{"repo": "psf/black", "name": "_merge_string_group", "language": "python", "path": "src/black/trans.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To combine adjacent strings into a single string within a line of code, ensuring that the merged result is syntactically correct.\n2. **Input**: A line of code and a list of indices indicating the starting points of groups of strings to be merged.\n3. **Output**: A modified line of code with the specified groups of strings merged, or an error if the merge cannot be performed.\n4. **Procedure**: \n   - The function iterates over the provided indices, each representing the start of a group of adjacent strings.\n   - For each index, it checks if the group of strings can be merged without violating syntax rules.\n   - If valid, it merges the strings and updates a dictionary with the new string and its position.\n   - If no strings can be merged, it returns an error.\n   - Otherwise, it constructs a new line of code with the merged strings and any remaining elements from the original line, including comments.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            msg_cant_transform = msg_result.err()\n            rblc_cant_transform = rblc_result.err()\n            cant_transform = CannotTransform(\n                \"StringMerger failed to merge any strings in this line.\"\n            )\n\n            # Chain the errors together using `__cause__`.\n            msg_cant_transform.__cause__ = rblc_cant_transform\n            cant_transform.__cause__ = msg_cant_transform\n\n            yield Err(cant_transform)\n        else:\n            yield Ok(new_line)\n\n    @staticmethod\n    def _remove_backslash_line_continuation_chars(\n        line: Line, string_indices: List[int]\n    ) -> TResult[Line]:\n        \"\"\"\n        Merge strings that were split across multiple lines using\n        line-continuation backslashes.\n\n        Returns:\n            Ok(new_line), if @line contains backslash line-continuation\n            characters.\n                OR\n            Err(CannotTransform), otherwise.\n        \"\"\"\n        LL = line.leaves\n\n        indices_to_transform = []\n        for string_idx in string_indices:\n            string_leaf = LL[string_idx]\n            if (\n                string_leaf.type == token.STRING\n                and \"\\\\\\n\" in string_leaf.value\n                and not has_triple_quotes(string_leaf.value)\n            ):\n                indices_to_transform.append(string_idx)\n\n        if not indices_to_transform:\n            return TErr(\n                \"Found no string leaves that contain backslash line continuation\"\n                \" characters.\"\n            )\n\n        new_line = line.clone()\n        new_line.comments = line.comments.copy()\n        append_leaves(new_line, line, LL)\n\n        for string_idx in indices_to_transform:\n            new_string_leaf = new_line.leaves[string_idx]\n            new_string_leaf.value = new_string_leaf.value.replace(\"\\\\\\n\", \"\")\n\n        return Ok(new_line)\n\n    \ndef _merge_string_group(\n        self, line: Line, string_indices: List[int]\n    ) -> TResult[Line]:\n        \"\"\"\n        Merges string groups (i.e. set of adjacent strings).\n\n        Each index from `string_indices` designates one string group's first\n        leaf in `line.leaves`.\n\n        Returns:\n            Ok(new_line), if ALL of the validation checks found in\n            _validate_msg(...) pass.\n                OR\n            Err(CannotTransform), otherwise.\n        \"\"\"\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.\n        merged_string_idx_dict: Dict[int, Tuple[int, Leaf]] = {}\n        for string_idx in string_indices:\n            vresult = self._validate_msg(line, string_idx)\n            if isinstance(vresult, Err):\n                continue\n            merged_string_idx_dict[string_idx] = self._merge_one_string_group(\n                LL, string_idx, is_valid_index\n            )\n\n        if not merged_string_idx_dict:\n            return TErr(\"No string group is merged\")\n\n        # Build the final line ('new_line') that this method will later return.\n        new_line = line.clone()\n        previous_merged_string_idx = -1\n        previous_merged_num_of_strings = -1\n        for i, leaf in enumerate(LL):\n            if i in merged_string_idx_dict:\n                previous_merged_string_idx = i\n                previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]\n                new_line.append(string_leaf)\n\n            if (\n                previous_merged_string_idx\n                <= i\n                < previous_merged_string_idx + previous_merged_num_of_strings\n            ):\n                for comment_leaf in line.comments_after(LL[i]):\n                    new_line.append(comment_leaf, preformatted=True)\n                continue\n\n            append_leaves(new_line, line, [leaf])\n\n        return Ok(new_line)\n\n    def _merge_one_string_group(\n        self, LL: List[Leaf], string_idx: int, is_valid_index: Callable[[int], bool]\n    ) -> Tuple[int, Leaf]:\n        \"\"\"\n        Merges one string group where the first string in the group is\n        `LL[string_idx]`.\n\n        Returns:\n            A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the\n            number of strings merged and `leaf` is the newly merged string\n            to be replaced in the new line.\n        \"\"\"\n        # If the string group is wrapped inside an Atom node, we must make sure\n        # to later replace that Atom with our new (merged) string leaf.\n        atom_node = LL[string_idx].parent\n\n        # We will place BREAK_MARK in between every two substrings that we\n        # merge. We will then later go through our final result and use the\n        # various instances of BREAK_MARK we find to add the right values to\n        # the custom split map.\n        BREAK_MARK = \"@@@@@ BLACK BREAKPOINT MARKER @@@@@\"\n\n        QUOTE = LL[string_idx].value[-1]\n\n        def make_naked(string: str, string_prefix: str) -> str:\n            \"\"\"Strip @string (i.e. make it a \"naked\" string)\n\n            Pre-conditions:\n                * assert_is_leaf_string(@string)\n\n            Returns:\n                A string that is identical to @string except that\n                @string_prefix has been stripped, the surrounding QUOTE\n                characters have been removed, and any remaining QUOTE\n                characters have been escaped.\n            \"\"\"\n            assert_is_leaf_string(string)\n            if \"f\" in string_prefix:\n                f_expressions = (\n                    string[span[0] + 1 : span[1] - 1]  # +-1 to get rid of curly braces\n                    for span in iter_fexpr_spans(string)\n                )\n                debug_expressions_contain_visible_quotes = any(\n                    re.search(r\".*[\\'\\\"].*(?<![!:=])={1}(?!=)(?![^\\s:])\", expression)\n                    for expression in f_expressions\n                )\n                if not debug_expressions_contain_visible_quotes:\n                    # We don't want to toggle visible quotes in debug f-strings, as\n                    # that would modify the AST\n                    string = _toggle_fexpr_quotes(string, QUOTE)\n                    # After quotes toggling, quotes in expressions won't be escaped\n                    # because quotes can't be reused in f-strings. So we can simply\n                    # let the escaping logic below run without knowing f-string\n                    # expressions.\n\n            RE_EVEN_BACKSLASHES = r\"(?:(?<!\\\\)(?:\\\\\\\\)*)\"\n            naked_string = string[len(string_prefix) + 1 : -1]\n            naked_string = re.sub(\n                \"(\" + RE_EVEN_BACKSLASHES + \")\" + QUOTE, r\"\\1\\\\\" + QUOTE, naked_string\n            )\n            return naked_string\n\n        # Holds the CustomSplit objects that will later be added to the custom\n        # split map.\n        custom_splits = []\n\n        # Temporary storage for the 'has_prefix' part of the CustomSplit objects.\n        prefix_tracker = []\n\n        # Sets the 'prefix' variable. This is the prefix that the final merged\n        # string will have.\n        next_str_idx = string_idx\n        prefix = \"\"\n        while (\n            not prefix\n            and is_valid_index(next_str_idx)\n            and LL[next_str_idx].type == token.STRING\n        ):\n            prefix = get_string_prefix(LL[next_str_idx].value).lower()\n            next_str_idx += 1\n\n        # The next loop merges the string group. The final string will be\n        # contained in 'S'.\n        #\n        # The following convenience variables are used:\n        #\n        #   S: string\n        #   NS: naked string\n        #   SS: next string\n        #   NSS: naked next string\n        S = \"\"\n        NS = \"\"\n        num_of_strings = 0\n        next_str_idx = string_idx\n        while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:\n            num_of_strings += 1\n\n            SS = LL[next_str_idx].value\n            next_prefix = get_string_prefix(SS).lower()\n\n            # If this is an f-string group but this substring is not prefixed\n            # with 'f'...\n            if \"f\" in prefix and \"f\" not in next_prefix:\n                # Then we must escape any braces contained in this substring.\n                SS = re.sub(r\"(\\{|\\})\", r\"\\1\\1\", SS)\n\n            NSS = make_naked(SS, next_prefix)\n\n            has_prefix = bool(next_prefix)\n            prefix_tracker.append(has_prefix)\n\n            S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE\n            NS = make_naked(S, prefix)\n\n            next_str_idx += 1\n\n        # Take a note on the index of the non-STRING leaf.\n        non_string_idx = next_str_idx\n\n        S_leaf = Leaf(token.STRING, S)\n        if self.normalize_strings:\n            S_leaf.value = normalize_string_quotes(S_leaf.value)\n\n        # Fill the 'custom_splits' list with the appropriate CustomSplit objects.\n        temp_string = S_leaf.value[len(prefix) + 1 : -1]\n        for has_prefix in prefix_tracker:\n            mark_idx = temp_string.find(BREAK_MARK)\n            assert (\n                mark_idx >= 0\n            ), \"Logic error while filling the custom string breakpoint cache.\"\n\n            temp_string = temp_string[mark_idx + len(BREAK_MARK) :]\n            breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1\n            custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))\n\n        string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, \"\"))\n\n        if atom_node is not None:\n            # If not all children of the atom node are merged (this can happen\n            # when there is a standalone comment in the middle) ...\n            if non_string_idx - string_idx < len(atom_node.children):\n                # We need to replace the old STRING leaves with the new string leaf.\n                first_child_idx = LL[string_idx].remove()\n                for idx in range(string_idx + 1, non_string_idx):\n                    LL[idx].remove()\n                if first_child_idx is not None:\n                    atom_node.insert_child(first_child_idx, string_leaf)\n            else:\n                # Else replace the atom node with the new string leaf.\n                replace_child(atom_node, string_leaf)\n\n        self.add_custom_splits(string_leaf.value, custom_splits)\n        return num_of_strings, string_leaf\n\n    @staticmethod\n    def _validate_msg(line: Line, string_idx: int) -> TResult[None]:\n        \"\"\"Validate (M)erge (S)tring (G)roup\n\n        Transform-time string validation logic for _merge_string_group(...).\n\n        Returns:\n            * Ok(None), if ALL validation checks (listed below) pass.\n                OR\n            * Err(CannotTransform), if any of the following are true:\n                - The target string group does not contain ANY stand-alone comments.\n                - The target string is not in a string group (i.e. it has no\n                  adjacent strings).\n                - The string group has more than one inline comment.\n                - The string group has an inline comment that appears to be a pragma.\n                - The set of all string prefixes in the string group is of\n                  length greater than one and is not equal to {\"\", \"f\"}.\n                - The string group consists of raw strings.\n                - The string group is stringified type annotations. We don't want to\n                  process stringified type annotations since pyright doesn't support\n                  them spanning multiple string values. (NOTE: mypy, pytype, pyre do\n                  support them, so we can change if pyright also gains support in the\n                  future. See https://github.com/microsoft/pyright/issues/4359.)\n        \"\"\"\n        # We first check for \"inner\" stand-alone comments (i.e. stand-alone\n        # comments that have a string leaf before them AND after them).\n        for inc in [1, -1]:\n            i = string_idx\n            found_sa_comment = False\n            is_valid_index = is_valid_index_factory(line.leaves)\n            while is_valid_index(i) and line.leaves[i].type in [\n                token.STRING,\n                STANDALONE_COMMENT,\n            ]:\n                if line.leaves[i].type == STANDALONE_COMMENT:\n                    found_sa_comment = True\n                elif found_sa_comment:\n                    return TErr(\n                        \"StringMerger does NOT merge string groups which contain \"\n                        \"stand-alone comments.\"\n                    )\n\n                i += inc\n\n        num_of_inline_string_comments = 0\n        set_of_prefixes = set()\n        num_of_strings = 0\n        for leaf in line.leaves[string_idx:]:\n            if leaf.type != token.STRING:\n                # If the string group is trailed by a comma, we count the\n                # comments trailing the comma to be one of the string group's\n                # comments.\n                if leaf.type == token.COMMA and id(leaf) in line.comments:\n                    num_of_inline_string_comments += 1\n                break\n\n            if has_triple_quotes(leaf.value):\n                return TErr(\"StringMerger does NOT merge multiline strings.\")\n\n            num_of_strings += 1\n            prefix = get_string_prefix(leaf.value).lower()\n            if \"r\" in prefix:\n                return TErr(\"StringMerger does NOT merge raw strings.\")\n\n            set_of_prefixes.add(prefix)\n\n            if id(leaf) in line.comments:\n                num_of_inline_string_comments += 1\n                if contains_pragma_comment(line.comments[id(leaf)]):\n                    return TErr(\"Cannot merge strings which have pragma comments.\")\n\n        if num_of_strings < 2:\n            return TErr(\n                f\"Not enough strings to merge (num_of_strings={num_of_strings}).\"\n            )\n\n        if num_of_inline_string_comments > 1:\n            return TErr(\n                f\"Too many inline string comments ({num_of_inline_string_comments}).\"\n            )\n\n        if len(set_of_prefixes) > 1 and set_of_prefixes != {\"\", \"f\"}:\n            return TErr(f\"Too many different prefixes ({set_of_prefixes}).\")\n\n        return Ok(None)\n\n\nclass StringParenStripper(StringTransformer):\n    \"\"\"StringTransformer that strips surrounding parentheses from strings.\n\n    Requirements:\n        The line contains a string which is surrounded by parentheses and:\n            - The target string is NOT the only argument to a function call.\n            - The target string is NOT a \"pointless\" string.\n            - If the target string contains a PERCENT, the brackets are not\n              preceded or followed by an operator with higher precedence than\n              PERCENT.\n\n    Transformations:\n        The parentheses mentioned in the 'Requirements' section are stripped.\n\n    Collaborations:\n        StringParenStripper has its own inherent usefulness, but it is also\n        relied on to clean up the parentheses created by StringParenWrapper (in\n        the event that they are no longer needed).\n    \"\"\"\n\n    def do_match(self, line: Line) -> TMatchResult:\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        string_indices = []\n\n        idx = -1\n        while True:\n            idx += 1\n            if idx >= len(LL):\n                break\n            leaf = LL[idx]\n\n            # Should be a string...\n            if leaf.type != token.STRING:\n                continue\n\n            # If this is a \"pointless\" string...\n            if (\n                leaf.parent\n                and leaf.parent.parent\n                and leaf.parent.parent.type == syms.simple_stmt\n            ):\n                continue\n\n            # Should be preceded by a non-empty LPAR...\n            if (\n                not is_valid_index(idx - 1)\n                or LL[idx - 1].type != token.LPAR\n                or is_empty_lpar(LL[idx - 1])\n            ):\n                continue\n\n            # That LPAR should NOT be preceded by a function name or a closing\n            # bracket (which could be a function which returns a function or a\n            # list/dictionary that contains a function)...\n            if is_valid_index(idx - 2) and (\n                LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS\n            ):\n                continue\n\n            string_idx = idx\n\n            # Skip the string trailer, if one exists.\n            string_parser = StringParser()\n            next_idx = string_parser.parse(LL, string_idx)\n\n            # if the leaves in the parsed string include a PERCENT, we need to\n            # make sure the initial LPAR is NOT preceded by an operator with\n            # higher or equal precedence to PERCENT\n            if is_valid_index(idx - 2):\n                # mypy can't quite follow unless we name this\n                before_lpar = LL[idx - 2]\n                if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and (\n                    (\n                        before_lpar.type\n                        in {\n                            token.STAR,\n                            token.AT,\n                            token.SLASH,\n                            token.DOUBLESLASH,\n                            token.PERCENT,\n                            token.TILDE,\n                            token.DOUBLESTAR,\n                            token.AWAIT,\n                            token.LSQB,\n                            token.LPAR,\n                        }\n                    )\n                    or (\n                        # only unary PLUS/MINUS\n                        before_lpar.parent\n                        and before_lpar.parent.type == syms.factor\n                        and (before_lpar.type in {token.PLUS, token.MINUS})\n                    )\n                ):\n                    continue\n\n            # Should be followed by a non-empty RPAR...\n            if (\n                is_valid_index(next_idx)\n                and LL[next_idx].type == token.RPAR\n                and not is_empty_rpar(LL[next_idx])\n            ):\n                # That RPAR should NOT be followed by anything with higher\n                # precedence than PERCENT\n                if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in {\n                    token.DOUBLESTAR,\n                    token.LSQB,\n                    token.LPAR,\n                    token.DOT,\n                }:\n                    continue\n\n                string_indices.append(string_idx)\n                idx = string_idx\n                while idx < len(LL) - 1 and LL[idx + 1].type == token.STRING:\n                    idx += 1\n\n        if string_indices:\n            return Ok(string_indices)\n        return TErr(\"This line has no strings wrapped in parens.\")\n\n    def do_transform(\n        self, line: Line, string_indices: List[int]\n    ) -> Iterator[TResult[Line]]:\n        LL = line.leaves\n\n        string_and_rpar_indices: List[int] = []\n        for string_idx in string_indices:\n            string_parser = StringParser()\n            rpar_idx = string_parser.parse(LL, string_idx)\n\n            should_transform = True\n            for leaf in (LL[string_idx - 1], LL[rpar_idx]):\n                if line.comments_after(leaf):\n                    # Should not strip parentheses which have comments attached\n                    # to them.\n                    should_transform = False\n                    break\n            if should_transform:\n                string_and_rpar_indices.extend((string_idx, rpar_idx))\n\n        if string_and_rpar_indices:\n            yield Ok(self._transform_to_new_line(line, string_and_rpar_indices))\n        else:\n            yield Err(\n                CannotTransform(\"All string groups have comments attached to them.\")\n            )\n\n    def _transform_to_new_line(\n        self, line: Line, string_and_rpar_indices: List[int]\n    ) -> Line:\n        LL = line.leaves\n\n        new_line = line.clone()\n        new_line.comments = line.comments.copy()\n\n        previous_idx = -1\n        # We need to sort the indices, since string_idx and its matching\n        # rpar_idx may not come in order, e.g. in\n        # `(\"outer\" % (\"inner\".join(items)))`, the \"inner\" string's\n        # string_idx is smaller than \"outer\" string's rpar_idx.\n        for idx in sorted(string_and_rpar_indices):\n            leaf = LL[idx]\n            lpar_or_rpar_idx = idx - 1 if leaf.type == token.STRING else idx\n            append_leaves(new_line, line, LL[previous_idx + 1 : lpar_or_rpar_idx])\n            if leaf.type == token.STRING:\n                string_leaf = Leaf(token.STRING, LL[idx].value)\n                LL[lpar_or_rpar_idx].remove()  # Remove lpar.\n                replace_child(LL[idx], string_leaf)\n                new_line.append(string_leaf)\n                # replace comments\n                old_comments = new_line.comments.pop(id(LL[idx]), [])\n                new_line.comments.setdefault(id(string_leaf), []).extend(old_comments)\n            else:\n                LL[lpar_or_rpar_idx].remove()  # This is a rpar.\n\n            previous_idx = idx\n\n        # Append the leaves after the last idx:\n        append_leaves(new_line, line, LL[idx + 1 :])\n\n        return new_line\n\n\nclass BaseStringSplitter(StringTransformer):\n    \"\"\"\n    Abstract class for StringTransformers which transform a Line's strings by splitting\n    them or placing them on their own lines where necessary to avoid going over\n    the configured line length.\n\n    Requirements:\n        * The target string value is responsible for the line going over the\n          line length limit. It follows that after all of black's other line\n          split methods have been exhausted, this line (or one of the resulting\n          lines after all line splits are performed) would still be over the\n          line_length limit unless we split this string.\n          AND\n\n        * The target string is NOT a \"pointless\" string (i.e. a string that has\n          no parent or siblings).\n          AND\n\n        * The target string is not followed by an inline comment that appears\n          to be a pragma.\n          AND\n\n        * The target string is not a multiline (i.e. triple-quote) string.\n    \"\"\"\n\n    STRING_OPERATORS: Final = [\n        token.EQEQUAL,\n        token.GREATER,\n        token.GREATEREQUAL,\n        token.LESS,\n        token.LESSEQUAL,\n        token.NOTEQUAL,\n        token.PERCENT,\n        token.PLUS,\n        token.STAR,\n    ]\n\n    @abstractmethod\n    def do_splitter_match(self, line: Line) -> TMatchResult:\n        \"\"\"\n        BaseStringSplitter asks its clients to override this method instead of\n        `StringTransformer.do_match(...)`.\n\n        Follows the same protocol as `StringTransformer.do_match(...)`.\n\n        Refer to `help(StringTransformer.do_match)` for more information.\n        \"\"\"\n\n    def do_match(self, line: Line) -> TMatchResult:\n        match_result = self.do_splitter_match(line)\n        if isinstance(match_result, Err):\n            return match_result\n\n        string_indices = match_result.ok()\n        assert len(string_indices) == 1, (\n            f\"{self.__class__.__name__} should only find one match at a time, found\"\n            f\" {len(string_indices)}\"\n        )\n        string_idx = string_indices[0]\n        vresult = self._validate(line, string_idx)\n        if isinstance(vresult, Err):\n            return vresult\n\n        return match_result\n\n    def _validate(self, line: Line, string_idx: int) -> TResult[None]:\n        \"\"\"\n        Checks that @line meets all of the requirements listed in this classes'\n        docstring. Refer to `help(BaseStringSplitter)` for a detailed\n        description of those requirements.\n\n        Returns:\n            * Ok(None), if ALL of the requirements are met.\n              OR\n            * Err(CannotTransform), if ANY of the requirements are NOT met.\n        \"\"\"\n        LL = line.leaves\n\n        string_leaf = LL[string_idx]\n\n        max_string_length = self._get_max_string_length(line, string_idx)\n        if len(string_leaf.value) <= max_string_length:\n            return TErr(\n                \"The string itself is not what is causing this line to be too long.\"\n            )\n\n        if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [\n            token.STRING,\n            token.NEWLINE,\n        ]:\n            return TErr(\n                f\"This string ({string_leaf.value}) appears to be pointless (i.e. has\"\n                \" no parent).\"\n            )\n\n        if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(\n            line.comments[id(line.leaves[string_idx])]\n        ):\n            return TErr(\n                \"Line appears to end with an inline pragma comment. Splitting the line\"\n                \" could modify the pragma's behavior.\"\n            )\n\n        if has_triple_quotes(string_leaf.value):\n            return TErr(\"We cannot split multiline strings.\")\n\n        return Ok(None)\n\n    def _get_max_string_length(self, line: Line, string_idx: int) -> int:\n        \"\"\"\n        Calculates the max string length used when attempting to determine\n        whether or not the target string is responsible for causing the line to\n        go over the line length limit.\n\n        WARNING: This method is tightly coupled to both StringSplitter and\n        (especially) StringParenWrapper. There is probably a better way to\n        accomplish what is being done here.\n\n        Returns:\n            max_string_length: such that `line.leaves[string_idx].value >\n            max_string_length` implies that the target string IS responsible\n            for causing this line to exceed the line length limit.\n        \"\"\"\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        # We use the shorthand \"WMA4\" in comments to abbreviate \"We must\n        # account for\". When giving examples, we use STRING to mean some/any\n        # valid string.\n        #\n        # Finally, we use the following convenience variables:\n        #\n        #   P:  The leaf that is before the target string leaf.\n        #   N:  The leaf that is after the target string leaf.\n        #   NN: The leaf that is after N.\n\n        # WMA4 the whitespace at the beginning of the line.\n        offset = line.depth * 4\n\n        if is_valid_index(string_idx - 1):\n            p_idx = string_idx - 1\n            if (\n                LL[string_idx - 1].type == token.LPAR\n                and LL[string_idx - 1].value == \"\"\n                and string_idx >= 2\n            ):\n                # If the previous leaf is an empty LPAR placeholder, we should skip it.\n                p_idx -= 1\n\n            P = LL[p_idx]\n            if P.type in self.STRING_OPERATORS:\n                # WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`).\n                offset += len(str(P)) + 1\n\n            if P.type == token.COMMA:\n                # WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].\n                offset += 3\n\n            if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]:\n                # This conditional branch is meant to handle dictionary keys,\n                # variable assignments, 'return STRING' statement lines, and\n                # 'else STRING' ternary expression lines.\n\n                # WMA4 a single space.\n                offset += 1\n\n                # WMA4 the lengths of any leaves that came before that space,\n                # but after any closing bracket before that space.\n                for leaf in reversed(LL[: p_idx + 1]):\n                    offset += len(str(leaf))\n                    if leaf.type in CLOSING_BRACKETS:\n                        break\n\n        if is_valid_index(string_idx + 1):\n            N = LL[string_idx + 1]\n            if N.type == token.RPAR and N.value == \"\" and len(LL) > string_idx + 2:\n                # If the next leaf is an empty RPAR placeholder, we should skip it.\n                N = LL[string_idx + 2]\n\n            if N.type == token.COMMA:\n                # WMA4 a single comma at the end of the string (e.g `STRING,`).\n                offset += 1\n\n            if is_valid_index(string_idx + 2):\n                NN = LL[string_idx + 2]\n\n                if N.type == token.DOT and NN.type == token.NAME:\n                    # This conditional branch is meant to handle method calls invoked\n                    # off of a string literal up to and including the LPAR character.\n\n                    # WMA4 the '.' character.\n                    offset += 1\n\n                    if (\n                        is_valid_index(string_idx + 3)\n                        and LL[string_idx + 3].type == token.LPAR\n                    ):\n                        # WMA4 the left parenthesis character.\n                        offset += 1\n\n                    # WMA4 the length of the method's name.\n                    offset += len(NN.value)\n\n        has_comments = False\n        for comment_leaf in line.comments_after(LL[string_idx]):\n            if not has_comments:\n                has_comments = True\n                # WMA4 two spaces before the '#' character.\n                offset += 2\n\n            # WMA4 the length of the inline comment.\n            offset += len(comment_leaf.value)\n\n        max_string_length = count_chars_in_width(str(line), self.line_length - offset)\n        return max_string_length\n\n    @staticmethod\n    def _prefer_paren_wrap_match(LL: List[Leaf]) -> Optional[int]:\n        \"\"\"\n        Returns:\n            string_idx such that @LL[string_idx] is equal to our target (i.e.\n            matched) string, if this line matches the \"prefer paren wrap\" statement\n            requirements listed in the 'Requirements' section of the StringParenWrapper\n            class's docstring.\n                OR\n            None, otherwise.\n        \"\"\"\n        # The line must start with a string.\n        if LL[0].type != token.STRING:\n            return None\n\n        matching_nodes = [\n            syms.listmaker,\n            syms.dictsetmaker,\n            syms.testlist_gexp,\n        ]\n        # If the string is an immediate child of a list/set/tuple literal...\n        if (\n            parent_type(LL[0]) in matching_nodes\n            or parent_type(LL[0].parent) in matching_nodes\n        ):\n            # And the string is surrounded by commas (or is the first/last child)...\n            prev_sibling = LL[0].prev_sibling\n            next_sibling = LL[0].next_sibling\n            if (\n                not prev_sibling\n                and not next_sibling\n                and parent_type(LL[0]) == syms.atom\n            ):\n                # If it's an atom string, we need to check the parent atom's siblings.\n                parent = LL[0].parent\n                assert parent is not None  # For type checkers.\n                prev_sibling = parent.prev_sibling\n                next_sibling = parent.next_sibling\n            if (not prev_sibling or prev_sibling.type == token.COMMA) and (\n                not next_sibling or next_sibling.type == token.COMMA\n            ):\n                return 0\n\n        return None\n\n\ndef iter_fexpr_spans(s: str) -> Iterator[Tuple[int, int]]:\n    \"\"\"\n    Yields spans corresponding to expressions in a given f-string.\n    Spans are half-open ranges (left inclusive, right exclusive).\n    Assumes the input string is a valid f-string, but will not crash if the input\n    string is invalid.\n    \"\"\"\n    stack: List[int] = []  # our curly paren stack\n    i = 0\n    while i < len(s):\n        if s[i] == \"{\":\n            # if we're in a string part of the f-string, ignore escaped curly braces\n            if not stack and i + 1 < len(s) and s[i + 1] == \"{\":\n                i += 2\n                continue\n            stack.append(i)\n            i += 1\n            continue\n\n        if s[i] == \"}\":\n            if not stack:\n                i += 1\n                continue\n            j = stack.pop()\n            # we've made it back out of the expression! yield the span\n            if not stack:\n                yield (j, i + 1)\n            i += 1\n            continue\n\n        # if we're in an expression part of the f-string, fast-forward through strings\n        # note that backslashes are not legal in the expression portion of f-strings\n        if stack:\n            delim = None\n            if s[i : i + 3] in (\"'''\", '\"\"\"'):\n                delim = s[i : i + 3]\n            elif s[i] in (\"'\", '\"'):\n                delim = s[i]\n            if delim:\n                i += len(delim)\n                while i < len(s) and s[i : i + len(delim)] != delim:\n                    i += 1\n                i += len(delim)\n                continue\n        i += 1\n\n\ndef fstring_contains_expr(s: str) -> bool:\n    return any(iter_fexpr_spans(s))\n\n\ndef _toggle_fexpr_quotes(fstring: str, old_quote: str) -> str:\n    \"\"\"\n    Toggles quotes used in f-string expressions that are `old_quote`.\n\n    f-string expressions can't contain backslashes, so we need to toggle the\n    quotes if the f-string itself will end up using the same quote. We can\n    simply toggle without escaping because, quotes can't be reused in f-string\n    expressions. They will fail to parse.\n\n    NOTE: If PEP 701 is accepted, above statement will no longer be true.\n    Though if quotes can be reused, we can simply reuse them without updates or\n    escaping, once Black figures out how to parse the new grammar.\n    \"\"\"\n    new_quote = \"'\" if old_quote == '\"' else '\"'\n    parts = []\n    previous_index = 0\n    for start, end in iter_fexpr_spans(fstring):\n        parts.append(fstring[previous_index:start])\n        parts.append(fstring[start:end].replace(old_quote, new_quote))\n        previous_index = end\n    parts.append(fstring[previous_index:])\n    return \"\".join(parts)\n\n\nclass StringSplitter(BaseStringSplitter, CustomSplitMapMixin):\n    \"\"\"\n    StringTransformer that splits \"atom\" strings (i.e. strings which exist on\n    lines by themselves).\n\n    Requirements:\n        * The line consists ONLY of a single string (possibly prefixed by a\n          string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE\n          a trailing comma.\n          AND\n        * All of the requirements listed in BaseStringSplitter's docstring.\n\n    Transformations:\n        The string mentioned in the 'Requirements' section is split into as\n        many substrings as necessary to adhere to the configured line length.\n\n        In the final set of substrings, no substring should be smaller than\n        MIN_SUBSTR_SIZE characters.\n\n        The string will ONLY be split on spaces (i.e. each new substring should\n        start with a space). Note that the string will NOT be split on a space\n        which is escaped with a backslash.\n\n        If the string is an f-string, it will NOT be split in the middle of an\n        f-expression (e.g. in f\"FooBar: {foo() if x else bar()}\", {foo() if x\n        else bar()} is an f-expression).\n\n        If the string that is being split has an associated set of custom split\n        records and those custom splits will NOT result in any line going over\n        the configured line length, those custom splits are used. Otherwise the\n        string is split as late as possible (from left-to-right) while still\n        adhering to the transformation rules listed above.\n\n    Collaborations:\n        StringSplitter relies on StringMerger to construct the appropriate\n        CustomSplit objects and add them to the custom split map.\n    \"\"\"\n\n    MIN_SUBSTR_SIZE: Final = 6\n\n    def do_splitter_match(self, line: Line) -> TMatchResult:\n        LL = line.leaves\n\n        if self._prefer_paren_wrap_match(LL) is not None:\n            return TErr(\"Line needs to be wrapped in parens first.\")\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        idx = 0\n\n        # The first two leaves MAY be the 'not in' keywords...\n        if (\n            is_valid_index(idx)\n            and is_valid_index(idx + 1)\n            and [LL[idx].type, LL[idx + 1].type] == [token.NAME, token.NAME]\n            and str(LL[idx]) + str(LL[idx + 1]) == \"not in\"\n        ):\n            idx += 2\n        # Else the first leaf MAY be a string operator symbol or the 'in' keyword...\n        elif is_valid_index(idx) and (\n            LL[idx].type in self.STRING_OPERATORS\n            or LL[idx].type == token.NAME\n            and str(LL[idx]) == \"in\"\n        ):\n            idx += 1\n\n        # The next/first leaf MAY be an empty LPAR...\n        if is_valid_index(idx) and is_empty_lpar(LL[idx]):\n            idx += 1\n\n        # The next/first leaf MUST be a string...\n        if not is_valid_index(idx) or LL[idx].type != token.STRING:\n            return TErr(\"Line does not start with a string.\")\n\n        string_idx = idx\n\n        # Skip the string trailer, if one exists.\n        string_parser = StringParser()\n        idx = string_parser.parse(LL, string_idx)\n\n        # That string MAY be followed by an empty RPAR...\n        if is_valid_index(idx) and is_empty_rpar(LL[idx]):\n            idx += 1\n\n        # That string / empty RPAR leaf MAY be followed by a comma...\n        if is_valid_index(idx) and LL[idx].type == token.COMMA:\n            idx += 1\n\n        # But no more leaves are allowed...\n        if is_valid_index(idx):\n            return TErr(\"This line does not end with a string.\")\n\n        return Ok([string_idx])\n\n    def do_transform(\n        self, line: Line, string_indices: List[int]\n    ) -> Iterator[TResult[Line]]:\n        LL = line.leaves\n        assert len(string_indices) == 1, (\n            f\"{self.__class__.__name__} should only find one match at a time, found\"\n            f\" {len(string_indices)}\"\n        )\n        string_idx = string_indices[0]\n\n        QUOTE = LL[string_idx].value[-1]\n\n        is_valid_index = is_valid_index_factory(LL)\n        insert_str_child = insert_str_child_factory(LL[string_idx])\n\n        prefix = get_string_prefix(LL[string_idx].value).lower()\n\n        # We MAY choose to drop the 'f' prefix from substrings that don't\n        # contain any f-expressions, but ONLY if the original f-string\n        # contains at least one f-expression. Otherwise, we will alter the AST\n        # of the program.\n        drop_pointless_f_prefix = (\"f\" in prefix) and fstring_contains_expr(\n            LL[string_idx].value\n        )\n\n        first_string_line = True\n\n        string_op_leaves = self._get_string_operator_leaves(LL)\n        string_op_leaves_length = (\n            sum(len(str(prefix_leaf)) for prefix_leaf in string_op_leaves) + 1\n            if string_op_leaves\n            else 0\n        )\n\n        def maybe_append_string_operators(new_line: Line) -> None:\n            \"\"\"\n            Side Effects:\n                If @line starts with a string operator and this is the first\n                line we are constructing, this function appends the string\n                operator to @new_line and replaces the old string operator leaf\n                in the node structure. Otherwise this function does nothing.\n            \"\"\"\n            maybe_prefix_leaves = string_op_leaves if first_string_line else []\n            for i, prefix_leaf in enumerate(maybe_prefix_leaves):\n                replace_child(LL[i], prefix_leaf)\n                new_line.append(prefix_leaf)\n\n        ends_with_comma = (\n            is_valid_index(string_idx + 1) and LL[string_idx + 1].type == token.COMMA\n        )\n\n        def max_last_string_column() -> int:\n            \"\"\"\n            Returns:\n                The max allowed width of the string value used for the last\n                line we will construct.  Note that this value means the width\n                rather than the number of characters (e.g., many East Asian\n                characters expand to two columns).\n            \"\"\"\n            result = self.line_length\n            result -= line.depth * 4\n            result -= 1 if ends_with_comma else 0\n            result -= string_op_leaves_length\n            return result\n\n        # --- Calculate Max Break Width (for string value)\n        # We start with the line length limit\n        max_break_width = self.line_length\n        # The last index of a string of length N is N-1.\n        max_break_width -= 1\n        # Leading whitespace is not present in the string value (e.g. Leaf.value).\n        max_break_width -= line.depth * 4\n        if max_break_width < 0:\n            yield TErr(\n                f\"Unable to split {LL[string_idx].value} at such high of a line depth:\"\n                f\" {line.depth}\"\n            )\n            return\n\n        # Check if StringMerger registered any custom splits.\n        custom_splits = self.pop_custom_splits(LL[string_idx].value)\n        # We use them ONLY if none of them would produce lines that exceed the\n        # line limit.\n        use_custom_breakpoints = bool(\n            custom_splits\n            and all(csplit.break_idx <= max_break_width for csplit in custom_splits)\n        )\n\n        # Temporary storage for the remaining chunk of the string line that\n        # can't fit onto the line currently being constructed.\n        rest_value = LL[string_idx].value\n\n        def more_splits_should_be_made() -> bool:\n            \"\"\"\n            Returns:\n                True iff `rest_value` (the remaining string value from the last\n                split), should be split again.\n            \"\"\"\n            if use_custom_breakpoints:\n                return len(custom_splits) > 1\n            else:\n                return str_width(rest_value) > max_last_string_column()\n\n        string_line_results: List[Ok[Line]] = []\n        while more_splits_should_be_made():\n            if use_custom_breakpoints:\n                # Custom User Split (manual)\n                csplit = custom_splits.pop(0)\n                break_idx = csplit.break_idx\n            else:\n                # Algorithmic Split (automatic)\n                max_bidx = (\n                    count_chars_in_width(rest_value, max_break_width)\n                    - string_op_leaves_length\n                )\n                maybe_break_idx = self._get_break_idx(rest_value, max_bidx)\n                if maybe_break_idx is None:\n                    # If we are unable to algorithmically determine a good split\n                    # and this string has custom splits registered to it, we\n                    # fall back to using them--which means we have to start\n                    # over from the beginning.\n                    if custom_splits:\n                        rest_value = LL[string_idx].value\n                        string_line_results = []\n                        first_string_line = True\n                        use_custom_breakpoints = True\n                        continue\n\n                    # Otherwise, we stop splitting here.\n                    break\n\n                break_idx = maybe_break_idx\n\n            # --- Construct `next_value`\n            next_value = rest_value[:break_idx] + QUOTE\n\n            # HACK: The following 'if' statement is a hack to fix the custom\n            # breakpoint index in the case of either: (a) substrings that were\n            # f-strings but will have the 'f' prefix removed OR (b) substrings\n            # that were not f-strings but will now become f-strings because of\n            # redundant use of the 'f' prefix (i.e. none of the substrings\n            # contain f-expressions but one or more of them had the 'f' prefix\n            # anyway; in which case, we will prepend 'f' to _all_ substrings).\n            #\n            # There is probably a better way to accomplish what is being done\n            # here...\n            #\n            # If this substring is an f-string, we _could_ remove the 'f'\n            # prefix, and the current custom split did NOT originally use a\n            # prefix...\n            if (\n                use_custom_breakpoints\n                and not csplit.has_prefix\n                and (\n                    # `next_value == prefix + QUOTE` happens when the custom\n                    # split is an empty string.\n                    next_value == prefix + QUOTE\n                    or next_value != self._normalize_f_string(next_value, prefix)\n                )\n            ):\n                # Then `csplit.break_idx` will be off by one after removing\n                # the 'f' prefix.\n                break_idx += 1\n                next_value = rest_value[:break_idx] + QUOTE\n\n            if drop_pointless_f_prefix:\n                next_value = self._normalize_f_string(next_value, prefix)\n\n            # --- Construct `next_leaf`\n            next_leaf = Leaf(token.STRING, next_value)\n            insert_str_child(next_leaf)\n            self._maybe_normalize_string_quotes(next_leaf)\n\n            # --- Construct `next_line`\n            next_line = line.clone()\n            maybe_append_string_operators(next_line)\n            next_line.append(next_leaf)\n            string_line_results.append(Ok(next_line))\n\n            rest_value = prefix + QUOTE + rest_value[break_idx:]\n            first_string_line = False\n\n        yield from string_line_results\n\n        if drop_pointless_f_prefix:\n            rest_value = self._normalize_f_string(rest_value, prefix)\n\n        rest_leaf = Leaf(token.STRING, rest_value)\n        insert_str_child(rest_leaf)\n\n        # NOTE: I could not find a test case that verifies that the following\n        # line is actually necessary, but it seems to be. Otherwise we risk\n        # not normalizing the last substring, right?\n        self._maybe_normalize_string_quotes(rest_leaf)\n\n        last_line = line.clone()\n        maybe_append_string_operators(last_line)\n\n        # If there are any leaves to the right of the target string...\n        if is_valid_index(string_idx + 1):\n            # We use `temp_value` here to determine how long the last line\n            # would be if we were to append all the leaves to the right of the\n            # target string to the last string line.\n            temp_value = rest_value\n            for leaf in LL[string_idx + 1 :]:\n                temp_value += str(leaf)\n                if leaf.type == token.LPAR:\n                    break\n\n            # Try to fit them all on the same line with the last substring...\n            if (\n                str_width(temp_value) <= max_last_string_column()\n                or LL[string_idx + 1].type == token.COMMA\n            ):\n                last_line.append(rest_leaf)\n                append_leaves(last_line, line, LL[string_idx + 1 :])\n                yield Ok(last_line)\n            # Otherwise, place the last substring on one line and everything\n            # else on a line below that...\n            else:\n                last_line.append(rest_leaf)\n                yield Ok(last_line)\n\n                non_string_line = line.clone()\n                append_leaves(non_string_line, line, LL[string_idx + 1 :])\n                yield Ok(non_string_line)\n        # Else the target string was the last leaf...\n        else:\n            last_line.append(rest_leaf)\n            last_line.comments = line.comments.copy()\n            yield Ok(last_line)\n\n    def _iter_nameescape_slices(self, string: str) -> Iterator[Tuple[Index, Index]]:\n        \"\"\"\n        Yields:\n            All ranges of @string which, if @string were to be split there,\n            would result in the splitting of an \\\\N{...} expression (which is NOT\n            allowed).\n        \"\"\"\n        # True - the previous backslash was unescaped\n        # False - the previous backslash was escaped *or* there was no backslash\n        previous_was_unescaped_backslash = False\n        it = iter(enumerate(string))\n        for idx, c in it:\n            if c == \"\\\\\":\n                previous_was_unescaped_backslash = not previous_was_unescaped_backslash\n                continue\n            if not previous_was_unescaped_backslash or c != \"N\":\n                previous_was_unescaped_backslash = False\n                continue\n            previous_was_unescaped_backslash = False\n\n            begin = idx - 1  # the position of backslash before \\N{...}\n            for idx, c in it:\n                if c == \"}\":\n                    end = idx\n                    break\n            else:\n                # malformed nameescape expression?\n                # should have been detected by AST parsing earlier...\n                raise RuntimeError(f\"{self.__class__.__name__} LOGIC ERROR!\")\n            yield begin, end\n\n    def _iter_fexpr_slices(self, string: str) -> Iterator[Tuple[Index, Index]]:\n        \"\"\"\n        Yields:\n            All ranges of @string which, if @string were to be split there,\n            would result in the splitting of an f-expression (which is NOT\n            allowed).\n        \"\"\"\n        if \"f\" not in get_string_prefix(string).lower():\n            return\n        yield from iter_fexpr_spans(string)\n\n    def _get_illegal_split_indices(self, string: str) -> Set[Index]:\n        illegal_indices: Set[Index] = set()\n        iterators = [\n            self._iter_fexpr_slices(string),\n            self._iter_nameescape_slices(string),\n        ]\n        for it in iterators:\n            for begin, end in it:\n                illegal_indices.update(range(begin, end + 1))\n        return illegal_indices\n\n    def _get_break_idx(self, string: str, max_break_idx: int) -> Optional[int]:\n        \"\"\"\n        This method contains the algorithm that StringSplitter uses to\n        determine which character to split each string at.\n\n        Args:\n            @string: The substring that we are attempting to split.\n            @max_break_idx: The ideal break index. We will return this value if it\n            meets all the necessary conditions. In the likely event that it\n            doesn't we will try to find the closest index BELOW @max_break_idx\n            that does. If that fails, we will expand our search by also\n            considering all valid indices ABOVE @max_break_idx.\n\n        Pre-Conditions:\n            * assert_is_leaf_string(@string)\n            * 0 <= @max_break_idx < len(@string)\n\n        Returns:\n            break_idx, if an index is able to be found that meets all of the\n            conditions listed in the 'Transformations' section of this classes'\n            docstring.\n                OR\n            None, otherwise.\n        \"\"\"\n        is_valid_index = is_valid_index_factory(string)\n\n        assert is_valid_index(max_break_idx)\n        assert_is_leaf_string(string)\n\n        _illegal_split_indices = self._get_illegal_split_indices(string)\n\n        def breaks_unsplittable_expression(i: Index) -> bool:\n            \"\"\"\n            Returns:\n                True iff returning @i would result in the splitting of an\n                unsplittable expression (which is NOT allowed).\n            \"\"\"\n            return i in _illegal_split_indices\n\n        def passes_all_checks(i: Index) -> bool:\n            \"\"\"\n            Returns:\n                True iff ALL of the conditions listed in the 'Transformations'\n                section of this classes' docstring would be met by returning @i.\n            \"\"\"\n            is_space = string[i] == \" \"\n            is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS\n\n            is_not_escaped = True\n            j = i - 1\n            while is_valid_index(j) and string[j] == \"\\\\\":\n                is_not_escaped = not is_not_escaped\n                j -= 1\n\n            is_big_enough = (\n                len(string[i:]) >= self.MIN_SUBSTR_SIZE\n                and len(string[:i]) >= self.MIN_SUBSTR_SIZE\n            )\n            return (\n                (is_space or is_split_safe)\n                and is_not_escaped\n                and is_big_enough\n                and not breaks_unsplittable_expression(i)\n            )\n\n        # First, we check all indices BELOW @max_break_idx.\n        break_idx = max_break_idx\n        while is_valid_index(break_idx - 1) and not passes_all_checks(break_idx):\n            break_idx -= 1\n\n        if not passes_all_checks(break_idx):\n            # If that fails, we check all indices ABOVE @max_break_idx.\n            #\n            # If we are able to find a valid index here, the next line is going\n            # to be longer than the specified line length, but it's probably\n            # better than doing nothing at all.\n            break_idx = max_break_idx + 1\n            while is_valid_index(break_idx + 1) and not passes_all_checks(break_idx):\n                break_idx += 1\n\n            if not is_valid_index(break_idx) or not passes_all_checks(break_idx):\n                return None\n\n        return break_idx\n\n    def _maybe_normalize_string_quotes(self, leaf: Leaf) -> None:\n        if self.normalize_strings:\n            leaf.value = normalize_string_quotes(leaf.value)\n\n    def _normalize_f_string(self, string: str, prefix: str) -> str:\n        \"\"\"\n        Pre-Conditions:\n            * assert_is_leaf_string(@string)\n\n        Returns:\n            * If @string is an f-string that contains no f-expressions, we\n            return a string identical to @string except that the 'f' prefix\n            has been stripped and all double braces (i.e. '{{' or '}}') have\n            been normalized (i.e. turned into '{' or '}').\n                OR\n            * Otherwise, we return @string.\n        \"\"\"\n        assert_is_leaf_string(string)\n\n        if \"f\" in prefix and not fstring_contains_expr(string):\n            new_prefix = prefix.replace(\"f\", \"\")\n\n            temp = string[len(prefix) :]\n            temp = re.sub(r\"\\{\\{\", \"{\", temp)\n            temp = re.sub(r\"\\}\\}\", \"}\", temp)\n            new_string = temp\n\n            return f\"{new_prefix}{new_string}\"\n        else:\n            return string\n\n    def _get_string_operator_leaves(self, leaves: Iterable[Leaf]) -> List[Leaf]:\n        LL = list(leaves)\n\n        string_op_leaves = []\n        i = 0\n        while LL[i].type in self.STRING_OPERATORS + [token.NAME]:\n            prefix_leaf = Leaf(LL[i].type, str(LL[i]).strip())\n            string_op_leaves.append(prefix_leaf)\n            i += 1\n        return string_op_leaves\n\n\nclass StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin):\n    \"\"\"\n    StringTransformer that wraps strings in parens and then splits at the LPAR.\n\n    Requirements:\n        All of the requirements listed in BaseStringSplitter's docstring in\n        addition to the requirements listed below:\n\n        * The line is a return/yield statement, which returns/yields a string.\n          OR\n        * The line is part of a ternary expression (e.g. `x = y if cond else\n          z`) such that the line starts with `else <string>`, where <string> is\n          some string.\n          OR\n        * The line is an assert statement, which ends with a string.\n          OR\n        * The line is an assignment statement (e.g. `x = <string>` or `x +=\n          <string>`) such that the variable is being assigned the value of some\n          string.\n          OR\n        * The line is a dictionary key assignment where some valid key is being\n          assigned the value of some string.\n          OR\n        * The line is an lambda expression and the value is a string.\n          OR\n        * The line starts with an \"atom\" string that prefers to be wrapped in\n          parens. It's preferred to be wrapped when it's is an immediate child of\n          a list/set/tuple literal, AND the string is surrounded by commas (or is\n          the first/last child).\n\n    Transformations:\n        The chosen string is wrapped in parentheses and then split at the LPAR.\n\n        We then have one line which ends with an LPAR and another line that\n        starts with the chosen string. The latter line is then split again at\n        the RPAR. This results in the RPAR (and possibly a trailing comma)\n        being placed on its own line.\n\n        NOTE: If any leaves exist to the right of the chosen string (except\n        for a trailing comma, which would be placed after the RPAR), those\n        leaves are placed inside the parentheses.  In effect, the chosen\n        string is not necessarily being \"wrapped\" by parentheses. We can,\n        however, count on the LPAR being placed directly before the chosen\n        string.\n\n        In other words, StringParenWrapper creates \"atom\" strings. These\n        can then be split again by StringSplitter, if necessary.\n\n    Collaborations:\n        In the event that a string line split by StringParenWrapper is\n        changed such that it no longer needs to be given its own line,\n        StringParenWrapper relies on StringParenStripper to clean up the\n        parentheses it created.\n\n        For \"atom\" strings that prefers to be wrapped in parens, it requires\n        StringSplitter to hold the split until the string is wrapped in parens.\n    \"\"\"\n\n    def do_splitter_match(self, line: Line) -> TMatchResult:\n        LL = line.leaves\n\n        if line.leaves[-1].type in OPENING_BRACKETS:\n            return TErr(\n                \"Cannot wrap parens around a line that ends in an opening bracket.\"\n            )\n\n        string_idx = (\n            self._return_match(LL)\n            or self._else_match(LL)\n            or self._assert_match(LL)\n            or self._assign_match(LL)\n            or self._dict_or_lambda_match(LL)\n            or self._prefer_paren_wrap_match(LL)\n        )\n\n        if string_idx is not None:\n            string_value = line.leaves[string_idx].value\n            # If the string has neither spaces nor East Asian stops...\n            if not any(\n                char == \" \" or char in SPLIT_SAFE_CHARS for char in string_value\n            ):\n                # And will still violate the line length limit when split...\n                max_string_width = self.line_length - ((line.depth + 1) * 4)\n                if str_width(string_value) > max_string_width:\n                    # And has no associated custom splits...\n                    if not self.has_custom_splits(string_value):\n                        # Then we should NOT put this string on its own line.\n                        return TErr(\n                            \"We do not wrap long strings in parentheses when the\"\n                            \" resultant line would still be over the specified line\"\n                            \" length and can't be split further by StringSplitter.\"\n                        )\n            return Ok([string_idx])\n\n        return TErr(\"This line does not contain any non-atomic strings.\")\n\n    @staticmethod\n    def _return_match(LL: List[Leaf]) -> Optional[int]:\n        \"\"\"\n        Returns:\n            string_idx such that @LL[string_idx] is equal to our target (i.e.\n            matched) string, if this line matches the return/yield statement\n            requirements listed in the 'Requirements' section of this classes'\n            docstring.\n                OR\n            None, otherwise.\n        \"\"\"\n        # If this line is a part of a return/yield statement and the first leaf\n        # contains either the \"return\" or \"yield\" keywords...\n        if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[\n            0\n        ].value in [\"return\", \"yield\"]:\n...\n", "needle_token_start": 508, "needle_token_end": 1066, "code_context_ntokens": 16122, "output": ["```python\ndef _merge_string_group(\n        self, line: Line, string_indices: List[int]\n    ) -> TResult[Line]:\n        \"\"\"\n        Merges string groups (i.e. set of adjacent strings).\n\n        Each index from `string_indices` designates one string group's first\n        leaf in `line.leaves`.\n\n        Returns:\n            Ok(new_line), if ALL of the validation checks found in\n            _validate_msg(...) pass.\n                OR\n            Err(CannotTransform), otherwise.\n        \"\"\"\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.\n        merged_string_idx_dict: Dict[int, Tuple[int, Leaf]] = {}\n        for string_idx in string_indices:\n            vresult = self._validate_msg(line, string_idx)\n            if isinstance(vresult, Err):\n                continue\n            merged_string_idx_dict[string_idx] = self._merge_one_string_group(\n                LL, string_idx, is_valid_index\n            )\n\n        if not merged_string_idx_dict:\n            return TErr(\"No string group is merged\")\n\n        # Build the final line ('new_line') that this method will later return.\n        new_line = line.clone()\n        previous_merged_string_idx = -1\n        previous_merged_num_of_strings = -1\n        for i, leaf in enumerate(LL):\n            if i in merged_string_idx_dict:\n                previous_merged_string_idx = i\n                previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]\n                new_line.append(string_leaf)\n\n            if (\n                previous_merged_string_idx\n                <= i\n                < previous_merged_string_idx + previous_merged_num_of_strings\n            ):\n                for comment_leaf in line.comments_after(LL[i]):\n                    new_line.append(comment_leaf, preformatted=True)\n                continue\n\n            append_leaves(new_line, line, [leaf])\n\n        return Ok(new_line)\n```"]}
{"repo": "psf/black", "name": "convert_unchanged_lines", "language": "python", "path": "src/black/ranges.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform specific lines of code that have not been modified into a special comment format that preserves their original formatting during further processing. This is particularly useful in scenarios where selective formatting is required, ensuring that unchanged parts of the code remain visually consistent and untouched.\n2. **Input**: The function takes two parameters: a node representing the parsed source code and a collection of line ranges that specify which lines have not been altered.\n3. **Output**: There is no direct output from this function as it modifies the input node in place. The changes involve converting specified lines into a format that will not be altered in subsequent formatting steps.\n4. **Procedure**: The function operates in two main phases:\n   - First, it identifies top-level statements within the unchanged lines and converts these blocks into a special comment type that preserves their formatting.\n   - Second, it processes individual unchanged lines within the node, converting them into the same special comment type. This includes normalizing comment prefixes and indentations to ensure consistency, even though the content itself remains unchanged.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/black/parsing.py\n\"\"\"\nParse Python code and perform AST validation.\n\"\"\"\n\nimport ast\nimport sys\nimport warnings\nfrom typing import Iterable, Iterator, List, Set, Tuple\n\nfrom black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature\nfrom black.nodes import syms\nfrom blib2to3 import pygram\nfrom blib2to3.pgen2 import driver\nfrom blib2to3.pgen2.grammar import Grammar\nfrom blib2to3.pgen2.parse import ParseError\nfrom blib2to3.pgen2.tokenize import TokenError\nfrom blib2to3.pytree import Leaf, Node\n\n\nclass InvalidInput(ValueError):\n    \"\"\"Raised when input source code fails all parse attempts.\"\"\"\n\n\ndef get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:\n    if not target_versions:\n        # No target_version specified, so try all grammars.\n        return [\n            # Python 3.7-3.9\n            pygram.python_grammar_async_keywords,\n            # Python 3.0-3.6\n            pygram.python_grammar,\n            # Python 3.10+\n            pygram.python_grammar_soft_keywords,\n        ]\n\n    grammars = []\n    # If we have to parse both, try to parse async as a keyword first\n    if not supports_feature(\n        target_versions, Feature.ASYNC_IDENTIFIERS\n    ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):\n        # Python 3.7-3.9\n        grammars.append(pygram.python_grammar_async_keywords)\n    if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):\n        # Python 3.0-3.6\n        grammars.append(pygram.python_grammar)\n    if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions):\n        # Python 3.10+\n        grammars.append(pygram.python_grammar_soft_keywords)\n\n    # At least one of the above branches must have been taken, because every Python\n    # version has exactly one of the two 'ASYNC_*' flags\n    return grammars\n\n\ndef lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:\n    \"\"\"Given a string with source, return the lib2to3 Node.\"\"\"\n    if not src_txt.endswith(\"\\n\"):\n        src_txt += \"\\n\"\n\n    grammars = get_grammars(set(target_versions))\n    errors = {}\n    for grammar in grammars:\n        drv = driver.Driver(grammar)\n        try:\n            result = drv.parse_string(src_txt, True)\n            break\n\n        except ParseError as pe:\n            lineno, column = pe.context[1]\n            lines = src_txt.splitlines()\n            try:\n                faulty_line = lines[lineno - 1]\n            except IndexError:\n                faulty_line = \"<line number missing in source>\"\n            errors[grammar.version] = InvalidInput(\n                f\"Cannot parse: {lineno}:{column}: {faulty_line}\"\n            )\n\n        except TokenError as te:\n            # In edge cases these are raised; and typically don't have a \"faulty_line\".\n            lineno, column = te.args[1]\n            errors[grammar.version] = InvalidInput(\n                f\"Cannot parse: {lineno}:{column}: {te.args[0]}\"\n            )\n\n    else:\n        # Choose the latest version when raising the actual parsing error.\n        assert len(errors) >= 1\n...\n# Path: src/black/ranges.py\n\"\"\"Functions related to Black's formatting by line ranges feature.\"\"\"\n\nimport difflib\nfrom dataclasses import dataclass\nfrom typing import Collection, Iterator, List, Sequence, Set, Tuple, Union\n\nfrom black.nodes import (\n    LN,\n    STANDALONE_COMMENT,\n    Leaf,\n    Node,\n    Visitor,\n    first_leaf,\n    furthest_ancestor_with_last_leaf,\n    last_leaf,\n    syms,\n)\nfrom blib2to3.pgen2.token import ASYNC, NEWLINE\n\n\ndef parse_line_ranges(line_ranges: Sequence[str]) -> List[Tuple[int, int]]:\n    lines: List[Tuple[int, int]] = []\n    for lines_str in line_ranges:\n        parts = lines_str.split(\"-\")\n        if len(parts) != 2:\n            raise ValueError(\n                \"Incorrect --line-ranges format, expect 'START-END', found\"\n                f\" {lines_str!r}\"\n            )\n        try:\n            start = int(parts[0])\n            end = int(parts[1])\n        except ValueError:\n            raise ValueError(\n                \"Incorrect --line-ranges value, expect integer ranges, found\"\n                f\" {lines_str!r}\"\n            ) from None\n        else:\n            lines.append((start, end))\n    return lines\n\n\ndef is_valid_line_range(lines: Tuple[int, int]) -> bool:\n    \"\"\"Returns whether the line range is valid.\"\"\"\n    return not lines or lines[0] <= lines[1]\n\n\ndef adjusted_lines(\n    lines: Collection[Tuple[int, int]],\n    original_source: str,\n    modified_source: str,\n) -> List[Tuple[int, int]]:\n    \"\"\"Returns the adjusted line ranges based on edits from the original code.\n\n    This computes the new line ranges by diffing original_source and\n    modified_source, and adjust each range based on how the range overlaps with\n    the diffs.\n\n    Note the diff can contain lines outside of the original line ranges. This can\n    happen when the formatting has to be done in adjacent to maintain consistent\n    local results. For example:\n\n    1. def my_func(arg1, arg2,\n    2.             arg3,):\n    3.   pass\n\n    If it restricts to line 2-2, it can't simply reformat line 2, it also has\n    to reformat line 1:\n\n    1. def my_func(\n    2.     arg1,\n    3.     arg2,\n    4.     arg3,\n    5. ):\n    6.   pass\n\n    In this case, we will expand the line ranges to also include the whole diff\n    block.\n\n    Args:\n      lines: a collection of line ranges.\n      original_source: the original source.\n      modified_source: the modified source.\n    \"\"\"\n    lines_mappings = _calculate_lines_mappings(original_source, modified_source)\n\n    new_lines = []\n    # Keep an index of the current search. Since the lines and lines_mappings are\n    # sorted, this makes the search complexity linear.\n    current_mapping_index = 0\n    for start, end in sorted(lines):\n        start_mapping_index = _find_lines_mapping_index(\n            start,\n            lines_mappings,\n            current_mapping_index,\n        )\n        end_mapping_index = _find_lines_mapping_index(\n            end,\n            lines_mappings,\n            start_mapping_index,\n        )\n        current_mapping_index = start_mapping_index\n        if start_mapping_index >= len(lines_mappings) or end_mapping_index >= len(\n            lines_mappings\n        ):\n            # Protect against invalid inputs.\n            continue\n        start_mapping = lines_mappings[start_mapping_index]\n        end_mapping = lines_mappings[end_mapping_index]\n        if start_mapping.is_changed_block:\n            # When the line falls into a changed block, expands to the whole block.\n            new_start = start_mapping.modified_start\n        else:\n            new_start = (\n                start - start_mapping.original_start + start_mapping.modified_start\n            )\n        if end_mapping.is_changed_block:\n            # When the line falls into a changed block, expands to the whole block.\n            new_end = end_mapping.modified_end\n        else:\n            new_end = end - end_mapping.original_start + end_mapping.modified_start\n        new_range = (new_start, new_end)\n        if is_valid_line_range(new_range):\n            new_lines.append(new_range)\n    return new_lines\n\n\n\ndef convert_unchanged_lines(src_node: Node, lines: Collection[Tuple[int, int]]) -> None:\n    \"\"\"Converts unchanged lines to STANDALONE_COMMENT.\n\n    The idea is similar to how `# fmt: on/off` is implemented. It also converts the\n    nodes between those markers as a single `STANDALONE_COMMENT` leaf node with\n    the unformatted code as its value. `STANDALONE_COMMENT` is a \"fake\" token\n    that will be formatted as-is with its prefix normalized.\n\n    Here we perform two passes:\n\n    1. Visit the top-level statements, and convert them to a single\n       `STANDALONE_COMMENT` when unchanged. This speeds up formatting when some\n       of the top-level statements aren't changed.\n    2. Convert unchanged \"unwrapped lines\" to `STANDALONE_COMMENT` nodes line by\n       line. \"unwrapped lines\" are divided by the `NEWLINE` token. e.g. a\n       multi-line statement is *one* \"unwrapped line\" that ends with `NEWLINE`,\n       even though this statement itself can span multiple lines, and the\n       tokenizer only sees the last '\\n' as the `NEWLINE` token.\n\n    NOTE: During pass (2), comment prefixes and indentations are ALWAYS\n    normalized even when the lines aren't changed. This is fixable by moving\n    more formatting to pass (1). However, it's hard to get it correct when\n    incorrect indentations are used. So we defer this to future optimizations.\n    \"\"\"\n    lines_set: Set[int] = set()\n    for start, end in lines:\n        lines_set.update(range(start, end + 1))\n    visitor = _TopLevelStatementsVisitor(lines_set)\n    _ = list(visitor.visit(src_node))  # Consume all results.\n    _convert_unchanged_line_by_line(src_node, lines_set)\n\n\ndef _contains_standalone_comment(node: LN) -> bool:\n    if isinstance(node, Leaf):\n        return node.type == STANDALONE_COMMENT\n    else:\n        for child in node.children:\n            if _contains_standalone_comment(child):\n                return True\n        return False\n\n\nclass _TopLevelStatementsVisitor(Visitor[None]):\n    \"\"\"\n    A node visitor that converts unchanged top-level statements to\n    STANDALONE_COMMENT.\n\n    This is used in addition to _convert_unchanged_line_by_line, to\n    speed up formatting when there are unchanged top-level\n    classes/functions/statements.\n    \"\"\"\n\n    def __init__(self, lines_set: Set[int]):\n        self._lines_set = lines_set\n\n    def visit_simple_stmt(self, node: Node) -> Iterator[None]:\n        # This is only called for top-level statements, since `visit_suite`\n        # won't visit its children nodes.\n        yield from []\n        newline_leaf = last_leaf(node)\n        if not newline_leaf:\n            return\n        assert (\n            newline_leaf.type == NEWLINE\n        ), f\"Unexpectedly found leaf.type={newline_leaf.type}\"\n        # We need to find the furthest ancestor with the NEWLINE as the last\n        # leaf, since a `suite` can simply be a `simple_stmt` when it puts\n        # its body on the same line. Example: `if cond: pass`.\n        ancestor = furthest_ancestor_with_last_leaf(newline_leaf)\n        if not _get_line_range(ancestor).intersection(self._lines_set):\n            _convert_node_to_standalone_comment(ancestor)\n\n    def visit_suite(self, node: Node) -> Iterator[None]:\n        yield from []\n        # If there is a STANDALONE_COMMENT node, it means parts of the node tree\n        # have fmt on/off/skip markers. Those STANDALONE_COMMENT nodes can't\n        # be simply converted by calling str(node). So we just don't convert\n        # here.\n        if _contains_standalone_comment(node):\n            return\n        # Find the semantic parent of this suite. For `async_stmt` and\n        # `async_funcdef`, the ASYNC token is defined on a separate level by the\n        # grammar.\n        semantic_parent = node.parent\n        if semantic_parent is not None:\n            if (\n                semantic_parent.prev_sibling is not None\n                and semantic_parent.prev_sibling.type == ASYNC\n            ):\n                semantic_parent = semantic_parent.parent\n        if semantic_parent is not None and not _get_line_range(\n            semantic_parent\n        ).intersection(self._lines_set):\n            _convert_node_to_standalone_comment(semantic_parent)\n\n\ndef _convert_unchanged_line_by_line(node: Node, lines_set: Set[int]) -> None:\n    \"\"\"Converts unchanged to STANDALONE_COMMENT line by line.\"\"\"\n    for leaf in node.leaves():\n        if leaf.type != NEWLINE:\n            # We only consider \"unwrapped lines\", which are divided by the NEWLINE\n            # token.\n            continue\n        if leaf.parent and leaf.parent.type == syms.match_stmt:\n            # The `suite` node is defined as:\n            #   match_stmt: \"match\" subject_expr ':' NEWLINE INDENT case_block+ DEDENT\n            # Here we need to check `subject_expr`. The `case_block+` will be\n            # checked by their own NEWLINEs.\n            nodes_to_ignore: List[LN] = []\n            prev_sibling = leaf.prev_sibling\n            while prev_sibling:\n                nodes_to_ignore.insert(0, prev_sibling)\n                prev_sibling = prev_sibling.prev_sibling\n            if not _get_line_range(nodes_to_ignore).intersection(lines_set):\n                _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf)\n        elif leaf.parent and leaf.parent.type == syms.suite:\n            # The `suite` node is defined as:\n            #   suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT\n            # We will check `simple_stmt` and `stmt+` separately against the lines set\n            parent_sibling = leaf.parent.prev_sibling\n            nodes_to_ignore = []\n            while parent_sibling and not parent_sibling.type == syms.suite:\n                # NOTE: Multiple suite nodes can exist as siblings in e.g. `if_stmt`.\n                nodes_to_ignore.insert(0, parent_sibling)\n                parent_sibling = parent_sibling.prev_sibling\n            # Special case for `async_stmt` and `async_funcdef` where the ASYNC\n            # token is on the grandparent node.\n            grandparent = leaf.parent.parent\n            if (\n                grandparent is not None\n                and grandparent.prev_sibling is not None\n                and grandparent.prev_sibling.type == ASYNC\n            ):\n                nodes_to_ignore.insert(0, grandparent.prev_sibling)\n            if not _get_line_range(nodes_to_ignore).intersection(lines_set):\n                _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf)\n        else:\n            ancestor = furthest_ancestor_with_last_leaf(leaf)\n            # Consider multiple decorators as a whole block, as their\n            # newlines have different behaviors than the rest of the grammar.\n            if (\n                ancestor.type == syms.decorator\n                and ancestor.parent\n                and ancestor.parent.type == syms.decorators\n            ):\n                ancestor = ancestor.parent\n            if not _get_line_range(ancestor).intersection(lines_set):\n                _convert_node_to_standalone_comment(ancestor)\n\n\ndef _convert_node_to_standalone_comment(node: LN) -> None:\n    \"\"\"Convert node to STANDALONE_COMMENT by modifying the tree inline.\"\"\"\n    parent = node.parent\n    if not parent:\n        return\n    first = first_leaf(node)\n    last = last_leaf(node)\n    if not first or not last:\n        return\n    if first is last:\n        # This can happen on the following edge cases:\n        # 1. A block of `# fmt: off/on` code except the `# fmt: on` is placed\n        #    on the end of the last line instead of on a new line.\n        # 2. A single backslash on its own line followed by a comment line.\n        # Ideally we don't want to format them when not requested, but fixing\n        # isn't easy. These cases are also badly formatted code, so it isn't\n        # too bad we reformat them.\n        return\n    # The prefix contains comments and indentation whitespaces. They are\n    # reformatted accordingly to the correct indentation level.\n    # This also means the indentation will be changed on the unchanged lines, and\n    # this is actually required to not break incremental reformatting.\n    prefix = first.prefix\n    first.prefix = \"\"\n    index = node.remove()\n    if index is not None:\n        # Remove the '\\n', as STANDALONE_COMMENT will have '\\n' appended when\n        # generating the formatted code.\n        value = str(node)[:-1]\n        parent.insert_child(\n            index,\n            Leaf(\n                STANDALONE_COMMENT,\n                value,\n                prefix=prefix,\n                fmt_pass_converted_first_leaf=first,\n            ),\n        )\n\n\ndef _convert_nodes_to_standalone_comment(nodes: Sequence[LN], *, newline: Leaf) -> None:\n    \"\"\"Convert nodes to STANDALONE_COMMENT by modifying the tree inline.\"\"\"\n    if not nodes:\n        return\n    parent = nodes[0].parent\n    first = first_leaf(nodes[0])\n    if not parent or not first:\n        return\n    prefix = first.prefix\n    first.prefix = \"\"\n    value = \"\".join(str(node) for node in nodes)\n    # The prefix comment on the NEWLINE leaf is the trailing comment of the statement.\n    if newline.prefix:\n        value += newline.prefix\n        newline.prefix = \"\"\n    index = nodes[0].remove()\n    for node in nodes[1:]:\n        node.remove()\n    if index is not None:\n        parent.insert_child(\n            index,\n            Leaf(\n                STANDALONE_COMMENT,\n                value,\n                prefix=prefix,\n                fmt_pass_converted_first_leaf=first,\n            ),\n        )\n\n\ndef _leaf_line_end(leaf: Leaf) -> int:\n    \"\"\"Returns the line number of the leaf node's last line.\"\"\"\n    if leaf.type == NEWLINE:\n        return leaf.lineno\n    else:\n        # Leaf nodes like multiline strings can occupy multiple lines.\n        return leaf.lineno + str(leaf).count(\"\\n\")\n\n\ndef _get_line_range(node_or_nodes: Union[LN, List[LN]]) -> Set[int]:\n    \"\"\"Returns the line range of this node or list of nodes.\"\"\"\n    if isinstance(node_or_nodes, list):\n        nodes = node_or_nodes\n        if not nodes:\n            return set()\n        first = first_leaf(nodes[0])\n        last = last_leaf(nodes[-1])\n        if first and last:\n            line_start = first.lineno\n            line_end = _leaf_line_end(last)\n            return set(range(line_start, line_end + 1))\n        else:\n            return set()\n    else:\n        node = node_or_nodes\n        if isinstance(node, Leaf):\n            return set(range(node.lineno, _leaf_line_end(node) + 1))\n        else:\n            first = first_leaf(node)\n            last = last_leaf(node)\n            if first and last:\n                return set(range(first.lineno, _leaf_line_end(last) + 1))\n            else:\n                return set()\n\n\n@dataclass\nclass _LinesMapping:\n    \"\"\"1-based lines mapping from original source to modified source.\n\n    Lines [original_start, original_end] from original source\n    are mapped to [modified_start, modified_end].\n\n    The ranges are inclusive on both ends.\n    \"\"\"\n\n    original_start: int\n    original_end: int\n    modified_start: int\n    modified_end: int\n    # Whether this range corresponds to a changed block, or an unchanged block.\n    is_changed_block: bool\n\n\ndef _calculate_lines_mappings(\n    original_source: str,\n    modified_source: str,\n) -> Sequence[_LinesMapping]:\n    \"\"\"Returns a sequence of _LinesMapping by diffing the sources.\n\n    For example, given the following diff:\n        import re\n      - def func(arg1,\n      -   arg2, arg3):\n      + def func(arg1, arg2, arg3):\n          pass\n    It returns the following mappings:\n      original -> modified\n       (1, 1)  ->  (1, 1), is_changed_block=False (the \"import re\" line)\n       (2, 3)  ->  (2, 2), is_changed_block=True (the diff)\n       (4, 4)  ->  (3, 3), is_changed_block=False (the \"pass\" line)\n\n    You can think of this visually as if it brings up a side-by-side diff, and tries\n    to map the line ranges from the left side to the right side:\n\n      (1, 1)->(1, 1)    1. import re          1. import re\n      (2, 3)->(2, 2)    2. def func(arg1,     2. def func(arg1, arg2, arg3):\n                        3.   arg2, arg3):\n      (4, 4)->(3, 3)    4.   pass             3.   pass\n\n    Args:\n      original_source: the original source.\n      modified_source: the modified source.\n    \"\"\"\n    matcher = difflib.SequenceMatcher(\n        None,\n        original_source.splitlines(keepends=True),\n        modified_source.splitlines(keepends=True),\n    )\n    matching_blocks = matcher.get_matching_blocks()\n    lines_mappings: List[_LinesMapping] = []\n    # matching_blocks is a sequence of \"same block of code ranges\", see\n    # https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_matching_blocks\n    # Each block corresponds to a _LinesMapping with is_changed_block=False,\n    # and the ranges between two blocks corresponds to a _LinesMapping with\n    # is_changed_block=True,\n    # NOTE: matching_blocks is 0-based, but _LinesMapping is 1-based.\n    for i, block in enumerate(matching_blocks):\n        if i == 0:\n            if block.a != 0 or block.b != 0:\n                lines_mappings.append(\n                    _LinesMapping(\n                        original_start=1,\n                        original_end=block.a,\n                        modified_start=1,\n                        modified_end=block.b,\n                        is_changed_block=False,\n                    )\n                )\n        else:\n            previous_block = matching_blocks[i - 1]\n            lines_mappings.append(\n                _LinesMapping(\n                    original_start=previous_block.a + previous_block.size + 1,\n                    original_end=block.a,\n                    modified_start=previous_block.b + previous_block.size + 1,\n                    modified_end=block.b,\n                    is_changed_block=True,\n                )\n            )\n        if i < len(matching_blocks) - 1:\n            lines_mappings.append(\n                _LinesMapping(\n                    original_start=block.a + 1,\n                    original_end=block.a + block.size,\n                    modified_start=block.b + 1,\n                    modified_end=block.b + block.size,\n                    is_changed_block=False,\n                )\n            )\n    return lines_mappings\n\n\ndef _find_lines_mapping_index(\n    original_line: int,\n    lines_mappings: Sequence[_LinesMapping],\n    start_index: int,\n) -> int:\n    \"\"\"Returns the original index of the lines mappings for the original line.\"\"\"\n    index = start_index\n    while index < len(lines_mappings):\n        mapping = lines_mappings[index]\n        if mapping.original_start <= original_line <= mapping.original_end:\n            return index\n        index += 1\n    return index\n\n# Path: src/black/__init__.py\nimport io\nimport json\nimport platform\nimport re\nimport sys\nimport tokenize\nimport traceback\nfrom contextlib import contextmanager\nfrom dataclasses import replace\nfrom datetime import datetime, timezone\nfrom enum import Enum\nfrom json.decoder import JSONDecodeError\nfrom pathlib import Path\nfrom typing import (\n    Any,\n    Collection,\n    Dict,\n    Generator,\n    Iterator,\n    List,\n    MutableMapping,\n    Optional,\n    Pattern,\n    Sequence,\n    Set,\n    Sized,\n    Tuple,\n    Union,\n)\n\nimport click\nfrom click.core import ParameterSource\nfrom mypy_extensions import mypyc_attr\nfrom pathspec import PathSpec\nfrom pathspec.patterns.gitwildmatch import GitWildMatchPatternError\n\nfrom _black_version import version as __version__\nfrom black.cache import Cache\nfrom black.comments import normalize_fmt_off\nfrom black.const import (\n    DEFAULT_EXCLUDES,\n    DEFAULT_INCLUDES,\n    DEFAULT_LINE_LENGTH,\n    STDIN_PLACEHOLDER,\n)\nfrom black.files import (\n    best_effort_relative_path,\n    find_project_root,\n    find_pyproject_toml,\n    find_user_pyproject_toml,\n    gen_python_files,\n    get_gitignore,\n    parse_pyproject_toml,\n    path_is_excluded,\n    resolves_outside_root_or_cannot_stat,\n    wrap_stream_for_windows,\n)\nfrom black.handle_ipynb_magics import (\n    PYTHON_CELL_MAGICS,\n    TRANSFORMED_MAGICS,\n    jupyter_dependencies_are_installed,\n    mask_cell,\n    put_trailing_semicolon_back,\n    remove_trailing_semicolon,\n    unmask_cell,\n)\nfrom black.linegen import LN, LineGenerator, transform_line\nfrom black.lines import EmptyLineTracker, LinesBlock\nfrom black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature\nfrom black.mode import Mode as Mode  # re-exported\nfrom black.mode import Preview, TargetVersion, supports_feature\nfrom black.nodes import (\n    STARS,\n    is_number_token,\n    is_simple_decorator_expression,\n    is_string_token,\n    syms,\n)\nfrom black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out\nfrom black.parsing import InvalidInput  # noqa F401\nfrom black.parsing import lib2to3_parse, parse_ast, stringify_ast\nfrom black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges\nfrom black.report import Changed, NothingChanged, Report\nfrom black.trans import iter_fexpr_spans\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\nCOMPILED = Path(__file__).suffix in (\".pyd\", \".so\")\n\n# types\nFileContent = str\nEncoding = str\nNewLine = str\n\n\nclass WriteBack(Enum):\n    NO = 0\n    YES = 1\n    DIFF = 2\n    CHECK = 3\n    COLOR_DIFF = 4\n\n    @classmethod\n    def from_configuration(\n        cls, *, check: bool, diff: bool, color: bool = False\n    ) -> \"WriteBack\":\n        if check and not diff:\n            return cls.CHECK\n\n        if diff and color:\n            return cls.COLOR_DIFF\n\n        return cls.DIFF if diff else cls.YES\n\n\n# Legacy name, left for integrations.\nFileMode = Mode\n\n\ndef read_pyproject_toml(\n    ctx: click.Context, param: click.Parameter, value: Optional[str]\n) -> Optional[str]:\n    \"\"\"Inject Black configuration from \"pyproject.toml\" into defaults in `ctx`.\n\n    Returns the path to a successfully found and read configuration file, None\n    otherwise.\n    \"\"\"\n    if not value:\n        value = find_pyproject_toml(\n            ctx.params.get(\"src\", ()), ctx.params.get(\"stdin_filename\", None)\n        )\n        if value is None:\n            return None\n\n    try:\n        config = parse_pyproject_toml(value)\n    except (OSError, ValueError) as e:\n        raise click.FileError(\n            filename=value, hint=f\"Error reading configuration file: {e}\"\n        ) from None\n\n    if not config:\n        return None\n    else:\n        spellcheck_pyproject_toml_keys(ctx, list(config), value)\n        # Sanitize the values to be Click friendly. For more information please see:\n        # https://github.com/psf/black/issues/1458\n        # https://github.com/pallets/click/issues/1567\n        config = {\n            k: str(v) if not isinstance(v, (list, dict)) else v\n            for k, v in config.items()\n        }\n\n    target_version = config.get(\"target_version\")\n    if target_version is not None and not isinstance(target_version, list):\n        raise click.BadOptionUsage(\n            \"target-version\", \"Config key target-version must be a list\"\n        )\n\n    exclude = config.get(\"exclude\")\n    if exclude is not None and not isinstance(exclude, str):\n        raise click.BadOptionUsage(\"exclude\", \"Config key exclude must be a string\")\n\n    extend_exclude = config.get(\"extend_exclude\")\n    if extend_exclude is not None and not isinstance(extend_exclude, str):\n        raise click.BadOptionUsage(\n            \"extend-exclude\", \"Config key extend-exclude must be a string\"\n        )\n\n    line_ranges = config.get(\"line_ranges\")\n    if line_ranges is not None:\n        raise click.BadOptionUsage(\n            \"line-ranges\", \"Cannot use line-ranges in the pyproject.toml file.\"\n        )\n\n    default_map: Dict[str, Any] = {}\n    if ctx.default_map:\n        default_map.update(ctx.default_map)\n    default_map.update(config)\n\n    ctx.default_map = default_map\n    return value\n\n\ndef spellcheck_pyproject_toml_keys(\n    ctx: click.Context, config_keys: List[str], config_file_path: str\n) -> None:\n    invalid_keys: List[str] = []\n    available_config_options = {param.name for param in ctx.command.params}\n    for key in config_keys:\n        if key not in available_config_options:\n            invalid_keys.append(key)\n    if invalid_keys:\n        keys_str = \", \".join(map(repr, invalid_keys))\n        out(\n            f\"Invalid config keys detected: {keys_str} (in {config_file_path})\",\n            fg=\"red\",\n        )\n\n\ndef target_version_option_callback(\n    c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]\n) -> List[TargetVersion]:\n    \"\"\"Compute the target versions from a --target-version flag.\n\n    This is its own function because mypy couldn't infer the type correctly\n    when it was a lambda, causing mypyc trouble.\n    \"\"\"\n    return [TargetVersion[val.upper()] for val in v]\n\n\ndef enable_unstable_feature_callback(\n    c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]\n) -> List[Preview]:\n    \"\"\"Compute the features from an --enable-unstable-feature flag.\"\"\"\n    return [Preview[val] for val in v]\n\n\ndef re_compile_maybe_verbose(regex: str) -> Pattern[str]:\n    \"\"\"Compile a regular expression string in `regex`.\n\n    If it contains newlines, use verbose mode.\n    \"\"\"\n    if \"\\n\" in regex:\n        regex = \"(?x)\" + regex\n    compiled: Pattern[str] = re.compile(regex)\n    return compiled\n\n\ndef validate_regex(\n    ctx: click.Context,\n    param: click.Parameter,\n    value: Optional[str],\n) -> Optional[Pattern[str]]:\n    try:\n        return re_compile_maybe_verbose(value) if value is not None else None\n    except re.error as e:\n        raise click.BadParameter(f\"Not a valid regular expression: {e}\") from None\n\n\n@click.command(\n    context_settings={\"help_option_names\": [\"-h\", \"--help\"]},\n    # While Click does set this field automatically using the docstring, mypyc\n    # (annoyingly) strips 'em so we need to set it here too.\n    help=\"The uncompromising code formatter.\",\n)\n@click.option(\"-c\", \"--code\", type=str, help=\"Format the code passed in as a string.\")\n@click.option(\n    \"-l\",\n    \"--line-length\",\n    type=int,\n    default=DEFAULT_LINE_LENGTH,\n    help=\"How many characters per line to allow.\",\n    show_default=True,\n)\n@click.option(\n    \"-t\",\n    \"--target-version\",\n    type=click.Choice([v.name.lower() for v in TargetVersion]),\n    callback=target_version_option_callback,\n    multiple=True,\n    help=(\n        \"Python versions that should be supported by Black's output. You should\"\n        \" include all versions that your code supports. By default, Black will infer\"\n        \" target versions from the project metadata in pyproject.toml. If this does\"\n        \" not yield conclusive results, Black will use per-file auto-detection.\"\n    ),\n)\n@click.option(\n    \"--pyi\",\n    is_flag=True,\n    help=(\n        \"Format all input files like typing stubs regardless of file extension. This\"\n        \" is useful when piping source on standard input.\"\n    ),\n)\n@click.option(\n    \"--ipynb\",\n    is_flag=True,\n    help=(\n        \"Format all input files like Jupyter Notebooks regardless of file extension.\"\n        \" This is useful when piping source on standard input.\"\n    ),\n)\n@click.option(\n    \"--python-cell-magics\",\n    multiple=True,\n    help=(\n        \"When processing Jupyter Notebooks, add the given magic to the list\"\n        f\" of known python-magics ({', '.join(sorted(PYTHON_CELL_MAGICS))}).\"\n        \" Useful for formatting cells with custom python magics.\"\n    ),\n    default=[],\n)\n@click.option(\n    \"-x\",\n    \"--skip-source-first-line\",\n    is_flag=True,\n    help=\"Skip the first line of the source code.\",\n)\n@click.option(\n    \"-S\",\n    \"--skip-string-normalization\",\n    is_flag=True,\n    help=\"Don't normalize string quotes or prefixes.\",\n)\n@click.option(\n    \"-C\",\n    \"--skip-magic-trailing-comma\",\n    is_flag=True,\n    help=\"Don't use trailing commas as a reason to split lines.\",\n)\n@click.option(\n    \"--preview\",\n    is_flag=True,\n    help=(\n        \"Enable potentially disruptive style changes that may be added to Black's main\"\n        \" functionality in the next major release.\"\n    ),\n)\n@click.option(\n    \"--unstable\",\n    is_flag=True,\n    help=(\n        \"Enable potentially disruptive style changes that have known bugs or are not\"\n        \" currently expected to make it into the stable style Black's next major\"\n        \" release. Implies --preview.\"\n    ),\n)\n@click.option(\n    \"--enable-unstable-feature\",\n    type=click.Choice([v.name for v in Preview]),\n    callback=enable_unstable_feature_callback,\n    multiple=True,\n    help=(\n        \"Enable specific features included in the `--unstable` style. Requires\"\n        \" `--preview`. No compatibility guarantees are provided on the behavior\"\n        \" or existence of any unstable features.\"\n    ),\n)\n@click.option(\n    \"--check\",\n    is_flag=True,\n    help=(\n        \"Don't write the files back, just return the status. Return code 0 means\"\n        \" nothing would change. Return code 1 means some files would be reformatted.\"\n        \" Return code 123 means there was an internal error.\"\n    ),\n)\n@click.option(\n    \"--diff\",\n    is_flag=True,\n    help=(\n        \"Don't write the files back, just output a diff to indicate what changes\"\n        \" Black would've made. They are printed to stdout so capturing them is simple.\"\n    ),\n)\n@click.option(\n    \"--color/--no-color\",\n    is_flag=True,\n    help=\"Show (or do not show) colored diff. Only applies when --diff is given.\",\n)\n@click.option(\n    \"--line-ranges\",\n    multiple=True,\n    metavar=\"START-END\",\n    help=(\n        \"When specified, Black will try its best to only format these lines. This\"\n        \" option can be specified multiple times, and a union of the lines will be\"\n        \" formatted. Each range must be specified as two integers connected by a `-`:\"\n        \" `<START>-<END>`. The `<START>` and `<END>` integer indices are 1-based and\"\n        \" inclusive on both ends.\"\n    ),\n    default=(),\n)\n@click.option(\n    \"--fast/--safe\",\n    is_flag=True,\n    help=(\n        \"By default, Black performs an AST safety check after formatting your code.\"\n        \" The --fast flag turns off this check and the --safe flag explicitly enables\"\n        \" it. [default: --safe]\"\n    ),\n)\n@click.option(\n    \"--required-version\",\n    type=str,\n    help=(\n        \"Require a specific version of Black to be running. This is useful for\"\n        \" ensuring that all contributors to your project are using the same\"\n        \" version, because different versions of Black may format code a little\"\n        \" differently. This option can be set in a configuration file for consistent\"\n        \" results across environments.\"\n    ),\n)\n@click.option(\n    \"--exclude\",\n    type=str,\n    callback=validate_regex,\n    help=(\n        \"A regular expression that matches files and directories that should be\"\n        \" excluded on recursive searches. An empty value means no paths are excluded.\"\n        \" Use forward slashes for directories on all platforms (Windows, too).\"\n        \" By default, Black also ignores all paths listed in .gitignore. Changing this\"\n        f\" value will override all default exclusions. [default: {DEFAULT_EXCLUDES}]\"\n    ),\n    show_default=False,\n)\n@click.option(\n    \"--extend-exclude\",\n    type=str,\n    callback=validate_regex,\n    help=(\n        \"Like --exclude, but adds additional files and directories on top of the\"\n        \" default values instead of overriding them.\"\n    ),\n)\n@click.option(\n    \"--force-exclude\",\n    type=str,\n    callback=validate_regex,\n    help=(\n        \"Like --exclude, but files and directories matching this regex will be excluded\"\n        \" even when they are passed explicitly as arguments. This is useful when\"\n        \" invoking Black programmatically on changed files, such as in a pre-commit\"\n        \" hook or editor plugin.\"\n    ),\n)\n@click.option(\n    \"--stdin-filename\",\n    type=str,\n    is_eager=True,\n    help=(\n        \"The name of the file when passing it through stdin. Useful to make sure Black\"\n        \" will respect the --force-exclude option on some editors that rely on using\"\n        \" stdin.\"\n    ),\n)\n@click.option(\n    \"--include\",\n    type=str,\n    default=DEFAULT_INCLUDES,\n    callback=validate_regex,\n    help=(\n        \"A regular expression that matches files and directories that should be\"\n        \" included on recursive searches. An empty value means all files are included\"\n        \" regardless of the name. Use forward slashes for directories on all platforms\"\n        \" (Windows, too). Overrides all exclusions, including from .gitignore and\"\n        \" command line options.\"\n    ),\n    show_default=True,\n)\n@click.option(\n    \"-W\",\n    \"--workers\",\n    type=click.IntRange(min=1),\n    default=None,\n    help=(\n        \"When Black formats multiple files, it may use a process pool to speed up\"\n        \" formatting. This option controls the number of parallel workers. This can\"\n        \" also be specified via the BLACK_NUM_WORKERS environment variable. Defaults\"\n        \" to the number of CPUs in the system.\"\n    ),\n)\n@click.option(\n    \"-q\",\n    \"--quiet\",\n    is_flag=True,\n    help=(\n        \"Stop emitting all non-critical output. Error messages will still be emitted\"\n        \" (which can silenced by 2>/dev/null).\"\n    ),\n)\n@click.option(\n    \"-v\",\n    \"--verbose\",\n    is_flag=True,\n    help=(\n        \"Emit messages about files that were not changed or were ignored due to\"\n        \" exclusion patterns. If Black is using a configuration file, a message\"\n        \" detailing which one it is using will be emitted.\"\n    ),\n)\n@click.version_option(\n    version=__version__,\n    message=(\n        f\"%(prog)s, %(version)s (compiled: {'yes' if COMPILED else 'no'})\\n\"\n        f\"Python ({platform.python_implementation()}) {platform.python_version()}\"\n    ),\n)\n@click.argument(\n    \"src\",\n    nargs=-1,\n    type=click.Path(\n        exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True\n    ),\n    is_eager=True,\n    metavar=\"SRC ...\",\n)\n@click.option(\n    \"--config\",\n    type=click.Path(\n        exists=True,\n        file_okay=True,\n        dir_okay=False,\n        readable=True,\n        allow_dash=False,\n        path_type=str,\n    ),\n    is_eager=True,\n    callback=read_pyproject_toml,\n    help=\"Read configuration options from a configuration file.\",\n)\n@click.pass_context\ndef main(  # noqa: C901\n    ctx: click.Context,\n    code: Optional[str],\n    line_length: int,\n    target_version: List[TargetVersion],\n    check: bool,\n    diff: bool,\n    line_ranges: Sequence[str],\n    color: bool,\n    fast: bool,\n    pyi: bool,\n    ipynb: bool,\n    python_cell_magics: Sequence[str],\n    skip_source_first_line: bool,\n    skip_string_normalization: bool,\n    skip_magic_trailing_comma: bool,\n    preview: bool,\n    unstable: bool,\n    enable_unstable_feature: List[Preview],\n    quiet: bool,\n    verbose: bool,\n    required_version: Optional[str],\n    include: Pattern[str],\n    exclude: Optional[Pattern[str]],\n    extend_exclude: Optional[Pattern[str]],\n    force_exclude: Optional[Pattern[str]],\n    stdin_filename: Optional[str],\n    workers: Optional[int],\n    src: Tuple[str, ...],\n    config: Optional[str],\n) -> None:\n    \"\"\"The uncompromising code formatter.\"\"\"\n    ctx.ensure_object(dict)\n\n    if src and code is not None:\n        out(\n            main.get_usage(ctx)\n            + \"\\n\\n'SRC' and 'code' cannot be passed simultaneously.\"\n        )\n        ctx.exit(1)\n    if not src and code is None:\n        out(main.get_usage(ctx) + \"\\n\\nOne of 'SRC' or 'code' is required.\")\n        ctx.exit(1)\n\n    # It doesn't do anything if --unstable is also passed, so just allow it.\n    if enable_unstable_feature and not (preview or unstable):\n        out(\n            main.get_usage(ctx)\n            + \"\\n\\n'--enable-unstable-feature' requires '--preview'.\"\n        )\n        ctx.exit(1)\n\n    root, method = (\n        find_project_root(src, stdin_filename) if code is None else (None, None)\n    )\n    ctx.obj[\"root\"] = root\n\n    if verbose:\n        if root:\n            out(\n                f\"Identified `{root}` as project root containing a {method}.\",\n                fg=\"blue\",\n            )\n\n        if config:\n            config_source = ctx.get_parameter_source(\"config\")\n            user_level_config = str(find_user_pyproject_toml())\n            if config == user_level_config:\n                out(\n                    \"Using configuration from user-level config at \"\n                    f\"'{user_level_config}'.\",\n                    fg=\"blue\",\n                )\n            elif config_source in (\n                ParameterSource.DEFAULT,\n                ParameterSource.DEFAULT_MAP,\n            ):\n                out(\"Using configuration from project root.\", fg=\"blue\")\n            else:\n                out(f\"Using configuration in '{config}'.\", fg=\"blue\")\n            if ctx.default_map:\n                for param, value in ctx.default_map.items():\n                    out(f\"{param}: {value}\")\n\n    error_msg = \"Oh no! \ud83d\udca5 \ud83d\udc94 \ud83d\udca5\"\n    if (\n        required_version\n        and required_version != __version__\n        and required_version != __version__.split(\".\")[0]\n    ):\n        err(\n            f\"{error_msg} The required version `{required_version}` does not match\"\n            f\" the running version `{__version__}`!\"\n        )\n        ctx.exit(1)\n    if ipynb and pyi:\n        err(\"Cannot pass both `pyi` and `ipynb` flags!\")\n        ctx.exit(1)\n\n    write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)\n    if target_version:\n        versions = set(target_version)\n    else:\n        # We'll autodetect later.\n        versions = set()\n    mode = Mode(\n        target_versions=versions,\n        line_length=line_length,\n        is_pyi=pyi,\n        is_ipynb=ipynb,\n        skip_source_first_line=skip_source_first_line,\n        string_normalization=not skip_string_normalization,\n        magic_trailing_comma=not skip_magic_trailing_comma,\n        preview=preview,\n        unstable=unstable,\n        python_cell_magics=set(python_cell_magics),\n        enabled_features=set(enable_unstable_feature),\n    )\n\n    lines: List[Tuple[int, int]] = []\n    if line_ranges:\n        if ipynb:\n            err(\"Cannot use --line-ranges with ipynb files.\")\n            ctx.exit(1)\n\n        try:\n            lines = parse_line_ranges(line_ranges)\n        except ValueError as e:\n            err(str(e))\n            ctx.exit(1)\n\n    if code is not None:\n        # Run in quiet mode by default with -c; the extra output isn't useful.\n        # You can still pass -v to get verbose output.\n        quiet = True\n\n    report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)\n\n    if code is not None:\n        reformat_code(\n            content=code,\n            fast=fast,\n            write_back=write_back,\n            mode=mode,\n            report=report,\n            lines=lines,\n        )\n    else:\n        assert root is not None  # root is only None if code is not None\n        try:\n            sources = get_sources(\n                root=root,\n                src=src,\n                quiet=quiet,\n                verbose=verbose,\n                include=include,\n                exclude=exclude,\n                extend_exclude=extend_exclude,\n                force_exclude=force_exclude,\n                report=report,\n                stdin_filename=stdin_filename,\n            )\n        except GitWildMatchPatternError:\n            ctx.exit(1)\n\n        path_empty(\n            sources,\n            \"No Python files are present to be formatted. Nothing to do \ud83d\ude34\",\n            quiet,\n            verbose,\n            ctx,\n        )\n\n        if len(sources) == 1:\n            reformat_one(\n                src=sources.pop(),\n                fast=fast,\n                write_back=write_back,\n                mode=mode,\n                report=report,\n                lines=lines,\n            )\n        else:\n            from black.concurrency import reformat_many\n\n            if lines:\n                err(\"Cannot use --line-ranges to format multiple files.\")\n                ctx.exit(1)\n            reformat_many(\n                sources=sources,\n                fast=fast,\n                write_back=write_back,\n                mode=mode,\n                report=report,\n                workers=workers,\n            )\n\n    if verbose or not quiet:\n        if code is None and (verbose or report.change_count or report.failure_count):\n            out()\n        out(error_msg if report.return_code else \"All done! \u2728 \ud83c\udf70 \u2728\")\n        if code is None:\n            click.echo(str(report), err=True)\n    ctx.exit(report.return_code)\n\n\ndef get_sources(\n    *,\n    root: Path,\n    src: Tuple[str, ...],\n    quiet: bool,\n    verbose: bool,\n    include: Pattern[str],\n    exclude: Optional[Pattern[str]],\n    extend_exclude: Optional[Pattern[str]],\n    force_exclude: Optional[Pattern[str]],\n    report: \"Report\",\n    stdin_filename: Optional[str],\n) -> Set[Path]:\n    \"\"\"Compute the set of files to be formatted.\"\"\"\n    sources: Set[Path] = set()\n\n    assert root.is_absolute(), f\"INTERNAL ERROR: `root` must be absolute but is {root}\"\n    using_default_exclude = exclude is None\n    exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude\n    gitignore: Optional[Dict[Path, PathSpec]] = None\n    root_gitignore = get_gitignore(root)\n\n    for s in src:\n        if s == \"-\" and stdin_filename:\n            path = Path(stdin_filename)\n            is_stdin = True\n        else:\n            path = Path(s)\n            is_stdin = False\n\n        # Compare the logic here to the logic in `gen_python_files`.\n        if is_stdin or path.is_file():\n            if resolves_outside_root_or_cannot_stat(path, root, report):\n                if verbose:\n                    out(f'Skipping invalid source: \"{path}\"', fg=\"red\")\n                continue\n\n            root_relative_path = best_effort_relative_path(path, root).as_posix()\n            root_relative_path = \"/\" + root_relative_path\n\n            # Hard-exclude any files that matches the `--force-exclude` regex.\n            if path_is_excluded(root_relative_path, force_exclude):\n                report.path_ignored(\n                    path, \"matches the --force-exclude regular expression\"\n                )\n                continue\n\n            if is_stdin:\n                path = Path(f\"{STDIN_PLACEHOLDER}{str(path)}\")\n\n            if path.suffix == \".ipynb\" and not jupyter_dependencies_are_installed(\n                warn=verbose or not quiet\n            ):\n                continue\n\n            if verbose:\n                out(f'Found input source: \"{path}\"', fg=\"blue\")\n            sources.add(path)\n        elif path.is_dir():\n            path = root / (path.resolve().relative_to(root))\n            if verbose:\n                out(f'Found input source directory: \"{path}\"', fg=\"blue\")\n\n            if using_default_exclude:\n                gitignore = {\n                    root: root_gitignore,\n                    path: get_gitignore(path),\n                }\n            sources.update(\n                gen_python_files(\n                    path.iterdir(),\n                    root,\n                    include,\n                    exclude,\n                    extend_exclude,\n                    force_exclude,\n                    report,\n                    gitignore,\n                    verbose=verbose,\n                    quiet=quiet,\n                )\n            )\n        elif s == \"-\":\n            if verbose:\n                out(\"Found input source stdin\", fg=\"blue\")\n            sources.add(path)\n        else:\n            err(f\"invalid path: {s}\")\n\n    return sources\n\n\ndef path_empty(\n    src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context\n) -> None:\n    \"\"\"\n    Exit if there is no `src` provided for formatting\n    \"\"\"\n    if not src:\n        if verbose or not quiet:\n            out(msg)\n        ctx.exit(0)\n\n\ndef reformat_code(\n    content: str,\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: Report,\n    *,\n    lines: Collection[Tuple[int, int]] = (),\n) -> None:\n    \"\"\"\n    Reformat and print out `content` without spawning child processes.\n    Similar to `reformat_one`, but for string content.\n\n    `fast`, `write_back`, and `mode` options are passed to\n    :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.\n    \"\"\"\n    path = Path(\"<string>\")\n    try:\n        changed = Changed.NO\n        if format_stdin_to_stdout(\n            content=content, fast=fast, write_back=write_back, mode=mode, lines=lines\n        ):\n            changed = Changed.YES\n        report.done(path, changed)\n    except Exception as exc:\n        if report.verbose:\n            traceback.print_exc()\n        report.failed(path, str(exc))\n\n\n# diff-shades depends on being to monkeypatch this function to operate. I know it's\n# not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26\n@mypyc_attr(patchable=True)\ndef reformat_one(\n    src: Path,\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: \"Report\",\n    *,\n    lines: Collection[Tuple[int, int]] = (),\n) -> None:\n    \"\"\"Reformat a single file under `src` without spawning child processes.\n\n    `fast`, `write_back`, and `mode` options are passed to\n    :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.\n    \"\"\"\n    try:\n        changed = Changed.NO\n\n        if str(src) == \"-\":\n            is_stdin = True\n        elif str(src).startswith(STDIN_PLACEHOLDER):\n            is_stdin = True\n            # Use the original name again in case we want to print something\n            # to the user\n            src = Path(str(src)[len(STDIN_PLACEHOLDER) :])\n        else:\n            is_stdin = False\n\n        if is_stdin:\n            if src.suffix == \".pyi\":\n                mode = replace(mode, is_pyi=True)\n            elif src.suffix == \".ipynb\":\n                mode = replace(mode, is_ipynb=True)\n            if format_stdin_to_stdout(\n                fast=fast, write_back=write_back, mode=mode, lines=lines\n            ):\n                changed = Changed.YES\n        else:\n            cache = Cache.read(mode)\n            if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n                if not cache.is_changed(src):\n                    changed = Changed.CACHED\n            if changed is not Changed.CACHED and format_file_in_place(\n                src, fast=fast, write_back=write_back, mode=mode, lines=lines\n            ):\n                changed = Changed.YES\n            if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (\n                write_back is WriteBack.CHECK and changed is Changed.NO\n            ):\n                cache.write([src])\n        report.done(src, changed)\n    except Exception as exc:\n        if report.verbose:\n            traceback.print_exc()\n        report.failed(src, str(exc))\n\n\ndef format_file_in_place(\n    src: Path,\n    fast: bool,\n    mode: Mode,\n    write_back: WriteBack = WriteBack.NO,\n    lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy\n    *,\n    lines: Collection[Tuple[int, int]] = (),\n) -> bool:\n    \"\"\"Format file under `src` path. Return True if changed.\n\n    If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted\n    code to the file.\n    `mode` and `fast` options are passed to :func:`format_file_contents`.\n    \"\"\"\n    if src.suffix == \".pyi\":\n        mode = replace(mode, is_pyi=True)\n    elif src.suffix == \".ipynb\":\n        mode = replace(mode, is_ipynb=True)\n\n    then = datetime.fromtimestamp(src.stat().st_mtime, timezone.utc)\n    header = b\"\"\n    with open(src, \"rb\") as buf:\n        if mode.skip_source_first_line:\n            header = buf.readline()\n        src_contents, encoding, newline = decode_bytes(buf.read())\n    try:\n        dst_contents = format_file_contents(\n            src_contents, fast=fast, mode=mode, lines=lines\n        )\n    except NothingChanged:\n        return False\n    except JSONDecodeError:\n        raise ValueError(\n            f\"File '{src}' cannot be parsed as valid Jupyter notebook.\"\n        ) from None\n    src_contents = header.decode(encoding) + src_contents\n    dst_contents = header.decode(encoding) + dst_contents\n\n    if write_back == WriteBack.YES:\n        with open(src, \"w\", encoding=encoding, newline=newline) as f:\n            f.write(dst_contents)\n    elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n        now = datetime.now(timezone.utc)\n        src_name = f\"{src}\\t{then}\"\n        dst_name = f\"{src}\\t{now}\"\n        if mode.is_ipynb:\n            diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name)\n        else:\n            diff_contents = diff(src_contents, dst_contents, src_name, dst_name)\n\n        if write_back == WriteBack.COLOR_DIFF:\n            diff_contents = color_diff(diff_contents)\n\n        with lock or nullcontext():\n            f = io.TextIOWrapper(\n                sys.stdout.buffer,\n                encoding=encoding,\n                newline=newline,\n                write_through=True,\n            )\n            f = wrap_stream_for_windows(f)\n            f.write(diff_contents)\n            f.detach()\n\n    return True\n\n\ndef format_stdin_to_stdout(\n    fast: bool,\n    *,\n    content: Optional[str] = None,\n    write_back: WriteBack = WriteBack.NO,\n    mode: Mode,\n    lines: Collection[Tuple[int, int]] = (),\n) -> bool:\n    \"\"\"Format file on stdin. Return True if changed.\n\n    If content is None, it's read from sys.stdin.\n\n    If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,\n    write a diff to stdout. The `mode` argument is passed to\n    :func:`format_file_contents`.\n    \"\"\"\n    then = datetime.now(timezone.utc)\n\n    if content is None:\n        src, encoding, newline = decode_bytes(sys.stdin.buffer.read())\n    else:\n        src, encoding, newline = content, \"utf-8\", \"\"\n\n    dst = src\n    try:\n        dst = format_file_contents(src, fast=fast, mode=mode, lines=lines)\n        return True\n\n    except NothingChanged:\n        return False\n\n    finally:\n        f = io.TextIOWrapper(\n            sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True\n        )\n        if write_back == WriteBack.YES:\n            # Make sure there's a newline after the content\n            if dst and dst[-1] != \"\\n\":\n                dst += \"\\n\"\n            f.write(dst)\n        elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n            now = datetime.now(timezone.utc)\n            src_name = f\"STDIN\\t{then}\"\n            dst_name = f\"STDOUT\\t{now}\"\n            d = diff(src, dst, src_name, dst_name)\n            if write_back == WriteBack.COLOR_DIFF:\n                d = color_diff(d)\n                f = wrap_stream_for_windows(f)\n            f.write(d)\n        f.detach()\n\n\ndef check_stability_and_equivalence(\n    src_contents: str,\n    dst_contents: str,\n    *,\n    mode: Mode,\n    lines: Collection[Tuple[int, int]] = (),\n) -> None:\n    \"\"\"Perform stability and equivalence checks.\n\n    Raise AssertionError if source and destination contents are not\n    equivalent, or if a second pass of the formatter would format the\n    content differently.\n    \"\"\"\n    assert_equivalent(src_contents, dst_contents)\n    assert_stable(src_contents, dst_contents, mode=mode, lines=lines)\n\n\ndef format_file_contents(\n...\n", "needle_token_start": 2176, "needle_token_end": 2667, "code_context_ntokens": 16222, "output": ["```python\ndef convert_unchanged_lines(src_node: Node, lines: Collection[Tuple[int, int]]) -> None:\n    \"\"\"Converts unchanged lines to STANDALONE_COMMENT.\n\n    The idea is similar to how `# fmt: on/off` is implemented. It also converts the\n    nodes between those markers as a single `STANDALONE_COMMENT` leaf node with\n    the unformatted code as its value. `STANDALONE_COMMENT` is a \"fake\" token\n    that will be formatted as-is with its prefix normalized.\n\n    Here we perform two passes:\n\n    1. Visit the top-level statements, and convert them to a single\n       `STANDALONE_COMMENT` when unchanged. This speeds up formatting when some\n       of the top-level statements aren't changed.\n    2. Convert unchanged \"unwrapped lines\" to `STANDALONE_COMMENT` nodes line by\n       line. \"unwrapped lines\" are divided by the `NEWLINE` token. e.g. a\n       multi-line statement is *one* \"unwrapped line\" that ends with `NEWLINE`,\n       even though this statement itself can span multiple lines, and the\n       tokenizer only sees the last '\\n' as the `NEWLINE` token.\n\n    NOTE: During pass (2), comment prefixes and indentations are ALWAYS\n    normalized even when the lines aren't changed. This is fixable by moving\n    more formatting to pass (1). However, it's hard to get it correct when\n    incorrect indentations are used. So we defer this to future optimizations.\n    \"\"\"\n    lines_set: Set[int] = set()\n    for start, end in lines:\n        lines_set.update(range(start, end + 1))\n    visitor = _TopLevelStatementsVisitor(lines_set)\n    _ = list(visitor.visit(src_node))  # Consume all results.\n    _convert_unchanged_line_by_line(src_node, lines_set)\n\n\ndef _contains_standalone_comment(node: LN) -> bool:\n    if isinstance(node, Leaf):\n        return node.type == STANDALONE_COMMENT\n    else:\n        for child in node.children:\n            if _contains_standalone_comment(child):\n                return True\n        return False\n\n\nclass _TopLevelStatementsVisitor(Visitor[None]):\n    \"\"\"\n    A node visitor that converts unchanged top-level statements to\n    STANDALONE_COMMENT.\n\n    This is used in addition to _convert_unchanged_line_by_line, to\n    speed up formatting when there are unchanged top-level\n    classes/functions/statements.\n    \"\"\"\n\n    def __init__(self, lines_set: Set[int]):\n        self._lines_set = lines_set\n\n    def visit_simple_stmt(self, node: Node) -> Iterator[None]:\n        # This is only called for top-level statements, since `visit_suite`\n        # won't visit its children nodes.\n        yield from []\n        newline_leaf = last_leaf(node)\n        if not newline_leaf:\n            return\n        assert (\n            newline_leaf.type == NEWLINE\n        ), f\"Unexpectedly found leaf.type={newline_leaf.type}\"\n        # We need to find the furthest ancestor with the NEWLINE as the last\n        # leaf, since a `suite` can simply be a `simple_stmt` when it puts\n        # its body on the same line. Example: `if cond: pass`.\n        ancestor = furthest_ancestor_with_last_leaf(newline_leaf)\n        if not _get_line_range(ancestor).intersection(self._lines_set):\n            _convert_node_to_standalone_comment(ancestor)\n\n    def visit_suite(self, node: Node) -> Iterator[None]:\n        yield from []\n        # If there is a STANDALONE_COMMENT node, it means parts of the node tree\n        # have fmt on/off/skip markers. Those STANDALONE_COMMENT nodes can't\n        # be simply converted by calling str(node). So we just don't convert\n        # here.\n        if _contains_standalone_comment(node):\n            return\n        # Find the semantic parent of this suite. For `async_stmt` and\n        # `async_funcdef`, the ASYNC token is defined on a separate level by the\n        # grammar.\n        semantic_parent = node.parent\n        if semantic_parent is not None:\n            if (\n                semantic_parent.prev_sibling is not None\n                and semantic_parent.prev_sibling.type == ASYNC\n            ):\n                semantic_parent = semantic_parent.parent\n        if semantic_parent is not None and not _get_line_range(\n            semantic_parent\n        ).intersection(self._lines_set):\n            _convert_node_to_standalone_comment(semantic_parent)\n\n\ndef _convert_unchanged_line_by_line(node: Node, lines_set: Set[int]) -> None:\n    \"\"\"Converts unchanged to STANDALONE_COMMENT line by line.\"\"\"\n    for leaf in node.leaves():\n        if leaf.type != NEWLINE:\n            # We only consider \"unwrapped lines\", which are divided"]}
{"repo": "psf/black", "name": "make_simple_prefix", "language": "python", "path": "src/black/nodes.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function generates a string used to format text with new lines and optionally a form feed character, typically used to control spacing in formatted output.\n2. **Input**: The function takes three parameters: an integer representing the number of new lines, a boolean indicating whether a form feed character should be included, and an optional string representing the line break character (defaulting to a newline).\n3. **Output**: It returns a string composed of the specified number of newline characters, and if requested, includes a form feed character followed by an additional newline.\n4. **Procedure**: The function first checks if the form feed should be included. If true, it concatenates the specified number of newline characters minus one with a form feed character and another newline. If false, it simply returns a string of newline characters multiplied by the specified integer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/black/strings.py\n\"\"\"\nSimple formatting on strings. Further string formatting code is in trans.py.\n\"\"\"\n\nimport re\nimport sys\nfrom functools import lru_cache\nfrom typing import Final, List, Match, Pattern\n\nfrom black._width_table import WIDTH_TABLE\nfrom blib2to3.pytree import Leaf\n\nSTRING_PREFIX_CHARS: Final = \"furbFURB\"  # All possible string prefix characters.\nSTRING_PREFIX_RE: Final = re.compile(\n    r\"^([\" + STRING_PREFIX_CHARS + r\"]*)(.*)$\", re.DOTALL\n)\nFIRST_NON_WHITESPACE_RE: Final = re.compile(r\"\\s*\\t+\\s*(\\S)\")\nUNICODE_ESCAPE_RE: Final = re.compile(\n    r\"(?P<backslashes>\\\\+)(?P<body>\"\n...\n# Path: src/black/nodes.py\n\"\"\"\nblib2to3 Node/Leaf transformation-related utility functions.\n\"\"\"\n\nimport sys\nfrom typing import (\n    Final,\n    Generic,\n    Iterator,\n    List,\n    Literal,\n    Optional,\n    Set,\n    Tuple,\n    TypeVar,\n    Union,\n)\n\nif sys.version_info >= (3, 10):\n    from typing import TypeGuard\nelse:\n    from typing_extensions import TypeGuard\n\nfrom mypy_extensions import mypyc_attr\n\nfrom black.cache import CACHE_DIR\nfrom black.mode import Mode, Preview\nfrom black.strings import get_string_prefix, has_triple_quotes\nfrom blib2to3 import pygram\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import NL, Leaf, Node, type_repr\n\npygram.initialize(CACHE_DIR)\nsyms: Final = pygram.python_symbols\n\n\n# types\nT = TypeVar(\"T\")\nLN = Union[Leaf, Node]\nLeafID = int\nNodeType = int\n\n\nWHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE}\nSTATEMENT: Final = {\n    syms.if_stmt,\n    syms.while_stmt,\n    syms.for_stmt,\n    syms.try_stmt,\n    syms.except_clause,\n    syms.with_stmt,\n    syms.funcdef,\n    syms.classdef,\n    syms.match_stmt,\n    syms.case_block,\n}\nSTANDALONE_COMMENT: Final = 153\ntoken.tok_name[STANDALONE_COMMENT] = \"STANDALONE_COMMENT\"\nLOGIC_OPERATORS: Final = {\"and\", \"or\"}\nCOMPARATORS: Final = {\n    token.LESS,\n    token.GREATER,\n    token.EQEQUAL,\n    token.NOTEQUAL,\n    token.LESSEQUAL,\n    token.GREATEREQUAL,\n}\nMATH_OPERATORS: Final = {\n    token.VBAR,\n    token.CIRCUMFLEX,\n    token.AMPER,\n    token.LEFTSHIFT,\n    token.RIGHTSHIFT,\n    token.PLUS,\n    token.MINUS,\n    token.STAR,\n    token.SLASH,\n    token.DOUBLESLASH,\n    token.PERCENT,\n    token.AT,\n    token.TILDE,\n    token.DOUBLESTAR,\n}\nSTARS: Final = {token.STAR, token.DOUBLESTAR}\nVARARGS_SPECIALS: Final = STARS | {token.SLASH}\nVARARGS_PARENTS: Final = {\n    syms.arglist,\n    syms.argument,  # double star in arglist\n    syms.trailer,  # single argument to call\n    syms.typedargslist,\n    syms.varargslist,  # lambdas\n}\nUNPACKING_PARENTS: Final = {\n    syms.atom,  # single element of a list or set literal\n    syms.dictsetmaker,\n    syms.listmaker,\n    syms.testlist_gexp,\n    syms.testlist_star_expr,\n    syms.subject_expr,\n    syms.pattern,\n}\nTEST_DESCENDANTS: Final = {\n    syms.test,\n    syms.lambdef,\n    syms.or_test,\n    syms.and_test,\n    syms.not_test,\n    syms.comparison,\n    syms.star_expr,\n    syms.expr,\n    syms.xor_expr,\n    syms.and_expr,\n    syms.shift_expr,\n    syms.arith_expr,\n    syms.trailer,\n    syms.term,\n    syms.power,\n    syms.namedexpr_test,\n}\nTYPED_NAMES: Final = {syms.tname, syms.tname_star}\nASSIGNMENTS: Final = {\n    \"=\",\n    \"+=\",\n    \"-=\",\n    \"*=\",\n    \"@=\",\n    \"/=\",\n    \"%=\",\n    \"&=\",\n    \"|=\",\n    \"^=\",\n    \"<<=\",\n    \">>=\",\n    \"**=\",\n    \"//=\",\n    \":\",\n}\n\nIMPLICIT_TUPLE: Final = {syms.testlist, syms.testlist_star_expr, syms.exprlist}\nBRACKET: Final = {\n    token.LPAR: token.RPAR,\n    token.LSQB: token.RSQB,\n    token.LBRACE: token.RBRACE,\n}\nOPENING_BRACKETS: Final = set(BRACKET.keys())\nCLOSING_BRACKETS: Final = set(BRACKET.values())\nBRACKETS: Final = OPENING_BRACKETS | CLOSING_BRACKETS\nALWAYS_NO_SPACE: Final = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}\n\nRARROW = 55\n\n\n@mypyc_attr(allow_interpreted_subclasses=True)\nclass Visitor(Generic[T]):\n    \"\"\"Basic lib2to3 visitor that yields things of type `T` on `visit()`.\"\"\"\n\n    def visit(self, node: LN) -> Iterator[T]:\n        \"\"\"Main method to visit `node` and its children.\n\n        It tries to find a `visit_*()` method for the given `node.type`, like\n        `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.\n        If no dedicated `visit_*()` method is found, chooses `visit_default()`\n        instead.\n\n        Then yields objects of type `T` from the selected visitor.\n        \"\"\"\n        if node.type < 256:\n            name = token.tok_name[node.type]\n        else:\n            name = str(type_repr(node.type))\n        # We explicitly branch on whether a visitor exists (instead of\n        # using self.visit_default as the default arg to getattr) in order\n        # to save needing to create a bound method object and so mypyc can\n        # generate a native call to visit_default.\n        visitf = getattr(self, f\"visit_{name}\", None)\n        if visitf:\n            yield from visitf(node)\n        else:\n            yield from self.visit_default(node)\n\n    def visit_default(self, node: LN) -> Iterator[T]:\n        \"\"\"Default `visit_*()` implementation. Recurses to children of `node`.\"\"\"\n        if isinstance(node, Node):\n            for child in node.children:\n                yield from self.visit(child)\n\n\ndef whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str:  # noqa: C901\n    \"\"\"Return whitespace prefix if needed for the given `leaf`.\n\n    `complex_subscript` signals whether the given leaf is part of a subscription\n    which has non-trivial arguments, like arithmetic expressions or function calls.\n    \"\"\"\n    NO: Final[str] = \"\"\n    SPACE: Final[str] = \" \"\n    DOUBLESPACE: Final[str] = \"  \"\n    t = leaf.type\n    p = leaf.parent\n    v = leaf.value\n    if t in ALWAYS_NO_SPACE:\n        return NO\n\n    if t == token.COMMENT:\n        return DOUBLESPACE\n\n    assert p is not None, f\"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}\"\n    if t == token.COLON and p.type not in {\n        syms.subscript,\n        syms.subscriptlist,\n        syms.sliceop,\n    }:\n        return NO\n\n    prev = leaf.prev_sibling\n    if not prev:\n        prevp = preceding_leaf(p)\n        if not prevp or prevp.type in OPENING_BRACKETS:\n            return NO\n\n        if t == token.COLON:\n            if prevp.type == token.COLON:\n                return NO\n\n            elif prevp.type != token.COMMA and not complex_subscript:\n                return NO\n\n            return SPACE\n\n        if prevp.type == token.EQUAL:\n            if prevp.parent:\n                if prevp.parent.type in {\n                    syms.arglist,\n                    syms.argument,\n                    syms.parameters,\n                    syms.varargslist,\n                }:\n                    return NO\n\n                elif prevp.parent.type == syms.typedargslist:\n                    # A bit hacky: if the equal sign has whitespace, it means we\n                    # previously found it's a typed argument.  So, we're using\n                    # that, too.\n                    return prevp.prefix\n\n        elif (\n            prevp.type == token.STAR\n            and parent_type(prevp) == syms.star_expr\n            and parent_type(prevp.parent) == syms.subscriptlist\n        ):\n            # No space between typevar tuples.\n            return NO\n\n        elif prevp.type in VARARGS_SPECIALS:\n            if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):\n                return NO\n\n        elif prevp.type == token.COLON:\n            if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:\n                return SPACE if complex_subscript else NO\n\n        elif (\n            prevp.parent\n            and prevp.parent.type == syms.factor\n            and prevp.type in MATH_OPERATORS\n        ):\n            return NO\n\n        elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator:\n            # no space in decorators\n            return NO\n\n    elif prev.type in OPENING_BRACKETS:\n        return NO\n\n    if p.type in {syms.parameters, syms.arglist}:\n        # untyped function signatures or calls\n        if not prev or prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.varargslist:\n        # lambdas\n        if prev and prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.typedargslist:\n        # typed function signatures\n        if not prev:\n            return NO\n\n        if t == token.EQUAL:\n            if prev.type not in TYPED_NAMES:\n                return NO\n\n        elif prev.type == token.EQUAL:\n            # A bit hacky: if the equal sign has whitespace, it means we\n            # previously found it's a typed argument.  So, we're using that, too.\n            return prev.prefix\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type in TYPED_NAMES:\n        # type names\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type != token.COMMA:\n                return NO\n\n    elif p.type == syms.trailer:\n        # attributes and calls\n        if t == token.LPAR or t == token.RPAR:\n            return NO\n\n        if not prev:\n            if t == token.DOT or t == token.LSQB:\n                return NO\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.argument:\n        # single argument\n        if t == token.EQUAL:\n            return NO\n\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type == token.LPAR:\n                return NO\n\n        elif prev.type in {token.EQUAL} | VARARGS_SPECIALS:\n            return NO\n\n    elif p.type == syms.decorator:\n        # decorators\n        return NO\n\n    elif p.type == syms.dotted_name:\n        if prev:\n            return NO\n\n        prevp = preceding_leaf(p)\n        if not prevp or prevp.type == token.AT or prevp.type == token.DOT:\n            return NO\n\n    elif p.type == syms.classdef:\n        if t == token.LPAR:\n            return NO\n\n        if prev and prev.type == token.LPAR:\n            return NO\n\n    elif p.type in {syms.subscript, syms.sliceop}:\n        # indexing\n        if not prev:\n            assert p.parent is not None, \"subscripts are always parented\"\n            if p.parent.type == syms.subscriptlist:\n                return SPACE\n\n            return NO\n\n        elif t == token.COLONEQUAL or prev.type == token.COLONEQUAL:\n            return SPACE\n\n        elif not complex_subscript:\n            return NO\n\n    elif p.type == syms.atom:\n        if prev and t == token.DOT:\n            # dots, but not the first one.\n            return NO\n\n    elif p.type == syms.dictsetmaker:\n        # dict unpacking\n        if prev and prev.type == token.DOUBLESTAR:\n            return NO\n\n    elif p.type in {syms.factor, syms.star_expr}:\n        # unary ops\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type in OPENING_BRACKETS:\n                return NO\n\n            prevp_parent = prevp.parent\n            assert prevp_parent is not None\n            if prevp.type == token.COLON and prevp_parent.type in {\n                syms.subscript,\n                syms.sliceop,\n            }:\n                return NO\n\n            elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:\n                return NO\n\n        elif t in {token.NAME, token.NUMBER, token.STRING}:\n            return NO\n\n    elif p.type == syms.import_from:\n        if t == token.DOT:\n            if prev and prev.type == token.DOT:\n                return NO\n\n        elif t == token.NAME:\n            if v == \"import\":\n                return SPACE\n\n            if prev and prev.type == token.DOT:\n                return NO\n\n    elif p.type == syms.sliceop:\n        return NO\n\n    elif p.type == syms.except_clause:\n        if t == token.STAR:\n            return NO\n\n    return SPACE\n\n\n\ndef make_simple_prefix(nl_count: int, form_feed: bool, empty_line: str = \"\\n\") -> str:\n    \"\"\"Generate a normalized prefix string.\"\"\"\n    if form_feed:\n        return (empty_line * (nl_count - 1)) + \"\\f\" + empty_line\n    return empty_line * nl_count\n\n\ndef preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:\n    \"\"\"Return the first leaf that precedes `node`, if any.\"\"\"\n    while node:\n        res = node.prev_sibling\n        if res:\n            if isinstance(res, Leaf):\n                return res\n\n            try:\n                return list(res.leaves())[-1]\n\n            except IndexError:\n                return None\n\n        node = node.parent\n    return None\n\n\ndef prev_siblings_are(node: Optional[LN], tokens: List[Optional[NodeType]]) -> bool:\n    \"\"\"Return if the `node` and its previous siblings match types against the provided\n    list of tokens; the provided `node`has its type matched against the last element in\n    the list.  `None` can be used as the first element to declare that the start of the\n    list is anchored at the start of its parent's children.\"\"\"\n    if not tokens:\n        return True\n    if tokens[-1] is None:\n        return node is None\n    if not node:\n        return False\n    if node.type != tokens[-1]:\n        return False\n    return prev_siblings_are(node.prev_sibling, tokens[:-1])\n\n\ndef parent_type(node: Optional[LN]) -> Optional[NodeType]:\n    \"\"\"\n    Returns:\n        @node.parent.type, if @node is not None and has a parent.\n            OR\n        None, otherwise.\n    \"\"\"\n    if node is None or node.parent is None:\n        return None\n\n    return node.parent.type\n\n\ndef child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:\n    \"\"\"Return the child of `ancestor` that contains `descendant`.\"\"\"\n    node: Optional[LN] = descendant\n    while node and node.parent != ancestor:\n        node = node.parent\n    return node\n\n\ndef replace_child(old_child: LN, new_child: LN) -> None:\n    \"\"\"\n    Side Effects:\n        * If @old_child.parent is set, replace @old_child with @new_child in\n        @old_child's underlying Node structure.\n            OR\n        * Otherwise, this function does nothing.\n    \"\"\"\n    parent = old_child.parent\n    if not parent:\n        return\n\n    child_idx = old_child.remove()\n    if child_idx is not None:\n        parent.insert_child(child_idx, new_child)\n\n\ndef container_of(leaf: Leaf) -> LN:\n    \"\"\"Return `leaf` or one of its ancestors that is the topmost container of it.\n\n    By \"container\" we mean a node where `leaf` is the very first child.\n    \"\"\"\n    same_prefix = leaf.prefix\n    container: LN = leaf\n    while container:\n        parent = container.parent\n        if parent is None:\n            break\n\n        if parent.children[0].prefix != same_prefix:\n            break\n\n        if parent.type == syms.file_input:\n            break\n\n        if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS:\n            break\n\n        container = parent\n    return container\n\n\ndef first_leaf_of(node: LN) -> Optional[Leaf]:\n    \"\"\"Returns the first leaf of the node tree.\"\"\"\n    if isinstance(node, Leaf):\n        return node\n    if node.children:\n        return first_leaf_of(node.children[0])\n    else:\n        return None\n\n\ndef is_arith_like(node: LN) -> bool:\n    \"\"\"Whether node is an arithmetic or a binary arithmetic expression\"\"\"\n    return node.type in {\n        syms.arith_expr,\n        syms.shift_expr,\n        syms.xor_expr,\n        syms.and_expr,\n    }\n\n\ndef is_docstring(leaf: Leaf, mode: Mode) -> bool:\n    if leaf.type != token.STRING:\n        return False\n\n    prefix = get_string_prefix(leaf.value)\n    if set(prefix).intersection(\"bBfF\"):\n        return False\n\n    if (\n        Preview.unify_docstring_detection in mode\n        and leaf.parent\n        and leaf.parent.type == syms.simple_stmt\n        and not leaf.parent.prev_sibling\n        and leaf.parent.parent\n        and leaf.parent.parent.type == syms.file_input\n    ):\n        return True\n\n    if prev_siblings_are(\n        leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt]\n    ):\n        return True\n\n    # Multiline docstring on the same line as the `def`.\n    if prev_siblings_are(leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt]):\n        # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python\n        # grammar. We're safe to return True without further checks.\n        return True\n\n    return False\n\n\ndef is_empty_tuple(node: LN) -> bool:\n    \"\"\"Return True if `node` holds an empty tuple.\"\"\"\n    return (\n        node.type == syms.atom\n        and len(node.children) == 2\n        and node.children[0].type == token.LPAR\n        and node.children[1].type == token.RPAR\n    )\n\n\ndef is_one_tuple(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a tuple with one element, with or without parens.\"\"\"\n    if node.type == syms.atom:\n        gexp = unwrap_singleton_parenthesis(node)\n        if gexp is None or gexp.type != syms.testlist_gexp:\n            return False\n\n        return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA\n\n    return (\n        node.type in IMPLICIT_TUPLE\n        and len(node.children) == 2\n        and node.children[1].type == token.COMMA\n    )\n\n\ndef is_tuple_containing_walrus(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a tuple that contains a walrus operator.\"\"\"\n    if node.type != syms.atom:\n        return False\n    gexp = unwrap_singleton_parenthesis(node)\n    if gexp is None or gexp.type != syms.testlist_gexp:\n        return False\n\n    return any(child.type == syms.namedexpr_test for child in gexp.children)\n\n\ndef is_one_sequence_between(\n    opening: Leaf,\n    closing: Leaf,\n    leaves: List[Leaf],\n    brackets: Tuple[int, int] = (token.LPAR, token.RPAR),\n) -> bool:\n    \"\"\"Return True if content between `opening` and `closing` is a one-sequence.\"\"\"\n    if (opening.type, closing.type) != brackets:\n        return False\n\n    depth = closing.bracket_depth + 1\n    for _opening_index, leaf in enumerate(leaves):\n        if leaf is opening:\n            break\n\n    else:\n        raise LookupError(\"Opening paren not found in `leaves`\")\n\n    commas = 0\n    _opening_index += 1\n    for leaf in leaves[_opening_index:]:\n        if leaf is closing:\n            break\n\n        bracket_depth = leaf.bracket_depth\n        if bracket_depth == depth and leaf.type == token.COMMA:\n            commas += 1\n            if leaf.parent and leaf.parent.type in {\n                syms.arglist,\n                syms.typedargslist,\n            }:\n                commas += 1\n                break\n\n    return commas < 2\n\n\ndef is_walrus_assignment(node: LN) -> bool:\n    \"\"\"Return True iff `node` is of the shape ( test := test )\"\"\"\n    inner = unwrap_singleton_parenthesis(node)\n    return inner is not None and inner.type == syms.namedexpr_test\n\n\ndef is_simple_decorator_trailer(node: LN, last: bool = False) -> bool:\n    \"\"\"Return True iff `node` is a trailer valid in a simple decorator\"\"\"\n    return node.type == syms.trailer and (\n        (\n            len(node.children) == 2\n            and node.children[0].type == token.DOT\n            and node.children[1].type == token.NAME\n        )\n        # last trailer can be an argument-less parentheses pair\n        or (\n            last\n            and len(node.children) == 2\n            and node.children[0].type == token.LPAR\n            and node.children[1].type == token.RPAR\n        )\n        # last trailer can be arguments\n        or (\n            last\n            and len(node.children) == 3\n            and node.children[0].type == token.LPAR\n            # and node.children[1].type == syms.argument\n            and node.children[2].type == token.RPAR\n        )\n    )\n\n\ndef is_simple_decorator_expression(node: LN) -> bool:\n    \"\"\"Return True iff `node` could be a 'dotted name' decorator\n\n    This function takes the node of the 'namedexpr_test' of the new decorator\n    grammar and test if it would be valid under the old decorator grammar.\n\n    The old grammar was: decorator: @ dotted_name [arguments] NEWLINE\n    The new grammar is : decorator: @ namedexpr_test NEWLINE\n    \"\"\"\n    if node.type == token.NAME:\n        return True\n    if node.type == syms.power:\n        if node.children:\n            return (\n                node.children[0].type == token.NAME\n                and all(map(is_simple_decorator_trailer, node.children[1:-1]))\n                and (\n                    len(node.children) < 2\n                    or is_simple_decorator_trailer(node.children[-1], last=True)\n                )\n            )\n    return False\n\n\ndef is_yield(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a `yield` or `yield from` expression.\"\"\"\n    if node.type == syms.yield_expr:\n        return True\n\n    if is_name_token(node) and node.value == \"yield\":\n        return True\n\n    if node.type != syms.atom:\n        return False\n\n    if len(node.children) != 3:\n        return False\n\n    lpar, expr, rpar = node.children\n    if lpar.type == token.LPAR and rpar.type == token.RPAR:\n        return is_yield(expr)\n\n    return False\n\n\ndef is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:\n    \"\"\"Return True if `leaf` is a star or double star in a vararg or kwarg.\n\n    If `within` includes VARARGS_PARENTS, this applies to function signatures.\n    If `within` includes UNPACKING_PARENTS, it applies to right hand-side\n    extended iterable unpacking (PEP 3132) and additional unpacking\n    generalizations (PEP 448).\n    \"\"\"\n    if leaf.type not in VARARGS_SPECIALS or not leaf.parent:\n        return False\n\n    p = leaf.parent\n    if p.type == syms.star_expr:\n        # Star expressions are also used as assignment targets in extended\n        # iterable unpacking (PEP 3132).  See what its parent is instead.\n        if not p.parent:\n            return False\n\n        p = p.parent\n\n    return p.type in within\n\n\ndef is_multiline_string(leaf: Leaf) -> bool:\n    \"\"\"Return True if `leaf` is a multiline string that actually spans many lines.\"\"\"\n    return has_triple_quotes(leaf.value) and \"\\n\" in leaf.value\n\n\ndef is_parent_function_or_class(node: Node) -> bool:\n    assert node.type in {syms.suite, syms.simple_stmt}\n    assert node.parent is not None\n    # Note this works for suites / simple_stmts in async def as well\n    return node.parent.type in {syms.funcdef, syms.classdef}\n\n\ndef is_function_or_class(node: Node) -> bool:\n    return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef}\n\n\ndef is_stub_suite(node: Node) -> bool:\n    \"\"\"Return True if `node` is a suite with a stub body.\"\"\"\n    if node.parent is not None and not is_parent_function_or_class(node):\n        return False\n\n    # If there is a comment, we want to keep it.\n    if node.prefix.strip():\n        return False\n\n    if (\n        len(node.children) != 4\n        or node.children[0].type != token.NEWLINE\n        or node.children[1].type != token.INDENT\n        or node.children[3].type != token.DEDENT\n    ):\n        return False\n\n    if node.children[3].prefix.strip():\n        return False\n\n    return is_stub_body(node.children[2])\n\n\ndef is_stub_body(node: LN) -> bool:\n    \"\"\"Return True if `node` is a simple statement containing an ellipsis.\"\"\"\n    if not isinstance(node, Node) or node.type != syms.simple_stmt:\n        return False\n\n    if len(node.children) != 2:\n        return False\n\n    child = node.children[0]\n    return (\n        not child.prefix.strip()\n        and child.type == syms.atom\n        and len(child.children) == 3\n        and all(leaf == Leaf(token.DOT, \".\") for leaf in child.children)\n    )\n\n\ndef is_atom_with_invisible_parens(node: LN) -> bool:\n    \"\"\"Given a `LN`, determines whether it's an atom `node` with invisible\n    parens. Useful in dedupe-ing and normalizing parens.\n    \"\"\"\n    if isinstance(node, Leaf) or node.type != syms.atom:\n        return False\n\n    first, last = node.children[0], node.children[-1]\n    return (\n        isinstance(first, Leaf)\n        and first.type == token.LPAR\n        and first.value == \"\"\n        and isinstance(last, Leaf)\n        and last.type == token.RPAR\n        and last.value == \"\"\n    )\n\n\ndef is_empty_par(leaf: Leaf) -> bool:\n    return is_empty_lpar(leaf) or is_empty_rpar(leaf)\n\n\ndef is_empty_lpar(leaf: Leaf) -> bool:\n    return leaf.type == token.LPAR and leaf.value == \"\"\n\n\ndef is_empty_rpar(leaf: Leaf) -> bool:\n    return leaf.type == token.RPAR and leaf.value == \"\"\n\n\ndef is_import(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf starts an import statement.\"\"\"\n    p = leaf.parent\n    t = leaf.type\n    v = leaf.value\n    return bool(\n        t == token.NAME\n        and (\n            (v == \"import\" and p and p.type == syms.import_name)\n            or (v == \"from\" and p and p.type == syms.import_from)\n        )\n    )\n\n\ndef is_with_or_async_with_stmt(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf starts a with or async with statement.\"\"\"\n    return bool(\n        leaf.type == token.NAME\n        and leaf.value == \"with\"\n        and leaf.parent\n        and leaf.parent.type == syms.with_stmt\n    ) or bool(\n        leaf.type == token.ASYNC\n        and leaf.next_sibling\n        and leaf.next_sibling.type == syms.with_stmt\n    )\n\n\ndef is_async_stmt_or_funcdef(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf starts an async def/for/with statement.\n\n    Note that `async def` can be either an `async_stmt` or `async_funcdef`,\n    the latter is used when it has decorators.\n    \"\"\"\n    return bool(\n        leaf.type == token.ASYNC\n        and leaf.parent\n        and leaf.parent.type in {syms.async_stmt, syms.async_funcdef}\n    )\n\n\ndef is_type_comment(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf is a type comment. This function should only\n    be used for general type comments (excluding ignore annotations, which should\n    use `is_type_ignore_comment`). Note that general type comments are no longer\n    used in modern version of Python, this function may be deprecated in the future.\"\"\"\n    t = leaf.type\n    v = leaf.value\n    return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith(\"# type:\")\n\n\ndef is_type_ignore_comment(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf is a type comment with ignore annotation.\"\"\"\n    t = leaf.type\n    v = leaf.value\n    return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v)\n\n\ndef is_type_ignore_comment_string(value: str) -> bool:\n    \"\"\"Return True if the given string match with type comment with\n    ignore annotation.\"\"\"\n    return value.startswith(\"# type: ignore\")\n\n\ndef wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None:\n    \"\"\"Wrap `child` in parentheses.\n\n    This replaces `child` with an atom holding the parentheses and the old\n    child.  That requires moving the prefix.\n\n    If `visible` is False, the leaves will be valueless (and thus invisible).\n    \"\"\"\n    lpar = Leaf(token.LPAR, \"(\" if visible else \"\")\n    rpar = Leaf(token.RPAR, \")\" if visible else \"\")\n    prefix = child.prefix\n    child.prefix = \"\"\n    index = child.remove() or 0\n    new_child = Node(syms.atom, [lpar, child, rpar])\n    new_child.prefix = prefix\n    parent.insert_child(index, new_child)\n\n\ndef unwrap_singleton_parenthesis(node: LN) -> Optional[LN]:\n    \"\"\"Returns `wrapped` if `node` is of the shape ( wrapped ).\n\n    Parenthesis can be optional. Returns None otherwise\"\"\"\n    if len(node.children) != 3:\n        return None\n\n    lpar, wrapped, rpar = node.children\n    if not (lpar.type == token.LPAR and rpar.type == token.RPAR):\n        return None\n\n    return wrapped\n\n\ndef ensure_visible(leaf: Leaf) -> None:\n    \"\"\"Make sure parentheses are visible.\n\n    They could be invisible as part of some statements (see\n    :func:`normalize_invisible_parens` and :func:`visit_import_from`).\n    \"\"\"\n    if leaf.type == token.LPAR:\n        leaf.value = \"(\"\n    elif leaf.type == token.RPAR:\n        leaf.value = \")\"\n\n\ndef is_name_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.NAME\n\n\ndef is_lpar_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.LPAR\n\n\ndef is_rpar_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.RPAR\n\n\ndef is_string_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.STRING\n\n\ndef is_number_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.NUMBER\n\n\ndef get_annotation_type(leaf: Leaf) -> Literal[\"return\", \"param\", None]:\n    \"\"\"Returns the type of annotation this leaf is part of, if any.\"\"\"\n    ancestor = leaf.parent\n    while ancestor is not None:\n        if ancestor.prev_sibling and ancestor.prev_sibling.type == token.RARROW:\n            return \"return\"\n        if ancestor.parent and ancestor.parent.type == syms.tname:\n            return \"param\"\n        ancestor = ancestor.parent\n    return None\n\n\ndef is_part_of_annotation(leaf: Leaf) -> bool:\n    \"\"\"Returns whether this leaf is part of a type annotation.\"\"\"\n    return get_annotation_type(leaf) is not None\n\n\ndef first_leaf(node: LN) -> Optional[Leaf]:\n    \"\"\"Returns the first leaf of the ancestor node.\"\"\"\n    if isinstance(node, Leaf):\n        return node\n    elif not node.children:\n        return None\n    else:\n        return first_leaf(node.children[0])\n\n\ndef last_leaf(node: LN) -> Optional[Leaf]:\n    \"\"\"Returns the last leaf of the ancestor node.\"\"\"\n    if isinstance(node, Leaf):\n        return node\n    elif not node.children:\n        return None\n    else:\n        return last_leaf(node.children[-1])\n\n\ndef furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN:\n    \"\"\"Returns the furthest ancestor that has this leaf node as the last leaf.\"\"\"\n    node: LN = leaf\n    while node.parent and node.parent.children and node is node.parent.children[-1]:\n        node = node.parent\n    return node\n\n# Path: src/black/comments.py\nimport re\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nfrom typing import Collection, Final, Iterator, List, Optional, Tuple, Union\n\nfrom black.mode import Mode, Preview\nfrom black.nodes import (\n    CLOSING_BRACKETS,\n    STANDALONE_COMMENT,\n    WHITESPACE,\n    container_of,\n    first_leaf_of,\n    make_simple_prefix,\n    preceding_leaf,\n    syms,\n)\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\n# types\nLN = Union[Leaf, Node]\n\nFMT_OFF: Final = {\"# fmt: off\", \"# fmt:off\", \"# yapf: disable\"}\nFMT_SKIP: Final = {\"# fmt: skip\", \"# fmt:skip\"}\nFMT_ON: Final = {\"# fmt: on\", \"# fmt:on\", \"# yapf: enable\"}\n\nCOMMENT_EXCEPTIONS = \" !:#'\"\n_COMMENT_PREFIX = \"# \"\n_COMMENT_LIST_SEPARATOR = \";\"\n\n\n@dataclass\nclass ProtoComment:\n    \"\"\"Describes a piece of syntax that is a comment.\n\n    It's not a :class:`blib2to3.pytree.Leaf` so that:\n\n    * it can be cached (`Leaf` objects should not be reused more than once as\n      they store their lineno, column, prefix, and parent information);\n    * `newlines` and `consumed` fields are kept separate from the `value`. This\n      simplifies handling of special marker comments like ``# fmt: off/on``.\n    \"\"\"\n\n    type: int  # token.COMMENT or STANDALONE_COMMENT\n    value: str  # content of the comment\n    newlines: int  # how many newlines before the comment\n    consumed: int  # how many characters of the original leaf's prefix did we consume\n    form_feed: bool  # is there a form feed before the comment\n    leading_whitespace: str  # leading whitespace before the comment, if any\n\n\ndef generate_comments(leaf: LN) -> Iterator[Leaf]:\n    \"\"\"Clean the prefix of the `leaf` and generate comments from it, if any.\n\n    Comments in lib2to3 are shoved into the whitespace prefix.  This happens\n    in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation\n    move because it does away with modifying the grammar to include all the\n    possible places in which comments can be placed.\n\n    The sad consequence for us though is that comments don't \"belong\" anywhere.\n    This is why this function generates simple parentless Leaf objects for\n    comments.  We simply don't know what the correct parent should be.\n\n    No matter though, we can live without this.  We really only need to\n    differentiate between inline and standalone comments.  The latter don't\n    share the line with any code.\n\n    Inline comments are emitted as regular token.COMMENT leaves.  Standalone\n    are emitted with a fake STANDALONE_COMMENT token identifier.\n    \"\"\"\n    total_consumed = 0\n    for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER):\n        total_consumed = pc.consumed\n        prefix = make_simple_prefix(pc.newlines, pc.form_feed)\n        yield Leaf(pc.type, pc.value, prefix=prefix)\n    normalize_trailing_prefix(leaf, total_consumed)\n\n\n@lru_cache(maxsize=4096)\ndef list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:\n    \"\"\"Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.\"\"\"\n    result: List[ProtoComment] = []\n    if not prefix or \"#\" not in prefix:\n        return result\n\n    consumed = 0\n    nlines = 0\n    ignored_lines = 0\n    form_feed = False\n    for index, full_line in enumerate(re.split(\"\\r?\\n\", prefix)):\n        consumed += len(full_line) + 1  # adding the length of the split '\\n'\n        match = re.match(r\"^(\\s*)(\\S.*|)$\", full_line)\n        assert match\n        whitespace, line = match.groups()\n        if not line:\n            nlines += 1\n            if \"\\f\" in full_line:\n                form_feed = True\n        if not line.startswith(\"#\"):\n            # Escaped newlines outside of a comment are not really newlines at\n            # all. We treat a single-line comment following an escaped newline\n            # as a simple trailing comment.\n            if line.endswith(\"\\\\\"):\n                ignored_lines += 1\n            continue\n\n        if index == ignored_lines and not is_endmarker:\n            comment_type = token.COMMENT  # simple trailing comment\n        else:\n            comment_type = STANDALONE_COMMENT\n        comment = make_comment(line)\n        result.append(\n            ProtoComment(\n                type=comment_type,\n                value=comment,\n                newlines=nlines,\n                consumed=consumed,\n                form_feed=form_feed,\n                leading_whitespace=whitespace,\n            )\n        )\n        form_feed = False\n        nlines = 0\n    return result\n\n\ndef normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None:\n    \"\"\"Normalize the prefix that's left over after generating comments.\n\n    Note: don't use backslashes for formatting or you'll lose your voting rights.\n    \"\"\"\n    remainder = leaf.prefix[total_consumed:]\n    if \"\\\\\" not in remainder:\n        nl_count = remainder.count(\"\\n\")\n        form_feed = \"\\f\" in remainder and remainder.endswith(\"\\n\")\n        leaf.prefix = make_simple_prefix(nl_count, form_feed)\n        return\n\n    leaf.prefix = \"\"\n\n\ndef make_comment(content: str) -> str:\n    \"\"\"Return a consistently formatted comment from the given `content` string.\n\n    All comments (except for \"##\", \"#!\", \"#:\", '#'\") should have a single\n    space between the hash sign and the content.\n\n    If `content` didn't start with a hash sign, one is provided.\n    \"\"\"\n    content = content.rstrip()\n    if not content:\n        return \"#\"\n\n    if content[0] == \"#\":\n        content = content[1:]\n    NON_BREAKING_SPACE = \"\u00a0\"\n    if (\n        content\n        and content[0] == NON_BREAKING_SPACE\n        and not content.lstrip().startswith(\"type:\")\n    ):\n        content = \" \" + content[1:]  # Replace NBSP by a simple space\n    if content and content[0] not in COMMENT_EXCEPTIONS:\n        content = \" \" + content\n    return \"#\" + content\n\n\ndef normalize_fmt_off(\n    node: Node, mode: Mode, lines: Collection[Tuple[int, int]]\n) -> None:\n    \"\"\"Convert content between `# fmt: off`/`# fmt: on` into standalone comments.\"\"\"\n    try_again = True\n    while try_again:\n        try_again = convert_one_fmt_off_pair(node, mode, lines)\n\n\ndef convert_one_fmt_off_pair(\n    node: Node, mode: Mode, lines: Collection[Tuple[int, int]]\n) -> bool:\n    \"\"\"Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.\n\n    Returns True if a pair was converted.\n    \"\"\"\n    for leaf in node.leaves():\n        previous_consumed = 0\n        for comment in list_comments(leaf.prefix, is_endmarker=False):\n            should_pass_fmt = comment.value in FMT_OFF or _contains_fmt_skip_comment(\n                comment.value, mode\n            )\n            if not should_pass_fmt:\n                previous_consumed = comment.consumed\n                continue\n            # We only want standalone comments. If there's no previous leaf or\n            # the previous leaf is indentation, it's a standalone comment in\n            # disguise.\n            if should_pass_fmt and comment.type != STANDALONE_COMMENT:\n                prev = preceding_leaf(leaf)\n                if prev:\n                    if comment.value in FMT_OFF and prev.type not in WHITESPACE:\n                        continue\n                    if (\n                        _contains_fmt_skip_comment(comment.value, mode)\n                        and prev.type in WHITESPACE\n                    ):\n                        continue\n\n            ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode))\n            if not ignored_nodes:\n                continue\n\n            first = ignored_nodes[0]  # Can be a container node with the `leaf`.\n            parent = first.parent\n            prefix = first.prefix\n            if comment.value in FMT_OFF:\n                first.prefix = prefix[comment.consumed :]\n            if _contains_fmt_skip_comment(comment.value, mode):\n                first.prefix = \"\"\n                standalone_comment_prefix = prefix\n            else:\n                standalone_comment_prefix = (\n                    prefix[:previous_consumed] + \"\\n\" * comment.newlines\n                )\n            hidden_value = \"\".join(str(n) for n in ignored_nodes)\n            comment_lineno = leaf.lineno - comment.newlines\n            if comment.value in FMT_OFF:\n                fmt_off_prefix = \"\"\n                if len(lines) > 0 and not any(\n                    line[0] <= comment_lineno <= line[1] for line in lines\n                ):\n                    # keeping indentation of comment by preserving original whitespaces.\n                    fmt_off_prefix = prefix.split(comment.value)[0]\n                    if \"\\n\" in fmt_off_prefix:\n                        fmt_off_prefix = fmt_off_prefix.split(\"\\n\")[-1]\n                standalone_comment_prefix += fmt_off_prefix\n                hidden_value = comment.value + \"\\n\" + hidden_value\n            if _contains_fmt_skip_comment(comment.value, mode):\n                hidden_value += (\n                    comment.leading_whitespace\n                    if Preview.no_normalize_fmt_skip_whitespace in mode\n                    else \"  \"\n                ) + comment.value\n            if hidden_value.endswith(\"\\n\"):\n                # That happens when one of the `ignored_nodes` ended with a NEWLINE\n                # leaf (possibly followed by a DEDENT).\n                hidden_value = hidden_value[:-1]\n            first_idx: Optional[int] = None\n            for ignored in ignored_nodes:\n                index = ignored.remove()\n                if first_idx is None:\n                    first_idx = index\n            assert parent is not None, \"INTERNAL ERROR: fmt: on/off handling (1)\"\n            assert first_idx is not None, \"INTERNAL ERROR: fmt: on/off handling (2)\"\n            parent.insert_child(\n                first_idx,\n                Leaf(\n                    STANDALONE_COMMENT,\n                    hidden_value,\n                    prefix=standalone_comment_prefix,\n                    fmt_pass_converted_first_leaf=first_leaf_of(first),\n                ),\n            )\n            return True\n\n    return False\n\n\ndef generate_ignored_nodes(\n    leaf: Leaf, comment: ProtoComment, mode: Mode\n) -> Iterator[LN]:\n    \"\"\"Starting from the container of `leaf`, generate all leaves until `# fmt: on`.\n\n    If comment is skip, returns leaf only.\n    Stops at the end of the block.\n    \"\"\"\n    if _contains_fmt_skip_comment(comment.value, mode):\n        yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment)\n        return\n    container: Optional[LN] = container_of(leaf)\n    while container is not None and container.type != token.ENDMARKER:\n        if is_fmt_on(container):\n            return\n\n        # fix for fmt: on in children\n        if children_contains_fmt_on(container):\n            for index, child in enumerate(container.children):\n                if isinstance(child, Leaf) and is_fmt_on(child):\n                    if child.type in CLOSING_BRACKETS:\n                        # This means `# fmt: on` is placed at a different bracket level\n                        # than `# fmt: off`. This is an invalid use, but as a courtesy,\n                        # we include this closing bracket in the ignored nodes.\n                        # The alternative is to fail the formatting.\n                        yield child\n                    return\n                if (\n                    child.type == token.INDENT\n                    and index < len(container.children) - 1\n                    and children_contains_fmt_on(container.children[index + 1])\n                ):\n                    # This means `# fmt: on` is placed right after an indentation\n                    # level, and we shouldn't swallow the previous INDENT token.\n                    return\n                if children_contains_fmt_on(child):\n                    return\n                yield child\n        else:\n            if container.type == token.DEDENT and container.next_sibling is None:\n                # This can happen when there is no matching `# fmt: on` comment at the\n                # same level as `# fmt: on`. We need to keep this DEDENT.\n                return\n            yield container\n            container = container.next_sibling\n\n\ndef _generate_ignored_nodes_from_fmt_skip(\n    leaf: Leaf, comment: ProtoComment\n) -> Iterator[LN]:\n    \"\"\"Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.\"\"\"\n    prev_sibling = leaf.prev_sibling\n    parent = leaf.parent\n    # Need to properly format the leaf prefix to compare it to comment.value,\n    # which is also formatted\n    comments = list_comments(leaf.prefix, is_endmarker=False)\n    if not comments or comment.value != comments[0].value:\n        return\n    if prev_sibling is not None:\n        leaf.prefix = \"\"\n        siblings = [prev_sibling]\n        while \"\\n\" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None:\n            prev_sibling = prev_sibling.prev_sibling\n            siblings.insert(0, prev_sibling)\n        yield from siblings\n    elif (\n        parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE\n    ):\n        # The `# fmt: skip` is on the colon line of the if/while/def/class/...\n        # statements. The ignored nodes should be previous siblings of the\n        # parent suite node.\n        leaf.prefix = \"\"\n        ignored_nodes: List[LN] = []\n        parent_sibling = parent.prev_sibling\n        while parent_sibling is not None and parent_sibling.type != syms.suite:\n            ignored_nodes.insert(0, parent_sibling)\n            parent_sibling = parent_sibling.prev_sibling\n        # Special case for `async_stmt` where the ASYNC token is on the\n        # grandparent node.\n        grandparent = parent.parent\n        if (\n            grandparent is not None\n            and grandparent.prev_sibling is not None\n            and grandparent.prev_sibling.type == token.ASYNC\n        ):\n            ignored_nodes.insert(0, grandparent.prev_sibling)\n        yield from iter(ignored_nodes)\n\n\ndef is_fmt_on(container: LN) -> bool:\n    \"\"\"Determine whether formatting is switched on within a container.\n    Determined by whether the last `# fmt:` comment is `on` or `off`.\n    \"\"\"\n    fmt_on = False\n    for comment in list_comments(container.prefix, is_endmarker=False):\n        if comment.value in FMT_ON:\n            fmt_on = True\n        elif comment.value in FMT_OFF:\n            fmt_on = False\n    return fmt_on\n\n\ndef children_contains_fmt_on(container: LN) -> bool:\n    \"\"\"Determine if children have formatting switched on.\"\"\"\n    for child in container.children:\n        leaf = first_leaf_of(child)\n        if leaf is not None and is_fmt_on(leaf):\n            return True\n\n    return False\n\n\ndef contains_pragma_comment(comment_list: List[Leaf]) -> bool:\n    \"\"\"\n    Returns:\n        True iff one of the comments in @comment_list is a pragma used by one\n        of the more common static analysis tools for python (e.g. mypy, flake8,\n        pylint).\n    \"\"\"\n    for comment in comment_list:\n        if comment.value.startswith((\"# type:\", \"# noqa\", \"# pylint:\")):\n            return True\n\n    return False\n\n\ndef _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool:\n    \"\"\"\n    Checks if the given comment contains FMT_SKIP alone or paired with other comments.\n    Matching styles:\n      # fmt:skip                           <-- single comment\n      # noqa:XXX # fmt:skip # a nice line  <-- multiple comments (Preview)\n      # pylint:XXX; fmt:skip               <-- list of comments (; separated, Preview)\n    \"\"\"\n    semantic_comment_blocks = [\n        comment_line,\n        *[\n            _COMMENT_PREFIX + comment.strip()\n            for comment in comment_line.split(_COMMENT_PREFIX)[1:]\n        ],\n        *[\n            _COMMENT_PREFIX + comment.strip()\n            for comment in comment_line.strip(_COMMENT_PREFIX).split(\n                _COMMENT_LIST_SEPARATOR\n            )\n        ],\n    ]\n\n    return any(comment in FMT_SKIP for comment in semantic_comment_blocks)\n\n# Path: src/black/report.py\n\"\"\"\nSummarize Black runs to users.\n\"\"\"\n\nfrom dataclasses import dataclass\nfrom enum import Enum\nfrom pathlib import Path\n\nfrom click import style\n\nfrom black.output import err, out\n\n\nclass Changed(Enum):\n    NO = 0\n    CACHED = 1\n    YES = 2\n\n\nclass NothingChanged(UserWarning):\n    \"\"\"Raised when reformatted code is the same as source.\"\"\"\n\n\n@dataclass\nclass Report:\n    \"\"\"Provides a reformatting counter. Can be rendered with `str(report)`.\"\"\"\n\n    check: bool = False\n    diff: bool = False\n    quiet: bool = False\n    verbose: bool = False\n    change_count: int = 0\n    same_count: int = 0\n    failure_count: int = 0\n\n    def done(self, src: Path, changed: Changed) -> None:\n        \"\"\"Increment the counter for successful reformatting. Write out a message.\"\"\"\n        if changed is Changed.YES:\n            reformatted = \"would reformat\" if self.check or self.diff else \"reformatted\"\n            if self.verbose or not self.quiet:\n                out(f\"{reformatted} {src}\")\n            self.change_count += 1\n        else:\n            if self.verbose:\n                if changed is Changed.NO:\n                    msg = f\"{src} already well formatted, good job.\"\n                else:\n                    msg = f\"{src} wasn't modified on disk since last run.\"\n                out(msg, bold=False)\n            self.same_count += 1\n\n    def failed(self, src: Path, message: str) -> None:\n        \"\"\"Increment the counter for failed reformatting. Write out a message.\"\"\"\n        err(f\"error: cannot format {src}: {message}\")\n        self.failure_count += 1\n\n    def path_ignored(self, path: Path, message: str) -> None:\n        if self.verbose:\n            out(f\"{path} ignored: {message}\", bold=False)\n\n    @property\n    def return_code(self) -> int:\n        \"\"\"Return the exit code that the app should use.\n\n        This considers the current state of changed files and failures:\n        - if there were any failures, return 123;\n        - if any files were changed and --check is being used, return 1;\n        - otherwise return 0.\n        \"\"\"\n        # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with\n        # 126 we have special return codes reserved by the shell.\n        if self.failure_count:\n            return 123\n\n        elif self.change_count and self.check:\n            return 1\n\n        return 0\n\n    def __str__(self) -> str:\n        \"\"\"Render a color report of the current state.\n\n        Use `click.unstyle` to remove colors.\n        \"\"\"\n        if self.check or self.diff:\n            reformatted = \"would be reformatted\"\n            unchanged = \"would be left unchanged\"\n            failed = \"would fail to reformat\"\n        else:\n            reformatted = \"reformatted\"\n            unchanged = \"left unchanged\"\n            failed = \"failed to reformat\"\n        report = []\n        if self.change_count:\n            s = \"s\" if self.change_count > 1 else \"\"\n            report.append(\n                style(f\"{self.change_count} file{s} \", bold=True, fg=\"blue\")\n                + style(f\"{reformatted}\", bold=True)\n            )\n\n        if self.same_count:\n            s = \"s\" if self.same_count > 1 else \"\"\n            report.append(style(f\"{self.same_count} file{s} \", fg=\"blue\") + unchanged)\n        if self.failure_count:\n            s = \"s\" if self.failure_count > 1 else \"\"\n            report.append(style(f\"{self.failure_count} file{s} {failed}\", fg=\"red\"))\n        return \", \".join(report) + \".\"\n\n# Path: src/black/concurrency.py\n\"\"\"\nFormatting many files at once via multiprocessing. Contains entrypoint and utilities.\n\nNOTE: this module is only imported if we need to format several files at once.\n\"\"\"\n\nimport asyncio\nimport logging\nimport os\nimport signal\nimport sys\nimport traceback\nfrom concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor\nfrom multiprocessing import Manager\nfrom pathlib import Path\nfrom typing import Any, Iterable, Optional, Set\n\nfrom mypy_extensions import mypyc_attr\n\nfrom black import WriteBack, format_file_in_place\nfrom black.cache import Cache\nfrom black.mode import Mode\nfrom black.output import err\nfrom black.report import Changed, Report\n\n\ndef maybe_install_uvloop() -> None:\n    \"\"\"If our environment has uvloop installed we use it.\n\n    This is called only from command-line entry points to avoid\n    interfering with the parent process if Black is used as a library.\n    \"\"\"\n    try:\n        import uvloop\n\n        uvloop.install()\n    except ImportError:\n        pass\n\n\ndef cancel(tasks: Iterable[\"asyncio.Future[Any]\"]) -> None:\n    \"\"\"asyncio signal handler that cancels all `tasks` and reports to stderr.\"\"\"\n    err(\"Aborted!\")\n    for task in tasks:\n        task.cancel()\n\n\ndef shutdown(loop: asyncio.AbstractEventLoop) -> None:\n    \"\"\"Cancel all pending tasks on `loop`, wait for them, and close the loop.\"\"\"\n    try:\n        # This part is borrowed from asyncio/runners.py in Python 3.7b2.\n        to_cancel = [task for task in asyncio.all_tasks(loop) if not task.done()]\n        if not to_cancel:\n            return\n\n        for task in to_cancel:\n            task.cancel()\n        loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))\n    finally:\n        # `concurrent.futures.Future` objects cannot be cancelled once they\n        # are already running. There might be some when the `shutdown()` happened.\n        # Silence their logger's spew about the event loop being closed.\n        cf_logger = logging.getLogger(\"concurrent.futures\")\n        cf_logger.setLevel(logging.CRITICAL)\n        loop.close()\n\n\n# diff-shades depends on being to monkeypatch this function to operate. I know it's\n# not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26\n@mypyc_attr(patchable=True)\ndef reformat_many(\n    sources: Set[Path],\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: Report,\n    workers: Optional[int],\n) -> None:\n    \"\"\"Reformat multiple files using a ProcessPoolExecutor.\"\"\"\n    maybe_install_uvloop()\n\n    executor: Executor\n    if workers is None:\n        workers = int(os.environ.get(\"BLACK_NUM_WORKERS\", 0))\n        workers = workers or os.cpu_count() or 1\n    if sys.platform == \"win32\":\n        # Work around https://bugs.python.org/issue26903\n        workers = min(workers, 60)\n    try:\n        executor = ProcessPoolExecutor(max_workers=workers)\n    except (ImportError, NotImplementedError, OSError):\n        # we arrive here if the underlying system does not support multi-processing\n        # like in AWS Lambda or Termux, in which case we gracefully fallback to\n        # a ThreadPoolExecutor with just a single worker (more workers would not do us\n        # any good due to the Global Interpreter Lock)\n...\n", "needle_token_start": 3951, "needle_token_end": 4036, "code_context_ntokens": 16048, "output": ["```python\ndef make_simple_prefix(nl_count: int, form_feed: bool, empty_line: str = \"\\n\") -> str:\n    \"\"\"Generate a normalized prefix string.\"\"\"\n    if form_feed:\n        return (empty_line * (nl_count - 1)) + \"\\f\" + empty_line\n    return empty_line * nl_count\n```"]}
{"repo": "psf/black", "name": "run_transformer", "language": "python", "path": "src/black/linegen.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to apply a transformation to a line of code based on specified rules and conditions, and then evaluate the transformed line to ensure it meets certain criteria. If the initial transformation does not meet the criteria, a secondary transformation is attempted with additional constraints.\n2. **Input**: The function accepts a line of code, a transformation function, a mode specifying formatting rules, a collection of features that may influence the transformation, and optionally a string representation of the line.\n3. **Output**: It returns a list of transformed lines of code. If the transformation is unsuccessful or unnecessary, the original line or its direct transformation is returned.\n4. **Procedure**: \n   - Convert the input line to a string if not provided.\n   - Apply the transformation function to the line.\n   - Check if the transformed line is unchanged; if so, raise an exception.\n   - Validate the transformed line against a set of criteria including line length and structural integrity.\n   - If the initial transformation does not meet all criteria, clone the original line and attempt a secondary transformation with an additional feature.\n   - Return the result of the transformation that best meets the criteria.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        for comment_after in line.comments_after(leaf):\n            yield from append_to_line(comment_after)\n\n    last_non_comment_leaf = _get_last_non_comment_leaf(line)\n    for leaf_idx, leaf in enumerate(line.leaves):\n        yield from append_to_line(leaf)\n\n        previous_priority = leaf_idx > 0 and bt.delimiters.get(\n            id(line.leaves[leaf_idx - 1])\n        )\n        if (\n            previous_priority != delimiter_priority\n            or delimiter_priority in MIGRATE_COMMENT_DELIMITERS\n        ):\n            yield from append_comments(leaf)\n\n        lowest_depth = min(lowest_depth, leaf.bracket_depth)\n        if trailing_comma_safe and leaf.bracket_depth == lowest_depth:\n            trailing_comma_safe = _can_add_trailing_comma(leaf, features)\n\n        if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf:\n            current_line = _safe_add_trailing_comma(\n                trailing_comma_safe, delimiter_priority, current_line\n            )\n\n        leaf_priority = bt.delimiters.get(id(leaf))\n        if leaf_priority == delimiter_priority:\n            if (\n                leaf_idx + 1 < len(line.leaves)\n                and delimiter_priority not in MIGRATE_COMMENT_DELIMITERS\n            ):\n                yield from append_comments(line.leaves[leaf_idx + 1])\n\n            yield current_line\n            current_line = Line(\n                mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets\n            )\n\n    if current_line:\n        current_line = _safe_add_trailing_comma(\n            trailing_comma_safe, delimiter_priority, current_line\n        )\n        yield current_line\n\n\n@dont_increase_indentation\ndef standalone_comment_split(\n    line: Line, features: Collection[Feature], mode: Mode\n) -> Iterator[Line]:\n    \"\"\"Split standalone comments from the rest of the line.\"\"\"\n    if not line.contains_standalone_comments():\n        raise CannotSplit(\"Line does not have any standalone comments\")\n\n    current_line = Line(\n        mode=line.mode, depth=line.depth, inside_brackets=line.inside_brackets\n    )\n\n    def append_to_line(leaf: Leaf) -> Iterator[Line]:\n        \"\"\"Append `leaf` to current line or to new line if appending impossible.\"\"\"\n        nonlocal current_line\n        try:\n            current_line.append_safe(leaf, preformatted=True)\n        except ValueError:\n            yield current_line\n\n            current_line = Line(\n                line.mode, depth=line.depth, inside_brackets=line.inside_brackets\n            )\n            current_line.append(leaf)\n\n    for leaf in line.leaves:\n        yield from append_to_line(leaf)\n\n        for comment_after in line.comments_after(leaf):\n            yield from append_to_line(comment_after)\n\n    if current_line:\n        yield current_line\n\n\ndef normalize_invisible_parens(  # noqa: C901\n    node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature]\n) -> None:\n    \"\"\"Make existing optional parentheses invisible or create new ones.\n\n    `parens_after` is a set of string leaf values immediately after which parens\n    should be put.\n\n    Standardizes on visible parentheses for single-element tuples, and keeps\n    existing visible parentheses for other tuples and generator expressions.\n    \"\"\"\n    for pc in list_comments(node.prefix, is_endmarker=False):\n        if pc.value in FMT_OFF:\n            # This `node` has a prefix with `# fmt: off`, don't mess with parens.\n            return\n\n    # The multiple context managers grammar has a different pattern, thus this is\n    # separate from the for-loop below. This possibly wraps them in invisible parens,\n    # and later will be removed in remove_with_parens when needed.\n    if node.type == syms.with_stmt:\n        _maybe_wrap_cms_in_parens(node, mode, features)\n\n    check_lpar = False\n    for index, child in enumerate(list(node.children)):\n        # Fixes a bug where invisible parens are not properly stripped from\n        # assignment statements that contain type annotations.\n        if isinstance(child, Node) and child.type == syms.annassign:\n            normalize_invisible_parens(\n                child, parens_after=parens_after, mode=mode, features=features\n            )\n\n        # Fixes a bug where invisible parens are not properly wrapped around\n        # case blocks.\n        if isinstance(child, Node) and child.type == syms.case_block:\n            normalize_invisible_parens(\n                child, parens_after={\"case\"}, mode=mode, features=features\n            )\n\n        # Add parentheses around long tuple unpacking in assignments.\n        if (\n            index == 0\n            and isinstance(child, Node)\n            and child.type == syms.testlist_star_expr\n        ):\n            check_lpar = True\n\n        if check_lpar:\n            if (\n                child.type == syms.atom\n                and node.type == syms.for_stmt\n                and isinstance(child.prev_sibling, Leaf)\n                and child.prev_sibling.type == token.NAME\n                and child.prev_sibling.value == \"for\"\n            ):\n                if maybe_make_parens_invisible_in_atom(\n                    child,\n                    parent=node,\n                    remove_brackets_around_comma=True,\n                ):\n                    wrap_in_parentheses(node, child, visible=False)\n            elif isinstance(child, Node) and node.type == syms.with_stmt:\n                remove_with_parens(child, node)\n            elif child.type == syms.atom:\n                if maybe_make_parens_invisible_in_atom(\n                    child,\n                    parent=node,\n                ):\n                    wrap_in_parentheses(node, child, visible=False)\n            elif is_one_tuple(child):\n                wrap_in_parentheses(node, child, visible=True)\n            elif node.type == syms.import_from:\n                _normalize_import_from(node, child, index)\n                break\n            elif (\n                index == 1\n                and child.type == token.STAR\n                and node.type == syms.except_clause\n            ):\n                # In except* (PEP 654), the star is actually part of\n                # of the keyword. So we need to skip the insertion of\n                # invisible parentheses to work more precisely.\n                continue\n\n            elif (\n                isinstance(child, Leaf)\n                and child.next_sibling is not None\n                and child.next_sibling.type == token.COLON\n                and child.value == \"case\"\n            ):\n                # A special patch for \"case case:\" scenario, the second occurrence\n                # of case will be not parsed as a Python keyword.\n                break\n\n            elif not (isinstance(child, Leaf) and is_multiline_string(child)):\n                wrap_in_parentheses(node, child, visible=False)\n\n        comma_check = child.type == token.COMMA\n\n        check_lpar = isinstance(child, Leaf) and (\n            child.value in parens_after or comma_check\n        )\n\n\ndef _normalize_import_from(parent: Node, child: LN, index: int) -> None:\n    # \"import from\" nodes store parentheses directly as part of\n    # the statement\n    if is_lpar_token(child):\n        assert is_rpar_token(parent.children[-1])\n        # make parentheses invisible\n        child.value = \"\"\n        parent.children[-1].value = \"\"\n    elif child.type != token.STAR:\n        # insert invisible parentheses\n        parent.insert_child(index, Leaf(token.LPAR, \"\"))\n        parent.append_child(Leaf(token.RPAR, \"\"))\n\n\ndef remove_await_parens(node: Node) -> None:\n    if node.children[0].type == token.AWAIT and len(node.children) > 1:\n        if (\n            node.children[1].type == syms.atom\n            and node.children[1].children[0].type == token.LPAR\n        ):\n            if maybe_make_parens_invisible_in_atom(\n                node.children[1],\n                parent=node,\n                remove_brackets_around_comma=True,\n            ):\n                wrap_in_parentheses(node, node.children[1], visible=False)\n\n            # Since await is an expression we shouldn't remove\n            # brackets in cases where this would change\n            # the AST due to operator precedence.\n            # Therefore we only aim to remove brackets around\n            # power nodes that aren't also await expressions themselves.\n            # https://peps.python.org/pep-0492/#updated-operator-precedence-table\n            # N.B. We've still removed any redundant nested brackets though :)\n            opening_bracket = cast(Leaf, node.children[1].children[0])\n            closing_bracket = cast(Leaf, node.children[1].children[-1])\n            bracket_contents = node.children[1].children[1]\n            if isinstance(bracket_contents, Node) and (\n                bracket_contents.type != syms.power\n                or bracket_contents.children[0].type == token.AWAIT\n                or any(\n                    isinstance(child, Leaf) and child.type == token.DOUBLESTAR\n                    for child in bracket_contents.children\n                )\n            ):\n                ensure_visible(opening_bracket)\n                ensure_visible(closing_bracket)\n\n\ndef _maybe_wrap_cms_in_parens(\n    node: Node, mode: Mode, features: Collection[Feature]\n) -> None:\n    \"\"\"When enabled and safe, wrap the multiple context managers in invisible parens.\n\n    It is only safe when `features` contain Feature.PARENTHESIZED_CONTEXT_MANAGERS.\n    \"\"\"\n    if (\n        Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features\n        or len(node.children) <= 2\n        # If it's an atom, it's already wrapped in parens.\n        or node.children[1].type == syms.atom\n    ):\n        return\n    colon_index: Optional[int] = None\n    for i in range(2, len(node.children)):\n        if node.children[i].type == token.COLON:\n            colon_index = i\n            break\n    if colon_index is not None:\n        lpar = Leaf(token.LPAR, \"\")\n        rpar = Leaf(token.RPAR, \"\")\n        context_managers = node.children[1:colon_index]\n        for child in context_managers:\n            child.remove()\n        # After wrapping, the with_stmt will look like this:\n        #   with_stmt\n        #     NAME 'with'\n        #     atom\n        #       LPAR ''\n        #       testlist_gexp\n        #         ... <-- context_managers\n        #       /testlist_gexp\n        #       RPAR ''\n        #     /atom\n        #     COLON ':'\n        new_child = Node(\n            syms.atom, [lpar, Node(syms.testlist_gexp, context_managers), rpar]\n        )\n        node.insert_child(1, new_child)\n\n\ndef remove_with_parens(node: Node, parent: Node) -> None:\n    \"\"\"Recursively hide optional parens in `with` statements.\"\"\"\n    # Removing all unnecessary parentheses in with statements in one pass is a tad\n    # complex as different variations of bracketed statements result in pretty\n    # different parse trees:\n    #\n    # with (open(\"file\")) as f:                       # this is an asexpr_test\n    #     ...\n    #\n    # with (open(\"file\") as f):                       # this is an atom containing an\n    #     ...                                         # asexpr_test\n    #\n    # with (open(\"file\")) as f, (open(\"file\")) as f:  # this is asexpr_test, COMMA,\n    #     ...                                         # asexpr_test\n    #\n    # with (open(\"file\") as f, open(\"file\") as f):    # an atom containing a\n    #     ...                                         # testlist_gexp which then\n    #                                                 # contains multiple asexpr_test(s)\n    if node.type == syms.atom:\n        if maybe_make_parens_invisible_in_atom(\n            node,\n            parent=parent,\n            remove_brackets_around_comma=True,\n        ):\n            wrap_in_parentheses(parent, node, visible=False)\n        if isinstance(node.children[1], Node):\n            remove_with_parens(node.children[1], node)\n    elif node.type == syms.testlist_gexp:\n        for child in node.children:\n            if isinstance(child, Node):\n                remove_with_parens(child, node)\n    elif node.type == syms.asexpr_test and not any(\n        leaf.type == token.COLONEQUAL for leaf in node.leaves()\n    ):\n        if maybe_make_parens_invisible_in_atom(\n            node.children[0],\n            parent=node,\n            remove_brackets_around_comma=True,\n        ):\n            wrap_in_parentheses(node, node.children[0], visible=False)\n\n\ndef maybe_make_parens_invisible_in_atom(\n    node: LN,\n    parent: LN,\n    remove_brackets_around_comma: bool = False,\n) -> bool:\n    \"\"\"If it's safe, make the parens in the atom `node` invisible, recursively.\n    Additionally, remove repeated, adjacent invisible parens from the atom `node`\n    as they are redundant.\n\n    Returns whether the node should itself be wrapped in invisible parentheses.\n    \"\"\"\n    if (\n        node.type not in (syms.atom, syms.expr)\n        or is_empty_tuple(node)\n        or is_one_tuple(node)\n        or (is_yield(node) and parent.type != syms.expr_stmt)\n        or (\n            # This condition tries to prevent removing non-optional brackets\n            # around a tuple, however, can be a bit overzealous so we provide\n            # and option to skip this check for `for` and `with` statements.\n            not remove_brackets_around_comma\n            and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY\n        )\n        or is_tuple_containing_walrus(node)\n    ):\n        return False\n\n    if is_walrus_assignment(node):\n        if parent.type in [\n            syms.annassign,\n            syms.expr_stmt,\n            syms.assert_stmt,\n            syms.return_stmt,\n            syms.except_clause,\n            syms.funcdef,\n            syms.with_stmt,\n            syms.tname,\n            # these ones aren't useful to end users, but they do please fuzzers\n            syms.for_stmt,\n            syms.del_stmt,\n            syms.for_stmt,\n        ]:\n            return False\n\n    first = node.children[0]\n    last = node.children[-1]\n    if is_lpar_token(first) and is_rpar_token(last):\n        middle = node.children[1]\n        # make parentheses invisible\n        if (\n            # If the prefix of `middle` includes a type comment with\n            # ignore annotation, then we do not remove the parentheses\n            not is_type_ignore_comment_string(middle.prefix.strip())\n        ):\n            first.value = \"\"\n            if first.prefix.strip():\n                # Preserve comments before first paren\n                middle.prefix = first.prefix + middle.prefix\n            last.value = \"\"\n        maybe_make_parens_invisible_in_atom(\n            middle,\n            parent=parent,\n            remove_brackets_around_comma=remove_brackets_around_comma,\n        )\n\n        if is_atom_with_invisible_parens(middle):\n            # Strip the invisible parens from `middle` by replacing\n            # it with the child in-between the invisible parens\n            middle.replace(middle.children[1])\n            if middle.children[-1].prefix.strip():\n                # Preserve comments before last paren\n                last.prefix = middle.children[-1].prefix + last.prefix\n\n        return False\n\n    return True\n\n\ndef should_split_line(line: Line, opening_bracket: Leaf) -> bool:\n    \"\"\"Should `line` be immediately split with `delimiter_split()` after RHS?\"\"\"\n\n    if not (opening_bracket.parent and opening_bracket.value in \"[{(\"):\n        return False\n\n    # We're essentially checking if the body is delimited by commas and there's more\n    # than one of them (we're excluding the trailing comma and if the delimiter priority\n    # is still commas, that means there's more).\n    exclude = set()\n    trailing_comma = False\n    try:\n        last_leaf = line.leaves[-1]\n        if last_leaf.type == token.COMMA:\n            trailing_comma = True\n            exclude.add(id(last_leaf))\n        max_priority = line.bracket_tracker.max_delimiter_priority(exclude=exclude)\n    except (IndexError, ValueError):\n        return False\n\n    return max_priority == COMMA_PRIORITY and (\n        (line.mode.magic_trailing_comma and trailing_comma)\n        # always explode imports\n        or opening_bracket.parent.type in {syms.atom, syms.import_from}\n    )\n\n\ndef generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[LeafID]]:\n    \"\"\"Generate sets of closing bracket IDs that should be omitted in a RHS.\n\n    Brackets can be omitted if the entire trailer up to and including\n    a preceding closing bracket fits in one line.\n\n    Yielded sets are cumulative (contain results of previous yields, too).  First\n    set is empty, unless the line should explode, in which case bracket pairs until\n    the one that needs to explode are omitted.\n    \"\"\"\n\n    omit: Set[LeafID] = set()\n    if not line.magic_trailing_comma:\n        yield omit\n\n    length = 4 * line.depth\n    opening_bracket: Optional[Leaf] = None\n    closing_bracket: Optional[Leaf] = None\n    inner_brackets: Set[LeafID] = set()\n    for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True):\n        length += leaf_length\n        if length > line_length:\n            break\n\n        has_inline_comment = leaf_length > len(leaf.value) + len(leaf.prefix)\n        if leaf.type == STANDALONE_COMMENT or has_inline_comment:\n            break\n\n        if opening_bracket:\n            if leaf is opening_bracket:\n                opening_bracket = None\n            elif leaf.type in CLOSING_BRACKETS:\n                prev = line.leaves[index - 1] if index > 0 else None\n                if (\n                    prev\n                    and prev.type == token.COMMA\n                    and leaf.opening_bracket is not None\n                    and not is_one_sequence_between(\n                        leaf.opening_bracket, leaf, line.leaves\n                    )\n                ):\n                    # Never omit bracket pairs with trailing commas.\n                    # We need to explode on those.\n                    break\n\n                inner_brackets.add(id(leaf))\n        elif leaf.type in CLOSING_BRACKETS:\n            prev = line.leaves[index - 1] if index > 0 else None\n            if prev and prev.type in OPENING_BRACKETS:\n                # Empty brackets would fail a split so treat them as \"inner\"\n                # brackets (e.g. only add them to the `omit` set if another\n                # pair of brackets was good enough.\n                inner_brackets.add(id(leaf))\n                continue\n\n            if closing_bracket:\n                omit.add(id(closing_bracket))\n                omit.update(inner_brackets)\n                inner_brackets.clear()\n                yield omit\n\n            if (\n                prev\n                and prev.type == token.COMMA\n                and leaf.opening_bracket is not None\n                and not is_one_sequence_between(leaf.opening_bracket, leaf, line.leaves)\n            ):\n                # Never omit bracket pairs with trailing commas.\n                # We need to explode on those.\n                break\n\n            if leaf.value:\n                opening_bracket = leaf.opening_bracket\n                closing_bracket = leaf\n\n\ndef run\n_transformer(\n    line: Line,\n    transform: Transformer,\n    mode: Mode,\n    features: Collection[Feature],\n    *,\n    line_str: str = \"\",\n) -> List[Line]:\n    if not line_str:\n        line_str = line_to_string(line)\n    result: List[Line] = []\n    for transformed_line in transform(line, features, mode):\n        if str(transformed_line).strip(\"\\n\") == line_str:\n            raise CannotTransform(\"Line transformer returned an unchanged result\")\n\n        result.extend(transform_line(transformed_line, mode=mode, features=features))\n\n    features_set = set(features)\n    if (\n        Feature.FORCE_OPTIONAL_PARENTHESES in features_set\n        or transform.__class__.__name__ != \"rhs\"\n        or not line.bracket_tracker.invisible\n        or any(bracket.value for bracket in line.bracket_tracker.invisible)\n        or line.contains_multiline_strings()\n        or result[0].contains_uncollapsable_type_comments()\n        or result[0].contains_unsplittable_type_ignore()\n        or is_line_short_enough(result[0], mode=mode)\n        # If any leaves have no parents (which _can_ occur since\n        # `transform(line)` potentially destroys the line's underlying node\n        # structure), then we can't proceed. Doing so would cause the below\n        # call to `append_leaves()` to fail.\n        or any(leaf.parent is None for leaf in line.leaves)\n    ):\n        return result\n\n    line_copy = line.clone()\n    append_leaves(line_copy, line, line.leaves)\n    features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}\n    second_opinion = run_transformer(\n        line_copy, transform, mode, features_fop, line_str=line_str\n    )\n    if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):\n        result = second_opinion\n    return result\n\n# Path: src/black/parsing.py\n\"\"\"\nParse Python code and perform AST validation.\n\"\"\"\n\nimport ast\nimport sys\nimport warnings\nfrom typing import Iterable, Iterator, List, Set, Tuple\n\nfrom black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature\nfrom black.nodes import syms\nfrom blib2to3 import pygram\nfrom blib2to3.pgen2 import driver\nfrom blib2to3.pgen2.grammar import Grammar\nfrom blib2to3.pgen2.parse import ParseError\nfrom blib2to3.pgen2.tokenize import TokenError\nfrom blib2to3.pytree import Leaf, Node\n\n\nclass InvalidInput(ValueError):\n    \"\"\"Raised when input source code fails all parse attempts.\"\"\"\n\n\ndef get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:\n    if not target_versions:\n        # No target_version specified, so try all grammars.\n        return [\n            # Python 3.7-3.9\n            pygram.python_grammar_async_keywords,\n            # Python 3.0-3.6\n            pygram.python_grammar,\n            # Python 3.10+\n            pygram.python_grammar_soft_keywords,\n        ]\n\n    grammars = []\n    # If we have to parse both, try to parse async as a keyword first\n    if not supports_feature(\n        target_versions, Feature.ASYNC_IDENTIFIERS\n    ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):\n        # Python 3.7-3.9\n        grammars.append(pygram.python_grammar_async_keywords)\n    if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):\n        # Python 3.0-3.6\n        grammars.append(pygram.python_grammar)\n    if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions):\n        # Python 3.10+\n        grammars.append(pygram.python_grammar_soft_keywords)\n\n    # At least one of the above branches must have been taken, because every Python\n    # version has exactly one of the two 'ASYNC_*' flags\n    return grammars\n\n\ndef lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:\n    \"\"\"Given a string with source, return the lib2to3 Node.\"\"\"\n    if not src_txt.endswith(\"\\n\"):\n        src_txt += \"\\n\"\n\n    grammars = get_grammars(set(target_versions))\n    errors = {}\n    for grammar in grammars:\n        drv = driver.Driver(grammar)\n        try:\n            result = drv.parse_string(src_txt, True)\n            break\n\n        except ParseError as pe:\n            lineno, column = pe.context[1]\n            lines = src_txt.splitlines()\n            try:\n                faulty_line = lines[lineno - 1]\n            except IndexError:\n                faulty_line = \"<line number missing in source>\"\n            errors[grammar.version] = InvalidInput(\n                f\"Cannot parse: {lineno}:{column}: {faulty_line}\"\n            )\n\n        except TokenError as te:\n            # In edge cases these are raised; and typically don't have a \"faulty_line\".\n            lineno, column = te.args[1]\n            errors[grammar.version] = InvalidInput(\n                f\"Cannot parse: {lineno}:{column}: {te.args[0]}\"\n            )\n\n    else:\n        # Choose the latest version when raising the actual parsing error.\n        assert len(errors) >= 1\n        exc = errors[max(errors)]\n        raise exc from None\n\n    if isinstance(result, Leaf):\n        result = Node(syms.file_input, [result])\n    return result\n\n\ndef matches_grammar(src_txt: str, grammar: Grammar) -> bool:\n    drv = driver.Driver(grammar)\n    try:\n        drv.parse_string(src_txt, True)\n    except (ParseError, TokenError, IndentationError):\n        return False\n    else:\n        return True\n\n\ndef lib2to3_unparse(node: Node) -> str:\n    \"\"\"Given a lib2to3 node, return its string representation.\"\"\"\n    code = str(node)\n    return code\n\n\ndef _parse_single_version(\n    src: str, version: Tuple[int, int], *, type_comments: bool\n) -> ast.AST:\n    filename = \"<unknown>\"\n    with warnings.catch_warnings():\n        warnings.simplefilter(\"ignore\", SyntaxWarning)\n        warnings.simplefilter(\"ignore\", DeprecationWarning)\n        return ast.parse(\n            src, filename, feature_version=version, type_comments=type_comments\n        )\n\n\ndef parse_ast(src: str) -> ast.AST:\n    # TODO: support Python 4+ ;)\n    versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]\n\n    first_error = \"\"\n    for version in sorted(versions, reverse=True):\n        try:\n            return _parse_single_version(src, version, type_comments=True)\n        except SyntaxError as e:\n            if not first_error:\n                first_error = str(e)\n\n    # Try to parse without type comments\n    for version in sorted(versions, reverse=True):\n        try:\n            return _parse_single_version(src, version, type_comments=False)\n        except SyntaxError:\n            pass\n\n    raise SyntaxError(first_error)\n\n\ndef _normalize(lineend: str, value: str) -> str:\n    # To normalize, we strip any leading and trailing space from\n    # each line...\n    stripped: List[str] = [i.strip() for i in value.splitlines()]\n    normalized = lineend.join(stripped)\n    # ...and remove any blank lines at the beginning and end of\n    # the whole string\n    return normalized.strip()\n\n\ndef stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]:\n    \"\"\"Simple visitor generating strings to compare ASTs by content.\"\"\"\n\n    if (\n        isinstance(node, ast.Constant)\n        and isinstance(node.value, str)\n        and node.kind == \"u\"\n    ):\n        # It's a quirk of history that we strip the u prefix over here. We used to\n        # rewrite the AST nodes for Python version compatibility and we never copied\n        # over the kind\n        node.kind = None\n\n    yield f\"{'  ' * depth}{node.__class__.__name__}(\"\n\n    for field in sorted(node._fields):  # noqa: F402\n        # TypeIgnore has only one field 'lineno' which breaks this comparison\n        if isinstance(node, ast.TypeIgnore):\n            break\n\n        try:\n            value: object = getattr(node, field)\n        except AttributeError:\n            continue\n\n        yield f\"{'  ' * (depth + 1)}{field}=\"\n\n        if isinstance(value, list):\n            for item in value:\n                # Ignore nested tuples within del statements, because we may insert\n                # parentheses and they change the AST.\n                if (\n                    field == \"targets\"\n                    and isinstance(node, ast.Delete)\n                    and isinstance(item, ast.Tuple)\n                ):\n                    for elt in item.elts:\n                        yield from stringify_ast(elt, depth + 2)\n\n                elif isinstance(item, ast.AST):\n                    yield from stringify_ast(item, depth + 2)\n\n        elif isinstance(value, ast.AST):\n            yield from stringify_ast(value, depth + 2)\n\n        else:\n            normalized: object\n            if (\n                isinstance(node, ast.Constant)\n                and field == \"value\"\n                and isinstance(value, str)\n            ):\n                # Constant strings may be indented across newlines, if they are\n                # docstrings; fold spaces after newlines when comparing. Similarly,\n                # trailing and leading space may be removed.\n                normalized = _normalize(\"\\n\", value)\n            elif field == \"type_comment\" and isinstance(value, str):\n                # Trailing whitespace in type comments is removed.\n                normalized = value.rstrip()\n            else:\n                normalized = value\n            yield f\"{'  ' * (depth + 2)}{normalized!r},  # {value.__class__.__name__}\"\n\n    yield f\"{'  ' * depth})  # /{node.__class__.__name__}\"\n\n# Path: src/black/ranges.py\n\"\"\"Functions related to Black's formatting by line ranges feature.\"\"\"\n\nimport difflib\nfrom dataclasses import dataclass\nfrom typing import Collection, Iterator, List, Sequence, Set, Tuple, Union\n\nfrom black.nodes import (\n    LN,\n    STANDALONE_COMMENT,\n    Leaf,\n    Node,\n    Visitor,\n    first_leaf,\n    furthest_ancestor_with_last_leaf,\n    last_leaf,\n    syms,\n)\nfrom blib2to3.pgen2.token import ASYNC, NEWLINE\n\n\ndef parse_line_ranges(line_ranges: Sequence[str]) -> List[Tuple[int, int]]:\n    lines: List[Tuple[int, int]] = []\n    for lines_str in line_ranges:\n        parts = lines_str.split(\"-\")\n        if len(parts) != 2:\n            raise ValueError(\n                \"Incorrect --line-ranges format, expect 'START-END', found\"\n                f\" {lines_str!r}\"\n            )\n        try:\n            start = int(parts[0])\n            end = int(parts[1])\n        except ValueError:\n            raise ValueError(\n                \"Incorrect --line-ranges value, expect integer ranges, found\"\n                f\" {lines_str!r}\"\n            ) from None\n        else:\n            lines.append((start, end))\n    return lines\n\n\ndef is_valid_line_range(lines: Tuple[int, int]) -> bool:\n    \"\"\"Returns whether the line range is valid.\"\"\"\n    return not lines or lines[0] <= lines[1]\n\n\ndef adjusted_lines(\n    lines: Collection[Tuple[int, int]],\n    original_source: str,\n    modified_source: str,\n) -> List[Tuple[int, int]]:\n    \"\"\"Returns the adjusted line ranges based on edits from the original code.\n\n    This computes the new line ranges by diffing original_source and\n    modified_source, and adjust each range based on how the range overlaps with\n    the diffs.\n\n    Note the diff can contain lines outside of the original line ranges. This can\n    happen when the formatting has to be done in adjacent to maintain consistent\n    local results. For example:\n\n    1. def my_func(arg1, arg2,\n    2.             arg3,):\n    3.   pass\n\n    If it restricts to line 2-2, it can't simply reformat line 2, it also has\n    to reformat line 1:\n\n    1. def my_func(\n    2.     arg1,\n    3.     arg2,\n    4.     arg3,\n    5. ):\n    6.   pass\n\n    In this case, we will expand the line ranges to also include the whole diff\n    block.\n\n    Args:\n      lines: a collection of line ranges.\n      original_source: the original source.\n      modified_source: the modified source.\n    \"\"\"\n    lines_mappings = _calculate_lines_mappings(original_source, modified_source)\n\n    new_lines = []\n    # Keep an index of the current search. Since the lines and lines_mappings are\n    # sorted, this makes the search complexity linear.\n    current_mapping_index = 0\n    for start, end in sorted(lines):\n        start_mapping_index = _find_lines_mapping_index(\n            start,\n            lines_mappings,\n            current_mapping_index,\n        )\n        end_mapping_index = _find_lines_mapping_index(\n            end,\n            lines_mappings,\n            start_mapping_index,\n        )\n        current_mapping_index = start_mapping_index\n        if start_mapping_index >= len(lines_mappings) or end_mapping_index >= len(\n            lines_mappings\n        ):\n            # Protect against invalid inputs.\n            continue\n        start_mapping = lines_mappings[start_mapping_index]\n        end_mapping = lines_mappings[end_mapping_index]\n        if start_mapping.is_changed_block:\n            # When the line falls into a changed block, expands to the whole block.\n            new_start = start_mapping.modified_start\n        else:\n            new_start = (\n                start - start_mapping.original_start + start_mapping.modified_start\n            )\n        if end_mapping.is_changed_block:\n            # When the line falls into a changed block, expands to the whole block.\n            new_end = end_mapping.modified_end\n        else:\n            new_end = end - end_mapping.original_start + end_mapping.modified_start\n        new_range = (new_start, new_end)\n        if is_valid_line_range(new_range):\n            new_lines.append(new_range)\n    return new_lines\n\n\ndef convert_unchanged_lines(src_node: Node, lines: Collection[Tuple[int, int]]) -> None:\n    \"\"\"Converts unchanged lines to STANDALONE_COMMENT.\n\n    The idea is similar to how `# fmt: on/off` is implemented. It also converts the\n    nodes between those markers as a single `STANDALONE_COMMENT` leaf node with\n    the unformatted code as its value. `STANDALONE_COMMENT` is a \"fake\" token\n    that will be formatted as-is with its prefix normalized.\n\n    Here we perform two passes:\n\n    1. Visit the top-level statements, and convert them to a single\n       `STANDALONE_COMMENT` when unchanged. This speeds up formatting when some\n       of the top-level statements aren't changed.\n    2. Convert unchanged \"unwrapped lines\" to `STANDALONE_COMMENT` nodes line by\n       line. \"unwrapped lines\" are divided by the `NEWLINE` token. e.g. a\n       multi-line statement is *one* \"unwrapped line\" that ends with `NEWLINE`,\n       even though this statement itself can span multiple lines, and the\n       tokenizer only sees the last '\\n' as the `NEWLINE` token.\n\n    NOTE: During pass (2), comment prefixes and indentations are ALWAYS\n    normalized even when the lines aren't changed. This is fixable by moving\n    more formatting to pass (1). However, it's hard to get it correct when\n    incorrect indentations are used. So we defer this to future optimizations.\n    \"\"\"\n    lines_set: Set[int] = set()\n    for start, end in lines:\n        lines_set.update(range(start, end + 1))\n    visitor = _TopLevelStatementsVisitor(lines_set)\n    _ = list(visitor.visit(src_node))  # Consume all results.\n    _convert_unchanged_line_by_line(src_node, lines_set)\n\n\ndef _contains_standalone_comment(node: LN) -> bool:\n    if isinstance(node, Leaf):\n        return node.type == STANDALONE_COMMENT\n    else:\n        for child in node.children:\n            if _contains_standalone_comment(child):\n                return True\n        return False\n\n\nclass _TopLevelStatementsVisitor(Visitor[None]):\n    \"\"\"\n    A node visitor that converts unchanged top-level statements to\n    STANDALONE_COMMENT.\n\n    This is used in addition to _convert_unchanged_line_by_line, to\n    speed up formatting when there are unchanged top-level\n    classes/functions/statements.\n    \"\"\"\n\n    def __init__(self, lines_set: Set[int]):\n        self._lines_set = lines_set\n\n    def visit_simple_stmt(self, node: Node) -> Iterator[None]:\n        # This is only called for top-level statements, since `visit_suite`\n        # won't visit its children nodes.\n        yield from []\n        newline_leaf = last_leaf(node)\n        if not newline_leaf:\n            return\n        assert (\n            newline_leaf.type == NEWLINE\n        ), f\"Unexpectedly found leaf.type={newline_leaf.type}\"\n        # We need to find the furthest ancestor with the NEWLINE as the last\n        # leaf, since a `suite` can simply be a `simple_stmt` when it puts\n        # its body on the same line. Example: `if cond: pass`.\n        ancestor = furthest_ancestor_with_last_leaf(newline_leaf)\n        if not _get_line_range(ancestor).intersection(self._lines_set):\n            _convert_node_to_standalone_comment(ancestor)\n\n    def visit_suite(self, node: Node) -> Iterator[None]:\n        yield from []\n        # If there is a STANDALONE_COMMENT node, it means parts of the node tree\n        # have fmt on/off/skip markers. Those STANDALONE_COMMENT nodes can't\n        # be simply converted by calling str(node). So we just don't convert\n        # here.\n        if _contains_standalone_comment(node):\n            return\n        # Find the semantic parent of this suite. For `async_stmt` and\n        # `async_funcdef`, the ASYNC token is defined on a separate level by the\n        # grammar.\n        semantic_parent = node.parent\n        if semantic_parent is not None:\n            if (\n                semantic_parent.prev_sibling is not None\n                and semantic_parent.prev_sibling.type == ASYNC\n            ):\n                semantic_parent = semantic_parent.parent\n        if semantic_parent is not None and not _get_line_range(\n            semantic_parent\n        ).intersection(self._lines_set):\n            _convert_node_to_standalone_comment(semantic_parent)\n\n\ndef _convert_unchanged_line_by_line(node: Node, lines_set: Set[int]) -> None:\n    \"\"\"Converts unchanged to STANDALONE_COMMENT line by line.\"\"\"\n    for leaf in node.leaves():\n        if leaf.type != NEWLINE:\n            # We only consider \"unwrapped lines\", which are divided by the NEWLINE\n            # token.\n            continue\n        if leaf.parent and leaf.parent.type == syms.match_stmt:\n            # The `suite` node is defined as:\n            #   match_stmt: \"match\" subject_expr ':' NEWLINE INDENT case_block+ DEDENT\n            # Here we need to check `subject_expr`. The `case_block+` will be\n            # checked by their own NEWLINEs.\n            nodes_to_ignore: List[LN] = []\n            prev_sibling = leaf.prev_sibling\n            while prev_sibling:\n                nodes_to_ignore.insert(0, prev_sibling)\n                prev_sibling = prev_sibling.prev_sibling\n            if not _get_line_range(nodes_to_ignore).intersection(lines_set):\n                _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf)\n        elif leaf.parent and leaf.parent.type == syms.suite:\n            # The `suite` node is defined as:\n            #   suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT\n            # We will check `simple_stmt` and `stmt+` separately against the lines set\n            parent_sibling = leaf.parent.prev_sibling\n            nodes_to_ignore = []\n            while parent_sibling and not parent_sibling.type == syms.suite:\n                # NOTE: Multiple suite nodes can exist as siblings in e.g. `if_stmt`.\n                nodes_to_ignore.insert(0, parent_sibling)\n                parent_sibling = parent_sibling.prev_sibling\n            # Special case for `async_stmt` and `async_funcdef` where the ASYNC\n            # token is on the grandparent node.\n            grandparent = leaf.parent.parent\n            if (\n                grandparent is not None\n                and grandparent.prev_sibling is not None\n                and grandparent.prev_sibling.type == ASYNC\n            ):\n                nodes_to_ignore.insert(0, grandparent.prev_sibling)\n            if not _get_line_range(nodes_to_ignore).intersection(lines_set):\n                _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf)\n        else:\n            ancestor = furthest_ancestor_with_last_leaf(leaf)\n            # Consider multiple decorators as a whole block, as their\n            # newlines have different behaviors than the rest of the grammar.\n            if (\n                ancestor.type == syms.decorator\n                and ancestor.parent\n                and ancestor.parent.type == syms.decorators\n            ):\n                ancestor = ancestor.parent\n            if not _get_line_range(ancestor).intersection(lines_set):\n                _convert_node_to_standalone_comment(ancestor)\n\n\ndef _convert_node_to_standalone_comment(node: LN) -> None:\n    \"\"\"Convert node to STANDALONE_COMMENT by modifying the tree inline.\"\"\"\n    parent = node.parent\n    if not parent:\n        return\n    first = first_leaf(node)\n    last = last_leaf(node)\n    if not first or not last:\n        return\n    if first is last:\n        # This can happen on the following edge cases:\n        # 1. A block of `# fmt: off/on` code except the `# fmt: on` is placed\n        #    on the end of the last line instead of on a new line.\n        # 2. A single backslash on its own line followed by a comment line.\n        # Ideally we don't want to format them when not requested, but fixing\n        # isn't easy. These cases are also badly formatted code, so it isn't\n        # too bad we reformat them.\n        return\n    # The prefix contains comments and indentation whitespaces. They are\n    # reformatted accordingly to the correct indentation level.\n    # This also means the indentation will be changed on the unchanged lines, and\n    # this is actually required to not break incremental reformatting.\n    prefix = first.prefix\n    first.prefix = \"\"\n    index = node.remove()\n    if index is not None:\n        # Remove the '\\n', as STANDALONE_COMMENT will have '\\n' appended when\n        # generating the formatted code.\n        value = str(node)[:-1]\n        parent.insert_child(\n            index,\n            Leaf(\n                STANDALONE_COMMENT,\n                value,\n                prefix=prefix,\n                fmt_pass_converted_first_leaf=first,\n            ),\n        )\n\n\ndef _convert_nodes_to_standalone_comment(nodes: Sequence[LN], *, newline: Leaf) -> None:\n    \"\"\"Convert nodes to STANDALONE_COMMENT by modifying the tree inline.\"\"\"\n    if not nodes:\n        return\n    parent = nodes[0].parent\n    first = first_leaf(nodes[0])\n    if not parent or not first:\n        return\n    prefix = first.prefix\n    first.prefix = \"\"\n    value = \"\".join(str(node) for node in nodes)\n    # The prefix comment on the NEWLINE leaf is the trailing comment of the statement.\n    if newline.prefix:\n        value += newline.prefix\n        newline.prefix = \"\"\n    index = nodes[0].remove()\n    for node in nodes[1:]:\n        node.remove()\n    if index is not None:\n        parent.insert_child(\n            index,\n            Leaf(\n                STANDALONE_COMMENT,\n                value,\n                prefix=prefix,\n                fmt_pass_converted_first_leaf=first,\n            ),\n        )\n\n\ndef _leaf_line_end(leaf: Leaf) -> int:\n    \"\"\"Returns the line number of the leaf node's last line.\"\"\"\n    if leaf.type == NEWLINE:\n        return leaf.lineno\n    else:\n        # Leaf nodes like multiline strings can occupy multiple lines.\n        return leaf.lineno + str(leaf).count(\"\\n\")\n\n\ndef _get_line_range(node_or_nodes: Union[LN, List[LN]]) -> Set[int]:\n    \"\"\"Returns the line range of this node or list of nodes.\"\"\"\n    if isinstance(node_or_nodes, list):\n        nodes = node_or_nodes\n        if not nodes:\n            return set()\n        first = first_leaf(nodes[0])\n        last = last_leaf(nodes[-1])\n        if first and last:\n            line_start = first.lineno\n            line_end = _leaf_line_end(last)\n            return set(range(line_start, line_end + 1))\n        else:\n            return set()\n    else:\n        node = node_or_nodes\n        if isinstance(node, Leaf):\n            return set(range(node.lineno, _leaf_line_end(node) + 1))\n        else:\n            first = first_leaf(node)\n            last = last_leaf(node)\n            if first and last:\n                return set(range(first.lineno, _leaf_line_end(last) + 1))\n            else:\n                return set()\n\n\n@dataclass\nclass _LinesMapping:\n    \"\"\"1-based lines mapping from original source to modified source.\n\n    Lines [original_start, original_end] from original source\n    are mapped to [modified_start, modified_end].\n\n    The ranges are inclusive on both ends.\n    \"\"\"\n\n    original_start: int\n    original_end: int\n    modified_start: int\n    modified_end: int\n    # Whether this range corresponds to a changed block, or an unchanged block.\n    is_changed_block: bool\n\n\ndef _calculate_lines_mappings(\n    original_source: str,\n    modified_source: str,\n) -> Sequence[_LinesMapping]:\n    \"\"\"Returns a sequence of _LinesMapping by diffing the sources.\n\n    For example, given the following diff:\n        import re\n      - def func(arg1,\n      -   arg2, arg3):\n      + def func(arg1, arg2, arg3):\n          pass\n    It returns the following mappings:\n      original -> modified\n       (1, 1)  ->  (1, 1), is_changed_block=False (the \"import re\" line)\n       (2, 3)  ->  (2, 2), is_changed_block=True (the diff)\n       (4, 4)  ->  (3, 3), is_changed_block=False (the \"pass\" line)\n\n    You can think of this visually as if it brings up a side-by-side diff, and tries\n    to map the line ranges from the left side to the right side:\n\n      (1, 1)->(1, 1)    1. import re          1. import re\n      (2, 3)->(2, 2)    2. def func(arg1,     2. def func(arg1, arg2, arg3):\n                        3.   arg2, arg3):\n      (4, 4)->(3, 3)    4.   pass             3.   pass\n\n    Args:\n      original_source: the original source.\n      modified_source: the modified source.\n    \"\"\"\n    matcher = difflib.SequenceMatcher(\n        None,\n        original_source.splitlines(keepends=True),\n        modified_source.splitlines(keepends=True),\n    )\n    matching_blocks = matcher.get_matching_blocks()\n    lines_mappings: List[_LinesMapping] = []\n    # matching_blocks is a sequence of \"same block of code ranges\", see\n    # https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_matching_blocks\n    # Each block corresponds to a _LinesMapping with is_changed_block=False,\n    # and the ranges between two blocks corresponds to a _LinesMapping with\n    # is_changed_block=True,\n    # NOTE: matching_blocks is 0-based, but _LinesMapping is 1-based.\n    for i, block in enumerate(matching_blocks):\n        if i == 0:\n            if block.a != 0 or block.b != 0:\n                lines_mappings.append(\n                    _LinesMapping(\n                        original_start=1,\n                        original_end=block.a,\n                        modified_start=1,\n                        modified_end=block.b,\n                        is_changed_block=False,\n                    )\n                )\n        else:\n            previous_block = matching_blocks[i - 1]\n            lines_mappings.append(\n                _LinesMapping(\n                    original_start=previous_block.a + previous_block.size + 1,\n                    original_end=block.a,\n                    modified_start=previous_block.b + previous_block.size + 1,\n                    modified_end=block.b,\n                    is_changed_block=True,\n                )\n            )\n        if i < len(matching_blocks) - 1:\n            lines_mappings.append(\n                _LinesMapping(\n                    original_start=block.a + 1,\n                    original_end=block.a + block.size,\n                    modified_start=block.b + 1,\n                    modified_end=block.b + block.size,\n                    is_changed_block=False,\n                )\n            )\n    return lines_mappings\n\n\ndef _find_lines_mapping_index(\n    original_line: int,\n    lines_mappings: Sequence[_LinesMapping],\n    start_index: int,\n) -> int:\n    \"\"\"Returns the original index of the lines mappings for the original line.\"\"\"\n    index = start_index\n    while index < len(lines_mappings):\n        mapping = lines_mappings[index]\n        if mapping.original_start <= original_line <= mapping.original_end:\n            return index\n        index += 1\n    return index\n\n# Path: src/black/__init__.py\nimport io\nimport json\nimport platform\nimport re\nimport sys\nimport tokenize\nimport traceback\nfrom contextlib import contextmanager\nfrom dataclasses import replace\nfrom datetime import datetime, timezone\nfrom enum import Enum\nfrom json.decoder import JSONDecodeError\nfrom pathlib import Path\nfrom typing import (\n    Any,\n    Collection,\n    Dict,\n    Generator,\n    Iterator,\n    List,\n    MutableMapping,\n    Optional,\n    Pattern,\n    Sequence,\n    Set,\n    Sized,\n    Tuple,\n    Union,\n)\n\nimport click\nfrom click.core import ParameterSource\nfrom mypy_extensions import mypyc_attr\nfrom pathspec import PathSpec\nfrom pathspec.patterns.gitwildmatch import GitWildMatchPatternError\n\nfrom _black_version import version as __version__\nfrom black.cache import Cache\nfrom black.comments import normalize_fmt_off\nfrom black.const import (\n    DEFAULT_EXCLUDES,\n    DEFAULT_INCLUDES,\n    DEFAULT_LINE_LENGTH,\n    STDIN_PLACEHOLDER,\n)\nfrom black.files import (\n    best_effort_relative_path,\n    find_project_root,\n    find_pyproject_toml,\n    find_user_pyproject_toml,\n    gen_python_files,\n    get_gitignore,\n    parse_pyproject_toml,\n    path_is_excluded,\n    resolves_outside_root_or_cannot_stat,\n    wrap_stream_for_windows,\n)\nfrom black.handle_ipynb_magics import (\n    PYTHON_CELL_MAGICS,\n    TRANSFORMED_MAGICS,\n    jupyter_dependencies_are_installed,\n    mask_cell,\n    put_trailing_semicolon_back,\n    remove_trailing_semicolon,\n    unmask_cell,\n)\nfrom black.linegen import LN, LineGenerator, transform_line\nfrom black.lines import EmptyLineTracker, LinesBlock\nfrom black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature\nfrom black.mode import Mode as Mode  # re-exported\nfrom black.mode import Preview, TargetVersion, supports_feature\nfrom black.nodes import (\n    STARS,\n    is_number_token,\n    is_simple_decorator_expression,\n    is_string_token,\n    syms,\n)\nfrom black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out\nfrom black.parsing import InvalidInput  # noqa F401\nfrom black.parsing import lib2to3_parse, parse_ast, stringify_ast\nfrom black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges\nfrom black.report import Changed, NothingChanged, Report\nfrom black.trans import iter_fexpr_spans\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\nCOMPILED = Path(__file__).suffix in (\".pyd\", \".so\")\n\n# types\nFileContent = str\nEncoding = str\nNewLine = str\n\n\nclass WriteBack(Enum):\n    NO = 0\n    YES = 1\n    DIFF = 2\n    CHECK = 3\n    COLOR_DIFF = 4\n\n    @classmethod\n    def from_configuration(\n        cls, *, check: bool, diff: bool, color: bool = False\n    ) -> \"WriteBack\":\n        if check and not diff:\n            return cls.CHECK\n\n        if diff and color:\n            return cls.COLOR_DIFF\n\n        return cls.DIFF if diff else cls.YES\n\n\n# Legacy name, left for integrations.\nFileMode = Mode\n\n\ndef read_pyproject_toml(\n    ctx: click.Context, param: click.Parameter, value: Optional[str]\n) -> Optional[str]:\n    \"\"\"Inject Black configuration from \"pyproject.toml\" into defaults in `ctx`.\n\n    Returns the path to a successfully found and read configuration file, None\n    otherwise.\n    \"\"\"\n    if not value:\n        value = find_pyproject_toml(\n            ctx.params.get(\"src\", ()), ctx.params.get(\"stdin_filename\", None)\n        )\n        if value is None:\n            return None\n\n    try:\n        config = parse_pyproject_toml(value)\n    except (OSError, ValueError) as e:\n        raise click.FileError(\n            filename=value, hint=f\"Error reading configuration file: {e}\"\n        ) from None\n\n    if not config:\n        return None\n    else:\n        spellcheck_pyproject_toml_keys(ctx, list(config), value)\n        # Sanitize the values to be Click friendly. For more information please see:\n        # https://github.com/psf/black/issues/1458\n        # https://github.com/pallets/click/issues/1567\n        config = {\n            k: str(v) if not isinstance(v, (list, dict)) else v\n            for k, v in config.items()\n        }\n\n    target_version = config.get(\"target_version\")\n    if target_version is not None and not isinstance(target_version, list):\n        raise click.BadOptionUsage(\n            \"target-version\", \"Config key target-version must be a list\"\n        )\n\n    exclude = config.get(\"exclude\")\n    if exclude is not None and not isinstance(exclude, str):\n        raise click.BadOptionUsage(\"exclude\", \"Config key exclude must be a string\")\n\n    extend_exclude = config.get(\"extend_exclude\")\n    if extend_exclude is not None and not isinstance(extend_exclude, str):\n        raise click.BadOptionUsage(\n            \"extend-exclude\", \"Config key extend-exclude must be a string\"\n        )\n\n    line_ranges = config.get(\"line_ranges\")\n    if line_ranges is not None:\n        raise click.BadOptionUsage(\n            \"line-ranges\", \"Cannot use line-ranges in the pyproject.toml file.\"\n        )\n\n    default_map: Dict[str, Any] = {}\n    if ctx.default_map:\n        default_map.update(ctx.default_map)\n    default_map.update(config)\n\n    ctx.default_map = default_map\n    return value\n\n\ndef spellcheck_pyproject_toml_keys(\n    ctx: click.Context, config_keys: List[str], config_file_path: str\n) -> None:\n    invalid_keys: List[str] = []\n    available_config_options = {param.name for param in ctx.command.params}\n    for key in config_keys:\n        if key not in available_config_options:\n            invalid_keys.append(key)\n    if invalid_keys:\n        keys_str = \", \".join(map(repr, invalid_keys))\n        out(\n            f\"Invalid config keys detected: {keys_str} (in {config_file_path})\",\n            fg=\"red\",\n        )\n\n\ndef target_version_option_callback(\n    c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]\n) -> List[TargetVersion]:\n    \"\"\"Compute the target versions from a --target-version flag.\n\n    This is its own function because mypy couldn't infer the type correctly\n    when it was a lambda, causing mypyc trouble.\n    \"\"\"\n    return [TargetVersion[val.upper()] for val in v]\n\n\ndef enable_unstable_feature_callback(\n    c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]\n) -> List[Preview]:\n    \"\"\"Compute the features from an --enable-unstable-feature flag.\"\"\"\n    return [Preview[val] for val in v]\n\n\ndef re_compile_maybe_verbose(regex: str) -> Pattern[str]:\n    \"\"\"Compile a regular expression string in `regex`.\n\n    If it contains newlines, use verbose mode.\n    \"\"\"\n    if \"\\n\" in regex:\n        regex = \"(?x)\" + regex\n    compiled: Pattern[str] = re.compile(regex)\n    return compiled\n\n\ndef validate_regex(\n    ctx: click.Context,\n    param: click.Parameter,\n    value: Optional[str],\n) -> Optional[Pattern[str]]:\n    try:\n        return re_compile_maybe_verbose(value) if value is not None else None\n    except re.error as e:\n        raise click.BadParameter(f\"Not a valid regular expression: {e}\") from None\n\n\n@click.command(\n    context_settings={\"help_option_names\": [\"-h\", \"--help\"]},\n    # While Click does set this field automatically using the docstring, mypyc\n    # (annoyingly) strips 'em so we need to set it here too.\n    help=\"The uncompromising code formatter.\",\n)\n@click.option(\"-c\", \"--code\", type=str, help=\"Format the code passed in as a string.\")\n@click.option(\n    \"-l\",\n    \"--line-length\",\n    type=int,\n    default=DEFAULT_LINE_LENGTH,\n    help=\"How many characters per line to allow.\",\n    show_default=True,\n)\n@click.option(\n    \"-t\",\n    \"--target-version\",\n    type=click.Choice([v.name.lower() for v in TargetVersion]),\n    callback=target_version_option_callback,\n    multiple=True,\n    help=(\n        \"Python versions that should be supported by Black's output. You should\"\n        \" include all versions that your code supports. By default, Black will infer\"\n        \" target versions from the project metadata in pyproject.toml. If this does\"\n        \" not yield conclusive results, Black will use per-file auto-detection.\"\n    ),\n)\n@click.option(\n    \"--pyi\",\n    is_flag=True,\n    help=(\n        \"Format all input files like typing stubs regardless of file extension. This\"\n        \" is useful when piping source on standard input.\"\n    ),\n)\n@click.option(\n    \"--ipynb\",\n...\n", "needle_token_start": 5377, "needle_token_end": 5921, "code_context_ntokens": 16166, "output": ["```python\ndef run_transformer(\n    line: Line,\n    transform: Transformer,\n    mode: Mode,\n    features: Collection[Feature],\n    *,\n    line_str: str = \"\",\n) -> List[Line]:\n    if not line_str:\n        line_str = line_to_string(line)\n    result: List[Line] = []\n    for transformed_line in transform(line, features, mode):\n        if str(transformed_line).strip(\"\\n\") == line_str:\n            raise CannotTransform(\"Line transformer returned an unchanged result\")\n\n        result.extend(transform_line(transformed_line, mode=mode, features=features))\n\n    features_set = set(features)\n    if (\n        Feature.FORCE_OPTIONAL_PARENTHESES in features_set\n        or transform.__class__.__name__ != \"rhs\"\n        or not line.bracket_tracker.invisible\n        or any(bracket.value for bracket in line.bracket_tracker.invisible)\n        or line.contains_multiline_strings()\n        or result[0].contains_uncollapsable_type_comments()\n        or result[0].contains_unsplittable_type_ignore()\n        or is_line_short_enough(result[0], mode=mode)\n        # If any leaves have no parents (which _can_ occur since\n        # `transform(line)` potentially destroys the line's underlying node\n        # structure), then we can't proceed. Doing so would cause the below\n        # call to `append_leaves()` to fail.\n        or any(leaf.parent is None for leaf in line.leaves)\n    ):\n        return result\n\n    line_copy = line.clone()\n    append_leaves(line_copy, line, line.leaves)\n    features_fop = features_set | {Feature.FORCE_OPTIONAL_PARENTHESES}\n    second_opinion = run_transformer(\n        line_copy, transform, mode, features_fop, line_str=line_str\n    )\n    if all(is_line_short_enough(ln, mode=mode) for ln in second_opinion):\n        result = second_opinion\n    return result\n```"]}
{"repo": "psf/black", "name": "is_part_of_annotation", "language": "python", "path": "src/black/nodes.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: Determines if a specified syntax element is part of a type annotation in the source code.\n2. **Input**: A syntax element from the parsed source code.\n3. **Output**: A boolean value indicating whether the given syntax element is involved in a type annotation.\n4. **Procedure**: The function checks the type of annotation associated with the syntax element by tracing its position in the syntax tree and examining its relationship with surrounding elements. If the element is identified as part of either a return type or parameter type annotation, the function returns true; otherwise, it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    DOUBLESPACE: Final[str] = \"  \"\n    t = leaf.type\n    p = leaf.parent\n    v = leaf.value\n    if t in ALWAYS_NO_SPACE:\n        return NO\n\n    if t == token.COMMENT:\n        return DOUBLESPACE\n\n    assert p is not None, f\"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}\"\n    if t == token.COLON and p.type not in {\n        syms.subscript,\n        syms.subscriptlist,\n        syms.sliceop,\n    }:\n        return NO\n\n    prev = leaf.prev_sibling\n    if not prev:\n        prevp = preceding_leaf(p)\n        if not prevp or prevp.type in OPENING_BRACKETS:\n            return NO\n\n        if t == token.COLON:\n            if prevp.type == token.COLON:\n                return NO\n\n            elif prevp.type != token.COMMA and not complex_subscript:\n                return NO\n\n            return SPACE\n\n        if prevp.type == token.EQUAL:\n            if prevp.parent:\n                if prevp.parent.type in {\n                    syms.arglist,\n                    syms.argument,\n                    syms.parameters,\n                    syms.varargslist,\n                }:\n                    return NO\n\n                elif prevp.parent.type == syms.typedargslist:\n                    # A bit hacky: if the equal sign has whitespace, it means we\n                    # previously found it's a typed argument.  So, we're using\n                    # that, too.\n                    return prevp.prefix\n\n        elif (\n            prevp.type == token.STAR\n            and parent_type(prevp) == syms.star_expr\n            and parent_type(prevp.parent) == syms.subscriptlist\n        ):\n            # No space between typevar tuples.\n            return NO\n\n        elif prevp.type in VARARGS_SPECIALS:\n            if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):\n                return NO\n\n        elif prevp.type == token.COLON:\n            if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:\n                return SPACE if complex_subscript else NO\n\n        elif (\n            prevp.parent\n            and prevp.parent.type == syms.factor\n            and prevp.type in MATH_OPERATORS\n        ):\n            return NO\n\n        elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator:\n            # no space in decorators\n            return NO\n\n    elif prev.type in OPENING_BRACKETS:\n        return NO\n\n    if p.type in {syms.parameters, syms.arglist}:\n        # untyped function signatures or calls\n        if not prev or prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.varargslist:\n        # lambdas\n        if prev and prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.typedargslist:\n        # typed function signatures\n        if not prev:\n            return NO\n\n        if t == token.EQUAL:\n            if prev.type not in TYPED_NAMES:\n                return NO\n\n        elif prev.type == token.EQUAL:\n            # A bit hacky: if the equal sign has whitespace, it means we\n            # previously found it's a typed argument.  So, we're using that, too.\n            return prev.prefix\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type in TYPED_NAMES:\n        # type names\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type != token.COMMA:\n                return NO\n\n    elif p.type == syms.trailer:\n        # attributes and calls\n        if t == token.LPAR or t == token.RPAR:\n            return NO\n\n        if not prev:\n            if t == token.DOT or t == token.LSQB:\n                return NO\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.argument:\n        # single argument\n        if t == token.EQUAL:\n            return NO\n\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type == token.LPAR:\n                return NO\n\n        elif prev.type in {token.EQUAL} | VARARGS_SPECIALS:\n            return NO\n\n    elif p.type == syms.decorator:\n        # decorators\n        return NO\n\n    elif p.type == syms.dotted_name:\n        if prev:\n            return NO\n\n        prevp = preceding_leaf(p)\n        if not prevp or prevp.type == token.AT or prevp.type == token.DOT:\n            return NO\n\n    elif p.type == syms.classdef:\n        if t == token.LPAR:\n            return NO\n\n        if prev and prev.type == token.LPAR:\n            return NO\n\n    elif p.type in {syms.subscript, syms.sliceop}:\n        # indexing\n        if not prev:\n            assert p.parent is not None, \"subscripts are always parented\"\n            if p.parent.type == syms.subscriptlist:\n                return SPACE\n\n            return NO\n\n        elif t == token.COLONEQUAL or prev.type == token.COLONEQUAL:\n            return SPACE\n\n        elif not complex_subscript:\n            return NO\n\n    elif p.type == syms.atom:\n        if prev and t == token.DOT:\n            # dots, but not the first one.\n            return NO\n\n    elif p.type == syms.dictsetmaker:\n        # dict unpacking\n        if prev and prev.type == token.DOUBLESTAR:\n            return NO\n\n    elif p.type in {syms.factor, syms.star_expr}:\n        # unary ops\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type in OPENING_BRACKETS:\n                return NO\n\n            prevp_parent = prevp.parent\n            assert prevp_parent is not None\n            if prevp.type == token.COLON and prevp_parent.type in {\n                syms.subscript,\n                syms.sliceop,\n            }:\n                return NO\n\n            elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:\n                return NO\n\n        elif t in {token.NAME, token.NUMBER, token.STRING}:\n            return NO\n\n    elif p.type == syms.import_from:\n        if t == token.DOT:\n            if prev and prev.type == token.DOT:\n                return NO\n\n        elif t == token.NAME:\n            if v == \"import\":\n                return SPACE\n\n            if prev and prev.type == token.DOT:\n                return NO\n\n    elif p.type == syms.sliceop:\n        return NO\n\n    elif p.type == syms.except_clause:\n        if t == token.STAR:\n            return NO\n\n    return SPACE\n\n\ndef make_simple_prefix(nl_count: int, form_feed: bool, empty_line: str = \"\\n\") -> str:\n    \"\"\"Generate a normalized prefix string.\"\"\"\n    if form_feed:\n        return (empty_line * (nl_count - 1)) + \"\\f\" + empty_line\n    return empty_line * nl_count\n\n\ndef preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:\n    \"\"\"Return the first leaf that precedes `node`, if any.\"\"\"\n    while node:\n        res = node.prev_sibling\n        if res:\n            if isinstance(res, Leaf):\n                return res\n\n            try:\n                return list(res.leaves())[-1]\n\n            except IndexError:\n                return None\n\n        node = node.parent\n    return None\n\n\ndef prev_siblings_are(node: Optional[LN], tokens: List[Optional[NodeType]]) -> bool:\n    \"\"\"Return if the `node` and its previous siblings match types against the provided\n    list of tokens; the provided `node`has its type matched against the last element in\n    the list.  `None` can be used as the first element to declare that the start of the\n    list is anchored at the start of its parent's children.\"\"\"\n    if not tokens:\n        return True\n    if tokens[-1] is None:\n        return node is None\n    if not node:\n        return False\n    if node.type != tokens[-1]:\n        return False\n    return prev_siblings_are(node.prev_sibling, tokens[:-1])\n\n\ndef parent_type(node: Optional[LN]) -> Optional[NodeType]:\n    \"\"\"\n    Returns:\n        @node.parent.type, if @node is not None and has a parent.\n            OR\n        None, otherwise.\n    \"\"\"\n    if node is None or node.parent is None:\n        return None\n\n    return node.parent.type\n\n\ndef child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:\n    \"\"\"Return the child of `ancestor` that contains `descendant`.\"\"\"\n    node: Optional[LN] = descendant\n    while node and node.parent != ancestor:\n        node = node.parent\n    return node\n\n\ndef replace_child(old_child: LN, new_child: LN) -> None:\n    \"\"\"\n    Side Effects:\n        * If @old_child.parent is set, replace @old_child with @new_child in\n        @old_child's underlying Node structure.\n            OR\n        * Otherwise, this function does nothing.\n    \"\"\"\n    parent = old_child.parent\n    if not parent:\n        return\n\n    child_idx = old_child.remove()\n    if child_idx is not None:\n        parent.insert_child(child_idx, new_child)\n\n\ndef container_of(leaf: Leaf) -> LN:\n    \"\"\"Return `leaf` or one of its ancestors that is the topmost container of it.\n\n    By \"container\" we mean a node where `leaf` is the very first child.\n    \"\"\"\n    same_prefix = leaf.prefix\n    container: LN = leaf\n    while container:\n        parent = container.parent\n        if parent is None:\n            break\n\n        if parent.children[0].prefix != same_prefix:\n            break\n\n        if parent.type == syms.file_input:\n            break\n\n        if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS:\n            break\n\n        container = parent\n    return container\n\n\ndef first_leaf_of(node: LN) -> Optional[Leaf]:\n    \"\"\"Returns the first leaf of the node tree.\"\"\"\n    if isinstance(node, Leaf):\n        return node\n    if node.children:\n        return first_leaf_of(node.children[0])\n    else:\n        return None\n\n\ndef is_arith_like(node: LN) -> bool:\n    \"\"\"Whether node is an arithmetic or a binary arithmetic expression\"\"\"\n    return node.type in {\n        syms.arith_expr,\n        syms.shift_expr,\n        syms.xor_expr,\n        syms.and_expr,\n    }\n\n\ndef is_docstring(leaf: Leaf, mode: Mode) -> bool:\n    if leaf.type != token.STRING:\n        return False\n\n    prefix = get_string_prefix(leaf.value)\n    if set(prefix).intersection(\"bBfF\"):\n        return False\n\n    if (\n        Preview.unify_docstring_detection in mode\n        and leaf.parent\n        and leaf.parent.type == syms.simple_stmt\n        and not leaf.parent.prev_sibling\n        and leaf.parent.parent\n        and leaf.parent.parent.type == syms.file_input\n    ):\n        return True\n\n    if prev_siblings_are(\n        leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt]\n    ):\n        return True\n\n    # Multiline docstring on the same line as the `def`.\n    if prev_siblings_are(leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt]):\n        # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python\n        # grammar. We're safe to return True without further checks.\n        return True\n\n    return False\n\n\ndef is_empty_tuple(node: LN) -> bool:\n    \"\"\"Return True if `node` holds an empty tuple.\"\"\"\n    return (\n        node.type == syms.atom\n        and len(node.children) == 2\n        and node.children[0].type == token.LPAR\n        and node.children[1].type == token.RPAR\n    )\n\n\ndef is_one_tuple(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a tuple with one element, with or without parens.\"\"\"\n    if node.type == syms.atom:\n        gexp = unwrap_singleton_parenthesis(node)\n        if gexp is None or gexp.type != syms.testlist_gexp:\n            return False\n\n        return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA\n\n    return (\n        node.type in IMPLICIT_TUPLE\n        and len(node.children) == 2\n        and node.children[1].type == token.COMMA\n    )\n\n\ndef is_tuple_containing_walrus(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a tuple that contains a walrus operator.\"\"\"\n    if node.type != syms.atom:\n        return False\n    gexp = unwrap_singleton_parenthesis(node)\n    if gexp is None or gexp.type != syms.testlist_gexp:\n        return False\n\n    return any(child.type == syms.namedexpr_test for child in gexp.children)\n\n\ndef is_one_sequence_between(\n    opening: Leaf,\n    closing: Leaf,\n    leaves: List[Leaf],\n    brackets: Tuple[int, int] = (token.LPAR, token.RPAR),\n) -> bool:\n    \"\"\"Return True if content between `opening` and `closing` is a one-sequence.\"\"\"\n    if (opening.type, closing.type) != brackets:\n        return False\n\n    depth = closing.bracket_depth + 1\n    for _opening_index, leaf in enumerate(leaves):\n        if leaf is opening:\n            break\n\n    else:\n        raise LookupError(\"Opening paren not found in `leaves`\")\n\n    commas = 0\n    _opening_index += 1\n    for leaf in leaves[_opening_index:]:\n        if leaf is closing:\n            break\n\n        bracket_depth = leaf.bracket_depth\n        if bracket_depth == depth and leaf.type == token.COMMA:\n            commas += 1\n            if leaf.parent and leaf.parent.type in {\n                syms.arglist,\n                syms.typedargslist,\n            }:\n                commas += 1\n                break\n\n    return commas < 2\n\n\ndef is_walrus_assignment(node: LN) -> bool:\n    \"\"\"Return True iff `node` is of the shape ( test := test )\"\"\"\n    inner = unwrap_singleton_parenthesis(node)\n    return inner is not None and inner.type == syms.namedexpr_test\n\n\ndef is_simple_decorator_trailer(node: LN, last: bool = False) -> bool:\n    \"\"\"Return True iff `node` is a trailer valid in a simple decorator\"\"\"\n    return node.type == syms.trailer and (\n        (\n            len(node.children) == 2\n            and node.children[0].type == token.DOT\n            and node.children[1].type == token.NAME\n        )\n        # last trailer can be an argument-less parentheses pair\n        or (\n            last\n            and len(node.children) == 2\n            and node.children[0].type == token.LPAR\n            and node.children[1].type == token.RPAR\n        )\n        # last trailer can be arguments\n        or (\n            last\n            and len(node.children) == 3\n            and node.children[0].type == token.LPAR\n            # and node.children[1].type == syms.argument\n            and node.children[2].type == token.RPAR\n        )\n    )\n\n\ndef is_simple_decorator_expression(node: LN) -> bool:\n    \"\"\"Return True iff `node` could be a 'dotted name' decorator\n\n    This function takes the node of the 'namedexpr_test' of the new decorator\n    grammar and test if it would be valid under the old decorator grammar.\n\n    The old grammar was: decorator: @ dotted_name [arguments] NEWLINE\n    The new grammar is : decorator: @ namedexpr_test NEWLINE\n    \"\"\"\n    if node.type == token.NAME:\n        return True\n    if node.type == syms.power:\n        if node.children:\n            return (\n                node.children[0].type == token.NAME\n                and all(map(is_simple_decorator_trailer, node.children[1:-1]))\n                and (\n                    len(node.children) < 2\n                    or is_simple_decorator_trailer(node.children[-1], last=True)\n                )\n            )\n    return False\n\n\ndef is_yield(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a `yield` or `yield from` expression.\"\"\"\n    if node.type == syms.yield_expr:\n        return True\n\n    if is_name_token(node) and node.value == \"yield\":\n        return True\n\n    if node.type != syms.atom:\n        return False\n\n    if len(node.children) != 3:\n        return False\n\n    lpar, expr, rpar = node.children\n    if lpar.type == token.LPAR and rpar.type == token.RPAR:\n        return is_yield(expr)\n\n    return False\n\n\ndef is_vararg(leaf: Leaf, within: Set[NodeType]) -> bool:\n    \"\"\"Return True if `leaf` is a star or double star in a vararg or kwarg.\n\n    If `within` includes VARARGS_PARENTS, this applies to function signatures.\n    If `within` includes UNPACKING_PARENTS, it applies to right hand-side\n    extended iterable unpacking (PEP 3132) and additional unpacking\n    generalizations (PEP 448).\n    \"\"\"\n    if leaf.type not in VARARGS_SPECIALS or not leaf.parent:\n        return False\n\n    p = leaf.parent\n    if p.type == syms.star_expr:\n        # Star expressions are also used as assignment targets in extended\n        # iterable unpacking (PEP 3132).  See what its parent is instead.\n        if not p.parent:\n            return False\n\n        p = p.parent\n\n    return p.type in within\n\n\ndef is_multiline_string(leaf: Leaf) -> bool:\n    \"\"\"Return True if `leaf` is a multiline string that actually spans many lines.\"\"\"\n    return has_triple_quotes(leaf.value) and \"\\n\" in leaf.value\n\n\ndef is_parent_function_or_class(node: Node) -> bool:\n    assert node.type in {syms.suite, syms.simple_stmt}\n    assert node.parent is not None\n    # Note this works for suites / simple_stmts in async def as well\n    return node.parent.type in {syms.funcdef, syms.classdef}\n\n\ndef is_function_or_class(node: Node) -> bool:\n    return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef}\n\n\ndef is_stub_suite(node: Node) -> bool:\n    \"\"\"Return True if `node` is a suite with a stub body.\"\"\"\n    if node.parent is not None and not is_parent_function_or_class(node):\n        return False\n\n    # If there is a comment, we want to keep it.\n    if node.prefix.strip():\n        return False\n\n    if (\n        len(node.children) != 4\n        or node.children[0].type != token.NEWLINE\n        or node.children[1].type != token.INDENT\n        or node.children[3].type != token.DEDENT\n    ):\n        return False\n\n    if node.children[3].prefix.strip():\n        return False\n\n    return is_stub_body(node.children[2])\n\n\ndef is_stub_body(node: LN) -> bool:\n    \"\"\"Return True if `node` is a simple statement containing an ellipsis.\"\"\"\n    if not isinstance(node, Node) or node.type != syms.simple_stmt:\n        return False\n\n    if len(node.children) != 2:\n        return False\n\n    child = node.children[0]\n    return (\n        not child.prefix.strip()\n        and child.type == syms.atom\n        and len(child.children) == 3\n        and all(leaf == Leaf(token.DOT, \".\") for leaf in child.children)\n    )\n\n\ndef is_atom_with_invisible_parens(node: LN) -> bool:\n    \"\"\"Given a `LN`, determines whether it's an atom `node` with invisible\n    parens. Useful in dedupe-ing and normalizing parens.\n    \"\"\"\n    if isinstance(node, Leaf) or node.type != syms.atom:\n        return False\n\n    first, last = node.children[0], node.children[-1]\n    return (\n        isinstance(first, Leaf)\n        and first.type == token.LPAR\n        and first.value == \"\"\n        and isinstance(last, Leaf)\n        and last.type == token.RPAR\n        and last.value == \"\"\n    )\n\n\ndef is_empty_par(leaf: Leaf) -> bool:\n    return is_empty_lpar(leaf) or is_empty_rpar(leaf)\n\n\ndef is_empty_lpar(leaf: Leaf) -> bool:\n    return leaf.type == token.LPAR and leaf.value == \"\"\n\n\ndef is_empty_rpar(leaf: Leaf) -> bool:\n    return leaf.type == token.RPAR and leaf.value == \"\"\n\n\ndef is_import(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf starts an import statement.\"\"\"\n    p = leaf.parent\n    t = leaf.type\n    v = leaf.value\n    return bool(\n        t == token.NAME\n        and (\n            (v == \"import\" and p and p.type == syms.import_name)\n            or (v == \"from\" and p and p.type == syms.import_from)\n        )\n    )\n\n\ndef is_with_or_async_with_stmt(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf starts a with or async with statement.\"\"\"\n    return bool(\n        leaf.type == token.NAME\n        and leaf.value == \"with\"\n        and leaf.parent\n        and leaf.parent.type == syms.with_stmt\n    ) or bool(\n        leaf.type == token.ASYNC\n        and leaf.next_sibling\n        and leaf.next_sibling.type == syms.with_stmt\n    )\n\n\ndef is_async_stmt_or_funcdef(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf starts an async def/for/with statement.\n\n    Note that `async def` can be either an `async_stmt` or `async_funcdef`,\n    the latter is used when it has decorators.\n    \"\"\"\n    return bool(\n        leaf.type == token.ASYNC\n        and leaf.parent\n        and leaf.parent.type in {syms.async_stmt, syms.async_funcdef}\n    )\n\n\ndef is_type_comment(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf is a type comment. This function should only\n    be used for general type comments (excluding ignore annotations, which should\n    use `is_type_ignore_comment`). Note that general type comments are no longer\n    used in modern version of Python, this function may be deprecated in the future.\"\"\"\n    t = leaf.type\n    v = leaf.value\n    return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith(\"# type:\")\n\n\ndef is_type_ignore_comment(leaf: Leaf) -> bool:\n    \"\"\"Return True if the given leaf is a type comment with ignore annotation.\"\"\"\n    t = leaf.type\n    v = leaf.value\n    return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v)\n\n\ndef is_type_ignore_comment_string(value: str) -> bool:\n    \"\"\"Return True if the given string match with type comment with\n    ignore annotation.\"\"\"\n    return value.startswith(\"# type: ignore\")\n\n\ndef wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None:\n    \"\"\"Wrap `child` in parentheses.\n\n    This replaces `child` with an atom holding the parentheses and the old\n    child.  That requires moving the prefix.\n\n    If `visible` is False, the leaves will be valueless (and thus invisible).\n    \"\"\"\n    lpar = Leaf(token.LPAR, \"(\" if visible else \"\")\n    rpar = Leaf(token.RPAR, \")\" if visible else \"\")\n    prefix = child.prefix\n    child.prefix = \"\"\n    index = child.remove() or 0\n    new_child = Node(syms.atom, [lpar, child, rpar])\n    new_child.prefix = prefix\n    parent.insert_child(index, new_child)\n\n\ndef unwrap_singleton_parenthesis(node: LN) -> Optional[LN]:\n    \"\"\"Returns `wrapped` if `node` is of the shape ( wrapped ).\n\n    Parenthesis can be optional. Returns None otherwise\"\"\"\n    if len(node.children) != 3:\n        return None\n\n    lpar, wrapped, rpar = node.children\n    if not (lpar.type == token.LPAR and rpar.type == token.RPAR):\n        return None\n\n    return wrapped\n\n\ndef ensure_visible(leaf: Leaf) -> None:\n    \"\"\"Make sure parentheses are visible.\n\n    They could be invisible as part of some statements (see\n    :func:`normalize_invisible_parens` and :func:`visit_import_from`).\n    \"\"\"\n    if leaf.type == token.LPAR:\n        leaf.value = \"(\"\n    elif leaf.type == token.RPAR:\n        leaf.value = \")\"\n\n\ndef is_name_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.NAME\n\n\ndef is_lpar_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.LPAR\n\n\ndef is_rpar_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.RPAR\n\n\ndef is_string_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.STRING\n\n\ndef is_number_token(nl: NL) -> TypeGuard[Leaf]:\n    return nl.type == token.NUMBER\n\n\ndef get_annotation_type(leaf: Leaf) -> Literal[\"return\", \"param\", None]:\n    \"\"\"Returns the type of annotation this leaf is part of, if any.\"\"\"\n    ancestor = leaf.parent\n    while ancestor is not None:\n        if ancestor.prev_sibling and ancestor.prev_sibling.type == token.RARROW:\n            return \"return\"\n        if ancestor.parent and ancestor.parent.type == syms.tname:\n            return \"param\"\n        ancestor = ancestor.parent\n    return None\n\n\n\ndef is_part_of_annotation(leaf: Leaf) -> bool:\n    \"\"\"Returns whether this leaf is part of a type annotation.\"\"\"\n    return get_annotation_type(leaf) is not None\n\n\ndef first_leaf(node: LN) -> Optional[Leaf]:\n    \"\"\"Returns the first leaf of the ancestor node.\"\"\"\n    if isinstance(node, Leaf):\n        return node\n    elif not node.children:\n        return None\n    else:\n        return first_leaf(node.children[0])\n\n\ndef last_leaf(node: LN) -> Optional[Leaf]:\n    \"\"\"Returns the last leaf of the ancestor node.\"\"\"\n    if isinstance(node, Leaf):\n        return node\n    elif not node.children:\n        return None\n    else:\n        return last_leaf(node.children[-1])\n\n\ndef furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN:\n    \"\"\"Returns the furthest ancestor that has this leaf node as the last leaf.\"\"\"\n    node: LN = leaf\n    while node.parent and node.parent.children and node is node.parent.children[-1]:\n        node = node.parent\n    return node\n\n# Path: src/black/comments.py\nimport re\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nfrom typing import Collection, Final, Iterator, List, Optional, Tuple, Union\n\nfrom black.mode import Mode, Preview\nfrom black.nodes import (\n    CLOSING_BRACKETS,\n    STANDALONE_COMMENT,\n    WHITESPACE,\n    container_of,\n    first_leaf_of,\n    make_simple_prefix,\n    preceding_leaf,\n    syms,\n)\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\n# types\nLN = Union[Leaf, Node]\n\nFMT_OFF: Final = {\"# fmt: off\", \"# fmt:off\", \"# yapf: disable\"}\nFMT_SKIP: Final = {\"# fmt: skip\", \"# fmt:skip\"}\nFMT_ON: Final = {\"# fmt: on\", \"# fmt:on\", \"# yapf: enable\"}\n\nCOMMENT_EXCEPTIONS = \" !:#'\"\n_COMMENT_PREFIX = \"# \"\n_COMMENT_LIST_SEPARATOR = \";\"\n\n\n@dataclass\nclass ProtoComment:\n    \"\"\"Describes a piece of syntax that is a comment.\n\n    It's not a :class:`blib2to3.pytree.Leaf` so that:\n\n    * it can be cached (`Leaf` objects should not be reused more than once as\n      they store their lineno, column, prefix, and parent information);\n    * `newlines` and `consumed` fields are kept separate from the `value`. This\n      simplifies handling of special marker comments like ``# fmt: off/on``.\n    \"\"\"\n\n    type: int  # token.COMMENT or STANDALONE_COMMENT\n    value: str  # content of the comment\n    newlines: int  # how many newlines before the comment\n    consumed: int  # how many characters of the original leaf's prefix did we consume\n    form_feed: bool  # is there a form feed before the comment\n    leading_whitespace: str  # leading whitespace before the comment, if any\n\n\ndef generate_comments(leaf: LN) -> Iterator[Leaf]:\n    \"\"\"Clean the prefix of the `leaf` and generate comments from it, if any.\n\n    Comments in lib2to3 are shoved into the whitespace prefix.  This happens\n    in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation\n    move because it does away with modifying the grammar to include all the\n    possible places in which comments can be placed.\n\n    The sad consequence for us though is that comments don't \"belong\" anywhere.\n    This is why this function generates simple parentless Leaf objects for\n    comments.  We simply don't know what the correct parent should be.\n\n    No matter though, we can live without this.  We really only need to\n    differentiate between inline and standalone comments.  The latter don't\n    share the line with any code.\n\n    Inline comments are emitted as regular token.COMMENT leaves.  Standalone\n    are emitted with a fake STANDALONE_COMMENT token identifier.\n    \"\"\"\n    total_consumed = 0\n    for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER):\n        total_consumed = pc.consumed\n        prefix = make_simple_prefix(pc.newlines, pc.form_feed)\n        yield Leaf(pc.type, pc.value, prefix=prefix)\n    normalize_trailing_prefix(leaf, total_consumed)\n\n\n@lru_cache(maxsize=4096)\ndef list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:\n    \"\"\"Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.\"\"\"\n    result: List[ProtoComment] = []\n    if not prefix or \"#\" not in prefix:\n        return result\n\n    consumed = 0\n    nlines = 0\n    ignored_lines = 0\n    form_feed = False\n    for index, full_line in enumerate(re.split(\"\\r?\\n\", prefix)):\n        consumed += len(full_line) + 1  # adding the length of the split '\\n'\n        match = re.match(r\"^(\\s*)(\\S.*|)$\", full_line)\n        assert match\n        whitespace, line = match.groups()\n        if not line:\n            nlines += 1\n            if \"\\f\" in full_line:\n                form_feed = True\n        if not line.startswith(\"#\"):\n            # Escaped newlines outside of a comment are not really newlines at\n            # all. We treat a single-line comment following an escaped newline\n            # as a simple trailing comment.\n            if line.endswith(\"\\\\\"):\n                ignored_lines += 1\n            continue\n\n        if index == ignored_lines and not is_endmarker:\n            comment_type = token.COMMENT  # simple trailing comment\n        else:\n            comment_type = STANDALONE_COMMENT\n        comment = make_comment(line)\n        result.append(\n            ProtoComment(\n                type=comment_type,\n                value=comment,\n                newlines=nlines,\n                consumed=consumed,\n                form_feed=form_feed,\n                leading_whitespace=whitespace,\n            )\n        )\n        form_feed = False\n        nlines = 0\n    return result\n\n\ndef normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None:\n    \"\"\"Normalize the prefix that's left over after generating comments.\n\n    Note: don't use backslashes for formatting or you'll lose your voting rights.\n    \"\"\"\n    remainder = leaf.prefix[total_consumed:]\n    if \"\\\\\" not in remainder:\n        nl_count = remainder.count(\"\\n\")\n        form_feed = \"\\f\" in remainder and remainder.endswith(\"\\n\")\n        leaf.prefix = make_simple_prefix(nl_count, form_feed)\n        return\n\n    leaf.prefix = \"\"\n\n\ndef make_comment(content: str) -> str:\n    \"\"\"Return a consistently formatted comment from the given `content` string.\n\n    All comments (except for \"##\", \"#!\", \"#:\", '#'\") should have a single\n    space between the hash sign and the content.\n\n    If `content` didn't start with a hash sign, one is provided.\n    \"\"\"\n    content = content.rstrip()\n    if not content:\n        return \"#\"\n\n    if content[0] == \"#\":\n        content = content[1:]\n    NON_BREAKING_SPACE = \"\u00a0\"\n    if (\n        content\n        and content[0] == NON_BREAKING_SPACE\n        and not content.lstrip().startswith(\"type:\")\n    ):\n        content = \" \" + content[1:]  # Replace NBSP by a simple space\n    if content and content[0] not in COMMENT_EXCEPTIONS:\n        content = \" \" + content\n    return \"#\" + content\n\n\ndef normalize_fmt_off(\n    node: Node, mode: Mode, lines: Collection[Tuple[int, int]]\n) -> None:\n    \"\"\"Convert content between `# fmt: off`/`# fmt: on` into standalone comments.\"\"\"\n    try_again = True\n    while try_again:\n        try_again = convert_one_fmt_off_pair(node, mode, lines)\n\n\ndef convert_one_fmt_off_pair(\n    node: Node, mode: Mode, lines: Collection[Tuple[int, int]]\n) -> bool:\n    \"\"\"Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.\n\n    Returns True if a pair was converted.\n    \"\"\"\n    for leaf in node.leaves():\n        previous_consumed = 0\n        for comment in list_comments(leaf.prefix, is_endmarker=False):\n            should_pass_fmt = comment.value in FMT_OFF or _contains_fmt_skip_comment(\n                comment.value, mode\n            )\n            if not should_pass_fmt:\n                previous_consumed = comment.consumed\n                continue\n            # We only want standalone comments. If there's no previous leaf or\n            # the previous leaf is indentation, it's a standalone comment in\n            # disguise.\n            if should_pass_fmt and comment.type != STANDALONE_COMMENT:\n                prev = preceding_leaf(leaf)\n                if prev:\n                    if comment.value in FMT_OFF and prev.type not in WHITESPACE:\n                        continue\n                    if (\n                        _contains_fmt_skip_comment(comment.value, mode)\n                        and prev.type in WHITESPACE\n                    ):\n                        continue\n\n            ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode))\n            if not ignored_nodes:\n                continue\n\n            first = ignored_nodes[0]  # Can be a container node with the `leaf`.\n            parent = first.parent\n            prefix = first.prefix\n            if comment.value in FMT_OFF:\n                first.prefix = prefix[comment.consumed :]\n            if _contains_fmt_skip_comment(comment.value, mode):\n                first.prefix = \"\"\n                standalone_comment_prefix = prefix\n            else:\n                standalone_comment_prefix = (\n                    prefix[:previous_consumed] + \"\\n\" * comment.newlines\n                )\n            hidden_value = \"\".join(str(n) for n in ignored_nodes)\n            comment_lineno = leaf.lineno - comment.newlines\n            if comment.value in FMT_OFF:\n                fmt_off_prefix = \"\"\n                if len(lines) > 0 and not any(\n                    line[0] <= comment_lineno <= line[1] for line in lines\n                ):\n                    # keeping indentation of comment by preserving original whitespaces.\n                    fmt_off_prefix = prefix.split(comment.value)[0]\n                    if \"\\n\" in fmt_off_prefix:\n                        fmt_off_prefix = fmt_off_prefix.split(\"\\n\")[-1]\n                standalone_comment_prefix += fmt_off_prefix\n                hidden_value = comment.value + \"\\n\" + hidden_value\n            if _contains_fmt_skip_comment(comment.value, mode):\n                hidden_value += (\n                    comment.leading_whitespace\n                    if Preview.no_normalize_fmt_skip_whitespace in mode\n                    else \"  \"\n                ) + comment.value\n            if hidden_value.endswith(\"\\n\"):\n                # That happens when one of the `ignored_nodes` ended with a NEWLINE\n                # leaf (possibly followed by a DEDENT).\n                hidden_value = hidden_value[:-1]\n            first_idx: Optional[int] = None\n            for ignored in ignored_nodes:\n                index = ignored.remove()\n                if first_idx is None:\n                    first_idx = index\n            assert parent is not None, \"INTERNAL ERROR: fmt: on/off handling (1)\"\n            assert first_idx is not None, \"INTERNAL ERROR: fmt: on/off handling (2)\"\n            parent.insert_child(\n                first_idx,\n                Leaf(\n                    STANDALONE_COMMENT,\n                    hidden_value,\n                    prefix=standalone_comment_prefix,\n                    fmt_pass_converted_first_leaf=first_leaf_of(first),\n                ),\n            )\n            return True\n\n    return False\n\n\ndef generate_ignored_nodes(\n    leaf: Leaf, comment: ProtoComment, mode: Mode\n) -> Iterator[LN]:\n    \"\"\"Starting from the container of `leaf`, generate all leaves until `# fmt: on`.\n\n    If comment is skip, returns leaf only.\n    Stops at the end of the block.\n    \"\"\"\n    if _contains_fmt_skip_comment(comment.value, mode):\n        yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment)\n        return\n    container: Optional[LN] = container_of(leaf)\n    while container is not None and container.type != token.ENDMARKER:\n        if is_fmt_on(container):\n            return\n\n        # fix for fmt: on in children\n        if children_contains_fmt_on(container):\n            for index, child in enumerate(container.children):\n                if isinstance(child, Leaf) and is_fmt_on(child):\n                    if child.type in CLOSING_BRACKETS:\n                        # This means `# fmt: on` is placed at a different bracket level\n                        # than `# fmt: off`. This is an invalid use, but as a courtesy,\n                        # we include this closing bracket in the ignored nodes.\n                        # The alternative is to fail the formatting.\n                        yield child\n                    return\n                if (\n                    child.type == token.INDENT\n                    and index < len(container.children) - 1\n                    and children_contains_fmt_on(container.children[index + 1])\n                ):\n                    # This means `# fmt: on` is placed right after an indentation\n                    # level, and we shouldn't swallow the previous INDENT token.\n                    return\n                if children_contains_fmt_on(child):\n                    return\n                yield child\n        else:\n            if container.type == token.DEDENT and container.next_sibling is None:\n                # This can happen when there is no matching `# fmt: on` comment at the\n                # same level as `# fmt: on`. We need to keep this DEDENT.\n                return\n            yield container\n            container = container.next_sibling\n\n\ndef _generate_ignored_nodes_from_fmt_skip(\n    leaf: Leaf, comment: ProtoComment\n) -> Iterator[LN]:\n    \"\"\"Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.\"\"\"\n    prev_sibling = leaf.prev_sibling\n    parent = leaf.parent\n    # Need to properly format the leaf prefix to compare it to comment.value,\n    # which is also formatted\n    comments = list_comments(leaf.prefix, is_endmarker=False)\n    if not comments or comment.value != comments[0].value:\n        return\n    if prev_sibling is not None:\n        leaf.prefix = \"\"\n        siblings = [prev_sibling]\n        while \"\\n\" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None:\n            prev_sibling = prev_sibling.prev_sibling\n            siblings.insert(0, prev_sibling)\n        yield from siblings\n    elif (\n        parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE\n    ):\n        # The `# fmt: skip` is on the colon line of the if/while/def/class/...\n        # statements. The ignored nodes should be previous siblings of the\n        # parent suite node.\n        leaf.prefix = \"\"\n        ignored_nodes: List[LN] = []\n        parent_sibling = parent.prev_sibling\n        while parent_sibling is not None and parent_sibling.type != syms.suite:\n            ignored_nodes.insert(0, parent_sibling)\n            parent_sibling = parent_sibling.prev_sibling\n        # Special case for `async_stmt` where the ASYNC token is on the\n        # grandparent node.\n        grandparent = parent.parent\n        if (\n            grandparent is not None\n            and grandparent.prev_sibling is not None\n            and grandparent.prev_sibling.type == token.ASYNC\n        ):\n            ignored_nodes.insert(0, grandparent.prev_sibling)\n        yield from iter(ignored_nodes)\n\n\ndef is_fmt_on(container: LN) -> bool:\n    \"\"\"Determine whether formatting is switched on within a container.\n    Determined by whether the last `# fmt:` comment is `on` or `off`.\n    \"\"\"\n    fmt_on = False\n    for comment in list_comments(container.prefix, is_endmarker=False):\n        if comment.value in FMT_ON:\n            fmt_on = True\n        elif comment.value in FMT_OFF:\n            fmt_on = False\n    return fmt_on\n\n\ndef children_contains_fmt_on(container: LN) -> bool:\n    \"\"\"Determine if children have formatting switched on.\"\"\"\n    for child in container.children:\n        leaf = first_leaf_of(child)\n        if leaf is not None and is_fmt_on(leaf):\n            return True\n\n    return False\n\n\ndef contains_pragma_comment(comment_list: List[Leaf]) -> bool:\n    \"\"\"\n    Returns:\n        True iff one of the comments in @comment_list is a pragma used by one\n        of the more common static analysis tools for python (e.g. mypy, flake8,\n        pylint).\n    \"\"\"\n    for comment in comment_list:\n        if comment.value.startswith((\"# type:\", \"# noqa\", \"# pylint:\")):\n            return True\n\n    return False\n\n\ndef _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool:\n    \"\"\"\n    Checks if the given comment contains FMT_SKIP alone or paired with other comments.\n    Matching styles:\n      # fmt:skip                           <-- single comment\n      # noqa:XXX # fmt:skip # a nice line  <-- multiple comments (Preview)\n      # pylint:XXX; fmt:skip               <-- list of comments (; separated, Preview)\n    \"\"\"\n    semantic_comment_blocks = [\n        comment_line,\n        *[\n            _COMMENT_PREFIX + comment.strip()\n            for comment in comment_line.split(_COMMENT_PREFIX)[1:]\n        ],\n        *[\n            _COMMENT_PREFIX + comment.strip()\n            for comment in comment_line.strip(_COMMENT_PREFIX).split(\n                _COMMENT_LIST_SEPARATOR\n            )\n        ],\n    ]\n\n    return any(comment in FMT_SKIP for comment in semantic_comment_blocks)\n\n# Path: src/black/report.py\n\"\"\"\nSummarize Black runs to users.\n\"\"\"\n\nfrom dataclasses import dataclass\nfrom enum import Enum\nfrom pathlib import Path\n\nfrom click import style\n\nfrom black.output import err, out\n\n\nclass Changed(Enum):\n    NO = 0\n    CACHED = 1\n    YES = 2\n\n\nclass NothingChanged(UserWarning):\n    \"\"\"Raised when reformatted code is the same as source.\"\"\"\n\n\n@dataclass\nclass Report:\n    \"\"\"Provides a reformatting counter. Can be rendered with `str(report)`.\"\"\"\n\n    check: bool = False\n    diff: bool = False\n    quiet: bool = False\n    verbose: bool = False\n    change_count: int = 0\n    same_count: int = 0\n    failure_count: int = 0\n\n    def done(self, src: Path, changed: Changed) -> None:\n        \"\"\"Increment the counter for successful reformatting. Write out a message.\"\"\"\n        if changed is Changed.YES:\n            reformatted = \"would reformat\" if self.check or self.diff else \"reformatted\"\n            if self.verbose or not self.quiet:\n                out(f\"{reformatted} {src}\")\n            self.change_count += 1\n        else:\n            if self.verbose:\n                if changed is Changed.NO:\n                    msg = f\"{src} already well formatted, good job.\"\n                else:\n                    msg = f\"{src} wasn't modified on disk since last run.\"\n                out(msg, bold=False)\n            self.same_count += 1\n\n    def failed(self, src: Path, message: str) -> None:\n        \"\"\"Increment the counter for failed reformatting. Write out a message.\"\"\"\n        err(f\"error: cannot format {src}: {message}\")\n        self.failure_count += 1\n\n    def path_ignored(self, path: Path, message: str) -> None:\n        if self.verbose:\n            out(f\"{path} ignored: {message}\", bold=False)\n\n    @property\n    def return_code(self) -> int:\n        \"\"\"Return the exit code that the app should use.\n\n        This considers the current state of changed files and failures:\n        - if there were any failures, return 123;\n        - if any files were changed and --check is being used, return 1;\n        - otherwise return 0.\n        \"\"\"\n        # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with\n        # 126 we have special return codes reserved by the shell.\n        if self.failure_count:\n            return 123\n\n        elif self.change_count and self.check:\n            return 1\n\n        return 0\n\n    def __str__(self) -> str:\n        \"\"\"Render a color report of the current state.\n\n        Use `click.unstyle` to remove colors.\n        \"\"\"\n        if self.check or self.diff:\n            reformatted = \"would be reformatted\"\n            unchanged = \"would be left unchanged\"\n            failed = \"would fail to reformat\"\n        else:\n            reformatted = \"reformatted\"\n            unchanged = \"left unchanged\"\n            failed = \"failed to reformat\"\n        report = []\n        if self.change_count:\n            s = \"s\" if self.change_count > 1 else \"\"\n            report.append(\n                style(f\"{self.change_count} file{s} \", bold=True, fg=\"blue\")\n                + style(f\"{reformatted}\", bold=True)\n            )\n\n        if self.same_count:\n            s = \"s\" if self.same_count > 1 else \"\"\n            report.append(style(f\"{self.same_count} file{s} \", fg=\"blue\") + unchanged)\n        if self.failure_count:\n            s = \"s\" if self.failure_count > 1 else \"\"\n            report.append(style(f\"{self.failure_count} file{s} {failed}\", fg=\"red\"))\n        return \", \".join(report) + \".\"\n\n# Path: src/black/concurrency.py\n\"\"\"\nFormatting many files at once via multiprocessing. Contains entrypoint and utilities.\n\nNOTE: this module is only imported if we need to format several files at once.\n\"\"\"\n\nimport asyncio\nimport logging\nimport os\nimport signal\nimport sys\nimport traceback\nfrom concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor\nfrom multiprocessing import Manager\nfrom pathlib import Path\nfrom typing import Any, Iterable, Optional, Set\n\nfrom mypy_extensions import mypyc_attr\n\nfrom black import WriteBack, format_file_in_place\nfrom black.cache import Cache\nfrom black.mode import Mode\nfrom black.output import err\nfrom black.report import Changed, Report\n\n\ndef maybe_install_uvloop() -> None:\n    \"\"\"If our environment has uvloop installed we use it.\n\n    This is called only from command-line entry points to avoid\n    interfering with the parent process if Black is used as a library.\n    \"\"\"\n    try:\n        import uvloop\n\n        uvloop.install()\n    except ImportError:\n        pass\n\n\ndef cancel(tasks: Iterable[\"asyncio.Future[Any]\"]) -> None:\n    \"\"\"asyncio signal handler that cancels all `tasks` and reports to stderr.\"\"\"\n    err(\"Aborted!\")\n    for task in tasks:\n        task.cancel()\n\n\ndef shutdown(loop: asyncio.AbstractEventLoop) -> None:\n    \"\"\"Cancel all pending tasks on `loop`, wait for them, and close the loop.\"\"\"\n    try:\n        # This part is borrowed from asyncio/runners.py in Python 3.7b2.\n        to_cancel = [task for task in asyncio.all_tasks(loop) if not task.done()]\n        if not to_cancel:\n            return\n\n        for task in to_cancel:\n            task.cancel()\n        loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))\n    finally:\n        # `concurrent.futures.Future` objects cannot be cancelled once they\n        # are already running. There might be some when the `shutdown()` happened.\n        # Silence their logger's spew about the event loop being closed.\n        cf_logger = logging.getLogger(\"concurrent.futures\")\n        cf_logger.setLevel(logging.CRITICAL)\n        loop.close()\n\n\n# diff-shades depends on being to monkeypatch this function to operate. I know it's\n# not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26\n@mypyc_attr(patchable=True)\ndef reformat_many(\n    sources: Set[Path],\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: Report,\n    workers: Optional[int],\n) -> None:\n    \"\"\"Reformat multiple files using a ProcessPoolExecutor.\"\"\"\n    maybe_install_uvloop()\n\n    executor: Executor\n    if workers is None:\n        workers = int(os.environ.get(\"BLACK_NUM_WORKERS\", 0))\n        workers = workers or os.cpu_count() or 1\n    if sys.platform == \"win32\":\n        # Work around https://bugs.python.org/issue26903\n        workers = min(workers, 60)\n    try:\n        executor = ProcessPoolExecutor(max_workers=workers)\n    except (ImportError, NotImplementedError, OSError):\n        # we arrive here if the underlying system does not support multi-processing\n        # like in AWS Lambda or Termux, in which case we gracefully fallback to\n        # a ThreadPoolExecutor with just a single worker (more workers would not do us\n        # any good due to the Global Interpreter Lock)\n        executor = ThreadPoolExecutor(max_workers=1)\n\n    loop = asyncio.new_event_loop()\n    asyncio.set_event_loop(loop)\n    try:\n        loop.run_until_complete(\n            schedule_formatting(\n                sources=sources,\n                fast=fast,\n                write_back=write_back,\n                mode=mode,\n                report=report,\n                loop=loop,\n                executor=executor,\n            )\n        )\n    finally:\n        try:\n            shutdown(loop)\n        finally:\n            asyncio.set_event_loop(None)\n        if executor is not None:\n            executor.shutdown()\n\n\nasync def schedule_formatting(\n    sources: Set[Path],\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: \"Report\",\n    loop: asyncio.AbstractEventLoop,\n    executor: \"Executor\",\n) -> None:\n    \"\"\"Run formatting of `sources` in parallel using the provided `executor`.\n\n    (Use ProcessPoolExecutors for actual parallelism.)\n\n    `write_back`, `fast`, and `mode` options are passed to\n    :func:`format_file_in_place`.\n    \"\"\"\n    cache = Cache.read(mode)\n    if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n        sources, cached = cache.filtered_cached(sources)\n        for src in sorted(cached):\n            report.done(src, Changed.CACHED)\n    if not sources:\n        return\n\n    cancelled = []\n    sources_to_cache = []\n    lock = None\n    if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n        # For diff output, we need locks to ensure we don't interleave output\n        # from different processes.\n        manager = Manager()\n        lock = manager.Lock()\n    tasks = {\n        asyncio.ensure_future(\n            loop.run_in_executor(\n                executor, format_file_in_place, src, fast, mode, write_back, lock\n            )\n        ): src\n        for src in sorted(sources)\n    }\n    pending = tasks.keys()\n    try:\n        loop.add_signal_handler(signal.SIGINT, cancel, pending)\n        loop.add_signal_handler(signal.SIGTERM, cancel, pending)\n    except NotImplementedError:\n        # There are no good alternatives for these on Windows.\n        pass\n    while pending:\n        done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)\n        for task in done:\n            src = tasks.pop(task)\n            if task.cancelled():\n                cancelled.append(task)\n            elif exc := task.exception():\n                if report.verbose:\n                    traceback.print_exception(type(exc), exc, exc.__traceback__)\n                report.failed(src, str(exc))\n            else:\n                changed = Changed.YES if task.result() else Changed.NO\n                # If the file was written back or was successfully checked as\n                # well-formatted, store this information in the cache.\n                if write_back is WriteBack.YES or (\n                    write_back is WriteBack.CHECK and changed is Changed.NO\n                ):\n                    sources_to_cache.append(src)\n                report.done(src, changed)\n    if cancelled:\n        await asyncio.gather(*cancelled, return_exceptions=True)\n    if sources_to_cache:\n        cache.write(sources_to_cache)\n\n# Path: src/black/handle_ipynb_magics.py\n\"\"\"Functions to process IPython magics with.\"\"\"\n\nimport ast\nimport collections\nimport dataclasses\nimport secrets\nimport sys\nfrom functools import lru_cache\nfrom importlib.util import find_spec\nfrom typing import Dict, List, Optional, Tuple\n\nif sys.version_info >= (3, 10):\n    from typing import TypeGuard\nelse:\n    from typing_extensions import TypeGuard\n\nfrom black.output import out\nfrom black.report import NothingChanged\n\nTRANSFORMED_MAGICS = frozenset((\n    \"get_ipython().run_cell_magic\",\n    \"get_ipython().system\",\n    \"get_ipython().getoutput\",\n    \"get_ipython().run_line_magic\",\n))\nTOKENS_TO_IGNORE = frozenset((\n    \"ENDMARKER\",\n    \"NL\",\n    \"NEWLINE\",\n    \"COMMENT\",\n    \"DEDENT\",\n    \"UNIMPORTANT_WS\",\n    \"ESCAPED_NL\",\n))\nPYTHON_CELL_MAGICS = frozenset((\n    \"capture\",\n    \"prun\",\n    \"pypy\",\n    \"python\",\n    \"python3\",\n    \"time\",\n    \"timeit\",\n))\nTOKEN_HEX = secrets.token_hex\n\n\n@dataclasses.dataclass(frozen=True)\nclass Replacement:\n    mask: str\n    src: str\n\n\n@lru_cache\ndef jupyter_dependencies_are_installed(*, warn: bool) -> bool:\n    installed = (\n        find_spec(\"tokenize_rt\") is not None and find_spec(\"IPython\") is not None\n    )\n    if not installed and warn:\n        msg = (\n            \"Skipping .ipynb files as Jupyter dependencies are not installed.\\n\"\n            'You can fix this by running ``pip install \"black[jupyter]\"``'\n        )\n        out(msg)\n    return installed\n\n\ndef remove_trailing_semicolon(src: str) -> Tuple[str, bool]:\n    \"\"\"Remove trailing semicolon from Jupyter notebook cell.\n\n    For example,\n\n        fig, ax = plt.subplots()\n        ax.plot(x_data, y_data);  # plot data\n\n    would become\n\n        fig, ax = plt.subplots()\n        ax.plot(x_data, y_data)  # plot data\n\n    Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses\n    ``tokenize_rt`` so that round-tripping works fine.\n    \"\"\"\n    from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src\n\n    tokens = src_to_tokens(src)\n    trailing_semicolon = False\n    for idx, token in reversed_enumerate(tokens):\n        if token.name in TOKENS_TO_IGNORE:\n            continue\n        if token.name == \"OP\" and token.src == \";\":\n            del tokens[idx]\n            trailing_semicolon = True\n        break\n    if not trailing_semicolon:\n        return src, False\n    return tokens_to_src(tokens), True\n\n\ndef put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str:\n    \"\"\"Put trailing semicolon back if cell originally had it.\n\n    Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses\n    ``tokenize_rt`` so that round-tripping works fine.\n    \"\"\"\n    if not has_trailing_semicolon:\n        return src\n    from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src\n\n    tokens = src_to_tokens(src)\n    for idx, token in reversed_enumerate(tokens):\n        if token.name in TOKENS_TO_IGNORE:\n            continue\n        tokens[idx] = token._replace(src=token.src + \";\")\n        break\n    else:  # pragma: nocover\n        raise AssertionError(\n            \"INTERNAL ERROR: Was not able to reinstate trailing semicolon. \"\n            \"Please report a bug on https://github.com/psf/black/issues.  \"\n        ) from None\n    return str(tokens_to_src(tokens))\n\n\ndef mask_cell(src: str) -> Tuple[str, List[Replacement]]:\n    \"\"\"Mask IPython magics so content becomes parseable Python code.\n\n    For example,\n\n        %matplotlib inline\n        'foo'\n\n    becomes\n\n        \"25716f358c32750e\"\n...\n", "needle_token_start": 7162, "needle_token_end": 7209, "code_context_ntokens": 16062, "output": ["```python\ndef is_part_of_annotation(leaf: Leaf) -> bool:\n    \"\"\"Returns whether this leaf is part of a type annotation.\"\"\"\n    ancestor = leaf.parent\n    while ancestor is not None:\n        if ancestor.prev_sibling and ancestor.prev_sibling.type == token.RARROW:\n            return \"return\"\n        if ancestor.parent and ancestor.parent.type == syms.tname:\n            return \"param\"\n        ancestor = ancestor.parent\n    return None\n```"]}
{"repo": "psf/black", "name": "maybe_install_uvloop", "language": "python", "path": "src/black/concurrency.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To enhance the event loop performance by optionally integrating a faster alternative if available in the environment.\n2. **Input**: None.\n3. **Output**: None, but the system's default event loop may be replaced.\n4. **Procedure**: The function attempts to import a specific high-performance event loop implementation. If the import is successful, it sets this implementation as the default for the entire application. If the import fails, the function does nothing, leaving the default event loop unchanged.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/black/nodes.py\n\"\"\"\nblib2to3 Node/Leaf transformation-related utility functions.\n\"\"\"\n\nimport sys\nfrom typing import (\n    Final,\n    Generic,\n    Iterator,\n    List,\n    Literal,\n    Optional,\n    Set,\n    Tuple,\n    TypeVar,\n    Union,\n)\n\nif sys.version_info >= (3, 10):\n    from typing import TypeGuard\nelse:\n    from typing_extensions import TypeGuard\n\nfrom mypy_extensions import mypyc_attr\n\nfrom black.cache import CACHE_DIR\nfrom black.mode import Mode, Preview\nfrom black.strings import get_string_prefix, has_triple_quotes\nfrom blib2to3 import pygram\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import NL, Leaf, Node, type_repr\n\npygram.initialize(CACHE_DIR)\nsyms: Final = pygram.python_symbols\n\n\n# types\nT = TypeVar(\"T\")\nLN = Union[Leaf, Node]\nLeafID = int\nNodeType = int\n\n\nWHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE}\nSTATEMENT: Final = {\n    syms.if_stmt,\n    syms.while_stmt,\n    syms.for_stmt,\n    syms.try_stmt,\n    syms.except_clause,\n    syms.with_stmt,\n    syms.funcdef,\n    syms.classdef,\n    syms.match_stmt,\n    syms.case_block,\n}\nSTANDALONE_COMMENT: Final = 153\ntoken.tok_name[STANDALONE_COMMENT] = \"STANDALONE_COMMENT\"\nLOGIC_OPERATORS: Final = {\"and\", \"or\"}\nCOMPARATORS: Final = {\n    token.LESS,\n    token.GREATER,\n    token.EQEQUAL,\n    token.NOTEQUAL,\n    token.LESSEQUAL,\n    token.GREATEREQUAL,\n}\nMATH_OPERATORS: Final = {\n    token.VBAR,\n    token.CIRCUMFLEX,\n    token.AMPER,\n    token.LEFTSHIFT,\n    token.RIGHTSHIFT,\n    token.PLUS,\n    token.MINUS,\n    token.STAR,\n    token.SLASH,\n    token.DOUBLESLASH,\n    token.PERCENT,\n    token.AT,\n    token.TILDE,\n    token.DOUBLESTAR,\n}\nSTARS: Final = {token.STAR, token.DOUBLESTAR}\nVARARGS_SPECIALS: Final = STARS | {token.SLASH}\nVARARGS_PARENTS: Final = {\n    syms.arglist,\n    syms.argument,  # double star in arglist\n    syms.trailer,  # single argument to call\n    syms.typedargslist,\n    syms.varargslist,  # lambdas\n}\nUNPACKING_PARENTS: Final = {\n    syms.atom,  # single element of a list or set literal\n    syms.dictsetmaker,\n    syms.listmaker,\n    syms.testlist_gexp,\n    syms.testlist_star_expr,\n    syms.subject_expr,\n    syms.pattern,\n}\nTEST_DESCENDANTS: Final = {\n    syms.test,\n    syms.lambdef,\n    syms.or_test,\n    syms.and_test,\n    syms.not_test,\n    syms.comparison,\n    syms.star_expr,\n    syms.expr,\n    syms.xor_expr,\n    syms.and_expr,\n    syms.shift_expr,\n    syms.arith_expr,\n    syms.trailer,\n    syms.term,\n    syms.power,\n    syms.namedexpr_test,\n}\nTYPED_NAMES: Final = {syms.tname, syms.tname_star}\nASSIGNMENTS: Final = {\n    \"=\",\n    \"+=\",\n    \"-=\",\n    \"*=\",\n    \"@=\",\n    \"/=\",\n    \"%=\",\n    \"&=\",\n    \"|=\",\n    \"^=\",\n    \"<<=\",\n    \">>=\",\n    \"**=\",\n    \"//=\",\n    \":\",\n}\n\nIMPLICIT_TUPLE: Final = {syms.testlist, syms.testlist_star_expr, syms.exprlist}\nBRACKET: Final = {\n    token.LPAR: token.RPAR,\n    token.LSQB: token.RSQB,\n    token.LBRACE: token.RBRACE,\n}\nOPENING_BRACKETS: Final = set(BRACKET.keys())\nCLOSING_BRACKETS: Final = set(BRACKET.values())\nBRACKETS: Final = OPENING_BRACKETS | CLOSING_BRACKETS\nALWAYS_NO_SPACE: Final = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}\n\nRARROW = 55\n\n\n@mypyc_attr(allow_interpreted_subclasses=True)\nclass Visitor(Generic[T]):\n    \"\"\"Basic lib2to3 visitor that yields things of type `T` on `visit()`.\"\"\"\n\n    def visit(self, node: LN) -> Iterator[T]:\n        \"\"\"Main method to visit `node` and its children.\n\n        It tries to find a `visit_*()` method for the given `node.type`, like\n        `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.\n        If no dedicated `visit_*()` method is found, chooses `visit_default()`\n        instead.\n\n        Then yields objects of type `T` from the selected visitor.\n        \"\"\"\n        if node.type < 256:\n            name = token.tok_name[node.type]\n        else:\n            name = str(type_repr(node.type))\n        # We explicitly branch on whether a visitor exists (instead of\n        # using self.visit_default as the default arg to getattr) in order\n        # to save needing to create a bound method object and so mypyc can\n        # generate a native call to visit_default.\n        visitf = getattr(self, f\"visit_{name}\", None)\n        if visitf:\n            yield from visitf(node)\n        else:\n            yield from self.visit_default(node)\n\n    def visit_default(self, node: LN) -> Iterator[T]:\n        \"\"\"Default `visit_*()` implementation. Recurses to children of `node`.\"\"\"\n        if isinstance(node, Node):\n            for child in node.children:\n                yield from self.visit(child)\n\n\ndef whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str:  # noqa: C901\n    \"\"\"Return whitespace prefix if needed for the given `leaf`.\n\n    `complex_subscript` signals whether the given leaf is part of a subscription\n    which has non-trivial arguments, like arithmetic expressions or function calls.\n    \"\"\"\n    NO: Final[str] = \"\"\n    SPACE: Final[str] = \" \"\n    DOUBLESPACE: Final[str] = \"  \"\n    t = leaf.type\n    p = leaf.parent\n    v = leaf.value\n    if t in ALWAYS_NO_SPACE:\n        return NO\n\n    if t == token.COMMENT:\n        return DOUBLESPACE\n\n    assert p is not None, f\"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}\"\n    if t == token.COLON and p.type not in {\n        syms.subscript,\n        syms.subscriptlist,\n        syms.sliceop,\n    }:\n        return NO\n\n    prev = leaf.prev_sibling\n    if not prev:\n        prevp = preceding_leaf(p)\n        if not prevp or prevp.type in OPENING_BRACKETS:\n            return NO\n\n        if t == token.COLON:\n            if prevp.type == token.COLON:\n                return NO\n\n            elif prevp.type != token.COMMA and not complex_subscript:\n                return NO\n\n            return SPACE\n\n        if prevp.type == token.EQUAL:\n            if prevp.parent:\n                if prevp.parent.type in {\n                    syms.arglist,\n                    syms.argument,\n                    syms.parameters,\n                    syms.varargslist,\n                }:\n                    return NO\n\n                elif prevp.parent.type == syms.typedargslist:\n                    # A bit hacky: if the equal sign has whitespace, it means we\n                    # previously found it's a typed argument.  So, we're using\n                    # that, too.\n                    return prevp.prefix\n\n        elif (\n            prevp.type == token.STAR\n            and parent_type(prevp) == syms.star_expr\n            and parent_type(prevp.parent) == syms.subscriptlist\n        ):\n            # No space between typevar tuples.\n            return NO\n\n        elif prevp.type in VARARGS_SPECIALS:\n            if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):\n                return NO\n\n        elif prevp.type == token.COLON:\n            if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:\n                return SPACE if complex_subscript else NO\n\n        elif (\n            prevp.parent\n            and prevp.parent.type == syms.factor\n            and prevp.type in MATH_OPERATORS\n        ):\n            return NO\n\n        elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator:\n            # no space in decorators\n            return NO\n\n    elif prev.type in OPENING_BRACKETS:\n        return NO\n\n    if p.type in {syms.parameters, syms.arglist}:\n        # untyped function signatures or calls\n        if not prev or prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.varargslist:\n        # lambdas\n        if prev and prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.typedargslist:\n        # typed function signatures\n        if not prev:\n            return NO\n\n        if t == token.EQUAL:\n            if prev.type not in TYPED_NAMES:\n                return NO\n\n        elif prev.type == token.EQUAL:\n            # A bit hacky: if the equal sign has whitespace, it means we\n            # previously found it's a typed argument.  So, we're using that, too.\n            return prev.prefix\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type in TYPED_NAMES:\n        # type names\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type != token.COMMA:\n                return NO\n\n    elif p.type == syms.trailer:\n        # attributes and calls\n        if t == token.LPAR or t == token.RPAR:\n            return NO\n\n        if not prev:\n            if t == token.DOT or t == token.LSQB:\n                return NO\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.argument:\n        # single argument\n        if t == token.EQUAL:\n            return NO\n\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type == token.LPAR:\n                return NO\n\n        elif prev.type in {token.EQUAL} | VARARGS_SPECIALS:\n            return NO\n\n    elif p.type == syms.decorator:\n        # decorators\n        return NO\n\n    elif p.type == syms.dotted_name:\n        if prev:\n            return NO\n...\n# Path: src/black/comments.py\nimport re\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nfrom typing import Collection, Final, Iterator, List, Optional, Tuple, Union\n\nfrom black.mode import Mode, Preview\nfrom black.nodes import (\n    CLOSING_BRACKETS,\n    STANDALONE_COMMENT,\n    WHITESPACE,\n    container_of,\n    first_leaf_of,\n    make_simple_prefix,\n    preceding_leaf,\n    syms,\n)\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\n# types\nLN = Union[Leaf, Node]\n\nFMT_OFF: Final = {\"# fmt: off\", \"# fmt:off\", \"# yapf: disable\"}\nFMT_SKIP: Final = {\"# fmt: skip\", \"# fmt:skip\"}\nFMT_ON: Final = {\"# fmt: on\", \"# fmt:on\", \"# yapf: enable\"}\n\nCOMMENT_EXCEPTIONS = \" !:#'\"\n_COMMENT_PREFIX = \"# \"\n_COMMENT_LIST_SEPARATOR = \";\"\n\n\n@dataclass\nclass ProtoComment:\n    \"\"\"Describes a piece of syntax that is a comment.\n\n    It's not a :class:`blib2to3.pytree.Leaf` so that:\n\n    * it can be cached (`Leaf` objects should not be reused more than once as\n      they store their lineno, column, prefix, and parent information);\n    * `newlines` and `consumed` fields are kept separate from the `value`. This\n      simplifies handling of special marker comments like ``# fmt: off/on``.\n    \"\"\"\n\n    type: int  # token.COMMENT or STANDALONE_COMMENT\n    value: str  # content of the comment\n    newlines: int  # how many newlines before the comment\n    consumed: int  # how many characters of the original leaf's prefix did we consume\n    form_feed: bool  # is there a form feed before the comment\n    leading_whitespace: str  # leading whitespace before the comment, if any\n\n\ndef generate_comments(leaf: LN) -> Iterator[Leaf]:\n    \"\"\"Clean the prefix of the `leaf` and generate comments from it, if any.\n\n    Comments in lib2to3 are shoved into the whitespace prefix.  This happens\n    in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation\n    move because it does away with modifying the grammar to include all the\n    possible places in which comments can be placed.\n\n    The sad consequence for us though is that comments don't \"belong\" anywhere.\n    This is why this function generates simple parentless Leaf objects for\n    comments.  We simply don't know what the correct parent should be.\n\n    No matter though, we can live without this.  We really only need to\n    differentiate between inline and standalone comments.  The latter don't\n    share the line with any code.\n\n    Inline comments are emitted as regular token.COMMENT leaves.  Standalone\n    are emitted with a fake STANDALONE_COMMENT token identifier.\n    \"\"\"\n    total_consumed = 0\n    for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER):\n        total_consumed = pc.consumed\n        prefix = make_simple_prefix(pc.newlines, pc.form_feed)\n        yield Leaf(pc.type, pc.value, prefix=prefix)\n    normalize_trailing_prefix(leaf, total_consumed)\n\n\n@lru_cache(maxsize=4096)\ndef list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:\n    \"\"\"Return a list of :class:`ProtoComment` objects parsed from the given `prefix`.\"\"\"\n    result: List[ProtoComment] = []\n    if not prefix or \"#\" not in prefix:\n        return result\n\n    consumed = 0\n    nlines = 0\n    ignored_lines = 0\n    form_feed = False\n    for index, full_line in enumerate(re.split(\"\\r?\\n\", prefix)):\n        consumed += len(full_line) + 1  # adding the length of the split '\\n'\n        match = re.match(r\"^(\\s*)(\\S.*|)$\", full_line)\n        assert match\n        whitespace, line = match.groups()\n        if not line:\n            nlines += 1\n            if \"\\f\" in full_line:\n                form_feed = True\n        if not line.startswith(\"#\"):\n            # Escaped newlines outside of a comment are not really newlines at\n            # all. We treat a single-line comment following an escaped newline\n            # as a simple trailing comment.\n            if line.endswith(\"\\\\\"):\n                ignored_lines += 1\n            continue\n\n        if index == ignored_lines and not is_endmarker:\n            comment_type = token.COMMENT  # simple trailing comment\n        else:\n            comment_type = STANDALONE_COMMENT\n        comment = make_comment(line)\n        result.append(\n            ProtoComment(\n                type=comment_type,\n                value=comment,\n                newlines=nlines,\n                consumed=consumed,\n                form_feed=form_feed,\n                leading_whitespace=whitespace,\n            )\n        )\n        form_feed = False\n        nlines = 0\n    return result\n\n\ndef normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None:\n    \"\"\"Normalize the prefix that's left over after generating comments.\n\n    Note: don't use backslashes for formatting or you'll lose your voting rights.\n    \"\"\"\n    remainder = leaf.prefix[total_consumed:]\n    if \"\\\\\" not in remainder:\n        nl_count = remainder.count(\"\\n\")\n        form_feed = \"\\f\" in remainder and remainder.endswith(\"\\n\")\n        leaf.prefix = make_simple_prefix(nl_count, form_feed)\n        return\n\n    leaf.prefix = \"\"\n\n\ndef make_comment(content: str) -> str:\n    \"\"\"Return a consistently formatted comment from the given `content` string.\n\n    All comments (except for \"##\", \"#!\", \"#:\", '#'\") should have a single\n    space between the hash sign and the content.\n\n    If `content` didn't start with a hash sign, one is provided.\n    \"\"\"\n    content = content.rstrip()\n    if not content:\n        return \"#\"\n\n    if content[0] == \"#\":\n        content = content[1:]\n    NON_BREAKING_SPACE = \"\u00a0\"\n    if (\n        content\n        and content[0] == NON_BREAKING_SPACE\n        and not content.lstrip().startswith(\"type:\")\n    ):\n        content = \" \" + content[1:]  # Replace NBSP by a simple space\n    if content and content[0] not in COMMENT_EXCEPTIONS:\n        content = \" \" + content\n    return \"#\" + content\n\n\ndef normalize_fmt_off(\n    node: Node, mode: Mode, lines: Collection[Tuple[int, int]]\n) -> None:\n    \"\"\"Convert content between `# fmt: off`/`# fmt: on` into standalone comments.\"\"\"\n    try_again = True\n    while try_again:\n        try_again = convert_one_fmt_off_pair(node, mode, lines)\n\n\ndef convert_one_fmt_off_pair(\n    node: Node, mode: Mode, lines: Collection[Tuple[int, int]]\n) -> bool:\n    \"\"\"Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.\n\n    Returns True if a pair was converted.\n    \"\"\"\n    for leaf in node.leaves():\n        previous_consumed = 0\n        for comment in list_comments(leaf.prefix, is_endmarker=False):\n            should_pass_fmt = comment.value in FMT_OFF or _contains_fmt_skip_comment(\n                comment.value, mode\n            )\n            if not should_pass_fmt:\n                previous_consumed = comment.consumed\n                continue\n            # We only want standalone comments. If there's no previous leaf or\n            # the previous leaf is indentation, it's a standalone comment in\n            # disguise.\n            if should_pass_fmt and comment.type != STANDALONE_COMMENT:\n                prev = preceding_leaf(leaf)\n                if prev:\n                    if comment.value in FMT_OFF and prev.type not in WHITESPACE:\n                        continue\n                    if (\n                        _contains_fmt_skip_comment(comment.value, mode)\n                        and prev.type in WHITESPACE\n                    ):\n                        continue\n\n            ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode))\n            if not ignored_nodes:\n                continue\n\n            first = ignored_nodes[0]  # Can be a container node with the `leaf`.\n            parent = first.parent\n            prefix = first.prefix\n            if comment.value in FMT_OFF:\n                first.prefix = prefix[comment.consumed :]\n            if _contains_fmt_skip_comment(comment.value, mode):\n                first.prefix = \"\"\n                standalone_comment_prefix = prefix\n            else:\n                standalone_comment_prefix = (\n                    prefix[:previous_consumed] + \"\\n\" * comment.newlines\n                )\n            hidden_value = \"\".join(str(n) for n in ignored_nodes)\n            comment_lineno = leaf.lineno - comment.newlines\n            if comment.value in FMT_OFF:\n                fmt_off_prefix = \"\"\n                if len(lines) > 0 and not any(\n                    line[0] <= comment_lineno <= line[1] for line in lines\n                ):\n                    # keeping indentation of comment by preserving original whitespaces.\n                    fmt_off_prefix = prefix.split(comment.value)[0]\n                    if \"\\n\" in fmt_off_prefix:\n                        fmt_off_prefix = fmt_off_prefix.split(\"\\n\")[-1]\n                standalone_comment_prefix += fmt_off_prefix\n                hidden_value = comment.value + \"\\n\" + hidden_value\n            if _contains_fmt_skip_comment(comment.value, mode):\n                hidden_value += (\n                    comment.leading_whitespace\n                    if Preview.no_normalize_fmt_skip_whitespace in mode\n                    else \"  \"\n                ) + comment.value\n            if hidden_value.endswith(\"\\n\"):\n                # That happens when one of the `ignored_nodes` ended with a NEWLINE\n                # leaf (possibly followed by a DEDENT).\n                hidden_value = hidden_value[:-1]\n            first_idx: Optional[int] = None\n            for ignored in ignored_nodes:\n                index = ignored.remove()\n                if first_idx is None:\n                    first_idx = index\n            assert parent is not None, \"INTERNAL ERROR: fmt: on/off handling (1)\"\n            assert first_idx is not None, \"INTERNAL ERROR: fmt: on/off handling (2)\"\n            parent.insert_child(\n                first_idx,\n                Leaf(\n                    STANDALONE_COMMENT,\n                    hidden_value,\n                    prefix=standalone_comment_prefix,\n                    fmt_pass_converted_first_leaf=first_leaf_of(first),\n                ),\n            )\n            return True\n\n    return False\n\n\ndef generate_ignored_nodes(\n    leaf: Leaf, comment: ProtoComment, mode: Mode\n) -> Iterator[LN]:\n    \"\"\"Starting from the container of `leaf`, generate all leaves until `# fmt: on`.\n\n    If comment is skip, returns leaf only.\n    Stops at the end of the block.\n    \"\"\"\n    if _contains_fmt_skip_comment(comment.value, mode):\n        yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment)\n        return\n    container: Optional[LN] = container_of(leaf)\n    while container is not None and container.type != token.ENDMARKER:\n        if is_fmt_on(container):\n            return\n\n        # fix for fmt: on in children\n        if children_contains_fmt_on(container):\n            for index, child in enumerate(container.children):\n                if isinstance(child, Leaf) and is_fmt_on(child):\n                    if child.type in CLOSING_BRACKETS:\n                        # This means `# fmt: on` is placed at a different bracket level\n                        # than `# fmt: off`. This is an invalid use, but as a courtesy,\n                        # we include this closing bracket in the ignored nodes.\n                        # The alternative is to fail the formatting.\n                        yield child\n                    return\n                if (\n                    child.type == token.INDENT\n                    and index < len(container.children) - 1\n                    and children_contains_fmt_on(container.children[index + 1])\n                ):\n                    # This means `# fmt: on` is placed right after an indentation\n                    # level, and we shouldn't swallow the previous INDENT token.\n                    return\n                if children_contains_fmt_on(child):\n                    return\n                yield child\n        else:\n            if container.type == token.DEDENT and container.next_sibling is None:\n                # This can happen when there is no matching `# fmt: on` comment at the\n                # same level as `# fmt: on`. We need to keep this DEDENT.\n                return\n            yield container\n            container = container.next_sibling\n\n\ndef _generate_ignored_nodes_from_fmt_skip(\n    leaf: Leaf, comment: ProtoComment\n) -> Iterator[LN]:\n    \"\"\"Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`.\"\"\"\n    prev_sibling = leaf.prev_sibling\n    parent = leaf.parent\n    # Need to properly format the leaf prefix to compare it to comment.value,\n    # which is also formatted\n    comments = list_comments(leaf.prefix, is_endmarker=False)\n    if not comments or comment.value != comments[0].value:\n        return\n    if prev_sibling is not None:\n        leaf.prefix = \"\"\n        siblings = [prev_sibling]\n        while \"\\n\" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None:\n            prev_sibling = prev_sibling.prev_sibling\n            siblings.insert(0, prev_sibling)\n        yield from siblings\n    elif (\n        parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE\n    ):\n        # The `# fmt: skip` is on the colon line of the if/while/def/class/...\n        # statements. The ignored nodes should be previous siblings of the\n        # parent suite node.\n        leaf.prefix = \"\"\n        ignored_nodes: List[LN] = []\n        parent_sibling = parent.prev_sibling\n        while parent_sibling is not None and parent_sibling.type != syms.suite:\n            ignored_nodes.insert(0, parent_sibling)\n            parent_sibling = parent_sibling.prev_sibling\n        # Special case for `async_stmt` where the ASYNC token is on the\n        # grandparent node.\n        grandparent = parent.parent\n        if (\n            grandparent is not None\n            and grandparent.prev_sibling is not None\n            and grandparent.prev_sibling.type == token.ASYNC\n        ):\n            ignored_nodes.insert(0, grandparent.prev_sibling)\n        yield from iter(ignored_nodes)\n\n\ndef is_fmt_on(container: LN) -> bool:\n    \"\"\"Determine whether formatting is switched on within a container.\n    Determined by whether the last `# fmt:` comment is `on` or `off`.\n    \"\"\"\n    fmt_on = False\n    for comment in list_comments(container.prefix, is_endmarker=False):\n        if comment.value in FMT_ON:\n            fmt_on = True\n        elif comment.value in FMT_OFF:\n            fmt_on = False\n    return fmt_on\n\n\ndef children_contains_fmt_on(container: LN) -> bool:\n    \"\"\"Determine if children have formatting switched on.\"\"\"\n    for child in container.children:\n        leaf = first_leaf_of(child)\n        if leaf is not None and is_fmt_on(leaf):\n            return True\n\n    return False\n\n\ndef contains_pragma_comment(comment_list: List[Leaf]) -> bool:\n    \"\"\"\n    Returns:\n        True iff one of the comments in @comment_list is a pragma used by one\n        of the more common static analysis tools for python (e.g. mypy, flake8,\n        pylint).\n    \"\"\"\n    for comment in comment_list:\n        if comment.value.startswith((\"# type:\", \"# noqa\", \"# pylint:\")):\n            return True\n\n    return False\n\n\ndef _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool:\n    \"\"\"\n    Checks if the given comment contains FMT_SKIP alone or paired with other comments.\n    Matching styles:\n      # fmt:skip                           <-- single comment\n      # noqa:XXX # fmt:skip # a nice line  <-- multiple comments (Preview)\n      # pylint:XXX; fmt:skip               <-- list of comments (; separated, Preview)\n    \"\"\"\n    semantic_comment_blocks = [\n        comment_line,\n        *[\n            _COMMENT_PREFIX + comment.strip()\n            for comment in comment_line.split(_COMMENT_PREFIX)[1:]\n        ],\n        *[\n            _COMMENT_PREFIX + comment.strip()\n            for comment in comment_line.strip(_COMMENT_PREFIX).split(\n                _COMMENT_LIST_SEPARATOR\n            )\n        ],\n    ]\n\n    return any(comment in FMT_SKIP for comment in semantic_comment_blocks)\n\n# Path: src/black/report.py\n\"\"\"\nSummarize Black runs to users.\n\"\"\"\n\nfrom dataclasses import dataclass\nfrom enum import Enum\nfrom pathlib import Path\n\nfrom click import style\n\nfrom black.output import err, out\n\n\nclass Changed(Enum):\n    NO = 0\n    CACHED = 1\n    YES = 2\n\n\nclass NothingChanged(UserWarning):\n    \"\"\"Raised when reformatted code is the same as source.\"\"\"\n\n\n@dataclass\nclass Report:\n    \"\"\"Provides a reformatting counter. Can be rendered with `str(report)`.\"\"\"\n\n    check: bool = False\n    diff: bool = False\n    quiet: bool = False\n    verbose: bool = False\n    change_count: int = 0\n    same_count: int = 0\n    failure_count: int = 0\n\n    def done(self, src: Path, changed: Changed) -> None:\n        \"\"\"Increment the counter for successful reformatting. Write out a message.\"\"\"\n        if changed is Changed.YES:\n            reformatted = \"would reformat\" if self.check or self.diff else \"reformatted\"\n            if self.verbose or not self.quiet:\n                out(f\"{reformatted} {src}\")\n            self.change_count += 1\n        else:\n            if self.verbose:\n                if changed is Changed.NO:\n                    msg = f\"{src} already well formatted, good job.\"\n                else:\n                    msg = f\"{src} wasn't modified on disk since last run.\"\n                out(msg, bold=False)\n            self.same_count += 1\n\n    def failed(self, src: Path, message: str) -> None:\n        \"\"\"Increment the counter for failed reformatting. Write out a message.\"\"\"\n        err(f\"error: cannot format {src}: {message}\")\n        self.failure_count += 1\n\n    def path_ignored(self, path: Path, message: str) -> None:\n        if self.verbose:\n            out(f\"{path} ignored: {message}\", bold=False)\n\n    @property\n    def return_code(self) -> int:\n        \"\"\"Return the exit code that the app should use.\n\n        This considers the current state of changed files and failures:\n        - if there were any failures, return 123;\n        - if any files were changed and --check is being used, return 1;\n        - otherwise return 0.\n        \"\"\"\n        # According to http://tldp.org/LDP/abs/html/exitcodes.html starting with\n        # 126 we have special return codes reserved by the shell.\n        if self.failure_count:\n            return 123\n\n        elif self.change_count and self.check:\n            return 1\n\n        return 0\n\n    def __str__(self) -> str:\n        \"\"\"Render a color report of the current state.\n\n        Use `click.unstyle` to remove colors.\n        \"\"\"\n        if self.check or self.diff:\n            reformatted = \"would be reformatted\"\n            unchanged = \"would be left unchanged\"\n            failed = \"would fail to reformat\"\n        else:\n            reformatted = \"reformatted\"\n            unchanged = \"left unchanged\"\n            failed = \"failed to reformat\"\n        report = []\n        if self.change_count:\n            s = \"s\" if self.change_count > 1 else \"\"\n            report.append(\n                style(f\"{self.change_count} file{s} \", bold=True, fg=\"blue\")\n                + style(f\"{reformatted}\", bold=True)\n            )\n\n        if self.same_count:\n            s = \"s\" if self.same_count > 1 else \"\"\n            report.append(style(f\"{self.same_count} file{s} \", fg=\"blue\") + unchanged)\n        if self.failure_count:\n            s = \"s\" if self.failure_count > 1 else \"\"\n            report.append(style(f\"{self.failure_count} file{s} {failed}\", fg=\"red\"))\n        return \", \".join(report) + \".\"\n\n# Path: src/black/concurrency.py\n\"\"\"\nFormatting many files at once via multiprocessing. Contains entrypoint and utilities.\n\nNOTE: this module is only imported if we need to format several files at once.\n\"\"\"\n\nimport asyncio\nimport logging\nimport os\nimport signal\nimport sys\nimport traceback\nfrom concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor\nfrom multiprocessing import Manager\nfrom pathlib import Path\nfrom typing import Any, Iterable, Optional, Set\n\nfrom mypy_extensions import mypyc_attr\n\nfrom black import WriteBack, format_file_in_place\nfrom black.cache import Cache\nfrom black.mode import Mode\nfrom black.output import err\nfrom black.report import Changed, Report\n\n\n\ndef maybe_install_uvloop() -> None:\n    \"\"\"If our environment has uvloop installed we use it.\n\n    This is called only from command-line entry points to avoid\n    interfering with the parent process if Black is used as a library.\n    \"\"\"\n    try:\n        import uvloop\n\n        uvloop.install()\n    except ImportError:\n        pass\n\n\ndef cancel(tasks: Iterable[\"asyncio.Future[Any]\"]) -> None:\n    \"\"\"asyncio signal handler that cancels all `tasks` and reports to stderr.\"\"\"\n    err(\"Aborted!\")\n    for task in tasks:\n        task.cancel()\n\n\ndef shutdown(loop: asyncio.AbstractEventLoop) -> None:\n    \"\"\"Cancel all pending tasks on `loop`, wait for them, and close the loop.\"\"\"\n    try:\n        # This part is borrowed from asyncio/runners.py in Python 3.7b2.\n        to_cancel = [task for task in asyncio.all_tasks(loop) if not task.done()]\n        if not to_cancel:\n            return\n\n        for task in to_cancel:\n            task.cancel()\n        loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True))\n    finally:\n        # `concurrent.futures.Future` objects cannot be cancelled once they\n        # are already running. There might be some when the `shutdown()` happened.\n        # Silence their logger's spew about the event loop being closed.\n        cf_logger = logging.getLogger(\"concurrent.futures\")\n        cf_logger.setLevel(logging.CRITICAL)\n        loop.close()\n\n\n# diff-shades depends on being to monkeypatch this function to operate. I know it's\n# not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26\n@mypyc_attr(patchable=True)\ndef reformat_many(\n    sources: Set[Path],\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: Report,\n    workers: Optional[int],\n) -> None:\n    \"\"\"Reformat multiple files using a ProcessPoolExecutor.\"\"\"\n    maybe_install_uvloop()\n\n    executor: Executor\n    if workers is None:\n        workers = int(os.environ.get(\"BLACK_NUM_WORKERS\", 0))\n        workers = workers or os.cpu_count() or 1\n    if sys.platform == \"win32\":\n        # Work around https://bugs.python.org/issue26903\n        workers = min(workers, 60)\n    try:\n        executor = ProcessPoolExecutor(max_workers=workers)\n    except (ImportError, NotImplementedError, OSError):\n        # we arrive here if the underlying system does not support multi-processing\n        # like in AWS Lambda or Termux, in which case we gracefully fallback to\n        # a ThreadPoolExecutor with just a single worker (more workers would not do us\n        # any good due to the Global Interpreter Lock)\n        executor = ThreadPoolExecutor(max_workers=1)\n\n    loop = asyncio.new_event_loop()\n    asyncio.set_event_loop(loop)\n    try:\n        loop.run_until_complete(\n            schedule_formatting(\n                sources=sources,\n                fast=fast,\n                write_back=write_back,\n                mode=mode,\n                report=report,\n                loop=loop,\n                executor=executor,\n            )\n        )\n    finally:\n        try:\n            shutdown(loop)\n        finally:\n            asyncio.set_event_loop(None)\n        if executor is not None:\n            executor.shutdown()\n\n\nasync def schedule_formatting(\n    sources: Set[Path],\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: \"Report\",\n    loop: asyncio.AbstractEventLoop,\n    executor: \"Executor\",\n) -> None:\n    \"\"\"Run formatting of `sources` in parallel using the provided `executor`.\n\n    (Use ProcessPoolExecutors for actual parallelism.)\n\n    `write_back`, `fast`, and `mode` options are passed to\n    :func:`format_file_in_place`.\n    \"\"\"\n    cache = Cache.read(mode)\n    if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n        sources, cached = cache.filtered_cached(sources)\n        for src in sorted(cached):\n            report.done(src, Changed.CACHED)\n    if not sources:\n        return\n\n    cancelled = []\n    sources_to_cache = []\n    lock = None\n    if write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n        # For diff output, we need locks to ensure we don't interleave output\n        # from different processes.\n        manager = Manager()\n        lock = manager.Lock()\n    tasks = {\n        asyncio.ensure_future(\n            loop.run_in_executor(\n                executor, format_file_in_place, src, fast, mode, write_back, lock\n            )\n        ): src\n        for src in sorted(sources)\n    }\n    pending = tasks.keys()\n    try:\n        loop.add_signal_handler(signal.SIGINT, cancel, pending)\n        loop.add_signal_handler(signal.SIGTERM, cancel, pending)\n    except NotImplementedError:\n        # There are no good alternatives for these on Windows.\n        pass\n    while pending:\n        done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)\n        for task in done:\n            src = tasks.pop(task)\n            if task.cancelled():\n                cancelled.append(task)\n            elif exc := task.exception():\n                if report.verbose:\n                    traceback.print_exception(type(exc), exc, exc.__traceback__)\n                report.failed(src, str(exc))\n            else:\n                changed = Changed.YES if task.result() else Changed.NO\n                # If the file was written back or was successfully checked as\n                # well-formatted, store this information in the cache.\n                if write_back is WriteBack.YES or (\n                    write_back is WriteBack.CHECK and changed is Changed.NO\n                ):\n                    sources_to_cache.append(src)\n                report.done(src, changed)\n    if cancelled:\n        await asyncio.gather(*cancelled, return_exceptions=True)\n    if sources_to_cache:\n        cache.write(sources_to_cache)\n\n# Path: src/black/handle_ipynb_magics.py\n\"\"\"Functions to process IPython magics with.\"\"\"\n\nimport ast\nimport collections\nimport dataclasses\nimport secrets\nimport sys\nfrom functools import lru_cache\nfrom importlib.util import find_spec\nfrom typing import Dict, List, Optional, Tuple\n\nif sys.version_info >= (3, 10):\n    from typing import TypeGuard\nelse:\n    from typing_extensions import TypeGuard\n\nfrom black.output import out\nfrom black.report import NothingChanged\n\nTRANSFORMED_MAGICS = frozenset((\n    \"get_ipython().run_cell_magic\",\n    \"get_ipython().system\",\n    \"get_ipython().getoutput\",\n    \"get_ipython().run_line_magic\",\n))\nTOKENS_TO_IGNORE = frozenset((\n    \"ENDMARKER\",\n    \"NL\",\n    \"NEWLINE\",\n    \"COMMENT\",\n    \"DEDENT\",\n    \"UNIMPORTANT_WS\",\n    \"ESCAPED_NL\",\n))\nPYTHON_CELL_MAGICS = frozenset((\n    \"capture\",\n    \"prun\",\n    \"pypy\",\n    \"python\",\n    \"python3\",\n    \"time\",\n    \"timeit\",\n))\nTOKEN_HEX = secrets.token_hex\n\n\n@dataclasses.dataclass(frozen=True)\nclass Replacement:\n    mask: str\n    src: str\n\n\n@lru_cache\ndef jupyter_dependencies_are_installed(*, warn: bool) -> bool:\n    installed = (\n        find_spec(\"tokenize_rt\") is not None and find_spec(\"IPython\") is not None\n    )\n    if not installed and warn:\n        msg = (\n            \"Skipping .ipynb files as Jupyter dependencies are not installed.\\n\"\n            'You can fix this by running ``pip install \"black[jupyter]\"``'\n        )\n        out(msg)\n    return installed\n\n\ndef remove_trailing_semicolon(src: str) -> Tuple[str, bool]:\n    \"\"\"Remove trailing semicolon from Jupyter notebook cell.\n\n    For example,\n\n        fig, ax = plt.subplots()\n        ax.plot(x_data, y_data);  # plot data\n\n    would become\n\n        fig, ax = plt.subplots()\n        ax.plot(x_data, y_data)  # plot data\n\n    Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses\n    ``tokenize_rt`` so that round-tripping works fine.\n    \"\"\"\n    from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src\n\n    tokens = src_to_tokens(src)\n    trailing_semicolon = False\n    for idx, token in reversed_enumerate(tokens):\n        if token.name in TOKENS_TO_IGNORE:\n            continue\n        if token.name == \"OP\" and token.src == \";\":\n            del tokens[idx]\n            trailing_semicolon = True\n        break\n    if not trailing_semicolon:\n        return src, False\n    return tokens_to_src(tokens), True\n\n\ndef put_trailing_semicolon_back(src: str, has_trailing_semicolon: bool) -> str:\n    \"\"\"Put trailing semicolon back if cell originally had it.\n\n    Mirrors the logic in `quiet` from `IPython.core.displayhook`, but uses\n    ``tokenize_rt`` so that round-tripping works fine.\n    \"\"\"\n    if not has_trailing_semicolon:\n        return src\n    from tokenize_rt import reversed_enumerate, src_to_tokens, tokens_to_src\n\n    tokens = src_to_tokens(src)\n    for idx, token in reversed_enumerate(tokens):\n        if token.name in TOKENS_TO_IGNORE:\n            continue\n        tokens[idx] = token._replace(src=token.src + \";\")\n        break\n    else:  # pragma: nocover\n        raise AssertionError(\n            \"INTERNAL ERROR: Was not able to reinstate trailing semicolon. \"\n            \"Please report a bug on https://github.com/psf/black/issues.  \"\n        ) from None\n    return str(tokens_to_src(tokens))\n\n\ndef mask_cell(src: str) -> Tuple[str, List[Replacement]]:\n    \"\"\"Mask IPython magics so content becomes parseable Python code.\n\n    For example,\n\n        %matplotlib inline\n        'foo'\n\n    becomes\n\n        \"25716f358c32750e\"\n        'foo'\n\n    The replacements are returned, along with the transformed code.\n    \"\"\"\n    replacements: List[Replacement] = []\n    try:\n        ast.parse(src)\n    except SyntaxError:\n        # Might have IPython magics, will process below.\n        pass\n    else:\n        # Syntax is fine, nothing to mask, early return.\n        return src, replacements\n\n    from IPython.core.inputtransformer2 import TransformerManager\n\n    transformer_manager = TransformerManager()\n    transformed = transformer_manager.transform_cell(src)\n    transformed, cell_magic_replacements = replace_cell_magics(transformed)\n    replacements += cell_magic_replacements\n    transformed = transformer_manager.transform_cell(transformed)\n    transformed, magic_replacements = replace_magics(transformed)\n    if len(transformed.splitlines()) != len(src.splitlines()):\n        # Multi-line magic, not supported.\n        raise NothingChanged\n    replacements += magic_replacements\n    return transformed, replacements\n\n\ndef get_token(src: str, magic: str) -> str:\n    \"\"\"Return randomly generated token to mask IPython magic with.\n\n    For example, if 'magic' was `%matplotlib inline`, then a possible\n    token to mask it with would be `\"43fdd17f7e5ddc83\"`. The token\n    will be the same length as the magic, and we make sure that it was\n    not already present anywhere else in the cell.\n    \"\"\"\n    assert magic\n    nbytes = max(len(magic) // 2 - 1, 1)\n    token = TOKEN_HEX(nbytes)\n    counter = 0\n    while token in src:\n        token = TOKEN_HEX(nbytes)\n        counter += 1\n        if counter > 100:\n            raise AssertionError(\n                \"INTERNAL ERROR: Black was not able to replace IPython magic. \"\n                \"Please report a bug on https://github.com/psf/black/issues.  \"\n                f\"The magic might be helpful: {magic}\"\n            ) from None\n    if len(token) + 2 < len(magic):\n        token = f\"{token}.\"\n    return f'\"{token}\"'\n\n\ndef replace_cell_magics(src: str) -> Tuple[str, List[Replacement]]:\n    \"\"\"Replace cell magic with token.\n\n    Note that 'src' will already have been processed by IPython's\n    TransformerManager().transform_cell.\n\n    Example,\n\n        get_ipython().run_cell_magic('t', '-n1', 'ls =!ls\\\\n')\n\n    becomes\n\n        \"a794.\"\n        ls =!ls\n\n    The replacement, along with the transformed code, is returned.\n    \"\"\"\n    replacements: List[Replacement] = []\n\n    tree = ast.parse(src)\n\n    cell_magic_finder = CellMagicFinder()\n    cell_magic_finder.visit(tree)\n    if cell_magic_finder.cell_magic is None:\n        return src, replacements\n    header = cell_magic_finder.cell_magic.header\n    mask = get_token(src, header)\n    replacements.append(Replacement(mask=mask, src=header))\n    return f\"{mask}\\n{cell_magic_finder.cell_magic.body}\", replacements\n\n\ndef replace_magics(src: str) -> Tuple[str, List[Replacement]]:\n    \"\"\"Replace magics within body of cell.\n\n    Note that 'src' will already have been processed by IPython's\n    TransformerManager().transform_cell.\n\n    Example, this\n\n        get_ipython().run_line_magic('matplotlib', 'inline')\n        'foo'\n\n    becomes\n\n        \"5e67db56d490fd39\"\n        'foo'\n\n    The replacement, along with the transformed code, are returned.\n    \"\"\"\n    replacements = []\n    magic_finder = MagicFinder()\n    magic_finder.visit(ast.parse(src))\n    new_srcs = []\n    for i, line in enumerate(src.splitlines(), start=1):\n        if i in magic_finder.magics:\n            offsets_and_magics = magic_finder.magics[i]\n            if len(offsets_and_magics) != 1:  # pragma: nocover\n                raise AssertionError(\n                    f\"Expecting one magic per line, got: {offsets_and_magics}\\n\"\n                    \"Please report a bug on https://github.com/psf/black/issues.\"\n                )\n            col_offset, magic = (\n                offsets_and_magics[0].col_offset,\n                offsets_and_magics[0].magic,\n            )\n            mask = get_token(src, magic)\n            replacements.append(Replacement(mask=mask, src=magic))\n            line = line[:col_offset] + mask\n        new_srcs.append(line)\n    return \"\\n\".join(new_srcs), replacements\n\n\ndef unmask_cell(src: str, replacements: List[Replacement]) -> str:\n    \"\"\"Remove replacements from cell.\n\n    For example\n\n        \"9b20\"\n        foo = bar\n\n    becomes\n\n        %%time\n        foo = bar\n    \"\"\"\n    for replacement in replacements:\n        src = src.replace(replacement.mask, replacement.src)\n    return src\n\n\ndef _is_ipython_magic(node: ast.expr) -> TypeGuard[ast.Attribute]:\n    \"\"\"Check if attribute is IPython magic.\n\n    Note that the source of the abstract syntax tree\n    will already have been processed by IPython's\n    TransformerManager().transform_cell.\n    \"\"\"\n    return (\n        isinstance(node, ast.Attribute)\n        and isinstance(node.value, ast.Call)\n        and isinstance(node.value.func, ast.Name)\n        and node.value.func.id == \"get_ipython\"\n    )\n\n\ndef _get_str_args(args: List[ast.expr]) -> List[str]:\n    str_args = []\n    for arg in args:\n        assert isinstance(arg, ast.Str)\n        str_args.append(arg.s)\n    return str_args\n\n\n@dataclasses.dataclass(frozen=True)\nclass CellMagic:\n    name: str\n    params: Optional[str]\n    body: str\n\n    @property\n    def header(self) -> str:\n        if self.params:\n            return f\"%%{self.name} {self.params}\"\n        return f\"%%{self.name}\"\n\n\n# ast.NodeVisitor + dataclass = breakage under mypyc.\nclass CellMagicFinder(ast.NodeVisitor):\n    \"\"\"Find cell magics.\n\n    Note that the source of the abstract syntax tree\n    will already have been processed by IPython's\n    TransformerManager().transform_cell.\n\n    For example,\n\n        %%time\\n\n        foo()\n\n    would have been transformed to\n\n        get_ipython().run_cell_magic('time', '', 'foo()\\\\n')\n\n    and we look for instances of the latter.\n    \"\"\"\n\n    def __init__(self, cell_magic: Optional[CellMagic] = None) -> None:\n        self.cell_magic = cell_magic\n\n    def visit_Expr(self, node: ast.Expr) -> None:\n        \"\"\"Find cell magic, extract header and body.\"\"\"\n        if (\n            isinstance(node.value, ast.Call)\n            and _is_ipython_magic(node.value.func)\n            and node.value.func.attr == \"run_cell_magic\"\n        ):\n            args = _get_str_args(node.value.args)\n            self.cell_magic = CellMagic(name=args[0], params=args[1], body=args[2])\n        self.generic_visit(node)\n\n\n@dataclasses.dataclass(frozen=True)\nclass OffsetAndMagic:\n    col_offset: int\n    magic: str\n\n\n# Unsurprisingly, subclassing ast.NodeVisitor means we can't use dataclasses here\n# as mypyc will generate broken code.\nclass MagicFinder(ast.NodeVisitor):\n    \"\"\"Visit cell to look for get_ipython calls.\n\n    Note that the source of the abstract syntax tree\n    will already have been processed by IPython's\n    TransformerManager().transform_cell.\n\n    For example,\n\n        %matplotlib inline\n\n    would have been transformed to\n\n        get_ipython().run_line_magic('matplotlib', 'inline')\n\n    and we look for instances of the latter (and likewise for other\n    types of magics).\n    \"\"\"\n\n    def __init__(self) -> None:\n        self.magics: Dict[int, List[OffsetAndMagic]] = collections.defaultdict(list)\n\n    def visit_Assign(self, node: ast.Assign) -> None:\n        \"\"\"Look for system assign magics.\n\n        For example,\n\n            black_version = !black --version\n            env = %env var\n\n        would have been (respectively) transformed to\n\n            black_version = get_ipython().getoutput('black --version')\n            env = get_ipython().run_line_magic('env', 'var')\n\n        and we look for instances of any of the latter.\n        \"\"\"\n        if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func):\n            args = _get_str_args(node.value.args)\n            if node.value.func.attr == \"getoutput\":\n                src = f\"!{args[0]}\"\n            elif node.value.func.attr == \"run_line_magic\":\n                src = f\"%{args[0]}\"\n                if args[1]:\n                    src += f\" {args[1]}\"\n            else:\n                raise AssertionError(\n                    f\"Unexpected IPython magic {node.value.func.attr!r} found. \"\n                    \"Please report a bug on https://github.com/psf/black/issues.\"\n                ) from None\n            self.magics[node.value.lineno].append(\n                OffsetAndMagic(node.value.col_offset, src)\n            )\n        self.generic_visit(node)\n\n    def visit_Expr(self, node: ast.Expr) -> None:\n        \"\"\"Look for magics in body of cell.\n\n        For examples,\n\n            !ls\n            !!ls\n            ?ls\n            ??ls\n\n        would (respectively) get transformed to\n\n            get_ipython().system('ls')\n            get_ipython().getoutput('ls')\n            get_ipython().run_line_magic('pinfo', 'ls')\n            get_ipython().run_line_magic('pinfo2', 'ls')\n\n        and we look for instances of any of the latter.\n        \"\"\"\n        if isinstance(node.value, ast.Call) and _is_ipython_magic(node.value.func):\n            args = _get_str_args(node.value.args)\n            if node.value.func.attr == \"run_line_magic\":\n                if args[0] == \"pinfo\":\n                    src = f\"?{args[1]}\"\n                elif args[0] == \"pinfo2\":\n                    src = f\"??{args[1]}\"\n                else:\n                    src = f\"%{args[0]}\"\n                    if args[1]:\n                        src += f\" {args[1]}\"\n            elif node.value.func.attr == \"system\":\n                src = f\"!{args[0]}\"\n            elif node.value.func.attr == \"getoutput\":\n                src = f\"!!{args[0]}\"\n            else:\n                raise NothingChanged  # unsupported magic.\n            self.magics[node.value.lineno].append(\n                OffsetAndMagic(node.value.col_offset, src)\n            )\n        self.generic_visit(node)\n\n# Path: src/black/files.py\nimport io\nimport os\nimport sys\nfrom functools import lru_cache\nfrom pathlib import Path\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Iterable,\n    Iterator,\n    List,\n    Optional,\n    Pattern,\n    Sequence,\n    Tuple,\n    Union,\n)\n\nfrom mypy_extensions import mypyc_attr\nfrom packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet\nfrom packaging.version import InvalidVersion, Version\nfrom pathspec import PathSpec\nfrom pathspec.patterns.gitwildmatch import GitWildMatchPatternError\n\nif sys.version_info >= (3, 11):\n    try:\n        import tomllib\n    except ImportError:\n        # Help users on older alphas\n        if not TYPE_CHECKING:\n            import tomli as tomllib\nelse:\n    import tomli as tomllib\n\nfrom black.handle_ipynb_magics import jupyter_dependencies_are_installed\nfrom black.mode import TargetVersion\nfrom black.output import err\nfrom black.report import Report\n\nif TYPE_CHECKING:\n    import colorama  # noqa: F401\n\n\n@lru_cache\ndef _load_toml(path: Union[Path, str]) -> Dict[str, Any]:\n    with open(path, \"rb\") as f:\n        return tomllib.load(f)\n\n\n@lru_cache\ndef _cached_resolve(path: Path) -> Path:\n    return path.resolve()\n\n\n@lru_cache\ndef find_project_root(\n    srcs: Sequence[str], stdin_filename: Optional[str] = None\n) -> Tuple[Path, str]:\n    \"\"\"Return a directory containing .git, .hg, or pyproject.toml.\n\n    That directory will be a common parent of all files and directories\n    passed in `srcs`.\n\n    If no directory in the tree contains a marker that would specify it's the\n    project root, the root of the file system is returned.\n\n    Returns a two-tuple with the first element as the project root path and\n    the second element as a string describing the method by which the\n    project root was discovered.\n    \"\"\"\n    if stdin_filename is not None:\n        srcs = tuple(stdin_filename if s == \"-\" else s for s in srcs)\n    if not srcs:\n        srcs = [str(_cached_resolve(Path.cwd()))]\n\n    path_srcs = [_cached_resolve(Path(Path.cwd(), src)) for src in srcs]\n\n    # A list of lists of parents for each 'src'. 'src' is included as a\n    # \"parent\" of itself if it is a directory\n    src_parents = [\n        list(path.parents) + ([path] if path.is_dir() else []) for path in path_srcs\n    ]\n\n    common_base = max(\n        set.intersection(*(set(parents) for parents in src_parents)),\n        key=lambda path: path.parts,\n    )\n\n    for directory in (common_base, *common_base.parents):\n        if (directory / \".git\").exists():\n            return directory, \".git directory\"\n\n        if (directory / \".hg\").is_dir():\n            return directory, \".hg directory\"\n\n        if (directory / \"pyproject.toml\").is_file():\n            pyproject_toml = _load_toml(directory / \"pyproject.toml\")\n            if \"black\" in pyproject_toml.get(\"tool\", {}):\n                return directory, \"pyproject.toml\"\n\n    return directory, \"file system root\"\n\n\ndef find_pyproject_toml(\n    path_search_start: Tuple[str, ...], stdin_filename: Optional[str] = None\n) -> Optional[str]:\n    \"\"\"Find the absolute filepath to a pyproject.toml if it exists\"\"\"\n    path_project_root, _ = find_project_root(path_search_start, stdin_filename)\n    path_pyproject_toml = path_project_root / \"pyproject.toml\"\n    if path_pyproject_toml.is_file():\n        return str(path_pyproject_toml)\n\n    try:\n        path_user_pyproject_toml = find_user_pyproject_toml()\n        return (\n            str(path_user_pyproject_toml)\n            if path_user_pyproject_toml.is_file()\n            else None\n        )\n    except (PermissionError, RuntimeError) as e:\n        # We do not have access to the user-level config directory, so ignore it.\n        err(f\"Ignoring user configuration directory due to {e!r}\")\n        return None\n\n\n@mypyc_attr(patchable=True)\ndef parse_pyproject_toml(path_config: str) -> Dict[str, Any]:\n    \"\"\"Parse a pyproject toml file, pulling out relevant parts for Black.\n\n    If parsing fails, will raise a tomllib.TOMLDecodeError.\n    \"\"\"\n    pyproject_toml = _load_toml(path_config)\n    config: Dict[str, Any] = pyproject_toml.get(\"tool\", {}).get(\"black\", {})\n    config = {k.replace(\"--\", \"\").replace(\"-\", \"_\"): v for k, v in config.items()}\n\n    if \"target_version\" not in config:\n...\n", "needle_token_start": 8833, "needle_token_end": 8922, "code_context_ntokens": 16105, "output": ["```python\ndef maybe_install_uvloop() -> None:\n    \"\"\"If our environment has uvloop installed we use it.\n\n    This is called only from command-line entry points to avoid\n    interfering with the parent process if Black is used as a library.\n    \"\"\"\n    try:\n        import uvloop\n\n        uvloop.install()\n    except ImportError:\n        pass\n```"]}
{"repo": "psf/black", "name": "check_stability_and_equivalence", "language": "python", "path": "src/black/__init__.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To ensure that the transformed content remains consistent after formatting and that the original and transformed contents are equivalent in terms of syntax and structure.\n2. **Input**: Two strings representing the original content and the transformed content, a configuration object specifying formatting rules, and an optional collection of line number ranges to focus the checks on specific parts of the content.\n3. **Output**: There is no return value; however, the function can raise an `AssertionError` if the transformed content does not meet the specified conditions.\n4. **Procedure**: The function first checks if the original and transformed contents are equivalent, ensuring no unintended changes have been introduced. It then verifies the stability of the transformation by checking if applying the transformation a second time results in the same output, ensuring idempotence of the formatting process.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/black/ranges.py\n\"\"\"Functions related to Black's formatting by line ranges feature.\"\"\"\n\nimport difflib\nfrom dataclasses import dataclass\nfrom typing import Collection, Iterator, List, Sequence, Set, Tuple, Union\n\nfrom black.nodes import (\n    LN,\n    STANDALONE_COMMENT,\n    Leaf,\n    Node,\n    Visitor,\n    first_leaf,\n    furthest_ancestor_with_last_leaf,\n    last_leaf,\n    syms,\n)\nfrom blib2to3.pgen2.token import ASYNC, NEWLINE\n\n\ndef parse_line_ranges(line_ranges: Sequence[str]) -> List[Tuple[int, int]]:\n    lines: List[Tuple[int, int]] = []\n    for lines_str in line_ranges:\n        parts = lines_str.split(\"-\")\n        if len(parts) != 2:\n            raise ValueError(\n                \"Incorrect --line-ranges format, expect 'START-END', found\"\n                f\" {lines_str!r}\"\n            )\n        try:\n            start = int(parts[0])\n            end = int(parts[1])\n        except ValueError:\n            raise ValueError(\n                \"Incorrect --line-ranges value, expect integer ranges, found\"\n                f\" {lines_str!r}\"\n            ) from None\n        else:\n            lines.append((start, end))\n    return lines\n\n\ndef is_valid_line_range(lines: Tuple[int, int]) -> bool:\n    \"\"\"Returns whether the line range is valid.\"\"\"\n    return not lines or lines[0] <= lines[1]\n\n\ndef adjusted_lines(\n    lines: Collection[Tuple[int, int]],\n    original_source: str,\n    modified_source: str,\n) -> List[Tuple[int, int]]:\n    \"\"\"Returns the adjusted line ranges based on edits from the original code.\n\n    This computes the new line ranges by diffing original_source and\n    modified_source, and adjust each range based on how the range overlaps with\n    the diffs.\n\n    Note the diff can contain lines outside of the original line ranges. This can\n    happen when the formatting has to be done in adjacent to maintain consistent\n    local results. For example:\n\n    1. def my_func(arg1, arg2,\n    2.             arg3,):\n    3.   pass\n\n    If it restricts to line 2-2, it can't simply reformat line 2, it also has\n    to reformat line 1:\n\n    1. def my_func(\n    2.     arg1,\n    3.     arg2,\n    4.     arg3,\n    5. ):\n    6.   pass\n\n    In this case, we will expand the line ranges to also include the whole diff\n    block.\n\n    Args:\n      lines: a collection of line ranges.\n      original_source: the original source.\n      modified_source: the modified source.\n    \"\"\"\n    lines_mappings = _calculate_lines_mappings(original_source, modified_source)\n\n    new_lines = []\n    # Keep an index of the current search. Since the lines and lines_mappings are\n    # sorted, this makes the search complexity linear.\n    current_mapping_index = 0\n    for start, end in sorted(lines):\n...\n# Path: src/black/__init__.py\nimport io\nimport json\nimport platform\nimport re\nimport sys\nimport tokenize\nimport traceback\nfrom contextlib import contextmanager\nfrom dataclasses import replace\nfrom datetime import datetime, timezone\nfrom enum import Enum\nfrom json.decoder import JSONDecodeError\nfrom pathlib import Path\nfrom typing import (\n    Any,\n    Collection,\n    Dict,\n    Generator,\n    Iterator,\n    List,\n    MutableMapping,\n    Optional,\n    Pattern,\n    Sequence,\n    Set,\n    Sized,\n    Tuple,\n    Union,\n)\n\nimport click\nfrom click.core import ParameterSource\nfrom mypy_extensions import mypyc_attr\nfrom pathspec import PathSpec\nfrom pathspec.patterns.gitwildmatch import GitWildMatchPatternError\n\nfrom _black_version import version as __version__\nfrom black.cache import Cache\nfrom black.comments import normalize_fmt_off\nfrom black.const import (\n    DEFAULT_EXCLUDES,\n    DEFAULT_INCLUDES,\n    DEFAULT_LINE_LENGTH,\n    STDIN_PLACEHOLDER,\n)\nfrom black.files import (\n    best_effort_relative_path,\n    find_project_root,\n    find_pyproject_toml,\n    find_user_pyproject_toml,\n    gen_python_files,\n    get_gitignore,\n    parse_pyproject_toml,\n    path_is_excluded,\n    resolves_outside_root_or_cannot_stat,\n    wrap_stream_for_windows,\n)\nfrom black.handle_ipynb_magics import (\n    PYTHON_CELL_MAGICS,\n    TRANSFORMED_MAGICS,\n    jupyter_dependencies_are_installed,\n    mask_cell,\n    put_trailing_semicolon_back,\n    remove_trailing_semicolon,\n    unmask_cell,\n)\nfrom black.linegen import LN, LineGenerator, transform_line\nfrom black.lines import EmptyLineTracker, LinesBlock\nfrom black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature\nfrom black.mode import Mode as Mode  # re-exported\nfrom black.mode import Preview, TargetVersion, supports_feature\nfrom black.nodes import (\n    STARS,\n    is_number_token,\n    is_simple_decorator_expression,\n    is_string_token,\n    syms,\n)\nfrom black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out\nfrom black.parsing import InvalidInput  # noqa F401\nfrom black.parsing import lib2to3_parse, parse_ast, stringify_ast\nfrom black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges\nfrom black.report import Changed, NothingChanged, Report\nfrom black.trans import iter_fexpr_spans\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\nCOMPILED = Path(__file__).suffix in (\".pyd\", \".so\")\n\n# types\nFileContent = str\nEncoding = str\nNewLine = str\n\n\nclass WriteBack(Enum):\n    NO = 0\n    YES = 1\n    DIFF = 2\n    CHECK = 3\n    COLOR_DIFF = 4\n\n    @classmethod\n    def from_configuration(\n        cls, *, check: bool, diff: bool, color: bool = False\n    ) -> \"WriteBack\":\n        if check and not diff:\n            return cls.CHECK\n\n        if diff and color:\n            return cls.COLOR_DIFF\n\n        return cls.DIFF if diff else cls.YES\n\n\n# Legacy name, left for integrations.\nFileMode = Mode\n\n\ndef read_pyproject_toml(\n    ctx: click.Context, param: click.Parameter, value: Optional[str]\n) -> Optional[str]:\n    \"\"\"Inject Black configuration from \"pyproject.toml\" into defaults in `ctx`.\n\n    Returns the path to a successfully found and read configuration file, None\n    otherwise.\n    \"\"\"\n    if not value:\n        value = find_pyproject_toml(\n            ctx.params.get(\"src\", ()), ctx.params.get(\"stdin_filename\", None)\n        )\n        if value is None:\n            return None\n\n    try:\n        config = parse_pyproject_toml(value)\n    except (OSError, ValueError) as e:\n        raise click.FileError(\n            filename=value, hint=f\"Error reading configuration file: {e}\"\n        ) from None\n\n    if not config:\n        return None\n    else:\n        spellcheck_pyproject_toml_keys(ctx, list(config), value)\n        # Sanitize the values to be Click friendly. For more information please see:\n        # https://github.com/psf/black/issues/1458\n        # https://github.com/pallets/click/issues/1567\n        config = {\n            k: str(v) if not isinstance(v, (list, dict)) else v\n            for k, v in config.items()\n        }\n\n    target_version = config.get(\"target_version\")\n    if target_version is not None and not isinstance(target_version, list):\n        raise click.BadOptionUsage(\n            \"target-version\", \"Config key target-version must be a list\"\n        )\n\n    exclude = config.get(\"exclude\")\n    if exclude is not None and not isinstance(exclude, str):\n        raise click.BadOptionUsage(\"exclude\", \"Config key exclude must be a string\")\n\n    extend_exclude = config.get(\"extend_exclude\")\n    if extend_exclude is not None and not isinstance(extend_exclude, str):\n        raise click.BadOptionUsage(\n            \"extend-exclude\", \"Config key extend-exclude must be a string\"\n        )\n\n    line_ranges = config.get(\"line_ranges\")\n    if line_ranges is not None:\n        raise click.BadOptionUsage(\n            \"line-ranges\", \"Cannot use line-ranges in the pyproject.toml file.\"\n        )\n\n    default_map: Dict[str, Any] = {}\n    if ctx.default_map:\n        default_map.update(ctx.default_map)\n    default_map.update(config)\n\n    ctx.default_map = default_map\n    return value\n\n\ndef spellcheck_pyproject_toml_keys(\n    ctx: click.Context, config_keys: List[str], config_file_path: str\n) -> None:\n    invalid_keys: List[str] = []\n    available_config_options = {param.name for param in ctx.command.params}\n    for key in config_keys:\n        if key not in available_config_options:\n            invalid_keys.append(key)\n    if invalid_keys:\n        keys_str = \", \".join(map(repr, invalid_keys))\n        out(\n            f\"Invalid config keys detected: {keys_str} (in {config_file_path})\",\n            fg=\"red\",\n        )\n\n\ndef target_version_option_callback(\n    c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]\n) -> List[TargetVersion]:\n    \"\"\"Compute the target versions from a --target-version flag.\n\n    This is its own function because mypy couldn't infer the type correctly\n    when it was a lambda, causing mypyc trouble.\n    \"\"\"\n    return [TargetVersion[val.upper()] for val in v]\n\n\ndef enable_unstable_feature_callback(\n    c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...]\n) -> List[Preview]:\n    \"\"\"Compute the features from an --enable-unstable-feature flag.\"\"\"\n    return [Preview[val] for val in v]\n\n\ndef re_compile_maybe_verbose(regex: str) -> Pattern[str]:\n    \"\"\"Compile a regular expression string in `regex`.\n\n    If it contains newlines, use verbose mode.\n    \"\"\"\n    if \"\\n\" in regex:\n        regex = \"(?x)\" + regex\n    compiled: Pattern[str] = re.compile(regex)\n    return compiled\n\n\ndef validate_regex(\n    ctx: click.Context,\n    param: click.Parameter,\n    value: Optional[str],\n) -> Optional[Pattern[str]]:\n    try:\n        return re_compile_maybe_verbose(value) if value is not None else None\n    except re.error as e:\n        raise click.BadParameter(f\"Not a valid regular expression: {e}\") from None\n\n\n@click.command(\n    context_settings={\"help_option_names\": [\"-h\", \"--help\"]},\n    # While Click does set this field automatically using the docstring, mypyc\n    # (annoyingly) strips 'em so we need to set it here too.\n    help=\"The uncompromising code formatter.\",\n)\n@click.option(\"-c\", \"--code\", type=str, help=\"Format the code passed in as a string.\")\n@click.option(\n    \"-l\",\n    \"--line-length\",\n    type=int,\n    default=DEFAULT_LINE_LENGTH,\n    help=\"How many characters per line to allow.\",\n    show_default=True,\n)\n@click.option(\n    \"-t\",\n    \"--target-version\",\n    type=click.Choice([v.name.lower() for v in TargetVersion]),\n    callback=target_version_option_callback,\n    multiple=True,\n    help=(\n        \"Python versions that should be supported by Black's output. You should\"\n        \" include all versions that your code supports. By default, Black will infer\"\n        \" target versions from the project metadata in pyproject.toml. If this does\"\n        \" not yield conclusive results, Black will use per-file auto-detection.\"\n    ),\n)\n@click.option(\n    \"--pyi\",\n    is_flag=True,\n    help=(\n        \"Format all input files like typing stubs regardless of file extension. This\"\n        \" is useful when piping source on standard input.\"\n    ),\n)\n@click.option(\n    \"--ipynb\",\n    is_flag=True,\n    help=(\n        \"Format all input files like Jupyter Notebooks regardless of file extension.\"\n        \" This is useful when piping source on standard input.\"\n    ),\n)\n@click.option(\n    \"--python-cell-magics\",\n    multiple=True,\n    help=(\n        \"When processing Jupyter Notebooks, add the given magic to the list\"\n        f\" of known python-magics ({', '.join(sorted(PYTHON_CELL_MAGICS))}).\"\n        \" Useful for formatting cells with custom python magics.\"\n    ),\n    default=[],\n)\n@click.option(\n    \"-x\",\n    \"--skip-source-first-line\",\n    is_flag=True,\n    help=\"Skip the first line of the source code.\",\n)\n@click.option(\n    \"-S\",\n    \"--skip-string-normalization\",\n    is_flag=True,\n    help=\"Don't normalize string quotes or prefixes.\",\n)\n@click.option(\n    \"-C\",\n    \"--skip-magic-trailing-comma\",\n    is_flag=True,\n    help=\"Don't use trailing commas as a reason to split lines.\",\n)\n@click.option(\n    \"--preview\",\n    is_flag=True,\n    help=(\n        \"Enable potentially disruptive style changes that may be added to Black's main\"\n        \" functionality in the next major release.\"\n    ),\n)\n@click.option(\n    \"--unstable\",\n    is_flag=True,\n    help=(\n        \"Enable potentially disruptive style changes that have known bugs or are not\"\n        \" currently expected to make it into the stable style Black's next major\"\n        \" release. Implies --preview.\"\n    ),\n)\n@click.option(\n    \"--enable-unstable-feature\",\n    type=click.Choice([v.name for v in Preview]),\n    callback=enable_unstable_feature_callback,\n    multiple=True,\n    help=(\n        \"Enable specific features included in the `--unstable` style. Requires\"\n        \" `--preview`. No compatibility guarantees are provided on the behavior\"\n        \" or existence of any unstable features.\"\n    ),\n)\n@click.option(\n    \"--check\",\n    is_flag=True,\n    help=(\n        \"Don't write the files back, just return the status. Return code 0 means\"\n        \" nothing would change. Return code 1 means some files would be reformatted.\"\n        \" Return code 123 means there was an internal error.\"\n    ),\n)\n@click.option(\n    \"--diff\",\n    is_flag=True,\n    help=(\n        \"Don't write the files back, just output a diff to indicate what changes\"\n        \" Black would've made. They are printed to stdout so capturing them is simple.\"\n    ),\n)\n@click.option(\n    \"--color/--no-color\",\n    is_flag=True,\n    help=\"Show (or do not show) colored diff. Only applies when --diff is given.\",\n)\n@click.option(\n    \"--line-ranges\",\n    multiple=True,\n    metavar=\"START-END\",\n    help=(\n        \"When specified, Black will try its best to only format these lines. This\"\n        \" option can be specified multiple times, and a union of the lines will be\"\n        \" formatted. Each range must be specified as two integers connected by a `-`:\"\n        \" `<START>-<END>`. The `<START>` and `<END>` integer indices are 1-based and\"\n        \" inclusive on both ends.\"\n    ),\n    default=(),\n)\n@click.option(\n    \"--fast/--safe\",\n    is_flag=True,\n    help=(\n        \"By default, Black performs an AST safety check after formatting your code.\"\n        \" The --fast flag turns off this check and the --safe flag explicitly enables\"\n        \" it. [default: --safe]\"\n    ),\n)\n@click.option(\n    \"--required-version\",\n    type=str,\n    help=(\n        \"Require a specific version of Black to be running. This is useful for\"\n        \" ensuring that all contributors to your project are using the same\"\n        \" version, because different versions of Black may format code a little\"\n        \" differently. This option can be set in a configuration file for consistent\"\n        \" results across environments.\"\n    ),\n)\n@click.option(\n    \"--exclude\",\n    type=str,\n    callback=validate_regex,\n    help=(\n        \"A regular expression that matches files and directories that should be\"\n        \" excluded on recursive searches. An empty value means no paths are excluded.\"\n        \" Use forward slashes for directories on all platforms (Windows, too).\"\n        \" By default, Black also ignores all paths listed in .gitignore. Changing this\"\n        f\" value will override all default exclusions. [default: {DEFAULT_EXCLUDES}]\"\n    ),\n    show_default=False,\n)\n@click.option(\n    \"--extend-exclude\",\n    type=str,\n    callback=validate_regex,\n    help=(\n        \"Like --exclude, but adds additional files and directories on top of the\"\n        \" default values instead of overriding them.\"\n    ),\n)\n@click.option(\n    \"--force-exclude\",\n    type=str,\n    callback=validate_regex,\n    help=(\n        \"Like --exclude, but files and directories matching this regex will be excluded\"\n        \" even when they are passed explicitly as arguments. This is useful when\"\n        \" invoking Black programmatically on changed files, such as in a pre-commit\"\n        \" hook or editor plugin.\"\n    ),\n)\n@click.option(\n    \"--stdin-filename\",\n    type=str,\n    is_eager=True,\n    help=(\n        \"The name of the file when passing it through stdin. Useful to make sure Black\"\n        \" will respect the --force-exclude option on some editors that rely on using\"\n        \" stdin.\"\n    ),\n)\n@click.option(\n    \"--include\",\n    type=str,\n    default=DEFAULT_INCLUDES,\n    callback=validate_regex,\n    help=(\n        \"A regular expression that matches files and directories that should be\"\n        \" included on recursive searches. An empty value means all files are included\"\n        \" regardless of the name. Use forward slashes for directories on all platforms\"\n        \" (Windows, too). Overrides all exclusions, including from .gitignore and\"\n        \" command line options.\"\n    ),\n    show_default=True,\n)\n@click.option(\n    \"-W\",\n    \"--workers\",\n    type=click.IntRange(min=1),\n    default=None,\n    help=(\n        \"When Black formats multiple files, it may use a process pool to speed up\"\n        \" formatting. This option controls the number of parallel workers. This can\"\n        \" also be specified via the BLACK_NUM_WORKERS environment variable. Defaults\"\n        \" to the number of CPUs in the system.\"\n    ),\n)\n@click.option(\n    \"-q\",\n    \"--quiet\",\n    is_flag=True,\n    help=(\n        \"Stop emitting all non-critical output. Error messages will still be emitted\"\n        \" (which can silenced by 2>/dev/null).\"\n    ),\n)\n@click.option(\n    \"-v\",\n    \"--verbose\",\n    is_flag=True,\n    help=(\n        \"Emit messages about files that were not changed or were ignored due to\"\n        \" exclusion patterns. If Black is using a configuration file, a message\"\n        \" detailing which one it is using will be emitted.\"\n    ),\n)\n@click.version_option(\n    version=__version__,\n    message=(\n        f\"%(prog)s, %(version)s (compiled: {'yes' if COMPILED else 'no'})\\n\"\n        f\"Python ({platform.python_implementation()}) {platform.python_version()}\"\n    ),\n)\n@click.argument(\n    \"src\",\n    nargs=-1,\n    type=click.Path(\n        exists=True, file_okay=True, dir_okay=True, readable=True, allow_dash=True\n    ),\n    is_eager=True,\n    metavar=\"SRC ...\",\n)\n@click.option(\n    \"--config\",\n    type=click.Path(\n        exists=True,\n        file_okay=True,\n        dir_okay=False,\n        readable=True,\n        allow_dash=False,\n        path_type=str,\n    ),\n    is_eager=True,\n    callback=read_pyproject_toml,\n    help=\"Read configuration options from a configuration file.\",\n)\n@click.pass_context\ndef main(  # noqa: C901\n    ctx: click.Context,\n    code: Optional[str],\n    line_length: int,\n    target_version: List[TargetVersion],\n    check: bool,\n    diff: bool,\n    line_ranges: Sequence[str],\n    color: bool,\n    fast: bool,\n    pyi: bool,\n    ipynb: bool,\n    python_cell_magics: Sequence[str],\n    skip_source_first_line: bool,\n    skip_string_normalization: bool,\n    skip_magic_trailing_comma: bool,\n    preview: bool,\n    unstable: bool,\n    enable_unstable_feature: List[Preview],\n    quiet: bool,\n    verbose: bool,\n    required_version: Optional[str],\n    include: Pattern[str],\n    exclude: Optional[Pattern[str]],\n    extend_exclude: Optional[Pattern[str]],\n    force_exclude: Optional[Pattern[str]],\n    stdin_filename: Optional[str],\n    workers: Optional[int],\n    src: Tuple[str, ...],\n    config: Optional[str],\n) -> None:\n    \"\"\"The uncompromising code formatter.\"\"\"\n    ctx.ensure_object(dict)\n\n    if src and code is not None:\n        out(\n            main.get_usage(ctx)\n            + \"\\n\\n'SRC' and 'code' cannot be passed simultaneously.\"\n        )\n        ctx.exit(1)\n    if not src and code is None:\n        out(main.get_usage(ctx) + \"\\n\\nOne of 'SRC' or 'code' is required.\")\n        ctx.exit(1)\n\n    # It doesn't do anything if --unstable is also passed, so just allow it.\n    if enable_unstable_feature and not (preview or unstable):\n        out(\n            main.get_usage(ctx)\n            + \"\\n\\n'--enable-unstable-feature' requires '--preview'.\"\n        )\n        ctx.exit(1)\n\n    root, method = (\n        find_project_root(src, stdin_filename) if code is None else (None, None)\n    )\n    ctx.obj[\"root\"] = root\n\n    if verbose:\n        if root:\n            out(\n                f\"Identified `{root}` as project root containing a {method}.\",\n                fg=\"blue\",\n            )\n\n        if config:\n            config_source = ctx.get_parameter_source(\"config\")\n            user_level_config = str(find_user_pyproject_toml())\n            if config == user_level_config:\n                out(\n                    \"Using configuration from user-level config at \"\n                    f\"'{user_level_config}'.\",\n                    fg=\"blue\",\n                )\n            elif config_source in (\n                ParameterSource.DEFAULT,\n                ParameterSource.DEFAULT_MAP,\n            ):\n                out(\"Using configuration from project root.\", fg=\"blue\")\n            else:\n                out(f\"Using configuration in '{config}'.\", fg=\"blue\")\n            if ctx.default_map:\n                for param, value in ctx.default_map.items():\n                    out(f\"{param}: {value}\")\n\n    error_msg = \"Oh no! \ud83d\udca5 \ud83d\udc94 \ud83d\udca5\"\n    if (\n        required_version\n        and required_version != __version__\n        and required_version != __version__.split(\".\")[0]\n    ):\n        err(\n            f\"{error_msg} The required version `{required_version}` does not match\"\n            f\" the running version `{__version__}`!\"\n        )\n        ctx.exit(1)\n    if ipynb and pyi:\n        err(\"Cannot pass both `pyi` and `ipynb` flags!\")\n        ctx.exit(1)\n\n    write_back = WriteBack.from_configuration(check=check, diff=diff, color=color)\n    if target_version:\n        versions = set(target_version)\n    else:\n        # We'll autodetect later.\n        versions = set()\n    mode = Mode(\n        target_versions=versions,\n        line_length=line_length,\n        is_pyi=pyi,\n        is_ipynb=ipynb,\n        skip_source_first_line=skip_source_first_line,\n        string_normalization=not skip_string_normalization,\n        magic_trailing_comma=not skip_magic_trailing_comma,\n        preview=preview,\n        unstable=unstable,\n        python_cell_magics=set(python_cell_magics),\n        enabled_features=set(enable_unstable_feature),\n    )\n\n    lines: List[Tuple[int, int]] = []\n    if line_ranges:\n        if ipynb:\n            err(\"Cannot use --line-ranges with ipynb files.\")\n            ctx.exit(1)\n\n        try:\n            lines = parse_line_ranges(line_ranges)\n        except ValueError as e:\n            err(str(e))\n            ctx.exit(1)\n\n    if code is not None:\n        # Run in quiet mode by default with -c; the extra output isn't useful.\n        # You can still pass -v to get verbose output.\n        quiet = True\n\n    report = Report(check=check, diff=diff, quiet=quiet, verbose=verbose)\n\n    if code is not None:\n        reformat_code(\n            content=code,\n            fast=fast,\n            write_back=write_back,\n            mode=mode,\n            report=report,\n            lines=lines,\n        )\n    else:\n        assert root is not None  # root is only None if code is not None\n        try:\n            sources = get_sources(\n                root=root,\n                src=src,\n                quiet=quiet,\n                verbose=verbose,\n                include=include,\n                exclude=exclude,\n                extend_exclude=extend_exclude,\n                force_exclude=force_exclude,\n                report=report,\n                stdin_filename=stdin_filename,\n            )\n        except GitWildMatchPatternError:\n            ctx.exit(1)\n\n        path_empty(\n            sources,\n            \"No Python files are present to be formatted. Nothing to do \ud83d\ude34\",\n            quiet,\n            verbose,\n            ctx,\n        )\n\n        if len(sources) == 1:\n            reformat_one(\n                src=sources.pop(),\n                fast=fast,\n                write_back=write_back,\n                mode=mode,\n                report=report,\n                lines=lines,\n            )\n        else:\n            from black.concurrency import reformat_many\n\n            if lines:\n                err(\"Cannot use --line-ranges to format multiple files.\")\n                ctx.exit(1)\n            reformat_many(\n                sources=sources,\n                fast=fast,\n                write_back=write_back,\n                mode=mode,\n                report=report,\n                workers=workers,\n            )\n\n    if verbose or not quiet:\n        if code is None and (verbose or report.change_count or report.failure_count):\n            out()\n        out(error_msg if report.return_code else \"All done! \u2728 \ud83c\udf70 \u2728\")\n        if code is None:\n            click.echo(str(report), err=True)\n    ctx.exit(report.return_code)\n\n\ndef get_sources(\n    *,\n    root: Path,\n    src: Tuple[str, ...],\n    quiet: bool,\n    verbose: bool,\n    include: Pattern[str],\n    exclude: Optional[Pattern[str]],\n    extend_exclude: Optional[Pattern[str]],\n    force_exclude: Optional[Pattern[str]],\n    report: \"Report\",\n    stdin_filename: Optional[str],\n) -> Set[Path]:\n    \"\"\"Compute the set of files to be formatted.\"\"\"\n    sources: Set[Path] = set()\n\n    assert root.is_absolute(), f\"INTERNAL ERROR: `root` must be absolute but is {root}\"\n    using_default_exclude = exclude is None\n    exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude\n    gitignore: Optional[Dict[Path, PathSpec]] = None\n    root_gitignore = get_gitignore(root)\n\n    for s in src:\n        if s == \"-\" and stdin_filename:\n            path = Path(stdin_filename)\n            is_stdin = True\n        else:\n            path = Path(s)\n            is_stdin = False\n\n        # Compare the logic here to the logic in `gen_python_files`.\n        if is_stdin or path.is_file():\n            if resolves_outside_root_or_cannot_stat(path, root, report):\n                if verbose:\n                    out(f'Skipping invalid source: \"{path}\"', fg=\"red\")\n                continue\n\n            root_relative_path = best_effort_relative_path(path, root).as_posix()\n            root_relative_path = \"/\" + root_relative_path\n\n            # Hard-exclude any files that matches the `--force-exclude` regex.\n            if path_is_excluded(root_relative_path, force_exclude):\n                report.path_ignored(\n                    path, \"matches the --force-exclude regular expression\"\n                )\n                continue\n\n            if is_stdin:\n                path = Path(f\"{STDIN_PLACEHOLDER}{str(path)}\")\n\n            if path.suffix == \".ipynb\" and not jupyter_dependencies_are_installed(\n                warn=verbose or not quiet\n            ):\n                continue\n\n            if verbose:\n                out(f'Found input source: \"{path}\"', fg=\"blue\")\n            sources.add(path)\n        elif path.is_dir():\n            path = root / (path.resolve().relative_to(root))\n            if verbose:\n                out(f'Found input source directory: \"{path}\"', fg=\"blue\")\n\n            if using_default_exclude:\n                gitignore = {\n                    root: root_gitignore,\n                    path: get_gitignore(path),\n                }\n            sources.update(\n                gen_python_files(\n                    path.iterdir(),\n                    root,\n                    include,\n                    exclude,\n                    extend_exclude,\n                    force_exclude,\n                    report,\n                    gitignore,\n                    verbose=verbose,\n                    quiet=quiet,\n                )\n            )\n        elif s == \"-\":\n            if verbose:\n                out(\"Found input source stdin\", fg=\"blue\")\n            sources.add(path)\n        else:\n            err(f\"invalid path: {s}\")\n\n    return sources\n\n\ndef path_empty(\n    src: Sized, msg: str, quiet: bool, verbose: bool, ctx: click.Context\n) -> None:\n    \"\"\"\n    Exit if there is no `src` provided for formatting\n    \"\"\"\n    if not src:\n        if verbose or not quiet:\n            out(msg)\n        ctx.exit(0)\n\n\ndef reformat_code(\n    content: str,\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: Report,\n    *,\n    lines: Collection[Tuple[int, int]] = (),\n) -> None:\n    \"\"\"\n    Reformat and print out `content` without spawning child processes.\n    Similar to `reformat_one`, but for string content.\n\n    `fast`, `write_back`, and `mode` options are passed to\n    :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.\n    \"\"\"\n    path = Path(\"<string>\")\n    try:\n        changed = Changed.NO\n        if format_stdin_to_stdout(\n            content=content, fast=fast, write_back=write_back, mode=mode, lines=lines\n        ):\n            changed = Changed.YES\n        report.done(path, changed)\n    except Exception as exc:\n        if report.verbose:\n            traceback.print_exc()\n        report.failed(path, str(exc))\n\n\n# diff-shades depends on being to monkeypatch this function to operate. I know it's\n# not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26\n@mypyc_attr(patchable=True)\ndef reformat_one(\n    src: Path,\n    fast: bool,\n    write_back: WriteBack,\n    mode: Mode,\n    report: \"Report\",\n    *,\n    lines: Collection[Tuple[int, int]] = (),\n) -> None:\n    \"\"\"Reformat a single file under `src` without spawning child processes.\n\n    `fast`, `write_back`, and `mode` options are passed to\n    :func:`format_file_in_place` or :func:`format_stdin_to_stdout`.\n    \"\"\"\n    try:\n        changed = Changed.NO\n\n        if str(src) == \"-\":\n            is_stdin = True\n        elif str(src).startswith(STDIN_PLACEHOLDER):\n            is_stdin = True\n            # Use the original name again in case we want to print something\n            # to the user\n            src = Path(str(src)[len(STDIN_PLACEHOLDER) :])\n        else:\n            is_stdin = False\n\n        if is_stdin:\n            if src.suffix == \".pyi\":\n                mode = replace(mode, is_pyi=True)\n            elif src.suffix == \".ipynb\":\n                mode = replace(mode, is_ipynb=True)\n            if format_stdin_to_stdout(\n                fast=fast, write_back=write_back, mode=mode, lines=lines\n            ):\n                changed = Changed.YES\n        else:\n            cache = Cache.read(mode)\n            if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n                if not cache.is_changed(src):\n                    changed = Changed.CACHED\n            if changed is not Changed.CACHED and format_file_in_place(\n                src, fast=fast, write_back=write_back, mode=mode, lines=lines\n            ):\n                changed = Changed.YES\n            if (write_back is WriteBack.YES and changed is not Changed.CACHED) or (\n                write_back is WriteBack.CHECK and changed is Changed.NO\n            ):\n                cache.write([src])\n        report.done(src, changed)\n    except Exception as exc:\n        if report.verbose:\n            traceback.print_exc()\n        report.failed(src, str(exc))\n\n\ndef format_file_in_place(\n    src: Path,\n    fast: bool,\n    mode: Mode,\n    write_back: WriteBack = WriteBack.NO,\n    lock: Any = None,  # multiprocessing.Manager().Lock() is some crazy proxy\n    *,\n    lines: Collection[Tuple[int, int]] = (),\n) -> bool:\n    \"\"\"Format file under `src` path. Return True if changed.\n\n    If `write_back` is DIFF, write a diff to stdout. If it is YES, write reformatted\n    code to the file.\n    `mode` and `fast` options are passed to :func:`format_file_contents`.\n    \"\"\"\n    if src.suffix == \".pyi\":\n        mode = replace(mode, is_pyi=True)\n    elif src.suffix == \".ipynb\":\n        mode = replace(mode, is_ipynb=True)\n\n    then = datetime.fromtimestamp(src.stat().st_mtime, timezone.utc)\n    header = b\"\"\n    with open(src, \"rb\") as buf:\n        if mode.skip_source_first_line:\n            header = buf.readline()\n        src_contents, encoding, newline = decode_bytes(buf.read())\n    try:\n        dst_contents = format_file_contents(\n            src_contents, fast=fast, mode=mode, lines=lines\n        )\n    except NothingChanged:\n        return False\n    except JSONDecodeError:\n        raise ValueError(\n            f\"File '{src}' cannot be parsed as valid Jupyter notebook.\"\n        ) from None\n    src_contents = header.decode(encoding) + src_contents\n    dst_contents = header.decode(encoding) + dst_contents\n\n    if write_back == WriteBack.YES:\n        with open(src, \"w\", encoding=encoding, newline=newline) as f:\n            f.write(dst_contents)\n    elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n        now = datetime.now(timezone.utc)\n        src_name = f\"{src}\\t{then}\"\n        dst_name = f\"{src}\\t{now}\"\n        if mode.is_ipynb:\n            diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name)\n        else:\n            diff_contents = diff(src_contents, dst_contents, src_name, dst_name)\n\n        if write_back == WriteBack.COLOR_DIFF:\n            diff_contents = color_diff(diff_contents)\n\n        with lock or nullcontext():\n            f = io.TextIOWrapper(\n                sys.stdout.buffer,\n                encoding=encoding,\n                newline=newline,\n                write_through=True,\n            )\n            f = wrap_stream_for_windows(f)\n            f.write(diff_contents)\n            f.detach()\n\n    return True\n\n\ndef format_stdin_to_stdout(\n    fast: bool,\n    *,\n    content: Optional[str] = None,\n    write_back: WriteBack = WriteBack.NO,\n    mode: Mode,\n    lines: Collection[Tuple[int, int]] = (),\n) -> bool:\n    \"\"\"Format file on stdin. Return True if changed.\n\n    If content is None, it's read from sys.stdin.\n\n    If `write_back` is YES, write reformatted code back to stdout. If it is DIFF,\n    write a diff to stdout. The `mode` argument is passed to\n    :func:`format_file_contents`.\n    \"\"\"\n    then = datetime.now(timezone.utc)\n\n    if content is None:\n        src, encoding, newline = decode_bytes(sys.stdin.buffer.read())\n    else:\n        src, encoding, newline = content, \"utf-8\", \"\"\n\n    dst = src\n    try:\n        dst = format_file_contents(src, fast=fast, mode=mode, lines=lines)\n        return True\n\n    except NothingChanged:\n        return False\n\n    finally:\n        f = io.TextIOWrapper(\n            sys.stdout.buffer, encoding=encoding, newline=newline, write_through=True\n        )\n        if write_back == WriteBack.YES:\n            # Make sure there's a newline after the content\n            if dst and dst[-1] != \"\\n\":\n                dst += \"\\n\"\n            f.write(dst)\n        elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF):\n            now = datetime.now(timezone.utc)\n            src_name = f\"STDIN\\t{then}\"\n            dst_name = f\"STDOUT\\t{now}\"\n            d = diff(src, dst, src_name, dst_name)\n            if write_back == WriteBack.COLOR_DIFF:\n                d = color_diff(d)\n                f = wrap_stream_for_windows(f)\n            f.write(d)\n        f.detach()\n\n\ndef check_stability\n_and_equivalence(\n    src_contents: str,\n    dst_contents: str,\n    *,\n    mode: Mode,\n    lines: Collection[Tuple[int, int]] = (),\n) -> None:\n    \"\"\"Perform stability and equivalence checks.\n\n    Raise AssertionError if source and destination contents are not\n    equivalent, or if a second pass of the formatter would format the\n    content differently.\n    \"\"\"\n    assert_equivalent(src_contents, dst_contents)\n    assert_stable(src_contents, dst_contents, mode=mode, lines=lines)\n\n\ndef format_file_contents(\n    src_contents: str,\n    *,\n    fast: bool,\n    mode: Mode,\n    lines: Collection[Tuple[int, int]] = (),\n) -> FileContent:\n    \"\"\"Reformat contents of a file and return new contents.\n\n    If `fast` is False, additionally confirm that the reformatted code is\n    valid by calling :func:`assert_equivalent` and :func:`assert_stable` on it.\n    `mode` is passed to :func:`format_str`.\n    \"\"\"\n    if mode.is_ipynb:\n        dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode)\n    else:\n        dst_contents = format_str(src_contents, mode=mode, lines=lines)\n    if src_contents == dst_contents:\n        raise NothingChanged\n\n    if not fast and not mode.is_ipynb:\n        # Jupyter notebooks will already have been checked above.\n        check_stability_and_equivalence(\n            src_contents, dst_contents, mode=mode, lines=lines\n        )\n    return dst_contents\n\n\ndef validate_cell(src: str, mode: Mode) -> None:\n    \"\"\"Check that cell does not already contain TransformerManager transformations,\n    or non-Python cell magics, which might cause tokenizer_rt to break because of\n    indentations.\n\n    If a cell contains ``!ls``, then it'll be transformed to\n    ``get_ipython().system('ls')``. However, if the cell originally contained\n    ``get_ipython().system('ls')``, then it would get transformed in the same way:\n\n        >>> TransformerManager().transform_cell(\"get_ipython().system('ls')\")\n        \"get_ipython().system('ls')\\n\"\n        >>> TransformerManager().transform_cell(\"!ls\")\n        \"get_ipython().system('ls')\\n\"\n\n    Due to the impossibility of safely roundtripping in such situations, cells\n    containing transformed magics will be ignored.\n    \"\"\"\n    if any(transformed_magic in src for transformed_magic in TRANSFORMED_MAGICS):\n        raise NothingChanged\n    if (\n        src[:2] == \"%%\"\n        and src.split()[0][2:] not in PYTHON_CELL_MAGICS | mode.python_cell_magics\n    ):\n        raise NothingChanged\n\n\ndef format_cell(src: str, *, fast: bool, mode: Mode) -> str:\n    \"\"\"Format code in given cell of Jupyter notebook.\n\n    General idea is:\n\n      - if cell has trailing semicolon, remove it;\n      - if cell has IPython magics, mask them;\n      - format cell;\n      - reinstate IPython magics;\n      - reinstate trailing semicolon (if originally present);\n      - strip trailing newlines.\n\n    Cells with syntax errors will not be processed, as they\n    could potentially be automagics or multi-line magics, which\n    are currently not supported.\n    \"\"\"\n    validate_cell(src, mode)\n    src_without_trailing_semicolon, has_trailing_semicolon = remove_trailing_semicolon(\n        src\n    )\n    try:\n        masked_src, replacements = mask_cell(src_without_trailing_semicolon)\n    except SyntaxError:\n        raise NothingChanged from None\n    masked_dst = format_str(masked_src, mode=mode)\n    if not fast:\n        check_stability_and_equivalence(masked_src, masked_dst, mode=mode)\n    dst_without_trailing_semicolon = unmask_cell(masked_dst, replacements)\n    dst = put_trailing_semicolon_back(\n        dst_without_trailing_semicolon, has_trailing_semicolon\n    )\n    dst = dst.rstrip(\"\\n\")\n    if dst == src:\n        raise NothingChanged from None\n    return dst\n\n\ndef validate_metadata(nb: MutableMapping[str, Any]) -> None:\n    \"\"\"If notebook is marked as non-Python, don't format it.\n\n    All notebook metadata fields are optional, see\n    https://nbformat.readthedocs.io/en/latest/format_description.html. So\n    if a notebook has empty metadata, we will try to parse it anyway.\n    \"\"\"\n    language = nb.get(\"metadata\", {}).get(\"language_info\", {}).get(\"name\", None)\n    if language is not None and language != \"python\":\n        raise NothingChanged from None\n\n\ndef format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileContent:\n    \"\"\"Format Jupyter notebook.\n\n    Operate cell-by-cell, only on code cells, only for Python notebooks.\n    If the ``.ipynb`` originally had a trailing newline, it'll be preserved.\n    \"\"\"\n    if not src_contents:\n        raise NothingChanged\n\n    trailing_newline = src_contents[-1] == \"\\n\"\n    modified = False\n    nb = json.loads(src_contents)\n    validate_metadata(nb)\n    for cell in nb[\"cells\"]:\n        if cell.get(\"cell_type\", None) == \"code\":\n            try:\n                src = \"\".join(cell[\"source\"])\n                dst = format_cell(src, fast=fast, mode=mode)\n            except NothingChanged:\n                pass\n            else:\n                cell[\"source\"] = dst.splitlines(keepends=True)\n                modified = True\n    if modified:\n        dst_contents = json.dumps(nb, indent=1, ensure_ascii=False)\n        if trailing_newline:\n            dst_contents = dst_contents + \"\\n\"\n        return dst_contents\n    else:\n        raise NothingChanged\n\n\ndef format_str(\n    src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = ()\n) -> str:\n    \"\"\"Reformat a string and return new contents.\n\n    `mode` determines formatting options, such as how many characters per line are\n    allowed.  Example:\n\n    >>> import black\n    >>> print(black.format_str(\"def f(arg:str='')->None:...\", mode=black.Mode()))\n    def f(arg: str = \"\") -> None:\n        ...\n\n    A more complex example:\n\n    >>> print(\n    ...   black.format_str(\n    ...     \"def f(arg:str='')->None: hey\",\n    ...     mode=black.Mode(\n    ...       target_versions={black.TargetVersion.PY36},\n    ...       line_length=10,\n    ...       string_normalization=False,\n    ...       is_pyi=False,\n    ...     ),\n    ...   ),\n    ... )\n    def f(\n        arg: str = '',\n    ) -> None:\n        hey\n\n    \"\"\"\n    dst_contents = _format_str_once(src_contents, mode=mode, lines=lines)\n    # Forced second pass to work around optional trailing commas (becoming\n    # forced trailing commas on pass 2) interacting differently with optional\n    # parentheses.  Admittedly ugly.\n    if src_contents != dst_contents:\n        if lines:\n            lines = adjusted_lines(lines, src_contents, dst_contents)\n        return _format_str_once(dst_contents, mode=mode, lines=lines)\n    return dst_contents\n\n\ndef _format_str_once(\n    src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = ()\n) -> str:\n    src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions)\n    dst_blocks: List[LinesBlock] = []\n    if mode.target_versions:\n        versions = mode.target_versions\n    else:\n        future_imports = get_future_imports(src_node)\n        versions = detect_target_versions(src_node, future_imports=future_imports)\n\n    context_manager_features = {\n        feature\n        for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS}\n        if supports_feature(versions, feature)\n    }\n    normalize_fmt_off(src_node, mode, lines)\n    if lines:\n        # This should be called after normalize_fmt_off.\n        convert_unchanged_lines(src_node, lines)\n\n    line_generator = LineGenerator(mode=mode, features=context_manager_features)\n    elt = EmptyLineTracker(mode=mode)\n    split_line_features = {\n        feature\n        for feature in {Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF}\n        if supports_feature(versions, feature)\n    }\n    block: Optional[LinesBlock] = None\n    for current_line in line_generator.visit(src_node):\n        block = elt.maybe_empty_lines(current_line)\n        dst_blocks.append(block)\n        for line in transform_line(\n            current_line, mode=mode, features=split_line_features\n        ):\n            block.content_lines.append(str(line))\n    if dst_blocks:\n        dst_blocks[-1].after = 0\n    dst_contents = []\n    for block in dst_blocks:\n        dst_contents.extend(block.all_lines())\n    if not dst_contents:\n        # Use decode_bytes to retrieve the correct source newline (CRLF or LF),\n        # and check if normalized_content has more than one line\n        normalized_content, _, newline = decode_bytes(src_contents.encode(\"utf-8\"))\n        if \"\\n\" in normalized_content:\n            return newline\n        return \"\"\n    return \"\".join(dst_contents)\n\n\ndef decode_bytes(src: bytes) -> Tuple[FileContent, Encoding, NewLine]:\n    \"\"\"Return a tuple of (decoded_contents, encoding, newline).\n\n    `newline` is either CRLF or LF but `decoded_contents` is decoded with\n    universal newlines (i.e. only contains LF).\n    \"\"\"\n    srcbuf = io.BytesIO(src)\n    encoding, lines = tokenize.detect_encoding(srcbuf.readline)\n    if not lines:\n        return \"\", encoding, \"\\n\"\n\n    newline = \"\\r\\n\" if b\"\\r\\n\" == lines[0][-2:] else \"\\n\"\n    srcbuf.seek(0)\n    with io.TextIOWrapper(srcbuf, encoding) as tiow:\n        return tiow.read(), encoding, newline\n\n\ndef get_features_used(  # noqa: C901\n    node: Node, *, future_imports: Optional[Set[str]] = None\n) -> Set[Feature]:\n    \"\"\"Return a set of (relatively) new Python features used in this file.\n\n    Currently looking for:\n    - f-strings;\n    - self-documenting expressions in f-strings (f\"{x=}\");\n    - underscores in numeric literals;\n    - trailing commas after * or ** in function signatures and calls;\n    - positional only arguments in function signatures and lambdas;\n    - assignment expression;\n    - relaxed decorator syntax;\n    - usage of __future__ flags (annotations);\n    - print / exec statements;\n    - parenthesized context managers;\n    - match statements;\n    - except* clause;\n    - variadic generics;\n    \"\"\"\n    features: Set[Feature] = set()\n    if future_imports:\n        features |= {\n            FUTURE_FLAG_TO_FEATURE[future_import]\n            for future_import in future_imports\n            if future_import in FUTURE_FLAG_TO_FEATURE\n        }\n\n    for n in node.pre_order():\n        if is_string_token(n):\n            value_head = n.value[:2]\n            if value_head in {'f\"', 'F\"', \"f'\", \"F'\", \"rf\", \"fr\", \"RF\", \"FR\"}:\n                features.add(Feature.F_STRINGS)\n                if Feature.DEBUG_F_STRINGS not in features:\n                    for span_beg, span_end in iter_fexpr_spans(n.value):\n                        if n.value[span_beg : span_end - 1].rstrip().endswith(\"=\"):\n                            features.add(Feature.DEBUG_F_STRINGS)\n                            break\n\n        elif is_number_token(n):\n            if \"_\" in n.value:\n                features.add(Feature.NUMERIC_UNDERSCORES)\n\n        elif n.type == token.SLASH:\n            if n.parent and n.parent.type in {\n                syms.typedargslist,\n                syms.arglist,\n                syms.varargslist,\n            }:\n                features.add(Feature.POS_ONLY_ARGUMENTS)\n\n        elif n.type == token.COLONEQUAL:\n            features.add(Feature.ASSIGNMENT_EXPRESSIONS)\n\n        elif n.type == syms.decorator:\n            if len(n.children) > 1 and not is_simple_decorator_expression(\n                n.children[1]\n            ):\n                features.add(Feature.RELAXED_DECORATORS)\n\n        elif (\n            n.type in {syms.typedargslist, syms.arglist}\n            and n.children\n            and n.children[-1].type == token.COMMA\n        ):\n            if n.type == syms.typedargslist:\n                feature = Feature.TRAILING_COMMA_IN_DEF\n            else:\n                feature = Feature.TRAILING_COMMA_IN_CALL\n\n            for ch in n.children:\n                if ch.type in STARS:\n                    features.add(feature)\n\n                if ch.type == syms.argument:\n                    for argch in ch.children:\n                        if argch.type in STARS:\n                            features.add(feature)\n\n        elif (\n            n.type in {syms.return_stmt, syms.yield_expr}\n            and len(n.children) >= 2\n            and n.children[1].type == syms.testlist_star_expr\n            and any(child.type == syms.star_expr for child in n.children[1].children)\n        ):\n            features.add(Feature.UNPACKING_ON_FLOW)\n\n        elif (\n            n.type == syms.annassign\n            and len(n.children) >= 4\n            and n.children[3].type == syms.testlist_star_expr\n        ):\n            features.add(Feature.ANN_ASSIGN_EXTENDED_RHS)\n\n        elif (\n            n.type == syms.with_stmt\n            and len(n.children) > 2\n            and n.children[1].type == syms.atom\n        ):\n            atom_children = n.children[1].children\n            if (\n                len(atom_children) == 3\n                and atom_children[0].type == token.LPAR\n                and _contains_asexpr(atom_children[1])\n                and atom_children[2].type == token.RPAR\n            ):\n                features.add(Feature.PARENTHESIZED_CONTEXT_MANAGERS)\n\n        elif n.type == syms.match_stmt:\n            features.add(Feature.PATTERN_MATCHING)\n\n        elif (\n            n.type == syms.except_clause\n            and len(n.children) >= 2\n            and n.children[1].type == token.STAR\n        ):\n            features.add(Feature.EXCEPT_STAR)\n\n        elif n.type in {syms.subscriptlist, syms.trailer} and any(\n            child.type == syms.star_expr for child in n.children\n        ):\n            features.add(Feature.VARIADIC_GENERICS)\n\n        elif (\n            n.type == syms.tname_star\n            and len(n.children) == 3\n            and n.children[2].type == syms.star_expr\n        ):\n            features.add(Feature.VARIADIC_GENERICS)\n\n        elif n.type in (syms.type_stmt, syms.typeparams):\n            features.add(Feature.TYPE_PARAMS)\n\n    return features\n\n\ndef _contains_asexpr(node: Union[Node, Leaf]) -> bool:\n    \"\"\"Return True if `node` contains an as-pattern.\"\"\"\n    if node.type == syms.asexpr_test:\n        return True\n    elif node.type == syms.atom:\n        if (\n            len(node.children) == 3\n            and node.children[0].type == token.LPAR\n            and node.children[2].type == token.RPAR\n        ):\n            return _contains_asexpr(node.children[1])\n    elif node.type == syms.testlist_gexp:\n        return any(_contains_asexpr(child) for child in node.children)\n    return False\n\n\ndef detect_target_versions(\n    node: Node, *, future_imports: Optional[Set[str]] = None\n) -> Set[TargetVersion]:\n    \"\"\"Detect the version to target based on the nodes used.\"\"\"\n    features = get_features_used(node, future_imports=future_imports)\n    return {\n        version for version in TargetVersion if features <= VERSION_TO_FEATURES[version]\n    }\n\n\ndef get_future_imports(node: Node) -> Set[str]:\n    \"\"\"Return a set of __future__ imports in the file.\"\"\"\n    imports: Set[str] = set()\n\n    def get_imports_from_children(children: List[LN]) -> Generator[str, None, None]:\n        for child in children:\n            if isinstance(child, Leaf):\n                if child.type == token.NAME:\n                    yield child.value\n\n            elif child.type == syms.import_as_name:\n                orig_name = child.children[0]\n                assert isinstance(orig_name, Leaf), \"Invalid syntax parsing imports\"\n                assert orig_name.type == token.NAME, \"Invalid syntax parsing imports\"\n                yield orig_name.value\n\n            elif child.type == syms.import_as_names:\n                yield from get_imports_from_children(child.children)\n\n            else:\n                raise AssertionError(\"Invalid syntax parsing imports\")\n\n    for child in node.children:\n        if child.type != syms.simple_stmt:\n            break\n\n        first_child = child.children[0]\n        if isinstance(first_child, Leaf):\n            # Continue looking if we see a docstring; otherwise stop.\n            if (\n                len(child.children) == 2\n                and first_child.type == token.STRING\n                and child.children[1].type == token.NEWLINE\n            ):\n                continue\n\n            break\n\n        elif first_child.type == syms.import_from:\n            module_name = first_child.children[1]\n            if not isinstance(module_name, Leaf) or module_name.value != \"__future__\":\n                break\n\n            imports |= set(get_imports_from_children(first_child.children[3:]))\n        else:\n            break\n\n    return imports\n\n\ndef assert_equivalent(src: str, dst: str) -> None:\n    \"\"\"Raise AssertionError if `src` and `dst` aren't equivalent.\"\"\"\n    try:\n        src_ast = parse_ast(src)\n    except Exception as exc:\n        raise AssertionError(\n            \"cannot use --safe with this file; failed to parse source file AST: \"\n            f\"{exc}\\n\"\n            \"This could be caused by running Black with an older Python version \"\n            \"that does not support new syntax used in your source file.\"\n        ) from exc\n\n    try:\n        dst_ast = parse_ast(dst)\n    except Exception as exc:\n        log = dump_to_file(\"\".join(traceback.format_tb(exc.__traceback__)), dst)\n        raise AssertionError(\n            f\"INTERNAL ERROR: Black produced invalid code: {exc}. \"\n            \"Please report a bug on https://github.com/psf/black/issues.  \"\n            f\"This invalid output might be helpful: {log}\"\n        ) from None\n\n    src_ast_str = \"\\n\".join(stringify_ast(src_ast))\n    dst_ast_str = \"\\n\".join(stringify_ast(dst_ast))\n    if src_ast_str != dst_ast_str:\n        log = dump_to_file(diff(src_ast_str, dst_ast_str, \"src\", \"dst\"))\n        raise AssertionError(\n            \"INTERNAL ERROR: Black produced code that is not equivalent to the\"\n            \" source.  Please report a bug on \"\n            f\"https://github.com/psf/black/issues.  This diff might be helpful: {log}\"\n        ) from None\n\n\ndef assert_stable(\n    src: str, dst: str, mode: Mode, *, lines: Collection[Tuple[int, int]] = ()\n) -> None:\n    \"\"\"Raise AssertionError if `dst` reformats differently the second time.\"\"\"\n    if lines:\n        # Formatting specified lines requires `adjusted_lines` to map original lines\n        # to the formatted lines before re-formatting the previously formatted result.\n        # Due to less-ideal diff algorithm, some edge cases produce incorrect new line\n        # ranges. Hence for now, we skip the stable check.\n        # See https://github.com/psf/black/issues/4033 for context.\n        return\n    # We shouldn't call format_str() here, because that formats the string\n    # twice and may hide a bug where we bounce back and forth between two\n    # versions.\n    newdst = _format_str_once(dst, mode=mode, lines=lines)\n    if dst != newdst:\n        log = dump_to_file(\n            str(mode),\n            diff(src, dst, \"source\", \"first pass\"),\n            diff(dst, newdst, \"first pass\", \"second pass\"),\n        )\n        raise AssertionError(\n            \"INTERNAL ERROR: Black produced different code on the second pass of the\"\n...\n", "needle_token_start": 10454, "needle_token_end": 10602, "code_context_ntokens": 16174, "output": ["```python\ndef check_stability_and_equivalence(\n    src_contents: str,\n    dst_contents: str,\n    *,\n    mode: Mode,\n    lines: Collection[Tuple[int, int]] = (),\n) -> None:\n    \"\"\"Perform stability and equivalence checks.\n\n    Raise AssertionError if source and destination contents are not\n    equivalent, or if a second pass of the formatter would format the\n    content differently.\n    \"\"\"\n    assert_equivalent(src_contents, dst_contents)\n    assert_stable(src_contents, dst_contents, mode=mode, lines=lines)\n```"]}
{"repo": "psf/black", "name": "visit", "language": "python", "path": "src/black/nodes.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to traverse a syntax tree node and its children, applying specific methods based on the node's type, and yielding results of a specified type.\n2. **Input**: A node from a syntax tree.\n3. **Output**: An iterator that yields objects of a specified type, derived from processing the node and its children.\n4. **Procedure**: The function first determines the type of the node. If the node type is less than 256, it uses a predefined mapping to get the method name; otherwise, it converts the node type to a string representation. It then attempts to find a method specifically tailored to handle the node type. If such a method exists, it is called and its results are yielded. If no specific method is found, a default method is called and its results are yielded.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/black/_width_table.py\n# Generated by make_width_table.py\n# wcwidth 0.2.6\n# Unicode 15.0.0\nfrom typing import Final, List, Tuple\n\nWIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [\n    (0, 0, 0),\n    (1, 31, -1),\n    (127, 159, -1),\n    (768, 879, 0),\n    (1155, 1161, 0),\n    (1425, 1469, 0),\n    (1471, 1471, 0),\n    (1473, 1474, 0),\n    (1476, 1477, 0),\n    (1479, 1479, 0),\n    (1552, 1562, 0),\n    (1611, 1631, 0),\n    (1648, 1648, 0),\n    (1750, 1756, 0),\n    (1759, 1764, 0),\n    (1767, 1768, 0),\n    (1770, 1773, 0),\n    (1809, 1809, 0),\n    (1840, 1866, 0),\n    (1958, 1968, 0),\n    (2027, 2035, 0),\n    (2045, 2045, 0),\n    (2070, 2073, 0),\n    (2075, 2083, 0),\n    (2085, 2087, 0),\n    (2089, 2093, 0),\n    (2137, 2139, 0),\n    (2200, 2207, 0),\n    (2250, 2273, 0),\n    (2275, 2306, 0),\n    (2362, 2362, 0),\n    (2364, 2364, 0),\n    (2369, 2376, 0),\n    (2381, 2381, 0),\n    (2385, 2391, 0),\n    (2402, 2403, 0),\n    (2433, 2433, 0),\n    (2492, 2492, 0),\n    (2497, 2500, 0),\n    (2509, 2509, 0),\n    (2530, 2531, 0),\n    (2558, 2558, 0),\n    (2561, 2562, 0),\n    (2620, 2620, 0),\n    (2625, 2626, 0),\n    (2631, 2632, 0),\n    (2635, 2637, 0),\n    (2641, 2641, 0),\n    (2672, 2673, 0),\n    (2677, 2677, 0),\n    (2689, 2690, 0),\n    (2748, 2748, 0),\n    (2753, 2757, 0),\n    (2759, 2760, 0),\n    (2765, 2765, 0),\n    (2786, 2787, 0),\n    (2810, 2815, 0),\n    (2817, 2817, 0),\n    (2876, 2876, 0),\n    (2879, 2879, 0),\n    (2881, 2884, 0),\n    (2893, 2893, 0),\n    (2901, 2902, 0),\n    (2914, 2915, 0),\n    (2946, 2946, 0),\n    (3008, 3008, 0),\n    (3021, 3021, 0),\n    (3072, 3072, 0),\n    (3076, 3076, 0),\n    (3132, 3132, 0),\n    (3134, 3136, 0),\n    (3142, 3144, 0),\n    (3146, 3149, 0),\n    (3157, 3158, 0),\n    (3170, 3171, 0),\n    (3201, 3201, 0),\n    (3260, 3260, 0),\n    (3263, 3263, 0),\n    (3270, 3270, 0),\n    (3276, 3277, 0),\n    (3298, 3299, 0),\n    (3328, 3329, 0),\n    (3387, 3388, 0),\n    (3393, 3396, 0),\n    (3405, 3405, 0),\n    (3426, 3427, 0),\n    (3457, 3457, 0),\n    (3530, 3530, 0),\n    (3538, 3540, 0),\n    (3542, 3542, 0),\n    (3633, 3633, 0),\n    (3636, 3642, 0),\n    (3655, 3662, 0),\n    (3761, 3761, 0),\n    (3764, 3772, 0),\n    (3784, 3790, 0),\n    (3864, 3865, 0),\n    (3893, 3893, 0),\n    (3895, 3895, 0),\n    (3897, 3897, 0),\n    (3953, 3966, 0),\n    (3968, 3972, 0),\n    (3974, 3975, 0),\n    (3981, 3991, 0),\n    (3993, 4028, 0),\n    (4038, 4038, 0),\n    (4141, 4144, 0),\n    (4146, 4151, 0),\n    (4153, 4154, 0),\n    (4157, 4158, 0),\n    (4184, 4185, 0),\n    (4190, 4192, 0),\n    (4209, 4212, 0),\n    (4226, 4226, 0),\n    (4229, 4230, 0),\n    (4237, 4237, 0),\n    (4253, 4253, 0),\n    (4352, 4447, 2),\n    (4957, 4959, 0),\n    (5906, 5908, 0),\n    (5938, 5939, 0),\n    (5970, 5971, 0),\n    (6002, 6003, 0),\n    (6068, 6069, 0),\n    (6071, 6077, 0),\n    (6086, 6086, 0),\n    (6089, 6099, 0),\n    (6109, 6109, 0),\n    (6155, 6157, 0),\n    (6159, 6159, 0),\n    (6277, 6278, 0),\n    (6313, 6313, 0),\n    (6432, 6434, 0),\n    (6439, 6440, 0),\n    (6450, 6450, 0),\n    (6457, 6459, 0),\n    (6679, 6680, 0),\n    (6683, 6683, 0),\n    (6742, 6742, 0),\n    (6744, 6750, 0),\n    (6752, 6752, 0),\n    (6754, 6754, 0),\n    (6757, 6764, 0),\n    (6771, 6780, 0),\n    (6783, 6783, 0),\n    (6832, 6862, 0),\n    (6912, 6915, 0),\n    (6964, 6964, 0),\n    (6966, 6970, 0),\n    (6972, 6972, 0),\n    (6978, 6978, 0),\n    (7019, 7027, 0),\n    (7040, 7041, 0),\n    (7074, 7077, 0),\n    (7080, 7081, 0),\n    (7083, 7085, 0),\n    (7142, 7142, 0),\n    (7144, 7145, 0),\n    (7149, 7149, 0),\n    (7151, 7153, 0),\n    (7212, 7219, 0),\n    (7222, 7223, 0),\n    (7376, 7378, 0),\n    (7380, 7392, 0),\n    (7394, 7400, 0),\n    (7405, 7405, 0),\n    (7412, 7412, 0),\n    (7416, 7417, 0),\n    (7616, 7679, 0),\n    (8203, 8207, 0),\n    (8232, 8238, 0),\n    (8288, 8291, 0),\n    (8400, 8432, 0),\n    (8986, 8987, 2),\n    (9001, 9002, 2),\n    (9193, 9196, 2),\n    (9200, 9200, 2),\n    (9203, 9203, 2),\n    (9725, 9726, 2),\n    (9748, 9749, 2),\n    (9800, 9811, 2),\n    (9855, 9855, 2),\n    (9875, 9875, 2),\n    (9889, 9889, 2),\n    (9898, 9899, 2),\n    (9917, 9918, 2),\n    (9924, 9925, 2),\n    (9934, 9934, 2),\n    (9940, 9940, 2),\n    (9962, 9962, 2),\n    (9970, 9971, 2),\n    (9973, 9973, 2),\n    (9978, 9978, 2),\n    (9981, 9981, 2),\n    (9989, 9989, 2),\n    (9994, 9995, 2),\n    (10024, 10024, 2),\n    (10060, 10060, 2),\n    (10062, 10062, 2),\n    (10067, 10069, 2),\n    (10071, 10071, 2),\n    (10133, 10135, 2),\n    (10160, 10160, 2),\n    (10175, 10175, 2),\n    (11035, 11036, 2),\n    (11088, 11088, 2),\n    (11093, 11093, 2),\n    (11503, 11505, 0),\n    (11647, 11647, 0),\n    (11744, 11775, 0),\n    (11904, 11929, 2),\n    (11931, 12019, 2),\n    (12032, 12245, 2),\n    (12272, 12283, 2),\n    (12288, 12329, 2),\n    (12330, 12333, 0),\n    (12334, 12350, 2),\n    (12353, 12438, 2),\n    (12441, 12442, 0),\n    (12443, 12543, 2),\n    (12549, 12591, 2),\n    (12593, 12686, 2),\n    (12688, 12771, 2),\n    (12784, 12830, 2),\n    (12832, 12871, 2),\n    (12880, 19903, 2),\n    (19968, 42124, 2),\n    (42128, 42182, 2),\n    (42607, 42610, 0),\n    (42612, 42621, 0),\n    (42654, 42655, 0),\n    (42736, 42737, 0),\n    (43010, 43010, 0),\n    (43014, 43014, 0),\n    (43019, 43019, 0),\n    (43045, 43046, 0),\n    (43052, 43052, 0),\n    (43204, 43205, 0),\n    (43232, 43249, 0),\n    (43263, 43263, 0),\n    (43302, 43309, 0),\n    (43335, 43345, 0),\n    (43360, 43388, 2),\n    (43392, 43394, 0),\n    (43443, 43443, 0),\n    (43446, 43449, 0),\n    (43452, 43453, 0),\n    (43493, 43493, 0),\n    (43561, 43566, 0),\n    (43569, 43570, 0),\n    (43573, 43574, 0),\n    (43587, 43587, 0),\n    (43596, 43596, 0),\n    (43644, 43644, 0),\n    (43696, 43696, 0),\n    (43698, 43700, 0),\n    (43703, 43704, 0),\n    (43710, 43711, 0),\n    (43713, 43713, 0),\n    (43756, 43757, 0),\n    (43766, 43766, 0),\n    (44005, 44005, 0),\n    (44008, 44008, 0),\n    (44013, 44013, 0),\n    (44032, 55203, 2),\n    (63744, 64255, 2),\n    (64286, 64286, 0),\n    (65024, 65039, 0),\n    (65040, 65049, 2),\n    (65056, 65071, 0),\n    (65072, 65106, 2),\n    (65108, 65126, 2),\n    (65128, 65131, 2),\n    (65281, 65376, 2),\n    (65504, 65510, 2),\n    (66045, 66045, 0),\n    (66272, 66272, 0),\n    (66422, 66426, 0),\n    (68097, 68099, 0),\n    (68101, 68102, 0),\n    (68108, 68111, 0),\n    (68152, 68154, 0),\n    (68159, 68159, 0),\n    (68325, 68326, 0),\n    (68900, 68903, 0),\n    (69291, 69292, 0),\n    (69373, 69375, 0),\n    (69446, 69456, 0),\n    (69506, 69509, 0),\n    (69633, 69633, 0),\n    (69688, 69702, 0),\n    (69744, 69744, 0),\n    (69747, 69748, 0),\n    (69759, 69761, 0),\n    (69811, 69814, 0),\n    (69817, 69818, 0),\n    (69826, 69826, 0),\n    (69888, 69890, 0),\n    (69927, 69931, 0),\n    (69933, 69940, 0),\n    (70003, 70003, 0),\n    (70016, 70017, 0),\n    (70070, 70078, 0),\n    (70089, 70092, 0),\n    (70095, 70095, 0),\n    (70191, 70193, 0),\n    (70196, 70196, 0),\n    (70198, 70199, 0),\n    (70206, 70206, 0),\n    (70209, 70209, 0),\n    (70367, 70367, 0),\n    (70371, 70378, 0),\n    (70400, 70401, 0),\n    (70459, 70460, 0),\n    (70464, 70464, 0),\n    (70502, 70508, 0),\n    (70512, 70516, 0),\n    (70712, 70719, 0),\n    (70722, 70724, 0),\n    (70726, 70726, 0),\n    (70750, 70750, 0),\n    (70835, 70840, 0),\n    (70842, 70842, 0),\n    (70847, 70848, 0),\n    (70850, 70851, 0),\n    (71090, 71093, 0),\n    (71100, 71101, 0),\n    (71103, 71104, 0),\n    (71132, 71133, 0),\n    (71219, 71226, 0),\n    (71229, 71229, 0),\n    (71231, 71232, 0),\n    (71339, 71339, 0),\n    (71341, 71341, 0),\n    (71344, 71349, 0),\n    (71351, 71351, 0),\n    (71453, 71455, 0),\n    (71458, 71461, 0),\n    (71463, 71467, 0),\n    (71727, 71735, 0),\n    (71737, 71738, 0),\n    (71995, 71996, 0),\n    (71998, 71998, 0),\n    (72003, 72003, 0),\n    (72148, 72151, 0),\n    (72154, 72155, 0),\n    (72160, 72160, 0),\n    (72193, 72202, 0),\n    (72243, 72248, 0),\n    (72251, 72254, 0),\n    (72263, 72263, 0),\n    (72273, 72278, 0),\n    (72281, 72283, 0),\n    (72330, 72342, 0),\n    (72344, 72345, 0),\n    (72752, 72758, 0),\n    (72760, 72765, 0),\n    (72767, 72767, 0),\n    (72850, 72871, 0),\n    (72874, 72880, 0),\n    (72882, 72883, 0),\n    (72885, 72886, 0),\n    (73009, 73014, 0),\n    (73018, 73018, 0),\n    (73020, 73021, 0),\n    (73023, 73029, 0),\n    (73031, 73031, 0),\n    (73104, 73105, 0),\n    (73109, 73109, 0),\n    (73111, 73111, 0),\n    (73459, 73460, 0),\n    (73472, 73473, 0),\n    (73526, 73530, 0),\n    (73536, 73536, 0),\n    (73538, 73538, 0),\n    (78912, 78912, 0),\n    (78919, 78933, 0),\n    (92912, 92916, 0),\n    (92976, 92982, 0),\n    (94031, 94031, 0),\n    (94095, 94098, 0),\n    (94176, 94179, 2),\n    (94180, 94180, 0),\n    (94192, 94193, 2),\n    (94208, 100343, 2),\n    (100352, 101589, 2),\n    (101632, 101640, 2),\n    (110576, 110579, 2),\n...\n# Path: src/black/strings.py\n\"\"\"\nSimple formatting on strings. Further string formatting code is in trans.py.\n\"\"\"\n\nimport re\nimport sys\nfrom functools import lru_cache\nfrom typing import Final, List, Match, Pattern\n\nfrom black._width_table import WIDTH_TABLE\nfrom blib2to3.pytree import Leaf\n\nSTRING_PREFIX_CHARS: Final = \"furbFURB\"  # All possible string prefix characters.\nSTRING_PREFIX_RE: Final = re.compile(\n    r\"^([\" + STRING_PREFIX_CHARS + r\"]*)(.*)$\", re.DOTALL\n)\nFIRST_NON_WHITESPACE_RE: Final = re.compile(r\"\\s*\\t+\\s*(\\S)\")\nUNICODE_ESCAPE_RE: Final = re.compile(\n    r\"(?P<backslashes>\\\\+)(?P<body>\"\n    r\"(u(?P<u>[a-fA-F0-9]{4}))\"  # Character with 16-bit hex value xxxx\n    r\"|(U(?P<U>[a-fA-F0-9]{8}))\"  # Character with 32-bit hex value xxxxxxxx\n    r\"|(x(?P<x>[a-fA-F0-9]{2}))\"  # Character with hex value hh\n    r\"|(N\\{(?P<N>[a-zA-Z0-9 \\-]{2,})\\})\"  # Character named name in the Unicode database\n    r\")\",\n    re.VERBOSE,\n)\n\n\ndef sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:\n    \"\"\"Replace `regex` with `replacement` twice on `original`.\n\n    This is used by string normalization to perform replaces on\n    overlapping matches.\n    \"\"\"\n    return regex.sub(replacement, regex.sub(replacement, original))\n\n\ndef has_triple_quotes(string: str) -> bool:\n    \"\"\"\n    Returns:\n        True iff @string starts with three quotation characters.\n    \"\"\"\n    raw_string = string.lstrip(STRING_PREFIX_CHARS)\n    return raw_string[:3] in {'\"\"\"', \"'''\"}\n\n\ndef lines_with_leading_tabs_expanded(s: str) -> List[str]:\n    \"\"\"\n    Splits string into lines and expands only leading tabs (following the normal\n    Python rules)\n    \"\"\"\n    lines = []\n    for line in s.splitlines():\n        # Find the index of the first non-whitespace character after a string of\n        # whitespace that includes at least one tab\n        match = FIRST_NON_WHITESPACE_RE.match(line)\n        if match:\n            first_non_whitespace_idx = match.start(1)\n\n            lines.append(\n                line[:first_non_whitespace_idx].expandtabs()\n                + line[first_non_whitespace_idx:]\n            )\n        else:\n            lines.append(line)\n    if s.endswith(\"\\n\"):\n        lines.append(\"\")\n    return lines\n\n\ndef fix_docstring(docstring: str, prefix: str) -> str:\n    # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation\n    if not docstring:\n        return \"\"\n    lines = lines_with_leading_tabs_expanded(docstring)\n    # Determine minimum indentation (first line doesn't count):\n    indent = sys.maxsize\n    for line in lines[1:]:\n        stripped = line.lstrip()\n        if stripped:\n            indent = min(indent, len(line) - len(stripped))\n    # Remove indentation (first line is special):\n    trimmed = [lines[0].strip()]\n    if indent < sys.maxsize:\n        last_line_idx = len(lines) - 2\n        for i, line in enumerate(lines[1:]):\n            stripped_line = line[indent:].rstrip()\n            if stripped_line or i == last_line_idx:\n                trimmed.append(prefix + stripped_line)\n            else:\n                trimmed.append(\"\")\n    return \"\\n\".join(trimmed)\n\n\ndef get_string_prefix(string: str) -> str:\n    \"\"\"\n    Pre-conditions:\n        * assert_is_leaf_string(@string)\n\n    Returns:\n        @string's prefix (e.g. '', 'r', 'f', or 'rf').\n    \"\"\"\n    assert_is_leaf_string(string)\n\n    prefix = \"\"\n    prefix_idx = 0\n    while string[prefix_idx] in STRING_PREFIX_CHARS:\n        prefix += string[prefix_idx]\n        prefix_idx += 1\n\n    return prefix\n\n\ndef assert_is_leaf_string(string: str) -> None:\n    \"\"\"\n    Checks the pre-condition that @string has the format that you would expect\n    of `leaf.value` where `leaf` is some Leaf such that `leaf.type ==\n    token.STRING`. A more precise description of the pre-conditions that are\n    checked are listed below.\n\n    Pre-conditions:\n        * @string starts with either ', \", <prefix>', or <prefix>\" where\n        `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`.\n        * @string ends with a quote character (' or \").\n\n    Raises:\n        AssertionError(...) if the pre-conditions listed above are not\n        satisfied.\n    \"\"\"\n    dquote_idx = string.find('\"')\n    squote_idx = string.find(\"'\")\n    if -1 in [dquote_idx, squote_idx]:\n        quote_idx = max(dquote_idx, squote_idx)\n    else:\n        quote_idx = min(squote_idx, dquote_idx)\n\n    assert (\n        0 <= quote_idx < len(string) - 1\n    ), f\"{string!r} is missing a starting quote character (' or \\\").\"\n    assert string[-1] in (\n        \"'\",\n        '\"',\n    ), f\"{string!r} is missing an ending quote character (' or \\\").\"\n    assert set(string[:quote_idx]).issubset(\n        set(STRING_PREFIX_CHARS)\n    ), f\"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}.\"\n\n\ndef normalize_string_prefix(s: str) -> str:\n    \"\"\"Make all string prefixes lowercase.\"\"\"\n    match = STRING_PREFIX_RE.match(s)\n    assert match is not None, f\"failed to match string {s!r}\"\n    orig_prefix = match.group(1)\n    new_prefix = (\n        orig_prefix.replace(\"F\", \"f\")\n        .replace(\"B\", \"b\")\n        .replace(\"U\", \"\")\n        .replace(\"u\", \"\")\n    )\n\n    # Python syntax guarantees max 2 prefixes and that one of them is \"r\"\n    if len(new_prefix) == 2 and \"r\" != new_prefix[0].lower():\n        new_prefix = new_prefix[::-1]\n    return f\"{new_prefix}{match.group(2)}\"\n\n\n# Re(gex) does actually cache patterns internally but this still improves\n# performance on a long list literal of strings by 5-9% since lru_cache's\n# caching overhead is much lower.\n@lru_cache(maxsize=64)\ndef _cached_compile(pattern: str) -> Pattern[str]:\n    return re.compile(pattern)\n\n\ndef normalize_string_quotes(s: str) -> str:\n    \"\"\"Prefer double quotes but only if it doesn't cause more escaping.\n\n    Adds or removes backslashes as appropriate. Doesn't parse and fix\n    strings nested in f-strings.\n    \"\"\"\n    value = s.lstrip(STRING_PREFIX_CHARS)\n    if value[:3] == '\"\"\"':\n        return s\n\n    elif value[:3] == \"'''\":\n        orig_quote = \"'''\"\n        new_quote = '\"\"\"'\n    elif value[0] == '\"':\n        orig_quote = '\"'\n        new_quote = \"'\"\n    else:\n        orig_quote = \"'\"\n        new_quote = '\"'\n    first_quote_pos = s.find(orig_quote)\n    if first_quote_pos == -1:\n        return s  # There's an internal error\n\n    prefix = s[:first_quote_pos]\n    unescaped_new_quote = _cached_compile(rf\"(([^\\\\]|^)(\\\\\\\\)*){new_quote}\")\n    escaped_new_quote = _cached_compile(rf\"([^\\\\]|^)\\\\((?:\\\\\\\\)*){new_quote}\")\n    escaped_orig_quote = _cached_compile(rf\"([^\\\\]|^)\\\\((?:\\\\\\\\)*){orig_quote}\")\n    body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)]\n    if \"r\" in prefix.casefold():\n        if unescaped_new_quote.search(body):\n            # There's at least one unescaped new_quote in this raw string\n            # so converting is impossible\n            return s\n\n        # Do not introduce or remove backslashes in raw strings\n        new_body = body\n    else:\n        # remove unnecessary escapes\n        new_body = sub_twice(escaped_new_quote, rf\"\\1\\2{new_quote}\", body)\n        if body != new_body:\n            # Consider the string without unnecessary escapes as the original\n            body = new_body\n            s = f\"{prefix}{orig_quote}{body}{orig_quote}\"\n        new_body = sub_twice(escaped_orig_quote, rf\"\\1\\2{orig_quote}\", new_body)\n        new_body = sub_twice(unescaped_new_quote, rf\"\\1\\\\{new_quote}\", new_body)\n    if \"f\" in prefix.casefold():\n        matches = re.findall(\n            r\"\"\"\n            (?:(?<!\\{)|^)\\{  # start of the string or a non-{ followed by a single {\n                ([^{].*?)  # contents of the brackets except if begins with {{\n            \\}(?:(?!\\})|$)  # A } followed by end of the string or a non-}\n            \"\"\",\n            new_body,\n            re.VERBOSE,\n        )\n        for m in matches:\n            if \"\\\\\" in str(m):\n                # Do not introduce backslashes in interpolated expressions\n                return s\n\n    if new_quote == '\"\"\"' and new_body[-1:] == '\"':\n        # edge case:\n        new_body = new_body[:-1] + '\\\\\"'\n    orig_escape_count = body.count(\"\\\\\")\n    new_escape_count = new_body.count(\"\\\\\")\n    if new_escape_count > orig_escape_count:\n        return s  # Do not introduce more escaping\n\n    if new_escape_count == orig_escape_count and orig_quote == '\"':\n        return s  # Prefer double quotes\n\n    return f\"{prefix}{new_quote}{new_body}{new_quote}\"\n\n\ndef normalize_unicode_escape_sequences(leaf: Leaf) -> None:\n    \"\"\"Replace hex codes in Unicode escape sequences with lowercase representation.\"\"\"\n    text = leaf.value\n    prefix = get_string_prefix(text)\n    if \"r\" in prefix.lower():\n        return\n\n    def replace(m: Match[str]) -> str:\n        groups = m.groupdict()\n        back_slashes = groups[\"backslashes\"]\n\n        if len(back_slashes) % 2 == 0:\n            return back_slashes + groups[\"body\"]\n\n        if groups[\"u\"]:\n            # \\u\n            return back_slashes + \"u\" + groups[\"u\"].lower()\n        elif groups[\"U\"]:\n            # \\U\n            return back_slashes + \"U\" + groups[\"U\"].lower()\n        elif groups[\"x\"]:\n            # \\x\n            return back_slashes + \"x\" + groups[\"x\"].lower()\n        else:\n            assert groups[\"N\"], f\"Unexpected match: {m}\"\n            # \\N{}\n            return back_slashes + \"N{\" + groups[\"N\"].upper() + \"}\"\n\n    leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text)\n\n\n@lru_cache(maxsize=4096)\ndef char_width(char: str) -> int:\n    \"\"\"Return the width of a single character as it would be displayed in a\n    terminal or editor (which respects Unicode East Asian Width).\n\n    Full width characters are counted as 2, while half width characters are\n    counted as 1.  Also control characters are counted as 0.\n    \"\"\"\n    table = WIDTH_TABLE\n    codepoint = ord(char)\n    highest = len(table) - 1\n    lowest = 0\n    idx = highest // 2\n    while True:\n        start_codepoint, end_codepoint, width = table[idx]\n        if codepoint < start_codepoint:\n            highest = idx - 1\n        elif codepoint > end_codepoint:\n            lowest = idx + 1\n        else:\n            return 0 if width < 0 else width\n        if highest < lowest:\n            break\n        idx = (highest + lowest) // 2\n    return 1\n\n\ndef str_width(line_str: str) -> int:\n    \"\"\"Return the width of `line_str` as it would be displayed in a terminal\n    or editor (which respects Unicode East Asian Width).\n\n    You could utilize this function to determine, for example, if a string\n    is too wide to display in a terminal or editor.\n    \"\"\"\n    if line_str.isascii():\n        # Fast path for a line consisting of only ASCII characters\n        return len(line_str)\n    return sum(map(char_width, line_str))\n\n\ndef count_chars_in_width(line_str: str, max_width: int) -> int:\n    \"\"\"Count the number of characters in `line_str` that would fit in a\n    terminal or editor of `max_width` (which respects Unicode East Asian\n    Width).\n    \"\"\"\n    total_width = 0\n    for i, char in enumerate(line_str):\n        width = char_width(char)\n        if width + total_width > max_width:\n            return i\n        total_width += width\n    return len(line_str)\n\n# Path: src/black/nodes.py\n\"\"\"\nblib2to3 Node/Leaf transformation-related utility functions.\n\"\"\"\n\nimport sys\nfrom typing import (\n    Final,\n    Generic,\n    Iterator,\n    List,\n    Literal,\n    Optional,\n    Set,\n    Tuple,\n    TypeVar,\n    Union,\n)\n\nif sys.version_info >= (3, 10):\n    from typing import TypeGuard\nelse:\n    from typing_extensions import TypeGuard\n\nfrom mypy_extensions import mypyc_attr\n\nfrom black.cache import CACHE_DIR\nfrom black.mode import Mode, Preview\nfrom black.strings import get_string_prefix, has_triple_quotes\nfrom blib2to3 import pygram\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import NL, Leaf, Node, type_repr\n\npygram.initialize(CACHE_DIR)\nsyms: Final = pygram.python_symbols\n\n\n# types\nT = TypeVar(\"T\")\nLN = Union[Leaf, Node]\nLeafID = int\nNodeType = int\n\n\nWHITESPACE: Final = {token.DEDENT, token.INDENT, token.NEWLINE}\nSTATEMENT: Final = {\n    syms.if_stmt,\n    syms.while_stmt,\n    syms.for_stmt,\n    syms.try_stmt,\n    syms.except_clause,\n    syms.with_stmt,\n    syms.funcdef,\n    syms.classdef,\n    syms.match_stmt,\n    syms.case_block,\n}\nSTANDALONE_COMMENT: Final = 153\ntoken.tok_name[STANDALONE_COMMENT] = \"STANDALONE_COMMENT\"\nLOGIC_OPERATORS: Final = {\"and\", \"or\"}\nCOMPARATORS: Final = {\n    token.LESS,\n    token.GREATER,\n    token.EQEQUAL,\n    token.NOTEQUAL,\n    token.LESSEQUAL,\n    token.GREATEREQUAL,\n}\nMATH_OPERATORS: Final = {\n    token.VBAR,\n    token.CIRCUMFLEX,\n    token.AMPER,\n    token.LEFTSHIFT,\n    token.RIGHTSHIFT,\n    token.PLUS,\n    token.MINUS,\n    token.STAR,\n    token.SLASH,\n    token.DOUBLESLASH,\n    token.PERCENT,\n    token.AT,\n    token.TILDE,\n    token.DOUBLESTAR,\n}\nSTARS: Final = {token.STAR, token.DOUBLESTAR}\nVARARGS_SPECIALS: Final = STARS | {token.SLASH}\nVARARGS_PARENTS: Final = {\n    syms.arglist,\n    syms.argument,  # double star in arglist\n    syms.trailer,  # single argument to call\n    syms.typedargslist,\n    syms.varargslist,  # lambdas\n}\nUNPACKING_PARENTS: Final = {\n    syms.atom,  # single element of a list or set literal\n    syms.dictsetmaker,\n    syms.listmaker,\n    syms.testlist_gexp,\n    syms.testlist_star_expr,\n    syms.subject_expr,\n    syms.pattern,\n}\nTEST_DESCENDANTS: Final = {\n    syms.test,\n    syms.lambdef,\n    syms.or_test,\n    syms.and_test,\n    syms.not_test,\n    syms.comparison,\n    syms.star_expr,\n    syms.expr,\n    syms.xor_expr,\n    syms.and_expr,\n    syms.shift_expr,\n    syms.arith_expr,\n    syms.trailer,\n    syms.term,\n    syms.power,\n    syms.namedexpr_test,\n}\nTYPED_NAMES: Final = {syms.tname, syms.tname_star}\nASSIGNMENTS: Final = {\n    \"=\",\n    \"+=\",\n    \"-=\",\n    \"*=\",\n    \"@=\",\n    \"/=\",\n    \"%=\",\n    \"&=\",\n    \"|=\",\n    \"^=\",\n    \"<<=\",\n    \">>=\",\n    \"**=\",\n    \"//=\",\n    \":\",\n}\n\nIMPLICIT_TUPLE: Final = {syms.testlist, syms.testlist_star_expr, syms.exprlist}\nBRACKET: Final = {\n    token.LPAR: token.RPAR,\n    token.LSQB: token.RSQB,\n    token.LBRACE: token.RBRACE,\n}\nOPENING_BRACKETS: Final = set(BRACKET.keys())\nCLOSING_BRACKETS: Final = set(BRACKET.values())\nBRACKETS: Final = OPENING_BRACKETS | CLOSING_BRACKETS\nALWAYS_NO_SPACE: Final = CLOSING_BRACKETS | {token.COMMA, STANDALONE_COMMENT}\n\nRARROW = 55\n\n\n@mypyc_attr(allow_interpreted_subclasses=True)\nclass Visitor(Generic[T]):\n    \"\"\"Basic lib2to3 visitor that yields things of type `T` on `visit()`.\"\"\"\n\n    \ndef visit(self, node: LN) -> Iterator[T]:\n        \"\"\"Main method to visit `node` and its children.\n\n        It tries to find a `visit_*()` method for the given `node.type`, like\n        `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.\n        If no dedicated `visit_*()` method is found, chooses `visit_default()`\n        instead.\n\n        Then yields objects of type `T` from the selected visitor.\n        \"\"\"\n        if node.type < 256:\n            name = token.tok_name[node.type]\n        else:\n            name = str(type_repr(node.type))\n        # We explicitly branch on whether a visitor exists (instead of\n        # using self.visit_default as the default arg to getattr) in order\n        # to save needing to create a bound method object and so mypyc can\n        # generate a native call to visit_default.\n        visitf = getattr(self, f\"visit_{name}\", None)\n        if visitf:\n            yield from visitf(node)\n        else:\n            yield from self.visit_default(node)\n\n    def visit_default(self, node: LN) -> Iterator[T]:\n        \"\"\"Default `visit_*()` implementation. Recurses to children of `node`.\"\"\"\n        if isinstance(node, Node):\n            for child in node.children:\n                yield from self.visit(child)\n\n\ndef whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str:  # noqa: C901\n    \"\"\"Return whitespace prefix if needed for the given `leaf`.\n\n    `complex_subscript` signals whether the given leaf is part of a subscription\n    which has non-trivial arguments, like arithmetic expressions or function calls.\n    \"\"\"\n    NO: Final[str] = \"\"\n    SPACE: Final[str] = \" \"\n    DOUBLESPACE: Final[str] = \"  \"\n    t = leaf.type\n    p = leaf.parent\n    v = leaf.value\n    if t in ALWAYS_NO_SPACE:\n        return NO\n\n    if t == token.COMMENT:\n        return DOUBLESPACE\n\n    assert p is not None, f\"INTERNAL ERROR: hand-made leaf without parent: {leaf!r}\"\n    if t == token.COLON and p.type not in {\n        syms.subscript,\n        syms.subscriptlist,\n        syms.sliceop,\n    }:\n        return NO\n\n    prev = leaf.prev_sibling\n    if not prev:\n        prevp = preceding_leaf(p)\n        if not prevp or prevp.type in OPENING_BRACKETS:\n            return NO\n\n        if t == token.COLON:\n            if prevp.type == token.COLON:\n                return NO\n\n            elif prevp.type != token.COMMA and not complex_subscript:\n                return NO\n\n            return SPACE\n\n        if prevp.type == token.EQUAL:\n            if prevp.parent:\n                if prevp.parent.type in {\n                    syms.arglist,\n                    syms.argument,\n                    syms.parameters,\n                    syms.varargslist,\n                }:\n                    return NO\n\n                elif prevp.parent.type == syms.typedargslist:\n                    # A bit hacky: if the equal sign has whitespace, it means we\n                    # previously found it's a typed argument.  So, we're using\n                    # that, too.\n                    return prevp.prefix\n\n        elif (\n            prevp.type == token.STAR\n            and parent_type(prevp) == syms.star_expr\n            and parent_type(prevp.parent) == syms.subscriptlist\n        ):\n            # No space between typevar tuples.\n            return NO\n\n        elif prevp.type in VARARGS_SPECIALS:\n            if is_vararg(prevp, within=VARARGS_PARENTS | UNPACKING_PARENTS):\n                return NO\n\n        elif prevp.type == token.COLON:\n            if prevp.parent and prevp.parent.type in {syms.subscript, syms.sliceop}:\n                return SPACE if complex_subscript else NO\n\n        elif (\n            prevp.parent\n            and prevp.parent.type == syms.factor\n            and prevp.type in MATH_OPERATORS\n        ):\n            return NO\n\n        elif prevp.type == token.AT and p.parent and p.parent.type == syms.decorator:\n            # no space in decorators\n            return NO\n\n    elif prev.type in OPENING_BRACKETS:\n        return NO\n\n    if p.type in {syms.parameters, syms.arglist}:\n        # untyped function signatures or calls\n        if not prev or prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.varargslist:\n        # lambdas\n        if prev and prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.typedargslist:\n        # typed function signatures\n        if not prev:\n            return NO\n\n        if t == token.EQUAL:\n            if prev.type not in TYPED_NAMES:\n                return NO\n\n        elif prev.type == token.EQUAL:\n            # A bit hacky: if the equal sign has whitespace, it means we\n            # previously found it's a typed argument.  So, we're using that, too.\n            return prev.prefix\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type in TYPED_NAMES:\n        # type names\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type != token.COMMA:\n                return NO\n\n    elif p.type == syms.trailer:\n        # attributes and calls\n        if t == token.LPAR or t == token.RPAR:\n            return NO\n\n        if not prev:\n            if t == token.DOT or t == token.LSQB:\n                return NO\n\n        elif prev.type != token.COMMA:\n            return NO\n\n    elif p.type == syms.argument:\n        # single argument\n        if t == token.EQUAL:\n            return NO\n\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type == token.LPAR:\n                return NO\n\n        elif prev.type in {token.EQUAL} | VARARGS_SPECIALS:\n            return NO\n\n    elif p.type == syms.decorator:\n        # decorators\n        return NO\n\n    elif p.type == syms.dotted_name:\n        if prev:\n            return NO\n\n        prevp = preceding_leaf(p)\n        if not prevp or prevp.type == token.AT or prevp.type == token.DOT:\n            return NO\n\n    elif p.type == syms.classdef:\n        if t == token.LPAR:\n            return NO\n\n        if prev and prev.type == token.LPAR:\n            return NO\n\n    elif p.type in {syms.subscript, syms.sliceop}:\n        # indexing\n        if not prev:\n            assert p.parent is not None, \"subscripts are always parented\"\n            if p.parent.type == syms.subscriptlist:\n                return SPACE\n\n            return NO\n\n        elif t == token.COLONEQUAL or prev.type == token.COLONEQUAL:\n            return SPACE\n\n        elif not complex_subscript:\n            return NO\n\n    elif p.type == syms.atom:\n        if prev and t == token.DOT:\n            # dots, but not the first one.\n            return NO\n\n    elif p.type == syms.dictsetmaker:\n        # dict unpacking\n        if prev and prev.type == token.DOUBLESTAR:\n            return NO\n\n    elif p.type in {syms.factor, syms.star_expr}:\n        # unary ops\n        if not prev:\n            prevp = preceding_leaf(p)\n            if not prevp or prevp.type in OPENING_BRACKETS:\n                return NO\n\n            prevp_parent = prevp.parent\n            assert prevp_parent is not None\n            if prevp.type == token.COLON and prevp_parent.type in {\n                syms.subscript,\n                syms.sliceop,\n            }:\n                return NO\n\n            elif prevp.type == token.EQUAL and prevp_parent.type == syms.argument:\n                return NO\n\n        elif t in {token.NAME, token.NUMBER, token.STRING}:\n            return NO\n\n    elif p.type == syms.import_from:\n        if t == token.DOT:\n            if prev and prev.type == token.DOT:\n                return NO\n\n        elif t == token.NAME:\n            if v == \"import\":\n                return SPACE\n\n            if prev and prev.type == token.DOT:\n                return NO\n\n    elif p.type == syms.sliceop:\n        return NO\n\n    elif p.type == syms.except_clause:\n        if t == token.STAR:\n            return NO\n\n    return SPACE\n\n\ndef make_simple_prefix(nl_count: int, form_feed: bool, empty_line: str = \"\\n\") -> str:\n    \"\"\"Generate a normalized prefix string.\"\"\"\n    if form_feed:\n        return (empty_line * (nl_count - 1)) + \"\\f\" + empty_line\n    return empty_line * nl_count\n\n\ndef preceding_leaf(node: Optional[LN]) -> Optional[Leaf]:\n    \"\"\"Return the first leaf that precedes `node`, if any.\"\"\"\n    while node:\n        res = node.prev_sibling\n        if res:\n            if isinstance(res, Leaf):\n                return res\n\n            try:\n                return list(res.leaves())[-1]\n\n            except IndexError:\n                return None\n\n        node = node.parent\n    return None\n\n\ndef prev_siblings_are(node: Optional[LN], tokens: List[Optional[NodeType]]) -> bool:\n    \"\"\"Return if the `node` and its previous siblings match types against the provided\n    list of tokens; the provided `node`has its type matched against the last element in\n    the list.  `None` can be used as the first element to declare that the start of the\n    list is anchored at the start of its parent's children.\"\"\"\n    if not tokens:\n        return True\n    if tokens[-1] is None:\n        return node is None\n    if not node:\n        return False\n    if node.type != tokens[-1]:\n        return False\n    return prev_siblings_are(node.prev_sibling, tokens[:-1])\n\n\ndef parent_type(node: Optional[LN]) -> Optional[NodeType]:\n    \"\"\"\n    Returns:\n        @node.parent.type, if @node is not None and has a parent.\n            OR\n        None, otherwise.\n    \"\"\"\n    if node is None or node.parent is None:\n        return None\n\n    return node.parent.type\n\n\ndef child_towards(ancestor: Node, descendant: LN) -> Optional[LN]:\n    \"\"\"Return the child of `ancestor` that contains `descendant`.\"\"\"\n    node: Optional[LN] = descendant\n    while node and node.parent != ancestor:\n        node = node.parent\n    return node\n\n\ndef replace_child(old_child: LN, new_child: LN) -> None:\n    \"\"\"\n    Side Effects:\n        * If @old_child.parent is set, replace @old_child with @new_child in\n        @old_child's underlying Node structure.\n            OR\n        * Otherwise, this function does nothing.\n    \"\"\"\n    parent = old_child.parent\n    if not parent:\n        return\n\n    child_idx = old_child.remove()\n    if child_idx is not None:\n        parent.insert_child(child_idx, new_child)\n\n\ndef container_of(leaf: Leaf) -> LN:\n    \"\"\"Return `leaf` or one of its ancestors that is the topmost container of it.\n\n    By \"container\" we mean a node where `leaf` is the very first child.\n    \"\"\"\n    same_prefix = leaf.prefix\n    container: LN = leaf\n    while container:\n        parent = container.parent\n        if parent is None:\n            break\n\n        if parent.children[0].prefix != same_prefix:\n            break\n\n        if parent.type == syms.file_input:\n            break\n\n        if parent.prev_sibling is not None and parent.prev_sibling.type in BRACKETS:\n            break\n\n        container = parent\n    return container\n\n\ndef first_leaf_of(node: LN) -> Optional[Leaf]:\n    \"\"\"Returns the first leaf of the node tree.\"\"\"\n    if isinstance(node, Leaf):\n        return node\n    if node.children:\n        return first_leaf_of(node.children[0])\n    else:\n        return None\n\n\ndef is_arith_like(node: LN) -> bool:\n    \"\"\"Whether node is an arithmetic or a binary arithmetic expression\"\"\"\n    return node.type in {\n        syms.arith_expr,\n        syms.shift_expr,\n        syms.xor_expr,\n        syms.and_expr,\n    }\n\n\ndef is_docstring(leaf: Leaf, mode: Mode) -> bool:\n    if leaf.type != token.STRING:\n        return False\n\n    prefix = get_string_prefix(leaf.value)\n    if set(prefix).intersection(\"bBfF\"):\n        return False\n\n    if (\n        Preview.unify_docstring_detection in mode\n        and leaf.parent\n        and leaf.parent.type == syms.simple_stmt\n        and not leaf.parent.prev_sibling\n        and leaf.parent.parent\n        and leaf.parent.parent.type == syms.file_input\n    ):\n        return True\n\n    if prev_siblings_are(\n        leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt]\n    ):\n        return True\n\n    # Multiline docstring on the same line as the `def`.\n    if prev_siblings_are(leaf.parent, [syms.parameters, token.COLON, syms.simple_stmt]):\n        # `syms.parameters` is only used in funcdefs and async_funcdefs in the Python\n        # grammar. We're safe to return True without further checks.\n        return True\n\n    return False\n\n\ndef is_empty_tuple(node: LN) -> bool:\n    \"\"\"Return True if `node` holds an empty tuple.\"\"\"\n    return (\n        node.type == syms.atom\n        and len(node.children) == 2\n        and node.children[0].type == token.LPAR\n        and node.children[1].type == token.RPAR\n    )\n\n\ndef is_one_tuple(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a tuple with one element, with or without parens.\"\"\"\n    if node.type == syms.atom:\n        gexp = unwrap_singleton_parenthesis(node)\n        if gexp is None or gexp.type != syms.testlist_gexp:\n            return False\n\n        return len(gexp.children) == 2 and gexp.children[1].type == token.COMMA\n\n    return (\n        node.type in IMPLICIT_TUPLE\n        and len(node.children) == 2\n        and node.children[1].type == token.COMMA\n    )\n\n\ndef is_tuple_containing_walrus(node: LN) -> bool:\n    \"\"\"Return True if `node` holds a tuple that contains a walrus operator.\"\"\"\n    if node.type != syms.atom:\n        return False\n    gexp = unwrap_singleton_parenthesis(node)\n    if gexp is None or gexp.type != syms.testlist_gexp:\n        return False\n\n    return any(child.type == syms.namedexpr_test for child in gexp.children)\n\n\ndef is_one_sequence_between(\n    opening: Leaf,\n    closing: Leaf,\n    leaves: List[Leaf],\n    brackets: Tuple[int, int] = (token.LPAR, token.RPAR),\n) -> bool:\n...\n", "needle_token_start": 12070, "needle_token_end": 12356, "code_context_ntokens": 16200, "output": ["```python\ndef visit(self, node: LN) -> Iterator[T]:\n    \"\"\"Main method to visit `node` and its children.\n\n    It tries to find a `visit_*()` method for the given `node.type`, like\n    `visit_simple_stmt` for Node objects or `visit_INDENT` for Leaf objects.\n    If no dedicated `visit_*()` method is found, chooses `visit_default()`\n    instead.\n\n    Then yields objects of type `T` from the selected visitor.\n    \"\"\"\n    if node.type < 256:\n        name = token.tok_name[node.type]\n    else:\n        name = str(type_repr(node.type))\n    # We explicitly branch on whether a visitor exists (instead of\n    # using self.visit_default as the default arg to getattr) in order\n    # to save needing to create a bound method object and so mypyc can\n    # generate a native call to visit_default.\n    visitf = getattr(self, f\"visit_{name}\", None)\n    if visitf:\n        yield from visitf(node)\n    else:\n        yield from self.visit_default(node)\n\ndef visit_default(self, node: LN) -> Iterator[T]:\n    \"\"\"Default `visit_*()` implementation. Recurses to children of `node`.\"\"\"\n    if isinstance(node, Node):\n        for child in node.children:\n            yield from self.visit(child)\n```"]}
{"repo": "psf/black", "name": "__post_init__", "language": "python", "path": "src/black/linegen.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: Initializes the state and configuration of an object after its creation, setting up necessary properties and methods for handling different types of statements in code formatting.\n2. **Input**: None explicitly, but operates on the instance itself.\n3. **Output**: None, as it modifies the instance state directly.\n4. **Procedure**: Sets an initial line configuration based on the mode, assigns specialized partial functions to handle various statement types like assertions, loops, and conditional statements, and configures handling for different syntactic elements based on the formatting mode.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/black/trans.py\n\"\"\"\nString transformers that can split and merge strings.\n\"\"\"\n\nimport re\nfrom abc import ABC, abstractmethod\nfrom collections import defaultdict\nfrom dataclasses import dataclass\nfrom typing import (\n    Any,\n    Callable,\n    ClassVar,\n    Collection,\n    Dict,\n    Final,\n    Iterable,\n    Iterator,\n    List,\n    Literal,\n    Optional,\n    Sequence,\n    Set,\n    Tuple,\n    TypeVar,\n    Union,\n)\n\nfrom mypy_extensions import trait\n\nfrom black.comments import contains_pragma_comment\nfrom black.lines import Line, append_leaves\nfrom black.mode import Feature, Mode, Preview\nfrom black.nodes import (\n    CLOSING_BRACKETS,\n    OPENING_BRACKETS,\n    STANDALONE_COMMENT,\n    is_empty_lpar,\n    is_empty_par,\n    is_empty_rpar,\n    is_part_of_annotation,\n    parent_type,\n    replace_child,\n    syms,\n)\nfrom black.rusty import Err, Ok, Result\nfrom black.strings import (\n    assert_is_leaf_string,\n    count_chars_in_width,\n    get_string_prefix,\n    has_triple_quotes,\n    normalize_string_quotes,\n    str_width,\n)\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\n\nclass CannotTransform(Exception):\n    \"\"\"Base class for errors raised by Transformers.\"\"\"\n\n\n# types\nT = TypeVar(\"T\")\nLN = Union[Leaf, Node]\nTransformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]]\nIndex = int\nNodeType = int\nParserState = int\nStringID = int\nTResult = Result[T, CannotTransform]  # (T)ransform Result\nTMatchResult = TResult[List[Index]]\n\nSPLIT_SAFE_CHARS = frozenset([\"\\u3001\", \"\\u3002\", \"\\uff0c\"])  # East Asian stops\n\n\ndef TErr(err_msg: str) -> Err[CannotTransform]:\n    \"\"\"(T)ransform Err\n\n    Convenience function used when working with the TResult type.\n    \"\"\"\n    cant_transform = CannotTransform(err_msg)\n    return Err(cant_transform)\n\n\ndef hug_power_op(\n    line: Line, features: Collection[Feature], mode: Mode\n) -> Iterator[Line]:\n    \"\"\"A transformer which normalizes spacing around power operators.\"\"\"\n\n    # Performance optimization to avoid unnecessary Leaf clones and other ops.\n    for leaf in line.leaves:\n        if leaf.type == token.DOUBLESTAR:\n            break\n    else:\n        raise CannotTransform(\"No doublestar token was found in the line.\")\n\n    def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool:\n        # Brackets and parentheses indicate calls, subscripts, etc. ...\n        # basically stuff that doesn't count as \"simple\". Only a NAME lookup\n        # or dotted lookup (eg. NAME.NAME) is OK.\n        if Preview.is_simple_lookup_for_doublestar_expression not in mode:\n            return original_is_simple_lookup_func(line, index, kind)\n\n        else:\n            if kind == -1:\n                return handle_is_simple_look_up_prev(\n                    line, index, {token.RPAR, token.RSQB}\n                )\n            else:\n                return handle_is_simple_lookup_forward(\n                    line, index, {token.LPAR, token.LSQB}\n                )\n\n    def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool:\n        # An operand is considered \"simple\" if's a NAME, a numeric CONSTANT, a simple\n        # lookup (see above), with or without a preceding unary operator.\n        start = line.leaves[index]\n        if start.type in {token.NAME, token.NUMBER}:\n            return is_simple_lookup(index, kind)\n\n        if start.type in {token.PLUS, token.MINUS, token.TILDE}:\n            if line.leaves[index + 1].type in {token.NAME, token.NUMBER}:\n                # kind is always one as bases with a preceding unary op will be checked\n                # for simplicity starting from the next token (so it'll hit the check\n                # above).\n                return is_simple_lookup(index + 1, kind=1)\n\n        return False\n\n    new_line = line.clone()\n    should_hug = False\n    for idx, leaf in enumerate(line.leaves):\n        new_leaf = leaf.clone()\n        if should_hug:\n            new_leaf.prefix = \"\"\n            should_hug = False\n\n        should_hug = (\n            (0 < idx < len(line.leaves) - 1)\n            and leaf.type == token.DOUBLESTAR\n            and is_simple_operand(idx - 1, kind=-1)\n            and line.leaves[idx - 1].value != \"lambda\"\n            and is_simple_operand(idx + 1, kind=1)\n        )\n        if should_hug:\n            new_leaf.prefix = \"\"\n\n        # We have to be careful to make a new line properly:\n        # - bracket related metadata must be maintained (handled by Line.append)\n        # - comments need to copied over, updating the leaf IDs they're attached to\n        new_line.append(new_leaf, preformatted=True)\n        for comment_leaf in line.comments_after(leaf):\n            new_line.append(comment_leaf, preformatted=True)\n\n    yield new_line\n\n\ndef original_is_simple_lookup_func(\n    line: Line, index: int, step: Literal[1, -1]\n) -> bool:\n    if step == -1:\n        disallowed = {token.RPAR, token.RSQB}\n    else:\n        disallowed = {token.LPAR, token.LSQB}\n\n    while 0 <= index < len(line.leaves):\n        current = line.leaves[index]\n        if current.type in disallowed:\n            return False\n        if current.type not in {token.NAME, token.DOT} or current.value == \"for\":\n            # If the current token isn't disallowed, we'll assume this is\n            # simple as only the disallowed tokens are semantically\n            # attached to this lookup expression we're checking. Also,\n            # stop early if we hit the 'for' bit of a comprehension.\n            return True\n\n        index += step\n\n    return True\n\n\ndef handle_is_simple_look_up_prev(line: Line, index: int, disallowed: Set[int]) -> bool:\n    \"\"\"\n    Handling the determination of is_simple_lookup for the lines prior to the doublestar\n    token. This is required because of the need to isolate the chained expression\n    to determine the bracket or parenthesis belong to the single expression.\n    \"\"\"\n    contains_disallowed = False\n    chain = []\n\n    while 0 <= index < len(line.leaves):\n        current = line.leaves[index]\n        chain.append(current)\n        if not contains_disallowed and current.type in disallowed:\n            contains_disallowed = True\n        if not is_expression_chained(chain):\n            return not contains_disallowed\n\n        index -= 1\n\n    return True\n\n\ndef handle_is_simple_lookup_forward(\n    line: Line, index: int, disallowed: Set[int]\n) -> bool:\n    \"\"\"\n    Handling decision is_simple_lookup for the lines behind the doublestar token.\n    This function is simplified to keep consistent with the prior logic and the forward\n    case are more straightforward and do not need to care about chained expressions.\n    \"\"\"\n    while 0 <= index < len(line.leaves):\n        current = line.leaves[index]\n        if current.type in disallowed:\n            return False\n        if current.type not in {token.NAME, token.DOT} or (\n            current.type == token.NAME and current.value == \"for\"\n        ):\n            # If the current token isn't disallowed, we'll assume this is simple as\n            # only the disallowed tokens are semantically attached to this lookup\n            # expression we're checking. Also, stop early if we hit the 'for' bit\n            # of a comprehension.\n            return True\n\n        index += 1\n\n    return True\n\n\ndef is_expression_chained(chained_leaves: List[Leaf]) -> bool:\n    \"\"\"\n    Function to determine if the variable is a chained call.\n    (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call)\n    \"\"\"\n    if len(chained_leaves) < 2:\n        return True\n\n    current_leaf = chained_leaves[-1]\n    past_leaf = chained_leaves[-2]\n\n    if past_leaf.type == token.NAME:\n        return current_leaf.type in {token.DOT}\n    elif past_leaf.type in {token.RPAR, token.RSQB}:\n        return current_leaf.type in {token.RSQB, token.RPAR}\n    elif past_leaf.type in {token.LPAR, token.LSQB}:\n        return current_leaf.type in {token.NAME, token.LPAR, token.LSQB}\n    else:\n        return False\n\n\nclass StringTransformer(ABC):\n    \"\"\"\n    An implementation of the Transformer protocol that relies on its\n    subclasses overriding the template methods `do_match(...)` and\n    `do_transform(...)`.\n\n    This Transformer works exclusively on strings (for example, by merging\n    or splitting them).\n\n    The following sections can be found among the docstrings of each concrete\n    StringTransformer subclass.\n\n    Requirements:\n        Which requirements must be met of the given Line for this\n        StringTransformer to be applied?\n\n    Transformations:\n        If the given Line meets all of the above requirements, which string\n        transformations can you expect to be applied to it by this\n        StringTransformer?\n\n    Collaborations:\n        What contractual agreements does this StringTransformer have with other\n        StringTransfomers? Such collaborations should be eliminated/minimized\n        as much as possible.\n    \"\"\"\n\n    __name__: Final = \"StringTransformer\"\n\n    # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with\n    # `abc.ABC`.\n    def __init__(self, line_length: int, normalize_strings: bool) -> None:\n        self.line_length = line_length\n        self.normalize_strings = normalize_strings\n\n    @abstractmethod\n    def do_match(self, line: Line) -> TMatchResult:\n        \"\"\"\n        Returns:\n            * Ok(string_indices) such that for each index, `line.leaves[index]`\n              is our target string if a match was able to be made. For\n              transformers that don't result in more lines (e.g. StringMerger,\n              StringParenStripper), multiple matches and transforms are done at\n              once to reduce the complexity.\n              OR\n            * Err(CannotTransform), if no match could be made.\n        \"\"\"\n\n    @abstractmethod\n    def do_transform(\n        self, line: Line, string_indices: List[int]\n    ) -> Iterator[TResult[Line]]:\n        \"\"\"\n        Yields:\n            * Ok(new_line) where new_line is the new transformed line.\n              OR\n            * Err(CannotTransform) if the transformation failed for some reason. The\n              `do_match(...)` template method should usually be used to reject\n              the form of the given Line, but in some cases it is difficult to\n              know whether or not a Line meets the StringTransformer's\n              requirements until the transformation is already midway.\n\n        Side Effects:\n            This method should NOT mutate @line directly, but it MAY mutate the\n            Line's underlying Node structure. (WARNING: If the underlying Node\n            structure IS altered, then this method should NOT be allowed to\n            yield an CannotTransform after that point.)\n        \"\"\"\n\n    def __call__(\n        self, line: Line, _features: Collection[Feature], _mode: Mode\n    ) -> Iterator[Line]:\n        \"\"\"\n        StringTransformer instances have a call signature that mirrors that of\n        the Transformer type.\n\n        Raises:\n            CannotTransform(...) if the concrete StringTransformer class is unable\n            to transform @line.\n        \"\"\"\n        # Optimization to avoid calling `self.do_match(...)` when the line does\n        # not contain any string.\n        if not any(leaf.type == token.STRING for leaf in line.leaves):\n            raise CannotTransform(\"There are no strings in this line.\")\n\n        match_result = self.do_match(line)\n\n        if isinstance(match_result, Err):\n            cant_transform = match_result.err()\n            raise CannotTransform(\n                f\"The string transformer {self.__class__.__name__} does not recognize\"\n                \" this line as one that it can transform.\"\n            ) from cant_transform\n\n        string_indices = match_result.ok()\n\n        for line_result in self.do_transform(line, string_indices):\n            if isinstance(line_result, Err):\n                cant_transform = line_result.err()\n                raise CannotTransform(\n                    \"StringTransformer failed while attempting to transform string.\"\n                ) from cant_transform\n            line = line_result.ok()\n            yield line\n\n\n@dataclass\nclass CustomSplit:\n    \"\"\"A custom (i.e. manual) string split.\n\n    A single CustomSplit instance represents a single substring.\n\n    Examples:\n        Consider the following string:\n        ```\n        \"Hi there friend.\"\n        \" This is a custom\"\n        f\" string {split}.\"\n        ```\n\n        This string will correspond to the following three CustomSplit instances:\n        ```\n        CustomSplit(False, 16)\n        CustomSplit(False, 17)\n        CustomSplit(True, 16)\n        ```\n    \"\"\"\n\n    has_prefix: bool\n    break_idx: int\n\n\n@trait\nclass CustomSplitMapMixin:\n    \"\"\"\n    This mixin class is used to map merged strings to a sequence of\n    CustomSplits, which will then be used to re-split the strings iff none of\n    the resultant substrings go over the configured max line length.\n    \"\"\"\n\n    _Key: ClassVar = Tuple[StringID, str]\n    _CUSTOM_SPLIT_MAP: ClassVar[Dict[_Key, Tuple[CustomSplit, ...]]] = defaultdict(\n        tuple\n    )\n\n    @staticmethod\n    def _get_key(string: str) -> \"CustomSplitMapMixin._Key\":\n        \"\"\"\n        Returns:\n            A unique identifier that is used internally to map @string to a\n            group of custom splits.\n        \"\"\"\n        return (id(string), string)\n\n    def add_custom_splits(\n        self, string: str, custom_splits: Iterable[CustomSplit]\n    ) -> None:\n        \"\"\"Custom Split Map Setter Method\n\n        Side Effects:\n            Adds a mapping from @string to the custom splits @custom_splits.\n        \"\"\"\n        key = self._get_key(string)\n        self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)\n\n    def pop_custom_splits(self, string: str) -> List[CustomSplit]:\n        \"\"\"Custom Split Map Getter Method\n\n        Returns:\n            * A list of the custom splits that are mapped to @string, if any\n              exist.\n              OR\n            * [], otherwise.\n\n        Side Effects:\n            Deletes the mapping between @string and its associated custom\n            splits (which are returned to the caller).\n        \"\"\"\n        key = self._get_key(string)\n\n        custom_splits = self._CUSTOM_SPLIT_MAP[key]\n        del self._CUSTOM_SPLIT_MAP[key]\n\n        return list(custom_splits)\n\n    def has_custom_splits(self, string: str) -> bool:\n        \"\"\"\n        Returns:\n            True iff @string is associated with a set of custom splits.\n        \"\"\"\n        key = self._get_key(string)\n        return key in self._CUSTOM_SPLIT_MAP\n\n\nclass StringMerger(StringTransformer, CustomSplitMapMixin):\n    \"\"\"StringTransformer that merges strings together.\n\n    Requirements:\n        (A) The line contains adjacent strings such that ALL of the validation checks\n        listed in StringMerger._validate_msg(...)'s docstring pass.\n        OR\n        (B) The line contains a string which uses line continuation backslashes.\n\n    Transformations:\n        Depending on which of the two requirements above where met, either:\n\n        (A) The string group associated with the target string is merged.\n        OR\n        (B) All line-continuation backslashes are removed from the target string.\n\n    Collaborations:\n        StringMerger provides custom split information to StringSplitter.\n    \"\"\"\n\n    def do_match(self, line: Line) -> TMatchResult:\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        string_indices = []\n        idx = 0\n        while is_valid_index(idx):\n            leaf = LL[idx]\n            if (\n                leaf.type == token.STRING\n                and is_valid_index(idx + 1)\n                and LL[idx + 1].type == token.STRING\n            ):\n                # Let's check if the string group contains an inline comment\n                # If we have a comment inline, we don't merge the strings\n                contains_comment = False\n                i = idx\n                while is_valid_index(i):\n                    if LL[i].type != token.STRING:\n                        break\n                    if line.comments_after(LL[i]):\n                        contains_comment = True\n                        break\n                    i += 1\n\n                if not is_part_of_annotation(leaf) and not contains_comment:\n                    string_indices.append(idx)\n\n                # Advance to the next non-STRING leaf.\n                idx += 2\n                while is_valid_index(idx) and LL[idx].type == token.STRING:\n                    idx += 1\n\n            elif leaf.type == token.STRING and \"\\\\\\n\" in leaf.value:\n                string_indices.append(idx)\n                # Advance to the next non-STRING leaf.\n                idx += 1\n                while is_valid_index(idx) and LL[idx].type == token.STRING:\n                    idx += 1\n\n            else:\n                idx += 1\n\n        if string_indices:\n            return Ok(string_indices)\n        else:\n            return TErr(\"This line has no strings that need merging.\")\n\n    def do_transform(\n        self, line: Line, string_indices: List[int]\n    ) -> Iterator[TResult[Line]]:\n        new_line = line\n\n        rblc_result = self._remove_backslash_line_continuation_chars(\n            new_line, string_indices\n        )\n        if isinstance(rblc_result, Ok):\n            new_line = rblc_result.ok()\n\n        msg_result = self._merge_string_group(new_line, string_indices)\n        if isinstance(msg_result, Ok):\n            new_line = msg_result.ok()\n\n        if isinstance(rblc_result, Err) and isinstance(msg_result, Err):\n            msg_cant_transform = msg_result.err()\n            rblc_cant_transform = rblc_result.err()\n            cant_transform = CannotTransform(\n                \"StringMerger failed to merge any strings in this line.\"\n            )\n\n            # Chain the errors together using `__cause__`.\n            msg_cant_transform.__cause__ = rblc_cant_transform\n            cant_transform.__cause__ = msg_cant_transform\n\n            yield Err(cant_transform)\n        else:\n            yield Ok(new_line)\n\n    @staticmethod\n    def _remove_backslash_line_continuation_chars(\n        line: Line, string_indices: List[int]\n    ) -> TResult[Line]:\n        \"\"\"\n        Merge strings that were split across multiple lines using\n        line-continuation backslashes.\n\n        Returns:\n            Ok(new_line), if @line contains backslash line-continuation\n            characters.\n                OR\n            Err(CannotTransform), otherwise.\n        \"\"\"\n        LL = line.leaves\n\n        indices_to_transform = []\n        for string_idx in string_indices:\n            string_leaf = LL[string_idx]\n            if (\n                string_leaf.type == token.STRING\n                and \"\\\\\\n\" in string_leaf.value\n                and not has_triple_quotes(string_leaf.value)\n            ):\n                indices_to_transform.append(string_idx)\n\n        if not indices_to_transform:\n            return TErr(\n                \"Found no string leaves that contain backslash line continuation\"\n                \" characters.\"\n            )\n\n        new_line = line.clone()\n        new_line.comments = line.comments.copy()\n        append_leaves(new_line, line, LL)\n\n        for string_idx in indices_to_transform:\n            new_string_leaf = new_line.leaves[string_idx]\n            new_string_leaf.value = new_string_leaf.value.replace(\"\\\\\\n\", \"\")\n\n        return Ok(new_line)\n\n    def _merge_string_group(\n        self, line: Line, string_indices: List[int]\n    ) -> TResult[Line]:\n        \"\"\"\n        Merges string groups (i.e. set of adjacent strings).\n\n        Each index from `string_indices` designates one string group's first\n        leaf in `line.leaves`.\n\n        Returns:\n            Ok(new_line), if ALL of the validation checks found in\n            _validate_msg(...) pass.\n                OR\n            Err(CannotTransform), otherwise.\n        \"\"\"\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.\n        merged_string_idx_dict: Dict[int, Tuple[int, Leaf]] = {}\n        for string_idx in string_indices:\n            vresult = self._validate_msg(line, string_idx)\n            if isinstance(vresult, Err):\n                continue\n            merged_string_idx_dict[string_idx] = self._merge_one_string_group(\n                LL, string_idx, is_valid_index\n            )\n\n        if not merged_string_idx_dict:\n            return TErr(\"No string group is merged\")\n\n        # Build the final line ('new_line') that this method will later return.\n        new_line = line.clone()\n        previous_merged_string_idx = -1\n        previous_merged_num_of_strings = -1\n        for i, leaf in enumerate(LL):\n            if i in merged_string_idx_dict:\n                previous_merged_string_idx = i\n                previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]\n                new_line.append(string_leaf)\n\n            if (\n                previous_merged_string_idx\n                <= i\n                < previous_merged_string_idx + previous_merged_num_of_strings\n            ):\n                for comment_leaf in line.comments_after(LL[i]):\n                    new_line.append(comment_leaf, preformatted=True)\n                continue\n\n            append_leaves(new_line, line, [leaf])\n\n        return Ok(new_line)\n\n    def _merge_one_string_group(\n        self, LL: List[Leaf], string_idx: int, is_valid_index: Callable[[int], bool]\n    ) -> Tuple[int, Leaf]:\n        \"\"\"\n        Merges one string group where the first string in the group is\n        `LL[string_idx]`.\n\n        Returns:\n            A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the\n            number of strings merged and `leaf` is the newly merged string\n            to be replaced in the new line.\n        \"\"\"\n        # If the string group is wrapped inside an Atom node, we must make sure\n        # to later replace that Atom with our new (merged) string leaf.\n        atom_node = LL[string_idx].parent\n\n        # We will place BREAK_MARK in between every two substrings that we\n        # merge. We will then later go through our final result and use the\n        # various instances of BREAK_MARK we find to add the right values to\n        # the custom split map.\n        BREAK_MARK = \"@@@@@ BLACK BREAKPOINT MARKER @@@@@\"\n\n        QUOTE = LL[string_idx].value[-1]\n\n        def make_naked(string: str, string_prefix: str) -> str:\n            \"\"\"Strip @string (i.e. make it a \"naked\" string)\n\n            Pre-conditions:\n                * assert_is_leaf_string(@string)\n\n            Returns:\n                A string that is identical to @string except that\n                @string_prefix has been stripped, the surrounding QUOTE\n                characters have been removed, and any remaining QUOTE\n                characters have been escaped.\n            \"\"\"\n            assert_is_leaf_string(string)\n            if \"f\" in string_prefix:\n                f_expressions = (\n                    string[span[0] + 1 : span[1] - 1]  # +-1 to get rid of curly braces\n                    for span in iter_fexpr_spans(string)\n                )\n                debug_expressions_contain_visible_quotes = any(\n                    re.search(r\".*[\\'\\\"].*(?<![!:=])={1}(?!=)(?![^\\s:])\", expression)\n                    for expression in f_expressions\n                )\n                if not debug_expressions_contain_visible_quotes:\n                    # We don't want to toggle visible quotes in debug f-strings, as\n                    # that would modify the AST\n                    string = _toggle_fexpr_quotes(string, QUOTE)\n                    # After quotes toggling, quotes in expressions won't be escaped\n                    # because quotes can't be reused in f-strings. So we can simply\n                    # let the escaping logic below run without knowing f-string\n                    # expressions.\n\n            RE_EVEN_BACKSLASHES = r\"(?:(?<!\\\\)(?:\\\\\\\\)*)\"\n            naked_string = string[len(string_prefix) + 1 : -1]\n            naked_string = re.sub(\n                \"(\" + RE_EVEN_BACKSLASHES + \")\" + QUOTE, r\"\\1\\\\\" + QUOTE, naked_string\n            )\n            return naked_string\n\n        # Holds the CustomSplit objects that will later be added to the custom\n        # split map.\n        custom_splits = []\n\n        # Temporary storage for the 'has_prefix' part of the CustomSplit objects.\n        prefix_tracker = []\n\n        # Sets the 'prefix' variable. This is the prefix that the final merged\n        # string will have.\n        next_str_idx = string_idx\n        prefix = \"\"\n        while (\n            not prefix\n            and is_valid_index(next_str_idx)\n            and LL[next_str_idx].type == token.STRING\n        ):\n            prefix = get_string_prefix(LL[next_str_idx].value).lower()\n            next_str_idx += 1\n\n        # The next loop merges the string group. The final string will be\n        # contained in 'S'.\n        #\n        # The following convenience variables are used:\n        #\n        #   S: string\n        #   NS: naked string\n        #   SS: next string\n        #   NSS: naked next string\n        S = \"\"\n        NS = \"\"\n        num_of_strings = 0\n        next_str_idx = string_idx\n        while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:\n            num_of_strings += 1\n\n            SS = LL[next_str_idx].value\n            next_prefix = get_string_prefix(SS).lower()\n\n            # If this is an f-string group but this substring is not prefixed\n            # with 'f'...\n            if \"f\" in prefix and \"f\" not in next_prefix:\n                # Then we must escape any braces contained in this substring.\n                SS = re.sub(r\"(\\{|\\})\", r\"\\1\\1\", SS)\n\n            NSS = make_naked(SS, next_prefix)\n\n            has_prefix = bool(next_prefix)\n            prefix_tracker.append(has_prefix)\n\n            S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE\n            NS = make_naked(S, prefix)\n\n            next_str_idx += 1\n\n        # Take a note on the index of the non-STRING leaf.\n        non_string_idx = next_str_idx\n\n        S_leaf = Leaf(token.STRING, S)\n        if self.normalize_strings:\n            S_leaf.value = normalize_string_quotes(S_leaf.value)\n\n        # Fill the 'custom_splits' list with the appropriate CustomSplit objects.\n        temp_string = S_leaf.value[len(prefix) + 1 : -1]\n        for has_prefix in prefix_tracker:\n            mark_idx = temp_string.find(BREAK_MARK)\n            assert (\n                mark_idx >= 0\n            ), \"Logic error while filling the custom string breakpoint cache.\"\n\n            temp_string = temp_string[mark_idx + len(BREAK_MARK) :]\n            breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1\n            custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))\n\n        string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, \"\"))\n\n        if atom_node is not None:\n            # If not all children of the atom node are merged (this can happen\n            # when there is a standalone comment in the middle) ...\n            if non_string_idx - string_idx < len(atom_node.children):\n                # We need to replace the old STRING leaves with the new string leaf.\n                first_child_idx = LL[string_idx].remove()\n                for idx in range(string_idx + 1, non_string_idx):\n                    LL[idx].remove()\n                if first_child_idx is not None:\n                    atom_node.insert_child(first_child_idx, string_leaf)\n            else:\n                # Else replace the atom node with the new string leaf.\n                replace_child(atom_node, string_leaf)\n\n        self.add_custom_splits(string_leaf.value, custom_splits)\n        return num_of_strings, string_leaf\n\n    @staticmethod\n    def _validate_msg(line: Line, string_idx: int) -> TResult[None]:\n        \"\"\"Validate (M)erge (S)tring (G)roup\n\n        Transform-time string validation logic for _merge_string_group(...).\n\n        Returns:\n            * Ok(None), if ALL validation checks (listed below) pass.\n                OR\n            * Err(CannotTransform), if any of the following are true:\n                - The target string group does not contain ANY stand-alone comments.\n                - The target string is not in a string group (i.e. it has no\n                  adjacent strings).\n                - The string group has more than one inline comment.\n...\n# Path: src/black/linegen.py\n\"\"\"\nGenerating lines of code.\n\"\"\"\n\nimport re\nimport sys\nfrom dataclasses import replace\nfrom enum import Enum, auto\nfrom functools import partial, wraps\nfrom typing import Collection, Iterator, List, Optional, Set, Union, cast\n\nfrom black.brackets import (\n    COMMA_PRIORITY,\n    DOT_PRIORITY,\n    STRING_PRIORITY,\n    get_leaves_inside_matching_brackets,\n    max_delimiter_priority_in_atom,\n)\nfrom black.comments import FMT_OFF, generate_comments, list_comments\nfrom black.lines import (\n    Line,\n    RHSResult,\n    append_leaves,\n    can_be_split,\n    can_omit_invisible_parens,\n    is_line_short_enough,\n    line_to_string,\n)\nfrom black.mode import Feature, Mode, Preview\nfrom black.nodes import (\n    ASSIGNMENTS,\n    BRACKETS,\n    CLOSING_BRACKETS,\n    OPENING_BRACKETS,\n    STANDALONE_COMMENT,\n    STATEMENT,\n    WHITESPACE,\n    Visitor,\n    ensure_visible,\n    get_annotation_type,\n    is_arith_like,\n    is_async_stmt_or_funcdef,\n    is_atom_with_invisible_parens,\n    is_docstring,\n    is_empty_tuple,\n    is_lpar_token,\n    is_multiline_string,\n    is_name_token,\n    is_one_sequence_between,\n    is_one_tuple,\n    is_parent_function_or_class,\n    is_part_of_annotation,\n    is_rpar_token,\n    is_stub_body,\n    is_stub_suite,\n    is_tuple_containing_walrus,\n    is_type_ignore_comment_string,\n    is_vararg,\n    is_walrus_assignment,\n    is_yield,\n    syms,\n    wrap_in_parentheses,\n)\nfrom black.numerics import normalize_numeric_literal\nfrom black.strings import (\n    fix_docstring,\n    get_string_prefix,\n    normalize_string_prefix,\n    normalize_string_quotes,\n    normalize_unicode_escape_sequences,\n)\nfrom black.trans import (\n    CannotTransform,\n    StringMerger,\n    StringParenStripper,\n    StringParenWrapper,\n    StringSplitter,\n    Transformer,\n    hug_power_op,\n)\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\n# types\nLeafID = int\nLN = Union[Leaf, Node]\n\n\nclass CannotSplit(CannotTransform):\n    \"\"\"A readable split that fits the allotted line length is impossible.\"\"\"\n\n\n# This isn't a dataclass because @dataclass + Generic breaks mypyc.\n# See also https://github.com/mypyc/mypyc/issues/827.\nclass LineGenerator(Visitor[Line]):\n    \"\"\"Generates reformatted Line objects.  Empty lines are not emitted.\n\n    Note: destroys the tree it's visiting by mutating prefixes of its leaves\n    in ways that will no longer stringify to valid Python code on the tree.\n    \"\"\"\n\n    def __init__(self, mode: Mode, features: Collection[Feature]) -> None:\n        self.mode = mode\n        self.features = features\n        self.current_line: Line\n        self.__post_init__()\n\n    def line(self, indent: int = 0) -> Iterator[Line]:\n        \"\"\"Generate a line.\n\n        If the line is empty, only emit if it makes sense.\n        If the line is too long, split it first and then generate.\n\n        If any lines were generated, set up a new current_line.\n        \"\"\"\n        if not self.current_line:\n            self.current_line.depth += indent\n            return  # Line is empty, don't emit. Creating a new one unnecessary.\n\n        if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(\n            self.current_line.leaves[0]\n        ):\n            # Special case for async def/for/with statements. `visit_async_stmt`\n            # adds an `ASYNC` leaf then visits the child def/for/with statement\n            # nodes. Line yields from those nodes shouldn't treat the former\n            # `ASYNC` leaf as a complete line.\n            return\n\n        complete_line = self.current_line\n        self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)\n        yield complete_line\n\n    def visit_default(self, node: LN) -> Iterator[Line]:\n        \"\"\"Default `visit_*()` implementation. Recurses to children of `node`.\"\"\"\n        if isinstance(node, Leaf):\n            any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()\n            for comment in generate_comments(node):\n                if any_open_brackets:\n                    # any comment within brackets is subject to splitting\n                    self.current_line.append(comment)\n                elif comment.type == token.COMMENT:\n                    # regular trailing comment\n                    self.current_line.append(comment)\n                    yield from self.line()\n\n                else:\n                    # regular standalone comment\n                    yield from self.line()\n\n                    self.current_line.append(comment)\n                    yield from self.line()\n\n            if any_open_brackets:\n                node.prefix = \"\"\n            if self.mode.string_normalization and node.type == token.STRING:\n                node.value = normalize_string_prefix(node.value)\n                node.value = normalize_string_quotes(node.value)\n            if node.type == token.NUMBER:\n                normalize_numeric_literal(node)\n            if node.type not in WHITESPACE:\n                self.current_line.append(node)\n        yield from super().visit_default(node)\n\n    def visit_test(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit an `x if y else z` test\"\"\"\n\n        already_parenthesized = (\n            node.prev_sibling and node.prev_sibling.type == token.LPAR\n        )\n\n        if not already_parenthesized:\n            # Similar to logic in wrap_in_parentheses\n            lpar = Leaf(token.LPAR, \"\")\n            rpar = Leaf(token.RPAR, \"\")\n            prefix = node.prefix\n            node.prefix = \"\"\n            lpar.prefix = prefix\n            node.insert_child(0, lpar)\n            node.append_child(rpar)\n\n        yield from self.visit_default(node)\n\n    def visit_INDENT(self, node: Leaf) -> Iterator[Line]:\n        \"\"\"Increase indentation level, maybe yield a line.\"\"\"\n        # In blib2to3 INDENT never holds comments.\n        yield from self.line(+1)\n        yield from self.visit_default(node)\n\n    def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:\n        \"\"\"Decrease indentation level, maybe yield a line.\"\"\"\n        # The current line might still wait for trailing comments.  At DEDENT time\n        # there won't be any (they would be prefixes on the preceding NEWLINE).\n        # Emit the line then.\n        yield from self.line()\n\n        # While DEDENT has no value, its prefix may contain standalone comments\n        # that belong to the current indentation level.  Get 'em.\n        yield from self.visit_default(node)\n\n        # Finally, emit the dedent.\n        yield from self.line(-1)\n\n    def visit_stmt(\n        self, node: Node, keywords: Set[str], parens: Set[str]\n    ) -> Iterator[Line]:\n        \"\"\"Visit a statement.\n\n        This implementation is shared for `if`, `while`, `for`, `try`, `except`,\n        `def`, `with`, `class`, `assert`, and assignments.\n\n        The relevant Python language `keywords` for a given statement will be\n        NAME leaves within it. This methods puts those on a separate line.\n\n        `parens` holds a set of string leaf values immediately after which\n        invisible parens should be put.\n        \"\"\"\n        normalize_invisible_parens(\n            node, parens_after=parens, mode=self.mode, features=self.features\n        )\n        for child in node.children:\n            if is_name_token(child) and child.value in keywords:\n                yield from self.line()\n\n            yield from self.visit(child)\n\n    def visit_typeparams(self, node: Node) -> Iterator[Line]:\n        yield from self.visit_default(node)\n        node.children[0].prefix = \"\"\n\n    def visit_typevartuple(self, node: Node) -> Iterator[Line]:\n        yield from self.visit_default(node)\n        node.children[1].prefix = \"\"\n\n    def visit_paramspec(self, node: Node) -> Iterator[Line]:\n        yield from self.visit_default(node)\n        node.children[1].prefix = \"\"\n\n    def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:\n        if Preview.wrap_long_dict_values_in_parens in self.mode:\n            for i, child in enumerate(node.children):\n                if i == 0:\n                    continue\n                if node.children[i - 1].type == token.COLON:\n                    if (\n                        child.type == syms.atom\n                        and child.children[0].type in OPENING_BRACKETS\n                        and not is_walrus_assignment(child)\n                    ):\n                        maybe_make_parens_invisible_in_atom(\n                            child,\n                            parent=node,\n                            remove_brackets_around_comma=False,\n                        )\n                    else:\n                        wrap_in_parentheses(node, child, visible=False)\n        yield from self.visit_default(node)\n\n    def visit_funcdef(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit function definition.\"\"\"\n        yield from self.line()\n\n        # Remove redundant brackets around return type annotation.\n        is_return_annotation = False\n        for child in node.children:\n            if child.type == token.RARROW:\n                is_return_annotation = True\n            elif is_return_annotation:\n                if child.type == syms.atom and child.children[0].type == token.LPAR:\n                    if maybe_make_parens_invisible_in_atom(\n                        child,\n                        parent=node,\n                        remove_brackets_around_comma=False,\n                    ):\n                        wrap_in_parentheses(node, child, visible=False)\n                else:\n                    wrap_in_parentheses(node, child, visible=False)\n                is_return_annotation = False\n\n        for child in node.children:\n            yield from self.visit(child)\n\n    def visit_match_case(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit either a match or case statement.\"\"\"\n        normalize_invisible_parens(\n            node, parens_after=set(), mode=self.mode, features=self.features\n        )\n\n        yield from self.line()\n        for child in node.children:\n            yield from self.visit(child)\n\n    def visit_suite(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit a suite.\"\"\"\n        if is_stub_suite(node):\n            yield from self.visit(node.children[2])\n        else:\n            yield from self.visit_default(node)\n\n    def visit_simple_stmt(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit a statement without nested statements.\"\"\"\n        prev_type: Optional[int] = None\n        for child in node.children:\n            if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):\n                wrap_in_parentheses(node, child, visible=False)\n            prev_type = child.type\n\n        if node.parent and node.parent.type in STATEMENT:\n            if is_parent_function_or_class(node) and is_stub_body(node):\n                yield from self.visit_default(node)\n            else:\n                yield from self.line(+1)\n                yield from self.visit_default(node)\n                yield from self.line(-1)\n\n        else:\n            if not node.parent or not is_stub_suite(node.parent):\n                yield from self.line()\n            yield from self.visit_default(node)\n\n    def visit_async_stmt(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit `async def`, `async for`, `async with`.\"\"\"\n        yield from self.line()\n\n        children = iter(node.children)\n        for child in children:\n            yield from self.visit(child)\n\n            if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:\n                # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async\n                # line.\n                break\n\n        internal_stmt = next(children)\n        yield from self.visit(internal_stmt)\n\n    def visit_decorators(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit decorators.\"\"\"\n        for child in node.children:\n            yield from self.line()\n            yield from self.visit(child)\n\n    def visit_power(self, node: Node) -> Iterator[Line]:\n        for idx, leaf in enumerate(node.children[:-1]):\n            next_leaf = node.children[idx + 1]\n\n            if not isinstance(leaf, Leaf):\n                continue\n\n            value = leaf.value.lower()\n            if (\n                leaf.type == token.NUMBER\n                and next_leaf.type == syms.trailer\n                # Ensure that we are in an attribute trailer\n                and next_leaf.children[0].type == token.DOT\n                # It shouldn't wrap hexadecimal, binary and octal literals\n                and not value.startswith((\"0x\", \"0b\", \"0o\"))\n                # It shouldn't wrap complex literals\n                and \"j\" not in value\n            ):\n                wrap_in_parentheses(node, leaf)\n\n        remove_await_parens(node)\n\n        yield from self.visit_default(node)\n\n    def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:\n        \"\"\"Remove a semicolon and put the other statement on a separate line.\"\"\"\n        yield from self.line()\n\n    def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:\n        \"\"\"End of file. Process outstanding comments and end with a newline.\"\"\"\n        yield from self.visit_default(leaf)\n        yield from self.line()\n\n    def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:\n        if not self.current_line.bracket_tracker.any_open_brackets():\n            yield from self.line()\n        yield from self.visit_default(leaf)\n\n    def visit_factor(self, node: Node) -> Iterator[Line]:\n        \"\"\"Force parentheses between a unary op and a binary power:\n\n        -2 ** 8 -> -(2 ** 8)\n        \"\"\"\n        _operator, operand = node.children\n        if (\n            operand.type == syms.power\n            and len(operand.children) == 3\n            and operand.children[1].type == token.DOUBLESTAR\n        ):\n            lpar = Leaf(token.LPAR, \"(\")\n            rpar = Leaf(token.RPAR, \")\")\n            index = operand.remove() or 0\n            node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))\n        yield from self.visit_default(node)\n\n    def visit_tname(self, node: Node) -> Iterator[Line]:\n        \"\"\"\n        Add potential parentheses around types in function parameter lists to be made\n        into real parentheses in case the type hint is too long to fit on a line\n        Examples:\n        def foo(a: int, b: float = 7): ...\n\n        ->\n\n        def foo(a: (int), b: (float) = 7): ...\n        \"\"\"\n        assert len(node.children) == 3\n        if maybe_make_parens_invisible_in_atom(node.children[2], parent=node):\n            wrap_in_parentheses(node, node.children[2], visible=False)\n\n        yield from self.visit_default(node)\n\n    def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:\n        if Preview.hex_codes_in_unicode_sequences in self.mode:\n            normalize_unicode_escape_sequences(leaf)\n\n        if is_docstring(leaf, self.mode) and not re.search(r\"\\\\\\s*\\n\", leaf.value):\n            # We're ignoring docstrings with backslash newline escapes because changing\n            # indentation of those changes the AST representation of the code.\n            if self.mode.string_normalization:\n                docstring = normalize_string_prefix(leaf.value)\n                # visit_default() does handle string normalization for us, but\n                # since this method acts differently depending on quote style (ex.\n                # see padding logic below), there's a possibility for unstable\n                # formatting as visit_default() is called *after*. To avoid a\n                # situation where this function formats a docstring differently on\n                # the second pass, normalize it early.\n                docstring = normalize_string_quotes(docstring)\n            else:\n                docstring = leaf.value\n            prefix = get_string_prefix(docstring)\n            docstring = docstring[len(prefix) :]  # Remove the prefix\n            quote_char = docstring[0]\n            # A natural way to remove the outer quotes is to do:\n            #   docstring = docstring.strip(quote_char)\n            # but that breaks on \"\"\"\"\"x\"\"\" (which is '\"\"x').\n            # So we actually need to remove the first character and the next two\n            # characters but only if they are the same as the first.\n            quote_len = 1 if docstring[1] != quote_char else 3\n            docstring = docstring[quote_len:-quote_len]\n            docstring_started_empty = not docstring\n            indent = \" \" * 4 * self.current_line.depth\n\n            if is_multiline_string(leaf):\n                docstring = fix_docstring(docstring, indent)\n            else:\n                docstring = docstring.strip()\n\n            has_trailing_backslash = False\n            if docstring:\n                # Add some padding if the docstring starts / ends with a quote mark.\n                if docstring[0] == quote_char:\n                    docstring = \" \" + docstring\n                if docstring[-1] == quote_char:\n                    docstring += \" \"\n                if docstring[-1] == \"\\\\\":\n                    backslash_count = len(docstring) - len(docstring.rstrip(\"\\\\\"))\n                    if backslash_count % 2:\n                        # Odd number of tailing backslashes, add some padding to\n                        # avoid escaping the closing string quote.\n                        docstring += \" \"\n                        has_trailing_backslash = True\n            elif not docstring_started_empty:\n                docstring = \" \"\n\n            # We could enforce triple quotes at this point.\n            quote = quote_char * quote_len\n\n            # It's invalid to put closing single-character quotes on a new line.\n            if quote_len == 3:\n                # We need to find the length of the last line of the docstring\n                # to find if we can add the closing quotes to the line without\n                # exceeding the maximum line length.\n                # If docstring is one line, we don't put the closing quotes on a\n                # separate line because it looks ugly (#3320).\n                lines = docstring.splitlines()\n                last_line_length = len(lines[-1]) if docstring else 0\n\n                # If adding closing quotes would cause the last line to exceed\n                # the maximum line length, and the closing quote is not\n                # prefixed by a newline then put a line break before\n                # the closing quotes\n                if (\n                    len(lines) > 1\n                    and last_line_length + quote_len > self.mode.line_length\n                    and len(indent) + quote_len <= self.mode.line_length\n                    and not has_trailing_backslash\n                ):\n                    if (\n                        Preview.docstring_check_for_newline in self.mode\n                        and leaf.value[-1 - quote_len] == \"\\n\"\n                    ):\n                        leaf.value = prefix + quote + docstring + quote\n                    else:\n                        leaf.value = prefix + quote + docstring + \"\\n\" + indent + quote\n                else:\n                    leaf.value = prefix + quote + docstring + quote\n            else:\n                leaf.value = prefix + quote + docstring + quote\n\n        yield from self.visit_default(leaf)\n\n    \ndef __post_init__(self) -> None:\n        \"\"\"You are in a twisty little maze of passages.\"\"\"\n        self.current_line = Line(mode=self.mode)\n\n        v = self.visit_stmt\n        \u00d8: Set[str] = set()\n        self.visit_assert_stmt = partial(v, keywords={\"assert\"}, parens={\"assert\", \",\"})\n        self.visit_if_stmt = partial(\n            v, keywords={\"if\", \"else\", \"elif\"}, parens={\"if\", \"elif\"}\n        )\n        self.visit_while_stmt = partial(v, keywords={\"while\", \"else\"}, parens={\"while\"})\n        self.visit_for_stmt = partial(v, keywords={\"for\", \"else\"}, parens={\"for\", \"in\"})\n        self.visit_try_stmt = partial(\n            v, keywords={\"try\", \"except\", \"else\", \"finally\"}, parens=\u00d8\n        )\n        self.visit_except_clause = partial(v, keywords={\"except\"}, parens={\"except\"})\n        self.visit_with_stmt = partial(v, keywords={\"with\"}, parens={\"with\"})\n        self.visit_classdef = partial(v, keywords={\"class\"}, parens=\u00d8)\n\n        self.visit_expr_stmt = partial(v, keywords=\u00d8, parens=ASSIGNMENTS)\n        self.visit_return_stmt = partial(v, keywords={\"return\"}, parens={\"return\"})\n        self.visit_import_from = partial(v, keywords=\u00d8, parens={\"import\"})\n        self.visit_del_stmt = partial(v, keywords=\u00d8, parens={\"del\"})\n        self.visit_async_funcdef = self.visit_async_stmt\n        self.visit_decorated = self.visit_decorators\n\n        # PEP 634\n        self.visit_match_stmt = self.visit_match_case\n        self.visit_case_block = self.visit_match_case\n        if Preview.remove_redundant_guard_parens in self.mode:\n            self.visit_guard = partial(v, keywords=\u00d8, parens={\"if\"})\n\n\ndef _hugging_power_ops_line_to_string(\n    line: Line,\n    features: Collection[Feature],\n    mode: Mode,\n) -> Optional[str]:\n    try:\n        return line_to_string(next(hug_power_op(line, features, mode)))\n    except CannotTransform:\n        return None\n\n\ndef transform_line(\n    line: Line, mode: Mode, features: Collection[Feature] = ()\n) -> Iterator[Line]:\n    \"\"\"Transform a `line`, potentially splitting it into many lines.\n\n    They should fit in the allotted `line_length` but might not be able to.\n\n    `features` are syntactical features that may be used in the output.\n    \"\"\"\n    if line.is_comment:\n        yield line\n        return\n\n    line_str = line_to_string(line)\n\n    # We need the line string when power operators are hugging to determine if we should\n    # split the line. Default to line_str, if no power operator are present on the line.\n    line_str_hugging_power_ops = (\n        _hugging_power_ops_line_to_string(line, features, mode) or line_str\n    )\n\n    ll = mode.line_length\n    sn = mode.string_normalization\n    string_merge = StringMerger(ll, sn)\n    string_paren_strip = StringParenStripper(ll, sn)\n    string_split = StringSplitter(ll, sn)\n    string_paren_wrap = StringParenWrapper(ll, sn)\n\n    transformers: List[Transformer]\n    if (\n        not line.contains_uncollapsable_type_comments()\n        and not line.should_split_rhs\n        and not line.magic_trailing_comma\n        and (\n            is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)\n            or line.contains_unsplittable_type_ignore()\n        )\n        and not (line.inside_brackets and line.contains_standalone_comments())\n        and not line.contains_implicit_multiline_string_with_comments()\n    ):\n        # Only apply basic string preprocessing, since lines shouldn't be split here.\n        if Preview.string_processing in mode:\n            transformers = [string_merge, string_paren_strip]\n        else:\n            transformers = []\n    elif line.is_def and not should_split_funcdef_with_rhs(line, mode):\n        transformers = [left_hand_split]\n    else:\n\n        def _rhs(\n            self: object, line: Line, features: Collection[Feature], mode: Mode\n        ) -> Iterator[Line]:\n            \"\"\"Wraps calls to `right_hand_split`.\n\n            The calls increasingly `omit` right-hand trailers (bracket pairs with\n            content), meaning the trailers get glued together to split on another\n            bracket pair instead.\n            \"\"\"\n            for omit in generate_trailers_to_omit(line, mode.line_length):\n                lines = list(right_hand_split(line, mode, features, omit=omit))\n                # Note: this check is only able to figure out if the first line of the\n                # *current* transformation fits in the line length.  This is true only\n                # for simple cases.  All others require running more transforms via\n                # `transform_line()`.  This check doesn't know if those would succeed.\n                if is_line_short_enough(lines[0], mode=mode):\n                    yield from lines\n                    return\n\n            # All splits failed, best effort split with no omits.\n            # This mostly happens to multiline strings that are by definition\n            # reported as not fitting a single line, as well as lines that contain\n            # trailing commas (those have to be exploded).\n            yield from right_hand_split(line, mode, features=features)\n\n        # HACK: nested functions (like _rhs) compiled by mypyc don't retain their\n        # __name__ attribute which is needed in `run_transformer` further down.\n        # Unfortunately a nested class breaks mypyc too. So a class must be created\n        # via type ... https://github.com/mypyc/mypyc/issues/884\n        rhs = type(\"rhs\", (), {\"__call__\": _rhs})()\n\n        if Preview.string_processing in mode:\n            if line.inside_brackets:\n                transformers = [\n                    string_merge,\n                    string_paren_strip,\n                    string_split,\n                    delimiter_split,\n                    standalone_comment_split,\n                    string_paren_wrap,\n                    rhs,\n                ]\n            else:\n                transformers = [\n                    string_merge,\n                    string_paren_strip,\n                    string_split,\n                    string_paren_wrap,\n                    rhs,\n                ]\n        else:\n            if line.inside_brackets:\n                transformers = [delimiter_split, standalone_comment_split, rhs]\n            else:\n                transformers = [rhs]\n    # It's always safe to attempt hugging of power operations and pretty much every line\n    # could match.\n    transformers.append(hug_power_op)\n\n    for transform in transformers:\n        # We are accumulating lines in `result` because we might want to abort\n        # mission and return the original line in the end, or attempt a different\n        # split altogether.\n        try:\n            result = run_transformer(line, transform, mode, features, line_str=line_str)\n        except CannotTransform:\n            continue\n        else:\n            yield from result\n            break\n\n    else:\n        yield line\n\n\ndef should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:\n    \"\"\"If a funcdef has a magic trailing comma in the return type, then we should first\n    split the line with rhs to respect the comma.\n    \"\"\"\n    return_type_leaves: List[Leaf] = []\n    in_return_type = False\n\n    for leaf in line.leaves:\n        if leaf.type == token.COLON:\n            in_return_type = False\n        if in_return_type:\n            return_type_leaves.append(leaf)\n        if leaf.type == token.RARROW:\n            in_return_type = True\n\n    # using `bracket_split_build_line` will mess with whitespace, so we duplicate a\n    # couple lines from it.\n    result = Line(mode=line.mode, depth=line.depth)\n    leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)\n    for leaf in return_type_leaves:\n        result.append(\n            leaf,\n            preformatted=True,\n            track_bracket=id(leaf) in leaves_to_track,\n        )\n\n    # we could also return true if the line is too long, and the return type is longer\n    # than the param list. Or if `should_split_rhs` returns True.\n    return result.magic_trailing_comma is not None\n\n\nclass _BracketSplitComponent(Enum):\n    head = auto()\n    body = auto()\n    tail = auto()\n\n\ndef left_hand_split(\n    line: Line, _features: Collection[Feature], mode: Mode\n) -> Iterator[Line]:\n    \"\"\"Split line into many lines, starting with the first matching bracket pair.\n\n    Note: this usually looks weird, only use this for function definitions.\n    Prefer RHS otherwise.  This is why this function is not symmetrical with\n    :func:`right_hand_split` which also handles optional parentheses.\n    \"\"\"\n    tail_leaves: List[Leaf] = []\n    body_leaves: List[Leaf] = []\n    head_leaves: List[Leaf] = []\n    current_leaves = head_leaves\n    matching_bracket: Optional[Leaf] = None\n    for leaf in line.leaves:\n        if (\n            current_leaves is body_leaves\n            and leaf.type in CLOSING_BRACKETS\n            and leaf.opening_bracket is matching_bracket\n            and isinstance(matching_bracket, Leaf)\n        ):\n            ensure_visible(leaf)\n            ensure_visible(matching_bracket)\n            current_leaves = tail_leaves if body_leaves else head_leaves\n        current_leaves.append(leaf)\n        if current_leaves is head_leaves:\n            if leaf.type in OPENING_BRACKETS:\n                matching_bracket = leaf\n                current_leaves = body_leaves\n...\n", "needle_token_start": 13424, "needle_token_end": 13982, "code_context_ntokens": 16131, "output": ["```python\ndef __post_init__(self) -> None:\n    \"\"\"You are in a twisty little maze of passages.\"\"\"\n    self.current_line = Line(mode=self.mode)\n\n    v = self.visit_stmt\n    \u00d8: Set[str] = set()\n    self.visit_assert_stmt = partial(v, keywords={\"assert\"}, parens={\"assert\", \",\"})\n    self.visit_if_stmt = partial(\n        v, keywords={\"if\", \"else\", \"elif\"}, parens={\"if\", \"elif\"}\n    )\n    self.visit_while_stmt = partial(v, keywords={\"while\", \"else\"}, parens={\"while\"})\n    self.visit_for_stmt = partial(v, keywords={\"for\", \"else\"}, parens={\"for\", \"in\"})\n    self.visit_try_stmt = partial(\n        v, keywords={\"try\", \"except\", \"else\", \"finally\"}, parens=\u00d8\n    )\n    self.visit_except_clause = partial(v, keywords={\"except\"}, parens={\"except\"})\n    self.visit_with_stmt = partial(v, keywords={\"with\"}, parens={\"with\"})\n    self.visit_classdef = partial(v, keywords={\"class\"}, parens=\u00d8)\n\n    self.visit_expr_stmt = partial(v, keywords=\u00d8, parens=ASSIGNMENTS)\n    self.visit_return_stmt = partial(v, keywords={\"return\"}, parens={\"return\"})\n    self.visit_import_from = partial(v, keywords=\u00d8, parens={\"import\"})\n    self.visit_del_stmt = partial(v, keywords=\u00d8, parens={\"del\"})\n    self.visit_async_funcdef = self.visit_async_stmt\n    self.visit_decorated = self.visit_decorators\n\n    # PEP 634\n    self.visit_match_stmt = self.visit_match_case\n    self.visit_case_block = self.visit_match_case\n    if Preview.remove_redundant_guard_parens in self.mode:\n        self.visit_guard = partial(v, keywords=\u00d8, parens={\"if\"})\n```"]}
{"repo": "psf/black", "name": "should_split_funcdef_with_rhs", "language": "python", "path": "src/black/linegen.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine if a function definition line should be split based on the presence of a specific punctuation mark in the return type.\n2. **Input**: A line of code and a formatting mode.\n3. **Output**: A boolean value indicating whether the line should be split.\n4. **Procedure**: The function scans through the tokens of the line to identify the section that specifies the return type. It then checks if this section contains a specific punctuation mark that necessitates splitting the line. If found, the function returns true, indicating that the line should be split to properly format the return type according to the style rules.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/black/trans.py\n\"\"\"\nString transformers that can split and merge strings.\n\"\"\"\n\nimport re\nfrom abc import ABC, abstractmethod\nfrom collections import defaultdict\nfrom dataclasses import dataclass\nfrom typing import (\n    Any,\n    Callable,\n    ClassVar,\n    Collection,\n    Dict,\n    Final,\n    Iterable,\n    Iterator,\n    List,\n    Literal,\n    Optional,\n    Sequence,\n    Set,\n    Tuple,\n    TypeVar,\n    Union,\n)\n\nfrom mypy_extensions import trait\n\nfrom black.comments import contains_pragma_comment\nfrom black.lines import Line, append_leaves\nfrom black.mode import Feature, Mode, Preview\nfrom black.nodes import (\n    CLOSING_BRACKETS,\n    OPENING_BRACKETS,\n    STANDALONE_COMMENT,\n    is_empty_lpar,\n    is_empty_par,\n    is_empty_rpar,\n    is_part_of_annotation,\n    parent_type,\n    replace_child,\n    syms,\n)\nfrom black.rusty import Err, Ok, Result\nfrom black.strings import (\n    assert_is_leaf_string,\n    count_chars_in_width,\n    get_string_prefix,\n    has_triple_quotes,\n    normalize_string_quotes,\n    str_width,\n)\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\n\nclass CannotTransform(Exception):\n    \"\"\"Base class for errors raised by Transformers.\"\"\"\n\n\n# types\nT = TypeVar(\"T\")\nLN = Union[Leaf, Node]\nTransformer = Callable[[Line, Collection[Feature], Mode], Iterator[Line]]\nIndex = int\nNodeType = int\nParserState = int\nStringID = int\nTResult = Result[T, CannotTransform]  # (T)ransform Result\nTMatchResult = TResult[List[Index]]\n\nSPLIT_SAFE_CHARS = frozenset([\"\\u3001\", \"\\u3002\", \"\\uff0c\"])  # East Asian stops\n\n\ndef TErr(err_msg: str) -> Err[CannotTransform]:\n    \"\"\"(T)ransform Err\n\n    Convenience function used when working with the TResult type.\n    \"\"\"\n    cant_transform = CannotTransform(err_msg)\n    return Err(cant_transform)\n\n\ndef hug_power_op(\n    line: Line, features: Collection[Feature], mode: Mode\n) -> Iterator[Line]:\n    \"\"\"A transformer which normalizes spacing around power operators.\"\"\"\n\n    # Performance optimization to avoid unnecessary Leaf clones and other ops.\n    for leaf in line.leaves:\n        if leaf.type == token.DOUBLESTAR:\n            break\n    else:\n        raise CannotTransform(\"No doublestar token was found in the line.\")\n\n    def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool:\n        # Brackets and parentheses indicate calls, subscripts, etc. ...\n        # basically stuff that doesn't count as \"simple\". Only a NAME lookup\n        # or dotted lookup (eg. NAME.NAME) is OK.\n        if Preview.is_simple_lookup_for_doublestar_expression not in mode:\n            return original_is_simple_lookup_func(line, index, kind)\n\n        else:\n            if kind == -1:\n                return handle_is_simple_look_up_prev(\n                    line, index, {token.RPAR, token.RSQB}\n                )\n            else:\n                return handle_is_simple_lookup_forward(\n                    line, index, {token.LPAR, token.LSQB}\n                )\n\n    def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool:\n        # An operand is considered \"simple\" if's a NAME, a numeric CONSTANT, a simple\n        # lookup (see above), with or without a preceding unary operator.\n        start = line.leaves[index]\n        if start.type in {token.NAME, token.NUMBER}:\n            return is_simple_lookup(index, kind)\n\n        if start.type in {token.PLUS, token.MINUS, token.TILDE}:\n            if line.leaves[index + 1].type in {token.NAME, token.NUMBER}:\n                # kind is always one as bases with a preceding unary op will be checked\n                # for simplicity starting from the next token (so it'll hit the check\n                # above).\n                return is_simple_lookup(index + 1, kind=1)\n\n        return False\n\n    new_line = line.clone()\n    should_hug = False\n    for idx, leaf in enumerate(line.leaves):\n        new_leaf = leaf.clone()\n        if should_hug:\n            new_leaf.prefix = \"\"\n            should_hug = False\n\n        should_hug = (\n            (0 < idx < len(line.leaves) - 1)\n            and leaf.type == token.DOUBLESTAR\n            and is_simple_operand(idx - 1, kind=-1)\n            and line.leaves[idx - 1].value != \"lambda\"\n            and is_simple_operand(idx + 1, kind=1)\n        )\n        if should_hug:\n            new_leaf.prefix = \"\"\n\n        # We have to be careful to make a new line properly:\n        # - bracket related metadata must be maintained (handled by Line.append)\n        # - comments need to copied over, updating the leaf IDs they're attached to\n        new_line.append(new_leaf, preformatted=True)\n        for comment_leaf in line.comments_after(leaf):\n            new_line.append(comment_leaf, preformatted=True)\n\n    yield new_line\n\n\ndef original_is_simple_lookup_func(\n    line: Line, index: int, step: Literal[1, -1]\n) -> bool:\n    if step == -1:\n        disallowed = {token.RPAR, token.RSQB}\n    else:\n        disallowed = {token.LPAR, token.LSQB}\n\n    while 0 <= index < len(line.leaves):\n        current = line.leaves[index]\n        if current.type in disallowed:\n            return False\n        if current.type not in {token.NAME, token.DOT} or current.value == \"for\":\n            # If the current token isn't disallowed, we'll assume this is\n            # simple as only the disallowed tokens are semantically\n            # attached to this lookup expression we're checking. Also,\n            # stop early if we hit the 'for' bit of a comprehension.\n            return True\n\n        index += step\n\n    return True\n\n\ndef handle_is_simple_look_up_prev(line: Line, index: int, disallowed: Set[int]) -> bool:\n    \"\"\"\n    Handling the determination of is_simple_lookup for the lines prior to the doublestar\n    token. This is required because of the need to isolate the chained expression\n    to determine the bracket or parenthesis belong to the single expression.\n    \"\"\"\n    contains_disallowed = False\n    chain = []\n\n    while 0 <= index < len(line.leaves):\n        current = line.leaves[index]\n        chain.append(current)\n        if not contains_disallowed and current.type in disallowed:\n            contains_disallowed = True\n        if not is_expression_chained(chain):\n            return not contains_disallowed\n\n        index -= 1\n\n    return True\n\n\ndef handle_is_simple_lookup_forward(\n    line: Line, index: int, disallowed: Set[int]\n) -> bool:\n    \"\"\"\n    Handling decision is_simple_lookup for the lines behind the doublestar token.\n    This function is simplified to keep consistent with the prior logic and the forward\n    case are more straightforward and do not need to care about chained expressions.\n    \"\"\"\n    while 0 <= index < len(line.leaves):\n        current = line.leaves[index]\n        if current.type in disallowed:\n            return False\n        if current.type not in {token.NAME, token.DOT} or (\n            current.type == token.NAME and current.value == \"for\"\n        ):\n            # If the current token isn't disallowed, we'll assume this is simple as\n            # only the disallowed tokens are semantically attached to this lookup\n            # expression we're checking. Also, stop early if we hit the 'for' bit\n            # of a comprehension.\n            return True\n\n        index += 1\n\n    return True\n\n\ndef is_expression_chained(chained_leaves: List[Leaf]) -> bool:\n    \"\"\"\n    Function to determine if the variable is a chained call.\n    (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call)\n    \"\"\"\n    if len(chained_leaves) < 2:\n        return True\n\n    current_leaf = chained_leaves[-1]\n    past_leaf = chained_leaves[-2]\n\n    if past_leaf.type == token.NAME:\n        return current_leaf.type in {token.DOT}\n    elif past_leaf.type in {token.RPAR, token.RSQB}:\n        return current_leaf.type in {token.RSQB, token.RPAR}\n    elif past_leaf.type in {token.LPAR, token.LSQB}:\n        return current_leaf.type in {token.NAME, token.LPAR, token.LSQB}\n    else:\n        return False\n\n\nclass StringTransformer(ABC):\n    \"\"\"\n    An implementation of the Transformer protocol that relies on its\n    subclasses overriding the template methods `do_match(...)` and\n    `do_transform(...)`.\n\n    This Transformer works exclusively on strings (for example, by merging\n    or splitting them).\n\n    The following sections can be found among the docstrings of each concrete\n    StringTransformer subclass.\n\n    Requirements:\n        Which requirements must be met of the given Line for this\n        StringTransformer to be applied?\n\n    Transformations:\n        If the given Line meets all of the above requirements, which string\n        transformations can you expect to be applied to it by this\n        StringTransformer?\n\n    Collaborations:\n        What contractual agreements does this StringTransformer have with other\n        StringTransfomers? Such collaborations should be eliminated/minimized\n        as much as possible.\n    \"\"\"\n\n    __name__: Final = \"StringTransformer\"\n\n    # Ideally this would be a dataclass, but unfortunately mypyc breaks when used with\n    # `abc.ABC`.\n    def __init__(self, line_length: int, normalize_strings: bool) -> None:\n        self.line_length = line_length\n        self.normalize_strings = normalize_strings\n\n    @abstractmethod\n    def do_match(self, line: Line) -> TMatchResult:\n        \"\"\"\n        Returns:\n            * Ok(string_indices) such that for each index, `line.leaves[index]`\n              is our target string if a match was able to be made. For\n              transformers that don't result in more lines (e.g. StringMerger,\n              StringParenStripper), multiple matches and transforms are done at\n              once to reduce the complexity.\n              OR\n            * Err(CannotTransform), if no match could be made.\n        \"\"\"\n\n    @abstractmethod\n    def do_transform(\n        self, line: Line, string_indices: List[int]\n    ) -> Iterator[TResult[Line]]:\n        \"\"\"\n        Yields:\n            * Ok(new_line) where new_line is the new transformed line.\n              OR\n            * Err(CannotTransform) if the transformation failed for some reason. The\n              `do_match(...)` template method should usually be used to reject\n              the form of the given Line, but in some cases it is difficult to\n              know whether or not a Line meets the StringTransformer's\n              requirements until the transformation is already midway.\n\n        Side Effects:\n            This method should NOT mutate @line directly, but it MAY mutate the\n            Line's underlying Node structure. (WARNING: If the underlying Node\n            structure IS altered, then this method should NOT be allowed to\n            yield an CannotTransform after that point.)\n        \"\"\"\n\n    def __call__(\n        self, line: Line, _features: Collection[Feature], _mode: Mode\n    ) -> Iterator[Line]:\n        \"\"\"\n        StringTransformer instances have a call signature that mirrors that of\n        the Transformer type.\n\n        Raises:\n            CannotTransform(...) if the concrete StringTransformer class is unable\n            to transform @line.\n        \"\"\"\n        # Optimization to avoid calling `self.do_match(...)` when the line does\n        # not contain any string.\n        if not any(leaf.type == token.STRING for leaf in line.leaves):\n            raise CannotTransform(\"There are no strings in this line.\")\n\n        match_result = self.do_match(line)\n\n        if isinstance(match_result, Err):\n            cant_transform = match_result.err()\n            raise CannotTransform(\n                f\"The string transformer {self.__class__.__name__} does not recognize\"\n                \" this line as one that it can transform.\"\n            ) from cant_transform\n\n        string_indices = match_result.ok()\n\n        for line_result in self.do_transform(line, string_indices):\n            if isinstance(line_result, Err):\n                cant_transform = line_result.err()\n                raise CannotTransform(\n                    \"StringTransformer failed while attempting to transform string.\"\n                ) from cant_transform\n            line = line_result.ok()\n            yield line\n\n\n@dataclass\nclass CustomSplit:\n    \"\"\"A custom (i.e. manual) string split.\n\n    A single CustomSplit instance represents a single substring.\n\n    Examples:\n        Consider the following string:\n        ```\n        \"Hi there friend.\"\n        \" This is a custom\"\n        f\" string {split}.\"\n        ```\n\n        This string will correspond to the following three CustomSplit instances:\n        ```\n        CustomSplit(False, 16)\n        CustomSplit(False, 17)\n        CustomSplit(True, 16)\n        ```\n    \"\"\"\n\n    has_prefix: bool\n    break_idx: int\n\n\n@trait\nclass CustomSplitMapMixin:\n    \"\"\"\n    This mixin class is used to map merged strings to a sequence of\n    CustomSplits, which will then be used to re-split the strings iff none of\n    the resultant substrings go over the configured max line length.\n    \"\"\"\n\n    _Key: ClassVar = Tuple[StringID, str]\n    _CUSTOM_SPLIT_MAP: ClassVar[Dict[_Key, Tuple[CustomSplit, ...]]] = defaultdict(\n        tuple\n    )\n\n    @staticmethod\n    def _get_key(string: str) -> \"CustomSplitMapMixin._Key\":\n        \"\"\"\n        Returns:\n            A unique identifier that is used internally to map @string to a\n            group of custom splits.\n        \"\"\"\n        return (id(string), string)\n\n    def add_custom_splits(\n        self, string: str, custom_splits: Iterable[CustomSplit]\n    ) -> None:\n        \"\"\"Custom Split Map Setter Method\n\n        Side Effects:\n            Adds a mapping from @string to the custom splits @custom_splits.\n        \"\"\"\n        key = self._get_key(string)\n        self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)\n\n    def pop_custom_splits(self, string: str) -> List[CustomSplit]:\n        \"\"\"Custom Split Map Getter Method\n\n        Returns:\n            * A list of the custom splits that are mapped to @string, if any\n              exist.\n              OR\n            * [], otherwise.\n\n        Side Effects:\n            Deletes the mapping between @string and its associated custom\n            splits (which are returned to the caller).\n        \"\"\"\n        key = self._get_key(string)\n\n        custom_splits = self._CUSTOM_SPLIT_MAP[key]\n        del self._CUSTOM_SPLIT_MAP[key]\n\n        return list(custom_splits)\n\n    def has_custom_splits(self, string: str) -> bool:\n        \"\"\"\n        Returns:\n            True iff @string is associated with a set of custom splits.\n        \"\"\"\n        key = self._get_key(string)\n        return key in self._CUSTOM_SPLIT_MAP\n\n\nclass StringMerger(StringTransformer, CustomSplitMapMixin):\n    \"\"\"StringTransformer that merges strings together.\n\n    Requirements:\n        (A) The line contains adjacent strings such that ALL of the validation checks\n        listed in StringMerger._validate_msg(...)'s docstring pass.\n        OR\n        (B) The line contains a string which uses line continuation backslashes.\n\n    Transformations:\n        Depending on which of the two requirements above where met, either:\n\n        (A) The string group associated with the target string is merged.\n        OR\n        (B) All line-continuation backslashes are removed from the target string.\n\n    Collaborations:\n        StringMerger provides custom split information to StringSplitter.\n    \"\"\"\n\n    def do_match(self, line: Line) -> TMatchResult:\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        string_indices = []\n        idx = 0\n        while is_valid_index(idx):\n            leaf = LL[idx]\n            if (\n                leaf.type == token.STRING\n                and is_valid_index(idx + 1)\n                and LL[idx + 1].type == token.STRING\n            ):\n                # Let's check if the string group contains an inline comment\n                # If we have a comment inline, we don't merge the strings\n                contains_comment = False\n                i = idx\n                while is_valid_index(i):\n                    if LL[i].type != token.STRING:\n                        break\n                    if line.comments_after(LL[i]):\n                        contains_comment = True\n                        break\n                    i += 1\n\n                if not is_part_of_annotation(leaf) and not contains_comment:\n                    string_indices.append(idx)\n\n                # Advance to the next non-STRING leaf.\n                idx += 2\n                while is_valid_index(idx) and LL[idx].type == token.STRING:\n                    idx += 1\n\n            elif leaf.type == token.STRING and \"\\\\\\n\" in leaf.value:\n                string_indices.append(idx)\n                # Advance to the next non-STRING leaf.\n                idx += 1\n                while is_valid_index(idx) and LL[idx].type == token.STRING:\n                    idx += 1\n\n            else:\n                idx += 1\n\n        if string_indices:\n            return Ok(string_indices)\n        else:\n            return TErr(\"This line has no strings that need merging.\")\n\n    def do_transform(\n        self, line: Line, string_indices: List[int]\n    ) -> Iterator[TResult[Line]]:\n        new_line = line\n\n        rblc_result = self._remove_backslash_line_continuation_chars(\n            new_line, string_indices\n        )\n        if isinstance(rblc_result, Ok):\n            new_line = rblc_result.ok()\n\n        msg_result = self._merge_string_group(new_line, string_indices)\n        if isinstance(msg_result, Ok):\n            new_line = msg_result.ok()\n\n        if isinstance(rblc_result, Err) and isinstance(msg_result, Err):\n            msg_cant_transform = msg_result.err()\n            rblc_cant_transform = rblc_result.err()\n            cant_transform = CannotTransform(\n                \"StringMerger failed to merge any strings in this line.\"\n            )\n\n            # Chain the errors together using `__cause__`.\n            msg_cant_transform.__cause__ = rblc_cant_transform\n            cant_transform.__cause__ = msg_cant_transform\n\n            yield Err(cant_transform)\n        else:\n            yield Ok(new_line)\n\n    @staticmethod\n    def _remove_backslash_line_continuation_chars(\n        line: Line, string_indices: List[int]\n    ) -> TResult[Line]:\n        \"\"\"\n        Merge strings that were split across multiple lines using\n        line-continuation backslashes.\n\n        Returns:\n            Ok(new_line), if @line contains backslash line-continuation\n            characters.\n                OR\n            Err(CannotTransform), otherwise.\n        \"\"\"\n        LL = line.leaves\n\n        indices_to_transform = []\n        for string_idx in string_indices:\n            string_leaf = LL[string_idx]\n            if (\n                string_leaf.type == token.STRING\n                and \"\\\\\\n\" in string_leaf.value\n                and not has_triple_quotes(string_leaf.value)\n            ):\n                indices_to_transform.append(string_idx)\n\n        if not indices_to_transform:\n            return TErr(\n                \"Found no string leaves that contain backslash line continuation\"\n                \" characters.\"\n            )\n\n        new_line = line.clone()\n        new_line.comments = line.comments.copy()\n        append_leaves(new_line, line, LL)\n\n        for string_idx in indices_to_transform:\n            new_string_leaf = new_line.leaves[string_idx]\n            new_string_leaf.value = new_string_leaf.value.replace(\"\\\\\\n\", \"\")\n\n        return Ok(new_line)\n\n    def _merge_string_group(\n        self, line: Line, string_indices: List[int]\n    ) -> TResult[Line]:\n        \"\"\"\n        Merges string groups (i.e. set of adjacent strings).\n\n        Each index from `string_indices` designates one string group's first\n        leaf in `line.leaves`.\n\n        Returns:\n            Ok(new_line), if ALL of the validation checks found in\n            _validate_msg(...) pass.\n                OR\n            Err(CannotTransform), otherwise.\n        \"\"\"\n        LL = line.leaves\n\n        is_valid_index = is_valid_index_factory(LL)\n\n        # A dict of {string_idx: tuple[num_of_strings, string_leaf]}.\n        merged_string_idx_dict: Dict[int, Tuple[int, Leaf]] = {}\n        for string_idx in string_indices:\n            vresult = self._validate_msg(line, string_idx)\n            if isinstance(vresult, Err):\n                continue\n            merged_string_idx_dict[string_idx] = self._merge_one_string_group(\n                LL, string_idx, is_valid_index\n            )\n\n        if not merged_string_idx_dict:\n            return TErr(\"No string group is merged\")\n\n        # Build the final line ('new_line') that this method will later return.\n        new_line = line.clone()\n        previous_merged_string_idx = -1\n        previous_merged_num_of_strings = -1\n        for i, leaf in enumerate(LL):\n            if i in merged_string_idx_dict:\n                previous_merged_string_idx = i\n                previous_merged_num_of_strings, string_leaf = merged_string_idx_dict[i]\n                new_line.append(string_leaf)\n\n            if (\n                previous_merged_string_idx\n                <= i\n                < previous_merged_string_idx + previous_merged_num_of_strings\n            ):\n                for comment_leaf in line.comments_after(LL[i]):\n                    new_line.append(comment_leaf, preformatted=True)\n                continue\n\n            append_leaves(new_line, line, [leaf])\n\n        return Ok(new_line)\n\n    def _merge_one_string_group(\n        self, LL: List[Leaf], string_idx: int, is_valid_index: Callable[[int], bool]\n    ) -> Tuple[int, Leaf]:\n        \"\"\"\n        Merges one string group where the first string in the group is\n        `LL[string_idx]`.\n\n        Returns:\n            A tuple of `(num_of_strings, leaf)` where `num_of_strings` is the\n            number of strings merged and `leaf` is the newly merged string\n            to be replaced in the new line.\n        \"\"\"\n        # If the string group is wrapped inside an Atom node, we must make sure\n        # to later replace that Atom with our new (merged) string leaf.\n        atom_node = LL[string_idx].parent\n\n        # We will place BREAK_MARK in between every two substrings that we\n        # merge. We will then later go through our final result and use the\n        # various instances of BREAK_MARK we find to add the right values to\n        # the custom split map.\n        BREAK_MARK = \"@@@@@ BLACK BREAKPOINT MARKER @@@@@\"\n\n        QUOTE = LL[string_idx].value[-1]\n\n        def make_naked(string: str, string_prefix: str) -> str:\n            \"\"\"Strip @string (i.e. make it a \"naked\" string)\n\n            Pre-conditions:\n                * assert_is_leaf_string(@string)\n\n            Returns:\n                A string that is identical to @string except that\n                @string_prefix has been stripped, the surrounding QUOTE\n                characters have been removed, and any remaining QUOTE\n                characters have been escaped.\n            \"\"\"\n            assert_is_leaf_string(string)\n            if \"f\" in string_prefix:\n                f_expressions = (\n                    string[span[0] + 1 : span[1] - 1]  # +-1 to get rid of curly braces\n                    for span in iter_fexpr_spans(string)\n                )\n                debug_expressions_contain_visible_quotes = any(\n                    re.search(r\".*[\\'\\\"].*(?<![!:=])={1}(?!=)(?![^\\s:])\", expression)\n                    for expression in f_expressions\n                )\n                if not debug_expressions_contain_visible_quotes:\n                    # We don't want to toggle visible quotes in debug f-strings, as\n                    # that would modify the AST\n                    string = _toggle_fexpr_quotes(string, QUOTE)\n                    # After quotes toggling, quotes in expressions won't be escaped\n                    # because quotes can't be reused in f-strings. So we can simply\n                    # let the escaping logic below run without knowing f-string\n                    # expressions.\n\n            RE_EVEN_BACKSLASHES = r\"(?:(?<!\\\\)(?:\\\\\\\\)*)\"\n            naked_string = string[len(string_prefix) + 1 : -1]\n            naked_string = re.sub(\n                \"(\" + RE_EVEN_BACKSLASHES + \")\" + QUOTE, r\"\\1\\\\\" + QUOTE, naked_string\n            )\n            return naked_string\n\n        # Holds the CustomSplit objects that will later be added to the custom\n        # split map.\n        custom_splits = []\n\n        # Temporary storage for the 'has_prefix' part of the CustomSplit objects.\n        prefix_tracker = []\n\n        # Sets the 'prefix' variable. This is the prefix that the final merged\n        # string will have.\n        next_str_idx = string_idx\n        prefix = \"\"\n        while (\n            not prefix\n            and is_valid_index(next_str_idx)\n            and LL[next_str_idx].type == token.STRING\n        ):\n            prefix = get_string_prefix(LL[next_str_idx].value).lower()\n            next_str_idx += 1\n\n        # The next loop merges the string group. The final string will be\n        # contained in 'S'.\n        #\n        # The following convenience variables are used:\n        #\n        #   S: string\n        #   NS: naked string\n        #   SS: next string\n        #   NSS: naked next string\n        S = \"\"\n        NS = \"\"\n        num_of_strings = 0\n        next_str_idx = string_idx\n        while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:\n            num_of_strings += 1\n\n            SS = LL[next_str_idx].value\n            next_prefix = get_string_prefix(SS).lower()\n\n            # If this is an f-string group but this substring is not prefixed\n            # with 'f'...\n            if \"f\" in prefix and \"f\" not in next_prefix:\n                # Then we must escape any braces contained in this substring.\n                SS = re.sub(r\"(\\{|\\})\", r\"\\1\\1\", SS)\n\n            NSS = make_naked(SS, next_prefix)\n\n            has_prefix = bool(next_prefix)\n            prefix_tracker.append(has_prefix)\n\n            S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE\n            NS = make_naked(S, prefix)\n\n            next_str_idx += 1\n\n        # Take a note on the index of the non-STRING leaf.\n        non_string_idx = next_str_idx\n\n        S_leaf = Leaf(token.STRING, S)\n        if self.normalize_strings:\n            S_leaf.value = normalize_string_quotes(S_leaf.value)\n\n        # Fill the 'custom_splits' list with the appropriate CustomSplit objects.\n        temp_string = S_leaf.value[len(prefix) + 1 : -1]\n        for has_prefix in prefix_tracker:\n            mark_idx = temp_string.find(BREAK_MARK)\n            assert (\n                mark_idx >= 0\n            ), \"Logic error while filling the custom string breakpoint cache.\"\n\n            temp_string = temp_string[mark_idx + len(BREAK_MARK) :]\n            breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1\n            custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))\n\n        string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, \"\"))\n\n        if atom_node is not None:\n            # If not all children of the atom node are merged (this can happen\n            # when there is a standalone comment in the middle) ...\n            if non_string_idx - string_idx < len(atom_node.children):\n                # We need to replace the old STRING leaves with the new string leaf.\n                first_child_idx = LL[string_idx].remove()\n                for idx in range(string_idx + 1, non_string_idx):\n                    LL[idx].remove()\n                if first_child_idx is not None:\n...\n# Path: src/black/linegen.py\n\"\"\"\nGenerating lines of code.\n\"\"\"\n\nimport re\nimport sys\nfrom dataclasses import replace\nfrom enum import Enum, auto\nfrom functools import partial, wraps\nfrom typing import Collection, Iterator, List, Optional, Set, Union, cast\n\nfrom black.brackets import (\n    COMMA_PRIORITY,\n    DOT_PRIORITY,\n    STRING_PRIORITY,\n    get_leaves_inside_matching_brackets,\n    max_delimiter_priority_in_atom,\n)\nfrom black.comments import FMT_OFF, generate_comments, list_comments\nfrom black.lines import (\n    Line,\n    RHSResult,\n    append_leaves,\n    can_be_split,\n    can_omit_invisible_parens,\n    is_line_short_enough,\n    line_to_string,\n)\nfrom black.mode import Feature, Mode, Preview\nfrom black.nodes import (\n    ASSIGNMENTS,\n    BRACKETS,\n    CLOSING_BRACKETS,\n    OPENING_BRACKETS,\n    STANDALONE_COMMENT,\n    STATEMENT,\n    WHITESPACE,\n    Visitor,\n    ensure_visible,\n    get_annotation_type,\n    is_arith_like,\n    is_async_stmt_or_funcdef,\n    is_atom_with_invisible_parens,\n    is_docstring,\n    is_empty_tuple,\n    is_lpar_token,\n    is_multiline_string,\n    is_name_token,\n    is_one_sequence_between,\n    is_one_tuple,\n    is_parent_function_or_class,\n    is_part_of_annotation,\n    is_rpar_token,\n    is_stub_body,\n    is_stub_suite,\n    is_tuple_containing_walrus,\n    is_type_ignore_comment_string,\n    is_vararg,\n    is_walrus_assignment,\n    is_yield,\n    syms,\n    wrap_in_parentheses,\n)\nfrom black.numerics import normalize_numeric_literal\nfrom black.strings import (\n    fix_docstring,\n    get_string_prefix,\n    normalize_string_prefix,\n    normalize_string_quotes,\n    normalize_unicode_escape_sequences,\n)\nfrom black.trans import (\n    CannotTransform,\n    StringMerger,\n    StringParenStripper,\n    StringParenWrapper,\n    StringSplitter,\n    Transformer,\n    hug_power_op,\n)\nfrom blib2to3.pgen2 import token\nfrom blib2to3.pytree import Leaf, Node\n\n# types\nLeafID = int\nLN = Union[Leaf, Node]\n\n\nclass CannotSplit(CannotTransform):\n    \"\"\"A readable split that fits the allotted line length is impossible.\"\"\"\n\n\n# This isn't a dataclass because @dataclass + Generic breaks mypyc.\n# See also https://github.com/mypyc/mypyc/issues/827.\nclass LineGenerator(Visitor[Line]):\n    \"\"\"Generates reformatted Line objects.  Empty lines are not emitted.\n\n    Note: destroys the tree it's visiting by mutating prefixes of its leaves\n    in ways that will no longer stringify to valid Python code on the tree.\n    \"\"\"\n\n    def __init__(self, mode: Mode, features: Collection[Feature]) -> None:\n        self.mode = mode\n        self.features = features\n        self.current_line: Line\n        self.__post_init__()\n\n    def line(self, indent: int = 0) -> Iterator[Line]:\n        \"\"\"Generate a line.\n\n        If the line is empty, only emit if it makes sense.\n        If the line is too long, split it first and then generate.\n\n        If any lines were generated, set up a new current_line.\n        \"\"\"\n        if not self.current_line:\n            self.current_line.depth += indent\n            return  # Line is empty, don't emit. Creating a new one unnecessary.\n\n        if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef(\n            self.current_line.leaves[0]\n        ):\n            # Special case for async def/for/with statements. `visit_async_stmt`\n            # adds an `ASYNC` leaf then visits the child def/for/with statement\n            # nodes. Line yields from those nodes shouldn't treat the former\n            # `ASYNC` leaf as a complete line.\n            return\n\n        complete_line = self.current_line\n        self.current_line = Line(mode=self.mode, depth=complete_line.depth + indent)\n        yield complete_line\n\n    def visit_default(self, node: LN) -> Iterator[Line]:\n        \"\"\"Default `visit_*()` implementation. Recurses to children of `node`.\"\"\"\n        if isinstance(node, Leaf):\n            any_open_brackets = self.current_line.bracket_tracker.any_open_brackets()\n            for comment in generate_comments(node):\n                if any_open_brackets:\n                    # any comment within brackets is subject to splitting\n                    self.current_line.append(comment)\n                elif comment.type == token.COMMENT:\n                    # regular trailing comment\n                    self.current_line.append(comment)\n                    yield from self.line()\n\n                else:\n                    # regular standalone comment\n                    yield from self.line()\n\n                    self.current_line.append(comment)\n                    yield from self.line()\n\n            if any_open_brackets:\n                node.prefix = \"\"\n            if self.mode.string_normalization and node.type == token.STRING:\n                node.value = normalize_string_prefix(node.value)\n                node.value = normalize_string_quotes(node.value)\n            if node.type == token.NUMBER:\n                normalize_numeric_literal(node)\n            if node.type not in WHITESPACE:\n                self.current_line.append(node)\n        yield from super().visit_default(node)\n\n    def visit_test(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit an `x if y else z` test\"\"\"\n\n        already_parenthesized = (\n            node.prev_sibling and node.prev_sibling.type == token.LPAR\n        )\n\n        if not already_parenthesized:\n            # Similar to logic in wrap_in_parentheses\n            lpar = Leaf(token.LPAR, \"\")\n            rpar = Leaf(token.RPAR, \"\")\n            prefix = node.prefix\n            node.prefix = \"\"\n            lpar.prefix = prefix\n            node.insert_child(0, lpar)\n            node.append_child(rpar)\n\n        yield from self.visit_default(node)\n\n    def visit_INDENT(self, node: Leaf) -> Iterator[Line]:\n        \"\"\"Increase indentation level, maybe yield a line.\"\"\"\n        # In blib2to3 INDENT never holds comments.\n        yield from self.line(+1)\n        yield from self.visit_default(node)\n\n    def visit_DEDENT(self, node: Leaf) -> Iterator[Line]:\n        \"\"\"Decrease indentation level, maybe yield a line.\"\"\"\n        # The current line might still wait for trailing comments.  At DEDENT time\n        # there won't be any (they would be prefixes on the preceding NEWLINE).\n        # Emit the line then.\n        yield from self.line()\n\n        # While DEDENT has no value, its prefix may contain standalone comments\n        # that belong to the current indentation level.  Get 'em.\n        yield from self.visit_default(node)\n\n        # Finally, emit the dedent.\n        yield from self.line(-1)\n\n    def visit_stmt(\n        self, node: Node, keywords: Set[str], parens: Set[str]\n    ) -> Iterator[Line]:\n        \"\"\"Visit a statement.\n\n        This implementation is shared for `if`, `while`, `for`, `try`, `except`,\n        `def`, `with`, `class`, `assert`, and assignments.\n\n        The relevant Python language `keywords` for a given statement will be\n        NAME leaves within it. This methods puts those on a separate line.\n\n        `parens` holds a set of string leaf values immediately after which\n        invisible parens should be put.\n        \"\"\"\n        normalize_invisible_parens(\n            node, parens_after=parens, mode=self.mode, features=self.features\n        )\n        for child in node.children:\n            if is_name_token(child) and child.value in keywords:\n                yield from self.line()\n\n            yield from self.visit(child)\n\n    def visit_typeparams(self, node: Node) -> Iterator[Line]:\n        yield from self.visit_default(node)\n        node.children[0].prefix = \"\"\n\n    def visit_typevartuple(self, node: Node) -> Iterator[Line]:\n        yield from self.visit_default(node)\n        node.children[1].prefix = \"\"\n\n    def visit_paramspec(self, node: Node) -> Iterator[Line]:\n        yield from self.visit_default(node)\n        node.children[1].prefix = \"\"\n\n    def visit_dictsetmaker(self, node: Node) -> Iterator[Line]:\n        if Preview.wrap_long_dict_values_in_parens in self.mode:\n            for i, child in enumerate(node.children):\n                if i == 0:\n                    continue\n                if node.children[i - 1].type == token.COLON:\n                    if (\n                        child.type == syms.atom\n                        and child.children[0].type in OPENING_BRACKETS\n                        and not is_walrus_assignment(child)\n                    ):\n                        maybe_make_parens_invisible_in_atom(\n                            child,\n                            parent=node,\n                            remove_brackets_around_comma=False,\n                        )\n                    else:\n                        wrap_in_parentheses(node, child, visible=False)\n        yield from self.visit_default(node)\n\n    def visit_funcdef(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit function definition.\"\"\"\n        yield from self.line()\n\n        # Remove redundant brackets around return type annotation.\n        is_return_annotation = False\n        for child in node.children:\n            if child.type == token.RARROW:\n                is_return_annotation = True\n            elif is_return_annotation:\n                if child.type == syms.atom and child.children[0].type == token.LPAR:\n                    if maybe_make_parens_invisible_in_atom(\n                        child,\n                        parent=node,\n                        remove_brackets_around_comma=False,\n                    ):\n                        wrap_in_parentheses(node, child, visible=False)\n                else:\n                    wrap_in_parentheses(node, child, visible=False)\n                is_return_annotation = False\n\n        for child in node.children:\n            yield from self.visit(child)\n\n    def visit_match_case(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit either a match or case statement.\"\"\"\n        normalize_invisible_parens(\n            node, parens_after=set(), mode=self.mode, features=self.features\n        )\n\n        yield from self.line()\n        for child in node.children:\n            yield from self.visit(child)\n\n    def visit_suite(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit a suite.\"\"\"\n        if is_stub_suite(node):\n            yield from self.visit(node.children[2])\n        else:\n            yield from self.visit_default(node)\n\n    def visit_simple_stmt(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit a statement without nested statements.\"\"\"\n        prev_type: Optional[int] = None\n        for child in node.children:\n            if (prev_type is None or prev_type == token.SEMI) and is_arith_like(child):\n                wrap_in_parentheses(node, child, visible=False)\n            prev_type = child.type\n\n        if node.parent and node.parent.type in STATEMENT:\n            if is_parent_function_or_class(node) and is_stub_body(node):\n                yield from self.visit_default(node)\n            else:\n                yield from self.line(+1)\n                yield from self.visit_default(node)\n                yield from self.line(-1)\n\n        else:\n            if not node.parent or not is_stub_suite(node.parent):\n                yield from self.line()\n            yield from self.visit_default(node)\n\n    def visit_async_stmt(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit `async def`, `async for`, `async with`.\"\"\"\n        yield from self.line()\n\n        children = iter(node.children)\n        for child in children:\n            yield from self.visit(child)\n\n            if child.type == token.ASYNC or child.type == STANDALONE_COMMENT:\n                # STANDALONE_COMMENT happens when `# fmt: skip` is applied on the async\n                # line.\n                break\n\n        internal_stmt = next(children)\n        yield from self.visit(internal_stmt)\n\n    def visit_decorators(self, node: Node) -> Iterator[Line]:\n        \"\"\"Visit decorators.\"\"\"\n        for child in node.children:\n            yield from self.line()\n            yield from self.visit(child)\n\n    def visit_power(self, node: Node) -> Iterator[Line]:\n        for idx, leaf in enumerate(node.children[:-1]):\n            next_leaf = node.children[idx + 1]\n\n            if not isinstance(leaf, Leaf):\n                continue\n\n            value = leaf.value.lower()\n            if (\n                leaf.type == token.NUMBER\n                and next_leaf.type == syms.trailer\n                # Ensure that we are in an attribute trailer\n                and next_leaf.children[0].type == token.DOT\n                # It shouldn't wrap hexadecimal, binary and octal literals\n                and not value.startswith((\"0x\", \"0b\", \"0o\"))\n                # It shouldn't wrap complex literals\n                and \"j\" not in value\n            ):\n                wrap_in_parentheses(node, leaf)\n\n        remove_await_parens(node)\n\n        yield from self.visit_default(node)\n\n    def visit_SEMI(self, leaf: Leaf) -> Iterator[Line]:\n        \"\"\"Remove a semicolon and put the other statement on a separate line.\"\"\"\n        yield from self.line()\n\n    def visit_ENDMARKER(self, leaf: Leaf) -> Iterator[Line]:\n        \"\"\"End of file. Process outstanding comments and end with a newline.\"\"\"\n        yield from self.visit_default(leaf)\n        yield from self.line()\n\n    def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]:\n        if not self.current_line.bracket_tracker.any_open_brackets():\n            yield from self.line()\n        yield from self.visit_default(leaf)\n\n    def visit_factor(self, node: Node) -> Iterator[Line]:\n        \"\"\"Force parentheses between a unary op and a binary power:\n\n        -2 ** 8 -> -(2 ** 8)\n        \"\"\"\n        _operator, operand = node.children\n        if (\n            operand.type == syms.power\n            and len(operand.children) == 3\n            and operand.children[1].type == token.DOUBLESTAR\n        ):\n            lpar = Leaf(token.LPAR, \"(\")\n            rpar = Leaf(token.RPAR, \")\")\n            index = operand.remove() or 0\n            node.insert_child(index, Node(syms.atom, [lpar, operand, rpar]))\n        yield from self.visit_default(node)\n\n    def visit_tname(self, node: Node) -> Iterator[Line]:\n        \"\"\"\n        Add potential parentheses around types in function parameter lists to be made\n        into real parentheses in case the type hint is too long to fit on a line\n        Examples:\n        def foo(a: int, b: float = 7): ...\n\n        ->\n\n        def foo(a: (int), b: (float) = 7): ...\n        \"\"\"\n        assert len(node.children) == 3\n        if maybe_make_parens_invisible_in_atom(node.children[2], parent=node):\n            wrap_in_parentheses(node, node.children[2], visible=False)\n\n        yield from self.visit_default(node)\n\n    def visit_STRING(self, leaf: Leaf) -> Iterator[Line]:\n        if Preview.hex_codes_in_unicode_sequences in self.mode:\n            normalize_unicode_escape_sequences(leaf)\n\n        if is_docstring(leaf, self.mode) and not re.search(r\"\\\\\\s*\\n\", leaf.value):\n            # We're ignoring docstrings with backslash newline escapes because changing\n            # indentation of those changes the AST representation of the code.\n            if self.mode.string_normalization:\n                docstring = normalize_string_prefix(leaf.value)\n                # visit_default() does handle string normalization for us, but\n                # since this method acts differently depending on quote style (ex.\n                # see padding logic below), there's a possibility for unstable\n                # formatting as visit_default() is called *after*. To avoid a\n                # situation where this function formats a docstring differently on\n                # the second pass, normalize it early.\n                docstring = normalize_string_quotes(docstring)\n            else:\n                docstring = leaf.value\n            prefix = get_string_prefix(docstring)\n            docstring = docstring[len(prefix) :]  # Remove the prefix\n            quote_char = docstring[0]\n            # A natural way to remove the outer quotes is to do:\n            #   docstring = docstring.strip(quote_char)\n            # but that breaks on \"\"\"\"\"x\"\"\" (which is '\"\"x').\n            # So we actually need to remove the first character and the next two\n            # characters but only if they are the same as the first.\n            quote_len = 1 if docstring[1] != quote_char else 3\n            docstring = docstring[quote_len:-quote_len]\n            docstring_started_empty = not docstring\n            indent = \" \" * 4 * self.current_line.depth\n\n            if is_multiline_string(leaf):\n                docstring = fix_docstring(docstring, indent)\n            else:\n                docstring = docstring.strip()\n\n            has_trailing_backslash = False\n            if docstring:\n                # Add some padding if the docstring starts / ends with a quote mark.\n                if docstring[0] == quote_char:\n                    docstring = \" \" + docstring\n                if docstring[-1] == quote_char:\n                    docstring += \" \"\n                if docstring[-1] == \"\\\\\":\n                    backslash_count = len(docstring) - len(docstring.rstrip(\"\\\\\"))\n                    if backslash_count % 2:\n                        # Odd number of tailing backslashes, add some padding to\n                        # avoid escaping the closing string quote.\n                        docstring += \" \"\n                        has_trailing_backslash = True\n            elif not docstring_started_empty:\n                docstring = \" \"\n\n            # We could enforce triple quotes at this point.\n            quote = quote_char * quote_len\n\n            # It's invalid to put closing single-character quotes on a new line.\n            if quote_len == 3:\n                # We need to find the length of the last line of the docstring\n                # to find if we can add the closing quotes to the line without\n                # exceeding the maximum line length.\n                # If docstring is one line, we don't put the closing quotes on a\n                # separate line because it looks ugly (#3320).\n                lines = docstring.splitlines()\n                last_line_length = len(lines[-1]) if docstring else 0\n\n                # If adding closing quotes would cause the last line to exceed\n                # the maximum line length, and the closing quote is not\n                # prefixed by a newline then put a line break before\n                # the closing quotes\n                if (\n                    len(lines) > 1\n                    and last_line_length + quote_len > self.mode.line_length\n                    and len(indent) + quote_len <= self.mode.line_length\n                    and not has_trailing_backslash\n                ):\n                    if (\n                        Preview.docstring_check_for_newline in self.mode\n                        and leaf.value[-1 - quote_len] == \"\\n\"\n                    ):\n                        leaf.value = prefix + quote + docstring + quote\n                    else:\n                        leaf.value = prefix + quote + docstring + \"\\n\" + indent + quote\n                else:\n                    leaf.value = prefix + quote + docstring + quote\n            else:\n                leaf.value = prefix + quote + docstring + quote\n\n        yield from self.visit_default(leaf)\n\n    def __post_init__(self) -> None:\n        \"\"\"You are in a twisty little maze of passages.\"\"\"\n        self.current_line = Line(mode=self.mode)\n\n        v = self.visit_stmt\n        \u00d8: Set[str] = set()\n        self.visit_assert_stmt = partial(v, keywords={\"assert\"}, parens={\"assert\", \",\"})\n        self.visit_if_stmt = partial(\n            v, keywords={\"if\", \"else\", \"elif\"}, parens={\"if\", \"elif\"}\n        )\n        self.visit_while_stmt = partial(v, keywords={\"while\", \"else\"}, parens={\"while\"})\n        self.visit_for_stmt = partial(v, keywords={\"for\", \"else\"}, parens={\"for\", \"in\"})\n        self.visit_try_stmt = partial(\n            v, keywords={\"try\", \"except\", \"else\", \"finally\"}, parens=\u00d8\n        )\n        self.visit_except_clause = partial(v, keywords={\"except\"}, parens={\"except\"})\n        self.visit_with_stmt = partial(v, keywords={\"with\"}, parens={\"with\"})\n        self.visit_classdef = partial(v, keywords={\"class\"}, parens=\u00d8)\n\n        self.visit_expr_stmt = partial(v, keywords=\u00d8, parens=ASSIGNMENTS)\n        self.visit_return_stmt = partial(v, keywords={\"return\"}, parens={\"return\"})\n        self.visit_import_from = partial(v, keywords=\u00d8, parens={\"import\"})\n        self.visit_del_stmt = partial(v, keywords=\u00d8, parens={\"del\"})\n        self.visit_async_funcdef = self.visit_async_stmt\n        self.visit_decorated = self.visit_decorators\n\n        # PEP 634\n        self.visit_match_stmt = self.visit_match_case\n        self.visit_case_block = self.visit_match_case\n        if Preview.remove_redundant_guard_parens in self.mode:\n            self.visit_guard = partial(v, keywords=\u00d8, parens={\"if\"})\n\n\ndef _hugging_power_ops_line_to_string(\n    line: Line,\n    features: Collection[Feature],\n    mode: Mode,\n) -> Optional[str]:\n    try:\n        return line_to_string(next(hug_power_op(line, features, mode)))\n    except CannotTransform:\n        return None\n\n\ndef transform_line(\n    line: Line, mode: Mode, features: Collection[Feature] = ()\n) -> Iterator[Line]:\n    \"\"\"Transform a `line`, potentially splitting it into many lines.\n\n    They should fit in the allotted `line_length` but might not be able to.\n\n    `features` are syntactical features that may be used in the output.\n    \"\"\"\n    if line.is_comment:\n        yield line\n        return\n\n    line_str = line_to_string(line)\n\n    # We need the line string when power operators are hugging to determine if we should\n    # split the line. Default to line_str, if no power operator are present on the line.\n    line_str_hugging_power_ops = (\n        _hugging_power_ops_line_to_string(line, features, mode) or line_str\n    )\n\n    ll = mode.line_length\n    sn = mode.string_normalization\n    string_merge = StringMerger(ll, sn)\n    string_paren_strip = StringParenStripper(ll, sn)\n    string_split = StringSplitter(ll, sn)\n    string_paren_wrap = StringParenWrapper(ll, sn)\n\n    transformers: List[Transformer]\n    if (\n        not line.contains_uncollapsable_type_comments()\n        and not line.should_split_rhs\n        and not line.magic_trailing_comma\n        and (\n            is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops)\n            or line.contains_unsplittable_type_ignore()\n        )\n        and not (line.inside_brackets and line.contains_standalone_comments())\n        and not line.contains_implicit_multiline_string_with_comments()\n    ):\n        # Only apply basic string preprocessing, since lines shouldn't be split here.\n        if Preview.string_processing in mode:\n            transformers = [string_merge, string_paren_strip]\n        else:\n            transformers = []\n    elif line.is_def and not should_split_funcdef_with_rhs(line, mode):\n        transformers = [left_hand_split]\n    else:\n\n        def _rhs(\n            self: object, line: Line, features: Collection[Feature], mode: Mode\n        ) -> Iterator[Line]:\n            \"\"\"Wraps calls to `right_hand_split`.\n\n            The calls increasingly `omit` right-hand trailers (bracket pairs with\n            content), meaning the trailers get glued together to split on another\n            bracket pair instead.\n            \"\"\"\n            for omit in generate_trailers_to_omit(line, mode.line_length):\n                lines = list(right_hand_split(line, mode, features, omit=omit))\n                # Note: this check is only able to figure out if the first line of the\n                # *current* transformation fits in the line length.  This is true only\n                # for simple cases.  All others require running more transforms via\n                # `transform_line()`.  This check doesn't know if those would succeed.\n                if is_line_short_enough(lines[0], mode=mode):\n                    yield from lines\n                    return\n\n            # All splits failed, best effort split with no omits.\n            # This mostly happens to multiline strings that are by definition\n            # reported as not fitting a single line, as well as lines that contain\n            # trailing commas (those have to be exploded).\n            yield from right_hand_split(line, mode, features=features)\n\n        # HACK: nested functions (like _rhs) compiled by mypyc don't retain their\n        # __name__ attribute which is needed in `run_transformer` further down.\n        # Unfortunately a nested class breaks mypyc too. So a class must be created\n        # via type ... https://github.com/mypyc/mypyc/issues/884\n        rhs = type(\"rhs\", (), {\"__call__\": _rhs})()\n\n        if Preview.string_processing in mode:\n            if line.inside_brackets:\n                transformers = [\n                    string_merge,\n                    string_paren_strip,\n                    string_split,\n                    delimiter_split,\n                    standalone_comment_split,\n                    string_paren_wrap,\n                    rhs,\n                ]\n            else:\n                transformers = [\n                    string_merge,\n                    string_paren_strip,\n                    string_split,\n                    string_paren_wrap,\n                    rhs,\n                ]\n        else:\n            if line.inside_brackets:\n                transformers = [delimiter_split, standalone_comment_split, rhs]\n            else:\n                transformers = [rhs]\n    # It's always safe to attempt hugging of power operations and pretty much every line\n    # could match.\n    transformers.append(hug_power_op)\n\n    for transform in transformers:\n        # We are accumulating lines in `result` because we might want to abort\n        # mission and return the original line in the end, or attempt a different\n        # split altogether.\n        try:\n            result = run_transformer(line, transform, mode, features, line_str=line_str)\n        except CannotTransform:\n            continue\n        else:\n            yield from result\n            break\n\n    else:\n        yield line\n\n\ndef sho\nuld_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:\n    \"\"\"If a funcdef has a magic trailing comma in the return type, then we should first\n    split the line with rhs to respect the comma.\n    \"\"\"\n    return_type_leaves: List[Leaf] = []\n    in_return_type = False\n\n    for leaf in line.leaves:\n        if leaf.type == token.COLON:\n            in_return_type = False\n        if in_return_type:\n            return_type_leaves.append(leaf)\n        if leaf.type == token.RARROW:\n            in_return_type = True\n\n    # using `bracket_split_build_line` will mess with whitespace, so we duplicate a\n    # couple lines from it.\n    result = Line(mode=line.mode, depth=line.depth)\n    leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)\n    for leaf in return_type_leaves:\n        result.append(\n            leaf,\n            preformatted=True,\n            track_bracket=id(leaf) in leaves_to_track,\n        )\n\n    # we could also return true if the line is too long, and the return type is longer\n    # than the param list. Or if `should_split_rhs` returns True.\n    return result.magic_trailing_comma is not None\n\n\nclass _BracketSplitComponent(Enum):\n    head = auto()\n    body = auto()\n    tail = auto()\n\n\ndef left_hand_split(\n    line: Line, _features: Collection[Feature], mode: Mode\n) -> Iterator[Line]:\n    \"\"\"Split line into many lines, starting with the first matching bracket pair.\n\n    Note: this usually looks weird, only use this for function definitions.\n    Prefer RHS otherwise.  This is why this function is not symmetrical with\n    :func:`right_hand_split` which also handles optional parentheses.\n    \"\"\"\n    tail_leaves: List[Leaf] = []\n    body_leaves: List[Leaf] = []\n    head_leaves: List[Leaf] = []\n    current_leaves = head_leaves\n    matching_bracket: Optional[Leaf] = None\n    for leaf in line.leaves:\n        if (\n            current_leaves is body_leaves\n            and leaf.type in CLOSING_BRACKETS\n            and leaf.opening_bracket is matching_bracket\n            and isinstance(matching_bracket, Leaf)\n        ):\n            ensure_visible(leaf)\n            ensure_visible(matching_bracket)\n            current_leaves = tail_leaves if body_leaves else head_leaves\n        current_leaves.append(leaf)\n        if current_leaves is head_leaves:\n            if leaf.type in OPENING_BRACKETS:\n                matching_bracket = leaf\n                current_leaves = body_leaves\n    if not matching_bracket or not tail_leaves:\n        raise CannotSplit(\"No brackets found\")\n\n    head = bracket_split_build_line(\n        head_leaves, line, matching_bracket, component=_BracketSplitComponent.head\n    )\n    body = bracket_split_build_line(\n        body_leaves, line, matching_bracket, component=_BracketSplitComponent.body\n    )\n    tail = bracket_split_build_line(\n        tail_leaves, line, matching_bracket, component=_BracketSplitComponent.tail\n    )\n    bracket_split_succeeded_or_raise(head, body, tail)\n    for result in (head, body, tail):\n        if result:\n            yield result\n\n\ndef right_hand_split(\n    line: Line,\n    mode: Mode,\n    features: Collection[Feature] = (),\n    omit: Collection[LeafID] = (),\n) -> Iterator[Line]:\n    \"\"\"Split line into many lines, starting with the last matching bracket pair.\n\n...\n", "needle_token_start": 15148, "needle_token_end": 15502, "code_context_ntokens": 16140, "output": ["```python\ndef should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool:\n    \"\"\"If a funcdef has a magic trailing comma in the return type, then we should first\n    split the line with rhs to respect the comma.\n    \"\"\"\n    return_type_leaves: List[Leaf] = []\n    in_return_type = False\n\n    for leaf in line.leaves:\n        if leaf.type == token.COLON:\n            in_return_type = False\n        if in_return_type:\n            return_type_leaves.append(leaf)\n        if leaf.type == token.RARROW:\n            in_return_type = True\n\n    # using `bracket_split_build_line` will mess with whitespace, so we duplicate a\n    # couple lines from it.\n    result = Line(mode=line.mode, depth=line.depth)\n    leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves)\n    for leaf in return_type_leaves:\n        result.append(\n            leaf,\n            preformatted=True,\n            track_bracket=id(leaf) in leaves_to_track,\n        )\n\n    # we could also return true if the line is too long, and the return type is longer\n    # than the param list. Or if `should_split_rhs` returns True.\n    return result.magic_trailing_comma is not None\n```"]}
{"repo": "python-poetry/poetry", "name": "_find_best_version_for_package", "language": "python", "path": "src/poetry/console/commands/init.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the most suitable version of a software package based on given criteria, ensuring compatibility and meeting specific requirements.\n2. **Input**: It accepts four parameters: the name of the package, an optional version requirement, a flag to include pre-release versions, and an optional source to specify where to fetch the package.\n3. **Output**: Returns a tuple containing the formatted name of the package and a version string prefixed with a caret (^), indicating compatibility with versions greater than or equal to the specified one.\n4. **Procedure**: The function utilizes a version selector tool to find the best candidate that matches the provided name and version criteria from a pool of available packages. If no suitable package is found, it raises an error. Otherwise, it returns the formatted package name and the compatible version string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                        info_string += \"\\nShowing the first 10 matches\"\n\n                    self.line(info_string)\n\n                    # Default to an empty value to signal no package was selected\n                    choices.append(\"\")\n\n                    package = self.choice(\n                        \"\\nEnter package # to add, or the complete package name if\"\n                        \" it is not listed\",\n                        choices,\n                        attempts=3,\n                        default=len(choices) - 1,\n                    )\n\n                    if not package:\n                        self.line(\"<warning>No package selected</warning>\")\n\n                    # package selected by user, set constraint name to package name\n                    if package:\n                        constraint[\"name\"] = package\n\n                # no constraint yet, determine the best version automatically\n                if package and \"version\" not in constraint:\n                    question = self.create_question(\n                        \"Enter the version constraint to require \"\n                        \"(or leave blank to use the latest version):\"\n                    )\n                    question.set_max_attempts(3)\n                    question.set_validator(lambda x: (x or \"\").strip() or None)\n\n                    package_constraint = self.ask(question)\n\n                    if package_constraint is None:\n                        _, package_constraint = self._find_best_version_for_package(\n                            package\n                        )\n\n                        self.line(\n                            f\"Using version <b>{package_constraint}</b> for\"\n                            f\" <c1>{package}</c1>\"\n                        )\n\n                    constraint[\"version\"] = package_constraint\n\n                if package:\n                    result.append(constraint)\n\n                if is_interactive:\n                    package = self.ask(follow_up_question)\n\n            return result\n\n        result = []\n        for requirement in self._parse_requirements(requires):\n            if \"git\" in requirement or \"url\" in requirement or \"path\" in requirement:\n                result.append(requirement)\n                continue\n            elif \"version\" not in requirement:\n                # determine the best version automatically\n                name, version = self._find_best_version_for_package(\n                    requirement[\"name\"],\n                    allow_prereleases=allow_prereleases,\n                    source=source,\n                )\n                requirement[\"version\"] = version\n                requirement[\"name\"] = name\n\n                self.line(f\"Using version <b>{version}</b> for <c1>{name}</c1>\")\n            else:\n                # check that the specified version/constraint exists\n                # before we proceed\n                name, _ = self._find_best_version_for_package(\n                    requirement[\"name\"],\n                    requirement[\"version\"],\n                    allow_prereleases=allow_prereleases,\n                    source=source,\n                )\n\n                requirement[\"name\"] = name\n\n            result.append(requirement)\n\n        return result\n\n    \ndef _find_best_version_for_package(\n        self,\n        name: str,\n        required_version: str | None = None,\n        allow_prereleases: bool = False,\n        source: str | None = None,\n    ) -> tuple[str, str]:\n        from poetry.version.version_selector import VersionSelector\n\n        selector = VersionSelector(self._get_pool())\n        package = selector.find_best_candidate(\n            name, required_version, allow_prereleases=allow_prereleases, source=source\n        )\n\n        if not package:\n            # TODO: find similar\n            raise ValueError(f\"Could not find a matching version of package {name}\")\n\n        return package.pretty_name, f\"^{package.version.to_string()}\"\n\n    def _parse_requirements(self, requirements: list[str]) -> list[dict[str, Any]]:\n        from poetry.core.pyproject.exceptions import PyProjectException\n\n        try:\n            cwd = self.poetry.file.path.parent\n            artifact_cache = self.poetry.pool.artifact_cache\n        except (PyProjectException, RuntimeError):\n            cwd = Path.cwd()\n            artifact_cache = self._get_pool().artifact_cache\n\n        parser = RequirementsParser(\n            artifact_cache=artifact_cache,\n            env=self.env if isinstance(self, EnvCommand) else None,\n            cwd=cwd,\n        )\n        return [parser.parse(requirement) for requirement in requirements]\n\n    def _format_requirements(self, requirements: list[dict[str, str]]) -> Requirements:\n        requires: Requirements = {}\n        for requirement in requirements:\n            name = requirement.pop(\"name\")\n            constraint: str | InlineTable\n            if \"version\" in requirement and len(requirement) == 1:\n                constraint = requirement[\"version\"]\n            else:\n                constraint = inline_table()\n                constraint.trivia.trail = \"\\n\"\n                constraint.update(requirement)\n\n            requires[name] = constraint\n\n        return requires\n\n    @staticmethod\n    def _validate_author(author: str, default: str) -> str | None:\n        from poetry.core.packages.package import AUTHOR_REGEX\n        from poetry.core.utils.helpers import combine_unicode\n\n        author = combine_unicode(author or default)\n\n        if author in [\"n\", \"no\"]:\n            return None\n\n        m = AUTHOR_REGEX.match(author)\n        if not m:\n            raise ValueError(\n                \"Invalid author string. Must be in the format: \"\n                \"John Smith <john@example.com>\"\n            )\n\n        return author\n\n    @staticmethod\n    def _validate_package(package: str | None) -> str | None:\n        if package and len(package.split()) > 2:\n            raise ValueError(\"Invalid package definition.\")\n\n        return package\n\n    def _get_pool(self) -> RepositoryPool:\n        from poetry.config.config import Config\n        from poetry.repositories import RepositoryPool\n        from poetry.repositories.pypi_repository import PyPiRepository\n\n        if isinstance(self, EnvCommand):\n            return self.poetry.pool\n\n        if self._pool is None:\n            self._pool = RepositoryPool()\n            pool_size = Config.create().installer_max_workers\n            self._pool.add_repository(PyPiRepository(pool_size=pool_size))\n\n        return self._pool\n\n# Path: src/poetry/console/commands/install.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass InstallCommand(InstallerCommand):\n    name = \"install\"\n    description = \"Installs the project dependencies.\"\n\n    options: ClassVar[list[Option]] = [\n        *InstallerCommand._group_dependency_options(),\n        option(\n            \"no-dev\",\n            None,\n            \"Do not install the development dependencies.\"\n            \" (<warning>Deprecated</warning>)\",\n        ),\n        option(\n            \"sync\",\n            None,\n            \"Synchronize the environment with the locked packages and the specified\"\n            \" groups.\",\n        ),\n        option(\n            \"no-root\", None, \"Do not install the root package (the current project).\"\n        ),\n        option(\n            \"no-directory\",\n            None,\n            \"Do not install any directory path dependencies; useful to install\"\n            \" dependencies without source code, e.g. for caching of Docker layers)\",\n            flag=True,\n            multiple=False,\n        ),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything \"\n            \"(implicitly enables --verbose).\",\n        ),\n        option(\n            \"remove-untracked\",\n            None,\n            \"Removes packages not present in the lock file.\"\n            \" (<warning>Deprecated</warning>)\",\n        ),\n        option(\n            \"extras\",\n            \"E\",\n            \"Extra sets of dependencies to install.\",\n            flag=False,\n            multiple=True,\n        ),\n        option(\"all-extras\", None, \"Install all extra dependencies.\"),\n        option(\"only-root\", None, \"Exclude all dependencies.\"),\n        option(\n            \"compile\",\n            None,\n            \"Compile Python source files to bytecode.\"\n            \" (This option has no effect if modern-installation is disabled\"\n            \" because the old installer always compiles.)\",\n        ),\n    ]\n\n    help = \"\"\"\\\nThe <info>install</info> command reads the <comment>poetry.lock</> file from\nthe current directory, processes it, and downloads and installs all the\nlibraries and dependencies outlined in that file. If the file does not\nexist it will look for <comment>pyproject.toml</> and do the same.\n\n<info>poetry install</info>\n\nBy default, the above command will also install the current project. To install only the\ndependencies and not including the current project, run the command with the\n<info>--no-root</info> option like below:\n\n<info> poetry install --no-root</info>\n\nIf you want to use Poetry only for dependency management but not for packaging,\nyou can set the \"package-mode\" to false in your pyproject.toml file.\n\"\"\"\n\n    _loggers: ClassVar[list[str]] = [\n        \"poetry.repositories.pypi_repository\",\n        \"poetry.inspection.info\",\n    ]\n\n    @property\n    def activated_groups(self) -> set[str]:\n        if self.option(\"only-root\"):\n            return set()\n        else:\n            return super().activated_groups\n\n    def handle(self) -> int:\n        from poetry.core.masonry.utils.module import ModuleOrPackageNotFound\n\n        from poetry.masonry.builders.editable import EditableBuilder\n\n        if self.option(\"extras\") and self.option(\"all-extras\"):\n            self.line_error(\n                \"<error>You cannot specify explicit\"\n                \" `<fg=yellow;options=bold>--extras</>` while installing\"\n                \" using `<fg=yellow;options=bold>--all-extras</>`.</error>\"\n            )\n            return 1\n\n        if self.option(\"only-root\") and any(\n            self.option(key) for key in {\"with\", \"without\", \"only\"}\n        ):\n            self.line_error(\n                \"<error>The `<fg=yellow;options=bold>--with</>`,\"\n                \" `<fg=yellow;options=bold>--without</>` and\"\n                \" `<fg=yellow;options=bold>--only</>` options cannot be used with\"\n                \" the `<fg=yellow;options=bold>--only-root</>`\"\n                \" option.</error>\"\n            )\n            return 1\n\n        if self.option(\"only-root\") and self.option(\"no-root\"):\n            self.line_error(\n                \"<error>You cannot specify `<fg=yellow;options=bold>--no-root</>`\"\n                \" when using `<fg=yellow;options=bold>--only-root</>`.</error>\"\n            )\n            return 1\n\n        extras: list[str]\n        if self.option(\"all-extras\"):\n            extras = list(self.poetry.package.extras.keys())\n        else:\n            extras = []\n            for extra in self.option(\"extras\", []):\n                extras += extra.split()\n\n        self.installer.extras(extras)\n\n        with_synchronization = self.option(\"sync\")\n        if self.option(\"remove-untracked\"):\n            self.line_error(\n                \"<warning>The `<fg=yellow;options=bold>--remove-untracked</>` option is\"\n                \" deprecated, use the `<fg=yellow;options=bold>--sync</>` option\"\n                \" instead.</warning>\"\n            )\n\n            with_synchronization = True\n\n        self.installer.only_groups(self.activated_groups)\n        self.installer.skip_directory(self.option(\"no-directory\"))\n        self.installer.dry_run(self.option(\"dry-run\"))\n        self.installer.requires_synchronization(with_synchronization)\n        self.installer.executor.enable_bytecode_compilation(self.option(\"compile\"))\n        self.installer.verbose(self.io.is_verbose())\n\n        return_code = self.installer.run()\n\n        if return_code != 0:\n            return return_code\n\n        if self.option(\"no-root\") or not self.poetry.is_package_mode:\n            return 0\n\n        log_install = (\n            \"<b>Installing</> the current project:\"\n            f\" <c1>{self.poetry.package.pretty_name}</c1>\"\n            f\" (<{{tag}}>{self.poetry.package.pretty_version}</>)\"\n        )\n        overwrite = self.io.output.is_decorated() and not self.io.is_debug()\n        self.line(\"\")\n        self.write(log_install.format(tag=\"c2\"))\n        if not overwrite:\n            self.line(\"\")\n\n        if self.option(\"dry-run\"):\n            self.line(\"\")\n            return 0\n\n        # Prior to https://github.com/python-poetry/poetry-core/pull/629\n        # the existence of a module/package was checked when creating the\n        # EditableBuilder. Afterwards, the existence is checked after\n        # executing the build script (if there is one),\n        # i.e. during EditableBuilder.build().\n        try:\n            builder = EditableBuilder(self.poetry, self.env, self.io)\n            builder.build()\n        except (ModuleOrPackageNotFound, FileNotFoundError) as e:\n            # This is likely due to the fact that the project is an application\n            # not following the structure expected by Poetry.\n            # No need for an editable install in this case.\n            self.line(\"\")\n            self.line_error(\n                f\"Warning: The current project could not be installed: {e}\\n\"\n                \"If you do not want to install the current project\"\n                \" use <c1>--no-root</c1>.\\n\"\n                \"If you want to use Poetry only for dependency management\"\n                \" but not for packaging, you can disable package mode by setting\"\n                \" <c1>package-mode = false</> in your pyproject.toml file.\\n\"\n                \"In a future version of Poetry this warning will become an error!\",\n                style=\"warning\",\n            )\n            return 0\n\n        if overwrite:\n            self.overwrite(log_install.format(tag=\"success\"))\n            self.line(\"\")\n\n        return 0\n\n# Path: src/poetry/console/commands/installer_command.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.console.commands.group_command import GroupCommand\n\n\nif TYPE_CHECKING:\n    from poetry.installation.installer import Installer\n\n\nclass InstallerCommand(GroupCommand, EnvCommand):\n    def __init__(self) -> None:\n        # Set in poetry.console.application.Application.configure_installer\n        self._installer: Installer | None = None\n\n        super().__init__()\n\n    def reset_poetry(self) -> None:\n        super().reset_poetry()\n\n        self.installer.set_package(self.poetry.package)\n        self.installer.set_locker(self.poetry.locker)\n\n    @property\n    def installer(self) -> Installer:\n        assert self._installer is not None\n        return self._installer\n\n    def set_installer(self, installer: Installer) -> None:\n        self._installer = installer\n\n# Path: src/poetry/console/commands/lock.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass LockCommand(InstallerCommand):\n    name = \"lock\"\n    description = \"Locks the project dependencies.\"\n\n    options: ClassVar[list[Option]] = [\n        option(\n            \"no-update\", None, \"Do not update locked versions, only refresh lock file.\"\n        ),\n        option(\n            \"check\",\n            None,\n            \"Check that the <comment>poetry.lock</> file corresponds to the current\"\n            \" version of <comment>pyproject.toml</>. (<warning>Deprecated</>) Use\"\n            \" <comment>poetry check --lock</> instead.\",\n        ),\n    ]\n\n    help = \"\"\"\nThe <info>lock</info> command reads the <comment>pyproject.toml</> file from the\ncurrent directory, processes it, and locks the dependencies in the\\\n <comment>poetry.lock</>\nfile.\n\n<info>poetry lock</info>\n\"\"\"\n\n    loggers: ClassVar[list[str]] = [\"poetry.repositories.pypi_repository\"]\n\n    def handle(self) -> int:\n        if self.option(\"check\"):\n            self.line_error(\n                \"<warning>poetry lock --check is deprecated, use `poetry\"\n                \" check --lock` instead.</warning>\"\n            )\n            if self.poetry.locker.is_locked() and self.poetry.locker.is_fresh():\n                self.line(\"poetry.lock is consistent with pyproject.toml.\")\n                return 0\n            self.line_error(\n                \"<error>\"\n                \"Error: pyproject.toml changed significantly since poetry.lock was last generated. \"\n                \"Run `poetry lock [--no-update]` to fix the lock file.\"\n                \"</error>\"\n            )\n            return 1\n\n        self.installer.lock(update=not self.option(\"no-update\"))\n\n        return self.installer.run()\n\n# Path: src/poetry/console/commands/new.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.init import InitCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass NewCommand(InitCommand):\n    name = \"new\"\n    description = \"Creates a new Python project at <path>.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"path\", \"The path to create the project at.\")\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\n            \"interactive\",\n            \"i\",\n            \"Allow interactive specification of project configuration.\",\n            flag=True,\n        ),\n        option(\"name\", None, \"Set the resulting package name.\", flag=False),\n        option(\"src\", None, \"Use the src layout for the project.\"),\n        option(\n            \"readme\",\n            None,\n            \"Specify the readme file format. One of md (default) or rst\",\n            flag=False,\n        ),\n        *[\n            o\n            for o in InitCommand.options\n            if o.name\n            in {\n                \"description\",\n                \"author\",\n                \"python\",\n                \"dependency\",\n                \"dev-dependency\",\n                \"license\",\n            }\n        ],\n    ]\n\n    def handle(self) -> int:\n        from pathlib import Path\n\n        if self.io.input.option(\"directory\"):\n            self.line_error(\n                \"<warning>--directory only makes sense with existing projects, and will\"\n                \" be ignored. You should consider the option --path instead.</warning>\"\n            )\n\n        path = Path(self.argument(\"path\"))\n        if not path.is_absolute():\n            # we do not use resolve here due to compatibility issues\n            # for path.resolve(strict=False)\n            path = Path.cwd().joinpath(path)\n\n        if path.exists() and list(path.glob(\"*\")):\n            # Directory is not empty. Aborting.\n            raise RuntimeError(\n                f\"Destination <fg=yellow>{path}</> exists and is not empty\"\n            )\n\n        return self._init_pyproject(\n            project_path=path,\n            allow_interactive=self.option(\"interactive\"),\n            layout_name=\"src\" if self.option(\"src\") else \"standard\",\n            readme_format=self.option(\"readme\") or \"md\",\n        )\n\n# Path: src/poetry/console/commands/publish.py\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass PublishCommand(Command):\n    name = \"publish\"\n    description = \"Publishes a package to a remote repository.\"\n\n    options: ClassVar[list[Option]] = [\n        option(\n            \"repository\", \"r\", \"The repository to publish the package to.\", flag=False\n        ),\n        option(\"username\", \"u\", \"The username to access the repository.\", flag=False),\n        option(\"password\", \"p\", \"The password to access the repository.\", flag=False),\n        option(\n            \"cert\", None, \"Certificate authority to access the repository.\", flag=False\n        ),\n        option(\n            \"client-cert\",\n            None,\n            \"Client certificate to access the repository.\",\n            flag=False,\n        ),\n        option(\n            \"dist-dir\",\n            None,\n            \"Dist directory where built artifact are stored. Default is `dist`.\",\n            default=\"dist\",\n            flag=False,\n        ),\n        option(\"build\", None, \"Build the package before publishing.\"),\n        option(\"dry-run\", None, \"Perform all actions except upload the package.\"),\n        option(\n            \"skip-existing\",\n            None,\n            \"Ignore errors from files already existing in the repository.\",\n        ),\n    ]\n\n    help = \"\"\"The publish command builds and uploads the package to a remote repository.\n\nBy default, it will upload to PyPI but if you pass the --repository option it will\nupload to it instead.\n\nThe --repository option should match the name of a configured repository using\nthe config command.\n\"\"\"\n\n    loggers: ClassVar[list[str]] = [\"poetry.publishing.publisher\"]\n\n    def handle(self) -> int:\n        from poetry.publishing.publisher import Publisher\n\n        if not self.poetry.is_package_mode:\n            self.line_error(\"Publishing a package is not possible in non-package mode.\")\n            return 1\n\n        dist_dir = self.option(\"dist-dir\")\n\n        publisher = Publisher(self.poetry, self.io, Path(dist_dir))\n\n        # Building package first, if told\n        if self.option(\"build\"):\n            if publisher.files and not self.confirm(\n                f\"There are <info>{len(publisher.files)}</info> files ready for\"\n                \" publishing. Build anyway?\"\n            ):\n                self.line_error(\"<error>Aborted!</error>\")\n\n                return 1\n\n            self.call(\"build\", args=f\"--output {dist_dir}\")\n\n        files = publisher.files\n        if not files:\n            self.line_error(\n                \"<error>No files to publish. \"\n                \"Run poetry build first or use the --build option.</error>\"\n            )\n\n            return 1\n\n        self.line(\"\")\n\n        cert = Path(self.option(\"cert\")) if self.option(\"cert\") else None\n        client_cert = (\n            Path(self.option(\"client-cert\")) if self.option(\"client-cert\") else None\n        )\n\n        publisher.publish(\n            self.option(\"repository\"),\n            self.option(\"username\"),\n            self.option(\"password\"),\n            cert,\n            client_cert,\n            self.option(\"dry-run\"),\n            self.option(\"skip-existing\"),\n        )\n\n        return 0\n\n# Path: src/poetry/console/commands/remove.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\nfrom poetry.core.packages.dependency_group import MAIN_GROUP\nfrom tomlkit.toml_document import TOMLDocument\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass RemoveCommand(InstallerCommand):\n    name = \"remove\"\n    description = \"Removes a package from the project dependencies.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"packages\", \"The packages to remove.\", multiple=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\"group\", \"G\", \"The group to remove the dependency from.\", flag=False),\n        option(\n            \"dev\",\n            \"D\",\n            \"Remove a package from the development dependencies.\"\n            \" (<warning>Deprecated</warning>)\"\n            \" Use --group=dev instead.\",\n        ),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything \"\n            \"(implicitly enables --verbose).\",\n        ),\n        option(\"lock\", None, \"Do not perform operations (only update the lockfile).\"),\n    ]\n\n    help = \"\"\"The <info>remove</info> command removes a package from the current\nlist of installed packages\n\n<info>poetry remove</info>\"\"\"\n\n    loggers: ClassVar[list[str]] = [\n        \"poetry.repositories.pypi_repository\",\n        \"poetry.inspection.info\",\n    ]\n\n    def handle(self) -> int:\n        packages = self.argument(\"packages\")\n\n        if self.option(\"dev\"):\n            self.line_error(\n                \"<warning>The --dev option is deprecated, \"\n                \"use the `--group dev` notation instead.</warning>\"\n            )\n            group = \"dev\"\n        else:\n            group = self.option(\"group\", self.default_group)\n\n        content: dict[str, Any] = self.poetry.file.read()\n        poetry_content = content[\"tool\"][\"poetry\"]\n\n        if group is None:\n            removed = []\n            group_sections = [\n                (group_name, group_section.get(\"dependencies\", {}))\n                for group_name, group_section in poetry_content.get(\"group\", {}).items()\n            ]\n\n            for group_name, section in [\n                (MAIN_GROUP, poetry_content[\"dependencies\"]),\n                *group_sections,\n            ]:\n                removed += self._remove_packages(packages, section, group_name)\n                if group_name != MAIN_GROUP:\n                    if not section:\n                        del poetry_content[\"group\"][group_name]\n                    else:\n                        poetry_content[\"group\"][group_name][\"dependencies\"] = section\n        elif group == \"dev\" and \"dev-dependencies\" in poetry_content:\n            # We need to account for the old `dev-dependencies` section\n            removed = self._remove_packages(\n                packages, poetry_content[\"dev-dependencies\"], \"dev\"\n            )\n\n            if not poetry_content[\"dev-dependencies\"]:\n                del poetry_content[\"dev-dependencies\"]\n        else:\n            removed = []\n            if \"group\" in poetry_content:\n                if group in poetry_content[\"group\"]:\n                    removed = self._remove_packages(\n                        packages,\n                        poetry_content[\"group\"][group].get(\"dependencies\", {}),\n                        group,\n                    )\n\n                if not poetry_content[\"group\"][group]:\n                    del poetry_content[\"group\"][group]\n\n        if \"group\" in poetry_content and not poetry_content[\"group\"]:\n            del poetry_content[\"group\"]\n\n        removed_set = set(removed)\n        not_found = set(packages).difference(removed_set)\n        if not_found:\n            raise ValueError(\n                \"The following packages were not found: \" + \", \".join(sorted(not_found))\n            )\n\n        # Refresh the locker\n        self.poetry.locker.set_local_config(poetry_content)\n        self.installer.set_locker(self.poetry.locker)\n        self.installer.set_package(self.poetry.package)\n        self.installer.dry_run(self.option(\"dry-run\", False))\n        self.installer.verbose(self.io.is_verbose())\n        self.installer.update(True)\n        self.installer.execute_operations(not self.option(\"lock\"))\n        self.installer.whitelist(removed_set)\n\n        status = self.installer.run()\n\n        if not self.option(\"dry-run\") and status == 0:\n            assert isinstance(content, TOMLDocument)\n            self.poetry.file.write(content)\n\n        return status\n\n    def _remove_packages(\n        self, packages: list[str], section: dict[str, Any], group_name: str\n    ) -> list[str]:\n        removed = []\n        group = self.poetry.package.dependency_group(group_name)\n        section_keys = list(section.keys())\n\n        for package in packages:\n            for existing_package in section_keys:\n                if canonicalize_name(existing_package) == canonicalize_name(package):\n                    del section[existing_package]\n                    removed.append(package)\n                    group.remove_dependency(package)\n\n        return removed\n\n# Path: src/poetry/console/commands/run.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\n\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.utils._compat import WINDOWS\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from poetry.core.masonry.utils.module import Module\n\n\nclass RunCommand(EnvCommand):\n    name = \"run\"\n    description = \"Runs a command in the appropriate environment.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"args\", \"The command and arguments/options to run.\", multiple=True)\n    ]\n\n    def handle(self) -> int:\n        args = self.argument(\"args\")\n        script = args[0]\n        scripts = self.poetry.local_config.get(\"scripts\")\n\n        if scripts and script in scripts:\n            return self.run_script(scripts[script], args)\n\n        try:\n            return self.env.execute(*args)\n        except FileNotFoundError:\n            self.line_error(f\"<error>Command not found: <c1>{script}</c1></error>\")\n            return 1\n\n    @property\n    def _module(self) -> Module:\n        from poetry.core.masonry.utils.module import Module\n\n        poetry = self.poetry\n        package = poetry.package\n        path = poetry.file.path.parent\n        module = Module(package.name, path.as_posix(), package.packages)\n\n        return module\n\n    def run_script(self, script: str | dict[str, str], args: list[str]) -> int:\n        \"\"\"Runs an entry point script defined in the section ``[tool.poetry.scripts]``.\n\n        When a script exists in the venv bin folder, i.e. after ``poetry install``,\n        then ``sys.argv[0]`` must be set to the full path of the executable, so\n        ``poetry run foo`` and ``poetry shell``, ``foo`` have the same ``sys.argv[0]``\n        that points to the full path.\n\n        Otherwise (when an entry point script does not exist), ``sys.argv[0]`` is the\n        script name only, i.e. ``poetry run foo`` has ``sys.argv == ['foo']``.\n        \"\"\"\n        for script_dir in self.env.script_dirs:\n            script_path = script_dir / args[0]\n            if WINDOWS:\n                script_path = script_path.with_suffix(\".cmd\")\n            if script_path.exists():\n                args = [str(script_path), *args[1:]]\n                break\n        else:\n            # If we reach this point, the script is not installed\n            self._warning_not_installed_script(args[0])\n\n        if isinstance(script, dict):\n            script = script[\"callable\"]\n\n        module, callable_ = script.split(\":\")\n\n        src_in_sys_path = \"sys.path.append('src'); \" if self._module.is_in_src() else \"\"\n\n        cmd = [\"python\", \"-c\"]\n\n        cmd += [\n            \"import sys; \"\n            \"from importlib import import_module; \"\n            f\"sys.argv = {args!r}; {src_in_sys_path}\"\n            f\"sys.exit(import_module('{module}').{callable_}())\"\n        ]\n\n        return self.env.execute(*cmd)\n\n    def _warning_not_installed_script(self, script: str) -> None:\n        message = f\"\"\"\\\nWarning: '{script}' is an entry point defined in pyproject.toml, but it's not \\\ninstalled as a script. You may get improper `sys.argv[0]`.\n\nThe support to run uninstalled scripts will be removed in a future release.\n\nRun `poetry install` to resolve and get rid of this message.\n\"\"\"\n        self.line_error(message, style=\"warning\")\n\n# Path: src/poetry/console/commands/search.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n\n\nclass SearchCommand(Command):\n    name = \"search\"\n    description = \"Searches for packages on remote repositories.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"tokens\", \"The tokens to search for.\", multiple=True)\n    ]\n\n    def handle(self) -> int:\n        from poetry.repositories.pypi_repository import PyPiRepository\n\n        results = PyPiRepository().search(self.argument(\"tokens\"))\n\n        for result in results:\n            self.line(\"\")\n            name = f\"<info>{result.name}</>\"\n\n            name += f\" (<comment>{result.version}</>)\"\n\n            self.line(name)\n\n            if result.description:\n                self.line(f\" {result.description}\")\n\n        return 0\n\n# Path: src/poetry/console/commands/shell.py\nfrom __future__ import annotations\n\nimport os\nimport sys\n\nfrom typing import TYPE_CHECKING\nfrom typing import cast\n\nfrom poetry.console.commands.env_command import EnvCommand\n\n\nif TYPE_CHECKING:\n    from poetry.utils.env import VirtualEnv\n\n\nclass ShellCommand(EnvCommand):\n    name = \"shell\"\n    description = \"Spawns a shell within the virtual environment.\"\n\n    help = f\"\"\"The <info>shell</> command spawns a shell within the project's virtual environment.\n\nBy default, the current active shell is detected and used. Failing that,\nthe shell defined via the environment variable <comment>{'COMSPEC' if os.name == 'nt' else 'SHELL'}</> is used.\n\nIf a virtual environment does not exist, it will be created.\n\"\"\"\n\n    def handle(self) -> int:\n        from poetry.utils.shell import Shell\n\n        # Check if it's already activated or doesn't exist and won't be created\n        if self._is_venv_activated():\n            self.line(\n                f\"Virtual environment already activated: <info>{self.env.path}</>\"\n            )\n\n            return 0\n\n        self.line(f\"Spawning shell within <info>{self.env.path}</>\")\n\n        # Be sure that we have the right type of environment.\n        env = self.env\n        assert env.is_venv()\n        env = cast(\"VirtualEnv\", env)\n\n        # Setting this to avoid spawning unnecessary nested shells\n        os.environ[\"POETRY_ACTIVE\"] = \"1\"\n        shell = Shell.get()\n        shell.activate(env)\n        os.environ.pop(\"POETRY_ACTIVE\")\n\n        return 0\n\n    def _is_venv_activated(self) -> bool:\n        return bool(os.environ.get(\"POETRY_ACTIVE\")) or getattr(\n            sys, \"real_prefix\", sys.prefix\n        ) == str(self.env.path)\n\n# Path: src/poetry/console/commands/show.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\n\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.console.commands.group_command import GroupCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n    from cleo.io.io import IO\n    from cleo.ui.table import Rows\n    from packaging.utils import NormalizedName\n    from poetry.core.packages.dependency import Dependency\n    from poetry.core.packages.package import Package\n    from poetry.core.packages.project_package import ProjectPackage\n\n    from poetry.repositories.repository import Repository\n\n\ndef reverse_deps(pkg: Package, repo: Repository) -> dict[str, str]:\n    required_by = {}\n    for locked in repo.packages:\n        dependencies = {d.name: d.pretty_constraint for d in locked.requires}\n\n        if pkg.name in dependencies:\n            required_by[locked.pretty_name] = dependencies[pkg.name]\n\n    return required_by\n\n\nclass ShowCommand(GroupCommand, EnvCommand):\n    name = \"show\"\n    description = \"Shows information about packages.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"package\", \"The package to inspect\", optional=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        *GroupCommand._group_dependency_options(),\n        option(\n            \"no-dev\",\n            None,\n            \"Do not list the development dependencies. (<warning>Deprecated</warning>)\",\n        ),\n        option(\"tree\", \"t\", \"List the dependencies as a tree.\"),\n        option(\n            \"why\",\n            None,\n            \"When showing the full list, or a <info>--tree</info> for a single package,\"\n            \" display whether they are a direct dependency or required by other\"\n            \" packages\",\n        ),\n        option(\"latest\", \"l\", \"Show the latest version.\"),\n        option(\n            \"outdated\",\n            \"o\",\n            \"Show the latest version but only for packages that are outdated.\",\n        ),\n        option(\n            \"all\",\n            \"a\",\n            \"Show all packages (even those not compatible with current system).\",\n        ),\n        option(\"top-level\", \"T\", \"Show only top-level dependencies.\"),\n    ]\n\n    help = \"\"\"The show command displays detailed information about a package, or\nlists all packages available.\"\"\"\n\n    colors: ClassVar[list[str]] = [\"cyan\", \"yellow\", \"green\", \"magenta\", \"blue\"]\n\n    def handle(self) -> int:\n        package = self.argument(\"package\")\n\n        if self.option(\"tree\"):\n            self.init_styles(self.io)\n\n        if self.option(\"top-level\"):\n            if self.option(\"tree\"):\n                self.line_error(\n                    \"<error>Error: Cannot use --tree and --top-level at the same\"\n                    \" time.</error>\"\n                )\n                return 1\n            if package is not None:\n                self.line_error(\n                    \"<error>Error: Cannot use --top-level when displaying a single\"\n                    \" package.</error>\"\n                )\n                return 1\n\n        if self.option(\"why\"):\n            if self.option(\"tree\") and package is None:\n                self.line_error(\n                    \"<error>Error: --why requires a package when combined with\"\n                    \" --tree.</error>\"\n                )\n\n                return 1\n\n            if not self.option(\"tree\") and package:\n                self.line_error(\n                    \"<error>Error: --why cannot be used without --tree when displaying\"\n                    \" a single package.</error>\"\n                )\n\n                return 1\n\n        if self.option(\"outdated\"):\n            self.io.input.set_option(\"latest\", True)\n\n        if not self.poetry.locker.is_locked():\n            self.line_error(\n                \"<error>Error: poetry.lock not found. Run `poetry lock` to create\"\n                \" it.</error>\"\n            )\n            return 1\n\n        locked_repo = self.poetry.locker.locked_repository()\n\n        if package:\n            return self._display_single_package_information(package, locked_repo)\n\n        root = self.project_with_activated_groups_only()\n\n        # Show tree view if requested\n        if self.option(\"tree\"):\n            return self._display_packages_tree_information(locked_repo, root)\n\n        return self._display_packages_information(locked_repo, root)\n\n    def _display_single_package_information(\n        self, package: str, locked_repository: Repository\n    ) -> int:\n        locked_packages = locked_repository.packages\n        canonicalized_package = canonicalize_name(package)\n        pkg = None\n\n        for locked in locked_packages:\n            if locked.name == canonicalized_package:\n                pkg = locked\n                break\n\n        if not pkg:\n            raise ValueError(f\"Package {package} not found\")\n\n        required_by = reverse_deps(pkg, locked_repository)\n\n        if self.option(\"tree\"):\n            if self.option(\"why\"):\n                # The default case if there's no reverse dependencies is to query\n                # the subtree for pkg but if any rev-deps exist we'll query for each\n                # of them in turn\n                packages = [pkg]\n                if required_by:\n                    packages = [\n                        p for p in locked_packages for r in required_by if p.name == r\n                    ]\n                else:\n                    # if no rev-deps exist we'll make this clear as it can otherwise\n                    # look very odd for packages that also have no or few direct\n                    # dependencies\n                    self.io.write_line(f\"Package {package} is a direct dependency.\")\n\n                for p in packages:\n                    self.display_package_tree(\n                        self.io, p, locked_packages, why_package=pkg\n                    )\n\n            else:\n                self.display_package_tree(self.io, pkg, locked_packages)\n\n            return 0\n\n        rows: Rows = [\n            [\"<info>name</>\", f\" : <c1>{pkg.pretty_name}</>\"],\n            [\"<info>version</>\", f\" : <b>{pkg.pretty_version}</b>\"],\n            [\"<info>description</>\", f\" : {pkg.description}\"],\n        ]\n\n        self.table(rows=rows, style=\"compact\").render()\n\n        if pkg.requires:\n            self.line(\"\")\n            self.line(\"<info>dependencies</info>\")\n            for dependency in pkg.requires:\n                self.line(\n                    f\" - <c1>{dependency.pretty_name}</c1>\"\n                    f\" <b>{dependency.pretty_constraint}</b>\"\n                )\n\n        if required_by:\n            self.line(\"\")\n            self.line(\"<info>required by</info>\")\n            for parent, requires_version in required_by.items():\n                self.line(f\" - <c1>{parent}</c1> <b>{requires_version}</b>\")\n\n        return 0\n\n    def _display_packages_information(\n        self, locked_repository: Repository, root: ProjectPackage\n    ) -> int:\n        import shutil\n\n        from cleo.io.null_io import NullIO\n\n        from poetry.puzzle.solver import Solver\n        from poetry.repositories.installed_repository import InstalledRepository\n        from poetry.repositories.repository_pool import RepositoryPool\n        from poetry.utils.helpers import get_package_version_display_string\n\n        locked_packages = locked_repository.packages\n        pool = RepositoryPool.from_packages(locked_packages, self.poetry.config)\n        solver = Solver(\n            root,\n            pool=pool,\n            installed=[],\n            locked=locked_packages,\n            io=NullIO(),\n        )\n        solver.provider.load_deferred(False)\n        with solver.use_environment(self.env):\n            ops = solver.solve().calculate_operations()\n\n        required_locked_packages = {op.package for op in ops if not op.skipped}\n\n        show_latest = self.option(\"latest\")\n        show_all = self.option(\"all\")\n        show_top_level = self.option(\"top-level\")\n        width = shutil.get_terminal_size().columns\n        name_length = version_length = latest_length = required_by_length = 0\n        latest_packages = {}\n        latest_statuses = {}\n        installed_repo = InstalledRepository.load(self.env)\n\n        # Computing widths\n        for locked in locked_packages:\n            if locked not in required_locked_packages and not show_all:\n                continue\n\n            current_length = len(locked.pretty_name)\n            if not self.io.output.is_decorated():\n                installed_status = self.get_installed_status(\n                    locked, installed_repo.packages\n                )\n\n                if installed_status == \"not-installed\":\n                    current_length += 4\n\n            if show_latest:\n                latest = self.find_latest_package(locked, root)\n                if not latest:\n                    latest = locked\n\n                latest_packages[locked.pretty_name] = latest\n                update_status = latest_statuses[locked.pretty_name] = (\n                    self.get_update_status(latest, locked)\n                )\n\n                if not self.option(\"outdated\") or update_status != \"up-to-date\":\n                    name_length = max(name_length, current_length)\n                    version_length = max(\n                        version_length,\n                        len(\n                            get_package_version_display_string(\n                                locked, root=self.poetry.file.path.parent\n                            )\n                        ),\n                    )\n                    latest_length = max(\n                        latest_length,\n                        len(\n                            get_package_version_display_string(\n                                latest, root=self.poetry.file.path.parent\n                            )\n                        ),\n                    )\n\n                    if self.option(\"why\"):\n                        required_by = reverse_deps(locked, locked_repository)\n                        required_by_length = max(\n                            required_by_length,\n                            len(\" from \" + \",\".join(required_by.keys())),\n                        )\n            else:\n                name_length = max(name_length, current_length)\n                version_length = max(\n                    version_length,\n                    len(\n                        get_package_version_display_string(\n                            locked, root=self.poetry.file.path.parent\n                        )\n                    ),\n                )\n\n                if self.option(\"why\"):\n                    required_by = reverse_deps(locked, locked_repository)\n                    required_by_length = max(\n                        required_by_length, len(\" from \" + \",\".join(required_by.keys()))\n                    )\n\n        write_version = name_length + version_length + 3 <= width\n        write_latest = name_length + version_length + latest_length + 3 <= width\n\n        why_end_column = (\n            name_length + version_length + latest_length + required_by_length\n        )\n        write_why = self.option(\"why\") and (why_end_column + 3) <= width\n        write_description = (why_end_column + 24) <= width\n\n        requires = root.all_requires\n\n        for locked in locked_packages:\n            color = \"cyan\"\n            name = locked.pretty_name\n            install_marker = \"\"\n\n            if show_top_level and not any(locked.satisfies(r) for r in requires):\n                continue\n\n            if locked not in required_locked_packages:\n                if not show_all:\n                    continue\n\n                color = \"black;options=bold\"\n            else:\n                installed_status = self.get_installed_status(\n                    locked, installed_repo.packages\n                )\n                if installed_status == \"not-installed\":\n                    color = \"red\"\n\n                    if not self.io.output.is_decorated():\n                        # Non installed in non decorated mode\n                        install_marker = \" (!)\"\n\n            if (\n                show_latest\n                and self.option(\"outdated\")\n                and latest_statuses[locked.pretty_name] == \"up-to-date\"\n            ):\n                continue\n\n            line = (\n                f\"<fg={color}>\"\n                f\"{name:{name_length - len(install_marker)}}{install_marker}</>\"\n            )\n            if write_version:\n                version = get_package_version_display_string(\n                    locked, root=self.poetry.file.path.parent\n                )\n                line += f\" <b>{version:{version_length}}</b>\"\n            if show_latest:\n                latest = latest_packages[locked.pretty_name]\n                update_status = latest_statuses[locked.pretty_name]\n\n                if write_latest:\n                    color = \"green\"\n                    if update_status == \"semver-safe-update\":\n                        color = \"red\"\n                    elif update_status == \"update-possible\":\n                        color = \"yellow\"\n\n                    version = get_package_version_display_string(\n                        latest, root=self.poetry.file.path.parent\n                    )\n                    line += f\" <fg={color}>{version:{latest_length}}</>\"\n\n            if write_why:\n                required_by = reverse_deps(locked, locked_repository)\n                if required_by:\n                    content = \",\".join(required_by.keys())\n                    # subtract 6 for ' from '\n                    line += f\" from {content:{required_by_length - 6}}\"\n                else:\n                    line += \" \" * required_by_length\n\n            if write_description:\n                description = locked.description\n                remaining = (\n                    width - name_length - version_length - required_by_length - 4\n                )\n\n                if show_latest:\n                    remaining -= latest_length\n\n                if len(locked.description) > remaining:\n                    description = description[: remaining - 3] + \"...\"\n\n                line += \" \" + description\n\n            self.line(line)\n\n        return 0\n\n    def _display_packages_tree_information(\n        self, locked_repository: Repository, root: ProjectPackage\n    ) -> int:\n        packages = locked_repository.packages\n\n        for p in packages:\n            for require in root.all_requires:\n                if p.name == require.name:\n                    self.display_package_tree(self.io, p, packages)\n                    break\n\n        return 0\n\n    def display_package_tree(\n        self,\n        io: IO,\n        package: Package,\n        installed_packages: list[Package],\n        why_package: Package | None = None,\n    ) -> None:\n        io.write(f\"<c1>{package.pretty_name}</c1>\")\n        description = \"\"\n        if package.description:\n            description = \" \" + package.description\n\n        io.write_line(f\" <b>{package.pretty_version}</b>{description}\")\n\n        if why_package is not None:\n            dependencies = [p for p in package.requires if p.name == why_package.name]\n        else:\n            dependencies = package.requires\n            dependencies = sorted(\n                dependencies,\n                key=lambda x: x.name,\n            )\n\n        tree_bar = \"\u251c\"\n        total = len(dependencies)\n        for i, dependency in enumerate(dependencies, 1):\n            if i == total:\n                tree_bar = \"\u2514\"\n\n            level = 1\n            color = self.colors[level]\n            info = (\n                f\"{tree_bar}\u2500\u2500 <{color}>{dependency.name}</{color}>\"\n                f\" {dependency.pretty_constraint}\"\n            )\n            self._write_tree_line(io, info)\n\n            tree_bar = tree_bar.replace(\"\u2514\", \" \")\n            packages_in_tree = [package.name, dependency.name]\n\n            self._display_tree(\n                io,\n                dependency,\n                installed_packages,\n                packages_in_tree,\n                tree_bar,\n                level + 1,\n            )\n\n    def _display_tree(\n        self,\n        io: IO,\n        dependency: Dependency,\n        installed_packages: list[Package],\n        packages_in_tree: list[NormalizedName],\n        previous_tree_bar: str = \"\u251c\",\n        level: int = 1,\n    ) -> None:\n        previous_tree_bar = previous_tree_bar.replace(\"\u251c\", \"\u2502\")\n\n        dependencies = []\n        for package in installed_packages:\n            if package.name == dependency.name:\n                dependencies = package.requires\n\n                break\n\n        dependencies = sorted(\n            dependencies,\n            key=lambda x: x.name,\n        )\n        tree_bar = previous_tree_bar + \"   \u251c\"\n        total = len(dependencies)\n        for i, dependency in enumerate(dependencies, 1):\n            current_tree = packages_in_tree\n            if i == total:\n                tree_bar = previous_tree_bar + \"   \u2514\"\n\n            color_ident = level % len(self.colors)\n            color = self.colors[color_ident]\n\n            circular_warn = \"\"\n            if dependency.name in current_tree:\n                circular_warn = \"(circular dependency aborted here)\"\n\n            info = (\n                f\"{tree_bar}\u2500\u2500 <{color}>{dependency.name}</{color}>\"\n                f\" {dependency.pretty_constraint} {circular_warn}\"\n            )\n            self._write_tree_line(io, info)\n\n            tree_bar = tree_bar.replace(\"\u2514\", \" \")\n\n            if dependency.name not in current_tree:\n                current_tree.append(dependency.name)\n\n                self._display_tree(\n                    io,\n                    dependency,\n                    installed_packages,\n                    current_tree,\n                    tree_bar,\n                    level + 1,\n                )\n\n    def _write_tree_line(self, io: IO, line: str) -> None:\n        if not io.output.supports_utf8():\n            line = line.replace(\"\u2514\", \"`-\")\n            line = line.replace(\"\u251c\", \"|-\")\n            line = line.replace(\"\u2500\u2500\", \"-\")\n            line = line.replace(\"\u2502\", \"|\")\n\n        io.write_line(line)\n\n    def init_styles(self, io: IO) -> None:\n        from cleo.formatters.style import Style\n\n        for color in self.colors:\n            style = Style(color)\n            io.output.formatter.set_style(color, style)\n            io.error_output.formatter.set_style(color, style)\n\n    def find_latest_package(\n        self, package: Package, root: ProjectPackage\n    ) -> Package | None:\n        from cleo.io.null_io import NullIO\n\n        from poetry.puzzle.provider import Provider\n        from poetry.version.version_selector import VersionSelector\n\n        # find the latest version allowed in this pool\n        requires = root.all_requires\n        if package.is_direct_origin():\n            for dep in requires:\n                if dep.name == package.name and dep.source_type == package.source_type:\n                    provider = Provider(root, self.poetry.pool, NullIO())\n                    return provider.search_for_direct_origin_dependency(dep)\n\n        allow_prereleases = False\n        for dep in requires:\n            if dep.name == package.name:\n                allow_prereleases = dep.allows_prereleases()\n                break\n\n        name = package.name\n        selector = VersionSelector(self.poetry.pool)\n\n        return selector.find_best_candidate(\n            name, f\">={package.pretty_version}\", allow_prereleases\n        )\n\n    def get_update_status(self, latest: Package, package: Package) -> str:\n        from poetry.core.constraints.version import parse_constraint\n\n        if latest.full_pretty_version == package.full_pretty_version:\n            return \"up-to-date\"\n\n        constraint = parse_constraint(\"^\" + package.pretty_version)\n\n        if constraint.allows(latest.version):\n            # It needs an immediate semver-compliant upgrade\n            return \"semver-safe-update\"\n\n        # it needs an upgrade but has potential BC breaks so is not urgent\n        return \"update-possible\"\n\n    def get_installed_status(\n        self, locked: Package, installed_packages: list[Package]\n    ) -> str:\n        for package in installed_packages:\n            if locked.name == package.name:\n                return \"installed\"\n\n        return \"not-installed\"\n\n# Path: src/poetry/console/commands/update.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass UpdateCommand(InstallerCommand):\n    name = \"update\"\n    description = (\n        \"Update the dependencies as according to the <comment>pyproject.toml</> file.\"\n    )\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"packages\", \"The packages to update\", optional=True, multiple=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        *InstallerCommand._group_dependency_options(),\n        option(\n            \"no-dev\",\n            None,\n            \"Do not update the development dependencies.\"\n            \" (<warning>Deprecated</warning>)\",\n        ),\n        option(\n            \"sync\",\n            None,\n            \"Synchronize the environment with the locked packages and the specified\"\n            \" groups.\",\n        ),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything \"\n            \"(implicitly enables --verbose).\",\n        ),\n        option(\"lock\", None, \"Do not perform operations (only update the lockfile).\"),\n    ]\n\n    loggers: ClassVar[list[str]] = [\"poetry.repositories.pypi_repository\"]\n\n    def handle(self) -> int:\n        packages = self.argument(\"packages\")\n        if packages:\n            self.installer.whitelist({name: \"*\" for name in packages})\n\n        self.installer.only_groups(self.activated_groups)\n        self.installer.dry_run(self.option(\"dry-run\"))\n        self.installer.requires_synchronization(self.option(\"sync\"))\n        self.installer.execute_operations(not self.option(\"lock\"))\n\n        # Force update\n        self.installer.update(True)\n\n        return self.installer.run()\n\n# Path: src/poetry/console/commands/version.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom poetry.core.version.exceptions import InvalidVersion\nfrom tomlkit.toml_document import TOMLDocument\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n    from poetry.core.constraints.version import Version\n\n\nclass VersionCommand(Command):\n    name = \"version\"\n    description = (\n        \"Shows the version of the project or bumps it when a valid \"\n        \"bump rule is provided.\"\n    )\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\n            \"version\",\n            \"The version number or the rule to update the version.\",\n            optional=True,\n        ),\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\"short\", \"s\", \"Output the version number only\"),\n        option(\n            \"dry-run\",\n            None,\n            \"Do not update pyproject.toml file\",\n        ),\n        option(\"next-phase\", None, \"Increment the phase of the current version\"),\n    ]\n\n    help = \"\"\"\\\nThe version command shows the current version of the project or bumps the version of\nthe project and writes the new version back to <comment>pyproject.toml</> if a valid\nbump rule is provided.\n\nThe new version should ideally be a valid semver string or a valid bump rule:\npatch, minor, major, prepatch, preminor, premajor, prerelease.\n\"\"\"\n\n    RESERVED: ClassVar[set[str]] = {\n        \"major\",\n        \"minor\",\n        \"patch\",\n        \"premajor\",\n        \"preminor\",\n        \"prepatch\",\n        \"prerelease\",\n    }\n\n    def handle(self) -> int:\n        version = self.argument(\"version\")\n\n        if version:\n            version = self.increment_version(\n                self.poetry.package.pretty_version, version, self.option(\"next-phase\")\n            )\n\n            if self.option(\"short\"):\n                self.line(version.to_string())\n            else:\n                self.line(\n                    f\"Bumping version from <b>{self.poetry.package.pretty_version}</>\"\n                    f\" to <fg=green>{version}</>\"\n                )\n\n            if not self.option(\"dry-run\"):\n                content: dict[str, Any] = self.poetry.file.read()\n                poetry_content = content[\"tool\"][\"poetry\"]\n                poetry_content[\"version\"] = version.text\n\n                assert isinstance(content, TOMLDocument)\n                self.poetry.file.write(content)\n        else:\n            if self.option(\"short\"):\n                self.line(self.poetry.package.pretty_version)\n            else:\n                self.line(\n                    f\"<comment>{self.poetry.package.pretty_name}</>\"\n                    f\" <info>{self.poetry.package.pretty_version}</>\"\n                )\n\n        return 0\n\n    def increment_version(\n        self, version: str, rule: str, next_phase: bool = False\n    ) -> Version:\n        from poetry.core.constraints.version import Version\n...\n", "needle_token_start": 705, "needle_token_end": 894, "code_context_ntokens": 16019, "output": ["```python\ndef _find_best_version_for_package(\n        self,\n        name: str,\n        required_version: str | None = None,\n        allow_prereleases: bool = False,\n        source: str | None = None,\n    ) -> tuple[str, str]:\n        from poetry.version.version_selector import VersionSelector\n\n        selector = VersionSelector(self._get_pool())\n        package = selector.find_best_candidate(\n            name, required_version, allow_prereleases=allow_prereleases, source=source\n        )\n\n        if not package:\n            # TODO: find similar\n            raise ValueError(f\"Could not find a matching version of package {name}\")\n\n        return package.pretty_name, f\"^{package.version.to_string()}\"\n```"]}
{"repo": "python-poetry/poetry", "name": "get_package_vcs_properties_from_path", "language": "python", "path": "src/poetry/repositories/installed_repository.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To retrieve version control system (VCS) details for a given package directory, specifically identifying the type of VCS, its remote origin URL, and the current revision hash.\n2. **Input**: A path to the directory where the package is located.\n3. **Output**: A tuple containing three strings: the type of VCS (e.g., 'git'), the URL of the remote origin, and the current revision hash.\n4. **Procedure**: The function utilizes a VCS utility to extract information from the specified directory. It first checks the VCS type and then gathers the remote origin URL and the current revision hash from the VCS metadata associated with the directory.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/repositories/http_repository.py\nfrom __future__ import annotations\n\nimport functools\nimport hashlib\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import Iterator\n\nimport requests\nimport requests.adapters\n\nfrom packaging.metadata import parse_email\nfrom poetry.core.constraints.version import parse_constraint\nfrom poetry.core.packages.dependency import Dependency\nfrom poetry.core.utils.helpers import temporary_directory\nfrom poetry.core.version.markers import parse_marker\n\nfrom poetry.config.config import Config\nfrom poetry.inspection.info import PackageInfo\nfrom poetry.inspection.lazy_wheel import LazyWheelUnsupportedError\nfrom poetry.inspection.lazy_wheel import metadata_from_wheel_url\nfrom poetry.repositories.cached_repository import CachedRepository\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.exceptions import RepositoryError\nfrom poetry.repositories.link_sources.html import HTMLPage\nfrom poetry.utils.authenticator import Authenticator\nfrom poetry.utils.constants import REQUESTS_TIMEOUT\nfrom poetry.utils.helpers import HTTPRangeRequestSupported\nfrom poetry.utils.helpers import download_file\nfrom poetry.utils.helpers import get_highest_priority_hash_type\nfrom poetry.utils.patterns import wheel_file_re\n\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n    from poetry.core.packages.utils.link import Link\n\n    from poetry.repositories.link_sources.base import LinkSource\n    from poetry.utils.authenticator import RepositoryCertificateConfig\n\n\nclass HTTPRepository(CachedRepository):\n    def __init__(\n        self,\n        name: str,\n        url: str,\n        config: Config | None = None,\n        disable_cache: bool = False,\n        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,\n    ) -> None:\n        super().__init__(name, disable_cache, config)\n        self._url = url\n        if config is None:\n            config = Config.create()\n        self._authenticator = Authenticator(\n            config=config,\n            cache_id=name,\n            disable_cache=disable_cache,\n            pool_size=pool_size,\n        )\n        self._authenticator.add_repository(name, url)\n        self.get_page = functools.lru_cache(maxsize=None)(self._get_page)\n\n        self._lazy_wheel = config.get(\"solver.lazy-wheel\", True)\n        # We are tracking if a domain supports range requests or not to avoid\n        # unnecessary requests.\n        # ATTENTION: A domain might support range requests only for some files, so the\n        # meaning is as follows:\n        # - Domain not in dict: We don't know anything.\n        # - True: The domain supports range requests for at least some files.\n        # - False: The domain does not support range requests for the files we tried.\n        self._supports_range_requests: dict[str, bool] = {}\n\n    @property\n    def session(self) -> Authenticator:\n        return self._authenticator\n\n    @property\n    def url(self) -> str:\n        return self._url\n\n    @property\n    def certificates(self) -> RepositoryCertificateConfig:\n        return self._authenticator.get_certs_for_url(self.url)\n\n    @property\n    def authenticated_url(self) -> str:\n        return self._authenticator.authenticated_url(url=self.url)\n\n    def _download(\n        self, url: str, dest: Path, *, raise_accepts_ranges: bool = False\n    ) -> None:\n        return download_file(\n            url, dest, session=self.session, raise_accepts_ranges=raise_accepts_ranges\n        )\n\n    @contextmanager\n    def _cached_or_downloaded_file(\n        self, link: Link, *, raise_accepts_ranges: bool = False\n    ) -> Iterator[Path]:\n        self._log(f\"Downloading: {link.url}\", level=\"debug\")\n        with temporary_directory() as temp_dir:\n            filepath = Path(temp_dir) / link.filename\n            self._download(\n                link.url, filepath, raise_accepts_ranges=raise_accepts_ranges\n            )\n            yield filepath\n\n    def _get_info_from_wheel(self, link: Link) -> PackageInfo:\n        from poetry.inspection.info import PackageInfo\n\n        netloc = link.netloc\n\n        # If \"lazy-wheel\" is enabled and the domain supports range requests\n        # or we don't know yet, we try range requests.\n        raise_accepts_ranges = self._lazy_wheel\n        if self._lazy_wheel and self._supports_range_requests.get(netloc, True):\n            try:\n                package_info = PackageInfo.from_metadata(\n                    metadata_from_wheel_url(link.filename, link.url, self.session)\n                )\n            except LazyWheelUnsupportedError as e:\n                # Do not set to False if we already know that the domain supports\n                # range requests for some URLs!\n                self._log(\n                    f\"Disabling lazy wheel support for {netloc}: {e}\",\n                    level=\"debug\",\n                )\n                raise_accepts_ranges = False\n                self._supports_range_requests.setdefault(netloc, False)\n            else:\n                self._supports_range_requests[netloc] = True\n                return package_info\n\n        try:\n            with self._cached_or_downloaded_file(\n                link, raise_accepts_ranges=raise_accepts_ranges\n            ) as filepath:\n                return PackageInfo.from_wheel(filepath)\n        except HTTPRangeRequestSupported:\n            # The domain did not support range requests for the first URL(s) we tried,\n            # but supports it for some URLs (especially the current URL),\n            # so we abort the download, update _supports_range_requests to try\n            # range requests for all files and use it for the current URL.\n            self._log(\n                f\"Abort downloading {link.url} because server supports range requests\",\n                level=\"debug\",\n            )\n            self._supports_range_requests[netloc] = True\n            return self._get_info_from_wheel(link)\n\n    def _get_info_from_sdist(self, link: Link) -> PackageInfo:\n...\n# Path: src/poetry/repositories/installed_repository.py\nfrom __future__ import annotations\n\nimport itertools\nimport json\nimport logging\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\n\nfrom packaging.utils import canonicalize_name\nfrom poetry.core.packages.package import Package\nfrom poetry.core.packages.utils.utils import url_to_path\nfrom poetry.core.utils.helpers import module_name\n\nfrom poetry.repositories.repository import Repository\nfrom poetry.utils._compat import metadata\n\n\nif TYPE_CHECKING:\n    from poetry.utils.env import Env\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass InstalledRepository(Repository):\n    def __init__(self) -> None:\n        super().__init__(\"poetry-installed\")\n\n    @classmethod\n    def get_package_paths(cls, env: Env, name: str) -> set[Path]:\n        \"\"\"\n        Process a .pth file within the site-packages directories, and return any valid\n        paths. We skip executable .pth files as there is no reliable means to do this\n        without side-effects to current run-time. Mo check is made that the item refers\n        to a directory rather than a file, however, in order to maintain backwards\n        compatibility, we allow non-existing paths to be discovered. The latter\n        behaviour is different to how Python's site-specific hook configuration works.\n\n        Reference: https://docs.python.org/3.8/library/site.html\n\n        :param env: The environment to search for the .pth file in.\n        :param name: The name of the package to search .pth file for.\n        :return: A `Set` of valid `Path` objects.\n        \"\"\"\n        paths = set()\n\n        # we identify the candidate pth files to check, this is done so to handle cases\n        # where the pth file for foo-bar might have been installed as either foo-bar.pth\n        # or foo_bar.pth (expected) in either pure or platform lib directories.\n        candidates = itertools.product(\n            {env.purelib, env.platlib},\n            {name, module_name(name)},\n        )\n\n        for lib, module in candidates:\n            pth_file = lib.joinpath(module).with_suffix(\".pth\")\n            if not pth_file.exists():\n                continue\n\n            with pth_file.open() as f:\n                for line in f:\n                    line = line.strip()\n                    if line and not line.startswith((\"#\", \"import \", \"import\\t\")):\n                        path = Path(line)\n                        if not path.is_absolute():\n                            path = lib.joinpath(path).resolve()\n                        paths.add(path)\n\n        src_path = env.path / \"src\" / name\n        if not paths and src_path.exists():\n            paths.add(src_path)\n\n        return paths\n\n    @classmethod\n    \ndef get_package_vcs_properties_from_path(cls, src: Path) -> tuple[str, str, str]:\n        from poetry.vcs.git import Git\n\n        info = Git.info(repo=src)\n        return \"git\", info.origin, info.revision\n\n    @classmethod\n    def is_vcs_package(cls, package: Path | Package, env: Env) -> bool:\n        # A VCS dependency should have been installed\n        # in the src directory.\n        src = env.path / \"src\"\n        if isinstance(package, Package):\n            return src.joinpath(package.name).is_dir()\n\n        try:\n            package.relative_to(env.path / \"src\")\n        except ValueError:\n            return False\n        else:\n            return True\n\n    @classmethod\n    def create_package_from_distribution(\n        cls, distribution: metadata.Distribution, env: Env\n    ) -> Package:\n        # We first check for a direct_url.json file to determine\n        # the type of package.\n        path = Path(str(distribution._path))  # type: ignore[attr-defined]\n\n        if (\n            path.name.endswith(\".dist-info\")\n            and path.joinpath(\"direct_url.json\").exists()\n        ):\n            return cls.create_package_from_pep610(distribution)\n\n        is_standard_package = env.is_path_relative_to_lib(path)\n\n        source_type = None\n        source_url = None\n        source_reference = None\n        source_resolved_reference = None\n        source_subdirectory = None\n        if is_standard_package:\n            if path.name.endswith(\".dist-info\"):\n                paths = cls.get_package_paths(\n                    env=env, name=distribution.metadata[\"name\"]\n                )\n                if paths:\n                    is_editable_package = False\n                    for src in paths:\n                        if cls.is_vcs_package(src, env):\n                            (\n                                source_type,\n                                source_url,\n                                source_reference,\n                            ) = cls.get_package_vcs_properties_from_path(src)\n                            break\n\n                        if not (\n                            is_editable_package or env.is_path_relative_to_lib(src)\n                        ):\n                            is_editable_package = True\n                    else:\n                        # TODO: handle multiple source directories?\n                        if is_editable_package:\n                            source_type = \"directory\"\n                            source_url = paths.pop().as_posix()\n        elif cls.is_vcs_package(path, env):\n            (\n                source_type,\n                source_url,\n                source_reference,\n            ) = cls.get_package_vcs_properties_from_path(\n                env.path / \"src\" / canonicalize_name(distribution.metadata[\"name\"])\n            )\n        else:\n            # If not, it's a path dependency\n            source_type = \"directory\"\n            source_url = str(path.parent)\n\n        package = Package(\n            distribution.metadata[\"name\"],\n            distribution.metadata[\"version\"],\n            source_type=source_type,\n            source_url=source_url,\n            source_reference=source_reference,\n            source_resolved_reference=source_resolved_reference,\n            source_subdirectory=source_subdirectory,\n        )\n\n        package.description = distribution.metadata.get(  # type: ignore[attr-defined]\n            \"summary\",\n            \"\",\n        )\n\n        return package\n\n    @classmethod\n    def create_package_from_pep610(cls, distribution: metadata.Distribution) -> Package:\n        path = Path(str(distribution._path))  # type: ignore[attr-defined]\n        source_type = None\n        source_url = None\n        source_reference = None\n        source_resolved_reference = None\n        source_subdirectory = None\n        develop = False\n\n        url_reference = json.loads(\n            path.joinpath(\"direct_url.json\").read_text(encoding=\"utf-8\")\n        )\n        if \"archive_info\" in url_reference:\n            # File or URL distribution\n            if url_reference[\"url\"].startswith(\"file:\"):\n                # File distribution\n                source_type = \"file\"\n                source_url = url_to_path(url_reference[\"url\"]).as_posix()\n            else:\n                # URL distribution\n                source_type = \"url\"\n                source_url = url_reference[\"url\"]\n        elif \"dir_info\" in url_reference:\n            # Directory distribution\n            source_type = \"directory\"\n            source_url = url_to_path(url_reference[\"url\"]).as_posix()\n            develop = url_reference[\"dir_info\"].get(\"editable\", False)\n        elif \"vcs_info\" in url_reference:\n            # VCS distribution\n            source_type = url_reference[\"vcs_info\"][\"vcs\"]\n            source_url = url_reference[\"url\"]\n            source_resolved_reference = url_reference[\"vcs_info\"][\"commit_id\"]\n            source_reference = url_reference[\"vcs_info\"].get(\n                \"requested_revision\", source_resolved_reference\n            )\n        source_subdirectory = url_reference.get(\"subdirectory\")\n\n        package = Package(\n            distribution.metadata[\"name\"],\n            distribution.metadata[\"version\"],\n            source_type=source_type,\n            source_url=source_url,\n            source_reference=source_reference,\n            source_resolved_reference=source_resolved_reference,\n            source_subdirectory=source_subdirectory,\n            develop=develop,\n        )\n\n        package.description = distribution.metadata.get(  # type: ignore[attr-defined]\n            \"summary\",\n            \"\",\n        )\n\n        return package\n\n    @classmethod\n    def load(cls, env: Env, with_dependencies: bool = False) -> InstalledRepository:\n        \"\"\"\n        Load installed packages.\n        \"\"\"\n        from poetry.core.packages.dependency import Dependency\n\n        repo = cls()\n        seen = set()\n        skipped = set()\n\n        for entry in reversed(env.sys_path):\n            if not entry.strip():\n                logger.debug(\n                    \"Project environment contains an empty path in <c1>sys_path</>,\"\n                    \" ignoring.\"\n                )\n                continue\n\n            for distribution in sorted(\n                metadata.distributions(path=[entry]),\n                key=lambda d: str(d._path),  # type: ignore[attr-defined]\n            ):\n                path = Path(str(distribution._path))  # type: ignore[attr-defined]\n\n                if path in skipped:\n                    continue\n\n                name = distribution.metadata.get(\"name\")  # type: ignore[attr-defined]\n                if name is None:\n                    logger.warning(\n                        \"Project environment contains an invalid distribution\"\n                        \" (<c1>%s</>). Consider removing it manually or recreate\"\n                        \" the environment.\",\n                        path,\n                    )\n                    skipped.add(path)\n                    continue\n\n                name = canonicalize_name(name)\n\n                if name in seen:\n                    continue\n\n                package = cls.create_package_from_distribution(distribution, env)\n\n                if with_dependencies:\n                    for require in distribution.metadata.get_all(\"requires-dist\", []):\n                        dep = Dependency.create_from_pep_508(require)\n                        package.add_dependency(dep)\n\n                seen.add(package.name)\n                repo.add_package(package)\n\n        return repo\n\n# Path: src/poetry/repositories/legacy_repository.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nimport requests.adapters\n\nfrom poetry.core.packages.package import Package\n\nfrom poetry.inspection.info import PackageInfo\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.http_repository import HTTPRepository\nfrom poetry.repositories.link_sources.html import SimpleRepositoryPage\n\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n    from poetry.core.constraints.version import Version\n    from poetry.core.constraints.version import VersionConstraint\n    from poetry.core.packages.utils.link import Link\n\n    from poetry.config.config import Config\n\n\nclass LegacyRepository(HTTPRepository):\n    def __init__(\n        self,\n        name: str,\n        url: str,\n        config: Config | None = None,\n        disable_cache: bool = False,\n        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,\n    ) -> None:\n        if name == \"pypi\":\n            raise ValueError(\"The name [pypi] is reserved for repositories\")\n\n        super().__init__(name, url.rstrip(\"/\"), config, disable_cache, pool_size)\n\n    @property\n    def packages(self) -> list[Package]:\n        # LegacyRepository._packages is not populated and other implementations\n        # implicitly rely on this (e.g. Pool.search via\n        # LegacyRepository.search). To avoid special-casing Pool or changing\n        # behavior, we stub and return an empty list.\n        #\n        # TODO: Rethinking search behaviour and design.\n        # Ref: https://github.com/python-poetry/poetry/issues/2446 and\n        # https://github.com/python-poetry/poetry/pull/6669#discussion_r990874908.\n        return []\n\n    def package(\n        self, name: str, version: Version, extras: list[str] | None = None\n    ) -> Package:\n        \"\"\"\n        Retrieve the release information.\n\n        This is a heavy task which takes time.\n        We have to download a package to get the dependencies.\n        We also need to download every file matching this release\n        to get the various hashes.\n\n        Note that this will be cached so the subsequent operations\n        should be much faster.\n        \"\"\"\n        try:\n            index = self._packages.index(Package(name, version))\n\n            return self._packages[index]\n        except ValueError:\n            package = super().package(name, version, extras)\n            package._source_type = \"legacy\"\n            package._source_url = self._url\n            package._source_reference = self.name\n\n            return package\n\n    def find_links_for_package(self, package: Package) -> list[Link]:\n        try:\n            page = self.get_page(package.name)\n        except PackageNotFound:\n            return []\n\n        return list(page.links_for_version(package.name, package.version))\n\n    def _find_packages(\n        self, name: NormalizedName, constraint: VersionConstraint\n    ) -> list[Package]:\n        \"\"\"\n        Find packages on the remote server.\n        \"\"\"\n        try:\n            page = self.get_page(name)\n        except PackageNotFound:\n            self._log(f\"No packages found for {name}\", level=\"debug\")\n            return []\n\n        versions = [\n            (version, page.yanked(name, version))\n            for version in page.versions(name)\n            if constraint.allows(version)\n        ]\n\n        return [\n            Package(\n                name,\n                version,\n                source_type=\"legacy\",\n                source_reference=self.name,\n                source_url=self._url,\n                yanked=yanked,\n            )\n            for version, yanked in versions\n        ]\n\n    def _get_release_info(\n        self, name: NormalizedName, version: Version\n    ) -> dict[str, Any]:\n        page = self.get_page(name)\n\n        links = list(page.links_for_version(name, version))\n        yanked = page.yanked(name, version)\n\n        return self._links_to_data(\n            links,\n            PackageInfo(\n                name=name,\n                version=version.text,\n                summary=\"\",\n                requires_dist=[],\n                requires_python=None,\n                files=[],\n                yanked=yanked,\n                cache_version=str(self.CACHE_VERSION),\n            ),\n        )\n\n    def _get_page(self, name: NormalizedName) -> SimpleRepositoryPage:\n        response = self._get_response(f\"/{name}/\")\n        if not response:\n            raise PackageNotFound(f\"Package [{name}] not found.\")\n        return SimpleRepositoryPage(response.url, response.text)\n\n# Path: src/poetry/repositories/lockfile_repository.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.repositories import Repository\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.package import Package\n\n\nclass LockfileRepository(Repository):\n    \"\"\"\n    Special repository that distinguishes packages not only by name and version,\n    but also by source type, url, etc.\n    \"\"\"\n\n    def __init__(self) -> None:\n        super().__init__(\"poetry-lockfile\")\n\n    def has_package(self, package: Package) -> bool:\n        return any(p == package for p in self.packages)\n\n    def remove_package(self, package: Package) -> None:\n        index = None\n        for i, repo_package in enumerate(self.packages):\n            if repo_package == package:\n                index = i\n                break\n\n        if index is not None:\n            del self._packages[index]\n\n# Path: src/poetry/repositories/pypi_repository.py\nfrom __future__ import annotations\n\nimport logging\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nimport requests\nimport requests.adapters\n\nfrom cachecontrol.controller import logger as cache_control_logger\nfrom poetry.core.packages.package import Package\nfrom poetry.core.packages.utils.link import Link\nfrom poetry.core.version.exceptions import InvalidVersion\n\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.http_repository import HTTPRepository\nfrom poetry.repositories.link_sources.json import SimpleJsonPage\nfrom poetry.repositories.parsers.pypi_search_parser import SearchResultParser\nfrom poetry.utils.constants import REQUESTS_TIMEOUT\n\n\ncache_control_logger.setLevel(logging.ERROR)\n\nlogger = logging.getLogger(__name__)\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n    from poetry.core.constraints.version import Version\n    from poetry.core.constraints.version import VersionConstraint\n\nSUPPORTED_PACKAGE_TYPES = {\"sdist\", \"bdist_wheel\"}\n\n\nclass PyPiRepository(HTTPRepository):\n    def __init__(\n        self,\n        url: str = \"https://pypi.org/\",\n        disable_cache: bool = False,\n        fallback: bool = True,\n        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,\n    ) -> None:\n        super().__init__(\n            \"PyPI\",\n            url.rstrip(\"/\") + \"/simple/\",\n            disable_cache=disable_cache,\n            pool_size=pool_size,\n        )\n\n        self._base_url = url\n        self._fallback = fallback\n\n    def search(self, query: str) -> list[Package]:\n        results = []\n\n        response = requests.get(\n            self._base_url + \"search\", params={\"q\": query}, timeout=REQUESTS_TIMEOUT\n        )\n        parser = SearchResultParser()\n        parser.feed(response.text)\n\n        for result in parser.results:\n            try:\n                package = Package(result.name, result.version)\n                package.description = result.description.strip()\n                results.append(package)\n            except InvalidVersion:\n                self._log(\n                    f'Unable to parse version \"{result.version}\" for the'\n                    f\" {result.name} package, skipping\",\n                    level=\"debug\",\n                )\n\n        return results\n\n    def get_package_info(self, name: NormalizedName) -> dict[str, Any]:\n        \"\"\"\n        Return the package information given its name.\n\n        The information is returned from the cache if it exists\n        or retrieved from the remote server.\n        \"\"\"\n        return self._get_package_info(name)\n\n    def _find_packages(\n        self, name: NormalizedName, constraint: VersionConstraint\n    ) -> list[Package]:\n        \"\"\"\n        Find packages on the remote server.\n        \"\"\"\n        try:\n            json_page = self.get_page(name)\n        except PackageNotFound:\n            self._log(f\"No packages found for {name}\", level=\"debug\")\n            return []\n\n        versions = [\n            (version, json_page.yanked(name, version))\n            for version in json_page.versions(name)\n            if constraint.allows(version)\n        ]\n\n        return [Package(name, version, yanked=yanked) for version, yanked in versions]\n\n    def _get_package_info(self, name: NormalizedName) -> dict[str, Any]:\n        headers = {\"Accept\": \"application/vnd.pypi.simple.v1+json\"}\n        info = self._get(f\"simple/{name}/\", headers=headers)\n        if info is None:\n            raise PackageNotFound(f\"Package [{name}] not found.\")\n\n        return info\n\n    def find_links_for_package(self, package: Package) -> list[Link]:\n        json_data = self._get(f\"pypi/{package.name}/{package.version}/json\")\n        if json_data is None:\n            return []\n\n        links = []\n        for url in json_data[\"urls\"]:\n            if url[\"packagetype\"] in SUPPORTED_PACKAGE_TYPES:\n                h = f\"sha256={url['digests']['sha256']}\"\n                links.append(Link(url[\"url\"] + \"#\" + h, yanked=self._get_yanked(url)))\n\n        return links\n\n    def _get_release_info(\n        self, name: NormalizedName, version: Version\n    ) -> dict[str, Any]:\n        from poetry.inspection.info import PackageInfo\n\n        self._log(f\"Getting info for {name} ({version}) from PyPI\", \"debug\")\n\n        json_data = self._get(f\"pypi/{name}/{version}/json\")\n        if json_data is None:\n            raise PackageNotFound(f\"Package [{name}] not found.\")\n\n        info = json_data[\"info\"]\n\n        data = PackageInfo(\n            name=info[\"name\"],\n            version=info[\"version\"],\n            summary=info[\"summary\"],\n            requires_dist=info[\"requires_dist\"],\n            requires_python=info[\"requires_python\"],\n            yanked=self._get_yanked(info),\n            cache_version=str(self.CACHE_VERSION),\n        )\n\n        try:\n            version_info = json_data[\"urls\"]\n        except KeyError:\n            version_info = []\n\n        files = info.get(\"files\", [])\n        for file_info in version_info:\n            if file_info[\"packagetype\"] in SUPPORTED_PACKAGE_TYPES:\n                files.append(\n                    {\n                        \"file\": file_info[\"filename\"],\n                        \"hash\": \"sha256:\" + file_info[\"digests\"][\"sha256\"],\n                    }\n                )\n        data.files = files\n\n        if self._fallback and data.requires_dist is None:\n            self._log(\n                \"No dependencies found, downloading metadata and/or archives\",\n                level=\"debug\",\n            )\n            # No dependencies set (along with other information)\n            # This might be due to actually no dependencies\n            # or badly set metadata when uploading.\n            # So, we need to make sure there is actually no\n            # dependencies by introspecting packages.\n            page = self.get_page(name)\n            links = list(page.links_for_version(name, version))\n            info = self._get_info_from_links(links)\n\n            data.requires_dist = info.requires_dist\n\n            if not data.requires_python:\n                data.requires_python = info.requires_python\n\n        return data.asdict()\n\n    def _get_page(self, name: NormalizedName) -> SimpleJsonPage:\n        source = self._base_url + f\"simple/{name}/\"\n        info = self.get_package_info(name)\n        return SimpleJsonPage(source, info)\n\n    def _get(\n        self, endpoint: str, headers: dict[str, str] | None = None\n    ) -> dict[str, Any] | None:\n        try:\n            json_response = self.session.get(\n                self._base_url + endpoint,\n                raise_for_status=False,\n                timeout=REQUESTS_TIMEOUT,\n                headers=headers,\n            )\n        except requests.exceptions.TooManyRedirects:\n            # Cache control redirect loop.\n            # We try to remove the cache and try again\n            self.session.delete_cache(self._base_url + endpoint)\n            json_response = self.session.get(\n                self._base_url + endpoint,\n                raise_for_status=False,\n                timeout=REQUESTS_TIMEOUT,\n                headers=headers,\n            )\n\n        if json_response.status_code != 200:\n            return None\n\n        json: dict[str, Any] = json_response.json()\n        return json\n\n    @staticmethod\n    def _get_yanked(json_data: dict[str, Any]) -> str | bool:\n        if json_data.get(\"yanked\", False):\n            return json_data.get(\"yanked_reason\") or True\n        return False\n\n# Path: src/poetry/repositories/repository.py\nfrom __future__ import annotations\n\nimport logging\n\nfrom typing import TYPE_CHECKING\n\nfrom packaging.utils import canonicalize_name\nfrom poetry.core.constraints.version import Version\n\nfrom poetry.repositories.abstract_repository import AbstractRepository\nfrom poetry.repositories.exceptions import PackageNotFound\n\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n    from poetry.core.constraints.version import VersionConstraint\n    from poetry.core.packages.dependency import Dependency\n    from poetry.core.packages.package import Package\n    from poetry.core.packages.utils.link import Link\n\n\nclass Repository(AbstractRepository):\n    def __init__(self, name: str, packages: list[Package] | None = None) -> None:\n        super().__init__(name)\n        self._packages: list[Package] = []\n\n        for package in packages or []:\n            self.add_package(package)\n\n    @property\n    def packages(self) -> list[Package]:\n        return self._packages\n\n    def find_packages(self, dependency: Dependency) -> list[Package]:\n        packages = []\n        ignored_pre_release_packages = []\n\n        constraint = dependency.constraint\n        allow_prereleases = dependency.allows_prereleases()\n        for package in self._find_packages(dependency.name, constraint):\n            if package.yanked and not isinstance(constraint, Version):\n                # PEP 592: yanked files are always ignored, unless they are the only\n                # file that matches a version specifier that \"pins\" to an exact\n                # version\n                continue\n            if (\n                package.is_prerelease()\n                and not allow_prereleases\n                and not package.is_direct_origin()\n            ):\n                ignored_pre_release_packages.append(package)\n                continue\n\n            packages.append(package)\n\n        self._log(\n            f\"{len(packages)} packages found for {dependency.name} {constraint!s}\",\n            level=\"debug\",\n        )\n\n        return packages or ignored_pre_release_packages\n\n    def has_package(self, package: Package) -> bool:\n        package_id = package.unique_name\n        return any(\n            package_id == repo_package.unique_name for repo_package in self.packages\n        )\n\n    def add_package(self, package: Package) -> None:\n        self._packages.append(package)\n\n    def remove_package(self, package: Package) -> None:\n        package_id = package.unique_name\n\n        index = None\n        for i, repo_package in enumerate(self.packages):\n            if package_id == repo_package.unique_name:\n                index = i\n                break\n\n        if index is not None:\n            del self._packages[index]\n\n    def search(self, query: str) -> list[Package]:\n        results: list[Package] = []\n\n        for package in self.packages:\n            if query in package.name:\n                results.append(package)\n\n        return results\n\n    def _find_packages(\n        self, name: NormalizedName, constraint: VersionConstraint\n    ) -> list[Package]:\n        return [\n            package\n            for package in self._packages\n            if package.name == name and constraint.allows(package.version)\n        ]\n\n    def _log(self, msg: str, level: str = \"info\") -> None:\n        logger = logging.getLogger(f\"{__name__}.{self.__class__.__name__}\")\n        getattr(logger, level)(f\"<c1>Source ({self.name}):</c1> {msg}\")\n\n    def __len__(self) -> int:\n        return len(self._packages)\n\n    def find_links_for_package(self, package: Package) -> list[Link]:\n        return []\n\n    def package(\n        self, name: str, version: Version, extras: list[str] | None = None\n    ) -> Package:\n        canonicalized_name = canonicalize_name(name)\n        for package in self.packages:\n            if canonicalized_name == package.name and package.version == version:\n                return package\n\n        raise PackageNotFound(f\"Package {name} ({version}) not found.\")\n\n# Path: src/poetry/repositories/repository_pool.py\nfrom __future__ import annotations\n\nimport enum\nimport warnings\n\nfrom collections import OrderedDict\nfrom dataclasses import dataclass\nfrom enum import IntEnum\nfrom typing import TYPE_CHECKING\n\nfrom poetry.config.config import Config\nfrom poetry.repositories.abstract_repository import AbstractRepository\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.repository import Repository\nfrom poetry.utils.cache import ArtifactCache\n\n\nif TYPE_CHECKING:\n    from poetry.core.constraints.version import Version\n    from poetry.core.packages.dependency import Dependency\n    from poetry.core.packages.package import Package\n\n_SENTINEL = object()\n\n\nclass Priority(IntEnum):\n    # The order of the members below dictates the actual priority. The first member has\n    # top priority.\n    DEFAULT = enum.auto()\n    PRIMARY = enum.auto()\n    SECONDARY = enum.auto()\n    SUPPLEMENTAL = enum.auto()\n    EXPLICIT = enum.auto()\n\n\n@dataclass(frozen=True)\nclass PrioritizedRepository:\n    repository: Repository\n    priority: Priority\n\n\nclass RepositoryPool(AbstractRepository):\n    def __init__(\n        self,\n        repositories: list[Repository] | None = None,\n        ignore_repository_names: object = _SENTINEL,\n        *,\n        config: Config | None = None,\n    ) -> None:\n        super().__init__(\"poetry-repository-pool\")\n        self._repositories: OrderedDict[str, PrioritizedRepository] = OrderedDict()\n\n        if repositories is None:\n            repositories = []\n        for repository in repositories:\n            self.add_repository(repository)\n\n        self._artifact_cache = ArtifactCache(\n            cache_dir=(config or Config.create()).artifacts_cache_directory\n        )\n\n        if ignore_repository_names is not _SENTINEL:\n            warnings.warn(\n                \"The 'ignore_repository_names' argument to 'RepositoryPool.__init__' is\"\n                \" deprecated. It has no effect anymore and will be removed in a future\"\n                \" version.\",\n                DeprecationWarning,\n                stacklevel=2,\n            )\n\n    @staticmethod\n    def from_packages(packages: list[Package], config: Config | None) -> RepositoryPool:\n        pool = RepositoryPool(config=config)\n        for package in packages:\n            if package.is_direct_origin():\n                continue\n\n            repo_name = package.source_reference or \"PyPI\"\n            try:\n                repo = pool.repository(repo_name)\n            except IndexError:\n                repo = Repository(repo_name)\n                pool.add_repository(repo)\n\n            if not repo.has_package(package):\n                repo.add_package(package)\n\n        return pool\n\n    @property\n    def repositories(self) -> list[Repository]:\n        \"\"\"\n        Returns the repositories in the pool,\n        in the order they will be searched for packages.\n\n        ATTENTION: For backwards compatibility and practical reasons,\n                   repositories with priority EXPLICIT are NOT included,\n                   because they will not be searched.\n        \"\"\"\n        sorted_repositories = self._sorted_repositories\n        return [\n            prio_repo.repository\n            for prio_repo in sorted_repositories\n            if prio_repo.priority is not Priority.EXPLICIT\n        ]\n\n    @property\n    def all_repositories(self) -> list[Repository]:\n        return [prio_repo.repository for prio_repo in self._sorted_repositories]\n\n    @property\n    def _sorted_repositories(self) -> list[PrioritizedRepository]:\n        return sorted(\n            self._repositories.values(), key=lambda prio_repo: prio_repo.priority\n        )\n\n    @property\n    def artifact_cache(self) -> ArtifactCache:\n        return self._artifact_cache\n\n    def has_default(self) -> bool:\n        return self._contains_priority(Priority.DEFAULT)\n\n    def has_primary_repositories(self) -> bool:\n        return self._contains_priority(Priority.PRIMARY)\n\n    def _contains_priority(self, priority: Priority) -> bool:\n        return any(\n            prio_repo.priority is priority for prio_repo in self._repositories.values()\n        )\n\n    def has_repository(self, name: str) -> bool:\n        return name.lower() in self._repositories\n\n    def repository(self, name: str) -> Repository:\n        return self._get_prioritized_repository(name).repository\n\n    def get_priority(self, name: str) -> Priority:\n        return self._get_prioritized_repository(name).priority\n\n    def _get_prioritized_repository(self, name: str) -> PrioritizedRepository:\n        name = name.lower()\n        if self.has_repository(name):\n            return self._repositories[name]\n        raise IndexError(f'Repository \"{name}\" does not exist.')\n\n    def add_repository(\n        self,\n        repository: Repository,\n        default: bool = False,\n        secondary: bool = False,\n        *,\n        priority: Priority = Priority.PRIMARY,\n    ) -> RepositoryPool:\n        \"\"\"\n        Adds a repository to the pool.\n        \"\"\"\n        repository_name = repository.name.lower()\n        if self.has_repository(repository_name):\n            raise ValueError(\n                f\"A repository with name {repository_name} was already added.\"\n            )\n\n        if default or secondary:\n            warnings.warn(\n                \"Parameters 'default' and 'secondary' to\"\n                \" 'RepositoryPool.add_repository' are deprecated. Please provide\"\n                \" the keyword-argument 'priority' instead.\",\n                DeprecationWarning,\n                stacklevel=2,\n            )\n            priority = Priority.DEFAULT if default else Priority.SECONDARY\n\n        if priority is Priority.DEFAULT and self.has_default():\n            raise ValueError(\"Only one repository can be the default.\")\n\n        self._repositories[repository_name] = PrioritizedRepository(\n            repository, priority\n        )\n        return self\n\n    def remove_repository(self, name: str) -> RepositoryPool:\n        if not self.has_repository(name):\n            raise IndexError(\n                f\"RepositoryPool can not remove unknown repository '{name}'.\"\n            )\n        del self._repositories[name.lower()]\n        return self\n\n    def package(\n        self,\n        name: str,\n        version: Version,\n        extras: list[str] | None = None,\n        repository_name: str | None = None,\n    ) -> Package:\n        if repository_name:\n            return self.repository(repository_name).package(\n                name, version, extras=extras\n            )\n\n        for repo in self.repositories:\n            try:\n                return repo.package(name, version, extras=extras)\n            except PackageNotFound:\n                continue\n        raise PackageNotFound(f\"Package {name} ({version}) not found.\")\n\n    def find_packages(self, dependency: Dependency) -> list[Package]:\n        repository_name = dependency.source_name\n        if repository_name:\n            return self.repository(repository_name).find_packages(dependency)\n\n        packages: list[Package] = []\n        for repo in self.repositories:\n            if packages and self.get_priority(repo.name) is Priority.SUPPLEMENTAL:\n                break\n            packages += repo.find_packages(dependency)\n        return packages\n\n    def search(self, query: str) -> list[Package]:\n        results: list[Package] = []\n        for repo in self.repositories:\n            results += repo.search(query)\n        return results\n\n# Path: src/poetry/repositories/single_page_repository.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.legacy_repository import LegacyRepository\nfrom poetry.repositories.link_sources.html import SimpleRepositoryPage\n\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n\n\nclass SinglePageRepository(LegacyRepository):\n    def _get_page(self, name: NormalizedName) -> SimpleRepositoryPage:\n        \"\"\"\n        Single page repositories only have one page irrespective of endpoint.\n        \"\"\"\n        response = self._get_response(\"\")\n        if not response:\n            raise PackageNotFound(f\"Package [{name}] not found.\")\n        return SimpleRepositoryPage(response.url, response.text)\n\n# Path: src/poetry/toml/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.toml.exceptions import TOMLError\nfrom poetry.toml.file import TOMLFile\n\n\n__all__ = [\"TOMLError\", \"TOMLFile\"]\n\n# Path: src/poetry/toml/exceptions.py\nfrom __future__ import annotations\n\nfrom poetry.core.exceptions import PoetryCoreException\nfrom tomlkit.exceptions import TOMLKitError\n\n\nclass TOMLError(TOMLKitError, PoetryCoreException):\n    pass\n\n# Path: src/poetry/toml/file.py\nfrom __future__ import annotations\n\nimport warnings\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom tomlkit.toml_file import TOMLFile as BaseTOMLFile\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n\n    from tomlkit.toml_document import TOMLDocument\n\n\nclass TOMLFile(BaseTOMLFile):\n    def __init__(self, path: Path) -> None:\n        super().__init__(path)\n        self.__path = path\n\n    @property\n    def path(self) -> Path:\n        return self.__path\n\n    def exists(self) -> bool:\n        return self.__path.exists()\n\n    def read(self) -> TOMLDocument:\n        from tomlkit.exceptions import TOMLKitError\n\n        from poetry.toml import TOMLError\n\n        try:\n            return super().read()\n        except (ValueError, TOMLKitError) as e:\n            raise TOMLError(f\"Invalid TOML file {self.path.as_posix()}: {e}\")\n\n    def __getattr__(self, item: str) -> Any:\n        warnings.warn(\n            \"`__getattr__` will be removed from the `TOMLFile` in a future release.\"\n            \"\\n\\nInstead of accessing properties of the underlying `Path` as \"\n            \"`tomlfile.whatever`, prefer `tomlfile.path.whatever`.\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        return getattr(self.__path, item)\n\n    def __str__(self) -> str:\n        return self.__path.as_posix()\n\n# Path: src/poetry/utils/__init__.py\n\n# Path: src/poetry/utils/_compat.py\nfrom __future__ import annotations\n\nimport sys\n\nfrom contextlib import suppress\n\n\n# TODO: use try/except ImportError when\n# https://github.com/python/mypy/issues/1393 is fixed\n\nif sys.version_info < (3, 11):\n    # compatibility for python <3.11\n    import tomli as tomllib\nelse:\n    import tomllib  # nopycln: import\n\n\nif sys.version_info < (3, 10):\n    # compatibility for python <3.10\n    import importlib_metadata as metadata\nelse:\n    from importlib import metadata\n\nWINDOWS = sys.platform == \"win32\"\n\n\ndef decode(string: bytes | str, encodings: list[str] | None = None) -> str:\n    if not isinstance(string, bytes):\n        return string\n\n    encodings = encodings or [\"utf-8\", \"latin1\", \"ascii\"]\n\n    for encoding in encodings:\n        with suppress(UnicodeEncodeError, UnicodeDecodeError):\n            return string.decode(encoding)\n\n    return string.decode(encodings[0], errors=\"ignore\")\n\n\ndef encode(string: str, encodings: list[str] | None = None) -> bytes:\n    if isinstance(string, bytes):\n        return string\n\n    encodings = encodings or [\"utf-8\", \"latin1\", \"ascii\"]\n\n    for encoding in encodings:\n        with suppress(UnicodeEncodeError, UnicodeDecodeError):\n            return string.encode(encoding)\n\n    return string.encode(encodings[0], errors=\"ignore\")\n\n\n__all__ = [\n    \"WINDOWS\",\n    \"decode\",\n    \"encode\",\n    \"metadata\",\n    \"tomllib\",\n]\n\n# Path: src/poetry/utils/authenticator.py\nfrom __future__ import annotations\n\nimport contextlib\nimport dataclasses\nimport functools\nimport logging\nimport time\nimport urllib.parse\n\nfrom os.path import commonprefix\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nimport requests\nimport requests.adapters\nimport requests.auth\nimport requests.exceptions\n\nfrom cachecontrol import CacheControlAdapter\nfrom cachecontrol.caches import FileCache\nfrom requests_toolbelt import user_agent\n\nfrom poetry.__version__ import __version__\nfrom poetry.config.config import Config\nfrom poetry.exceptions import PoetryException\nfrom poetry.utils.constants import REQUESTS_TIMEOUT\nfrom poetry.utils.constants import RETRY_AFTER_HEADER\nfrom poetry.utils.constants import STATUS_FORCELIST\nfrom poetry.utils.password_manager import HTTPAuthCredential\nfrom poetry.utils.password_manager import PasswordManager\n\n\nif TYPE_CHECKING:\n    from cleo.io.io import IO\n\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclasses.dataclass(frozen=True)\nclass RepositoryCertificateConfig:\n    cert: Path | None = dataclasses.field(default=None)\n    client_cert: Path | None = dataclasses.field(default=None)\n    verify: bool = dataclasses.field(default=True)\n\n    @classmethod\n    def create(\n        cls, repository: str, config: Config | None\n    ) -> RepositoryCertificateConfig:\n        config = config if config else Config.create()\n\n        verify: str | bool = config.get(\n            f\"certificates.{repository}.verify\",\n            config.get(f\"certificates.{repository}.cert\", True),\n        )\n        client_cert: str = config.get(f\"certificates.{repository}.client-cert\")\n\n        return cls(\n            cert=Path(verify) if isinstance(verify, str) else None,\n            client_cert=Path(client_cert) if client_cert else None,\n            verify=verify if isinstance(verify, bool) else True,\n        )\n\n\n@dataclasses.dataclass\nclass AuthenticatorRepositoryConfig:\n    name: str\n    url: str\n    netloc: str = dataclasses.field(init=False)\n    path: str = dataclasses.field(init=False)\n\n    def __post_init__(self) -> None:\n        parsed_url = urllib.parse.urlsplit(self.url)\n        self.netloc = parsed_url.netloc\n        self.path = parsed_url.path\n\n    def certs(self, config: Config) -> RepositoryCertificateConfig:\n        return RepositoryCertificateConfig.create(self.name, config)\n\n    @property\n    def http_credential_keys(self) -> list[str]:\n        return [self.url, self.netloc, self.name]\n\n    def get_http_credentials(\n        self, password_manager: PasswordManager, username: str | None = None\n    ) -> HTTPAuthCredential:\n        # try with the repository name via the password manager\n        credential = HTTPAuthCredential(\n            **(password_manager.get_http_auth(self.name) or {})\n        )\n\n        if credential.password is not None:\n            return credential\n\n        if password_manager.use_keyring:\n            # fallback to url and netloc based keyring entries\n            credential = password_manager.get_credential(\n                self.url, self.netloc, username=credential.username\n            )\n\n        return credential\n\n\nclass Authenticator:\n    def __init__(\n        self,\n        config: Config | None = None,\n        io: IO | None = None,\n        cache_id: str | None = None,\n        disable_cache: bool = False,\n        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,\n    ) -> None:\n        self._config = config or Config.create()\n        self._io = io\n        self._sessions_for_netloc: dict[str, requests.Session] = {}\n        self._credentials: dict[str, HTTPAuthCredential] = {}\n        self._certs: dict[str, RepositoryCertificateConfig] = {}\n        self._configured_repositories: (\n            dict[str, AuthenticatorRepositoryConfig] | None\n        ) = None\n        self._password_manager = PasswordManager(self._config)\n        self._cache_control = (\n            FileCache(\n                self._config.repository_cache_directory\n                / (cache_id or \"_default_cache\")\n                / \"_http\"\n            )\n            if not disable_cache\n            else None\n        )\n        self.get_repository_config_for_url = functools.lru_cache(maxsize=None)(\n            self._get_repository_config_for_url\n        )\n        self._pool_size = pool_size\n        self._user_agent = user_agent(\"poetry\", __version__)\n\n    def create_session(self) -> requests.Session:\n        session = requests.Session()\n        session.headers[\"User-Agent\"] = self._user_agent\n\n        if self._cache_control is None:\n            return session\n\n        adapter = CacheControlAdapter(\n            cache=self._cache_control,\n            pool_maxsize=self._pool_size,\n        )\n        session.mount(\"http://\", adapter)\n        session.mount(\"https://\", adapter)\n\n        return session\n\n    def get_session(self, url: str | None = None) -> requests.Session:\n        if not url:\n            return self.create_session()\n\n        parsed_url = urllib.parse.urlsplit(url)\n        netloc = parsed_url.netloc\n\n        if netloc not in self._sessions_for_netloc:\n            logger.debug(\"Creating new session for %s\", netloc)\n            self._sessions_for_netloc[netloc] = self.create_session()\n\n        return self._sessions_for_netloc[netloc]\n\n    def close(self) -> None:\n        for session in self._sessions_for_netloc.values():\n            if session is not None:\n                with contextlib.suppress(AttributeError):\n                    session.close()\n\n    def __del__(self) -> None:\n        self.close()\n\n    def delete_cache(self, url: str) -> None:\n        if self._cache_control is not None:\n            self._cache_control.delete(key=url)\n\n    def authenticated_url(self, url: str) -> str:\n        parsed = urllib.parse.urlparse(url)\n        credential = self.get_credentials_for_url(url)\n\n        if credential.username is not None and credential.password is not None:\n            username = urllib.parse.quote(credential.username, safe=\"\")\n            password = urllib.parse.quote(credential.password, safe=\"\")\n\n            return (\n                f\"{parsed.scheme}://{username}:{password}@{parsed.netloc}{parsed.path}\"\n            )\n\n        return url\n\n    def request(\n        self, method: str, url: str, raise_for_status: bool = True, **kwargs: Any\n    ) -> requests.Response:\n        headers = kwargs.get(\"headers\")\n        request = requests.Request(method, url, headers=headers)\n        credential = self.get_credentials_for_url(url)\n\n        if credential.username is not None or credential.password is not None:\n            request = requests.auth.HTTPBasicAuth(\n                credential.username or \"\", credential.password or \"\"\n            )(request)\n\n        session = self.get_session(url=url)\n        prepared_request = session.prepare_request(request)\n\n        proxies: dict[str, str] = kwargs.get(\"proxies\", {})\n        stream: bool | None = kwargs.get(\"stream\")\n\n        certs = self.get_certs_for_url(url)\n        verify: bool | str | Path = kwargs.get(\"verify\") or certs.cert or certs.verify\n        cert: str | Path | None = kwargs.get(\"cert\") or certs.client_cert\n\n        if cert is not None:\n            cert = str(cert)\n\n        verify = str(verify) if isinstance(verify, Path) else verify\n\n        settings = session.merge_environment_settings(\n            prepared_request.url, proxies, stream, verify, cert\n        )\n\n        # Send the request.\n        send_kwargs = {\n            \"timeout\": kwargs.get(\"timeout\", REQUESTS_TIMEOUT),\n            \"allow_redirects\": kwargs.get(\"allow_redirects\", True),\n        }\n        send_kwargs.update(settings)\n\n        attempt = 0\n        resp = None\n\n        while True:\n            is_last_attempt = attempt >= 5\n            try:\n                resp = session.send(prepared_request, **send_kwargs)\n            except (requests.exceptions.ConnectionError, OSError) as e:\n                if is_last_attempt:\n                    raise e\n            else:\n                if resp.status_code not in STATUS_FORCELIST or is_last_attempt:\n                    if raise_for_status:\n                        resp.raise_for_status()\n                    return resp\n\n            if not is_last_attempt:\n                attempt += 1\n                delay = self._get_backoff(resp, attempt)\n                logger.debug(\"Retrying HTTP request in %s seconds.\", delay)\n                time.sleep(delay)\n                continue\n\n        # this should never really be hit under any sane circumstance\n        raise PoetryException(\"Failed HTTP {} request\", method.upper())\n\n    def _get_backoff(self, response: requests.Response | None, attempt: int) -> float:\n        if response is not None:\n            retry_after = response.headers.get(RETRY_AFTER_HEADER, \"\")\n            if retry_after:\n                return float(retry_after)\n\n        return 0.5 * attempt\n\n    def get(self, url: str, **kwargs: Any) -> requests.Response:\n        return self.request(\"get\", url, **kwargs)\n\n    def head(self, url: str, **kwargs: Any) -> requests.Response:\n        kwargs.setdefault(\"allow_redirects\", False)\n        return self.request(\"head\", url, **kwargs)\n\n    def post(self, url: str, **kwargs: Any) -> requests.Response:\n        return self.request(\"post\", url, **kwargs)\n\n    def _get_credentials_for_repository(\n        self, repository: AuthenticatorRepositoryConfig, username: str | None = None\n    ) -> HTTPAuthCredential:\n        # cache repository credentials by repository url to avoid multiple keyring\n        # backend queries when packages are being downloaded from the same source\n        key = f\"{repository.url}#username={username or ''}\"\n\n        if key not in self._credentials:\n            self._credentials[key] = repository.get_http_credentials(\n                password_manager=self._password_manager, username=username\n            )\n\n        return self._credentials[key]\n\n    def _get_credentials_for_url(\n        self, url: str, exact_match: bool = False\n    ) -> HTTPAuthCredential:\n        repository = self.get_repository_config_for_url(url, exact_match)\n\n        credential = (\n            self._get_credentials_for_repository(repository=repository)\n            if repository is not None\n            else HTTPAuthCredential()\n        )\n\n        if credential.password is None:\n            parsed_url = urllib.parse.urlsplit(url)\n            netloc = parsed_url.netloc\n            credential = self._password_manager.get_credential(\n                url, netloc, username=credential.username\n            )\n\n            return HTTPAuthCredential(\n                username=credential.username, password=credential.password\n            )\n\n        return credential\n\n    def get_credentials_for_git_url(self, url: str) -> HTTPAuthCredential:\n        parsed_url = urllib.parse.urlsplit(url)\n\n        if parsed_url.scheme not in {\"http\", \"https\"}:\n            return HTTPAuthCredential()\n\n        key = f\"git+{url}\"\n\n        if key not in self._credentials:\n            self._credentials[key] = self._get_credentials_for_url(url, True)\n\n        return self._credentials[key]\n\n    def get_credentials_for_url(self, url: str) -> HTTPAuthCredential:\n        parsed_url = urllib.parse.urlsplit(url)\n        netloc = parsed_url.netloc\n\n        if url not in self._credentials:\n            if \"@\" not in netloc:\n                # no credentials were provided in the url, try finding the\n                # best repository configuration\n                self._credentials[url] = self._get_credentials_for_url(url)\n            else:\n                # Split from the right because that's how urllib.parse.urlsplit()\n                # behaves if more than one @ is present (which can be checked using\n                # the password attribute of urlsplit()'s return value).\n                auth, netloc = netloc.rsplit(\"@\", 1)\n                # Split from the left because that's how urllib.parse.urlsplit()\n                # behaves if more than one : is present (which again can be checked\n                # using the password attribute of the return value)\n                user, password = auth.split(\":\", 1) if \":\" in auth else (auth, \"\")\n                self._credentials[url] = HTTPAuthCredential(\n                    urllib.parse.unquote(user),\n                    urllib.parse.unquote(password),\n                )\n\n        return self._credentials[url]\n\n    def get_pypi_token(self, name: str) -> str | None:\n        return self._password_manager.get_pypi_token(name)\n\n    def get_http_auth(\n        self, name: str, username: str | None = None\n    ) -> HTTPAuthCredential | None:\n        if name == \"pypi\":\n            repository = AuthenticatorRepositoryConfig(\n                name, \"https://upload.pypi.org/legacy/\"\n            )\n        else:\n            if name not in self.configured_repositories:\n                return None\n            repository = self.configured_repositories[name]\n\n        return self._get_credentials_for_repository(\n            repository=repository, username=username\n        )\n\n    def get_certs_for_repository(self, name: str) -> RepositoryCertificateConfig:\n        if name.lower() == \"pypi\" or name not in self.configured_repositories:\n            return RepositoryCertificateConfig()\n        return self.configured_repositories[name].certs(self._config)\n\n    @property\n    def configured_repositories(self) -> dict[str, AuthenticatorRepositoryConfig]:\n        if self._configured_repositories is None:\n            self._configured_repositories = {}\n            for repository_name in self._config.get(\"repositories\", []):\n                url = self._config.get(f\"repositories.{repository_name}.url\")\n                self._configured_repositories[repository_name] = (\n                    AuthenticatorRepositoryConfig(repository_name, url)\n                )\n\n        return self._configured_repositories\n\n    def reset_credentials_cache(self) -> None:\n        self.get_repository_config_for_url.cache_clear()\n        self._credentials = {}\n\n    def add_repository(self, name: str, url: str) -> None:\n        self.configured_repositories[name] = AuthenticatorRepositoryConfig(name, url)\n        self.reset_credentials_cache()\n\n...\n", "needle_token_start": 2374, "needle_token_end": 2441, "code_context_ntokens": 16013, "output": ["```python\nclass InstalledRepository(Repository):\n    @classmethod\n    def get_package_vcs_properties_from_path(cls, src: Path) -> tuple[str, str, str]:\n        from poetry.vcs.git import Git\n\n        info = Git.info(repo=src)\n        return \"git\", info.origin, info.revision\n```"]}
{"repo": "python-poetry/poetry", "name": "_validate_group_options", "language": "python", "path": "src/poetry/console/commands/group_command.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function ensures that specified dependency groups are valid according to the project configuration.\n2. **Input**: A dictionary where keys are option types and values are sets of group names.\n3. **Output**: The function does not return any value but raises an error if any group is not recognized.\n4. **Procedure**: The function iterates over each group specified in the input dictionary. For each group, it checks if the group exists in the project's dependency configuration. If a group does not exist, it is added to a list of invalid groups. If there are any invalid groups, an error message is constructed and an exception is raised, indicating which groups are invalid and through which options they were specified.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/console/commands/config.py\nfrom __future__ import annotations\n\nimport json\nimport re\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\nfrom typing import cast\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n\nfrom poetry.config.config import PackageFilterPolicy\nfrom poetry.config.config import boolean_normalizer\nfrom poetry.config.config import boolean_validator\nfrom poetry.config.config import int_normalizer\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n    from poetry.config.config_source import ConfigSource\n\n\nclass ConfigCommand(Command):\n    name = \"config\"\n    description = \"Manages configuration settings.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"key\", \"Setting key.\", optional=True),\n        argument(\"value\", \"Setting value.\", optional=True, multiple=True),\n    ]\n\n    options: ClassVar[list[Option]] = [\n        option(\"list\", None, \"List configuration settings.\"),\n        option(\"unset\", None, \"Unset configuration setting.\"),\n        option(\"local\", None, \"Set/Get from the project's local configuration.\"),\n    ]\n\n    help = \"\"\"\\\nThis command allows you to edit the poetry config settings and repositories.\n\nTo add a repository:\n\n    <comment>poetry config repositories.foo https://bar.com/simple/</comment>\n\nTo remove a repository (repo is a short alias for repositories):\n\n    <comment>poetry config --unset repo.foo</comment>\"\"\"\n\n    LIST_PROHIBITED_SETTINGS: ClassVar[set[str]] = {\"http-basic\", \"pypi-token\"}\n\n    @property\n    def unique_config_values(self) -> dict[str, tuple[Any, Any]]:\n        unique_config_values = {\n            \"cache-dir\": (str, lambda val: str(Path(val))),\n            \"virtualenvs.create\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.in-project\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.options.always-copy\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.options.system-site-packages\": (\n                boolean_validator,\n                boolean_normalizer,\n            ),\n            \"virtualenvs.options.no-pip\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.options.no-setuptools\": (\n                boolean_validator,\n                boolean_normalizer,\n            ),\n            \"virtualenvs.path\": (str, lambda val: str(Path(val))),\n            \"virtualenvs.prefer-active-python\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.prompt\": (str, str),\n            \"experimental.system-git-client\": (boolean_validator, boolean_normalizer),\n            \"installer.modern-installation\": (boolean_validator, boolean_normalizer),\n            \"installer.parallel\": (boolean_validator, boolean_normalizer),\n            \"installer.max-workers\": (lambda val: int(val) > 0, int_normalizer),\n            \"installer.no-binary\": (\n                PackageFilterPolicy.validator,\n                PackageFilterPolicy.normalize,\n            ),\n            \"solver.lazy-wheel\": (boolean_validator, boolean_normalizer),\n            \"warnings.export\": (boolean_validator, boolean_normalizer),\n            \"keyring.enabled\": (boolean_validator, boolean_normalizer),\n        }\n\n        return unique_config_values\n\n    def handle(self) -> int:\n        from pathlib import Path\n\n        from poetry.core.pyproject.exceptions import PyProjectException\n\n        from poetry.config.config import Config\n        from poetry.config.file_config_source import FileConfigSource\n        from poetry.locations import CONFIG_DIR\n        from poetry.toml.file import TOMLFile\n\n        config = Config.create()\n        config_file = TOMLFile(CONFIG_DIR / \"config.toml\")\n\n        try:\n            local_config_file = TOMLFile(self.poetry.file.path.parent / \"poetry.toml\")\n            if local_config_file.exists():\n                config.merge(local_config_file.read())\n        except (RuntimeError, PyProjectException):\n            local_config_file = TOMLFile(Path.cwd() / \"poetry.toml\")\n\n        if self.option(\"local\"):\n            config.set_config_source(FileConfigSource(local_config_file))\n\n        if not config_file.exists():\n            config_file.path.parent.mkdir(parents=True, exist_ok=True)\n            config_file.path.touch(mode=0o0600)\n\n        if self.option(\"list\"):\n            self._list_configuration(config.all(), config.raw())\n\n            return 0\n\n        setting_key = self.argument(\"key\")\n        if not setting_key:\n            return 0\n\n        if self.argument(\"value\") and self.option(\"unset\"):\n            raise RuntimeError(\"You can not combine a setting value with --unset\")\n\n        # show the value if no value is provided\n        if not self.argument(\"value\") and not self.option(\"unset\"):\n            if setting_key.split(\".\")[0] in self.LIST_PROHIBITED_SETTINGS:\n                raise ValueError(f\"Expected a value for {setting_key} setting.\")\n\n            m = re.match(r\"^repos?(?:itories)?(?:\\.(.+))?\", self.argument(\"key\"))\n            value: str | dict[str, Any]\n            if m:\n                if not m.group(1):\n                    value = {}\n                    if config.get(\"repositories\") is not None:\n                        value = config.get(\"repositories\")\n                else:\n                    repo = config.get(f\"repositories.{m.group(1)}\")\n                    if repo is None:\n                        raise ValueError(f\"There is no {m.group(1)} repository defined\")\n\n                    value = repo\n\n                self.line(str(value))\n            else:\n                if setting_key not in self.unique_config_values:\n                    raise ValueError(f\"There is no {setting_key} setting.\")\n\n                value = config.get(setting_key)\n\n                if not isinstance(value, str):\n                    value = json.dumps(value)\n\n                self.line(value)\n\n            return 0\n\n        values: list[str] = self.argument(\"value\")\n\n        if setting_key in self.unique_config_values:\n            if self.option(\"unset\"):\n                config.config_source.remove_property(setting_key)\n                return 0\n\n            return self._handle_single_value(\n                config.config_source,\n                setting_key,\n                self.unique_config_values[setting_key],\n                values,\n            )\n\n        # handle repositories\n        m = re.match(r\"^repos?(?:itories)?(?:\\.(.+))?\", self.argument(\"key\"))\n        if m:\n            if not m.group(1):\n                raise ValueError(\"You cannot remove the [repositories] section\")\n\n            if self.option(\"unset\"):\n                repo = config.get(f\"repositories.{m.group(1)}\")\n                if repo is None:\n                    raise ValueError(f\"There is no {m.group(1)} repository defined\")\n\n                config.config_source.remove_property(f\"repositories.{m.group(1)}\")\n\n                return 0\n\n            if len(values) == 1:\n                url = values[0]\n\n                config.config_source.add_property(f\"repositories.{m.group(1)}.url\", url)\n\n                return 0\n\n            raise ValueError(\n                \"You must pass the url. \"\n                \"Example: poetry config repositories.foo https://bar.com\"\n            )\n\n        # handle auth\n        m = re.match(r\"^(http-basic|pypi-token)\\.(.+)\", self.argument(\"key\"))\n        if m:\n            from poetry.utils.password_manager import PasswordManager\n\n            password_manager = PasswordManager(config)\n            if self.option(\"unset\"):\n                if m.group(1) == \"http-basic\":\n                    password_manager.delete_http_password(m.group(2))\n                elif m.group(1) == \"pypi-token\":\n                    password_manager.delete_pypi_token(m.group(2))\n\n                return 0\n\n            if m.group(1) == \"http-basic\":\n                if len(values) == 1:\n                    username = values[0]\n                    # Only username, so we prompt for password\n                    password = self.secret(\"Password:\")\n                    assert isinstance(password, str)\n                elif len(values) != 2:\n                    raise ValueError(\n                        \"Expected one or two arguments \"\n                        f\"(username, password), got {len(values)}\"\n                    )\n                else:\n                    username = values[0]\n                    password = values[1]\n\n                password_manager.set_http_password(m.group(2), username, password)\n            elif m.group(1) == \"pypi-token\":\n                if len(values) != 1:\n                    raise ValueError(\n                        f\"Expected only one argument (token), got {len(values)}\"\n                    )\n\n                token = values[0]\n\n                password_manager.set_pypi_token(m.group(2), token)\n\n            return 0\n\n        # handle certs\n        m = re.match(r\"certificates\\.([^.]+)\\.(cert|client-cert)\", self.argument(\"key\"))\n        if m:\n            repository = m.group(1)\n            key = m.group(2)\n\n...\n# Path: src/poetry/console/commands/env_command.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from poetry.utils.env import Env\n\n\nclass EnvCommand(Command):\n    def __init__(self) -> None:\n        # Set in poetry.console.application.Application.configure_env\n        self._env: Env | None = None\n\n        super().__init__()\n\n    @property\n    def env(self) -> Env:\n        assert self._env is not None\n        return self._env\n\n    def set_env(self, env: Env) -> None:\n        self._env = env\n\n# Path: src/poetry/console/commands/export.py\nfrom __future__ import annotations\n\nfrom poetry_plugin_export.command import (  # type: ignore[import-untyped]\n    ExportCommand as BaseExportCommand,\n)\n\n\nclass ExportCommand(BaseExportCommand):  # type: ignore[misc]\n    def handle(self) -> int:\n        if self.poetry.config.get(\"warnings.export\"):\n            self.line_error(\n                \"Warning: poetry-plugin-export will not be installed by default in a\"\n                \" future version of Poetry.\\n\"\n                \"In order to avoid a breaking change and make your automation\"\n                \" forward-compatible, please install poetry-plugin-export explicitly.\"\n                \" See https://python-poetry.org/docs/plugins/#using-plugins for details\"\n                \" on how to install a plugin.\\n\"\n                \"To disable this warning run 'poetry config warnings.export false'.\",\n                style=\"warning\",\n            )\n        return super().handle()  # type: ignore[no-any-return]\n\n# Path: src/poetry/console/commands/group_command.py\nfrom __future__ import annotations\n\nfrom collections import defaultdict\nfrom typing import TYPE_CHECKING\n\nfrom cleo.helpers import option\nfrom poetry.core.packages.dependency_group import MAIN_GROUP\n\nfrom poetry.console.commands.command import Command\nfrom poetry.console.exceptions import GroupNotFound\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n    from poetry.core.packages.project_package import ProjectPackage\n\n\nclass GroupCommand(Command):\n    @staticmethod\n    def _group_dependency_options() -> list[Option]:\n        return [\n            option(\n                \"without\",\n                None,\n                \"The dependency groups to ignore.\",\n                flag=False,\n                multiple=True,\n            ),\n            option(\n                \"with\",\n                None,\n                \"The optional dependency groups to include.\",\n                flag=False,\n                multiple=True,\n            ),\n            option(\n                \"only\",\n                None,\n                \"The only dependency groups to include.\",\n                flag=False,\n                multiple=True,\n            ),\n        ]\n\n    @property\n    def non_optional_groups(self) -> set[str]:\n        # TODO: this should move into poetry-core\n        return {\n            group.name\n            for group in self.poetry.package._dependency_groups.values()\n            if not group.is_optional()\n        }\n\n    @property\n    def default_group(self) -> str | None:\n        \"\"\"\n        The default group to use when no group is specified. This is useful\n        for command that have the `--group` option, eg: add, remove.\n\n        Can be overridden to adapt behavior.\n        \"\"\"\n        return None\n\n    @property\n    def default_groups(self) -> set[str]:\n        \"\"\"\n        The groups that are considered by the command by default.\n\n        Can be overridden to adapt behavior.\n        \"\"\"\n        return self.non_optional_groups\n\n    @property\n    def activated_groups(self) -> set[str]:\n        groups = {}\n\n        for key in {\"with\", \"without\", \"only\"}:\n            groups[key] = {\n                group.strip()\n                for groups in self.option(key, \"\")\n                for group in groups.split(\",\")\n            }\n        self._validate_group_options(groups)\n\n        for opt, new, group in [\n            (\"no-dev\", \"only\", MAIN_GROUP),\n            (\"dev\", \"with\", \"dev\"),\n        ]:\n            if self.io.input.has_option(opt) and self.option(opt):\n                self.line_error(\n                    f\"<warning>The `<fg=yellow;options=bold>--{opt}</>` option is\"\n                    f\" deprecated, use the `<fg=yellow;options=bold>--{new} {group}</>`\"\n                    \" notation instead.</warning>\"\n                )\n                groups[new].add(group)\n\n        if groups[\"only\"] and (groups[\"with\"] or groups[\"without\"]):\n            self.line_error(\n                \"<warning>The `<fg=yellow;options=bold>--with</>` and \"\n                \"`<fg=yellow;options=bold>--without</>` options are ignored when used\"\n                \" along with the `<fg=yellow;options=bold>--only</>` option.\"\n                \"</warning>\"\n            )\n\n        return groups[\"only\"] or self.default_groups.union(groups[\"with\"]).difference(\n            groups[\"without\"]\n        )\n\n    def project_with_activated_groups_only(self) -> ProjectPackage:\n        return self.poetry.package.with_dependency_groups(\n            list(self.activated_groups), only=True\n        )\n\n    \ndef _validate_group_options(self, group_options: dict[str, set[str]]) -> None:\n        \"\"\"\n        Raises an error if it detects that a group is not part of pyproject.toml\n        \"\"\"\n        invalid_options = defaultdict(set)\n        for opt, groups in group_options.items():\n            for group in groups:\n                if not self.poetry.package.has_dependency_group(group):\n                    invalid_options[group].add(opt)\n        if invalid_options:\n            message_parts = []\n            for group in sorted(invalid_options):\n                opts = \", \".join(\n                    f\"<fg=yellow;options=bold>--{opt}</>\"\n                    for opt in sorted(invalid_options[group])\n                )\n                message_parts.append(f\"{group} (via {opts})\")\n            raise GroupNotFound(f\"Group(s) not found: {', '.join(message_parts)}\")\n\n# Path: src/poetry/console/commands/init.py\nfrom __future__ import annotations\n\nfrom contextlib import suppress\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\nfrom typing import Dict\nfrom typing import Mapping\nfrom typing import Union\n\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\nfrom tomlkit import inline_table\n\nfrom poetry.console.commands.command import Command\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.utils.dependency_specification import RequirementsParser\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n    from packaging.utils import NormalizedName\n    from poetry.core.packages.package import Package\n    from tomlkit.items import InlineTable\n\n    from poetry.repositories import RepositoryPool\n\nRequirements = Dict[str, Union[str, Mapping[str, Any]]]\n\n\nclass InitCommand(Command):\n    name = \"init\"\n    description = (\n        \"Creates a basic <comment>pyproject.toml</> file in the current directory.\"\n    )\n\n    options: ClassVar[list[Option]] = [\n        option(\"name\", None, \"Name of the package.\", flag=False),\n        option(\"description\", None, \"Description of the package.\", flag=False),\n        option(\"author\", None, \"Author name of the package.\", flag=False),\n        option(\"python\", None, \"Compatible Python versions.\", flag=False),\n        option(\n            \"dependency\",\n            None,\n            \"Package to require, with an optional version constraint, \"\n            \"e.g. requests:^2.10.0 or requests=2.11.1.\",\n            flag=False,\n            multiple=True,\n        ),\n        option(\n            \"dev-dependency\",\n            None,\n            \"Package to require for development, with an optional version\"\n            \" constraint, e.g. requests:^2.10.0 or requests=2.11.1.\",\n            flag=False,\n            multiple=True,\n        ),\n        option(\"license\", \"l\", \"License of the package.\", flag=False),\n    ]\n\n    help = \"\"\"\\\nThe <c1>init</c1> command creates a basic <comment>pyproject.toml</> file in the\\\n current directory.\n\"\"\"\n\n    def __init__(self) -> None:\n        super().__init__()\n\n        self._pool: RepositoryPool | None = None\n\n    def handle(self) -> int:\n        from pathlib import Path\n\n        project_path = Path.cwd()\n\n        if self.io.input.option(\"directory\"):\n            project_path = Path(self.io.input.option(\"directory\"))\n            if not project_path.exists() or not project_path.is_dir():\n                self.line_error(\n                    \"<error>The --directory path is not a directory.</error>\"\n                )\n                return 1\n\n        return self._init_pyproject(project_path=project_path)\n\n    def _init_pyproject(\n        self,\n        project_path: Path,\n        allow_interactive: bool = True,\n        layout_name: str = \"standard\",\n        readme_format: str = \"md\",\n    ) -> int:\n        from poetry.core.vcs.git import GitConfig\n\n        from poetry.config.config import Config\n        from poetry.layouts import layout\n        from poetry.pyproject.toml import PyProjectTOML\n        from poetry.utils.env import EnvManager\n\n        is_interactive = self.io.is_interactive() and allow_interactive\n\n        pyproject = PyProjectTOML(project_path / \"pyproject.toml\")\n\n        if pyproject.file.exists():\n            if pyproject.is_poetry_project():\n                self.line_error(\n                    \"<error>A pyproject.toml file with a poetry section already\"\n                    \" exists.</error>\"\n                )\n                return 1\n\n            if pyproject.data.get(\"build-system\"):\n                self.line_error(\n                    \"<error>A pyproject.toml file with a defined build-system already\"\n                    \" exists.</error>\"\n                )\n                return 1\n\n        vcs_config = GitConfig()\n\n        if is_interactive:\n            self.line(\"\")\n            self.line(\n                \"This command will guide you through creating your\"\n                \" <info>pyproject.toml</> config.\"\n            )\n            self.line(\"\")\n\n        name = self.option(\"name\")\n        if not name:\n            name = project_path.name.lower()\n\n            if is_interactive:\n                question = self.create_question(\n                    f\"Package name [<comment>{name}</comment>]: \", default=name\n                )\n                name = self.ask(question)\n\n        version = \"0.1.0\"\n\n        if is_interactive:\n            question = self.create_question(\n                f\"Version [<comment>{version}</comment>]: \", default=version\n            )\n            version = self.ask(question)\n\n        description = self.option(\"description\") or \"\"\n        if not description and is_interactive:\n            description = self.ask(self.create_question(\"Description []: \", default=\"\"))\n\n        author = self.option(\"author\")\n        if not author and vcs_config.get(\"user.name\"):\n            author = vcs_config[\"user.name\"]\n            author_email = vcs_config.get(\"user.email\")\n            if author_email:\n                author += f\" <{author_email}>\"\n\n        if is_interactive:\n            question = self.create_question(\n                f\"Author [<comment>{author}</comment>, n to skip]: \", default=author\n            )\n            question.set_validator(lambda v: self._validate_author(v, author))\n            author = self.ask(question)\n\n        authors = [author] if author else []\n\n        license_name = self.option(\"license\")\n        if not license_name and is_interactive:\n            license_name = self.ask(self.create_question(\"License []: \", default=\"\"))\n\n        python = self.option(\"python\")\n        if not python:\n            config = Config.create()\n            python = (\n                \"^\"\n                + EnvManager.get_python_version(\n                    precision=2,\n                    prefer_active_python=config.get(\"virtualenvs.prefer-active-python\"),\n                    io=self.io,\n                ).to_string()\n            )\n\n            if is_interactive:\n                question = self.create_question(\n                    f\"Compatible Python versions [<comment>{python}</comment>]: \",\n                    default=python,\n                )\n                python = self.ask(question)\n\n        if is_interactive:\n            self.line(\"\")\n\n        requirements: Requirements = {}\n        if self.option(\"dependency\"):\n            requirements = self._format_requirements(\n                self._determine_requirements(self.option(\"dependency\"))\n            )\n\n        question_text = \"Would you like to define your main dependencies interactively?\"\n        help_message = \"\"\"\\\n        You can specify a package in the following forms:\n          - A single name (<b>requests</b>): this will search for matches on PyPI\n          - A name and a constraint (<b>requests@^2.23.0</b>)\n          - A git url (<b>git+https://github.com/python-poetry/poetry.git</b>)\n          - A git url with a revision\\\n         (<b>git+https://github.com/python-poetry/poetry.git#develop</b>)\n          - A file path (<b>../my-package/my-package.whl</b>)\n          - A directory (<b>../my-package/</b>)\n          - A url (<b>https://example.com/packages/my-package-0.1.0.tar.gz</b>)\n        \"\"\"\n\n        help_displayed = False\n        if is_interactive and self.confirm(question_text, True):\n            self.line(help_message)\n            help_displayed = True\n            requirements.update(\n                self._format_requirements(self._determine_requirements([]))\n            )\n            self.line(\"\")\n\n        dev_requirements: Requirements = {}\n        if self.option(\"dev-dependency\"):\n            dev_requirements = self._format_requirements(\n                self._determine_requirements(self.option(\"dev-dependency\"))\n            )\n\n        question_text = (\n            \"Would you like to define your development dependencies interactively?\"\n        )\n        if is_interactive and self.confirm(question_text, True):\n            if not help_displayed:\n                self.line(help_message)\n\n            dev_requirements.update(\n                self._format_requirements(self._determine_requirements([]))\n            )\n\n            self.line(\"\")\n\n        layout_ = layout(layout_name)(\n            name,\n            version,\n            description=description,\n            author=authors[0] if authors else None,\n            readme_format=readme_format,\n            license=license_name,\n            python=python,\n            dependencies=requirements,\n            dev_dependencies=dev_requirements,\n        )\n\n        create_layout = not project_path.exists()\n\n        if create_layout:\n            layout_.create(project_path, with_pyproject=False)\n\n        content = layout_.generate_poetry_content()\n        for section, item in content.items():\n            pyproject.data.append(section, item)\n\n        if is_interactive:\n            self.line(\"<info>Generated file</info>\")\n            self.line(\"\")\n            self.line(pyproject.data.as_string().replace(\"\\r\\n\", \"\\n\"))\n            self.line(\"\")\n\n        if is_interactive and not self.confirm(\"Do you confirm generation?\", True):\n            self.line_error(\"<error>Command aborted</error>\")\n\n            return 1\n\n        pyproject.save()\n\n        if create_layout:\n            path = project_path.resolve()\n\n            with suppress(ValueError):\n                path = path.relative_to(Path.cwd())\n\n            self.line(\n                f\"Created package <info>{layout_._package_name}</> in\"\n                f\" <fg=blue>{path.as_posix()}</>\"\n            )\n\n        return 0\n\n    def _generate_choice_list(\n        self, matches: list[Package], canonicalized_name: NormalizedName\n    ) -> list[str]:\n        choices = []\n        matches_names = [p.name for p in matches]\n        exact_match = canonicalized_name in matches_names\n        if exact_match:\n            choices.append(matches[matches_names.index(canonicalized_name)].pretty_name)\n\n        for found_package in matches:\n            if len(choices) >= 10:\n                break\n\n            if found_package.name == canonicalized_name:\n                continue\n\n            choices.append(found_package.pretty_name)\n\n        return choices\n\n    def _determine_requirements(\n        self,\n        requires: list[str],\n        allow_prereleases: bool = False,\n        source: str | None = None,\n        is_interactive: bool | None = None,\n    ) -> list[dict[str, Any]]:\n        if is_interactive is None:\n            is_interactive = self.io.is_interactive()\n\n        if not requires:\n            result = []\n\n            question = self.create_question(\n                \"Package to add or search for (leave blank to skip):\"\n            )\n            question.set_validator(self._validate_package)\n\n            follow_up_question = self.create_question(\n                \"\\nAdd a package (leave blank to skip):\"\n            )\n            follow_up_question.set_validator(self._validate_package)\n\n            package = self.ask(question)\n            while package:\n                constraint = self._parse_requirements([package])[0]\n                if (\n                    \"git\" in constraint\n                    or \"url\" in constraint\n                    or \"path\" in constraint\n                    or \"version\" in constraint\n                ):\n                    self.line(f\"Adding <info>{package}</info>\")\n                    result.append(constraint)\n                    package = self.ask(follow_up_question)\n                    continue\n\n                canonicalized_name = canonicalize_name(constraint[\"name\"])\n                matches = self._get_pool().search(canonicalized_name)\n                if not matches:\n                    self.line_error(\"<error>Unable to find package</error>\")\n                    package = False\n                else:\n                    choices = self._generate_choice_list(matches, canonicalized_name)\n\n                    info_string = (\n                        f\"Found <info>{len(matches)}</info> packages matching\"\n                        f\" <c1>{package}</c1>\"\n                    )\n\n                    if len(matches) > 10:\n                        info_string += \"\\nShowing the first 10 matches\"\n\n                    self.line(info_string)\n\n                    # Default to an empty value to signal no package was selected\n                    choices.append(\"\")\n\n                    package = self.choice(\n                        \"\\nEnter package # to add, or the complete package name if\"\n                        \" it is not listed\",\n                        choices,\n                        attempts=3,\n                        default=len(choices) - 1,\n                    )\n\n                    if not package:\n                        self.line(\"<warning>No package selected</warning>\")\n\n                    # package selected by user, set constraint name to package name\n                    if package:\n                        constraint[\"name\"] = package\n\n                # no constraint yet, determine the best version automatically\n                if package and \"version\" not in constraint:\n                    question = self.create_question(\n                        \"Enter the version constraint to require \"\n                        \"(or leave blank to use the latest version):\"\n                    )\n                    question.set_max_attempts(3)\n                    question.set_validator(lambda x: (x or \"\").strip() or None)\n\n                    package_constraint = self.ask(question)\n\n                    if package_constraint is None:\n                        _, package_constraint = self._find_best_version_for_package(\n                            package\n                        )\n\n                        self.line(\n                            f\"Using version <b>{package_constraint}</b> for\"\n                            f\" <c1>{package}</c1>\"\n                        )\n\n                    constraint[\"version\"] = package_constraint\n\n                if package:\n                    result.append(constraint)\n\n                if is_interactive:\n                    package = self.ask(follow_up_question)\n\n            return result\n\n        result = []\n        for requirement in self._parse_requirements(requires):\n            if \"git\" in requirement or \"url\" in requirement or \"path\" in requirement:\n                result.append(requirement)\n                continue\n            elif \"version\" not in requirement:\n                # determine the best version automatically\n                name, version = self._find_best_version_for_package(\n                    requirement[\"name\"],\n                    allow_prereleases=allow_prereleases,\n                    source=source,\n                )\n                requirement[\"version\"] = version\n                requirement[\"name\"] = name\n\n                self.line(f\"Using version <b>{version}</b> for <c1>{name}</c1>\")\n            else:\n                # check that the specified version/constraint exists\n                # before we proceed\n                name, _ = self._find_best_version_for_package(\n                    requirement[\"name\"],\n                    requirement[\"version\"],\n                    allow_prereleases=allow_prereleases,\n                    source=source,\n                )\n\n                requirement[\"name\"] = name\n\n            result.append(requirement)\n\n        return result\n\n    def _find_best_version_for_package(\n        self,\n        name: str,\n        required_version: str | None = None,\n        allow_prereleases: bool = False,\n        source: str | None = None,\n    ) -> tuple[str, str]:\n        from poetry.version.version_selector import VersionSelector\n\n        selector = VersionSelector(self._get_pool())\n        package = selector.find_best_candidate(\n            name, required_version, allow_prereleases=allow_prereleases, source=source\n        )\n\n        if not package:\n            # TODO: find similar\n            raise ValueError(f\"Could not find a matching version of package {name}\")\n\n        return package.pretty_name, f\"^{package.version.to_string()}\"\n\n    def _parse_requirements(self, requirements: list[str]) -> list[dict[str, Any]]:\n        from poetry.core.pyproject.exceptions import PyProjectException\n\n        try:\n            cwd = self.poetry.file.path.parent\n            artifact_cache = self.poetry.pool.artifact_cache\n        except (PyProjectException, RuntimeError):\n            cwd = Path.cwd()\n            artifact_cache = self._get_pool().artifact_cache\n\n        parser = RequirementsParser(\n            artifact_cache=artifact_cache,\n            env=self.env if isinstance(self, EnvCommand) else None,\n            cwd=cwd,\n        )\n        return [parser.parse(requirement) for requirement in requirements]\n\n    def _format_requirements(self, requirements: list[dict[str, str]]) -> Requirements:\n        requires: Requirements = {}\n        for requirement in requirements:\n            name = requirement.pop(\"name\")\n            constraint: str | InlineTable\n            if \"version\" in requirement and len(requirement) == 1:\n                constraint = requirement[\"version\"]\n            else:\n                constraint = inline_table()\n                constraint.trivia.trail = \"\\n\"\n                constraint.update(requirement)\n\n            requires[name] = constraint\n\n        return requires\n\n    @staticmethod\n    def _validate_author(author: str, default: str) -> str | None:\n        from poetry.core.packages.package import AUTHOR_REGEX\n        from poetry.core.utils.helpers import combine_unicode\n\n        author = combine_unicode(author or default)\n\n        if author in [\"n\", \"no\"]:\n            return None\n\n        m = AUTHOR_REGEX.match(author)\n        if not m:\n            raise ValueError(\n                \"Invalid author string. Must be in the format: \"\n                \"John Smith <john@example.com>\"\n            )\n\n        return author\n\n    @staticmethod\n    def _validate_package(package: str | None) -> str | None:\n        if package and len(package.split()) > 2:\n            raise ValueError(\"Invalid package definition.\")\n\n        return package\n\n    def _get_pool(self) -> RepositoryPool:\n        from poetry.config.config import Config\n        from poetry.repositories import RepositoryPool\n        from poetry.repositories.pypi_repository import PyPiRepository\n\n        if isinstance(self, EnvCommand):\n            return self.poetry.pool\n\n        if self._pool is None:\n            self._pool = RepositoryPool()\n            pool_size = Config.create().installer_max_workers\n            self._pool.add_repository(PyPiRepository(pool_size=pool_size))\n\n        return self._pool\n\n# Path: src/poetry/console/commands/install.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass InstallCommand(InstallerCommand):\n    name = \"install\"\n    description = \"Installs the project dependencies.\"\n\n    options: ClassVar[list[Option]] = [\n        *InstallerCommand._group_dependency_options(),\n        option(\n            \"no-dev\",\n            None,\n            \"Do not install the development dependencies.\"\n            \" (<warning>Deprecated</warning>)\",\n        ),\n        option(\n            \"sync\",\n            None,\n            \"Synchronize the environment with the locked packages and the specified\"\n            \" groups.\",\n        ),\n        option(\n            \"no-root\", None, \"Do not install the root package (the current project).\"\n        ),\n        option(\n            \"no-directory\",\n            None,\n            \"Do not install any directory path dependencies; useful to install\"\n            \" dependencies without source code, e.g. for caching of Docker layers)\",\n            flag=True,\n            multiple=False,\n        ),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything \"\n            \"(implicitly enables --verbose).\",\n        ),\n        option(\n            \"remove-untracked\",\n            None,\n            \"Removes packages not present in the lock file.\"\n            \" (<warning>Deprecated</warning>)\",\n        ),\n        option(\n            \"extras\",\n            \"E\",\n            \"Extra sets of dependencies to install.\",\n            flag=False,\n            multiple=True,\n        ),\n        option(\"all-extras\", None, \"Install all extra dependencies.\"),\n        option(\"only-root\", None, \"Exclude all dependencies.\"),\n        option(\n            \"compile\",\n            None,\n            \"Compile Python source files to bytecode.\"\n            \" (This option has no effect if modern-installation is disabled\"\n            \" because the old installer always compiles.)\",\n        ),\n    ]\n\n    help = \"\"\"\\\nThe <info>install</info> command reads the <comment>poetry.lock</> file from\nthe current directory, processes it, and downloads and installs all the\nlibraries and dependencies outlined in that file. If the file does not\nexist it will look for <comment>pyproject.toml</> and do the same.\n\n<info>poetry install</info>\n\nBy default, the above command will also install the current project. To install only the\ndependencies and not including the current project, run the command with the\n<info>--no-root</info> option like below:\n\n<info> poetry install --no-root</info>\n\nIf you want to use Poetry only for dependency management but not for packaging,\nyou can set the \"package-mode\" to false in your pyproject.toml file.\n\"\"\"\n\n    _loggers: ClassVar[list[str]] = [\n        \"poetry.repositories.pypi_repository\",\n        \"poetry.inspection.info\",\n    ]\n\n    @property\n    def activated_groups(self) -> set[str]:\n        if self.option(\"only-root\"):\n            return set()\n        else:\n            return super().activated_groups\n\n    def handle(self) -> int:\n        from poetry.core.masonry.utils.module import ModuleOrPackageNotFound\n\n        from poetry.masonry.builders.editable import EditableBuilder\n\n        if self.option(\"extras\") and self.option(\"all-extras\"):\n            self.line_error(\n                \"<error>You cannot specify explicit\"\n                \" `<fg=yellow;options=bold>--extras</>` while installing\"\n                \" using `<fg=yellow;options=bold>--all-extras</>`.</error>\"\n            )\n            return 1\n\n        if self.option(\"only-root\") and any(\n            self.option(key) for key in {\"with\", \"without\", \"only\"}\n        ):\n            self.line_error(\n                \"<error>The `<fg=yellow;options=bold>--with</>`,\"\n                \" `<fg=yellow;options=bold>--without</>` and\"\n                \" `<fg=yellow;options=bold>--only</>` options cannot be used with\"\n                \" the `<fg=yellow;options=bold>--only-root</>`\"\n                \" option.</error>\"\n            )\n            return 1\n\n        if self.option(\"only-root\") and self.option(\"no-root\"):\n            self.line_error(\n                \"<error>You cannot specify `<fg=yellow;options=bold>--no-root</>`\"\n                \" when using `<fg=yellow;options=bold>--only-root</>`.</error>\"\n            )\n            return 1\n\n        extras: list[str]\n        if self.option(\"all-extras\"):\n            extras = list(self.poetry.package.extras.keys())\n        else:\n            extras = []\n            for extra in self.option(\"extras\", []):\n                extras += extra.split()\n\n        self.installer.extras(extras)\n\n        with_synchronization = self.option(\"sync\")\n        if self.option(\"remove-untracked\"):\n            self.line_error(\n                \"<warning>The `<fg=yellow;options=bold>--remove-untracked</>` option is\"\n                \" deprecated, use the `<fg=yellow;options=bold>--sync</>` option\"\n                \" instead.</warning>\"\n            )\n\n            with_synchronization = True\n\n        self.installer.only_groups(self.activated_groups)\n        self.installer.skip_directory(self.option(\"no-directory\"))\n        self.installer.dry_run(self.option(\"dry-run\"))\n        self.installer.requires_synchronization(with_synchronization)\n        self.installer.executor.enable_bytecode_compilation(self.option(\"compile\"))\n        self.installer.verbose(self.io.is_verbose())\n\n        return_code = self.installer.run()\n\n        if return_code != 0:\n            return return_code\n\n        if self.option(\"no-root\") or not self.poetry.is_package_mode:\n            return 0\n\n        log_install = (\n            \"<b>Installing</> the current project:\"\n            f\" <c1>{self.poetry.package.pretty_name}</c1>\"\n            f\" (<{{tag}}>{self.poetry.package.pretty_version}</>)\"\n        )\n        overwrite = self.io.output.is_decorated() and not self.io.is_debug()\n        self.line(\"\")\n        self.write(log_install.format(tag=\"c2\"))\n        if not overwrite:\n            self.line(\"\")\n\n        if self.option(\"dry-run\"):\n            self.line(\"\")\n            return 0\n\n        # Prior to https://github.com/python-poetry/poetry-core/pull/629\n        # the existence of a module/package was checked when creating the\n        # EditableBuilder. Afterwards, the existence is checked after\n        # executing the build script (if there is one),\n        # i.e. during EditableBuilder.build().\n        try:\n            builder = EditableBuilder(self.poetry, self.env, self.io)\n            builder.build()\n        except (ModuleOrPackageNotFound, FileNotFoundError) as e:\n            # This is likely due to the fact that the project is an application\n            # not following the structure expected by Poetry.\n            # No need for an editable install in this case.\n            self.line(\"\")\n            self.line_error(\n                f\"Warning: The current project could not be installed: {e}\\n\"\n                \"If you do not want to install the current project\"\n                \" use <c1>--no-root</c1>.\\n\"\n                \"If you want to use Poetry only for dependency management\"\n                \" but not for packaging, you can disable package mode by setting\"\n                \" <c1>package-mode = false</> in your pyproject.toml file.\\n\"\n                \"In a future version of Poetry this warning will become an error!\",\n                style=\"warning\",\n            )\n            return 0\n\n        if overwrite:\n            self.overwrite(log_install.format(tag=\"success\"))\n            self.line(\"\")\n\n        return 0\n\n# Path: src/poetry/console/commands/installer_command.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.console.commands.group_command import GroupCommand\n\n\nif TYPE_CHECKING:\n    from poetry.installation.installer import Installer\n\n\nclass InstallerCommand(GroupCommand, EnvCommand):\n    def __init__(self) -> None:\n        # Set in poetry.console.application.Application.configure_installer\n        self._installer: Installer | None = None\n\n        super().__init__()\n\n    def reset_poetry(self) -> None:\n        super().reset_poetry()\n\n        self.installer.set_package(self.poetry.package)\n        self.installer.set_locker(self.poetry.locker)\n\n    @property\n    def installer(self) -> Installer:\n        assert self._installer is not None\n        return self._installer\n\n    def set_installer(self, installer: Installer) -> None:\n        self._installer = installer\n\n# Path: src/poetry/console/commands/lock.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass LockCommand(InstallerCommand):\n    name = \"lock\"\n    description = \"Locks the project dependencies.\"\n\n    options: ClassVar[list[Option]] = [\n        option(\n            \"no-update\", None, \"Do not update locked versions, only refresh lock file.\"\n        ),\n        option(\n            \"check\",\n            None,\n            \"Check that the <comment>poetry.lock</> file corresponds to the current\"\n            \" version of <comment>pyproject.toml</>. (<warning>Deprecated</>) Use\"\n            \" <comment>poetry check --lock</> instead.\",\n        ),\n    ]\n\n    help = \"\"\"\nThe <info>lock</info> command reads the <comment>pyproject.toml</> file from the\ncurrent directory, processes it, and locks the dependencies in the\\\n <comment>poetry.lock</>\nfile.\n\n<info>poetry lock</info>\n\"\"\"\n\n    loggers: ClassVar[list[str]] = [\"poetry.repositories.pypi_repository\"]\n\n    def handle(self) -> int:\n        if self.option(\"check\"):\n            self.line_error(\n                \"<warning>poetry lock --check is deprecated, use `poetry\"\n                \" check --lock` instead.</warning>\"\n            )\n            if self.poetry.locker.is_locked() and self.poetry.locker.is_fresh():\n                self.line(\"poetry.lock is consistent with pyproject.toml.\")\n                return 0\n            self.line_error(\n                \"<error>\"\n                \"Error: pyproject.toml changed significantly since poetry.lock was last generated. \"\n                \"Run `poetry lock [--no-update]` to fix the lock file.\"\n                \"</error>\"\n            )\n            return 1\n\n        self.installer.lock(update=not self.option(\"no-update\"))\n\n        return self.installer.run()\n\n# Path: src/poetry/console/commands/new.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.init import InitCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass NewCommand(InitCommand):\n    name = \"new\"\n    description = \"Creates a new Python project at <path>.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"path\", \"The path to create the project at.\")\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\n            \"interactive\",\n            \"i\",\n            \"Allow interactive specification of project configuration.\",\n            flag=True,\n        ),\n        option(\"name\", None, \"Set the resulting package name.\", flag=False),\n        option(\"src\", None, \"Use the src layout for the project.\"),\n        option(\n            \"readme\",\n            None,\n            \"Specify the readme file format. One of md (default) or rst\",\n            flag=False,\n        ),\n        *[\n            o\n            for o in InitCommand.options\n            if o.name\n            in {\n                \"description\",\n                \"author\",\n                \"python\",\n                \"dependency\",\n                \"dev-dependency\",\n                \"license\",\n            }\n        ],\n    ]\n\n    def handle(self) -> int:\n        from pathlib import Path\n\n        if self.io.input.option(\"directory\"):\n            self.line_error(\n                \"<warning>--directory only makes sense with existing projects, and will\"\n                \" be ignored. You should consider the option --path instead.</warning>\"\n            )\n\n        path = Path(self.argument(\"path\"))\n        if not path.is_absolute():\n            # we do not use resolve here due to compatibility issues\n            # for path.resolve(strict=False)\n            path = Path.cwd().joinpath(path)\n\n        if path.exists() and list(path.glob(\"*\")):\n            # Directory is not empty. Aborting.\n            raise RuntimeError(\n                f\"Destination <fg=yellow>{path}</> exists and is not empty\"\n            )\n\n        return self._init_pyproject(\n            project_path=path,\n            allow_interactive=self.option(\"interactive\"),\n            layout_name=\"src\" if self.option(\"src\") else \"standard\",\n            readme_format=self.option(\"readme\") or \"md\",\n        )\n\n# Path: src/poetry/console/commands/publish.py\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass PublishCommand(Command):\n    name = \"publish\"\n    description = \"Publishes a package to a remote repository.\"\n\n    options: ClassVar[list[Option]] = [\n        option(\n            \"repository\", \"r\", \"The repository to publish the package to.\", flag=False\n        ),\n        option(\"username\", \"u\", \"The username to access the repository.\", flag=False),\n        option(\"password\", \"p\", \"The password to access the repository.\", flag=False),\n        option(\n            \"cert\", None, \"Certificate authority to access the repository.\", flag=False\n        ),\n        option(\n            \"client-cert\",\n            None,\n            \"Client certificate to access the repository.\",\n            flag=False,\n        ),\n        option(\n            \"dist-dir\",\n            None,\n            \"Dist directory where built artifact are stored. Default is `dist`.\",\n            default=\"dist\",\n            flag=False,\n        ),\n        option(\"build\", None, \"Build the package before publishing.\"),\n        option(\"dry-run\", None, \"Perform all actions except upload the package.\"),\n        option(\n            \"skip-existing\",\n            None,\n            \"Ignore errors from files already existing in the repository.\",\n        ),\n    ]\n\n    help = \"\"\"The publish command builds and uploads the package to a remote repository.\n\nBy default, it will upload to PyPI but if you pass the --repository option it will\nupload to it instead.\n\nThe --repository option should match the name of a configured repository using\nthe config command.\n\"\"\"\n\n    loggers: ClassVar[list[str]] = [\"poetry.publishing.publisher\"]\n\n    def handle(self) -> int:\n        from poetry.publishing.publisher import Publisher\n\n        if not self.poetry.is_package_mode:\n            self.line_error(\"Publishing a package is not possible in non-package mode.\")\n            return 1\n\n        dist_dir = self.option(\"dist-dir\")\n\n        publisher = Publisher(self.poetry, self.io, Path(dist_dir))\n\n        # Building package first, if told\n        if self.option(\"build\"):\n            if publisher.files and not self.confirm(\n                f\"There are <info>{len(publisher.files)}</info> files ready for\"\n                \" publishing. Build anyway?\"\n            ):\n                self.line_error(\"<error>Aborted!</error>\")\n\n                return 1\n\n            self.call(\"build\", args=f\"--output {dist_dir}\")\n\n        files = publisher.files\n        if not files:\n            self.line_error(\n                \"<error>No files to publish. \"\n                \"Run poetry build first or use the --build option.</error>\"\n            )\n\n            return 1\n\n        self.line(\"\")\n\n        cert = Path(self.option(\"cert\")) if self.option(\"cert\") else None\n        client_cert = (\n            Path(self.option(\"client-cert\")) if self.option(\"client-cert\") else None\n        )\n\n        publisher.publish(\n            self.option(\"repository\"),\n            self.option(\"username\"),\n            self.option(\"password\"),\n            cert,\n            client_cert,\n            self.option(\"dry-run\"),\n            self.option(\"skip-existing\"),\n        )\n\n        return 0\n\n# Path: src/poetry/console/commands/remove.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\nfrom poetry.core.packages.dependency_group import MAIN_GROUP\nfrom tomlkit.toml_document import TOMLDocument\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass RemoveCommand(InstallerCommand):\n    name = \"remove\"\n    description = \"Removes a package from the project dependencies.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"packages\", \"The packages to remove.\", multiple=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\"group\", \"G\", \"The group to remove the dependency from.\", flag=False),\n        option(\n            \"dev\",\n            \"D\",\n            \"Remove a package from the development dependencies.\"\n            \" (<warning>Deprecated</warning>)\"\n            \" Use --group=dev instead.\",\n        ),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything \"\n            \"(implicitly enables --verbose).\",\n        ),\n        option(\"lock\", None, \"Do not perform operations (only update the lockfile).\"),\n    ]\n\n    help = \"\"\"The <info>remove</info> command removes a package from the current\nlist of installed packages\n\n<info>poetry remove</info>\"\"\"\n\n    loggers: ClassVar[list[str]] = [\n        \"poetry.repositories.pypi_repository\",\n        \"poetry.inspection.info\",\n    ]\n\n    def handle(self) -> int:\n        packages = self.argument(\"packages\")\n\n        if self.option(\"dev\"):\n            self.line_error(\n                \"<warning>The --dev option is deprecated, \"\n                \"use the `--group dev` notation instead.</warning>\"\n            )\n            group = \"dev\"\n        else:\n            group = self.option(\"group\", self.default_group)\n\n        content: dict[str, Any] = self.poetry.file.read()\n        poetry_content = content[\"tool\"][\"poetry\"]\n\n        if group is None:\n            removed = []\n            group_sections = [\n                (group_name, group_section.get(\"dependencies\", {}))\n                for group_name, group_section in poetry_content.get(\"group\", {}).items()\n            ]\n\n            for group_name, section in [\n                (MAIN_GROUP, poetry_content[\"dependencies\"]),\n                *group_sections,\n            ]:\n                removed += self._remove_packages(packages, section, group_name)\n                if group_name != MAIN_GROUP:\n                    if not section:\n                        del poetry_content[\"group\"][group_name]\n                    else:\n                        poetry_content[\"group\"][group_name][\"dependencies\"] = section\n        elif group == \"dev\" and \"dev-dependencies\" in poetry_content:\n            # We need to account for the old `dev-dependencies` section\n            removed = self._remove_packages(\n                packages, poetry_content[\"dev-dependencies\"], \"dev\"\n            )\n\n            if not poetry_content[\"dev-dependencies\"]:\n                del poetry_content[\"dev-dependencies\"]\n        else:\n            removed = []\n            if \"group\" in poetry_content:\n                if group in poetry_content[\"group\"]:\n                    removed = self._remove_packages(\n                        packages,\n                        poetry_content[\"group\"][group].get(\"dependencies\", {}),\n                        group,\n                    )\n\n                if not poetry_content[\"group\"][group]:\n                    del poetry_content[\"group\"][group]\n\n        if \"group\" in poetry_content and not poetry_content[\"group\"]:\n            del poetry_content[\"group\"]\n\n        removed_set = set(removed)\n        not_found = set(packages).difference(removed_set)\n        if not_found:\n            raise ValueError(\n                \"The following packages were not found: \" + \", \".join(sorted(not_found))\n            )\n\n        # Refresh the locker\n        self.poetry.locker.set_local_config(poetry_content)\n        self.installer.set_locker(self.poetry.locker)\n        self.installer.set_package(self.poetry.package)\n        self.installer.dry_run(self.option(\"dry-run\", False))\n        self.installer.verbose(self.io.is_verbose())\n        self.installer.update(True)\n        self.installer.execute_operations(not self.option(\"lock\"))\n        self.installer.whitelist(removed_set)\n\n        status = self.installer.run()\n\n        if not self.option(\"dry-run\") and status == 0:\n            assert isinstance(content, TOMLDocument)\n            self.poetry.file.write(content)\n\n        return status\n\n    def _remove_packages(\n        self, packages: list[str], section: dict[str, Any], group_name: str\n    ) -> list[str]:\n        removed = []\n        group = self.poetry.package.dependency_group(group_name)\n        section_keys = list(section.keys())\n\n        for package in packages:\n            for existing_package in section_keys:\n                if canonicalize_name(existing_package) == canonicalize_name(package):\n                    del section[existing_package]\n                    removed.append(package)\n                    group.remove_dependency(package)\n\n        return removed\n\n# Path: src/poetry/console/commands/run.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\n\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.utils._compat import WINDOWS\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from poetry.core.masonry.utils.module import Module\n\n\nclass RunCommand(EnvCommand):\n    name = \"run\"\n    description = \"Runs a command in the appropriate environment.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"args\", \"The command and arguments/options to run.\", multiple=True)\n    ]\n\n    def handle(self) -> int:\n        args = self.argument(\"args\")\n        script = args[0]\n        scripts = self.poetry.local_config.get(\"scripts\")\n\n        if scripts and script in scripts:\n            return self.run_script(scripts[script], args)\n\n        try:\n            return self.env.execute(*args)\n        except FileNotFoundError:\n            self.line_error(f\"<error>Command not found: <c1>{script}</c1></error>\")\n            return 1\n\n    @property\n    def _module(self) -> Module:\n        from poetry.core.masonry.utils.module import Module\n\n        poetry = self.poetry\n        package = poetry.package\n        path = poetry.file.path.parent\n        module = Module(package.name, path.as_posix(), package.packages)\n\n        return module\n\n    def run_script(self, script: str | dict[str, str], args: list[str]) -> int:\n        \"\"\"Runs an entry point script defined in the section ``[tool.poetry.scripts]``.\n\n        When a script exists in the venv bin folder, i.e. after ``poetry install``,\n        then ``sys.argv[0]`` must be set to the full path of the executable, so\n        ``poetry run foo`` and ``poetry shell``, ``foo`` have the same ``sys.argv[0]``\n        that points to the full path.\n\n        Otherwise (when an entry point script does not exist), ``sys.argv[0]`` is the\n        script name only, i.e. ``poetry run foo`` has ``sys.argv == ['foo']``.\n        \"\"\"\n        for script_dir in self.env.script_dirs:\n            script_path = script_dir / args[0]\n            if WINDOWS:\n                script_path = script_path.with_suffix(\".cmd\")\n            if script_path.exists():\n                args = [str(script_path), *args[1:]]\n                break\n        else:\n            # If we reach this point, the script is not installed\n            self._warning_not_installed_script(args[0])\n\n        if isinstance(script, dict):\n            script = script[\"callable\"]\n\n        module, callable_ = script.split(\":\")\n\n        src_in_sys_path = \"sys.path.append('src'); \" if self._module.is_in_src() else \"\"\n\n        cmd = [\"python\", \"-c\"]\n\n        cmd += [\n            \"import sys; \"\n            \"from importlib import import_module; \"\n            f\"sys.argv = {args!r}; {src_in_sys_path}\"\n            f\"sys.exit(import_module('{module}').{callable_}())\"\n        ]\n\n        return self.env.execute(*cmd)\n\n    def _warning_not_installed_script(self, script: str) -> None:\n        message = f\"\"\"\\\nWarning: '{script}' is an entry point defined in pyproject.toml, but it's not \\\ninstalled as a script. You may get improper `sys.argv[0]`.\n\nThe support to run uninstalled scripts will be removed in a future release.\n\nRun `poetry install` to resolve and get rid of this message.\n\"\"\"\n        self.line_error(message, style=\"warning\")\n\n# Path: src/poetry/console/commands/search.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n\n\nclass SearchCommand(Command):\n    name = \"search\"\n    description = \"Searches for packages on remote repositories.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"tokens\", \"The tokens to search for.\", multiple=True)\n    ]\n\n...\n", "needle_token_start": 3886, "needle_token_end": 4116, "code_context_ntokens": 16018, "output": ["```python\ndef _validate_group_options(self, group_options: dict[str, set[str]]) -> None:\n    \"\"\"\n    Raises an error if it detects that a group is not part of pyproject.toml\n    \"\"\"\n    invalid_options = defaultdict(set)\n    for opt, groups in group_options.items():\n        for group in groups:\n            if not self.poetry.package.has_dependency_group(group):\n                invalid_options[group].add(opt)\n    if invalid_options:\n        message_parts = []\n        for group in sorted(invalid_options):\n            opts = \", \".join(\n                f\"<fg=yellow;options=bold>--{opt}</>\"\n                for opt in sorted(invalid_options[group])\n            )\n            message_parts.append(f\"{group} (via {opts})\")\n        raise GroupNotFound(f\"Group(s) not found: {', '.join(message_parts)}\")\n```"]}
{"repo": "python-poetry/poetry", "name": "candidates", "language": "python", "path": "src/poetry/utils/env/site_packages.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To compile a list of potential directories where Python packages can be installed.\n2. **Input**: Directories specified for pure Python libraries and platform-specific libraries, along with optional fallback directories.\n3. **Output**: A list of unique directory paths that are considered as potential installation targets.\n4. **Procedure**: The method combines the specified directories for pure and platform-specific libraries with any provided fallback directories. It ensures each directory is unique in the resulting list to avoid duplication.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/utils/env/env_manager.py\nfrom __future__ import annotations\n\nimport base64\nimport hashlib\nimport os\nimport plistlib\nimport re\nimport shutil\nimport subprocess\nimport sys\n\nfrom functools import cached_property\nfrom pathlib import Path\nfrom subprocess import CalledProcessError\nfrom typing import TYPE_CHECKING\n\nimport tomlkit\nimport virtualenv\n\nfrom cleo.io.null_io import NullIO\nfrom cleo.io.outputs.output import Verbosity\nfrom poetry.core.constraints.version import Version\nfrom poetry.core.constraints.version import parse_constraint\n\nfrom poetry.toml.file import TOMLFile\nfrom poetry.utils._compat import WINDOWS\nfrom poetry.utils._compat import encode\nfrom poetry.utils.env.exceptions import EnvCommandError\nfrom poetry.utils.env.exceptions import IncorrectEnvError\nfrom poetry.utils.env.exceptions import InvalidCurrentPythonVersionError\nfrom poetry.utils.env.exceptions import NoCompatiblePythonVersionFound\nfrom poetry.utils.env.exceptions import PythonVersionNotFound\nfrom poetry.utils.env.generic_env import GenericEnv\nfrom poetry.utils.env.script_strings import GET_ENV_PATH_ONELINER\nfrom poetry.utils.env.script_strings import GET_PYTHON_VERSION_ONELINER\nfrom poetry.utils.env.system_env import SystemEnv\nfrom poetry.utils.env.virtual_env import VirtualEnv\nfrom poetry.utils.helpers import get_real_windows_path\nfrom poetry.utils.helpers import remove_directory\n\n\nif TYPE_CHECKING:\n    from cleo.io.io import IO\n\n    from poetry.poetry import Poetry\n    from poetry.utils.env.base_env import Env\n\n\nclass EnvsFile(TOMLFile):\n    \"\"\"\n    This file contains one section per project with the project's base env name\n    as section name. Each section contains the minor and patch version of the\n    python executable used to create the currently active virtualenv.\n\n    Example:\n\n    [poetry-QRErDmmj]\n    minor = \"3.9\"\n    patch = \"3.9.13\"\n\n    [poetry-core-m5r7DkRA]\n    minor = \"3.11\"\n    patch = \"3.11.6\"\n    \"\"\"\n\n    def remove_section(self, name: str, minor: str | None = None) -> str | None:\n        \"\"\"\n        Remove a section from the envs file.\n\n        If \"minor\" is given, the section is only removed if its minor value\n        matches \"minor\".\n\n        Returns the \"minor\" value of the removed section.\n        \"\"\"\n        envs = self.read()\n        current_env = envs.get(name)\n        if current_env is not None and (not minor or current_env[\"minor\"] == minor):\n            del envs[name]\n            self.write(envs)\n            minor = current_env[\"minor\"]\n            assert isinstance(minor, str)\n            return minor\n\n        return None\n\n\nclass EnvManager:\n    \"\"\"\n    Environments manager\n    \"\"\"\n\n    _env = None\n\n    ENVS_FILE = \"envs.toml\"\n\n    def __init__(self, poetry: Poetry, io: None | IO = None) -> None:\n        self._poetry = poetry\n        self._io = io or NullIO()\n\n    @staticmethod\n    def _full_python_path(python: str) -> Path | None:\n        # eg first find pythonXY.bat on windows.\n        path_python = shutil.which(python)\n        if path_python is None:\n            return None\n\n        try:\n            executable = subprocess.check_output(\n                [path_python, \"-c\", \"import sys; print(sys.executable)\"], text=True\n            ).strip()\n            return Path(executable)\n\n        except CalledProcessError:\n            return None\n\n    @staticmethod\n    def _detect_active_python(io: None | IO = None) -> Path | None:\n        io = io or NullIO()\n        io.write_error_line(\n            \"Trying to detect current active python executable as specified in\"\n            \" the config.\",\n            verbosity=Verbosity.VERBOSE,\n        )\n\n        executable = EnvManager._full_python_path(\"python\")\n\n        if executable is not None:\n            io.write_error_line(f\"Found: {executable}\", verbosity=Verbosity.VERBOSE)\n        else:\n            io.write_error_line(\n                \"Unable to detect the current active python executable. Falling\"\n                \" back to default.\",\n                verbosity=Verbosity.VERBOSE,\n            )\n\n        return executable\n\n    @staticmethod\n    def get_python_version(\n        precision: int = 3,\n        prefer_active_python: bool = False,\n...\n# Path: src/poetry/utils/env/exceptions.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.utils._compat import decode\n\n\nif TYPE_CHECKING:\n    from subprocess import CalledProcessError\n\n\nclass EnvError(Exception):\n    pass\n\n\nclass IncorrectEnvError(EnvError):\n    def __init__(self, env_name: str) -> None:\n        message = f\"Env {env_name} doesn't belong to this project.\"\n        super().__init__(message)\n\n\nclass EnvCommandError(EnvError):\n    def __init__(self, e: CalledProcessError) -> None:\n        self.e = e\n\n        message_parts = [\n            f\"Command {e.cmd} errored with the following return code {e.returncode}\"\n        ]\n        if e.output:\n            message_parts.append(f\"Output:\\n{decode(e.output)}\")\n        if e.stderr:\n            message_parts.append(f\"Error output:\\n{decode(e.stderr)}\")\n        super().__init__(\"\\n\\n\".join(message_parts))\n\n\nclass PythonVersionNotFound(EnvError):\n    def __init__(self, expected: str) -> None:\n        super().__init__(f\"Could not find the python executable {expected}\")\n\n\nclass NoCompatiblePythonVersionFound(EnvError):\n    def __init__(self, expected: str, given: str | None = None) -> None:\n        if given:\n            message = (\n                f\"The specified Python version ({given}) \"\n                f\"is not supported by the project ({expected}).\\n\"\n                \"Please choose a compatible version \"\n                \"or loosen the python constraint specified \"\n                \"in the pyproject.toml file.\"\n            )\n        else:\n            message = (\n                \"Poetry was unable to find a compatible version. \"\n                \"If you have one, you can explicitly use it \"\n                'via the \"env use\" command.'\n            )\n\n        super().__init__(message)\n\n\nclass InvalidCurrentPythonVersionError(EnvError):\n    def __init__(self, expected: str, given: str) -> None:\n        message = (\n            f\"Current Python version ({given}) \"\n            f\"is not allowed by the project ({expected}).\\n\"\n            'Please change python executable via the \"env use\" command.'\n        )\n\n        super().__init__(message)\n\n# Path: src/poetry/utils/env/generic_env.py\nfrom __future__ import annotations\n\nimport json\nimport os\nimport re\nimport subprocess\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom poetry.utils.env.script_strings import GET_PATHS_FOR_GENERIC_ENVS\nfrom poetry.utils.env.virtual_env import VirtualEnv\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n\n    from poetry.utils.env.base_env import Env\n\n\nclass GenericEnv(VirtualEnv):\n    def __init__(\n        self, path: Path, base: Path | None = None, child_env: Env | None = None\n    ) -> None:\n        self._child_env = child_env\n\n        super().__init__(path, base=base)\n\n    def find_executables(self) -> None:\n        patterns = [(\"python*\", \"pip*\")]\n\n        if self._child_env:\n            minor_version = (\n                f\"{self._child_env.version_info[0]}.{self._child_env.version_info[1]}\"\n            )\n            major_version = f\"{self._child_env.version_info[0]}\"\n            patterns = [\n                (f\"python{minor_version}\", f\"pip{minor_version}\"),\n                (f\"python{major_version}\", f\"pip{major_version}\"),\n            ]\n\n        python_executable = None\n        pip_executable = None\n\n        for python_pattern, pip_pattern in patterns:\n            if python_executable and pip_executable:\n                break\n\n            if not python_executable:\n                python_executables = sorted(\n                    p.name\n                    for p in self._bin_dir.glob(python_pattern)\n                    if re.match(r\"python(?:\\d+(?:\\.\\d+)?)?(?:\\.exe)?$\", p.name)\n                )\n\n                if python_executables:\n                    executable = python_executables[0]\n                    if executable.endswith(\".exe\"):\n                        executable = executable[:-4]\n\n                    python_executable = executable\n\n            if not pip_executable:\n                pip_executables = sorted(\n                    p.name\n                    for p in self._bin_dir.glob(pip_pattern)\n                    if re.match(r\"pip(?:\\d+(?:\\.\\d+)?)?(?:\\.exe)?$\", p.name)\n                )\n                if pip_executables:\n                    pip_executable = pip_executables[0]\n                    if pip_executable.endswith(\".exe\"):\n                        pip_executable = pip_executable[:-4]\n\n            if python_executable:\n                self._executable = python_executable\n\n            if pip_executable:\n                self._pip_executable = pip_executable\n\n    def get_paths(self) -> dict[str, str]:\n        output = self.run_python_script(GET_PATHS_FOR_GENERIC_ENVS)\n\n        paths: dict[str, str] = json.loads(output)\n        return paths\n\n    def execute(self, bin: str, *args: str, **kwargs: Any) -> int:\n        command = self.get_command_from_bin(bin) + list(args)\n        env = kwargs.pop(\"env\", dict(os.environ))\n\n        if not self._is_windows:\n            return os.execvpe(command[0], command, env=env)\n\n        exe = subprocess.Popen(command, env=env, **kwargs)\n        exe.communicate()\n\n        return exe.returncode\n\n    def _run(self, cmd: list[str], **kwargs: Any) -> str:\n        return super(VirtualEnv, self)._run(cmd, **kwargs)\n\n    def is_venv(self) -> bool:\n        return self._path != self._base\n\n# Path: src/poetry/utils/env/mock_env.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom poetry.utils.env.null_env import NullEnv\n\n\nif TYPE_CHECKING:\n    from packaging.tags import Tag\n\n\nclass MockEnv(NullEnv):\n    def __init__(\n        self,\n        version_info: tuple[int, int, int] = (3, 7, 0),\n        *,\n        python_implementation: str = \"CPython\",\n        platform: str = \"darwin\",\n        platform_machine: str = \"amd64\",\n        os_name: str = \"posix\",\n        is_venv: bool = False,\n        sys_path: list[str] | None = None,\n        marker_env: dict[str, Any] | None = None,\n        supported_tags: list[Tag] | None = None,\n        **kwargs: Any,\n    ) -> None:\n        super().__init__(**kwargs)\n\n        self._version_info = version_info\n        self._python_implementation = python_implementation\n        self._platform = platform\n        self._platform_machine = platform_machine\n        self._os_name = os_name\n        self._is_venv = is_venv\n        self._sys_path = sys_path\n        self._mock_marker_env = marker_env\n        self._supported_tags = supported_tags\n\n    @property\n    def platform(self) -> str:\n        return self._platform\n\n    @property\n    def platform_machine(self) -> str:\n        return self._platform_machine\n\n    @property\n    def os(self) -> str:\n        return self._os_name\n\n    @property\n    def sys_path(self) -> list[str]:\n        if self._sys_path is None:\n            return super().sys_path\n\n        return self._sys_path\n\n    def get_marker_env(self) -> dict[str, Any]:\n        if self._mock_marker_env is not None:\n            return self._mock_marker_env\n\n        marker_env = super().get_marker_env()\n        marker_env[\"python_implementation\"] = self._python_implementation\n        marker_env[\"version_info\"] = self._version_info\n        marker_env[\"python_version\"] = \".\".join(str(v) for v in self._version_info[:2])\n        marker_env[\"python_full_version\"] = \".\".join(str(v) for v in self._version_info)\n        marker_env[\"sys_platform\"] = self._platform\n        marker_env[\"platform_machine\"] = self._platform_machine\n        marker_env[\"interpreter_name\"] = self._python_implementation.lower()\n        marker_env[\"interpreter_version\"] = \"cp\" + \"\".join(\n            str(v) for v in self._version_info[:2]\n        )\n\n        return marker_env\n\n    def is_venv(self) -> bool:\n        return self._is_venv\n\n# Path: src/poetry/utils/env/null_env.py\nfrom __future__ import annotations\n\nimport sys\n\nfrom pathlib import Path\nfrom typing import Any\n\nfrom poetry.utils.env.system_env import SystemEnv\n\n\nclass NullEnv(SystemEnv):\n    def __init__(\n        self, path: Path | None = None, base: Path | None = None, execute: bool = False\n    ) -> None:\n        if path is None:\n            path = Path(sys.prefix)\n\n        super().__init__(path, base=base)\n\n        self._execute = execute\n        self.executed: list[list[str]] = []\n\n    @property\n    def paths(self) -> dict[str, str]:\n        if self._paths is None:\n            self._paths = self.get_paths()\n            self._paths[\"platlib\"] = str(self._path / \"platlib\")\n            self._paths[\"purelib\"] = str(self._path / \"purelib\")\n            self._paths[\"scripts\"] = str(self._path / \"scripts\")\n            self._paths[\"data\"] = str(self._path / \"data\")\n\n        return self._paths\n\n    def _run(self, cmd: list[str], **kwargs: Any) -> str:\n        self.executed.append(cmd)\n\n        if self._execute:\n            return super()._run(cmd, **kwargs)\n        return \"\"\n\n    def execute(self, bin: str, *args: str, **kwargs: Any) -> int:\n        self.executed.append([bin, *list(args)])\n\n        if self._execute:\n            return super().execute(bin, *args, **kwargs)\n        return 0\n\n    def _bin(self, bin: str) -> str:\n        return bin\n\n# Path: src/poetry/utils/env/script_strings.py\nfrom __future__ import annotations\n\nimport packaging.tags\n\n\nGET_SYS_TAGS = f\"\"\"\nimport importlib.util\nimport json\nimport sys\n\nfrom pathlib import Path\n\nspec = importlib.util.spec_from_file_location(\n    \"packaging\", Path(r\"{packaging.__file__}\")\n)\npackaging = importlib.util.module_from_spec(spec)\nsys.modules[spec.name] = packaging\n\nspec = importlib.util.spec_from_file_location(\n    \"packaging.tags\", Path(r\"{packaging.tags.__file__}\")\n)\npackaging_tags = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(packaging_tags)\n\nprint(\n    json.dumps([(t.interpreter, t.abi, t.platform) for t in packaging_tags.sys_tags()])\n)\n\"\"\"\n\nGET_ENVIRONMENT_INFO = \"\"\"\\\nimport json\nimport os\nimport platform\nimport sys\nimport sysconfig\n\nINTERPRETER_SHORT_NAMES = {\n    \"python\": \"py\",\n    \"cpython\": \"cp\",\n    \"pypy\": \"pp\",\n    \"ironpython\": \"ip\",\n    \"jython\": \"jy\",\n}\n\n\ndef interpreter_version():\n    version = sysconfig.get_config_var(\"interpreter_version\")\n    if version:\n        version = str(version)\n    else:\n        version = _version_nodot(sys.version_info[:2])\n\n    return version\n\n\ndef _version_nodot(version):\n    if any(v >= 10 for v in version):\n        sep = \"_\"\n    else:\n        sep = \"\"\n\n    return sep.join(map(str, version))\n\n\nif hasattr(sys, \"implementation\"):\n    info = sys.implementation.version\n    iver = \"{0.major}.{0.minor}.{0.micro}\".format(info)\n    kind = info.releaselevel\n    if kind != \"final\":\n        iver += kind[0] + str(info.serial)\n\n    implementation_name = sys.implementation.name\nelse:\n    iver = \"0\"\n    implementation_name = platform.python_implementation().lower()\n\nenv = {\n    \"implementation_name\": implementation_name,\n    \"implementation_version\": iver,\n    \"os_name\": os.name,\n    \"platform_machine\": platform.machine(),\n    \"platform_release\": platform.release(),\n    \"platform_system\": platform.system(),\n    \"platform_version\": platform.version(),\n    \"python_full_version\": platform.python_version(),\n    \"platform_python_implementation\": platform.python_implementation(),\n    \"python_version\": \".\".join(platform.python_version_tuple()[:2]),\n    \"sys_platform\": sys.platform,\n    \"version_info\": tuple(sys.version_info),\n    # Extra information\n    \"interpreter_name\": INTERPRETER_SHORT_NAMES.get(\n        implementation_name, implementation_name\n    ),\n    \"interpreter_version\": interpreter_version(),\n}\n\nprint(json.dumps(env))\n\"\"\"\n\nGET_BASE_PREFIX = \"\"\"\\\nimport sys\n\nif hasattr(sys, \"real_prefix\"):\n    print(sys.real_prefix)\nelif hasattr(sys, \"base_prefix\"):\n    print(sys.base_prefix)\nelse:\n    print(sys.prefix)\n\"\"\"\n\nGET_PYTHON_VERSION = \"\"\"\\\nimport sys\n\nprint('.'.join([str(s) for s in sys.version_info[:3]]))\n\"\"\"\n\nGET_PYTHON_VERSION_ONELINER = (\n    \"import sys; print('.'.join([str(s) for s in sys.version_info[:3]]))\"\n)\nGET_ENV_PATH_ONELINER = \"import sys; print(sys.prefix)\"\n\nGET_SYS_PATH = \"\"\"\\\nimport json\nimport sys\n\nprint(json.dumps(sys.path))\n\"\"\"\n\nGET_PATHS = \"\"\"\\\nimport json\nimport sysconfig\n\nprint(json.dumps(sysconfig.get_paths()))\n\"\"\"\n\nGET_PATHS_FOR_GENERIC_ENVS = \"\"\"\\\nimport json\nimport site\nimport sysconfig\n\npaths = sysconfig.get_paths().copy()\n\nif site.check_enableusersite():\n    paths[\"usersite\"] = site.getusersitepackages()\n    paths[\"userbase\"] = site.getuserbase()\n\nprint(json.dumps(paths))\n\"\"\"\n\n# Path: src/poetry/utils/env/site_packages.py\nfrom __future__ import annotations\n\nimport contextlib\nimport itertools\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom poetry.utils._compat import metadata\nfrom poetry.utils.helpers import is_dir_writable\nfrom poetry.utils.helpers import paths_csv\nfrom poetry.utils.helpers import remove_directory\n\n\nif TYPE_CHECKING:\n    from collections.abc import Iterable\n\n\nclass SitePackages:\n    def __init__(\n        self,\n        purelib: Path,\n        platlib: Path | None = None,\n        fallbacks: list[Path] | None = None,\n        skip_write_checks: bool = False,\n    ) -> None:\n        self._purelib = purelib\n        self._platlib = platlib or purelib\n\n        if platlib and platlib.resolve() == purelib.resolve():\n            self._platlib = purelib\n\n        self._fallbacks = fallbacks or []\n        self._skip_write_checks = skip_write_checks\n\n        self._candidates: list[Path] = []\n        for path in itertools.chain([self._purelib, self._platlib], self._fallbacks):\n            if path not in self._candidates:\n                self._candidates.append(path)\n\n        self._writable_candidates = None if not skip_write_checks else self._candidates\n\n    @property\n    def path(self) -> Path:\n        return self._purelib\n\n    @property\n    def purelib(self) -> Path:\n        return self._purelib\n\n    @property\n    def platlib(self) -> Path:\n        return self._platlib\n\n    @property\n    \ndef candidates(self) -> list[Path]:\n        return self._candidates\n\n    @property\n    def writable_candidates(self) -> list[Path]:\n        if self._writable_candidates is not None:\n            return self._writable_candidates\n\n        self._writable_candidates = []\n        for candidate in self._candidates:\n            if not is_dir_writable(path=candidate, create=True):\n                continue\n            self._writable_candidates.append(candidate)\n\n        return self._writable_candidates\n\n    def make_candidates(\n        self, path: Path, writable_only: bool = False, strict: bool = False\n    ) -> list[Path]:\n        candidates = self._candidates if not writable_only else self.writable_candidates\n        if path.is_absolute():\n            for candidate in candidates:\n                with contextlib.suppress(ValueError):\n                    path.relative_to(candidate)\n                    return [path]\n            site_type = \"writable \" if writable_only else \"\"\n            raise ValueError(\n                f\"{path} is not relative to any discovered {site_type}sites\"\n            )\n\n        results = [candidate / path for candidate in candidates]\n\n        if not results and strict:\n            raise RuntimeError(\n                f'Unable to find a suitable destination for \"{path}\" in'\n                f\" {paths_csv(self._candidates)}\"\n            )\n\n        return results\n\n    def distributions(\n        self, name: str | None = None, writable_only: bool = False\n    ) -> Iterable[metadata.Distribution]:\n        path = list(\n            map(\n                str, self._candidates if not writable_only else self.writable_candidates\n            )\n        )\n\n        yield from metadata.PathDistribution.discover(name=name, path=path)\n\n    def find_distribution(\n        self, name: str, writable_only: bool = False\n    ) -> metadata.Distribution | None:\n        for distribution in self.distributions(name=name, writable_only=writable_only):\n            return distribution\n        return None\n\n    def find_distribution_files_with_suffix(\n        self, distribution_name: str, suffix: str, writable_only: bool = False\n    ) -> Iterable[Path]:\n        for distribution in self.distributions(\n            name=distribution_name, writable_only=writable_only\n        ):\n            files = [] if distribution.files is None else distribution.files\n            for file in files:\n                if file.name.endswith(suffix):\n                    path = distribution.locate_file(file)\n                    assert isinstance(path, Path)\n                    yield path\n\n    def find_distribution_files_with_name(\n        self, distribution_name: str, name: str, writable_only: bool = False\n    ) -> Iterable[Path]:\n        for distribution in self.distributions(\n            name=distribution_name, writable_only=writable_only\n        ):\n            files = [] if distribution.files is None else distribution.files\n            for file in files:\n                if file.name == name:\n                    path = distribution.locate_file(file)\n                    assert isinstance(path, Path)\n                    yield path\n\n    def find_distribution_direct_url_json_files(\n        self, distribution_name: str, writable_only: bool = False\n    ) -> Iterable[Path]:\n        return self.find_distribution_files_with_name(\n            distribution_name=distribution_name,\n            name=\"direct_url.json\",\n            writable_only=writable_only,\n        )\n\n    def remove_distribution_files(self, distribution_name: str) -> list[Path]:\n        paths = []\n\n        for distribution in self.distributions(\n            name=distribution_name, writable_only=True\n        ):\n            files = [] if distribution.files is None else distribution.files\n            for file in files:\n                path = distribution.locate_file(file)\n                assert isinstance(path, Path)\n                path.unlink(missing_ok=True)\n\n            distribution_path: Path = distribution._path  # type: ignore[attr-defined]\n            if distribution_path.exists():\n                remove_directory(distribution_path, force=True)\n\n            paths.append(distribution_path)\n\n        return paths\n\n    def _path_method_wrapper(\n        self,\n        path: Path,\n        method: str,\n        *args: Any,\n        return_first: bool = True,\n        writable_only: bool = False,\n        **kwargs: Any,\n    ) -> tuple[Path, Any] | list[tuple[Path, Any]]:\n        candidates = self.make_candidates(\n            path, writable_only=writable_only, strict=True\n        )\n\n        results = []\n\n        for candidate in candidates:\n            try:\n                result = candidate, getattr(candidate, method)(*args, **kwargs)\n                if return_first:\n                    return result\n                results.append(result)\n            except OSError:\n                # TODO: Replace with PermissionError\n                pass\n\n        if results:\n            return results\n\n        raise OSError(f\"Unable to access any of {paths_csv(candidates)}\")\n\n    def write_text(self, path: Path, *args: Any, **kwargs: Any) -> Path:\n        paths = self._path_method_wrapper(path, \"write_text\", *args, **kwargs)\n        assert isinstance(paths, tuple)\n        return paths[0]\n\n    def mkdir(self, path: Path, *args: Any, **kwargs: Any) -> Path:\n        paths = self._path_method_wrapper(path, \"mkdir\", *args, **kwargs)\n        assert isinstance(paths, tuple)\n        return paths[0]\n\n    def exists(self, path: Path) -> bool:\n        return any(\n            value[-1]\n            for value in self._path_method_wrapper(path, \"exists\", return_first=False)\n        )\n\n    def find(\n        self,\n        path: Path,\n        writable_only: bool = False,\n    ) -> list[Path]:\n        return [\n            value[0]\n            for value in self._path_method_wrapper(\n                path, \"exists\", return_first=False, writable_only=writable_only\n            )\n            if value[-1] is True\n        ]\n\n# Path: src/poetry/utils/env/system_env.py\nfrom __future__ import annotations\n\nimport os\nimport platform\nimport site\nimport sys\nimport sysconfig\n\nfrom pathlib import Path\nfrom typing import Any\n\nfrom packaging.tags import Tag\nfrom packaging.tags import interpreter_name\nfrom packaging.tags import interpreter_version\nfrom packaging.tags import sys_tags\n\nfrom poetry.utils.env.base_env import Env\n\n\nclass SystemEnv(Env):\n    \"\"\"\n    A system (i.e. not a virtualenv) Python environment.\n    \"\"\"\n\n    @property\n    def python(self) -> Path:\n        return Path(sys.executable)\n\n    @property\n    def sys_path(self) -> list[str]:\n        return sys.path\n\n    def get_version_info(self) -> tuple[Any, ...]:\n        return tuple(sys.version_info)\n\n    def get_python_implementation(self) -> str:\n        return platform.python_implementation()\n\n    def get_paths(self) -> dict[str, str]:\n        import site\n\n        paths = sysconfig.get_paths().copy()\n\n        if site.check_enableusersite():\n            paths[\"usersite\"] = site.getusersitepackages()\n            paths[\"userbase\"] = site.getuserbase()\n\n        return paths\n\n    def get_supported_tags(self) -> list[Tag]:\n        return list(sys_tags())\n\n    def get_marker_env(self) -> dict[str, Any]:\n        if hasattr(sys, \"implementation\"):\n            info = sys.implementation.version\n            iver = f\"{info.major}.{info.minor}.{info.micro}\"\n            kind = info.releaselevel\n            if kind != \"final\":\n                iver += kind[0] + str(info.serial)\n\n            implementation_name = sys.implementation.name\n        else:\n            iver = \"0\"\n            implementation_name = \"\"\n\n        return {\n            \"implementation_name\": implementation_name,\n            \"implementation_version\": iver,\n            \"os_name\": os.name,\n            \"platform_machine\": platform.machine(),\n            \"platform_release\": platform.release(),\n            \"platform_system\": platform.system(),\n            \"platform_version\": platform.version(),\n            \"python_full_version\": platform.python_version(),\n            \"platform_python_implementation\": platform.python_implementation(),\n            \"python_version\": \".\".join(platform.python_version().split(\".\")[:2]),\n            \"sys_platform\": sys.platform,\n            \"version_info\": sys.version_info,\n            \"interpreter_name\": interpreter_name(),\n            \"interpreter_version\": interpreter_version(),\n        }\n\n    def is_venv(self) -> bool:\n        return self._path != self._base\n\n    def _get_lib_dirs(self) -> list[Path]:\n        return super()._get_lib_dirs() + [Path(d) for d in site.getsitepackages()]\n\n# Path: src/poetry/utils/env/virtual_env.py\nfrom __future__ import annotations\n\nimport json\nimport os\nimport re\nimport sys\n\nfrom contextlib import contextmanager\nfrom copy import deepcopy\nfrom functools import cached_property\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom packaging.tags import Tag\n\nfrom poetry.utils.env.base_env import Env\nfrom poetry.utils.env.script_strings import GET_BASE_PREFIX\nfrom poetry.utils.env.script_strings import GET_ENVIRONMENT_INFO\nfrom poetry.utils.env.script_strings import GET_PATHS\nfrom poetry.utils.env.script_strings import GET_PYTHON_VERSION\nfrom poetry.utils.env.script_strings import GET_SYS_PATH\nfrom poetry.utils.env.script_strings import GET_SYS_TAGS\nfrom poetry.utils.env.system_env import SystemEnv\n\n\nif TYPE_CHECKING:\n    from collections.abc import Iterator\n\n\nclass VirtualEnv(Env):\n    \"\"\"\n    A virtual Python environment.\n    \"\"\"\n\n    def __init__(self, path: Path, base: Path | None = None) -> None:\n        super().__init__(path, base)\n\n        # If base is None, it probably means this is\n        # a virtualenv created from VIRTUAL_ENV.\n        # In this case we need to get sys.base_prefix\n        # from inside the virtualenv.\n        if base is None:\n            output = self.run_python_script(GET_BASE_PREFIX)\n            self._base = Path(output.strip())\n\n    @property\n    def sys_path(self) -> list[str]:\n        output = self.run_python_script(GET_SYS_PATH)\n        paths: list[str] = json.loads(output)\n        return paths\n\n    def get_version_info(self) -> tuple[Any, ...]:\n        output = self.run_python_script(GET_PYTHON_VERSION)\n        assert isinstance(output, str)\n\n        return tuple(int(s) for s in output.strip().split(\".\"))\n\n    def get_python_implementation(self) -> str:\n        implementation: str = self.marker_env[\"platform_python_implementation\"]\n        return implementation\n\n    def get_supported_tags(self) -> list[Tag]:\n        output = self.run_python_script(GET_SYS_TAGS)\n\n        return [Tag(*t) for t in json.loads(output)]\n\n    def get_marker_env(self) -> dict[str, Any]:\n        output = self.run_python_script(GET_ENVIRONMENT_INFO)\n\n        env: dict[str, Any] = json.loads(output)\n        return env\n\n    def get_paths(self) -> dict[str, str]:\n        output = self.run_python_script(GET_PATHS)\n        paths: dict[str, str] = json.loads(output)\n        return paths\n\n    def is_venv(self) -> bool:\n        return True\n\n    def is_sane(self) -> bool:\n        # A virtualenv is considered sane if \"python\" exists.\n        return os.path.exists(self.python)\n\n    def _run(self, cmd: list[str], **kwargs: Any) -> str:\n        kwargs[\"env\"] = self.get_temp_environ(environ=kwargs.get(\"env\"))\n        return super()._run(cmd, **kwargs)\n\n    def get_temp_environ(\n        self,\n        environ: dict[str, str] | None = None,\n        exclude: list[str] | None = None,\n        **kwargs: str,\n    ) -> dict[str, str]:\n        exclude = exclude or []\n        exclude.extend([\"PYTHONHOME\", \"__PYVENV_LAUNCHER__\"])\n\n        if environ:\n            environ = deepcopy(environ)\n            for key in exclude:\n                environ.pop(key, None)\n        else:\n            environ = {k: v for k, v in os.environ.items() if k not in exclude}\n\n        environ.update(kwargs)\n\n        environ[\"PATH\"] = self._updated_path()\n        environ[\"VIRTUAL_ENV\"] = str(self._path)\n\n        return environ\n\n    def execute(self, bin: str, *args: str, **kwargs: Any) -> int:\n        kwargs[\"env\"] = self.get_temp_environ(environ=kwargs.get(\"env\"))\n        return super().execute(bin, *args, **kwargs)\n\n    @contextmanager\n    def temp_environ(self) -> Iterator[None]:\n        environ = dict(os.environ)\n        try:\n            yield\n        finally:\n            os.environ.clear()\n            os.environ.update(environ)\n\n    def _updated_path(self) -> str:\n        return os.pathsep.join([str(self._bin_dir), os.environ.get(\"PATH\", \"\")])\n\n    @cached_property\n    def includes_system_site_packages(self) -> bool:\n        pyvenv_cfg = self._path / \"pyvenv.cfg\"\n        return pyvenv_cfg.exists() and (\n            re.search(\n                r\"^\\s*include-system-site-packages\\s*=\\s*true\\s*$\",\n                pyvenv_cfg.read_text(),\n                re.IGNORECASE | re.MULTILINE,\n            )\n            is not None\n        )\n\n    def is_path_relative_to_lib(self, path: Path) -> bool:\n        return super().is_path_relative_to_lib(path) or (\n            self.includes_system_site_packages\n            and SystemEnv(Path(sys.prefix)).is_path_relative_to_lib(path)\n        )\n\n# Path: src/poetry/vcs/git/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.vcs.git.backend import Git\n\n\n__all__ = [\"Git\"]\n\n# Path: src/poetry/vcs/git/backend.py\nfrom __future__ import annotations\n\nimport dataclasses\nimport logging\nimport re\n\nfrom pathlib import Path\nfrom subprocess import CalledProcessError\nfrom typing import TYPE_CHECKING\nfrom urllib.parse import urljoin\nfrom urllib.parse import urlparse\nfrom urllib.parse import urlunparse\n\nfrom dulwich import porcelain\nfrom dulwich.client import HTTPUnauthorized\nfrom dulwich.client import get_transport_and_path\nfrom dulwich.config import ConfigFile\nfrom dulwich.config import parse_submodules\nfrom dulwich.errors import NotGitRepository\nfrom dulwich.index import IndexEntry\nfrom dulwich.refs import ANNOTATED_TAG_SUFFIX\nfrom dulwich.repo import Repo\n\nfrom poetry.console.exceptions import PoetryConsoleError\nfrom poetry.utils.authenticator import get_default_authenticator\nfrom poetry.utils.helpers import remove_directory\n\n\nif TYPE_CHECKING:\n    from dulwich.client import FetchPackResult\n    from dulwich.client import GitClient\n\n\nlogger = logging.getLogger(__name__)\n\n# A relative URL by definition starts with ../ or ./\nRELATIVE_SUBMODULE_REGEX = re.compile(r\"^\\.{1,2}/\")\n\n\ndef is_revision_sha(revision: str | None) -> bool:\n    return re.match(r\"^\\b[0-9a-f]{5,40}\\b$\", revision or \"\") is not None\n\n\ndef annotated_tag(ref: str | bytes) -> bytes:\n    if isinstance(ref, str):\n        ref = ref.encode(\"utf-8\")\n    return ref + ANNOTATED_TAG_SUFFIX\n\n\n@dataclasses.dataclass\nclass GitRefSpec:\n    branch: str | None = None\n    revision: str | None = None\n    tag: str | None = None\n    ref: bytes = dataclasses.field(default_factory=lambda: b\"HEAD\")\n\n    def resolve(self, remote_refs: FetchPackResult) -> None:\n        \"\"\"\n        Resolve the ref using the provided remote refs.\n        \"\"\"\n        self._normalise(remote_refs=remote_refs)\n        self._set_head(remote_refs=remote_refs)\n\n    def _normalise(self, remote_refs: FetchPackResult) -> None:\n        \"\"\"\n        Internal helper method to determine if given revision is\n            1. a branch or tag; if so, set corresponding properties.\n            2. a short sha; if so, resolve full sha and set as revision\n        \"\"\"\n        if self.revision:\n            ref = f\"refs/tags/{self.revision}\".encode()\n            if ref in remote_refs.refs or annotated_tag(ref) in remote_refs.refs:\n                # this is a tag, incorrectly specified as a revision, tags take priority\n                self.tag = self.revision\n                self.revision = None\n            elif (\n                self.revision.encode(\"utf-8\") in remote_refs.refs\n                or f\"refs/heads/{self.revision}\".encode() in remote_refs.refs\n            ):\n                # this is most likely a ref spec or a branch incorrectly specified\n                self.branch = self.revision\n                self.revision = None\n        elif (\n            self.branch\n            and f\"refs/heads/{self.branch}\".encode() not in remote_refs.refs\n            and (\n                f\"refs/tags/{self.branch}\".encode() in remote_refs.refs\n                or annotated_tag(f\"refs/tags/{self.branch}\") in remote_refs.refs\n            )\n        ):\n            # this is a tag incorrectly specified as a branch\n            self.tag = self.branch\n            self.branch = None\n\n        if self.revision and self.is_sha_short:\n            # revision is a short sha, resolve to full sha\n            short_sha = self.revision.encode(\"utf-8\")\n            for sha in remote_refs.refs.values():\n                if sha.startswith(short_sha):\n                    self.revision = sha.decode(\"utf-8\")\n                    break\n\n    def _set_head(self, remote_refs: FetchPackResult) -> None:\n        \"\"\"\n        Internal helper method to populate ref and set it's sha as the remote's head\n        and default ref.\n        \"\"\"\n        self.ref = remote_refs.symrefs[b\"HEAD\"]\n\n        if self.revision:\n            head = self.revision.encode(\"utf-8\")\n        else:\n            if self.tag:\n                ref = f\"refs/tags/{self.tag}\".encode()\n                annotated = annotated_tag(ref)\n                self.ref = annotated if annotated in remote_refs.refs else ref\n            elif self.branch:\n                self.ref = (\n                    self.branch.encode(\"utf-8\")\n                    if self.is_ref\n                    else f\"refs/heads/{self.branch}\".encode()\n                )\n            head = remote_refs.refs[self.ref]\n\n        remote_refs.refs[self.ref] = remote_refs.refs[b\"HEAD\"] = head\n\n    @property\n    def key(self) -> str:\n        return self.revision or self.branch or self.tag or self.ref.decode(\"utf-8\")\n\n    @property\n    def is_sha(self) -> bool:\n        return is_revision_sha(revision=self.revision)\n\n    @property\n    def is_ref(self) -> bool:\n        return self.branch is not None and (\n            self.branch.startswith(\"refs/\") or self.branch == \"HEAD\"\n        )\n\n    @property\n    def is_sha_short(self) -> bool:\n        return self.revision is not None and self.is_sha and len(self.revision) < 40\n\n\n@dataclasses.dataclass\nclass GitRepoLocalInfo:\n    repo: dataclasses.InitVar[Repo | Path]\n    origin: str = dataclasses.field(init=False)\n    revision: str = dataclasses.field(init=False)\n\n    def __post_init__(self, repo: Repo | Path) -> None:\n        repo = Git.as_repo(repo=repo) if not isinstance(repo, Repo) else repo\n        self.origin = Git.get_remote_url(repo=repo, remote=\"origin\")\n        self.revision = Git.get_revision(repo=repo)\n\n\nclass Git:\n    @staticmethod\n    def as_repo(repo: Path) -> Repo:\n        return Repo(str(repo))\n\n    @staticmethod\n    def get_remote_url(repo: Repo, remote: str = \"origin\") -> str:\n        with repo:\n            config = repo.get_config()\n            section = (b\"remote\", remote.encode(\"utf-8\"))\n\n            url = \"\"\n            if config.has_section(section):\n                value = config.get(section, b\"url\")\n                url = value.decode(\"utf-8\")\n\n            return url\n\n    @staticmethod\n    def get_revision(repo: Repo) -> str:\n        with repo:\n            return repo.head().decode(\"utf-8\")\n\n    @classmethod\n    def info(cls, repo: Repo | Path) -> GitRepoLocalInfo:\n        return GitRepoLocalInfo(repo=repo)\n\n    @staticmethod\n    def get_name_from_source_url(url: str) -> str:\n        return re.sub(r\"(.git)?$\", \"\", url.rsplit(\"/\", 1)[-1])\n\n    @classmethod\n    def _fetch_remote_refs(cls, url: str, local: Repo) -> FetchPackResult:\n        \"\"\"\n        Helper method to fetch remote refs.\n        \"\"\"\n        client: GitClient\n        path: str\n\n        kwargs: dict[str, str] = {}\n        credentials = get_default_authenticator().get_credentials_for_git_url(url=url)\n\n        if credentials.password and credentials.username:\n            # we do this conditionally as otherwise, dulwich might complain if these\n            # parameters are passed in for an ssh url\n            kwargs[\"username\"] = credentials.username\n            kwargs[\"password\"] = credentials.password\n\n        config = local.get_config_stack()\n        client, path = get_transport_and_path(url, config=config, **kwargs)\n\n        with local:\n            result: FetchPackResult = client.fetch(\n                path,\n                local,\n                determine_wants=local.object_store.determine_wants_all,\n            )\n            return result\n\n    @staticmethod\n    def _clone_legacy(url: str, refspec: GitRefSpec, target: Path) -> Repo:\n        \"\"\"\n        Helper method to facilitate fallback to using system provided git client via\n        subprocess calls.\n        \"\"\"\n        from poetry.vcs.git.system import SystemGit\n\n        logger.debug(\"Cloning '%s' using system git client\", url)\n\n        if target.exists():\n            remove_directory(path=target, force=True)\n\n        revision = refspec.tag or refspec.branch or refspec.revision or \"HEAD\"\n\n        try:\n            SystemGit.clone(url, target)\n        except CalledProcessError:\n            raise PoetryConsoleError(\n                f\"Failed to clone {url}, check your git configuration and permissions\"\n                \" for this repository.\"\n            )\n\n        if revision:\n            revision.replace(\"refs/head/\", \"\")\n            revision.replace(\"refs/tags/\", \"\")\n\n        try:\n            SystemGit.checkout(revision, target)\n        except CalledProcessError:\n            raise PoetryConsoleError(f\"Failed to checkout {url} at '{revision}'\")\n\n        repo = Repo(str(target))\n        return repo\n\n    @classmethod\n    def _clone(cls, url: str, refspec: GitRefSpec, target: Path) -> Repo:\n        \"\"\"\n        Helper method to clone a remove repository at the given `url` at the specified\n        ref spec.\n        \"\"\"\n        local: Repo\n        if not target.exists():\n            local = Repo.init(str(target), mkdir=True)\n            porcelain.remote_add(local, \"origin\", url)\n        else:\n            local = Repo(str(target))\n\n        remote_refs = cls._fetch_remote_refs(url=url, local=local)\n\n        logger.debug(\n            \"Cloning <c2>%s</> at '<c2>%s</>' to <c1>%s</>\", url, refspec.key, target\n        )\n\n        try:\n            refspec.resolve(remote_refs=remote_refs)\n        except KeyError:  # branch / ref does not exist\n            raise PoetryConsoleError(\n                f\"Failed to clone {url} at '{refspec.key}', verify ref exists on\"\n                \" remote.\"\n            )\n\n        # ensure local HEAD matches remote\n        local.refs[b\"HEAD\"] = remote_refs.refs[b\"HEAD\"]\n\n        if refspec.is_ref:\n            # set ref to current HEAD\n            local.refs[refspec.ref] = local.refs[b\"HEAD\"]\n\n        for base, prefix in {\n            (b\"refs/remotes/origin\", b\"refs/heads/\"),\n            (b\"refs/tags\", b\"refs/tags\"),\n        }:\n            local.refs.import_refs(\n                base=base,\n                other={\n                    n[len(prefix) :]: v\n                    for (n, v) in remote_refs.refs.items()\n                    if n.startswith(prefix) and not n.endswith(ANNOTATED_TAG_SUFFIX)\n                },\n            )\n\n        try:\n            with local:\n                local.reset_index()\n        except (AssertionError, KeyError) as e:\n            # this implies the ref we need does not exist or is invalid\n            if isinstance(e, KeyError):\n                # the local copy is at a bad state, lets remove it\n                logger.debug(\n                    \"Removing local clone (<c1>%s</>) of repository as it is in a\"\n                    \" broken state.\",\n                    local.path,\n                )\n                remove_directory(Path(local.path), force=True)\n\n            if isinstance(e, AssertionError) and \"Invalid object name\" not in str(e):\n                raise\n\n            logger.debug(\n                \"\\nRequested ref (<c2>%s</c2>) was not fetched to local copy and\"\n                \" cannot be used. The following error was\"\n                \" raised:\\n\\n\\t<warning>%s</>\",\n                refspec.key,\n                e,\n            )\n\n            raise PoetryConsoleError(\n                f\"Failed to clone {url} at '{refspec.key}', verify ref exists on\"\n                \" remote.\"\n            )\n\n        return local\n\n    @classmethod\n    def _clone_submodules(cls, repo: Repo) -> None:\n        \"\"\"\n        Helper method to identify configured submodules and clone them recursively.\n        \"\"\"\n        repo_root = Path(repo.path)\n        for submodule in cls._get_submodules(repo):\n            path_absolute = repo_root / submodule.path\n            source_root = path_absolute.parent\n            source_root.mkdir(parents=True, exist_ok=True)\n            cls.clone(\n                url=submodule.url,\n                source_root=source_root,\n                name=path_absolute.name,\n                revision=submodule.revision,\n                clean=path_absolute.exists()\n                and not path_absolute.joinpath(\".git\").is_dir(),\n            )\n\n    @classmethod\n    def _get_submodules(cls, repo: Repo) -> list[SubmoduleInfo]:\n        modules_config = Path(repo.path, \".gitmodules\")\n\n        if not modules_config.exists():\n            return []\n\n        config = ConfigFile.from_path(str(modules_config))\n\n        submodules: list[SubmoduleInfo] = []\n        for path, url, name in parse_submodules(config):\n            url_str = url.decode(\"utf-8\")\n            path_str = path.decode(\"utf-8\")\n            name_str = name.decode(\"utf-8\")\n\n            if RELATIVE_SUBMODULE_REGEX.search(url_str):\n                url_str = urlpathjoin(f\"{cls.get_remote_url(repo)}/\", url_str)\n\n            with repo:\n                index = repo.open_index()\n\n                try:\n                    entry = index[path]\n                except KeyError:\n                    logger.debug(\n                        \"Skip submodule %s in %s, path %s not found\",\n                        name,\n                        repo.path,\n                        path,\n                    )\n                    continue\n\n                assert isinstance(entry, IndexEntry)\n                revision = entry.sha.decode(\"utf-8\")\n\n            submodules.append(\n                SubmoduleInfo(\n                    path=path_str,\n                    url=url_str,\n                    name=name_str,\n                    revision=revision,\n                )\n            )\n\n        return submodules\n\n    @staticmethod\n    def is_using_legacy_client() -> bool:\n        from poetry.config.config import Config\n\n        legacy_client: bool = Config.create().get(\n            \"experimental.system-git-client\", False\n        )\n        return legacy_client\n\n    @staticmethod\n    def get_default_source_root() -> Path:\n        from poetry.config.config import Config\n\n        return Path(Config.create().get(\"cache-dir\")) / \"src\"\n\n    @classmethod\n    def clone(\n        cls,\n        url: str,\n        name: str | None = None,\n        branch: str | None = None,\n        tag: str | None = None,\n        revision: str | None = None,\n        source_root: Path | None = None,\n        clean: bool = False,\n    ) -> Repo:\n        source_root = source_root or cls.get_default_source_root()\n        source_root.mkdir(parents=True, exist_ok=True)\n\n        name = name or cls.get_name_from_source_url(url=url)\n        target = source_root / name\n        refspec = GitRefSpec(branch=branch, revision=revision, tag=tag)\n\n        if target.exists():\n            if clean:\n                # force clean the local copy if it exists, do not reuse\n                remove_directory(target, force=True)\n            else:\n                # check if the current local copy matches the requested ref spec\n                try:\n                    current_repo = Repo(str(target))\n\n                    with current_repo:\n                        current_sha = current_repo.head().decode(\"utf-8\")\n                except (NotGitRepository, AssertionError, KeyError):\n                    # something is wrong with the current checkout, clean it\n                    remove_directory(target, force=True)\n                else:\n                    if not is_revision_sha(revision=current_sha):\n                        # head is not a sha, this will cause issues later, lets reset\n                        remove_directory(target, force=True)\n                    elif (\n                        refspec.is_sha\n                        and refspec.revision is not None\n                        and current_sha.startswith(refspec.revision)\n                    ):\n                        # if revision is used short-circuit remote fetch head matches\n                        return current_repo\n\n        try:\n            if not cls.is_using_legacy_client():\n                local = cls._clone(url=url, refspec=refspec, target=target)\n                cls._clone_submodules(repo=local)\n                return local\n        except HTTPUnauthorized:\n            # we do this here to handle http authenticated repositories as dulwich\n            # does not currently support using credentials from git-credential helpers.\n            # upstream issue: https://github.com/jelmer/dulwich/issues/873\n            #\n            # this is a little inefficient, however preferred as this is transparent\n            # without additional configuration or changes for existing projects that\n            # use http basic auth credentials.\n            logger.debug(\n                \"Unable to fetch from private repository '%s', falling back to\"\n                \" system git\",\n                url,\n            )\n\n        # fallback to legacy git client\n        return cls._clone_legacy(url=url, refspec=refspec, target=target)\n\n\ndef urlpathjoin(base: str, path: str) -> str:\n    \"\"\"\n    Allow any URL to be joined with a path\n\n    This works around an issue with urllib.parse.urljoin where it only handles\n    relative URLs for protocols contained in urllib.parse.uses_relative. As it\n    happens common protocols used with git, like ssh or git+ssh are not in that\n    list.\n\n    Thus we need to implement our own version of urljoin that handles all URLs\n    protocols. This is accomplished by using urlparse and urlunparse to split\n    the URL into its components, join the path, and then reassemble the URL.\n\n    See: https://github.com/python-poetry/poetry/issues/6499#issuecomment-1564712609\n    \"\"\"\n    parsed_base = urlparse(base)\n    new = parsed_base._replace(path=urljoin(parsed_base.path, path))\n    return urlunparse(new)\n\n\n@dataclasses.dataclass\nclass SubmoduleInfo:\n    path: str\n    url: str\n    name: str\n    revision: str\n\n# Path: src/poetry/vcs/git/system.py\nfrom __future__ import annotations\n\nimport os\nimport subprocess\n\nfrom typing import TYPE_CHECKING\n\nfrom dulwich.client import find_git_command\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n    from typing import Any\n\n\nclass SystemGit:\n    @classmethod\n    def clone(cls, repository: str, dest: Path) -> None:\n        cls._check_parameter(repository)\n\n        cls.run(\"clone\", \"--recurse-submodules\", \"--\", repository, str(dest))\n\n    @classmethod\n    def checkout(cls, rev: str, target: Path | None = None) -> None:\n        cls._check_parameter(rev)\n        cls.run(\"checkout\", rev, folder=target)\n\n    @staticmethod\n    def run(*args: Any, **kwargs: Any) -> None:\n        folder = kwargs.pop(\"folder\", None)\n        if folder:\n            args = (\n                \"--git-dir\",\n                (folder / \".git\").as_posix(),\n                \"--work-tree\",\n                folder.as_posix(),\n                *args,\n            )\n\n        git_command = find_git_command()\n        env = os.environ.copy()\n        env[\"GIT_TERMINAL_PROMPT\"] = \"0\"\n        subprocess.check_call(\n            git_command + list(args),\n            stderr=subprocess.DEVNULL,\n            stdout=subprocess.DEVNULL,\n            env=env,\n            text=True,\n        )\n\n    @staticmethod\n    def _check_parameter(parameter: str) -> None:\n        \"\"\"\n        Checks a git parameter to avoid unwanted code execution.\n        \"\"\"\n        if parameter.strip().startswith(\"-\"):\n            raise RuntimeError(f\"Invalid Git parameter: {parameter}\")\n\n# Path: src/poetry/console/commands/cache/__init__.py\n\n# Path: src/poetry/console/commands/cache/clear.py\nfrom __future__ import annotations\n\nimport os\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\n\nfrom poetry.config.config import Config\nfrom poetry.console.commands.command import Command\nfrom poetry.utils.cache import FileCache\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass CacheClearCommand(Command):\n    name = \"cache clear\"\n    description = \"Clears a Poetry cache by name.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"cache\", description=\"The name of the cache to clear.\")\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\"all\", description=\"Clear all entries in the cache.\")\n    ]\n\n    def handle(self) -> int:\n        cache = self.argument(\"cache\")\n\n        parts = cache.split(\":\")\n        root = parts[0]\n\n        config = Config.create()\n        cache_dir = config.repository_cache_directory / root\n\n        try:\n            cache_dir.relative_to(config.repository_cache_directory)\n        except ValueError:\n            raise ValueError(f\"{root} is not a valid repository cache\")\n\n        cache = FileCache(cache_dir)\n\n        if len(parts) == 1:\n            if not self.option(\"all\"):\n                raise RuntimeError(\n                    f\"Add the --all option if you want to clear all {parts[0]} caches\"\n                )\n\n            if not cache_dir.exists():\n                self.line(f\"No cache entries for {parts[0]}\")\n                return 0\n\n            # Calculate number of entries\n            entries_count = sum(\n                len(files) for _path, _dirs, files in os.walk(str(cache_dir))\n            )\n\n            delete = self.confirm(f\"<question>Delete {entries_count} entries?</>\", True)\n            if not delete:\n                return 0\n\n            cache.flush()\n        elif len(parts) == 2:\n            raise RuntimeError(\n                \"Only specifying the package name is not yet supported. \"\n                \"Add a specific version to clear\"\n            )\n        elif len(parts) == 3:\n            package = canonicalize_name(parts[1])\n            version = parts[2]\n\n            if not cache.has(f\"{package}:{version}\"):\n                self.line(f\"No cache entries for {package}:{version}\")\n                return 0\n\n            delete = self.confirm(f\"Delete cache entry {package}:{version}\", True)\n            if not delete:\n                return 0\n\n            cache.forget(f\"{package}:{version}\")\n        else:\n            raise ValueError(\"Invalid cache key\")\n\n        return 0\n\n# Path: src/poetry/console/commands/cache/list.py\nfrom __future__ import annotations\n\nfrom poetry.config.config import Config\nfrom poetry.console.commands.command import Command\n\n\nclass CacheListCommand(Command):\n    name = \"cache list\"\n    description = \"List Poetry's caches.\"\n\n    def handle(self) -> int:\n        config = Config.create()\n        if config.repository_cache_directory.exists():\n            caches = sorted(config.repository_cache_directory.iterdir())\n            if caches:\n                for cache in caches:\n                    self.line(f\"<info>{cache.name}</>\")\n                return 0\n\n        self.line_error(\"<warning>No caches found</>\")\n        return 0\n\n# Path: src/poetry/console/commands/debug/__init__.py\n\n# Path: src/poetry/console/commands/debug/info.py\nfrom __future__ import annotations\n\nimport sys\n\nfrom poetry.console.commands.command import Command\n\n\nclass DebugInfoCommand(Command):\n    name = \"debug info\"\n    description = \"Shows debug information.\"\n\n    def handle(self) -> int:\n        poetry_python_version = \".\".join(str(s) for s in sys.version_info[:3])\n\n        self.line(\"\")\n        self.line(\"<b>Poetry</b>\")\n        self.line(\n            \"\\n\".join(\n                [\n                    f\"<info>Version</info>: <comment>{self.poetry.VERSION}</>\",\n                    f\"<info>Python</info>:  <comment>{poetry_python_version}</>\",\n                ]\n            )\n        )\n        command = self.get_application().get(\"env info\")\n\n        exit_code: int = command.run(self.io)\n        return exit_code\n\n# Path: src/poetry/console/commands/debug/resolve.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n...\n", "needle_token_start": 5574, "needle_token_end": 5592, "code_context_ntokens": 15996, "output": ["```python\ndef candidates(self) -> list[Path]:\n    return self._candidates\n\n@cached_property\ndef writable_candidates(self) -> list[Path]:\n    if self._writable_candidates is not None:\n        return self._writable_candidates\n\n    self._writable_candidates = []\n    for candidate in self._candidates:\n        if not is_dir_writable(path=candidate, create=True):\n            continue\n        self._writable_candidates.append(candidate)\n\n    return self._writable_candidates\n```"]}
{"repo": "python-poetry/poetry", "name": "_display_complete_info", "language": "python", "path": "src/poetry/console/commands/env/info.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to display detailed information about the current Python environment and its base environment, including versions, paths, and other relevant data.\n2. **Input**: It accepts an environment object that encapsulates details about a Python environment.\n3. **Output**: There is no return value; instead, it outputs formatted text directly to the console, presenting various attributes of the environment and its base.\n4. **Procedure**: The function first extracts and formats details such as the Python version, implementation, and paths from the provided environment object. It checks if the environment is a virtual environment and displays additional validity information if true. It then retrieves and displays similar information for the base environment associated with the provided environment.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/vcs/git/backend.py\nfrom __future__ import annotations\n\nimport dataclasses\nimport logging\nimport re\n\nfrom pathlib import Path\nfrom subprocess import CalledProcessError\nfrom typing import TYPE_CHECKING\nfrom urllib.parse import urljoin\nfrom urllib.parse import urlparse\nfrom urllib.parse import urlunparse\n\nfrom dulwich import porcelain\nfrom dulwich.client import HTTPUnauthorized\nfrom dulwich.client import get_transport_and_path\nfrom dulwich.config import ConfigFile\nfrom dulwich.config import parse_submodules\nfrom dulwich.errors import NotGitRepository\nfrom dulwich.index import IndexEntry\nfrom dulwich.refs import ANNOTATED_TAG_SUFFIX\nfrom dulwich.repo import Repo\n\nfrom poetry.console.exceptions import PoetryConsoleError\nfrom poetry.utils.authenticator import get_default_authenticator\nfrom poetry.utils.helpers import remove_directory\n\n\nif TYPE_CHECKING:\n    from dulwich.client import FetchPackResult\n    from dulwich.client import GitClient\n\n\nlogger = logging.getLogger(__name__)\n\n# A relative URL by definition starts with ../ or ./\nRELATIVE_SUBMODULE_REGEX = re.compile(r\"^\\.{1,2}/\")\n\n\ndef is_revision_sha(revision: str | None) -> bool:\n    return re.match(r\"^\\b[0-9a-f]{5,40}\\b$\", revision or \"\") is not None\n\n\ndef annotated_tag(ref: str | bytes) -> bytes:\n    if isinstance(ref, str):\n        ref = ref.encode(\"utf-8\")\n    return ref + ANNOTATED_TAG_SUFFIX\n\n\n@dataclasses.dataclass\nclass GitRefSpec:\n    branch: str | None = None\n    revision: str | None = None\n    tag: str | None = None\n    ref: bytes = dataclasses.field(default_factory=lambda: b\"HEAD\")\n\n    def resolve(self, remote_refs: FetchPackResult) -> None:\n        \"\"\"\n        Resolve the ref using the provided remote refs.\n        \"\"\"\n        self._normalise(remote_refs=remote_refs)\n        self._set_head(remote_refs=remote_refs)\n\n    def _normalise(self, remote_refs: FetchPackResult) -> None:\n        \"\"\"\n        Internal helper method to determine if given revision is\n            1. a branch or tag; if so, set corresponding properties.\n            2. a short sha; if so, resolve full sha and set as revision\n        \"\"\"\n        if self.revision:\n            ref = f\"refs/tags/{self.revision}\".encode()\n            if ref in remote_refs.refs or annotated_tag(ref) in remote_refs.refs:\n                # this is a tag, incorrectly specified as a revision, tags take priority\n                self.tag = self.revision\n                self.revision = None\n            elif (\n                self.revision.encode(\"utf-8\") in remote_refs.refs\n                or f\"refs/heads/{self.revision}\".encode() in remote_refs.refs\n            ):\n                # this is most likely a ref spec or a branch incorrectly specified\n                self.branch = self.revision\n                self.revision = None\n        elif (\n            self.branch\n            and f\"refs/heads/{self.branch}\".encode() not in remote_refs.refs\n            and (\n                f\"refs/tags/{self.branch}\".encode() in remote_refs.refs\n                or annotated_tag(f\"refs/tags/{self.branch}\") in remote_refs.refs\n            )\n        ):\n            # this is a tag incorrectly specified as a branch\n            self.tag = self.branch\n            self.branch = None\n\n        if self.revision and self.is_sha_short:\n            # revision is a short sha, resolve to full sha\n            short_sha = self.revision.encode(\"utf-8\")\n            for sha in remote_refs.refs.values():\n                if sha.startswith(short_sha):\n                    self.revision = sha.decode(\"utf-8\")\n                    break\n\n    def _set_head(self, remote_refs: FetchPackResult) -> None:\n        \"\"\"\n        Internal helper method to populate ref and set it's sha as the remote's head\n        and default ref.\n        \"\"\"\n        self.ref = remote_refs.symrefs[b\"HEAD\"]\n\n        if self.revision:\n            head = self.revision.encode(\"utf-8\")\n        else:\n            if self.tag:\n                ref = f\"refs/tags/{self.tag}\".encode()\n                annotated = annotated_tag(ref)\n                self.ref = annotated if annotated in remote_refs.refs else ref\n            elif self.branch:\n                self.ref = (\n                    self.branch.encode(\"utf-8\")\n                    if self.is_ref\n                    else f\"refs/heads/{self.branch}\".encode()\n                )\n            head = remote_refs.refs[self.ref]\n\n        remote_refs.refs[self.ref] = remote_refs.refs[b\"HEAD\"] = head\n\n    @property\n    def key(self) -> str:\n        return self.revision or self.branch or self.tag or self.ref.decode(\"utf-8\")\n\n    @property\n    def is_sha(self) -> bool:\n        return is_revision_sha(revision=self.revision)\n\n    @property\n    def is_ref(self) -> bool:\n        return self.branch is not None and (\n            self.branch.startswith(\"refs/\") or self.branch == \"HEAD\"\n        )\n\n    @property\n    def is_sha_short(self) -> bool:\n        return self.revision is not None and self.is_sha and len(self.revision) < 40\n\n\n@dataclasses.dataclass\nclass GitRepoLocalInfo:\n    repo: dataclasses.InitVar[Repo | Path]\n    origin: str = dataclasses.field(init=False)\n    revision: str = dataclasses.field(init=False)\n\n    def __post_init__(self, repo: Repo | Path) -> None:\n        repo = Git.as_repo(repo=repo) if not isinstance(repo, Repo) else repo\n        self.origin = Git.get_remote_url(repo=repo, remote=\"origin\")\n        self.revision = Git.get_revision(repo=repo)\n\n\nclass Git:\n    @staticmethod\n    def as_repo(repo: Path) -> Repo:\n        return Repo(str(repo))\n\n    @staticmethod\n    def get_remote_url(repo: Repo, remote: str = \"origin\") -> str:\n        with repo:\n            config = repo.get_config()\n            section = (b\"remote\", remote.encode(\"utf-8\"))\n\n            url = \"\"\n            if config.has_section(section):\n                value = config.get(section, b\"url\")\n                url = value.decode(\"utf-8\")\n\n            return url\n\n    @staticmethod\n    def get_revision(repo: Repo) -> str:\n        with repo:\n            return repo.head().decode(\"utf-8\")\n\n    @classmethod\n    def info(cls, repo: Repo | Path) -> GitRepoLocalInfo:\n        return GitRepoLocalInfo(repo=repo)\n\n    @staticmethod\n    def get_name_from_source_url(url: str) -> str:\n        return re.sub(r\"(.git)?$\", \"\", url.rsplit(\"/\", 1)[-1])\n\n    @classmethod\n    def _fetch_remote_refs(cls, url: str, local: Repo) -> FetchPackResult:\n        \"\"\"\n        Helper method to fetch remote refs.\n        \"\"\"\n        client: GitClient\n        path: str\n\n        kwargs: dict[str, str] = {}\n        credentials = get_default_authenticator().get_credentials_for_git_url(url=url)\n\n        if credentials.password and credentials.username:\n            # we do this conditionally as otherwise, dulwich might complain if these\n            # parameters are passed in for an ssh url\n            kwargs[\"username\"] = credentials.username\n            kwargs[\"password\"] = credentials.password\n\n        config = local.get_config_stack()\n        client, path = get_transport_and_path(url, config=config, **kwargs)\n\n        with local:\n            result: FetchPackResult = client.fetch(\n                path,\n                local,\n                determine_wants=local.object_store.determine_wants_all,\n            )\n            return result\n\n    @staticmethod\n    def _clone_legacy(url: str, refspec: GitRefSpec, target: Path) -> Repo:\n        \"\"\"\n        Helper method to facilitate fallback to using system provided git client via\n        subprocess calls.\n        \"\"\"\n        from poetry.vcs.git.system import SystemGit\n\n        logger.debug(\"Cloning '%s' using system git client\", url)\n\n        if target.exists():\n            remove_directory(path=target, force=True)\n\n        revision = refspec.tag or refspec.branch or refspec.revision or \"HEAD\"\n\n        try:\n            SystemGit.clone(url, target)\n        except CalledProcessError:\n            raise PoetryConsoleError(\n                f\"Failed to clone {url}, check your git configuration and permissions\"\n                \" for this repository.\"\n            )\n\n        if revision:\n            revision.replace(\"refs/head/\", \"\")\n            revision.replace(\"refs/tags/\", \"\")\n\n        try:\n            SystemGit.checkout(revision, target)\n        except CalledProcessError:\n            raise PoetryConsoleError(f\"Failed to checkout {url} at '{revision}'\")\n\n        repo = Repo(str(target))\n        return repo\n\n    @classmethod\n    def _clone(cls, url: str, refspec: GitRefSpec, target: Path) -> Repo:\n        \"\"\"\n        Helper method to clone a remove repository at the given `url` at the specified\n        ref spec.\n        \"\"\"\n        local: Repo\n        if not target.exists():\n            local = Repo.init(str(target), mkdir=True)\n            porcelain.remote_add(local, \"origin\", url)\n        else:\n            local = Repo(str(target))\n\n        remote_refs = cls._fetch_remote_refs(url=url, local=local)\n\n        logger.debug(\n            \"Cloning <c2>%s</> at '<c2>%s</>' to <c1>%s</>\", url, refspec.key, target\n        )\n\n        try:\n            refspec.resolve(remote_refs=remote_refs)\n        except KeyError:  # branch / ref does not exist\n            raise PoetryConsoleError(\n                f\"Failed to clone {url} at '{refspec.key}', verify ref exists on\"\n                \" remote.\"\n            )\n\n        # ensure local HEAD matches remote\n        local.refs[b\"HEAD\"] = remote_refs.refs[b\"HEAD\"]\n\n        if refspec.is_ref:\n            # set ref to current HEAD\n            local.refs[refspec.ref] = local.refs[b\"HEAD\"]\n\n        for base, prefix in {\n            (b\"refs/remotes/origin\", b\"refs/heads/\"),\n            (b\"refs/tags\", b\"refs/tags\"),\n        }:\n            local.refs.import_refs(\n                base=base,\n                other={\n                    n[len(prefix) :]: v\n                    for (n, v) in remote_refs.refs.items()\n                    if n.startswith(prefix) and not n.endswith(ANNOTATED_TAG_SUFFIX)\n                },\n            )\n\n        try:\n            with local:\n                local.reset_index()\n        except (AssertionError, KeyError) as e:\n            # this implies the ref we need does not exist or is invalid\n            if isinstance(e, KeyError):\n                # the local copy is at a bad state, lets remove it\n                logger.debug(\n                    \"Removing local clone (<c1>%s</>) of repository as it is in a\"\n                    \" broken state.\",\n                    local.path,\n                )\n                remove_directory(Path(local.path), force=True)\n\n            if isinstance(e, AssertionError) and \"Invalid object name\" not in str(e):\n                raise\n\n            logger.debug(\n                \"\\nRequested ref (<c2>%s</c2>) was not fetched to local copy and\"\n                \" cannot be used. The following error was\"\n                \" raised:\\n\\n\\t<warning>%s</>\",\n                refspec.key,\n                e,\n            )\n\n            raise PoetryConsoleError(\n                f\"Failed to clone {url} at '{refspec.key}', verify ref exists on\"\n                \" remote.\"\n            )\n\n        return local\n\n    @classmethod\n    def _clone_submodules(cls, repo: Repo) -> None:\n        \"\"\"\n        Helper method to identify configured submodules and clone them recursively.\n        \"\"\"\n        repo_root = Path(repo.path)\n        for submodule in cls._get_submodules(repo):\n            path_absolute = repo_root / submodule.path\n            source_root = path_absolute.parent\n            source_root.mkdir(parents=True, exist_ok=True)\n            cls.clone(\n                url=submodule.url,\n                source_root=source_root,\n                name=path_absolute.name,\n                revision=submodule.revision,\n                clean=path_absolute.exists()\n                and not path_absolute.joinpath(\".git\").is_dir(),\n            )\n\n    @classmethod\n    def _get_submodules(cls, repo: Repo) -> list[SubmoduleInfo]:\n        modules_config = Path(repo.path, \".gitmodules\")\n\n        if not modules_config.exists():\n            return []\n\n        config = ConfigFile.from_path(str(modules_config))\n\n        submodules: list[SubmoduleInfo] = []\n        for path, url, name in parse_submodules(config):\n            url_str = url.decode(\"utf-8\")\n            path_str = path.decode(\"utf-8\")\n            name_str = name.decode(\"utf-8\")\n\n            if RELATIVE_SUBMODULE_REGEX.search(url_str):\n                url_str = urlpathjoin(f\"{cls.get_remote_url(repo)}/\", url_str)\n\n            with repo:\n                index = repo.open_index()\n...\n# Path: src/poetry/vcs/git/system.py\nfrom __future__ import annotations\n\nimport os\nimport subprocess\n\nfrom typing import TYPE_CHECKING\n\nfrom dulwich.client import find_git_command\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n    from typing import Any\n\n\nclass SystemGit:\n    @classmethod\n    def clone(cls, repository: str, dest: Path) -> None:\n        cls._check_parameter(repository)\n\n        cls.run(\"clone\", \"--recurse-submodules\", \"--\", repository, str(dest))\n\n    @classmethod\n    def checkout(cls, rev: str, target: Path | None = None) -> None:\n        cls._check_parameter(rev)\n        cls.run(\"checkout\", rev, folder=target)\n\n    @staticmethod\n    def run(*args: Any, **kwargs: Any) -> None:\n        folder = kwargs.pop(\"folder\", None)\n        if folder:\n            args = (\n                \"--git-dir\",\n                (folder / \".git\").as_posix(),\n                \"--work-tree\",\n                folder.as_posix(),\n                *args,\n            )\n\n        git_command = find_git_command()\n        env = os.environ.copy()\n        env[\"GIT_TERMINAL_PROMPT\"] = \"0\"\n        subprocess.check_call(\n            git_command + list(args),\n            stderr=subprocess.DEVNULL,\n            stdout=subprocess.DEVNULL,\n            env=env,\n            text=True,\n        )\n\n    @staticmethod\n    def _check_parameter(parameter: str) -> None:\n        \"\"\"\n        Checks a git parameter to avoid unwanted code execution.\n        \"\"\"\n        if parameter.strip().startswith(\"-\"):\n            raise RuntimeError(f\"Invalid Git parameter: {parameter}\")\n\n# Path: src/poetry/console/commands/cache/__init__.py\n\n# Path: src/poetry/console/commands/cache/clear.py\nfrom __future__ import annotations\n\nimport os\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\n\nfrom poetry.config.config import Config\nfrom poetry.console.commands.command import Command\nfrom poetry.utils.cache import FileCache\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass CacheClearCommand(Command):\n    name = \"cache clear\"\n    description = \"Clears a Poetry cache by name.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"cache\", description=\"The name of the cache to clear.\")\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\"all\", description=\"Clear all entries in the cache.\")\n    ]\n\n    def handle(self) -> int:\n        cache = self.argument(\"cache\")\n\n        parts = cache.split(\":\")\n        root = parts[0]\n\n        config = Config.create()\n        cache_dir = config.repository_cache_directory / root\n\n        try:\n            cache_dir.relative_to(config.repository_cache_directory)\n        except ValueError:\n            raise ValueError(f\"{root} is not a valid repository cache\")\n\n        cache = FileCache(cache_dir)\n\n        if len(parts) == 1:\n            if not self.option(\"all\"):\n                raise RuntimeError(\n                    f\"Add the --all option if you want to clear all {parts[0]} caches\"\n                )\n\n            if not cache_dir.exists():\n                self.line(f\"No cache entries for {parts[0]}\")\n                return 0\n\n            # Calculate number of entries\n            entries_count = sum(\n                len(files) for _path, _dirs, files in os.walk(str(cache_dir))\n            )\n\n            delete = self.confirm(f\"<question>Delete {entries_count} entries?</>\", True)\n            if not delete:\n                return 0\n\n            cache.flush()\n        elif len(parts) == 2:\n            raise RuntimeError(\n                \"Only specifying the package name is not yet supported. \"\n                \"Add a specific version to clear\"\n            )\n        elif len(parts) == 3:\n            package = canonicalize_name(parts[1])\n            version = parts[2]\n\n            if not cache.has(f\"{package}:{version}\"):\n                self.line(f\"No cache entries for {package}:{version}\")\n                return 0\n\n            delete = self.confirm(f\"Delete cache entry {package}:{version}\", True)\n            if not delete:\n                return 0\n\n            cache.forget(f\"{package}:{version}\")\n        else:\n            raise ValueError(\"Invalid cache key\")\n\n        return 0\n\n# Path: src/poetry/console/commands/cache/list.py\nfrom __future__ import annotations\n\nfrom poetry.config.config import Config\nfrom poetry.console.commands.command import Command\n\n\nclass CacheListCommand(Command):\n    name = \"cache list\"\n    description = \"List Poetry's caches.\"\n\n    def handle(self) -> int:\n        config = Config.create()\n        if config.repository_cache_directory.exists():\n            caches = sorted(config.repository_cache_directory.iterdir())\n            if caches:\n                for cache in caches:\n                    self.line(f\"<info>{cache.name}</>\")\n                return 0\n\n        self.line_error(\"<warning>No caches found</>\")\n        return 0\n\n# Path: src/poetry/console/commands/debug/__init__.py\n\n# Path: src/poetry/console/commands/debug/info.py\nfrom __future__ import annotations\n\nimport sys\n\nfrom poetry.console.commands.command import Command\n\n\nclass DebugInfoCommand(Command):\n    name = \"debug info\"\n    description = \"Shows debug information.\"\n\n    def handle(self) -> int:\n        poetry_python_version = \".\".join(str(s) for s in sys.version_info[:3])\n\n        self.line(\"\")\n        self.line(\"<b>Poetry</b>\")\n        self.line(\n            \"\\n\".join(\n                [\n                    f\"<info>Version</info>: <comment>{self.poetry.VERSION}</>\",\n                    f\"<info>Python</info>:  <comment>{poetry_python_version}</>\",\n                ]\n            )\n        )\n        command = self.get_application().get(\"env info\")\n\n        exit_code: int = command.run(self.io)\n        return exit_code\n\n# Path: src/poetry/console/commands/debug/resolve.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom cleo.io.outputs.output import Verbosity\n\nfrom poetry.console.commands.init import InitCommand\nfrom poetry.console.commands.show import ShowCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n    from cleo.ui.table import Rows\n\n\nclass DebugResolveCommand(InitCommand):\n    name = \"debug resolve\"\n    description = \"Debugs dependency resolution.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"package\", \"The packages to resolve.\", optional=True, multiple=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\n            \"extras\",\n            \"E\",\n            \"Extras to activate for the dependency.\",\n            flag=False,\n            multiple=True,\n        ),\n        option(\"python\", None, \"Python version(s) to use for resolution.\", flag=False),\n        option(\"tree\", None, \"Display the dependency tree.\"),\n        option(\"install\", None, \"Show what would be installed for the current system.\"),\n    ]\n\n    loggers: ClassVar[list[str]] = [\n        \"poetry.repositories.pypi_repository\",\n        \"poetry.inspection.info\",\n    ]\n\n    def handle(self) -> int:\n        from cleo.io.null_io import NullIO\n        from poetry.core.packages.project_package import ProjectPackage\n\n        from poetry.factory import Factory\n        from poetry.puzzle.solver import Solver\n        from poetry.repositories.repository import Repository\n        from poetry.repositories.repository_pool import RepositoryPool\n        from poetry.utils.env import EnvManager\n\n        packages = self.argument(\"package\")\n\n        if not packages:\n            package = self.poetry.package\n        else:\n            # Using current pool for determine_requirements()\n            self._pool = self.poetry.pool\n\n            package = ProjectPackage(\n                self.poetry.package.name, self.poetry.package.version\n            )\n\n            # Silencing output\n            verbosity = self.io.output.verbosity\n            self.io.output.set_verbosity(Verbosity.QUIET)\n\n            requirements = self._determine_requirements(packages)\n\n            self.io.output.set_verbosity(verbosity)\n\n            for constraint in requirements:\n                name = constraint.pop(\"name\")\n                assert isinstance(name, str)\n                extras = []\n                for extra in self.option(\"extras\"):\n                    extras += extra.split()\n\n                constraint[\"extras\"] = extras\n\n                package.add_dependency(Factory.create_dependency(name, constraint))\n\n        package.python_versions = self.option(\"python\") or (\n            self.poetry.package.python_versions\n        )\n\n        pool = self.poetry.pool\n\n        solver = Solver(package, pool, [], [], self.io)\n\n        ops = solver.solve().calculate_operations()\n\n        self.line(\"\")\n        self.line(\"Resolution results:\")\n        self.line(\"\")\n\n        if self.option(\"tree\"):\n            show_command = self.get_application().find(\"show\")\n            assert isinstance(show_command, ShowCommand)\n            show_command.init_styles(self.io)\n\n            packages = [op.package for op in ops]\n\n            requires = package.all_requires\n            for pkg in packages:\n                for require in requires:\n                    if pkg.name == require.name:\n                        show_command.display_package_tree(self.io, pkg, packages)\n                        break\n\n            return 0\n\n        table = self.table(style=\"compact\")\n        table.style.set_vertical_border_chars(\"\", \" \")\n        rows: Rows = []\n\n        if self.option(\"install\"):\n            env = EnvManager(self.poetry).get()\n            pool = RepositoryPool(config=self.poetry.config)\n            locked_repository = Repository(\"poetry-locked\")\n            for op in ops:\n                locked_repository.add_package(op.package)\n\n            pool.add_repository(locked_repository)\n\n            solver = Solver(package, pool, [], [], NullIO())\n            with solver.use_environment(env):\n                ops = solver.solve().calculate_operations()\n\n        for op in ops:\n            if self.option(\"install\") and op.skipped:\n                continue\n\n            pkg = op.package\n            row = [\n                f\"<c1>{pkg.complete_name}</c1>\",\n                f\"<b>{pkg.version}</b>\",\n            ]\n\n            if not pkg.marker.is_any():\n                row[2] = str(pkg.marker)\n\n            rows.append(row)\n\n        table.set_rows(rows)\n        table.render()\n\n        return 0\n\n# Path: src/poetry/console/commands/env/__init__.py\n\n# Path: src/poetry/console/commands/env/info.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n    from poetry.utils.env import Env\n\n\nclass EnvInfoCommand(Command):\n    name = \"env info\"\n    description = \"Displays information about the current environment.\"\n\n    options: ClassVar[list[Option]] = [\n        option(\"path\", \"p\", \"Only display the environment's path.\"),\n        option(\n            \"executable\", \"e\", \"Only display the environment's python executable path.\"\n        ),\n    ]\n\n    def handle(self) -> int:\n        from poetry.utils.env import EnvManager\n\n        env = EnvManager(self.poetry).get()\n\n        if self.option(\"path\"):\n            if not env.is_venv():\n                return 1\n\n            self.line(str(env.path))\n\n            return 0\n\n        if self.option(\"executable\"):\n            if not env.is_venv():\n                return 1\n\n            self.line(str(env.python))\n\n            return 0\n\n        self._display_complete_info(env)\n        return 0\n\n    \ndef _display_complete_info(self, env: Env) -> None:\n        env_python_version = \".\".join(str(s) for s in env.version_info[:3])\n        self.line(\"\")\n        self.line(\"<b>Virtualenv</b>\")\n        listing = [\n            f\"<info>Python</info>:         <comment>{env_python_version}</>\",\n            f\"<info>Implementation</info>: <comment>{env.python_implementation}</>\",\n            (\n                \"<info>Path</info>:          \"\n                f\" <comment>{env.path if env.is_venv() else 'NA'}</>\"\n            ),\n            (\n                \"<info>Executable</info>:    \"\n                f\" <comment>{env.python if env.is_venv() else 'NA'}</>\"\n            ),\n        ]\n        if env.is_venv():\n            listing.append(\n                \"<info>Valid</info>:         \"\n                f\" <{'comment' if env.is_sane() else 'error'}>{env.is_sane()}</>\"\n            )\n        self.line(\"\\n\".join(listing))\n\n        self.line(\"\")\n\n        base_env = env.parent_env\n        python = \".\".join(str(v) for v in base_env.version_info[:3])\n        self.line(\"<b>Base</b>\")\n        self.line(\n            \"\\n\".join(\n                [\n                    f\"<info>Platform</info>:   <comment>{env.platform}</>\",\n                    f\"<info>OS</info>:         <comment>{env.os}</>\",\n                    f\"<info>Python</info>:     <comment>{python}</>\",\n                    f\"<info>Path</info>:       <comment>{base_env.path}</>\",\n                    f\"<info>Executable</info>: <comment>{base_env.python}</>\",\n                ]\n            )\n        )\n\n# Path: src/poetry/console/commands/env/list.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass EnvListCommand(Command):\n    name = \"env list\"\n    description = \"Lists all virtualenvs associated with the current project.\"\n\n    options: ClassVar[list[Option]] = [\n        option(\"full-path\", None, \"Output the full paths of the virtualenvs.\")\n    ]\n\n    def handle(self) -> int:\n        from poetry.utils.env import EnvManager\n\n        manager = EnvManager(self.poetry)\n        current_env = manager.get()\n\n        for venv in manager.list():\n            name = venv.path.name\n            if self.option(\"full-path\"):\n                name = str(venv.path)\n\n            if venv == current_env:\n                self.line(f\"<info>{name} (Activated)</info>\")\n\n                continue\n\n            self.line(name)\n\n        return 0\n\n# Path: src/poetry/console/commands/env/remove.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass EnvRemoveCommand(Command):\n    name = \"env remove\"\n    description = \"Remove virtual environments associated with the project.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\n            \"python\",\n            \"The python executables associated with, or names of the virtual\"\n            \" environments which are to be removed.\",\n            optional=True,\n            multiple=True,\n        )\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\n            \"all\",\n            description=(\n                \"Remove all managed virtual environments associated with the project.\"\n            ),\n        ),\n    ]\n\n    def handle(self) -> int:\n        from poetry.utils.env import EnvManager\n\n        pythons = self.argument(\"python\")\n        all = self.option(\"all\")\n        if not (pythons or all):\n            self.line(\"No virtualenv provided.\")\n\n        manager = EnvManager(self.poetry)\n        # TODO: refactor env.py to allow removal with one loop\n        for python in pythons:\n            venv = manager.remove(python)\n            self.line(f\"Deleted virtualenv: <comment>{venv.path}</comment>\")\n        if all:\n            for venv in manager.list():\n                manager.remove_venv(venv.path)\n                self.line(f\"Deleted virtualenv: <comment>{venv.path}</comment>\")\n            # Since we remove all the virtualenvs, we can also remove the entry\n            # in the envs file. (Strictly speaking, we should do this explicitly,\n            # in case it points to a virtualenv that had been removed manually before.)\n            if manager.envs_file.exists():\n                manager.envs_file.remove_section(manager.base_env_name)\n\n        return 0\n\n# Path: src/poetry/console/commands/env/use.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n\n\nclass EnvUseCommand(Command):\n    name = \"env use\"\n    description = \"Activates or creates a new virtualenv for the current project.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"python\", \"The python executable to use.\")\n    ]\n\n    def handle(self) -> int:\n        from poetry.utils.env import EnvManager\n\n        manager = EnvManager(self.poetry, io=self.io)\n\n        if self.argument(\"python\") == \"system\":\n            manager.deactivate()\n\n            return 0\n\n        env = manager.activate(self.argument(\"python\"))\n\n        self.line(f\"Using virtualenv: <comment>{env.path}</>\")\n\n        return 0\n\n# Path: src/poetry/console/commands/self/__init__.py\n\n# Path: src/poetry/console/commands/self/add.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom poetry.core.constraints.version import Version\n\nfrom poetry.__version__ import __version__\nfrom poetry.console.commands.add import AddCommand\nfrom poetry.console.commands.self.self_command import SelfCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass SelfAddCommand(SelfCommand, AddCommand):\n    name = \"self add\"\n    description = \"Add additional packages to Poetry's runtime environment.\"\n    options: ClassVar[list[Option]] = [\n        o\n        for o in AddCommand.options\n        if o.name in {\"editable\", \"extras\", \"source\", \"dry-run\", \"allow-prereleases\"}\n    ]\n    help = f\"\"\"\\\nThe <c1>self add</c1> command installs additional packages to Poetry's runtime \\\nenvironment.\n\nThis is managed in the <comment>{SelfCommand.get_default_system_pyproject_file()}</> \\\nfile.\n\n{AddCommand.examples}\n\"\"\"\n\n    @property\n    def _hint_update_packages(self) -> str:\n        version = Version.parse(__version__)\n        flags = \"\"\n\n        if not version.is_stable():\n            flags = \" --preview\"\n\n        return (\n            \"\\nIf you want to update it to the latest compatible version, you can use\"\n            f\" `poetry self update{flags}`.\\nIf you prefer to upgrade it to the latest\"\n            \" available version, you can use `poetry self add package@latest`.\\n\"\n        )\n\n# Path: src/poetry/console/commands/self/install.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom poetry.core.packages.dependency_group import MAIN_GROUP\n\nfrom poetry.console.commands.install import InstallCommand\nfrom poetry.console.commands.self.self_command import SelfCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass SelfInstallCommand(SelfCommand, InstallCommand):\n    name = \"self install\"\n    description = (\n        \"Install locked packages (incl. addons) required by this Poetry installation.\"\n    )\n    options: ClassVar[list[Option]] = [\n        o for o in InstallCommand.options if o.name in {\"sync\", \"dry-run\"}\n    ]\n    help = f\"\"\"\\\nThe <c1>self install</c1> command ensures all additional packages specified are \\\ninstalled in the current runtime environment.\n\nThis is managed in the <comment>{SelfCommand.get_default_system_pyproject_file()}</> \\\nfile.\n\nYou can add more packages using the <c1>self add</c1> command and remove them using \\\nthe <c1>self remove</c1> command.\n\"\"\"\n\n    @property\n    def activated_groups(self) -> set[str]:\n        return {MAIN_GROUP, self.default_group}\n\n# Path: src/poetry/console/commands/self/lock.py\nfrom __future__ import annotations\n\nfrom poetry.console.commands.lock import LockCommand\nfrom poetry.console.commands.self.self_command import SelfCommand\n\n\nclass SelfLockCommand(SelfCommand, LockCommand):\n    name = \"self lock\"\n    description = \"Lock the Poetry installation's system requirements.\"\n    help = f\"\"\"\\\nThe <c1>self lock</c1> command reads this Poetry installation's system requirements as \\\nspecified in the <comment>{SelfCommand.get_default_system_pyproject_file()}</> file.\n\nThe system dependencies are locked in the <comment>\\\n{SelfCommand.get_default_system_pyproject_file().parent.joinpath(\"poetry.lock\")}</> \\\nfile.\n\"\"\"\n\n# Path: src/poetry/console/commands/self/remove.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom poetry.console.commands.remove import RemoveCommand\nfrom poetry.console.commands.self.self_command import SelfCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass SelfRemoveCommand(SelfCommand, RemoveCommand):\n    name = \"self remove\"\n    description = \"Remove additional packages from Poetry's runtime environment.\"\n    options: ClassVar[list[Option]] = [\n        o for o in RemoveCommand.options if o.name in {\"dry-run\"}\n    ]\n    help = f\"\"\"\\\nThe <c1>self remove</c1> command removes additional package's to Poetry's runtime \\\nenvironment.\n\nThis is managed in the <comment>{SelfCommand.get_default_system_pyproject_file()}</> \\\nfile.\n\"\"\"\n\n# Path: src/poetry/console/commands/self/self_command.py\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\n\nfrom poetry.core.packages.dependency import Dependency\nfrom poetry.core.packages.project_package import ProjectPackage\n\nfrom poetry.__version__ import __version__\nfrom poetry.console.commands.installer_command import InstallerCommand\nfrom poetry.factory import Factory\nfrom poetry.pyproject.toml import PyProjectTOML\nfrom poetry.utils.env import EnvManager\nfrom poetry.utils.env import SystemEnv\nfrom poetry.utils.helpers import directory\n\n\nif TYPE_CHECKING:\n    from poetry.poetry import Poetry\n    from poetry.utils.env import Env\n\n\nclass SelfCommand(InstallerCommand):\n    ADDITIONAL_PACKAGE_GROUP = \"additional\"\n\n    @staticmethod\n    def get_default_system_pyproject_file() -> Path:\n        # We separate this out to avoid unwanted side effect during testing while\n        # maintaining dynamic use in help text.\n        #\n        # This is not ideal, but is the simplest solution for now.\n        from poetry.locations import CONFIG_DIR\n\n        return Path(CONFIG_DIR).joinpath(\"pyproject.toml\")\n\n    @property\n    def system_pyproject(self) -> Path:\n        file = self.get_default_system_pyproject_file()\n        file.parent.mkdir(parents=True, exist_ok=True)\n        return file\n\n    def reset_env(self) -> None:\n        self._env = EnvManager.get_system_env(naive=True)\n\n    @property\n    def env(self) -> Env:\n        if not isinstance(self._env, SystemEnv):\n            self.reset_env()\n        assert self._env is not None\n        return self._env\n\n    @property\n    def default_group(self) -> str:\n        return self.ADDITIONAL_PACKAGE_GROUP\n\n    @property\n    def activated_groups(self) -> set[str]:\n        return {self.default_group}\n\n    def generate_system_pyproject(self) -> None:\n        preserved = {}\n\n        if self.system_pyproject.exists():\n            content = PyProjectTOML(self.system_pyproject).poetry_config\n\n            for key in {\"group\", \"source\"}:\n                if key in content:\n                    preserved[key] = content[key]\n\n        package = ProjectPackage(name=\"poetry-instance\", version=__version__)\n        package.add_dependency(Dependency(name=\"poetry\", constraint=f\"{__version__}\"))\n\n        package.python_versions = \".\".join(str(v) for v in self.env.version_info[:3])\n\n        content = Factory.create_pyproject_from_package(package=package)\n\n        for key in preserved:\n            content[\"tool\"][\"poetry\"][key] = preserved[key]  # type: ignore[index]\n\n        pyproject = PyProjectTOML(self.system_pyproject)\n        pyproject.file.write(content)\n\n    def reset_poetry(self) -> None:\n        with directory(self.system_pyproject.parent):\n            self.generate_system_pyproject()\n            self._poetry = Factory().create_poetry(\n                self.system_pyproject.parent, io=self.io, disable_plugins=True\n            )\n\n    @property\n    def poetry(self) -> Poetry:\n        if self._poetry is None:\n            self.reset_poetry()\n\n        assert self._poetry is not None\n        return self._poetry\n\n    def _system_project_handle(self) -> int:\n        \"\"\"\n        This is a helper method that by default calls the handle method implemented in\n        the child class's next MRO sibling. Override this if you want special handling\n        either before calling the handle() from the super class or have custom logic\n        to handle the command.\n\n        The default implementations handles cases where a `self` command delegates\n        handling to an existing command. Eg: `SelfAddCommand(SelfCommand, AddCommand)`.\n        \"\"\"\n        return_code: int = super().handle()\n        return return_code\n\n    def reset(self) -> None:\n        \"\"\"\n        Reset current command instance's environment and poetry instances to ensure\n        use of the system specific ones.\n        \"\"\"\n        self.reset_env()\n        self.reset_poetry()\n\n    def handle(self) -> int:\n        # We override the base class's handle() method to ensure that poetry and env\n        # are reset to work within the system project instead of current context.\n        # Further, during execution, the working directory is temporarily changed\n        # to parent directory of Poetry system pyproject.toml file.\n        #\n        # This method **should not** be overridden in child classes as it may have\n        # unexpected consequences.\n\n        self.reset()\n\n        with directory(self.system_pyproject.parent):\n            return self._system_project_handle()\n\n# Path: src/poetry/console/commands/self/update.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom cleo.io.inputs.string_input import StringInput\nfrom cleo.io.io import IO\n\nfrom poetry.console.commands.add import AddCommand\nfrom poetry.console.commands.self.self_command import SelfCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass SelfUpdateCommand(SelfCommand):\n    name = \"self update\"\n    description = \"Updates Poetry to the latest version.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\n            \"version\", \"The version to update to.\", optional=True, default=\"latest\"\n        )\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\"preview\", None, \"Allow the installation of pre-release versions.\"),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything \"\n            \"(implicitly enables --verbose).\",\n        ),\n    ]\n    help = \"\"\"\\\nThe <c1>self update</c1> command updates Poetry version in its current runtime \\\nenvironment.\n\"\"\"\n\n    def _system_project_handle(self) -> int:\n        self.write(\"<info>Updating Poetry version ...</info>\\n\\n\")\n        application = self.get_application()\n        add_command = application.find(\"add\")\n        assert isinstance(add_command, AddCommand)\n        add_command.set_env(self.env)\n        application.configure_installer_for_command(add_command, self.io)\n\n        argv = [\"add\", f\"poetry@{self.argument('version')}\"]\n\n        if self.option(\"dry-run\"):\n            argv.append(\"--dry-run\")\n\n        if self.option(\"preview\"):\n            argv.append(\"--allow-prereleases\")\n\n        exit_code: int = add_command.run(\n            IO(\n                StringInput(\" \".join(argv)),\n                self.io.output,\n                self.io.error_output,\n            )\n        )\n        return exit_code\n\n# Path: src/poetry/console/commands/source/__init__.py\n\n# Path: src/poetry/console/commands/source/add.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom cleo.io.null_io import NullIO\nfrom tomlkit.items import AoT\n\nfrom poetry.config.source import Source\nfrom poetry.console.commands.command import Command\nfrom poetry.repositories.repository_pool import Priority\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass SourceAddCommand(Command):\n    name = \"source add\"\n    description = \"Add source configuration for project.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\n            \"name\",\n            \"Source repository name.\",\n        ),\n        argument(\n            \"url\",\n            \"Source repository URL.\"\n            \" Required, except for PyPI, for which it is not allowed.\",\n            optional=True,\n        ),\n    ]\n\n    options: ClassVar[list[Option]] = [\n        option(\n            \"default\",\n            \"d\",\n            \"Set this source as the default (disable PyPI). A \"\n            \"default source will also be the fallback source if \"\n            \"you add other sources. (<warning>Deprecated</warning>, use --priority)\",\n        ),\n        option(\n            \"secondary\",\n            \"s\",\n            \"Set this source as secondary. (<warning>Deprecated</warning>, use\"\n            \" --priority)\",\n        ),\n        option(\n            \"priority\",\n            \"p\",\n            \"Set the priority of this source. One of:\"\n            f\" {', '.join(p.name.lower() for p in Priority)}. Defaults to\"\n            f\" {Priority.PRIMARY.name.lower()}.\",\n            flag=False,\n        ),\n    ]\n\n    def handle(self) -> int:\n        from poetry.factory import Factory\n        from poetry.utils.source import source_to_table\n\n        name: str = self.argument(\"name\")\n        lower_name = name.lower()\n        url: str = self.argument(\"url\")\n        is_default: bool = self.option(\"default\", False)\n        is_secondary: bool = self.option(\"secondary\", False)\n        priority_str: str | None = self.option(\"priority\", None)\n\n        if lower_name == \"pypi\":\n            name = \"PyPI\"\n            if url:\n                self.line_error(\n                    \"<error>The URL of PyPI is fixed and cannot be set.</error>\"\n                )\n                return 1\n        elif not url:\n            self.line_error(\n                \"<error>A custom source cannot be added without a URL.</error>\"\n            )\n            return 1\n\n        if is_default and is_secondary:\n            self.line_error(\n                \"<error>Cannot configure a source as both <c1>default</c1> and\"\n                \" <c1>secondary</c1>.</error>\"\n            )\n            return 1\n\n        if is_default or is_secondary:\n            if priority_str is not None:\n                self.line_error(\n                    \"<error>Priority was passed through both --priority and a\"\n                    \" deprecated flag (--default or --secondary). Please only provide\"\n                    \" one of these.</error>\"\n                )\n                return 1\n            else:\n                self.line_error(\n                    \"<warning>Warning: Priority was set through a deprecated flag\"\n                    \" (--default or --secondary). Consider using --priority next\"\n                    \" time.</warning>\"\n                )\n\n        if is_default:\n            priority = Priority.DEFAULT\n        elif is_secondary:\n            priority = Priority.SECONDARY\n        elif priority_str is None:\n            priority = Priority.PRIMARY\n        else:\n            priority = Priority[priority_str.upper()]\n\n        if priority is Priority.SECONDARY:\n            allowed_prios = (\n                p for p in Priority if p not in {Priority.DEFAULT, Priority.SECONDARY}\n            )\n            self.line_error(\n                \"<warning>Warning: Priority 'secondary' is deprecated. Consider\"\n                \" changing the priority to one of the non-deprecated values:\"\n                f\" {', '.join(repr(p.name.lower()) for p in allowed_prios)}.</warning>\"\n            )\n        if priority is Priority.DEFAULT:\n            self.line_error(\n                \"<warning>Warning: Priority 'default' is deprecated. You can achieve\"\n                \" the same effect by changing the priority to 'primary' and putting\"\n                \" the source first.</warning>\"\n            )\n\n        sources = AoT([])\n        new_source = Source(name=name, url=url, priority=priority)\n        is_new_source = True\n\n        for source in self.poetry.get_sources():\n            if source.priority is Priority.DEFAULT and priority is Priority.DEFAULT:\n                self.line_error(\n                    f\"<error>Source with name <c1>{source.name}</c1> is already set to\"\n                    \" default. Only one default source can be configured at a\"\n                    \" time.</error>\"\n                )\n                return 1\n\n            if source.name.lower() == lower_name:\n                source = new_source\n                is_new_source = False\n\n            sources.append(source_to_table(source))\n\n        if is_new_source:\n            self.line(f\"Adding source with name <c1>{name}</c1>.\")\n            sources.append(source_to_table(new_source))\n        else:\n            self.line(f\"Source with name <c1>{name}</c1> already exists. Updating.\")\n\n        # ensure new source is valid. eg: invalid name etc.\n        try:\n            pool = Factory.create_pool(self.poetry.config, sources, NullIO())\n            pool.repository(name)\n        except ValueError as e:\n            self.line_error(\n                f\"<error>Failed to validate addition of <c1>{name}</c1>: {e}</error>\"\n            )\n            return 1\n\n        self.poetry.pyproject.poetry_config[\"source\"] = sources\n        self.poetry.pyproject.save()\n\n        return 0\n\n# Path: src/poetry/console/commands/source/remove.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom tomlkit.items import AoT\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n\n\nclass SourceRemoveCommand(Command):\n    name = \"source remove\"\n    description = \"Remove source configured for the project.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\n            \"name\",\n            \"Source repository name.\",\n        ),\n    ]\n\n    def handle(self) -> int:\n        from poetry.utils.source import source_to_table\n\n        name = self.argument(\"name\")\n        lower_name = name.lower()\n\n        sources = AoT([])\n        removed = False\n\n        for source in self.poetry.get_sources():\n            if source.name.lower() == lower_name:\n                self.line(f\"Removing source with name <c1>{source.name}</c1>.\")\n                removed = True\n                continue\n            sources.append(source_to_table(source))\n\n        if not removed:\n            self.line_error(\n                f\"<error>Source with name <c1>{name}</c1> was not found.</error>\"\n            )\n            return 1\n\n        self.poetry.pyproject.poetry_config[\"source\"] = sources\n        self.poetry.pyproject.save()\n\n        return 0\n\n# Path: src/poetry/console/commands/source/show.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.ui.table import Rows\n\n\nclass SourceShowCommand(Command):\n    name = \"source show\"\n    description = \"Show information about sources configured for the project.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\n            \"source\",\n            \"Source(s) to show information for. Defaults to showing all sources.\",\n            optional=True,\n            multiple=True,\n        ),\n    ]\n\n    def handle(self) -> int:\n        sources = self.poetry.get_sources()\n        names = self.argument(\"source\")\n        lower_names = [name.lower() for name in names]\n\n        if not sources:\n            self.line(\"No sources configured for this project.\")\n            return 0\n\n        if names and not any(s.name.lower() in lower_names for s in sources):\n            self.line_error(\n                f\"No source found with name(s): {', '.join(names)}\",\n                style=\"error\",\n            )\n            return 1\n\n        for source in sources:\n            if names and source.name.lower() not in lower_names:\n                continue\n\n            table = self.table(style=\"compact\")\n            rows: Rows = [[\"<info>name</>\", f\" : <c1>{source.name}</>\"]]\n            if source.url:\n                rows.append([\"<info>url</>\", f\" : {source.url}\"])\n            rows.append([\"<info>priority</>\", f\" : {source.priority.name.lower()}\"])\n            table.add_rows(rows)\n            table.render()\n            self.line(\"\")\n\n        return 0\n\n# Path: src/poetry/console/io/inputs/__init__.py\n\n# Path: src/poetry/console/io/inputs/run_argv_input.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom cleo.io.inputs.argv_input import ArgvInput\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.definition import Definition\n\n\nclass RunArgvInput(ArgvInput):\n    def __init__(\n        self,\n        argv: list[str] | None = None,\n        definition: Definition | None = None,\n    ) -> None:\n        super().__init__(argv, definition=definition)\n\n        self._parameter_options: list[str] = []\n\n    @property\n    def first_argument(self) -> str | None:\n        return \"run\"\n\n    def add_parameter_option(self, name: str) -> None:\n        self._parameter_options.append(name)\n\n    def has_parameter_option(\n        self, values: str | list[str], only_params: bool = False\n    ) -> bool:\n        if not isinstance(values, list):\n            values = [values]\n\n        for token in self._tokens:\n            if only_params and token == \"--\":\n                return False\n\n            for value in values:\n                if value not in self._parameter_options:\n                    continue\n\n                # Options with values:\n                # For long options, test for '--option=' at beginning\n                # For short options, test for '-o' at beginning\n                leading = value + \"=\" if value.startswith(\"--\") else value\n\n                if token == value or leading != \"\" and token.startswith(leading):\n                    return True\n\n        return False\n\n    def _parse(self) -> None:\n        parse_options = True\n        self._parsed = self._tokens[:]\n\n        try:\n            token = self._parsed.pop(0)\n        except IndexError:\n            token = None\n\n        while token is not None:\n            if parse_options and token == \"\":\n                self._parse_argument(token)\n            elif parse_options and token == \"--\":\n                parse_options = False\n            elif parse_options and token.find(\"--\") == 0:\n                if token in self._parameter_options:\n                    self._parse_long_option(token)\n                else:\n                    self._parse_argument(token)\n            elif parse_options and token[0] == \"-\" and token != \"-\":\n                if token in self._parameter_options:\n                    self._parse_short_option(token)\n                else:\n                    self._parse_argument(token)\n            else:\n                self._parse_argument(token)\n\n            try:\n                token = self._parsed.pop(0)\n            except IndexError:\n                token = None\n\n# Path: src/poetry/console/logging/formatters/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.console.logging.formatters.builder_formatter import BuilderLogFormatter\n\n\nFORMATTERS = {\n    \"poetry.core.masonry.builders.builder\": BuilderLogFormatter(),\n    \"poetry.core.masonry.builders.sdist\": BuilderLogFormatter(),\n    \"poetry.core.masonry.builders.wheel\": BuilderLogFormatter(),\n}\n\n# Path: src/poetry/console/logging/formatters/builder_formatter.py\nfrom __future__ import annotations\n\nimport re\n\nfrom poetry.console.logging.formatters.formatter import Formatter\n\n\nclass BuilderLogFormatter(Formatter):\n    def format(self, msg: str) -> str:\n        if msg.startswith(\"Building \"):\n            msg = re.sub(\"Building (.+)\", \"  - Building <info>\\\\1</info>\", msg)\n        elif msg.startswith(\"Built \"):\n            msg = re.sub(\"Built (.+)\", \"  - Built <success>\\\\1</success>\", msg)\n        elif msg.startswith(\"Adding: \"):\n            msg = re.sub(\"Adding: (.+)\", \"  - Adding: <b>\\\\1</b>\", msg)\n        elif msg.startswith(\"Executing build script: \"):\n            msg = re.sub(\n                \"Executing build script: (.+)\",\n                \"  - Executing build script: <b>\\\\1</b>\",\n                msg,\n            )\n\n        return msg\n\n# Path: src/poetry/console/logging/formatters/formatter.py\nfrom __future__ import annotations\n\n\nclass Formatter:\n    def format(self, record: str) -> str:\n        raise NotImplementedError()\n\n# Path: src/poetry/console/commands/self/show/__init__.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.self.self_command import SelfCommand\nfrom poetry.console.commands.show import ShowCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass SelfShowCommand(SelfCommand, ShowCommand):\n    name = \"self show\"\n    options: ClassVar[list[Option]] = [\n        option(\"addons\", None, \"List only add-on packages installed.\"),\n        *[o for o in ShowCommand.options if o.name in {\"tree\", \"latest\", \"outdated\"}],\n    ]\n    description = \"Show packages from Poetry's runtime environment.\"\n    help = f\"\"\"\\\nThe <c1>self show</c1> command behaves similar to the <c1>show</c1> command, but\nworking within Poetry's runtime environment. This lists all packages installed within\nthe Poetry install environment.\n\nTo show only additional packages that have been added via <c1>self add</c1> and their\ndependencies use <c1>self show --addons</c1>.\n\nThis is managed in the <comment>{SelfCommand.get_default_system_pyproject_file()}</> \\\nfile.\n\"\"\"\n\n    @property\n    def activated_groups(self) -> set[str]:\n        if self.option(\"addons\", False):\n            return {SelfCommand.ADDITIONAL_PACKAGE_GROUP}\n\n        groups: set[str] = super(ShowCommand, self).activated_groups\n        return groups\n\n# Path: src/poetry/console/commands/self/show/plugins.py\nfrom __future__ import annotations\n\nimport dataclasses\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.console.commands.self.self_command import SelfCommand\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.package import Package\n\n    from poetry.utils._compat import metadata\n\n\n@dataclasses.dataclass\nclass PluginPackage:\n    package: Package\n    plugins: list[metadata.EntryPoint] = dataclasses.field(default_factory=list)\n    application_plugins: list[metadata.EntryPoint] = dataclasses.field(\n        default_factory=list\n    )\n\n    def append(self, entry_point: metadata.EntryPoint) -> None:\n        from poetry.plugins.application_plugin import ApplicationPlugin\n        from poetry.plugins.plugin import Plugin\n\n        group = entry_point.group\n\n        if group == ApplicationPlugin.group:\n            self.application_plugins.append(entry_point)\n        elif group == Plugin.group:\n            self.plugins.append(entry_point)\n        else:\n            name = entry_point.name\n            raise ValueError(f\"Unknown plugin group ({group}) for {name}\")\n\n\n...\n", "needle_token_start": 6960, "needle_token_end": 7426, "code_context_ntokens": 15965, "output": ["```python\ndef _display_complete_info(self, env: Env) -> None:\n    env_python_version = \".\".join(str(s) for s in env.version_info[:3])\n    self.line(\"\")\n    self.line(\"<b>Virtualenv</b>\")\n    listing = [\n        f\"<info>Python</info>:         <comment>{env_python_version}</>\",\n        f\"<info>Implementation</info>: <comment>{env.python_implementation}</>\",\n        (\n            \"<info>Path</info>:          \"\n            f\" <comment>{env.path if env.is_venv() else 'NA'}</>\"\n        ),\n        (\n            \"<info>Executable</info>:    \"\n            f\" <comment>{env.python if env.is_venv() else 'NA'}</>\"\n        ),\n    ]\n    if env.is_venv():\n        listing.append(\n            \"<info>Valid</info>:         \"\n            f\" <{'comment' if env.is_sane() else 'error'}>{env.is_sane()}</>\"\n        )\n    self.line(\"\\n\".join(listing))\n\n    self.line(\"\")\n\n    base_env = env.parent_env\n    python = \".\".join(str(v) for v in base_env.version_info[:3])\n    self.line(\"<b>Base</b>\")\n    self.line(\n        \"\\n\".join(\n            [\n                f\"<info>Platform</info>:   <comment>{env.platform}</>\",\n                f\"<info>OS</info>:         <comment>{env.os}</>\",\n                f\"<info>Python</info>:     <comment>{python}</>\",\n                f\"<info>Path</info>:       <comment>{base_env.path}</>\",\n                f\"<info>Executable</info>: <comment>{base_env.python}</>\",\n            ]\n        )\n    )\n```"]}
{"repo": "python-poetry/poetry", "name": "_write_lock_file", "language": "python", "path": "src/poetry/installation/installer.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to update and save the project's dependency lock file based on the current state of a repository that tracks locked dependencies, ensuring that the project's dependencies are consistently managed and reproducible.\n2. **Input**: It takes a repository object that contains the current state of locked dependencies and an optional boolean flag that forces the update regardless of other conditions.\n3. **Output**: There is no return value; the function's effect is the potential modification of the lock file on disk.\n4. **Procedure**: The function first checks if the operation is not a dry run and if an update is necessary or forced. If conditions are met, it updates the lock data with the current state of dependencies from the provided repository. If the lock data is successfully updated, it outputs a message indicating that the lock file is being written.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/installation/executor.py\nfrom __future__ import annotations\n\nimport contextlib\nimport csv\nimport functools\nimport itertools\nimport json\nimport threading\n\nfrom concurrent.futures import ThreadPoolExecutor\nfrom concurrent.futures import wait\nfrom pathlib import Path\nfrom subprocess import CalledProcessError\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom cleo.io.null_io import NullIO\nfrom poetry.core.packages.utils.link import Link\n\nfrom poetry.installation.chef import Chef\nfrom poetry.installation.chef import ChefBuildError\nfrom poetry.installation.chef import ChefInstallError\nfrom poetry.installation.chooser import Chooser\nfrom poetry.installation.operations import Install\nfrom poetry.installation.operations import Uninstall\nfrom poetry.installation.operations import Update\nfrom poetry.installation.wheel_installer import WheelInstaller\nfrom poetry.puzzle.exceptions import SolverProblemError\nfrom poetry.utils._compat import decode\nfrom poetry.utils.authenticator import Authenticator\nfrom poetry.utils.env import EnvCommandError\nfrom poetry.utils.helpers import Downloader\nfrom poetry.utils.helpers import get_file_hash\nfrom poetry.utils.helpers import get_highest_priority_hash_type\nfrom poetry.utils.helpers import pluralize\nfrom poetry.utils.helpers import remove_directory\nfrom poetry.utils.pip import pip_install\n\n\nif TYPE_CHECKING:\n    from cleo.io.io import IO\n    from cleo.io.outputs.section_output import SectionOutput\n    from poetry.core.masonry.builders.builder import Builder\n    from poetry.core.packages.package import Package\n\n    from poetry.config.config import Config\n    from poetry.installation.operations.operation import Operation\n    from poetry.repositories import RepositoryPool\n    from poetry.utils.env import Env\n\n\nclass Executor:\n    def __init__(\n        self,\n        env: Env,\n        pool: RepositoryPool,\n        config: Config,\n        io: IO,\n        parallel: bool | None = None,\n        disable_cache: bool = False,\n    ) -> None:\n        self._env = env\n        self._io = io\n        self._dry_run = False\n        self._enabled = True\n        self._verbose = False\n        self._wheel_installer = WheelInstaller(self._env)\n        self._use_modern_installation = config.get(\n            \"installer.modern-installation\", True\n        )\n        if not self._use_modern_installation:\n            self._io.write_line(\n                \"<warning>Warning: Setting `installer.modern-installation` to `false` \"\n                \"is deprecated.</>\"\n            )\n            self._io.write_line(\n                \"<warning>The pip-based installer will be removed in a future release.</>\"\n            )\n            self._io.write_line(\n                \"<warning>See https://github.com/python-poetry/poetry/issues/8987.</>\"\n            )\n\n        if parallel is None:\n            parallel = config.get(\"installer.parallel\", True)\n\n        if parallel:\n            self._max_workers = config.installer_max_workers\n        else:\n            self._max_workers = 1\n\n        self._artifact_cache = pool.artifact_cache\n        self._authenticator = Authenticator(\n            config, self._io, disable_cache=disable_cache, pool_size=self._max_workers\n        )\n        self._chef = Chef(self._artifact_cache, self._env, pool)\n        self._chooser = Chooser(pool, self._env, config)\n\n        self._executor = ThreadPoolExecutor(max_workers=self._max_workers)\n        self._total_operations = 0\n        self._executed_operations = 0\n        self._executed = {\"install\": 0, \"update\": 0, \"uninstall\": 0}\n        self._skipped = {\"install\": 0, \"update\": 0, \"uninstall\": 0}\n        self._sections: dict[int, SectionOutput] = {}\n        self._yanked_warnings: list[str] = []\n        self._lock = threading.Lock()\n        self._shutdown = False\n        self._hashes: dict[str, str] = {}\n\n    @property\n    def installations_count(self) -> int:\n        return self._executed[\"install\"]\n\n    @property\n    def updates_count(self) -> int:\n        return self._executed[\"update\"]\n\n    @property\n    def removals_count(self) -> int:\n        return self._executed[\"uninstall\"]\n\n    @property\n    def enabled(self) -> bool:\n        return self._enabled\n\n    def supports_fancy_output(self) -> bool:\n        return self._io.output.is_decorated() and not self._dry_run\n\n    def disable(self) -> Executor:\n        self._enabled = False\n\n        return self\n\n    def dry_run(self, dry_run: bool = True) -> Executor:\n        self._dry_run = dry_run\n\n        return self\n\n    def verbose(self, verbose: bool = True) -> Executor:\n        self._verbose = verbose\n\n        return self\n\n    def enable_bytecode_compilation(self, enable: bool = True) -> None:\n        self._wheel_installer.enable_bytecode_compilation(enable)\n\n    def pip_install(\n        self, req: Path, upgrade: bool = False, editable: bool = False\n    ) -> int:\n        try:\n            pip_install(req, self._env, upgrade=upgrade, editable=editable)\n        except EnvCommandError as e:\n            output = decode(e.e.output)\n            if (\n                \"KeyboardInterrupt\" in output\n                or \"ERROR: Operation cancelled by user\" in output\n            ):\n                return -2\n            raise\n\n        return 0\n\n    def execute(self, operations: list[Operation]) -> int:\n        self._total_operations = len(operations)\n        for job_type in self._executed:\n            self._executed[job_type] = 0\n            self._skipped[job_type] = 0\n\n        if operations and (self._enabled or self._dry_run):\n            self._display_summary(operations)\n\n        self._sections = {}\n        self._yanked_warnings = []\n\n        # pip has to be installed first without parallelism if we install via pip\n        for i, op in enumerate(operations):\n            if op.package.name == \"pip\":\n                wait([self._executor.submit(self._execute_operation, op)])\n                del operations[i]\n                break\n\n        # We group operations by priority\n        groups = itertools.groupby(operations, key=lambda o: -o.priority)\n        for _, group in groups:\n            tasks = []\n            serial_operations = []\n            for operation in group:\n                if self._shutdown:\n                    break\n\n                # Some operations are unsafe, we must execute them serially in a group\n                # https://github.com/python-poetry/poetry/issues/3086\n                # https://github.com/python-poetry/poetry/issues/2658\n                #\n                # We need to explicitly check source type here, see:\n                # https://github.com/python-poetry/poetry-core/pull/98\n                is_parallel_unsafe = operation.job_type == \"uninstall\" or (\n                    operation.package.develop\n                    and operation.package.source_type in {\"directory\", \"git\"}\n                )\n                if not operation.skipped and is_parallel_unsafe:\n                    serial_operations.append(operation)\n                    continue\n\n                tasks.append(self._executor.submit(self._execute_operation, operation))\n\n            try:\n                wait(tasks)\n\n                for operation in serial_operations:\n                    wait([self._executor.submit(self._execute_operation, operation)])\n\n            except KeyboardInterrupt:\n                self._shutdown = True\n\n            if self._shutdown:\n                # Cancelling further tasks from being executed\n                [task.cancel() for task in tasks]\n                self._executor.shutdown(wait=True)\n\n                break\n\n        for warning in self._yanked_warnings:\n            self._io.write_error_line(f\"<warning>Warning: {warning}</warning>\")\n        for path, issues in self._wheel_installer.invalid_wheels.items():\n            formatted_issues = \"\\n\".join(issues)\n            warning = (\n                f\"Validation of the RECORD file of {path.name} failed.\"\n                \" Please report to the maintainers of that package so they can fix\"\n                f\" their build process. Details:\\n{formatted_issues}\\n\"\n            )\n            self._io.write_error_line(f\"<warning>Warning: {warning}</warning>\")\n\n        return 1 if self._shutdown else 0\n\n    def _write(self, operation: Operation, line: str) -> None:\n        if not self.supports_fancy_output() or not self._should_write_operation(\n            operation\n        ):\n            return\n\n        if self._io.is_debug():\n            with self._lock:\n                section = self._sections[id(operation)]\n                section.write_line(line)\n\n            return\n\n        with self._lock:\n            section = self._sections[id(operation)]\n            section.clear()\n            section.write(line)\n\n    def _execute_operation(self, operation: Operation) -> None:\n        try:\n            op_message = self.get_operation_message(operation)\n            if self.supports_fancy_output():\n                if id(operation) not in self._sections and self._should_write_operation(\n                    operation\n                ):\n                    with self._lock:\n                        self._sections[id(operation)] = self._io.section()\n                        self._sections[id(operation)].write_line(\n                            f\"  <fg=blue;options=bold>-</> {op_message}:\"\n                            \" <fg=blue>Pending...</>\"\n                        )\n            else:\n                if self._should_write_operation(operation):\n                    if not operation.skipped:\n                        self._io.write_line(\n                            f\"  <fg=blue;options=bold>-</> {op_message}\"\n                        )\n                    else:\n                        self._io.write_line(\n                            f\"  <fg=default;options=bold,dark>-</> {op_message}: \"\n                            \"<fg=default;options=bold,dark>Skipped</> \"\n                            \"<fg=default;options=dark>for the following reason:</> \"\n                            f\"<fg=default;options=bold,dark>{operation.skip_reason}</>\"\n                        )\n\n            try:\n                result = self._do_execute_operation(operation)\n            except EnvCommandError as e:\n                if e.e.returncode == -2:\n                    result = -2\n                else:\n                    raise\n\n            # If we have a result of -2 it means a KeyboardInterrupt\n            # in the any python subprocess, so we raise a KeyboardInterrupt\n            # error to be picked up by the error handler.\n            if result == -2:\n                raise KeyboardInterrupt\n        except Exception as e:\n            try:\n                from cleo.ui.exception_trace import ExceptionTrace\n\n                io: IO | SectionOutput\n                if not self.supports_fancy_output():\n                    io = self._io\n                else:\n                    message = (\n                        \"  <error>-</error>\"\n                        f\" {self.get_operation_message(operation, error=True)}:\"\n                        \" <error>Failed</error>\"\n                    )\n                    self._write(operation, message)\n                    io = self._sections.get(id(operation), self._io)\n\n                with self._lock:\n                    trace = ExceptionTrace(e)\n                    trace.render(io)\n                    pkg = operation.package\n                    if isinstance(e, ChefBuildError):\n                        pip_command = \"pip wheel --no-cache-dir --use-pep517\"\n                        if pkg.develop:\n                            requirement = pkg.source_url\n                            pip_command += \" --editable\"\n                        else:\n                            requirement = (\n                                pkg.to_dependency().to_pep_508().split(\";\")[0].strip()\n                            )\n                        message = (\n                            \"<info>\"\n                            \"Note: This error originates from the build backend,\"\n                            \" and is likely not a problem with poetry\"\n                            f\" but with {pkg.pretty_name} ({pkg.full_pretty_version})\"\n                            \" not supporting PEP 517 builds. You can verify this by\"\n                            f\" running '{pip_command} \\\"{requirement}\\\"'.\"\n                            \"</info>\"\n                        )\n                    elif isinstance(e, ChefInstallError):\n                        message = (\n                            \"<error>\"\n                            \"Cannot install build-system.requires\"\n                            f\" for {pkg.pretty_name}.\"\n                            \"</error>\"\n                        )\n                    elif isinstance(e, SolverProblemError):\n                        message = (\n                            \"<error>\"\n                            \"Cannot resolve build-system.requires\"\n                            f\" for {pkg.pretty_name}.\"\n                            \"</error>\"\n                        )\n                    else:\n                        message = f\"<error>Cannot install {pkg.pretty_name}.</error>\"\n\n                    io.write_line(\"\")\n                    io.write_line(message)\n                    io.write_line(\"\")\n            finally:\n                with self._lock:\n                    self._shutdown = True\n\n        except KeyboardInterrupt:\n            try:\n                message = (\n                    \"  <warning>-</warning>\"\n                    f\" {self.get_operation_message(operation, warning=True)}:\"\n                    \" <warning>Cancelled</warning>\"\n                )\n                if not self.supports_fancy_output():\n                    self._io.write_line(message)\n                else:\n                    self._write(operation, message)\n            finally:\n                with self._lock:\n                    self._shutdown = True\n\n    def _do_execute_operation(self, operation: Operation) -> int:\n        method = operation.job_type\n\n        operation_message = self.get_operation_message(operation)\n        if operation.skipped:\n            if self.supports_fancy_output():\n                self._write(\n                    operation,\n                    f\"  <fg=default;options=bold,dark>-</> {operation_message}: \"\n                    \"<fg=default;options=bold,dark>Skipped</> \"\n                    \"<fg=default;options=dark>for the following reason:</> \"\n                    f\"<fg=default;options=bold,dark>{operation.skip_reason}</>\",\n                )\n\n            self._skipped[operation.job_type] += 1\n\n            return 0\n\n        if not self._enabled or self._dry_run:\n            return 0\n\n        result: int = getattr(self, f\"_execute_{method}\")(operation)\n\n        if result != 0:\n            return result\n\n        operation_message = self.get_operation_message(operation, done=True)\n        message = f\"  <fg=green;options=bold>-</> {operation_message}\"\n        self._write(operation, message)\n\n        self._increment_operations_count(operation, True)\n\n        return result\n\n    def _increment_operations_count(self, operation: Operation, executed: bool) -> None:\n        with self._lock:\n            if executed:\n                self._executed_operations += 1\n                self._executed[operation.job_type] += 1\n            else:\n                self._skipped[operation.job_type] += 1\n\n    def run_pip(self, *args: Any, **kwargs: Any) -> int:\n        try:\n            self._env.run_pip(*args, **kwargs)\n        except EnvCommandError as e:\n            output = decode(e.e.output)\n            if (\n                \"KeyboardInterrupt\" in output\n                or \"ERROR: Operation cancelled by user\" in output\n            ):\n                return -2\n\n            raise\n\n        return 0\n\n    def get_operation_message(\n        self,\n        operation: Operation,\n        done: bool = False,\n        error: bool = False,\n        warning: bool = False,\n    ) -> str:\n        base_tag = \"fg=default\"\n        operation_color = \"c2\"\n        source_operation_color = \"c2\"\n        package_color = \"c1\"\n\n        if error:\n            operation_color = \"error\"\n        elif warning:\n            operation_color = \"warning\"\n        elif done:\n            operation_color = \"success\"\n\n        if operation.skipped:\n            base_tag = \"fg=default;options=dark\"\n            operation_color += \"_dark\"\n            source_operation_color += \"_dark\"\n            package_color += \"_dark\"\n\n        if isinstance(operation, Install):\n            return (\n                f\"<{base_tag}>Installing\"\n                f\" <{package_color}>{operation.package.name}</{package_color}>\"\n                f\" (<{operation_color}>{operation.package.full_pretty_version}</>)</>\"\n            )\n\n        if isinstance(operation, Uninstall):\n            return (\n                f\"<{base_tag}>Removing\"\n                f\" <{package_color}>{operation.package.name}</{package_color}>\"\n                f\" (<{operation_color}>{operation.package.full_pretty_version}</>)</>\"\n            )\n\n        if isinstance(operation, Update):\n            initial_version = (initial_pkg := operation.initial_package).version\n            target_version = (target_pkg := operation.target_package).version\n            update_kind = (\n                \"Updating\" if target_version >= initial_version else \"Downgrading\"\n            )\n            return (\n                f\"<{base_tag}>{update_kind}\"\n                f\" <{package_color}>{initial_pkg.name}</{package_color}> \"\n                f\"(<{source_operation_color}>\"\n                f\"{initial_pkg.full_pretty_version}\"\n                f\"</{source_operation_color}> -> <{operation_color}>\"\n                f\"{target_pkg.full_pretty_version}</>)</>\"\n            )\n        return \"\"\n\n    def _display_summary(self, operations: list[Operation]) -> None:\n        installs = 0\n        updates = 0\n        uninstalls = 0\n        skipped = 0\n        for op in operations:\n            if op.skipped:\n                skipped += 1\n                continue\n\n            if op.job_type == \"install\":\n                installs += 1\n            elif op.job_type == \"update\":\n                updates += 1\n            elif op.job_type == \"uninstall\":\n                uninstalls += 1\n\n        if not installs and not updates and not uninstalls and not self._verbose:\n            self._io.write_line(\"\")\n            self._io.write_line(\"No dependencies to install or update\")\n\n            return\n\n        self._io.write_line(\"\")\n        self._io.write(\"<b>Package operations</b>: \")\n        self._io.write(f\"<info>{installs}</> install{pluralize(installs)}, \")\n        self._io.write(f\"<info>{updates}</> update{pluralize(updates)}, \")\n        self._io.write(f\"<info>{uninstalls}</> removal{pluralize(uninstalls)}\")\n        if skipped and self._verbose:\n            self._io.write(f\", <info>{skipped}</> skipped\")\n        self._io.write_line(\"\")\n        self._io.write_line(\"\")\n\n    def _execute_install(self, operation: Install | Update) -> int:\n        status_code = self._install(operation)\n\n        self._save_url_reference(operation)\n\n        return status_code\n\n    def _execute_update(self, operation: Install | Update) -> int:\n        status_code = self._update(operation)\n\n        self._save_url_reference(operation)\n\n        return status_code\n\n    def _execute_uninstall(self, operation: Uninstall) -> int:\n        op_msg = self.get_operation_message(operation)\n        message = f\"  <fg=blue;options=bold>-</> {op_msg}: <info>Removing...</info>\"\n        self._write(operation, message)\n\n        return self._remove(operation.package)\n\n    def _install(self, operation: Install | Update) -> int:\n        package = operation.package\n        if package.source_type == \"directory\" and not self._use_modern_installation:\n            return self._install_directory_without_wheel_installer(operation)\n\n        cleanup_archive: bool = False\n        if package.source_type == \"git\":\n            archive = self._prepare_git_archive(operation)\n            cleanup_archive = operation.package.develop\n        elif package.source_type == \"file\":\n            archive = self._prepare_archive(operation)\n        elif package.source_type == \"directory\":\n            archive = self._prepare_archive(operation)\n            cleanup_archive = True\n        elif package.source_type == \"url\":\n            assert package.source_url is not None\n            archive = self._download_link(operation, Link(package.source_url))\n        else:\n            archive = self._download(operation)\n\n        operation_message = self.get_operation_message(operation)\n        message = (\n            f\"  <fg=blue;options=bold>-</> {operation_message}:\"\n            \" <info>Installing...</info>\"\n        )\n        self._write(operation, message)\n\n        if not self._use_modern_installation:\n            return self.pip_install(archive, upgrade=operation.job_type == \"update\")\n\n        try:\n            if operation.job_type == \"update\":\n                # Uninstall first\n                # TODO: Make an uninstaller and find a way to rollback in case\n                # the new package can't be installed\n                assert isinstance(operation, Update)\n                self._remove(operation.initial_package)\n\n            self._wheel_installer.install(archive)\n        finally:\n            if cleanup_archive:\n                archive.unlink()\n\n        return 0\n\n    def _update(self, operation: Install | Update) -> int:\n        return self._install(operation)\n\n    def _remove(self, package: Package) -> int:\n        # If we have a VCS package, remove its source directory\n        if package.source_type == \"git\":\n            src_dir = self._env.path / \"src\" / package.name\n...\n# Path: src/poetry/installation/installer.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import cast\n\nfrom cleo.io.null_io import NullIO\nfrom packaging.utils import canonicalize_name\n\nfrom poetry.installation.executor import Executor\nfrom poetry.installation.operations import Install\nfrom poetry.installation.operations import Uninstall\nfrom poetry.installation.operations import Update\nfrom poetry.repositories import Repository\nfrom poetry.repositories import RepositoryPool\nfrom poetry.repositories.installed_repository import InstalledRepository\nfrom poetry.repositories.lockfile_repository import LockfileRepository\nfrom poetry.utils.extras import get_extra_package_names\n\n\nif TYPE_CHECKING:\n    from collections.abc import Iterable\n\n    from cleo.io.io import IO\n    from packaging.utils import NormalizedName\n    from poetry.core.packages.path_dependency import PathDependency\n    from poetry.core.packages.project_package import ProjectPackage\n\n    from poetry.config.config import Config\n    from poetry.installation.operations.operation import Operation\n    from poetry.packages import Locker\n    from poetry.utils.env import Env\n\n\nclass Installer:\n    def __init__(\n        self,\n        io: IO,\n        env: Env,\n        package: ProjectPackage,\n        locker: Locker,\n        pool: RepositoryPool,\n        config: Config,\n        installed: Repository | None = None,\n        executor: Executor | None = None,\n        disable_cache: bool = False,\n    ) -> None:\n        self._io = io\n        self._env = env\n        self._package = package\n        self._locker = locker\n        self._pool = pool\n        self._config = config\n\n        self._dry_run = False\n        self._requires_synchronization = False\n        self._update = False\n        self._verbose = False\n        self._groups: Iterable[str] | None = None\n        self._skip_directory = False\n        self._lock = False\n\n        self._whitelist: list[NormalizedName] = []\n\n        self._extras: list[NormalizedName] = []\n\n        if executor is None:\n            executor = Executor(\n                self._env, self._pool, config, self._io, disable_cache=disable_cache\n            )\n\n        self._executor = executor\n\n        if installed is None:\n            installed = self._get_installed()\n\n        self._installed_repository = installed\n\n    @property\n    def executor(self) -> Executor:\n        return self._executor\n\n    def set_package(self, package: ProjectPackage) -> Installer:\n        self._package = package\n\n        return self\n\n    def set_locker(self, locker: Locker) -> Installer:\n        self._locker = locker\n\n        return self\n\n    def run(self) -> int:\n        # Check if refresh\n        if not self._update and self._lock and self._locker.is_locked():\n            return self._do_refresh()\n\n        # Force update if there is no lock file present\n        if not self._update and not self._locker.is_locked():\n            self._update = True\n\n        if self.is_dry_run():\n            self.verbose(True)\n\n        return self._do_install()\n\n    def dry_run(self, dry_run: bool = True) -> Installer:\n        self._dry_run = dry_run\n        self._executor.dry_run(dry_run)\n\n        return self\n\n    def is_dry_run(self) -> bool:\n        return self._dry_run\n\n    def requires_synchronization(\n        self, requires_synchronization: bool = True\n    ) -> Installer:\n        self._requires_synchronization = requires_synchronization\n\n        return self\n\n    def verbose(self, verbose: bool = True) -> Installer:\n        self._verbose = verbose\n        self._executor.verbose(verbose)\n\n        return self\n\n    def is_verbose(self) -> bool:\n        return self._verbose\n\n    def only_groups(self, groups: Iterable[str]) -> Installer:\n        self._groups = groups\n\n        return self\n\n    def update(self, update: bool = True) -> Installer:\n        self._update = update\n\n        return self\n\n    def skip_directory(self, skip_directory: bool = False) -> Installer:\n        self._skip_directory = skip_directory\n\n        return self\n\n    def lock(self, update: bool = True) -> Installer:\n        \"\"\"\n        Prepare the installer for locking only.\n        \"\"\"\n        self.update(update=update)\n        self.execute_operations(False)\n        self._lock = True\n\n        return self\n\n    def is_updating(self) -> bool:\n        return self._update\n\n    def execute_operations(self, execute: bool = True) -> Installer:\n        if not execute:\n            self._executor.disable()\n\n        return self\n\n    def whitelist(self, packages: Iterable[str]) -> Installer:\n        self._whitelist = [canonicalize_name(p) for p in packages]\n\n        return self\n\n    def extras(self, extras: list[str]) -> Installer:\n        self._extras = [canonicalize_name(extra) for extra in extras]\n\n        return self\n\n    def _do_refresh(self) -> int:\n        from poetry.puzzle.solver import Solver\n\n        # Checking extras\n        for extra in self._extras:\n            if extra not in self._package.extras:\n                raise ValueError(f\"Extra [{extra}] is not specified.\")\n\n        locked_repository = self._locker.locked_repository()\n        solver = Solver(\n            self._package,\n            self._pool,\n            locked_repository.packages,\n            locked_repository.packages,\n            self._io,\n        )\n\n        # Always re-solve directory dependencies, otherwise we can't determine\n        # if anything has changed (and the lock file contains an invalid version).\n        use_latest = [\n            p.name for p in locked_repository.packages if p.source_type == \"directory\"\n        ]\n\n        with solver.provider.use_source_root(\n            source_root=self._env.path.joinpath(\"src\")\n        ):\n            ops = solver.solve(use_latest=use_latest).calculate_operations()\n\n        lockfile_repo = LockfileRepository()\n        self._populate_lockfile_repo(lockfile_repo, ops)\n\n        self._write_lock_file(lockfile_repo, force=True)\n\n        return 0\n\n    def _do_install(self) -> int:\n        from poetry.puzzle.solver import Solver\n\n        locked_repository = Repository(\"poetry-locked\")\n        if self._update:\n            if not self._lock and self._locker.is_locked():\n                locked_repository = self._locker.locked_repository()\n\n                # If no packages have been whitelisted (The ones we want to update),\n                # we whitelist every package in the lock file.\n                if not self._whitelist:\n                    for pkg in locked_repository.packages:\n                        self._whitelist.append(pkg.name)\n\n            # Checking extras\n            for extra in self._extras:\n                if extra not in self._package.extras:\n                    raise ValueError(f\"Extra [{extra}] is not specified.\")\n\n            self._io.write_line(\"<info>Updating dependencies</>\")\n            solver = Solver(\n                self._package,\n                self._pool,\n                self._installed_repository.packages,\n                locked_repository.packages,\n                self._io,\n            )\n\n            with solver.provider.use_source_root(\n                source_root=self._env.path.joinpath(\"src\")\n            ):\n                ops = solver.solve(use_latest=self._whitelist).calculate_operations()\n        else:\n            self._io.write_line(\"<info>Installing dependencies from lock file</>\")\n\n            locked_repository = self._locker.locked_repository()\n\n            if not self._locker.is_fresh():\n                raise ValueError(\n                    \"pyproject.toml changed significantly since poetry.lock was last generated. \"\n                    \"Run `poetry lock [--no-update]` to fix the lock file.\"\n                )\n\n            locker_extras = {\n                canonicalize_name(extra)\n                for extra in self._locker.lock_data.get(\"extras\", {})\n            }\n            for extra in self._extras:\n                if extra not in locker_extras:\n                    raise ValueError(f\"Extra [{extra}] is not specified.\")\n\n            # If we are installing from lock\n            # Filter the operations by comparing it with what is\n            # currently installed\n            ops = self._get_operations_from_lock(locked_repository)\n\n        lockfile_repo = LockfileRepository()\n        uninstalls = self._populate_lockfile_repo(lockfile_repo, ops)\n\n        if not self.executor.enabled:\n            # If we are only in lock mode, no need to go any further\n            self._write_lock_file(lockfile_repo)\n            return 0\n\n        if self._groups is not None:\n            root = self._package.with_dependency_groups(list(self._groups), only=True)\n        else:\n            root = self._package.without_optional_dependency_groups()\n\n        if self._io.is_verbose():\n            self._io.write_line(\"\")\n            self._io.write_line(\n                \"<info>Finding the necessary packages for the current system</>\"\n            )\n\n        # We resolve again by only using the lock file\n        packages = lockfile_repo.packages + locked_repository.packages\n        pool = RepositoryPool.from_packages(packages, self._config)\n\n        solver = Solver(\n            root,\n            pool,\n            self._installed_repository.packages,\n            locked_repository.packages,\n            NullIO(),\n        )\n        # Everything is resolved at this point, so we no longer need\n        # to load deferred dependencies (i.e. VCS, URL and path dependencies)\n        solver.provider.load_deferred(False)\n\n        with solver.use_environment(self._env):\n            ops = solver.solve(use_latest=self._whitelist).calculate_operations(\n                with_uninstalls=self._requires_synchronization,\n                synchronize=self._requires_synchronization,\n                skip_directory=self._skip_directory,\n            )\n\n        if not self._requires_synchronization:\n            # If no packages synchronisation has been requested we need\n            # to calculate the uninstall operations\n            from poetry.puzzle.transaction import Transaction\n\n            transaction = Transaction(\n                locked_repository.packages,\n                [(package, 0) for package in lockfile_repo.packages],\n                installed_packages=self._installed_repository.packages,\n                root_package=root,\n            )\n\n            ops = [\n                op\n                for op in transaction.calculate_operations(with_uninstalls=True)\n                if op.job_type == \"uninstall\"\n            ] + ops\n        else:\n            ops = uninstalls + ops\n\n        # We need to filter operations so that packages\n        # not compatible with the current system,\n        # or optional and not requested, are dropped\n        self._filter_operations(ops, lockfile_repo)\n\n        # Validate the dependencies\n        for op in ops:\n            dep = op.package.to_dependency()\n            if dep.is_file() or dep.is_directory():\n                dep = cast(\"PathDependency\", dep)\n                dep.validate(raise_error=not op.skipped)\n\n        # Execute operations\n        status = self._execute(ops)\n\n        if status == 0 and self._update:\n            # Only write lock file when installation is success\n            self._write_lock_file(lockfile_repo)\n\n        return status\n\n    \ndef _write_lock_file(self, repo: LockfileRepository, force: bool = False) -> None:\n        if not self.is_dry_run() and (force or self._update):\n            updated_lock = self._locker.set_lock_data(self._package, repo.packages)\n\n            if updated_lock:\n                self._io.write_line(\"\")\n                self._io.write_line(\"<info>Writing lock file</>\")\n\n    def _execute(self, operations: list[Operation]) -> int:\n        return self._executor.execute(operations)\n\n    def _populate_lockfile_repo(\n        self, repo: LockfileRepository, ops: Iterable[Operation]\n    ) -> list[Uninstall]:\n        uninstalls = []\n        for op in ops:\n            if isinstance(op, Uninstall):\n                uninstalls.append(op)\n                continue\n\n            package = op.target_package if isinstance(op, Update) else op.package\n            if not repo.has_package(package):\n                repo.add_package(package)\n\n        return uninstalls\n\n    def _get_operations_from_lock(\n        self, locked_repository: Repository\n    ) -> list[Operation]:\n        installed_repo = self._installed_repository\n        ops: list[Operation] = []\n\n        extra_packages = self._get_extra_packages(locked_repository)\n        for locked in locked_repository.packages:\n            is_installed = False\n            for installed in installed_repo.packages:\n                if locked.name == installed.name:\n                    is_installed = True\n                    if locked.optional and locked.name not in extra_packages:\n                        # Installed but optional and not requested in extras\n                        ops.append(Uninstall(locked))\n                    elif locked.version != installed.version:\n                        ops.append(Update(installed, locked))\n\n            # If it's optional and not in required extras\n            # we do not install\n            if locked.optional and locked.name not in extra_packages:\n                continue\n\n            op = Install(locked)\n            if is_installed:\n                op.skip(\"Already installed\")\n\n            ops.append(op)\n\n        return ops\n\n    def _filter_operations(self, ops: Iterable[Operation], repo: Repository) -> None:\n        extra_packages = self._get_extra_packages(repo)\n        for op in ops:\n            package = op.target_package if isinstance(op, Update) else op.package\n\n            if op.job_type == \"uninstall\":\n                continue\n\n            if not self._env.is_valid_for_marker(package.marker):\n                op.skip(\"Not needed for the current environment\")\n                continue\n\n            # If a package is optional and not requested\n            # in any extra we skip it\n            if package.optional and package.name not in extra_packages:\n                op.skip(\"Not required\")\n\n    def _get_extra_packages(self, repo: Repository) -> set[NormalizedName]:\n        \"\"\"\n        Returns all package names required by extras.\n\n        Maybe we just let the solver handle it?\n        \"\"\"\n        extras: dict[NormalizedName, list[NormalizedName]]\n        if self._update:\n            extras = {k: [d.name for d in v] for k, v in self._package.extras.items()}\n        else:\n            raw_extras = self._locker.lock_data.get(\"extras\", {})\n            extras = {\n                canonicalize_name(extra): [\n                    canonicalize_name(dependency) for dependency in dependencies\n                ]\n                for extra, dependencies in raw_extras.items()\n            }\n\n        return get_extra_package_names(repo.packages, extras, self._extras)\n\n    def _get_installed(self) -> InstalledRepository:\n        return InstalledRepository.load(self._env)\n\n# Path: src/poetry/installation/wheel_installer.py\nfrom __future__ import annotations\n\nimport logging\nimport platform\nimport sys\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\n\nfrom installer import install\nfrom installer.destinations import SchemeDictionaryDestination\nfrom installer.sources import WheelFile\nfrom installer.sources import _WheelFileValidationError\n\nfrom poetry.__version__ import __version__\nfrom poetry.utils._compat import WINDOWS\n\n\nlogger = logging.getLogger(__name__)\n\nif TYPE_CHECKING:\n    from collections.abc import Collection\n    from typing import BinaryIO\n\n    from installer.records import RecordEntry\n    from installer.scripts import LauncherKind\n    from installer.utils import Scheme\n\n    from poetry.utils.env import Env\n\n\nclass WheelDestination(SchemeDictionaryDestination):\n    \"\"\" \"\"\"\n\n    def write_to_fs(\n        self,\n        scheme: Scheme,\n        path: str,\n        stream: BinaryIO,\n        is_executable: bool,\n    ) -> RecordEntry:\n        from installer.records import Hash\n        from installer.records import RecordEntry\n        from installer.utils import copyfileobj_with_hashing\n        from installer.utils import make_file_executable\n\n        target_path = Path(self.scheme_dict[scheme]) / path\n        if target_path.exists():\n            # Contrary to the base library we don't raise an error here since it can\n            # break pkgutil-style and pkg_resource-style namespace packages.\n            logger.warning(f\"Installing {target_path} over existing file\")\n\n        parent_folder = target_path.parent\n        if not parent_folder.exists():\n            # Due to the parallel installation it can happen\n            # that two threads try to create the directory.\n            parent_folder.mkdir(parents=True, exist_ok=True)\n\n        with target_path.open(\"wb\") as f:\n            hash_, size = copyfileobj_with_hashing(stream, f, self.hash_algorithm)\n\n        if is_executable:\n            make_file_executable(target_path)\n\n        return RecordEntry(path, Hash(self.hash_algorithm, hash_), size)\n\n\nclass WheelInstaller:\n    def __init__(self, env: Env) -> None:\n        self._env = env\n\n        script_kind: LauncherKind\n        if not WINDOWS:\n            script_kind = \"posix\"\n        else:\n            if platform.uname()[4].startswith(\"arm\"):\n                script_kind = \"win-arm64\" if sys.maxsize > 2**32 else \"win-arm\"\n            else:\n                script_kind = \"win-amd64\" if sys.maxsize > 2**32 else \"win-ia32\"\n        self._script_kind = script_kind\n\n        self._bytecode_optimization_levels: Collection[int] = ()\n        self.invalid_wheels: dict[Path, list[str]] = {}\n\n    def enable_bytecode_compilation(self, enable: bool = True) -> None:\n        self._bytecode_optimization_levels = (-1,) if enable else ()\n\n    def install(self, wheel: Path) -> None:\n        with WheelFile.open(wheel) as source:\n            try:\n                # Content validation is temporarily disabled because of\n                # pypa/installer's out of memory issues with big wheels. See\n                # https://github.com/python-poetry/poetry/issues/7983\n                source.validate_record(validate_contents=False)\n            except _WheelFileValidationError as e:\n                self.invalid_wheels[wheel] = e.issues\n\n            scheme_dict = self._env.paths.copy()\n            scheme_dict[\"headers\"] = str(\n                Path(scheme_dict[\"include\"]) / source.distribution\n            )\n            destination = WheelDestination(\n                scheme_dict,\n                interpreter=str(self._env.python),\n                script_kind=self._script_kind,\n                bytecode_optimization_levels=self._bytecode_optimization_levels,\n            )\n\n            install(\n                source=source,\n                destination=destination,\n                # Additional metadata that is generated by the installation tool.\n                additional_metadata={\n                    \"INSTALLER\": f\"Poetry {__version__}\".encode(),\n                },\n            )\n\n# Path: src/poetry/json/__init__.py\nfrom __future__ import annotations\n\nimport json\n\nfrom pathlib import Path\nfrom typing import Any\n\nimport fastjsonschema\n\nfrom fastjsonschema.exceptions import JsonSchemaException\nfrom poetry.core.json import SCHEMA_DIR as CORE_SCHEMA_DIR\n\n\nSCHEMA_DIR = Path(__file__).parent / \"schemas\"\n\n\nclass ValidationError(ValueError):\n    pass\n\n\ndef validate_object(obj: dict[str, Any]) -> list[str]:\n    schema_file = Path(SCHEMA_DIR, \"poetry.json\")\n    schema = json.loads(schema_file.read_text(encoding=\"utf-8\"))\n\n    validate = fastjsonschema.compile(schema)\n\n    errors = []\n    try:\n        validate(obj)\n    except JsonSchemaException as e:\n        errors = [e.message]\n\n    core_schema = json.loads(\n        (CORE_SCHEMA_DIR / \"poetry-schema.json\").read_text(encoding=\"utf-8\")\n    )\n\n    properties = {*schema[\"properties\"].keys(), *core_schema[\"properties\"].keys()}\n    additional_properties = set(obj.keys()) - properties\n    for key in additional_properties:\n        errors.append(f\"Additional properties are not allowed ('{key}' was unexpected)\")\n\n    return errors\n\n# Path: src/poetry/layouts/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.layouts.layout import Layout\nfrom poetry.layouts.src import SrcLayout\n\n\n_LAYOUTS = {\"src\": SrcLayout, \"standard\": Layout}\n\n\ndef layout(name: str) -> type[Layout]:\n    if name not in _LAYOUTS:\n        raise ValueError(\"Invalid layout\")\n\n    return _LAYOUTS[name]\n\n# Path: src/poetry/layouts/layout.py\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom packaging.utils import canonicalize_name\nfrom poetry.core.utils.helpers import module_name\nfrom tomlkit import inline_table\nfrom tomlkit import loads\nfrom tomlkit import table\nfrom tomlkit.toml_document import TOMLDocument\n\nfrom poetry.pyproject.toml import PyProjectTOML\n\n\nif TYPE_CHECKING:\n    from collections.abc import Mapping\n\n    from tomlkit.items import InlineTable\n\n\nPOETRY_DEFAULT = \"\"\"\\\n[tool.poetry]\nname = \"\"\nversion = \"\"\ndescription = \"\"\nauthors = []\nlicense = \"\"\nreadme = \"\"\npackages = []\n\n[tool.poetry.dependencies]\n\n[tool.poetry.group.dev.dependencies]\n\"\"\"\n\nBUILD_SYSTEM_MIN_VERSION: str | None = None\nBUILD_SYSTEM_MAX_VERSION: str | None = None\n\n\nclass Layout:\n    def __init__(\n        self,\n        project: str,\n        version: str = \"0.1.0\",\n        description: str = \"\",\n        readme_format: str = \"md\",\n        author: str | None = None,\n        license: str | None = None,\n        python: str = \"*\",\n        dependencies: Mapping[str, str | Mapping[str, Any]] | None = None,\n        dev_dependencies: Mapping[str, str | Mapping[str, Any]] | None = None,\n    ) -> None:\n        self._project = canonicalize_name(project)\n        self._package_path_relative = Path(\n            *(module_name(part) for part in project.split(\".\"))\n        )\n        self._package_name = \".\".join(self._package_path_relative.parts)\n        self._version = version\n        self._description = description\n\n        self._readme_format = readme_format.lower()\n\n        self._license = license\n        self._python = python\n        self._dependencies = dependencies or {}\n        self._dev_dependencies = dev_dependencies or {}\n\n        if not author:\n            author = \"Your Name <you@example.com>\"\n\n        self._author = author\n\n    @property\n    def basedir(self) -> Path:\n        return Path()\n\n    @property\n    def package_path(self) -> Path:\n        return self.basedir / self._package_path_relative\n\n    def get_package_include(self) -> InlineTable | None:\n        package = inline_table()\n\n        # If a project is created in the root directory (this is reasonable inside a\n        # docker container, eg <https://github.com/python-poetry/poetry/issues/5103>)\n        # then parts will be empty.\n        parts = self._package_path_relative.parts\n        if not parts:\n            return None\n\n        include = parts[0]\n        package.append(\"include\", include)\n\n        if self.basedir != Path():\n            package.append(\"from\", self.basedir.as_posix())\n        else:\n            if module_name(self._project) == include:\n                # package include and package name are the same,\n                # packages table is redundant here.\n                return None\n\n        return package\n\n    def create(\n        self, path: Path, with_tests: bool = True, with_pyproject: bool = True\n    ) -> None:\n        path.mkdir(parents=True, exist_ok=True)\n\n        self._create_default(path)\n        self._create_readme(path)\n\n        if with_tests:\n            self._create_tests(path)\n\n        if with_pyproject:\n            self._write_poetry(path)\n\n    def generate_poetry_content(self) -> TOMLDocument:\n        template = POETRY_DEFAULT\n\n        content: dict[str, Any] = loads(template)\n\n        poetry_content = content[\"tool\"][\"poetry\"]\n        poetry_content[\"name\"] = self._project\n        poetry_content[\"version\"] = self._version\n        poetry_content[\"description\"] = self._description\n        poetry_content[\"authors\"].append(self._author)\n\n        if self._license:\n            poetry_content[\"license\"] = self._license\n        else:\n            poetry_content.remove(\"license\")\n\n        poetry_content[\"readme\"] = f\"README.{self._readme_format}\"\n        packages = self.get_package_include()\n        if packages:\n            poetry_content[\"packages\"].append(packages)\n        else:\n            poetry_content.remove(\"packages\")\n\n        poetry_content[\"dependencies\"][\"python\"] = self._python\n\n        for dep_name, dep_constraint in self._dependencies.items():\n            poetry_content[\"dependencies\"][dep_name] = dep_constraint\n\n        if self._dev_dependencies:\n            for dep_name, dep_constraint in self._dev_dependencies.items():\n                poetry_content[\"group\"][\"dev\"][\"dependencies\"][dep_name] = (\n                    dep_constraint\n                )\n        else:\n            del poetry_content[\"group\"]\n\n        # Add build system\n        build_system = table()\n        build_system_version = \"\"\n\n        if BUILD_SYSTEM_MIN_VERSION is not None:\n            build_system_version = \">=\" + BUILD_SYSTEM_MIN_VERSION\n        if BUILD_SYSTEM_MAX_VERSION is not None:\n            if build_system_version:\n                build_system_version += \",\"\n            build_system_version += \"<\" + BUILD_SYSTEM_MAX_VERSION\n\n        build_system.add(\"requires\", [\"poetry-core\" + build_system_version])\n        build_system.add(\"build-backend\", \"poetry.core.masonry.api\")\n\n        assert isinstance(content, TOMLDocument)\n        content.add(\"build-system\", build_system)\n\n        return content\n\n    def _create_default(self, path: Path, src: bool = True) -> None:\n        package_path = path / self.package_path\n        package_path.mkdir(parents=True)\n\n        package_init = package_path / \"__init__.py\"\n        package_init.touch()\n\n    def _create_readme(self, path: Path) -> Path:\n        readme_file = path.joinpath(f\"README.{self._readme_format}\")\n        readme_file.touch()\n        return readme_file\n\n    @staticmethod\n    def _create_tests(path: Path) -> None:\n        tests = path / \"tests\"\n        tests.mkdir()\n\n        tests_init = tests / \"__init__.py\"\n        tests_init.touch(exist_ok=False)\n\n    def _write_poetry(self, path: Path) -> None:\n        pyproject = PyProjectTOML(path / \"pyproject.toml\")\n        content = self.generate_poetry_content()\n        for section, item in content.items():\n            pyproject.data.append(section, item)\n        pyproject.save()\n\n# Path: src/poetry/layouts/src.py\nfrom __future__ import annotations\n\nfrom pathlib import Path\n\nfrom poetry.layouts.layout import Layout\n\n\nclass SrcLayout(Layout):\n    @property\n    def basedir(self) -> Path:\n        return Path(\"src\")\n\n# Path: src/poetry/layouts/standard.py\n\n# Path: src/poetry/masonry/__init__.py\n\n# Path: src/poetry/masonry/api.py\nfrom __future__ import annotations\n\nfrom poetry.core.masonry.api import build_sdist\nfrom poetry.core.masonry.api import build_wheel\nfrom poetry.core.masonry.api import get_requires_for_build_sdist\nfrom poetry.core.masonry.api import get_requires_for_build_wheel\nfrom poetry.core.masonry.api import prepare_metadata_for_build_wheel\n\n\n__all__ = [\n    \"build_sdist\",\n    \"build_wheel\",\n    \"get_requires_for_build_sdist\",\n    \"get_requires_for_build_wheel\",\n    \"prepare_metadata_for_build_wheel\",\n]\n\n# Path: src/poetry/mixology/__init__.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.mixology.version_solver import VersionSolver\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.project_package import ProjectPackage\n\n    from poetry.mixology.result import SolverResult\n    from poetry.puzzle.provider import Provider\n\n\ndef resolve_version(root: ProjectPackage, provider: Provider) -> SolverResult:\n    solver = VersionSolver(root, provider)\n\n    return solver.solve()\n\n# Path: src/poetry/mixology/assignment.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.mixology.term import Term\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.dependency import Dependency\n    from poetry.core.packages.package import Package\n\n    from poetry.mixology.incompatibility import Incompatibility\n\n\nclass Assignment(Term):\n    \"\"\"\n    A term in a PartialSolution that tracks some additional metadata.\n    \"\"\"\n\n    def __init__(\n        self,\n        dependency: Dependency,\n        is_positive: bool,\n        decision_level: int,\n        index: int,\n        cause: Incompatibility | None = None,\n    ) -> None:\n        super().__init__(dependency, is_positive)\n\n        self._decision_level = decision_level\n        self._index = index\n        self._cause = cause\n\n    @property\n    def decision_level(self) -> int:\n        return self._decision_level\n\n    @property\n    def index(self) -> int:\n        return self._index\n\n    @property\n    def cause(self) -> Incompatibility | None:\n        return self._cause\n\n    @classmethod\n    def decision(cls, package: Package, decision_level: int, index: int) -> Assignment:\n        return cls(package.to_dependency(), True, decision_level, index)\n\n    @classmethod\n    def derivation(\n        cls,\n        dependency: Dependency,\n        is_positive: bool,\n        cause: Incompatibility,\n        decision_level: int,\n        index: int,\n    ) -> Assignment:\n        return cls(dependency, is_positive, decision_level, index, cause)\n\n    def is_decision(self) -> bool:\n        return self._cause is None\n\n# Path: src/poetry/mixology/failure.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.core.constraints.version import parse_constraint\n\nfrom poetry.mixology.incompatibility_cause import ConflictCause\nfrom poetry.mixology.incompatibility_cause import PythonCause\n\n\nif TYPE_CHECKING:\n    from poetry.mixology.incompatibility import Incompatibility\n\n\nclass SolveFailure(Exception):\n    def __init__(self, incompatibility: Incompatibility) -> None:\n        self._incompatibility = incompatibility\n\n    @property\n    def message(self) -> str:\n        return str(self)\n\n    def __str__(self) -> str:\n        return _Writer(self._incompatibility).write()\n\n\nclass _Writer:\n    def __init__(self, root: Incompatibility) -> None:\n        self._root = root\n        self._derivations: dict[Incompatibility, int] = {}\n        self._lines: list[tuple[str, int | None]] = []\n        self._line_numbers: dict[Incompatibility, int] = {}\n\n        self._count_derivations(self._root)\n\n    def write(self) -> str:\n        buffer = []\n        version_solutions = []\n        required_python_version_notification = False\n        for incompatibility in self._root.external_incompatibilities:\n            if isinstance(incompatibility.cause, PythonCause):\n                root_constraint = parse_constraint(\n                    incompatibility.cause.root_python_version\n                )\n                constraint = parse_constraint(incompatibility.cause.python_version)\n\n                version_solutions.append(\n                    \"For <fg=default;options=bold>\"\n                    f\"{incompatibility.terms[0].dependency.name}</>,\"\n                    \" a possible solution would be to set the\"\n                    \" `<fg=default;options=bold>python</>` property to\"\n                    f' <fg=yellow>\"{root_constraint.intersect(constraint)}\"</>'\n                )\n                if not required_python_version_notification:\n                    buffer.append(\n                        \"The current project's supported Python range\"\n                        f\" ({incompatibility.cause.root_python_version}) is not\"\n                        \" compatible with some of the required packages Python\"\n                        \" requirement:\"\n                    )\n                    required_python_version_notification = True\n\n                root_constraint = parse_constraint(\n                    incompatibility.cause.root_python_version\n                )\n                constraint = parse_constraint(incompatibility.cause.python_version)\n                buffer.append(\n                    f\"  - {incompatibility.terms[0].dependency.name} requires Python\"\n                    f\" {incompatibility.cause.python_version}, so it will not be\"\n                    f\" satisfied for Python {root_constraint.difference(constraint)}\"\n                )\n\n        if required_python_version_notification:\n            buffer.append(\"\")\n\n        if isinstance(self._root.cause, ConflictCause):\n            self._visit(self._root)\n        else:\n            self._write(self._root, f\"Because {self._root}, version solving failed.\")\n\n        padding = (\n            0\n            if not self._line_numbers\n            else len(f\"({list(self._line_numbers.values())[-1]}) \")\n        )\n\n        last_was_empty = False\n        for line in self._lines:\n            message = line[0]\n            if not message:\n                if not last_was_empty:\n                    buffer.append(\"\")\n\n                last_was_empty = True\n                continue\n\n            last_was_empty = False\n\n            number = line[-1]\n            if number is not None:\n                message = f\"({number})\".ljust(padding) + message\n            else:\n                message = \" \" * padding + message\n\n            buffer.append(message)\n        if required_python_version_notification:\n            # Add suggested solution\n            links = \",\".join(\n                f\"\\n    <fg=blue>https://python-poetry.org/docs/dependency-specification/#{section}</>\"\n                for section in [\n                    \"python-restricted-dependencies\",\n                    \"using-environment-markers\",\n                ]\n            )\n\n            description = (\n                \"The Python requirement can be specified via the\"\n                \" `<fg=default;options=bold>python</>` or\"\n                \" `<fg=default;options=bold>markers</>` properties\"\n            )\n            if version_solutions:\n                description += \"\\n\\n    \" + \"\\n\".join(version_solutions)\n\n            description = description.strip(\" \")\n\n            buffer.append(\n                f\"\\n  <fg=blue;options=bold>* </>\"\n                f\"<fg=default;options=bold>Check your dependencies Python requirement</>:\"\n                f\" {description}\\n{links}\\n\",\n            )\n        return \"\\n\".join(buffer)\n\n    def _write(\n        self, incompatibility: Incompatibility, message: str, numbered: bool = False\n    ) -> None:\n        if numbered:\n            number = len(self._line_numbers) + 1\n            self._line_numbers[incompatibility] = number\n            self._lines.append((message, number))\n        else:\n            self._lines.append((message, None))\n\n    def _visit(\n        self,\n        incompatibility: Incompatibility,\n        conclusion: bool = False,\n    ) -> None:\n        numbered = conclusion or self._derivations[incompatibility] > 1\n        conjunction = \"So,\" if conclusion or incompatibility == self._root else \"And\"\n        incompatibility_string = str(incompatibility)\n\n        cause = incompatibility.cause\n        assert isinstance(cause, ConflictCause)\n\n        if isinstance(cause.conflict.cause, ConflictCause) and isinstance(\n            cause.other.cause, ConflictCause\n        ):\n            conflict_line = self._line_numbers.get(cause.conflict)\n            other_line = self._line_numbers.get(cause.other)\n\n            if conflict_line is not None and other_line is not None:\n                reason = cause.conflict.and_to_string(\n                    cause.other, conflict_line, other_line\n                )\n                self._write(\n                    incompatibility,\n                    f\"Because {reason}, {incompatibility_string}.\",\n                    numbered=numbered,\n                )\n            elif conflict_line is not None or other_line is not None:\n                if conflict_line is not None:\n                    with_line = cause.conflict\n                    without_line = cause.other\n                    line = conflict_line\n                elif other_line is not None:\n                    with_line = cause.other\n                    without_line = cause.conflict\n                    line = other_line\n\n                self._visit(without_line)\n                self._write(\n                    incompatibility,\n                    f\"{conjunction} because {with_line!s} ({line}),\"\n                    f\" {incompatibility_string}.\",\n                    numbered=numbered,\n                )\n            else:\n                single_line_conflict = self._is_single_line(cause.conflict.cause)\n                single_line_other = self._is_single_line(cause.other.cause)\n\n                if single_line_other or single_line_conflict:\n                    first = cause.conflict if single_line_other else cause.other\n                    second = cause.other if single_line_other else cause.conflict\n                    self._visit(first)\n                    self._visit(second)\n                    self._write(\n                        incompatibility,\n                        f\"Thus, {incompatibility_string}.\",\n                        numbered=numbered,\n                    )\n                else:\n                    self._visit(cause.conflict, conclusion=True)\n                    self._lines.append((\"\", None))\n\n                    self._visit(cause.other)\n\n                    self._write(\n                        incompatibility,\n...\n", "needle_token_start": 8778, "needle_token_end": 8888, "code_context_ntokens": 16020, "output": ["```python\ndef _write_lock_file(self, repo: LockfileRepository, force: bool = False) -> None:\n    if not self.is_dry_run() and (force or self._update):\n        updated_lock = self._locker.set_lock_data(self._package, repo.packages)\n\n        if updated_lock:\n            self._io.write_line(\"\")\n            self._io.write_line(\"<info>Writing lock file</>\")\n```"]}
{"repo": "python-poetry/poetry", "name": "_do_execute_operation", "language": "python", "path": "src/poetry/installation/executor.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to manage the execution of a specified operation, handling its completion, skipping, or failure based on the operation's properties and the system's state.\n2. **Input**: The function takes an operation object that contains details about the job to be executed, including its type and whether it should be skipped.\n3. **Output**: Returns an integer indicating the result of the operation execution, where a value of 0 typically denotes successful completion or intentional skipping.\n4. **Procedure**: The function first checks if the operation is marked as skipped and, if so, logs this status and returns 0. If not skipped, it checks if the system is in a dry run or disabled state, returning 0 if true. Otherwise, it dynamically calls the appropriate method based on the operation type to execute the job. After execution, it logs the completion and updates the count of successful operations, returning the result of the execution.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/inspection/lazy_wheel.py\n\"\"\"Lazy ZIP over HTTP\"\"\"\n\nfrom __future__ import annotations\n\nimport io\nimport logging\nimport re\n\nfrom bisect import bisect_left\nfrom bisect import bisect_right\nfrom contextlib import contextmanager\nfrom tempfile import NamedTemporaryFile\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import BinaryIO\nfrom typing import ClassVar\nfrom typing import TypeVar\nfrom typing import cast\nfrom urllib.parse import urlparse\nfrom zipfile import BadZipFile\nfrom zipfile import ZipFile\n\nfrom packaging.metadata import parse_email\nfrom requests.models import CONTENT_CHUNK_SIZE\nfrom requests.models import HTTPError\nfrom requests.models import Response\nfrom requests.status_codes import codes\n\n\nif TYPE_CHECKING:\n    from collections.abc import Iterable\n    from collections.abc import Iterator\n    from types import TracebackType\n\n    from packaging.metadata import RawMetadata\n    from requests import Session\n\n    from poetry.utils.authenticator import Authenticator\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass LazyWheelUnsupportedError(Exception):\n    \"\"\"Raised when a lazy wheel is unsupported.\"\"\"\n\n\nclass HTTPRangeRequestUnsupported(LazyWheelUnsupportedError):\n    \"\"\"Raised when the remote server appears unable to support byte ranges.\"\"\"\n\n\nclass HTTPRangeRequestNotRespected(LazyWheelUnsupportedError):\n    \"\"\"Raised when the remote server tells us that it supports byte ranges\n    but does not respect a respective request.\"\"\"\n\n\nclass UnsupportedWheel(LazyWheelUnsupportedError):\n    \"\"\"Unsupported wheel.\"\"\"\n\n\nclass InvalidWheel(LazyWheelUnsupportedError):\n    \"\"\"Invalid (e.g. corrupt) wheel.\"\"\"\n\n    def __init__(self, location: str, name: str) -> None:\n        self.location = location\n        self.name = name\n\n    def __str__(self) -> str:\n        return f\"Wheel {self.name} located at {self.location} is invalid.\"\n\n\ndef metadata_from_wheel_url(\n    name: str, url: str, session: Session | Authenticator\n) -> RawMetadata:\n    \"\"\"Fetch metadata from the given wheel URL.\n\n    This uses HTTP range requests to only fetch the portion of the wheel\n    containing metadata, just enough for the object to be constructed.\n\n    :raises HTTPRangeRequestUnsupported: if range requests are unsupported for ``url``.\n    :raises InvalidWheel: if the zip file contents could not be parsed.\n    \"\"\"\n    try:\n        # After context manager exit, wheel.name will point to a deleted file path.\n        # Add `delete_backing_file=False` to disable this for debugging.\n        with LazyWheelOverHTTP(url, session) as lazy_file:\n            metadata_bytes = lazy_file.read_metadata(name)\n\n        metadata, _ = parse_email(metadata_bytes)\n        return metadata\n\n    except (BadZipFile, UnsupportedWheel):\n        # We assume that these errors have occurred because the wheel contents\n        # themselves are invalid, not because we've messed up our bookkeeping\n        # and produced an invalid file.\n        raise InvalidWheel(url, name)\n    except Exception as e:\n        if isinstance(e, LazyWheelUnsupportedError):\n            # this is expected when the code handles issues with lazy wheel metadata retrieval correctly\n            raise e\n\n        logger.debug(\n            \"There was an unexpected %s when handling lazy wheel metadata retrieval for %s from %s: %s\",\n            type(e).__name__,\n            name,\n            url,\n            e,\n        )\n\n        # Catch all exception to handle any issues that may have occurred during\n        # attempts to use Lazy Wheel.\n        raise LazyWheelUnsupportedError(\n            f\"Attempts to use lazy wheel metadata retrieval for {name} from {url} failed\"\n        ) from e\n\n\nclass MergeIntervals:\n    \"\"\"Stateful bookkeeping to merge interval graphs.\"\"\"\n\n    def __init__(self, *, left: Iterable[int] = (), right: Iterable[int] = ()) -> None:\n        self._left = list(left)\n        self._right = list(right)\n\n    def __repr__(self) -> str:\n        return (\n            f\"{type(self).__name__}\"\n            f\"(left={tuple(self._left)}, right={tuple(self._right)})\"\n        )\n\n    def _merge(\n        self, start: int, end: int, left: int, right: int\n    ) -> Iterator[tuple[int, int]]:\n        \"\"\"Return an iterator of intervals to be fetched.\n\n        Args:\n            start: Start of needed interval\n            end: End of needed interval\n            left: Index of first overlapping downloaded data\n            right: Index after last overlapping downloaded data\n        \"\"\"\n        lslice, rslice = self._left[left:right], self._right[left:right]\n        i = start = min([start] + lslice[:1])\n        end = max([end] + rslice[-1:])\n        for j, k in zip(lslice, rslice):\n            if j > i:\n                yield i, j - 1\n            i = k + 1\n        if i <= end:\n            yield i, end\n        self._left[left:right], self._right[left:right] = [start], [end]\n\n    def minimal_intervals_covering(\n        self, start: int, end: int\n    ) -> Iterator[tuple[int, int]]:\n        \"\"\"Provide the intervals needed to cover from ``start <= x <= end``.\n\n        This method mutates internal state so that later calls only return intervals not\n        covered by prior calls. The first call to this method will always return exactly\n        one interval, which was exactly the one requested. Later requests for\n        intervals overlapping that first requested interval will yield only the ranges\n        not previously covered (which may be empty, e.g. if the same interval is\n        requested twice).\n\n        This may be used e.g. to download substrings of remote files on demand.\n        \"\"\"\n        left = bisect_left(self._right, start)\n        right = bisect_right(self._left, end)\n        yield from self._merge(start, end, left, right)\n\n\nT = TypeVar(\"T\", bound=\"ReadOnlyIOWrapper\")\n\n\nclass ReadOnlyIOWrapper(BinaryIO):\n    \"\"\"Implement read-side ``BinaryIO`` methods wrapping an inner ``BinaryIO``.\n\n    This wrapper is useful because Python currently does not distinguish read-only\n    streams at the type level.\n    \"\"\"\n\n    def __init__(self, inner: BinaryIO) -> None:\n        self._file = inner\n\n    def __enter__(self: T) -> T:\n        self._file.__enter__()\n        return self\n\n    def __exit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        self._file.__exit__(exc_type, exc_value, traceback)\n\n    def __iter__(self) -> Iterator[bytes]:\n        raise NotImplementedError\n\n    def __next__(self) -> bytes:\n        raise NotImplementedError\n\n    @property\n    def mode(self) -> str:\n        \"\"\"Opening mode, which is always rb.\"\"\"\n        return \"rb\"\n\n    @property\n    def name(self) -> str:\n        \"\"\"Path to the underlying file.\"\"\"\n        return self._file.name\n\n    def seekable(self) -> bool:\n        \"\"\"Return whether random access is supported, which is True.\"\"\"\n        return True\n\n    def close(self) -> None:\n        \"\"\"Close the file.\"\"\"\n        self._file.close()\n\n    @property\n    def closed(self) -> bool:\n        \"\"\"Whether the file is closed.\"\"\"\n        return self._file.closed\n\n    def fileno(self) -> int:\n        return self._file.fileno()\n\n    def flush(self) -> None:\n        self._file.flush()\n\n    def isatty(self) -> bool:\n        return False\n\n    def readable(self) -> bool:\n        \"\"\"Return whether the file is readable, which is True.\"\"\"\n        return True\n\n    def read(self, size: int = -1) -> bytes:\n        \"\"\"Read up to size bytes from the object and return them.\n\n        As a convenience, if size is unspecified or -1,\n        all bytes until EOF are returned.  Fewer than\n        size bytes may be returned if EOF is reached.\n        \"\"\"\n        return self._file.read(size)\n\n    def readline(self, limit: int = -1) -> bytes:\n        # Explicit impl needed to satisfy mypy.\n        raise NotImplementedError\n\n    def readlines(self, hint: int = -1) -> list[bytes]:\n        raise NotImplementedError\n\n    def seek(self, offset: int, whence: int = 0) -> int:\n        \"\"\"Change stream position and return the new absolute position.\n\n        Seek to offset relative position indicated by whence:\n        * 0: Start of stream (the default).  pos should be >= 0;\n        * 1: Current position - pos may be negative;\n        * 2: End of stream - pos usually negative.\n        \"\"\"\n        return self._file.seek(offset, whence)\n\n    def tell(self) -> int:\n        \"\"\"Return the current position.\"\"\"\n        return self._file.tell()\n\n    def truncate(self, size: int | None = None) -> int:\n        \"\"\"Resize the stream to the given size in bytes.\n\n        If size is unspecified resize to the current position.\n        The current stream position isn't changed.\n\n        Return the new file size.\n        \"\"\"\n        return self._file.truncate(size)\n\n    def writable(self) -> bool:\n        \"\"\"Return False.\"\"\"\n        return False\n\n    def write(self, s: Any) -> int:\n        raise NotImplementedError\n\n    def writelines(self, lines: Iterable[Any]) -> None:\n        raise NotImplementedError\n\n\nU = TypeVar(\"U\", bound=\"LazyFileOverHTTP\")\n\n\nclass LazyFileOverHTTP(ReadOnlyIOWrapper):\n    \"\"\"File-like object representing a fixed-length file over HTTP.\n\n    This uses HTTP range requests to lazily fetch the file's content into a temporary\n    file. If such requests are not supported by the server, raises\n    ``HTTPRangeRequestUnsupported`` in the ``__enter__`` method.\"\"\"\n\n    def __init__(\n        self,\n        url: str,\n        session: Session | Authenticator,\n        delete_backing_file: bool = True,\n    ) -> None:\n        super().__init__(cast(BinaryIO, NamedTemporaryFile(delete=delete_backing_file)))\n\n        self._merge_intervals: MergeIntervals | None = None\n        self._length: int | None = None\n\n        self._request_count = 0\n...\n# Path: src/poetry/installation/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.installation.installer import Installer\n\n\n__all__ = [\"Installer\"]\n\n# Path: src/poetry/installation/chef.py\nfrom __future__ import annotations\n\nimport os\nimport tempfile\n\nfrom contextlib import redirect_stdout\nfrom io import StringIO\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\n\nfrom build import BuildBackendException\nfrom build import ProjectBuilder\nfrom build.env import IsolatedEnv as BaseIsolatedEnv\nfrom poetry.core.utils.helpers import temporary_directory\nfrom pyproject_hooks import quiet_subprocess_runner  # type: ignore[import-untyped]\n\nfrom poetry.utils._compat import decode\nfrom poetry.utils.env import ephemeral_environment\nfrom poetry.utils.helpers import extractall\n\n\nif TYPE_CHECKING:\n    from collections.abc import Collection\n\n    from poetry.repositories import RepositoryPool\n    from poetry.utils.cache import ArtifactCache\n    from poetry.utils.env import Env\n\n\nclass ChefError(Exception): ...\n\n\nclass ChefBuildError(ChefError): ...\n\n\nclass ChefInstallError(ChefError):\n    def __init__(self, requirements: Collection[str], output: str, error: str) -> None:\n        message = \"\\n\\n\".join(\n            (\n                f\"Failed to install {', '.join(requirements)}.\",\n                f\"Output:\\n{output}\",\n                f\"Error:\\n{error}\",\n            )\n        )\n        super().__init__(message)\n        self._requirements = requirements\n\n    @property\n    def requirements(self) -> Collection[str]:\n        return self._requirements\n\n\nclass IsolatedEnv(BaseIsolatedEnv):\n    def __init__(self, env: Env, pool: RepositoryPool) -> None:\n        self._env = env\n        self._pool = pool\n\n    @property\n    def python_executable(self) -> str:\n        return str(self._env.python)\n\n    def make_extra_environ(self) -> dict[str, str]:\n        path = os.environ.get(\"PATH\")\n        scripts_dir = str(self._env._bin_dir)\n        return {\n            \"PATH\": (\n                os.pathsep.join([scripts_dir, path])\n                if path is not None\n                else scripts_dir\n            )\n        }\n\n    def install(self, requirements: Collection[str]) -> None:\n        from cleo.io.buffered_io import BufferedIO\n        from poetry.core.packages.dependency import Dependency\n        from poetry.core.packages.project_package import ProjectPackage\n\n        from poetry.config.config import Config\n        from poetry.installation.installer import Installer\n        from poetry.packages.locker import Locker\n        from poetry.repositories.installed_repository import InstalledRepository\n\n        # We build Poetry dependencies from the requirements\n        package = ProjectPackage(\"__root__\", \"0.0.0\")\n        package.python_versions = \".\".join(str(v) for v in self._env.version_info[:3])\n        for requirement in requirements:\n            dependency = Dependency.create_from_pep_508(requirement)\n            package.add_dependency(dependency)\n\n        io = BufferedIO()\n        installer = Installer(\n            io,\n            self._env,\n            package,\n            Locker(self._env.path.joinpath(\"poetry.lock\"), {}),\n            self._pool,\n            Config.create(),\n            InstalledRepository.load(self._env),\n        )\n        installer.update(True)\n        if installer.run() != 0:\n            raise ChefInstallError(requirements, io.fetch_output(), io.fetch_error())\n\n\nclass Chef:\n    def __init__(\n        self, artifact_cache: ArtifactCache, env: Env, pool: RepositoryPool\n    ) -> None:\n        self._env = env\n        self._pool = pool\n        self._artifact_cache = artifact_cache\n\n    def prepare(\n        self, archive: Path, output_dir: Path | None = None, *, editable: bool = False\n    ) -> Path:\n        if not self._should_prepare(archive):\n            return archive\n\n        if archive.is_dir():\n            destination = output_dir or Path(tempfile.mkdtemp(prefix=\"poetry-chef-\"))\n            return self._prepare(archive, destination=destination, editable=editable)\n\n        return self._prepare_sdist(archive, destination=output_dir)\n\n    def _prepare(\n        self, directory: Path, destination: Path, *, editable: bool = False\n    ) -> Path:\n        from subprocess import CalledProcessError\n\n        with ephemeral_environment(\n            self._env.python,\n            flags={\"no-pip\": True, \"no-setuptools\": True, \"no-wheel\": True},\n        ) as venv:\n            env = IsolatedEnv(venv, self._pool)\n            builder = ProjectBuilder.from_isolated_env(\n                env, directory, runner=quiet_subprocess_runner\n            )\n            env.install(builder.build_system_requires)\n\n            stdout = StringIO()\n            error: Exception | None = None\n            try:\n                with redirect_stdout(stdout):\n                    dist_format = \"wheel\" if not editable else \"editable\"\n                    env.install(\n                        builder.build_system_requires\n                        | builder.get_requires_for_build(dist_format)\n                    )\n                    path = Path(\n                        builder.build(\n                            dist_format,\n                            destination.as_posix(),\n                        )\n                    )\n            except BuildBackendException as e:\n                message_parts = [str(e)]\n                if isinstance(e.exception, CalledProcessError):\n                    text = e.exception.stderr or e.exception.stdout\n                    if text is not None:\n                        message_parts.append(decode(text))\n                else:\n                    message_parts.append(str(e.exception))\n\n                error = ChefBuildError(\"\\n\\n\".join(message_parts))\n\n            if error is not None:\n                raise error from None\n\n            return path\n\n    def _prepare_sdist(self, archive: Path, destination: Path | None = None) -> Path:\n        from poetry.core.packages.utils.link import Link\n\n        suffix = archive.suffix\n        zip = suffix == \".zip\"\n\n        with temporary_directory() as tmp_dir:\n            archive_dir = Path(tmp_dir)\n            extractall(source=archive, dest=archive_dir, zip=zip)\n\n            elements = list(archive_dir.glob(\"*\"))\n\n            if len(elements) == 1 and elements[0].is_dir():\n                sdist_dir = elements[0]\n            else:\n                sdist_dir = archive_dir / archive.name.rstrip(suffix)\n                if not sdist_dir.is_dir():\n                    sdist_dir = archive_dir\n\n            if destination is None:\n                destination = self._artifact_cache.get_cache_directory_for_link(\n                    Link(archive.as_uri())\n                )\n\n            destination.mkdir(parents=True, exist_ok=True)\n\n            return self._prepare(\n                sdist_dir,\n                destination,\n            )\n\n    def _should_prepare(self, archive: Path) -> bool:\n        return archive.is_dir() or not self._is_wheel(archive)\n\n    @classmethod\n    def _is_wheel(cls, archive: Path) -> bool:\n        return archive.suffix == \".whl\"\n\n# Path: src/poetry/installation/chooser.py\nfrom __future__ import annotations\n\nimport logging\nimport re\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom poetry.config.config import Config\nfrom poetry.config.config import PackageFilterPolicy\nfrom poetry.repositories.http_repository import HTTPRepository\nfrom poetry.utils.helpers import get_highest_priority_hash_type\nfrom poetry.utils.wheel import Wheel\n\n\nif TYPE_CHECKING:\n    from poetry.core.constraints.version import Version\n    from poetry.core.packages.package import Package\n    from poetry.core.packages.utils.link import Link\n\n    from poetry.repositories.repository_pool import RepositoryPool\n    from poetry.utils.env import Env\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass Chooser:\n    \"\"\"\n    A Chooser chooses an appropriate release archive for packages.\n    \"\"\"\n\n    def __init__(\n        self, pool: RepositoryPool, env: Env, config: Config | None = None\n    ) -> None:\n        self._pool = pool\n        self._env = env\n        self._config = config or Config.create()\n        self._no_binary_policy: PackageFilterPolicy = PackageFilterPolicy(\n            self._config.get(\"installer.no-binary\", [])\n        )\n\n    def choose_for(self, package: Package) -> Link:\n        \"\"\"\n        Return the url of the selected archive for a given package.\n        \"\"\"\n        links = []\n        for link in self._get_links(package):\n            if link.is_wheel:\n                if not self._no_binary_policy.allows(package.name):\n                    logger.debug(\n                        \"Skipping wheel for %s as requested in no binary policy for\"\n                        \" package (%s)\",\n                        link.filename,\n                        package.name,\n                    )\n                    continue\n\n                if not Wheel(link.filename).is_supported_by_environment(self._env):\n                    logger.debug(\n                        \"Skipping wheel %s as this is not supported by the current\"\n                        \" environment\",\n                        link.filename,\n                    )\n                    continue\n\n            if link.ext in {\".egg\", \".exe\", \".msi\", \".rpm\", \".srpm\"}:\n                logger.debug(\"Skipping unsupported distribution %s\", link.filename)\n                continue\n\n            links.append(link)\n\n        if not links:\n            raise RuntimeError(f\"Unable to find installation candidates for {package}\")\n\n        # Get the best link\n        chosen = max(links, key=lambda link: self._sort_key(package, link))\n\n        return chosen\n\n    def _get_links(self, package: Package) -> list[Link]:\n        if package.source_type:\n            assert package.source_reference is not None\n            repository = self._pool.repository(package.source_reference)\n\n        elif not self._pool.has_repository(\"pypi\"):\n            repository = self._pool.repositories[0]\n        else:\n            repository = self._pool.repository(\"pypi\")\n        links = repository.find_links_for_package(package)\n\n        locked_hashes = {f[\"hash\"] for f in package.files}\n        if not locked_hashes:\n            return links\n\n        selected_links = []\n        skipped = []\n        locked_hash_names = {h.split(\":\")[0] for h in locked_hashes}\n        for link in links:\n            if not link.hashes:\n                selected_links.append(link)\n                continue\n\n            link_hash: str | None = None\n            if (candidates := locked_hash_names.intersection(link.hashes.keys())) and (\n                hash_name := get_highest_priority_hash_type(candidates, link.filename)\n            ):\n                link_hash = f\"{hash_name}:{link.hashes[hash_name]}\"\n\n            elif isinstance(repository, HTTPRepository):\n                link_hash = repository.calculate_sha256(link)\n\n            if link_hash not in locked_hashes:\n                skipped.append((link.filename, link_hash))\n                logger.debug(\n                    \"Skipping %s as %s checksum does not match expected value\",\n                    link.filename,\n                    link_hash,\n                )\n                continue\n\n            selected_links.append(link)\n\n        if links and not selected_links:\n            links_str = \", \".join(f\"{link}({h})\" for link, h in skipped)\n            raise RuntimeError(\n                f\"Retrieved digests for links {links_str} not in poetry.lock\"\n                f\" metadata {locked_hashes}\"\n            )\n\n        return selected_links\n\n    def _sort_key(\n        self, package: Package, link: Link\n    ) -> tuple[int, int, int, Version, tuple[Any, ...], int]:\n        \"\"\"\n        Function to pass as the `key` argument to a call to sorted() to sort\n        InstallationCandidates by preference.\n        Returns a tuple such that tuples sorting as greater using Python's\n        default comparison operator are more preferred.\n        The preference is as follows:\n        First and foremost, candidates with allowed (matching) hashes are\n        always preferred over candidates without matching hashes. This is\n        because e.g. if the only candidate with an allowed hash is yanked,\n        we still want to use that candidate.\n        Second, excepting hash considerations, candidates that have been\n        yanked (in the sense of PEP 592) are always less preferred than\n        candidates that haven't been yanked. Then:\n        If not finding wheels, they are sorted by version only.\n        If finding wheels, then the sort order is by version, then:\n          1. existing installs\n          2. wheels ordered via Wheel.support_index_min(self._supported_tags)\n          3. source archives\n        If prefer_binary was set, then all wheels are sorted above sources.\n        Note: it was considered to embed this logic into the Link\n              comparison operators, but then different sdist links\n              with the same version, would have to be considered equal\n        \"\"\"\n        build_tag: tuple[Any, ...] = ()\n        binary_preference = 0\n        if link.is_wheel:\n            wheel = Wheel(link.filename)\n            if not wheel.is_supported_by_environment(self._env):\n                raise RuntimeError(\n                    f\"{wheel.filename} is not a supported wheel for this platform. It \"\n                    \"can't be sorted.\"\n                )\n\n            # TODO: Binary preference\n            pri = -(wheel.get_minimum_supported_index(self._env.supported_tags) or 0)\n            if wheel.build_tag is not None:\n                match = re.match(r\"^(\\d+)(.*)$\", wheel.build_tag)\n                if not match:\n                    raise ValueError(f\"Unable to parse build tag: {wheel.build_tag}\")\n                build_tag_groups = match.groups()\n                build_tag = (int(build_tag_groups[0]), build_tag_groups[1])\n        else:  # sdist\n            support_num = len(self._env.supported_tags)\n            pri = -support_num\n\n        has_allowed_hash = int(self._is_link_hash_allowed_for_package(link, package))\n\n        yank_value = int(not link.yanked)\n\n        return (\n            has_allowed_hash,\n            yank_value,\n            binary_preference,\n            package.version,\n            build_tag,\n            pri,\n        )\n\n    def _is_link_hash_allowed_for_package(self, link: Link, package: Package) -> bool:\n        if not link.hashes:\n            return True\n\n        link_hashes = {f\"{name}:{h}\" for name, h in link.hashes.items()}\n        locked_hashes = {f[\"hash\"] for f in package.files}\n\n        return bool(link_hashes & locked_hashes)\n\n# Path: src/poetry/installation/executor.py\nfrom __future__ import annotations\n\nimport contextlib\nimport csv\nimport functools\nimport itertools\nimport json\nimport threading\n\nfrom concurrent.futures import ThreadPoolExecutor\nfrom concurrent.futures import wait\nfrom pathlib import Path\nfrom subprocess import CalledProcessError\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom cleo.io.null_io import NullIO\nfrom poetry.core.packages.utils.link import Link\n\nfrom poetry.installation.chef import Chef\nfrom poetry.installation.chef import ChefBuildError\nfrom poetry.installation.chef import ChefInstallError\nfrom poetry.installation.chooser import Chooser\nfrom poetry.installation.operations import Install\nfrom poetry.installation.operations import Uninstall\nfrom poetry.installation.operations import Update\nfrom poetry.installation.wheel_installer import WheelInstaller\nfrom poetry.puzzle.exceptions import SolverProblemError\nfrom poetry.utils._compat import decode\nfrom poetry.utils.authenticator import Authenticator\nfrom poetry.utils.env import EnvCommandError\nfrom poetry.utils.helpers import Downloader\nfrom poetry.utils.helpers import get_file_hash\nfrom poetry.utils.helpers import get_highest_priority_hash_type\nfrom poetry.utils.helpers import pluralize\nfrom poetry.utils.helpers import remove_directory\nfrom poetry.utils.pip import pip_install\n\n\nif TYPE_CHECKING:\n    from cleo.io.io import IO\n    from cleo.io.outputs.section_output import SectionOutput\n    from poetry.core.masonry.builders.builder import Builder\n    from poetry.core.packages.package import Package\n\n    from poetry.config.config import Config\n    from poetry.installation.operations.operation import Operation\n    from poetry.repositories import RepositoryPool\n    from poetry.utils.env import Env\n\n\nclass Executor:\n    def __init__(\n        self,\n        env: Env,\n        pool: RepositoryPool,\n        config: Config,\n        io: IO,\n        parallel: bool | None = None,\n        disable_cache: bool = False,\n    ) -> None:\n        self._env = env\n        self._io = io\n        self._dry_run = False\n        self._enabled = True\n        self._verbose = False\n        self._wheel_installer = WheelInstaller(self._env)\n        self._use_modern_installation = config.get(\n            \"installer.modern-installation\", True\n        )\n        if not self._use_modern_installation:\n            self._io.write_line(\n                \"<warning>Warning: Setting `installer.modern-installation` to `false` \"\n                \"is deprecated.</>\"\n            )\n            self._io.write_line(\n                \"<warning>The pip-based installer will be removed in a future release.</>\"\n            )\n            self._io.write_line(\n                \"<warning>See https://github.com/python-poetry/poetry/issues/8987.</>\"\n            )\n\n        if parallel is None:\n            parallel = config.get(\"installer.parallel\", True)\n\n        if parallel:\n            self._max_workers = config.installer_max_workers\n        else:\n            self._max_workers = 1\n\n        self._artifact_cache = pool.artifact_cache\n        self._authenticator = Authenticator(\n            config, self._io, disable_cache=disable_cache, pool_size=self._max_workers\n        )\n        self._chef = Chef(self._artifact_cache, self._env, pool)\n        self._chooser = Chooser(pool, self._env, config)\n\n        self._executor = ThreadPoolExecutor(max_workers=self._max_workers)\n        self._total_operations = 0\n        self._executed_operations = 0\n        self._executed = {\"install\": 0, \"update\": 0, \"uninstall\": 0}\n        self._skipped = {\"install\": 0, \"update\": 0, \"uninstall\": 0}\n        self._sections: dict[int, SectionOutput] = {}\n        self._yanked_warnings: list[str] = []\n        self._lock = threading.Lock()\n        self._shutdown = False\n        self._hashes: dict[str, str] = {}\n\n    @property\n    def installations_count(self) -> int:\n        return self._executed[\"install\"]\n\n    @property\n    def updates_count(self) -> int:\n        return self._executed[\"update\"]\n\n    @property\n    def removals_count(self) -> int:\n        return self._executed[\"uninstall\"]\n\n    @property\n    def enabled(self) -> bool:\n        return self._enabled\n\n    def supports_fancy_output(self) -> bool:\n        return self._io.output.is_decorated() and not self._dry_run\n\n    def disable(self) -> Executor:\n        self._enabled = False\n\n        return self\n\n    def dry_run(self, dry_run: bool = True) -> Executor:\n        self._dry_run = dry_run\n\n        return self\n\n    def verbose(self, verbose: bool = True) -> Executor:\n        self._verbose = verbose\n\n        return self\n\n    def enable_bytecode_compilation(self, enable: bool = True) -> None:\n        self._wheel_installer.enable_bytecode_compilation(enable)\n\n    def pip_install(\n        self, req: Path, upgrade: bool = False, editable: bool = False\n    ) -> int:\n        try:\n            pip_install(req, self._env, upgrade=upgrade, editable=editable)\n        except EnvCommandError as e:\n            output = decode(e.e.output)\n            if (\n                \"KeyboardInterrupt\" in output\n                or \"ERROR: Operation cancelled by user\" in output\n            ):\n                return -2\n            raise\n\n        return 0\n\n    def execute(self, operations: list[Operation]) -> int:\n        self._total_operations = len(operations)\n        for job_type in self._executed:\n            self._executed[job_type] = 0\n            self._skipped[job_type] = 0\n\n        if operations and (self._enabled or self._dry_run):\n            self._display_summary(operations)\n\n        self._sections = {}\n        self._yanked_warnings = []\n\n        # pip has to be installed first without parallelism if we install via pip\n        for i, op in enumerate(operations):\n            if op.package.name == \"pip\":\n                wait([self._executor.submit(self._execute_operation, op)])\n                del operations[i]\n                break\n\n        # We group operations by priority\n        groups = itertools.groupby(operations, key=lambda o: -o.priority)\n        for _, group in groups:\n            tasks = []\n            serial_operations = []\n            for operation in group:\n                if self._shutdown:\n                    break\n\n                # Some operations are unsafe, we must execute them serially in a group\n                # https://github.com/python-poetry/poetry/issues/3086\n                # https://github.com/python-poetry/poetry/issues/2658\n                #\n                # We need to explicitly check source type here, see:\n                # https://github.com/python-poetry/poetry-core/pull/98\n                is_parallel_unsafe = operation.job_type == \"uninstall\" or (\n                    operation.package.develop\n                    and operation.package.source_type in {\"directory\", \"git\"}\n                )\n                if not operation.skipped and is_parallel_unsafe:\n                    serial_operations.append(operation)\n                    continue\n\n                tasks.append(self._executor.submit(self._execute_operation, operation))\n\n            try:\n                wait(tasks)\n\n                for operation in serial_operations:\n                    wait([self._executor.submit(self._execute_operation, operation)])\n\n            except KeyboardInterrupt:\n                self._shutdown = True\n\n            if self._shutdown:\n                # Cancelling further tasks from being executed\n                [task.cancel() for task in tasks]\n                self._executor.shutdown(wait=True)\n\n                break\n\n        for warning in self._yanked_warnings:\n            self._io.write_error_line(f\"<warning>Warning: {warning}</warning>\")\n        for path, issues in self._wheel_installer.invalid_wheels.items():\n            formatted_issues = \"\\n\".join(issues)\n            warning = (\n                f\"Validation of the RECORD file of {path.name} failed.\"\n                \" Please report to the maintainers of that package so they can fix\"\n                f\" their build process. Details:\\n{formatted_issues}\\n\"\n            )\n            self._io.write_error_line(f\"<warning>Warning: {warning}</warning>\")\n\n        return 1 if self._shutdown else 0\n\n    def _write(self, operation: Operation, line: str) -> None:\n        if not self.supports_fancy_output() or not self._should_write_operation(\n            operation\n        ):\n            return\n\n        if self._io.is_debug():\n            with self._lock:\n                section = self._sections[id(operation)]\n                section.write_line(line)\n\n            return\n\n        with self._lock:\n            section = self._sections[id(operation)]\n            section.clear()\n            section.write(line)\n\n    def _execute_operation(self, operation: Operation) -> None:\n        try:\n            op_message = self.get_operation_message(operation)\n            if self.supports_fancy_output():\n                if id(operation) not in self._sections and self._should_write_operation(\n                    operation\n                ):\n                    with self._lock:\n                        self._sections[id(operation)] = self._io.section()\n                        self._sections[id(operation)].write_line(\n                            f\"  <fg=blue;options=bold>-</> {op_message}:\"\n                            \" <fg=blue>Pending...</>\"\n                        )\n            else:\n                if self._should_write_operation(operation):\n                    if not operation.skipped:\n                        self._io.write_line(\n                            f\"  <fg=blue;options=bold>-</> {op_message}\"\n                        )\n                    else:\n                        self._io.write_line(\n                            f\"  <fg=default;options=bold,dark>-</> {op_message}: \"\n                            \"<fg=default;options=bold,dark>Skipped</> \"\n                            \"<fg=default;options=dark>for the following reason:</> \"\n                            f\"<fg=default;options=bold,dark>{operation.skip_reason}</>\"\n                        )\n\n            try:\n                result = self._do_execute_operation(operation)\n            except EnvCommandError as e:\n                if e.e.returncode == -2:\n                    result = -2\n                else:\n                    raise\n\n            # If we have a result of -2 it means a KeyboardInterrupt\n            # in the any python subprocess, so we raise a KeyboardInterrupt\n            # error to be picked up by the error handler.\n            if result == -2:\n                raise KeyboardInterrupt\n        except Exception as e:\n            try:\n                from cleo.ui.exception_trace import ExceptionTrace\n\n                io: IO | SectionOutput\n                if not self.supports_fancy_output():\n                    io = self._io\n                else:\n                    message = (\n                        \"  <error>-</error>\"\n                        f\" {self.get_operation_message(operation, error=True)}:\"\n                        \" <error>Failed</error>\"\n                    )\n                    self._write(operation, message)\n                    io = self._sections.get(id(operation), self._io)\n\n                with self._lock:\n                    trace = ExceptionTrace(e)\n                    trace.render(io)\n                    pkg = operation.package\n                    if isinstance(e, ChefBuildError):\n                        pip_command = \"pip wheel --no-cache-dir --use-pep517\"\n                        if pkg.develop:\n                            requirement = pkg.source_url\n                            pip_command += \" --editable\"\n                        else:\n                            requirement = (\n                                pkg.to_dependency().to_pep_508().split(\";\")[0].strip()\n                            )\n                        message = (\n                            \"<info>\"\n                            \"Note: This error originates from the build backend,\"\n                            \" and is likely not a problem with poetry\"\n                            f\" but with {pkg.pretty_name} ({pkg.full_pretty_version})\"\n                            \" not supporting PEP 517 builds. You can verify this by\"\n                            f\" running '{pip_command} \\\"{requirement}\\\"'.\"\n                            \"</info>\"\n                        )\n                    elif isinstance(e, ChefInstallError):\n                        message = (\n                            \"<error>\"\n                            \"Cannot install build-system.requires\"\n                            f\" for {pkg.pretty_name}.\"\n                            \"</error>\"\n                        )\n                    elif isinstance(e, SolverProblemError):\n                        message = (\n                            \"<error>\"\n                            \"Cannot resolve build-system.requires\"\n                            f\" for {pkg.pretty_name}.\"\n                            \"</error>\"\n                        )\n                    else:\n                        message = f\"<error>Cannot install {pkg.pretty_name}.</error>\"\n\n                    io.write_line(\"\")\n                    io.write_line(message)\n                    io.write_line(\"\")\n            finally:\n                with self._lock:\n                    self._shutdown = True\n\n        except KeyboardInterrupt:\n            try:\n                message = (\n                    \"  <warning>-</warning>\"\n                    f\" {self.get_operation_message(operation, warning=True)}:\"\n                    \" <warning>Cancelled</warning>\"\n                )\n                if not self.supports_fancy_output():\n                    self._io.write_line(message)\n                else:\n                    self._write(operation, message)\n            finally:\n                with self._lock:\n                    self._shutdown = True\n\n    \ndef _do_execute_operation(self, operation: Operation) -> int:\n        method = operation.job_type\n\n        operation_message = self.get_operation_message(operation)\n        if operation.skipped:\n            if self.supports_fancy_output():\n                self._write(\n                    operation,\n                    f\"  <fg=default;options=bold,dark>-</> {operation_message}: \"\n                    \"<fg=default;options=bold,dark>Skipped</> \"\n                    \"<fg=default;options=dark>for the following reason:</> \"\n                    f\"<fg=default;options=bold,dark>{operation.skip_reason}</>\",\n                )\n\n            self._skipped[operation.job_type] += 1\n\n            return 0\n\n        if not self._enabled or self._dry_run:\n            return 0\n\n        result: int = getattr(self, f\"_execute_{method}\")(operation)\n\n        if result != 0:\n            return result\n\n        operation_message = self.get_operation_message(operation, done=True)\n        message = f\"  <fg=green;options=bold>-</> {operation_message}\"\n        self._write(operation, message)\n\n        self._increment_operations_count(operation, True)\n\n        return result\n\n    def _increment_operations_count(self, operation: Operation, executed: bool) -> None:\n        with self._lock:\n            if executed:\n                self._executed_operations += 1\n                self._executed[operation.job_type] += 1\n            else:\n                self._skipped[operation.job_type] += 1\n\n    def run_pip(self, *args: Any, **kwargs: Any) -> int:\n        try:\n            self._env.run_pip(*args, **kwargs)\n        except EnvCommandError as e:\n            output = decode(e.e.output)\n            if (\n                \"KeyboardInterrupt\" in output\n                or \"ERROR: Operation cancelled by user\" in output\n            ):\n                return -2\n\n            raise\n\n        return 0\n\n    def get_operation_message(\n        self,\n        operation: Operation,\n        done: bool = False,\n        error: bool = False,\n        warning: bool = False,\n    ) -> str:\n        base_tag = \"fg=default\"\n        operation_color = \"c2\"\n        source_operation_color = \"c2\"\n        package_color = \"c1\"\n\n        if error:\n            operation_color = \"error\"\n        elif warning:\n            operation_color = \"warning\"\n        elif done:\n            operation_color = \"success\"\n\n        if operation.skipped:\n            base_tag = \"fg=default;options=dark\"\n            operation_color += \"_dark\"\n            source_operation_color += \"_dark\"\n            package_color += \"_dark\"\n\n        if isinstance(operation, Install):\n            return (\n                f\"<{base_tag}>Installing\"\n                f\" <{package_color}>{operation.package.name}</{package_color}>\"\n                f\" (<{operation_color}>{operation.package.full_pretty_version}</>)</>\"\n            )\n\n        if isinstance(operation, Uninstall):\n            return (\n                f\"<{base_tag}>Removing\"\n                f\" <{package_color}>{operation.package.name}</{package_color}>\"\n                f\" (<{operation_color}>{operation.package.full_pretty_version}</>)</>\"\n            )\n\n        if isinstance(operation, Update):\n            initial_version = (initial_pkg := operation.initial_package).version\n            target_version = (target_pkg := operation.target_package).version\n            update_kind = (\n                \"Updating\" if target_version >= initial_version else \"Downgrading\"\n            )\n            return (\n                f\"<{base_tag}>{update_kind}\"\n                f\" <{package_color}>{initial_pkg.name}</{package_color}> \"\n                f\"(<{source_operation_color}>\"\n                f\"{initial_pkg.full_pretty_version}\"\n                f\"</{source_operation_color}> -> <{operation_color}>\"\n                f\"{target_pkg.full_pretty_version}</>)</>\"\n            )\n        return \"\"\n\n    def _display_summary(self, operations: list[Operation]) -> None:\n        installs = 0\n        updates = 0\n        uninstalls = 0\n        skipped = 0\n        for op in operations:\n            if op.skipped:\n                skipped += 1\n                continue\n\n            if op.job_type == \"install\":\n                installs += 1\n            elif op.job_type == \"update\":\n                updates += 1\n            elif op.job_type == \"uninstall\":\n                uninstalls += 1\n\n        if not installs and not updates and not uninstalls and not self._verbose:\n            self._io.write_line(\"\")\n            self._io.write_line(\"No dependencies to install or update\")\n\n            return\n\n        self._io.write_line(\"\")\n        self._io.write(\"<b>Package operations</b>: \")\n        self._io.write(f\"<info>{installs}</> install{pluralize(installs)}, \")\n        self._io.write(f\"<info>{updates}</> update{pluralize(updates)}, \")\n        self._io.write(f\"<info>{uninstalls}</> removal{pluralize(uninstalls)}\")\n        if skipped and self._verbose:\n            self._io.write(f\", <info>{skipped}</> skipped\")\n        self._io.write_line(\"\")\n        self._io.write_line(\"\")\n\n    def _execute_install(self, operation: Install | Update) -> int:\n        status_code = self._install(operation)\n\n        self._save_url_reference(operation)\n\n        return status_code\n\n    def _execute_update(self, operation: Install | Update) -> int:\n        status_code = self._update(operation)\n\n        self._save_url_reference(operation)\n\n        return status_code\n\n    def _execute_uninstall(self, operation: Uninstall) -> int:\n        op_msg = self.get_operation_message(operation)\n        message = f\"  <fg=blue;options=bold>-</> {op_msg}: <info>Removing...</info>\"\n        self._write(operation, message)\n\n        return self._remove(operation.package)\n\n    def _install(self, operation: Install | Update) -> int:\n        package = operation.package\n        if package.source_type == \"directory\" and not self._use_modern_installation:\n            return self._install_directory_without_wheel_installer(operation)\n\n        cleanup_archive: bool = False\n        if package.source_type == \"git\":\n            archive = self._prepare_git_archive(operation)\n            cleanup_archive = operation.package.develop\n        elif package.source_type == \"file\":\n            archive = self._prepare_archive(operation)\n        elif package.source_type == \"directory\":\n            archive = self._prepare_archive(operation)\n            cleanup_archive = True\n        elif package.source_type == \"url\":\n            assert package.source_url is not None\n            archive = self._download_link(operation, Link(package.source_url))\n        else:\n            archive = self._download(operation)\n\n        operation_message = self.get_operation_message(operation)\n        message = (\n            f\"  <fg=blue;options=bold>-</> {operation_message}:\"\n            \" <info>Installing...</info>\"\n        )\n        self._write(operation, message)\n\n        if not self._use_modern_installation:\n            return self.pip_install(archive, upgrade=operation.job_type == \"update\")\n\n        try:\n            if operation.job_type == \"update\":\n                # Uninstall first\n                # TODO: Make an uninstaller and find a way to rollback in case\n                # the new package can't be installed\n                assert isinstance(operation, Update)\n                self._remove(operation.initial_package)\n\n            self._wheel_installer.install(archive)\n        finally:\n            if cleanup_archive:\n                archive.unlink()\n\n        return 0\n\n    def _update(self, operation: Install | Update) -> int:\n        return self._install(operation)\n\n    def _remove(self, package: Package) -> int:\n        # If we have a VCS package, remove its source directory\n        if package.source_type == \"git\":\n            src_dir = self._env.path / \"src\" / package.name\n            if src_dir.exists():\n                remove_directory(src_dir, force=True)\n\n        try:\n            return self.run_pip(\"uninstall\", package.name, \"-y\")\n        except CalledProcessError as e:\n            if \"not installed\" in str(e):\n                return 0\n\n            raise\n\n    def _prepare_archive(\n        self, operation: Install | Update, *, output_dir: Path | None = None\n    ) -> Path:\n        package = operation.package\n        operation_message = self.get_operation_message(operation)\n\n        message = (\n            f\"  <fg=blue;options=bold>-</> {operation_message}:\"\n            \" <info>Preparing...</info>\"\n        )\n        self._write(operation, message)\n\n        assert package.source_url is not None\n        archive = Path(package.source_url)\n        if package.source_subdirectory:\n            archive = archive / package.source_subdirectory\n        if not Path(package.source_url).is_absolute() and package.root_dir:\n            archive = package.root_dir / archive\n\n        self._populate_hashes_dict(archive, package)\n\n        return self._chef.prepare(\n            archive, editable=package.develop, output_dir=output_dir\n        )\n\n    def _prepare_git_archive(self, operation: Install | Update) -> Path:\n        from poetry.vcs.git import Git\n\n        package = operation.package\n        assert package.source_url is not None\n\n        if package.source_resolved_reference and not package.develop:\n            # Only cache git archives when we know precise reference hash,\n            # otherwise we might get stale archives\n            cached_archive = self._artifact_cache.get_cached_archive_for_git(\n                package.source_url,\n                package.source_resolved_reference,\n                package.source_subdirectory,\n                env=self._env,\n            )\n            if cached_archive is not None:\n                return cached_archive\n\n        operation_message = self.get_operation_message(operation)\n\n        message = (\n            f\"  <fg=blue;options=bold>-</> {operation_message}: <info>Cloning...</info>\"\n        )\n        self._write(operation, message)\n\n        source = Git.clone(\n            url=package.source_url,\n            source_root=self._env.path / \"src\",\n            revision=package.source_resolved_reference or package.source_reference,\n        )\n\n        # Now we just need to install from the source directory\n        original_url = package.source_url\n        package._source_url = str(source.path)\n\n        output_dir = None\n        if package.source_resolved_reference and not package.develop:\n            output_dir = self._artifact_cache.get_cache_directory_for_git(\n                original_url,\n                package.source_resolved_reference,\n                package.source_subdirectory,\n            )\n\n        archive = self._prepare_archive(operation, output_dir=output_dir)\n        if not package.develop:\n            package._source_url = original_url\n\n        if output_dir is not None and output_dir.is_dir():\n            # Mark directories with cached git packages, to distinguish from\n            # \"normal\" cache\n            (output_dir / \".created_from_git_dependency\").touch()\n\n        return archive\n\n    def _install_directory_without_wheel_installer(\n        self, operation: Install | Update\n    ) -> int:\n        from poetry.factory import Factory\n        from poetry.pyproject.toml import PyProjectTOML\n\n        package = operation.package\n        operation_message = self.get_operation_message(operation)\n\n        message = (\n            f\"  <fg=blue;options=bold>-</> {operation_message}:\"\n            \" <info>Building...</info>\"\n        )\n        self._write(operation, message)\n\n        assert package.source_url is not None\n        if package.root_dir:\n            req = package.root_dir / package.source_url\n        else:\n            req = Path(package.source_url).resolve(strict=False)\n\n        if package.source_subdirectory:\n            req /= package.source_subdirectory\n\n        pyproject = PyProjectTOML(req / \"pyproject.toml\")\n\n        package_poetry = None\n        if pyproject.is_poetry_project():\n            with contextlib.suppress(RuntimeError):\n                package_poetry = Factory().create_poetry(pyproject.file.path.parent)\n\n        if package_poetry is not None:\n            builder: Builder\n            if package.develop and not package_poetry.package.build_script:\n                from poetry.masonry.builders.editable import EditableBuilder\n\n                # This is a Poetry package in editable mode\n                # we can use the EditableBuilder without going through pip\n                # to install it, unless it has a build script.\n                builder = EditableBuilder(package_poetry, self._env, NullIO())\n                builder.build()\n\n                return 0\n\n            if package_poetry.package.build_script:\n                from poetry.core.masonry.builders.sdist import SdistBuilder\n\n                builder = SdistBuilder(package_poetry)\n                with builder.setup_py():\n                    return self.pip_install(req, upgrade=True, editable=package.develop)\n\n        return self.pip_install(req, upgrade=True, editable=package.develop)\n\n    def _download(self, operation: Install | Update) -> Path:\n        link = self._chooser.choose_for(operation.package)\n\n        if link.yanked:\n            # Store yanked warnings in a list and print after installing, so they can't\n            # be overlooked. Further, printing them in the concerning section would have\n            # the risk of overwriting the warning, so it is only briefly visible.\n            message = (\n                f\"The file chosen for install of {operation.package.pretty_name} \"\n                f\"{operation.package.pretty_version} ({link.show_url}) is yanked.\"\n            )\n            if link.yanked_reason:\n                message += f\" Reason for being yanked: {link.yanked_reason}\"\n            self._yanked_warnings.append(message)\n\n        return self._download_link(operation, link)\n\n    def _download_link(self, operation: Install | Update, link: Link) -> Path:\n        package = operation.package\n\n        # Get original package for the link provided\n        download_func = functools.partial(self._download_archive, operation)\n        original_archive = self._artifact_cache.get_cached_archive_for_link(\n            link, strict=True, download_func=download_func\n        )\n\n        # Get potential higher prioritized cached archive, otherwise it will fall back\n        # to the original archive.\n        archive = self._artifact_cache.get_cached_archive_for_link(\n            link,\n            strict=False,\n            env=self._env,\n        )\n        if archive is None:\n            # Since we previously downloaded an archive, we now should have\n            # something cached that we can use here. The only case in which\n            # archive is None is if the original archive is not valid for the\n            # current environment.\n            raise RuntimeError(\n                f\"Package {link.url} cannot be installed in the current environment\"\n                f\" {self._env.marker_env}\"\n            )\n\n        if archive.suffix != \".whl\":\n            message = (\n                f\"  <fg=blue;options=bold>-</> {self.get_operation_message(operation)}:\"\n                \" <info>Preparing...</info>\"\n            )\n            self._write(operation, message)\n\n            archive = self._chef.prepare(archive, output_dir=original_archive.parent)\n\n        # Use the original archive to provide the correct hash.\n        self._populate_hashes_dict(original_archive, package)\n\n        return archive\n\n    def _populate_hashes_dict(self, archive: Path, package: Package) -> None:\n        if package.files and archive.name in {f[\"file\"] for f in package.files}:\n            archive_hash = self._validate_archive_hash(archive, package)\n            self._hashes[package.name] = archive_hash\n\n    @staticmethod\n    def _validate_archive_hash(archive: Path, package: Package) -> str:\n        known_hashes = {f[\"hash\"] for f in package.files if f[\"file\"] == archive.name}\n        hash_types = {t.split(\":\")[0] for t in known_hashes}\n        hash_type = get_highest_priority_hash_type(hash_types, archive.name)\n\n        if hash_type is None:\n            raise RuntimeError(\n                f\"No usable hash type(s) for {package} from archive\"\n                f\" {archive.name} found (known hashes: {known_hashes!s})\"\n            )\n\n        archive_hash = f\"{hash_type}:{get_file_hash(archive, hash_type)}\"\n\n        if archive_hash not in known_hashes:\n            raise RuntimeError(\n                f\"Hash for {package} from archive {archive.name} not found in\"\n                f\" known hashes (was: {archive_hash})\"\n            )\n\n        return archive_hash\n\n    def _download_archive(\n        self,\n        operation: Install | Update,\n        url: str,\n        dest: Path,\n    ) -> None:\n        downloader = Downloader(url, dest, self._authenticator)\n        wheel_size = downloader.total_size\n\n        operation_message = self.get_operation_message(operation)\n        message = (\n            f\"  <fg=blue;options=bold>-</> {operation_message}: <info>Downloading...</>\"\n        )\n        progress = None\n        if self.supports_fancy_output():\n            if wheel_size is None:\n                self._write(operation, message)\n            else:\n                from cleo.ui.progress_bar import ProgressBar\n\n                progress = ProgressBar(\n                    self._sections[id(operation)], max=int(wheel_size)\n                )\n                progress.set_format(message + \" <b>%percent%%</b>\")\n\n        if progress:\n            with self._lock:\n                self._sections[id(operation)].clear()\n                progress.start()\n\n        for fetched_size in downloader.download_with_progress(chunk_size=4096):\n            if progress:\n                with self._lock:\n                    progress.set_progress(fetched_size)\n\n        if progress:\n            with self._lock:\n                progress.finish()\n\n    def _should_write_operation(self, operation: Operation) -> bool:\n        return (\n            not operation.skipped or self._dry_run or self._verbose or not self._enabled\n        )\n\n    def _save_url_reference(self, operation: Operation) -> None:\n        \"\"\"\n        Create and store a PEP-610 `direct_url.json` file, if needed.\n        \"\"\"\n        if operation.job_type not in {\"install\", \"update\"}:\n            return\n\n        package = operation.package\n\n        if not package.source_url or package.source_type == \"legacy\":\n            if not self._use_modern_installation:\n                # Since we are installing from our own distribution cache pip will write\n                # a `direct_url.json` file pointing to the cache distribution.\n                #\n                # That's not what we want, so we remove the direct_url.json file, if it\n                # exists.\n                for (\n                    direct_url_json\n                ) in self._env.site_packages.find_distribution_direct_url_json_files(\n                    distribution_name=package.name, writable_only=True\n                ):\n                    direct_url_json.unlink(missing_ok=True)\n            return\n\n        url_reference: dict[str, Any] | None = None\n\n        if package.source_type == \"git\" and not package.develop:\n            url_reference = self._create_git_url_reference(package)\n        elif package.source_type in (\"directory\", \"git\"):\n            url_reference = self._create_directory_url_reference(package)\n        elif package.source_type == \"url\":\n            url_reference = self._create_url_url_reference(package)\n        elif package.source_type == \"file\":\n            url_reference = self._create_file_url_reference(package)\n\n        if url_reference:\n            for dist in self._env.site_packages.distributions(\n                name=package.name, writable_only=True\n            ):\n                dist_path = dist._path  # type: ignore[attr-defined]\n                assert isinstance(dist_path, Path)\n                url = dist_path / \"direct_url.json\"\n                url.write_text(json.dumps(url_reference), encoding=\"utf-8\")\n\n                record = dist_path / \"RECORD\"\n                if record.exists():\n                    with record.open(mode=\"a\", encoding=\"utf-8\", newline=\"\") as f:\n                        writer = csv.writer(f)\n                        path = url.relative_to(record.parent.parent)\n                        writer.writerow([str(path), \"\", \"\"])\n\n    def _create_git_url_reference(self, package: Package) -> dict[str, Any]:\n        reference = {\n            \"url\": package.source_url,\n            \"vcs_info\": {\n                \"vcs\": \"git\",\n                \"requested_revision\": package.source_reference,\n                \"commit_id\": package.source_resolved_reference,\n            },\n        }\n        if package.source_subdirectory:\n            reference[\"subdirectory\"] = package.source_subdirectory\n\n        return reference\n\n    def _create_url_url_reference(self, package: Package) -> dict[str, Any]:\n        archive_info = self._get_archive_info(package)\n\n        return {\"url\": package.source_url, \"archive_info\": archive_info}\n\n    def _create_file_url_reference(self, package: Package) -> dict[str, Any]:\n        archive_info = self._get_archive_info(package)\n\n        assert package.source_url is not None\n        return {\n            \"url\": Path(package.source_url).as_uri(),\n            \"archive_info\": archive_info,\n        }\n\n    def _create_directory_url_reference(self, package: Package) -> dict[str, Any]:\n        dir_info = {}\n\n        if package.develop:\n...\n", "needle_token_start": 10271, "needle_token_end": 10593, "code_context_ntokens": 16057, "output": ["```python\ndef _do_execute_operation(self, operation: Operation) -> int:\n    method = operation.job_type\n\n    operation_message = self.get_operation_message(operation)\n    if operation.skipped:\n        if self.supports_fancy_output():\n            self._write(\n                operation,\n                f\"  <fg=default;options=bold,dark>-</> {operation_message}: \"\n                \"<fg=default;options=bold,dark>Skipped</> \"\n                \"<fg=default;options=dark>for the following reason:</> \"\n                f\"<fg=default;options=bold,dark>{operation.skip_reason}</>\",\n            )\n\n        self._skipped[operation.job_type] += 1\n\n        return 0\n\n    if not self._enabled or self._dry_run:\n        return 0\n\n    result: int = getattr(self, f\"_execute_{method}\")(operation)\n\n    if result != 0:\n        return result\n\n    operation_message = self.get_operation_message(operation, done=True)\n    message = f\"  <fg=green;options=bold>-</> {operation_message}\"\n    self._write(operation, message)\n\n    self._increment_operations_count(operation, True)\n\n    return result\n```"]}
{"repo": "python-poetry/poetry", "name": "_get_directory_from_hash", "language": "python", "path": "src/poetry/utils/cache.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To generate a directory path based on a hashed representation of key components, ensuring organized and consistent storage of cached data.\n2. **Input**: A dictionary containing key-value pairs that uniquely identify a cache item.\n3. **Output**: A `Path` object representing the directory where the cache item should be stored.\n4. **Procedure**: The function serializes the input dictionary into a JSON string, ensuring keys are sorted and special characters are handled properly. This string is then hashed using SHA-256. The hash is split into segments to create a nested directory structure, which is combined with the base cache directory to form the full path.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/repositories/pypi_repository.py\nfrom __future__ import annotations\n\nimport logging\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nimport requests\nimport requests.adapters\n\nfrom cachecontrol.controller import logger as cache_control_logger\nfrom poetry.core.packages.package import Package\nfrom poetry.core.packages.utils.link import Link\nfrom poetry.core.version.exceptions import InvalidVersion\n\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.http_repository import HTTPRepository\nfrom poetry.repositories.link_sources.json import SimpleJsonPage\nfrom poetry.repositories.parsers.pypi_search_parser import SearchResultParser\nfrom poetry.utils.constants import REQUESTS_TIMEOUT\n\n\ncache_control_logger.setLevel(logging.ERROR)\n\nlogger = logging.getLogger(__name__)\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n    from poetry.core.constraints.version import Version\n    from poetry.core.constraints.version import VersionConstraint\n\nSUPPORTED_PACKAGE_TYPES = {\"sdist\", \"bdist_wheel\"}\n\n\nclass PyPiRepository(HTTPRepository):\n    def __init__(\n        self,\n        url: str = \"https://pypi.org/\",\n        disable_cache: bool = False,\n        fallback: bool = True,\n        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,\n    ) -> None:\n        super().__init__(\n            \"PyPI\",\n            url.rstrip(\"/\") + \"/simple/\",\n            disable_cache=disable_cache,\n            pool_size=pool_size,\n        )\n\n        self._base_url = url\n        self._fallback = fallback\n\n    def search(self, query: str) -> list[Package]:\n        results = []\n\n        response = requests.get(\n            self._base_url + \"search\", params={\"q\": query}, timeout=REQUESTS_TIMEOUT\n        )\n        parser = SearchResultParser()\n        parser.feed(response.text)\n\n        for result in parser.results:\n            try:\n                package = Package(result.name, result.version)\n                package.description = result.description.strip()\n                results.append(package)\n            except InvalidVersion:\n                self._log(\n                    f'Unable to parse version \"{result.version}\" for the'\n                    f\" {result.name} package, skipping\",\n                    level=\"debug\",\n                )\n\n        return results\n\n    def get_package_info(self, name: NormalizedName) -> dict[str, Any]:\n        \"\"\"\n        Return the package information given its name.\n\n        The information is returned from the cache if it exists\n        or retrieved from the remote server.\n        \"\"\"\n        return self._get_package_info(name)\n\n    def _find_packages(\n        self, name: NormalizedName, constraint: VersionConstraint\n    ) -> list[Package]:\n        \"\"\"\n        Find packages on the remote server.\n        \"\"\"\n        try:\n            json_page = self.get_page(name)\n        except PackageNotFound:\n            self._log(f\"No packages found for {name}\", level=\"debug\")\n            return []\n\n        versions = [\n            (version, json_page.yanked(name, version))\n            for version in json_page.versions(name)\n            if constraint.allows(version)\n        ]\n\n        return [Package(name, version, yanked=yanked) for version, yanked in versions]\n\n    def _get_package_info(self, name: NormalizedName) -> dict[str, Any]:\n        headers = {\"Accept\": \"application/vnd.pypi.simple.v1+json\"}\n        info = self._get(f\"simple/{name}/\", headers=headers)\n        if info is None:\n            raise PackageNotFound(f\"Package [{name}] not found.\")\n\n        return info\n\n    def find_links_for_package(self, package: Package) -> list[Link]:\n        json_data = self._get(f\"pypi/{package.name}/{package.version}/json\")\n        if json_data is None:\n            return []\n\n        links = []\n        for url in json_data[\"urls\"]:\n            if url[\"packagetype\"] in SUPPORTED_PACKAGE_TYPES:\n                h = f\"sha256={url['digests']['sha256']}\"\n                links.append(Link(url[\"url\"] + \"#\" + h, yanked=self._get_yanked(url)))\n\n        return links\n\n    def _get_release_info(\n        self, name: NormalizedName, version: Version\n    ) -> dict[str, Any]:\n        from poetry.inspection.info import PackageInfo\n\n        self._log(f\"Getting info for {name} ({version}) from PyPI\", \"debug\")\n\n        json_data = self._get(f\"pypi/{name}/{version}/json\")\n        if json_data is None:\n            raise PackageNotFound(f\"Package [{name}] not found.\")\n\n        info = json_data[\"info\"]\n\n        data = PackageInfo(\n            name=info[\"name\"],\n            version=info[\"version\"],\n            summary=info[\"summary\"],\n            requires_dist=info[\"requires_dist\"],\n            requires_python=info[\"requires_python\"],\n            yanked=self._get_yanked(info),\n            cache_version=str(self.CACHE_VERSION),\n        )\n\n        try:\n...\n# Path: src/poetry/repositories/repository.py\nfrom __future__ import annotations\n\nimport logging\n\nfrom typing import TYPE_CHECKING\n\nfrom packaging.utils import canonicalize_name\nfrom poetry.core.constraints.version import Version\n\nfrom poetry.repositories.abstract_repository import AbstractRepository\nfrom poetry.repositories.exceptions import PackageNotFound\n\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n    from poetry.core.constraints.version import VersionConstraint\n    from poetry.core.packages.dependency import Dependency\n    from poetry.core.packages.package import Package\n    from poetry.core.packages.utils.link import Link\n\n\nclass Repository(AbstractRepository):\n    def __init__(self, name: str, packages: list[Package] | None = None) -> None:\n        super().__init__(name)\n        self._packages: list[Package] = []\n\n        for package in packages or []:\n            self.add_package(package)\n\n    @property\n    def packages(self) -> list[Package]:\n        return self._packages\n\n    def find_packages(self, dependency: Dependency) -> list[Package]:\n        packages = []\n        ignored_pre_release_packages = []\n\n        constraint = dependency.constraint\n        allow_prereleases = dependency.allows_prereleases()\n        for package in self._find_packages(dependency.name, constraint):\n            if package.yanked and not isinstance(constraint, Version):\n                # PEP 592: yanked files are always ignored, unless they are the only\n                # file that matches a version specifier that \"pins\" to an exact\n                # version\n                continue\n            if (\n                package.is_prerelease()\n                and not allow_prereleases\n                and not package.is_direct_origin()\n            ):\n                ignored_pre_release_packages.append(package)\n                continue\n\n            packages.append(package)\n\n        self._log(\n            f\"{len(packages)} packages found for {dependency.name} {constraint!s}\",\n            level=\"debug\",\n        )\n\n        return packages or ignored_pre_release_packages\n\n    def has_package(self, package: Package) -> bool:\n        package_id = package.unique_name\n        return any(\n            package_id == repo_package.unique_name for repo_package in self.packages\n        )\n\n    def add_package(self, package: Package) -> None:\n        self._packages.append(package)\n\n    def remove_package(self, package: Package) -> None:\n        package_id = package.unique_name\n\n        index = None\n        for i, repo_package in enumerate(self.packages):\n            if package_id == repo_package.unique_name:\n                index = i\n                break\n\n        if index is not None:\n            del self._packages[index]\n\n    def search(self, query: str) -> list[Package]:\n        results: list[Package] = []\n\n        for package in self.packages:\n            if query in package.name:\n                results.append(package)\n\n        return results\n\n    def _find_packages(\n        self, name: NormalizedName, constraint: VersionConstraint\n    ) -> list[Package]:\n        return [\n            package\n            for package in self._packages\n            if package.name == name and constraint.allows(package.version)\n        ]\n\n    def _log(self, msg: str, level: str = \"info\") -> None:\n        logger = logging.getLogger(f\"{__name__}.{self.__class__.__name__}\")\n        getattr(logger, level)(f\"<c1>Source ({self.name}):</c1> {msg}\")\n\n    def __len__(self) -> int:\n        return len(self._packages)\n\n    def find_links_for_package(self, package: Package) -> list[Link]:\n        return []\n\n    def package(\n        self, name: str, version: Version, extras: list[str] | None = None\n    ) -> Package:\n        canonicalized_name = canonicalize_name(name)\n        for package in self.packages:\n            if canonicalized_name == package.name and package.version == version:\n                return package\n\n        raise PackageNotFound(f\"Package {name} ({version}) not found.\")\n\n# Path: src/poetry/repositories/repository_pool.py\nfrom __future__ import annotations\n\nimport enum\nimport warnings\n\nfrom collections import OrderedDict\nfrom dataclasses import dataclass\nfrom enum import IntEnum\nfrom typing import TYPE_CHECKING\n\nfrom poetry.config.config import Config\nfrom poetry.repositories.abstract_repository import AbstractRepository\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.repository import Repository\nfrom poetry.utils.cache import ArtifactCache\n\n\nif TYPE_CHECKING:\n    from poetry.core.constraints.version import Version\n    from poetry.core.packages.dependency import Dependency\n    from poetry.core.packages.package import Package\n\n_SENTINEL = object()\n\n\nclass Priority(IntEnum):\n    # The order of the members below dictates the actual priority. The first member has\n    # top priority.\n    DEFAULT = enum.auto()\n    PRIMARY = enum.auto()\n    SECONDARY = enum.auto()\n    SUPPLEMENTAL = enum.auto()\n    EXPLICIT = enum.auto()\n\n\n@dataclass(frozen=True)\nclass PrioritizedRepository:\n    repository: Repository\n    priority: Priority\n\n\nclass RepositoryPool(AbstractRepository):\n    def __init__(\n        self,\n        repositories: list[Repository] | None = None,\n        ignore_repository_names: object = _SENTINEL,\n        *,\n        config: Config | None = None,\n    ) -> None:\n        super().__init__(\"poetry-repository-pool\")\n        self._repositories: OrderedDict[str, PrioritizedRepository] = OrderedDict()\n\n        if repositories is None:\n            repositories = []\n        for repository in repositories:\n            self.add_repository(repository)\n\n        self._artifact_cache = ArtifactCache(\n            cache_dir=(config or Config.create()).artifacts_cache_directory\n        )\n\n        if ignore_repository_names is not _SENTINEL:\n            warnings.warn(\n                \"The 'ignore_repository_names' argument to 'RepositoryPool.__init__' is\"\n                \" deprecated. It has no effect anymore and will be removed in a future\"\n                \" version.\",\n                DeprecationWarning,\n                stacklevel=2,\n            )\n\n    @staticmethod\n    def from_packages(packages: list[Package], config: Config | None) -> RepositoryPool:\n        pool = RepositoryPool(config=config)\n        for package in packages:\n            if package.is_direct_origin():\n                continue\n\n            repo_name = package.source_reference or \"PyPI\"\n            try:\n                repo = pool.repository(repo_name)\n            except IndexError:\n                repo = Repository(repo_name)\n                pool.add_repository(repo)\n\n            if not repo.has_package(package):\n                repo.add_package(package)\n\n        return pool\n\n    @property\n    def repositories(self) -> list[Repository]:\n        \"\"\"\n        Returns the repositories in the pool,\n        in the order they will be searched for packages.\n\n        ATTENTION: For backwards compatibility and practical reasons,\n                   repositories with priority EXPLICIT are NOT included,\n                   because they will not be searched.\n        \"\"\"\n        sorted_repositories = self._sorted_repositories\n        return [\n            prio_repo.repository\n            for prio_repo in sorted_repositories\n            if prio_repo.priority is not Priority.EXPLICIT\n        ]\n\n    @property\n    def all_repositories(self) -> list[Repository]:\n        return [prio_repo.repository for prio_repo in self._sorted_repositories]\n\n    @property\n    def _sorted_repositories(self) -> list[PrioritizedRepository]:\n        return sorted(\n            self._repositories.values(), key=lambda prio_repo: prio_repo.priority\n        )\n\n    @property\n    def artifact_cache(self) -> ArtifactCache:\n        return self._artifact_cache\n\n    def has_default(self) -> bool:\n        return self._contains_priority(Priority.DEFAULT)\n\n    def has_primary_repositories(self) -> bool:\n        return self._contains_priority(Priority.PRIMARY)\n\n    def _contains_priority(self, priority: Priority) -> bool:\n        return any(\n            prio_repo.priority is priority for prio_repo in self._repositories.values()\n        )\n\n    def has_repository(self, name: str) -> bool:\n        return name.lower() in self._repositories\n\n    def repository(self, name: str) -> Repository:\n        return self._get_prioritized_repository(name).repository\n\n    def get_priority(self, name: str) -> Priority:\n        return self._get_prioritized_repository(name).priority\n\n    def _get_prioritized_repository(self, name: str) -> PrioritizedRepository:\n        name = name.lower()\n        if self.has_repository(name):\n            return self._repositories[name]\n        raise IndexError(f'Repository \"{name}\" does not exist.')\n\n    def add_repository(\n        self,\n        repository: Repository,\n        default: bool = False,\n        secondary: bool = False,\n        *,\n        priority: Priority = Priority.PRIMARY,\n    ) -> RepositoryPool:\n        \"\"\"\n        Adds a repository to the pool.\n        \"\"\"\n        repository_name = repository.name.lower()\n        if self.has_repository(repository_name):\n            raise ValueError(\n                f\"A repository with name {repository_name} was already added.\"\n            )\n\n        if default or secondary:\n            warnings.warn(\n                \"Parameters 'default' and 'secondary' to\"\n                \" 'RepositoryPool.add_repository' are deprecated. Please provide\"\n                \" the keyword-argument 'priority' instead.\",\n                DeprecationWarning,\n                stacklevel=2,\n            )\n            priority = Priority.DEFAULT if default else Priority.SECONDARY\n\n        if priority is Priority.DEFAULT and self.has_default():\n            raise ValueError(\"Only one repository can be the default.\")\n\n        self._repositories[repository_name] = PrioritizedRepository(\n            repository, priority\n        )\n        return self\n\n    def remove_repository(self, name: str) -> RepositoryPool:\n        if not self.has_repository(name):\n            raise IndexError(\n                f\"RepositoryPool can not remove unknown repository '{name}'.\"\n            )\n        del self._repositories[name.lower()]\n        return self\n\n    def package(\n        self,\n        name: str,\n        version: Version,\n        extras: list[str] | None = None,\n        repository_name: str | None = None,\n    ) -> Package:\n        if repository_name:\n            return self.repository(repository_name).package(\n                name, version, extras=extras\n            )\n\n        for repo in self.repositories:\n            try:\n                return repo.package(name, version, extras=extras)\n            except PackageNotFound:\n                continue\n        raise PackageNotFound(f\"Package {name} ({version}) not found.\")\n\n    def find_packages(self, dependency: Dependency) -> list[Package]:\n        repository_name = dependency.source_name\n        if repository_name:\n            return self.repository(repository_name).find_packages(dependency)\n\n        packages: list[Package] = []\n        for repo in self.repositories:\n            if packages and self.get_priority(repo.name) is Priority.SUPPLEMENTAL:\n                break\n            packages += repo.find_packages(dependency)\n        return packages\n\n    def search(self, query: str) -> list[Package]:\n        results: list[Package] = []\n        for repo in self.repositories:\n            results += repo.search(query)\n        return results\n\n# Path: src/poetry/repositories/single_page_repository.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.repositories.exceptions import PackageNotFound\nfrom poetry.repositories.legacy_repository import LegacyRepository\nfrom poetry.repositories.link_sources.html import SimpleRepositoryPage\n\n\nif TYPE_CHECKING:\n    from packaging.utils import NormalizedName\n\n\nclass SinglePageRepository(LegacyRepository):\n    def _get_page(self, name: NormalizedName) -> SimpleRepositoryPage:\n        \"\"\"\n        Single page repositories only have one page irrespective of endpoint.\n        \"\"\"\n        response = self._get_response(\"\")\n        if not response:\n            raise PackageNotFound(f\"Package [{name}] not found.\")\n        return SimpleRepositoryPage(response.url, response.text)\n\n# Path: src/poetry/toml/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.toml.exceptions import TOMLError\nfrom poetry.toml.file import TOMLFile\n\n\n__all__ = [\"TOMLError\", \"TOMLFile\"]\n\n# Path: src/poetry/toml/exceptions.py\nfrom __future__ import annotations\n\nfrom poetry.core.exceptions import PoetryCoreException\nfrom tomlkit.exceptions import TOMLKitError\n\n\nclass TOMLError(TOMLKitError, PoetryCoreException):\n    pass\n\n# Path: src/poetry/toml/file.py\nfrom __future__ import annotations\n\nimport warnings\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom tomlkit.toml_file import TOMLFile as BaseTOMLFile\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n\n    from tomlkit.toml_document import TOMLDocument\n\n\nclass TOMLFile(BaseTOMLFile):\n    def __init__(self, path: Path) -> None:\n        super().__init__(path)\n        self.__path = path\n\n    @property\n    def path(self) -> Path:\n        return self.__path\n\n    def exists(self) -> bool:\n        return self.__path.exists()\n\n    def read(self) -> TOMLDocument:\n        from tomlkit.exceptions import TOMLKitError\n\n        from poetry.toml import TOMLError\n\n        try:\n            return super().read()\n        except (ValueError, TOMLKitError) as e:\n            raise TOMLError(f\"Invalid TOML file {self.path.as_posix()}: {e}\")\n\n    def __getattr__(self, item: str) -> Any:\n        warnings.warn(\n            \"`__getattr__` will be removed from the `TOMLFile` in a future release.\"\n            \"\\n\\nInstead of accessing properties of the underlying `Path` as \"\n            \"`tomlfile.whatever`, prefer `tomlfile.path.whatever`.\",\n            DeprecationWarning,\n            stacklevel=2,\n        )\n        return getattr(self.__path, item)\n\n    def __str__(self) -> str:\n        return self.__path.as_posix()\n\n# Path: src/poetry/utils/__init__.py\n\n# Path: src/poetry/utils/_compat.py\nfrom __future__ import annotations\n\nimport sys\n\nfrom contextlib import suppress\n\n\n# TODO: use try/except ImportError when\n# https://github.com/python/mypy/issues/1393 is fixed\n\nif sys.version_info < (3, 11):\n    # compatibility for python <3.11\n    import tomli as tomllib\nelse:\n    import tomllib  # nopycln: import\n\n\nif sys.version_info < (3, 10):\n    # compatibility for python <3.10\n    import importlib_metadata as metadata\nelse:\n    from importlib import metadata\n\nWINDOWS = sys.platform == \"win32\"\n\n\ndef decode(string: bytes | str, encodings: list[str] | None = None) -> str:\n    if not isinstance(string, bytes):\n        return string\n\n    encodings = encodings or [\"utf-8\", \"latin1\", \"ascii\"]\n\n    for encoding in encodings:\n        with suppress(UnicodeEncodeError, UnicodeDecodeError):\n            return string.decode(encoding)\n\n    return string.decode(encodings[0], errors=\"ignore\")\n\n\ndef encode(string: str, encodings: list[str] | None = None) -> bytes:\n    if isinstance(string, bytes):\n        return string\n\n    encodings = encodings or [\"utf-8\", \"latin1\", \"ascii\"]\n\n    for encoding in encodings:\n        with suppress(UnicodeEncodeError, UnicodeDecodeError):\n            return string.encode(encoding)\n\n    return string.encode(encodings[0], errors=\"ignore\")\n\n\n__all__ = [\n    \"WINDOWS\",\n    \"decode\",\n    \"encode\",\n    \"metadata\",\n    \"tomllib\",\n]\n\n# Path: src/poetry/utils/authenticator.py\nfrom __future__ import annotations\n\nimport contextlib\nimport dataclasses\nimport functools\nimport logging\nimport time\nimport urllib.parse\n\nfrom os.path import commonprefix\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nimport requests\nimport requests.adapters\nimport requests.auth\nimport requests.exceptions\n\nfrom cachecontrol import CacheControlAdapter\nfrom cachecontrol.caches import FileCache\nfrom requests_toolbelt import user_agent\n\nfrom poetry.__version__ import __version__\nfrom poetry.config.config import Config\nfrom poetry.exceptions import PoetryException\nfrom poetry.utils.constants import REQUESTS_TIMEOUT\nfrom poetry.utils.constants import RETRY_AFTER_HEADER\nfrom poetry.utils.constants import STATUS_FORCELIST\nfrom poetry.utils.password_manager import HTTPAuthCredential\nfrom poetry.utils.password_manager import PasswordManager\n\n\nif TYPE_CHECKING:\n    from cleo.io.io import IO\n\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclasses.dataclass(frozen=True)\nclass RepositoryCertificateConfig:\n    cert: Path | None = dataclasses.field(default=None)\n    client_cert: Path | None = dataclasses.field(default=None)\n    verify: bool = dataclasses.field(default=True)\n\n    @classmethod\n    def create(\n        cls, repository: str, config: Config | None\n    ) -> RepositoryCertificateConfig:\n        config = config if config else Config.create()\n\n        verify: str | bool = config.get(\n            f\"certificates.{repository}.verify\",\n            config.get(f\"certificates.{repository}.cert\", True),\n        )\n        client_cert: str = config.get(f\"certificates.{repository}.client-cert\")\n\n        return cls(\n            cert=Path(verify) if isinstance(verify, str) else None,\n            client_cert=Path(client_cert) if client_cert else None,\n            verify=verify if isinstance(verify, bool) else True,\n        )\n\n\n@dataclasses.dataclass\nclass AuthenticatorRepositoryConfig:\n    name: str\n    url: str\n    netloc: str = dataclasses.field(init=False)\n    path: str = dataclasses.field(init=False)\n\n    def __post_init__(self) -> None:\n        parsed_url = urllib.parse.urlsplit(self.url)\n        self.netloc = parsed_url.netloc\n        self.path = parsed_url.path\n\n    def certs(self, config: Config) -> RepositoryCertificateConfig:\n        return RepositoryCertificateConfig.create(self.name, config)\n\n    @property\n    def http_credential_keys(self) -> list[str]:\n        return [self.url, self.netloc, self.name]\n\n    def get_http_credentials(\n        self, password_manager: PasswordManager, username: str | None = None\n    ) -> HTTPAuthCredential:\n        # try with the repository name via the password manager\n        credential = HTTPAuthCredential(\n            **(password_manager.get_http_auth(self.name) or {})\n        )\n\n        if credential.password is not None:\n            return credential\n\n        if password_manager.use_keyring:\n            # fallback to url and netloc based keyring entries\n            credential = password_manager.get_credential(\n                self.url, self.netloc, username=credential.username\n            )\n\n        return credential\n\n\nclass Authenticator:\n    def __init__(\n        self,\n        config: Config | None = None,\n        io: IO | None = None,\n        cache_id: str | None = None,\n        disable_cache: bool = False,\n        pool_size: int = requests.adapters.DEFAULT_POOLSIZE,\n    ) -> None:\n        self._config = config or Config.create()\n        self._io = io\n        self._sessions_for_netloc: dict[str, requests.Session] = {}\n        self._credentials: dict[str, HTTPAuthCredential] = {}\n        self._certs: dict[str, RepositoryCertificateConfig] = {}\n        self._configured_repositories: (\n            dict[str, AuthenticatorRepositoryConfig] | None\n        ) = None\n        self._password_manager = PasswordManager(self._config)\n        self._cache_control = (\n            FileCache(\n                self._config.repository_cache_directory\n                / (cache_id or \"_default_cache\")\n                / \"_http\"\n            )\n            if not disable_cache\n            else None\n        )\n        self.get_repository_config_for_url = functools.lru_cache(maxsize=None)(\n            self._get_repository_config_for_url\n        )\n        self._pool_size = pool_size\n        self._user_agent = user_agent(\"poetry\", __version__)\n\n    def create_session(self) -> requests.Session:\n        session = requests.Session()\n        session.headers[\"User-Agent\"] = self._user_agent\n\n        if self._cache_control is None:\n            return session\n\n        adapter = CacheControlAdapter(\n            cache=self._cache_control,\n            pool_maxsize=self._pool_size,\n        )\n        session.mount(\"http://\", adapter)\n        session.mount(\"https://\", adapter)\n\n        return session\n\n    def get_session(self, url: str | None = None) -> requests.Session:\n        if not url:\n            return self.create_session()\n\n        parsed_url = urllib.parse.urlsplit(url)\n        netloc = parsed_url.netloc\n\n        if netloc not in self._sessions_for_netloc:\n            logger.debug(\"Creating new session for %s\", netloc)\n            self._sessions_for_netloc[netloc] = self.create_session()\n\n        return self._sessions_for_netloc[netloc]\n\n    def close(self) -> None:\n        for session in self._sessions_for_netloc.values():\n            if session is not None:\n                with contextlib.suppress(AttributeError):\n                    session.close()\n\n    def __del__(self) -> None:\n        self.close()\n\n    def delete_cache(self, url: str) -> None:\n        if self._cache_control is not None:\n            self._cache_control.delete(key=url)\n\n    def authenticated_url(self, url: str) -> str:\n        parsed = urllib.parse.urlparse(url)\n        credential = self.get_credentials_for_url(url)\n\n        if credential.username is not None and credential.password is not None:\n            username = urllib.parse.quote(credential.username, safe=\"\")\n            password = urllib.parse.quote(credential.password, safe=\"\")\n\n            return (\n                f\"{parsed.scheme}://{username}:{password}@{parsed.netloc}{parsed.path}\"\n            )\n\n        return url\n\n    def request(\n        self, method: str, url: str, raise_for_status: bool = True, **kwargs: Any\n    ) -> requests.Response:\n        headers = kwargs.get(\"headers\")\n        request = requests.Request(method, url, headers=headers)\n        credential = self.get_credentials_for_url(url)\n\n        if credential.username is not None or credential.password is not None:\n            request = requests.auth.HTTPBasicAuth(\n                credential.username or \"\", credential.password or \"\"\n            )(request)\n\n        session = self.get_session(url=url)\n        prepared_request = session.prepare_request(request)\n\n        proxies: dict[str, str] = kwargs.get(\"proxies\", {})\n        stream: bool | None = kwargs.get(\"stream\")\n\n        certs = self.get_certs_for_url(url)\n        verify: bool | str | Path = kwargs.get(\"verify\") or certs.cert or certs.verify\n        cert: str | Path | None = kwargs.get(\"cert\") or certs.client_cert\n\n        if cert is not None:\n            cert = str(cert)\n\n        verify = str(verify) if isinstance(verify, Path) else verify\n\n        settings = session.merge_environment_settings(\n            prepared_request.url, proxies, stream, verify, cert\n        )\n\n        # Send the request.\n        send_kwargs = {\n            \"timeout\": kwargs.get(\"timeout\", REQUESTS_TIMEOUT),\n            \"allow_redirects\": kwargs.get(\"allow_redirects\", True),\n        }\n        send_kwargs.update(settings)\n\n        attempt = 0\n        resp = None\n\n        while True:\n            is_last_attempt = attempt >= 5\n            try:\n                resp = session.send(prepared_request, **send_kwargs)\n            except (requests.exceptions.ConnectionError, OSError) as e:\n                if is_last_attempt:\n                    raise e\n            else:\n                if resp.status_code not in STATUS_FORCELIST or is_last_attempt:\n                    if raise_for_status:\n                        resp.raise_for_status()\n                    return resp\n\n            if not is_last_attempt:\n                attempt += 1\n                delay = self._get_backoff(resp, attempt)\n                logger.debug(\"Retrying HTTP request in %s seconds.\", delay)\n                time.sleep(delay)\n                continue\n\n        # this should never really be hit under any sane circumstance\n        raise PoetryException(\"Failed HTTP {} request\", method.upper())\n\n    def _get_backoff(self, response: requests.Response | None, attempt: int) -> float:\n        if response is not None:\n            retry_after = response.headers.get(RETRY_AFTER_HEADER, \"\")\n            if retry_after:\n                return float(retry_after)\n\n        return 0.5 * attempt\n\n    def get(self, url: str, **kwargs: Any) -> requests.Response:\n        return self.request(\"get\", url, **kwargs)\n\n    def head(self, url: str, **kwargs: Any) -> requests.Response:\n        kwargs.setdefault(\"allow_redirects\", False)\n        return self.request(\"head\", url, **kwargs)\n\n    def post(self, url: str, **kwargs: Any) -> requests.Response:\n        return self.request(\"post\", url, **kwargs)\n\n    def _get_credentials_for_repository(\n        self, repository: AuthenticatorRepositoryConfig, username: str | None = None\n    ) -> HTTPAuthCredential:\n        # cache repository credentials by repository url to avoid multiple keyring\n        # backend queries when packages are being downloaded from the same source\n        key = f\"{repository.url}#username={username or ''}\"\n\n        if key not in self._credentials:\n            self._credentials[key] = repository.get_http_credentials(\n                password_manager=self._password_manager, username=username\n            )\n\n        return self._credentials[key]\n\n    def _get_credentials_for_url(\n        self, url: str, exact_match: bool = False\n    ) -> HTTPAuthCredential:\n        repository = self.get_repository_config_for_url(url, exact_match)\n\n        credential = (\n            self._get_credentials_for_repository(repository=repository)\n            if repository is not None\n            else HTTPAuthCredential()\n        )\n\n        if credential.password is None:\n            parsed_url = urllib.parse.urlsplit(url)\n            netloc = parsed_url.netloc\n            credential = self._password_manager.get_credential(\n                url, netloc, username=credential.username\n            )\n\n            return HTTPAuthCredential(\n                username=credential.username, password=credential.password\n            )\n\n        return credential\n\n    def get_credentials_for_git_url(self, url: str) -> HTTPAuthCredential:\n        parsed_url = urllib.parse.urlsplit(url)\n\n        if parsed_url.scheme not in {\"http\", \"https\"}:\n            return HTTPAuthCredential()\n\n        key = f\"git+{url}\"\n\n        if key not in self._credentials:\n            self._credentials[key] = self._get_credentials_for_url(url, True)\n\n        return self._credentials[key]\n\n    def get_credentials_for_url(self, url: str) -> HTTPAuthCredential:\n        parsed_url = urllib.parse.urlsplit(url)\n        netloc = parsed_url.netloc\n\n        if url not in self._credentials:\n            if \"@\" not in netloc:\n                # no credentials were provided in the url, try finding the\n                # best repository configuration\n                self._credentials[url] = self._get_credentials_for_url(url)\n            else:\n                # Split from the right because that's how urllib.parse.urlsplit()\n                # behaves if more than one @ is present (which can be checked using\n                # the password attribute of urlsplit()'s return value).\n                auth, netloc = netloc.rsplit(\"@\", 1)\n                # Split from the left because that's how urllib.parse.urlsplit()\n                # behaves if more than one : is present (which again can be checked\n                # using the password attribute of the return value)\n                user, password = auth.split(\":\", 1) if \":\" in auth else (auth, \"\")\n                self._credentials[url] = HTTPAuthCredential(\n                    urllib.parse.unquote(user),\n                    urllib.parse.unquote(password),\n                )\n\n        return self._credentials[url]\n\n    def get_pypi_token(self, name: str) -> str | None:\n        return self._password_manager.get_pypi_token(name)\n\n    def get_http_auth(\n        self, name: str, username: str | None = None\n    ) -> HTTPAuthCredential | None:\n        if name == \"pypi\":\n            repository = AuthenticatorRepositoryConfig(\n                name, \"https://upload.pypi.org/legacy/\"\n            )\n        else:\n            if name not in self.configured_repositories:\n                return None\n            repository = self.configured_repositories[name]\n\n        return self._get_credentials_for_repository(\n            repository=repository, username=username\n        )\n\n    def get_certs_for_repository(self, name: str) -> RepositoryCertificateConfig:\n        if name.lower() == \"pypi\" or name not in self.configured_repositories:\n            return RepositoryCertificateConfig()\n        return self.configured_repositories[name].certs(self._config)\n\n    @property\n    def configured_repositories(self) -> dict[str, AuthenticatorRepositoryConfig]:\n        if self._configured_repositories is None:\n            self._configured_repositories = {}\n            for repository_name in self._config.get(\"repositories\", []):\n                url = self._config.get(f\"repositories.{repository_name}.url\")\n                self._configured_repositories[repository_name] = (\n                    AuthenticatorRepositoryConfig(repository_name, url)\n                )\n\n        return self._configured_repositories\n\n    def reset_credentials_cache(self) -> None:\n        self.get_repository_config_for_url.cache_clear()\n        self._credentials = {}\n\n    def add_repository(self, name: str, url: str) -> None:\n        self.configured_repositories[name] = AuthenticatorRepositoryConfig(name, url)\n        self.reset_credentials_cache()\n\n    def get_certs_for_url(self, url: str) -> RepositoryCertificateConfig:\n        if url not in self._certs:\n            self._certs[url] = self._get_certs_for_url(url)\n        return self._certs[url]\n\n    def _get_repository_config_for_url(\n        self, url: str, exact_match: bool = False\n    ) -> AuthenticatorRepositoryConfig | None:\n        parsed_url = urllib.parse.urlsplit(url)\n        candidates_netloc_only = []\n        candidates_path_match = []\n\n        for repository in self.configured_repositories.values():\n            if exact_match:\n                if parsed_url.path == repository.path:\n                    return repository\n                continue\n\n            if repository.netloc == parsed_url.netloc:\n                if parsed_url.path.startswith(repository.path) or commonprefix(\n                    (parsed_url.path, repository.path)\n                ):\n                    candidates_path_match.append(repository)\n                    continue\n                candidates_netloc_only.append(repository)\n\n        if candidates_path_match:\n            candidates = candidates_path_match\n        elif candidates_netloc_only:\n            candidates = candidates_netloc_only\n        else:\n            return None\n\n        if len(candidates) > 1:\n            logger.debug(\n                \"Multiple source configurations found for %s - %s\",\n                parsed_url.netloc,\n                \", \".join(c.name for c in candidates),\n            )\n            # prefer the more specific path\n            candidates.sort(\n                key=lambda c: len(commonprefix([parsed_url.path, c.path])), reverse=True\n            )\n\n        return candidates[0]\n\n    def _get_certs_for_url(self, url: str) -> RepositoryCertificateConfig:\n        selected = self.get_repository_config_for_url(url)\n        if selected:\n            return selected.certs(config=self._config)\n        return RepositoryCertificateConfig()\n\n\n_authenticator: Authenticator | None = None\n\n\ndef get_default_authenticator() -> Authenticator:\n    global _authenticator\n\n    if _authenticator is None:\n        _authenticator = Authenticator()\n\n    return _authenticator\n\n# Path: src/poetry/utils/cache.py\nfrom __future__ import annotations\n\nimport dataclasses\nimport hashlib\nimport json\nimport logging\nimport shutil\nimport threading\nimport time\n\nfrom collections import defaultdict\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import Generic\nfrom typing import TypeVar\nfrom typing import overload\n\nfrom poetry.utils._compat import decode\nfrom poetry.utils._compat import encode\nfrom poetry.utils.helpers import get_highest_priority_hash_type\nfrom poetry.utils.wheel import InvalidWheelName\nfrom poetry.utils.wheel import Wheel\n\n\nif TYPE_CHECKING:\n    from collections.abc import Callable\n\n    from poetry.core.packages.utils.link import Link\n\n    from poetry.utils.env import Env\n\n\n# Used by FileCache for items that do not expire.\nMAX_DATE = 9999999999\nT = TypeVar(\"T\")\n\nlogger = logging.getLogger(__name__)\n\n\ndef _expiration(minutes: int) -> int:\n    \"\"\"\n    Calculates the time in seconds since epoch that occurs 'minutes' from now.\n\n    :param minutes: The number of minutes to count forward\n    \"\"\"\n    return round(time.time()) + minutes * 60\n\n\n_HASHES = {\n    \"md5\": (hashlib.md5, 2),\n    \"sha1\": (hashlib.sha1, 4),\n    \"sha256\": (hashlib.sha256, 8),\n}\n\n\n@dataclasses.dataclass(frozen=True)\nclass CacheItem(Generic[T]):\n    \"\"\"\n    Stores data and metadata for cache items.\n    \"\"\"\n\n    data: T\n    expires: int | None = None\n\n    @property\n    def expired(self) -> bool:\n        \"\"\"\n        Return true if the cache item has exceeded its expiration period.\n        \"\"\"\n        return self.expires is not None and time.time() >= self.expires\n\n\n@dataclasses.dataclass(frozen=True)\nclass FileCache(Generic[T]):\n    \"\"\"\n    Cachy-compatible minimal file cache. Stores subsequent data in a JSON format.\n\n    :param path: The path that the cache starts at.\n    :param hash_type: The hash to use for encoding keys/building directories.\n    \"\"\"\n\n    path: Path\n    hash_type: str = \"sha256\"\n\n    def __post_init__(self) -> None:\n        if self.hash_type not in _HASHES:\n            raise ValueError(\n                f\"FileCache.hash_type is unknown value: '{self.hash_type}'.\"\n            )\n\n    def get(self, key: str) -> T | None:\n        return self._get_payload(key)\n\n    def has(self, key: str) -> bool:\n        \"\"\"\n        Determine if a file exists and has not expired in the cache.\n        :param key: The cache key\n        :returns: True if the key exists in the cache\n        \"\"\"\n        return self.get(key) is not None\n\n    def put(self, key: str, value: Any, minutes: int | None = None) -> None:\n        \"\"\"\n        Store an item in the cache.\n\n        :param key: The cache key\n        :param value: The cache value\n        :param minutes: The lifetime in minutes of the cached value\n        \"\"\"\n        payload: CacheItem[Any] = CacheItem(\n            value, expires=_expiration(minutes) if minutes is not None else None\n        )\n        path = self._path(key)\n        path.parent.mkdir(parents=True, exist_ok=True)\n        with path.open(\"wb\") as f:\n            f.write(self._serialize(payload))\n\n    def forget(self, key: str) -> None:\n        \"\"\"\n        Remove an item from the cache.\n\n        :param key: The cache key\n        \"\"\"\n        path = self._path(key)\n        if path.exists():\n            path.unlink()\n\n    def flush(self) -> None:\n        \"\"\"\n        Clear the cache.\n        \"\"\"\n        shutil.rmtree(self.path)\n\n    def remember(\n        self, key: str, callback: T | Callable[[], T], minutes: int | None = None\n    ) -> T:\n        \"\"\"\n        Get an item from the cache, or use a default from callback.\n\n        :param key: The cache key\n        :param callback: Callback function providing default value\n        :param minutes: The lifetime in minutes of the cached value\n        \"\"\"\n        value = self.get(key)\n        if value is None:\n            value = callback() if callable(callback) else callback\n            self.put(key, value, minutes)\n        return value\n\n    def _get_payload(self, key: str) -> T | None:\n        path = self._path(key)\n\n        if not path.exists():\n            return None\n\n        with path.open(\"rb\") as f:\n            file_content = f.read()\n\n        try:\n            payload = self._deserialize(file_content)\n        except (json.JSONDecodeError, ValueError):\n            self.forget(key)\n            logger.warning(\"Corrupt cache file was detected and cleaned up.\")\n            return None\n\n        if payload.expired:\n            self.forget(key)\n            return None\n        else:\n            return payload.data\n\n    def _path(self, key: str) -> Path:\n        hash_type, parts_count = _HASHES[self.hash_type]\n        h = hash_type(encode(key)).hexdigest()\n        parts = [h[i : i + 2] for i in range(0, len(h), 2)][:parts_count]\n        return Path(self.path, *parts, h)\n\n    def _serialize(self, payload: CacheItem[T]) -> bytes:\n        expires = payload.expires or MAX_DATE\n        data = json.dumps(payload.data)\n        return encode(f\"{expires:010d}{data}\")\n\n    def _deserialize(self, data_raw: bytes) -> CacheItem[T]:\n        data_str = decode(data_raw)\n        data = json.loads(data_str[10:])\n        expires = int(data_str[:10])\n        return CacheItem(data, expires)\n\n\nclass ArtifactCache:\n    def __init__(self, *, cache_dir: Path) -> None:\n        self._cache_dir = cache_dir\n        self._archive_locks: defaultdict[Path, threading.Lock] = defaultdict(\n            threading.Lock\n        )\n\n    def get_cache_directory_for_link(self, link: Link) -> Path:\n        key_parts = {\"url\": link.url_without_fragment}\n\n        if hash_name := get_highest_priority_hash_type(\n            set(link.hashes.keys()), link.filename\n        ):\n            key_parts[hash_name] = link.hashes[hash_name]\n\n        if link.subdirectory_fragment:\n            key_parts[\"subdirectory\"] = link.subdirectory_fragment\n\n        return self._get_directory_from_hash(key_parts)\n\n    \ndef _get_directory_from_hash(self, key_parts: object) -> Path:\n        key = hashlib.sha256(\n            json.dumps(\n                key_parts, sort_keys=True, separators=(\",\", \":\"), ensure_ascii=True\n            ).encode(\"ascii\")\n        ).hexdigest()\n\n        split_key = [key[:2], key[2:4], key[4:6], key[6:]]\n        return self._cache_dir.joinpath(*split_key)\n\n    def get_cache_directory_for_git(\n        self, url: str, ref: str, subdirectory: str | None\n    ) -> Path:\n        key_parts = {\"url\": url, \"ref\": ref}\n        if subdirectory:\n            key_parts[\"subdirectory\"] = subdirectory\n\n        return self._get_directory_from_hash(key_parts)\n\n    @overload\n    def get_cached_archive_for_link(\n        self,\n        link: Link,\n        *,\n        strict: bool,\n        env: Env | None = ...,\n        download_func: Callable[[str, Path], None],\n    ) -> Path: ...\n\n    @overload\n    def get_cached_archive_for_link(\n        self,\n        link: Link,\n        *,\n        strict: bool,\n        env: Env | None = ...,\n        download_func: None = ...,\n    ) -> Path | None: ...\n\n    def get_cached_archive_for_link(\n        self,\n        link: Link,\n        *,\n        strict: bool,\n        env: Env | None = None,\n        download_func: Callable[[str, Path], None] | None = None,\n    ) -> Path | None:\n        cache_dir = self.get_cache_directory_for_link(link)\n\n        cached_archive = self._get_cached_archive(\n            cache_dir, strict=strict, filename=link.filename, env=env\n        )\n        if cached_archive is None and strict and download_func is not None:\n            cached_archive = cache_dir / link.filename\n            with self._archive_locks[cached_archive]:\n                # Check again if the archive exists (under the lock) to avoid\n                # duplicate downloads because it may have already been downloaded\n                # by another thread in the meantime\n                if not cached_archive.exists():\n                    cache_dir.mkdir(parents=True, exist_ok=True)\n                    try:\n                        download_func(link.url, cached_archive)\n                    except BaseException:\n                        cached_archive.unlink(missing_ok=True)\n                        raise\n\n        return cached_archive\n\n    def get_cached_archive_for_git(\n        self, url: str, reference: str, subdirectory: str | None, env: Env\n    ) -> Path | None:\n        cache_dir = self.get_cache_directory_for_git(url, reference, subdirectory)\n\n        return self._get_cached_archive(cache_dir, strict=False, env=env)\n\n    def _get_cached_archive(\n        self,\n        cache_dir: Path,\n        *,\n        strict: bool,\n        filename: str | None = None,\n        env: Env | None = None,\n    ) -> Path | None:\n        # implication \"not strict -> env must not be None\"\n        assert strict or env is not None\n        # implication \"strict -> filename must not be None\"\n        assert not strict or filename is not None\n\n        archives = self._get_cached_archives(cache_dir)\n        if not archives:\n            return None\n\n        candidates: list[tuple[float | None, Path]] = []\n\n        for archive in archives:\n            if strict:\n                # in strict mode return the original cached archive instead of the\n                # prioritized archive type.\n                if filename == archive.name:\n                    return archive\n                continue\n\n            assert env is not None\n\n            if archive.suffix != \".whl\":\n                candidates.append((float(\"inf\"), archive))\n                continue\n\n            try:\n                wheel = Wheel(archive.name)\n            except InvalidWheelName:\n                continue\n\n            if not wheel.is_supported_by_environment(env):\n                continue\n\n            candidates.append(\n                (wheel.get_minimum_supported_index(env.supported_tags), archive),\n            )\n\n        if not candidates:\n            return None\n\n        return min(candidates)[1]\n\n    def _get_cached_archives(self, cache_dir: Path) -> list[Path]:\n        archive_types = [\"whl\", \"tar.gz\", \"tar.bz2\", \"bz2\", \"zip\"]\n        paths: list[Path] = []\n        for archive_type in archive_types:\n            paths += cache_dir.glob(f\"*.{archive_type}\")\n\n        return paths\n\n# Path: src/poetry/utils/constants.py\nfrom __future__ import annotations\n\nimport os\n\n\n# Timeout for HTTP requests using the requests library.\nREQUESTS_TIMEOUT = int(os.getenv(\"POETRY_REQUESTS_TIMEOUT\", 15))\n\nRETRY_AFTER_HEADER = \"retry-after\"\n\n# Server response codes to retry requests on.\nSTATUS_FORCELIST = [429, 500, 501, 502, 503, 504]\n\n# Path: src/poetry/utils/dependency_specification.py\nfrom __future__ import annotations\n\nimport contextlib\nimport os\nimport re\nimport urllib.parse\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Dict\nfrom typing import List\nfrom typing import TypeVar\nfrom typing import Union\nfrom typing import cast\n\nfrom poetry.core.packages.dependency import Dependency\nfrom tomlkit.items import InlineTable\n\nfrom poetry.packages.direct_origin import DirectOrigin\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.vcs_dependency import VCSDependency\n\n    from poetry.utils.cache import ArtifactCache\n    from poetry.utils.env import Env\n\n\nDependencySpec = Dict[str, Union[str, bool, Dict[str, Union[str, bool]], List[str]]]\nBaseSpec = TypeVar(\"BaseSpec\", DependencySpec, InlineTable)\n\nGIT_URL_SCHEMES = {\"git+http\", \"git+https\", \"git+ssh\"}\n\n\ndef dependency_to_specification(\n    dependency: Dependency, specification: BaseSpec\n) -> BaseSpec:\n    if dependency.is_vcs():\n        dependency = cast(\"VCSDependency\", dependency)\n        assert dependency.source_url is not None\n        specification[dependency.vcs] = dependency.source_url\n        if dependency.reference:\n            specification[\"rev\"] = dependency.reference\n    elif dependency.is_file() or dependency.is_directory():\n        assert dependency.source_url is not None\n        specification[\"path\"] = dependency.source_url\n    elif dependency.is_url():\n        assert dependency.source_url is not None\n        specification[\"url\"] = dependency.source_url\n    elif dependency.pretty_constraint != \"*\" and not dependency.constraint.is_empty():\n        specification[\"version\"] = dependency.pretty_constraint\n\n    if not dependency.marker.is_any():\n        specification[\"markers\"] = str(dependency.marker)\n\n    if dependency.extras:\n        specification[\"extras\"] = sorted(dependency.extras)\n\n    return specification\n\n\nclass RequirementsParser:\n    def __init__(\n        self,\n        *,\n        artifact_cache: ArtifactCache,\n        env: Env | None = None,\n        cwd: Path | None = None,\n    ) -> None:\n        self._direct_origin = DirectOrigin(artifact_cache)\n        self._env = env\n        self._cwd = cwd or Path.cwd()\n\n    def parse(self, requirement: str) -> DependencySpec:\n        requirement = requirement.strip()\n\n        specification = self._parse_pep508(requirement)\n\n        if specification is not None:\n            return specification\n\n        extras = []\n        extras_m = re.search(r\"\\[([\\w\\d,-_ ]+)\\]$\", requirement)\n        if extras_m:\n            extras = [e.strip() for e in extras_m.group(1).split(\",\")]\n            requirement, _ = requirement.split(\"[\")\n\n        specification = (\n            self._parse_url(requirement)\n            or self._parse_path(requirement)\n            or self._parse_simple(requirement)\n        )\n\n        if specification:\n            if extras:\n                specification.setdefault(\"extras\", extras)\n            return specification\n\n        raise ValueError(f\"Invalid dependency specification: {requirement}\")\n\n    def _parse_pep508(self, requirement: str) -> DependencySpec | None:\n        if \" ; \" not in requirement and re.search(r\"@[\\^~!=<>\\d]\", requirement):\n            # this is of the form package@<semver>, do not attempt to parse it\n            return None\n\n        with contextlib.suppress(ValueError):\n            dependency = Dependency.create_from_pep_508(requirement)\n            specification: DependencySpec = {}\n            specification = dependency_to_specification(dependency, specification)\n\n            if specification:\n                specification[\"name\"] = dependency.name\n                return specification\n\n        return None\n\n    def _parse_git_url(self, requirement: str) -> DependencySpec | None:\n        from poetry.core.vcs.git import Git\n        from poetry.core.vcs.git import ParsedUrl\n\n        parsed = ParsedUrl.parse(requirement)\n        url = Git.normalize_url(requirement)\n\n        pair = {\"name\": parsed.name, \"git\": url.url}\n\n        if parsed.rev:\n            pair[\"rev\"] = url.revision\n\n        if parsed.subdirectory:\n            pair[\"subdirectory\"] = parsed.subdirectory\n\n        source_root = self._env.path.joinpath(\"src\") if self._env else None\n        package = self._direct_origin.get_package_from_vcs(\n            \"git\",\n            url=url.url,\n            rev=pair.get(\"rev\"),\n            subdirectory=parsed.subdirectory,\n            source_root=source_root,\n        )\n        pair[\"name\"] = package.name\n        return pair\n\n    def _parse_url(self, requirement: str) -> DependencySpec | None:\n        url_parsed = urllib.parse.urlparse(requirement)\n        if not (url_parsed.scheme and url_parsed.netloc):\n            return None\n\n        if url_parsed.scheme in GIT_URL_SCHEMES:\n            return self._parse_git_url(requirement)\n\n        if url_parsed.scheme in [\"http\", \"https\"]:\n            package = self._direct_origin.get_package_from_url(requirement)\n            assert package.source_url is not None\n            return {\"name\": package.name, \"url\": package.source_url}\n\n        return None\n\n    def _parse_path(self, requirement: str) -> DependencySpec | None:\n        if (os.path.sep in requirement or \"/\" in requirement) and (\n            self._cwd.joinpath(requirement).exists()\n            or Path(requirement).expanduser().exists()\n            and Path(requirement).expanduser().is_absolute()\n        ):\n            path = Path(requirement).expanduser()\n            is_absolute = path.is_absolute()\n\n            if not path.is_absolute():\n                path = self._cwd.joinpath(requirement)\n\n            if path.is_file():\n                package = self._direct_origin.get_package_from_file(path.resolve())\n            else:\n                package = self._direct_origin.get_package_from_directory(path.resolve())\n\n            return {\n                \"name\": package.name,\n                \"path\": (\n                    path.relative_to(self._cwd).as_posix()\n                    if not is_absolute\n                    else path.as_posix()\n                ),\n            }\n\n        return None\n\n    def _parse_simple(\n        self,\n        requirement: str,\n    ) -> DependencySpec | None:\n        extras: list[str] = []\n        pair = re.sub(\n            \"^([^@=: ]+)(?:@|==|(?<![<>~!])=|:| )(.*)$\", \"\\\\1 \\\\2\", requirement\n        )\n        pair = pair.strip()\n\n        require: DependencySpec = {}\n\n        if \" \" in pair:\n            name, version = pair.split(\" \", 1)\n            extras_m = re.search(r\"\\[([\\w\\d,-_]+)\\]$\", name)\n            if extras_m:\n                extras = [e.strip() for e in extras_m.group(1).split(\",\")]\n                name, _ = name.split(\"[\")\n\n            require[\"name\"] = name\n            if version != \"latest\":\n                require[\"version\"] = version\n        else:\n            m = re.match(\n                r\"^([^><=!: ]+)((?:>=|<=|>|<|!=|~=|~|\\^).*)$\", requirement.strip()\n            )\n            if m:\n                name, constraint = m.group(1), m.group(2)\n                extras_m = re.search(r\"\\[([\\w\\d,-_]+)\\]$\", name)\n                if extras_m:\n                    extras = [e.strip() for e in extras_m.group(1).split(\",\")]\n                    name, _ = name.split(\"[\")\n\n                require[\"name\"] = name\n                require[\"version\"] = constraint\n            else:\n                extras_m = re.search(r\"\\[([\\w\\d,-_]+)\\]$\", pair)\n                if extras_m:\n                    extras = [e.strip() for e in extras_m.group(1).split(\",\")]\n                    pair, _ = pair.split(\"[\")\n\n                require[\"name\"] = pair\n\n        if extras:\n            require[\"extras\"] = extras\n\n        return require\n\n# Path: src/poetry/utils/extras.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\n\nif TYPE_CHECKING:\n    from collections.abc import Collection\n    from collections.abc import Iterable\n    from collections.abc import Mapping\n\n    from packaging.utils import NormalizedName\n    from poetry.core.packages.package import Package\n\n\ndef get_extra_package_names(\n    packages: Iterable[Package],\n    extras: Mapping[NormalizedName, Iterable[NormalizedName]],\n    extra_names: Collection[NormalizedName],\n) -> set[NormalizedName]:\n    \"\"\"\n    Returns all package names required by the given extras.\n\n    :param packages: A collection of packages, such as from Repository.packages\n    :param extras: A mapping of `extras` names to lists of package names, as defined\n        in the `extras` section of `poetry.lock`.\n    :param extra_names: A list of strings specifying names of extra groups to resolve.\n    \"\"\"\n    from packaging.utils import canonicalize_name\n\n    if not extra_names:\n        return set()\n\n    # lookup for packages by name, faster than looping over packages repeatedly\n    packages_by_name = {package.name: package for package in packages}\n\n    # Depth-first search, with our entry points being the packages directly required by\n    # extras.\n    seen_package_names = set()\n    stack = [\n        canonicalize_name(extra_package_name)\n        for extra_name in extra_names\n        for extra_package_name in extras.get(extra_name, ())\n    ]\n\n    while stack:\n        package_name = stack.pop()\n\n        # We expect to find all packages, but can just carry on if we don't.\n        package = packages_by_name.get(package_name)\n        if package is None or package.name in seen_package_names:\n            continue\n\n        seen_package_names.add(package.name)\n\n        stack += [dependency.name for dependency in package.requires]\n\n    return seen_package_names\n\n...\n", "needle_token_start": 11917, "needle_token_end": 12042, "code_context_ntokens": 15975, "output": ["```python\ndef _get_directory_from_hash(self, key_parts: object) -> Path:\n    key = hashlib.sha256(\n        json.dumps(\n            key_parts, sort_keys=True, separators=(\",\", \":\"), ensure_ascii=True\n        ).encode(\"ascii\")\n    ).hexdigest()\n\n    split_key = [key[:2], key[2:4], key[4:6], key[6:]]\n    return self._cache_dir.joinpath(*split_key)\n```"]}
{"repo": "python-poetry/poetry", "name": "_validate_readme", "language": "python", "path": "src/poetry/console/commands/check.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To verify the existence of README files referenced in a project configuration file.\n2. **Input**: A string or list of strings representing README filenames, and a path object pointing to the project configuration file.\n3. **Output**: A list of error messages for each README file that is declared but does not exist in the specified location.\n4. **Procedure**: The function checks if each README file listed exists in the directory of the project configuration file. If a file does not exist, an error message is generated and added to the list of errors, which is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/utils/password_manager.py\nfrom __future__ import annotations\n\nimport dataclasses\nimport functools\nimport logging\n\nfrom contextlib import suppress\nfrom typing import TYPE_CHECKING\n\n\nif TYPE_CHECKING:\n    from keyring.backend import KeyringBackend\n\n    from poetry.config.config import Config\n\nlogger = logging.getLogger(__name__)\n\n\nclass PasswordManagerError(Exception):\n    pass\n\n\nclass PoetryKeyringError(Exception):\n    pass\n\n\n@dataclasses.dataclass\nclass HTTPAuthCredential:\n    username: str | None = dataclasses.field(default=None)\n    password: str | None = dataclasses.field(default=None)\n\n\nclass PoetryKeyring:\n    def __init__(self, namespace: str) -> None:\n        self._namespace = namespace\n\n    def get_credential(\n        self, *names: str, username: str | None = None\n    ) -> HTTPAuthCredential:\n        import keyring\n\n        from keyring.errors import KeyringError\n        from keyring.errors import KeyringLocked\n\n        for name in names:\n            credential = None\n            try:\n                credential = keyring.get_credential(name, username)\n            except KeyringLocked:\n                logger.debug(\"Keyring %s is locked\", name)\n            except (KeyringError, RuntimeError):\n                logger.debug(\"Accessing keyring %s failed\", name, exc_info=True)\n\n            if credential:\n                return HTTPAuthCredential(\n                    username=credential.username, password=credential.password\n                )\n\n        return HTTPAuthCredential(username=username, password=None)\n\n    def get_password(self, name: str, username: str) -> str | None:\n        import keyring\n        import keyring.errors\n\n        name = self.get_entry_name(name)\n\n        try:\n            return keyring.get_password(name, username)\n        except (RuntimeError, keyring.errors.KeyringError):\n            raise PoetryKeyringError(\n                f\"Unable to retrieve the password for {name} from the key ring\"\n            )\n\n    def set_password(self, name: str, username: str, password: str) -> None:\n        import keyring\n        import keyring.errors\n\n        name = self.get_entry_name(name)\n\n        try:\n            keyring.set_password(name, username, password)\n        except (RuntimeError, keyring.errors.KeyringError) as e:\n            raise PoetryKeyringError(\n                f\"Unable to store the password for {name} in the key ring: {e}\"\n            )\n\n    def delete_password(self, name: str, username: str) -> None:\n        import keyring.errors\n\n        name = self.get_entry_name(name)\n\n        try:\n            keyring.delete_password(name, username)\n        except (RuntimeError, keyring.errors.KeyringError):\n            raise PoetryKeyringError(\n                f\"Unable to delete the password for {name} from the key ring\"\n            )\n\n    def get_entry_name(self, name: str) -> str:\n        return f\"{self._namespace}-{name}\"\n\n    @classmethod\n    def is_available(cls) -> bool:\n        logger.debug(\"Checking if keyring is available\")\n        try:\n            import keyring\n            import keyring.backend\n        except ImportError as e:\n            logger.debug(\"An error occurred while importing keyring: %s\", e)\n            return False\n\n        def backend_name(backend: KeyringBackend) -> str:\n            name: str = backend.name\n            return name.split(\" \")[0]\n\n        def backend_is_valid(backend: KeyringBackend) -> bool:\n            name = backend_name(backend)\n            if name in (\"chainer\", \"fail\", \"null\"):\n                logger.debug(f\"Backend {backend.name!r} is not suitable\")\n                return False\n            elif \"plaintext\" in backend.name.lower():\n                logger.debug(f\"Not using plaintext keyring backend {backend.name!r}\")\n                return False\n\n            return True\n\n        backend = keyring.get_keyring()\n        if backend_name(backend) == \"chainer\":\n            backends = keyring.backend.get_all_keyring()\n            valid_backend = next((b for b in backends if backend_is_valid(b)), None)\n        else:\n            valid_backend = backend if backend_is_valid(backend) else None\n\n        if valid_backend is None:\n            logger.debug(\"No valid keyring backend was found\")\n            return False\n        else:\n            logger.debug(f\"Using keyring backend {backend.name!r}\")\n            return True\n\n\nclass PasswordManager:\n    def __init__(self, config: Config) -> None:\n        self._config = config\n\n    @functools.cached_property\n    def use_keyring(self) -> bool:\n        return self._config.get(\"keyring.enabled\") and PoetryKeyring.is_available()\n\n    @functools.cached_property\n    def keyring(self) -> PoetryKeyring:\n        if not self.use_keyring:\n            raise PoetryKeyringError(\n                \"Access to keyring was requested, but it is not available\"\n            )\n\n        return PoetryKeyring(\"poetry-repository\")\n\n    @staticmethod\n    def warn_plaintext_credentials_stored() -> None:\n        logger.warning(\"Using a plaintext file to store credentials\")\n\n    def set_pypi_token(self, repo_name: str, token: str) -> None:\n        if not self.use_keyring:\n            self.warn_plaintext_credentials_stored()\n            self._config.auth_config_source.add_property(\n                f\"pypi-token.{repo_name}\", token\n            )\n        else:\n            self.keyring.set_password(repo_name, \"__token__\", token)\n\n    def get_pypi_token(self, repo_name: str) -> str | None:\n        \"\"\"Get PyPi token.\n\n        First checks the environment variables for a token,\n        then the configured username/password and the\n        available keyring.\n\n        :param repo_name:  Name of repository.\n        :return: Returns a token as a string if found, otherwise None.\n        \"\"\"\n        token: str | None = self._config.get(f\"pypi-token.{repo_name}\")\n        if token:\n            return token\n\n        if self.use_keyring:\n            return self.keyring.get_password(repo_name, \"__token__\")\n        else:\n            return None\n\n    def delete_pypi_token(self, repo_name: str) -> None:\n        if not self.use_keyring:\n            return self._config.auth_config_source.remove_property(\n                f\"pypi-token.{repo_name}\"\n            )\n\n        self.keyring.delete_password(repo_name, \"__token__\")\n\n    def get_http_auth(self, repo_name: str) -> dict[str, str | None] | None:\n        username = self._config.get(f\"http-basic.{repo_name}.username\")\n        password = self._config.get(f\"http-basic.{repo_name}.password\")\n        if not username and not password:\n            return None\n\n        if not password:\n            if self.use_keyring:\n                password = self.keyring.get_password(repo_name, username)\n            else:\n                return None\n\n        return {\n            \"username\": username,\n            \"password\": password,\n        }\n\n    def set_http_password(self, repo_name: str, username: str, password: str) -> None:\n        auth = {\"username\": username}\n\n        if not self.use_keyring:\n            self.warn_plaintext_credentials_stored()\n...\n# Path: src/poetry/utils/patterns.py\nfrom __future__ import annotations\n\nimport re\n\n\nwheel_file_re = re.compile(\n    r\"^(?P<namever>(?P<name>.+?)-(?P<ver>\\d.*?))\"\n    r\"(-(?P<build>\\d.*?))?\"\n    r\"-(?P<pyver>.+?)\"\n    r\"-(?P<abi>.+?)\"\n    r\"-(?P<plat>.+?)\"\n    r\"\\.whl|\\.dist-info$\",\n    re.VERBOSE,\n)\n\nsdist_file_re = re.compile(\n    r\"^(?P<namever>(?P<name>.+?)-(?P<ver>\\d.*?))\"\n    r\"(\\.sdist)?\\.(?P<format>(zip|tar(\\.(gz|bz2|xz|Z))?))$\"\n)\n\n# Path: src/poetry/utils/pip.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.exceptions import PoetryException\nfrom poetry.utils.env import EnvCommandError\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n\n    from poetry.utils.env import Env\n\n\ndef pip_install(\n    path: Path,\n    environment: Env,\n    editable: bool = False,\n    deps: bool = False,\n    upgrade: bool = False,\n) -> str:\n    is_wheel = path.suffix == \".whl\"\n\n    # We disable version check here as we are already pinning to version available in\n    # either the virtual environment or the virtualenv package embedded wheel. Version\n    # checks are a wasteful network call that adds a lot of wait time when installing a\n    # lot of packages.\n    args = [\n        \"install\",\n        \"--disable-pip-version-check\",\n        \"--isolated\",\n        \"--no-input\",\n        \"--prefix\",\n        str(environment.path),\n    ]\n\n    if not is_wheel and not editable:\n        args.insert(1, \"--use-pep517\")\n\n    if upgrade:\n        args.append(\"--upgrade\")\n\n    if not deps:\n        args.append(\"--no-deps\")\n\n    if editable:\n        if not path.is_dir():\n            raise PoetryException(\n                \"Cannot install non directory dependencies in editable mode\"\n            )\n        args.append(\"-e\")\n\n    args.append(str(path))\n\n    try:\n        return environment.run_pip(*args)\n    except EnvCommandError as e:\n        raise PoetryException(f\"Failed to install {path}\") from e\n\n# Path: src/poetry/utils/setup_reader.py\nfrom __future__ import annotations\n\nimport ast\n\nfrom configparser import ConfigParser\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom poetry.core.constraints.version import Version\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n\n\nclass SetupReaderError(Exception):\n    pass\n\n\nclass SetupReader:\n    \"\"\"\n    Class that reads a setup.py file without executing it.\n    \"\"\"\n\n    DEFAULT: ClassVar[dict[str, Any]] = {\n        \"name\": None,\n        \"version\": None,\n        \"description\": None,\n        \"install_requires\": [],\n        \"extras_require\": {},\n        \"python_requires\": None,\n    }\n\n    FILES: ClassVar[list[str]] = [\"setup.py\", \"setup.cfg\"]\n\n    @classmethod\n    def read_from_directory(cls, directory: Path) -> dict[str, Any]:\n        result = cls.DEFAULT.copy()\n        for filename in cls.FILES:\n            filepath = directory / filename\n            if not filepath.exists():\n                continue\n\n            read_file_func = getattr(cls(), \"read_\" + filename.replace(\".\", \"_\"))\n            new_result = read_file_func(filepath)\n\n            for key in result:\n                if new_result[key]:\n                    result[key] = new_result[key]\n\n        return result\n\n    def read_setup_py(self, filepath: Path) -> dict[str, Any]:\n        with filepath.open(encoding=\"utf-8\") as f:\n            content = f.read()\n\n        result: dict[str, Any] = {}\n\n        body = ast.parse(content).body\n\n        setup_call = self._find_setup_call(body)\n        if setup_call is None:\n            return self.DEFAULT\n\n        # Inspecting keyword arguments\n        call, body = setup_call\n        result[\"name\"] = self._find_single_string(call, body, \"name\")\n        result[\"version\"] = self._find_single_string(call, body, \"version\")\n        result[\"description\"] = self._find_single_string(call, body, \"description\")\n        result[\"install_requires\"] = self._find_install_requires(call, body)\n        result[\"extras_require\"] = self._find_extras_require(call, body)\n        result[\"python_requires\"] = self._find_single_string(\n            call, body, \"python_requires\"\n        )\n\n        return result\n\n    def read_setup_cfg(self, filepath: Path) -> dict[str, Any]:\n        parser = ConfigParser()\n\n        parser.read(str(filepath))\n\n        name = None\n        version = None\n        description = None\n        if parser.has_option(\"metadata\", \"name\"):\n            name = parser.get(\"metadata\", \"name\")\n\n        if parser.has_option(\"metadata\", \"version\"):\n            version = Version.parse(parser.get(\"metadata\", \"version\")).text\n\n        if parser.has_option(\"metadata\", \"description\"):\n            description = parser.get(\"metadata\", \"description\")\n\n        install_requires = []\n        extras_require: dict[str, list[str]] = {}\n        python_requires = None\n        if parser.has_section(\"options\"):\n            if parser.has_option(\"options\", \"install_requires\"):\n                for dep in parser.get(\"options\", \"install_requires\").split(\"\\n\"):\n                    dep = dep.strip()\n                    if not dep:\n                        continue\n\n                    install_requires.append(dep)\n\n            if parser.has_option(\"options\", \"python_requires\"):\n                python_requires = parser.get(\"options\", \"python_requires\")\n\n        if parser.has_section(\"options.extras_require\"):\n            for group in parser.options(\"options.extras_require\"):\n                extras_require[group] = []\n                deps = parser.get(\"options.extras_require\", group)\n                for dep in deps.split(\"\\n\"):\n                    dep = dep.strip()\n                    if not dep:\n                        continue\n\n                    extras_require[group].append(dep)\n\n        return {\n            \"name\": name,\n            \"version\": version,\n            \"description\": description,\n            \"install_requires\": install_requires,\n            \"extras_require\": extras_require,\n            \"python_requires\": python_requires,\n        }\n\n    def _find_setup_call(\n        self, elements: list[ast.stmt]\n    ) -> tuple[ast.Call, list[ast.stmt]] | None:\n        funcdefs: list[ast.stmt] = []\n        for i, element in enumerate(elements):\n            if isinstance(element, ast.If) and i == len(elements) - 1:\n                # Checking if the last element is an if statement\n                # and if it is 'if __name__ == \"__main__\"' which\n                # could contain the call to setup()\n                test = element.test\n                if not isinstance(test, ast.Compare):\n                    continue\n\n                left = test.left\n                if not isinstance(left, ast.Name):\n                    continue\n\n                if left.id != \"__name__\":\n                    continue\n\n                setup_call = self._find_sub_setup_call([element])\n                if setup_call is None:\n                    continue\n\n                call, body = setup_call\n                return call, body + elements\n\n            if not isinstance(element, ast.Expr):\n                if isinstance(element, ast.FunctionDef):\n                    funcdefs.append(element)\n\n                continue\n\n            value = element.value\n            if not isinstance(value, ast.Call):\n                continue\n\n            func = value.func\n            if not (isinstance(func, ast.Name) and func.id == \"setup\") and not (\n                isinstance(func, ast.Attribute)\n                and getattr(func.value, \"id\", None) == \"setuptools\"\n                and func.attr == \"setup\"\n            ):\n                continue\n\n            return value, elements\n\n        # Nothing, we inspect the function definitions\n        return self._find_sub_setup_call(funcdefs)\n\n    def _find_sub_setup_call(\n        self, elements: list[ast.stmt]\n    ) -> tuple[ast.Call, list[ast.stmt]] | None:\n        for element in elements:\n            if not isinstance(element, (ast.FunctionDef, ast.If)):\n                continue\n\n            setup_call = self._find_setup_call(element.body)\n            if setup_call is not None:\n                sub_call, body = setup_call\n\n                body = elements + body\n\n                return sub_call, body\n\n        return None\n\n    def _find_install_requires(self, call: ast.Call, body: list[ast.stmt]) -> list[str]:\n        value = self._find_in_call(call, \"install_requires\")\n        if value is None:\n            # Trying to find in kwargs\n            kwargs = self._find_call_kwargs(call)\n\n            if kwargs is None or not isinstance(kwargs, ast.Name):\n                return []\n\n            variable = self._find_variable_in_body(body, kwargs.id)\n\n            if isinstance(variable, ast.Dict):\n                value = self._find_in_dict(variable, \"install_requires\")\n\n            elif (\n                isinstance(variable, ast.Call)\n                and isinstance(variable.func, ast.Name)\n                and variable.func.id == \"dict\"\n            ):\n                value = self._find_in_call(variable, \"install_requires\")\n\n            else:\n                raise SetupReaderError(f\"Cannot handle variable {variable}\")\n\n        if value is None:\n            return []\n\n        if isinstance(value, ast.Name):\n            value = self._find_variable_in_body(body, value.id)\n\n        if isinstance(value, ast.Constant) and value.value is None:\n            return []\n\n        if isinstance(value, ast.List):\n            return string_list_values(value)\n\n        raise SetupReaderError(f\"Cannot handle value of type {type(value)}\")\n\n    def _find_extras_require(\n        self, call: ast.Call, body: list[ast.stmt]\n    ) -> dict[str, list[str]]:\n        value = self._find_in_call(call, \"extras_require\")\n        if value is None:\n            # Trying to find in kwargs\n            kwargs = self._find_call_kwargs(call)\n\n            if kwargs is None or not isinstance(kwargs, ast.Name):\n                return {}\n\n            variable = self._find_variable_in_body(body, kwargs.id)\n            if isinstance(variable, ast.Dict):\n                value = self._find_in_dict(variable, \"extras_require\")\n\n            elif (\n                isinstance(variable, ast.Call)\n                and isinstance(variable.func, ast.Name)\n                and variable.func.id == \"dict\"\n            ):\n                value = self._find_in_call(variable, \"extras_require\")\n\n            else:\n                raise SetupReaderError(f\"Cannot handle variable {variable}\")\n\n        if value is None:\n            return {}\n\n        if isinstance(value, ast.Name):\n            value = self._find_variable_in_body(body, value.id)\n\n        if isinstance(value, ast.Constant) and value.value is None:\n            return {}\n\n        if isinstance(value, ast.Dict):\n            extras_require: dict[str, list[str]] = {}\n            val: ast.expr | None\n            for key, val in zip(value.keys, value.values):\n                if not isinstance(key, ast.Constant) or not isinstance(key.value, str):\n                    raise SetupReaderError(f\"Cannot handle key {key}\")\n\n                if isinstance(val, ast.Name):\n                    val = self._find_variable_in_body(body, val.id)\n\n                if not isinstance(val, ast.List):\n                    raise SetupReaderError(f\"Cannot handle value of type {type(val)}\")\n\n                extras_require[key.value] = string_list_values(val)\n\n            return extras_require\n\n        raise SetupReaderError(f\"Cannot handle value of type {type(value)}\")\n\n    def _find_single_string(\n        self, call: ast.Call, body: list[ast.stmt], name: str\n    ) -> str | None:\n        value = self._find_in_call(call, name)\n        if value is None:\n            # Trying to find in kwargs\n            kwargs = self._find_call_kwargs(call)\n\n            if kwargs is None or not isinstance(kwargs, ast.Name):\n                return None\n\n            variable = self._find_variable_in_body(body, kwargs.id)\n            if not isinstance(variable, (ast.Dict, ast.Call)):\n                return None\n\n            if isinstance(variable, ast.Call):\n                if not isinstance(variable.func, ast.Name):\n                    return None\n\n                if variable.func.id != \"dict\":\n                    return None\n\n                value = self._find_in_call(variable, name)\n            else:\n                value = self._find_in_dict(variable, name)\n\n        if value is None:\n            return None\n\n        if isinstance(value, ast.Constant) and isinstance(value.value, str):\n            return value.value\n        elif isinstance(value, ast.Name):\n            variable = self._find_variable_in_body(body, value.id)\n\n            if (\n                variable is not None\n                and isinstance(variable, ast.Constant)\n                and isinstance(variable.value, str)\n            ):\n                return variable.value\n\n        return None\n\n    def _find_in_call(self, call: ast.Call, name: str) -> Any | None:\n        for keyword in call.keywords:\n            if keyword.arg == name:\n                return keyword.value\n        return None\n\n    def _find_call_kwargs(self, call: ast.Call) -> Any | None:\n        kwargs = None\n        for keyword in call.keywords:\n            if keyword.arg is None:\n                kwargs = keyword.value\n\n        return kwargs\n\n    def _find_variable_in_body(\n        self, body: list[ast.stmt], name: str\n    ) -> ast.expr | None:\n        for elem in body:\n            if not isinstance(elem, ast.Assign):\n                continue\n\n            for target in elem.targets:\n                if not isinstance(target, ast.Name):\n                    continue\n\n                if target.id == name:\n                    return elem.value\n\n        return None\n\n    def _find_in_dict(self, dict_: ast.Dict, name: str) -> ast.expr | None:\n        for key, val in zip(dict_.keys, dict_.values):\n            if (\n                isinstance(key, ast.Constant)\n                and isinstance(key.value, str)\n                and key.value == name\n            ):\n                return val\n\n        return None\n\n\ndef string_list_values(value: ast.List) -> list[str]:\n    strings = []\n    for element in value.elts:\n        if isinstance(element, ast.Constant) and isinstance(element.value, str):\n            strings.append(element.value)\n\n        else:\n            raise SetupReaderError(\"Found non-string element in list\")\n\n    return strings\n\n# Path: src/poetry/utils/shell.py\nfrom __future__ import annotations\n\nimport os\nimport shutil\nimport signal\nimport subprocess\nimport sys\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nimport pexpect\n\nfrom shellingham import ShellDetectionFailure\nfrom shellingham import detect_shell\n\nfrom poetry.utils._compat import WINDOWS\n\n\nif TYPE_CHECKING:\n    from poetry.utils.env import VirtualEnv\n\n\nclass Shell:\n    \"\"\"\n    Represents the current shell.\n    \"\"\"\n\n    _shell = None\n\n    def __init__(self, name: str, path: str) -> None:\n        self._name = name\n        self._path = path\n\n    @property\n    def name(self) -> str:\n        return self._name\n\n    @property\n    def path(self) -> str:\n        return self._path\n\n    @classmethod\n    def get(cls) -> Shell:\n        \"\"\"\n        Retrieve the current shell.\n        \"\"\"\n        if cls._shell is not None:\n            return cls._shell\n\n        try:\n            name, path = detect_shell(os.getpid())\n        except (RuntimeError, ShellDetectionFailure):\n            shell = None\n\n            if os.name == \"posix\":\n                shell = os.environ.get(\"SHELL\")\n            elif os.name == \"nt\":\n                shell = os.environ.get(\"COMSPEC\")\n\n            if not shell:\n                raise RuntimeError(\"Unable to detect the current shell.\")\n\n            name, path = Path(shell).stem, shell\n\n        cls._shell = cls(name, path)\n\n        return cls._shell\n\n    def activate(self, env: VirtualEnv) -> int | None:\n        activate_script = self._get_activate_script()\n        if WINDOWS:\n            bin_path = env.path / \"Scripts\"\n            # Python innstalled via msys2 on Windows might produce a POSIX-like venv\n            # See https://github.com/python-poetry/poetry/issues/8638\n            bin_dir = \"Scripts\" if bin_path.exists() else \"bin\"\n        else:\n            bin_dir = \"bin\"\n        activate_path = env.path / bin_dir / activate_script\n\n        # mypy requires using sys.platform instead of WINDOWS constant\n        # in if statements to properly type check on Windows\n        if sys.platform == \"win32\":\n            args = None\n            if self._name in (\"powershell\", \"pwsh\"):\n                args = [\"-NoExit\", \"-File\", str(activate_path)]\n            elif self._name == \"cmd\":\n                # /K will execute the bat file and\n                # keep the cmd process from terminating\n                args = [\"/K\", str(activate_path)]\n\n            if args:\n                completed_proc = subprocess.run([self.path, *args])\n                return completed_proc.returncode\n            else:\n                # If no args are set, execute the shell within the venv\n                # This activates it, but there could be some features missing:\n                # deactivate command might not work\n                # shell prompt will not be modified.\n                return env.execute(self._path)\n\n        import shlex\n\n        terminal = shutil.get_terminal_size()\n        cmd = f\"{self._get_source_command()} {shlex.quote(str(activate_path))}\"\n\n        with env.temp_environ():\n            if self._name == \"nu\":\n                args = [\"-e\", cmd]\n            elif self._name == \"fish\":\n                args = [\"-i\", \"--init-command\", cmd]\n            else:\n                args = [\"-i\"]\n\n            c = pexpect.spawn(\n                self._path, args, dimensions=(terminal.lines, terminal.columns)\n            )\n\n        if self._name in [\"zsh\"]:\n            c.setecho(False)\n\n        if self._name == \"zsh\":\n            # Under ZSH the source command should be invoked in zsh's bash emulator\n            quoted_activate_path = shlex.quote(str(activate_path))\n            c.sendline(f\"emulate bash -c {shlex.quote(f'. {quoted_activate_path}')}\")\n        elif self._name == \"xonsh\":\n            c.sendline(f\"vox activate {shlex.quote(str(env.path))}\")\n        elif self._name in [\"nu\", \"fish\"]:\n            # If this is nu or fish, we don't want to send the activation command to the\n            # command line since we already ran it via the shell's invocation.\n            pass\n        else:\n            c.sendline(cmd)\n\n        def resize(sig: Any, data: Any) -> None:\n            terminal = shutil.get_terminal_size()\n            c.setwinsize(terminal.lines, terminal.columns)\n\n        signal.signal(signal.SIGWINCH, resize)\n\n        # Interact with the new shell.\n        c.interact(escape_character=None)\n        c.close()\n\n        sys.exit(c.exitstatus)\n\n    def _get_activate_script(self) -> str:\n        if self._name == \"fish\":\n            suffix = \".fish\"\n        elif self._name in (\"csh\", \"tcsh\"):\n            suffix = \".csh\"\n        elif self._name in (\"powershell\", \"pwsh\"):\n            suffix = \".ps1\"\n        elif self._name == \"cmd\":\n            suffix = \".bat\"\n        elif self._name == \"nu\":\n            suffix = \".nu\"\n        else:\n            suffix = \"\"\n\n        return \"activate\" + suffix\n\n    def _get_source_command(self) -> str:\n        if self._name in (\"fish\", \"csh\", \"tcsh\"):\n            return \"source\"\n        elif self._name == \"nu\":\n            return \"overlay use\"\n        return \".\"\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}(\"{self._name}\", \"{self._path}\")'\n\n# Path: src/poetry/utils/source.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\n\nif TYPE_CHECKING:\n    from tomlkit.items import Table\n\n    from poetry.config.source import Source\n\n\ndef source_to_table(source: Source) -> Table:\n    from tomlkit import nl\n    from tomlkit import table\n\n    source_table: Table = table()\n    for key, value in source.to_dict().items():\n        source_table.add(key, value)\n    source_table.add(nl())\n    return source_table\n\n# Path: src/poetry/utils/wheel.py\nfrom __future__ import annotations\n\nimport logging\n\nfrom typing import TYPE_CHECKING\n\nfrom packaging.tags import Tag\n\nfrom poetry.utils.patterns import wheel_file_re\n\n\nif TYPE_CHECKING:\n    from poetry.utils.env import Env\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass InvalidWheelName(Exception):\n    pass\n\n\nclass Wheel:\n    def __init__(self, filename: str) -> None:\n        wheel_info = wheel_file_re.match(filename)\n        if not wheel_info:\n            raise InvalidWheelName(f\"{filename} is not a valid wheel filename.\")\n\n        self.filename = filename\n        self.name = wheel_info.group(\"name\").replace(\"_\", \"-\")\n        self.version = wheel_info.group(\"ver\").replace(\"_\", \"-\")\n        self.build_tag = wheel_info.group(\"build\")\n        self.pyversions = wheel_info.group(\"pyver\").split(\".\")\n        self.abis = wheel_info.group(\"abi\").split(\".\")\n        self.plats = wheel_info.group(\"plat\").split(\".\")\n\n        self.tags = {\n            Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats\n        }\n\n    def get_minimum_supported_index(self, tags: list[Tag]) -> int | None:\n        indexes = [tags.index(t) for t in self.tags if t in tags]\n\n        return min(indexes) if indexes else None\n\n    def is_supported_by_environment(self, env: Env) -> bool:\n        return bool(set(env.supported_tags).intersection(self.tags))\n\n# Path: src/poetry/vcs/__init__.py\n\n# Path: src/poetry/version/__init__.py\n\n# Path: src/poetry/version/version_selector.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.package import Package\n\n    from poetry.repositories import RepositoryPool\n\n\nclass VersionSelector:\n    def __init__(self, pool: RepositoryPool) -> None:\n        self._pool = pool\n\n    def find_best_candidate(\n        self,\n        package_name: str,\n        target_package_version: str | None = None,\n        allow_prereleases: bool = False,\n        source: str | None = None,\n    ) -> Package | None:\n        \"\"\"\n        Given a package name and optional version,\n        returns the latest Package that matches\n        \"\"\"\n        from poetry.factory import Factory\n\n        dependency = Factory.create_dependency(\n            package_name,\n            {\n                \"version\": target_package_version or \"*\",\n                \"allow-prereleases\": allow_prereleases,\n                \"source\": source,\n            },\n        )\n        candidates = self._pool.find_packages(dependency)\n        only_prereleases = all(c.version.is_unstable() for c in candidates)\n\n        if not candidates:\n            return None\n\n        package = None\n        for candidate in candidates:\n            if (\n                candidate.is_prerelease()\n                and not dependency.allows_prereleases()\n                and not only_prereleases\n            ):\n                continue\n\n            # Select highest version of the two\n            if package is None or package.version < candidate.version:\n                package = candidate\n\n        return package\n\n# Path: src/poetry/console/commands/__init__.py\n\n# Path: src/poetry/console/commands/about.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from collections.abc import Callable\n\n\nclass AboutCommand(Command):\n    name = \"about\"\n\n    description = \"Shows information about Poetry.\"\n\n    def handle(self) -> int:\n        from poetry.utils._compat import metadata\n\n        # The metadata.version that we import for Python 3.7 is untyped, work around\n        # that.\n        version: Callable[[str], str] = metadata.version\n\n        self.line(\n            f\"\"\"\\\n<info>Poetry - Package Management for Python\n\nVersion: {version('poetry')}\nPoetry-Core Version: {version('poetry-core')}</info>\n\n<comment>Poetry is a dependency manager tracking local dependencies of your projects\\\n and libraries.\nSee <fg=blue>https://github.com/python-poetry/poetry</> for more information.</comment>\\\n\"\"\"\n        )\n\n        return 0\n\n# Path: src/poetry/console/commands/add.py\nfrom __future__ import annotations\n\nimport contextlib\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\nfrom poetry.core.packages.dependency_group import MAIN_GROUP\nfrom tomlkit.toml_document import TOMLDocument\n\nfrom poetry.console.commands.init import InitCommand\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass AddCommand(InstallerCommand, InitCommand):\n    name = \"add\"\n    description = \"Adds a new dependency to <comment>pyproject.toml</>.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"name\", \"The packages to add.\", multiple=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\n            \"group\",\n            \"-G\",\n            \"The group to add the dependency to.\",\n            flag=False,\n            default=MAIN_GROUP,\n        ),\n        option(\n            \"dev\",\n            \"D\",\n            \"Add as a development dependency. (<warning>Deprecated</warning>) Use\"\n            \" --group=dev instead.\",\n        ),\n        option(\"editable\", \"e\", \"Add vcs/path dependencies as editable.\"),\n        option(\n            \"extras\",\n            \"E\",\n            \"Extras to activate for the dependency.\",\n            flag=False,\n            multiple=True,\n        ),\n        option(\"optional\", None, \"Add as an optional dependency.\"),\n        option(\n            \"python\",\n            None,\n            \"Python version for which the dependency must be installed.\",\n            flag=False,\n        ),\n        option(\n            \"platform\",\n            None,\n            \"Platforms for which the dependency must be installed.\",\n            flag=False,\n        ),\n        option(\n            \"source\",\n            None,\n            \"Name of the source to use to install the package.\",\n            flag=False,\n        ),\n        option(\"allow-prereleases\", None, \"Accept prereleases.\"),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything (implicitly enables\"\n            \" --verbose).\",\n        ),\n        option(\"lock\", None, \"Do not perform operations (only update the lockfile).\"),\n    ]\n    examples = \"\"\"\\\nIf you do not specify a version constraint, poetry will choose a suitable one based on\\\n the available package versions.\n\nYou can specify a package in the following forms:\n  - A single name (<b>requests</b>)\n  - A name and a constraint (<b>requests@^2.23.0</b>)\n  - A git url (<b>git+https://github.com/python-poetry/poetry.git</b>)\n  - A git url with a revision\\\n (<b>git+https://github.com/python-poetry/poetry.git#develop</b>)\n  - A subdirectory of a git repository\\\n (<b>git+https://github.com/python-poetry/poetry.git#subdirectory=tests/fixtures/sample_project</b>)\n  - A git SSH url (<b>git+ssh://github.com/python-poetry/poetry.git</b>)\n  - A git SSH url with a revision\\\n (<b>git+ssh://github.com/python-poetry/poetry.git#develop</b>)\n  - A file path (<b>../my-package/my-package.whl</b>)\n  - A directory (<b>../my-package/</b>)\n  - A url (<b>https://example.com/packages/my-package-0.1.0.tar.gz</b>)\n\"\"\"\n    help = f\"\"\"\\\nThe add command adds required packages to your <comment>pyproject.toml</> and installs\\\n them.\n\n{examples}\n\"\"\"\n\n    loggers: ClassVar[list[str]] = [\n        \"poetry.repositories.pypi_repository\",\n        \"poetry.inspection.info\",\n    ]\n\n    def handle(self) -> int:\n        from poetry.core.constraints.version import parse_constraint\n        from tomlkit import inline_table\n        from tomlkit import parse as parse_toml\n        from tomlkit import table\n\n        from poetry.factory import Factory\n\n        packages = self.argument(\"name\")\n        if self.option(\"dev\"):\n            self.line_error(\n                \"<warning>The --dev option is deprecated, \"\n                \"use the `--group dev` notation instead.</warning>\"\n            )\n            group = \"dev\"\n        else:\n            group = self.option(\"group\", self.default_group or MAIN_GROUP)\n\n        if self.option(\"extras\") and len(packages) > 1:\n            raise ValueError(\n                \"You can only specify one package when using the --extras option\"\n            )\n\n        # tomlkit types are awkward to work with, treat content as a mostly untyped\n        # dictionary.\n        content: dict[str, Any] = self.poetry.file.read()\n        poetry_content = content[\"tool\"][\"poetry\"]\n        project_name = (\n            canonicalize_name(name) if (name := poetry_content.get(\"name\")) else None\n        )\n\n        if group == MAIN_GROUP:\n            if \"dependencies\" not in poetry_content:\n                poetry_content[\"dependencies\"] = table()\n\n            section = poetry_content[\"dependencies\"]\n        else:\n            if \"group\" not in poetry_content:\n                poetry_content[\"group\"] = table(is_super_table=True)\n\n            groups = poetry_content[\"group\"]\n            if group not in groups:\n                dependencies_toml: dict[str, Any] = parse_toml(\n                    f\"[tool.poetry.group.{group}.dependencies]\\n\\n\"\n                )\n                group_table = dependencies_toml[\"tool\"][\"poetry\"][\"group\"][group]\n                poetry_content[\"group\"][group] = group_table\n\n            if \"dependencies\" not in poetry_content[\"group\"][group]:\n                poetry_content[\"group\"][group][\"dependencies\"] = table()\n\n            section = poetry_content[\"group\"][group][\"dependencies\"]\n\n        existing_packages = self.get_existing_packages_from_input(packages, section)\n\n        if existing_packages:\n            self.notify_about_existing_packages(existing_packages)\n\n        packages = [name for name in packages if name not in existing_packages]\n\n        if not packages:\n            self.line(\"Nothing to add.\")\n            return 0\n\n        requirements = self._determine_requirements(\n            packages,\n            allow_prereleases=self.option(\"allow-prereleases\"),\n            source=self.option(\"source\"),\n        )\n\n        for _constraint in requirements:\n            version = _constraint.get(\"version\")\n            if version is not None:\n                # Validate version constraint\n                assert isinstance(version, str)\n                parse_constraint(version)\n\n            constraint: dict[str, Any] = inline_table()\n            for name, value in _constraint.items():\n                if name == \"name\":\n                    continue\n\n                constraint[name] = value\n\n            if self.option(\"optional\"):\n                constraint[\"optional\"] = True\n\n            if self.option(\"allow-prereleases\"):\n                constraint[\"allow-prereleases\"] = True\n\n            if self.option(\"extras\"):\n                extras = []\n                for extra in self.option(\"extras\"):\n                    extras += extra.split()\n\n                constraint[\"extras\"] = extras\n\n            if self.option(\"editable\"):\n                if \"git\" in _constraint or \"path\" in _constraint:\n                    constraint[\"develop\"] = True\n                else:\n                    self.line_error(\n                        \"\\n\"\n                        \"<error>Failed to add packages. \"\n                        \"Only vcs/path dependencies support editable installs. \"\n                        f\"<c1>{_constraint['name']}</c1> is neither.\"\n                    )\n                    self.line_error(\"\\nNo changes were applied.\")\n                    return 1\n\n            if self.option(\"python\"):\n                constraint[\"python\"] = self.option(\"python\")\n\n            if self.option(\"platform\"):\n                constraint[\"platform\"] = self.option(\"platform\")\n\n            if self.option(\"source\"):\n                constraint[\"source\"] = self.option(\"source\")\n\n            if len(constraint) == 1 and \"version\" in constraint:\n                constraint = constraint[\"version\"]\n\n            constraint_name = _constraint[\"name\"]\n            assert isinstance(constraint_name, str)\n\n            canonical_constraint_name = canonicalize_name(constraint_name)\n\n            if canonical_constraint_name == project_name:\n                self.line_error(\n                    f\"<error>Cannot add dependency on <c1>{constraint_name}</c1> to\"\n                    \" project with the same name.\"\n                )\n                self.line_error(\"\\nNo changes were applied.\")\n                return 1\n\n            for key in section:\n                if canonicalize_name(key) == canonical_constraint_name:\n                    section[key] = constraint\n                    break\n            else:\n                section[constraint_name] = constraint\n\n            with contextlib.suppress(ValueError):\n                self.poetry.package.dependency_group(group).remove_dependency(\n                    constraint_name\n                )\n\n            self.poetry.package.add_dependency(\n                Factory.create_dependency(\n                    constraint_name,\n                    constraint,\n                    groups=[group],\n                    root_dir=self.poetry.file.path.parent,\n                )\n            )\n\n        # Refresh the locker\n        self.poetry.locker.set_local_config(poetry_content)\n        self.installer.set_locker(self.poetry.locker)\n\n        # Cosmetic new line\n        self.line(\"\")\n\n        self.installer.set_package(self.poetry.package)\n        self.installer.dry_run(self.option(\"dry-run\"))\n        self.installer.verbose(self.io.is_verbose())\n        self.installer.update(True)\n        self.installer.execute_operations(not self.option(\"lock\"))\n\n        self.installer.whitelist([r[\"name\"] for r in requirements])\n\n        status = self.installer.run()\n\n        if status == 0 and not self.option(\"dry-run\"):\n            assert isinstance(content, TOMLDocument)\n            self.poetry.file.write(content)\n\n        return status\n\n    def get_existing_packages_from_input(\n        self, packages: list[str], section: dict[str, Any]\n    ) -> list[str]:\n        existing_packages = []\n\n        for name in packages:\n            for key in section:\n                if canonicalize_name(key) == canonicalize_name(name):\n                    existing_packages.append(name)\n\n        return existing_packages\n\n    @property\n    def _hint_update_packages(self) -> str:\n        return (\n            \"\\nIf you want to update it to the latest compatible version, you can use\"\n            \" `poetry update package`.\\nIf you prefer to upgrade it to the latest\"\n            \" available version, you can use `poetry add package@latest`.\\n\"\n        )\n\n    def notify_about_existing_packages(self, existing_packages: list[str]) -> None:\n        self.line(\n            \"The following packages are already present in the pyproject.toml and will\"\n            \" be skipped:\\n\"\n        )\n        for name in existing_packages:\n            self.line(f\"  - <c1>{name}</c1>\")\n        self.line(self._hint_update_packages)\n\n# Path: src/poetry/console/commands/build.py\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.utils.env import build_environment\nfrom poetry.utils.helpers import remove_directory\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.option import Option\n\n\nclass BuildCommand(EnvCommand):\n    name = \"build\"\n    description = \"Builds a package, as a tarball and a wheel by default.\"\n\n    options: ClassVar[list[Option]] = [\n        option(\"format\", \"f\", \"Limit the format to either sdist or wheel.\", flag=False),\n        option(\n            \"clean\",\n            \"Clean output directory before building.\",\n            flag=True,\n        ),\n        option(\n            \"local-version\",\n            \"l\",\n            \"Add or replace a local version label to the build.\",\n            flag=False,\n        ),\n        option(\n            \"output\",\n            \"o\",\n            \"Set output directory for build artifacts. Default is `dist`.\",\n            default=\"dist\",\n            flag=False,\n        ),\n    ]\n\n    loggers: ClassVar[list[str]] = [\n        \"poetry.core.masonry.builders.builder\",\n        \"poetry.core.masonry.builders.sdist\",\n        \"poetry.core.masonry.builders.wheel\",\n    ]\n\n    def _build(\n        self,\n        fmt: str,\n        executable: str | Path | None = None,\n        *,\n        target_dir: Path | None = None,\n    ) -> None:\n        from poetry.masonry.builders import BUILD_FORMATS\n\n        if fmt in BUILD_FORMATS:\n            builders = [BUILD_FORMATS[fmt]]\n        elif fmt == \"all\":\n            builders = list(BUILD_FORMATS.values())\n        else:\n            raise ValueError(f\"Invalid format: {fmt}\")\n\n        if local_version_label := self.option(\"local-version\"):\n            self.poetry.package.version = self.poetry.package.version.replace(\n                local=local_version_label\n            )\n\n        for builder in builders:\n            builder(self.poetry, executable=executable).build(target_dir)\n\n    def handle(self) -> int:\n        if not self.poetry.is_package_mode:\n            self.line_error(\"Building a package is not possible in non-package mode.\")\n            return 1\n\n        with build_environment(poetry=self.poetry, env=self.env, io=self.io) as env:\n            fmt = self.option(\"format\") or \"all\"\n            dist_dir = Path(self.option(\"output\"))\n            package = self.poetry.package\n            self.line(\n                f\"Building <c1>{package.pretty_name}</c1> (<c2>{package.version}</c2>)\"\n            )\n\n            if not dist_dir.is_absolute():\n                dist_dir = self.poetry.pyproject_path.parent / dist_dir\n\n            if self.option(\"clean\"):\n                remove_directory(path=dist_dir, force=True)\n\n            self._build(fmt, executable=env.python, target_dir=dist_dir)\n\n        return 0\n\n# Path: src/poetry/console/commands/check.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from pathlib import Path\n\n    from cleo.io.inputs.option import Option\n\n\nclass CheckCommand(Command):\n    name = \"check\"\n    description = (\n        \"Validates the content of the <comment>pyproject.toml</> file and its\"\n        \" consistency with the poetry.lock file.\"\n    )\n\n    options: ClassVar[list[Option]] = [\n        option(\n            \"lock\",\n            None,\n            \"Checks that <comment>poetry.lock</> exists for the current\"\n            \" version of <comment>pyproject.toml</>.\",\n        ),\n    ]\n\n    def _validate_classifiers(\n        self, project_classifiers: set[str]\n    ) -> tuple[list[str], list[str]]:\n        \"\"\"Identify unrecognized and deprecated trove classifiers.\n\n        A fully-qualified classifier is a string delimited by `` :: `` separators. To\n        make the error message more readable we need to have visual clues to\n        materialize the start and end of a classifier string. That way the user can\n        easily copy and paste it from the messages while reducing mistakes because of\n        extra spaces.\n\n        We use ``!r`` (``repr()``) for classifiers and list of classifiers for\n        consistency. That way all strings will be rendered with the same kind of quotes\n        (i.e. simple tick: ``'``).\n        \"\"\"\n        from trove_classifiers import classifiers\n        from trove_classifiers import deprecated_classifiers\n\n        errors = []\n        warnings = []\n\n        unrecognized = sorted(\n            project_classifiers - set(classifiers) - set(deprecated_classifiers)\n        )\n        # Allow \"Private ::\" classifiers as recommended on PyPI and the packaging guide\n        # to allow users to avoid accidentally publishing private packages to PyPI.\n        # https://pypi.org/classifiers/\n        unrecognized = [u for u in unrecognized if not u.startswith(\"Private ::\")]\n        if unrecognized:\n            errors.append(f\"Unrecognized classifiers: {unrecognized!r}.\")\n\n        deprecated = sorted(\n            project_classifiers.intersection(set(deprecated_classifiers))\n        )\n        if deprecated:\n            for old_classifier in deprecated:\n                new_classifiers = deprecated_classifiers[old_classifier]\n                if new_classifiers:\n                    message = (\n                        f\"Deprecated classifier {old_classifier!r}. \"\n                        f\"Must be replaced by {new_classifiers!r}.\"\n                    )\n                else:\n                    message = (\n                        f\"Deprecated classifier {old_classifier!r}. Must be removed.\"\n                    )\n                warnings.append(message)\n\n        return errors, warnings\n\n    \ndef _validate_readme(self, readme: str | list[str], poetry_file: Path) -> list[str]:\n        \"\"\"Check existence of referenced readme files\"\"\"\n\n        readmes = [readme] if isinstance(readme, str) else readme\n\n        errors = []\n        for name in readmes:\n            if not (poetry_file.parent / name).exists():\n                errors.append(f\"Declared README file does not exist: {name}\")\n        return errors\n\n    def _validate_dependencies_source(self, config: dict[str, Any]) -> list[str]:\n        \"\"\"Check dependencies's source are valid\"\"\"\n        sources = {k[\"name\"] for k in config.get(\"source\", [])}\n\n        dependency_declarations: list[\n            dict[str, str | dict[str, str] | list[dict[str, str]]]\n        ] = []\n        # scan dependencies and group dependencies settings in pyproject.toml\n        if \"dependencies\" in config:\n            dependency_declarations.append(config[\"dependencies\"])\n\n        for group in config.get(\"group\", {}).values():\n            if \"dependencies\" in group:\n                dependency_declarations.append(group[\"dependencies\"])\n\n        all_referenced_sources: set[str] = set()\n\n        for dependency_declaration in dependency_declarations:\n            for declaration in dependency_declaration.values():\n                if isinstance(declaration, list):\n                    for item in declaration:\n                        if \"source\" in item:\n                            all_referenced_sources.add(item[\"source\"])\n                elif isinstance(declaration, dict) and \"source\" in declaration:\n                    all_referenced_sources.add(declaration[\"source\"])\n\n        return [\n            f'Invalid source \"{source}\" referenced in dependencies.'\n            for source in sorted(all_referenced_sources - sources)\n        ]\n\n    def handle(self) -> int:\n        from poetry.core.pyproject.toml import PyProjectTOML\n\n        from poetry.factory import Factory\n\n        # Load poetry config and display errors, if any\n        poetry_file = self.poetry.file.path\n        config = PyProjectTOML(poetry_file).poetry_config\n        check_result = Factory.validate(config, strict=True)\n\n        # Validate trove classifiers\n        project_classifiers = set(config.get(\"classifiers\", []))\n        errors, warnings = self._validate_classifiers(project_classifiers)\n        check_result[\"errors\"].extend(errors)\n        check_result[\"warnings\"].extend(warnings)\n\n        # Validate readme (files must exist)\n        if \"readme\" in config:\n            errors = self._validate_readme(config[\"readme\"], poetry_file)\n            check_result[\"errors\"].extend(errors)\n\n        check_result[\"errors\"] += self._validate_dependencies_source(config)\n\n        # Verify that lock file is consistent\n        if self.option(\"lock\") and not self.poetry.locker.is_locked():\n            check_result[\"errors\"] += [\"poetry.lock was not found.\"]\n        if self.poetry.locker.is_locked() and not self.poetry.locker.is_fresh():\n            check_result[\"errors\"] += [\n                \"pyproject.toml changed significantly since poetry.lock was last generated. \"\n                \"Run `poetry lock [--no-update]` to fix the lock file.\"\n            ]\n\n        if not check_result[\"errors\"] and not check_result[\"warnings\"]:\n            self.info(\"All set!\")\n\n            return 0\n\n        for error in check_result[\"errors\"]:\n            self.line_error(f\"<error>Error: {error}</error>\")\n\n        for error in check_result[\"warnings\"]:\n            self.line_error(f\"<warning>Warning: {error}</warning>\")\n\n        return 1\n\n# Path: src/poetry/console/commands/command.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom cleo.commands.command import Command as BaseCommand\nfrom cleo.exceptions import CleoValueError\n\n\nif TYPE_CHECKING:\n    from poetry.console.application import Application\n    from poetry.poetry import Poetry\n\n\nclass Command(BaseCommand):\n    loggers: ClassVar[list[str]] = []\n\n    _poetry: Poetry | None = None\n\n    @property\n    def poetry(self) -> Poetry:\n        if self._poetry is None:\n            return self.get_application().poetry\n\n        return self._poetry\n\n    def set_poetry(self, poetry: Poetry) -> None:\n        self._poetry = poetry\n\n    def get_application(self) -> Application:\n        from poetry.console.application import Application\n\n        application = self.application\n        assert isinstance(application, Application)\n        return application\n\n    def reset_poetry(self) -> None:\n        self.get_application().reset_poetry()\n\n    def option(self, name: str, default: Any = None) -> Any:\n        try:\n            return super().option(name)\n        except CleoValueError:\n            return default\n\n# Path: src/poetry/console/commands/config.py\nfrom __future__ import annotations\n\nimport json\nimport re\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\nfrom typing import cast\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n\nfrom poetry.config.config import PackageFilterPolicy\nfrom poetry.config.config import boolean_normalizer\nfrom poetry.config.config import boolean_validator\nfrom poetry.config.config import int_normalizer\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n    from poetry.config.config_source import ConfigSource\n\n\nclass ConfigCommand(Command):\n    name = \"config\"\n    description = \"Manages configuration settings.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"key\", \"Setting key.\", optional=True),\n        argument(\"value\", \"Setting value.\", optional=True, multiple=True),\n    ]\n\n    options: ClassVar[list[Option]] = [\n        option(\"list\", None, \"List configuration settings.\"),\n        option(\"unset\", None, \"Unset configuration setting.\"),\n        option(\"local\", None, \"Set/Get from the project's local configuration.\"),\n    ]\n\n    help = \"\"\"\\\nThis command allows you to edit the poetry config settings and repositories.\n\nTo add a repository:\n\n    <comment>poetry config repositories.foo https://bar.com/simple/</comment>\n\nTo remove a repository (repo is a short alias for repositories):\n\n    <comment>poetry config --unset repo.foo</comment>\"\"\"\n\n    LIST_PROHIBITED_SETTINGS: ClassVar[set[str]] = {\"http-basic\", \"pypi-token\"}\n\n    @property\n    def unique_config_values(self) -> dict[str, tuple[Any, Any]]:\n        unique_config_values = {\n            \"cache-dir\": (str, lambda val: str(Path(val))),\n            \"virtualenvs.create\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.in-project\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.options.always-copy\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.options.system-site-packages\": (\n                boolean_validator,\n                boolean_normalizer,\n            ),\n            \"virtualenvs.options.no-pip\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.options.no-setuptools\": (\n                boolean_validator,\n                boolean_normalizer,\n            ),\n            \"virtualenvs.path\": (str, lambda val: str(Path(val))),\n            \"virtualenvs.prefer-active-python\": (boolean_validator, boolean_normalizer),\n            \"virtualenvs.prompt\": (str, str),\n            \"experimental.system-git-client\": (boolean_validator, boolean_normalizer),\n            \"installer.modern-installation\": (boolean_validator, boolean_normalizer),\n            \"installer.parallel\": (boolean_validator, boolean_normalizer),\n            \"installer.max-workers\": (lambda val: int(val) > 0, int_normalizer),\n            \"installer.no-binary\": (\n                PackageFilterPolicy.validator,\n                PackageFilterPolicy.normalize,\n            ),\n            \"solver.lazy-wheel\": (boolean_validator, boolean_normalizer),\n            \"warnings.export\": (boolean_validator, boolean_normalizer),\n            \"keyring.enabled\": (boolean_validator, boolean_normalizer),\n        }\n\n        return unique_config_values\n\n    def handle(self) -> int:\n        from pathlib import Path\n\n        from poetry.core.pyproject.exceptions import PyProjectException\n\n        from poetry.config.config import Config\n        from poetry.config.file_config_source import FileConfigSource\n        from poetry.locations import CONFIG_DIR\n        from poetry.toml.file import TOMLFile\n\n        config = Config.create()\n        config_file = TOMLFile(CONFIG_DIR / \"config.toml\")\n\n        try:\n            local_config_file = TOMLFile(self.poetry.file.path.parent / \"poetry.toml\")\n            if local_config_file.exists():\n                config.merge(local_config_file.read())\n        except (RuntimeError, PyProjectException):\n            local_config_file = TOMLFile(Path.cwd() / \"poetry.toml\")\n\n        if self.option(\"local\"):\n...\n", "needle_token_start": 13503, "needle_token_end": 13622, "code_context_ntokens": 15951, "output": ["```python\ndef _validate_readme(self, readme: str | list[str], poetry_file: Path) -> list[str]:\n    \"\"\"Check existence of referenced readme files\"\"\"\n\n    readmes = [readme] if isinstance(readme, str) else readme\n\n    errors = []\n    for name in readmes:\n        if not (poetry_file.parent / name).exists():\n            errors.append(f\"Declared README file does not exist: {name}\")\n    return errors\n```"]}
{"repo": "python-poetry/poetry", "name": "base", "language": "python", "path": "src/poetry/utils/env/base_env.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this attribute is to provide the root directory of the Python environment, which may differ from the active environment's directory if the environment is nested or linked to a different base environment.\n2. **Input**: There is no direct input required as it is an attribute that retrieves the stored path.\n3. **Output**: The output is the path to the base directory of the Python environment.\n4. **Procedure**: The procedure involves checking if a base path was explicitly provided during the initialization of the environment object. If not provided, it defaults to the path of the current environment. This path is then adjusted for Windows systems to ensure compatibility.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/poetry/console/commands/show.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom packaging.utils import canonicalize_name\n\nfrom poetry.console.commands.env_command import EnvCommand\nfrom poetry.console.commands.group_command import GroupCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n    from cleo.io.io import IO\n    from cleo.ui.table import Rows\n    from packaging.utils import NormalizedName\n    from poetry.core.packages.dependency import Dependency\n    from poetry.core.packages.package import Package\n    from poetry.core.packages.project_package import ProjectPackage\n\n    from poetry.repositories.repository import Repository\n\n\ndef reverse_deps(pkg: Package, repo: Repository) -> dict[str, str]:\n    required_by = {}\n    for locked in repo.packages:\n        dependencies = {d.name: d.pretty_constraint for d in locked.requires}\n\n        if pkg.name in dependencies:\n            required_by[locked.pretty_name] = dependencies[pkg.name]\n\n    return required_by\n\n\nclass ShowCommand(GroupCommand, EnvCommand):\n    name = \"show\"\n    description = \"Shows information about packages.\"\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"package\", \"The package to inspect\", optional=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        *GroupCommand._group_dependency_options(),\n        option(\n            \"no-dev\",\n            None,\n            \"Do not list the development dependencies. (<warning>Deprecated</warning>)\",\n        ),\n        option(\"tree\", \"t\", \"List the dependencies as a tree.\"),\n        option(\n            \"why\",\n            None,\n            \"When showing the full list, or a <info>--tree</info> for a single package,\"\n            \" display whether they are a direct dependency or required by other\"\n            \" packages\",\n        ),\n        option(\"latest\", \"l\", \"Show the latest version.\"),\n        option(\n            \"outdated\",\n            \"o\",\n            \"Show the latest version but only for packages that are outdated.\",\n        ),\n        option(\n            \"all\",\n            \"a\",\n            \"Show all packages (even those not compatible with current system).\",\n        ),\n        option(\"top-level\", \"T\", \"Show only top-level dependencies.\"),\n    ]\n\n    help = \"\"\"The show command displays detailed information about a package, or\nlists all packages available.\"\"\"\n\n    colors: ClassVar[list[str]] = [\"cyan\", \"yellow\", \"green\", \"magenta\", \"blue\"]\n\n    def handle(self) -> int:\n        package = self.argument(\"package\")\n\n        if self.option(\"tree\"):\n            self.init_styles(self.io)\n\n        if self.option(\"top-level\"):\n            if self.option(\"tree\"):\n                self.line_error(\n                    \"<error>Error: Cannot use --tree and --top-level at the same\"\n                    \" time.</error>\"\n                )\n                return 1\n            if package is not None:\n                self.line_error(\n                    \"<error>Error: Cannot use --top-level when displaying a single\"\n                    \" package.</error>\"\n                )\n                return 1\n\n        if self.option(\"why\"):\n            if self.option(\"tree\") and package is None:\n                self.line_error(\n                    \"<error>Error: --why requires a package when combined with\"\n                    \" --tree.</error>\"\n                )\n\n                return 1\n\n            if not self.option(\"tree\") and package:\n                self.line_error(\n                    \"<error>Error: --why cannot be used without --tree when displaying\"\n                    \" a single package.</error>\"\n                )\n\n                return 1\n\n        if self.option(\"outdated\"):\n            self.io.input.set_option(\"latest\", True)\n\n        if not self.poetry.locker.is_locked():\n            self.line_error(\n                \"<error>Error: poetry.lock not found. Run `poetry lock` to create\"\n                \" it.</error>\"\n            )\n            return 1\n\n        locked_repo = self.poetry.locker.locked_repository()\n\n        if package:\n            return self._display_single_package_information(package, locked_repo)\n\n        root = self.project_with_activated_groups_only()\n\n        # Show tree view if requested\n        if self.option(\"tree\"):\n            return self._display_packages_tree_information(locked_repo, root)\n\n        return self._display_packages_information(locked_repo, root)\n\n    def _display_single_package_information(\n        self, package: str, locked_repository: Repository\n    ) -> int:\n        locked_packages = locked_repository.packages\n        canonicalized_package = canonicalize_name(package)\n        pkg = None\n\n        for locked in locked_packages:\n            if locked.name == canonicalized_package:\n                pkg = locked\n                break\n\n        if not pkg:\n            raise ValueError(f\"Package {package} not found\")\n\n        required_by = reverse_deps(pkg, locked_repository)\n\n        if self.option(\"tree\"):\n            if self.option(\"why\"):\n                # The default case if there's no reverse dependencies is to query\n                # the subtree for pkg but if any rev-deps exist we'll query for each\n                # of them in turn\n                packages = [pkg]\n                if required_by:\n                    packages = [\n                        p for p in locked_packages for r in required_by if p.name == r\n                    ]\n                else:\n                    # if no rev-deps exist we'll make this clear as it can otherwise\n                    # look very odd for packages that also have no or few direct\n                    # dependencies\n                    self.io.write_line(f\"Package {package} is a direct dependency.\")\n\n                for p in packages:\n                    self.display_package_tree(\n                        self.io, p, locked_packages, why_package=pkg\n                    )\n\n            else:\n                self.display_package_tree(self.io, pkg, locked_packages)\n\n            return 0\n\n        rows: Rows = [\n            [\"<info>name</>\", f\" : <c1>{pkg.pretty_name}</>\"],\n            [\"<info>version</>\", f\" : <b>{pkg.pretty_version}</b>\"],\n            [\"<info>description</>\", f\" : {pkg.description}\"],\n        ]\n\n        self.table(rows=rows, style=\"compact\").render()\n\n        if pkg.requires:\n            self.line(\"\")\n            self.line(\"<info>dependencies</info>\")\n            for dependency in pkg.requires:\n                self.line(\n                    f\" - <c1>{dependency.pretty_name}</c1>\"\n                    f\" <b>{dependency.pretty_constraint}</b>\"\n                )\n\n        if required_by:\n            self.line(\"\")\n            self.line(\"<info>required by</info>\")\n            for parent, requires_version in required_by.items():\n                self.line(f\" - <c1>{parent}</c1> <b>{requires_version}</b>\")\n\n        return 0\n\n    def _display_packages_information(\n        self, locked_repository: Repository, root: ProjectPackage\n    ) -> int:\n        import shutil\n\n        from cleo.io.null_io import NullIO\n\n        from poetry.puzzle.solver import Solver\n        from poetry.repositories.installed_repository import InstalledRepository\n        from poetry.repositories.repository_pool import RepositoryPool\n        from poetry.utils.helpers import get_package_version_display_string\n\n        locked_packages = locked_repository.packages\n        pool = RepositoryPool.from_packages(locked_packages, self.poetry.config)\n        solver = Solver(\n            root,\n            pool=pool,\n            installed=[],\n            locked=locked_packages,\n            io=NullIO(),\n        )\n        solver.provider.load_deferred(False)\n        with solver.use_environment(self.env):\n            ops = solver.solve().calculate_operations()\n\n        required_locked_packages = {op.package for op in ops if not op.skipped}\n\n        show_latest = self.option(\"latest\")\n        show_all = self.option(\"all\")\n        show_top_level = self.option(\"top-level\")\n        width = shutil.get_terminal_size().columns\n        name_length = version_length = latest_length = required_by_length = 0\n        latest_packages = {}\n        latest_statuses = {}\n        installed_repo = InstalledRepository.load(self.env)\n\n        # Computing widths\n        for locked in locked_packages:\n            if locked not in required_locked_packages and not show_all:\n                continue\n\n            current_length = len(locked.pretty_name)\n            if not self.io.output.is_decorated():\n                installed_status = self.get_installed_status(\n                    locked, installed_repo.packages\n                )\n\n                if installed_status == \"not-installed\":\n                    current_length += 4\n\n            if show_latest:\n                latest = self.find_latest_package(locked, root)\n                if not latest:\n                    latest = locked\n\n                latest_packages[locked.pretty_name] = latest\n                update_status = latest_statuses[locked.pretty_name] = (\n                    self.get_update_status(latest, locked)\n                )\n\n                if not self.option(\"outdated\") or update_status != \"up-to-date\":\n                    name_length = max(name_length, current_length)\n                    version_length = max(\n                        version_length,\n                        len(\n                            get_package_version_display_string(\n                                locked, root=self.poetry.file.path.parent\n                            )\n                        ),\n...\n# Path: src/poetry/console/commands/update.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\n\nfrom poetry.console.commands.installer_command import InstallerCommand\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n\n\nclass UpdateCommand(InstallerCommand):\n    name = \"update\"\n    description = (\n        \"Update the dependencies as according to the <comment>pyproject.toml</> file.\"\n    )\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\"packages\", \"The packages to update\", optional=True, multiple=True)\n    ]\n    options: ClassVar[list[Option]] = [\n        *InstallerCommand._group_dependency_options(),\n        option(\n            \"no-dev\",\n            None,\n            \"Do not update the development dependencies.\"\n            \" (<warning>Deprecated</warning>)\",\n        ),\n        option(\n            \"sync\",\n            None,\n            \"Synchronize the environment with the locked packages and the specified\"\n            \" groups.\",\n        ),\n        option(\n            \"dry-run\",\n            None,\n            \"Output the operations but do not execute anything \"\n            \"(implicitly enables --verbose).\",\n        ),\n        option(\"lock\", None, \"Do not perform operations (only update the lockfile).\"),\n    ]\n\n    loggers: ClassVar[list[str]] = [\"poetry.repositories.pypi_repository\"]\n\n    def handle(self) -> int:\n        packages = self.argument(\"packages\")\n        if packages:\n            self.installer.whitelist({name: \"*\" for name in packages})\n\n        self.installer.only_groups(self.activated_groups)\n        self.installer.dry_run(self.option(\"dry-run\"))\n        self.installer.requires_synchronization(self.option(\"sync\"))\n        self.installer.execute_operations(not self.option(\"lock\"))\n\n        # Force update\n        self.installer.update(True)\n\n        return self.installer.run()\n\n# Path: src/poetry/console/commands/version.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import Any\nfrom typing import ClassVar\n\nfrom cleo.helpers import argument\nfrom cleo.helpers import option\nfrom poetry.core.version.exceptions import InvalidVersion\nfrom tomlkit.toml_document import TOMLDocument\n\nfrom poetry.console.commands.command import Command\n\n\nif TYPE_CHECKING:\n    from cleo.io.inputs.argument import Argument\n    from cleo.io.inputs.option import Option\n    from poetry.core.constraints.version import Version\n\n\nclass VersionCommand(Command):\n    name = \"version\"\n    description = (\n        \"Shows the version of the project or bumps it when a valid \"\n        \"bump rule is provided.\"\n    )\n\n    arguments: ClassVar[list[Argument]] = [\n        argument(\n            \"version\",\n            \"The version number or the rule to update the version.\",\n            optional=True,\n        ),\n    ]\n    options: ClassVar[list[Option]] = [\n        option(\"short\", \"s\", \"Output the version number only\"),\n        option(\n            \"dry-run\",\n            None,\n            \"Do not update pyproject.toml file\",\n        ),\n        option(\"next-phase\", None, \"Increment the phase of the current version\"),\n    ]\n\n    help = \"\"\"\\\nThe version command shows the current version of the project or bumps the version of\nthe project and writes the new version back to <comment>pyproject.toml</> if a valid\nbump rule is provided.\n\nThe new version should ideally be a valid semver string or a valid bump rule:\npatch, minor, major, prepatch, preminor, premajor, prerelease.\n\"\"\"\n\n    RESERVED: ClassVar[set[str]] = {\n        \"major\",\n        \"minor\",\n        \"patch\",\n        \"premajor\",\n        \"preminor\",\n        \"prepatch\",\n        \"prerelease\",\n    }\n\n    def handle(self) -> int:\n        version = self.argument(\"version\")\n\n        if version:\n            version = self.increment_version(\n                self.poetry.package.pretty_version, version, self.option(\"next-phase\")\n            )\n\n            if self.option(\"short\"):\n                self.line(version.to_string())\n            else:\n                self.line(\n                    f\"Bumping version from <b>{self.poetry.package.pretty_version}</>\"\n                    f\" to <fg=green>{version}</>\"\n                )\n\n            if not self.option(\"dry-run\"):\n                content: dict[str, Any] = self.poetry.file.read()\n                poetry_content = content[\"tool\"][\"poetry\"]\n                poetry_content[\"version\"] = version.text\n\n                assert isinstance(content, TOMLDocument)\n                self.poetry.file.write(content)\n        else:\n            if self.option(\"short\"):\n                self.line(self.poetry.package.pretty_version)\n            else:\n                self.line(\n                    f\"<comment>{self.poetry.package.pretty_name}</>\"\n                    f\" <info>{self.poetry.package.pretty_version}</>\"\n                )\n\n        return 0\n\n    def increment_version(\n        self, version: str, rule: str, next_phase: bool = False\n    ) -> Version:\n        from poetry.core.constraints.version import Version\n\n        try:\n            parsed = Version.parse(version)\n        except InvalidVersion:\n            raise ValueError(\"The project's version doesn't seem to follow semver\")\n\n        if rule in {\"major\", \"premajor\"}:\n            new = parsed.next_major()\n            if rule == \"premajor\":\n                new = new.first_prerelease()\n        elif rule in {\"minor\", \"preminor\"}:\n            new = parsed.next_minor()\n            if rule == \"preminor\":\n                new = new.first_prerelease()\n        elif rule in {\"patch\", \"prepatch\"}:\n            new = parsed.next_patch()\n            if rule == \"prepatch\":\n                new = new.first_prerelease()\n        elif rule == \"prerelease\":\n            if parsed.is_unstable():\n                pre = parsed.pre\n                assert pre is not None\n                pre = pre.next_phase() if next_phase else pre.next()\n                new = Version(parsed.epoch, parsed.release, pre)\n            else:\n                new = parsed.next_patch().first_prerelease()\n        else:\n            new = Version.parse(rule)\n\n        return new\n\n# Path: src/poetry/console/events/__init__.py\n\n# Path: src/poetry/console/events/console_events.py\n\n# Path: src/poetry/console/io/__init__.py\n\n# Path: src/poetry/console/logging/__init__.py\n\n# Path: src/poetry/console/logging/filters.py\nfrom __future__ import annotations\n\nimport logging\n\n\nPOETRY_FILTER = logging.Filter(name=\"poetry\")\n\n# Path: src/poetry/console/logging/io_formatter.py\nfrom __future__ import annotations\n\nimport logging\nimport sys\nimport textwrap\n\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\n\nfrom poetry.console.logging.filters import POETRY_FILTER\nfrom poetry.console.logging.formatters import FORMATTERS\n\n\nif TYPE_CHECKING:\n    from logging import LogRecord\n\n\nclass IOFormatter(logging.Formatter):\n    _colors: ClassVar[dict[str, str]] = {\n        \"error\": \"fg=red\",\n        \"warning\": \"fg=yellow\",\n        \"debug\": \"debug\",\n        \"info\": \"fg=blue\",\n    }\n\n    def format(self, record: LogRecord) -> str:\n        if not record.exc_info:\n            level = record.levelname.lower()\n            msg = record.msg\n\n            if record.name in FORMATTERS:\n                msg = FORMATTERS[record.name].format(msg)\n            elif level in self._colors:\n                msg = f\"<{self._colors[level]}>{msg}</>\"\n\n            record.msg = msg\n\n        formatted = super().format(record)\n\n        if not POETRY_FILTER.filter(record):\n            # prefix all lines from third-party packages for easier debugging\n            formatted = textwrap.indent(\n                formatted, f\"[{_log_prefix(record)}] \", lambda line: True\n            )\n\n        return formatted\n\n\ndef _log_prefix(record: LogRecord) -> str:\n    prefix = _path_to_package(Path(record.pathname)) or record.module\n    if record.name != \"root\":\n        prefix = \":\".join([prefix, record.name])\n    return prefix\n\n\ndef _path_to_package(path: Path) -> str | None:\n    \"\"\"Return main package name from the LogRecord.pathname.\"\"\"\n    prefix: Path | None = None\n    # Find the most specific prefix in sys.path.\n    # We have to search the entire sys.path because a subsequent path might be\n    # a sub path of the first match and thereby a better match.\n    for syspath in sys.path:\n        if (\n            prefix and prefix in (p := Path(syspath)).parents and p in path.parents\n        ) or (not prefix and (p := Path(syspath)) in path.parents):\n            prefix = p\n    if not prefix:\n        # this is unexpected, but let's play it safe\n        return None\n    path = path.relative_to(prefix)\n    return path.parts[0]  # main package name\n\n# Path: src/poetry/console/logging/io_handler.py\nfrom __future__ import annotations\n\nimport logging\n\nfrom typing import TYPE_CHECKING\n\n\nif TYPE_CHECKING:\n    from logging import LogRecord\n\n    from cleo.io.io import IO\n\n\nclass IOHandler(logging.Handler):\n    def __init__(self, io: IO) -> None:\n        self._io = io\n\n        super().__init__()\n\n    def emit(self, record: LogRecord) -> None:\n        try:\n            msg = self.format(record)\n            level = record.levelname.lower()\n            err = level in (\"warning\", \"error\", \"exception\", \"critical\")\n            if err:\n                self._io.write_error_line(msg)\n            else:\n                self._io.write_line(msg)\n        except Exception:\n            self.handleError(record)\n\n# Path: src/poetry/installation/operations/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.installation.operations.install import Install\nfrom poetry.installation.operations.uninstall import Uninstall\nfrom poetry.installation.operations.update import Update\n\n\n__all__ = [\"Install\", \"Uninstall\", \"Update\"]\n\n# Path: src/poetry/installation/operations/install.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.installation.operations.operation import Operation\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.package import Package\n\n\nclass Install(Operation):\n    def __init__(\n        self, package: Package, reason: str | None = None, priority: int = 0\n    ) -> None:\n        super().__init__(reason, priority=priority)\n\n        self._package = package\n\n    @property\n    def package(self) -> Package:\n        return self._package\n\n    @property\n    def job_type(self) -> str:\n        return \"install\"\n\n    def __str__(self) -> str:\n        return (\n            \"Installing\"\n            f\" {self.package.pretty_name} ({self.format_version(self.package)})\"\n        )\n\n    def __repr__(self) -> str:\n        return (\n            \"<Install\"\n            f\" {self.package.pretty_name} ({self.format_version(self.package)})>\"\n        )\n\n# Path: src/poetry/installation/operations/operation.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom typing import TypeVar\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.package import Package\n\nT = TypeVar(\"T\", bound=\"Operation\")\n\n\nclass Operation:\n    def __init__(self, reason: str | None = None, priority: float = 0) -> None:\n        self._reason = reason\n\n        self._skipped = False\n        self._skip_reason: str | None = None\n        self._priority = priority\n\n    @property\n    def job_type(self) -> str:\n        raise NotImplementedError\n\n    @property\n    def reason(self) -> str | None:\n        return self._reason\n\n    @property\n    def skipped(self) -> bool:\n        return self._skipped\n\n    @property\n    def skip_reason(self) -> str | None:\n        return self._skip_reason\n\n    @property\n    def priority(self) -> float:\n        return self._priority\n\n    @property\n    def package(self) -> Package:\n        raise NotImplementedError()\n\n    def format_version(self, package: Package) -> str:\n        version: str = package.full_pretty_version\n        return version\n\n    def skip(self: T, reason: str) -> T:\n        self._skipped = True\n        self._skip_reason = reason\n\n        return self\n\n    def unskip(self: T) -> T:\n        self._skipped = False\n        self._skip_reason = None\n\n        return self\n\n# Path: src/poetry/installation/operations/uninstall.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.installation.operations.operation import Operation\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.package import Package\n\n\nclass Uninstall(Operation):\n    def __init__(\n        self,\n        package: Package,\n        reason: str | None = None,\n        priority: float = float(\"inf\"),\n    ) -> None:\n        super().__init__(reason, priority=priority)\n\n        self._package = package\n\n    @property\n    def package(self) -> Package:\n        return self._package\n\n    @property\n    def job_type(self) -> str:\n        return \"uninstall\"\n\n    def __str__(self) -> str:\n        return (\n            \"Uninstalling\"\n            f\" {self.package.pretty_name} ({self.format_version(self._package)})\"\n        )\n\n    def __repr__(self) -> str:\n        return (\n            \"<Uninstall\"\n            f\" {self.package.pretty_name} ({self.format_version(self.package)})>\"\n        )\n\n# Path: src/poetry/installation/operations/update.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\n\nfrom poetry.installation.operations.operation import Operation\n\n\nif TYPE_CHECKING:\n    from poetry.core.packages.package import Package\n\n\nclass Update(Operation):\n    def __init__(\n        self,\n        initial: Package,\n        target: Package,\n        reason: str | None = None,\n        priority: int = 0,\n    ) -> None:\n        self._initial_package = initial\n        self._target_package = target\n\n        super().__init__(reason, priority=priority)\n\n    @property\n    def initial_package(self) -> Package:\n        return self._initial_package\n\n    @property\n    def target_package(self) -> Package:\n        return self._target_package\n\n    @property\n    def package(self) -> Package:\n        return self._target_package\n\n    @property\n    def job_type(self) -> str:\n        return \"update\"\n\n    def __str__(self) -> str:\n        init_version = self.format_version(self.initial_package)\n        target_version = self.format_version(self.target_package)\n        return (\n            f\"Updating {self.initial_package.pretty_name} ({init_version}) \"\n            f\"to {self.target_package.pretty_name} ({target_version})\"\n        )\n\n    def __repr__(self) -> str:\n        init_version = self.format_version(self.initial_package)\n        target_version = self.format_version(self.target_package)\n        return (\n            f\"<Update {self.initial_package.pretty_name} ({init_version}) \"\n            f\"to {self.target_package.pretty_name} ({target_version})>\"\n        )\n\n# Path: src/poetry/masonry/builders/__init__.py\nfrom __future__ import annotations\n\nfrom poetry.core.masonry.builders.sdist import SdistBuilder\nfrom poetry.core.masonry.builders.wheel import WheelBuilder\n\nfrom poetry.masonry.builders.editable import EditableBuilder\n\n\n__all__ = [\"BUILD_FORMATS\", \"EditableBuilder\"]\n\n\n# might be extended by plugins\nBUILD_FORMATS = {\n    \"sdist\": SdistBuilder,\n    \"wheel\": WheelBuilder,\n}\n\n# Path: src/poetry/masonry/builders/editable.py\nfrom __future__ import annotations\n\nimport csv\nimport hashlib\nimport json\nimport locale\nimport os\n\nfrom base64 import urlsafe_b64encode\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\n\nfrom poetry.core.masonry.builders.builder import Builder\nfrom poetry.core.masonry.builders.sdist import SdistBuilder\nfrom poetry.core.masonry.utils.package_include import PackageInclude\n\nfrom poetry.utils._compat import WINDOWS\nfrom poetry.utils._compat import decode\nfrom poetry.utils.env import build_environment\nfrom poetry.utils.helpers import is_dir_writable\nfrom poetry.utils.pip import pip_install\n\n\nif TYPE_CHECKING:\n    from cleo.io.io import IO\n\n    from poetry.poetry import Poetry\n    from poetry.utils.env import Env\n\nSCRIPT_TEMPLATE = \"\"\"\\\n#!{python}\nimport sys\nfrom {module} import {callable_holder}\n\nif __name__ == '__main__':\n    sys.exit({callable_}())\n\"\"\"\n\nWINDOWS_CMD_TEMPLATE = \"\"\"\\\n@echo off\\r\\n\"{python}\" \"%~dp0\\\\{script}\" %*\\r\\n\n\"\"\"\n\n\nclass EditableBuilder(Builder):\n    def __init__(self, poetry: Poetry, env: Env, io: IO) -> None:\n        self._poetry: Poetry\n        super().__init__(poetry)\n\n        self._env = env\n        self._io = io\n\n    def build(self, target_dir: Path | None = None) -> Path:\n        self._debug(\n            f\"  - Building package <c1>{self._package.name}</c1> in\"\n            \" <info>editable</info> mode\"\n        )\n\n        if self._package.build_script:\n            if self._package.build_should_generate_setup():\n                self._debug(\n                    \"  - <warning>Falling back on using a <b>setup.py</b></warning>\"\n                )\n                self._setup_build()\n                return self._path\n\n            self._run_build_script(self._package.build_script)\n\n        for removed in self._env.site_packages.remove_distribution_files(\n            distribution_name=self._package.name\n        ):\n            self._debug(\n                f\"  - Removed <c2>{removed.name}</c2> directory from\"\n                f\" <b>{removed.parent}</b>\"\n            )\n\n        added_files = []\n        added_files += self._add_pth()\n        added_files += self._add_scripts()\n        self._add_dist_info(added_files)\n\n        return self._path\n\n    def _run_build_script(self, build_script: str) -> None:\n        with build_environment(poetry=self._poetry, env=self._env, io=self._io) as env:\n            self._debug(f\"  - Executing build script: <b>{build_script}</b>\")\n            env.run(\"python\", str(self._path.joinpath(build_script)), call=True)\n\n    def _setup_build(self) -> None:\n        builder = SdistBuilder(self._poetry)\n        setup = self._path / \"setup.py\"\n        has_setup = setup.exists()\n\n        if has_setup:\n            self._io.write_error_line(\n                \"<warning>A setup.py file already exists. Using it.</warning>\"\n            )\n        else:\n            with setup.open(\"w\", encoding=\"utf-8\") as f:\n                f.write(decode(builder.build_setup()))\n\n        try:\n            pip_install(self._path, self._env, upgrade=True, editable=True)\n        finally:\n            if not has_setup:\n                os.remove(setup)\n\n    def _add_pth(self) -> list[Path]:\n        paths = {\n            include.base.resolve().as_posix()\n            for include in self._module.includes\n            if isinstance(include, PackageInclude)\n            and (include.is_module() or include.is_package())\n        }\n\n        content = \"\".join(decode(path + os.linesep) for path in paths)\n        pth_file = Path(self._module.name).with_suffix(\".pth\")\n\n        # remove any pre-existing pth files for this package\n        for file in self._env.site_packages.find(path=pth_file, writable_only=True):\n            self._debug(\n                f\"  - Removing existing <c2>{file.name}</c2> from <b>{file.parent}</b>\"\n                f\" for {self._poetry.file.path.parent}\"\n            )\n            file.unlink(missing_ok=True)\n\n        try:\n            pth_file = self._env.site_packages.write_text(\n                pth_file, content, encoding=locale.getpreferredencoding()\n            )\n            self._debug(\n                f\"  - Adding <c2>{pth_file.name}</c2> to <b>{pth_file.parent}</b> for\"\n                f\" {self._poetry.file.path.parent}\"\n            )\n            return [pth_file]\n        except OSError:\n            # TODO: Replace with PermissionError\n            self._io.write_error_line(\n                f\"  - Failed to create <c2>{pth_file.name}</c2> for\"\n                f\" {self._poetry.file.path.parent}\"\n            )\n            return []\n\n    def _add_scripts(self) -> list[Path]:\n        added = []\n        entry_points = self.convert_entry_points()\n\n        for scripts_path in self._env.script_dirs:\n            if is_dir_writable(path=scripts_path, create=True):\n                break\n        else:\n            self._io.write_error_line(\n                \"  - Failed to find a suitable script installation directory for\"\n                f\" {self._poetry.file.path.parent}\"\n            )\n            return []\n\n        scripts = entry_points.get(\"console_scripts\", [])\n        for script in scripts:\n            name, script_with_extras = script.split(\" = \")\n            script_without_extras = script_with_extras.split(\"[\")[0]\n            try:\n                module, callable_ = script_without_extras.split(\":\")\n            except ValueError as exc:\n                msg = (\n                    f\"Bad script ({name}): script needs to specify a function within a\"\n                    \" module like: module(.submodule):function\\nInstead got:\"\n                    f\" {script_with_extras}\"\n                )\n                if \"not enough values\" in str(exc):\n                    msg += (\n                        \"\\nHint: If the script depends on module-level code, try\"\n                        \" wrapping it in a main() function and modifying your script\"\n                        f' like:\\n{name} = \"{script_with_extras}:main\"'\n                    )\n                elif \"too many values\" in str(exc):\n                    msg += '\\nToo many \":\" found!'\n\n                raise ValueError(msg)\n\n            callable_holder = callable_.split(\".\", 1)[0]\n\n            script_file = scripts_path.joinpath(name)\n            self._debug(\n                f\"  - Adding the <c2>{name}</c2> script to <b>{scripts_path}</b>\"\n            )\n            with script_file.open(\"w\", encoding=\"utf-8\") as f:\n                f.write(\n                    decode(\n                        SCRIPT_TEMPLATE.format(\n                            python=self._env.python,\n                            module=module,\n                            callable_holder=callable_holder,\n                            callable_=callable_,\n                        )\n                    )\n                )\n\n            script_file.chmod(0o755)\n\n            added.append(script_file)\n\n            if WINDOWS:\n                cmd_script = script_file.with_suffix(\".cmd\")\n                cmd = WINDOWS_CMD_TEMPLATE.format(python=self._env.python, script=name)\n                self._debug(\n                    f\"  - Adding the <c2>{cmd_script.name}</c2> script wrapper to\"\n                    f\" <b>{scripts_path}</b>\"\n                )\n\n                with cmd_script.open(\"w\", encoding=\"utf-8\") as f:\n                    f.write(decode(cmd))\n\n                added.append(cmd_script)\n\n        return added\n\n    def _add_dist_info(self, added_files: list[Path]) -> None:\n        from poetry.core.masonry.builders.wheel import WheelBuilder\n\n        added_files = added_files[:]\n\n        builder = WheelBuilder(self._poetry)\n        dist_info = self._env.site_packages.mkdir(Path(builder.dist_info))\n\n        self._debug(\n            f\"  - Adding the <c2>{dist_info.name}</c2> directory to\"\n            f\" <b>{dist_info.parent}</b>\"\n        )\n\n        with dist_info.joinpath(\"METADATA\").open(\"w\", encoding=\"utf-8\") as f:\n            builder._write_metadata_file(f)\n\n        added_files.append(dist_info.joinpath(\"METADATA\"))\n\n        with dist_info.joinpath(\"INSTALLER\").open(\"w\", encoding=\"utf-8\") as f:\n            f.write(\"poetry\")\n\n        added_files.append(dist_info.joinpath(\"INSTALLER\"))\n\n        if self.convert_entry_points():\n            with dist_info.joinpath(\"entry_points.txt\").open(\n                \"w\", encoding=\"utf-8\"\n            ) as f:\n                builder._write_entry_points(f)\n\n            added_files.append(dist_info.joinpath(\"entry_points.txt\"))\n\n        # write PEP 610 metadata\n        direct_url_json = dist_info.joinpath(\"direct_url.json\")\n        direct_url_json.write_text(\n            json.dumps(\n                {\n                    \"dir_info\": {\"editable\": True},\n                    \"url\": self._poetry.file.path.parent.absolute().as_uri(),\n                }\n            )\n        )\n        added_files.append(direct_url_json)\n\n        record = dist_info.joinpath(\"RECORD\")\n        with record.open(\"w\", encoding=\"utf-8\", newline=\"\") as f:\n            csv_writer = csv.writer(f)\n            for path in added_files:\n                hash = self._get_file_hash(path)\n                size = path.stat().st_size\n                csv_writer.writerow((path, f\"sha256={hash}\", size))\n\n            # RECORD itself is recorded with no hash or size\n            csv_writer.writerow((record, \"\", \"\"))\n\n    def _get_file_hash(self, filepath: Path) -> str:\n        hashsum = hashlib.sha256()\n        with filepath.open(\"rb\") as src:\n            while True:\n                buf = src.read(1024 * 8)\n                if not buf:\n                    break\n                hashsum.update(buf)\n\n            src.seek(0)\n\n        return urlsafe_b64encode(hashsum.digest()).decode(\"ascii\").rstrip(\"=\")\n\n    def _debug(self, msg: str) -> None:\n        if self._io.is_debug():\n            self._io.write_line(msg)\n\n# Path: src/poetry/repositories/link_sources/__init__.py\n\n# Path: src/poetry/repositories/link_sources/base.py\nfrom __future__ import annotations\n\nimport logging\nimport re\n\nfrom functools import cached_property\nfrom typing import TYPE_CHECKING\nfrom typing import ClassVar\nfrom typing import DefaultDict\nfrom typing import List\n\nfrom poetry.core.constraints.version import Version\nfrom poetry.core.packages.package import Package\nfrom poetry.core.version.exceptions import InvalidVersion\n\nfrom poetry.utils.patterns import sdist_file_re\nfrom poetry.utils.patterns import wheel_file_re\n\n\nif TYPE_CHECKING:\n    from collections.abc import Iterator\n\n    from packaging.utils import NormalizedName\n    from poetry.core.packages.utils.link import Link\n\n    LinkCache = DefaultDict[NormalizedName, DefaultDict[Version, List[Link]]]\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass LinkSource:\n    VERSION_REGEX = re.compile(r\"(?i)([a-z0-9_\\-.]+?)-(?=\\d)([a-z0-9_.!+-]+)\")\n    CLEAN_REGEX = re.compile(r\"[^a-z0-9$&+,/:;=?@.#%_\\\\|-]\", re.I)\n    SUPPORTED_FORMATS: ClassVar[list[str]] = [\n        \".tar.gz\",\n        \".whl\",\n        \".zip\",\n        \".tar.bz2\",\n        \".tar.xz\",\n        \".tar.Z\",\n        \".tar\",\n    ]\n\n    def __init__(self, url: str) -> None:\n        self._url = url\n\n    @property\n    def url(self) -> str:\n        return self._url\n\n    def versions(self, name: NormalizedName) -> Iterator[Version]:\n        yield from self._link_cache[name]\n\n    @property\n    def packages(self) -> Iterator[Package]:\n        for link in self.links:\n            pkg = self.link_package_data(link)\n\n            if pkg:\n                yield pkg\n\n    @property\n    def links(self) -> Iterator[Link]:\n        for links_per_version in self._link_cache.values():\n            for links in links_per_version.values():\n                yield from links\n\n    @classmethod\n    def link_package_data(cls, link: Link) -> Package | None:\n        name: str | None = None\n        version_string: str | None = None\n        version: Version | None = None\n        m = wheel_file_re.match(link.filename) or sdist_file_re.match(link.filename)\n\n        if m:\n            name = m.group(\"name\")\n            version_string = m.group(\"ver\")\n        else:\n            info, ext = link.splitext()\n            match = cls.VERSION_REGEX.match(info)\n            if match:\n                name = match.group(1)\n                version_string = match.group(2)\n\n        if version_string:\n            try:\n                version = Version.parse(version_string)\n            except InvalidVersion:\n                logger.debug(\n                    \"Skipping url (%s) due to invalid version (%s)\", link.url, version\n                )\n                return None\n\n        pkg = None\n        if name and version:\n            pkg = Package(name, version, source_url=link.url)\n        return pkg\n\n    def links_for_version(\n        self, name: NormalizedName, version: Version\n    ) -> Iterator[Link]:\n        yield from self._link_cache[name][version]\n\n    def clean_link(self, url: str) -> str:\n        \"\"\"Makes sure a link is fully encoded.  That is, if a ' ' shows up in\n        the link, it will be rewritten to %20 (while not over-quoting\n        % or other characters).\"\"\"\n        return self.CLEAN_REGEX.sub(lambda match: f\"%{ord(match.group(0)):02x}\", url)\n\n    def yanked(self, name: NormalizedName, version: Version) -> str | bool:\n        reasons = set()\n        for link in self.links_for_version(name, version):\n            if link.yanked:\n                if link.yanked_reason:\n                    reasons.add(link.yanked_reason)\n            else:\n                # release is not yanked if at least one file is not yanked\n                return False\n        # if all files are yanked (or there are no files) the release is yanked\n        if reasons:\n            return \"\\n\".join(sorted(reasons))\n        return True\n\n    @cached_property\n    def _link_cache(self) -> LinkCache:\n        raise NotImplementedError()\n\n# Path: src/poetry/repositories/link_sources/html.py\nfrom __future__ import annotations\n\nimport urllib.parse\n\nfrom collections import defaultdict\nfrom functools import cached_property\nfrom html import unescape\nfrom typing import TYPE_CHECKING\n\nfrom poetry.core.packages.utils.link import Link\n\nfrom poetry.repositories.link_sources.base import LinkSource\nfrom poetry.repositories.parsers.html_page_parser import HTMLPageParser\n\n\nif TYPE_CHECKING:\n    from poetry.repositories.link_sources.base import LinkCache\n\n\nclass HTMLPage(LinkSource):\n    def __init__(self, url: str, content: str) -> None:\n        super().__init__(url=url)\n\n        parser = HTMLPageParser()\n        parser.feed(content)\n        self._parsed = parser.anchors\n        self._base_url: str | None = parser.base_url\n\n    @cached_property\n    def _link_cache(self) -> LinkCache:\n        links: LinkCache = defaultdict(lambda: defaultdict(list))\n        for anchor in self._parsed:\n            if href := anchor.get(\"href\"):\n                url = self.clean_link(\n                    urllib.parse.urljoin(self._base_url or self._url, href)\n                )\n                pyrequire = anchor.get(\"data-requires-python\")\n                pyrequire = unescape(pyrequire) if pyrequire else None\n                yanked_value = anchor.get(\"data-yanked\")\n                yanked: str | bool\n                if yanked_value:\n                    yanked = unescape(yanked_value)\n                else:\n                    yanked = \"data-yanked\" in anchor\n\n                # see https://peps.python.org/pep-0714/#clients\n                # and https://peps.python.org/pep-0658/#specification\n                metadata: str | bool\n                for metadata_key in (\"data-core-metadata\", \"data-dist-info-metadata\"):\n                    metadata_value = anchor.get(metadata_key)\n                    if metadata_value:\n                        metadata = unescape(metadata_value)\n                    else:\n                        metadata = metadata_key in anchor\n                    if metadata:\n                        break\n                link = Link(\n                    url, requires_python=pyrequire, yanked=yanked, metadata=metadata\n                )\n\n                if link.ext not in self.SUPPORTED_FORMATS:\n                    continue\n\n                pkg = self.link_package_data(link)\n                if pkg:\n                    links[pkg.name][pkg.version].append(link)\n\n        return links\n\n\nclass SimpleRepositoryPage(HTMLPage):\n    def __init__(self, url: str, content: str) -> None:\n        if not url.endswith(\"/\"):\n            url += \"/\"\n        super().__init__(url=url, content=content)\n\n# Path: src/poetry/repositories/link_sources/json.py\nfrom __future__ import annotations\n\nfrom collections import defaultdict\nfrom functools import cached_property\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom poetry.core.packages.utils.link import Link\n\nfrom poetry.repositories.link_sources.base import LinkSource\n\n\nif TYPE_CHECKING:\n    from poetry.repositories.link_sources.base import LinkCache\n\n\nclass SimpleJsonPage(LinkSource):\n    \"\"\"Links as returned by PEP 691 compatible JSON-based Simple API.\"\"\"\n\n    def __init__(self, url: str, content: dict[str, Any]) -> None:\n        super().__init__(url=url)\n        self.content = content\n\n    @cached_property\n    def _link_cache(self) -> LinkCache:\n        links: LinkCache = defaultdict(lambda: defaultdict(list))\n        for file in self.content[\"files\"]:\n            url = file[\"url\"]\n            requires_python = file.get(\"requires-python\")\n            yanked = file.get(\"yanked\", False)\n\n            # see https://peps.python.org/pep-0714/#clients\n            # and https://peps.python.org/pep-0691/#project-detail\n            metadata: dict[str, str] | bool = False\n            for metadata_key in (\"core-metadata\", \"dist-info-metadata\"):\n                if metadata_key in file:\n                    metadata_value = file[metadata_key]\n                    if metadata_value and isinstance(metadata_value, dict):\n                        metadata = metadata_value\n                    else:\n                        metadata = bool(metadata_value)\n                    break\n\n            link = Link(\n                url, requires_python=requires_python, yanked=yanked, metadata=metadata\n            )\n\n            if link.ext not in self.SUPPORTED_FORMATS:\n                continue\n\n            pkg = self.link_package_data(link)\n            if pkg:\n                links[pkg.name][pkg.version].append(link)\n\n        return links\n\n# Path: src/poetry/repositories/parsers/__init__.py\n\n# Path: src/poetry/repositories/parsers/html_page_parser.py\nfrom __future__ import annotations\n\nfrom html.parser import HTMLParser\n\n\nclass HTMLPageParser(HTMLParser):\n    def __init__(self) -> None:\n        super().__init__()\n        self.base_url: str | None = None\n        self.anchors: list[dict[str, str | None]] = []\n\n    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:\n        if tag == \"base\" and self.base_url is None:\n            base_url = dict(attrs).get(\"href\")\n            if base_url is not None:\n                self.base_url = base_url\n        elif tag == \"a\":\n            self.anchors.append(dict(attrs))\n\n# Path: src/poetry/repositories/parsers/pypi_search_parser.py\nfrom __future__ import annotations\n\nimport functools\n\nfrom dataclasses import dataclass\nfrom html.parser import HTMLParser\nfrom typing import Callable\n\n\n# The following code was originally written for PDM project\n# https://github.com/pdm-project/pdm/blob/1f4f48a35cdded064def85df117bebf713f7c17a/src/pdm/models/search.py\n# and later changed to fit Poetry needs\n\n\n@dataclass\nclass Result:\n    name: str = \"\"\n    version: str = \"\"\n    description: str = \"\"\n\n\nclass SearchResultParser(HTMLParser):\n    \"\"\"A simple HTML parser for pypi.org search results.\"\"\"\n\n    def __init__(self) -> None:\n        super().__init__()\n        self.results: list[Result] = []\n        self._current: Result | None = None\n        self._nest_anchors = 0\n        self._data_callback: Callable[[str], None] | None = None\n\n    @staticmethod\n    def _match_class(attrs: list[tuple[str, str | None]], name: str) -> bool:\n        attrs_map = dict(attrs)\n        return name in (attrs_map.get(\"class\") or \"\").split()\n\n    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:\n        if not self._current:\n            if tag == \"a\" and self._match_class(attrs, \"package-snippet\"):\n                self._current = Result()\n                self._nest_anchors = 1\n        else:\n            if tag == \"span\" and self._match_class(attrs, \"package-snippet__name\"):\n                self._data_callback = functools.partial(setattr, self._current, \"name\")\n            elif tag == \"span\" and self._match_class(attrs, \"package-snippet__version\"):\n                self._data_callback = functools.partial(\n                    setattr, self._current, \"version\"\n                )\n            elif tag == \"p\" and self._match_class(\n                attrs, \"package-snippet__description\"\n            ):\n                self._data_callback = functools.partial(\n                    setattr, self._current, \"description\"\n                )\n            elif tag == \"a\":\n                self._nest_anchors += 1\n\n    def handle_data(self, data: str) -> None:\n        if self._data_callback is not None:\n            self._data_callback(data)\n            self._data_callback = None\n\n    def handle_endtag(self, tag: str) -> None:\n        if tag != \"a\" or self._current is None:\n            return\n        self._nest_anchors -= 1\n        if self._nest_anchors == 0:\n            if self._current.name and self._current.version:\n                self.results.append(self._current)\n            self._current = None\n\n# Path: src/poetry/utils/env/__init__.py\nfrom __future__ import annotations\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING\n\nfrom poetry.core.utils.helpers import temporary_directory\n\nfrom poetry.utils.env.base_env import Env\nfrom poetry.utils.env.env_manager import EnvManager\nfrom poetry.utils.env.exceptions import EnvCommandError\nfrom poetry.utils.env.exceptions import EnvError\nfrom poetry.utils.env.exceptions import IncorrectEnvError\nfrom poetry.utils.env.exceptions import InvalidCurrentPythonVersionError\nfrom poetry.utils.env.exceptions import NoCompatiblePythonVersionFound\nfrom poetry.utils.env.exceptions import PythonVersionNotFound\nfrom poetry.utils.env.generic_env import GenericEnv\nfrom poetry.utils.env.mock_env import MockEnv\nfrom poetry.utils.env.null_env import NullEnv\nfrom poetry.utils.env.script_strings import GET_BASE_PREFIX\nfrom poetry.utils.env.script_strings import GET_ENV_PATH_ONELINER\nfrom poetry.utils.env.script_strings import GET_ENVIRONMENT_INFO\nfrom poetry.utils.env.script_strings import GET_PATHS\nfrom poetry.utils.env.script_strings import GET_PATHS_FOR_GENERIC_ENVS\nfrom poetry.utils.env.script_strings import GET_PYTHON_VERSION\nfrom poetry.utils.env.script_strings import GET_PYTHON_VERSION_ONELINER\nfrom poetry.utils.env.script_strings import GET_SYS_PATH\nfrom poetry.utils.env.script_strings import GET_SYS_TAGS\nfrom poetry.utils.env.site_packages import SitePackages\nfrom poetry.utils.env.system_env import SystemEnv\nfrom poetry.utils.env.virtual_env import VirtualEnv\n\n\nif TYPE_CHECKING:\n    from collections.abc import Iterator\n\n    from cleo.io.io import IO\n    from poetry.core.poetry import Poetry as CorePoetry\n\n\n@contextmanager\ndef ephemeral_environment(\n    executable: Path | None = None,\n    flags: dict[str, str | bool] | None = None,\n) -> Iterator[VirtualEnv]:\n    with temporary_directory() as tmp_dir:\n        # TODO: cache PEP 517 build environment corresponding to each project venv\n        venv_dir = Path(tmp_dir) / \".venv\"\n        EnvManager.build_venv(\n            path=venv_dir,\n            executable=executable,\n            flags=flags,\n        )\n        yield VirtualEnv(venv_dir, venv_dir)\n\n\n@contextmanager\ndef build_environment(\n    poetry: CorePoetry, env: Env | None = None, io: IO | None = None\n) -> Iterator[Env]:\n    \"\"\"\n    If a build script is specified for the project, there could be additional build\n    time dependencies, eg: cython, setuptools etc. In these cases, we create an\n    ephemeral build environment with all requirements specified under\n    `build-system.requires` and return this. Otherwise, the given default project\n    environment is returned.\n    \"\"\"\n    if not env or poetry.package.build_script:\n        with ephemeral_environment(executable=env.python if env else None) as venv:\n            if io:\n                requires = [\n                    f\"<c1>{requirement}</c1>\"\n                    for requirement in poetry.pyproject.build_system.requires\n                ]\n\n                io.write_error_line(\n                    \"<b>Preparing</b> build environment with build-system requirements\"\n                    f\" {', '.join(requires)}\"\n                )\n\n            output = venv.run_pip(\n                \"install\",\n                \"--disable-pip-version-check\",\n                \"--ignore-installed\",\n                \"--no-input\",\n                *poetry.pyproject.build_system.requires,\n            )\n\n            if io and io.is_debug() and output:\n                io.write_error(output)\n\n            yield venv\n    else:\n        yield env\n\n\n__all__ = [\n    \"GET_BASE_PREFIX\",\n    \"GET_ENVIRONMENT_INFO\",\n    \"GET_PATHS\",\n    \"GET_PYTHON_VERSION\",\n    \"GET_SYS_PATH\",\n    \"GET_SYS_TAGS\",\n    \"GET_ENV_PATH_ONELINER\",\n    \"GET_PYTHON_VERSION_ONELINER\",\n    \"GET_PATHS_FOR_GENERIC_ENVS\",\n    \"EnvError\",\n    \"EnvCommandError\",\n    \"IncorrectEnvError\",\n    \"InvalidCurrentPythonVersionError\",\n    \"NoCompatiblePythonVersionFound\",\n    \"PythonVersionNotFound\",\n    \"Env\",\n    \"EnvManager\",\n    \"GenericEnv\",\n    \"MockEnv\",\n    \"NullEnv\",\n    \"SystemEnv\",\n    \"VirtualEnv\",\n    \"SitePackages\",\n    \"build_environment\",\n    \"ephemeral_environment\",\n]\n\n# Path: src/poetry/utils/env/base_env.py\nfrom __future__ import annotations\n\nimport contextlib\nimport os\nimport re\nimport subprocess\nimport sys\nimport sysconfig\n\nfrom pathlib import Path\nfrom subprocess import CalledProcessError\nfrom typing import TYPE_CHECKING\nfrom typing import Any\n\nfrom virtualenv.seed.wheels.embed import get_embed_wheel\n\nfrom poetry.utils.env.exceptions import EnvCommandError\nfrom poetry.utils.env.site_packages import SitePackages\nfrom poetry.utils.helpers import get_real_windows_path\n\n\nif TYPE_CHECKING:\n    from packaging.tags import Tag\n    from poetry.core.version.markers import BaseMarker\n    from virtualenv.seed.wheels.util import Wheel\n\n    from poetry.utils.env.generic_env import GenericEnv\n\n\nclass Env:\n    \"\"\"\n    An abstract Python environment.\n    \"\"\"\n\n    def __init__(self, path: Path, base: Path | None = None) -> None:\n        self._is_windows = sys.platform == \"win32\"\n        self._is_mingw = sysconfig.get_platform().startswith(\"mingw\")\n        self._is_conda = bool(os.environ.get(\"CONDA_DEFAULT_ENV\"))\n\n        if self._is_windows:\n            path = get_real_windows_path(path)\n            base = get_real_windows_path(base) if base else None\n\n        bin_dir = \"bin\" if not self._is_windows or self._is_mingw else \"Scripts\"\n        self._path = path\n        self._bin_dir = self._path / bin_dir\n\n        self._executable = \"python\"\n        self._pip_executable = \"pip\"\n\n        self.find_executables()\n\n        self._base = base or path\n\n        self._marker_env: dict[str, Any] | None = None\n        self._site_packages: SitePackages | None = None\n        self._paths: dict[str, str] | None = None\n        self._supported_tags: list[Tag] | None = None\n        self._purelib: Path | None = None\n        self._platlib: Path | None = None\n        self._script_dirs: list[Path] | None = None\n\n        self._embedded_pip_path: Path | None = None\n\n    @property\n    def path(self) -> Path:\n        return self._path\n\n    @property\n    \ndef base(self) -> Path:\n        return self._base\n\n    @property\n    def version_info(self) -> tuple[int, int, int, str, int]:\n        version_info: tuple[int, int, int, str, int] = self.marker_env[\"version_info\"]\n        return version_info\n\n    @property\n    def python_implementation(self) -> str:\n        implementation: str = self.marker_env[\"platform_python_implementation\"]\n        return implementation\n\n    @property\n    def python(self) -> Path:\n        \"\"\"\n        Path to current python executable\n        \"\"\"\n        return Path(self._bin(self._executable))\n\n    @property\n    def marker_env(self) -> dict[str, Any]:\n        if self._marker_env is None:\n            self._marker_env = self.get_marker_env()\n\n        return self._marker_env\n\n    @property\n    def parent_env(self) -> GenericEnv:\n        from poetry.utils.env.generic_env import GenericEnv\n\n        return GenericEnv(self.base, child_env=self)\n\n    def _find_python_executable(self) -> None:\n        bin_dir = self._bin_dir\n\n        if self._is_windows and self._is_conda:\n            bin_dir = self._path\n\n        python_executables = sorted(\n            p.name\n            for p in bin_dir.glob(\"python*\")\n            if re.match(r\"python(?:\\d+(?:\\.\\d+)?)?(?:\\.exe)?$\", p.name)\n        )\n        if python_executables:\n            executable = python_executables[0]\n            if executable.endswith(\".exe\"):\n                executable = executable[:-4]\n\n            self._executable = executable\n\n    def _find_pip_executable(self) -> None:\n        pip_executables = sorted(\n            p.name\n            for p in self._bin_dir.glob(\"pip*\")\n            if re.match(r\"pip(?:\\d+(?:\\.\\d+)?)?(?:\\.exe)?$\", p.name)\n        )\n        if pip_executables:\n            pip_executable = pip_executables[0]\n            if pip_executable.endswith(\".exe\"):\n                pip_executable = pip_executable[:-4]\n\n            self._pip_executable = pip_executable\n\n    def find_executables(self) -> None:\n        self._find_python_executable()\n        self._find_pip_executable()\n\n    def get_embedded_wheel(self, distribution: str) -> Path:\n        wheel: Wheel = get_embed_wheel(\n            distribution, f\"{self.version_info[0]}.{self.version_info[1]}\"\n        )\n        path: Path = wheel.path\n        return path\n\n    @property\n    def pip_embedded(self) -> Path:\n        if self._embedded_pip_path is None:\n            self._embedded_pip_path = self.get_embedded_wheel(\"pip\") / \"pip\"\n        return self._embedded_pip_path\n\n    @property\n    def pip(self) -> Path:\n        \"\"\"\n        Path to current pip executable\n        \"\"\"\n...\n", "needle_token_start": 15193, "needle_token_end": 15207, "code_context_ntokens": 15994, "output": ["```python\n@property\ndef base(self) -> Path:\n    return self._base\n```"]}
{"repo": "locustio/locust", "name": "test_user_classes_with_same_name_is_error", "language": "python", "path": "locust/test/test_env.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To verify that an error is raised when two user classes with identical names are used in the same environment, ensuring unique identification of user classes.\n2. **Input**: Two user classes imported from different modules but with the same class name.\n3. **Output**: A `ValueError` exception indicating that user classes with duplicate names cannot be used together.\n4. **Procedure**: The test attempts to create an environment with the two conflicting user classes. It checks that the correct exception is raised and that the error message specifies the exact issue with the class names and their originating modules.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: locust/test/test_dispatch.py\nfrom __future__ import annotations\n\nfrom locust import User\nfrom locust.dispatch import UsersDispatcher\nfrom locust.runners import WorkerNode\nfrom locust.test.util import clear_all_functools_lru_cache\n\nimport time\nimport unittest\nfrom operator import attrgetter\n\n_TOLERANCE = 0.025\n\n\nclass TestRampUpUsersFromZero(unittest.TestCase):\n    def test_ramp_up_users_to_3_workers_with_spawn_rate_of_0_5(self):\n        \"\"\"Final distribution should be {\"User1\": 3, \"User2\": 3, \"User3\": 3}\"\"\"\n\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        worker_node1 = WorkerNode(\"1\")\n        worker_node2 = WorkerNode(\"2\")\n        worker_node3 = WorkerNode(\"3\")\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher = UsersDispatcher(\n            worker_nodes=[worker_node1, worker_node2, worker_node3], user_classes=[User1, User2, User3]\n        )\n        users_dispatcher.new_dispatch(target_user_count=9, spawn_rate=0.5)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n...\n# Path: locust/test/test_env.py\nfrom locust import (\n    constant,\n)\nfrom locust.dispatch import UsersDispatcher\nfrom locust.env import Environment, LoadTestShape\nfrom locust.user import (\n    User,\n    task,\n)\nfrom locust.user.task import TaskSet\n\nfrom .fake_module1_for_env_test import MyUserWithSameName as MyUserWithSameName1\nfrom .fake_module2_for_env_test import MyUserWithSameName as MyUserWithSameName2\nfrom .testcases import LocustTestCase\n\n\nclass TestEnvironment(LocustTestCase):\n    def test_user_classes_count(self):\n        class MyUser1(User):\n            wait_time = constant(0)\n\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            wait_time = constant(0)\n\n            @task\n            def my_task(self):\n                pass\n\n        environment = Environment(user_classes=[MyUser1, MyUser2])\n\n        self.assertDictEqual({\"MyUser1\": MyUser1, \"MyUser2\": MyUser2}, environment.user_classes_by_name)\n\n    \ndef test_user_classes_with_same_name_is_error(self):\n        with self.assertRaises(ValueError) as e:\n            environment = Environment(user_classes=[MyUserWithSameName1, MyUserWithSameName2])\n\n        self.assertEqual(\n            e.exception.args[0],\n            \"The following user classes have the same class name: locust.test.fake_module1_for_env_test.MyUserWithSameName, locust.test.fake_module2_for_env_test.MyUserWithSameName\",\n        )\n\n    def test_assign_equal_weights(self):\n        def verify_tasks(u, target_tasks):\n            self.assertEqual(len(u.tasks), len(target_tasks))\n            tasks = [t.__name__ for t in u.tasks]\n            self.assertEqual(len(tasks), len(set(tasks)))\n            self.assertEqual(set(tasks), set(target_tasks))\n\n        # Base case\n        class MyUser1(User):\n            wait_time = constant(0)\n\n            @task(4)\n            def my_task(self):\n                pass\n\n            @task(1)\n            def my_task_2(self):\n                pass\n\n        environment = Environment(user_classes=[MyUser1])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"my_task\", \"my_task_2\"])\n\n        # Testing nested task sets\n        class MyUser2(User):\n            @task\n            class TopLevelTaskSet(TaskSet):\n                @task\n                class IndexTaskSet(TaskSet):\n                    @task(10)\n                    def index(self):\n                        self.client.get(\"/\")\n\n                    @task\n                    def stop(self):\n                        self.client.get(\"/hi\")\n\n                @task(2)\n                def stats(self):\n                    self.client.get(\"/stats/requests\")\n\n        environment = Environment(user_classes=[MyUser2])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"index\", \"stop\", \"stats\"])\n\n        # Testing task assignment via instance variable\n        def outside_task():\n            pass\n\n        def outside_task_2():\n            pass\n\n        class SingleTaskSet(TaskSet):\n            tasks = [outside_task, outside_task, outside_task_2]\n\n        class MyUser3(User):\n            tasks = [SingleTaskSet, outside_task]\n\n        environment = Environment(user_classes=[MyUser3])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"outside_task\", \"outside_task_2\"])\n\n        # Testing task assignment via dict\n        class DictTaskSet(TaskSet):\n            def dict_task_1():\n                pass\n\n            def dict_task_2():\n                pass\n\n            def dict_task_3():\n                pass\n\n            tasks = {\n                dict_task_1: 5,\n                dict_task_2: 3,\n                dict_task_3: 1,\n            }\n\n        class MyUser4(User):\n            tasks = [DictTaskSet, SingleTaskSet, SingleTaskSet]\n\n        # Assign user tasks in dict\n        environment = Environment(user_classes=[MyUser4])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"outside_task\", \"outside_task_2\", \"dict_task_1\", \"dict_task_2\", \"dict_task_3\"])\n\n        class MyUser5(User):\n            tasks = {\n                DictTaskSet: 5,\n                SingleTaskSet: 3,\n                outside_task: 6,\n            }\n\n        environment = Environment(user_classes=[MyUser5])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"outside_task\", \"outside_task_2\", \"dict_task_1\", \"dict_task_2\", \"dict_task_3\"])\n\n    def test_user_classes_with_zero_weight_are_removed(self):\n        class MyUser1(User):\n            wait_time = constant(0)\n            weight = 0\n\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            wait_time = constant(0)\n            weight = 1\n\n            @task\n            def my_task(self):\n                pass\n\n        environment = Environment(user_classes=[MyUser1, MyUser2])\n\n        self.assertEqual(len(environment.user_classes), 1)\n        self.assertIs(environment.user_classes[0], MyUser2)\n\n    def test_all_user_classes_with_zero_weight_raises_exception(self):\n        class MyUser1(User):\n            wait_time = constant(0)\n            weight = 0\n\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            wait_time = constant(0)\n            weight = 0\n\n            @task\n            def my_task(self):\n                pass\n\n        with self.assertRaises(ValueError) as e:\n            environment = Environment(user_classes=[MyUser1, MyUser2])\n\n        self.assertEqual(\n            e.exception.args[0],\n            \"There are no users with weight > 0.\",\n        )\n\n    def test_shape_class_attribute(self):\n        class SubLoadTestShape(LoadTestShape):\n            \"\"\"Inherited from locust.env.LoadTestShape\"\"\"\n\n        with self.assertRaisesRegex(\n            ValueError, r\"instance of LoadTestShape or subclass LoadTestShape\", msg=\"exception message is mismatching\"\n        ):\n            Environment(user_classes=[MyUserWithSameName1], shape_class=SubLoadTestShape)\n\n    def test_dispatcher_class_attribute(self):\n        environment = Environment(user_classes=[MyUserWithSameName1])\n\n        self.assertEqual(environment.dispatcher_class, UsersDispatcher)\n\n        class MyUsersDispatcher(UsersDispatcher):\n            pass\n\n        environment = Environment(user_classes=[MyUserWithSameName1], dispatcher_class=MyUsersDispatcher)\n\n        self.assertEqual(environment.dispatcher_class, MyUsersDispatcher)\n\n    def test_update_user_class(self):\n        class MyUser1(User):\n            @task\n            def my_task(self):\n                pass\n\n            @task\n            def my_task_2(self):\n                pass\n\n        class MyUser2(User):\n            @task\n            def my_task(self):\n                pass\n\n        environment = Environment(\n            user_classes=[MyUser1, MyUser2],\n            available_user_classes={\"User1\": MyUser1, \"User2\": MyUser2},\n            available_user_tasks={\"User1\": MyUser1.tasks, \"User2\": MyUser2.tasks},\n        )\n\n        environment.update_user_class({\"user_class_name\": \"User1\", \"host\": \"http://localhost\", \"tasks\": [\"my_task_2\"]})\n\n        self.assertEqual(\n            environment.available_user_classes[\"User1\"].json(),\n            {\"host\": \"http://localhost\", \"tasks\": [\"my_task_2\"], \"fixed_count\": 0, \"weight\": 1},\n        )\n\n# Path: locust/test/test_fasthttp.py\nfrom locust import FastHttpUser\nfrom locust.argument_parser import parse_options\nfrom locust.contrib.fasthttp import FastHttpSession\nfrom locust.exception import CatchResponseError, InterruptTaskSet, LocustError, ResponseError\nfrom locust.user import TaskSet, task\nfrom locust.util.load_locustfile import is_user_class\n\nimport socket\nimport time\nfrom tempfile import NamedTemporaryFile\n\nimport gevent\nfrom geventhttpclient.client import HTTPClientPool\n\nfrom .testcases import LocustTestCase, WebserverTestCase\nfrom .util import create_tls_cert\n\n\nclass TestFastHttpSession(WebserverTestCase):\n    def get_client(self):\n        return FastHttpSession(self.environment, base_url=\"http://127.0.0.1:%i\" % self.port, user=None)\n\n    def test_get(self):\n        s = self.get_client()\n        r = s.get(\"/ultra_fast\")\n        self.assertEqual(200, r.status_code)\n\n    def test_connection_error(self):\n        s = FastHttpSession(self.environment, \"http://localhost:1\", user=None)\n        r = s.get(\"/\", headers={\"X-Test-Headers\": \"hello\"})\n        self.assertEqual(r.status_code, 0)\n        self.assertEqual(None, r.content)\n        self.assertEqual(1, len(self.runner.stats.errors))\n        self.assertTrue(isinstance(r.error, ConnectionRefusedError))\n        self.assertTrue(isinstance(next(iter(self.runner.stats.errors.values())).error, ConnectionRefusedError))\n        self.assertEqual(r.url, \"http://localhost:1/\")\n        self.assertEqual(r.request.url, r.url)\n        self.assertEqual(r.request.headers.get(\"X-Test-Headers\", \"\"), \"hello\")\n\n    def test_404(self):\n        s = self.get_client()\n        r = s.get(\"/does_not_exist\")\n        self.assertEqual(404, r.status_code)\n        self.assertEqual(1, self.runner.stats.get(\"/does_not_exist\", \"GET\").num_failures)\n\n    def test_204(self):\n        s = self.get_client()\n        r = s.get(\"/status/204\")\n        self.assertEqual(204, r.status_code)\n        self.assertEqual(1, self.runner.stats.get(\"/status/204\", \"GET\").num_requests)\n        self.assertEqual(0, self.runner.stats.get(\"/status/204\", \"GET\").num_failures)\n        self.assertEqual(r.url, \"http://127.0.0.1:%i/status/204\" % self.port)\n        self.assertEqual(r.request.url, r.url)\n\n    def test_streaming_response(self):\n        \"\"\"\n        Test a request to an endpoint that returns a streaming response\n        \"\"\"\n        s = self.get_client()\n        r = s.get(\"/streaming/30\")\n\n        # verify that the time reported includes the download time of the whole streamed response\n        self.assertGreater(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n        self.runner.stats.clear_all()\n\n        # verify that response time does NOT include whole download time, when using stream=True\n        r = s.get(\"/streaming/30\", stream=True)\n        self.assertGreaterEqual(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 0)\n        self.assertLess(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n\n        # download the content of the streaming response (so we don't get an ugly exception in the log)\n        _ = r.content\n\n    def test_slow_redirect(self):\n        s = self.get_client()\n        url = \"/redirect?url=/redirect&delay=0.5\"\n        r = s.get(url)\n        stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, stats.num_requests)\n        self.assertGreater(stats.avg_response_time, 500)\n\n    def test_post_redirect(self):\n        s = self.get_client()\n        url = \"/redirect\"\n        r = s.post(url)\n        self.assertEqual(200, r.status_code)\n        post_stats = self.runner.stats.get(url, method=\"POST\")\n        get_stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, post_stats.num_requests)\n        self.assertEqual(0, get_stats.num_requests)\n\n    def test_cookie(self):\n        s = self.get_client()\n        r = s.post(\"/set_cookie?name=testcookie&value=1337\")\n        self.assertEqual(200, r.status_code)\n        r = s.get(\"/get_cookie?name=testcookie\")\n        self.assertEqual(\"1337\", r.content.decode())\n        self.assertEqual(\"1337\", r.text)\n\n    def test_head(self):\n        s = self.get_client()\n        r = s.head(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n\n    def test_delete(self):\n        s = self.get_client()\n        r = s.delete(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"DELETE\", r.content.decode())\n\n    def test_patch(self):\n        s = self.get_client()\n        r = s.patch(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"PATCH\", r.content.decode())\n\n    def test_options(self):\n        s = self.get_client()\n        r = s.options(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n        self.assertEqual(\n            {\"OPTIONS\", \"DELETE\", \"PUT\", \"GET\", \"POST\", \"HEAD\", \"PATCH\"},\n            set(r.headers[\"allow\"].split(\", \")),\n        )\n\n    def test_json_payload(self):\n        s = self.get_client()\n        r = s.post(\"/request_method\", json={\"foo\": \"bar\"})\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(r.request.body, '{\"foo\": \"bar\"}')\n        self.assertEqual(r.request.headers.get(\"Content-Type\"), \"application/json\")\n\n    def test_catch_response_fail_successful_request(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure(\"nope\")\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n\n    def test_catch_response_pass_failed_request(self):\n        s = self.get_client()\n        with s.get(\"/fail\", catch_response=True) as r:\n            r.success()\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_multiple_failure_and_success(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure(\"nope\")\n            r.success()\n            r.failure(\"nooo\")\n            r.success()\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_pass_failed_request_with_other_exception_within_block(self):\n        class OtherException(Exception):\n            pass\n\n        s = self.get_client()\n        try:\n            with s.get(\"/fail\", catch_response=True) as r:\n                r.success()\n                raise OtherException(\"wtf\")\n        except OtherException as e:\n            pass\n        else:\n            self.fail(\"OtherException should have been raised\")\n\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_default_success(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            pass\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(0, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n\n    def test_catch_response_default_fail(self):\n        s = self.get_client()\n        with s.get(\"/fail\", catch_response=True) as r:\n            pass\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(1, self.environment.stats.total.num_failures)\n\n    def test_error_message_with_name_replacement(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            self.assertIsNotNone(kw[\"exception\"])\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        before_request = time.time()\n        s.request(\"get\", \"/wrong_url/01\", name=\"replaced_url_name\", context={\"foo\": \"bar\"})\n        after_request = time.time()\n        # self.assertIn(\"for url: replaced_url_name\", str(kwargs[\"exception\"])) # this is actually broken for FastHttpUser right now...\n        self.assertAlmostEqual(before_request, kwargs[\"start_time\"], delta=0.01)\n        self.assertAlmostEqual(after_request, kwargs[\"start_time\"] + kwargs[\"response_time\"] / 1000, delta=0.01)\n        self.assertEqual(s.base_url + \"/wrong_url/01\", kwargs[\"url\"])  # url is unaffected by name\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    def test_custom_ssl_context_fail_with_bad_context(self):\n        \"\"\"\n        Test FastHttpSession with a custom SSLContext factory that will fail as\n        we can not set verify_mode to CERT_NONE when check_hostname is enabled\n        \"\"\"\n\n        def create_custom_context():\n            context = gevent.ssl.create_default_context()\n            context.check_hostname = True\n            context.verify_mode = gevent.ssl.CERT_NONE\n            return context\n\n        s = FastHttpSession(\n            self.environment,\n            \"https://127.0.0.1:%i\" % self.port,\n            ssl_context_factory=create_custom_context,\n            user=None,\n        )\n        with self.assertRaises(ValueError) as e:\n            s.get(\"/\")\n        self.assertEqual(e.exception.args, (\"Cannot set verify_mode to CERT_NONE when check_hostname is enabled.\",))\n\n    def test_custom_ssl_context_passed_correct_to_client_pool(self):\n        \"\"\"\n        Test FastHttpSession with a custom SSLContext factory with a options.name\n        that will be passed correctly to the ClientPool. It will also test a 2nd\n        factory which is not the correct one.\n        \"\"\"\n\n        def custom_ssl_context():\n            context = gevent.ssl.create_default_context()\n            context.check_hostname = False\n            context.verify_mode = gevent.ssl.CERT_NONE\n            context.options.name = \"FAKEOPTION\"\n            return context\n\n        def custom_context_with_wrong_option():\n            context = gevent.ssl.create_default_context()\n            context.check_hostname = False\n            context.verify_mode = gevent.ssl.CERT_NONE\n            context.options.name = \"OPTIONFAKED\"\n            return context\n\n        s = FastHttpSession(\n            self.environment,\n            \"https://127.0.0.1:%i\" % self.port,\n            ssl_context_factory=custom_ssl_context,\n            user=None,\n        )\n        self.assertEqual(s.client.clientpool.client_args[\"ssl_context_factory\"], custom_ssl_context)\n        self.assertNotEqual(s.client.clientpool.client_args[\"ssl_context_factory\"], custom_context_with_wrong_option)\n\n\nclass TestRequestStatsWithWebserver(WebserverTestCase):\n    def test_request_stats_content_length(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.get(\"/ultra_fast\")\n        self.assertEqual(\n            self.runner.stats.get(\"/ultra_fast\", \"GET\").avg_content_length, len(\"This is an ultra fast response\")\n        )\n        locust.client.get(\"/ultra_fast\")\n        self.assertEqual(\n            self.runner.stats.get(\"/ultra_fast\", \"GET\").avg_content_length, len(\"This is an ultra fast response\")\n        )\n\n    def test_request_stats_no_content_length(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        path = \"/no_content_length\"\n        r = l.client.get(path)\n        self.assertEqual(\n            self.runner.stats.get(path, \"GET\").avg_content_length,\n            len(\"This response does not have content-length in the header\"),\n        )\n\n    def test_request_stats_no_content_length_streaming(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        path = \"/no_content_length\"\n        r = l.client.get(path, stream=True)\n        self.assertEqual(0, self.runner.stats.get(path, \"GET\").avg_content_length)\n\n    def test_request_stats_named_endpoint(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.get(\"/ultra_fast\", name=\"my_custom_name\")\n        self.assertEqual(1, self.runner.stats.get(\"my_custom_name\", \"GET\").num_requests)\n\n    def test_request_stats_query_variables(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.get(\"/ultra_fast?query=1\")\n        self.assertEqual(1, self.runner.stats.get(\"/ultra_fast?query=1\", \"GET\").num_requests)\n\n    def test_request_stats_put(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.put(\"/put\")\n        self.assertEqual(1, self.runner.stats.get(\"/put\", \"PUT\").num_requests)\n\n    def test_request_connection_error(self):\n        class MyUser(FastHttpUser):\n            host = \"http://localhost:1\"\n\n        locust = MyUser(self.environment)\n        response = locust.client.get(\"/\")\n        self.assertEqual(response.status_code, 0)\n        self.assertEqual(1, self.runner.stats.get(\"/\", \"GET\").num_failures)\n        self.assertEqual(1, self.runner.stats.get(\"/\", \"GET\").num_requests)\n\n\nclass TestFastHttpUserClass(WebserverTestCase):\n    def test_is_abstract(self):\n        self.assertTrue(FastHttpUser.abstract)\n        self.assertFalse(is_user_class(FastHttpUser))\n\n    def test_class_context(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n            def context(self):\n                return {\"user\": self.username}\n\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        user = MyUser(self.environment)\n        user.username = \"foo\"\n        user.client.request(\"get\", \"/request_method\")\n        self.assertDictEqual({\"user\": \"foo\"}, kwargs[\"context\"])\n        self.assertEqual(\"GET\", kwargs[\"response\"].text)\n        user.client.request(\"get\", \"/request_method\", context={\"user\": \"bar\"})\n        self.assertDictEqual({\"user\": \"bar\"}, kwargs[\"context\"])\n\n    def test_get_request(self):\n        self.response = \"\"\n\n        def t1(l):\n            self.response = l.client.get(\"/ultra_fast\")\n\n        class MyUser(FastHttpUser):\n            tasks = [t1]\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        my_locust = MyUser(self.environment)\n        t1(my_locust)\n        self.assertEqual(self.response.text, \"This is an ultra fast response\")\n\n    def test_client_request_headers(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        r = locust.client.get(\"/request_header_test\", headers={\"X-Header-Test\": \"hello\"})\n        self.assertEqual(\"hello\", r.text)\n        self.assertEqual(\"hello\", r.headers.get(\"X-Header-Test\"))\n        self.assertEqual(\"hello\", r.request.headers.get(\"X-Header-Test\"))\n\n    def test_client_get(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"GET\", locust.client.get(\"/request_method\").text)\n\n    def test_client_get_absolute_url(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"GET\", locust.client.get(\"http://127.0.0.1:%i/request_method\" % self.port).text)\n\n    def test_client_post(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"POST\", locust.client.post(\"/request_method\", {\"arg\": \"hello world\"}).text)\n        self.assertEqual(\"hello world\", locust.client.post(\"/post\", {\"arg\": \"hello world\"}).text)\n\n    def test_client_put(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"PUT\", locust.client.put(\"/request_method\", {\"arg\": \"hello world\"}).text)\n        self.assertEqual(\"hello world\", locust.client.put(\"/put\", {\"arg\": \"hello world\"}).text)\n\n    def test_client_delete(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"DELETE\", locust.client.delete(\"/request_method\").text)\n        self.assertEqual(200, locust.client.delete(\"/request_method\").status_code)\n\n    def test_client_head(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(200, locust.client.head(\"/request_method\").status_code)\n\n    def test_complex_content_type(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n\n        self.assertEqual(\"stuff\", locust.client.get(\"/content_type_missing_charset\").text)\n        self.assertEqual(\"stuff\", locust.client.get(\"/content_type_regular\").text)\n        self.assertEqual(\"stuff\", locust.client.get(\"/content_type_with_extra_stuff\").text)\n\n    def test_log_request_name_argument(self):\n        self.response = \"\"\n\n        class MyUser(FastHttpUser):\n            tasks = []\n            host = \"http://127.0.0.1:%i\" % self.port\n\n            @task()\n            def t1(l):\n                self.response = l.client.get(\"/ultra_fast\", name=\"new name!\")\n\n        my_locust = MyUser(self.environment)\n        my_locust.t1()\n\n        self.assertEqual(1, self.runner.stats.get(\"new name!\", \"GET\").num_requests)\n        self.assertEqual(0, self.runner.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n\n    def test_redirect_url_original_path_as_name(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        l.client.get(\"/redirect\")\n\n        self.assertEqual(1, len(self.runner.stats.entries))\n        self.assertEqual(1, self.runner.stats.get(\"/redirect\", \"GET\").num_requests)\n        self.assertEqual(0, self.runner.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n\n    def test_network_timeout_setting(self):\n        class MyUser(FastHttpUser):\n            network_timeout = 0.5\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n\n        timeout = gevent.Timeout(\n            seconds=0.6,\n            exception=AssertionError(\n                \"Request took longer than 0.6 even though FastHttpUser.network_timeout was set to 0.5\"\n            ),\n        )\n        timeout.start()\n        r = l.client.get(\"/redirect?url=/redirect&delay=5.0\")\n        timeout.cancel()\n\n        self.assertTrue(isinstance(r.error.original, socket.timeout))\n        self.assertEqual(1, self.runner.stats.get(\"/redirect?url=/redirect&delay=5.0\", \"GET\").num_failures)\n\n    def test_max_redirect_setting(self):\n        class MyUser(FastHttpUser):\n            max_redirects = 1  # max_redirects and max_retries are funny names, because they are actually max attempts\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        l.client.get(\"/redirect\")\n        self.assertEqual(1, self.runner.stats.get(\"/redirect\", \"GET\").num_failures)\n\n    def test_allow_redirects_override(self):\n        class MyLocust(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyLocust(self.environment)\n        resp = l.client.get(\"/redirect\", allow_redirects=False)\n        self.assertTrue(resp.headers[\"location\"].endswith(\"/ultra_fast\"))\n        resp = l.client.get(\"/redirect\")  # ensure redirect still works\n        self.assertFalse(\"location\" in resp.headers)\n\n    def test_slow_redirect(self):\n        s = FastHttpSession(self.environment, \"http://127.0.0.1:%i\" % self.port, user=None)\n        url = \"/redirect?url=/redirect&delay=0.5\"\n        r = s.get(url)\n        stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, stats.num_requests)\n        self.assertGreater(stats.avg_response_time, 500)\n\n    def test_client_basic_auth(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        class MyAuthorizedUser(FastHttpUser):\n            host = \"http://locust:menace@127.0.0.1:%i\" % self.port\n\n        class MyUnauthorizedUser(FastHttpUser):\n            host = \"http://locust:wrong@127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        unauthorized = MyUnauthorizedUser(self.environment)\n        authorized = MyAuthorizedUser(self.environment)\n        response = authorized.client.get(\"/basic_auth\")\n        self.assertEqual(200, response.status_code)\n        self.assertEqual(\"Authorized\", response.text)\n        self.assertEqual(401, locust.client.get(\"/basic_auth\").status_code)\n        self.assertEqual(401, unauthorized.client.get(\"/basic_auth\").status_code)\n\n    def test_shared_client_pool(self):\n        shared_client_pool = HTTPClientPool(concurrency=1)\n\n        class MyUserA(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n            client_pool = shared_client_pool\n\n        class MyUserB(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n            client_pool = shared_client_pool\n\n        user_a = MyUserA(self.environment)\n        user_b = MyUserB(self.environment)\n\n        user_a.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_a.client.get(\"/ultra_fast\")\n\n        self.assertEqual(1, self.connections_count)\n        self.assertEqual(4, self.requests_count)\n\n    def test_client_pool_per_user_instance(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        user_a = MyUser(self.environment)\n        user_b = MyUser(self.environment)\n\n        user_a.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_a.client.get(\"/ultra_fast\")\n\n        self.assertEqual(2, self.connections_count)\n        self.assertEqual(4, self.requests_count)\n\n    def test_client_pool_concurrency(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n            @task\n            def t(self):\n                def concurrent_request(url):\n                    response = self.client.get(url)\n                    assert response.status_code == 200\n\n                pool = gevent.pool.Pool()\n                urls = [\"/slow?delay=0.2\"] * 20  # these urls are all the same, but they could be different\n                for url in urls:\n                    pool.spawn(concurrent_request, url)\n                pool.join()\n\n        user = MyUser(self.environment)\n        before_requests = time.time()\n        user.t()\n        after_requests = time.time()\n        expected_delta = 0.4  # 20 requests with concurrency 10 and response time 0.2\n        self.assertAlmostEqual(before_requests + expected_delta, after_requests, delta=0.1)\n\n\nclass TestFastHttpCatchResponse(WebserverTestCase):\n    def setUp(self):\n        super().setUp()\n\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        self.user = MyUser(self.environment)\n\n        self.num_failures = 0\n        self.num_success = 0\n\n        def on_request(exception, **kwargs):\n            if exception:\n                self.num_failures += 1\n                self.last_failure_exception = exception\n            else:\n                self.num_success += 1\n\n        self.environment.events.request.add_listener(on_request)\n\n    def test_catch_response(self):\n        self.assertEqual(500, self.user.client.get(\"/fail\").status_code)\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n        with self.user.client.get(\"/ultra_fast\", catch_response=True) as response:\n            pass\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(1, self.num_success)\n        self.assertIn(\"ultra fast\", str(response.content))\n\n        with self.user.client.get(\"/ultra_fast\", catch_response=True) as response:\n            raise ResponseError(\"Not working\")\n\n        self.assertEqual(2, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_catch_response_http_fail(self):\n        with self.user.client.get(\"/fail\", catch_response=True) as response:\n            pass\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n    def test_catch_response_http_manual_fail(self):\n        with self.user.client.get(\"/ultra_fast\", catch_response=True) as response:\n            response.failure(\"Haha!\")\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n        self.assertTrue(\n            isinstance(self.last_failure_exception, CatchResponseError),\n            \"Failure event handler should have been passed a CatchResponseError instance\",\n        )\n\n    def test_catch_response_http_manual_success(self):\n        with self.user.client.get(\"/fail\", catch_response=True) as response:\n            response.success()\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_catch_response_allow_404(self):\n        with self.user.client.get(\"/does/not/exist\", catch_response=True) as response:\n            self.assertEqual(404, response.status_code)\n            if response.status_code == 404:\n                response.success()\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_interrupt_taskset_with_catch_response(self):\n        class MyTaskSet(TaskSet):\n            @task\n            def interrupted_task(self):\n                with self.client.get(\"/ultra_fast\", catch_response=True) as r:\n                    raise InterruptTaskSet()\n\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n            tasks = [MyTaskSet]\n\n        l = MyUser(self.environment)\n        ts = MyTaskSet(l)\n        self.assertRaises(InterruptTaskSet, lambda: ts.interrupted_task())\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n    def test_catch_response_connection_error_success(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:1\"\n\n        l = MyUser(self.environment)\n        with l.client.get(\"/\", catch_response=True) as r:\n            self.assertEqual(r.status_code, 0)\n            self.assertEqual(None, r.content)\n            r.success()\n        self.assertEqual(1, self.num_success)\n        self.assertEqual(0, self.num_failures)\n\n    def test_catch_response_connection_error_fail(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:1\"\n\n        l = MyUser(self.environment)\n        with l.client.get(\"/\", catch_response=True) as r:\n            self.assertEqual(r.status_code, 0)\n            self.assertEqual(None, r.content)\n            r.failure(\"Manual fail\")\n        self.assertEqual(0, self.num_success)\n        self.assertEqual(1, self.num_failures)\n\n    def test_catch_response_missing_with_block(self):\n        # incorrect usage, missing with-block\n        r = self.user.client.get(\"/fail\", catch_response=True)\n        self.assertRaises(LocustError, r.success)\n        self.assertRaises(LocustError, r.failure, \"\")\n\n    def test_missing_catch_response_true(self):\n        # incorrect usage, missing catch_response=True\n        with self.user.client.get(\"/fail\") as resp:\n            self.assertRaises(LocustError, resp.success)\n\n    def test_rest_success(self):\n        self.last_failure_exception = None\n        with self.user.rest(\"POST\", \"/rest\", json={\"foo\": \"bar\"}) as response:\n            assert response.js[\"foo\"] == \"bar\"\n\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_rest_fail(self):\n        with self.user.rest(\"POST\", \"/rest\", json={\"foo\": \"bar\"}) as response:\n            assert response.js[\"foo\"] == \"NOPE\"\n\n        self.assertTrue(\n            isinstance(self.last_failure_exception, CatchResponseError),\n            \"Failure event handler should have been passed a CatchResponseError instance\",\n        )\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n\nclass TestFastHttpSsl(LocustTestCase):\n    def setUp(self):\n        super().setUp()\n        tls_cert, tls_key = create_tls_cert(\"127.0.0.1\")\n        self.tls_cert_file = NamedTemporaryFile()\n        self.tls_key_file = NamedTemporaryFile()\n        with open(self.tls_cert_file.name, \"w\") as f:\n            f.write(tls_cert.decode())\n        with open(self.tls_key_file.name, \"w\") as f:\n            f.write(tls_key.decode())\n\n        self.web_ui = self.environment.create_web_ui(\n            \"127.0.0.1\",\n            0,\n            tls_cert=self.tls_cert_file.name,\n            tls_key=self.tls_key_file.name,\n        )\n        gevent.sleep(0.01)\n        self.web_port = self.web_ui.server.server_port\n\n    def tearDown(self):\n        super().tearDown()\n        self.web_ui.stop()\n\n    def test_ssl_request_insecure(self):\n        s = FastHttpSession(self.environment, \"https://127.0.0.1:%i\" % self.web_port, insecure=True, user=None)\n        r = s.get(\"/\")\n        self.assertEqual(200, r.status_code)\n        self.assertIn(\"<title>Locust for None</title>\", r.content.decode(\"utf-8\"))\n        self.assertIn(\"<p>Script: <span>None</span></p>\", r.text)\n\n# Path: locust/test/test_http.py\nfrom locust.clients import HttpSession\nfrom locust.exception import LocustError, ResponseError\nfrom locust.user.users import HttpUser\n\nimport time\n\nfrom requests.exceptions import InvalidSchema, InvalidURL, MissingSchema, RequestException\n\nfrom .testcases import WebserverTestCase\n\n\nclass TestHttpSession(WebserverTestCase):\n    def get_client(self, base_url=None):\n        if base_url is None:\n            base_url = \"http://127.0.0.1:%i\" % self.port\n        return HttpSession(\n            base_url=base_url,\n            request_event=self.environment.events.request,\n            user=None,\n        )\n\n    def test_get(self):\n        s = self.get_client()\n        r = s.get(\"/ultra_fast\")\n        self.assertEqual(200, r.status_code)\n\n    def test_connection_error(self):\n        s = self.get_client(base_url=\"http://localhost:1\")\n        r = s.get(\"/\", timeout=0.1)\n        self.assertEqual(r.status_code, 0)\n        self.assertEqual(None, r.content)\n        self.assertRaises(RequestException, r.raise_for_status)\n\n    def test_wrong_url(self):\n        for url, exception in (\n            (\"http://\\x94\", InvalidURL),\n            (\"telnet://127.0.0.1\", InvalidSchema),\n            (\"127.0.0.1\", MissingSchema),\n        ):\n            s = self.get_client(base_url=url)\n            try:\n                self.assertRaises(exception, s.get, \"/\")\n            except KeyError:\n                self.fail(f\"Invalid URL {url} was not propagated\")\n\n    def test_streaming_response(self):\n        \"\"\"\n        Test a request to an endpoint that returns a streaming response\n        \"\"\"\n        s = self.get_client()\n        r = s.get(\"/streaming/30\")\n\n        # verify that the time reported includes the download time of the whole streamed response\n        self.assertGreater(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n        self.runner.stats.clear_all()\n\n        # verify that response time does NOT include whole download time, when using stream=True\n        r = s.get(\"/streaming/30\", stream=True)\n        self.assertGreater(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 0)\n        self.assertLess(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n\n        # download the content of the streaming response (so we don't get an ugly exception in the log)\n        _ = r.content\n\n    def test_slow_redirect(self):\n        s = self.get_client()\n        url = \"/redirect?url=/redirect&delay=0.5\"\n        r = s.get(url)\n        stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, stats.num_requests)\n        self.assertGreater(stats.avg_response_time, 500)\n\n    def test_post_redirect(self):\n        s = self.get_client()\n        url = \"/redirect\"\n        r = s.post(url)\n        self.assertEqual(200, r.status_code)\n        post_stats = self.runner.stats.get(url, method=\"POST\")\n        get_stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, post_stats.num_requests)\n        self.assertEqual(0, get_stats.num_requests)\n\n    def test_cookie(self):\n        s = self.get_client()\n        r = s.post(\"/set_cookie?name=testcookie&value=1337\")\n        self.assertEqual(200, r.status_code)\n        r = s.get(\"/get_cookie?name=testcookie\")\n        self.assertEqual(\"1337\", r.content.decode())\n\n    def test_head(self):\n        s = self.get_client()\n        r = s.head(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n\n    def test_delete(self):\n        s = self.get_client()\n        r = s.delete(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"DELETE\", r.content.decode())\n\n    def test_options(self):\n        s = self.get_client()\n        r = s.options(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n        self.assertEqual(\n            {\"OPTIONS\", \"DELETE\", \"PUT\", \"GET\", \"POST\", \"HEAD\", \"PATCH\"},\n            set(r.headers[\"allow\"].split(\", \")),\n        )\n\n    def test_error_message(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        s.request(\"get\", \"/wrong_url\", context={\"foo\": \"bar\"})\n        self.assertIn(\"/wrong_url\", str(kwargs[\"exception\"]))\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    def test_context_in_success(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(exception, **kw):\n            self.assertIsNone(exception)\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        s.request(\"get\", \"/request_method\", context={\"foo\": \"bar\"})\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    def test_response_parameter(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        s.request(\"get\", \"/request_method\")\n        self.assertEqual(\"GET\", kwargs[\"response\"].text)\n        s.request(\"get\", \"/wrong_url\")\n        self.assertEqual(\"Not Found\", kwargs[\"response\"].text)\n\n    def test_error_message_with_name_replacement(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            self.assertIsNotNone(kw[\"exception\"])\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        before_request = time.time()\n        s.request(\"get\", \"/wrong_url/01\", name=\"replaced_url_name\", context={\"foo\": \"bar\"})\n        after_request = time.time()\n        self.assertIn(\"for url: replaced_url_name\", str(kwargs[\"exception\"]))\n        self.assertAlmostEqual(before_request, kwargs[\"start_time\"], delta=0.01)\n        self.assertAlmostEqual(after_request, kwargs[\"start_time\"] + kwargs[\"response_time\"] / 1000, delta=0.01)\n        self.assertEqual(s.base_url + \"/wrong_url/01\", kwargs[\"url\"])  # url is unaffected by name\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    def test_get_with_params(self):\n        s = self.get_client()\n        r = s.get(\"/get_arg\", params={\"arg\": \"test_123\"})\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"test_123\", r.text)\n\n    def test_catch_response_fail_successful_request(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure(\"nope\")\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n\n    def test_catch_response_fail_successful_request_with_non_string_error_message(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure({\"other types are also wrapped as exceptions\": True})\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n\n    def test_catch_response_pass_failed_request(self):\n        s = self.get_client()\n        with s.get(\"/fail\", catch_response=True) as r:\n            r.success()\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_multiple_failure_and_success(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure(\"nope\")\n            r.success()\n            r.failure(\"nooo\")\n            r.success()\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_timeout(self):\n        s = self.get_client()\n        with s.get(\"/slow\", catch_response=True, timeout=0.1) as r:\n            self.assertAlmostEqual(r.request_meta[\"response_time\"], 100, delta=50)\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(1, self.environment.stats.total.num_failures)\n\n    def test_catch_response_pass_failed_request_with_other_exception_within_block(self):\n        class OtherException(Exception):\n            pass\n\n        s = self.get_client()\n        try:\n            with s.get(\"/fail\", catch_response=True) as r:\n                r.success()\n                raise OtherException(\"wtf\")\n        except OtherException as e:\n            pass\n        else:\n            self.fail(\"OtherException should have been raised\")\n\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_response_error(self):\n        s = self.get_client()\n        try:\n            with s.get(\"/fail\", catch_response=True) as r:\n                raise ResponseError(\"response error\")\n        except ResponseError as e:\n            self.fail(\"ResponseError should not have been raised\")\n\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(1, self.environment.stats.total.num_failures)\n\n    def test_catch_response_default_success(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            pass\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(0, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n\n    def test_catch_response_default_fail(self):\n        s = self.get_client()\n        with s.get(\"/fail\", catch_response=True) as r:\n            pass\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(1, self.environment.stats.total.num_failures)\n\n    def test_catch_response_with_name_replacement(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            self.assertIsNotNone(kw[\"exception\"])\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n\n        with s.get(\"/wrong_url/01\", name=\"replaced_url_name\") as r:\n            pass\n\n        self.assertIn(\"for url: replaced_url_name\", str(kwargs[\"exception\"]))\n        self.assertEqual(s.base_url + \"/wrong_url/01\", kwargs[\"url\"])  # url is unaffected by name\n\n    def test_catch_response_missing_with_block(self):\n        s = self.get_client()\n        # incorrect usage, missing with-block\n        r = s.get(\"/fail\", catch_response=True)\n        self.assertRaises(LocustError, r.success)\n        self.assertRaises(LocustError, r.failure, \"\")\n\n    def test_missing_catch_response_true(self):\n        s = self.get_client()\n        # incorrect usage, missing catch_response=True\n        with s.get(\"/fail\") as resp:\n            self.assertRaises(LocustError, resp.success)\n\n    def test_event_measure(self):\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(**kw)\n\n        self.environment.events.request.add_listener(on_request)\n\n        with self.environment.events.request.measure(\"GET\", \"/test\") as request_meta:\n            time.sleep(0.001)\n\n        self.assertTrue(1 <= kwargs[\"response_time\"] <= 1.5, kwargs[\"response_time\"])\n        self.assertEqual(kwargs[\"name\"], \"/test\")\n        self.assertIsNone(kwargs[\"exception\"])\n\n        with self.environment.events.request.measure(\"GET\", \"/test\") as request_meta:\n            request_meta[\"foo\"] = \"bar\"\n            raise Exception(\"nooo\")\n\n        self.assertEqual(kwargs[\"name\"], \"/test\")\n        self.assertEqual(kwargs[\"foo\"], \"bar\")\n        self.assertEqual(str(kwargs[\"exception\"]), \"nooo\")\n\n    def test_user_context(self):\n        class TestUser(HttpUser):\n            host = f\"http://127.0.0.1:{self.port}\"\n\n            def context(self):\n                return {\"user\": self.username}\n\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n\n        user = TestUser(self.environment)\n        user.username = \"foo\"\n        user.client.request(\"get\", \"/request_method\")\n        self.assertDictEqual({\"user\": \"foo\"}, kwargs[\"context\"])\n        self.assertEqual(\"GET\", kwargs[\"response\"].text)\n        user.client.request(\"get\", \"/request_method\", context={\"user\": \"bar\"})  # override User context\n...\n", "needle_token_start": 709, "needle_token_end": 852, "code_context_ntokens": 16084, "output": ["```python\ndef test_user_classes_with_same_name_is_error(self):\n    with self.assertRaises(ValueError) as e:\n        environment = Environment(user_classes=[MyUserWithSameName1, MyUserWithSameName2])\n\n    self.assertEqual(\n        e.exception.args[0],\n        \"The following user classes have the same class name: locust.test.fake_module1_for_env_test.MyUserWithSameName, locust.test.fake_module2_for_env_test.MyUserWithSameName\",\n    )\n```"]}
{"repo": "locustio/locust", "name": "rename_request", "language": "python", "path": "locust/clients.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To temporarily change the label used for grouping performance statistics of web requests in a session, facilitating the aggregation of different URLs under a common name in statistical reports.\n2. **Input**: A string representing the new label name for the grouping of requests.\n3. **Output**: This method does not return any value; it acts as a context manager that temporarily changes an internal state.\n4. **Procedure**: The method sets an internal label to the provided name, executes the block of code within its context, and then resets the label to its original state, ensuring that all requests made within the context block are grouped under the specified name in performance statistics.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: locust/user/task.py\nfrom __future__ import annotations\n\nfrom locust.exception import InterruptTaskSet, MissingWaitTimeError, RescheduleTask, RescheduleTaskImmediately, StopUser\n\nimport logging\nimport random\nimport traceback\nfrom time import time\nfrom typing import (\n    TYPE_CHECKING,\n    Callable,\n    Protocol,\n    Type,\n    TypeVar,\n    final,\n    overload,\n    runtime_checkable,\n)\n\nimport gevent\nfrom gevent import GreenletExit\n\nif TYPE_CHECKING:\n    from locust import User\n\n\nlogger = logging.getLogger(__name__)\nTaskT = TypeVar(\"TaskT\", Callable[..., None], Type[\"TaskSet\"])\n\nLOCUST_STATE_RUNNING, LOCUST_STATE_WAITING, LOCUST_STATE_STOPPING = [\"running\", \"waiting\", \"stopping\"]\n\n\n@runtime_checkable\nclass TaskHolder(Protocol[TaskT]):\n    tasks: list[TaskT]\n\n\n@overload\ndef task(weight: TaskT) -> TaskT: ...\n\n\n@overload\ndef task(weight: int) -> Callable[[TaskT], TaskT]: ...\n\n\ndef task(weight: TaskT | int = 1) -> TaskT | Callable[[TaskT], TaskT]:\n    \"\"\"\n    Used as a convenience decorator to be able to declare tasks for a User or a TaskSet\n    inline in the class. Example::\n\n        class ForumPage(TaskSet):\n            @task(100)\n            def read_thread(self):\n                pass\n\n            @task(7)\n            def create_thread(self):\n                pass\n\n            @task(25)\n            class ForumThread(TaskSet):\n                @task\n                def get_author(self):\n                    pass\n\n                @task\n                def get_created(self):\n                    pass\n    \"\"\"\n\n    def decorator_func(func):\n        if func.__name__ in [\"on_stop\", \"on_start\"]:\n            logging.warning(\n                \"You have tagged your on_stop/start function with @task. This will make the method get called both as a task AND on stop/start.\"\n            )  # this is usually not what the user intended\n        if func.__name__ == \"run\":\n            raise Exception(\n                \"User.run() is a method used internally by Locust, and you must not override it or register it as a task\"\n            )\n        func.locust_task_weight = weight\n        return func\n\n    \"\"\"\n    Check if task was used without parentheses (not called), like this::\n\n        @task\n        def my_task(self)\n            pass\n    \"\"\"\n    if callable(weight):\n        func = weight\n        weight = 1\n        return decorator_func(func)\n    else:\n        return decorator_func\n\n\ndef tag(*tags: str) -> Callable[[TaskT], TaskT]:\n    \"\"\"\n    Decorator for tagging tasks and TaskSets with the given tag name. You can\n    then limit the test to only execute tasks that are tagged with any of the\n    tags provided by the :code:`--tags` command-line argument. Example::\n\n        class ForumPage(TaskSet):\n            @tag('thread')\n            @task(100)\n            def read_thread(self):\n                pass\n\n            @tag('thread')\n            @tag('post')\n            @task(7)\n            def create_thread(self):\n                pass\n\n            @tag('post')\n            @task(11)\n            def comment(self):\n                pass\n    \"\"\"\n\n    def decorator_func(decorated):\n        if hasattr(decorated, \"tasks\"):\n            decorated.tasks = list(map(tag(*tags), decorated.tasks))\n        else:\n            if \"locust_tag_set\" not in decorated.__dict__:\n                decorated.locust_tag_set = set()\n            decorated.locust_tag_set |= set(tags)\n        return decorated\n\n    if len(tags) == 0 or callable(tags[0]):\n        raise ValueError(\"No tag name was supplied\")\n\n    return decorator_func\n\n\ndef get_tasks_from_base_classes(bases, class_dict):\n    \"\"\"\n    Function used by both TaskSetMeta and UserMeta for collecting all declared tasks\n    on the TaskSet/User class and all its base classes\n    \"\"\"\n    new_tasks = []\n    for base in bases:\n        if hasattr(base, \"tasks\") and base.tasks:\n            new_tasks += base.tasks\n\n    if \"tasks\" in class_dict and class_dict[\"tasks\"] is not None:\n        tasks = class_dict[\"tasks\"]\n        if isinstance(tasks, dict):\n            tasks = tasks.items()\n\n        for task in tasks:\n            if isinstance(task, tuple):\n                task, count = task\n                for _ in range(count):\n                    new_tasks.append(task)\n            else:\n                new_tasks.append(task)\n\n    for item in class_dict.values():\n        if \"locust_task_weight\" in dir(item):\n            for i in range(item.locust_task_weight):\n                new_tasks.append(item)\n\n    return new_tasks\n\n\ndef filter_tasks_by_tags(\n    task_holder: type[TaskHolder],\n...\n# Path: locust/clients.py\nfrom __future__ import annotations\n\nimport re\nimport time\nfrom contextlib import contextmanager\nfrom typing import Generator\nfrom urllib.parse import urlparse, urlunparse\n\nimport requests\nfrom requests import Request, Response\nfrom requests.adapters import HTTPAdapter\nfrom requests.auth import HTTPBasicAuth\nfrom requests.exceptions import InvalidSchema, InvalidURL, MissingSchema, RequestException\nfrom urllib3 import PoolManager\n\nfrom .exception import CatchResponseError, LocustError, ResponseError\n\nabsolute_http_url_regexp = re.compile(r\"^https?://\", re.I)\n\n\nclass LocustResponse(Response):\n    def raise_for_status(self):\n        if hasattr(self, \"error\") and self.error:\n            raise self.error\n        Response.raise_for_status(self)\n\n\nclass HttpSession(requests.Session):\n    \"\"\"\n    Class for performing web requests and holding (session-) cookies between requests (in order\n    to be able to log in and out of websites). Each request is logged so that locust can display\n    statistics.\n\n    This is a slightly extended version of `python-request <http://python-requests.org>`_'s\n    :py:class:`requests.Session` class and mostly this class works exactly the same. However\n    the methods for making requests (get, post, delete, put, head, options, patch, request)\n    can now take a *url* argument that's only the path part of the URL, in which case the host\n    part of the URL will be prepended with the HttpSession.base_url which is normally inherited\n    from a User class' host attribute.\n\n    Each of the methods for making requests also takes two additional optional arguments which\n    are Locust specific and doesn't exist in python-requests. These are:\n\n    :param name: (optional) An argument that can be specified to use as label in Locust's statistics instead of the URL path.\n                 This can be used to group different URL's that are requested into a single entry in Locust's statistics.\n    :param catch_response: (optional) Boolean argument that, if set, can be used to make a request return a context manager\n                           to work as argument to a with statement. This will allow the request to be marked as a fail based on the content of the\n                           response, even if the response code is ok (2xx). The opposite also works, one can use catch_response to catch a request\n                           and then mark it as successful even if the response code was not (i.e 500 or 404).\n    \"\"\"\n\n    def __init__(self, base_url, request_event, user, *args, pool_manager: PoolManager | None = None, **kwargs):\n        super().__init__(*args, **kwargs)\n\n        self.base_url = base_url\n        self.request_event = request_event\n        self.user = user\n\n        # User can group name, or use the group context manager to gather performance statistics under a specific name\n        # This is an alternative to passing in the \"name\" parameter to the requests function\n        self.request_name: str | None = None\n\n        # Check for basic authentication\n        parsed_url = urlparse(self.base_url)\n        if parsed_url.username and parsed_url.password:\n            netloc = parsed_url.hostname\n            if parsed_url.port:\n                netloc += \":%d\" % parsed_url.port\n\n            # remove username and password from the base_url\n            self.base_url = urlunparse(\n                (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)\n            )\n            # configure requests to use basic auth\n            self.auth = HTTPBasicAuth(parsed_url.username, parsed_url.password)\n\n        self.mount(\"https://\", LocustHttpAdapter(pool_manager=pool_manager))\n        self.mount(\"http://\", LocustHttpAdapter(pool_manager=pool_manager))\n\n    def _build_url(self, path):\n        \"\"\"prepend url with hostname unless it's already an absolute URL\"\"\"\n        if absolute_http_url_regexp.match(path):\n            return path\n        else:\n            return f\"{self.base_url}{path}\"\n\n    @contextmanager\n    \ndef rename_request(self, name: str) -> Generator[None, None, None]:\n        \"\"\"Group requests using the \"with\" keyword\"\"\"\n\n        self.request_name = name\n        try:\n            yield\n        finally:\n            self.request_name = None\n\n    def request(self, method, url, name=None, catch_response=False, context={}, **kwargs):\n        \"\"\"\n        Constructs and sends a :py:class:`requests.Request`.\n        Returns :py:class:`requests.Response` object.\n\n        :param method: method for the new :class:`Request` object.\n        :param url: URL for the new :class:`Request` object.\n        :param name: (optional) An argument that can be specified to use as label in Locust's statistics instead of the URL path.\n          This can be used to group different URL's that are requested into a single entry in Locust's statistics.\n        :param catch_response: (optional) Boolean argument that, if set, can be used to make a request return a context manager\n          to work as argument to a with statement. This will allow the request to be marked as a fail based on the content of the\n          response, even if the response code is ok (2xx). The opposite also works, one can use catch_response to catch a request\n          and then mark it as successful even if the response code was not (i.e 500 or 404).\n        :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.\n        :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.\n        :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.\n        :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.\n        :param files: (optional) Dictionary of ``'filename': file-like-objects`` for multipart encoding upload.\n        :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth.\n        :param timeout: (optional) How long in seconds to wait for the server to send data before giving up, as a float,\n            or a (`connect timeout, read timeout <user/advanced.html#timeouts>`_) tuple.\n        :type timeout: float or tuple\n        :param allow_redirects: (optional) Set to True by default.\n        :type allow_redirects: bool\n        :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.\n        :param stream: (optional) whether to immediately download the response content. Defaults to ``False``.\n        :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.\n        :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.\n        \"\"\"\n\n        # if group name has been set and no name parameter has been passed in; set the name parameter to group_name\n        if self.request_name and not name:\n            name = self.request_name\n\n        # prepend url with hostname unless it's already an absolute URL\n        url = self._build_url(url)\n\n        start_time = time.time()\n        start_perf_counter = time.perf_counter()\n        response = self._send_request_safe_mode(method, url, **kwargs)\n        response_time = (time.perf_counter() - start_perf_counter) * 1000\n\n        request_before_redirect = (response.history and response.history[0] or response).request\n        url = request_before_redirect.url\n\n        if not name:\n            name = request_before_redirect.path_url\n\n        if self.user:\n            context = {**self.user.context(), **context}\n\n        # store meta data that is used when reporting the request to locust's statistics\n        request_meta = {\n            \"request_type\": method,\n            \"response_time\": response_time,\n            \"name\": name,\n            \"context\": context,\n            \"response\": response,\n            \"exception\": None,\n            \"start_time\": start_time,\n            \"url\": url,\n        }\n\n        # get the length of the content, but if the argument stream is set to True, we take\n        # the size from the content-length header, in order to not trigger fetching of the body\n        if kwargs.get(\"stream\", False):\n            request_meta[\"response_length\"] = int(response.headers.get(\"content-length\") or 0)\n        else:\n            request_meta[\"response_length\"] = len(response.content or b\"\")\n\n        if catch_response:\n            return ResponseContextManager(response, request_event=self.request_event, request_meta=request_meta)\n        else:\n            with ResponseContextManager(response, request_event=self.request_event, request_meta=request_meta):\n                pass\n            return response\n\n    def _send_request_safe_mode(self, method, url, **kwargs):\n        \"\"\"\n        Send an HTTP request, and catch any exception that might occur due to connection problems.\n\n        Safe mode has been removed from requests 1.x.\n        \"\"\"\n        try:\n            return super().request(method, url, **kwargs)\n        except (MissingSchema, InvalidSchema, InvalidURL):\n            raise\n        except RequestException as e:\n            r = LocustResponse()\n            r.error = e\n            r.status_code = 0  # with this status_code, content returns None\n            r.request = Request(method, url).prepare()\n            return r\n\n\nclass ResponseContextManager(LocustResponse):\n    \"\"\"\n    A Response class that also acts as a context manager that provides the ability to manually\n    control if an HTTP request should be marked as successful or a failure in Locust's statistics\n\n    This class is a subclass of :py:class:`Response <requests.Response>` with two additional\n    methods: :py:meth:`success <locust.clients.ResponseContextManager.success>` and\n    :py:meth:`failure <locust.clients.ResponseContextManager.failure>`.\n    \"\"\"\n\n    _manual_result: bool | Exception | None = None\n    _entered = False\n\n    def __init__(self, response, request_event, request_meta):\n        # copy data from response to this object\n        self.__dict__ = response.__dict__\n        self._request_event = request_event\n        self.request_meta = request_meta\n\n    def __enter__(self):\n        self._entered = True\n        return self\n\n    def __exit__(self, exc, value, traceback):\n        # if the user has already manually marked this response as failure or success\n        # we can ignore the default behaviour of letting the response code determine the outcome\n        if self._manual_result is not None:\n            if self._manual_result is True:\n                self.request_meta[\"exception\"] = None\n            elif isinstance(self._manual_result, Exception):\n                self.request_meta[\"exception\"] = self._manual_result\n            self._report_request()\n            return exc is None\n\n        if exc:\n            if isinstance(value, ResponseError):\n                self.request_meta[\"exception\"] = value\n                self._report_request()\n            else:\n                # we want other unknown exceptions to be raised\n                return False\n        else:\n            # Since we use the Exception message when grouping failures, in order to not get\n            # multiple failure entries for different URLs for the same name argument, we need\n            # to temporarily override the response.url attribute\n            orig_url = self.url\n            self.url = self.request_meta[\"name\"]\n\n            try:\n                self.raise_for_status()\n            except requests.exceptions.RequestException as e:\n                while (\n                    isinstance(\n                        e,\n                        (\n                            requests.exceptions.ConnectionError,\n                            requests.packages.urllib3.exceptions.ProtocolError,\n                            requests.packages.urllib3.exceptions.MaxRetryError,\n                            requests.packages.urllib3.exceptions.NewConnectionError,\n                        ),\n                    )\n                    and e.__context__  # Not sure if the above exceptions can ever be the lowest level, but it is good to be sure\n                ):\n                    e = e.__context__\n                self.request_meta[\"exception\"] = e\n\n            self._report_request()\n            self.url = orig_url\n\n        return True\n\n    def _report_request(self, exc=None):\n        self._request_event.fire(**self.request_meta)\n\n    def success(self):\n        \"\"\"\n        Report the response as successful\n\n        Example::\n\n            with self.client.get(\"/does/not/exist\", catch_response=True) as response:\n                if response.status_code == 404:\n                    response.success()\n        \"\"\"\n        if not self._entered:\n            raise LocustError(\n                \"Tried to set status on a request that has not yet been made. Make sure you use a with-block, like this:\\n\\nwith self.client.request(..., catch_response=True) as response:\\n    response.success()\"\n            )\n        self._manual_result = True\n\n    def failure(self, exc):\n        \"\"\"\n        Report the response as a failure.\n\n        if exc is anything other than a python exception (like a string) it will\n        be wrapped inside a CatchResponseError.\n\n        Example::\n\n            with self.client.get(\"/\", catch_response=True) as response:\n                if response.content == b\"\":\n                    response.failure(\"No data\")\n        \"\"\"\n        if not self._entered:\n            raise LocustError(\n                \"Tried to set status on a request that has not yet been made. Make sure you use a with-block, like this:\\n\\nwith self.client.request(..., catch_response=True) as response:\\n    response.failure(...)\"\n            )\n        if not isinstance(exc, Exception):\n            exc = CatchResponseError(exc)\n        self._manual_result = exc\n\n\nclass LocustHttpAdapter(HTTPAdapter):\n    def __init__(self, pool_manager: PoolManager | None, *args, **kwargs):\n        self.poolmanager = pool_manager\n        super().__init__(*args, **kwargs)\n\n    def init_poolmanager(self, *args, **kwargs):\n        if self.poolmanager is None:\n            super().init_poolmanager(*args, **kwargs)\n\n\n# Monkey patch Response class to give some guidance\ndef _success(self):\n    raise LocustError(\n        \"If you want to change the state of the request, you must pass catch_response=True. See http://docs.locust.io/en/stable/writing-a-locustfile.html#validating-responses\"\n    )\n\n\ndef _failure(self):\n    raise LocustError(\n        \"If you want to change the state of the request, you must pass catch_response=True. See http://docs.locust.io/en/stable/writing-a-locustfile.html#validating-responses\"\n    )\n\n\nResponse.success = _success  # type: ignore[attr-defined]\nResponse.failure = _failure  # type: ignore[attr-defined]\n\n# Path: locust/user/wait_time.py\nimport random\nfrom time import time\n\n\ndef between(min_wait, max_wait):\n    \"\"\"\n    Returns a function that will return a random number between min_wait and max_wait.\n\n    Example::\n\n        class MyUser(User):\n            # wait between 3.0 and 10.5 seconds after each task\n            wait_time = between(3.0, 10.5)\n    \"\"\"\n    return lambda instance: min_wait + random.random() * (max_wait - min_wait)\n\n\ndef constant(wait_time):\n    \"\"\"\n    Returns a function that just returns the number specified by the wait_time argument\n\n    Example::\n\n        class MyUser(User):\n            wait_time = constant(3)\n    \"\"\"\n    return lambda instance: wait_time\n\n\ndef constant_pacing(wait_time):\n    \"\"\"\n    Returns a function that will track the run time of the tasks, and for each time it's\n    called it will return a wait time that will try to make the total time between task\n    execution equal to the time specified by the wait_time argument.\n\n    In the following example the task will always be executed once every 10 seconds, no matter\n    the task execution time::\n\n        class MyUser(User):\n            wait_time = constant_pacing(10)\n            @task\n            def my_task(self):\n                time.sleep(random.random())\n\n    If a task execution exceeds the specified wait_time, the wait will be 0 before starting\n    the next task.\n    \"\"\"\n\n    def wait_time_func(self):\n        if not hasattr(self, \"_cp_last_wait_time\"):\n            self._cp_last_wait_time = 0\n        run_time = time() - self._cp_last_run - self._cp_last_wait_time\n        self._cp_last_wait_time = max(0, wait_time - run_time)\n        self._cp_last_run = time()\n        return self._cp_last_wait_time\n\n    return wait_time_func\n\n\ndef constant_throughput(task_runs_per_second):\n    \"\"\"\n    Returns a function that will track the run time of the tasks, and for each time it's\n    called it will return a wait time that will try to make the number of task runs per second\n    execution equal to the time specified by the task_runs_per_second argument.\n\n    If you have multiple requests in a task your RPS will of course be higher than the\n    specified throughput.\n\n    This is the mathematical inverse of constant_pacing.\n\n    In the following example the task will always be executed once every 10 seconds, no matter\n    the task execution time::\n\n        class MyUser(User):\n            wait_time = constant_throughput(0.1)\n            @task\n            def my_task(self):\n                time.sleep(random.random())\n\n    If a task execution exceeds the specified wait_time, the wait will be 0 before starting\n    the next task.\n    \"\"\"\n    return constant_pacing(1 / task_runs_per_second)\n\n# Path: locust/util/deprecation.py\nimport warnings\n\n# Show deprecation warnings\nwarnings.filterwarnings(\"always\", category=DeprecationWarning, module=\"locust\")\n\n\ndef check_for_deprecated_task_set_attribute(class_dict):\n    from locust.user.task import TaskSet\n\n    if \"task_set\" in class_dict:\n        task_set = class_dict[\"task_set\"]\n        if issubclass(task_set, TaskSet) and not hasattr(task_set, \"locust_task_weight\"):\n            warnings.warn(\n                \"Usage of User.task_set is deprecated since version 1.0. Set the tasks attribute instead \"\n                \"(tasks = [%s])\" % task_set.__name__,\n                DeprecationWarning,\n            )\n\n\ndef deprecated_locust_meta_class(deprecation_message):\n    class MetaClass(type):\n        def __new__(mcs, classname, bases, class_dict):\n            if classname in [\"DeprecatedLocustClass\", \"DeprecatedHttpLocustClass\", \"DeprecatedFastHttpLocustClass\"]:\n                return super().__new__(mcs, classname, bases, class_dict)\n            else:\n                raise ImportError(deprecation_message)\n\n    return MetaClass\n\n\n# PEP 484 specifies \"Generic metaclasses are not supported\", see https://github.com/python/mypy/issues/3602, ignore typing errors\nclass DeprecatedLocustClass(\n    metaclass=deprecated_locust_meta_class(  # type: ignore\n        \"The Locust class has been renamed to User in version 1.0. \"\n        \"For more info see: https://docs.locust.io/en/latest/changelog.html#changelog-1-0\"\n    )\n):\n    pass\n\n\nclass DeprecatedHttpLocustClass(\n    metaclass=deprecated_locust_meta_class(  # type: ignore\n        \"The HttpLocust class has been renamed to HttpUser in version 1.0. \"\n        \"For more info see: https://docs.locust.io/en/latest/changelog.html#changelog-1-0\"\n    )\n):\n    pass\n\n\nclass DeprecatedFastHttpLocustClass(\n    metaclass=deprecated_locust_meta_class(  # type: ignore\n        \"The FastHttpLocust class has been renamed to FastHttpUser in version 1.0. \"\n        \"For more info see: https://docs.locust.io/en/latest/changelog.html#changelog-1-0\"\n    )\n):\n    pass\n\n# Path: locust/user/users.py\nfrom __future__ import annotations\n\nimport logging\nimport time\nimport traceback\nfrom typing import Callable, final\n\nfrom gevent import GreenletExit, greenlet\nfrom gevent.pool import Group\nfrom urllib3 import PoolManager\n\nlogger = logging.getLogger(__name__)\nfrom locust.clients import HttpSession\nfrom locust.exception import LocustError, StopUser\nfrom locust.user.wait_time import constant\nfrom locust.util import deprecation\n\nfrom .task import (\n    LOCUST_STATE_RUNNING,\n    LOCUST_STATE_STOPPING,\n    LOCUST_STATE_WAITING,\n    DefaultTaskSet,\n    TaskSet,\n    get_tasks_from_base_classes,\n)\n\nlogger = logging.getLogger(__name__)\n\n\nclass UserMeta(type):\n    \"\"\"\n    Meta class for the main User class. It's used to allow User classes to specify task execution\n    ratio using an {task:int} dict, or a [(task0,int), ..., (taskN,int)] list.\n    \"\"\"\n\n    def __new__(mcs, classname, bases, class_dict):\n        # gather any tasks that is declared on the class (or it's bases)\n        tasks = get_tasks_from_base_classes(bases, class_dict)\n        class_dict[\"tasks\"] = tasks\n\n        if not class_dict.get(\"abstract\"):\n            # Not a base class\n            class_dict[\"abstract\"] = False\n\n        deprecation.check_for_deprecated_task_set_attribute(class_dict)\n\n        return type.__new__(mcs, classname, bases, class_dict)\n\n\nclass User(metaclass=UserMeta):\n    \"\"\"\n    Represents a \"user\" which is to be spawned and attack the system that is to be load tested.\n\n    The behaviour of this user is defined by its tasks. Tasks can be declared either directly on the\n    class by using the :py:func:`@task decorator <locust.task>` on methods, or by setting\n    the :py:attr:`tasks attribute <locust.User.tasks>`.\n\n    This class should usually be subclassed by a class that defines some kind of client. For\n    example when load testing an HTTP system, you probably want to use the\n    :py:class:`HttpUser <locust.HttpUser>` class.\n    \"\"\"\n\n    host: str | None = None\n    \"\"\"Base hostname to swarm. i.e: http://127.0.0.1:1234\"\"\"\n\n    min_wait = None\n    \"\"\"Deprecated: Use wait_time instead. Minimum waiting time between the execution of locust tasks\"\"\"\n\n    max_wait = None\n    \"\"\"Deprecated: Use wait_time instead. Maximum waiting time between the execution of locust tasks\"\"\"\n\n    wait_time = constant(0)\n    \"\"\"\n    Method that returns the time (in seconds) between the execution of locust tasks.\n    Can be overridden for individual TaskSets.\n\n    Example::\n\n        from locust import User, between\n        class MyUser(User):\n            wait_time = between(3, 25)\n    \"\"\"\n\n    wait_function = None\n    \"\"\"\n    .. warning::\n\n        DEPRECATED: Use wait_time instead. Note that the new wait_time method should return seconds and not milliseconds.\n\n    Method that returns the time between the execution of locust tasks in milliseconds\n    \"\"\"\n\n    tasks: list[TaskSet | Callable] = []\n    \"\"\"\n    Collection of python callables and/or TaskSet classes that the Locust user(s) will run.\n\n    If tasks is a list, the task to be performed will be picked randomly.\n\n    If tasks is a *(callable,int)* list of two-tuples, or a {callable:int} dict,\n    the task to be performed will be picked randomly, but each task will be weighted\n    according to its corresponding int value. So in the following case, *ThreadPage* will\n    be fifteen times more likely to be picked than *write_post*::\n\n        class ForumPage(TaskSet):\n            tasks = {ThreadPage:15, write_post:1}\n    \"\"\"\n\n    weight = 1\n    \"\"\"Probability of user class being chosen. The higher the weight, the greater the chance of it being chosen.\"\"\"\n\n    fixed_count = 0\n    \"\"\"\n    If the value > 0, the weight property will be ignored and the 'fixed_count'-instances will be spawned.\n    These Users are spawned first. If the total target count (specified by the --users arg) is not enough\n    to spawn all instances of each User class with the defined property, the final count of each User is undefined.\n    \"\"\"\n\n    abstract = True\n    \"\"\"If abstract is True, the class is meant to be subclassed, and locust will not spawn users of this class during a test.\"\"\"\n\n    def __init__(self, environment):\n        super().__init__()\n        self.environment = environment\n        \"\"\"A reference to the :py:class:`Environment <locust.env.Environment>` in which this user is running\"\"\"\n        self._state = None\n        self._greenlet: greenlet.Greenlet = None\n        self._group: Group\n        self._taskset_instance: TaskSet = None\n        self._cp_last_run = time.time()  # used by constant_pacing wait_time\n\n    def on_start(self) -> None:\n        \"\"\"\n        Called when a User starts running.\n        \"\"\"\n        pass\n\n    def on_stop(self):\n        \"\"\"\n        Called when a User stops running (is killed)\n        \"\"\"\n        pass\n\n    @final\n    def run(self):\n        self._state = LOCUST_STATE_RUNNING\n        self._taskset_instance = DefaultTaskSet(self)\n        try:\n            # run the TaskSet on_start method, if it has one\n            try:\n                self.on_start()\n            except Exception as e:\n                # unhandled exceptions inside tasks are logged in TaskSet.run, but since we're not yet there...\n                logger.error(\"%s\\n%s\", e, traceback.format_exc())\n                raise\n\n            self._taskset_instance.run()\n        except (GreenletExit, StopUser):\n            # run the on_stop method, if it has one\n            self.on_stop()\n\n    def wait(self):\n        \"\"\"\n        Make the running user sleep for a duration defined by the User.wait_time\n        function.\n\n        The user can also be killed gracefully while it's sleeping, so calling this\n        method within a task makes it possible for a user to be killed mid-task even if you've\n        set a stop_timeout. If this behaviour is not desired, you should make the user wait using\n        gevent.sleep() instead.\n        \"\"\"\n        self._taskset_instance.wait()\n\n    def start(self, group: Group):\n        \"\"\"\n        Start a greenlet that runs this User instance.\n\n        :param group: Group instance where the greenlet will be spawned.\n        :type group: gevent.pool.Group\n        :returns: The spawned greenlet.\n        \"\"\"\n\n        def run_user(user):\n            \"\"\"\n            Main function for User greenlet. It's important that this function takes the user\n            instance as an argument, since we use greenlet_instance.args[0] to retrieve a reference to the\n            User instance.\n            \"\"\"\n            user.run()\n\n        self._greenlet = group.spawn(run_user, self)\n        self._group = group\n        return self._greenlet\n\n    def stop(self, force=False):\n        \"\"\"\n        Stop the user greenlet.\n\n        :param force: If False (the default) the stopping is done gracefully by setting the state to LOCUST_STATE_STOPPING\n                      which will make the User instance stop once any currently running task is complete and on_stop\n                      methods are called. If force is True the greenlet will be killed immediately.\n        :returns: True if the greenlet was killed immediately, otherwise False\n        \"\"\"\n        if force or self._state == LOCUST_STATE_WAITING:\n            self._group.killone(self._greenlet)\n            return True\n        elif self._state == LOCUST_STATE_RUNNING:\n            self._state = LOCUST_STATE_STOPPING\n            return False\n        else:\n            raise Exception(f\"Tried to stop User in an unexpected state: {self._state}. This should never happen.\")\n\n    @property\n    def group(self):\n        return self._group\n\n    @property\n    def greenlet(self):\n        return self._greenlet\n\n    def context(self) -> dict:\n        \"\"\"\n        Adds the returned value (a dict) to the context for :ref:`request event <request_context>`.\n        Override this in your User class to customize the context.\n        \"\"\"\n        return {}\n\n    @classmethod\n    def json(cls):\n        return {\n            \"host\": cls.host,\n            \"weight\": cls.weight,\n            \"fixed_count\": cls.fixed_count,\n            \"tasks\": [task.__name__ for task in cls.tasks],\n        }\n\n    @classmethod\n    def fullname(cls) -> str:\n        \"\"\"Fully qualified name of the user class, e.g. my_package.my_module.MyUserClass\"\"\"\n        return \".\".join(filter(lambda x: x != \"<locals>\", (cls.__module__ + \".\" + cls.__qualname__).split(\".\")))\n\n\nclass HttpUser(User):\n    \"\"\"\n    Represents an HTTP \"user\" which is to be spawned and attack the system that is to be load tested.\n\n    The behaviour of this user is defined by its tasks. Tasks can be declared either directly on the\n    class by using the :py:func:`@task decorator <locust.task>` on methods, or by setting\n    the :py:attr:`tasks attribute <locust.User.tasks>`.\n\n    This class creates a *client* attribute on instantiation which is an HTTP client with support\n    for keeping a user session between requests.\n    \"\"\"\n\n    abstract = True\n    \"\"\"If abstract is True, the class is meant to be subclassed, and users will not choose this locust during a test\"\"\"\n\n    pool_manager: PoolManager | None = None\n    \"\"\"Connection pool manager to use. If not given, a new manager is created per single user.\"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        if self.host is None:\n            raise LocustError(\n                \"You must specify the base host. Either in the host attribute in the User class, or on the command line using the --host option.\"\n            )\n\n        self.client = HttpSession(\n            base_url=self.host,\n            request_event=self.environment.events.request,\n            user=self,\n            pool_manager=self.pool_manager,\n        )\n        \"\"\"\n        Instance of HttpSession that is created upon instantiation of Locust.\n        The client supports cookies, and therefore keeps the session between HTTP requests.\n        \"\"\"\n        self.client.trust_env = False\n\n# Path: locust/user/__init__.py\nfrom .task import TaskSet, tag, task\nfrom .users import HttpUser, User\n\n# Path: locust/user/inspectuser.py\nfrom __future__ import annotations\n\nimport inspect\nfrom collections import defaultdict\nfrom json import dumps\n\nfrom .task import TaskSet\nfrom .users import User\n\n\ndef print_task_ratio(user_classes, num_users, total):\n    \"\"\"\n    This function calculates the task ratio of users based on the user total count.\n    \"\"\"\n    d = get_ratio(user_classes, _calc_distribution(user_classes, num_users), total)\n    _print_task_ratio(d)\n\n\ndef print_task_ratio_json(user_classes, num_users):\n    d = _calc_distribution(user_classes, num_users)\n    task_data = {\n        \"per_class\": get_ratio(user_classes, d, False),\n        \"total\": get_ratio(user_classes, d, True),\n    }\n\n    print(dumps(task_data, indent=4))\n\n\ndef _calc_distribution(user_classes, num_users):\n    fixed_count = sum(u.fixed_count for u in user_classes if u.fixed_count)\n    total_weight = sum(u.weight for u in user_classes if not u.fixed_count)\n    num_users = num_users or (total_weight if not fixed_count else 1)\n    weighted_count = num_users - fixed_count\n    weighted_count = weighted_count if weighted_count > 0 else 0\n    user_classes_count = {}\n\n    for u in user_classes:\n        count = u.fixed_count if u.fixed_count else (u.weight / total_weight) * weighted_count\n        user_classes_count[u.__name__] = round(count)\n\n    return user_classes_count\n\n\ndef _print_task_ratio(x, level=0):\n    padding = 2 * \" \" * level\n    for k, v in x.items():\n        ratio = v.get(\"ratio\", 1)\n        print(\" %-10s %-50s\" % (padding + \"%-6.1f\" % (ratio * 100), padding + k))\n        if \"tasks\" in v:\n            _print_task_ratio(v[\"tasks\"], level + 1)\n\n\ndef get_ratio(user_classes: list[type[User]], user_spawned: dict[str, int], total: bool) -> dict[str, dict[str, float]]:\n    user_count = sum(user_spawned.values()) or 1\n    ratio_percent: dict[type[User], float] = {u: user_spawned.get(u.__name__, 0) / user_count for u in user_classes}\n\n    task_dict: dict[str, dict[str, float]] = {}\n    for u, r in ratio_percent.items():\n        d = {\"ratio\": r}\n        d[\"tasks\"] = _get_task_ratio(u.tasks, total, r)\n        task_dict[u.__name__] = d\n\n    return task_dict\n\n\ndef _get_task_ratio(tasks, total, parent_ratio):\n    parent_ratio = parent_ratio if total else 1.0\n    ratio = defaultdict(int)\n    for task in tasks:\n        ratio[task] += 1\n\n    ratio_percent = {t: r * parent_ratio / len(tasks) for t, r in ratio.items()}\n\n    task_dict = {}\n    for t, r in ratio_percent.items():\n        d = {\"ratio\": r}\n        if inspect.isclass(t) and issubclass(t, TaskSet):\n            d[\"tasks\"] = _get_task_ratio(t.tasks, total, r)\n        task_dict[t.__name__] = d\n\n    return task_dict\n\n# Path: locust/html.py\nimport datetime\nimport glob\nimport os\nimport pathlib\nfrom html import escape\nfrom itertools import chain\nfrom json import dumps\n\nfrom jinja2 import Environment, FileSystemLoader\n\nfrom . import stats as stats_module\nfrom .runners import STATE_STOPPED, STATE_STOPPING, MasterRunner\nfrom .stats import sort_stats\nfrom .user.inspectuser import get_ratio\n\nPERCENTILES_FOR_HTML_REPORT = [0.50, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1.0]\n\n\ndef render_template(file, template_path, **kwargs):\n    env = Environment(loader=FileSystemLoader(template_path), extensions=[\"jinja2.ext.do\"])\n    template = env.get_template(file)\n    return template.render(**kwargs)\n\n\ndef get_html_report(\n    environment,\n    show_download_link=True,\n    use_modern_ui=False,\n    theme=\"\",\n):\n    root_path = os.path.dirname(os.path.abspath(__file__))\n    if use_modern_ui:\n        static_path = os.path.join(root_path, \"webui\", \"dist\", \"assets\")\n        template_path = os.path.join(root_path, \"webui\", \"dist\")\n    else:\n        static_path = os.path.join(root_path, \"static\")\n        template_path = os.path.join(root_path, \"templates\")\n\n    stats = environment.runner.stats\n\n    start_ts = stats.start_time\n    start_time = datetime.datetime.utcfromtimestamp(start_ts).strftime(\"%Y-%m-%d %H:%M:%S\")\n\n    end_ts = stats.last_request_timestamp\n    if end_ts:\n        end_time = datetime.datetime.utcfromtimestamp(end_ts).strftime(\"%Y-%m-%d %H:%M:%S\")\n    else:\n        end_time = start_time\n\n    host = None\n    if environment.host:\n        host = environment.host\n    elif environment.runner.user_classes:\n        all_hosts = {l.host for l in environment.runner.user_classes}\n        if len(all_hosts) == 1:\n            host = list(all_hosts)[0]\n\n    requests_statistics = list(chain(sort_stats(stats.entries), [stats.total]))\n    failures_statistics = sort_stats(stats.errors)\n    exceptions_statistics = [\n        {**exc, \"nodes\": \", \".join(exc[\"nodes\"])} for exc in environment.runner.exceptions.values()\n    ]\n\n    history = stats.history\n\n    static_js = []\n    if use_modern_ui:\n        js_files = [os.path.basename(filepath) for filepath in glob.glob(os.path.join(static_path, \"*.js\"))]\n    else:\n        js_files = [\"jquery-1.11.3.min.js\", \"echarts.common.min.js\", \"vintage.js\", \"chart.js\", \"tasks.js\"]\n\n    for js_file in js_files:\n        path = os.path.join(static_path, js_file)\n        static_js.append(\"// \" + js_file + \"\\n\")\n        with open(path, encoding=\"utf8\") as f:\n            static_js.append(f.read())\n        static_js.extend([\"\", \"\"])\n\n    if not use_modern_ui:\n        static_css = []\n        css_files = [\"tables.css\"]\n        for css_file in css_files:\n            path = os.path.join(static_path, \"css\", css_file)\n            static_css.append(\"/* \" + css_file + \" */\")\n            with open(path, encoding=\"utf8\") as f:\n                static_css.append(f.read())\n            static_css.extend([\"\", \"\"])\n\n    is_distributed = isinstance(environment.runner, MasterRunner)\n    user_spawned = (\n        environment.runner.reported_user_classes_count if is_distributed else environment.runner.user_classes_count\n    )\n\n    if environment.runner.state in [STATE_STOPPED, STATE_STOPPING]:\n        user_spawned = environment.runner.final_user_classes_count\n\n    task_data = {\n        \"per_class\": get_ratio(environment.user_classes, user_spawned, False),\n        \"total\": get_ratio(environment.user_classes, user_spawned, True),\n    }\n\n    if use_modern_ui:\n        res = render_template(\n            \"report.html\",\n            template_path,\n            template_args={\n                \"is_report\": True,\n                \"requests_statistics\": [stat.to_dict(escape_string_values=True) for stat in requests_statistics],\n                \"failures_statistics\": [stat.to_dict() for stat in failures_statistics],\n                \"exceptions_statistics\": [stat for stat in exceptions_statistics],\n                \"response_time_statistics\": [\n                    {\n                        \"name\": escape(stat.name),\n                        \"method\": escape(stat.method or \"\"),\n                        **{\n                            str(percentile): stat.get_response_time_percentile(percentile)\n                            for percentile in PERCENTILES_FOR_HTML_REPORT\n                        },\n                    }\n                    for stat in requests_statistics\n                ],\n                \"start_time\": start_time,\n                \"end_time\": end_time,\n                \"host\": escape(str(host)),\n                \"history\": history,\n                \"show_download_link\": show_download_link,\n                \"locustfile\": escape(str(environment.locustfile)),\n                \"tasks\": task_data,\n                \"percentiles_to_chart\": stats_module.MODERN_UI_PERCENTILES_TO_CHART,\n            },\n            theme=theme,\n            static_js=\"\\n\".join(static_js),\n        )\n    else:\n        res = render_template(\n            \"report.html\",\n            template_path,\n            int=int,\n            round=round,\n            escape=escape,\n            str=str,\n            requests_statistics=requests_statistics,\n            failures_statistics=failures_statistics,\n            exceptions_statistics=exceptions_statistics,\n            start_time=start_time,\n            end_time=end_time,\n            host=host,\n            history=history,\n            static_js=\"\\n\".join(static_js),\n            static_css=\"\\n\".join(static_css),\n            show_download_link=show_download_link,\n            locustfile=environment.locustfile,\n            tasks=dumps(task_data),\n            percentile1=stats_module.PERCENTILES_TO_CHART[0],\n            percentile2=stats_module.PERCENTILES_TO_CHART[1],\n        )\n\n    return res\n\n# Path: locust/util/cache.py\nimport functools\nfrom time import time\n\n\ndef memoize(timeout, dynamic_timeout=False):\n    \"\"\"\n    Memoization decorator with support for timeout.\n\n    If dynamic_timeout is set, the cache timeout is doubled if the cached function\n    takes longer time to run than the timeout time\n    \"\"\"\n    cache = {\"timeout\": timeout}\n\n    def decorator(func):\n        @functools.wraps(func)\n        def wrapper(*args, **kwargs):\n            start = time()\n            if (not \"time\" in cache) or (start - cache[\"time\"] > cache[\"timeout\"]):\n                # cache miss\n                cache[\"result\"] = func(*args, **kwargs)\n                cache[\"time\"] = time()\n                if dynamic_timeout and cache[\"time\"] - start > cache[\"timeout\"]:\n                    cache[\"timeout\"] *= 2\n            return cache[\"result\"]\n\n        def clear_cache():\n            if \"time\" in cache:\n                del cache[\"time\"]\n            if \"result\" in cache:\n                del cache[\"result\"]\n\n        wrapper.clear_cache = clear_cache\n        return wrapper\n\n    return decorator\n\n# Path: locust/util/timespan.py\nimport re\nfrom datetime import timedelta\n\n\ndef parse_timespan(time_str):\n    \"\"\"\n    Parse a string representing a time span and return the number of seconds.\n    Valid formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\n    \"\"\"\n    if not time_str:\n        raise ValueError(\"Invalid time span format\")\n\n    if re.match(r\"^\\d+$\", time_str):\n        # if an int is specified we assume they want seconds\n        return int(time_str)\n\n    timespan_regex = re.compile(r\"((?P<hours>\\d+?)h)?((?P<minutes>\\d+?)m)?((?P<seconds>\\d+?)s)?\")\n    parts = timespan_regex.match(time_str)\n    if not parts:\n        raise ValueError(\"Invalid time span format. Valid formats: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n    parts = parts.groupdict()\n    time_params = {name: int(value) for name, value in parts.items() if value}\n    if not time_params:\n        raise ValueError(\"Invalid time span format. Valid formats: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n    return int(timedelta(**time_params).total_seconds())\n\n# Path: locust/web.py\nfrom __future__ import annotations\n\nimport csv\nimport json\nimport logging\nimport os.path\nfrom functools import wraps\nfrom html import escape\nfrom io import StringIO\nfrom itertools import chain\nfrom json import dumps\nfrom time import time\nfrom typing import TYPE_CHECKING, Any\n\nimport gevent\nfrom flask import (\n    Flask,\n    Response,\n    jsonify,\n    make_response,\n    redirect,\n    render_template,\n    request,\n    send_file,\n    send_from_directory,\n    url_for,\n)\nfrom flask_cors import CORS\nfrom flask_login import LoginManager, login_required\nfrom gevent import pywsgi\n\nfrom . import __version__ as version\nfrom . import argument_parser\nfrom . import stats as stats_module\nfrom .html import get_html_report\nfrom .log import greenlet_exception_logger\nfrom .runners import STATE_MISSING, STATE_RUNNING, MasterRunner\nfrom .stats import StatsCSV, StatsCSVFileWriter, StatsErrorDict, sort_stats\nfrom .user.inspectuser import get_ratio\nfrom .util.cache import memoize\nfrom .util.timespan import parse_timespan\n\nif TYPE_CHECKING:\n    from .env import Environment\n\n\nlogger = logging.getLogger(__name__)\ngreenlet_exception_handler = greenlet_exception_logger(logger)\n\nDEFAULT_CACHE_TIME = 2.0\n\n\nclass WebUI:\n    \"\"\"\n    Sets up and runs a Flask web app that can start and stop load tests using the\n    :attr:`environment.runner <locust.env.Environment.runner>` as well as show the load test statistics\n    in :attr:`environment.stats <locust.env.Environment.stats>`\n    \"\"\"\n\n    app: Flask | None = None\n    \"\"\"\n    Reference to the :class:`flask.Flask` app. Can be used to add additional web routes and customize\n    the Flask app in other various ways. Example::\n\n        from flask import request\n\n        @web_ui.app.route(\"/my_custom_route\")\n        def my_custom_route():\n            return \"your IP is: %s\" % request.remote_addr\n    \"\"\"\n\n    greenlet: gevent.Greenlet | None = None\n    \"\"\"\n    Greenlet of the running web server\n    \"\"\"\n\n    server: pywsgi.WSGIServer | None = None\n    \"\"\"Reference to the :class:`pyqsgi.WSGIServer` instance\"\"\"\n\n    template_args: dict[str, Any]\n    \"\"\"Arguments used to render index.html for the web UI. Must be used with custom templates\n    extending index.html.\"\"\"\n\n    auth_args: dict[str, Any]\n    \"\"\"Arguments used to render auth.html for the web UI auth page. Must be used when configuring auth\"\"\"\n\n    def __init__(\n        self,\n        environment: Environment,\n        host: str,\n        port: int,\n        web_login: bool = False,\n        tls_cert: str | None = None,\n        tls_key: str | None = None,\n        stats_csv_writer: StatsCSV | None = None,\n        delayed_start=False,\n        userclass_picker_is_active=False,\n        modern_ui=False,\n    ):\n        \"\"\"\n        Create WebUI instance and start running the web server in a separate greenlet (self.greenlet)\n\n        Arguments:\n        environment: Reference to the current Locust Environment\n        host: Host/interface that the web server should accept connections to\n        port: Port that the web server should listen to\n        web_login:  Enables a login page for the modern UI\n        tls_cert: A path to a TLS certificate\n        tls_key: A path to a TLS private key\n        delayed_start: Whether or not to delay starting web UI until `start()` is called. Delaying web UI start\n                       allows for adding Flask routes or Blueprints before accepting requests, avoiding errors.\n        \"\"\"\n        environment.web_ui = self\n        self.stats_csv_writer = stats_csv_writer or StatsCSV(environment, stats_module.PERCENTILES_TO_REPORT)\n        self.environment = environment\n        self.host = host\n        self.port = port\n        self.tls_cert = tls_cert\n        self.tls_key = tls_key\n        self.userclass_picker_is_active = userclass_picker_is_active\n        self.modern_ui = modern_ui\n        self.web_login = web_login\n        app = Flask(__name__)\n        CORS(app)\n        self.app = app\n        app.jinja_env.add_extension(\"jinja2.ext.do\")\n        app.debug = True\n        root_path = os.path.dirname(os.path.abspath(__file__))\n        app.root_path = root_path\n        self.webui_build_path = os.path.join(root_path, \"webui\", \"dist\")\n        self.greenlet: gevent.Greenlet | None = None\n        self._swarm_greenlet: gevent.Greenlet | None = None\n        self.template_args = {}\n        self.auth_args = {}\n\n        if self.web_login:\n            self.login_manager = LoginManager()\n            self.login_manager.init_app(app)\n            self.login_manager.login_view = \"login\"\n\n        if environment.runner:\n            self.update_template_args()\n        if not delayed_start:\n            self.start()\n\n        @app.errorhandler(Exception)\n        def handle_exception(error):\n            error_message = str(error)\n            logger.log(logging.CRITICAL, error_message)\n            return make_response(error_message, 500)\n\n        @app.route(\"/assets/<path:path>\")\n        def send_assets(path):\n            return send_from_directory(os.path.join(self.webui_build_path, \"assets\"), path)\n\n        @app.route(\"/\")\n        @self.auth_required_if_enabled\n        def index() -> str | Response:\n            if not environment.runner:\n                return make_response(\"Error: Locust Environment does not have any runner\", 500)\n            self.update_template_args()\n\n            if self.modern_ui:\n                self.set_static_modern_ui()\n\n                return render_template(\"index.html\", template_args=self.template_args)\n            return render_template(\"index.html\", **self.template_args)\n\n        @app.route(\"/swarm\", methods=[\"POST\"])\n        @self.auth_required_if_enabled\n        def swarm() -> Response:\n            assert request.method == \"POST\"\n\n            # Loading UserClasses & ShapeClasses if Locust is running with UserClass Picker\n            if self.userclass_picker_is_active:\n                if not self.environment.available_user_classes:\n                    err_msg = \"UserClass picker is active but there are no available UserClasses\"\n                    return jsonify({\"success\": False, \"message\": err_msg, \"host\": environment.host})\n\n                # Getting Specified User Classes\n                form_data_user_class_names = request.form.getlist(\"user_classes\")\n\n                # Updating UserClasses\n                if form_data_user_class_names:\n                    user_classes = {}\n                    for user_class_name, user_class_object in self.environment.available_user_classes.items():\n                        if user_class_name in form_data_user_class_names:\n                            user_classes[user_class_name] = user_class_object\n\n                else:\n                    if self.environment.runner and self.environment.runner.state == STATE_RUNNING:\n                        # Test is already running\n                        # Using the user classes that have already been selected\n                        user_classes = {\n                            key: value\n                            for (key, value) in self.environment.available_user_classes.items()\n                            if value in self.environment.user_classes\n                        }\n                    else:\n                        # Starting test with no user class selection\n                        # Defaulting to using all available user classes\n                        user_classes = self.environment.available_user_classes\n\n                self._update_user_classes(user_classes)\n\n                # Updating ShapeClass if specified in WebUI Form\n                form_data_shape_class_name = request.form.get(\"shape_class\", \"Default\")\n                if form_data_shape_class_name == \"Default\":\n                    self._update_shape_class(None)\n                else:\n                    self._update_shape_class(form_data_shape_class_name)\n\n            parsed_options_dict = vars(environment.parsed_options) if environment.parsed_options else {}\n            run_time = None\n            for key, value in request.form.items():\n                if key == \"user_count\":  # if we just renamed this field to \"users\" we wouldn't need this\n                    user_count = int(value)\n                elif key == \"spawn_rate\":\n                    spawn_rate = float(value)\n                elif key == \"host\":\n                    # Replace < > to guard against XSS\n                    environment.host = str(request.form[\"host\"]).replace(\"<\", \"\").replace(\">\", \"\")\n                elif key == \"user_classes\":\n                    # Set environment.parsed_options.user_classes to the selected user_classes\n                    parsed_options_dict[key] = request.form.getlist(\"user_classes\")\n                elif key == \"run_time\":\n                    if not value:\n                        continue\n                    try:\n                        run_time = parse_timespan(value)\n                    except ValueError:\n                        err_msg = \"Valid run_time formats are : 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\"\n                        logger.error(err_msg)\n                        return jsonify({\"success\": False, \"message\": err_msg, \"host\": environment.host})\n                elif key in parsed_options_dict:\n                    # update the value in environment.parsed_options, but dont change the type.\n                    parsed_options_value = parsed_options_dict[key]\n\n                    if isinstance(parsed_options_value, bool):\n                        parsed_options_dict[key] = value == \"true\"\n                    elif parsed_options_value is None:\n                        parsed_options_dict[key] = parsed_options_value\n                    else:\n                        parsed_options_dict[key] = type(parsed_options_dict[key])(value)\n\n            if environment.shape_class and environment.runner is not None:\n                environment.runner.start_shape()\n                return jsonify(\n                    {\n                        \"success\": True,\n                        \"message\": f\"Swarming started using shape class '{type(environment.shape_class).__name__}'\",\n                        \"host\": environment.host,\n                    }\n                )\n\n            if self._swarm_greenlet is not None:\n                self._swarm_greenlet.kill(block=True)\n                self._swarm_greenlet = None\n\n            if environment.runner is not None:\n                self._swarm_greenlet = gevent.spawn(environment.runner.start, user_count, spawn_rate)\n                self._swarm_greenlet.link_exception(greenlet_exception_handler)\n                response_data = {\n                    \"success\": True,\n                    \"message\": \"Swarming started\",\n                    \"host\": environment.host,\n                }\n                if run_time:\n                    gevent.spawn_later(run_time, self._stop_runners).link_exception(greenlet_exception_handler)\n                    response_data[\"run_time\"] = run_time\n\n                if self.userclass_picker_is_active:\n                    response_data[\"user_classes\"] = sorted(user_classes.keys())\n\n                return jsonify(response_data)\n            else:\n                return jsonify({\"success\": False, \"message\": \"No runner\", \"host\": environment.host})\n\n        @app.route(\"/stop\")\n        @self.auth_required_if_enabled\n        def stop() -> Response:\n            if self._swarm_greenlet is not None:\n                self._swarm_greenlet.kill(block=True)\n                self._swarm_greenlet = None\n            if environment.runner is not None:\n                environment.runner.stop()\n            return jsonify({\"success\": True, \"message\": \"Test stopped\"})\n\n        @app.route(\"/stats/reset\")\n        @self.auth_required_if_enabled\n        def reset_stats() -> str:\n            environment.events.reset_stats.fire()\n            if environment.runner is not None:\n                environment.runner.stats.reset_all()\n                environment.runner.exceptions = {}\n            return \"ok\"\n\n        @app.route(\"/stats/report\")\n        @self.auth_required_if_enabled\n        def stats_report() -> Response:\n            theme = request.args.get(\"theme\", \"\")\n            res = get_html_report(\n                self.environment,\n                show_download_link=not request.args.get(\"download\"),\n                use_modern_ui=self.modern_ui,\n                theme=theme,\n            )\n            if request.args.get(\"download\"):\n                res = app.make_response(res)\n                res.headers[\"Content-Disposition\"] = f\"attachment;filename=report_{time()}.html\"\n            return res\n\n...\n", "needle_token_start": 2358, "needle_token_end": 2422, "code_context_ntokens": 16064, "output": ["```python\ndef rename_request(self, name: str) -> Generator[None, None, None]:\n    \"\"\"Group requests using the \"with\" keyword\"\"\"\n\n    self.request_name = name\n    try:\n        yield\n    finally:\n        self.request_name = None\n```"]}
{"repo": "locustio/locust", "name": "get_poller", "language": "python", "path": "locust/input_events.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to select and return an appropriate keyboard input polling mechanism based on the operating system the script is running on. It chooses between two different classes that handle keyboard input for Unix-like systems and Windows systems.\n2. **Input**: There are no parameters required for this function.\n3. **Output**: The function returns an instance of a class that is capable of polling keyboard input. The specific class returned depends on the operating system.\n4. **Procedure**: The function checks the name of the operating system. If the operating system is Windows, it returns an instance of the class designed for handling keyboard input on Windows. If the operating system is not Windows, it assumes a Unix-like environment and returns an instance of the class designed for Unix-like systems.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: locust/contrib/fasthttp.py\nfrom __future__ import annotations\n\nfrom locust.env import Environment\nfrom locust.exception import CatchResponseError, LocustError, ResponseError\nfrom locust.user import User\nfrom locust.util.deprecation import DeprecatedFastHttpLocustClass as FastHttpLocust\n\nimport json\nimport json as unshadowed_json  # some methods take a named parameter called json\nimport re\nimport socket\nimport time\nimport traceback\nfrom base64 import b64encode\nfrom contextlib import contextmanager\nfrom http.cookiejar import CookieJar\nfrom json.decoder import JSONDecodeError\nfrom ssl import SSLError\nfrom typing import Any, Callable, Generator, cast\nfrom urllib.parse import urlparse, urlunparse\n\nimport gevent\nfrom charset_normalizer import detect\nfrom gevent.timeout import Timeout\nfrom geventhttpclient._parser import HTTPParseError\nfrom geventhttpclient.client import HTTPClientPool\nfrom geventhttpclient.header import Headers\nfrom geventhttpclient.response import HTTPConnectionClosed, HTTPSocketPoolResponse\nfrom geventhttpclient.useragent import CompatRequest, CompatResponse, ConnectionError, UserAgent\n\n# borrow requests's content-type header parsing\nfrom requests.utils import get_encoding_from_headers\n\n# Monkey patch geventhttpclient.useragent.CompatRequest so that Cookiejar works with Python >= 3.3\n# More info: https://github.com/requests/requests/pull/871\nCompatRequest.unverifiable = False\n\n# Workaround for AttributeError: 'CompatRequest' object has no attribute 'type' in Cookiejar\n# https://github.com/locustio/locust/issues/1138\n# Might allow secure cookies over non-secure connections but that is a minor concern in a load testing tool\nCompatRequest.type = \"https\"\n\n# Regexp for checking if an absolute URL was specified\nabsolute_http_url_regexp = re.compile(r\"^https?://\", re.I)\n\n# List of exceptions that can be raised by geventhttpclient when sending an HTTP request,\n# and that should result in a Locust failure\nFAILURE_EXCEPTIONS = (\n    ConnectionError,\n    ConnectionRefusedError,\n    ConnectionResetError,\n    socket.error,\n    SSLError,\n    Timeout,\n    HTTPConnectionClosed,\n)\n\n\n...\n# Path: locust/debug.py\nfrom __future__ import annotations\n\nimport locust\nimport locust.log\nfrom locust import argument_parser\nfrom locust.env import Environment\nfrom locust.exception import CatchResponseError, RescheduleTask\n\nimport inspect\nimport os\nfrom datetime import datetime, timezone\nfrom typing import TYPE_CHECKING\n\nif TYPE_CHECKING:\n    from locust import User\n\n\ndef _print_t(s):\n    \"\"\"\n    Print something with a tab instead of newline at the end\n    \"\"\"\n    print(str(s), end=\"\\t\")\n\n\nclass PrintListener:\n    \"\"\"\n    Print every response (useful when debugging a single locust)\n    \"\"\"\n\n    def __init__(\n        self,\n        env: Environment,\n        include_length=False,\n        include_time=False,\n        include_context=False,\n        include_payload=False,\n    ):\n        env.events.request.add_listener(self.on_request)\n\n        self.include_length = \"length\\t\" if include_length else \"\"\n        self.include_time = \"time                    \\t\" if include_time else \"\"\n        self.include_context = \"context\\t\" if include_context else \"\"\n        self.include_payload = \"payload\\t\" if include_payload else \"\"\n\n        print(\n            f\"\\n{self.include_time}type\\t{'name'.ljust(50)}\\tresp_ms\\t{self.include_length}exception\\t{self.include_context}\\t{self.include_payload}\"\n        )\n\n    def on_request(\n        self,\n        request_type,\n        name,\n        response_time,\n        response_length,\n        exception,\n        context: dict,\n        start_time=None,\n        response=None,\n        **_kwargs,\n    ):\n        if exception:\n            if isinstance(exception, RescheduleTask):\n                pass\n            if isinstance(exception, CatchResponseError):\n                e = str(exception)\n            else:\n                try:\n                    e = repr(exception)\n                except AttributeError:\n                    e = f\"{exception.__class__} (and it has no string representation)\"\n            errortext = e[:500].replace(\"\\n\", \" \")\n        else:\n            errortext = \"\"\n\n        if response_time is None:\n            response_time = -1\n        n = name.ljust(30) if name else \"\"\n\n        if self.include_time:\n            if start_time:\n                _print_t(datetime.fromtimestamp(start_time, tz=timezone.utc))\n            else:\n                _print_t(datetime.now())\n\n        _print_t(request_type)\n        _print_t(n.ljust(50))\n        _print_t(str(round(response_time)).ljust(7))\n\n        if self.include_length:\n            _print_t(response_length)\n\n        _print_t(errortext.ljust(9))\n\n        if self.include_context:\n            _print_t(context or \"\")\n\n        if self.include_payload:\n            _print_t(response._request.payload)\n\n        print()\n\n\n_env: Environment | None = None  # minimal Environment for debugging\n\n\ndef run_single_user(\n    user_class: type[User],\n    include_length=False,\n    include_time=False,\n    include_context=False,\n    include_payload=False,\n    loglevel: str | None = \"WARNING\",\n):\n    \"\"\"\n    Runs a single User. Useful when you want to run a debugger.\n\n    It creates in a new locust :py:attr:`Environment <locust.env.Environment>` and triggers any ``init`` or ``test_start`` :ref:`events <extending_locust>` as normal.\n\n    It does **not** trigger ``test_stop`` or ``quit`` when you quit the debugger.\n\n    It prints some info about every request to stdout, and you can get additional info using the `include_*` flags\n\n    It also initiates logging on WARNING level (not INFO, because it could interfere with the printing of requests),\n    but you can change that by passing a log level (or disabling logging entirely by passing None)\n    \"\"\"\n    global _env\n\n    if loglevel:\n        locust.log.setup_logging(loglevel)\n\n    if not _env:\n        options = argument_parser.parse_options()\n\n        # in case your test goes looking for the file name of your locustfile\n        frame = inspect.stack()[1]\n        locustfile = os.path.basename(frame[0].f_code.co_filename)\n        options.locustfile = locustfile\n\n        _env = Environment(events=locust.events, locustfile=locustfile, host=options.host, parsed_options=options)\n\n        # log requests to stdout\n        PrintListener(\n            _env,\n            include_length=include_length,\n            include_time=include_time,\n            include_context=include_context,\n            include_payload=include_payload,\n        )\n        # fire various events (quit and test_stop will never get called, sorry about that)\n        _env.events.init.fire(environment=_env, runner=None, web_ui=None)\n        # uncaught events will be suppressed, so check if that happened\n        if locust.log.unhandled_greenlet_exception:\n            raise Exception(\"Unhandled exception in init\")\n\n    # do the things that the Runner usually does\n    _env.user_classes = [user_class]\n    _env._filter_tasks_by_tags()\n    _env.events.test_start.fire(environment=_env)\n    if _env.host:\n        user_class.host = _env.host\n\n    # create a single user\n    user = user_class(_env)\n    setattr(_env, \"single_user_instance\", user)  # if you happen to need access to this from the Environment instance\n    user.run()\n\n# Path: locust/user/sequential_taskset.py\nfrom locust.exception import LocustError\n\nimport logging\n\nfrom .task import TaskSet, TaskSetMeta\n\n\nclass SequentialTaskSetMeta(TaskSetMeta):\n    \"\"\"\n    Meta class for SequentialTaskSet. It's used to allow SequentialTaskSet classes to specify\n    task execution in both a list as the tasks attribute or using the @task decorator\n\n    We use the fact that class_dict order is the order of declaration in Python 3.6\n    (See https://www.python.org/dev/peps/pep-0520/)\n    \"\"\"\n\n    def __new__(mcs, classname, bases, class_dict):\n        new_tasks = []\n        for base in bases:\n            # first get tasks from base classes\n            if hasattr(base, \"tasks\") and base.tasks:\n                new_tasks += base.tasks\n        for key, value in class_dict.items():\n            if key == \"tasks\":\n                # we want to insert tasks from the tasks attribute at the point of it's declaration\n                # compared to methods declared with @task\n                if isinstance(value, list):\n                    new_tasks.extend(value)\n                else:\n                    raise ValueError(\"On SequentialTaskSet the task attribute can only be set to a list\")\n\n            if \"locust_task_weight\" in dir(value):\n                # method decorated with @task\n                for _ in range(value.locust_task_weight):\n                    new_tasks.append(value)\n\n        class_dict[\"tasks\"] = new_tasks\n        return type.__new__(mcs, classname, bases, class_dict)\n\n\nclass SequentialTaskSet(TaskSet, metaclass=SequentialTaskSetMeta):\n    \"\"\"\n    Class defining a sequence of tasks that a User will execute.\n\n    Works like TaskSet, but task weight is ignored, and all tasks are executed in order. Tasks can\n    either be specified by setting the *tasks* attribute to a list of tasks, or by declaring tasks\n    as methods using the @task decorator. The order of declaration decides the order of execution.\n\n    It's possible to combine a task list in the *tasks* attribute, with some tasks declared using\n    the @task decorator. The order of declaration is respected also in that case.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self._task_index = 0\n\n    def get_next_task(self):\n        if not self.tasks:\n            raise LocustError(\n                \"No tasks defined. Use the @task decorator or set the 'tasks' attribute of the SequentialTaskSet\"\n            )\n        task = self.tasks[self._task_index % len(self.tasks)]\n        self._task_index += 1\n        return task\n\n# Path: locust/__init__.py\nimport os\n\nif os.getenv(\"LOCUST_PLAYWRIGHT\", None):\n    print(\"LOCUST_PLAYWRIGHT setting is no longer needed (because locust-plugins no longer installs trio)\")\n    print(\"Uninstall trio package and remove the setting.\")\n    try:\n        # preserve backwards compatibility for now\n        import trio\n    except ModuleNotFoundError:\n        # dont show a massive callstack if trio is not installed\n        os._exit(1)\n\nfrom gevent import monkey\n\nmonkey.patch_all()\n\nfrom ._version import version as __version__\nfrom .contrib.fasthttp import FastHttpUser\nfrom .debug import run_single_user\nfrom .event import Events\nfrom .shape import LoadTestShape\nfrom .user import wait_time\nfrom .user.sequential_taskset import SequentialTaskSet\nfrom .user.task import TaskSet, tag, task\nfrom .user.users import HttpUser, User\nfrom .user.wait_time import between, constant, constant_pacing, constant_throughput\n\nevents = Events()\n\n__all__ = (\n    \"SequentialTaskSet\",\n    \"wait_time\",\n    \"task\",\n    \"tag\",\n    \"TaskSet\",\n    \"HttpUser\",\n    \"FastHttpUser\",\n    \"User\",\n    \"between\",\n    \"constant\",\n    \"constant_pacing\",\n    \"constant_throughput\",\n    \"events\",\n    \"LoadTestShape\",\n    \"run_single_user\",\n)\n\n# Used for raising a DeprecationWarning if old Locust/HttpLocust is used\nfrom .util.deprecation import DeprecatedHttpLocustClass as HttpLocust\nfrom .util.deprecation import DeprecatedLocustClass as Locust\n\n# Path: locust/input_events.py\nfrom __future__ import annotations\n\nimport logging\nimport os\nimport sys\nfrom typing import Callable\n\nimport gevent\n\nif os.name == \"nt\":\n    import pywintypes\n    from win32api import STD_INPUT_HANDLE\n    from win32console import (\n        ENABLE_ECHO_INPUT,\n        ENABLE_LINE_INPUT,\n        ENABLE_PROCESSED_INPUT,\n        KEY_EVENT,\n        GetStdHandle,\n    )\nelse:\n    import select\n    import termios\n    import tty\n\n\nclass InitError(Exception):\n    pass\n\n\nclass UnixKeyPoller:\n    def __init__(self):\n        if sys.stdin.isatty():\n            self.stdin = sys.stdin.fileno()\n            self.tattr = termios.tcgetattr(self.stdin)\n            tty.setcbreak(self.stdin, termios.TCSANOW)\n        else:\n            raise InitError(\"Terminal was not a tty. Keyboard input disabled\")\n\n    def cleanup(self):\n        termios.tcsetattr(self.stdin, termios.TCSANOW, self.tattr)\n\n    def poll(_self):\n        dr, dw, de = select.select([sys.stdin], [], [], 0)\n        if not dr == []:\n            return sys.stdin.read(1)\n        return None\n\n\nclass WindowsKeyPoller:\n    def __init__(self):\n        if sys.stdin.isatty():\n            try:\n                self.read_handle = GetStdHandle(STD_INPUT_HANDLE)\n                self.read_handle.SetConsoleMode(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT)\n                self.cur_event_length = 0\n                self.cur_keys_length = 0\n                self.captured_chars = []\n            except pywintypes.error:\n                raise InitError(\"Terminal says its a tty but we couldnt enable line input. Keyboard input disabled.\")\n        else:\n            raise InitError(\"Terminal was not a tty. Keyboard input disabled\")\n\n    def cleanup(self):\n        pass\n\n    def poll(self):\n        if self.captured_chars:\n            return self.captured_chars.pop(0)\n\n        events_peek = self.read_handle.PeekConsoleInput(10000)\n\n        if not events_peek:\n            return None\n\n        if not len(events_peek) == self.cur_event_length:\n            for cur_event in events_peek[self.cur_event_length :]:\n                if cur_event.EventType == KEY_EVENT:\n                    if ord(cur_event.Char) and cur_event.KeyDown:\n                        cur_char = str(cur_event.Char)\n                        self.captured_chars.append(cur_char)\n\n            self.cur_event_length = len(events_peek)\n\n        if self.captured_chars:\n            return self.captured_chars.pop(0)\n        else:\n            return None\n\n\n\ndef get_poller():\n    if os.name == \"nt\":\n        return WindowsKeyPoller()\n    else:\n        return UnixKeyPoller()\n\n\ndef input_listener(key_to_func_map: dict[str, Callable]):\n    def input_listener_func():\n        try:\n            poller = get_poller()\n        except InitError as e:\n            logging.debug(e)\n            return\n\n        try:\n            while True:\n                input = poller.poll()\n                if input:\n                    for key in key_to_func_map:\n                        if input == key:\n                            key_to_func_map[key]()\n                else:\n                    gevent.sleep(0.2)\n        except Exception as e:\n            logging.warning(f\"Exception in keyboard input poller: {e}\")\n        finally:\n            poller.cleanup()\n\n    return input_listener_func\n\n# Path: locust/util/load_locustfile.py\nfrom __future__ import annotations\n\nimport importlib\nimport importlib.util\nimport inspect\nimport os\nimport sys\n\nfrom ..shape import LoadTestShape\nfrom ..user import User\n\n\ndef is_user_class(item):\n    \"\"\"\n    Check if a variable is a runnable (non-abstract) User class\n    \"\"\"\n    return bool(inspect.isclass(item) and issubclass(item, User) and item.abstract is False)\n\n\ndef is_shape_class(item):\n    \"\"\"\n    Check if a class is a LoadTestShape\n    \"\"\"\n    return bool(inspect.isclass(item) and issubclass(item, LoadTestShape) and not getattr(item, \"abstract\", True))\n\n\ndef load_locustfile(path) -> tuple[str | None, dict[str, User], list[LoadTestShape]]:\n    \"\"\"\n    Import given locustfile path and return (docstring, callables).\n\n    Specifically, the locustfile's ``__doc__`` attribute (a string) and a\n    dictionary of ``{'name': callable}`` containing all callables which pass\n    the \"is a Locust\" test.\n    \"\"\"\n\n    # Start with making sure the current working dir is in the sys.path\n    sys.path.insert(0, os.getcwd())\n    # Get directory and locustfile name\n    directory, locustfile = os.path.split(path)\n    # If the directory isn't in the PYTHONPATH, add it so our import will work\n    added_to_path = False\n    index = None\n    if directory not in sys.path:\n        sys.path.insert(0, directory)\n        added_to_path = True\n    # If the directory IS in the PYTHONPATH, move it to the front temporarily,\n    # otherwise other locustfiles -- like Locusts's own -- may scoop the intended\n    # one.\n    else:\n        i = sys.path.index(directory)\n        if i != 0:\n            # Store index for later restoration\n            index = i\n            # Add to front, then remove from original position\n            sys.path.insert(0, directory)\n            del sys.path[i + 1]\n\n    # Perform the import\n    module_name = os.path.splitext(locustfile)[0]\n    if module_name == \"locust\":\n        module_name = \"locustfile\"  # Avoid conflict with locust package\n    loader = importlib.machinery.SourceFileLoader(module_name, path)\n    spec = importlib.util.spec_from_file_location(module_name, path, loader=loader)\n    if spec is None:\n        sys.stderr.write(f\"Unable to get module spec for {module_name} in {path}\")\n        sys.exit(1)\n\n    imported = importlib.util.module_from_spec(spec)\n    sys.modules[imported.__name__] = imported\n    loader.exec_module(imported)\n\n    # Remove directory from path if we added it ourselves (just to be neat)\n    if added_to_path:\n        del sys.path[0]\n    # Put back in original index if we moved it\n    if index is not None:\n        sys.path.insert(index + 1, directory)\n        del sys.path[0]\n    # Return our two-tuple\n    user_classes = {name: value for name, value in vars(imported).items() if is_user_class(value)}\n\n    # Find shape class, if any, return it\n    shape_classes = [value() for value in vars(imported).values() if is_shape_class(value)]\n\n    return imported.__doc__, user_classes, shape_classes\n\n# Path: locust/main.py\nfrom __future__ import annotations\n\nimport locust\n\nimport atexit\nimport errno\nimport gc\nimport inspect\nimport json\nimport logging\nimport os\nimport signal\nimport sys\nimport time\nimport traceback\n\nimport gevent\n\nfrom . import log, stats\nfrom .argument_parser import parse_locustfile_option, parse_options\nfrom .env import Environment\nfrom .html import get_html_report\nfrom .input_events import input_listener\nfrom .log import greenlet_exception_logger, setup_logging\nfrom .stats import (\n    StatsCSV,\n    StatsCSVFileWriter,\n    print_error_report,\n    print_percentile_stats,\n    print_stats,\n    print_stats_json,\n    stats_history,\n    stats_printer,\n)\nfrom .user.inspectuser import print_task_ratio, print_task_ratio_json\nfrom .util.load_locustfile import load_locustfile\nfrom .util.timespan import parse_timespan\n\ntry:\n    # import locust_plugins if it is installed, to allow it to register custom arguments etc\n    import locust_plugins  # pyright: ignore[reportMissingImports]\nexcept ModuleNotFoundError:\n    pass\n\nversion = locust.__version__\n\n# Options to ignore when using a custom shape class without `use_common_options=True`\n# See: https://docs.locust.io/en/stable/custom-load-shape.html#use-common-options\nCOMMON_OPTIONS = {\n    \"num_users\": \"users\",\n    \"spawn_rate\": \"spawn-rate\",\n    \"run_time\": \"run-time\",\n}\n\n\ndef create_environment(\n    user_classes,\n    options,\n    events=None,\n    shape_class=None,\n    locustfile=None,\n    available_user_classes=None,\n    available_shape_classes=None,\n    available_user_tasks=None,\n):\n    \"\"\"\n    Create an Environment instance from options\n    \"\"\"\n    return Environment(\n        locustfile=locustfile,\n        user_classes=user_classes,\n        shape_class=shape_class,\n        events=events,\n        host=options.host,\n        reset_stats=options.reset_stats,\n        parsed_options=options,\n        available_user_classes=available_user_classes,\n        available_shape_classes=available_shape_classes,\n        available_user_tasks=available_user_tasks,\n    )\n\n\ndef main():\n    # find specified locustfile(s) and make sure it exists, using a very simplified\n    # command line parser that is only used to parse the -f option.\n    locustfiles = parse_locustfile_option()\n    locustfiles_length = len(locustfiles)\n\n    # Grabbing the Locustfile if only one was provided. Otherwise, allowing users to select the locustfile in the UI\n    # If --headless or --autostart and multiple locustfiles, all provided UserClasses will be ran\n    locustfile = locustfiles[0] if locustfiles_length == 1 else None\n\n    # Importing Locustfile(s) - setting available UserClasses and ShapeClasses to choose from in UI\n    user_classes: dict[str, locust.User] = {}\n    available_user_classes = {}\n    available_shape_classes = {}\n    available_user_tasks = {}\n    shape_class = None\n    for _locustfile in locustfiles:\n        docstring, _user_classes, shape_classes = load_locustfile(_locustfile)\n\n        # Setting Available Shape Classes\n        if shape_classes:\n            shape_class = shape_classes[0]\n            for shape_class in shape_classes:\n                shape_class_name = type(shape_class).__name__\n                if shape_class_name in available_shape_classes.keys():\n                    sys.stderr.write(f\"Duplicate shape classes: {shape_class_name}\\n\")\n                    sys.exit(1)\n\n                available_shape_classes[shape_class_name] = shape_class\n\n        # Setting Available User Classes\n        for key, value in _user_classes.items():\n            if key in available_user_classes.keys():\n                previous_path = inspect.getfile(user_classes[key])\n                new_path = inspect.getfile(value)\n                if previous_path == new_path:\n                    # The same User class was defined in two locustfiles but one probably imported the other, so we just ignore it\n                    continue\n                else:\n                    sys.stderr.write(\n                        f\"Duplicate user class names: {key} is defined in both {previous_path} and {new_path}\\n\"\n                    )\n                    sys.exit(1)\n\n            user_classes[key] = value\n            available_user_classes[key] = value\n            available_user_tasks[key] = value.tasks or None\n\n    if len(stats.PERCENTILES_TO_CHART) != 2:\n        logging.error(\"stats.PERCENTILES_TO_CHART parameter should be 2 parameters \\n\")\n        sys.exit(1)\n\n    if len(stats.MODERN_UI_PERCENTILES_TO_CHART) > 6:\n        logging.error(\"stats.MODERN_UI_PERCENTILES_TO_CHART parameter should be a maximum of 6 parameters \\n\")\n        sys.exit(1)\n\n    def is_valid_percentile(parameter):\n        try:\n            if 0 < float(parameter) < 1:\n                return True\n            return False\n        except ValueError:\n            return False\n\n    for percentile in stats.PERCENTILES_TO_CHART:\n        if not is_valid_percentile(percentile):\n            logging.error(\n                \"stats.PERCENTILES_TO_CHART parameter need to be float and value between. 0 < percentile < 1 Eg 0.95\\n\"\n            )\n            sys.exit(1)\n\n    for percentile in stats.PERCENTILES_TO_STATISTICS:\n        if not is_valid_percentile(percentile):\n            logging.error(\n                \"stats.PERCENTILES_TO_STATISTICS parameter need to be float and value between. 0 < percentile < 1 Eg 0.95\\n\"\n            )\n            sys.exit(1)\n\n    # parse all command line options\n    options = parse_options()\n\n    if options.headful:\n        options.headless = False\n\n    if options.slave or options.expect_slaves:\n        sys.stderr.write(\"The --slave/--expect-slaves parameters have been renamed --worker/--expect-workers\\n\")\n        sys.exit(1)\n\n    if options.web_auth:\n        sys.stderr.write(\n            \"The --web-auth parameters has been replaced with --web-login. See https://docs.locust.io/en/stable/extending-locust.html#authentication for details\\n\"\n        )\n        sys.exit(1)\n\n    if options.autoquit != -1 and not options.autostart:\n        sys.stderr.write(\"--autoquit is only meaningful in combination with --autostart\\n\")\n        sys.exit(1)\n\n    if options.hatch_rate:\n        sys.stderr.write(\"[DEPRECATED] The --hatch-rate parameter has been renamed --spawn-rate\\n\")\n        options.spawn_rate = options.hatch_rate\n\n    # setup logging\n    if not options.skip_log_setup:\n        if options.loglevel.upper() in [\"DEBUG\", \"INFO\", \"WARNING\", \"ERROR\", \"CRITICAL\"]:\n            setup_logging(options.loglevel, options.logfile)\n        else:\n            sys.stderr.write(\"Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\\n\")\n            sys.exit(1)\n\n    children = []\n\n    if options.processes:\n        if os.name == \"nt\":\n            sys.stderr.write(\"--processes is not supported in Windows (except in WSL)\\n\")\n            sys.exit(1)\n        if options.processes == -1:\n            options.processes = os.cpu_count()\n            if not options.processes:\n                sys.stderr.write(\"--processes failed to detect number of cpus!?\\n\")\n                sys.exit(1)\n        elif options.processes < -1:\n            sys.stderr.write(f\"Invalid --processes count {options.processes}\\n\")\n            sys.exit(1)\n        elif options.master:\n            sys.stderr.write(\n                \"--master cannot be combined with --processes. Remove --master, as it is implicit as long as --worker is not set.\\n\"\n            )\n            sys.exit(1)\n        # Optimize copy-on-write-behavior to save some memory (aprx 26MB -> 15MB rss) in child processes\n        gc.collect()  # avoid freezing garbage\n        gc.freeze()  # move all objects to perm gen so ref counts dont get updated\n        for _ in range(options.processes):\n            child_pid = gevent.fork()\n            if child_pid:\n                children.append(child_pid)\n                logging.debug(f\"Started child worker with pid #{child_pid}\")\n            else:\n                # child is always a worker, even when it wasnt set on command line\n                options.worker = True\n                # remove options that dont make sense on worker\n                options.run_time = None\n                options.autostart = None\n                break\n        else:\n            # we're in the parent process\n            if options.worker:\n                # ignore the first sigint in parent, and wait for the children to handle sigint\n                def sigint_handler(_signal, _frame):\n                    if getattr(sigint_handler, \"has_run\", False):\n                        # if parent gets repeated sigint, we kill the children hard\n                        for child_pid in children:\n                            try:\n                                logging.debug(f\"Sending SIGKILL to child with pid {child_pid}\")\n                                os.kill(child_pid, signal.SIGKILL)\n                            except ProcessLookupError:\n                                pass  # process already dead\n                            except Exception:\n                                logging.error(traceback.format_exc())\n                        sys.exit(1)\n                    sigint_handler.has_run = True\n\n                signal.signal(signal.SIGINT, sigint_handler)\n                exit_code = 0\n                # nothing more to do, just wait for the children to exit\n                for child_pid in children:\n                    _, child_status = os.waitpid(child_pid, 0)\n                    try:\n                        if sys.version_info >= (3, 9):\n                            child_exit_code = os.waitstatus_to_exitcode(child_status)\n                            exit_code = max(exit_code, child_exit_code)\n                    except AttributeError:\n                        pass  # dammit python 3.8...\n                sys.exit(exit_code)\n            else:\n                options.master = True\n                options.expect_workers = options.processes\n\n                def kill_workers(children):\n                    exit_code = 0\n                    start_time = time.time()\n                    # give children some time to finish up (in case they had an error parsing arguments etc)\n                    for child_pid in children[:]:\n                        while time.time() < start_time + 3:\n                            try:\n                                _, child_status = os.waitpid(child_pid, os.WNOHANG)\n                                children.remove(child_pid)\n                                try:\n                                    if sys.version_info >= (3, 9):\n                                        child_exit_code = os.waitstatus_to_exitcode(child_status)\n                                        exit_code = max(exit_code, child_exit_code)\n                                except AttributeError:\n                                    pass  # dammit python 3.8...\n                            except OSError as e:\n                                if e.errno == errno.EINTR:\n                                    time.sleep(0.1)\n                                else:\n                                    logging.error(traceback.format_exc())\n                            else:\n                                break\n                    for child_pid in children:\n                        try:\n                            logging.debug(f\"Sending SIGINT to child with pid {child_pid}\")\n                            os.kill(child_pid, signal.SIGINT)\n                        except ProcessLookupError:\n                            pass  # never mind, process was already dead\n                    for child_pid in children:\n                        _, child_status = os.waitpid(child_pid, 0)\n                        try:\n                            if sys.version_info >= (3, 9):\n                                child_exit_code = os.waitstatus_to_exitcode(child_status)\n                                exit_code = max(exit_code, child_exit_code)\n                        except AttributeError:\n                            pass  # dammit python 3.8...\n                    if exit_code > 1:\n                        logging.error(f\"Bad response code from worker children: {exit_code}\")\n                    # ensure master doesnt finish until output from workers has arrived\n                    # otherwise the terminal might look weird.\n                    time.sleep(0.1)\n\n                atexit.register(kill_workers, children)\n\n    logger = logging.getLogger(__name__)\n    greenlet_exception_handler = greenlet_exception_logger(logger)\n\n    if options.stop_timeout:\n        try:\n            options.stop_timeout = parse_timespan(options.stop_timeout)\n        except ValueError:\n            logger.error(\"Valid --stop-timeout formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n            sys.exit(1)\n\n    if options.list_commands:\n        print(\"Available Users:\")\n        for name in user_classes:\n            print(\"    \" + name)\n        sys.exit(0)\n\n    if not user_classes:\n        logger.error(\"No User class found!\")\n        sys.exit(1)\n\n    # make sure specified User exists\n    if options.user_classes:\n        missing = set(options.user_classes) - set(user_classes.keys())\n        if missing:\n            logger.error(f\"Unknown User(s): {', '.join(missing)}\\n\")\n            sys.exit(1)\n        else:\n            names = set(options.user_classes) & set(user_classes.keys())\n            user_classes = [user_classes[n] for n in names]\n    else:\n        # list() call is needed to consume the dict_view object in Python 3\n        user_classes = list(user_classes.values())\n\n    if os.name != \"nt\" and not options.master:\n        try:\n            import resource\n\n            minimum_open_file_limit = 10000\n            (soft_limit, hard_limit) = resource.getrlimit(resource.RLIMIT_NOFILE)\n\n            if soft_limit < minimum_open_file_limit:\n                # Increasing the limit to 10000 within a running process should work on at least MacOS.\n                # It does not work on all OS:es, but we should be no worse off for trying.\n                resource.setrlimit(resource.RLIMIT_NOFILE, [minimum_open_file_limit, hard_limit])\n        except BaseException:\n            logger.warning(\n                f\"\"\"System open file limit '{soft_limit} is below minimum setting '{minimum_open_file_limit}'.\nIt's not high enough for load testing, and the OS didn't allow locust to increase it by itself.\nSee https://github.com/locustio/locust/wiki/Installation#increasing-maximum-number-of-open-files-limit for more info.\"\"\"\n            )\n\n    if sys.version_info < (3, 9):\n        logger.warning(\"Python 3.8 support is deprecated and will be removed soon\")\n\n    # create locust Environment\n    locustfile_path = None if not locustfile else os.path.basename(locustfile)\n\n    environment = create_environment(\n        user_classes,\n        options,\n        events=locust.events,\n        shape_class=shape_class,\n        locustfile=locustfile_path,\n        available_user_classes=available_user_classes,\n        available_shape_classes=available_shape_classes,\n        available_user_tasks=available_user_tasks,\n    )\n\n    if options.config_users:\n        for json_user_config in options.config_users:\n            try:\n                if json_user_config.endswith(\".json\"):\n                    with open(json_user_config) as file:\n                        user_config = json.load(file)\n                else:\n                    user_config = json.loads(json_user_config)\n\n                def ensure_user_class_name(config):\n                    if \"user_class_name\" not in config:\n                        logger.error(\"The user config must specify a user_class_name\")\n                        sys.exit(-1)\n\n                if isinstance(user_config, list):\n                    for config in user_config:\n                        ensure_user_class_name(config)\n\n                        environment.update_user_class(config)\n                else:\n                    ensure_user_class_name(user_config)\n\n                    environment.update_user_class(user_config)\n            except Exception as e:\n                logger.error(f\"The --config-users arugment must be in valid JSON string or file: {e}\")\n                sys.exit(-1)\n\n    if (\n        shape_class\n        and not shape_class.use_common_options\n        and any(getattr(options, opt, None) for opt in COMMON_OPTIONS)\n    ):\n        logger.warning(\n            \"--run-time, --users or --spawn-rate have no impact on LoadShapes unless the shape class explicitly reads them. \"\n            \"See: docs.locust.io/en/stable/custom-load-shape.html#use-common-options\"\n        )\n        ignored = [f\"--{arg}\" for opt, arg in COMMON_OPTIONS.items() if getattr(options, opt, None)]\n        logger.warning(f\"The following option(s) will be ignored: {', '.join(ignored)}\")\n\n    if options.show_task_ratio:\n        print(\"\\n Task ratio per User class\")\n        print(\"-\" * 80)\n        print_task_ratio(user_classes, options.num_users, False)\n        print(\"\\n Total task ratio\")\n        print(\"-\" * 80)\n        print_task_ratio(user_classes, options.num_users, True)\n        sys.exit(0)\n    if options.show_task_ratio_json:\n        print_task_ratio_json(user_classes, options.num_users)\n        sys.exit(0)\n\n    if options.master:\n        if options.worker:\n            logger.error(\"The --master argument cannot be combined with --worker\")\n            sys.exit(-1)\n        if options.expect_workers_max_wait and not options.expect_workers:\n            logger.error(\"The --expect-workers-max-wait argument only makes sense when combined with --expect-workers\")\n            sys.exit(-1)\n        runner = environment.create_master_runner(\n            master_bind_host=options.master_bind_host,\n            master_bind_port=options.master_bind_port,\n        )\n    elif options.worker:\n        try:\n            runner = environment.create_worker_runner(options.master_host, options.master_port)\n            logger.debug(\"Connected to locust master: %s:%s\", options.master_host, options.master_port)\n        except OSError as e:\n            logger.error(\"Failed to connect to the Locust master: %s\", e)\n            sys.exit(-1)\n    else:\n        runner = environment.create_local_runner()\n\n    # main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode\n    main_greenlet = runner.greenlet\n\n    if options.run_time:\n        if options.worker:\n            logger.error(\"--run-time should be specified on the master node, and not on worker nodes\")\n            sys.exit(1)\n        try:\n            options.run_time = parse_timespan(options.run_time)\n        except ValueError:\n            logger.error(\"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n            sys.exit(1)\n\n    if options.csv_prefix:\n        base_csv_file = os.path.basename(options.csv_prefix)\n        base_csv_dir = options.csv_prefix[: -len(base_csv_file)]\n        if not os.path.exists(base_csv_dir) and len(base_csv_dir) != 0:\n            os.makedirs(base_csv_dir)\n        stats_csv_writer = StatsCSVFileWriter(\n            environment, stats.PERCENTILES_TO_REPORT, options.csv_prefix, options.stats_history_enabled\n        )\n    else:\n        stats_csv_writer = StatsCSV(environment, stats.PERCENTILES_TO_REPORT)\n\n    # start Web UI\n    if not options.headless and not options.worker:\n        # spawn web greenlet\n        protocol = \"https\" if options.tls_cert and options.tls_key else \"http\"\n\n        if options.web_host == \"*\":\n            # special check for \"*\" so that we're consistent with --master-bind-host\n            web_host = \"\"\n        else:\n            web_host = options.web_host\n        if web_host:\n            logger.info(f\"Starting web interface at {protocol}://{web_host}:{options.web_port}\")\n        else:\n            if os.name == \"nt\":\n                logger.info(\n                    f\"Starting web interface at {protocol}://localhost:{options.web_port} (accepting connections from all network interfaces)\"\n                )\n            else:\n                logger.info(f\"Starting web interface at {protocol}://0.0.0.0:{options.web_port}\")\n\n        web_ui = environment.create_web_ui(\n            host=web_host,\n            port=options.web_port,\n            web_login=options.web_login,\n            tls_cert=options.tls_cert,\n            tls_key=options.tls_key,\n            stats_csv_writer=stats_csv_writer,\n            delayed_start=True,\n            userclass_picker_is_active=options.class_picker,\n            modern_ui=not options.legacy_ui,\n        )\n    else:\n        web_ui = None\n\n    if options.autostart and options.headless:\n        logger.warning(\"The --autostart argument is implied by --headless, no need to set both.\")\n\n    if options.autostart and options.worker:\n        logger.warning(\"The --autostart argument has no meaning on a worker.\")\n\n    def assign_equal_weights(environment, **kwargs):\n        environment.assign_equal_weights()\n\n    if options.equal_weights:\n        environment.events.init.add_listener(assign_equal_weights)\n\n    # Fire locust init event which can be used by end-users' code to run setup code that\n    # need access to the Environment, Runner or WebUI.\n    environment.events.init.fire(environment=environment, runner=runner, web_ui=web_ui)\n\n    if web_ui:\n        web_ui.start()\n        main_greenlet = web_ui.greenlet\n\n    def stop_and_optionally_quit():\n        if options.autostart and not options.headless:\n            logger.info(\"--run-time limit reached, stopping test\")\n            runner.stop()\n            if options.autoquit != -1:\n                logger.debug(\"Autoquit time limit set to %s seconds\" % options.autoquit)\n                time.sleep(options.autoquit)\n                logger.info(\"--autoquit time reached, shutting down\")\n                runner.quit()\n                if web_ui:\n                    web_ui.stop()\n            else:\n                logger.info(\"--autoquit not specified, leaving web ui running indefinitely\")\n        else:  # --headless run\n            logger.info(\"--run-time limit reached, shutting down\")\n            runner.quit()\n\n    def spawn_run_time_quit_greenlet():\n        gevent.spawn_later(options.run_time, stop_and_optionally_quit).link_exception(greenlet_exception_handler)\n\n    headless_master_greenlet = None\n    stats_printer_greenlet = None\n    if not options.only_summary and (options.print_stats or (options.headless and not options.worker)):\n        # spawn stats printing greenlet\n        stats_printer_greenlet = gevent.spawn(stats_printer(runner.stats))\n        stats_printer_greenlet.link_exception(greenlet_exception_handler)\n\n    gevent.spawn(stats_history, runner)\n\n    def start_automatic_run():\n        if options.master:\n            # wait for worker nodes to connect\n            start_time = time.monotonic()\n            while len(runner.clients.ready) < options.expect_workers:\n                if options.expect_workers_max_wait and options.expect_workers_max_wait < time.monotonic() - start_time:\n                    logger.error(\"Gave up waiting for workers to connect\")\n                    runner.quit()\n                    sys.exit(1)\n                logging.info(\n                    \"Waiting for workers to be ready, %s of %s connected\",\n                    len(runner.clients.ready),\n                    options.expect_workers,\n                )\n                # TODO: Handle KeyboardInterrupt and send quit signal to workers that are started.\n                #       Right now, if the user sends a ctrl+c, the master will not gracefully\n                #       shutdown resulting in all the already started workers to stay active.\n                time.sleep(1)\n        if not options.worker:\n            # apply headless mode defaults\n            if options.num_users is None:\n                options.num_users = 1\n            if options.spawn_rate is None:\n                options.spawn_rate = 1\n\n            # start the test\n            if environment.shape_class:\n                try:\n                    environment.runner.start_shape()\n                    environment.runner.shape_greenlet.join()\n                except KeyboardInterrupt:\n                    logging.info(\"Exiting due to CTRL+C interruption\")\n                finally:\n                    stop_and_optionally_quit()\n            else:\n                headless_master_greenlet = gevent.spawn(runner.start, options.num_users, options.spawn_rate)\n                headless_master_greenlet.link_exception(greenlet_exception_handler)\n\n        if options.run_time:\n            logger.info(f\"Run time limit set to {options.run_time} seconds\")\n            spawn_run_time_quit_greenlet()\n        elif not options.worker and not environment.shape_class:\n            logger.info(\"No run time limit set, use CTRL+C to interrupt\")\n\n    if options.csv_prefix:\n        gevent.spawn(stats_csv_writer.stats_writer).link_exception(greenlet_exception_handler)\n\n    if options.headless:\n        start_automatic_run()\n\n    input_listener_greenlet = None\n    if not options.worker:\n        # spawn input listener greenlet\n        input_listener_greenlet = gevent.spawn(\n            input_listener(\n                {\n                    \"w\": lambda: runner.start(runner.user_count + 1, 100)\n                    if runner.state != \"spawning\"\n                    else logging.warning(\"Already spawning users, can't spawn more right now\"),\n                    \"W\": lambda: runner.start(runner.user_count + 10, 100)\n                    if runner.state != \"spawning\"\n                    else logging.warning(\"Already spawning users, can't spawn more right now\"),\n                    \"s\": lambda: runner.start(max(0, runner.user_count - 1), 100)\n                    if runner.state != \"spawning\"\n                    else logging.warning(\"Spawning users, can't stop right now\"),\n                    \"S\": lambda: runner.start(max(0, runner.user_count - 10), 100)\n                    if runner.state != \"spawning\"\n                    else logging.warning(\"Spawning users, can't stop right now\"),\n                },\n            )\n        )\n        input_listener_greenlet.link_exception(greenlet_exception_handler)\n        # ensure terminal is reset, even if there is an unhandled exception in locust or someone\n        # does something wild, like calling sys.exit() in the locustfile\n        atexit.register(input_listener_greenlet.kill, block=True)\n\n    def shutdown():\n        \"\"\"\n        Shut down locust by firing quitting event, printing/writing stats and exiting\n        \"\"\"\n        logger.debug(\"Running teardowns...\")\n\n        if input_listener_greenlet is not None:\n            input_listener_greenlet.kill(block=False)\n\n        environment.events.quitting.fire(environment=environment, reverse=True)\n\n        # determine the process exit code\n        if environment.process_exit_code is not None:\n            code = environment.process_exit_code\n        elif len(runner.errors) or len(runner.exceptions):\n            code = options.exit_code_on_error\n        elif log.unhandled_greenlet_exception:\n            code = 2\n        else:\n            code = 0\n\n        logger.info(f\"Shutting down (exit code {code})\")\n\n        if stats_printer_greenlet is not None:\n            stats_printer_greenlet.kill(block=False)\n        if headless_master_greenlet is not None:\n            headless_master_greenlet.kill(block=False)\n        logger.debug(\"Cleaning up runner...\")\n        if runner is not None:\n            runner.quit()\n        if options.json:\n            print_stats_json(runner.stats)\n        elif not isinstance(runner, locust.runners.WorkerRunner):\n            print_stats(runner.stats, current=False)\n            print_percentile_stats(runner.stats)\n            print_error_report(runner.stats)\n        environment.events.quit.fire(exit_code=code)\n        sys.exit(code)\n\n    # install SIGTERM handler\n    def sig_term_handler():\n        logger.info(\"Got SIGTERM signal\")\n        shutdown()\n\n    def save_html_report(use_modern_ui=False):\n        html_report = get_html_report(environment, show_download_link=False, use_modern_ui=use_modern_ui)\n        logger.info(\"writing html report to file: %s\", options.html_file)\n        with open(options.html_file, \"w\", encoding=\"utf-8\") as file:\n            file.write(html_report)\n\n    gevent.signal_handler(signal.SIGTERM, sig_term_handler)\n\n    try:\n        logger.info(f\"Starting Locust {version}\")\n        if options.class_picker:\n            logger.info(\"Locust is running with the UserClass Picker Enabled\")\n        if options.autostart and not options.headless:\n            start_automatic_run()\n\n        main_greenlet.join()\n        if options.html_file:\n            save_html_report(not options.legacy_ui)\n    except KeyboardInterrupt:\n        if options.html_file:\n            save_html_report(not options.legacy_ui)\n    except Exception:\n        raise\n    shutdown()\n\n# Path: locust/__main__.py\nfrom .main import main\n\nmain()\n\n# Path: locust/test/fake_module1_for_env_test.py\n\"\"\"Module for locust.test.test_env.TestEnvironment.test_user_classes_with_same_name_is_error\"\"\"\n\nfrom locust import User\n\n\nclass MyUserWithSameName(User):\n    pass\n\n# Path: locust/test/fake_module2_for_env_test.py\n\"\"\"Module for locust.test.test_env.TestEnvironment.test_user_classes_with_same_name_is_error\"\"\"\n\nfrom locust import User\n\n\nclass MyUserWithSameName(User):\n    pass\n\n# Path: locust/test/__init__.py\ntry:\n    import resource\n\n    # work around occasional \"zmq.error.ZMQError: Too many open files\"\n    # this is done in main.py when running locust proper so we need to do it here as well\n    resource.setrlimit(\n        resource.RLIMIT_NOFILE,\n        (\n            10000,\n            resource.RLIM_INFINITY,\n        ),\n    )\n    changed_rlimit = True\nexcept Exception:\n    changed_rlimit = False\n\n# Path: locust/test/mock_logging.py\nfrom __future__ import annotations\n\nimport logging\n\nfrom typing import List, Union, Dict\nfrom types import TracebackType\n\nLogMessage = List[Union[str, Dict[str, TracebackType]]]\n\n\nclass MockedLoggingHandler(logging.Handler):\n    debug: list[LogMessage] = []\n    warning: list[LogMessage] = []\n    info: list[LogMessage] = []\n    error: list[LogMessage] = []\n    critical: list[LogMessage] = []\n\n    def emit(self, record):\n        if record.exc_info:\n            value = {\"message\": record.getMessage(), \"exc_info\": record.exc_info}\n        else:\n            value = record.getMessage()\n        getattr(self.__class__, record.levelname.lower()).append(value)\n\n    @classmethod\n    def reset(cls):\n        for attr in dir(cls):\n            if isinstance(getattr(cls, attr), list):\n                setattr(cls, attr, [])\n\n# Path: locust/test/util.py\nimport datetime\nimport functools\nimport gc\nimport os\nimport socket\nimport warnings\nfrom contextlib import contextmanager\nfrom tempfile import NamedTemporaryFile\n\nfrom cryptography import x509\nfrom cryptography.hazmat.backends import default_backend\nfrom cryptography.hazmat.primitives import hashes, serialization\nfrom cryptography.hazmat.primitives.asymmetric import rsa\nfrom cryptography.x509.oid import NameOID\n\n\n@contextmanager\ndef temporary_file(content, suffix=\"_locustfile.py\", dir=None):\n    f = NamedTemporaryFile(suffix=suffix, delete=False, dir=dir)\n    f.write(content.encode(\"utf-8\"))\n    f.close()\n    try:\n        yield f.name\n    finally:\n        if os.path.exists(f.name):\n            os.remove(f.name)\n\n\n@contextmanager\ndef patch_env(name: str, value: str):\n    prev_value = os.getenv(name)\n    os.environ[name] = value\n    try:\n        yield\n    finally:\n        if prev_value is None:\n            del os.environ[name]\n        else:\n            os.environ[name] = prev_value\n\n\ndef get_free_tcp_port():\n    \"\"\"\n    Find an unused TCP port\n    \"\"\"\n    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n    s.bind((\"127.0.0.1\", 0))\n    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n    port = s.getsockname()[1]\n    s.close()\n    return port\n\n\ndef create_tls_cert(hostname):\n    \"\"\"Generate a TLS cert and private key to serve over https\"\"\"\n    key = rsa.generate_private_key(public_exponent=2**16 + 1, key_size=2048, backend=default_backend())\n    name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostname)])\n    now = datetime.datetime.now(tz=datetime.timezone.utc)\n    cert = (\n        x509.CertificateBuilder()\n        .subject_name(name)\n        .issuer_name(name)\n        .public_key(key.public_key())\n        .serial_number(1000)\n        .not_valid_before(now)\n        .not_valid_after(now + datetime.timedelta(days=10 * 365))\n        .sign(key, hashes.SHA256(), default_backend())\n    )\n    cert_pem = cert.public_bytes(encoding=serialization.Encoding.PEM)\n    key_pem = key.private_bytes(\n        encoding=serialization.Encoding.PEM,\n        format=serialization.PrivateFormat.TraditionalOpenSSL,\n        encryption_algorithm=serialization.NoEncryption(),\n    )\n\n    return cert_pem, key_pem\n\n\ndef clear_all_functools_lru_cache() -> None:\n    # Clear all `functools.lru_cache` to ensure that no state are persisted from one test to another.\n    # Taken from https://stackoverflow.com/a/50699209.\n    with warnings.catch_warnings():\n        warnings.simplefilter(action=\"ignore\", category=ResourceWarning)\n        gc.collect()\n    wrappers = [a for a in gc.get_objects() if isinstance(a, functools._lru_cache_wrapper)]\n    assert len(wrappers) > 0\n    for wrapper in wrappers:\n        wrapper.cache_clear()\n\n# Path: locust/test/testcases.py\nimport locust\nfrom locust import log\nfrom locust.env import Environment\nfrom locust.event import Events\nfrom locust.test.mock_logging import MockedLoggingHandler\nfrom locust.test.util import clear_all_functools_lru_cache\n\nimport base64\nimport logging\nimport random\nimport sys\nimport unittest\nimport warnings\nfrom io import BytesIO\n\nimport gevent\nimport gevent.pywsgi\nfrom flask import Flask, Response, make_response, redirect, request, send_file, stream_with_context\n\napp = Flask(__name__)\napp.jinja_env.add_extension(\"jinja2.ext.do\")\n\n\n@app.route(\"/ultra_fast\")\ndef ultra_fast():\n    return \"This is an ultra fast response\"\n\n\n@app.route(\"/fast\")\ndef fast():\n    gevent.sleep(random.choice([0.1, 0.2, 0.3]))\n    return \"This is a fast response\"\n\n\n@app.route(\"/slow\")\ndef slow():\n    delay = request.args.get(\"delay\")\n    if delay:\n        gevent.sleep(float(delay))\n    else:\n        gevent.sleep(random.choice([0.5, 1, 1.5]))\n    return \"This is a slow response\"\n\n\n@app.route(\"/consistent\")\ndef consistent():\n    gevent.sleep(0.2)\n    return \"This is a consistent response\"\n\n\n@app.route(\"/request_method\", methods=[\"POST\", \"GET\", \"HEAD\", \"PUT\", \"DELETE\", \"PATCH\"])\ndef request_method():\n    return request.method\n\n\n@app.route(\"/request_header_test\")\ndef request_header_test():\n    x_header_test = request.headers[\"X-Header-Test\"]\n    response = Response(x_header_test)\n    response.headers[\"X-Header-Test\"] = x_header_test\n\n    return response\n\n\n@app.route(\"/post\", methods=[\"POST\"])\n@app.route(\"/put\", methods=[\"PUT\"])\ndef manipulate():\n    return str(request.form.get(\"arg\", \"\"))\n\n\n@app.route(\"/get_arg\", methods=[\"GET\"])\ndef get_arg():\n    return request.args.get(\"arg\")\n\n\n@app.route(\"/fail\")\ndef failed_request():\n    return \"This response failed\", 500\n\n\n@app.route(\"/status/204\")\ndef status_204():\n    return \"\", 204\n\n\n@app.route(\"/redirect\", methods=[\"GET\", \"POST\"])\ndef do_redirect():\n    delay = request.args.get(\"delay\")\n    if delay:\n        gevent.sleep(float(delay))\n    url = request.args.get(\"url\", \"/ultra_fast\")\n    return redirect(url)\n\n\n@app.route(\"/basic_auth\")\ndef basic_auth():\n    auth = base64.b64decode(request.headers.get(\"Authorization\", \"\").replace(\"Basic \", \"\")).decode(\"utf-8\")\n    if auth == \"locust:menace\":\n        return \"Authorized\"\n    resp = make_response(\"401 Authorization Required\", 401)\n    resp.headers[\"WWW-Authenticate\"] = 'Basic realm=\"Locust\"'\n    return resp\n\n\n@app.route(\"/no_content_length\")\ndef no_content_length():\n    r = send_file(\n        BytesIO(b\"This response does not have content-length in the header\"),\n        etag=False,\n        mimetype=\"text/plain\",\n    )\n    r.headers.remove(\"Content-Length\")\n    return r\n\n\n@app.errorhandler(404)\ndef not_found(error):\n    return \"Not Found\", 404\n\n\n@app.route(\"/streaming/<int:iterations>\")\ndef streaming_response(iterations):\n    import time\n\n    def generate():\n        yield \"<html><body><h1>streaming response</h1>\"\n        for i in range(iterations):\n            yield f\"<span>{i}</span>\\n\"\n            time.sleep(0.01)\n        yield \"</body></html>\"\n\n    return Response(stream_with_context(generate()), mimetype=\"text/html\")\n\n\n@app.route(\"/set_cookie\", methods=[\"POST\"])\ndef set_cookie():\n    response = make_response(\"ok\")\n    response.set_cookie(request.args.get(\"name\"), request.args.get(\"value\"))\n    return response\n\n\n@app.route(\"/get_cookie\")\ndef get_cookie():\n    return make_response(request.cookies.get(request.args.get(\"name\"), \"\"))\n\n\n@app.route(\"/rest\", methods=[\"POST\"])\ndef rest():\n    return request.json\n\n\n@app.route(\"/content_type_missing_charset\")\ndef content_type_missing_charset():\n    resp = make_response(\"stuff\")\n    resp.headers[\"Content-Type\"] = \"Content-Type: application/json;\"\n    return resp\n\n\n@app.route(\"/content_type_regular\")\ndef content_type_regular():\n    resp = make_response(\"stuff\")\n    resp.headers[\"Content-Type\"] = \"Content-Type: application/json; charset=utf-8;\"\n    return resp\n\n\n@app.route(\"/content_type_with_extra_stuff\")\ndef content_type_with_extra_stuff():\n    resp = make_response(\"stuff\")\n    resp.headers[\"Content-Type\"] = \"Content-Type: application/json; charset=utf-8; api-version=3.0\"\n    return resp\n\n\n...\n", "needle_token_start": 3984, "needle_token_end": 4020, "code_context_ntokens": 16090, "output": ["```python\ndef get_poller():\n    if os.name == \"nt\":\n        return WindowsKeyPoller()\n    else:\n        return UnixKeyPoller()\n```"]}
{"repo": "locustio/locust", "name": "test_ramp_down_users_to_3_workers_with_spawn_rate_of_9", "language": "python", "path": "locust/test/test_dispatch.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To verify that a system correctly reduces the number of active users across three worker nodes to zero using a specific rate of user deactivation.\n2. **Input**: A setup with three worker nodes and three types of users, initially having a total of nine active users distributed across the nodes.\n3. **Output**: Assertions check that all users are deactivated in one step and that no further users remain to be deactivated, confirming the rapid deactivation process.\n4. **Procedure**: \n   - Initialize the system with three worker nodes and an equal distribution of nine users.\n   - Command the system to deactivate all users at a rate of nine users per step.\n   - Verify through assertions that all users are deactivated in one step and that attempting to get the next state results in no further users, confirming the end of the deactivation process.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 1},\n                \"3\": {\"User1\": 1, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 1},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_2(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=2)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_2_4(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=2.4)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_3(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=3)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_4(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=4)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    \ndef test_ramp_down_users_to_3_workers_with_spawn_rate_of_9(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=9)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n\n@unittest.skip(reason=\"takes too long. run this manually if you change dispatch logic.\")\nclass TestRampUpThenDownThenUp(unittest.TestCase):\n    def test_ramp_up_then_down_then_up(self):\n        for user1_weight, user2_weight, user3_weight, user4_weight, user5_weight in [\n            (1, 1, 1, 1, 1),\n            (1, 2, 3, 4, 5),\n            (1, 3, 5, 7, 9),\n        ]:\n\n            class User1(User):\n                weight = user1_weight\n\n            class User2(User):\n                weight = user2_weight\n\n            class User3(User):\n                weight = user3_weight\n\n            class User4(User):\n                weight = user4_weight\n\n            class User5(User):\n                weight = user5_weight\n\n            all_user_classes = [User1, User2, User3, User4, User5]\n\n            for number_of_user_classes in range(1, len(all_user_classes) + 1):\n                user_classes = all_user_classes[:number_of_user_classes]\n\n                for max_user_count, min_user_count in [(30, 15), (54, 21), (14165, 1476)]:\n                    for worker_count in [1, 3, 5, 9]:\n                        workers = [WorkerNode(str(i + 1)) for i in range(worker_count)]\n\n                        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n\n                        # Ramp-up to go to `min_user_count` #########\n\n                        users_dispatcher.new_dispatch(target_user_count=min_user_count, spawn_rate=1)\n                        users_dispatcher._wait_between_dispatch = 0\n\n                        all_dispatched_users_ramp_up_to_min_user_count = list(users_dispatcher)\n\n                        # Ramp-up to go to `max_user_count` #########\n\n                        users_dispatcher.new_dispatch(target_user_count=max_user_count, spawn_rate=1)\n                        users_dispatcher._wait_between_dispatch = 0\n\n                        list(users_dispatcher)\n\n                        # Ramp-down go back to `min_user_count` #########\n\n                        users_dispatcher.new_dispatch(target_user_count=min_user_count, spawn_rate=1)\n                        users_dispatcher._wait_between_dispatch = 0\n\n                        all_dispatched_users_ramp_down_to_min_user_count = list(users_dispatcher)\n\n                        # Assertions #########\n\n                        self.assertDictEqual(\n                            all_dispatched_users_ramp_up_to_min_user_count[-1],\n                            all_dispatched_users_ramp_down_to_min_user_count[-1],\n                        )\n\n\nclass TestDispatchUsersToWorkersHavingTheSameUsersAsTheTarget(unittest.TestCase):\n    def test_dispatch_users_to_3_workers(self):\n        \"\"\"Final distribution should be {\"User1\": 3, \"User2\": 3, \"User3\": 3}\"\"\"\n\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        user_count = 9\n\n        for spawn_rate in [0.15, 0.5, 1, 2, 2.4, 3, 4, 9]:\n            workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n            users_dispatcher.new_dispatch(target_user_count=user_count, spawn_rate=user_count)\n            users_dispatcher._wait_between_dispatch = 0\n            list(users_dispatcher)\n\n            sleep_time = 0.2  # Speed-up test\n\n            users_dispatcher.new_dispatch(target_user_count=user_count, spawn_rate=spawn_rate)\n            users_dispatcher._wait_between_dispatch = sleep_time\n\n            ts = time.perf_counter()\n            self.assertDictEqual(\n                next(users_dispatcher),\n                {\n                    \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                    \"2\": {\"User1\": 0, \"User2\": 3, \"User3\": 0},\n                    \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 3},\n                },\n            )\n            delta = time.perf_counter() - ts\n            self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n            ts = time.perf_counter()\n            self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n            delta = time.perf_counter() - ts\n            self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n            clear_all_functools_lru_cache()\n\n\nclass TestDistributionIsRespectedDuringDispatch(unittest.TestCase):\n    def test_dispatch_75_users_to_4_workers_with_spawn_rate_of_5(self):\n        \"\"\"\n        Test case covering reported issue in https://github.com/locustio/locust/pull/1621#issuecomment-853624275.\n\n        The case is to ramp-up from 0 to 75 users with two user classes. `User1` has a weight of 1 and `User2`\n        has a weight of 2. The original issue was with 500 users, but to keep the test shorter, we use 75 users.\n\n        Final distribution should be {\"User1\": 25, \"User2\": 50}\n        \"\"\"\n\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 2\n\n        worker_node1 = WorkerNode(\"1\")\n        worker_node2 = WorkerNode(\"2\")\n        worker_node3 = WorkerNode(\"3\")\n        worker_node4 = WorkerNode(\"4\")\n\n        users_dispatcher = UsersDispatcher(\n            worker_nodes=[worker_node1, worker_node2, worker_node3, worker_node4], user_classes=[User1, User2]\n        )\n        users_dispatcher.new_dispatch(target_user_count=75, spawn_rate=5)\n        users_dispatcher._wait_between_dispatch = 0\n\n        # total user count = 5\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 3})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1},\n                \"2\": {\"User1\": 1, \"User2\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 1},\n                \"4\": {\"User1\": 0, \"User2\": 1},\n            },\n        )\n\n        # total user count = 10\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 3, \"User2\": 7})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 1, \"User2\": 2},\n                \"2\": {\"User1\": 1, \"User2\": 2},\n                \"3\": {\"User1\": 0, \"User2\": 2},\n                \"4\": {\"User1\": 1, \"User2\": 1},\n            },\n        )\n\n        # total user count = 15\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 5, \"User2\": 10})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 1, \"User2\": 3},\n                \"2\": {\"User1\": 2, \"User2\": 2},\n                \"3\": {\"User1\": 1, \"User2\": 3},\n                \"4\": {\"User1\": 1, \"User2\": 2},\n            },\n        )\n\n        # total user count = 20\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 7, \"User2\": 13})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 2, \"User2\": 3},\n                \"2\": {\"User1\": 2, \"User2\": 3},\n                \"3\": {\"User1\": 1, \"User2\": 4},\n                \"4\": {\"User1\": 2, \"User2\": 3},\n            },\n        )\n\n        # total user count = 25\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 8, \"User2\": 17})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 2, \"User2\": 5},\n                \"2\": {\"User1\": 2, \"User2\": 4},\n                \"3\": {\"User1\": 2, \"User2\": 4},\n                \"4\": {\"User1\": 2, \"User2\": 4},\n            },\n        )\n\n        # total user count = 30\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 10, \"User2\": 20})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 3, \"User2\": 5},\n                \"2\": {\"User1\": 3, \"User2\": 5},\n                \"3\": {\"User1\": 2, \"User2\": 5},\n                \"4\": {\"User1\": 2, \"User2\": 5},\n            },\n        )\n\n        # total user count = 35\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 12, \"User2\": 23})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 3, \"User2\": 6},\n                \"2\": {\"User1\": 3, \"User2\": 6},\n                \"3\": {\"User1\": 3, \"User2\": 6},\n                \"4\": {\"User1\": 3, \"User2\": 5},\n            },\n        )\n\n        # total user count = 40\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 13, \"User2\": 27})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 3, \"User2\": 7},\n                \"2\": {\"User1\": 4, \"User2\": 6},\n                \"3\": {\"User1\": 3, \"User2\": 7},\n                \"4\": {\"User1\": 3, \"User2\": 7},\n            },\n        )\n\n        # total user count = 45\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 15, \"User2\": 30})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 4, \"User2\": 8},\n                \"2\": {\"User1\": 4, \"User2\": 7},\n                \"3\": {\"User1\": 3, \"User2\": 8},\n                \"4\": {\"User1\": 4, \"User2\": 7},\n            },\n        )\n\n        # total user count = 50\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 17, \"User2\": 33})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 4, \"User2\": 9},\n                \"2\": {\"User1\": 5, \"User2\": 8},\n                \"3\": {\"User1\": 4, \"User2\": 8},\n                \"4\": {\"User1\": 4, \"User2\": 8},\n            },\n        )\n\n        # total user count = 55\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 18, \"User2\": 37})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 5, \"User2\": 9},\n                \"2\": {\"User1\": 5, \"User2\": 9},\n                \"3\": {\"User1\": 4, \"User2\": 10},\n                \"4\": {\"User1\": 4, \"User2\": 9},\n            },\n        )\n\n        # total user count = 60\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 20, \"User2\": 40})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 5, \"User2\": 10},\n                \"2\": {\"User1\": 5, \"User2\": 10},\n                \"3\": {\"User1\": 5, \"User2\": 10},\n                \"4\": {\"User1\": 5, \"User2\": 10},\n            },\n        )\n\n        # total user count = 65\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 22, \"User2\": 43})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 6, \"User2\": 11},\n                \"2\": {\"User1\": 6, \"User2\": 10},\n                \"3\": {\"User1\": 5, \"User2\": 11},\n                \"4\": {\"User1\": 5, \"User2\": 11},\n            },\n        )\n\n        # total user count = 70\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 23, \"User2\": 47})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 6, \"User2\": 12},\n                \"2\": {\"User1\": 6, \"User2\": 12},\n                \"3\": {\"User1\": 5, \"User2\": 12},\n                \"4\": {\"User1\": 6, \"User2\": 11},\n            },\n        )\n\n        # total user count = 75, User1 = 25, User2 = 50\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 25, \"User2\": 50})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 6, \"User2\": 13},\n                \"2\": {\"User1\": 7, \"User2\": 12},\n                \"3\": {\"User1\": 6, \"User2\": 13},\n                \"4\": {\"User1\": 6, \"User2\": 12},\n            },\n        )\n\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n\n\nclass TestLargeScale(unittest.TestCase):\n    # fmt: off\n    weights = [\n        5, 55, 37, 2, 97, 41, 33, 19, 19, 34, 78, 76, 28, 62, 69, 5, 55, 37, 2, 97, 41, 33, 19, 19, 34,\n        78, 76, 28, 62, 69, 41, 33, 19, 19, 34, 78, 76, 28, 62, 69, 41, 33, 19, 19, 34, 78, 76, 28, 62, 69\n    ]\n    # fmt: on\n    numerated_weights = dict(zip(range(len(weights)), weights))\n\n    weighted_user_classes = [type(f\"User{i}\", (User,), {\"weight\": w}) for i, w in numerated_weights.items()]\n    fixed_user_classes_10k = [type(f\"FixedUser10k{i}\", (User,), {\"fixed_count\": 2000}) for i in range(50)]\n    fixed_user_classes_1M = [type(f\"FixedUser1M{i}\", (User,), {\"fixed_count\": 20000}) for i in range(50)]\n    mixed_users = weighted_user_classes[:25] + fixed_user_classes_10k[25:]\n\n    def test_distribute_users(self):\n        for user_classes in [self.weighted_user_classes, self.fixed_user_classes_1M, self.mixed_users]:\n            workers = [WorkerNode(str(i)) for i in range(10_000)]\n\n            target_user_count = 1_000_000\n\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n\n            ts = time.perf_counter()\n            users_on_workers, user_gen, worker_gen, active_users = users_dispatcher._distribute_users(\n                target_user_count=target_user_count\n            )\n            delta = time.perf_counter() - ts\n\n            # Because tests are run with coverage, the code will be slower.\n            # We set the pass criterion to 7000ms, but in real life, the\n            # `_distribute_users` method runs faster than this.\n            self.assertLessEqual(1000 * delta, 7000)\n\n            self.assertEqual(_user_count(users_on_workers), target_user_count)\n\n    def test_ramp_up_from_0_to_100_000_users_with_50_user_classes_and_1000_workers_and_5000_spawn_rate(self):\n        for user_classes in [\n            self.weighted_user_classes,\n            self.fixed_user_classes_1M,\n            self.fixed_user_classes_10k,\n            self.mixed_users,\n        ]:\n            workers = [WorkerNode(str(i)) for i in range(1000)]\n\n            target_user_count = 100_000\n\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n            users_dispatcher.new_dispatch(target_user_count=target_user_count, spawn_rate=5_000)\n            users_dispatcher._wait_between_dispatch = 0\n\n            all_dispatched_users = list(users_dispatcher)\n\n            tol = 0.2\n            self.assertTrue(\n                all(\n                    dispatch_iteration_duration <= tol\n                    for dispatch_iteration_duration in users_dispatcher.dispatch_iteration_durations\n                ),\n                \"One or more dispatch took more than {:.0f}s to compute (max = {}ms)\".format(\n                    tol * 1000, 1000 * max(users_dispatcher.dispatch_iteration_durations)\n                ),\n            )\n\n            self.assertEqual(_user_count(all_dispatched_users[-1]), target_user_count)\n\n            for dispatch_users in all_dispatched_users:\n                user_count_on_workers = [\n                    sum(user_classes_count.values()) for user_classes_count in dispatch_users.values()\n                ]\n                self.assertLessEqual(\n                    max(user_count_on_workers) - min(user_count_on_workers),\n                    1,\n                    \"One or more workers have too much users compared to the other workers when user count is {}\".format(\n                        _user_count(dispatch_users)\n                    ),\n                )\n\n            for i, dispatch_users in enumerate(all_dispatched_users):\n                aggregated_dispatched_users = _aggregate_dispatched_users(dispatch_users)\n                for user_class in [u for u in user_classes if not u.fixed_count]:\n                    target_relative_weight = user_class.weight / sum(\n                        map(attrgetter(\"weight\"), [u for u in user_classes if not u.fixed_count])\n                    )\n                    relative_weight = aggregated_dispatched_users[user_class.__name__] / _user_count(dispatch_users)\n                    error_percent = 100 * (relative_weight - target_relative_weight) / target_relative_weight\n                    if i == len(all_dispatched_users) - 1:\n                        # We want the distribution to be as good as possible at the end of the ramp-up\n                        tol = 0.5\n                    else:\n                        tol = 15\n                    self.assertLessEqual(\n                        error_percent,\n                        tol,\n                        \"Distribution for user class {} is off by more than {}% when user count is {}\".format(\n                            user_class, tol, _user_count(dispatch_users)\n                        ),\n                    )\n\n    def test_ramp_down_from_100_000_to_0_users_with_50_user_classes_and_1000_workers_and_5000_spawn_rate(self):\n        for user_classes in [\n            self.weighted_user_classes,\n            self.fixed_user_classes_1M,\n            self.fixed_user_classes_10k,\n            self.mixed_users,\n        ]:\n            initial_user_count = 100_000\n\n            workers = [WorkerNode(str(i)) for i in range(1000)]\n\n            # Ramp-up\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n            users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n            users_dispatcher._wait_between_dispatch = 0\n            list(users_dispatcher)\n\n            # Ramp-down\n            users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=5000)\n            users_dispatcher._wait_between_dispatch = 0\n\n            all_dispatched_users = list(users_dispatcher)\n\n            tol = 0.2\n            self.assertTrue(\n                all(\n                    dispatch_iteration_duration <= tol\n                    for dispatch_iteration_duration in users_dispatcher.dispatch_iteration_durations\n                ),\n                \"One or more dispatch took more than {:.0f}ms to compute (max = {}ms)\".format(\n                    tol * 1000, 1000 * max(users_dispatcher.dispatch_iteration_durations)\n                ),\n            )\n\n            self.assertEqual(_user_count(all_dispatched_users[-1]), 0)\n\n            for dispatch_users in all_dispatched_users[:-1]:\n                user_count_on_workers = [\n                    sum(user_classes_count.values()) for user_classes_count in dispatch_users.values()\n                ]\n                self.assertLessEqual(\n                    max(user_count_on_workers) - min(user_count_on_workers),\n                    1,\n                    \"One or more workers have too much users compared to the other workers when user count is {}\".format(\n                        _user_count(dispatch_users)\n                    ),\n                )\n\n            for dispatch_users in all_dispatched_users[:-1]:\n                aggregated_dispatched_users = _aggregate_dispatched_users(dispatch_users)\n                for user_class in [u for u in user_classes if not u.fixed_count]:\n                    target_relative_weight = user_class.weight / sum(\n                        map(attrgetter(\"weight\"), [u for u in user_classes if not u.fixed_count])\n                    )\n                    relative_weight = aggregated_dispatched_users[user_class.__name__] / _user_count(dispatch_users)\n                    error_percent = 100 * (relative_weight - target_relative_weight) / target_relative_weight\n                    tol = 15\n                    self.assertLessEqual(\n                        error_percent,\n                        tol,\n                        \"Distribution for user class {} is off by more than {}% when user count is {}\".format(\n                            user_class, tol, _user_count(dispatch_users)\n                        ),\n                    )\n\n\nclass TestSmallConsecutiveRamping(unittest.TestCase):\n    def test_consecutive_ramp_up_and_ramp_down(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        user_classes = [User1, User2]\n\n        worker_node1 = WorkerNode(\"1\")\n        worker_node2 = WorkerNode(\"2\")\n\n        worker_nodes = [worker_node1, worker_node2]\n\n        users_dispatcher = UsersDispatcher(worker_nodes=worker_nodes, user_classes=user_classes)\n\n        # user count = 1\n        users_dispatcher.new_dispatch(target_user_count=1, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 1, \"User2\": 0})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 0)\n\n        # user count = 2\n        users_dispatcher.new_dispatch(target_user_count=2, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 1, \"User2\": 1})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 1)\n\n        # user count = 3\n        users_dispatcher.new_dispatch(target_user_count=3, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 1})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 2)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 1)\n\n        # user count = 4\n        users_dispatcher.new_dispatch(target_user_count=4, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 2})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 2)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 2)\n\n        # user count = 3\n        users_dispatcher.new_dispatch(target_user_count=3, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 1})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 2)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 1)\n\n        # user count = 2\n        users_dispatcher.new_dispatch(target_user_count=2, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 1, \"User2\": 1})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 1)\n\n        # user count = 1\n        users_dispatcher.new_dispatch(target_user_count=1, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 1, \"User2\": 0})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 0)\n\n        # user count = 0\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = 0\n\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 0, \"User2\": 0})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node1.id), 0)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_node2.id), 0)\n\n\nclass TestRampingMiscellaneous(unittest.TestCase):\n    def test_spawn_rate_greater_than_target_user_count(self):\n        class User1(User):\n            weight = 1\n\n        user_classes = [User1]\n\n        worker_nodes = [WorkerNode(str(i + 1)) for i in range(1)]\n\n        users_dispatcher = UsersDispatcher(worker_nodes=worker_nodes, user_classes=user_classes)\n\n        users_dispatcher.new_dispatch(target_user_count=1, spawn_rate=100)\n        users_dispatcher._wait_between_dispatch = 0\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(dispatched_users, {\"1\": {\"User1\": 1}})\n\n        users_dispatcher.new_dispatch(target_user_count=11, spawn_rate=100)\n        users_dispatcher._wait_between_dispatch = 0\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(dispatched_users, {\"1\": {\"User1\": 11}})\n\n        users_dispatcher.new_dispatch(target_user_count=10, spawn_rate=100)\n        users_dispatcher._wait_between_dispatch = 0\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(dispatched_users, {\"1\": {\"User1\": 10}})\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=100)\n        users_dispatcher._wait_between_dispatch = 0\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(dispatched_users, {\"1\": {\"User1\": 0}})\n\n\nclass TestRemoveWorker(unittest.TestCase):\n    def test_remove_worker_during_ramp_up(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        worker_nodes = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        users_dispatcher = UsersDispatcher(worker_nodes=worker_nodes, user_classes=user_classes)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=9, spawn_rate=3)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        # Dispatch iteration 1\n        ts = time.perf_counter()\n        dispatched_users = next(users_dispatcher)\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 1, \"User2\": 1, \"User3\": 1})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[0].id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[1].id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[2].id), 1)\n\n        # Dispatch iteration 2\n        ts = time.perf_counter()\n        dispatched_users = next(users_dispatcher)\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 2, \"User3\": 2})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[0].id), 2)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[1].id), 2)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[2].id), 2)\n\n        self.assertFalse(users_dispatcher._rebalance)\n\n        users_dispatcher.remove_worker(worker_nodes[1])\n\n        self.assertTrue(users_dispatcher._rebalance)\n\n        # Re-balance\n        ts = time.perf_counter()\n        dispatched_users = next(users_dispatcher)\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, f\"Expected re-balance dispatch to be instantaneous but got {delta}s\")\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 2, \"User3\": 2})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[0].id), 3)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[2].id), 3)\n\n        self.assertFalse(users_dispatcher._rebalance)\n\n        # Dispatch iteration 3\n        ts = time.perf_counter()\n        dispatched_users = next(users_dispatcher)\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 3, \"User2\": 3, \"User3\": 3})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[0].id), 5)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[2].id), 4)\n\n    def test_remove_two_workers_during_ramp_up(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        worker_nodes = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        users_dispatcher = UsersDispatcher(worker_nodes=worker_nodes, user_classes=user_classes)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=9, spawn_rate=3)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        # Dispatch iteration 1\n        ts = time.perf_counter()\n        dispatched_users = next(users_dispatcher)\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 1, \"User2\": 1, \"User3\": 1})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[0].id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[1].id), 1)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[2].id), 1)\n\n        # Dispatch iteration 2\n        ts = time.perf_counter()\n        dispatched_users = next(users_dispatcher)\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 2, \"User3\": 2})\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[0].id), 2)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[1].id), 2)\n        self.assertEqual(_user_count_on_worker(dispatched_users, worker_nodes[2].id), 2)\n\n        self.assertFalse(users_dispatcher._rebalance)\n\n        users_dispatcher.remove_worker(worker_nodes[1])\n        users_dispatcher.remove_worker(worker_nodes[2])\n\n        self.assertTrue(users_dispatcher._rebalance)\n\n        # Re-balance\n...\n", "needle_token_start": 5418, "needle_token_end": 5899, "code_context_ntokens": 16157, "output": ["```python\ndef test_ramp_down_users_to_3_workers_with_spawn_rate_of_9(self):\n    class User1(User):\n        weight = 1\n\n    class User2(User):\n        weight = 1\n\n    class User3(User):\n        weight = 1\n\n    user_classes = [User1, User2, User3]\n\n    workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n    initial_user_count = 9\n\n    users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n    users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n    users_dispatcher._wait_between_dispatch = 0\n    list(users_dispatcher)\n\n    sleep_time = 0.2  # Speed-up test\n\n    users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=9)\n    users_dispatcher._wait_between_dispatch = sleep_time\n\n    ts = time.perf_counter()\n    self.assertDictEqual(\n        next(users_dispatcher),\n        {\n            \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n        },\n    )\n    delta = time.perf_counter() - ts\n    self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    ts = time.perf_counter()\n    self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n    delta = time.perf_counter() - ts\n    self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n```"]}
{"repo": "locustio/locust", "name": "test_custom_ssl_context_fail_with_bad_context", "language": "python", "path": "locust/test/test_fasthttp.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To verify that the system correctly handles an error when attempting to use an SSL context with incompatible settings (hostname checks enabled but certificate verification disabled).\n2. **Input**: A custom SSL context configuration where hostname verification is enabled but the SSL certificate verification mode is set to none.\n3. **Output**: An exception is expected to be raised, specifically a `ValueError` indicating the conflict in SSL context settings.\n4. **Procedure**: \n   - A custom SSL context is created with conflicting settings: hostname checks are enabled, but certificate verification is disabled.\n   - An HTTP session is initiated using this custom SSL context to make a request to a predefined server address.\n   - The test checks that making a request with this configuration raises the expected `ValueError`, confirming that the system properly identifies and rejects the invalid SSL context settings.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: locust/test/test_dispatch.py\nfrom __future__ import annotations\n\nfrom locust import User\nfrom locust.dispatch import UsersDispatcher\nfrom locust.runners import WorkerNode\nfrom locust.test.util import clear_all_functools_lru_cache\n\nimport time\nimport unittest\nfrom operator import attrgetter\n\n_TOLERANCE = 0.025\n\n\nclass TestRampUpUsersFromZero(unittest.TestCase):\n    def test_ramp_up_users_to_3_workers_with_spawn_rate_of_0_5(self):\n        \"\"\"Final distribution should be {\"User1\": 3, \"User2\": 3, \"User3\": 3}\"\"\"\n\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        worker_node1 = WorkerNode(\"1\")\n        worker_node2 = WorkerNode(\"2\")\n        worker_node3 = WorkerNode(\"3\")\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher = UsersDispatcher(\n            worker_nodes=[worker_node1, worker_node2, worker_node3], user_classes=[User1, User2, User3]\n        )\n        users_dispatcher.new_dispatch(target_user_count=9, spawn_rate=0.5)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 3, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 3, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 3},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_up_users_to_3_workers_with_spawn_rate_of_1(self):\n        \"\"\"Final distribution should be {\"User1\": 3, \"User2\": 3, \"User3\": 3}\"\"\"\n\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        worker_node1 = WorkerNode(\"1\")\n        worker_node2 = WorkerNode(\"2\")\n        worker_node3 = WorkerNode(\"3\")\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher = UsersDispatcher(\n            worker_nodes=[worker_node1, worker_node2, worker_node3], user_classes=[User1, User2, User3]\n        )\n        users_dispatcher.new_dispatch(target_user_count=9, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n...\n# Path: locust/test/test_env.py\nfrom locust import (\n    constant,\n)\nfrom locust.dispatch import UsersDispatcher\nfrom locust.env import Environment, LoadTestShape\nfrom locust.user import (\n    User,\n    task,\n)\nfrom locust.user.task import TaskSet\n\nfrom .fake_module1_for_env_test import MyUserWithSameName as MyUserWithSameName1\nfrom .fake_module2_for_env_test import MyUserWithSameName as MyUserWithSameName2\nfrom .testcases import LocustTestCase\n\n\nclass TestEnvironment(LocustTestCase):\n    def test_user_classes_count(self):\n        class MyUser1(User):\n            wait_time = constant(0)\n\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            wait_time = constant(0)\n\n            @task\n            def my_task(self):\n                pass\n\n        environment = Environment(user_classes=[MyUser1, MyUser2])\n\n        self.assertDictEqual({\"MyUser1\": MyUser1, \"MyUser2\": MyUser2}, environment.user_classes_by_name)\n\n    def test_user_classes_with_same_name_is_error(self):\n        with self.assertRaises(ValueError) as e:\n            environment = Environment(user_classes=[MyUserWithSameName1, MyUserWithSameName2])\n\n        self.assertEqual(\n            e.exception.args[0],\n            \"The following user classes have the same class name: locust.test.fake_module1_for_env_test.MyUserWithSameName, locust.test.fake_module2_for_env_test.MyUserWithSameName\",\n        )\n\n    def test_assign_equal_weights(self):\n        def verify_tasks(u, target_tasks):\n            self.assertEqual(len(u.tasks), len(target_tasks))\n            tasks = [t.__name__ for t in u.tasks]\n            self.assertEqual(len(tasks), len(set(tasks)))\n            self.assertEqual(set(tasks), set(target_tasks))\n\n        # Base case\n        class MyUser1(User):\n            wait_time = constant(0)\n\n            @task(4)\n            def my_task(self):\n                pass\n\n            @task(1)\n            def my_task_2(self):\n                pass\n\n        environment = Environment(user_classes=[MyUser1])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"my_task\", \"my_task_2\"])\n\n        # Testing nested task sets\n        class MyUser2(User):\n            @task\n            class TopLevelTaskSet(TaskSet):\n                @task\n                class IndexTaskSet(TaskSet):\n                    @task(10)\n                    def index(self):\n                        self.client.get(\"/\")\n\n                    @task\n                    def stop(self):\n                        self.client.get(\"/hi\")\n\n                @task(2)\n                def stats(self):\n                    self.client.get(\"/stats/requests\")\n\n        environment = Environment(user_classes=[MyUser2])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"index\", \"stop\", \"stats\"])\n\n        # Testing task assignment via instance variable\n        def outside_task():\n            pass\n\n        def outside_task_2():\n            pass\n\n        class SingleTaskSet(TaskSet):\n            tasks = [outside_task, outside_task, outside_task_2]\n\n        class MyUser3(User):\n            tasks = [SingleTaskSet, outside_task]\n\n        environment = Environment(user_classes=[MyUser3])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"outside_task\", \"outside_task_2\"])\n\n        # Testing task assignment via dict\n        class DictTaskSet(TaskSet):\n            def dict_task_1():\n                pass\n\n            def dict_task_2():\n                pass\n\n            def dict_task_3():\n                pass\n\n            tasks = {\n                dict_task_1: 5,\n                dict_task_2: 3,\n                dict_task_3: 1,\n            }\n\n        class MyUser4(User):\n            tasks = [DictTaskSet, SingleTaskSet, SingleTaskSet]\n\n        # Assign user tasks in dict\n        environment = Environment(user_classes=[MyUser4])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"outside_task\", \"outside_task_2\", \"dict_task_1\", \"dict_task_2\", \"dict_task_3\"])\n\n        class MyUser5(User):\n            tasks = {\n                DictTaskSet: 5,\n                SingleTaskSet: 3,\n                outside_task: 6,\n            }\n\n        environment = Environment(user_classes=[MyUser5])\n        environment.assign_equal_weights()\n        u = environment.user_classes[0]\n        verify_tasks(u, [\"outside_task\", \"outside_task_2\", \"dict_task_1\", \"dict_task_2\", \"dict_task_3\"])\n\n    def test_user_classes_with_zero_weight_are_removed(self):\n        class MyUser1(User):\n            wait_time = constant(0)\n            weight = 0\n\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            wait_time = constant(0)\n            weight = 1\n\n            @task\n            def my_task(self):\n                pass\n\n        environment = Environment(user_classes=[MyUser1, MyUser2])\n\n        self.assertEqual(len(environment.user_classes), 1)\n        self.assertIs(environment.user_classes[0], MyUser2)\n\n    def test_all_user_classes_with_zero_weight_raises_exception(self):\n        class MyUser1(User):\n            wait_time = constant(0)\n            weight = 0\n\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            wait_time = constant(0)\n            weight = 0\n\n            @task\n            def my_task(self):\n                pass\n\n        with self.assertRaises(ValueError) as e:\n            environment = Environment(user_classes=[MyUser1, MyUser2])\n\n        self.assertEqual(\n            e.exception.args[0],\n            \"There are no users with weight > 0.\",\n        )\n\n    def test_shape_class_attribute(self):\n        class SubLoadTestShape(LoadTestShape):\n            \"\"\"Inherited from locust.env.LoadTestShape\"\"\"\n\n        with self.assertRaisesRegex(\n            ValueError, r\"instance of LoadTestShape or subclass LoadTestShape\", msg=\"exception message is mismatching\"\n        ):\n            Environment(user_classes=[MyUserWithSameName1], shape_class=SubLoadTestShape)\n\n    def test_dispatcher_class_attribute(self):\n        environment = Environment(user_classes=[MyUserWithSameName1])\n\n        self.assertEqual(environment.dispatcher_class, UsersDispatcher)\n\n        class MyUsersDispatcher(UsersDispatcher):\n            pass\n\n        environment = Environment(user_classes=[MyUserWithSameName1], dispatcher_class=MyUsersDispatcher)\n\n        self.assertEqual(environment.dispatcher_class, MyUsersDispatcher)\n\n    def test_update_user_class(self):\n        class MyUser1(User):\n            @task\n            def my_task(self):\n                pass\n\n            @task\n            def my_task_2(self):\n                pass\n\n        class MyUser2(User):\n            @task\n            def my_task(self):\n                pass\n\n        environment = Environment(\n            user_classes=[MyUser1, MyUser2],\n            available_user_classes={\"User1\": MyUser1, \"User2\": MyUser2},\n            available_user_tasks={\"User1\": MyUser1.tasks, \"User2\": MyUser2.tasks},\n        )\n\n        environment.update_user_class({\"user_class_name\": \"User1\", \"host\": \"http://localhost\", \"tasks\": [\"my_task_2\"]})\n\n        self.assertEqual(\n            environment.available_user_classes[\"User1\"].json(),\n            {\"host\": \"http://localhost\", \"tasks\": [\"my_task_2\"], \"fixed_count\": 0, \"weight\": 1},\n        )\n\n# Path: locust/test/test_fasthttp.py\nfrom locust import FastHttpUser\nfrom locust.argument_parser import parse_options\nfrom locust.contrib.fasthttp import FastHttpSession\nfrom locust.exception import CatchResponseError, InterruptTaskSet, LocustError, ResponseError\nfrom locust.user import TaskSet, task\nfrom locust.util.load_locustfile import is_user_class\n\nimport socket\nimport time\nfrom tempfile import NamedTemporaryFile\n\nimport gevent\nfrom geventhttpclient.client import HTTPClientPool\n\nfrom .testcases import LocustTestCase, WebserverTestCase\nfrom .util import create_tls_cert\n\n\nclass TestFastHttpSession(WebserverTestCase):\n    def get_client(self):\n        return FastHttpSession(self.environment, base_url=\"http://127.0.0.1:%i\" % self.port, user=None)\n\n    def test_get(self):\n        s = self.get_client()\n        r = s.get(\"/ultra_fast\")\n        self.assertEqual(200, r.status_code)\n\n    def test_connection_error(self):\n        s = FastHttpSession(self.environment, \"http://localhost:1\", user=None)\n        r = s.get(\"/\", headers={\"X-Test-Headers\": \"hello\"})\n        self.assertEqual(r.status_code, 0)\n        self.assertEqual(None, r.content)\n        self.assertEqual(1, len(self.runner.stats.errors))\n        self.assertTrue(isinstance(r.error, ConnectionRefusedError))\n        self.assertTrue(isinstance(next(iter(self.runner.stats.errors.values())).error, ConnectionRefusedError))\n        self.assertEqual(r.url, \"http://localhost:1/\")\n        self.assertEqual(r.request.url, r.url)\n        self.assertEqual(r.request.headers.get(\"X-Test-Headers\", \"\"), \"hello\")\n\n    def test_404(self):\n        s = self.get_client()\n        r = s.get(\"/does_not_exist\")\n        self.assertEqual(404, r.status_code)\n        self.assertEqual(1, self.runner.stats.get(\"/does_not_exist\", \"GET\").num_failures)\n\n    def test_204(self):\n        s = self.get_client()\n        r = s.get(\"/status/204\")\n        self.assertEqual(204, r.status_code)\n        self.assertEqual(1, self.runner.stats.get(\"/status/204\", \"GET\").num_requests)\n        self.assertEqual(0, self.runner.stats.get(\"/status/204\", \"GET\").num_failures)\n        self.assertEqual(r.url, \"http://127.0.0.1:%i/status/204\" % self.port)\n        self.assertEqual(r.request.url, r.url)\n\n    def test_streaming_response(self):\n        \"\"\"\n        Test a request to an endpoint that returns a streaming response\n        \"\"\"\n        s = self.get_client()\n        r = s.get(\"/streaming/30\")\n\n        # verify that the time reported includes the download time of the whole streamed response\n        self.assertGreater(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n        self.runner.stats.clear_all()\n\n        # verify that response time does NOT include whole download time, when using stream=True\n        r = s.get(\"/streaming/30\", stream=True)\n        self.assertGreaterEqual(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 0)\n        self.assertLess(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n\n        # download the content of the streaming response (so we don't get an ugly exception in the log)\n        _ = r.content\n\n    def test_slow_redirect(self):\n        s = self.get_client()\n        url = \"/redirect?url=/redirect&delay=0.5\"\n        r = s.get(url)\n        stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, stats.num_requests)\n        self.assertGreater(stats.avg_response_time, 500)\n\n    def test_post_redirect(self):\n        s = self.get_client()\n        url = \"/redirect\"\n        r = s.post(url)\n        self.assertEqual(200, r.status_code)\n        post_stats = self.runner.stats.get(url, method=\"POST\")\n        get_stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, post_stats.num_requests)\n        self.assertEqual(0, get_stats.num_requests)\n\n    def test_cookie(self):\n        s = self.get_client()\n        r = s.post(\"/set_cookie?name=testcookie&value=1337\")\n        self.assertEqual(200, r.status_code)\n        r = s.get(\"/get_cookie?name=testcookie\")\n        self.assertEqual(\"1337\", r.content.decode())\n        self.assertEqual(\"1337\", r.text)\n\n    def test_head(self):\n        s = self.get_client()\n        r = s.head(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n\n    def test_delete(self):\n        s = self.get_client()\n        r = s.delete(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"DELETE\", r.content.decode())\n\n    def test_patch(self):\n        s = self.get_client()\n        r = s.patch(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"PATCH\", r.content.decode())\n\n    def test_options(self):\n        s = self.get_client()\n        r = s.options(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n        self.assertEqual(\n            {\"OPTIONS\", \"DELETE\", \"PUT\", \"GET\", \"POST\", \"HEAD\", \"PATCH\"},\n            set(r.headers[\"allow\"].split(\", \")),\n        )\n\n    def test_json_payload(self):\n        s = self.get_client()\n        r = s.post(\"/request_method\", json={\"foo\": \"bar\"})\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(r.request.body, '{\"foo\": \"bar\"}')\n        self.assertEqual(r.request.headers.get(\"Content-Type\"), \"application/json\")\n\n    def test_catch_response_fail_successful_request(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure(\"nope\")\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n\n    def test_catch_response_pass_failed_request(self):\n        s = self.get_client()\n        with s.get(\"/fail\", catch_response=True) as r:\n            r.success()\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_multiple_failure_and_success(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure(\"nope\")\n            r.success()\n            r.failure(\"nooo\")\n            r.success()\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_pass_failed_request_with_other_exception_within_block(self):\n        class OtherException(Exception):\n            pass\n\n        s = self.get_client()\n        try:\n            with s.get(\"/fail\", catch_response=True) as r:\n                r.success()\n                raise OtherException(\"wtf\")\n        except OtherException as e:\n            pass\n        else:\n            self.fail(\"OtherException should have been raised\")\n\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(0, self.environment.stats.total.num_failures)\n\n    def test_catch_response_default_success(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            pass\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(0, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n\n    def test_catch_response_default_fail(self):\n        s = self.get_client()\n        with s.get(\"/fail\", catch_response=True) as r:\n            pass\n        self.assertEqual(1, self.environment.stats.total.num_requests)\n        self.assertEqual(1, self.environment.stats.total.num_failures)\n\n    def test_error_message_with_name_replacement(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            self.assertIsNotNone(kw[\"exception\"])\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        before_request = time.time()\n        s.request(\"get\", \"/wrong_url/01\", name=\"replaced_url_name\", context={\"foo\": \"bar\"})\n        after_request = time.time()\n        # self.assertIn(\"for url: replaced_url_name\", str(kwargs[\"exception\"])) # this is actually broken for FastHttpUser right now...\n        self.assertAlmostEqual(before_request, kwargs[\"start_time\"], delta=0.01)\n        self.assertAlmostEqual(after_request, kwargs[\"start_time\"] + kwargs[\"response_time\"] / 1000, delta=0.01)\n        self.assertEqual(s.base_url + \"/wrong_url/01\", kwargs[\"url\"])  # url is unaffected by name\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    \ndef test_custom_ssl_context_fail_with_bad_context(self):\n        \"\"\"\n        Test FastHttpSession with a custom SSLContext factory that will fail as\n        we can not set verify_mode to CERT_NONE when check_hostname is enabled\n        \"\"\"\n\n        def create_custom_context():\n            context = gevent.ssl.create_default_context()\n            context.check_hostname = True\n            context.verify_mode = gevent.ssl.CERT_NONE\n            return context\n\n        s = FastHttpSession(\n            self.environment,\n            \"https://127.0.0.1:%i\" % self.port,\n            ssl_context_factory=create_custom_context,\n            user=None,\n        )\n        with self.assertRaises(ValueError) as e:\n            s.get(\"/\")\n        self.assertEqual(e.exception.args, (\"Cannot set verify_mode to CERT_NONE when check_hostname is enabled.\",))\n\n    def test_custom_ssl_context_passed_correct_to_client_pool(self):\n        \"\"\"\n        Test FastHttpSession with a custom SSLContext factory with a options.name\n        that will be passed correctly to the ClientPool. It will also test a 2nd\n        factory which is not the correct one.\n        \"\"\"\n\n        def custom_ssl_context():\n            context = gevent.ssl.create_default_context()\n            context.check_hostname = False\n            context.verify_mode = gevent.ssl.CERT_NONE\n            context.options.name = \"FAKEOPTION\"\n            return context\n\n        def custom_context_with_wrong_option():\n            context = gevent.ssl.create_default_context()\n            context.check_hostname = False\n            context.verify_mode = gevent.ssl.CERT_NONE\n            context.options.name = \"OPTIONFAKED\"\n            return context\n\n        s = FastHttpSession(\n            self.environment,\n            \"https://127.0.0.1:%i\" % self.port,\n            ssl_context_factory=custom_ssl_context,\n            user=None,\n        )\n        self.assertEqual(s.client.clientpool.client_args[\"ssl_context_factory\"], custom_ssl_context)\n        self.assertNotEqual(s.client.clientpool.client_args[\"ssl_context_factory\"], custom_context_with_wrong_option)\n\n\nclass TestRequestStatsWithWebserver(WebserverTestCase):\n    def test_request_stats_content_length(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.get(\"/ultra_fast\")\n        self.assertEqual(\n            self.runner.stats.get(\"/ultra_fast\", \"GET\").avg_content_length, len(\"This is an ultra fast response\")\n        )\n        locust.client.get(\"/ultra_fast\")\n        self.assertEqual(\n            self.runner.stats.get(\"/ultra_fast\", \"GET\").avg_content_length, len(\"This is an ultra fast response\")\n        )\n\n    def test_request_stats_no_content_length(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        path = \"/no_content_length\"\n        r = l.client.get(path)\n        self.assertEqual(\n            self.runner.stats.get(path, \"GET\").avg_content_length,\n            len(\"This response does not have content-length in the header\"),\n        )\n\n    def test_request_stats_no_content_length_streaming(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        path = \"/no_content_length\"\n        r = l.client.get(path, stream=True)\n        self.assertEqual(0, self.runner.stats.get(path, \"GET\").avg_content_length)\n\n    def test_request_stats_named_endpoint(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.get(\"/ultra_fast\", name=\"my_custom_name\")\n        self.assertEqual(1, self.runner.stats.get(\"my_custom_name\", \"GET\").num_requests)\n\n    def test_request_stats_query_variables(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.get(\"/ultra_fast?query=1\")\n        self.assertEqual(1, self.runner.stats.get(\"/ultra_fast?query=1\", \"GET\").num_requests)\n\n    def test_request_stats_put(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        locust.client.put(\"/put\")\n        self.assertEqual(1, self.runner.stats.get(\"/put\", \"PUT\").num_requests)\n\n    def test_request_connection_error(self):\n        class MyUser(FastHttpUser):\n            host = \"http://localhost:1\"\n\n        locust = MyUser(self.environment)\n        response = locust.client.get(\"/\")\n        self.assertEqual(response.status_code, 0)\n        self.assertEqual(1, self.runner.stats.get(\"/\", \"GET\").num_failures)\n        self.assertEqual(1, self.runner.stats.get(\"/\", \"GET\").num_requests)\n\n\nclass TestFastHttpUserClass(WebserverTestCase):\n    def test_is_abstract(self):\n        self.assertTrue(FastHttpUser.abstract)\n        self.assertFalse(is_user_class(FastHttpUser))\n\n    def test_class_context(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n            def context(self):\n                return {\"user\": self.username}\n\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        user = MyUser(self.environment)\n        user.username = \"foo\"\n        user.client.request(\"get\", \"/request_method\")\n        self.assertDictEqual({\"user\": \"foo\"}, kwargs[\"context\"])\n        self.assertEqual(\"GET\", kwargs[\"response\"].text)\n        user.client.request(\"get\", \"/request_method\", context={\"user\": \"bar\"})\n        self.assertDictEqual({\"user\": \"bar\"}, kwargs[\"context\"])\n\n    def test_get_request(self):\n        self.response = \"\"\n\n        def t1(l):\n            self.response = l.client.get(\"/ultra_fast\")\n\n        class MyUser(FastHttpUser):\n            tasks = [t1]\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        my_locust = MyUser(self.environment)\n        t1(my_locust)\n        self.assertEqual(self.response.text, \"This is an ultra fast response\")\n\n    def test_client_request_headers(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        r = locust.client.get(\"/request_header_test\", headers={\"X-Header-Test\": \"hello\"})\n        self.assertEqual(\"hello\", r.text)\n        self.assertEqual(\"hello\", r.headers.get(\"X-Header-Test\"))\n        self.assertEqual(\"hello\", r.request.headers.get(\"X-Header-Test\"))\n\n    def test_client_get(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"GET\", locust.client.get(\"/request_method\").text)\n\n    def test_client_get_absolute_url(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"GET\", locust.client.get(\"http://127.0.0.1:%i/request_method\" % self.port).text)\n\n    def test_client_post(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"POST\", locust.client.post(\"/request_method\", {\"arg\": \"hello world\"}).text)\n        self.assertEqual(\"hello world\", locust.client.post(\"/post\", {\"arg\": \"hello world\"}).text)\n\n    def test_client_put(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"PUT\", locust.client.put(\"/request_method\", {\"arg\": \"hello world\"}).text)\n        self.assertEqual(\"hello world\", locust.client.put(\"/put\", {\"arg\": \"hello world\"}).text)\n\n    def test_client_delete(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(\"DELETE\", locust.client.delete(\"/request_method\").text)\n        self.assertEqual(200, locust.client.delete(\"/request_method\").status_code)\n\n    def test_client_head(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        self.assertEqual(200, locust.client.head(\"/request_method\").status_code)\n\n    def test_complex_content_type(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n\n        self.assertEqual(\"stuff\", locust.client.get(\"/content_type_missing_charset\").text)\n        self.assertEqual(\"stuff\", locust.client.get(\"/content_type_regular\").text)\n        self.assertEqual(\"stuff\", locust.client.get(\"/content_type_with_extra_stuff\").text)\n\n    def test_log_request_name_argument(self):\n        self.response = \"\"\n\n        class MyUser(FastHttpUser):\n            tasks = []\n            host = \"http://127.0.0.1:%i\" % self.port\n\n            @task()\n            def t1(l):\n                self.response = l.client.get(\"/ultra_fast\", name=\"new name!\")\n\n        my_locust = MyUser(self.environment)\n        my_locust.t1()\n\n        self.assertEqual(1, self.runner.stats.get(\"new name!\", \"GET\").num_requests)\n        self.assertEqual(0, self.runner.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n\n    def test_redirect_url_original_path_as_name(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        l.client.get(\"/redirect\")\n\n        self.assertEqual(1, len(self.runner.stats.entries))\n        self.assertEqual(1, self.runner.stats.get(\"/redirect\", \"GET\").num_requests)\n        self.assertEqual(0, self.runner.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n\n    def test_network_timeout_setting(self):\n        class MyUser(FastHttpUser):\n            network_timeout = 0.5\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n\n        timeout = gevent.Timeout(\n            seconds=0.6,\n            exception=AssertionError(\n                \"Request took longer than 0.6 even though FastHttpUser.network_timeout was set to 0.5\"\n            ),\n        )\n        timeout.start()\n        r = l.client.get(\"/redirect?url=/redirect&delay=5.0\")\n        timeout.cancel()\n\n        self.assertTrue(isinstance(r.error.original, socket.timeout))\n        self.assertEqual(1, self.runner.stats.get(\"/redirect?url=/redirect&delay=5.0\", \"GET\").num_failures)\n\n    def test_max_redirect_setting(self):\n        class MyUser(FastHttpUser):\n            max_redirects = 1  # max_redirects and max_retries are funny names, because they are actually max attempts\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyUser(self.environment)\n        l.client.get(\"/redirect\")\n        self.assertEqual(1, self.runner.stats.get(\"/redirect\", \"GET\").num_failures)\n\n    def test_allow_redirects_override(self):\n        class MyLocust(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        l = MyLocust(self.environment)\n        resp = l.client.get(\"/redirect\", allow_redirects=False)\n        self.assertTrue(resp.headers[\"location\"].endswith(\"/ultra_fast\"))\n        resp = l.client.get(\"/redirect\")  # ensure redirect still works\n        self.assertFalse(\"location\" in resp.headers)\n\n    def test_slow_redirect(self):\n        s = FastHttpSession(self.environment, \"http://127.0.0.1:%i\" % self.port, user=None)\n        url = \"/redirect?url=/redirect&delay=0.5\"\n        r = s.get(url)\n        stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, stats.num_requests)\n        self.assertGreater(stats.avg_response_time, 500)\n\n    def test_client_basic_auth(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        class MyAuthorizedUser(FastHttpUser):\n            host = \"http://locust:menace@127.0.0.1:%i\" % self.port\n\n        class MyUnauthorizedUser(FastHttpUser):\n            host = \"http://locust:wrong@127.0.0.1:%i\" % self.port\n\n        locust = MyUser(self.environment)\n        unauthorized = MyUnauthorizedUser(self.environment)\n        authorized = MyAuthorizedUser(self.environment)\n        response = authorized.client.get(\"/basic_auth\")\n        self.assertEqual(200, response.status_code)\n        self.assertEqual(\"Authorized\", response.text)\n        self.assertEqual(401, locust.client.get(\"/basic_auth\").status_code)\n        self.assertEqual(401, unauthorized.client.get(\"/basic_auth\").status_code)\n\n    def test_shared_client_pool(self):\n        shared_client_pool = HTTPClientPool(concurrency=1)\n\n        class MyUserA(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n            client_pool = shared_client_pool\n\n        class MyUserB(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n            client_pool = shared_client_pool\n\n        user_a = MyUserA(self.environment)\n        user_b = MyUserB(self.environment)\n\n        user_a.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_a.client.get(\"/ultra_fast\")\n\n        self.assertEqual(1, self.connections_count)\n        self.assertEqual(4, self.requests_count)\n\n    def test_client_pool_per_user_instance(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        user_a = MyUser(self.environment)\n        user_b = MyUser(self.environment)\n\n        user_a.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_b.client.get(\"/ultra_fast\")\n        user_a.client.get(\"/ultra_fast\")\n\n        self.assertEqual(2, self.connections_count)\n        self.assertEqual(4, self.requests_count)\n\n    def test_client_pool_concurrency(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n            @task\n            def t(self):\n                def concurrent_request(url):\n                    response = self.client.get(url)\n                    assert response.status_code == 200\n\n                pool = gevent.pool.Pool()\n                urls = [\"/slow?delay=0.2\"] * 20  # these urls are all the same, but they could be different\n                for url in urls:\n                    pool.spawn(concurrent_request, url)\n                pool.join()\n\n        user = MyUser(self.environment)\n        before_requests = time.time()\n        user.t()\n        after_requests = time.time()\n        expected_delta = 0.4  # 20 requests with concurrency 10 and response time 0.2\n        self.assertAlmostEqual(before_requests + expected_delta, after_requests, delta=0.1)\n\n\nclass TestFastHttpCatchResponse(WebserverTestCase):\n    def setUp(self):\n        super().setUp()\n\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        self.user = MyUser(self.environment)\n\n        self.num_failures = 0\n        self.num_success = 0\n\n        def on_request(exception, **kwargs):\n            if exception:\n                self.num_failures += 1\n                self.last_failure_exception = exception\n            else:\n                self.num_success += 1\n\n        self.environment.events.request.add_listener(on_request)\n\n    def test_catch_response(self):\n        self.assertEqual(500, self.user.client.get(\"/fail\").status_code)\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n        with self.user.client.get(\"/ultra_fast\", catch_response=True) as response:\n            pass\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(1, self.num_success)\n        self.assertIn(\"ultra fast\", str(response.content))\n\n        with self.user.client.get(\"/ultra_fast\", catch_response=True) as response:\n            raise ResponseError(\"Not working\")\n\n        self.assertEqual(2, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_catch_response_http_fail(self):\n        with self.user.client.get(\"/fail\", catch_response=True) as response:\n            pass\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n    def test_catch_response_http_manual_fail(self):\n        with self.user.client.get(\"/ultra_fast\", catch_response=True) as response:\n            response.failure(\"Haha!\")\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n        self.assertTrue(\n            isinstance(self.last_failure_exception, CatchResponseError),\n            \"Failure event handler should have been passed a CatchResponseError instance\",\n        )\n\n    def test_catch_response_http_manual_success(self):\n        with self.user.client.get(\"/fail\", catch_response=True) as response:\n            response.success()\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_catch_response_allow_404(self):\n        with self.user.client.get(\"/does/not/exist\", catch_response=True) as response:\n            self.assertEqual(404, response.status_code)\n            if response.status_code == 404:\n                response.success()\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_interrupt_taskset_with_catch_response(self):\n        class MyTaskSet(TaskSet):\n            @task\n            def interrupted_task(self):\n                with self.client.get(\"/ultra_fast\", catch_response=True) as r:\n                    raise InterruptTaskSet()\n\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n            tasks = [MyTaskSet]\n\n        l = MyUser(self.environment)\n        ts = MyTaskSet(l)\n        self.assertRaises(InterruptTaskSet, lambda: ts.interrupted_task())\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n    def test_catch_response_connection_error_success(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:1\"\n\n        l = MyUser(self.environment)\n        with l.client.get(\"/\", catch_response=True) as r:\n            self.assertEqual(r.status_code, 0)\n            self.assertEqual(None, r.content)\n            r.success()\n        self.assertEqual(1, self.num_success)\n        self.assertEqual(0, self.num_failures)\n\n    def test_catch_response_connection_error_fail(self):\n        class MyUser(FastHttpUser):\n            host = \"http://127.0.0.1:1\"\n\n        l = MyUser(self.environment)\n        with l.client.get(\"/\", catch_response=True) as r:\n            self.assertEqual(r.status_code, 0)\n            self.assertEqual(None, r.content)\n            r.failure(\"Manual fail\")\n        self.assertEqual(0, self.num_success)\n        self.assertEqual(1, self.num_failures)\n\n    def test_catch_response_missing_with_block(self):\n        # incorrect usage, missing with-block\n        r = self.user.client.get(\"/fail\", catch_response=True)\n        self.assertRaises(LocustError, r.success)\n        self.assertRaises(LocustError, r.failure, \"\")\n\n    def test_missing_catch_response_true(self):\n        # incorrect usage, missing catch_response=True\n        with self.user.client.get(\"/fail\") as resp:\n            self.assertRaises(LocustError, resp.success)\n\n    def test_rest_success(self):\n        self.last_failure_exception = None\n        with self.user.rest(\"POST\", \"/rest\", json={\"foo\": \"bar\"}) as response:\n            assert response.js[\"foo\"] == \"bar\"\n\n        self.assertEqual(0, self.num_failures)\n        self.assertEqual(1, self.num_success)\n\n    def test_rest_fail(self):\n        with self.user.rest(\"POST\", \"/rest\", json={\"foo\": \"bar\"}) as response:\n            assert response.js[\"foo\"] == \"NOPE\"\n\n        self.assertTrue(\n            isinstance(self.last_failure_exception, CatchResponseError),\n            \"Failure event handler should have been passed a CatchResponseError instance\",\n        )\n        self.assertEqual(1, self.num_failures)\n        self.assertEqual(0, self.num_success)\n\n\nclass TestFastHttpSsl(LocustTestCase):\n    def setUp(self):\n        super().setUp()\n        tls_cert, tls_key = create_tls_cert(\"127.0.0.1\")\n        self.tls_cert_file = NamedTemporaryFile()\n        self.tls_key_file = NamedTemporaryFile()\n        with open(self.tls_cert_file.name, \"w\") as f:\n            f.write(tls_cert.decode())\n        with open(self.tls_key_file.name, \"w\") as f:\n            f.write(tls_key.decode())\n\n        self.web_ui = self.environment.create_web_ui(\n            \"127.0.0.1\",\n            0,\n            tls_cert=self.tls_cert_file.name,\n            tls_key=self.tls_key_file.name,\n        )\n        gevent.sleep(0.01)\n        self.web_port = self.web_ui.server.server_port\n\n    def tearDown(self):\n        super().tearDown()\n        self.web_ui.stop()\n\n    def test_ssl_request_insecure(self):\n        s = FastHttpSession(self.environment, \"https://127.0.0.1:%i\" % self.web_port, insecure=True, user=None)\n        r = s.get(\"/\")\n        self.assertEqual(200, r.status_code)\n        self.assertIn(\"<title>Locust for None</title>\", r.content.decode(\"utf-8\"))\n        self.assertIn(\"<p>Script: <span>None</span></p>\", r.text)\n\n# Path: locust/test/test_http.py\nfrom locust.clients import HttpSession\nfrom locust.exception import LocustError, ResponseError\nfrom locust.user.users import HttpUser\n\nimport time\n\nfrom requests.exceptions import InvalidSchema, InvalidURL, MissingSchema, RequestException\n\nfrom .testcases import WebserverTestCase\n\n\nclass TestHttpSession(WebserverTestCase):\n    def get_client(self, base_url=None):\n        if base_url is None:\n            base_url = \"http://127.0.0.1:%i\" % self.port\n        return HttpSession(\n            base_url=base_url,\n            request_event=self.environment.events.request,\n            user=None,\n        )\n\n    def test_get(self):\n        s = self.get_client()\n        r = s.get(\"/ultra_fast\")\n        self.assertEqual(200, r.status_code)\n\n    def test_connection_error(self):\n        s = self.get_client(base_url=\"http://localhost:1\")\n        r = s.get(\"/\", timeout=0.1)\n        self.assertEqual(r.status_code, 0)\n        self.assertEqual(None, r.content)\n        self.assertRaises(RequestException, r.raise_for_status)\n\n    def test_wrong_url(self):\n        for url, exception in (\n            (\"http://\\x94\", InvalidURL),\n            (\"telnet://127.0.0.1\", InvalidSchema),\n            (\"127.0.0.1\", MissingSchema),\n        ):\n            s = self.get_client(base_url=url)\n            try:\n                self.assertRaises(exception, s.get, \"/\")\n            except KeyError:\n                self.fail(f\"Invalid URL {url} was not propagated\")\n\n    def test_streaming_response(self):\n        \"\"\"\n        Test a request to an endpoint that returns a streaming response\n        \"\"\"\n        s = self.get_client()\n        r = s.get(\"/streaming/30\")\n\n        # verify that the time reported includes the download time of the whole streamed response\n        self.assertGreater(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n        self.runner.stats.clear_all()\n\n        # verify that response time does NOT include whole download time, when using stream=True\n        r = s.get(\"/streaming/30\", stream=True)\n        self.assertGreater(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 0)\n        self.assertLess(self.runner.stats.get(\"/streaming/30\", method=\"GET\").avg_response_time, 250)\n\n        # download the content of the streaming response (so we don't get an ugly exception in the log)\n        _ = r.content\n\n    def test_slow_redirect(self):\n        s = self.get_client()\n        url = \"/redirect?url=/redirect&delay=0.5\"\n        r = s.get(url)\n        stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, stats.num_requests)\n        self.assertGreater(stats.avg_response_time, 500)\n\n    def test_post_redirect(self):\n        s = self.get_client()\n        url = \"/redirect\"\n        r = s.post(url)\n        self.assertEqual(200, r.status_code)\n        post_stats = self.runner.stats.get(url, method=\"POST\")\n        get_stats = self.runner.stats.get(url, method=\"GET\")\n        self.assertEqual(1, post_stats.num_requests)\n        self.assertEqual(0, get_stats.num_requests)\n\n    def test_cookie(self):\n        s = self.get_client()\n        r = s.post(\"/set_cookie?name=testcookie&value=1337\")\n        self.assertEqual(200, r.status_code)\n        r = s.get(\"/get_cookie?name=testcookie\")\n        self.assertEqual(\"1337\", r.content.decode())\n\n    def test_head(self):\n        s = self.get_client()\n        r = s.head(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n\n    def test_delete(self):\n        s = self.get_client()\n        r = s.delete(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"DELETE\", r.content.decode())\n\n    def test_options(self):\n        s = self.get_client()\n        r = s.options(\"/request_method\")\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"\", r.content.decode())\n        self.assertEqual(\n            {\"OPTIONS\", \"DELETE\", \"PUT\", \"GET\", \"POST\", \"HEAD\", \"PATCH\"},\n            set(r.headers[\"allow\"].split(\", \")),\n        )\n\n    def test_error_message(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        s.request(\"get\", \"/wrong_url\", context={\"foo\": \"bar\"})\n        self.assertIn(\"/wrong_url\", str(kwargs[\"exception\"]))\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    def test_context_in_success(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(exception, **kw):\n            self.assertIsNone(exception)\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        s.request(\"get\", \"/request_method\", context={\"foo\": \"bar\"})\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    def test_response_parameter(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        s.request(\"get\", \"/request_method\")\n        self.assertEqual(\"GET\", kwargs[\"response\"].text)\n        s.request(\"get\", \"/wrong_url\")\n        self.assertEqual(\"Not Found\", kwargs[\"response\"].text)\n\n    def test_error_message_with_name_replacement(self):\n        s = self.get_client()\n        kwargs = {}\n\n        def on_request(**kw):\n            self.assertIsNotNone(kw[\"exception\"])\n            kwargs.update(kw)\n\n        self.environment.events.request.add_listener(on_request)\n        before_request = time.time()\n        s.request(\"get\", \"/wrong_url/01\", name=\"replaced_url_name\", context={\"foo\": \"bar\"})\n        after_request = time.time()\n        self.assertIn(\"for url: replaced_url_name\", str(kwargs[\"exception\"]))\n        self.assertAlmostEqual(before_request, kwargs[\"start_time\"], delta=0.01)\n        self.assertAlmostEqual(after_request, kwargs[\"start_time\"] + kwargs[\"response_time\"] / 1000, delta=0.01)\n        self.assertEqual(s.base_url + \"/wrong_url/01\", kwargs[\"url\"])  # url is unaffected by name\n        self.assertDictEqual({\"foo\": \"bar\"}, kwargs[\"context\"])\n\n    def test_get_with_params(self):\n        s = self.get_client()\n        r = s.get(\"/get_arg\", params={\"arg\": \"test_123\"})\n        self.assertEqual(200, r.status_code)\n        self.assertEqual(\"test_123\", r.text)\n\n    def test_catch_response_fail_successful_request(self):\n        s = self.get_client()\n        with s.get(\"/ultra_fast\", catch_response=True) as r:\n            r.failure(\"nope\")\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_requests)\n        self.assertEqual(1, self.environment.stats.get(\"/ultra_fast\", \"GET\").num_failures)\n...\n", "needle_token_start": 7129, "needle_token_end": 7363, "code_context_ntokens": 16112, "output": ["```python\ndef test_custom_ssl_context_fail_with_bad_context(self):\n    \"\"\"\n    Test FastHttpSession with a custom SSLContext factory that will fail as\n    we can not set verify_mode to CERT_NONE when check_hostname is enabled\n    \"\"\"\n\n    def create_custom_context():\n        context = gevent.ssl.create_default_context()\n        context.check_hostname = True\n        context.verify_mode = gevent.ssl.CERT_NONE\n        return context\n\n    s = FastHttpSession(\n        self.environment,\n        \"https://127.0.0.1:%i\" % self.port,\n        ssl_context_factory=create_custom_context,\n        user=None,\n    )\n    with self.assertRaises(ValueError) as e:\n        s.get(\"/\")\n    self.assertEqual(e.exception.args, (\"Cannot set verify_mode to CERT_NONE when check_hostname is enabled.\",))\n```"]}
{"repo": "locustio/locust", "name": "test_worker_heartbeat_messages_sent_to_master", "language": "python", "path": "locust/test/test_runners.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To verify that the correct heartbeat data is sent from a worker to the master, including the worker's state and resource usage metrics.\n2. **Input**: The test does not require any external inputs as it uses a mocked RPC client and a predefined user class.\n3. **Output**: There is no direct output from the test other than the validation of conditions within the test itself, ensuring that the heartbeat message contains the expected keys and values.\n4. **Procedure**: \n   - A worker instance is created with a mocked RPC client and a user class designed for the test.\n   - The test waits for a heartbeat message to be sent to the master, ensuring it happens within a specified timeout.\n   - It then checks the last heartbeat message to confirm it includes the correct data fields: state, CPU usage, and memory usage.\n   - The test concludes by shutting down the worker.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        even number of the connected workers\n        \"\"\"\n\n        class TestUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[TestUser])\n\n            for i in range(5):\n                server.mocked_send(Message(\"client_ready\", __version__, \"fake_client%i\" % i))\n\n            master.start(7, 7)\n            self.assertEqual(10, len(server.outbox))\n\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n\n            self.assertEqual(7, num_users, \"Total number of locusts that would have been spawned is not 7\")\n\n    def test_spawn_fewer_locusts_than_workers(self):\n        class TestUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[TestUser])\n\n            for i in range(5):\n                server.mocked_send(Message(\"client_ready\", __version__, \"fake_client%i\" % i))\n\n            master.start(2, 2)\n            self.assertEqual(10, len(server.outbox))\n\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n\n            self.assertEqual(2, num_users, \"Total number of locusts that would have been spawned is not 2\")\n\n    def test_spawn_correct_worker_indexes(self):\n        \"\"\"\n        Tests that workers would receive a monotonic sequence of ordinal IDs.\n        \"\"\"\n\n        class TestUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[TestUser])\n\n            USERS_COUNT = 5\n\n            for i in range(USERS_COUNT):\n                server.mocked_send(Message(\"client_ready\", __version__, \"fake_client%i\" % i))\n\n            master.start(USERS_COUNT, USERS_COUNT)\n            self.assertEqual(USERS_COUNT * 2, len(server.outbox))\n\n            indexes = []\n            for _, msg in server.outbox:\n                if msg.type == \"ack\":\n                    indexes.append(msg.data[\"index\"])\n            self.assertEqual(USERS_COUNT, len(indexes), \"Total number of locusts/workers is not 5\")\n\n            indexes.sort()\n            for i in range(USERS_COUNT):\n                self.assertEqual(indexes[i], i, \"Worker index mismatch\")\n\n    def test_custom_shape_scale_interval(self):\n        class MyUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        class TestShape(LoadTestShape):\n            def __init__(self):\n                super().__init__()\n                self._users_num = [1, 1, 1, 2, 2, 3, 3, 3, 4]\n                self._index = 0\n\n            def tick(self):\n                if self._index >= len(self._users_num):\n                    return None\n                users_num = self._users_num[self._index]\n                self._index += 1\n                return users_num, users_num\n\n        self.environment.shape_class = TestShape()\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[MyUser])\n            for i in range(5):\n                server.mocked_send(Message(\"client_ready\", __version__, \"fake_client%i\" % i))\n\n            # Start the shape_worker\n            self.environment.shape_class.reset_time()\n            master.start_shape()\n\n            # Wait for shape_worker to update user_count\n            sleep(0.5)\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n            self.assertEqual(\n                1, num_users, \"Total number of users in first stage of shape test is not 1: %i\" % num_users\n            )\n\n            # Wait for shape_worker to update user_count again\n            sleep(1.5)\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n            self.assertEqual(\n                1, num_users, \"Total number of users in second stage of shape test is not 1: %i\" % num_users\n            )\n\n            # Wait for shape_worker to update user_count few times but not reach the end yet\n            sleep(2.5)\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n            self.assertEqual(\n                3, num_users, \"Total number of users in second stage of shape test is not 3: %i\" % num_users\n            )\n\n            # Wait to ensure shape_worker has stopped the test\n            sleep(3)\n            self.assertEqual(\"stopped\", master.state, \"The test has not been stopped by the shape class\")\n\n    def test_custom_shape_scale_up(self):\n        class MyUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        class TestShape(LoadTestShape):\n            def tick(self):\n                run_time = self.get_run_time()\n                if run_time < 2:\n                    return 1, 1\n                elif run_time < 4:\n                    return 2, 2\n                else:\n                    return None\n\n        self.environment.shape_class = TestShape()\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[MyUser])\n            for i in range(5):\n                server.mocked_send(Message(\"client_ready\", __version__, \"fake_client%i\" % i))\n\n            # Start the shape_worker\n            self.environment.shape_class.reset_time()\n            master.start_shape()\n            sleep(0.5)\n\n            # Wait for shape_worker to update user_count\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n            self.assertEqual(\n                1, num_users, \"Total number of users in first stage of shape test is not 1: %i\" % num_users\n            )\n\n            # Wait for shape_worker to update user_count again\n            sleep(2)\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n            self.assertEqual(\n                3, num_users, \"Total number of users in second stage of shape test is not 3: %i\" % num_users\n            )\n\n            # Wait to ensure shape_worker has stopped the test\n            sleep(3)\n            self.assertEqual(\"stopped\", master.state, \"The test has not been stopped by the shape class\")\n\n    def test_custom_shape_scale_down(self):\n        class MyUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        class TestShape(LoadTestShape):\n            def tick(self):\n                run_time = self.get_run_time()\n                if run_time < 2:\n                    return 5, 5\n                elif run_time < 4:\n                    return 1, 5\n                else:\n                    return None\n\n        self.environment.shape_class = TestShape()\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[MyUser])\n            for i in range(5):\n                server.mocked_send(Message(\"client_ready\", __version__, \"fake_client%i\" % i))\n\n            # Start the shape_worker\n            self.environment.shape_class.reset_time()\n            master.start_shape()\n            sleep(0.5)\n\n            # Wait for shape_worker to update user_count\n            num_users = sum(\n                sum(msg.data[\"user_classes_count\"].values()) for _, msg in server.outbox if msg.type != \"ack\"\n            )\n            self.assertEqual(\n                5, num_users, \"Total number of users in first stage of shape test is not 5: %i\" % num_users\n            )\n\n            # Wait for shape_worker to update user_count again\n            sleep(2)\n            msgs = defaultdict(dict)\n            for _, msg in server.outbox:\n                if msg.type == \"ack\":\n                    continue\n                msgs[msg.node_id][msg.data[\"timestamp\"]] = sum(msg.data[\"user_classes_count\"].values())\n            # Count users for the last received messages\n            num_users = sum(v[max(v.keys())] for v in msgs.values())\n            self.assertEqual(\n                1, num_users, \"Total number of users in second stage of shape test is not 1: %i\" % num_users\n            )\n\n            # Wait to ensure shape_worker has stopped the test\n            sleep(3)\n            self.assertEqual(\"stopped\", master.state, \"The test has not been stopped by the shape class\")\n\n    def test_exception_in_task(self):\n        class MyUser(User):\n            @task\n            def will_error(self):\n                raise HeyAnException(\":(\")\n\n        self.environment.user_classes = [MyUser]\n        runner = self.environment.create_local_runner()\n\n        l = MyUser(self.environment)\n\n        self.assertRaises(HeyAnException, l.run)\n        self.assertRaises(HeyAnException, l.run)\n        self.assertEqual(1, len(runner.exceptions))\n\n        hash_key, exception = runner.exceptions.popitem()\n        self.assertIn(\"traceback\", exception)\n        self.assertIn(\"HeyAnException\", exception[\"traceback\"])\n        self.assertEqual(2, exception[\"count\"])\n\n    def test_exception_is_caught(self):\n        \"\"\"Test that exceptions are stored, and execution continues\"\"\"\n\n        class MyTaskSet(TaskSet):\n            def __init__(self, *a, **kw):\n                super().__init__(*a, **kw)\n                self._task_queue = [self.will_error, self.will_stop]\n\n            @task(1)\n            def will_error(self):\n                raise HeyAnException(\":(\")\n\n            @task(1)\n            def will_stop(self):\n                raise StopUser()\n\n        class MyUser(User):\n            wait_time = constant(0.01)\n            tasks = [MyTaskSet]\n\n        # set config to catch exceptions in locust users\n        self.environment.catch_exceptions = True\n        self.environment.user_classes = [MyUser]\n        runner = LocalRunner(self.environment)\n        l = MyUser(self.environment)\n\n        # make sure HeyAnException isn't raised\n        l.run()\n        l.run()\n        # make sure we got two entries in the error log\n        self.assertEqual(2, len(self.mocked_log.error))\n\n        # make sure exception was stored\n        self.assertEqual(1, len(runner.exceptions))\n        hash_key, exception = runner.exceptions.popitem()\n        self.assertTrue(\"traceback\" in exception)\n        self.assertTrue(\"HeyAnException\" in exception[\"traceback\"])\n        self.assertEqual(2, exception[\"count\"])\n\n    def test_master_reset_connection(self):\n        \"\"\"Test that connection will be reset when network issues found\"\"\"\n        with mock.patch(\"locust.runners.FALLBACK_INTERVAL\", new=0.1):\n            with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc(raise_on_close=False)) as server:\n                master = self.get_runner()\n                self.assertEqual(0, len(master.clients))\n                server.mocked_send(Message(\"client_ready\", NETWORK_BROKEN, \"fake_client\"))\n                self.assertTrue(master.connection_broken)\n                server.mocked_send(Message(\"client_ready\", __version__, \"fake_client\"))\n                sleep(1)\n                self.assertFalse(master.connection_broken)\n                self.assertEqual(1, len(master.clients))\n                master.quit()\n\n    def test_reset_connection_after_RPCError(self):\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc(raise_on_close=False)) as server:\n            master = self.get_runner()\n            server.mocked_send(Message(\"client_ready\", __version__, \"fake_client\"))\n            sleep(0.2)\n            self.assertFalse(master.connection_broken)\n            self.assertEqual(1, len(master.clients))\n\n            # Trigger RPCError\n            server.mocked_send(Message(\"lets_trigger_RPCError\", NETWORK_BROKEN, \"fake_client\"))\n            self.assertTrue(master.connection_broken)\n            sleep(1)\n            self.assertFalse(master.connection_broken)\n            master.quit()\n\n    def test_attributes_populated_when_calling_start(self):\n        class MyUser1(User):\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[MyUser1, MyUser2])\n\n            server.mocked_send(Message(\"client_ready\", __version__, \"fake_client1\"))\n\n            master.start(7, 7)\n            self.assertEqual({\"MyUser1\": 4, \"MyUser2\": 3}, master.target_user_classes_count)\n            self.assertEqual(7, master.target_user_count)\n            self.assertEqual(7, master.spawn_rate)\n\n            master.start(10, 10)\n            self.assertEqual({\"MyUser1\": 5, \"MyUser2\": 5}, master.target_user_classes_count)\n            self.assertEqual(10, master.target_user_count)\n            self.assertEqual(10, master.spawn_rate)\n\n            master.start(1, 3)\n            self.assertEqual({\"MyUser1\": 1, \"MyUser2\": 0}, master.target_user_classes_count)\n            self.assertEqual(1, master.target_user_count)\n            self.assertEqual(3, master.spawn_rate)\n\n    def test_custom_message_send(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner()\n            for i in range(5):\n                master.clients[i] = WorkerNode(str(i))\n            master.send_message(\"test_custom_msg\", {\"test_data\": 123})\n\n            self.assertEqual(5, len(server.outbox))\n            for _, msg in server.outbox:\n                self.assertEqual(\"test_custom_msg\", msg.type)\n                self.assertEqual(123, msg.data[\"test_data\"])\n\n    def test_custom_message_receive(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            test_custom_msg = [False]\n            test_custom_msg_data = [{}]\n\n            def on_custom_msg(msg, **kw):\n                test_custom_msg[0] = True\n                test_custom_msg_data[0] = msg.data\n\n            master = self.get_runner()\n            master.register_message(\"test_custom_msg\", on_custom_msg)\n\n            server.mocked_send(Message(\"test_custom_msg\", {\"test_data\": 123}, \"dummy_id\"))\n\n            self.assertTrue(test_custom_msg[0])\n            self.assertEqual(123, test_custom_msg_data[0][\"test_data\"])\n\n    def test_undefined_custom_message_receive(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            test_custom_msg = [False]\n\n            def on_custom_msg(msg, **kw):\n                test_custom_msg[0] = True\n\n            master = self.get_runner()\n            master.register_message(\"test_custom_msg\", on_custom_msg)\n\n            server.mocked_send(Message(\"unregistered_custom_msg\", {}, \"dummy_id\"))\n\n            self.assertFalse(test_custom_msg[0])\n            self.assertEqual(1, len(self.mocked_log.warning))\n            msg = self.mocked_log.warning[0]\n            self.assertIn(\"Unknown message type received from worker\", msg)\n\n    def test_wait_for_workers_report_after_ramp_up(self):\n        def assert_cache_hits():\n            self.assertEqual(master._wait_for_workers_report_after_ramp_up.cache_info().hits, 0)\n            master._wait_for_workers_report_after_ramp_up()\n            self.assertEqual(master._wait_for_workers_report_after_ramp_up.cache_info().hits, 1)\n\n        master = self.get_runner()\n\n        master._wait_for_workers_report_after_ramp_up.cache_clear()\n        self.assertEqual(master._wait_for_workers_report_after_ramp_up(), 1.0)\n        assert_cache_hits()\n\n        master._wait_for_workers_report_after_ramp_up.cache_clear()\n        with patch_env(\"LOCUST_WAIT_FOR_WORKERS_REPORT_AFTER_RAMP_UP\", \"5.7\"):\n            self.assertEqual(master._wait_for_workers_report_after_ramp_up(), 5.7)\n            assert_cache_hits()\n\n        master._wait_for_workers_report_after_ramp_up.cache_clear()\n        with mock.patch(\"locust.runners.WORKER_REPORT_INTERVAL\", new=1.5), patch_env(\n            \"LOCUST_WAIT_FOR_WORKERS_REPORT_AFTER_RAMP_UP\", \"5.7 * WORKER_REPORT_INTERVAL\"\n        ):\n            self.assertEqual(master._wait_for_workers_report_after_ramp_up(), 5.7 * 1.5)\n            assert_cache_hits()\n\n        master._wait_for_workers_report_after_ramp_up.cache_clear()\n\n    def test_master_discard_first_client_ready(self):\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            server.mocked_send(Message(\"client_ready\", __version__, \"dummy_client\"))\n            # discard first client_ready msg\n            server.queue.get()\n            master = self.get_runner()\n            server.mocked_send(Message(\"client_ready\", __version__, \"dummy_client\"))\n\n            self.assertEqual(1, len(master.clients))\n            self.assertEqual(\"ack\", server.outbox[0][1].type)\n            self.assertEqual(1, len(server.outbox))\n            self.assertEqual(0, server.outbox[0][1].data[\"index\"])\n\n    def test_worker_sends_bad_message_to_master(self):\n        \"\"\"\n        Validate master sends reconnect message to worker when it receives a bad message.\n        \"\"\"\n\n        class TestUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[TestUser])\n            server.mocked_send(Message(\"client_ready\", __version__, \"zeh_fake_client1\"))\n            self.assertEqual(1, len(master.clients))\n            self.assertTrue(\n                \"zeh_fake_client1\" in master.clients, \"Could not find fake client in master instance's clients dict\"\n            )\n\n            master.start(10, 10)\n            sleep(0.1)\n            server.mocked_send(Message(\"stats\", BAD_MESSAGE, \"zeh_fake_client1\"))\n            self.assertEqual(4, len(server.outbox))\n\n            # Expected message order in outbox: ack, spawn, reconnect, ack\n            self.assertEqual(\n                \"reconnect\", server.outbox[2][1].type, \"Master didn't send worker reconnect message when expected.\"\n            )\n\n    def test_worker_sends_unrecognized_message_to_master(self):\n        \"\"\"\n        Validate master ignores message from worker when it cannot parse adddress info.\n        \"\"\"\n\n        class TestUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[TestUser])\n            server.mocked_send(Message(\"client_ready\", __version__, \"zeh_fake_client1\"))\n            self.assertEqual(1, len(master.clients))\n            self.assertTrue(\n                \"zeh_fake_client1\" in master.clients, \"Could not find fake client in master instance's clients dict\"\n            )\n\n            master.start(10, 10)\n            sleep(0.1)\n            server.mocked_send(Message(\"stats\", UNRECOGNIZED_MESSAGE, \"zeh_fake_client1\"))\n            self.assertEqual(2, len(server.outbox))\n\n    def test_unknown_host_sends_message_to_master(self):\n        \"\"\"\n        Validate master ignores message that is sent from unknown host\n        \"\"\"\n\n        class TestUser(User):\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            master = self.get_runner(user_classes=[TestUser])\n            server.mocked_send(Message(\"client_ready\", __version__, \"zeh_fake_client1\"))\n            self.assertEqual(1, len(master.clients))\n            self.assertTrue(\n                \"zeh_fake_client1\" in master.clients, \"Could not find fake client in master instance's clients dict\"\n            )\n\n            master.start(10, 10)\n            sleep(0.1)\n            server.mocked_send(Message(\"stats\", UNRECOGNIZED_HOST_MESSAGE, \"unknown_host\"))\n            self.assertEqual(2, len(server.outbox))\n\n\nclass TestWorkerRunner(LocustTestCase):\n    def setUp(self):\n        super().setUp()\n        # self._report_to_master_event_handlers = [h for h in events.report_to_master._handlers]\n\n    def tearDown(self):\n        # events.report_to_master._handlers = self._report_to_master_event_handlers\n        super().tearDown()\n\n    def get_runner(self, client, environment=None, user_classes=None, auto_connect=True):\n        if auto_connect:\n            client.mocked_send(Message(\"ack\", {\"index\": 0}, \"dummy_client_id\"))\n        if environment is None:\n            environment = self.environment\n        user_classes = user_classes or []\n        environment.user_classes = user_classes\n        return WorkerRunner(environment, master_host=\"localhost\", master_port=5557)\n\n    def test_worker_stop_timeout(self):\n        class MyTestUser(User):\n            _test_state = 0\n\n            @task\n            def the_task(self):\n                MyTestUser._test_state = 1\n                gevent.sleep(0.2)\n                MyTestUser._test_state = 2\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyTestUser], client=client)\n            self.assertEqual(1, len(client.outbox))\n            self.assertEqual(\"client_ready\", client.outbox[0].type)\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyTestUser\": 1},\n                        \"host\": \"\",\n                        \"stop_timeout\": 1,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            # wait for worker to spawn locusts\n            self.assertIn(\"spawning\", [m.type for m in client.outbox])\n            worker.spawning_greenlet.join()\n            self.assertEqual(1, len(worker.user_greenlets))\n            # check that locust has started running\n            gevent.sleep(0.01)\n            self.assertEqual(1, MyTestUser._test_state)\n            # send stop message\n            client.mocked_send(Message(\"stop\", None, \"dummy_client_id\"))\n            worker.user_greenlets.join()\n            # check that locust user got to finish\n            self.assertEqual(2, MyTestUser._test_state)\n\n    def test_worker_without_stop_timeout(self):\n        class MyTestUser(User):\n            _test_state = 0\n\n            @task\n            def the_task(self):\n                MyTestUser._test_state = 1\n                gevent.sleep(0.2)\n                MyTestUser._test_state = 2\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyTestUser], client=client)\n            self.assertEqual(1, len(client.outbox))\n            self.assertEqual(\"client_ready\", client.outbox[0].type)\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyTestUser\": 1},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            # print(\"outbox:\", client.outbox)\n            # wait for worker to spawn locusts\n            self.assertIn(\"spawning\", [m.type for m in client.outbox])\n            worker.spawning_greenlet.join()\n            self.assertEqual(1, len(worker.user_greenlets))\n            # check that locust has started running\n            gevent.sleep(0.01)\n            self.assertEqual(1, MyTestUser._test_state)\n            # send stop message\n            client.mocked_send(Message(\"stop\", None, \"dummy_client_id\"))\n            worker.user_greenlets.join()\n            # check that locust user did not get to finish\n            self.assertEqual(1, MyTestUser._test_state)\n\n    def test_spawn_message_with_older_timestamp_is_rejected(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            def start(self, group: Group):\n                # We do this so that the spawning does not finish\n                # too quickly\n                gevent.sleep(0.1)\n                return super().start(group)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyUser\": 10},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            sleep(0.6)\n            self.assertEqual(STATE_SPAWNING, worker.state)\n            worker.spawning_greenlet.join()\n            self.assertEqual(10, worker.user_count)\n\n            # Send same timestamp as the first message\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyUser\": 9},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            worker.spawning_greenlet.join()\n            # Still 10 users\n            self.assertEqual(10, worker.user_count)\n\n            # Send older timestamp than the first message\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538583,\n                        \"user_classes_count\": {\"MyUser\": 2},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            worker.spawning_greenlet.join()\n            # Still 10 users\n            self.assertEqual(10, worker.user_count)\n\n            # Send newer timestamp than the first message\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538585,\n                        \"user_classes_count\": {\"MyUser\": 2},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            worker.spawning_greenlet.join()\n            self.assertEqual(2, worker.user_count)\n\n            worker.quit()\n\n    def test_worker_messages_sent_to_master(self):\n        \"\"\"\n        Ensure that worker includes both \"user_count\" and \"user_classes_count\"\n        when reporting to the master.\n        \"\"\"\n\n        class MyUser(User):\n            wait_time = constant(1)\n\n            def start(self, group: Group):\n                # We do this so that the spawning does not finish\n                # too quickly\n                gevent.sleep(0.1)\n                return super().start(group)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyUser\": 10},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            sleep(0.6)\n            self.assertEqual(STATE_SPAWNING, worker.state)\n            worker.spawning_greenlet.join()\n            self.assertEqual(10, worker.user_count)\n\n            sleep(2)\n\n            message = next((m for m in reversed(client.outbox) if m.type == \"stats\"), None)\n            self.assertIsNotNone(message)\n            self.assertIn(\"user_count\", message.data)\n            self.assertIn(\"user_classes_count\", message.data)\n            self.assertEqual(message.data[\"user_count\"], 10)\n            self.assertEqual(message.data[\"user_classes_count\"][\"MyUser\"], 10)\n\n            message = next((m for m in client.outbox if m.type == \"spawning_complete\"), None)\n            self.assertIsNotNone(message)\n            self.assertIn(\"user_count\", message.data)\n            self.assertIn(\"user_classes_count\", message.data)\n            self.assertEqual(message.data[\"user_count\"], 10)\n            self.assertEqual(message.data[\"user_classes_count\"][\"MyUser\"], 10)\n\n            worker.quit()\n\n    \ndef test_worker_heartbeat_messages_sent_to_master(self):\n        \"\"\"\n        Validate content of the heartbeat payload sent to the master.\n        \"\"\"\n\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n\n            t0 = time.perf_counter()\n            while len([m for m in client.outbox if m.type == \"heartbeat\"]) == 0:\n                self.assertLessEqual(time.perf_counter() - t0, 3)\n                sleep(0.1)\n\n            message = next(m for m in reversed(client.outbox) if m.type == \"heartbeat\")\n            self.assertEqual(len(message.data), 3)\n            self.assertIn(\"state\", message.data)\n            self.assertIn(\"current_cpu_usage\", message.data)\n            self.assertIn(\"current_memory_usage\", message.data)\n\n            worker.quit()\n\n    def test_reset_rpc_connection_to_master(self):\n        \"\"\"\n        Validate worker resets RPC connection to master on \"reconnect\" message.\n        \"\"\"\n\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc(raise_on_close=False)) as client:\n            client_id = id(client)\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyUser\": 10},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            sleep(0.6)\n            self.assertEqual(STATE_RUNNING, worker.state)\n            with self.assertLogs(\"locust.runners\") as capture:\n                with mock.patch(\"locust.rpc.rpc.Client.close\") as close:\n                    client.mocked_send(\n                        Message(\n                            \"reconnect\",\n                            None,\n                            \"dummy_client_id\",\n                        )\n                    )\n                    sleep(0)\n                    worker.spawning_greenlet.join()\n                    worker.quit()\n                    close.assert_called_once()\n            self.assertIn(\n                \"WARNING:locust.runners:Received reconnect message from master. Resetting RPC connection.\",\n                capture.output,\n            )\n\n    def test_change_user_count_during_spawning(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            def start(self, group: Group):\n                # We do this so that the spawning does not finish\n                # too quickly\n                gevent.sleep(0.1)\n                return super().start(group)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyUser\": 10},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            sleep(0.6)\n            self.assertEqual(STATE_SPAWNING, worker.state)\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538585,\n                        \"user_classes_count\": {\"MyUser\": 9},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            sleep(0)\n            worker.spawning_greenlet.join()\n            self.assertEqual(9, len(worker.user_greenlets))\n            worker.quit()\n\n    def test_computed_properties(self):\n        class MyUser1(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        class MyUser2(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser1, MyUser2], client=client)\n\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538584,\n                        \"user_classes_count\": {\"MyUser1\": 10, \"MyUser2\": 10},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            worker.spawning_greenlet.join()\n            self.assertDictEqual(worker.user_classes_count, {\"MyUser1\": 10, \"MyUser2\": 10})\n            self.assertDictEqual(worker.target_user_classes_count, {\"MyUser1\": 10, \"MyUser2\": 10})\n            self.assertEqual(worker.target_user_count, 20)\n\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538585,\n                        \"user_classes_count\": {\"MyUser1\": 1, \"MyUser2\": 2},\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            worker.spawning_greenlet.join()\n            self.assertDictEqual(worker.user_classes_count, {\"MyUser1\": 1, \"MyUser2\": 2})\n            self.assertDictEqual(worker.target_user_classes_count, {\"MyUser1\": 1, \"MyUser2\": 2})\n            self.assertEqual(worker.target_user_count, 3)\n\n            worker.quit()\n\n    def test_custom_message_send(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n            client.outbox.clear()\n            worker.send_message(\"test_custom_msg\", {\"test_data\": 123})\n            self.assertEqual(\"test_custom_msg\", client.outbox[0].type)\n            self.assertEqual(123, client.outbox[0].data[\"test_data\"])\n            worker.quit()\n\n    def test_custom_message_receive(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            test_custom_msg = [False]\n            test_custom_msg_data = [{}]\n\n            def on_custom_msg(msg, **kw):\n                test_custom_msg[0] = True\n                test_custom_msg_data[0] = msg.data\n\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n            worker.register_message(\"test_custom_msg\", on_custom_msg)\n\n            client.mocked_send(Message(\"test_custom_msg\", {\"test_data\": 123}, \"dummy_client_id\"))\n\n            self.assertTrue(test_custom_msg[0])\n            self.assertEqual(123, test_custom_msg_data[0][\"test_data\"])\n            worker.quit()\n\n    def test_undefined_custom_message_receive(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            test_custom_msg = [False]\n\n            def on_custom_msg(msg, **kw):\n                test_custom_msg[0] = True\n\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n            worker.register_message(\"test_custom_msg\", on_custom_msg)\n\n            client.mocked_send(Message(\"unregistered_custom_msg\", {}, \"dummy_id\"))\n\n            self.assertFalse(test_custom_msg[0])\n            self.assertEqual(1, len(self.mocked_log.warning))\n            msg = self.mocked_log.warning[0]\n            self.assertIn(\"Unknown message type received\", msg)\n\n    def test_start_event(self):\n        class MyTestUser(User):\n            _test_state = 0\n\n            @task\n            def the_task(self):\n                MyTestUser._test_state = 1\n                gevent.sleep(0.2)\n                MyTestUser._test_state = 2\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            environment = Environment()\n            run_count = [0]\n\n            @environment.events.test_start.add_listener\n            def on_test_start(*args, **kw):\n                run_count[0] += 1\n\n            worker = self.get_runner(environment=environment, user_classes=[MyTestUser], client=client)\n            self.assertEqual(1, len(client.outbox))\n            self.assertEqual(\"client_ready\", client.outbox[0].type)\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538585,\n                        \"user_classes_count\": {\"MyTestUser\": 1},\n                        \"spawn_rate\": 1,\n                        \"num_users\": 1,\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            # wait for worker to spawn locusts\n            self.assertIn(\"spawning\", [m.type for m in client.outbox])\n            worker.spawning_greenlet.join()\n            self.assertEqual(1, len(worker.user_greenlets))\n            self.assertEqual(1, run_count[0])\n\n            # check that locust has started running\n            gevent.sleep(0.01)\n            self.assertEqual(1, MyTestUser._test_state)\n\n            # change number of users and check that test_start isn't fired again\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538586,\n                        \"user_classes_count\": {\"MyTestUser\": 1},\n                        \"spawn_rate\": 1,\n                        \"num_users\": 1,\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            self.assertEqual(1, run_count[0])\n\n            # stop and start to make sure test_start is fired again\n            client.mocked_send(Message(\"stop\", None, \"dummy_client_id\"))\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538587,\n                        \"user_classes_count\": {\"MyTestUser\": 1},\n                        \"spawn_rate\": 1,\n                        \"num_users\": 1,\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            gevent.sleep(0.01)\n            self.assertEqual(2, run_count[0])\n\n            client.mocked_send(Message(\"stop\", None, \"dummy_client_id\"))\n\n    def test_stop_event(self):\n        class MyTestUser(User):\n            _test_state = 0\n\n            @task\n            def the_task(self):\n                MyTestUser._test_state = 1\n                gevent.sleep(0.2)\n                MyTestUser._test_state = 2\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            environment = Environment()\n            run_count = [0]\n\n            @environment.events.test_stop.add_listener\n            def on_test_stop(*args, **kw):\n                run_count[0] += 1\n\n            worker = self.get_runner(environment=environment, user_classes=[MyTestUser], client=client)\n            self.assertEqual(1, len(client.outbox))\n            self.assertEqual(\"client_ready\", client.outbox[0].type)\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538585,\n                        \"user_classes_count\": {\"MyTestUser\": 1},\n                        \"spawn_rate\": 1,\n                        \"num_users\": 1,\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n\n            # wait for worker to spawn locusts\n            self.assertIn(\"spawning\", [m.type for m in client.outbox])\n            worker.spawning_greenlet.join()\n            self.assertEqual(1, len(worker.user_greenlets))\n\n            # check that locust has started running\n            gevent.sleep(0.01)\n            self.assertEqual(1, MyTestUser._test_state)\n\n            # stop and make sure test_stop is fired\n            client.mocked_send(Message(\"stop\", None, \"dummy_client_id\"))\n            gevent.sleep(0.01)\n            self.assertEqual(1, run_count[0])\n\n            # stop while stopped and make sure the event isn't fired again\n            client.mocked_send(Message(\"stop\", None, \"dummy_client_id\"))\n            gevent.sleep(0.01)\n            self.assertEqual(1, run_count[0])\n\n            # start and stop to check that the event is fired again\n            client.mocked_send(\n                Message(\n                    \"spawn\",\n                    {\n                        \"timestamp\": 1605538586,\n                        \"user_classes_count\": {\"MyTestUser\": 1},\n                        \"spawn_rate\": 1,\n                        \"num_users\": 1,\n                        \"host\": \"\",\n                        \"stop_timeout\": None,\n                        \"parsed_options\": {},\n                    },\n                    \"dummy_client_id\",\n                )\n            )\n            client.mocked_send(Message(\"stop\", None, \"dummy_client_id\"))\n            gevent.sleep(0.01)\n            self.assertEqual(2, run_count[0])\n\n    def test_worker_connect_success(self):\n        class MyTestUser(User):\n            @task\n            def the_task(self):\n                pass\n\n        with mock.patch(\"locust.runners.CONNECT_TIMEOUT\", new=1):\n            with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n                worker = self.get_runner(environment=Environment(), user_classes=[MyTestUser], client=client)\n\n                self.assertEqual(\"client_ready\", client.outbox[0].type)\n                self.assertEqual(1, len(client.outbox))\n                self.assertTrue(worker.connected)\n\n    def test_worker_connect_failure(self):\n        class MyTestUser(User):\n            @task\n            def the_task(self):\n                pass\n\n        with mock.patch(\"locust.runners.CONNECT_TIMEOUT\", new=0.01):\n            with mock.patch(\"locust.runners.CONNECT_RETRY_COUNT\", new=1):\n                with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n                    with self.assertRaises(ConnectionError):\n                        self.get_runner(\n                            environment=Environment(), user_classes=[MyTestUser], client=client, auto_connect=False\n                        )\n                    self.assertEqual(2, len(client.outbox))\n\n\nclass TestMessageSerializing(unittest.TestCase):\n    def test_message_serialize(self):\n        msg = Message(\"client_ready\", __version__, \"my_id\")\n        rebuilt = Message.unserialize(msg.serialize())\n        self.assertEqual(msg.type, rebuilt.type)\n        self.assertEqual(msg.data, rebuilt.data)\n        self.assertEqual(msg.node_id, rebuilt.node_id)\n\n\nclass TestStopTimeout(LocustTestCase):\n    def test_stop_timeout(self):\n        short_time = 0.05\n\n        class MyTaskSet(TaskSet):\n            @task\n            def my_task(self):\n                MyTaskSet.state = \"first\"\n                gevent.sleep(short_time)\n                MyTaskSet.state = \"second\"  # should only run when run time + stop_timeout is > short_time\n                gevent.sleep(short_time)\n                MyTaskSet.state = \"third\"  # should only run when run time + stop_timeout is > short_time * 2\n\n        class MyTestUser(User):\n            tasks = [MyTaskSet]\n\n        environment = Environment(user_classes=[MyTestUser])\n        runner = environment.create_local_runner()\n        runner.start(1, 1, wait=False)\n        gevent.sleep(short_time / 2)\n        runner.quit()\n        self.assertEqual(\"first\", MyTaskSet.state)\n\n        # exit with timeout\n        environment = Environment(user_classes=[MyTestUser], stop_timeout=short_time / 2)\n        runner = environment.create_local_runner()\n        runner.start(1, 1, wait=False)\n        gevent.sleep(short_time)\n        runner.quit()\n        self.assertEqual(\"second\", MyTaskSet.state)\n\n        # allow task iteration to complete, with some margin\n        environment = Environment(user_classes=[MyTestUser], stop_timeout=short_time * 3)\n        runner = environment.create_local_runner()\n        runner.start(1, 1, wait=False)\n        gevent.sleep(short_time)\n        timeout = gevent.Timeout(short_time * 2)\n        timeout.start()\n        try:\n            runner.quit()\n            runner.greenlet.join()\n        except gevent.Timeout:\n            self.fail(\"Got Timeout exception. Some locusts must have kept running after iteration finish\")\n        finally:\n            timeout.cancel()\n        self.assertEqual(\"third\", MyTaskSet.state)\n\n    def test_stop_timeout_during_on_start(self):\n        short_time = 0.05\n\n        class MyTaskSet(TaskSet):\n            finished_on_start = False\n            my_task_run = False\n\n            def on_start(self):\n                gevent.sleep(short_time)\n                MyTaskSet.finished_on_start = True\n\n            @task\n            def my_task(self):\n                MyTaskSet.my_task_run = True\n\n        class MyTestUser(User):\n            tasks = [MyTaskSet]\n\n        environment = create_environment([MyTestUser], mocked_options())\n        environment.stop_timeout = short_time\n        runner = environment.create_local_runner()\n        runner.start(1, 1)\n        gevent.sleep(short_time / 2)\n        runner.quit()\n\n        self.assertTrue(MyTaskSet.finished_on_start)\n        self.assertFalse(MyTaskSet.my_task_run)\n\n    def test_stop_timeout_exit_during_wait(self):\n        short_time = 0.05\n\n        class MyTaskSet(TaskSet):\n            @task\n            def my_task(self):\n                pass\n\n        class MyTestUser(User):\n            tasks = [MyTaskSet]\n            wait_time = constant(1)\n\n        environment = Environment(user_classes=[MyTestUser], stop_timeout=short_time)\n        runner = environment.create_local_runner()\n        runner.start(1, 1)\n        gevent.sleep(short_time)  # sleep to make sure locust has had time to start waiting\n        timeout = gevent.Timeout(short_time)\n        timeout.start()\n        try:\n            runner.quit()\n            runner.greenlet.join()\n        except gevent.Timeout:\n            self.fail(\"Got Timeout exception. Waiting locusts should stop immediately, even when using stop_timeout.\")\n        finally:\n            timeout.cancel()\n\n    def test_stop_timeout_with_interrupt(self):\n        short_time = 0.05\n\n        class MySubTaskSet(TaskSet):\n            @task\n            def a_task(self):\n                gevent.sleep(0)\n                self.interrupt(reschedule=True)\n\n        class MyTaskSet(TaskSet):\n            tasks = [MySubTaskSet]\n\n        class MyTestUser(User):\n            tasks = [MyTaskSet]\n\n        environment = create_environment([MyTestUser], mocked_options())\n        environment.stop_timeout = short_time\n        runner = environment.create_local_runner()\n        runner.start(1, 1, wait=True)\n        gevent.sleep(0)\n        timeout = gevent.Timeout(short_time)\n        timeout.start()\n        try:\n            runner.quit()\n            runner.greenlet.join()\n        except gevent.Timeout:\n            self.fail(\"Got Timeout exception. Interrupted locusts should exit immediately during stop_timeout.\")\n        finally:\n            timeout.cancel()\n\n    def test_stop_timeout_with_interrupt_no_reschedule(self):\n        state = [0]\n\n        class MySubTaskSet(TaskSet):\n            @task\n            def a_task(self):\n                gevent.sleep(0.1)\n                state[0] = 1\n                self.interrupt(reschedule=False)\n\n        class MyTestUser(User):\n            tasks = [MySubTaskSet]\n            wait_time = constant(3)\n\n        options = mocked_options()\n        options.stop_timeout = 0.3\n        environment = create_environment([MyTestUser], options)\n        runner = environment.create_local_runner()\n        runner.start(1, 1, wait=True)\n        gevent.sleep(0)\n        timeout = gevent.Timeout(0.11)\n        timeout.start()\n        try:\n            runner.quit()\n            runner.greenlet.join()\n        except gevent.Timeout:\n            self.fail(\"Got Timeout exception. Interrupted locusts should exit immediately during stop_timeout.\")\n        finally:\n            timeout.cancel()\n        self.assertEqual(1, state[0])\n\n    def test_kill_locusts_with_stop_timeout(self):\n        short_time = 0.05\n\n        class MyTaskSet(TaskSet):\n            @task\n            def my_task(self):\n                MyTaskSet.state = \"first\"\n                gevent.sleep(short_time)\n                MyTaskSet.state = \"second\"  # should only run when run time + stop_timeout is > short_time\n                gevent.sleep(short_time)\n                MyTaskSet.state = \"third\"  # should only run when run time + stop_timeout is > short_time * 2\n\n        class MyTestUser(User):\n            tasks = [MyTaskSet]\n\n        environment = create_environment([MyTestUser], mocked_options())\n        runner = environment.create_local_runner()\n        runner.start(1, 1)\n        gevent.sleep(short_time / 2)\n        runner.stop_users({MyTestUser.__name__: 1})\n        self.assertEqual(\"first\", MyTaskSet.state)\n        runner.quit()\n        environment.runner = None\n\n        environment.stop_timeout = short_time / 2  # exit with timeout\n        runner = environment.create_local_runner()\n        runner.start(1, 1)\n        gevent.sleep(short_time)\n        runner.stop_users({MyTestUser.__name__: 1})\n        self.assertEqual(\"second\", MyTaskSet.state)\n        runner.quit()\n        environment.runner = None\n\n        environment.stop_timeout = short_time * 3  # allow task iteration to complete, with some margin\n        runner = environment.create_local_runner()\n        runner.start(1, 1)\n        gevent.sleep(short_time)\n        timeout = gevent.Timeout(short_time * 2)\n        timeout.start()\n        try:\n            runner.stop_users({MyTestUser.__name__: 1})\n            runner.user_greenlets.join()\n        except gevent.Timeout:\n            self.fail(\"Got Timeout exception. Some locusts must have kept running after iteration finish\")\n        finally:\n            timeout.cancel()\n        self.assertEqual(\"third\", MyTaskSet.state)\n\n    def test_users_can_call_runner_quit_with_stop_timeout(self):\n        class BaseUser(User):\n            wait_time = constant(1)\n\n            @task\n            def trigger(self):\n                self.environment.runner.quit()\n\n        runner = Environment(user_classes=[BaseUser]).create_local_runner()\n        runner.environment.stop_timeout = 1\n        runner.spawn_users({BaseUser.__name__: 1}, wait=False)\n        timeout = gevent.Timeout(0.5)\n        timeout.start()\n        try:\n            runner.greenlet.join()\n        except gevent.Timeout:\n            self.fail(\"Got Timeout exception, runner must have hung somehow.\")\n        finally:\n            timeout.cancel()\n\n    def test_gracefully_handle_exceptions_in_listener(self):\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        test_stop_run = [0]\n        environment = Environment(user_classes=[MyUser])\n\n        def on_test_stop_ok(*args, **kwargs):\n            test_stop_run[0] += 1\n\n        def on_test_stop_fail(*args, **kwargs):\n            assert False\n\n        environment.events.test_stop.add_listener(on_test_stop_ok)\n        environment.events.test_stop.add_listener(on_test_stop_fail)\n        environment.events.test_stop.add_listener(on_test_stop_ok)\n\n        runner = LocalRunner(environment)\n        runner.start(user_count=3, spawn_rate=3, wait=False)\n        self.assertEqual(0, test_stop_run[0])\n        runner.stop()\n        self.assertEqual(2, test_stop_run[0])\n\n    def test_stop_timeout_with_ramp_down(self):\n        class MyTaskSet(TaskSet):\n            @task\n            def my_task(self):\n                gevent.sleep(1)\n\n        class MyTestUser(User):\n            tasks = [MyTaskSet]\n\n        environment = Environment(user_classes=[MyTestUser], stop_timeout=2)\n        runner = environment.create_local_runner()\n\n        # Start load test, wait for users to start, then trigger ramp down\n        ts = time.perf_counter()\n        runner.start(10, 10, wait=False)\n        runner.spawning_greenlet.join()\n        delta = time.perf_counter() - ts\n        self.assertTrue(\n            0 <= delta <= 0.05, f\"Expected user count to increase to 10 instantaneously, instead it took {delta:f}\"\n        )\n        self.assertTrue(\n...\n", "needle_token_start": 8705, "needle_token_end": 9005, "code_context_ntokens": 16102, "output": ["```python\ndef test_worker_heartbeat_messages_sent_to_master(self):\n        \"\"\"\n        Validate content of the heartbeat payload sent to the master.\n        \"\"\"\n\n        class MyUser(User):\n            wait_time = constant(1)\n\n            @task\n            def my_task(self):\n                pass\n\n        with mock.patch(\"locust.rpc.rpc.Client\", mocked_rpc()) as client:\n            worker = self.get_runner(environment=Environment(), user_classes=[MyUser], client=client)\n\n            t0 = time.perf_counter()\n            while len([m for m in client.outbox if m.type == \"heartbeat\"]) == 0:\n                self.assertLessEqual(time.perf_counter() - t0, 3)\n                sleep(0.1)\n\n            message = next(m for m in reversed(client.outbox) if m.type == \"heartbeat\")\n            self.assertEqual(len(message.data), 3)\n            self.assertIn(\"state\", message.data)\n            self.assertIn(\"current_cpu_usage\", message.data)\n            self.assertIn(\"current_memory_usage\", message.data)\n\n            worker.quit()\n```"]}
{"repo": "locustio/locust", "name": "test_request_stats_named_endpoint_rename_request", "language": "python", "path": "locust/test/test_stats.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To verify that renaming a request endpoint correctly updates the statistics tracking system with the new name.\n2. **Input**: A GET request to a specified URL under a temporarily assigned custom name.\n3. **Output**: The test checks if the statistics entry for the custom name records exactly one request.\n4. **Procedure**: The test initiates a context where the request name is temporarily changed. Within this context, a GET request is made to a specific URL. After exiting the context, the test asserts that the statistics for the custom name correctly reflect the number of requests made under that name.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        log(35, 1)\n        log(79, 1)\n        log(None, 1)\n        log_error(Exception(\"dummy fail\"))\n        self.s = self.stats.entries[(\"test_entry\", \"GET\")]\n\n    def test_percentile(self):\n        s = StatsEntry(self.stats, \"percentile_test\", \"GET\")\n        for x in range(100):\n            s.log(x, 0)\n\n        self.assertEqual(s.get_response_time_percentile(0.5), 50)\n        self.assertEqual(s.get_response_time_percentile(0.6), 60)\n        self.assertEqual(s.get_response_time_percentile(0.95), 95)\n\n    def test_median(self):\n        self.assertEqual(self.s.median_response_time, 79)\n\n    def test_median_out_of_min_max_bounds(self):\n        s = StatsEntry(self.stats, \"median_test\", \"GET\")\n        s.log(6034, 0)\n        self.assertEqual(s.median_response_time, 6034)\n        s.reset()\n        s.log(6099, 0)\n        self.assertEqual(s.median_response_time, 6099)\n\n    def test_total_rps(self):\n        self.stats.log_request(\"GET\", \"other_endpoint\", 1337, 1337)\n        s2 = self.stats.entries[(\"other_endpoint\", \"GET\")]\n        s2.start_time = 2.0\n        s2.last_request_timestamp = 6.0\n        self.s.start_time = 1.0\n        self.s.last_request_timestamp = 4.0\n        self.stats.total.start_time = 1.0\n        self.stats.total.last_request_timestamp = 6.0\n        self.assertEqual(self.s.total_rps, 9 / 5.0)\n        self.assertAlmostEqual(s2.total_rps, 1 / 5.0)\n        self.assertEqual(self.stats.total.total_rps, 10 / 5.0)\n\n    def test_rps_less_than_one_second(self):\n        s = StatsEntry(self.stats, \"percentile_test\", \"GET\")\n        for i in range(10):\n            s.log(i, 0)\n        self.assertGreater(s.total_rps, 10)\n\n    def test_current_rps(self):\n        self.stats.total.last_request_timestamp = int(time.time()) + 4\n        self.assertEqual(self.s.current_rps, 4.5)\n\n        self.stats.total.last_request_timestamp = int(time.time()) + 25\n        self.assertEqual(self.s.current_rps, 0)\n\n    def test_current_fail_per_sec(self):\n        self.stats.total.last_request_timestamp = int(time.time()) + 4\n        self.assertEqual(self.s.current_fail_per_sec, 1.5)\n\n        self.stats.total.last_request_timestamp = int(time.time()) + 12\n        self.assertEqual(self.s.current_fail_per_sec, 0.3)\n\n        self.stats.total.last_request_timestamp = int(time.time()) + 25\n        self.assertEqual(self.s.current_fail_per_sec, 0)\n\n    def test_num_reqs_fails(self):\n        self.assertEqual(self.s.num_requests, 9)\n        self.assertEqual(self.s.num_failures, 3)\n\n    def test_avg(self):\n        self.assertEqual(self.s.avg_response_time, 187.71428571428572)\n\n    def test_total_content_length(self):\n        self.assertEqual(self.s.total_content_length, 9)\n\n    def test_reset(self):\n        self.s.reset()\n        self.s.log(756, 0)\n        self.s.log_error(Exception(\"dummy fail after reset\"))\n        self.s.log(85, 0)\n\n        self.assertGreater(self.s.total_rps, 2)\n        self.assertEqual(self.s.num_requests, 2)\n        self.assertEqual(self.s.num_failures, 1)\n        self.assertEqual(self.s.avg_response_time, 420.5)\n        self.assertEqual(self.s.median_response_time, 85)\n        self.assertNotEqual(None, self.s.last_request_timestamp)\n        self.s.reset()\n        self.assertEqual(None, self.s.last_request_timestamp)\n\n    def test_avg_only_none(self):\n        self.s.reset()\n        self.s.log(None, 123)\n        self.assertEqual(self.s.avg_response_time, 0)\n        self.assertEqual(self.s.median_response_time, 0)\n        self.assertEqual(self.s.get_response_time_percentile(0.5), 0)\n\n    def test_reset_min_response_time(self):\n        self.s.reset()\n        self.s.log(756, 0)\n        self.assertEqual(756, self.s.min_response_time)\n\n    def test_aggregation(self):\n        s1 = StatsEntry(self.stats, \"aggregate me!\", \"GET\")\n        s1.log(12, 0)\n        s1.log(12, 0)\n        s1.log(38, 0)\n        s1.log_error(\"Dummy exception\")\n\n        s2 = StatsEntry(self.stats, \"aggregate me!\", \"GET\")\n        s2.log_error(\"Dummy exception\")\n        s2.log_error(\"Dummy exception\")\n        s2.log(12, 0)\n        s2.log(99, 0)\n        s2.log(14, 0)\n        s2.log(55, 0)\n        s2.log(38, 0)\n        s2.log(55, 0)\n        s2.log(97, 0)\n\n        s = StatsEntry(self.stats, \"GET\", \"\")\n        s.extend(s1)\n        s.extend(s2)\n\n        self.assertEqual(s.num_requests, 10)\n        self.assertEqual(s.num_failures, 3)\n        self.assertEqual(s.median_response_time, 38)\n        self.assertEqual(s.avg_response_time, 43.2)\n\n    def test_aggregation_with_rounding(self):\n        s1 = StatsEntry(self.stats, \"round me!\", \"GET\")\n        s1.log(122, 0)  # (rounded 120) min\n        s1.log(992, 0)  # (rounded 990) max\n        s1.log(142, 0)  # (rounded 140)\n        s1.log(552, 0)  # (rounded 550)\n        s1.log(557, 0)  # (rounded 560)\n        s1.log(387, 0)  # (rounded 390)\n        s1.log(557, 0)  # (rounded 560)\n        s1.log(977, 0)  # (rounded 980)\n\n        self.assertEqual(s1.num_requests, 8)\n        self.assertEqual(s1.median_response_time, 550)\n        self.assertEqual(s1.avg_response_time, 535.75)\n        self.assertEqual(s1.min_response_time, 122)\n        self.assertEqual(s1.max_response_time, 992)\n\n    def test_aggregation_with_decimal_rounding(self):\n        s1 = StatsEntry(self.stats, \"round me!\", \"GET\")\n        s1.log(1.1, 0)\n        s1.log(1.99, 0)\n        s1.log(3.1, 0)\n        self.assertEqual(s1.num_requests, 3)\n        self.assertEqual(s1.median_response_time, 2)\n        self.assertEqual(s1.avg_response_time, (1.1 + 1.99 + 3.1) / 3)\n        self.assertEqual(s1.min_response_time, 1.1)\n        self.assertEqual(s1.max_response_time, 3.1)\n\n    def test_aggregation_min_response_time(self):\n        s1 = StatsEntry(self.stats, \"min\", \"GET\")\n        s1.log(10, 0)\n        self.assertEqual(10, s1.min_response_time)\n        s2 = StatsEntry(self.stats, \"min\", \"GET\")\n        s1.extend(s2)\n        self.assertEqual(10, s1.min_response_time)\n\n    def test_aggregation_last_request_timestamp(self):\n        s1 = StatsEntry(self.stats, \"r\", \"GET\")\n        s2 = StatsEntry(self.stats, \"r\", \"GET\")\n        s1.extend(s2)\n        self.assertEqual(None, s1.last_request_timestamp)\n        s1 = StatsEntry(self.stats, \"r\", \"GET\")\n        s2 = StatsEntry(self.stats, \"r\", \"GET\")\n        s1.last_request_timestamp = 666\n        s1.extend(s2)\n        self.assertEqual(666, s1.last_request_timestamp)\n        s1 = StatsEntry(self.stats, \"r\", \"GET\")\n        s2 = StatsEntry(self.stats, \"r\", \"GET\")\n        s2.last_request_timestamp = 666\n        s1.extend(s2)\n        self.assertEqual(666, s1.last_request_timestamp)\n        s1 = StatsEntry(self.stats, \"r\", \"GET\")\n        s2 = StatsEntry(self.stats, \"r\", \"GET\")\n        s1.last_request_timestamp = 666\n        s1.last_request_timestamp = 700\n        s1.extend(s2)\n        self.assertEqual(700, s1.last_request_timestamp)\n\n    def test_percentile_rounded_down(self):\n        s1 = StatsEntry(self.stats, \"rounding down!\", \"GET\")\n        s1.log(122, 0)  # (rounded 120) min\n        actual_percentile = s1.percentile().split()\n\n        self.assertEqual(actual_percentile, [\"GET\", \"rounding\", \"down!\"] + [\"120\"] * len(PERCENTILES_TO_REPORT) + [\"1\"])\n\n    def test_percentile_rounded_up(self):\n        s2 = StatsEntry(self.stats, \"rounding up!\", \"GET\")\n        s2.log(127, 0)  # (rounded 130) min\n        actual_percentile = s2.percentile().split()\n        self.assertEqual(actual_percentile, [\"GET\", \"rounding\", \"up!\"] + [\"130\"] * len(PERCENTILES_TO_REPORT) + [\"1\"])\n\n    def test_custom_percentile_list(self):\n        s = StatsEntry(self.stats, \"custom_percentiles\", \"GET\")\n        custom_percentile_list = [0.50, 0.90, 0.95, 0.99]\n        locust.stats.PERCENTILES_TO_REPORT = custom_percentile_list\n        s.log(150, 0)\n        actual_percentile = s.percentile().split()\n        self.assertEqual(\n            actual_percentile, [\"GET\", \"custom_percentiles\"] + [\"150\"] * len(custom_percentile_list) + [\"1\"]\n        )\n\n    def test_error_grouping(self):\n        # reset stats\n        self.stats = RequestStats()\n\n        self.stats.log_error(\"GET\", \"/some-path\", Exception(\"Exception!\"))\n        self.stats.log_error(\"GET\", \"/some-path\", Exception(\"Exception!\"))\n\n        self.assertEqual(1, len(self.stats.errors))\n        self.assertEqual(2, list(self.stats.errors.values())[0].occurrences)\n\n        self.stats.log_error(\"GET\", \"/some-path\", Exception(\"Another exception!\"))\n        self.stats.log_error(\"GET\", \"/some-path\", Exception(\"Another exception!\"))\n        self.stats.log_error(\"GET\", \"/some-path\", Exception(\"Third exception!\"))\n        self.assertEqual(3, len(self.stats.errors))\n\n    def test_error_grouping_errors_with_memory_addresses(self):\n        # reset stats\n        self.stats = RequestStats()\n\n        class Dummy:\n            pass\n\n        self.stats.log_error(\"GET\", \"/\", Exception(f\"Error caused by {Dummy()!r}\"))\n        self.assertEqual(1, len(self.stats.errors))\n\n    def test_serialize_through_message(self):\n        \"\"\"\n        Serialize a RequestStats instance, then serialize it through a Message,\n        and unserialize the whole thing again. This is done \"IRL\" when stats are sent\n        from workers to master.\n        \"\"\"\n        s1 = StatsEntry(self.stats, \"test\", \"GET\")\n        s1.log(10, 0)\n        s1.log(20, 0)\n        s1.log(40, 0)\n        u1 = StatsEntry.unserialize(s1.serialize())\n\n        data = Message.unserialize(Message(\"dummy\", s1.serialize(), \"none\").serialize()).data\n        u1 = StatsEntry.unserialize(data)\n\n        self.assertEqual(20, u1.median_response_time)\n\n\nclass TestStatsPrinting(LocustTestCase):\n    def setUp(self):\n        super().setUp()\n\n        self.stats = RequestStats()\n        for i in range(100):\n            for method, name, freq in [\n                (\n                    \"GET\",\n                    \"test_entry\",\n                    5,\n                ),\n                (\n                    \"DELETE\",\n                    \"test\" * int((STATS_NAME_WIDTH - STATS_TYPE_WIDTH + 4) / len(\"test\")),\n                    3,\n                ),\n            ]:\n                self.stats.log_request(method, name, i, 2000 + i)\n                if i % freq == 0:\n                    self.stats.log_error(method, name, RuntimeError(f\"{method} error\"))\n\n    def test_print_percentile_stats(self):\n        locust.stats.print_percentile_stats(self.stats)\n        info = self.mocked_log.info\n        self.assertEqual(8, len(info))\n        self.assertEqual(\"Response time percentiles (approximated)\", info[0])\n        # check that headline contains same number of column as the value rows\n        headlines = info[1].replace(\"# reqs\", \"#reqs\").split()\n        self.assertEqual(len(headlines), len(info[3].split()))\n        self.assertEqual(len(headlines) - 1, len(info[-2].split()))  # Aggregated, no \"Type\"\n        self.assertEqual(info[2], info[-3])  # table ascii separators\n\n    def test_print_stats(self):\n        locust.stats.print_stats(self.stats)\n        info = self.mocked_log.info\n        self.assertEqual(7, len(info))\n\n        headlines = info[0].replace(\"# \", \"#\").split()\n\n        # check number of columns in separator\n        self.assertEqual(len(headlines), len(info[1].split(\"|\")) + 2)\n        # check entry row\n        self.assertEqual(len(headlines), len(info[2].split()))\n        # check aggregated row, which is missing value in \"type\"-column\n        self.assertEqual(len(headlines) - 1, len(info[-2].split()))\n        # table ascii separators\n        self.assertEqual(info[1], info[-3])\n\n    def test_print_error_report(self):\n        locust.stats.print_error_report(self.stats)\n        info = self.mocked_log.info\n        self.assertEqual(7, len(info))\n        self.assertEqual(\"Error report\", info[0])\n\n        headlines = info[1].replace(\"# \", \"#\").split()\n        # check number of columns in headlines vs table ascii separator\n        self.assertEqual(len(headlines), len(info[2].split(\"|\")))\n        # table ascii seprators\n        self.assertEqual(info[2], info[-2])\n\n\nclass TestCsvStats(LocustTestCase):\n    STATS_BASE_NAME = \"test\"\n    STATS_FILENAME = f\"{STATS_BASE_NAME}_stats.csv\"\n    STATS_HISTORY_FILENAME = f\"{STATS_BASE_NAME}_stats_history.csv\"\n    STATS_FAILURES_FILENAME = f\"{STATS_BASE_NAME}_failures.csv\"\n    STATS_EXCEPTIONS_FILENAME = f\"{STATS_BASE_NAME}_exceptions.csv\"\n\n    def setUp(self):\n        super().setUp()\n        self.remove_file_if_exists(self.STATS_FILENAME)\n        self.remove_file_if_exists(self.STATS_HISTORY_FILENAME)\n        self.remove_file_if_exists(self.STATS_FAILURES_FILENAME)\n        self.remove_file_if_exists(self.STATS_EXCEPTIONS_FILENAME)\n\n    def tearDown(self):\n        self.remove_file_if_exists(self.STATS_FILENAME)\n        self.remove_file_if_exists(self.STATS_HISTORY_FILENAME)\n        self.remove_file_if_exists(self.STATS_FAILURES_FILENAME)\n        self.remove_file_if_exists(self.STATS_EXCEPTIONS_FILENAME)\n\n    def remove_file_if_exists(self, filename):\n        if os.path.exists(filename):\n            os.remove(filename)\n\n    def test_write_csv_files(self):\n        _write_csv_files(self.environment, self.STATS_BASE_NAME)\n        self.assertTrue(os.path.exists(self.STATS_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_HISTORY_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_FAILURES_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_EXCEPTIONS_FILENAME))\n\n    def test_write_csv_files_full_history(self):\n        _write_csv_files(self.environment, self.STATS_BASE_NAME, full_history=True)\n        self.assertTrue(os.path.exists(self.STATS_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_HISTORY_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_FAILURES_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_EXCEPTIONS_FILENAME))\n\n    @mock.patch(\"locust.stats.CSV_STATS_INTERVAL_SEC\", new=_TEST_CSV_STATS_INTERVAL_SEC)\n    def test_csv_stats_writer(self):\n        _write_csv_files(self.environment, self.STATS_BASE_NAME)\n\n        self.assertTrue(os.path.exists(self.STATS_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_HISTORY_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_FAILURES_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_EXCEPTIONS_FILENAME))\n\n        with open(self.STATS_HISTORY_FILENAME) as f:\n            reader = csv.DictReader(f)\n            rows = [r for r in reader]\n\n        self.assertEqual(2, len(rows))\n        self.assertEqual(\"Aggregated\", rows[0][\"Name\"])\n        self.assertEqual(\"Aggregated\", rows[1][\"Name\"])\n\n    @mock.patch(\"locust.stats.CSV_STATS_INTERVAL_SEC\", new=_TEST_CSV_STATS_INTERVAL_SEC)\n    def test_csv_stats_writer_full_history(self):\n        stats_writer = StatsCSVFileWriter(\n            self.environment, PERCENTILES_TO_REPORT, self.STATS_BASE_NAME, full_history=True\n        )\n\n        for i in range(10):\n            self.runner.stats.log_request(\"GET\", \"/\", 100, content_length=666)\n\n        greenlet = gevent.spawn(stats_writer)\n        gevent.sleep(10)\n\n        for i in range(10):\n            self.runner.stats.log_request(\"GET\", \"/\", 10, content_length=666)\n\n        gevent.sleep(5)\n\n        gevent.sleep(_TEST_CSV_STATS_INTERVAL_WAIT_SEC)\n        gevent.kill(greenlet)\n        stats_writer.close_files()\n\n        self.assertTrue(os.path.exists(self.STATS_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_HISTORY_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_FAILURES_FILENAME))\n        self.assertTrue(os.path.exists(self.STATS_EXCEPTIONS_FILENAME))\n\n        with open(self.STATS_HISTORY_FILENAME) as f:\n            reader = csv.DictReader(f)\n            rows = [r for r in reader]\n\n        self.assertGreaterEqual(len(rows), 130)\n\n        self.assertEqual(\"/\", rows[0][\"Name\"])\n        self.assertEqual(\"Aggregated\", rows[1][\"Name\"])\n        self.assertEqual(\"/\", rows[2][\"Name\"])\n        self.assertEqual(\"Aggregated\", rows[3][\"Name\"])\n        self.assertEqual(\"20\", rows[-1][\"Total Request Count\"])\n\n        saw100 = False\n        saw10 = False\n\n        for row in rows:\n            if not saw100 and row[\"95%\"] == \"100\":\n                saw100 = True\n            elif saw100 and row[\"95%\"] == \"10\":\n                saw10 = True\n                break\n\n        self.assertTrue(saw100, \"Never saw 95th percentile increase to 100\")\n        self.assertTrue(saw10, \"Never saw 95th percentile decrease to 10\")\n\n    def test_csv_stats_on_master_from_aggregated_stats(self):\n        # Failing test for: https://github.com/locustio/locust/issues/1315\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            environment = Environment()\n            stats_writer = StatsCSVFileWriter(\n                environment, PERCENTILES_TO_REPORT, self.STATS_BASE_NAME, full_history=True\n            )\n            master = environment.create_master_runner(master_bind_host=\"*\", master_bind_port=0)\n            greenlet = gevent.spawn(stats_writer)\n            gevent.sleep(_TEST_CSV_STATS_INTERVAL_WAIT_SEC)\n\n            server.mocked_send(Message(\"client_ready\", __version__, \"fake_client\"))\n\n            master.stats.entries[(\"/\", \"GET\")].log(100, 23455)\n            master.stats.entries[(\"/\", \"GET\")].log(800, 23455)\n            master.stats.entries[(\"/\", \"GET\")].log(700, 23455)\n\n            data = {\"user_count\": 1}\n            environment.events.report_to_master.fire(client_id=\"fake_client\", data=data)\n            master.stats.clear_all()\n\n            server.mocked_send(Message(\"stats\", data, \"fake_client\"))\n            s = master.stats.entries[(\"/\", \"GET\")]\n            self.assertEqual(700, s.median_response_time)\n\n            gevent.kill(greenlet)\n            stats_writer.close_files()\n\n            self.assertTrue(os.path.exists(self.STATS_FILENAME))\n            self.assertTrue(os.path.exists(self.STATS_HISTORY_FILENAME))\n            self.assertTrue(os.path.exists(self.STATS_FAILURES_FILENAME))\n            self.assertTrue(os.path.exists(self.STATS_EXCEPTIONS_FILENAME))\n\n    @mock.patch(\"locust.stats.CSV_STATS_INTERVAL_SEC\", new=_TEST_CSV_STATS_INTERVAL_SEC)\n    def test_user_count_in_csv_history_stats(self):\n        start_time = int(time.time())\n\n        class TestUser(User):\n            wait_time = constant(10)\n\n            @task\n            def t(self):\n                self.environment.runner.stats.log_request(\"GET\", \"/\", 10, 10)\n\n        environment = Environment(user_classes=[TestUser])\n        stats_writer = StatsCSVFileWriter(environment, PERCENTILES_TO_REPORT, self.STATS_BASE_NAME, full_history=True)\n        runner = environment.create_local_runner()\n        # spawn a user every _TEST_CSV_STATS_INTERVAL_SEC second\n        user_count = 15\n        spawn_rate = 5\n        assert 1 / 5 == _TEST_CSV_STATS_INTERVAL_SEC\n        runner_greenlet = gevent.spawn(runner.start, user_count, spawn_rate)\n        gevent.sleep(0.1)\n\n        greenlet = gevent.spawn(stats_writer)\n        gevent.sleep(user_count / spawn_rate)\n        gevent.kill(greenlet)\n        stats_writer.close_files()\n        runner.stop()\n        gevent.kill(runner_greenlet)\n\n        with open(self.STATS_HISTORY_FILENAME) as f:\n            reader = csv.DictReader(f)\n            rows = [r for r in reader]\n\n        self.assertEqual(2 * user_count, len(rows))\n        for i in range(int(user_count / spawn_rate)):\n            for _ in range(spawn_rate):\n                row = rows.pop(0)\n                self.assertEqual(\"%i\" % ((i + 1) * spawn_rate), row[\"User Count\"])\n                self.assertEqual(\"/\", row[\"Name\"])\n                self.assertEqual(\"%i\" % ((i + 1) * spawn_rate), row[\"Total Request Count\"])\n                self.assertGreaterEqual(int(row[\"Timestamp\"]), start_time)\n                row = rows.pop(0)\n                self.assertEqual(\"%i\" % ((i + 1) * spawn_rate), row[\"User Count\"])\n                self.assertEqual(\"Aggregated\", row[\"Name\"])\n                self.assertEqual(\"%i\" % ((i + 1) * spawn_rate), row[\"Total Request Count\"])\n                self.assertGreaterEqual(int(row[\"Timestamp\"]), start_time)\n\n    def test_requests_csv_quote_escaping(self):\n        with mock.patch(\"locust.rpc.rpc.Server\", mocked_rpc()) as server:\n            environment = Environment()\n            master = environment.create_master_runner(master_bind_host=\"*\", master_bind_port=0)\n            server.mocked_send(Message(\"client_ready\", __version__, \"fake_client\"))\n\n            request_name_dict = {\n                \"scenario\": \"get cashes\",\n                \"path\": \"/cash/[amount]\",\n                \"arguments\": [{\"size\": 1}],\n            }\n            request_name_str = json.dumps(request_name_dict)\n\n            master.stats.entries[(request_name_str, \"GET\")].log(100, 23455)\n            data = {\"user_count\": 1}\n            environment.events.report_to_master.fire(client_id=\"fake_client\", data=data)\n            master.stats.clear_all()\n            server.mocked_send(Message(\"stats\", data, \"fake_client\"))\n\n            _write_csv_files(environment, self.STATS_BASE_NAME, full_history=True)\n            with open(self.STATS_FILENAME) as f:\n                reader = csv.DictReader(f)\n                rows = [r for r in reader]\n                csv_request_name = rows[0].get(\"Name\")\n                self.assertEqual(request_name_str, csv_request_name)\n\n    def test_stats_history(self):\n        env1 = Environment(events=locust.events, catch_exceptions=False)\n        runner1 = env1.create_master_runner(\"127.0.0.1\", 5558)\n        env2 = Environment(events=locust.events, catch_exceptions=False)\n        runner2 = env2.create_worker_runner(\"127.0.0.1\", 5558)\n        greenlet1 = gevent.spawn(stats_history, runner1)\n        greenlet2 = gevent.spawn(stats_history, runner2)\n        gevent.sleep(1)\n        hs1 = runner1.stats.history\n        hs2 = runner2.stats.history\n        gevent.kill(greenlet1)\n        gevent.kill(greenlet2)\n        self.assertEqual(1, len(hs1))\n        self.assertEqual(0, len(hs2))\n\n\nclass TestStatsEntryResponseTimesCache(unittest.TestCase):\n    def setUp(self, *args, **kwargs):\n        super().setUp(*args, **kwargs)\n        self.stats = RequestStats()\n\n    def test_response_times_cached(self):\n        s = StatsEntry(self.stats, \"/\", \"GET\", use_response_times_cache=True)\n        self.assertEqual(1, len(s.response_times_cache))\n        s.log(11, 1337)\n        self.assertEqual(1, len(s.response_times_cache))\n        s.last_request_timestamp -= 1\n        s.log(666, 1337)\n        self.assertEqual(2, len(s.response_times_cache))\n        self.assertEqual(\n            CachedResponseTimes(\n                response_times={11: 1},\n                num_requests=1,\n            ),\n            s.response_times_cache[int(s.last_request_timestamp) - 1],\n        )\n\n    def test_response_times_not_cached_if_not_enabled(self):\n        s = StatsEntry(self.stats, \"/\", \"GET\")\n        s.log(11, 1337)\n        self.assertEqual(None, s.response_times_cache)\n        s.last_request_timestamp -= 1\n        s.log(666, 1337)\n        self.assertEqual(None, s.response_times_cache)\n\n    def test_latest_total_response_times_pruned(self):\n        \"\"\"\n        Check that RequestStats.latest_total_response_times are pruned when exceeding 20 entries\n        \"\"\"\n        s = StatsEntry(self.stats, \"/\", \"GET\", use_response_times_cache=True)\n        t = int(time.time())\n        for i in reversed(range(2, 30)):\n            s.response_times_cache[t - i] = CachedResponseTimes(response_times={}, num_requests=0)\n        self.assertEqual(29, len(s.response_times_cache))\n        s.log(17, 1337)\n        s.last_request_timestamp -= 1\n        s.log(1, 1)\n        self.assertEqual(20, len(s.response_times_cache))\n        self.assertEqual(\n            CachedResponseTimes(response_times={17: 1}, num_requests=1),\n            s.response_times_cache.popitem(last=True)[1],\n        )\n\n    def test_get_current_response_time_percentile(self):\n        s = StatsEntry(self.stats, \"/\", \"GET\", use_response_times_cache=True)\n        t = int(time.time())\n        s.response_times_cache[t - 10] = CachedResponseTimes(\n            response_times={i: 1 for i in range(100)}, num_requests=200\n        )\n        s.response_times_cache[t - 10].response_times[1] = 201\n\n        s.response_times = {i: 2 for i in range(100)}\n        s.response_times[1] = 202\n        s.num_requests = 300\n\n        self.assertEqual(95, s.get_current_response_time_percentile(0.95))\n\n    def test_get_current_response_time_percentile_outside_cache_window(self):\n        s = StatsEntry(self.stats, \"/\", \"GET\", use_response_times_cache=True)\n        # an empty response times cache, current time will not be in this cache\n        s.response_times_cache = {}\n        self.assertEqual(None, s.get_current_response_time_percentile(0.95))\n\n    def test_diff_response_times_dicts(self):\n        self.assertEqual(\n            {1: 5, 6: 8},\n            diff_response_time_dicts(\n                {1: 6, 6: 16, 2: 2},\n                {1: 1, 6: 8, 2: 2},\n            ),\n        )\n        self.assertEqual(\n            {},\n            diff_response_time_dicts(\n                {},\n                {},\n            ),\n        )\n        self.assertEqual(\n            {10: 15},\n            diff_response_time_dicts(\n                {10: 15},\n                {},\n            ),\n        )\n        self.assertEqual(\n            {10: 10},\n            diff_response_time_dicts(\n                {10: 10},\n                {},\n            ),\n        )\n        self.assertEqual(\n            {},\n            diff_response_time_dicts(\n                {1: 1},\n                {1: 1},\n            ),\n        )\n\n\nclass TestStatsEntry(unittest.TestCase):\n    def parse_string_output(self, text):\n        tokenlist = re.split(r\"[\\s\\(\\)%|]+\", text.strip())\n        tokens = {\n            \"method\": tokenlist[0],\n            \"name\": tokenlist[1],\n            \"request_count\": int(tokenlist[2]),\n            \"failure_count\": int(tokenlist[3]),\n            \"failure_percentage\": float(tokenlist[4]),\n        }\n        return tokens\n\n    def setUp(self, *args, **kwargs):\n        super().setUp(*args, **kwargs)\n        self.stats = RequestStats()\n\n    def test_fail_ratio_with_no_failures(self):\n        REQUEST_COUNT = 10\n        FAILURE_COUNT = 0\n        EXPECTED_FAIL_RATIO = 0.0\n\n        s = StatsEntry(self.stats, \"/\", \"GET\")\n        s.num_requests = REQUEST_COUNT\n        s.num_failures = FAILURE_COUNT\n\n        self.assertAlmostEqual(s.fail_ratio, EXPECTED_FAIL_RATIO)\n        output_fields = self.parse_string_output(str(s))\n        self.assertEqual(output_fields[\"request_count\"], REQUEST_COUNT)\n        self.assertEqual(output_fields[\"failure_count\"], FAILURE_COUNT)\n        self.assertAlmostEqual(output_fields[\"failure_percentage\"], EXPECTED_FAIL_RATIO * 100)\n\n    def test_fail_ratio_with_all_failures(self):\n        REQUEST_COUNT = 10\n        FAILURE_COUNT = 10\n        EXPECTED_FAIL_RATIO = 1.0\n\n        s = StatsEntry(self.stats, \"/\", \"GET\")\n        s.num_requests = REQUEST_COUNT\n        s.num_failures = FAILURE_COUNT\n\n        self.assertAlmostEqual(s.fail_ratio, EXPECTED_FAIL_RATIO)\n        output_fields = self.parse_string_output(str(s))\n        self.assertEqual(output_fields[\"request_count\"], REQUEST_COUNT)\n        self.assertEqual(output_fields[\"failure_count\"], FAILURE_COUNT)\n        self.assertAlmostEqual(output_fields[\"failure_percentage\"], EXPECTED_FAIL_RATIO * 100)\n\n    def test_fail_ratio_with_half_failures(self):\n        REQUEST_COUNT = 10\n        FAILURE_COUNT = 5\n        EXPECTED_FAIL_RATIO = 0.5\n\n        s = StatsEntry(self.stats, \"/\", \"GET\")\n        s.num_requests = REQUEST_COUNT\n        s.num_failures = FAILURE_COUNT\n\n        self.assertAlmostEqual(s.fail_ratio, EXPECTED_FAIL_RATIO)\n        output_fields = self.parse_string_output(str(s))\n        self.assertEqual(output_fields[\"request_count\"], REQUEST_COUNT)\n        self.assertEqual(output_fields[\"failure_count\"], FAILURE_COUNT)\n        self.assertAlmostEqual(output_fields[\"failure_percentage\"], EXPECTED_FAIL_RATIO * 100)\n\n\nclass TestRequestStatsWithWebserver(WebserverTestCase):\n    def setUp(self):\n        super().setUp()\n\n        class MyUser(HttpUser):\n            host = \"http://127.0.0.1:%i\" % self.port\n\n        self.locust = MyUser(self.environment)\n\n    def test_request_stats_content_length(self):\n        self.locust.client.get(\"/ultra_fast\")\n        self.assertEqual(\n            self.runner.stats.entries[(\"/ultra_fast\", \"GET\")].avg_content_length, len(\"This is an ultra fast response\")\n        )\n        self.locust.client.get(\"/ultra_fast\")\n        # test legacy stats.get() function sometimes too\n        self.assertEqual(\n            self.runner.stats.get(\"/ultra_fast\", \"GET\").avg_content_length, len(\"This is an ultra fast response\")\n        )\n\n    def test_request_stats_no_content_length(self):\n        path = \"/no_content_length\"\n        self.locust.client.get(path)\n        self.assertEqual(\n            self.runner.stats.entries[(path, \"GET\")].avg_content_length,\n            len(\"This response does not have content-length in the header\"),\n        )\n\n    def test_request_stats_no_content_length_streaming(self):\n        path = \"/no_content_length\"\n        self.locust.client.get(path, stream=True)\n        self.assertEqual(0, self.runner.stats.entries[(path, \"GET\")].avg_content_length)\n\n    def test_request_stats_named_endpoint(self):\n        self.locust.client.get(\"/ultra_fast\", name=\"my_custom_name\")\n        self.assertEqual(1, self.runner.stats.entries[(\"my_custom_name\", \"GET\")].num_requests)\n\n    def test_request_stats_named_endpoint_request_name(self):\n        self.locust.client.request_name = \"my_custom_name_1\"\n        self.locust.client.get(\"/ultra_fast\")\n        self.assertEqual(1, self.runner.stats.entries[(\"my_custom_name_1\", \"GET\")].num_requests)\n        self.locust.client.request_name = None\n\n    \ndef test_request_stats_named_endpoint_rename_request(self):\n        with self.locust.client.rename_request(\"my_custom_name_3\"):\n            self.locust.client.get(\"/ultra_fast\")\n        self.assertEqual(1, self.runner.stats.entries[(\"my_custom_name_3\", \"GET\")].num_requests)\n\n    def test_request_stats_query_variables(self):\n        self.locust.client.get(\"/ultra_fast?query=1\")\n        self.assertEqual(1, self.runner.stats.entries[(\"/ultra_fast?query=1\", \"GET\")].num_requests)\n\n    def test_request_stats_put(self):\n        self.locust.client.put(\"/put\")\n        self.assertEqual(1, self.runner.stats.entries[(\"/put\", \"PUT\")].num_requests)\n\n    def test_request_connection_error(self):\n        class MyUser(HttpUser):\n            host = \"http://localhost:1\"\n\n        locust = MyUser(self.environment)\n        response = locust.client.get(\"/\", timeout=0.1)\n        self.assertEqual(response.status_code, 0)\n        self.assertEqual(1, self.runner.stats.entries[(\"/\", \"GET\")].num_failures)\n        self.assertEqual(1, self.runner.stats.entries[(\"/\", \"GET\")].num_requests)\n\n\nclass MyTaskSet(TaskSet):\n    @task(75)\n    def root_task(self):\n        pass\n\n    @task(25)\n    class MySubTaskSet(TaskSet):\n        @task\n        def task1(self):\n            pass\n\n        @task\n        def task2(self):\n            pass\n\n\nclass TestInspectUser(unittest.TestCase):\n    def test_get_task_ratio_relative(self):\n        ratio = _get_task_ratio([MyTaskSet], False, 1.0)\n        self.assertEqual(1.0, ratio[\"MyTaskSet\"][\"ratio\"])\n        self.assertEqual(0.75, ratio[\"MyTaskSet\"][\"tasks\"][\"root_task\"][\"ratio\"])\n        self.assertEqual(0.25, ratio[\"MyTaskSet\"][\"tasks\"][\"MySubTaskSet\"][\"ratio\"])\n        self.assertEqual(0.5, ratio[\"MyTaskSet\"][\"tasks\"][\"MySubTaskSet\"][\"tasks\"][\"task1\"][\"ratio\"])\n        self.assertEqual(0.5, ratio[\"MyTaskSet\"][\"tasks\"][\"MySubTaskSet\"][\"tasks\"][\"task2\"][\"ratio\"])\n\n    def test_get_task_ratio_total(self):\n        ratio = _get_task_ratio([MyTaskSet], True, 1.0)\n        self.assertEqual(1.0, ratio[\"MyTaskSet\"][\"ratio\"])\n        self.assertEqual(0.75, ratio[\"MyTaskSet\"][\"tasks\"][\"root_task\"][\"ratio\"])\n        self.assertEqual(0.25, ratio[\"MyTaskSet\"][\"tasks\"][\"MySubTaskSet\"][\"ratio\"])\n        self.assertEqual(0.125, ratio[\"MyTaskSet\"][\"tasks\"][\"MySubTaskSet\"][\"tasks\"][\"task1\"][\"ratio\"])\n        self.assertEqual(0.125, ratio[\"MyTaskSet\"][\"tasks\"][\"MySubTaskSet\"][\"tasks\"][\"task2\"][\"ratio\"])\n\n# Path: locust/test/test_tags.py\nfrom locust import TaskSet, User, tag, task\nfrom locust.env import Environment\nfrom locust.user.task import filter_tasks_by_tags\n\nfrom .testcases import LocustTestCase\n\n\nclass TestTags(LocustTestCase):\n    def test_tagging(self):\n        @tag(\"tag1\")\n        @task\n        def tagged():\n            pass\n\n        self.assertIn(\"locust_tag_set\", dir(tagged))\n        self.assertEqual({\"tag1\"}, tagged.locust_tag_set)\n\n        @tag(\"tag2\", \"tag3\")\n        @task\n        def tagged_multiple_args():\n            pass\n\n        self.assertIn(\"locust_tag_set\", dir(tagged_multiple_args))\n        self.assertEqual({\"tag2\", \"tag3\"}, tagged_multiple_args.locust_tag_set)\n\n        @tag(\"tag4\")\n        @tag(\"tag5\")\n        @task\n        def tagged_multiple_times():\n            pass\n\n        self.assertIn(\"locust_tag_set\", dir(tagged_multiple_times))\n        self.assertEqual({\"tag4\", \"tag5\"}, tagged_multiple_times.locust_tag_set)\n\n    def test_tagging_taskset(self):\n        @tag(\"taskset\")\n        @task\n        class MyTaskSet(TaskSet):\n            @task\n            def tagged(self):\n                pass\n\n            @tag(\"task\")\n            @task\n            def tagged_again(self):\n                pass\n\n            @tag(\"taskset2\")\n            @task\n            class NestedTaskSet(TaskSet):\n                @task\n                def nested_task(self):\n                    pass\n\n        # when tagging taskset, its tasks receive the tag\n        self.assertIn(\"locust_tag_set\", dir(MyTaskSet.tagged))\n        self.assertEqual({\"taskset\"}, MyTaskSet.tagged.locust_tag_set)\n\n        # tagging inner task receives both\n        self.assertIn(\"locust_tag_set\", dir(MyTaskSet.tagged_again))\n        self.assertEqual({\"taskset\", \"task\"}, MyTaskSet.tagged_again.locust_tag_set)\n\n        # when tagging nested taskset, its tasks receives both\n        self.assertIn(\"locust_tag_set\", dir(MyTaskSet.NestedTaskSet.nested_task))\n        self.assertEqual({\"taskset\", \"taskset2\"}, MyTaskSet.NestedTaskSet.nested_task.locust_tag_set)\n\n    def test_tagging_without_args_fails(self):\n        @task\n        def dummy_task(self):\n            pass\n\n        # task is tagged without parens\n        self.assertRaises(ValueError, lambda: tag(dummy_task))\n\n        # task is tagged with empty parens\n        self.assertRaises(ValueError, lambda: tag()(dummy_task))\n\n    def test_including_tags(self):\n        class MyTaskSet(TaskSet):\n            @tag(\"include this\", \"other tag\")\n            @task\n            def included(self):\n                pass\n\n            @tag(\"dont include this\", \"other tag\")\n            @task\n            def not_included(self):\n                pass\n\n            @task\n            def dont_include_this_either(self):\n                pass\n\n        self.assertListEqual(\n            MyTaskSet.tasks, [MyTaskSet.included, MyTaskSet.not_included, MyTaskSet.dont_include_this_either]\n        )\n\n        filter_tasks_by_tags(MyTaskSet, tags={\"include this\"})\n        self.assertListEqual(MyTaskSet.tasks, [MyTaskSet.included])\n\n    def test_excluding_tags(self):\n        class MyTaskSet(TaskSet):\n            @tag(\"exclude this\", \"other tag\")\n            @task\n            def excluded(self):\n                pass\n\n            @tag(\"dont exclude this\", \"other tag\")\n            @task\n            def not_excluded(self):\n                pass\n\n            @task\n            def dont_exclude_this_either(self):\n                pass\n\n        self.assertListEqual(\n            MyTaskSet.tasks, [MyTaskSet.excluded, MyTaskSet.not_excluded, MyTaskSet.dont_exclude_this_either]\n        )\n\n        filter_tasks_by_tags(MyTaskSet, exclude_tags={\"exclude this\"})\n        self.assertListEqual(MyTaskSet.tasks, [MyTaskSet.not_excluded, MyTaskSet.dont_exclude_this_either])\n\n    def test_including_and_excluding(self):\n        class MyTaskSet(TaskSet):\n            @task\n            def not_included_or_excluded(self):\n                pass\n\n            @tag(\"included\")\n            @task\n            def included(self):\n                pass\n\n            @tag(\"excluded\")\n            @task\n            def excluded(self):\n                pass\n\n            @tag(\"included\", \"excluded\")\n            @task\n            def included_and_excluded(self):\n                pass\n\n        filter_tasks_by_tags(MyTaskSet, tags={\"included\"}, exclude_tags={\"excluded\"})\n        self.assertListEqual(MyTaskSet.tasks, [MyTaskSet.included])\n\n    def test_including_tasksets(self):\n        class MyTaskSet(TaskSet):\n            @task\n            class MixedNestedTaskSet(TaskSet):\n                @tag(\"included\")\n                @task\n                def included(self):\n                    pass\n\n                @task\n                def not_included(self):\n                    pass\n\n            @tag(\"included\")\n            @task\n            class TaggedNestedTaskSet(TaskSet):\n                @task\n                def included(self):\n                    pass\n\n            @task\n            class NormalNestedTaskSet(TaskSet):\n                @task\n                def not_included(self):\n                    pass\n\n        filter_tasks_by_tags(MyTaskSet, tags={\"included\"})\n        self.assertListEqual(MyTaskSet.tasks, [MyTaskSet.MixedNestedTaskSet, MyTaskSet.TaggedNestedTaskSet])\n        self.assertListEqual(MyTaskSet.MixedNestedTaskSet.tasks, [MyTaskSet.MixedNestedTaskSet.included])\n\n    def test_excluding_tasksets(self):\n        class MyTaskSet(TaskSet):\n            @task\n            class MixedNestedTaskSet(TaskSet):\n                @tag(\"excluded\")\n                @task\n                def excluded(self):\n                    pass\n\n                @task\n                def not_excluded(self):\n                    pass\n\n            @task\n            class ExcludedNestedTaskSet(TaskSet):\n                @tag(\"excluded\")\n                @task\n                def excluded(self):\n                    pass\n\n            @tag(\"excluded\")\n            @task\n            class TaggedNestedTaskSet(TaskSet):\n                @task\n                def excluded(self):\n                    pass\n\n            @task\n            class NormalNestedTaskSet(TaskSet):\n                @task\n                def not_excluded(self):\n                    pass\n\n        filter_tasks_by_tags(MyTaskSet, exclude_tags={\"excluded\"})\n        self.assertListEqual(MyTaskSet.tasks, [MyTaskSet.MixedNestedTaskSet, MyTaskSet.NormalNestedTaskSet])\n        self.assertListEqual(MyTaskSet.MixedNestedTaskSet.tasks, [MyTaskSet.MixedNestedTaskSet.not_excluded])\n\n    def test_including_tags_with_weights(self):\n        class MyTaskSet(TaskSet):\n            @tag(\"included\")\n            @task(2)\n            def include_twice(self):\n                pass\n\n            @tag(\"included\")\n            @task(3)\n            def include_3_times(self):\n                pass\n\n            @tag(\"dont include this\")\n            @task(4)\n            def dont_include_4_times(self):\n                pass\n\n            @task(5)\n            def dont_include_5_times(self):\n                pass\n\n        self.assertListEqual(\n            MyTaskSet.tasks,\n            [\n                MyTaskSet.include_twice,\n                MyTaskSet.include_twice,\n                MyTaskSet.include_3_times,\n                MyTaskSet.include_3_times,\n                MyTaskSet.include_3_times,\n                MyTaskSet.dont_include_4_times,\n                MyTaskSet.dont_include_4_times,\n                MyTaskSet.dont_include_4_times,\n                MyTaskSet.dont_include_4_times,\n                MyTaskSet.dont_include_5_times,\n                MyTaskSet.dont_include_5_times,\n                MyTaskSet.dont_include_5_times,\n                MyTaskSet.dont_include_5_times,\n                MyTaskSet.dont_include_5_times,\n            ],\n        )\n\n        filter_tasks_by_tags(MyTaskSet, tags={\"included\"})\n\n        self.assertListEqual(\n            MyTaskSet.tasks,\n            [\n                MyTaskSet.include_twice,\n                MyTaskSet.include_twice,\n                MyTaskSet.include_3_times,\n                MyTaskSet.include_3_times,\n                MyTaskSet.include_3_times,\n            ],\n        )\n\n    def test_excluding_tags_with_weights(self):\n        class MyTaskSet(TaskSet):\n            @tag(\"dont exclude this\")\n            @task(2)\n            def dont_exclude_twice(self):\n                pass\n\n            @task(3)\n            def dont_exclude_3_times(self):\n                pass\n\n            @tag(\"excluded\")\n            @task(4)\n            def exclude_4_times(self):\n                pass\n\n            @tag(\"excluded\")\n            @task(5)\n            def exclude_5_times(self):\n                pass\n\n        self.assertListEqual(\n            MyTaskSet.tasks,\n            [\n                MyTaskSet.dont_exclude_twice,\n                MyTaskSet.dont_exclude_twice,\n                MyTaskSet.dont_exclude_3_times,\n                MyTaskSet.dont_exclude_3_times,\n                MyTaskSet.dont_exclude_3_times,\n                MyTaskSet.exclude_4_times,\n                MyTaskSet.exclude_4_times,\n                MyTaskSet.exclude_4_times,\n                MyTaskSet.exclude_4_times,\n                MyTaskSet.exclude_5_times,\n                MyTaskSet.exclude_5_times,\n                MyTaskSet.exclude_5_times,\n                MyTaskSet.exclude_5_times,\n                MyTaskSet.exclude_5_times,\n            ],\n        )\n\n        filter_tasks_by_tags(MyTaskSet, exclude_tags={\"excluded\"})\n\n        self.assertListEqual(\n            MyTaskSet.tasks,\n            [\n                MyTaskSet.dont_exclude_twice,\n                MyTaskSet.dont_exclude_twice,\n                MyTaskSet.dont_exclude_3_times,\n                MyTaskSet.dont_exclude_3_times,\n                MyTaskSet.dont_exclude_3_times,\n            ],\n        )\n\n    def test_tagged_tasks_shared_across_tasksets(self):\n        @tag(\"tagged\")\n        def shared_task():\n            pass\n\n        def untagged_shared_task():\n            pass\n\n        @tag(\"tagged\")\n        class SharedTaskSet(TaskSet):\n            @task\n            def inner_task(self):\n                pass\n\n        class IncludeTaskSet(TaskSet):\n            tasks = [shared_task, untagged_shared_task, SharedTaskSet]\n\n        class ExcludeTaskSet(TaskSet):\n            tasks = [shared_task, untagged_shared_task, SharedTaskSet]\n\n        filter_tasks_by_tags(IncludeTaskSet, tags={\"tagged\"})\n\n        self.assertListEqual(IncludeTaskSet.tasks, [shared_task, SharedTaskSet])\n        self.assertListEqual(IncludeTaskSet.tasks[1].tasks, [SharedTaskSet.inner_task])\n\n        filter_tasks_by_tags(ExcludeTaskSet, exclude_tags={\"tagged\"})\n\n        self.assertListEqual(ExcludeTaskSet.tasks, [untagged_shared_task])\n\n    def test_include_tags_under_user(self):\n        class MyUser(User):\n            @tag(\"include this\")\n            @task\n            def included(self):\n                pass\n\n            @tag(\"dont include this\")\n            @task\n            def not_included(self):\n                pass\n\n            @task\n            def dont_include_this_either(self):\n                pass\n\n        filter_tasks_by_tags(MyUser, tags={\"include this\"})\n\n        self.assertListEqual(MyUser.tasks, [MyUser.included])\n\n    def test_exclude_tags_under_user(self):\n        class MyUser(User):\n            @tag(\"exclude this\")\n            @task\n            def excluded(self):\n                pass\n\n            @tag(\"dont exclude this\")\n            @task\n            def not_excluded(self):\n                pass\n\n            @task\n            def dont_exclude_this_either(self):\n                pass\n\n        filter_tasks_by_tags(MyUser, exclude_tags={\"exclude this\"})\n\n        self.assertListEqual(MyUser.tasks, [MyUser.not_excluded, MyUser.dont_exclude_this_either])\n\n    def test_env_include_tags(self):\n        class MyTaskSet(TaskSet):\n            @tag(\"include this\")\n            @task\n            def included(self):\n                pass\n\n            @tag(\"dont include this\")\n            @task\n            def not_included(self):\n                pass\n\n            @task\n            def dont_include_this_either(self):\n                pass\n\n        class MyUser(User):\n            tasks = [MyTaskSet]\n\n        env = Environment(user_classes=[MyUser], tags=[\"include this\"])\n        env._filter_tasks_by_tags()\n\n        self.assertListEqual(MyUser.tasks, [MyTaskSet])\n        self.assertListEqual(MyUser.tasks[0].tasks, [MyTaskSet.included])\n\n    def test_env_exclude_tags(self):\n        class MyTaskSet(User):\n            @tag(\"exclude this\")\n            @task\n            def excluded(self):\n                pass\n\n            @tag(\"dont exclude this\")\n            @task\n            def not_excluded(self):\n                pass\n\n            @task\n            def dont_exclude_this_either(self):\n                pass\n\n        class MyUser(User):\n            tasks = [MyTaskSet]\n\n        env = Environment(user_classes=[MyUser], exclude_tags=[\"exclude this\"])\n        env._filter_tasks_by_tags()\n\n        self.assertListEqual(MyUser.tasks, [MyTaskSet])\n        self.assertListEqual(MyUser.tasks[0].tasks, [MyTaskSet.not_excluded, MyTaskSet.dont_exclude_this_either])\n\n# Path: locust/test/test_taskratio.py\nfrom locust.user import TaskSet, User, task\nfrom locust.user.inspectuser import _get_task_ratio, get_ratio\n\nimport unittest\n\n\nclass TestTaskRatio(unittest.TestCase):\n    def test_task_ratio_command(self):\n        class Tasks(TaskSet):\n            @task\n            def root_task1(self):\n                pass\n\n            @task\n            class SubTasks(TaskSet):\n                @task\n                def task1(self):\n                    pass\n\n                @task\n                def task2(self):\n                    pass\n\n        class MyUser(User):\n            tasks = [Tasks]\n\n        ratio_dict = _get_task_ratio(Tasks.tasks, True, 1.0)\n\n        self.assertEqual(\n            {\n                \"SubTasks\": {\"tasks\": {\"task1\": {\"ratio\": 0.25}, \"task2\": {\"ratio\": 0.25}}, \"ratio\": 0.5},\n                \"root_task1\": {\"ratio\": 0.5},\n            },\n            ratio_dict,\n        )\n\n    def test_task_ratio_command_with_locust_weight(self):\n        class Tasks(TaskSet):\n            @task(1)\n            def task1(self):\n                pass\n\n            @task(3)\n            def task3(self):\n                pass\n\n        class UnlikelyUser(User):\n            weight = 1\n            tasks = [Tasks]\n\n        class MoreLikelyUser(User):\n            weight = 3\n            tasks = [Tasks]\n\n        ratio_dict = get_ratio([UnlikelyUser, MoreLikelyUser], {\"UnlikelyUser\": 1, \"MoreLikelyUser\": 3}, True)\n\n        self.assertDictEqual(\n            {\n                \"UnlikelyUser\": {\n                    \"ratio\": 0.25,\n                    \"tasks\": {\n                        \"Tasks\": {\n                            \"tasks\": {\n                                \"task1\": {\"ratio\": 0.25 * 0.25},\n                                \"task3\": {\"ratio\": 0.25 * 0.75},\n                            },\n                            \"ratio\": 0.25,\n                        }\n                    },\n                },\n                \"MoreLikelyUser\": {\n                    \"ratio\": 0.75,\n                    \"tasks\": {\n                        \"Tasks\": {\n                            \"tasks\": {\n                                \"task1\": {\"ratio\": 0.75 * 0.25},\n                                \"task3\": {\"ratio\": 0.75 * 0.75},\n                            },\n                            \"ratio\": 0.75,\n                        },\n                    },\n                },\n            },\n            ratio_dict,\n        )\n        unlikely = ratio_dict[\"UnlikelyUser\"][\"tasks\"][\"Tasks\"][\"tasks\"]\n        likely = ratio_dict[\"MoreLikelyUser\"][\"tasks\"][\"Tasks\"][\"tasks\"]\n        assert (\n            unlikely[\"task1\"][\"ratio\"]\n            + unlikely[\"task3\"][\"ratio\"]\n            + likely[\"task1\"][\"ratio\"]\n            + likely[\"task3\"][\"ratio\"]\n            == 1\n        )\n\n# Path: locust/test/test_users.py\nfrom locust import HttpUser, User\nfrom locust.test.testcases import WebserverTestCase\n\nimport unittest\n\nfrom urllib3 import PoolManager\n\n\nclass TestUserClass(unittest.TestCase):\n    class MyClassScopedUser(User):\n        pass\n\n    def test_fullname_module_scoped(self):\n        self.assertEqual(MyModuleScopedUser.fullname(), \"locust.test.test_users.MyModuleScopedUser\")\n\n    def test_fullname_class_scoped(self):\n...\n", "needle_token_start": 10468, "needle_token_end": 10560, "code_context_ntokens": 16108, "output": ["```python\ndef test_request_stats_named_endpoint_rename_request(self):\n    with self.locust.client.rename_request(\"my_custom_name_3\"):\n        self.locust.client.get(\"/ultra_fast\")\n    self.assertEqual(1, self.runner.stats.entries[(\"my_custom_name_3\", \"GET\")].num_requests)\n```"]}
{"repo": "locustio/locust", "name": "is_valid_percentile", "language": "python", "path": "locust/main.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To verify if a given parameter is a valid percentile value, specifically ensuring it is a decimal between 0 and 1 (exclusive).\n2. **Input**: A single parameter, expected to be a numeric value or a string that can be converted to a float.\n3. **Output**: A boolean value; `True` if the input is a valid percentile, otherwise `False`.\n4. **Procedure**: The function first attempts to convert the input to a float. If successful, it checks whether the float value is greater than 0 and less than 1. If both conditions are met, it returns `True`. If the conversion fails or the conditions are not met, it returns `False`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: locust/contrib/fasthttp.py\nfrom __future__ import annotations\n\nfrom locust.env import Environment\nfrom locust.exception import CatchResponseError, LocustError, ResponseError\nfrom locust.user import User\nfrom locust.util.deprecation import DeprecatedFastHttpLocustClass as FastHttpLocust\n\nimport json\nimport json as unshadowed_json  # some methods take a named parameter called json\nimport re\nimport socket\nimport time\nimport traceback\nfrom base64 import b64encode\nfrom contextlib import contextmanager\nfrom http.cookiejar import CookieJar\nfrom json.decoder import JSONDecodeError\nfrom ssl import SSLError\nfrom typing import Any, Callable, Generator, cast\nfrom urllib.parse import urlparse, urlunparse\n\nimport gevent\nfrom charset_normalizer import detect\nfrom gevent.timeout import Timeout\nfrom geventhttpclient._parser import HTTPParseError\nfrom geventhttpclient.client import HTTPClientPool\nfrom geventhttpclient.header import Headers\nfrom geventhttpclient.response import HTTPConnectionClosed, HTTPSocketPoolResponse\nfrom geventhttpclient.useragent import CompatRequest, CompatResponse, ConnectionError, UserAgent\n\n# borrow requests's content-type header parsing\nfrom requests.utils import get_encoding_from_headers\n\n# Monkey patch geventhttpclient.useragent.CompatRequest so that Cookiejar works with Python >= 3.3\n# More info: https://github.com/requests/requests/pull/871\nCompatRequest.unverifiable = False\n\n# Workaround for AttributeError: 'CompatRequest' object has no attribute 'type' in Cookiejar\n# https://github.com/locustio/locust/issues/1138\n# Might allow secure cookies over non-secure connections but that is a minor concern in a load testing tool\nCompatRequest.type = \"https\"\n\n# Regexp for checking if an absolute URL was specified\nabsolute_http_url_regexp = re.compile(r\"^https?://\", re.I)\n\n# List of exceptions that can be raised by geventhttpclient when sending an HTTP request,\n# and that should result in a Locust failure\nFAILURE_EXCEPTIONS = (\n    ConnectionError,\n    ConnectionRefusedError,\n    ConnectionResetError,\n    socket.error,\n    SSLError,\n    Timeout,\n    HTTPConnectionClosed,\n)\n\n\ndef _construct_basic_auth_str(username, password):\n    \"\"\"Construct Authorization header value to be used in HTTP Basic Auth\"\"\"\n    if isinstance(username, str):\n        username = username.encode(\"latin1\")\n    if isinstance(password, str):\n        password = password.encode(\"latin1\")\n    return \"Basic \" + b64encode(b\":\".join((username, password))).strip().decode(\"ascii\")\n\n\ndef insecure_ssl_context_factory():\n    context = gevent.ssl.create_default_context()\n    context.check_hostname = False\n    context.verify_mode = gevent.ssl.CERT_NONE\n    return context\n\n\nclass FastHttpSession:\n    auth_header = None\n\n    def __init__(\n        self,\n        environment: Environment,\n        base_url: str,\n        user: User | None,\n        insecure=True,\n        client_pool: HTTPClientPool | None = None,\n        ssl_context_factory: Callable | None = None,\n        **kwargs,\n    ):\n        self.environment = environment\n        self.base_url = base_url\n        self.cookiejar = CookieJar()\n        self.user = user\n        if not ssl_context_factory:\n            if insecure:\n                ssl_context_factory = insecure_ssl_context_factory\n            else:\n                ssl_context_factory = gevent.ssl.create_default_context\n        self.client = LocustUserAgent(\n            cookiejar=self.cookiejar,\n            ssl_context_factory=ssl_context_factory,\n            insecure=insecure,\n            client_pool=client_pool,\n            **kwargs,\n        )\n\n        # Check for basic authentication\n        parsed_url = urlparse(self.base_url)\n        if parsed_url.username and parsed_url.password:\n            netloc = parsed_url.hostname or \"\"\n            if parsed_url.port:\n                netloc += \":%d\" % parsed_url.port\n\n            # remove username and password from the base_url\n            self.base_url = urlunparse(\n                (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)\n            )\n            # store authentication header (we construct this by using _basic_auth_str() function from requests.auth)\n            self.auth_header = _construct_basic_auth_str(parsed_url.username, parsed_url.password)\n\n    def _build_url(self, path):\n        \"\"\"prepend url with hostname unless it's already an absolute URL\"\"\"\n        if absolute_http_url_regexp.match(path):\n            return path\n        else:\n            return f\"{self.base_url}{path}\"\n\n    def _send_request_safe_mode(self, method, url, **kwargs):\n        \"\"\"\n        Send an HTTP request, and catch any exception that might occur due to either\n        connection problems, or invalid HTTP status codes\n        \"\"\"\n        try:\n            return self.client.urlopen(url, method=method, **kwargs)\n        except FAILURE_EXCEPTIONS as e:\n            if hasattr(e, \"response\"):\n                r = e.response\n            else:\n                req = self.client._make_request(\n                    url,\n                    method=method,\n                    headers=kwargs.get(\"headers\"),\n                    payload=kwargs.get(\"payload\"),\n                    params=kwargs.get(\"params\"),\n                )\n                r = ErrorResponse(url=url, request=req)\n            r.error = e\n            return r\n\n    def request(\n        self,\n        method: str,\n        url: str,\n        name: str | None = None,\n        data: str | dict | None = None,\n        catch_response: bool = False,\n        stream: bool = False,\n        headers: dict | None = None,\n        auth=None,\n        json: dict | None = None,\n        allow_redirects=True,\n        context: dict = {},\n        **kwargs,\n    ) -> ResponseContextManager | FastResponse:\n        \"\"\"\n        Send and HTTP request\n        Returns :py:class:`locust.contrib.fasthttp.FastResponse` object.\n\n        :param method: method for the new :class:`Request` object.\n        :param url: path that will be concatenated with the base host URL that has been specified.\n            Can also be a full URL, in which case the full URL will be requested, and the base host\n            is ignored.\n        :param name: (optional) An argument that can be specified to use as label in Locust's\n            statistics instead of the URL path. This can be used to group different URL's\n            that are requested into a single entry in Locust's statistics.\n        :param catch_response: (optional) Boolean argument that, if set, can be used to make a request\n            return a context manager to work as argument to a with statement. This will allow the\n            request to be marked as a fail based on the content of the response, even if the response\n            code is ok (2xx). The opposite also works, one can use catch_response to catch a request\n            and then mark it as successful even if the response code was not (i.e 500 or 404).\n        :param data: (optional) String/bytes to send in the body of the request.\n        :param json: (optional) Dictionary to send in the body of the request.\n            Automatically sets Content-Type and Accept headers to \"application/json\".\n            Only used if data is not set.\n        :param headers: (optional) Dictionary of HTTP Headers to send with the request.\n        :param auth: (optional) Auth (username, password) tuple to enable Basic HTTP Auth.\n        :param stream: (optional) If set to true the response body will not be consumed immediately\n            and can instead be consumed by accessing the stream attribute on the Response object.\n            Another side effect of setting stream to True is that the time for downloading the response\n            content will not be accounted for in the request time that is reported by Locust.\n        \"\"\"\n        # prepend url with hostname unless it's already an absolute URL\n        built_url = self._build_url(url)\n\n        start_time = time.time()  # seconds since epoch\n\n        if self.user:\n            context = {**self.user.context(), **context}\n\n        headers = headers or {}\n        if auth:\n            headers[\"Authorization\"] = _construct_basic_auth_str(auth[0], auth[1])\n        elif self.auth_header:\n            headers[\"Authorization\"] = self.auth_header\n        if \"Accept-Encoding\" not in headers and \"accept-encoding\" not in headers:\n            headers[\"Accept-Encoding\"] = \"gzip, deflate\"\n\n        if not data and json is not None:\n            data = unshadowed_json.dumps(json)\n            if \"Content-Type\" not in headers and \"content-type\" not in headers:\n                headers[\"Content-Type\"] = \"application/json\"\n            if \"Accept\" not in headers and \"accept\" not in headers:\n                headers[\"Accept\"] = \"application/json\"\n\n        if not allow_redirects:\n            old_redirect_response_codes = self.client.redirect_resonse_codes\n            self.client.redirect_resonse_codes = []\n\n        start_perf_counter = time.perf_counter()\n        # send request, and catch any exceptions\n        response = self._send_request_safe_mode(method, built_url, payload=data, headers=headers, **kwargs)\n        request_meta = {\n            \"request_type\": method,\n            \"name\": name or url,\n            \"context\": context,\n            \"response\": response,\n            \"exception\": None,\n            \"start_time\": start_time,\n            \"url\": built_url,  # this is a small deviation from HttpSession, which gets the final (possibly redirected) URL\n        }\n\n        if not allow_redirects:\n            self.client.redirect_resonse_codes = old_redirect_response_codes\n\n        # get the length of the content, but if the argument stream is set to True, we take\n        # the size from the content-length header, in order to not trigger fetching of the body\n        if stream:\n            request_meta[\"response_length\"] = int(response.headers.get(\"response_length\") or 0)\n        else:\n            try:\n                request_meta[\"response_length\"] = len(response.content or \"\")\n            except HTTPParseError as e:\n                request_meta[\"response_time\"] = (time.perf_counter() - start_perf_counter) * 1000\n                request_meta[\"response_length\"] = 0\n                request_meta[\"exception\"] = e\n                self.environment.events.request.fire(**request_meta)\n                return response\n\n        # Record the consumed time\n        # Note: This is intentionally placed after we record the content_size above, since\n        # we'll then trigger fetching of the body (unless stream=True)\n        request_meta[\"response_time\"] = int((time.perf_counter() - start_perf_counter) * 1000)\n\n        if catch_response:\n            return ResponseContextManager(response, environment=self.environment, request_meta=request_meta)\n        else:\n            try:\n                response.raise_for_status()\n            except FAILURE_EXCEPTIONS as e:\n                request_meta[\"exception\"] = e\n\n            self.environment.events.request.fire(**request_meta)\n            return response\n\n    def delete(self, url, **kwargs):\n        return self.request(\"DELETE\", url, **kwargs)\n\n    def get(self, url, **kwargs):\n        \"\"\"Sends a GET request\"\"\"\n        return self.request(\"GET\", url, **kwargs)\n\n    def head(self, url, **kwargs):\n        \"\"\"Sends a HEAD request\"\"\"\n        return self.request(\"HEAD\", url, **kwargs)\n\n    def options(self, url, **kwargs):\n        \"\"\"Sends a OPTIONS request\"\"\"\n        return self.request(\"OPTIONS\", url, **kwargs)\n\n    def patch(self, url, data=None, **kwargs):\n        \"\"\"Sends a POST request\"\"\"\n        return self.request(\"PATCH\", url, data=data, **kwargs)\n\n    def post(self, url, data=None, **kwargs):\n        \"\"\"Sends a POST request\"\"\"\n        return self.request(\"POST\", url, data=data, **kwargs)\n\n    def put(self, url, data=None, **kwargs):\n        \"\"\"Sends a PUT request\"\"\"\n        return self.request(\"PUT\", url, data=data, **kwargs)\n\n\nclass FastHttpUser(User):\n    \"\"\"\n    FastHttpUser provides the same API as HttpUser, but uses geventhttpclient instead of python-requests\n    as its underlying client. It uses considerably less CPU on the load generator, and should work\n    as a simple drop-in-replacement in most cases.\n    \"\"\"\n\n    # Below are various UserAgent settings. Change these in your subclass to alter FastHttpUser's behaviour.\n    # It needs to be done before FastHttpUser is instantiated, changing them later will have no effect\n\n    network_timeout: float = 60.0\n    \"\"\"Parameter passed to FastHttpSession\"\"\"\n\n    connection_timeout: float = 60.0\n    \"\"\"Parameter passed to FastHttpSession\"\"\"\n\n    max_redirects: int = 5\n    \"\"\"Parameter passed to FastHttpSession. Default 5, meaning 4 redirects.\"\"\"\n\n    max_retries: int = 1\n    \"\"\"Parameter passed to FastHttpSession. Default 1, meaning zero retries.\"\"\"\n\n    insecure: bool = True\n    \"\"\"Parameter passed to FastHttpSession. Default True, meaning no SSL verification.\"\"\"\n\n    default_headers: dict | None = None\n    \"\"\"Parameter passed to FastHttpSession. Adds the listed headers to every request.\"\"\"\n\n    concurrency: int = 10\n    \"\"\"Parameter passed to FastHttpSession. Describes number of concurrent requests allowed by the FastHttpSession. Default 10.\n    Note that setting this value has no effect when custom client_pool was given, and you need to spawn a your own gevent pool\n    to use it (as Users only have one greenlet). See test_fasthttp.py / test_client_pool_concurrency for an example.\"\"\"\n\n    client_pool: HTTPClientPool | None = None\n    \"\"\"HTTP client pool to use. If not given, a new pool is created per single user.\"\"\"\n\n    ssl_context_factory: Callable | None = None\n    \"\"\"A callable that return a SSLContext for overriding the default context created by the FastHttpSession.\"\"\"\n\n    abstract = True\n    \"\"\"Dont register this as a User class that can be run by itself\"\"\"\n\n    _callstack_regex = re.compile(r'  File \"(\\/.[^\"]*)\", line (\\d*),(.*)')\n\n    def __init__(self, environment):\n        super().__init__(environment)\n        if self.host is None:\n            raise LocustError(\n                \"You must specify the base host. Either in the host attribute in the User class, or on the command line using the --host option.\"\n            )\n\n        self.client: FastHttpSession = FastHttpSession(\n            self.environment,\n            base_url=self.host,\n            network_timeout=self.network_timeout,\n            connection_timeout=self.connection_timeout,\n            max_redirects=self.max_redirects,\n            max_retries=self.max_retries,\n            insecure=self.insecure,\n            concurrency=self.concurrency,\n            user=self,\n            client_pool=self.client_pool,\n            ssl_context_factory=self.ssl_context_factory,\n            headers=self.default_headers,\n        )\n        \"\"\"\n        Instance of HttpSession that is created upon instantiation of User.\n        The client support cookies, and therefore keeps the session between HTTP requests.\n        \"\"\"\n\n    @contextmanager\n    def rest(\n        self, method, url, headers: dict | None = None, **kwargs\n    ) -> Generator[RestResponseContextManager, None, None]:\n        \"\"\"\n        A wrapper for self.client.request that:\n\n        * Parses the JSON response to a dict called ``js`` in the response object. Marks the request as failed if the response was not valid JSON.\n        * Defaults ``Content-Type`` and ``Accept`` headers to ``application/json``\n        * Sets ``catch_response=True`` (so always use a :ref:`with-block <catch-response>`)\n        * Catches any unhandled exceptions thrown inside your with-block, marking the sample as failed (instead of exiting the task immediately without even firing the request event)\n        \"\"\"\n        headers = headers or {}\n        if not (\"Content-Type\" in headers or \"content-type\" in headers):\n            headers[\"Content-Type\"] = \"application/json\"\n        if not (\"Accept\" in headers or \"accept\" in headers):\n            headers[\"Accept\"] = \"application/json\"\n        with self.client.request(method, url, catch_response=True, headers=headers, **kwargs) as r:\n            resp = cast(RestResponseContextManager, r)\n            resp.js = None  # type: ignore\n            if resp.content is None:\n                resp.failure(str(resp.error))\n            elif resp.text:\n                try:\n                    resp.js = resp.json()\n                except JSONDecodeError as e:\n                    resp.failure(\n                        f\"Could not parse response as JSON. {resp.text[:250]}, response code {resp.status_code}, error {e}\"\n                    )\n            try:\n                yield resp\n            except AssertionError as e:\n                if e.args:\n                    if e.args[0].endswith(\",\"):\n                        short_resp = resp.text[:200] if resp.text else resp.text\n                        resp.failure(f\"{e.args[0][:-1]}, response was {short_resp}\")\n                    else:\n                        resp.failure(e.args[0])\n                else:\n                    resp.failure(\"Assertion failed\")\n\n            except Exception as e:\n                error_lines = []\n                for l in traceback.format_exc().split(\"\\n\"):\n                    m = self._callstack_regex.match(l)\n                    if m:\n                        filename = re.sub(r\"/(home|Users/\\w*)/\", \"~/\", m.group(1))\n                        error_lines.append(filename + \":\" + m.group(2) + m.group(3))\n                    short_resp = resp.text[:200] if resp.text else resp.text\n                    resp.failure(f\"{e.__class__.__name__}: {e} at {', '.join(error_lines)}. Response was {short_resp}\")\n\n    @contextmanager\n    def rest_(self, method, url, name=None, **kwargs) -> Generator[RestResponseContextManager, None, None]:\n        \"\"\"\n        Some REST api:s use a timestamp as part of their query string (mainly to break through caches).\n        This is a convenience method for that, appending a _=<timestamp> parameter automatically\n        \"\"\"\n        separator = \"&\" if \"?\" in url else \"?\"\n        if name is None:\n            name = url + separator + \"_=...\"\n        with self.rest(method, f\"{url}{separator}_={int(time.time()*1000)}\", name=name, **kwargs) as resp:\n            yield resp\n\n\nclass FastRequest(CompatRequest):\n    payload: str | None = None\n\n    @property\n    def body(self) -> str | None:\n        return self.payload\n\n\nclass FastResponse(CompatResponse):\n    headers: Headers | None = None\n    \"\"\"Dict like object containing the response headers\"\"\"\n\n    _response: HTTPSocketPoolResponse | None = None\n\n    encoding: str | None = None\n    \"\"\"In some cases setting the encoding explicitly is needed. If so, do it before calling .text\"\"\"\n\n    request: FastRequest | None = None\n\n    def __init__(\n        self,\n        ghc_response: HTTPSocketPoolResponse,\n        request: FastRequest | None = None,\n        sent_request: str | None = None,\n    ):\n        super().__init__(ghc_response, request, sent_request)\n\n        self.request = request\n\n    @property\n    def text(self) -> str | None:\n        \"\"\"\n        Returns the text content of the response as a decoded string\n        \"\"\"\n        if self.content is None:\n            return None\n        if self.encoding is None:\n            if self.headers is None:\n                # No information, try to detect\n                self.encoding = detect(self.content)[\"encoding\"]\n            else:\n                self.encoding = get_encoding_from_headers(self.headers)\n                # No information, try to detect\n                if not self.encoding:\n                    self.encoding = detect(self.content)[\"encoding\"]\n        if self.encoding is None:\n            return None\n        return str(self.content, str(self.encoding), errors=\"replace\")\n\n    @property\n    def url(self) -> str | None:\n        \"\"\"\n        Get \"response\" URL, which is the same as the request URL. This is a small deviation from HttpSession, which gets the final (possibly redirected) URL.\n        \"\"\"\n        if self.request is not None:\n            return self.request.url\n\n        return None\n\n    def json(self) -> dict:\n        \"\"\"\n        Parses the response as json and returns a dict\n        \"\"\"\n        return json.loads(self.text)  # type: ignore\n\n    def raise_for_status(self):\n        \"\"\"Raise any connection errors that occurred during the request\"\"\"\n        if hasattr(self, \"error\") and self.error:\n            raise self.error\n\n    @property\n    def status_code(self) -> int:\n        \"\"\"\n        We override status_code in order to return None if no valid response was\n        returned. E.g. in the case of connection errors\n        \"\"\"\n        return self._response.get_code() if self._response is not None else 0\n\n    @property\n    def ok(self):\n        \"\"\"Returns True if :attr:`status_code` is less than 400, False if not.\"\"\"\n        return self.status_code < 400\n\n    def _content(self):\n        if self.headers is None:\n            return None\n        return super()._content()\n\n    def success(self):\n        raise LocustError(\n            \"If you want to change the state of the request, you must pass catch_response=True. See http://docs.locust.io/en/stable/writing-a-locustfile.html#validating-responses\"\n        )\n\n    def failure(self):\n        raise LocustError(\n            \"If you want to change the state of the request, you must pass catch_response=True. See http://docs.locust.io/en/stable/writing-a-locustfile.html#validating-responses\"\n        )\n\n\nclass ErrorResponse:\n    \"\"\"\n    This is used as a dummy response object when geventhttpclient raises an error\n    that doesn't have a real Response object attached. E.g. a socket error or similar\n    \"\"\"\n\n    headers: Headers | None = None\n    content = None\n    status_code = 0\n    error: Exception | None = None\n    text: str | None = None\n    request: CompatRequest\n\n    def __init__(self, url: str, request: CompatRequest):\n        self.url = url\n        self.request = request\n\n    def raise_for_status(self):\n        raise self.error\n\n\nclass LocustUserAgent(UserAgent):\n    response_type = FastResponse\n    request_type = FastRequest\n    valid_response_codes = frozenset([200, 201, 202, 203, 204, 205, 206, 207, 208, 226, 301, 302, 303, 304, 307])\n\n    def __init__(self, client_pool: HTTPClientPool | None = None, **kwargs):\n        super().__init__(**kwargs)\n\n        if client_pool is not None:\n            self.clientpool = client_pool\n\n    def _urlopen(self, request):\n        \"\"\"Override _urlopen() in order to make it use the response_type attribute\"\"\"\n        client = self.clientpool.get_client(request.url_split)\n        resp = client.request(\n            request.method, request.url_split.request_uri, body=request.payload, headers=request.headers\n        )\n        return self.response_type(resp, request=request, sent_request=resp._sent_request)\n\n\n...\n# Path: locust/debug.py\nfrom __future__ import annotations\n\nimport locust\nimport locust.log\nfrom locust import argument_parser\nfrom locust.env import Environment\nfrom locust.exception import CatchResponseError, RescheduleTask\n\nimport inspect\nimport os\nfrom datetime import datetime, timezone\nfrom typing import TYPE_CHECKING\n\nif TYPE_CHECKING:\n    from locust import User\n\n\ndef _print_t(s):\n    \"\"\"\n    Print something with a tab instead of newline at the end\n    \"\"\"\n    print(str(s), end=\"\\t\")\n\n\nclass PrintListener:\n    \"\"\"\n    Print every response (useful when debugging a single locust)\n    \"\"\"\n\n    def __init__(\n        self,\n        env: Environment,\n        include_length=False,\n        include_time=False,\n        include_context=False,\n        include_payload=False,\n    ):\n        env.events.request.add_listener(self.on_request)\n\n        self.include_length = \"length\\t\" if include_length else \"\"\n        self.include_time = \"time                    \\t\" if include_time else \"\"\n        self.include_context = \"context\\t\" if include_context else \"\"\n        self.include_payload = \"payload\\t\" if include_payload else \"\"\n\n        print(\n            f\"\\n{self.include_time}type\\t{'name'.ljust(50)}\\tresp_ms\\t{self.include_length}exception\\t{self.include_context}\\t{self.include_payload}\"\n        )\n\n    def on_request(\n        self,\n        request_type,\n        name,\n        response_time,\n        response_length,\n        exception,\n        context: dict,\n        start_time=None,\n        response=None,\n        **_kwargs,\n    ):\n        if exception:\n            if isinstance(exception, RescheduleTask):\n                pass\n            if isinstance(exception, CatchResponseError):\n                e = str(exception)\n            else:\n                try:\n                    e = repr(exception)\n                except AttributeError:\n                    e = f\"{exception.__class__} (and it has no string representation)\"\n            errortext = e[:500].replace(\"\\n\", \" \")\n        else:\n            errortext = \"\"\n\n        if response_time is None:\n            response_time = -1\n        n = name.ljust(30) if name else \"\"\n\n        if self.include_time:\n            if start_time:\n                _print_t(datetime.fromtimestamp(start_time, tz=timezone.utc))\n            else:\n                _print_t(datetime.now())\n\n        _print_t(request_type)\n        _print_t(n.ljust(50))\n        _print_t(str(round(response_time)).ljust(7))\n\n        if self.include_length:\n            _print_t(response_length)\n\n        _print_t(errortext.ljust(9))\n\n        if self.include_context:\n            _print_t(context or \"\")\n\n        if self.include_payload:\n            _print_t(response._request.payload)\n\n        print()\n\n\n_env: Environment | None = None  # minimal Environment for debugging\n\n\ndef run_single_user(\n    user_class: type[User],\n    include_length=False,\n    include_time=False,\n    include_context=False,\n    include_payload=False,\n    loglevel: str | None = \"WARNING\",\n):\n    \"\"\"\n    Runs a single User. Useful when you want to run a debugger.\n\n    It creates in a new locust :py:attr:`Environment <locust.env.Environment>` and triggers any ``init`` or ``test_start`` :ref:`events <extending_locust>` as normal.\n\n    It does **not** trigger ``test_stop`` or ``quit`` when you quit the debugger.\n\n    It prints some info about every request to stdout, and you can get additional info using the `include_*` flags\n\n    It also initiates logging on WARNING level (not INFO, because it could interfere with the printing of requests),\n    but you can change that by passing a log level (or disabling logging entirely by passing None)\n    \"\"\"\n    global _env\n\n    if loglevel:\n        locust.log.setup_logging(loglevel)\n\n    if not _env:\n        options = argument_parser.parse_options()\n\n        # in case your test goes looking for the file name of your locustfile\n        frame = inspect.stack()[1]\n        locustfile = os.path.basename(frame[0].f_code.co_filename)\n        options.locustfile = locustfile\n\n        _env = Environment(events=locust.events, locustfile=locustfile, host=options.host, parsed_options=options)\n\n        # log requests to stdout\n        PrintListener(\n            _env,\n            include_length=include_length,\n            include_time=include_time,\n            include_context=include_context,\n            include_payload=include_payload,\n        )\n        # fire various events (quit and test_stop will never get called, sorry about that)\n        _env.events.init.fire(environment=_env, runner=None, web_ui=None)\n        # uncaught events will be suppressed, so check if that happened\n        if locust.log.unhandled_greenlet_exception:\n            raise Exception(\"Unhandled exception in init\")\n\n    # do the things that the Runner usually does\n    _env.user_classes = [user_class]\n    _env._filter_tasks_by_tags()\n    _env.events.test_start.fire(environment=_env)\n    if _env.host:\n        user_class.host = _env.host\n\n    # create a single user\n    user = user_class(_env)\n    setattr(_env, \"single_user_instance\", user)  # if you happen to need access to this from the Environment instance\n    user.run()\n\n# Path: locust/user/sequential_taskset.py\nfrom locust.exception import LocustError\n\nimport logging\n\nfrom .task import TaskSet, TaskSetMeta\n\n\nclass SequentialTaskSetMeta(TaskSetMeta):\n    \"\"\"\n    Meta class for SequentialTaskSet. It's used to allow SequentialTaskSet classes to specify\n    task execution in both a list as the tasks attribute or using the @task decorator\n\n    We use the fact that class_dict order is the order of declaration in Python 3.6\n    (See https://www.python.org/dev/peps/pep-0520/)\n    \"\"\"\n\n    def __new__(mcs, classname, bases, class_dict):\n        new_tasks = []\n        for base in bases:\n            # first get tasks from base classes\n            if hasattr(base, \"tasks\") and base.tasks:\n                new_tasks += base.tasks\n        for key, value in class_dict.items():\n            if key == \"tasks\":\n                # we want to insert tasks from the tasks attribute at the point of it's declaration\n                # compared to methods declared with @task\n                if isinstance(value, list):\n                    new_tasks.extend(value)\n                else:\n                    raise ValueError(\"On SequentialTaskSet the task attribute can only be set to a list\")\n\n            if \"locust_task_weight\" in dir(value):\n                # method decorated with @task\n                for _ in range(value.locust_task_weight):\n                    new_tasks.append(value)\n\n        class_dict[\"tasks\"] = new_tasks\n        return type.__new__(mcs, classname, bases, class_dict)\n\n\nclass SequentialTaskSet(TaskSet, metaclass=SequentialTaskSetMeta):\n    \"\"\"\n    Class defining a sequence of tasks that a User will execute.\n\n    Works like TaskSet, but task weight is ignored, and all tasks are executed in order. Tasks can\n    either be specified by setting the *tasks* attribute to a list of tasks, or by declaring tasks\n    as methods using the @task decorator. The order of declaration decides the order of execution.\n\n    It's possible to combine a task list in the *tasks* attribute, with some tasks declared using\n    the @task decorator. The order of declaration is respected also in that case.\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self._task_index = 0\n\n    def get_next_task(self):\n        if not self.tasks:\n            raise LocustError(\n                \"No tasks defined. Use the @task decorator or set the 'tasks' attribute of the SequentialTaskSet\"\n            )\n        task = self.tasks[self._task_index % len(self.tasks)]\n        self._task_index += 1\n        return task\n\n# Path: locust/__init__.py\nimport os\n\nif os.getenv(\"LOCUST_PLAYWRIGHT\", None):\n    print(\"LOCUST_PLAYWRIGHT setting is no longer needed (because locust-plugins no longer installs trio)\")\n    print(\"Uninstall trio package and remove the setting.\")\n    try:\n        # preserve backwards compatibility for now\n        import trio\n    except ModuleNotFoundError:\n        # dont show a massive callstack if trio is not installed\n        os._exit(1)\n\nfrom gevent import monkey\n\nmonkey.patch_all()\n\nfrom ._version import version as __version__\nfrom .contrib.fasthttp import FastHttpUser\nfrom .debug import run_single_user\nfrom .event import Events\nfrom .shape import LoadTestShape\nfrom .user import wait_time\nfrom .user.sequential_taskset import SequentialTaskSet\nfrom .user.task import TaskSet, tag, task\nfrom .user.users import HttpUser, User\nfrom .user.wait_time import between, constant, constant_pacing, constant_throughput\n\nevents = Events()\n\n__all__ = (\n    \"SequentialTaskSet\",\n    \"wait_time\",\n    \"task\",\n    \"tag\",\n    \"TaskSet\",\n    \"HttpUser\",\n    \"FastHttpUser\",\n    \"User\",\n    \"between\",\n    \"constant\",\n    \"constant_pacing\",\n    \"constant_throughput\",\n    \"events\",\n    \"LoadTestShape\",\n    \"run_single_user\",\n)\n\n# Used for raising a DeprecationWarning if old Locust/HttpLocust is used\nfrom .util.deprecation import DeprecatedHttpLocustClass as HttpLocust\nfrom .util.deprecation import DeprecatedLocustClass as Locust\n\n# Path: locust/input_events.py\nfrom __future__ import annotations\n\nimport logging\nimport os\nimport sys\nfrom typing import Callable\n\nimport gevent\n\nif os.name == \"nt\":\n    import pywintypes\n    from win32api import STD_INPUT_HANDLE\n    from win32console import (\n        ENABLE_ECHO_INPUT,\n        ENABLE_LINE_INPUT,\n        ENABLE_PROCESSED_INPUT,\n        KEY_EVENT,\n        GetStdHandle,\n    )\nelse:\n    import select\n    import termios\n    import tty\n\n\nclass InitError(Exception):\n    pass\n\n\nclass UnixKeyPoller:\n    def __init__(self):\n        if sys.stdin.isatty():\n            self.stdin = sys.stdin.fileno()\n            self.tattr = termios.tcgetattr(self.stdin)\n            tty.setcbreak(self.stdin, termios.TCSANOW)\n        else:\n            raise InitError(\"Terminal was not a tty. Keyboard input disabled\")\n\n    def cleanup(self):\n        termios.tcsetattr(self.stdin, termios.TCSANOW, self.tattr)\n\n    def poll(_self):\n        dr, dw, de = select.select([sys.stdin], [], [], 0)\n        if not dr == []:\n            return sys.stdin.read(1)\n        return None\n\n\nclass WindowsKeyPoller:\n    def __init__(self):\n        if sys.stdin.isatty():\n            try:\n                self.read_handle = GetStdHandle(STD_INPUT_HANDLE)\n                self.read_handle.SetConsoleMode(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT)\n                self.cur_event_length = 0\n                self.cur_keys_length = 0\n                self.captured_chars = []\n            except pywintypes.error:\n                raise InitError(\"Terminal says its a tty but we couldnt enable line input. Keyboard input disabled.\")\n        else:\n            raise InitError(\"Terminal was not a tty. Keyboard input disabled\")\n\n    def cleanup(self):\n        pass\n\n    def poll(self):\n        if self.captured_chars:\n            return self.captured_chars.pop(0)\n\n        events_peek = self.read_handle.PeekConsoleInput(10000)\n\n        if not events_peek:\n            return None\n\n        if not len(events_peek) == self.cur_event_length:\n            for cur_event in events_peek[self.cur_event_length :]:\n                if cur_event.EventType == KEY_EVENT:\n                    if ord(cur_event.Char) and cur_event.KeyDown:\n                        cur_char = str(cur_event.Char)\n                        self.captured_chars.append(cur_char)\n\n            self.cur_event_length = len(events_peek)\n\n        if self.captured_chars:\n            return self.captured_chars.pop(0)\n        else:\n            return None\n\n\ndef get_poller():\n    if os.name == \"nt\":\n        return WindowsKeyPoller()\n    else:\n        return UnixKeyPoller()\n\n\ndef input_listener(key_to_func_map: dict[str, Callable]):\n    def input_listener_func():\n        try:\n            poller = get_poller()\n        except InitError as e:\n            logging.debug(e)\n            return\n\n        try:\n            while True:\n                input = poller.poll()\n                if input:\n                    for key in key_to_func_map:\n                        if input == key:\n                            key_to_func_map[key]()\n                else:\n                    gevent.sleep(0.2)\n        except Exception as e:\n            logging.warning(f\"Exception in keyboard input poller: {e}\")\n        finally:\n            poller.cleanup()\n\n    return input_listener_func\n\n# Path: locust/util/load_locustfile.py\nfrom __future__ import annotations\n\nimport importlib\nimport importlib.util\nimport inspect\nimport os\nimport sys\n\nfrom ..shape import LoadTestShape\nfrom ..user import User\n\n\ndef is_user_class(item):\n    \"\"\"\n    Check if a variable is a runnable (non-abstract) User class\n    \"\"\"\n    return bool(inspect.isclass(item) and issubclass(item, User) and item.abstract is False)\n\n\ndef is_shape_class(item):\n    \"\"\"\n    Check if a class is a LoadTestShape\n    \"\"\"\n    return bool(inspect.isclass(item) and issubclass(item, LoadTestShape) and not getattr(item, \"abstract\", True))\n\n\ndef load_locustfile(path) -> tuple[str | None, dict[str, User], list[LoadTestShape]]:\n    \"\"\"\n    Import given locustfile path and return (docstring, callables).\n\n    Specifically, the locustfile's ``__doc__`` attribute (a string) and a\n    dictionary of ``{'name': callable}`` containing all callables which pass\n    the \"is a Locust\" test.\n    \"\"\"\n\n    # Start with making sure the current working dir is in the sys.path\n    sys.path.insert(0, os.getcwd())\n    # Get directory and locustfile name\n    directory, locustfile = os.path.split(path)\n    # If the directory isn't in the PYTHONPATH, add it so our import will work\n    added_to_path = False\n    index = None\n    if directory not in sys.path:\n        sys.path.insert(0, directory)\n        added_to_path = True\n    # If the directory IS in the PYTHONPATH, move it to the front temporarily,\n    # otherwise other locustfiles -- like Locusts's own -- may scoop the intended\n    # one.\n    else:\n        i = sys.path.index(directory)\n        if i != 0:\n            # Store index for later restoration\n            index = i\n            # Add to front, then remove from original position\n            sys.path.insert(0, directory)\n            del sys.path[i + 1]\n\n    # Perform the import\n    module_name = os.path.splitext(locustfile)[0]\n    if module_name == \"locust\":\n        module_name = \"locustfile\"  # Avoid conflict with locust package\n    loader = importlib.machinery.SourceFileLoader(module_name, path)\n    spec = importlib.util.spec_from_file_location(module_name, path, loader=loader)\n    if spec is None:\n        sys.stderr.write(f\"Unable to get module spec for {module_name} in {path}\")\n        sys.exit(1)\n\n    imported = importlib.util.module_from_spec(spec)\n    sys.modules[imported.__name__] = imported\n    loader.exec_module(imported)\n\n    # Remove directory from path if we added it ourselves (just to be neat)\n    if added_to_path:\n        del sys.path[0]\n    # Put back in original index if we moved it\n    if index is not None:\n        sys.path.insert(index + 1, directory)\n        del sys.path[0]\n    # Return our two-tuple\n    user_classes = {name: value for name, value in vars(imported).items() if is_user_class(value)}\n\n    # Find shape class, if any, return it\n    shape_classes = [value() for value in vars(imported).values() if is_shape_class(value)]\n\n    return imported.__doc__, user_classes, shape_classes\n\n# Path: locust/main.py\nfrom __future__ import annotations\n\nimport locust\n\nimport atexit\nimport errno\nimport gc\nimport inspect\nimport json\nimport logging\nimport os\nimport signal\nimport sys\nimport time\nimport traceback\n\nimport gevent\n\nfrom . import log, stats\nfrom .argument_parser import parse_locustfile_option, parse_options\nfrom .env import Environment\nfrom .html import get_html_report\nfrom .input_events import input_listener\nfrom .log import greenlet_exception_logger, setup_logging\nfrom .stats import (\n    StatsCSV,\n    StatsCSVFileWriter,\n    print_error_report,\n    print_percentile_stats,\n    print_stats,\n    print_stats_json,\n    stats_history,\n    stats_printer,\n)\nfrom .user.inspectuser import print_task_ratio, print_task_ratio_json\nfrom .util.load_locustfile import load_locustfile\nfrom .util.timespan import parse_timespan\n\ntry:\n    # import locust_plugins if it is installed, to allow it to register custom arguments etc\n    import locust_plugins  # pyright: ignore[reportMissingImports]\nexcept ModuleNotFoundError:\n    pass\n\nversion = locust.__version__\n\n# Options to ignore when using a custom shape class without `use_common_options=True`\n# See: https://docs.locust.io/en/stable/custom-load-shape.html#use-common-options\nCOMMON_OPTIONS = {\n    \"num_users\": \"users\",\n    \"spawn_rate\": \"spawn-rate\",\n    \"run_time\": \"run-time\",\n}\n\n\ndef create_environment(\n    user_classes,\n    options,\n    events=None,\n    shape_class=None,\n    locustfile=None,\n    available_user_classes=None,\n    available_shape_classes=None,\n    available_user_tasks=None,\n):\n    \"\"\"\n    Create an Environment instance from options\n    \"\"\"\n    return Environment(\n        locustfile=locustfile,\n        user_classes=user_classes,\n        shape_class=shape_class,\n        events=events,\n        host=options.host,\n        reset_stats=options.reset_stats,\n        parsed_options=options,\n        available_user_classes=available_user_classes,\n        available_shape_classes=available_shape_classes,\n        available_user_tasks=available_user_tasks,\n    )\n\n\ndef main():\n    # find specified locustfile(s) and make sure it exists, using a very simplified\n    # command line parser that is only used to parse the -f option.\n    locustfiles = parse_locustfile_option()\n    locustfiles_length = len(locustfiles)\n\n    # Grabbing the Locustfile if only one was provided. Otherwise, allowing users to select the locustfile in the UI\n    # If --headless or --autostart and multiple locustfiles, all provided UserClasses will be ran\n    locustfile = locustfiles[0] if locustfiles_length == 1 else None\n\n    # Importing Locustfile(s) - setting available UserClasses and ShapeClasses to choose from in UI\n    user_classes: dict[str, locust.User] = {}\n    available_user_classes = {}\n    available_shape_classes = {}\n    available_user_tasks = {}\n    shape_class = None\n    for _locustfile in locustfiles:\n        docstring, _user_classes, shape_classes = load_locustfile(_locustfile)\n\n        # Setting Available Shape Classes\n        if shape_classes:\n            shape_class = shape_classes[0]\n            for shape_class in shape_classes:\n                shape_class_name = type(shape_class).__name__\n                if shape_class_name in available_shape_classes.keys():\n                    sys.stderr.write(f\"Duplicate shape classes: {shape_class_name}\\n\")\n                    sys.exit(1)\n\n                available_shape_classes[shape_class_name] = shape_class\n\n        # Setting Available User Classes\n        for key, value in _user_classes.items():\n            if key in available_user_classes.keys():\n                previous_path = inspect.getfile(user_classes[key])\n                new_path = inspect.getfile(value)\n                if previous_path == new_path:\n                    # The same User class was defined in two locustfiles but one probably imported the other, so we just ignore it\n                    continue\n                else:\n                    sys.stderr.write(\n                        f\"Duplicate user class names: {key} is defined in both {previous_path} and {new_path}\\n\"\n                    )\n                    sys.exit(1)\n\n            user_classes[key] = value\n            available_user_classes[key] = value\n            available_user_tasks[key] = value.tasks or None\n\n    if len(stats.PERCENTILES_TO_CHART) != 2:\n        logging.error(\"stats.PERCENTILES_TO_CHART parameter should be 2 parameters \\n\")\n        sys.exit(1)\n\n    if len(stats.MODERN_UI_PERCENTILES_TO_CHART) > 6:\n        logging.error(\"stats.MODERN_UI_PERCENTILES_TO_CHART parameter should be a maximum of 6 parameters \\n\")\n        sys.exit(1)\n\n    \ndef is_valid_percentile(parameter):\n        try:\n            if 0 < float(parameter) < 1:\n                return True\n            return False\n        except ValueError:\n            return False\n\n    for percentile in stats.PERCENTILES_TO_CHART:\n        if not is_valid_percentile(percentile):\n            logging.error(\n                \"stats.PERCENTILES_TO_CHART parameter need to be float and value between. 0 < percentile < 1 Eg 0.95\\n\"\n            )\n            sys.exit(1)\n\n    for percentile in stats.PERCENTILES_TO_STATISTICS:\n        if not is_valid_percentile(percentile):\n            logging.error(\n                \"stats.PERCENTILES_TO_STATISTICS parameter need to be float and value between. 0 < percentile < 1 Eg 0.95\\n\"\n            )\n            sys.exit(1)\n\n    # parse all command line options\n    options = parse_options()\n\n    if options.headful:\n        options.headless = False\n\n    if options.slave or options.expect_slaves:\n        sys.stderr.write(\"The --slave/--expect-slaves parameters have been renamed --worker/--expect-workers\\n\")\n        sys.exit(1)\n\n    if options.web_auth:\n        sys.stderr.write(\n            \"The --web-auth parameters has been replaced with --web-login. See https://docs.locust.io/en/stable/extending-locust.html#authentication for details\\n\"\n        )\n        sys.exit(1)\n\n    if options.autoquit != -1 and not options.autostart:\n        sys.stderr.write(\"--autoquit is only meaningful in combination with --autostart\\n\")\n        sys.exit(1)\n\n    if options.hatch_rate:\n        sys.stderr.write(\"[DEPRECATED] The --hatch-rate parameter has been renamed --spawn-rate\\n\")\n        options.spawn_rate = options.hatch_rate\n\n    # setup logging\n    if not options.skip_log_setup:\n        if options.loglevel.upper() in [\"DEBUG\", \"INFO\", \"WARNING\", \"ERROR\", \"CRITICAL\"]:\n            setup_logging(options.loglevel, options.logfile)\n        else:\n            sys.stderr.write(\"Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\\n\")\n            sys.exit(1)\n\n    children = []\n\n    if options.processes:\n        if os.name == \"nt\":\n            sys.stderr.write(\"--processes is not supported in Windows (except in WSL)\\n\")\n            sys.exit(1)\n        if options.processes == -1:\n            options.processes = os.cpu_count()\n            if not options.processes:\n                sys.stderr.write(\"--processes failed to detect number of cpus!?\\n\")\n                sys.exit(1)\n        elif options.processes < -1:\n            sys.stderr.write(f\"Invalid --processes count {options.processes}\\n\")\n            sys.exit(1)\n        elif options.master:\n            sys.stderr.write(\n                \"--master cannot be combined with --processes. Remove --master, as it is implicit as long as --worker is not set.\\n\"\n            )\n            sys.exit(1)\n        # Optimize copy-on-write-behavior to save some memory (aprx 26MB -> 15MB rss) in child processes\n        gc.collect()  # avoid freezing garbage\n        gc.freeze()  # move all objects to perm gen so ref counts dont get updated\n        for _ in range(options.processes):\n            child_pid = gevent.fork()\n            if child_pid:\n                children.append(child_pid)\n                logging.debug(f\"Started child worker with pid #{child_pid}\")\n            else:\n                # child is always a worker, even when it wasnt set on command line\n                options.worker = True\n                # remove options that dont make sense on worker\n                options.run_time = None\n                options.autostart = None\n                break\n        else:\n            # we're in the parent process\n            if options.worker:\n                # ignore the first sigint in parent, and wait for the children to handle sigint\n                def sigint_handler(_signal, _frame):\n                    if getattr(sigint_handler, \"has_run\", False):\n                        # if parent gets repeated sigint, we kill the children hard\n                        for child_pid in children:\n                            try:\n                                logging.debug(f\"Sending SIGKILL to child with pid {child_pid}\")\n                                os.kill(child_pid, signal.SIGKILL)\n                            except ProcessLookupError:\n                                pass  # process already dead\n                            except Exception:\n                                logging.error(traceback.format_exc())\n                        sys.exit(1)\n                    sigint_handler.has_run = True\n\n                signal.signal(signal.SIGINT, sigint_handler)\n                exit_code = 0\n                # nothing more to do, just wait for the children to exit\n                for child_pid in children:\n                    _, child_status = os.waitpid(child_pid, 0)\n                    try:\n                        if sys.version_info >= (3, 9):\n                            child_exit_code = os.waitstatus_to_exitcode(child_status)\n                            exit_code = max(exit_code, child_exit_code)\n                    except AttributeError:\n                        pass  # dammit python 3.8...\n                sys.exit(exit_code)\n            else:\n                options.master = True\n                options.expect_workers = options.processes\n\n                def kill_workers(children):\n                    exit_code = 0\n                    start_time = time.time()\n                    # give children some time to finish up (in case they had an error parsing arguments etc)\n                    for child_pid in children[:]:\n                        while time.time() < start_time + 3:\n                            try:\n                                _, child_status = os.waitpid(child_pid, os.WNOHANG)\n                                children.remove(child_pid)\n                                try:\n                                    if sys.version_info >= (3, 9):\n                                        child_exit_code = os.waitstatus_to_exitcode(child_status)\n                                        exit_code = max(exit_code, child_exit_code)\n                                except AttributeError:\n                                    pass  # dammit python 3.8...\n                            except OSError as e:\n                                if e.errno == errno.EINTR:\n                                    time.sleep(0.1)\n                                else:\n                                    logging.error(traceback.format_exc())\n                            else:\n                                break\n                    for child_pid in children:\n                        try:\n                            logging.debug(f\"Sending SIGINT to child with pid {child_pid}\")\n                            os.kill(child_pid, signal.SIGINT)\n                        except ProcessLookupError:\n                            pass  # never mind, process was already dead\n                    for child_pid in children:\n                        _, child_status = os.waitpid(child_pid, 0)\n                        try:\n                            if sys.version_info >= (3, 9):\n                                child_exit_code = os.waitstatus_to_exitcode(child_status)\n                                exit_code = max(exit_code, child_exit_code)\n                        except AttributeError:\n                            pass  # dammit python 3.8...\n                    if exit_code > 1:\n                        logging.error(f\"Bad response code from worker children: {exit_code}\")\n                    # ensure master doesnt finish until output from workers has arrived\n                    # otherwise the terminal might look weird.\n                    time.sleep(0.1)\n\n                atexit.register(kill_workers, children)\n\n    logger = logging.getLogger(__name__)\n    greenlet_exception_handler = greenlet_exception_logger(logger)\n\n    if options.stop_timeout:\n        try:\n            options.stop_timeout = parse_timespan(options.stop_timeout)\n        except ValueError:\n            logger.error(\"Valid --stop-timeout formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n            sys.exit(1)\n\n    if options.list_commands:\n        print(\"Available Users:\")\n        for name in user_classes:\n            print(\"    \" + name)\n        sys.exit(0)\n\n    if not user_classes:\n        logger.error(\"No User class found!\")\n        sys.exit(1)\n\n    # make sure specified User exists\n    if options.user_classes:\n        missing = set(options.user_classes) - set(user_classes.keys())\n        if missing:\n            logger.error(f\"Unknown User(s): {', '.join(missing)}\\n\")\n            sys.exit(1)\n        else:\n            names = set(options.user_classes) & set(user_classes.keys())\n            user_classes = [user_classes[n] for n in names]\n    else:\n        # list() call is needed to consume the dict_view object in Python 3\n        user_classes = list(user_classes.values())\n\n    if os.name != \"nt\" and not options.master:\n        try:\n            import resource\n\n            minimum_open_file_limit = 10000\n            (soft_limit, hard_limit) = resource.getrlimit(resource.RLIMIT_NOFILE)\n\n            if soft_limit < minimum_open_file_limit:\n                # Increasing the limit to 10000 within a running process should work on at least MacOS.\n                # It does not work on all OS:es, but we should be no worse off for trying.\n                resource.setrlimit(resource.RLIMIT_NOFILE, [minimum_open_file_limit, hard_limit])\n        except BaseException:\n            logger.warning(\n                f\"\"\"System open file limit '{soft_limit} is below minimum setting '{minimum_open_file_limit}'.\nIt's not high enough for load testing, and the OS didn't allow locust to increase it by itself.\nSee https://github.com/locustio/locust/wiki/Installation#increasing-maximum-number-of-open-files-limit for more info.\"\"\"\n            )\n\n    if sys.version_info < (3, 9):\n        logger.warning(\"Python 3.8 support is deprecated and will be removed soon\")\n\n    # create locust Environment\n    locustfile_path = None if not locustfile else os.path.basename(locustfile)\n\n    environment = create_environment(\n        user_classes,\n        options,\n        events=locust.events,\n        shape_class=shape_class,\n        locustfile=locustfile_path,\n        available_user_classes=available_user_classes,\n        available_shape_classes=available_shape_classes,\n        available_user_tasks=available_user_tasks,\n    )\n\n    if options.config_users:\n        for json_user_config in options.config_users:\n            try:\n                if json_user_config.endswith(\".json\"):\n                    with open(json_user_config) as file:\n                        user_config = json.load(file)\n                else:\n                    user_config = json.loads(json_user_config)\n\n                def ensure_user_class_name(config):\n                    if \"user_class_name\" not in config:\n                        logger.error(\"The user config must specify a user_class_name\")\n                        sys.exit(-1)\n\n                if isinstance(user_config, list):\n                    for config in user_config:\n                        ensure_user_class_name(config)\n\n                        environment.update_user_class(config)\n                else:\n                    ensure_user_class_name(user_config)\n\n                    environment.update_user_class(user_config)\n            except Exception as e:\n                logger.error(f\"The --config-users arugment must be in valid JSON string or file: {e}\")\n                sys.exit(-1)\n\n    if (\n        shape_class\n        and not shape_class.use_common_options\n        and any(getattr(options, opt, None) for opt in COMMON_OPTIONS)\n    ):\n        logger.warning(\n            \"--run-time, --users or --spawn-rate have no impact on LoadShapes unless the shape class explicitly reads them. \"\n            \"See: docs.locust.io/en/stable/custom-load-shape.html#use-common-options\"\n        )\n        ignored = [f\"--{arg}\" for opt, arg in COMMON_OPTIONS.items() if getattr(options, opt, None)]\n        logger.warning(f\"The following option(s) will be ignored: {', '.join(ignored)}\")\n\n    if options.show_task_ratio:\n        print(\"\\n Task ratio per User class\")\n        print(\"-\" * 80)\n        print_task_ratio(user_classes, options.num_users, False)\n        print(\"\\n Total task ratio\")\n        print(\"-\" * 80)\n        print_task_ratio(user_classes, options.num_users, True)\n        sys.exit(0)\n    if options.show_task_ratio_json:\n        print_task_ratio_json(user_classes, options.num_users)\n        sys.exit(0)\n\n    if options.master:\n        if options.worker:\n            logger.error(\"The --master argument cannot be combined with --worker\")\n            sys.exit(-1)\n        if options.expect_workers_max_wait and not options.expect_workers:\n            logger.error(\"The --expect-workers-max-wait argument only makes sense when combined with --expect-workers\")\n            sys.exit(-1)\n        runner = environment.create_master_runner(\n            master_bind_host=options.master_bind_host,\n            master_bind_port=options.master_bind_port,\n        )\n    elif options.worker:\n        try:\n            runner = environment.create_worker_runner(options.master_host, options.master_port)\n            logger.debug(\"Connected to locust master: %s:%s\", options.master_host, options.master_port)\n        except OSError as e:\n            logger.error(\"Failed to connect to the Locust master: %s\", e)\n            sys.exit(-1)\n    else:\n        runner = environment.create_local_runner()\n\n    # main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode\n    main_greenlet = runner.greenlet\n\n    if options.run_time:\n        if options.worker:\n            logger.error(\"--run-time should be specified on the master node, and not on worker nodes\")\n            sys.exit(1)\n        try:\n            options.run_time = parse_timespan(options.run_time)\n        except ValueError:\n            logger.error(\"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n            sys.exit(1)\n\n    if options.csv_prefix:\n        base_csv_file = os.path.basename(options.csv_prefix)\n        base_csv_dir = options.csv_prefix[: -len(base_csv_file)]\n        if not os.path.exists(base_csv_dir) and len(base_csv_dir) != 0:\n            os.makedirs(base_csv_dir)\n        stats_csv_writer = StatsCSVFileWriter(\n            environment, stats.PERCENTILES_TO_REPORT, options.csv_prefix, options.stats_history_enabled\n        )\n    else:\n        stats_csv_writer = StatsCSV(environment, stats.PERCENTILES_TO_REPORT)\n\n    # start Web UI\n    if not options.headless and not options.worker:\n        # spawn web greenlet\n        protocol = \"https\" if options.tls_cert and options.tls_key else \"http\"\n\n        if options.web_host == \"*\":\n...\n", "needle_token_start": 12053, "needle_token_end": 12099, "code_context_ntokens": 16136, "output": ["```python\ndef is_valid_percentile(parameter):\n    try:\n        if 0 < float(parameter) < 1:\n            return True\n        return False\n    except ValueError:\n        return False\n```"]}
{"repo": "locustio/locust", "name": "_construct_basic_auth_str", "language": "python", "path": "locust/contrib/fasthttp.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To create an encoded string for HTTP Basic Authentication that can be used in an HTTP request header.\n2. **Input**: Two strings representing a username and a password.\n3. **Output**: A string that combines the username and password in a format suitable for HTTP Basic Authentication, encoded in Base64.\n4. **Procedure**: The function first ensures both the username and password are bytes, encoding them if they are strings. It then concatenates them with a colon in between, encodes the resulting string in Base64, and finally formats it with the prefix required for HTTP Basic Authentication headers.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: locust/html.py\nimport datetime\nimport glob\nimport os\nimport pathlib\nfrom html import escape\nfrom itertools import chain\nfrom json import dumps\n\nfrom jinja2 import Environment, FileSystemLoader\n\nfrom . import stats as stats_module\nfrom .runners import STATE_STOPPED, STATE_STOPPING, MasterRunner\nfrom .stats import sort_stats\nfrom .user.inspectuser import get_ratio\n\nPERCENTILES_FOR_HTML_REPORT = [0.50, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1.0]\n\n\ndef render_template(file, template_path, **kwargs):\n    env = Environment(loader=FileSystemLoader(template_path), extensions=[\"jinja2.ext.do\"])\n    template = env.get_template(file)\n    return template.render(**kwargs)\n\n\ndef get_html_report(\n    environment,\n    show_download_link=True,\n    use_modern_ui=False,\n    theme=\"\",\n):\n    root_path = os.path.dirname(os.path.abspath(__file__))\n    if use_modern_ui:\n        static_path = os.path.join(root_path, \"webui\", \"dist\", \"assets\")\n        template_path = os.path.join(root_path, \"webui\", \"dist\")\n    else:\n        static_path = os.path.join(root_path, \"static\")\n        template_path = os.path.join(root_path, \"templates\")\n\n    stats = environment.runner.stats\n\n    start_ts = stats.start_time\n    start_time = datetime.datetime.utcfromtimestamp(start_ts).strftime(\"%Y-%m-%d %H:%M:%S\")\n\n    end_ts = stats.last_request_timestamp\n    if end_ts:\n        end_time = datetime.datetime.utcfromtimestamp(end_ts).strftime(\"%Y-%m-%d %H:%M:%S\")\n    else:\n        end_time = start_time\n\n    host = None\n    if environment.host:\n        host = environment.host\n    elif environment.runner.user_classes:\n        all_hosts = {l.host for l in environment.runner.user_classes}\n        if len(all_hosts) == 1:\n            host = list(all_hosts)[0]\n\n    requests_statistics = list(chain(sort_stats(stats.entries), [stats.total]))\n    failures_statistics = sort_stats(stats.errors)\n    exceptions_statistics = [\n        {**exc, \"nodes\": \", \".join(exc[\"nodes\"])} for exc in environment.runner.exceptions.values()\n    ]\n\n    history = stats.history\n\n    static_js = []\n    if use_modern_ui:\n        js_files = [os.path.basename(filepath) for filepath in glob.glob(os.path.join(static_path, \"*.js\"))]\n    else:\n        js_files = [\"jquery-1.11.3.min.js\", \"echarts.common.min.js\", \"vintage.js\", \"chart.js\", \"tasks.js\"]\n\n    for js_file in js_files:\n        path = os.path.join(static_path, js_file)\n        static_js.append(\"// \" + js_file + \"\\n\")\n        with open(path, encoding=\"utf8\") as f:\n            static_js.append(f.read())\n        static_js.extend([\"\", \"\"])\n\n    if not use_modern_ui:\n        static_css = []\n        css_files = [\"tables.css\"]\n        for css_file in css_files:\n            path = os.path.join(static_path, \"css\", css_file)\n            static_css.append(\"/* \" + css_file + \" */\")\n            with open(path, encoding=\"utf8\") as f:\n                static_css.append(f.read())\n            static_css.extend([\"\", \"\"])\n\n    is_distributed = isinstance(environment.runner, MasterRunner)\n    user_spawned = (\n        environment.runner.reported_user_classes_count if is_distributed else environment.runner.user_classes_count\n    )\n\n    if environment.runner.state in [STATE_STOPPED, STATE_STOPPING]:\n        user_spawned = environment.runner.final_user_classes_count\n\n    task_data = {\n        \"per_class\": get_ratio(environment.user_classes, user_spawned, False),\n        \"total\": get_ratio(environment.user_classes, user_spawned, True),\n    }\n\n    if use_modern_ui:\n        res = render_template(\n            \"report.html\",\n            template_path,\n            template_args={\n                \"is_report\": True,\n                \"requests_statistics\": [stat.to_dict(escape_string_values=True) for stat in requests_statistics],\n                \"failures_statistics\": [stat.to_dict() for stat in failures_statistics],\n                \"exceptions_statistics\": [stat for stat in exceptions_statistics],\n                \"response_time_statistics\": [\n                    {\n                        \"name\": escape(stat.name),\n                        \"method\": escape(stat.method or \"\"),\n                        **{\n                            str(percentile): stat.get_response_time_percentile(percentile)\n                            for percentile in PERCENTILES_FOR_HTML_REPORT\n                        },\n                    }\n                    for stat in requests_statistics\n                ],\n                \"start_time\": start_time,\n                \"end_time\": end_time,\n                \"host\": escape(str(host)),\n                \"history\": history,\n                \"show_download_link\": show_download_link,\n                \"locustfile\": escape(str(environment.locustfile)),\n                \"tasks\": task_data,\n                \"percentiles_to_chart\": stats_module.MODERN_UI_PERCENTILES_TO_CHART,\n            },\n            theme=theme,\n            static_js=\"\\n\".join(static_js),\n        )\n    else:\n        res = render_template(\n            \"report.html\",\n            template_path,\n            int=int,\n            round=round,\n            escape=escape,\n            str=str,\n            requests_statistics=requests_statistics,\n            failures_statistics=failures_statistics,\n...\n# Path: locust/util/cache.py\nimport functools\nfrom time import time\n\n\ndef memoize(timeout, dynamic_timeout=False):\n    \"\"\"\n    Memoization decorator with support for timeout.\n\n    If dynamic_timeout is set, the cache timeout is doubled if the cached function\n    takes longer time to run than the timeout time\n    \"\"\"\n    cache = {\"timeout\": timeout}\n\n    def decorator(func):\n        @functools.wraps(func)\n        def wrapper(*args, **kwargs):\n            start = time()\n            if (not \"time\" in cache) or (start - cache[\"time\"] > cache[\"timeout\"]):\n                # cache miss\n                cache[\"result\"] = func(*args, **kwargs)\n                cache[\"time\"] = time()\n                if dynamic_timeout and cache[\"time\"] - start > cache[\"timeout\"]:\n                    cache[\"timeout\"] *= 2\n            return cache[\"result\"]\n\n        def clear_cache():\n            if \"time\" in cache:\n                del cache[\"time\"]\n            if \"result\" in cache:\n                del cache[\"result\"]\n\n        wrapper.clear_cache = clear_cache\n        return wrapper\n\n    return decorator\n\n# Path: locust/util/timespan.py\nimport re\nfrom datetime import timedelta\n\n\ndef parse_timespan(time_str):\n    \"\"\"\n    Parse a string representing a time span and return the number of seconds.\n    Valid formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\n    \"\"\"\n    if not time_str:\n        raise ValueError(\"Invalid time span format\")\n\n    if re.match(r\"^\\d+$\", time_str):\n        # if an int is specified we assume they want seconds\n        return int(time_str)\n\n    timespan_regex = re.compile(r\"((?P<hours>\\d+?)h)?((?P<minutes>\\d+?)m)?((?P<seconds>\\d+?)s)?\")\n    parts = timespan_regex.match(time_str)\n    if not parts:\n        raise ValueError(\"Invalid time span format. Valid formats: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n    parts = parts.groupdict()\n    time_params = {name: int(value) for name, value in parts.items() if value}\n    if not time_params:\n        raise ValueError(\"Invalid time span format. Valid formats: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\")\n    return int(timedelta(**time_params).total_seconds())\n\n# Path: locust/web.py\nfrom __future__ import annotations\n\nimport csv\nimport json\nimport logging\nimport os.path\nfrom functools import wraps\nfrom html import escape\nfrom io import StringIO\nfrom itertools import chain\nfrom json import dumps\nfrom time import time\nfrom typing import TYPE_CHECKING, Any\n\nimport gevent\nfrom flask import (\n    Flask,\n    Response,\n    jsonify,\n    make_response,\n    redirect,\n    render_template,\n    request,\n    send_file,\n    send_from_directory,\n    url_for,\n)\nfrom flask_cors import CORS\nfrom flask_login import LoginManager, login_required\nfrom gevent import pywsgi\n\nfrom . import __version__ as version\nfrom . import argument_parser\nfrom . import stats as stats_module\nfrom .html import get_html_report\nfrom .log import greenlet_exception_logger\nfrom .runners import STATE_MISSING, STATE_RUNNING, MasterRunner\nfrom .stats import StatsCSV, StatsCSVFileWriter, StatsErrorDict, sort_stats\nfrom .user.inspectuser import get_ratio\nfrom .util.cache import memoize\nfrom .util.timespan import parse_timespan\n\nif TYPE_CHECKING:\n    from .env import Environment\n\n\nlogger = logging.getLogger(__name__)\ngreenlet_exception_handler = greenlet_exception_logger(logger)\n\nDEFAULT_CACHE_TIME = 2.0\n\n\nclass WebUI:\n    \"\"\"\n    Sets up and runs a Flask web app that can start and stop load tests using the\n    :attr:`environment.runner <locust.env.Environment.runner>` as well as show the load test statistics\n    in :attr:`environment.stats <locust.env.Environment.stats>`\n    \"\"\"\n\n    app: Flask | None = None\n    \"\"\"\n    Reference to the :class:`flask.Flask` app. Can be used to add additional web routes and customize\n    the Flask app in other various ways. Example::\n\n        from flask import request\n\n        @web_ui.app.route(\"/my_custom_route\")\n        def my_custom_route():\n            return \"your IP is: %s\" % request.remote_addr\n    \"\"\"\n\n    greenlet: gevent.Greenlet | None = None\n    \"\"\"\n    Greenlet of the running web server\n    \"\"\"\n\n    server: pywsgi.WSGIServer | None = None\n    \"\"\"Reference to the :class:`pyqsgi.WSGIServer` instance\"\"\"\n\n    template_args: dict[str, Any]\n    \"\"\"Arguments used to render index.html for the web UI. Must be used with custom templates\n    extending index.html.\"\"\"\n\n    auth_args: dict[str, Any]\n    \"\"\"Arguments used to render auth.html for the web UI auth page. Must be used when configuring auth\"\"\"\n\n    def __init__(\n        self,\n        environment: Environment,\n        host: str,\n        port: int,\n        web_login: bool = False,\n        tls_cert: str | None = None,\n        tls_key: str | None = None,\n        stats_csv_writer: StatsCSV | None = None,\n        delayed_start=False,\n        userclass_picker_is_active=False,\n        modern_ui=False,\n    ):\n        \"\"\"\n        Create WebUI instance and start running the web server in a separate greenlet (self.greenlet)\n\n        Arguments:\n        environment: Reference to the current Locust Environment\n        host: Host/interface that the web server should accept connections to\n        port: Port that the web server should listen to\n        web_login:  Enables a login page for the modern UI\n        tls_cert: A path to a TLS certificate\n        tls_key: A path to a TLS private key\n        delayed_start: Whether or not to delay starting web UI until `start()` is called. Delaying web UI start\n                       allows for adding Flask routes or Blueprints before accepting requests, avoiding errors.\n        \"\"\"\n        environment.web_ui = self\n        self.stats_csv_writer = stats_csv_writer or StatsCSV(environment, stats_module.PERCENTILES_TO_REPORT)\n        self.environment = environment\n        self.host = host\n        self.port = port\n        self.tls_cert = tls_cert\n        self.tls_key = tls_key\n        self.userclass_picker_is_active = userclass_picker_is_active\n        self.modern_ui = modern_ui\n        self.web_login = web_login\n        app = Flask(__name__)\n        CORS(app)\n        self.app = app\n        app.jinja_env.add_extension(\"jinja2.ext.do\")\n        app.debug = True\n        root_path = os.path.dirname(os.path.abspath(__file__))\n        app.root_path = root_path\n        self.webui_build_path = os.path.join(root_path, \"webui\", \"dist\")\n        self.greenlet: gevent.Greenlet | None = None\n        self._swarm_greenlet: gevent.Greenlet | None = None\n        self.template_args = {}\n        self.auth_args = {}\n\n        if self.web_login:\n            self.login_manager = LoginManager()\n            self.login_manager.init_app(app)\n            self.login_manager.login_view = \"login\"\n\n        if environment.runner:\n            self.update_template_args()\n        if not delayed_start:\n            self.start()\n\n        @app.errorhandler(Exception)\n        def handle_exception(error):\n            error_message = str(error)\n            logger.log(logging.CRITICAL, error_message)\n            return make_response(error_message, 500)\n\n        @app.route(\"/assets/<path:path>\")\n        def send_assets(path):\n            return send_from_directory(os.path.join(self.webui_build_path, \"assets\"), path)\n\n        @app.route(\"/\")\n        @self.auth_required_if_enabled\n        def index() -> str | Response:\n            if not environment.runner:\n                return make_response(\"Error: Locust Environment does not have any runner\", 500)\n            self.update_template_args()\n\n            if self.modern_ui:\n                self.set_static_modern_ui()\n\n                return render_template(\"index.html\", template_args=self.template_args)\n            return render_template(\"index.html\", **self.template_args)\n\n        @app.route(\"/swarm\", methods=[\"POST\"])\n        @self.auth_required_if_enabled\n        def swarm() -> Response:\n            assert request.method == \"POST\"\n\n            # Loading UserClasses & ShapeClasses if Locust is running with UserClass Picker\n            if self.userclass_picker_is_active:\n                if not self.environment.available_user_classes:\n                    err_msg = \"UserClass picker is active but there are no available UserClasses\"\n                    return jsonify({\"success\": False, \"message\": err_msg, \"host\": environment.host})\n\n                # Getting Specified User Classes\n                form_data_user_class_names = request.form.getlist(\"user_classes\")\n\n                # Updating UserClasses\n                if form_data_user_class_names:\n                    user_classes = {}\n                    for user_class_name, user_class_object in self.environment.available_user_classes.items():\n                        if user_class_name in form_data_user_class_names:\n                            user_classes[user_class_name] = user_class_object\n\n                else:\n                    if self.environment.runner and self.environment.runner.state == STATE_RUNNING:\n                        # Test is already running\n                        # Using the user classes that have already been selected\n                        user_classes = {\n                            key: value\n                            for (key, value) in self.environment.available_user_classes.items()\n                            if value in self.environment.user_classes\n                        }\n                    else:\n                        # Starting test with no user class selection\n                        # Defaulting to using all available user classes\n                        user_classes = self.environment.available_user_classes\n\n                self._update_user_classes(user_classes)\n\n                # Updating ShapeClass if specified in WebUI Form\n                form_data_shape_class_name = request.form.get(\"shape_class\", \"Default\")\n                if form_data_shape_class_name == \"Default\":\n                    self._update_shape_class(None)\n                else:\n                    self._update_shape_class(form_data_shape_class_name)\n\n            parsed_options_dict = vars(environment.parsed_options) if environment.parsed_options else {}\n            run_time = None\n            for key, value in request.form.items():\n                if key == \"user_count\":  # if we just renamed this field to \"users\" we wouldn't need this\n                    user_count = int(value)\n                elif key == \"spawn_rate\":\n                    spawn_rate = float(value)\n                elif key == \"host\":\n                    # Replace < > to guard against XSS\n                    environment.host = str(request.form[\"host\"]).replace(\"<\", \"\").replace(\">\", \"\")\n                elif key == \"user_classes\":\n                    # Set environment.parsed_options.user_classes to the selected user_classes\n                    parsed_options_dict[key] = request.form.getlist(\"user_classes\")\n                elif key == \"run_time\":\n                    if not value:\n                        continue\n                    try:\n                        run_time = parse_timespan(value)\n                    except ValueError:\n                        err_msg = \"Valid run_time formats are : 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.\"\n                        logger.error(err_msg)\n                        return jsonify({\"success\": False, \"message\": err_msg, \"host\": environment.host})\n                elif key in parsed_options_dict:\n                    # update the value in environment.parsed_options, but dont change the type.\n                    parsed_options_value = parsed_options_dict[key]\n\n                    if isinstance(parsed_options_value, bool):\n                        parsed_options_dict[key] = value == \"true\"\n                    elif parsed_options_value is None:\n                        parsed_options_dict[key] = parsed_options_value\n                    else:\n                        parsed_options_dict[key] = type(parsed_options_dict[key])(value)\n\n            if environment.shape_class and environment.runner is not None:\n                environment.runner.start_shape()\n                return jsonify(\n                    {\n                        \"success\": True,\n                        \"message\": f\"Swarming started using shape class '{type(environment.shape_class).__name__}'\",\n                        \"host\": environment.host,\n                    }\n                )\n\n            if self._swarm_greenlet is not None:\n                self._swarm_greenlet.kill(block=True)\n                self._swarm_greenlet = None\n\n            if environment.runner is not None:\n                self._swarm_greenlet = gevent.spawn(environment.runner.start, user_count, spawn_rate)\n                self._swarm_greenlet.link_exception(greenlet_exception_handler)\n                response_data = {\n                    \"success\": True,\n                    \"message\": \"Swarming started\",\n                    \"host\": environment.host,\n                }\n                if run_time:\n                    gevent.spawn_later(run_time, self._stop_runners).link_exception(greenlet_exception_handler)\n                    response_data[\"run_time\"] = run_time\n\n                if self.userclass_picker_is_active:\n                    response_data[\"user_classes\"] = sorted(user_classes.keys())\n\n                return jsonify(response_data)\n            else:\n                return jsonify({\"success\": False, \"message\": \"No runner\", \"host\": environment.host})\n\n        @app.route(\"/stop\")\n        @self.auth_required_if_enabled\n        def stop() -> Response:\n            if self._swarm_greenlet is not None:\n                self._swarm_greenlet.kill(block=True)\n                self._swarm_greenlet = None\n            if environment.runner is not None:\n                environment.runner.stop()\n            return jsonify({\"success\": True, \"message\": \"Test stopped\"})\n\n        @app.route(\"/stats/reset\")\n        @self.auth_required_if_enabled\n        def reset_stats() -> str:\n            environment.events.reset_stats.fire()\n            if environment.runner is not None:\n                environment.runner.stats.reset_all()\n                environment.runner.exceptions = {}\n            return \"ok\"\n\n        @app.route(\"/stats/report\")\n        @self.auth_required_if_enabled\n        def stats_report() -> Response:\n            theme = request.args.get(\"theme\", \"\")\n            res = get_html_report(\n                self.environment,\n                show_download_link=not request.args.get(\"download\"),\n                use_modern_ui=self.modern_ui,\n                theme=theme,\n            )\n            if request.args.get(\"download\"):\n                res = app.make_response(res)\n                res.headers[\"Content-Disposition\"] = f\"attachment;filename=report_{time()}.html\"\n            return res\n\n        def _download_csv_suggest_file_name(suggest_filename_prefix: str) -> str:\n            \"\"\"Generate csv file download attachment filename suggestion.\n\n            Arguments:\n            suggest_filename_prefix: Prefix of the filename to suggest for saving the download. Will be appended with timestamp.\n            \"\"\"\n\n            return f\"{suggest_filename_prefix}_{time()}.csv\"\n\n        def _download_csv_response(csv_data: str, filename_prefix: str) -> Response:\n            \"\"\"Generate csv file download response with 'csv_data'.\n\n            Arguments:\n            csv_data: CSV header and data rows.\n            filename_prefix: Prefix of the filename to suggest for saving the download. Will be appended with timestamp.\n            \"\"\"\n\n            response = make_response(csv_data)\n            response.headers[\"Content-type\"] = \"text/csv\"\n            response.headers[\"Content-disposition\"] = (\n                f\"attachment;filename={_download_csv_suggest_file_name(filename_prefix)}\"\n            )\n            return response\n\n        @app.route(\"/stats/requests/csv\")\n        @self.auth_required_if_enabled\n        def request_stats_csv() -> Response:\n            data = StringIO()\n            writer = csv.writer(data)\n            self.stats_csv_writer.requests_csv(writer)\n            return _download_csv_response(data.getvalue(), \"requests\")\n\n        @app.route(\"/stats/requests_full_history/csv\")\n        @self.auth_required_if_enabled\n        def request_stats_full_history_csv() -> Response:\n            options = self.environment.parsed_options\n            if options and options.stats_history_enabled and isinstance(self.stats_csv_writer, StatsCSVFileWriter):\n                return send_file(\n                    os.path.abspath(self.stats_csv_writer.stats_history_file_name()),\n                    mimetype=\"text/csv\",\n                    as_attachment=True,\n                    download_name=_download_csv_suggest_file_name(\"requests_full_history\"),\n                    etag=True,\n                    max_age=0,\n                    conditional=True,\n                    last_modified=None,\n                )\n\n            return make_response(\"Error: Server was not started with option to generate full history.\", 404)\n\n        @app.route(\"/stats/failures/csv\")\n        @self.auth_required_if_enabled\n        def failures_stats_csv() -> Response:\n            data = StringIO()\n            writer = csv.writer(data)\n            self.stats_csv_writer.failures_csv(writer)\n            return _download_csv_response(data.getvalue(), \"failures\")\n\n        @app.route(\"/stats/requests\")\n        @self.auth_required_if_enabled\n        @memoize(timeout=DEFAULT_CACHE_TIME, dynamic_timeout=True)\n        def request_stats() -> Response:\n            stats: list[dict[str, Any]] = []\n            errors: list[StatsErrorDict] = []\n\n            if environment.runner is None:\n                report = {\n                    \"stats\": stats,\n                    \"errors\": errors,\n                    \"total_rps\": 0.0,\n                    \"total_fail_per_sec\": 0.0,\n                    \"fail_ratio\": 0.0,\n                    \"current_response_time_percentile_1\": None,\n                    \"current_response_time_percentile_2\": None,\n                    \"state\": STATE_MISSING,\n                    \"user_count\": 0,\n                }\n\n                if isinstance(environment.runner, MasterRunner):\n                    report.update({\"workers\": []})\n\n                return jsonify(report)\n\n            for s in chain(sort_stats(environment.runner.stats.entries), [environment.runner.stats.total]):\n                stats.append(s.to_dict())\n\n            for e in environment.runner.errors.values():\n                err_dict = e.serialize()\n                err_dict[\"name\"] = escape(err_dict[\"name\"])\n                err_dict[\"error\"] = escape(err_dict[\"error\"])\n                errors.append(err_dict)\n\n            # Truncate the total number of stats and errors displayed since a large number of rows will cause the app\n            # to render extremely slowly. Aggregate stats should be preserved.\n            truncated_stats = stats[:500]\n            if len(stats) > 500:\n                truncated_stats += [stats[-1]]\n\n            report = {\"stats\": truncated_stats, \"errors\": errors[:500]}\n            total_stats = stats[-1]\n\n            if stats:\n                report[\"total_rps\"] = total_stats[\"current_rps\"]\n                report[\"total_fail_per_sec\"] = total_stats[\"current_fail_per_sec\"]\n                report[\"total_avg_response_time\"] = total_stats[\"avg_response_time\"]\n                report[\"fail_ratio\"] = environment.runner.stats.total.fail_ratio\n\n                if self.modern_ui:\n                    report[\"current_response_time_percentiles\"] = {\n                        f\"response_time_percentile_{percentile}\": environment.runner.stats.total.get_current_response_time_percentile(\n                            percentile\n                        )\n                        for percentile in stats_module.MODERN_UI_PERCENTILES_TO_CHART\n                    }\n                else:\n                    report[\"current_response_time_percentile_1\"] = (\n                        environment.runner.stats.total.get_current_response_time_percentile(\n                            stats_module.PERCENTILES_TO_CHART[0]\n                        )\n                    )\n                    report[\"current_response_time_percentile_2\"] = (\n                        environment.runner.stats.total.get_current_response_time_percentile(\n                            stats_module.PERCENTILES_TO_CHART[1]\n                        )\n                    )\n\n            if isinstance(environment.runner, MasterRunner):\n                workers = []\n                for worker in environment.runner.clients.values():\n                    workers.append(\n                        {\n                            \"id\": worker.id,\n                            \"state\": worker.state,\n                            \"user_count\": worker.user_count,\n                            \"cpu_usage\": worker.cpu_usage,\n                            \"memory_usage\": worker.memory_usage,\n                        }\n                    )\n\n                report[\"workers\"] = workers\n\n            report[\"state\"] = environment.runner.state\n            report[\"user_count\"] = environment.runner.user_count\n\n            return jsonify(report)\n\n        @app.route(\"/exceptions\")\n        @self.auth_required_if_enabled\n        def exceptions() -> Response:\n            return jsonify(\n                {\n                    \"exceptions\": [\n                        {\n                            \"count\": row[\"count\"],\n                            \"msg\": escape(row[\"msg\"]),\n                            \"traceback\": escape(row[\"traceback\"]),\n                            \"nodes\": \", \".join(row[\"nodes\"]),\n                        }\n                        for row in (environment.runner.exceptions.values() if environment.runner is not None else [])\n                    ]\n                }\n            )\n\n        @app.route(\"/exceptions/csv\")\n        @self.auth_required_if_enabled\n        def exceptions_csv() -> Response:\n            data = StringIO()\n            writer = csv.writer(data)\n            self.stats_csv_writer.exceptions_csv(writer)\n            return _download_csv_response(data.getvalue(), \"exceptions\")\n\n        @app.route(\"/tasks\")\n        @self.auth_required_if_enabled\n        def tasks() -> dict[str, dict[str, dict[str, float]]]:\n            runner = self.environment.runner\n            user_spawned: dict[str, int]\n            if runner is None:\n                user_spawned = {}\n            else:\n                user_spawned = (\n                    runner.reported_user_classes_count\n                    if isinstance(runner, MasterRunner)\n                    else runner.user_classes_count\n                )\n\n            task_data = {\n                \"per_class\": get_ratio(self.environment.user_classes, user_spawned, False),\n                \"total\": get_ratio(self.environment.user_classes, user_spawned, True),\n            }\n            return task_data\n\n        @app.route(\"/logs\")\n        @self.auth_required_if_enabled\n        def logs():\n            log_reader_handler = [\n                handler for handler in logging.getLogger(\"root\").handlers if handler.name == \"log_reader\"\n            ]\n\n            if log_reader_handler:\n                logs = log_reader_handler[0].logs\n            else:\n                logs = []\n\n            return jsonify({\"logs\": logs})\n\n        @app.route(\"/login\")\n        def login():\n            if not self.web_login:\n                return redirect(url_for(\"index\"))\n\n            if self.modern_ui:\n                self.set_static_modern_ui()\n\n                return render_template(\n                    \"auth.html\",\n                    auth_args=self.auth_args,\n                )\n            else:\n                return \"Web Auth is only available on the modern web ui.\"\n\n        @app.route(\"/user\", methods=[\"POST\"])\n        def update_user():\n            assert request.method == \"POST\"\n\n            user_settings = json.loads(request.data)\n            self.environment.update_user_class(user_settings)\n\n            return {}, 201\n\n    def start(self):\n        self.greenlet = gevent.spawn(self.start_server)\n        self.greenlet.link_exception(greenlet_exception_handler)\n\n    def start_server(self):\n        if self.tls_cert and self.tls_key:\n            self.server = pywsgi.WSGIServer(\n                (self.host, self.port), self.app, log=None, keyfile=self.tls_key, certfile=self.tls_cert\n            )\n        else:\n            self.server = pywsgi.WSGIServer((self.host, self.port), self.app, log=None)\n        self.server.serve_forever()\n\n    def stop(self):\n        \"\"\"\n        Stop the running web server\n        \"\"\"\n        self.server.stop()\n\n    def auth_required_if_enabled(self, view_func):\n        \"\"\"\n        Decorator that can be used on custom route methods that will turn on Flask Login\n        authentication if the ``--web-login`` flag is used. Example::\n\n            @web_ui.app.route(\"/my_custom_route\")\n            @web_ui.auth_required_if_enabled\n            def my_custom_route():\n                return \"custom response\"\n        \"\"\"\n\n        @wraps(view_func)\n        def wrapper(*args, **kwargs):\n            if self.web_login:\n                try:\n                    return login_required(view_func)(*args, **kwargs)\n                except Exception as e:\n                    return f\"Locust auth exception: {e} See https://docs.locust.io/en/stable/extending-locust.html#authentication for configuring authentication.\"\n            else:\n                return view_func(*args, **kwargs)\n\n        return wrapper\n\n    def set_static_modern_ui(self):\n        self.app.template_folder = self.webui_build_path\n        self.app.static_folder = os.path.join(self.webui_build_path, \"assets\")\n        self.app.static_url_path = \"/assets/\"\n\n    def update_template_args(self):\n        override_host_warning = False\n        if self.environment.host:\n            host = self.environment.host\n        elif self.environment.runner.user_classes:\n            all_hosts = {l.host for l in self.environment.runner.user_classes}\n            if len(all_hosts) == 1:\n                host = list(all_hosts)[0]\n            else:\n                # since we have multiple User classes with different host attributes, we'll\n                # inform that specifying host will override the host for all User classes\n                override_host_warning = True\n                host = None\n        else:\n            host = None\n\n        options = self.environment.parsed_options\n\n        is_distributed = isinstance(self.environment.runner, MasterRunner)\n        if is_distributed:\n            worker_count = self.environment.runner.worker_count\n        else:\n            worker_count = 0\n\n        stats = self.environment.runner.stats\n        extra_options = argument_parser.ui_extra_args_dict()\n\n        available_user_classes = None\n        users = None\n        if self.environment.available_user_classes:\n            available_user_classes = sorted(self.environment.available_user_classes)\n            users = {\n                user_class_name: user_class.json()\n                for (user_class_name, user_class) in self.environment.available_user_classes.items()\n            }\n\n        available_shape_classes = [\"Default\"]\n        if self.environment.available_shape_classes:\n            available_shape_classes += sorted(self.environment.available_shape_classes.keys())\n\n        available_user_tasks = (\n            {\n                user_class_name: [task.__name__ for task in user_class]\n                for (user_class_name, user_class) in self.environment.available_user_tasks.items()\n            }\n            if self.environment.available_user_tasks\n            else None\n        )\n\n        if self.modern_ui:\n            percentiles = {\n                \"percentiles_to_chart\": stats_module.MODERN_UI_PERCENTILES_TO_CHART,\n                \"percentiles_to_statistics\": stats_module.PERCENTILES_TO_STATISTICS,\n            }\n        else:\n            percentiles = {\n                \"percentile1\": stats_module.PERCENTILES_TO_CHART[0],\n                \"percentile2\": stats_module.PERCENTILES_TO_CHART[1],\n            }\n\n        self.template_args = {\n            \"locustfile\": self.environment.locustfile,\n            \"state\": self.environment.runner.state,\n            \"is_distributed\": is_distributed,\n            \"user_count\": self.environment.runner.user_count,\n            \"version\": version,\n            \"host\": host if host else \"\",\n            \"history\": stats.history if stats.num_requests > 0 else {},\n            \"override_host_warning\": override_host_warning,\n            \"num_users\": options and options.num_users,\n            \"spawn_rate\": options and options.spawn_rate,\n            \"worker_count\": worker_count,\n            \"hide_common_options\": (\n                self.environment.shape_class\n                and not (self.userclass_picker_is_active or self.environment.shape_class.use_common_options)\n            ),\n            \"stats_history_enabled\": options and options.stats_history_enabled,\n            \"tasks\": dumps({}),\n            \"extra_options\": extra_options,\n            \"run_time\": options and options.run_time,\n            \"show_userclass_picker\": self.userclass_picker_is_active,\n            \"available_user_classes\": available_user_classes,\n            \"available_shape_classes\": available_shape_classes,\n            \"available_user_tasks\": available_user_tasks,\n            \"users\": users,\n            **percentiles,\n        }\n\n    def _update_shape_class(self, shape_class_name):\n        if shape_class_name:\n            shape_class = self.environment.available_shape_classes[shape_class_name]\n            shape_class.runner = self.environment.runner\n        else:\n            shape_class = None\n\n        # Validating ShapeClass\n        self.environment.shape_class = shape_class\n        self.environment._validate_shape_class_instance()\n\n    def _update_user_classes(self, user_classes):\n        self.environment.user_classes = list(user_classes.values())\n        # populate the locustfile which used in web ui title only\n        if self.environment.locustfile is None:\n            self.environment.locustfile = \",\".join(self.environment.user_classes_by_name.keys())\n\n        # Validating UserClasses\n        self.environment._remove_user_classes_with_weight_zero()\n        self.environment._validate_user_class_name_uniqueness()\n\n    def _stop_runners(self):\n        self.environment.runner.stop()\n\n# Path: locust/env.py\nfrom __future__ import annotations\n\nfrom operator import methodcaller\nfrom typing import Callable, TypeVar\n\nfrom configargparse import Namespace\n\nfrom .dispatch import UsersDispatcher\nfrom .event import Events\nfrom .exception import RunnerAlreadyExistsError\nfrom .runners import LocalRunner, MasterRunner, Runner, WorkerRunner\nfrom .shape import LoadTestShape\nfrom .stats import RequestStats, StatsCSV\nfrom .user import User\nfrom .user.task import TaskHolder, TaskSet, filter_tasks_by_tags\nfrom .web import WebUI\n\nRunnerType = TypeVar(\"RunnerType\", bound=Runner)\n\n\nclass Environment:\n    def __init__(\n        self,\n        *,\n        user_classes: list[type[User]] | None = None,\n        shape_class: LoadTestShape | None = None,\n        tags: list[str] | None = None,\n        locustfile: str | None = None,\n        exclude_tags: list[str] | None = None,\n        events: Events | None = None,\n        host: str | None = None,\n        reset_stats=False,\n        stop_timeout: float | None = None,\n        catch_exceptions=True,\n        parsed_options: Namespace | None = None,\n        available_user_classes: dict[str, User] | None = None,\n        available_shape_classes: dict[str, LoadTestShape] | None = None,\n        available_user_tasks: dict[str, list[TaskSet | Callable]] | None = None,\n        dispatcher_class: type[UsersDispatcher] = UsersDispatcher,\n    ):\n        self.runner: Runner | None = None\n        \"\"\"Reference to the :class:`Runner <locust.runners.Runner>` instance\"\"\"\n\n        self.web_ui: WebUI | None = None\n        \"\"\"Reference to the WebUI instance\"\"\"\n\n        self.process_exit_code: int | None = None\n        \"\"\"\n        If set it'll be the exit code of the Locust process\n        \"\"\"\n\n        if events:\n            self.events = events\n            \"\"\"\n            Event hooks used by Locust internally, as well as to extend Locust's functionality\n            See :ref:`events` for available events.\n            \"\"\"\n        else:\n            self.events = Events()\n\n        self.locustfile = locustfile\n        \"\"\"Filename (not path) of locustfile\"\"\"\n        self.user_classes: list[type[User]] = user_classes or []\n        \"\"\"User classes that the runner will run\"\"\"\n        self.shape_class = shape_class\n        \"\"\"A shape class to control the shape of the load test\"\"\"\n        self.tags = tags\n        \"\"\"If set, only tasks that are tagged by tags in this list will be executed. Leave this as None to use the one from parsed_options\"\"\"\n        self.exclude_tags = exclude_tags\n        \"\"\"If set, only tasks that aren't tagged by tags in this list will be executed. Leave this as None to use the one from parsed_options\"\"\"\n        self.stats = RequestStats()\n        \"\"\"Reference to RequestStats instance\"\"\"\n        self.host = host\n        \"\"\"Base URL of the target system\"\"\"\n        self.reset_stats = reset_stats\n        \"\"\"Determines if stats should be reset once all simulated users have been spawned\"\"\"\n        if stop_timeout is not None:\n            self.stop_timeout = stop_timeout\n        elif parsed_options:\n            self.stop_timeout = float(getattr(parsed_options, \"stop_timeout\", 0.0))\n        else:\n            self.stop_timeout = 0.0\n        \"\"\"\n        If set, the runner will try to stop the running users gracefully and wait this many seconds\n        before killing them hard.\n        \"\"\"\n        self.catch_exceptions = catch_exceptions\n        \"\"\"\n        If True exceptions that happen within running users will be caught (and reported in UI/console).\n        If False, exceptions will be raised.\n        \"\"\"\n        self.parsed_options = parsed_options\n        \"\"\"Reference to the parsed command line options (used to pre-populate fields in Web UI). When using Locust as a library, this should either be `None` or an object created by `argument_parser.parse_args()`\"\"\"\n        self.available_user_classes = available_user_classes\n        \"\"\"List of the available User Classes to pick from in the UserClass Picker\"\"\"\n        self.available_shape_classes = available_shape_classes\n        \"\"\"List of the available Shape Classes to pick from in the ShapeClass Picker\"\"\"\n        self.available_user_tasks = available_user_tasks\n        \"\"\"List of the available Tasks per User Classes to pick from in the Task Picker\"\"\"\n        self.dispatcher_class = dispatcher_class\n        \"\"\"A user dispatcher class that decides how users are spawned, default :class:`UsersDispatcher <locust.dispatch.UsersDispatcher>`\"\"\"\n\n        self._remove_user_classes_with_weight_zero()\n        self._validate_user_class_name_uniqueness()\n        self._validate_shape_class_instance()\n\n    def _create_runner(\n        self,\n        runner_class: type[RunnerType],\n        *args,\n        **kwargs,\n    ) -> RunnerType:\n        if self.runner is not None:\n            raise RunnerAlreadyExistsError(f\"Environment.runner already exists ({self.runner})\")\n        self.runner = runner_class(self, *args, **kwargs)\n\n        # Attach the runner to the shape class so that the shape class can access user count state\n        if self.shape_class:\n            self.shape_class.runner = self.runner\n\n        return self.runner\n\n    def create_local_runner(self) -> LocalRunner:\n        \"\"\"\n        Create a :class:`LocalRunner <locust.runners.LocalRunner>` instance for this Environment\n        \"\"\"\n        return self._create_runner(LocalRunner)\n\n    def create_master_runner(self, master_bind_host=\"*\", master_bind_port=5557) -> MasterRunner:\n        \"\"\"\n        Create a :class:`MasterRunner <locust.runners.MasterRunner>` instance for this Environment\n\n        :param master_bind_host: Interface/host that the master should use for incoming worker connections.\n                                 Defaults to \"*\" which means all interfaces.\n        :param master_bind_port: Port that the master should listen for incoming worker connections on\n        \"\"\"\n        return self._create_runner(\n            MasterRunner,\n            master_bind_host=master_bind_host,\n            master_bind_port=master_bind_port,\n        )\n\n    def create_worker_runner(self, master_host: str, master_port: int) -> WorkerRunner:\n        \"\"\"\n        Create a :class:`WorkerRunner <locust.runners.WorkerRunner>` instance for this Environment\n\n        :param master_host: Host/IP of a running master node\n        :param master_port: Port on master node to connect to\n        \"\"\"\n        # Create a new RequestStats with use_response_times_cache set to False to save some memory\n        # and CPU cycles, since the response_times_cache is not needed for Worker nodes\n        self.stats = RequestStats(use_response_times_cache=False)\n        return self._create_runner(\n            WorkerRunner,\n            master_host=master_host,\n            master_port=master_port,\n        )\n\n    def create_web_ui(\n        self,\n        host=\"\",\n        port=8089,\n        web_login: bool = False,\n        tls_cert: str | None = None,\n        tls_key: str | None = None,\n        stats_csv_writer: StatsCSV | None = None,\n        delayed_start=False,\n        userclass_picker_is_active=False,\n        modern_ui=False,\n    ) -> WebUI:\n        \"\"\"\n        Creates a :class:`WebUI <locust.web.WebUI>` instance for this Environment and start running the web server\n\n        :param host: Host/interface that the web server should accept connections to. Defaults to \"\"\n                     which means all interfaces\n        :param port: Port that the web server should listen to\n        :param web_login: If provided, an authentication page will protect the app\n        :param tls_cert: An optional path (str) to a TLS cert. If this is provided the web UI will be\n                         served over HTTPS\n        :param tls_key: An optional path (str) to a TLS private key. If this is provided the web UI will be\n                        served over HTTPS\n        :param stats_csv_writer: `StatsCSV <stats_csv.StatsCSV>` instance.\n        :param delayed_start: Whether or not to delay starting web UI until `start()` is called. Delaying web UI start\n                              allows for adding Flask routes or Blueprints before accepting requests, avoiding errors.\n        \"\"\"\n        self.web_ui = WebUI(\n            self,\n            host,\n            port,\n            web_login=web_login,\n            tls_cert=tls_cert,\n            tls_key=tls_key,\n            stats_csv_writer=stats_csv_writer,\n            delayed_start=delayed_start,\n            userclass_picker_is_active=userclass_picker_is_active,\n            modern_ui=modern_ui,\n        )\n        return self.web_ui\n\n    def update_user_class(self, user_settings):\n        user_class_name = user_settings.get(\"user_class_name\")\n        user_class = self.available_user_classes[user_class_name]\n        user_tasks = self.available_user_tasks[user_class_name]\n\n        for key, value in user_settings.items():\n            if key not in [\"user_class_name\", \"tasks\"]:\n                setattr(user_class, key, value)\n            if key == \"tasks\":\n                user_class.tasks = [task for task in user_tasks if task.__name__ in value]\n\n    def _filter_tasks_by_tags(self) -> None:\n        \"\"\"\n        Filter the tasks on all the user_classes recursively, according to the tags and\n        exclude_tags attributes\n        \"\"\"\n        if getattr(self, \"_tasks_filtered\", False):\n            return  # only filter once\n        self._tasks_filtered = True\n\n        if self.tags is not None:\n            tags = set(self.tags)\n        elif self.parsed_options and self.parsed_options.tags:\n            tags = set(self.parsed_options.tags)\n        else:\n            tags = None\n\n        if self.exclude_tags is not None:\n            exclude_tags = set(self.exclude_tags)\n        elif self.parsed_options and self.parsed_options.exclude_tags:\n            exclude_tags = set(self.parsed_options.exclude_tags)\n        else:\n            exclude_tags = None\n\n        for user_class in self.user_classes:\n            filter_tasks_by_tags(user_class, tags, exclude_tags)\n\n    def _remove_user_classes_with_weight_zero(self) -> None:\n        \"\"\"\n        Remove user classes having a weight of zero.\n        \"\"\"\n        if len(self.user_classes) == 0:\n            # Preserve previous behaviour that allowed no user classes to be specified.\n            return\n        filtered_user_classes = [\n            user_class for user_class in self.user_classes if user_class.weight > 0 or user_class.fixed_count > 0\n        ]\n        if len(filtered_user_classes) == 0:\n            # TODO: Better exception than `ValueError`?\n            raise ValueError(\"There are no users with weight > 0.\")\n        self.user_classes[:] = filtered_user_classes\n\n    def assign_equal_weights(self) -> None:\n        \"\"\"\n        Update the user classes such that each user runs their specified tasks with equal\n        probability.\n        \"\"\"\n        for u in self.user_classes:\n            u.weight = 1\n            user_tasks: list[TaskSet | Callable] = []\n            tasks_frontier = u.tasks\n            while len(tasks_frontier) != 0:\n                t = tasks_frontier.pop()\n                if isinstance(t, TaskHolder):\n                    tasks_frontier.extend(t.tasks)\n                elif callable(t):\n                    if t not in user_tasks:\n                        user_tasks.append(t)\n                else:\n                    raise ValueError(\"Unrecognized task type in user\")\n            u.tasks = user_tasks\n\n    def _validate_user_class_name_uniqueness(self):\n        # Validate there's no class with the same name but in different modules\n        if len({user_class.__name__ for user_class in self.user_classes}) != len(self.user_classes):\n            raise ValueError(\n                \"The following user classes have the same class name: {}\".format(\n                    \", \".join(map(methodcaller(\"fullname\"), self.user_classes))\n                )\n            )\n\n    def _validate_shape_class_instance(self):\n        if self.shape_class is not None and not isinstance(self.shape_class, LoadTestShape):\n            raise ValueError(\n                \"shape_class should be instance of LoadTestShape or subclass LoadTestShape, but got: %s\"\n                % self.shape_class\n            )\n\n    @property\n    def user_classes_by_name(self) -> dict[str, type[User]]:\n        return {u.__name__: u for u in self.user_classes}\n\n# Path: locust/contrib/fasthttp.py\nfrom __future__ import annotations\n\nfrom locust.env import Environment\nfrom locust.exception import CatchResponseError, LocustError, ResponseError\nfrom locust.user import User\nfrom locust.util.deprecation import DeprecatedFastHttpLocustClass as FastHttpLocust\n\nimport json\nimport json as unshadowed_json  # some methods take a named parameter called json\nimport re\nimport socket\nimport time\nimport traceback\nfrom base64 import b64encode\nfrom contextlib import contextmanager\nfrom http.cookiejar import CookieJar\nfrom json.decoder import JSONDecodeError\nfrom ssl import SSLError\nfrom typing import Any, Callable, Generator, cast\nfrom urllib.parse import urlparse, urlunparse\n\nimport gevent\nfrom charset_normalizer import detect\nfrom gevent.timeout import Timeout\nfrom geventhttpclient._parser import HTTPParseError\nfrom geventhttpclient.client import HTTPClientPool\nfrom geventhttpclient.header import Headers\nfrom geventhttpclient.response import HTTPConnectionClosed, HTTPSocketPoolResponse\nfrom geventhttpclient.useragent import CompatRequest, CompatResponse, ConnectionError, UserAgent\n\n# borrow requests's content-type header parsing\nfrom requests.utils import get_encoding_from_headers\n\n# Monkey patch geventhttpclient.useragent.CompatRequest so that Cookiejar works with Python >= 3.3\n# More info: https://github.com/requests/requests/pull/871\nCompatRequest.unverifiable = False\n\n# Workaround for AttributeError: 'CompatRequest' object has no attribute 'type' in Cookiejar\n# https://github.com/locustio/locust/issues/1138\n# Might allow secure cookies over non-secure connections but that is a minor concern in a load testing tool\nCompatRequest.type = \"https\"\n\n# Regexp for checking if an absolute URL was specified\nabsolute_http_url_regexp = re.compile(r\"^https?://\", re.I)\n\n# List of exceptions that can be raised by geventhttpclient when sending an HTTP request,\n# and that should result in a Locust failure\nFAILURE_EXCEPTIONS = (\n    ConnectionError,\n    ConnectionRefusedError,\n    ConnectionResetError,\n    socket.error,\n    SSLError,\n    Timeout,\n    HTTPConnectionClosed,\n)\n\n\n\ndef _construct_basic_auth_str(username, password):\n    \"\"\"Construct Authorization header value to be used in HTTP Basic Auth\"\"\"\n    if isinstance(username, str):\n        username = username.encode(\"latin1\")\n    if isinstance(password, str):\n        password = password.encode(\"latin1\")\n    return \"Basic \" + b64encode(b\":\".join((username, password))).strip().decode(\"ascii\")\n\n\ndef insecure_ssl_context_factory():\n    context = gevent.ssl.create_default_context()\n    context.check_hostname = False\n    context.verify_mode = gevent.ssl.CERT_NONE\n    return context\n\n\nclass FastHttpSession:\n    auth_header = None\n\n    def __init__(\n        self,\n        environment: Environment,\n        base_url: str,\n        user: User | None,\n        insecure=True,\n        client_pool: HTTPClientPool | None = None,\n        ssl_context_factory: Callable | None = None,\n        **kwargs,\n    ):\n        self.environment = environment\n        self.base_url = base_url\n        self.cookiejar = CookieJar()\n        self.user = user\n        if not ssl_context_factory:\n            if insecure:\n                ssl_context_factory = insecure_ssl_context_factory\n            else:\n                ssl_context_factory = gevent.ssl.create_default_context\n        self.client = LocustUserAgent(\n            cookiejar=self.cookiejar,\n            ssl_context_factory=ssl_context_factory,\n            insecure=insecure,\n            client_pool=client_pool,\n            **kwargs,\n        )\n\n        # Check for basic authentication\n        parsed_url = urlparse(self.base_url)\n        if parsed_url.username and parsed_url.password:\n            netloc = parsed_url.hostname or \"\"\n            if parsed_url.port:\n                netloc += \":%d\" % parsed_url.port\n\n            # remove username and password from the base_url\n            self.base_url = urlunparse(\n                (parsed_url.scheme, netloc, parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)\n            )\n            # store authentication header (we construct this by using _basic_auth_str() function from requests.auth)\n            self.auth_header = _construct_basic_auth_str(parsed_url.username, parsed_url.password)\n\n    def _build_url(self, path):\n        \"\"\"prepend url with hostname unless it's already an absolute URL\"\"\"\n        if absolute_http_url_regexp.match(path):\n            return path\n        else:\n            return f\"{self.base_url}{path}\"\n\n    def _send_request_safe_mode(self, method, url, **kwargs):\n        \"\"\"\n        Send an HTTP request, and catch any exception that might occur due to either\n        connection problems, or invalid HTTP status codes\n        \"\"\"\n        try:\n            return self.client.urlopen(url, method=method, **kwargs)\n        except FAILURE_EXCEPTIONS as e:\n            if hasattr(e, \"response\"):\n                r = e.response\n            else:\n                req = self.client._make_request(\n                    url,\n                    method=method,\n                    headers=kwargs.get(\"headers\"),\n                    payload=kwargs.get(\"payload\"),\n                    params=kwargs.get(\"params\"),\n                )\n                r = ErrorResponse(url=url, request=req)\n            r.error = e\n            return r\n\n    def request(\n        self,\n        method: str,\n        url: str,\n        name: str | None = None,\n        data: str | dict | None = None,\n        catch_response: bool = False,\n        stream: bool = False,\n        headers: dict | None = None,\n        auth=None,\n        json: dict | None = None,\n        allow_redirects=True,\n        context: dict = {},\n        **kwargs,\n    ) -> ResponseContextManager | FastResponse:\n        \"\"\"\n        Send and HTTP request\n        Returns :py:class:`locust.contrib.fasthttp.FastResponse` object.\n\n        :param method: method for the new :class:`Request` object.\n        :param url: path that will be concatenated with the base host URL that has been specified.\n            Can also be a full URL, in which case the full URL will be requested, and the base host\n            is ignored.\n        :param name: (optional) An argument that can be specified to use as label in Locust's\n            statistics instead of the URL path. This can be used to group different URL's\n            that are requested into a single entry in Locust's statistics.\n        :param catch_response: (optional) Boolean argument that, if set, can be used to make a request\n            return a context manager to work as argument to a with statement. This will allow the\n            request to be marked as a fail based on the content of the response, even if the response\n            code is ok (2xx). The opposite also works, one can use catch_response to catch a request\n            and then mark it as successful even if the response code was not (i.e 500 or 404).\n        :param data: (optional) String/bytes to send in the body of the request.\n        :param json: (optional) Dictionary to send in the body of the request.\n            Automatically sets Content-Type and Accept headers to \"application/json\".\n            Only used if data is not set.\n        :param headers: (optional) Dictionary of HTTP Headers to send with the request.\n        :param auth: (optional) Auth (username, password) tuple to enable Basic HTTP Auth.\n        :param stream: (optional) If set to true the response body will not be consumed immediately\n            and can instead be consumed by accessing the stream attribute on the Response object.\n            Another side effect of setting stream to True is that the time for downloading the response\n            content will not be accounted for in the request time that is reported by Locust.\n        \"\"\"\n        # prepend url with hostname unless it's already an absolute URL\n        built_url = self._build_url(url)\n\n        start_time = time.time()  # seconds since epoch\n\n        if self.user:\n            context = {**self.user.context(), **context}\n\n        headers = headers or {}\n        if auth:\n            headers[\"Authorization\"] = _construct_basic_auth_str(auth[0], auth[1])\n        elif self.auth_header:\n            headers[\"Authorization\"] = self.auth_header\n        if \"Accept-Encoding\" not in headers and \"accept-encoding\" not in headers:\n            headers[\"Accept-Encoding\"] = \"gzip, deflate\"\n\n        if not data and json is not None:\n            data = unshadowed_json.dumps(json)\n            if \"Content-Type\" not in headers and \"content-type\" not in headers:\n                headers[\"Content-Type\"] = \"application/json\"\n            if \"Accept\" not in headers and \"accept\" not in headers:\n                headers[\"Accept\"] = \"application/json\"\n\n        if not allow_redirects:\n            old_redirect_response_codes = self.client.redirect_resonse_codes\n            self.client.redirect_resonse_codes = []\n\n        start_perf_counter = time.perf_counter()\n        # send request, and catch any exceptions\n        response = self._send_request_safe_mode(method, built_url, payload=data, headers=headers, **kwargs)\n        request_meta = {\n            \"request_type\": method,\n            \"name\": name or url,\n            \"context\": context,\n            \"response\": response,\n            \"exception\": None,\n            \"start_time\": start_time,\n            \"url\": built_url,  # this is a small deviation from HttpSession, which gets the final (possibly redirected) URL\n        }\n\n        if not allow_redirects:\n            self.client.redirect_resonse_codes = old_redirect_response_codes\n\n        # get the length of the content, but if the argument stream is set to True, we take\n        # the size from the content-length header, in order to not trigger fetching of the body\n        if stream:\n            request_meta[\"response_length\"] = int(response.headers.get(\"response_length\") or 0)\n        else:\n            try:\n                request_meta[\"response_length\"] = len(response.content or \"\")\n            except HTTPParseError as e:\n                request_meta[\"response_time\"] = (time.perf_counter() - start_perf_counter) * 1000\n                request_meta[\"response_length\"] = 0\n                request_meta[\"exception\"] = e\n                self.environment.events.request.fire(**request_meta)\n                return response\n\n        # Record the consumed time\n        # Note: This is intentionally placed after we record the content_size above, since\n        # we'll then trigger fetching of the body (unless stream=True)\n        request_meta[\"response_time\"] = int((time.perf_counter() - start_perf_counter) * 1000)\n\n        if catch_response:\n            return ResponseContextManager(response, environment=self.environment, request_meta=request_meta)\n        else:\n            try:\n                response.raise_for_status()\n            except FAILURE_EXCEPTIONS as e:\n                request_meta[\"exception\"] = e\n\n            self.environment.events.request.fire(**request_meta)\n            return response\n\n    def delete(self, url, **kwargs):\n        return self.request(\"DELETE\", url, **kwargs)\n\n    def get(self, url, **kwargs):\n        \"\"\"Sends a GET request\"\"\"\n        return self.request(\"GET\", url, **kwargs)\n\n    def head(self, url, **kwargs):\n        \"\"\"Sends a HEAD request\"\"\"\n        return self.request(\"HEAD\", url, **kwargs)\n\n    def options(self, url, **kwargs):\n        \"\"\"Sends a OPTIONS request\"\"\"\n        return self.request(\"OPTIONS\", url, **kwargs)\n\n    def patch(self, url, data=None, **kwargs):\n        \"\"\"Sends a POST request\"\"\"\n...\n", "needle_token_start": 13678, "needle_token_end": 13783, "code_context_ntokens": 16164, "output": ["```python\ndef _construct_basic_auth_str(username, password):\n    \"\"\"Construct Authorization header value to be used in HTTP Basic Auth\"\"\"\n    if isinstance(username, str):\n        username = username.encode(\"latin1\")\n    if isinstance(password, str):\n        password = password.encode(\"latin1\")\n    return \"Basic \" + b64encode(b\":\".join((username, password))).strip().decode(\"ascii\")\n```"]}
{"repo": "locustio/locust", "name": "test_distribute_users", "language": "python", "path": "locust/test/test_dispatch.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate the distribution mechanism of users across multiple worker nodes, ensuring that the total number of users distributed matches a predefined target while measuring the efficiency and performance of the distribution process.\n2. **Input**: The function takes different sets of user classes (with varying distribution weights and fixed counts) and a large number of worker nodes as input parameters.\n3. **Output**: The primary output is a validation of whether the total number of users distributed across all worker nodes matches the expected target. Additionally, it outputs performance metrics indicating the time taken for the distribution process.\n4. **Procedure**: The function initiates by creating a list of worker nodes. It then iterates over different configurations of user classes, setting up a user distribution system for each configuration. For each setup, it measures the time taken to distribute a target number of users across the worker nodes, checks the total count of distributed users, and asserts that the distribution time is within acceptable limits.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=spawn_rate)\n            self.assertEqual(users_dispatcher._wait_between_dispatch, expected_wait_between_dispatch)\n\n\nclass TestRampDownUsersToZero(unittest.TestCase):\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_0_5(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=0.5)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 3, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    # def test_ramp_down_users_on_workers_respecting_weight(self):\n    #     class User1(User):\n    #         weight = 1\n    #\n    #     class User2(User):\n    #         weight = 1\n    #\n    #     class User3(User):\n    #         weight = 1\n    #\n    #     user_classes = [User1, User2, User3]\n    #     workers = [WorkerNode(str(i + 1)) for i in range(3)]\n    #\n    #     user_dispatcher = UsersDispatcher(worker_nodes= workers, user_classes = user_classes)\n    #     user_dispatcher.new_dispatch(target_user_count=7, spawn_rate=7)\n    #\n    #     dispatched_users = next(user_dispatcher)\n    #     self.assertDictEqual(dispatched_users,\n    #                          {\n    #                              \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n    #                              \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n    #                              \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2}\n    #                          })\n    #\n    #     user_dispatcher.new_dispatch(target_user_count=16, spawn_rate=9)\n    #     dispatched_users = next(user_dispatcher)\n    #     self.assertDictEqual(dispatched_users,\n    #                          {\n    #                              \"1\": {\"User1\": 6, \"User2\": 0, \"User3\": 0},\n    #                              \"2\": {\"User1\": 0, \"User2\": 5, \"User3\": 0},\n    #                              \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 5}\n    #                          })\n    #\n    #     user_dispatcher.new_dispatch(target_user_count=3, spawn_rate=15)\n    #     dispatched_users = next(user_dispatcher)\n    #     self.assertDictEqual(dispatched_users,\n    #                          {\n    #                              \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n    #                              \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n    #                              \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1}\n    #                          })\n    #\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_1(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 3, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_4_workers_with_spawn_rate_of_1(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(4)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=1)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 1},\n                \"3\": {\"User1\": 1, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 1},\n                \"3\": {\"User1\": 1, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 1},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"4\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_2(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=2)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_2_4(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=2.4)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_3(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=3)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 2},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 1, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_4(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=4)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 2, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 2, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 1},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 1, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(sleep_time - _TOLERANCE <= delta <= sleep_time + _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n    def test_ramp_down_users_to_3_workers_with_spawn_rate_of_9(self):\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n        initial_user_count = 9\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n        users_dispatcher.new_dispatch(target_user_count=initial_user_count, spawn_rate=initial_user_count)\n        users_dispatcher._wait_between_dispatch = 0\n        list(users_dispatcher)\n\n        sleep_time = 0.2  # Speed-up test\n\n        users_dispatcher.new_dispatch(target_user_count=0, spawn_rate=9)\n        users_dispatcher._wait_between_dispatch = sleep_time\n\n        ts = time.perf_counter()\n        self.assertDictEqual(\n            next(users_dispatcher),\n            {\n                \"1\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"2\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 0},\n            },\n        )\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n        ts = time.perf_counter()\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n        delta = time.perf_counter() - ts\n        self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n\n@unittest.skip(reason=\"takes too long. run this manually if you change dispatch logic.\")\nclass TestRampUpThenDownThenUp(unittest.TestCase):\n    def test_ramp_up_then_down_then_up(self):\n        for user1_weight, user2_weight, user3_weight, user4_weight, user5_weight in [\n            (1, 1, 1, 1, 1),\n            (1, 2, 3, 4, 5),\n            (1, 3, 5, 7, 9),\n        ]:\n\n            class User1(User):\n                weight = user1_weight\n\n            class User2(User):\n                weight = user2_weight\n\n            class User3(User):\n                weight = user3_weight\n\n            class User4(User):\n                weight = user4_weight\n\n            class User5(User):\n                weight = user5_weight\n\n            all_user_classes = [User1, User2, User3, User4, User5]\n\n            for number_of_user_classes in range(1, len(all_user_classes) + 1):\n                user_classes = all_user_classes[:number_of_user_classes]\n\n                for max_user_count, min_user_count in [(30, 15), (54, 21), (14165, 1476)]:\n                    for worker_count in [1, 3, 5, 9]:\n                        workers = [WorkerNode(str(i + 1)) for i in range(worker_count)]\n\n                        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n\n                        # Ramp-up to go to `min_user_count` #########\n\n                        users_dispatcher.new_dispatch(target_user_count=min_user_count, spawn_rate=1)\n                        users_dispatcher._wait_between_dispatch = 0\n\n                        all_dispatched_users_ramp_up_to_min_user_count = list(users_dispatcher)\n\n                        # Ramp-up to go to `max_user_count` #########\n\n                        users_dispatcher.new_dispatch(target_user_count=max_user_count, spawn_rate=1)\n                        users_dispatcher._wait_between_dispatch = 0\n\n                        list(users_dispatcher)\n\n                        # Ramp-down go back to `min_user_count` #########\n\n                        users_dispatcher.new_dispatch(target_user_count=min_user_count, spawn_rate=1)\n                        users_dispatcher._wait_between_dispatch = 0\n\n                        all_dispatched_users_ramp_down_to_min_user_count = list(users_dispatcher)\n\n                        # Assertions #########\n\n                        self.assertDictEqual(\n                            all_dispatched_users_ramp_up_to_min_user_count[-1],\n                            all_dispatched_users_ramp_down_to_min_user_count[-1],\n                        )\n\n\nclass TestDispatchUsersToWorkersHavingTheSameUsersAsTheTarget(unittest.TestCase):\n    def test_dispatch_users_to_3_workers(self):\n        \"\"\"Final distribution should be {\"User1\": 3, \"User2\": 3, \"User3\": 3}\"\"\"\n\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 1\n\n        class User3(User):\n            weight = 1\n\n        user_classes = [User1, User2, User3]\n\n        user_count = 9\n\n        for spawn_rate in [0.15, 0.5, 1, 2, 2.4, 3, 4, 9]:\n            workers = [WorkerNode(str(i + 1)) for i in range(3)]\n\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n            users_dispatcher.new_dispatch(target_user_count=user_count, spawn_rate=user_count)\n            users_dispatcher._wait_between_dispatch = 0\n            list(users_dispatcher)\n\n            sleep_time = 0.2  # Speed-up test\n\n            users_dispatcher.new_dispatch(target_user_count=user_count, spawn_rate=spawn_rate)\n            users_dispatcher._wait_between_dispatch = sleep_time\n\n            ts = time.perf_counter()\n            self.assertDictEqual(\n                next(users_dispatcher),\n                {\n                    \"1\": {\"User1\": 3, \"User2\": 0, \"User3\": 0},\n                    \"2\": {\"User1\": 0, \"User2\": 3, \"User3\": 0},\n                    \"3\": {\"User1\": 0, \"User2\": 0, \"User3\": 3},\n                },\n            )\n            delta = time.perf_counter() - ts\n            self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n            ts = time.perf_counter()\n            self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n            delta = time.perf_counter() - ts\n            self.assertTrue(0 <= delta <= _TOLERANCE, delta)\n\n            clear_all_functools_lru_cache()\n\n\nclass TestDistributionIsRespectedDuringDispatch(unittest.TestCase):\n    def test_dispatch_75_users_to_4_workers_with_spawn_rate_of_5(self):\n        \"\"\"\n        Test case covering reported issue in https://github.com/locustio/locust/pull/1621#issuecomment-853624275.\n\n        The case is to ramp-up from 0 to 75 users with two user classes. `User1` has a weight of 1 and `User2`\n        has a weight of 2. The original issue was with 500 users, but to keep the test shorter, we use 75 users.\n\n        Final distribution should be {\"User1\": 25, \"User2\": 50}\n        \"\"\"\n\n        class User1(User):\n            weight = 1\n\n        class User2(User):\n            weight = 2\n\n        worker_node1 = WorkerNode(\"1\")\n        worker_node2 = WorkerNode(\"2\")\n        worker_node3 = WorkerNode(\"3\")\n        worker_node4 = WorkerNode(\"4\")\n\n        users_dispatcher = UsersDispatcher(\n            worker_nodes=[worker_node1, worker_node2, worker_node3, worker_node4], user_classes=[User1, User2]\n        )\n        users_dispatcher.new_dispatch(target_user_count=75, spawn_rate=5)\n        users_dispatcher._wait_between_dispatch = 0\n\n        # total user count = 5\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 2, \"User2\": 3})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 1, \"User2\": 1},\n                \"2\": {\"User1\": 1, \"User2\": 0},\n                \"3\": {\"User1\": 0, \"User2\": 1},\n                \"4\": {\"User1\": 0, \"User2\": 1},\n            },\n        )\n\n        # total user count = 10\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 3, \"User2\": 7})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 1, \"User2\": 2},\n                \"2\": {\"User1\": 1, \"User2\": 2},\n                \"3\": {\"User1\": 0, \"User2\": 2},\n                \"4\": {\"User1\": 1, \"User2\": 1},\n            },\n        )\n\n        # total user count = 15\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 5, \"User2\": 10})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 1, \"User2\": 3},\n                \"2\": {\"User1\": 2, \"User2\": 2},\n                \"3\": {\"User1\": 1, \"User2\": 3},\n                \"4\": {\"User1\": 1, \"User2\": 2},\n            },\n        )\n\n        # total user count = 20\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 7, \"User2\": 13})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 2, \"User2\": 3},\n                \"2\": {\"User1\": 2, \"User2\": 3},\n                \"3\": {\"User1\": 1, \"User2\": 4},\n                \"4\": {\"User1\": 2, \"User2\": 3},\n            },\n        )\n\n        # total user count = 25\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 8, \"User2\": 17})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 2, \"User2\": 5},\n                \"2\": {\"User1\": 2, \"User2\": 4},\n                \"3\": {\"User1\": 2, \"User2\": 4},\n                \"4\": {\"User1\": 2, \"User2\": 4},\n            },\n        )\n\n        # total user count = 30\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 10, \"User2\": 20})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 3, \"User2\": 5},\n                \"2\": {\"User1\": 3, \"User2\": 5},\n                \"3\": {\"User1\": 2, \"User2\": 5},\n                \"4\": {\"User1\": 2, \"User2\": 5},\n            },\n        )\n\n        # total user count = 35\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 12, \"User2\": 23})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 3, \"User2\": 6},\n                \"2\": {\"User1\": 3, \"User2\": 6},\n                \"3\": {\"User1\": 3, \"User2\": 6},\n                \"4\": {\"User1\": 3, \"User2\": 5},\n            },\n        )\n\n        # total user count = 40\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 13, \"User2\": 27})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 3, \"User2\": 7},\n                \"2\": {\"User1\": 4, \"User2\": 6},\n                \"3\": {\"User1\": 3, \"User2\": 7},\n                \"4\": {\"User1\": 3, \"User2\": 7},\n            },\n        )\n\n        # total user count = 45\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 15, \"User2\": 30})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 4, \"User2\": 8},\n                \"2\": {\"User1\": 4, \"User2\": 7},\n                \"3\": {\"User1\": 3, \"User2\": 8},\n                \"4\": {\"User1\": 4, \"User2\": 7},\n            },\n        )\n\n        # total user count = 50\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 17, \"User2\": 33})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 4, \"User2\": 9},\n                \"2\": {\"User1\": 5, \"User2\": 8},\n                \"3\": {\"User1\": 4, \"User2\": 8},\n                \"4\": {\"User1\": 4, \"User2\": 8},\n            },\n        )\n\n        # total user count = 55\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 18, \"User2\": 37})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 5, \"User2\": 9},\n                \"2\": {\"User1\": 5, \"User2\": 9},\n                \"3\": {\"User1\": 4, \"User2\": 10},\n                \"4\": {\"User1\": 4, \"User2\": 9},\n            },\n        )\n\n        # total user count = 60\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 20, \"User2\": 40})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 5, \"User2\": 10},\n                \"2\": {\"User1\": 5, \"User2\": 10},\n                \"3\": {\"User1\": 5, \"User2\": 10},\n                \"4\": {\"User1\": 5, \"User2\": 10},\n            },\n        )\n\n        # total user count = 65\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 22, \"User2\": 43})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 6, \"User2\": 11},\n                \"2\": {\"User1\": 6, \"User2\": 10},\n                \"3\": {\"User1\": 5, \"User2\": 11},\n                \"4\": {\"User1\": 5, \"User2\": 11},\n            },\n        )\n\n        # total user count = 70\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 23, \"User2\": 47})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 6, \"User2\": 12},\n                \"2\": {\"User1\": 6, \"User2\": 12},\n                \"3\": {\"User1\": 5, \"User2\": 12},\n                \"4\": {\"User1\": 6, \"User2\": 11},\n            },\n        )\n\n        # total user count = 75, User1 = 25, User2 = 50\n        dispatched_users = next(users_dispatcher)\n        self.assertDictEqual(_aggregate_dispatched_users(dispatched_users), {\"User1\": 25, \"User2\": 50})\n        self.assertDictEqual(\n            dispatched_users,\n            {\n                \"1\": {\"User1\": 6, \"User2\": 13},\n                \"2\": {\"User1\": 7, \"User2\": 12},\n                \"3\": {\"User1\": 6, \"User2\": 13},\n                \"4\": {\"User1\": 6, \"User2\": 12},\n            },\n        )\n\n        self.assertRaises(StopIteration, lambda: next(users_dispatcher))\n\n\nclass TestLargeScale(unittest.TestCase):\n    # fmt: off\n    weights = [\n        5, 55, 37, 2, 97, 41, 33, 19, 19, 34, 78, 76, 28, 62, 69, 5, 55, 37, 2, 97, 41, 33, 19, 19, 34,\n        78, 76, 28, 62, 69, 41, 33, 19, 19, 34, 78, 76, 28, 62, 69, 41, 33, 19, 19, 34, 78, 76, 28, 62, 69\n    ]\n    # fmt: on\n    numerated_weights = dict(zip(range(len(weights)), weights))\n\n    weighted_user_classes = [type(f\"User{i}\", (User,), {\"weight\": w}) for i, w in numerated_weights.items()]\n    fixed_user_classes_10k = [type(f\"FixedUser10k{i}\", (User,), {\"fixed_count\": 2000}) for i in range(50)]\n    fixed_user_classes_1M = [type(f\"FixedUser1M{i}\", (User,), {\"fixed_count\": 20000}) for i in range(50)]\n    mixed_users = weighted_user_classes[:25] + fixed_user_classes_10k[25:]\n\n    \ndef test_distribute_users(self):\n        for user_classes in [self.weighted_user_classes, self.fixed_user_classes_1M, self.mixed_users]:\n            workers = [WorkerNode(str(i)) for i in range(10_000)]\n\n            target_user_count = 1_000_000\n\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n\n            ts = time.perf_counter()\n            users_on_workers, user_gen, worker_gen, active_users = users_dispatcher._distribute_users(\n                target_user_count=target_user_count\n            )\n            delta = time.perf_counter() - ts\n\n            # Because tests are run with coverage, the code will be slower.\n            # We set the pass criterion to 7000ms, but in real life, the\n            # `_distribute_users` method runs faster than this.\n            self.assertLessEqual(1000 * delta, 7000)\n\n            self.assertEqual(_user_count(users_on_workers), target_user_count)\n\n    def test_ramp_up_from_0_to_100_000_users_with_50_user_classes_and_1000_workers_and_5000_spawn_rate(self):\n        for user_classes in [\n            self.weighted_user_classes,\n            self.fixed_user_classes_1M,\n            self.fixed_user_classes_10k,\n            self.mixed_users,\n        ]:\n            workers = [WorkerNode(str(i)) for i in range(1000)]\n\n            target_user_count = 100_000\n\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n            users_dispatcher.new_dispatch(target_user_count=target_user_count, spawn_rate=5_000)\n            users_dispatcher._wait_between_dispatch = 0\n\n            all_dispatched_users = list(users_dispatcher)\n\n            tol = 0.2\n            self.assertTrue(\n                all(\n                    dispatch_iteration_duration <= tol\n                    for dispatch_iteration_duration in users_dispatcher.dispatch_iteration_durations\n                ),\n                \"One or more dispatch took more than {:.0f}s to compute (max = {}ms)\".format(\n                    tol * 1000, 1000 * max(users_dispatcher.dispatch_iteration_durations)\n                ),\n            )\n\n            self.assertEqual(_user_count(all_dispatched_users[-1]), target_user_count)\n\n            for dispatch_users in all_dispatched_users:\n                user_count_on_workers = [\n                    sum(user_classes_count.values()) for user_classes_count in dispatch_users.values()\n                ]\n                self.assertLessEqual(\n                    max(user_count_on_workers) - min(user_count_on_workers),\n                    1,\n                    \"One or more workers have too much users compared to the other workers when user count is {}\".format(\n                        _user_count(dispatch_users)\n                    ),\n                )\n\n            for i, dispatch_users in enumerate(all_dispatched_users):\n                aggregated_dispatched_users = _aggregate_dispatched_users(dispatch_users)\n                for user_class in [u for u in user_classes if not u.fixed_count]:\n                    target_relative_weight = user_class.weight / sum(\n                        map(attrgetter(\"weight\"), [u for u in user_classes if not u.fixed_count])\n                    )\n...\n", "needle_token_start": 15216, "needle_token_end": 15504, "code_context_ntokens": 16164, "output": ["```python\ndef test_distribute_users(self):\n    for user_classes in [self.weighted_user_classes, self.fixed_user_classes_1M, self.mixed_users]:\n        workers = [WorkerNode(str(i)) for i in range(10_000)]\n\n        target_user_count = 1_000_000\n\n        users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n\n        ts = time.perf_counter()\n        users_on_workers, user_gen, worker_gen, active_users = users_dispatcher._distribute_users(\n            target_user_count=target_user_count\n        )\n        delta = time.perf_counter() - ts\n\n        # Because tests are run with coverage, the code will be slower.\n        # We set the pass criterion to 7000ms, but in real life, the\n        # `_distribute_users` method runs faster than this.\n        self.assertLessEqual(1000 * delta, 7000)\n\n        self.assertEqual(_user_count(users_on_workers), target_user_count)\n\n    def test_ramp_up_from_0_to_100_000_users_with_50_user_classes_and_1000_workers_and_5000_spawn_rate(self):\n        for user_classes in [\n            self.weighted_user_classes,\n            self.fixed_user_classes_1M,\n            self.fixed_user_classes_10k,\n            self.mixed_users,\n        ]:\n            workers = [WorkerNode(str(i)) for i in range(1000)]\n\n            target_user_count = 100_000\n\n            users_dispatcher = UsersDispatcher(worker_nodes=workers, user_classes=user_classes)\n            users_dispatcher.new_dispatch(target_user_count=target_user_count, spawn_rate=5_000)\n            users_dispatcher._wait_between_dispatch = 0\n\n            all_dispatched_users = list(users_dispatcher)\n\n            tol = 0.2\n            self.assertTrue(\n                all(\n                    dispatch_iteration_duration <= tol\n                    for dispatch_iteration_duration in users_dispatcher.dispatch_iteration_durations\n                ),\n                \"One or more dispatch took more than {:.0f}s to compute (max = {}ms)\".format(\n                    tol * 1000, 1000 * max(users_dispatcher.dispatch_iteration_durations)\n                ),\n            )\n\n            self.assertEqual(_user_count(all_dispatched_users[-1]), target_user_count)\n\n            for dispatch_users in all_dispatched_users:\n                user_count_on_workers = [\n                    sum(user_classes_count.values()) for user_classes_count in dispatch_users.values()\n                ]\n                self.assertLessEqual(\n                    max(user_count_on_workers) - min(user_count_on_workers),\n                    1,\n                    \"One or more workers have too much users compared to the other workers when user count is {}\".format(\n                        _user_count(dispatch_users)\n                    ),\n                )\n\n            for i, dispatch_users in enumerate(all_dispatched_users):\n                aggregated_dispatched_users = _aggregate_dispatched_users(dispatch_users)\n                for user_class in [u for u in user_classes if not u.fixed_count]:\n                    target_relative_weight = user_class.weight / sum(\n                        map(attrgetter(\"weight\"), [u for u in user_classes if not u.fixed_count])\n                    )\n...\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "maybe_init_stream", "language": "python", "path": "torch_geometric/loader/prefetch.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function initializes a GPU stream if the device being used supports GPU operations. This allows for asynchronous operations on the GPU, which can improve performance by overlapping computation and data transfer.\n2. **Input**: There are no direct inputs as the function uses the class's internal state to determine if a stream should be initialized.\n3. **Output**: There is no direct output returned by the function; however, it modifies the internal state of the class by potentially creating a new GPU stream and setting up a context manager for that stream.\n4. **Procedure**: The function first checks if the current device is a GPU. If it is, the function creates a new GPU stream and sets up a context manager that will allow other methods to execute operations within this stream, thereby enabling asynchronous GPU operations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/loader/neighbor_sampler.py\nfrom typing import Callable, List, NamedTuple, Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import SparseTensor\n\n\nclass EdgeIndex(NamedTuple):\n    edge_index: Tensor\n    e_id: Optional[Tensor]\n    size: Tuple[int, int]\n\n    def to(self, *args, **kwargs):\n        edge_index = self.edge_index.to(*args, **kwargs)\n        e_id = self.e_id.to(*args, **kwargs) if self.e_id is not None else None\n        return EdgeIndex(edge_index, e_id, self.size)\n\n\nclass Adj(NamedTuple):\n    adj_t: SparseTensor\n    e_id: Optional[Tensor]\n    size: Tuple[int, int]\n\n    def to(self, *args, **kwargs):\n        adj_t = self.adj_t.to(*args, **kwargs)\n        e_id = self.e_id.to(*args, **kwargs) if self.e_id is not None else None\n        return Adj(adj_t, e_id, self.size)\n\n\nclass NeighborSampler(torch.utils.data.DataLoader):\n    r\"\"\"The neighbor sampler from the `\"Inductive Representation Learning on\n    Large Graphs\" <https://arxiv.org/abs/1706.02216>`_ paper, which allows\n...\n# Path: torch_geometric/loader/prefetch.py\nimport warnings\nfrom contextlib import nullcontext\nfrom functools import partial\nfrom typing import Any, Optional\n\nimport torch\nfrom torch.utils.data import DataLoader\n\nfrom torch_geometric.typing import WITH_IPEX\n\n\nclass DeviceHelper:\n    def __init__(self, device: Optional[torch.device] = None):\n        with_cuda = torch.cuda.is_available()\n        with_xpu = torch.xpu.is_available() if WITH_IPEX else False\n\n        if device is None:\n            if with_cuda:\n                device = 'cuda'\n            elif with_xpu:\n                device = 'xpu'\n            else:\n                device = 'cpu'\n\n        self.device = torch.device(device)\n        self.is_gpu = self.device.type in ['cuda', 'xpu']\n\n        if ((self.device.type == 'cuda' and not with_cuda)\n                or (self.device.type == 'xpu' and not with_xpu)):\n            warnings.warn(f\"Requested device '{self.device.type}' is not \"\n                          f\"available, falling back to CPU\")\n            self.device = torch.device('cpu')\n\n        self.stream = None\n        self.stream_context = nullcontext\n        self.module = getattr(torch, self.device.type) if self.is_gpu else None\n\n    \ndef maybe_init_stream(self) -> None:\n        if self.is_gpu:\n            self.stream = self.module.Stream()\n            self.stream_context = partial(\n                self.module.stream,\n                stream=self.stream,\n            )\n\n    def maybe_wait_stream(self) -> None:\n        if self.stream is not None:\n            self.module.current_stream().wait_stream(self.stream)\n\n\nclass PrefetchLoader:\n    r\"\"\"A GPU prefetcher class for asynchronously transferring data of a\n    :class:`torch.utils.data.DataLoader` from host memory to device memory.\n\n    Args:\n        loader (torch.utils.data.DataLoader): The data loader.\n        device (torch.device, optional): The device to load the data to.\n            (default: :obj:`None`)\n    \"\"\"\n    def __init__(\n        self,\n        loader: DataLoader,\n        device: Optional[torch.device] = None,\n    ):\n        self.loader = loader\n        self.device_helper = DeviceHelper(device)\n\n    def non_blocking_transfer(self, batch: Any) -> Any:\n        if not self.device_helper.is_gpu:\n            return batch\n        if isinstance(batch, (list, tuple)):\n            return [self.non_blocking_transfer(v) for v in batch]\n        if isinstance(batch, dict):\n            return {k: self.non_blocking_transfer(v) for k, v in batch.items()}\n\n        batch = batch.pin_memory(self.device_helper.device)\n        return batch.to(self.device_helper.device, non_blocking=True)\n\n    def __iter__(self) -> Any:\n        first = True\n        self.device_helper.maybe_init_stream()\n\n        batch = None\n        for next_batch in self.loader:\n\n            with self.device_helper.stream_context():\n                next_batch = self.non_blocking_transfer(next_batch)\n\n            if not first:\n                yield batch\n            else:\n                first = False\n\n            self.device_helper.maybe_wait_stream()\n\n            batch = next_batch\n\n        yield batch\n\n    def __len__(self) -> int:\n        return len(self.loader)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}({self.loader})'\n\n# Path: torch_geometric/loader/random_node_loader.py\nimport math\nfrom typing import Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.data.hetero_data import to_homogeneous_edge_index\n\n\nclass RandomNodeLoader(torch.utils.data.DataLoader):\n    r\"\"\"A data loader that randomly samples nodes within a graph and returns\n    their induced subgraph.\n\n    .. note::\n\n        For an example of using\n        :class:`~torch_geometric.loader.RandomNodeLoader`, see\n        `examples/ogbn_proteins_deepgcn.py\n        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/\n        ogbn_proteins_deepgcn.py>`_.\n\n    Args:\n        data (torch_geometric.data.Data or torch_geometric.data.HeteroData):\n            The :class:`~torch_geometric.data.Data` or\n            :class:`~torch_geometric.data.HeteroData` graph object.\n        num_parts (int): The number of partitions.\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`num_workers`.\n    \"\"\"\n    def __init__(\n        self,\n        data: Union[Data, HeteroData],\n        num_parts: int,\n        **kwargs,\n    ):\n        self.data = data\n        self.num_parts = num_parts\n\n        if isinstance(data, HeteroData):\n            edge_index, node_dict, edge_dict = to_homogeneous_edge_index(data)\n            self.node_dict, self.edge_dict = node_dict, edge_dict\n        else:\n            edge_index = data.edge_index\n\n        self.edge_index = edge_index\n        self.num_nodes = data.num_nodes\n\n        super().__init__(\n            range(self.num_nodes),\n            batch_size=math.ceil(self.num_nodes / num_parts),\n            collate_fn=self.collate_fn,\n            **kwargs,\n        )\n\n    def collate_fn(self, index):\n        if not isinstance(index, Tensor):\n            index = torch.tensor(index)\n\n        if isinstance(self.data, Data):\n            return self.data.subgraph(index)\n\n        elif isinstance(self.data, HeteroData):\n            node_dict = {\n                key: index[(index >= start) & (index < end)] - start\n                for key, (start, end) in self.node_dict.items()\n            }\n            return self.data.subgraph(node_dict)\n\n# Path: torch_geometric/loader/shadow.py\nimport copy\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Batch, Data\nfrom torch_geometric.typing import WITH_TORCH_SPARSE, SparseTensor\n\n\nclass ShaDowKHopSampler(torch.utils.data.DataLoader):\n    r\"\"\"The ShaDow :math:`k`-hop sampler from the `\"Decoupling the Depth and\n    Scope of Graph Neural Networks\" <https://arxiv.org/abs/2201.07858>`_ paper.\n    Given a graph in a :obj:`data` object, the sampler will create shallow,\n    localized subgraphs.\n    A deep GNN on this local graph then smooths the informative local signals.\n\n    .. note::\n\n        For an example of using :class:`ShaDowKHopSampler`, see\n        `examples/shadow.py <https://github.com/pyg-team/\n        pytorch_geometric/blob/master/examples/shadow.py>`_.\n\n    Args:\n        data (torch_geometric.data.Data): The graph data object.\n        depth (int): The depth/number of hops of the localized subgraph.\n        num_neighbors (int): The number of neighbors to sample for each node in\n            each hop.\n        node_idx (LongTensor or BoolTensor, optional): The nodes that should be\n            considered for creating mini-batches.\n            If set to :obj:`None`, all nodes will be\n            considered.\n        replace (bool, optional): If set to :obj:`True`, will sample neighbors\n            with replacement. (default: :obj:`False`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`batch_size` or\n            :obj:`num_workers`.\n    \"\"\"\n    def __init__(self, data: Data, depth: int, num_neighbors: int,\n                 node_idx: Optional[Tensor] = None, replace: bool = False,\n                 **kwargs):\n\n        if not WITH_TORCH_SPARSE:\n            raise ImportError(\n                f\"'{self.__class__.__name__}' requires 'torch-sparse'\")\n\n        self.data = copy.copy(data)\n        self.depth = depth\n        self.num_neighbors = num_neighbors\n        self.replace = replace\n\n        if data.edge_index is not None:\n            self.is_sparse_tensor = False\n            row, col = data.edge_index.cpu()\n            self.adj_t = SparseTensor(\n                row=row, col=col, value=torch.arange(col.size(0)),\n                sparse_sizes=(data.num_nodes, data.num_nodes)).t()\n        else:\n            self.is_sparse_tensor = True\n            self.adj_t = data.adj_t.cpu()\n\n        if node_idx is None:\n            node_idx = torch.arange(self.adj_t.sparse_size(0))\n        elif node_idx.dtype == torch.bool:\n            node_idx = node_idx.nonzero(as_tuple=False).view(-1)\n        self.node_idx = node_idx\n\n        super().__init__(node_idx.tolist(), collate_fn=self.__collate__,\n                         **kwargs)\n\n    def __collate__(self, n_id):\n        n_id = torch.tensor(n_id)\n\n        rowptr, col, value = self.adj_t.csr()\n        out = torch.ops.torch_sparse.ego_k_hop_sample_adj(\n            rowptr, col, n_id, self.depth, self.num_neighbors, self.replace)\n        rowptr, col, n_id, e_id, ptr, root_n_id = out\n\n        adj_t = SparseTensor(rowptr=rowptr, col=col,\n                             value=value[e_id] if value is not None else None,\n                             sparse_sizes=(n_id.numel(), n_id.numel()),\n                             is_sorted=True, trust_data=True)\n\n        batch = Batch(batch=torch.ops.torch_sparse.ptr2ind(ptr, n_id.numel()),\n                      ptr=ptr)\n        batch.root_n_id = root_n_id\n\n        if self.is_sparse_tensor:\n            batch.adj_t = adj_t\n        else:\n            row, col, e_id = adj_t.t().coo()\n            batch.edge_index = torch.stack([row, col], dim=0)\n\n        for k, v in self.data:\n            if k in ['edge_index', 'adj_t', 'num_nodes', 'batch', 'ptr']:\n                continue\n            if k == 'y' and v.size(0) == self.data.num_nodes:\n                batch[k] = v[n_id][root_n_id]\n            elif isinstance(v, Tensor) and v.size(0) == self.data.num_nodes:\n                batch[k] = v[n_id]\n            elif isinstance(v, Tensor) and v.size(0) == self.data.num_edges:\n                batch[k] = v[e_id]\n            else:\n                batch[k] = v\n\n        return batch\n\n# Path: torch_geometric/loader/temporal_dataloader.py\nfrom typing import List\n\nimport torch\n\nfrom torch_geometric.data import TemporalData\n\n\nclass TemporalDataLoader(torch.utils.data.DataLoader):\n    r\"\"\"A data loader which merges succesive events of a\n    :class:`torch_geometric.data.TemporalData` to a mini-batch.\n\n    Args:\n        data (TemporalData): The :obj:`~torch_geometric.data.TemporalData`\n            from which to load the data.\n        batch_size (int, optional): How many samples per batch to load.\n            (default: :obj:`1`)\n        neg_sampling_ratio (float, optional): The ratio of sampled negative\n            destination nodes to the number of postive destination nodes.\n            (default: :obj:`0.0`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`.\n    \"\"\"\n    def __init__(\n        self,\n        data: TemporalData,\n        batch_size: int = 1,\n        neg_sampling_ratio: float = 0.0,\n        **kwargs,\n    ):\n        # Remove for PyTorch Lightning:\n        kwargs.pop('dataset', None)\n        kwargs.pop('collate_fn', None)\n        kwargs.pop('shuffle', None)\n\n        self.data = data\n        self.events_per_batch = batch_size\n        self.neg_sampling_ratio = neg_sampling_ratio\n\n        if neg_sampling_ratio > 0:\n            self.min_dst = int(data.dst.min())\n            self.max_dst = int(data.dst.max())\n\n        if kwargs.get('drop_last', False) and len(data) % batch_size != 0:\n            arange = range(0, len(data) - batch_size, batch_size)\n        else:\n            arange = range(0, len(data), batch_size)\n\n        super().__init__(arange, 1, shuffle=False, collate_fn=self, **kwargs)\n\n    def __call__(self, arange: List[int]) -> TemporalData:\n        batch = self.data[arange[0]:arange[0] + self.events_per_batch]\n\n        n_ids = [batch.src, batch.dst]\n\n        if self.neg_sampling_ratio > 0:\n            batch.neg_dst = torch.randint(\n                low=self.min_dst,\n                high=self.max_dst + 1,\n                size=(round(self.neg_sampling_ratio * batch.dst.size(0)), ),\n                dtype=batch.dst.dtype,\n                device=batch.dst.device,\n            )\n            n_ids += [batch.neg_dst]\n\n        batch.n_id = torch.cat(n_ids, dim=0).unique()\n\n        return batch\n\n# Path: torch_geometric/loader/zip_loader.py\nfrom typing import Any, Iterator, List, Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.loader import LinkLoader, NodeLoader\nfrom torch_geometric.loader.base import DataLoaderIterator\nfrom torch_geometric.loader.utils import infer_filter_per_worker\n\n\nclass ZipLoader(torch.utils.data.DataLoader):\n    r\"\"\"A loader that returns a tuple of data objects by sampling from multiple\n    :class:`NodeLoader` or :class:`LinkLoader` instances.\n\n    Args:\n        loaders (List[NodeLoader] or List[LinkLoader]): The loader instances.\n        filter_per_worker (bool, optional): If set to :obj:`True`, will filter\n            the returned data in each worker's subprocess.\n            If set to :obj:`False`, will filter the returned data in the main\n            process.\n            If set to :obj:`None`, will automatically infer the decision based\n            on whether data partially lives on the GPU\n            (:obj:`filter_per_worker=True`) or entirely on the CPU\n            (:obj:`filter_per_worker=False`).\n            There exists different trade-offs for setting this option.\n            Specifically, setting this option to :obj:`True` for in-memory\n            datasets will move all features to shared memory, which may result\n            in too many open file handles. (default: :obj:`None`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`batch_size`,\n            :obj:`shuffle`, :obj:`drop_last` or :obj:`num_workers`.\n    \"\"\"\n    def __init__(\n        self,\n        loaders: Union[List[NodeLoader], List[LinkLoader]],\n        filter_per_worker: Optional[bool] = None,\n        **kwargs,\n    ):\n        if filter_per_worker is None:\n            filter_per_worker = infer_filter_per_worker(loaders[0].data)\n\n        # Remove for PyTorch Lightning:\n        kwargs.pop('dataset', None)\n        kwargs.pop('collate_fn', None)\n\n        for loader in loaders:\n            if not callable(getattr(loader, 'collate_fn', None)):\n                raise ValueError(\"'{loader.__class__.__name__}' does not have \"\n                                 \"a 'collate_fn' method\")\n            if not callable(getattr(loader, 'filter_fn', None)):\n                raise ValueError(\"'{loader.__class__.__name__}' does not have \"\n                                 \"a 'filter_fn' method\")\n            loader.filter_per_worker = filter_per_worker\n\n        iterator = range(min([len(loader.dataset) for loader in loaders]))\n        super().__init__(iterator, collate_fn=self.collate_fn, **kwargs)\n\n        self.loaders = loaders\n        self.filter_per_worker = filter_per_worker\n\n    def collate_fn(self, index: List[int]) -> Tuple[Any, ...]:\n        if not isinstance(index, Tensor):\n            index = torch.tensor(index, dtype=torch.long)\n\n        return tuple(loader.collate_fn(index) for loader in self.loaders)\n\n    def filter_fn(\n        self,\n        outs: Tuple[Any, ...],\n    ) -> Tuple[Union[Data, HeteroData], ...]:\n        loaders = self.loaders\n        return tuple(loader.filter_fn(v) for loader, v in zip(loaders, outs))\n\n    def _get_iterator(self) -> Iterator:\n        if self.filter_per_worker:\n            return super()._get_iterator()\n\n        # Execute `filter_fn` in the main process:\n        return DataLoaderIterator(super()._get_iterator(), self.filter_fn)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}(loaders={self.loaders})'\n\n# Path: torch_geometric/loader/__init__.py\nfrom torch_geometric.deprecation import deprecated\n\nfrom .dataloader import DataLoader\nfrom .node_loader import NodeLoader\nfrom .link_loader import LinkLoader\nfrom .neighbor_loader import NeighborLoader\nfrom .link_neighbor_loader import LinkNeighborLoader\nfrom .hgt_loader import HGTLoader\nfrom .cluster import ClusterData, ClusterLoader\nfrom .graph_saint import (GraphSAINTSampler, GraphSAINTNodeSampler,\n                          GraphSAINTEdgeSampler, GraphSAINTRandomWalkSampler)\nfrom .shadow import ShaDowKHopSampler\nfrom .random_node_loader import RandomNodeLoader\n# from .ibmb_loader import IBMBBatchLoader, IBMBNodeLoader\nfrom .zip_loader import ZipLoader\nfrom .data_list_loader import DataListLoader\nfrom .dense_data_loader import DenseDataLoader\nfrom .temporal_dataloader import TemporalDataLoader\nfrom .neighbor_sampler import NeighborSampler\nfrom .imbalanced_sampler import ImbalancedSampler\nfrom .dynamic_batch_sampler import DynamicBatchSampler\nfrom .prefetch import PrefetchLoader\nfrom .cache import CachedLoader\nfrom .mixin import AffinityMixin\n\n__all__ = classes = [\n    'DataLoader',\n    'NodeLoader',\n    'LinkLoader',\n    'NeighborLoader',\n    'LinkNeighborLoader',\n    'HGTLoader',\n    'ClusterData',\n    'ClusterLoader',\n    'GraphSAINTSampler',\n    'GraphSAINTNodeSampler',\n    'GraphSAINTEdgeSampler',\n    'GraphSAINTRandomWalkSampler',\n    'ShaDowKHopSampler',\n    'RandomNodeLoader',\n    # 'IBMBBatchLoader',\n    # 'IBMBNodeLoader',\n    'ZipLoader',\n    'DataListLoader',\n    'DenseDataLoader',\n    'TemporalDataLoader',\n    'NeighborSampler',\n    'ImbalancedSampler',\n    'DynamicBatchSampler',\n    'PrefetchLoader',\n    'CachedLoader',\n    'AffinityMixin',\n]\n\nRandomNodeSampler = deprecated(\n    details=\"use 'loader.RandomNodeLoader' instead\",\n    func_name='loader.RandomNodeSampler',\n)(RandomNodeLoader)\n\n# Path: torch_geometric/data/__init__.py\n# flake8: noqa\n\nfrom .feature_store import FeatureStore, TensorAttr\nfrom .graph_store import GraphStore, EdgeAttr\nfrom .data import Data\nfrom .hetero_data import HeteroData\nfrom .batch import Batch\nfrom .temporal import TemporalData\nfrom .database import Database, SQLiteDatabase, RocksDatabase\nfrom .dataset import Dataset\nfrom .in_memory_dataset import InMemoryDataset\nfrom .on_disk_dataset import OnDiskDataset\nfrom .makedirs import makedirs\nfrom .download import download_url, download_google_url\nfrom .extract import extract_tar, extract_zip, extract_bz2, extract_gz\n\nfrom torch_geometric.lazy_loader import LazyLoader\n\ndata_classes = [\n    'Data',\n    'HeteroData',\n    'Batch',\n    'TemporalData',\n    'Dataset',\n    'InMemoryDataset',\n    'OnDiskDataset',\n]\n\nremote_backend_classes = [\n    'FeatureStore',\n    'GraphStore',\n    'TensorAttr',\n    'EdgeAttr',\n]\n\ndatabase_classes = [\n    'Database',\n    'SQLiteDatabase',\n    'RocksDatabase',\n]\n\nhelper_functions = [\n    'makedirs',\n    'download_url',\n    'download_google_url',\n    'extract_tar',\n    'extract_zip',\n    'extract_bz2',\n    'extract_gz',\n]\n\n__all__ = data_classes + remote_backend_classes + helper_functions\n\nlightning = LazyLoader('lightning', globals(),\n                       'torch_geometric.data.lightning')\n\nfrom torch_geometric.deprecation import deprecated\nfrom torch_geometric.loader import NeighborSampler\nfrom torch_geometric.loader import ClusterData\nfrom torch_geometric.loader import ClusterLoader\nfrom torch_geometric.loader import GraphSAINTSampler\nfrom torch_geometric.loader import GraphSAINTNodeSampler\nfrom torch_geometric.loader import GraphSAINTEdgeSampler\nfrom torch_geometric.loader import GraphSAINTRandomWalkSampler\nfrom torch_geometric.loader import ShaDowKHopSampler\nfrom torch_geometric.loader import RandomNodeLoader\nfrom torch_geometric.loader import DataLoader\nfrom torch_geometric.loader import DataListLoader\nfrom torch_geometric.loader import DenseDataLoader\n\nNeighborSampler = deprecated(  # type: ignore\n    details=\"use 'loader.NeighborSampler' instead\",\n    func_name='data.NeighborSampler',\n)(NeighborSampler)\nClusterData = deprecated(  # type: ignore\n    details=\"use 'loader.ClusterData' instead\",\n    func_name='data.ClusterData',\n)(ClusterData)\nClusterLoader = deprecated(  # type: ignore\n    details=\"use 'loader.ClusterLoader' instead\",\n    func_name='data.ClusterLoader',\n)(ClusterLoader)\nGraphSAINTSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTSampler' instead\",\n    func_name='data.GraphSAINTSampler',\n)(GraphSAINTSampler)\nGraphSAINTNodeSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTNodeSampler' instead\",\n    func_name='data.GraphSAINTNodeSampler',\n)(GraphSAINTNodeSampler)\nGraphSAINTEdgeSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTEdgeSampler' instead\",\n    func_name='data.GraphSAINTEdgeSampler',\n)(GraphSAINTEdgeSampler)\nGraphSAINTRandomWalkSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTRandomWalkSampler' instead\",\n    func_name='data.GraphSAINTRandomWalkSampler',\n)(GraphSAINTRandomWalkSampler)\nShaDowKHopSampler = deprecated(  # type: ignore\n    details=\"use 'loader.ShaDowKHopSampler' instead\",\n    func_name='data.ShaDowKHopSampler',\n)(ShaDowKHopSampler)\nRandomNodeSampler = deprecated(\n    details=\"use 'loader.RandomNodeLoader' instead\",\n    func_name='data.RandomNodeSampler',\n)(RandomNodeLoader)\nDataLoader = deprecated(  # type: ignore\n    details=\"use 'loader.DataLoader' instead\",\n    func_name='data.DataLoader',\n)(DataLoader)\nDataListLoader = deprecated(  # type: ignore\n    details=\"use 'loader.DataListLoader' instead\",\n    func_name='data.DataListLoader',\n)(DataListLoader)\nDenseDataLoader = deprecated(  # type: ignore\n    details=\"use 'loader.DenseDataLoader' instead\",\n    func_name='data.DenseDataLoader',\n)(DenseDataLoader)\n\n# Path: torch_geometric/visualization/graph.py\nfrom math import sqrt\nfrom typing import Any, List, Optional\n\nimport torch\nfrom torch import Tensor\n\nBACKENDS = {'graphviz', 'networkx'}\n\n\ndef has_graphviz() -> bool:\n    try:\n        import graphviz\n    except ImportError:\n        return False\n\n    try:\n        graphviz.Digraph().pipe()\n    except graphviz.backend.ExecutableNotFound:\n        return False\n\n    return True\n\n\ndef visualize_graph(\n    edge_index: Tensor,\n    edge_weight: Optional[Tensor] = None,\n    path: Optional[str] = None,\n    backend: Optional[str] = None,\n    node_labels: Optional[List[str]] = None,\n) -> Any:\n    r\"\"\"Visualizes the graph given via :obj:`edge_index` and (optional)\n    :obj:`edge_weight`.\n\n    Args:\n        edge_index (torch.Tensor): The edge indices.\n        edge_weight (torch.Tensor, optional): The edge weights.\n        path (str, optional): The path to where the plot is saved.\n            If set to :obj:`None`, will visualize the plot on-the-fly.\n            (default: :obj:`None`)\n        backend (str, optional): The graph drawing backend to use for\n            visualization (:obj:`\"graphviz\"`, :obj:`\"networkx\"`).\n            If set to :obj:`None`, will use the most appropriate\n            visualization backend based on available system packages.\n            (default: :obj:`None`)\n        node_labels (List[str], optional): The labels/IDs of nodes.\n            (default: :obj:`None`)\n    \"\"\"\n    if edge_weight is not None:  # Normalize edge weights.\n        edge_weight = edge_weight - edge_weight.min()\n        edge_weight = edge_weight / edge_weight.max()\n\n    if edge_weight is not None:  # Discard any edges with zero edge weight:\n        mask = edge_weight > 1e-7\n        edge_index = edge_index[:, mask]\n        edge_weight = edge_weight[mask]\n\n    if edge_weight is None:\n        edge_weight = torch.ones(edge_index.size(1))\n\n    if backend is None:\n        backend = 'graphviz' if has_graphviz() else 'networkx'\n\n    if backend.lower() == 'networkx':\n        return _visualize_graph_via_networkx(edge_index, edge_weight, path,\n                                             node_labels)\n    elif backend.lower() == 'graphviz':\n        return _visualize_graph_via_graphviz(edge_index, edge_weight, path,\n                                             node_labels)\n\n    raise ValueError(f\"Expected graph drawing backend to be in \"\n                     f\"{BACKENDS} (got '{backend}')\")\n\n\ndef _visualize_graph_via_graphviz(\n    edge_index: Tensor,\n    edge_weight: Tensor,\n    path: Optional[str] = None,\n    node_labels: Optional[List[str]] = None,\n) -> Any:\n    import graphviz\n\n    suffix = path.split('.')[-1] if path is not None else None\n    g = graphviz.Digraph('graph', format=suffix)\n    g.attr('node', shape='circle', fontsize='11pt')\n\n    for node in edge_index.view(-1).unique().tolist():\n        g.node(str(node) if node_labels is None else node_labels[node])\n\n    for (src, dst), w in zip(edge_index.t().tolist(), edge_weight.tolist()):\n        hex_color = hex(255 - round(255 * w))[2:]\n        hex_color = f'{hex_color}0' if len(hex_color) == 1 else hex_color\n        if node_labels is not None:\n            src = node_labels[src]\n            dst = node_labels[dst]\n        g.edge(str(src), str(dst), color=f'#{hex_color}{hex_color}{hex_color}')\n\n    if path is not None:\n        path = '.'.join(path.split('.')[:-1])\n        g.render(path, cleanup=True)\n    else:\n        g.view()\n\n    return g\n\n\ndef _visualize_graph_via_networkx(\n    edge_index: Tensor,\n    edge_weight: Tensor,\n    path: Optional[str] = None,\n    node_labels: Optional[List[str]] = None,\n) -> Any:\n    import matplotlib.pyplot as plt\n    import networkx as nx\n\n    g = nx.DiGraph()\n    node_size = 800\n\n    for node in edge_index.view(-1).unique().tolist():\n        g.add_node(node if node_labels is None else node_labels[node])\n\n    for (src, dst), w in zip(edge_index.t().tolist(), edge_weight.tolist()):\n        if node_labels is not None:\n            src = node_labels[src]\n            dst = node_labels[dst]\n        g.add_edge(src, dst, alpha=w)\n\n    ax = plt.gca()\n    pos = nx.spring_layout(g)\n    for src, dst, data in g.edges(data=True):\n        ax.annotate(\n            '',\n            xy=pos[src],\n            xytext=pos[dst],\n            arrowprops=dict(\n                arrowstyle=\"->\",\n                alpha=data['alpha'],\n                shrinkA=sqrt(node_size) / 2.0,\n                shrinkB=sqrt(node_size) / 2.0,\n                connectionstyle=\"arc3,rad=0.1\",\n            ),\n        )\n\n    nodes = nx.draw_networkx_nodes(g, pos, node_size=node_size,\n                                   node_color='white', margins=0.1)\n    nodes.set_edgecolor('black')\n    nx.draw_networkx_labels(g, pos, font_size=10)\n\n    if path is not None:\n        plt.savefig(path)\n    else:\n        plt.show()\n\n    plt.close()\n\n# Path: torch_geometric/visualization/influence.py\nfrom typing import Any\n\nimport torch\nfrom torch import Tensor\nfrom torch.autograd import grad\n\n\ndef influence(model: torch.nn.Module, src: Tensor, *args: Any) -> Tensor:\n    x = src.clone().requires_grad_()\n    out = model(x, *args).sum(dim=-1)\n\n    influences = []\n    for j in range(src.size(0)):\n        influence = grad([out[j]], [x], retain_graph=True)[0].abs().sum(dim=-1)\n        influences.append(influence / influence.sum())\n\n    return torch.stack(influences, dim=0)\n\n# Path: torch_geometric/visualization/__init__.py\nr\"\"\"Visualization package.\"\"\"\n\nfrom .graph import visualize_graph\nfrom .influence import influence\n\n__all__ = [\n    'visualize_graph',\n    'influence',\n]\n\n# Path: torch_geometric/explain/explanation.py\nimport copy\nfrom typing import Dict, List, Optional, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data.data import Data, warn_or_raise\nfrom torch_geometric.data.hetero_data import HeteroData\nfrom torch_geometric.explain.config import ThresholdConfig, ThresholdType\nfrom torch_geometric.typing import EdgeType, NodeType\nfrom torch_geometric.visualization import visualize_graph\n\n\nclass ExplanationMixin:\n    @property\n    def available_explanations(self) -> List[str]:\n        \"\"\"Returns the available explanation masks.\"\"\"\n        return [key for key in self.keys() if key.endswith('_mask')]\n\n    def validate_masks(self, raise_on_error: bool = True) -> bool:\n        r\"\"\"Validates the correctness of the :class:`Explanation` masks.\"\"\"\n        status = True\n\n        for store in self.node_stores:\n            if 'node_mask' not in store:\n                continue\n\n            if store.node_mask.dim() != 2:\n                status = False\n                warn_or_raise(\n                    f\"Expected a 'node_mask' with two dimensions (got \"\n                    f\"{store.node_mask.dim()} dimensions)\", raise_on_error)\n\n            if store.node_mask.size(0) not in {1, store.num_nodes}:\n                status = False\n                warn_or_raise(\n                    f\"Expected a 'node_mask' with {store.num_nodes} nodes \"\n                    f\"(got {store.node_mask.size(0)} nodes)\", raise_on_error)\n\n            if 'x' in store:\n                num_features = store.x.size(-1)\n            else:\n                num_features = store.node_mask.size(-1)\n\n            if store.node_mask.size(1) not in {1, num_features}:\n                status = False\n                warn_or_raise(\n                    f\"Expected a 'node_mask' with {num_features} features (\"\n                    f\"got {store.node_mask.size(1)} features)\", raise_on_error)\n\n        for store in self.edge_stores:\n            if 'edge_mask' not in store:\n                continue\n\n            if store.edge_mask.dim() != 1:\n                status = False\n                warn_or_raise(\n                    f\"Expected an 'edge_mask' with one dimension (got \"\n                    f\"{store.edge_mask.dim()} dimensions)\", raise_on_error)\n\n            if store.edge_mask.size(0) != store.num_edges:\n                status = False\n                warn_or_raise(\n                    f\"Expected an 'edge_mask' with {store.num_edges} edges \"\n                    f\"(got {store.edge_mask.size(0)} edges)\", raise_on_error)\n\n        return status\n\n    def _threshold_mask(\n        self,\n        mask: Optional[Tensor],\n        threshold_config: ThresholdConfig,\n    ) -> Optional[Tensor]:\n\n        if mask is None:\n            return None\n\n        if threshold_config.type == ThresholdType.hard:\n            return (mask > threshold_config.value).float()\n\n        if threshold_config.type in [\n                ThresholdType.topk,\n                ThresholdType.topk_hard,\n        ]:\n            if threshold_config.value >= mask.numel():\n                if threshold_config.type == ThresholdType.topk:\n                    return mask\n                else:\n                    return torch.ones_like(mask)\n\n            value, index = torch.topk(\n                mask.flatten(),\n                k=threshold_config.value,\n            )\n\n            out = torch.zeros_like(mask.flatten())\n            if threshold_config.type == ThresholdType.topk:\n                out[index] = value\n            else:\n                out[index] = 1.0\n            return out.view(mask.size())\n\n        assert False\n\n    def threshold(\n        self,\n        *args,\n        **kwargs,\n    ) -> Union['Explanation', 'HeteroExplanation']:\n        \"\"\"Thresholds the explanation masks according to the thresholding\n        method.\n\n        Args:\n            *args: Arguments passed to :class:`ThresholdConfig`.\n            **kwargs: Keyword arguments passed to :class:`ThresholdConfig`.\n        \"\"\"\n        threshold_config = ThresholdConfig.cast(*args, **kwargs)\n\n        if threshold_config is None:\n            return self\n\n        # Avoid modification of the original explanation:\n        out = copy.copy(self)\n\n        for store in out.node_stores:\n            store.node_mask = self._threshold_mask(store.get('node_mask'),\n                                                   threshold_config)\n\n        for store in out.edge_stores:\n            store.edge_mask = self._threshold_mask(store.get('edge_mask'),\n                                                   threshold_config)\n\n        return out\n\n\nclass Explanation(Data, ExplanationMixin):\n    r\"\"\"Holds all the obtained explanations of a homogeneous graph.\n\n    The explanation object is a :obj:`~torch_geometric.data.Data` object and\n    can hold node attributions and edge attributions.\n    It can also hold the original graph if needed.\n\n    Args:\n        node_mask (Tensor, optional): Node-level mask with shape\n            :obj:`[num_nodes, 1]`, :obj:`[1, num_features]` or\n            :obj:`[num_nodes, num_features]`. (default: :obj:`None`)\n        edge_mask (Tensor, optional): Edge-level mask with shape\n            :obj:`[num_edges]`. (default: :obj:`None`)\n        **kwargs (optional): Additional attributes.\n    \"\"\"\n    def validate(self, raise_on_error: bool = True) -> bool:\n        r\"\"\"Validates the correctness of the :class:`Explanation` object.\"\"\"\n        status = super().validate(raise_on_error)\n        status &= self.validate_masks(raise_on_error)\n        return status\n\n    def get_explanation_subgraph(self) -> 'Explanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with\n        zero attribution are masked out.\n        \"\"\"\n        node_mask = self.get('node_mask')\n        if node_mask is not None:\n            node_mask = node_mask.sum(dim=-1) > 0\n        edge_mask = self.get('edge_mask')\n        if edge_mask is not None:\n            edge_mask = edge_mask > 0\n        return self._apply_masks(node_mask, edge_mask)\n\n    def get_complement_subgraph(self) -> 'Explanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with any\n        attribution are masked out.\n        \"\"\"\n        node_mask = self.get('node_mask')\n        if node_mask is not None:\n            node_mask = node_mask.sum(dim=-1) == 0\n        edge_mask = self.get('edge_mask')\n        if edge_mask is not None:\n            edge_mask = edge_mask == 0\n        return self._apply_masks(node_mask, edge_mask)\n\n    def _apply_masks(\n        self,\n        node_mask: Optional[Tensor] = None,\n        edge_mask: Optional[Tensor] = None,\n    ) -> 'Explanation':\n        out = copy.copy(self)\n\n        if edge_mask is not None:\n            for key, value in self.items():\n                if key == 'edge_index':\n                    out.edge_index = value[:, edge_mask]\n                elif self.is_edge_attr(key):\n                    out[key] = value[edge_mask]\n\n        if node_mask is not None:\n            out = out.subgraph(node_mask)\n\n        return out\n\n    def visualize_feature_importance(\n        self,\n        path: Optional[str] = None,\n        feat_labels: Optional[List[str]] = None,\n        top_k: Optional[int] = None,\n    ):\n        r\"\"\"Creates a bar plot of the node feature importances by summing up\n        the node mask across all nodes.\n\n        Args:\n            path (str, optional): The path to where the plot is saved.\n                If set to :obj:`None`, will visualize the plot on-the-fly.\n                (default: :obj:`None`)\n            feat_labels (List[str], optional): The labels of features.\n                (default :obj:`None`)\n            top_k (int, optional): Top k features to plot. If :obj:`None`\n                plots all features. (default: :obj:`None`)\n        \"\"\"\n        node_mask = self.get('node_mask')\n        if node_mask is None:\n            raise ValueError(f\"The attribute 'node_mask' is not available \"\n                             f\"in '{self.__class__.__name__}' \"\n                             f\"(got {self.available_explanations})\")\n        if node_mask.dim() != 2 or node_mask.size(1) <= 1:\n            raise ValueError(f\"Cannot compute feature importance for \"\n                             f\"object-level 'node_mask' \"\n                             f\"(got shape {node_mask.size()})\")\n\n        if feat_labels is None:\n            feat_labels = range(node_mask.size(1))\n\n        score = node_mask.sum(dim=0)\n\n        return _visualize_score(score, feat_labels, path, top_k)\n\n    def visualize_graph(\n        self,\n        path: Optional[str] = None,\n        backend: Optional[str] = None,\n        node_labels: Optional[List[str]] = None,\n    ) -> None:\n        r\"\"\"Visualizes the explanation graph with edge opacity corresponding to\n        edge importance.\n\n        Args:\n            path (str, optional): The path to where the plot is saved.\n                If set to :obj:`None`, will visualize the plot on-the-fly.\n                (default: :obj:`None`)\n            backend (str, optional): The graph drawing backend to use for\n                visualization (:obj:`\"graphviz\"`, :obj:`\"networkx\"`).\n                If set to :obj:`None`, will use the most appropriate\n                visualization backend based on available system packages.\n                (default: :obj:`None`)\n            node_labels (list[str], optional): The labels/IDs of nodes.\n                (default: :obj:`None`)\n        \"\"\"\n        edge_mask = self.get('edge_mask')\n        if edge_mask is None:\n            raise ValueError(f\"The attribute 'edge_mask' is not available \"\n                             f\"in '{self.__class__.__name__}' \"\n                             f\"(got {self.available_explanations})\")\n        visualize_graph(self.edge_index, edge_mask, path, backend, node_labels)\n\n\nclass HeteroExplanation(HeteroData, ExplanationMixin):\n    r\"\"\"Holds all the obtained explanations of a heterogeneous graph.\n\n    The explanation object is a :obj:`~torch_geometric.data.HeteroData` object\n    and can hold node attributions and edge attributions.\n    It can also hold the original graph if needed.\n    \"\"\"\n    def validate(self, raise_on_error: bool = True) -> bool:\n        r\"\"\"Validates the correctness of the :class:`Explanation` object.\"\"\"\n        status = super().validate(raise_on_error)\n        status &= self.validate_masks(raise_on_error)\n        return status\n\n    def get_explanation_subgraph(self) -> 'HeteroExplanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with\n        zero attribution are masked out.\n        \"\"\"\n        return self._apply_masks(\n            node_mask_dict={\n                key: mask.sum(dim=-1) > 0\n                for key, mask in self.collect('node_mask', True).items()\n            },\n            edge_mask_dict={\n                key: mask > 0\n                for key, mask in self.collect('edge_mask', True).items()\n            },\n        )\n\n    def get_complement_subgraph(self) -> 'HeteroExplanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with any\n        attribution are masked out.\n        \"\"\"\n        return self._apply_masks(\n            node_mask_dict={\n                key: mask.sum(dim=-1) == 0\n                for key, mask in self.collect('node_mask', True).items()\n            },\n            edge_mask_dict={\n                key: mask == 0\n                for key, mask in self.collect('edge_mask', True).items()\n            },\n        )\n\n    def _apply_masks(\n        self,\n        node_mask_dict: Dict[NodeType, Tensor],\n        edge_mask_dict: Dict[EdgeType, Tensor],\n    ) -> 'HeteroExplanation':\n        out = copy.copy(self)\n\n        for edge_type, edge_mask in edge_mask_dict.items():\n            for key, value in self[edge_type].items():\n                if key == 'edge_index':\n                    out[edge_type].edge_index = value[:, edge_mask]\n                elif self[edge_type].is_edge_attr(key):\n                    out[edge_type][key] = value[edge_mask]\n\n        return out.subgraph(node_mask_dict)\n\n    def visualize_feature_importance(\n        self,\n        path: Optional[str] = None,\n        feat_labels: Optional[Dict[NodeType, List[str]]] = None,\n        top_k: Optional[int] = None,\n    ):\n        r\"\"\"Creates a bar plot of the node feature importances by summing up\n        node masks across all nodes for each node type.\n\n        Args:\n            path (str, optional): The path to where the plot is saved.\n                If set to :obj:`None`, will visualize the plot on-the-fly.\n                (default: :obj:`None`)\n            feat_labels (Dict[NodeType, List[str]], optional): The labels of\n                features for each node type. (default :obj:`None`)\n            top_k (int, optional): Top k features to plot. If :obj:`None`\n                plots all features. (default: :obj:`None`)\n        \"\"\"\n        node_mask_dict = self.node_mask_dict\n        for node_mask in node_mask_dict.values():\n            if node_mask.dim() != 2 or node_mask.size(1) <= 1:\n                raise ValueError(f\"Cannot compute feature importance for \"\n                                 f\"object-level 'node_mask' \"\n                                 f\"(got shape {node_mask_dict.size()})\")\n\n        if feat_labels is None:\n            feat_labels = {}\n            for node_type, node_mask in node_mask_dict.items():\n                feat_labels[node_type] = range(node_mask.size(1))\n\n        score = torch.cat(\n            [node_mask.sum(dim=0) for node_mask in node_mask_dict.values()],\n            dim=0)\n\n        all_feat_labels = []\n        for node_type in node_mask_dict.keys():\n            all_feat_labels += [\n                f'{node_type}#{label}' for label in feat_labels[node_type]\n            ]\n\n        return _visualize_score(score, all_feat_labels, path, top_k)\n\n\ndef _visualize_score(\n    score: torch.Tensor,\n    labels: List[str],\n    path: Optional[str] = None,\n    top_k: Optional[int] = None,\n):\n    import matplotlib.pyplot as plt\n    import pandas as pd\n\n    if len(labels) != score.numel():\n        raise ValueError(f\"The number of labels (got {len(labels)}) must \"\n                         f\"match the number of scores (got {score.numel()})\")\n\n    score = score.cpu().numpy()\n\n    df = pd.DataFrame({'score': score}, index=labels)\n    df = df.sort_values('score', ascending=False)\n    df = df.round(decimals=3)\n\n    if top_k is not None:\n        df = df.head(top_k)\n        title = f\"Feature importance for top {len(df)} features\"\n    else:\n        title = f\"Feature importance for {len(df)} features\"\n\n    ax = df.plot(\n        kind='barh',\n        figsize=(10, 7),\n        title=title,\n        ylabel='Feature label',\n        xlim=[0, float(df['score'].max()) + 0.3],\n        legend=False,\n    )\n    plt.gca().invert_yaxis()\n    ax.bar_label(container=ax.containers[0], label_type='edge')\n\n    if path is not None:\n        plt.savefig(path)\n    else:\n        plt.show()\n\n    plt.close()\n\n# Path: torch_geometric/explain/metric/basic.py\nfrom typing import List, Optional, Tuple, Union\n\nfrom torch import Tensor\n\nMETRICS = ['accuracy', 'recall', 'precision', 'f1_score', 'auroc']\n\n\ndef groundtruth_metrics(\n    pred_mask: Tensor,\n    target_mask: Tensor,\n    metrics: Optional[Union[str, List[str]]] = None,\n    threshold: float = 0.5,\n) -> Union[float, Tuple[float, ...]]:\n    r\"\"\"Compares and evaluates an explanation mask with the ground-truth\n    explanation mask.\n\n    Args:\n        pred_mask (torch.Tensor): The prediction mask to evaluate.\n        target_mask (torch.Tensor): The ground-truth target mask.\n        metrics (str or List[str], optional): The metrics to return\n            (:obj:`\"accuracy\"`, :obj:`\"recall\"`, :obj:`\"precision\"`,\n            :obj:`\"f1_score\"`, :obj:`\"auroc\"`). (default: :obj:`[\"accuracy\",\n            \"recall\", \"precision\", \"f1_score\", \"auroc\"]`)\n        threshold (float, optional): The threshold value to perform hard\n            thresholding of :obj:`mask` and :obj:`groundtruth`.\n            (default: :obj:`0.5`)\n    \"\"\"\n    import torchmetrics\n\n    if metrics is None:\n        metrics = METRICS\n\n    if isinstance(metrics, str):\n        metrics = [metrics]\n\n    if not isinstance(metrics, (tuple, list)):\n        raise ValueError(f\"Expected metrics to be a string or a list of \"\n                         f\"strings (got {type(metrics)})\")\n\n    pred_mask = pred_mask.view(-1)\n    target_mask = (target_mask >= threshold).view(-1)\n\n    outs = []\n    for metric in metrics:\n        if metric not in METRICS:\n            raise ValueError(f\"Encountered invalid metric {metric}\")\n\n        fn = getattr(torchmetrics.functional, metric)\n        if metric in {'auroc'}:\n            out = fn(pred_mask, target_mask, 'binary')\n        else:\n            out = fn(pred_mask, target_mask, 'binary', threshold)\n\n        outs.append(float(out))\n\n    return tuple(outs) if len(outs) > 1 else outs[0]\n\n# Path: torch_geometric/explain/metric/faithfulness.py\nfrom typing import Optional\n\nimport torch\nimport torch.nn.functional as F\n\nfrom torch_geometric.explain import Explainer, Explanation\nfrom torch_geometric.explain.config import MaskType, ModelMode, ModelReturnType\n\n\ndef unfaithfulness(\n    explainer: Explainer,\n    explanation: Explanation,\n    top_k: Optional[int] = None,\n) -> float:\n    r\"\"\"Evaluates how faithful an :class:`~torch_geometric.explain.Explanation`\n    is to an underyling GNN predictor, as described in the\n    `\"Evaluating Explainability for Graph Neural Networks\"\n    <https://arxiv.org/abs/2208.09339>`_ paper.\n\n    In particular, the graph explanation unfaithfulness metric is defined as\n\n    .. math::\n        \\textrm{GEF}(y, \\hat{y}) = 1 - \\exp(- \\textrm{KL}(y || \\hat{y}))\n\n    where :math:`y` refers to the prediction probability vector obtained from\n    the original graph, and :math:`\\hat{y}` refers to the prediction\n    probability vector obtained from the masked subgraph.\n    Finally, the Kullback-Leibler (KL) divergence score quantifies the distance\n    between the two probability distributions.\n\n    Args:\n        explainer (Explainer): The explainer to evaluate.\n        explanation (Explanation): The explanation to evaluate.\n        top_k (int, optional): If set, will only keep the original values of\n            the top-:math:`k` node features identified by an explanation.\n            If set to :obj:`None`, will use :obj:`explanation.node_mask` as it\n            is for masking node features. (default: :obj:`None`)\n    \"\"\"\n    if explainer.model_config.mode == ModelMode.regression:\n        raise ValueError(\"Fidelity not defined for 'regression' models\")\n\n    if top_k is not None and explainer.node_mask_type == MaskType.object:\n        raise ValueError(\"Cannot apply top-k feature selection based on a \"\n                         \"node mask of type 'object'\")\n\n    node_mask = explanation.get('node_mask')\n    edge_mask = explanation.get('edge_mask')\n    x, edge_index = explanation.x, explanation.edge_index\n    kwargs = {key: explanation[key] for key in explanation._model_args}\n\n    y = explanation.get('prediction')\n    if y is None:  # == ExplanationType.phenomenon\n        y = explainer.get_prediction(x, edge_index, **kwargs)\n\n    if node_mask is not None and top_k is not None:\n        feat_importance = node_mask.sum(dim=0)\n        _, top_k_index = feat_importance.topk(top_k)\n        node_mask = torch.zeros_like(node_mask)\n        node_mask[:, top_k_index] = 1.0\n\n    y_hat = explainer.get_masked_prediction(x, edge_index, node_mask,\n                                            edge_mask, **kwargs)\n\n    if explanation.get('index') is not None:\n        y, y_hat = y[explanation.index], y_hat[explanation.index]\n\n    if explainer.model_config.return_type == ModelReturnType.raw:\n        y, y_hat = y.softmax(dim=-1), y_hat.softmax(dim=-1)\n    elif explainer.model_config.return_type == ModelReturnType.log_probs:\n        y, y_hat = y.exp(), y_hat.exp()\n\n    kl_div = F.kl_div(y.log(), y_hat, reduction='batchmean')\n    return 1 - float(torch.exp(-kl_div))\n\n# Path: torch_geometric/explain/metric/fidelity.py\nfrom typing import Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.explain import Explainer, Explanation\nfrom torch_geometric.explain.config import ExplanationType, ModelMode\n\n\ndef fidelity(\n    explainer: Explainer,\n    explanation: Explanation,\n) -> Tuple[float, float]:\n    r\"\"\"Evaluates the fidelity of an\n    :class:`~torch_geometric.explain.Explainer` given an\n    :class:`~torch_geometric.explain.Explanation`, as described in the\n    `\"GraphFramEx: Towards Systematic Evaluation of Explainability Methods for\n    Graph Neural Networks\" <https://arxiv.org/abs/2206.09677>`_ paper.\n\n    Fidelity evaluates the contribution of the produced explanatory subgraph\n    to the initial prediction, either by giving only the subgraph to the model\n    (fidelity-) or by removing it from the entire graph (fidelity+).\n    The fidelity scores capture how good an explainable model reproduces the\n    natural phenomenon or the GNN model logic.\n\n    For **phenomenon** explanations, the fidelity scores are given by:\n\n    .. math::\n        \\textrm{fid}_{+} &= \\frac{1}{N} \\sum_{i = 1}^N\n        \\| \\mathbb{1}(\\hat{y}_i = y_i) -\n        \\mathbb{1}( \\hat{y}_i^{G_{C \\setminus S}} = y_i) \\|\n\n        \\textrm{fid}_{-} &= \\frac{1}{N} \\sum_{i = 1}^N\n        \\| \\mathbb{1}(\\hat{y}_i = y_i) -\n        \\mathbb{1}( \\hat{y}_i^{G_S} = y_i) \\|\n\n    For **model** explanations, the fidelity scores are given by:\n\n    .. math::\n        \\textrm{fid}_{+} &= 1 - \\frac{1}{N} \\sum_{i = 1}^N\n        \\mathbb{1}( \\hat{y}_i^{G_{C \\setminus S}} = \\hat{y}_i)\n\n        \\textrm{fid}_{-} &= 1 - \\frac{1}{N} \\sum_{i = 1}^N\n        \\mathbb{1}( \\hat{y}_i^{G_S} = \\hat{y}_i)\n\n    Args:\n        explainer (Explainer): The explainer to evaluate.\n        explanation (Explanation): The explanation to evaluate.\n    \"\"\"\n    if explainer.model_config.mode == ModelMode.regression:\n        raise ValueError(\"Fidelity not defined for 'regression' models\")\n\n    node_mask = explanation.get('node_mask')\n    edge_mask = explanation.get('edge_mask')\n    kwargs = {key: explanation[key] for key in explanation._model_args}\n\n    y = explanation.target\n    if explainer.explanation_type == ExplanationType.phenomenon:\n        y_hat = explainer.get_prediction(\n            explanation.x,\n            explanation.edge_index,\n            **kwargs,\n        )\n        y_hat = explainer.get_target(y_hat)\n\n    explain_y_hat = explainer.get_masked_prediction(\n        explanation.x,\n        explanation.edge_index,\n        node_mask,\n        edge_mask,\n        **kwargs,\n    )\n    explain_y_hat = explainer.get_target(explain_y_hat)\n\n    complement_y_hat = explainer.get_masked_prediction(\n        explanation.x,\n        explanation.edge_index,\n        1. - node_mask if node_mask is not None else None,\n        1. - edge_mask if edge_mask is not None else None,\n        **kwargs,\n    )\n    complement_y_hat = explainer.get_target(complement_y_hat)\n\n    if explanation.get('index') is not None:\n        y = y[explanation.index]\n        if explainer.explanation_type == ExplanationType.phenomenon:\n            y_hat = y_hat[explanation.index]\n        explain_y_hat = explain_y_hat[explanation.index]\n        complement_y_hat = complement_y_hat[explanation.index]\n\n    if explainer.explanation_type == ExplanationType.model:\n        pos_fidelity = 1. - (complement_y_hat == y).float().mean()\n        neg_fidelity = 1. - (explain_y_hat == y).float().mean()\n    else:\n        pos_fidelity = ((y_hat == y).float() -\n                        (complement_y_hat == y).float()).abs().mean()\n        neg_fidelity = ((y_hat == y).float() -\n                        (explain_y_hat == y).float()).abs().mean()\n\n...\n", "needle_token_start": 756, "needle_token_end": 819, "code_context_ntokens": 16078, "output": ["```python\ndef maybe_init_stream(self) -> None:\n    if self.is_gpu:\n        self.stream = self.module.Stream()\n        self.stream_context = partial(\n            self.module.stream,\n            stream=self.stream,\n        )\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "positive_sampling", "language": "python", "path": "torch_geometric/nn/conv/supergat_conv.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to randomly select a subset of existing edges from a graph, which can be used to train models that require a sample of positive interactions.\n2. **Input**: It takes a tensor representing the indices of edges in a graph.\n3. **Output**: It returns a tensor containing a subset of the input edge indices, representing the sampled positive edges.\n4. **Procedure**: The function first applies a dropout to the edges based on a predefined sampling ratio, effectively removing a certain percentage of edges randomly. The remaining edges are then output as the sampled positive edges, which are used during the training phase to compare against negative samples.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        negative_slope (float, optional): LeakyReLU angle of the negative\n            slope. (default: :obj:`0.2`)\n        dropout (float, optional): Dropout probability of the normalized\n            attention coefficients which exposes each node to a stochastically\n            sampled neighborhood during training. (default: :obj:`0`)\n        add_self_loops (bool, optional): If set to :obj:`False`, will not add\n            self-loops to the input graph. (default: :obj:`True`)\n        bias (bool, optional): If set to :obj:`False`, the layer will not learn\n            an additive bias. (default: :obj:`True`)\n        attention_type (str, optional): Type of attention to use\n            (:obj:`'MX'`, :obj:`'SD'`). (default: :obj:`'MX'`)\n        neg_sample_ratio (float, optional): The ratio of the number of sampled\n            negative edges to the number of positive edges.\n            (default: :obj:`0.5`)\n        edge_sample_ratio (float, optional): The ratio of samples to use for\n            training among the number of training edges. (default: :obj:`1.0`)\n        is_undirected (bool, optional): Whether the input graph is undirected.\n            If not given, will be automatically computed with the input graph\n            when negative sampling is performed. (default: :obj:`False`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch_geometric.nn.conv.MessagePassing`.\n\n    Shapes:\n        - **input:**\n          node features :math:`(|\\mathcal{V}|, F_{in})`,\n          edge indices :math:`(2, |\\mathcal{E}|)`,\n          negative edge indices :math:`(2, |\\mathcal{E}^{(-)}|)` *(optional)*\n        - **output:** node features :math:`(|\\mathcal{V}|, H * F_{out})`\n    \"\"\"\n    att_x: OptTensor\n    att_y: OptTensor\n\n    def __init__(self, in_channels: int, out_channels: int, heads: int = 1,\n                 concat: bool = True, negative_slope: float = 0.2,\n                 dropout: float = 0.0, add_self_loops: bool = True,\n                 bias: bool = True, attention_type: str = 'MX',\n                 neg_sample_ratio: float = 0.5, edge_sample_ratio: float = 1.0,\n                 is_undirected: bool = False, **kwargs):\n        kwargs.setdefault('aggr', 'add')\n        super().__init__(node_dim=0, **kwargs)\n\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.heads = heads\n        self.concat = concat\n        self.negative_slope = negative_slope\n        self.dropout = dropout\n        self.add_self_loops = add_self_loops\n        self.attention_type = attention_type\n        self.neg_sample_ratio = neg_sample_ratio\n        self.edge_sample_ratio = edge_sample_ratio\n        self.is_undirected = is_undirected\n\n        assert attention_type in ['MX', 'SD']\n        assert 0.0 < neg_sample_ratio and 0.0 < edge_sample_ratio <= 1.0\n\n        self.lin = Linear(in_channels, heads * out_channels, bias=False,\n                          weight_initializer='glorot')\n\n        if self.attention_type == 'MX':\n            self.att_l = Parameter(torch.empty(1, heads, out_channels))\n            self.att_r = Parameter(torch.empty(1, heads, out_channels))\n        else:  # self.attention_type == 'SD'\n            self.register_parameter('att_l', None)\n            self.register_parameter('att_r', None)\n\n        self.att_x = self.att_y = None  # x/y for self-supervision\n\n        if bias and concat:\n            self.bias = Parameter(torch.empty(heads * out_channels))\n        elif bias and not concat:\n            self.bias = Parameter(torch.empty(out_channels))\n        else:\n            self.register_parameter('bias', None)\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        super().reset_parameters()\n        self.lin.reset_parameters()\n        glorot(self.att_l)\n        glorot(self.att_r)\n        zeros(self.bias)\n\n    def forward(\n        self,\n        x: Tensor,\n        edge_index: Adj,\n        neg_edge_index: OptTensor = None,\n        batch: OptTensor = None,\n    ) -> Tensor:\n        r\"\"\"Runs the forward pass of the module.\n\n        Args:\n            x (torch.Tensor): The input node features.\n            edge_index (torch.Tensor or SparseTensor): The edge indices.\n            neg_edge_index (torch.Tensor, optional): The negative edges to\n                train against. If not given, uses negative sampling to\n                calculate negative edges. (default: :obj:`None`)\n            batch (torch.Tensor, optional): The batch vector\n                :math:`\\mathbf{b} \\in {\\{ 0, \\ldots, B-1\\}}^N`, which assigns\n                each element to a specific example.\n                Used when sampling negatives on-the-fly in mini-batch\n                scenarios. (default: :obj:`None`)\n        \"\"\"\n        N, H, C = x.size(0), self.heads, self.out_channels\n\n        if self.add_self_loops:\n            if isinstance(edge_index, SparseTensor):\n                edge_index = torch_sparse.fill_diag(edge_index, 1.)\n            else:\n                edge_index, _ = remove_self_loops(edge_index)\n                edge_index, _ = add_self_loops(edge_index, num_nodes=N)\n\n        x = self.lin(x).view(-1, H, C)\n\n        # propagate_type: (x: Tensor)\n        out = self.propagate(edge_index, x=x)\n\n        if self.training:\n            if isinstance(edge_index, SparseTensor):\n                col, row, _ = edge_index.coo()\n                edge_index = torch.stack([row, col], dim=0)\n            pos_edge_index = self.positive_sampling(edge_index)\n\n            pos_att = self.get_attention(\n                edge_index_i=pos_edge_index[1],\n                x_i=x[pos_edge_index[1]],\n                x_j=x[pos_edge_index[0]],\n                num_nodes=x.size(0),\n                return_logits=True,\n            )\n\n            if neg_edge_index is None:\n                neg_edge_index = self.negative_sampling(edge_index, N, batch)\n\n            neg_att = self.get_attention(\n                edge_index_i=neg_edge_index[1],\n                x_i=x[neg_edge_index[1]],\n                x_j=x[neg_edge_index[0]],\n                num_nodes=x.size(0),\n                return_logits=True,\n            )\n\n            self.att_x = torch.cat([pos_att, neg_att], dim=0)\n            self.att_y = self.att_x.new_zeros(self.att_x.size(0))\n            self.att_y[:pos_edge_index.size(1)] = 1.\n\n        if self.concat is True:\n            out = out.view(-1, self.heads * self.out_channels)\n        else:\n            out = out.mean(dim=1)\n\n        if self.bias is not None:\n            out = out + self.bias\n\n        return out\n\n    def message(self, edge_index_i: Tensor, x_i: Tensor, x_j: Tensor,\n                size_i: Optional[int]) -> Tensor:\n        alpha = self.get_attention(edge_index_i, x_i, x_j, num_nodes=size_i)\n        alpha = F.dropout(alpha, p=self.dropout, training=self.training)\n        return x_j * alpha.view(-1, self.heads, 1)\n\n    def negative_sampling(self, edge_index: Tensor, num_nodes: int,\n                          batch: OptTensor = None) -> Tensor:\n\n        num_neg_samples = int(self.neg_sample_ratio * self.edge_sample_ratio *\n                              edge_index.size(1))\n\n        if not self.is_undirected and not is_undirected(\n                edge_index, num_nodes=num_nodes):\n            edge_index = to_undirected(edge_index, num_nodes=num_nodes)\n\n        if batch is None:\n            neg_edge_index = negative_sampling(edge_index, num_nodes,\n                                               num_neg_samples=num_neg_samples)\n        else:\n            neg_edge_index = batched_negative_sampling(\n                edge_index, batch, num_neg_samples=num_neg_samples)\n\n        return neg_edge_index\n\n    \ndef positive_sampling(self, edge_index: Tensor) -> Tensor:\n        pos_edge_index, _ = dropout_edge(edge_index,\n                                         p=1. - self.edge_sample_ratio,\n                                         training=self.training)\n        return pos_edge_index\n\n    def get_attention(self, edge_index_i: Tensor, x_i: Tensor, x_j: Tensor,\n                      num_nodes: Optional[int],\n                      return_logits: bool = False) -> Tensor:\n\n        if self.attention_type == 'MX':\n            logits = (x_i * x_j).sum(dim=-1)\n            if return_logits:\n                return logits\n\n            alpha = (x_j * self.att_l).sum(-1) + (x_i * self.att_r).sum(-1)\n            alpha = alpha * logits.sigmoid()\n\n        else:  # self.attention_type == 'SD'\n            alpha = (x_i * x_j).sum(dim=-1) / math.sqrt(self.out_channels)\n            if return_logits:\n                return alpha\n\n        alpha = F.leaky_relu(alpha, self.negative_slope)\n        alpha = softmax(alpha, edge_index_i, num_nodes=num_nodes)\n        return alpha\n\n    def get_attention_loss(self) -> Tensor:\n        r\"\"\"Computes the self-supervised graph attention loss.\"\"\"\n        if not self.training:\n            return torch.tensor([0], device=self.lin.weight.device)\n\n        return F.binary_cross_entropy_with_logits(\n            self.att_x.mean(dim=-1),\n            self.att_y,\n        )\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}({self.in_channels}, '\n                f'{self.out_channels}, heads={self.heads}, '\n                f'type={self.attention_type})')\n\n# Path: torch_geometric/nn/conv/tag_conv.py\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.nn.conv import MessagePassing\nfrom torch_geometric.nn.conv.gcn_conv import gcn_norm\nfrom torch_geometric.nn.dense.linear import Linear\nfrom torch_geometric.nn.inits import zeros\nfrom torch_geometric.typing import Adj, OptTensor, SparseTensor\nfrom torch_geometric.utils import spmm\n\n\nclass TAGConv(MessagePassing):\n    r\"\"\"The topology adaptive graph convolutional networks operator from the\n    `\"Topology Adaptive Graph Convolutional Networks\"\n    <https://arxiv.org/abs/1710.10370>`_ paper.\n\n    .. math::\n        \\mathbf{X}^{\\prime} = \\sum_{k=0}^K \\left( \\mathbf{D}^{-1/2} \\mathbf{A}\n        \\mathbf{D}^{-1/2} \\right)^k \\mathbf{X} \\mathbf{W}_{k},\n\n    where :math:`\\mathbf{A}` denotes the adjacency matrix and\n    :math:`D_{ii} = \\sum_{j=0} A_{ij}` its diagonal degree matrix.\n    The adjacency matrix can include other values than :obj:`1` representing\n    edge weights via the optional :obj:`edge_weight` tensor.\n\n    Args:\n        in_channels (int): Size of each input sample, or :obj:`-1` to derive\n            the size from the first input(s) to the forward method.\n        out_channels (int): Size of each output sample.\n        K (int, optional): Number of hops :math:`K`. (default: :obj:`3`)\n        bias (bool, optional): If set to :obj:`False`, the layer will not learn\n            an additive bias. (default: :obj:`True`)\n        normalize (bool, optional): Whether to apply symmetric normalization.\n            (default: :obj:`True`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch_geometric.nn.conv.MessagePassing`.\n\n    Shapes:\n        - **input:**\n          node_features :math:`(|\\mathcal{V}|, F_{in})`,\n          edge_index :math:`(2, |\\mathcal{E}|)`,\n          edge_weights :math:`(|\\mathcal{E}|)` *(optional)*\n        - **output:** node features :math:`(|\\mathcal{V}|, F_{out})`\n    \"\"\"\n    def __init__(self, in_channels: int, out_channels: int, K: int = 3,\n                 bias: bool = True, normalize: bool = True, **kwargs):\n        kwargs.setdefault('aggr', 'add')\n        super().__init__(**kwargs)\n\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.K = K\n        self.normalize = normalize\n\n        self.lins = torch.nn.ModuleList([\n            Linear(in_channels, out_channels, bias=False) for _ in range(K + 1)\n        ])\n\n        if bias:\n            self.bias = torch.nn.Parameter(torch.empty(out_channels))\n        else:\n            self.register_parameter('bias', None)\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        super().reset_parameters()\n        for lin in self.lins:\n            lin.reset_parameters()\n        zeros(self.bias)\n\n    def forward(self, x: Tensor, edge_index: Adj,\n                edge_weight: OptTensor = None) -> Tensor:\n\n        if self.normalize:\n            if isinstance(edge_index, Tensor):\n                edge_index, edge_weight = gcn_norm(  # yapf: disable\n                    edge_index, edge_weight, x.size(self.node_dim),\n                    improved=False, add_self_loops=False, flow=self.flow,\n                    dtype=x.dtype)\n\n            elif isinstance(edge_index, SparseTensor):\n                edge_index = gcn_norm(  # yapf: disable\n                    edge_index, edge_weight, x.size(self.node_dim),\n                    add_self_loops=False, flow=self.flow, dtype=x.dtype)\n\n        out = self.lins[0](x)\n        for lin in self.lins[1:]:\n            # propagate_type: (x: Tensor, edge_weight: OptTensor)\n            x = self.propagate(edge_index, x=x, edge_weight=edge_weight)\n            out = out + lin.forward(x)\n\n        if self.bias is not None:\n            out = out + self.bias\n\n        return out\n\n    def message(self, x_j: Tensor, edge_weight: OptTensor) -> Tensor:\n        return x_j if edge_weight is None else edge_weight.view(-1, 1) * x_j\n\n    def message_and_aggregate(self, adj_t: Adj, x: Tensor) -> Tensor:\n        return spmm(adj_t, x, reduce=self.aggr)\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}({self.in_channels}, '\n                f'{self.out_channels}, K={self.K})')\n\n# Path: torch_geometric/nn/conv/transformer_conv.py\nimport math\nimport typing\nfrom typing import Optional, Tuple, Union\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import Tensor\n\nfrom torch_geometric.nn.conv import MessagePassing\nfrom torch_geometric.nn.dense.linear import Linear\nfrom torch_geometric.typing import (\n    Adj,\n    NoneType,\n    OptTensor,\n    PairTensor,\n    SparseTensor,\n)\nfrom torch_geometric.utils import softmax\n\nif typing.TYPE_CHECKING:\n    from typing import overload\nelse:\n    from torch.jit import _overload_method as overload\n\n\nclass TransformerConv(MessagePassing):\n    r\"\"\"The graph transformer operator from the `\"Masked Label Prediction:\n    Unified Message Passing Model for Semi-Supervised Classification\"\n    <https://arxiv.org/abs/2009.03509>`_ paper.\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\mathbf{W}_1 \\mathbf{x}_i +\n        \\sum_{j \\in \\mathcal{N}(i)} \\alpha_{i,j} \\mathbf{W}_2 \\mathbf{x}_{j},\n\n    where the attention coefficients :math:`\\alpha_{i,j}` are computed via\n    multi-head dot product attention:\n\n    .. math::\n        \\alpha_{i,j} = \\textrm{softmax} \\left(\n        \\frac{(\\mathbf{W}_3\\mathbf{x}_i)^{\\top} (\\mathbf{W}_4\\mathbf{x}_j)}\n        {\\sqrt{d}} \\right)\n\n    Args:\n        in_channels (int or tuple): Size of each input sample, or :obj:`-1` to\n            derive the size from the first input(s) to the forward method.\n            A tuple corresponds to the sizes of source and target\n            dimensionalities.\n        out_channels (int): Size of each output sample.\n        heads (int, optional): Number of multi-head-attentions.\n            (default: :obj:`1`)\n        concat (bool, optional): If set to :obj:`False`, the multi-head\n            attentions are averaged instead of concatenated.\n            (default: :obj:`True`)\n        beta (bool, optional): If set, will combine aggregation and\n            skip information via\n\n            .. math::\n                \\mathbf{x}^{\\prime}_i = \\beta_i \\mathbf{W}_1 \\mathbf{x}_i +\n                (1 - \\beta_i) \\underbrace{\\left(\\sum_{j \\in \\mathcal{N}(i)}\n                \\alpha_{i,j} \\mathbf{W}_2 \\vec{x}_j \\right)}_{=\\mathbf{m}_i}\n\n            with :math:`\\beta_i = \\textrm{sigmoid}(\\mathbf{w}_5^{\\top}\n            [ \\mathbf{W}_1 \\mathbf{x}_i, \\mathbf{m}_i, \\mathbf{W}_1\n            \\mathbf{x}_i - \\mathbf{m}_i ])` (default: :obj:`False`)\n        dropout (float, optional): Dropout probability of the normalized\n            attention coefficients which exposes each node to a stochastically\n            sampled neighborhood during training. (default: :obj:`0`)\n        edge_dim (int, optional): Edge feature dimensionality (in case\n            there are any). Edge features are added to the keys after\n            linear transformation, that is, prior to computing the\n            attention dot product. They are also added to final values\n            after the same linear transformation. The model is:\n\n            .. math::\n                \\mathbf{x}^{\\prime}_i = \\mathbf{W}_1 \\mathbf{x}_i +\n                \\sum_{j \\in \\mathcal{N}(i)} \\alpha_{i,j} \\left(\n                \\mathbf{W}_2 \\mathbf{x}_{j} + \\mathbf{W}_6 \\mathbf{e}_{ij}\n                \\right),\n\n            where the attention coefficients :math:`\\alpha_{i,j}` are now\n            computed via:\n\n            .. math::\n                \\alpha_{i,j} = \\textrm{softmax} \\left(\n                \\frac{(\\mathbf{W}_3\\mathbf{x}_i)^{\\top}\n                (\\mathbf{W}_4\\mathbf{x}_j + \\mathbf{W}_6 \\mathbf{e}_{ij})}\n                {\\sqrt{d}} \\right)\n\n            (default :obj:`None`)\n        bias (bool, optional): If set to :obj:`False`, the layer will not learn\n            an additive bias. (default: :obj:`True`)\n        root_weight (bool, optional): If set to :obj:`False`, the layer will\n            not add the transformed root node features to the output and the\n            option  :attr:`beta` is set to :obj:`False`. (default: :obj:`True`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch_geometric.nn.conv.MessagePassing`.\n    \"\"\"\n    _alpha: OptTensor\n\n    def __init__(\n        self,\n        in_channels: Union[int, Tuple[int, int]],\n        out_channels: int,\n        heads: int = 1,\n        concat: bool = True,\n        beta: bool = False,\n        dropout: float = 0.,\n        edge_dim: Optional[int] = None,\n        bias: bool = True,\n        root_weight: bool = True,\n        **kwargs,\n    ):\n        kwargs.setdefault('aggr', 'add')\n        super().__init__(node_dim=0, **kwargs)\n\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        self.heads = heads\n        self.beta = beta and root_weight\n        self.root_weight = root_weight\n        self.concat = concat\n        self.dropout = dropout\n        self.edge_dim = edge_dim\n        self._alpha = None\n\n        if isinstance(in_channels, int):\n            in_channels = (in_channels, in_channels)\n\n        self.lin_key = Linear(in_channels[0], heads * out_channels)\n        self.lin_query = Linear(in_channels[1], heads * out_channels)\n        self.lin_value = Linear(in_channels[0], heads * out_channels)\n        if edge_dim is not None:\n            self.lin_edge = Linear(edge_dim, heads * out_channels, bias=False)\n        else:\n            self.lin_edge = self.register_parameter('lin_edge', None)\n\n        if concat:\n            self.lin_skip = Linear(in_channels[1], heads * out_channels,\n                                   bias=bias)\n            if self.beta:\n                self.lin_beta = Linear(3 * heads * out_channels, 1, bias=False)\n            else:\n                self.lin_beta = self.register_parameter('lin_beta', None)\n        else:\n            self.lin_skip = Linear(in_channels[1], out_channels, bias=bias)\n            if self.beta:\n                self.lin_beta = Linear(3 * out_channels, 1, bias=False)\n            else:\n                self.lin_beta = self.register_parameter('lin_beta', None)\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        super().reset_parameters()\n        self.lin_key.reset_parameters()\n        self.lin_query.reset_parameters()\n        self.lin_value.reset_parameters()\n        if self.edge_dim:\n            self.lin_edge.reset_parameters()\n        self.lin_skip.reset_parameters()\n        if self.beta:\n            self.lin_beta.reset_parameters()\n\n    @overload\n    def forward(\n        self,\n        x: Union[Tensor, PairTensor],\n        edge_index: Adj,\n        edge_attr: OptTensor = None,\n        return_attention_weights: NoneType = None,\n    ) -> Tensor:\n        pass\n\n    @overload\n    def forward(  # noqa: F811\n        self,\n        x: Union[Tensor, PairTensor],\n        edge_index: Tensor,\n        edge_attr: OptTensor = None,\n        return_attention_weights: bool = None,\n    ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]:\n        pass\n\n    @overload\n    def forward(  # noqa: F811\n        self,\n        x: Union[Tensor, PairTensor],\n        edge_index: SparseTensor,\n        edge_attr: OptTensor = None,\n        return_attention_weights: bool = None,\n    ) -> Tuple[Tensor, SparseTensor]:\n        pass\n\n    def forward(  # noqa: F811\n        self,\n        x: Union[Tensor, PairTensor],\n        edge_index: Adj,\n        edge_attr: OptTensor = None,\n        return_attention_weights: Optional[bool] = None,\n    ) -> Union[\n            Tensor,\n            Tuple[Tensor, Tuple[Tensor, Tensor]],\n            Tuple[Tensor, SparseTensor],\n    ]:\n        r\"\"\"Runs the forward pass of the module.\n\n        Args:\n            x (torch.Tensor or (torch.Tensor, torch.Tensor)): The input node\n                features.\n            edge_index (torch.Tensor or SparseTensor): The edge indices.\n            edge_attr (torch.Tensor, optional): The edge features.\n                (default: :obj:`None`)\n            return_attention_weights (bool, optional): If set to :obj:`True`,\n                will additionally return the tuple\n                :obj:`(edge_index, attention_weights)`, holding the computed\n                attention weights for each edge. (default: :obj:`None`)\n        \"\"\"\n        H, C = self.heads, self.out_channels\n\n        if isinstance(x, Tensor):\n            x = (x, x)\n\n        query = self.lin_query(x[1]).view(-1, H, C)\n        key = self.lin_key(x[0]).view(-1, H, C)\n        value = self.lin_value(x[0]).view(-1, H, C)\n\n        # propagate_type: (query: Tensor, key:Tensor, value: Tensor,\n        #                  edge_attr: OptTensor)\n        out = self.propagate(edge_index, query=query, key=key, value=value,\n                             edge_attr=edge_attr)\n\n        alpha = self._alpha\n        self._alpha = None\n\n        if self.concat:\n            out = out.view(-1, self.heads * self.out_channels)\n        else:\n            out = out.mean(dim=1)\n\n        if self.root_weight:\n            x_r = self.lin_skip(x[1])\n            if self.lin_beta is not None:\n                beta = self.lin_beta(torch.cat([out, x_r, out - x_r], dim=-1))\n                beta = beta.sigmoid()\n                out = beta * x_r + (1 - beta) * out\n            else:\n                out = out + x_r\n\n        if isinstance(return_attention_weights, bool):\n            assert alpha is not None\n            if isinstance(edge_index, Tensor):\n                return out, (edge_index, alpha)\n            elif isinstance(edge_index, SparseTensor):\n                return out, edge_index.set_value(alpha, layout='coo')\n        else:\n            return out\n\n    def message(self, query_i: Tensor, key_j: Tensor, value_j: Tensor,\n                edge_attr: OptTensor, index: Tensor, ptr: OptTensor,\n                size_i: Optional[int]) -> Tensor:\n\n        if self.lin_edge is not None:\n            assert edge_attr is not None\n            edge_attr = self.lin_edge(edge_attr).view(-1, self.heads,\n                                                      self.out_channels)\n            key_j = key_j + edge_attr\n\n        alpha = (query_i * key_j).sum(dim=-1) / math.sqrt(self.out_channels)\n        alpha = softmax(alpha, index, ptr, size_i)\n        self._alpha = alpha\n        alpha = F.dropout(alpha, p=self.dropout, training=self.training)\n\n        out = value_j\n        if edge_attr is not None:\n            out = out + edge_attr\n\n        out = out * alpha.view(-1, self.heads, 1)\n        return out\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}({self.in_channels}, '\n                f'{self.out_channels}, heads={self.heads})')\n\n# Path: torch_geometric/nn/conv/utils/cheatsheet.py\nimport importlib\nimport inspect\nimport re\nfrom typing import Optional\n\n\ndef paper_title(cls: str) -> Optional[str]:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    match = re.search('`\\\".+?\\\"', inspect.getdoc(cls), flags=re.DOTALL)\n    return None if match is None else match.group().replace('\\n', ' ')[2:-1]\n\n\ndef paper_link(cls: str) -> Optional[str]:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    match = re.search('<.+?>', inspect.getdoc(cls), flags=re.DOTALL)\n    return None if match is None else match.group().replace('\\n', ' ')[1:-1]\n\n\ndef supports_sparse_tensor(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'SparseTensor' in str(signature)\n\n\ndef supports_edge_weights(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'edge_weight' in str(signature)\n\n\ndef supports_edge_features(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'edge_attr' in str(signature)\n\n\ndef supports_bipartite_graphs(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'Union[torch.Tensor, Tuple[torch.Tensor' in str(signature)\n\n\ndef supports_static_graphs(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    return 'node_dim=' not in inspect.getsource(cls.__init__)\n\n\ndef supports_lazy_initialization(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    doc = re.sub(' +', ' ', inspect.getdoc(cls).replace('\\n', ' '))\n    match = re.search('or :obj:`-1` to derive the size from the first', doc)\n    return match is not None\n\n\ndef processes_heterogeneous_graphs(cls: str) -> bool:\n    if 'hetero' in cls.lower():\n        return True\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'edge_index_dict' in str(signature) or 'edge_type' in str(signature)\n\n\ndef processes_hypergraphs(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'hyperedge_index' in str(signature)\n\n\ndef processes_point_clouds(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return (('edge_index' not in str(signature)\n             and 'csc' not in str(signature)) or 'pos' in str(signature))\n\n# Path: torch_geometric/nn/conv/utils/__init__.py\nr\"\"\"GNN utility package.\"\"\"\n\nfrom .cheatsheet import paper_title, paper_link\nfrom .cheatsheet import supports_sparse_tensor\nfrom .cheatsheet import supports_edge_weights\nfrom .cheatsheet import supports_edge_features\nfrom .cheatsheet import supports_bipartite_graphs\nfrom .cheatsheet import supports_static_graphs\nfrom .cheatsheet import supports_lazy_initialization\nfrom .cheatsheet import processes_heterogeneous_graphs\nfrom .cheatsheet import processes_hypergraphs\nfrom .cheatsheet import processes_point_clouds\n\n__all__ = [\n    'paper_title',\n    'paper_link',\n    'supports_sparse_tensor',\n    'supports_edge_weights',\n    'supports_edge_features',\n    'supports_bipartite_graphs',\n    'supports_static_graphs',\n    'supports_lazy_initialization',\n    'processes_heterogeneous_graphs',\n    'processes_hypergraphs',\n    'processes_point_clouds',\n]\n\n# Path: torch_geometric/nn/conv/wl_conv.py\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import Adj\nfrom torch_geometric.utils import (\n    degree,\n    is_sparse,\n    scatter,\n    sort_edge_index,\n    to_edge_index,\n)\n\n\nclass WLConv(torch.nn.Module):\n    r\"\"\"The Weisfeiler Lehman (WL) operator from the `\"A Reduction of a Graph\n    to a Canonical Form and an Algebra Arising During this Reduction\"\n    <https://www.iti.zcu.cz/wl2018/pdf/wl_paper_translation.pdf>`_ paper.\n\n    :class:`WLConv` iteratively refines node colorings according to:\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\textrm{hash} \\left( \\mathbf{x}_i, \\{\n        \\mathbf{x}_j \\colon j \\in \\mathcal{N}(i) \\} \\right)\n\n    Shapes:\n        - **input:**\n          node coloring :math:`(|\\mathcal{V}|, F_{in})` *(one-hot encodings)*\n          or :math:`(|\\mathcal{V}|)` *(integer-based)*,\n          edge indices :math:`(2, |\\mathcal{E}|)`\n        - **output:** node coloring :math:`(|\\mathcal{V}|)` *(integer-based)*\n    \"\"\"\n    def __init__(self):\n        super().__init__()\n        self.hashmap = {}\n\n    def reset_parameters(self):\n        r\"\"\"Resets all learnable parameters of the module.\"\"\"\n        self.hashmap = {}\n\n    @torch.no_grad()\n    def forward(self, x: Tensor, edge_index: Adj) -> Tensor:\n        r\"\"\"Runs the forward pass of the module.\"\"\"\n        if x.dim() > 1:\n            assert (x.sum(dim=-1) == 1).sum() == x.size(0)\n            x = x.argmax(dim=-1)  # one-hot -> integer.\n        assert x.dtype == torch.long\n\n        if is_sparse(edge_index):\n            col_and_row, _ = to_edge_index(edge_index)\n            col = col_and_row[0]\n            row = col_and_row[1]\n        else:\n            edge_index = sort_edge_index(edge_index, num_nodes=x.size(0),\n                                         sort_by_row=False)\n            row, col = edge_index[0], edge_index[1]\n\n        # `col` is sorted, so we can use it to `split` neighbors to groups:\n        deg = degree(col, x.size(0), dtype=torch.long).tolist()\n\n        out = []\n        for node, neighbors in zip(x.tolist(), x[row].split(deg)):\n            idx = hash(tuple([node] + neighbors.sort()[0].tolist()))\n            if idx not in self.hashmap:\n                self.hashmap[idx] = len(self.hashmap)\n            out.append(self.hashmap[idx])\n\n        return torch.tensor(out, device=x.device)\n\n    def histogram(self, x: Tensor, batch: Optional[Tensor] = None,\n                  norm: bool = False) -> Tensor:\n        r\"\"\"Given a node coloring :obj:`x`, computes the color histograms of\n        the respective graphs (separated by :obj:`batch`).\n        \"\"\"\n        if batch is None:\n            batch = torch.zeros(x.size(0), dtype=torch.long, device=x.device)\n\n        num_colors = len(self.hashmap)\n        batch_size = int(batch.max()) + 1\n\n        index = batch * num_colors + x\n        out = scatter(torch.ones_like(index), index, dim=0,\n                      dim_size=num_colors * batch_size, reduce='sum')\n        out = out.view(batch_size, num_colors)\n\n        if norm:\n            out = out.to(torch.float)\n            out /= out.norm(dim=-1, keepdim=True)\n\n        return out\n\n# Path: torch_geometric/nn/conv/wl_conv_continuous.py\nfrom typing import Union\n\nfrom torch import Tensor\n\nfrom torch_geometric.nn.conv import MessagePassing\nfrom torch_geometric.typing import (\n    Adj,\n    OptPairTensor,\n    OptTensor,\n    Size,\n    SparseTensor,\n)\nfrom torch_geometric.utils import scatter, spmm\n\n\nclass WLConvContinuous(MessagePassing):\n    r\"\"\"The Weisfeiler Lehman operator from the `\"Wasserstein\n    Weisfeiler-Lehman Graph Kernels\" <https://arxiv.org/abs/1906.01277>`_\n    paper.\n\n    Refinement is done though a degree-scaled mean aggregation and works on\n    nodes with continuous attributes:\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\frac{1}{2}\\big(\\mathbf{x}_i +\n        \\frac{1}{\\textrm{deg}(i)}\n        \\sum_{j \\in \\mathcal{N}(i)} e_{j,i} \\cdot \\mathbf{x}_j \\big)\n\n    where :math:`e_{j,i}` denotes the edge weight from source node :obj:`j` to\n    target node :obj:`i` (default: :obj:`1`)\n\n    Args:\n        **kwargs (optional): Additional arguments of\n            :class:`torch_geometric.nn.conv.MessagePassing`.\n\n    Shapes:\n        - **input:**\n          node features :math:`(|\\mathcal{V}|, F)` or\n          :math:`((|\\mathcal{V_s}|, F), (|\\mathcal{V_t}|, F))` if bipartite,\n          edge indices :math:`(2, |\\mathcal{E}|)`,\n          edge weights :math:`(|\\mathcal{E}|)` *(optional)*\n        - **output:** node features :math:`(|\\mathcal{V}|, F)` or\n          :math:`(|\\mathcal{V}_t|, F)` if bipartite\n    \"\"\"\n    def __init__(self, **kwargs):\n        super().__init__(aggr='add', **kwargs)\n\n    def forward(\n        self,\n        x: Union[Tensor, OptPairTensor],\n        edge_index: Adj,\n        edge_weight: OptTensor = None,\n        size: Size = None,\n    ) -> Tensor:\n\n        if isinstance(x, Tensor):\n            x = (x, x)\n\n        # propagate_type: (x: OptPairTensor, edge_weight: OptTensor)\n        out = self.propagate(edge_index, x=x, edge_weight=edge_weight,\n                             size=size)\n\n        if isinstance(edge_index, SparseTensor):\n            assert edge_weight is None\n            dst_index, _, edge_weight = edge_index.coo()\n        else:\n            dst_index = edge_index[1]\n\n        if edge_weight is None:\n            edge_weight = x[0].new_ones(dst_index.numel())\n\n        deg = scatter(edge_weight, dst_index, 0, out.size(0), reduce='sum')\n        deg_inv = 1. / deg\n        deg_inv.masked_fill_(deg_inv == float('inf'), 0)\n        out = deg_inv.view(-1, 1) * out\n\n        x_dst = x[1]\n        if x_dst is not None:\n            out = 0.5 * (x_dst + out)\n\n        return out\n\n    def message(self, x_j: Tensor, edge_weight: OptTensor) -> Tensor:\n        return x_j if edge_weight is None else edge_weight.view(-1, 1) * x_j\n\n    def message_and_aggregate(self, adj_t: Adj, x: OptPairTensor) -> Tensor:\n        return spmm(adj_t, x[0], reduce=self.aggr)\n\n# Path: torch_geometric/nn/conv/x_conv.py\nfrom math import ceil\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\nfrom torch.nn import ELU\nfrom torch.nn import BatchNorm1d as BN\nfrom torch.nn import Conv1d\nfrom torch.nn import Linear as L\nfrom torch.nn import Sequential as S\n\nfrom torch_geometric.nn import Reshape\nfrom torch_geometric.nn.inits import reset\n\ntry:\n    from torch_cluster import knn_graph\nexcept ImportError:\n    knn_graph = None\n\n\nclass XConv(torch.nn.Module):\n    r\"\"\"The convolutional operator on :math:`\\mathcal{X}`-transformed points\n    from the `\"PointCNN: Convolution On X-Transformed Points\"\n    <https://arxiv.org/abs/1801.07791>`_ paper.\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\mathrm{Conv}\\left(\\mathbf{K},\n        \\gamma_{\\mathbf{\\Theta}}(\\mathbf{P}_i - \\mathbf{p}_i) \\times\n        \\left( h_\\mathbf{\\Theta}(\\mathbf{P}_i - \\mathbf{p}_i) \\, \\Vert \\,\n        \\mathbf{x}_i \\right) \\right),\n\n    where :math:`\\mathbf{K}` and :math:`\\mathbf{P}_i` denote the trainable\n    filter and neighboring point positions of :math:`\\mathbf{x}_i`,\n    respectively.\n    :math:`\\gamma_{\\mathbf{\\Theta}}` and :math:`h_{\\mathbf{\\Theta}}` describe\n    neural networks, *i.e.* MLPs, where :math:`h_{\\mathbf{\\Theta}}`\n    individually lifts each point into a higher-dimensional space, and\n    :math:`\\gamma_{\\mathbf{\\Theta}}` computes the :math:`\\mathcal{X}`-\n    transformation matrix based on *all* points in a neighborhood.\n\n    Args:\n        in_channels (int): Size of each input sample.\n        out_channels (int): Size of each output sample.\n        dim (int): Point cloud dimensionality.\n        kernel_size (int): Size of the convolving kernel, *i.e.* number of\n            neighbors including self-loops.\n        hidden_channels (int, optional): Output size of\n            :math:`h_{\\mathbf{\\Theta}}`, *i.e.* dimensionality of lifted\n            points. If set to :obj:`None`, will be automatically set to\n            :obj:`in_channels / 4`. (default: :obj:`None`)\n        dilation (int, optional): The factor by which the neighborhood is\n            extended, from which :obj:`kernel_size` neighbors are then\n            uniformly sampled. Can be interpreted as the dilation rate of\n            classical convolutional operators. (default: :obj:`1`)\n        bias (bool, optional): If set to :obj:`False`, the layer will not learn\n            an additive bias. (default: :obj:`True`)\n        num_workers (int): Number of workers to use for k-NN computation.\n            Has no effect in case :obj:`batch` is not :obj:`None`, or the input\n            lies on the GPU. (default: :obj:`1`)\n\n    Shapes:\n        - **input:**\n          node features :math:`(|\\mathcal{V}|, F_{in})`,\n          positions :math:`(|\\mathcal{V}|, D)`,\n          batch vector :math:`(|\\mathcal{V}|)` *(optional)*\n        - **output:**\n          node features :math:`(|\\mathcal{V}|, F_{out})`\n    \"\"\"\n    def __init__(self, in_channels: int, out_channels: int, dim: int,\n                 kernel_size: int, hidden_channels: Optional[int] = None,\n                 dilation: int = 1, bias: bool = True, num_workers: int = 1):\n        super().__init__()\n\n        if knn_graph is None:\n            raise ImportError('`XConv` requires `torch-cluster`.')\n\n        self.in_channels = in_channels\n        if hidden_channels is None:\n            hidden_channels = in_channels // 4\n        assert hidden_channels > 0\n        self.hidden_channels = hidden_channels\n        self.out_channels = out_channels\n        self.dim = dim\n        self.kernel_size = kernel_size\n        self.dilation = dilation\n        self.num_workers = num_workers\n\n        C_in, C_delta, C_out = in_channels, hidden_channels, out_channels\n        D, K = dim, kernel_size\n\n        self.mlp1 = S(\n            L(dim, C_delta),\n            ELU(),\n            BN(C_delta),\n            L(C_delta, C_delta),\n            ELU(),\n            BN(C_delta),\n            Reshape(-1, K, C_delta),\n        )\n\n        self.mlp2 = S(\n            L(D * K, K**2),\n            ELU(),\n            BN(K**2),\n            Reshape(-1, K, K),\n            Conv1d(K, K**2, K, groups=K),\n            ELU(),\n            BN(K**2),\n            Reshape(-1, K, K),\n            Conv1d(K, K**2, K, groups=K),\n            BN(K**2),\n            Reshape(-1, K, K),\n        )\n\n        C_in = C_in + C_delta\n        depth_multiplier = int(ceil(C_out / C_in))\n        self.conv = S(\n            Conv1d(C_in, C_in * depth_multiplier, K, groups=C_in),\n            Reshape(-1, C_in * depth_multiplier),\n            L(C_in * depth_multiplier, C_out, bias=bias),\n        )\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        r\"\"\"Resets all learnable parameters of the module.\"\"\"\n        reset(self.mlp1)\n        reset(self.mlp2)\n        reset(self.conv)\n\n    def forward(self, x: Tensor, pos: Tensor, batch: Optional[Tensor] = None):\n        r\"\"\"Runs the forward pass of the module.\"\"\"\n        pos = pos.unsqueeze(-1) if pos.dim() == 1 else pos\n        (N, D), K = pos.size(), self.kernel_size\n\n        edge_index = knn_graph(pos, K * self.dilation, batch, loop=True,\n                               flow='target_to_source',\n                               num_workers=self.num_workers)\n\n        if self.dilation > 1:\n            edge_index = edge_index[:, ::self.dilation]\n\n        row, col = edge_index[0], edge_index[1]\n\n        pos = pos[col] - pos[row]\n\n        x_star = self.mlp1(pos)\n        if x is not None:\n            x = x.unsqueeze(-1) if x.dim() == 1 else x\n            x = x[col].view(N, K, self.in_channels)\n            x_star = torch.cat([x_star, x], dim=-1)\n        x_star = x_star.transpose(1, 2).contiguous()\n\n        transform_matrix = self.mlp2(pos.view(N, K * D))\n\n        x_transformed = torch.matmul(x_star, transform_matrix)\n\n        out = self.conv(x_transformed)\n\n        return out\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}({self.in_channels}, '\n                f'{self.out_channels})')\n\n# Path: torch_geometric/nn/conv/__init__.py\nfrom .message_passing import MessagePassing\nfrom .simple_conv import SimpleConv\nfrom .gcn_conv import GCNConv\nfrom .cheb_conv import ChebConv\nfrom .sage_conv import SAGEConv\nfrom .cugraph.sage_conv import CuGraphSAGEConv\nfrom .graph_conv import GraphConv\nfrom .gravnet_conv import GravNetConv\nfrom .gated_graph_conv import GatedGraphConv\nfrom .res_gated_graph_conv import ResGatedGraphConv\nfrom .gat_conv import GATConv\nfrom .cugraph.gat_conv import CuGraphGATConv\nfrom .fused_gat_conv import FusedGATConv\nfrom .gatv2_conv import GATv2Conv\nfrom .transformer_conv import TransformerConv\nfrom .agnn_conv import AGNNConv\nfrom .tag_conv import TAGConv\nfrom .gin_conv import GINConv, GINEConv\nfrom .arma_conv import ARMAConv\nfrom .sg_conv import SGConv\nfrom .appnp import APPNP\nfrom .mf_conv import MFConv\nfrom .rgcn_conv import RGCNConv, FastRGCNConv\nfrom .cugraph.rgcn_conv import CuGraphRGCNConv\nfrom .rgat_conv import RGATConv\nfrom .signed_conv import SignedConv\nfrom .dna_conv import DNAConv\nfrom .point_conv import PointNetConv\nfrom .gmm_conv import GMMConv\nfrom .spline_conv import SplineConv\nfrom .nn_conv import NNConv\nfrom .cg_conv import CGConv\nfrom .edge_conv import EdgeConv, DynamicEdgeConv\nfrom .x_conv import XConv\nfrom .ppf_conv import PPFConv\nfrom .feast_conv import FeaStConv\nfrom .point_transformer_conv import PointTransformerConv\nfrom .hypergraph_conv import HypergraphConv\nfrom .le_conv import LEConv\nfrom .pna_conv import PNAConv\nfrom .cluster_gcn_conv import ClusterGCNConv\nfrom .gen_conv import GENConv\nfrom .gcn2_conv import GCN2Conv\nfrom .pan_conv import PANConv\nfrom .wl_conv import WLConv\nfrom .wl_conv_continuous import WLConvContinuous\nfrom .film_conv import FiLMConv\nfrom .supergat_conv import SuperGATConv\nfrom .fa_conv import FAConv\nfrom .eg_conv import EGConv\nfrom .pdn_conv import PDNConv\nfrom .general_conv import GeneralConv\nfrom .hgt_conv import HGTConv\nfrom .heat_conv import HEATConv\nfrom .hetero_conv import HeteroConv\nfrom .han_conv import HANConv\nfrom .lg_conv import LGConv\nfrom .ssg_conv import SSGConv\nfrom .point_gnn_conv import PointGNNConv\nfrom .gps_conv import GPSConv\nfrom .antisymmetric_conv import AntiSymmetricConv\nfrom .dir_gnn_conv import DirGNNConv\nfrom .mixhop_conv import MixHopConv\n\nimport torch_geometric.nn.conv.utils  # noqa\n\n__all__ = [\n    'MessagePassing',\n    'SimpleConv',\n    'GCNConv',\n    'ChebConv',\n    'SAGEConv',\n    'CuGraphSAGEConv',\n    'GraphConv',\n    'GravNetConv',\n    'GatedGraphConv',\n    'ResGatedGraphConv',\n    'GATConv',\n    'CuGraphGATConv',\n    'FusedGATConv',\n    'GATv2Conv',\n    'TransformerConv',\n    'AGNNConv',\n    'TAGConv',\n    'GINConv',\n    'GINEConv',\n    'ARMAConv',\n    'SGConv',\n    'SSGConv',\n    'APPNP',\n    'MFConv',\n    'RGCNConv',\n    'FastRGCNConv',\n    'CuGraphRGCNConv',\n    'RGATConv',\n    'SignedConv',\n    'DNAConv',\n    'PointNetConv',\n    'GMMConv',\n    'SplineConv',\n    'NNConv',\n    'CGConv',\n    'EdgeConv',\n    'DynamicEdgeConv',\n    'XConv',\n    'PPFConv',\n    'FeaStConv',\n    'PointTransformerConv',\n    'HypergraphConv',\n    'LEConv',\n    'PNAConv',\n    'ClusterGCNConv',\n    'GENConv',\n    'GCN2Conv',\n    'PANConv',\n    'WLConv',\n    'WLConvContinuous',\n    'FiLMConv',\n    'SuperGATConv',\n    'FAConv',\n    'EGConv',\n    'PDNConv',\n    'GeneralConv',\n    'HGTConv',\n    'HEATConv',\n    'HeteroConv',\n    'HANConv',\n    'LGConv',\n    'PointGNNConv',\n    'GPSConv',\n    'AntiSymmetricConv',\n    'DirGNNConv',\n    'MixHopConv',\n]\n\nclasses = __all__\n\nECConv = NNConv\nPointConv = PointNetConv\n\n# Path: torch_geometric/explain/algorithm/attention_explainer.py\nimport logging\nfrom typing import List, Optional, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.explain import Explanation\nfrom torch_geometric.explain.algorithm import ExplainerAlgorithm\nfrom torch_geometric.explain.config import ExplanationType, ModelTaskLevel\nfrom torch_geometric.nn.conv.message_passing import MessagePassing\n\n\nclass AttentionExplainer(ExplainerAlgorithm):\n    r\"\"\"An explainer that uses the attention coefficients produced by an\n    attention-based GNN (*e.g.*,\n    :class:`~torch_geometric.nn.conv.GATConv`,\n    :class:`~torch_geometric.nn.conv.GATv2Conv`, or\n    :class:`~torch_geometric.nn.conv.TransformerConv`) as edge explanation.\n    Attention scores across layers and heads will be aggregated according to\n    the :obj:`reduce` argument.\n\n    Args:\n        reduce (str, optional): The method to reduce the attention scores\n            across layers and heads. (default: :obj:`\"max\"`)\n    \"\"\"\n    def __init__(self, reduce: str = 'max'):\n        super().__init__()\n        self.reduce = reduce\n\n    def forward(\n        self,\n        model: torch.nn.Module,\n        x: Tensor,\n        edge_index: Tensor,\n        *,\n        target: Tensor,\n        index: Optional[Union[int, Tensor]] = None,\n        **kwargs,\n    ) -> Explanation:\n        if isinstance(x, dict) or isinstance(edge_index, dict):\n            raise ValueError(f\"Heterogeneous graphs not yet supported in \"\n                             f\"'{self.__class__.__name__}'\")\n\n        hard_edge_mask = None\n        if self.model_config.task_level == ModelTaskLevel.node:\n            # We need to compute the hard edge mask to properly clean up edge\n            # attributions not involved during message passing:\n            _, hard_edge_mask = self._get_hard_masks(model, index, edge_index,\n                                                     num_nodes=x.size(0))\n\n        alphas: List[Tensor] = []\n\n        def hook(module, msg_kwargs, out):\n            if 'alpha' in msg_kwargs[0]:\n                alphas.append(msg_kwargs[0]['alpha'].detach())\n            elif getattr(module, '_alpha', None) is not None:\n                alphas.append(module._alpha.detach())\n\n        hook_handles = []\n        for module in model.modules():  # Register message forward hooks:\n            if (isinstance(module, MessagePassing)\n                    and module.explain is not False):\n                hook_handles.append(module.register_message_forward_hook(hook))\n\n        model(x, edge_index, **kwargs)\n\n        for handle in hook_handles:  # Remove hooks:\n            handle.remove()\n\n        if len(alphas) == 0:\n            raise ValueError(\"Could not collect any attention coefficients. \"\n                             \"Please ensure that your model is using \"\n                             \"attention-based GNN layers.\")\n\n        for i, alpha in enumerate(alphas):\n            alpha = alpha[:edge_index.size(1)]  # Respect potential self-loops.\n            if alpha.dim() == 2:\n                alpha = getattr(torch, self.reduce)(alpha, dim=-1)\n                if isinstance(alpha, tuple):  # Respect `torch.max`:\n                    alpha = alpha[0]\n            elif alpha.dim() > 2:\n                raise ValueError(f\"Can not reduce attention coefficients of \"\n                                 f\"shape {list(alpha.size())}\")\n            alphas[i] = alpha\n\n        if len(alphas) > 1:\n            alpha = torch.stack(alphas, dim=-1)\n            alpha = getattr(torch, self.reduce)(alpha, dim=-1)\n            if isinstance(alpha, tuple):  # Respect `torch.max`:\n                alpha = alpha[0]\n        else:\n            alpha = alphas[0]\n\n        alpha = self._post_process_mask(alpha, hard_edge_mask,\n                                        apply_sigmoid=False)\n\n        return Explanation(edge_mask=alpha)\n\n    def supports(self) -> bool:\n        explanation_type = self.explainer_config.explanation_type\n        if explanation_type != ExplanationType.model:\n            logging.error(f\"'{self.__class__.__name__}' only supports \"\n                          f\"model explanations \"\n                          f\"got (`explanation_type={explanation_type.value}`)\")\n            return False\n\n        node_mask_type = self.explainer_config.node_mask_type\n        if node_mask_type is not None:\n            logging.error(f\"'{self.__class__.__name__}' does not support \"\n                          f\"explaining input node features \"\n                          f\"got (`node_mask_type={node_mask_type.value}`)\")\n            return False\n\n        return True\n\n# Path: torch_geometric/explain/algorithm/base.py\nfrom abc import abstractmethod\nfrom typing import Dict, Optional, Tuple, Union\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import Tensor\n\nfrom torch_geometric.explain import Explanation, HeteroExplanation\nfrom torch_geometric.explain.config import (\n    ExplainerConfig,\n    ModelConfig,\n    ModelReturnType,\n)\nfrom torch_geometric.nn import MessagePassing\nfrom torch_geometric.typing import EdgeType, NodeType\nfrom torch_geometric.utils import k_hop_subgraph\n\n\nclass ExplainerAlgorithm(torch.nn.Module):\n    r\"\"\"An abstract base class for implementing explainer algorithms.\"\"\"\n    @abstractmethod\n    def forward(\n        self,\n        model: torch.nn.Module,\n        x: Union[Tensor, Dict[NodeType, Tensor]],\n        edge_index: Union[Tensor, Dict[EdgeType, Tensor]],\n        *,\n        target: Tensor,\n        index: Optional[Union[int, Tensor]] = None,\n        **kwargs,\n    ) -> Union[Explanation, HeteroExplanation]:\n        r\"\"\"Computes the explanation.\n\n        Args:\n            model (torch.nn.Module): The model to explain.\n            x (Union[torch.Tensor, Dict[NodeType, torch.Tensor]]): The input\n                node features of a homogeneous or heterogeneous graph.\n            edge_index (Union[torch.Tensor, Dict[NodeType, torch.Tensor]]): The\n                input edge indices of a homogeneous or heterogeneous graph.\n            target (torch.Tensor): The target of the model.\n            index (Union[int, Tensor], optional): The index of the model\n                output to explain. Can be a single index or a tensor of\n                indices. (default: :obj:`None`)\n            **kwargs (optional): Additional keyword arguments passed to\n                :obj:`model`.\n        \"\"\"\n\n    @abstractmethod\n    def supports(self) -> bool:\n        r\"\"\"Checks if the explainer supports the user-defined settings provided\n        in :obj:`self.explainer_config`, :obj:`self.model_config`.\n        \"\"\"\n        pass\n\n    ###########################################################################\n...\n", "needle_token_start": 2380, "needle_token_end": 2454, "code_context_ntokens": 16143, "output": ["```python\ndef positive_sampling(self, edge_index: Tensor) -> Tensor:\n    pos_edge_index, _ = dropout_edge(edge_index,\n                                     p=1. - self.edge_sample_ratio,\n                                     training=self.training)\n    return pos_edge_index\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "read_from_memlab", "language": "python", "path": "torch_geometric/profile/profile.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To extract and convert specific memory usage statistics from profiling data into a more readable format, specifically in megabytes.\n2. **Input**: A profiler object that contains raw memory profiling records.\n3. **Output**: A list of floats, each representing a memory statistic (peak allocated, reserved, and active bytes) converted from bytes to megabytes.\n4. **Procedure**: The function retrieves raw line records from the profiler, specifies the memory statistics to track, and then processes these records to extract the maximum values of the specified statistics. These byte values are then converted to megabytes and returned as a list.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/profile/utils.py\nimport gc\nimport os\nimport os.path as osp\nimport random\nimport subprocess as sp\nimport sys\nimport warnings\nfrom collections.abc import Mapping, Sequence\nfrom typing import Any, Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data.data import BaseData\nfrom torch_geometric.typing import SparseTensor\n\n\ndef count_parameters(model: torch.nn.Module) -> int:\n    r\"\"\"Given a :class:`torch.nn.Module`, count its trainable parameters.\n\n    Args:\n        model (torch.nn.Model): The model.\n    \"\"\"\n    return sum([p.numel() for p in model.parameters() if p.requires_grad])\n\n\ndef get_model_size(model: torch.nn.Module) -> int:\n    r\"\"\"Given a :class:`torch.nn.Module`, get its actual disk size in bytes.\n\n    Args:\n        model (torch model): The model.\n    \"\"\"\n    path = f'{random.randrange(sys.maxsize)}.pt'\n    torch.save(model.state_dict(), path)\n    model_size = osp.getsize(path)\n    os.remove(path)\n    return model_size\n\n\ndef get_data_size(data: BaseData) -> int:\n    r\"\"\"Given a :class:`torch_geometric.data.Data` object, get its theoretical\n    memory usage in bytes.\n\n    Args:\n        data (torch_geometric.data.Data or torch_geometric.data.HeteroData):\n            The :class:`~torch_geometric.data.Data` or\n            :class:`~torch_geometric.data.HeteroData` graph object.\n    \"\"\"\n    data_ptrs = set()\n\n    def _get_size(obj: Any) -> int:\n        if isinstance(obj, Tensor):\n            if obj.data_ptr() in data_ptrs:\n                return 0\n            data_ptrs.add(obj.data_ptr())\n            return obj.numel() * obj.element_size()\n        elif isinstance(obj, SparseTensor):\n            return _get_size(obj.csr())\n        elif isinstance(obj, Sequence) and not isinstance(obj, str):\n            return sum([_get_size(x) for x in obj])\n        elif isinstance(obj, Mapping):\n            return sum([_get_size(x) for x in obj.values()])\n        else:\n            return 0\n\n    return sum([_get_size(store) for store in data.stores])\n\n\ndef get_cpu_memory_from_gc() -> int:\n    r\"\"\"Returns the used CPU memory in bytes, as reported by the\n    :python:`Python` garbage collector.\n    \"\"\"\n    warnings.filterwarnings('ignore', '.*torch.distributed.reduce_op.*')\n\n    mem = 0\n    for obj in gc.get_objects():\n        try:\n            if isinstance(obj, Tensor) and not obj.is_cuda:\n                mem += obj.numel() * obj.element_size()\n        except Exception:\n            pass\n    return mem\n\n\ndef get_gpu_memory_from_gc(device: int = 0) -> int:  # pragma: no cover\n    r\"\"\"Returns the used GPU memory in bytes, as reported by the\n    :python:`Python` garbage collector.\n\n    Args:\n        device (int, optional): The GPU device identifier. (default: :obj:`1`)\n    \"\"\"\n    warnings.filterwarnings('ignore', '.*torch.distributed.reduce_op.*')\n\n    mem = 0\n    for obj in gc.get_objects():\n        try:\n            if isinstance(obj, Tensor) and obj.get_device() == device:\n                mem += obj.numel() * obj.element_size()\n        except Exception:\n            pass\n    return mem\n\n\ndef get_gpu_memory_from_nvidia_smi(  # pragma: no cover\n    device: int = 0,\n    digits: int = 2,\n) -> Tuple[float, float]:\n    r\"\"\"Returns the free and used GPU memory in megabytes, as reported by\n    :obj:`nivdia-smi`.\n\n    .. note::\n\n        :obj:`nvidia-smi` will generally overestimate the amount of memory used\n        by the actual program, see `here <https://pytorch.org/docs/stable/\n        notes/faq.html#my-gpu-memory-isn-t-freed-properly>`__.\n\n    Args:\n        device (int, optional): The GPU device identifier. (default: :obj:`1`)\n        digits (int): The number of decimals to use for megabytes.\n            (default: :obj:`2`)\n    \"\"\"\n    CMD = 'nvidia-smi --query-gpu=memory.free --format=csv'\n    free_out = sp.check_output(CMD.split()).decode('utf-8').split('\\n')[1:-1]\n\n    CMD = 'nvidia-smi --query-gpu=memory.used --format=csv'\n    used_out = sp.check_output(CMD.split()).decode('utf-8').split('\\n')[1:-1]\n\n    if device < 0 or device >= len(free_out):\n        raise AttributeError(\n            f'GPU {device} not available (found {len(free_out)} GPUs)')\n\n    free_mem = medibyte_to_megabyte(int(free_out[device].split()[0]), digits)\n    used_mem = medibyte_to_megabyte(int(used_out[device].split()[0]), digits)\n\n    return free_mem, used_mem\n\n\ndef get_gpu_memory_from_ipex(\n        device: int = 0,\n        digits=2) -> Tuple[float, float, float]:  # pragma: no cover\n    r\"\"\"Returns the XPU memory statistics.\n\n...\n# Path: torch_geometric/profile/profile.py\nimport os\nimport pathlib\nimport time\nfrom contextlib import ContextDecorator, contextmanager\nfrom dataclasses import dataclass\nfrom typing import Any, List, Tuple, Union\n\nimport torch\nfrom torch.autograd.profiler import EventList\nfrom torch.profiler import ProfilerActivity, profile\n\nfrom torch_geometric.profile.utils import (\n    byte_to_megabyte,\n    get_gpu_memory_from_ipex,\n    get_gpu_memory_from_nvidia_smi,\n)\n\n\n@dataclass\nclass GPUStats:\n    time: float\n    max_allocated_gpu: float\n    max_reserved_gpu: float\n    max_active_gpu: float\n\n\n@dataclass\nclass CUDAStats(GPUStats):\n    nvidia_smi_free_cuda: float\n    nvidia_smi_used_cuda: float\n\n\n@dataclass\nclass GPUStatsSummary:\n    time_mean: float\n    time_std: float\n    max_allocated_gpu: float\n    max_reserved_gpu: float\n    max_active_gpu: float\n\n\n@dataclass\nclass CUDAStatsSummary(GPUStatsSummary):\n    min_nvidia_smi_free_cuda: float\n    max_nvidia_smi_used_cuda: float\n\n\ndef profileit(device: str):  # pragma: no cover\n    r\"\"\"A decorator to facilitate profiling a function, *e.g.*, obtaining\n    training runtime and memory statistics of a specific model on a specific\n    dataset.\n    Returns a :obj:`GPUStats` if :obj:`device` is :obj:`xpu` or extended\n    object :obj:`CUDAStats`, if :obj:`device` is :obj:`cuda`.\n\n    Args:\n        device (str): Target device for profiling. Options are:\n            :obj:`cuda` and obj:`xpu`.\n\n    .. code-block:: python\n\n        @profileit(\"cuda\")\n        def train(model, optimizer, x, edge_index, y):\n            optimizer.zero_grad()\n            out = model(x, edge_index)\n            loss = criterion(out, y)\n            loss.backward()\n            optimizer.step()\n            return float(loss)\n\n        loss, stats = train(model, x, edge_index, y)\n    \"\"\"\n    def decorator(func):\n        def wrapper(\n                *args, **kwargs\n        ) -> Union[Tuple[Any, GPUStats], Tuple[Any, CUDAStats]]:\n            model = args[0]\n            if not isinstance(model, torch.nn.Module):\n                raise AttributeError(\n                    'First argument for profiling needs to be torch.nn.Module')\n            if device not in ['cuda', 'xpu']:\n                raise AttributeError(\n                    \"The profiling decorator supports only CUDA and \"\n                    \"XPU devices\")\n\n            device_id = None\n            for arg in list(args) + list(kwargs.values()):\n                if isinstance(arg, torch.Tensor):\n                    device_id = arg.get_device()\n                    break\n            if device_id is None:\n                raise AttributeError(\n                    \"Could not infer GPU device from the args in the \"\n                    \"function being profiled\")\n            if device_id == -1:\n                raise RuntimeError(\n                    \"The profiling decorator does not support profiling \"\n                    \"on non GPU devices\")\n\n            is_cuda = device == 'cuda'\n            torch_gpu = torch.cuda if is_cuda else torch.xpu\n\n            # `pytorch_memlab` supports only CUDA devices\n            if is_cuda:\n                from pytorch_memlab import LineProfiler\n\n                # Init `pytorch_memlab` for analyzing the model forward pass:\n                line_profiler = LineProfiler(target_gpu=device_id)\n                line_profiler.enable()\n                line_profiler.add_function(args[0].forward)\n\n            start = torch_gpu.Event(enable_timing=True)\n            end = torch_gpu.Event(enable_timing=True)\n            start.record()\n\n            out = func(*args, **kwargs)\n\n            end.record()\n            torch_gpu.synchronize()\n            time = start.elapsed_time(end) / 1000\n\n            if is_cuda:\n                # Get the global memory statistics collected\n                # by `pytorch_memlab`:\n                memlab = read_from_memlab(line_profiler)\n                max_allocated, max_reserved, max_active = memlab\n                line_profiler.disable()\n\n                # Get additional information from `nvidia-smi`:\n                free_cuda, used_cuda = get_gpu_memory_from_nvidia_smi(\n                    device=device_id)\n\n                stats = CUDAStats(time, max_allocated, max_reserved,\n                                  max_active, free_cuda, used_cuda)\n                return out, stats\n            else:\n                stats = GPUStats(time, *get_gpu_memory_from_ipex(device_id))\n                return out, stats\n\n        return wrapper\n\n    return decorator\n\n\nclass timeit(ContextDecorator):\n    r\"\"\"A context decorator to facilitate timing a function, *e.g.*, obtaining\n    the runtime of a specific model on a specific dataset.\n\n    .. code-block:: python\n\n        @torch.no_grad()\n        def test(model, x, edge_index):\n            return model(x, edge_index)\n\n        with timeit() as t:\n            z = test(model, x, edge_index)\n        time = t.duration\n\n    Args:\n        log (bool, optional): If set to :obj:`False`, will not log any runtime\n            to the console. (default: :obj:`True`)\n        avg_time_divisor (int, optional): If set to a value greater than\n            :obj:`1`, will divide the total time by this value. Useful for\n            calculating the average of runtimes within a for-loop.\n            (default: :obj:`0`)\n    \"\"\"\n    def __init__(self, log: bool = True, avg_time_divisor: int = 0):\n        self.log = log\n        self.avg_time_divisor = avg_time_divisor\n\n    def __enter__(self):\n        if torch.cuda.is_available():\n            torch.cuda.synchronize()\n        self.t_start = time.time()\n        return self\n\n    def __exit__(self, *args):\n        if torch.cuda.is_available():\n            torch.cuda.synchronize()\n        self.t_end = time.time()\n        self.duration = self.t_end - self.t_start\n        if self.avg_time_divisor > 1:\n            self.duration = self.duration / self.avg_time_divisor\n        if self.log:  # pragma: no cover\n            print(f'Time: {self.duration:.8f}s', flush=True)\n\n    def reset(self):\n        r\"\"\"Prints the duration and resets current timer.\"\"\"\n        if self.t_start is None:\n            raise RuntimeError(\"Timer wasn't started.\")\n        else:\n            self.__exit__()\n            self.__enter__()\n\n\ndef get_stats_summary(\n    stats_list: Union[List[GPUStats], List[CUDAStats]]\n) -> Union[GPUStatsSummary, CUDAStatsSummary]:  # pragma: no cover\n    r\"\"\"Creates a summary of collected runtime and memory statistics.\n    Returns a :obj:`GPUStatsSummary` if list of :obj:`GPUStats` was passed,\n    otherwise (list of :obj:`CUDAStats` was passed),\n    returns a :obj:`CUDAStatsSummary`.\n\n    Args:\n        stats_list (Union[List[GPUStats], List[CUDAStats]]): A list of\n            :obj:`GPUStats` or :obj:`CUDAStats` objects, as returned by\n            :meth:`~torch_geometric.profile.profileit`.\n    \"\"\"\n    # calculate common statistics\n    kwargs = dict(\n        time_mean=float(torch.tensor([s.time for s in stats_list]).mean()),\n        time_std=float(torch.tensor([s.time for s in stats_list]).std()),\n        max_allocated_gpu=max([s.max_allocated_gpu for s in stats_list]),\n        max_reserved_gpu=max([s.max_reserved_gpu for s in stats_list]),\n        max_active_gpu=max([s.max_active_gpu for s in stats_list]))\n\n    if all(isinstance(s, CUDAStats) for s in stats_list):\n        return CUDAStatsSummary(\n            **kwargs,\n            min_nvidia_smi_free_cuda=min(\n                [s.nvidia_smi_free_cuda for s in stats_list]),\n            max_nvidia_smi_used_cuda=max(\n                [s.nvidia_smi_used_cuda for s in stats_list]),\n        )\n    else:\n        return GPUStatsSummary(**kwargs)\n\n\n###############################################################################\n\n\n\ndef read_from_memlab(line_profiler: Any) -> List[float]:  # pragma: no cover\n    from pytorch_memlab.line_profiler.line_records import LineRecords\n\n    # See: https://pytorch.org/docs/stable/cuda.html#torch.cuda.memory_stats\n\n    track_stats = [  # Different statistic can be collected as needed.\n        'allocated_bytes.all.peak',\n        'reserved_bytes.all.peak',\n        'active_bytes.all.peak',\n    ]\n\n    records = LineRecords(line_profiler._raw_line_records,\n                          line_profiler._code_infos)\n    stats = records.display(None, track_stats)._line_records\n    return [byte_to_megabyte(x) for x in stats.values.max(axis=0).tolist()]\n\n\ndef trace_handler(p):\n    print_time_total(p)\n    profile_dir = str(pathlib.Path.cwd()) + '/'\n    timeline_file = profile_dir + 'timeline' + '.json'\n    p.export_chrome_trace(timeline_file)\n\n\ndef print_time_total(p):\n    if torch.cuda.is_available():\n        profile_sort = 'self_cuda_time_total'\n    else:\n        profile_sort = 'self_cpu_time_total'\n    output = p.key_averages().table(sort_by=profile_sort)\n    print(output)\n\n\ndef rename_profile_file(*args):\n    profile_dir = str(pathlib.Path.cwd()) + '/'\n    timeline_file = profile_dir + 'profile'\n    for arg in args:\n        timeline_file += '-' + arg\n    timeline_file += '.json'\n    os.rename('timeline.json', timeline_file)\n\n\n@contextmanager\ndef torch_profile(export_chrome_trace=True, csv_data=None, write_csv=None):\n    use_cuda = torch.cuda.is_available()\n\n    activities = [ProfilerActivity.CPU]\n    if use_cuda:\n        activities.append(ProfilerActivity.CUDA)\n\n    if export_chrome_trace:\n        p_trace_handler = trace_handler\n    else:\n        p_trace_handler = print_time_total\n\n    p = profile(activities=activities, on_trace_ready=p_trace_handler)\n\n    with p:\n        yield\n        p.step()\n\n    if csv_data is not None and write_csv == 'prof':\n        if use_cuda:\n            profile_sort = 'self_cuda_time_total'\n        else:\n            profile_sort = 'self_cpu_time_total'\n        events = EventList(\n            sorted(\n                p.key_averages(),\n                key=lambda evt: getattr(evt, profile_sort),\n                reverse=True,\n            ), use_cuda=use_cuda)\n\n        save_profile_data(csv_data, events, use_cuda)\n\n\n@contextmanager\ndef xpu_profile(export_chrome_trace=True):\n    with torch.autograd.profiler_legacy.profile(use_xpu=True) as profile:\n        yield\n    print(profile.key_averages().table(sort_by='self_xpu_time_total'))\n    if export_chrome_trace:\n        profile.export_chrome_trace('timeline.json')\n\n\ndef format_prof_time(time):\n    # Profile time is in micro seconds, so format it appropriately:\n    return round(time / 1e6, 3)\n\n\ndef save_profile_data(csv_data, events, use_cuda):\n    sum_self_cpu_time_total = sum(\n        [event.self_cpu_time_total for event in events])\n    sum_cpu_time_total = sum([event.self_cpu_time_total for event in events])\n    sum_self_cuda_time_total = sum(\n        [event.self_cuda_time_total for event in events]) if use_cuda else 0\n\n    for e in events[:5]:  # Save top 5 most time consuming operations:\n        csv_data['NAME'].append(e.key)\n        csv_data['SELF CPU %'].append(\n            round(e.self_cpu_time_total * 100.0 / sum_self_cpu_time_total, 3))\n        csv_data['SELF CPU'].append(format_prof_time(e.self_cpu_time_total))\n        csv_data['CPU TOTAL %'].append(\n            round(e.cpu_time_total * 100.0 / sum_cpu_time_total, 3))\n        csv_data['CPU TOTAL'].append(format_prof_time(e.cpu_time_total))\n        csv_data['CPU TIME AVG'].append(format_prof_time(e.cpu_time_total))\n        if use_cuda:\n            csv_data['SELF CUDA %'].append(e.self_cuda_time_total * 100.0 /\n                                           sum_self_cuda_time_total)\n            csv_data['SELF CUDA'].append(\n                format_prof_time(e.self_cuda_time_total))\n            csv_data['CUDA TOTAL'].append(format_prof_time(e.cpu_time_total))\n            csv_data['CUDA TIME AVG'].append(format_prof_time(\n                e.cpu_time_total))\n        csv_data['# OF CALLS'].append(e.count)\n\n# Path: torch_geometric/profile/__init__.py\nr\"\"\"GNN profiling package.\"\"\"\n\nfrom .benchmark import benchmark\nfrom .profile import (\n    get_stats_summary,\n    print_time_total,\n    profileit,\n    rename_profile_file,\n    timeit,\n    torch_profile,\n    trace_handler,\n    xpu_profile,\n)\nfrom .utils import (\n    count_parameters,\n    get_cpu_memory_from_gc,\n    get_data_size,\n    get_gpu_memory_from_gc,\n    get_gpu_memory_from_ipex,\n    get_gpu_memory_from_nvidia_smi,\n    get_model_size,\n)\n\n__all__ = [\n    'profileit',\n    'timeit',\n    'get_stats_summary',\n    'trace_handler',\n    'print_time_total',\n    'rename_profile_file',\n    'torch_profile',\n    'xpu_profile',\n    'count_parameters',\n    'get_model_size',\n    'get_data_size',\n    'get_cpu_memory_from_gc',\n    'get_gpu_memory_from_gc',\n    'get_gpu_memory_from_nvidia_smi',\n    'get_gpu_memory_from_ipex',\n    'benchmark',\n]\n\nclasses = __all__\n\n# Path: torch_geometric/seed.py\nimport random\n\nimport numpy as np\nimport torch\n\n\ndef seed_everything(seed: int) -> None:\n    r\"\"\"Sets the seed for generating random numbers in :pytorch:`PyTorch`,\n    :obj:`numpy` and :python:`Python`.\n\n    Args:\n        seed (int): The desired seed.\n    \"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\n# Path: torch_geometric/transforms/add_metapaths.py\nimport warnings\nfrom typing import List, Optional, Tuple, Union, cast\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric import EdgeIndex\nfrom torch_geometric.data import HeteroData\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\nfrom torch_geometric.typing import EdgeType\nfrom torch_geometric.utils import coalesce, degree\n\n\n@functional_transform('add_metapaths')\nclass AddMetaPaths(BaseTransform):\n    r\"\"\"Adds additional edge types to a\n    :class:`~torch_geometric.data.HeteroData` object between the source node\n    type and the destination node type of a given :obj:`metapath`, as described\n    in the `\"Heterogenous Graph Attention Networks\"\n    <https://arxiv.org/abs/1903.07293>`_ paper\n    (functional name: :obj:`add_metapaths`).\n\n    Meta-path based neighbors can exploit different aspects of structure\n    information in heterogeneous graphs.\n    Formally, a metapath is a path of the form\n\n    .. math::\n\n        \\mathcal{V}_1 \\xrightarrow{R_1} \\mathcal{V}_2 \\xrightarrow{R_2} \\ldots\n        \\xrightarrow{R_{\\ell-1}} \\mathcal{V}_{\\ell}\n\n    in which :math:`\\mathcal{V}_i` represents node types, and :math:`R_j`\n    represents the edge type connecting two node types.\n    The added edge type is given by the sequential multiplication  of\n    adjacency matrices along the metapath, and is added to the\n    :class:`~torch_geometric.data.HeteroData` object as edge type\n    :obj:`(src_node_type, \"metapath_*\", dst_node_type)`, where\n    :obj:`src_node_type` and :obj:`dst_node_type` denote :math:`\\mathcal{V}_1`\n    and :math:`\\mathcal{V}_{\\ell}`, repectively.\n\n    In addition, a :obj:`metapath_dict` object is added to the\n    :class:`~torch_geometric.data.HeteroData` object which maps the\n    metapath-based edge type to its original metapath.\n\n    .. code-block:: python\n\n        from torch_geometric.datasets import DBLP\n        from torch_geometric.data import HeteroData\n        from torch_geometric.transforms import AddMetaPaths\n\n        data = DBLP(root)[0]\n        # 4 node types: \"paper\", \"author\", \"conference\", and \"term\"\n        # 6 edge types: (\"paper\",\"author\"), (\"author\", \"paper\"),\n        #               (\"paper, \"term\"), (\"paper\", \"conference\"),\n        #               (\"term, \"paper\"), (\"conference\", \"paper\")\n\n        # Add two metapaths:\n        # 1. From \"paper\" to \"paper\" through \"conference\"\n        # 2. From \"author\" to \"conference\" through \"paper\"\n        metapaths = [[(\"paper\", \"conference\"), (\"conference\", \"paper\")],\n                     [(\"author\", \"paper\"), (\"paper\", \"conference\")]]\n        data = AddMetaPaths(metapaths)(data)\n\n        print(data.edge_types)\n        >>> [(\"author\", \"to\", \"paper\"), (\"paper\", \"to\", \"author\"),\n             (\"paper\", \"to\", \"term\"), (\"paper\", \"to\", \"conference\"),\n             (\"term\", \"to\", \"paper\"), (\"conference\", \"to\", \"paper\"),\n             (\"paper\", \"metapath_0\", \"paper\"),\n             (\"author\", \"metapath_1\", \"conference\")]\n\n        print(data.metapath_dict)\n        >>> {(\"paper\", \"metapath_0\", \"paper\"): [(\"paper\", \"conference\"),\n                                                (\"conference\", \"paper\")],\n             (\"author\", \"metapath_1\", \"conference\"): [(\"author\", \"paper\"),\n                                                      (\"paper\", \"conference\")]}\n\n    Args:\n        metapaths (List[List[Tuple[str, str, str]]]): The metapaths described\n            by a list of lists of\n            :obj:`(src_node_type, rel_type, dst_node_type)` tuples.\n        drop_orig_edge_types (bool, optional): If set to :obj:`True`, existing\n            edge types will be dropped. (default: :obj:`False`)\n        keep_same_node_type (bool, optional): If set to :obj:`True`, existing\n            edge types between the same node type are not dropped even in case\n            :obj:`drop_orig_edge_types` is set to :obj:`True`.\n            (default: :obj:`False`)\n        drop_unconnected_node_types (bool, optional): If set to :obj:`True`,\n            will drop node types not connected by any edge type.\n            (default: :obj:`False`)\n        max_sample (int, optional): If set, will sample at maximum\n            :obj:`max_sample` neighbors within metapaths. Useful in order to\n            tackle very dense metapath edges. (default: :obj:`None`)\n        weighted (bool, optional): If set to :obj:`True`, computes weights for\n            each metapath edge and stores them in :obj:`edge_weight`. The\n            weight of each metapath edge is computed as the number of metapaths\n            from the start to the end of the metapath edge.\n            (default :obj:`False`)\n    \"\"\"\n    def __init__(\n        self,\n        metapaths: List[List[EdgeType]],\n        drop_orig_edge_types: bool = False,\n        keep_same_node_type: bool = False,\n        drop_unconnected_node_types: bool = False,\n        max_sample: Optional[int] = None,\n        weighted: bool = False,\n        **kwargs: bool,\n    ) -> None:\n        if 'drop_orig_edges' in kwargs:\n            warnings.warn(\"'drop_orig_edges' is dprecated. Use \"\n                          \"'drop_orig_edge_types' instead\")\n            drop_orig_edge_types = kwargs['drop_orig_edges']\n\n        if 'drop_unconnected_nodes' in kwargs:\n            warnings.warn(\"'drop_unconnected_nodes' is dprecated. Use \"\n                          \"'drop_unconnected_node_types' instead\")\n            drop_unconnected_node_types = kwargs['drop_unconnected_nodes']\n\n        for path in metapaths:\n            assert len(path) >= 2, f\"Invalid metapath '{path}'\"\n            assert all([\n                j[-1] == path[i + 1][0] for i, j in enumerate(path[:-1])\n            ]), f\"Invalid sequence of node types in '{path}'\"\n\n        self.metapaths = metapaths\n        self.drop_orig_edge_types = drop_orig_edge_types\n        self.keep_same_node_type = keep_same_node_type\n        self.drop_unconnected_node_types = drop_unconnected_node_types\n        self.max_sample = max_sample\n        self.weighted = weighted\n\n    def forward(self, data: HeteroData) -> HeteroData:\n        edge_types = data.edge_types  # Save original edge types.\n        data.metapath_dict = {}\n\n        for j, metapath in enumerate(self.metapaths):\n            for edge_type in metapath:\n                assert data._to_canonical(edge_type) in edge_types\n\n            edge_type = metapath[0]\n            edge_index, edge_weight = self._edge_index(data, edge_type)\n\n            if self.max_sample is not None:\n                edge_index, edge_weight = self._sample(edge_index, edge_weight)\n\n            for i, edge_type in enumerate(metapath[1:]):\n                edge_index2, edge_weight2 = self._edge_index(data, edge_type)\n\n                edge_index, edge_weight = edge_index.matmul(\n                    edge_index2, edge_weight, edge_weight2)\n\n                if not self.weighted:\n                    edge_weight = None\n\n                if self.max_sample is not None:\n                    edge_index, edge_weight = self._sample(\n                        edge_index, edge_weight)\n\n            new_edge_type = (metapath[0][0], f'metapath_{j}', metapath[-1][-1])\n            data[new_edge_type].edge_index = edge_index\n            if self.weighted:\n                data[new_edge_type].edge_weight = edge_weight\n            data.metapath_dict[new_edge_type] = metapath\n\n        postprocess(data, edge_types, self.drop_orig_edge_types,\n                    self.keep_same_node_type, self.drop_unconnected_node_types)\n\n        return data\n\n    def _edge_index(\n        self,\n        data: HeteroData,\n        edge_type: EdgeType,\n    ) -> Tuple[EdgeIndex, Optional[Tensor]]:\n\n        edge_index = EdgeIndex(\n            data[edge_type].edge_index,\n            sparse_size=data[edge_type].size(),\n        )\n        edge_index, perm = edge_index.sort_by('row')\n\n        if not self.weighted:\n            return edge_index, None\n\n        edge_weight = data[edge_type].get('edge_weight')\n        if edge_weight is not None:\n            assert edge_weight.dim() == 1\n            edge_weight = edge_weight[perm]\n\n        return edge_index, edge_weight\n\n    def _sample(\n        self,\n        edge_index: EdgeIndex,\n        edge_weight: Optional[Tensor],\n    ) -> Tuple[EdgeIndex, Optional[Tensor]]:\n\n        deg = degree(edge_index[0], num_nodes=edge_index.get_sparse_size(0))\n        prob = (self.max_sample * (1. / deg))[edge_index[0]]\n        mask = torch.rand_like(prob) < prob\n\n        edge_index = cast(EdgeIndex, edge_index[:, mask])\n        assert isinstance(edge_index, EdgeIndex)\n        if edge_weight is not None:\n            edge_weight = edge_weight[mask]\n\n        return edge_index, edge_weight\n\n\n@functional_transform('add_random_metapaths')\nclass AddRandomMetaPaths(BaseTransform):\n    r\"\"\"Adds additional edge types similar to :class:`AddMetaPaths`.\n    The key difference is that the added edge type is given by\n    multiple random walks along the metapath.\n    One might want to increase the number of random walks\n    via :obj:`walks_per_node` to achieve competitive performance with\n    :class:`AddMetaPaths`.\n\n    Args:\n        metapaths (List[List[Tuple[str, str, str]]]): The metapaths described\n            by a list of lists of\n            :obj:`(src_node_type, rel_type, dst_node_type)` tuples.\n        drop_orig_edge_types (bool, optional): If set to :obj:`True`, existing\n            edge types will be dropped. (default: :obj:`False`)\n        keep_same_node_type (bool, optional): If set to :obj:`True`, existing\n            edge types between the same node type are not dropped even in case\n            :obj:`drop_orig_edge_types` is set to :obj:`True`.\n            (default: :obj:`False`)\n        drop_unconnected_node_types (bool, optional): If set to :obj:`True`,\n            will drop node types not connected by any edge type.\n            (default: :obj:`False`)\n        walks_per_node (int, List[int], optional): The number of random walks\n            for each starting node in a metapth. (default: :obj:`1`)\n        sample_ratio (float, optional): The ratio of source nodes to start\n            random walks from. (default: :obj:`1.0`)\n    \"\"\"\n    def __init__(\n        self,\n        metapaths: List[List[EdgeType]],\n        drop_orig_edge_types: bool = False,\n        keep_same_node_type: bool = False,\n        drop_unconnected_node_types: bool = False,\n        walks_per_node: Union[int, List[int]] = 1,\n        sample_ratio: float = 1.0,\n    ):\n\n        for path in metapaths:\n            assert len(path) >= 2, f\"Invalid metapath '{path}'\"\n            assert all([\n                j[-1] == path[i + 1][0] for i, j in enumerate(path[:-1])\n            ]), f\"Invalid sequence of node types in '{path}'\"\n\n        self.metapaths = metapaths\n        self.drop_orig_edge_types = drop_orig_edge_types\n        self.keep_same_node_type = keep_same_node_type\n        self.drop_unconnected_node_types = drop_unconnected_node_types\n        self.sample_ratio = sample_ratio\n        if isinstance(walks_per_node, int):\n            walks_per_node = [walks_per_node] * len(metapaths)\n        assert len(walks_per_node) == len(metapaths)\n        self.walks_per_node = walks_per_node\n\n    def forward(self, data: HeteroData) -> HeteroData:\n        edge_types = data.edge_types  # save original edge types\n        data.metapath_dict = {}\n\n        for j, metapath in enumerate(self.metapaths):\n            for edge_type in metapath:\n                assert data._to_canonical(\n                    edge_type) in edge_types, f\"'{edge_type}' not present\"\n\n            src_node = metapath[0][0]\n            num_nodes = data[src_node].num_nodes\n            num_starts = round(num_nodes * self.sample_ratio)\n            row = start = torch.randperm(num_nodes)[:num_starts].repeat(\n                self.walks_per_node[j])\n\n            for i, edge_type in enumerate(metapath):\n                edge_index = EdgeIndex(\n                    data[edge_type].edge_index,\n                    sparse_size=data[edge_type].size(),\n                )\n                col, mask = self.sample(edge_index, start)\n                row, col = row[mask], col[mask]\n                start = col\n\n            new_edge_type = (metapath[0][0], f'metapath_{j}', metapath[-1][-1])\n            data[new_edge_type].edge_index = coalesce(torch.vstack([row, col]))\n            data.metapath_dict[new_edge_type] = metapath\n\n        postprocess(data, edge_types, self.drop_orig_edge_types,\n                    self.keep_same_node_type, self.drop_unconnected_node_types)\n\n        return data\n\n    @staticmethod\n    def sample(edge_index: EdgeIndex, subset: Tensor) -> Tuple[Tensor, Tensor]:\n        \"\"\"Sample neighbors from :obj:`edge_index` for each node in\n        :obj:`subset`.\n        \"\"\"\n        edge_index, _ = edge_index.sort_by('row')\n        rowptr = edge_index.get_indptr()\n        rowcount = rowptr.diff()[subset]\n\n        mask = rowcount > 0\n        offset = torch.zeros_like(subset)\n        offset[mask] = rowptr[subset[mask]]\n\n        rand = torch.rand((rowcount.size(0), 1), device=subset.device)\n        rand.mul_(rowcount.to(rand.dtype).view(-1, 1))\n        rand = rand.to(torch.long)\n        rand.add_(offset.view(-1, 1))\n        col = edge_index[1][rand].squeeze()\n        return col, mask\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}('\n                f'sample_ratio={self.sample_ratio}, '\n                f'walks_per_node={self.walks_per_node})')\n\n\ndef postprocess(\n    data: HeteroData,\n    edge_types: List[EdgeType],\n    drop_orig_edge_types: bool,\n    keep_same_node_type: bool,\n    drop_unconnected_node_types: bool,\n) -> None:\n\n    if drop_orig_edge_types:\n        for i in edge_types:\n            if keep_same_node_type and i[0] == i[-1]:\n                continue\n            else:\n                del data[i]\n\n    # Remove nodes not connected by any edge type:\n    if drop_unconnected_node_types:\n        new_edge_types = data.edge_types\n        node_types = data.node_types\n        connected_nodes = set()\n        for i in new_edge_types:\n            connected_nodes.add(i[0])\n            connected_nodes.add(i[-1])\n        for node in node_types:\n            if node not in connected_nodes:\n                del data[node]\n\n# Path: torch_geometric/transforms/add_positional_encoding.py\nfrom typing import Any, Optional\n\nimport numpy as np\nimport torch\nfrom torch import Tensor\n\nimport torch_geometric.typing\nfrom torch_geometric.data import Data\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\nfrom torch_geometric.utils import (\n    get_laplacian,\n    get_self_loop_attr,\n    is_torch_sparse_tensor,\n    scatter,\n    to_edge_index,\n    to_scipy_sparse_matrix,\n    to_torch_coo_tensor,\n    to_torch_csr_tensor,\n)\n\n\ndef add_node_attr(\n    data: Data,\n    value: Any,\n    attr_name: Optional[str] = None,\n) -> Data:\n    # TODO Move to `BaseTransform`.\n    if attr_name is None:\n        if data.x is not None:\n            x = data.x.view(-1, 1) if data.x.dim() == 1 else data.x\n            data.x = torch.cat([x, value.to(x.device, x.dtype)], dim=-1)\n        else:\n            data.x = value\n    else:\n        data[attr_name] = value\n\n    return data\n\n\n@functional_transform('add_laplacian_eigenvector_pe')\nclass AddLaplacianEigenvectorPE(BaseTransform):\n    r\"\"\"Adds the Laplacian eigenvector positional encoding from the\n    `\"Benchmarking Graph Neural Networks\" <https://arxiv.org/abs/2003.00982>`_\n    paper to the given graph\n    (functional name: :obj:`add_laplacian_eigenvector_pe`).\n\n    Args:\n        k (int): The number of non-trivial eigenvectors to consider.\n        attr_name (str, optional): The attribute name of the data object to add\n            positional encodings to. If set to :obj:`None`, will be\n            concatenated to :obj:`data.x`.\n            (default: :obj:`\"laplacian_eigenvector_pe\"`)\n        is_undirected (bool, optional): If set to :obj:`True`, this transform\n            expects undirected graphs as input, and can hence speed up the\n            computation of eigenvectors. (default: :obj:`False`)\n        **kwargs (optional): Additional arguments of\n            :meth:`scipy.sparse.linalg.eigs` (when :attr:`is_undirected` is\n            :obj:`False`) or :meth:`scipy.sparse.linalg.eigsh` (when\n            :attr:`is_undirected` is :obj:`True`).\n    \"\"\"\n    # Number of nodes from which to use sparse eigenvector computation:\n    SPARSE_THRESHOLD: int = 100\n\n    def __init__(\n        self,\n        k: int,\n        attr_name: Optional[str] = 'laplacian_eigenvector_pe',\n        is_undirected: bool = False,\n        **kwargs: Any,\n    ) -> None:\n        self.k = k\n        self.attr_name = attr_name\n        self.is_undirected = is_undirected\n        self.kwargs = kwargs\n\n    def forward(self, data: Data) -> Data:\n        assert data.edge_index is not None\n        num_nodes = data.num_nodes\n        assert num_nodes is not None\n\n        edge_index, edge_weight = get_laplacian(\n            data.edge_index,\n            data.edge_weight,\n            normalization='sym',\n            num_nodes=num_nodes,\n        )\n\n        L = to_scipy_sparse_matrix(edge_index, edge_weight, num_nodes)\n\n        if num_nodes < self.SPARSE_THRESHOLD:\n            from numpy.linalg import eig, eigh\n            eig_fn = eig if not self.is_undirected else eigh\n\n            eig_vals, eig_vecs = eig_fn(L.todense())  # type: ignore\n        else:\n            from scipy.sparse.linalg import eigs, eigsh\n            eig_fn = eigs if not self.is_undirected else eigsh\n\n            eig_vals, eig_vecs = eig_fn(  # type: ignore\n                L,\n                k=self.k + 1,\n                which='SR' if not self.is_undirected else 'SA',\n                return_eigenvectors=True,\n                **self.kwargs,\n            )\n\n        eig_vecs = np.real(eig_vecs[:, eig_vals.argsort()])\n        pe = torch.from_numpy(eig_vecs[:, 1:self.k + 1])\n        sign = -1 + 2 * torch.randint(0, 2, (self.k, ))\n        pe *= sign\n\n        data = add_node_attr(data, pe, attr_name=self.attr_name)\n        return data\n\n\n@functional_transform('add_random_walk_pe')\nclass AddRandomWalkPE(BaseTransform):\n    r\"\"\"Adds the random walk positional encoding from the `\"Graph Neural\n    Networks with Learnable Structural and Positional Representations\"\n    <https://arxiv.org/abs/2110.07875>`_ paper to the given graph\n    (functional name: :obj:`add_random_walk_pe`).\n\n    Args:\n        walk_length (int): The number of random walk steps.\n        attr_name (str, optional): The attribute name of the data object to add\n            positional encodings to. If set to :obj:`None`, will be\n            concatenated to :obj:`data.x`.\n            (default: :obj:`\"random_walk_pe\"`)\n    \"\"\"\n    def __init__(\n        self,\n        walk_length: int,\n        attr_name: Optional[str] = 'random_walk_pe',\n    ) -> None:\n        self.walk_length = walk_length\n        self.attr_name = attr_name\n\n    def forward(self, data: Data) -> Data:\n        assert data.edge_index is not None\n        row, col = data.edge_index\n        N = data.num_nodes\n        assert N is not None\n\n        if data.edge_weight is None:\n            value = torch.ones(data.num_edges, device=row.device)\n        else:\n            value = data.edge_weight\n        value = scatter(value, row, dim_size=N, reduce='sum').clamp(min=1)[row]\n        value = 1.0 / value\n\n        if N <= 2_000:  # Dense code path for faster computation:\n            adj = torch.zeros((N, N), device=row.device)\n            adj[row, col] = value\n            loop_index = torch.arange(N, device=row.device)\n        elif torch_geometric.typing.WITH_MKL:\n            adj = to_torch_csr_tensor(data.edge_index, value, size=data.size())\n        else:\n            adj = to_torch_coo_tensor(data.edge_index, value, size=data.size())\n\n        def get_pe(out: Tensor) -> Tensor:\n            if is_torch_sparse_tensor(out):\n                return get_self_loop_attr(*to_edge_index(out), num_nodes=N)\n            return out[loop_index, loop_index]\n\n        out = adj\n        pe_list = [get_pe(out)]\n        for _ in range(self.walk_length - 1):\n            out = out @ adj\n            pe_list.append(get_pe(out))\n\n        pe = torch.stack(pe_list, dim=-1)\n        data = add_node_attr(data, pe, attr_name=self.attr_name)\n\n        return data\n\n# Path: torch_geometric/transforms/add_remaining_self_loops.py\nfrom typing import Union\n\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\nfrom torch_geometric.utils import add_remaining_self_loops\n\n\n@functional_transform('add_remaining_self_loops')\nclass AddRemainingSelfLoops(BaseTransform):\n    r\"\"\"Adds remaining self-loops to the given homogeneous or heterogeneous\n    graph (functional name: :obj:`add_remaining_self_loops`).\n\n    Args:\n        attr (str, optional): The name of the attribute of edge weights\n            or multi-dimensional edge features to pass to\n            :meth:`torch_geometric.utils.add_remaining_self_loops`.\n            (default: :obj:`\"edge_weight\"`)\n        fill_value (float or Tensor or str, optional): The way to generate\n            edge features of self-loops (in case :obj:`attr != None`).\n            If given as :obj:`float` or :class:`torch.Tensor`, edge features of\n            self-loops will be directly given by :obj:`fill_value`.\n            If given as :obj:`str`, edge features of self-loops are computed by\n            aggregating all features of edges that point to the specific node,\n            according to a reduce operation. (:obj:`\"add\"`, :obj:`\"mean\"`,\n            :obj:`\"min\"`, :obj:`\"max\"`, :obj:`\"mul\"`). (default: :obj:`1.`)\n    \"\"\"\n    def __init__(\n        self,\n        attr: str = 'edge_weight',\n        fill_value: Union[float, Tensor, str] = 1.0,\n    ):\n        self.attr = attr\n        self.fill_value = fill_value\n\n    def forward(\n        self,\n        data: Union[Data, HeteroData],\n    ) -> Union[Data, HeteroData]:\n        for store in data.edge_stores:\n            if store.is_bipartite() or 'edge_index' not in store:\n                continue\n\n            store.edge_index, store[self.attr] = add_remaining_self_loops(\n                store.edge_index,\n                edge_attr=store.get(self.attr, None),\n                fill_value=self.fill_value,\n                num_nodes=store.size(0),\n            )\n\n        return data\n\n# Path: torch_geometric/transforms/add_self_loops.py\nfrom typing import Union\n\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\nfrom torch_geometric.utils import add_self_loops\n\n\n@functional_transform('add_self_loops')\nclass AddSelfLoops(BaseTransform):\n    r\"\"\"Adds self-loops to the given homogeneous or heterogeneous graph\n    (functional name: :obj:`add_self_loops`).\n\n    Args:\n        attr (str, optional): The name of the attribute of edge weights\n            or multi-dimensional edge features to pass to\n            :meth:`torch_geometric.utils.add_self_loops`.\n            (default: :obj:`\"edge_weight\"`)\n        fill_value (float or Tensor or str, optional): The way to generate\n            edge features of self-loops (in case :obj:`attr != None`).\n            If given as :obj:`float` or :class:`torch.Tensor`, edge features of\n            self-loops will be directly given by :obj:`fill_value`.\n            If given as :obj:`str`, edge features of self-loops are computed by\n            aggregating all features of edges that point to the specific node,\n            according to a reduce operation. (:obj:`\"add\"`, :obj:`\"mean\"`,\n            :obj:`\"min\"`, :obj:`\"max\"`, :obj:`\"mul\"`). (default: :obj:`1.`)\n    \"\"\"\n    def __init__(\n        self,\n        attr: str = 'edge_weight',\n        fill_value: Union[float, Tensor, str] = 1.0,\n    ) -> None:\n        self.attr = attr\n        self.fill_value = fill_value\n\n    def forward(\n        self,\n        data: Union[Data, HeteroData],\n    ) -> Union[Data, HeteroData]:\n        for store in data.edge_stores:\n            if store.is_bipartite() or 'edge_index' not in store:\n                continue\n\n            store.edge_index, store[self.attr] = add_self_loops(\n                store.edge_index,\n                edge_attr=store.get(self.attr, None),\n                fill_value=self.fill_value,\n                num_nodes=store.size(0),\n            )\n\n        return data\n\n# Path: torch_geometric/transforms/base_transform.py\nimport copy\nfrom abc import ABC\nfrom typing import Any\n\n\nclass BaseTransform(ABC):\n    r\"\"\"An abstract base class for writing transforms.\n\n    Transforms are a general way to modify and customize\n    :class:`~torch_geometric.data.Data` or\n    :class:`~torch_geometric.data.HeteroData` objects, either by implicitly\n    passing them as an argument to a :class:`~torch_geometric.data.Dataset`, or\n    by applying them explicitly to individual\n    :class:`~torch_geometric.data.Data` or\n    :class:`~torch_geometric.data.HeteroData` objects:\n\n    .. code-block:: python\n\n        import torch_geometric.transforms as T\n        from torch_geometric.datasets import TUDataset\n\n        transform = T.Compose([T.ToUndirected(), T.AddSelfLoops()])\n\n        dataset = TUDataset(path, name='MUTAG', transform=transform)\n        data = dataset[0]  # Implicitly transform data on every access.\n\n        data = TUDataset(path, name='MUTAG')[0]\n        data = transform(data)  # Explicitly transform data.\n    \"\"\"\n    def __call__(self, data: Any) -> Any:\n        # Shallow-copy the data so that we prevent in-place data modification.\n        return self.forward(copy.copy(data))\n\n    def forward(self, data: Any) -> Any:\n        pass\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}()'\n\n# Path: torch_geometric/transforms/cartesian.py\nfrom typing import Optional, Tuple\n\nimport torch\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\n\n\n@functional_transform('cartesian')\nclass Cartesian(BaseTransform):\n    r\"\"\"Saves the relative Cartesian coordinates of linked nodes in its edge\n    attributes (functional name: :obj:`cartesian`). Each coordinate gets\n    globally normalized to a specified interval (:math:`[0, 1]` by default).\n\n    Args:\n        norm (bool, optional): If set to :obj:`False`, the output will not be\n            normalized. (default: :obj:`True`)\n        max_value (float, optional): If set and :obj:`norm=True`, normalization\n            will be performed based on this value instead of the maximum value\n            found in the data. (default: :obj:`None`)\n        cat (bool, optional): If set to :obj:`False`, all existing edge\n            attributes will be replaced. (default: :obj:`True`)\n        interval ((float, float), optional): A tuple specifying the lower and\n            upper bound for normalization. (default: :obj:`(0.0, 1.0)`)\n    \"\"\"\n    def __init__(\n            self,\n            norm: bool = True,\n            max_value: Optional[float] = None,\n            cat: bool = True,\n            interval: Tuple[float, float] = (0.0, 1.0),\n    ):\n        self.norm = norm\n        self.max = max_value\n        self.cat = cat\n        self.interval = interval\n\n    def forward(self, data: Data) -> Data:\n        assert data.pos is not None\n        assert data.edge_index is not None\n        (row, col), pos, pseudo = data.edge_index, data.pos, data.edge_attr\n\n        cart = pos[row] - pos[col]\n        cart = cart.view(-1, 1) if cart.dim() == 1 else cart\n\n        if self.norm and cart.numel() > 0:\n            max_val = float(cart.abs().max()) if self.max is None else self.max\n\n            length = self.interval[1] - self.interval[0]\n            center = (self.interval[0] + self.interval[1]) / 2\n            cart = length * cart / (2 * max_val) + center\n\n        if pseudo is not None and self.cat:\n            pseudo = pseudo.view(-1, 1) if pseudo.dim() == 1 else pseudo\n            data.edge_attr = torch.cat([pseudo, cart.type_as(pseudo)], dim=-1)\n        else:\n            data.edge_attr = cart\n\n        return data\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(norm={self.norm}, '\n                f'max_value={self.max})')\n\n# Path: torch_geometric/transforms/center.py\nfrom typing import Union\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\n\n\n@functional_transform('center')\nclass Center(BaseTransform):\n    r\"\"\"Centers node positions :obj:`data.pos` around the origin\n    (functional name: :obj:`center`).\n    \"\"\"\n    def forward(\n        self,\n        data: Union[Data, HeteroData],\n    ) -> Union[Data, HeteroData]:\n        for store in data.node_stores:\n            if hasattr(store, 'pos'):\n                store.pos = store.pos - store.pos.mean(dim=-2, keepdim=True)\n        return data\n\n# Path: torch_geometric/transforms/compose.py\nfrom typing import Callable, List, Union\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.transforms import BaseTransform\n\n\nclass Compose(BaseTransform):\n    r\"\"\"Composes several transforms together.\n\n    Args:\n        transforms (List[Callable]): List of transforms to compose.\n    \"\"\"\n    def __init__(self, transforms: List[Callable]):\n        self.transforms = transforms\n\n    def forward(\n        self,\n        data: Union[Data, HeteroData],\n    ) -> Union[Data, HeteroData]:\n        for transform in self.transforms:\n            if isinstance(data, (list, tuple)):\n                data = [transform(d) for d in data]\n            else:\n                data = transform(data)\n        return data\n\n    def __repr__(self) -> str:\n        args = [f'  {transform}' for transform in self.transforms]\n        return '{}([\\n{}\\n])'.format(self.__class__.__name__, ',\\n'.join(args))\n\n\nclass ComposeFilters:\n    r\"\"\"Composes several filters together.\n\n    Args:\n        filters (List[Callable]): List of filters to compose.\n    \"\"\"\n    def __init__(self, filters: List[Callable]):\n        self.filters = filters\n\n    def __call__(\n        self,\n        data: Union[Data, HeteroData],\n    ) -> bool:\n        for filter_fn in self.filters:\n            if isinstance(data, (list, tuple)):\n                if not all([filter_fn(d) for d in data]):\n                    return False\n            elif not filter_fn(data):\n                return False\n        return True\n\n    def __repr__(self) -> str:\n        args = [f'  {filter_fn}' for filter_fn in self.filters]\n        return '{}([\\n{}\\n])'.format(self.__class__.__name__, ',\\n'.join(args))\n\n# Path: torch_geometric/transforms/constant.py\nfrom typing import List, Optional, Union\n\nimport torch\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\n\n\n@functional_transform('constant')\nclass Constant(BaseTransform):\n    r\"\"\"Appends a constant value to each node feature :obj:`x`\n    (functional name: :obj:`constant`).\n\n    Args:\n        value (float, optional): The value to add. (default: :obj:`1.0`)\n        cat (bool, optional): If set to :obj:`False`, existing node features\n            will be replaced. (default: :obj:`True`)\n        node_types (str or List[str], optional): The specified node type(s) to\n            append constant values for if used on heterogeneous graphs.\n            If set to :obj:`None`, constants will be added to each node feature\n            :obj:`x` for all existing node types. (default: :obj:`None`)\n    \"\"\"\n    def __init__(\n        self,\n        value: float = 1.0,\n        cat: bool = True,\n        node_types: Optional[Union[str, List[str]]] = None,\n    ):\n        if isinstance(node_types, str):\n            node_types = [node_types]\n\n        self.value = value\n        self.cat = cat\n        self.node_types = node_types\n\n    def forward(\n        self,\n        data: Union[Data, HeteroData],\n    ) -> Union[Data, HeteroData]:\n\n        for store in data.node_stores:\n            if self.node_types is None or store._key in self.node_types:\n                num_nodes = store.num_nodes\n                assert num_nodes is not None\n                c = torch.full((num_nodes, 1), self.value, dtype=torch.float)\n\n                if hasattr(store, 'x') and self.cat:\n                    x = store.x.view(-1, 1) if store.x.dim() == 1 else store.x\n                    store.x = torch.cat([x, c.to(x.device, x.dtype)], dim=-1)\n                else:\n                    store.x = c\n\n        return data\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}(value={self.value})'\n\n# Path: torch_geometric/transforms/delaunay.py\nimport scipy.spatial\nimport torch\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.data.datapipes import functional_transform\nfrom torch_geometric.transforms import BaseTransform\n\n\n@functional_transform('delaunay')\nclass Delaunay(BaseTransform):\n    r\"\"\"Computes the delaunay triangulation of a set of points\n    (functional name: :obj:`delaunay`).\n...\n", "needle_token_start": 3903, "needle_token_end": 4135, "code_context_ntokens": 16115, "output": ["```python\ndef read_from_memlab(line_profiler: Any) -> List[float]:  # pragma: no cover\n    from pytorch_memlab.line_profiler.line_records import LineRecords\n\n    # See: https://pytorch.org/docs/stable/cuda.html#torch.cuda.memory_stats\n\n    track_stats = [  # Different statistic can be collected as needed.\n        'allocated_bytes.all.peak',\n        'reserved_bytes.all.peak',\n        'active_bytes.all.peak',\n    ]\n\n    records = LineRecords(line_profiler._raw_line_records,\n                          line_profiler._code_infos)\n    stats = records.display(None, track_stats)._line_records\n    return [byte_to_megabyte(x) for x in stats.values.max(axis=0).tolist()]\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "num_node_labels", "language": "python", "path": "torch_geometric/datasets/tu_dataset.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the total number of distinct labels that can be assigned to nodes within the graphs of a dataset.\n2. **Input**: This method does not require any external inputs as it operates on the internal state of the dataset object.\n3. **Output**: Returns an integer representing the count of unique node labels available in the dataset.\n4. **Procedure**: The method accesses a pre-stored dictionary that contains various dataset properties, including the count of unique node labels, and returns the value associated with this specific property.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/datasets/snap_dataset.py\nimport os\nimport os.path as osp\nfrom typing import Any, Callable, Dict, List, Optional, Union\n\nimport fsspec\nimport numpy as np\nimport torch\n\n...\n# Path: torch_geometric/datasets/suite_sparse.py\nimport os.path as osp\nfrom typing import Callable, Optional\n\nimport fsspec\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset\nfrom torch_geometric.io import fs\n\n\nclass SuiteSparseMatrixCollection(InMemoryDataset):\n    r\"\"\"A suite of sparse matrix benchmarks known as the `Suite Sparse Matrix\n    Collection <https://sparse.tamu.edu>`_ collected from a wide range of\n    applications.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        group (str): The group of the sparse matrix.\n        name (str): The name of the sparse matrix.\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n\n    url = 'https://sparse.tamu.edu/mat/{}/{}.mat'\n\n    def __init__(\n        self,\n        root: str,\n        group: str,\n        name: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        self.group = group\n        self.name = name\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_dir(self) -> str:\n        return osp.join(self.root, self.group, self.name, 'raw')\n\n    @property\n    def processed_dir(self) -> str:\n        return osp.join(self.root, self.group, self.name, 'processed')\n\n    @property\n    def raw_file_names(self) -> str:\n        return f'{self.name}.mat'\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        fs.cp(self.url.format(self.group, self.name), self.raw_dir)\n\n    def process(self) -> None:\n        from scipy.io import loadmat\n\n        with fsspec.open(self.raw_paths[0], 'rb') as f:\n            mat = loadmat(f)['Problem'][0][0][2].tocsr().tocoo()\n\n        row = torch.from_numpy(mat.row).to(torch.long)\n        col = torch.from_numpy(mat.col).to(torch.long)\n        edge_index = torch.stack([row, col], dim=0)\n\n        value = torch.from_numpy(mat.data).to(torch.float)\n        edge_attr = None if torch.all(value == 1.0) else value\n\n        size: Optional[torch.Size] = torch.Size(mat.shape)\n        if mat.shape[0] == mat.shape[1]:\n            size = None\n\n        num_nodes = mat.shape[0]\n\n        data = Data(edge_index=edge_index, edge_attr=edge_attr, size=size,\n                    num_nodes=num_nodes)\n\n        if self.pre_transform is not None:\n            data = self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(group={self.group}, '\n                f'name={self.name})')\n\n# Path: torch_geometric/datasets/taobao.py\nimport os\nfrom typing import Callable, Optional\n\nimport numpy as np\nimport torch\n\nfrom torch_geometric.data import (\n    HeteroData,\n    InMemoryDataset,\n    download_url,\n    extract_zip,\n)\n\n\nclass Taobao(InMemoryDataset):\n    r\"\"\"Taobao is a dataset of user behaviors from Taobao offered by Alibaba,\n    provided by the `Tianchi Alicloud platform\n    <https://tianchi.aliyun.com/dataset/649>`_.\n\n    Taobao is a heterogeneous graph for recommendation.\n    Nodes represent users with user IDs, items with item IDs, and categories\n    with category ID.\n    Edges between users and items represent different types of user behaviors\n    towards items (alongside with timestamps).\n    Edges between items and categories assign each item to its set of\n    categories.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.HeteroData` object and returns a\n            transformed version. The data object will be transformed before\n            every access. (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.HeteroData` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n\n    \"\"\"\n    url = ('https://alicloud-dev.oss-cn-hangzhou.aliyuncs.com/'\n           'UserBehavior.csv.zip')\n\n    def __init__(\n        self,\n        root: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0], data_cls=HeteroData)\n\n    @property\n    def raw_file_names(self) -> str:\n        return 'UserBehavior.csv'\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        path = download_url(self.url, self.raw_dir)\n        extract_zip(path, self.raw_dir)\n        os.remove(path)\n\n    def process(self) -> None:\n        import pandas as pd\n\n        cols = ['userId', 'itemId', 'categoryId', 'behaviorType', 'timestamp']\n        df = pd.read_csv(self.raw_paths[0], names=cols)\n\n        # Time representation (YYYY.MM.DD-HH:MM:SS -> Integer)\n        # start: 1511539200 = 2017.11.25-00:00:00\n        # end:   1512316799 = 2017.12.03-23:59:59\n        start = 1511539200\n        end = 1512316799\n        df = df[(df[\"timestamp\"] >= start) & (df[\"timestamp\"] <= end)]\n\n        df = df.drop_duplicates()\n\n        behavior_dict = {'pv': 0, 'cart': 1, 'buy': 2, 'fav': 3}\n        df['behaviorType'] = df['behaviorType'].map(behavior_dict)\n\n        num_entries = {}\n        for name in ['userId', 'itemId', 'categoryId']:\n            # Map IDs to consecutive integers:\n            value, df[name] = np.unique(df[[name]].values, return_inverse=True)\n            num_entries[name] = value.shape[0]\n\n        data = HeteroData()\n\n        data['user'].num_nodes = num_entries['userId']\n        data['item'].num_nodes = num_entries['itemId']\n        data['category'].num_nodes = num_entries['categoryId']\n\n        row = torch.from_numpy(df['userId'].values)\n        col = torch.from_numpy(df['itemId'].values)\n        data['user', 'item'].edge_index = torch.stack([row, col], dim=0)\n        data['user', 'item'].time = torch.from_numpy(df['timestamp'].values)\n        behavior = torch.from_numpy(df['behaviorType'].values)\n        data['user', 'item'].behavior = behavior\n\n        df = df[['itemId', 'categoryId']].drop_duplicates()\n        row = torch.from_numpy(df['itemId'].values)\n        col = torch.from_numpy(df['categoryId'].values)\n        data['item', 'category'].edge_index = torch.stack([row, col], dim=0)\n\n        data = data if self.pre_transform is None else self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n# Path: torch_geometric/datasets/tosca.py\nimport glob\nimport os\nimport os.path as osp\nfrom typing import Callable, List, Optional\n\nimport torch\n\nfrom torch_geometric.data import (\n    Data,\n    InMemoryDataset,\n    download_url,\n    extract_zip,\n)\nfrom torch_geometric.io import read_txt_array\n\n\nclass TOSCA(InMemoryDataset):\n    r\"\"\"The TOSCA dataset from the `\"Numerical Geometry of Non-Ridig Shapes\"\n    <https://www.amazon.com/Numerical-Geometry-Non-Rigid-Monographs-Computer/\n    dp/0387733000>`_ book, containing 80 meshes.\n    Meshes within the same category have the same triangulation and an equal\n    number of vertices numbered in a compatible way.\n\n    .. note::\n\n        Data objects hold mesh faces instead of edge indices.\n        To convert the mesh to a graph, use the\n        :obj:`torch_geometric.transforms.FaceToEdge` as :obj:`pre_transform`.\n        To convert the mesh to a point cloud, use the\n        :obj:`torch_geometric.transforms.SamplePoints` as :obj:`transform` to\n        sample a fixed number of points on the mesh faces according to their\n        face area.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        categories (list, optional): List of categories to include in the\n            dataset. Can include the categories :obj:`\"Cat\"`, :obj:`\"Centaur\"`,\n            :obj:`\"David\"`, :obj:`\"Dog\"`, :obj:`\"Gorilla\"`, :obj:`\"Horse\"`,\n            :obj:`\"Michael\"`, :obj:`\"Victoria\"`, :obj:`\"Wolf\"`. If set to\n            :obj:`None`, the dataset will contain all categories. (default:\n            :obj:`None`)\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        pre_filter (callable, optional): A function that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a boolean\n            value, indicating whether the data object should be included in the\n            final dataset. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n\n    url = 'http://tosca.cs.technion.ac.il/data/toscahires-asci.zip'\n\n    categories = [\n        'cat', 'centaur', 'david', 'dog', 'gorilla', 'horse', 'michael',\n        'victoria', 'wolf'\n    ]\n\n    def __init__(\n        self,\n        root: str,\n        categories: Optional[List[str]] = None,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        pre_filter: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        categories = self.categories if categories is None else categories\n        categories = [cat.lower() for cat in categories]\n        for cat in categories:\n            assert cat in self.categories\n        self.categories = categories\n        super().__init__(root, transform, pre_transform, pre_filter,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        return ['cat0.vert', 'cat0.tri']\n\n    @property\n    def processed_file_names(self) -> str:\n        name = '_'.join([cat[:2] for cat in self.categories])\n        return f'{name}.pt'\n\n    def download(self) -> None:\n        path = download_url(self.url, self.raw_dir)\n        extract_zip(path, self.raw_dir)\n        os.unlink(path)\n\n    def process(self) -> None:\n        data_list = []\n        for cat in self.categories:\n            paths = glob.glob(osp.join(self.raw_dir, f'{cat}*.tri'))\n            paths = [path[:-4] for path in paths]\n            paths = sorted(paths, key=lambda e: (len(e), e))\n\n            for path in paths:\n                pos = read_txt_array(f'{path}.vert')\n                face = read_txt_array(f'{path}.tri', dtype=torch.long)\n                face = face - face.min()  # Ensure zero-based index.\n                data = Data(pos=pos, face=face.t().contiguous())\n                if self.pre_filter is not None and not self.pre_filter(data):\n                    continue\n                if self.pre_transform is not None:\n                    data = self.pre_transform(data)\n                data_list.append(data)\n\n        self.save(data_list, self.processed_paths[0])\n\n# Path: torch_geometric/datasets/tu_dataset.py\nimport os.path as osp\nfrom typing import Callable, List, Optional\n\nfrom torch_geometric.data import Data, InMemoryDataset\nfrom torch_geometric.io import fs, read_tu_data\n\n\nclass TUDataset(InMemoryDataset):\n    r\"\"\"A variety of graph kernel benchmark datasets, *.e.g.*,\n    :obj:`\"IMDB-BINARY\"`, :obj:`\"REDDIT-BINARY\"` or :obj:`\"PROTEINS\"`,\n    collected from the `TU Dortmund University\n    <https://chrsmrrs.github.io/datasets>`_.\n    In addition, this dataset wrapper provides `cleaned dataset versions\n    <https://github.com/nd7141/graph_datasets>`_ as motivated by the\n    `\"Understanding Isomorphism Bias in Graph Data Sets\"\n    <https://arxiv.org/abs/1910.12091>`_ paper, containing only non-isomorphic\n    graphs.\n\n    .. note::\n        Some datasets may not come with any node labels.\n        You can then either make use of the argument :obj:`use_node_attr`\n        to load additional continuous node attributes (if present) or provide\n        synthetic node features using transforms such as\n        :class:`torch_geometric.transforms.Constant` or\n        :class:`torch_geometric.transforms.OneHotDegree`.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        name (str): The `name\n            <https://chrsmrrs.github.io/datasets/docs/datasets/>`_ of the\n            dataset.\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        pre_filter (callable, optional): A function that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a boolean\n            value, indicating whether the data object should be included in the\n            final dataset. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n        use_node_attr (bool, optional): If :obj:`True`, the dataset will\n            contain additional continuous node attributes (if present).\n            (default: :obj:`False`)\n        use_edge_attr (bool, optional): If :obj:`True`, the dataset will\n            contain additional continuous edge attributes (if present).\n            (default: :obj:`False`)\n        cleaned (bool, optional): If :obj:`True`, the dataset will\n            contain only non-isomorphic graphs. (default: :obj:`False`)\n\n    **STATS:**\n\n    .. list-table::\n        :widths: 20 10 10 10 10 10\n        :header-rows: 1\n\n        * - Name\n          - #graphs\n          - #nodes\n          - #edges\n          - #features\n          - #classes\n        * - MUTAG\n          - 188\n          - ~17.9\n          - ~39.6\n          - 7\n          - 2\n        * - ENZYMES\n          - 600\n          - ~32.6\n          - ~124.3\n          - 3\n          - 6\n        * - PROTEINS\n          - 1,113\n          - ~39.1\n          - ~145.6\n          - 3\n          - 2\n        * - COLLAB\n          - 5,000\n          - ~74.5\n          - ~4914.4\n          - 0\n          - 3\n        * - IMDB-BINARY\n          - 1,000\n          - ~19.8\n          - ~193.1\n          - 0\n          - 2\n        * - REDDIT-BINARY\n          - 2,000\n          - ~429.6\n          - ~995.5\n          - 0\n          - 2\n        * - ...\n          -\n          -\n          -\n          -\n          -\n    \"\"\"\n\n    url = 'https://www.chrsmrrs.com/graphkerneldatasets'\n    cleaned_url = ('https://raw.githubusercontent.com/nd7141/'\n                   'graph_datasets/master/datasets')\n\n    def __init__(\n        self,\n        root: str,\n        name: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        pre_filter: Optional[Callable] = None,\n        force_reload: bool = False,\n        use_node_attr: bool = False,\n        use_edge_attr: bool = False,\n        cleaned: bool = False,\n    ) -> None:\n        self.name = name\n        self.cleaned = cleaned\n        super().__init__(root, transform, pre_transform, pre_filter,\n                         force_reload=force_reload)\n\n        out = fs.torch_load(self.processed_paths[0])\n        if not isinstance(out, tuple) or len(out) < 3:\n            raise RuntimeError(\n                \"The 'data' object was created by an older version of PyG. \"\n                \"If this error occurred while loading an already existing \"\n                \"dataset, remove the 'processed/' directory in the dataset's \"\n                \"root folder and try again.\")\n        assert len(out) == 3 or len(out) == 4\n\n        if len(out) == 3:  # Backward compatibility.\n            data, self.slices, self.sizes = out\n            data_cls = Data\n        else:\n            data, self.slices, self.sizes, data_cls = out\n\n        if not isinstance(data, dict):  # Backward compatibility.\n            self.data = data\n        else:\n            self.data = data_cls.from_dict(data)\n\n        assert isinstance(self._data, Data)\n        if self._data.x is not None and not use_node_attr:\n            num_node_attributes = self.num_node_attributes\n            self._data.x = self._data.x[:, num_node_attributes:]\n        if self._data.edge_attr is not None and not use_edge_attr:\n            num_edge_attrs = self.num_edge_attributes\n            self._data.edge_attr = self._data.edge_attr[:, num_edge_attrs:]\n\n    @property\n    def raw_dir(self) -> str:\n        name = f'raw{\"_cleaned\" if self.cleaned else \"\"}'\n        return osp.join(self.root, self.name, name)\n\n    @property\n    def processed_dir(self) -> str:\n        name = f'processed{\"_cleaned\" if self.cleaned else \"\"}'\n        return osp.join(self.root, self.name, name)\n\n    @property\n    \ndef num_node_labels(self) -> int:\n        return self.sizes['num_node_labels']\n\n    @property\n    def num_node_attributes(self) -> int:\n        return self.sizes['num_node_attributes']\n\n    @property\n    def num_edge_labels(self) -> int:\n        return self.sizes['num_edge_labels']\n\n    @property\n    def num_edge_attributes(self) -> int:\n        return self.sizes['num_edge_attributes']\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        names = ['A', 'graph_indicator']\n        return [f'{self.name}_{name}.txt' for name in names]\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        url = self.cleaned_url if self.cleaned else self.url\n        fs.cp(f'{url}/{self.name}.zip', self.raw_dir, extract=True)\n        for filename in fs.ls(osp.join(self.raw_dir, self.name)):\n            fs.mv(filename, osp.join(self.raw_dir, osp.basename(filename)))\n        fs.rm(osp.join(self.raw_dir, self.name))\n\n    def process(self) -> None:\n        self.data, self.slices, sizes = read_tu_data(self.raw_dir, self.name)\n\n        if self.pre_filter is not None or self.pre_transform is not None:\n            data_list = [self.get(idx) for idx in range(len(self))]\n\n            if self.pre_filter is not None:\n                data_list = [d for d in data_list if self.pre_filter(d)]\n\n            if self.pre_transform is not None:\n                data_list = [self.pre_transform(d) for d in data_list]\n\n            self.data, self.slices = self.collate(data_list)\n            self._data_list = None  # Reset cache.\n\n        assert isinstance(self._data, Data)\n        fs.torch_save(\n            (self._data.to_dict(), self.slices, sizes, self._data.__class__),\n            self.processed_paths[0],\n        )\n\n    def __repr__(self) -> str:\n        return f'{self.name}({len(self)})'\n\n# Path: torch_geometric/datasets/twitch.py\nimport os.path as osp\nfrom typing import Callable, Optional\n\nimport numpy as np\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset, download_url\n\n\nclass Twitch(InMemoryDataset):\n    r\"\"\"The Twitch Gamer networks introduced in the\n    `\"Multi-scale Attributed Node Embedding\"\n    <https://arxiv.org/abs/1909.13021>`_ paper.\n    Nodes represent gamers on Twitch and edges are followerships between them.\n    Node features represent embeddings of games played by the Twitch users.\n    The task is to predict whether a user streams mature content.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        name (str): The name of the dataset (:obj:`\"DE\"`, :obj:`\"EN\"`,\n            :obj:`\"ES\"`, :obj:`\"FR\"`, :obj:`\"PT\"`, :obj:`\"RU\"`).\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n\n    **STATS:**\n\n    .. list-table::\n        :widths: 10 10 10 10 10\n        :header-rows: 1\n\n        * - Name\n          - #nodes\n          - #edges\n          - #features\n          - #classes\n        * - DE\n          - 9,498\n          - 315,774\n          - 128\n          - 2\n        * - EN\n          - 7,126\n          - 77,774\n          - 128\n          - 2\n        * - ES\n          - 4,648\n          - 123,412\n          - 128\n          - 2\n        * - FR\n          - 6,551\n          - 231,883\n          - 128\n          - 2\n        * - PT\n          - 1,912\n          - 64,510\n          - 128\n          - 2\n        * - RU\n          - 4,385\n          - 78,993\n          - 128\n          - 2\n    \"\"\"\n\n    url = 'https://graphmining.ai/datasets/ptg/twitch'\n\n    def __init__(\n        self,\n        root: str,\n        name: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        self.name = name\n        assert self.name in ['DE', 'EN', 'ES', 'FR', 'PT', 'RU']\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_dir(self) -> str:\n        return osp.join(self.root, self.name, 'raw')\n\n    @property\n    def processed_dir(self) -> str:\n        return osp.join(self.root, self.name, 'processed')\n\n    @property\n    def raw_file_names(self) -> str:\n        return f'{self.name}.npz'\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        download_url(f'{self.url}/{self.name}.npz', self.raw_dir)\n\n    def process(self) -> None:\n        data = np.load(self.raw_paths[0], 'r', allow_pickle=True)\n        x = torch.from_numpy(data['features']).to(torch.float)\n        y = torch.from_numpy(data['target']).to(torch.long)\n\n        edge_index = torch.from_numpy(data['edges']).to(torch.long)\n        edge_index = edge_index.t().contiguous()\n\n        data = Data(x=x, y=y, edge_index=edge_index)\n\n        if self.pre_transform is not None:\n            data = self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n# Path: torch_geometric/datasets/upfd.py\nimport os\nimport os.path as osp\nfrom typing import Callable, List, Optional\n\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\n\nfrom torch_geometric.data import (\n    Data,\n    InMemoryDataset,\n    download_google_url,\n    extract_zip,\n)\nfrom torch_geometric.io import read_txt_array\nfrom torch_geometric.utils import coalesce, cumsum\n\n\nclass UPFD(InMemoryDataset):\n    r\"\"\"The tree-structured fake news propagation graph classification dataset\n    from the `\"User Preference-aware Fake News Detection\"\n    <https://arxiv.org/abs/2104.12259>`_ paper.\n    It includes two sets of tree-structured fake & real news propagation graphs\n    extracted from Twitter.\n    For a single graph, the root node represents the source news, and leaf\n    nodes represent Twitter users who retweeted the same root news.\n    A user node has an edge to the news node if and only if the user retweeted\n    the root news directly.\n    Two user nodes have an edge if and only if one user retweeted the root news\n    from the other user.\n    Four different node features are encoded using different encoders.\n    Please refer to `GNN-FakeNews\n    <https://github.com/safe-graph/GNN-FakeNews>`_ repo for more details.\n\n    .. note::\n\n        For an example of using UPFD, see `examples/upfd.py\n        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/\n        upfd.py>`_.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        name (str): The name of the graph set (:obj:`\"politifact\"`,\n            :obj:`\"gossipcop\"`).\n        feature (str): The node feature type (:obj:`\"profile\"`, :obj:`\"spacy\"`,\n            :obj:`\"bert\"`, :obj:`\"content\"`).\n            If set to :obj:`\"profile\"`, the 10-dimensional node feature\n            is composed of ten Twitter user profile attributes.\n            If set to :obj:`\"spacy\"`, the 300-dimensional node feature is\n            composed of Twitter user historical tweets encoded by\n            the `spaCy word2vec encoder\n            <https://spacy.io/models/en#en_core_web_lg>`_.\n            If set to :obj:`\"bert\"`, the 768-dimensional node feature is\n            composed of Twitter user historical tweets encoded by the\n            `bert-as-service <https://github.com/hanxiao/bert-as-service>`_.\n            If set to :obj:`\"content\"`, the 310-dimensional node feature is\n            composed of a 300-dimensional \"spacy\" vector plus a\n            10-dimensional \"profile\" vector.\n        split (str, optional): If :obj:`\"train\"`, loads the training dataset.\n            If :obj:`\"val\"`, loads the validation dataset.\n            If :obj:`\"test\"`, loads the test dataset.\n            (default: :obj:`\"train\"`)\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        pre_filter (callable, optional): A function that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a boolean\n            value, indicating whether the data object should be included in the\n            final dataset. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n    file_ids = {\n        'politifact': '1KOmSrlGcC50PjkvRVbyb_WoWHVql06J-',\n        'gossipcop': '1VskhAQ92PrT4sWEKQ2v2-AJhEcpp4A81',\n    }\n\n    def __init__(\n        self,\n        root: str,\n        name: str,\n        feature: str,\n        split: str = \"train\",\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        pre_filter: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        assert name in ['politifact', 'gossipcop']\n        assert split in ['train', 'val', 'test']\n\n        self.root = root\n        self.name = name\n        self.feature = feature\n\n        super().__init__(root, transform, pre_transform, pre_filter,\n                         force_reload=force_reload)\n\n        path = self.processed_paths[['train', 'val', 'test'].index(split)]\n        self.load(path)\n\n    @property\n    def raw_dir(self) -> str:\n        return osp.join(self.root, self.name, 'raw')\n\n    @property\n    def processed_dir(self) -> str:\n        return osp.join(self.root, self.name, 'processed', self.feature)\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        return [\n            'node_graph_id.npy', 'graph_labels.npy', 'A.txt', 'train_idx.npy',\n            'val_idx.npy', 'test_idx.npy', f'new_{self.feature}_feature.npz'\n        ]\n\n    @property\n    def processed_file_names(self) -> List[str]:\n        return ['train.pt', 'val.pt', 'test.pt']\n\n    def download(self) -> None:\n        id = self.file_ids[self.name]\n        path = download_google_url(id, self.raw_dir, 'data.zip')\n        extract_zip(path, self.raw_dir)\n        os.remove(path)\n\n    def process(self) -> None:\n        x = sp.load_npz(\n            osp.join(self.raw_dir, f'new_{self.feature}_feature.npz'))\n        x = torch.from_numpy(x.todense()).to(torch.float)\n\n        edge_index = read_txt_array(osp.join(self.raw_dir, 'A.txt'), sep=',',\n                                    dtype=torch.long).t()\n        edge_index = coalesce(edge_index, num_nodes=x.size(0))\n\n        y = np.load(osp.join(self.raw_dir, 'graph_labels.npy'))\n        y = torch.from_numpy(y).to(torch.long)\n        _, y = y.unique(sorted=True, return_inverse=True)\n\n        batch = np.load(osp.join(self.raw_dir, 'node_graph_id.npy'))\n        batch = torch.from_numpy(batch).to(torch.long)\n\n        node_slice = cumsum(batch.bincount())\n        edge_slice = cumsum(batch[edge_index[0]].bincount())\n        graph_slice = torch.arange(y.size(0) + 1)\n        self.slices = {\n            'x': node_slice,\n            'edge_index': edge_slice,\n            'y': graph_slice\n        }\n\n        edge_index -= node_slice[batch[edge_index[0]]].view(1, -1)\n        self.data = Data(x=x, edge_index=edge_index, y=y)\n\n        for path, split in zip(self.processed_paths, ['train', 'val', 'test']):\n            idx = np.load(osp.join(self.raw_dir, f'{split}_idx.npy')).tolist()\n            data_list = [self.get(i) for i in idx]\n            if self.pre_filter is not None:\n                data_list = [d for d in data_list if self.pre_filter(d)]\n            if self.pre_transform is not None:\n                data_list = [self.pre_transform(d) for d in data_list]\n            self.save(data_list, path)\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}({len(self)}, name={self.name}, '\n                f'feature={self.feature})')\n\n# Path: torch_geometric/datasets/utils/cheatsheet.py\nimport importlib\nimport inspect\nimport re\nfrom typing import Any, List, Optional\n\n\ndef paper_link(cls: str) -> Optional[str]:\n    cls = importlib.import_module('torch_geometric.datasets').__dict__[cls]\n    doc = inspect.getdoc(cls)\n    assert doc is not None\n    match = re.search('<.+?>', doc, flags=re.DOTALL)\n    return None if match is None else match.group().replace('\\n', ' ')[1:-1]\n\n\ndef get_stats_table(cls: str) -> str:\n    cls = importlib.import_module('torch_geometric.datasets').__dict__[cls]\n    doc = inspect.getdoc(cls)\n    assert doc is not None\n    match = re.search(r'\\*\\*STATS:\\*\\*\\n.*$', doc, flags=re.DOTALL)\n    return '' if match is None else match.group()\n\n\ndef has_stats(cls: str) -> bool:\n    return len(get_stats_table(cls)) > 0\n\n\ndef get_type(cls: str) -> str:\n    return 'Edge' if '-' in cls else 'Node'\n\n\ndef get_stat(cls: str, name: str, child: Optional[str] = None,\n             default: Any = None) -> str:\n    if child is None and len(get_children(cls)) > 0:\n        return ''\n\n    stats_table = get_stats_table(cls)\n\n    if len(stats_table) > 0:\n        stats_table = '\\n'.join(stats_table.split('\\n')[2:])\n\n    match = re.search(f'^.*- {name}', stats_table, flags=re.DOTALL)\n    if match is None:\n        return default\n\n    column = match.group().count(' -')\n\n    if child is not None:\n        child = child.replace('(', r'\\(').replace(')', r'\\)')\n        match = re.search(f'[*] - {child}\\n.*$', stats_table, flags=re.DOTALL)\n        assert match is not None\n        stats_row = match.group()\n    else:\n        stats_row = '*' + stats_table.split('*')[2]\n\n    return stats_row.split(' -')[column].split('\\n')[0].strip()\n\n\ndef get_children(cls: str) -> List[str]:\n    matches = re.findall('[*] -.*', get_stats_table(cls))\n    return [match[4:] for match in matches[1:]] if len(matches) > 2 else []\n\n# Path: torch_geometric/datasets/utils/__init__.py\nfrom .cheatsheet import paper_link, has_stats, get_stat, get_children, get_type\n\n__all__ = [\n    'paper_link',\n    'has_stats',\n    'get_stat',\n    'get_children',\n    'get_type',\n]\n\n# Path: torch_geometric/datasets/webkb.py\nimport os.path as osp\nfrom typing import Callable, List, Optional\n\nimport numpy as np\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset, download_url\nfrom torch_geometric.utils import coalesce\n\n\nclass WebKB(InMemoryDataset):\n    r\"\"\"The WebKB datasets used in the\n    `\"Geom-GCN: Geometric Graph Convolutional Networks\"\n    <https://openreview.net/forum?id=S1e2agrFvS>`_ paper.\n    Nodes represent web pages and edges represent hyperlinks between them.\n    Node features are the bag-of-words representation of web pages.\n    The task is to classify the nodes into one of the five categories, student,\n    project, course, staff, and faculty.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        name (str): The name of the dataset (:obj:`\"Cornell\"`, :obj:`\"Texas\"`,\n            :obj:`\"Wisconsin\"`).\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n\n    **STATS:**\n\n    .. list-table::\n        :widths: 10 10 10 10 10\n        :header-rows: 1\n\n        * - Name\n          - #nodes\n          - #edges\n          - #features\n          - #classes\n        * - Cornell\n          - 183\n          - 298\n          - 1,703\n          - 5\n        * - Texas\n          - 183\n          - 325\n          - 1,703\n          - 5\n        * - Wisconsin\n          - 251\n          - 515\n          - 1,703\n          - 5\n    \"\"\"\n\n    url = 'https://raw.githubusercontent.com/graphdml-uiuc-jlu/geom-gcn/master'\n\n    def __init__(\n        self,\n        root: str,\n        name: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        self.name = name.lower()\n        assert self.name in ['cornell', 'texas', 'wisconsin']\n\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_dir(self) -> str:\n        return osp.join(self.root, self.name, 'raw')\n\n    @property\n    def processed_dir(self) -> str:\n        return osp.join(self.root, self.name, 'processed')\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        out = ['out1_node_feature_label.txt', 'out1_graph_edges.txt']\n        out += [f'{self.name}_split_0.6_0.2_{i}.npz' for i in range(10)]\n        return out\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        for f in self.raw_file_names[:2]:\n            download_url(f'{self.url}/new_data/{self.name}/{f}', self.raw_dir)\n        for f in self.raw_file_names[2:]:\n            download_url(f'{self.url}/splits/{f}', self.raw_dir)\n\n    def process(self) -> None:\n        with open(self.raw_paths[0], 'r') as f:\n            lines = f.read().split('\\n')[1:-1]\n            xs = [[float(value) for value in line.split('\\t')[1].split(',')]\n                  for line in lines]\n            x = torch.tensor(xs, dtype=torch.float)\n\n            ys = [int(line.split('\\t')[2]) for line in lines]\n            y = torch.tensor(ys, dtype=torch.long)\n\n        with open(self.raw_paths[1], 'r') as f:\n            lines = f.read().split('\\n')[1:-1]\n            edge_indices = [[int(value) for value in line.split('\\t')]\n                            for line in lines]\n            edge_index = torch.tensor(edge_indices).t().contiguous()\n            edge_index = coalesce(edge_index, num_nodes=x.size(0))\n\n        train_masks, val_masks, test_masks = [], [], []\n        for path in self.raw_paths[2:]:\n            tmp = np.load(path)\n            train_masks += [torch.from_numpy(tmp['train_mask']).to(torch.bool)]\n            val_masks += [torch.from_numpy(tmp['val_mask']).to(torch.bool)]\n            test_masks += [torch.from_numpy(tmp['test_mask']).to(torch.bool)]\n        train_mask = torch.stack(train_masks, dim=1)\n        val_mask = torch.stack(val_masks, dim=1)\n        test_mask = torch.stack(test_masks, dim=1)\n\n        data = Data(x=x, edge_index=edge_index, y=y, train_mask=train_mask,\n                    val_mask=val_mask, test_mask=test_mask)\n        data = data if self.pre_transform is None else self.pre_transform(data)\n        self.save([data], self.processed_paths[0])\n\n    def __repr__(self) -> str:\n        return f'{self.name}()'\n\n# Path: torch_geometric/datasets/wikics.py\nimport json\nimport warnings\nfrom itertools import chain\nfrom typing import Callable, List, Optional\n\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset, download_url\nfrom torch_geometric.utils import to_undirected\n\n\nclass WikiCS(InMemoryDataset):\n    r\"\"\"The semi-supervised Wikipedia-based dataset from the\n    `\"Wiki-CS: A Wikipedia-Based Benchmark for Graph Neural Networks\"\n    <https://arxiv.org/abs/2007.02901>`_ paper, containing 11,701 nodes,\n    216,123 edges, 10 classes and 20 different training splits.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        is_undirected (bool, optional): Whether the graph is undirected.\n            (default: :obj:`True`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n\n    url = 'https://github.com/pmernyei/wiki-cs-dataset/raw/master/dataset'\n\n    def __init__(\n        self,\n        root: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        is_undirected: Optional[bool] = None,\n        force_reload: bool = False,\n    ) -> None:\n        if is_undirected is None:\n            warnings.warn(\n                f\"The {self.__class__.__name__} dataset now returns an \"\n                f\"undirected graph by default. Please explicitly specify \"\n                f\"'is_undirected=False' to restore the old behavior.\")\n            is_undirected = True\n        self.is_undirected = is_undirected\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        return ['data.json']\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data_undirected.pt' if self.is_undirected else 'data.pt'\n\n    def download(self) -> None:\n        for name in self.raw_file_names:\n            download_url(f'{self.url}/{name}', self.raw_dir)\n\n    def process(self) -> None:\n        with open(self.raw_paths[0], 'r') as f:\n            data = json.load(f)\n\n        x = torch.tensor(data['features'], dtype=torch.float)\n        y = torch.tensor(data['labels'], dtype=torch.long)\n\n        edges = [[(i, j) for j in js] for i, js in enumerate(data['links'])]\n        edges = list(chain(*edges))  # type: ignore\n        edge_index = torch.tensor(edges, dtype=torch.long).t().contiguous()\n        if self.is_undirected:\n            edge_index = to_undirected(edge_index, num_nodes=x.size(0))\n\n        train_mask = torch.tensor(data['train_masks'], dtype=torch.bool)\n        train_mask = train_mask.t().contiguous()\n\n        val_mask = torch.tensor(data['val_masks'], dtype=torch.bool)\n        val_mask = val_mask.t().contiguous()\n\n        test_mask = torch.tensor(data['test_mask'], dtype=torch.bool)\n\n        stopping_mask = torch.tensor(data['stopping_masks'], dtype=torch.bool)\n        stopping_mask = stopping_mask.t().contiguous()\n\n        data = Data(x=x, y=y, edge_index=edge_index, train_mask=train_mask,\n                    val_mask=val_mask, test_mask=test_mask,\n                    stopping_mask=stopping_mask)\n\n        if self.pre_transform is not None:\n            data = self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n# Path: torch_geometric/datasets/wikidata.py\nimport os\nimport os.path as osp\nfrom typing import Callable, Dict, List, Optional\n\nimport torch\n\nfrom torch_geometric.data import (\n    Data,\n    InMemoryDataset,\n    download_url,\n    extract_tar,\n)\n\n\nclass Wikidata5M(InMemoryDataset):\n    r\"\"\"The Wikidata-5M dataset from the `\"KEPLER: A Unified Model for\n    Knowledge Embedding and Pre-trained Language Representation\"\n    <https://arxiv.org/abs/1911.06136>`_ paper,\n    containing 4,594,485 entities, 822 relations,\n    20,614,279 train triples, 5,163 validation triples, and 5,133 test triples.\n\n    `Wikidata-5M <https://deepgraphlearning.github.io/project/wikidata5m>`_\n    is a large-scale knowledge graph dataset with aligned corpus\n    extracted form Wikidata.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        setting (str, optional):\n            If :obj:`\"transductive\"`, loads the transductive dataset.\n            If :obj:`\"inductive\"`, loads the inductive dataset.\n            (default: :obj:`\"transductive\"`)\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n    def __init__(\n        self,\n        root: str,\n        setting: str = 'transductive',\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        if setting not in {'transductive', 'inductive'}:\n            raise ValueError(f\"Invalid 'setting' argument (got '{setting}')\")\n\n        self.setting = setting\n\n        self.urls = [\n            ('https://www.dropbox.com/s/7jp4ib8zo3i6m10/'\n             'wikidata5m_text.txt.gz?dl=1'),\n            'https://uni-bielefeld.sciebo.de/s/yuBKzBxsEc9j3hy/download',\n        ]\n        if self.setting == 'inductive':\n            self.urls.append('https://www.dropbox.com/s/csed3cgal3m7rzo/'\n                             'wikidata5m_inductive.tar.gz?dl=1')\n        else:\n            self.urls.append('https://www.dropbox.com/s/6sbhm0rwo4l73jq/'\n                             'wikidata5m_transductive.tar.gz?dl=1')\n\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        return [\n            'wikidata5m_text.txt.gz',\n            'download',\n            f'wikidata5m_{self.setting}_train.txt',\n            f'wikidata5m_{self.setting}_valid.txt',\n            f'wikidata5m_{self.setting}_test.txt',\n        ]\n\n    @property\n    def processed_file_names(self) -> str:\n        return f'{self.setting}_data.pt'\n\n    def download(self) -> None:\n        for url in self.urls:\n            download_url(url, self.raw_dir)\n        path = osp.join(self.raw_dir, f'wikidata5m_{self.setting}.tar.gz')\n        extract_tar(path, self.raw_dir)\n        os.remove(path)\n\n    def process(self) -> None:\n        import gzip\n\n        entity_to_id: Dict[str, int] = {}\n        with gzip.open(self.raw_paths[0], 'rt') as f:\n            for i, line in enumerate(f):\n                values = line.strip().split('\\t')\n                entity_to_id[values[0]] = i\n\n        x = torch.load(self.raw_paths[1])\n\n        edge_indices = []\n        edge_types = []\n        split_indices = []\n\n        rel_to_id: Dict[str, int] = {}\n        for split, path in enumerate(self.raw_paths[2:]):\n            with open(path, 'r') as f:\n                for line in f:\n                    head, rel, tail = line[:-1].split('\\t')\n                    src = entity_to_id[head]\n                    dst = entity_to_id[tail]\n                    edge_indices.append([src, dst])\n                    if rel not in rel_to_id:\n                        rel_to_id[rel] = len(rel_to_id)\n                    edge_types.append(rel_to_id[rel])\n                    split_indices.append(split)\n\n        edge_index = torch.tensor(edge_indices).t().contiguous()\n        edge_type = torch.tensor(edge_types)\n        split_index = torch.tensor(split_indices)\n\n        data = Data(\n            x=x,\n            edge_index=edge_index,\n            edge_type=edge_type,\n            train_mask=split_index == 0,\n            val_mask=split_index == 1,\n            test_mask=split_index == 2,\n        )\n\n        if self.pre_transform is not None:\n            data = self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n# Path: torch_geometric/datasets/wikipedia_network.py\nimport os.path as osp\nfrom typing import Callable, List, Optional, Union\n\nimport numpy as np\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset, download_url\nfrom torch_geometric.utils import coalesce\n\n\nclass WikipediaNetwork(InMemoryDataset):\n    r\"\"\"The Wikipedia networks introduced in the\n    `\"Multi-scale Attributed Node Embedding\"\n    <https://arxiv.org/abs/1909.13021>`_ paper.\n    Nodes represent web pages and edges represent hyperlinks between them.\n    Node features represent several informative nouns in the Wikipedia pages.\n    The task is to predict the average daily traffic of the web page.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        name (str): The name of the dataset (:obj:`\"chameleon\"`,\n            :obj:`\"crocodile\"`, :obj:`\"squirrel\"`).\n        geom_gcn_preprocess (bool): If set to :obj:`True`, will load the\n            pre-processed data as introduced in the `\"Geom-GCN: Geometric\n            Graph Convolutional Networks\" <https://arxiv.org/abs/2002.05287>_`,\n            in which the average monthly traffic of the web page is converted\n            into five categories to predict.\n            If set to :obj:`True`, the dataset :obj:`\"crocodile\"` is not\n            available.\n            If set to :obj:`True`, train/validation/test splits will be\n            available as masks for multiple splits with shape\n            :obj:`[num_nodes, num_splits]`. (default: :obj:`True`)\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n\n    \"\"\"\n\n    raw_url = 'https://graphmining.ai/datasets/ptg/wiki'\n    processed_url = ('https://raw.githubusercontent.com/graphdml-uiuc-jlu/'\n                     'geom-gcn/f1fc0d14b3b019c562737240d06ec83b07d16a8f')\n\n    def __init__(\n        self,\n        root: str,\n        name: str,\n        geom_gcn_preprocess: bool = True,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        self.name = name.lower()\n        self.geom_gcn_preprocess = geom_gcn_preprocess\n        assert self.name in ['chameleon', 'crocodile', 'squirrel']\n        if geom_gcn_preprocess and self.name == 'crocodile':\n            raise AttributeError(\"The dataset 'crocodile' is not available in \"\n                                 \"case 'geom_gcn_preprocess=True'\")\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_dir(self) -> str:\n        if self.geom_gcn_preprocess:\n            return osp.join(self.root, self.name, 'geom_gcn', 'raw')\n        else:\n            return osp.join(self.root, self.name, 'raw')\n\n    @property\n    def processed_dir(self) -> str:\n        if self.geom_gcn_preprocess:\n            return osp.join(self.root, self.name, 'geom_gcn', 'processed')\n        else:\n            return osp.join(self.root, self.name, 'processed')\n\n    @property\n    def raw_file_names(self) -> Union[List[str], str]:\n        if self.geom_gcn_preprocess:\n            return (['out1_node_feature_label.txt', 'out1_graph_edges.txt'] +\n                    [f'{self.name}_split_0.6_0.2_{i}.npz' for i in range(10)])\n        else:\n            return f'{self.name}.npz'\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        if self.geom_gcn_preprocess:\n            for filename in self.raw_file_names[:2]:\n...\n", "needle_token_start": 5629, "needle_token_end": 5655, "code_context_ntokens": 16113, "output": ["```python\ndef num_node_labels(self) -> int:\n    return self.sizes['num_node_labels']\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "__collate__", "language": "python", "path": "torch_geometric/loader/shadow.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate mini-batches of subgraphs from a larger graph, focusing on a localized neighborhood around specified nodes. This is particularly useful for training graph neural networks on sections of a graph to reduce computational complexity and focus learning on local structures.\n2. **Input**: A list of node indices for which localized subgraphs are to be generated.\n3. **Output**: A batch object containing the subgraphs corresponding to the input node indices. This batch includes the adjacency information, node features, and any other relevant graph data restricted to the subgraph.\n4. **Procedure**: \n   - Convert the input node indices into a tensor if they are not already.\n   - Extract the adjacency matrix for the specified nodes using a k-hop sampling method, which considers a specified number of neighbor nodes across a defined number of hops.\n   - Construct a new sparse tensor representing the adjacency matrix of the subgraph.\n   - Populate a batch object with the subgraph's data, including node features and edge attributes, ensuring that only data relevant to the subgraph's nodes and edges is included.\n   - Return the populated batch object, ready for use in graph neural network training or evaluation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/loader/neighbor_loader.py\nfrom typing import Callable, Dict, List, Optional, Tuple, Union\n\nfrom torch_geometric.data import Data, FeatureStore, GraphStore, HeteroData\nfrom torch_geometric.loader.node_loader import NodeLoader\nfrom torch_geometric.sampler import NeighborSampler\nfrom torch_geometric.sampler.base import SubgraphType\nfrom torch_geometric.typing import EdgeType, InputNodes, OptTensor\n\n\nclass NeighborLoader(NodeLoader):\n    r\"\"\"A data loader that performs neighbor sampling as introduced in the\n    `\"Inductive Representation Learning on Large Graphs\"\n    <https://arxiv.org/abs/1706.02216>`_ paper.\n    This loader allows for mini-batch training of GNNs on large-scale graphs\n    where full-batch training is not feasible.\n\n    More specifically, :obj:`num_neighbors` denotes how much neighbors are\n    sampled for each node in each iteration.\n    :class:`~torch_geometric.loader.NeighborLoader` takes in this list of\n    :obj:`num_neighbors` and iteratively samples :obj:`num_neighbors[i]` for\n    each node involved in iteration :obj:`i - 1`.\n\n    Sampled nodes are sorted based on the order in which they were sampled.\n    In particular, the first :obj:`batch_size` nodes represent the set of\n    original mini-batch nodes.\n\n    .. code-block:: python\n\n        from torch_geometric.datasets import Planetoid\n        from torch_geometric.loader import NeighborLoader\n\n        data = Planetoid(path, name='Cora')[0]\n\n        loader = NeighborLoader(\n            data,\n            # Sample 30 neighbors for each node for 2 iterations\n            num_neighbors=[30] * 2,\n            # Use a batch size of 128 for sampling training nodes\n            batch_size=128,\n            input_nodes=data.train_mask,\n        )\n\n        sampled_data = next(iter(loader))\n        print(sampled_data.batch_size)\n        >>> 128\n\n    By default, the data loader will only include the edges that were\n    originally sampled (:obj:`directed = True`).\n    This option should only be used in case the number of hops is equivalent to\n    the number of GNN layers.\n    In case the number of GNN layers is greater than the number of hops,\n    consider setting :obj:`directed = False`, which will include all edges\n    between all sampled nodes (but is slightly slower as a result).\n\n    Furthermore, :class:`~torch_geometric.loader.NeighborLoader` works for both\n    **homogeneous** graphs stored via :class:`~torch_geometric.data.Data` as\n    well as **heterogeneous** graphs stored via\n    :class:`~torch_geometric.data.HeteroData`.\n    When operating in heterogeneous graphs, up to :obj:`num_neighbors`\n    neighbors will be sampled for each :obj:`edge_type`.\n    However, more fine-grained control over\n    the amount of sampled neighbors of individual edge types is possible:\n\n    .. code-block:: python\n\n        from torch_geometric.datasets import OGB_MAG\n        from torch_geometric.loader import NeighborLoader\n\n        hetero_data = OGB_MAG(path)[0]\n\n        loader = NeighborLoader(\n            hetero_data,\n            # Sample 30 neighbors for each node and edge type for 2 iterations\n            num_neighbors={key: [30] * 2 for key in hetero_data.edge_types},\n            # Use a batch size of 128 for sampling training nodes of type paper\n            batch_size=128,\n            input_nodes=('paper', hetero_data['paper'].train_mask),\n        )\n\n        sampled_hetero_data = next(iter(loader))\n        print(sampled_hetero_data['paper'].batch_size)\n        >>> 128\n\n    .. note::\n\n        For an example of using\n        :class:`~torch_geometric.loader.NeighborLoader`, see\n        `examples/hetero/to_hetero_mag.py <https://github.com/pyg-team/\n        pytorch_geometric/blob/master/examples/hetero/to_hetero_mag.py>`_.\n\n    The :class:`~torch_geometric.loader.NeighborLoader` will return subgraphs\n    where global node indices are mapped to local indices corresponding to this\n    specific subgraph. However, often times it is desired to map the nodes of\n    the current subgraph back to the global node indices. The\n    :class:`~torch_geometric.loader.NeighborLoader` will include this mapping\n    as part of the :obj:`data` object:\n\n    .. code-block:: python\n\n        loader = NeighborLoader(data, ...)\n        sampled_data = next(iter(loader))\n        print(sampled_data.n_id)  # Global node index of each node in batch.\n\n    In particular, the data loader will add the following attributes to the\n    returned mini-batch:\n\n    * :obj:`batch_size` The number of seed nodes (first nodes in the batch)\n    * :obj:`n_id` The global node index for every sampled node\n    * :obj:`e_id` The global edge index for every sampled edge\n    * :obj:`input_id`: The global index of the :obj:`input_nodes`\n    * :obj:`num_sampled_nodes`: The number of sampled nodes in each hop\n    * :obj:`num_sampled_edges`: The number of sampled edges in each hop\n\n    Args:\n        data (Any): A :class:`~torch_geometric.data.Data`,\n            :class:`~torch_geometric.data.HeteroData`, or\n            (:class:`~torch_geometric.data.FeatureStore`,\n            :class:`~torch_geometric.data.GraphStore`) data object.\n        num_neighbors (List[int] or Dict[Tuple[str, str, str], List[int]]): The\n            number of neighbors to sample for each node in each iteration.\n            If an entry is set to :obj:`-1`, all neighbors will be included.\n            In heterogeneous graphs, may also take in a dictionary denoting\n            the amount of neighbors to sample for each individual edge type.\n        input_nodes (torch.Tensor or str or Tuple[str, torch.Tensor]): The\n            indices of nodes for which neighbors are sampled to create\n            mini-batches.\n            Needs to be either given as a :obj:`torch.LongTensor` or\n            :obj:`torch.BoolTensor`.\n            If set to :obj:`None`, all nodes will be considered.\n            In heterogeneous graphs, needs to be passed as a tuple that holds\n            the node type and node indices. (default: :obj:`None`)\n        input_time (torch.Tensor, optional): Optional values to override the\n            timestamp for the input nodes given in :obj:`input_nodes`. If not\n            set, will use the timestamps in :obj:`time_attr` as default (if\n            present). The :obj:`time_attr` needs to be set for this to work.\n            (default: :obj:`None`)\n        replace (bool, optional): If set to :obj:`True`, will sample with\n...\n# Path: torch_geometric/loader/neighbor_sampler.py\nfrom typing import Callable, List, NamedTuple, Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import SparseTensor\n\n\nclass EdgeIndex(NamedTuple):\n    edge_index: Tensor\n    e_id: Optional[Tensor]\n    size: Tuple[int, int]\n\n    def to(self, *args, **kwargs):\n        edge_index = self.edge_index.to(*args, **kwargs)\n        e_id = self.e_id.to(*args, **kwargs) if self.e_id is not None else None\n        return EdgeIndex(edge_index, e_id, self.size)\n\n\nclass Adj(NamedTuple):\n    adj_t: SparseTensor\n    e_id: Optional[Tensor]\n    size: Tuple[int, int]\n\n    def to(self, *args, **kwargs):\n        adj_t = self.adj_t.to(*args, **kwargs)\n        e_id = self.e_id.to(*args, **kwargs) if self.e_id is not None else None\n        return Adj(adj_t, e_id, self.size)\n\n\nclass NeighborSampler(torch.utils.data.DataLoader):\n    r\"\"\"The neighbor sampler from the `\"Inductive Representation Learning on\n    Large Graphs\" <https://arxiv.org/abs/1706.02216>`_ paper, which allows\n    for mini-batch training of GNNs on large-scale graphs where full-batch\n    training is not feasible.\n\n    Given a GNN with :math:`L` layers and a specific mini-batch of nodes\n    :obj:`node_idx` for which we want to compute embeddings, this module\n    iteratively samples neighbors and constructs bipartite graphs that simulate\n    the actual computation flow of GNNs.\n\n    More specifically, :obj:`sizes` denotes how much neighbors we want to\n    sample for each node in each layer.\n    This module then takes in these :obj:`sizes` and iteratively samples\n    :obj:`sizes[l]` for each node involved in layer :obj:`l`.\n    In the next layer, sampling is repeated for the union of nodes that were\n    already encountered.\n    The actual computation graphs are then returned in reverse-mode, meaning\n    that we pass messages from a larger set of nodes to a smaller one, until we\n    reach the nodes for which we originally wanted to compute embeddings.\n\n    Hence, an item returned by :class:`NeighborSampler` holds the current\n    :obj:`batch_size`, the IDs :obj:`n_id` of all nodes involved in the\n    computation, and a list of bipartite graph objects via the tuple\n    :obj:`(edge_index, e_id, size)`, where :obj:`edge_index` represents the\n    bipartite edges between source and target nodes, :obj:`e_id` denotes the\n    IDs of original edges in the full graph, and :obj:`size` holds the shape\n    of the bipartite graph.\n    For each bipartite graph, target nodes are also included at the beginning\n    of the list of source nodes so that one can easily apply skip-connections\n    or add self-loops.\n\n    .. warning::\n\n        :class:`~torch_geometric.loader.NeighborSampler` is deprecated and will\n        be removed in a future release.\n        Use :class:`torch_geometric.loader.NeighborLoader` instead.\n\n    .. note::\n\n        For an example of using :obj:`NeighborSampler`, see\n        `examples/reddit.py\n        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/\n        reddit.py>`_ or\n        `examples/ogbn_products_sage.py\n        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/\n        ogbn_products_sage.py>`_.\n\n    Args:\n        edge_index (Tensor or SparseTensor): A :obj:`torch.LongTensor` or a\n            :class:`torch_sparse.SparseTensor` that defines the underlying\n            graph connectivity/message passing flow.\n            :obj:`edge_index` holds the indices of a (sparse) symmetric\n            adjacency matrix.\n            If :obj:`edge_index` is of type :obj:`torch.LongTensor`, its shape\n            must be defined as :obj:`[2, num_edges]`, where messages from nodes\n            :obj:`edge_index[0]` are sent to nodes in :obj:`edge_index[1]`\n            (in case :obj:`flow=\"source_to_target\"`).\n            If :obj:`edge_index` is of type :class:`torch_sparse.SparseTensor`,\n            its sparse indices :obj:`(row, col)` should relate to\n            :obj:`row = edge_index[1]` and :obj:`col = edge_index[0]`.\n            The major difference between both formats is that we need to input\n            the *transposed* sparse adjacency matrix.\n        sizes ([int]): The number of neighbors to sample for each node in each\n            layer. If set to :obj:`sizes[l] = -1`, all neighbors are included\n            in layer :obj:`l`.\n        node_idx (LongTensor, optional): The nodes that should be considered\n            for creating mini-batches. If set to :obj:`None`, all nodes will be\n            considered.\n        num_nodes (int, optional): The number of nodes in the graph.\n            (default: :obj:`None`)\n        return_e_id (bool, optional): If set to :obj:`False`, will not return\n            original edge indices of sampled edges. This is only useful in case\n            when operating on graphs without edge features to save memory.\n            (default: :obj:`True`)\n        transform (callable, optional): A function/transform that takes in\n            a sampled mini-batch and returns a transformed version.\n            (default: :obj:`None`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`batch_size`,\n            :obj:`shuffle`, :obj:`drop_last` or :obj:`num_workers`.\n    \"\"\"\n    def __init__(self, edge_index: Union[Tensor, SparseTensor],\n                 sizes: List[int], node_idx: Optional[Tensor] = None,\n                 num_nodes: Optional[int] = None, return_e_id: bool = True,\n                 transform: Callable = None, **kwargs):\n\n        edge_index = edge_index.to('cpu')\n\n        # Remove for PyTorch Lightning:\n        kwargs.pop('dataset', None)\n        kwargs.pop('collate_fn', None)\n\n        # Save for Pytorch Lightning < 1.6:\n        self.edge_index = edge_index\n        self.node_idx = node_idx\n        self.num_nodes = num_nodes\n\n        self.sizes = sizes\n        self.return_e_id = return_e_id\n        self.transform = transform\n        self.is_sparse_tensor = isinstance(edge_index, SparseTensor)\n        self.__val__ = None\n\n        # Obtain a *transposed* `SparseTensor` instance.\n        if not self.is_sparse_tensor:\n            if (num_nodes is None and node_idx is not None\n                    and node_idx.dtype == torch.bool):\n                num_nodes = node_idx.size(0)\n            if (num_nodes is None and node_idx is not None\n                    and node_idx.dtype == torch.long):\n                num_nodes = max(int(edge_index.max()), int(node_idx.max())) + 1\n            if num_nodes is None:\n                num_nodes = int(edge_index.max()) + 1\n\n            value = torch.arange(edge_index.size(1)) if return_e_id else None\n            self.adj_t = SparseTensor(row=edge_index[0], col=edge_index[1],\n                                      value=value,\n                                      sparse_sizes=(num_nodes, num_nodes)).t()\n        else:\n            adj_t = edge_index\n            if return_e_id:\n                self.__val__ = adj_t.storage.value()\n                value = torch.arange(adj_t.nnz())\n                adj_t = adj_t.set_value(value, layout='coo')\n            self.adj_t = adj_t\n\n        self.adj_t.storage.rowptr()\n\n        if node_idx is None:\n            node_idx = torch.arange(self.adj_t.sparse_size(0))\n        elif node_idx.dtype == torch.bool:\n            node_idx = node_idx.nonzero(as_tuple=False).view(-1)\n\n        super().__init__(\n            node_idx.view(-1).tolist(), collate_fn=self.sample, **kwargs)\n\n    def sample(self, batch):\n        if not isinstance(batch, Tensor):\n            batch = torch.tensor(batch)\n\n        batch_size: int = len(batch)\n\n        adjs = []\n        n_id = batch\n        for size in self.sizes:\n            adj_t, n_id = self.adj_t.sample_adj(n_id, size, replace=False)\n            e_id = adj_t.storage.value()\n            size = adj_t.sparse_sizes()[::-1]\n            if self.__val__ is not None:\n                adj_t.set_value_(self.__val__[e_id], layout='coo')\n\n            if self.is_sparse_tensor:\n                adjs.append(Adj(adj_t, e_id, size))\n            else:\n                row, col, _ = adj_t.coo()\n                edge_index = torch.stack([col, row], dim=0)\n                adjs.append(EdgeIndex(edge_index, e_id, size))\n\n        adjs = adjs[0] if len(adjs) == 1 else adjs[::-1]\n        out = (batch_size, n_id, adjs)\n        out = self.transform(*out) if self.transform is not None else out\n        return out\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}(sizes={self.sizes})'\n\n# Path: torch_geometric/loader/prefetch.py\nimport warnings\nfrom contextlib import nullcontext\nfrom functools import partial\nfrom typing import Any, Optional\n\nimport torch\nfrom torch.utils.data import DataLoader\n\nfrom torch_geometric.typing import WITH_IPEX\n\n\nclass DeviceHelper:\n    def __init__(self, device: Optional[torch.device] = None):\n        with_cuda = torch.cuda.is_available()\n        with_xpu = torch.xpu.is_available() if WITH_IPEX else False\n\n        if device is None:\n            if with_cuda:\n                device = 'cuda'\n            elif with_xpu:\n                device = 'xpu'\n            else:\n                device = 'cpu'\n\n        self.device = torch.device(device)\n        self.is_gpu = self.device.type in ['cuda', 'xpu']\n\n        if ((self.device.type == 'cuda' and not with_cuda)\n                or (self.device.type == 'xpu' and not with_xpu)):\n            warnings.warn(f\"Requested device '{self.device.type}' is not \"\n                          f\"available, falling back to CPU\")\n            self.device = torch.device('cpu')\n\n        self.stream = None\n        self.stream_context = nullcontext\n        self.module = getattr(torch, self.device.type) if self.is_gpu else None\n\n    def maybe_init_stream(self) -> None:\n        if self.is_gpu:\n            self.stream = self.module.Stream()\n            self.stream_context = partial(\n                self.module.stream,\n                stream=self.stream,\n            )\n\n    def maybe_wait_stream(self) -> None:\n        if self.stream is not None:\n            self.module.current_stream().wait_stream(self.stream)\n\n\nclass PrefetchLoader:\n    r\"\"\"A GPU prefetcher class for asynchronously transferring data of a\n    :class:`torch.utils.data.DataLoader` from host memory to device memory.\n\n    Args:\n        loader (torch.utils.data.DataLoader): The data loader.\n        device (torch.device, optional): The device to load the data to.\n            (default: :obj:`None`)\n    \"\"\"\n    def __init__(\n        self,\n        loader: DataLoader,\n        device: Optional[torch.device] = None,\n    ):\n        self.loader = loader\n        self.device_helper = DeviceHelper(device)\n\n    def non_blocking_transfer(self, batch: Any) -> Any:\n        if not self.device_helper.is_gpu:\n            return batch\n        if isinstance(batch, (list, tuple)):\n            return [self.non_blocking_transfer(v) for v in batch]\n        if isinstance(batch, dict):\n            return {k: self.non_blocking_transfer(v) for k, v in batch.items()}\n\n        batch = batch.pin_memory(self.device_helper.device)\n        return batch.to(self.device_helper.device, non_blocking=True)\n\n    def __iter__(self) -> Any:\n        first = True\n        self.device_helper.maybe_init_stream()\n\n        batch = None\n        for next_batch in self.loader:\n\n            with self.device_helper.stream_context():\n                next_batch = self.non_blocking_transfer(next_batch)\n\n            if not first:\n                yield batch\n            else:\n                first = False\n\n            self.device_helper.maybe_wait_stream()\n\n            batch = next_batch\n\n        yield batch\n\n    def __len__(self) -> int:\n        return len(self.loader)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}({self.loader})'\n\n# Path: torch_geometric/loader/random_node_loader.py\nimport math\nfrom typing import Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.data.hetero_data import to_homogeneous_edge_index\n\n\nclass RandomNodeLoader(torch.utils.data.DataLoader):\n    r\"\"\"A data loader that randomly samples nodes within a graph and returns\n    their induced subgraph.\n\n    .. note::\n\n        For an example of using\n        :class:`~torch_geometric.loader.RandomNodeLoader`, see\n        `examples/ogbn_proteins_deepgcn.py\n        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/\n        ogbn_proteins_deepgcn.py>`_.\n\n    Args:\n        data (torch_geometric.data.Data or torch_geometric.data.HeteroData):\n            The :class:`~torch_geometric.data.Data` or\n            :class:`~torch_geometric.data.HeteroData` graph object.\n        num_parts (int): The number of partitions.\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`num_workers`.\n    \"\"\"\n    def __init__(\n        self,\n        data: Union[Data, HeteroData],\n        num_parts: int,\n        **kwargs,\n    ):\n        self.data = data\n        self.num_parts = num_parts\n\n        if isinstance(data, HeteroData):\n            edge_index, node_dict, edge_dict = to_homogeneous_edge_index(data)\n            self.node_dict, self.edge_dict = node_dict, edge_dict\n        else:\n            edge_index = data.edge_index\n\n        self.edge_index = edge_index\n        self.num_nodes = data.num_nodes\n\n        super().__init__(\n            range(self.num_nodes),\n            batch_size=math.ceil(self.num_nodes / num_parts),\n            collate_fn=self.collate_fn,\n            **kwargs,\n        )\n\n    def collate_fn(self, index):\n        if not isinstance(index, Tensor):\n            index = torch.tensor(index)\n\n        if isinstance(self.data, Data):\n            return self.data.subgraph(index)\n\n        elif isinstance(self.data, HeteroData):\n            node_dict = {\n                key: index[(index >= start) & (index < end)] - start\n                for key, (start, end) in self.node_dict.items()\n            }\n            return self.data.subgraph(node_dict)\n\n# Path: torch_geometric/loader/shadow.py\nimport copy\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Batch, Data\nfrom torch_geometric.typing import WITH_TORCH_SPARSE, SparseTensor\n\n\nclass ShaDowKHopSampler(torch.utils.data.DataLoader):\n    r\"\"\"The ShaDow :math:`k`-hop sampler from the `\"Decoupling the Depth and\n    Scope of Graph Neural Networks\" <https://arxiv.org/abs/2201.07858>`_ paper.\n    Given a graph in a :obj:`data` object, the sampler will create shallow,\n    localized subgraphs.\n    A deep GNN on this local graph then smooths the informative local signals.\n\n    .. note::\n\n        For an example of using :class:`ShaDowKHopSampler`, see\n        `examples/shadow.py <https://github.com/pyg-team/\n        pytorch_geometric/blob/master/examples/shadow.py>`_.\n\n    Args:\n        data (torch_geometric.data.Data): The graph data object.\n        depth (int): The depth/number of hops of the localized subgraph.\n        num_neighbors (int): The number of neighbors to sample for each node in\n            each hop.\n        node_idx (LongTensor or BoolTensor, optional): The nodes that should be\n            considered for creating mini-batches.\n            If set to :obj:`None`, all nodes will be\n            considered.\n        replace (bool, optional): If set to :obj:`True`, will sample neighbors\n            with replacement. (default: :obj:`False`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`batch_size` or\n            :obj:`num_workers`.\n    \"\"\"\n    def __init__(self, data: Data, depth: int, num_neighbors: int,\n                 node_idx: Optional[Tensor] = None, replace: bool = False,\n                 **kwargs):\n\n        if not WITH_TORCH_SPARSE:\n            raise ImportError(\n                f\"'{self.__class__.__name__}' requires 'torch-sparse'\")\n\n        self.data = copy.copy(data)\n        self.depth = depth\n        self.num_neighbors = num_neighbors\n        self.replace = replace\n\n        if data.edge_index is not None:\n            self.is_sparse_tensor = False\n            row, col = data.edge_index.cpu()\n            self.adj_t = SparseTensor(\n                row=row, col=col, value=torch.arange(col.size(0)),\n                sparse_sizes=(data.num_nodes, data.num_nodes)).t()\n        else:\n            self.is_sparse_tensor = True\n            self.adj_t = data.adj_t.cpu()\n\n        if node_idx is None:\n            node_idx = torch.arange(self.adj_t.sparse_size(0))\n        elif node_idx.dtype == torch.bool:\n            node_idx = node_idx.nonzero(as_tuple=False).view(-1)\n        self.node_idx = node_idx\n\n        super().__init__(node_idx.tolist(), collate_fn=self.__collate__,\n                         **kwargs)\n\n    \ndef __collate__(self, n_id):\n        n_id = torch.tensor(n_id)\n\n        rowptr, col, value = self.adj_t.csr()\n        out = torch.ops.torch_sparse.ego_k_hop_sample_adj(\n            rowptr, col, n_id, self.depth, self.num_neighbors, self.replace)\n        rowptr, col, n_id, e_id, ptr, root_n_id = out\n\n        adj_t = SparseTensor(rowptr=rowptr, col=col,\n                             value=value[e_id] if value is not None else None,\n                             sparse_sizes=(n_id.numel(), n_id.numel()),\n                             is_sorted=True, trust_data=True)\n\n        batch = Batch(batch=torch.ops.torch_sparse.ptr2ind(ptr, n_id.numel()),\n                      ptr=ptr)\n        batch.root_n_id = root_n_id\n\n        if self.is_sparse_tensor:\n            batch.adj_t = adj_t\n        else:\n            row, col, e_id = adj_t.t().coo()\n            batch.edge_index = torch.stack([row, col], dim=0)\n\n        for k, v in self.data:\n            if k in ['edge_index', 'adj_t', 'num_nodes', 'batch', 'ptr']:\n                continue\n            if k == 'y' and v.size(0) == self.data.num_nodes:\n                batch[k] = v[n_id][root_n_id]\n            elif isinstance(v, Tensor) and v.size(0) == self.data.num_nodes:\n                batch[k] = v[n_id]\n            elif isinstance(v, Tensor) and v.size(0) == self.data.num_edges:\n                batch[k] = v[e_id]\n            else:\n                batch[k] = v\n\n        return batch\n\n# Path: torch_geometric/loader/temporal_dataloader.py\nfrom typing import List\n\nimport torch\n\nfrom torch_geometric.data import TemporalData\n\n\nclass TemporalDataLoader(torch.utils.data.DataLoader):\n    r\"\"\"A data loader which merges succesive events of a\n    :class:`torch_geometric.data.TemporalData` to a mini-batch.\n\n    Args:\n        data (TemporalData): The :obj:`~torch_geometric.data.TemporalData`\n            from which to load the data.\n        batch_size (int, optional): How many samples per batch to load.\n            (default: :obj:`1`)\n        neg_sampling_ratio (float, optional): The ratio of sampled negative\n            destination nodes to the number of postive destination nodes.\n            (default: :obj:`0.0`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`.\n    \"\"\"\n    def __init__(\n        self,\n        data: TemporalData,\n        batch_size: int = 1,\n        neg_sampling_ratio: float = 0.0,\n        **kwargs,\n    ):\n        # Remove for PyTorch Lightning:\n        kwargs.pop('dataset', None)\n        kwargs.pop('collate_fn', None)\n        kwargs.pop('shuffle', None)\n\n        self.data = data\n        self.events_per_batch = batch_size\n        self.neg_sampling_ratio = neg_sampling_ratio\n\n        if neg_sampling_ratio > 0:\n            self.min_dst = int(data.dst.min())\n            self.max_dst = int(data.dst.max())\n\n        if kwargs.get('drop_last', False) and len(data) % batch_size != 0:\n            arange = range(0, len(data) - batch_size, batch_size)\n        else:\n            arange = range(0, len(data), batch_size)\n\n        super().__init__(arange, 1, shuffle=False, collate_fn=self, **kwargs)\n\n    def __call__(self, arange: List[int]) -> TemporalData:\n        batch = self.data[arange[0]:arange[0] + self.events_per_batch]\n\n        n_ids = [batch.src, batch.dst]\n\n        if self.neg_sampling_ratio > 0:\n            batch.neg_dst = torch.randint(\n                low=self.min_dst,\n                high=self.max_dst + 1,\n                size=(round(self.neg_sampling_ratio * batch.dst.size(0)), ),\n                dtype=batch.dst.dtype,\n                device=batch.dst.device,\n            )\n            n_ids += [batch.neg_dst]\n\n        batch.n_id = torch.cat(n_ids, dim=0).unique()\n\n        return batch\n\n# Path: torch_geometric/loader/zip_loader.py\nfrom typing import Any, Iterator, List, Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data, HeteroData\nfrom torch_geometric.loader import LinkLoader, NodeLoader\nfrom torch_geometric.loader.base import DataLoaderIterator\nfrom torch_geometric.loader.utils import infer_filter_per_worker\n\n\nclass ZipLoader(torch.utils.data.DataLoader):\n    r\"\"\"A loader that returns a tuple of data objects by sampling from multiple\n    :class:`NodeLoader` or :class:`LinkLoader` instances.\n\n    Args:\n        loaders (List[NodeLoader] or List[LinkLoader]): The loader instances.\n        filter_per_worker (bool, optional): If set to :obj:`True`, will filter\n            the returned data in each worker's subprocess.\n            If set to :obj:`False`, will filter the returned data in the main\n            process.\n            If set to :obj:`None`, will automatically infer the decision based\n            on whether data partially lives on the GPU\n            (:obj:`filter_per_worker=True`) or entirely on the CPU\n            (:obj:`filter_per_worker=False`).\n            There exists different trade-offs for setting this option.\n            Specifically, setting this option to :obj:`True` for in-memory\n            datasets will move all features to shared memory, which may result\n            in too many open file handles. (default: :obj:`None`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`batch_size`,\n            :obj:`shuffle`, :obj:`drop_last` or :obj:`num_workers`.\n    \"\"\"\n    def __init__(\n        self,\n        loaders: Union[List[NodeLoader], List[LinkLoader]],\n        filter_per_worker: Optional[bool] = None,\n        **kwargs,\n    ):\n        if filter_per_worker is None:\n            filter_per_worker = infer_filter_per_worker(loaders[0].data)\n\n        # Remove for PyTorch Lightning:\n        kwargs.pop('dataset', None)\n        kwargs.pop('collate_fn', None)\n\n        for loader in loaders:\n            if not callable(getattr(loader, 'collate_fn', None)):\n                raise ValueError(\"'{loader.__class__.__name__}' does not have \"\n                                 \"a 'collate_fn' method\")\n            if not callable(getattr(loader, 'filter_fn', None)):\n                raise ValueError(\"'{loader.__class__.__name__}' does not have \"\n                                 \"a 'filter_fn' method\")\n            loader.filter_per_worker = filter_per_worker\n\n        iterator = range(min([len(loader.dataset) for loader in loaders]))\n        super().__init__(iterator, collate_fn=self.collate_fn, **kwargs)\n\n        self.loaders = loaders\n        self.filter_per_worker = filter_per_worker\n\n    def collate_fn(self, index: List[int]) -> Tuple[Any, ...]:\n        if not isinstance(index, Tensor):\n            index = torch.tensor(index, dtype=torch.long)\n\n        return tuple(loader.collate_fn(index) for loader in self.loaders)\n\n    def filter_fn(\n        self,\n        outs: Tuple[Any, ...],\n    ) -> Tuple[Union[Data, HeteroData], ...]:\n        loaders = self.loaders\n        return tuple(loader.filter_fn(v) for loader, v in zip(loaders, outs))\n\n    def _get_iterator(self) -> Iterator:\n        if self.filter_per_worker:\n            return super()._get_iterator()\n\n        # Execute `filter_fn` in the main process:\n        return DataLoaderIterator(super()._get_iterator(), self.filter_fn)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}(loaders={self.loaders})'\n\n# Path: torch_geometric/loader/__init__.py\nfrom torch_geometric.deprecation import deprecated\n\nfrom .dataloader import DataLoader\nfrom .node_loader import NodeLoader\nfrom .link_loader import LinkLoader\nfrom .neighbor_loader import NeighborLoader\nfrom .link_neighbor_loader import LinkNeighborLoader\nfrom .hgt_loader import HGTLoader\nfrom .cluster import ClusterData, ClusterLoader\nfrom .graph_saint import (GraphSAINTSampler, GraphSAINTNodeSampler,\n                          GraphSAINTEdgeSampler, GraphSAINTRandomWalkSampler)\nfrom .shadow import ShaDowKHopSampler\nfrom .random_node_loader import RandomNodeLoader\n# from .ibmb_loader import IBMBBatchLoader, IBMBNodeLoader\nfrom .zip_loader import ZipLoader\nfrom .data_list_loader import DataListLoader\nfrom .dense_data_loader import DenseDataLoader\nfrom .temporal_dataloader import TemporalDataLoader\nfrom .neighbor_sampler import NeighborSampler\nfrom .imbalanced_sampler import ImbalancedSampler\nfrom .dynamic_batch_sampler import DynamicBatchSampler\nfrom .prefetch import PrefetchLoader\nfrom .cache import CachedLoader\nfrom .mixin import AffinityMixin\n\n__all__ = classes = [\n    'DataLoader',\n    'NodeLoader',\n    'LinkLoader',\n    'NeighborLoader',\n    'LinkNeighborLoader',\n    'HGTLoader',\n    'ClusterData',\n    'ClusterLoader',\n    'GraphSAINTSampler',\n    'GraphSAINTNodeSampler',\n    'GraphSAINTEdgeSampler',\n    'GraphSAINTRandomWalkSampler',\n    'ShaDowKHopSampler',\n    'RandomNodeLoader',\n    # 'IBMBBatchLoader',\n    # 'IBMBNodeLoader',\n    'ZipLoader',\n    'DataListLoader',\n    'DenseDataLoader',\n    'TemporalDataLoader',\n    'NeighborSampler',\n    'ImbalancedSampler',\n    'DynamicBatchSampler',\n    'PrefetchLoader',\n    'CachedLoader',\n    'AffinityMixin',\n]\n\nRandomNodeSampler = deprecated(\n    details=\"use 'loader.RandomNodeLoader' instead\",\n    func_name='loader.RandomNodeSampler',\n)(RandomNodeLoader)\n\n# Path: torch_geometric/data/__init__.py\n# flake8: noqa\n\nfrom .feature_store import FeatureStore, TensorAttr\nfrom .graph_store import GraphStore, EdgeAttr\nfrom .data import Data\nfrom .hetero_data import HeteroData\nfrom .batch import Batch\nfrom .temporal import TemporalData\nfrom .database import Database, SQLiteDatabase, RocksDatabase\nfrom .dataset import Dataset\nfrom .in_memory_dataset import InMemoryDataset\nfrom .on_disk_dataset import OnDiskDataset\nfrom .makedirs import makedirs\nfrom .download import download_url, download_google_url\nfrom .extract import extract_tar, extract_zip, extract_bz2, extract_gz\n\nfrom torch_geometric.lazy_loader import LazyLoader\n\ndata_classes = [\n    'Data',\n    'HeteroData',\n    'Batch',\n    'TemporalData',\n    'Dataset',\n    'InMemoryDataset',\n    'OnDiskDataset',\n]\n\nremote_backend_classes = [\n    'FeatureStore',\n    'GraphStore',\n    'TensorAttr',\n    'EdgeAttr',\n]\n\ndatabase_classes = [\n    'Database',\n    'SQLiteDatabase',\n    'RocksDatabase',\n]\n\nhelper_functions = [\n    'makedirs',\n    'download_url',\n    'download_google_url',\n    'extract_tar',\n    'extract_zip',\n    'extract_bz2',\n    'extract_gz',\n]\n\n__all__ = data_classes + remote_backend_classes + helper_functions\n\nlightning = LazyLoader('lightning', globals(),\n                       'torch_geometric.data.lightning')\n\nfrom torch_geometric.deprecation import deprecated\nfrom torch_geometric.loader import NeighborSampler\nfrom torch_geometric.loader import ClusterData\nfrom torch_geometric.loader import ClusterLoader\nfrom torch_geometric.loader import GraphSAINTSampler\nfrom torch_geometric.loader import GraphSAINTNodeSampler\nfrom torch_geometric.loader import GraphSAINTEdgeSampler\nfrom torch_geometric.loader import GraphSAINTRandomWalkSampler\nfrom torch_geometric.loader import ShaDowKHopSampler\nfrom torch_geometric.loader import RandomNodeLoader\nfrom torch_geometric.loader import DataLoader\nfrom torch_geometric.loader import DataListLoader\nfrom torch_geometric.loader import DenseDataLoader\n\nNeighborSampler = deprecated(  # type: ignore\n    details=\"use 'loader.NeighborSampler' instead\",\n    func_name='data.NeighborSampler',\n)(NeighborSampler)\nClusterData = deprecated(  # type: ignore\n    details=\"use 'loader.ClusterData' instead\",\n    func_name='data.ClusterData',\n)(ClusterData)\nClusterLoader = deprecated(  # type: ignore\n    details=\"use 'loader.ClusterLoader' instead\",\n    func_name='data.ClusterLoader',\n)(ClusterLoader)\nGraphSAINTSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTSampler' instead\",\n    func_name='data.GraphSAINTSampler',\n)(GraphSAINTSampler)\nGraphSAINTNodeSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTNodeSampler' instead\",\n    func_name='data.GraphSAINTNodeSampler',\n)(GraphSAINTNodeSampler)\nGraphSAINTEdgeSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTEdgeSampler' instead\",\n    func_name='data.GraphSAINTEdgeSampler',\n)(GraphSAINTEdgeSampler)\nGraphSAINTRandomWalkSampler = deprecated(  # type: ignore\n    details=\"use 'loader.GraphSAINTRandomWalkSampler' instead\",\n    func_name='data.GraphSAINTRandomWalkSampler',\n)(GraphSAINTRandomWalkSampler)\nShaDowKHopSampler = deprecated(  # type: ignore\n    details=\"use 'loader.ShaDowKHopSampler' instead\",\n    func_name='data.ShaDowKHopSampler',\n)(ShaDowKHopSampler)\nRandomNodeSampler = deprecated(\n    details=\"use 'loader.RandomNodeLoader' instead\",\n    func_name='data.RandomNodeSampler',\n)(RandomNodeLoader)\nDataLoader = deprecated(  # type: ignore\n    details=\"use 'loader.DataLoader' instead\",\n    func_name='data.DataLoader',\n)(DataLoader)\nDataListLoader = deprecated(  # type: ignore\n    details=\"use 'loader.DataListLoader' instead\",\n    func_name='data.DataListLoader',\n)(DataListLoader)\nDenseDataLoader = deprecated(  # type: ignore\n    details=\"use 'loader.DenseDataLoader' instead\",\n    func_name='data.DenseDataLoader',\n)(DenseDataLoader)\n\n# Path: torch_geometric/visualization/graph.py\nfrom math import sqrt\nfrom typing import Any, List, Optional\n\nimport torch\nfrom torch import Tensor\n\nBACKENDS = {'graphviz', 'networkx'}\n\n\ndef has_graphviz() -> bool:\n    try:\n        import graphviz\n    except ImportError:\n        return False\n\n    try:\n        graphviz.Digraph().pipe()\n    except graphviz.backend.ExecutableNotFound:\n        return False\n\n    return True\n\n\ndef visualize_graph(\n    edge_index: Tensor,\n    edge_weight: Optional[Tensor] = None,\n    path: Optional[str] = None,\n    backend: Optional[str] = None,\n    node_labels: Optional[List[str]] = None,\n) -> Any:\n    r\"\"\"Visualizes the graph given via :obj:`edge_index` and (optional)\n    :obj:`edge_weight`.\n\n    Args:\n        edge_index (torch.Tensor): The edge indices.\n        edge_weight (torch.Tensor, optional): The edge weights.\n        path (str, optional): The path to where the plot is saved.\n            If set to :obj:`None`, will visualize the plot on-the-fly.\n            (default: :obj:`None`)\n        backend (str, optional): The graph drawing backend to use for\n            visualization (:obj:`\"graphviz\"`, :obj:`\"networkx\"`).\n            If set to :obj:`None`, will use the most appropriate\n            visualization backend based on available system packages.\n            (default: :obj:`None`)\n        node_labels (List[str], optional): The labels/IDs of nodes.\n            (default: :obj:`None`)\n    \"\"\"\n    if edge_weight is not None:  # Normalize edge weights.\n        edge_weight = edge_weight - edge_weight.min()\n        edge_weight = edge_weight / edge_weight.max()\n\n    if edge_weight is not None:  # Discard any edges with zero edge weight:\n        mask = edge_weight > 1e-7\n        edge_index = edge_index[:, mask]\n        edge_weight = edge_weight[mask]\n\n    if edge_weight is None:\n        edge_weight = torch.ones(edge_index.size(1))\n\n    if backend is None:\n        backend = 'graphviz' if has_graphviz() else 'networkx'\n\n    if backend.lower() == 'networkx':\n        return _visualize_graph_via_networkx(edge_index, edge_weight, path,\n                                             node_labels)\n    elif backend.lower() == 'graphviz':\n        return _visualize_graph_via_graphviz(edge_index, edge_weight, path,\n                                             node_labels)\n\n    raise ValueError(f\"Expected graph drawing backend to be in \"\n                     f\"{BACKENDS} (got '{backend}')\")\n\n\ndef _visualize_graph_via_graphviz(\n    edge_index: Tensor,\n    edge_weight: Tensor,\n    path: Optional[str] = None,\n    node_labels: Optional[List[str]] = None,\n) -> Any:\n    import graphviz\n\n    suffix = path.split('.')[-1] if path is not None else None\n    g = graphviz.Digraph('graph', format=suffix)\n    g.attr('node', shape='circle', fontsize='11pt')\n\n    for node in edge_index.view(-1).unique().tolist():\n        g.node(str(node) if node_labels is None else node_labels[node])\n\n    for (src, dst), w in zip(edge_index.t().tolist(), edge_weight.tolist()):\n        hex_color = hex(255 - round(255 * w))[2:]\n        hex_color = f'{hex_color}0' if len(hex_color) == 1 else hex_color\n        if node_labels is not None:\n            src = node_labels[src]\n            dst = node_labels[dst]\n        g.edge(str(src), str(dst), color=f'#{hex_color}{hex_color}{hex_color}')\n\n    if path is not None:\n        path = '.'.join(path.split('.')[:-1])\n        g.render(path, cleanup=True)\n    else:\n        g.view()\n\n    return g\n\n\ndef _visualize_graph_via_networkx(\n    edge_index: Tensor,\n    edge_weight: Tensor,\n    path: Optional[str] = None,\n    node_labels: Optional[List[str]] = None,\n) -> Any:\n    import matplotlib.pyplot as plt\n    import networkx as nx\n\n    g = nx.DiGraph()\n    node_size = 800\n\n    for node in edge_index.view(-1).unique().tolist():\n        g.add_node(node if node_labels is None else node_labels[node])\n\n    for (src, dst), w in zip(edge_index.t().tolist(), edge_weight.tolist()):\n        if node_labels is not None:\n            src = node_labels[src]\n            dst = node_labels[dst]\n        g.add_edge(src, dst, alpha=w)\n\n    ax = plt.gca()\n    pos = nx.spring_layout(g)\n    for src, dst, data in g.edges(data=True):\n        ax.annotate(\n            '',\n            xy=pos[src],\n            xytext=pos[dst],\n            arrowprops=dict(\n                arrowstyle=\"->\",\n                alpha=data['alpha'],\n                shrinkA=sqrt(node_size) / 2.0,\n                shrinkB=sqrt(node_size) / 2.0,\n                connectionstyle=\"arc3,rad=0.1\",\n            ),\n        )\n\n    nodes = nx.draw_networkx_nodes(g, pos, node_size=node_size,\n                                   node_color='white', margins=0.1)\n    nodes.set_edgecolor('black')\n    nx.draw_networkx_labels(g, pos, font_size=10)\n\n    if path is not None:\n        plt.savefig(path)\n    else:\n        plt.show()\n\n    plt.close()\n\n# Path: torch_geometric/visualization/influence.py\nfrom typing import Any\n\nimport torch\nfrom torch import Tensor\nfrom torch.autograd import grad\n\n\ndef influence(model: torch.nn.Module, src: Tensor, *args: Any) -> Tensor:\n    x = src.clone().requires_grad_()\n    out = model(x, *args).sum(dim=-1)\n\n    influences = []\n    for j in range(src.size(0)):\n        influence = grad([out[j]], [x], retain_graph=True)[0].abs().sum(dim=-1)\n        influences.append(influence / influence.sum())\n\n    return torch.stack(influences, dim=0)\n\n# Path: torch_geometric/visualization/__init__.py\nr\"\"\"Visualization package.\"\"\"\n\nfrom .graph import visualize_graph\nfrom .influence import influence\n\n__all__ = [\n    'visualize_graph',\n    'influence',\n]\n\n# Path: torch_geometric/explain/explanation.py\nimport copy\nfrom typing import Dict, List, Optional, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data.data import Data, warn_or_raise\nfrom torch_geometric.data.hetero_data import HeteroData\nfrom torch_geometric.explain.config import ThresholdConfig, ThresholdType\nfrom torch_geometric.typing import EdgeType, NodeType\nfrom torch_geometric.visualization import visualize_graph\n\n\nclass ExplanationMixin:\n    @property\n    def available_explanations(self) -> List[str]:\n        \"\"\"Returns the available explanation masks.\"\"\"\n        return [key for key in self.keys() if key.endswith('_mask')]\n\n    def validate_masks(self, raise_on_error: bool = True) -> bool:\n        r\"\"\"Validates the correctness of the :class:`Explanation` masks.\"\"\"\n        status = True\n\n        for store in self.node_stores:\n            if 'node_mask' not in store:\n                continue\n\n            if store.node_mask.dim() != 2:\n                status = False\n                warn_or_raise(\n                    f\"Expected a 'node_mask' with two dimensions (got \"\n                    f\"{store.node_mask.dim()} dimensions)\", raise_on_error)\n\n            if store.node_mask.size(0) not in {1, store.num_nodes}:\n                status = False\n                warn_or_raise(\n                    f\"Expected a 'node_mask' with {store.num_nodes} nodes \"\n                    f\"(got {store.node_mask.size(0)} nodes)\", raise_on_error)\n\n            if 'x' in store:\n                num_features = store.x.size(-1)\n            else:\n                num_features = store.node_mask.size(-1)\n\n            if store.node_mask.size(1) not in {1, num_features}:\n                status = False\n                warn_or_raise(\n                    f\"Expected a 'node_mask' with {num_features} features (\"\n                    f\"got {store.node_mask.size(1)} features)\", raise_on_error)\n\n        for store in self.edge_stores:\n            if 'edge_mask' not in store:\n                continue\n\n            if store.edge_mask.dim() != 1:\n                status = False\n                warn_or_raise(\n                    f\"Expected an 'edge_mask' with one dimension (got \"\n                    f\"{store.edge_mask.dim()} dimensions)\", raise_on_error)\n\n            if store.edge_mask.size(0) != store.num_edges:\n                status = False\n                warn_or_raise(\n                    f\"Expected an 'edge_mask' with {store.num_edges} edges \"\n                    f\"(got {store.edge_mask.size(0)} edges)\", raise_on_error)\n\n        return status\n\n    def _threshold_mask(\n        self,\n        mask: Optional[Tensor],\n        threshold_config: ThresholdConfig,\n    ) -> Optional[Tensor]:\n\n        if mask is None:\n            return None\n\n        if threshold_config.type == ThresholdType.hard:\n            return (mask > threshold_config.value).float()\n\n        if threshold_config.type in [\n                ThresholdType.topk,\n                ThresholdType.topk_hard,\n        ]:\n            if threshold_config.value >= mask.numel():\n                if threshold_config.type == ThresholdType.topk:\n                    return mask\n                else:\n                    return torch.ones_like(mask)\n\n            value, index = torch.topk(\n                mask.flatten(),\n                k=threshold_config.value,\n            )\n\n            out = torch.zeros_like(mask.flatten())\n            if threshold_config.type == ThresholdType.topk:\n                out[index] = value\n            else:\n                out[index] = 1.0\n            return out.view(mask.size())\n\n        assert False\n\n    def threshold(\n        self,\n        *args,\n        **kwargs,\n    ) -> Union['Explanation', 'HeteroExplanation']:\n        \"\"\"Thresholds the explanation masks according to the thresholding\n        method.\n\n        Args:\n            *args: Arguments passed to :class:`ThresholdConfig`.\n            **kwargs: Keyword arguments passed to :class:`ThresholdConfig`.\n        \"\"\"\n        threshold_config = ThresholdConfig.cast(*args, **kwargs)\n\n        if threshold_config is None:\n            return self\n\n        # Avoid modification of the original explanation:\n        out = copy.copy(self)\n\n        for store in out.node_stores:\n            store.node_mask = self._threshold_mask(store.get('node_mask'),\n                                                   threshold_config)\n\n        for store in out.edge_stores:\n            store.edge_mask = self._threshold_mask(store.get('edge_mask'),\n                                                   threshold_config)\n\n        return out\n\n\nclass Explanation(Data, ExplanationMixin):\n    r\"\"\"Holds all the obtained explanations of a homogeneous graph.\n\n    The explanation object is a :obj:`~torch_geometric.data.Data` object and\n    can hold node attributions and edge attributions.\n    It can also hold the original graph if needed.\n\n    Args:\n        node_mask (Tensor, optional): Node-level mask with shape\n            :obj:`[num_nodes, 1]`, :obj:`[1, num_features]` or\n            :obj:`[num_nodes, num_features]`. (default: :obj:`None`)\n        edge_mask (Tensor, optional): Edge-level mask with shape\n            :obj:`[num_edges]`. (default: :obj:`None`)\n        **kwargs (optional): Additional attributes.\n    \"\"\"\n    def validate(self, raise_on_error: bool = True) -> bool:\n        r\"\"\"Validates the correctness of the :class:`Explanation` object.\"\"\"\n        status = super().validate(raise_on_error)\n        status &= self.validate_masks(raise_on_error)\n        return status\n\n    def get_explanation_subgraph(self) -> 'Explanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with\n        zero attribution are masked out.\n        \"\"\"\n        node_mask = self.get('node_mask')\n        if node_mask is not None:\n            node_mask = node_mask.sum(dim=-1) > 0\n        edge_mask = self.get('edge_mask')\n        if edge_mask is not None:\n            edge_mask = edge_mask > 0\n        return self._apply_masks(node_mask, edge_mask)\n\n    def get_complement_subgraph(self) -> 'Explanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with any\n        attribution are masked out.\n        \"\"\"\n        node_mask = self.get('node_mask')\n        if node_mask is not None:\n            node_mask = node_mask.sum(dim=-1) == 0\n        edge_mask = self.get('edge_mask')\n        if edge_mask is not None:\n            edge_mask = edge_mask == 0\n        return self._apply_masks(node_mask, edge_mask)\n\n    def _apply_masks(\n        self,\n        node_mask: Optional[Tensor] = None,\n        edge_mask: Optional[Tensor] = None,\n    ) -> 'Explanation':\n        out = copy.copy(self)\n\n        if edge_mask is not None:\n            for key, value in self.items():\n                if key == 'edge_index':\n                    out.edge_index = value[:, edge_mask]\n                elif self.is_edge_attr(key):\n                    out[key] = value[edge_mask]\n\n        if node_mask is not None:\n            out = out.subgraph(node_mask)\n\n        return out\n\n    def visualize_feature_importance(\n        self,\n        path: Optional[str] = None,\n        feat_labels: Optional[List[str]] = None,\n        top_k: Optional[int] = None,\n    ):\n        r\"\"\"Creates a bar plot of the node feature importances by summing up\n        the node mask across all nodes.\n\n        Args:\n            path (str, optional): The path to where the plot is saved.\n                If set to :obj:`None`, will visualize the plot on-the-fly.\n                (default: :obj:`None`)\n            feat_labels (List[str], optional): The labels of features.\n                (default :obj:`None`)\n            top_k (int, optional): Top k features to plot. If :obj:`None`\n                plots all features. (default: :obj:`None`)\n        \"\"\"\n        node_mask = self.get('node_mask')\n        if node_mask is None:\n            raise ValueError(f\"The attribute 'node_mask' is not available \"\n                             f\"in '{self.__class__.__name__}' \"\n                             f\"(got {self.available_explanations})\")\n        if node_mask.dim() != 2 or node_mask.size(1) <= 1:\n            raise ValueError(f\"Cannot compute feature importance for \"\n                             f\"object-level 'node_mask' \"\n                             f\"(got shape {node_mask.size()})\")\n\n        if feat_labels is None:\n            feat_labels = range(node_mask.size(1))\n\n        score = node_mask.sum(dim=0)\n\n        return _visualize_score(score, feat_labels, path, top_k)\n\n    def visualize_graph(\n        self,\n        path: Optional[str] = None,\n        backend: Optional[str] = None,\n        node_labels: Optional[List[str]] = None,\n    ) -> None:\n        r\"\"\"Visualizes the explanation graph with edge opacity corresponding to\n        edge importance.\n\n        Args:\n            path (str, optional): The path to where the plot is saved.\n                If set to :obj:`None`, will visualize the plot on-the-fly.\n                (default: :obj:`None`)\n            backend (str, optional): The graph drawing backend to use for\n                visualization (:obj:`\"graphviz\"`, :obj:`\"networkx\"`).\n                If set to :obj:`None`, will use the most appropriate\n                visualization backend based on available system packages.\n                (default: :obj:`None`)\n            node_labels (list[str], optional): The labels/IDs of nodes.\n                (default: :obj:`None`)\n        \"\"\"\n        edge_mask = self.get('edge_mask')\n        if edge_mask is None:\n            raise ValueError(f\"The attribute 'edge_mask' is not available \"\n                             f\"in '{self.__class__.__name__}' \"\n                             f\"(got {self.available_explanations})\")\n        visualize_graph(self.edge_index, edge_mask, path, backend, node_labels)\n\n\nclass HeteroExplanation(HeteroData, ExplanationMixin):\n    r\"\"\"Holds all the obtained explanations of a heterogeneous graph.\n\n    The explanation object is a :obj:`~torch_geometric.data.HeteroData` object\n    and can hold node attributions and edge attributions.\n    It can also hold the original graph if needed.\n    \"\"\"\n    def validate(self, raise_on_error: bool = True) -> bool:\n        r\"\"\"Validates the correctness of the :class:`Explanation` object.\"\"\"\n        status = super().validate(raise_on_error)\n        status &= self.validate_masks(raise_on_error)\n        return status\n\n    def get_explanation_subgraph(self) -> 'HeteroExplanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with\n        zero attribution are masked out.\n        \"\"\"\n        return self._apply_masks(\n            node_mask_dict={\n                key: mask.sum(dim=-1) > 0\n                for key, mask in self.collect('node_mask', True).items()\n            },\n            edge_mask_dict={\n                key: mask > 0\n                for key, mask in self.collect('edge_mask', True).items()\n            },\n        )\n\n    def get_complement_subgraph(self) -> 'HeteroExplanation':\n        r\"\"\"Returns the induced subgraph, in which all nodes and edges with any\n        attribution are masked out.\n        \"\"\"\n        return self._apply_masks(\n...\n", "needle_token_start": 7004, "needle_token_end": 7512, "code_context_ntokens": 16118, "output": ["```python\nclass ShaDowKHopSampler(torch.utils.data.DataLoader):\n    r\"\"\"The ShaDow :math:`k`-hop sampler from the `\"Decoupling the Depth and\n    Scope of Graph Neural Networks\" <https://arxiv.org/abs/2201.07858>`_ paper.\n    Given a graph in a :obj:`data` object, the sampler will create shallow,\n    localized subgraphs.\n    A deep GNN on this local graph then smooths the informative local signals.\n\n    .. note::\n\n        For an example of using :class:`ShaDowKHopSampler`, see\n        `examples/shadow.py <https://github.com/pyg-team/\n        pytorch_geometric/blob/master/examples/shadow.py>`_.\n\n    Args:\n        data (torch_geometric.data.Data): The graph data object.\n        depth (int): The depth/number of hops of the localized subgraph.\n        num_neighbors (int): The number of neighbors to sample for each node in\n            each hop.\n        node_idx (LongTensor or BoolTensor, optional): The nodes that should be\n            considered for creating mini-batches.\n            If set to :obj:`None`, all nodes will be\n            considered.\n        replace (bool, optional): If set to :obj:`True`, will sample neighbors\n            with replacement. (default: :obj:`False`)\n        **kwargs (optional): Additional arguments of\n            :class:`torch.utils.data.DataLoader`, such as :obj:`batch_size` or\n            :obj:`num_workers`.\n    \"\"\"\n    def __init__(self, data: Data, depth: int, num_neighbors: int,\n                 node_idx: Optional[Tensor] = None, replace: bool = False,\n                 **kwargs):\n\n        if not WITH_TORCH_SPARSE:\n            raise ImportError(\n                f\"'{self.__class__.__name__}' requires 'torch-sparse'\")\n\n        self.data = copy.copy(data)\n        self.depth = depth\n        self.num_neighbors = num_neighbors\n        self.replace = replace\n\n        if data.edge_index is not None:\n            self.is_sparse_tensor = False\n            row, col = data.edge_index.cpu()\n            self.adj_t = SparseTensor(\n                row=row, col=col, value=torch.arange(col.size(0)),\n                sparse_sizes=(data.num_nodes, data.num_nodes)).t()\n        else:\n            self.is_sparse_tensor = True\n            self.adj_t = data.adj_t.cpu()\n\n        if node_idx is None:\n            node_idx = torch.arange(self.adj_t.sparse_size(0))\n        elif node_idx.dtype == torch.bool:\n            node_idx = node_idx.nonzero(as_tuple=False).view(-1)\n        self.node_idx = node_idx\n\n        super().__init__(node_idx.tolist(), collate_fn=self.__collate__,\n                         **kwargs)\n\n    \ndef __collate__(self, n_id):\n        n_id = torch.tensor(n_id)\n\n        rowptr, col, value = self.adj_t.csr()\n        out = torch.ops.torch_sparse.ego_k_hop_sample_adj(\n            rowptr, col, n_id, self.depth, self.num_neighbors, self.replace)\n        rowptr, col, n_id, e_id, ptr, root_n_id = out\n\n        adj_t = SparseTensor(rowptr=rowptr, col=col,\n                             value=value[e_id] if value is not None else None,\n                             sparse_sizes=(n_id.numel(), n_id.numel()),\n                             is_sorted=True, trust_data=True)\n\n        batch = Batch(batch=torch.ops.torch_sparse.ptr2ind(ptr, n_id.numel()),\n                      ptr=ptr)\n        batch.root_n_id = root_n_id\n\n        if self.is_sparse_tensor:\n            batch.adj_t = adj_t\n        else:\n            row, col, e_id = adj_t.t().coo()\n            batch.edge_index = torch.stack([row, col], dim=0)\n\n        for k, v in self.data:\n            if k in ['edge_index', 'adj_t', 'num_nodes', 'batch', 'ptr']:\n                continue\n            if k == 'y' and v.size(0) == self.data.num_nodes:\n                batch[k] = v[n_id][root_n_id]\n            elif isinstance(v, Tensor) and v.size(0) == self.data.num_nodes:\n                batch[k] = v[n_id]\n            elif isinstance(v, Tensor) and v.size(0) == self.data.num_edges:\n                batch[k] = v[e_id]\n            else:\n                batch[k] = v\n\n        return batch\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "tree", "language": "python", "path": "torch_geometric/datasets/graph_generator/tree_graph.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function generates a tree graph with specified depth and branching factor, optionally making it undirected, and provides node-level depth information.\n2. **Input**: \n   - An integer representing the depth of the tree.\n   - An optional integer for the branching factor of each node (default is 2).\n   - An optional boolean to determine if the graph should be undirected (default is False).\n   - An optional device specification for where the tensors should be computed (default is CPU).\n3. **Output**: \n   - A tensor representing the edge connections between nodes in the tree.\n   - A tensor indicating the depth level of each node in the tree.\n4. **Procedure**: \n   - Initialize lists to store edges and node depths.\n   - Define a recursive function to add edges and node depths based on the current node and depth, iterating until the specified tree depth is reached.\n   - Convert the list of edges into a tensor and, if specified, make the graph undirected by adding reciprocal edges.\n   - Return the edge tensor and the node depth tensor.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/datasets/dynamic_faust.py\nfrom itertools import product\nfrom typing import Callable, List, Optional\n\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset\n\n\nclass DynamicFAUST(InMemoryDataset):\n    r\"\"\"The dynamic FAUST humans dataset from the `\"Dynamic FAUST: Registering\n    Human Bodies in Motion\"\n    <http://files.is.tue.mpg.de/black/papers/dfaust2017.pdf>`_ paper.\n\n    .. note::\n\n        Data objects hold mesh faces instead of edge indices.\n        To convert the mesh to a graph, use the\n        :obj:`torch_geometric.transforms.FaceToEdge` as :obj:`pre_transform`.\n        To convert the mesh to a point cloud, use the\n        :obj:`torch_geometric.transforms.SamplePoints` as :obj:`transform` to\n        sample a fixed number of points on the mesh faces according to their\n        face area.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        subjects (list, optional): List of subjects to include in the\n            dataset. Can include the subjects :obj:`\"50002\"`, :obj:`\"50004\"`,\n            :obj:`\"50007\"`, :obj:`\"50009\"`, :obj:`\"50020\"`, :obj:`\"50021\"`,\n            :obj:`\"50022\"`, :obj:`\"50025\"`, :obj:`\"50026\"`, :obj:`\"50027\"`.\n            If set to :obj:`None`, the dataset will contain all subjects.\n            (default: :obj:`None`)\n        categories (list, optional): List of categories to include in the\n            dataset. Can include the categories :obj:`\"chicken_wings\"`,\n            :obj:`\"hips\"`, :obj:`\"jiggle_on_toes\"`, :obj:`\"jumping_jacks\"`,\n            :obj:`\"knees\"`, :obj:`\"light_hopping_loose\"`,\n            :obj:`\"light_hopping_stiff\"`, :obj:`\"one_leg_jump\"`,\n            :obj:`\"one_leg_loose\"`, :obj:`\"personal_move\"`, :obj:`\"punching\"`,\n            :obj:`\"running_on_spot\"`, :obj:`\"running_on_spot_bugfix\"`,\n            :obj:`\"shake_arms\"`, :obj:`\"shake_hips\"`, :obj:`\"shoulders\"`.\n            If set to :obj:`None`, the dataset will contain all categories.\n            (default: :obj:`None`)\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        pre_filter (callable, optional): A function that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a boolean\n            value, indicating whether the data object should be included in the\n            final dataset. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n\n    url = 'http://dfaust.is.tue.mpg.de/'\n\n    subjects = [\n        '50002', '50004', '50007', '50009', '50020', '50021', '50022', '50025',\n        '50026', '50027'\n    ]\n    categories = [\n        'chicken_wings', 'hips', 'jiggle_on_toes', 'jumping_jacks', 'knees',\n        'light_hopping_loose', 'light_hopping_stiff', 'one_leg_jump',\n        'one_leg_loose', 'personal_move', 'punching', 'running_on_spot',\n        'running_on_spot_bugfix', 'shake_arms', 'shake_hips', 'shake_shoulders'\n    ]\n\n    def __init__(\n        self,\n        root: str,\n        subjects: Optional[List[str]] = None,\n        categories: Optional[List[str]] = None,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        pre_filter: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n\n        subjects = self.subjects if subjects is None else subjects\n        subjects = [sid.lower() for sid in subjects]\n        for sid in subjects:\n            assert sid in self.subjects\n        self.subjects = subjects\n\n        categories = self.categories if categories is None else categories\n        categories = [cat.lower() for cat in categories]\n        for cat in categories:\n            assert cat in self.categories\n        self.categories = categories\n\n        super().__init__(root, transform, pre_transform, pre_filter,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n...\n# Path: torch_geometric/datasets/elliptic.py\nfrom typing import Any, Callable, List, Optional, Tuple\n\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset\nfrom torch_geometric.io import fs\n\n\nclass EllipticBitcoinDataset(InMemoryDataset):\n    r\"\"\"The Elliptic Bitcoin dataset of Bitcoin transactions from the\n    `\"Anti-Money Laundering in Bitcoin: Experimenting with Graph Convolutional\n    Networks for Financial Forensics\" <https://arxiv.org/abs/1908.02591>`_\n    paper.\n\n    :class:`EllipticBitcoinDataset` maps Bitcoin transactions to real entities\n    belonging to licit categories (exchanges, wallet providers, miners,\n    licit services, etc.) versus illicit ones (scams, malware, terrorist\n    organizations, ransomware, Ponzi schemes, etc.)\n\n    There exists 203,769 node transactions and 234,355 directed edge payments\n    flows, with two percent of nodes (4,545) labelled as illicit, and\n    twenty-one percent of nodes (42,019) labelled as licit.\n    The remaining transactions are unknown.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n\n    **STATS:**\n\n    .. list-table::\n        :widths: 10 10 10 10\n        :header-rows: 1\n\n        * - #nodes\n          - #edges\n          - #features\n          - #classes\n        * - 203,769\n          - 234,355\n          - 165\n          - 2\n    \"\"\"\n    url = 'https://data.pyg.org/datasets/elliptic'\n\n    def __init__(\n        self,\n        root: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        return [\n            'elliptic_txs_features.csv',\n            'elliptic_txs_edgelist.csv',\n            'elliptic_txs_classes.csv',\n        ]\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        for file_name in self.raw_file_names:\n            fs.cp(f'{self.url}/{file_name}.zip', self.raw_dir, extract=True)\n\n    def _process_df(self, feat_df: Any, edge_df: Any,\n                    class_df: Any) -> Tuple[Any, Any, Any]:\n        return feat_df, edge_df, class_df\n\n    def process(self) -> None:\n        import pandas as pd\n\n        feat_df = pd.read_csv(self.raw_paths[0], header=None)\n        edge_df = pd.read_csv(self.raw_paths[1])\n        class_df = pd.read_csv(self.raw_paths[2])\n\n        columns = {0: 'txId', 1: 'time_step'}\n        feat_df = feat_df.rename(columns=columns)\n\n        feat_df, edge_df, class_df = self._process_df(\n            feat_df,\n            edge_df,\n            class_df,\n        )\n\n        x = torch.from_numpy(feat_df.loc[:, 2:].values).to(torch.float)\n\n        # There exists 3 different classes in the dataset:\n        # 0=licit,  1=illicit, 2=unknown\n        mapping = {'unknown': 2, '1': 1, '2': 0}\n        class_df['class'] = class_df['class'].map(mapping)\n        y = torch.from_numpy(class_df['class'].values)\n\n        mapping = {idx: i for i, idx in enumerate(feat_df['txId'].values)}\n        edge_df['txId1'] = edge_df['txId1'].map(mapping)\n        edge_df['txId2'] = edge_df['txId2'].map(mapping)\n        edge_index = torch.from_numpy(edge_df.values).t().contiguous()\n\n        # Timestamp based split:\n        # train_mask: 1 - 34 time_step, test_mask: 35-49 time_step\n        time_step = torch.from_numpy(feat_df['time_step'].values)\n        train_mask = (time_step < 35) & (y != 2)\n        test_mask = (time_step >= 35) & (y != 2)\n\n        data = Data(x=x, edge_index=edge_index, y=y, train_mask=train_mask,\n                    test_mask=test_mask)\n\n        if self.pre_transform is not None:\n            data = self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n    @property\n    def num_classes(self) -> int:\n        return 2\n\n# Path: torch_geometric/datasets/elliptic_temporal.py\nfrom typing import Any, Callable, Optional, Tuple\n\nfrom torch_geometric.datasets import EllipticBitcoinDataset\n\n\nclass EllipticBitcoinTemporalDataset(EllipticBitcoinDataset):\n    r\"\"\"The time-step aware Elliptic Bitcoin dataset of Bitcoin transactions\n    from the `\"Anti-Money Laundering in Bitcoin: Experimenting with Graph\n    Convolutional Networks for Financial Forensics\"\n    <https://arxiv.org/abs/1908.02591>`_ paper.\n\n    :class:`EllipticBitcoinTemporalDataset` maps Bitcoin transactions to real\n    entities belonging to licit categories (exchanges, wallet providers,\n    miners, licit services, etc.) versus illicit ones (scams, malware,\n    terrorist organizations, ransomware, Ponzi schemes, etc.)\n\n    There exists 203,769 node transactions and 234,355 directed edge payments\n    flows, with two percent of nodes (4,545) labelled as illicit, and\n    twenty-one percent of nodes (42,019) labelled as licit.\n    The remaining transactions are unknown.\n\n    .. note::\n\n        In contrast to :class:`EllipticBitcoinDataset`, this dataset returns\n        Bitcoin transactions only for a given timestamp :obj:`t`.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        t (int): The Timestep for which nodes should be selected (from :obj:`1`\n            to :obj:`49`).\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n\n    **STATS:**\n\n    .. list-table::\n        :widths: 10 10 10 10\n        :header-rows: 1\n\n        * - #nodes\n          - #edges\n          - #features\n          - #classes\n        * - 203,769\n          - 234,355\n          - 165\n          - 2\n    \"\"\"\n    def __init__(\n        self,\n        root: str,\n        t: int,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ):\n        if t < 1 or t > 49:\n            raise ValueError(\"'t' needs to be between 1 and 49\")\n\n        self.t = t\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n\n    @property\n    def processed_file_names(self) -> str:\n        return f'data_t_{self.t}.pt'\n\n    def _process_df(self, feat_df: Any, edge_df: Any,\n                    class_df: Any) -> Tuple[Any, Any, Any]:\n\n        feat_df = feat_df[feat_df['time_step'] == self.t]\n\n        mask = edge_df['txId1'].isin(feat_df['txId'].values)\n        edge_df = edge_df[mask]\n\n        class_df = class_df.merge(feat_df[['txId']], how='right',\n                                  left_on='txId', right_on='txId')\n\n        return feat_df, edge_df, class_df\n\n# Path: torch_geometric/datasets/email_eu_core.py\nimport os\nfrom typing import Callable, List, Optional\n\nimport torch\n\nfrom torch_geometric.data import (\n    Data,\n    InMemoryDataset,\n    download_url,\n    extract_gz,\n)\n\n\nclass EmailEUCore(InMemoryDataset):\n    r\"\"\"An e-mail communication network of a large European research\n    institution, taken from the `\"Local Higher-order Graph Clustering\"\n    <https://www-cs.stanford.edu/~jure/pubs/mappr-kdd17.pdf>`_ paper.\n    Nodes indicate members of the institution.\n    An edge between a pair of members indicates that they exchanged at least\n    one email.\n    Node labels indicate membership to one of the 42 departments.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n\n    urls = [\n        'https://snap.stanford.edu/data/email-Eu-core.txt.gz',\n        'https://snap.stanford.edu/data/email-Eu-core-department-labels.txt.gz'\n    ]\n\n    def __init__(\n        self,\n        root: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        return ['email-Eu-core.txt', 'email-Eu-core-department-labels.txt']\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        for url in self.urls:\n            path = download_url(url, self.raw_dir)\n            extract_gz(path, self.raw_dir)\n            os.unlink(path)\n\n    def process(self) -> None:\n        import pandas as pd\n\n        edge_index = pd.read_csv(self.raw_paths[0], sep=' ', header=None)\n        edge_index = torch.from_numpy(edge_index.values).t().contiguous()\n\n        y = pd.read_csv(self.raw_paths[1], sep=' ', header=None, usecols=[1])\n        y = torch.from_numpy(y.values).view(-1)\n\n        data = Data(edge_index=edge_index, y=y, num_nodes=y.size(0))\n\n        if self.pre_transform is not None:\n            data = self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n# Path: torch_geometric/datasets/entities.py\nimport logging\nimport os\nimport os.path as osp\nfrom collections import Counter\nfrom typing import Any, Callable, List, Optional\n\nimport torch\n\nfrom torch_geometric.data import (\n    Data,\n    HeteroData,\n    InMemoryDataset,\n    download_url,\n    extract_tar,\n)\nfrom torch_geometric.utils import index_sort\n\n\nclass Entities(InMemoryDataset):\n    r\"\"\"The relational entities networks :obj:`\"AIFB\"`, :obj:`\"MUTAG\"`,\n    :obj:`\"BGS\"` and :obj:`\"AM\"` from the `\"Modeling Relational Data with Graph\n    Convolutional Networks\" <https://arxiv.org/abs/1703.06103>`_ paper.\n    Training and test splits are given by node indices.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        name (str): The name of the dataset (:obj:`\"AIFB\"`, :obj:`\"MUTAG\"`,\n            :obj:`\"BGS\"`, :obj:`\"AM\"`).\n        hetero (bool, optional): If set to :obj:`True`, will save the dataset\n            as a :class:`~torch_geometric.data.HeteroData` object.\n            (default: :obj:`False`)\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n\n    **STATS:**\n\n    .. list-table::\n        :widths: 10 10 10 10 10\n        :header-rows: 1\n\n        * - Name\n          - #nodes\n          - #edges\n          - #features\n          - #classes\n        * - AIFB\n          - 8,285\n          - 58,086\n          - 0\n          - 4\n        * - AM\n          - 1,666,764\n          - 11,976,642\n          - 0\n          - 11\n        * - MUTAG\n          - 23,644\n          - 148,454\n          - 0\n          - 2\n        * - BGS\n          - 333,845\n          - 1,832,398\n          - 0\n          - 2\n    \"\"\"\n\n    url = 'https://data.dgl.ai/dataset/{}.tgz'\n\n    def __init__(\n        self,\n        root: str,\n        name: str,\n        hetero: bool = False,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        self.name = name.lower()\n        self.hetero = hetero\n        assert self.name in ['aifb', 'am', 'mutag', 'bgs']\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        if hetero:\n            self.load(self.processed_paths[0], data_cls=HeteroData)\n        else:\n            self.load(self.processed_paths[0], data_cls=Data)\n\n    @property\n    def raw_dir(self) -> str:\n        return osp.join(self.root, self.name, 'raw')\n\n    @property\n    def processed_dir(self) -> str:\n        return osp.join(self.root, self.name, 'processed')\n\n    @property\n    def num_relations(self) -> int:\n        return int(self._data.edge_type.max()) + 1  # type: ignore\n\n    @property\n    def num_classes(self) -> int:\n        return int(self._data.train_y.max()) + 1  # type: ignore\n\n    @property\n    def raw_file_names(self) -> List[str]:\n        return [\n            f'{self.name}_stripped.nt.gz',\n            'completeDataset.tsv',\n            'trainingSet.tsv',\n            'testSet.tsv',\n        ]\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'hetero_data.pt' if self.hetero else 'data.pt'\n\n    def download(self) -> None:\n        path = download_url(self.url.format(self.name), self.root)\n        extract_tar(path, self.raw_dir)\n        os.unlink(path)\n\n    def process(self) -> None:\n        import gzip\n\n        import pandas as pd\n        import rdflib as rdf\n\n        graph_file, task_file, train_file, test_file = self.raw_paths\n\n        with hide_stdout():\n            g = rdf.Graph()\n            with gzip.open(graph_file, 'rb') as f:\n                g.parse(file=f, format='nt')  # type: ignore\n\n        freq = Counter(g.predicates())\n\n        relations = sorted(set(g.predicates()), key=lambda p: -freq.get(p, 0))\n        subjects = set(g.subjects())\n        objects = set(g.objects())\n        nodes = list(subjects.union(objects))\n\n        N = len(nodes)\n        R = 2 * len(relations)\n\n        relations_dict = {rel: i for i, rel in enumerate(relations)}\n        nodes_dict = {str(node): i for i, node in enumerate(nodes)}\n\n        edges = []\n        for s, p, o in g.triples((None, None, None)):\n            src, dst = nodes_dict[str(s)], nodes_dict[str(o)]\n            rel = relations_dict[p]\n            edges.append([src, dst, 2 * rel])\n            edges.append([dst, src, 2 * rel + 1])\n\n        edge = torch.tensor(edges, dtype=torch.long).t().contiguous()\n        _, perm = index_sort(N * R * edge[0] + R * edge[1] + edge[2])\n        edge = edge[:, perm]\n\n        edge_index, edge_type = edge[:2], edge[2]\n\n        if self.name == 'am':\n            label_header = 'label_cateogory'\n            nodes_header = 'proxy'\n        elif self.name == 'aifb':\n            label_header = 'label_affiliation'\n            nodes_header = 'person'\n        elif self.name == 'mutag':\n            label_header = 'label_mutagenic'\n            nodes_header = 'bond'\n        elif self.name == 'bgs':\n            label_header = 'label_lithogenesis'\n            nodes_header = 'rock'\n\n        labels_df = pd.read_csv(task_file, sep='\\t')\n        labels_set = set(labels_df[label_header].values.tolist())\n        labels_dict = {lab: i for i, lab in enumerate(list(labels_set))}\n\n        train_labels_df = pd.read_csv(train_file, sep='\\t')\n        train_indices, train_labels = [], []\n        for nod, lab in zip(train_labels_df[nodes_header].values,\n                            train_labels_df[label_header].values):\n            train_indices.append(nodes_dict[nod])\n            train_labels.append(labels_dict[lab])\n\n        train_idx = torch.tensor(train_indices, dtype=torch.long)\n        train_y = torch.tensor(train_labels, dtype=torch.long)\n\n        test_labels_df = pd.read_csv(test_file, sep='\\t')\n        test_indices, test_labels = [], []\n        for nod, lab in zip(test_labels_df[nodes_header].values,\n                            test_labels_df[label_header].values):\n            test_indices.append(nodes_dict[nod])\n            test_labels.append(labels_dict[lab])\n\n        test_idx = torch.tensor(test_indices, dtype=torch.long)\n        test_y = torch.tensor(test_labels, dtype=torch.long)\n\n        data = Data(edge_index=edge_index, edge_type=edge_type,\n                    train_idx=train_idx, train_y=train_y, test_idx=test_idx,\n                    test_y=test_y, num_nodes=N)\n\n        if self.hetero:\n            data = data.to_heterogeneous(node_type_names=['v'])\n\n        self.save([data], self.processed_paths[0])\n\n    def __repr__(self) -> str:\n        return f'{self.name.upper()}{self.__class__.__name__}()'\n\n\nclass hide_stdout:\n    def __enter__(self) -> None:\n        self.level = logging.getLogger().level\n        logging.getLogger().setLevel(logging.ERROR)\n\n    def __exit__(self, *args: Any) -> None:\n        logging.getLogger().setLevel(self.level)\n\n# Path: torch_geometric/datasets/graph_generator/ba_graph.py\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.graph_generator import GraphGenerator\nfrom torch_geometric.utils import barabasi_albert_graph\n\n\nclass BAGraph(GraphGenerator):\n    r\"\"\"Generates random Barabasi-Albert (BA) graphs.\n    See :meth:`~torch_geometric.utils.barabasi_albert_graph` for more\n    information.\n\n    Args:\n        num_nodes (int): The number of nodes.\n        num_edges (int): The number of edges from a new node to existing nodes.\n    \"\"\"\n    def __init__(self, num_nodes: int, num_edges: int):\n        super().__init__()\n        self.num_nodes = num_nodes\n        self.num_edges = num_edges\n\n    def __call__(self) -> Data:\n        edge_index = barabasi_albert_graph(self.num_nodes, self.num_edges)\n        return Data(num_nodes=self.num_nodes, edge_index=edge_index)\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(num_nodes={self.num_nodes}, '\n                f'num_edges={self.num_edges})')\n\n# Path: torch_geometric/datasets/graph_generator/base.py\nfrom abc import ABC, abstractmethod\nfrom typing import Any\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.resolver import resolver\n\n\nclass GraphGenerator(ABC):\n    r\"\"\"An abstract base class for generating synthetic graphs.\"\"\"\n    @abstractmethod\n    def __call__(self) -> Data:\n        r\"\"\"To be implemented by :class:`GraphGenerator` subclasses.\"\"\"\n        raise NotImplementedError\n\n    @staticmethod\n    def resolve(query: Any, *args: Any, **kwargs: Any) -> 'GraphGenerator':\n        import torch_geometric.datasets.graph_generator as _graph_generators\n        graph_generators = [\n            gen for gen in vars(_graph_generators).values()\n            if isinstance(gen, type) and issubclass(gen, GraphGenerator)\n        ]\n        return resolver(graph_generators, {}, query, GraphGenerator, 'Graph',\n                        *args, **kwargs)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}()'\n\n# Path: torch_geometric/datasets/graph_generator/er_graph.py\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.graph_generator import GraphGenerator\nfrom torch_geometric.utils import erdos_renyi_graph\n\n\nclass ERGraph(GraphGenerator):\n    r\"\"\"Generates random Erdos-Renyi (ER) graphs.\n    See :meth:`~torch_geometric.utils.erdos_renyi_graph` for more information.\n\n    Args:\n        num_nodes (int): The number of nodes.\n        edge_prob (float): Probability of an edge.\n    \"\"\"\n    def __init__(self, num_nodes: int, edge_prob: float):\n        super().__init__()\n        self.num_nodes = num_nodes\n        self.edge_prob = edge_prob\n\n    def __call__(self) -> Data:\n        edge_index = erdos_renyi_graph(self.num_nodes, self.edge_prob)\n        return Data(num_nodes=self.num_nodes, edge_index=edge_index)\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(num_nodes={self.num_nodes}, '\n                f'edge_prob={self.edge_prob})')\n\n# Path: torch_geometric/datasets/graph_generator/grid_graph.py\nfrom typing import Optional\n\nimport torch\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.graph_generator import GraphGenerator\nfrom torch_geometric.utils import grid\n\n\nclass GridGraph(GraphGenerator):\n    r\"\"\"Generates two-dimensional grid graphs.\n    See :meth:`~torch_geometric.utils.grid` for more information.\n\n    Args:\n        height (int): The height of the grid.\n        width (int): The width of the grid.\n        dtype (:obj:`torch.dtype`, optional): The desired data type of the\n            returned position tensor. (default: :obj:`None`)\n    \"\"\"\n    def __init__(\n        self,\n        height: int,\n        width: int,\n        dtype: Optional[torch.dtype] = None,\n    ):\n        super().__init__()\n        self.height = height\n        self.width = width\n        self.dtype = dtype\n\n    def __call__(self) -> Data:\n        edge_index, pos = grid(height=self.height, width=self.width,\n                               dtype=self.dtype)\n        return Data(edge_index=edge_index, pos=pos)\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(height={self.height}, '\n                f'width={self.width})')\n\n# Path: torch_geometric/datasets/graph_generator/tree_graph.py\nfrom typing import List, Optional, Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.graph_generator import GraphGenerator\nfrom torch_geometric.utils import to_undirected\n\n\n\ndef tree(\n    depth: int,\n    branch: int = 2,\n    undirected: bool = False,\n    device: Optional[torch.device] = None,\n) -> Tuple[Tensor, Tensor]:\n    \"\"\"Generates a tree graph with the given depth and branch size, along with\n    node-level depth indicators.\n\n    Args:\n        depth (int): The depth of the tree.\n        branch (int, optional): The branch size of the tree.\n            (default: :obj:`2`)\n        undirected (bool, optional): If set to :obj:`True`, the tree graph will\n            be undirected. (default: :obj:`False`)\n        device (torch.device, optional): The desired device of the returned\n            tensors. (default: :obj:`None`)\n    \"\"\"\n    edges: List[Tuple[int, int]] = []\n    depths: List[int] = [0]\n\n    def add_edges(node: int, current_depth: int) -> None:\n        node_count = len(depths)\n\n        if current_depth < depth:\n            for i in range(branch):\n                edges.append((node, node_count + i))\n                depths.append(current_depth + 1)\n\n            for i in range(branch):\n                add_edges(node=node_count + i, current_depth=current_depth + 1)\n\n    add_edges(node=0, current_depth=0)\n\n    edge_index = torch.tensor(edges, device=device).t().contiguous()\n    if undirected:\n        edge_index = to_undirected(edge_index, num_nodes=len(depths))\n\n    return edge_index, torch.tensor(depths, device=device)\n\n\nclass TreeGraph(GraphGenerator):\n    r\"\"\"Generates tree graphs.\n\n    Args:\n        depth (int): The depth of the tree.\n        branch (int, optional): The branch size of the tree.\n            (default: :obj:`2`)\n        undirected (bool, optional): If set to :obj:`True`, the tree graph will\n            be undirected. (default: :obj:`False`)\n    \"\"\"\n    def __init__(\n        self,\n        depth: int,\n        branch: int = 2,\n        undirected: bool = False,\n    ) -> None:\n        super().__init__()\n        self.depth = depth\n        self.branch = branch\n        self.undirected = undirected\n\n    def __call__(self) -> Data:\n        edge_index, depth = tree(self.depth, self.branch, self.undirected)\n        num_nodes = depth.numel()\n        return Data(edge_index=edge_index, depth=depth, num_nodes=num_nodes)\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(depth={self.depth}, '\n                f'branch={self.branch}, undirected={self.undirected})')\n\n# Path: torch_geometric/datasets/graph_generator/__init__.py\nfrom .base import GraphGenerator\nfrom .ba_graph import BAGraph\nfrom .er_graph import ERGraph\nfrom .grid_graph import GridGraph\nfrom .tree_graph import TreeGraph\n\n__all__ = classes = [\n    'GraphGenerator',\n    'BAGraph',\n    'ERGraph',\n    'GridGraph',\n    'TreeGraph',\n]\n\n# Path: torch_geometric/datasets/motif_generator/base.py\nfrom abc import ABC, abstractmethod\nfrom typing import Any\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.resolver import resolver\n\n\nclass MotifGenerator(ABC):\n    r\"\"\"An abstract base class for generating a motif.\"\"\"\n    @abstractmethod\n    def __call__(self) -> Data:\n        r\"\"\"To be implemented by :class:`Motif` subclasses.\"\"\"\n        pass\n\n    @staticmethod\n    def resolve(query: Any, *args: Any, **kwargs: Any) -> 'MotifGenerator':\n        import torch_geometric.datasets.motif_generator as _motif_generators\n        motif_generators = [\n            gen for gen in vars(_motif_generators).values()\n            if isinstance(gen, type) and issubclass(gen, MotifGenerator)\n        ]\n        return resolver(motif_generators, {}, query, MotifGenerator, 'Motif',\n                        *args, **kwargs)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}()'\n\n# Path: torch_geometric/datasets/motif_generator/custom.py\nfrom typing import Any, Optional\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.motif_generator import MotifGenerator\nfrom torch_geometric.utils import from_networkx\n\n\nclass CustomMotif(MotifGenerator):\n    r\"\"\"Generates a motif based on a custom structure coming from a\n    :class:`torch_geometric.data.Data` or :class:`networkx.Graph` object.\n\n    Args:\n        structure (torch_geometric.data.Data or networkx.Graph): The structure\n            to use as a motif.\n    \"\"\"\n    def __init__(self, structure: Any):\n        super().__init__()\n\n        self.structure: Optional[Data] = None\n\n        if isinstance(structure, Data):\n            self.structure = structure\n        else:\n            try:\n                import networkx as nx\n                if isinstance(structure, nx.Graph):\n                    self.structure = from_networkx(structure)\n            except ImportError:\n                pass\n\n        if self.structure is None:\n            raise ValueError(f\"Expected a motif structure of type \"\n                             f\"'torch_geometric.data.Data' or 'networkx.Graph'\"\n                             f\"(got {type(structure)})\")\n\n    def __call__(self) -> Data:\n        assert isinstance(self.structure, Data)\n        return self.structure\n\n# Path: torch_geometric/datasets/motif_generator/cycle.py\nimport torch\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.motif_generator import CustomMotif\n\n\nclass CycleMotif(CustomMotif):\n    r\"\"\"Generates the cycle motif from the `\"GNNExplainer:\n    Generating Explanations for Graph Neural Networks\"\n    <https://arxiv.org/abs/1903.03894>`__ paper.\n\n    Args:\n        num_nodes (int): The number of nodes in the cycle.\n    \"\"\"\n    def __init__(self, num_nodes: int):\n        self.num_nodes = num_nodes\n\n        row = torch.arange(num_nodes).view(-1, 1).repeat(1, 2).view(-1)\n        col1 = torch.arange(-1, num_nodes - 1) % num_nodes\n        col2 = torch.arange(1, num_nodes + 1) % num_nodes\n        col = torch.stack([col1, col2], dim=1).sort(dim=-1)[0].view(-1)\n\n        structure = Data(\n            num_nodes=num_nodes,\n            edge_index=torch.stack([row, col], dim=0),\n        )\n        super().__init__(structure)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}({self.num_nodes})'\n\n# Path: torch_geometric/datasets/motif_generator/grid.py\nimport torch\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.motif_generator import CustomMotif\n\n\nclass GridMotif(CustomMotif):\n    r\"\"\"Generates the grid-structured motif from the\n    `\"GNNExplainer: Generating Explanations for Graph Neural Networks\"\n    <https://arxiv.org/abs/1903.03894>`__ paper.\n    \"\"\"\n    def __init__(self) -> None:\n        edge_indices = [\n            [0, 1],\n            [0, 3],\n            [1, 4],\n            [3, 4],\n            [1, 2],\n            [2, 5],\n            [4, 5],\n            [3, 6],\n            [6, 7],\n            [4, 7],\n            [5, 8],\n            [7, 8],\n            [1, 0],\n            [3, 0],\n            [4, 1],\n            [4, 3],\n            [2, 1],\n            [5, 2],\n            [5, 4],\n            [6, 3],\n            [7, 6],\n            [7, 4],\n            [8, 5],\n            [8, 7],\n        ]\n        structure = Data(\n            num_nodes=9,\n            edge_index=torch.tensor(edge_indices).t().contiguous(),\n            y=torch.tensor([0, 1, 0, 1, 2, 1, 0, 1, 0]),\n        )\n        super().__init__(structure)\n\n# Path: torch_geometric/datasets/motif_generator/house.py\nimport torch\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.motif_generator import CustomMotif\n\n\nclass HouseMotif(CustomMotif):\n    r\"\"\"Generates the house-structured motif from the `\"GNNExplainer:\n    Generating Explanations for Graph Neural Networks\"\n    <https://arxiv.org/abs/1903.03894>`__ paper, containing 5 nodes and 6\n    undirected edges. Nodes are labeled according to their structural role:\n    the top, middle and bottom of the house.\n    \"\"\"\n    def __init__(self) -> None:\n        structure = Data(\n            num_nodes=5,\n            edge_index=torch.tensor([\n                [0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4],\n                [1, 3, 4, 4, 2, 0, 1, 3, 2, 0, 0, 1],\n            ]),\n            y=torch.tensor([0, 0, 1, 1, 2]),\n        )\n        super().__init__(structure)\n\n# Path: torch_geometric/datasets/motif_generator/__init__.py\nfrom .base import MotifGenerator\nfrom .custom import CustomMotif\nfrom .house import HouseMotif\nfrom .cycle import CycleMotif\nfrom .grid import GridMotif\n\n__all__ = classes = [\n    'MotifGenerator',\n    'CustomMotif',\n    'HouseMotif',\n    'CycleMotif',\n    'GridMotif',\n]\n\n# Path: torch_geometric/datasets/explainer_dataset.py\nfrom typing import Any, Callable, Dict, Optional, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import InMemoryDataset\nfrom torch_geometric.datasets.graph_generator import GraphGenerator\nfrom torch_geometric.datasets.motif_generator import MotifGenerator\nfrom torch_geometric.explain import Explanation\n\n\nclass ExplainerDataset(InMemoryDataset):\n    r\"\"\"Generates a synthetic dataset for evaluating explainabilty algorithms,\n    as described in the `\"GNNExplainer: Generating Explanations for Graph\n    Neural Networks\" <https://arxiv.org/abs/1903.03894>`__ paper.\n    The :class:`~torch_geometric.datasets.ExplainerDataset` creates synthetic\n    graphs coming from a\n    :class:`~torch_geometric.datasets.graph_generator.GraphGenerator`, and\n    randomly attaches :obj:`num_motifs` many motifs to it coming from a\n    :class:`~torch_geometric.datasets.graph_generator.MotifGenerator`.\n    Ground-truth node-level and edge-level explainabilty masks are given based\n    on whether nodes and edges are part of a certain motif or not.\n\n    For example, to generate a random Barabasi-Albert (BA) graph with 300\n    nodes, in which we want to randomly attach 80 :obj:`\"house\"` motifs, write:\n\n    .. code-block:: python\n\n        from torch_geometric.datasets import ExplainerDataset\n        from torch_geometric.datasets.graph_generator import BAGraph\n\n        dataset = ExplainerDataset(\n            graph_generator=BAGraph(num_nodes=300, num_edges=5),\n            motif_generator='house',\n            num_motifs=80,\n        )\n\n    .. note::\n\n        For an example of using :class:`ExplainerDataset`, see\n        `examples/explain/gnn_explainer_ba_shapes.py\n        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/\n        /explain/gnn_explainer_ba_shapes.py>`_.\n\n    Args:\n        graph_generator (GraphGenerator or str): The graph generator to be\n            used, *e.g.*,\n            :class:`torch.geometric.datasets.graph_generator.BAGraph`\n            (or any string that automatically resolves to it).\n        motif_generator (MotifGenerator): The motif generator to be used,\n            *e.g.*,\n            :class:`torch_geometric.datasets.motif_generator.HouseMotif`\n            (or any string that automatically resolves to it).\n        num_motifs (int): The number of motifs to attach to the graph.\n        num_graphs (int, optional): The number of graphs to generate.\n            (default: :obj:`1`)\n        graph_generator_kwargs (Dict[str, Any], optional): Arguments passed to\n            the respective graph generator module in case it gets automatically\n            resolved. (default: :obj:`None`)\n        motif_generator_kwargs (Dict[str, Any], optional): Arguments passed to\n            the respective motif generator module in case it gets automatically\n            resolved. (default: :obj:`None`)\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n    \"\"\"\n    def __init__(\n        self,\n        graph_generator: Union[GraphGenerator, str],\n        motif_generator: Union[MotifGenerator, str],\n        num_motifs: int,\n        num_graphs: int = 1,\n        graph_generator_kwargs: Optional[Dict[str, Any]] = None,\n        motif_generator_kwargs: Optional[Dict[str, Any]] = None,\n        transform: Optional[Callable] = None,\n    ):\n        super().__init__(root=None, transform=transform)\n\n        if num_motifs <= 0:\n            raise ValueError(f\"At least one motif needs to be attached to the \"\n                             f\"graph (got {num_motifs})\")\n\n        self.graph_generator = GraphGenerator.resolve(\n            graph_generator,\n            **(graph_generator_kwargs or {}),\n        )\n        self.motif_generator = MotifGenerator.resolve(\n            motif_generator,\n            **(motif_generator_kwargs or {}),\n        )\n        self.num_motifs = num_motifs\n\n        # TODO (matthias) support on-the-fly graph generation.\n        data_list = [self.get_graph() for _ in range(num_graphs)]\n        self.data, self.slices = self.collate(data_list)\n\n    def get_graph(self) -> Explanation:\n        data = self.graph_generator()\n        assert data.num_nodes is not None\n        assert data.edge_index is not None\n\n        edge_indices = [data.edge_index]\n        num_nodes = data.num_nodes\n        node_masks = [torch.zeros(data.num_nodes)]\n        edge_masks = [torch.zeros(data.num_edges)]\n        ys = [torch.zeros(num_nodes, dtype=torch.long)]\n\n        connecting_nodes = torch.randperm(num_nodes)[:self.num_motifs]\n        for i in connecting_nodes.tolist():\n            motif = self.motif_generator()\n            assert motif.num_nodes is not None\n            assert motif.edge_index is not None\n\n            # Add motif to the graph.\n            edge_indices.append(motif.edge_index + num_nodes)\n            node_masks.append(torch.ones(motif.num_nodes))\n            edge_masks.append(torch.ones(motif.num_edges))\n\n            # Add random motif connection to the graph.\n            j = int(torch.randint(0, motif.num_nodes, (1, ))) + num_nodes\n            edge_indices.append(torch.tensor([[i, j], [j, i]]))\n            edge_masks.append(torch.zeros(2))\n\n            if isinstance(motif.y, Tensor):\n                ys.append(motif.y + 1 if motif.y.min() == 0 else motif.y)\n            else:\n                ys.append(torch.ones(motif.num_nodes, dtype=torch.long))\n\n            num_nodes += motif.num_nodes\n\n        return Explanation(\n            edge_index=torch.cat(edge_indices, dim=1),\n            y=torch.cat(ys, dim=0),\n            edge_mask=torch.cat(edge_masks, dim=0),\n            node_mask=torch.cat(node_masks, dim=0),\n        )\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}({len(self)}, '\n                f'graph_generator={self.graph_generator}, '\n                f'motif_generator={self.motif_generator}, '\n                f'num_motifs={self.num_motifs})')\n\n# Path: torch_geometric/datasets/facebook.py\nfrom typing import Callable, Optional\n\nimport numpy as np\nimport torch\n\nfrom torch_geometric.data import Data, InMemoryDataset, download_url\n\n\nclass FacebookPagePage(InMemoryDataset):\n    r\"\"\"The Facebook Page-Page network dataset introduced in the\n    `\"Multi-scale Attributed Node Embedding\"\n    <https://arxiv.org/abs/1909.13021>`_ paper.\n    Nodes represent verified pages on Facebook and edges are mutual likes.\n    It contains 22,470 nodes, 342,004 edges, 128 node features and 4 classes.\n\n    Args:\n        root (str): Root directory where the dataset should be saved.\n        transform (callable, optional): A function/transform that takes in an\n            :obj:`torch_geometric.data.Data` object and returns a transformed\n            version. The data object will be transformed before every access.\n            (default: :obj:`None`)\n        pre_transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            being saved to disk. (default: :obj:`None`)\n        force_reload (bool, optional): Whether to re-process the dataset.\n            (default: :obj:`False`)\n    \"\"\"\n\n    url = 'https://graphmining.ai/datasets/ptg/facebook.npz'\n\n    def __init__(\n        self,\n        root: str,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        force_reload: bool = False,\n    ) -> None:\n        super().__init__(root, transform, pre_transform,\n                         force_reload=force_reload)\n        self.load(self.processed_paths[0])\n\n    @property\n    def raw_file_names(self) -> str:\n        return 'facebook.npz'\n\n    @property\n    def processed_file_names(self) -> str:\n        return 'data.pt'\n\n    def download(self) -> None:\n        download_url(self.url, self.raw_dir)\n\n    def process(self) -> None:\n        data = np.load(self.raw_paths[0], 'r', allow_pickle=True)\n        x = torch.from_numpy(data['features']).to(torch.float)\n        y = torch.from_numpy(data['target']).to(torch.long)\n        edge_index = torch.from_numpy(data['edges']).to(torch.long)\n        edge_index = edge_index.t().contiguous()\n\n        data = Data(x=x, y=y, edge_index=edge_index)\n\n        if self.pre_transform is not None:\n            data = self.pre_transform(data)\n\n        self.save([data], self.processed_paths[0])\n\n# Path: torch_geometric/datasets/fake.py\nimport random\nfrom collections import defaultdict\nfrom itertools import product\nfrom typing import Callable, Dict, List, Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data, HeteroData, InMemoryDataset\nfrom torch_geometric.utils import coalesce, remove_self_loops, to_undirected\n\n\nclass FakeDataset(InMemoryDataset):\n    r\"\"\"A fake dataset that returns randomly generated\n    :class:`~torch_geometric.data.Data` objects.\n\n    Args:\n        num_graphs (int, optional): The number of graphs. (default: :obj:`1`)\n        avg_num_nodes (int, optional): The average number of nodes in a graph.\n            (default: :obj:`1000`)\n        avg_degree (float, optional): The average degree per node.\n            (default: :obj:`10.0`)\n        num_channels (int, optional): The number of node features.\n            (default: :obj:`64`)\n        edge_dim (int, optional): The number of edge features.\n            (default: :obj:`0`)\n        num_classes (int, optional): The number of classes in the dataset.\n            (default: :obj:`10`)\n        task (str, optional): Whether to return node-level or graph-level\n            labels (:obj:`\"node\"`, :obj:`\"graph\"`, :obj:`\"auto\"`).\n            If set to :obj:`\"auto\"`, will return graph-level labels if\n            :obj:`num_graphs > 1`, and node-level labels other-wise.\n            (default: :obj:`\"auto\"`)\n        is_undirected (bool, optional): Whether the graphs to generate are\n            undirected. (default: :obj:`True`)\n        transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.Data` object and returns a\n            transformed version. The data object will be transformed before\n            every access. (default: :obj:`None`)\n        **kwargs (optional): Additional attributes and their shapes\n            *e.g.* :obj:`global_features=5`.\n    \"\"\"\n    def __init__(\n        self,\n        num_graphs: int = 1,\n        avg_num_nodes: int = 1000,\n        avg_degree: float = 10.0,\n        num_channels: int = 64,\n        edge_dim: int = 0,\n        num_classes: int = 10,\n        task: str = 'auto',\n        is_undirected: bool = True,\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        **kwargs: Union[int, Tuple[int, ...]],\n    ) -> None:\n        super().__init__(None, transform)\n\n        if task == 'auto':\n            task = 'graph' if num_graphs > 1 else 'node'\n        assert task in ['node', 'graph']\n\n        self.avg_num_nodes = max(avg_num_nodes, int(avg_degree))\n        self.avg_degree = max(avg_degree, 1)\n        self.num_channels = num_channels\n        self.edge_dim = edge_dim\n        self._num_classes = num_classes\n        self.task = task\n        self.is_undirected = is_undirected\n        self.kwargs = kwargs\n\n        data_list = [self.generate_data() for _ in range(max(num_graphs, 1))]\n        self.data, self.slices = self.collate(data_list)\n\n    def generate_data(self) -> Data:\n        num_nodes = get_num_nodes(self.avg_num_nodes, self.avg_degree)\n\n        data = Data()\n\n        if self._num_classes > 0 and self.task == 'node':\n            data.y = torch.randint(self._num_classes, (num_nodes, ))\n        elif self._num_classes > 0 and self.task == 'graph':\n            data.y = torch.tensor([random.randint(0, self._num_classes - 1)])\n\n        data.edge_index = get_edge_index(num_nodes, num_nodes, self.avg_degree,\n                                         self.is_undirected, remove_loops=True)\n\n        if self.num_channels > 0:\n            x = torch.randn(num_nodes, self.num_channels)\n            if self._num_classes > 0 and self.task == 'node':\n                assert isinstance(data.y, Tensor)\n                x = x + data.y.unsqueeze(1)\n            elif self._num_classes > 0 and self.task == 'graph':\n                assert isinstance(data.y, Tensor)\n                x = x + data.y\n            data.x = x\n        else:\n            data.num_nodes = num_nodes\n\n        if self.edge_dim > 1:\n            data.edge_attr = torch.rand(data.num_edges, self.edge_dim)\n        elif self.edge_dim == 1:\n            data.edge_weight = torch.rand(data.num_edges)\n\n        for feature_name, feature_shape in self.kwargs.items():\n            setattr(data, feature_name, torch.randn(feature_shape))\n\n        return data\n\n\nclass FakeHeteroDataset(InMemoryDataset):\n    r\"\"\"A fake dataset that returns randomly generated\n    :class:`~torch_geometric.data.HeteroData` objects.\n\n    Args:\n        num_graphs (int, optional): The number of graphs. (default: :obj:`1`)\n        num_node_types (int, optional): The number of node types.\n            (default: :obj:`3`)\n        num_edge_types (int, optional): The number of edge types.\n            (default: :obj:`6`)\n        avg_num_nodes (int, optional): The average number of nodes in a graph.\n            (default: :obj:`1000`)\n        avg_degree (float, optional): The average degree per node.\n            (default: :obj:`10.0`)\n        avg_num_channels (int, optional): The average number of node features.\n            (default: :obj:`64`)\n        edge_dim (int, optional): The number of edge features.\n            (default: :obj:`0`)\n        num_classes (int, optional): The number of classes in the dataset.\n            (default: :obj:`10`)\n        task (str, optional): Whether to return node-level or graph-level\n            labels (:obj:`\"node\"`, :obj:`\"graph\"`, :obj:`\"auto\"`).\n            If set to :obj:`\"auto\"`, will return graph-level labels if\n            :obj:`num_graphs > 1`, and node-level labels other-wise.\n            (default: :obj:`\"auto\"`)\n        transform (callable, optional): A function/transform that takes in\n            an :obj:`torch_geometric.data.HeteroData` object and returns a\n            transformed version. The data object will be transformed before\n            every access. (default: :obj:`None`)\n        **kwargs (optional): Additional attributes and their shapes\n            *e.g.* :obj:`global_features=5`.\n    \"\"\"\n    def __init__(\n        self,\n        num_graphs: int = 1,\n        num_node_types: int = 3,\n        num_edge_types: int = 6,\n        avg_num_nodes: int = 1000,\n        avg_degree: float = 10.0,\n        avg_num_channels: int = 64,\n        edge_dim: int = 0,\n        num_classes: int = 10,\n        task: str = \"auto\",\n        transform: Optional[Callable] = None,\n        pre_transform: Optional[Callable] = None,\n        **kwargs: Union[int, Tuple[int, ...]],\n    ) -> None:\n        super().__init__(None, transform)\n\n        if task == 'auto':\n            task = 'graph' if num_graphs > 1 else 'node'\n        assert task in ['node', 'graph']\n\n...\n", "needle_token_start": 8646, "needle_token_end": 9079, "code_context_ntokens": 16120, "output": ["```python\nfrom typing import List, Optional, Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.data import Data\nfrom torch_geometric.datasets.graph_generator import GraphGenerator\nfrom torch_geometric.utils import to_undirected\n\n\ndef tree(\n    depth: int,\n    branch: int = 2,\n    undirected: bool = False,\n    device: Optional[torch.device] = None,\n) -> Tuple[Tensor, Tensor]:\n    \"\"\"Generates a tree graph with the given depth and branch size, along with\n    node-level depth indicators.\n\n    Args:\n        depth (int): The depth of the tree.\n        branch (int, optional): The branch size of the tree.\n            (default: :obj:`2`)\n        undirected (bool, optional): If set to :obj:`True`, the tree graph will\n            be undirected. (default: :obj:`False`)\n        device (torch.device, optional): The desired device of the returned\n            tensors. (default: :obj:`None`)\n    \"\"\"\n    edges: List[Tuple[int, int]] = []\n    depths: List[int] = [0]\n\n    def add_edges(node: int, current_depth: int) -> None:\n        node_count = len(depths)\n\n        if current_depth < depth:\n            for i in range(branch):\n                edges.append((node, node_count + i))\n                depths.append(current_depth + 1)\n\n            for i in range(branch):\n                add_edges(node=node_count + i, current_depth=current_depth + 1)\n\n    add_edges(node=0, current_depth=0)\n\n    edge_index = torch.tensor(edges, device=device).t().contiguous()\n    if undirected:\n        edge_index = to_undirected(edge_index, num_nodes=len(depths))\n\n    return edge_index, torch.tensor(depths, device=device)\n\n\nclass TreeGraph(GraphGenerator):\n    r\"\"\"Generates tree graphs.\n\n    Args:\n        depth (int): The depth of the tree.\n        branch (int, optional): The branch size of the tree.\n            (default: :obj:`2`)\n        undirected (bool, optional): If set to :obj:`True`, the tree graph will\n            be undirected. (default: :obj:`False`)\n    \"\"\"\n    def __init__(\n        self,\n        depth: int,\n        branch: int = 2,\n        undirected: bool = False,\n    ) -> None:\n        super().__init__()\n        self.depth = depth\n        self.branch = branch\n        self.undirected = undirected\n\n    def __call__(self) -> Data:\n        edge_index, depth = tree(self.depth, self.branch, self.undirected)\n        num_nodes = depth.numel()\n        return Data(edge_index=edge_index, depth=depth, num_nodes=num_nodes)\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}(depth={self.depth}, '\n                f'branch={self.branch}, undirected={self.undirected})')\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "to_scipy_sparse_matrix", "language": "python", "path": "torch_geometric/utils/convert.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a graph representation from edge indices and optional edge attributes into a sparse matrix format, which is commonly used for efficient mathematical and scientific computations.\n2. **Input**: The function takes three parameters: a tensor of edge indices indicating the connections between nodes, an optional tensor of edge attributes that can represent weights or multi-dimensional features of the edges, and an optional integer specifying the total number of nodes in the graph.\n3. **Output**: It produces a sparse matrix in COOrdinate format from the scipy library, representing the graph where rows and columns correspond to nodes and non-zero entries represent edges, optionally weighted by the edge attributes.\n4. **Procedure**: The function first extracts the rows and columns from the edge indices tensor. If no edge attributes are provided, it assigns a weight of one to all edges. It then determines the number of nodes, either from the input or by calculating the maximum index in the edge indices. Finally, it constructs and returns the sparse matrix using the node indices and edge attributes (or default weights).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/utils/_subgraph.py\nfrom typing import List, Literal, Optional, Tuple, Union, overload\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import OptTensor, PairTensor\nfrom torch_geometric.utils import scatter\nfrom torch_geometric.utils.map import map_index\nfrom torch_geometric.utils.mask import index_to_mask\nfrom torch_geometric.utils.num_nodes import maybe_num_nodes\n\n\ndef get_num_hops(model: torch.nn.Module) -> int:\n    r\"\"\"Returns the number of hops the model is aggregating information\n    from.\n\n    .. note::\n\n        This function counts the number of message passing layers as an\n        approximation of the total number of hops covered by the model.\n        Its output may not necessarily be correct in case message passing\n        layers perform multi-hop aggregation, *e.g.*, as in\n        :class:`~torch_geometric.nn.conv.ChebConv`.\n\n    Example:\n        >>> class GNN(torch.nn.Module):\n        ...     def __init__(self):\n        ...         super().__init__()\n        ...         self.conv1 = GCNConv(3, 16)\n        ...         self.conv2 = GCNConv(16, 16)\n        ...         self.lin = Linear(16, 2)\n        ...\n        ...     def forward(self, x, edge_index):\n        ...         x = self.conv1(x, edge_index).relu()\n        ...         x = self.conv2(x, edge_index).relu()\n        ...         return self.lin(x)\n        >>> get_num_hops(GNN())\n        2\n    \"\"\"\n    from torch_geometric.nn.conv import MessagePassing\n    num_hops = 0\n    for module in model.modules():\n        if isinstance(module, MessagePassing):\n            num_hops += 1\n    return num_hops\n\n\n@overload\ndef subgraph(\n    subset: Union[Tensor, List[int]],\n    edge_index: Tensor,\n    edge_attr: OptTensor = ...,\n    relabel_nodes: bool = ...,\n    num_nodes: Optional[int] = ...,\n) -> Tuple[Tensor, OptTensor]:\n    pass\n\n\n@overload\ndef subgraph(\n    subset: Union[Tensor, List[int]],\n    edge_index: Tensor,\n    edge_attr: OptTensor = ...,\n    relabel_nodes: bool = ...,\n    num_nodes: Optional[int] = ...,\n    *,\n    return_edge_mask: Literal[False],\n) -> Tuple[Tensor, OptTensor]:\n    pass\n\n\n@overload\ndef subgraph(\n    subset: Union[Tensor, List[int]],\n    edge_index: Tensor,\n    edge_attr: OptTensor = ...,\n    relabel_nodes: bool = ...,\n    num_nodes: Optional[int] = ...,\n    *,\n    return_edge_mask: Literal[True],\n) -> Tuple[Tensor, OptTensor, Tensor]:\n    pass\n\n\ndef subgraph(\n    subset: Union[Tensor, List[int]],\n    edge_index: Tensor,\n    edge_attr: OptTensor = None,\n    relabel_nodes: bool = False,\n    num_nodes: Optional[int] = None,\n    *,\n    return_edge_mask: bool = False,\n) -> Union[Tuple[Tensor, OptTensor], Tuple[Tensor, OptTensor, Tensor]]:\n    r\"\"\"Returns the induced subgraph of :obj:`(edge_index, edge_attr)`\n    containing the nodes in :obj:`subset`.\n\n    Args:\n        subset (LongTensor, BoolTensor or [int]): The nodes to keep.\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional\n            edge features. (default: :obj:`None`)\n        relabel_nodes (bool, optional): If set to :obj:`True`, the resulting\n            :obj:`edge_index` will be relabeled to hold consecutive indices\n            starting from zero. (default: :obj:`False`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max(edge_index) + 1`. (default: :obj:`None`)\n        return_edge_mask (bool, optional): If set to :obj:`True`, will return\n            the edge mask to filter out additional edge features.\n            (default: :obj:`False`)\n\n    :rtype: (:class:`LongTensor`, :class:`Tensor`)\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6],\n        ...                            [1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5]])\n        >>> edge_attr = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])\n        >>> subset = torch.tensor([3, 4, 5])\n        >>> subgraph(subset, edge_index, edge_attr)\n...\n# Path: torch_geometric/utils/_to_dense_batch.py\nfrom typing import Optional, Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.experimental import (\n    disable_dynamic_shapes,\n    is_experimental_mode_enabled,\n)\nfrom torch_geometric.utils import cumsum, scatter\n\n\n@disable_dynamic_shapes(required_args=['batch_size', 'max_num_nodes'])\ndef to_dense_batch(\n    x: Tensor,\n    batch: Optional[Tensor] = None,\n    fill_value: float = 0.0,\n    max_num_nodes: Optional[int] = None,\n    batch_size: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Given a sparse batch of node features\n    :math:`\\mathbf{X} \\in \\mathbb{R}^{(N_1 + \\ldots + N_B) \\times F}` (with\n    :math:`N_i` indicating the number of nodes in graph :math:`i`), creates a\n    dense node feature tensor\n    :math:`\\mathbf{X} \\in \\mathbb{R}^{B \\times N_{\\max} \\times F}` (with\n    :math:`N_{\\max} = \\max_i^B N_i`).\n    In addition, a mask of shape :math:`\\mathbf{M} \\in \\{ 0, 1 \\}^{B \\times\n    N_{\\max}}` is returned, holding information about the existence of\n    fake-nodes in the dense representation.\n\n    Args:\n        x (Tensor): Node feature matrix\n            :math:`\\mathbf{X} \\in \\mathbb{R}^{(N_1 + \\ldots + N_B) \\times F}`.\n        batch (LongTensor, optional): Batch vector\n            :math:`\\mathbf{b} \\in {\\{ 0, \\ldots, B-1\\}}^N`, which assigns each\n            node to a specific example. Must be ordered. (default: :obj:`None`)\n        fill_value (float, optional): The value for invalid entries in the\n            resulting dense output tensor. (default: :obj:`0`)\n        max_num_nodes (int, optional): The size of the output node dimension.\n            (default: :obj:`None`)\n        batch_size (int, optional): The batch size. (default: :obj:`None`)\n\n    :rtype: (:class:`Tensor`, :class:`BoolTensor`)\n\n    Examples:\n        >>> x = torch.arange(12).view(6, 2)\n        >>> x\n        tensor([[ 0,  1],\n                [ 2,  3],\n                [ 4,  5],\n                [ 6,  7],\n                [ 8,  9],\n                [10, 11]])\n\n        >>> out, mask = to_dense_batch(x)\n        >>> mask\n        tensor([[True, True, True, True, True, True]])\n\n        >>> batch = torch.tensor([0, 0, 1, 2, 2, 2])\n        >>> out, mask = to_dense_batch(x, batch)\n        >>> out\n        tensor([[[ 0,  1],\n                [ 2,  3],\n                [ 0,  0]],\n                [[ 4,  5],\n                [ 0,  0],\n                [ 0,  0]],\n                [[ 6,  7],\n                [ 8,  9],\n                [10, 11]]])\n        >>> mask\n        tensor([[ True,  True, False],\n                [ True, False, False],\n                [ True,  True,  True]])\n\n        >>> out, mask = to_dense_batch(x, batch, max_num_nodes=4)\n        >>> out\n        tensor([[[ 0,  1],\n                [ 2,  3],\n                [ 0,  0],\n                [ 0,  0]],\n                [[ 4,  5],\n                [ 0,  0],\n                [ 0,  0],\n                [ 0,  0]],\n                [[ 6,  7],\n                [ 8,  9],\n                [10, 11],\n                [ 0,  0]]])\n\n        >>> mask\n        tensor([[ True,  True, False, False],\n                [ True, False, False, False],\n                [ True,  True,  True, False]])\n    \"\"\"\n    if batch is None and max_num_nodes is None:\n        mask = torch.ones(1, x.size(0), dtype=torch.bool, device=x.device)\n        return x.unsqueeze(0), mask\n\n    if batch is None:\n        batch = x.new_zeros(x.size(0), dtype=torch.long)\n\n    if batch_size is None:\n        batch_size = int(batch.max()) + 1\n\n    num_nodes = scatter(batch.new_ones(x.size(0)), batch, dim=0,\n                        dim_size=batch_size, reduce='sum')\n    cum_nodes = cumsum(num_nodes)\n\n    filter_nodes = False\n    dynamic_shapes_disabled = is_experimental_mode_enabled(\n        'disable_dynamic_shapes')\n\n    if max_num_nodes is None:\n        max_num_nodes = int(num_nodes.max())\n    elif not dynamic_shapes_disabled and num_nodes.max() > max_num_nodes:\n        filter_nodes = True\n\n    tmp = torch.arange(batch.size(0), device=x.device) - cum_nodes[batch]\n    idx = tmp + (batch * max_num_nodes)\n    if filter_nodes:\n        mask = tmp < max_num_nodes\n        x, idx = x[mask], idx[mask]\n\n    size = [batch_size * max_num_nodes] + list(x.size())[1:]\n    out = torch.as_tensor(fill_value, device=x.device)\n    out = out.to(x.dtype).repeat(size)\n    out[idx] = x\n    out = out.view([batch_size, max_num_nodes] + list(x.size())[1:])\n\n    mask = torch.zeros(batch_size * max_num_nodes, dtype=torch.bool,\n                       device=x.device)\n    mask[idx] = 1\n    mask = mask.view(batch_size, max_num_nodes)\n\n    return out, mask\n\n# Path: torch_geometric/utils/_train_test_split_edges.py\nimport math\n\nimport torch\n\nimport torch_geometric\nfrom torch_geometric.deprecation import deprecated\nfrom torch_geometric.utils import to_undirected\n\n\n@deprecated(\"use 'transforms.RandomLinkSplit' instead\")\ndef train_test_split_edges(\n    data: 'torch_geometric.data.Data',\n    val_ratio: float = 0.05,\n    test_ratio: float = 0.1,\n) -> 'torch_geometric.data.Data':\n    r\"\"\"Splits the edges of a :class:`torch_geometric.data.Data` object\n    into positive and negative train/val/test edges.\n    As such, it will replace the :obj:`edge_index` attribute with\n    :obj:`train_pos_edge_index`, :obj:`train_pos_neg_adj_mask`,\n    :obj:`val_pos_edge_index`, :obj:`val_neg_edge_index` and\n    :obj:`test_pos_edge_index` attributes.\n    If :obj:`data` has edge features named :obj:`edge_attr`, then\n    :obj:`train_pos_edge_attr`, :obj:`val_pos_edge_attr` and\n    :obj:`test_pos_edge_attr` will be added as well.\n\n    .. warning::\n\n        :meth:`~torch_geometric.utils.train_test_split_edges` is deprecated and\n        will be removed in a future release.\n        Use :class:`torch_geometric.transforms.RandomLinkSplit` instead.\n\n    Args:\n        data (Data): The data object.\n        val_ratio (float, optional): The ratio of positive validation edges.\n            (default: :obj:`0.05`)\n        test_ratio (float, optional): The ratio of positive test edges.\n            (default: :obj:`0.1`)\n\n    :rtype: :class:`torch_geometric.data.Data`\n    \"\"\"\n    assert 'batch' not in data  # No batch-mode.\n\n    assert data.num_nodes is not None\n    assert data.edge_index is not None\n\n    num_nodes = data.num_nodes\n    row, col = data.edge_index\n    edge_attr = data.edge_attr\n    del data.edge_index\n    del data.edge_attr\n\n    # Return upper triangular portion.\n    mask = row < col\n    row, col = row[mask], col[mask]\n\n    if edge_attr is not None:\n        edge_attr = edge_attr[mask]\n\n    n_v = int(math.floor(val_ratio * row.size(0)))\n    n_t = int(math.floor(test_ratio * row.size(0)))\n\n    # Positive edges.\n    perm = torch.randperm(row.size(0))\n    row, col = row[perm], col[perm]\n    if edge_attr is not None:\n        edge_attr = edge_attr[perm]\n\n    r, c = row[:n_v], col[:n_v]\n    data.val_pos_edge_index = torch.stack([r, c], dim=0)\n    if edge_attr is not None:\n        data.val_pos_edge_attr = edge_attr[:n_v]\n\n    r, c = row[n_v:n_v + n_t], col[n_v:n_v + n_t]\n    data.test_pos_edge_index = torch.stack([r, c], dim=0)\n    if edge_attr is not None:\n        data.test_pos_edge_attr = edge_attr[n_v:n_v + n_t]\n\n    r, c = row[n_v + n_t:], col[n_v + n_t:]\n    data.train_pos_edge_index = torch.stack([r, c], dim=0)\n    if edge_attr is not None:\n        out = to_undirected(data.train_pos_edge_index, edge_attr[n_v + n_t:])\n        data.train_pos_edge_index, data.train_pos_edge_attr = out\n    else:\n        data.train_pos_edge_index = to_undirected(data.train_pos_edge_index)\n\n    # Negative edges.\n    neg_adj_mask = torch.ones(num_nodes, num_nodes, dtype=torch.uint8)\n    neg_adj_mask = neg_adj_mask.triu(diagonal=1).to(torch.bool)\n    neg_adj_mask[row, col] = 0\n\n    neg_row, neg_col = neg_adj_mask.nonzero(as_tuple=False).t()\n    perm = torch.randperm(neg_row.size(0))[:n_v + n_t]\n    neg_row, neg_col = neg_row[perm], neg_col[perm]\n\n    neg_adj_mask[neg_row, neg_col] = 0\n    data.train_neg_adj_mask = neg_adj_mask\n\n    row, col = neg_row[:n_v], neg_col[:n_v]\n    data.val_neg_edge_index = torch.stack([row, col], dim=0)\n\n    row, col = neg_row[n_v:n_v + n_t], neg_col[n_v:n_v + n_t]\n    data.test_neg_edge_index = torch.stack([row, col], dim=0)\n\n    return data\n\n# Path: torch_geometric/utils/_tree_decomposition.py\nfrom itertools import chain\nfrom typing import Any, List, Literal, Tuple, Union, overload\n\nimport torch\nfrom scipy.sparse.csgraph import minimum_spanning_tree\nfrom torch import Tensor\n\nfrom torch_geometric.utils import (\n    from_scipy_sparse_matrix,\n    to_scipy_sparse_matrix,\n    to_undirected,\n)\n\n\n@overload\ndef tree_decomposition(mol: Any) -> Tuple[Tensor, Tensor, int]:\n    pass\n\n\n@overload\ndef tree_decomposition(\n    mol: Any,\n    return_vocab: Literal[False],\n) -> Tuple[Tensor, Tensor, int]:\n    pass\n\n\n@overload\ndef tree_decomposition(\n    mol: Any,\n    return_vocab: Literal[True],\n) -> Tuple[Tensor, Tensor, int, Tensor]:\n    pass\n\n\ndef tree_decomposition(\n    mol: Any,\n    return_vocab: bool = False,\n) -> Union[Tuple[Tensor, Tensor, int], Tuple[Tensor, Tensor, int, Tensor]]:\n    r\"\"\"The tree decomposition algorithm of molecules from the\n    `\"Junction Tree Variational Autoencoder for Molecular Graph Generation\"\n    <https://arxiv.org/abs/1802.04364>`_ paper.\n    Returns the graph connectivity of the junction tree, the assignment\n    mapping of each atom to the clique in the junction tree, and the number\n    of cliques.\n\n    Args:\n        mol (rdkit.Chem.Mol): An :obj:`rdkit` molecule.\n        return_vocab (bool, optional): If set to :obj:`True`, will return an\n            identifier for each clique (ring, bond, bridged compounds, single).\n            (default: :obj:`False`)\n\n    :rtype: :obj:`(LongTensor, LongTensor, int)` if :obj:`return_vocab` is\n        :obj:`False`, else :obj:`(LongTensor, LongTensor, int, LongTensor)`\n    \"\"\"\n    import rdkit.Chem as Chem\n\n    # Cliques = rings and bonds.\n    cliques: List[List[int]] = [list(x) for x in Chem.GetSymmSSSR(mol)]\n    xs: List[int] = [0] * len(cliques)\n    for bond in mol.GetBonds():\n        if not bond.IsInRing():\n            cliques.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()])\n            xs.append(1)\n\n    # Generate `atom2cliques` mappings.\n    atom2cliques: List[List[int]] = [[] for i in range(mol.GetNumAtoms())]\n    for c in range(len(cliques)):\n        for atom in cliques[c]:\n            atom2cliques[atom].append(c)\n\n    # Merge rings that share more than 2 atoms as they form bridged compounds.\n    for c1 in range(len(cliques)):\n        for atom in cliques[c1]:\n            for c2 in atom2cliques[atom]:\n                if c1 >= c2 or len(cliques[c1]) <= 2 or len(cliques[c2]) <= 2:\n                    continue\n                if len(set(cliques[c1]) & set(cliques[c2])) > 2:\n                    cliques[c1] = list(set(cliques[c1]) | set(cliques[c2]))\n                    xs[c1] = 2\n                    cliques[c2] = []\n                    xs[c2] = -1\n    cliques = [c for c in cliques if len(c) > 0]\n    xs = [x for x in xs if x >= 0]\n\n    # Update `atom2cliques` mappings.\n    atom2cliques = [[] for i in range(mol.GetNumAtoms())]\n    for c in range(len(cliques)):\n        for atom in cliques[c]:\n            atom2cliques[atom].append(c)\n\n    # Add singleton cliques in case there are more than 2 intersecting\n    # cliques. We further compute the \"initial\" clique graph.\n    edges = {}\n    for atom in range(mol.GetNumAtoms()):\n        cs = atom2cliques[atom]\n        if len(cs) <= 1:\n            continue\n\n        # Number of bond clusters that the atom lies in.\n        bonds = [c for c in cs if len(cliques[c]) == 2]\n        # Number of ring clusters that the atom lies in.\n        rings = [c for c in cs if len(cliques[c]) > 4]\n\n        if len(bonds) > 2 or (len(bonds) == 2 and len(cs) > 2):\n            cliques.append([atom])\n            xs.append(3)\n            c2 = len(cliques) - 1\n            for c1 in cs:\n                edges[(c1, c2)] = 1\n\n        elif len(rings) > 2:\n            cliques.append([atom])\n            xs.append(3)\n            c2 = len(cliques) - 1\n            for c1 in cs:\n                edges[(c1, c2)] = 99\n\n        else:\n            for i in range(len(cs)):\n                for j in range(i + 1, len(cs)):\n                    c1, c2 = cs[i], cs[j]\n                    count = len(set(cliques[c1]) & set(cliques[c2]))\n                    edges[(c1, c2)] = min(count, edges.get((c1, c2), 99))\n\n    # Update `atom2cliques` mappings.\n    atom2cliques = [[] for i in range(mol.GetNumAtoms())]\n    for c in range(len(cliques)):\n        for atom in cliques[c]:\n            atom2cliques[atom].append(c)\n\n    if len(edges) > 0:\n        edge_index_T, weight = zip(*edges.items())\n        edge_index = torch.tensor(edge_index_T).t()\n        inv_weight = 100 - torch.tensor(weight)\n        graph = to_scipy_sparse_matrix(edge_index, inv_weight, len(cliques))\n        junc_tree = minimum_spanning_tree(graph)\n        edge_index, _ = from_scipy_sparse_matrix(junc_tree)\n        edge_index = to_undirected(edge_index, num_nodes=len(cliques))\n    else:\n        edge_index = torch.empty((2, 0), dtype=torch.long)\n\n    rows = [[i] * len(atom2cliques[i]) for i in range(mol.GetNumAtoms())]\n    row = torch.tensor(list(chain.from_iterable(rows)))\n    col = torch.tensor(list(chain.from_iterable(atom2cliques)))\n    atom2clique = torch.stack([row, col], dim=0).to(torch.long)\n\n    if return_vocab:\n        vocab = torch.tensor(xs, dtype=torch.long)\n        return edge_index, atom2clique, len(cliques), vocab\n    else:\n        return edge_index, atom2clique, len(cliques)\n\n# Path: torch_geometric/utils/_unbatch.py\nfrom typing import List, Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.utils import cumsum, degree\n\n\ndef unbatch(\n    src: Tensor,\n    batch: Tensor,\n    dim: int = 0,\n    batch_size: Optional[int] = None,\n) -> List[Tensor]:\n    r\"\"\"Splits :obj:`src` according to a :obj:`batch` vector along dimension\n    :obj:`dim`.\n\n    Args:\n        src (Tensor): The source tensor.\n        batch (LongTensor): The batch vector\n            :math:`\\mathbf{b} \\in {\\{ 0, \\ldots, B-1\\}}^N`, which assigns each\n            entry in :obj:`src` to a specific example. Must be ordered.\n        dim (int, optional): The dimension along which to split the :obj:`src`\n            tensor. (default: :obj:`0`)\n        batch_size (int, optional): The batch size. (default: :obj:`None`)\n\n    :rtype: :class:`List[Tensor]`\n\n    Example:\n        >>> src = torch.arange(7)\n        >>> batch = torch.tensor([0, 0, 0, 1, 1, 2, 2])\n        >>> unbatch(src, batch)\n        (tensor([0, 1, 2]), tensor([3, 4]), tensor([5, 6]))\n    \"\"\"\n    sizes = degree(batch, batch_size, dtype=torch.long).tolist()\n    return src.split(sizes, dim)\n\n\ndef unbatch_edge_index(\n    edge_index: Tensor,\n    batch: Tensor,\n    batch_size: Optional[int] = None,\n) -> List[Tensor]:\n    r\"\"\"Splits the :obj:`edge_index` according to a :obj:`batch` vector.\n\n    Args:\n        edge_index (Tensor): The edge_index tensor. Must be ordered.\n        batch (LongTensor): The batch vector\n            :math:`\\mathbf{b} \\in {\\{ 0, \\ldots, B-1\\}}^N`, which assigns each\n            node to a specific example. Must be ordered.\n        batch_size (int, optional): The batch size. (default: :obj:`None`)\n\n    :rtype: :class:`List[Tensor]`\n\n    Example:\n        >>> edge_index = torch.tensor([[0, 1, 1, 2, 2, 3, 4, 5, 5, 6],\n        ...                            [1, 0, 2, 1, 3, 2, 5, 4, 6, 5]])\n        >>> batch = torch.tensor([0, 0, 0, 0, 1, 1, 1])\n        >>> unbatch_edge_index(edge_index, batch)\n        (tensor([[0, 1, 1, 2, 2, 3],\n                [1, 0, 2, 1, 3, 2]]),\n        tensor([[0, 1, 1, 2],\n                [1, 0, 2, 1]]))\n    \"\"\"\n    deg = degree(batch, batch_size, dtype=torch.long)\n    ptr = cumsum(deg)\n\n    edge_batch = batch[edge_index[0]]\n    edge_index = edge_index - ptr[edge_batch]\n    sizes = degree(edge_batch, batch_size, dtype=torch.long).cpu().tolist()\n    return edge_index.split(sizes, dim=1)\n\n# Path: torch_geometric/utils/augmentation.py\nfrom typing import Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.utils import cumsum, negative_sampling, scatter\n\n\ndef shuffle_node(\n    x: Tensor,\n    batch: Optional[Tensor] = None,\n    training: bool = True,\n) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Randomly shuffle the feature matrix :obj:`x` along the\n    first dimmension.\n\n    The method returns (1) the shuffled :obj:`x`, (2) the permutation\n    indicating the orders of original nodes after shuffling.\n\n    Args:\n        x (FloatTensor): The feature matrix.\n        batch (LongTensor, optional): Batch vector\n            :math:`\\mathbf{b} \\in {\\{ 0, \\ldots, B-1\\}}^N`, which assigns each\n            node to a specific example. Must be ordered. (default: :obj:`None`)\n        training (bool, optional): If set to :obj:`False`, this operation is a\n            no-op. (default: :obj:`True`)\n\n    :rtype: (:class:`FloatTensor`, :class:`LongTensor`)\n\n    Example:\n        >>> # Standard case\n        >>> x = torch.tensor([[0, 1, 2],\n        ...                   [3, 4, 5],\n        ...                   [6, 7, 8],\n        ...                   [9, 10, 11]], dtype=torch.float)\n        >>> x, node_perm = shuffle_node(x)\n        >>> x\n        tensor([[ 3.,  4.,  5.],\n                [ 9., 10., 11.],\n                [ 0.,  1.,  2.],\n                [ 6.,  7.,  8.]])\n        >>> node_perm\n        tensor([1, 3, 0, 2])\n\n        >>> # For batched graphs as inputs\n        >>> batch = torch.tensor([0, 0, 1, 1])\n        >>> x, node_perm = shuffle_node(x, batch)\n        >>> x\n        tensor([[ 3.,  4.,  5.],\n                [ 0.,  1.,  2.],\n                [ 9., 10., 11.],\n                [ 6.,  7.,  8.]])\n        >>> node_perm\n        tensor([1, 0, 3, 2])\n    \"\"\"\n    if not training:\n        perm = torch.arange(x.size(0), device=x.device)\n        return x, perm\n    if batch is None:\n        perm = torch.randperm(x.size(0), device=x.device)\n        return x[perm], perm\n    num_nodes = scatter(batch.new_ones(x.size(0)), batch, dim=0, reduce='sum')\n    ptr = cumsum(num_nodes)\n    perm = torch.cat([\n        torch.randperm(n, device=x.device) + offset\n        for offset, n in zip(ptr[:-1], num_nodes)\n    ])\n    return x[perm], perm\n\n\ndef mask_feature(\n    x: Tensor,\n    p: float = 0.5,\n    mode: str = 'col',\n    fill_value: float = 0.,\n    training: bool = True,\n) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Randomly masks feature from the feature matrix\n    :obj:`x` with probability :obj:`p` using samples from\n    a Bernoulli distribution.\n\n    The method returns (1) the retained :obj:`x`, (2) the feature\n    mask broadcastable with :obj:`x` (:obj:`mode='row'` and :obj:`mode='col'`)\n    or with the same shape as :obj:`x` (:obj:`mode='all'`),\n    indicating where features are retained.\n\n    Args:\n        x (FloatTensor): The feature matrix.\n        p (float, optional): The masking ratio. (default: :obj:`0.5`)\n        mode (str, optional): The masked scheme to use for feature masking.\n            (:obj:`\"row\"`, :obj:`\"col\"` or :obj:`\"all\"`).\n            If :obj:`mode='col'`, will mask entire features of all nodes\n            from the feature matrix. If :obj:`mode='row'`, will mask entire\n            nodes from the feature matrix. If :obj:`mode='all'`, will mask\n            individual features across all nodes. (default: :obj:`'col'`)\n        fill_value (float, optional): The value for masked features in the\n            output tensor. (default: :obj:`0`)\n        training (bool, optional): If set to :obj:`False`, this operation is a\n            no-op. (default: :obj:`True`)\n\n    :rtype: (:class:`FloatTensor`, :class:`BoolTensor`)\n\n    Examples:\n        >>> # Masked features are column-wise sampled\n        >>> x = torch.tensor([[1, 2, 3],\n        ...                   [4, 5, 6],\n        ...                   [7, 8, 9]], dtype=torch.float)\n        >>> x, feat_mask = mask_feature(x)\n        >>> x\n        tensor([[1., 0., 3.],\n                [4., 0., 6.],\n                [7., 0., 9.]]),\n        >>> feat_mask\n        tensor([[True, False, True]])\n\n        >>> # Masked features are row-wise sampled\n        >>> x, feat_mask = mask_feature(x, mode='row')\n        >>> x\n        tensor([[1., 2., 3.],\n                [0., 0., 0.],\n                [7., 8., 9.]]),\n        >>> feat_mask\n        tensor([[True], [False], [True]])\n\n        >>> # Masked features are uniformly sampled\n        >>> x, feat_mask = mask_feature(x, mode='all')\n        >>> x\n        tensor([[0., 0., 0.],\n                [4., 0., 6.],\n                [0., 0., 9.]])\n        >>> feat_mask\n        tensor([[False, False, False],\n                [True, False,  True],\n                [False, False,  True]])\n    \"\"\"\n    if p < 0. or p > 1.:\n        raise ValueError(f'Masking ratio has to be between 0 and 1 '\n                         f'(got {p}')\n    if not training or p == 0.0:\n        return x, torch.ones_like(x, dtype=torch.bool)\n    assert mode in ['row', 'col', 'all']\n\n    if mode == 'row':\n        mask = torch.rand(x.size(0), device=x.device) >= p\n        mask = mask.view(-1, 1)\n    elif mode == 'col':\n        mask = torch.rand(x.size(1), device=x.device) >= p\n        mask = mask.view(1, -1)\n    else:\n        mask = torch.rand_like(x) >= p\n\n    x = x.masked_fill(~mask, fill_value)\n    return x, mask\n\n\ndef add_random_edge(\n    edge_index: Tensor,\n    p: float = 0.5,\n    force_undirected: bool = False,\n    num_nodes: Optional[Union[int, Tuple[int, int]]] = None,\n    training: bool = True,\n) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Randomly adds edges to :obj:`edge_index`.\n\n    The method returns (1) the retained :obj:`edge_index`, (2) the added\n    edge indices.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        p (float): Ratio of added edges to the existing edges.\n            (default: :obj:`0.5`)\n        force_undirected (bool, optional): If set to :obj:`True`,\n            added edges will be undirected.\n            (default: :obj:`False`)\n        num_nodes (int, Tuple[int], optional): The overall number of nodes,\n            *i.e.* :obj:`max_val + 1`, or the number of source and\n            destination nodes, *i.e.* :obj:`(max_src_val + 1, max_dst_val + 1)`\n            of :attr:`edge_index`. (default: :obj:`None`)\n        training (bool, optional): If set to :obj:`False`, this operation is a\n            no-op. (default: :obj:`True`)\n\n    :rtype: (:class:`LongTensor`, :class:`LongTensor`)\n\n    Examples:\n        >>> # Standard case\n        >>> edge_index = torch.tensor([[0, 1, 1, 2, 2, 3],\n        ...                            [1, 0, 2, 1, 3, 2]])\n        >>> edge_index, added_edges = add_random_edge(edge_index, p=0.5)\n        >>> edge_index\n        tensor([[0, 1, 1, 2, 2, 3, 2, 1, 3],\n                [1, 0, 2, 1, 3, 2, 0, 2, 1]])\n        >>> added_edges\n        tensor([[2, 1, 3],\n                [0, 2, 1]])\n\n        >>> # The returned graph is kept undirected\n        >>> edge_index, added_edges = add_random_edge(edge_index, p=0.5,\n        ...                                           force_undirected=True)\n        >>> edge_index\n        tensor([[0, 1, 1, 2, 2, 3, 2, 1, 3, 0, 2, 1],\n                [1, 0, 2, 1, 3, 2, 0, 2, 1, 2, 1, 3]])\n        >>> added_edges\n        tensor([[2, 1, 3, 0, 2, 1],\n                [0, 2, 1, 2, 1, 3]])\n\n        >>> # For bipartite graphs\n        >>> edge_index = torch.tensor([[0, 1, 2, 3, 4, 5],\n        ...                            [2, 3, 1, 4, 2, 1]])\n        >>> edge_index, added_edges = add_random_edge(edge_index, p=0.5,\n        ...                                           num_nodes=(6, 5))\n        >>> edge_index\n        tensor([[0, 1, 2, 3, 4, 5, 3, 4, 1],\n                [2, 3, 1, 4, 2, 1, 1, 3, 2]])\n        >>> added_edges\n        tensor([[3, 4, 1],\n                [1, 3, 2]])\n    \"\"\"\n    if p < 0. or p > 1.:\n        raise ValueError(f\"Ratio of added edges has to be between 0 and 1 \"\n                         f\"(got '{p}')\")\n    if force_undirected and isinstance(num_nodes, (tuple, list)):\n        raise RuntimeError(\"'force_undirected' is not supported for \"\n                           \"bipartite graphs\")\n\n    device = edge_index.device\n    if not training or p == 0.0:\n        edge_index_to_add = torch.tensor([[], []], device=device)\n        return edge_index, edge_index_to_add\n\n    edge_index_to_add = negative_sampling(\n        edge_index=edge_index,\n        num_nodes=num_nodes,\n        num_neg_samples=round(edge_index.size(1) * p),\n        force_undirected=force_undirected,\n    )\n\n    edge_index = torch.cat([edge_index, edge_index_to_add], dim=1)\n\n    return edge_index, edge_index_to_add\n\n# Path: torch_geometric/utils/convert.py\nfrom collections import defaultdict\nfrom typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Union\n\nimport scipy.sparse\nimport torch\nfrom torch import Tensor\nfrom torch.utils.dlpack import from_dlpack, to_dlpack\n\nimport torch_geometric\nfrom torch_geometric.utils.num_nodes import maybe_num_nodes\n\n\n\ndef to_scipy_sparse_matrix(\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> scipy.sparse.coo_matrix:\n    r\"\"\"Converts a graph given by edge indices and edge attributes to a scipy\n    sparse matrix.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional\n            edge features. (default: :obj:`None`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`index`. (default: :obj:`None`)\n\n    Examples:\n        >>> edge_index = torch.tensor([\n        ...     [0, 1, 1, 2, 2, 3],\n        ...     [1, 0, 2, 1, 3, 2],\n        ... ])\n        >>> to_scipy_sparse_matrix(edge_index)\n        <4x4 sparse matrix of type '<class 'numpy.float32'>'\n            with 6 stored elements in COOrdinate format>\n    \"\"\"\n    row, col = edge_index.cpu()\n\n    if edge_attr is None:\n        edge_attr = torch.ones(row.size(0))\n    else:\n        edge_attr = edge_attr.view(-1).cpu()\n        assert edge_attr.size(0) == row.size(0)\n\n    N = maybe_num_nodes(edge_index, num_nodes)\n    out = scipy.sparse.coo_matrix(\n        (edge_attr.numpy(), (row.numpy(), col.numpy())), (N, N))\n    return out\n\n\ndef from_scipy_sparse_matrix(\n        A: scipy.sparse.spmatrix) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Converts a scipy sparse matrix to edge indices and edge attributes.\n\n    Args:\n        A (scipy.sparse): A sparse matrix.\n\n    Examples:\n        >>> edge_index = torch.tensor([\n        ...     [0, 1, 1, 2, 2, 3],\n        ...     [1, 0, 2, 1, 3, 2],\n        ... ])\n        >>> adj = to_scipy_sparse_matrix(edge_index)\n        >>> # `edge_index` and `edge_weight` are both returned\n        >>> from_scipy_sparse_matrix(adj)\n        (tensor([[0, 1, 1, 2, 2, 3],\n                [1, 0, 2, 1, 3, 2]]),\n        tensor([1., 1., 1., 1., 1., 1.]))\n    \"\"\"\n    A = A.tocoo()\n    row = torch.from_numpy(A.row).to(torch.long)\n    col = torch.from_numpy(A.col).to(torch.long)\n    edge_index = torch.stack([row, col], dim=0)\n    edge_weight = torch.from_numpy(A.data)\n    return edge_index, edge_weight\n\n\ndef to_networkx(\n    data: Union[\n        'torch_geometric.data.Data',\n        'torch_geometric.data.HeteroData',\n    ],\n    node_attrs: Optional[Iterable[str]] = None,\n    edge_attrs: Optional[Iterable[str]] = None,\n    graph_attrs: Optional[Iterable[str]] = None,\n    to_undirected: Optional[Union[bool, str]] = False,\n    to_multi: bool = False,\n    remove_self_loops: bool = False,\n) -> Any:\n    r\"\"\"Converts a :class:`torch_geometric.data.Data` instance to a\n    :obj:`networkx.Graph` if :attr:`to_undirected` is set to :obj:`True`, or\n    a directed :obj:`networkx.DiGraph` otherwise.\n\n    Args:\n        data (torch_geometric.data.Data or torch_geometric.data.HeteroData): A\n            homogeneous or heterogeneous data object.\n        node_attrs (iterable of str, optional): The node attributes to be\n            copied. (default: :obj:`None`)\n        edge_attrs (iterable of str, optional): The edge attributes to be\n            copied. (default: :obj:`None`)\n        graph_attrs (iterable of str, optional): The graph attributes to be\n            copied. (default: :obj:`None`)\n        to_undirected (bool or str, optional): If set to :obj:`True`, will\n            return a :class:`networkx.Graph` instead of a\n            :class:`networkx.DiGraph`.\n            By default, will include all edges and make them undirected.\n            If set to :obj:`\"upper\"`, the undirected graph will only correspond\n            to the upper triangle of the input adjacency matrix.\n            If set to :obj:`\"lower\"`, the undirected graph will only correspond\n            to the lower triangle of the input adjacency matrix.\n            Only applicable in case the :obj:`data` object holds a homogeneous\n            graph. (default: :obj:`False`)\n        to_multi (bool, optional): if set to :obj:`True`, will return a\n            :class:`networkx.MultiGraph` or a :class:`networkx:MultiDiGraph`\n            (depending on the :obj:`to_undirected` option), which will not drop\n            duplicated edges that may exist in :obj:`data`.\n            (default: :obj:`False`)\n        remove_self_loops (bool, optional): If set to :obj:`True`, will not\n            include self-loops in the resulting graph. (default: :obj:`False`)\n\n    Examples:\n        >>> edge_index = torch.tensor([\n        ...     [0, 1, 1, 2, 2, 3],\n        ...     [1, 0, 2, 1, 3, 2],\n        ... ])\n        >>> data = Data(edge_index=edge_index, num_nodes=4)\n        >>> to_networkx(data)\n        <networkx.classes.digraph.DiGraph at 0x2713fdb40d0>\n\n    \"\"\"\n    import networkx as nx\n\n    from torch_geometric.data import HeteroData\n\n    to_undirected_upper: bool = to_undirected == 'upper'\n    to_undirected_lower: bool = to_undirected == 'lower'\n\n    to_undirected = to_undirected is True\n    to_undirected |= to_undirected_upper or to_undirected_lower\n    assert isinstance(to_undirected, bool)\n\n    if isinstance(data, HeteroData) and to_undirected:\n        raise ValueError(\"'to_undirected' is not supported in \"\n                         \"'to_networkx' for heterogeneous graphs\")\n\n    if to_undirected:\n        G = nx.MultiGraph() if to_multi else nx.Graph()\n    else:\n        G = nx.MultiDiGraph() if to_multi else nx.DiGraph()\n\n    def to_networkx_value(value: Any) -> Any:\n        return value.tolist() if isinstance(value, Tensor) else value\n\n    for key in graph_attrs or []:\n        G.graph[key] = to_networkx_value(data[key])\n\n    node_offsets = data.node_offsets\n    for node_store in data.node_stores:\n        start = node_offsets[node_store._key]\n        assert node_store.num_nodes is not None\n        for i in range(node_store.num_nodes):\n            node_kwargs: Dict[str, Any] = {}\n            if isinstance(data, HeteroData):\n                node_kwargs['type'] = node_store._key\n            for key in node_attrs or []:\n                node_kwargs[key] = to_networkx_value(node_store[key][i])\n\n            G.add_node(start + i, **node_kwargs)\n\n    for edge_store in data.edge_stores:\n        for i, (v, w) in enumerate(edge_store.edge_index.t().tolist()):\n            if to_undirected_upper and v > w:\n                continue\n            elif to_undirected_lower and v < w:\n                continue\n            elif remove_self_loops and v == w and not edge_store.is_bipartite(\n            ):\n                continue\n\n            edge_kwargs: Dict[str, Any] = {}\n            if isinstance(data, HeteroData):\n                v = v + node_offsets[edge_store._key[0]]\n                w = w + node_offsets[edge_store._key[-1]]\n                edge_kwargs['type'] = edge_store._key\n            for key in edge_attrs or []:\n                edge_kwargs[key] = to_networkx_value(edge_store[key][i])\n\n            G.add_edge(v, w, **edge_kwargs)\n\n    return G\n\n\ndef from_networkx(\n    G: Any,\n    group_node_attrs: Optional[Union[List[str], Literal['all']]] = None,\n    group_edge_attrs: Optional[Union[List[str], Literal['all']]] = None,\n) -> 'torch_geometric.data.Data':\n    r\"\"\"Converts a :obj:`networkx.Graph` or :obj:`networkx.DiGraph` to a\n    :class:`torch_geometric.data.Data` instance.\n\n    Args:\n        G (networkx.Graph or networkx.DiGraph): A networkx graph.\n        group_node_attrs (List[str] or \"all\", optional): The node attributes to\n            be concatenated and added to :obj:`data.x`. (default: :obj:`None`)\n        group_edge_attrs (List[str] or \"all\", optional): The edge attributes to\n            be concatenated and added to :obj:`data.edge_attr`.\n            (default: :obj:`None`)\n\n    .. note::\n\n        All :attr:`group_node_attrs` and :attr:`group_edge_attrs` values must\n        be numeric.\n\n    Examples:\n        >>> edge_index = torch.tensor([\n        ...     [0, 1, 1, 2, 2, 3],\n        ...     [1, 0, 2, 1, 3, 2],\n        ... ])\n        >>> data = Data(edge_index=edge_index, num_nodes=4)\n        >>> g = to_networkx(data)\n        >>> # A `Data` object is returned\n        >>> from_networkx(g)\n        Data(edge_index=[2, 6], num_nodes=4)\n    \"\"\"\n    import networkx as nx\n\n    from torch_geometric.data import Data\n\n    G = G.to_directed() if not nx.is_directed(G) else G\n\n    mapping = dict(zip(G.nodes(), range(G.number_of_nodes())))\n    edge_index = torch.empty((2, G.number_of_edges()), dtype=torch.long)\n    for i, (src, dst) in enumerate(G.edges()):\n        edge_index[0, i] = mapping[src]\n        edge_index[1, i] = mapping[dst]\n\n    data_dict: Dict[str, Any] = defaultdict(list)\n    data_dict['edge_index'] = edge_index\n\n    node_attrs: List[str] = []\n    if G.number_of_nodes() > 0:\n        node_attrs = list(next(iter(G.nodes(data=True)))[-1].keys())\n\n    edge_attrs: List[str] = []\n    if G.number_of_edges() > 0:\n        edge_attrs = list(next(iter(G.edges(data=True)))[-1].keys())\n\n    if group_node_attrs is not None and not isinstance(group_node_attrs, list):\n        group_node_attrs = node_attrs\n\n    if group_edge_attrs is not None and not isinstance(group_edge_attrs, list):\n        group_edge_attrs = edge_attrs\n\n    for i, (_, feat_dict) in enumerate(G.nodes(data=True)):\n        if set(feat_dict.keys()) != set(node_attrs):\n            raise ValueError('Not all nodes contain the same attributes')\n        for key, value in feat_dict.items():\n            data_dict[str(key)].append(value)\n\n    for i, (_, _, feat_dict) in enumerate(G.edges(data=True)):\n        if set(feat_dict.keys()) != set(edge_attrs):\n            raise ValueError('Not all edges contain the same attributes')\n        for key, value in feat_dict.items():\n            key = f'edge_{key}' if key in node_attrs else key\n            data_dict[str(key)].append(value)\n\n    for key, value in G.graph.items():\n        if key == 'node_default' or key == 'edge_default':\n            continue  # Do not load default attributes.\n        key = f'graph_{key}' if key in node_attrs else key\n        data_dict[str(key)] = value\n\n    for key, value in data_dict.items():\n        if isinstance(value, (tuple, list)) and isinstance(value[0], Tensor):\n            data_dict[key] = torch.stack(value, dim=0)\n        else:\n            try:\n                data_dict[key] = torch.as_tensor(value)\n            except Exception:\n                pass\n\n    data = Data.from_dict(data_dict)\n\n    if group_node_attrs is not None:\n        xs = []\n        for key in group_node_attrs:\n            x = data[key]\n            x = x.view(-1, 1) if x.dim() <= 1 else x\n            xs.append(x)\n            del data[key]\n        data.x = torch.cat(xs, dim=-1)\n\n    if group_edge_attrs is not None:\n        xs = []\n        for key in group_edge_attrs:\n            key = f'edge_{key}' if key in node_attrs else key\n            x = data[key]\n            x = x.view(-1, 1) if x.dim() <= 1 else x\n            xs.append(x)\n            del data[key]\n        data.edge_attr = torch.cat(xs, dim=-1)\n\n    if data.x is None and data.pos is None:\n        data.num_nodes = G.number_of_nodes()\n\n    return data\n\n\ndef to_networkit(\n    edge_index: Tensor,\n    edge_weight: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n    directed: bool = True,\n) -> Any:\n    r\"\"\"Converts a :obj:`(edge_index, edge_weight)` tuple to a\n    :class:`networkit.Graph`.\n\n    Args:\n        edge_index (torch.Tensor): The edge indices of the graph.\n        edge_weight (torch.Tensor, optional): The edge weights of the graph.\n            (default: :obj:`None`)\n        num_nodes (int, optional): The number of nodes in the graph.\n            (default: :obj:`None`)\n        directed (bool, optional): If set to :obj:`False`, the graph will be\n            undirected. (default: :obj:`True`)\n    \"\"\"\n    import networkit as nk\n\n    num_nodes = maybe_num_nodes(edge_index, num_nodes)\n\n    g = nk.graph.Graph(\n        num_nodes,\n        weighted=edge_weight is not None,\n        directed=directed,\n    )\n\n    if edge_weight is None:\n        edge_weight = torch.ones(edge_index.size(1))\n\n    if not directed:\n        mask = edge_index[0] <= edge_index[1]\n        edge_index = edge_index[:, mask]\n        edge_weight = edge_weight[mask]\n\n    for (u, v), w in zip(edge_index.t().tolist(), edge_weight.tolist()):\n        g.addEdge(u, v, w)\n\n    return g\n\n\ndef from_networkit(g: Any) -> Tuple[Tensor, Optional[Tensor]]:\n    r\"\"\"Converts a :class:`networkit.Graph` to a\n    :obj:`(edge_index, edge_weight)` tuple.\n    If the :class:`networkit.Graph` is not weighted, the returned\n    :obj:`edge_weight` will be :obj:`None`.\n\n    Args:\n        g (networkkit.graph.Graph): A :obj:`networkit` graph object.\n    \"\"\"\n    is_directed = g.isDirected()\n    is_weighted = g.isWeighted()\n\n    edge_indices, edge_weights = [], []\n    for u, v, w in g.iterEdgesWeights():\n        edge_indices.append([u, v])\n        edge_weights.append(w)\n        if not is_directed:\n            edge_indices.append([v, u])\n            edge_weights.append(w)\n\n    edge_index = torch.tensor(edge_indices).t().contiguous()\n    edge_weight = torch.tensor(edge_weights) if is_weighted else None\n\n    return edge_index, edge_weight\n\n\ndef to_trimesh(data: 'torch_geometric.data.Data') -> Any:\n    r\"\"\"Converts a :class:`torch_geometric.data.Data` instance to a\n    :obj:`trimesh.Trimesh`.\n\n    Args:\n        data (torch_geometric.data.Data): The data object.\n\n    Example:\n        >>> pos = torch.tensor([[0, 0, 0], [1, 0, 0], [0, 1, 0], [1, 1, 0]],\n        ...                    dtype=torch.float)\n        >>> face = torch.tensor([[0, 1, 2], [1, 2, 3]]).t()\n\n        >>> data = Data(pos=pos, face=face)\n        >>> to_trimesh(data)\n        <trimesh.Trimesh(vertices.shape=(4, 3), faces.shape=(2, 3))>\n    \"\"\"\n    import trimesh\n\n    assert data.pos is not None\n    assert data.face is not None\n\n    return trimesh.Trimesh(\n        vertices=data.pos.detach().cpu().numpy(),\n        faces=data.face.detach().t().cpu().numpy(),\n        process=False,\n    )\n\n\ndef from_trimesh(mesh: Any) -> 'torch_geometric.data.Data':\n    r\"\"\"Converts a :obj:`trimesh.Trimesh` to a\n    :class:`torch_geometric.data.Data` instance.\n\n    Args:\n        mesh (trimesh.Trimesh): A :obj:`trimesh` mesh.\n\n    Example:\n        >>> pos = torch.tensor([[0, 0, 0], [1, 0, 0], [0, 1, 0], [1, 1, 0]],\n        ...                    dtype=torch.float)\n        >>> face = torch.tensor([[0, 1, 2], [1, 2, 3]]).t()\n\n        >>> data = Data(pos=pos, face=face)\n        >>> mesh = to_trimesh(data)\n        >>> from_trimesh(mesh)\n        Data(pos=[4, 3], face=[3, 2])\n    \"\"\"\n    from torch_geometric.data import Data\n\n    pos = torch.from_numpy(mesh.vertices).to(torch.float)\n    face = torch.from_numpy(mesh.faces).t().contiguous()\n\n    return Data(pos=pos, face=face)\n\n\ndef to_cugraph(\n    edge_index: Tensor,\n    edge_weight: Optional[Tensor] = None,\n    relabel_nodes: bool = True,\n    directed: bool = True,\n) -> Any:\n    r\"\"\"Converts a graph given by :obj:`edge_index` and optional\n    :obj:`edge_weight` into a :obj:`cugraph` graph object.\n\n    Args:\n        edge_index (torch.Tensor): The edge indices of the graph.\n        edge_weight (torch.Tensor, optional): The edge weights of the graph.\n            (default: :obj:`None`)\n        relabel_nodes (bool, optional): If set to :obj:`True`,\n            :obj:`cugraph` will remove any isolated nodes, leading to a\n            relabeling of nodes. (default: :obj:`True`)\n        directed (bool, optional): If set to :obj:`False`, the graph will be\n            undirected. (default: :obj:`True`)\n    \"\"\"\n    import cudf\n    import cugraph\n\n    g = cugraph.Graph(directed=directed)\n    df = cudf.from_dlpack(to_dlpack(edge_index.t()))\n\n    if edge_weight is not None:\n        assert edge_weight.dim() == 1\n        df['2'] = cudf.from_dlpack(to_dlpack(edge_weight))\n\n    g.from_cudf_edgelist(\n        df,\n        source=0,\n        destination=1,\n        edge_attr='2' if edge_weight is not None else None,\n        renumber=relabel_nodes,\n    )\n\n    return g\n\n\ndef from_cugraph(g: Any) -> Tuple[Tensor, Optional[Tensor]]:\n    r\"\"\"Converts a :obj:`cugraph` graph object into :obj:`edge_index` and\n    optional :obj:`edge_weight` tensors.\n\n    Args:\n        g (cugraph.Graph): A :obj:`cugraph` graph object.\n    \"\"\"\n    df = g.view_edge_list()\n\n    src = from_dlpack(df[0].to_dlpack()).long()\n    dst = from_dlpack(df[1].to_dlpack()).long()\n    edge_index = torch.stack([src, dst], dim=0)\n\n    edge_weight = None\n    if '2' in df:\n        edge_weight = from_dlpack(df['2'].to_dlpack())\n\n    return edge_index, edge_weight\n\n\ndef to_dgl(\n    data: Union['torch_geometric.data.Data', 'torch_geometric.data.HeteroData']\n) -> Any:\n    r\"\"\"Converts a :class:`torch_geometric.data.Data` or\n    :class:`torch_geometric.data.HeteroData` instance to a :obj:`dgl` graph\n    object.\n\n    Args:\n        data (torch_geometric.data.Data or torch_geometric.data.HeteroData):\n            The data object.\n\n    Example:\n...\n", "needle_token_start": 10274, "needle_token_end": 10716, "code_context_ntokens": 16127, "output": ["```python\nfrom scipy.sparse.csgraph import minimum_spanning_tree\nfrom torch import Tensor\n\nfrom torch_geometric.utils import (\n    from_scipy_sparse_matrix,\n    to_scipy_sparse_matrix,\n    to_undirected,\n)\n\n\n@overload\ndef tree_decomposition(mol: Any) -> Tuple[Tensor, Tensor, int]:\n    pass\n\n\n@overload\ndef tree_decomposition(\n    mol: Any,\n    return_vocab: Literal[False],\n) -> Tuple[Tensor, Tensor, int]:\n    pass\n\n\n@overload\ndef tree_decomposition(\n    mol: Any,\n    return_vocab: Literal[True],\n) -> Tuple[Tensor, Tensor, int, Tensor]:\n    pass\n\n\ndef tree_decomposition(\n    mol: Any,\n    return_vocab: bool = False,\n) -> Union[Tuple[Tensor, Tensor, int], Tuple[Tensor, Tensor, int, Tensor]]:\n    r\"\"\"The tree decomposition algorithm of molecules from the\n    `\"Junction Tree Variational Autoencoder for Molecular Graph Generation\"\n    <https://arxiv.org/abs/1802.04364>`_ paper.\n    Returns the graph connectivity of the junction tree, the assignment\n    mapping of each atom to the clique in the junction tree, and the number\n    of cliques.\n\n    Args:\n        mol (rdkit.Chem.Mol): An :obj:`rdkit` molecule.\n        return_vocab (bool, optional): If set to :obj:`True`, will return an\n            identifier for each clique (ring, bond, bridged compounds, single).\n            (default: :obj:`False`)\n\n    :rtype: :obj:`(LongTensor, LongTensor, int)` if :obj:`return_vocab` is\n        :obj:`False`, else :obj:`(LongTensor, LongTensor, int, LongTensor)`\n    \"\"\"\n    import rdkit.Chem as Chem\n\n    # Cliques = rings and bonds.\n    cliques: List[List[int]] = [list(x) for x in Chem.GetSymmSSSR(mol)]\n    xs: List[int] = [0] * len(cliques)\n    for bond in mol.GetBonds():\n        if not bond.IsInRing():\n            cliques.append([bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()])\n            xs.append(1)\n\n    # Generate `atom2cliques` mappings.\n    atom2cliques: List[List[int]] = [[] for i in range(mol.GetNumAtoms())]\n    for c in range(len(cliques)):\n        for atom in cliques[c]:\n            atom2cliques[atom].append(c)\n\n    # Merge rings that share more than 2 atoms as they form bridged compounds.\n    for c1 in range(len(cliques)):\n        for atom in cliques[c1]:\n            for c2 in atom2cliques[atom]:\n                if c1 >= c2 or len(cliques[c1]) <= 2 or len(cliques[c2]) <= 2:\n                    continue\n                if len(set(cliques[c1]) & set(cliques[c2])) > 2:\n                    cliques[c1] = list(set(cliques[c1]) | set(cliques[c2]))\n                    xs[c1] = 2\n                    cliques[c2] = []\n                    xs[c2] = -1\n    cliques = [c for c in cliques if len(c) > 0]\n    xs = [x for x in xs if x >= 0]\n\n    # Update `atom2cliques` mappings.\n    atom2cliques = [[] for i in range(mol.GetNumAtoms())]\n    for c in range(len(cliques)):\n        for atom in cliques[c]:\n            atom2cliques[atom].append(c)\n\n    # Add singleton cliques in case there are more than 2 intersecting\n    # cliques. We further compute the \"initial\" clique graph.\n    edges = {}\n    for atom in range(mol.GetNumAtoms()):\n        cs = atom2cliques[atom]\n        if len(cs) <= 1:\n            continue\n\n        # Number of bond clusters that the atom lies in.\n        bonds = [c for c in cs if len(cliques[c]) == 2]\n        # Number of ring clusters that the atom lies in.\n        rings = [c for c in cs if len(cliques[c]) > 4]\n\n        if len(bonds) > 2 or (len(bonds) == 2 and len(cs) > 2):\n            cliques.append([atom])\n            xs.append(3)\n            c2 = len(cliques) - 1\n            for c1 in cs:\n                edges[(c1, c2)] = 1\n\n        elif len(rings) > 2:\n            cliques.append([atom])\n            xs.append(3)\n            c2 = len(cliques) - 1\n            for"]}
{"repo": "pyg-team/pytorch_geometric", "name": "set_tuple_item", "language": "python", "path": "torch_geometric/edge_index.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to replace an item in a tuple with a new value at a specified index, since tuples are immutable in Python.\n2. **Input**: The function takes three parameters: a tuple containing any types of values, an integer representing the index at which the replacement should occur, and the new value to be inserted at that index.\n3. **Output**: It returns a new tuple that is a copy of the original tuple but with the specified item replaced by the new value.\n4. **Procedure**: The function first checks if the provided index is within the valid range for the tuple. If the index is out of range, it raises an `IndexError`. If the index is negative, it adjusts it to access the tuple in reverse order. The function then constructs a new tuple by slicing the original tuple up to the specified index, adding the new value, and then appending the remainder of the original tuple after the specified index.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/utils/_to_dense_adj.py\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import OptTensor\nfrom torch_geometric.utils import cumsum, scatter\n\n\ndef to_dense_adj(\n    edge_index: Tensor,\n    batch: OptTensor = None,\n    edge_attr: OptTensor = None,\n    max_num_nodes: Optional[int] = None,\n    batch_size: Optional[int] = None,\n) -> Tensor:\n    r\"\"\"Converts batched sparse adjacency matrices given by edge indices and\n    edge attributes to a single dense batched adjacency matrix.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        batch (LongTensor, optional): Batch vector\n            :math:`\\mathbf{b} \\in {\\{ 0, \\ldots, B-1\\}}^N`, which assigns each\n            node to a specific example. (default: :obj:`None`)\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional edge\n            features.\n            If :obj:`edge_index` contains duplicated edges, the dense adjacency\n            matrix output holds the summed up entries of :obj:`edge_attr` for\n            duplicated edges. (default: :obj:`None`)\n        max_num_nodes (int, optional): The size of the output node dimension.\n            (default: :obj:`None`)\n        batch_size (int, optional): The batch size. (default: :obj:`None`)\n\n    :rtype: :class:`Tensor`\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 0, 1, 2, 3],\n        ...                            [0, 1, 0, 3, 0]])\n        >>> batch = torch.tensor([0, 0, 1, 1])\n        >>> to_dense_adj(edge_index, batch)\n        tensor([[[1., 1.],\n                [1., 0.]],\n                [[0., 1.],\n                [1., 0.]]])\n\n        >>> to_dense_adj(edge_index, batch, max_num_nodes=4)\n        tensor([[[1., 1., 0., 0.],\n                [1., 0., 0., 0.],\n                [0., 0., 0., 0.],\n                [0., 0., 0., 0.]],\n                [[0., 1., 0., 0.],\n                [1., 0., 0., 0.],\n                [0., 0., 0., 0.],\n                [0., 0., 0., 0.]]])\n\n        >>> edge_attr = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0])\n        >>> to_dense_adj(edge_index, batch, edge_attr)\n        tensor([[[1., 2.],\n                [3., 0.]],\n                [[0., 4.],\n                [5., 0.]]])\n    \"\"\"\n    if batch is None:\n        max_index = int(edge_index.max()) + 1 if edge_index.numel() > 0 else 0\n        batch = edge_index.new_zeros(max_index)\n\n    if batch_size is None:\n        batch_size = int(batch.max()) + 1 if batch.numel() > 0 else 1\n\n    one = batch.new_ones(batch.size(0))\n    num_nodes = scatter(one, batch, dim=0, dim_size=batch_size, reduce='sum')\n    cum_nodes = cumsum(num_nodes)\n\n    idx0 = batch[edge_index[0]]\n    idx1 = edge_index[0] - cum_nodes[batch][edge_index[0]]\n    idx2 = edge_index[1] - cum_nodes[batch][edge_index[1]]\n\n    if max_num_nodes is None:\n        max_num_nodes = int(num_nodes.max())\n\n    elif ((idx1.numel() > 0 and idx1.max() >= max_num_nodes)\n          or (idx2.numel() > 0 and idx2.max() >= max_num_nodes)):\n        mask = (idx1 < max_num_nodes) & (idx2 < max_num_nodes)\n        idx0 = idx0[mask]\n        idx1 = idx1[mask]\n        idx2 = idx2[mask]\n        edge_attr = None if edge_attr is None else edge_attr[mask]\n\n    if edge_attr is None:\n        edge_attr = torch.ones(idx0.numel(), device=edge_index.device)\n\n    size = [batch_size, max_num_nodes, max_num_nodes]\n    size += list(edge_attr.size())[1:]\n    flattened_size = batch_size * max_num_nodes * max_num_nodes\n\n    idx = idx0 * max_num_nodes * max_num_nodes + idx1 * max_num_nodes + idx2\n    adj = scatter(edge_attr, idx, dim=0, dim_size=flattened_size, reduce='sum')\n    adj = adj.view(size)\n\n    return adj\n\n# Path: torch_geometric/utils/_assortativity.py\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import Adj, SparseTensor\nfrom torch_geometric.utils import coalesce, degree\nfrom torch_geometric.utils._to_dense_adj import to_dense_adj\n\n\ndef assortativity(edge_index: Adj) -> float:\n    r\"\"\"The degree assortativity coefficient from the\n    `\"Mixing patterns in networks\"\n    <https://arxiv.org/abs/cond-mat/0209450>`_ paper.\n    Assortativity in a network refers to the tendency of nodes to\n    connect with other similar nodes over dissimilar nodes.\n    It is computed from Pearson correlation coefficient of the node degrees.\n\n    Args:\n        edge_index (Tensor or SparseTensor): The graph connectivity.\n\n    Returns:\n        The value of the degree assortativity coefficient for the input\n        graph :math:`\\in [-1, 1]`\n\n    Example:\n        >>> edge_index = torch.tensor([[0, 1, 2, 3, 2],\n        ...                            [1, 2, 0, 1, 3]])\n        >>> assortativity(edge_index)\n        -0.666667640209198\n    \"\"\"\n    if isinstance(edge_index, SparseTensor):\n        adj: SparseTensor = edge_index\n        row, col, _ = adj.coo()\n    else:\n        assert isinstance(edge_index, Tensor)\n        row, col = edge_index\n\n    device = row.device\n    out_deg = degree(row, dtype=torch.long)\n    in_deg = degree(col, dtype=torch.long)\n    degrees = torch.unique(torch.cat([out_deg, in_deg]))\n    mapping = row.new_zeros(degrees.max().item() + 1)\n    mapping[degrees] = torch.arange(degrees.size(0), device=device)\n\n    # Compute degree mixing matrix (joint probability distribution) `M`\n    num_degrees = degrees.size(0)\n    src_deg = mapping[out_deg[row]]\n    dst_deg = mapping[in_deg[col]]\n\n    pairs = torch.stack([src_deg, dst_deg], dim=0)\n    occurrence = torch.ones(pairs.size(1), device=device)\n    pairs, occurrence = coalesce(pairs, occurrence)\n    M = to_dense_adj(pairs, edge_attr=occurrence, max_num_nodes=num_degrees)[0]\n    # normalization\n    M /= M.sum()\n\n    # numeric assortativity coefficient, computed by\n    # Pearson correlation coefficient of the node degrees\n    x = y = degrees.float()\n    a, b = M.sum(0), M.sum(1)\n\n    vara = (a * x**2).sum() - ((a * x).sum())**2\n    varb = (b * x**2).sum() - ((b * x).sum())**2\n    xy = torch.outer(x, y)\n    ab = torch.outer(a, b)\n    out = (xy * (M - ab)).sum() / (vara * varb).sqrt()\n    return out.item()\n\n# Path: torch_geometric/edge_index.py\nimport functools\nimport typing\nfrom enum import Enum\nfrom typing import (\n    Any,\n    Callable,\n    Dict,\n    List,\n    Literal,\n    NamedTuple,\n    Optional,\n    Sequence,\n    Set,\n    Tuple,\n    Type,\n    Union,\n    get_args,\n    overload,\n)\n\nimport torch\nfrom torch import Tensor\n\nimport torch_geometric.typing\nfrom torch_geometric import is_compiling\nfrom torch_geometric.typing import SparseTensor\n\nHANDLED_FUNCTIONS: Dict[Callable, Callable] = {}\n\nif torch_geometric.typing.WITH_PT20:\n    SUPPORTED_DTYPES: Set[torch.dtype] = {\n        torch.int32,\n        torch.int64,\n    }\nelif not typing.TYPE_CHECKING:  # pragma: no cover\n    SUPPORTED_DTYPES: Set[torch.dtype] = {\n        torch.int64,\n    }\n\nReduceType = Literal['sum', 'mean', 'amin', 'amax', 'add', 'min', 'max']\nPYG_REDUCE: Dict[ReduceType, ReduceType] = {\n    'add': 'sum',\n    'amin': 'min',\n    'amax': 'max'\n}\nTORCH_REDUCE: Dict[ReduceType, ReduceType] = {\n    'add': 'sum',\n    'min': 'amin',\n    'max': 'amax'\n}\n\n\nclass SortOrder(Enum):\n    ROW = 'row'\n    COL = 'col'\n\n\nclass CatMetadata(NamedTuple):\n    nnz: List[int]\n    sparse_size: List[Tuple[Optional[int], Optional[int]]]\n    sort_order: List[Optional[SortOrder]]\n    is_undirected: List[bool]\n\n\ndef implements(torch_function: Callable) -> Callable:\n    r\"\"\"Registers a :pytorch:`PyTorch` function override.\"\"\"\n    @functools.wraps(torch_function)\n    def decorator(my_function: Callable) -> Callable:\n        HANDLED_FUNCTIONS[torch_function] = my_function\n        return my_function\n\n    return decorator\n\n\n\ndef set_tuple_item(\n    values: Tuple[Any, ...],\n    dim: int,\n    value: Any,\n) -> Tuple[Any, ...]:\n    if dim < -len(values) or dim >= len(values):\n        raise IndexError(\"tuple index out of range\")\n\n    dim = dim + len(values) if dim < 0 else dim\n    return values[:dim] + (value, ) + values[dim + 1:]\n\n\ndef maybe_add(\n    value: Sequence[Optional[int]],\n    other: Union[int, Sequence[Optional[int]]],\n    alpha: int = 1,\n) -> Tuple[Optional[int], ...]:\n\n    if isinstance(other, int):\n        return tuple(v + alpha * other if v is not None else None\n                     for v in value)\n\n    assert len(value) == len(other)\n    return tuple(v + alpha * o if v is not None and o is not None else None\n                 for v, o in zip(value, other))\n\n\ndef maybe_sub(\n    value: Sequence[Optional[int]],\n    other: Union[int, Sequence[Optional[int]]],\n    alpha: int = 1,\n) -> Tuple[Optional[int], ...]:\n\n    if isinstance(other, int):\n        return tuple(v - alpha * other if v is not None else None\n                     for v in value)\n\n    assert len(value) == len(other)\n    return tuple(v - alpha * o if v is not None and o is not None else None\n                 for v, o in zip(value, other))\n\n\ndef ptr2index(ptr: Tensor, output_size: Optional[int] = None) -> Tensor:\n    index = torch.arange(ptr.numel() - 1, dtype=ptr.dtype, device=ptr.device)\n    return index.repeat_interleave(ptr.diff(), output_size=output_size)\n\n\ndef assert_valid_dtype(tensor: Tensor) -> None:\n    if tensor.dtype not in SUPPORTED_DTYPES:\n        raise ValueError(f\"'EdgeIndex' holds an unsupported data type \"\n                         f\"(got '{tensor.dtype}', but expected one of \"\n                         f\"{SUPPORTED_DTYPES})\")\n\n\ndef assert_two_dimensional(tensor: Tensor) -> None:\n    if tensor.dim() != 2:\n        raise ValueError(f\"'EdgeIndex' needs to be two-dimensional \"\n                         f\"(got {tensor.dim()} dimensions)\")\n    if not torch.jit.is_tracing() and tensor.size(0) != 2:\n        raise ValueError(f\"'EdgeIndex' needs to have a shape of \"\n                         f\"[2, *] (got {list(tensor.size())})\")\n\n\ndef assert_contiguous(tensor: Tensor) -> None:\n    if not tensor.is_contiguous():\n        raise ValueError(\"'EdgeIndex' needs to be contiguous. Please call \"\n                         \"`edge_index.contiguous()` before proceeding.\")\n\n\ndef assert_symmetric(size: Tuple[Optional[int], Optional[int]]) -> None:\n    if (not torch.jit.is_tracing() and size[0] is not None\n            and size[1] is not None and size[0] != size[1]):\n        raise ValueError(f\"'EdgeIndex' is undirected but received a \"\n                         f\"non-symmetric size (got {list(size)})\")\n\n\ndef assert_sorted(func: Callable) -> Callable:\n    @functools.wraps(func)\n    def wrapper(*args: Any, **kwargs: Any) -> Any:\n        if not args[0].is_sorted:\n            cls_name = args[0].__class__.__name__\n            raise ValueError(\n                f\"Cannot call '{func.__name__}' since '{cls_name}' is not \"\n                f\"sorted. Please call `{cls_name}.sort_by(...)` first.\")\n        return func(*args, **kwargs)\n\n    return wrapper\n\n\nclass EdgeIndex(Tensor):\n    r\"\"\"A COO :obj:`edge_index` tensor with additional (meta)data attached.\n\n    :class:`EdgeIndex` is a :pytorch:`null` :class:`torch.Tensor`, that holds\n    an :obj:`edge_index` representation of shape :obj:`[2, num_edges]`.\n    Edges are given as pairwise source and destination node indices in sparse\n    COO format.\n\n    While :class:`EdgeIndex` sub-classes a general :pytorch:`null`\n    :class:`torch.Tensor`, it can hold additional (meta)data, *i.e.*:\n\n    * :obj:`sparse_size`: The underlying sparse matrix size\n    * :obj:`sort_order`: The sort order (if present), either by row or column.\n    * :obj:`is_undirected`: Whether edges are bidirectional.\n\n    Additionally, :class:`EdgeIndex` caches data for fast CSR or CSC conversion\n    in case its representation is sorted, such as its :obj:`rowptr` or\n    :obj:`colptr`, or the permutation vector for going from CSR to CSC or vice\n    versa.\n    Caches are filled based on demand (*e.g.*, when calling\n    :meth:`EdgeIndex.sort_by`), or when explicitly requested via\n    :meth:`EdgeIndex.fill_cache_`, and are maintained and adjusted over its\n    lifespan (*e.g.*, when calling :meth:`EdgeIndex.flip`).\n\n    This representation ensures for optimal computation in GNN message passing\n    schemes, while preserving the ease-of-use of regular COO-based :pyg:`PyG`\n    workflows.\n\n    .. code-block:: python\n\n        from torch_geometric import EdgeIndex\n\n        edge_index = EdgeIndex(\n            [[0, 1, 1, 2],\n             [1, 0, 2, 1]]\n            sparse_size=(3, 3),\n            sort_order='row',\n            is_undirected=True,\n            device='cpu',\n        )\n        >>> EdgeIndex([[0, 1, 1, 2],\n        ...            [1, 0, 2, 1]])\n        assert edge_index.is_sorted_by_row\n        assert edge_index.is_undirected\n\n        # Flipping order:\n        edge_index = edge_index.flip(0)\n        >>> EdgeIndex([[1, 0, 2, 1],\n        ...            [0, 1, 1, 2]])\n        assert edge_index.is_sorted_by_col\n        assert edge_index.is_undirected\n\n        # Filtering:\n        mask = torch.tensor([True, True, True, False])\n        edge_index = edge_index[:, mask]\n        >>> EdgeIndex([[1, 0, 2],\n        ...            [0, 1, 1]])\n        assert edge_index.is_sorted_by_col\n        assert not edge_index.is_undirected\n\n        # Sparse-Dense Matrix Multiplication:\n        out = edge_index.flip(0) @\u00a0torch.randn(3, 16)\n        assert out.size() == (3, 16)\n    \"\"\"\n    # See \"https://pytorch.org/docs/stable/notes/extending.html\"\n    # for a basic tutorial on how to subclass `torch.Tensor`.\n\n    # The underlying tensor representation:\n    _data: Optional[Tensor] = None\n\n    # The size of the underlying sparse matrix:\n    _sparse_size: Tuple[Optional[int], Optional[int]] = (None, None)\n\n    # Whether the `edge_index` represented is non-sorted (`None`), or sorted\n    # based on row or column values.\n    _sort_order: Optional[SortOrder] = None\n\n    # Whether the `edge_index` is undirected:\n    # NOTE `is_undirected` allows us to assume symmetric adjacency matrix size\n    # and to share compressed pointer representations, however, it does not\n    # allow us get rid of CSR/CSC permutation vectors since ordering within\n    # neighborhoods is not necessarily deterministic.\n    _is_undirected: bool = False\n\n    # A cache for its compressed representation:\n    _indptr: Optional[Tensor] = None\n\n    # A cache for its transposed representation:\n    _T_perm: Optional[Tensor] = None\n    _T_index: Tuple[Optional[Tensor], Optional[Tensor]] = (None, None)\n    _T_indptr: Optional[Tensor] = None\n\n    # A cached \"1\"-value vector for `torch.sparse` matrix multiplication:\n    _value: Optional[Tensor] = None\n\n    # Whenever we perform a concatenation of edge indices, we cache the\n    # original metadata to be able to reconstruct individual edge indices:\n    _cat_metadata: Optional[CatMetadata] = None\n\n    def __new__(\n        cls: Type,\n        data: Any,\n        *args: Any,\n        sparse_size: Optional[Tuple[Optional[int], Optional[int]]] = None,\n        sort_order: Optional[Union[str, SortOrder]] = None,\n        is_undirected: bool = False,\n        **kwargs: Any,\n    ) -> 'EdgeIndex':\n        if not isinstance(data, Tensor):\n            data = torch.tensor(data, *args, **kwargs)\n        elif len(args) > 0:\n            raise TypeError(\n                f\"new() received an invalid combination of arguments - got \"\n                f\"(Tensor, {', '.join(str(type(arg)) for arg in args)})\")\n        elif len(kwargs) > 0:\n            raise TypeError(f\"new() received invalid keyword arguments - got \"\n                            f\"{set(kwargs.keys())})\")\n\n        assert isinstance(data, Tensor)\n\n        indptr: Optional[Tensor] = None\n\n        if isinstance(data, cls):  # If passed `EdgeIndex`, inherit metadata:\n            indptr = data._indptr\n            sparse_size = sparse_size or data.sparse_size()\n            sort_order = sort_order or data.sort_order\n            is_undirected = is_undirected or data.is_undirected\n\n        # Convert `torch.sparse` tensors to `EdgeIndex` representation:\n        if data.layout == torch.sparse_coo:\n            sort_order = SortOrder.ROW\n            sparse_size = sparse_size or (data.size(0), data.size(1))\n            data = data.indices()\n\n        if data.layout == torch.sparse_csr:\n            indptr = data.crow_indices()\n            col = data.col_indices()\n\n            assert isinstance(indptr, Tensor)\n            row = ptr2index(indptr, output_size=col.numel())\n\n            sort_order = SortOrder.ROW\n            sparse_size = sparse_size or (data.size(0), data.size(1))\n            if sparse_size[0] is not None and sparse_size[0] != data.size(0):\n                indptr = None\n            data = torch.stack([row, col], dim=0)\n\n        if (torch_geometric.typing.WITH_PT112\n                and data.layout == torch.sparse_csc):\n            row = data.row_indices()\n            indptr = data.ccol_indices()\n\n            assert isinstance(indptr, Tensor)\n            col = ptr2index(indptr, output_size=row.numel())\n\n            sort_order = SortOrder.COL\n            sparse_size = sparse_size or (data.size(0), data.size(1))\n            if sparse_size[1] is not None and sparse_size[1] != data.size(1):\n                indptr = None\n            data = torch.stack([row, col], dim=0)\n\n        assert_valid_dtype(data)\n        assert_two_dimensional(data)\n        assert_contiguous(data)\n\n        if sparse_size is None:\n            sparse_size = (None, None)\n\n        if is_undirected:\n            assert_symmetric(sparse_size)\n            if sparse_size[0] is not None and sparse_size[1] is None:\n                sparse_size = (sparse_size[0], sparse_size[0])\n            elif sparse_size[0] is None and sparse_size[1] is not None:\n                sparse_size = (sparse_size[1], sparse_size[1])\n\n        if torch_geometric.typing.WITH_PT112:\n            out = super().__new__(cls, data)\n        else:\n            out = Tensor._make_subclass(cls, data)\n\n        # Attach metadata:\n        assert isinstance(out, EdgeIndex)\n        if torch_geometric.typing.WITH_PT22:\n            out._data = data\n        out._sparse_size = sparse_size\n        out._sort_order = None if sort_order is None else SortOrder(sort_order)\n        out._is_undirected = is_undirected\n        out._indptr = indptr\n\n        if isinstance(data, cls):  # If passed `EdgeIndex`, inherit metadata:\n            out._T_perm = data._T_perm\n            out._T_index = data._T_index\n            out._T_indptr = data._T_indptr\n            out._value = out._value\n\n            # Reset metadata if cache is invalidated:\n            num_rows = sparse_size[0]\n            if num_rows is not None and num_rows != data.sparse_size(0):\n                out._indptr = None\n\n            num_cols = sparse_size[1]\n            if num_cols is not None and num_cols != data.sparse_size(1):\n                out._T_indptr = None\n\n        return out\n\n    # Validation ##############################################################\n\n    def validate(self) -> 'EdgeIndex':\n        r\"\"\"Validates the :class:`EdgeIndex` representation.\n\n        In particular, it ensures that\n\n        * it only holds valid indices.\n        * the sort order is correctly set.\n        * indices are bidirectional in case it is specified as undirected.\n        \"\"\"\n        assert_valid_dtype(self)\n        assert_two_dimensional(self)\n        assert_contiguous(self)\n        if self.is_undirected:\n            assert_symmetric(self.sparse_size())\n\n        if self.numel() > 0 and self.min() < 0:\n            raise ValueError(f\"'{self.__class__.__name__}' contains negative \"\n                             f\"indices (got {int(self.min())})\")\n\n        if (self.numel() > 0 and self.num_rows is not None\n                and self[0].max() >= self.num_rows):\n            raise ValueError(f\"'{self.__class__.__name__}' contains larger \"\n                             f\"indices than its number of rows \"\n                             f\"(got {int(self[0].max())}, but expected values \"\n                             f\"smaller than {self.num_rows})\")\n\n        if (self.numel() > 0 and self.num_cols is not None\n                and self[1].max() >= self.num_cols):\n            raise ValueError(f\"'{self.__class__.__name__}' contains larger \"\n                             f\"indices than its number of columns \"\n                             f\"(got {int(self[1].max())}, but expected values \"\n                             f\"smaller than {self.num_cols})\")\n\n        if self.is_sorted_by_row and (self[0].diff() < 0).any():\n            raise ValueError(f\"'{self.__class__.__name__}' is not sorted by \"\n                             f\"row indices\")\n\n        if self.is_sorted_by_col and (self[1].diff() < 0).any():\n            raise ValueError(f\"'{self.__class__.__name__}' is not sorted by \"\n                             f\"column indices\")\n\n        if self.is_undirected:\n            flat_index1 = (self[0] * self.get_num_rows() + self[1]).sort()[0]\n...\n", "needle_token_start": 2771, "needle_token_end": 2878, "code_context_ntokens": 6825, "output": ["```python\ndef set_tuple_item(\n    values: Tuple[Any, ...],\n    dim: int,\n    value: Any,\n) -> Tuple[Any, ...]:\n    if dim < -len(values) or dim >= len(values):\n        raise IndexError(\"tuple index out of range\")\n\n    dim = dim + len(values) if dim < 0 else dim\n    return values[:dim] + (value, ) + values[dim + 1:]\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "_to_edge_mask", "language": "python", "path": "torch_geometric/explain/algorithm/captum.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function generates a tensor representing a mask for edges in a graph, where each edge is initially considered equally important.\n2. **Input**: A tensor representing the indices of edges in a graph.\n3. **Output**: A tensor of ones with the same length as the number of edges, with gradient tracking enabled.\n4. **Procedure**: The function calculates the number of edges based on the input tensor's second dimension, then creates and returns a tensor filled with ones, sized according to the number of edges, and configured to track gradients and match the device of the input tensor.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/nn/conv/transformer_conv.py\nimport math\nimport typing\nfrom typing import Optional, Tuple, Union\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import Tensor\n\nfrom torch_geometric.nn.conv import MessagePassing\nfrom torch_geometric.nn.dense.linear import Linear\nfrom torch_geometric.typing import (\n    Adj,\n    NoneType,\n    OptTensor,\n    PairTensor,\n    SparseTensor,\n)\nfrom torch_geometric.utils import softmax\n\nif typing.TYPE_CHECKING:\n    from typing import overload\nelse:\n    from torch.jit import _overload_method as overload\n\n\nclass TransformerConv(MessagePassing):\n    r\"\"\"The graph transformer operator from the `\"Masked Label Prediction:\n    Unified Message Passing Model for Semi-Supervised Classification\"\n    <https://arxiv.org/abs/2009.03509>`_ paper.\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\mathbf{W}_1 \\mathbf{x}_i +\n        \\sum_{j \\in \\mathcal{N}(i)} \\alpha_{i,j} \\mathbf{W}_2 \\mathbf{x}_{j},\n\n    where the attention coefficients :math:`\\alpha_{i,j}` are computed via\n    multi-head dot product attention:\n\n    .. math::\n        \\alpha_{i,j} = \\textrm{softmax} \\left(\n        \\frac{(\\mathbf{W}_3\\mathbf{x}_i)^{\\top} (\\mathbf{W}_4\\mathbf{x}_j)}\n        {\\sqrt{d}} \\right)\n\n    Args:\n        in_channels (int or tuple): Size of each input sample, or :obj:`-1` to\n            derive the size from the first input(s) to the forward method.\n            A tuple corresponds to the sizes of source and target\n            dimensionalities.\n        out_channels (int): Size of each output sample.\n        heads (int, optional): Number of multi-head-attentions.\n            (default: :obj:`1`)\n        concat (bool, optional): If set to :obj:`False`, the multi-head\n            attentions are averaged instead of concatenated.\n            (default: :obj:`True`)\n        beta (bool, optional): If set, will combine aggregation and\n            skip information via\n\n            .. math::\n                \\mathbf{x}^{\\prime}_i = \\beta_i \\mathbf{W}_1 \\mathbf{x}_i +\n                (1 - \\beta_i) \\underbrace{\\left(\\sum_{j \\in \\mathcal{N}(i)}\n                \\alpha_{i,j} \\mathbf{W}_2 \\vec{x}_j \\right)}_{=\\mathbf{m}_i}\n\n            with :math:`\\beta_i = \\textrm{sigmoid}(\\mathbf{w}_5^{\\top}\n            [ \\mathbf{W}_1 \\mathbf{x}_i, \\mathbf{m}_i, \\mathbf{W}_1\n            \\mathbf{x}_i - \\mathbf{m}_i ])` (default: :obj:`False`)\n...\n# Path: torch_geometric/nn/conv/utils/cheatsheet.py\nimport importlib\nimport inspect\nimport re\nfrom typing import Optional\n\n\ndef paper_title(cls: str) -> Optional[str]:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    match = re.search('`\\\".+?\\\"', inspect.getdoc(cls), flags=re.DOTALL)\n    return None if match is None else match.group().replace('\\n', ' ')[2:-1]\n\n\ndef paper_link(cls: str) -> Optional[str]:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    match = re.search('<.+?>', inspect.getdoc(cls), flags=re.DOTALL)\n    return None if match is None else match.group().replace('\\n', ' ')[1:-1]\n\n\ndef supports_sparse_tensor(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'SparseTensor' in str(signature)\n\n\ndef supports_edge_weights(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'edge_weight' in str(signature)\n\n\ndef supports_edge_features(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'edge_attr' in str(signature)\n\n\ndef supports_bipartite_graphs(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'Union[torch.Tensor, Tuple[torch.Tensor' in str(signature)\n\n\ndef supports_static_graphs(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    return 'node_dim=' not in inspect.getsource(cls.__init__)\n\n\ndef supports_lazy_initialization(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    doc = re.sub(' +', ' ', inspect.getdoc(cls).replace('\\n', ' '))\n    match = re.search('or :obj:`-1` to derive the size from the first', doc)\n    return match is not None\n\n\ndef processes_heterogeneous_graphs(cls: str) -> bool:\n    if 'hetero' in cls.lower():\n        return True\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'edge_index_dict' in str(signature) or 'edge_type' in str(signature)\n\n\ndef processes_hypergraphs(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return 'hyperedge_index' in str(signature)\n\n\ndef processes_point_clouds(cls: str) -> bool:\n    cls = importlib.import_module('torch_geometric.nn.conv').__dict__[cls]\n    signature = inspect.signature(cls.forward)\n    return (('edge_index' not in str(signature)\n             and 'csc' not in str(signature)) or 'pos' in str(signature))\n\n# Path: torch_geometric/nn/conv/utils/__init__.py\nr\"\"\"GNN utility package.\"\"\"\n\nfrom .cheatsheet import paper_title, paper_link\nfrom .cheatsheet import supports_sparse_tensor\nfrom .cheatsheet import supports_edge_weights\nfrom .cheatsheet import supports_edge_features\nfrom .cheatsheet import supports_bipartite_graphs\nfrom .cheatsheet import supports_static_graphs\nfrom .cheatsheet import supports_lazy_initialization\nfrom .cheatsheet import processes_heterogeneous_graphs\nfrom .cheatsheet import processes_hypergraphs\nfrom .cheatsheet import processes_point_clouds\n\n__all__ = [\n    'paper_title',\n    'paper_link',\n    'supports_sparse_tensor',\n    'supports_edge_weights',\n    'supports_edge_features',\n    'supports_bipartite_graphs',\n    'supports_static_graphs',\n    'supports_lazy_initialization',\n    'processes_heterogeneous_graphs',\n    'processes_hypergraphs',\n    'processes_point_clouds',\n]\n\n# Path: torch_geometric/nn/conv/wl_conv.py\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import Adj\nfrom torch_geometric.utils import (\n    degree,\n    is_sparse,\n    scatter,\n    sort_edge_index,\n    to_edge_index,\n)\n\n\nclass WLConv(torch.nn.Module):\n    r\"\"\"The Weisfeiler Lehman (WL) operator from the `\"A Reduction of a Graph\n    to a Canonical Form and an Algebra Arising During this Reduction\"\n    <https://www.iti.zcu.cz/wl2018/pdf/wl_paper_translation.pdf>`_ paper.\n\n    :class:`WLConv` iteratively refines node colorings according to:\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\textrm{hash} \\left( \\mathbf{x}_i, \\{\n        \\mathbf{x}_j \\colon j \\in \\mathcal{N}(i) \\} \\right)\n\n    Shapes:\n        - **input:**\n          node coloring :math:`(|\\mathcal{V}|, F_{in})` *(one-hot encodings)*\n          or :math:`(|\\mathcal{V}|)` *(integer-based)*,\n          edge indices :math:`(2, |\\mathcal{E}|)`\n        - **output:** node coloring :math:`(|\\mathcal{V}|)` *(integer-based)*\n    \"\"\"\n    def __init__(self):\n        super().__init__()\n        self.hashmap = {}\n\n    def reset_parameters(self):\n        r\"\"\"Resets all learnable parameters of the module.\"\"\"\n        self.hashmap = {}\n\n    @torch.no_grad()\n    def forward(self, x: Tensor, edge_index: Adj) -> Tensor:\n        r\"\"\"Runs the forward pass of the module.\"\"\"\n        if x.dim() > 1:\n            assert (x.sum(dim=-1) == 1).sum() == x.size(0)\n            x = x.argmax(dim=-1)  # one-hot -> integer.\n        assert x.dtype == torch.long\n\n        if is_sparse(edge_index):\n            col_and_row, _ = to_edge_index(edge_index)\n            col = col_and_row[0]\n            row = col_and_row[1]\n        else:\n            edge_index = sort_edge_index(edge_index, num_nodes=x.size(0),\n                                         sort_by_row=False)\n            row, col = edge_index[0], edge_index[1]\n\n        # `col` is sorted, so we can use it to `split` neighbors to groups:\n        deg = degree(col, x.size(0), dtype=torch.long).tolist()\n\n        out = []\n        for node, neighbors in zip(x.tolist(), x[row].split(deg)):\n            idx = hash(tuple([node] + neighbors.sort()[0].tolist()))\n            if idx not in self.hashmap:\n                self.hashmap[idx] = len(self.hashmap)\n            out.append(self.hashmap[idx])\n\n        return torch.tensor(out, device=x.device)\n\n    def histogram(self, x: Tensor, batch: Optional[Tensor] = None,\n                  norm: bool = False) -> Tensor:\n        r\"\"\"Given a node coloring :obj:`x`, computes the color histograms of\n        the respective graphs (separated by :obj:`batch`).\n        \"\"\"\n        if batch is None:\n            batch = torch.zeros(x.size(0), dtype=torch.long, device=x.device)\n\n        num_colors = len(self.hashmap)\n        batch_size = int(batch.max()) + 1\n\n        index = batch * num_colors + x\n        out = scatter(torch.ones_like(index), index, dim=0,\n                      dim_size=num_colors * batch_size, reduce='sum')\n        out = out.view(batch_size, num_colors)\n\n        if norm:\n            out = out.to(torch.float)\n            out /= out.norm(dim=-1, keepdim=True)\n\n        return out\n\n# Path: torch_geometric/nn/conv/wl_conv_continuous.py\nfrom typing import Union\n\nfrom torch import Tensor\n\nfrom torch_geometric.nn.conv import MessagePassing\nfrom torch_geometric.typing import (\n    Adj,\n    OptPairTensor,\n    OptTensor,\n    Size,\n    SparseTensor,\n)\nfrom torch_geometric.utils import scatter, spmm\n\n\nclass WLConvContinuous(MessagePassing):\n    r\"\"\"The Weisfeiler Lehman operator from the `\"Wasserstein\n    Weisfeiler-Lehman Graph Kernels\" <https://arxiv.org/abs/1906.01277>`_\n    paper.\n\n    Refinement is done though a degree-scaled mean aggregation and works on\n    nodes with continuous attributes:\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\frac{1}{2}\\big(\\mathbf{x}_i +\n        \\frac{1}{\\textrm{deg}(i)}\n        \\sum_{j \\in \\mathcal{N}(i)} e_{j,i} \\cdot \\mathbf{x}_j \\big)\n\n    where :math:`e_{j,i}` denotes the edge weight from source node :obj:`j` to\n    target node :obj:`i` (default: :obj:`1`)\n\n    Args:\n        **kwargs (optional): Additional arguments of\n            :class:`torch_geometric.nn.conv.MessagePassing`.\n\n    Shapes:\n        - **input:**\n          node features :math:`(|\\mathcal{V}|, F)` or\n          :math:`((|\\mathcal{V_s}|, F), (|\\mathcal{V_t}|, F))` if bipartite,\n          edge indices :math:`(2, |\\mathcal{E}|)`,\n          edge weights :math:`(|\\mathcal{E}|)` *(optional)*\n        - **output:** node features :math:`(|\\mathcal{V}|, F)` or\n          :math:`(|\\mathcal{V}_t|, F)` if bipartite\n    \"\"\"\n    def __init__(self, **kwargs):\n        super().__init__(aggr='add', **kwargs)\n\n    def forward(\n        self,\n        x: Union[Tensor, OptPairTensor],\n        edge_index: Adj,\n        edge_weight: OptTensor = None,\n        size: Size = None,\n    ) -> Tensor:\n\n        if isinstance(x, Tensor):\n            x = (x, x)\n\n        # propagate_type: (x: OptPairTensor, edge_weight: OptTensor)\n        out = self.propagate(edge_index, x=x, edge_weight=edge_weight,\n                             size=size)\n\n        if isinstance(edge_index, SparseTensor):\n            assert edge_weight is None\n            dst_index, _, edge_weight = edge_index.coo()\n        else:\n            dst_index = edge_index[1]\n\n        if edge_weight is None:\n            edge_weight = x[0].new_ones(dst_index.numel())\n\n        deg = scatter(edge_weight, dst_index, 0, out.size(0), reduce='sum')\n        deg_inv = 1. / deg\n        deg_inv.masked_fill_(deg_inv == float('inf'), 0)\n        out = deg_inv.view(-1, 1) * out\n\n        x_dst = x[1]\n        if x_dst is not None:\n            out = 0.5 * (x_dst + out)\n\n        return out\n\n    def message(self, x_j: Tensor, edge_weight: OptTensor) -> Tensor:\n        return x_j if edge_weight is None else edge_weight.view(-1, 1) * x_j\n\n    def message_and_aggregate(self, adj_t: Adj, x: OptPairTensor) -> Tensor:\n        return spmm(adj_t, x[0], reduce=self.aggr)\n\n# Path: torch_geometric/nn/conv/x_conv.py\nfrom math import ceil\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\nfrom torch.nn import ELU\nfrom torch.nn import BatchNorm1d as BN\nfrom torch.nn import Conv1d\nfrom torch.nn import Linear as L\nfrom torch.nn import Sequential as S\n\nfrom torch_geometric.nn import Reshape\nfrom torch_geometric.nn.inits import reset\n\ntry:\n    from torch_cluster import knn_graph\nexcept ImportError:\n    knn_graph = None\n\n\nclass XConv(torch.nn.Module):\n    r\"\"\"The convolutional operator on :math:`\\mathcal{X}`-transformed points\n    from the `\"PointCNN: Convolution On X-Transformed Points\"\n    <https://arxiv.org/abs/1801.07791>`_ paper.\n\n    .. math::\n        \\mathbf{x}^{\\prime}_i = \\mathrm{Conv}\\left(\\mathbf{K},\n        \\gamma_{\\mathbf{\\Theta}}(\\mathbf{P}_i - \\mathbf{p}_i) \\times\n        \\left( h_\\mathbf{\\Theta}(\\mathbf{P}_i - \\mathbf{p}_i) \\, \\Vert \\,\n        \\mathbf{x}_i \\right) \\right),\n\n    where :math:`\\mathbf{K}` and :math:`\\mathbf{P}_i` denote the trainable\n    filter and neighboring point positions of :math:`\\mathbf{x}_i`,\n    respectively.\n    :math:`\\gamma_{\\mathbf{\\Theta}}` and :math:`h_{\\mathbf{\\Theta}}` describe\n    neural networks, *i.e.* MLPs, where :math:`h_{\\mathbf{\\Theta}}`\n    individually lifts each point into a higher-dimensional space, and\n    :math:`\\gamma_{\\mathbf{\\Theta}}` computes the :math:`\\mathcal{X}`-\n    transformation matrix based on *all* points in a neighborhood.\n\n    Args:\n        in_channels (int): Size of each input sample.\n        out_channels (int): Size of each output sample.\n        dim (int): Point cloud dimensionality.\n        kernel_size (int): Size of the convolving kernel, *i.e.* number of\n            neighbors including self-loops.\n        hidden_channels (int, optional): Output size of\n            :math:`h_{\\mathbf{\\Theta}}`, *i.e.* dimensionality of lifted\n            points. If set to :obj:`None`, will be automatically set to\n            :obj:`in_channels / 4`. (default: :obj:`None`)\n        dilation (int, optional): The factor by which the neighborhood is\n            extended, from which :obj:`kernel_size` neighbors are then\n            uniformly sampled. Can be interpreted as the dilation rate of\n            classical convolutional operators. (default: :obj:`1`)\n        bias (bool, optional): If set to :obj:`False`, the layer will not learn\n            an additive bias. (default: :obj:`True`)\n        num_workers (int): Number of workers to use for k-NN computation.\n            Has no effect in case :obj:`batch` is not :obj:`None`, or the input\n            lies on the GPU. (default: :obj:`1`)\n\n    Shapes:\n        - **input:**\n          node features :math:`(|\\mathcal{V}|, F_{in})`,\n          positions :math:`(|\\mathcal{V}|, D)`,\n          batch vector :math:`(|\\mathcal{V}|)` *(optional)*\n        - **output:**\n          node features :math:`(|\\mathcal{V}|, F_{out})`\n    \"\"\"\n    def __init__(self, in_channels: int, out_channels: int, dim: int,\n                 kernel_size: int, hidden_channels: Optional[int] = None,\n                 dilation: int = 1, bias: bool = True, num_workers: int = 1):\n        super().__init__()\n\n        if knn_graph is None:\n            raise ImportError('`XConv` requires `torch-cluster`.')\n\n        self.in_channels = in_channels\n        if hidden_channels is None:\n            hidden_channels = in_channels // 4\n        assert hidden_channels > 0\n        self.hidden_channels = hidden_channels\n        self.out_channels = out_channels\n        self.dim = dim\n        self.kernel_size = kernel_size\n        self.dilation = dilation\n        self.num_workers = num_workers\n\n        C_in, C_delta, C_out = in_channels, hidden_channels, out_channels\n        D, K = dim, kernel_size\n\n        self.mlp1 = S(\n            L(dim, C_delta),\n            ELU(),\n            BN(C_delta),\n            L(C_delta, C_delta),\n            ELU(),\n            BN(C_delta),\n            Reshape(-1, K, C_delta),\n        )\n\n        self.mlp2 = S(\n            L(D * K, K**2),\n            ELU(),\n            BN(K**2),\n            Reshape(-1, K, K),\n            Conv1d(K, K**2, K, groups=K),\n            ELU(),\n            BN(K**2),\n            Reshape(-1, K, K),\n            Conv1d(K, K**2, K, groups=K),\n            BN(K**2),\n            Reshape(-1, K, K),\n        )\n\n        C_in = C_in + C_delta\n        depth_multiplier = int(ceil(C_out / C_in))\n        self.conv = S(\n            Conv1d(C_in, C_in * depth_multiplier, K, groups=C_in),\n            Reshape(-1, C_in * depth_multiplier),\n            L(C_in * depth_multiplier, C_out, bias=bias),\n        )\n\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        r\"\"\"Resets all learnable parameters of the module.\"\"\"\n        reset(self.mlp1)\n        reset(self.mlp2)\n        reset(self.conv)\n\n    def forward(self, x: Tensor, pos: Tensor, batch: Optional[Tensor] = None):\n        r\"\"\"Runs the forward pass of the module.\"\"\"\n        pos = pos.unsqueeze(-1) if pos.dim() == 1 else pos\n        (N, D), K = pos.size(), self.kernel_size\n\n        edge_index = knn_graph(pos, K * self.dilation, batch, loop=True,\n                               flow='target_to_source',\n                               num_workers=self.num_workers)\n\n        if self.dilation > 1:\n            edge_index = edge_index[:, ::self.dilation]\n\n        row, col = edge_index[0], edge_index[1]\n\n        pos = pos[col] - pos[row]\n\n        x_star = self.mlp1(pos)\n        if x is not None:\n            x = x.unsqueeze(-1) if x.dim() == 1 else x\n            x = x[col].view(N, K, self.in_channels)\n            x_star = torch.cat([x_star, x], dim=-1)\n        x_star = x_star.transpose(1, 2).contiguous()\n\n        transform_matrix = self.mlp2(pos.view(N, K * D))\n\n        x_transformed = torch.matmul(x_star, transform_matrix)\n\n        out = self.conv(x_transformed)\n\n        return out\n\n    def __repr__(self) -> str:\n        return (f'{self.__class__.__name__}({self.in_channels}, '\n                f'{self.out_channels})')\n\n# Path: torch_geometric/nn/conv/__init__.py\nfrom .message_passing import MessagePassing\nfrom .simple_conv import SimpleConv\nfrom .gcn_conv import GCNConv\nfrom .cheb_conv import ChebConv\nfrom .sage_conv import SAGEConv\nfrom .cugraph.sage_conv import CuGraphSAGEConv\nfrom .graph_conv import GraphConv\nfrom .gravnet_conv import GravNetConv\nfrom .gated_graph_conv import GatedGraphConv\nfrom .res_gated_graph_conv import ResGatedGraphConv\nfrom .gat_conv import GATConv\nfrom .cugraph.gat_conv import CuGraphGATConv\nfrom .fused_gat_conv import FusedGATConv\nfrom .gatv2_conv import GATv2Conv\nfrom .transformer_conv import TransformerConv\nfrom .agnn_conv import AGNNConv\nfrom .tag_conv import TAGConv\nfrom .gin_conv import GINConv, GINEConv\nfrom .arma_conv import ARMAConv\nfrom .sg_conv import SGConv\nfrom .appnp import APPNP\nfrom .mf_conv import MFConv\nfrom .rgcn_conv import RGCNConv, FastRGCNConv\nfrom .cugraph.rgcn_conv import CuGraphRGCNConv\nfrom .rgat_conv import RGATConv\nfrom .signed_conv import SignedConv\nfrom .dna_conv import DNAConv\nfrom .point_conv import PointNetConv\nfrom .gmm_conv import GMMConv\nfrom .spline_conv import SplineConv\nfrom .nn_conv import NNConv\nfrom .cg_conv import CGConv\nfrom .edge_conv import EdgeConv, DynamicEdgeConv\nfrom .x_conv import XConv\nfrom .ppf_conv import PPFConv\nfrom .feast_conv import FeaStConv\nfrom .point_transformer_conv import PointTransformerConv\nfrom .hypergraph_conv import HypergraphConv\nfrom .le_conv import LEConv\nfrom .pna_conv import PNAConv\nfrom .cluster_gcn_conv import ClusterGCNConv\nfrom .gen_conv import GENConv\nfrom .gcn2_conv import GCN2Conv\nfrom .pan_conv import PANConv\nfrom .wl_conv import WLConv\nfrom .wl_conv_continuous import WLConvContinuous\nfrom .film_conv import FiLMConv\nfrom .supergat_conv import SuperGATConv\nfrom .fa_conv import FAConv\nfrom .eg_conv import EGConv\nfrom .pdn_conv import PDNConv\nfrom .general_conv import GeneralConv\nfrom .hgt_conv import HGTConv\nfrom .heat_conv import HEATConv\nfrom .hetero_conv import HeteroConv\nfrom .han_conv import HANConv\nfrom .lg_conv import LGConv\nfrom .ssg_conv import SSGConv\nfrom .point_gnn_conv import PointGNNConv\nfrom .gps_conv import GPSConv\nfrom .antisymmetric_conv import AntiSymmetricConv\nfrom .dir_gnn_conv import DirGNNConv\nfrom .mixhop_conv import MixHopConv\n\nimport torch_geometric.nn.conv.utils  # noqa\n\n__all__ = [\n    'MessagePassing',\n    'SimpleConv',\n    'GCNConv',\n    'ChebConv',\n    'SAGEConv',\n    'CuGraphSAGEConv',\n    'GraphConv',\n    'GravNetConv',\n    'GatedGraphConv',\n    'ResGatedGraphConv',\n    'GATConv',\n    'CuGraphGATConv',\n    'FusedGATConv',\n    'GATv2Conv',\n    'TransformerConv',\n    'AGNNConv',\n    'TAGConv',\n    'GINConv',\n    'GINEConv',\n    'ARMAConv',\n    'SGConv',\n    'SSGConv',\n    'APPNP',\n    'MFConv',\n    'RGCNConv',\n    'FastRGCNConv',\n    'CuGraphRGCNConv',\n    'RGATConv',\n    'SignedConv',\n    'DNAConv',\n    'PointNetConv',\n    'GMMConv',\n    'SplineConv',\n    'NNConv',\n    'CGConv',\n    'EdgeConv',\n    'DynamicEdgeConv',\n    'XConv',\n    'PPFConv',\n    'FeaStConv',\n    'PointTransformerConv',\n    'HypergraphConv',\n    'LEConv',\n    'PNAConv',\n    'ClusterGCNConv',\n    'GENConv',\n    'GCN2Conv',\n    'PANConv',\n    'WLConv',\n    'WLConvContinuous',\n    'FiLMConv',\n    'SuperGATConv',\n    'FAConv',\n    'EGConv',\n    'PDNConv',\n    'GeneralConv',\n    'HGTConv',\n    'HEATConv',\n    'HeteroConv',\n    'HANConv',\n    'LGConv',\n    'PointGNNConv',\n    'GPSConv',\n    'AntiSymmetricConv',\n    'DirGNNConv',\n    'MixHopConv',\n]\n\nclasses = __all__\n\nECConv = NNConv\nPointConv = PointNetConv\n\n# Path: torch_geometric/explain/algorithm/attention_explainer.py\nimport logging\nfrom typing import List, Optional, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.explain import Explanation\nfrom torch_geometric.explain.algorithm import ExplainerAlgorithm\nfrom torch_geometric.explain.config import ExplanationType, ModelTaskLevel\nfrom torch_geometric.nn.conv.message_passing import MessagePassing\n\n\nclass AttentionExplainer(ExplainerAlgorithm):\n    r\"\"\"An explainer that uses the attention coefficients produced by an\n    attention-based GNN (*e.g.*,\n    :class:`~torch_geometric.nn.conv.GATConv`,\n    :class:`~torch_geometric.nn.conv.GATv2Conv`, or\n    :class:`~torch_geometric.nn.conv.TransformerConv`) as edge explanation.\n    Attention scores across layers and heads will be aggregated according to\n    the :obj:`reduce` argument.\n\n    Args:\n        reduce (str, optional): The method to reduce the attention scores\n            across layers and heads. (default: :obj:`\"max\"`)\n    \"\"\"\n    def __init__(self, reduce: str = 'max'):\n        super().__init__()\n        self.reduce = reduce\n\n    def forward(\n        self,\n        model: torch.nn.Module,\n        x: Tensor,\n        edge_index: Tensor,\n        *,\n        target: Tensor,\n        index: Optional[Union[int, Tensor]] = None,\n        **kwargs,\n    ) -> Explanation:\n        if isinstance(x, dict) or isinstance(edge_index, dict):\n            raise ValueError(f\"Heterogeneous graphs not yet supported in \"\n                             f\"'{self.__class__.__name__}'\")\n\n        hard_edge_mask = None\n        if self.model_config.task_level == ModelTaskLevel.node:\n            # We need to compute the hard edge mask to properly clean up edge\n            # attributions not involved during message passing:\n            _, hard_edge_mask = self._get_hard_masks(model, index, edge_index,\n                                                     num_nodes=x.size(0))\n\n        alphas: List[Tensor] = []\n\n        def hook(module, msg_kwargs, out):\n            if 'alpha' in msg_kwargs[0]:\n                alphas.append(msg_kwargs[0]['alpha'].detach())\n            elif getattr(module, '_alpha', None) is not None:\n                alphas.append(module._alpha.detach())\n\n        hook_handles = []\n        for module in model.modules():  # Register message forward hooks:\n            if (isinstance(module, MessagePassing)\n                    and module.explain is not False):\n                hook_handles.append(module.register_message_forward_hook(hook))\n\n        model(x, edge_index, **kwargs)\n\n        for handle in hook_handles:  # Remove hooks:\n            handle.remove()\n\n        if len(alphas) == 0:\n            raise ValueError(\"Could not collect any attention coefficients. \"\n                             \"Please ensure that your model is using \"\n                             \"attention-based GNN layers.\")\n\n        for i, alpha in enumerate(alphas):\n            alpha = alpha[:edge_index.size(1)]  # Respect potential self-loops.\n            if alpha.dim() == 2:\n                alpha = getattr(torch, self.reduce)(alpha, dim=-1)\n                if isinstance(alpha, tuple):  # Respect `torch.max`:\n                    alpha = alpha[0]\n            elif alpha.dim() > 2:\n                raise ValueError(f\"Can not reduce attention coefficients of \"\n                                 f\"shape {list(alpha.size())}\")\n            alphas[i] = alpha\n\n        if len(alphas) > 1:\n            alpha = torch.stack(alphas, dim=-1)\n            alpha = getattr(torch, self.reduce)(alpha, dim=-1)\n            if isinstance(alpha, tuple):  # Respect `torch.max`:\n                alpha = alpha[0]\n        else:\n            alpha = alphas[0]\n\n        alpha = self._post_process_mask(alpha, hard_edge_mask,\n                                        apply_sigmoid=False)\n\n        return Explanation(edge_mask=alpha)\n\n    def supports(self) -> bool:\n        explanation_type = self.explainer_config.explanation_type\n        if explanation_type != ExplanationType.model:\n            logging.error(f\"'{self.__class__.__name__}' only supports \"\n                          f\"model explanations \"\n                          f\"got (`explanation_type={explanation_type.value}`)\")\n            return False\n\n        node_mask_type = self.explainer_config.node_mask_type\n        if node_mask_type is not None:\n            logging.error(f\"'{self.__class__.__name__}' does not support \"\n                          f\"explaining input node features \"\n                          f\"got (`node_mask_type={node_mask_type.value}`)\")\n            return False\n\n        return True\n\n# Path: torch_geometric/explain/algorithm/base.py\nfrom abc import abstractmethod\nfrom typing import Dict, Optional, Tuple, Union\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import Tensor\n\nfrom torch_geometric.explain import Explanation, HeteroExplanation\nfrom torch_geometric.explain.config import (\n    ExplainerConfig,\n    ModelConfig,\n    ModelReturnType,\n)\nfrom torch_geometric.nn import MessagePassing\nfrom torch_geometric.typing import EdgeType, NodeType\nfrom torch_geometric.utils import k_hop_subgraph\n\n\nclass ExplainerAlgorithm(torch.nn.Module):\n    r\"\"\"An abstract base class for implementing explainer algorithms.\"\"\"\n    @abstractmethod\n    def forward(\n        self,\n        model: torch.nn.Module,\n        x: Union[Tensor, Dict[NodeType, Tensor]],\n        edge_index: Union[Tensor, Dict[EdgeType, Tensor]],\n        *,\n        target: Tensor,\n        index: Optional[Union[int, Tensor]] = None,\n        **kwargs,\n    ) -> Union[Explanation, HeteroExplanation]:\n        r\"\"\"Computes the explanation.\n\n        Args:\n            model (torch.nn.Module): The model to explain.\n            x (Union[torch.Tensor, Dict[NodeType, torch.Tensor]]): The input\n                node features of a homogeneous or heterogeneous graph.\n            edge_index (Union[torch.Tensor, Dict[NodeType, torch.Tensor]]): The\n                input edge indices of a homogeneous or heterogeneous graph.\n            target (torch.Tensor): The target of the model.\n            index (Union[int, Tensor], optional): The index of the model\n                output to explain. Can be a single index or a tensor of\n                indices. (default: :obj:`None`)\n            **kwargs (optional): Additional keyword arguments passed to\n                :obj:`model`.\n        \"\"\"\n\n    @abstractmethod\n    def supports(self) -> bool:\n        r\"\"\"Checks if the explainer supports the user-defined settings provided\n        in :obj:`self.explainer_config`, :obj:`self.model_config`.\n        \"\"\"\n        pass\n\n    ###########################################################################\n\n    @property\n    def explainer_config(self) -> ExplainerConfig:\n        r\"\"\"Returns the connected explainer configuration.\"\"\"\n        if not hasattr(self, '_explainer_config'):\n            raise ValueError(\n                f\"The explanation algorithm '{self.__class__.__name__}' is \"\n                f\"not yet connected to any explainer configuration. Please \"\n                f\"call `{self.__class__.__name__}.connect(...)` before \"\n                f\"proceeding.\")\n        return self._explainer_config\n\n    @property\n    def model_config(self) -> ModelConfig:\n        r\"\"\"Returns the connected model configuration.\"\"\"\n        if not hasattr(self, '_model_config'):\n            raise ValueError(\n                f\"The explanation algorithm '{self.__class__.__name__}' is \"\n                f\"not yet connected to any model configuration. Please call \"\n                f\"`{self.__class__.__name__}.connect(...)` before \"\n                f\"proceeding.\")\n        return self._model_config\n\n    def connect(\n        self,\n        explainer_config: ExplainerConfig,\n        model_config: ModelConfig,\n    ):\n        r\"\"\"Connects an explainer and model configuration to the explainer\n        algorithm.\n        \"\"\"\n        self._explainer_config = ExplainerConfig.cast(explainer_config)\n        self._model_config = ModelConfig.cast(model_config)\n\n        if not self.supports():\n            raise ValueError(\n                f\"The explanation algorithm '{self.__class__.__name__}' does \"\n                f\"not support the given explanation settings.\")\n\n    # Helper functions ########################################################\n\n    @staticmethod\n    def _post_process_mask(\n        mask: Optional[Tensor],\n        hard_mask: Optional[Tensor] = None,\n        apply_sigmoid: bool = True,\n    ) -> Optional[Tensor]:\n        r\"\"\"\"Post processes any mask to not include any attributions of\n        elements not involved during message passing.\n        \"\"\"\n        if mask is None:\n            return mask\n\n        mask = mask.detach()\n\n        if apply_sigmoid:\n            mask = mask.sigmoid()\n\n        if hard_mask is not None and mask.size(0) == hard_mask.size(0):\n            mask[~hard_mask] = 0.\n\n        return mask\n\n    @staticmethod\n    def _get_hard_masks(\n        model: torch.nn.Module,\n        node_index: Optional[Union[int, Tensor]],\n        edge_index: Tensor,\n        num_nodes: int,\n    ) -> Tuple[Optional[Tensor], Optional[Tensor]]:\n        r\"\"\"Returns hard node and edge masks that only include the nodes and\n        edges visited during message passing.\n        \"\"\"\n        if node_index is None:\n            return None, None  # Consider all nodes and edges.\n\n        index, _, _, edge_mask = k_hop_subgraph(\n            node_index,\n            num_hops=ExplainerAlgorithm._num_hops(model),\n            edge_index=edge_index,\n            num_nodes=num_nodes,\n            flow=ExplainerAlgorithm._flow(model),\n        )\n\n        node_mask = edge_index.new_zeros(num_nodes, dtype=torch.bool)\n        node_mask[index] = True\n\n        return node_mask, edge_mask\n\n    @staticmethod\n    def _num_hops(model: torch.nn.Module) -> int:\n        r\"\"\"Returns the number of hops the :obj:`model` is aggregating\n        information from.\n        \"\"\"\n        num_hops = 0\n        for module in model.modules():\n            if isinstance(module, MessagePassing):\n                num_hops += 1\n        return num_hops\n\n    @staticmethod\n    def _flow(model: torch.nn.Module) -> str:\n        r\"\"\"Determines the message passing flow of the :obj:`model`.\"\"\"\n        for module in model.modules():\n            if isinstance(module, MessagePassing):\n                return module.flow\n        return 'source_to_target'\n\n    def _loss_binary_classification(self, y_hat: Tensor, y: Tensor) -> Tensor:\n        if self.model_config.return_type == ModelReturnType.raw:\n            loss_fn = F.binary_cross_entropy_with_logits\n        elif self.model_config.return_type == ModelReturnType.probs:\n            loss_fn = F.binary_cross_entropy\n        else:\n            assert False\n\n        return loss_fn(y_hat.view_as(y), y.float())\n\n    def _loss_multiclass_classification(\n        self,\n        y_hat: Tensor,\n        y: Tensor,\n    ) -> Tensor:\n        if self.model_config.return_type == ModelReturnType.raw:\n            loss_fn = F.cross_entropy\n        elif self.model_config.return_type == ModelReturnType.probs:\n            loss_fn = F.nll_loss\n            y_hat = y_hat.log()\n        elif self.model_config.return_type == ModelReturnType.log_probs:\n            loss_fn = F.nll_loss\n        else:\n            assert False\n\n        return loss_fn(y_hat, y)\n\n    def _loss_regression(self, y_hat: Tensor, y: Tensor) -> Tensor:\n        assert self.model_config.return_type == ModelReturnType.raw\n        return F.mse_loss(y_hat, y)\n\n    def __repr__(self) -> str:\n        return f'{self.__class__.__name__}()'\n\n# Path: torch_geometric/explain/algorithm/utils.py\nfrom typing import Dict, Union\n\nimport torch\nfrom torch import Tensor\nfrom torch.nn import Parameter\n\nfrom torch_geometric.nn import MessagePassing\nfrom torch_geometric.typing import EdgeType\n\n\ndef set_masks(\n    model: torch.nn.Module,\n    mask: Union[Tensor, Parameter],\n    edge_index: Tensor,\n    apply_sigmoid: bool = True,\n):\n    r\"\"\"Apply mask to every graph layer in the :obj:`model`.\"\"\"\n    loop_mask = edge_index[0] != edge_index[1]\n\n    # Loop over layers and set masks on MessagePassing layers:\n    for module in model.modules():\n        if isinstance(module, MessagePassing):\n            # Skip layers that have been explicitly set to `False`:\n            if module.explain is False:\n                continue\n\n            # Convert mask to a param if it was previously registered as one.\n            # This is a workaround for the fact that PyTorch does not allow\n            # assignments of pure tensors to parameter attributes:\n            if (not isinstance(mask, Parameter)\n                    and '_edge_mask' in module._parameters):\n                mask = Parameter(mask)\n\n            module.explain = True\n            module._edge_mask = mask\n            module._loop_mask = loop_mask\n            module._apply_sigmoid = apply_sigmoid\n\n\ndef set_hetero_masks(\n    model: torch.nn.Module,\n    mask_dict: Dict[EdgeType, Union[Tensor, Parameter]],\n    edge_index_dict: Dict[EdgeType, Tensor],\n    apply_sigmoid: bool = True,\n):\n    r\"\"\"Apply masks to every heterogeneous graph layer in the :obj:`model`\n    according to edge types.\n    \"\"\"\n    for module in model.modules():\n        if isinstance(module, torch.nn.ModuleDict):\n            for edge_type in mask_dict.keys():\n                if edge_type in module:\n                    edge_level_module = module[edge_type]\n                elif '__'.join(edge_type) in module:\n                    edge_level_module = module['__'.join(edge_type)]\n                else:\n                    continue\n\n                set_masks(\n                    edge_level_module,\n                    mask_dict[edge_type],\n                    edge_index_dict[edge_type],\n                    apply_sigmoid=apply_sigmoid,\n                )\n\n\ndef clear_masks(model: torch.nn.Module):\n    r\"\"\"Clear all masks from the model.\"\"\"\n    for module in model.modules():\n        if isinstance(module, MessagePassing):\n            if module.explain is True:\n                module.explain = None\n            module._edge_mask = None\n            module._loop_mask = None\n            module._apply_sigmoid = True\n    return module\n\n# Path: torch_geometric/explain/algorithm/captum.py\nfrom enum import Enum\nfrom typing import Dict, Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.explain.algorithm.utils import (\n    clear_masks,\n    set_hetero_masks,\n    set_masks,\n)\nfrom torch_geometric.explain.config import (\n    ModelConfig,\n    ModelMode,\n    ModelReturnType,\n)\nfrom torch_geometric.typing import EdgeType, Metadata, NodeType\n\n\nclass MaskLevelType(Enum):\n    \"\"\"Enum class for the mask level type.\"\"\"\n    node = 'node'\n    edge = 'edge'\n    node_and_edge = 'node_and_edge'\n\n    @property\n    def with_edge(self) -> bool:\n        return self in [MaskLevelType.edge, MaskLevelType.node_and_edge]\n\n\nclass CaptumModel(torch.nn.Module):\n    def __init__(\n        self,\n        model: torch.nn.Module,\n        mask_type: Union[str, MaskLevelType],\n        output_idx: Optional[Union[int, Tensor]] = None,\n        model_config: Optional[ModelConfig] = None,\n    ):\n        super().__init__()\n\n        self.mask_type = MaskLevelType(mask_type)\n        self.model = model\n        self.output_idx = output_idx\n        self.model_config = model_config\n\n    def forward(self, mask, *args):\n        \"\"\"\"\"\"  # noqa: D419\n        # The mask tensor, which comes from Captum's attribution methods,\n        # contains the number of samples in dimension 0. Since we are\n        # working with only one sample, we squeeze the tensors below.\n        assert mask.shape[0] == 1, \"Dimension 0 of input should be 1\"\n        if self.mask_type == MaskLevelType.edge:\n            assert len(args) >= 2, \"Expects at least x and edge_index as args.\"\n        if self.mask_type == MaskLevelType.node:\n            assert len(args) >= 1, \"Expects at least edge_index as args.\"\n        if self.mask_type == MaskLevelType.node_and_edge:\n            assert args[0].shape[0] == 1, \"Dimension 0 of input should be 1\"\n            assert len(args[1:]) >= 1, \"Expects at least edge_index as args.\"\n\n        # Set edge mask:\n        if self.mask_type == MaskLevelType.edge:\n            set_masks(self.model, mask.squeeze(0), args[1],\n                      apply_sigmoid=False)\n        elif self.mask_type == MaskLevelType.node_and_edge:\n            set_masks(self.model, args[0].squeeze(0), args[1],\n                      apply_sigmoid=False)\n            args = args[1:]\n\n        if self.mask_type == MaskLevelType.edge:\n            x = self.model(*args)\n\n        else:\n            x = self.model(mask.squeeze(0), *args)\n\n        return self.postprocess(x)\n\n    def postprocess(self, x: Tensor) -> Tensor:\n        if self.mask_type.with_edge:\n            clear_masks(self.model)\n\n        if self.output_idx is not None:  # Filter by output index:\n            x = x[self.output_idx]\n            if (isinstance(self.output_idx, int)\n                    or self.output_idx.dim() == 0):\n                x = x.unsqueeze(0)\n\n        # Convert binary classification to multi-class classification:\n        if (self.model_config is not None\n                and self.model_config.mode == ModelMode.binary_classification):\n            assert self.model_config.return_type == ModelReturnType.probs\n            x = x.view(-1, 1)\n            x = torch.cat([1 - x, x], dim=-1)\n\n        return x\n\n\n# TODO(jinu) Is there any point of inheriting from `CaptumModel`\nclass CaptumHeteroModel(CaptumModel):\n    def __init__(\n        self,\n        model: torch.nn.Module,\n        mask_type: Union[str, MaskLevelType],\n        output_idx: Optional[Union[int, Tensor]],\n        metadata: Metadata,\n        model_config: Optional[ModelConfig] = None,\n    ):\n        super().__init__(model, mask_type, output_idx, model_config)\n        self.node_types = metadata[0]\n        self.edge_types = metadata[1]\n        self.num_node_types = len(self.node_types)\n        self.num_edge_types = len(self.edge_types)\n\n    def _captum_data_to_hetero_data(\n        self, *args\n    ) -> Tuple[Dict[NodeType, Tensor], Dict[EdgeType, Tensor], Optional[Dict[\n            EdgeType, Tensor]]]:\n        \"\"\"Converts tuple of tensors to `x_dict`, `edge_index_dict` and\n        `edge_mask_dict`.\n        \"\"\"\n        if self.mask_type == MaskLevelType.node:\n            node_tensors = args[:self.num_node_types]\n            node_tensors = [mask.squeeze(0) for mask in node_tensors]\n            x_dict = dict(zip(self.node_types, node_tensors))\n            edge_index_dict = args[self.num_node_types]\n        elif self.mask_type == MaskLevelType.edge:\n            edge_mask_tensors = args[:self.num_edge_types]\n            x_dict = args[self.num_edge_types]\n            edge_index_dict = args[self.num_edge_types + 1]\n        else:\n            node_tensors = args[:self.num_node_types]\n            node_tensors = [mask.squeeze(0) for mask in node_tensors]\n            x_dict = dict(zip(self.node_types, node_tensors))\n            edge_mask_tensors = args[self.num_node_types:self.num_node_types +\n                                     self.num_edge_types]\n            edge_index_dict = args[self.num_node_types + self.num_edge_types]\n\n        if self.mask_type.with_edge:\n            edge_mask_tensors = [mask.squeeze(0) for mask in edge_mask_tensors]\n            edge_mask_dict = dict(zip(self.edge_types, edge_mask_tensors))\n        else:\n            edge_mask_dict = None\n        return x_dict, edge_index_dict, edge_mask_dict\n\n    def forward(self, *args):\n        # Validate args:\n        if self.mask_type == MaskLevelType.node:\n            assert len(args) >= self.num_node_types + 1\n            len_remaining_args = len(args) - (self.num_node_types + 1)\n        elif self.mask_type == MaskLevelType.edge:\n            assert len(args) >= self.num_edge_types + 2\n            len_remaining_args = len(args) - (self.num_edge_types + 2)\n        else:\n            assert len(args) >= self.num_node_types + self.num_edge_types + 1\n            len_remaining_args = len(args) - (self.num_node_types +\n                                              self.num_edge_types + 1)\n\n        # Get main args:\n        (x_dict, edge_index_dict,\n         edge_mask_dict) = self._captum_data_to_hetero_data(*args)\n\n        if self.mask_type.with_edge:\n            set_hetero_masks(self.model, edge_mask_dict, edge_index_dict)\n\n        if len_remaining_args > 0:\n            # If there are args other than `x_dict` and `edge_index_dict`\n            x = self.model(x_dict, edge_index_dict,\n                           *args[-len_remaining_args:])\n        else:\n            x = self.model(x_dict, edge_index_dict)\n\n        return self.postprocess(x)\n\n\n\ndef _to_edge_mask(edge_index: Tensor) -> Tensor:\n    num_edges = edge_index.shape[1]\n    return torch.ones(num_edges, requires_grad=True, device=edge_index.device)\n\n\ndef to_captum_input(\n    x: Union[Tensor, Dict[NodeType, Tensor]],\n    edge_index: Union[Tensor, Dict[EdgeType, Tensor]],\n    mask_type: Union[str, MaskLevelType],\n    *args,\n) -> Tuple[Tuple[Tensor, ...], Tuple[Tensor, ...]]:\n    r\"\"\"Given :obj:`x`, :obj:`edge_index` and :obj:`mask_type`, converts it\n    to a format to use in `Captum <https://captum.ai/>`_ attribution\n    methods. Returns :obj:`inputs` and :obj:`additional_forward_args`\n    required for :captum:`Captum's` :obj:`attribute` functions.\n    See :meth:`~torch_geometric.nn.models.to_captum_model` for example usage.\n\n    Args:\n        x (torch.Tensor or Dict[NodeType, torch.Tensor]): The node features.\n            For heterogeneous graphs this is a dictionary holding node featues\n            for each node type.\n        edge_index(torch.Tensor or Dict[EdgeType, torch.Tensor]): The edge\n            indices. For heterogeneous graphs this is a dictionary holding the\n            :obj:`edge index` for each edge type.\n        mask_type (str): Denotes the type of mask to be created with\n            a Captum explainer. Valid inputs are :obj:`\"edge\"`, :obj:`\"node\"`,\n            and :obj:`\"node_and_edge\"`.\n        *args: Additional forward arguments of the model being explained\n            which will be added to :obj:`additional_forward_args`.\n    \"\"\"\n    mask_type = MaskLevelType(mask_type)\n\n    additional_forward_args = []\n    if isinstance(x, Tensor) and isinstance(edge_index, Tensor):\n        if mask_type == MaskLevelType.node:\n            inputs = [x.unsqueeze(0)]\n        elif mask_type == MaskLevelType.edge:\n            inputs = [_to_edge_mask(edge_index).unsqueeze(0)]\n            additional_forward_args.append(x)\n        else:\n            inputs = [x.unsqueeze(0), _to_edge_mask(edge_index).unsqueeze(0)]\n        additional_forward_args.append(edge_index)\n\n    elif isinstance(x, Dict) and isinstance(edge_index, Dict):\n        node_types = x.keys()\n        edge_types = edge_index.keys()\n        inputs = []\n        if mask_type == MaskLevelType.node:\n            for key in node_types:\n                inputs.append(x[key].unsqueeze(0))\n        elif mask_type == MaskLevelType.edge:\n            for key in edge_types:\n                inputs.append(_to_edge_mask(edge_index[key]).unsqueeze(0))\n            additional_forward_args.append(x)\n        else:\n            for key in node_types:\n                inputs.append(x[key].unsqueeze(0))\n            for key in edge_types:\n                inputs.append(_to_edge_mask(edge_index[key]).unsqueeze(0))\n        additional_forward_args.append(edge_index)\n\n    else:\n        raise ValueError(\n            \"'x' and 'edge_index' need to be either\"\n            f\"'Dict' or 'Tensor' got({type(x)}, {type(edge_index)})\")\n\n    additional_forward_args.extend(args)\n\n    return tuple(inputs), tuple(additional_forward_args)\n\n\ndef captum_output_to_dicts(\n    captum_attrs: Tuple[Tensor, ...],\n    mask_type: Union[str, MaskLevelType],\n    metadata: Metadata,\n) -> Tuple[Optional[Dict[NodeType, Tensor]], Optional[Dict[EdgeType, Tensor]]]:\n    r\"\"\"Convert the output of `Captum <https://captum.ai/>`_ attribution\n    methods which is a tuple of attributions to two dictionaries with node and\n    edge attribution tensors. This function is used while explaining\n    :class:`~torch_geometric.data.HeteroData` objects.\n    See :meth:`~torch_geometric.nn.models.to_captum_model` for example usage.\n\n    Args:\n        captum_attrs (tuple[torch.Tensor]): The output of attribution methods.\n        mask_type (str): Denotes the type of mask to be created with\n            a Captum explainer. Valid inputs are :obj:`\"edge\"`, :obj:`\"node\"`,\n            and :obj:`\"node_and_edge\"`:\n\n            1. :obj:`\"edge\"`: :obj:`captum_attrs` contains only edge\n               attributions. The returned tuple has no node attributions, and\n               an edge attribution dictionary edge types as keys and edge mask\n               tensors of shape :obj:`[num_edges]` as values.\n\n            2. :obj:`\"node\"`: :obj:`captum_attrs` contains only node\n               attributions. The returned tuple has a node attribution\n               dictionary with node types as keys and node mask tensors of\n               shape :obj:`[num_nodes, num_features]` as values, and no edge\n               attributions.\n\n            3. :obj:`\"node_and_edge\"`: :obj:`captum_attrs` contains node and\n                edge attributions.\n\n        metadata (Metadata): The metadata of the heterogeneous graph.\n    \"\"\"\n    mask_type = MaskLevelType(mask_type)\n    node_types = metadata[0]\n    edge_types = metadata[1]\n    x_attr_dict, edge_attr_dict = None, None\n    captum_attrs = [captum_attr.squeeze(0) for captum_attr in captum_attrs]\n    if mask_type == MaskLevelType.node:\n        assert len(node_types) == len(captum_attrs)\n        x_attr_dict = dict(zip(node_types, captum_attrs))\n    elif mask_type == MaskLevelType.edge:\n        assert len(edge_types) == len(captum_attrs)\n        edge_attr_dict = dict(zip(edge_types, captum_attrs))\n    elif mask_type == MaskLevelType.node_and_edge:\n        assert len(edge_types) + len(node_types) == len(captum_attrs)\n        x_attr_dict = dict(zip(node_types, captum_attrs[:len(node_types)]))\n        edge_attr_dict = dict(zip(edge_types, captum_attrs[len(node_types):]))\n    return x_attr_dict, edge_attr_dict\n\n\ndef convert_captum_output(\n    captum_attrs: Tuple[Tensor, ...],\n    mask_type: Union[str, MaskLevelType],\n    metadata: Optional[Metadata] = None,\n):\n    r\"\"\"Convert the output of `Captum.ai <https://captum.ai/>`_ attribution\n    methods which is a tuple of attributions to either\n    :obj:`(node_mask, edge_mask)` or :obj:`(node_mask_dict, edge_mask_dict)`.\n    \"\"\"\n    mask_type = MaskLevelType(mask_type)\n    if metadata is not None:\n        return captum_output_to_dicts(captum_attrs, mask_type, metadata)\n\n    node_mask = edge_mask = None\n    if mask_type == MaskLevelType.edge:\n        edge_mask = captum_attrs[0].squeeze(0)\n    elif mask_type == MaskLevelType.node:\n        node_mask = captum_attrs[0].squeeze(0)\n    else:\n        node_mask = captum_attrs[0].squeeze(0)\n        edge_mask = captum_attrs[1].squeeze(0)\n\n    return node_mask, edge_mask\n\n# Path: torch_geometric/explain/algorithm/captum_explainer.py\nimport inspect\nimport logging\nimport warnings\nfrom typing import Any, Dict, Optional, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.explain import Explanation, HeteroExplanation\nfrom torch_geometric.explain.algorithm import ExplainerAlgorithm\nfrom torch_geometric.explain.algorithm.captum import (\n    CaptumHeteroModel,\n    CaptumModel,\n    MaskLevelType,\n    convert_captum_output,\n    to_captum_input,\n)\nfrom torch_geometric.explain.config import MaskType, ModelMode, ModelReturnType\nfrom torch_geometric.typing import EdgeType, NodeType\n\n\nclass CaptumExplainer(ExplainerAlgorithm):\n    \"\"\"A `Captum <https://captum.ai>`__-based explainer for identifying compact\n    subgraph structures and node features that play a crucial role in the\n    predictions made by a GNN.\n\n    This explainer algorithm uses :captum:`null` `Captum <https://captum.ai/>`_\n    to compute attributions.\n\n    Currently, the following attribution methods are supported:\n\n    * :class:`captum.attr.IntegratedGradients`\n    * :class:`captum.attr.Saliency`\n    * :class:`captum.attr.InputXGradient`\n    * :class:`captum.attr.Deconvolution`\n    * :class:`captum.attr.ShapleyValueSampling`\n...\n", "needle_token_start": 13687, "needle_token_end": 13748, "code_context_ntokens": 16140, "output": ["```python\ndef _to_edge_mask(edge_index: Tensor) -> Tensor:\n    num_edges = edge_index.shape[1]\n    return torch.ones(num_edges, requires_grad=True, device=edge_index.device)\n```"]}
{"repo": "pyg-team/pytorch_geometric", "name": "get_self_loop_attr", "language": "python", "path": "torch_geometric/utils/loop.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function retrieves the attributes or weights of self-loops (edges that connect a node to itself) for each node in a graph. If a self-loop is absent for a node, the attribute is set to zero. If no attributes are provided, a default attribute of one is used for each self-loop.\n2. **Input**: \n   - A tensor representing the indices of edges in the graph.\n   - An optional tensor for the attributes or weights of these edges.\n   - An optional integer specifying the total number of nodes in the graph.\n3. **Output**: A tensor containing the attributes or weights of self-loops for each node. If a node does not have a self-loop, the corresponding value is zero.\n4. **Procedure**: \n   - First, identify which edges are self-loops by checking where the start and end nodes of each edge are the same.\n   - Extract the indices and attributes of these self-loops.\n   - Create a tensor initialized with zeros (or ones if no attributes are provided) to represent the attributes of self-loops for all nodes.\n   - Place the extracted attributes into this tensor at the positions corresponding to nodes with self-loops.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: torch_geometric/utils/dropout.py\nfrom typing import Optional, Tuple\n\nimport torch\nfrom torch import Tensor\n\nimport torch_geometric.typing\nfrom torch_geometric import is_compiling\nfrom torch_geometric.deprecation import deprecated\nfrom torch_geometric.typing import OptTensor\nfrom torch_geometric.utils import cumsum, degree, sort_edge_index, subgraph\nfrom torch_geometric.utils.num_nodes import maybe_num_nodes\n\n\ndef filter_adj(row: Tensor, col: Tensor, edge_attr: OptTensor,\n               mask: Tensor) -> Tuple[Tensor, Tensor, OptTensor]:\n    return row[mask], col[mask], None if edge_attr is None else edge_attr[mask]\n\n\n@deprecated(\"use 'dropout_edge' instead\")\ndef dropout_adj(\n    edge_index: Tensor,\n    edge_attr: OptTensor = None,\n    p: float = 0.5,\n    force_undirected: bool = False,\n    num_nodes: Optional[int] = None,\n    training: bool = True,\n) -> Tuple[Tensor, OptTensor]:\n    r\"\"\"Randomly drops edges from the adjacency matrix\n    :obj:`(edge_index, edge_attr)` with probability :obj:`p` using samples from\n    a Bernoulli distribution.\n\n    .. warning::\n\n        :class:`~torch_geometric.utils.dropout_adj` is deprecated and will\n        be removed in a future release.\n        Use :class:`torch_geometric.utils.dropout_edge` instead.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional\n            edge features. (default: :obj:`None`)\n        p (float, optional): Dropout probability. (default: :obj:`0.5`)\n        force_undirected (bool, optional): If set to :obj:`True`, will either\n            drop or keep both edges of an undirected edge.\n            (default: :obj:`False`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n        training (bool, optional): If set to :obj:`False`, this operation is a\n            no-op. (default: :obj:`True`)\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 1, 2, 2, 3],\n        ...                            [1, 0, 2, 1, 3, 2]])\n        >>> edge_attr = torch.tensor([1, 2, 3, 4, 5, 6])\n        >>> dropout_adj(edge_index, edge_attr)\n        (tensor([[0, 1, 2, 3],\n                [1, 2, 3, 2]]),\n        tensor([1, 3, 5, 6]))\n\n        >>> # The returned graph is kept undirected\n        >>> dropout_adj(edge_index, edge_attr, force_undirected=True)\n        (tensor([[0, 1, 2, 1, 2, 3],\n                [1, 2, 3, 0, 1, 2]]),\n        tensor([1, 3, 5, 1, 3, 5]))\n    \"\"\"\n    if p < 0. or p > 1.:\n        raise ValueError(f'Dropout probability has to be between 0 and 1 '\n                         f'(got {p}')\n\n    if not training or p == 0.0:\n        return edge_index, edge_attr\n\n    row, col = edge_index\n\n    mask = torch.rand(row.size(0), device=edge_index.device) >= p\n\n    if force_undirected:\n        mask[row > col] = False\n\n    row, col, edge_attr = filter_adj(row, col, edge_attr, mask)\n\n    if force_undirected:\n        edge_index = torch.stack(\n            [torch.cat([row, col], dim=0),\n             torch.cat([col, row], dim=0)], dim=0)\n        if edge_attr is not None:\n            edge_attr = torch.cat([edge_attr, edge_attr], dim=0)\n    else:\n        edge_index = torch.stack([row, col], dim=0)\n\n    return edge_index, edge_attr\n\n\ndef dropout_node(\n    edge_index: Tensor,\n    p: float = 0.5,\n    num_nodes: Optional[int] = None,\n    training: bool = True,\n    relabel_nodes: bool = False,\n) -> Tuple[Tensor, Tensor, Tensor]:\n    r\"\"\"Randomly drops nodes from the adjacency matrix\n    :obj:`edge_index` with probability :obj:`p` using samples from\n    a Bernoulli distribution.\n\n    The method returns (1) the retained :obj:`edge_index`, (2) the edge mask\n    indicating which edges were retained. (3) the node mask indicating\n    which nodes were retained.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        p (float, optional): Dropout probability. (default: :obj:`0.5`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n        training (bool, optional): If set to :obj:`False`, this operation is a\n            no-op. (default: :obj:`True`)\n        relabel_nodes (bool, optional): If set to `True`, the resulting\n            `edge_index` will be relabeled to hold consecutive indices\n            starting from zero.\n\n    :rtype: (:class:`LongTensor`, :class:`BoolTensor`, :class:`BoolTensor`)\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 1, 2, 2, 3],\n        ...                            [1, 0, 2, 1, 3, 2]])\n        >>> edge_index, edge_mask, node_mask = dropout_node(edge_index)\n        >>> edge_index\n        tensor([[0, 1],\n                [1, 0]])\n        >>> edge_mask\n        tensor([ True,  True, False, False, False, False])\n        >>> node_mask\n        tensor([ True,  True, False, False])\n    \"\"\"\n    if p < 0. or p > 1.:\n        raise ValueError(f'Dropout probability has to be between 0 and 1 '\n                         f'(got {p}')\n\n    num_nodes = maybe_num_nodes(edge_index, num_nodes)\n\n    if not training or p == 0.0:\n        node_mask = edge_index.new_ones(num_nodes, dtype=torch.bool)\n        edge_mask = edge_index.new_ones(edge_index.size(1), dtype=torch.bool)\n        return edge_index, edge_mask, node_mask\n\n    prob = torch.rand(num_nodes, device=edge_index.device)\n    node_mask = prob > p\n    edge_index, _, edge_mask = subgraph(\n        node_mask,\n        edge_index,\n        relabel_nodes=relabel_nodes,\n        num_nodes=num_nodes,\n        return_edge_mask=True,\n    )\n    return edge_index, edge_mask, node_mask\n\n\ndef dropout_edge(edge_index: Tensor, p: float = 0.5,\n                 force_undirected: bool = False,\n                 training: bool = True) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Randomly drops edges from the adjacency matrix\n    :obj:`edge_index` with probability :obj:`p` using samples from\n    a Bernoulli distribution.\n\n    The method returns (1) the retained :obj:`edge_index`, (2) the edge mask\n    or index indicating which edges were retained, depending on the argument\n    :obj:`force_undirected`.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        p (float, optional): Dropout probability. (default: :obj:`0.5`)\n        force_undirected (bool, optional): If set to :obj:`True`, will either\n            drop or keep both edges of an undirected edge.\n            (default: :obj:`False`)\n        training (bool, optional): If set to :obj:`False`, this operation is a\n            no-op. (default: :obj:`True`)\n\n    :rtype: (:class:`LongTensor`, :class:`BoolTensor` or :class:`LongTensor`)\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 1, 2, 2, 3],\n        ...                            [1, 0, 2, 1, 3, 2]])\n        >>> edge_index, edge_mask = dropout_edge(edge_index)\n        >>> edge_index\n        tensor([[0, 1, 2, 2],\n                [1, 2, 1, 3]])\n        >>> edge_mask # masks indicating which edges are retained\n        tensor([ True, False,  True,  True,  True, False])\n\n        >>> edge_index, edge_id = dropout_edge(edge_index,\n        ...                                    force_undirected=True)\n        >>> edge_index\n        tensor([[0, 1, 2, 1, 2, 3],\n                [1, 2, 3, 0, 1, 2]])\n        >>> edge_id # indices indicating which edges are retained\n        tensor([0, 2, 4, 0, 2, 4])\n    \"\"\"\n    if p < 0. or p > 1.:\n        raise ValueError(f'Dropout probability has to be between 0 and 1 '\n                         f'(got {p}')\n\n    if not training or p == 0.0:\n        edge_mask = edge_index.new_ones(edge_index.size(1), dtype=torch.bool)\n        return edge_index, edge_mask\n\n    row, col = edge_index\n\n    edge_mask = torch.rand(row.size(0), device=edge_index.device) >= p\n\n    if force_undirected:\n        edge_mask[row > col] = False\n\n    edge_index = edge_index[:, edge_mask]\n\n    if force_undirected:\n        edge_index = torch.cat([edge_index, edge_index.flip(0)], dim=1)\n        edge_mask = edge_mask.nonzero().repeat((2, 1)).squeeze()\n\n    return edge_index, edge_mask\n\n\ndef dropout_path(edge_index: Tensor, p: float = 0.2, walks_per_node: int = 1,\n                 walk_length: int = 3, num_nodes: Optional[int] = None,\n                 is_sorted: bool = False,\n                 training: bool = True) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Drops edges from the adjacency matrix :obj:`edge_index`\n    based on random walks. The source nodes to start random walks from are\n    sampled from :obj:`edge_index` with probability :obj:`p`, following\n    a Bernoulli distribution.\n\n    The method returns (1) the retained :obj:`edge_index`, (2) the edge mask\n    indicating which edges were retained.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        p (float, optional): Sample probability. (default: :obj:`0.2`)\n        walks_per_node (int, optional): The number of walks per node, same as\n            :class:`~torch_geometric.nn.models.Node2Vec`. (default: :obj:`1`)\n        walk_length (int, optional): The walk length, same as\n            :class:`~torch_geometric.nn.models.Node2Vec`. (default: :obj:`3`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n        is_sorted (bool, optional): If set to :obj:`True`, will expect\n            :obj:`edge_index` to be already sorted row-wise.\n            (default: :obj:`False`)\n        training (bool, optional): If set to :obj:`False`, this operation is a\n            no-op. (default: :obj:`True`)\n\n    :rtype: (:class:`LongTensor`, :class:`BoolTensor`)\n\n    Example:\n        >>> edge_index = torch.tensor([[0, 1, 1, 2, 2, 3],\n...\n# Path: torch_geometric/utils/embedding.py\nimport warnings\nfrom typing import Any, List\n\nimport torch\nfrom torch import Tensor\n\n\ndef get_embeddings(\n    model: torch.nn.Module,\n    *args: Any,\n    **kwargs: Any,\n) -> List[Tensor]:\n    \"\"\"Returns the output embeddings of all\n    :class:`~torch_geometric.nn.conv.MessagePassing` layers in\n    :obj:`model`.\n\n    Internally, this method registers forward hooks on all\n    :class:`~torch_geometric.nn.conv.MessagePassing` layers of a :obj:`model`,\n    and runs the forward pass of the :obj:`model` by calling\n    :obj:`model(*args, **kwargs)`.\n\n    Args:\n        model (torch.nn.Module): The message passing model.\n        *args: Arguments passed to the model.\n        **kwargs (optional): Additional keyword arguments passed to the model.\n    \"\"\"\n    from torch_geometric.nn import MessagePassing\n\n    embeddings: List[Tensor] = []\n\n    def hook(model: torch.nn.Module, inputs: Any, outputs: Any) -> None:\n        # Clone output in case it will be later modified in-place:\n        outputs = outputs[0] if isinstance(outputs, tuple) else outputs\n        assert isinstance(outputs, Tensor)\n        embeddings.append(outputs.clone())\n\n    hook_handles = []\n    for module in model.modules():  # Register forward hooks:\n        if isinstance(module, MessagePassing):\n            hook_handles.append(module.register_forward_hook(hook))\n\n    if len(hook_handles) == 0:\n        warnings.warn(\"The 'model' does not have any 'MessagePassing' layers\")\n\n    training = model.training\n    model.eval()\n    with torch.no_grad():\n        model(*args, **kwargs)\n    model.train(training)\n\n    for handle in hook_handles:  # Remove hooks:\n        handle.remove()\n\n    return embeddings\n\n# Path: torch_geometric/utils/geodesic.py\nimport multiprocessing as mp\nimport warnings\nfrom typing import Optional\n\nimport torch\nfrom torch import Tensor\n\n\ndef geodesic_distance(  # noqa: D417\n    pos: Tensor,\n    face: Tensor,\n    src: Optional[Tensor] = None,\n    dst: Optional[Tensor] = None,\n    norm: bool = True,\n    max_distance: Optional[float] = None,\n    num_workers: int = 0,\n    # Backward compatibility for `dest`:\n    **kwargs: Optional[Tensor],\n) -> Tensor:\n    r\"\"\"Computes (normalized) geodesic distances of a mesh given by :obj:`pos`\n    and :obj:`face`. If :obj:`src` and :obj:`dst` are given, this method only\n    computes the geodesic distances for the respective source and target\n    node-pairs.\n\n    .. note::\n\n        This function requires the :obj:`gdist` package.\n        To install, run :obj:`pip install cython && pip install gdist`.\n\n    Args:\n        pos (torch.Tensor): The node positions.\n        face (torch.Tensor): The face indices.\n        src (torch.Tensor, optional): If given, only compute geodesic distances\n            for the specified source indices. (default: :obj:`None`)\n        dst (torch.Tensor, optional): If given, only compute geodesic distances\n            for the specified target indices. (default: :obj:`None`)\n        norm (bool, optional): Normalizes geodesic distances by\n            :math:`\\sqrt{\\textrm{area}(\\mathcal{M})}`. (default: :obj:`True`)\n        max_distance (float, optional): If given, only yields results for\n            geodesic distances less than :obj:`max_distance`. This will speed\n            up runtime dramatically. (default: :obj:`None`)\n        num_workers (int, optional): How many subprocesses to use for\n            calculating geodesic distances.\n            :obj:`0` means that computation takes place in the main process.\n            :obj:`-1` means that the available amount of CPU cores is used.\n            (default: :obj:`0`)\n\n    :rtype: :class:`Tensor`\n\n    Example:\n        >>> pos = torch.tensor([[0.0, 0.0, 0.0],\n        ...                     [2.0, 0.0, 0.0],\n        ...                     [0.0, 2.0, 0.0],\n        ...                     [2.0, 2.0, 0.0]])\n        >>> face = torch.tensor([[0, 0],\n        ...                      [1, 2],\n        ...                      [3, 3]])\n        >>> geodesic_distance(pos, face)\n        [[0, 1, 1, 1.4142135623730951],\n        [1, 0, 1.4142135623730951, 1],\n        [1, 1.4142135623730951, 0, 1],\n        [1.4142135623730951, 1, 1, 0]]\n    \"\"\"\n    import gdist\n\n    if 'dest' in kwargs:\n        dst = kwargs['dest']\n        warnings.warn(\"'dest' attribute in 'geodesic_distance' is deprecated \"\n                      \"and will be removed in a future release. Use the 'dst' \"\n                      \"argument instead.\")\n\n    max_distance = float('inf') if max_distance is None else max_distance\n\n    if norm:\n        area = (pos[face[1]] - pos[face[0]]).cross(\n            pos[face[2]] - pos[face[0]],\n            dim=1,\n        )\n        scale = float((area.norm(p=2, dim=1) / 2).sum().sqrt())\n    else:\n        scale = 1.0\n\n    dtype = pos.dtype\n\n    pos = pos.detach().cpu().to(torch.double).numpy()\n    face = face.detach().t().cpu().to(torch.int).numpy()\n\n    if src is None and dst is None:\n        out = gdist.local_gdist_matrix(pos, face,\n                                       max_distance * scale).toarray() / scale\n        return torch.from_numpy(out).to(dtype)\n\n    if src is None:\n        src = torch.arange(pos.shape[0], dtype=torch.int).numpy()\n    else:\n        src = src.detach().cpu().to(torch.int).numpy()\n    assert src is not None\n\n    dst = None if dst is None else dst.detach().cpu().to(torch.int).numpy()\n\n    def _parallel_loop(\n        pos: Tensor,\n        face: Tensor,\n        src: Tensor,\n        dst: Optional[Tensor],\n        max_distance: float,\n        scale: float,\n        i: int,\n        dtype: torch.dtype,\n    ) -> Tensor:\n        s = src[i:i + 1]\n        d = None if dst is None else dst[i:i + 1]\n        out = gdist.compute_gdist(pos, face, s, d, max_distance * scale)\n        out = out / scale\n        return torch.from_numpy(out).to(dtype)\n\n    num_workers = mp.cpu_count() if num_workers <= -1 else num_workers\n    if num_workers > 0:\n        with mp.Pool(num_workers) as pool:\n            outs = pool.starmap(\n                _parallel_loop,\n                [(pos, face, src, dst, max_distance, scale, i, dtype)\n                 for i in range(len(src))])\n    else:\n        outs = [\n            _parallel_loop(pos, face, src, dst, max_distance, scale, i, dtype)\n            for i in range(len(src))\n        ]\n\n    out = torch.cat(outs, dim=0)\n\n    if dst is None:\n        out = out.view(-1, pos.shape[0])\n\n    return out\n\n# Path: torch_geometric/utils/isolated.py\nfrom typing import Optional, Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.utils import remove_self_loops, segregate_self_loops\nfrom torch_geometric.utils.num_nodes import maybe_num_nodes\n\n\ndef contains_isolated_nodes(\n    edge_index: Tensor,\n    num_nodes: Optional[int] = None,\n) -> bool:\n    r\"\"\"Returns :obj:`True` if the graph given by :attr:`edge_index` contains\n    isolated nodes.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n\n    :rtype: bool\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 0],\n        ...                            [1, 0, 0]])\n        >>> contains_isolated_nodes(edge_index)\n        False\n\n        >>> contains_isolated_nodes(edge_index, num_nodes=3)\n        True\n    \"\"\"\n    num_nodes = maybe_num_nodes(edge_index, num_nodes)\n    edge_index, _ = remove_self_loops(edge_index)\n    return torch.unique(edge_index.view(-1)).numel() < num_nodes\n\n\ndef remove_isolated_nodes(\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor], Tensor]:\n    r\"\"\"Removes the isolated nodes from the graph given by :attr:`edge_index`\n    with optional edge attributes :attr:`edge_attr`.\n    In addition, returns a mask of shape :obj:`[num_nodes]` to manually filter\n    out isolated node features later on.\n    Self-loops are preserved for non-isolated nodes.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional\n            edge features. (default: :obj:`None`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n\n    :rtype: (LongTensor, Tensor, BoolTensor)\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 0],\n        ...                            [1, 0, 0]])\n        >>> edge_index, edge_attr, mask = remove_isolated_nodes(edge_index)\n        >>> mask # node mask (2 nodes)\n        tensor([True, True])\n\n        >>> edge_index, edge_attr, mask = remove_isolated_nodes(edge_index,\n        ...                                                     num_nodes=3)\n        >>> mask # node mask (3 nodes)\n        tensor([True, True, False])\n    \"\"\"\n    num_nodes = maybe_num_nodes(edge_index, num_nodes)\n\n    out = segregate_self_loops(edge_index, edge_attr)\n    edge_index, edge_attr, loop_edge_index, loop_edge_attr = out\n\n    mask = torch.zeros(num_nodes, dtype=torch.bool, device=edge_index.device)\n    mask[edge_index.view(-1)] = 1\n\n    assoc = torch.full((num_nodes, ), -1, dtype=torch.long, device=mask.device)\n    assoc[mask] = torch.arange(mask.sum(), device=assoc.device)  # type: ignore\n    edge_index = assoc[edge_index]\n\n    loop_mask = torch.zeros_like(mask)\n    loop_mask[loop_edge_index[0]] = 1\n    loop_mask = loop_mask & mask\n    loop_assoc = torch.full_like(assoc, -1)\n    loop_assoc[loop_edge_index[0]] = torch.arange(loop_edge_index.size(1),\n                                                  device=loop_assoc.device)\n    loop_idx = loop_assoc[loop_mask]\n    loop_edge_index = assoc[loop_edge_index[:, loop_idx]]\n\n    edge_index = torch.cat([edge_index, loop_edge_index], dim=1)\n\n    if edge_attr is not None:\n        assert loop_edge_attr is not None\n        loop_edge_attr = loop_edge_attr[loop_idx]\n        edge_attr = torch.cat([edge_attr, loop_edge_attr], dim=0)\n\n    return edge_index, edge_attr, mask\n\n# Path: torch_geometric/utils/laplacian.py\nfrom typing import Optional, Tuple\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric.typing import OptTensor\nfrom torch_geometric.utils import add_self_loops, remove_self_loops, scatter\nfrom torch_geometric.utils.num_nodes import maybe_num_nodes\n\n\ndef get_laplacian(\n    edge_index: Tensor,\n    edge_weight: OptTensor = None,\n    normalization: Optional[str] = None,\n    dtype: Optional[torch.dtype] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    r\"\"\"Computes the graph Laplacian of the graph given by :obj:`edge_index`\n    and optional :obj:`edge_weight`.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_weight (Tensor, optional): One-dimensional edge weights.\n            (default: :obj:`None`)\n        normalization (str, optional): The normalization scheme for the graph\n            Laplacian (default: :obj:`None`):\n\n            1. :obj:`None`: No normalization\n            :math:`\\mathbf{L} = \\mathbf{D} - \\mathbf{A}`\n\n            2. :obj:`\"sym\"`: Symmetric normalization\n            :math:`\\mathbf{L} = \\mathbf{I} - \\mathbf{D}^{-1/2} \\mathbf{A}\n            \\mathbf{D}^{-1/2}`\n\n            3. :obj:`\"rw\"`: Random-walk normalization\n            :math:`\\mathbf{L} = \\mathbf{I} - \\mathbf{D}^{-1} \\mathbf{A}`\n        dtype (torch.dtype, optional): The desired data type of returned tensor\n            in case :obj:`edge_weight=None`. (default: :obj:`None`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 1, 2],\n        ...                            [1, 0, 2, 1]])\n        >>> edge_weight = torch.tensor([1., 2., 2., 4.])\n\n        >>> # No normalization\n        >>> lap = get_laplacian(edge_index, edge_weight)\n\n        >>> # Symmetric normalization\n        >>> lap_sym = get_laplacian(edge_index, edge_weight,\n                                    normalization='sym')\n\n        >>> # Random-walk normalization\n        >>> lap_rw = get_laplacian(edge_index, edge_weight, normalization='rw')\n    \"\"\"\n    if normalization is not None:\n        assert normalization in ['sym', 'rw']  # 'Invalid normalization'\n\n    edge_index, edge_weight = remove_self_loops(edge_index, edge_weight)\n\n    if edge_weight is None:\n        edge_weight = torch.ones(edge_index.size(1), dtype=dtype,\n                                 device=edge_index.device)\n\n    num_nodes = maybe_num_nodes(edge_index, num_nodes)\n\n    row, col = edge_index[0], edge_index[1]\n    deg = scatter(edge_weight, row, 0, dim_size=num_nodes, reduce='sum')\n\n    if normalization is None:\n        # L = D - A.\n        edge_index, _ = add_self_loops(edge_index, num_nodes=num_nodes)\n        edge_weight = torch.cat([-edge_weight, deg], dim=0)\n    elif normalization == 'sym':\n        # Compute A_norm = -D^{-1/2} A D^{-1/2}.\n        deg_inv_sqrt = deg.pow_(-0.5)\n        deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0)\n        edge_weight = deg_inv_sqrt[row] * edge_weight * deg_inv_sqrt[col]\n\n        # L = I - A_norm.\n        assert isinstance(edge_weight, Tensor)\n        edge_index, edge_weight = add_self_loops(  #\n            edge_index, -edge_weight, fill_value=1., num_nodes=num_nodes)\n    else:\n        # Compute A_norm = -D^{-1} A.\n        deg_inv = 1.0 / deg\n        deg_inv.masked_fill_(deg_inv == float('inf'), 0)\n        edge_weight = deg_inv[row] * edge_weight\n\n        # L = I - A_norm.\n        assert isinstance(edge_weight, Tensor)\n        edge_index, edge_weight = add_self_loops(  #\n            edge_index, -edge_weight, fill_value=1., num_nodes=num_nodes)\n\n    return edge_index, edge_weight\n\n# Path: torch_geometric/utils/loop.py\nimport typing\nfrom typing import Optional, Tuple, Union\n\nimport torch\nfrom torch import Tensor\n\nfrom torch_geometric import EdgeIndex\nfrom torch_geometric.utils import scatter\nfrom torch_geometric.utils.num_nodes import maybe_num_nodes\nfrom torch_geometric.utils.sparse import (\n    is_torch_sparse_tensor,\n    to_edge_index,\n    to_torch_coo_tensor,\n    to_torch_csr_tensor,\n)\n\nif typing.TYPE_CHECKING:\n    from typing import overload\nelse:\n    from torch.jit import _overload as overload\n\n\ndef contains_self_loops(edge_index: Tensor) -> bool:\n    r\"\"\"Returns :obj:`True` if the graph given by :attr:`edge_index` contains\n    self-loops.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n\n    :rtype: bool\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 0],\n        ...                            [1, 0, 0]])\n        >>> contains_self_loops(edge_index)\n        True\n\n        >>> edge_index = torch.tensor([[0, 1, 1],\n        ...                            [1, 0, 2]])\n        >>> contains_self_loops(edge_index)\n        False\n    \"\"\"\n    mask = edge_index[0] == edge_index[1]\n    return mask.sum().item() > 0\n\n\n@overload\ndef remove_self_loops(\n    edge_index: Tensor,\n    edge_attr: None = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef remove_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef remove_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\ndef remove_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    r\"\"\"Removes every self-loop in the graph given by :attr:`edge_index`, so\n    that :math:`(i,i) \\not\\in \\mathcal{E}` for every :math:`i \\in \\mathcal{V}`.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional\n            edge features. (default: :obj:`None`)\n\n    :rtype: (:class:`LongTensor`, :class:`Tensor`)\n\n    Example:\n        >>> edge_index = torch.tensor([[0, 1, 0],\n        ...                            [1, 0, 0]])\n        >>> edge_attr = [[1, 2], [3, 4], [5, 6]]\n        >>> edge_attr = torch.tensor(edge_attr)\n        >>> remove_self_loops(edge_index, edge_attr)\n        (tensor([[0, 1],\n                [1, 0]]),\n        tensor([[1, 2],\n                [3, 4]]))\n    \"\"\"\n    size: Optional[Tuple[int, int]] = None\n    if not typing.TYPE_CHECKING and torch.jit.is_scripting():\n        layout: Optional[int] = None\n    else:\n        layout: Optional[torch.layout] = None\n\n    value: Optional[Tensor] = None\n    if is_torch_sparse_tensor(edge_index):\n        layout = edge_index.layout\n        size = (edge_index.size(0), edge_index.size(1))\n        edge_index, value = to_edge_index(edge_index)\n\n    is_undirected = False\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        is_undirected = edge_index.is_undirected\n\n    mask = edge_index[0] != edge_index[1]\n    edge_index = edge_index[:, mask]\n\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        edge_index._is_undirected = is_undirected\n\n    if layout is not None:\n        assert edge_attr is None\n        assert value is not None\n        value = value[mask]\n        if str(layout) == 'torch.sparse_coo':  # str(...) for TorchScript :(\n            return to_torch_coo_tensor(edge_index, value, size, True), None\n        elif str(layout) == 'torch.sparse_csr':\n            return to_torch_csr_tensor(edge_index, value, size, True), None\n        raise ValueError(f\"Unexpected sparse tensor layout (got '{layout}')\")\n\n    if edge_attr is None:\n        return edge_index, None\n    else:\n        return edge_index, edge_attr[mask]\n\n\n@overload\ndef segregate_self_loops(\n    edge_index: Tensor,\n    edge_attr: None = None,\n) -> Tuple[Tensor, None, Tensor, None]:\n    pass\n\n\n@overload\ndef segregate_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n) -> Tuple[Tensor, Tensor, Tensor, Tensor]:\n    pass\n\n\n@overload\ndef segregate_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n) -> Tuple[Tensor, Optional[Tensor], Tensor, Optional[Tensor]]:\n    pass\n\n\ndef segregate_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n) -> Tuple[Tensor, Optional[Tensor], Tensor, Optional[Tensor]]:\n    r\"\"\"Segregates self-loops from the graph.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional\n            edge features. (default: :obj:`None`)\n\n    :rtype: (:class:`LongTensor`, :class:`Tensor`, :class:`LongTensor`,\n        :class:`Tensor`)\n\n    Example:\n        >>> edge_index = torch.tensor([[0, 0, 1],\n        ...                            [0, 1, 0]])\n        >>> (edge_index, edge_attr,\n        ...  loop_edge_index,\n        ...  loop_edge_attr) = segregate_self_loops(edge_index)\n        >>>  loop_edge_index\n        tensor([[0],\n                [0]])\n    \"\"\"\n    mask = edge_index[0] != edge_index[1]\n    inv_mask = ~mask\n\n    is_undirected = False\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        is_undirected = edge_index.is_undirected\n\n    loop_edge_index = edge_index[:, inv_mask]\n    loop_edge_attr = None if edge_attr is None else edge_attr[inv_mask]\n    edge_index = edge_index[:, mask]\n    edge_attr = None if edge_attr is None else edge_attr[mask]\n\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        assert isinstance(loop_edge_index, EdgeIndex)\n        edge_index._is_undirected = is_undirected\n        loop_edge_index._is_undirected = is_undirected\n\n    return edge_index, edge_attr, loop_edge_index, loop_edge_attr\n\n\n@overload\ndef add_self_loops(\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\n@overload\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[Tuple[int, int]] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\ndef add_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n    fill_value: Optional[Union[float, Tensor, str]] = None,\n    num_nodes: Optional[Union[int, Tuple[int, int]]] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    r\"\"\"Adds a self-loop :math:`(i,i) \\in \\mathcal{E}` to every node\n    :math:`i \\in \\mathcal{V}` in the graph given by :attr:`edge_index`.\n    In case the graph is weighted or has multi-dimensional edge features\n    (:obj:`edge_attr != None`), edge features of self-loops will be added\n    according to :obj:`fill_value`.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional edge\n            features. (default: :obj:`None`)\n        fill_value (float or Tensor or str, optional): The way to generate\n            edge features of self-loops (in case :obj:`edge_attr != None`).\n            If given as :obj:`float` or :class:`torch.Tensor`, edge features of\n            self-loops will be directly given by :obj:`fill_value`.\n            If given as :obj:`str`, edge features of self-loops are computed by\n            aggregating all features of edges that point to the specific node,\n            according to a reduce operation. (:obj:`\"add\"`, :obj:`\"mean\"`,\n            :obj:`\"min\"`, :obj:`\"max\"`, :obj:`\"mul\"`). (default: :obj:`1.`)\n        num_nodes (int or Tuple[int, int], optional): The number of nodes,\n            *i.e.* :obj:`max_val + 1` of :attr:`edge_index`.\n            If given as a tuple, then :obj:`edge_index` is interpreted as a\n            bipartite graph with shape :obj:`(num_src_nodes, num_dst_nodes)`.\n            (default: :obj:`None`)\n\n    :rtype: (:class:`LongTensor`, :class:`Tensor`)\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 0],\n        ...                            [1, 0, 0]])\n        >>> edge_weight = torch.tensor([0.5, 0.5, 0.5])\n        >>> add_self_loops(edge_index)\n        (tensor([[0, 1, 0, 0, 1],\n                [1, 0, 0, 0, 1]]),\n        None)\n\n        >>> add_self_loops(edge_index, edge_weight)\n        (tensor([[0, 1, 0, 0, 1],\n                [1, 0, 0, 0, 1]]),\n        tensor([0.5000, 0.5000, 0.5000, 1.0000, 1.0000]))\n\n        >>> # edge features of self-loops are filled by constant `2.0`\n        >>> add_self_loops(edge_index, edge_weight,\n        ...                fill_value=2.)\n        (tensor([[0, 1, 0, 0, 1],\n                [1, 0, 0, 0, 1]]),\n        tensor([0.5000, 0.5000, 0.5000, 2.0000, 2.0000]))\n\n        >>> # Use 'add' operation to merge edge features for self-loops\n        >>> add_self_loops(edge_index, edge_weight,\n        ...                fill_value='add')\n        (tensor([[0, 1, 0, 0, 1],\n                [1, 0, 0, 0, 1]]),\n        tensor([0.5000, 0.5000, 0.5000, 1.0000, 0.5000]))\n    \"\"\"\n    if not typing.TYPE_CHECKING and torch.jit.is_scripting():\n        layout: Optional[int] = None\n    else:\n        layout: Optional[torch.layout] = None\n    is_sparse = is_torch_sparse_tensor(edge_index)\n\n    value: Optional[Tensor] = None\n    if is_sparse:\n        assert edge_attr is None\n        layout = edge_index.layout\n        size = (edge_index.size(0), edge_index.size(1))\n        N = min(size)\n        edge_index, value = to_edge_index(edge_index)\n    elif isinstance(num_nodes, (tuple, list)):\n        size = (num_nodes[0], num_nodes[1])\n        N = min(size)\n    else:\n        N = maybe_num_nodes(edge_index, num_nodes)\n        size = (N, N)\n\n    device = edge_index.device\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        loop_index: Tensor = EdgeIndex(\n            torch.arange(0, N, device=device).view(1, -1).repeat(2, 1),\n            sparse_size=(N, N),\n            is_undirected=True,\n        )\n    else:\n        loop_index = torch.arange(0, N, device=device).view(1, -1).repeat(2, 1)\n\n    full_edge_index = torch.cat([edge_index, loop_index], dim=1)\n\n    if is_sparse:\n        assert edge_attr is None\n        assert value is not None\n        loop_attr = compute_loop_attr(  #\n            edge_index, value, N, is_sparse, fill_value)\n        value = torch.cat([value, loop_attr], dim=0)\n\n        if str(layout) == 'torch.sparse_coo':  # str(...) for TorchScript :(\n            return to_torch_coo_tensor(full_edge_index, value, size), None\n        elif str(layout) == 'torch.sparse_csr':\n            return to_torch_csr_tensor(full_edge_index, value, size), None\n        raise ValueError(f\"Unexpected sparse tensor layout (got '{layout}')\")\n\n    if edge_attr is not None:\n        loop_attr = compute_loop_attr(  #\n            edge_index, edge_attr, N, is_sparse, fill_value)\n        edge_attr = torch.cat([edge_attr, loop_attr], dim=0)\n\n    return full_edge_index, edge_attr\n\n\n@overload\ndef add_remaining_self_loops(\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: None = None,\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, None]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Tensor]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[float] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\n@overload\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor],\n    fill_value: Optional[str] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    pass\n\n\ndef add_remaining_self_loops(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n    fill_value: Optional[Union[float, Tensor, str]] = None,\n    num_nodes: Optional[int] = None,\n) -> Tuple[Tensor, Optional[Tensor]]:\n    r\"\"\"Adds remaining self-loop :math:`(i,i) \\in \\mathcal{E}` to every node\n    :math:`i \\in \\mathcal{V}` in the graph given by :attr:`edge_index`.\n    In case the graph is weighted or has multi-dimensional edge features\n    (:obj:`edge_attr != None`), edge features of non-existing self-loops will\n    be added according to :obj:`fill_value`.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional edge\n            features. (default: :obj:`None`)\n        fill_value (float or Tensor or str, optional): The way to generate\n            edge features of self-loops (in case :obj:`edge_attr != None`).\n            If given as :obj:`float` or :class:`torch.Tensor`, edge features of\n            self-loops will be directly given by :obj:`fill_value`.\n            If given as :obj:`str`, edge features of self-loops are computed by\n            aggregating all features of edges that point to the specific node,\n            according to a reduce operation. (:obj:`\"add\"`, :obj:`\"mean\"`,\n            :obj:`\"min\"`, :obj:`\"max\"`, :obj:`\"mul\"`). (default: :obj:`1.`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n\n    :rtype: (:class:`LongTensor`, :class:`Tensor`)\n\n    Example:\n        >>> edge_index = torch.tensor([[0, 1],\n        ...                            [1, 0]])\n        >>> edge_weight = torch.tensor([0.5, 0.5])\n        >>> add_remaining_self_loops(edge_index, edge_weight)\n        (tensor([[0, 1, 0, 1],\n                [1, 0, 0, 1]]),\n        tensor([0.5000, 0.5000, 1.0000, 1.0000]))\n    \"\"\"\n    N = maybe_num_nodes(edge_index, num_nodes)\n    mask = edge_index[0] != edge_index[1]\n\n    device = edge_index.device\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        loop_index: Tensor = EdgeIndex(\n            torch.arange(0, N, device=device).view(1, -1).repeat(2, 1),\n            sparse_size=(N, N),\n            is_undirected=True,\n        )\n    else:\n        loop_index = torch.arange(0, N, device=device).view(1, -1).repeat(2, 1)\n\n    if edge_attr is not None:\n\n        loop_attr = compute_loop_attr(  #\n            edge_index, edge_attr, N, False, fill_value)\n\n        inv_mask = ~mask\n        loop_attr[edge_index[0][inv_mask]] = edge_attr[inv_mask]\n\n        edge_attr = torch.cat([edge_attr[mask], loop_attr], dim=0)\n\n    is_undirected = False\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        is_undirected = edge_index.is_undirected\n\n    edge_index = edge_index[:, mask]\n\n    if not torch.jit.is_scripting() and isinstance(edge_index, EdgeIndex):\n        edge_index._is_undirected = is_undirected\n\n    edge_index = torch.cat([edge_index, loop_index], dim=1)\n\n    return edge_index, edge_attr\n\n\n\ndef get_self_loop_attr(\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tensor:\n    r\"\"\"Returns the edge features or weights of self-loops\n    :math:`(i, i)` of every node :math:`i \\in \\mathcal{V}` in the\n    graph given by :attr:`edge_index`. Edge features of missing self-loops not\n    present in :attr:`edge_index` will be filled with zeros. If\n    :attr:`edge_attr` is not given, it will be the vector of ones.\n\n    .. note::\n        This operation is analogous to getting the diagonal elements of the\n        dense adjacency matrix.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional edge\n            features. (default: :obj:`None`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n\n    :rtype: :class:`Tensor`\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 0],\n        ...                            [1, 0, 0]])\n        >>> edge_weight = torch.tensor([0.2, 0.3, 0.5])\n        >>> get_self_loop_attr(edge_index, edge_weight)\n        tensor([0.5000, 0.0000])\n\n        >>> get_self_loop_attr(edge_index, edge_weight, num_nodes=4)\n        tensor([0.5000, 0.0000, 0.0000, 0.0000])\n    \"\"\"\n    loop_mask = edge_index[0] == edge_index[1]\n    loop_index = edge_index[0][loop_mask]\n\n    if edge_attr is not None:\n        loop_attr = edge_attr[loop_mask]\n    else:  # A vector of ones:\n        loop_attr = torch.ones(loop_index.numel(), device=edge_index.device)\n\n    num_nodes = maybe_num_nodes(edge_index, num_nodes)\n    full_loop_attr = loop_attr.new_zeros((num_nodes, ) + loop_attr.size()[1:])\n    full_loop_attr[loop_index] = loop_attr\n\n    return full_loop_attr\n\n\n@overload\ndef compute_loop_attr(\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    num_nodes: int,\n    is_sparse: bool,\n    fill_value: Optional[float] = None,\n) -> Tensor:\n    pass\n\n\n@overload\ndef compute_loop_attr(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    num_nodes: int,\n    is_sparse: bool,\n    fill_value: Optional[Tensor] = None,\n) -> Tensor:\n    pass\n\n\n@overload\ndef compute_loop_attr(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    num_nodes: int,\n    is_sparse: bool,\n    fill_value: Optional[str] = None,\n) -> Tensor:\n    pass\n\n\ndef compute_loop_attr(  # noqa: F811\n    edge_index: Tensor,\n    edge_attr: Tensor,\n    num_nodes: int,\n    is_sparse: bool,\n    fill_value: Optional[Union[float, Tensor, str]] = None,\n) -> Tensor:\n\n    if fill_value is None:\n        size = (num_nodes, ) + edge_attr.size()[1:]\n        return edge_attr.new_ones(size)\n\n    elif isinstance(fill_value, (int, float)):\n        size = (num_nodes, ) + edge_attr.size()[1:]\n        return edge_attr.new_full(size, fill_value)\n\n    elif isinstance(fill_value, Tensor):\n        size = (num_nodes, ) + edge_attr.size()[1:]\n        loop_attr = fill_value.to(edge_attr.device, edge_attr.dtype)\n        if edge_attr.dim() != loop_attr.dim():\n...\n", "needle_token_start": 14965, "needle_token_end": 15603, "code_context_ntokens": 16090, "output": ["```python\ndef get_self_loop_attr(\n    edge_index: Tensor,\n    edge_attr: Optional[Tensor] = None,\n    num_nodes: Optional[int] = None,\n) -> Tensor:\n    r\"\"\"Returns the edge features or weights of self-loops\n    :math:`(i, i)` of every node :math:`i \\in \\mathcal{V}` in the\n    graph given by :attr:`edge_index`. Edge features of missing self-loops not\n    present in :attr:`edge_index` will be filled with zeros. If\n    :attr:`edge_attr` is not given, it will be the vector of ones.\n\n    .. note::\n        This operation is analogous to getting the diagonal elements of the\n        dense adjacency matrix.\n\n    Args:\n        edge_index (LongTensor): The edge indices.\n        edge_attr (Tensor, optional): Edge weights or multi-dimensional edge\n            features. (default: :obj:`None`)\n        num_nodes (int, optional): The number of nodes, *i.e.*\n            :obj:`max_val + 1` of :attr:`edge_index`. (default: :obj:`None`)\n\n    :rtype: :class:`Tensor`\n\n    Examples:\n        >>> edge_index = torch.tensor([[0, 1, 0],\n        ...                            [1, 0, 0]])\n        >>> edge_weight = torch.tensor([0.2, 0.3, 0.5])\n        >>> get_self_loop_attr(edge_index, edge_weight)\n        tensor([0.5000, 0.0000])\n\n        >>> get_self_loop_attr(edge_index, edge_weight, num_nodes=4)\n        tensor([0.5000, 0.0000, 0.0000, 0.0000])\n    \"\"\"\n    loop_mask = edge_index[0] == edge_index[1]\n    loop_index = edge_index[0][loop_mask]\n\n    if edge_attr is not None:\n        loop_attr = edge_attr[loop_mask]\n    else:  # A vector of ones:\n        loop_attr = torch.ones(loop_index.numel(), device=edge_index.device)\n\n    num_nodes = maybe_num_nodes(edge_index, num_nodes)\n    full_loop_attr = loop_attr.new_zeros((num_nodes, ) + loop_attr.size()[1:])\n    full_loop_attr[loop_index] = loop_attr\n\n    return full_loop_attr\n```"]}
{"repo": "openai/openai-python", "name": "_get_arch", "language": "python", "path": "src/openai/cli/_tools/migrate.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function determines the architecture of the system on which the script is running and maps it to a standardized format used for identifying compatible binaries.\n2. **Input**: There are no explicit inputs taken by the function as it retrieves the system architecture internally.\n3. **Output**: Returns a string representing the standardized architecture name.\n4. **Procedure**: The function fetches the current system's architecture, converts it to a lower case format, and then maps it to a corresponding standardized name using a predefined dictionary. If the architecture does not exist in the dictionary, the original architecture name is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    \"\"\"Installs the Grit CLI and returns the location of the binary\"\"\"\n    if sys.platform == \"win32\":\n        raise CLIError(\"Windows is not supported yet in the migration CLI\")\n\n    platform = \"macos\" if sys.platform == \"darwin\" else \"linux\"\n\n    dir_name = _cache_dir() / \"openai-python\"\n    install_dir = dir_name / \".install\"\n    target_dir = install_dir / \"bin\"\n\n    target_path = target_dir / \"marzano\"\n    temp_file = target_dir / \"marzano.tmp\"\n\n    if target_path.exists():\n        _debug(f\"{target_path} already exists\")\n        sys.stdout.flush()\n        return target_path\n\n    _debug(f\"Using Grit CLI path: {target_path}\")\n\n    target_dir.mkdir(parents=True, exist_ok=True)\n\n    if temp_file.exists():\n        temp_file.unlink()\n\n    arch = _get_arch()\n    _debug(f\"Using architecture {arch}\")\n\n    file_name = f\"marzano-{platform}-{arch}\"\n    meta_url = f\"https://api.keygen.sh/v1/accounts/{KEYGEN_ACCOUNT}/artifacts/{file_name}\"\n\n    sys.stdout.write(f\"Retrieving Grit CLI metadata from {meta_url}\\n\")\n    with httpx.Client() as client:\n        response = client.get(meta_url)  # pyright: ignore[reportUnknownMemberType]\n\n        data = response.json()\n        errors = data.get(\"errors\")\n        if errors:\n            for error in errors:\n                sys.stdout.write(f\"{error}\\n\")\n\n            raise CLIError(\"Could not locate Grit CLI binary - see above errors\")\n\n        write_manifest(install_dir, data[\"data\"][\"relationships\"][\"release\"][\"data\"][\"id\"])\n\n        link = data[\"data\"][\"links\"][\"redirect\"]\n        _debug(f\"Redirect URL {link}\")\n\n        download_response = client.get(link)  # pyright: ignore[reportUnknownMemberType]\n        with open(temp_file, \"wb\") as file:\n            for chunk in download_response.iter_bytes():\n                file.write(chunk)\n\n    unpacked_dir = target_dir / \"cli-bin\"\n    unpacked_dir.mkdir(parents=True, exist_ok=True)\n\n    with tarfile.open(temp_file, \"r:gz\") as archive:\n        archive.extractall(unpacked_dir, filter=\"data\")\n\n    for item in unpacked_dir.iterdir():\n        item.rename(target_dir / item.name)\n\n    shutil.rmtree(unpacked_dir)\n    os.remove(temp_file)\n    os.chmod(target_path, 0o755)\n\n    sys.stdout.flush()\n\n    return target_path\n\n\n\ndef _get_arch() -> str:\n    architecture = platform.machine().lower()\n\n    # Map the architecture names to Node.js equivalents\n    arch_map = {\n        \"x86_64\": \"x64\",\n        \"amd64\": \"x64\",\n        \"armv7l\": \"arm\",\n        \"aarch64\": \"arm64\",\n    }\n\n    return arch_map.get(architecture, architecture)\n\n\ndef write_manifest(install_path: Path, release: str) -> None:\n    manifest = {\n        \"installPath\": str(install_path),\n        \"binaries\": {\n            \"marzano\": {\n                \"name\": \"marzano\",\n                \"release\": release,\n            },\n        },\n    }\n    manifest_path = Path(install_path) / \"manifests.json\"\n    with open(manifest_path, \"w\") as f:\n        json.dump(manifest, f, indent=2)\n\n# Path: src/openai/cli/_tools/_main.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom argparse import ArgumentParser\n\nfrom . import migrate, fine_tunes\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register_commands(parser: ArgumentParser, subparser: _SubParsersAction[ArgumentParser]) -> None:\n    migrate.register(subparser)\n\n    namespaced = parser.add_subparsers(title=\"Tools\", help=\"Convenience client side tools\")\n\n    fine_tunes.register(namespaced)\n\n# Path: src/openai/cli/_tools/__init__.py\nfrom ._main import register_commands as register_commands\n\n# Path: src/openai/cli/_cli.py\nfrom __future__ import annotations\n\nimport sys\nimport logging\nimport argparse\nfrom typing import Any, List, Type, Optional\nfrom typing_extensions import ClassVar\n\nimport httpx\nimport pydantic\n\nimport openai\n\nfrom . import _tools\nfrom .. import _ApiType, __version__\nfrom ._api import register_commands\nfrom ._utils import can_use_http2\nfrom .._types import ProxiesDict\nfrom ._errors import CLIError, display_error\nfrom .._compat import PYDANTIC_V2, ConfigDict, model_parse\nfrom .._models import BaseModel\nfrom .._exceptions import APIError\n\nlogger = logging.getLogger()\nformatter = logging.Formatter(\"[%(asctime)s] %(message)s\")\nhandler = logging.StreamHandler(sys.stderr)\nhandler.setFormatter(formatter)\nlogger.addHandler(handler)\n\n\nclass Arguments(BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(\n            extra=\"ignore\",\n        )\n    else:\n\n        class Config(pydantic.BaseConfig):  # type: ignore\n            extra: Any = pydantic.Extra.ignore  # type: ignore\n\n    verbosity: int\n    version: Optional[str] = None\n\n    api_key: Optional[str]\n    api_base: Optional[str]\n    organization: Optional[str]\n    proxy: Optional[List[str]]\n    api_type: Optional[_ApiType] = None\n    api_version: Optional[str] = None\n\n    # azure\n    azure_endpoint: Optional[str] = None\n    azure_ad_token: Optional[str] = None\n\n    # internal, set by subparsers to parse their specific args\n    args_model: Optional[Type[BaseModel]] = None\n\n    # internal, used so that subparsers can forward unknown arguments\n    unknown_args: List[str] = []\n    allow_unknown_args: bool = False\n\n\ndef _build_parser() -> argparse.ArgumentParser:\n    parser = argparse.ArgumentParser(description=None, prog=\"openai\")\n    parser.add_argument(\n        \"-v\",\n        \"--verbose\",\n        action=\"count\",\n        dest=\"verbosity\",\n        default=0,\n        help=\"Set verbosity.\",\n    )\n    parser.add_argument(\"-b\", \"--api-base\", help=\"What API base url to use.\")\n    parser.add_argument(\"-k\", \"--api-key\", help=\"What API key to use.\")\n    parser.add_argument(\"-p\", \"--proxy\", nargs=\"+\", help=\"What proxy to use.\")\n    parser.add_argument(\n        \"-o\",\n        \"--organization\",\n        help=\"Which organization to run as (will use your default organization if not specified)\",\n    )\n    parser.add_argument(\n        \"-t\",\n        \"--api-type\",\n        type=str,\n        choices=(\"openai\", \"azure\"),\n        help=\"The backend API to call, must be `openai` or `azure`\",\n    )\n    parser.add_argument(\n        \"--api-version\",\n        help=\"The Azure API version, e.g. 'https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning'\",\n    )\n\n    # azure\n    parser.add_argument(\n        \"--azure-endpoint\",\n        help=\"The Azure endpoint, e.g. 'https://endpoint.openai.azure.com'\",\n    )\n    parser.add_argument(\n        \"--azure-ad-token\",\n        help=\"A token from Azure Active Directory, https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id\",\n    )\n\n    # prints the package version\n    parser.add_argument(\n        \"-V\",\n        \"--version\",\n        action=\"version\",\n        version=\"%(prog)s \" + __version__,\n    )\n\n    def help() -> None:\n        parser.print_help()\n\n    parser.set_defaults(func=help)\n\n    subparsers = parser.add_subparsers()\n    sub_api = subparsers.add_parser(\"api\", help=\"Direct API calls\")\n\n    register_commands(sub_api)\n\n    sub_tools = subparsers.add_parser(\"tools\", help=\"Client side tools for convenience\")\n    _tools.register_commands(sub_tools, subparsers)\n\n    return parser\n\n\ndef main() -> int:\n    try:\n        _main()\n    except (APIError, CLIError, pydantic.ValidationError) as err:\n        display_error(err)\n        return 1\n    except KeyboardInterrupt:\n        sys.stderr.write(\"\\n\")\n        return 1\n    return 0\n\n\ndef _parse_args(parser: argparse.ArgumentParser) -> tuple[argparse.Namespace, Arguments, list[str]]:\n    # argparse by default will strip out the `--` but we want to keep it for unknown arguments\n    if \"--\" in sys.argv:\n        idx = sys.argv.index(\"--\")\n        known_args = sys.argv[1:idx]\n        unknown_args = sys.argv[idx:]\n    else:\n        known_args = sys.argv[1:]\n        unknown_args = []\n\n    parsed, remaining_unknown = parser.parse_known_args(known_args)\n\n    # append any remaining unknown arguments from the initial parsing\n    remaining_unknown.extend(unknown_args)\n\n    args = model_parse(Arguments, vars(parsed))\n    if not args.allow_unknown_args:\n        # we have to parse twice to ensure any unknown arguments\n        # result in an error if that behaviour is desired\n        parser.parse_args()\n\n    return parsed, args, remaining_unknown\n\n\ndef _main() -> None:\n    parser = _build_parser()\n    parsed, args, unknown = _parse_args(parser)\n\n    if args.verbosity != 0:\n        sys.stderr.write(\"Warning: --verbosity isn't supported yet\\n\")\n\n    proxies: ProxiesDict = {}\n    if args.proxy is not None:\n        for proxy in args.proxy:\n            key = \"https://\" if proxy.startswith(\"https\") else \"http://\"\n            if key in proxies:\n                raise CLIError(f\"Multiple {key} proxies given - only the last one would be used\")\n\n            proxies[key] = proxy\n\n    http_client = httpx.Client(\n        proxies=proxies or None,\n        http2=can_use_http2(),\n    )\n    openai.http_client = http_client\n\n    if args.organization:\n        openai.organization = args.organization\n\n    if args.api_key:\n        openai.api_key = args.api_key\n\n    if args.api_base:\n        openai.base_url = args.api_base\n\n    # azure\n    if args.api_type is not None:\n        openai.api_type = args.api_type\n\n    if args.azure_endpoint is not None:\n        openai.azure_endpoint = args.azure_endpoint\n\n    if args.api_version is not None:\n        openai.api_version = args.api_version\n\n    if args.azure_ad_token is not None:\n        openai.azure_ad_token = args.azure_ad_token\n\n    try:\n        if args.args_model:\n            parsed.func(\n                model_parse(\n                    args.args_model,\n                    {\n                        **{\n                            # we omit None values so that they can be defaulted to `NotGiven`\n                            # and we'll strip it from the API request\n                            key: value\n                            for key, value in vars(parsed).items()\n                            if value is not None\n                        },\n                        \"unknown_args\": unknown,\n                    },\n                )\n            )\n        else:\n            parsed.func()\n    finally:\n        try:\n            http_client.close()\n        except Exception:\n            pass\n\n\nif __name__ == \"__main__\":\n    sys.exit(main())\n\n# Path: src/openai/cli/__init__.py\nfrom ._cli import main as main\n\n# Path: src/openai/__main__.py\nfrom .cli import main\n\nmain()\n\n# Path: src/openai/types/beta/thread.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Optional\nfrom typing_extensions import Literal\n\nfrom ..._models import BaseModel\n\n__all__ = [\"Thread\"]\n\n\nclass Thread(BaseModel):\n    id: str\n    \"\"\"The identifier, which can be referenced in API endpoints.\"\"\"\n\n    created_at: int\n    \"\"\"The Unix timestamp (in seconds) for when the thread was created.\"\"\"\n\n    metadata: Optional[object] = None\n    \"\"\"Set of 16 key-value pairs that can be attached to an object.\n\n    This can be useful for storing additional information about the object in a\n    structured format. Keys can be a maximum of 64 characters long and values can be\n    a maxium of 512 characters long.\n    \"\"\"\n\n    object: Literal[\"thread\"]\n    \"\"\"The object type, which is always `thread`.\"\"\"\n\n# Path: src/openai/version.py\nfrom ._version import __version__\n\nVERSION: str = __version__\n\n# Path: src/openai/lib/_old_api.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any\nfrom typing_extensions import override\n\nfrom .._utils import LazyProxy\nfrom .._exceptions import OpenAIError\n\nINSTRUCTIONS = \"\"\"\n\nYou tried to access openai.{symbol}, but this is no longer supported in openai>=1.0.0 - see the README at https://github.com/openai/openai-python for the API.\n\nYou can run `openai migrate` to automatically upgrade your codebase to use the 1.0.0 interface. \n\nAlternatively, you can pin your installation to the old version, e.g. `pip install openai==0.28`\n\nA detailed migration guide is available here: https://github.com/openai/openai-python/discussions/742\n\"\"\"\n\n\nclass APIRemovedInV1(OpenAIError):\n    def __init__(self, *, symbol: str) -> None:\n        super().__init__(INSTRUCTIONS.format(symbol=symbol))\n\n\nclass APIRemovedInV1Proxy(LazyProxy[Any]):\n    def __init__(self, *, symbol: str) -> None:\n        super().__init__()\n        self._symbol = symbol\n\n    @override\n    def __load__(self) -> Any:\n        # return the proxy until it is eventually called so that\n        # we don't break people that are just checking the attributes\n        # of a module\n        return self\n\n    def __call__(self, *_args: Any, **_kwargs: Any) -> Any:\n        raise APIRemovedInV1(symbol=self._symbol)\n\n\nSYMBOLS = [\n    \"Edit\",\n    \"File\",\n    \"Audio\",\n    \"Image\",\n    \"Model\",\n    \"Engine\",\n    \"Customer\",\n    \"FineTune\",\n    \"Embedding\",\n    \"Completion\",\n    \"Deployment\",\n    \"Moderation\",\n    \"ErrorObject\",\n    \"FineTuningJob\",\n    \"ChatCompletion\",\n]\n\n# we explicitly tell type checkers that nothing is exported\n# from this file so that when we re-export the old symbols\n# in `openai/__init__.py` they aren't added to the auto-complete\n# suggestions given by editors\nif TYPE_CHECKING:\n    __all__: list[str] = []\nelse:\n    __all__ = SYMBOLS\n\n\n__locals = locals()\nfor symbol in SYMBOLS:\n    __locals[symbol] = APIRemovedInV1Proxy(symbol=symbol)\n\n# Path: src/openai/lib/_validators.py\n# pyright: basic\nfrom __future__ import annotations\n\nimport os\nimport sys\nfrom typing import Any, TypeVar, Callable, Optional, NamedTuple\nfrom typing_extensions import TypeAlias\n\nfrom .._extras import pandas as pd\n\n\nclass Remediation(NamedTuple):\n    name: str\n    immediate_msg: Optional[str] = None\n    necessary_msg: Optional[str] = None\n    necessary_fn: Optional[Callable[[Any], Any]] = None\n    optional_msg: Optional[str] = None\n    optional_fn: Optional[Callable[[Any], Any]] = None\n    error_msg: Optional[str] = None\n\n\nOptionalDataFrameT = TypeVar(\"OptionalDataFrameT\", bound=\"Optional[pd.DataFrame]\")\n\n\ndef num_examples_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will only print out the number of examples and recommend to the user to increase the number of examples if less than 100.\n    \"\"\"\n    MIN_EXAMPLES = 100\n    optional_suggestion = (\n        \"\"\n        if len(df) >= MIN_EXAMPLES\n        else \". In general, we recommend having at least a few hundred examples. We've found that performance tends to linearly increase for every doubling of the number of examples\"\n    )\n    immediate_msg = f\"\\n- Your file contains {len(df)} prompt-completion pairs{optional_suggestion}\"\n    return Remediation(name=\"num_examples\", immediate_msg=immediate_msg)\n\n\ndef necessary_column_validator(df: pd.DataFrame, necessary_column: str) -> Remediation:\n    \"\"\"\n    This validator will ensure that the necessary column is present in the dataframe.\n    \"\"\"\n\n    def lower_case_column(df: pd.DataFrame, column: Any) -> pd.DataFrame:\n        cols = [c for c in df.columns if str(c).lower() == column]\n        df.rename(columns={cols[0]: column.lower()}, inplace=True)\n        return df\n\n    immediate_msg = None\n    necessary_fn = None\n    necessary_msg = None\n    error_msg = None\n\n    if necessary_column not in df.columns:\n        if necessary_column in [str(c).lower() for c in df.columns]:\n\n            def lower_case_column_creator(df: pd.DataFrame) -> pd.DataFrame:\n                return lower_case_column(df, necessary_column)\n\n            necessary_fn = lower_case_column_creator\n            immediate_msg = f\"\\n- The `{necessary_column}` column/key should be lowercase\"\n            necessary_msg = f\"Lower case column name to `{necessary_column}`\"\n        else:\n            error_msg = f\"`{necessary_column}` column/key is missing. Please make sure you name your columns/keys appropriately, then retry\"\n\n    return Remediation(\n        name=\"necessary_column\",\n        immediate_msg=immediate_msg,\n        necessary_msg=necessary_msg,\n        necessary_fn=necessary_fn,\n        error_msg=error_msg,\n    )\n\n\ndef additional_column_validator(df: pd.DataFrame, fields: list[str] = [\"prompt\", \"completion\"]) -> Remediation:\n    \"\"\"\n    This validator will remove additional columns from the dataframe.\n    \"\"\"\n    additional_columns = []\n    necessary_msg = None\n    immediate_msg = None\n    necessary_fn = None  # type: ignore\n\n    if len(df.columns) > 2:\n        additional_columns = [c for c in df.columns if c not in fields]\n        warn_message = \"\"\n        for ac in additional_columns:\n            dups = [c for c in additional_columns if ac in c]\n            if len(dups) > 0:\n                warn_message += f\"\\n  WARNING: Some of the additional columns/keys contain `{ac}` in their name. These will be ignored, and the column/key `{ac}` will be used instead. This could also result from a duplicate column/key in the provided file.\"\n        immediate_msg = f\"\\n- The input file should contain exactly two columns/keys per row. Additional columns/keys present are: {additional_columns}{warn_message}\"\n        necessary_msg = f\"Remove additional columns/keys: {additional_columns}\"\n\n        def necessary_fn(x: Any) -> Any:\n            return x[fields]\n\n    return Remediation(\n        name=\"additional_column\",\n        immediate_msg=immediate_msg,\n        necessary_msg=necessary_msg,\n        necessary_fn=necessary_fn,\n    )\n\n\ndef non_empty_field_validator(df: pd.DataFrame, field: str = \"completion\") -> Remediation:\n    \"\"\"\n    This validator will ensure that no completion is empty.\n    \"\"\"\n    necessary_msg = None\n    necessary_fn = None  # type: ignore\n    immediate_msg = None\n\n    if df[field].apply(lambda x: x == \"\").any() or df[field].isnull().any():\n        empty_rows = (df[field] == \"\") | (df[field].isnull())\n        empty_indexes = df.reset_index().index[empty_rows].tolist()\n        immediate_msg = f\"\\n- `{field}` column/key should not contain empty strings. These are rows: {empty_indexes}\"\n\n        def necessary_fn(x: Any) -> Any:\n            return x[x[field] != \"\"].dropna(subset=[field])\n\n        necessary_msg = f\"Remove {len(empty_indexes)} rows with empty {field}s\"\n\n    return Remediation(\n        name=f\"empty_{field}\",\n        immediate_msg=immediate_msg,\n        necessary_msg=necessary_msg,\n        necessary_fn=necessary_fn,\n    )\n\n\ndef duplicated_rows_validator(df: pd.DataFrame, fields: list[str] = [\"prompt\", \"completion\"]) -> Remediation:\n    \"\"\"\n    This validator will suggest to the user to remove duplicate rows if they exist.\n    \"\"\"\n    duplicated_rows = df.duplicated(subset=fields)\n    duplicated_indexes = df.reset_index().index[duplicated_rows].tolist()\n    immediate_msg = None\n    optional_msg = None\n    optional_fn = None  # type: ignore\n\n    if len(duplicated_indexes) > 0:\n        immediate_msg = f\"\\n- There are {len(duplicated_indexes)} duplicated {'-'.join(fields)} sets. These are rows: {duplicated_indexes}\"\n        optional_msg = f\"Remove {len(duplicated_indexes)} duplicate rows\"\n\n        def optional_fn(x: Any) -> Any:\n            return x.drop_duplicates(subset=fields)\n\n    return Remediation(\n        name=\"duplicated_rows\",\n        immediate_msg=immediate_msg,\n        optional_msg=optional_msg,\n        optional_fn=optional_fn,\n    )\n\n\ndef long_examples_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will suggest to the user to remove examples that are too long.\n    \"\"\"\n    immediate_msg = None\n    optional_msg = None\n    optional_fn = None  # type: ignore\n\n    ft_type = infer_task_type(df)\n    if ft_type != \"open-ended generation\":\n\n        def get_long_indexes(d: pd.DataFrame) -> Any:\n            long_examples = d.apply(lambda x: len(x.prompt) + len(x.completion) > 10000, axis=1)\n            return d.reset_index().index[long_examples].tolist()\n\n        long_indexes = get_long_indexes(df)\n\n        if len(long_indexes) > 0:\n            immediate_msg = f\"\\n- There are {len(long_indexes)} examples that are very long. These are rows: {long_indexes}\\nFor conditional generation, and for classification the examples shouldn't be longer than 2048 tokens.\"\n            optional_msg = f\"Remove {len(long_indexes)} long examples\"\n\n            def optional_fn(x: Any) -> Any:\n                long_indexes_to_drop = get_long_indexes(x)\n                if long_indexes != long_indexes_to_drop:\n                    sys.stdout.write(\n                        f\"The indices of the long examples has changed as a result of a previously applied recommendation.\\nThe {len(long_indexes_to_drop)} long examples to be dropped are now at the following indices: {long_indexes_to_drop}\\n\"\n                    )\n                return x.drop(long_indexes_to_drop)\n\n    return Remediation(\n        name=\"long_examples\",\n        immediate_msg=immediate_msg,\n        optional_msg=optional_msg,\n        optional_fn=optional_fn,\n    )\n\n\ndef common_prompt_suffix_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will suggest to add a common suffix to the prompt if one doesn't already exist in case of classification or conditional generation.\n    \"\"\"\n    error_msg = None\n    immediate_msg = None\n    optional_msg = None\n    optional_fn = None  # type: ignore\n\n    # Find a suffix which is not contained within the prompt otherwise\n    suggested_suffix = \"\\n\\n### =>\\n\\n\"\n    suffix_options = [\n        \" ->\",\n        \"\\n\\n###\\n\\n\",\n        \"\\n\\n===\\n\\n\",\n        \"\\n\\n---\\n\\n\",\n        \"\\n\\n===>\\n\\n\",\n        \"\\n\\n--->\\n\\n\",\n    ]\n    for suffix_option in suffix_options:\n        if suffix_option == \" ->\":\n            if df.prompt.str.contains(\"\\n\").any():\n                continue\n        if df.prompt.str.contains(suffix_option, regex=False).any():\n            continue\n        suggested_suffix = suffix_option\n        break\n    display_suggested_suffix = suggested_suffix.replace(\"\\n\", \"\\\\n\")\n\n    ft_type = infer_task_type(df)\n    if ft_type == \"open-ended generation\":\n        return Remediation(name=\"common_suffix\")\n\n    def add_suffix(x: Any, suffix: Any) -> Any:\n        x[\"prompt\"] += suffix\n        return x\n\n    common_suffix = get_common_xfix(df.prompt, xfix=\"suffix\")\n    if (df.prompt == common_suffix).all():\n        error_msg = f\"All prompts are identical: `{common_suffix}`\\nConsider leaving the prompts blank if you want to do open-ended generation, otherwise ensure prompts are different\"\n        return Remediation(name=\"common_suffix\", error_msg=error_msg)\n\n    if common_suffix != \"\":\n        common_suffix_new_line_handled = common_suffix.replace(\"\\n\", \"\\\\n\")\n        immediate_msg = f\"\\n- All prompts end with suffix `{common_suffix_new_line_handled}`\"\n        if len(common_suffix) > 10:\n            immediate_msg += f\". This suffix seems very long. Consider replacing with a shorter suffix, such as `{display_suggested_suffix}`\"\n        if df.prompt.str[: -len(common_suffix)].str.contains(common_suffix, regex=False).any():\n            immediate_msg += f\"\\n  WARNING: Some of your prompts contain the suffix `{common_suffix}` more than once. We strongly suggest that you review your prompts and add a unique suffix\"\n\n    else:\n        immediate_msg = \"\\n- Your data does not contain a common separator at the end of your prompts. Having a separator string appended to the end of the prompt makes it clearer to the fine-tuned model where the completion should begin. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples. If you intend to do open-ended generation, then you should leave the prompts empty\"\n\n    if common_suffix == \"\":\n        optional_msg = f\"Add a suffix separator `{display_suggested_suffix}` to all prompts\"\n\n        def optional_fn(x: Any) -> Any:\n            return add_suffix(x, suggested_suffix)\n\n    return Remediation(\n        name=\"common_completion_suffix\",\n        immediate_msg=immediate_msg,\n        optional_msg=optional_msg,\n        optional_fn=optional_fn,\n        error_msg=error_msg,\n    )\n\n\ndef common_prompt_prefix_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will suggest to remove a common prefix from the prompt if a long one exist.\n    \"\"\"\n    MAX_PREFIX_LEN = 12\n\n    immediate_msg = None\n    optional_msg = None\n    optional_fn = None  # type: ignore\n\n    common_prefix = get_common_xfix(df.prompt, xfix=\"prefix\")\n    if common_prefix == \"\":\n        return Remediation(name=\"common_prefix\")\n\n    def remove_common_prefix(x: Any, prefix: Any) -> Any:\n        x[\"prompt\"] = x[\"prompt\"].str[len(prefix) :]\n        return x\n\n    if (df.prompt == common_prefix).all():\n        # already handled by common_suffix_validator\n        return Remediation(name=\"common_prefix\")\n\n    if common_prefix != \"\":\n        immediate_msg = f\"\\n- All prompts start with prefix `{common_prefix}`\"\n        if MAX_PREFIX_LEN < len(common_prefix):\n            immediate_msg += \". Fine-tuning doesn't require the instruction specifying the task, or a few-shot example scenario. Most of the time you should only add the input data into the prompt, and the desired output into the completion\"\n            optional_msg = f\"Remove prefix `{common_prefix}` from all prompts\"\n\n            def optional_fn(x: Any) -> Any:\n                return remove_common_prefix(x, common_prefix)\n\n    return Remediation(\n        name=\"common_prompt_prefix\",\n        immediate_msg=immediate_msg,\n        optional_msg=optional_msg,\n        optional_fn=optional_fn,\n    )\n\n\ndef common_completion_prefix_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will suggest to remove a common prefix from the completion if a long one exist.\n    \"\"\"\n    MAX_PREFIX_LEN = 5\n\n    common_prefix = get_common_xfix(df.completion, xfix=\"prefix\")\n    ws_prefix = len(common_prefix) > 0 and common_prefix[0] == \" \"\n    if len(common_prefix) < MAX_PREFIX_LEN:\n        return Remediation(name=\"common_prefix\")\n\n    def remove_common_prefix(x: Any, prefix: Any, ws_prefix: Any) -> Any:\n        x[\"completion\"] = x[\"completion\"].str[len(prefix) :]\n        if ws_prefix:\n            # keep the single whitespace as prefix\n            x[\"completion\"] = f\" {x['completion']}\"\n        return x\n\n    if (df.completion == common_prefix).all():\n        # already handled by common_suffix_validator\n        return Remediation(name=\"common_prefix\")\n\n    immediate_msg = f\"\\n- All completions start with prefix `{common_prefix}`. Most of the time you should only add the output data into the completion, without any prefix\"\n    optional_msg = f\"Remove prefix `{common_prefix}` from all completions\"\n\n    def optional_fn(x: Any) -> Any:\n        return remove_common_prefix(x, common_prefix, ws_prefix)\n\n    return Remediation(\n        name=\"common_completion_prefix\",\n        immediate_msg=immediate_msg,\n        optional_msg=optional_msg,\n        optional_fn=optional_fn,\n    )\n\n\ndef common_completion_suffix_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n    \"\"\"\n    error_msg = None\n    immediate_msg = None\n    optional_msg = None\n    optional_fn = None  # type: ignore\n\n    ft_type = infer_task_type(df)\n    if ft_type == \"open-ended generation\" or ft_type == \"classification\":\n        return Remediation(name=\"common_suffix\")\n\n    common_suffix = get_common_xfix(df.completion, xfix=\"suffix\")\n    if (df.completion == common_suffix).all():\n        error_msg = f\"All completions are identical: `{common_suffix}`\\nEnsure completions are different, otherwise the model will just repeat `{common_suffix}`\"\n        return Remediation(name=\"common_suffix\", error_msg=error_msg)\n\n    # Find a suffix which is not contained within the completion otherwise\n    suggested_suffix = \" [END]\"\n    suffix_options = [\n        \"\\n\",\n        \".\",\n        \" END\",\n        \"***\",\n        \"+++\",\n        \"&&&\",\n        \"$$$\",\n        \"@@@\",\n        \"%%%\",\n    ]\n    for suffix_option in suffix_options:\n        if df.completion.str.contains(suffix_option, regex=False).any():\n            continue\n        suggested_suffix = suffix_option\n        break\n    display_suggested_suffix = suggested_suffix.replace(\"\\n\", \"\\\\n\")\n\n    def add_suffix(x: Any, suffix: Any) -> Any:\n        x[\"completion\"] += suffix\n        return x\n\n    if common_suffix != \"\":\n        common_suffix_new_line_handled = common_suffix.replace(\"\\n\", \"\\\\n\")\n        immediate_msg = f\"\\n- All completions end with suffix `{common_suffix_new_line_handled}`\"\n        if len(common_suffix) > 10:\n            immediate_msg += f\". This suffix seems very long. Consider replacing with a shorter suffix, such as `{display_suggested_suffix}`\"\n        if df.completion.str[: -len(common_suffix)].str.contains(common_suffix, regex=False).any():\n            immediate_msg += f\"\\n  WARNING: Some of your completions contain the suffix `{common_suffix}` more than once. We suggest that you review your completions and add a unique ending\"\n\n    else:\n        immediate_msg = \"\\n- Your data does not contain a common ending at the end of your completions. Having a common ending string appended to the end of the completion makes it clearer to the fine-tuned model where the completion should end. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more detail and examples.\"\n\n    if common_suffix == \"\":\n        optional_msg = f\"Add a suffix ending `{display_suggested_suffix}` to all completions\"\n\n        def optional_fn(x: Any) -> Any:\n            return add_suffix(x, suggested_suffix)\n\n    return Remediation(\n        name=\"common_completion_suffix\",\n        immediate_msg=immediate_msg,\n        optional_msg=optional_msg,\n        optional_fn=optional_fn,\n        error_msg=error_msg,\n    )\n\n\ndef completions_space_start_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will suggest to add a space at the start of the completion if it doesn't already exist. This helps with tokenization.\n    \"\"\"\n\n    def add_space_start(x: Any) -> Any:\n        x[\"completion\"] = x[\"completion\"].apply(lambda s: (\"\" if s.startswith(\" \") else \" \") + s)\n        return x\n\n    optional_msg = None\n    optional_fn = None\n    immediate_msg = None\n\n    if df.completion.str[:1].nunique() != 1 or df.completion.values[0][0] != \" \":\n        immediate_msg = \"\\n- The completion should start with a whitespace character (` `). This tends to produce better results due to the tokenization we use. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details\"\n        optional_msg = \"Add a whitespace character to the beginning of the completion\"\n        optional_fn = add_space_start\n    return Remediation(\n        name=\"completion_space_start\",\n        immediate_msg=immediate_msg,\n        optional_msg=optional_msg,\n        optional_fn=optional_fn,\n    )\n\n\ndef lower_case_validator(df: pd.DataFrame, column: Any) -> Remediation | None:\n    \"\"\"\n    This validator will suggest to lowercase the column values, if more than a third of letters are uppercase.\n    \"\"\"\n\n    def lower_case(x: Any) -> Any:\n        x[column] = x[column].str.lower()\n        return x\n\n    count_upper = df[column].apply(lambda x: sum(1 for c in x if c.isalpha() and c.isupper())).sum()\n    count_lower = df[column].apply(lambda x: sum(1 for c in x if c.isalpha() and c.islower())).sum()\n\n    if count_upper * 2 > count_lower:\n        return Remediation(\n            name=\"lower_case\",\n            immediate_msg=f\"\\n- More than a third of your `{column}` column/key is uppercase. Uppercase {column}s tends to perform worse than a mixture of case encountered in normal language. We recommend to lower case the data if that makes sense in your domain. See https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset for more details\",\n            optional_msg=f\"Lowercase all your data in column/key `{column}`\",\n            optional_fn=lower_case,\n        )\n    return None\n\n\ndef read_any_format(\n    fname: str, fields: list[str] = [\"prompt\", \"completion\"]\n) -> tuple[pd.DataFrame | None, Remediation]:\n    \"\"\"\n    This function will read a file saved in .csv, .json, .txt, .xlsx or .tsv format using pandas.\n     - for .xlsx it will read the first sheet\n     - for .txt it will assume completions and split on newline\n    \"\"\"\n    remediation = None\n    necessary_msg = None\n    immediate_msg = None\n    error_msg = None\n    df = None\n\n    if os.path.isfile(fname):\n        try:\n            if fname.lower().endswith(\".csv\") or fname.lower().endswith(\".tsv\"):\n                file_extension_str, separator = (\"CSV\", \",\") if fname.lower().endswith(\".csv\") else (\"TSV\", \"\\t\")\n                immediate_msg = (\n                    f\"\\n- Based on your file extension, your file is formatted as a {file_extension_str} file\"\n                )\n                necessary_msg = f\"Your format `{file_extension_str}` will be converted to `JSONL`\"\n                df = pd.read_csv(fname, sep=separator, dtype=str).fillna(\"\")\n            elif fname.lower().endswith(\".xlsx\"):\n                immediate_msg = \"\\n- Based on your file extension, your file is formatted as an Excel file\"\n                necessary_msg = \"Your format `XLSX` will be converted to `JSONL`\"\n                xls = pd.ExcelFile(fname)\n                sheets = xls.sheet_names\n                if len(sheets) > 1:\n                    immediate_msg += \"\\n- Your Excel file contains more than one sheet. Please either save as csv or ensure all data is present in the first sheet. WARNING: Reading only the first sheet...\"\n                df = pd.read_excel(fname, dtype=str).fillna(\"\")\n            elif fname.lower().endswith(\".txt\"):\n                immediate_msg = \"\\n- Based on your file extension, you provided a text file\"\n                necessary_msg = \"Your format `TXT` will be converted to `JSONL`\"\n                with open(fname, \"r\") as f:\n                    content = f.read()\n                    df = pd.DataFrame(\n                        [[\"\", line] for line in content.split(\"\\n\")],\n                        columns=fields,\n                        dtype=str,\n                    ).fillna(\"\")\n            elif fname.lower().endswith(\".jsonl\"):\n                df = pd.read_json(fname, lines=True, dtype=str).fillna(\"\")  # type: ignore\n                if len(df) == 1:  # type: ignore\n                    # this is NOT what we expect for a .jsonl file\n                    immediate_msg = \"\\n- Your JSONL file appears to be in a JSON format. Your file will be converted to JSONL format\"\n                    necessary_msg = \"Your format `JSON` will be converted to `JSONL`\"\n                    df = pd.read_json(fname, dtype=str).fillna(\"\")  # type: ignore\n                else:\n                    pass  # this is what we expect for a .jsonl file\n            elif fname.lower().endswith(\".json\"):\n                try:\n                    # to handle case where .json file is actually a .jsonl file\n                    df = pd.read_json(fname, lines=True, dtype=str).fillna(\"\")  # type: ignore\n                    if len(df) == 1:  # type: ignore\n                        # this code path corresponds to a .json file that has one line\n                        df = pd.read_json(fname, dtype=str).fillna(\"\")  # type: ignore\n                    else:\n                        # this is NOT what we expect for a .json file\n                        immediate_msg = \"\\n- Your JSON file appears to be in a JSONL format. Your file will be converted to JSONL format\"\n                        necessary_msg = \"Your format `JSON` will be converted to `JSONL`\"\n                except ValueError:\n                    # this code path corresponds to a .json file that has multiple lines (i.e. it is indented)\n                    df = pd.read_json(fname, dtype=str).fillna(\"\")  # type: ignore\n            else:\n                error_msg = (\n                    \"Your file must have one of the following extensions: .CSV, .TSV, .XLSX, .TXT, .JSON or .JSONL\"\n                )\n                if \".\" in fname:\n                    error_msg += f\" Your file `{fname}` ends with the extension `.{fname.split('.')[-1]}` which is not supported.\"\n                else:\n                    error_msg += f\" Your file `{fname}` is missing a file extension.\"\n\n        except (ValueError, TypeError):\n            file_extension_str = fname.split(\".\")[-1].upper()\n            error_msg = f\"Your file `{fname}` does not appear to be in valid {file_extension_str} format. Please ensure your file is formatted as a valid {file_extension_str} file.\"\n\n    else:\n        error_msg = f\"File {fname} does not exist.\"\n\n    remediation = Remediation(\n        name=\"read_any_format\",\n        necessary_msg=necessary_msg,\n        immediate_msg=immediate_msg,\n        error_msg=error_msg,\n    )\n    return df, remediation\n\n\ndef format_inferrer_validator(df: pd.DataFrame) -> Remediation:\n    \"\"\"\n    This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n    It will also suggest to use ada and explain train/validation split benefits.\n    \"\"\"\n    ft_type = infer_task_type(df)\n    immediate_msg = None\n    if ft_type == \"classification\":\n        immediate_msg = f\"\\n- Based on your data it seems like you're trying to fine-tune a model for {ft_type}\\n- For classification, we recommend you try one of the faster and cheaper models, such as `ada`\\n- For classification, you can estimate the expected model performance by keeping a held out dataset, which is not used for training\"\n    return Remediation(name=\"num_examples\", immediate_msg=immediate_msg)\n\n\ndef apply_necessary_remediation(df: OptionalDataFrameT, remediation: Remediation) -> OptionalDataFrameT:\n    \"\"\"\n    This function will apply a necessary remediation to a dataframe, or print an error message if one exists.\n    \"\"\"\n    if remediation.error_msg is not None:\n        sys.stderr.write(f\"\\n\\nERROR in {remediation.name} validator: {remediation.error_msg}\\n\\nAborting...\")\n        sys.exit(1)\n    if remediation.immediate_msg is not None:\n        sys.stdout.write(remediation.immediate_msg)\n    if remediation.necessary_fn is not None:\n        df = remediation.necessary_fn(df)\n    return df\n\n\ndef accept_suggestion(input_text: str, auto_accept: bool) -> bool:\n    sys.stdout.write(input_text)\n    if auto_accept:\n        sys.stdout.write(\"Y\\n\")\n        return True\n    return input().lower() != \"n\"\n\n\ndef apply_optional_remediation(\n    df: pd.DataFrame, remediation: Remediation, auto_accept: bool\n) -> tuple[pd.DataFrame, bool]:\n    \"\"\"\n    This function will apply an optional remediation to a dataframe, based on the user input.\n    \"\"\"\n    optional_applied = False\n    input_text = f\"- [Recommended] {remediation.optional_msg} [Y/n]: \"\n    if remediation.optional_msg is not None:\n        if accept_suggestion(input_text, auto_accept):\n            assert remediation.optional_fn is not None\n            df = remediation.optional_fn(df)\n            optional_applied = True\n    if remediation.necessary_msg is not None:\n        sys.stdout.write(f\"- [Necessary] {remediation.necessary_msg}\\n\")\n    return df, optional_applied\n\n\ndef estimate_fine_tuning_time(df: pd.DataFrame) -> None:\n    \"\"\"\n    Estimate the time it'll take to fine-tune the dataset\n    \"\"\"\n    ft_format = infer_task_type(df)\n    expected_time = 1.0\n    if ft_format == \"classification\":\n        num_examples = len(df)\n        expected_time = num_examples * 1.44\n    else:\n        size = df.memory_usage(index=True).sum()\n        expected_time = size * 0.0515\n\n    def format_time(time: float) -> str:\n        if time < 60:\n            return f\"{round(time, 2)} seconds\"\n        elif time < 3600:\n            return f\"{round(time / 60, 2)} minutes\"\n        elif time < 86400:\n            return f\"{round(time / 3600, 2)} hours\"\n        else:\n            return f\"{round(time / 86400, 2)} days\"\n\n    time_string = format_time(expected_time + 140)\n    sys.stdout.write(\n        f\"Once your model starts training, it'll approximately take {time_string} to train a `curie` model, and less for `ada` and `babbage`. Queue will approximately take half an hour per job ahead of you.\\n\"\n    )\n\n\ndef get_outfnames(fname: str, split: bool) -> list[str]:\n    suffixes = [\"_train\", \"_valid\"] if split else [\"\"]\n    i = 0\n    while True:\n        index_suffix = f\" ({i})\" if i > 0 else \"\"\n        candidate_fnames = [f\"{os.path.splitext(fname)[0]}_prepared{suffix}{index_suffix}.jsonl\" for suffix in suffixes]\n        if not any(os.path.isfile(f) for f in candidate_fnames):\n            return candidate_fnames\n        i += 1\n\n\ndef get_classification_hyperparams(df: pd.DataFrame) -> tuple[int, object]:\n    n_classes = df.completion.nunique()\n    pos_class = None\n    if n_classes == 2:\n        pos_class = df.completion.value_counts().index[0]\n    return n_classes, pos_class\n\n\ndef write_out_file(df: pd.DataFrame, fname: str, any_remediations: bool, auto_accept: bool) -> None:\n    \"\"\"\n    This function will write out a dataframe to a file, if the user would like to proceed, and also offer a fine-tuning command with the newly created file.\n    For classification it will optionally ask the user if they would like to split the data into train/valid files, and modify the suggested command to include the valid set.\n    \"\"\"\n    ft_format = infer_task_type(df)\n    common_prompt_suffix = get_common_xfix(df.prompt, xfix=\"suffix\")\n    common_completion_suffix = get_common_xfix(df.completion, xfix=\"suffix\")\n\n    split = False\n    input_text = \"- [Recommended] Would you like to split into training and validation set? [Y/n]: \"\n    if ft_format == \"classification\":\n        if accept_suggestion(input_text, auto_accept):\n            split = True\n\n    additional_params = \"\"\n    common_prompt_suffix_new_line_handled = common_prompt_suffix.replace(\"\\n\", \"\\\\n\")\n    common_completion_suffix_new_line_handled = common_completion_suffix.replace(\"\\n\", \"\\\\n\")\n    optional_ending_string = (\n        f' Make sure to include `stop=[\"{common_completion_suffix_new_line_handled}\"]` so that the generated texts ends at the expected place.'\n        if len(common_completion_suffix_new_line_handled) > 0\n        else \"\"\n    )\n\n    input_text = \"\\n\\nYour data will be written to a new JSONL file. Proceed [Y/n]: \"\n\n    if not any_remediations and not split:\n        sys.stdout.write(\n            f'\\nYou can use your file for fine-tuning:\\n> openai api fine_tunes.create -t \"{fname}\"{additional_params}\\n\\nAfter you\u2019ve fine-tuned a model, remember that your prompt has to end with the indicator string `{common_prompt_suffix_new_line_handled}` for the model to start generating completions, rather than continuing with the prompt.{optional_ending_string}\\n'\n        )\n        estimate_fine_tuning_time(df)\n\n    elif accept_suggestion(input_text, auto_accept):\n        fnames = get_outfnames(fname, split)\n        if split:\n            assert len(fnames) == 2 and \"train\" in fnames[0] and \"valid\" in fnames[1]\n            MAX_VALID_EXAMPLES = 1000\n            n_train = max(len(df) - MAX_VALID_EXAMPLES, int(len(df) * 0.8))\n            df_train = df.sample(n=n_train, random_state=42)\n            df_valid = df.drop(df_train.index)\n            df_train[[\"prompt\", \"completion\"]].to_json(  # type: ignore\n                fnames[0], lines=True, orient=\"records\", force_ascii=False\n            )\n            df_valid[[\"prompt\", \"completion\"]].to_json(fnames[1], lines=True, orient=\"records\", force_ascii=False)\n\n            n_classes, pos_class = get_classification_hyperparams(df)\n            additional_params += \" --compute_classification_metrics\"\n            if n_classes == 2:\n                additional_params += f' --classification_positive_class \"{pos_class}\"'\n            else:\n                additional_params += f\" --classification_n_classes {n_classes}\"\n        else:\n            assert len(fnames) == 1\n            df[[\"prompt\", \"completion\"]].to_json(fnames[0], lines=True, orient=\"records\", force_ascii=False)\n\n        # Add -v VALID_FILE if we split the file into train / valid\n        files_string = (\"s\" if split else \"\") + \" to `\" + (\"` and `\".join(fnames))\n        valid_string = f' -v \"{fnames[1]}\"' if split else \"\"\n        separator_reminder = (\n            \"\"\n            if len(common_prompt_suffix_new_line_handled) == 0\n            else f\"After you\u2019ve fine-tuned a model, remember that your prompt has to end with the indicator string `{common_prompt_suffix_new_line_handled}` for the model to start generating completions, rather than continuing with the prompt.\"\n        )\n        sys.stdout.write(\n            f'\\nWrote modified file{files_string}`\\nFeel free to take a look!\\n\\nNow use that file when fine-tuning:\\n> openai api fine_tunes.create -t \"{fnames[0]}\"{valid_string}{additional_params}\\n\\n{separator_reminder}{optional_ending_string}\\n'\n        )\n        estimate_fine_tuning_time(df)\n    else:\n        sys.stdout.write(\"Aborting... did not write the file\\n\")\n\n\ndef infer_task_type(df: pd.DataFrame) -> str:\n    \"\"\"\n    Infer the likely fine-tuning task type from the data\n    \"\"\"\n    CLASSIFICATION_THRESHOLD = 3  # min_average instances of each class\n    if sum(df.prompt.str.len()) == 0:\n        return \"open-ended generation\"\n\n    if len(df.completion.unique()) < len(df) / CLASSIFICATION_THRESHOLD:\n        return \"classification\"\n\n    return \"conditional generation\"\n\n\ndef get_common_xfix(series: Any, xfix: str = \"suffix\") -> str:\n    \"\"\"\n    Finds the longest common suffix or prefix of all the values in a series\n    \"\"\"\n    common_xfix = \"\"\n    while True:\n        common_xfixes = (\n            series.str[-(len(common_xfix) + 1) :] if xfix == \"suffix\" else series.str[: len(common_xfix) + 1]\n        )  # first few or last few characters\n        if common_xfixes.nunique() != 1:  # we found the character at which we don't have a unique xfix anymore\n            break\n        elif common_xfix == common_xfixes.values[0]:  # the entire first row is a prefix of every other row\n            break\n        else:  # the first or last few characters are still common across all rows - let's try to add one more\n            common_xfix = common_xfixes.values[0]\n    return common_xfix\n\n\nValidator: TypeAlias = \"Callable[[pd.DataFrame], Remediation | None]\"\n\n\ndef get_validators() -> list[Validator]:\n    return [\n        num_examples_validator,\n        lambda x: necessary_column_validator(x, \"prompt\"),\n        lambda x: necessary_column_validator(x, \"completion\"),\n        additional_column_validator,\n        non_empty_field_validator,\n        format_inferrer_validator,\n        duplicated_rows_validator,\n        long_examples_validator,\n        lambda x: lower_case_validator(x, \"prompt\"),\n        lambda x: lower_case_validator(x, \"completion\"),\n        common_prompt_suffix_validator,\n        common_prompt_prefix_validator,\n        common_completion_prefix_validator,\n        common_completion_suffix_validator,\n        completions_space_start_validator,\n    ]\n\n\ndef apply_validators(\n    df: pd.DataFrame,\n    fname: str,\n    remediation: Remediation | None,\n    validators: list[Validator],\n    auto_accept: bool,\n    write_out_file_func: Callable[..., Any],\n) -> None:\n    optional_remediations: list[Remediation] = []\n    if remediation is not None:\n        optional_remediations.append(remediation)\n    for validator in validators:\n        remediation = validator(df)\n        if remediation is not None:\n            optional_remediations.append(remediation)\n            df = apply_necessary_remediation(df, remediation)\n\n    any_optional_or_necessary_remediations = any(\n        [\n            remediation\n            for remediation in optional_remediations\n            if remediation.optional_msg is not None or remediation.necessary_msg is not None\n        ]\n    )\n    any_necessary_applied = any(\n        [remediation for remediation in optional_remediations if remediation.necessary_msg is not None]\n    )\n    any_optional_applied = False\n\n    if any_optional_or_necessary_remediations:\n        sys.stdout.write(\"\\n\\nBased on the analysis we will perform the following actions:\\n\")\n        for remediation in optional_remediations:\n            df, optional_applied = apply_optional_remediation(df, remediation, auto_accept)\n            any_optional_applied = any_optional_applied or optional_applied\n    else:\n        sys.stdout.write(\"\\n\\nNo remediations found.\\n\")\n\n    any_optional_or_necessary_applied = any_optional_applied or any_necessary_applied\n\n    write_out_file_func(df, fname, any_optional_or_necessary_applied, auto_accept)\n\n# Path: src/openai/lib/azure.py\nfrom __future__ import annotations\n\nimport os\nimport inspect\nfrom typing import Any, Union, Mapping, TypeVar, Callable, Awaitable, overload\nfrom typing_extensions import Self, override\n\nimport httpx\n\nfrom .._types import NOT_GIVEN, Omit, Timeout, NotGiven\nfrom .._utils import is_given, is_mapping\nfrom .._client import OpenAI, AsyncOpenAI\nfrom .._models import FinalRequestOptions\nfrom .._streaming import Stream, AsyncStream\nfrom .._exceptions import OpenAIError\nfrom .._base_client import DEFAULT_MAX_RETRIES, BaseClient\n\n_deployments_endpoints = set(\n    [\n        \"/completions\",\n        \"/chat/completions\",\n        \"/embeddings\",\n        \"/audio/transcriptions\",\n        \"/audio/translations\",\n        \"/audio/speech\",\n        \"/images/generations\",\n    ]\n)\n\n\nAzureADTokenProvider = Callable[[], str]\nAsyncAzureADTokenProvider = Callable[[], \"str | Awaitable[str]\"]\n_HttpxClientT = TypeVar(\"_HttpxClientT\", bound=Union[httpx.Client, httpx.AsyncClient])\n_DefaultStreamT = TypeVar(\"_DefaultStreamT\", bound=Union[Stream[Any], AsyncStream[Any]])\n\n\n# we need to use a sentinel API key value for Azure AD\n# as we don't want to make the `api_key` in the main client Optional\n# and Azure AD tokens may be retrieved on a per-request basis\nAPI_KEY_SENTINEL = \"\".join([\"<\", \"missing API key\", \">\"])\n\n\nclass MutuallyExclusiveAuthError(OpenAIError):\n    def __init__(self) -> None:\n        super().__init__(\n            \"The `api_key`, `azure_ad_token` and `azure_ad_token_provider` arguments are mutually exclusive; Only one can be passed at a time\"\n        )\n\n\nclass BaseAzureClient(BaseClient[_HttpxClientT, _DefaultStreamT]):\n    @override\n    def _build_request(\n        self,\n        options: FinalRequestOptions,\n    ) -> httpx.Request:\n        if options.url in _deployments_endpoints and is_mapping(options.json_data):\n            model = options.json_data.get(\"model\")\n            if model is not None and not \"/deployments\" in str(self.base_url):\n                options.url = f\"/deployments/{model}{options.url}\"\n\n        return super()._build_request(options)\n\n\nclass AzureOpenAI(BaseAzureClient[httpx.Client, Stream[Any]], OpenAI):\n    @overload\n    def __init__(\n        self,\n        *,\n        azure_endpoint: str,\n        azure_deployment: str | None = None,\n        api_version: str | None = None,\n        api_key: str | None = None,\n        azure_ad_token: str | None = None,\n        azure_ad_token_provider: AzureADTokenProvider | None = None,\n        organization: str | None = None,\n        timeout: float | Timeout | None | NotGiven = NOT_GIVEN,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        default_headers: Mapping[str, str] | None = None,\n        default_query: Mapping[str, object] | None = None,\n        http_client: httpx.Client | None = None,\n        _strict_response_validation: bool = False,\n    ) -> None:\n        ...\n\n    @overload\n    def __init__(\n        self,\n        *,\n        azure_deployment: str | None = None,\n        api_version: str | None = None,\n        api_key: str | None = None,\n        azure_ad_token: str | None = None,\n        azure_ad_token_provider: AzureADTokenProvider | None = None,\n        organization: str | None = None,\n        timeout: float | Timeout | None | NotGiven = NOT_GIVEN,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        default_headers: Mapping[str, str] | None = None,\n        default_query: Mapping[str, object] | None = None,\n        http_client: httpx.Client | None = None,\n        _strict_response_validation: bool = False,\n    ) -> None:\n        ...\n\n    @overload\n    def __init__(\n        self,\n        *,\n        base_url: str,\n        api_version: str | None = None,\n        api_key: str | None = None,\n        azure_ad_token: str | None = None,\n        azure_ad_token_provider: AzureADTokenProvider | None = None,\n        organization: str | None = None,\n        timeout: float | Timeout | None | NotGiven = NOT_GIVEN,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        default_headers: Mapping[str, str] | None = None,\n        default_query: Mapping[str, object] | None = None,\n        http_client: httpx.Client | None = None,\n        _strict_response_validation: bool = False,\n    ) -> None:\n        ...\n\n    def __init__(\n        self,\n        *,\n        api_version: str | None = None,\n...\n", "needle_token_start": 735, "needle_token_end": 843, "code_context_ntokens": 16096, "output": ["```python\ndef _get_arch() -> str:\n    architecture = platform.machine().lower()\n\n    # Map the architecture names to Node.js equivalents\n    arch_map = {\n        \"x86_64\": \"x64\",\n        \"amd64\": \"x64\",\n        \"armv7l\": \"arm\",\n        \"aarch64\": \"arm64\",\n    }\n\n    return arch_map.get(architecture, architecture)\n```"]}
{"repo": "openai/openai-python", "name": "to_custom_raw_response_wrapper", "language": "python", "path": "src/openai/_response.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to modify an API method so that it returns a specific type of API response directly, rather than the default response format.\n2. **Input**: It takes an API method and a specific API response class that should be used to format the response.\n3. **Output**: The output is a modified version of the original API method that, when called, returns an instance of the specified API response class.\n4. **Procedure**: The function first creates a wrapper around the original API method. It modifies the method's behavior by adding specific headers to the request to indicate that the response should be returned as a raw data type and specifies the class that should be used to cast the response. The modified API method is then returned, ready to be used with the new settings applied.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\nclass MissingStreamClassError(TypeError):\n    def __init__(self) -> None:\n        super().__init__(\n            \"The `stream` argument was set to `True` but the `stream_cls` argument was not given. See `openai._streaming` for reference\",\n        )\n\n\nclass StreamAlreadyConsumed(OpenAIError):\n    \"\"\"\n    Attempted to read or stream content, but the content has already\n    been streamed.\n\n    This can happen if you use a method like `.iter_lines()` and then attempt\n    to read th entire response body afterwards, e.g.\n\n    ```py\n    response = await client.post(...)\n    async for line in response.iter_lines():\n        ...  # do something with `line`\n\n    content = await response.read()\n    # ^ error\n    ```\n\n    If you want this behaviour you'll need to either manually accumulate the response\n    content or call `await response.read()` before iterating over the stream.\n    \"\"\"\n\n    def __init__(self) -> None:\n        message = (\n            \"Attempted to read or stream some content, but the content has \"\n            \"already been streamed. \"\n            \"This could be due to attempting to stream the response \"\n            \"content more than once.\"\n            \"\\n\\n\"\n            \"You can fix this by manually accumulating the response content while streaming \"\n            \"or by calling `.read()` before starting to stream.\"\n        )\n        super().__init__(message)\n\n\nclass ResponseContextManager(Generic[_APIResponseT]):\n    \"\"\"Context manager for ensuring that a request is not made\n    until it is entered and that the response will always be closed\n    when the context manager exits\n    \"\"\"\n\n    def __init__(self, request_func: Callable[[], _APIResponseT]) -> None:\n        self._request_func = request_func\n        self.__response: _APIResponseT | None = None\n\n    def __enter__(self) -> _APIResponseT:\n        self.__response = self._request_func()\n        return self.__response\n\n    def __exit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        if self.__response is not None:\n            self.__response.close()\n\n\nclass AsyncResponseContextManager(Generic[_AsyncAPIResponseT]):\n    \"\"\"Context manager for ensuring that a request is not made\n    until it is entered and that the response will always be closed\n    when the context manager exits\n    \"\"\"\n\n    def __init__(self, api_request: Awaitable[_AsyncAPIResponseT]) -> None:\n        self._api_request = api_request\n        self.__response: _AsyncAPIResponseT | None = None\n\n    async def __aenter__(self) -> _AsyncAPIResponseT:\n        self.__response = await self._api_request\n        return self.__response\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        if self.__response is not None:\n            await self.__response.close()\n\n\ndef to_streamed_response_wrapper(func: Callable[P, R]) -> Callable[P, ResponseContextManager[APIResponse[R]]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support streaming and returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> ResponseContextManager[APIResponse[R]]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = functools.partial(func, *args, **kwargs)\n\n        return ResponseContextManager(cast(Callable[[], APIResponse[R]], make_request))\n\n    return wrapped\n\n\ndef async_to_streamed_response_wrapper(\n    func: Callable[P, Awaitable[R]],\n) -> Callable[P, AsyncResponseContextManager[AsyncAPIResponse[R]]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support streaming and returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> AsyncResponseContextManager[AsyncAPIResponse[R]]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = func(*args, **kwargs)\n\n        return AsyncResponseContextManager(cast(Awaitable[AsyncAPIResponse[R]], make_request))\n\n    return wrapped\n\n\ndef to_custom_streamed_response_wrapper(\n    func: Callable[P, object],\n    response_cls: type[_APIResponseT],\n) -> Callable[P, ResponseContextManager[_APIResponseT]]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support streaming and returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> ResponseContextManager[_APIResponseT]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = functools.partial(func, *args, **kwargs)\n\n        return ResponseContextManager(cast(Callable[[], _APIResponseT], make_request))\n\n    return wrapped\n\n\ndef async_to_custom_streamed_response_wrapper(\n    func: Callable[P, Awaitable[object]],\n    response_cls: type[_AsyncAPIResponseT],\n) -> Callable[P, AsyncResponseContextManager[_AsyncAPIResponseT]]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support streaming and returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> AsyncResponseContextManager[_AsyncAPIResponseT]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = func(*args, **kwargs)\n\n        return AsyncResponseContextManager(cast(Awaitable[_AsyncAPIResponseT], make_request))\n\n    return wrapped\n\n\ndef to_raw_response_wrapper(func: Callable[P, R]) -> Callable[P, APIResponse[R]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> APIResponse[R]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(APIResponse[R], func(*args, **kwargs))\n\n    return wrapped\n\n\ndef async_to_raw_response_wrapper(func: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[AsyncAPIResponse[R]]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    async def wrapped(*args: P.args, **kwargs: P.kwargs) -> AsyncAPIResponse[R]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(AsyncAPIResponse[R], await func(*args, **kwargs))\n\n    return wrapped\n\n\n\ndef to_custom_raw_response_wrapper(\n    func: Callable[P, object],\n    response_cls: type[_APIResponseT],\n) -> Callable[P, _APIResponseT]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> _APIResponseT:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(_APIResponseT, func(*args, **kwargs))\n\n    return wrapped\n\n\ndef async_to_custom_raw_response_wrapper(\n    func: Callable[P, Awaitable[object]],\n    response_cls: type[_AsyncAPIResponseT],\n) -> Callable[P, Awaitable[_AsyncAPIResponseT]]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> Awaitable[_AsyncAPIResponseT]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(Awaitable[_AsyncAPIResponseT], func(*args, **kwargs))\n\n    return wrapped\n\n\ndef extract_response_type(typ: type[BaseAPIResponse[Any]]) -> type:\n    \"\"\"Given a type like `APIResponse[T]`, returns the generic type variable `T`.\n\n    This also handles the case where a concrete subclass is given, e.g.\n    ```py\n    class MyResponse(APIResponse[bytes]):\n        ...\n\n    extract_response_type(MyResponse) -> bytes\n    ```\n    \"\"\"\n    return extract_type_var_from_base(\n        typ,\n        generic_bases=cast(\"tuple[type, ...]\", (BaseAPIResponse, APIResponse, AsyncAPIResponse)),\n        index=0,\n    )\n\n# Path: src/openai/types/completion_choice.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Dict, List, Optional\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\n\n__all__ = [\"CompletionChoice\", \"Logprobs\"]\n\n\nclass Logprobs(BaseModel):\n    text_offset: Optional[List[int]] = None\n\n    token_logprobs: Optional[List[float]] = None\n\n    tokens: Optional[List[str]] = None\n\n    top_logprobs: Optional[List[Dict[str, float]]] = None\n\n\nclass CompletionChoice(BaseModel):\n    finish_reason: Literal[\"stop\", \"length\", \"content_filter\"]\n    \"\"\"The reason the model stopped generating tokens.\n\n    This will be `stop` if the model hit a natural stop point or a provided stop\n    sequence, `length` if the maximum number of tokens specified in the request was\n    reached, or `content_filter` if content was omitted due to a flag from our\n    content filters.\n    \"\"\"\n\n    index: int\n\n    logprobs: Optional[Logprobs] = None\n\n    text: str\n\n# Path: src/openai/types/completion_usage.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom .._models import BaseModel\n\n__all__ = [\"CompletionUsage\"]\n\n\nclass CompletionUsage(BaseModel):\n    completion_tokens: int\n    \"\"\"Number of tokens in the generated completion.\"\"\"\n\n    prompt_tokens: int\n    \"\"\"Number of tokens in the prompt.\"\"\"\n\n    total_tokens: int\n    \"\"\"Total number of tokens used in the request (prompt + completion).\"\"\"\n\n# Path: src/openai/types/completion.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import List, Optional\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\nfrom .completion_usage import CompletionUsage\nfrom .completion_choice import CompletionChoice\n\n__all__ = [\"Completion\"]\n\n\nclass Completion(BaseModel):\n    id: str\n    \"\"\"A unique identifier for the completion.\"\"\"\n\n    choices: List[CompletionChoice]\n    \"\"\"The list of completion choices the model generated for the input prompt.\"\"\"\n\n    created: int\n    \"\"\"The Unix timestamp (in seconds) of when the completion was created.\"\"\"\n\n    model: str\n    \"\"\"The model used for completion.\"\"\"\n\n    object: Literal[\"text_completion\"]\n    \"\"\"The object type, which is always \"text_completion\" \"\"\"\n\n    system_fingerprint: Optional[str] = None\n    \"\"\"This fingerprint represents the backend configuration that the model runs with.\n\n    Can be used in conjunction with the `seed` request parameter to understand when\n    backend changes have been made that might impact determinism.\n    \"\"\"\n\n    usage: Optional[CompletionUsage] = None\n    \"\"\"Usage statistics for the completion request.\"\"\"\n\n# Path: src/openai/types/completion_create_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Dict, List, Union, Iterable, Optional\nfrom typing_extensions import Literal, Required, TypedDict\n\n__all__ = [\"CompletionCreateParamsBase\", \"CompletionCreateParamsNonStreaming\", \"CompletionCreateParamsStreaming\"]\n\n\nclass CompletionCreateParamsBase(TypedDict, total=False):\n    model: Required[Union[str, Literal[\"gpt-3.5-turbo-instruct\", \"davinci-002\", \"babbage-002\"]]]\n    \"\"\"ID of the model to use.\n\n    You can use the\n    [List models](https://platform.openai.com/docs/api-reference/models/list) API to\n    see all of your available models, or see our\n    [Model overview](https://platform.openai.com/docs/models/overview) for\n    descriptions of them.\n    \"\"\"\n\n    prompt: Required[Union[str, List[str], Iterable[int], Iterable[Iterable[int]], None]]\n    \"\"\"\n    The prompt(s) to generate completions for, encoded as a string, array of\n    strings, array of tokens, or array of token arrays.\n\n    Note that <|endoftext|> is the document separator that the model sees during\n    training, so if a prompt is not specified the model will generate as if from the\n    beginning of a new document.\n    \"\"\"\n\n    best_of: Optional[int]\n    \"\"\"\n    Generates `best_of` completions server-side and returns the \"best\" (the one with\n    the highest log probability per token). Results cannot be streamed.\n\n    When used with `n`, `best_of` controls the number of candidate completions and\n    `n` specifies how many to return \u2013 `best_of` must be greater than `n`.\n\n    **Note:** Because this parameter generates many completions, it can quickly\n    consume your token quota. Use carefully and ensure that you have reasonable\n    settings for `max_tokens` and `stop`.\n    \"\"\"\n\n    echo: Optional[bool]\n    \"\"\"Echo back the prompt in addition to the completion\"\"\"\n\n    frequency_penalty: Optional[float]\n    \"\"\"Number between -2.0 and 2.0.\n\n    Positive values penalize new tokens based on their existing frequency in the\n    text so far, decreasing the model's likelihood to repeat the same line verbatim.\n\n    [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation/parameter-details)\n    \"\"\"\n\n    logit_bias: Optional[Dict[str, int]]\n    \"\"\"Modify the likelihood of specified tokens appearing in the completion.\n\n    Accepts a JSON object that maps tokens (specified by their token ID in the GPT\n    tokenizer) to an associated bias value from -100 to 100. You can use this\n    [tokenizer tool](/tokenizer?view=bpe) to convert text to token IDs.\n    Mathematically, the bias is added to the logits generated by the model prior to\n    sampling. The exact effect will vary per model, but values between -1 and 1\n    should decrease or increase likelihood of selection; values like -100 or 100\n    should result in a ban or exclusive selection of the relevant token.\n\n    As an example, you can pass `{\"50256\": -100}` to prevent the <|endoftext|> token\n    from being generated.\n    \"\"\"\n\n    logprobs: Optional[int]\n    \"\"\"\n    Include the log probabilities on the `logprobs` most likely output tokens, as\n    well the chosen tokens. For example, if `logprobs` is 5, the API will return a\n    list of the 5 most likely tokens. The API will always return the `logprob` of\n    the sampled token, so there may be up to `logprobs+1` elements in the response.\n\n    The maximum value for `logprobs` is 5.\n    \"\"\"\n\n    max_tokens: Optional[int]\n    \"\"\"\n    The maximum number of [tokens](/tokenizer) that can be generated in the\n    completion.\n\n    The token count of your prompt plus `max_tokens` cannot exceed the model's\n    context length.\n    [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken)\n    for counting tokens.\n    \"\"\"\n\n    n: Optional[int]\n    \"\"\"How many completions to generate for each prompt.\n\n    **Note:** Because this parameter generates many completions, it can quickly\n    consume your token quota. Use carefully and ensure that you have reasonable\n    settings for `max_tokens` and `stop`.\n    \"\"\"\n\n    presence_penalty: Optional[float]\n    \"\"\"Number between -2.0 and 2.0.\n\n    Positive values penalize new tokens based on whether they appear in the text so\n    far, increasing the model's likelihood to talk about new topics.\n\n    [See more information about frequency and presence penalties.](https://platform.openai.com/docs/guides/text-generation/parameter-details)\n    \"\"\"\n\n    seed: Optional[int]\n    \"\"\"\n    If specified, our system will make a best effort to sample deterministically,\n    such that repeated requests with the same `seed` and parameters should return\n    the same result.\n\n    Determinism is not guaranteed, and you should refer to the `system_fingerprint`\n    response parameter to monitor changes in the backend.\n    \"\"\"\n\n    stop: Union[Optional[str], List[str], None]\n    \"\"\"Up to 4 sequences where the API will stop generating further tokens.\n\n    The returned text will not contain the stop sequence.\n    \"\"\"\n\n    suffix: Optional[str]\n    \"\"\"The suffix that comes after a completion of inserted text.\"\"\"\n\n    temperature: Optional[float]\n    \"\"\"What sampling temperature to use, between 0 and 2.\n\n    Higher values like 0.8 will make the output more random, while lower values like\n    0.2 will make it more focused and deterministic.\n\n    We generally recommend altering this or `top_p` but not both.\n    \"\"\"\n\n    top_p: Optional[float]\n    \"\"\"\n    An alternative to sampling with temperature, called nucleus sampling, where the\n    model considers the results of the tokens with top_p probability mass. So 0.1\n    means only the tokens comprising the top 10% probability mass are considered.\n\n    We generally recommend altering this or `temperature` but not both.\n    \"\"\"\n\n    user: str\n    \"\"\"\n    A unique identifier representing your end-user, which can help OpenAI to monitor\n    and detect abuse.\n    [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n    \"\"\"\n\n\nclass CompletionCreateParamsNonStreaming(CompletionCreateParamsBase):\n    stream: Optional[Literal[False]]\n    \"\"\"Whether to stream back partial progress.\n\n    If set, tokens will be sent as data-only\n    [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format)\n    as they become available, with the stream terminated by a `data: [DONE]`\n    message.\n    [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).\n    \"\"\"\n\n\nclass CompletionCreateParamsStreaming(CompletionCreateParamsBase):\n    stream: Required[Literal[True]]\n    \"\"\"Whether to stream back partial progress.\n\n    If set, tokens will be sent as data-only\n    [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format)\n    as they become available, with the stream terminated by a `data: [DONE]`\n    message.\n    [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).\n    \"\"\"\n\n\nCompletionCreateParams = Union[CompletionCreateParamsNonStreaming, CompletionCreateParamsStreaming]\n\n# Path: src/openai/types/embedding.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import List\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\n\n__all__ = [\"Embedding\"]\n\n\nclass Embedding(BaseModel):\n    embedding: List[float]\n    \"\"\"The embedding vector, which is a list of floats.\n\n    The length of vector depends on the model as listed in the\n    [embedding guide](https://platform.openai.com/docs/guides/embeddings).\n    \"\"\"\n\n    index: int\n    \"\"\"The index of the embedding in the list of embeddings.\"\"\"\n\n    object: Literal[\"embedding\"]\n    \"\"\"The object type, which is always \"embedding\".\"\"\"\n\n# Path: src/openai/types/create_embedding_response.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import List\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\nfrom .embedding import Embedding\n\n__all__ = [\"CreateEmbeddingResponse\", \"Usage\"]\n\n\nclass Usage(BaseModel):\n    prompt_tokens: int\n    \"\"\"The number of tokens used by the prompt.\"\"\"\n\n    total_tokens: int\n    \"\"\"The total number of tokens used by the request.\"\"\"\n\n\nclass CreateEmbeddingResponse(BaseModel):\n    data: List[Embedding]\n    \"\"\"The list of embeddings generated by the model.\"\"\"\n\n    model: str\n    \"\"\"The name of the model used to generate the embedding.\"\"\"\n\n    object: Literal[\"list\"]\n    \"\"\"The object type, which is always \"list\".\"\"\"\n\n    usage: Usage\n    \"\"\"The usage information for the request.\"\"\"\n\n# Path: src/openai/types/embedding_create_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import List, Union, Iterable\nfrom typing_extensions import Literal, Required, TypedDict\n\n__all__ = [\"EmbeddingCreateParams\"]\n\n\nclass EmbeddingCreateParams(TypedDict, total=False):\n    input: Required[Union[str, List[str], Iterable[int], Iterable[Iterable[int]]]]\n    \"\"\"Input text to embed, encoded as a string or array of tokens.\n\n    To embed multiple inputs in a single request, pass an array of strings or array\n    of token arrays. The input must not exceed the max input tokens for the model\n    (8192 tokens for `text-embedding-ada-002`), cannot be an empty string, and any\n    array must be 2048 dimensions or less.\n    [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken)\n    for counting tokens.\n    \"\"\"\n\n    model: Required[Union[str, Literal[\"text-embedding-ada-002\", \"text-embedding-3-small\", \"text-embedding-3-large\"]]]\n    \"\"\"ID of the model to use.\n\n    You can use the\n    [List models](https://platform.openai.com/docs/api-reference/models/list) API to\n    see all of your available models, or see our\n    [Model overview](https://platform.openai.com/docs/models/overview) for\n    descriptions of them.\n    \"\"\"\n\n    dimensions: int\n    \"\"\"The number of dimensions the resulting output embeddings should have.\n\n    Only supported in `text-embedding-3` and later models.\n    \"\"\"\n\n    encoding_format: Literal[\"float\", \"base64\"]\n    \"\"\"The format to return the embeddings in.\n\n    Can be either `float` or [`base64`](https://pypi.org/project/pybase64/).\n    \"\"\"\n\n    user: str\n    \"\"\"\n    A unique identifier representing your end-user, which can help OpenAI to monitor\n    and detect abuse.\n    [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n    \"\"\"\n\n# Path: src/openai/types/file_content.py\n# File generated from our OpenAPI spec by Stainless.\n\n\n__all__ = [\"FileContent\"]\n\nFileContent = str\n\n# Path: src/openai/types/file_create_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing_extensions import Literal, Required, TypedDict\n\nfrom .._types import FileTypes\n\n__all__ = [\"FileCreateParams\"]\n\n\nclass FileCreateParams(TypedDict, total=False):\n    file: Required[FileTypes]\n    \"\"\"The File object (not file name) to be uploaded.\"\"\"\n\n    purpose: Required[Literal[\"fine-tune\", \"assistants\"]]\n    \"\"\"The intended purpose of the uploaded file.\n\n    Use \"fine-tune\" for\n    [Fine-tuning](https://platform.openai.com/docs/api-reference/fine-tuning) and\n    \"assistants\" for\n    [Assistants](https://platform.openai.com/docs/api-reference/assistants) and\n    [Messages](https://platform.openai.com/docs/api-reference/messages). This allows\n    us to validate the format of the uploaded file is correct for fine-tuning.\n    \"\"\"\n\n# Path: src/openai/types/file_deleted.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\n\n__all__ = [\"FileDeleted\"]\n\n\nclass FileDeleted(BaseModel):\n    id: str\n\n    deleted: bool\n\n    object: Literal[\"file\"]\n\n# Path: src/openai/types/file_list_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing_extensions import TypedDict\n\n__all__ = [\"FileListParams\"]\n\n\nclass FileListParams(TypedDict, total=False):\n    purpose: str\n    \"\"\"Only return files with the given purpose.\"\"\"\n\n# Path: src/openai/types/file_object.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Optional\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\n\n__all__ = [\"FileObject\"]\n\n\nclass FileObject(BaseModel):\n    id: str\n    \"\"\"The file identifier, which can be referenced in the API endpoints.\"\"\"\n\n    bytes: int\n    \"\"\"The size of the file, in bytes.\"\"\"\n\n    created_at: int\n    \"\"\"The Unix timestamp (in seconds) for when the file was created.\"\"\"\n\n    filename: str\n    \"\"\"The name of the file.\"\"\"\n\n    object: Literal[\"file\"]\n    \"\"\"The object type, which is always `file`.\"\"\"\n\n    purpose: Literal[\"fine-tune\", \"fine-tune-results\", \"assistants\", \"assistants_output\"]\n    \"\"\"The intended purpose of the file.\n\n    Supported values are `fine-tune`, `fine-tune-results`, `assistants`, and\n    `assistants_output`.\n    \"\"\"\n\n    status: Literal[\"uploaded\", \"processed\", \"error\"]\n    \"\"\"Deprecated.\n\n    The current status of the file, which can be either `uploaded`, `processed`, or\n    `error`.\n    \"\"\"\n\n    status_details: Optional[str] = None\n    \"\"\"Deprecated.\n\n    For details on why a fine-tuning training file failed validation, see the\n    `error` field on `fine_tuning.job`.\n    \"\"\"\n\n# Path: src/openai/types/image.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Optional\n\nfrom .._models import BaseModel\n\n__all__ = [\"Image\"]\n\n\nclass Image(BaseModel):\n    b64_json: Optional[str] = None\n    \"\"\"\n    The base64-encoded JSON of the generated image, if `response_format` is\n    `b64_json`.\n    \"\"\"\n\n    revised_prompt: Optional[str] = None\n    \"\"\"\n    The prompt that was used to generate the image, if there was any revision to the\n    prompt.\n    \"\"\"\n\n    url: Optional[str] = None\n    \"\"\"The URL of the generated image, if `response_format` is `url` (default).\"\"\"\n\n# Path: src/openai/types/image_create_variation_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Union, Optional\nfrom typing_extensions import Literal, Required, TypedDict\n\nfrom .._types import FileTypes\n\n__all__ = [\"ImageCreateVariationParams\"]\n\n\nclass ImageCreateVariationParams(TypedDict, total=False):\n    image: Required[FileTypes]\n    \"\"\"The image to use as the basis for the variation(s).\n\n    Must be a valid PNG file, less than 4MB, and square.\n    \"\"\"\n\n    model: Union[str, Literal[\"dall-e-2\"], None]\n    \"\"\"The model to use for image generation.\n\n    Only `dall-e-2` is supported at this time.\n    \"\"\"\n\n    n: Optional[int]\n    \"\"\"The number of images to generate.\n\n    Must be between 1 and 10. For `dall-e-3`, only `n=1` is supported.\n    \"\"\"\n\n    response_format: Optional[Literal[\"url\", \"b64_json\"]]\n    \"\"\"The format in which the generated images are returned.\n\n    Must be one of `url` or `b64_json`.\n    \"\"\"\n\n    size: Optional[Literal[\"256x256\", \"512x512\", \"1024x1024\"]]\n    \"\"\"The size of the generated images.\n\n    Must be one of `256x256`, `512x512`, or `1024x1024`.\n    \"\"\"\n\n    user: str\n    \"\"\"\n    A unique identifier representing your end-user, which can help OpenAI to monitor\n    and detect abuse.\n    [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n    \"\"\"\n\n# Path: src/openai/types/image_edit_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Union, Optional\nfrom typing_extensions import Literal, Required, TypedDict\n\nfrom .._types import FileTypes\n\n__all__ = [\"ImageEditParams\"]\n\n\nclass ImageEditParams(TypedDict, total=False):\n    image: Required[FileTypes]\n    \"\"\"The image to edit.\n\n    Must be a valid PNG file, less than 4MB, and square. If mask is not provided,\n    image must have transparency, which will be used as the mask.\n    \"\"\"\n\n    prompt: Required[str]\n    \"\"\"A text description of the desired image(s).\n\n    The maximum length is 1000 characters.\n    \"\"\"\n\n    mask: FileTypes\n    \"\"\"An additional image whose fully transparent areas (e.g.\n\n    where alpha is zero) indicate where `image` should be edited. Must be a valid\n    PNG file, less than 4MB, and have the same dimensions as `image`.\n    \"\"\"\n\n    model: Union[str, Literal[\"dall-e-2\"], None]\n    \"\"\"The model to use for image generation.\n\n    Only `dall-e-2` is supported at this time.\n    \"\"\"\n\n    n: Optional[int]\n    \"\"\"The number of images to generate. Must be between 1 and 10.\"\"\"\n\n    response_format: Optional[Literal[\"url\", \"b64_json\"]]\n    \"\"\"The format in which the generated images are returned.\n\n    Must be one of `url` or `b64_json`.\n    \"\"\"\n\n    size: Optional[Literal[\"256x256\", \"512x512\", \"1024x1024\"]]\n    \"\"\"The size of the generated images.\n\n    Must be one of `256x256`, `512x512`, or `1024x1024`.\n    \"\"\"\n\n    user: str\n    \"\"\"\n    A unique identifier representing your end-user, which can help OpenAI to monitor\n    and detect abuse.\n    [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n    \"\"\"\n\n# Path: src/openai/types/image_generate_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Union, Optional\nfrom typing_extensions import Literal, Required, TypedDict\n\n__all__ = [\"ImageGenerateParams\"]\n\n\nclass ImageGenerateParams(TypedDict, total=False):\n    prompt: Required[str]\n    \"\"\"A text description of the desired image(s).\n\n    The maximum length is 1000 characters for `dall-e-2` and 4000 characters for\n    `dall-e-3`.\n    \"\"\"\n\n    model: Union[str, Literal[\"dall-e-2\", \"dall-e-3\"], None]\n    \"\"\"The model to use for image generation.\"\"\"\n\n    n: Optional[int]\n    \"\"\"The number of images to generate.\n\n    Must be between 1 and 10. For `dall-e-3`, only `n=1` is supported.\n    \"\"\"\n\n    quality: Literal[\"standard\", \"hd\"]\n    \"\"\"The quality of the image that will be generated.\n\n    `hd` creates images with finer details and greater consistency across the image.\n    This param is only supported for `dall-e-3`.\n    \"\"\"\n\n    response_format: Optional[Literal[\"url\", \"b64_json\"]]\n    \"\"\"The format in which the generated images are returned.\n\n    Must be one of `url` or `b64_json`.\n    \"\"\"\n\n    size: Optional[Literal[\"256x256\", \"512x512\", \"1024x1024\", \"1792x1024\", \"1024x1792\"]]\n    \"\"\"The size of the generated images.\n\n    Must be one of `256x256`, `512x512`, or `1024x1024` for `dall-e-2`. Must be one\n    of `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3` models.\n    \"\"\"\n\n    style: Optional[Literal[\"vivid\", \"natural\"]]\n    \"\"\"The style of the generated images.\n\n    Must be one of `vivid` or `natural`. Vivid causes the model to lean towards\n    generating hyper-real and dramatic images. Natural causes the model to produce\n    more natural, less hyper-real looking images. This param is only supported for\n    `dall-e-3`.\n    \"\"\"\n\n    user: str\n    \"\"\"\n    A unique identifier representing your end-user, which can help OpenAI to monitor\n    and detect abuse.\n    [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n    \"\"\"\n\n# Path: src/openai/types/images_response.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import List\n\nfrom .image import Image\nfrom .._models import BaseModel\n\n__all__ = [\"ImagesResponse\"]\n\n\nclass ImagesResponse(BaseModel):\n    created: int\n\n    data: List[Image]\n\n# Path: src/openai/types/model.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\n\n__all__ = [\"Model\"]\n\n\nclass Model(BaseModel):\n    id: str\n    \"\"\"The model identifier, which can be referenced in the API endpoints.\"\"\"\n\n    created: int\n    \"\"\"The Unix timestamp (in seconds) when the model was created.\"\"\"\n\n    object: Literal[\"model\"]\n    \"\"\"The object type, which is always \"model\".\"\"\"\n\n    owned_by: str\n    \"\"\"The organization that owns the model.\"\"\"\n\n# Path: src/openai/types/model_deleted.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom .._models import BaseModel\n\n__all__ = [\"ModelDeleted\"]\n\n\nclass ModelDeleted(BaseModel):\n    id: str\n\n    deleted: bool\n\n    object: str\n\n# Path: src/openai/types/moderation.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom pydantic import Field as FieldInfo\n\nfrom .._models import BaseModel\n\n__all__ = [\"Moderation\", \"Categories\", \"CategoryScores\"]\n\n\nclass Categories(BaseModel):\n    harassment: bool\n    \"\"\"\n    Content that expresses, incites, or promotes harassing language towards any\n    target.\n    \"\"\"\n\n    harassment_threatening: bool = FieldInfo(alias=\"harassment/threatening\")\n    \"\"\"\n    Harassment content that also includes violence or serious harm towards any\n    target.\n    \"\"\"\n\n    hate: bool\n    \"\"\"\n    Content that expresses, incites, or promotes hate based on race, gender,\n    ethnicity, religion, nationality, sexual orientation, disability status, or\n    caste. Hateful content aimed at non-protected groups (e.g., chess players) is\n    harassment.\n    \"\"\"\n\n    hate_threatening: bool = FieldInfo(alias=\"hate/threatening\")\n    \"\"\"\n    Hateful content that also includes violence or serious harm towards the targeted\n    group based on race, gender, ethnicity, religion, nationality, sexual\n    orientation, disability status, or caste.\n    \"\"\"\n\n    self_harm: bool = FieldInfo(alias=\"self-harm\")\n    \"\"\"\n    Content that promotes, encourages, or depicts acts of self-harm, such as\n    suicide, cutting, and eating disorders.\n    \"\"\"\n\n    self_harm_instructions: bool = FieldInfo(alias=\"self-harm/instructions\")\n    \"\"\"\n    Content that encourages performing acts of self-harm, such as suicide, cutting,\n    and eating disorders, or that gives instructions or advice on how to commit such\n    acts.\n    \"\"\"\n\n    self_harm_intent: bool = FieldInfo(alias=\"self-harm/intent\")\n    \"\"\"\n    Content where the speaker expresses that they are engaging or intend to engage\n    in acts of self-harm, such as suicide, cutting, and eating disorders.\n    \"\"\"\n\n    sexual: bool\n    \"\"\"\n    Content meant to arouse sexual excitement, such as the description of sexual\n    activity, or that promotes sexual services (excluding sex education and\n    wellness).\n    \"\"\"\n\n    sexual_minors: bool = FieldInfo(alias=\"sexual/minors\")\n    \"\"\"Sexual content that includes an individual who is under 18 years old.\"\"\"\n\n    violence: bool\n    \"\"\"Content that depicts death, violence, or physical injury.\"\"\"\n\n    violence_graphic: bool = FieldInfo(alias=\"violence/graphic\")\n    \"\"\"Content that depicts death, violence, or physical injury in graphic detail.\"\"\"\n\n\nclass CategoryScores(BaseModel):\n    harassment: float\n    \"\"\"The score for the category 'harassment'.\"\"\"\n\n    harassment_threatening: float = FieldInfo(alias=\"harassment/threatening\")\n    \"\"\"The score for the category 'harassment/threatening'.\"\"\"\n\n    hate: float\n    \"\"\"The score for the category 'hate'.\"\"\"\n\n    hate_threatening: float = FieldInfo(alias=\"hate/threatening\")\n    \"\"\"The score for the category 'hate/threatening'.\"\"\"\n\n    self_harm: float = FieldInfo(alias=\"self-harm\")\n    \"\"\"The score for the category 'self-harm'.\"\"\"\n\n    self_harm_instructions: float = FieldInfo(alias=\"self-harm/instructions\")\n    \"\"\"The score for the category 'self-harm/instructions'.\"\"\"\n\n    self_harm_intent: float = FieldInfo(alias=\"self-harm/intent\")\n    \"\"\"The score for the category 'self-harm/intent'.\"\"\"\n\n    sexual: float\n    \"\"\"The score for the category 'sexual'.\"\"\"\n\n    sexual_minors: float = FieldInfo(alias=\"sexual/minors\")\n    \"\"\"The score for the category 'sexual/minors'.\"\"\"\n\n    violence: float\n    \"\"\"The score for the category 'violence'.\"\"\"\n\n    violence_graphic: float = FieldInfo(alias=\"violence/graphic\")\n    \"\"\"The score for the category 'violence/graphic'.\"\"\"\n\n\nclass Moderation(BaseModel):\n    categories: Categories\n    \"\"\"A list of the categories, and whether they are flagged or not.\"\"\"\n\n    category_scores: CategoryScores\n    \"\"\"A list of the categories along with their scores as predicted by model.\"\"\"\n\n    flagged: bool\n    \"\"\"\n    Whether the content violates\n    [OpenAI's usage policies](/policies/usage-policies).\n    \"\"\"\n\n# Path: src/openai/types/moderation_create_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import List, Union\nfrom typing_extensions import Literal, Required, TypedDict\n\n__all__ = [\"ModerationCreateParams\"]\n\n\nclass ModerationCreateParams(TypedDict, total=False):\n    input: Required[Union[str, List[str]]]\n    \"\"\"The input text to classify\"\"\"\n\n    model: Union[str, Literal[\"text-moderation-latest\", \"text-moderation-stable\"]]\n    \"\"\"\n    Two content moderations models are available: `text-moderation-stable` and\n    `text-moderation-latest`.\n\n    The default is `text-moderation-latest` which will be automatically upgraded\n    over time. This ensures you are always using our most accurate model. If you use\n    `text-moderation-stable`, we will provide advanced notice before updating the\n    model. Accuracy of `text-moderation-stable` may be slightly lower than for\n    `text-moderation-latest`.\n    \"\"\"\n\n# Path: src/openai/types/moderation_create_response.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import List\n\nfrom .._models import BaseModel\nfrom .moderation import Moderation\n\n__all__ = [\"ModerationCreateResponse\"]\n\n\nclass ModerationCreateResponse(BaseModel):\n    id: str\n    \"\"\"The unique identifier for the moderation request.\"\"\"\n\n    model: str\n    \"\"\"The model used to generate the moderation results.\"\"\"\n\n    results: List[Moderation]\n    \"\"\"A list of moderation objects.\"\"\"\n\n# Path: src/openai/types/shared/function_parameters.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Dict\n\n__all__ = [\"FunctionParameters\"]\n\nFunctionParameters = Dict[str, object]\n\n# Path: src/openai/types/shared/function_definition.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Optional\n\nfrom ..._models import BaseModel\nfrom .function_parameters import FunctionParameters\n\n__all__ = [\"FunctionDefinition\"]\n\n\nclass FunctionDefinition(BaseModel):\n    name: str\n    \"\"\"The name of the function to be called.\n\n    Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length\n    of 64.\n    \"\"\"\n\n    description: Optional[str] = None\n    \"\"\"\n    A description of what the function does, used by the model to choose when and\n    how to call the function.\n    \"\"\"\n\n    parameters: Optional[FunctionParameters] = None\n    \"\"\"The parameters the functions accepts, described as a JSON Schema object.\n\n    See the\n    [guide](https://platform.openai.com/docs/guides/text-generation/function-calling)\n    for examples, and the\n    [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for\n    documentation about the format.\n\n    Omitting `parameters` defines a function with an empty parameter list.\n    \"\"\"\n\n# Path: src/openai/types/shared/__init__.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom .function_definition import FunctionDefinition as FunctionDefinition\nfrom .function_parameters import FunctionParameters as FunctionParameters\n\n# Path: src/openai/types/__init__.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom .image import Image as Image\nfrom .model import Model as Model\nfrom .shared import FunctionDefinition as FunctionDefinition, FunctionParameters as FunctionParameters\nfrom .embedding import Embedding as Embedding\nfrom .completion import Completion as Completion\nfrom .moderation import Moderation as Moderation\nfrom .file_object import FileObject as FileObject\nfrom .file_content import FileContent as FileContent\nfrom .file_deleted import FileDeleted as FileDeleted\nfrom .model_deleted import ModelDeleted as ModelDeleted\nfrom .images_response import ImagesResponse as ImagesResponse\nfrom .completion_usage import CompletionUsage as CompletionUsage\nfrom .file_list_params import FileListParams as FileListParams\nfrom .completion_choice import CompletionChoice as CompletionChoice\nfrom .image_edit_params import ImageEditParams as ImageEditParams\nfrom .file_create_params import FileCreateParams as FileCreateParams\nfrom .image_generate_params import ImageGenerateParams as ImageGenerateParams\nfrom .embedding_create_params import EmbeddingCreateParams as EmbeddingCreateParams\nfrom .completion_create_params import CompletionCreateParams as CompletionCreateParams\nfrom .moderation_create_params import ModerationCreateParams as ModerationCreateParams\nfrom .create_embedding_response import CreateEmbeddingResponse as CreateEmbeddingResponse\nfrom .moderation_create_response import ModerationCreateResponse as ModerationCreateResponse\nfrom .image_create_variation_params import ImageCreateVariationParams as ImageCreateVariationParams\n\n# Path: src/openai/types/audio/speech_create_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Union\nfrom typing_extensions import Literal, Required, TypedDict\n\n__all__ = [\"SpeechCreateParams\"]\n\n\nclass SpeechCreateParams(TypedDict, total=False):\n    input: Required[str]\n    \"\"\"The text to generate audio for. The maximum length is 4096 characters.\"\"\"\n\n    model: Required[Union[str, Literal[\"tts-1\", \"tts-1-hd\"]]]\n    \"\"\"\n    One of the available [TTS models](https://platform.openai.com/docs/models/tts):\n    `tts-1` or `tts-1-hd`\n    \"\"\"\n\n    voice: Required[Literal[\"alloy\", \"echo\", \"fable\", \"onyx\", \"nova\", \"shimmer\"]]\n    \"\"\"The voice to use when generating the audio.\n\n    Supported voices are `alloy`, `echo`, `fable`, `onyx`, `nova`, and `shimmer`.\n    Previews of the voices are available in the\n    [Text to speech guide](https://platform.openai.com/docs/guides/text-to-speech/voice-options).\n    \"\"\"\n\n    response_format: Literal[\"mp3\", \"opus\", \"aac\", \"flac\", \"pcm\", \"wav\"]\n    \"\"\"The format to return audio in.\n\n    Supported formats are `mp3`, `opus`, `aac`, `flac`, `pcm`, and `wav`.\n\n    The `pcm` audio format, similar to `wav` but without a header, utilizes a 24kHz\n    sample rate, mono channel, and 16-bit depth in signed little-endian format.\n    \"\"\"\n\n    speed: float\n    \"\"\"The speed of the generated audio.\n\n    Select a value from `0.25` to `4.0`. `1.0` is the default.\n    \"\"\"\n\n# Path: src/openai/types/audio/transcription.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom ..._models import BaseModel\n\n__all__ = [\"Transcription\"]\n\n\nclass Transcription(BaseModel):\n    text: str\n\n# Path: src/openai/types/audio/transcription_create_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import List, Union\nfrom typing_extensions import Literal, Required, TypedDict\n\nfrom ..._types import FileTypes\n\n__all__ = [\"TranscriptionCreateParams\"]\n\n\nclass TranscriptionCreateParams(TypedDict, total=False):\n    file: Required[FileTypes]\n    \"\"\"\n    The audio file object (not file name) to transcribe, in one of these formats:\n    flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.\n    \"\"\"\n\n    model: Required[Union[str, Literal[\"whisper-1\"]]]\n    \"\"\"ID of the model to use. Only `whisper-1` is currently available.\"\"\"\n\n    language: str\n    \"\"\"The language of the input audio.\n\n    Supplying the input language in\n    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will\n    improve accuracy and latency.\n    \"\"\"\n\n    prompt: str\n    \"\"\"An optional text to guide the model's style or continue a previous audio\n    segment.\n\n    The [prompt](https://platform.openai.com/docs/guides/speech-to-text/prompting)\n    should match the audio language.\n    \"\"\"\n\n    response_format: Literal[\"json\", \"text\", \"srt\", \"verbose_json\", \"vtt\"]\n    \"\"\"\n    The format of the transcript output, in one of these options: `json`, `text`,\n    `srt`, `verbose_json`, or `vtt`.\n    \"\"\"\n\n    temperature: float\n    \"\"\"The sampling temperature, between 0 and 1.\n\n    Higher values like 0.8 will make the output more random, while lower values like\n    0.2 will make it more focused and deterministic. If set to 0, the model will use\n    [log probability](https://en.wikipedia.org/wiki/Log_probability) to\n    automatically increase the temperature until certain thresholds are hit.\n    \"\"\"\n\n    timestamp_granularities: List[Literal[\"word\", \"segment\"]]\n    \"\"\"The timestamp granularities to populate for this transcription.\n\n    Any of these options: `word`, or `segment`. Note: There is no additional latency\n    for segment timestamps, but generating word timestamps incurs additional\n    latency.\n    \"\"\"\n\n# Path: src/openai/types/audio/translation.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom ..._models import BaseModel\n\n__all__ = [\"Translation\"]\n\n\nclass Translation(BaseModel):\n    text: str\n\n# Path: src/openai/types/audio/translation_create_params.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Union\nfrom typing_extensions import Literal, Required, TypedDict\n\nfrom ..._types import FileTypes\n\n__all__ = [\"TranslationCreateParams\"]\n\n\nclass TranslationCreateParams(TypedDict, total=False):\n    file: Required[FileTypes]\n    \"\"\"\n    The audio file object (not file name) translate, in one of these formats: flac,\n    mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.\n    \"\"\"\n\n    model: Required[Union[str, Literal[\"whisper-1\"]]]\n    \"\"\"ID of the model to use. Only `whisper-1` is currently available.\"\"\"\n\n    prompt: str\n    \"\"\"An optional text to guide the model's style or continue a previous audio\n    segment.\n\n    The [prompt](https://platform.openai.com/docs/guides/speech-to-text/prompting)\n    should be in English.\n    \"\"\"\n\n    response_format: str\n    \"\"\"\n    The format of the transcript output, in one of these options: `json`, `text`,\n    `srt`, `verbose_json`, or `vtt`.\n    \"\"\"\n\n    temperature: float\n    \"\"\"The sampling temperature, between 0 and 1.\n\n    Higher values like 0.8 will make the output more random, while lower values like\n    0.2 will make it more focused and deterministic. If set to 0, the model will use\n    [log probability](https://en.wikipedia.org/wiki/Log_probability) to\n    automatically increase the temperature until certain thresholds are hit.\n    \"\"\"\n\n# Path: src/openai/types/audio/__init__.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom .translation import Translation as Translation\nfrom .transcription import Transcription as Transcription\nfrom .speech_create_params import SpeechCreateParams as SpeechCreateParams\nfrom .translation_create_params import TranslationCreateParams as TranslationCreateParams\nfrom .transcription_create_params import TranscriptionCreateParams as TranscriptionCreateParams\n\n# Path: src/openai/resources/audio/speech.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Union\nfrom typing_extensions import Literal\n\nimport httpx\n\nfrom ... import _legacy_response\nfrom ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven\nfrom ..._utils import maybe_transform\nfrom ..._compat import cached_property\nfrom ..._resource import SyncAPIResource, AsyncAPIResource\nfrom ..._response import (\n    StreamedBinaryAPIResponse,\n    AsyncStreamedBinaryAPIResponse,\n    to_custom_streamed_response_wrapper,\n    async_to_custom_streamed_response_wrapper,\n)\nfrom ...types.audio import speech_create_params\nfrom ..._base_client import (\n    make_request_options,\n)\n\n__all__ = [\"Speech\", \"AsyncSpeech\"]\n\n\nclass Speech(SyncAPIResource):\n    @cached_property\n    def with_raw_response(self) -> SpeechWithRawResponse:\n        return SpeechWithRawResponse(self)\n\n    @cached_property\n    def with_streaming_response(self) -> SpeechWithStreamingResponse:\n        return SpeechWithStreamingResponse(self)\n\n    def create(\n        self,\n        *,\n        input: str,\n        model: Union[str, Literal[\"tts-1\", \"tts-1-hd\"]],\n        voice: Literal[\"alloy\", \"echo\", \"fable\", \"onyx\", \"nova\", \"shimmer\"],\n        response_format: Literal[\"mp3\", \"opus\", \"aac\", \"flac\", \"pcm\", \"wav\"] | NotGiven = NOT_GIVEN,\n        speed: float | NotGiven = NOT_GIVEN,\n        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.\n        # The extra values given here take precedence over values defined on the client or passed to this method.\n        extra_headers: Headers | None = None,\n        extra_query: Query | None = None,\n        extra_body: Body | None = None,\n        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,\n    ) -> _legacy_response.HttpxBinaryResponseContent:\n        \"\"\"\n        Generates audio from the input text.\n\n        Args:\n          input: The text to generate audio for. The maximum length is 4096 characters.\n\n          model:\n              One of the available [TTS models](https://platform.openai.com/docs/models/tts):\n              `tts-1` or `tts-1-hd`\n\n          voice: The voice to use when generating the audio. Supported voices are `alloy`,\n              `echo`, `fable`, `onyx`, `nova`, and `shimmer`. Previews of the voices are\n              available in the\n              [Text to speech guide](https://platform.openai.com/docs/guides/text-to-speech/voice-options).\n\n          response_format: The format to return audio in. Supported formats are `mp3`, `opus`, `aac`,\n              `flac`, `pcm`, and `wav`.\n\n              The `pcm` audio format, similar to `wav` but without a header, utilizes a 24kHz\n              sample rate, mono channel, and 16-bit depth in signed little-endian format.\n\n          speed: The speed of the generated audio. Select a value from `0.25` to `4.0`. `1.0` is\n              the default.\n\n          extra_headers: Send extra headers\n\n          extra_query: Add additional query parameters to the request\n\n          extra_body: Add additional JSON properties to the request\n\n          timeout: Override the client-level default timeout for this request, in seconds\n        \"\"\"\n        extra_headers = {\"Accept\": \"application/octet-stream\", **(extra_headers or {})}\n        return self._post(\n            \"/audio/speech\",\n            body=maybe_transform(\n                {\n                    \"input\": input,\n                    \"model\": model,\n                    \"voice\": voice,\n                    \"response_format\": response_format,\n                    \"speed\": speed,\n                },\n                speech_create_params.SpeechCreateParams,\n            ),\n            options=make_request_options(\n                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout\n            ),\n            cast_to=_legacy_response.HttpxBinaryResponseContent,\n        )\n\n\nclass AsyncSpeech(AsyncAPIResource):\n    @cached_property\n    def with_raw_response(self) -> AsyncSpeechWithRawResponse:\n        return AsyncSpeechWithRawResponse(self)\n\n    @cached_property\n    def with_streaming_response(self) -> AsyncSpeechWithStreamingResponse:\n        return AsyncSpeechWithStreamingResponse(self)\n\n    async def create(\n        self,\n        *,\n        input: str,\n        model: Union[str, Literal[\"tts-1\", \"tts-1-hd\"]],\n        voice: Literal[\"alloy\", \"echo\", \"fable\", \"onyx\", \"nova\", \"shimmer\"],\n        response_format: Literal[\"mp3\", \"opus\", \"aac\", \"flac\", \"pcm\", \"wav\"] | NotGiven = NOT_GIVEN,\n        speed: float | NotGiven = NOT_GIVEN,\n        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.\n        # The extra values given here take precedence over values defined on the client or passed to this method.\n        extra_headers: Headers | None = None,\n        extra_query: Query | None = None,\n        extra_body: Body | None = None,\n        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,\n    ) -> _legacy_response.HttpxBinaryResponseContent:\n        \"\"\"\n        Generates audio from the input text.\n\n        Args:\n          input: The text to generate audio for. The maximum length is 4096 characters.\n\n          model:\n              One of the available [TTS models](https://platform.openai.com/docs/models/tts):\n              `tts-1` or `tts-1-hd`\n\n          voice: The voice to use when generating the audio. Supported voices are `alloy`,\n              `echo`, `fable`, `onyx`, `nova`, and `shimmer`. Previews of the voices are\n              available in the\n              [Text to speech guide](https://platform.openai.com/docs/guides/text-to-speech/voice-options).\n\n          response_format: The format to return audio in. Supported formats are `mp3`, `opus`, `aac`,\n              `flac`, `pcm`, and `wav`.\n\n...\n", "needle_token_start": 2261, "needle_token_end": 2527, "code_context_ntokens": 15891, "output": ["```python\ndef to_custom_raw_response_wrapper(\n    func: Callable[P, object],\n    response_cls: type[_APIResponseT],\n) -> Callable[P, _APIResponseT]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> _APIResponseT:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(_APIResponseT, func(*args, **kwargs))\n\n    return wrapped\n```"]}
{"repo": "openai/openai-python", "name": "transcribe", "language": "python", "path": "src/openai/cli/_api/audio.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: Converts spoken language in an audio file into written text using a specified machine learning model.\n2. **Input**: An audio file, a model identifier, and optional parameters such as language, temperature, prompt, and response format.\n3. **Output**: The transcribed text from the audio file displayed in a structured format.\n4. **Procedure**: The function reads the audio file, sends it to a machine learning API using the specified model and optional parameters, and then displays the transcription result.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/openai/__init__.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nimport os as _os\nfrom typing_extensions import override\n\nfrom . import types\nfrom ._types import NoneType, Transport, ProxiesTypes\nfrom ._utils import file_from_path\nfrom ._client import Client, OpenAI, Stream, Timeout, Transport, AsyncClient, AsyncOpenAI, AsyncStream, RequestOptions\nfrom ._models import BaseModel\nfrom ._version import __title__, __version__\nfrom ._response import APIResponse as APIResponse, AsyncAPIResponse as AsyncAPIResponse\nfrom ._exceptions import (\n    APIError,\n    OpenAIError,\n    ConflictError,\n    NotFoundError,\n    APIStatusError,\n    RateLimitError,\n    APITimeoutError,\n    BadRequestError,\n    APIConnectionError,\n    AuthenticationError,\n    InternalServerError,\n    PermissionDeniedError,\n    UnprocessableEntityError,\n    APIResponseValidationError,\n)\nfrom ._utils._logs import setup_logging as _setup_logging\n\n__all__ = [\n    \"types\",\n    \"__version__\",\n    \"__title__\",\n    \"NoneType\",\n    \"Transport\",\n    \"ProxiesTypes\",\n    \"OpenAIError\",\n    \"APIError\",\n    \"APIStatusError\",\n    \"APITimeoutError\",\n    \"APIConnectionError\",\n    \"APIResponseValidationError\",\n    \"BadRequestError\",\n    \"AuthenticationError\",\n    \"PermissionDeniedError\",\n    \"NotFoundError\",\n    \"ConflictError\",\n    \"UnprocessableEntityError\",\n    \"RateLimitError\",\n    \"InternalServerError\",\n    \"Timeout\",\n    \"RequestOptions\",\n    \"Client\",\n    \"AsyncClient\",\n    \"Stream\",\n    \"AsyncStream\",\n    \"OpenAI\",\n    \"AsyncOpenAI\",\n    \"file_from_path\",\n    \"BaseModel\",\n]\n\nfrom .lib import azure as _azure\nfrom .version import VERSION as VERSION\nfrom .lib.azure import AzureOpenAI as AzureOpenAI, AsyncAzureOpenAI as AsyncAzureOpenAI\nfrom .lib._old_api import *\n\n_setup_logging()\n\n# Update the __module__ attribute for exported symbols so that\n# error messages point to this module instead of the module\n# it was originally defined in, e.g.\n# openai._exceptions.NotFoundError -> openai.NotFoundError\n__locals = locals()\nfor __name in __all__:\n    if not __name.startswith(\"__\"):\n        try:\n            __locals[__name].__module__ = \"openai\"\n        except (TypeError, AttributeError):\n            # Some of our exported symbols are builtins which we can't set attributes for.\n            pass\n\n# ------ Module level client ------\nimport typing as _t\nimport typing_extensions as _te\n\nimport httpx as _httpx\n\nfrom ._base_client import DEFAULT_TIMEOUT, DEFAULT_MAX_RETRIES\n\napi_key: str | None = None\n\norganization: str | None = None\n\nbase_url: str | _httpx.URL | None = None\n\ntimeout: float | Timeout | None = DEFAULT_TIMEOUT\n\nmax_retries: int = DEFAULT_MAX_RETRIES\n\ndefault_headers: _t.Mapping[str, str] | None = None\n\ndefault_query: _t.Mapping[str, object] | None = None\n\nhttp_client: _httpx.Client | None = None\n\n_ApiType = _te.Literal[\"openai\", \"azure\"]\n\napi_type: _ApiType | None = _t.cast(_ApiType, _os.environ.get(\"OPENAI_API_TYPE\"))\n\napi_version: str | None = _os.environ.get(\"OPENAI_API_VERSION\")\n\nazure_endpoint: str | None = _os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n\nazure_ad_token: str | None = _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\n\nazure_ad_token_provider: _azure.AzureADTokenProvider | None = None\n\n\nclass _ModuleClient(OpenAI):\n    # Note: we have to use type: ignores here as overriding class members\n    # with properties is technically unsafe but it is fine for our use case\n\n    @property  # type: ignore\n    @override\n    def api_key(self) -> str | None:\n        return api_key\n\n    @api_key.setter  # type: ignore\n    def api_key(self, value: str | None) -> None:  # type: ignore\n        global api_key\n\n        api_key = value\n\n    @property  # type: ignore\n    @override\n    def organization(self) -> str | None:\n        return organization\n\n    @organization.setter  # type: ignore\n    def organization(self, value: str | None) -> None:  # type: ignore\n        global organization\n\n        organization = value\n\n    @property\n    @override\n    def base_url(self) -> _httpx.URL:\n        if base_url is not None:\n            return _httpx.URL(base_url)\n\n        return super().base_url\n\n    @base_url.setter\n    def base_url(self, url: _httpx.URL | str) -> None:\n        super().base_url = url  # type: ignore[misc]\n\n    @property  # type: ignore\n    @override\n    def timeout(self) -> float | Timeout | None:\n        return timeout\n\n    @timeout.setter  # type: ignore\n    def timeout(self, value: float | Timeout | None) -> None:  # type: ignore\n        global timeout\n\n        timeout = value\n\n    @property  # type: ignore\n    @override\n    def max_retries(self) -> int:\n        return max_retries\n\n    @max_retries.setter  # type: ignore\n    def max_retries(self, value: int) -> None:  # type: ignore\n        global max_retries\n\n        max_retries = value\n\n    @property  # type: ignore\n    @override\n    def _custom_headers(self) -> _t.Mapping[str, str] | None:\n        return default_headers\n\n    @_custom_headers.setter  # type: ignore\n    def _custom_headers(self, value: _t.Mapping[str, str] | None) -> None:  # type: ignore\n        global default_headers\n\n        default_headers = value\n\n    @property  # type: ignore\n    @override\n    def _custom_query(self) -> _t.Mapping[str, object] | None:\n        return default_query\n\n    @_custom_query.setter  # type: ignore\n    def _custom_query(self, value: _t.Mapping[str, object] | None) -> None:  # type: ignore\n        global default_query\n\n        default_query = value\n\n    @property  # type: ignore\n    @override\n    def _client(self) -> _httpx.Client:\n        return http_client or super()._client\n\n    @_client.setter  # type: ignore\n    def _client(self, value: _httpx.Client) -> None:  # type: ignore\n        global http_client\n\n        http_client = value\n\n\nclass _AzureModuleClient(_ModuleClient, AzureOpenAI):  # type: ignore\n    ...\n\n\nclass _AmbiguousModuleClientUsageError(OpenAIError):\n    def __init__(self) -> None:\n        super().__init__(\n            \"Ambiguous use of module client; please set `openai.api_type` or the `OPENAI_API_TYPE` environment variable to `openai` or `azure`\"\n        )\n\n\ndef _has_openai_credentials() -> bool:\n    return _os.environ.get(\"OPENAI_API_KEY\") is not None\n\n\ndef _has_azure_credentials() -> bool:\n    return azure_endpoint is not None or _os.environ.get(\"AZURE_OPENAI_API_KEY\") is not None\n\n\ndef _has_azure_ad_credentials() -> bool:\n    return (\n        _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\") is not None\n        or azure_ad_token is not None\n        or azure_ad_token_provider is not None\n    )\n\n\n_client: OpenAI | None = None\n\n\ndef _load_client() -> OpenAI:  # type: ignore[reportUnusedFunction]\n    global _client\n\n    if _client is None:\n        global api_type, azure_endpoint, azure_ad_token, api_version\n\n        if azure_endpoint is None:\n            azure_endpoint = _os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n\n        if azure_ad_token is None:\n            azure_ad_token = _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\n\n        if api_version is None:\n            api_version = _os.environ.get(\"OPENAI_API_VERSION\")\n\n        if api_type is None:\n...\n# Path: src/openai/cli/_models.py\nfrom typing import Any\nfrom typing_extensions import ClassVar\n\nimport pydantic\n\nfrom .. import _models\nfrom .._compat import PYDANTIC_V2, ConfigDict\n\n\nclass BaseModel(_models.BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(extra=\"ignore\", arbitrary_types_allowed=True)\n    else:\n\n        class Config(pydantic.BaseConfig):  # type: ignore\n            extra: Any = pydantic.Extra.ignore  # type: ignore\n            arbitrary_types_allowed: bool = True\n\n# Path: src/openai/cli/_progress.py\nfrom __future__ import annotations\n\nimport io\nfrom typing import Callable\nfrom typing_extensions import override\n\n\nclass CancelledError(Exception):\n    def __init__(self, msg: str) -> None:\n        self.msg = msg\n        super().__init__(msg)\n\n    @override\n    def __str__(self) -> str:\n        return self.msg\n\n    __repr__ = __str__\n\n\nclass BufferReader(io.BytesIO):\n    def __init__(self, buf: bytes = b\"\", desc: str | None = None) -> None:\n        super().__init__(buf)\n        self._len = len(buf)\n        self._progress = 0\n        self._callback = progress(len(buf), desc=desc)\n\n    def __len__(self) -> int:\n        return self._len\n\n    @override\n    def read(self, n: int | None = -1) -> bytes:\n        chunk = io.BytesIO.read(self, n)\n        self._progress += len(chunk)\n\n        try:\n            self._callback(self._progress)\n        except Exception as e:  # catches exception from the callback\n            raise CancelledError(\"The upload was cancelled: {}\".format(e)) from e\n\n        return chunk\n\n\ndef progress(total: float, desc: str | None) -> Callable[[float], None]:\n    import tqdm\n\n    meter = tqdm.tqdm(total=total, unit_scale=True, desc=desc)\n\n    def incr(progress: float) -> None:\n        meter.n = progress\n        if progress == total:\n            meter.close()\n        else:\n            meter.refresh()\n\n    return incr\n\n\ndef MB(i: int) -> int:\n    return int(i // 1024**2)\n\n# Path: src/openai/cli/_utils.py\nfrom __future__ import annotations\n\nimport sys\n\nimport openai\n\nfrom .. import OpenAI, _load_client\nfrom .._compat import model_json\nfrom .._models import BaseModel\n\n\nclass Colors:\n    HEADER = \"\\033[95m\"\n    OKBLUE = \"\\033[94m\"\n    OKGREEN = \"\\033[92m\"\n    WARNING = \"\\033[93m\"\n    FAIL = \"\\033[91m\"\n    ENDC = \"\\033[0m\"\n    BOLD = \"\\033[1m\"\n    UNDERLINE = \"\\033[4m\"\n\n\ndef get_client() -> OpenAI:\n    return _load_client()\n\n\ndef organization_info() -> str:\n    organization = openai.organization\n    if organization is not None:\n        return \"[organization={}] \".format(organization)\n\n    return \"\"\n\n\ndef print_model(model: BaseModel) -> None:\n    sys.stdout.write(model_json(model, indent=2) + \"\\n\")\n\n\ndef can_use_http2() -> bool:\n    try:\n        import h2  # type: ignore  # noqa\n    except ImportError:\n        return False\n\n    return True\n\n# Path: src/openai/cli/_api/audio.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any, Optional, cast\nfrom argparse import ArgumentParser\n\nfrom .._utils import get_client, print_model\nfrom ..._types import NOT_GIVEN\nfrom .._models import BaseModel\nfrom .._progress import BufferReader\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    # transcriptions\n    sub = subparser.add_parser(\"audio.transcriptions.create\")\n\n    # Required\n    sub.add_argument(\"-m\", \"--model\", type=str, default=\"whisper-1\")\n    sub.add_argument(\"-f\", \"--file\", type=str, required=True)\n    # Optional\n    sub.add_argument(\"--response-format\", type=str)\n    sub.add_argument(\"--language\", type=str)\n    sub.add_argument(\"-t\", \"--temperature\", type=float)\n    sub.add_argument(\"--prompt\", type=str)\n    sub.set_defaults(func=CLIAudio.transcribe, args_model=CLITranscribeArgs)\n\n    # translations\n    sub = subparser.add_parser(\"audio.translations.create\")\n\n    # Required\n    sub.add_argument(\"-f\", \"--file\", type=str, required=True)\n    # Optional\n    sub.add_argument(\"-m\", \"--model\", type=str, default=\"whisper-1\")\n    sub.add_argument(\"--response-format\", type=str)\n    # TODO: doesn't seem to be supported by the API\n    # sub.add_argument(\"--language\", type=str)\n    sub.add_argument(\"-t\", \"--temperature\", type=float)\n    sub.add_argument(\"--prompt\", type=str)\n    sub.set_defaults(func=CLIAudio.translate, args_model=CLITranslationArgs)\n\n\nclass CLITranscribeArgs(BaseModel):\n    model: str\n    file: str\n    response_format: Optional[str] = None\n    language: Optional[str] = None\n    temperature: Optional[float] = None\n    prompt: Optional[str] = None\n\n\nclass CLITranslationArgs(BaseModel):\n    model: str\n    file: str\n    response_format: Optional[str] = None\n    language: Optional[str] = None\n    temperature: Optional[float] = None\n    prompt: Optional[str] = None\n\n\nclass CLIAudio:\n    @staticmethod\n    \ndef transcribe(args: CLITranscribeArgs) -> None:\n        with open(args.file, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        model = get_client().audio.transcriptions.create(\n            file=(args.file, buffer_reader),\n            model=args.model,\n            language=args.language or NOT_GIVEN,\n            temperature=args.temperature or NOT_GIVEN,\n            prompt=args.prompt or NOT_GIVEN,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(model)\n\n    @staticmethod\n    def translate(args: CLITranslationArgs) -> None:\n        with open(args.file, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        model = get_client().audio.translations.create(\n            file=(args.file, buffer_reader),\n            model=args.model,\n            temperature=args.temperature or NOT_GIVEN,\n            prompt=args.prompt or NOT_GIVEN,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(model)\n\n# Path: src/openai/cli/_api/chat/completions.py\nfrom __future__ import annotations\n\nimport sys\nfrom typing import TYPE_CHECKING, List, Optional, cast\nfrom argparse import ArgumentParser\nfrom typing_extensions import Literal, NamedTuple\n\nfrom ..._utils import get_client\nfrom ..._models import BaseModel\nfrom ...._streaming import Stream\nfrom ....types.chat import (\n    ChatCompletionRole,\n    ChatCompletionChunk,\n    CompletionCreateParams,\n)\nfrom ....types.chat.completion_create_params import (\n    CompletionCreateParamsStreaming,\n    CompletionCreateParamsNonStreaming,\n)\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"chat.completions.create\")\n\n    sub._action_groups.pop()\n    req = sub.add_argument_group(\"required arguments\")\n    opt = sub.add_argument_group(\"optional arguments\")\n\n    req.add_argument(\n        \"-g\",\n        \"--message\",\n        action=\"append\",\n        nargs=2,\n        metavar=(\"ROLE\", \"CONTENT\"),\n        help=\"A message in `{role} {content}` format. Use this argument multiple times to add multiple messages.\",\n        required=True,\n    )\n    req.add_argument(\n        \"-m\",\n        \"--model\",\n        help=\"The model to use.\",\n        required=True,\n    )\n\n    opt.add_argument(\n        \"-n\",\n        \"--n\",\n        help=\"How many completions to generate for the conversation.\",\n        type=int,\n    )\n    opt.add_argument(\"-M\", \"--max-tokens\", help=\"The maximum number of tokens to generate.\", type=int)\n    opt.add_argument(\n        \"-t\",\n        \"--temperature\",\n        help=\"\"\"What sampling temperature to use. Higher values means the model will take more risks. Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer.\n\nMutually exclusive with `top_p`.\"\"\",\n        type=float,\n    )\n    opt.add_argument(\n        \"-P\",\n        \"--top_p\",\n        help=\"\"\"An alternative to sampling with temperature, called nucleus sampling, where the considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10%% probability mass are considered.\n\n            Mutually exclusive with `temperature`.\"\"\",\n        type=float,\n    )\n    opt.add_argument(\n        \"--stop\",\n        help=\"A stop sequence at which to stop generating tokens for the message.\",\n    )\n    opt.add_argument(\"--stream\", help=\"Stream messages as they're ready.\", action=\"store_true\")\n    sub.set_defaults(func=CLIChatCompletion.create, args_model=CLIChatCompletionCreateArgs)\n\n\nclass CLIMessage(NamedTuple):\n    role: ChatCompletionRole\n    content: str\n\n\nclass CLIChatCompletionCreateArgs(BaseModel):\n    message: List[CLIMessage]\n    model: str\n    n: Optional[int] = None\n    max_tokens: Optional[int] = None\n    temperature: Optional[float] = None\n    top_p: Optional[float] = None\n    stop: Optional[str] = None\n    stream: bool = False\n\n\nclass CLIChatCompletion:\n    @staticmethod\n    def create(args: CLIChatCompletionCreateArgs) -> None:\n        params: CompletionCreateParams = {\n            \"model\": args.model,\n            \"messages\": [\n                {\"role\": cast(Literal[\"user\"], message.role), \"content\": message.content} for message in args.message\n            ],\n            \"n\": args.n,\n            \"temperature\": args.temperature,\n            \"top_p\": args.top_p,\n            \"stop\": args.stop,\n            # type checkers are not good at inferring union types so we have to set stream afterwards\n            \"stream\": False,\n        }\n        if args.stream:\n            params[\"stream\"] = args.stream  # type: ignore\n        if args.max_tokens is not None:\n            params[\"max_tokens\"] = args.max_tokens\n\n        if args.stream:\n            return CLIChatCompletion._stream_create(cast(CompletionCreateParamsStreaming, params))\n\n        return CLIChatCompletion._create(cast(CompletionCreateParamsNonStreaming, params))\n\n    @staticmethod\n    def _create(params: CompletionCreateParamsNonStreaming) -> None:\n        completion = get_client().chat.completions.create(**params)\n        should_print_header = len(completion.choices) > 1\n        for choice in completion.choices:\n            if should_print_header:\n                sys.stdout.write(\"===== Chat Completion {} =====\\n\".format(choice.index))\n\n            content = choice.message.content if choice.message.content is not None else \"None\"\n            sys.stdout.write(content)\n\n            if should_print_header or not content.endswith(\"\\n\"):\n                sys.stdout.write(\"\\n\")\n\n            sys.stdout.flush()\n\n    @staticmethod\n    def _stream_create(params: CompletionCreateParamsStreaming) -> None:\n        # cast is required for mypy\n        stream = cast(  # pyright: ignore[reportUnnecessaryCast]\n            Stream[ChatCompletionChunk], get_client().chat.completions.create(**params)\n        )\n        for chunk in stream:\n            should_print_header = len(chunk.choices) > 1\n            for choice in chunk.choices:\n                if should_print_header:\n                    sys.stdout.write(\"===== Chat Completion {} =====\\n\".format(choice.index))\n\n                content = choice.delta.content or \"\"\n                sys.stdout.write(content)\n\n                if should_print_header:\n                    sys.stdout.write(\"\\n\")\n\n                sys.stdout.flush()\n\n        sys.stdout.write(\"\\n\")\n\n# Path: src/openai/cli/_api/chat/__init__.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom argparse import ArgumentParser\n\nfrom . import completions\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    completions.register(subparser)\n\n# Path: src/openai/cli/_errors.py\nfrom __future__ import annotations\n\nimport sys\n\nimport pydantic\n\nfrom ._utils import Colors, organization_info\nfrom .._exceptions import APIError, OpenAIError\n\n\nclass CLIError(OpenAIError):\n    ...\n\n\nclass SilentCLIError(CLIError):\n    ...\n\n\ndef display_error(err: CLIError | APIError | pydantic.ValidationError) -> None:\n    if isinstance(err, SilentCLIError):\n        return\n\n    sys.stderr.write(\"{}{}Error:{} {}\\n\".format(organization_info(), Colors.FAIL, Colors.ENDC, err))\n\n# Path: src/openai/cli/_api/completions.py\nfrom __future__ import annotations\n\nimport sys\nfrom typing import TYPE_CHECKING, Optional, cast\nfrom argparse import ArgumentParser\nfrom functools import partial\n\nfrom openai.types.completion import Completion\n\nfrom .._utils import get_client\nfrom ..._types import NOT_GIVEN, NotGivenOr\nfrom ..._utils import is_given\nfrom .._errors import CLIError\nfrom .._models import BaseModel\nfrom ..._streaming import Stream\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"completions.create\")\n\n    # Required\n    sub.add_argument(\n        \"-m\",\n        \"--model\",\n        help=\"The model to use\",\n        required=True,\n    )\n\n    # Optional\n    sub.add_argument(\"-p\", \"--prompt\", help=\"An optional prompt to complete from\")\n    sub.add_argument(\"--stream\", help=\"Stream tokens as they're ready.\", action=\"store_true\")\n    sub.add_argument(\"-M\", \"--max-tokens\", help=\"The maximum number of tokens to generate\", type=int)\n    sub.add_argument(\n        \"-t\",\n        \"--temperature\",\n        help=\"\"\"What sampling temperature to use. Higher values means the model will take more risks. Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer.\n\nMutually exclusive with `top_p`.\"\"\",\n        type=float,\n    )\n    sub.add_argument(\n        \"-P\",\n        \"--top_p\",\n        help=\"\"\"An alternative to sampling with temperature, called nucleus sampling, where the considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10%% probability mass are considered.\n\n            Mutually exclusive with `temperature`.\"\"\",\n        type=float,\n    )\n    sub.add_argument(\n        \"-n\",\n        \"--n\",\n        help=\"How many sub-completions to generate for each prompt.\",\n        type=int,\n    )\n    sub.add_argument(\n        \"--logprobs\",\n        help=\"Include the log probabilities on the `logprobs` most likely tokens, as well the chosen tokens. So for example, if `logprobs` is 10, the API will return a list of the 10 most likely tokens. If `logprobs` is 0, only the chosen tokens will have logprobs returned.\",\n        type=int,\n    )\n    sub.add_argument(\n        \"--best_of\",\n        help=\"Generates `best_of` completions server-side and returns the 'best' (the one with the highest log probability per token). Results cannot be streamed.\",\n        type=int,\n    )\n    sub.add_argument(\n        \"--echo\",\n        help=\"Echo back the prompt in addition to the completion\",\n        action=\"store_true\",\n    )\n    sub.add_argument(\n        \"--frequency_penalty\",\n        help=\"Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.\",\n        type=float,\n    )\n    sub.add_argument(\n        \"--presence_penalty\",\n        help=\"Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.\",\n        type=float,\n    )\n    sub.add_argument(\"--suffix\", help=\"The suffix that comes after a completion of inserted text.\")\n    sub.add_argument(\"--stop\", help=\"A stop sequence at which to stop generating tokens.\")\n    sub.add_argument(\n        \"--user\",\n        help=\"A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse.\",\n    )\n    # TODO: add support for logit_bias\n    sub.set_defaults(func=CLICompletions.create, args_model=CLICompletionCreateArgs)\n\n\nclass CLICompletionCreateArgs(BaseModel):\n    model: str\n    stream: bool = False\n\n    prompt: Optional[str] = None\n    n: NotGivenOr[int] = NOT_GIVEN\n    stop: NotGivenOr[str] = NOT_GIVEN\n    user: NotGivenOr[str] = NOT_GIVEN\n    echo: NotGivenOr[bool] = NOT_GIVEN\n    suffix: NotGivenOr[str] = NOT_GIVEN\n    best_of: NotGivenOr[int] = NOT_GIVEN\n    top_p: NotGivenOr[float] = NOT_GIVEN\n    logprobs: NotGivenOr[int] = NOT_GIVEN\n    max_tokens: NotGivenOr[int] = NOT_GIVEN\n    temperature: NotGivenOr[float] = NOT_GIVEN\n    presence_penalty: NotGivenOr[float] = NOT_GIVEN\n    frequency_penalty: NotGivenOr[float] = NOT_GIVEN\n\n\nclass CLICompletions:\n    @staticmethod\n    def create(args: CLICompletionCreateArgs) -> None:\n        if is_given(args.n) and args.n > 1 and args.stream:\n            raise CLIError(\"Can't stream completions with n>1 with the current CLI\")\n\n        make_request = partial(\n            get_client().completions.create,\n            n=args.n,\n            echo=args.echo,\n            stop=args.stop,\n            user=args.user,\n            model=args.model,\n            top_p=args.top_p,\n            prompt=args.prompt,\n            suffix=args.suffix,\n            best_of=args.best_of,\n            logprobs=args.logprobs,\n            max_tokens=args.max_tokens,\n            temperature=args.temperature,\n            presence_penalty=args.presence_penalty,\n            frequency_penalty=args.frequency_penalty,\n        )\n\n        if args.stream:\n            return CLICompletions._stream_create(\n                # mypy doesn't understand the `partial` function but pyright does\n                cast(Stream[Completion], make_request(stream=True))  # pyright: ignore[reportUnnecessaryCast]\n            )\n\n        return CLICompletions._create(make_request())\n\n    @staticmethod\n    def _create(completion: Completion) -> None:\n        should_print_header = len(completion.choices) > 1\n        for choice in completion.choices:\n            if should_print_header:\n                sys.stdout.write(\"===== Completion {} =====\\n\".format(choice.index))\n\n            sys.stdout.write(choice.text)\n\n            if should_print_header or not choice.text.endswith(\"\\n\"):\n                sys.stdout.write(\"\\n\")\n\n            sys.stdout.flush()\n\n    @staticmethod\n    def _stream_create(stream: Stream[Completion]) -> None:\n        for completion in stream:\n            should_print_header = len(completion.choices) > 1\n            for choice in sorted(completion.choices, key=lambda c: c.index):\n                if should_print_header:\n                    sys.stdout.write(\"===== Chat Completion {} =====\\n\".format(choice.index))\n\n                sys.stdout.write(choice.text)\n\n                if should_print_header:\n                    sys.stdout.write(\"\\n\")\n\n                sys.stdout.flush()\n\n        sys.stdout.write(\"\\n\")\n\n# Path: src/openai/cli/_api/files.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any, cast\nfrom argparse import ArgumentParser\n\nfrom .._utils import get_client, print_model\nfrom .._models import BaseModel\nfrom .._progress import BufferReader\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"files.create\")\n\n    sub.add_argument(\n        \"-f\",\n        \"--file\",\n        required=True,\n        help=\"File to upload\",\n    )\n    sub.add_argument(\n        \"-p\",\n        \"--purpose\",\n        help=\"Why are you uploading this file? (see https://platform.openai.com/docs/api-reference/ for purposes)\",\n        required=True,\n    )\n    sub.set_defaults(func=CLIFile.create, args_model=CLIFileCreateArgs)\n\n    sub = subparser.add_parser(\"files.retrieve\")\n    sub.add_argument(\"-i\", \"--id\", required=True, help=\"The files ID\")\n    sub.set_defaults(func=CLIFile.get, args_model=CLIFileCreateArgs)\n\n    sub = subparser.add_parser(\"files.delete\")\n    sub.add_argument(\"-i\", \"--id\", required=True, help=\"The files ID\")\n    sub.set_defaults(func=CLIFile.delete, args_model=CLIFileCreateArgs)\n\n    sub = subparser.add_parser(\"files.list\")\n    sub.set_defaults(func=CLIFile.list)\n\n\nclass CLIFileIDArgs(BaseModel):\n    id: str\n\n\nclass CLIFileCreateArgs(BaseModel):\n    file: str\n    purpose: str\n\n\nclass CLIFile:\n    @staticmethod\n    def create(args: CLIFileCreateArgs) -> None:\n        with open(args.file, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        file = get_client().files.create(\n            file=(args.file, buffer_reader),\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            purpose=cast(Any, args.purpose),\n        )\n        print_model(file)\n\n    @staticmethod\n    def get(args: CLIFileIDArgs) -> None:\n        file = get_client().files.retrieve(file_id=args.id)\n        print_model(file)\n\n    @staticmethod\n    def delete(args: CLIFileIDArgs) -> None:\n        file = get_client().files.delete(file_id=args.id)\n        print_model(file)\n\n    @staticmethod\n    def list() -> None:\n        files = get_client().files.list()\n        for file in files:\n            print_model(file)\n\n# Path: src/openai/cli/_api/image.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any, cast\nfrom argparse import ArgumentParser\n\nfrom .._utils import get_client, print_model\nfrom ..._types import NOT_GIVEN, NotGiven, NotGivenOr\nfrom .._models import BaseModel\nfrom .._progress import BufferReader\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"images.generate\")\n    sub.add_argument(\"-m\", \"--model\", type=str)\n    sub.add_argument(\"-p\", \"--prompt\", type=str, required=True)\n    sub.add_argument(\"-n\", \"--num-images\", type=int, default=1)\n    sub.add_argument(\"-s\", \"--size\", type=str, default=\"1024x1024\", help=\"Size of the output image\")\n    sub.add_argument(\"--response-format\", type=str, default=\"url\")\n    sub.set_defaults(func=CLIImage.create, args_model=CLIImageCreateArgs)\n\n    sub = subparser.add_parser(\"images.edit\")\n    sub.add_argument(\"-m\", \"--model\", type=str)\n    sub.add_argument(\"-p\", \"--prompt\", type=str, required=True)\n    sub.add_argument(\"-n\", \"--num-images\", type=int, default=1)\n    sub.add_argument(\n        \"-I\",\n        \"--image\",\n        type=str,\n        required=True,\n        help=\"Image to modify. Should be a local path and a PNG encoded image.\",\n    )\n    sub.add_argument(\"-s\", \"--size\", type=str, default=\"1024x1024\", help=\"Size of the output image\")\n    sub.add_argument(\"--response-format\", type=str, default=\"url\")\n    sub.add_argument(\n        \"-M\",\n        \"--mask\",\n        type=str,\n        required=False,\n        help=\"Path to a mask image. It should be the same size as the image you're editing and a RGBA PNG image. The Alpha channel acts as the mask.\",\n    )\n    sub.set_defaults(func=CLIImage.edit, args_model=CLIImageEditArgs)\n\n    sub = subparser.add_parser(\"images.create_variation\")\n    sub.add_argument(\"-m\", \"--model\", type=str)\n    sub.add_argument(\"-n\", \"--num-images\", type=int, default=1)\n    sub.add_argument(\n        \"-I\",\n        \"--image\",\n        type=str,\n        required=True,\n        help=\"Image to modify. Should be a local path and a PNG encoded image.\",\n    )\n    sub.add_argument(\"-s\", \"--size\", type=str, default=\"1024x1024\", help=\"Size of the output image\")\n    sub.add_argument(\"--response-format\", type=str, default=\"url\")\n    sub.set_defaults(func=CLIImage.create_variation, args_model=CLIImageCreateVariationArgs)\n\n\nclass CLIImageCreateArgs(BaseModel):\n    prompt: str\n    num_images: int\n    size: str\n    response_format: str\n    model: NotGivenOr[str] = NOT_GIVEN\n\n\nclass CLIImageCreateVariationArgs(BaseModel):\n    image: str\n    num_images: int\n    size: str\n    response_format: str\n    model: NotGivenOr[str] = NOT_GIVEN\n\n\nclass CLIImageEditArgs(BaseModel):\n    image: str\n    num_images: int\n    size: str\n    response_format: str\n    prompt: str\n    mask: NotGivenOr[str] = NOT_GIVEN\n    model: NotGivenOr[str] = NOT_GIVEN\n\n\nclass CLIImage:\n    @staticmethod\n    def create(args: CLIImageCreateArgs) -> None:\n        image = get_client().images.generate(\n            model=args.model,\n            prompt=args.prompt,\n            n=args.num_images,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            size=cast(Any, args.size),\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(image)\n\n    @staticmethod\n    def create_variation(args: CLIImageCreateVariationArgs) -> None:\n        with open(args.image, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        image = get_client().images.create_variation(\n            model=args.model,\n            image=(\"image\", buffer_reader),\n            n=args.num_images,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            size=cast(Any, args.size),\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(image)\n\n    @staticmethod\n    def edit(args: CLIImageEditArgs) -> None:\n        with open(args.image, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Image upload progress\")\n\n        if isinstance(args.mask, NotGiven):\n            mask: NotGivenOr[BufferReader] = NOT_GIVEN\n        else:\n            with open(args.mask, \"rb\") as file_reader:\n                mask = BufferReader(file_reader.read(), desc=\"Mask progress\")\n\n        image = get_client().images.edit(\n            model=args.model,\n            prompt=args.prompt,\n            image=(\"image\", buffer_reader),\n            n=args.num_images,\n            mask=(\"mask\", mask) if not isinstance(mask, NotGiven) else mask,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            size=cast(Any, args.size),\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(image)\n\n# Path: src/openai/cli/_api/models.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom argparse import ArgumentParser\n\nfrom .._utils import get_client, print_model\nfrom .._models import BaseModel\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"models.list\")\n    sub.set_defaults(func=CLIModels.list)\n\n    sub = subparser.add_parser(\"models.retrieve\")\n    sub.add_argument(\"-i\", \"--id\", required=True, help=\"The model ID\")\n    sub.set_defaults(func=CLIModels.get, args_model=CLIModelIDArgs)\n\n    sub = subparser.add_parser(\"models.delete\")\n    sub.add_argument(\"-i\", \"--id\", required=True, help=\"The model ID\")\n    sub.set_defaults(func=CLIModels.delete, args_model=CLIModelIDArgs)\n\n\nclass CLIModelIDArgs(BaseModel):\n    id: str\n\n\nclass CLIModels:\n    @staticmethod\n    def get(args: CLIModelIDArgs) -> None:\n        model = get_client().models.retrieve(model=args.id)\n        print_model(model)\n\n    @staticmethod\n    def delete(args: CLIModelIDArgs) -> None:\n        model = get_client().models.delete(model=args.id)\n        print_model(model)\n\n    @staticmethod\n    def list() -> None:\n        models = get_client().models.list()\n        for model in models:\n            print_model(model)\n\n# Path: src/openai/cli/_api/_main.py\nfrom __future__ import annotations\n\nfrom argparse import ArgumentParser\n\nfrom . import chat, audio, files, image, models, completions\n\n\ndef register_commands(parser: ArgumentParser) -> None:\n    subparsers = parser.add_subparsers(help=\"All API subcommands\")\n\n    chat.register(subparsers)\n    image.register(subparsers)\n    audio.register(subparsers)\n    files.register(subparsers)\n    models.register(subparsers)\n    completions.register(subparsers)\n\n# Path: src/openai/cli/_api/__init__.py\nfrom ._main import register_commands as register_commands\n\n# Path: src/openai/cli/_tools/fine_tunes.py\nfrom __future__ import annotations\n\nimport sys\nfrom typing import TYPE_CHECKING\nfrom argparse import ArgumentParser\n\nfrom .._models import BaseModel\nfrom ...lib._validators import (\n    get_validators,\n    write_out_file,\n    read_any_format,\n    apply_validators,\n    apply_necessary_remediation,\n)\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"fine_tunes.prepare_data\")\n    sub.add_argument(\n        \"-f\",\n        \"--file\",\n        required=True,\n        help=\"JSONL, JSON, CSV, TSV, TXT or XLSX file containing prompt-completion examples to be analyzed.\"\n        \"This should be the local file path.\",\n    )\n    sub.add_argument(\n        \"-q\",\n        \"--quiet\",\n        required=False,\n        action=\"store_true\",\n        help=\"Auto accepts all suggestions, without asking for user input. To be used within scripts.\",\n    )\n    sub.set_defaults(func=prepare_data, args_model=PrepareDataArgs)\n\n\nclass PrepareDataArgs(BaseModel):\n    file: str\n\n    quiet: bool\n\n\ndef prepare_data(args: PrepareDataArgs) -> None:\n    sys.stdout.write(\"Analyzing...\\n\")\n    fname = args.file\n    auto_accept = args.quiet\n    df, remediation = read_any_format(fname)\n    apply_necessary_remediation(None, remediation)\n\n    validators = get_validators()\n\n    assert df is not None\n\n    apply_validators(\n        df,\n        fname,\n        remediation,\n        validators,\n        auto_accept,\n        write_out_file_func=write_out_file,\n    )\n\n# Path: src/openai/cli/_tools/migrate.py\nfrom __future__ import annotations\n\nimport os\nimport sys\nimport json\nimport shutil\nimport tarfile\nimport platform\nimport subprocess\nfrom typing import TYPE_CHECKING, List\nfrom pathlib import Path\nfrom argparse import ArgumentParser\n\nimport httpx\n\nfrom .._errors import CLIError, SilentCLIError\nfrom .._models import BaseModel\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"migrate\")\n    sub.set_defaults(func=migrate, args_model=MigrateArgs, allow_unknown_args=True)\n\n    sub = subparser.add_parser(\"grit\")\n    sub.set_defaults(func=grit, args_model=GritArgs, allow_unknown_args=True)\n\n\nclass GritArgs(BaseModel):\n    # internal\n    unknown_args: List[str] = []\n\n\ndef grit(args: GritArgs) -> None:\n    grit_path = install()\n\n    try:\n        subprocess.check_call([grit_path, *args.unknown_args])\n    except subprocess.CalledProcessError:\n        # stdout and stderr are forwarded by subprocess so an error will already\n        # have been displayed\n        raise SilentCLIError() from None\n\n\nclass MigrateArgs(BaseModel):\n    # internal\n    unknown_args: List[str] = []\n\n\ndef migrate(args: MigrateArgs) -> None:\n    grit_path = install()\n\n    try:\n        subprocess.check_call([grit_path, \"apply\", \"openai\", *args.unknown_args])\n    except subprocess.CalledProcessError:\n        # stdout and stderr are forwarded by subprocess so an error will already\n        # have been displayed\n        raise SilentCLIError() from None\n\n\n# handles downloading the Grit CLI until they provide their own PyPi package\n\nKEYGEN_ACCOUNT = \"custodian-dev\"\n\n\ndef _cache_dir() -> Path:\n    xdg = os.environ.get(\"XDG_CACHE_HOME\")\n    if xdg is not None:\n        return Path(xdg)\n\n    return Path.home() / \".cache\"\n\n\ndef _debug(message: str) -> None:\n    if not os.environ.get(\"DEBUG\"):\n        return\n\n    sys.stdout.write(f\"[DEBUG]: {message}\\n\")\n\n\ndef install() -> Path:\n    \"\"\"Installs the Grit CLI and returns the location of the binary\"\"\"\n    if sys.platform == \"win32\":\n        raise CLIError(\"Windows is not supported yet in the migration CLI\")\n\n    platform = \"macos\" if sys.platform == \"darwin\" else \"linux\"\n\n    dir_name = _cache_dir() / \"openai-python\"\n    install_dir = dir_name / \".install\"\n    target_dir = install_dir / \"bin\"\n\n    target_path = target_dir / \"marzano\"\n    temp_file = target_dir / \"marzano.tmp\"\n\n    if target_path.exists():\n        _debug(f\"{target_path} already exists\")\n        sys.stdout.flush()\n        return target_path\n\n    _debug(f\"Using Grit CLI path: {target_path}\")\n\n    target_dir.mkdir(parents=True, exist_ok=True)\n\n    if temp_file.exists():\n        temp_file.unlink()\n\n    arch = _get_arch()\n    _debug(f\"Using architecture {arch}\")\n\n    file_name = f\"marzano-{platform}-{arch}\"\n    meta_url = f\"https://api.keygen.sh/v1/accounts/{KEYGEN_ACCOUNT}/artifacts/{file_name}\"\n\n    sys.stdout.write(f\"Retrieving Grit CLI metadata from {meta_url}\\n\")\n    with httpx.Client() as client:\n        response = client.get(meta_url)  # pyright: ignore[reportUnknownMemberType]\n\n        data = response.json()\n        errors = data.get(\"errors\")\n        if errors:\n            for error in errors:\n                sys.stdout.write(f\"{error}\\n\")\n\n            raise CLIError(\"Could not locate Grit CLI binary - see above errors\")\n\n        write_manifest(install_dir, data[\"data\"][\"relationships\"][\"release\"][\"data\"][\"id\"])\n\n        link = data[\"data\"][\"links\"][\"redirect\"]\n        _debug(f\"Redirect URL {link}\")\n\n        download_response = client.get(link)  # pyright: ignore[reportUnknownMemberType]\n        with open(temp_file, \"wb\") as file:\n            for chunk in download_response.iter_bytes():\n                file.write(chunk)\n\n    unpacked_dir = target_dir / \"cli-bin\"\n    unpacked_dir.mkdir(parents=True, exist_ok=True)\n\n    with tarfile.open(temp_file, \"r:gz\") as archive:\n        archive.extractall(unpacked_dir, filter=\"data\")\n\n    for item in unpacked_dir.iterdir():\n        item.rename(target_dir / item.name)\n\n    shutil.rmtree(unpacked_dir)\n    os.remove(temp_file)\n    os.chmod(target_path, 0o755)\n\n    sys.stdout.flush()\n\n    return target_path\n\n\ndef _get_arch() -> str:\n    architecture = platform.machine().lower()\n\n    # Map the architecture names to Node.js equivalents\n    arch_map = {\n        \"x86_64\": \"x64\",\n        \"amd64\": \"x64\",\n        \"armv7l\": \"arm\",\n        \"aarch64\": \"arm64\",\n    }\n\n    return arch_map.get(architecture, architecture)\n\n\ndef write_manifest(install_path: Path, release: str) -> None:\n    manifest = {\n        \"installPath\": str(install_path),\n        \"binaries\": {\n            \"marzano\": {\n                \"name\": \"marzano\",\n                \"release\": release,\n            },\n        },\n    }\n    manifest_path = Path(install_path) / \"manifests.json\"\n    with open(manifest_path, \"w\") as f:\n        json.dump(manifest, f, indent=2)\n\n# Path: src/openai/cli/_tools/_main.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom argparse import ArgumentParser\n\nfrom . import migrate, fine_tunes\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register_commands(parser: ArgumentParser, subparser: _SubParsersAction[ArgumentParser]) -> None:\n    migrate.register(subparser)\n\n    namespaced = parser.add_subparsers(title=\"Tools\", help=\"Convenience client side tools\")\n\n    fine_tunes.register(namespaced)\n\n# Path: src/openai/cli/_tools/__init__.py\nfrom ._main import register_commands as register_commands\n\n# Path: src/openai/cli/_cli.py\nfrom __future__ import annotations\n\nimport sys\nimport logging\nimport argparse\nfrom typing import Any, List, Type, Optional\nfrom typing_extensions import ClassVar\n\nimport httpx\nimport pydantic\n\nimport openai\n\nfrom . import _tools\nfrom .. import _ApiType, __version__\nfrom ._api import register_commands\nfrom ._utils import can_use_http2\nfrom .._types import ProxiesDict\nfrom ._errors import CLIError, display_error\nfrom .._compat import PYDANTIC_V2, ConfigDict, model_parse\nfrom .._models import BaseModel\nfrom .._exceptions import APIError\n\nlogger = logging.getLogger()\nformatter = logging.Formatter(\"[%(asctime)s] %(message)s\")\nhandler = logging.StreamHandler(sys.stderr)\nhandler.setFormatter(formatter)\nlogger.addHandler(handler)\n\n\nclass Arguments(BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(\n            extra=\"ignore\",\n        )\n    else:\n\n        class Config(pydantic.BaseConfig):  # type: ignore\n            extra: Any = pydantic.Extra.ignore  # type: ignore\n\n    verbosity: int\n    version: Optional[str] = None\n\n    api_key: Optional[str]\n    api_base: Optional[str]\n    organization: Optional[str]\n    proxy: Optional[List[str]]\n    api_type: Optional[_ApiType] = None\n    api_version: Optional[str] = None\n\n    # azure\n    azure_endpoint: Optional[str] = None\n    azure_ad_token: Optional[str] = None\n\n    # internal, set by subparsers to parse their specific args\n    args_model: Optional[Type[BaseModel]] = None\n\n    # internal, used so that subparsers can forward unknown arguments\n    unknown_args: List[str] = []\n    allow_unknown_args: bool = False\n\n\ndef _build_parser() -> argparse.ArgumentParser:\n    parser = argparse.ArgumentParser(description=None, prog=\"openai\")\n    parser.add_argument(\n        \"-v\",\n        \"--verbose\",\n        action=\"count\",\n        dest=\"verbosity\",\n        default=0,\n        help=\"Set verbosity.\",\n    )\n    parser.add_argument(\"-b\", \"--api-base\", help=\"What API base url to use.\")\n    parser.add_argument(\"-k\", \"--api-key\", help=\"What API key to use.\")\n    parser.add_argument(\"-p\", \"--proxy\", nargs=\"+\", help=\"What proxy to use.\")\n    parser.add_argument(\n        \"-o\",\n        \"--organization\",\n        help=\"Which organization to run as (will use your default organization if not specified)\",\n    )\n    parser.add_argument(\n        \"-t\",\n        \"--api-type\",\n        type=str,\n        choices=(\"openai\", \"azure\"),\n        help=\"The backend API to call, must be `openai` or `azure`\",\n    )\n    parser.add_argument(\n        \"--api-version\",\n        help=\"The Azure API version, e.g. 'https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning'\",\n    )\n\n    # azure\n    parser.add_argument(\n        \"--azure-endpoint\",\n        help=\"The Azure endpoint, e.g. 'https://endpoint.openai.azure.com'\",\n    )\n    parser.add_argument(\n        \"--azure-ad-token\",\n        help=\"A token from Azure Active Directory, https://www.microsoft.com/en-us/security/business/identity-access/microsoft-entra-id\",\n    )\n\n    # prints the package version\n    parser.add_argument(\n        \"-V\",\n        \"--version\",\n        action=\"version\",\n        version=\"%(prog)s \" + __version__,\n    )\n\n    def help() -> None:\n        parser.print_help()\n\n    parser.set_defaults(func=help)\n\n    subparsers = parser.add_subparsers()\n    sub_api = subparsers.add_parser(\"api\", help=\"Direct API calls\")\n\n    register_commands(sub_api)\n\n    sub_tools = subparsers.add_parser(\"tools\", help=\"Client side tools for convenience\")\n    _tools.register_commands(sub_tools, subparsers)\n\n    return parser\n\n\ndef main() -> int:\n    try:\n        _main()\n    except (APIError, CLIError, pydantic.ValidationError) as err:\n        display_error(err)\n        return 1\n    except KeyboardInterrupt:\n        sys.stderr.write(\"\\n\")\n        return 1\n    return 0\n\n\ndef _parse_args(parser: argparse.ArgumentParser) -> tuple[argparse.Namespace, Arguments, list[str]]:\n    # argparse by default will strip out the `--` but we want to keep it for unknown arguments\n    if \"--\" in sys.argv:\n        idx = sys.argv.index(\"--\")\n        known_args = sys.argv[1:idx]\n        unknown_args = sys.argv[idx:]\n    else:\n        known_args = sys.argv[1:]\n        unknown_args = []\n\n    parsed, remaining_unknown = parser.parse_known_args(known_args)\n\n    # append any remaining unknown arguments from the initial parsing\n    remaining_unknown.extend(unknown_args)\n\n    args = model_parse(Arguments, vars(parsed))\n    if not args.allow_unknown_args:\n        # we have to parse twice to ensure any unknown arguments\n        # result in an error if that behaviour is desired\n        parser.parse_args()\n\n    return parsed, args, remaining_unknown\n\n\ndef _main() -> None:\n    parser = _build_parser()\n    parsed, args, unknown = _parse_args(parser)\n\n    if args.verbosity != 0:\n        sys.stderr.write(\"Warning: --verbosity isn't supported yet\\n\")\n\n    proxies: ProxiesDict = {}\n    if args.proxy is not None:\n        for proxy in args.proxy:\n            key = \"https://\" if proxy.startswith(\"https\") else \"http://\"\n            if key in proxies:\n                raise CLIError(f\"Multiple {key} proxies given - only the last one would be used\")\n\n            proxies[key] = proxy\n\n    http_client = httpx.Client(\n        proxies=proxies or None,\n        http2=can_use_http2(),\n    )\n    openai.http_client = http_client\n\n    if args.organization:\n        openai.organization = args.organization\n\n    if args.api_key:\n        openai.api_key = args.api_key\n\n    if args.api_base:\n        openai.base_url = args.api_base\n\n    # azure\n    if args.api_type is not None:\n        openai.api_type = args.api_type\n\n    if args.azure_endpoint is not None:\n        openai.azure_endpoint = args.azure_endpoint\n\n    if args.api_version is not None:\n        openai.api_version = args.api_version\n\n    if args.azure_ad_token is not None:\n        openai.azure_ad_token = args.azure_ad_token\n\n    try:\n        if args.args_model:\n            parsed.func(\n                model_parse(\n                    args.args_model,\n                    {\n                        **{\n                            # we omit None values so that they can be defaulted to `NotGiven`\n                            # and we'll strip it from the API request\n                            key: value\n                            for key, value in vars(parsed).items()\n                            if value is not None\n                        },\n                        \"unknown_args\": unknown,\n                    },\n                )\n            )\n        else:\n            parsed.func()\n    finally:\n        try:\n            http_client.close()\n        except Exception:\n            pass\n\n\nif __name__ == \"__main__\":\n    sys.exit(main())\n\n# Path: src/openai/cli/__init__.py\nfrom ._cli import main as main\n\n# Path: src/openai/__main__.py\nfrom .cli import main\n\nmain()\n\n# Path: src/openai/types/beta/thread.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Optional\nfrom typing_extensions import Literal\n\nfrom ..._models import BaseModel\n\n__all__ = [\"Thread\"]\n\n\nclass Thread(BaseModel):\n    id: str\n    \"\"\"The identifier, which can be referenced in API endpoints.\"\"\"\n\n    created_at: int\n    \"\"\"The Unix timestamp (in seconds) for when the thread was created.\"\"\"\n\n    metadata: Optional[object] = None\n    \"\"\"Set of 16 key-value pairs that can be attached to an object.\n\n    This can be useful for storing additional information about the object in a\n    structured format. Keys can be a maximum of 64 characters long and values can be\n    a maxium of 512 characters long.\n    \"\"\"\n\n    object: Literal[\"thread\"]\n    \"\"\"The object type, which is always `thread`.\"\"\"\n\n# Path: src/openai/version.py\nfrom ._version import __version__\n\nVERSION: str = __version__\n\n# Path: src/openai/lib/_old_api.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any\nfrom typing_extensions import override\n\nfrom .._utils import LazyProxy\nfrom .._exceptions import OpenAIError\n\nINSTRUCTIONS = \"\"\"\n\nYou tried to access openai.{symbol}, but this is no longer supported in openai>=1.0.0 - see the README at https://github.com/openai/openai-python for the API.\n\nYou can run `openai migrate` to automatically upgrade your codebase to use the 1.0.0 interface. \n\nAlternatively, you can pin your installation to the old version, e.g. `pip install openai==0.28`\n\nA detailed migration guide is available here: https://github.com/openai/openai-python/discussions/742\n\"\"\"\n\n\nclass APIRemovedInV1(OpenAIError):\n    def __init__(self, *, symbol: str) -> None:\n        super().__init__(INSTRUCTIONS.format(symbol=symbol))\n\n\nclass APIRemovedInV1Proxy(LazyProxy[Any]):\n    def __init__(self, *, symbol: str) -> None:\n        super().__init__()\n        self._symbol = symbol\n\n    @override\n    def __load__(self) -> Any:\n        # return the proxy until it is eventually called so that\n        # we don't break people that are just checking the attributes\n        # of a module\n        return self\n\n...\n", "needle_token_start": 3867, "needle_token_end": 4060, "code_context_ntokens": 15951, "output": ["```python\nclass CLIAudio:\n    @staticmethod\n    def transcribe(args: CLITranscribeArgs) -> None:\n        with open(args.file, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        model = get_client().audio.transcriptions.create(\n            file=(args.file, buffer_reader),\n            model=args.model,\n            language=args.language or NOT_GIVEN,\n            temperature=args.temperature or NOT_GIVEN,\n            prompt=args.prompt or NOT_GIVEN,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(model)\n\n    @staticmethod\n    def translate(args: CLITranslationArgs) -> None:\n        with open(args.file, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        model = get_client().audio.translations.create(\n            file=(args.file, buffer_reader),\n            model=args.model,\n            temperature=args.temperature or NOT_GIVEN,\n            prompt=args.prompt or NOT_GIVEN,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(model)\n```"]}
{"repo": "openai/openai-python", "name": "make_request_options", "language": "python", "path": "src/openai/_base_client.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To construct a dictionary of request options for HTTP operations, filtering out any parameters that are not provided or are marked as not given.\n2. **Input**: Optional parameters including query parameters, additional headers, extra query parameters, extra body content, an idempotency key, a timeout setting, and a post-parser function. Each of these can be omitted or marked as not given.\n3. **Output**: A dictionary tailored for configuring HTTP requests, containing only the parameters that were explicitly provided and valid.\n4. **Procedure**: The function initializes an empty dictionary and conditionally adds entries based on the presence and validity of input parameters. It merges additional query parameters with base query parameters if both are provided, and ensures that no entries for parameters marked as not given are included in the output dictionary.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: Literal[False] = False,\n    ) -> ResponseT:\n        ...\n\n    @overload\n    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: Literal[True],\n        stream_cls: type[_StreamT],\n    ) -> _StreamT:\n        ...\n\n    @overload\n    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: bool,\n        stream_cls: type[_StreamT] | None = None,\n    ) -> ResponseT | _StreamT:\n        ...\n\n    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: bool = False,\n        stream_cls: type[_StreamT] | None = None,\n    ) -> ResponseT | _StreamT:\n        opts = FinalRequestOptions.construct(\n            method=\"post\", url=path, json_data=body, files=to_httpx_files(files), **options\n        )\n        return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n\n    def patch(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n    ) -> ResponseT:\n        opts = FinalRequestOptions.construct(method=\"patch\", url=path, json_data=body, **options)\n        return self.request(cast_to, opts)\n\n    def put(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        files: RequestFiles | None = None,\n        options: RequestOptions = {},\n    ) -> ResponseT:\n        opts = FinalRequestOptions.construct(\n            method=\"put\", url=path, json_data=body, files=to_httpx_files(files), **options\n        )\n        return self.request(cast_to, opts)\n\n    def delete(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n    ) -> ResponseT:\n        opts = FinalRequestOptions.construct(method=\"delete\", url=path, json_data=body, **options)\n        return self.request(cast_to, opts)\n\n    def get_api_list(\n        self,\n        path: str,\n        *,\n        model: Type[object],\n        page: Type[SyncPageT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        method: str = \"get\",\n    ) -> SyncPageT:\n        opts = FinalRequestOptions.construct(method=method, url=path, json_data=body, **options)\n        return self._request_api_list(model, page, opts)\n\n\nclass AsyncHttpxClientWrapper(httpx.AsyncClient):\n    def __del__(self) -> None:\n        try:\n            # TODO(someday): support non asyncio runtimes here\n            asyncio.get_running_loop().create_task(self.aclose())\n        except Exception:\n            pass\n\n\nclass AsyncAPIClient(BaseClient[httpx.AsyncClient, AsyncStream[Any]]):\n    _client: httpx.AsyncClient\n    _default_stream_cls: type[AsyncStream[Any]] | None = None\n\n    def __init__(\n        self,\n        *,\n        version: str,\n        base_url: str | URL,\n        _strict_response_validation: bool,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        timeout: float | Timeout | None | NotGiven = NOT_GIVEN,\n        transport: AsyncTransport | None = None,\n        proxies: ProxiesTypes | None = None,\n        limits: Limits | None = None,\n        http_client: httpx.AsyncClient | None = None,\n        custom_headers: Mapping[str, str] | None = None,\n        custom_query: Mapping[str, object] | None = None,\n    ) -> None:\n        if limits is not None:\n            warnings.warn(\n                \"The `connection_pool_limits` argument is deprecated. The `http_client` argument should be passed instead\",\n                category=DeprecationWarning,\n                stacklevel=3,\n            )\n            if http_client is not None:\n                raise ValueError(\"The `http_client` argument is mutually exclusive with `connection_pool_limits`\")\n        else:\n            limits = DEFAULT_LIMITS\n\n        if transport is not None:\n            warnings.warn(\n                \"The `transport` argument is deprecated. The `http_client` argument should be passed instead\",\n                category=DeprecationWarning,\n                stacklevel=3,\n            )\n            if http_client is not None:\n                raise ValueError(\"The `http_client` argument is mutually exclusive with `transport`\")\n\n        if proxies is not None:\n            warnings.warn(\n                \"The `proxies` argument is deprecated. The `http_client` argument should be passed instead\",\n                category=DeprecationWarning,\n                stacklevel=3,\n            )\n            if http_client is not None:\n                raise ValueError(\"The `http_client` argument is mutually exclusive with `proxies`\")\n\n        if not is_given(timeout):\n            # if the user passed in a custom http client with a non-default\n            # timeout set then we use that timeout.\n            #\n            # note: there is an edge case here where the user passes in a client\n            # where they've explicitly set the timeout to match the default timeout\n            # as this check is structural, meaning that we'll think they didn't\n            # pass in a timeout and will ignore it\n            if http_client and http_client.timeout != HTTPX_DEFAULT_TIMEOUT:\n                timeout = http_client.timeout\n            else:\n                timeout = DEFAULT_TIMEOUT\n\n        super().__init__(\n            version=version,\n            base_url=base_url,\n            limits=limits,\n            # cast to a valid type because mypy doesn't understand our type narrowing\n            timeout=cast(Timeout, timeout),\n            proxies=proxies,\n            transport=transport,\n            max_retries=max_retries,\n            custom_query=custom_query,\n            custom_headers=custom_headers,\n            _strict_response_validation=_strict_response_validation,\n        )\n        self._client = http_client or AsyncHttpxClientWrapper(\n            base_url=base_url,\n            # cast to a valid type because mypy doesn't understand our type narrowing\n            timeout=cast(Timeout, timeout),\n            proxies=proxies,\n            transport=transport,\n            limits=limits,\n            follow_redirects=True,\n        )\n\n    def is_closed(self) -> bool:\n        return self._client.is_closed\n\n    async def close(self) -> None:\n        \"\"\"Close the underlying HTTPX client.\n\n        The client will *not* be usable after this.\n        \"\"\"\n        await self._client.aclose()\n\n    async def __aenter__(self: _T) -> _T:\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        await self.close()\n\n    async def _prepare_options(\n        self,\n        options: FinalRequestOptions,  # noqa: ARG002\n    ) -> None:\n        \"\"\"Hook for mutating the given options\"\"\"\n        return None\n\n    async def _prepare_request(\n        self,\n        request: httpx.Request,  # noqa: ARG002\n    ) -> None:\n        \"\"\"This method is used as a callback for mutating the `Request` object\n        after it has been constructed.\n        This is useful for cases where you want to add certain headers based off of\n        the request properties, e.g. `url`, `method` etc.\n        \"\"\"\n        return None\n\n    @overload\n    async def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        *,\n        stream: Literal[False] = False,\n        remaining_retries: Optional[int] = None,\n    ) -> ResponseT:\n        ...\n\n    @overload\n    async def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        *,\n        stream: Literal[True],\n        stream_cls: type[_AsyncStreamT],\n        remaining_retries: Optional[int] = None,\n    ) -> _AsyncStreamT:\n        ...\n\n    @overload\n    async def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        *,\n        stream: bool,\n        stream_cls: type[_AsyncStreamT] | None = None,\n        remaining_retries: Optional[int] = None,\n    ) -> ResponseT | _AsyncStreamT:\n        ...\n\n    async def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        *,\n        stream: bool = False,\n        stream_cls: type[_AsyncStreamT] | None = None,\n        remaining_retries: Optional[int] = None,\n    ) -> ResponseT | _AsyncStreamT:\n        return await self._request(\n            cast_to=cast_to,\n            options=options,\n            stream=stream,\n            stream_cls=stream_cls,\n            remaining_retries=remaining_retries,\n        )\n\n    async def _request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        *,\n        stream: bool,\n        stream_cls: type[_AsyncStreamT] | None,\n        remaining_retries: int | None,\n    ) -> ResponseT | _AsyncStreamT:\n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        await self._prepare_options(options)\n\n        retries = self._remaining_retries(remaining_retries, options)\n        request = self._build_request(options)\n        await self._prepare_request(request)\n\n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n\n        try:\n            response = await self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n\n            if retries > 0:\n                return await self._retry_request(\n                    options,\n                    cast_to,\n                    retries,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n\n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n\n            if retries > 0:\n                return await self._retry_request(\n                    options,\n                    cast_to,\n                    retries,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n\n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n\n        log.debug(\n            'HTTP Request: %s %s \"%i %s\"', request.method, request.url, response.status_code, response.reason_phrase\n        )\n\n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n\n            if retries > 0 and self._should_retry(err.response):\n                await err.response.aclose()\n                return await self._retry_request(\n                    options,\n                    cast_to,\n                    retries,\n                    err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n\n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                await err.response.aread()\n\n            log.debug(\"Re-raising status error\")\n            raise self._make_status_error_from_response(err.response) from None\n\n        return await self._process_response(\n            cast_to=cast_to,\n            options=options,\n            response=response,\n            stream=stream,\n            stream_cls=stream_cls,\n        )\n\n    async def _retry_request(\n        self,\n        options: FinalRequestOptions,\n        cast_to: Type[ResponseT],\n        remaining_retries: int,\n        response_headers: httpx.Headers | None,\n        *,\n        stream: bool,\n        stream_cls: type[_AsyncStreamT] | None,\n    ) -> ResponseT | _AsyncStreamT:\n        remaining = remaining_retries - 1\n        if remaining == 1:\n            log.debug(\"1 retry left\")\n        else:\n            log.debug(\"%i retries left\", remaining)\n\n        timeout = self._calculate_retry_timeout(remaining, options, response_headers)\n        log.info(\"Retrying request to %s in %f seconds\", options.url, timeout)\n\n        await anyio.sleep(timeout)\n\n        return await self._request(\n            options=options,\n            cast_to=cast_to,\n            remaining_retries=remaining,\n            stream=stream,\n            stream_cls=stream_cls,\n        )\n\n    async def _process_response(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        response: httpx.Response,\n        stream: bool,\n        stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None,\n    ) -> ResponseT:\n        if response.request.headers.get(RAW_RESPONSE_HEADER) == \"true\":\n            return cast(\n                ResponseT,\n                LegacyAPIResponse(\n                    raw=response,\n                    client=self,\n                    cast_to=cast_to,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    options=options,\n                ),\n            )\n\n        origin = get_origin(cast_to) or cast_to\n\n        if inspect.isclass(origin) and issubclass(origin, BaseAPIResponse):\n            if not issubclass(origin, AsyncAPIResponse):\n                raise TypeError(f\"API Response types must subclass {AsyncAPIResponse}; Received {origin}\")\n\n            response_cls = cast(\"type[BaseAPIResponse[Any]]\", cast_to)\n            return cast(\n                \"ResponseT\",\n                response_cls(\n                    raw=response,\n                    client=self,\n                    cast_to=extract_response_type(response_cls),\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    options=options,\n                ),\n            )\n\n        if cast_to == httpx.Response:\n            return cast(ResponseT, response)\n\n        api_response = AsyncAPIResponse(\n            raw=response,\n            client=self,\n            cast_to=cast(\"type[ResponseT]\", cast_to),  # pyright: ignore[reportUnnecessaryCast]\n            stream=stream,\n            stream_cls=stream_cls,\n            options=options,\n        )\n        if bool(response.request.headers.get(RAW_RESPONSE_HEADER)):\n            return cast(ResponseT, api_response)\n\n        return await api_response.parse()\n\n    def _request_api_list(\n        self,\n        model: Type[_T],\n        page: Type[AsyncPageT],\n        options: FinalRequestOptions,\n    ) -> AsyncPaginator[_T, AsyncPageT]:\n        return AsyncPaginator(client=self, options=options, page_cls=page, model=model)\n\n    @overload\n    async def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: Literal[False] = False,\n    ) -> ResponseT:\n        ...\n\n    @overload\n    async def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: Literal[True],\n        stream_cls: type[_AsyncStreamT],\n    ) -> _AsyncStreamT:\n        ...\n\n    @overload\n    async def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: bool,\n        stream_cls: type[_AsyncStreamT] | None = None,\n    ) -> ResponseT | _AsyncStreamT:\n        ...\n\n    async def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: bool = False,\n        stream_cls: type[_AsyncStreamT] | None = None,\n    ) -> ResponseT | _AsyncStreamT:\n        opts = FinalRequestOptions.construct(method=\"get\", url=path, **options)\n        return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)\n\n    @overload\n    async def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        files: RequestFiles | None = None,\n        options: RequestOptions = {},\n        stream: Literal[False] = False,\n    ) -> ResponseT:\n        ...\n\n    @overload\n    async def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        files: RequestFiles | None = None,\n        options: RequestOptions = {},\n        stream: Literal[True],\n        stream_cls: type[_AsyncStreamT],\n    ) -> _AsyncStreamT:\n        ...\n\n    @overload\n    async def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        files: RequestFiles | None = None,\n        options: RequestOptions = {},\n        stream: bool,\n        stream_cls: type[_AsyncStreamT] | None = None,\n    ) -> ResponseT | _AsyncStreamT:\n        ...\n\n    async def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        files: RequestFiles | None = None,\n        options: RequestOptions = {},\n        stream: bool = False,\n        stream_cls: type[_AsyncStreamT] | None = None,\n    ) -> ResponseT | _AsyncStreamT:\n        opts = FinalRequestOptions.construct(\n            method=\"post\", url=path, json_data=body, files=await async_to_httpx_files(files), **options\n        )\n        return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)\n\n    async def patch(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n    ) -> ResponseT:\n        opts = FinalRequestOptions.construct(method=\"patch\", url=path, json_data=body, **options)\n        return await self.request(cast_to, opts)\n\n    async def put(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        files: RequestFiles | None = None,\n        options: RequestOptions = {},\n    ) -> ResponseT:\n        opts = FinalRequestOptions.construct(\n            method=\"put\", url=path, json_data=body, files=await async_to_httpx_files(files), **options\n        )\n        return await self.request(cast_to, opts)\n\n    async def delete(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n    ) -> ResponseT:\n        opts = FinalRequestOptions.construct(method=\"delete\", url=path, json_data=body, **options)\n        return await self.request(cast_to, opts)\n\n    def get_api_list(\n        self,\n        path: str,\n        *,\n        model: Type[_T],\n        page: Type[AsyncPageT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        method: str = \"get\",\n    ) -> AsyncPaginator[_T, AsyncPageT]:\n        opts = FinalRequestOptions.construct(method=method, url=path, json_data=body, **options)\n        return self._request_api_list(model, page, opts)\n\n\n\ndef make_request_options(\n    *,\n    query: Query | None = None,\n    extra_headers: Headers | None = None,\n    extra_query: Query | None = None,\n    extra_body: Body | None = None,\n    idempotency_key: str | None = None,\n    timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,\n    post_parser: PostParser | NotGiven = NOT_GIVEN,\n) -> RequestOptions:\n    \"\"\"Create a dict of type RequestOptions without keys of NotGiven values.\"\"\"\n    options: RequestOptions = {}\n    if extra_headers is not None:\n        options[\"headers\"] = extra_headers\n\n    if extra_body is not None:\n        options[\"extra_json\"] = cast(AnyMapping, extra_body)\n\n    if query is not None:\n        options[\"params\"] = query\n\n    if extra_query is not None:\n        options[\"params\"] = {**options.get(\"params\", {}), **extra_query}\n\n    if not isinstance(timeout, NotGiven):\n        options[\"timeout\"] = timeout\n\n    if idempotency_key is not None:\n        options[\"idempotency_key\"] = idempotency_key\n\n    if is_given(post_parser):\n        # internal\n        options[\"post_parser\"] = post_parser  # type: ignore\n\n    return options\n\n\nclass OtherPlatform:\n    def __init__(self, name: str) -> None:\n        self.name = name\n\n    @override\n    def __str__(self) -> str:\n        return f\"Other:{self.name}\"\n\n\nPlatform = Union[\n    OtherPlatform,\n    Literal[\n        \"MacOS\",\n        \"Linux\",\n        \"Windows\",\n        \"FreeBSD\",\n        \"OpenBSD\",\n        \"iOS\",\n        \"Android\",\n        \"Unknown\",\n    ],\n]\n\n\ndef get_platform() -> Platform:\n    try:\n        system = platform.system().lower()\n        platform_name = platform.platform().lower()\n    except Exception:\n        return \"Unknown\"\n\n    if \"iphone\" in platform_name or \"ipad\" in platform_name:\n        # Tested using Python3IDE on an iPhone 11 and Pythonista on an iPad 7\n        # system is Darwin and platform_name is a string like:\n        # - Darwin-21.6.0-iPhone12,1-64bit\n        # - Darwin-21.6.0-iPad7,11-64bit\n        return \"iOS\"\n\n    if system == \"darwin\":\n        return \"MacOS\"\n\n    if system == \"windows\":\n        return \"Windows\"\n\n    if \"android\" in platform_name:\n        # Tested using Pydroid 3\n        # system is Linux and platform_name is a string like 'Linux-5.10.81-android12-9-00001-geba40aecb3b7-ab8534902-aarch64-with-libc'\n        return \"Android\"\n\n    if system == \"linux\":\n        # https://distro.readthedocs.io/en/latest/#distro.id\n        distro_id = distro.id()\n        if distro_id == \"freebsd\":\n            return \"FreeBSD\"\n\n        if distro_id == \"openbsd\":\n            return \"OpenBSD\"\n\n        return \"Linux\"\n\n    if platform_name:\n        return OtherPlatform(platform_name)\n\n    return \"Unknown\"\n\n\n@lru_cache(maxsize=None)\ndef platform_headers(version: str) -> Dict[str, str]:\n    return {\n        \"X-Stainless-Lang\": \"python\",\n        \"X-Stainless-Package-Version\": version,\n        \"X-Stainless-OS\": str(get_platform()),\n        \"X-Stainless-Arch\": str(get_architecture()),\n        \"X-Stainless-Runtime\": get_python_runtime(),\n        \"X-Stainless-Runtime-Version\": get_python_version(),\n    }\n\n\nclass OtherArch:\n    def __init__(self, name: str) -> None:\n        self.name = name\n\n    @override\n    def __str__(self) -> str:\n        return f\"other:{self.name}\"\n\n\nArch = Union[OtherArch, Literal[\"x32\", \"x64\", \"arm\", \"arm64\", \"unknown\"]]\n\n\ndef get_python_runtime() -> str:\n    try:\n        return platform.python_implementation()\n    except Exception:\n        return \"unknown\"\n\n\ndef get_python_version() -> str:\n    try:\n        return platform.python_version()\n    except Exception:\n        return \"unknown\"\n\n\ndef get_architecture() -> Arch:\n    try:\n        python_bitness, _ = platform.architecture()\n        machine = platform.machine().lower()\n    except Exception:\n        return \"unknown\"\n\n    if machine in (\"arm64\", \"aarch64\"):\n        return \"arm64\"\n\n    # TODO: untested\n    if machine == \"arm\":\n        return \"arm\"\n\n    if machine == \"x86_64\":\n        return \"x64\"\n\n    # TODO: untested\n    if python_bitness == \"32bit\":\n        return \"x32\"\n\n    if machine:\n        return OtherArch(machine)\n\n    return \"unknown\"\n\n\ndef _merge_mappings(\n    obj1: Mapping[_T_co, Union[_T, Omit]],\n    obj2: Mapping[_T_co, Union[_T, Omit]],\n) -> Dict[_T_co, _T]:\n    \"\"\"Merge two mappings of the same type, removing any values that are instances of `Omit`.\n\n    In cases with duplicate keys the second mapping takes precedence.\n    \"\"\"\n    merged = {**obj1, **obj2}\n    return {key: value for key, value in merged.items() if not isinstance(value, Omit)}\n\n# Path: src/openai/_module_client.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing_extensions import override\n\nfrom . import resources, _load_client\nfrom ._utils import LazyProxy\n\n\nclass ChatProxy(LazyProxy[resources.Chat]):\n    @override\n    def __load__(self) -> resources.Chat:\n        return _load_client().chat\n\n\nclass BetaProxy(LazyProxy[resources.Beta]):\n    @override\n    def __load__(self) -> resources.Beta:\n        return _load_client().beta\n\n\nclass FilesProxy(LazyProxy[resources.Files]):\n    @override\n    def __load__(self) -> resources.Files:\n        return _load_client().files\n\n\nclass AudioProxy(LazyProxy[resources.Audio]):\n    @override\n    def __load__(self) -> resources.Audio:\n        return _load_client().audio\n\n\nclass ImagesProxy(LazyProxy[resources.Images]):\n    @override\n    def __load__(self) -> resources.Images:\n        return _load_client().images\n\n\nclass ModelsProxy(LazyProxy[resources.Models]):\n    @override\n    def __load__(self) -> resources.Models:\n        return _load_client().models\n\n\nclass EmbeddingsProxy(LazyProxy[resources.Embeddings]):\n    @override\n    def __load__(self) -> resources.Embeddings:\n        return _load_client().embeddings\n\n\nclass CompletionsProxy(LazyProxy[resources.Completions]):\n    @override\n    def __load__(self) -> resources.Completions:\n        return _load_client().completions\n\n\nclass ModerationsProxy(LazyProxy[resources.Moderations]):\n    @override\n    def __load__(self) -> resources.Moderations:\n        return _load_client().moderations\n\n\nclass FineTuningProxy(LazyProxy[resources.FineTuning]):\n    @override\n    def __load__(self) -> resources.FineTuning:\n        return _load_client().fine_tuning\n\n\nchat: resources.Chat = ChatProxy().__as_proxied__()\nbeta: resources.Beta = BetaProxy().__as_proxied__()\nfiles: resources.Files = FilesProxy().__as_proxied__()\naudio: resources.Audio = AudioProxy().__as_proxied__()\nimages: resources.Images = ImagesProxy().__as_proxied__()\nmodels: resources.Models = ModelsProxy().__as_proxied__()\nembeddings: resources.Embeddings = EmbeddingsProxy().__as_proxied__()\ncompletions: resources.Completions = CompletionsProxy().__as_proxied__()\nmoderations: resources.Moderations = ModerationsProxy().__as_proxied__()\nfine_tuning: resources.FineTuning = FineTuningProxy().__as_proxied__()\n\n# Path: src/openai/_utils/_logs.py\nimport os\nimport logging\n\nlogger: logging.Logger = logging.getLogger(\"openai\")\nhttpx_logger: logging.Logger = logging.getLogger(\"httpx\")\n\n\ndef _basic_config() -> None:\n    # e.g. [2023-10-05 14:12:26 - openai._base_client:818 - DEBUG] HTTP Request: POST http://127.0.0.1:4010/foo/bar \"200 OK\"\n    logging.basicConfig(\n        format=\"[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s\",\n        datefmt=\"%Y-%m-%d %H:%M:%S\",\n    )\n\n\ndef setup_logging() -> None:\n    env = os.environ.get(\"OPENAI_LOG\")\n    if env == \"debug\":\n        _basic_config()\n        logger.setLevel(logging.DEBUG)\n        httpx_logger.setLevel(logging.DEBUG)\n    elif env == \"info\":\n        _basic_config()\n        logger.setLevel(logging.INFO)\n        httpx_logger.setLevel(logging.INFO)\n\n# Path: src/openai/__init__.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nimport os as _os\nfrom typing_extensions import override\n\nfrom . import types\nfrom ._types import NoneType, Transport, ProxiesTypes\nfrom ._utils import file_from_path\nfrom ._client import Client, OpenAI, Stream, Timeout, Transport, AsyncClient, AsyncOpenAI, AsyncStream, RequestOptions\nfrom ._models import BaseModel\nfrom ._version import __title__, __version__\nfrom ._response import APIResponse as APIResponse, AsyncAPIResponse as AsyncAPIResponse\nfrom ._exceptions import (\n    APIError,\n    OpenAIError,\n    ConflictError,\n    NotFoundError,\n    APIStatusError,\n    RateLimitError,\n    APITimeoutError,\n    BadRequestError,\n    APIConnectionError,\n    AuthenticationError,\n    InternalServerError,\n    PermissionDeniedError,\n    UnprocessableEntityError,\n    APIResponseValidationError,\n)\nfrom ._utils._logs import setup_logging as _setup_logging\n\n__all__ = [\n    \"types\",\n    \"__version__\",\n    \"__title__\",\n    \"NoneType\",\n    \"Transport\",\n    \"ProxiesTypes\",\n    \"OpenAIError\",\n    \"APIError\",\n    \"APIStatusError\",\n    \"APITimeoutError\",\n    \"APIConnectionError\",\n    \"APIResponseValidationError\",\n    \"BadRequestError\",\n    \"AuthenticationError\",\n    \"PermissionDeniedError\",\n    \"NotFoundError\",\n    \"ConflictError\",\n    \"UnprocessableEntityError\",\n    \"RateLimitError\",\n    \"InternalServerError\",\n    \"Timeout\",\n    \"RequestOptions\",\n    \"Client\",\n    \"AsyncClient\",\n    \"Stream\",\n    \"AsyncStream\",\n    \"OpenAI\",\n    \"AsyncOpenAI\",\n    \"file_from_path\",\n    \"BaseModel\",\n]\n\nfrom .lib import azure as _azure\nfrom .version import VERSION as VERSION\nfrom .lib.azure import AzureOpenAI as AzureOpenAI, AsyncAzureOpenAI as AsyncAzureOpenAI\nfrom .lib._old_api import *\n\n_setup_logging()\n\n# Update the __module__ attribute for exported symbols so that\n# error messages point to this module instead of the module\n# it was originally defined in, e.g.\n# openai._exceptions.NotFoundError -> openai.NotFoundError\n__locals = locals()\nfor __name in __all__:\n    if not __name.startswith(\"__\"):\n        try:\n            __locals[__name].__module__ = \"openai\"\n        except (TypeError, AttributeError):\n            # Some of our exported symbols are builtins which we can't set attributes for.\n            pass\n\n# ------ Module level client ------\nimport typing as _t\nimport typing_extensions as _te\n\nimport httpx as _httpx\n\nfrom ._base_client import DEFAULT_TIMEOUT, DEFAULT_MAX_RETRIES\n\napi_key: str | None = None\n\norganization: str | None = None\n\nbase_url: str | _httpx.URL | None = None\n\ntimeout: float | Timeout | None = DEFAULT_TIMEOUT\n\nmax_retries: int = DEFAULT_MAX_RETRIES\n\ndefault_headers: _t.Mapping[str, str] | None = None\n\ndefault_query: _t.Mapping[str, object] | None = None\n\nhttp_client: _httpx.Client | None = None\n\n_ApiType = _te.Literal[\"openai\", \"azure\"]\n\napi_type: _ApiType | None = _t.cast(_ApiType, _os.environ.get(\"OPENAI_API_TYPE\"))\n\napi_version: str | None = _os.environ.get(\"OPENAI_API_VERSION\")\n\nazure_endpoint: str | None = _os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n\nazure_ad_token: str | None = _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\n\nazure_ad_token_provider: _azure.AzureADTokenProvider | None = None\n\n\nclass _ModuleClient(OpenAI):\n    # Note: we have to use type: ignores here as overriding class members\n    # with properties is technically unsafe but it is fine for our use case\n\n    @property  # type: ignore\n    @override\n    def api_key(self) -> str | None:\n        return api_key\n\n    @api_key.setter  # type: ignore\n    def api_key(self, value: str | None) -> None:  # type: ignore\n        global api_key\n\n        api_key = value\n\n    @property  # type: ignore\n    @override\n    def organization(self) -> str | None:\n        return organization\n\n    @organization.setter  # type: ignore\n    def organization(self, value: str | None) -> None:  # type: ignore\n        global organization\n\n        organization = value\n\n    @property\n    @override\n    def base_url(self) -> _httpx.URL:\n        if base_url is not None:\n            return _httpx.URL(base_url)\n\n        return super().base_url\n\n    @base_url.setter\n    def base_url(self, url: _httpx.URL | str) -> None:\n        super().base_url = url  # type: ignore[misc]\n\n    @property  # type: ignore\n    @override\n    def timeout(self) -> float | Timeout | None:\n        return timeout\n\n    @timeout.setter  # type: ignore\n    def timeout(self, value: float | Timeout | None) -> None:  # type: ignore\n        global timeout\n\n        timeout = value\n\n    @property  # type: ignore\n    @override\n    def max_retries(self) -> int:\n        return max_retries\n\n    @max_retries.setter  # type: ignore\n    def max_retries(self, value: int) -> None:  # type: ignore\n        global max_retries\n\n        max_retries = value\n\n    @property  # type: ignore\n    @override\n    def _custom_headers(self) -> _t.Mapping[str, str] | None:\n        return default_headers\n\n    @_custom_headers.setter  # type: ignore\n    def _custom_headers(self, value: _t.Mapping[str, str] | None) -> None:  # type: ignore\n        global default_headers\n\n        default_headers = value\n\n    @property  # type: ignore\n    @override\n    def _custom_query(self) -> _t.Mapping[str, object] | None:\n        return default_query\n\n    @_custom_query.setter  # type: ignore\n    def _custom_query(self, value: _t.Mapping[str, object] | None) -> None:  # type: ignore\n        global default_query\n\n        default_query = value\n\n    @property  # type: ignore\n    @override\n    def _client(self) -> _httpx.Client:\n        return http_client or super()._client\n\n    @_client.setter  # type: ignore\n    def _client(self, value: _httpx.Client) -> None:  # type: ignore\n        global http_client\n\n        http_client = value\n\n\nclass _AzureModuleClient(_ModuleClient, AzureOpenAI):  # type: ignore\n    ...\n\n\nclass _AmbiguousModuleClientUsageError(OpenAIError):\n    def __init__(self) -> None:\n        super().__init__(\n            \"Ambiguous use of module client; please set `openai.api_type` or the `OPENAI_API_TYPE` environment variable to `openai` or `azure`\"\n        )\n\n\ndef _has_openai_credentials() -> bool:\n    return _os.environ.get(\"OPENAI_API_KEY\") is not None\n\n\ndef _has_azure_credentials() -> bool:\n    return azure_endpoint is not None or _os.environ.get(\"AZURE_OPENAI_API_KEY\") is not None\n\n\ndef _has_azure_ad_credentials() -> bool:\n    return (\n        _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\") is not None\n        or azure_ad_token is not None\n        or azure_ad_token_provider is not None\n    )\n\n\n_client: OpenAI | None = None\n\n\ndef _load_client() -> OpenAI:  # type: ignore[reportUnusedFunction]\n    global _client\n\n    if _client is None:\n        global api_type, azure_endpoint, azure_ad_token, api_version\n\n        if azure_endpoint is None:\n            azure_endpoint = _os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n\n        if azure_ad_token is None:\n            azure_ad_token = _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\n\n        if api_version is None:\n            api_version = _os.environ.get(\"OPENAI_API_VERSION\")\n\n        if api_type is None:\n            has_openai = _has_openai_credentials()\n            has_azure = _has_azure_credentials()\n            has_azure_ad = _has_azure_ad_credentials()\n\n            if has_openai and (has_azure or has_azure_ad):\n                raise _AmbiguousModuleClientUsageError()\n\n            if (azure_ad_token is not None or azure_ad_token_provider is not None) and _os.environ.get(\n                \"AZURE_OPENAI_API_KEY\"\n            ) is not None:\n                raise _AmbiguousModuleClientUsageError()\n\n            if has_azure or has_azure_ad:\n                api_type = \"azure\"\n            else:\n                api_type = \"openai\"\n\n        if api_type == \"azure\":\n            _client = _AzureModuleClient(  # type: ignore\n                api_version=api_version,\n                azure_endpoint=azure_endpoint,\n                api_key=api_key,\n                azure_ad_token=azure_ad_token,\n                azure_ad_token_provider=azure_ad_token_provider,\n                organization=organization,\n                base_url=base_url,\n                timeout=timeout,\n                max_retries=max_retries,\n                default_headers=default_headers,\n                default_query=default_query,\n                http_client=http_client,\n            )\n            return _client\n\n        _client = _ModuleClient(\n            api_key=api_key,\n            organization=organization,\n            base_url=base_url,\n            timeout=timeout,\n            max_retries=max_retries,\n            default_headers=default_headers,\n            default_query=default_query,\n            http_client=http_client,\n        )\n        return _client\n\n    return _client\n\n\ndef _reset_client() -> None:  # type: ignore[reportUnusedFunction]\n    global _client\n\n    _client = None\n\n\nfrom ._module_client import (\n    beta as beta,\n    chat as chat,\n    audio as audio,\n    files as files,\n    images as images,\n    models as models,\n    embeddings as embeddings,\n    completions as completions,\n    fine_tuning as fine_tuning,\n    moderations as moderations,\n)\n\n# Path: src/openai/cli/_models.py\nfrom typing import Any\nfrom typing_extensions import ClassVar\n\nimport pydantic\n\nfrom .. import _models\nfrom .._compat import PYDANTIC_V2, ConfigDict\n\n\nclass BaseModel(_models.BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(extra=\"ignore\", arbitrary_types_allowed=True)\n    else:\n\n        class Config(pydantic.BaseConfig):  # type: ignore\n            extra: Any = pydantic.Extra.ignore  # type: ignore\n            arbitrary_types_allowed: bool = True\n\n# Path: src/openai/cli/_progress.py\nfrom __future__ import annotations\n\nimport io\nfrom typing import Callable\nfrom typing_extensions import override\n\n\nclass CancelledError(Exception):\n    def __init__(self, msg: str) -> None:\n        self.msg = msg\n        super().__init__(msg)\n\n    @override\n    def __str__(self) -> str:\n        return self.msg\n\n    __repr__ = __str__\n\n\nclass BufferReader(io.BytesIO):\n    def __init__(self, buf: bytes = b\"\", desc: str | None = None) -> None:\n        super().__init__(buf)\n        self._len = len(buf)\n        self._progress = 0\n        self._callback = progress(len(buf), desc=desc)\n\n    def __len__(self) -> int:\n        return self._len\n\n    @override\n    def read(self, n: int | None = -1) -> bytes:\n        chunk = io.BytesIO.read(self, n)\n        self._progress += len(chunk)\n\n        try:\n            self._callback(self._progress)\n        except Exception as e:  # catches exception from the callback\n            raise CancelledError(\"The upload was cancelled: {}\".format(e)) from e\n\n        return chunk\n\n\ndef progress(total: float, desc: str | None) -> Callable[[float], None]:\n    import tqdm\n\n    meter = tqdm.tqdm(total=total, unit_scale=True, desc=desc)\n\n    def incr(progress: float) -> None:\n        meter.n = progress\n        if progress == total:\n            meter.close()\n        else:\n            meter.refresh()\n\n    return incr\n\n\ndef MB(i: int) -> int:\n    return int(i // 1024**2)\n\n# Path: src/openai/cli/_utils.py\nfrom __future__ import annotations\n\nimport sys\n\nimport openai\n\nfrom .. import OpenAI, _load_client\nfrom .._compat import model_json\nfrom .._models import BaseModel\n\n\nclass Colors:\n    HEADER = \"\\033[95m\"\n    OKBLUE = \"\\033[94m\"\n    OKGREEN = \"\\033[92m\"\n    WARNING = \"\\033[93m\"\n    FAIL = \"\\033[91m\"\n    ENDC = \"\\033[0m\"\n    BOLD = \"\\033[1m\"\n    UNDERLINE = \"\\033[4m\"\n\n\ndef get_client() -> OpenAI:\n    return _load_client()\n\n\ndef organization_info() -> str:\n    organization = openai.organization\n    if organization is not None:\n        return \"[organization={}] \".format(organization)\n\n    return \"\"\n\n\ndef print_model(model: BaseModel) -> None:\n    sys.stdout.write(model_json(model, indent=2) + \"\\n\")\n\n\ndef can_use_http2() -> bool:\n    try:\n        import h2  # type: ignore  # noqa\n    except ImportError:\n        return False\n\n    return True\n\n# Path: src/openai/cli/_api/audio.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any, Optional, cast\nfrom argparse import ArgumentParser\n\nfrom .._utils import get_client, print_model\nfrom ..._types import NOT_GIVEN\nfrom .._models import BaseModel\nfrom .._progress import BufferReader\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    # transcriptions\n    sub = subparser.add_parser(\"audio.transcriptions.create\")\n\n    # Required\n    sub.add_argument(\"-m\", \"--model\", type=str, default=\"whisper-1\")\n    sub.add_argument(\"-f\", \"--file\", type=str, required=True)\n    # Optional\n    sub.add_argument(\"--response-format\", type=str)\n    sub.add_argument(\"--language\", type=str)\n    sub.add_argument(\"-t\", \"--temperature\", type=float)\n    sub.add_argument(\"--prompt\", type=str)\n    sub.set_defaults(func=CLIAudio.transcribe, args_model=CLITranscribeArgs)\n\n    # translations\n    sub = subparser.add_parser(\"audio.translations.create\")\n\n    # Required\n    sub.add_argument(\"-f\", \"--file\", type=str, required=True)\n    # Optional\n    sub.add_argument(\"-m\", \"--model\", type=str, default=\"whisper-1\")\n    sub.add_argument(\"--response-format\", type=str)\n    # TODO: doesn't seem to be supported by the API\n    # sub.add_argument(\"--language\", type=str)\n    sub.add_argument(\"-t\", \"--temperature\", type=float)\n    sub.add_argument(\"--prompt\", type=str)\n    sub.set_defaults(func=CLIAudio.translate, args_model=CLITranslationArgs)\n\n\nclass CLITranscribeArgs(BaseModel):\n    model: str\n    file: str\n    response_format: Optional[str] = None\n    language: Optional[str] = None\n    temperature: Optional[float] = None\n    prompt: Optional[str] = None\n\n\nclass CLITranslationArgs(BaseModel):\n    model: str\n    file: str\n    response_format: Optional[str] = None\n    language: Optional[str] = None\n    temperature: Optional[float] = None\n    prompt: Optional[str] = None\n\n\nclass CLIAudio:\n    @staticmethod\n    def transcribe(args: CLITranscribeArgs) -> None:\n        with open(args.file, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        model = get_client().audio.transcriptions.create(\n            file=(args.file, buffer_reader),\n            model=args.model,\n            language=args.language or NOT_GIVEN,\n            temperature=args.temperature or NOT_GIVEN,\n            prompt=args.prompt or NOT_GIVEN,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(model)\n\n    @staticmethod\n    def translate(args: CLITranslationArgs) -> None:\n        with open(args.file, \"rb\") as file_reader:\n            buffer_reader = BufferReader(file_reader.read(), desc=\"Upload progress\")\n\n        model = get_client().audio.translations.create(\n            file=(args.file, buffer_reader),\n            model=args.model,\n            temperature=args.temperature or NOT_GIVEN,\n            prompt=args.prompt or NOT_GIVEN,\n            # casts required because the API is typed for enums\n            # but we don't want to validate that here for forwards-compat\n            response_format=cast(Any, args.response_format),\n        )\n        print_model(model)\n\n# Path: src/openai/cli/_api/chat/completions.py\nfrom __future__ import annotations\n\nimport sys\nfrom typing import TYPE_CHECKING, List, Optional, cast\nfrom argparse import ArgumentParser\nfrom typing_extensions import Literal, NamedTuple\n\nfrom ..._utils import get_client\nfrom ..._models import BaseModel\nfrom ...._streaming import Stream\nfrom ....types.chat import (\n    ChatCompletionRole,\n    ChatCompletionChunk,\n    CompletionCreateParams,\n)\nfrom ....types.chat.completion_create_params import (\n    CompletionCreateParamsStreaming,\n    CompletionCreateParamsNonStreaming,\n)\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"chat.completions.create\")\n\n    sub._action_groups.pop()\n    req = sub.add_argument_group(\"required arguments\")\n    opt = sub.add_argument_group(\"optional arguments\")\n\n    req.add_argument(\n        \"-g\",\n        \"--message\",\n        action=\"append\",\n        nargs=2,\n        metavar=(\"ROLE\", \"CONTENT\"),\n        help=\"A message in `{role} {content}` format. Use this argument multiple times to add multiple messages.\",\n        required=True,\n    )\n    req.add_argument(\n        \"-m\",\n        \"--model\",\n        help=\"The model to use.\",\n        required=True,\n    )\n\n    opt.add_argument(\n        \"-n\",\n        \"--n\",\n        help=\"How many completions to generate for the conversation.\",\n        type=int,\n    )\n    opt.add_argument(\"-M\", \"--max-tokens\", help=\"The maximum number of tokens to generate.\", type=int)\n    opt.add_argument(\n        \"-t\",\n        \"--temperature\",\n        help=\"\"\"What sampling temperature to use. Higher values means the model will take more risks. Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer.\n\nMutually exclusive with `top_p`.\"\"\",\n        type=float,\n    )\n    opt.add_argument(\n        \"-P\",\n        \"--top_p\",\n        help=\"\"\"An alternative to sampling with temperature, called nucleus sampling, where the considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10%% probability mass are considered.\n\n            Mutually exclusive with `temperature`.\"\"\",\n        type=float,\n    )\n    opt.add_argument(\n        \"--stop\",\n        help=\"A stop sequence at which to stop generating tokens for the message.\",\n    )\n    opt.add_argument(\"--stream\", help=\"Stream messages as they're ready.\", action=\"store_true\")\n    sub.set_defaults(func=CLIChatCompletion.create, args_model=CLIChatCompletionCreateArgs)\n\n\nclass CLIMessage(NamedTuple):\n    role: ChatCompletionRole\n    content: str\n\n\nclass CLIChatCompletionCreateArgs(BaseModel):\n    message: List[CLIMessage]\n    model: str\n    n: Optional[int] = None\n    max_tokens: Optional[int] = None\n    temperature: Optional[float] = None\n    top_p: Optional[float] = None\n    stop: Optional[str] = None\n    stream: bool = False\n\n\nclass CLIChatCompletion:\n    @staticmethod\n    def create(args: CLIChatCompletionCreateArgs) -> None:\n        params: CompletionCreateParams = {\n            \"model\": args.model,\n            \"messages\": [\n                {\"role\": cast(Literal[\"user\"], message.role), \"content\": message.content} for message in args.message\n            ],\n            \"n\": args.n,\n            \"temperature\": args.temperature,\n            \"top_p\": args.top_p,\n            \"stop\": args.stop,\n            # type checkers are not good at inferring union types so we have to set stream afterwards\n            \"stream\": False,\n        }\n        if args.stream:\n            params[\"stream\"] = args.stream  # type: ignore\n        if args.max_tokens is not None:\n            params[\"max_tokens\"] = args.max_tokens\n\n        if args.stream:\n            return CLIChatCompletion._stream_create(cast(CompletionCreateParamsStreaming, params))\n\n        return CLIChatCompletion._create(cast(CompletionCreateParamsNonStreaming, params))\n\n    @staticmethod\n    def _create(params: CompletionCreateParamsNonStreaming) -> None:\n        completion = get_client().chat.completions.create(**params)\n        should_print_header = len(completion.choices) > 1\n        for choice in completion.choices:\n            if should_print_header:\n                sys.stdout.write(\"===== Chat Completion {} =====\\n\".format(choice.index))\n\n            content = choice.message.content if choice.message.content is not None else \"None\"\n            sys.stdout.write(content)\n\n            if should_print_header or not content.endswith(\"\\n\"):\n                sys.stdout.write(\"\\n\")\n\n            sys.stdout.flush()\n\n    @staticmethod\n    def _stream_create(params: CompletionCreateParamsStreaming) -> None:\n        # cast is required for mypy\n        stream = cast(  # pyright: ignore[reportUnnecessaryCast]\n            Stream[ChatCompletionChunk], get_client().chat.completions.create(**params)\n        )\n        for chunk in stream:\n            should_print_header = len(chunk.choices) > 1\n            for choice in chunk.choices:\n                if should_print_header:\n                    sys.stdout.write(\"===== Chat Completion {} =====\\n\".format(choice.index))\n\n                content = choice.delta.content or \"\"\n                sys.stdout.write(content)\n\n                if should_print_header:\n                    sys.stdout.write(\"\\n\")\n\n                sys.stdout.flush()\n\n        sys.stdout.write(\"\\n\")\n\n# Path: src/openai/cli/_api/chat/__init__.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING\nfrom argparse import ArgumentParser\n\nfrom . import completions\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    completions.register(subparser)\n\n# Path: src/openai/cli/_errors.py\nfrom __future__ import annotations\n\nimport sys\n\nimport pydantic\n\nfrom ._utils import Colors, organization_info\nfrom .._exceptions import APIError, OpenAIError\n\n\nclass CLIError(OpenAIError):\n    ...\n\n\nclass SilentCLIError(CLIError):\n    ...\n\n\ndef display_error(err: CLIError | APIError | pydantic.ValidationError) -> None:\n    if isinstance(err, SilentCLIError):\n        return\n\n    sys.stderr.write(\"{}{}Error:{} {}\\n\".format(organization_info(), Colors.FAIL, Colors.ENDC, err))\n\n# Path: src/openai/cli/_api/completions.py\nfrom __future__ import annotations\n\nimport sys\nfrom typing import TYPE_CHECKING, Optional, cast\nfrom argparse import ArgumentParser\nfrom functools import partial\n\nfrom openai.types.completion import Completion\n\nfrom .._utils import get_client\nfrom ..._types import NOT_GIVEN, NotGivenOr\nfrom ..._utils import is_given\nfrom .._errors import CLIError\nfrom .._models import BaseModel\nfrom ..._streaming import Stream\n\nif TYPE_CHECKING:\n    from argparse import _SubParsersAction\n\n\ndef register(subparser: _SubParsersAction[ArgumentParser]) -> None:\n    sub = subparser.add_parser(\"completions.create\")\n\n    # Required\n    sub.add_argument(\n        \"-m\",\n        \"--model\",\n        help=\"The model to use\",\n        required=True,\n    )\n\n    # Optional\n    sub.add_argument(\"-p\", \"--prompt\", help=\"An optional prompt to complete from\")\n    sub.add_argument(\"--stream\", help=\"Stream tokens as they're ready.\", action=\"store_true\")\n    sub.add_argument(\"-M\", \"--max-tokens\", help=\"The maximum number of tokens to generate\", type=int)\n    sub.add_argument(\n        \"-t\",\n        \"--temperature\",\n        help=\"\"\"What sampling temperature to use. Higher values means the model will take more risks. Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer.\n\nMutually exclusive with `top_p`.\"\"\",\n        type=float,\n    )\n    sub.add_argument(\n        \"-P\",\n        \"--top_p\",\n        help=\"\"\"An alternative to sampling with temperature, called nucleus sampling, where the considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10%% probability mass are considered.\n\n            Mutually exclusive with `temperature`.\"\"\",\n        type=float,\n    )\n    sub.add_argument(\n        \"-n\",\n        \"--n\",\n        help=\"How many sub-completions to generate for each prompt.\",\n        type=int,\n    )\n    sub.add_argument(\n        \"--logprobs\",\n        help=\"Include the log probabilities on the `logprobs` most likely tokens, as well the chosen tokens. So for example, if `logprobs` is 10, the API will return a list of the 10 most likely tokens. If `logprobs` is 0, only the chosen tokens will have logprobs returned.\",\n        type=int,\n    )\n    sub.add_argument(\n        \"--best_of\",\n        help=\"Generates `best_of` completions server-side and returns the 'best' (the one with the highest log probability per token). Results cannot be streamed.\",\n        type=int,\n    )\n    sub.add_argument(\n        \"--echo\",\n        help=\"Echo back the prompt in addition to the completion\",\n        action=\"store_true\",\n    )\n    sub.add_argument(\n        \"--frequency_penalty\",\n        help=\"Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.\",\n        type=float,\n    )\n    sub.add_argument(\n        \"--presence_penalty\",\n        help=\"Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.\",\n        type=float,\n    )\n    sub.add_argument(\"--suffix\", help=\"The suffix that comes after a completion of inserted text.\")\n    sub.add_argument(\"--stop\", help=\"A stop sequence at which to stop generating tokens.\")\n    sub.add_argument(\n        \"--user\",\n        help=\"A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse.\",\n    )\n    # TODO: add support for logit_bias\n    sub.set_defaults(func=CLICompletions.create, args_model=CLICompletionCreateArgs)\n\n\nclass CLICompletionCreateArgs(BaseModel):\n    model: str\n    stream: bool = False\n\n    prompt: Optional[str] = None\n    n: NotGivenOr[int] = NOT_GIVEN\n    stop: NotGivenOr[str] = NOT_GIVEN\n    user: NotGivenOr[str] = NOT_GIVEN\n    echo: NotGivenOr[bool] = NOT_GIVEN\n    suffix: NotGivenOr[str] = NOT_GIVEN\n    best_of: NotGivenOr[int] = NOT_GIVEN\n    top_p: NotGivenOr[float] = NOT_GIVEN\n    logprobs: NotGivenOr[int] = NOT_GIVEN\n    max_tokens: NotGivenOr[int] = NOT_GIVEN\n    temperature: NotGivenOr[float] = NOT_GIVEN\n    presence_penalty: NotGivenOr[float] = NOT_GIVEN\n    frequency_penalty: NotGivenOr[float] = NOT_GIVEN\n\n\nclass CLICompletions:\n    @staticmethod\n    def create(args: CLICompletionCreateArgs) -> None:\n        if is_given(args.n) and args.n > 1 and args.stream:\n            raise CLIError(\"Can't stream completions with n>1 with the current CLI\")\n\n        make_request = partial(\n            get_client().completions.create,\n            n=args.n,\n            echo=args.echo,\n            stop=args.stop,\n...\n", "needle_token_start": 5496, "needle_token_end": 5836, "code_context_ntokens": 16052, "output": ["```python\ndef make_request_options(\n    *,\n    query: Query | None = None,\n    extra_headers: Headers | None = None,\n    extra_query: Query | None = None,\n    extra_body: Body | None = None,\n    idempotency_key: str | None = None,\n    timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,\n    post_parser: PostParser | NotGiven = NOT_GIVEN,\n) -> RequestOptions:\n    \"\"\"Create a dict of type RequestOptions without keys of NotGiven values.\"\"\"\n    options: RequestOptions = {}\n    if extra_headers is not None:\n        options[\"headers\"] = extra_headers\n\n    if extra_body is not None:\n        options[\"extra_json\"] = cast(AnyMapping, extra_body)\n\n    if query is not None:\n        options[\"params\"] = query\n\n    if extra_query is not None:\n        options[\"params\"] = {**options.get(\"params\", {}), **extra_query}\n\n    if not isinstance(timeout, NotGiven):\n        options[\"timeout\"] = timeout\n\n    if idempotency_key is not None:\n        options[\"idempotency_key\"] = idempotency_key\n\n    if is_given(post_parser):\n        # internal\n        options[\"post_parser\"] = post_parser  # type: ignore\n\n    return options\n```"]}
{"repo": "openai/openai-python", "name": "_primitive_value_to_str", "language": "python", "path": "src/openai/_qs.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: Converts basic data types to their string representations for use in URL query strings.\n2. **Input**: A single value which can be a boolean, None, or other primitive data types.\n3. **Output**: A string representation of the input value, tailored for URL encoding.\n4. **Procedure**: The function checks the type of the input value:\n   - If the value is `True`, it returns the string \"true\".\n   - If the value is `False`, it returns the string \"false\".\n   - If the value is `None`, it returns an empty string.\n   - For all other data types, it converts the value to a string using Python's built-in `str()` function.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/openai/_utils/__init__.py\nfrom ._sync import asyncify as asyncify\nfrom ._proxy import LazyProxy as LazyProxy\nfrom ._utils import (\n    flatten as flatten,\n    is_dict as is_dict,\n    is_list as is_list,\n    is_given as is_given,\n    is_tuple as is_tuple,\n    is_mapping as is_mapping,\n    is_tuple_t as is_tuple_t,\n    parse_date as parse_date,\n    is_iterable as is_iterable,\n    is_sequence as is_sequence,\n    coerce_float as coerce_float,\n    is_mapping_t as is_mapping_t,\n    removeprefix as removeprefix,\n    removesuffix as removesuffix,\n    extract_files as extract_files,\n    is_sequence_t as is_sequence_t,\n    required_args as required_args,\n    coerce_boolean as coerce_boolean,\n    coerce_integer as coerce_integer,\n    file_from_path as file_from_path,\n    parse_datetime as parse_datetime,\n    strip_not_given as strip_not_given,\n    deepcopy_minimal as deepcopy_minimal,\n    get_async_library as get_async_library,\n...\n# Path: src/openai/_exceptions.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Any, Optional, cast\nfrom typing_extensions import Literal\n\nimport httpx\n\nfrom ._utils import is_dict\n\n__all__ = [\n    \"BadRequestError\",\n    \"AuthenticationError\",\n    \"PermissionDeniedError\",\n    \"NotFoundError\",\n    \"ConflictError\",\n    \"UnprocessableEntityError\",\n    \"RateLimitError\",\n    \"InternalServerError\",\n]\n\n\nclass OpenAIError(Exception):\n    pass\n\n\nclass APIError(OpenAIError):\n    message: str\n    request: httpx.Request\n\n    body: object | None\n    \"\"\"The API response body.\n\n    If the API responded with a valid JSON structure then this property will be the\n    decoded result.\n\n    If it isn't a valid JSON structure then this will be the raw response.\n\n    If there was no response associated with this error then it will be `None`.\n    \"\"\"\n\n    code: Optional[str] = None\n    param: Optional[str] = None\n    type: Optional[str]\n\n    def __init__(self, message: str, request: httpx.Request, *, body: object | None) -> None:\n        super().__init__(message)\n        self.request = request\n        self.message = message\n        self.body = body\n\n        if is_dict(body):\n            self.code = cast(Any, body.get(\"code\"))\n            self.param = cast(Any, body.get(\"param\"))\n            self.type = cast(Any, body.get(\"type\"))\n        else:\n            self.code = None\n            self.param = None\n            self.type = None\n\n\nclass APIResponseValidationError(APIError):\n    response: httpx.Response\n    status_code: int\n\n    def __init__(self, response: httpx.Response, body: object | None, *, message: str | None = None) -> None:\n        super().__init__(message or \"Data returned by API invalid for expected schema.\", response.request, body=body)\n        self.response = response\n        self.status_code = response.status_code\n\n\nclass APIStatusError(APIError):\n    \"\"\"Raised when an API response has a status code of 4xx or 5xx.\"\"\"\n\n    response: httpx.Response\n    status_code: int\n\n    def __init__(self, message: str, *, response: httpx.Response, body: object | None) -> None:\n        super().__init__(message, response.request, body=body)\n        self.response = response\n        self.status_code = response.status_code\n\n\nclass APIConnectionError(APIError):\n    def __init__(self, *, message: str = \"Connection error.\", request: httpx.Request) -> None:\n        super().__init__(message, request, body=None)\n\n\nclass APITimeoutError(APIConnectionError):\n    def __init__(self, request: httpx.Request) -> None:\n        super().__init__(message=\"Request timed out.\", request=request)\n\n\nclass BadRequestError(APIStatusError):\n    status_code: Literal[400] = 400  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass AuthenticationError(APIStatusError):\n    status_code: Literal[401] = 401  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass PermissionDeniedError(APIStatusError):\n    status_code: Literal[403] = 403  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass NotFoundError(APIStatusError):\n    status_code: Literal[404] = 404  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass ConflictError(APIStatusError):\n    status_code: Literal[409] = 409  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass UnprocessableEntityError(APIStatusError):\n    status_code: Literal[422] = 422  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass RateLimitError(APIStatusError):\n    status_code: Literal[429] = 429  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass InternalServerError(APIStatusError):\n    pass\n\n# Path: src/openai/_models.py\nfrom __future__ import annotations\n\nimport inspect\nfrom typing import TYPE_CHECKING, Any, Type, Union, Generic, TypeVar, Callable, cast\nfrom datetime import date, datetime\nfrom typing_extensions import (\n    Unpack,\n    Literal,\n    ClassVar,\n    Protocol,\n    Required,\n    TypedDict,\n    final,\n    override,\n    runtime_checkable,\n)\n\nimport pydantic\nimport pydantic.generics\nfrom pydantic.fields import FieldInfo\n\nfrom ._types import (\n    Body,\n    IncEx,\n    Query,\n    ModelT,\n    Headers,\n    Timeout,\n    NotGiven,\n    AnyMapping,\n    HttpxRequestFiles,\n)\nfrom ._utils import is_list, is_given, is_mapping, parse_date, parse_datetime, strip_not_given\nfrom ._compat import (\n    PYDANTIC_V2,\n    ConfigDict,\n    GenericModel as BaseGenericModel,\n    get_args,\n    is_union,\n    parse_obj,\n    get_origin,\n    is_literal_type,\n    get_model_config,\n    get_model_fields,\n    field_get_default,\n)\nfrom ._constants import RAW_RESPONSE_HEADER\n\n__all__ = [\"BaseModel\", \"GenericModel\"]\n\n_T = TypeVar(\"_T\")\n\n\n@runtime_checkable\nclass _ConfigProtocol(Protocol):\n    allow_population_by_field_name: bool\n\n\nclass BaseModel(pydantic.BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(extra=\"allow\")\n    else:\n\n        @property\n        @override\n        def model_fields_set(self) -> set[str]:\n            # a forwards-compat shim for pydantic v2\n            return self.__fields_set__  # type: ignore\n\n        class Config(pydantic.BaseConfig):  # pyright: ignore[reportDeprecated]\n            extra: Any = pydantic.Extra.allow  # type: ignore\n\n    @override\n    def __str__(self) -> str:\n        # mypy complains about an invalid self arg\n        return f'{self.__repr_name__()}({self.__repr_str__(\", \")})'  # type: ignore[misc]\n\n    # Override the 'construct' method in a way that supports recursive parsing without validation.\n    # Based on https://github.com/samuelcolvin/pydantic/issues/1168#issuecomment-817742836.\n    @classmethod\n    @override\n    def construct(\n        cls: Type[ModelT],\n        _fields_set: set[str] | None = None,\n        **values: object,\n    ) -> ModelT:\n        m = cls.__new__(cls)\n        fields_values: dict[str, object] = {}\n\n        config = get_model_config(cls)\n        populate_by_name = (\n            config.allow_population_by_field_name\n            if isinstance(config, _ConfigProtocol)\n            else config.get(\"populate_by_name\")\n        )\n\n        if _fields_set is None:\n            _fields_set = set()\n\n        model_fields = get_model_fields(cls)\n        for name, field in model_fields.items():\n            key = field.alias\n            if key is None or (key not in values and populate_by_name):\n                key = name\n\n            if key in values:\n                fields_values[name] = _construct_field(value=values[key], field=field, key=key)\n                _fields_set.add(name)\n            else:\n                fields_values[name] = field_get_default(field)\n\n        _extra = {}\n        for key, value in values.items():\n            if key not in model_fields:\n                if PYDANTIC_V2:\n                    _extra[key] = value\n                else:\n                    _fields_set.add(key)\n                    fields_values[key] = value\n\n        object.__setattr__(m, \"__dict__\", fields_values)\n\n        if PYDANTIC_V2:\n            # these properties are copied from Pydantic's `model_construct()` method\n            object.__setattr__(m, \"__pydantic_private__\", None)\n            object.__setattr__(m, \"__pydantic_extra__\", _extra)\n            object.__setattr__(m, \"__pydantic_fields_set__\", _fields_set)\n        else:\n            # init_private_attributes() does not exist in v2\n            m._init_private_attributes()  # type: ignore\n\n            # copied from Pydantic v1's `construct()` method\n            object.__setattr__(m, \"__fields_set__\", _fields_set)\n\n        return m\n\n    if not TYPE_CHECKING:\n        # type checkers incorrectly complain about this assignment\n        # because the type signatures are technically different\n        # although not in practice\n        model_construct = construct\n\n    if not PYDANTIC_V2:\n        # we define aliases for some of the new pydantic v2 methods so\n        # that we can just document these methods without having to specify\n        # a specific pydantic version as some users may not know which\n        # pydantic version they are currently using\n\n        @override\n        def model_dump(\n            self,\n            *,\n            mode: Literal[\"json\", \"python\"] | str = \"python\",\n            include: IncEx = None,\n            exclude: IncEx = None,\n            by_alias: bool = False,\n            exclude_unset: bool = False,\n            exclude_defaults: bool = False,\n            exclude_none: bool = False,\n            round_trip: bool = False,\n            warnings: bool = True,\n        ) -> dict[str, Any]:\n            \"\"\"Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump\n\n            Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.\n\n            Args:\n                mode: The mode in which `to_python` should run.\n                    If mode is 'json', the dictionary will only contain JSON serializable types.\n                    If mode is 'python', the dictionary may contain any Python objects.\n                include: A list of fields to include in the output.\n                exclude: A list of fields to exclude from the output.\n                by_alias: Whether to use the field's alias in the dictionary key if defined.\n                exclude_unset: Whether to exclude fields that are unset or None from the output.\n                exclude_defaults: Whether to exclude fields that are set to their default value from the output.\n                exclude_none: Whether to exclude fields that have a value of `None` from the output.\n                round_trip: Whether to enable serialization and deserialization round-trip support.\n                warnings: Whether to log warnings when invalid fields are encountered.\n\n            Returns:\n                A dictionary representation of the model.\n            \"\"\"\n            if mode != \"python\":\n                raise ValueError(\"mode is only supported in Pydantic v2\")\n            if round_trip != False:\n                raise ValueError(\"round_trip is only supported in Pydantic v2\")\n            if warnings != True:\n                raise ValueError(\"warnings is only supported in Pydantic v2\")\n            return super().dict(  # pyright: ignore[reportDeprecated]\n                include=include,\n                exclude=exclude,\n                by_alias=by_alias,\n                exclude_unset=exclude_unset,\n                exclude_defaults=exclude_defaults,\n                exclude_none=exclude_none,\n            )\n\n        @override\n        def model_dump_json(\n            self,\n            *,\n            indent: int | None = None,\n            include: IncEx = None,\n            exclude: IncEx = None,\n            by_alias: bool = False,\n            exclude_unset: bool = False,\n            exclude_defaults: bool = False,\n            exclude_none: bool = False,\n            round_trip: bool = False,\n            warnings: bool = True,\n        ) -> str:\n            \"\"\"Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump_json\n\n            Generates a JSON representation of the model using Pydantic's `to_json` method.\n\n            Args:\n                indent: Indentation to use in the JSON output. If None is passed, the output will be compact.\n                include: Field(s) to include in the JSON output. Can take either a string or set of strings.\n                exclude: Field(s) to exclude from the JSON output. Can take either a string or set of strings.\n                by_alias: Whether to serialize using field aliases.\n                exclude_unset: Whether to exclude fields that have not been explicitly set.\n                exclude_defaults: Whether to exclude fields that have the default value.\n                exclude_none: Whether to exclude fields that have a value of `None`.\n                round_trip: Whether to use serialization/deserialization between JSON and class instance.\n                warnings: Whether to show any warnings that occurred during serialization.\n\n            Returns:\n                A JSON string representation of the model.\n            \"\"\"\n            if round_trip != False:\n                raise ValueError(\"round_trip is only supported in Pydantic v2\")\n            if warnings != True:\n                raise ValueError(\"warnings is only supported in Pydantic v2\")\n            return super().json(  # type: ignore[reportDeprecated]\n                indent=indent,\n                include=include,\n                exclude=exclude,\n                by_alias=by_alias,\n                exclude_unset=exclude_unset,\n                exclude_defaults=exclude_defaults,\n                exclude_none=exclude_none,\n            )\n\n\ndef _construct_field(value: object, field: FieldInfo, key: str) -> object:\n    if value is None:\n        return field_get_default(field)\n\n    if PYDANTIC_V2:\n        type_ = field.annotation\n    else:\n        type_ = cast(type, field.outer_type_)  # type: ignore\n\n    if type_ is None:\n        raise RuntimeError(f\"Unexpected field type is None for {key}\")\n\n    return construct_type(value=value, type_=type_)\n\n\ndef is_basemodel(type_: type) -> bool:\n    \"\"\"Returns whether or not the given type is either a `BaseModel` or a union of `BaseModel`\"\"\"\n    origin = get_origin(type_) or type_\n    if is_union(type_):\n        for variant in get_args(type_):\n            if is_basemodel(variant):\n                return True\n\n        return False\n\n    return issubclass(origin, BaseModel) or issubclass(origin, GenericModel)\n\n\ndef construct_type(*, value: object, type_: type) -> object:\n    \"\"\"Loose coercion to the expected type with construction of nested values.\n\n    If the given value does not match the expected type then it is returned as-is.\n    \"\"\"\n\n    # we need to use the origin class for any types that are subscripted generics\n    # e.g. Dict[str, object]\n    origin = get_origin(type_) or type_\n    args = get_args(type_)\n\n    if is_union(origin):\n        try:\n            return validate_type(type_=cast(\"type[object]\", type_), value=value)\n        except Exception:\n            pass\n\n        # if the data is not valid, use the first variant that doesn't fail while deserializing\n        for variant in args:\n            try:\n                return construct_type(value=value, type_=variant)\n            except Exception:\n                continue\n\n        raise RuntimeError(f\"Could not convert data into a valid instance of {type_}\")\n\n    if origin == dict:\n        if not is_mapping(value):\n            return value\n\n        _, items_type = get_args(type_)  # Dict[_, items_type]\n        return {key: construct_type(value=item, type_=items_type) for key, item in value.items()}\n\n    if not is_literal_type(type_) and (issubclass(origin, BaseModel) or issubclass(origin, GenericModel)):\n        if is_list(value):\n            return [cast(Any, type_).construct(**entry) if is_mapping(entry) else entry for entry in value]\n\n        if is_mapping(value):\n            if issubclass(type_, BaseModel):\n                return type_.construct(**value)  # type: ignore[arg-type]\n\n            return cast(Any, type_).construct(**value)\n\n    if origin == list:\n        if not is_list(value):\n            return value\n\n        inner_type = args[0]  # List[inner_type]\n        return [construct_type(value=entry, type_=inner_type) for entry in value]\n\n    if origin == float:\n        if isinstance(value, int):\n            coerced = float(value)\n            if coerced != value:\n                return value\n            return coerced\n\n        return value\n\n    if type_ == datetime:\n        try:\n            return parse_datetime(value)  # type: ignore\n        except Exception:\n            return value\n\n    if type_ == date:\n        try:\n            return parse_date(value)  # type: ignore\n        except Exception:\n            return value\n\n    return value\n\n\ndef validate_type(*, type_: type[_T], value: object) -> _T:\n    \"\"\"Strict validation that the given value matches the expected type\"\"\"\n    if inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n        return cast(_T, parse_obj(type_, value))\n\n    return cast(_T, _validate_non_model_type(type_=type_, value=value))\n\n\n# our use of subclasssing here causes weirdness for type checkers,\n# so we just pretend that we don't subclass\nif TYPE_CHECKING:\n    GenericModel = BaseModel\nelse:\n\n    class GenericModel(BaseGenericModel, BaseModel):\n        pass\n\n\nif PYDANTIC_V2:\n    from pydantic import TypeAdapter\n\n    def _validate_non_model_type(*, type_: type[_T], value: object) -> _T:\n        return TypeAdapter(type_).validate_python(value)\n\nelif not TYPE_CHECKING:  # TODO: condition is weird\n\n    class RootModel(GenericModel, Generic[_T]):\n        \"\"\"Used as a placeholder to easily convert runtime types to a Pydantic format\n        to provide validation.\n\n        For example:\n        ```py\n        validated = RootModel[int](__root__=\"5\").__root__\n        # validated: 5\n        ```\n        \"\"\"\n\n        __root__: _T\n\n    def _validate_non_model_type(*, type_: type[_T], value: object) -> _T:\n        model = _create_pydantic_model(type_).validate(value)\n        return cast(_T, model.__root__)\n\n    def _create_pydantic_model(type_: _T) -> Type[RootModel[_T]]:\n        return RootModel[type_]  # type: ignore\n\n\nclass FinalRequestOptionsInput(TypedDict, total=False):\n    method: Required[str]\n    url: Required[str]\n    params: Query\n    headers: Headers\n    max_retries: int\n    timeout: float | Timeout | None\n    files: HttpxRequestFiles | None\n    idempotency_key: str\n    json_data: Body\n    extra_json: AnyMapping\n\n\n@final\nclass FinalRequestOptions(pydantic.BaseModel):\n    method: str\n    url: str\n    params: Query = {}\n    headers: Union[Headers, NotGiven] = NotGiven()\n    max_retries: Union[int, NotGiven] = NotGiven()\n    timeout: Union[float, Timeout, None, NotGiven] = NotGiven()\n    files: Union[HttpxRequestFiles, None] = None\n    idempotency_key: Union[str, None] = None\n    post_parser: Union[Callable[[Any], Any], NotGiven] = NotGiven()\n\n    # It should be noted that we cannot use `json` here as that would override\n    # a BaseModel method in an incompatible fashion.\n    json_data: Union[Body, None] = None\n    extra_json: Union[AnyMapping, None] = None\n\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)\n    else:\n\n        class Config(pydantic.BaseConfig):  # pyright: ignore[reportDeprecated]\n            arbitrary_types_allowed: bool = True\n\n    def get_max_retries(self, max_retries: int) -> int:\n        if isinstance(self.max_retries, NotGiven):\n            return max_retries\n        return self.max_retries\n\n    def _strip_raw_response_header(self) -> None:\n        if not is_given(self.headers):\n            return\n\n        if self.headers.get(RAW_RESPONSE_HEADER):\n            self.headers = {**self.headers}\n            self.headers.pop(RAW_RESPONSE_HEADER)\n\n    # override the `construct` method so that we can run custom transformations.\n    # this is necessary as we don't want to do any actual runtime type checking\n    # (which means we can't use validators) but we do want to ensure that `NotGiven`\n    # values are not present\n    #\n    # type ignore required because we're adding explicit types to `**values`\n    @classmethod\n    def construct(  # type: ignore\n        cls,\n        _fields_set: set[str] | None = None,\n        **values: Unpack[FinalRequestOptionsInput],\n    ) -> FinalRequestOptions:\n        kwargs: dict[str, Any] = {\n            # we unconditionally call `strip_not_given` on any value\n            # as it will just ignore any non-mapping types\n            key: strip_not_given(value)\n            for key, value in values.items()\n        }\n        if PYDANTIC_V2:\n            return super().model_construct(_fields_set, **kwargs)\n        return cast(FinalRequestOptions, super().construct(_fields_set, **kwargs))  # pyright: ignore[reportDeprecated]\n\n    if not TYPE_CHECKING:\n        # type checkers incorrectly complain about this assignment\n        model_construct = construct\n\n# Path: src/openai/_qs.py\nfrom __future__ import annotations\n\nfrom typing import Any, List, Tuple, Union, Mapping, TypeVar\nfrom urllib.parse import parse_qs, urlencode\nfrom typing_extensions import Literal, get_args\n\nfrom ._types import NOT_GIVEN, NotGiven, NotGivenOr\nfrom ._utils import flatten\n\n_T = TypeVar(\"_T\")\n\n\nArrayFormat = Literal[\"comma\", \"repeat\", \"indices\", \"brackets\"]\nNestedFormat = Literal[\"dots\", \"brackets\"]\n\nPrimitiveData = Union[str, int, float, bool, None]\n# this should be Data = Union[PrimitiveData, \"List[Data]\", \"Tuple[Data]\", \"Mapping[str, Data]\"]\n# https://github.com/microsoft/pyright/issues/3555\nData = Union[PrimitiveData, List[Any], Tuple[Any], \"Mapping[str, Any]\"]\nParams = Mapping[str, Data]\n\n\nclass Querystring:\n    array_format: ArrayFormat\n    nested_format: NestedFormat\n\n    def __init__(\n        self,\n        *,\n        array_format: ArrayFormat = \"repeat\",\n        nested_format: NestedFormat = \"brackets\",\n    ) -> None:\n        self.array_format = array_format\n        self.nested_format = nested_format\n\n    def parse(self, query: str) -> Mapping[str, object]:\n        # Note: custom format syntax is not supported yet\n        return parse_qs(query)\n\n    def stringify(\n        self,\n        params: Params,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n        nested_format: NotGivenOr[NestedFormat] = NOT_GIVEN,\n    ) -> str:\n        return urlencode(\n            self.stringify_items(\n                params,\n                array_format=array_format,\n                nested_format=nested_format,\n            )\n        )\n\n    def stringify_items(\n        self,\n        params: Params,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n        nested_format: NotGivenOr[NestedFormat] = NOT_GIVEN,\n    ) -> list[tuple[str, str]]:\n        opts = Options(\n            qs=self,\n            array_format=array_format,\n            nested_format=nested_format,\n        )\n        return flatten([self._stringify_item(key, value, opts) for key, value in params.items()])\n\n    def _stringify_item(\n        self,\n        key: str,\n        value: Data,\n        opts: Options,\n    ) -> list[tuple[str, str]]:\n        if isinstance(value, Mapping):\n            items: list[tuple[str, str]] = []\n            nested_format = opts.nested_format\n            for subkey, subvalue in value.items():\n                items.extend(\n                    self._stringify_item(\n                        # TODO: error if unknown format\n                        f\"{key}.{subkey}\" if nested_format == \"dots\" else f\"{key}[{subkey}]\",\n                        subvalue,\n                        opts,\n                    )\n                )\n            return items\n\n        if isinstance(value, (list, tuple)):\n            array_format = opts.array_format\n            if array_format == \"comma\":\n                return [\n                    (\n                        key,\n                        \",\".join(self._primitive_value_to_str(item) for item in value if item is not None),\n                    ),\n                ]\n            elif array_format == \"repeat\":\n                items = []\n                for item in value:\n                    items.extend(self._stringify_item(key, item, opts))\n                return items\n            elif array_format == \"indices\":\n                raise NotImplementedError(\"The array indices format is not supported yet\")\n            elif array_format == \"brackets\":\n                items = []\n                key = key + \"[]\"\n                for item in value:\n                    items.extend(self._stringify_item(key, item, opts))\n                return items\n            else:\n                raise NotImplementedError(\n                    f\"Unknown array_format value: {array_format}, choose from {', '.join(get_args(ArrayFormat))}\"\n                )\n\n        serialised = self._primitive_value_to_str(value)\n        if not serialised:\n            return []\n        return [(key, serialised)]\n\n    \ndef _primitive_value_to_str(self, value: PrimitiveData) -> str:\n        # copied from httpx\n        if value is True:\n            return \"true\"\n        elif value is False:\n            return \"false\"\n        elif value is None:\n            return \"\"\n        return str(value)\n\n\n_qs = Querystring()\nparse = _qs.parse\nstringify = _qs.stringify\nstringify_items = _qs.stringify_items\n\n\nclass Options:\n    array_format: ArrayFormat\n    nested_format: NestedFormat\n\n    def __init__(\n        self,\n        qs: Querystring = _qs,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n        nested_format: NotGivenOr[NestedFormat] = NOT_GIVEN,\n    ) -> None:\n        self.array_format = qs.array_format if isinstance(array_format, NotGiven) else array_format\n        self.nested_format = qs.nested_format if isinstance(nested_format, NotGiven) else nested_format\n\n# Path: src/openai/_version.py\n# File generated from our OpenAPI spec by Stainless.\n\n__title__ = \"openai\"\n__version__ = \"1.13.3\"  # x-release-please-version\n\n# Path: src/openai/_resource.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nimport time\nfrom typing import TYPE_CHECKING\n\nimport anyio\n\nif TYPE_CHECKING:\n    from ._client import OpenAI, AsyncOpenAI\n\n\nclass SyncAPIResource:\n    _client: OpenAI\n\n    def __init__(self, client: OpenAI) -> None:\n        self._client = client\n        self._get = client.get\n        self._post = client.post\n        self._patch = client.patch\n        self._put = client.put\n        self._delete = client.delete\n        self._get_api_list = client.get_api_list\n\n    def _sleep(self, seconds: float) -> None:\n        time.sleep(seconds)\n\n\nclass AsyncAPIResource:\n    _client: AsyncOpenAI\n\n    def __init__(self, client: AsyncOpenAI) -> None:\n        self._client = client\n        self._get = client.get\n        self._post = client.post\n        self._patch = client.patch\n        self._put = client.put\n        self._delete = client.delete\n        self._get_api_list = client.get_api_list\n\n    async def _sleep(self, seconds: float) -> None:\n        await anyio.sleep(seconds)\n\n# Path: src/openai/_response.py\nfrom __future__ import annotations\n\nimport os\nimport inspect\nimport logging\nimport datetime\nimport functools\nfrom types import TracebackType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Union,\n    Generic,\n    TypeVar,\n    Callable,\n    Iterator,\n    AsyncIterator,\n    cast,\n    overload,\n)\nfrom typing_extensions import Awaitable, ParamSpec, override, get_origin\n\nimport anyio\nimport httpx\nimport pydantic\n\nfrom ._types import NoneType\nfrom ._utils import is_given, extract_type_var_from_base\nfrom ._models import BaseModel, is_basemodel\nfrom ._constants import RAW_RESPONSE_HEADER, OVERRIDE_CAST_TO_HEADER\nfrom ._streaming import Stream, AsyncStream, is_stream_class_type, extract_stream_chunk_type\nfrom ._exceptions import OpenAIError, APIResponseValidationError\n\nif TYPE_CHECKING:\n    from ._models import FinalRequestOptions\n    from ._base_client import BaseClient\n\n\nP = ParamSpec(\"P\")\nR = TypeVar(\"R\")\n_T = TypeVar(\"_T\")\n_APIResponseT = TypeVar(\"_APIResponseT\", bound=\"APIResponse[Any]\")\n_AsyncAPIResponseT = TypeVar(\"_AsyncAPIResponseT\", bound=\"AsyncAPIResponse[Any]\")\n\nlog: logging.Logger = logging.getLogger(__name__)\n\n\nclass BaseAPIResponse(Generic[R]):\n    _cast_to: type[R]\n    _client: BaseClient[Any, Any]\n    _parsed_by_type: dict[type[Any], Any]\n    _is_sse_stream: bool\n    _stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None\n    _options: FinalRequestOptions\n\n    http_response: httpx.Response\n\n    def __init__(\n        self,\n        *,\n        raw: httpx.Response,\n        cast_to: type[R],\n        client: BaseClient[Any, Any],\n        stream: bool,\n        stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None,\n        options: FinalRequestOptions,\n    ) -> None:\n        self._cast_to = cast_to\n        self._client = client\n        self._parsed_by_type = {}\n        self._is_sse_stream = stream\n        self._stream_cls = stream_cls\n        self._options = options\n        self.http_response = raw\n\n    @property\n    def headers(self) -> httpx.Headers:\n        return self.http_response.headers\n\n    @property\n    def http_request(self) -> httpx.Request:\n        \"\"\"Returns the httpx Request instance associated with the current response.\"\"\"\n        return self.http_response.request\n\n    @property\n    def status_code(self) -> int:\n        return self.http_response.status_code\n\n    @property\n    def url(self) -> httpx.URL:\n        \"\"\"Returns the URL for which the request was made.\"\"\"\n        return self.http_response.url\n\n    @property\n    def method(self) -> str:\n        return self.http_request.method\n\n    @property\n    def http_version(self) -> str:\n        return self.http_response.http_version\n\n    @property\n    def elapsed(self) -> datetime.timedelta:\n        \"\"\"The time taken for the complete request/response cycle to complete.\"\"\"\n        return self.http_response.elapsed\n\n    @property\n    def is_closed(self) -> bool:\n        \"\"\"Whether or not the response body has been closed.\n\n        If this is False then there is response data that has not been read yet.\n        You must either fully consume the response body or call `.close()`\n        before discarding the response to prevent resource leaks.\n        \"\"\"\n        return self.http_response.is_closed\n\n    @override\n    def __repr__(self) -> str:\n        return (\n            f\"<{self.__class__.__name__} [{self.status_code} {self.http_response.reason_phrase}] type={self._cast_to}>\"\n        )\n\n    def _parse(self, *, to: type[_T] | None = None) -> R | _T:\n        if self._is_sse_stream:\n            if to:\n                if not is_stream_class_type(to):\n                    raise TypeError(f\"Expected custom parse type to be a subclass of {Stream} or {AsyncStream}\")\n\n                return cast(\n                    _T,\n                    to(\n                        cast_to=extract_stream_chunk_type(\n                            to,\n                            failure_message=\"Expected custom stream type to be passed with a type argument, e.g. Stream[ChunkType]\",\n                        ),\n                        response=self.http_response,\n                        client=cast(Any, self._client),\n                    ),\n                )\n\n            if self._stream_cls:\n                return cast(\n                    R,\n                    self._stream_cls(\n                        cast_to=extract_stream_chunk_type(self._stream_cls),\n                        response=self.http_response,\n                        client=cast(Any, self._client),\n                    ),\n                )\n\n            stream_cls = cast(\"type[Stream[Any]] | type[AsyncStream[Any]] | None\", self._client._default_stream_cls)\n            if stream_cls is None:\n                raise MissingStreamClassError()\n\n            return cast(\n                R,\n                stream_cls(\n                    cast_to=self._cast_to,\n                    response=self.http_response,\n                    client=cast(Any, self._client),\n                ),\n            )\n\n        cast_to = to if to is not None else self._cast_to\n        if cast_to is NoneType:\n            return cast(R, None)\n\n        response = self.http_response\n        if cast_to == str:\n            return cast(R, response.text)\n\n        if cast_to == bytes:\n            return cast(R, response.content)\n\n        origin = get_origin(cast_to) or cast_to\n\n        # handle the legacy binary response case\n        if inspect.isclass(cast_to) and cast_to.__name__ == \"HttpxBinaryResponseContent\":\n            return cast(R, cast_to(response))  # type: ignore\n\n        if origin == APIResponse:\n            raise RuntimeError(\"Unexpected state - cast_to is `APIResponse`\")\n\n        if inspect.isclass(origin) and issubclass(origin, httpx.Response):\n            # Because of the invariance of our ResponseT TypeVar, users can subclass httpx.Response\n            # and pass that class to our request functions. We cannot change the variance to be either\n            # covariant or contravariant as that makes our usage of ResponseT illegal. We could construct\n            # the response class ourselves but that is something that should be supported directly in httpx\n            # as it would be easy to incorrectly construct the Response object due to the multitude of arguments.\n            if cast_to != httpx.Response:\n                raise ValueError(f\"Subclasses of httpx.Response cannot be passed to `cast_to`\")\n            return cast(R, response)\n\n        if inspect.isclass(origin) and not issubclass(origin, BaseModel) and issubclass(origin, pydantic.BaseModel):\n            raise TypeError(\"Pydantic models must subclass our base model type, e.g. `from openai import BaseModel`\")\n\n        if (\n            cast_to is not object\n            and not origin is list\n            and not origin is dict\n            and not origin is Union\n            and not issubclass(origin, BaseModel)\n        ):\n            raise RuntimeError(\n                f\"Unsupported type, expected {cast_to} to be a subclass of {BaseModel}, {dict}, {list}, {Union}, {NoneType}, {str} or {httpx.Response}.\"\n            )\n\n        # split is required to handle cases where additional information is included\n        # in the response, e.g. application/json; charset=utf-8\n        content_type, *_ = response.headers.get(\"content-type\", \"*\").split(\";\")\n        if content_type != \"application/json\":\n            if is_basemodel(cast_to):\n                try:\n                    data = response.json()\n                except Exception as exc:\n                    log.debug(\"Could not read JSON from response data due to %s - %s\", type(exc), exc)\n                else:\n                    return self._client._process_response_data(\n                        data=data,\n                        cast_to=cast_to,  # type: ignore\n                        response=response,\n                    )\n\n            if self._client._strict_response_validation:\n                raise APIResponseValidationError(\n                    response=response,\n                    message=f\"Expected Content-Type response header to be `application/json` but received `{content_type}` instead.\",\n                    body=response.text,\n                )\n\n            # If the API responds with content that isn't JSON then we just return\n            # the (decoded) text without performing any parsing so that you can still\n            # handle the response however you need to.\n            return response.text  # type: ignore\n\n        data = response.json()\n\n        return self._client._process_response_data(\n            data=data,\n            cast_to=cast_to,  # type: ignore\n            response=response,\n        )\n\n\nclass APIResponse(BaseAPIResponse[R]):\n    @overload\n    def parse(self, *, to: type[_T]) -> _T:\n        ...\n\n    @overload\n    def parse(self) -> R:\n        ...\n\n    def parse(self, *, to: type[_T] | None = None) -> R | _T:\n        \"\"\"Returns the rich python representation of this response's data.\n\n        For lower-level control, see `.read()`, `.json()`, `.iter_bytes()`.\n\n        You can customise the type that the response is parsed into through\n        the `to` argument, e.g.\n\n        ```py\n        from openai import BaseModel\n\n\n        class MyModel(BaseModel):\n            foo: str\n\n\n        obj = response.parse(to=MyModel)\n        print(obj.foo)\n        ```\n\n        We support parsing:\n          - `BaseModel`\n          - `dict`\n          - `list`\n          - `Union`\n          - `str`\n          - `httpx.Response`\n        \"\"\"\n        cache_key = to if to is not None else self._cast_to\n        cached = self._parsed_by_type.get(cache_key)\n        if cached is not None:\n            return cached  # type: ignore[no-any-return]\n\n        if not self._is_sse_stream:\n            self.read()\n\n        parsed = self._parse(to=to)\n        if is_given(self._options.post_parser):\n            parsed = self._options.post_parser(parsed)\n\n        self._parsed_by_type[cache_key] = parsed\n        return parsed\n\n    def read(self) -> bytes:\n        \"\"\"Read and return the binary response content.\"\"\"\n        try:\n            return self.http_response.read()\n        except httpx.StreamConsumed as exc:\n            # The default error raised by httpx isn't very\n            # helpful in our case so we re-raise it with\n            # a different error message.\n            raise StreamAlreadyConsumed() from exc\n\n    def text(self) -> str:\n        \"\"\"Read and decode the response content into a string.\"\"\"\n        self.read()\n        return self.http_response.text\n\n    def json(self) -> object:\n        \"\"\"Read and decode the JSON response content.\"\"\"\n        self.read()\n        return self.http_response.json()\n\n    def close(self) -> None:\n        \"\"\"Close the response and release the connection.\n\n        Automatically called if the response body is read to completion.\n        \"\"\"\n        self.http_response.close()\n\n    def iter_bytes(self, chunk_size: int | None = None) -> Iterator[bytes]:\n        \"\"\"\n        A byte-iterator over the decoded response content.\n\n        This automatically handles gzip, deflate and brotli encoded responses.\n        \"\"\"\n        for chunk in self.http_response.iter_bytes(chunk_size):\n            yield chunk\n\n    def iter_text(self, chunk_size: int | None = None) -> Iterator[str]:\n        \"\"\"A str-iterator over the decoded response content\n        that handles both gzip, deflate, etc but also detects the content's\n        string encoding.\n        \"\"\"\n        for chunk in self.http_response.iter_text(chunk_size):\n            yield chunk\n\n    def iter_lines(self) -> Iterator[str]:\n        \"\"\"Like `iter_text()` but will only yield chunks for each line\"\"\"\n        for chunk in self.http_response.iter_lines():\n            yield chunk\n\n\nclass AsyncAPIResponse(BaseAPIResponse[R]):\n    @overload\n    async def parse(self, *, to: type[_T]) -> _T:\n        ...\n\n    @overload\n    async def parse(self) -> R:\n        ...\n\n    async def parse(self, *, to: type[_T] | None = None) -> R | _T:\n        \"\"\"Returns the rich python representation of this response's data.\n\n        For lower-level control, see `.read()`, `.json()`, `.iter_bytes()`.\n\n        You can customise the type that the response is parsed into through\n        the `to` argument, e.g.\n\n        ```py\n        from openai import BaseModel\n\n\n        class MyModel(BaseModel):\n            foo: str\n\n\n        obj = response.parse(to=MyModel)\n        print(obj.foo)\n        ```\n\n        We support parsing:\n          - `BaseModel`\n          - `dict`\n          - `list`\n          - `Union`\n          - `str`\n          - `httpx.Response`\n        \"\"\"\n        cache_key = to if to is not None else self._cast_to\n        cached = self._parsed_by_type.get(cache_key)\n        if cached is not None:\n            return cached  # type: ignore[no-any-return]\n\n        if not self._is_sse_stream:\n            await self.read()\n\n        parsed = self._parse(to=to)\n        if is_given(self._options.post_parser):\n            parsed = self._options.post_parser(parsed)\n\n        self._parsed_by_type[cache_key] = parsed\n        return parsed\n\n    async def read(self) -> bytes:\n        \"\"\"Read and return the binary response content.\"\"\"\n        try:\n            return await self.http_response.aread()\n        except httpx.StreamConsumed as exc:\n            # the default error raised by httpx isn't very\n            # helpful in our case so we re-raise it with\n            # a different error message\n            raise StreamAlreadyConsumed() from exc\n\n    async def text(self) -> str:\n        \"\"\"Read and decode the response content into a string.\"\"\"\n        await self.read()\n        return self.http_response.text\n\n    async def json(self) -> object:\n        \"\"\"Read and decode the JSON response content.\"\"\"\n        await self.read()\n        return self.http_response.json()\n\n    async def close(self) -> None:\n        \"\"\"Close the response and release the connection.\n\n        Automatically called if the response body is read to completion.\n        \"\"\"\n        await self.http_response.aclose()\n\n    async def iter_bytes(self, chunk_size: int | None = None) -> AsyncIterator[bytes]:\n        \"\"\"\n        A byte-iterator over the decoded response content.\n\n        This automatically handles gzip, deflate and brotli encoded responses.\n        \"\"\"\n        async for chunk in self.http_response.aiter_bytes(chunk_size):\n            yield chunk\n\n    async def iter_text(self, chunk_size: int | None = None) -> AsyncIterator[str]:\n        \"\"\"A str-iterator over the decoded response content\n        that handles both gzip, deflate, etc but also detects the content's\n        string encoding.\n        \"\"\"\n        async for chunk in self.http_response.aiter_text(chunk_size):\n            yield chunk\n\n    async def iter_lines(self) -> AsyncIterator[str]:\n        \"\"\"Like `iter_text()` but will only yield chunks for each line\"\"\"\n        async for chunk in self.http_response.aiter_lines():\n            yield chunk\n\n\nclass BinaryAPIResponse(APIResponse[bytes]):\n    \"\"\"Subclass of APIResponse providing helpers for dealing with binary data.\n\n    Note: If you want to stream the response data instead of eagerly reading it\n    all at once then you should use `.with_streaming_response` when making\n    the API request, e.g. `.with_streaming_response.get_binary_response()`\n    \"\"\"\n\n    def write_to_file(\n        self,\n        file: str | os.PathLike[str],\n    ) -> None:\n        \"\"\"Write the output to the given file.\n\n        Accepts a filename or any path-like object, e.g. pathlib.Path\n\n        Note: if you want to stream the data to the file instead of writing\n        all at once then you should use `.with_streaming_response` when making\n        the API request, e.g. `.with_streaming_response.get_binary_response()`\n        \"\"\"\n        with open(file, mode=\"wb\") as f:\n            for data in self.iter_bytes():\n                f.write(data)\n\n\nclass AsyncBinaryAPIResponse(AsyncAPIResponse[bytes]):\n    \"\"\"Subclass of APIResponse providing helpers for dealing with binary data.\n\n    Note: If you want to stream the response data instead of eagerly reading it\n    all at once then you should use `.with_streaming_response` when making\n    the API request, e.g. `.with_streaming_response.get_binary_response()`\n    \"\"\"\n\n    async def write_to_file(\n        self,\n        file: str | os.PathLike[str],\n    ) -> None:\n        \"\"\"Write the output to the given file.\n\n        Accepts a filename or any path-like object, e.g. pathlib.Path\n\n        Note: if you want to stream the data to the file instead of writing\n        all at once then you should use `.with_streaming_response` when making\n        the API request, e.g. `.with_streaming_response.get_binary_response()`\n        \"\"\"\n        path = anyio.Path(file)\n        async with await path.open(mode=\"wb\") as f:\n            async for data in self.iter_bytes():\n                await f.write(data)\n\n\nclass StreamedBinaryAPIResponse(APIResponse[bytes]):\n    def stream_to_file(\n        self,\n        file: str | os.PathLike[str],\n        *,\n        chunk_size: int | None = None,\n    ) -> None:\n        \"\"\"Streams the output to the given file.\n\n        Accepts a filename or any path-like object, e.g. pathlib.Path\n        \"\"\"\n        with open(file, mode=\"wb\") as f:\n            for data in self.iter_bytes(chunk_size):\n                f.write(data)\n\n\nclass AsyncStreamedBinaryAPIResponse(AsyncAPIResponse[bytes]):\n    async def stream_to_file(\n        self,\n        file: str | os.PathLike[str],\n        *,\n        chunk_size: int | None = None,\n    ) -> None:\n        \"\"\"Streams the output to the given file.\n\n        Accepts a filename or any path-like object, e.g. pathlib.Path\n        \"\"\"\n        path = anyio.Path(file)\n        async with await path.open(mode=\"wb\") as f:\n            async for data in self.iter_bytes(chunk_size):\n                await f.write(data)\n\n\nclass MissingStreamClassError(TypeError):\n    def __init__(self) -> None:\n        super().__init__(\n            \"The `stream` argument was set to `True` but the `stream_cls` argument was not given. See `openai._streaming` for reference\",\n        )\n\n\nclass StreamAlreadyConsumed(OpenAIError):\n    \"\"\"\n    Attempted to read or stream content, but the content has already\n    been streamed.\n\n    This can happen if you use a method like `.iter_lines()` and then attempt\n    to read th entire response body afterwards, e.g.\n\n    ```py\n    response = await client.post(...)\n    async for line in response.iter_lines():\n        ...  # do something with `line`\n\n    content = await response.read()\n    # ^ error\n    ```\n\n    If you want this behaviour you'll need to either manually accumulate the response\n    content or call `await response.read()` before iterating over the stream.\n    \"\"\"\n\n    def __init__(self) -> None:\n        message = (\n            \"Attempted to read or stream some content, but the content has \"\n            \"already been streamed. \"\n            \"This could be due to attempting to stream the response \"\n            \"content more than once.\"\n            \"\\n\\n\"\n            \"You can fix this by manually accumulating the response content while streaming \"\n            \"or by calling `.read()` before starting to stream.\"\n        )\n        super().__init__(message)\n\n\nclass ResponseContextManager(Generic[_APIResponseT]):\n    \"\"\"Context manager for ensuring that a request is not made\n    until it is entered and that the response will always be closed\n    when the context manager exits\n    \"\"\"\n\n    def __init__(self, request_func: Callable[[], _APIResponseT]) -> None:\n        self._request_func = request_func\n        self.__response: _APIResponseT | None = None\n\n    def __enter__(self) -> _APIResponseT:\n        self.__response = self._request_func()\n        return self.__response\n\n    def __exit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        if self.__response is not None:\n            self.__response.close()\n\n\nclass AsyncResponseContextManager(Generic[_AsyncAPIResponseT]):\n    \"\"\"Context manager for ensuring that a request is not made\n    until it is entered and that the response will always be closed\n    when the context manager exits\n    \"\"\"\n\n    def __init__(self, api_request: Awaitable[_AsyncAPIResponseT]) -> None:\n        self._api_request = api_request\n        self.__response: _AsyncAPIResponseT | None = None\n\n    async def __aenter__(self) -> _AsyncAPIResponseT:\n        self.__response = await self._api_request\n        return self.__response\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        if self.__response is not None:\n            await self.__response.close()\n\n\ndef to_streamed_response_wrapper(func: Callable[P, R]) -> Callable[P, ResponseContextManager[APIResponse[R]]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support streaming and returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> ResponseContextManager[APIResponse[R]]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = functools.partial(func, *args, **kwargs)\n\n        return ResponseContextManager(cast(Callable[[], APIResponse[R]], make_request))\n\n    return wrapped\n\n\ndef async_to_streamed_response_wrapper(\n    func: Callable[P, Awaitable[R]],\n) -> Callable[P, AsyncResponseContextManager[AsyncAPIResponse[R]]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support streaming and returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> AsyncResponseContextManager[AsyncAPIResponse[R]]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = func(*args, **kwargs)\n\n        return AsyncResponseContextManager(cast(Awaitable[AsyncAPIResponse[R]], make_request))\n\n    return wrapped\n\n\ndef to_custom_streamed_response_wrapper(\n    func: Callable[P, object],\n    response_cls: type[_APIResponseT],\n) -> Callable[P, ResponseContextManager[_APIResponseT]]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support streaming and returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> ResponseContextManager[_APIResponseT]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = functools.partial(func, *args, **kwargs)\n\n        return ResponseContextManager(cast(Callable[[], _APIResponseT], make_request))\n\n    return wrapped\n\n\ndef async_to_custom_streamed_response_wrapper(\n    func: Callable[P, Awaitable[object]],\n    response_cls: type[_AsyncAPIResponseT],\n) -> Callable[P, AsyncResponseContextManager[_AsyncAPIResponseT]]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support streaming and returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> AsyncResponseContextManager[_AsyncAPIResponseT]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"stream\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        make_request = func(*args, **kwargs)\n\n        return AsyncResponseContextManager(cast(Awaitable[_AsyncAPIResponseT], make_request))\n\n    return wrapped\n\n\ndef to_raw_response_wrapper(func: Callable[P, R]) -> Callable[P, APIResponse[R]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> APIResponse[R]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(APIResponse[R], func(*args, **kwargs))\n\n    return wrapped\n\n\ndef async_to_raw_response_wrapper(func: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[AsyncAPIResponse[R]]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    async def wrapped(*args: P.args, **kwargs: P.kwargs) -> AsyncAPIResponse[R]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(AsyncAPIResponse[R], await func(*args, **kwargs))\n\n    return wrapped\n\n\ndef to_custom_raw_response_wrapper(\n    func: Callable[P, object],\n    response_cls: type[_APIResponseT],\n) -> Callable[P, _APIResponseT]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> _APIResponseT:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(_APIResponseT, func(*args, **kwargs))\n\n    return wrapped\n\n\ndef async_to_custom_raw_response_wrapper(\n    func: Callable[P, Awaitable[object]],\n    response_cls: type[_AsyncAPIResponseT],\n) -> Callable[P, Awaitable[_AsyncAPIResponseT]]:\n    \"\"\"Higher order function that takes one of our bound API methods and an `APIResponse` class\n    and wraps the method to support returning the given response class directly.\n\n    Note: the given `response_cls` *must* be concrete, e.g. `class BinaryAPIResponse(APIResponse[bytes])`\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> Awaitable[_AsyncAPIResponseT]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"raw\"\n        extra_headers[OVERRIDE_CAST_TO_HEADER] = response_cls\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(Awaitable[_AsyncAPIResponseT], func(*args, **kwargs))\n\n    return wrapped\n\n\ndef extract_response_type(typ: type[BaseAPIResponse[Any]]) -> type:\n    \"\"\"Given a type like `APIResponse[T]`, returns the generic type variable `T`.\n\n    This also handles the case where a concrete subclass is given, e.g.\n    ```py\n    class MyResponse(APIResponse[bytes]):\n        ...\n\n    extract_response_type(MyResponse) -> bytes\n    ```\n    \"\"\"\n    return extract_type_var_from_base(\n        typ,\n        generic_bases=cast(\"tuple[type, ...]\", (BaseAPIResponse, APIResponse, AsyncAPIResponse)),\n        index=0,\n    )\n\n# Path: src/openai/types/completion_choice.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing import Dict, List, Optional\nfrom typing_extensions import Literal\n\nfrom .._models import BaseModel\n\n__all__ = [\"CompletionChoice\", \"Logprobs\"]\n\n\nclass Logprobs(BaseModel):\n    text_offset: Optional[List[int]] = None\n\n    token_logprobs: Optional[List[float]] = None\n\n    tokens: Optional[List[str]] = None\n\n    top_logprobs: Optional[List[Dict[str, float]]] = None\n\n\nclass CompletionChoice(BaseModel):\n    finish_reason: Literal[\"stop\", \"length\", \"content_filter\"]\n    \"\"\"The reason the model stopped generating tokens.\n\n    This will be `stop` if the model hit a natural stop point or a provided stop\n    sequence, `length` if the maximum number of tokens specified in the request was\n    reached, or `content_filter` if content was omitted due to a flag from our\n    content filters.\n    \"\"\"\n\n...\n", "needle_token_start": 7176, "needle_token_end": 7249, "code_context_ntokens": 16001, "output": ["```python\ndef _primitive_value_to_str(self, value: PrimitiveData) -> str:\n    # copied from httpx\n    if value is True:\n        return \"true\"\n    elif value is False:\n        return \"false\"\n    elif value is None:\n        return \"\"\n    return str(value)\n```"]}
{"repo": "openai/openai-python", "name": "_transform_value", "language": "python", "path": "src/openai/_utils/_transform.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to modify data based on specific formatting rules defined in type annotations.\n2. **Input**: The function takes two parameters: the data to be transformed and the type information which may include formatting annotations.\n3. **Output**: It returns the transformed data, which might be formatted according to the specified rules in the type annotations, or the original data if no formatting is applicable.\n4. **Procedure**: \n   - First, the function checks if the type information includes annotations.\n   - If annotations are present, it iterates through them to find any formatting instructions.\n   - If a formatting instruction is found, it applies the specified format to the data using a helper function.\n   - If no relevant annotations are found, the original data is returned unchanged.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/openai/_utils/_proxy.py\nfrom __future__ import annotations\n\nfrom abc import ABC, abstractmethod\nfrom typing import Generic, TypeVar, Iterable, cast\nfrom typing_extensions import override\n\nT = TypeVar(\"T\")\n\n\nclass LazyProxy(Generic[T], ABC):\n    \"\"\"Implements data methods to pretend that an instance is another instance.\n\n    This includes forwarding attribute access and othe methods.\n    \"\"\"\n\n    # Note: we have to special case proxies that themselves return proxies\n    # to support using a proxy as a catch-all for any random access, e.g. `proxy.foo.bar.baz`\n\n    def __getattr__(self, attr: str) -> object:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return proxied  # pyright: ignore\n        return getattr(proxied, attr)\n\n    @override\n    def __repr__(self) -> str:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return proxied.__class__.__name__\n        return repr(self.__get_proxied__())\n\n    @override\n    def __str__(self) -> str:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return proxied.__class__.__name__\n        return str(proxied)\n\n    @override\n    def __dir__(self) -> Iterable[str]:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return []\n        return proxied.__dir__()\n\n    @property  # type: ignore\n    @override\n    def __class__(self) -> type:  # pyright: ignore\n        proxied = self.__get_proxied__()\n        if issubclass(type(proxied), LazyProxy):\n            return type(proxied)\n...\n# Path: src/openai/_utils/_streams.py\nfrom typing import Any\nfrom typing_extensions import Iterator, AsyncIterator\n\n\ndef consume_sync_iterator(iterator: Iterator[Any]) -> None:\n    for _ in iterator:\n        ...\n\n\nasync def consume_async_iterator(iterator: AsyncIterator[Any]) -> None:\n    async for _ in iterator:\n        ...\n\n# Path: src/openai/_utils/_sync.py\nfrom __future__ import annotations\n\nimport functools\nfrom typing import TypeVar, Callable, Awaitable\nfrom typing_extensions import ParamSpec\n\nimport anyio\nimport anyio.to_thread\n\nT_Retval = TypeVar(\"T_Retval\")\nT_ParamSpec = ParamSpec(\"T_ParamSpec\")\n\n\n# copied from `asyncer`, https://github.com/tiangolo/asyncer\ndef asyncify(\n    function: Callable[T_ParamSpec, T_Retval],\n    *,\n    cancellable: bool = False,\n    limiter: anyio.CapacityLimiter | None = None,\n) -> Callable[T_ParamSpec, Awaitable[T_Retval]]:\n    \"\"\"\n    Take a blocking function and create an async one that receives the same\n    positional and keyword arguments, and that when called, calls the original function\n    in a worker thread using `anyio.to_thread.run_sync()`. Internally,\n    `asyncer.asyncify()` uses the same `anyio.to_thread.run_sync()`, but it supports\n    keyword arguments additional to positional arguments and it adds better support for\n    autocompletion and inline errors for the arguments of the function called and the\n    return value.\n\n    If the `cancellable` option is enabled and the task waiting for its completion is\n    cancelled, the thread will still run its course but its return value (or any raised\n    exception) will be ignored.\n\n    Use it like this:\n\n    ```Python\n    def do_work(arg1, arg2, kwarg1=\"\", kwarg2=\"\") -> str:\n        # Do work\n        return \"Some result\"\n\n\n    result = await to_thread.asyncify(do_work)(\"spam\", \"ham\", kwarg1=\"a\", kwarg2=\"b\")\n    print(result)\n    ```\n\n    ## Arguments\n\n    `function`: a blocking regular callable (e.g. a function)\n    `cancellable`: `True` to allow cancellation of the operation\n    `limiter`: capacity limiter to use to limit the total amount of threads running\n        (if omitted, the default limiter is used)\n\n    ## Return\n\n    An async function that takes the same positional and keyword arguments as the\n    original one, that when called runs the same original function in a thread worker\n    and returns the result.\n    \"\"\"\n\n    async def wrapper(*args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs) -> T_Retval:\n        partial_f = functools.partial(function, *args, **kwargs)\n        return await anyio.to_thread.run_sync(partial_f, cancellable=cancellable, limiter=limiter)\n\n    return wrapper\n\n# Path: src/openai/_utils/_typing.py\nfrom __future__ import annotations\n\nfrom typing import Any, TypeVar, Iterable, cast\nfrom collections import abc as _c_abc\nfrom typing_extensions import Required, Annotated, get_args, get_origin\n\nfrom .._types import InheritsGeneric\nfrom .._compat import is_union as _is_union\n\n\ndef is_annotated_type(typ: type) -> bool:\n    return get_origin(typ) == Annotated\n\n\ndef is_list_type(typ: type) -> bool:\n    return (get_origin(typ) or typ) == list\n\n\ndef is_iterable_type(typ: type) -> bool:\n    \"\"\"If the given type is `typing.Iterable[T]`\"\"\"\n    origin = get_origin(typ) or typ\n    return origin == Iterable or origin == _c_abc.Iterable\n\n\ndef is_union_type(typ: type) -> bool:\n    return _is_union(get_origin(typ))\n\n\ndef is_required_type(typ: type) -> bool:\n    return get_origin(typ) == Required\n\n\ndef is_typevar(typ: type) -> bool:\n    # type ignore is required because type checkers\n    # think this expression will always return False\n    return type(typ) == TypeVar  # type: ignore\n\n\n# Extracts T from Annotated[T, ...] or from Required[Annotated[T, ...]]\ndef strip_annotated_type(typ: type) -> type:\n    if is_required_type(typ) or is_annotated_type(typ):\n        return strip_annotated_type(cast(type, get_args(typ)[0]))\n\n    return typ\n\n\ndef extract_type_arg(typ: type, index: int) -> type:\n    args = get_args(typ)\n    try:\n        return cast(type, args[index])\n    except IndexError as err:\n        raise RuntimeError(f\"Expected type {typ} to have a type argument at index {index} but it did not\") from err\n\n\ndef extract_type_var_from_base(\n    typ: type,\n    *,\n    generic_bases: tuple[type, ...],\n    index: int,\n    failure_message: str | None = None,\n) -> type:\n    \"\"\"Given a type like `Foo[T]`, returns the generic type variable `T`.\n\n    This also handles the case where a concrete subclass is given, e.g.\n    ```py\n    class MyResponse(Foo[bytes]):\n        ...\n\n    extract_type_var(MyResponse, bases=(Foo,), index=0) -> bytes\n    ```\n\n    And where a generic subclass is given:\n    ```py\n    _T = TypeVar('_T')\n    class MyResponse(Foo[_T]):\n        ...\n\n    extract_type_var(MyResponse[bytes], bases=(Foo,), index=0) -> bytes\n    ```\n    \"\"\"\n    cls = cast(object, get_origin(typ) or typ)\n    if cls in generic_bases:\n        # we're given the class directly\n        return extract_type_arg(typ, index)\n\n    # if a subclass is given\n    # ---\n    # this is needed as __orig_bases__ is not present in the typeshed stubs\n    # because it is intended to be for internal use only, however there does\n    # not seem to be a way to resolve generic TypeVars for inherited subclasses\n    # without using it.\n    if isinstance(cls, InheritsGeneric):\n        target_base_class: Any | None = None\n        for base in cls.__orig_bases__:\n            if base.__origin__ in generic_bases:\n                target_base_class = base\n                break\n\n        if target_base_class is None:\n            raise RuntimeError(\n                \"Could not find the generic base class;\\n\"\n                \"This should never happen;\\n\"\n                f\"Does {cls} inherit from one of {generic_bases} ?\"\n            )\n\n        extracted = extract_type_arg(target_base_class, index)\n        if is_typevar(extracted):\n            # If the extracted type argument is itself a type variable\n            # then that means the subclass itself is generic, so we have\n            # to resolve the type argument from the class itself, not\n            # the base class.\n            #\n            # Note: if there is more than 1 type argument, the subclass could\n            # change the ordering of the type arguments, this is not currently\n            # supported.\n            return extract_type_arg(typ, index)\n\n        return extracted\n\n    raise RuntimeError(failure_message or f\"Could not resolve inner type variable at index {index} for {typ}\")\n\n# Path: src/openai/_files.py\nfrom __future__ import annotations\n\nimport io\nimport os\nimport pathlib\nfrom typing import overload\nfrom typing_extensions import TypeGuard\n\nimport anyio\n\nfrom ._types import (\n    FileTypes,\n    FileContent,\n    RequestFiles,\n    HttpxFileTypes,\n    HttpxFileContent,\n    HttpxRequestFiles,\n)\nfrom ._utils import is_tuple_t, is_mapping_t, is_sequence_t\n\n\ndef is_file_content(obj: object) -> TypeGuard[FileContent]:\n    return (\n        isinstance(obj, bytes) or isinstance(obj, tuple) or isinstance(obj, io.IOBase) or isinstance(obj, os.PathLike)\n    )\n\n\ndef assert_is_file_content(obj: object, *, key: str | None = None) -> None:\n    if not is_file_content(obj):\n        prefix = f\"Expected entry at `{key}`\" if key is not None else f\"Expected file input `{obj!r}`\"\n        raise RuntimeError(\n            f\"{prefix} to be bytes, an io.IOBase instance, PathLike or a tuple but received {type(obj)} instead. See https://github.com/openai/openai-python/tree/main#file-uploads\"\n        ) from None\n\n\n@overload\ndef to_httpx_files(files: None) -> None:\n    ...\n\n\n@overload\ndef to_httpx_files(files: RequestFiles) -> HttpxRequestFiles:\n    ...\n\n\ndef to_httpx_files(files: RequestFiles | None) -> HttpxRequestFiles | None:\n    if files is None:\n        return None\n\n    if is_mapping_t(files):\n        files = {key: _transform_file(file) for key, file in files.items()}\n    elif is_sequence_t(files):\n        files = [(key, _transform_file(file)) for key, file in files]\n    else:\n        raise TypeError(f\"Unexpected file type input {type(files)}, expected mapping or sequence\")\n\n    return files\n\n\ndef _transform_file(file: FileTypes) -> HttpxFileTypes:\n    if is_file_content(file):\n        if isinstance(file, os.PathLike):\n            path = pathlib.Path(file)\n            return (path.name, path.read_bytes())\n\n        return file\n\n    if is_tuple_t(file):\n        return (file[0], _read_file_content(file[1]), *file[2:])\n\n    raise TypeError(f\"Expected file types input to be a FileContent type or to be a tuple\")\n\n\ndef _read_file_content(file: FileContent) -> HttpxFileContent:\n    if isinstance(file, os.PathLike):\n        return pathlib.Path(file).read_bytes()\n    return file\n\n\n@overload\nasync def async_to_httpx_files(files: None) -> None:\n    ...\n\n\n@overload\nasync def async_to_httpx_files(files: RequestFiles) -> HttpxRequestFiles:\n    ...\n\n\nasync def async_to_httpx_files(files: RequestFiles | None) -> HttpxRequestFiles | None:\n    if files is None:\n        return None\n\n    if is_mapping_t(files):\n        files = {key: await _async_transform_file(file) for key, file in files.items()}\n    elif is_sequence_t(files):\n        files = [(key, await _async_transform_file(file)) for key, file in files]\n    else:\n        raise TypeError(\"Unexpected file type input {type(files)}, expected mapping or sequence\")\n\n    return files\n\n\nasync def _async_transform_file(file: FileTypes) -> HttpxFileTypes:\n    if is_file_content(file):\n        if isinstance(file, os.PathLike):\n            path = anyio.Path(file)\n            return (path.name, await path.read_bytes())\n\n        return file\n\n    if is_tuple_t(file):\n        return (file[0], await _async_read_file_content(file[1]), *file[2:])\n\n    raise TypeError(f\"Expected file types input to be a FileContent type or to be a tuple\")\n\n\nasync def _async_read_file_content(file: FileContent) -> HttpxFileContent:\n    if isinstance(file, os.PathLike):\n        return await anyio.Path(file).read_bytes()\n\n    return file\n\n# Path: src/openai/_utils/_utils.py\nfrom __future__ import annotations\n\nimport os\nimport re\nimport inspect\nimport functools\nfrom typing import (\n    Any,\n    Tuple,\n    Mapping,\n    TypeVar,\n    Callable,\n    Iterable,\n    Sequence,\n    cast,\n    overload,\n)\nfrom pathlib import Path\nfrom typing_extensions import TypeGuard\n\nimport sniffio\n\nfrom .._types import Headers, NotGiven, FileTypes, NotGivenOr, HeadersLike\nfrom .._compat import parse_date as parse_date, parse_datetime as parse_datetime\n\n_T = TypeVar(\"_T\")\n_TupleT = TypeVar(\"_TupleT\", bound=Tuple[object, ...])\n_MappingT = TypeVar(\"_MappingT\", bound=Mapping[str, object])\n_SequenceT = TypeVar(\"_SequenceT\", bound=Sequence[object])\nCallableT = TypeVar(\"CallableT\", bound=Callable[..., Any])\n\n\ndef flatten(t: Iterable[Iterable[_T]]) -> list[_T]:\n    return [item for sublist in t for item in sublist]\n\n\ndef extract_files(\n    # TODO: this needs to take Dict but variance issues.....\n    # create protocol type ?\n    query: Mapping[str, object],\n    *,\n    paths: Sequence[Sequence[str]],\n) -> list[tuple[str, FileTypes]]:\n    \"\"\"Recursively extract files from the given dictionary based on specified paths.\n\n    A path may look like this ['foo', 'files', '<array>', 'data'].\n\n    Note: this mutates the given dictionary.\n    \"\"\"\n    files: list[tuple[str, FileTypes]] = []\n    for path in paths:\n        files.extend(_extract_items(query, path, index=0, flattened_key=None))\n    return files\n\n\ndef _extract_items(\n    obj: object,\n    path: Sequence[str],\n    *,\n    index: int,\n    flattened_key: str | None,\n) -> list[tuple[str, FileTypes]]:\n    try:\n        key = path[index]\n    except IndexError:\n        if isinstance(obj, NotGiven):\n            # no value was provided - we can safely ignore\n            return []\n\n        # cyclical import\n        from .._files import assert_is_file_content\n\n        # We have exhausted the path, return the entry we found.\n        assert_is_file_content(obj, key=flattened_key)\n        assert flattened_key is not None\n        return [(flattened_key, cast(FileTypes, obj))]\n\n    index += 1\n    if is_dict(obj):\n        try:\n            # We are at the last entry in the path so we must remove the field\n            if (len(path)) == index:\n                item = obj.pop(key)\n            else:\n                item = obj[key]\n        except KeyError:\n            # Key was not present in the dictionary, this is not indicative of an error\n            # as the given path may not point to a required field. We also do not want\n            # to enforce required fields as the API may differ from the spec in some cases.\n            return []\n        if flattened_key is None:\n            flattened_key = key\n        else:\n            flattened_key += f\"[{key}]\"\n        return _extract_items(\n            item,\n            path,\n            index=index,\n            flattened_key=flattened_key,\n        )\n    elif is_list(obj):\n        if key != \"<array>\":\n            return []\n\n        return flatten(\n            [\n                _extract_items(\n                    item,\n                    path,\n                    index=index,\n                    flattened_key=flattened_key + \"[]\" if flattened_key is not None else \"[]\",\n                )\n                for item in obj\n            ]\n        )\n\n    # Something unexpected was passed, just ignore it.\n    return []\n\n\ndef is_given(obj: NotGivenOr[_T]) -> TypeGuard[_T]:\n    return not isinstance(obj, NotGiven)\n\n\n# Type safe methods for narrowing types with TypeVars.\n# The default narrowing for isinstance(obj, dict) is dict[unknown, unknown],\n# however this cause Pyright to rightfully report errors. As we know we don't\n# care about the contained types we can safely use `object` in it's place.\n#\n# There are two separate functions defined, `is_*` and `is_*_t` for different use cases.\n# `is_*` is for when you're dealing with an unknown input\n# `is_*_t` is for when you're narrowing a known union type to a specific subset\n\n\ndef is_tuple(obj: object) -> TypeGuard[tuple[object, ...]]:\n    return isinstance(obj, tuple)\n\n\ndef is_tuple_t(obj: _TupleT | object) -> TypeGuard[_TupleT]:\n    return isinstance(obj, tuple)\n\n\ndef is_sequence(obj: object) -> TypeGuard[Sequence[object]]:\n    return isinstance(obj, Sequence)\n\n\ndef is_sequence_t(obj: _SequenceT | object) -> TypeGuard[_SequenceT]:\n    return isinstance(obj, Sequence)\n\n\ndef is_mapping(obj: object) -> TypeGuard[Mapping[str, object]]:\n    return isinstance(obj, Mapping)\n\n\ndef is_mapping_t(obj: _MappingT | object) -> TypeGuard[_MappingT]:\n    return isinstance(obj, Mapping)\n\n\ndef is_dict(obj: object) -> TypeGuard[dict[object, object]]:\n    return isinstance(obj, dict)\n\n\ndef is_list(obj: object) -> TypeGuard[list[object]]:\n    return isinstance(obj, list)\n\n\ndef is_iterable(obj: object) -> TypeGuard[Iterable[object]]:\n    return isinstance(obj, Iterable)\n\n\ndef deepcopy_minimal(item: _T) -> _T:\n    \"\"\"Minimal reimplementation of copy.deepcopy() that will only copy certain object types:\n\n    - mappings, e.g. `dict`\n    - list\n\n    This is done for performance reasons.\n    \"\"\"\n    if is_mapping(item):\n        return cast(_T, {k: deepcopy_minimal(v) for k, v in item.items()})\n    if is_list(item):\n        return cast(_T, [deepcopy_minimal(entry) for entry in item])\n    return item\n\n\n# copied from https://github.com/Rapptz/RoboDanny\ndef human_join(seq: Sequence[str], *, delim: str = \", \", final: str = \"or\") -> str:\n    size = len(seq)\n    if size == 0:\n        return \"\"\n\n    if size == 1:\n        return seq[0]\n\n    if size == 2:\n        return f\"{seq[0]} {final} {seq[1]}\"\n\n    return delim.join(seq[:-1]) + f\" {final} {seq[-1]}\"\n\n\ndef quote(string: str) -> str:\n    \"\"\"Add single quotation marks around the given string. Does *not* do any escaping.\"\"\"\n    return f\"'{string}'\"\n\n\ndef required_args(*variants: Sequence[str]) -> Callable[[CallableT], CallableT]:\n    \"\"\"Decorator to enforce a given set of arguments or variants of arguments are passed to the decorated function.\n\n    Useful for enforcing runtime validation of overloaded functions.\n\n    Example usage:\n    ```py\n    @overload\n    def foo(*, a: str) -> str:\n        ...\n\n\n    @overload\n    def foo(*, b: bool) -> str:\n        ...\n\n\n    # This enforces the same constraints that a static type checker would\n    # i.e. that either a or b must be passed to the function\n    @required_args([\"a\"], [\"b\"])\n    def foo(*, a: str | None = None, b: bool | None = None) -> str:\n        ...\n    ```\n    \"\"\"\n\n    def inner(func: CallableT) -> CallableT:\n        params = inspect.signature(func).parameters\n        positional = [\n            name\n            for name, param in params.items()\n            if param.kind\n            in {\n                param.POSITIONAL_ONLY,\n                param.POSITIONAL_OR_KEYWORD,\n            }\n        ]\n\n        @functools.wraps(func)\n        def wrapper(*args: object, **kwargs: object) -> object:\n            given_params: set[str] = set()\n            for i, _ in enumerate(args):\n                try:\n                    given_params.add(positional[i])\n                except IndexError:\n                    raise TypeError(\n                        f\"{func.__name__}() takes {len(positional)} argument(s) but {len(args)} were given\"\n                    ) from None\n\n            for key in kwargs.keys():\n                given_params.add(key)\n\n            for variant in variants:\n                matches = all((param in given_params for param in variant))\n                if matches:\n                    break\n            else:  # no break\n                if len(variants) > 1:\n                    variations = human_join(\n                        [\"(\" + human_join([quote(arg) for arg in variant], final=\"and\") + \")\" for variant in variants]\n                    )\n                    msg = f\"Missing required arguments; Expected either {variations} arguments to be given\"\n                else:\n                    # TODO: this error message is not deterministic\n                    missing = list(set(variants[0]) - given_params)\n                    if len(missing) > 1:\n                        msg = f\"Missing required arguments: {human_join([quote(arg) for arg in missing])}\"\n                    else:\n                        msg = f\"Missing required argument: {quote(missing[0])}\"\n                raise TypeError(msg)\n            return func(*args, **kwargs)\n\n        return wrapper  # type: ignore\n\n    return inner\n\n\n_K = TypeVar(\"_K\")\n_V = TypeVar(\"_V\")\n\n\n@overload\ndef strip_not_given(obj: None) -> None:\n    ...\n\n\n@overload\ndef strip_not_given(obj: Mapping[_K, _V | NotGiven]) -> dict[_K, _V]:\n    ...\n\n\n@overload\ndef strip_not_given(obj: object) -> object:\n    ...\n\n\ndef strip_not_given(obj: object | None) -> object:\n    \"\"\"Remove all top-level keys where their values are instances of `NotGiven`\"\"\"\n    if obj is None:\n        return None\n\n    if not is_mapping(obj):\n        return obj\n\n    return {key: value for key, value in obj.items() if not isinstance(value, NotGiven)}\n\n\ndef coerce_integer(val: str) -> int:\n    return int(val, base=10)\n\n\ndef coerce_float(val: str) -> float:\n    return float(val)\n\n\ndef coerce_boolean(val: str) -> bool:\n    return val == \"true\" or val == \"1\" or val == \"on\"\n\n\ndef maybe_coerce_integer(val: str | None) -> int | None:\n    if val is None:\n        return None\n    return coerce_integer(val)\n\n\ndef maybe_coerce_float(val: str | None) -> float | None:\n    if val is None:\n        return None\n    return coerce_float(val)\n\n\ndef maybe_coerce_boolean(val: str | None) -> bool | None:\n    if val is None:\n        return None\n    return coerce_boolean(val)\n\n\ndef removeprefix(string: str, prefix: str) -> str:\n    \"\"\"Remove a prefix from a string.\n\n    Backport of `str.removeprefix` for Python < 3.9\n    \"\"\"\n    if string.startswith(prefix):\n        return string[len(prefix) :]\n    return string\n\n\ndef removesuffix(string: str, suffix: str) -> str:\n    \"\"\"Remove a suffix from a string.\n\n    Backport of `str.removesuffix` for Python < 3.9\n    \"\"\"\n    if string.endswith(suffix):\n        return string[: -len(suffix)]\n    return string\n\n\ndef file_from_path(path: str) -> FileTypes:\n    contents = Path(path).read_bytes()\n    file_name = os.path.basename(path)\n    return (file_name, contents)\n\n\ndef get_required_header(headers: HeadersLike, header: str) -> str:\n    lower_header = header.lower()\n    if isinstance(headers, Mapping):\n        headers = cast(Headers, headers)\n        for k, v in headers.items():\n            if k.lower() == lower_header and isinstance(v, str):\n                return v\n\n    \"\"\" to deal with the case where the header looks like Stainless-Event-Id \"\"\"\n    intercaps_header = re.sub(r\"([^\\w])(\\w)\", lambda pat: pat.group(1) + pat.group(2).upper(), header.capitalize())\n\n    for normalized_header in [header, lower_header, header.upper(), intercaps_header]:\n        value = headers.get(normalized_header)\n        if value:\n            return value\n\n    raise ValueError(f\"Could not find {header} header\")\n\n\ndef get_async_library() -> str:\n    try:\n        return sniffio.current_async_library()\n    except Exception:\n        return \"false\"\n\n# Path: src/openai/_utils/_transform.py\nfrom __future__ import annotations\n\nfrom typing import Any, Mapping, TypeVar, cast\nfrom datetime import date, datetime\nfrom typing_extensions import Literal, get_args, override, get_type_hints\n\nimport pydantic\n\nfrom ._utils import (\n    is_list,\n    is_mapping,\n    is_iterable,\n)\nfrom ._typing import (\n    is_list_type,\n    is_union_type,\n    extract_type_arg,\n    is_iterable_type,\n    is_required_type,\n    is_annotated_type,\n    strip_annotated_type,\n)\nfrom .._compat import model_dump, is_typeddict\n\n_T = TypeVar(\"_T\")\n\n\n# TODO: support for drilling globals() and locals()\n# TODO: ensure works correctly with forward references in all cases\n\n\nPropertyFormat = Literal[\"iso8601\", \"custom\"]\n\n\nclass PropertyInfo:\n    \"\"\"Metadata class to be used in Annotated types to provide information about a given type.\n\n    For example:\n\n    class MyParams(TypedDict):\n        account_holder_name: Annotated[str, PropertyInfo(alias='accountHolderName')]\n\n    This means that {'account_holder_name': 'Robert'} will be transformed to {'accountHolderName': 'Robert'} before being sent to the API.\n    \"\"\"\n\n    alias: str | None\n    format: PropertyFormat | None\n    format_template: str | None\n\n    def __init__(\n        self,\n        *,\n        alias: str | None = None,\n        format: PropertyFormat | None = None,\n        format_template: str | None = None,\n    ) -> None:\n        self.alias = alias\n        self.format = format\n        self.format_template = format_template\n\n    @override\n    def __repr__(self) -> str:\n        return f\"{self.__class__.__name__}(alias='{self.alias}', format={self.format}, format_template='{self.format_template}')\"\n\n\ndef maybe_transform(\n    data: object,\n    expected_type: object,\n) -> Any | None:\n    \"\"\"Wrapper over `transform()` that allows `None` to be passed.\n\n    See `transform()` for more details.\n    \"\"\"\n    if data is None:\n        return None\n    return transform(data, expected_type)\n\n\n# Wrapper over _transform_recursive providing fake types\ndef transform(\n    data: _T,\n    expected_type: object,\n) -> _T:\n    \"\"\"Transform dictionaries based off of type information from the given type, for example:\n\n    ```py\n    class Params(TypedDict, total=False):\n        card_id: Required[Annotated[str, PropertyInfo(alias=\"cardID\")]]\n\n\n    transformed = transform({\"card_id\": \"<my card ID>\"}, Params)\n    # {'cardID': '<my card ID>'}\n    ```\n\n    Any keys / data that does not have type information given will be included as is.\n\n    It should be noted that the transformations that this function does are not represented in the type system.\n    \"\"\"\n    transformed = _transform_recursive(data, annotation=cast(type, expected_type))\n    return cast(_T, transformed)\n\n\ndef _get_annotated_type(type_: type) -> type | None:\n    \"\"\"If the given type is an `Annotated` type then it is returned, if not `None` is returned.\n\n    This also unwraps the type when applicable, e.g. `Required[Annotated[T, ...]]`\n    \"\"\"\n    if is_required_type(type_):\n        # Unwrap `Required[Annotated[T, ...]]` to `Annotated[T, ...]`\n        type_ = get_args(type_)[0]\n\n    if is_annotated_type(type_):\n        return type_\n\n    return None\n\n\ndef _maybe_transform_key(key: str, type_: type) -> str:\n    \"\"\"Transform the given `data` based on the annotations provided in `type_`.\n\n    Note: this function only looks at `Annotated` types that contain `PropertInfo` metadata.\n    \"\"\"\n    annotated_type = _get_annotated_type(type_)\n    if annotated_type is None:\n        # no `Annotated` definition for this type, no transformation needed\n        return key\n\n    # ignore the first argument as it is the actual type\n    annotations = get_args(annotated_type)[1:]\n    for annotation in annotations:\n        if isinstance(annotation, PropertyInfo) and annotation.alias is not None:\n            return annotation.alias\n\n    return key\n\n\ndef _transform_recursive(\n    data: object,\n    *,\n    annotation: type,\n    inner_type: type | None = None,\n) -> object:\n    \"\"\"Transform the given data against the expected type.\n\n    Args:\n        annotation: The direct type annotation given to the particular piece of data.\n            This may or may not be wrapped in metadata types, e.g. `Required[T]`, `Annotated[T, ...]` etc\n\n        inner_type: If applicable, this is the \"inside\" type. This is useful in certain cases where the outside type\n            is a container type such as `List[T]`. In that case `inner_type` should be set to `T` so that each entry in\n            the list can be transformed using the metadata from the container type.\n\n            Defaults to the same value as the `annotation` argument.\n    \"\"\"\n    if inner_type is None:\n        inner_type = annotation\n\n    stripped_type = strip_annotated_type(inner_type)\n    if is_typeddict(stripped_type) and is_mapping(data):\n        return _transform_typeddict(data, stripped_type)\n\n    if (\n        # List[T]\n        (is_list_type(stripped_type) and is_list(data))\n        # Iterable[T]\n        or (is_iterable_type(stripped_type) and is_iterable(data) and not isinstance(data, str))\n    ):\n        inner_type = extract_type_arg(stripped_type, 0)\n        return [_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data]\n\n    if is_union_type(stripped_type):\n        # For union types we run the transformation against all subtypes to ensure that everything is transformed.\n        #\n        # TODO: there may be edge cases where the same normalized field name will transform to two different names\n        # in different subtypes.\n        for subtype in get_args(stripped_type):\n            data = _transform_recursive(data, annotation=annotation, inner_type=subtype)\n        return data\n\n    if isinstance(data, pydantic.BaseModel):\n        return model_dump(data, exclude_unset=True)\n\n    return _transform_value(data, annotation)\n\n\n\ndef _transform_value(data: object, type_: type) -> object:\n    annotated_type = _get_annotated_type(type_)\n    if annotated_type is None:\n        return data\n\n    # ignore the first argument as it is the actual type\n    annotations = get_args(annotated_type)[1:]\n    for annotation in annotations:\n        if isinstance(annotation, PropertyInfo) and annotation.format is not None:\n            return _format_data(data, annotation.format, annotation.format_template)\n\n    return data\n\n\ndef _format_data(data: object, format_: PropertyFormat, format_template: str | None) -> object:\n    if isinstance(data, (date, datetime)):\n        if format_ == \"iso8601\":\n            return data.isoformat()\n\n        if format_ == \"custom\" and format_template is not None:\n            return data.strftime(format_template)\n\n    return data\n\n\ndef _transform_typeddict(\n    data: Mapping[str, object],\n    expected_type: type,\n) -> Mapping[str, object]:\n    result: dict[str, object] = {}\n    annotations = get_type_hints(expected_type, include_extras=True)\n    for key, value in data.items():\n        type_ = annotations.get(key)\n        if type_ is None:\n            # we do not have a type annotation for this field, leave it as is\n            result[key] = value\n        else:\n            result[_maybe_transform_key(key, type_)] = _transform_recursive(value, annotation=type_)\n    return result\n\n# Path: src/openai/_utils/__init__.py\nfrom ._sync import asyncify as asyncify\nfrom ._proxy import LazyProxy as LazyProxy\nfrom ._utils import (\n    flatten as flatten,\n    is_dict as is_dict,\n    is_list as is_list,\n    is_given as is_given,\n    is_tuple as is_tuple,\n    is_mapping as is_mapping,\n    is_tuple_t as is_tuple_t,\n    parse_date as parse_date,\n    is_iterable as is_iterable,\n    is_sequence as is_sequence,\n    coerce_float as coerce_float,\n    is_mapping_t as is_mapping_t,\n    removeprefix as removeprefix,\n    removesuffix as removesuffix,\n    extract_files as extract_files,\n    is_sequence_t as is_sequence_t,\n    required_args as required_args,\n    coerce_boolean as coerce_boolean,\n    coerce_integer as coerce_integer,\n    file_from_path as file_from_path,\n    parse_datetime as parse_datetime,\n    strip_not_given as strip_not_given,\n    deepcopy_minimal as deepcopy_minimal,\n    get_async_library as get_async_library,\n    maybe_coerce_float as maybe_coerce_float,\n    get_required_header as get_required_header,\n    maybe_coerce_boolean as maybe_coerce_boolean,\n    maybe_coerce_integer as maybe_coerce_integer,\n)\nfrom ._typing import (\n    is_list_type as is_list_type,\n    is_union_type as is_union_type,\n    extract_type_arg as extract_type_arg,\n    is_iterable_type as is_iterable_type,\n    is_required_type as is_required_type,\n    is_annotated_type as is_annotated_type,\n    strip_annotated_type as strip_annotated_type,\n    extract_type_var_from_base as extract_type_var_from_base,\n)\nfrom ._streams import consume_sync_iterator as consume_sync_iterator, consume_async_iterator as consume_async_iterator\nfrom ._transform import (\n    PropertyInfo as PropertyInfo,\n    transform as transform,\n    maybe_transform as maybe_transform,\n)\n\n# Path: src/openai/_exceptions.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Any, Optional, cast\nfrom typing_extensions import Literal\n\nimport httpx\n\nfrom ._utils import is_dict\n\n__all__ = [\n    \"BadRequestError\",\n    \"AuthenticationError\",\n    \"PermissionDeniedError\",\n    \"NotFoundError\",\n    \"ConflictError\",\n    \"UnprocessableEntityError\",\n    \"RateLimitError\",\n    \"InternalServerError\",\n]\n\n\nclass OpenAIError(Exception):\n    pass\n\n\nclass APIError(OpenAIError):\n    message: str\n    request: httpx.Request\n\n    body: object | None\n    \"\"\"The API response body.\n\n    If the API responded with a valid JSON structure then this property will be the\n    decoded result.\n\n    If it isn't a valid JSON structure then this will be the raw response.\n\n    If there was no response associated with this error then it will be `None`.\n    \"\"\"\n\n    code: Optional[str] = None\n    param: Optional[str] = None\n    type: Optional[str]\n\n    def __init__(self, message: str, request: httpx.Request, *, body: object | None) -> None:\n        super().__init__(message)\n        self.request = request\n        self.message = message\n        self.body = body\n\n        if is_dict(body):\n            self.code = cast(Any, body.get(\"code\"))\n            self.param = cast(Any, body.get(\"param\"))\n            self.type = cast(Any, body.get(\"type\"))\n        else:\n            self.code = None\n            self.param = None\n            self.type = None\n\n\nclass APIResponseValidationError(APIError):\n    response: httpx.Response\n    status_code: int\n\n    def __init__(self, response: httpx.Response, body: object | None, *, message: str | None = None) -> None:\n        super().__init__(message or \"Data returned by API invalid for expected schema.\", response.request, body=body)\n        self.response = response\n        self.status_code = response.status_code\n\n\nclass APIStatusError(APIError):\n    \"\"\"Raised when an API response has a status code of 4xx or 5xx.\"\"\"\n\n    response: httpx.Response\n    status_code: int\n\n    def __init__(self, message: str, *, response: httpx.Response, body: object | None) -> None:\n        super().__init__(message, response.request, body=body)\n        self.response = response\n        self.status_code = response.status_code\n\n\nclass APIConnectionError(APIError):\n    def __init__(self, *, message: str = \"Connection error.\", request: httpx.Request) -> None:\n        super().__init__(message, request, body=None)\n\n\nclass APITimeoutError(APIConnectionError):\n    def __init__(self, request: httpx.Request) -> None:\n        super().__init__(message=\"Request timed out.\", request=request)\n\n\nclass BadRequestError(APIStatusError):\n    status_code: Literal[400] = 400  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass AuthenticationError(APIStatusError):\n    status_code: Literal[401] = 401  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass PermissionDeniedError(APIStatusError):\n    status_code: Literal[403] = 403  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass NotFoundError(APIStatusError):\n    status_code: Literal[404] = 404  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass ConflictError(APIStatusError):\n    status_code: Literal[409] = 409  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass UnprocessableEntityError(APIStatusError):\n    status_code: Literal[422] = 422  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass RateLimitError(APIStatusError):\n    status_code: Literal[429] = 429  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass InternalServerError(APIStatusError):\n    pass\n\n# Path: src/openai/_models.py\nfrom __future__ import annotations\n\nimport inspect\nfrom typing import TYPE_CHECKING, Any, Type, Union, Generic, TypeVar, Callable, cast\nfrom datetime import date, datetime\nfrom typing_extensions import (\n    Unpack,\n    Literal,\n    ClassVar,\n    Protocol,\n    Required,\n    TypedDict,\n    final,\n    override,\n    runtime_checkable,\n)\n\nimport pydantic\nimport pydantic.generics\nfrom pydantic.fields import FieldInfo\n\nfrom ._types import (\n    Body,\n    IncEx,\n    Query,\n    ModelT,\n    Headers,\n    Timeout,\n    NotGiven,\n    AnyMapping,\n    HttpxRequestFiles,\n)\nfrom ._utils import is_list, is_given, is_mapping, parse_date, parse_datetime, strip_not_given\nfrom ._compat import (\n    PYDANTIC_V2,\n    ConfigDict,\n    GenericModel as BaseGenericModel,\n    get_args,\n    is_union,\n    parse_obj,\n    get_origin,\n    is_literal_type,\n    get_model_config,\n    get_model_fields,\n    field_get_default,\n)\nfrom ._constants import RAW_RESPONSE_HEADER\n\n__all__ = [\"BaseModel\", \"GenericModel\"]\n\n_T = TypeVar(\"_T\")\n\n\n@runtime_checkable\nclass _ConfigProtocol(Protocol):\n    allow_population_by_field_name: bool\n\n\nclass BaseModel(pydantic.BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(extra=\"allow\")\n    else:\n\n        @property\n        @override\n        def model_fields_set(self) -> set[str]:\n            # a forwards-compat shim for pydantic v2\n            return self.__fields_set__  # type: ignore\n\n        class Config(pydantic.BaseConfig):  # pyright: ignore[reportDeprecated]\n            extra: Any = pydantic.Extra.allow  # type: ignore\n\n    @override\n    def __str__(self) -> str:\n        # mypy complains about an invalid self arg\n        return f'{self.__repr_name__()}({self.__repr_str__(\", \")})'  # type: ignore[misc]\n\n    # Override the 'construct' method in a way that supports recursive parsing without validation.\n    # Based on https://github.com/samuelcolvin/pydantic/issues/1168#issuecomment-817742836.\n    @classmethod\n    @override\n    def construct(\n        cls: Type[ModelT],\n        _fields_set: set[str] | None = None,\n        **values: object,\n    ) -> ModelT:\n        m = cls.__new__(cls)\n        fields_values: dict[str, object] = {}\n\n        config = get_model_config(cls)\n        populate_by_name = (\n            config.allow_population_by_field_name\n            if isinstance(config, _ConfigProtocol)\n            else config.get(\"populate_by_name\")\n        )\n\n        if _fields_set is None:\n            _fields_set = set()\n\n        model_fields = get_model_fields(cls)\n        for name, field in model_fields.items():\n            key = field.alias\n            if key is None or (key not in values and populate_by_name):\n                key = name\n\n            if key in values:\n                fields_values[name] = _construct_field(value=values[key], field=field, key=key)\n                _fields_set.add(name)\n            else:\n                fields_values[name] = field_get_default(field)\n\n        _extra = {}\n        for key, value in values.items():\n            if key not in model_fields:\n                if PYDANTIC_V2:\n                    _extra[key] = value\n                else:\n                    _fields_set.add(key)\n                    fields_values[key] = value\n\n        object.__setattr__(m, \"__dict__\", fields_values)\n\n        if PYDANTIC_V2:\n            # these properties are copied from Pydantic's `model_construct()` method\n            object.__setattr__(m, \"__pydantic_private__\", None)\n            object.__setattr__(m, \"__pydantic_extra__\", _extra)\n            object.__setattr__(m, \"__pydantic_fields_set__\", _fields_set)\n        else:\n            # init_private_attributes() does not exist in v2\n            m._init_private_attributes()  # type: ignore\n\n            # copied from Pydantic v1's `construct()` method\n            object.__setattr__(m, \"__fields_set__\", _fields_set)\n\n        return m\n\n    if not TYPE_CHECKING:\n        # type checkers incorrectly complain about this assignment\n        # because the type signatures are technically different\n        # although not in practice\n        model_construct = construct\n\n    if not PYDANTIC_V2:\n        # we define aliases for some of the new pydantic v2 methods so\n        # that we can just document these methods without having to specify\n        # a specific pydantic version as some users may not know which\n        # pydantic version they are currently using\n\n        @override\n        def model_dump(\n            self,\n            *,\n            mode: Literal[\"json\", \"python\"] | str = \"python\",\n            include: IncEx = None,\n            exclude: IncEx = None,\n            by_alias: bool = False,\n            exclude_unset: bool = False,\n            exclude_defaults: bool = False,\n            exclude_none: bool = False,\n            round_trip: bool = False,\n            warnings: bool = True,\n        ) -> dict[str, Any]:\n            \"\"\"Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump\n\n            Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.\n\n            Args:\n                mode: The mode in which `to_python` should run.\n                    If mode is 'json', the dictionary will only contain JSON serializable types.\n                    If mode is 'python', the dictionary may contain any Python objects.\n                include: A list of fields to include in the output.\n                exclude: A list of fields to exclude from the output.\n                by_alias: Whether to use the field's alias in the dictionary key if defined.\n                exclude_unset: Whether to exclude fields that are unset or None from the output.\n                exclude_defaults: Whether to exclude fields that are set to their default value from the output.\n                exclude_none: Whether to exclude fields that have a value of `None` from the output.\n                round_trip: Whether to enable serialization and deserialization round-trip support.\n                warnings: Whether to log warnings when invalid fields are encountered.\n\n            Returns:\n                A dictionary representation of the model.\n            \"\"\"\n            if mode != \"python\":\n                raise ValueError(\"mode is only supported in Pydantic v2\")\n            if round_trip != False:\n                raise ValueError(\"round_trip is only supported in Pydantic v2\")\n            if warnings != True:\n                raise ValueError(\"warnings is only supported in Pydantic v2\")\n            return super().dict(  # pyright: ignore[reportDeprecated]\n                include=include,\n                exclude=exclude,\n                by_alias=by_alias,\n                exclude_unset=exclude_unset,\n                exclude_defaults=exclude_defaults,\n                exclude_none=exclude_none,\n            )\n\n        @override\n        def model_dump_json(\n            self,\n            *,\n            indent: int | None = None,\n            include: IncEx = None,\n            exclude: IncEx = None,\n            by_alias: bool = False,\n            exclude_unset: bool = False,\n            exclude_defaults: bool = False,\n            exclude_none: bool = False,\n            round_trip: bool = False,\n            warnings: bool = True,\n        ) -> str:\n            \"\"\"Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump_json\n\n            Generates a JSON representation of the model using Pydantic's `to_json` method.\n\n            Args:\n                indent: Indentation to use in the JSON output. If None is passed, the output will be compact.\n                include: Field(s) to include in the JSON output. Can take either a string or set of strings.\n                exclude: Field(s) to exclude from the JSON output. Can take either a string or set of strings.\n                by_alias: Whether to serialize using field aliases.\n                exclude_unset: Whether to exclude fields that have not been explicitly set.\n                exclude_defaults: Whether to exclude fields that have the default value.\n                exclude_none: Whether to exclude fields that have a value of `None`.\n                round_trip: Whether to use serialization/deserialization between JSON and class instance.\n                warnings: Whether to show any warnings that occurred during serialization.\n\n            Returns:\n                A JSON string representation of the model.\n            \"\"\"\n            if round_trip != False:\n                raise ValueError(\"round_trip is only supported in Pydantic v2\")\n            if warnings != True:\n                raise ValueError(\"warnings is only supported in Pydantic v2\")\n            return super().json(  # type: ignore[reportDeprecated]\n                indent=indent,\n                include=include,\n                exclude=exclude,\n                by_alias=by_alias,\n                exclude_unset=exclude_unset,\n                exclude_defaults=exclude_defaults,\n                exclude_none=exclude_none,\n            )\n\n\ndef _construct_field(value: object, field: FieldInfo, key: str) -> object:\n    if value is None:\n        return field_get_default(field)\n\n    if PYDANTIC_V2:\n        type_ = field.annotation\n    else:\n        type_ = cast(type, field.outer_type_)  # type: ignore\n\n    if type_ is None:\n        raise RuntimeError(f\"Unexpected field type is None for {key}\")\n\n    return construct_type(value=value, type_=type_)\n\n\ndef is_basemodel(type_: type) -> bool:\n    \"\"\"Returns whether or not the given type is either a `BaseModel` or a union of `BaseModel`\"\"\"\n    origin = get_origin(type_) or type_\n    if is_union(type_):\n        for variant in get_args(type_):\n            if is_basemodel(variant):\n                return True\n\n        return False\n\n    return issubclass(origin, BaseModel) or issubclass(origin, GenericModel)\n\n\ndef construct_type(*, value: object, type_: type) -> object:\n    \"\"\"Loose coercion to the expected type with construction of nested values.\n\n    If the given value does not match the expected type then it is returned as-is.\n    \"\"\"\n\n    # we need to use the origin class for any types that are subscripted generics\n    # e.g. Dict[str, object]\n    origin = get_origin(type_) or type_\n    args = get_args(type_)\n\n    if is_union(origin):\n        try:\n            return validate_type(type_=cast(\"type[object]\", type_), value=value)\n        except Exception:\n            pass\n\n        # if the data is not valid, use the first variant that doesn't fail while deserializing\n        for variant in args:\n            try:\n                return construct_type(value=value, type_=variant)\n            except Exception:\n                continue\n\n        raise RuntimeError(f\"Could not convert data into a valid instance of {type_}\")\n\n    if origin == dict:\n        if not is_mapping(value):\n            return value\n\n        _, items_type = get_args(type_)  # Dict[_, items_type]\n        return {key: construct_type(value=item, type_=items_type) for key, item in value.items()}\n\n    if not is_literal_type(type_) and (issubclass(origin, BaseModel) or issubclass(origin, GenericModel)):\n        if is_list(value):\n            return [cast(Any, type_).construct(**entry) if is_mapping(entry) else entry for entry in value]\n\n        if is_mapping(value):\n            if issubclass(type_, BaseModel):\n                return type_.construct(**value)  # type: ignore[arg-type]\n\n            return cast(Any, type_).construct(**value)\n\n    if origin == list:\n        if not is_list(value):\n            return value\n\n        inner_type = args[0]  # List[inner_type]\n        return [construct_type(value=entry, type_=inner_type) for entry in value]\n\n    if origin == float:\n        if isinstance(value, int):\n            coerced = float(value)\n            if coerced != value:\n                return value\n            return coerced\n\n        return value\n\n    if type_ == datetime:\n        try:\n            return parse_datetime(value)  # type: ignore\n        except Exception:\n            return value\n\n    if type_ == date:\n        try:\n            return parse_date(value)  # type: ignore\n        except Exception:\n            return value\n\n    return value\n\n\ndef validate_type(*, type_: type[_T], value: object) -> _T:\n    \"\"\"Strict validation that the given value matches the expected type\"\"\"\n    if inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n        return cast(_T, parse_obj(type_, value))\n\n    return cast(_T, _validate_non_model_type(type_=type_, value=value))\n\n\n# our use of subclasssing here causes weirdness for type checkers,\n# so we just pretend that we don't subclass\nif TYPE_CHECKING:\n    GenericModel = BaseModel\nelse:\n\n    class GenericModel(BaseGenericModel, BaseModel):\n        pass\n\n\nif PYDANTIC_V2:\n    from pydantic import TypeAdapter\n\n    def _validate_non_model_type(*, type_: type[_T], value: object) -> _T:\n        return TypeAdapter(type_).validate_python(value)\n\nelif not TYPE_CHECKING:  # TODO: condition is weird\n\n    class RootModel(GenericModel, Generic[_T]):\n        \"\"\"Used as a placeholder to easily convert runtime types to a Pydantic format\n        to provide validation.\n\n        For example:\n        ```py\n        validated = RootModel[int](__root__=\"5\").__root__\n        # validated: 5\n        ```\n        \"\"\"\n\n        __root__: _T\n\n    def _validate_non_model_type(*, type_: type[_T], value: object) -> _T:\n        model = _create_pydantic_model(type_).validate(value)\n        return cast(_T, model.__root__)\n\n    def _create_pydantic_model(type_: _T) -> Type[RootModel[_T]]:\n        return RootModel[type_]  # type: ignore\n\n\nclass FinalRequestOptionsInput(TypedDict, total=False):\n    method: Required[str]\n    url: Required[str]\n    params: Query\n    headers: Headers\n    max_retries: int\n    timeout: float | Timeout | None\n    files: HttpxRequestFiles | None\n    idempotency_key: str\n    json_data: Body\n    extra_json: AnyMapping\n\n\n@final\nclass FinalRequestOptions(pydantic.BaseModel):\n    method: str\n    url: str\n    params: Query = {}\n    headers: Union[Headers, NotGiven] = NotGiven()\n    max_retries: Union[int, NotGiven] = NotGiven()\n    timeout: Union[float, Timeout, None, NotGiven] = NotGiven()\n    files: Union[HttpxRequestFiles, None] = None\n    idempotency_key: Union[str, None] = None\n    post_parser: Union[Callable[[Any], Any], NotGiven] = NotGiven()\n\n    # It should be noted that we cannot use `json` here as that would override\n    # a BaseModel method in an incompatible fashion.\n    json_data: Union[Body, None] = None\n    extra_json: Union[AnyMapping, None] = None\n\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)\n    else:\n\n        class Config(pydantic.BaseConfig):  # pyright: ignore[reportDeprecated]\n            arbitrary_types_allowed: bool = True\n\n    def get_max_retries(self, max_retries: int) -> int:\n        if isinstance(self.max_retries, NotGiven):\n            return max_retries\n        return self.max_retries\n\n    def _strip_raw_response_header(self) -> None:\n        if not is_given(self.headers):\n            return\n\n        if self.headers.get(RAW_RESPONSE_HEADER):\n            self.headers = {**self.headers}\n            self.headers.pop(RAW_RESPONSE_HEADER)\n\n    # override the `construct` method so that we can run custom transformations.\n    # this is necessary as we don't want to do any actual runtime type checking\n    # (which means we can't use validators) but we do want to ensure that `NotGiven`\n    # values are not present\n    #\n    # type ignore required because we're adding explicit types to `**values`\n    @classmethod\n    def construct(  # type: ignore\n        cls,\n        _fields_set: set[str] | None = None,\n        **values: Unpack[FinalRequestOptionsInput],\n    ) -> FinalRequestOptions:\n        kwargs: dict[str, Any] = {\n            # we unconditionally call `strip_not_given` on any value\n            # as it will just ignore any non-mapping types\n            key: strip_not_given(value)\n            for key, value in values.items()\n        }\n        if PYDANTIC_V2:\n            return super().model_construct(_fields_set, **kwargs)\n        return cast(FinalRequestOptions, super().construct(_fields_set, **kwargs))  # pyright: ignore[reportDeprecated]\n\n    if not TYPE_CHECKING:\n        # type checkers incorrectly complain about this assignment\n        model_construct = construct\n\n# Path: src/openai/_qs.py\nfrom __future__ import annotations\n\nfrom typing import Any, List, Tuple, Union, Mapping, TypeVar\nfrom urllib.parse import parse_qs, urlencode\nfrom typing_extensions import Literal, get_args\n\nfrom ._types import NOT_GIVEN, NotGiven, NotGivenOr\nfrom ._utils import flatten\n\n_T = TypeVar(\"_T\")\n\n\nArrayFormat = Literal[\"comma\", \"repeat\", \"indices\", \"brackets\"]\nNestedFormat = Literal[\"dots\", \"brackets\"]\n\nPrimitiveData = Union[str, int, float, bool, None]\n# this should be Data = Union[PrimitiveData, \"List[Data]\", \"Tuple[Data]\", \"Mapping[str, Data]\"]\n# https://github.com/microsoft/pyright/issues/3555\nData = Union[PrimitiveData, List[Any], Tuple[Any], \"Mapping[str, Any]\"]\nParams = Mapping[str, Data]\n\n\nclass Querystring:\n    array_format: ArrayFormat\n    nested_format: NestedFormat\n\n    def __init__(\n        self,\n        *,\n        array_format: ArrayFormat = \"repeat\",\n        nested_format: NestedFormat = \"brackets\",\n    ) -> None:\n        self.array_format = array_format\n        self.nested_format = nested_format\n\n    def parse(self, query: str) -> Mapping[str, object]:\n        # Note: custom format syntax is not supported yet\n        return parse_qs(query)\n\n    def stringify(\n        self,\n        params: Params,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n        nested_format: NotGivenOr[NestedFormat] = NOT_GIVEN,\n    ) -> str:\n        return urlencode(\n            self.stringify_items(\n                params,\n                array_format=array_format,\n                nested_format=nested_format,\n            )\n        )\n\n    def stringify_items(\n        self,\n        params: Params,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n...\n", "needle_token_start": 8684, "needle_token_end": 8813, "code_context_ntokens": 15954, "output": ["```python\ndef maybe_transform(\n    data: object,\n    expected_type: object,\n) -> Any | None:\n    \"\"\"Wrapper over `transform()` that allows `None` to be passed.\n\n    See `transform()` for more details.\n    \"\"\"\n    if data is None:\n        return None\n    return transform(data, expected_type)\n```"]}
{"repo": "openai/openai-python", "name": "_construct_field", "language": "python", "path": "src/openai/_models.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to construct and return an object of a specified type based on the provided value and field metadata.\n2. **Input**: The function takes three parameters: a value (any object), field metadata (which includes type annotations and default values), and a key (string representing the field name).\n3. **Output**: Returns an object that is constructed according to the specified type in the field metadata. If the input value is `None`, it returns the default value defined in the field metadata.\n4. **Procedure**: \n   - First, it checks if the input value is `None`. If true, it retrieves and returns the default value from the field metadata.\n   - It then determines the type of the field. If a certain version condition is met, it uses the type annotation directly from the field metadata; otherwise, it uses a type conversion method to get the appropriate type.\n   - If the determined type is `None`, it raises an error indicating an unexpected `None` type for the field.\n   - Finally, it constructs and returns an object of the determined type using the provided value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/openai/_files.py\nfrom __future__ import annotations\n\nimport io\nimport os\nimport pathlib\nfrom typing import overload\nfrom typing_extensions import TypeGuard\n\nimport anyio\n\nfrom ._types import (\n    FileTypes,\n    FileContent,\n    RequestFiles,\n    HttpxFileTypes,\n    HttpxFileContent,\n    HttpxRequestFiles,\n)\nfrom ._utils import is_tuple_t, is_mapping_t, is_sequence_t\n\n\ndef is_file_content(obj: object) -> TypeGuard[FileContent]:\n    return (\n        isinstance(obj, bytes) or isinstance(obj, tuple) or isinstance(obj, io.IOBase) or isinstance(obj, os.PathLike)\n    )\n\n\ndef assert_is_file_content(obj: object, *, key: str | None = None) -> None:\n    if not is_file_content(obj):\n        prefix = f\"Expected entry at `{key}`\" if key is not None else f\"Expected file input `{obj!r}`\"\n        raise RuntimeError(\n            f\"{prefix} to be bytes, an io.IOBase instance, PathLike or a tuple but received {type(obj)} instead. See https://github.com/openai/openai-python/tree/main#file-uploads\"\n        ) from None\n\n\n@overload\ndef to_httpx_files(files: None) -> None:\n    ...\n\n\n@overload\ndef to_httpx_files(files: RequestFiles) -> HttpxRequestFiles:\n    ...\n\n\ndef to_httpx_files(files: RequestFiles | None) -> HttpxRequestFiles | None:\n    if files is None:\n        return None\n\n    if is_mapping_t(files):\n        files = {key: _transform_file(file) for key, file in files.items()}\n    elif is_sequence_t(files):\n        files = [(key, _transform_file(file)) for key, file in files]\n    else:\n        raise TypeError(f\"Unexpected file type input {type(files)}, expected mapping or sequence\")\n\n    return files\n\n\ndef _transform_file(file: FileTypes) -> HttpxFileTypes:\n    if is_file_content(file):\n        if isinstance(file, os.PathLike):\n            path = pathlib.Path(file)\n            return (path.name, path.read_bytes())\n\n        return file\n\n    if is_tuple_t(file):\n        return (file[0], _read_file_content(file[1]), *file[2:])\n\n    raise TypeError(f\"Expected file types input to be a FileContent type or to be a tuple\")\n\n\n...\n# Path: src/openai/_utils/_utils.py\nfrom __future__ import annotations\n\nimport os\nimport re\nimport inspect\nimport functools\nfrom typing import (\n    Any,\n    Tuple,\n    Mapping,\n    TypeVar,\n    Callable,\n    Iterable,\n    Sequence,\n    cast,\n    overload,\n)\nfrom pathlib import Path\nfrom typing_extensions import TypeGuard\n\nimport sniffio\n\nfrom .._types import Headers, NotGiven, FileTypes, NotGivenOr, HeadersLike\nfrom .._compat import parse_date as parse_date, parse_datetime as parse_datetime\n\n_T = TypeVar(\"_T\")\n_TupleT = TypeVar(\"_TupleT\", bound=Tuple[object, ...])\n_MappingT = TypeVar(\"_MappingT\", bound=Mapping[str, object])\n_SequenceT = TypeVar(\"_SequenceT\", bound=Sequence[object])\nCallableT = TypeVar(\"CallableT\", bound=Callable[..., Any])\n\n\ndef flatten(t: Iterable[Iterable[_T]]) -> list[_T]:\n    return [item for sublist in t for item in sublist]\n\n\ndef extract_files(\n    # TODO: this needs to take Dict but variance issues.....\n    # create protocol type ?\n    query: Mapping[str, object],\n    *,\n    paths: Sequence[Sequence[str]],\n) -> list[tuple[str, FileTypes]]:\n    \"\"\"Recursively extract files from the given dictionary based on specified paths.\n\n    A path may look like this ['foo', 'files', '<array>', 'data'].\n\n    Note: this mutates the given dictionary.\n    \"\"\"\n    files: list[tuple[str, FileTypes]] = []\n    for path in paths:\n        files.extend(_extract_items(query, path, index=0, flattened_key=None))\n    return files\n\n\ndef _extract_items(\n    obj: object,\n    path: Sequence[str],\n    *,\n    index: int,\n    flattened_key: str | None,\n) -> list[tuple[str, FileTypes]]:\n    try:\n        key = path[index]\n    except IndexError:\n        if isinstance(obj, NotGiven):\n            # no value was provided - we can safely ignore\n            return []\n\n        # cyclical import\n        from .._files import assert_is_file_content\n\n        # We have exhausted the path, return the entry we found.\n        assert_is_file_content(obj, key=flattened_key)\n        assert flattened_key is not None\n        return [(flattened_key, cast(FileTypes, obj))]\n\n    index += 1\n    if is_dict(obj):\n        try:\n            # We are at the last entry in the path so we must remove the field\n            if (len(path)) == index:\n                item = obj.pop(key)\n            else:\n                item = obj[key]\n        except KeyError:\n            # Key was not present in the dictionary, this is not indicative of an error\n            # as the given path may not point to a required field. We also do not want\n            # to enforce required fields as the API may differ from the spec in some cases.\n            return []\n        if flattened_key is None:\n            flattened_key = key\n        else:\n            flattened_key += f\"[{key}]\"\n        return _extract_items(\n            item,\n            path,\n            index=index,\n            flattened_key=flattened_key,\n        )\n    elif is_list(obj):\n        if key != \"<array>\":\n            return []\n\n        return flatten(\n            [\n                _extract_items(\n                    item,\n                    path,\n                    index=index,\n                    flattened_key=flattened_key + \"[]\" if flattened_key is not None else \"[]\",\n                )\n                for item in obj\n            ]\n        )\n\n    # Something unexpected was passed, just ignore it.\n    return []\n\n\ndef is_given(obj: NotGivenOr[_T]) -> TypeGuard[_T]:\n    return not isinstance(obj, NotGiven)\n\n\n# Type safe methods for narrowing types with TypeVars.\n# The default narrowing for isinstance(obj, dict) is dict[unknown, unknown],\n# however this cause Pyright to rightfully report errors. As we know we don't\n# care about the contained types we can safely use `object` in it's place.\n#\n# There are two separate functions defined, `is_*` and `is_*_t` for different use cases.\n# `is_*` is for when you're dealing with an unknown input\n# `is_*_t` is for when you're narrowing a known union type to a specific subset\n\n\ndef is_tuple(obj: object) -> TypeGuard[tuple[object, ...]]:\n    return isinstance(obj, tuple)\n\n\ndef is_tuple_t(obj: _TupleT | object) -> TypeGuard[_TupleT]:\n    return isinstance(obj, tuple)\n\n\ndef is_sequence(obj: object) -> TypeGuard[Sequence[object]]:\n    return isinstance(obj, Sequence)\n\n\ndef is_sequence_t(obj: _SequenceT | object) -> TypeGuard[_SequenceT]:\n    return isinstance(obj, Sequence)\n\n\ndef is_mapping(obj: object) -> TypeGuard[Mapping[str, object]]:\n    return isinstance(obj, Mapping)\n\n\ndef is_mapping_t(obj: _MappingT | object) -> TypeGuard[_MappingT]:\n    return isinstance(obj, Mapping)\n\n\ndef is_dict(obj: object) -> TypeGuard[dict[object, object]]:\n    return isinstance(obj, dict)\n\n\ndef is_list(obj: object) -> TypeGuard[list[object]]:\n    return isinstance(obj, list)\n\n\ndef is_iterable(obj: object) -> TypeGuard[Iterable[object]]:\n    return isinstance(obj, Iterable)\n\n\ndef deepcopy_minimal(item: _T) -> _T:\n    \"\"\"Minimal reimplementation of copy.deepcopy() that will only copy certain object types:\n\n    - mappings, e.g. `dict`\n    - list\n\n    This is done for performance reasons.\n    \"\"\"\n    if is_mapping(item):\n        return cast(_T, {k: deepcopy_minimal(v) for k, v in item.items()})\n    if is_list(item):\n        return cast(_T, [deepcopy_minimal(entry) for entry in item])\n    return item\n\n\n# copied from https://github.com/Rapptz/RoboDanny\ndef human_join(seq: Sequence[str], *, delim: str = \", \", final: str = \"or\") -> str:\n    size = len(seq)\n    if size == 0:\n        return \"\"\n\n    if size == 1:\n        return seq[0]\n\n    if size == 2:\n        return f\"{seq[0]} {final} {seq[1]}\"\n\n    return delim.join(seq[:-1]) + f\" {final} {seq[-1]}\"\n\n\ndef quote(string: str) -> str:\n    \"\"\"Add single quotation marks around the given string. Does *not* do any escaping.\"\"\"\n    return f\"'{string}'\"\n\n\ndef required_args(*variants: Sequence[str]) -> Callable[[CallableT], CallableT]:\n    \"\"\"Decorator to enforce a given set of arguments or variants of arguments are passed to the decorated function.\n\n    Useful for enforcing runtime validation of overloaded functions.\n\n    Example usage:\n    ```py\n    @overload\n    def foo(*, a: str) -> str:\n        ...\n\n\n    @overload\n    def foo(*, b: bool) -> str:\n        ...\n\n\n    # This enforces the same constraints that a static type checker would\n    # i.e. that either a or b must be passed to the function\n    @required_args([\"a\"], [\"b\"])\n    def foo(*, a: str | None = None, b: bool | None = None) -> str:\n        ...\n    ```\n    \"\"\"\n\n    def inner(func: CallableT) -> CallableT:\n        params = inspect.signature(func).parameters\n        positional = [\n            name\n            for name, param in params.items()\n            if param.kind\n            in {\n                param.POSITIONAL_ONLY,\n                param.POSITIONAL_OR_KEYWORD,\n            }\n        ]\n\n        @functools.wraps(func)\n        def wrapper(*args: object, **kwargs: object) -> object:\n            given_params: set[str] = set()\n            for i, _ in enumerate(args):\n                try:\n                    given_params.add(positional[i])\n                except IndexError:\n                    raise TypeError(\n                        f\"{func.__name__}() takes {len(positional)} argument(s) but {len(args)} were given\"\n                    ) from None\n\n            for key in kwargs.keys():\n                given_params.add(key)\n\n            for variant in variants:\n                matches = all((param in given_params for param in variant))\n                if matches:\n                    break\n            else:  # no break\n                if len(variants) > 1:\n                    variations = human_join(\n                        [\"(\" + human_join([quote(arg) for arg in variant], final=\"and\") + \")\" for variant in variants]\n                    )\n                    msg = f\"Missing required arguments; Expected either {variations} arguments to be given\"\n                else:\n                    # TODO: this error message is not deterministic\n                    missing = list(set(variants[0]) - given_params)\n                    if len(missing) > 1:\n                        msg = f\"Missing required arguments: {human_join([quote(arg) for arg in missing])}\"\n                    else:\n                        msg = f\"Missing required argument: {quote(missing[0])}\"\n                raise TypeError(msg)\n            return func(*args, **kwargs)\n\n        return wrapper  # type: ignore\n\n    return inner\n\n\n_K = TypeVar(\"_K\")\n_V = TypeVar(\"_V\")\n\n\n@overload\ndef strip_not_given(obj: None) -> None:\n    ...\n\n\n@overload\ndef strip_not_given(obj: Mapping[_K, _V | NotGiven]) -> dict[_K, _V]:\n    ...\n\n\n@overload\ndef strip_not_given(obj: object) -> object:\n    ...\n\n\ndef strip_not_given(obj: object | None) -> object:\n    \"\"\"Remove all top-level keys where their values are instances of `NotGiven`\"\"\"\n    if obj is None:\n        return None\n\n    if not is_mapping(obj):\n        return obj\n\n    return {key: value for key, value in obj.items() if not isinstance(value, NotGiven)}\n\n\ndef coerce_integer(val: str) -> int:\n    return int(val, base=10)\n\n\ndef coerce_float(val: str) -> float:\n    return float(val)\n\n\ndef coerce_boolean(val: str) -> bool:\n    return val == \"true\" or val == \"1\" or val == \"on\"\n\n\ndef maybe_coerce_integer(val: str | None) -> int | None:\n    if val is None:\n        return None\n    return coerce_integer(val)\n\n\ndef maybe_coerce_float(val: str | None) -> float | None:\n    if val is None:\n        return None\n    return coerce_float(val)\n\n\ndef maybe_coerce_boolean(val: str | None) -> bool | None:\n    if val is None:\n        return None\n    return coerce_boolean(val)\n\n\ndef removeprefix(string: str, prefix: str) -> str:\n    \"\"\"Remove a prefix from a string.\n\n    Backport of `str.removeprefix` for Python < 3.9\n    \"\"\"\n    if string.startswith(prefix):\n        return string[len(prefix) :]\n    return string\n\n\ndef removesuffix(string: str, suffix: str) -> str:\n    \"\"\"Remove a suffix from a string.\n\n    Backport of `str.removesuffix` for Python < 3.9\n    \"\"\"\n    if string.endswith(suffix):\n        return string[: -len(suffix)]\n    return string\n\n\ndef file_from_path(path: str) -> FileTypes:\n    contents = Path(path).read_bytes()\n    file_name = os.path.basename(path)\n    return (file_name, contents)\n\n\ndef get_required_header(headers: HeadersLike, header: str) -> str:\n    lower_header = header.lower()\n    if isinstance(headers, Mapping):\n        headers = cast(Headers, headers)\n        for k, v in headers.items():\n            if k.lower() == lower_header and isinstance(v, str):\n                return v\n\n    \"\"\" to deal with the case where the header looks like Stainless-Event-Id \"\"\"\n    intercaps_header = re.sub(r\"([^\\w])(\\w)\", lambda pat: pat.group(1) + pat.group(2).upper(), header.capitalize())\n\n    for normalized_header in [header, lower_header, header.upper(), intercaps_header]:\n        value = headers.get(normalized_header)\n        if value:\n            return value\n\n    raise ValueError(f\"Could not find {header} header\")\n\n\ndef get_async_library() -> str:\n    try:\n        return sniffio.current_async_library()\n    except Exception:\n        return \"false\"\n\n# Path: src/openai/_utils/_transform.py\nfrom __future__ import annotations\n\nfrom typing import Any, Mapping, TypeVar, cast\nfrom datetime import date, datetime\nfrom typing_extensions import Literal, get_args, override, get_type_hints\n\nimport pydantic\n\nfrom ._utils import (\n    is_list,\n    is_mapping,\n    is_iterable,\n)\nfrom ._typing import (\n    is_list_type,\n    is_union_type,\n    extract_type_arg,\n    is_iterable_type,\n    is_required_type,\n    is_annotated_type,\n    strip_annotated_type,\n)\nfrom .._compat import model_dump, is_typeddict\n\n_T = TypeVar(\"_T\")\n\n\n# TODO: support for drilling globals() and locals()\n# TODO: ensure works correctly with forward references in all cases\n\n\nPropertyFormat = Literal[\"iso8601\", \"custom\"]\n\n\nclass PropertyInfo:\n    \"\"\"Metadata class to be used in Annotated types to provide information about a given type.\n\n    For example:\n\n    class MyParams(TypedDict):\n        account_holder_name: Annotated[str, PropertyInfo(alias='accountHolderName')]\n\n    This means that {'account_holder_name': 'Robert'} will be transformed to {'accountHolderName': 'Robert'} before being sent to the API.\n    \"\"\"\n\n    alias: str | None\n    format: PropertyFormat | None\n    format_template: str | None\n\n    def __init__(\n        self,\n        *,\n        alias: str | None = None,\n        format: PropertyFormat | None = None,\n        format_template: str | None = None,\n    ) -> None:\n        self.alias = alias\n        self.format = format\n        self.format_template = format_template\n\n    @override\n    def __repr__(self) -> str:\n        return f\"{self.__class__.__name__}(alias='{self.alias}', format={self.format}, format_template='{self.format_template}')\"\n\n\ndef maybe_transform(\n    data: object,\n    expected_type: object,\n) -> Any | None:\n    \"\"\"Wrapper over `transform()` that allows `None` to be passed.\n\n    See `transform()` for more details.\n    \"\"\"\n    if data is None:\n        return None\n    return transform(data, expected_type)\n\n\n# Wrapper over _transform_recursive providing fake types\ndef transform(\n    data: _T,\n    expected_type: object,\n) -> _T:\n    \"\"\"Transform dictionaries based off of type information from the given type, for example:\n\n    ```py\n    class Params(TypedDict, total=False):\n        card_id: Required[Annotated[str, PropertyInfo(alias=\"cardID\")]]\n\n\n    transformed = transform({\"card_id\": \"<my card ID>\"}, Params)\n    # {'cardID': '<my card ID>'}\n    ```\n\n    Any keys / data that does not have type information given will be included as is.\n\n    It should be noted that the transformations that this function does are not represented in the type system.\n    \"\"\"\n    transformed = _transform_recursive(data, annotation=cast(type, expected_type))\n    return cast(_T, transformed)\n\n\ndef _get_annotated_type(type_: type) -> type | None:\n    \"\"\"If the given type is an `Annotated` type then it is returned, if not `None` is returned.\n\n    This also unwraps the type when applicable, e.g. `Required[Annotated[T, ...]]`\n    \"\"\"\n    if is_required_type(type_):\n        # Unwrap `Required[Annotated[T, ...]]` to `Annotated[T, ...]`\n        type_ = get_args(type_)[0]\n\n    if is_annotated_type(type_):\n        return type_\n\n    return None\n\n\ndef _maybe_transform_key(key: str, type_: type) -> str:\n    \"\"\"Transform the given `data` based on the annotations provided in `type_`.\n\n    Note: this function only looks at `Annotated` types that contain `PropertInfo` metadata.\n    \"\"\"\n    annotated_type = _get_annotated_type(type_)\n    if annotated_type is None:\n        # no `Annotated` definition for this type, no transformation needed\n        return key\n\n    # ignore the first argument as it is the actual type\n    annotations = get_args(annotated_type)[1:]\n    for annotation in annotations:\n        if isinstance(annotation, PropertyInfo) and annotation.alias is not None:\n            return annotation.alias\n\n    return key\n\n\ndef _transform_recursive(\n    data: object,\n    *,\n    annotation: type,\n    inner_type: type | None = None,\n) -> object:\n    \"\"\"Transform the given data against the expected type.\n\n    Args:\n        annotation: The direct type annotation given to the particular piece of data.\n            This may or may not be wrapped in metadata types, e.g. `Required[T]`, `Annotated[T, ...]` etc\n\n        inner_type: If applicable, this is the \"inside\" type. This is useful in certain cases where the outside type\n            is a container type such as `List[T]`. In that case `inner_type` should be set to `T` so that each entry in\n            the list can be transformed using the metadata from the container type.\n\n            Defaults to the same value as the `annotation` argument.\n    \"\"\"\n    if inner_type is None:\n        inner_type = annotation\n\n    stripped_type = strip_annotated_type(inner_type)\n    if is_typeddict(stripped_type) and is_mapping(data):\n        return _transform_typeddict(data, stripped_type)\n\n    if (\n        # List[T]\n        (is_list_type(stripped_type) and is_list(data))\n        # Iterable[T]\n        or (is_iterable_type(stripped_type) and is_iterable(data) and not isinstance(data, str))\n    ):\n        inner_type = extract_type_arg(stripped_type, 0)\n        return [_transform_recursive(d, annotation=annotation, inner_type=inner_type) for d in data]\n\n    if is_union_type(stripped_type):\n        # For union types we run the transformation against all subtypes to ensure that everything is transformed.\n        #\n        # TODO: there may be edge cases where the same normalized field name will transform to two different names\n        # in different subtypes.\n        for subtype in get_args(stripped_type):\n            data = _transform_recursive(data, annotation=annotation, inner_type=subtype)\n        return data\n\n    if isinstance(data, pydantic.BaseModel):\n        return model_dump(data, exclude_unset=True)\n\n    return _transform_value(data, annotation)\n\n\ndef _transform_value(data: object, type_: type) -> object:\n    annotated_type = _get_annotated_type(type_)\n    if annotated_type is None:\n        return data\n\n    # ignore the first argument as it is the actual type\n    annotations = get_args(annotated_type)[1:]\n    for annotation in annotations:\n        if isinstance(annotation, PropertyInfo) and annotation.format is not None:\n            return _format_data(data, annotation.format, annotation.format_template)\n\n    return data\n\n\ndef _format_data(data: object, format_: PropertyFormat, format_template: str | None) -> object:\n    if isinstance(data, (date, datetime)):\n        if format_ == \"iso8601\":\n            return data.isoformat()\n\n        if format_ == \"custom\" and format_template is not None:\n            return data.strftime(format_template)\n\n    return data\n\n\ndef _transform_typeddict(\n    data: Mapping[str, object],\n    expected_type: type,\n) -> Mapping[str, object]:\n    result: dict[str, object] = {}\n    annotations = get_type_hints(expected_type, include_extras=True)\n    for key, value in data.items():\n        type_ = annotations.get(key)\n        if type_ is None:\n            # we do not have a type annotation for this field, leave it as is\n            result[key] = value\n        else:\n            result[_maybe_transform_key(key, type_)] = _transform_recursive(value, annotation=type_)\n    return result\n\n# Path: src/openai/_utils/__init__.py\nfrom ._sync import asyncify as asyncify\nfrom ._proxy import LazyProxy as LazyProxy\nfrom ._utils import (\n    flatten as flatten,\n    is_dict as is_dict,\n    is_list as is_list,\n    is_given as is_given,\n    is_tuple as is_tuple,\n    is_mapping as is_mapping,\n    is_tuple_t as is_tuple_t,\n    parse_date as parse_date,\n    is_iterable as is_iterable,\n    is_sequence as is_sequence,\n    coerce_float as coerce_float,\n    is_mapping_t as is_mapping_t,\n    removeprefix as removeprefix,\n    removesuffix as removesuffix,\n    extract_files as extract_files,\n    is_sequence_t as is_sequence_t,\n    required_args as required_args,\n    coerce_boolean as coerce_boolean,\n    coerce_integer as coerce_integer,\n    file_from_path as file_from_path,\n    parse_datetime as parse_datetime,\n    strip_not_given as strip_not_given,\n    deepcopy_minimal as deepcopy_minimal,\n    get_async_library as get_async_library,\n    maybe_coerce_float as maybe_coerce_float,\n    get_required_header as get_required_header,\n    maybe_coerce_boolean as maybe_coerce_boolean,\n    maybe_coerce_integer as maybe_coerce_integer,\n)\nfrom ._typing import (\n    is_list_type as is_list_type,\n    is_union_type as is_union_type,\n    extract_type_arg as extract_type_arg,\n    is_iterable_type as is_iterable_type,\n    is_required_type as is_required_type,\n    is_annotated_type as is_annotated_type,\n    strip_annotated_type as strip_annotated_type,\n    extract_type_var_from_base as extract_type_var_from_base,\n)\nfrom ._streams import consume_sync_iterator as consume_sync_iterator, consume_async_iterator as consume_async_iterator\nfrom ._transform import (\n    PropertyInfo as PropertyInfo,\n    transform as transform,\n    maybe_transform as maybe_transform,\n)\n\n# Path: src/openai/_exceptions.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nfrom typing import Any, Optional, cast\nfrom typing_extensions import Literal\n\nimport httpx\n\nfrom ._utils import is_dict\n\n__all__ = [\n    \"BadRequestError\",\n    \"AuthenticationError\",\n    \"PermissionDeniedError\",\n    \"NotFoundError\",\n    \"ConflictError\",\n    \"UnprocessableEntityError\",\n    \"RateLimitError\",\n    \"InternalServerError\",\n]\n\n\nclass OpenAIError(Exception):\n    pass\n\n\nclass APIError(OpenAIError):\n    message: str\n    request: httpx.Request\n\n    body: object | None\n    \"\"\"The API response body.\n\n    If the API responded with a valid JSON structure then this property will be the\n    decoded result.\n\n    If it isn't a valid JSON structure then this will be the raw response.\n\n    If there was no response associated with this error then it will be `None`.\n    \"\"\"\n\n    code: Optional[str] = None\n    param: Optional[str] = None\n    type: Optional[str]\n\n    def __init__(self, message: str, request: httpx.Request, *, body: object | None) -> None:\n        super().__init__(message)\n        self.request = request\n        self.message = message\n        self.body = body\n\n        if is_dict(body):\n            self.code = cast(Any, body.get(\"code\"))\n            self.param = cast(Any, body.get(\"param\"))\n            self.type = cast(Any, body.get(\"type\"))\n        else:\n            self.code = None\n            self.param = None\n            self.type = None\n\n\nclass APIResponseValidationError(APIError):\n    response: httpx.Response\n    status_code: int\n\n    def __init__(self, response: httpx.Response, body: object | None, *, message: str | None = None) -> None:\n        super().__init__(message or \"Data returned by API invalid for expected schema.\", response.request, body=body)\n        self.response = response\n        self.status_code = response.status_code\n\n\nclass APIStatusError(APIError):\n    \"\"\"Raised when an API response has a status code of 4xx or 5xx.\"\"\"\n\n    response: httpx.Response\n    status_code: int\n\n    def __init__(self, message: str, *, response: httpx.Response, body: object | None) -> None:\n        super().__init__(message, response.request, body=body)\n        self.response = response\n        self.status_code = response.status_code\n\n\nclass APIConnectionError(APIError):\n    def __init__(self, *, message: str = \"Connection error.\", request: httpx.Request) -> None:\n        super().__init__(message, request, body=None)\n\n\nclass APITimeoutError(APIConnectionError):\n    def __init__(self, request: httpx.Request) -> None:\n        super().__init__(message=\"Request timed out.\", request=request)\n\n\nclass BadRequestError(APIStatusError):\n    status_code: Literal[400] = 400  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass AuthenticationError(APIStatusError):\n    status_code: Literal[401] = 401  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass PermissionDeniedError(APIStatusError):\n    status_code: Literal[403] = 403  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass NotFoundError(APIStatusError):\n    status_code: Literal[404] = 404  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass ConflictError(APIStatusError):\n    status_code: Literal[409] = 409  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass UnprocessableEntityError(APIStatusError):\n    status_code: Literal[422] = 422  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass RateLimitError(APIStatusError):\n    status_code: Literal[429] = 429  # pyright: ignore[reportIncompatibleVariableOverride]\n\n\nclass InternalServerError(APIStatusError):\n    pass\n\n# Path: src/openai/_models.py\nfrom __future__ import annotations\n\nimport inspect\nfrom typing import TYPE_CHECKING, Any, Type, Union, Generic, TypeVar, Callable, cast\nfrom datetime import date, datetime\nfrom typing_extensions import (\n    Unpack,\n    Literal,\n    ClassVar,\n    Protocol,\n    Required,\n    TypedDict,\n    final,\n    override,\n    runtime_checkable,\n)\n\nimport pydantic\nimport pydantic.generics\nfrom pydantic.fields import FieldInfo\n\nfrom ._types import (\n    Body,\n    IncEx,\n    Query,\n    ModelT,\n    Headers,\n    Timeout,\n    NotGiven,\n    AnyMapping,\n    HttpxRequestFiles,\n)\nfrom ._utils import is_list, is_given, is_mapping, parse_date, parse_datetime, strip_not_given\nfrom ._compat import (\n    PYDANTIC_V2,\n    ConfigDict,\n    GenericModel as BaseGenericModel,\n    get_args,\n    is_union,\n    parse_obj,\n    get_origin,\n    is_literal_type,\n    get_model_config,\n    get_model_fields,\n    field_get_default,\n)\nfrom ._constants import RAW_RESPONSE_HEADER\n\n__all__ = [\"BaseModel\", \"GenericModel\"]\n\n_T = TypeVar(\"_T\")\n\n\n@runtime_checkable\nclass _ConfigProtocol(Protocol):\n    allow_population_by_field_name: bool\n\n\nclass BaseModel(pydantic.BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(extra=\"allow\")\n    else:\n\n        @property\n        @override\n        def model_fields_set(self) -> set[str]:\n            # a forwards-compat shim for pydantic v2\n            return self.__fields_set__  # type: ignore\n\n        class Config(pydantic.BaseConfig):  # pyright: ignore[reportDeprecated]\n            extra: Any = pydantic.Extra.allow  # type: ignore\n\n    @override\n    def __str__(self) -> str:\n        # mypy complains about an invalid self arg\n        return f'{self.__repr_name__()}({self.__repr_str__(\", \")})'  # type: ignore[misc]\n\n    # Override the 'construct' method in a way that supports recursive parsing without validation.\n    # Based on https://github.com/samuelcolvin/pydantic/issues/1168#issuecomment-817742836.\n    @classmethod\n    @override\n    def construct(\n        cls: Type[ModelT],\n        _fields_set: set[str] | None = None,\n        **values: object,\n    ) -> ModelT:\n        m = cls.__new__(cls)\n        fields_values: dict[str, object] = {}\n\n        config = get_model_config(cls)\n        populate_by_name = (\n            config.allow_population_by_field_name\n            if isinstance(config, _ConfigProtocol)\n            else config.get(\"populate_by_name\")\n        )\n\n        if _fields_set is None:\n            _fields_set = set()\n\n        model_fields = get_model_fields(cls)\n        for name, field in model_fields.items():\n            key = field.alias\n            if key is None or (key not in values and populate_by_name):\n                key = name\n\n            if key in values:\n                fields_values[name] = _construct_field(value=values[key], field=field, key=key)\n                _fields_set.add(name)\n            else:\n                fields_values[name] = field_get_default(field)\n\n        _extra = {}\n        for key, value in values.items():\n            if key not in model_fields:\n                if PYDANTIC_V2:\n                    _extra[key] = value\n                else:\n                    _fields_set.add(key)\n                    fields_values[key] = value\n\n        object.__setattr__(m, \"__dict__\", fields_values)\n\n        if PYDANTIC_V2:\n            # these properties are copied from Pydantic's `model_construct()` method\n            object.__setattr__(m, \"__pydantic_private__\", None)\n            object.__setattr__(m, \"__pydantic_extra__\", _extra)\n            object.__setattr__(m, \"__pydantic_fields_set__\", _fields_set)\n        else:\n            # init_private_attributes() does not exist in v2\n            m._init_private_attributes()  # type: ignore\n\n            # copied from Pydantic v1's `construct()` method\n            object.__setattr__(m, \"__fields_set__\", _fields_set)\n\n        return m\n\n    if not TYPE_CHECKING:\n        # type checkers incorrectly complain about this assignment\n        # because the type signatures are technically different\n        # although not in practice\n        model_construct = construct\n\n    if not PYDANTIC_V2:\n        # we define aliases for some of the new pydantic v2 methods so\n        # that we can just document these methods without having to specify\n        # a specific pydantic version as some users may not know which\n        # pydantic version they are currently using\n\n        @override\n        def model_dump(\n            self,\n            *,\n            mode: Literal[\"json\", \"python\"] | str = \"python\",\n            include: IncEx = None,\n            exclude: IncEx = None,\n            by_alias: bool = False,\n            exclude_unset: bool = False,\n            exclude_defaults: bool = False,\n            exclude_none: bool = False,\n            round_trip: bool = False,\n            warnings: bool = True,\n        ) -> dict[str, Any]:\n            \"\"\"Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump\n\n            Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.\n\n            Args:\n                mode: The mode in which `to_python` should run.\n                    If mode is 'json', the dictionary will only contain JSON serializable types.\n                    If mode is 'python', the dictionary may contain any Python objects.\n                include: A list of fields to include in the output.\n                exclude: A list of fields to exclude from the output.\n                by_alias: Whether to use the field's alias in the dictionary key if defined.\n                exclude_unset: Whether to exclude fields that are unset or None from the output.\n                exclude_defaults: Whether to exclude fields that are set to their default value from the output.\n                exclude_none: Whether to exclude fields that have a value of `None` from the output.\n                round_trip: Whether to enable serialization and deserialization round-trip support.\n                warnings: Whether to log warnings when invalid fields are encountered.\n\n            Returns:\n                A dictionary representation of the model.\n            \"\"\"\n            if mode != \"python\":\n                raise ValueError(\"mode is only supported in Pydantic v2\")\n            if round_trip != False:\n                raise ValueError(\"round_trip is only supported in Pydantic v2\")\n            if warnings != True:\n                raise ValueError(\"warnings is only supported in Pydantic v2\")\n            return super().dict(  # pyright: ignore[reportDeprecated]\n                include=include,\n                exclude=exclude,\n                by_alias=by_alias,\n                exclude_unset=exclude_unset,\n                exclude_defaults=exclude_defaults,\n                exclude_none=exclude_none,\n            )\n\n        @override\n        def model_dump_json(\n            self,\n            *,\n            indent: int | None = None,\n            include: IncEx = None,\n            exclude: IncEx = None,\n            by_alias: bool = False,\n            exclude_unset: bool = False,\n            exclude_defaults: bool = False,\n            exclude_none: bool = False,\n            round_trip: bool = False,\n            warnings: bool = True,\n        ) -> str:\n            \"\"\"Usage docs: https://docs.pydantic.dev/2.4/concepts/serialization/#modelmodel_dump_json\n\n            Generates a JSON representation of the model using Pydantic's `to_json` method.\n\n            Args:\n                indent: Indentation to use in the JSON output. If None is passed, the output will be compact.\n                include: Field(s) to include in the JSON output. Can take either a string or set of strings.\n                exclude: Field(s) to exclude from the JSON output. Can take either a string or set of strings.\n                by_alias: Whether to serialize using field aliases.\n                exclude_unset: Whether to exclude fields that have not been explicitly set.\n                exclude_defaults: Whether to exclude fields that have the default value.\n                exclude_none: Whether to exclude fields that have a value of `None`.\n                round_trip: Whether to use serialization/deserialization between JSON and class instance.\n                warnings: Whether to show any warnings that occurred during serialization.\n\n            Returns:\n                A JSON string representation of the model.\n            \"\"\"\n            if round_trip != False:\n                raise ValueError(\"round_trip is only supported in Pydantic v2\")\n            if warnings != True:\n                raise ValueError(\"warnings is only supported in Pydantic v2\")\n            return super().json(  # type: ignore[reportDeprecated]\n                indent=indent,\n                include=include,\n                exclude=exclude,\n                by_alias=by_alias,\n                exclude_unset=exclude_unset,\n                exclude_defaults=exclude_defaults,\n                exclude_none=exclude_none,\n            )\n\n\n\ndef _construct_field(value: object, field: FieldInfo, key: str) -> object:\n    if value is None:\n        return field_get_default(field)\n\n    if PYDANTIC_V2:\n        type_ = field.annotation\n    else:\n        type_ = cast(type, field.outer_type_)  # type: ignore\n\n    if type_ is None:\n        raise RuntimeError(f\"Unexpected field type is None for {key}\")\n\n    return construct_type(value=value, type_=type_)\n\n\ndef is_basemodel(type_: type) -> bool:\n    \"\"\"Returns whether or not the given type is either a `BaseModel` or a union of `BaseModel`\"\"\"\n    origin = get_origin(type_) or type_\n    if is_union(type_):\n        for variant in get_args(type_):\n            if is_basemodel(variant):\n                return True\n\n        return False\n\n    return issubclass(origin, BaseModel) or issubclass(origin, GenericModel)\n\n\ndef construct_type(*, value: object, type_: type) -> object:\n    \"\"\"Loose coercion to the expected type with construction of nested values.\n\n    If the given value does not match the expected type then it is returned as-is.\n    \"\"\"\n\n    # we need to use the origin class for any types that are subscripted generics\n    # e.g. Dict[str, object]\n    origin = get_origin(type_) or type_\n    args = get_args(type_)\n\n    if is_union(origin):\n        try:\n            return validate_type(type_=cast(\"type[object]\", type_), value=value)\n        except Exception:\n            pass\n\n        # if the data is not valid, use the first variant that doesn't fail while deserializing\n        for variant in args:\n            try:\n                return construct_type(value=value, type_=variant)\n            except Exception:\n                continue\n\n        raise RuntimeError(f\"Could not convert data into a valid instance of {type_}\")\n\n    if origin == dict:\n        if not is_mapping(value):\n            return value\n\n        _, items_type = get_args(type_)  # Dict[_, items_type]\n        return {key: construct_type(value=item, type_=items_type) for key, item in value.items()}\n\n    if not is_literal_type(type_) and (issubclass(origin, BaseModel) or issubclass(origin, GenericModel)):\n        if is_list(value):\n            return [cast(Any, type_).construct(**entry) if is_mapping(entry) else entry for entry in value]\n\n        if is_mapping(value):\n            if issubclass(type_, BaseModel):\n                return type_.construct(**value)  # type: ignore[arg-type]\n\n            return cast(Any, type_).construct(**value)\n\n    if origin == list:\n        if not is_list(value):\n            return value\n\n        inner_type = args[0]  # List[inner_type]\n        return [construct_type(value=entry, type_=inner_type) for entry in value]\n\n    if origin == float:\n        if isinstance(value, int):\n            coerced = float(value)\n            if coerced != value:\n                return value\n            return coerced\n\n        return value\n\n    if type_ == datetime:\n        try:\n            return parse_datetime(value)  # type: ignore\n        except Exception:\n            return value\n\n    if type_ == date:\n        try:\n            return parse_date(value)  # type: ignore\n        except Exception:\n            return value\n\n    return value\n\n\ndef validate_type(*, type_: type[_T], value: object) -> _T:\n    \"\"\"Strict validation that the given value matches the expected type\"\"\"\n    if inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n        return cast(_T, parse_obj(type_, value))\n\n    return cast(_T, _validate_non_model_type(type_=type_, value=value))\n\n\n# our use of subclasssing here causes weirdness for type checkers,\n# so we just pretend that we don't subclass\nif TYPE_CHECKING:\n    GenericModel = BaseModel\nelse:\n\n    class GenericModel(BaseGenericModel, BaseModel):\n        pass\n\n\nif PYDANTIC_V2:\n    from pydantic import TypeAdapter\n\n    def _validate_non_model_type(*, type_: type[_T], value: object) -> _T:\n        return TypeAdapter(type_).validate_python(value)\n\nelif not TYPE_CHECKING:  # TODO: condition is weird\n\n    class RootModel(GenericModel, Generic[_T]):\n        \"\"\"Used as a placeholder to easily convert runtime types to a Pydantic format\n        to provide validation.\n\n        For example:\n        ```py\n        validated = RootModel[int](__root__=\"5\").__root__\n        # validated: 5\n        ```\n        \"\"\"\n\n        __root__: _T\n\n    def _validate_non_model_type(*, type_: type[_T], value: object) -> _T:\n        model = _create_pydantic_model(type_).validate(value)\n        return cast(_T, model.__root__)\n\n    def _create_pydantic_model(type_: _T) -> Type[RootModel[_T]]:\n        return RootModel[type_]  # type: ignore\n\n\nclass FinalRequestOptionsInput(TypedDict, total=False):\n    method: Required[str]\n    url: Required[str]\n    params: Query\n    headers: Headers\n    max_retries: int\n    timeout: float | Timeout | None\n    files: HttpxRequestFiles | None\n    idempotency_key: str\n    json_data: Body\n    extra_json: AnyMapping\n\n\n@final\nclass FinalRequestOptions(pydantic.BaseModel):\n    method: str\n    url: str\n    params: Query = {}\n    headers: Union[Headers, NotGiven] = NotGiven()\n    max_retries: Union[int, NotGiven] = NotGiven()\n    timeout: Union[float, Timeout, None, NotGiven] = NotGiven()\n    files: Union[HttpxRequestFiles, None] = None\n    idempotency_key: Union[str, None] = None\n    post_parser: Union[Callable[[Any], Any], NotGiven] = NotGiven()\n\n    # It should be noted that we cannot use `json` here as that would override\n    # a BaseModel method in an incompatible fashion.\n    json_data: Union[Body, None] = None\n    extra_json: Union[AnyMapping, None] = None\n\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)\n    else:\n\n        class Config(pydantic.BaseConfig):  # pyright: ignore[reportDeprecated]\n            arbitrary_types_allowed: bool = True\n\n    def get_max_retries(self, max_retries: int) -> int:\n        if isinstance(self.max_retries, NotGiven):\n            return max_retries\n        return self.max_retries\n\n    def _strip_raw_response_header(self) -> None:\n        if not is_given(self.headers):\n            return\n\n        if self.headers.get(RAW_RESPONSE_HEADER):\n            self.headers = {**self.headers}\n            self.headers.pop(RAW_RESPONSE_HEADER)\n\n    # override the `construct` method so that we can run custom transformations.\n    # this is necessary as we don't want to do any actual runtime type checking\n    # (which means we can't use validators) but we do want to ensure that `NotGiven`\n    # values are not present\n    #\n    # type ignore required because we're adding explicit types to `**values`\n    @classmethod\n    def construct(  # type: ignore\n        cls,\n        _fields_set: set[str] | None = None,\n        **values: Unpack[FinalRequestOptionsInput],\n    ) -> FinalRequestOptions:\n        kwargs: dict[str, Any] = {\n            # we unconditionally call `strip_not_given` on any value\n            # as it will just ignore any non-mapping types\n            key: strip_not_given(value)\n            for key, value in values.items()\n        }\n        if PYDANTIC_V2:\n            return super().model_construct(_fields_set, **kwargs)\n        return cast(FinalRequestOptions, super().construct(_fields_set, **kwargs))  # pyright: ignore[reportDeprecated]\n\n    if not TYPE_CHECKING:\n        # type checkers incorrectly complain about this assignment\n        model_construct = construct\n\n# Path: src/openai/_qs.py\nfrom __future__ import annotations\n\nfrom typing import Any, List, Tuple, Union, Mapping, TypeVar\nfrom urllib.parse import parse_qs, urlencode\nfrom typing_extensions import Literal, get_args\n\nfrom ._types import NOT_GIVEN, NotGiven, NotGivenOr\nfrom ._utils import flatten\n\n_T = TypeVar(\"_T\")\n\n\nArrayFormat = Literal[\"comma\", \"repeat\", \"indices\", \"brackets\"]\nNestedFormat = Literal[\"dots\", \"brackets\"]\n\nPrimitiveData = Union[str, int, float, bool, None]\n# this should be Data = Union[PrimitiveData, \"List[Data]\", \"Tuple[Data]\", \"Mapping[str, Data]\"]\n# https://github.com/microsoft/pyright/issues/3555\nData = Union[PrimitiveData, List[Any], Tuple[Any], \"Mapping[str, Any]\"]\nParams = Mapping[str, Data]\n\n\nclass Querystring:\n    array_format: ArrayFormat\n    nested_format: NestedFormat\n\n    def __init__(\n        self,\n        *,\n        array_format: ArrayFormat = \"repeat\",\n        nested_format: NestedFormat = \"brackets\",\n    ) -> None:\n        self.array_format = array_format\n        self.nested_format = nested_format\n\n    def parse(self, query: str) -> Mapping[str, object]:\n        # Note: custom format syntax is not supported yet\n        return parse_qs(query)\n\n    def stringify(\n        self,\n        params: Params,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n        nested_format: NotGivenOr[NestedFormat] = NOT_GIVEN,\n    ) -> str:\n        return urlencode(\n            self.stringify_items(\n                params,\n                array_format=array_format,\n                nested_format=nested_format,\n            )\n        )\n\n    def stringify_items(\n        self,\n        params: Params,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n        nested_format: NotGivenOr[NestedFormat] = NOT_GIVEN,\n    ) -> list[tuple[str, str]]:\n        opts = Options(\n            qs=self,\n            array_format=array_format,\n            nested_format=nested_format,\n        )\n        return flatten([self._stringify_item(key, value, opts) for key, value in params.items()])\n\n    def _stringify_item(\n        self,\n        key: str,\n        value: Data,\n        opts: Options,\n    ) -> list[tuple[str, str]]:\n        if isinstance(value, Mapping):\n            items: list[tuple[str, str]] = []\n            nested_format = opts.nested_format\n            for subkey, subvalue in value.items():\n                items.extend(\n                    self._stringify_item(\n                        # TODO: error if unknown format\n                        f\"{key}.{subkey}\" if nested_format == \"dots\" else f\"{key}[{subkey}]\",\n                        subvalue,\n                        opts,\n                    )\n                )\n            return items\n\n        if isinstance(value, (list, tuple)):\n            array_format = opts.array_format\n            if array_format == \"comma\":\n                return [\n                    (\n                        key,\n                        \",\".join(self._primitive_value_to_str(item) for item in value if item is not None),\n                    ),\n                ]\n            elif array_format == \"repeat\":\n                items = []\n                for item in value:\n                    items.extend(self._stringify_item(key, item, opts))\n                return items\n            elif array_format == \"indices\":\n                raise NotImplementedError(\"The array indices format is not supported yet\")\n            elif array_format == \"brackets\":\n                items = []\n                key = key + \"[]\"\n                for item in value:\n                    items.extend(self._stringify_item(key, item, opts))\n                return items\n            else:\n                raise NotImplementedError(\n                    f\"Unknown array_format value: {array_format}, choose from {', '.join(get_args(ArrayFormat))}\"\n                )\n\n        serialised = self._primitive_value_to_str(value)\n        if not serialised:\n            return []\n        return [(key, serialised)]\n\n    def _primitive_value_to_str(self, value: PrimitiveData) -> str:\n        # copied from httpx\n        if value is True:\n            return \"true\"\n        elif value is False:\n            return \"false\"\n        elif value is None:\n            return \"\"\n        return str(value)\n\n\n_qs = Querystring()\nparse = _qs.parse\nstringify = _qs.stringify\nstringify_items = _qs.stringify_items\n\n\nclass Options:\n    array_format: ArrayFormat\n    nested_format: NestedFormat\n\n    def __init__(\n        self,\n        qs: Querystring = _qs,\n        *,\n        array_format: NotGivenOr[ArrayFormat] = NOT_GIVEN,\n        nested_format: NotGivenOr[NestedFormat] = NOT_GIVEN,\n    ) -> None:\n        self.array_format = qs.array_format if isinstance(array_format, NotGiven) else array_format\n        self.nested_format = qs.nested_format if isinstance(nested_format, NotGiven) else nested_format\n\n# Path: src/openai/_version.py\n# File generated from our OpenAPI spec by Stainless.\n\n__title__ = \"openai\"\n__version__ = \"1.13.3\"  # x-release-please-version\n\n# Path: src/openai/_resource.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nimport time\nfrom typing import TYPE_CHECKING\n\nimport anyio\n\nif TYPE_CHECKING:\n    from ._client import OpenAI, AsyncOpenAI\n\n\nclass SyncAPIResource:\n    _client: OpenAI\n\n    def __init__(self, client: OpenAI) -> None:\n        self._client = client\n        self._get = client.get\n        self._post = client.post\n        self._patch = client.patch\n        self._put = client.put\n        self._delete = client.delete\n        self._get_api_list = client.get_api_list\n\n    def _sleep(self, seconds: float) -> None:\n        time.sleep(seconds)\n\n\nclass AsyncAPIResource:\n    _client: AsyncOpenAI\n\n    def __init__(self, client: AsyncOpenAI) -> None:\n        self._client = client\n        self._get = client.get\n        self._post = client.post\n        self._patch = client.patch\n        self._put = client.put\n        self._delete = client.delete\n        self._get_api_list = client.get_api_list\n\n    async def _sleep(self, seconds: float) -> None:\n        await anyio.sleep(seconds)\n\n# Path: src/openai/_response.py\nfrom __future__ import annotations\n\nimport os\nimport inspect\nimport logging\nimport datetime\nimport functools\nfrom types import TracebackType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Union,\n    Generic,\n    TypeVar,\n    Callable,\n    Iterator,\n    AsyncIterator,\n    cast,\n    overload,\n)\nfrom typing_extensions import Awaitable, ParamSpec, override, get_origin\n\nimport anyio\nimport httpx\nimport pydantic\n\nfrom ._types import NoneType\nfrom ._utils import is_given, extract_type_var_from_base\nfrom ._models import BaseModel, is_basemodel\nfrom ._constants import RAW_RESPONSE_HEADER, OVERRIDE_CAST_TO_HEADER\nfrom ._streaming import Stream, AsyncStream, is_stream_class_type, extract_stream_chunk_type\nfrom ._exceptions import OpenAIError, APIResponseValidationError\n\nif TYPE_CHECKING:\n    from ._models import FinalRequestOptions\n    from ._base_client import BaseClient\n\n\nP = ParamSpec(\"P\")\nR = TypeVar(\"R\")\n_T = TypeVar(\"_T\")\n_APIResponseT = TypeVar(\"_APIResponseT\", bound=\"APIResponse[Any]\")\n_AsyncAPIResponseT = TypeVar(\"_AsyncAPIResponseT\", bound=\"AsyncAPIResponse[Any]\")\n\nlog: logging.Logger = logging.getLogger(__name__)\n\n\nclass BaseAPIResponse(Generic[R]):\n    _cast_to: type[R]\n    _client: BaseClient[Any, Any]\n    _parsed_by_type: dict[type[Any], Any]\n    _is_sse_stream: bool\n    _stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None\n    _options: FinalRequestOptions\n\n    http_response: httpx.Response\n\n    def __init__(\n        self,\n        *,\n        raw: httpx.Response,\n        cast_to: type[R],\n        client: BaseClient[Any, Any],\n        stream: bool,\n        stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None,\n        options: FinalRequestOptions,\n    ) -> None:\n        self._cast_to = cast_to\n        self._client = client\n        self._parsed_by_type = {}\n        self._is_sse_stream = stream\n        self._stream_cls = stream_cls\n        self._options = options\n        self.http_response = raw\n\n    @property\n    def headers(self) -> httpx.Headers:\n        return self.http_response.headers\n\n    @property\n    def http_request(self) -> httpx.Request:\n        \"\"\"Returns the httpx Request instance associated with the current response.\"\"\"\n        return self.http_response.request\n\n    @property\n    def status_code(self) -> int:\n        return self.http_response.status_code\n\n    @property\n    def url(self) -> httpx.URL:\n        \"\"\"Returns the URL for which the request was made.\"\"\"\n        return self.http_response.url\n\n    @property\n    def method(self) -> str:\n        return self.http_request.method\n\n    @property\n    def http_version(self) -> str:\n        return self.http_response.http_version\n\n    @property\n    def elapsed(self) -> datetime.timedelta:\n        \"\"\"The time taken for the complete request/response cycle to complete.\"\"\"\n        return self.http_response.elapsed\n\n    @property\n    def is_closed(self) -> bool:\n        \"\"\"Whether or not the response body has been closed.\n\n        If this is False then there is response data that has not been read yet.\n        You must either fully consume the response body or call `.close()`\n        before discarding the response to prevent resource leaks.\n        \"\"\"\n        return self.http_response.is_closed\n\n    @override\n    def __repr__(self) -> str:\n        return (\n            f\"<{self.__class__.__name__} [{self.status_code} {self.http_response.reason_phrase}] type={self._cast_to}>\"\n        )\n\n    def _parse(self, *, to: type[_T] | None = None) -> R | _T:\n        if self._is_sse_stream:\n            if to:\n                if not is_stream_class_type(to):\n                    raise TypeError(f\"Expected custom parse type to be a subclass of {Stream} or {AsyncStream}\")\n\n                return cast(\n                    _T,\n                    to(\n                        cast_to=extract_stream_chunk_type(\n                            to,\n                            failure_message=\"Expected custom stream type to be passed with a type argument, e.g. Stream[ChunkType]\",\n                        ),\n                        response=self.http_response,\n                        client=cast(Any, self._client),\n                    ),\n                )\n\n            if self._stream_cls:\n                return cast(\n                    R,\n                    self._stream_cls(\n                        cast_to=extract_stream_chunk_type(self._stream_cls),\n                        response=self.http_response,\n                        client=cast(Any, self._client),\n                    ),\n                )\n\n            stream_cls = cast(\"type[Stream[Any]] | type[AsyncStream[Any]] | None\", self._client._default_stream_cls)\n            if stream_cls is None:\n                raise MissingStreamClassError()\n\n            return cast(\n                R,\n                stream_cls(\n                    cast_to=self._cast_to,\n                    response=self.http_response,\n                    client=cast(Any, self._client),\n                ),\n            )\n\n        cast_to = to if to is not None else self._cast_to\n        if cast_to is NoneType:\n            return cast(R, None)\n\n        response = self.http_response\n        if cast_to == str:\n            return cast(R, response.text)\n\n        if cast_to == bytes:\n            return cast(R, response.content)\n\n        origin = get_origin(cast_to) or cast_to\n\n        # handle the legacy binary response case\n        if inspect.isclass(cast_to) and cast_to.__name__ == \"HttpxBinaryResponseContent\":\n            return cast(R, cast_to(response))  # type: ignore\n\n        if origin == APIResponse:\n            raise RuntimeError(\"Unexpected state - cast_to is `APIResponse`\")\n\n        if inspect.isclass(origin) and issubclass(origin, httpx.Response):\n            # Because of the invariance of our ResponseT TypeVar, users can subclass httpx.Response\n            # and pass that class to our request functions. We cannot change the variance to be either\n...\n", "needle_token_start": 10316, "needle_token_end": 10446, "code_context_ntokens": 15992, "output": ["```python\ndef _construct_field(value: object, field: FieldInfo, key: str) -> object:\n    if value is None:\n        return field_get_default(field)\n\n    if PYDANTIC_V2:\n        type_ = field.annotation\n    else:\n        type_ = cast(type, field.outer_type_)  # type: ignore\n\n    if type_ is None:\n        raise RuntimeError(f\"Unexpected field type is None for {key}\")\n\n    return construct_type(value=value, type_=type_)\n```"]}
{"repo": "openai/openai-python", "name": "_basic_config", "language": "python", "path": "src/openai/_utils/_logs.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To configure the default logging settings for the application, ensuring that log messages are formatted and timestamped consistently.\n2. **Input**: None, as it does not require any parameters to be passed.\n3. **Output**: None, as it does not return any values but configures the logging system.\n4. **Procedure**: The function sets up the logging format to include the timestamp, logger name, line number, log level, and the log message itself. It also specifies the date and time format to be used in the log messages.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/openai/_base_client.py\nfrom __future__ import annotations\n\nimport json\nimport time\nimport uuid\nimport email\nimport asyncio\nimport inspect\nimport logging\nimport platform\nimport warnings\nimport email.utils\nfrom types import TracebackType\nfrom random import random\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Type,\n    Union,\n    Generic,\n    Mapping,\n    TypeVar,\n    Iterable,\n    Iterator,\n    Optional,\n    Generator,\n    AsyncIterator,\n    cast,\n    overload,\n)\nfrom functools import lru_cache\nfrom typing_extensions import Literal, override, get_origin\n\nimport anyio\nimport httpx\nimport distro\nimport pydantic\nfrom httpx import URL, Limits\nfrom pydantic import PrivateAttr\n\nfrom . import _exceptions\nfrom ._qs import Querystring\nfrom ._files import to_httpx_files, async_to_httpx_files\nfrom ._types import (\n    NOT_GIVEN,\n    Body,\n    Omit,\n    Query,\n    Headers,\n    Timeout,\n    NotGiven,\n    ResponseT,\n    Transport,\n    AnyMapping,\n    PostParser,\n    ProxiesTypes,\n    RequestFiles,\n    HttpxSendArgs,\n    AsyncTransport,\n    RequestOptions,\n    ModelBuilderProtocol,\n)\nfrom ._utils import is_dict, is_list, is_given, is_mapping\nfrom ._compat import model_copy, model_dump\nfrom ._models import GenericModel, FinalRequestOptions, validate_type, construct_type\nfrom ._response import (\n    APIResponse,\n    BaseAPIResponse,\n    AsyncAPIResponse,\n    extract_response_type,\n)\nfrom ._constants import (\n    DEFAULT_LIMITS,\n    DEFAULT_TIMEOUT,\n    MAX_RETRY_DELAY,\n    DEFAULT_MAX_RETRIES,\n    INITIAL_RETRY_DELAY,\n    RAW_RESPONSE_HEADER,\n    OVERRIDE_CAST_TO_HEADER,\n)\nfrom ._streaming import Stream, AsyncStream\nfrom ._exceptions import (\n    APIStatusError,\n    APITimeoutError,\n    APIConnectionError,\n    APIResponseValidationError,\n)\nfrom ._legacy_response import LegacyAPIResponse\n\nlog: logging.Logger = logging.getLogger(__name__)\n\n# TODO: make base page type vars covariant\nSyncPageT = TypeVar(\"SyncPageT\", bound=\"BaseSyncPage[Any]\")\nAsyncPageT = TypeVar(\"AsyncPageT\", bound=\"BaseAsyncPage[Any]\")\n\n\n_T = TypeVar(\"_T\")\n_T_co = TypeVar(\"_T_co\", covariant=True)\n\n_StreamT = TypeVar(\"_StreamT\", bound=Stream[Any])\n_AsyncStreamT = TypeVar(\"_AsyncStreamT\", bound=AsyncStream[Any])\n\nif TYPE_CHECKING:\n    from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT\nelse:\n    try:\n        from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT\n    except ImportError:\n        # taken from https://github.com/encode/httpx/blob/3ba5fe0d7ac70222590e759c31442b1cab263791/httpx/_config.py#L366\n        HTTPX_DEFAULT_TIMEOUT = Timeout(5.0)\n\n\nclass PageInfo:\n    \"\"\"Stores the necessary information to build the request to retrieve the next page.\n\n    Either `url` or `params` must be set.\n    \"\"\"\n\n    url: URL | NotGiven\n    params: Query | NotGiven\n\n    @overload\n    def __init__(\n        self,\n        *,\n        url: URL,\n    ) -> None:\n        ...\n\n    @overload\n    def __init__(\n        self,\n        *,\n        params: Query,\n    ) -> None:\n        ...\n\n    def __init__(\n        self,\n        *,\n        url: URL | NotGiven = NOT_GIVEN,\n        params: Query | NotGiven = NOT_GIVEN,\n    ) -> None:\n        self.url = url\n        self.params = params\n\n\nclass BasePage(GenericModel, Generic[_T]):\n    \"\"\"\n    Defines the core interface for pagination.\n\n    Type Args:\n        ModelT: The pydantic model that represents an item in the response.\n\n    Methods:\n        has_next_page(): Check if there is another page available\n        next_page_info(): Get the necessary information to make a request for the next page\n    \"\"\"\n\n    _options: FinalRequestOptions = PrivateAttr()\n    _model: Type[_T] = PrivateAttr()\n\n    def has_next_page(self) -> bool:\n        items = self._get_page_items()\n        if not items:\n            return False\n        return self.next_page_info() is not None\n\n    def next_page_info(self) -> Optional[PageInfo]:\n        ...\n\n    def _get_page_items(self) -> Iterable[_T]:  # type: ignore[empty-body]\n        ...\n\n    def _params_from_url(self, url: URL) -> httpx.QueryParams:\n        # TODO: do we have to preprocess params here?\n        return httpx.QueryParams(cast(Any, self._options.params)).merge(url.params)\n\n    def _info_to_options(self, info: PageInfo) -> FinalRequestOptions:\n        options = model_copy(self._options)\n        options._strip_raw_response_header()\n\n        if not isinstance(info.params, NotGiven):\n            options.params = {**options.params, **info.params}\n            return options\n\n        if not isinstance(info.url, NotGiven):\n            params = self._params_from_url(info.url)\n            url = info.url.copy_with(params=params)\n            options.params = dict(url.params)\n            options.url = str(url)\n            return options\n\n        raise ValueError(\"Unexpected PageInfo state\")\n\n\nclass BaseSyncPage(BasePage[_T], Generic[_T]):\n    _client: SyncAPIClient = pydantic.PrivateAttr()\n\n    def _set_private_attributes(\n        self,\n        client: SyncAPIClient,\n        model: Type[_T],\n        options: FinalRequestOptions,\n    ) -> None:\n        self._model = model\n        self._client = client\n        self._options = options\n\n    # Pydantic uses a custom `__iter__` method to support casting BaseModels\n    # to dictionaries. e.g. dict(model).\n    # As we want to support `for item in page`, this is inherently incompatible\n    # with the default pydantic behaviour. It is not possible to support both\n    # use cases at once. Fortunately, this is not a big deal as all other pydantic\n    # methods should continue to work as expected as there is an alternative method\n    # to cast a model to a dictionary, model.dict(), which is used internally\n    # by pydantic.\n    def __iter__(self) -> Iterator[_T]:  # type: ignore\n        for page in self.iter_pages():\n            for item in page._get_page_items():\n                yield item\n\n    def iter_pages(self: SyncPageT) -> Iterator[SyncPageT]:\n        page = self\n        while True:\n            yield page\n            if page.has_next_page():\n                page = page.get_next_page()\n            else:\n                return\n\n    def get_next_page(self: SyncPageT) -> SyncPageT:\n        info = self.next_page_info()\n        if not info:\n            raise RuntimeError(\n                \"No next page expected; please check `.has_next_page()` before calling `.get_next_page()`.\"\n            )\n\n        options = self._info_to_options(info)\n        return self._client._request_api_list(self._model, page=self.__class__, options=options)\n\n\nclass AsyncPaginator(Generic[_T, AsyncPageT]):\n    def __init__(\n        self,\n        client: AsyncAPIClient,\n        options: FinalRequestOptions,\n        page_cls: Type[AsyncPageT],\n        model: Type[_T],\n    ) -> None:\n        self._model = model\n        self._client = client\n        self._options = options\n        self._page_cls = page_cls\n\n    def __await__(self) -> Generator[Any, None, AsyncPageT]:\n        return self._get_page().__await__()\n\n    async def _get_page(self) -> AsyncPageT:\n        def _parser(resp: AsyncPageT) -> AsyncPageT:\n            resp._set_private_attributes(\n                model=self._model,\n                options=self._options,\n                client=self._client,\n            )\n            return resp\n\n        self._options.post_parser = _parser\n\n        return await self._client.request(self._page_cls, self._options)\n\n    async def __aiter__(self) -> AsyncIterator[_T]:\n        # https://github.com/microsoft/pyright/issues/3464\n        page = cast(\n            AsyncPageT,\n            await self,  # type: ignore\n        )\n        async for item in page:\n            yield item\n\n\nclass BaseAsyncPage(BasePage[_T], Generic[_T]):\n    _client: AsyncAPIClient = pydantic.PrivateAttr()\n\n    def _set_private_attributes(\n        self,\n        model: Type[_T],\n        client: AsyncAPIClient,\n        options: FinalRequestOptions,\n    ) -> None:\n        self._model = model\n        self._client = client\n        self._options = options\n\n    async def __aiter__(self) -> AsyncIterator[_T]:\n        async for page in self.iter_pages():\n            for item in page._get_page_items():\n                yield item\n\n    async def iter_pages(self: AsyncPageT) -> AsyncIterator[AsyncPageT]:\n        page = self\n        while True:\n            yield page\n            if page.has_next_page():\n                page = await page.get_next_page()\n            else:\n                return\n\n    async def get_next_page(self: AsyncPageT) -> AsyncPageT:\n        info = self.next_page_info()\n        if not info:\n            raise RuntimeError(\n                \"No next page expected; please check `.has_next_page()` before calling `.get_next_page()`.\"\n            )\n\n        options = self._info_to_options(info)\n        return await self._client._request_api_list(self._model, page=self.__class__, options=options)\n\n\n_HttpxClientT = TypeVar(\"_HttpxClientT\", bound=Union[httpx.Client, httpx.AsyncClient])\n_DefaultStreamT = TypeVar(\"_DefaultStreamT\", bound=Union[Stream[Any], AsyncStream[Any]])\n\n\nclass BaseClient(Generic[_HttpxClientT, _DefaultStreamT]):\n    _client: _HttpxClientT\n    _version: str\n    _base_url: URL\n    max_retries: int\n    timeout: Union[float, Timeout, None]\n    _limits: httpx.Limits\n    _proxies: ProxiesTypes | None\n    _transport: Transport | AsyncTransport | None\n    _strict_response_validation: bool\n    _idempotency_header: str | None\n    _default_stream_cls: type[_DefaultStreamT] | None = None\n\n    def __init__(\n        self,\n        *,\n        version: str,\n        base_url: str | URL,\n        _strict_response_validation: bool,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        timeout: float | Timeout | None = DEFAULT_TIMEOUT,\n        limits: httpx.Limits,\n        transport: Transport | AsyncTransport | None,\n        proxies: ProxiesTypes | None,\n        custom_headers: Mapping[str, str] | None = None,\n        custom_query: Mapping[str, object] | None = None,\n    ) -> None:\n        self._version = version\n        self._base_url = self._enforce_trailing_slash(URL(base_url))\n        self.max_retries = max_retries\n        self.timeout = timeout\n        self._limits = limits\n        self._proxies = proxies\n        self._transport = transport\n        self._custom_headers = custom_headers or {}\n        self._custom_query = custom_query or {}\n        self._strict_response_validation = _strict_response_validation\n        self._idempotency_header = None\n\n    def _enforce_trailing_slash(self, url: URL) -> URL:\n        if url.raw_path.endswith(b\"/\"):\n            return url\n        return url.copy_with(raw_path=url.raw_path + b\"/\")\n\n    def _make_status_error_from_response(\n        self,\n        response: httpx.Response,\n    ) -> APIStatusError:\n        if response.is_closed and not response.is_stream_consumed:\n            # We can't read the response body as it has been closed\n            # before it was read. This can happen if an event hook\n            # raises a status error.\n            body = None\n            err_msg = f\"Error code: {response.status_code}\"\n        else:\n            err_text = response.text.strip()\n            body = err_text\n\n            try:\n                body = json.loads(err_text)\n                err_msg = f\"Error code: {response.status_code} - {body}\"\n            except Exception:\n                err_msg = err_text or f\"Error code: {response.status_code}\"\n\n        return self._make_status_error(err_msg, body=body, response=response)\n\n    def _make_status_error(\n        self,\n        err_msg: str,\n        *,\n        body: object,\n        response: httpx.Response,\n    ) -> _exceptions.APIStatusError:\n        raise NotImplementedError()\n\n    def _remaining_retries(\n        self,\n        remaining_retries: Optional[int],\n        options: FinalRequestOptions,\n    ) -> int:\n        return remaining_retries if remaining_retries is not None else options.get_max_retries(self.max_retries)\n\n    def _build_headers(self, options: FinalRequestOptions) -> httpx.Headers:\n        custom_headers = options.headers or {}\n        headers_dict = _merge_mappings(self.default_headers, custom_headers)\n        self._validate_headers(headers_dict, custom_headers)\n\n        # headers are case-insensitive while dictionaries are not.\n        headers = httpx.Headers(headers_dict)\n\n        idempotency_header = self._idempotency_header\n        if idempotency_header and options.method.lower() != \"get\" and idempotency_header not in headers:\n            headers[idempotency_header] = options.idempotency_key or self._idempotency_key()\n\n        return headers\n\n    def _prepare_url(self, url: str) -> URL:\n        \"\"\"\n        Merge a URL argument together with any 'base_url' on the client,\n        to create the URL used for the outgoing request.\n        \"\"\"\n        # Copied from httpx's `_merge_url` method.\n        merge_url = URL(url)\n        if merge_url.is_relative_url:\n            merge_raw_path = self.base_url.raw_path + merge_url.raw_path.lstrip(b\"/\")\n            return self.base_url.copy_with(raw_path=merge_raw_path)\n\n        return merge_url\n\n    def _build_request(\n        self,\n        options: FinalRequestOptions,\n    ) -> httpx.Request:\n        if log.isEnabledFor(logging.DEBUG):\n            log.debug(\"Request options: %s\", model_dump(options, exclude_unset=True))\n\n        kwargs: dict[str, Any] = {}\n\n        json_data = options.json_data\n        if options.extra_json is not None:\n            if json_data is None:\n                json_data = cast(Body, options.extra_json)\n            elif is_mapping(json_data):\n                json_data = _merge_mappings(json_data, options.extra_json)\n            else:\n                raise RuntimeError(f\"Unexpected JSON data type, {type(json_data)}, cannot merge with `extra_body`\")\n\n        headers = self._build_headers(options)\n        params = _merge_mappings(self._custom_query, options.params)\n        content_type = headers.get(\"Content-Type\")\n\n        # If the given Content-Type header is multipart/form-data then it\n        # has to be removed so that httpx can generate the header with\n        # additional information for us as it has to be in this form\n        # for the server to be able to correctly parse the request:\n        # multipart/form-data; boundary=---abc--\n        if content_type is not None and content_type.startswith(\"multipart/form-data\"):\n            if \"boundary\" not in content_type:\n                # only remove the header if the boundary hasn't been explicitly set\n                # as the caller doesn't want httpx to come up with their own boundary\n                headers.pop(\"Content-Type\")\n\n            # As we are now sending multipart/form-data instead of application/json\n            # we need to tell httpx to use it, https://www.python-httpx.org/advanced/#multipart-file-encoding\n            if json_data:\n                if not is_dict(json_data):\n                    raise TypeError(\n                        f\"Expected query input to be a dictionary for multipart requests but got {type(json_data)} instead.\"\n                    )\n                kwargs[\"data\"] = self._serialize_multipartform(json_data)\n\n        # TODO: report this error to httpx\n        return self._client.build_request(  # pyright: ignore[reportUnknownMemberType]\n            headers=headers,\n            timeout=self.timeout if isinstance(options.timeout, NotGiven) else options.timeout,\n            method=options.method,\n            url=self._prepare_url(options.url),\n            # the `Query` type that we use is incompatible with qs'\n            # `Params` type as it needs to be typed as `Mapping[str, object]`\n            # so that passing a `TypedDict` doesn't cause an error.\n            # https://github.com/microsoft/pyright/issues/3526#event-6715453066\n            params=self.qs.stringify(cast(Mapping[str, Any], params)) if params else None,\n            json=json_data,\n            files=options.files,\n            **kwargs,\n        )\n\n    def _serialize_multipartform(self, data: Mapping[object, object]) -> dict[str, object]:\n        items = self.qs.stringify_items(\n            # TODO: type ignore is required as stringify_items is well typed but we can't be\n            # well typed without heavy validation.\n            data,  # type: ignore\n            array_format=\"brackets\",\n        )\n        serialized: dict[str, object] = {}\n        for key, value in items:\n            existing = serialized.get(key)\n\n            if not existing:\n                serialized[key] = value\n                continue\n\n            # If a value has already been set for this key then that\n            # means we're sending data like `array[]=[1, 2, 3]` and we\n            # need to tell httpx that we want to send multiple values with\n            # the same key which is done by using a list or a tuple.\n            #\n            # Note: 2d arrays should never result in the same key at both\n            # levels so it's safe to assume that if the value is a list,\n            # it was because we changed it to be a list.\n            if is_list(existing):\n                existing.append(value)\n            else:\n                serialized[key] = [existing, value]\n\n        return serialized\n\n    def _maybe_override_cast_to(self, cast_to: type[ResponseT], options: FinalRequestOptions) -> type[ResponseT]:\n        if not is_given(options.headers):\n            return cast_to\n\n        # make a copy of the headers so we don't mutate user-input\n        headers = dict(options.headers)\n\n        # we internally support defining a temporary header to override the\n        # default `cast_to` type for use with `.with_raw_response` and `.with_streaming_response`\n        # see _response.py for implementation details\n        override_cast_to = headers.pop(OVERRIDE_CAST_TO_HEADER, NOT_GIVEN)\n        if is_given(override_cast_to):\n            options.headers = headers\n            return cast(Type[ResponseT], override_cast_to)\n\n        return cast_to\n\n    def _should_stream_response_body(self, request: httpx.Request) -> bool:\n        return request.headers.get(RAW_RESPONSE_HEADER) == \"stream\"  # type: ignore[no-any-return]\n\n    def _process_response_data(\n        self,\n        *,\n        data: object,\n        cast_to: type[ResponseT],\n        response: httpx.Response,\n    ) -> ResponseT:\n        if data is None:\n            return cast(ResponseT, None)\n\n        if cast_to is object:\n            return cast(ResponseT, data)\n\n        try:\n            if inspect.isclass(cast_to) and issubclass(cast_to, ModelBuilderProtocol):\n                return cast(ResponseT, cast_to.build(response=response, data=data))\n\n            if self._strict_response_validation:\n                return cast(ResponseT, validate_type(type_=cast_to, value=data))\n\n            return cast(ResponseT, construct_type(type_=cast_to, value=data))\n        except pydantic.ValidationError as err:\n            raise APIResponseValidationError(response=response, body=data) from err\n\n    @property\n    def qs(self) -> Querystring:\n        return Querystring()\n\n    @property\n    def custom_auth(self) -> httpx.Auth | None:\n        return None\n\n    @property\n    def auth_headers(self) -> dict[str, str]:\n        return {}\n\n    @property\n    def default_headers(self) -> dict[str, str | Omit]:\n        return {\n            \"Accept\": \"application/json\",\n            \"Content-Type\": \"application/json\",\n            \"User-Agent\": self.user_agent,\n            **self.platform_headers(),\n            **self.auth_headers,\n            **self._custom_headers,\n        }\n\n    def _validate_headers(\n        self,\n        headers: Headers,  # noqa: ARG002\n        custom_headers: Headers,  # noqa: ARG002\n    ) -> None:\n        \"\"\"Validate the given default headers and custom headers.\n\n        Does nothing by default.\n        \"\"\"\n        return\n\n    @property\n    def user_agent(self) -> str:\n        return f\"{self.__class__.__name__}/Python {self._version}\"\n\n    @property\n    def base_url(self) -> URL:\n        return self._base_url\n\n    @base_url.setter\n    def base_url(self, url: URL | str) -> None:\n        self._base_url = self._enforce_trailing_slash(url if isinstance(url, URL) else URL(url))\n\n    def platform_headers(self) -> Dict[str, str]:\n        return platform_headers(self._version)\n\n    def _parse_retry_after_header(self, response_headers: Optional[httpx.Headers] = None) -> float | None:\n        \"\"\"Returns a float of the number of seconds (not milliseconds) to wait after retrying, or None if unspecified.\n\n        About the Retry-After header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After\n        See also  https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After#syntax\n        \"\"\"\n        if response_headers is None:\n            return None\n\n        # First, try the non-standard `retry-after-ms` header for milliseconds,\n        # which is more precise than integer-seconds `retry-after`\n        try:\n            retry_ms_header = response_headers.get(\"retry-after-ms\", None)\n            return float(retry_ms_header) / 1000\n        except (TypeError, ValueError):\n            pass\n\n        # Next, try parsing `retry-after` header as seconds (allowing nonstandard floats).\n        retry_header = response_headers.get(\"retry-after\")\n        try:\n            # note: the spec indicates that this should only ever be an integer\n            # but if someone sends a float there's no reason for us to not respect it\n            return float(retry_header)\n        except (TypeError, ValueError):\n            pass\n\n        # Last, try parsing `retry-after` as a date.\n        retry_date_tuple = email.utils.parsedate_tz(retry_header)\n        if retry_date_tuple is None:\n            return None\n\n        retry_date = email.utils.mktime_tz(retry_date_tuple)\n        return float(retry_date - time.time())\n\n    def _calculate_retry_timeout(\n        self,\n        remaining_retries: int,\n        options: FinalRequestOptions,\n        response_headers: Optional[httpx.Headers] = None,\n    ) -> float:\n        max_retries = options.get_max_retries(self.max_retries)\n\n        # If the API asks us to wait a certain amount of time (and it's a reasonable amount), just do what it says.\n        retry_after = self._parse_retry_after_header(response_headers)\n        if retry_after is not None and 0 < retry_after <= 60:\n            return retry_after\n\n        nb_retries = max_retries - remaining_retries\n\n        # Apply exponential backoff, but not more than the max.\n        sleep_seconds = min(INITIAL_RETRY_DELAY * pow(2.0, nb_retries), MAX_RETRY_DELAY)\n\n        # Apply some jitter, plus-or-minus half a second.\n        jitter = 1 - 0.25 * random()\n        timeout = sleep_seconds * jitter\n        return timeout if timeout >= 0 else 0\n\n    def _should_retry(self, response: httpx.Response) -> bool:\n        # Note: this is not a standard header\n        should_retry_header = response.headers.get(\"x-should-retry\")\n\n        # If the server explicitly says whether or not to retry, obey.\n        if should_retry_header == \"true\":\n            log.debug(\"Retrying as header `x-should-retry` is set to `true`\")\n            return True\n        if should_retry_header == \"false\":\n            log.debug(\"Not retrying as header `x-should-retry` is set to `false`\")\n            return False\n\n        # Retry on request timeouts.\n        if response.status_code == 408:\n            log.debug(\"Retrying due to status code %i\", response.status_code)\n            return True\n\n        # Retry on lock timeouts.\n        if response.status_code == 409:\n            log.debug(\"Retrying due to status code %i\", response.status_code)\n            return True\n\n        # Retry on rate limits.\n        if response.status_code == 429:\n            log.debug(\"Retrying due to status code %i\", response.status_code)\n            return True\n\n        # Retry internal errors.\n        if response.status_code >= 500:\n            log.debug(\"Retrying due to status code %i\", response.status_code)\n            return True\n\n        log.debug(\"Not retrying\")\n        return False\n\n    def _idempotency_key(self) -> str:\n        return f\"stainless-python-retry-{uuid.uuid4()}\"\n\n\nclass SyncHttpxClientWrapper(httpx.Client):\n    def __del__(self) -> None:\n        try:\n            self.close()\n        except Exception:\n            pass\n\n\nclass SyncAPIClient(BaseClient[httpx.Client, Stream[Any]]):\n    _client: httpx.Client\n    _default_stream_cls: type[Stream[Any]] | None = None\n\n    def __init__(\n        self,\n        *,\n        version: str,\n        base_url: str | URL,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        timeout: float | Timeout | None | NotGiven = NOT_GIVEN,\n        transport: Transport | None = None,\n        proxies: ProxiesTypes | None = None,\n        limits: Limits | None = None,\n        http_client: httpx.Client | None = None,\n        custom_headers: Mapping[str, str] | None = None,\n        custom_query: Mapping[str, object] | None = None,\n        _strict_response_validation: bool,\n    ) -> None:\n        if limits is not None:\n            warnings.warn(\n                \"The `connection_pool_limits` argument is deprecated. The `http_client` argument should be passed instead\",\n                category=DeprecationWarning,\n                stacklevel=3,\n            )\n            if http_client is not None:\n                raise ValueError(\"The `http_client` argument is mutually exclusive with `connection_pool_limits`\")\n        else:\n            limits = DEFAULT_LIMITS\n\n        if transport is not None:\n            warnings.warn(\n                \"The `transport` argument is deprecated. The `http_client` argument should be passed instead\",\n                category=DeprecationWarning,\n                stacklevel=3,\n            )\n            if http_client is not None:\n                raise ValueError(\"The `http_client` argument is mutually exclusive with `transport`\")\n\n        if proxies is not None:\n            warnings.warn(\n                \"The `proxies` argument is deprecated. The `http_client` argument should be passed instead\",\n                category=DeprecationWarning,\n                stacklevel=3,\n            )\n            if http_client is not None:\n                raise ValueError(\"The `http_client` argument is mutually exclusive with `proxies`\")\n\n        if not is_given(timeout):\n            # if the user passed in a custom http client with a non-default\n            # timeout set then we use that timeout.\n            #\n            # note: there is an edge case here where the user passes in a client\n            # where they've explicitly set the timeout to match the default timeout\n            # as this check is structural, meaning that we'll think they didn't\n            # pass in a timeout and will ignore it\n            if http_client and http_client.timeout != HTTPX_DEFAULT_TIMEOUT:\n                timeout = http_client.timeout\n            else:\n                timeout = DEFAULT_TIMEOUT\n\n        super().__init__(\n            version=version,\n            limits=limits,\n            # cast to a valid type because mypy doesn't understand our type narrowing\n            timeout=cast(Timeout, timeout),\n            proxies=proxies,\n            base_url=base_url,\n            transport=transport,\n            max_retries=max_retries,\n            custom_query=custom_query,\n            custom_headers=custom_headers,\n            _strict_response_validation=_strict_response_validation,\n        )\n        self._client = http_client or SyncHttpxClientWrapper(\n            base_url=base_url,\n            # cast to a valid type because mypy doesn't understand our type narrowing\n            timeout=cast(Timeout, timeout),\n            proxies=proxies,\n            transport=transport,\n            limits=limits,\n            follow_redirects=True,\n        )\n\n    def is_closed(self) -> bool:\n        return self._client.is_closed\n\n    def close(self) -> None:\n        \"\"\"Close the underlying HTTPX client.\n\n        The client will *not* be usable after this.\n        \"\"\"\n        # If an error is thrown while constructing a client, self._client\n        # may not be present\n        if hasattr(self, \"_client\"):\n            self._client.close()\n\n    def __enter__(self: _T) -> _T:\n        return self\n\n    def __exit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        self.close()\n\n    def _prepare_options(\n        self,\n        options: FinalRequestOptions,  # noqa: ARG002\n    ) -> None:\n        \"\"\"Hook for mutating the given options\"\"\"\n        return None\n\n    def _prepare_request(\n        self,\n        request: httpx.Request,  # noqa: ARG002\n    ) -> None:\n        \"\"\"This method is used as a callback for mutating the `Request` object\n        after it has been constructed.\n        This is useful for cases where you want to add certain headers based off of\n        the request properties, e.g. `url`, `method` etc.\n        \"\"\"\n        return None\n\n    @overload\n    def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        remaining_retries: Optional[int] = None,\n        *,\n        stream: Literal[True],\n        stream_cls: Type[_StreamT],\n    ) -> _StreamT:\n        ...\n\n    @overload\n    def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        remaining_retries: Optional[int] = None,\n        *,\n        stream: Literal[False] = False,\n    ) -> ResponseT:\n        ...\n\n    @overload\n    def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        remaining_retries: Optional[int] = None,\n        *,\n        stream: bool = False,\n        stream_cls: Type[_StreamT] | None = None,\n    ) -> ResponseT | _StreamT:\n        ...\n\n    def request(\n        self,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        remaining_retries: Optional[int] = None,\n        *,\n        stream: bool = False,\n        stream_cls: type[_StreamT] | None = None,\n    ) -> ResponseT | _StreamT:\n        return self._request(\n            cast_to=cast_to,\n            options=options,\n            stream=stream,\n            stream_cls=stream_cls,\n            remaining_retries=remaining_retries,\n        )\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        remaining_retries: int | None,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        self._prepare_options(options)\n\n        retries = self._remaining_retries(remaining_retries, options)\n        request = self._build_request(options)\n        self._prepare_request(request)\n\n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n\n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n\n            if retries > 0:\n                return self._retry_request(\n                    options,\n                    cast_to,\n                    retries,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n\n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n\n            if retries > 0:\n                return self._retry_request(\n                    options,\n                    cast_to,\n                    retries,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n\n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n\n        log.debug(\n            'HTTP Request: %s %s \"%i %s\"', request.method, request.url, response.status_code, response.reason_phrase\n        )\n\n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n\n            if retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    options,\n                    cast_to,\n                    retries,\n                    err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n\n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n\n            log.debug(\"Re-raising status error\")\n            raise self._make_status_error_from_response(err.response) from None\n\n        return self._process_response(\n            cast_to=cast_to,\n            options=options,\n            response=response,\n            stream=stream,\n            stream_cls=stream_cls,\n        )\n\n    def _retry_request(\n        self,\n        options: FinalRequestOptions,\n        cast_to: Type[ResponseT],\n        remaining_retries: int,\n        response_headers: httpx.Headers | None,\n        *,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        remaining = remaining_retries - 1\n        if remaining == 1:\n            log.debug(\"1 retry left\")\n        else:\n            log.debug(\"%i retries left\", remaining)\n\n        timeout = self._calculate_retry_timeout(remaining, options, response_headers)\n        log.info(\"Retrying request to %s in %f seconds\", options.url, timeout)\n\n        # In a synchronous context we are blocking the entire thread. Up to the library user to run the client in a\n        # different thread if necessary.\n        time.sleep(timeout)\n\n        return self._request(\n            options=options,\n            cast_to=cast_to,\n            remaining_retries=remaining,\n            stream=stream,\n            stream_cls=stream_cls,\n        )\n\n    def _process_response(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        response: httpx.Response,\n        stream: bool,\n        stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None,\n    ) -> ResponseT:\n        if response.request.headers.get(RAW_RESPONSE_HEADER) == \"true\":\n            return cast(\n                ResponseT,\n                LegacyAPIResponse(\n                    raw=response,\n                    client=self,\n                    cast_to=cast_to,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    options=options,\n                ),\n            )\n\n        origin = get_origin(cast_to) or cast_to\n\n        if inspect.isclass(origin) and issubclass(origin, BaseAPIResponse):\n            if not issubclass(origin, APIResponse):\n                raise TypeError(f\"API Response types must subclass {APIResponse}; Received {origin}\")\n\n            response_cls = cast(\"type[BaseAPIResponse[Any]]\", cast_to)\n            return cast(\n                ResponseT,\n                response_cls(\n                    raw=response,\n                    client=self,\n                    cast_to=extract_response_type(response_cls),\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    options=options,\n                ),\n            )\n\n        if cast_to == httpx.Response:\n            return cast(ResponseT, response)\n\n        api_response = APIResponse(\n            raw=response,\n            client=self,\n            cast_to=cast(\"type[ResponseT]\", cast_to),  # pyright: ignore[reportUnnecessaryCast]\n            stream=stream,\n            stream_cls=stream_cls,\n            options=options,\n        )\n        if bool(response.request.headers.get(RAW_RESPONSE_HEADER)):\n            return cast(ResponseT, api_response)\n\n        return api_response.parse()\n\n    def _request_api_list(\n        self,\n        model: Type[object],\n        page: Type[SyncPageT],\n        options: FinalRequestOptions,\n    ) -> SyncPageT:\n        def _parser(resp: SyncPageT) -> SyncPageT:\n            resp._set_private_attributes(\n                client=self,\n                model=model,\n                options=options,\n            )\n            return resp\n\n        options.post_parser = _parser\n\n        return self.request(page, options, stream=False)\n\n    @overload\n    def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: Literal[False] = False,\n    ) -> ResponseT:\n        ...\n\n    @overload\n    def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: Literal[True],\n        stream_cls: type[_StreamT],\n    ) -> _StreamT:\n        ...\n\n    @overload\n    def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: bool,\n        stream_cls: type[_StreamT] | None = None,\n    ) -> ResponseT | _StreamT:\n        ...\n\n    def get(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        options: RequestOptions = {},\n        stream: bool = False,\n        stream_cls: type[_StreamT] | None = None,\n    ) -> ResponseT | _StreamT:\n        opts = FinalRequestOptions.construct(method=\"get\", url=path, **options)\n        # cast is required because mypy complains about returning Any even though\n        # it understands the type variables\n        return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n\n    @overload\n    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: Literal[False] = False,\n    ) -> ResponseT:\n        ...\n\n    @overload\n    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: Literal[True],\n        stream_cls: type[_StreamT],\n    ) -> _StreamT:\n        ...\n\n    @overload\n    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: bool,\n        stream_cls: type[_StreamT] | None = None,\n    ) -> ResponseT | _StreamT:\n        ...\n\n    def post(\n        self,\n        path: str,\n        *,\n        cast_to: Type[ResponseT],\n        body: Body | None = None,\n        options: RequestOptions = {},\n        files: RequestFiles | None = None,\n        stream: bool = False,\n...\n# Path: src/openai/_module_client.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom typing_extensions import override\n\nfrom . import resources, _load_client\nfrom ._utils import LazyProxy\n\n\nclass ChatProxy(LazyProxy[resources.Chat]):\n    @override\n    def __load__(self) -> resources.Chat:\n        return _load_client().chat\n\n\nclass BetaProxy(LazyProxy[resources.Beta]):\n    @override\n    def __load__(self) -> resources.Beta:\n        return _load_client().beta\n\n\nclass FilesProxy(LazyProxy[resources.Files]):\n    @override\n    def __load__(self) -> resources.Files:\n        return _load_client().files\n\n\nclass AudioProxy(LazyProxy[resources.Audio]):\n    @override\n    def __load__(self) -> resources.Audio:\n        return _load_client().audio\n\n\nclass ImagesProxy(LazyProxy[resources.Images]):\n    @override\n    def __load__(self) -> resources.Images:\n        return _load_client().images\n\n\nclass ModelsProxy(LazyProxy[resources.Models]):\n    @override\n    def __load__(self) -> resources.Models:\n        return _load_client().models\n\n\nclass EmbeddingsProxy(LazyProxy[resources.Embeddings]):\n    @override\n    def __load__(self) -> resources.Embeddings:\n        return _load_client().embeddings\n\n\nclass CompletionsProxy(LazyProxy[resources.Completions]):\n    @override\n    def __load__(self) -> resources.Completions:\n        return _load_client().completions\n\n\nclass ModerationsProxy(LazyProxy[resources.Moderations]):\n    @override\n    def __load__(self) -> resources.Moderations:\n        return _load_client().moderations\n\n\nclass FineTuningProxy(LazyProxy[resources.FineTuning]):\n    @override\n    def __load__(self) -> resources.FineTuning:\n        return _load_client().fine_tuning\n\n\nchat: resources.Chat = ChatProxy().__as_proxied__()\nbeta: resources.Beta = BetaProxy().__as_proxied__()\nfiles: resources.Files = FilesProxy().__as_proxied__()\naudio: resources.Audio = AudioProxy().__as_proxied__()\nimages: resources.Images = ImagesProxy().__as_proxied__()\nmodels: resources.Models = ModelsProxy().__as_proxied__()\nembeddings: resources.Embeddings = EmbeddingsProxy().__as_proxied__()\ncompletions: resources.Completions = CompletionsProxy().__as_proxied__()\nmoderations: resources.Moderations = ModerationsProxy().__as_proxied__()\nfine_tuning: resources.FineTuning = FineTuningProxy().__as_proxied__()\n\n# Path: src/openai/_utils/_logs.py\nimport os\nimport logging\n\nlogger: logging.Logger = logging.getLogger(\"openai\")\nhttpx_logger: logging.Logger = logging.getLogger(\"httpx\")\n\n\n\ndef _basic_config() -> None:\n    # e.g. [2023-10-05 14:12:26 - openai._base_client:818 - DEBUG] HTTP Request: POST http://127.0.0.1:4010/foo/bar \"200 OK\"\n    logging.basicConfig(\n        format=\"[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s\",\n        datefmt=\"%Y-%m-%d %H:%M:%S\",\n    )\n\n\ndef setup_logging() -> None:\n    env = os.environ.get(\"OPENAI_LOG\")\n    if env == \"debug\":\n        _basic_config()\n        logger.setLevel(logging.DEBUG)\n        httpx_logger.setLevel(logging.DEBUG)\n    elif env == \"info\":\n        _basic_config()\n        logger.setLevel(logging.INFO)\n        httpx_logger.setLevel(logging.INFO)\n\n# Path: src/openai/__init__.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nimport os as _os\nfrom typing_extensions import override\n\nfrom . import types\nfrom ._types import NoneType, Transport, ProxiesTypes\nfrom ._utils import file_from_path\nfrom ._client import Client, OpenAI, Stream, Timeout, Transport, AsyncClient, AsyncOpenAI, AsyncStream, RequestOptions\nfrom ._models import BaseModel\nfrom ._version import __title__, __version__\nfrom ._response import APIResponse as APIResponse, AsyncAPIResponse as AsyncAPIResponse\nfrom ._exceptions import (\n    APIError,\n    OpenAIError,\n    ConflictError,\n    NotFoundError,\n    APIStatusError,\n    RateLimitError,\n    APITimeoutError,\n    BadRequestError,\n    APIConnectionError,\n    AuthenticationError,\n    InternalServerError,\n    PermissionDeniedError,\n    UnprocessableEntityError,\n    APIResponseValidationError,\n)\nfrom ._utils._logs import setup_logging as _setup_logging\n\n__all__ = [\n    \"types\",\n    \"__version__\",\n    \"__title__\",\n    \"NoneType\",\n    \"Transport\",\n    \"ProxiesTypes\",\n    \"OpenAIError\",\n    \"APIError\",\n    \"APIStatusError\",\n    \"APITimeoutError\",\n    \"APIConnectionError\",\n    \"APIResponseValidationError\",\n    \"BadRequestError\",\n    \"AuthenticationError\",\n    \"PermissionDeniedError\",\n    \"NotFoundError\",\n    \"ConflictError\",\n    \"UnprocessableEntityError\",\n    \"RateLimitError\",\n    \"InternalServerError\",\n    \"Timeout\",\n    \"RequestOptions\",\n    \"Client\",\n    \"AsyncClient\",\n    \"Stream\",\n    \"AsyncStream\",\n    \"OpenAI\",\n    \"AsyncOpenAI\",\n    \"file_from_path\",\n    \"BaseModel\",\n]\n\nfrom .lib import azure as _azure\nfrom .version import VERSION as VERSION\nfrom .lib.azure import AzureOpenAI as AzureOpenAI, AsyncAzureOpenAI as AsyncAzureOpenAI\nfrom .lib._old_api import *\n\n_setup_logging()\n\n# Update the __module__ attribute for exported symbols so that\n# error messages point to this module instead of the module\n# it was originally defined in, e.g.\n# openai._exceptions.NotFoundError -> openai.NotFoundError\n__locals = locals()\nfor __name in __all__:\n    if not __name.startswith(\"__\"):\n        try:\n            __locals[__name].__module__ = \"openai\"\n        except (TypeError, AttributeError):\n            # Some of our exported symbols are builtins which we can't set attributes for.\n            pass\n\n# ------ Module level client ------\nimport typing as _t\nimport typing_extensions as _te\n\nimport httpx as _httpx\n\nfrom ._base_client import DEFAULT_TIMEOUT, DEFAULT_MAX_RETRIES\n\napi_key: str | None = None\n\norganization: str | None = None\n\nbase_url: str | _httpx.URL | None = None\n\ntimeout: float | Timeout | None = DEFAULT_TIMEOUT\n\nmax_retries: int = DEFAULT_MAX_RETRIES\n\ndefault_headers: _t.Mapping[str, str] | None = None\n\ndefault_query: _t.Mapping[str, object] | None = None\n\nhttp_client: _httpx.Client | None = None\n\n_ApiType = _te.Literal[\"openai\", \"azure\"]\n\napi_type: _ApiType | None = _t.cast(_ApiType, _os.environ.get(\"OPENAI_API_TYPE\"))\n\napi_version: str | None = _os.environ.get(\"OPENAI_API_VERSION\")\n\nazure_endpoint: str | None = _os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n\nazure_ad_token: str | None = _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\n\nazure_ad_token_provider: _azure.AzureADTokenProvider | None = None\n\n\nclass _ModuleClient(OpenAI):\n    # Note: we have to use type: ignores here as overriding class members\n    # with properties is technically unsafe but it is fine for our use case\n\n    @property  # type: ignore\n    @override\n    def api_key(self) -> str | None:\n        return api_key\n\n    @api_key.setter  # type: ignore\n    def api_key(self, value: str | None) -> None:  # type: ignore\n        global api_key\n\n        api_key = value\n\n    @property  # type: ignore\n    @override\n    def organization(self) -> str | None:\n        return organization\n\n    @organization.setter  # type: ignore\n    def organization(self, value: str | None) -> None:  # type: ignore\n        global organization\n\n        organization = value\n\n    @property\n    @override\n    def base_url(self) -> _httpx.URL:\n        if base_url is not None:\n            return _httpx.URL(base_url)\n\n        return super().base_url\n\n    @base_url.setter\n    def base_url(self, url: _httpx.URL | str) -> None:\n        super().base_url = url  # type: ignore[misc]\n\n    @property  # type: ignore\n    @override\n    def timeout(self) -> float | Timeout | None:\n        return timeout\n\n    @timeout.setter  # type: ignore\n    def timeout(self, value: float | Timeout | None) -> None:  # type: ignore\n        global timeout\n\n        timeout = value\n\n    @property  # type: ignore\n    @override\n    def max_retries(self) -> int:\n        return max_retries\n\n    @max_retries.setter  # type: ignore\n    def max_retries(self, value: int) -> None:  # type: ignore\n        global max_retries\n\n        max_retries = value\n\n    @property  # type: ignore\n    @override\n    def _custom_headers(self) -> _t.Mapping[str, str] | None:\n        return default_headers\n\n    @_custom_headers.setter  # type: ignore\n    def _custom_headers(self, value: _t.Mapping[str, str] | None) -> None:  # type: ignore\n        global default_headers\n\n        default_headers = value\n\n    @property  # type: ignore\n    @override\n    def _custom_query(self) -> _t.Mapping[str, object] | None:\n        return default_query\n\n    @_custom_query.setter  # type: ignore\n    def _custom_query(self, value: _t.Mapping[str, object] | None) -> None:  # type: ignore\n        global default_query\n\n        default_query = value\n\n    @property  # type: ignore\n    @override\n    def _client(self) -> _httpx.Client:\n        return http_client or super()._client\n\n    @_client.setter  # type: ignore\n    def _client(self, value: _httpx.Client) -> None:  # type: ignore\n        global http_client\n\n        http_client = value\n\n\nclass _AzureModuleClient(_ModuleClient, AzureOpenAI):  # type: ignore\n    ...\n\n\nclass _AmbiguousModuleClientUsageError(OpenAIError):\n    def __init__(self) -> None:\n        super().__init__(\n            \"Ambiguous use of module client; please set `openai.api_type` or the `OPENAI_API_TYPE` environment variable to `openai` or `azure`\"\n        )\n\n\ndef _has_openai_credentials() -> bool:\n    return _os.environ.get(\"OPENAI_API_KEY\") is not None\n\n\ndef _has_azure_credentials() -> bool:\n    return azure_endpoint is not None or _os.environ.get(\"AZURE_OPENAI_API_KEY\") is not None\n\n\ndef _has_azure_ad_credentials() -> bool:\n    return (\n        _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\") is not None\n        or azure_ad_token is not None\n        or azure_ad_token_provider is not None\n    )\n\n\n_client: OpenAI | None = None\n\n\ndef _load_client() -> OpenAI:  # type: ignore[reportUnusedFunction]\n    global _client\n\n    if _client is None:\n        global api_type, azure_endpoint, azure_ad_token, api_version\n\n        if azure_endpoint is None:\n            azure_endpoint = _os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n\n        if azure_ad_token is None:\n            azure_ad_token = _os.environ.get(\"AZURE_OPENAI_AD_TOKEN\")\n\n        if api_version is None:\n            api_version = _os.environ.get(\"OPENAI_API_VERSION\")\n\n        if api_type is None:\n            has_openai = _has_openai_credentials()\n            has_azure = _has_azure_credentials()\n            has_azure_ad = _has_azure_ad_credentials()\n\n            if has_openai and (has_azure or has_azure_ad):\n                raise _AmbiguousModuleClientUsageError()\n\n            if (azure_ad_token is not None or azure_ad_token_provider is not None) and _os.environ.get(\n                \"AZURE_OPENAI_API_KEY\"\n            ) is not None:\n                raise _AmbiguousModuleClientUsageError()\n\n            if has_azure or has_azure_ad:\n                api_type = \"azure\"\n            else:\n                api_type = \"openai\"\n\n        if api_type == \"azure\":\n            _client = _AzureModuleClient(  # type: ignore\n                api_version=api_version,\n                azure_endpoint=azure_endpoint,\n                api_key=api_key,\n                azure_ad_token=azure_ad_token,\n                azure_ad_token_provider=azure_ad_token_provider,\n                organization=organization,\n                base_url=base_url,\n                timeout=timeout,\n                max_retries=max_retries,\n                default_headers=default_headers,\n                default_query=default_query,\n                http_client=http_client,\n            )\n            return _client\n\n        _client = _ModuleClient(\n            api_key=api_key,\n            organization=organization,\n            base_url=base_url,\n            timeout=timeout,\n            max_retries=max_retries,\n            default_headers=default_headers,\n            default_query=default_query,\n            http_client=http_client,\n        )\n        return _client\n\n    return _client\n\n\ndef _reset_client() -> None:  # type: ignore[reportUnusedFunction]\n    global _client\n\n    _client = None\n\n\nfrom ._module_client import (\n    beta as beta,\n    chat as chat,\n    audio as audio,\n    files as files,\n    images as images,\n    models as models,\n    embeddings as embeddings,\n    completions as completions,\n    fine_tuning as fine_tuning,\n    moderations as moderations,\n)\n\n# Path: src/openai/cli/_models.py\nfrom typing import Any\nfrom typing_extensions import ClassVar\n\nimport pydantic\n\nfrom .. import _models\nfrom .._compat import PYDANTIC_V2, ConfigDict\n\n\nclass BaseModel(_models.BaseModel):\n    if PYDANTIC_V2:\n        model_config: ClassVar[ConfigDict] = ConfigDict(extra=\"ignore\", arbitrary_types_allowed=True)\n    else:\n\n        class Config(pydantic.BaseConfig):  # type: ignore\n            extra: Any = pydantic.Extra.ignore  # type: ignore\n            arbitrary_types_allowed: bool = True\n\n# Path: src/openai/cli/_progress.py\nfrom __future__ import annotations\n\nimport io\nfrom typing import Callable\nfrom typing_extensions import override\n\n\nclass CancelledError(Exception):\n    def __init__(self, msg: str) -> None:\n        self.msg = msg\n        super().__init__(msg)\n\n    @override\n    def __str__(self) -> str:\n        return self.msg\n\n    __repr__ = __str__\n\n\nclass BufferReader(io.BytesIO):\n    def __init__(self, buf: bytes = b\"\", desc: str | None = None) -> None:\n        super().__init__(buf)\n        self._len = len(buf)\n        self._progress = 0\n        self._callback = progress(len(buf), desc=desc)\n\n    def __len__(self) -> int:\n        return self._len\n\n    @override\n    def read(self, n: int | None = -1) -> bytes:\n        chunk = io.BytesIO.read(self, n)\n        self._progress += len(chunk)\n\n        try:\n            self._callback(self._progress)\n        except Exception as e:  # catches exception from the callback\n            raise CancelledError(\"The upload was cancelled: {}\".format(e)) from e\n\n        return chunk\n\n\ndef progress(total: float, desc: str | None) -> Callable[[float], None]:\n    import tqdm\n\n    meter = tqdm.tqdm(total=total, unit_scale=True, desc=desc)\n\n    def incr(progress: float) -> None:\n        meter.n = progress\n        if progress == total:\n            meter.close()\n        else:\n            meter.refresh()\n\n    return incr\n\n\ndef MB(i: int) -> int:\n    return int(i // 1024**2)\n\n# Path: src/openai/cli/_utils.py\nfrom __future__ import annotations\n\nimport sys\n\nimport openai\n\nfrom .. import OpenAI, _load_client\nfrom .._compat import model_json\nfrom .._models import BaseModel\n\n\nclass Colors:\n    HEADER = \"\\033[95m\"\n    OKBLUE = \"\\033[94m\"\n    OKGREEN = \"\\033[92m\"\n    WARNING = \"\\033[93m\"\n    FAIL = \"\\033[91m\"\n    ENDC = \"\\033[0m\"\n    BOLD = \"\\033[1m\"\n    UNDERLINE = \"\\033[4m\"\n\n\ndef get_client() -> OpenAI:\n    return _load_client()\n\n\ndef organization_info() -> str:\n    organization = openai.organization\n    if organization is not None:\n        return \"[organization={}] \".format(organization)\n\n    return \"\"\n\n\ndef print_model(model: BaseModel) -> None:\n    sys.stdout.write(model_json(model, indent=2) + \"\\n\")\n\n\ndef can_use_http2() -> bool:\n    try:\n        import h2  # type: ignore  # noqa\n    except ImportError:\n        return False\n\n    return True\n\n...\n", "needle_token_start": 12002, "needle_token_end": 12148, "code_context_ntokens": 16036, "output": ["```python\nimport os\nimport logging\n\nlogger: logging.Logger = logging.getLogger(\"openai\")\nhttpx_logger: logging.Logger = logging.getLogger(\"httpx\")\n\ndef _basic_config() -> None:\n    # e.g. [2023-10-05 14:12:26 - openai._base_client:818 - DEBUG] HTTP Request: POST http://127.0.0.1:4010/foo/bar \"200 OK\"\n    logging.basicConfig(\n        format=\"[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s\",\n        datefmt=\"%Y-%m-%d %H:%M:%S\",\n    )\n\ndef setup_logging() -> None:\n    env = os.environ.get(\"OPENAI_LOG\")\n    if env == \"debug\":\n        _basic_config()\n        logger.setLevel(logging.DEBUG)\n        httpx_logger.setLevel(logging.DEBUG)\n    elif env == \"info\":\n        _basic_config()\n        logger.setLevel(logging.INFO)\n        httpx_logger.setLevel(logging.INFO)\n```"]}
{"repo": "openai/openai-python", "name": "__getattr__", "language": "python", "path": "src/openai/_utils/_proxy.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To dynamically handle attribute access on instances that do not explicitly define those attributes, effectively forwarding the attribute access to another object.\n2. **Input**: A string representing the name of the attribute being accessed.\n3. **Output**: The value of the attribute from the proxied object, or the proxied object itself if it is an instance of the same class.\n4. **Procedure**: When an attribute is accessed that isn't directly found on the instance, this method retrieves the proxied object. If this proxied object is an instance of the same class, it returns the proxied object itself. Otherwise, it retrieves and returns the attribute from the proxied object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/openai/_constants.py\n# File generated from our OpenAPI spec by Stainless.\n\nimport httpx\n\nRAW_RESPONSE_HEADER = \"X-Stainless-Raw-Response\"\nOVERRIDE_CAST_TO_HEADER = \"____stainless_override_cast_to\"\n\n# default timeout is 10 minutes\nDEFAULT_TIMEOUT = httpx.Timeout(timeout=600.0, connect=5.0)\nDEFAULT_MAX_RETRIES = 2\nDEFAULT_LIMITS = httpx.Limits(max_connections=100, max_keepalive_connections=20)\n\nINITIAL_RETRY_DELAY = 0.5\nMAX_RETRY_DELAY = 8.0\n\n# Path: src/openai/_utils/_proxy.py\nfrom __future__ import annotations\n\nfrom abc import ABC, abstractmethod\nfrom typing import Generic, TypeVar, Iterable, cast\nfrom typing_extensions import override\n\nT = TypeVar(\"T\")\n\n\nclass LazyProxy(Generic[T], ABC):\n    \"\"\"Implements data methods to pretend that an instance is another instance.\n\n    This includes forwarding attribute access and othe methods.\n    \"\"\"\n\n    # Note: we have to special case proxies that themselves return proxies\n    # to support using a proxy as a catch-all for any random access, e.g. `proxy.foo.bar.baz`\n\n    \ndef __getattr__(self, attr: str) -> object:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return proxied  # pyright: ignore\n        return getattr(proxied, attr)\n\n    @override\n    def __repr__(self) -> str:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return proxied.__class__.__name__\n        return repr(self.__get_proxied__())\n\n    @override\n    def __str__(self) -> str:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return proxied.__class__.__name__\n        return str(proxied)\n\n    @override\n    def __dir__(self) -> Iterable[str]:\n        proxied = self.__get_proxied__()\n        if isinstance(proxied, LazyProxy):\n            return []\n        return proxied.__dir__()\n\n    @property  # type: ignore\n    @override\n    def __class__(self) -> type:  # pyright: ignore\n        proxied = self.__get_proxied__()\n        if issubclass(type(proxied), LazyProxy):\n            return type(proxied)\n        return proxied.__class__\n\n    def __get_proxied__(self) -> T:\n        return self.__load__()\n\n    def __as_proxied__(self) -> T:\n        \"\"\"Helper method that returns the current proxy, typed as the loaded object\"\"\"\n        return cast(T, self)\n\n    @abstractmethod\n    def __load__(self) -> T:\n        ...\n\n# Path: src/openai/_utils/_streams.py\nfrom typing import Any\nfrom typing_extensions import Iterator, AsyncIterator\n\n\ndef consume_sync_iterator(iterator: Iterator[Any]) -> None:\n    for _ in iterator:\n        ...\n\n\nasync def consume_async_iterator(iterator: AsyncIterator[Any]) -> None:\n    async for _ in iterator:\n        ...\n\n# Path: src/openai/_utils/_sync.py\nfrom __future__ import annotations\n\nimport functools\nfrom typing import TypeVar, Callable, Awaitable\nfrom typing_extensions import ParamSpec\n\nimport anyio\nimport anyio.to_thread\n\nT_Retval = TypeVar(\"T_Retval\")\nT_ParamSpec = ParamSpec(\"T_ParamSpec\")\n\n\n# copied from `asyncer`, https://github.com/tiangolo/asyncer\ndef asyncify(\n    function: Callable[T_ParamSpec, T_Retval],\n    *,\n    cancellable: bool = False,\n    limiter: anyio.CapacityLimiter | None = None,\n) -> Callable[T_ParamSpec, Awaitable[T_Retval]]:\n    \"\"\"\n    Take a blocking function and create an async one that receives the same\n    positional and keyword arguments, and that when called, calls the original function\n    in a worker thread using `anyio.to_thread.run_sync()`. Internally,\n    `asyncer.asyncify()` uses the same `anyio.to_thread.run_sync()`, but it supports\n    keyword arguments additional to positional arguments and it adds better support for\n    autocompletion and inline errors for the arguments of the function called and the\n    return value.\n\n    If the `cancellable` option is enabled and the task waiting for its completion is\n    cancelled, the thread will still run its course but its return value (or any raised\n    exception) will be ignored.\n\n    Use it like this:\n\n    ```Python\n    def do_work(arg1, arg2, kwarg1=\"\", kwarg2=\"\") -> str:\n        # Do work\n        return \"Some result\"\n\n\n    result = await to_thread.asyncify(do_work)(\"spam\", \"ham\", kwarg1=\"a\", kwarg2=\"b\")\n    print(result)\n    ```\n\n    ## Arguments\n\n    `function`: a blocking regular callable (e.g. a function)\n    `cancellable`: `True` to allow cancellation of the operation\n    `limiter`: capacity limiter to use to limit the total amount of threads running\n        (if omitted, the default limiter is used)\n\n    ## Return\n\n    An async function that takes the same positional and keyword arguments as the\n    original one, that when called runs the same original function in a thread worker\n    and returns the result.\n    \"\"\"\n\n    async def wrapper(*args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs) -> T_Retval:\n        partial_f = functools.partial(function, *args, **kwargs)\n        return await anyio.to_thread.run_sync(partial_f, cancellable=cancellable, limiter=limiter)\n\n    return wrapper\n\n# Path: src/openai/_utils/_typing.py\nfrom __future__ import annotations\n\nfrom typing import Any, TypeVar, Iterable, cast\nfrom collections import abc as _c_abc\nfrom typing_extensions import Required, Annotated, get_args, get_origin\n\nfrom .._types import InheritsGeneric\nfrom .._compat import is_union as _is_union\n\n\ndef is_annotated_type(typ: type) -> bool:\n    return get_origin(typ) == Annotated\n\n\ndef is_list_type(typ: type) -> bool:\n    return (get_origin(typ) or typ) == list\n\n\ndef is_iterable_type(typ: type) -> bool:\n    \"\"\"If the given type is `typing.Iterable[T]`\"\"\"\n    origin = get_origin(typ) or typ\n    return origin == Iterable or origin == _c_abc.Iterable\n\n\ndef is_union_type(typ: type) -> bool:\n    return _is_union(get_origin(typ))\n\n\ndef is_required_type(typ: type) -> bool:\n    return get_origin(typ) == Required\n\n\ndef is_typevar(typ: type) -> bool:\n    # type ignore is required because type checkers\n    # think this expression will always return False\n    return type(typ) == TypeVar  # type: ignore\n\n\n# Extracts T from Annotated[T, ...] or from Required[Annotated[T, ...]]\ndef strip_annotated_type(typ: type) -> type:\n    if is_required_type(typ) or is_annotated_type(typ):\n        return strip_annotated_type(cast(type, get_args(typ)[0]))\n\n    return typ\n\n\ndef extract_type_arg(typ: type, index: int) -> type:\n    args = get_args(typ)\n    try:\n        return cast(type, args[index])\n    except IndexError as err:\n        raise RuntimeError(f\"Expected type {typ} to have a type argument at index {index} but it did not\") from err\n\n\ndef extract_type_var_from_base(\n    typ: type,\n    *,\n    generic_bases: tuple[type, ...],\n    index: int,\n    failure_message: str | None = None,\n) -> type:\n    \"\"\"Given a type like `Foo[T]`, returns the generic type variable `T`.\n\n    This also handles the case where a concrete subclass is given, e.g.\n    ```py\n    class MyResponse(Foo[bytes]):\n        ...\n\n    extract_type_var(MyResponse, bases=(Foo,), index=0) -> bytes\n    ```\n\n    And where a generic subclass is given:\n    ```py\n    _T = TypeVar('_T')\n    class MyResponse(Foo[_T]):\n        ...\n\n    extract_type_var(MyResponse[bytes], bases=(Foo,), index=0) -> bytes\n    ```\n    \"\"\"\n    cls = cast(object, get_origin(typ) or typ)\n    if cls in generic_bases:\n        # we're given the class directly\n        return extract_type_arg(typ, index)\n\n    # if a subclass is given\n    # ---\n    # this is needed as __orig_bases__ is not present in the typeshed stubs\n    # because it is intended to be for internal use only, however there does\n    # not seem to be a way to resolve generic TypeVars for inherited subclasses\n    # without using it.\n    if isinstance(cls, InheritsGeneric):\n        target_base_class: Any | None = None\n        for base in cls.__orig_bases__:\n            if base.__origin__ in generic_bases:\n                target_base_class = base\n                break\n\n        if target_base_class is None:\n            raise RuntimeError(\n                \"Could not find the generic base class;\\n\"\n                \"This should never happen;\\n\"\n                f\"Does {cls} inherit from one of {generic_bases} ?\"\n            )\n\n        extracted = extract_type_arg(target_base_class, index)\n        if is_typevar(extracted):\n            # If the extracted type argument is itself a type variable\n            # then that means the subclass itself is generic, so we have\n            # to resolve the type argument from the class itself, not\n            # the base class.\n            #\n            # Note: if there is more than 1 type argument, the subclass could\n            # change the ordering of the type arguments, this is not currently\n            # supported.\n            return extract_type_arg(typ, index)\n\n        return extracted\n\n    raise RuntimeError(failure_message or f\"Could not resolve inner type variable at index {index} for {typ}\")\n\n# Path: src/openai/_files.py\nfrom __future__ import annotations\n\nimport io\nimport os\nimport pathlib\nfrom typing import overload\nfrom typing_extensions import TypeGuard\n\nimport anyio\n\nfrom ._types import (\n    FileTypes,\n    FileContent,\n...\n", "needle_token_start": 358, "needle_token_end": 425, "code_context_ntokens": 2787, "output": ["```python\ndef __getattr__(self, attr: str) -> object:\n    proxied = self.__get_proxied__()\n    if isinstance(proxied, LazyProxy):\n        return proxied  # pyright: ignore\n    return getattr(proxied, attr)\n```"]}
{"repo": "openai/openai-python", "name": "_enforce_trailing_slash", "language": "python", "path": "src/openai/_base_client.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: Ensures that a given URL ends with a slash, which is often required for proper URL formation in web requests.\n2. **Input**: A URL object representing a web address.\n3. **Output**: A URL object that is guaranteed to end with a slash.\n4. **Procedure**: The function checks if the last character of the URL's path is a slash. If not, it appends a slash to the end of the path and returns the modified URL. If the path already ends with a slash, the original URL is returned unchanged.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/openai/_client.py\n# File generated from our OpenAPI spec by Stainless.\n\nfrom __future__ import annotations\n\nimport os\nfrom typing import Any, Union, Mapping\nfrom typing_extensions import Self, override\n\nimport httpx\n\nfrom . import resources, _exceptions\nfrom ._qs import Querystring\nfrom ._types import (\n    NOT_GIVEN,\n    Omit,\n    Timeout,\n    NotGiven,\n    Transport,\n    ProxiesTypes,\n    RequestOptions,\n)\nfrom ._utils import (\n    is_given,\n    is_mapping,\n    get_async_library,\n)\nfrom ._version import __version__\nfrom ._streaming import Stream as Stream, AsyncStream as AsyncStream\nfrom ._exceptions import OpenAIError, APIStatusError\nfrom ._base_client import (\n    DEFAULT_MAX_RETRIES,\n    SyncAPIClient,\n    AsyncAPIClient,\n)\n\n__all__ = [\n    \"Timeout\",\n    \"Transport\",\n    \"ProxiesTypes\",\n    \"RequestOptions\",\n    \"resources\",\n    \"OpenAI\",\n    \"AsyncOpenAI\",\n    \"Client\",\n    \"AsyncClient\",\n]\n\n\nclass OpenAI(SyncAPIClient):\n    completions: resources.Completions\n    chat: resources.Chat\n    embeddings: resources.Embeddings\n    files: resources.Files\n    images: resources.Images\n    audio: resources.Audio\n    moderations: resources.Moderations\n    models: resources.Models\n    fine_tuning: resources.FineTuning\n    beta: resources.Beta\n    with_raw_response: OpenAIWithRawResponse\n    with_streaming_response: OpenAIWithStreamedResponse\n\n    # client options\n    api_key: str\n    organization: str | None\n\n    def __init__(\n        self,\n        *,\n        api_key: str | None = None,\n        organization: str | None = None,\n        base_url: str | httpx.URL | None = None,\n        timeout: Union[float, Timeout, None, NotGiven] = NOT_GIVEN,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        default_headers: Mapping[str, str] | None = None,\n        default_query: Mapping[str, object] | None = None,\n        # Configure a custom httpx client. See the [httpx documentation](https://www.python-httpx.org/api/#client) for more details.\n        http_client: httpx.Client | None = None,\n        # Enable or disable schema validation for data returned by the API.\n        # When enabled an error APIResponseValidationError is raised\n        # if the API responds with invalid data for the expected schema.\n        #\n        # This parameter may be removed or changed in the future.\n        # If you rely on this feature, please open a GitHub issue\n        # outlining your use-case to help us decide if it should be\n        # part of our public interface in the future.\n        _strict_response_validation: bool = False,\n    ) -> None:\n        \"\"\"Construct a new synchronous openai client instance.\n\n        This automatically infers the following arguments from their corresponding environment variables if they are not provided:\n        - `api_key` from `OPENAI_API_KEY`\n        - `organization` from `OPENAI_ORG_ID`\n        \"\"\"\n        if api_key is None:\n            api_key = os.environ.get(\"OPENAI_API_KEY\")\n        if api_key is None:\n            raise OpenAIError(\n                \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\"\n            )\n        self.api_key = api_key\n\n        if organization is None:\n            organization = os.environ.get(\"OPENAI_ORG_ID\")\n        self.organization = organization\n\n        if base_url is None:\n            base_url = os.environ.get(\"OPENAI_BASE_URL\")\n        if base_url is None:\n            base_url = f\"https://api.openai.com/v1\"\n\n        super().__init__(\n            version=__version__,\n            base_url=base_url,\n            max_retries=max_retries,\n            timeout=timeout,\n            http_client=http_client,\n            custom_headers=default_headers,\n            custom_query=default_query,\n            _strict_response_validation=_strict_response_validation,\n        )\n\n        self._default_stream_cls = Stream\n\n        self.completions = resources.Completions(self)\n        self.chat = resources.Chat(self)\n        self.embeddings = resources.Embeddings(self)\n        self.files = resources.Files(self)\n...\n# Path: src/openai/_streaming.py\n# Note: initially copied from https://github.com/florimondmanca/httpx-sse/blob/master/src/httpx_sse/_decoders.py\nfrom __future__ import annotations\n\nimport json\nimport inspect\nfrom types import TracebackType\nfrom typing import TYPE_CHECKING, Any, Generic, TypeVar, Iterator, AsyncIterator, cast\nfrom typing_extensions import Self, TypeGuard, override, get_origin\n\nimport httpx\n\nfrom ._utils import is_mapping, extract_type_var_from_base\nfrom ._exceptions import APIError\n\nif TYPE_CHECKING:\n    from ._client import OpenAI, AsyncOpenAI\n\n\n_T = TypeVar(\"_T\")\n\n\nclass Stream(Generic[_T]):\n    \"\"\"Provides the core interface to iterate over a synchronous stream response.\"\"\"\n\n    response: httpx.Response\n\n    def __init__(\n        self,\n        *,\n        cast_to: type[_T],\n        response: httpx.Response,\n        client: OpenAI,\n    ) -> None:\n        self.response = response\n        self._cast_to = cast_to\n        self._client = client\n        self._decoder = SSEDecoder()\n        self._iterator = self.__stream__()\n\n    def __next__(self) -> _T:\n        return self._iterator.__next__()\n\n    def __iter__(self) -> Iterator[_T]:\n        for item in self._iterator:\n            yield item\n\n    def _iter_events(self) -> Iterator[ServerSentEvent]:\n        yield from self._decoder.iter(self.response.iter_lines())\n\n    def __stream__(self) -> Iterator[_T]:\n        cast_to = cast(Any, self._cast_to)\n        response = self.response\n        process_data = self._client._process_response_data\n        iterator = self._iter_events()\n\n        for sse in iterator:\n            if sse.data.startswith(\"[DONE]\"):\n                break\n\n            if sse.event is None:\n                data = sse.json()\n                if is_mapping(data) and data.get(\"error\"):\n                    raise APIError(\n                        message=\"An error occurred during streaming\",\n                        request=self.response.request,\n                        body=data[\"error\"],\n                    )\n\n                yield process_data(data=data, cast_to=cast_to, response=response)\n\n        # Ensure the entire stream is consumed\n        for _sse in iterator:\n            ...\n\n    def __enter__(self) -> Self:\n        return self\n\n    def __exit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        self.close()\n\n    def close(self) -> None:\n        \"\"\"\n        Close the response and release the connection.\n\n        Automatically called if the response body is read to completion.\n        \"\"\"\n        self.response.close()\n\n\nclass AsyncStream(Generic[_T]):\n    \"\"\"Provides the core interface to iterate over an asynchronous stream response.\"\"\"\n\n    response: httpx.Response\n\n    def __init__(\n        self,\n        *,\n        cast_to: type[_T],\n        response: httpx.Response,\n        client: AsyncOpenAI,\n    ) -> None:\n        self.response = response\n        self._cast_to = cast_to\n        self._client = client\n        self._decoder = SSEDecoder()\n        self._iterator = self.__stream__()\n\n    async def __anext__(self) -> _T:\n        return await self._iterator.__anext__()\n\n    async def __aiter__(self) -> AsyncIterator[_T]:\n        async for item in self._iterator:\n            yield item\n\n    async def _iter_events(self) -> AsyncIterator[ServerSentEvent]:\n        async for sse in self._decoder.aiter(self.response.aiter_lines()):\n            yield sse\n\n    async def __stream__(self) -> AsyncIterator[_T]:\n        cast_to = cast(Any, self._cast_to)\n        response = self.response\n        process_data = self._client._process_response_data\n        iterator = self._iter_events()\n\n        async for sse in iterator:\n            if sse.data.startswith(\"[DONE]\"):\n                break\n\n            if sse.event is None:\n                data = sse.json()\n                if is_mapping(data) and data.get(\"error\"):\n                    raise APIError(\n                        message=\"An error occurred during streaming\",\n                        request=self.response.request,\n                        body=data[\"error\"],\n                    )\n\n                yield process_data(data=data, cast_to=cast_to, response=response)\n\n        # Ensure the entire stream is consumed\n        async for _sse in iterator:\n            ...\n\n    async def __aenter__(self) -> Self:\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc: BaseException | None,\n        exc_tb: TracebackType | None,\n    ) -> None:\n        await self.close()\n\n    async def close(self) -> None:\n        \"\"\"\n        Close the response and release the connection.\n\n        Automatically called if the response body is read to completion.\n        \"\"\"\n        await self.response.aclose()\n\n\nclass ServerSentEvent:\n    def __init__(\n        self,\n        *,\n        event: str | None = None,\n        data: str | None = None,\n        id: str | None = None,\n        retry: int | None = None,\n    ) -> None:\n        if data is None:\n            data = \"\"\n\n        self._id = id\n        self._data = data\n        self._event = event or None\n        self._retry = retry\n\n    @property\n    def event(self) -> str | None:\n        return self._event\n\n    @property\n    def id(self) -> str | None:\n        return self._id\n\n    @property\n    def retry(self) -> int | None:\n        return self._retry\n\n    @property\n    def data(self) -> str:\n        return self._data\n\n    def json(self) -> Any:\n        return json.loads(self.data)\n\n    @override\n    def __repr__(self) -> str:\n        return f\"ServerSentEvent(event={self.event}, data={self.data}, id={self.id}, retry={self.retry})\"\n\n\nclass SSEDecoder:\n    _data: list[str]\n    _event: str | None\n    _retry: int | None\n    _last_event_id: str | None\n\n    def __init__(self) -> None:\n        self._event = None\n        self._data = []\n        self._last_event_id = None\n        self._retry = None\n\n    def iter(self, iterator: Iterator[str]) -> Iterator[ServerSentEvent]:\n        \"\"\"Given an iterator that yields lines, iterate over it & yield every event encountered\"\"\"\n        for line in iterator:\n            line = line.rstrip(\"\\n\")\n            sse = self.decode(line)\n            if sse is not None:\n                yield sse\n\n    async def aiter(self, iterator: AsyncIterator[str]) -> AsyncIterator[ServerSentEvent]:\n        \"\"\"Given an async iterator that yields lines, iterate over it & yield every event encountered\"\"\"\n        async for line in iterator:\n            line = line.rstrip(\"\\n\")\n            sse = self.decode(line)\n            if sse is not None:\n                yield sse\n\n    def decode(self, line: str) -> ServerSentEvent | None:\n        # See: https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation  # noqa: E501\n\n        if not line:\n            if not self._event and not self._data and not self._last_event_id and self._retry is None:\n                return None\n\n            sse = ServerSentEvent(\n                event=self._event,\n                data=\"\\n\".join(self._data),\n                id=self._last_event_id,\n                retry=self._retry,\n            )\n\n            # NOTE: as per the SSE spec, do not reset last_event_id.\n            self._event = None\n            self._data = []\n            self._retry = None\n\n            return sse\n\n        if line.startswith(\":\"):\n            return None\n\n        fieldname, _, value = line.partition(\":\")\n\n        if value.startswith(\" \"):\n            value = value[1:]\n\n        if fieldname == \"event\":\n            self._event = value\n        elif fieldname == \"data\":\n            self._data.append(value)\n        elif fieldname == \"id\":\n            if \"\\0\" in value:\n                pass\n            else:\n                self._last_event_id = value\n        elif fieldname == \"retry\":\n            try:\n                self._retry = int(value)\n            except (TypeError, ValueError):\n                pass\n        else:\n            pass  # Field is ignored.\n\n        return None\n\n\ndef is_stream_class_type(typ: type) -> TypeGuard[type[Stream[object]] | type[AsyncStream[object]]]:\n    \"\"\"TypeGuard for determining whether or not the given type is a subclass of `Stream` / `AsyncStream`\"\"\"\n    origin = get_origin(typ) or typ\n    return inspect.isclass(origin) and issubclass(origin, (Stream, AsyncStream))\n\n\ndef extract_stream_chunk_type(\n    stream_cls: type,\n    *,\n    failure_message: str | None = None,\n) -> type:\n    \"\"\"Given a type like `Stream[T]`, returns the generic type variable `T`.\n\n    This also handles the case where a concrete subclass is given, e.g.\n    ```py\n    class MyStream(Stream[bytes]):\n        ...\n\n    extract_stream_chunk_type(MyStream) -> bytes\n    ```\n    \"\"\"\n    from ._base_client import Stream, AsyncStream\n\n    return extract_type_var_from_base(\n        stream_cls,\n        index=0,\n        generic_bases=cast(\"tuple[type, ...]\", (Stream, AsyncStream)),\n        failure_message=failure_message,\n    )\n\n# Path: src/openai/_legacy_response.py\nfrom __future__ import annotations\n\nimport os\nimport inspect\nimport logging\nimport datetime\nimport functools\nfrom typing import TYPE_CHECKING, Any, Union, Generic, TypeVar, Callable, Iterator, AsyncIterator, cast, overload\nfrom typing_extensions import Awaitable, ParamSpec, override, deprecated, get_origin\n\nimport anyio\nimport httpx\nimport pydantic\n\nfrom ._types import NoneType\nfrom ._utils import is_given\nfrom ._models import BaseModel, is_basemodel\nfrom ._constants import RAW_RESPONSE_HEADER\nfrom ._streaming import Stream, AsyncStream, is_stream_class_type, extract_stream_chunk_type\nfrom ._exceptions import APIResponseValidationError\n\nif TYPE_CHECKING:\n    from ._models import FinalRequestOptions\n    from ._base_client import BaseClient\n\n\nP = ParamSpec(\"P\")\nR = TypeVar(\"R\")\n_T = TypeVar(\"_T\")\n\nlog: logging.Logger = logging.getLogger(__name__)\n\n\nclass LegacyAPIResponse(Generic[R]):\n    \"\"\"This is a legacy class as it will be replaced by `APIResponse`\n    and `AsyncAPIResponse` in the `_response.py` file in the next major\n    release.\n\n    For the sync client this will mostly be the same with the exception\n    of `content` & `text` will be methods instead of properties. In the\n    async client, all methods will be async.\n\n    A migration script will be provided & the migration in general should\n    be smooth.\n    \"\"\"\n\n    _cast_to: type[R]\n    _client: BaseClient[Any, Any]\n    _parsed_by_type: dict[type[Any], Any]\n    _stream: bool\n    _stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None\n    _options: FinalRequestOptions\n\n    http_response: httpx.Response\n\n    def __init__(\n        self,\n        *,\n        raw: httpx.Response,\n        cast_to: type[R],\n        client: BaseClient[Any, Any],\n        stream: bool,\n        stream_cls: type[Stream[Any]] | type[AsyncStream[Any]] | None,\n        options: FinalRequestOptions,\n    ) -> None:\n        self._cast_to = cast_to\n        self._client = client\n        self._parsed_by_type = {}\n        self._stream = stream\n        self._stream_cls = stream_cls\n        self._options = options\n        self.http_response = raw\n\n    @overload\n    def parse(self, *, to: type[_T]) -> _T:\n        ...\n\n    @overload\n    def parse(self) -> R:\n        ...\n\n    def parse(self, *, to: type[_T] | None = None) -> R | _T:\n        \"\"\"Returns the rich python representation of this response's data.\n\n        NOTE: For the async client: this will become a coroutine in the next major version.\n\n        For lower-level control, see `.read()`, `.json()`, `.iter_bytes()`.\n\n        You can customise the type that the response is parsed into through\n        the `to` argument, e.g.\n\n        ```py\n        from openai import BaseModel\n\n\n        class MyModel(BaseModel):\n            foo: str\n\n\n        obj = response.parse(to=MyModel)\n        print(obj.foo)\n        ```\n\n        We support parsing:\n          - `BaseModel`\n          - `dict`\n          - `list`\n          - `Union`\n          - `str`\n          - `httpx.Response`\n        \"\"\"\n        cache_key = to if to is not None else self._cast_to\n        cached = self._parsed_by_type.get(cache_key)\n        if cached is not None:\n            return cached  # type: ignore[no-any-return]\n\n        parsed = self._parse(to=to)\n        if is_given(self._options.post_parser):\n            parsed = self._options.post_parser(parsed)\n\n        self._parsed_by_type[cache_key] = parsed\n        return parsed\n\n    @property\n    def headers(self) -> httpx.Headers:\n        return self.http_response.headers\n\n    @property\n    def http_request(self) -> httpx.Request:\n        return self.http_response.request\n\n    @property\n    def status_code(self) -> int:\n        return self.http_response.status_code\n\n    @property\n    def url(self) -> httpx.URL:\n        return self.http_response.url\n\n    @property\n    def method(self) -> str:\n        return self.http_request.method\n\n    @property\n    def content(self) -> bytes:\n        \"\"\"Return the binary response content.\n\n        NOTE: this will be removed in favour of `.read()` in the\n        next major version.\n        \"\"\"\n        return self.http_response.content\n\n    @property\n    def text(self) -> str:\n        \"\"\"Return the decoded response content.\n\n        NOTE: this will be turned into a method in the next major version.\n        \"\"\"\n        return self.http_response.text\n\n    @property\n    def http_version(self) -> str:\n        return self.http_response.http_version\n\n    @property\n    def is_closed(self) -> bool:\n        return self.http_response.is_closed\n\n    @property\n    def elapsed(self) -> datetime.timedelta:\n        \"\"\"The time taken for the complete request/response cycle to complete.\"\"\"\n        return self.http_response.elapsed\n\n    def _parse(self, *, to: type[_T] | None = None) -> R | _T:\n        if self._stream:\n            if to:\n                if not is_stream_class_type(to):\n                    raise TypeError(f\"Expected custom parse type to be a subclass of {Stream} or {AsyncStream}\")\n\n                return cast(\n                    _T,\n                    to(\n                        cast_to=extract_stream_chunk_type(\n                            to,\n                            failure_message=\"Expected custom stream type to be passed with a type argument, e.g. Stream[ChunkType]\",\n                        ),\n                        response=self.http_response,\n                        client=cast(Any, self._client),\n                    ),\n                )\n\n            if self._stream_cls:\n                return cast(\n                    R,\n                    self._stream_cls(\n                        cast_to=extract_stream_chunk_type(self._stream_cls),\n                        response=self.http_response,\n                        client=cast(Any, self._client),\n                    ),\n                )\n\n            stream_cls = cast(\"type[Stream[Any]] | type[AsyncStream[Any]] | None\", self._client._default_stream_cls)\n            if stream_cls is None:\n                raise MissingStreamClassError()\n\n            return cast(\n                R,\n                stream_cls(\n                    cast_to=self._cast_to,\n                    response=self.http_response,\n                    client=cast(Any, self._client),\n                ),\n            )\n\n        cast_to = to if to is not None else self._cast_to\n        if cast_to is NoneType:\n            return cast(R, None)\n\n        response = self.http_response\n        if cast_to == str:\n            return cast(R, response.text)\n\n        origin = get_origin(cast_to) or cast_to\n\n        if inspect.isclass(origin) and issubclass(origin, HttpxBinaryResponseContent):\n            return cast(R, cast_to(response))  # type: ignore\n\n        if origin == LegacyAPIResponse:\n            raise RuntimeError(\"Unexpected state - cast_to is `APIResponse`\")\n\n        if inspect.isclass(origin) and issubclass(origin, httpx.Response):\n            # Because of the invariance of our ResponseT TypeVar, users can subclass httpx.Response\n            # and pass that class to our request functions. We cannot change the variance to be either\n            # covariant or contravariant as that makes our usage of ResponseT illegal. We could construct\n            # the response class ourselves but that is something that should be supported directly in httpx\n            # as it would be easy to incorrectly construct the Response object due to the multitude of arguments.\n            if cast_to != httpx.Response:\n                raise ValueError(f\"Subclasses of httpx.Response cannot be passed to `cast_to`\")\n            return cast(R, response)\n\n        if inspect.isclass(origin) and not issubclass(origin, BaseModel) and issubclass(origin, pydantic.BaseModel):\n            raise TypeError(\"Pydantic models must subclass our base model type, e.g. `from openai import BaseModel`\")\n\n        if (\n            cast_to is not object\n            and not origin is list\n            and not origin is dict\n            and not origin is Union\n            and not issubclass(origin, BaseModel)\n        ):\n            raise RuntimeError(\n                f\"Unsupported type, expected {cast_to} to be a subclass of {BaseModel}, {dict}, {list}, {Union}, {NoneType}, {str} or {httpx.Response}.\"\n            )\n\n        # split is required to handle cases where additional information is included\n        # in the response, e.g. application/json; charset=utf-8\n        content_type, *_ = response.headers.get(\"content-type\", \"*\").split(\";\")\n        if content_type != \"application/json\":\n            if is_basemodel(cast_to):\n                try:\n                    data = response.json()\n                except Exception as exc:\n                    log.debug(\"Could not read JSON from response data due to %s - %s\", type(exc), exc)\n                else:\n                    return self._client._process_response_data(\n                        data=data,\n                        cast_to=cast_to,  # type: ignore\n                        response=response,\n                    )\n\n            if self._client._strict_response_validation:\n                raise APIResponseValidationError(\n                    response=response,\n                    message=f\"Expected Content-Type response header to be `application/json` but received `{content_type}` instead.\",\n                    body=response.text,\n                )\n\n            # If the API responds with content that isn't JSON then we just return\n            # the (decoded) text without performing any parsing so that you can still\n            # handle the response however you need to.\n            return response.text  # type: ignore\n\n        data = response.json()\n\n        return self._client._process_response_data(\n            data=data,\n            cast_to=cast_to,  # type: ignore\n            response=response,\n        )\n\n    @override\n    def __repr__(self) -> str:\n        return f\"<APIResponse [{self.status_code} {self.http_response.reason_phrase}] type={self._cast_to}>\"\n\n\nclass MissingStreamClassError(TypeError):\n    def __init__(self) -> None:\n        super().__init__(\n            \"The `stream` argument was set to `True` but the `stream_cls` argument was not given. See `openai._streaming` for reference\",\n        )\n\n\ndef to_raw_response_wrapper(func: Callable[P, R]) -> Callable[P, LegacyAPIResponse[R]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    def wrapped(*args: P.args, **kwargs: P.kwargs) -> LegacyAPIResponse[R]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"true\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(LegacyAPIResponse[R], func(*args, **kwargs))\n\n    return wrapped\n\n\ndef async_to_raw_response_wrapper(func: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[LegacyAPIResponse[R]]]:\n    \"\"\"Higher order function that takes one of our bound API methods and wraps it\n    to support returning the raw `APIResponse` object directly.\n    \"\"\"\n\n    @functools.wraps(func)\n    async def wrapped(*args: P.args, **kwargs: P.kwargs) -> LegacyAPIResponse[R]:\n        extra_headers = {**(cast(Any, kwargs.get(\"extra_headers\")) or {})}\n        extra_headers[RAW_RESPONSE_HEADER] = \"true\"\n\n        kwargs[\"extra_headers\"] = extra_headers\n\n        return cast(LegacyAPIResponse[R], await func(*args, **kwargs))\n\n    return wrapped\n\n\nclass HttpxBinaryResponseContent:\n    response: httpx.Response\n\n    def __init__(self, response: httpx.Response) -> None:\n        self.response = response\n\n    @property\n    def content(self) -> bytes:\n        return self.response.content\n\n    @property\n    def text(self) -> str:\n        return self.response.text\n\n    @property\n    def encoding(self) -> str | None:\n        return self.response.encoding\n\n    @property\n    def charset_encoding(self) -> str | None:\n        return self.response.charset_encoding\n\n    def json(self, **kwargs: Any) -> Any:\n        return self.response.json(**kwargs)\n\n    def read(self) -> bytes:\n        return self.response.read()\n\n    def iter_bytes(self, chunk_size: int | None = None) -> Iterator[bytes]:\n        return self.response.iter_bytes(chunk_size)\n\n    def iter_text(self, chunk_size: int | None = None) -> Iterator[str]:\n        return self.response.iter_text(chunk_size)\n\n    def iter_lines(self) -> Iterator[str]:\n        return self.response.iter_lines()\n\n    def iter_raw(self, chunk_size: int | None = None) -> Iterator[bytes]:\n        return self.response.iter_raw(chunk_size)\n\n    def write_to_file(\n        self,\n        file: str | os.PathLike[str],\n    ) -> None:\n        \"\"\"Write the output to the given file.\n\n        Accepts a filename or any path-like object, e.g. pathlib.Path\n\n        Note: if you want to stream the data to the file instead of writing\n        all at once then you should use `.with_streaming_response` when making\n        the API request, e.g. `client.with_streaming_response.foo().stream_to_file('my_filename.txt')`\n        \"\"\"\n        with open(file, mode=\"wb\") as f:\n            for data in self.response.iter_bytes():\n                f.write(data)\n\n    @deprecated(\n        \"Due to a bug, this method doesn't actually stream the response content, `.with_streaming_response.method()` should be used instead\"\n    )\n    def stream_to_file(\n        self,\n        file: str | os.PathLike[str],\n        *,\n        chunk_size: int | None = None,\n    ) -> None:\n        with open(file, mode=\"wb\") as f:\n            for data in self.response.iter_bytes(chunk_size):\n                f.write(data)\n\n    def close(self) -> None:\n        return self.response.close()\n\n    async def aread(self) -> bytes:\n        return await self.response.aread()\n\n    async def aiter_bytes(self, chunk_size: int | None = None) -> AsyncIterator[bytes]:\n        return self.response.aiter_bytes(chunk_size)\n\n    async def aiter_text(self, chunk_size: int | None = None) -> AsyncIterator[str]:\n        return self.response.aiter_text(chunk_size)\n\n    async def aiter_lines(self) -> AsyncIterator[str]:\n        return self.response.aiter_lines()\n\n    async def aiter_raw(self, chunk_size: int | None = None) -> AsyncIterator[bytes]:\n        return self.response.aiter_raw(chunk_size)\n\n    @deprecated(\n        \"Due to a bug, this method doesn't actually stream the response content, `.with_streaming_response.method()` should be used instead\"\n    )\n    async def astream_to_file(\n        self,\n        file: str | os.PathLike[str],\n        *,\n        chunk_size: int | None = None,\n    ) -> None:\n        path = anyio.Path(file)\n        async with await path.open(mode=\"wb\") as f:\n            async for data in self.response.aiter_bytes(chunk_size):\n                await f.write(data)\n\n    async def aclose(self) -> None:\n        return await self.response.aclose()\n\n# Path: src/openai/_types.py\nfrom __future__ import annotations\n\nfrom os import PathLike\nfrom typing import (\n    IO,\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    List,\n    Type,\n    Tuple,\n    Union,\n    Mapping,\n    TypeVar,\n    Callable,\n    Optional,\n    Sequence,\n)\nfrom typing_extensions import Literal, Protocol, TypeAlias, TypedDict, override, runtime_checkable\n\nimport httpx\nimport pydantic\nfrom httpx import URL, Proxy, Timeout, Response, BaseTransport, AsyncBaseTransport\n\nif TYPE_CHECKING:\n    from ._models import BaseModel\n    from ._response import APIResponse, AsyncAPIResponse\n    from ._legacy_response import HttpxBinaryResponseContent\n\nTransport = BaseTransport\nAsyncTransport = AsyncBaseTransport\nQuery = Mapping[str, object]\nBody = object\nAnyMapping = Mapping[str, object]\nModelT = TypeVar(\"ModelT\", bound=pydantic.BaseModel)\n_T = TypeVar(\"_T\")\n\n\n# Approximates httpx internal ProxiesTypes and RequestFiles types\n# while adding support for `PathLike` instances\nProxiesDict = Dict[\"str | URL\", Union[None, str, URL, Proxy]]\nProxiesTypes = Union[str, Proxy, ProxiesDict]\nif TYPE_CHECKING:\n    FileContent = Union[IO[bytes], bytes, PathLike[str]]\nelse:\n    FileContent = Union[IO[bytes], bytes, PathLike]  # PathLike is not subscriptable in Python 3.8.\nFileTypes = Union[\n    # file (or bytes)\n    FileContent,\n    # (filename, file (or bytes))\n    Tuple[Optional[str], FileContent],\n    # (filename, file (or bytes), content_type)\n    Tuple[Optional[str], FileContent, Optional[str]],\n    # (filename, file (or bytes), content_type, headers)\n    Tuple[Optional[str], FileContent, Optional[str], Mapping[str, str]],\n]\nRequestFiles = Union[Mapping[str, FileTypes], Sequence[Tuple[str, FileTypes]]]\n\n# duplicate of the above but without our custom file support\nHttpxFileContent = Union[IO[bytes], bytes]\nHttpxFileTypes = Union[\n    # file (or bytes)\n    HttpxFileContent,\n    # (filename, file (or bytes))\n    Tuple[Optional[str], HttpxFileContent],\n    # (filename, file (or bytes), content_type)\n    Tuple[Optional[str], HttpxFileContent, Optional[str]],\n    # (filename, file (or bytes), content_type, headers)\n    Tuple[Optional[str], HttpxFileContent, Optional[str], Mapping[str, str]],\n]\nHttpxRequestFiles = Union[Mapping[str, HttpxFileTypes], Sequence[Tuple[str, HttpxFileTypes]]]\n\n# Workaround to support (cast_to: Type[ResponseT]) -> ResponseT\n# where ResponseT includes `None`. In order to support directly\n# passing `None`, overloads would have to be defined for every\n# method that uses `ResponseT` which would lead to an unacceptable\n# amount of code duplication and make it unreadable. See _base_client.py\n# for example usage.\n#\n# This unfortunately means that you will either have\n# to import this type and pass it explicitly:\n#\n# from openai import NoneType\n# client.get('/foo', cast_to=NoneType)\n#\n# or build it yourself:\n#\n# client.get('/foo', cast_to=type(None))\nif TYPE_CHECKING:\n    NoneType: Type[None]\nelse:\n    NoneType = type(None)\n\n\nclass RequestOptions(TypedDict, total=False):\n    headers: Headers\n    max_retries: int\n    timeout: float | Timeout | None\n    params: Query\n    extra_json: AnyMapping\n    idempotency_key: str\n\n\n# Sentinel class used until PEP 0661 is accepted\nclass NotGiven:\n    \"\"\"\n    A sentinel singleton class used to distinguish omitted keyword arguments\n    from those passed in with the value None (which may have different behavior).\n\n    For example:\n\n    ```py\n    def get(timeout: Union[int, NotGiven, None] = NotGiven()) -> Response:\n        ...\n\n\n    get(timeout=1)  # 1s timeout\n    get(timeout=None)  # No timeout\n    get()  # Default timeout behavior, which may not be statically known at the method definition.\n    ```\n    \"\"\"\n\n    def __bool__(self) -> Literal[False]:\n        return False\n\n    @override\n    def __repr__(self) -> str:\n        return \"NOT_GIVEN\"\n\n\nNotGivenOr = Union[_T, NotGiven]\nNOT_GIVEN = NotGiven()\n\n\nclass Omit:\n    \"\"\"In certain situations you need to be able to represent a case where a default value has\n    to be explicitly removed and `None` is not an appropriate substitute, for example:\n\n    ```py\n    # as the default `Content-Type` header is `application/json` that will be sent\n    client.post(\"/upload/files\", files={\"file\": b\"my raw file content\"})\n\n    # you can't explicitly override the header as it has to be dynamically generated\n    # to look something like: 'multipart/form-data; boundary=0d8382fcf5f8c3be01ca2e11002d2983'\n    client.post(..., headers={\"Content-Type\": \"multipart/form-data\"})\n\n    # instead you can remove the default `application/json` header by passing Omit\n    client.post(..., headers={\"Content-Type\": Omit()})\n    ```\n    \"\"\"\n\n    def __bool__(self) -> Literal[False]:\n        return False\n\n\n@runtime_checkable\nclass ModelBuilderProtocol(Protocol):\n    @classmethod\n    def build(\n        cls: type[_T],\n        *,\n        response: Response,\n        data: object,\n    ) -> _T:\n        ...\n\n\nHeaders = Mapping[str, Union[str, Omit]]\n\n\nclass HeadersLikeProtocol(Protocol):\n    def get(self, __key: str) -> str | None:\n        ...\n\n\nHeadersLike = Union[Headers, HeadersLikeProtocol]\n\nResponseT = TypeVar(\n    \"ResponseT\",\n    bound=Union[\n        object,\n        str,\n        None,\n        \"BaseModel\",\n        List[Any],\n        Dict[str, Any],\n        Response,\n        ModelBuilderProtocol,\n        \"APIResponse[Any]\",\n        \"AsyncAPIResponse[Any]\",\n        \"HttpxBinaryResponseContent\",\n    ],\n)\n\nStrBytesIntFloat = Union[str, bytes, int, float]\n\n# Note: copied from Pydantic\n# https://github.com/pydantic/pydantic/blob/32ea570bf96e84234d2992e1ddf40ab8a565925a/pydantic/main.py#L49\nIncEx: TypeAlias = \"set[int] | set[str] | dict[int, Any] | dict[str, Any] | None\"\n\nPostParser = Callable[[Any], Any]\n\n\n@runtime_checkable\nclass InheritsGeneric(Protocol):\n    \"\"\"Represents a type that has inherited from `Generic`\n\n    The `__orig_bases__` property can be used to determine the resolved\n    type variable for a given base class.\n    \"\"\"\n\n    __orig_bases__: tuple[_GenericAlias]\n\n\nclass _GenericAlias(Protocol):\n    __origin__: type[object]\n\n\nclass HttpxSendArgs(TypedDict, total=False):\n    auth: httpx.Auth\n\n# Path: src/openai/_compat.py\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any, Union, Generic, TypeVar, Callable, cast, overload\nfrom datetime import date, datetime\nfrom typing_extensions import Self\n\nimport pydantic\nfrom pydantic.fields import FieldInfo\n\nfrom ._types import StrBytesIntFloat\n\n_T = TypeVar(\"_T\")\n_ModelT = TypeVar(\"_ModelT\", bound=pydantic.BaseModel)\n\n# --------------- Pydantic v2 compatibility ---------------\n\n# Pyright incorrectly reports some of our functions as overriding a method when they don't\n# pyright: reportIncompatibleMethodOverride=false\n\nPYDANTIC_V2 = pydantic.VERSION.startswith(\"2.\")\n\n# v1 re-exports\nif TYPE_CHECKING:\n\n    def parse_date(value: date | StrBytesIntFloat) -> date:  # noqa: ARG001\n        ...\n\n    def parse_datetime(value: Union[datetime, StrBytesIntFloat]) -> datetime:  # noqa: ARG001\n        ...\n\n    def get_args(t: type[Any]) -> tuple[Any, ...]:  # noqa: ARG001\n        ...\n\n    def is_union(tp: type[Any] | None) -> bool:  # noqa: ARG001\n        ...\n\n    def get_origin(t: type[Any]) -> type[Any] | None:  # noqa: ARG001\n        ...\n\n    def is_literal_type(type_: type[Any]) -> bool:  # noqa: ARG001\n        ...\n\n    def is_typeddict(type_: type[Any]) -> bool:  # noqa: ARG001\n        ...\n\nelse:\n    if PYDANTIC_V2:\n        from pydantic.v1.typing import (\n            get_args as get_args,\n            is_union as is_union,\n            get_origin as get_origin,\n            is_typeddict as is_typeddict,\n            is_literal_type as is_literal_type,\n        )\n        from pydantic.v1.datetime_parse import parse_date as parse_date, parse_datetime as parse_datetime\n    else:\n        from pydantic.typing import (\n            get_args as get_args,\n            is_union as is_union,\n            get_origin as get_origin,\n            is_typeddict as is_typeddict,\n            is_literal_type as is_literal_type,\n        )\n        from pydantic.datetime_parse import parse_date as parse_date, parse_datetime as parse_datetime\n\n\n# refactored config\nif TYPE_CHECKING:\n    from pydantic import ConfigDict as ConfigDict\nelse:\n    if PYDANTIC_V2:\n        from pydantic import ConfigDict\n    else:\n        # TODO: provide an error message here?\n        ConfigDict = None\n\n\n# renamed methods / properties\ndef parse_obj(model: type[_ModelT], value: object) -> _ModelT:\n    if PYDANTIC_V2:\n        return model.model_validate(value)\n    else:\n        return cast(_ModelT, model.parse_obj(value))  # pyright: ignore[reportDeprecated, reportUnnecessaryCast]\n\n\ndef field_is_required(field: FieldInfo) -> bool:\n    if PYDANTIC_V2:\n        return field.is_required()\n    return field.required  # type: ignore\n\n\ndef field_get_default(field: FieldInfo) -> Any:\n    value = field.get_default()\n    if PYDANTIC_V2:\n        from pydantic_core import PydanticUndefined\n\n        if value == PydanticUndefined:\n            return None\n        return value\n    return value\n\n\ndef field_outer_type(field: FieldInfo) -> Any:\n    if PYDANTIC_V2:\n        return field.annotation\n    return field.outer_type_  # type: ignore\n\n\ndef get_model_config(model: type[pydantic.BaseModel]) -> Any:\n    if PYDANTIC_V2:\n        return model.model_config\n    return model.__config__  # type: ignore\n\n\ndef get_model_fields(model: type[pydantic.BaseModel]) -> dict[str, FieldInfo]:\n    if PYDANTIC_V2:\n        return model.model_fields\n    return model.__fields__  # type: ignore\n\n\ndef model_copy(model: _ModelT) -> _ModelT:\n    if PYDANTIC_V2:\n        return model.model_copy()\n    return model.copy()  # type: ignore\n\n\ndef model_json(model: pydantic.BaseModel, *, indent: int | None = None) -> str:\n    if PYDANTIC_V2:\n        return model.model_dump_json(indent=indent)\n    return model.json(indent=indent)  # type: ignore\n\n\ndef model_dump(\n    model: pydantic.BaseModel,\n    *,\n    exclude_unset: bool = False,\n    exclude_defaults: bool = False,\n) -> dict[str, Any]:\n    if PYDANTIC_V2:\n        return model.model_dump(\n            exclude_unset=exclude_unset,\n            exclude_defaults=exclude_defaults,\n        )\n    return cast(\n        \"dict[str, Any]\",\n        model.dict(  # pyright: ignore[reportDeprecated, reportUnnecessaryCast]\n            exclude_unset=exclude_unset,\n            exclude_defaults=exclude_defaults,\n        ),\n    )\n\n\ndef model_parse(model: type[_ModelT], data: Any) -> _ModelT:\n    if PYDANTIC_V2:\n        return model.model_validate(data)\n    return model.parse_obj(data)  # pyright: ignore[reportDeprecated]\n\n\n# generic models\nif TYPE_CHECKING:\n\n    class GenericModel(pydantic.BaseModel):\n        ...\n\nelse:\n    if PYDANTIC_V2:\n        # there no longer needs to be a distinction in v2 but\n        # we still have to create our own subclass to avoid\n        # inconsistent MRO ordering errors\n        class GenericModel(pydantic.BaseModel):\n            ...\n\n    else:\n        import pydantic.generics\n\n        class GenericModel(pydantic.generics.GenericModel, pydantic.BaseModel):\n            ...\n\n\n# cached properties\nif TYPE_CHECKING:\n    cached_property = property\n\n    # we define a separate type (copied from typeshed)\n    # that represents that `cached_property` is `set`able\n    # at runtime, which differs from `@property`.\n    #\n    # this is a separate type as editors likely special case\n    # `@property` and we don't want to cause issues just to have\n    # more helpful internal types.\n\n    class typed_cached_property(Generic[_T]):\n        func: Callable[[Any], _T]\n        attrname: str | None\n\n        def __init__(self, func: Callable[[Any], _T]) -> None:\n            ...\n\n        @overload\n        def __get__(self, instance: None, owner: type[Any] | None = None) -> Self:\n            ...\n\n        @overload\n        def __get__(self, instance: object, owner: type[Any] | None = None) -> _T:\n            ...\n\n        def __get__(self, instance: object, owner: type[Any] | None = None) -> _T | Self:\n            raise NotImplementedError()\n\n        def __set_name__(self, owner: type[Any], name: str) -> None:\n            ...\n\n        # __set__ is not defined at runtime, but @cached_property is designed to be settable\n        def __set__(self, instance: object, value: _T) -> None:\n            ...\nelse:\n    try:\n        from functools import cached_property as cached_property\n    except ImportError:\n        from cached_property import cached_property as cached_property\n\n    typed_cached_property = cached_property\n\n# Path: src/openai/_base_client.py\nfrom __future__ import annotations\n\nimport json\nimport time\nimport uuid\nimport email\nimport asyncio\nimport inspect\nimport logging\nimport platform\nimport warnings\nimport email.utils\nfrom types import TracebackType\nfrom random import random\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Type,\n    Union,\n    Generic,\n    Mapping,\n    TypeVar,\n    Iterable,\n    Iterator,\n    Optional,\n    Generator,\n    AsyncIterator,\n    cast,\n    overload,\n)\nfrom functools import lru_cache\nfrom typing_extensions import Literal, override, get_origin\n\nimport anyio\nimport httpx\nimport distro\nimport pydantic\nfrom httpx import URL, Limits\nfrom pydantic import PrivateAttr\n\nfrom . import _exceptions\nfrom ._qs import Querystring\nfrom ._files import to_httpx_files, async_to_httpx_files\nfrom ._types import (\n    NOT_GIVEN,\n    Body,\n    Omit,\n    Query,\n    Headers,\n    Timeout,\n    NotGiven,\n    ResponseT,\n    Transport,\n    AnyMapping,\n    PostParser,\n    ProxiesTypes,\n    RequestFiles,\n    HttpxSendArgs,\n    AsyncTransport,\n    RequestOptions,\n    ModelBuilderProtocol,\n)\nfrom ._utils import is_dict, is_list, is_given, is_mapping\nfrom ._compat import model_copy, model_dump\nfrom ._models import GenericModel, FinalRequestOptions, validate_type, construct_type\nfrom ._response import (\n    APIResponse,\n    BaseAPIResponse,\n    AsyncAPIResponse,\n    extract_response_type,\n)\nfrom ._constants import (\n    DEFAULT_LIMITS,\n    DEFAULT_TIMEOUT,\n    MAX_RETRY_DELAY,\n    DEFAULT_MAX_RETRIES,\n    INITIAL_RETRY_DELAY,\n    RAW_RESPONSE_HEADER,\n    OVERRIDE_CAST_TO_HEADER,\n)\nfrom ._streaming import Stream, AsyncStream\nfrom ._exceptions import (\n    APIStatusError,\n    APITimeoutError,\n    APIConnectionError,\n    APIResponseValidationError,\n)\nfrom ._legacy_response import LegacyAPIResponse\n\nlog: logging.Logger = logging.getLogger(__name__)\n\n# TODO: make base page type vars covariant\nSyncPageT = TypeVar(\"SyncPageT\", bound=\"BaseSyncPage[Any]\")\nAsyncPageT = TypeVar(\"AsyncPageT\", bound=\"BaseAsyncPage[Any]\")\n\n\n_T = TypeVar(\"_T\")\n_T_co = TypeVar(\"_T_co\", covariant=True)\n\n_StreamT = TypeVar(\"_StreamT\", bound=Stream[Any])\n_AsyncStreamT = TypeVar(\"_AsyncStreamT\", bound=AsyncStream[Any])\n\nif TYPE_CHECKING:\n    from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT\nelse:\n    try:\n        from httpx._config import DEFAULT_TIMEOUT_CONFIG as HTTPX_DEFAULT_TIMEOUT\n    except ImportError:\n        # taken from https://github.com/encode/httpx/blob/3ba5fe0d7ac70222590e759c31442b1cab263791/httpx/_config.py#L366\n        HTTPX_DEFAULT_TIMEOUT = Timeout(5.0)\n\n\nclass PageInfo:\n    \"\"\"Stores the necessary information to build the request to retrieve the next page.\n\n    Either `url` or `params` must be set.\n    \"\"\"\n\n    url: URL | NotGiven\n    params: Query | NotGiven\n\n    @overload\n    def __init__(\n        self,\n        *,\n        url: URL,\n    ) -> None:\n        ...\n\n    @overload\n    def __init__(\n        self,\n        *,\n        params: Query,\n    ) -> None:\n        ...\n\n    def __init__(\n        self,\n        *,\n        url: URL | NotGiven = NOT_GIVEN,\n        params: Query | NotGiven = NOT_GIVEN,\n    ) -> None:\n        self.url = url\n        self.params = params\n\n\nclass BasePage(GenericModel, Generic[_T]):\n    \"\"\"\n    Defines the core interface for pagination.\n\n    Type Args:\n        ModelT: The pydantic model that represents an item in the response.\n\n    Methods:\n        has_next_page(): Check if there is another page available\n        next_page_info(): Get the necessary information to make a request for the next page\n    \"\"\"\n\n    _options: FinalRequestOptions = PrivateAttr()\n    _model: Type[_T] = PrivateAttr()\n\n    def has_next_page(self) -> bool:\n        items = self._get_page_items()\n        if not items:\n            return False\n        return self.next_page_info() is not None\n\n    def next_page_info(self) -> Optional[PageInfo]:\n        ...\n\n    def _get_page_items(self) -> Iterable[_T]:  # type: ignore[empty-body]\n        ...\n\n    def _params_from_url(self, url: URL) -> httpx.QueryParams:\n        # TODO: do we have to preprocess params here?\n        return httpx.QueryParams(cast(Any, self._options.params)).merge(url.params)\n\n    def _info_to_options(self, info: PageInfo) -> FinalRequestOptions:\n        options = model_copy(self._options)\n        options._strip_raw_response_header()\n\n        if not isinstance(info.params, NotGiven):\n            options.params = {**options.params, **info.params}\n            return options\n\n        if not isinstance(info.url, NotGiven):\n            params = self._params_from_url(info.url)\n            url = info.url.copy_with(params=params)\n            options.params = dict(url.params)\n            options.url = str(url)\n            return options\n\n        raise ValueError(\"Unexpected PageInfo state\")\n\n\nclass BaseSyncPage(BasePage[_T], Generic[_T]):\n    _client: SyncAPIClient = pydantic.PrivateAttr()\n\n    def _set_private_attributes(\n        self,\n        client: SyncAPIClient,\n        model: Type[_T],\n        options: FinalRequestOptions,\n    ) -> None:\n        self._model = model\n        self._client = client\n        self._options = options\n\n    # Pydantic uses a custom `__iter__` method to support casting BaseModels\n    # to dictionaries. e.g. dict(model).\n    # As we want to support `for item in page`, this is inherently incompatible\n    # with the default pydantic behaviour. It is not possible to support both\n    # use cases at once. Fortunately, this is not a big deal as all other pydantic\n    # methods should continue to work as expected as there is an alternative method\n    # to cast a model to a dictionary, model.dict(), which is used internally\n    # by pydantic.\n    def __iter__(self) -> Iterator[_T]:  # type: ignore\n        for page in self.iter_pages():\n            for item in page._get_page_items():\n                yield item\n\n    def iter_pages(self: SyncPageT) -> Iterator[SyncPageT]:\n        page = self\n        while True:\n            yield page\n            if page.has_next_page():\n                page = page.get_next_page()\n            else:\n                return\n\n    def get_next_page(self: SyncPageT) -> SyncPageT:\n        info = self.next_page_info()\n        if not info:\n            raise RuntimeError(\n                \"No next page expected; please check `.has_next_page()` before calling `.get_next_page()`.\"\n            )\n\n        options = self._info_to_options(info)\n        return self._client._request_api_list(self._model, page=self.__class__, options=options)\n\n\nclass AsyncPaginator(Generic[_T, AsyncPageT]):\n    def __init__(\n        self,\n        client: AsyncAPIClient,\n        options: FinalRequestOptions,\n        page_cls: Type[AsyncPageT],\n        model: Type[_T],\n    ) -> None:\n        self._model = model\n        self._client = client\n        self._options = options\n        self._page_cls = page_cls\n\n    def __await__(self) -> Generator[Any, None, AsyncPageT]:\n        return self._get_page().__await__()\n\n    async def _get_page(self) -> AsyncPageT:\n        def _parser(resp: AsyncPageT) -> AsyncPageT:\n            resp._set_private_attributes(\n                model=self._model,\n                options=self._options,\n                client=self._client,\n            )\n            return resp\n\n        self._options.post_parser = _parser\n\n        return await self._client.request(self._page_cls, self._options)\n\n    async def __aiter__(self) -> AsyncIterator[_T]:\n        # https://github.com/microsoft/pyright/issues/3464\n        page = cast(\n            AsyncPageT,\n            await self,  # type: ignore\n        )\n        async for item in page:\n            yield item\n\n\nclass BaseAsyncPage(BasePage[_T], Generic[_T]):\n    _client: AsyncAPIClient = pydantic.PrivateAttr()\n\n    def _set_private_attributes(\n        self,\n        model: Type[_T],\n        client: AsyncAPIClient,\n        options: FinalRequestOptions,\n    ) -> None:\n        self._model = model\n        self._client = client\n        self._options = options\n\n    async def __aiter__(self) -> AsyncIterator[_T]:\n        async for page in self.iter_pages():\n            for item in page._get_page_items():\n                yield item\n\n    async def iter_pages(self: AsyncPageT) -> AsyncIterator[AsyncPageT]:\n        page = self\n        while True:\n            yield page\n            if page.has_next_page():\n                page = await page.get_next_page()\n            else:\n                return\n\n    async def get_next_page(self: AsyncPageT) -> AsyncPageT:\n        info = self.next_page_info()\n        if not info:\n            raise RuntimeError(\n                \"No next page expected; please check `.has_next_page()` before calling `.get_next_page()`.\"\n            )\n\n        options = self._info_to_options(info)\n        return await self._client._request_api_list(self._model, page=self.__class__, options=options)\n\n\n_HttpxClientT = TypeVar(\"_HttpxClientT\", bound=Union[httpx.Client, httpx.AsyncClient])\n_DefaultStreamT = TypeVar(\"_DefaultStreamT\", bound=Union[Stream[Any], AsyncStream[Any]])\n\n\nclass BaseClient(Generic[_HttpxClientT, _DefaultStreamT]):\n    _client: _HttpxClientT\n    _version: str\n    _base_url: URL\n    max_retries: int\n    timeout: Union[float, Timeout, None]\n    _limits: httpx.Limits\n    _proxies: ProxiesTypes | None\n    _transport: Transport | AsyncTransport | None\n    _strict_response_validation: bool\n    _idempotency_header: str | None\n    _default_stream_cls: type[_DefaultStreamT] | None = None\n\n    def __init__(\n        self,\n        *,\n        version: str,\n        base_url: str | URL,\n        _strict_response_validation: bool,\n        max_retries: int = DEFAULT_MAX_RETRIES,\n        timeout: float | Timeout | None = DEFAULT_TIMEOUT,\n        limits: httpx.Limits,\n        transport: Transport | AsyncTransport | None,\n        proxies: ProxiesTypes | None,\n        custom_headers: Mapping[str, str] | None = None,\n        custom_query: Mapping[str, object] | None = None,\n    ) -> None:\n        self._version = version\n        self._base_url = self._enforce_trailing_slash(URL(base_url))\n        self.max_retries = max_retries\n        self.timeout = timeout\n        self._limits = limits\n        self._proxies = proxies\n        self._transport = transport\n        self._custom_headers = custom_headers or {}\n        self._custom_query = custom_query or {}\n        self._strict_response_validation = _strict_response_validation\n        self._idempotency_header = None\n\n    \ndef _enforce_trailing_slash(self, url: URL) -> URL:\n        if url.raw_path.endswith(b\"/\"):\n            return url\n        return url.copy_with(raw_path=url.raw_path + b\"/\")\n\n    def _make_status_error_from_response(\n        self,\n        response: httpx.Response,\n    ) -> APIStatusError:\n        if response.is_closed and not response.is_stream_consumed:\n            # We can't read the response body as it has been closed\n            # before it was read. This can happen if an event hook\n            # raises a status error.\n            body = None\n            err_msg = f\"Error code: {response.status_code}\"\n        else:\n            err_text = response.text.strip()\n            body = err_text\n\n            try:\n                body = json.loads(err_text)\n                err_msg = f\"Error code: {response.status_code} - {body}\"\n            except Exception:\n                err_msg = err_text or f\"Error code: {response.status_code}\"\n\n        return self._make_status_error(err_msg, body=body, response=response)\n\n    def _make_status_error(\n        self,\n        err_msg: str,\n        *,\n        body: object,\n        response: httpx.Response,\n    ) -> _exceptions.APIStatusError:\n        raise NotImplementedError()\n\n    def _remaining_retries(\n        self,\n        remaining_retries: Optional[int],\n        options: FinalRequestOptions,\n    ) -> int:\n        return remaining_retries if remaining_retries is not None else options.get_max_retries(self.max_retries)\n\n    def _build_headers(self, options: FinalRequestOptions) -> httpx.Headers:\n        custom_headers = options.headers or {}\n        headers_dict = _merge_mappings(self.default_headers, custom_headers)\n        self._validate_headers(headers_dict, custom_headers)\n\n        # headers are case-insensitive while dictionaries are not.\n        headers = httpx.Headers(headers_dict)\n\n        idempotency_header = self._idempotency_header\n        if idempotency_header and options.method.lower() != \"get\" and idempotency_header not in headers:\n            headers[idempotency_header] = options.idempotency_key or self._idempotency_key()\n\n        return headers\n\n    def _prepare_url(self, url: str) -> URL:\n        \"\"\"\n        Merge a URL argument together with any 'base_url' on the client,\n        to create the URL used for the outgoing request.\n        \"\"\"\n        # Copied from httpx's `_merge_url` method.\n        merge_url = URL(url)\n        if merge_url.is_relative_url:\n            merge_raw_path = self.base_url.raw_path + merge_url.raw_path.lstrip(b\"/\")\n            return self.base_url.copy_with(raw_path=merge_raw_path)\n\n        return merge_url\n\n    def _build_request(\n        self,\n        options: FinalRequestOptions,\n    ) -> httpx.Request:\n        if log.isEnabledFor(logging.DEBUG):\n...\n", "needle_token_start": 15188, "needle_token_end": 15249, "code_context_ntokens": 16006, "output": ["```python\ndef _enforce_trailing_slash(self, url: URL) -> URL:\n    if url.raw_path.endswith(b\"/\"):\n        return url\n    return url.copy_with(raw_path=url.raw_path + b\"/\")\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "fused_decode3_matmul1_fp16_before", "language": "python", "path": "mlc_llm/dispatch/llama/main.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function performs a decoding operation followed by a matrix multiplication, primarily working with 16-bit floating-point precision. This is typically used in neural network layers where decoding of inputs is followed by a weighted sum computation.\n2. **Input**: The function takes buffers containing 32-bit unsigned integers and 16-bit floating-point numbers, which represent encoded data and matrix values for the computation.\n3. **Output**: It outputs a buffer of 16-bit floating-point numbers, which are the results of the matrix multiplication after the decoding process.\n4. **Procedure**: The function first decodes the input data using bitwise operations and arithmetic, storing the results in an intermediate buffer. It then performs a matrix multiplication using the decoded data and another input matrix, initializing the output buffer to zero before accumulating the products.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                        with T.block(\"var_matmul_intermediate_local\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                            T.reads(lv4[v0, v1, v2], var_matmul_intermediate_local[v0, v1, v2])\n                            T.writes(p_output0_intermediate[v0, v1, v2])\n                            p_output0_intermediate[v0, v1, v2] = lv4[v0, v1, v2] + var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\n\ndef fused_decode3_matmul1_fp16_before(lv5865: T.Buffer((T.int64(512), T.int64(32000)), \"uint32\"), lv5866: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv5867: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv2705: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(32000)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(32000)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv5865[v_i // T.int64(8), v_j], lv5866[v_i // T.int64(32), v_j], lv5867[v_i // T.int64(32), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv5865[v_i // T.int64(8), v_j], T.Cast(\"uint32\", v_i % T.int64(8) * T.int64(4))), T.uint32(15))) * lv5866[v_i // T.int64(32), v_j] + lv5867[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(32000), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv2705[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv2705[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n\n\n@T.prim_func\ndef fused_decode3_matmul1_fp16_after(lv1123: T.Buffer((T.int64(512), T.int64(32000)), \"uint32\"), lv5866: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv5867: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv1511: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    # with T.block(\"root\"):\n    var_decode_intermediate_pad_local = T.alloc_buffer((T.int64(4096), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_pad_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    lv1511_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(125), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(4)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        for ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                            with T.block(\"lv1511_shared\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v2 = T.axis.spatial(T.int64(4096), ax1_ax2_fused_0 * T.int64(1024) + ax1_ax2_fused_1 * T.int64(4) + ax1_ax2_fused_2)\n                                T.reads(lv1511[v0, v1, v2])\n                                T.writes(lv1511_shared[v0, v1, v2])\n                                T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                                lv1511_shared[v0, v1, v2] = lv1511[v0, v1, v2]\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(64)):\n                    for ax0_0 in range(T.int64(8)):\n                        for ax0_1 in T.unroll(T.int64(8)):\n                            for ax1 in range(T.int64(1)):\n                                with T.block(\"var_decode_intermediate_pad\"):\n                                    v0 = T.axis.spatial(T.int64(4096), k_0_0 * T.int64(64) + ax0_0 * T.int64(8) + ax0_1)\n                                    v1 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                    T.reads(lv1123[v0 // T.int64(8), v1], lv5866[v0 // T.int64(32), v1], lv5867[v0 // T.int64(32), v1])\n                                    T.writes(var_decode_intermediate_pad_local[v0, v1])\n                                    var_decode_intermediate_pad_local[v0, v1] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1123[v0 // T.int64(8), v1], T.Cast(\"uint32\", v0 % T.int64(8) * T.int64(4))), T.uint32(15))) * lv5866[v0 // T.int64(32), v1] + lv5867[v0 // T.int64(32), v1]\n                    for k_0_1_k_1_fused in range(T.int64(64)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4096), k_0_0 * T.int64(64) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2], lv1511_shared[v_i0, v_i1, v_k], var_decode_intermediate_pad_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] + lv1511_shared[v_i0, v_i1, v_k] * var_decode_intermediate_pad_local[v_k, v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_pad_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_pad_local[v0, v1, v2])\n                        T.writes(var_matmul_intermediate[v0, v1, v2])\n                        var_matmul_intermediate[v0, v1, v2] = var_matmul_intermediate_pad_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode3_matmul1_cast_fp16_before(lv1803: T.Buffer((T.int64(512), T.int64(32000)), \"uint32\"), lv1804: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv1805: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv3025: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(32000)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(32000)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1803[v_i // T.int64(8), v_j], lv1804[v_i // T.int64(32), v_j], lv1805[v_i // T.int64(32), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1803[v_i // T.int64(8), v_j], T.Cast(\"uint32\", v_i % T.int64(8) * T.int64(4))), T.uint32(15))) * lv1804[v_i // T.int64(32), v_j] + lv1805[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(32000), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv3025[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv3025[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(32000)):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(p_output0_intermediate[v_i0, v_i1, v_i2])\n            p_output0_intermediate[v_i0, v_i1, v_i2] = T.Cast(\"float32\", var_matmul_intermediate[v_i0, v_i1, v_i2])\n\n\n@T.prim_func\ndef fused_decode3_matmul1_cast_fp16_after(lv1123: T.Buffer((T.int64(512), T.int64(32000)), \"uint32\"), lv5866: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv5867: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv1511: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    # with T.block(\"root\"):\n    var_decode_intermediate_pad_local = T.alloc_buffer((T.int64(4096), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_pad_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    lv1511_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(125), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(4)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        for ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                            with T.block(\"lv1511_shared\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v2 = T.axis.spatial(T.int64(4096), ax1_ax2_fused_0 * T.int64(1024) + ax1_ax2_fused_1 * T.int64(4) + ax1_ax2_fused_2)\n                                T.reads(lv1511[v0, v1, v2])\n                                T.writes(lv1511_shared[v0, v1, v2])\n                                T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                                lv1511_shared[v0, v1, v2] = lv1511[v0, v1, v2]\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(64)):\n                    for ax0_0 in range(T.int64(8)):\n                        for ax0_1 in T.unroll(T.int64(8)):\n                            for ax1 in range(T.int64(1)):\n                                with T.block(\"var_decode_intermediate_pad\"):\n                                    v0 = T.axis.spatial(T.int64(4096), k_0_0 * T.int64(64) + ax0_0 * T.int64(8) + ax0_1)\n                                    v1 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                    T.reads(lv1123[v0 // T.int64(8), v1], lv5866[v0 // T.int64(32), v1], lv5867[v0 // T.int64(32), v1])\n                                    T.writes(var_decode_intermediate_pad_local[v0, v1])\n                                    var_decode_intermediate_pad_local[v0, v1] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1123[v0 // T.int64(8), v1], T.Cast(\"uint32\", v0 % T.int64(8) * T.int64(4))), T.uint32(15))) * lv5866[v0 // T.int64(32), v1] + lv5867[v0 // T.int64(32), v1]\n                    for k_0_1_k_1_fused in range(T.int64(64)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4096), k_0_0 * T.int64(64) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2], lv1511_shared[v_i0, v_i1, v_k], var_decode_intermediate_pad_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] + lv1511_shared[v_i0, v_i1, v_k] * var_decode_intermediate_pad_local[v_k, v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_pad_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_pad_local[v0, v1, v2])\n                        T.writes(var_matmul_intermediate[v0, v1, v2])\n                        var_matmul_intermediate[v0, v1, v2] = T.Cast(\"float32\", var_matmul_intermediate_pad_local[v0, v1, v2])\n\n\n@T.prim_func\ndef fused_decode4_fused_matmul5_add3_fp16_before(lv35: T.Buffer((T.int64(512), T.int64(4096)), \"uint32\"), lv36: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv37: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv2: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv2710: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(4096)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv35[v_i // T.int64(8), v_j], lv36[v_i // T.int64(32), v_j], lv37[v_i // T.int64(32), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv35[v_i // T.int64(8), v_j], T.Cast(\"uint32\", v_i % T.int64(8) * T.int64(4))), T.uint32(15))) * lv36[v_i // T.int64(32), v_j] + lv37[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv2[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv2[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(4096)):\n        with T.block(\"T_add\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv2710[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = lv2710[v_ax0, v_ax1, v_ax2] + var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_decode4_fused_matmul5_add3_fp16_after(lv1143: T.Buffer((T.int64(512), T.int64(4096)), \"uint32\"), lv36: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv37: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv3: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv2710: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4096), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    lv3_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(4)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        for ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                            with T.block(\"lv3_shared\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v2 = T.axis.spatial(T.int64(4096), ax1_ax2_fused_0 * T.int64(1024) + ax1_ax2_fused_1 * T.int64(4) + ax1_ax2_fused_2)\n                                T.reads(lv3[v0, v1, v2])\n                                T.writes(lv3_shared[v0, v1, v2])\n                                T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                                lv3_shared[v0, v1, v2] = lv3[v0, v1, v2]\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(64)):\n                    for ax0_0 in range(T.int64(8)):\n                        for ax0_1 in T.unroll(T.int64(8)):\n                            for ax1 in range(T.int64(1)):\n                                with T.block(\"decode\"):\n                                    v_j = T.axis.spatial(T.int64(4096), k_0_0 * T.int64(64) + ax0_0 * T.int64(8) + ax0_1)\n                                    v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                    T.reads(lv1143[v_j // T.int64(8), v_i], lv36[v_j // T.int64(32), v_i], lv37[v_j // T.int64(32), v_i])\n                                    T.writes(var_decode_intermediate_local[v_j, v_i])\n                                    var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1143[v_j // T.int64(8), v_i], T.Cast(\"uint32\", v_j % T.int64(8) * T.int64(4))), T.uint32(15))) * lv36[v_j // T.int64(32), v_i] + lv37[v_j // T.int64(32), v_i]\n                    for k_0_1_k_1_fused in range(T.int64(64)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4096), k_0_0 * T.int64(64) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv3_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv3_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(lv2710[v0, v1, v2], var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(p_output0_intermediate[v0, v1, v2])\n                        p_output0_intermediate[v0, v1, v2] = lv2710[v0, v1, v2] + var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode4_matmul5_fp16_before(lv11: T.Buffer((T.int64(512), T.int64(4096)), \"uint32\"), lv12: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv13: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv2712: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv11[v_i // T.int64(8), v_j], lv12[v_i // T.int64(32), v_j], lv13[v_i // T.int64(32), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv11[v_i // T.int64(8), v_j], T.Cast(\"uint32\", v_i % T.int64(8) * T.int64(4))), T.uint32(15))) * lv12[v_i // T.int64(32), v_j] + lv13[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv2712[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv2712[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n\n\n@T.prim_func\ndef fused_decode4_matmul5_fp16_after(lv1128: T.Buffer((T.int64(512), T.int64(4096)), \"uint32\"), lv12: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv13: T.Buffer((T.int64(128), T.int64(4096)), \"float16\"), lv2712: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4096), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    lv2712_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(4)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        for ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                            with T.block(\"lv2712_shared\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v2 = T.axis.spatial(T.int64(4096), ax1_ax2_fused_0 * T.int64(1024) + ax1_ax2_fused_1 * T.int64(4) + ax1_ax2_fused_2)\n                                T.reads(lv2712[v0, v1, v2])\n                                T.writes(lv2712_shared[v0, v1, v2])\n                                T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                                lv2712_shared[v0, v1, v2] = lv2712[v0, v1, v2]\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(64)):\n                    for ax0_0 in range(T.int64(8)):\n                        for ax0_1 in T.unroll(T.int64(8)):\n                            for ax1 in range(T.int64(1)):\n                                with T.block(\"decode\"):\n                                    v_j = T.axis.spatial(T.int64(4096), k_0_0 * T.int64(64) + ax0_0 * T.int64(8) + ax0_1)\n                                    v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                    T.reads(lv1128[v_j // T.int64(8), v_i], lv12[v_j // T.int64(32), v_i], lv13[v_j // T.int64(32), v_i])\n                                    T.writes(var_decode_intermediate_local[v_j, v_i])\n                                    var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1128[v_j // T.int64(8), v_i], T.Cast(\"uint32\", v_j % T.int64(8) * T.int64(4))), T.uint32(15))) * lv12[v_j // T.int64(32), v_i] + lv13[v_j // T.int64(32), v_i]\n                    for k_0_1_k_1_fused in range(T.int64(64)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4096), k_0_0 * T.int64(64) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv2712_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv2712_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(var_matmul_intermediate[v0, v1, v2])\n                        var_matmul_intermediate[v0, v1, v2] = var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode5_fused_matmul8_multiply1_fp16_before(lv51: T.Buffer((T.int64(512), T.int64(11008)), \"uint32\"), lv52: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"), lv53: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"), lv2749: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv5: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(11008)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(11008)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv51[v_i // T.int64(8), v_j], lv52[v_i // T.int64(32), v_j], lv53[v_i // T.int64(32), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv51[v_i // T.int64(8), v_j], T.Cast(\"uint32\", v_i % T.int64(8) * T.int64(4))), T.uint32(15))) * lv52[v_i // T.int64(32), v_j] + lv53[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(11008), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv2749[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv2749[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv5[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = lv5[v_ax0, v_ax1, v_ax2] * var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_decode5_fused_matmul8_multiply1_fp16_after(lv1153: T.Buffer((T.int64(512), T.int64(11008)), \"uint32\"), lv52: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"), lv53: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"), lv2749: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv5: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4096), T.int64(11008)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), scope=\"local\", dtype=\"float16\")\n    lv2749_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(43), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(4)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        for ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                            with T.block(\"lv2749_shared\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v2 = T.axis.spatial(T.int64(4096), ax1_ax2_fused_0 * T.int64(1024) + ax1_ax2_fused_1 * T.int64(4) + ax1_ax2_fused_2)\n                                T.reads(lv2749[v0, v1, v2])\n                                T.writes(lv2749_shared[v0, v1, v2])\n                                T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                                lv2749_shared[v0, v1, v2] = lv2749[v0, v1, v2]\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(11008), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(64)):\n...\n", "needle_token_start": 270, "needle_token_end": 1194, "code_context_ntokens": 16276, "output": ["```python\n@T.prim_func\ndef fused_decode3_matmul1_fp16_before(lv5865: T.Buffer((T.int64(512), T.int64(32000)), \"uint32\"), lv5866: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv5867: T.Buffer((T.int64(128), T.int64(32000)), \"float16\"), lv2705: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(32000)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(32000)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv5865[v_i // T.int64(8), v_j], lv5866[v_i // T.int64(32), v_j], lv5867[v_i // T.int64(32), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv5865[v_i // T.int64(8), v_j], T.Cast(\"uint32\", v_i % T.int64(8) * T.int64(4))), T.uint32(15))) * lv5866[v_i // T.int64(32), v_j] + lv5867[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(32000), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv2705[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv2705[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "fused_fused_decode7_matmul4", "language": "python", "path": "mlc_llm/dispatch/dispatch_tir_operator_adreno.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to perform a matrix multiplication operation after decoding and transforming the input data. It is likely used in a neural network or similar computational model where such transformations and multiplications are common for processing the input data.\n2. **Input**: The function takes three primary inputs:\n   - A 2D buffer of unsigned 32-bit integers,\n   - A 2D buffer of 16-bit floating-point numbers,\n   - A 3D buffer of 16-bit floating-point numbers.\n3. **Output**: It outputs a 3D buffer of 16-bit floating-point numbers, which represents the result of the matrix multiplication after the initial decoding and transformation of the input data.\n4. **Procedure**: The function proceeds through several steps:\n   - **Decoding and Transformation**: Each element of the first input buffer is decoded and transformed using bitwise operations and then multiplied by corresponding elements from the second input buffer.\n   - **Matrix Multiplication**: The transformed data is then used in a matrix multiplication with the third input buffer.\n   - **Aggregation**: The results of the multiplication are aggregated into the output buffer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(\n                                        lv20_local[v0, v1],\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ],\n                                    )\n                                    T.writes(\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                    )\n                                    var_matmul_intermediate_local[v_i0, v_i1, v_i2k] = (\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                        + var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ]\n                                        * lv20_local[v0, v1]\n                                    )\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_update\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(352256),\n                                i0_i1_i2_fused_0 * T.int64(2048)\n                                + i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.reads(var_matmul_intermediate_local[v0, v1, v_i2k])\n                            T.writes(lv1654_shared[v0, v1, v2])\n                            lv1654_shared[v0, v1, v2] = var_matmul_intermediate_local[\n                                v0, v1, v_i2k\n                            ]\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"reduction_1\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v_i2k = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(8))\n                            T.reads(lv1654_shared[v0, v1, v_i2k])\n                            T.writes(lv1654_shared[v0, v1, v_i2k])\n                            lv1654_shared[v0, v1, v_i2k] = (\n                                lv1654_shared[v0, v1, v_i2k]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(32)]\n                            )\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"reduction_2\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v_i2k = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(4))\n                            T.reads(lv1654_shared[v0, v1, v_i2k])\n                            T.writes(lv1654_shared[v0, v1, v_i2k])\n                            lv1654_shared[v0, v1, v_i2k] = (\n                                lv1654_shared[v0, v1, v_i2k]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(16)]\n                            )\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_local\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(22016),\n                                i0_i1_i2_fused_0 * T.int64(128)\n                                + i0_i1_i2_fused_1 * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(1))\n                            T.reads(lv1654_shared[v0, v1, v_i2k])\n                            T.writes(var_matmul_intermediate[v0, v1, v2])\n                            var_matmul_intermediate[v0, v1, v2] = (\n                                lv1654_shared[v0, v1, v_i2k]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(4)]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(8)]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(12)]\n                            )\n\n\n@T.prim_func(private=True)\n\ndef fused_fused_decode7_matmul4(\n    lv3: T.Buffer((T.int64(512), T.int64(12288)), \"uint32\"),\n    lv4: T.Buffer((T.int64(128), T.int64(12288)), \"float16\"),\n    lv1615: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    var_matmul_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(12288)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    p_output0_intermediate = T.alloc_buffer((T.int64(4096), T.int64(12288)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(12288)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv3[v_i // T.int64(8), v_j], lv4[v_i // T.int64(32), v_j])\n            T.writes(p_output0_intermediate[v_i, v_j])\n            p_output0_intermediate[v_i, v_j] = (\n                T.Cast(\n                    \"float16\",\n                    T.bitwise_and(\n                        T.shift_right(\n                            lv3[v_i // T.int64(8), v_j],\n                            T.Cast(\"uint32\", v_i % T.int64(8)) * T.uint32(4),\n                        ),\n                        T.uint32(15),\n                    ),\n                )\n                - T.float16(7)\n            ) * lv4[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(12288), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1615[v_i0, v_i1, v_k], p_output0_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = (\n                var_matmul_intermediate[v_i0, v_i1, v_i2]\n                + lv1615[v_i0, v_i1, v_k] * p_output0_intermediate[v_k, v_i2]\n            )\n\n\n@T.prim_func(private=True)\ndef fused_fused_decode7_matmul4_after(\n    lv3: T.Buffer((T.int64(512), T.int64(12288)), \"uint32\"),\n    lv4: T.Buffer((T.int64(128), T.int64(12288)), \"float16\"),\n    lv1615: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    var_matmul_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(12288)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_matmul_intermediate_local = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(24576)), \"float16\", scope=\"local\"\n    )\n    var_matmul_intermediate_local_batch = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(24576)), \"float16\", scope=\"local\"\n    )\n    lv3_local = T.alloc_buffer((T.int64(512), T.int64(12288)), \"uint32\", scope=\"local\")\n    lv4_local = T.alloc_buffer((T.int64(128), T.int64(12288)), \"float16\", scope=\"local\")\n    lv1615_shared = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(1024)), \"float16\", scope=\"shared\"\n    )\n    for i0_i1_i2_fused_0 in T.thread_binding(T.int64(48), thread=\"blockIdx.x\"):\n        for i0_i1_i2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for i0_i1_i2_fused_2_init in T.vectorized(T.int64(4)):\n                    with T.block(\"matmul_init\"):\n                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i2 = T.axis.spatial(\n                            T.int64(24576),\n                            i0_i1_i2_fused_0 * T.int64(512)\n                            + i0_i1_i2_fused_1 * T.int64(8)\n                            + ax2_y * T.int64(4)\n                            + i0_i1_i2_fused_2_init,\n                        )\n                        T.reads()\n                        T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                        var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float16(0)\n            for k_0 in range(T.int64(4)):\n                for ax2_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                            for ax2_2 in T.vectorized(T.int64(8)):\n                                with T.block(\"lv1615_shared\"):\n                                    v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                                    v2 = T.axis.spatial(\n                                        T.int64(4096),\n                                        k_0 * T.int64(1024)\n                                        + ax2_y * T.int64(512)\n                                        + ax2_1 * T.int64(8)\n                                        + ax2_2,\n                                    )\n                                    v2k = T.axis.spatial(\n                                        T.int64(1024),\n                                        (\n                                            ax2_y * T.int64(512)\n                                            + ax2_1 * T.int64(8)\n                                            + ax2_2\n                                        ),\n                                    )\n                                    T.reads(lv1615[v0, v1, v2])\n                                    T.writes(lv1615_shared[v0, v1, v2k])\n                                    lv1615_shared[v0, v1, v2k] = lv1615[v0, v1, v2]\n                for k_1 in range(T.int64(16)):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax1 in T.vectorized(T.int64(4)):\n                            with T.block(\"matmul_init_local\"):\n                                v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i2k = T.axis.spatial(\n                                    T.int64(24576),\n                                    i0_i1_i2_fused_0 * T.int64(512)\n                                    + i0_i1_i2_fused_1 * T.int64(8)\n                                    + ax2_y * T.int64(4)\n                                    + ax1,\n                                )\n                                T.reads()\n                                T.writes(\n                                    var_matmul_intermediate_local_batch[\n                                        v_i0, v_i1, v_i2k\n                                    ]\n                                )\n                                var_matmul_intermediate_local_batch[\n                                    v_i0, v_i1, v_i2k\n                                ] = T.float16(0)\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"lv4_local\"):\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(32)\n                                        + (k_1 * T.int64(2) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(12288),\n                                        i0_i1_i2_fused_0 * T.int64(256)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(lv4[v0, v1])\n                                    T.writes(lv4_local[v0, v1])\n                                    lv4_local[v0, v1] = lv4[v0, v1]\n                        for k_2 in range(T.int64(4)):\n                            for ax0 in range(T.int64(1)):\n                                for ax1 in T.vectorized(T.int64(4)):\n                                    with T.block(\"lv3_local\"):\n                                        v0 = T.axis.spatial(\n                                            T.int64(512),\n                                            k_0 * T.int64(128)\n                                            + (k_1 * T.int64(2) + ax2_y) * T.int64(4)\n                                            + k_2\n                                            + ax0,\n                                        )\n                                        v1 = T.axis.spatial(\n                                            T.int64(12288),\n                                            i0_i1_i2_fused_0 * T.int64(256)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + ax1,\n                                        )\n                                        T.reads(lv3[v0, v1])\n                                        T.writes(lv3_local[v0, v1])\n                                        lv3_local[v0, v1] = lv3[v0, v1]\n                            for k_3 in range(T.int64(8)):\n                                for i0_i1_i2_fused_2 in T.vectorized(T.int64(4)):\n                                    with T.block(\"matmul_update\"):\n                                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i2 = T.axis.spatial(\n                                            T.int64(12288),\n                                            i0_i1_i2_fused_0 * T.int64(256)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_i2k = T.axis.spatial(\n                                            T.int64(24576),\n                                            i0_i1_i2_fused_0 * T.int64(512)\n                                            + i0_i1_i2_fused_1 * T.int64(8)\n                                            + ax2_y * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_k = T.axis.reduce(\n                                            T.int64(4096),\n                                            k_0 * T.int64(1024)\n                                            + (k_1 * T.int64(2) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        v_ki = T.axis.reduce(\n                                            T.int64(1024),\n                                            (k_1 * T.int64(2) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        T.reads(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ],\n                                            lv1615_shared[v_i0, v_i1, v_ki],\n                                            lv3_local[v_k // T.int64(8), v_i2],\n                                        )\n                                        T.writes(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ]\n                                        )\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] = var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] + lv1615_shared[\n                                            v_i0, v_i1, v_ki\n                                        ] * (\n                                            (\n                                                T.Cast(\n                                                    \"float16\",\n                                                    T.bitwise_and(\n                                                        T.shift_right(\n                                                            lv3_local[\n                                                                v_k // T.int64(8), v_i2\n                                                            ],\n                                                            T.Cast(\n                                                                \"uint32\",\n                                                                v_k % T.int64(8),\n                                                            )\n                                                            * T.uint32(4),\n                                                        ),\n                                                        T.uint32(15),\n                                                    ),\n                                                )\n                                                - T.float16(7)\n                                            )\n                                        )\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"multiple_scale\"):\n                                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i2k = T.axis.spatial(\n                                        T.int64(24576),\n                                        i0_i1_i2_fused_0 * T.int64(512)\n                                        + i0_i1_i2_fused_1 * T.int64(8)\n                                        + ax2_y * T.int64(4)\n                                        + ax1,\n                                    )\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(32)\n                                        + (k_1 * T.int64(2) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(12288),\n                                        i0_i1_i2_fused_0 * T.int64(256)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(\n                                        lv4_local[v0, v1],\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ],\n                                    )\n                                    T.writes(\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                    )\n                                    var_matmul_intermediate_local[v_i0, v_i1, v_i2k] = (\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                        + var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ]\n                                        * lv4_local[v0, v1]\n                                    )\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_update\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(512),\n                                i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(24576),\n                                i0_i1_i2_fused_0 * T.int64(512)\n                                + i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.reads(var_matmul_intermediate_local[v0, v1, v_i2k])\n                            T.writes(lv1615_shared[v0, v1, v2])\n                            lv1615_shared[v0, v1, v2] = var_matmul_intermediate_local[\n                                v0, v1, v_i2k\n                            ]\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_local\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(12288),\n                                i0_i1_i2_fused_0 * T.int64(256)\n                                + i0_i1_i2_fused_1 * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(512),\n                                i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(1))\n                            T.reads(lv1615_shared[v0, v1, v_i2k])\n                            T.writes(var_matmul_intermediate[v0, v1, v2])\n                            var_matmul_intermediate[v0, v1, v2] = (\n                                lv1615_shared[v0, v1, v_i2k]\n                                + lv1615_shared[v0, v1, v_i2k + T.int64(4)]\n                            )\n\n\n@T.prim_func(private=True)\ndef fused_decode5_fused_matmul6_silu1(\n    lv1611: T.Buffer((T.int64(512), T.int64(11008)), \"uint32\"),\n    lv1612: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"),\n    lv1622: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    p_output0_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(11008)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(11008)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(11008)), \"float16\"\n    )\n    compute = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(11008)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1611[v_i // T.int64(8), v_j], lv1612[v_i // T.int64(32), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (\n                T.Cast(\n                    \"float16\",\n                    T.bitwise_and(\n                        T.shift_right(\n                            lv1611[v_i // T.int64(8), v_j],\n                            T.Cast(\"uint32\", v_i % T.int64(8)) * T.uint32(4),\n                        ),\n                        T.uint32(15),\n                    ),\n                )\n                - T.float16(7)\n            ) * lv1612[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(11008), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1622[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = (\n                var_matmul_intermediate[v_i0, v_i1, v_i2]\n                + lv1622[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n            )\n    for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(compute[v_i0, v_i1, v_i2])\n            compute[v_i0, v_i1, v_i2] = T.sigmoid(\n                var_matmul_intermediate[v_i0, v_i1, v_i2]\n            )\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(\n                var_matmul_intermediate[v_ax0, v_ax1, v_ax2],\n                compute[v_ax0, v_ax1, v_ax2],\n            )\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = (\n                var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n                * compute[v_ax0, v_ax1, v_ax2]\n            )\n\n\n@T.prim_func(private=True)\ndef fused_decode5_fused_matmul6_silu1_after(\n    lv1611: T.Buffer((T.int64(512), T.int64(11008)), \"uint32\"),\n    lv1612: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"),\n    lv1622: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    p_output0_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(11008)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_matmul_intermediate_local = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\", scope=\"local\"\n    )\n    var_matmul_intermediate_local_batch = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\", scope=\"local\"\n    )\n    lv1611_local = T.alloc_buffer(\n        (T.int64(512), T.int64(11008)), \"uint32\", scope=\"local\"\n    )\n    lv1612_local = T.alloc_buffer(\n        (T.int64(128), T.int64(11008)), \"float16\", scope=\"local\"\n    )\n    lv1622_shared = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(1024)), \"float16\", scope=\"shared\"\n    )\n    for i0_i1_i2_fused_0 in T.thread_binding(T.int64(43), thread=\"blockIdx.x\"):\n        for i0_i1_i2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for i0_i1_i2_fused_2_init in T.vectorized(T.int64(4)):\n                    with T.block(\"matmul_init\"):\n                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i2 = T.axis.spatial(\n                            T.int64(22016),\n                            i0_i1_i2_fused_0 * T.int64(512)\n                            + i0_i1_i2_fused_1 * T.int64(8)\n                            + ax2_y * T.int64(4)\n                            + i0_i1_i2_fused_2_init,\n                        )\n                        T.reads()\n                        T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                        var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float16(0)\n            for k_0 in range(T.int64(4)):\n                for ax2_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                            for ax2_2 in T.vectorized(T.int64(8)):\n                                with T.block(\"lv1622_shared\"):\n                                    v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                                    v2 = T.axis.spatial(\n                                        T.int64(4096),\n                                        k_0 * T.int64(1024)\n                                        + ax2_y * T.int64(512)\n                                        + ax2_1 * T.int64(8)\n                                        + ax2_2,\n                                    )\n                                    v2k = T.axis.spatial(\n                                        T.int64(1024),\n                                        (\n                                            ax2_y * T.int64(512)\n                                            + ax2_1 * T.int64(8)\n                                            + ax2_2\n                                        ),\n                                    )\n                                    T.reads(lv1622[v0, v1, v2])\n                                    T.writes(lv1622_shared[v0, v1, v2k])\n                                    lv1622_shared[v0, v1, v2k] = lv1622[v0, v1, v2]\n                for k_1 in range(T.int64(16)):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax1 in T.vectorized(T.int64(4)):\n                            with T.block(\"matmul_init_local\"):\n                                v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i2k = T.axis.spatial(\n                                    T.int64(22016),\n                                    i0_i1_i2_fused_0 * T.int64(512)\n                                    + i0_i1_i2_fused_1 * T.int64(8)\n                                    + ax2_y * T.int64(4)\n                                    + ax1,\n                                )\n                                T.reads()\n                                T.writes(\n                                    var_matmul_intermediate_local_batch[\n                                        v_i0, v_i1, v_i2k\n                                    ]\n                                )\n                                var_matmul_intermediate_local_batch[\n                                    v_i0, v_i1, v_i2k\n                                ] = T.float16(0)\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"lv1612_local\"):\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(32)\n                                        + (k_1 * T.int64(2) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(11008),\n                                        i0_i1_i2_fused_0 * T.int64(256)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(lv1612[v0, v1])\n                                    T.writes(lv1612_local[v0, v1])\n                                    lv1612_local[v0, v1] = lv1612[v0, v1]\n                        for k_2 in range(T.int64(4)):\n                            for ax0 in range(T.int64(1)):\n                                for ax1 in T.vectorized(T.int64(4)):\n                                    with T.block(\"lv1611_local\"):\n                                        v0 = T.axis.spatial(\n                                            T.int64(512),\n                                            k_0 * T.int64(128)\n                                            + (k_1 * T.int64(2) + ax2_y) * T.int64(4)\n                                            + k_2\n                                            + ax0,\n                                        )\n                                        v1 = T.axis.spatial(\n                                            T.int64(11008),\n                                            i0_i1_i2_fused_0 * T.int64(256)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + ax1,\n                                        )\n                                        T.reads(lv1611[v0, v1])\n                                        T.writes(lv1611_local[v0, v1])\n                                        lv1611_local[v0, v1] = lv1611[v0, v1]\n                            for k_3 in range(T.int64(8)):\n                                for i0_i1_i2_fused_2 in T.vectorized(T.int64(4)):\n                                    with T.block(\"matmul_update\"):\n                                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i2 = T.axis.spatial(\n                                            T.int64(11008),\n                                            i0_i1_i2_fused_0 * T.int64(256)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_i2k = T.axis.spatial(\n                                            T.int64(22016),\n                                            i0_i1_i2_fused_0 * T.int64(512)\n                                            + i0_i1_i2_fused_1 * T.int64(8)\n                                            + ax2_y * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_k = T.axis.reduce(\n                                            T.int64(4096),\n                                            k_0 * T.int64(1024)\n                                            + (k_1 * T.int64(2) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        v_ki = T.axis.reduce(\n                                            T.int64(1024),\n                                            (k_1 * T.int64(2) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        T.reads(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ],\n                                            lv1622_shared[v_i0, v_i1, v_ki],\n                                            lv1611_local[v_k // T.int64(8), v_i2],\n                                        )\n                                        T.writes(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ]\n                                        )\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] = var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] + lv1622_shared[\n                                            v_i0, v_i1, v_ki\n                                        ] * (\n                                            (\n                                                T.Cast(\n                                                    \"float16\",\n                                                    T.bitwise_and(\n                                                        T.shift_right(\n                                                            lv1611_local[\n                                                                v_k // T.int64(8), v_i2\n                                                            ],\n                                                            T.Cast(\n                                                                \"uint32\",\n                                                                v_k % T.int64(8),\n                                                            )\n                                                            * T.uint32(4),\n                                                        ),\n                                                        T.uint32(15),\n                                                    ),\n                                                )\n                                                - T.float16(7)\n                                            )\n                                        )\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"multiple_scale\"):\n                                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i2k = T.axis.spatial(\n                                        T.int64(22016),\n                                        i0_i1_i2_fused_0 * T.int64(512)\n                                        + i0_i1_i2_fused_1 * T.int64(8)\n                                        + ax2_y * T.int64(4)\n                                        + ax1,\n                                    )\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(32)\n                                        + (k_1 * T.int64(2) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(11008),\n                                        i0_i1_i2_fused_0 * T.int64(256)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(\n                                        lv1612_local[v0, v1],\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ],\n                                    )\n                                    T.writes(\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                    )\n                                    var_matmul_intermediate_local[v_i0, v_i1, v_i2k] = (\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                        + var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ]\n                                        * lv1612_local[v0, v1]\n                                    )\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_update\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(512),\n                                i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(22016),\n                                i0_i1_i2_fused_0 * T.int64(512)\n                                + i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.reads(var_matmul_intermediate_local[v0, v1, v_i2k])\n                            T.writes(lv1622_shared[v0, v1, v2])\n                            lv1622_shared[v0, v1, v2] = var_matmul_intermediate_local[\n                                v0, v1, v_i2k\n                            ]\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"reduction\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(512),\n                                i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(1))\n                            T.reads(lv1622_shared[v0, v1, v2])\n                            T.writes(lv1622_shared[v0, v1, v2])\n                            lv1622_shared[v0, v1, v2] = (\n                                lv1622_shared[v0, v1, v2]\n                                + lv1622_shared[v0, v1, v2 + T.int64(4)]\n                            )\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_local\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(11008),\n                                i0_i1_i2_fused_0 * T.int64(256)\n                                + i0_i1_i2_fused_1 * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(512),\n                                i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(1))\n                            T.reads(lv1622_shared[v0, v1, v_i2k])\n                            T.writes(p_output0_intermediate[v0, v1, v2])\n                            p_output0_intermediate[v0, v1, v2] = lv1622_shared[\n                                v0, v1, v_i2k\n                            ] * T.sigmoid(lv1622_shared[v0, v1, v_i2k])\n\n\ndef sch_fused_decode5_fused_matmul6_silu1(func):\n    sch = tvm.tir.Schedule(func)\n    b0 = sch.get_block(name=\"decode\", func_name=\"main\")\n    b1 = sch.get_block(name=\"matmul\", func_name=\"main\")\n    l2, l3, l4, l5 = sch.get_loops(block=b1)\n    l6 = sch.fuse(l2, l3, l4, preserve_unit_iters=True)\n    v7, v8, v9 = sch.sample_perfect_tile(\n        loop=l6, n=3, max_innermost_factor=4, decision=[43, 64, 4]\n    )\n    l10, l11, l12 = sch.split(loop=l6, factors=[v7, v8, v9], preserve_unit_iters=True)\n    v13, v14, v15 = sch.sample_perfect_tile(\n        loop=l5, n=3, max_innermost_factor=8, decision=[128, 4, 8]\n    )\n    l16, l17, l18 = sch.split(\n        loop=l5, factors=[v13, v14, v15], preserve_unit_iters=True\n    )\n    sch.reorder(l10, l11, l16, l17, l18, l12)\n    sch.bind(loop=l10, thread_axis=\"blockIdx.x\")\n    sch.bind(loop=l11, thread_axis=\"threadIdx.x\")\n    sch.compute_inline(block=b0)\n    b19 = sch.cache_write(block=b1, write_buffer_index=0, storage_scope=\"local\")\n    sch.reverse_compute_at(block=b19, loop=l11, preserve_unit_loops=True, index=-1)\n    b20 = sch.cache_read(block=b1, read_buffer_index=1, storage_scope=\"local\")\n    b21 = sch.cache_read(block=b1, read_buffer_index=2, storage_scope=\"local\")\n    b22 = sch.cache_read(block=b1, read_buffer_index=0, storage_scope=\"shared\")\n    sch.compute_at(block=b22, loop=l11, preserve_unit_loops=True, index=-1)\n    v23 = sch.sample_categorical(\n        candidates=[1, 2, 4, 8], probs=[0.25, 0.25, 0.25, 0.25], decision=3\n    )\n    sch.annotate(\n        block_or_loop=b22, ann_key=\"meta_schedule.cooperative_fetch\", ann_val=v23\n    )\n    sch.compute_at(block=b20, loop=l17, preserve_unit_loops=True, index=-1)\n    sch.compute_at(block=b21, loop=l16, preserve_unit_loops=True, index=-1)\n    l24, l25, l26, l27, l28, l29 = sch.get_loops(block=b20)\n    sch.vectorize(loop=l29)\n    l30, l31, l32, l33, l34 = sch.get_loops(block=b21)\n    sch.vectorize(loop=l34)\n    l35, l36, l37, l38, l39 = sch.get_loops(block=b19)\n    sch.vectorize(loop=l39)\n    sch.vectorize(loop=l12)\n    b40 = sch.decompose_reduction(block=b1, loop=l16)\n    b41 = sch.get_block(name=\"compute\", func_name=\"main\")\n    sch.compute_inline(block=b41)\n    b42 = sch.get_block(name=\"T_multiply\", func_name=\"main\")\n    sch.reverse_compute_inline(block=b42)\n    sch.enter_postproc()\n    sch.unannotate(block_or_loop=b22, ann_key=\"meta_schedule.cooperative_fetch\")\n    l43, l44, l45, l46, l47 = sch.get_loops(block=b22)\n    l48, l49, l50 = sch.split(loop=l47, factors=[None, 64, 8], preserve_unit_iters=True)\n    sch.vectorize(loop=l50)\n    sch.bind(loop=l49, thread_axis=\"threadIdx.x\")\n    return sch.mod[\"main\"].with_attr(\"tir.is_scheduled\", 1)\n\n@T.prim_func(private=True)\ndef fused_decode81_fused_matmul1_cast2(\n    lv1576: T.Buffer((T.int64(512), T.int64(64000)), \"uint32\"),\n    lv1577: T.Buffer((T.int64(128), T.int64(64000)), \"float16\"),\n    lv1575: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    p_output0_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(64000)), \"float32\"\n    ),\n):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(64000)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(64000)), \"float16\"\n    )\n    for i, j in T.grid(T.int64(4096), T.int64(64000)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n...\n", "needle_token_start": 1982, "needle_token_end": 2874, "code_context_ntokens": 16324, "output": ["```python\n@T.prim_func(private=True)\ndef fused_decode81_fused_matmul1_cast2(\n    lv1576: T.Buffer((T.int64(512), T.int64(64000)), \"uint32\"),\n    lv1577: T.Buffer((T.int64(128), T.int64(64000)), \"float16\"),\n    lv1575: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    p_output0_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(64000)), \"float32\"\n    ),\n):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(64000)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(64000)), \"float16\"\n    )\n    for i, j in T.grid(T.int64(4096), T.int64(64000)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            var_decode_intermediate[v_i, v_j] = (\n                T.Cast(\n                    \"float16\",\n                    T.bitwise_and(\n                        T.shift_right(\n                            lv1576[v_i // T.int64(8), v_j],\n                            T.Cast(\"uint32\", v_i % T.int64(8)) * T.uint32(4),\n                        ),\n                        T.uint32(15),\n                    ),\n                )\n                - T.float16(7)\n            ) * lv1577[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(64000), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1575[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = (\n                var_matmul_intermediate[v_i0, v_i1, v_i2]\n                + lv1575[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n            )\n    for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(64000)):\n        with T.block(\"T_cast\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(p_output0_intermediate[v_i0, v_i1, v_i2])\n            p_output0_intermediate[v_i0, v_i1, v_i2] = T.cast(\n                \"float32\", var_matmul_intermediate[v_i0, v_i1, v_i2]\n            )\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "contain_symbolic_var", "language": "python", "path": "mlc_llm/transform/lift_tir_global_buffer_alloc.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function checks if a tensor's shape contains any symbolic (non-constant) variables.\n2. **Input**: A structure containing information about a tensor, specifically its shape.\n3. **Output**: A boolean value indicating whether the tensor's shape includes any symbolic variables.\n4. **Procedure**: The function iterates through the values defining the tensor's shape. If any of these values are not integer constants, it returns true, indicating the presence of symbolic variables; otherwise, it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: mlc_llm/transform/fuse_split_rotary_embedding.py\nimport tvm\nfrom tvm import relax\nfrom tvm.relax.dpl import (\n    PatternContext,\n    is_op,\n    rewrite_bindings,\n    wildcard,\n    is_tuple_get_item,\n    GlobalVarPattern,\n    TuplePattern,\n    is_shape,\n)\nfrom tvm.script import relax as R, tir as T\n\n\ndef get_dynamic_split_rotary():\n    \"\"\"Implementation of R.split(rotary_embedding(fused_qkv))\n\n    Implementation is generic over the number of query heads,\n    key/value heads, sequence length, head dimension, and position\n    embedding base.  These parameters can be replaced with static\n    values using `PrimFunc.specialize`.\n    \"\"\"\n\n    @T.prim_func(private=True)\n    def split_rotary(\n        fused_qkv_handle: T.handle,\n        embedded_query_handle: T.handle,\n        embedded_key_handle: T.handle,\n        value_handle: T.handle,\n        rotary_offset: T.int64,\n        batch_size: T.int64,\n        seq_len: T.int64,\n        num_query_heads: T.int64,\n        num_kv_heads: T.int64,\n        head_dim: T.int64,\n        position_embedding_base: T.float32,\n    ):\n        Fused_QKV = T.match_buffer(\n            fused_qkv_handle,\n            [batch_size, seq_len, num_query_heads + num_kv_heads * 2, head_dim],\n            dtype=\"float16\",\n        )\n        EmbeddedQuery = T.match_buffer(\n            embedded_query_handle,\n            [batch_size, seq_len, num_query_heads, head_dim],\n            dtype=\"float16\",\n        )\n        EmbeddedKey = T.match_buffer(\n            embedded_key_handle,\n            [batch_size, seq_len, num_kv_heads, head_dim],\n            dtype=\"float16\",\n        )\n        Value = T.match_buffer(\n            value_handle,\n            [batch_size, seq_len, num_kv_heads, head_dim],\n            dtype=\"float16\",\n        )\n\n        T.func_attr({\"op_pattern\": 2, \"tir.noalias\": T.bool(True)})\n\n        for iters in T.grid(batch_size, seq_len, num_query_heads + num_kv_heads * 2, head_dim):\n            with T.block(\"FusedRotaryEmbeddingAndSplitQKV\"):\n                batch_i, seq_i, head_num, head_i = T.axis.remap(\"SSSS\", iters)\n                pos: T.float32 = T.Cast(\"float32\", rotary_offset + seq_i - seq_len)\n\n                inv_freq: T.float32 = T.float32(1) / T.pow(\n                    position_embedding_base,\n                    T.Cast(\"float32\", (head_i * 2) % head_dim) / T.float32(head_dim),\n                )\n                freq: T.float32 = pos * inv_freq\n                cos_value: T.float16 = T.Cast(\"float16\", T.cos(freq))\n                sin_value: T.float16 = T.Cast(\"float16\", T.sin(freq))\n\n                input_value = Fused_QKV[batch_i, seq_i, head_num, head_i]\n                embedded_value = cos_value * input_value + sin_value * T.Select(\n                    head_i < T.int64(head_dim // 2),\n                    Fused_QKV[batch_i, seq_i, head_num, head_i + T.int64(head_dim // 2)]\n                    * T.float16(-1),\n                    Fused_QKV[batch_i, seq_i, head_num, head_i - T.int64(head_dim // 2)],\n                )\n                if head_num < num_query_heads:\n                    EmbeddedQuery[batch_i, seq_i, head_num, head_i] = embedded_value\n                elif head_num < num_query_heads + num_kv_heads:\n                    EmbeddedKey[batch_i, seq_i, head_num - num_query_heads, head_i] = embedded_value\n                else:\n                    Value[\n                        batch_i, seq_i, head_num - num_query_heads - num_kv_heads, head_i\n                    ] = input_value\n\n    param_sinfo = []\n    for param in split_rotary.params:\n        if param in split_rotary.buffer_map:\n            buf = split_rotary.buffer_map[param]\n            sinfo = relax.TensorStructInfo(shape=buf.shape, dtype=buf.dtype)\n        else:\n            sinfo = relax.PrimStructInfo(param.dtype)\n        param_sinfo.append(sinfo)\n\n    relax.expr._update_struct_info(\n        split_rotary,\n        tvm.relax.FuncStructInfo(\n            params=param_sinfo,\n            ret=relax.TupleStructInfo([]),\n            purity=False,\n        ),\n    )\n\n    return split_rotary\n\n\ndef fuse_split_rotary_embedding(\n    num_query_heads, num_kv_heads, hidden_size, position_embedding_base\n):\n    @tvm.ir.transform.module_pass(opt_level=0, name=\"fuse_split_rotary_embedding\")\n    def ir_module_pass(mod: tvm.IRModule, _pass_context) -> tvm.IRModule:\n        head_dim = hidden_size // num_query_heads\n        split_rotary = get_dynamic_split_rotary()\n\n        (\n            dyn_batch_size,\n            dyn_seq_len,\n            dyn_num_query_heads,\n            dyn_num_kv_heads,\n            dyn_head_dim,\n            dyn_position_embedding_base,\n        ) = split_rotary.params[-6:]\n\n        split_rotary = split_rotary.specialize(\n            {\n                # Static model parameters\n                dyn_batch_size: T.int64(1),\n                dyn_num_query_heads: T.int64(num_query_heads),\n                dyn_num_kv_heads: T.int64(num_kv_heads),\n                dyn_head_dim: T.int64(head_dim),\n                dyn_position_embedding_base: T.float32(position_embedding_base),\n                # Dynamic parameters, to be inferred from TIR Buffer shapes\n                dyn_seq_len: tvm.tir.Var(\"query_sequence_length\", \"int64\"),\n            }\n        )\n\n        mod[\"split_rotary\"] = split_rotary\n\n        split_rotary_gvar = mod.get_global_var(\"split_rotary\")\n        relax.expr._update_struct_info(split_rotary_gvar, mod[\"split_rotary\"].struct_info)\n\n        with PatternContext() as ctx:\n            # flat_qkv_tuple: R.Tuple(\n            #     R.Tensor((batch_size, seq_len, 4096), dtype=\"float16\"),\n            #     R.Tensor((batch_size, seq_len, 4096), dtype=\"float16\"),\n            #     R.Tensor((batch_size, seq_len, 4096), dtype=\"float16\"),\n            # ) = R.split(flat_fused_qkv, indices_or_sections=[4096, 8192], axis=2)\n            #\n            # flat_query: R.Tensor((batch_size, seq_len, 4096), dtype=\"float16\") = flat_qkv_tuple[0]\n            # query: R.Tensor((batch_size, seq_len, 32, 128), dtype=\"float16\") = R.reshape(\n            #     flat_query, R.shape([batch_size, seq_len, 32, 128])\n            # )\n            # flat_key: R.Tensor((batch_size, seq_len, 4096), dtype=\"float16\") = flat_qkv_tuple[1]\n            # key: R.Tensor((batch_size, seq_len, 32, 128), dtype=\"float16\") = R.reshape(\n            #     flat_key, R.shape([batch_size, seq_len, 32, 128])\n            # )\n            # flat_value: R.Tensor((batch_size, seq_len, 4096), dtype=\"float16\") = flat_qkv_tuple[2]\n            # value: R.Tensor((batch_size, seq_len, 32, 128), dtype=\"float16\") = R.reshape(\n            #     flat_value, R.shape([batch_size, seq_len, 32, 128])\n            # )\n            # embedded_query = R.call_tir(\n            #     cls.rotary_embedding1,\n            #     [query],\n            #     out_sinfo=R.Tensor((batch_size, seq_len, 32, 128), dtype=\"float16\"),\n            #     tir_vars=R.shape([n]),\n            # )\n            # embedded_key = R.call_tir(\n            #     cls.rotary_embedding1,\n            #     [key],\n            #     out_sinfo=R.Tensor((batch_size, seq_len, 32, 128), dtype=\"float16\"),\n            #     tir_vars=R.shape([n]),\n            # )\n\n            pat_rotary_embedding_gvar = GlobalVarPattern()\n\n            pat_flat_fused_qkv = wildcard()\n            pat_offset = wildcard()\n\n            # query_shape = is_shape([1, seq_len, num_query_heads, head_dim])\n            pat_query_shape = wildcard()\n            # value_shape = is_shape([1, seq_len, num_kv_heads, head_dim])\n            pat_key_shape = wildcard()\n            # value_shape = is_shape([1, seq_len, num_kv_heads, head_dim])\n            pat_value_shape = wildcard()\n\n            pat_flat_qkv_tuple = is_op(\"relax.split\")(pat_flat_fused_qkv)\n            pat_flat_query = is_tuple_get_item(pat_flat_qkv_tuple, 0)\n            pat_query = is_op(\"relax.reshape\")(\n                pat_flat_query, pat_query_shape, add_constraint=False\n            )\n            pat_flat_query.used_by(pat_query)\n            pat_flat_key = is_tuple_get_item(pat_flat_qkv_tuple, 1)\n            pat_key = is_op(\"relax.reshape\")(pat_flat_key, pat_key_shape, add_constraint=False)\n            pat_flat_key.used_by(pat_key)\n            pat_flat_value = is_tuple_get_item(pat_flat_qkv_tuple, 2)\n            pat_value = is_op(\"relax.reshape\")(\n                pat_flat_value, pat_value_shape, add_constraint=False\n            )\n            pat_flat_value.used_by(pat_value)\n\n            pat_embedded_query = is_op(\"relax.call_tir\")(\n                pat_rotary_embedding_gvar,\n                TuplePattern([pat_query]),\n                pat_offset,\n                add_constraint=False,\n            )\n            pat_embedded_key = is_op(\"relax.call_tir\")(\n                pat_rotary_embedding_gvar,\n                TuplePattern([pat_key]),\n                pat_offset,\n                add_constraint=False,\n            )\n\n            pat_flat_qkv_tuple.used_by(pat_flat_query)\n            pat_flat_qkv_tuple.used_by(pat_flat_key)\n            pat_flat_qkv_tuple.used_by(pat_flat_value)\n            pat_query.used_by(pat_embedded_query)\n            pat_key.used_by(pat_embedded_key)\n\n        def rewriter(matchings, bindings):\n            # Extracting all the relax and TIR variables that we'll need\n            flat_fused_qkv = matchings[pat_flat_fused_qkv]\n            flat_qkv_tuple = matchings[pat_flat_qkv_tuple]\n\n            flat_query = matchings[pat_flat_query]\n...\n# Path: mlc_llm/transform/lift_tir_global_buffer_alloc.py\n\"\"\"Lift global buffer allocation in TIR to graph level\"\"\"\n\nfrom typing import Dict, List, Tuple, Optional\n\nimport tvm\nfrom tvm import relax, tir\nfrom tvm.ir.module import IRModule\nfrom tvm.relax.analysis import remove_all_unused\nfrom tvm.relax.expr_functor import PyExprMutator, mutator\n\n\ndef remove_global_buf_alloc(\n    func: tir.PrimFunc,\n) -> Optional[Tuple[tir.PrimFunc, List[relax.TensorStructInfo]]]:\n    \"\"\"Remove the global buffer allocation for a given TIR PrimFunc.\"\"\"\n    if not isinstance(func.body, tir.BlockRealize):\n        return None\n\n    params = list(func.params)\n    buffer_map = dict(func.buffer_map)\n    tensor_sinfo = []\n    alloc_buffers = []\n\n    insertion_point = len(params)\n    while params[insertion_point - 1].dtype != \"handle\":\n        insertion_point -= 1\n        assert insertion_point >= 1\n\n    prev_root_block = func.body.block\n    for buf_alloc in func.body.block.alloc_buffers:\n        if buf_alloc.scope() == \"global\":\n            param = tir.Var(\"var_\" + buf_alloc.name, \"handle\")\n            params.insert(insertion_point, param)\n            insertion_point += 1\n            buffer_map[param] = buf_alloc\n            tensor_sinfo.append(relax.TensorStructInfo(buf_alloc.shape, buf_alloc.dtype))\n        else:\n            alloc_buffers.append(buf_alloc)\n\n    if len(tensor_sinfo) == 0:\n        return None\n\n    assert len(prev_root_block.iter_vars) == 0\n    assert len(prev_root_block.reads) == 0\n    assert len(prev_root_block.writes) == 0\n    assert len(prev_root_block.match_buffers) == 0\n    assert prev_root_block.name_hint == \"root\"\n    assert prev_root_block.init is None\n    root_block = tir.Block(\n        iter_vars=[],\n        reads=[],\n        writes=[],\n        name_hint=\"root\",\n        body=prev_root_block.body,\n        alloc_buffers=alloc_buffers,\n        annotations=prev_root_block.annotations,\n    )\n\n    updated_func = tir.PrimFunc(\n        params=params,\n        body=tir.BlockRealize(iter_values=[], predicate=True, block=root_block),\n        ret_type=func.ret_type,\n        buffer_map=buffer_map,\n        attrs=func.attrs,\n    )\n    return updated_func, tensor_sinfo\n\n\n\ndef contain_symbolic_var(tensor_sinfo: relax.TensorStructInfo) -> bool:\n    assert isinstance(tensor_sinfo.shape, relax.ShapeExpr)\n    for v in tensor_sinfo.shape.values:\n        if not isinstance(v, tir.IntImm):\n            return True\n    return False\n\n\ndef resolve_tir_var_mapping(\n    func: tir.PrimFunc, call: relax.Call, tensor_sinfo: List[relax.TensorStructInfo]\n) -> Tuple[List[relax.TensorStructInfo], bool]:\n    \"\"\"Resolve the TIR symbolic var relationship across sides of PrimFunc and Relax Function\"\"\"\n    var_map: Dict[tir.Var, tir.PrimExpr] = dict()\n\n    n_arg = len(call.args[1].fields)\n    for i in range(n_arg):\n        buffer_shape = func.buffer_map[func.params[i]].shape\n        arg_shape = call.args[1][i].struct_info.shape.values\n        assert len(buffer_shape) == len(arg_shape)\n        for vl, vr in zip(buffer_shape, arg_shape):\n            if isinstance(vl, tir.Var):\n                var_map[vl] = vr\n            elif not isinstance(vl, tir.IntImm):\n                return [], False\n\n    ret_tensors = call.sinfo_args[0]\n    ret_tensors = (\n        [ret_tensors]\n        if isinstance(ret_tensors, relax.TensorStructInfo)\n        else list(ret_tensors.fields)\n    )\n    for i in range(len(ret_tensors)):\n        buffer_shape = func.buffer_map[func.params[n_arg + i]].shape\n        ret_tensor_shape = ret_tensors[i].shape.values\n        assert len(buffer_shape) == len(ret_tensor_shape)\n        for vl, vr in zip(buffer_shape, ret_tensor_shape):\n            if isinstance(vl, tir.Var):\n                var_map[vl] = vr\n            elif not isinstance(vl, tir.IntImm):\n                return [], False\n\n    updated_tensor_sinfo = []\n    for sinfo in tensor_sinfo:\n        if not contain_symbolic_var(sinfo):\n            updated_tensor_sinfo.append(sinfo)\n            continue\n\n        new_shape = []\n        for v in sinfo.shape.values:\n            new_shape.append(tir.stmt_functor.substitute(v, var_map))\n        updated_tensor_sinfo.append(relax.TensorStructInfo(new_shape, sinfo.dtype))\n    return updated_tensor_sinfo, True\n\n\ndef LiftTIRGlobalBufferAlloc():\n    @mutator\n    class TIRGlobalAllocRewriter(PyExprMutator):\n        def __init__(self, mod: IRModule):\n            super().__init__(mod)\n            self.mod = mod\n\n        def transform(self) -> IRModule:\n            self.mod = self.builder_.get()\n            for gv, func in self.mod.functions.items():\n                if isinstance(func, relax.Function):\n                    updated_func = self.visit_expr(func)\n                    self.builder_.update_func(gv, updated_func)\n            return self.builder_.get()\n\n        def visit_call_(self, call: relax.Call):\n            call = self.visit_expr_post_order(call)\n            if call.op != tvm.ir.Op.get(\"relax.call_tir\"):\n                return call\n\n            old_gvar = call.args[0]\n\n            func_before_update = self.mod.functions[old_gvar]\n            updates = remove_global_buf_alloc(func_before_update)\n            if updates is None:\n                return call\n            updated_func, tensor_sinfo = updates\n\n            assert len(call.sinfo_args) == 1\n            if any(contain_symbolic_var(sinfo) for sinfo in tensor_sinfo):\n                tensor_sinfo, success = resolve_tir_var_mapping(\n                    func_before_update, call, tensor_sinfo\n                )\n                if not success:\n                    # Cannot resolve TIR var mapping. Fall back to no lifting.\n                    return call\n\n            new_gvar = self.builder_.add_func(updated_func, old_gvar.name_hint)\n            new_args = [new_gvar, *call.args[1:]]\n\n            if isinstance(call.sinfo_args[0], relax.TensorStructInfo):\n                new_call = relax.Call(\n                    call.op,\n                    args=new_args,\n                    sinfo_args=[relax.TupleStructInfo(list(call.sinfo_args) + tensor_sinfo)],\n                    attrs=call.attrs,\n                )\n                emitted_tuple = self.builder_.emit(new_call)\n                return relax.TupleGetItem(emitted_tuple, 0)\n            elif isinstance(call.sinfo_args[0], relax.TupleStructInfo):\n                return relax.Call(\n                    call.op,\n                    args=new_args,\n                    sinfo_args=[\n                        relax.TupleStructInfo(list(call.sinfo_args[0].fields) + tensor_sinfo)\n                    ],\n                    attrs=call.attrs,\n                )\n            else:\n                raise TypeError(\n                    f\"Expected {call.op} to return either R.Tensor or R.Tuple, \"\n                    f\"but instead returned {call.sinfo_args[0]}\"\n                )\n\n    @tvm.transform.module_pass(opt_level=0, name=\"LiftTIRGlobalBufferAlloc.Inner\")\n    def transform_module(mod: IRModule, _: tvm.transform.PassContext) -> IRModule:\n        return TIRGlobalAllocRewriter(mod).transform()\n\n    return tvm.ir.transform.Sequential(\n        [\n            transform_module,\n            tvm.relax.transform.DeadCodeElimination(),\n        ],\n        name=\"LiftTIRGlobalBufferAlloc\",\n    )\n\n# Path: mlc_llm/transform/reorder_transform_func.py\nfrom typing import Callable, Dict, List, Set, Tuple, Optional\n\nimport tvm\nfrom tvm import relax\nfrom tvm.ir.module import IRModule\n\n\"\"\"\nThis pass in this file reorders the bindings of the weight transform function\naccording to the weight location in binary files. The goal of the reorder is to\nreduce the memory pressure when loading the raw model weights and processing\nthem. In the ideal case, with this pass, the highest CPU memory usage will\naround the size of the largest raw weight binary file.\n\nRegarding the implementation, the bindings of fetching a raw weight in the\nweight transform function are all in the form of `lv = params[idx]`. Here, each\nindex specifies a raw weight tensor, and the raw weight tensor resides in a\nbinary file on the disk.\n\nWe group such `lv = params[idx]` into multiple groups, such that all raw weight\ntensors in a group come from a same binary file. We reorder the bindings\naccording to the grouping result based on topological sort.\n\nIn ideal case, after reordering the weight transform function has the following\nprocess during execution:\n* load a weight binary file,\n* process all weights in this file,\n* load another weight binary file,\n* process all weights in this file,\n* ...\n\nSo the maximum CPU memory usage will be the size of the largest raw weight\nbinary file, since we process and release all the raw weight tensors immediately\nafter loading them from the file.\n\"\"\"\n\n\ndef analyze_func(\n    func: relax.Function,\n    pidx2binname: Dict[int, str],\n) -> Tuple[List[relax.Binding], Dict[relax.Var, List[relax.Binding]], Dict[relax.Binding, int],]:\n    \"\"\"Binding grouping analysis function.\n    It takes the function to be analyzed, and mapping from each raw tensor index\n    to the name of the binary file where it resides.\n\n    This analysis function\n    * computes a new order of weight fetching bindings (the bindings in form\n    `lv = params[idx]`) based on weight location on disk.\n    * collects the dataflow def-use information of the given function for\n    topological sort (particularly, it collects the consumers of each binding\n    variables and the number of variables each binding depends on).\n\n    Parameters\n    ----------\n    func : relax.Function\n        The weight transform function to be analyzed.\n\n    pidx2binname : Dict[int, str]\n        The mapping from each raw tensor index to the name of the binary\n        file where it resides.\n\n    Returns\n    -------\n    get_param_bindings : List[relax.Binding]\n        The weight fetching bindings (`lv = params[idx]`) in the new order.\n\n    var_users : Dict[relax.Var, List[relax.Binding]]\n        The consumer bindings of each binding variable.\n        Used for topological sort.\n\n    num_depending_vars : Dict[relax.Binding, int]\n        The number of variables each binding depends on.\n        Used for topological sort.\n    \"\"\"\n\n    # The mapping of the weight fetching bindings in each binary file.\n    # Here empty string means the weight is not in any binary file (e.g., cached\n    # sin and cos values for rotary embeddings).\n    binname2get_param_bindings: Dict[str, List[relax.Binding]] = {\"\": []}\n    # The set of binding variables.\n    binding_var_set: Set[relax.Var] = set()\n    var_users: Dict[relax.Var, List[relax.Binding]] = {}\n    num_depending_vars: Dict[relax.Binding, int] = {}\n\n    if func.attrs is not None and \"num_input\" in func.attrs:\n        num_input = func.attrs[\"num_input\"].value\n    else:\n        num_input = 0\n\n    # Sanity check on the function pattern.\n    assert isinstance(func.body, relax.SeqExpr)\n    assert len(func.body.blocks) == 1\n    assert isinstance(func.body.blocks[0], relax.DataflowBlock)\n    assert func.body.blocks[0].bindings[-1].var.same_as(func.body.body)\n\n    if isinstance(func.params[num_input].struct_info, relax.TupleStructInfo):\n        model_param_tuple = func.params[num_input]\n    else:\n        model_param_tuple = None\n        for i, var in enumerate(func.params[num_input:]):\n            binname = pidx2binname.get(i, var.name_hint)\n            if binname not in binname2get_param_bindings:\n                binname2get_param_bindings[binname] = []\n            binname2get_param_bindings[binname].append(var)\n\n    bindings = list(func.body.blocks[0].bindings)\n\n    # Go through each binding except the last one. (The last one is the output\n    # binding `gv = (lv, lv1, ...)`) which we ignore for analysis.\n    for binding in bindings[:-1]:\n        value = binding.value\n        binding_var_set.add(binding.var)\n        var_users[binding.var] = []\n\n        if (\n            model_param_tuple is not None\n            and isinstance(value, relax.TupleGetItem)\n            and value.tuple_value.same_as(model_param_tuple)\n        ):\n            # For weight fetching bindings (`lv = params[idx]`), we group them\n            # according to the binary file name.\n            pidx = value.index\n            if pidx not in pidx2binname:\n                binname2get_param_bindings[\"\"].append(binding)\n                continue\n\n            binname = pidx2binname[pidx]\n            if binname in binname2get_param_bindings:\n                binname2get_param_bindings[binname].append(binding)\n            else:\n                binname2get_param_bindings[binname] = [binding]\n        else:\n            # For other bindings, we collect the use-def information for\n            # topological sort.\n            num_depending_vars[binding] = 0\n\n            def fvisit(obj):\n                if isinstance(obj, relax.Var) and obj in binding_var_set:\n                    assert obj in var_users\n                    var_users[obj].append(binding)\n                    num_depending_vars[binding] += 1\n\n            relax.analysis.post_order_visit(value, fvisit)\n\n    # Get the weight fetching bindings in new order according to the group results.\n    get_param_bindings: List[relax.Binding] = []\n    for bindings in binname2get_param_bindings.values():\n        get_param_bindings += bindings\n\n    return get_param_bindings, var_users, num_depending_vars\n\n\ndef reorder_func(\n    func: relax.Function,\n    pidx2binname: Optional[Dict[int, str]] = None,\n) -> relax.Function:\n    \"\"\"Reorder the bindings of the input weight transform Relax function\n    according the weight location in binary files.\n\n    This function first analyzes the input function and gets the reordered\n    weight fetching bindings and the use-def information for topological sort.\n    It then reorders all bindings in the function with topological sort.\n\n    Parameters\n    ----------\n    func : relax.Function\n        The weight transform function to be analyzed.\n\n    pidx2binname : Optional[Dict[int, str]]\n\n        The mapping from each raw tensor index to the name of the\n        binary file where it resides.  If a relax dataflow graph has\n        multiple valid topological sorts, the order that minimizes the\n        number of simultaneously open files will be produced\n\n        If `None` (default), the existing order of relax bindings is\n        preserved in these cases.\n\n    Returns\n    -------\n    func_updated : relax.Function\n        The returned function where the bindings are updated with the new order.\n\n    \"\"\"\n\n    if pidx2binname is None:\n        pidx2binname = {}\n\n    bindings_to_visit = list(func.body.blocks[0].bindings)\n    param_lookup = {param: i for i, param in enumerate(func.params)}\n    binding_lookup = {}\n    previously_defined = set(func.params)\n    new_binding_order = []\n\n    param_tuple = None\n    if len(func.params) == 1 and isinstance(func.params[0].struct_info, relax.TupleStructInfo):\n        param_tuple = func.params[0]\n\n    def sort_key(i):\n        binding = bindings_to_visit[i]\n        upstream_vars = relax.analysis.free_vars(binding.value)\n\n        valid_ordering = all(var in previously_defined for var in upstream_vars)\n        last_param_used = max(\n            (param_lookup[var] for var in upstream_vars if var in param_lookup), default=-1\n        )\n        earliest_binding_used = min(\n            (binding_lookup[var] for var in upstream_vars if var in binding_lookup), default=-1\n        )\n        if (\n            param_tuple\n            and isinstance(binding.value, relax.TupleGetItem)\n            and binding.value.tuple_value.same_as(param_tuple)\n            and binding.value.index in pidx2binname\n        ):\n            tuple_param_group = pidx2binname[binding.value.index]\n        else:\n            tuple_param_group = \"\"\n\n        return [\n            # First, sort by valid orderings, so the min element will\n            # always be a binding that would be legal to use.\n            -valid_ordering,\n            # Next, sort by the function parameter used by this\n            # binding, in increasing order.  That way, we start by\n            # computing everything that required just the first\n            # parameter, then move on to variables that can be\n            # computed with the first two parameters, and so on.\n            last_param_used,\n            # Next, sort by the other bindings used.  This way, for\n            # variables that are only used as input in a single\n            # downstream binding, the variable's required live range\n            # is minimized.\n            -earliest_binding_used,\n            # Finally, if this is a `TupleGetItem(param_tuple, i)`,\n            # select the option that uses an already-open file.  This\n            # is mainly used relevant when loading from pytorch, which\n            # require loading the entire file at once.\n            tuple_param_group,\n        ]\n\n    while bindings_to_visit:\n        i_binding = min(range(len(bindings_to_visit)), key=sort_key)\n        binding = bindings_to_visit.pop(i_binding)\n\n        assert all(var in previously_defined for var in relax.analysis.free_vars(binding.value))\n        new_binding_order.append(binding)\n        previously_defined.add(binding.var)\n\n    assert len(new_binding_order) == len(func.body.blocks[0].bindings)\n\n    return relax.Function(\n        func.params,\n        relax.SeqExpr(\n            blocks=[relax.DataflowBlock(new_binding_order)],\n            body=func.body.body,\n        ),\n        func.ret_struct_info,\n        func.is_pure,\n        func.attrs,\n    )\n\n\n@tvm.transform.module_pass(opt_level=0, name=\"ReorderTransformFunc\")\nclass ReorderTransformFunc:\n    def __init__(self, pidx2binname: Optional[Dict[int, str]] = None):\n        if pidx2binname is None:\n            pidx2binname = {}\n        self.pidx2binname = pidx2binname\n\n    def transform_module(\n        self,\n        mod: IRModule,\n        ctx: tvm.transform.PassContext,\n    ) -> IRModule:\n        mod = mod.clone()\n        for gv, func in list(mod.functions.items()):\n            if isinstance(func, relax.Function) and func.attrs and \"global_symbol\" in func.attrs:\n                assert gv.name_hint.endswith(\"transform_params\")\n                func_updated = reorder_func(func, self.pidx2binname)\n                mod[gv] = func_updated\n        return mod\n\n# Path: mlc_llm/transform/rewrite_attention.py\nimport tvm\nfrom tvm.relax.dpl import PatternContext, is_const, is_op, rewrite_call, wildcard\nfrom tvm.script import relax as R\n\n\ndef rewrite_attention(use_flash_mqa=False):\n    @tvm.ir.transform.module_pass(opt_level=0, name=\"mlc_llm.transform.rewrite_attention\")\n    def ir_module_transform(mod: tvm.IRModule, context) -> tvm.IRModule:\n        Q = wildcard()\n        K = wildcard()\n        V = wildcard()\n\n        Q_BNSH = is_op(\"relax.permute_dims\")(Q)\n\n        if use_flash_mqa:\n            K_BNSH = is_op(\"relax.permute_dims\")(is_op(\"relax.repeat\")(K))\n            V_BNSH = is_op(\"relax.permute_dims\")(is_op(\"relax.repeat\")(V))\n        else:\n            K_BNSH = is_op(\"relax.permute_dims\")(K)\n            V_BNSH = is_op(\"relax.permute_dims\")(V)\n\n        K_BNSH_T = is_op(\"relax.permute_dims\")(K_BNSH)\n\n        matmul1 = is_op(\"relax.matmul\")(Q_BNSH, K_BNSH_T)\n        divide = is_op(\"relax.divide\")(matmul1, is_const())\n        max = is_op(\"relax.maximum\")(divide, is_const())\n        min = is_op(\"relax.minimum\")(max, wildcard())\n        softmax = is_op(\"relax.nn.softmax\")(is_op(\"relax.astype\")(min))\n        matmul2 = is_op(\"relax.matmul\")(is_op(\"relax.astype\")(softmax), V_BNSH)\n\n        pattern = is_op(\"relax.permute_dims\")(matmul2)\n\n        def callback(_, matchings):\n            return R.nn.attention(\n                matchings[Q], matchings[K], matchings[V], causal_mask=\"BottomRight\"\n            )\n\n        new_module = {}\n        for gvar, func in mod.functions.items():\n            if isinstance(func, tvm.relax.Function):\n                func = rewrite_call(pattern, callback, func)\n            new_module[gvar] = func\n\n        return tvm.IRModule(new_module, mod.type_definitions, mod.attrs, mod.global_infos)\n\n    return ir_module_transform\n\n# Path: mlc_llm/transform/set_entry_funcs.py\nimport re\n\nfrom typing import List, Union\n\nimport tvm\nfrom tvm.ir import GlobalVar\n\n\ndef SetEntryFuncs(*entry_funcs: List[Union[GlobalVar, str]]) -> tvm.ir.transform.Pass:\n    \"\"\"Update which functions are externally-exposed\n\n    All functions whose GlobalVar is contained `entry_funcs` list, or\n    whose name matches a regular expression in `entry_funcs`, are set\n    as externally exposed.  All other functions are set as internal.\n\n    This pass does not add or remove any functions from the\n    `IRModule`.  This pass may result in functions no longer being\n    used by any externally-exposed function.  In these cases, users\n    may use the `relax.transform.DeadCodeElimination` pass to remove\n    any unnecessary functions.\n\n    Parameters\n    ----------\n    entry_funcs: List[Union[GlobalVar, str]]\n\n        Specifies which functions that should be externally exposed,\n        either by GlobalVar or by regular expression.\n\n    Returns\n    -------\n    transform: tvm.ir.transform.Pass\n\n        The IRModule-to-IRModule transformation\n    \"\"\"\n\n    def is_entry_func(gvar: GlobalVar) -> bool:\n        for entry_func in entry_funcs:\n            if isinstance(entry_func, GlobalVar):\n                if entry_func.same_as(gvar):\n                    return True\n            elif isinstance(entry_func, str):\n                if re.fullmatch(entry_func, gvar.name_hint):\n                    return True\n            else:\n                raise TypeError(\n                    f\"SetEntryFuncs requires all arguments to be a GlobalVar or a str.  \"\n                    f\"However, argument {entry_func} has type {type(entry_func)}.\"\n                )\n\n    def is_exposed(func: tvm.ir.BaseFunc) -> bool:\n        return func.attrs is not None and \"global_symbol\" in func.attrs\n\n    @tvm.ir.transform.module_pass(opt_level=0, name=\"SetEntryFuncs\")\n    def transform(mod: tvm.IRModule, _pass_context) -> tvm.IRModule:\n        updates = {}\n        for gvar, func in mod.functions.items():\n            if is_entry_func(gvar):\n                if not is_exposed(func):\n                    updates[gvar] = func.with_attr(\"global_symbol\", gvar.name_hint)\n            else:\n                if is_exposed(func):\n                    updates[gvar] = func.without_attr(\"global_symbol\")\n\n        if updates:\n            mod = mod.clone()\n            mod.update(updates)\n\n        return mod\n\n    return transform\n\n# Path: mlc_llm/transform/transpose_matmul.py\nimport tvm\nfrom tvm import IRModule, relax, te, tir\nfrom tvm.relax.dpl.pattern import is_op, wildcard\n\n\n@relax.expr_functor.mutator\nclass TransposeMatmulCodeGenerator(relax.PyExprMutator):\n    def __init__(self, mod):\n        super().__init__(mod)\n\n    @staticmethod\n    def pattern():\n        w = wildcard()\n        x = wildcard()\n        wT = is_op(\"relax.permute_dims\")(w)\n        o = is_op(\"relax.matmul\")(x, wT)\n        annotations = {\"o\": o, \"w\": w, \"x\": x, \"wT\": wT}\n\n        def _check(context: relax.transform.PatternCheckContext) -> bool:\n            transpose_call = context.annotated_expr[\"wT\"]\n            ndim = transpose_call.args[0].struct_info.ndim\n            if ndim == -1:\n                return False\n            if ndim == 2 and transpose_call.attrs.axes is None:\n                return True\n            axes = list(range(ndim))\n            axes[-1], axes[-2] = axes[-2], axes[-1]\n            return list(transpose_call.attrs.axes) == axes\n\n        return o, annotations, _check\n\n    def visit_call_(self, call: relax.Call) -> relax.Expr:\n        out_dtype = None\n\n        def te_transposed_matmul(a: te.Tensor, b: te.Tensor) -> te.Tensor:\n            nonlocal out_dtype\n            a_shape = list(a.shape)\n            b_shape = list(b.shape)\n            a_prepended = False\n            b_appended = False\n            if len(a_shape) == 1:\n                a_prepended = True\n                a_shape.insert(0, 1)\n            if len(b_shape) == 1:\n                b_appended = True\n                b_shape.append(1)\n\n            is_a_larger = len(a_shape) > len(b_shape)\n            offset = len(a_shape) - len(b_shape) if is_a_larger else len(b_shape) - len(a_shape)\n\n            a_relax = relax.Var(\"a\", relax.TensorStructInfo(a.shape))\n            bT_shape = list(b.shape)\n            bT_shape[-1], bT_shape[-2] = bT_shape[-2], bT_shape[-1]\n            bT_relax = relax.Var(\"b\", relax.TensorStructInfo(bT_shape))\n            output_shape = self.builder_.normalize(\n                relax.op.matmul(a_relax, bT_relax)\n            ).struct_info.shape\n\n            def matmul_compute(*idx_spatial):\n                k = te.reduce_axis((0, a_shape[-1]), name=\"k\")\n\n                def multiply_compute(idx_reduce):\n                    a_indices = []\n                    b_indices = []\n\n                    for i in range(offset):\n                        if is_a_larger:\n                            a_indices.append(idx_spatial[i])\n                        else:\n                            b_indices.append(idx_spatial[i])\n                    for i in range(offset, len(output_shape) - (2 - a_prepended - b_appended)):\n                        a_dim = a_shape[i if is_a_larger else i - offset]\n                        b_dim = b_shape[i if not is_a_larger else i - offset]\n                        dim_equal = a_dim == b_dim\n                        if not isinstance(dim_equal, tir.IntImm) or dim_equal == 0:\n                            a_dim_is_one = isinstance(a_dim, tir.IntImm) and a_dim == 1\n                            b_dim_is_one = isinstance(b_dim, tir.IntImm) and b_dim == 1\n                            a_indices.append(0 if a_dim_is_one else idx_spatial[i])\n                            b_indices.append(0 if b_dim_is_one else idx_spatial[i])\n                        else:\n                            a_indices.append(idx_spatial[i])\n                            b_indices.append(idx_spatial[i])\n\n                    if not a_prepended:\n                        a_indices.append(idx_spatial[-2 + b_appended])\n                    a_indices.append(idx_reduce)\n                    if not b_appended:\n                        b_indices.append(idx_spatial[-1])\n                    b_indices.append(idx_reduce)\n\n                    dtype = out_dtype\n                    if dtype != \"\":\n                        return a(*a_indices).astype(dtype) * b(*b_indices).astype(dtype)\n                    return a(*a_indices) * b(*b_indices)\n\n                return te.sum(multiply_compute(k), axis=k)\n\n            return te.compute(\n                output_shape,\n                lambda *idx: matmul_compute(*idx),  # pylint: disable=unnecessary-lambda\n                name=\"NT_matmul\",\n            )\n\n        if isinstance(call.op, relax.GlobalVar):\n            function = self.builder_.get()[call.op]\n            if (\n                function.attrs\n                and \"Composite\" in function.attrs\n                and function.attrs[\"Composite\"] == \"transpose_matmul_fuse\"\n            ):\n                out_dtype = function.ret_struct_info.dtype\n                return self.builder_.call_te(\n                    te_transposed_matmul,\n                    call.args[1],\n                    call.args[0],\n                    primfunc_name_hint=\"NT_matmul\",\n                )\n\n        return super().visit_call_(call)\n\n\n@tvm.transform.module_pass(opt_level=0, name=\"FuseTransposeMatmul\")\nclass FuseTransposeMatmul:\n    def transform_module(self, mod: IRModule, ctx: tvm.transform.PassContext) -> IRModule:\n        mod = relax.transform.FuseOpsByPattern(\n            [(\"transpose_matmul_fuse\", *TransposeMatmulCodeGenerator.pattern())]\n        )(mod)\n\n        transpose_matmul_codegen = TransposeMatmulCodeGenerator(mod)\n        for gv in mod.functions:\n            func = mod[gv]\n            if not isinstance(func, relax.Function):\n                continue\n            func = transpose_matmul_codegen.visit_expr(func)\n            transpose_matmul_codegen.builder_.update_func(gv, func)\n\n        return transpose_matmul_codegen.builder_.get()\n\n@relax.expr_functor.mutator\nclass Transpose1MatmulCodeGenerator(relax.PyExprMutator):\n    def __init__(self, mod):\n        super().__init__(mod)\n\n    @staticmethod\n    def pattern():\n        w = wildcard()\n        x = wildcard()\n        xT = is_op(\"relax.permute_dims\")(x)\n        wT = is_op(\"relax.permute_dims\")(w)\n        o = is_op(\"relax.matmul\")(xT, wT)\n        annotations = {\"o\": o, \"w\": w, \"x\": x, \"xT\": xT, \"wT\": wT}\n\n        def _check(context: relax.transform.PatternCheckContext) -> bool:\n            x_transpose_call = context.annotated_expr[\"o\"]\n            w_transpose_call = context.annotated_expr[\"o\"]\n            x_shape = context.annotated_expr[\"x\"].struct_info.shape\n            w_shape = context.annotated_expr[\"w\"].struct_info.shape\n            xT_shape = x_transpose_call.args[0].struct_info.shape\n            wT_shape = w_transpose_call.args[1].struct_info.shape\n\n            if not (\n                xT_shape[0] == x_shape[0] and xT_shape[1] == x_shape[2]\n                and xT_shape[2] == x_shape[1] and xT_shape[3] == x_shape[3]\n            ):\n                return False\n\n            if not (\n                wT_shape[0] == w_shape[0] and wT_shape[1] == w_shape[2]\n                and wT_shape[2] == w_shape[3] and wT_shape[3] == w_shape[1]\n            ):\n                return False\n\n            return True\n\n        return o, annotations, _check\n\n    def visit_call_(self, call: relax.Call) -> relax.Expr:\n        out_dtype = None\n\n        def te_transposed_matmul(a: te.Tensor, b: te.Tensor) -> te.Tensor:\n            nonlocal out_dtype\n            a_shape = list(a.shape)\n            b_shape = list(b.shape)\n\n            aT_shape = list(a.shape)\n            aT_shape[-2], aT_shape[-3] = aT_shape[-3], aT_shape[-2]\n            aT_relax = relax.Var(\"a\", relax.TensorStructInfo(aT_shape))\n            bT_shape = list(b.shape)\n            bT_shape[-1], bT_shape[-2], bT_shape[-3] = bT_shape[-3], bT_shape[-1], bT_shape[-2]\n            bT_relax = relax.Var(\"b\", relax.TensorStructInfo(bT_shape))\n            output_shape = self.builder_.normalize(\n                relax.op.matmul(aT_relax, bT_relax)\n            ).struct_info.shape\n            def matmul_compute(*idx_spatial):\n                k = te.reduce_axis((0, a_shape[-1]), name=\"k\")\n                def multiply_compute(idx_reduce):\n                    a_indices = [idx_spatial[0], idx_spatial[2], idx_spatial[1], idx_reduce]\n                    b_indices = [idx_spatial[0], idx_spatial[3], idx_spatial[1], idx_reduce]\n                    dtype = out_dtype\n                    if dtype != \"\":\n                        return a(*a_indices).astype(dtype) * b(*b_indices).astype(dtype)\n                    return a(*a_indices) * b(*b_indices)\n\n                return te.sum(multiply_compute(k), axis=k)\n\n            return te.compute(\n                output_shape,\n                lambda *idx: matmul_compute(*idx),  # pylint: disable=unnecessary-lambda\n                name=\"NT_matmul\",\n            )\n\n        if isinstance(call.op, relax.GlobalVar):\n            function = self.builder_.get()[call.op]\n            if (\n                \"Composite\" in function.attrs\n                and function.attrs[\"Composite\"] == \"transpose1_matmul_fuse\"\n            ):\n                out_dtype = function.ret_struct_info.dtype\n                return self.builder_.call_te(\n                    te_transposed_matmul,\n                    call.args[0],\n                    call.args[1],\n                    primfunc_name_hint=\"NT_matmul\",\n                )\n\n        return super().visit_call_(call)\n\n\n@tvm.transform.module_pass(opt_level=0, name=\"FuseTranspose1Matmul\")\nclass FuseTranspose1Matmul:\n    def transform_module(\n        self, mod: IRModule, ctx: tvm.transform.PassContext\n    ) -> IRModule:\n        mod = relax.transform.FuseOpsByPattern(\n            [(\"transpose1_matmul_fuse\", *Transpose1MatmulCodeGenerator.pattern())]\n        )(mod)\n\n        transpose_matmul_codegen = Transpose1MatmulCodeGenerator(mod)\n        for gv in mod.functions:\n            func = mod[gv]\n            if not isinstance(func, relax.Function):\n                continue\n            func = transpose_matmul_codegen.visit_expr(func)\n            transpose_matmul_codegen.builder_.update_func(gv, func)\n\n        return transpose_matmul_codegen.builder_.get()\n\n\n@relax.expr_functor.mutator\nclass Transpose2MatmulCodeGenerator(relax.PyExprMutator):\n    def __init__(self, mod):\n        super().__init__(mod)\n\n    @staticmethod\n    def pattern():\n        w = wildcard()\n        x = wildcard()\n        wT = is_op(\"relax.permute_dims\")(w)\n        o = is_op(\"relax.permute_dims\")(is_op(\"relax.matmul\")(x, wT))\n        #oT = is_op(\"relax.permute_dims\")(o)\n        annotations = {\"o\": o, \"w\": w, \"x\": x, \"wT\": wT}\n\n        def _check(context: relax.transform.PatternCheckContext) -> bool:\n            w_transpose_call = context.annotated_expr[\"wT\"]\n            w_shape = w_transpose_call.args[0].struct_info.shape\n            wT_shape = w_transpose_call.struct_info.shape\n            oT_call = context.annotated_expr[\"o\"]\n            o_shape = oT_call.args[0].struct_info.shape\n            oT_shape = oT_call.struct_info.shape\n\n            if not (\n                wT_shape[0] == w_shape[0] and wT_shape[1] == w_shape[2]\n                and wT_shape[2] == w_shape[1] and wT_shape[3] == w_shape[3]\n            ):\n                return False\n\n            if not (\n                oT_shape[0] == o_shape[0] and oT_shape[1] == o_shape[2]\n                and oT_shape[2] == o_shape[1] and oT_shape[3] == o_shape[3]\n            ):\n                return False\n\n            return True\n\n        return o, annotations, _check\n\n    def visit_call_(self, call: relax.Call) -> relax.Expr:\n        out_dtype = None\n\n        def te_transposed_matmul(a: te.Tensor, b: te.Tensor) -> te.Tensor:\n            nonlocal out_dtype\n            a_shape = list(a.shape)\n            b_shape = list(b.shape)\n            output_shape = [a_shape[0], b_shape[-2], a_shape[2], a_shape[3]]\n            def matmul_compute(*idx_spatial):\n                k = te.reduce_axis((0, b_shape[-1]), name=\"k\")\n                def multiply_compute(idx_reduce):\n                    a_indices = [idx_spatial[0], idx_reduce, idx_spatial[2], idx_spatial[3]]\n                    b_indices = [idx_spatial[0], idx_spatial[2], idx_spatial[1], idx_reduce]\n\n                    dtype = out_dtype\n                    if dtype != \"\":\n                        return a(*a_indices).astype(dtype) * b(*b_indices).astype(dtype)\n                    return a(*a_indices) * b(*b_indices)\n\n                return te.sum(multiply_compute(k), axis=k)\n\n            return te.compute(\n                output_shape,\n                lambda *idx: matmul_compute(*idx),  # pylint: disable=unnecessary-lambda\n                name=\"NT_matmul\",\n            )\n\n        if isinstance(call.op, relax.GlobalVar):\n            function = self.builder_.get()[call.op]\n            if (\n                \"Composite\" in function.attrs\n                and function.attrs[\"Composite\"] == \"transpose2_matmul_fuse\"\n            ):\n                out_dtype = function.ret_struct_info.dtype\n                #NT_output_shape = function.ret_struct_info.shape\n                return self.builder_.call_te(\n                    te_transposed_matmul,\n                    call.args[0],\n                    call.args[1],\n                    primfunc_name_hint=\"NT_matmul\",\n                )\n\n        return super().visit_call_(call)\n\n\n@tvm.transform.module_pass(opt_level=0, name=\"FuseTranspose2Matmul\")\nclass FuseTranspose2Matmul:\n    def transform_module(\n        self, mod: IRModule, ctx: tvm.transform.PassContext\n    ) -> IRModule:\n        mod = relax.transform.FuseOpsByPattern(\n            [(\"transpose2_matmul_fuse\", *Transpose2MatmulCodeGenerator.pattern())]\n        )(mod)\n\n        transpose_matmul_codegen = Transpose2MatmulCodeGenerator(mod)\n        for gv in mod.functions:\n            func = mod[gv]\n            if not isinstance(func, relax.Function):\n                continue\n            func = transpose_matmul_codegen.visit_expr(func)\n            transpose_matmul_codegen.builder_.update_func(gv, func)\n\n        return transpose_matmul_codegen.builder_.get()\n\n# Path: mlc_llm/transform/__init__.py\nfrom .clean_up_tir_attrs import CleanUpTIRAttrs\nfrom .decode_matmul_ewise import FuseDecodeMatmulEwise\nfrom .decode_take import FuseDecodeTake\nfrom .decode_transpose import FuseDecodeTranspose\nfrom .fuse_split_rotary_embedding import fuse_split_rotary_embedding\nfrom .lift_tir_global_buffer_alloc import LiftTIRGlobalBufferAlloc\nfrom .reorder_transform_func import ReorderTransformFunc\nfrom .rewrite_attention import rewrite_attention\nfrom .transpose_matmul import FuseTransposeMatmul, FuseTranspose1Matmul, FuseTranspose2Matmul\nfrom .set_entry_funcs import SetEntryFuncs\n\n# Path: mlc_llm/relax_model/param_manager.py\nimport json\nimport os\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union\n\nimport tvm\nfrom torch import Tensor as torchTensor\nfrom tvm import relax, tir\nfrom tvm._ffi.runtime_ctypes import Device\nfrom tvm.relax.analysis import remove_all_unused\nfrom tvm.relax.expr import Expr, Function, Var\nfrom tvm.relax.expr_functor import PyExprMutator, mutator\nfrom tvm.relax.testing import nn\n\nfrom .. import quantization\nfrom .modules import named_parameters\nfrom ..transform import ReorderTransformFunc\n\n\ndef f_default_compute_relax_param(relax_pname: str, torch_params: List[Any]) -> Any:\n    \"\"\"The defualt `f_compute_relax_param` for ParamManager.\n    See ParamManager for more details.\n    \"\"\"\n    raise NotImplementedError()\n\n\nclass Parameter:\n    \"\"\"The abstraction of weight tensors (e.g., linear layer weight, embedding\n    table, etc.) in a model.\n\n    Attributes\n    ----------\n    name : str\n        The name of the parameter.\n        The name of a weight is got by `named_parameters()` method, similar to\n        PyTorch's `named_parameters()` function.\n        An example name is `model.layers.11.self_attn.k_proj.weight`.\n        In a model, the name is the **unique** identifier of a parameter.\n\n    param_info_dict : Dict[str, relax.TensorStructInfo]\n        The shape and dtype of the parameter in each function.\n        The shape can be accessed by `param_info_dict[func_name].shape`, which is\n        a relax.ShapeExpr instance.\n        And the dtype can be accessed by `param_info_dict[func_name].dtype`,\n        which is a Python string.\n\n    quant_spec : quantization.QuantizationSpec\n        The quantization specification of this parameter.\n        It specifies the algorithm to quantize and dequantize this parameter (or\n        this parameter does not need quantization).\n\n    shard_dim : Optional[int]\n        The dimension to be sharded.\n\n    shard_strategy : Optional[str]\n        The strategy to shard the parameter.\n    \"\"\"\n\n    name: str\n    param_info_dict: Dict[str, relax.TensorStructInfo]\n    quant_spec: quantization.QuantizationSpec\n    shard_dim: Optional[int]\n    shard_strategy: Optional[str]\n\n    def __init__(\n        self,\n        name: str,\n        quant_spec: quantization.QuantizationSpec,\n        shard_dim: Optional[int],\n        shard_strategy: Optional[str],\n    ) -> None:\n        self.name = name\n        self.param_info_dict = dict()\n        self.quant_spec = quant_spec\n        self.shard_dim = shard_dim\n        self.shard_strategy = shard_strategy\n\n    def register_func(self, func_name: str, param_info: relax.TensorStructInfo):\n        self.param_info_dict[func_name] = param_info\n\n    @property\n    def param_info(self):\n        \"\"\"Return the shape and dtype of the parameter (in some arbitrary function).\"\"\"\n        return next(iter(self.param_info_dict.values()))\n\n\nclass ParamManager:\n    \"\"\"The model-wise data structure which contains the information of every\n    weight in the model and is in charge of applying quantization and dequantization\n    to the parameters at the entire model level.\n\n    Attributes\n    ----------\n    params : Dict[str, Parameter]\n        The mapping from parameter names to parameters.\n\n    param_names : List[str]\n        The name list of all the parameters.\n        To enforce a unique order or all the parameters for determinism, the\n        parameter names are kept in the list, and the parameter order is\n        uniquely determined by the parameter name list.\n\n    func_raw_param_map : Dict[relax.Var, Tuple[str, Parameter]]\n        The mapping from each relax.Var that denotes a weight parameter to the\n        name of the function the var is in (e.g., \"prefill\" or \"decode\"), and\n        the Parameter it corresponds to.\n        This mapping is used for applying quantization transformation to the\n        Relax functions (e.g., the \"prefill\", \"decode\", etc.) in the model.\n\n    param2qrange : Dict[Parameter, range]\n        The mapping from each parameter to the range of its quantized tensors\n        in the list of quantized tensors of all parameters.\n        Each parameter is quantized into multiple tensors.\n        For example, assume we have parameters `p0`, `p1`, `p2`.\n        - `p0` is quantized into `t0_0`, `t0_1`,\n        - `p1` is quantized into `t1_0`, and\n        - `p2` is quantized into `t2_0`, `t2_1` and `t2_2`.\n        Then the list of all quantized tensors is `[t0_0, t0_1, t1_0, t2_0, t2_1, t2_2]`,\n        and the dict `param2qrange` is\n        `{p0: range(0, 2), p1: range(2, 3), p2: range(3, 6)}`.\n\n    f_convert_pname_fwd : Callable[[str], List[str]]\n        The function which converts Relax parameter name (ours) to torch's\n        parameter names, suggesting \"to load this Relax parameter, which torch\n        parameter(s) are needed\".\n        - Usually, the function maps a name to itself. For example, in LLaMA we\n        map `lm_head.weight` itself, as the parameter has the same name on both\n        Relax side and torch side.\n        - In some cases we map a name to multiple names. For example, if we\n        support combined QKV computing when the torch side separates them, on\n        Relax side we only have one QKV weight, while on torch side we have\n        one weight for each of Q, K, V. In this case, we map one name to three\n...\n", "needle_token_start": 4008, "needle_token_end": 4086, "code_context_ntokens": 16163, "output": ["```python\ndef contain_symbolic_var(tensor_sinfo: relax.TensorStructInfo) -> bool:\n    assert isinstance(tensor_sinfo.shape, relax.ShapeExpr)\n    for v in tensor_sinfo.shape.values:\n        if not isinstance(v, tir.IntImm):\n            return True\n    return False\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "get_param_set_item", "language": "python", "path": "mlc_llm/relax_model/param_manager.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide a mechanism for lazy loading of parameters in a machine learning model, specifically for use with a parameter transformation and loading framework. It facilitates the registration of a function that can be used to set parameters at specific indices during model execution.\n2. **Input**: This function does not take any direct inputs when called.\n3. **Output**: It returns a tuple containing a callable function and a list. The callable function is used to set a parameter at a specified index, and the list is intended to store these parameters once they are loaded and set.\n4. **Procedure**: \n   - Initializes an empty list to store parameters.\n   - Defines a nested function that takes an index and a parameter. This function checks if the list is long enough to accommodate the parameter at the given index, extends the list if necessary, and then sets the parameter at the specified index.\n   - Returns the nested function and the initially empty list. The list is populated with parameters as the nested function is called during the execution of a compiled function in a machine learning framework.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    f_convert_param_bkwd: Callable[[str, Any], Optional[List[Tuple[str, Any]]]]\n    f_compute_relax_param: Callable[[str, List[Any]], Any]\n    f_run_prequantize: Optional[Callable[[str], str]]\n\n    model_path: str\n    use_safetensors: bool\n    safetensors_load_func: Callable[[Union[str, os.PathLike], str], Dict[str, torchTensor]]\n    pidx2pname: Dict[int, str]\n    torch_pname2binname: Dict[str, str]\n\n    def __init__(self) -> None:\n        self.params = {}\n        self.param_names = []\n        self.params_in_func = {}\n\n        self.func_raw_param_map = {}\n        self.param2qrange = None\n\n        self.nparam_to_load = None\n        self.f_convert_pname_fwd = None\n        self.f_convert_param_bkwd = None\n        self.f_compute_relax_param = None\n        self.f_run_prequantize = None\n\n        self.qspec_updater_classes = []\n\n    def register_params(\n        self,\n        model: nn.Module,\n        func_name: str,\n        quantization_scheme: quantization.QuantizationScheme,\n        f_get_param_quant_kind: Callable[\n            [str, relax.TensorStructInfo], quantization.ParamQuantKind\n        ],\n    ) -> None:\n        \"\"\"Register the parameters of the input model (within the context of the\n        input function) in the parameter manager.\n\n        Parameters\n        ----------\n        model : nn.Module\n            The input model whose parameters are registered.\n\n        func_name : str\n            The name of the function the input model is in.\n            For example, the \"prefill\" function or the \"decode\" function.\n\n        quantization_scheme : quantization.QuantizationScheme\n            The quantization scheme of the input model, which describes how\n            to quantize the model.\n\n        f_get_param_quant_kind: Callable[[str, relax.TensorStructInfo], quantization.ParamQuantKind]\n            A function which takes the name and StructInfo (effectively shape\n            and dtype) of a parameter, and returns which quantization kind this\n            parameter uses.\n            This is used for applying quantization to the parameters.\n        \"\"\"\n        if quantization_scheme.qspec_updater_class is not None:\n            self.qspec_updater_classes.append(quantization_scheme.qspec_updater_class)\n        if quantization_scheme.f_convert_param_bkwd is not None:\n            self.f_convert_param_bkwd = quantization_scheme.f_convert_param_bkwd\n        if quantization_scheme.f_compute_relax_param is not None:\n            self.f_compute_relax_param = quantization_scheme.f_compute_relax_param\n        if quantization_scheme.f_run_prequantize is not None:\n            self.f_run_prequantize = quantization_scheme.f_run_prequantize\n\n        self.params_in_func[func_name] = []\n        # For each parameter in the input model, get its quantization kind and\n        # register the parameter with its name and quantization kind.\n        for name, relax_param in named_parameters(model).items():\n            quant_kind = f_get_param_quant_kind(name, relax_param.struct_info)\n            param = self._register_param(\n                name,\n                relax_param,\n                getattr(quantization_scheme, quant_kind.name),\n                func_name,\n                relax_param.__dict__.get(\"shard_dim\", None),\n                relax_param.__dict__.get(\"shard_strategy\", None),\n            )\n\n            self.params_in_func[func_name].append(param)\n\n    def run_pre_quantize(self, model_path: str):\n        if self.f_run_prequantize is not None:\n            model_path = self.f_run_prequantize(model_path)\n\n        self.model_path = model_path\n        return model_path\n\n    def init_torch_pname_to_bin_name(self, use_safetensors: bool):\n        assert hasattr(self, \"model_path\"), (\n            \"Must call either set_param_loading_func or run_pre_quantize \"\n            \"before init_torch_pname_to_bin_name\"\n        )\n\n        if self.pidx2pname:\n            mapping = load_torch_pname2binname_map(\n                self.model_path,\n                use_safetensors,\n                set(self.pidx2pname.values()),\n                self.f_convert_pname_fwd,\n            )\n        else:\n            mapping = {}\n\n        self.torch_pname2binname = mapping\n\n    def set_param_loading_func(\n        self,\n        model_path: str,\n        use_safetensors: bool,\n        f_convert_pname_fwd: Callable[[str], List[str]] = lambda pname: [pname],\n        f_convert_param_bkwd: Callable[\n            [str, Any], Optional[List[Tuple[str, Any]]]\n        ] = lambda pname, torch_param: [(pname, torch_param)],\n        f_compute_relax_param: Callable[[str, List[Any]], Any] = f_default_compute_relax_param,\n        *,\n        no_lazy_param_loading: bool = False,\n    ) -> None:\n        \"\"\"Set the parameter loading functions.\n\n        Parameters\n        ----------\n        model_path : str\n            The path of the Hugging Face model on disk.\n\n        use_safetensors : bool\n            Whether to use ``.safetensors`` instead of ``.bin`` to load model.\n\n        f_convert_pname_fwd : Callable[[str], List[str]]\n            The function which converts Relax parameter name (ours) to torch's\n            parameter names. See the document of ParamManager for more details.\n\n        f_convert_param_bkwd : Callable[[str, Any], Optional[List[Tuple[str, Any]]]]\n            The function which converts torch parameter and param name back to\n            Relax parameters with names. `Any` here stands for numpy.ndarray.\n            See the document of ParamManager for more details.\n\n        f_compute_relax_param : Callable[[str, List[Any]], Any]\n            The function which computes a Relax parameter from a list of torch\n            parameters. `Any` here stands for numpy.ndarray.\n            See the document of ParamManager for more details.\n\n        no_lazy_param_loading : bool\n            A boolean indicating that no lazy parameter loading from torch is needed.\n            This needs to be set as True when all the model weights are loaded\n            at the time of constructing the model.\n        \"\"\"\n        self.f_convert_pname_fwd = f_convert_pname_fwd\n        if self.f_convert_param_bkwd is None:\n            self.f_convert_param_bkwd = f_convert_param_bkwd\n        if self.f_compute_relax_param is None:\n            self.f_compute_relax_param = f_compute_relax_param\n\n        self.model_path = model_path\n        self.use_safetensors = use_safetensors\n        if self.use_safetensors:\n            # Use a pointer here to prevent repeated import in tvm registered function\n            from safetensors.torch import (\n                load_file,  # pylint: disable=import-outside-toplevel\n            )\n\n            def load_safetensors_func(*args):\n                params = load_file(*args)\n                for name, param in params.items():\n                    dtype = str(param.dtype)\n                    if dtype == \"torch.bfloat16\":\n                        param = param.float()\n                    params[name] = param\n                return params\n\n            self.safetensors_load_func = load_safetensors_func\n\n        pnames_to_load = []\n        for param_name in self.param_names:\n            param = self.params[param_name]\n            loaded_names, _ = param.quant_spec.get_loaded_tensor_info(param_name, param.param_info)\n            pnames_to_load += loaded_names\n\n        self.nparam_to_load = len(pnames_to_load)\n        if not no_lazy_param_loading:\n            self.pidx2pname = {pidx: pname for pidx, pname in enumerate(pnames_to_load)}\n        else:\n            self.pidx2pname = dict()\n\n    def transform_dequantize(self) -> tvm.ir.transform.Pass:\n        \"\"\"Apply dequantization to the input IRModule.\n\n        Parameters\n        ----------\n        mod : tvm.IRModule\n            The input IRModule to be applied dequantization.\n            The IRModule contains all the constructed Relax functions\n            (e.g., the \"prefill\"/\"decode\" functions) and is expected to\n            have all of its parameters registered in the ParamManager.\n\n        Returns\n        -------\n        updated_mod : tvm.IRModule\n            The IRModule updated with the dequantization computation.\n        \"\"\"\n\n        @tvm.ir.transform.module_pass(opt_level=0, name=\"ParamManager.transform_dequantize\")\n        def transform_func(mod: tvm.IRModule, _context) -> tvm.IRModule:\n            # For each Relax function in the input IRModule (e.g., \"prefill\"),\n            # we create its input relax.Var of all the quantized data, and\n            # store the mapping from function name to the var.\n            func_name_to_quantized_params: Dict[str, List[relax.Var]] = {}\n\n            for gv, func in mod.functions.items():\n                if isinstance(func, relax.Function) and func.attrs and \"num_input\" in func.attrs:\n                    func_name_to_quantized_params[gv.name_hint] = self.get_quantized_params(\n                        gv.name_hint\n                    )\n\n            # Cache mapping to avoid duplicate dequantization.\n            dequantized_cache: Dict[relax.Var, relax.Var] = {}\n\n            # Define a var replacement function for applying dequantization.\n            def f_replace(var: relax.Var, bb: relax.BlockBuilder) -> relax.Var:\n                if var in dequantized_cache:\n                    return dequantized_cache[var]\n                assert var in self.func_raw_param_map\n\n                func_name, param = self.func_raw_param_map[var]\n                quantized_params = func_name_to_quantized_params[func_name]\n                relevant_quantized_params = [quantized_params[i] for i in self.param2qrange[param]]\n\n                dequantized = self._dequantize(param, relevant_quantized_params, bb, func_name)\n\n                dequantized_cache[var] = dequantized\n                return dequantized\n\n            # Create the function mutator for applying dequantization.\n            replacer = ParamReplacer(mod, func_name_to_quantized_params, f_replace)\n            # Update the input IRModule with dequantization.\n            mod = replacer.transform()\n\n            return mod\n\n        return transform_func\n\n    def get_quantized_params(self, func_name: str) -> List[relax.Var]:\n        quantized_params: List[relax.Var] = []\n\n        bb = relax.BlockBuilder()\n        with bb.function(\"main\", []):\n            self.param2qrange = dict()\n\n            for name in self.param_names:\n                param = self.params[name]\n                param_info = None\n                if func_name in param.param_info_dict:\n                    param_info = param.param_info_dict[func_name]\n                else:\n                    param_info = relax.TensorStructInfo(\n                        tvm.ir.load_json(tvm.ir.save_json(param.param_info.shape)),\n                        param.param_info.dtype,\n                    )\n\n                loaded_tensor_names, loaded_tensor_info = param.quant_spec.get_loaded_tensor_info(\n                    name, param_info\n                )\n\n                provided_tensor_vars: List[relax.Var] = [\n                    relax.Var(name, sinfo)\n                    for name, sinfo in zip(loaded_tensor_names, loaded_tensor_info)\n                ]\n\n                # Get the quantization function of this parameter.\n                f_quantize = param.quant_spec.get_quantize_func(param_info)\n                if f_quantize is None:\n                    # If the parameter does not have a quantization function, either it\n                    # does not need quantization or it is pre-quantized.\n                    self.param2qrange[param] = range(\n                        len(quantized_params),\n                        len(quantized_params) + len(provided_tensor_vars),\n                    )\n                    quantized_params.extend(provided_tensor_vars)\n                else:\n                    # If the parameter has a quantization function, it is not expected\n                    # to be pre-quantized.\n                    assert len(provided_tensor_vars) == 1, (\n                        \"A parameter with quantization function is not expected \"\n                        \"to be pre-quantized.\"\n                    )\n\n                    # Apply the quantization function.\n                    quantized_data = bb.normalize(f_quantize(bb, provided_tensor_vars))\n                    if isinstance(quantized_data.struct_info, relax.TupleStructInfo):\n                        fields = quantized_data.struct_info.fields\n                        n_tensor = len(fields)\n                        assert n_tensor > 1\n                        # Record the range of quantized tensors of this parameter.\n                        self.param2qrange[param] = range(\n                            len(quantized_params),\n                            len(quantized_params) + n_tensor,\n                        )\n                        # Collect the quantized tensors to return.\n                        quantized_params.extend(\n                            relax.Var(f\"{name}.{field.dtype}.{i}\", field)\n                            for i, field in enumerate(fields)\n                        )\n\n                    else:\n                        field = quantized_data.struct_info\n                        assert isinstance(field, relax.TensorStructInfo)\n                        self.param2qrange[param] = range(\n                            len(quantized_params), len(quantized_params) + 1\n                        )\n                        quantized_params.append(relax.Var(f\"{name}.{field.dtype}\", field))\n            bb.emit_func_output(relax.const(0, \"int64\"))\n\n        return quantized_params\n\n    def get_param_get_item(\n        self, device: Device, model_params: List[Optional[tvm.nd.NDArray]] = []\n    ) -> Callable:\n        \"\"\"A wrapper function which returns the `get_item`\n        functions for parameter lazy loading.\n\n        The return value of this function is intended to be registered\n        as `\"get_item\"`, for use in a module built with\n        `LazyTransformParams`.\n\n        .. code-block:: python\n\n            get_item = manager.get_param_get_item(tvm.cuda())\n            tvm.register_func(func_name=\"get_item\", f=get_item, override=True)\n            compiled_function()\n\n        Parameters\n        ----------\n        device : Device\n\n            The device onto which tensor parameters should be loaded.\n\n        model_params : List[Optional[tvm.nd.NDArray]]\n\n            Any pre-loaded model parameters.  For parameter at index\n            `i`, if `model_params[i]` already contains an array, that\n            array will be returned from `get_item`.  Otherwise, the\n            parameter will be loaded either from disk, or from an\n            internal cache.\n\n        Returns\n        -------\n        get_item: Callable[[int], tvm.nd.NDArray]\n\n            A function that accepts an index, and returns the tensor\n            parameter located at that index, loaded onto `device`.\n\n        \"\"\"\n        import torch  # pylint: disable=import-outside-toplevel\n\n        assert self.f_convert_pname_fwd is not None\n        assert self.f_convert_param_bkwd is not None\n        assert self.f_compute_relax_param is not None\n        pname2pidx: Dict[str, int] = {pname: pidx for pidx, pname in self.pidx2pname.items()}\n\n        # The set of indices of loaded parameters, serving for\n        # robustness guarantee to avoid one parameter being loaded for\n        # multiple times.\n        loaded_idx_set: Set[int] = set()\n\n        # The set of torch binary filenames, serving for robustness guarantee\n        # to avoid one torch binary file being loaded for multiple times.\n        loaded_torch_bins: Set[str] = set()\n\n        # The set of cached Relax parameters.\n        cached_relax_params: Dict[int, tvm.nd.NDArray] = {}\n\n        # The set of cached torch parameters. `Any` here stands for\n        # numpy.ndarray.\n        cached_torch_params: Dict[str, Any] = {}\n\n        device_cpu = tvm.cpu()\n\n        def fetch_torch_param(torch_param):\n            if str(torch_param.dtype) == \"torch.bfloat16\":\n                # Convert to float32 first.\n                return torch_param.detach().cpu().float().numpy()\n            else:\n                return torch_param.detach().cpu().numpy()\n\n        def load_torch_params_from_bin(torch_binname: str):\n            torch_binpath = os.path.join(self.model_path, torch_binname)\n            torch_params = None\n            if self.use_safetensors:\n                torch_params = self.safetensors_load_func(torch_binpath)\n            else:\n                torch_params = torch.load(\n                    torch_binpath,\n                    map_location=torch.device(\"cpu\"),\n                )\n            torch_param_names = list(torch_params.keys())\n            for torch_param_name in torch_param_names:\n                torch_param = fetch_torch_param(torch_params[torch_param_name])\n                del torch_params[torch_param_name]\n\n                relax_params = self.f_convert_param_bkwd(torch_param_name, torch_param)\n                if relax_params is not None:\n                    for param_name, param in relax_params:\n                        if param_name not in pname2pidx.keys():\n                            continue\n                        pidx = pname2pidx[param_name]\n                        assert pidx not in cached_relax_params\n                        cached_relax_params[pidx] = tvm.nd.array(param, device_cpu)\n                else:\n                    assert torch_param_name not in cached_torch_params\n                    cached_torch_params[torch_param_name] = torch_param\n                del torch_param\n\n        def get_item(i):\n            # If the weight is already provided by `model_params`, directly use it\n            # and no need to load from binary file.\n            if model_params and len(model_params) > i and model_params[i] is not None:\n                assert i not in cached_relax_params\n                return tvm.nd.array(model_params[i], device=device)\n\n            # Otherwise, we load the weight from its corresponding binary file.\n            assert i in self.pidx2pname\n            relax_pname = self.pidx2pname[i]\n            torch_pnames = self.f_convert_pname_fwd(relax_pname)\n\n            if i not in cached_relax_params:\n                for torch_binname in [\n                    self.torch_pname2binname[torch_pname] for torch_pname in torch_pnames\n                ]:\n                    if torch_binname in loaded_torch_bins:\n                        continue\n                    load_torch_params_from_bin(torch_binname)\n                    loaded_torch_bins.add(torch_binname)\n\n            if i not in cached_relax_params:\n                assert len(torch_pnames) > 1\n                assert all([torch_pname in cached_torch_params] for torch_pname in torch_pnames)\n                cached_relax_params[i] = self.f_compute_relax_param(\n                    relax_pname,\n                    [cached_torch_params[torch_pname] for torch_pname in torch_pnames],\n                )\n                for torch_pname in torch_pnames:\n                    del cached_torch_params[torch_pname]\n\n            assert i in cached_relax_params\n            assert i not in loaded_idx_set\n            param_on_device = tvm.nd.array(cached_relax_params[i], device=device)\n            loaded_idx_set.add(i)\n            del cached_relax_params[i]\n            return param_on_device\n\n        return get_item\n\n    \ndef get_param_set_item(self) -> Tuple[Callable, List[tvm.nd.NDArray]]:\n        \"\"\"A wrapper function which returns the `set_item`\n        functions for parameter lazy loading.\n\n        The return value of this function is intended to be registered\n        as `\"set_item\"`, for use in a module built with\n        `LazyTransformParams`.\n\n        .. code-block:: python\n\n            set_item,loaded_params = manager.get_param_set_item()\n            tvm.register_func(func_name=\"set_item\", f=set_item, override=True)\n            compiled_function()\n            # `loaded_params` is now fully populated\n\n        Returns\n        -------\n        set_item: Callable[[int,tvm.nd.NDArray]]\n\n            A function that accepts an index and the return value at\n            that index.\n\n        loaded_params: List[tvm.nd.NDArray]\n\n            A list of loaded parameters, populated by `set_item`.\n            When initially returned, this list is empty.  After\n            executing the compiled function with\n            `LazyTransformParams`, `loaded_params` will be\n            populated.\n        \"\"\"\n        device_cpu = tvm.cpu()\n        loaded_params: List[tvm.nd.NDArray] = []\n\n        def set_item(i: int, computed_param: tvm.nd.NDArray):\n            if len(loaded_params) <= i:\n                loaded_params.extend([None for _ in range(i - len(loaded_params) + 1)])\n            loaded_params[i] = tvm.nd.array(computed_param, device=device_cpu)\n\n        return set_item, loaded_params\n\n    #################### Below are internally called methods ####################\n\n    def _register_param(\n        self,\n        name: str,\n        var: relax.Var,\n        quant_spec: quantization.QuantizationSpec,\n        func_name: str,\n        shard_dim: Optional[int],\n        shard_strategy: Optional[str],\n    ) -> Parameter:\n        \"\"\"Register a single parameter in the parameter manager.\n        In most cases, this method is not directly used outside this class:\n        it is called by `register_params` above.\n\n        Parameters\n        ----------\n        name : str\n            The name of the parameter to register.\n            Name serves as the unique identifier of the parameter.\n\n        var : relax.Var\n            The parameter relax.Var on the nn.Module side.\n\n        quant_spec : quantization.QuantizationSpec\n            The quantization specification of the parameter\n\n        func_name : str\n            The name of the function the input var is in.\n            For example, the \"prefill\" function or the \"decode\" function.\n\n        shard_dim : Optional[int]\n            The dimension along which the parameter is sharded.\n\n        shard_strategy : Optional[str]\n            The strategy of sharding the parameter.\n\n        Returns\n        -------\n        param : Parameter\n            The registered Parameter.\n        \"\"\"\n        assert (\n            var not in self.func_raw_param_map\n        ), \"The input var is not supposed to be already registered.\"\n        assert isinstance(\n            var.struct_info.shape, relax.ShapeExpr\n        ), \"The parameter to register is expected to have shape as a tuple\"\n\n        if name in self.params:\n            # When the input name appears in `self.params`, it means the input\n            # parameter has been previously registered in some other function.\n            # Thus, we check if the dtype, shape and the quantization specification\n            # of both sides are consistent.\n            param = self.params[name]\n            assert (\n                param.quant_spec == quant_spec\n            ), \"One parameter is expected to be quantized by single specification in all functions.\"\n            assert (\n                param.param_info.dtype == var.struct_info.dtype\n            ), \"Dtype mismatch of one parameter in two functions.\"\n            assert (\n                param.param_info.ndim == var.struct_info.ndim\n            ), \"Shape mismatch of one parameter in two functions.\"\n            for len0, len1 in zip(param.param_info.shape.values, var.struct_info.shape.values):\n                if isinstance(len0, tir.IntImm) and isinstance(len1, tir.IntImm):\n                    assert (\n                        len0.value == len1.value\n                    ), \"Shape mismatch of one parameter in two functions.\"\n        else:\n            # Otherwise, the parameter is registered for the first time.\n            param = Parameter(name, quant_spec, shard_dim, shard_strategy)\n            self.params[name] = param\n            self.param_names.append(name)\n\n        param.register_func(func_name, var.struct_info)\n        # Record the mapping from the input relax.Var to the function name and\n        # the parameter in the manager.\n        self.func_raw_param_map[var] = (func_name, param)\n        return param\n\n    def _dequantize(\n        self,\n        param: Parameter,\n        qparams: List[relax.Var],\n        bb: relax.BlockBuilder,\n        func_name: str,\n    ) -> relax.Var:\n        \"\"\"Applying dequantization to the input parameter.\n        This method is called by `transform_module` below, and is not\n        directly invoked outside the class.\n\n        Parameters\n        ----------\n        param : Parameter\n            The parameter whose quantized tensors are to be dequantized.\n\n        qparams : List[relax.Var]\n            The relax.Var of the quantized tensors of all parameters in the model.\n\n        Returns\n        -------\n        The dequantized parameter, in the form of a relax.Var.\n        \"\"\"\n        # Get the dequantization function of this parameter.\n        f_dequantize = param.quant_spec.get_dequantize_func(\n            param_info=param.param_info_dict[func_name],\n            qparam_info=[qparam.struct_info for qparam in qparams],\n        )\n        if f_dequantize is None:\n            # If the parameter does not have a dequantization function, its \"quantized\n            # data\" is expected to have only one element.\n            assert len(qparams) == 1, (\n                \"A parameter without dequantization function is expected not to have \"\n                'more than one \"quantized data\".'\n            )\n            return qparams[0]\n        else:\n            # Apply the dequantization function.\n            return bb.emit(f_dequantize(bb, qparams))\n\n    def create_parameter_transformation(self, optimize_parameter_order: bool = True):\n        \"\"\"Produce an IRModule that can transform the parameters\n\n        Parameters\n        ----------\n        optimize_parameter_order: bool\n\n            If true, reorder the parameter transformations to\n            prioritize operations that use a currently-open file.  If\n            false, transform the parameters in their default order.\n\n        Returns\n        -------\n        tvm.IRModule\n            The transformation module\n\n        \"\"\"\n        mod = _create_quantize_func(self)\n        if optimize_parameter_order:\n            mod = self.optimize_transform_param_order()(mod)\n        return mod\n\n    def optimize_transform_param_order(self) -> tvm.transform.Pass:\n        \"\"\"Produce an transformation that optimizes for minimal memory footprint\n\n        Returns\n        -------\n        tvm.transform.Pass\n            The transformation\n        \"\"\"\n\n        pidx2binname: Dict[int, str] = {\n            pidx: self.torch_pname2binname[self.f_convert_pname_fwd(pname)[0]]\n            for pidx, pname in self.pidx2pname.items()\n            if self.f_convert_pname_fwd(pname)[0] in self.torch_pname2binname\n        }\n        return ReorderTransformFunc(pidx2binname)\n\n\n@mutator\nclass ParamReplacer(PyExprMutator):\n    \"\"\"The function mutator that updates the model with dequantization.\n\n    Attributes\n    ----------\n    mod : tvm.IRModule\n        The IRModule of the model to be updated.\n\n    func_name_to_quantized_params : Dict[str, List[relax.Var]]\n        The mapping from each function name to its input var of quantized data tuple.\n\n    f_replace : Callable[[relax.Var, relax.BlockBuilder], relax.Var]\n        The function for updating a previous parameter in functions with dequantization.\n\n    param_set : Set[relax.Var]\n        The set of previous parameters (before applying quantization and dequantization)\n        in the relax functions.\n    \"\"\"\n\n    mod: tvm.IRModule\n    func_name_to_quantized_params: Dict[str, List[relax.Var]]\n    f_replace: Callable[[relax.Var, relax.BlockBuilder], relax.Var]\n    param_set: Set[relax.Var]\n\n    cur_func_name: str\n\n    def __init__(\n        self,\n        mod: tvm.IRModule,\n        func_name_to_quantized_params: Dict[str, relax.Var],\n        f_replace: Callable[[relax.Var, relax.BlockBuilder], relax.Var],\n    ):\n        super().__init__(mod)\n        self.mod = mod\n        self.func_name_to_quantized_params = func_name_to_quantized_params\n        self.f_replace = f_replace\n        self.cur_func_name = \"\"\n\n    def transform(self) -> tvm.IRModule:\n        for gv, func in self.mod.functions.items():\n            if not isinstance(func, relax.Function):\n                continue\n            if func.attrs is None or not \"num_input\" in func.attrs:\n                continue\n\n            assert (\n                gv.name_hint in self.func_name_to_quantized_params\n            ), f\"{gv.name_hint} not in {self.func_name_to_quantized_params}\"\n            updated_func = self.rewrite_func(func, self.func_name_to_quantized_params[gv.name_hint])\n            updated_func = remove_all_unused(updated_func)\n            self.builder_.update_func(gv, updated_func)\n        return self.builder_.get()\n\n    def rewrite_func(self, func: Function, quantized_params: List[relax.Var]) -> relax.Function:\n        num_input = int(func.attrs[\"num_input\"])\n        self.param_set = set(func.params[num_input:])\n\n        body = self.visit_expr(func.body)\n        return relax.Function(\n            params=func.params[:num_input] + quantized_params,\n            body=body,\n            ret_struct_info=func.ret_struct_info,\n            is_pure=func.is_pure,\n            attrs=func.attrs,\n        )\n\n    def visit_var_(self, var: Var) -> Expr:\n        if var in self.param_set:\n            return self.f_replace(var, self.builder_)\n        else:\n            return super().visit_var_(var)\n\n\n##################################################################\n\n\ndef load_torch_pname2binname_map(\n    model_path: str,\n    use_safetensors: bool,\n    relax_pnames: Set[str],\n    f_convert_pname_fwd: Callable[[str], List[str]] = lambda pname: [pname],\n) -> Dict[str, str]:\n    \"\"\"Constructing the dictionary from each torch parameter's name to\n    the name of the binary shard where the torch parameter is saved.\n\n    Parameters\n    ----------\n    model_path : str\n        The path of the Hugging Face model on disk.\n\n    use_safetensors: bool\n        Whether to use ``.safetensors`` instead of ``.bin`` to load model.\n\n    relax_pnames: Set[str]\n        The name of the Relax parameters.\n\n    f_convert_pname_fwd: Callable[[str], List[str]]\n        The function which converts Relax parameter name to torch's\n        parameter names. See ParamManager for more details.\n    \"\"\"\n    bin_idx_path = None\n    single_shard_file_name = None\n    if use_safetensors:\n        bin_idx_path = os.path.join(model_path, \"model.safetensors.index.json\")\n        single_shard_file_name = \"model.safetensors\"\n    else:\n        bin_idx_path = os.path.join(model_path, \"pytorch_model.bin.index.json\")\n        single_shard_file_name = \"pytorch_model.bin\"\n    single_shard_path = os.path.join(model_path, single_shard_file_name)\n\n    if os.path.isfile(bin_idx_path):\n        # Multiple weight shards.\n        with open(bin_idx_path, \"r\") as f_torch_json:\n            torch_bin_json = json.load(f_torch_json)\n            torch_pname2binname = torch_bin_json[\"weight_map\"]\n    elif os.path.isfile(single_shard_path):\n        # Single weight shard.\n        torch_pname2binname = {\n            torch_pname: single_shard_file_name\n            for relax_pname in relax_pnames\n            for torch_pname in f_convert_pname_fwd(relax_pname)\n        }\n    else:\n        suffix = \".safetensors\" if use_safetensors else \".bin\"\n        shard_names = []\n        # Collect Scan every single file with the suffix\n        for filename in os.listdir(model_path):\n            if filename.endswith(suffix):\n                shard_names.append(filename)\n        if len(shard_names) == 1:\n            torch_pname2binname = {\n                torch_pname: shard_names[0]\n                for relax_pname in relax_pnames\n                for torch_pname in f_convert_pname_fwd(relax_pname)\n            }\n        else:\n            raise ValueError(\"Multiple weight shard files without json map is not supported\")\n    return torch_pname2binname\n\n\ndef _create_quantize_func(param_manager: ParamManager) -> tvm.IRModule:\n    \"\"\"Construct the Relax function which computes quantization.\n    This method is called by `transform_module` below, and is not\n    directly invoked outside the class.\n\n    Parameters\n    ----------\n    param_manager : ParamManager\n        The parameter manager which has all the parameter information.\n\n    Returns\n    -------\n    The created function which computes quantization.\n    Precisely, an IRModule which contains the main quantization Relax function\n    and a series of TIR functions is returned.\n    \"\"\"\n    bb = relax.BlockBuilder()\n    param2qrange = dict()\n\n    # Construct the input of the function.\n    # We need a list of ranges for each\n    # parameter to get its corresponding tensors loaded from disk.\n    input_tensor_info: List[relax.TensorStructInfo] = []\n    loaded_tensor_ranges: List[range] = []\n    for name in param_manager.param_names:\n        param = param_manager.params[name]\n        _, loaded_tensor_info = param.quant_spec.get_loaded_tensor_info(name, param.param_info)\n        loaded_tensor_ranges.append(\n            range(\n                len(input_tensor_info),\n                len(input_tensor_info) + len(loaded_tensor_info),\n            )\n        )\n        input_tensor_info += loaded_tensor_info\n    raw_param_tuple = relax.Var(\"params\", relax.TupleStructInfo(input_tensor_info))\n\n    with bb.function(\"transform_params\", params=[raw_param_tuple]):\n        with bb.dataflow():\n            quantized_params: List[relax.Var] = []\n            for pidx, name in enumerate(param_manager.param_names):\n                param = param_manager.params[name]\n                param_vars: List[relax.Var] = []\n                # Emit relax.TupleGetItem to get the raw parameters or pre-quantized params.\n                for loaded_tensor_idx in loaded_tensor_ranges[pidx]:\n                    param_vars.append(\n                        bb.emit(relax.TupleGetItem(raw_param_tuple, loaded_tensor_idx))\n                    )\n\n                # Get the quantization function of this parameter.\n                f_quantize = param.quant_spec.get_quantize_func(param.param_info)\n                if f_quantize is None:\n                    # If the parameter does not have a quantization function, either it\n                    # does not need quantization or it is pre-quantized.\n                    param2qrange[param] = range(\n                        len(quantized_params),\n                        len(quantized_params) + len(param_vars),\n                    )\n                    quantized_params += param_vars\n                else:\n                    # If the parameter has a quantization function, it is not expected\n                    # to be pre-quantized.\n                    assert len(param_vars) == 1, (\n                        \"A parameter with quantization function is not expected \"\n                        \"to be pre-quantized.\"\n                    )\n\n                    # Apply the quantization function.\n                    quantized_data = bb.emit(f_quantize(bb, param_vars))\n\n                    if isinstance(quantized_data.struct_info, relax.TupleStructInfo):\n                        n_tensor = len(quantized_data.struct_info.fields)\n                        assert n_tensor > 1\n                        # Record the range of quantized tensors of this parameter.\n                        param2qrange[param] = range(\n                            len(quantized_params), len(quantized_params) + n_tensor\n                        )\n                        # Collect the quantized tensors to return.\n                        for i in range(n_tensor):\n                            quantized_params.append(bb.emit(relax.TupleGetItem(quantized_data, i)))\n                    else:\n                        assert isinstance(quantized_data.struct_info, relax.TensorStructInfo)\n                        param2qrange[param] = range(\n                            len(quantized_params), len(quantized_params) + 1\n                        )\n                        quantized_params.append(quantized_data)\n\n            output = bb.emit_output(relax.Tuple(quantized_params))\n        bb.emit_func_output(output)\n\n    mod = bb.get()\n    param_manager.param2qrange = param2qrange\n    # Return the created IRModule.\n    return bb.get()\n\n\ndef transform_params_for_each_rank(\n    num_shards: int, rank_argument_name: str = \"rank_arg\"\n) -> tvm.ir.transform.Pass:\n    \"\"\"Update a parameter transform to apply across all ranks\n\n    For use in generating a pre-sharded set of weights.  Given a\n    parameter transformation that generates sharded model weights for\n    a single shard, produce a parameter transformation that generates\n    sharded model weights for each shard.\n\n    Parameters\n    ----------\n    mod: tvm.IRModule\n\n        A module containing the parameter transformation function,\n        named \"transform_params\", along with any subroutines called by\n        the parameter transformation.\n\n    num_shards: int\n\n        The number of shards to generate.\n\n    rank_argument_name: str\n\n        The name of the argument that specifies the rank.  Should be a\n        R.ShapeTuple with a single R.PrimStructInfo('int64').\n\n    Returns\n    -------\n    tvm.IRModule\n\n        The modified parameter transformation\n    \"\"\"\n\n    @tvm.ir.transform.module_pass(opt_level=0, name=\"ParamManager.transform_params_for_each_rank\")\n    def transform_func(mod: tvm.IRModule, _context) -> tvm.IRModule:\n        generic_transform = mod[\"transform_params\"]\n\n        if generic_transform.attrs is not None and \"num_input\" in generic_transform.attrs:\n            num_input = generic_transform.attrs[\"num_input\"].value\n        else:\n            num_input = 0\n\n        if num_input == 0:\n            return mod\n\n        tensor_params = generic_transform.params[num_input:]\n        attrs = {\"num_input\": num_input - 1}\n\n        bb = relax.BlockBuilder()\n\n        with bb.function(\"transform_params\", params=tensor_params, attrs=attrs):\n            output = []\n            for rank in range(num_shards):\n                # TODO(Lunderberg): Implement this in terms of a\n                # generic utility that inlines local functions.\n                func = generic_transform\n                func = func.bind_params({rank_argument_name: relax.ShapeExpr([rank])})\n                func = relax.utils.copy_with_new_vars(func)\n                func = func.bind_params(\n                    {var: tensor_param for (var, tensor_param) in zip(func.params, tensor_params)}\n                )\n                shard_tuple = func.body\n                output.extend([shard_tuple[i] for i in range(len(tensor_params))])\n\n            with bb.dataflow():\n                gv = bb.emit_output(relax.Tuple(output))\n            bb.emit_func_output(gv)\n\n        mod = mod.clone()\n        mod[\"transform_params\"] = bb.get()[\"transform_params\"]\n        return mod\n\n    return transform_func\n\n\ndef chain_parameter_transforms(mod_a: tvm.IRModule, mod_b: tvm.IRModule) -> tvm.IRModule:\n    \"\"\"Chain two sequential parameter transformations\n\n    For use in manipulating sets of model weights.  Given two\n    parameter transformations that could be applied sequentially,\n    produce a single parameter transformation whose output is the same\n    as applying the parameter transformations sequentially.\n\n\n    .. code-block:: python\n\n        # Before\n        params_after_a = mod_a['transform_params'](orig_params)\n        params_after_b = mod_b['transform_params'](params_after_a)\n\n        # After\n        mod_ab = chain_parameter_transforms(mod_a, mod_b)\n        params_after_b = mod_ab['transform_params'](orig_params)\n\n    Parameters\n    ----------\n    mod_a: tvm.IRModule\n\n        The module containing the first parameter transformation.\n\n    mod_b: tvm.IRModule\n\n        The module containing the second parameter transformation.\n\n    Returns\n    -------\n    tvm.IRModule\n\n        The module containing the output\n\n    \"\"\"\n    func_a = mod_a[\"transform_params\"]\n    func_b = mod_b[\"transform_params\"]\n\n    bb = relax.BlockBuilder()\n\n    def get_num_input_attr(func):\n        if func.attrs is None:\n            return 0\n\n        attrs = func.attrs\n        if \"num_input\" not in attrs:\n            return 0\n        num_input = attrs[\"num_input\"]\n\n        assert isinstance(num_input, tvm.tir.IntImm)\n        return num_input.value\n\n    # Either func_a or func_b may have parameters that are provided at\n    # a later point.  The chaining of parameter transforms assumes\n    # that all model weights accepted by func_b are produced by\n    # func_a.  If func_b accepts non-weight parameters (e.g. the GPU\n    # rank), these must still be provided.\n    func_a_num_input = get_num_input_attr(func_a)\n    func_b_num_input = get_num_input_attr(func_b)\n\n    output_num_input = func_a_num_input + func_b_num_input\n    output_params = [\n        *func_a.params[:func_a_num_input],\n        *func_b.params[:func_b_num_input],\n        *func_a.params[func_a_num_input:],\n    ]\n\n    with bb.function(\n        \"transform_params\", params=output_params, attrs={\"num_input\": output_num_input}\n    ):\n        with bb.dataflow():\n            # TODO(Lunderberg): Implement this in terms of a\n            # generic utility that inlines local functions.\n            func_a_output = bb.emit(func_a.body)\n            func_b_param_map = {\n                param: expr\n                for (param, expr) in zip(func_b.params[func_b_num_input:], func_a_output)\n            }\n            func_b_output = func_b.bind_params(func_b_param_map).body\n            gv = bb.emit_output(func_b_output)\n        bb.emit_func_output(gv)\n\n    merged_transform_func = bb.get()[\"transform_params\"]\n\n    new_mod = {\n        **{\n            gvar: func\n            for gvar, func in mod_a.functions.items()\n            if gvar.name_hint != \"transform_params\"\n        },\n        **{\n            gvar: func\n            for gvar, func in mod_b.functions.items()\n            if gvar.name_hint != \"transform_params\"\n        },\n        \"transform_params\": merged_transform_func,\n    }\n    return tvm.IRModule(new_mod)\n\n# Path: mlc_llm/relax_model/llama.py\nimport math\nfrom dataclasses import dataclass\nfrom typing import Any, List, Optional, Tuple, Union\n\nimport numpy as np\nimport tvm\nfrom tvm import relax, te, tir\nfrom tvm.relax.op import ccl\nfrom tvm.relax.testing import nn\nfrom tvm.script import relax as R\n\nfrom ..quantization import ParamQuantKind, QuantizationScheme\nfrom .commons import create_metadata_func\nfrom .modules import ModuleList\nfrom .param_manager import ParamManager\n\n\n@dataclass\nclass LlamaConfig:\n    def __init__(\n        self,\n        dtype=\"float32\",\n        max_sequence_length=2048,\n        vocab_size=32000,  # some models like WizardMath can have 32001\n        hidden_size=4096,\n        intermediate_size=11008,\n        num_hidden_layers=32,\n        num_attention_heads=32,\n        num_key_value_heads=None,\n        hidden_act=\"silu\",\n        initializer_range=0.02,\n        rms_norm_eps=1e-6,\n        pad_token_id=-1,\n        bos_token_id=0,\n        eos_token_id=1,\n        tie_word_embeddings=False,\n        position_embedding_base=10000,\n        combine_matmul=True,\n        build_model_only=False,\n        num_shards=1,\n        sliding_window=None,\n        target_kind=None,\n        **kwargs,\n    ):\n        self.dtype = dtype\n        self.max_sequence_length = max_sequence_length\n        self.vocab_size = vocab_size\n        self.hidden_size = hidden_size\n        self.intermediate_size = intermediate_size\n        self.num_hidden_layers = num_hidden_layers\n        self.num_attention_heads = num_attention_heads\n        self.num_key_value_heads = num_key_value_heads\n        self.hidden_act = hidden_act\n        self.initializer_range = initializer_range\n        self.rms_norm_eps = rms_norm_eps\n        self.pad_token_id = pad_token_id\n        self.bos_token_id = bos_token_id\n        self.eos_token_id = eos_token_id\n        self.tie_word_embeddings = tie_word_embeddings\n        self.position_embedding_base = position_embedding_base\n        self.combine_matmul = combine_matmul\n        self.sliding_window = sliding_window\n        self.target_kind = target_kind\n\n        if build_model_only and num_shards > 1:\n            self.num_shards = num_shards\n        else:\n            self.num_shards = 1\n        self.kwargs = kwargs\n\n    def get_num_key_value_heads(self):\n        if self.num_key_value_heads is None:\n            return self.num_attention_heads\n\n        return self.num_key_value_heads\n\n\nclass Linear(nn.Module):\n    def __init__(self, in_features, out_features, dtype: str, bias=True):\n        self.in_features = in_features\n        self.out_features = out_features\n        self.weight = nn.Parameter((out_features, in_features), dtype=dtype, name=\"linear_weight\")\n        if bias:\n            self.bias = nn.Parameter((out_features,), dtype=dtype, name=\"linear_bias\")\n        else:\n            self.bias = None\n\n    def forward(self, input: relax.Expr) -> relax.Var:\n        return nn.emit(relax.op.linear(input, self.weight, self.bias))\n\n\nclass Embedding(nn.Module):\n    def __init__(self, num_embeddings, embedding_dim, dtype: str):\n        self.num_embeddings = num_embeddings\n        self.embedding_dim = embedding_dim\n        self.weight = nn.Parameter(\n            (num_embeddings, embedding_dim), dtype=dtype, name=\"embedding_weight\"\n        )\n\n    def forward(self, x: relax.Expr) -> relax.Var:\n        from tvm.relax.op import reshape, take\n\n        ndim = x.struct_info.ndim\n        if ndim == 1:\n            return nn.emit(take(self.weight, x, axis=0))\n        else:\n            x_shape = x.struct_info.shape.values\n            emb_size = self.weight.struct_info.shape.values[-1]\n            x = nn.emit(reshape(x, shape=[-1]))\n            embedding = nn.emit(take(self.weight, x, axis=0))\n            return nn.emit(reshape(embedding, [*x_shape, emb_size]))\n\n\nclass LlamaRMSNorm(nn.Module):\n    def __init__(self, hidden_size, dtype, eps=1e-6):\n        self.weight = nn.Parameter((hidden_size,), dtype=dtype, name=\"rms_norm_weight\")\n        self.variance_epsilon = tvm.tir.const(eps, dtype)\n\n    def forward(self, hidden_states):\n        from tvm import te, tir\n\n        def f_rms_norm(x, weight):\n            is_float32 = x.dtype == \"float32\"\n\n            def f_square(x):\n                return tir.Cast(\"float32\", x) * tir.Cast(\"float32\", x) if not is_float32 else x * x\n\n            def f_mul_cast(x, y):\n                value = x * y\n                if not is_float32:\n                    value = tir.Cast(x.dtype, value)\n                return value\n\n            def f_div_cast_2d(i, k):\n                x_val = x[i, k]\n                if not is_float32:\n                    x_val = tir.Cast(\"float32\", x_val)\n                return x_val / tir.sqrt(square_sum[i] / x.shape[1] + self.variance_epsilon)\n\n            def f_div_cast_3d(bsz, i, k):\n                x_val = x[bsz, i, k]\n                if not is_float32:\n                    x_val = tir.Cast(\"float32\", x_val)\n                return x_val / tir.sqrt(square_sum[bsz, i] / x.shape[2] + self.variance_epsilon)\n\n            k = te.reduce_axis((0, x.shape[-1]), name=\"k\")\n\n            if len(x.shape) == 2:\n                square_sum = te.compute(\n                    (x.shape[0],),\n                    lambda i: te.sum(f_square(x[i, k]), axis=k),\n                    name=x.op.name + \"red_temp\",\n                )\n\n                return te.compute(\n                    x.shape,\n                    lambda i, k: f_mul_cast(weight(k), f_div_cast_2d(i, k)),\n                    name=\"rms_norm\",\n                )\n            else:\n                square_sum = te.compute(\n                    (x.shape[0], x.shape[1]),\n                    lambda bsz, i: te.sum(f_square(x[bsz, i, k]), axis=k),\n                    name=x.op.name + \"red_temp\",\n                )\n\n                return te.compute(\n                    x.shape,\n                    lambda bsz, i, k: f_mul_cast(weight(k), f_div_cast_3d(bsz, i, k)),\n                    name=\"rms_norm\",\n                )\n\n        return nn.emit_te(f_rms_norm, hidden_states, self.weight, primfunc_name_hint=\"rms_norm\")\n\n\nclass LlamaMLP(nn.Module):\n    def __init__(self, config: LlamaConfig):\n        self.combine_matmul = config.combine_matmul\n        self.num_shards = config.num_shards\n        hidden_size = config.hidden_size\n        intermediate_size = config.intermediate_size // self.num_shards\n        dtype = config.dtype\n        if self.combine_matmul:\n            self.gate_up_proj = Linear(hidden_size, 2 * intermediate_size, dtype=dtype, bias=False)\n            self.down_proj = Linear(intermediate_size, hidden_size, dtype=dtype, bias=False)\n            self.gate_up_proj.weight.shard_dim = 0\n            self.gate_up_proj.weight.shard_strategy = \"shard_gate_up\"\n            self.down_proj.weight.shard_dim = 1\n            self.down_proj.weight.shard_strategy = \"shard_mlp_k\"\n        else:\n            self.gate_proj = Linear(hidden_size, intermediate_size, dtype=dtype, bias=False)\n            self.down_proj = Linear(intermediate_size, hidden_size, dtype=dtype, bias=False)\n            self.up_proj = Linear(hidden_size, intermediate_size, dtype=dtype, bias=False)\n            self.gate_proj.weight.shard_dim = 0\n            self.gate_proj.weight.shard_strategy = \"shard_axis_0\"\n            self.down_proj.weight.shard_dim = 1\n            self.down_proj.weight.shard_strategy = \"shard_axis_1\"\n            self.up_proj.weight.shard_dim = 0\n            self.up_proj.weight.shard_strategy = \"shard_axis_0\"\n\n    def forward(self, x):\n        if self.combine_matmul:\n            gate_up_results = nn.emit(\n                relax.op.split(\n                    self.gate_up_proj(x),\n                    indices_or_sections=2,\n                    axis=-1,\n                )\n            )\n            gate_result = relax.TupleGetItem(gate_up_results, 0)\n            up_result = relax.TupleGetItem(gate_up_results, 1)\n        else:\n            gate_result = self.gate_proj(x)\n            up_result = self.up_proj(x)\n\n        result = self.down_proj(relax.op.nn.silu(gate_result) * up_result)\n        return result\n\n\ndef rotary_modulate_by_freq(tensor, idx, pos, position_embedding_base):\n    head_dim = tensor.shape[-1]\n    dtype = tensor.dtype\n    n_feat_half = head_dim // 2\n    feat_idx = idx[-1]\n    inv_freq = te.const(1, \"float32\") / (\n        te.power(\n            te.const(position_embedding_base, \"float32\"),\n            ((2 * feat_idx) % head_dim).astype(\"float32\") / head_dim.astype(\"float32\"),\n        )\n    )\n    freq = pos * inv_freq\n    left_indices = idx[:-1] + (feat_idx - n_feat_half,)\n    right_indices = idx[:-1] + (feat_idx + n_feat_half,)\n    return te.cos(freq).astype(dtype) * tensor(*idx) + te.sin(freq).astype(dtype) * tvm.tir.Select(\n        feat_idx >= n_feat_half,\n        tensor[(*left_indices,)],\n        -tensor[(*right_indices,)],\n    )\n\n\ndef apply_rotary_pos_emb(q, k, position_embedding_base, offset: int = 0):\n    def f_rotary_embedding(tensor, offset):\n        def rotary_compute(*idx):\n            pos = (offset + idx[-3]).astype(\"float32\")\n            return rotary_modulate_by_freq(\n                tensor,\n                idx,\n                pos,\n                position_embedding_base,\n            )\n\n        return tvm.te.compute(tensor.shape, rotary_compute, name=\"rotary\")\n\n    q_embed = nn.emit_te(f_rotary_embedding, q, offset, primfunc_name_hint=\"rotary_embedding\")\n    k_embed = nn.emit_te(f_rotary_embedding, k, offset, primfunc_name_hint=\"rotary_embedding\")\n    return q_embed, k_embed\n\n\nclass LlamaAttentionBase(nn.Module):\n    \"\"\"Multi-headed attention from 'Attention Is All You Need' paper\"\"\"\n\n    def __init__(self, config: LlamaConfig):\n        dtype = config.dtype\n        self.num_shards = config.num_shards\n        self.hidden_size = config.hidden_size\n        self.num_key_value_heads = config.get_num_key_value_heads() // config.num_shards\n        self.num_query_heads = config.num_attention_heads // self.num_shards\n        self.head_dim = self.hidden_size // config.num_attention_heads\n        self.position_embedding_base = config.position_embedding_base\n\n        self.combine_matmul = config.combine_matmul\n        if self.combine_matmul:\n            self.query_key_value_proj = Linear(\n                self.hidden_size,\n                (self.num_query_heads + 2 * self.num_key_value_heads) * self.head_dim,\n                dtype=dtype,\n                bias=False,\n            )\n            self.query_key_value_proj.weight.shard_dim = 0\n            self.query_key_value_proj.weight.shard_strategy = \"shard_qkv\"\n        else:\n            self.q_proj = Linear(\n                self.hidden_size,\n                self.num_query_heads * self.head_dim,\n                dtype=dtype,\n                bias=False,\n            )\n            self.k_proj = Linear(\n                self.hidden_size,\n                self.num_key_value_heads * self.head_dim,\n                dtype=dtype,\n                bias=False,\n            )\n            self.v_proj = Linear(\n                self.hidden_size,\n                self.num_key_value_heads * self.head_dim,\n                dtype=dtype,\n                bias=False,\n            )\n            self.q_proj.weight.shard_dim = 0\n            self.k_proj.weight.shard_dim = 0\n            self.v_proj.weight.shard_dim = 0\n            self.q_proj.weight.shard_strategy = \"shard_axis_0\"\n            self.k_proj.weight.shard_strategy = \"shard_axis_0\"\n            self.v_proj.weight.shard_strategy = \"shard_axis_0\"\n\n        self.o_proj = Linear(\n            self.head_dim * self.num_query_heads, self.hidden_size, dtype=dtype, bias=False\n        )\n        self.o_proj.weight.shard_dim = 1\n        self.o_proj.weight.shard_strategy = \"shard_o_proj_k\"\n\n    def project_qkv(self, hidden_states, query_output_shape, kv_output_shape):\n        from tvm.relax.op import reshape, split\n\n        if self.combine_matmul:\n            qkv_states = nn.emit(\n                split(\n                    self.query_key_value_proj(hidden_states),\n                    indices_or_sections=[\n                        self.num_query_heads * self.head_dim,\n                        (self.num_query_heads + self.num_key_value_heads) * self.head_dim,\n                    ],\n                    axis=-1,\n                )\n            )\n            query_states = relax.TupleGetItem(qkv_states, 0)\n            key_states = relax.TupleGetItem(qkv_states, 1)\n            value_states = relax.TupleGetItem(qkv_states, 2)\n        else:\n            query_states = self.q_proj(hidden_states)\n            key_states = self.k_proj(hidden_states)\n            value_states = self.v_proj(hidden_states)\n\n        query_states = nn.emit(\n            reshape(query_states, query_output_shape),\n        )\n        key_states = nn.emit(\n            reshape(key_states, kv_output_shape),\n        )\n        value_states = nn.emit(\n...\n", "needle_token_start": 5447, "needle_token_end": 5862, "code_context_ntokens": 16145, "output": ["```python\ndef get_param_set_item(self) -> Tuple[Callable, List[tvm.nd.NDArray]]:\n    \"\"\"A wrapper function which returns the `set_item`\n    functions for parameter lazy loading.\n\n    The return value of this function is intended to be registered\n    as `\"set_item\"`, for use in a module built with\n    `LazyTransformParams`.\n\n    .. code-block:: python\n\n        set_item,loaded_params = manager.get_param_set_item()\n        tvm.register_func(func_name=\"set_item\", f=set_item, override=True)\n        compiled_function()\n        # `loaded_params` is now fully populated\n\n    Returns\n    -------\n    set_item: Callable[[int,tvm.nd.NDArray]]\n\n        A function that accepts an index and the return value at\n        that index.\n\n    loaded_params: List[tvm.nd.NDArray]\n\n        A list of loaded parameters, populated by `set_item`.\n        When initially returned, this list is empty.  After\n        executing the compiled function with\n        `LazyTransformParams`, `loaded_params` will be\n        populated.\n    \"\"\"\n    device_cpu = tvm.cpu()\n    loaded_params: List[tvm.nd.NDArray] = []\n\n    def set_item(i: int, computed_param: tvm.nd.NDArray):\n        if len(loaded_params) <= i:\n            loaded_params.extend([None for _ in range(i - len(loaded_params) + 1)])\n        loaded_params[i] = tvm.nd.array(computed_param, device=device_cpu)\n\n    return set_item, loaded_params\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "fused_fused_decode9_matmul7", "language": "python", "path": "mlc_llm/dispatch/dispatch_tir_operator_adreno.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function performs a decoding operation followed by a matrix multiplication. The decoding involves extracting and transforming elements from an input buffer, and the matrix multiplication aggregates these transformed elements with another input buffer.\n2. **Input**: \n   - A buffer containing 32-bit unsigned integers, structured in a 2-dimensional array.\n   - A buffer containing 16-bit floating-point numbers, also structured in a 2-dimensional array.\n   - A buffer containing 16-bit floating-point numbers, structured in a 3-dimensional array.\n3. **Output**: A buffer containing 16-bit floating-point numbers, structured in a 3-dimensional array, which holds the result of the matrix multiplication.\n4. **Procedure**: \n   - The function iterates over a grid to decode values from the first input buffer using bitwise operations and scaling, and multiplies the decoded values with corresponding values from the second input buffer.\n   - The results are stored temporarily.\n   - Another loop iterates over a grid to perform matrix multiplication using the temporary results and the third input buffer.\n   - The final results of the matrix multiplication are stored in the output buffer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(11008), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1622[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = (\n                var_matmul_intermediate[v_i0, v_i1, v_i2]\n                + lv1622[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n            )\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(\n                lv4[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n            )\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = (\n                lv4[v_ax0, v_ax1, v_ax2] * var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n            )\n\n\n@T.prim_func(private=True)\ndef fused_decode5_fused_matmul6_multiply1_after(\n    lv1617: T.Buffer((T.int64(512), T.int64(11008)), \"uint32\"),\n    lv1618: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"),\n    lv1622: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    lv4: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"),\n    p_output0_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(11008)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_matmul_intermediate_local = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\", scope=\"local\"\n    )\n    var_matmul_intermediate_local_batch = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\", scope=\"local\"\n    )\n    lv1617_local = T.alloc_buffer(\n        (T.int64(512), T.int64(11008)), \"uint32\", scope=\"local\"\n    )\n    lv1618_local = T.alloc_buffer(\n        (T.int64(128), T.int64(11008)), \"float16\", scope=\"local\"\n    )\n    lv1622_shared = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(1024)), \"float16\", scope=\"shared\"\n    )\n    for i0_i1_i2_fused_0 in T.thread_binding(T.int64(43), thread=\"blockIdx.x\"):\n        for i0_i1_i2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for i0_i1_i2_fused_2_init in T.vectorized(T.int64(4)):\n                    with T.block(\"matmul_init\"):\n                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i2 = T.axis.spatial(\n                            T.int64(22016),\n                            i0_i1_i2_fused_0 * T.int64(512)\n                            + i0_i1_i2_fused_1 * T.int64(8)\n                            + ax2_y * T.int64(4)\n                            + i0_i1_i2_fused_2_init,\n                        )\n                        T.reads()\n                        T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                        var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float16(0)\n            for k_0 in range(T.int64(4)):\n                for ax2_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                            for ax2_2 in T.vectorized(T.int64(8)):\n                                with T.block(\"lv1622_shared\"):\n                                    v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                                    v2 = T.axis.spatial(\n                                        T.int64(4096),\n                                        k_0 * T.int64(1024)\n                                        + ax2_y * T.int64(512)\n                                        + ax2_1 * T.int64(8)\n                                        + ax2_2,\n                                    )\n                                    v2k = T.axis.spatial(\n                                        T.int64(1024),\n                                        (\n                                            ax2_y * T.int64(512)\n                                            + ax2_1 * T.int64(8)\n                                            + ax2_2\n                                        ),\n                                    )\n                                    T.reads(lv1622[v0, v1, v2])\n                                    T.writes(lv1622_shared[v0, v1, v2k])\n                                    lv1622_shared[v0, v1, v2k] = lv1622[v0, v1, v2]\n                for k_1 in range(T.int64(16)):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax1 in T.vectorized(T.int64(4)):\n                            with T.block(\"matmul_init_local\"):\n                                v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i2k = T.axis.spatial(\n                                    T.int64(22016),\n                                    i0_i1_i2_fused_0 * T.int64(512)\n                                    + i0_i1_i2_fused_1 * T.int64(8)\n                                    + ax2_y * T.int64(4)\n                                    + ax1,\n                                )\n                                T.reads()\n                                T.writes(\n                                    var_matmul_intermediate_local_batch[\n                                        v_i0, v_i1, v_i2k\n                                    ]\n                                )\n                                var_matmul_intermediate_local_batch[\n                                    v_i0, v_i1, v_i2k\n                                ] = T.float16(0)\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"lv1618_local\"):\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(32)\n                                        + (k_1 * T.int64(2) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(11008),\n                                        i0_i1_i2_fused_0 * T.int64(256)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(lv1618[v0, v1])\n                                    T.writes(lv1618_local[v0, v1])\n                                    lv1618_local[v0, v1] = lv1618[v0, v1]\n                        for k_2 in range(T.int64(4)):\n                            for ax0 in range(T.int64(1)):\n                                for ax1 in T.vectorized(T.int64(4)):\n                                    with T.block(\"lv1617_local\"):\n                                        v0 = T.axis.spatial(\n                                            T.int64(512),\n                                            k_0 * T.int64(128)\n                                            + (k_1 * T.int64(2) + ax2_y) * T.int64(4)\n                                            + k_2\n                                            + ax0,\n                                        )\n                                        v1 = T.axis.spatial(\n                                            T.int64(11008),\n                                            i0_i1_i2_fused_0 * T.int64(256)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + ax1,\n                                        )\n                                        T.reads(lv1617[v0, v1])\n                                        T.writes(lv1617_local[v0, v1])\n                                        lv1617_local[v0, v1] = lv1617[v0, v1]\n                            for k_3 in range(T.int64(8)):\n                                for i0_i1_i2_fused_2 in T.vectorized(T.int64(4)):\n                                    with T.block(\"matmul_update\"):\n                                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i2 = T.axis.spatial(\n                                            T.int64(11008),\n                                            i0_i1_i2_fused_0 * T.int64(256)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_i2k = T.axis.spatial(\n                                            T.int64(22016),\n                                            i0_i1_i2_fused_0 * T.int64(512)\n                                            + i0_i1_i2_fused_1 * T.int64(8)\n                                            + ax2_y * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_k = T.axis.reduce(\n                                            T.int64(4096),\n                                            k_0 * T.int64(1024)\n                                            + (k_1 * T.int64(2) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        v_ki = T.axis.reduce(\n                                            T.int64(1024),\n                                            (k_1 * T.int64(2) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        T.reads(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ],\n                                            lv1622_shared[v_i0, v_i1, v_ki],\n                                            lv1617_local[v_k // T.int64(8), v_i2],\n                                        )\n                                        T.writes(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ]\n                                        )\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] = var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] + lv1622_shared[\n                                            v_i0, v_i1, v_ki\n                                        ] * (\n                                            (\n                                                T.Cast(\n                                                    \"float16\",\n                                                    T.bitwise_and(\n                                                        T.shift_right(\n                                                            lv1617_local[\n                                                                v_k // T.int64(8), v_i2\n                                                            ],\n                                                            T.Cast(\n                                                                \"uint32\",\n                                                                v_k % T.int64(8),\n                                                            )\n                                                            * T.uint32(4),\n                                                        ),\n                                                        T.uint32(15),\n                                                    ),\n                                                )\n                                                - T.float16(7)\n                                            )\n                                        )\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"multiple_scale\"):\n                                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i2k = T.axis.spatial(\n                                        T.int64(22016),\n                                        i0_i1_i2_fused_0 * T.int64(512)\n                                        + i0_i1_i2_fused_1 * T.int64(8)\n                                        + ax2_y * T.int64(4)\n                                        + ax1,\n                                    )\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(32)\n                                        + (k_1 * T.int64(2) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(11008),\n                                        i0_i1_i2_fused_0 * T.int64(256)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(\n                                        lv1618_local[v0, v1],\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ],\n                                    )\n                                    T.writes(\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                    )\n                                    var_matmul_intermediate_local[v_i0, v_i1, v_i2k] = (\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                        + var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ]\n                                        * lv1618_local[v0, v1]\n                                    )\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_update\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(512),\n                                i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(22016),\n                                i0_i1_i2_fused_0 * T.int64(512)\n                                + i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.reads(var_matmul_intermediate_local[v0, v1, v_i2k])\n                            T.writes(lv1622_shared[v0, v1, v2])\n                            lv1622_shared[v0, v1, v2] = var_matmul_intermediate_local[\n                                v0, v1, v_i2k\n                            ]\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_local\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(11008),\n                                i0_i1_i2_fused_0 * T.int64(256)\n                                + i0_i1_i2_fused_1 * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(512),\n                                i0_i1_i2_fused_1 * T.int64(8)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(1))\n                            T.reads(lv1622_shared[v0, v1, v_i2k], lv4[v0, v1, v2])\n                            T.writes(p_output0_intermediate[v0, v1, v2])\n                            p_output0_intermediate[v0, v1, v2] = lv4[v0, v1, v2] * (\n                                lv1622_shared[v0, v1, v_i2k]\n                                + lv1622_shared[v0, v1, v_i2k + T.int64(4)]\n                            )\n\n\ndef sch_fused_decode5_fused_matmul6_multiply1(func):\n    sch = tvm.tir.Schedule(func)\n    b0 = sch.get_block(name=\"decode\", func_name=\"main\")\n    b1 = sch.get_block(name=\"matmul\", func_name=\"main\")\n    l2, l3, l4, l5 = sch.get_loops(block=b1)\n    l6 = sch.fuse(l2, l3, l4, preserve_unit_iters=True)\n    v7, v8, v9 = sch.sample_perfect_tile(\n        loop=l6, n=3, max_innermost_factor=4, decision=[43, 64, 4]\n    )\n    l10, l11, l12 = sch.split(loop=l6, factors=[v7, v8, v9], preserve_unit_iters=True)\n    v13, v14, v15 = sch.sample_perfect_tile(\n        loop=l5, n=3, max_innermost_factor=8, decision=[128, 4, 8]\n    )\n    l16, l17, l18 = sch.split(\n        loop=l5, factors=[v13, v14, v15], preserve_unit_iters=True\n    )\n    sch.reorder(l10, l11, l16, l17, l18, l12)\n    sch.bind(loop=l10, thread_axis=\"blockIdx.x\")\n    sch.bind(loop=l11, thread_axis=\"threadIdx.x\")\n    sch.compute_inline(block=b0)\n    b19 = sch.cache_write(block=b1, write_buffer_index=0, storage_scope=\"local\")\n    sch.reverse_compute_at(block=b19, loop=l11, preserve_unit_loops=True, index=-1)\n    b20 = sch.cache_read(block=b1, read_buffer_index=1, storage_scope=\"local\")\n    b21 = sch.cache_read(block=b1, read_buffer_index=2, storage_scope=\"local\")\n    b22 = sch.cache_read(block=b1, read_buffer_index=0, storage_scope=\"shared\")\n    sch.compute_at(block=b22, loop=l11, preserve_unit_loops=True, index=-1)\n    v23 = sch.sample_categorical(\n        candidates=[1, 2, 4, 8], probs=[0.25, 0.25, 0.25, 0.25], decision=3\n    )\n    sch.annotate(\n        block_or_loop=b22, ann_key=\"meta_schedule.cooperative_fetch\", ann_val=v23\n    )\n    sch.compute_at(block=b20, loop=l17, preserve_unit_loops=True, index=-1)\n    sch.compute_at(block=b21, loop=l16, preserve_unit_loops=True, index=-1)\n    l24, l25, l26, l27, l28, l29 = sch.get_loops(block=b20)\n    sch.vectorize(loop=l29)\n    l30, l31, l32, l33, l34 = sch.get_loops(block=b21)\n    sch.vectorize(loop=l34)\n    l35, l36, l37, l38, l39 = sch.get_loops(block=b19)\n    sch.vectorize(loop=l39)\n    sch.vectorize(loop=l12)\n    b40 = sch.decompose_reduction(block=b1, loop=l16)\n    b41 = sch.get_block(name=\"T_multiply\", func_name=\"main\")\n    sch.reverse_compute_inline(block=b41)\n    sch.enter_postproc()\n    sch.unannotate(block_or_loop=b22, ann_key=\"meta_schedule.cooperative_fetch\")\n    l42, l43, l44, l45, l46 = sch.get_loops(block=b22)\n    l47, l48, l49 = sch.split(loop=l46, factors=[None, 64, 8], preserve_unit_iters=True)\n    sch.vectorize(loop=l49)\n    sch.bind(loop=l48, thread_axis=\"threadIdx.x\")\n    return sch.mod[\"main\"].with_attr(\"tir.is_scheduled\", 1)\n\n\n@T.prim_func(private=True)\n\ndef fused_fused_decode9_matmul7(\n    lv19: T.Buffer((T.int64(512), T.int64(22016)), \"uint32\"),\n    lv20: T.Buffer((T.int64(128), T.int64(22016)), \"float16\"),\n    lv1654: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    var_matmul_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    p_output0_intermediate = T.alloc_buffer((T.int64(4096), T.int64(22016)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(22016)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv19[v_i // T.int64(8), v_j], lv20[v_i // T.int64(32), v_j])\n            T.writes(p_output0_intermediate[v_i, v_j])\n            p_output0_intermediate[v_i, v_j] = (\n                T.Cast(\n                    \"float16\",\n                    T.bitwise_and(\n                        T.shift_right(\n                            lv19[v_i // T.int64(8), v_j],\n                            T.Cast(\"uint32\", v_i % T.int64(8)) * T.uint32(4),\n                        ),\n                        T.uint32(15),\n                    ),\n                )\n                - T.float16(7)\n            ) * lv20[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(22016), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1654[v_i0, v_i1, v_k], p_output0_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = (\n                var_matmul_intermediate[v_i0, v_i1, v_i2]\n                + lv1654[v_i0, v_i1, v_k] * p_output0_intermediate[v_k, v_i2]\n            )\n\n\n@T.prim_func(private=True)\ndef fused_fused_decode9_matmul7_after(\n    lv19: T.Buffer((T.int64(512), T.int64(22016)), \"uint32\"),\n    lv20: T.Buffer((T.int64(128), T.int64(22016)), \"float16\"),\n    lv1654: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    var_matmul_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_matmul_intermediate_local = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(352256)), \"float16\", scope=\"local\"\n    )\n    var_matmul_intermediate_local_batch = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(352256)), \"float16\", scope=\"local\"\n    )\n    lv19_local = T.alloc_buffer((T.int64(512), T.int64(22016)), \"uint32\", scope=\"local\")\n    lv20_local = T.alloc_buffer(\n        (T.int64(128), T.int64(22016)), \"float16\", scope=\"local\"\n    )\n    lv1654_shared = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(4096)), \"float16\", scope=\"shared\"\n    )\n    for i0_i1_i2_fused_0 in T.thread_binding(T.int64(172), thread=\"blockIdx.x\"):\n        for i0_i1_i2_fused_1 in T.thread_binding(T.int64(32), thread=\"threadIdx.x\"):\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for i0_i1_i2_fused_2_init in T.vectorized(T.int64(4)):\n                    with T.block(\"matmul_init\"):\n                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i2 = T.axis.spatial(\n                            T.int64(352256),\n                            i0_i1_i2_fused_0 * T.int64(2048)\n                            + i0_i1_i2_fused_1 * T.int64(64)\n                            + ax2_y * T.int64(4)\n                            + i0_i1_i2_fused_2_init\n                        )\n                        T.reads()\n                        T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                        var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float16(0)\n            for k_0 in range(T.int64(1)):\n                for ax2_1 in T.thread_binding(T.int64(32), thread=\"threadIdx.x\"):\n                    for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                        for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                            for ax2_2 in T.vectorized(T.int64(8)):\n                                with T.block(\"lv1654_shared\"):\n                                    v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                                    v2 = T.axis.spatial(\n                                        T.int64(4096),\n                                        k_0 * T.int64(4096)\n                                        + ax2_y * T.int64(256)\n                                        + ax2_1 * T.int64(8)\n                                        + ax2_2,\n                                    )\n                                    v2k = T.axis.spatial(\n                                        T.int64(4096),\n                                        (\n                                            ax2_y * T.int64(256)\n                                            + ax2_1 * T.int64(8)\n                                            + ax2_2\n                                        ),\n                                    )\n                                    T.reads(lv1654[v0, v1, v2])\n                                    T.writes(lv1654_shared[v0, v1, v2k])\n                                    lv1654_shared[v0, v1, v2k] = lv1654[v0, v1, v2]\n                for k_1 in range(T.int64(8)):\n                    for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                        for ax1 in T.vectorized(T.int64(4)):\n                            with T.block(\"matmul_init_local\"):\n                                v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i2k = T.axis.spatial(\n                                    T.int64(352256),\n                                    i0_i1_i2_fused_0 * T.int64(2048)\n                                    + i0_i1_i2_fused_1 * T.int64(64)\n                                    + ax2_y * T.int64(4)\n                                    + ax1,\n                                )\n                                T.reads()\n                                T.writes(\n                                    var_matmul_intermediate_local_batch[\n                                        v_i0, v_i1, v_i2k\n                                    ]\n                                )\n                                var_matmul_intermediate_local_batch[\n                                    v_i0, v_i1, v_i2k\n                                ] = T.float16(0)\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"lv20_local\"):\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(128)\n                                        + (k_1 * T.int64(16) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(22016),\n                                        i0_i1_i2_fused_0 * T.int64(128)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(lv20[v0, v1])\n                                    T.writes(lv20_local[v0, v1])\n                                    lv20_local[v0, v1] = lv20[v0, v1]\n                        for k_2 in range(T.int64(4)):\n                            for ax0 in range(T.int64(1)):\n                                for ax1 in T.vectorized(T.int64(4)):\n                                    with T.block(\"lv19_local\"):\n                                        v0 = T.axis.spatial(\n                                            T.int64(512),\n                                            k_0 * T.int64(512)\n                                            + (k_1 * T.int64(16) + ax2_y) * T.int64(4)\n                                            + k_2\n                                            + ax0,\n                                        )\n                                        v1 = T.axis.spatial(\n                                            T.int64(22016),\n                                            i0_i1_i2_fused_0 * T.int64(128)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + ax1,\n                                        )\n                                        T.reads(lv19[v0, v1])\n                                        T.writes(lv19_local[v0, v1])\n                                        lv19_local[v0, v1] = lv19[v0, v1]\n                            for k_3 in range(T.int64(8)):\n                                for i0_i1_i2_fused_2 in T.vectorized(T.int64(4)):\n                                    with T.block(\"matmul_update\"):\n                                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                        v_i2 = T.axis.spatial(\n                                            T.int64(22016),\n                                            i0_i1_i2_fused_0 * T.int64(128)\n                                            + i0_i1_i2_fused_1 * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_i2k = T.axis.spatial(\n                                            T.int64(352256),\n                                            i0_i1_i2_fused_0 * T.int64(2048)\n                                            + i0_i1_i2_fused_1 * T.int64(64)\n                                            + ax2_y * T.int64(4)\n                                            + i0_i1_i2_fused_2,\n                                        )\n                                        v_k = T.axis.reduce(\n                                            T.int64(4096),\n                                            k_0 * T.int64(4096)\n                                            + (k_1 * T.int64(16) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        v_ki = T.axis.reduce(\n                                            T.int64(4096),\n                                            (k_1 * T.int64(16) + ax2_y) * T.int64(32)\n                                            + k_2 * T.int64(8)\n                                            + k_3,\n                                        )\n                                        T.reads(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ],\n                                            lv1654_shared[v_i0, v_i1, v_ki],\n                                            lv19_local[v_k // T.int64(8), v_i2],\n                                        )\n                                        T.writes(\n                                            var_matmul_intermediate_local_batch[\n                                                v_i0, v_i1, v_i2k\n                                            ]\n                                        )\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] = var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ] + lv1654_shared[\n                                            v_i0, v_i1, v_ki\n                                        ] * (\n                                            (\n                                                T.Cast(\n                                                    \"float16\",\n                                                    T.bitwise_and(\n                                                        T.shift_right(\n                                                            lv19_local[\n                                                                v_k // T.int64(8), v_i2\n                                                            ],\n                                                            T.Cast(\n                                                                \"uint32\",\n                                                                v_k % T.int64(8),\n                                                            )\n                                                            * T.uint32(4),\n                                                        ),\n                                                        T.uint32(15),\n                                                    ),\n                                                )\n                                                - T.float16(7)\n                                            )\n                                        )\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n                                with T.block(\"multiple_scale\"):\n                                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i2k = T.axis.spatial(\n                                        T.int64(352256),\n                                        i0_i1_i2_fused_0 * T.int64(2048)\n                                        + i0_i1_i2_fused_1 * T.int64(64)\n                                        + ax2_y * T.int64(4)\n                                        + ax1,\n                                    )\n                                    v0 = T.axis.spatial(\n                                        T.int64(128),\n                                        k_0 * T.int64(128)\n                                        + (k_1 * T.int64(16) + ax2_y)\n                                        + ax0,\n                                    )\n                                    v1 = T.axis.spatial(\n                                        T.int64(22016),\n                                        i0_i1_i2_fused_0 * T.int64(128)\n                                        + i0_i1_i2_fused_1 * T.int64(4)\n                                        + ax1,\n                                    )\n                                    T.reads(\n                                        lv20_local[v0, v1],\n                                        var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ],\n                                    )\n                                    T.writes(\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                    )\n                                    var_matmul_intermediate_local[v_i0, v_i1, v_i2k] = (\n                                        var_matmul_intermediate_local[v_i0, v_i1, v_i2k]\n                                        + var_matmul_intermediate_local_batch[\n                                            v_i0, v_i1, v_i2k\n                                        ]\n                                        * lv20_local[v0, v1]\n                                    )\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_update\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(352256),\n                                i0_i1_i2_fused_0 * T.int64(2048)\n                                + i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.reads(var_matmul_intermediate_local[v0, v1, v_i2k])\n                            T.writes(lv1654_shared[v0, v1, v2])\n                            lv1654_shared[v0, v1, v2] = var_matmul_intermediate_local[\n                                v0, v1, v_i2k\n                            ]\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"reduction_1\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v_i2k = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(8))\n                            T.reads(lv1654_shared[v0, v1, v_i2k])\n                            T.writes(lv1654_shared[v0, v1, v_i2k])\n                            lv1654_shared[v0, v1, v_i2k] = (\n                                lv1654_shared[v0, v1, v_i2k]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(32)]\n                            )\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"reduction_2\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v_i2k = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(4))\n                            T.reads(lv1654_shared[v0, v1, v_i2k])\n                            T.writes(lv1654_shared[v0, v1, v_i2k])\n                            lv1654_shared[v0, v1, v_i2k] = (\n                                lv1654_shared[v0, v1, v_i2k]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(16)]\n                            )\n            for ax2_y in T.thread_binding(T.int64(16), thread=\"threadIdx.y\"):\n                for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                    for ax2 in T.vectorized(T.int64(4)):\n                        with T.block(\"var_matmul_intermediate_local\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(\n                                T.int64(22016),\n                                i0_i1_i2_fused_0 * T.int64(128)\n                                + i0_i1_i2_fused_1 * T.int64(4)\n                                + ax2,\n                            )\n                            v_i2k = T.axis.spatial(\n                                T.int64(2048),\n                                i0_i1_i2_fused_1 * T.int64(64)\n                                + ax2_y * T.int64(4)\n                                + ax2,\n                            )\n                            T.where(ax2_y < T.int64(1))\n                            T.reads(lv1654_shared[v0, v1, v_i2k])\n                            T.writes(var_matmul_intermediate[v0, v1, v2])\n                            var_matmul_intermediate[v0, v1, v2] = (\n                                lv1654_shared[v0, v1, v_i2k]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(4)]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(8)]\n                                + lv1654_shared[v0, v1, v_i2k + T.int64(12)]\n                            )\n\n\n@T.prim_func(private=True)\ndef fused_fused_decode7_matmul4(\n    lv3: T.Buffer((T.int64(512), T.int64(12288)), \"uint32\"),\n    lv4: T.Buffer((T.int64(128), T.int64(12288)), \"float16\"),\n    lv1615: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    var_matmul_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(12288)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    p_output0_intermediate = T.alloc_buffer((T.int64(4096), T.int64(12288)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(12288)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv3[v_i // T.int64(8), v_j], lv4[v_i // T.int64(32), v_j])\n            T.writes(p_output0_intermediate[v_i, v_j])\n            p_output0_intermediate[v_i, v_j] = (\n                T.Cast(\n                    \"float16\",\n                    T.bitwise_and(\n                        T.shift_right(\n                            lv3[v_i // T.int64(8), v_j],\n                            T.Cast(\"uint32\", v_i % T.int64(8)) * T.uint32(4),\n                        ),\n                        T.uint32(15),\n                    ),\n                )\n                - T.float16(7)\n            ) * lv4[v_i // T.int64(32), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(12288), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1615[v_i0, v_i1, v_k], p_output0_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = (\n                var_matmul_intermediate[v_i0, v_i1, v_i2]\n                + lv1615[v_i0, v_i1, v_k] * p_output0_intermediate[v_k, v_i2]\n            )\n\n\n@T.prim_func(private=True)\ndef fused_fused_decode7_matmul4_after(\n    lv3: T.Buffer((T.int64(512), T.int64(12288)), \"uint32\"),\n    lv4: T.Buffer((T.int64(128), T.int64(12288)), \"float16\"),\n    lv1615: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    var_matmul_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(12288)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_matmul_intermediate_local = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(24576)), \"float16\", scope=\"local\"\n    )\n    var_matmul_intermediate_local_batch = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(24576)), \"float16\", scope=\"local\"\n    )\n    lv3_local = T.alloc_buffer((T.int64(512), T.int64(12288)), \"uint32\", scope=\"local\")\n    lv4_local = T.alloc_buffer((T.int64(128), T.int64(12288)), \"float16\", scope=\"local\")\n    lv1615_shared = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(1024)), \"float16\", scope=\"shared\"\n    )\n    for i0_i1_i2_fused_0 in T.thread_binding(T.int64(48), thread=\"blockIdx.x\"):\n        for i0_i1_i2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for i0_i1_i2_fused_2_init in T.vectorized(T.int64(4)):\n                    with T.block(\"matmul_init\"):\n                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i2 = T.axis.spatial(\n                            T.int64(24576),\n                            i0_i1_i2_fused_0 * T.int64(512)\n                            + i0_i1_i2_fused_1 * T.int64(8)\n                            + ax2_y * T.int64(4)\n                            + i0_i1_i2_fused_2_init,\n                        )\n                        T.reads()\n                        T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                        var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float16(0)\n            for k_0 in range(T.int64(4)):\n                for ax2_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                            for ax2_2 in T.vectorized(T.int64(8)):\n                                with T.block(\"lv1615_shared\"):\n                                    v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                                    v2 = T.axis.spatial(\n                                        T.int64(4096),\n                                        k_0 * T.int64(1024)\n                                        + ax2_y * T.int64(512)\n                                        + ax2_1 * T.int64(8)\n                                        + ax2_2,\n                                    )\n                                    v2k = T.axis.spatial(\n                                        T.int64(1024),\n                                        (\n                                            ax2_y * T.int64(512)\n                                            + ax2_1 * T.int64(8)\n                                            + ax2_2\n                                        ),\n                                    )\n                                    T.reads(lv1615[v0, v1, v2])\n                                    T.writes(lv1615_shared[v0, v1, v2k])\n                                    lv1615_shared[v0, v1, v2k] = lv1615[v0, v1, v2]\n                for k_1 in range(T.int64(16)):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax1 in T.vectorized(T.int64(4)):\n                            with T.block(\"matmul_init_local\"):\n                                v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i2k = T.axis.spatial(\n                                    T.int64(24576),\n                                    i0_i1_i2_fused_0 * T.int64(512)\n                                    + i0_i1_i2_fused_1 * T.int64(8)\n                                    + ax2_y * T.int64(4)\n                                    + ax1,\n                                )\n                                T.reads()\n                                T.writes(\n                                    var_matmul_intermediate_local_batch[\n                                        v_i0, v_i1, v_i2k\n                                    ]\n                                )\n                                var_matmul_intermediate_local_batch[\n                                    v_i0, v_i1, v_i2k\n                                ] = T.float16(0)\n                        for ax0 in range(T.int64(1)):\n                            for ax1 in T.vectorized(T.int64(4)):\n...\n", "needle_token_start": 6907, "needle_token_end": 7805, "code_context_ntokens": 16361, "output": ["```python\n@T.prim_func(private=True)\ndef fused_decode5_fused_matmul6_multiply1_after(\n    lv1617: T.Buffer((T.int64(512), T.int64(11008)), \"uint32\"),\n    lv1618: T.Buffer((T.int64(128), T.int64(11008)), \"float16\"),\n    lv1622: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"),\n    lv4: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"),\n    p_output0_intermediate: T.Buffer(\n        (T.int64(1), T.int64(1), T.int64(11008)), \"float16\"\n    ),\n):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_matmul_intermediate_local = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\", scope=\"local\"\n    )\n    var_matmul_intermediate_local_batch = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(22016)), \"float16\", scope=\"local\"\n    )\n    lv1617_local = T.alloc_buffer(\n        (T.int64(512), T.int64(11008)), \"uint32\", scope=\"local\"\n    )\n    lv1618_local = T.alloc_buffer(\n        (T.int64(128), T.int64(11008)), \"float16\", scope=\"local\"\n    )\n    lv1622_shared = T.alloc_buffer(\n        (T.int64(1), T.int64(1), T.int64(1024)), \"float16\", scope=\"shared\"\n    )\n    for i0_i1_i2_fused_0 in T.thread_binding(T.int64(43), thread=\"blockIdx.x\"):\n        for i0_i1_i2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n            for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                for i0_i1_i2_fused_2_init in T.vectorized(T.int64(4)):\n                    with T.block(\"matmul_init\"):\n                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i2 = T.axis.spatial(\n                            T.int64(22016),\n                            i0_i1_i2_fused_0 * T.int64(512)\n                            + i0_i1_i2_fused_1 * T.int64(8)\n                            + ax2_y * T.int64(4)\n                            + i0_i1_i2_fused_2_init,\n                        )\n                        T.reads()\n                        T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                        var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float16(0)\n            for k_0 in range(T.int64(4)):\n                for ax2_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                    for ax2_y in T.thread_binding(T.int64(2), thread=\"threadIdx.y\"):\n                        for ax0, ax1 in T.grid(T.int64(1), T.int64(1)):\n                            for ax2_2 in T.vectorized(T.int64(8)):\n                                with T.block(\"lv1622_shared\"):\n                                    v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                                    v2 = T.axis.spatial(\n                                        T.int64(4096),\n                                        k_0 * T.int64(1024)\n                                        + ax2_y * T.int64(512)\n                                        + ax2_1 * T.int64(8)\n                                        + ax2_2,\n                                    )\n                                    v2k = T.axis.spatial(\n                                        T.int64(1024),\n                                        (\n                                            ax2_y *"]}
{"repo": "mlc-ai/mlc-llm", "name": "fused_decode4_matmul5_int3_fp16_before", "language": "python", "path": "mlc_llm/dispatch/llama/main.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function performs a decoding operation followed by a matrix multiplication, primarily working with 16-bit floating-point and 32-bit unsigned integer data types. This is typically used in neural network operations where decoding of compressed data is followed by a transformation via matrix multiplication.\n2. **Input**: The function takes four buffers: a 2D buffer of 32-bit unsigned integers, a 2D buffer of 16-bit floats, and two 3D buffers of 16-bit floats.\n3. **Output**: A 3D buffer of 16-bit floats that stores the result of the matrix multiplication.\n4. **Procedure**: \n   - First, a decoding operation is performed where each element of the integer buffer is processed with bitwise operations and combined with elements from the first float buffer to produce a new 2D float buffer.\n   - Next, a matrix multiplication is carried out between a slice of the second 3D float buffer and the newly created 2D float buffer, initializing the result buffer to zero before accumulation.\n   - The result of this multiplication is stored in the output 3D float buffer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                        T.reads()\n                        T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                        var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                    for k_0_0 in range(T.int64(2)):\n                        for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(22)):\n                            for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                                with T.block(\"lv6_shared\"):\n                                    v0 = T.axis.spatial(T.int64(1), ax0)\n                                    v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v2 = T.axis.spatial(T.int64(11008), k_0_0 * T.int64(5504) + (ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1))\n                                    T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(5504))\n                                    T.reads(lv6[v0, v1, v2])\n                                    T.writes(lv6_shared[v0, v1, v2])\n                                    T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                                    lv6_shared[v0, v1, v2] = lv6[v0, v1, v2]\n                        for k_0_1 in range(T.int64(86)):\n                            for ax0_0 in range(T.int64(8)):\n                                for ax0_1 in T.unroll(T.int64(8)):\n                                    for ax1 in range(T.int64(1)):\n                                        with T.block(\"decode\"):\n                                            v_j = T.axis.spatial(T.int64(11008), k_0_0 * T.int64(5504) + k_0_1 * T.int64(64) + ax0_0 * T.int64(8) + ax0_1)\n                                            v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                            T.reads(lv1158[v_j // T.int64(8), v_i], lv60[v_j // T.int64(32), v_i], lv61[v_j // T.int64(32), v_i])\n                                            T.writes(var_decode_intermediate_local[v_j, v_i])\n                                            var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1158[v_j // T.int64(8), v_i], T.Cast(\"uint32\", v_j % T.int64(8) * T.int64(4))), T.uint32(15))) * lv60[v_j // T.int64(32), v_i] + lv61[v_j // T.int64(32), v_i]\n                            for k_0_2_k_1_fused in range(T.int64(64)):\n                                with T.block(\"matmul_update\"):\n                                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                                    v_k = T.axis.reduce(T.int64(11008), k_0_0 * T.int64(5504) + k_0_1 * T.int64(64) + k_0_2_k_1_fused)\n                                    T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv6_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv6_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2]\n                    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                        with T.block(\"var_matmul_intermediate_local\"):\n                            v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                            v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                            T.reads(lv4[v0, v1, v2], var_matmul_intermediate_local[v0, v1, v2])\n                            T.writes(p_output0_intermediate[v0, v1, v2])\n                            p_output0_intermediate[v0, v1, v2] = lv4[v0, v1, v2] + var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode3_matmul1_cast_int3_fp16_before(lv2931: T.Buffer((T.int64(412), T.int64(32000)), \"uint32\"), lv2932: T.Buffer((T.int64(103), T.int64(32000)), \"float16\"), lv3025: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(32000)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(32000)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv2931[v_i // T.int64(10), v_j], lv2932[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv2931[v_i // T.int64(10), v_j], T.Cast(\"uint32\", v_i % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv2932[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(32000), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv3025[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv3025[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(32000)):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(p_output0_intermediate[v_i0, v_i1, v_i2])\n            p_output0_intermediate[v_i0, v_i1, v_i2] = T.Cast(\"float32\", var_matmul_intermediate[v_i0, v_i1, v_i2])\n\n\n@T.prim_func\ndef fused_decode3_matmul1_cast_int3_fp16_after(lv1123: T.Buffer((T.int64(412), T.int64(32000)), \"uint32\"), lv5866: T.Buffer((T.int64(103), T.int64(32000)), \"float16\"), lv1511: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    # with T.block(\"root\"):\n    var_decode_intermediate_pad_local = T.alloc_buffer((T.int64(4120), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_pad_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    lv1511_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4120)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(125), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(17)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        with T.block(\"lv1511_shared\"):\n                            v0 = T.axis.spatial(T.int64(1), ax0)\n                            v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v2 = T.axis.spatial(T.int64(4120), ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1)\n                            T.reads(lv1511[v0, v1, v2])\n                            T.writes(lv1511_shared[v0, v1, v2])\n                            T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(4120))\n                            T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                            lv1511_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(4096), lv1511[v0, v1, v2], T.float16(0))\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(103)):\n                    for ax0_0 in T.unroll(T.int64(40)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"var_decode_intermediate_pad\"):\n                                v0 = T.axis.spatial(T.int64(4120), k_0_0 * T.int64(40) + ax0_0)\n                                v1 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv1123[v0 // T.int64(10), v1], lv5866[v0 // T.int64(40), v1])\n                                T.writes(var_decode_intermediate_pad_local[v0, v1])\n                                var_decode_intermediate_pad_local[v0, v1] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1123[v0 // T.int64(10), v1], T.Cast(\"uint32\", v0 % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2], lv1511_shared[v_i0, v_i1, v_k], var_decode_intermediate_pad_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] + lv1511_shared[v_i0, v_i1, v_k] * var_decode_intermediate_pad_local[v_k, v_i2] * lv5866[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_pad_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_pad_local[v0, v1, v2])\n                        T.writes(var_matmul_intermediate[v0, v1, v2])\n                        var_matmul_intermediate[v0, v1, v2] = T.Cast(\"float32\", var_matmul_intermediate_pad_local[v0, v1, v2])\n\n\n@T.prim_func\ndef fused_decode4_fused_matmul5_add3_int3_fp16_before(lv1605: T.Buffer((T.int64(412), T.int64(4096)), \"uint32\"), lv1606: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv164: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv1518: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(4096)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1605[v_i // T.int64(10), v_j], lv1606[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1605[v_i // T.int64(10), v_j], T.Cast(\"uint32\", v_i % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1606[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv164[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv164[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(4096)):\n        with T.block(\"T_add\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv1518[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = lv1518[v_ax0, v_ax1, v_ax2] + var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_decode4_fused_matmul5_add3_int3_fp16_after(lv1143: T.Buffer((T.int64(412), T.int64(4096)), \"uint32\"), lv36: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv3: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv2710: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4120), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    lv3_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4120)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(17)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        with T.block(\"lv3_shared\"):\n                            v0 = T.axis.spatial(T.int64(1), ax0)\n                            v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v2 = T.axis.spatial(T.int64(4120), ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1)\n                            T.reads(lv3[v0, v1, v2])\n                            T.writes(lv3_shared[v0, v1, v2])\n                            T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(4120))\n                            T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                            lv3_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(4096), lv3[v0, v1, v2], T.float16(0))\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(103)):\n                    for ax0_0 in T.unroll(T.int64(40)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"decode\"):\n                                v_j = T.axis.spatial(T.int64(4120), k_0_0 * T.int64(40) + ax0_0)\n                                v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv1143[v_j // T.int64(10), v_i], lv36[v_j // T.int64(40), v_i])\n                                T.writes(var_decode_intermediate_local[v_j, v_i])\n                                var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1143[v_j // T.int64(10), v_i], T.Cast(\"uint32\", v_j % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv3_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv3_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2] * lv36[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(lv2710[v0, v1, v2], var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(p_output0_intermediate[v0, v1, v2])\n                        p_output0_intermediate[v0, v1, v2] = lv2710[v0, v1, v2] + var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\n\ndef fused_decode4_matmul5_int3_fp16_before(lv1587: T.Buffer((T.int64(412), T.int64(4096)), \"uint32\"), lv1588: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv1520: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1587[v_i // T.int64(10), v_j], lv1588[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1587[v_i // T.int64(10), v_j], T.Cast(\"uint32\", v_i % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1588[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1520[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv1520[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n\n\n@T.prim_func\ndef fused_decode4_matmul5_int3_fp16_after(lv1128: T.Buffer((T.int64(412), T.int64(4096)), \"uint32\"), lv12: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv2712: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4120), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    lv2712_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4120)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(17)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        with T.block(\"lv2712_shared\"):\n                            v0 = T.axis.spatial(T.int64(1), ax0)\n                            v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v2 = T.axis.spatial(T.int64(4120), ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1)\n                            T.reads(lv2712[v0, v1, v2])\n                            T.writes(lv2712_shared[v0, v1, v2])\n                            T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(4120))\n                            T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                            lv2712_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(4096), lv2712[v0, v1, v2], T.float16(0))\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(103)):\n                    for ax0_0 in T.unroll(T.int64(40)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"decode\"):\n                                v_j = T.axis.spatial(T.int64(4120), k_0_0 * T.int64(40) + ax0_0)\n                                v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv1128[v_j // T.int64(10), v_i], lv12[v_j // T.int64(40), v_i])\n                                T.writes(var_decode_intermediate_local[v_j, v_i])\n                                var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1128[v_j // T.int64(10), v_i], T.Cast(\"uint32\", v_j % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv2712_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv2712_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2] * lv12[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(var_matmul_intermediate[v0, v1, v2])\n                        var_matmul_intermediate[v0, v1, v2] = var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode5_fused_matmul8_multiply1_int3_fp16_before(lv1617: T.Buffer((T.int64(412), T.int64(11008)), \"uint32\"), lv1618: T.Buffer((T.int64(103), T.int64(11008)), \"float16\"), lv1557: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv3: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(11008)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(11008)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1617[v_i // T.int64(10), v_j], lv1618[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1617[v_i // T.int64(10), v_j], T.Cast(\"uint32\", v_i % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1618[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(11008), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1557[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv1557[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv3[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = lv3[v_ax0, v_ax1, v_ax2] * var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_decode5_fused_matmul8_multiply1_int3_fp16_after(lv1153: T.Buffer((T.int64(412), T.int64(11008)), \"uint32\"), lv52: T.Buffer((T.int64(103), T.int64(11008)), \"float16\"), lv2749: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv5: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4120), T.int64(11008)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), scope=\"local\", dtype=\"float16\")\n    lv2749_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4120)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(43), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(17)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        with T.block(\"lv2749_shared\"):\n                            v0 = T.axis.spatial(T.int64(1), ax0)\n                            v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v2 = T.axis.spatial(T.int64(4120), ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1)\n                            T.reads(lv2749[v0, v1, v2])\n                            T.writes(lv2749_shared[v0, v1, v2])\n                            T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(4120))\n                            T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                            lv2749_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(4096), lv2749[v0, v1, v2], T.float16(0))\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(11008), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(103)):\n                    for ax0_0 in T.unroll(T.int64(40)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"decode\"):\n                                v_j = T.axis.spatial(T.int64(4120), k_0_0 * T.int64(40) + ax0_0)\n                                v_i = T.axis.spatial(T.int64(11008), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv1153[v_j // T.int64(10), v_i], lv52[v_j // T.int64(40), v_i])\n                                T.writes(var_decode_intermediate_local[v_j, v_i])\n                                var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1153[v_j // T.int64(10), v_i], T.Cast(\"uint32\", v_j % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(11008), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv2749_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv2749_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2] * lv52[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(11008), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(lv5[v0, v1, v2], var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(p_output0_intermediate[v0, v1, v2])\n                        p_output0_intermediate[v0, v1, v2] = lv5[v0, v1, v2] * var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode5_fused_matmul8_silu1_int3_fp16_before(lv1611: T.Buffer((T.int64(412), T.int64(11008)), \"uint32\"), lv1612: T.Buffer((T.int64(103), T.int64(11008)), \"float16\"), lv1557: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(11008)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    compute = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(11008)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1611[v_i // T.int64(10), v_j], lv1612[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1611[v_i // T.int64(10), v_j], T.Cast(\"uint32\", v_i % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1612[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(11008), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1557[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv1557[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(compute[v_i0, v_i1, v_i2])\n            compute[v_i0, v_i1, v_i2] = T.sigmoid(var_matmul_intermediate[v_i0, v_i1, v_i2])\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"T_multiply\"):\n...\n", "needle_token_start": 8523, "needle_token_end": 9368, "code_context_ntokens": 16281, "output": ["```python\n@T.prim_func\ndef fused_decode5_fused_matmul8_silu1_int3_fp16_before(lv1611: T.Buffer((T.int64(412), T.int64(11008)), \"uint32\"), lv1612: T.Buffer((T.int64(103), T.int64(11008)), \"float16\"), lv1557: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(11008)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    compute = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(11008)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1611[v_i // T.int64(10), v_j], lv1612[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1611[v_i // T.int64(10), v_j], T.Cast(\"uint32\", v_i % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1612[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(11008), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1557[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv1557[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(compute[v_i0, v_i1, v_i2])\n            compute[v_i0, v_i1, v_i2] = T.sigmoid(var_matmul_intermediate[v_i0, v_i1, v_i2])\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(compute[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v"]}
{"repo": "mlc-ai/mlc-llm", "name": "fused_NT_matmul3_add1_before", "language": "python", "path": "mlc_llm/dispatch/llama/main.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function performs a matrix multiplication followed by an addition operation. It is designed to compute the result of multiplying a matrix with another transposed matrix and then adding a third matrix to the result.\n2. **Input**: Three matrices are provided as input. The first matrix has dimensions (1, n, 11008), the second matrix is transposed with dimensions (4096, 11008), and the third matrix has dimensions (1, n, 4096).\n3. **Output**: The output is a matrix of dimensions (1, n, 4096), which is the result of the matrix multiplication followed by the addition with the third input matrix.\n4. **Procedure**: \n   - The function first initializes an intermediate matrix to store the result of the matrix multiplication.\n   - It then iterates over the elements of the input matrices, performing the multiplication of the first matrix and the transposed second matrix, and accumulates the results in the intermediate matrix.\n   - After completing the matrix multiplication, the function iterates over the elements of the intermediate matrix and the third input matrix, adding corresponding elements together to produce the final output matrix.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                v_ax2 = T.axis.spatial(T.int64(1), T.int64(0))\n                v_ax3 = T.axis.spatial(n, (ax0_ax1_ax2_ax3_fused_0 * T.int64(32) + ax0_ax1_ax2_ax3_fused_1) % n)\n                T.reads(var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2, v_ax3], lv2709[v_ax0, T.int64(0), v_ax2, v_ax3])\n                T.writes(var_T_maximum_intermediate[v_ax0, v_ax1, v_ax2, v_ax3])\n                var_T_maximum_intermediate[v_ax0, v_ax1, v_ax2, v_ax3] = T.max(var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2, v_ax3] * T.float32(0.088388349161020605) + lv2709[v_ax0, T.int64(0), v_ax2, v_ax3], T.float32(-3.4028234663852886e+38))\n\n\n@T.prim_func\ndef fused_NT_matmul2_multiply_before(p_lv43: T.handle, linear_weight6: T.Buffer((T.int64(11008), T.int64(4096)), \"float32\"), p_lv48: T.handle, p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    lv43 = T.match_buffer(p_lv43, (T.int64(1), n, T.int64(4096)))\n    lv48 = T.match_buffer(p_lv48, (T.int64(1), n, T.int64(11008)))\n    var_T_multiply_intermediate = T.match_buffer(p_output0, (T.int64(1), n, T.int64(11008)))\n    # with T.block(\"root\"):\n    var_NT_matmul_intermediate = T.alloc_buffer((T.int64(1), n, T.int64(11008)))\n    for i0, i1, i2, k in T.grid(T.int64(1), n, T.int64(11008), T.int64(4096)):\n        with T.block(\"NT_matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv43[v_i0, v_i1, v_k], linear_weight6[v_i2, v_k])\n            T.writes(var_NT_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0)\n            var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = var_NT_matmul_intermediate[v_i0, v_i1, v_i2] + lv43[v_i0, v_i1, v_k] * linear_weight6[v_i2, v_k]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), n, T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv48[v_ax0, v_ax1, v_ax2], var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(var_T_multiply_intermediate[v_ax0, v_ax1, v_ax2])\n            var_T_multiply_intermediate[v_ax0, v_ax1, v_ax2] = lv48[v_ax0, v_ax1, v_ax2] * var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_NT_matmul2_multiply_after(p_lv37: T.handle, linear_weight6: T.Buffer((T.int64(11008), T.int64(4096)), \"float32\"), p_lv42: T.handle, p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    n = T.int64()\n    lv37 = T.match_buffer(p_lv37, (T.int64(1), n, T.int64(4096)))\n    lv42 = T.match_buffer(p_lv42, (T.int64(1), n, T.int64(11008)))\n    var_T_multiply_intermediate = T.match_buffer(p_output0, (T.int64(1), n, T.int64(11008)))\n    # with T.block(\"root\"):\n    for i1_0 in T.thread_binding((n + T.int64(31)) // T.int64(32), thread=\"blockIdx.y\"):\n        with T.block(\"NT_matmul_o\"):\n            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n            v_i1_o = T.axis.spatial((n + T.int64(31)) // T.int64(32), i1_0)\n            T.reads(lv37[T.Add(v_i0, T.int64(0)), v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(4096)], linear_weight6[T.int64(0):T.int64(11008), T.int64(0):T.int64(4096)], lv42[v_i0, v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(11008)])\n            T.writes(var_T_multiply_intermediate[v_i0, v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(11008)])\n            var_NT_matmul_intermediate_pad_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(11008)), scope=\"local\")\n            lv37_pad_shared = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(4096)), scope=\"shared\")\n            linear_weight6_shared = T.alloc_buffer((T.int64(11008), T.int64(4096)), scope=\"shared\")\n            for i0_0_i1_1_0_i2_0_fused in T.thread_binding(T.int64(344), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n                for i0_1_i1_1_1_i2_1_fused in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n                    for i0_2_i1_1_2_i2_2_fused in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                        for i1_1_3_init, i2_3_init, i1_1_4_init, i2_4_init in T.grid(T.int64(2), T.int64(2), T.int64(2), T.int64(2)):\n                            with T.block(\"NT_matmul_init\"):\n                                v_i1_i = T.axis.spatial(T.int64(32), i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(4) + i1_1_3_init * T.int64(2) + i1_1_4_init)\n                                v_i2_i = T.axis.spatial(T.int64(11008), i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(4) + i2_3_init * T.int64(2) + i2_4_init)\n                                T.reads()\n                                T.writes(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i])\n                                T.block_attr({\"meta_schedule.thread_extent_high_inclusive\": 256, \"meta_schedule.thread_extent_low_inclusive\": 32, \"meta_schedule.tiling_structure\": \"SSSRRSRS\"})\n                                var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] = T.float32(0)\n                        for k_0 in range(T.int64(128)):\n                            for ax0_ax1_ax2_fused_0 in range(T.int64(4)):\n                                for ax0_ax1_ax2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                                    for ax0_ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                                        with T.block(\"lv37_pad_shared\"):\n                                            v0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                            v1 = T.axis.spatial(T.int64(32), (ax0_ax1_ax2_fused_0 * T.int64(256) + ax0_ax1_ax2_fused_1 * T.int64(4) + ax0_ax1_ax2_fused_2) // T.int64(32))\n                                            v2 = T.axis.spatial(T.int64(4096), k_0 * T.int64(32) + (ax0_ax1_ax2_fused_0 * T.int64(256) + ax0_ax1_ax2_fused_1 * T.int64(4) + ax0_ax1_ax2_fused_2) % T.int64(32))\n                                            T.reads(lv37[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2])\n                                            T.writes(lv37_pad_shared[v0, v1, v2])\n                                            lv37_pad_shared[v0, v1, v2] = T.if_then_else(v_i1_o * T.int64(32) + v1 < n, lv37[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2], T.float32(0))\n                            for ax0_ax1_fused_0 in range(T.int64(8)):\n                                for ax0_ax1_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                                    for ax0_ax1_fused_2 in T.vectorized(T.int64(2)):\n                                        with T.block(\"linear_weight6_shared\"):\n                                            v0 = T.axis.spatial(T.int64(11008), i0_0_i1_1_0_i2_0_fused * T.int64(32) + (ax0_ax1_fused_0 * T.int64(128) + ax0_ax1_fused_1 * T.int64(2) + ax0_ax1_fused_2) // T.int64(32))\n                                            v1 = T.axis.spatial(T.int64(4096), k_0 * T.int64(32) + (ax0_ax1_fused_0 * T.int64(128) + ax0_ax1_fused_1 * T.int64(2) + ax0_ax1_fused_2) % T.int64(32))\n                                            T.reads(linear_weight6[v0, v1])\n                                            T.writes(linear_weight6_shared[v0, v1])\n                                            linear_weight6_shared[v0, v1] = linear_weight6[v0, v1]\n                            for k_1, i0_3, i1_1_3, i2_3, k_2, i0_4, i1_1_4, i2_4 in T.grid(T.int64(8), T.int64(1), T.int64(2), T.int64(2), T.int64(4), T.int64(1), T.int64(2), T.int64(2)):\n                                with T.block(\"NT_matmul_update\"):\n                                    v_i1_i = T.axis.spatial(T.int64(32), i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(4) + i1_1_3 * T.int64(2) + i1_1_4)\n                                    v_i2_i = T.axis.spatial(T.int64(11008), i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(4) + i2_3 * T.int64(2) + i2_4)\n                                    v_k_i = T.axis.reduce(T.int64(4096), k_0 * T.int64(32) + k_1 * T.int64(4) + k_2)\n                                    T.reads(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i], lv37_pad_shared[T.int64(0), v_i1_i, v_k_i], linear_weight6_shared[v_i2_i, v_k_i])\n                                    T.writes(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i])\n                                    T.block_attr({\"meta_schedule.thread_extent_high_inclusive\": 256, \"meta_schedule.thread_extent_low_inclusive\": 32, \"meta_schedule.tiling_structure\": \"SSSRRSRS\"})\n                                    var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] = var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] + lv37_pad_shared[T.int64(0), v_i1_i, v_k_i] * linear_weight6_shared[v_i2_i, v_k_i]\n                        for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(4), T.int64(4)):\n                            with T.block(\"var_NT_matmul_intermediate_pad_local\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(32), i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(4) + ax1)\n                                v2 = T.axis.spatial(T.int64(11008), i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(4) + ax2)\n                                T.reads(lv42[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2], var_NT_matmul_intermediate_pad_local[v0, v1, v2])\n                                T.writes(var_T_multiply_intermediate[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2])\n                                # if T.int64(0) <= v_i0 and v_i0 < T.int64(1) and T.int64(0) <= v_i1_o * T.int64(32) + v1 and v_i1_o * T.int64(32) + v1 < n:\n                                if v_i1_o * T.int64(32) + v1 < n:\n                                    var_T_multiply_intermediate[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2] = lv42[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2] * var_NT_matmul_intermediate_pad_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_NT_matmul2_silu_before(p_lv43: T.handle, linear_weight4: T.Buffer((T.int64(11008), T.int64(4096)), \"float32\"), p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    lv43 = T.match_buffer(p_lv43, (T.int64(1), n, T.int64(4096)))\n    var_T_multiply_intermediate = T.match_buffer(p_output0, (T.int64(1), n, T.int64(11008)))\n    # with T.block(\"root\"):\n    var_NT_matmul_intermediate = T.alloc_buffer((T.int64(1), n, T.int64(11008)))\n    compute = T.alloc_buffer((T.int64(1), n, T.int64(11008)))\n    for i0, i1, i2, k in T.grid(T.int64(1), n, T.int64(11008), T.int64(4096)):\n        with T.block(\"NT_matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv43[v_i0, v_i1, v_k], linear_weight4[v_i2, v_k])\n            T.writes(var_NT_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0)\n            var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = var_NT_matmul_intermediate[v_i0, v_i1, v_i2] + lv43[v_i0, v_i1, v_k] * linear_weight4[v_i2, v_k]\n    for i0, i1, i2 in T.grid(T.int64(1), n, T.int64(11008)):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_NT_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(compute[v_i0, v_i1, v_i2])\n            compute[v_i0, v_i1, v_i2] = T.sigmoid(var_NT_matmul_intermediate[v_i0, v_i1, v_i2])\n    for ax0, ax1, ax2 in T.grid(T.int64(1), n, T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2], compute[v_ax0, v_ax1, v_ax2])\n            T.writes(var_T_multiply_intermediate[v_ax0, v_ax1, v_ax2])\n            var_T_multiply_intermediate[v_ax0, v_ax1, v_ax2] = var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] * compute[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_NT_matmul2_silu_after(p_lv37: T.handle, linear_weight4: T.Buffer((T.int64(11008), T.int64(4096)), \"float32\"), p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    n = T.int64()\n    lv37 = T.match_buffer(p_lv37, (T.int64(1), n, T.int64(4096)))\n    var_T_multiply_intermediate = T.match_buffer(p_output0, (T.int64(1), n, T.int64(11008)))\n    # with T.block(\"root\"):\n    var_NT_matmul_intermediate = T.alloc_buffer((T.int64(1), n, T.int64(11008)))\n    for i1_0 in T.thread_binding((n + T.int64(31)) // T.int64(32), thread=\"blockIdx.y\"):\n        with T.block(\"NT_matmul_o\"):\n            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n            v_i1_o = T.axis.spatial((n + T.int64(31)) // T.int64(32), i1_0)\n            T.reads(lv37[T.Add(v_i0, T.int64(0)), v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(4096)], linear_weight4[T.int64(0):T.int64(11008), T.int64(0):T.int64(4096)])\n            T.writes(var_NT_matmul_intermediate[v_i0, v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(11008)])\n            var_NT_matmul_intermediate_pad_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(11008)), scope=\"local\")\n            lv37_pad_shared = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(4096)), scope=\"shared\")\n            linear_weight4_shared = T.alloc_buffer((T.int64(11008), T.int64(4096)), scope=\"shared\")\n            for i0_0_i1_1_0_i2_0_fused in T.thread_binding(T.int64(344), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n                for i0_1_i1_1_1_i2_1_fused in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n                    for i0_2_i1_1_2_i2_2_fused in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                        for i1_1_3_init, i2_3_init, i1_1_4_init, i2_4_init in T.grid(T.int64(2), T.int64(4), T.int64(2), T.int64(1)):\n                            with T.block(\"NT_matmul_init\"):\n                                v_i1_i = T.axis.spatial(T.int64(32), i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(4) + i1_1_3_init * T.int64(2) + i1_1_4_init)\n                                v_i2_i = T.axis.spatial(T.int64(11008), i2_4_init + i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(4) + i2_3_init)\n                                T.reads()\n                                T.writes(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i])\n                                T.block_attr({\"meta_schedule.thread_extent_high_inclusive\": 256, \"meta_schedule.thread_extent_low_inclusive\": 32, \"meta_schedule.tiling_structure\": \"SSSRRSRS\"})\n                                var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] = T.float32(0)\n                        for k_0 in range(T.int64(128)):\n                            for ax0_ax1_ax2_fused_0 in range(T.int64(4)):\n                                for ax0_ax1_ax2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                                    for ax0_ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                                        with T.block(\"lv37_pad_shared\"):\n                                            v0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                            v1 = T.axis.spatial(T.int64(32), (ax0_ax1_ax2_fused_0 * T.int64(256) + ax0_ax1_ax2_fused_1 * T.int64(4) + ax0_ax1_ax2_fused_2) // T.int64(32))\n                                            v2 = T.axis.spatial(T.int64(4096), k_0 * T.int64(32) + (ax0_ax1_ax2_fused_0 * T.int64(256) + ax0_ax1_ax2_fused_1 * T.int64(4) + ax0_ax1_ax2_fused_2) % T.int64(32))\n                                            T.reads(lv37[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2])\n                                            T.writes(lv37_pad_shared[v0, v1, v2])\n                                            lv37_pad_shared[v0, v1, v2] = T.if_then_else(v_i1_o * T.int64(32) + v1 < n, lv37[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2], T.float32(0))\n                            for ax0_ax1_fused_0 in range(T.int64(8)):\n                                for ax0_ax1_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                                    for ax0_ax1_fused_2 in T.vectorized(T.int64(2)):\n                                        with T.block(\"linear_weight4_shared\"):\n                                            v0 = T.axis.spatial(T.int64(11008), i0_0_i1_1_0_i2_0_fused * T.int64(32) + (ax0_ax1_fused_0 * T.int64(128) + ax0_ax1_fused_1 * T.int64(2) + ax0_ax1_fused_2) // T.int64(32))\n                                            v1 = T.axis.spatial(T.int64(4096), k_0 * T.int64(32) + (ax0_ax1_fused_0 * T.int64(128) + ax0_ax1_fused_1 * T.int64(2) + ax0_ax1_fused_2) % T.int64(32))\n                                            T.reads(linear_weight4[v0, v1])\n                                            T.writes(linear_weight4_shared[v0, v1])\n                                            linear_weight4_shared[v0, v1] = linear_weight4[v0, v1]\n                            for k_1, i0_3, i1_1_3, i2_3, k_2, i0_4, i1_1_4, i2_4 in T.grid(T.int64(8), T.int64(1), T.int64(2), T.int64(4), T.int64(4), T.int64(1), T.int64(2), T.int64(1)):\n                                with T.block(\"NT_matmul_update\"):\n                                    v_i1_i = T.axis.spatial(T.int64(32), i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(4) + i1_1_3 * T.int64(2) + i1_1_4)\n                                    v_i2_i = T.axis.spatial(T.int64(11008), i2_4 + i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(4) + i2_3)\n                                    v_k_i = T.axis.reduce(T.int64(4096), k_0 * T.int64(32) + k_1 * T.int64(4) + k_2)\n                                    T.reads(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i], lv37_pad_shared[T.int64(0), v_i1_i, v_k_i], linear_weight4_shared[v_i2_i, v_k_i])\n                                    T.writes(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i])\n                                    T.block_attr({\"meta_schedule.thread_extent_high_inclusive\": 256, \"meta_schedule.thread_extent_low_inclusive\": 32, \"meta_schedule.tiling_structure\": \"SSSRRSRS\"})\n                                    var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] = var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] + lv37_pad_shared[T.int64(0), v_i1_i, v_k_i] * linear_weight4_shared[v_i2_i, v_k_i]\n                        for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(4), T.int64(4)):\n                            with T.block(\"var_NT_matmul_intermediate_pad_local\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(32), i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(4) + ax1)\n                                v2 = T.axis.spatial(T.int64(11008), i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(4) + ax2)\n                                T.reads(var_NT_matmul_intermediate_pad_local[v0, v1, v2])\n                                T.writes(var_NT_matmul_intermediate[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2])\n                                # if T.int64(0) <= v_i0 and v_i0 < T.int64(1) and T.int64(0) <= v_i1_o * T.int64(32) + v1 and v_i1_o * T.int64(32) + v1 < n:\n                                if v_i1_o * T.int64(32) + v1 < n:\n                                    var_NT_matmul_intermediate[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2] = var_NT_matmul_intermediate_pad_local[v0, v1, v2]\n    for ax0_ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for ax0_ax1_ax2_fused_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n            for ax0_ax1_ax2_fused_0 in range((n * T.int64(11008) + T.int64(65535)) // T.int64(65536)):\n                with T.block(\"T_multiply\"):\n                    v_ax0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_ax1 = T.axis.spatial(n, (ax0_ax1_ax2_fused_0 * T.int64(65536) + ax0_ax1_ax2_fused_1 * T.int64(256) + ax0_ax1_ax2_fused_2) // T.int64(11008))\n                    v_ax2 = T.axis.spatial(T.int64(11008), (ax0_ax1_ax2_fused_0 * T.int64(65536) + ax0_ax1_ax2_fused_1 * T.int64(256) + ax0_ax1_ax2_fused_2) % T.int64(11008))\n                    T.where((ax0_ax1_ax2_fused_0 * T.int64(256) + ax0_ax1_ax2_fused_1) * T.int64(256) + ax0_ax1_ax2_fused_2 < n * T.int64(11008))\n                    T.reads(var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n                    T.writes(var_T_multiply_intermediate[v_ax0, v_ax1, v_ax2])\n                    var_T_multiply_intermediate[v_ax0, v_ax1, v_ax2] = var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2] * T.sigmoid(var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n\n\n@T.prim_func\n\ndef fused_NT_matmul3_add1_before(p_lv49: T.handle, linear_weight5: T.Buffer((T.int64(4096), T.int64(11008)), \"float32\"), p_lv42: T.handle, p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    lv49 = T.match_buffer(p_lv49, (T.int64(1), n, T.int64(11008)))\n    lv42 = T.match_buffer(p_lv42, (T.int64(1), n, T.int64(4096)))\n    var_T_add_intermediate = T.match_buffer(p_output0, (T.int64(1), n, T.int64(4096)))\n    # with T.block(\"root\"):\n    var_NT_matmul_intermediate = T.alloc_buffer((T.int64(1), n, T.int64(4096)))\n    for i0, i1, i2, k in T.grid(T.int64(1), n, T.int64(4096), T.int64(11008)):\n        with T.block(\"NT_matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv49[v_i0, v_i1, v_k], linear_weight5[v_i2, v_k])\n            T.writes(var_NT_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0)\n            var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = var_NT_matmul_intermediate[v_i0, v_i1, v_i2] + lv49[v_i0, v_i1, v_k] * linear_weight5[v_i2, v_k]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), n, T.int64(4096)):\n        with T.block(\"T_add\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv42[v_ax0, v_ax1, v_ax2], var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(var_T_add_intermediate[v_ax0, v_ax1, v_ax2])\n            var_T_add_intermediate[v_ax0, v_ax1, v_ax2] = lv42[v_ax0, v_ax1, v_ax2] + var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_NT_matmul3_add1_after(p_lv43: T.handle, linear_weight5: T.Buffer((T.int64(4096), T.int64(11008)), \"float32\"), p_lv36: T.handle, p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    n = T.int64()\n    lv43 = T.match_buffer(p_lv43, (T.int64(1), n, T.int64(11008)))\n    lv36 = T.match_buffer(p_lv36, (T.int64(1), n, T.int64(4096)))\n    var_T_add_intermediate = T.match_buffer(p_output0, (T.int64(1), n, T.int64(4096)))\n    # with T.block(\"root\"):\n    for i1_0 in T.thread_binding((n + T.int64(31)) // T.int64(32), thread=\"blockIdx.y\"):\n        with T.block(\"NT_matmul_o\"):\n            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n            v_i1_o = T.axis.spatial((n + T.int64(31)) // T.int64(32), i1_0)\n            T.reads(lv43[T.Add(v_i0, T.int64(0)), v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(11008)], linear_weight5[T.int64(0):T.int64(4096), T.int64(0):T.int64(11008)], lv36[v_i0, v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(4096)])\n            T.writes(var_T_add_intermediate[v_i0, v_i1_o * T.int64(32):v_i1_o * T.int64(32) + T.int64(32), T.int64(0):T.int64(4096)])\n            var_NT_matmul_intermediate_pad_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(4096)), scope=\"local\")\n            lv43_pad_shared = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(11008)), scope=\"shared\")\n            linear_weight5_shared = T.alloc_buffer((T.int64(4096), T.int64(11008)), scope=\"shared\")\n            for i0_0_i1_1_0_i2_0_fused in T.thread_binding(T.int64(128), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n                for i0_1_i1_1_1_i2_1_fused in T.thread_binding(T.int64(4), thread=\"vthread.x\"):\n                    for i0_2_i1_1_2_i2_2_fused in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                        for i1_1_3_init, i2_3_init, i1_1_4_init, i2_4_init in T.grid(T.int64(2), T.int64(2), T.int64(1), T.int64(1)):\n                            with T.block(\"NT_matmul_init\"):\n                                v_i1_i = T.axis.spatial(T.int64(32), i0_1_i1_1_1_i2_1_fused // T.int64(2) * T.int64(16) + i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(2) + i1_1_3_init + i1_1_4_init)\n                                v_i2_i = T.axis.spatial(T.int64(4096), i2_4_init + i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_1_i1_1_1_i2_1_fused % T.int64(2) * T.int64(16) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(2) + i2_3_init)\n                                T.reads()\n                                T.writes(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i])\n                                T.block_attr({\"meta_schedule.thread_extent_high_inclusive\": 256, \"meta_schedule.thread_extent_low_inclusive\": 32, \"meta_schedule.tiling_structure\": \"SSSRRSRS\"})\n                                var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] = T.float32(0)\n                        for k_0 in range(T.int64(344)):\n                            for ax0_ax1_ax2_fused_0 in range(T.int64(4)):\n                                for ax0_ax1_ax2_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                                    for ax0_ax1_ax2_fused_2 in T.vectorized(T.int64(4)):\n                                        with T.block(\"lv43_pad_shared\"):\n                                            v0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                            v1 = T.axis.spatial(T.int64(32), (ax0_ax1_ax2_fused_0 * T.int64(256) + ax0_ax1_ax2_fused_1 * T.int64(4) + ax0_ax1_ax2_fused_2) // T.int64(32))\n                                            v2 = T.axis.spatial(T.int64(11008), k_0 * T.int64(32) + (ax0_ax1_ax2_fused_0 * T.int64(256) + ax0_ax1_ax2_fused_1 * T.int64(4) + ax0_ax1_ax2_fused_2) % T.int64(32))\n                                            T.reads(lv43[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2])\n                                            T.writes(lv43_pad_shared[v0, v1, v2])\n                                            lv43_pad_shared[v0, v1, v2] = T.if_then_else(v_i1_o * T.int64(32) + v1 < n, lv43[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2], T.float32(0))\n                            for ax0_ax1_fused_0 in range(T.int64(8)):\n                                for ax0_ax1_fused_1 in T.thread_binding(T.int64(64), thread=\"threadIdx.x\"):\n                                    for ax0_ax1_fused_2 in T.vectorized(T.int64(2)):\n                                        with T.block(\"linear_weight5_shared\"):\n                                            v0 = T.axis.spatial(T.int64(4096), i0_0_i1_1_0_i2_0_fused * T.int64(32) + (ax0_ax1_fused_0 * T.int64(128) + ax0_ax1_fused_1 * T.int64(2) + ax0_ax1_fused_2) // T.int64(32))\n                                            v1 = T.axis.spatial(T.int64(11008), k_0 * T.int64(32) + (ax0_ax1_fused_0 * T.int64(128) + ax0_ax1_fused_1 * T.int64(2) + ax0_ax1_fused_2) % T.int64(32))\n                                            T.reads(linear_weight5[v0, v1])\n                                            T.writes(linear_weight5_shared[v0, v1])\n                                            linear_weight5_shared[v0, v1] = linear_weight5[v0, v1]\n                            for k_1, i0_3, i1_1_3, i2_3, k_2, i0_4, i1_1_4, i2_4 in T.grid(T.int64(8), T.int64(1), T.int64(2), T.int64(2), T.int64(4), T.int64(1), T.int64(1), T.int64(1)):\n                                with T.block(\"NT_matmul_update\"):\n                                    v_i1_i = T.axis.spatial(T.int64(32), i0_1_i1_1_1_i2_1_fused // T.int64(2) * T.int64(16) + i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(2) + i1_1_3 + i1_1_4)\n                                    v_i2_i = T.axis.spatial(T.int64(4096), i2_4 + i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_1_i1_1_1_i2_1_fused % T.int64(2) * T.int64(16) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(2) + i2_3)\n                                    v_k_i = T.axis.reduce(T.int64(11008), k_0 * T.int64(32) + k_1 * T.int64(4) + k_2)\n                                    T.reads(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i], lv43_pad_shared[T.int64(0), v_i1_i, v_k_i], linear_weight5_shared[v_i2_i, v_k_i])\n                                    T.writes(var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i])\n                                    T.block_attr({\"meta_schedule.thread_extent_high_inclusive\": 256, \"meta_schedule.thread_extent_low_inclusive\": 32, \"meta_schedule.tiling_structure\": \"SSSRRSRS\"})\n                                    var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] = var_NT_matmul_intermediate_pad_local[T.int64(0), v_i1_i, v_i2_i] + lv43_pad_shared[T.int64(0), v_i1_i, v_k_i] * linear_weight5_shared[v_i2_i, v_k_i]\n                        for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(2), T.int64(2)):\n                            with T.block(\"var_NT_matmul_intermediate_pad_local\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(32), i0_1_i1_1_1_i2_1_fused // T.int64(2) * T.int64(16) + i0_2_i1_1_2_i2_2_fused // T.int64(8) * T.int64(2) + ax1)\n                                v2 = T.axis.spatial(T.int64(4096), i0_0_i1_1_0_i2_0_fused * T.int64(32) + i0_1_i1_1_1_i2_1_fused % T.int64(2) * T.int64(16) + i0_2_i1_1_2_i2_2_fused % T.int64(8) * T.int64(2) + ax2)\n                                T.reads(lv36[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2], var_NT_matmul_intermediate_pad_local[v0, v1, v2])\n                                T.writes(var_T_add_intermediate[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2])\n                                # if T.int64(0) <= v_i0 and v_i0 < T.int64(1) and T.int64(0) <= v_i1_o * T.int64(32) + v1 and v_i1_o * T.int64(32) + v1 < n:\n                                if v_i1_o * T.int64(32) + v1 < n:\n                                    var_T_add_intermediate[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2] = lv36[v_i0 + v0, v_i1_o * T.int64(32) + v1, v2] + var_NT_matmul_intermediate_pad_local[v0, v1, v2]\n\n\n\n@T.prim_func\ndef fused_NT_matmul_divide_maximum_minimum_cast_before(lv1605: T.Buffer((T.int64(1), T.int64(32), T.int64(1), T.int64(128)), \"float16\"), p_lv1606: T.handle, p_lv1582: T.handle, p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    lv1606 = T.match_buffer(p_lv1606, (T.int64(1), T.int64(32), n, T.int64(128)), \"float16\")\n    lv1582 = T.match_buffer(p_lv1582, (T.int64(1), T.int64(1), T.int64(1), n), \"float16\")\n    var_compute_intermediate = T.match_buffer(p_output0, (T.int64(1), T.int64(32), T.int64(1), n))\n    # with T.block(\"root\"):\n    var_NT_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    var_T_divide_intermediate = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    var_T_maximum_intermediate = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    var_T_minimum_intermediate = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    for i0, i1, i2, i3, k in T.grid(T.int64(1), T.int64(32), T.int64(1), n, T.int64(128)):\n        with T.block(\"NT_matmul\"):\n            v_i0, v_i1, v_i2, v_i3, v_k = T.axis.remap(\"SSSSR\", [i0, i1, i2, i3, k])\n            T.reads(lv1605[v_i0, v_i1, v_i2, v_k], lv1606[v_i0, v_i1, v_i3, v_k])\n            T.writes(var_NT_matmul_intermediate[v_i0, v_i1, v_i2, v_i3])\n            with T.init():\n                var_NT_matmul_intermediate[v_i0, v_i1, v_i2, v_i3] = T.float16(0)\n            var_NT_matmul_intermediate[v_i0, v_i1, v_i2, v_i3] = var_NT_matmul_intermediate[v_i0, v_i1, v_i2, v_i3] + lv1605[v_i0, v_i1, v_i2, v_k] * lv1606[v_i0, v_i1, v_i3, v_k]\n    for ax0, ax1, ax2, ax3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_divide\"):\n            v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap(\"SSSS\", [ax0, ax1, ax2, ax3])\n            T.reads(var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2, v_ax3])\n            T.writes(var_T_divide_intermediate[v_ax0, v_ax1, v_ax2, v_ax3])\n            var_T_divide_intermediate[v_ax0, v_ax1, v_ax2, v_ax3] = var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2, v_ax3] * T.float16(0.088397790055248615)\n    for ax0, ax1, ax2, ax3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_maximum\"):\n            v_ax0, v_ax1, v_ax2, v_ax3 = T.axis.remap(\"SSSS\", [ax0, ax1, ax2, ax3])\n            T.reads(var_T_divide_intermediate[v_ax0, v_ax1, v_ax2, v_ax3])\n            T.writes(var_T_maximum_intermediate[v_ax0, v_ax1, v_ax2, v_ax3])\n...\n", "needle_token_start": 10214, "needle_token_end": 11053, "code_context_ntokens": 16341, "output": ["```python\n@T.prim_func\ndef fused_NT_matmul3_add1_before(p_lv49: T.handle, linear_weight5: T.Buffer((T.int64(4096), T.int64(11008)), \"float32\"), p_lv42: T.handle, p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    lv49 = T.match_buffer(p_lv49, (T.int64(1), n, T.int64(11008)))\n    lv42 = T.match_buffer(p_lv42, (T.int64(1), n, T.int64(4096)))\n    var_T_add_intermediate = T.match_buffer(p_output0, (T.int64(1), n, T.int64(4096)))\n    # with T.block(\"root\"):\n    var_NT_matmul_intermediate = T.alloc_buffer((T.int64(1), n, T.int64(4096)))\n    for i0, i1, i2, k in T.grid(T.int64(1), n, T.int64(4096), T.int64(11008)):\n        with T.block(\"NT_matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv49[v_i0, v_i1, v_k], linear_weight5[v_i2, v_k])\n            T.writes(var_NT_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = T.float32(0)\n            var_NT_matmul_intermediate[v_i0, v_i1, v_i2] = var_NT_matmul_intermediate[v_i0, v_i1, v_i2] + lv49[v_i0, v_i1, v_k] * linear_weight5[v_i2, v_k]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), n, T.int64(4096)):\n        with T.block(\"T_add\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv42[v_ax0, v_ax1, v_ax2], var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(var_T_add_intermediate[v_ax0, v_ax1, v_ax2])\n            var_T_add_intermediate[v_ax0, v_ax1, v_ax2] = lv42[v_ax0, v_ax1, v_ax2] + var_NT_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "fused_decode4_matmul5_int3_int16_fp16_before", "language": "python", "path": "mlc_llm/dispatch/llama/main.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function performs a decoding operation followed by a matrix multiplication. The decoding involves extracting and transforming elements from an input buffer, which are then used in a matrix multiplication with another input buffer.\n2. **Input**: \n   - A buffer of 16-bit unsigned integers with dimensions (824, 4096), representing encoded data.\n   - A buffer of 16-bit floating-point numbers with dimensions (103, 4096), representing scaling factors for the decoding.\n   - A buffer of 16-bit floating-point numbers with dimensions (1, 1, 4096), used as one of the multiplicands in the matrix multiplication.\n3. **Output**: \n   - A buffer of 16-bit floating-point numbers with dimensions (1, 1, 4096), storing the result of the matrix multiplication.\n4. **Procedure**: \n   - The function first allocates a buffer for intermediate decoded values.\n   - It iterates over the dimensions of the encoded data buffer, decoding each element using bitwise operations and scaling it by corresponding values from the scaling factors buffer.\n   - The decoded data is then used in a matrix multiplication operation with the provided multiplicand buffer.\n   - The result of the matrix multiplication is stored in the output buffer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                                var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1148[v_j // T.int64(10), v_i], T.Cast(\"uint32\", v_j % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(11008), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv2749_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv2749_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2] * lv44[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(11008), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(p_output0_intermediate[v0, v1, v2])\n                        p_output0_intermediate[v0, v1, v2] = var_matmul_intermediate_local[v0, v1, v2] * T.sigmoid(var_matmul_intermediate_local[v0, v1, v2])\n\n\n@T.prim_func\ndef fused_decode6_fused_matmul9_add3_int3_fp16_before(lv1623: T.Buffer((T.int64(1104), T.int64(4096)), \"uint32\"), lv1624: T.Buffer((T.int64(276), T.int64(4096)), \"float16\"), lv167: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"), lv165: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(11008), T.int64(4096)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(11008), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1623[v_i // T.int64(10), v_j], lv1624[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1623[v_i // T.int64(10), v_j], T.Cast(\"uint32\", v_i % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1624[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(11008)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv167[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv167[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(4096)):\n        with T.block(\"T_add\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv165[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = lv165[v_ax0, v_ax1, v_ax2] + var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_decode6_fused_matmul9_add3_int3_fp16_after(lv1158: T.Buffer((T.int64(1104), T.int64(4096)), \"uint32\"), lv60: T.Buffer((T.int64(276), T.int64(4096)), \"float16\"), lv6: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"), lv4: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(11040), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    lv6_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11040)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(2)):\n                    for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(22)):\n                        for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                            with T.block(\"lv6_shared\"):\n                                v0 = T.axis.spatial(T.int64(1), ax0)\n                                v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v2 = T.axis.spatial(T.int64(11040), k_0_0 * T.int64(5520) + (ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1))\n                                T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(5520))\n                                T.reads(lv6[v0, v1, v2])\n                                T.writes(lv6_shared[v0, v1, v2])\n                                T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                                lv6_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(11008), lv6[v0, v1, v2], T.float16(0))\n                    for k_0_1 in range(T.int64(69)):\n                        for ax0_0 in T.unroll(T.int64(80)):\n                            for ax1 in range(T.int64(1)):\n                                with T.block(\"decode\"):\n                                    v_j = T.axis.spatial(T.int64(11040), k_0_0 * T.int64(5520) + k_0_1 * T.int64(80) + ax0_0)\n                                    v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                    T.reads(lv1158[v_j // T.int64(10), v_i], lv60[v_j // T.int64(40), v_i])\n                                    T.writes(var_decode_intermediate_local[v_j, v_i])\n                                    var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.bitwise_and(T.shift_right(lv1158[v_j // T.int64(10), v_i], T.Cast(\"uint32\", v_j % T.int64(10)) * T.uint32(3)), T.uint32(7))) - T.float16(3)\n                        for k_0_2_k_1_fused in range(T.int64(80)):\n                            with T.block(\"matmul_update\"):\n                                v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                                v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                                v_k = T.axis.reduce(T.int64(11040), k_0_0 * T.int64(5520) + k_0_1 * T.int64(80) + k_0_2_k_1_fused)\n                                T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv6_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2])\n                                T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                                var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv6_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2] * lv60[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(lv4[v0, v1, v2], var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(p_output0_intermediate[v0, v1, v2])\n                        p_output0_intermediate[v0, v1, v2] = lv4[v0, v1, v2] + var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode3_matmul1_cast_int3_int16_fp16_before(lv2931: T.Buffer((T.int64(824), T.int64(32000)), \"uint16\"), lv2932: T.Buffer((T.int64(103), T.int64(32000)), \"float16\"), lv3025: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(32000)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(32000)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv2931[v_i // T.int64(5), v_j], lv2932[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(T.Cast(\"uint32\", lv2931[v_i // T.int64(5), v_j]), T.Cast(\"uint32\", v_i % T.int64(5)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv2932[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(32000), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv3025[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv3025[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for i0, i1, i2 in T.grid(T.int64(1), T.int64(1), T.int64(32000)):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2 = T.axis.remap(\"SSS\", [i0, i1, i2])\n            T.reads(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            T.writes(p_output0_intermediate[v_i0, v_i1, v_i2])\n            p_output0_intermediate[v_i0, v_i1, v_i2] = T.Cast(\"float32\", var_matmul_intermediate[v_i0, v_i1, v_i2])\n\n\n@T.prim_func\ndef fused_decode3_matmul1_cast_int3_int16_fp16_after(lv1123: T.Buffer((T.int64(824), T.int64(32000)), \"uint16\"), lv5866: T.Buffer((T.int64(103), T.int64(32000)), \"float16\"), lv1511: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(32000)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    # with T.block(\"root\"):\n    var_decode_intermediate_pad_local = T.alloc_buffer((T.int64(4120), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    var_scale_intermediate_local = T.alloc_buffer((T.int64(103), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_pad_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(32000)), scope=\"local\", dtype=\"float16\")\n    lv1511_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4120)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(125), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(17)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        with T.block(\"lv1511_shared\"):\n                            v0 = T.axis.spatial(T.int64(1), ax0)\n                            v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v2 = T.axis.spatial(T.int64(4120), ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1)\n                            T.reads(lv1511[v0, v1, v2])\n                            T.writes(lv1511_shared[v0, v1, v2])\n                            T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(4120))\n                            T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                            lv1511_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(4096), lv1511[v0, v1, v2], T.float16(0))\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(103)):\n                    for ax0_0 in T.unroll(T.int64(40)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"var_decode_intermediate_pad\"):\n                                v0 = T.axis.spatial(T.int64(4120), k_0_0 * T.int64(40) + ax0_0)\n                                v1 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv1123[v0 // T.int64(5), v1])\n                                T.writes(var_decode_intermediate_pad_local[v0, v1])\n                                var_decode_intermediate_pad_local[v0, v1] = T.Cast(\"float16\", T.Cast(\"int16\", T.bitwise_and(T.shift_right(T.Cast(\"uint16\", lv1123[v0 // T.int64(5), v1]), T.Cast(\"uint16\", v0 % T.int64(5)) * T.uint16(3)), T.uint16(7))) - T.int16(3))\n                    for ax0_0 in range(T.int64(1)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"scale\"):\n                                v_j = T.axis.spatial(T.int64(103), k_0_0 + ax0_0)\n                                v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv5866[v_j, v_i])\n                                T.writes(var_scale_intermediate_local[v_j, v_i])\n                                var_scale_intermediate_local[v_j, v_i] = lv5866[v_j, v_i]\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2], lv1511_shared[v_i0, v_i1, v_k], var_decode_intermediate_pad_local[v_k, v_i2], var_scale_intermediate_local[v_k // T.int64(40), v_i2])\n                            T.writes(var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_pad_local[v_i0, v_i1, v_i2] + lv1511_shared[v_i0, v_i1, v_k] * var_decode_intermediate_pad_local[v_k, v_i2] * var_scale_intermediate_local[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_pad_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(32000), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_pad_local[v0, v1, v2])\n                        T.writes(var_matmul_intermediate[v0, v1, v2])\n                        var_matmul_intermediate[v0, v1, v2] = T.Cast(\"float32\", var_matmul_intermediate_pad_local[v0, v1, v2])\n\n\n@T.prim_func\ndef fused_decode4_fused_matmul5_add3_int3_int16_fp16_before(lv1605: T.Buffer((T.int64(824), T.int64(4096)), \"uint16\"), lv1606: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv164: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv1518: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(4096)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1605[v_i // T.int64(5), v_j], lv1606[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(T.Cast(\"uint32\", lv1605[v_i // T.int64(5), v_j]), T.Cast(\"uint32\", v_i % T.int64(5)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1606[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv164[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv164[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(4096)):\n        with T.block(\"T_add\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv1518[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n            p_output0_intermediate[v_ax0, v_ax1, v_ax2] = lv1518[v_ax0, v_ax1, v_ax2] + var_matmul_intermediate[v_ax0, v_ax1, v_ax2]\n\n\n@T.prim_func\ndef fused_decode4_fused_matmul5_add3_int3_int16_fp16_after(lv1143: T.Buffer((T.int64(824), T.int64(4096)), \"uint16\"), lv36: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv3: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv2710: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4120), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_scale_intermediate_local = T.alloc_buffer((T.int64(103), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    lv3_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4120)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(17)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        with T.block(\"lv3_shared\"):\n                            v0 = T.axis.spatial(T.int64(1), ax0)\n                            v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v2 = T.axis.spatial(T.int64(4120), ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1)\n                            T.reads(lv3[v0, v1, v2])\n                            T.writes(lv3_shared[v0, v1, v2])\n                            T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(4120))\n                            T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                            lv3_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(4096), lv3[v0, v1, v2], T.float16(0))\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(103)):\n                    for ax0_0 in T.unroll(T.int64(40)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"decode\"):\n                                v_j = T.axis.spatial(T.int64(4120), k_0_0 * T.int64(40) + ax0_0)\n                                v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv1143[v_j // T.int64(5), v_i])\n                                T.writes(var_decode_intermediate_local[v_j, v_i])\n                                var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.Cast(\"int16\", T.bitwise_and(T.shift_right(T.Cast(\"uint16\", lv1143[v_j // T.int64(5), v_i]), T.Cast(\"uint16\", v_j % T.int64(5)) * T.uint16(3)), T.uint16(7))) - T.int16(3))\n                    for ax0_0 in range(T.int64(1)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"scale\"):\n                                v_j = T.axis.spatial(T.int64(103), k_0_0 + ax0_0)\n                                v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv36[v_j, v_i])\n                                T.writes(var_scale_intermediate_local[v_j, v_i])\n                                var_scale_intermediate_local[v_j, v_i] = lv36[v_j, v_i]\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv3_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2], var_scale_intermediate_local[v_k // T.int64(40), v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv3_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2] * var_scale_intermediate_local[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(lv2710[v0, v1, v2], var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(p_output0_intermediate[v0, v1, v2])\n                        p_output0_intermediate[v0, v1, v2] = lv2710[v0, v1, v2] + var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\n\ndef fused_decode4_matmul5_int3_int16_fp16_before(lv1587: T.Buffer((T.int64(824), T.int64(4096)), \"uint16\"), lv1588: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv1520: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1587[v_i // T.int64(5), v_j], lv1588[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(T.Cast(\"uint32\", lv1587[v_i // T.int64(5), v_j]), T.Cast(\"uint32\", v_i % T.int64(5)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1588[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1520[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv1520[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n\n\n@T.prim_func\ndef fused_decode4_matmul5_int3_int16_fp16_after(lv1128: T.Buffer((T.int64(824), T.int64(4096)), \"uint16\"), lv12: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv2712: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.is_scheduled\": 1, \"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate_local = T.alloc_buffer((T.int64(4120), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_scale_intermediate_local = T.alloc_buffer((T.int64(103), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    var_matmul_intermediate_local = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4096)), scope=\"local\", dtype=\"float16\")\n    lv2712_shared = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(4120)), scope=\"shared\", dtype=\"float16\")\n    for i0_i1_i2_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 16, \"pragma_unroll_explicit\": 1}):\n        for i2_1 in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i2_2 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                for ax0, ax1_ax2_fused_0 in T.grid(T.int64(1), T.int64(17)):\n                    for ax1_ax2_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n                        with T.block(\"lv2712_shared\"):\n                            v0 = T.axis.spatial(T.int64(1), ax0)\n                            v1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v2 = T.axis.spatial(T.int64(4120), ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1)\n                            T.reads(lv2712[v0, v1, v2])\n                            T.writes(lv2712_shared[v0, v1, v2])\n                            T.where(ax1_ax2_fused_0 * T.int64(256) + ax1_ax2_fused_1 < T.int64(4120))\n                            T.block_attr({\"buffer_dim_align\": [[0, 1, 32, 8]]})\n                            lv2712_shared[v0, v1, v2] = T.if_then_else(v2 < T.int64(4096), lv2712[v0, v1, v2], T.float16(0))\n                with T.block(\"matmul_init\"):\n                    v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                    v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                    T.reads()\n                    T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                    var_matmul_intermediate_local[v_i0, v_i1, v_i2] = T.float32(0)\n                for k_0_0 in range(T.int64(103)):\n                    for ax0_0 in T.unroll(T.int64(40)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"decode\"):\n                                v_j = T.axis.spatial(T.int64(4120), k_0_0 * T.int64(40) + ax0_0)\n                                v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv1128[v_j // T.int64(5), v_i])\n                                T.writes(var_decode_intermediate_local[v_j, v_i])\n                                var_decode_intermediate_local[v_j, v_i] = T.Cast(\"float16\", T.Cast(\"int16\", T.bitwise_and(T.shift_right(T.Cast(\"uint16\", lv1128[v_j // T.int64(5), v_i]), T.Cast(\"uint16\", v_j % T.int64(5)) * T.uint16(3)), T.uint16(7))) - T.int16(3))\n                    for ax0_0 in range(T.int64(1)):\n                        for ax1 in range(T.int64(1)):\n                            with T.block(\"scale\"):\n                                v_j = T.axis.spatial(T.int64(103), k_0_0 + ax0_0)\n                                v_i = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax1)\n                                T.reads(lv12[v_j, v_i])\n                                T.writes(var_scale_intermediate_local[v_j, v_i])\n                                var_scale_intermediate_local[v_j, v_i] = lv12[v_j, v_i]\n                    for k_0_1_k_1_fused in range(T.int64(40)):\n                        with T.block(\"matmul_update\"):\n                            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i1 = T.axis.spatial(T.int64(1), T.int64(0))\n                            v_i2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_1 * T.int64(256) + i2_2)\n                            v_k = T.axis.reduce(T.int64(4120), k_0_0 * T.int64(40) + k_0_1_k_1_fused)\n                            T.reads(var_matmul_intermediate_local[v_i0, v_i1, v_i2], lv2712_shared[v_i0, v_i1, v_k], var_decode_intermediate_local[v_k, v_i2], var_scale_intermediate_local[v_k // T.int64(40), v_i2])\n                            T.writes(var_matmul_intermediate_local[v_i0, v_i1, v_i2])\n                            var_matmul_intermediate_local[v_i0, v_i1, v_i2] = var_matmul_intermediate_local[v_i0, v_i1, v_i2] + lv2712_shared[v_i0, v_i1, v_k] * var_decode_intermediate_local[v_k, v_i2] * var_scale_intermediate_local[v_k // T.int64(40), v_i2]\n                for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(1)):\n                    with T.block(\"var_matmul_intermediate_local\"):\n                        v0, v1 = T.axis.remap(\"SS\", [ax0, ax1])\n                        v2 = T.axis.spatial(T.int64(4096), i0_i1_i2_0_fused * T.int64(256) + i2_2 + ax2)\n                        T.reads(var_matmul_intermediate_local[v0, v1, v2])\n                        T.writes(var_matmul_intermediate[v0, v1, v2])\n                        var_matmul_intermediate[v0, v1, v2] = var_matmul_intermediate_local[v0, v1, v2]\n\n\n@T.prim_func\ndef fused_decode5_fused_matmul8_multiply1_int3_int16_fp16_before(lv1617: T.Buffer((T.int64(824), T.int64(11008)), \"uint16\"), lv1618: T.Buffer((T.int64(103), T.int64(11008)), \"float16\"), lv1557: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), lv3: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\"), p_output0_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(11008)), \"float16\")\n    var_matmul_intermediate = T.alloc_buffer((T.int64(1), T.int64(1), T.int64(11008)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(11008)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1617[v_i // T.int64(5), v_j], lv1618[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(T.Cast(\"uint32\", lv1617[v_i // T.int64(5), v_j]), T.Cast(\"uint32\", v_i % T.int64(5)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1618[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(11008), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1557[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv1557[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n    for ax0, ax1, ax2 in T.grid(T.int64(1), T.int64(1), T.int64(11008)):\n        with T.block(\"T_multiply\"):\n            v_ax0, v_ax1, v_ax2 = T.axis.remap(\"SSS\", [ax0, ax1, ax2])\n            T.reads(lv3[v_ax0, v_ax1, v_ax2], var_matmul_intermediate[v_ax0, v_ax1, v_ax2])\n            T.writes(p_output0_intermediate[v_ax0, v_ax1, v_ax2])\n...\n", "needle_token_start": 11821, "needle_token_end": 12676, "code_context_ntokens": 16322, "output": ["```python\n@T.prim_func\ndef fused_decode4_matmul5_int3_int16_fp16_before(lv1587: T.Buffer((T.int64(824), T.int64(4096)), \"uint16\"), lv1588: T.Buffer((T.int64(103), T.int64(4096)), \"float16\"), lv1520: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\"), var_matmul_intermediate: T.Buffer((T.int64(1), T.int64(1), T.int64(4096)), \"float16\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    # with T.block(\"root\"):\n    var_decode_intermediate = T.alloc_buffer((T.int64(4096), T.int64(4096)), \"float16\")\n    for i, j in T.grid(T.int64(4096), T.int64(4096)):\n        with T.block(\"decode\"):\n            v_i, v_j = T.axis.remap(\"SS\", [i, j])\n            T.reads(lv1587[v_i // T.int64(5), v_j], lv1588[v_i // T.int64(40), v_j])\n            T.writes(var_decode_intermediate[v_i, v_j])\n            var_decode_intermediate[v_i, v_j] = (T.Cast(\"float16\", T.bitwise_and(T.shift_right(T.Cast(\"uint32\", lv1587[v_i // T.int64(5), v_j]), T.Cast(\"uint32\", v_i % T.int64(5)) * T.uint32(3)), T.uint32(7))) - T.float16(3)) * lv1588[v_i // T.int64(40), v_j]\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(1), T.int64(4096), T.int64(4096)):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1520[v_i0, v_i1, v_k], var_decode_intermediate[v_k, v_i2])\n            T.writes(var_matmul_intermediate[v_i0, v_i1, v_i2])\n            with T.init():\n                var_matmul_intermediate[v_i0, v_i1, v_i2] = T.float16(0)\n            var_matmul_intermediate[v_i0, v_i1, v_i2] = var_matmul_intermediate[v_i0, v_i1, v_i2] + lv1520[v_i0, v_i1, v_k] * var_decode_intermediate[v_k, v_i2]\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "softmax_1xn_sch_func", "language": "python", "path": "mlc_llm/dispatch/llama/main.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to optimize the execution of a softmax operation on a 1xN input tensor, potentially casting the results to a lower precision format based on a flag.\n2. **Input**: It accepts a primary function representing the softmax computation and a boolean flag indicating whether the output should be cast to half-precision floating-point format.\n3. **Output**: The output is a modified computational graph that includes optimizations such as inlining computations, splitting and binding loops to hardware threads, and setting memory scopes for efficient execution on parallel hardware.\n4. **Procedure**: The procedure involves:\n   - Optionally inlining a casting computation based on the input flag.\n   - Inlining the exponential computation directly into the normalization step to reduce memory overhead.\n   - Splitting the normalization loop to facilitate parallel execution on GPU threads.\n   - Computing the exponential sum and maximum element computations at optimal points in the loop hierarchy to enhance data locality and reduce latency.\n   - Binding loops to GPU block and thread identifiers to ensure efficient distribution of work across the GPU's parallel execution units.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            T_softmax_maxelem[v_i0, v_i1, v_i2] = T.max(T_softmax_maxelem[v_i0, v_i1, v_i2], rxplaceholder[v_i0, v_i1, v_i2, v_k])\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), n, m):\n        with T.block(\"T_softmax_exp\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(rxplaceholder[v_i0, v_i1, v_i2, v_i3], T_softmax_maxelem[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_exp[v_i0, v_i1, v_i2, v_i3])\n            T_softmax_exp[v_i0, v_i1, v_i2, v_i3] = T.exp(rxplaceholder[v_i0, v_i1, v_i2, v_i3] - T_softmax_maxelem[v_i0, v_i1, v_i2])\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), n, m):\n        with T.block(\"T_softmax_expsum\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_expsum[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_expsum[v_i0, v_i1, v_i2] = T.float16(0)\n            T_softmax_expsum[v_i0, v_i1, v_i2] = T_softmax_expsum[v_i0, v_i1, v_i2] + T_softmax_exp[v_i0, v_i1, v_i2, v_k]\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), n, m):\n        with T.block(\"T_softmax_norm\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_i3], T_softmax_expsum[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_norm[v_i0, v_i1, v_i2, v_i3])\n            T.block_attr({\"axis\": 3})\n            T_softmax_norm[v_i0, v_i1, v_i2, v_i3] = T_softmax_exp[v_i0, v_i1, v_i2, v_i3] / T_softmax_expsum[v_i0, v_i1, v_i2]\n\n@T.prim_func\ndef softmax_mxn_fp16_after(var_A: T.handle, var_T_softmax_norm: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    n = T.int64()\n    m = T.int64()\n    A = T.match_buffer(var_A, (T.int64(1), T.int64(32), n, m), dtype=\"float16\")\n    T_softmax_norm = T.match_buffer(var_T_softmax_norm, (T.int64(1), T.int64(32), n, m), dtype=\"float16\")\n    # with T.block(\"root\"):\n    for i2_0 in T.thread_binding((n + T.int64(31)) // T.int64(32), thread=\"blockIdx.x\"):\n        with T.block(\"T_softmax_maxelem_o\"):\n            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n            v_i2_o = T.axis.spatial((n + T.int64(31)) // T.int64(32), i2_0)\n            T.reads(A[v_i0, T.int64(0):T.int64(32), v_i2_o * T.int64(32):v_i2_o * T.int64(32) + T.int64(32), T.int64(0):(m + T.int64(127)) // T.int64(128) * T.int64(128)])\n            T.writes(T_softmax_norm[v_i0, T.int64(0):T.int64(32), v_i2_o * T.int64(32):v_i2_o * T.int64(32) + T.int64(32), T.int64(0):m])\n            T_softmax_maxelem_pad_0_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(32)), scope=\"shared\", dtype=\"float16\")\n            T_softmax_expsum_pad_0_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(32)), scope=\"shared\", dtype=\"float16\")\n            for i0, i1, i2_1, k_0 in T.grid(T.int64(1), T.int64(32), T.int64(32), (m + T.int64(127)) // T.int64(128)):\n                for k_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                    with T.block(\"T_softmax_maxelem\"):\n                        v_i1_i, v_i2_i = T.axis.remap(\"SS\", [i1, i2_1])\n                        v_k_i = T.axis.reduce(T.int64(32) * ((m + T.int64(127)) // T.int64(128)), k_0 * T.int64(128) + k_1)\n                        T.reads(A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i])\n                        T.writes(T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i])\n                        with T.init():\n                            T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i] = T.float16(-65504)\n                        T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i] = T.max(T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i], T.if_then_else(v_i2_o * T.int64(32) + v_i2_i < n and v_k_i < m, A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i], T.float16(-65504)))\n            for i0, i1, i2_1, k_0 in T.grid(T.int64(1), T.int64(32), T.int64(32), (m + T.int64(127)) // T.int64(128)):\n                for k_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                    with T.block(\"T_softmax_expsum\"):\n                        v_i1_i, v_i2_i = T.axis.remap(\"SS\", [i1, i2_1])\n                        v_k_i = T.axis.reduce(T.int64(32) * ((m + T.int64(127)) // T.int64(128)), k_0 * T.int64(128) + k_1)\n                        T.reads(A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i], T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i])\n                        T.writes(T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i])\n                        with T.init():\n                            T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i] = T.float16(0)\n                        T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i] = T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i] + T.if_then_else(v_i2_o * T.int64(32) + v_i2_i < n and v_k_i < m, T.exp(A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i] - T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i]), T.float16(0))\n            for i0_i1_i2_1_i3_fused_0 in range((T.int64(32) * T.int64(32) * m) // T.int64(128)):\n                for i0_i1_i2_1_i3_fused_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                    with T.block(\"T_softmax_norm\"):\n                        v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                        v_i1 = T.axis.spatial(T.int64(32), (i0_i1_i2_1_i3_fused_0 * T.int64(128) + i0_i1_i2_1_i3_fused_1) // T.int64(32) // m)\n                        v_i2_i = T.axis.spatial(T.int64(32), (i0_i1_i2_1_i3_fused_0 * T.int64(128) + i0_i1_i2_1_i3_fused_1) // m % T.int64(32))\n                        v_i3 = T.axis.spatial(m, (i0_i1_i2_1_i3_fused_0 * T.int64(128) + i0_i1_i2_1_i3_fused_1) % m)\n                        T.where(i0_i1_i2_1_i3_fused_0 * T.int64(128) + i0_i1_i2_1_i3_fused_1 < T.int64(32) * T.int64(32) * m)\n                        T.reads(T_softmax_expsum_pad_0_local[v_i0, v_i1, v_i2_i], A[v_i0, v_i1, v_i2_o * T.int64(32) + v_i2_i, v_i3], T_softmax_maxelem_pad_0_local[v_i0, v_i1, v_i2_i])\n                        T.writes(T_softmax_norm[v_i0, v_i1, v_i2_o * T.int64(32) + v_i2_i, v_i3])\n                        if v_i2_o * T.int64(32) + v_i2_i < n:\n                            T_softmax_norm[v_i0, v_i1, v_i2_o * T.int64(32) + v_i2_i, v_i3] = T.exp(A[v_i0, v_i1, v_i2_o * T.int64(32) + v_i2_i, v_i3] - T_softmax_maxelem_pad_0_local[v_i0, v_i1, v_i2_i]) / T_softmax_expsum_pad_0_local[v_i0, v_i1, v_i2_i]\n\n\n@T.prim_func\ndef softmax_fp16_before(var_rxplaceholder: T.handle, var_T_softmax_norm: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    rxplaceholder = T.match_buffer(var_rxplaceholder, (T.int64(1), T.int64(32), n, n), \"float16\")\n    T_softmax_norm = T.match_buffer(var_T_softmax_norm, (T.int64(1), T.int64(32), n, n), \"float16\")\n    # with T.block(\"root\"):\n    T_softmax_maxelem = T.alloc_buffer((T.int64(1), T.int64(32), n), \"float16\")\n    T_softmax_exp = T.alloc_buffer((T.int64(1), T.int64(32), n, n), \"float16\")\n    T_softmax_expsum = T.alloc_buffer((T.int64(1), T.int64(32), n), \"float16\")\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), n, n):\n        with T.block(\"T_softmax_maxelem\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(rxplaceholder[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_maxelem[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_maxelem[v_i0, v_i1, v_i2] = T.float16(-65504)\n            T_softmax_maxelem[v_i0, v_i1, v_i2] = T.max(T_softmax_maxelem[v_i0, v_i1, v_i2], rxplaceholder[v_i0, v_i1, v_i2, v_k])\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), n, n):\n        with T.block(\"T_softmax_exp\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(rxplaceholder[v_i0, v_i1, v_i2, v_i3], T_softmax_maxelem[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_exp[v_i0, v_i1, v_i2, v_i3])\n            T_softmax_exp[v_i0, v_i1, v_i2, v_i3] = T.exp(rxplaceholder[v_i0, v_i1, v_i2, v_i3] - T_softmax_maxelem[v_i0, v_i1, v_i2])\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), n, n):\n        with T.block(\"T_softmax_expsum\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_expsum[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_expsum[v_i0, v_i1, v_i2] = T.float16(0)\n            T_softmax_expsum[v_i0, v_i1, v_i2] = T_softmax_expsum[v_i0, v_i1, v_i2] + T_softmax_exp[v_i0, v_i1, v_i2, v_k]\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), n, n):\n        with T.block(\"T_softmax_norm\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_i3], T_softmax_expsum[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_norm[v_i0, v_i1, v_i2, v_i3])\n            T.block_attr({\"axis\": 3})\n            T_softmax_norm[v_i0, v_i1, v_i2, v_i3] = T_softmax_exp[v_i0, v_i1, v_i2, v_i3] / T_softmax_expsum[v_i0, v_i1, v_i2]\n\n\n@T.prim_func\ndef softmax_fp16_after(var_A: T.handle, var_T_softmax_norm: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    n = T.int64()\n    A = T.match_buffer(var_A, (T.int64(1), T.int64(32), n, n), dtype=\"float16\")\n    T_softmax_norm = T.match_buffer(var_T_softmax_norm, (T.int64(1), T.int64(32), n, n), dtype=\"float16\")\n    # with T.block(\"root\"):\n    T_softmax_maxelem = T.alloc_buffer((T.int64(1), T.int64(32), n), dtype=\"float16\")\n    T_softmax_expsum = T.alloc_buffer((T.int64(1), T.int64(32), n), dtype=\"float16\")\n    for i2_0 in T.thread_binding((n + T.int64(31)) // T.int64(32), thread=\"blockIdx.x\"):\n        with T.block(\"T_softmax_maxelem_o\"):\n            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n            v_i2_o = T.axis.spatial((n + T.int64(31)) // T.int64(32), i2_0)\n            T.reads(A[v_i0, T.int64(0):T.int64(32), v_i2_o * T.int64(32):v_i2_o * T.int64(32) + T.int64(32), T.int64(0):(n + T.int64(127)) // T.int64(128) * T.int64(128)])\n            T.writes(T_softmax_maxelem[v_i0, T.int64(0):T.int64(32), v_i2_o * T.int64(32):v_i2_o * T.int64(32) + T.int64(32)])\n            T_softmax_maxelem_pad_0_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(32)), scope=\"shared\", dtype=\"float16\")\n            for i0, i1, i2_1, k_0 in T.grid(T.int64(1), T.int64(32), T.int64(32), (n + T.int64(127)) // T.int64(128)):\n                for k_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                    with T.block(\"T_softmax_maxelem\"):\n                        v_i1_i, v_i2_i = T.axis.remap(\"SS\", [i1, i2_1])\n                        v_k_i = T.axis.reduce(T.int64(32) * ((n + T.int64(127)) // T.int64(128)), k_0 * T.int64(128) + k_1)\n                        T.reads(A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i])\n                        T.writes(T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i])\n                        with T.init():\n                            T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i] = T.float16(-65504)\n                        T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i] = T.max(T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i], T.if_then_else(v_i2_o * T.int64(32) + v_i2_i < n and v_k_i < n, A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i], T.float16(-65504)))\n            for i0_i1_i2_1_fused_0 in range(T.int64(8)):\n                for i0_i1_i2_1_fused_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                    with T.block(\"T_softmax_maxelem_cache_write\"):\n                        v_i1_i = T.axis.spatial(T.int64(32), (i0_i1_i2_1_fused_0 * T.int64(128) + i0_i1_i2_1_fused_1) // T.int64(32))\n                        v_i2_i = T.axis.spatial(T.int64(32), (i0_i1_i2_1_fused_0 * T.int64(128) + i0_i1_i2_1_fused_1) % T.int64(32))\n                        T.where(v_i2_o * T.int64(32) + (i0_i1_i2_1_fused_0 * T.int64(128) + i0_i1_i2_1_fused_1) % T.int64(32) < n)\n                        T.reads(T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i])\n                        T.writes(T_softmax_maxelem[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i])\n                        T_softmax_maxelem[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i] = T_softmax_maxelem_pad_0_local[v_i0, v_i1_i, v_i2_i]\n    for i2_0 in T.thread_binding((n + T.int64(31)) // T.int64(32), thread=\"blockIdx.x\"):\n        with T.block(\"T_softmax_expsum_o\"):\n            v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n            v_i2_o = T.axis.spatial((n + T.int64(31)) // T.int64(32), i2_0)\n            T.reads(A[v_i0, T.int64(0):T.int64(32), v_i2_o * T.int64(32):v_i2_o * T.int64(32) + T.int64(32), T.int64(0):(n + T.int64(127)) // T.int64(128) * T.int64(128)], T_softmax_maxelem[v_i0, T.int64(0):T.int64(32), v_i2_o * T.int64(32):v_i2_o * T.int64(32) + T.int64(32)])\n            T.writes(T_softmax_expsum[v_i0, T.int64(0):T.int64(32), v_i2_o * T.int64(32):v_i2_o * T.int64(32) + T.int64(32)])\n            T_softmax_expsum_pad_0_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(32)), scope=\"shared\", dtype=\"float16\")\n            for i0, i1, i2_1, k_0 in T.grid(T.int64(1), T.int64(32), T.int64(32), (n + T.int64(127)) // T.int64(128)):\n                for k_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                    with T.block(\"T_softmax_expsum\"):\n                        v_i1_i, v_i2_i = T.axis.remap(\"SS\", [i1, i2_1])\n                        v_k_i = T.axis.reduce(T.int64(32) * ((n + T.int64(127)) // T.int64(128)), k_0 * T.int64(128) + k_1)\n                        T.reads(A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i], T_softmax_maxelem[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i])\n                        T.writes(T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i])\n                        with T.init():\n                            T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i] = T.float16(0)\n                        T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i] = T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i] + T.if_then_else(v_i2_o * T.int64(32) + v_i2_i < n and v_k_i < n, T.exp(A[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i, v_k_i] - T_softmax_maxelem[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i]), T.float16(0))\n            for i0_i1_i2_1_fused_0 in range(T.int64(8)):\n                for i0_i1_i2_1_fused_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                    with T.block(\"T_softmax_expsum_cache_write\"):\n                        v_i1_i = T.axis.spatial(T.int64(32), (i0_i1_i2_1_fused_0 * T.int64(128) + i0_i1_i2_1_fused_1) // T.int64(32))\n                        v_i2_i = T.axis.spatial(T.int64(32), (i0_i1_i2_1_fused_0 * T.int64(128) + i0_i1_i2_1_fused_1) % T.int64(32))\n                        T.where(v_i2_o * T.int64(32) + (i0_i1_i2_1_fused_0 * T.int64(128) + i0_i1_i2_1_fused_1) % T.int64(32) < n)\n                        T.reads(T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i])\n                        T.writes(T_softmax_expsum[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i])\n                        T_softmax_expsum[v_i0, v_i1_i, v_i2_o * T.int64(32) + v_i2_i] = T_softmax_expsum_pad_0_local[v_i0, v_i1_i, v_i2_i]\n    for i0_i1_i2_fused_i3_fused_0 in T.thread_binding((n * T.int64(32) * n + T.int64(255)) // T.int64(256), thread=\"blockIdx.x\"):\n        for i0_i1_i2_fused_i3_fused_1 in T.thread_binding(T.int64(256), thread=\"threadIdx.x\"):\n            with T.block(\"T_softmax_norm\"):\n                v_i0 = T.axis.spatial(T.int64(1), T.int64(0))\n                v_i1 = T.axis.spatial(T.int64(32), (i0_i1_i2_fused_i3_fused_0 * T.int64(256) + i0_i1_i2_fused_i3_fused_1) // n // n)\n                v_i2 = T.axis.spatial(n, (i0_i1_i2_fused_i3_fused_0 * T.int64(256) + i0_i1_i2_fused_i3_fused_1) // n % n)\n                v_i3 = T.axis.spatial(n, (i0_i1_i2_fused_i3_fused_0 * T.int64(256) + i0_i1_i2_fused_i3_fused_1) % n)\n                T.where(i0_i1_i2_fused_i3_fused_0 * T.int64(256) + i0_i1_i2_fused_i3_fused_1 < n * T.int64(32) * n)\n                T.reads(T_softmax_expsum[v_i0, v_i1, v_i2], A[v_i0, v_i1, v_i2, v_i3], T_softmax_maxelem[v_i0, v_i1, v_i2])\n                T.writes(T_softmax_norm[v_i0, v_i1, v_i2, v_i3])\n                T_softmax_norm[v_i0, v_i1, v_i2, v_i3] = T.exp(A[v_i0, v_i1, v_i2, v_i3] - T_softmax_maxelem[v_i0, v_i1, v_i2]) / T_softmax_expsum[v_i0, v_i1, v_i2]\n\n\n@T.prim_func\ndef softmax_1xn_before(var_inp0: T.handle, var_T_softmax_norm: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    inp0 = T.match_buffer(var_inp0, (T.int64(1), T.int64(32), T.int64(1), n))\n    T_softmax_norm = T.match_buffer(var_T_softmax_norm, (T.int64(1), T.int64(32), T.int64(1), n))\n    # with T.block(\"root\"):\n    T_softmax_maxelem = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1)))\n    T_softmax_exp = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n))\n    T_softmax_expsum = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1)))\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_maxelem\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(inp0[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_maxelem[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_maxelem[v_i0, v_i1, v_i2] = T.float32(-3.4028234663852886e+38)\n            T_softmax_maxelem[v_i0, v_i1, v_i2] = T.max(T_softmax_maxelem[v_i0, v_i1, v_i2], inp0[v_i0, v_i1, v_i2, v_k])\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_exp\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(inp0[v_i0, v_i1, v_i2, v_i3], T_softmax_maxelem[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_exp[v_i0, v_i1, v_i2, v_i3])\n            T_softmax_exp[v_i0, v_i1, v_i2, v_i3] = T.exp(inp0[v_i0, v_i1, v_i2, v_i3] - T_softmax_maxelem[v_i0, v_i1, v_i2])\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_expsum\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_expsum[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_expsum[v_i0, v_i1, v_i2] = T.float32(0)\n            T_softmax_expsum[v_i0, v_i1, v_i2] = T_softmax_expsum[v_i0, v_i1, v_i2] + T_softmax_exp[v_i0, v_i1, v_i2, v_k]\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_norm\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_i3], T_softmax_expsum[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_norm[v_i0, v_i1, v_i2, v_i3])\n            T.block_attr({\"axis\": 3})\n            T_softmax_norm[v_i0, v_i1, v_i2, v_i3] = T_softmax_exp[v_i0, v_i1, v_i2, v_i3] / T_softmax_expsum[v_i0, v_i1, v_i2]\n\n\n@T.prim_func\ndef softmax_cast_1xn_before(p_lv1614: T.handle, p_output0: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    lv1614 = T.match_buffer(p_lv1614, (T.int64(1), T.int64(32), T.int64(1), n))\n    var_compute_intermediate = T.match_buffer(p_output0, (T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    # with T.block(\"root\"):\n    T_softmax_maxelem = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1)))\n    T_softmax_exp = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n))\n    T_softmax_expsum = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1)))\n    var_T_softmax_norm_intermediate = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n))\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_maxelem\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(lv1614[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_maxelem[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_maxelem[v_i0, v_i1, v_i2] = T.float32(-3.4028234663852886e+38)\n            T_softmax_maxelem[v_i0, v_i1, v_i2] = T.max(T_softmax_maxelem[v_i0, v_i1, v_i2], lv1614[v_i0, v_i1, v_i2, v_k])\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_exp\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(lv1614[v_i0, v_i1, v_i2, v_i3], T_softmax_maxelem[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_exp[v_i0, v_i1, v_i2, v_i3])\n            T_softmax_exp[v_i0, v_i1, v_i2, v_i3] = T.exp(lv1614[v_i0, v_i1, v_i2, v_i3] - T_softmax_maxelem[v_i0, v_i1, v_i2])\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_expsum\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_expsum[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_expsum[v_i0, v_i1, v_i2] = T.float32(0)\n            T_softmax_expsum[v_i0, v_i1, v_i2] = T_softmax_expsum[v_i0, v_i1, v_i2] + T_softmax_exp[v_i0, v_i1, v_i2, v_k]\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_norm\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_i3], T_softmax_expsum[v_i0, v_i1, v_i2])\n            T.writes(var_T_softmax_norm_intermediate[v_i0, v_i1, v_i2, v_i3])\n            T.block_attr({\"axis\": 3})\n            var_T_softmax_norm_intermediate[v_i0, v_i1, v_i2, v_i3] = T_softmax_exp[v_i0, v_i1, v_i2, v_i3] / T_softmax_expsum[v_i0, v_i1, v_i2]\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"compute\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(var_T_softmax_norm_intermediate[v_i0, v_i1, v_i2, v_i3])\n            T.writes(var_compute_intermediate[v_i0, v_i1, v_i2, v_i3])\n            var_compute_intermediate[v_i0, v_i1, v_i2, v_i3] = T.Cast(\"float16\", var_T_softmax_norm_intermediate[v_i0, v_i1, v_i2, v_i3])\n\n\n@T.prim_func\ndef softmax_1xn_fp16_before(var_rxplaceholder: T.handle, var_T_softmax_norm: T.handle):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    rxplaceholder = T.match_buffer(var_rxplaceholder, (T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    T_softmax_norm = T.match_buffer(var_T_softmax_norm, (T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    # with T.block(\"root\"):\n    T_softmax_maxelem = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1)), \"float16\")\n    T_softmax_exp = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), n), \"float16\")\n    T_softmax_expsum = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1)), \"float16\")\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_maxelem\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(rxplaceholder[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_maxelem[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_maxelem[v_i0, v_i1, v_i2] = T.float16(-65504)\n            T_softmax_maxelem[v_i0, v_i1, v_i2] = T.max(T_softmax_maxelem[v_i0, v_i1, v_i2], rxplaceholder[v_i0, v_i1, v_i2, v_k])\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_exp\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(rxplaceholder[v_i0, v_i1, v_i2, v_i3], T_softmax_maxelem[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_exp[v_i0, v_i1, v_i2, v_i3])\n            T_softmax_exp[v_i0, v_i1, v_i2, v_i3] = T.exp(rxplaceholder[v_i0, v_i1, v_i2, v_i3] - T_softmax_maxelem[v_i0, v_i1, v_i2])\n    for i0, i1, i2, k in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_expsum\"):\n            v_i0, v_i1, v_i2, v_k = T.axis.remap(\"SSSR\", [i0, i1, i2, k])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_k])\n            T.writes(T_softmax_expsum[v_i0, v_i1, v_i2])\n            with T.init():\n                T_softmax_expsum[v_i0, v_i1, v_i2] = T.float16(0)\n            T_softmax_expsum[v_i0, v_i1, v_i2] = T_softmax_expsum[v_i0, v_i1, v_i2] + T_softmax_exp[v_i0, v_i1, v_i2, v_k]\n    for i0, i1, i2, i3 in T.grid(T.int64(1), T.int64(32), T.int64(1), n):\n        with T.block(\"T_softmax_norm\"):\n            v_i0, v_i1, v_i2, v_i3 = T.axis.remap(\"SSSS\", [i0, i1, i2, i3])\n            T.reads(T_softmax_exp[v_i0, v_i1, v_i2, v_i3], T_softmax_expsum[v_i0, v_i1, v_i2])\n            T.writes(T_softmax_norm[v_i0, v_i1, v_i2, v_i3])\n            T.block_attr({\"axis\": 3})\n            T_softmax_norm[v_i0, v_i1, v_i2, v_i3] = T_softmax_exp[v_i0, v_i1, v_i2, v_i3] / T_softmax_expsum[v_i0, v_i1, v_i2]\n\n\n\ndef softmax_1xn_sch_func(f_softmax, cast_to_fp16: bool = False):\n    sch = tvm.tir.Schedule(f_softmax)\n    if cast_to_fp16:\n        b_cast = sch.get_block(\"compute\")\n        sch.reverse_compute_inline(b_cast)\n\n    b0 = sch.get_block(\"T_softmax_exp\")\n    sch.compute_inline(b0)\n    b1 = sch.get_block(\"T_softmax_norm\")\n    l2, l3, l4, l5 = sch.get_loops(b1)\n    l6, l7 = sch.split(l5, [None, 128])\n    sch.bind(l7, \"threadIdx.x\")\n    b8 = sch.get_block(\"T_softmax_expsum\")\n    sch.compute_at(b8, l4)\n    sch.set_scope(b8, 0, \"shared\")\n    l9, l10, l11, l12 = sch.get_loops(b8)\n    l13, l14 = sch.split(l12, [None, 128])\n    sch.bind(l14, \"threadIdx.x\")\n    b15 = sch.get_block(\"T_softmax_maxelem\")\n    sch.compute_at(b15, l4)\n    sch.set_scope(b15, 0, \"shared\")\n    l16, l17, l18, l19 = sch.get_loops(b15)\n    l20, l21 = sch.split(l19, [None, 128])\n    sch.bind(l21, \"threadIdx.x\")\n    l22 = sch.fuse(l2, l3, l4)\n    sch.bind(l22, \"blockIdx.x\")\n    return sch.mod[\"main\"].with_attr(\"tir.is_scheduled\", 1)\n\n\n@T.prim_func\ndef matmul1_before(var_rxplaceholder: T.handle, var_rxplaceholder_1: T.handle, matmul: T.Buffer((T.int64(1), T.int64(32), T.int64(1), T.int64(128)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True)})\n    n = T.int64()\n    rxplaceholder = T.match_buffer(var_rxplaceholder, (T.int64(1), T.int64(32), T.int64(1), n))\n    rxplaceholder_1 = T.match_buffer(var_rxplaceholder_1, (T.int64(1), T.int64(32), n, T.int64(128)))\n    # with T.block(\"root\"):\n    for i0, i1, i2, i3, k in T.grid(T.int64(1), T.int64(32), T.int64(1), T.int64(128), n):\n        with T.block(\"matmul\"):\n            v_i0, v_i1, v_i2, v_i3, v_k = T.axis.remap(\"SSSSR\", [i0, i1, i2, i3, k])\n            T.reads(rxplaceholder[v_i0, v_i1, v_i2, v_k], rxplaceholder_1[v_i0, v_i1, v_k, v_i3])\n            T.writes(matmul[v_i0, v_i1, v_i2, v_i3])\n            with T.init():\n                matmul[v_i0, v_i1, v_i2, v_i3] = T.float32(0)\n            matmul[v_i0, v_i1, v_i2, v_i3] = matmul[v_i0, v_i1, v_i2, v_i3] + rxplaceholder[v_i0, v_i1, v_i2, v_k] * rxplaceholder_1[v_i0, v_i1, v_k, v_i3]\n\n\n@T.prim_func\ndef matmul1_after(var_rxplaceholder: T.handle, var_rxplaceholder_1: T.handle, matmul: T.Buffer((T.int64(1), T.int64(32), T.int64(1), T.int64(128)), \"float32\")):\n    T.func_attr({\"tir.noalias\": T.bool(True), \"tir.is_scheduled\": 1})\n    n = T.int64()\n    rxplaceholder = T.match_buffer(var_rxplaceholder, (T.int64(1), T.int64(32), T.int64(1), n))\n    rxplaceholder_1 = T.match_buffer(var_rxplaceholder_1, (T.int64(1), T.int64(32), n, T.int64(128)))\n    # with T.block(\"root\"):\n    matmul_local = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), T.int64(128)), scope=\"local\")\n    rxplaceholder_pad_shared = T.alloc_buffer((T.int64(1), T.int64(32), T.int64(1), (n + T.int64(127)) // T.int64(128) * T.int64(128)), scope=\"shared\")\n    rxplaceholder_1_pad_shared = T.alloc_buffer((T.int64(1), T.int64(32), (n + T.int64(127)) // T.int64(128) * T.int64(128), T.int64(128)), scope=\"shared\")\n    for i0_0_i1_0_i2_0_i3_0_fused in T.thread_binding(T.int64(16), thread=\"blockIdx.x\", annotations={\"pragma_auto_unroll_max_step\": 512, \"pragma_unroll_explicit\": 1}):\n        for i0_1_i1_1_i2_1_i3_1_fused in T.thread_binding(T.int64(1), thread=\"vthread.x\"):\n            for i0_2_i1_2_i2_2_i3_2_fused in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                for i0_3_init, i1_3_init, i2_3_init, i3_3_init, i0_4_init, i1_4_init, i2_4_init, i3_4_init in T.grid(T.int64(1), T.int64(1), T.int64(1), T.int64(1), T.int64(1), T.int64(2), T.int64(1), T.int64(1)):\n                    with T.block(\"matmul_init\"):\n                        v_i0 = T.axis.spatial(T.int64(1), i0_3_init + i0_4_init)\n                        v_i1 = T.axis.spatial(T.int64(32), i0_2_i1_2_i2_2_i3_2_fused // T.int64(8) * T.int64(2) + i1_3_init * T.int64(2) + i1_4_init)\n                        v_i2 = T.axis.spatial(T.int64(1), i2_3_init + i2_4_init)\n                        v_i3 = T.axis.spatial(T.int64(128), i0_0_i1_0_i2_0_i3_0_fused * T.int64(8) + i0_2_i1_2_i2_2_i3_2_fused % T.int64(8) + i3_3_init + i3_4_init)\n                        T.reads()\n                        T.writes(matmul_local[v_i0, v_i1, v_i2, v_i3])\n                        T.block_attr({\"meta_schedule.thread_extent_high_inclusive\": 256, \"meta_schedule.thread_extent_low_inclusive\": 32, \"meta_schedule.tiling_structure\": \"SSSRRSRS\"})\n                        matmul_local[v_i0, v_i1, v_i2, v_i3] = T.float32(0)\n                for k_0, k_1_0 in T.grid((n + T.int64(127)) // T.int64(128), T.int64(8)):\n                    for ax0_ax1_ax2_ax3_fused_0 in range(T.int64(1)):\n                        for ax0_ax1_ax2_ax3_fused_1 in T.thread_binding(T.int64(128), thread=\"threadIdx.x\"):\n                            for ax0_ax1_ax2_ax3_fused_2 in T.vectorized(T.int64(4)):\n                                with T.block(\"rxplaceholder_pad_shared\"):\n                                    v0 = T.axis.spatial(T.int64(1), T.int64(0))\n                                    v1 = T.axis.spatial(T.int64(32), (ax0_ax1_ax2_ax3_fused_0 * T.int64(512) + ax0_ax1_ax2_ax3_fused_1 * T.int64(4) + ax0_ax1_ax2_ax3_fused_2) // T.int64(16))\n                                    v2 = T.axis.spatial(T.int64(1), T.int64(0))\n...\n", "needle_token_start": 13651, "needle_token_end": 14154, "code_context_ntokens": 16251, "output": ["```python\ndef softmax_1xn_sch_func(f_softmax, cast_to_fp16: bool = False):\n    sch = tvm.tir.Schedule(f_softmax)\n    if cast_to_fp16:\n        b_cast = sch.get_block(\"compute\")\n        sch.reverse_compute_inline(b_cast)\n\n    b0 = sch.get_block(\"T_softmax_exp\")\n    sch.compute_inline(b0)\n    b1 = sch.get_block(\"T_softmax_norm\")\n    l2, l3, l4, l5 = sch.get_loops(b1)\n    l6, l7 = sch.split(l5, [None, 128])\n    sch.bind(l7, \"threadIdx.x\")\n    b8 = sch.get_block(\"T_softmax_expsum\")\n    sch.compute_at(b8, l4)\n    sch.set_scope(b8, 0, \"shared\")\n    l9, l10, l11, l12 = sch.get_loops(b8)\n    l13, l14 = sch.split(l12, [None, 128])\n    sch.bind(l14, \"threadIdx.x\")\n    b15 = sch.get_block(\"T_softmax_maxelem\")\n    sch.compute_at(b15, l4)\n    sch.set_scope(b15, 0, \"shared\")\n    l16, l17, l18, l19 = sch.get_loops(b15)\n    l20, l21 = sch.split(l19, [None, 128])\n    sch.bind(l21, \"threadIdx.x\")\n    l22 = sch.fuse(l2, l3, l4)\n    sch.bind(l22, \"blockIdx.x\")\n    return sch.mod[\"main\"].with_attr(\"tir.is_scheduled\", 1)\n```"]}
{"repo": "mlc-ai/mlc-llm", "name": "f_quantize", "language": "python", "path": "mlc_llm/quantization/ft_quantization.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to quantize and preprocess neural network weights for optimized processing on specific GPU architectures, particularly for use in deep learning inference.\n2. **Input**: The function takes a list of expressions representing neural network weights.\n3. **Output**: It returns a tuple containing the preprocessed and quantized weights along with additional encoded data.\n4. **Procedure**: \n   - The function first quantizes the input weights using a specified bit precision and group size.\n   - It then optionally preprocesses the quantized weights using a CUDA kernel tailored for specific NVIDIA GPU architectures, enhancing compatibility and performance for deep learning computations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: mlc_llm/quantization/quantization.py\nimport enum\nfrom dataclasses import dataclass\nfrom typing import Any, Callable, List, Literal, Optional, Tuple, Type, Union\n\nimport tvm\nfrom tvm import relax, te\nfrom tvm.relax.expr_functor import PyExprVisitor, visitor\n\nFQuantize = Callable[[relax.BlockBuilder, List[relax.Expr]], relax.Var]\nFTEQuantize = Callable[[te.Tensor], List[te.Tensor]]\nFTEDequantize = Callable[[List[te.Tensor]], te.Tensor]\n\n\n@dataclass\nclass QuantizationSpec:\n    \"\"\"The base dataclass of quantization specification.\n    A specification describes how a parameter is quantized and dequantized.\n\n    A subclass of QuantizationSpec\n      - contains more data fields (e.g., the \"group size\" in group quantization)\n      which instruct the quantization/dequantization,\n      - defines the `get_quantize_func` method, which returns a function\n      (`Callable[[relax.BlockBuilder, List[relax.Expr]], relax.Var]`) that takes a\n      Relax BlockBuilder and the weight relax Var to be quantized, computes\n      the quantization and returns the relax Var of quantized results.\n      algorithm of the quantization.\n      - defines the `get_dequantize_func` method, which returns function\n      (`Callable[[relax.BlockBuilder, List[relax.Expr]], relax.Var]`) that takes\n      the quantized results, computes and returns the dequantization result.\n      - optionally overloads the `get_loaded_tensor_info` when the parameter is\n      pre-quantized, in which case `get_loaded_tensor_info` needs to be overloaded\n      so that we know how many quantized data tensors there are, and the dtype\n      and shape of each quantized data tensor.\n    \"\"\"\n\n    dtype: str\n\n    def get_loaded_tensor_info(\n        self, pname: str, param_info: relax.TensorStructInfo\n    ) -> Tuple[List[str], List[relax.TensorStructInfo]]:\n        \"\"\"Returns the names and shapes and dtypes of the tensors that need to\n        be loaded from the disk.\n\n        It is useful when the parameter is pre-quantized. In such cases, we need\n        to know how many tensors the parameter is quantized into, and together\n        with the dtype and shape of each tensor, so that we can load the\n        pre-quantized tensors in.\n        \"\"\"\n        return [pname], [param_info]\n\n    def get_quantize_func(self, param_info: relax.TensorStructInfo) -> Optional[FQuantize]:\n        \"\"\"Returns the function which computes quantization.\n        Returning `None` means the parameter does not need quantization or is\n        pre-quantized.\n\n        The returned function takes a Relax BlockBuilder and a (list of) weight\n        relax Var to be quantized, computes the quantization and returns the\n        quantization result Relax Var(s).\n\n        You can use `convert_TE_func` to convert a TE function to the function\n        of the desired return format. See `group_quantization.py` for examples.\n        \"\"\"\n        return NotImplementedError()\n\n    def get_dequantize_func(\n        self,\n        param_info: relax.TensorStructInfo,\n        qparam_info: List[relax.TensorStructInfo],\n    ) -> Optional[FQuantize]:\n        \"\"\"Returns the function which computes dequantization.\n        Returning `None` means the parameter does not need dequantization.\n\n        The returned function takes a Relax BlockBuilder and a (list of)\n        quantized weight relax Var, computes the dequantization and returns the\n        result Relax Var(s).\n\n        You can use `convert_TE_func` to convert a TE function to the function\n        of the desired return format. See `group_quantization.py` for examples.\n        \"\"\"\n        return NotImplementedError()\n\n\n@dataclass\nclass NoQuantizationSpec(QuantizationSpec):\n    \"\"\"The quantization specification that describes doing no quantization.\"\"\"\n\n    def get_quantize_func(self, param_info: relax.TensorStructInfo) -> Optional[FQuantize]:\n        return None\n\n    def get_dequantize_func(\n        self,\n        param_info: relax.TensorStructInfo,\n        qparam_info: List[relax.TensorStructInfo],\n    ) -> Optional[FQuantize]:\n        return None\n\n\nclass ParamQuantKind(enum.IntEnum):\n    \"\"\"The parameter quantization kind class.\n\n    We categorized all the parameters in a model into four kinds:\n    - the weights of the internal linear layers, which are the main targets of quantization,\n    - the embedding table of every token,\n    - the weight of the fully-connected layer at the end of the model, which is\n    used for computes the logits of each input token,\n    - other parameters (e.g., the weight of layer normalization, etc.).\n    \"\"\"\n\n    linear_weight = 0\n    embedding_table = 1\n    final_fc_weight = 2\n    others = 3\n\n\nclass QuantizationScheme:\n    \"\"\"The quantization scheme class describes how an entire model is quantized.\n    It contains the quantization specification for each parameter quantization kind.\n\n    Besides, it has an optional field for a visitor class which has the ability to\n    take the constructed model (in format of IRModule) as input, go through the\n    model and update the QuantizationSpec for certain parameters.\n    \"\"\"\n\n    name: str\n    linear_weight: QuantizationSpec\n    embedding_table: QuantizationSpec\n    final_fc_weight: QuantizationSpec\n    others: QuantizationSpec\n\n    qspec_updater_class: Optional[Type[\"QuantSpecUpdater\"]]\n    f_convert_param_bkwd: Optional[Callable[[str, Any], Optional[List[Tuple[str, Any]]]]]\n    f_compute_relax_param: Optional[Callable[[str, List[Any]], Any]]\n    f_run_prequantize: Optional[Callable[[str], str]]\n\n    def __init__(\n        self,\n        name: str,\n        linear_weight: QuantizationSpec,\n        *,\n        embedding_table: Optional[Union[QuantizationSpec, Literal[\"same_as_linear_weight\"]]] = None,\n        final_fc_weight: Optional[Union[QuantizationSpec, Literal[\"same_as_linear_weight\"]]] = None,\n        others: Optional[QuantizationSpec] = None,\n        qspec_updater_class: Optional[Type[\"QuantSpecUpdater\"]] = None,\n    ) -> None:\n        self.name = name\n        self.linear_weight = linear_weight\n        self.others = others if others is not None else NoQuantizationSpec(self.model_dtype)\n\n        if embedding_table is None:\n            self.embedding_table = self.others\n        elif embedding_table == \"same_as_linear_weight\":\n            self.embedding_table = self.linear_weight\n        else:\n            self.embedding_table = embedding_table\n\n        if final_fc_weight is None:\n            self.final_fc_weight = self.others\n        elif final_fc_weight == \"same_as_linear_weight\":\n            self.final_fc_weight = self.linear_weight\n        else:\n            self.final_fc_weight = final_fc_weight\n\n        self.qspec_updater_class = qspec_updater_class\n        self.f_convert_param_bkwd = None\n        self.f_compute_relax_param = None\n        self.f_run_prequantize = None\n\n        for spec in [self.linear_weight, self.embedding_table, self.final_fc_weight, self.others]:\n            if hasattr(spec, \"convert_param_bkwd\"):\n                self.f_convert_param_bkwd = spec.convert_param_bkwd\n            if hasattr(spec, \"compute_relax_param\"):\n                self.f_compute_relax_param = spec.compute_relax_param\n            if hasattr(spec, \"run_prequantize\"):\n                self.f_run_prequantize = spec.run_prequantize\n\n    @property\n    def model_dtype(self) -> str:\n        \"\"\"Returns the overall model dtype, which is defined as the dtype of\n        the linear layers.\n        \"\"\"\n        return self.linear_weight.dtype\n\n\ndef convert_TE_func(te_func: Union[FTEQuantize, FTEDequantize], func_name: str) -> FQuantize:\n    def func(bb: relax.BlockBuilder, inputs: List[relax.Expr]) -> relax.Var:\n        return bb.call_te(te_func, *inputs, primfunc_name_hint=func_name)\n\n    return func\n\n\n@visitor\nclass QuantSpecUpdater(PyExprVisitor):\n    def __init__(self, param_manager) -> None:\n        super().__init__()\n        self.param_manager = param_manager\n        self.param_map = None\n        self.builder = relax.BlockBuilder()\n\n    def lookup_binding(self, var: relax.Var):\n        return self.builder.lookup_binding(var)\n\n    def visit_module(self, mod: tvm.IRModule):\n        for gv, func in mod.functions.items():\n            if not isinstance(func, relax.Function):\n                continue\n            if func.attrs is None or not \"num_input\" in func.attrs:\n                continue\n\n            self.param_map = dict()\n            num_input = int(func.attrs[\"num_input\"])\n            params_in_func = self.param_manager.params_in_func[gv.name_hint]\n            assert len(func.params) - num_input == len(params_in_func)\n            for i, relax_param in enumerate(func.params[num_input:]):\n                self.param_map[relax_param] = params_in_func[i]\n\n            self.builder.normalize(func)\n            self.visit_expr(func)\n\n# Path: mlc_llm/quantization/tir_utils.py\n\"\"\"TIR computation utilities for quantization.\"\"\"\n\nimport tvm\nfrom tvm import tir\n\n# fmt: off\ndef _tir_f32x2_to_bf16x2_to_u32(v0: tir.PrimExpr, v1: tir.PrimExpr, round_to_even: bool=True):\n    mask = tir.const((1 << 16) - 1, \"uint32\")\n    res = []\n    for data in [v0, v1]:\n        u32_val = tir.reinterpret(\"uint32\", data)\n        if round_to_even:\n            rounding_bias = ((u32_val >> tir.const(16, \"uint32\")) & tir.const(1, \"uint32\")) + tir.const(0x7FFF, \"uint32\")\n            u32_val += rounding_bias\n        res.append((u32_val >> tir.const(16, \"uint32\")) & mask)\n    return res[0] | (res[1] << tir.const(16, \"uint32\"))\n\n\ndef _tir_u32_to_bf16x2_to_f32x2(x: tir.PrimExpr):\n    mask = tir.const((1 << 16) - 1, \"uint32\")\n    x0 = x & mask\n    x1 = (x >> 16) & mask\n    return (tir.reinterpret(\"float32\", x << tir.const(16, \"uint32\")) for x in [x0, x1])\n\n\ndef _tir_u32_to_int_to_float(nbit: int, val: tir.PrimExpr, pos: tir.PrimExpr, dtype: str):\n    assert val.dtype == \"uint32\"\n    mask = tvm.tir.const((1 << nbit) - 1, \"uint32\")\n    return tir.Cast(dtype, (val >> (pos * nbit).astype(\"uint32\")) & mask)\n\n\ndef _tir_packed_uint_to_uint_to_float(storage_nbit: int):\n    storage_dtype = \"uint\" + str(storage_nbit)\n\n    def f_convert(nbit: int, val: tir.PrimExpr, pos: tir.PrimExpr, dtype: str):\n        assert val.dtype == storage_dtype\n        max_int_value = (1 << (nbit - 1)) - 1\n        return ((val >> (pos.astype(\"uint32\") * tir.const(nbit, \"uint32\"))) & tir.const((1 << nbit) - 1, \"uint32\")).astype(dtype) - tir.const(max_int_value, dtype)\n\n    return f_convert\n\n\ndef _tir_packed_int_to_int_to_float(storage_nbit: int):\n    storage_dtype = \"int\" + str(storage_nbit)\n\n    def f_convert(nbit: int, val: tir.PrimExpr, pos: tir.PrimExpr, dtype: str):\n        assert val.dtype == storage_dtype\n        mask = tir.const((1 << nbit) - 1, \"int32\")\n        unextended = (val >> (pos.astype(\"int32\") * tir.const(nbit, \"int32\"))) & mask\n        return tir.Cast(dtype, (unextended << tir.const(32 - nbit, \"int32\")) >> tir.const(32 - nbit, \"int32\"))\n\n    return f_convert\n\n\ndef _tir_f32_to_uint_to_f4(val: tir.PrimExpr):\n    assert val.dtype == \"float32\"\n    val_u32 = tir.reinterpret(\"uint32\", val)\n    # e_f32 >  120 -> e_f4 = min(e_f32 - 120 + M_h, 7)\n    # e_f32 == 120 -> e_f4 = 1\n    # e_f32 < 120 -> e_f4 = 0\n    m_h = (val_u32 >> tir.const(22, \"uint32\")) & tir.const(1, \"uint32\")\n    e_f32 = (val_u32 >> tir.const(23, \"uint32\")) & tir.const(255, \"uint32\")\n    s = (val_u32 >> tir.const(31, \"uint32\"))\n    e_f4 = tir.Select(e_f32 > tir.const(120, \"uint32\"), tir.Min(e_f32 - tir.const(120, \"uint32\") + m_h, tir.const(7, \"uint32\")), tir.Select(e_f32 == tir.const(120, \"uint32\"), tir.const(1, \"uint32\"), tir.const(0, \"uint32\")))\n    return (s << tir.const(3, \"uint32\")) | e_f4\n\n\ndef _tir_f16_to_uint_to_f4(val: tir.PrimExpr):\n    assert val.dtype == \"float16\"\n    val_u32 = tir.Cast(\"uint32\", tir.reinterpret(\"uint16\", val))\n    m_h = (val_u32 >> tir.const(9, \"uint32\")) & tir.const(1, \"uint32\")\n    e_f16 = (val_u32 >> tir.const(10, \"uint32\")) & tir.const(31, \"uint32\")\n    s = (val_u32 >> tir.const(15, \"uint32\"))\n    e_f4 = tir.Select(e_f16 > tir.const(8, \"uint32\"), tir.Min(e_f16 - tir.const(8, \"uint32\") + m_h, tir.const(7, \"uint32\")), tir.Select(e_f16 == tir.const(8, \"uint32\"), tir.const(1, \"uint32\"), tir.const(0, \"uint32\")))\n    return (s << tir.const(3, \"uint32\")) | e_f4\n\n\ndef _tir_u32_to_f4_to_f32(nbit: int, val: tir.PrimExpr, pos: tir.PrimExpr, dtype: str):\n    assert nbit == 4\n    assert dtype == \"float32\"\n    assert val.dtype == \"uint32\"\n    # e_f4 == 0 -> e_f32 = 0\n    # e_f4 != 0 -> e_f32 = e_f4 + 120 = e_f4 | (1111000)_2\n    mask = tvm.tir.const((1 << nbit) - 1, \"uint32\")\n    f4 = (val >> (pos.astype(\"uint32\") * tir.const(nbit, \"uint32\"))) & mask\n    s = f4 >> tir.const(3, \"uint32\")\n    e_f4 = f4 & tir.const(7, \"uint32\")\n    e_f32 = e_f4 | tir.const(120, \"uint32\")\n    val_f32 = tir.reinterpret(\"float32\", (e_f32 | (s << tir.const(8, \"uint32\"))) << tir.const(23, \"uint32\"))\n    return tir.Select(e_f4 == tir.const(0, \"uint32\"), tir.const(0, \"float32\"), val_f32)\n\n\ndef _tir_u32_to_f4_to_f16(nbit: int, val: tir.PrimExpr, pos: tir.PrimExpr, dtype: str):\n    assert nbit == 4\n    assert dtype == \"float16\"\n    assert val.dtype == \"uint32\"\n    # e_f4 == 0 -> e_f16 = 0\n    # e_f4 != 0 -> e_f16 = e_f4 + 8 = e_f4 | (1000)_2\n    mask = tvm.tir.const((1 << nbit) - 1, \"uint32\")\n    f4 = (val >> (pos.astype(\"uint32\") * tir.const(nbit, \"uint32\"))) & mask\n    s = f4 >> tir.const(3, \"uint32\")\n    e_f4 = f4 & tir.const(7, \"uint32\")\n    e_f16 = e_f4 | tir.const(8, \"uint32\")\n    val_f16 = tir.reinterpret(\"float16\", (e_f16 | (s << tir.const(5, \"uint32\"))) << tir.const(10, \"uint32\"))\n    return tir.Select(e_f4 == tir.const(0, \"uint32\"), tir.const(0, \"float16\"), val_f16)\n# fmt: on\n\n# Path: mlc_llm/quantization/autogptq_quantization.py\nfrom dataclasses import dataclass\nfrom typing import Any, List, Literal, Optional, Tuple\nfrom tvm import relax, te, tir, topi\nfrom . import tir_utils\nfrom .quantization import QuantizationSpec\nfrom .quantization import FQuantize, FTEDequantize, convert_TE_func\n\n\n@dataclass\nclass AutogptqQuantizationSpec(QuantizationSpec):\n    \"\"\"The quantization specification for group quantization algorithm.\"\"\"\n\n    mode: Literal[\"int2\", \"int3\", \"int4\", \"int8\"]\n    sym: bool\n    group_size: int\n    storage_nbit: int = 32\n\n    quantized_suffix = [\"qweight\", \"qzeros\", \"scales\", \"g_idx\"]\n\n    def get_loaded_tensor_info(\n        self, pname: str, param_info: relax.TensorStructInfo\n    ) -> Tuple[List[str], List[relax.TensorStructInfo]]:\n        assert self.storage_nbit == 32, \"Only support 32bit storage currently\"\n\n        quantized_pnames = self.quant_convert_pname_fwd(pname)\n        if len(quantized_pnames) == 1:\n            return quantized_pnames, [param_info]\n        else:\n            assert len(quantized_pnames) == 4\n            assert param_info.ndim == 2\n            nbit = int(self.mode[-1])\n            tensor_info = []\n            outfeatures, infeatures = param_info.shape.values\n            group_size = self.group_size if self.group_size != -1 else infeatures\n\n            def get_quantized_shape_dtype(quantized_pname: str):\n                if quantized_pname.endswith(\"qweight\"):\n                    return (infeatures // self.storage_nbit * nbit, outfeatures), \"uint32\"\n                elif quantized_pname.endswith(\"qzeros\"):\n                    return (\n                        infeatures // group_size,\n                        outfeatures // self.storage_nbit * nbit,\n                    ), \"uint32\"\n                elif quantized_pname.endswith(\"scales\"):\n                    return (infeatures // group_size, outfeatures), \"float16\"\n                elif quantized_pname.endswith(\"g_idx\"):\n                    return (infeatures,), \"uint32\"\n                else:\n                    raise ValueError(f\"Unrecognized quantized parameter name {quantized_pname}\")\n\n            for quantized_pname in quantized_pnames:\n                shape, dtype = get_quantized_shape_dtype(quantized_pname)\n                tensor_info.append(relax.TensorStructInfo(shape, dtype))\n\n        return quantized_pnames, tensor_info\n\n    def quant_convert_pname_fwd(self, torch_pname: str) -> List[str]:\n        # For Llama:\n        if \"_proj.weight\" in torch_pname:\n            return [torch_pname.replace(\"weight\", suffix) for suffix in self.quantized_suffix]\n        return [torch_pname]\n\n    def run_prequantize(self, model_path: str) -> str:\n        # with auto-gptq >= 0.2.0\n        try:\n            import auto_gptq  # pylint: disable=import-outside-toplevel\n            import transformers  # pylint: disable=import-outside-toplevel\n        except ImportError:\n            raise ImportError(\n                \"Please install auto_gptq package (version >= 0.2.0) and \"\n                \"transformers package to use AutoGPTQ quantization.\"\n            )\n        import os\n        from transformers import AutoTokenizer\n        from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig\n\n        quantized_model_path = (\n            model_path\n            + f\"-gptq-i{self.mode[-1]}\"\n            + (\"-sym\" if self.sym else \"\")\n            + f\"-g{self.group_size}\"\n        )\n        if os.path.isdir(quantized_model_path):\n            return quantized_model_path\n\n        tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)\n        examples = [\n            tokenizer(\n                \"MLC LLM is a universal solution that allows any language models \"\n                \"to be deployed natively on a diverse set of hardware backends and \"\n                \"native applications, plus a productive framework for everyone to \"\n                \"further optimize model performance for their own use cases.\"\n            )\n        ]\n        quantize_config = BaseQuantizeConfig(\n            bits=int(self.mode[-1]),  # quantize bits\n            desc_act=False,  # disable activation description\n            group_size=self.group_size,  # disable group quantization\n        )\n\n        model = AutoGPTQForCausalLM.from_pretrained(model_path, quantize_config)\n        model.quantize(examples)\n\n        # save quantized model\n        model.save_quantized(quantized_model_path)\n        tokenizer.save_pretrained(quantized_model_path)\n        return quantized_model_path\n\n    def get_quantize_func(self, param_info: relax.TensorStructInfo) -> Optional[FQuantize]:\n        return None\n\n    def get_dequantize_func(\n        self,\n        param_info: relax.TensorStructInfo,\n        qparam_info: List[relax.TensorStructInfo],\n    ) -> Optional[FQuantize]:\n        return convert_TE_func(\n            decoding_func(\n                sym=self.sym,\n                nbit=int(self.mode[-1]),\n                storage_nbit=self.storage_nbit,\n                dim_length=param_info.shape.values[-1],\n                dtype=self.dtype,\n            ),\n            func_name=\"decode\",\n        )\n\n    def convert_param_bkwd(self, torch_pname: str, torch_param):\n        target_dtype = (\n            self.dtype if \"_proj.\" not in torch_pname or \"scales\" in torch_pname else \"uint32\"\n        )\n\n        # For Llama\n        combined_layers = [\"q_proj\", \"k_proj\", \"v_proj\", \"gate_proj\", \"up_proj\"]\n        if any([name in torch_pname for name in combined_layers]):\n            return None\n        return [(torch_pname, torch_param.astype(target_dtype))]\n\n    def compute_relax_param(self, relax_pname: str, torch_params: List[Any]):\n        import numpy as np\n\n        # For Llama\n        if \"query_key_value_proj\" in relax_pname:\n            assert len(torch_params) == 3\n        elif \"gate_up_proj\" in relax_pname:\n            assert len(torch_params) == 2\n        else:\n            raise ValueError(\"Unexpected param loading\")\n\n        if \"g_idx\" in relax_pname:\n            return torch_params[0].astype(\"uint32\")\n        else:\n            target_dtype = self.dtype if \"scales\" in relax_pname else \"uint32\"\n            return np.concatenate(torch_params, axis=-1).astype(target_dtype)\n\n\ndef decoding_func(\n    sym: bool,\n    nbit: int,\n    storage_nbit: int,\n    dim_length: tir.PrimExpr,\n    dtype: str = \"float16\",\n) -> FTEDequantize:\n    assert dtype in [\"float16\"], \"Only support float16 currently\"\n    assert sym == False, \"Only support sym=False currently\"\n    assert storage_nbit == 32, \"Only support storage_nbit=32 currently\"\n\n    def te_decode_asym(qweight, qzeros, scales, g_idx):\n        n_float_per_u32 = 32 // nbit\n\n        def f_decode_asym(i, j):\n            zeros = tir_utils._tir_u32_to_int_to_float(\n                nbit,\n                qzeros[g_idx[i], j // n_float_per_u32],\n                j % n_float_per_u32,\n                dtype=dtype,\n            )\n            data_float = tir_utils._tir_u32_to_int_to_float(\n                nbit,\n                qweight[i // n_float_per_u32, j],\n                i % n_float_per_u32,\n                dtype=dtype,\n            )\n            scale_float, bias_float = scales[g_idx[i], j], zeros + 1\n            w = (data_float - bias_float) * scale_float\n            return w\n\n        shape = (dim_length, qweight.shape[1])\n        w = te.compute(shape=shape, fcompute=f_decode_asym, name=\"decode\")\n        w = topi.transpose(w)\n        return w\n\n    return te_decode_asym\n\n# Path: mlc_llm/quantization/group_quantization.py\nfrom dataclasses import dataclass\nfrom typing import List, Literal, Optional\n\nimport tvm\nfrom tvm import relax, te, tir, topi\nfrom tvm.script import tir as T\nfrom tvm.relax.expr_functor import visitor\n\nfrom . import tir_utils\nfrom .quantization import QuantizationSpec, QuantSpecUpdater\nfrom .quantization import NoQuantizationSpec\nfrom .quantization import FQuantize, FTEQuantize, FTEDequantize, convert_TE_func\n\n\n@dataclass\nclass GroupQuantizationSpec(QuantizationSpec):\n    \"\"\"The quantization specification for group quantization algorithm.\"\"\"\n\n    mode: Literal[\"int3\", \"int4\"]\n    sym: bool\n    storage_nbit: int\n    group_size: int\n    transpose: bool\n\n    def get_quantize_func(self, param_info: relax.TensorStructInfo) -> Optional[FQuantize]:\n        return convert_TE_func(\n            encoding_func(\n                sym=self.sym,\n                group_size=self.group_size,\n                nbit=int(self.mode[-1]),\n                mode=self.mode,\n                storage_nbit=self.storage_nbit,\n                transpose=self.transpose,\n                dtype=self.dtype,\n            ),\n            func_name=\"encode\",\n        )\n\n    def get_dequantize_func(\n        self,\n        param_info: relax.TensorStructInfo,\n        qparam_info: List[relax.TensorStructInfo],\n    ) -> Optional[FQuantize]:\n        return convert_TE_func(\n            decoding_func(\n                sym=self.sym,\n                group_size=self.group_size,\n                nbit=int(self.mode[-1]),\n                mode=self.mode,\n                storage_nbit=self.storage_nbit,\n                dim_length=param_info.shape.values[-1],\n                data_transposed=self.transpose,\n                transpose_output=self.transpose,\n                dtype=self.dtype,\n            ),\n            func_name=\"decode\",\n        )\n\n\n# fmt: off\ndef encoding_func(sym: bool, group_size: int, nbit: int, mode: str, storage_nbit: int, transpose: bool=True, dtype: str = \"float32\") -> FTEQuantize:\n    def te_encode_asym(weight: te.Tensor):\n        assert weight.shape[1] % group_size == 0\n        n_group = weight.shape[1] // group_size\n        n_float_per_u32 = 32 // nbit\n\n        scale_min_shape = (weight.shape[0], n_group)\n        k = te.reduce_axis((0, group_size), name=\"k\")\n        min_value = te.compute(shape=scale_min_shape, fcompute=lambda i, j: te.min(weight[i, j * group_size + k], axis=k), name=\"min_value\")\n        max_value = te.compute(shape=scale_min_shape, fcompute=lambda i, j: te.max(weight[i, j * group_size + k], axis=k), name=\"max_value\")\n        scale = te.compute(shape=scale_min_shape, fcompute=lambda i, j: (max_value[i, j] - min_value[i, j]) / tir.const((1 << nbit) - 1, dtype), name=\"scale\")\n\n        def f_scale_weight(i, j):\n            group_idx = j // group_size\n            w_scaled = tir.round((weight[i, j] - min_value[i, group_idx]) / scale[i, group_idx]).astype(\"int32\")\n            w_scaled = T.min(T.max(w_scaled, tir.const(0, \"int32\")), tir.const((1 << nbit) - 1, \"int32\"))\n            w_scaled = w_scaled.astype(\"uint32\")\n            return w_scaled\n\n        k = te.reduce_axis((0, n_float_per_u32), name=\"k\")\n        reducer = te.comm_reducer(fcombine=lambda x, y: tir.bitwise_or(x, y), fidentity=lambda dtype: tir.const(0, dtype), name=\"bitwise_or\")\n        if dtype == \"float32\":\n            if transpose:\n                w_gathered = te.compute(shape=(weight.shape[1] // n_float_per_u32, weight.shape[0]), fcompute=lambda j, i: reducer(f_scale_weight(i, j * n_float_per_u32 + k) << (k * nbit).astype(\"uint32\"), axis=k), name=\"w_gathered\")\n                scale_bias = te.compute(shape=(n_group, weight.shape[0]), fcompute=lambda j, i: tir_utils._tir_f32x2_to_bf16x2_to_u32(scale[i, j], min_value[i, j], round_to_even=True), name=\"scale_min\")\n            else:\n                w_gathered = te.compute(shape=(weight.shape[0], weight.shape[1] // n_float_per_u32), fcompute=lambda i, j: reducer(f_scale_weight(i, j * n_float_per_u32 + k) << (k * nbit).astype(\"uint32\"), axis=k), name=\"w_gathered\")\n                scale_bias = te.compute(shape=(weight.shape[0], n_group), fcompute=lambda i, j: tir_utils._tir_f32x2_to_bf16x2_to_u32(scale[i, j], min_value[i, j], round_to_even=True), name=\"scale_min\")\n            return w_gathered, scale_bias\n        else:\n            if transpose:\n                w_gathered = te.compute(shape=(weight.shape[1] // n_float_per_u32, weight.shape[0]), fcompute=lambda j, i: reducer(f_scale_weight(i, j * n_float_per_u32 + k) << (k * nbit).astype(\"uint32\"), axis=k), name=\"w_gathered\")\n                scale = te.compute(shape=(n_group, weight.shape[0]), fcompute=lambda j, i: scale[i, j], name=\"scale_transpose\")\n                min_value = te.compute(shape=(n_group, weight.shape[0]), fcompute=lambda j, i: min_value[i, j], name=\"min_transpose\")\n            else:\n                w_gathered = te.compute(shape=(weight.shape[0], weight.shape[1] // n_float_per_u32), fcompute=lambda i, j: reducer(f_scale_weight(i, j * n_float_per_u32 + k) << (k * nbit).astype(\"uint32\"), axis=k), name=\"w_gathered\")\n            return w_gathered, scale, min_value\n\n    def te_encode_sym(weight: te.Tensor):\n        n_group = tir.ceildiv(weight.shape[1], group_size)\n        n_float_per_int = storage_nbit // nbit\n        max_int_value = (1 << (nbit - 1)) - 1\n        assert group_size % n_float_per_int == 0\n\n        scale_min_shape = (weight.shape[0], n_group)\n        k = te.reduce_axis((0, group_size), name=\"k\")\n        max_abs_value = te.compute(shape=scale_min_shape, fcompute=lambda i, j: te.max(tir.if_then_else(j * group_size + k < weight.shape[1], te.abs(weight[i, j * group_size + k]), tir.min_value(dtype)), axis=k), name=\"max_abs_value\")\n\n        def f_compute_scale(i, j):\n            max_value = tir.max(max_abs_value[i, j], tir.const(1e-4, dtype))\n            return (max_value / tir.const(max_int_value, dtype)) if mode.startswith(\"int\") else max_value\n\n        scale = te.compute(shape=scale_min_shape, fcompute=f_compute_scale, name=\"scale\")\n        storage_dtype = (\"uint\" + str(storage_nbit)) if mode.startswith(\"int\") else \"uint32\"\n\n        def f_scale_weight(i, j):\n            group_idx = j // group_size\n            if mode.startswith(\"int\"):\n                w_scaled = tir.round(weight[i, j] / scale[i, group_idx] + tir.const(max_int_value, dtype))\n                w_scaled = T.min(T.max(w_scaled, tir.const(0, dtype)), tir.const(max_int_value * 2, dtype)).astype(storage_dtype)\n                return w_scaled\n            else:\n                f_convert = tir_utils._tir_f32_to_uint_to_f4 if dtype == \"float32\" else tir_utils._tir_f16_to_uint_to_f4\n                return f_convert(weight[i, j] / scale[i, group_idx])\n\n        k = te.reduce_axis((0, n_float_per_int), name=\"k\")\n        reducer = te.comm_reducer(fcombine=lambda x, y: tir.bitwise_or(x, y), fidentity=lambda dtype: tir.const(0, dtype), name=\"bitwise_or\")\n        n_i32 = tir.ceildiv(group_size, n_float_per_int) * n_group\n        if transpose:\n            w_gathered = te.compute(shape=(n_i32, weight.shape[0]), fcompute=lambda j, i: reducer(tir.if_then_else(j * n_float_per_int + k < weight.shape[1], f_scale_weight(i, j * n_float_per_int + k) << (k.astype(storage_dtype) * tir.const(nbit, storage_dtype)), tir.const(0, storage_dtype)), axis=k), name=\"w_gathered\")\n            scale = te.compute(shape=(n_group, weight.shape[0]), fcompute=lambda j, i: scale[i, j])\n        else:\n            w_gathered = te.compute(shape=(weight.shape[0], n_i32), fcompute=lambda i, j: reducer(tir.if_then_else(j * n_float_per_int + k < weight.shape[1], f_scale_weight(i, j * n_float_per_int + k) << (k.astype(storage_dtype) * tir.const(nbit, storage_dtype)), tir.const(0, storage_dtype)), axis=k), name=\"w_gathered\")\n        return w_gathered, scale\n\n    return te_encode_sym if sym else te_encode_asym\n\n\ndef decoding_func(sym: bool, group_size: int, nbit: int, mode: str, storage_nbit: int, dim_length: tir.PrimExpr, data_transposed: bool=True, transpose_output: bool=False, dtype: str = \"float32\") -> FTEDequantize:\n    def te_decode_asym(*args):\n        n_float_per_u32 = 32 // nbit\n        data = args[0]\n        if dtype == \"float32\":\n            scale_bias_bf16x2 = args[1]\n        else:\n            scale, min_value = args[1], args[2]\n\n        def f_decode_asym(i, j):\n            if data_transposed:\n                data_float = tir_utils._tir_u32_to_int_to_float(nbit, data[i // n_float_per_u32, j], i % n_float_per_u32, dtype=dtype)\n                if dtype == \"float32\":\n                    scale_float, bias_float = tir_utils._tir_u32_to_bf16x2_to_f32x2(scale_bias_bf16x2[i // group_size, j])\n                else:\n                    scale_float, bias_float = scale[i // group_size, j], min_value[i // group_size, j]\n            else:\n                data_float = tir_utils._tir_u32_to_int_to_float(nbit, data[i, j // n_float_per_u32], j % n_float_per_u32, dtype=dtype)\n                if dtype == \"float32\":\n                    scale_float, bias_float = tir_utils._tir_u32_to_bf16x2_to_f32x2(scale_bias_bf16x2[i, j // group_size])\n                else:\n                    scale_float, bias_float = scale[i, j // group_size], min_value[i, j // group_size]\n            w = data_float * scale_float + bias_float\n            return w\n\n        shape = (dim_length, data.shape[1]) if data_transposed else (data.shape[0], dim_length)\n        w = te.compute(shape=shape, fcompute=f_decode_asym, name=\"decode\")\n        if transpose_output:\n            w = topi.transpose(w)\n        return w\n\n    def te_decode_sym(data, scale):\n        n_float_per_int = storage_nbit // nbit\n\n        def f_decode_sym(i, j):\n            f_convert = tir_utils._tir_packed_uint_to_uint_to_float(storage_nbit) if mode.startswith(\"int\") else (tir_utils._tir_u32_to_f4_to_f32 if dtype == \"float32\" else tir_utils._tir_u32_to_f4_to_f16)\n            if data_transposed:\n                data_float = f_convert(nbit, data[i // n_float_per_int, j], i % n_float_per_int, dtype=dtype)\n                scale_float = scale[i // group_size, j]\n            else:\n                data_float = f_convert(nbit, data[i, j // n_float_per_int], j % n_float_per_int, dtype=dtype)\n                scale_float = scale[i, j // group_size]\n            return data_float * scale_float\n\n        shape = (dim_length, data.shape[1]) if data_transposed else (data.shape[0], dim_length)\n        w = te.compute(shape=shape, fcompute=f_decode_sym, name=\"decode\")\n        if transpose_output:\n            w = topi.transpose(w)\n        return w\n\n    return te_decode_sym if sym else te_decode_asym\n# fmt: on\n\n\n# A simple example demo showing how QuantSpecUpdater is used.\n# NOTE: This visitor is only for demo purpose and should not be put into real use.\n@visitor\nclass GroupQuantDemoUpdater(QuantSpecUpdater._cls):\n    def visit_call_(self, call: relax.Call):\n        if call.op != tvm.ir.Op.get(\"relax.matmul\"):\n            return\n        rhs = self.lookup_binding(call.args[1])\n        assert rhs is not None\n        if (\n            rhs.op != tvm.ir.Op.get(\"relax.permute_dims\")\n            or rhs.attrs.axes is not None\n            or rhs.args[0].struct_info.ndim != 2\n        ):\n            return\n\n        if rhs.args[0] not in self.param_map:\n            return\n        param = self.param_map[rhs.args[0]]\n        # Update to no quantization for matmul with float32 output dtype.\n        if call.struct_info.dtype == \"float32\":\n            param.quant_spec = NoQuantizationSpec(param.param_info.dtype)\n\n# Path: mlc_llm/quantization/ft_quantization.py\nfrom dataclasses import dataclass\nfrom typing import List, Optional\n\nimport tvm\nfrom tvm.contrib.nvcc import parse_compute_version\nfrom tvm import relax, te, tir, topi\nfrom tvm.script import tir as T\nfrom tvm.relax.expr_functor import visitor\n\nfrom . import tir_utils\nfrom .quantization import QuantizationSpec, QuantSpecUpdater\nfrom .quantization import FQuantize, convert_TE_func\nfrom .group_quantization import GroupQuantizationSpec\n\n\n@dataclass\nclass FTQuantizationSpec(QuantizationSpec):\n    \"\"\"The quantization specification for the FasterTransformer kernel.\"\"\"\n\n    def __init__(self, dtype, nbit, group_size=-1):\n        super().__init__(dtype)\n        self.nbit = nbit\n        assert group_size in [-1, 64, 128], f\"Group size {group_size} is not supported.\"\n        self.group_size = group_size\n\n        if tvm.cuda(0).exist:\n            major, minor = parse_compute_version(tvm.cuda(0).compute_version)\n            if major == 8:\n                self.sm = 80\n            else:\n                self.sm = 10 * major + minor\n        else:\n            self.sm = None\n\n        self.do_preprocess = True\n\n    def get_quantize_func(self, param_info: relax.TensorStructInfo) -> Optional[FQuantize]:\n        assert self.sm is not None\n\n        \ndef f_quantize(bb: relax.BlockBuilder, inputs: List[relax.Expr]):\n            encoded_data = bb.emit_te(\n                encoding_func(\n                    self.nbit,\n                    8,\n                    group_size=self.group_size,\n                    dtype=self.dtype,\n                ),\n                inputs[0],\n                primfunc_name_hint=\"encode\",\n            )\n\n            packed_weight = bb.normalize(encoded_data[0])\n\n            if self.do_preprocess:\n                encoded_weight = bb.emit(\n                    relax.call_pure_packed(\n                        \"cutlass.ft_preprocess_weight\",\n                        packed_weight,\n                        self.sm,\n                        self.nbit == 4,\n                        sinfo_args=packed_weight.struct_info,\n                    )\n                )\n            else:\n                encoded_weight = packed_weight\n\n            return bb.emit(relax.Tuple([encoded_weight, encoded_data[1]]))\n\n        return f_quantize\n\n    def get_dequantize_func(\n        self,\n        param_info: relax.TensorStructInfo,\n        qparam_info: List[relax.TensorStructInfo],\n    ) -> Optional[FQuantize]:\n        return convert_TE_func(\n            decoding_func(\n                self.nbit,\n                storage_nbit=8,\n                group_size=self.group_size,\n            ),\n            func_name=\"decode\",\n        )\n\n\ndef encoding_func(nbit: int, storage_nbit: int, group_size: int, dtype: str = \"float32\"):\n    def te_encode_sym(weight: te.Tensor):\n        \"\"\"Encode the weight tensor of shape [N, K] into a quantized weight tensor of shape\n        [K, N // float_per_int] and a scale tensor of shape [K // group_size, N]\n        \"\"\"\n        n_float_per_int = storage_nbit // nbit\n        max_int_value = (1 << (nbit - 1)) - 1\n\n        cur_group_size = weight.shape[1] if group_size == -1 else group_size\n        scale_min_shape = (tir.ceildiv(weight.shape[1], cur_group_size), weight.shape[0])\n        k = te.reduce_axis((0, cur_group_size), name=\"k\")\n        max_abs_value = te.compute(\n            shape=scale_min_shape,\n            fcompute=lambda group, i: te.max(\n                te.abs(\n                    tir.if_then_else(\n                        group * cur_group_size + k < weight.shape[1],\n                        weight[i, group * cur_group_size + k],\n                        tir.const(0, dtype=weight.dtype),\n                    )\n                ),\n                axis=k,\n            ),\n            name=\"max_abs_value\",\n        )\n\n        def f_compute_scale(*idx):\n            max_value = tir.max(tir.Cast(dtype, max_abs_value(*idx)), tir.const(1e-4, dtype))\n            return max_value / tir.const(max_int_value, dtype)\n\n        scale = te.compute(shape=scale_min_shape, fcompute=f_compute_scale, name=\"scale\")\n        storage_dtype = \"int\" + str(storage_nbit)\n\n        def f_scale_weight(i, j):\n            w_scaled = tir.round(tir.Cast(dtype, weight[i, j]) / scale[j // cur_group_size, i])\n            w_scaled = T.min(\n...\n", "needle_token_start": 11602, "needle_token_end": 11861, "code_context_ntokens": 12525, "output": ["```python\ndef f_quantize(bb: relax.BlockBuilder, inputs: List[relax.Expr]):\n    encoded_data = bb.emit_te(\n        encoding_func(\n            self.nbit,\n            8,\n            group_size=self.group_size,\n            dtype=self.dtype,\n        ),\n        inputs[0],\n        primfunc_name_hint=\"encode\",\n    )\n\n    packed_weight = bb.normalize(encoded_data[0])\n\n    if self.do_preprocess:\n        encoded_weight = bb.emit(\n            relax.call_pure_packed(\n                \"cutlass.ft_preprocess_weight\",\n                packed_weight,\n                self.sm,\n                self.nbit == 4,\n                sinfo_args=packed_weight.struct_info,\n            )\n        )\n    else:\n        encoded_weight = packed_weight\n\n    return bb.emit(relax.Tuple([encoded_weight, encoded_data[1]]))\n```"]}
{"repo": "reactive-python/reactpy", "name": "sync_function", "language": "python", "path": "src/py/reactpy/reactpy/core/hooks.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To execute a synchronous version of an asynchronous effect function, potentially including a cleanup process.\n2. **Input**: An asynchronous function that returns a coroutine, which when executed, may return a cleanup function.\n3. **Output**: A synchronous function that, when called, handles the execution of the asynchronous function and its cleanup process.\n4. **Procedure**: \n   - The asynchronous function is wrapped into a synchronous function.\n   - A task is created to run the asynchronous function using asyncio.\n   - A cleanup function is defined within the synchronous function to handle the cancellation of the task and to execute any cleanup returned by the asynchronous function if the task completes successfully.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    return State(current_state.value, current_state.dispatch)\n\n\nclass _CurrentState(Generic[_Type]):\n    __slots__ = \"value\", \"dispatch\"\n\n    def __init__(\n        self,\n        initial_value: _Type | Callable[[], _Type],\n    ) -> None:\n        if callable(initial_value):\n            self.value = initial_value()\n        else:\n            self.value = initial_value\n\n        hook = current_hook()\n\n        def dispatch(new: _Type | Callable[[_Type], _Type]) -> None:\n            if callable(new):\n                next_value = new(self.value)\n            else:\n                next_value = new\n            if not strictly_equal(next_value, self.value):\n                self.value = next_value\n                hook.schedule_render()\n\n        self.dispatch = dispatch\n\n\n_EffectCleanFunc: TypeAlias = \"Callable[[], None]\"\n_SyncEffectFunc: TypeAlias = \"Callable[[], _EffectCleanFunc | None]\"\n_AsyncEffectFunc: TypeAlias = (\n    \"Callable[[], Coroutine[None, None, _EffectCleanFunc | None]]\"\n)\n_EffectApplyFunc: TypeAlias = \"_SyncEffectFunc | _AsyncEffectFunc\"\n\n\n@overload\ndef use_effect(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None]: ...\n\n\n@overload\ndef use_effect(\n    function: _EffectApplyFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None: ...\n\n\ndef use_effect(\n    function: _EffectApplyFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None] | None:\n    \"\"\"See the full :ref:`Use Effect` docs for details\n\n    Parameters:\n        function:\n            Applies the effect and can return a clean-up function\n        dependencies:\n            Dependencies for the effect. The effect will only trigger if the identity\n            of any value in the given sequence changes (i.e. their :func:`id` is\n            different). By default these are inferred based on local variables that are\n            referenced by the given function.\n\n    Returns:\n        If not function is provided, a decorator. Otherwise ``None``.\n    \"\"\"\n    hook = current_hook()\n\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n    last_clean_callback: Ref[_EffectCleanFunc | None] = use_ref(None)\n\n    def add_effect(function: _EffectApplyFunc) -> None:\n        if not asyncio.iscoroutinefunction(function):\n            sync_function = cast(_SyncEffectFunc, function)\n        else:\n            async_function = cast(_AsyncEffectFunc, function)\n\n            \ndef sync_function() -> _EffectCleanFunc | None:\n                task = asyncio.create_task(async_function())\n\n                def clean_future() -> None:\n                    if not task.cancel():\n                        try:\n                            clean = task.result()\n                        except asyncio.CancelledError:\n                            pass\n                        else:\n                            if clean is not None:\n                                clean()\n\n                return clean_future\n\n        async def effect(stop: asyncio.Event) -> None:\n            if last_clean_callback.current is not None:\n                last_clean_callback.current()\n                last_clean_callback.current = None\n            clean = last_clean_callback.current = sync_function()\n            await stop.wait()\n            if clean is not None:\n                clean()\n\n        return memoize(lambda: hook.add_effect(effect))\n\n    if function is not None:\n        add_effect(function)\n        return None\n    else:\n        return add_effect\n\n\ndef use_debug_value(\n    message: Any | Callable[[], Any],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None:\n    \"\"\"Log debug information when the given message changes.\n\n    .. note::\n        This hook only logs if :data:`~reactpy.config.REACTPY_DEBUG_MODE` is active.\n\n    Unlike other hooks, a message is considered to have changed if the old and new\n    values are ``!=``. Because this comparison is performed on every render of the\n    component, it may be worth considering the performance cost in some situations.\n\n    Parameters:\n        message:\n            The value to log or a memoized function for generating the value.\n        dependencies:\n            Dependencies for the memoized function. The message will only be recomputed\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n    \"\"\"\n    old: Ref[Any] = _use_const(lambda: Ref(object()))\n    memo_func = message if callable(message) else lambda: message\n    new = use_memo(memo_func, dependencies)\n\n    if REACTPY_DEBUG_MODE.current and old.current != new:\n        old.current = new\n        logger.debug(f\"{current_hook().component} {new}\")\n\n\ndef create_context(default_value: _Type) -> Context[_Type]:\n    \"\"\"Return a new context type for use in :func:`use_context`\"\"\"\n\n    def context(\n        *children: Any,\n        value: _Type = default_value,\n        key: Key | None = None,\n    ) -> _ContextProvider[_Type]:\n        return _ContextProvider(\n            *children,\n            value=value,\n            key=key,\n            type=context,\n        )\n\n    context.__qualname__ = \"context\"\n\n    return context\n\n\ndef use_context(context: Context[_Type]) -> _Type:\n    \"\"\"Get the current value for the given context type.\n\n    See the full :ref:`Use Context` docs for more information.\n    \"\"\"\n    hook = current_hook()\n    provider = hook.get_context_provider(context)\n\n    if provider is None:\n        # same assertions but with normal exceptions\n        if not isinstance(context, FunctionType):\n            raise TypeError(f\"{context} is not a Context\")  # nocov\n        if context.__kwdefaults__ is None:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        if \"value\" not in context.__kwdefaults__:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        return cast(_Type, context.__kwdefaults__[\"value\"])\n\n    return provider.value\n\n\nclass _ContextProvider(Generic[_Type]):\n    def __init__(\n        self,\n        *children: Any,\n        value: _Type,\n        key: Key | None,\n        type: Context[_Type],\n    ) -> None:\n        self.children = children\n        self.key = key\n        self.type = type\n        self.value = value\n\n    def render(self) -> VdomDict:\n        current_hook().set_context_provider(self)\n        return {\"tagName\": \"\", \"children\": self.children}\n\n    def __repr__(self) -> str:\n        return f\"ContextProvider({self.type})\"\n\n\n_ActionType = TypeVar(\"_ActionType\")\n\n\ndef use_reducer(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    initial_value: _Type,\n) -> tuple[_Type, Callable[[_ActionType], None]]:\n    \"\"\"See the full :ref:`Use Reducer` docs for details\n\n    Parameters:\n        reducer:\n            A function which applies an action to the current state in order to\n            produce the next state.\n        initial_value:\n            The initial state value (same as for :func:`use_state`)\n\n    Returns:\n        A tuple containing the current state and a function to change it with an action\n    \"\"\"\n    state, set_state = use_state(initial_value)\n    return state, _use_const(lambda: _create_dispatcher(reducer, set_state))\n\n\ndef _create_dispatcher(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    set_state: Callable[[Callable[[_Type], _Type]], None],\n) -> Callable[[_ActionType], None]:\n    def dispatch(action: _ActionType) -> None:\n        set_state(lambda last_state: reducer(last_state, action))\n\n    return dispatch\n\n\n_CallbackFunc = TypeVar(\"_CallbackFunc\", bound=Callable[..., Any])\n\n\n@overload\ndef use_callback(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_CallbackFunc], _CallbackFunc]: ...\n\n\n@overload\ndef use_callback(\n    function: _CallbackFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc: ...\n\n\ndef use_callback(\n    function: _CallbackFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc | Callable[[_CallbackFunc], _CallbackFunc]:\n    \"\"\"See the full :ref:`Use Callback` docs for details\n\n    Parameters:\n        function:\n            The function whose identity will be preserved\n        dependencies:\n            Dependencies of the callback. The identity the ``function`` will be updated\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current function\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n\n    def setup(function: _CallbackFunc) -> _CallbackFunc:\n        return memoize(lambda: function)\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _LambdaCaller(Protocol):\n    \"\"\"MyPy doesn't know how to deal with TypeVars only used in function return\"\"\"\n\n    def __call__(self, func: Callable[[], _Type]) -> _Type: ...\n\n\n@overload\ndef use_memo(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _LambdaCaller: ...\n\n\n@overload\ndef use_memo(\n    function: Callable[[], _Type],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type: ...\n\n\ndef use_memo(\n    function: Callable[[], _Type] | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type | Callable[[Callable[[], _Type]], _Type]:\n    \"\"\"See the full :ref:`Use Memo` docs for details\n\n    Parameters:\n        function:\n            The function to be memoized.\n        dependencies:\n            Dependencies for the memoized function. The memo will only be recomputed if\n            the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current state\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n\n    memo: _Memo[_Type] = _use_const(_Memo)\n\n    if memo.empty():\n        # we need to initialize on the first run\n        changed = True\n        memo.deps = () if dependencies is None else dependencies\n    elif dependencies is None:\n        changed = True\n        memo.deps = ()\n    elif (\n        len(memo.deps) != len(dependencies)\n        # if deps are same length check identity for each item\n        or not all(\n            strictly_equal(current, new)\n            for current, new in zip(memo.deps, dependencies)\n        )\n    ):\n        memo.deps = dependencies\n        changed = True\n    else:\n        changed = False\n\n    setup: Callable[[Callable[[], _Type]], _Type]\n\n    if changed:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            current_value = memo.value = function()\n            return current_value\n\n    else:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            return memo.value\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _Memo(Generic[_Type]):\n    \"\"\"Simple object for storing memoization data\"\"\"\n\n    __slots__ = \"value\", \"deps\"\n\n    value: _Type\n    deps: Sequence[Any]\n\n    def empty(self) -> bool:\n        try:\n            self.value  # noqa: B018\n        except AttributeError:\n            return True\n        else:\n            return False\n\n\ndef use_ref(initial_value: _Type) -> Ref[_Type]:\n    \"\"\"See the full :ref:`Use State` docs for details\n\n    Parameters:\n        initial_value: The value initially assigned to the reference.\n\n    Returns:\n        A :class:`Ref` object.\n    \"\"\"\n    return _use_const(lambda: Ref(initial_value))\n\n\ndef _use_const(function: Callable[[], _Type]) -> _Type:\n    return current_hook().use_state(function)\n\n\ndef _try_to_infer_closure_values(\n    func: Callable[..., Any] | None,\n    values: Sequence[Any] | ellipsis | None,\n) -> Sequence[Any] | None:\n    if values is ...:\n        if isinstance(func, FunctionType):\n            return (\n                [cell.cell_contents for cell in func.__closure__]\n                if func.__closure__\n                else []\n            )\n        else:\n            return None\n    else:\n        return values\n\n\ndef strictly_equal(x: Any, y: Any) -> bool:\n    \"\"\"Check if two values are identical or, for a limited set or types, equal.\n\n    Only the following types are checked for equality rather than identity:\n\n    - ``int``\n    - ``float``\n    - ``complex``\n    - ``str``\n    - ``bytes``\n    - ``bytearray``\n    - ``memoryview``\n    \"\"\"\n    return x is y or (type(x) in _NUMERIC_TEXT_BINARY_TYPES and x == y)\n\n\n_NUMERIC_TEXT_BINARY_TYPES = {\n    # numeric\n    int,\n    float,\n    complex,\n    # text\n    str,\n    # binary types\n    bytes,\n    bytearray,\n    memoryview,\n}\n\n# Path: src/py/reactpy/reactpy/backend/hooks.py\nfrom __future__ import annotations\n\nfrom collections.abc import MutableMapping\nfrom typing import Any\n\nfrom reactpy.backend.types import Connection, Location\nfrom reactpy.core.hooks import create_context, use_context\nfrom reactpy.core.types import Context\n\n# backend implementations should establish this context at the root of an app\nConnectionContext: Context[Connection[Any] | None] = create_context(None)\n\n\ndef use_connection() -> Connection[Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`.\"\"\"\n    conn = use_context(ConnectionContext)\n    if conn is None:  # nocov\n        msg = \"No backend established a connection.\"\n        raise RuntimeError(msg)\n    return conn\n\n\ndef use_scope() -> MutableMapping[str, Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s scope.\"\"\"\n    return use_connection().scope\n\n\ndef use_location() -> Location:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s location.\"\"\"\n    return use_connection().location\n\n# Path: src/py/reactpy/reactpy/core/component.py\nfrom __future__ import annotations\n\nimport inspect\nfrom functools import wraps\nfrom typing import Any, Callable\n\nfrom reactpy.core.types import ComponentType, VdomDict\n\n\ndef component(\n    function: Callable[..., ComponentType | VdomDict | str | None]\n) -> Callable[..., Component]:\n    \"\"\"A decorator for defining a new component.\n\n    Parameters:\n        function: The component's :meth:`reactpy.core.proto.ComponentType.render` function.\n    \"\"\"\n    sig = inspect.signature(function)\n\n    if \"key\" in sig.parameters and sig.parameters[\"key\"].kind in (\n        inspect.Parameter.KEYWORD_ONLY,\n        inspect.Parameter.POSITIONAL_OR_KEYWORD,\n    ):\n        msg = f\"Component render function {function} uses reserved parameter 'key'\"\n        raise TypeError(msg)\n\n    @wraps(function)\n    def constructor(*args: Any, key: Any | None = None, **kwargs: Any) -> Component:\n        return Component(function, key, args, kwargs, sig)\n\n    return constructor\n\n\nclass Component:\n    \"\"\"An object for rending component models.\"\"\"\n\n    __slots__ = \"__weakref__\", \"_func\", \"_args\", \"_kwargs\", \"_sig\", \"key\", \"type\"\n\n    def __init__(\n        self,\n        function: Callable[..., ComponentType | VdomDict | str | None],\n        key: Any | None,\n        args: tuple[Any, ...],\n        kwargs: dict[str, Any],\n        sig: inspect.Signature,\n    ) -> None:\n        self.key = key\n        self.type = function\n        self._args = args\n        self._kwargs = kwargs\n        self._sig = sig\n\n    def render(self) -> ComponentType | VdomDict | str | None:\n        return self.type(*self._args, **self._kwargs)\n\n    def __repr__(self) -> str:\n        try:\n            args = self._sig.bind(*self._args, **self._kwargs).arguments\n        except TypeError:\n            return f\"{self.type.__name__}(...)\"\n        else:\n            items = \", \".join(f\"{k}={v!r}\" for k, v in args.items())\n            if items:\n                return f\"{self.type.__name__}({id(self):02x}, {items})\"\n            else:\n                return f\"{self.type.__name__}({id(self):02x})\"\n\n# Path: src/py/reactpy/reactpy/types.py\n\"\"\"Exports common types from:\n\n- :mod:`reactpy.core.types`\n- :mod:`reactpy.backend.types`\n\"\"\"\n\nfrom reactpy.backend.types import BackendType, Connection, Location\nfrom reactpy.core.component import Component\nfrom reactpy.core.types import (\n    ComponentConstructor,\n    ComponentType,\n    Context,\n    EventHandlerDict,\n    EventHandlerFunc,\n    EventHandlerMapping,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    LayoutType,\n    RootComponentConstructor,\n    State,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomJson,\n)\n\n__all__ = [\n    \"BackendType\",\n    \"Component\",\n    \"ComponentConstructor\",\n    \"ComponentType\",\n    \"Connection\",\n    \"Context\",\n    \"EventHandlerDict\",\n    \"EventHandlerFunc\",\n    \"EventHandlerMapping\",\n    \"EventHandlerType\",\n    \"ImportSourceDict\",\n    \"Key\",\n    \"LayoutType\",\n    \"Location\",\n    \"RootComponentConstructor\",\n    \"State\",\n    \"VdomAttributes\",\n    \"VdomChild\",\n    \"VdomChildren\",\n    \"VdomDict\",\n    \"VdomJson\",\n]\n\n# Path: src/py/reactpy/reactpy/backend/utils.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nimport socket\nimport sys\nfrom collections.abc import Iterator\nfrom contextlib import closing\nfrom importlib import import_module\nfrom typing import Any\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.types import RootComponentConstructor\n\nlogger = logging.getLogger(__name__)\n\nSUPPORTED_BACKENDS = (\n    \"fastapi\",\n    \"sanic\",\n    \"tornado\",\n    \"flask\",\n    \"starlette\",\n)\n\n\ndef run(\n    component: RootComponentConstructor,\n    host: str = \"127.0.0.1\",\n    port: int | None = None,\n    implementation: BackendType[Any] | None = None,\n) -> None:\n    \"\"\"Run a component with a development server\"\"\"\n    logger.warning(_DEVELOPMENT_RUN_FUNC_WARNING)\n\n    implementation = implementation or import_module(\"reactpy.backend.default\")\n    app = implementation.create_development_app()\n    implementation.configure(app, component)\n    port = port or find_available_port(host)\n    app_cls = type(app)\n\n    logger.info(\n        \"ReactPy is running with '%s.%s' at http://%s:%s\",\n        app_cls.__module__,\n        app_cls.__name__,\n        host,\n        port,\n    )\n    asyncio.run(implementation.serve_development_app(app, host, port))\n\n\ndef find_available_port(host: str, port_min: int = 8000, port_max: int = 9000) -> int:\n    \"\"\"Get a port that's available for the given host and port range\"\"\"\n    for port in range(port_min, port_max):\n        with closing(socket.socket()) as sock:\n            try:\n                if sys.platform in (\"linux\", \"darwin\"):\n                    # Fixes bug on Unix-like systems where every time you restart the\n                    # server you'll get a different port on Linux. This cannot be set\n                    # on Windows otherwise address will always be reused.\n                    # Ref: https://stackoverflow.com/a/19247688/3159288\n                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n                sock.bind((host, port))\n            except OSError:\n                pass\n            else:\n                return port\n    msg = f\"Host {host!r} has no available port in range {port_max}-{port_max}\"\n    raise RuntimeError(msg)\n\n\ndef all_implementations() -> Iterator[BackendType[Any]]:\n    \"\"\"Yield all available server implementations\"\"\"\n    for name in SUPPORTED_BACKENDS:\n        try:\n            import_module(name)\n        except ImportError:  # nocov\n            logger.debug(\"Failed to import %s\", name, exc_info=True)\n            continue\n\n        reactpy_backend_name = f\"{__name__.rsplit('.', 1)[0]}.{name}\"\n        yield import_module(reactpy_backend_name)\n\n\n_DEVELOPMENT_RUN_FUNC_WARNING = \"\"\"\\\nThe `run()` function is only intended for testing during development! To run \\\nin production, refer to the docs on how to use reactpy.backend.*.configure.\\\n\"\"\"\n\n# Path: src/py/reactpy/reactpy/core/layout.py\nfrom __future__ import annotations\n\nimport abc\nfrom asyncio import (\n    FIRST_COMPLETED,\n    CancelledError,\n    Queue,\n    Task,\n    create_task,\n    get_running_loop,\n    wait,\n)\nfrom collections import Counter\nfrom collections.abc import Sequence\nfrom contextlib import AsyncExitStack\nfrom logging import getLogger\nfrom typing import (\n    Any,\n    Callable,\n    Generic,\n    NamedTuple,\n    NewType,\n    TypeVar,\n    cast,\n)\nfrom uuid import uuid4\nfrom weakref import ref as weakref\n\nfrom anyio import Semaphore\nfrom typing_extensions import TypeAlias\n\nfrom reactpy.config import (\n    REACTPY_ASYNC_RENDERING,\n    REACTPY_CHECK_VDOM_SPEC,\n    REACTPY_DEBUG_MODE,\n)\nfrom reactpy.core._life_cycle_hook import LifeCycleHook\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    Key,\n    LayoutEventMessage,\n    LayoutUpdateMessage,\n    VdomChild,\n    VdomDict,\n    VdomJson,\n)\nfrom reactpy.core.vdom import validate_vdom_json\nfrom reactpy.utils import Ref\n\nlogger = getLogger(__name__)\n\n\nclass Layout:\n    \"\"\"Responsible for \"rendering\" components. That is, turning them into VDOM.\"\"\"\n\n    __slots__: tuple[str, ...] = (\n        \"root\",\n        \"_event_handlers\",\n        \"_rendering_queue\",\n        \"_render_tasks\",\n        \"_render_tasks_ready\",\n        \"_root_life_cycle_state_id\",\n        \"_model_states_by_life_cycle_state_id\",\n    )\n\n    if not hasattr(abc.ABC, \"__weakref__\"):  # nocov\n        __slots__ += (\"__weakref__\",)\n\n    def __init__(self, root: ComponentType) -> None:\n        super().__init__()\n        if not isinstance(root, ComponentType):\n            msg = f\"Expected a ComponentType, not {type(root)!r}.\"\n            raise TypeError(msg)\n        self.root = root\n\n    async def __aenter__(self) -> Layout:\n        # create attributes here to avoid access before entering context manager\n        self._event_handlers: EventHandlerDict = {}\n        self._render_tasks: set[Task[LayoutUpdateMessage]] = set()\n        self._render_tasks_ready: Semaphore = Semaphore(0)\n\n        self._rendering_queue: _ThreadSafeQueue[_LifeCycleStateId] = _ThreadSafeQueue()\n        root_model_state = _new_root_model_state(self.root, self._schedule_render_task)\n\n        self._root_life_cycle_state_id = root_id = root_model_state.life_cycle_state.id\n        self._model_states_by_life_cycle_state_id = {root_id: root_model_state}\n        self._schedule_render_task(root_id)\n\n        return self\n\n    async def __aexit__(self, *exc: Any) -> None:\n        root_csid = self._root_life_cycle_state_id\n        root_model_state = self._model_states_by_life_cycle_state_id[root_csid]\n\n        for t in self._render_tasks:\n            t.cancel()\n            try:\n                await t\n            except CancelledError:\n                pass\n\n        await self._unmount_model_states([root_model_state])\n\n        # delete attributes here to avoid access after exiting context manager\n        del self._event_handlers\n        del self._rendering_queue\n        del self._root_life_cycle_state_id\n        del self._model_states_by_life_cycle_state_id\n\n    async def deliver(self, event: LayoutEventMessage) -> None:\n        \"\"\"Dispatch an event to the targeted handler\"\"\"\n        # It is possible for an element in the frontend to produce an event\n        # associated with a backend model that has been deleted. We only handle\n        # events if the element and the handler exist in the backend. Otherwise\n        # we just ignore the event.\n        handler = self._event_handlers.get(event[\"target\"])\n\n        if handler is not None:\n            try:\n                await handler.function(event[\"data\"])\n            except Exception:\n                logger.exception(f\"Failed to execute event handler {handler}\")\n        else:\n            logger.info(\n                f\"Ignored event - handler {event['target']!r} \"\n                \"does not exist or its component unmounted\"\n            )\n\n    async def render(self) -> LayoutUpdateMessage:\n        if REACTPY_ASYNC_RENDERING.current:\n            return await self._concurrent_render()\n        else:  # nocov\n            return await self._serial_render()\n\n    async def _serial_render(self) -> LayoutUpdateMessage:  # nocov\n        \"\"\"Await the next available render. This will block until a component is updated\"\"\"\n        while True:\n            model_state_id = await self._rendering_queue.get()\n            try:\n                model_state = self._model_states_by_life_cycle_state_id[model_state_id]\n            except KeyError:\n                logger.debug(\n                    \"Did not render component with model state ID \"\n                    f\"{model_state_id!r} - component already unmounted\"\n                )\n            else:\n                return await self._create_layout_update(model_state)\n\n    async def _concurrent_render(self) -> LayoutUpdateMessage:\n        \"\"\"Await the next available render. This will block until a component is updated\"\"\"\n        await self._render_tasks_ready.acquire()\n        done, _ = await wait(self._render_tasks, return_when=FIRST_COMPLETED)\n        update_task: Task[LayoutUpdateMessage] = done.pop()\n        self._render_tasks.remove(update_task)\n        return update_task.result()\n\n    async def _create_layout_update(\n        self, old_state: _ModelState\n    ) -> LayoutUpdateMessage:\n        new_state = _copy_component_model_state(old_state)\n        component = new_state.life_cycle_state.component\n\n        async with AsyncExitStack() as exit_stack:\n            await self._render_component(exit_stack, old_state, new_state, component)\n\n        if REACTPY_CHECK_VDOM_SPEC.current:\n            validate_vdom_json(new_state.model.current)\n\n        return {\n            \"type\": \"layout-update\",\n            \"path\": new_state.patch_path,\n            \"model\": new_state.model.current,\n        }\n\n    async def _render_component(\n        self,\n        exit_stack: AsyncExitStack,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n        component: ComponentType,\n    ) -> None:\n        life_cycle_state = new_state.life_cycle_state\n        life_cycle_hook = life_cycle_state.hook\n\n        self._model_states_by_life_cycle_state_id[life_cycle_state.id] = new_state\n\n        await life_cycle_hook.affect_component_will_render(component)\n        exit_stack.push_async_callback(life_cycle_hook.affect_layout_did_render)\n        try:\n            raw_model = component.render()\n            # wrap the model in a fragment (i.e. tagName=\"\") to ensure components have\n            # a separate node in the model state tree. This could be removed if this\n            # components are given a node in the tree some other way\n            wrapper_model: VdomDict = {\"tagName\": \"\", \"children\": [raw_model]}\n            await self._render_model(exit_stack, old_state, new_state, wrapper_model)\n        except Exception as error:\n            logger.exception(f\"Failed to render {component}\")\n            new_state.model.current = {\n                \"tagName\": \"\",\n                \"error\": (\n                    f\"{type(error).__name__}: {error}\"\n                    if REACTPY_DEBUG_MODE.current\n                    else \"\"\n                ),\n            }\n        finally:\n            await life_cycle_hook.affect_component_did_render()\n\n        try:\n            parent = new_state.parent\n        except AttributeError:\n            pass  # only happens for root component\n        else:\n            key, index = new_state.key, new_state.index\n            parent.children_by_key[key] = new_state\n            # need to add this model to parent's children without mutating parent model\n            old_parent_model = parent.model.current\n            old_parent_children = old_parent_model[\"children\"]\n            parent.model.current = {\n                **old_parent_model,\n                \"children\": [\n                    *old_parent_children[:index],\n                    new_state.model.current,\n                    *old_parent_children[index + 1 :],\n                ],\n            }\n\n    async def _render_model(\n        self,\n        exit_stack: AsyncExitStack,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n        raw_model: Any,\n    ) -> None:\n        try:\n            new_state.model.current = {\"tagName\": raw_model[\"tagName\"]}\n        except Exception as e:  # nocov\n            msg = f\"Expected a VDOM element dict, not {raw_model}\"\n            raise ValueError(msg) from e\n        if \"key\" in raw_model:\n            new_state.key = new_state.model.current[\"key\"] = raw_model[\"key\"]\n        if \"importSource\" in raw_model:\n            new_state.model.current[\"importSource\"] = raw_model[\"importSource\"]\n        self._render_model_attributes(old_state, new_state, raw_model)\n        await self._render_model_children(\n            exit_stack, old_state, new_state, raw_model.get(\"children\", [])\n        )\n\n    def _render_model_attributes(\n        self,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n        raw_model: dict[str, Any],\n    ) -> None:\n        # extract event handlers from 'eventHandlers' and 'attributes'\n        handlers_by_event: EventHandlerDict = raw_model.get(\"eventHandlers\", {})\n\n        if \"attributes\" in raw_model:\n            attrs = raw_model[\"attributes\"].copy()\n            new_state.model.current[\"attributes\"] = attrs\n\n        if old_state is None:\n            self._render_model_event_handlers_without_old_state(\n                new_state, handlers_by_event\n            )\n            return None\n\n        for old_event in set(old_state.targets_by_event).difference(handlers_by_event):\n            old_target = old_state.targets_by_event[old_event]\n            del self._event_handlers[old_target]\n\n        if not handlers_by_event:\n            return None\n\n        model_event_handlers = new_state.model.current[\"eventHandlers\"] = {}\n        for event, handler in handlers_by_event.items():\n            if event in old_state.targets_by_event:\n                target = old_state.targets_by_event[event]\n            else:\n                target = uuid4().hex if handler.target is None else handler.target\n            new_state.targets_by_event[event] = target\n            self._event_handlers[target] = handler\n            model_event_handlers[event] = {\n                \"target\": target,\n                \"preventDefault\": handler.prevent_default,\n                \"stopPropagation\": handler.stop_propagation,\n            }\n\n        return None\n\n    def _render_model_event_handlers_without_old_state(\n        self,\n        new_state: _ModelState,\n        handlers_by_event: EventHandlerDict,\n    ) -> None:\n        if not handlers_by_event:\n            return None\n\n        model_event_handlers = new_state.model.current[\"eventHandlers\"] = {}\n        for event, handler in handlers_by_event.items():\n            target = uuid4().hex if handler.target is None else handler.target\n            new_state.targets_by_event[event] = target\n            self._event_handlers[target] = handler\n            model_event_handlers[event] = {\n                \"target\": target,\n                \"preventDefault\": handler.prevent_default,\n                \"stopPropagation\": handler.stop_propagation,\n            }\n\n        return None\n\n    async def _render_model_children(\n        self,\n        exit_stack: AsyncExitStack,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n        raw_children: Any,\n    ) -> None:\n        if not isinstance(raw_children, (list, tuple)):\n            raw_children = [raw_children]\n\n        if old_state is None:\n            if raw_children:\n                await self._render_model_children_without_old_state(\n                    exit_stack, new_state, raw_children\n                )\n            return None\n        elif not raw_children:\n            await self._unmount_model_states(list(old_state.children_by_key.values()))\n            return None\n\n        children_info = _get_children_info(raw_children)\n\n        new_keys = {k for _, _, k in children_info}\n        if len(new_keys) != len(children_info):\n            key_counter = Counter(item[2] for item in children_info)\n            duplicate_keys = [key for key, count in key_counter.items() if count > 1]\n            msg = f\"Duplicate keys {duplicate_keys} at {new_state.patch_path or '/'!r}\"\n            raise ValueError(msg)\n\n        old_keys = set(old_state.children_by_key).difference(new_keys)\n        if old_keys:\n            await self._unmount_model_states(\n                [old_state.children_by_key[key] for key in old_keys]\n            )\n\n        new_state.model.current[\"children\"] = []\n        for index, (child, child_type, key) in enumerate(children_info):\n            old_child_state = old_state.children_by_key.get(key)\n            if child_type is _DICT_TYPE:\n                old_child_state = old_state.children_by_key.get(key)\n                if old_child_state is None:\n                    new_child_state = _make_element_model_state(\n                        new_state,\n                        index,\n                        key,\n                    )\n                elif old_child_state.is_component_state:\n                    await self._unmount_model_states([old_child_state])\n                    new_child_state = _make_element_model_state(\n                        new_state,\n                        index,\n                        key,\n                    )\n                    old_child_state = None\n                else:\n                    new_child_state = _update_element_model_state(\n                        old_child_state,\n                        new_state,\n                        index,\n                    )\n                await self._render_model(\n                    exit_stack, old_child_state, new_child_state, child\n                )\n                new_state.append_child(new_child_state.model.current)\n                new_state.children_by_key[key] = new_child_state\n            elif child_type is _COMPONENT_TYPE:\n                child = cast(ComponentType, child)\n                old_child_state = old_state.children_by_key.get(key)\n                if old_child_state is None:\n                    new_child_state = _make_component_model_state(\n                        new_state,\n                        index,\n                        key,\n                        child,\n                        self._schedule_render_task,\n                    )\n                elif old_child_state.is_component_state and (\n                    old_child_state.life_cycle_state.component.type != child.type\n                ):\n                    await self._unmount_model_states([old_child_state])\n                    old_child_state = None\n                    new_child_state = _make_component_model_state(\n                        new_state,\n                        index,\n                        key,\n                        child,\n                        self._schedule_render_task,\n                    )\n                else:\n                    new_child_state = _update_component_model_state(\n                        old_child_state,\n                        new_state,\n                        index,\n                        child,\n                        self._schedule_render_task,\n                    )\n                await self._render_component(\n                    exit_stack, old_child_state, new_child_state, child\n                )\n            else:\n                old_child_state = old_state.children_by_key.get(key)\n                if old_child_state is not None:\n                    await self._unmount_model_states([old_child_state])\n                new_state.append_child(child)\n\n    async def _render_model_children_without_old_state(\n        self,\n        exit_stack: AsyncExitStack,\n        new_state: _ModelState,\n        raw_children: list[Any],\n    ) -> None:\n        children_info = _get_children_info(raw_children)\n\n        new_keys = {k for _, _, k in children_info}\n        if len(new_keys) != len(children_info):\n            key_counter = Counter(k for _, _, k in children_info)\n            duplicate_keys = [key for key, count in key_counter.items() if count > 1]\n            msg = f\"Duplicate keys {duplicate_keys} at {new_state.patch_path or '/'!r}\"\n            raise ValueError(msg)\n\n        new_state.model.current[\"children\"] = []\n        for index, (child, child_type, key) in enumerate(children_info):\n            if child_type is _DICT_TYPE:\n                child_state = _make_element_model_state(new_state, index, key)\n                await self._render_model(exit_stack, None, child_state, child)\n                new_state.append_child(child_state.model.current)\n                new_state.children_by_key[key] = child_state\n            elif child_type is _COMPONENT_TYPE:\n                child_state = _make_component_model_state(\n                    new_state, index, key, child, self._schedule_render_task\n                )\n                await self._render_component(exit_stack, None, child_state, child)\n            else:\n                new_state.append_child(child)\n\n    async def _unmount_model_states(self, old_states: list[_ModelState]) -> None:\n        to_unmount = old_states[::-1]  # unmount in reversed order of rendering\n        while to_unmount:\n            model_state = to_unmount.pop()\n\n            for target in model_state.targets_by_event.values():\n                del self._event_handlers[target]\n\n            if model_state.is_component_state:\n                life_cycle_state = model_state.life_cycle_state\n                del self._model_states_by_life_cycle_state_id[life_cycle_state.id]\n                await life_cycle_state.hook.affect_component_will_unmount()\n\n            to_unmount.extend(model_state.children_by_key.values())\n\n    def _schedule_render_task(self, lcs_id: _LifeCycleStateId) -> None:\n        if not REACTPY_ASYNC_RENDERING.current:\n            self._rendering_queue.put(lcs_id)\n            return None\n        try:\n            model_state = self._model_states_by_life_cycle_state_id[lcs_id]\n        except KeyError:\n            logger.debug(\n                \"Did not render component with model state ID \"\n                f\"{lcs_id!r} - component already unmounted\"\n            )\n        else:\n            self._render_tasks.add(create_task(self._create_layout_update(model_state)))\n            self._render_tasks_ready.release()\n\n    def __repr__(self) -> str:\n        return f\"{type(self).__name__}({self.root})\"\n\n\ndef _new_root_model_state(\n    component: ComponentType, schedule_render: Callable[[_LifeCycleStateId], None]\n) -> _ModelState:\n    return _ModelState(\n        parent=None,\n        index=-1,\n        key=None,\n        model=Ref(),\n        patch_path=\"\",\n        children_by_key={},\n        targets_by_event={},\n        life_cycle_state=_make_life_cycle_state(component, schedule_render),\n    )\n\n\ndef _make_component_model_state(\n    parent: _ModelState,\n    index: int,\n    key: Any,\n    component: ComponentType,\n    schedule_render: Callable[[_LifeCycleStateId], None],\n) -> _ModelState:\n    return _ModelState(\n        parent=parent,\n        index=index,\n        key=key,\n        model=Ref(),\n        patch_path=f\"{parent.patch_path}/children/{index}\",\n        children_by_key={},\n        targets_by_event={},\n        life_cycle_state=_make_life_cycle_state(component, schedule_render),\n    )\n\n\ndef _copy_component_model_state(old_model_state: _ModelState) -> _ModelState:\n    # use try/except here because not having a parent is rare (only the root state)\n    try:\n        parent: _ModelState | None = old_model_state.parent\n    except AttributeError:\n        parent = None\n\n    return _ModelState(\n        parent=parent,\n        index=old_model_state.index,\n        key=old_model_state.key,\n        model=Ref(),  # does not copy the model\n        patch_path=old_model_state.patch_path,\n        children_by_key={},\n        targets_by_event={},\n        life_cycle_state=old_model_state.life_cycle_state,\n    )\n\n\ndef _update_component_model_state(\n    old_model_state: _ModelState,\n    new_parent: _ModelState,\n    new_index: int,\n    new_component: ComponentType,\n    schedule_render: Callable[[_LifeCycleStateId], None],\n) -> _ModelState:\n    return _ModelState(\n        parent=new_parent,\n        index=new_index,\n        key=old_model_state.key,\n        model=Ref(),  # does not copy the model\n        patch_path=f\"{new_parent.patch_path}/children/{new_index}\",\n        children_by_key={},\n        targets_by_event={},\n        life_cycle_state=(\n            _update_life_cycle_state(old_model_state.life_cycle_state, new_component)\n            if old_model_state.is_component_state\n            else _make_life_cycle_state(new_component, schedule_render)\n        ),\n    )\n\n\ndef _make_element_model_state(\n    parent: _ModelState,\n    index: int,\n    key: Any,\n) -> _ModelState:\n    return _ModelState(\n        parent=parent,\n        index=index,\n        key=key,\n        model=Ref(),\n        patch_path=f\"{parent.patch_path}/children/{index}\",\n        children_by_key={},\n        targets_by_event={},\n    )\n\n\ndef _update_element_model_state(\n    old_model_state: _ModelState,\n    new_parent: _ModelState,\n    new_index: int,\n) -> _ModelState:\n    return _ModelState(\n        parent=new_parent,\n        index=new_index,\n        key=old_model_state.key,\n        model=Ref(),  # does not copy the model\n        patch_path=old_model_state.patch_path,\n        children_by_key={},\n        targets_by_event={},\n    )\n\n\nclass _ModelState:\n    \"\"\"State that is bound to a particular element within the layout\"\"\"\n\n    __slots__ = (\n        \"__weakref__\",\n        \"_parent_ref\",\n        \"_render_semaphore\",\n        \"children_by_key\",\n        \"index\",\n        \"key\",\n        \"life_cycle_state\",\n        \"model\",\n        \"patch_path\",\n        \"targets_by_event\",\n    )\n\n    def __init__(\n        self,\n        parent: _ModelState | None,\n        index: int,\n        key: Any,\n        model: Ref[VdomJson],\n        patch_path: str,\n        children_by_key: dict[Key, _ModelState],\n        targets_by_event: dict[str, str],\n        life_cycle_state: _LifeCycleState | None = None,\n    ):\n        self.index = index\n        \"\"\"The index of the element amongst its siblings\"\"\"\n\n        self.key = key\n        \"\"\"A key that uniquely identifies the element amongst its siblings\"\"\"\n\n        self.model = model\n        \"\"\"The actual model of the element\"\"\"\n\n        self.patch_path = patch_path\n        \"\"\"A \"/\" delimited path to the element within the greater layout\"\"\"\n\n        self.children_by_key = children_by_key\n        \"\"\"Child model states indexed by their unique keys\"\"\"\n\n        self.targets_by_event = targets_by_event\n        \"\"\"The element's event handler target strings indexed by their event name\"\"\"\n\n        # === Conditionally Available Attributes ===\n        # It's easier to conditionally assign than to force a null check on every usage\n\n        if parent is not None:\n            self._parent_ref = weakref(parent)\n            \"\"\"The parent model state\"\"\"\n\n        if life_cycle_state is not None:\n            self.life_cycle_state = life_cycle_state\n            \"\"\"The state for the element's component (if it has one)\"\"\"\n\n    @property\n    def is_component_state(self) -> bool:\n        return hasattr(self, \"life_cycle_state\")\n\n    @property\n    def parent(self) -> _ModelState:\n        parent = self._parent_ref()\n        if parent is None:\n            raise RuntimeError(\"detached model state\")  # nocov\n        return parent\n\n    def append_child(self, child: Any) -> None:\n        self.model.current[\"children\"].append(child)\n\n    def __repr__(self) -> str:  # nocov\n        return f\"ModelState({ {s: getattr(self, s, None) for s in self.__slots__} })\"\n\n\ndef _make_life_cycle_state(\n    component: ComponentType,\n    schedule_render: Callable[[_LifeCycleStateId], None],\n) -> _LifeCycleState:\n    life_cycle_state_id = _LifeCycleStateId(uuid4().hex)\n    return _LifeCycleState(\n        life_cycle_state_id,\n        LifeCycleHook(lambda: schedule_render(life_cycle_state_id)),\n        component,\n    )\n\n\ndef _update_life_cycle_state(\n    old_life_cycle_state: _LifeCycleState,\n    new_component: ComponentType,\n) -> _LifeCycleState:\n    return _LifeCycleState(\n        old_life_cycle_state.id,\n        # the hook is preserved across renders because it holds the state\n        old_life_cycle_state.hook,\n        new_component,\n    )\n\n\n_LifeCycleStateId = NewType(\"_LifeCycleStateId\", str)\n\n\nclass _LifeCycleState(NamedTuple):\n    \"\"\"Component state for :class:`_ModelState`\"\"\"\n\n    id: _LifeCycleStateId\n    \"\"\"A unique identifier used in the :class:`~reactpy.core.hooks.LifeCycleHook` callback\"\"\"\n\n    hook: LifeCycleHook\n    \"\"\"The life cycle hook\"\"\"\n\n    component: ComponentType\n    \"\"\"The current component instance\"\"\"\n\n\n_Type = TypeVar(\"_Type\")\n\n\nclass _ThreadSafeQueue(Generic[_Type]):\n    def __init__(self) -> None:\n        self._loop = get_running_loop()\n        self._queue: Queue[_Type] = Queue()\n        self._pending: set[_Type] = set()\n\n    def put(self, value: _Type) -> None:\n        if value not in self._pending:\n            self._pending.add(value)\n            self._loop.call_soon_threadsafe(self._queue.put_nowait, value)\n\n    async def get(self) -> _Type:\n        value = await self._queue.get()\n        self._pending.remove(value)\n        return value\n\n\ndef _get_children_info(children: list[VdomChild]) -> Sequence[_ChildInfo]:\n    infos: list[_ChildInfo] = []\n    for index, child in enumerate(children):\n        if child is None:\n            continue\n        elif isinstance(child, dict):\n            child_type = _DICT_TYPE\n            key = child.get(\"key\")\n        elif isinstance(child, ComponentType):\n            child_type = _COMPONENT_TYPE\n            key = child.key\n        else:\n            child = f\"{child}\"\n            child_type = _STRING_TYPE\n            key = None\n\n        if key is None:\n            key = index\n\n        infos.append((child, child_type, key))\n\n    return infos\n\n\n_ChildInfo: TypeAlias = tuple[Any, \"_ElementType\", Key]\n\n# used in _process_child_type_and_key\n_ElementType = NewType(\"_ElementType\", int)\n_DICT_TYPE = _ElementType(1)\n_COMPONENT_TYPE = _ElementType(2)\n_STRING_TYPE = _ElementType(3)\n\n# Path: src/py/reactpy/reactpy/html.py\n\"\"\"\n\n**Fragment**\n\n- :func:`_`\n\n**Document metadata**\n\n- :func:`base`\n- :func:`head`\n- :func:`link`\n- :func:`meta`\n- :func:`style`\n- :func:`title`\n\n**Content sectioning**\n\n- :func:`address`\n- :func:`article`\n- :func:`aside`\n- :func:`footer`\n- :func:`header`\n- :func:`h1`\n- :func:`h2`\n- :func:`h3`\n- :func:`h4`\n- :func:`h5`\n- :func:`h6`\n- :func:`main`\n- :func:`nav`\n- :func:`section`\n\n**Text content**\n\n- :func:`blockquote`\n- :func:`dd`\n- :func:`div`\n- :func:`dl`\n- :func:`dt`\n- :func:`figcaption`\n- :func:`figure`\n- :func:`hr`\n- :func:`li`\n- :func:`ol`\n- :func:`p`\n- :func:`pre`\n- :func:`ul`\n\n**Inline text semantics**\n\n- :func:`a`\n- :func:`abbr`\n- :func:`b`\n- :func:`bdi`\n- :func:`bdo`\n- :func:`br`\n- :func:`cite`\n- :func:`code`\n- :func:`data`\n- :func:`em`\n- :func:`i`\n- :func:`kbd`\n- :func:`mark`\n- :func:`q`\n- :func:`rp`\n- :func:`rt`\n- :func:`ruby`\n- :func:`s`\n- :func:`samp`\n- :func:`small`\n- :func:`span`\n- :func:`strong`\n- :func:`sub`\n- :func:`sup`\n- :func:`time`\n- :func:`u`\n- :func:`var`\n- :func:`wbr`\n\n**Image and video**\n\n- :func:`area`\n- :func:`audio`\n- :func:`img`\n- :func:`map`\n- :func:`track`\n- :func:`video`\n\n**Embedded content**\n\n- :func:`embed`\n- :func:`iframe`\n- :func:`object`\n- :func:`param`\n- :func:`picture`\n- :func:`portal`\n- :func:`source`\n\n**SVG and MathML**\n\n- :func:`svg`\n- :func:`math`\n\n**Scripting**\n\n- :func:`canvas`\n- :func:`noscript`\n- :func:`script`\n\n**Demarcating edits**\n\n- :func:`del_`\n- :func:`ins`\n\n**Table content**\n\n- :func:`caption`\n- :func:`col`\n- :func:`colgroup`\n- :func:`table`\n- :func:`tbody`\n- :func:`td`\n- :func:`tfoot`\n- :func:`th`\n- :func:`thead`\n- :func:`tr`\n\n**Forms**\n\n- :func:`button`\n- :func:`fieldset`\n- :func:`form`\n- :func:`input`\n- :func:`label`\n- :func:`legend`\n- :func:`meter`\n- :func:`option`\n- :func:`output`\n- :func:`progress`\n- :func:`select`\n- :func:`textarea`\n\n**Interactive elements**\n\n- :func:`details`\n- :func:`dialog`\n- :func:`menu`\n- :func:`menuitem`\n- :func:`summary`\n\n**Web components**\n\n- :func:`slot`\n- :func:`template`\n\n.. autofunction:: _\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom collections.abc import Sequence\n\nfrom reactpy.core.types import (\n    EventHandlerDict,\n    Key,\n    VdomAttributes,\n    VdomChild,\n    VdomDict,\n)\nfrom reactpy.core.vdom import custom_vdom_constructor, make_vdom_constructor\n\n__all__ = (\n    \"_\",\n    \"a\",\n    \"abbr\",\n    \"address\",\n    \"area\",\n    \"article\",\n    \"aside\",\n    \"audio\",\n    \"b\",\n    \"base\",\n    \"bdi\",\n    \"bdo\",\n    \"blockquote\",\n    \"br\",\n    \"button\",\n    \"canvas\",\n    \"caption\",\n    \"cite\",\n    \"code\",\n    \"col\",\n    \"colgroup\",\n    \"data\",\n    \"dd\",\n    \"del_\",\n    \"details\",\n    \"dialog\",\n    \"div\",\n    \"dl\",\n    \"dt\",\n    \"em\",\n    \"embed\",\n    \"fieldset\",\n    \"figcaption\",\n    \"figure\",\n    \"footer\",\n    \"form\",\n    \"h1\",\n    \"h2\",\n    \"h3\",\n    \"h4\",\n    \"h5\",\n    \"h6\",\n    \"head\",\n    \"header\",\n    \"hr\",\n    \"i\",\n    \"iframe\",\n    \"img\",\n    \"input\",\n    \"ins\",\n    \"kbd\",\n    \"label\",\n    \"legend\",\n    \"li\",\n    \"link\",\n    \"main\",\n    \"map\",\n    \"mark\",\n    \"math\",\n    \"menu\",\n    \"menuitem\",\n    \"meta\",\n    \"meter\",\n    \"nav\",\n    \"noscript\",\n    \"object\",\n    \"ol\",\n    \"option\",\n    \"output\",\n    \"p\",\n    \"param\",\n    \"picture\",\n    \"portal\",\n    \"pre\",\n    \"progress\",\n    \"q\",\n    \"rp\",\n    \"rt\",\n    \"ruby\",\n    \"s\",\n    \"samp\",\n    \"script\",\n    \"section\",\n    \"select\",\n    \"slot\",\n    \"small\",\n    \"source\",\n    \"span\",\n    \"strong\",\n    \"style\",\n    \"sub\",\n    \"summary\",\n    \"sup\",\n    \"svg\",\n    \"table\",\n    \"tbody\",\n    \"td\",\n    \"template\",\n    \"textarea\",\n    \"tfoot\",\n    \"th\",\n    \"thead\",\n    \"time\",\n    \"title\",\n    \"tr\",\n    \"track\",\n    \"u\",\n    \"ul\",\n    \"var\",\n    \"video\",\n    \"wbr\",\n)\n\n\ndef _fragment(\n    attributes: VdomAttributes,\n    children: Sequence[VdomChild],\n    key: Key | None,\n    event_handlers: EventHandlerDict,\n) -> VdomDict:\n    \"\"\"An HTML fragment - this element will not appear in the DOM\"\"\"\n    if attributes or event_handlers:\n        msg = \"Fragments cannot have attributes besides 'key'\"\n        raise TypeError(msg)\n    model: VdomDict = {\"tagName\": \"\"}\n\n    if children:\n        model[\"children\"] = children\n\n    if key is not None:\n        model[\"key\"] = key\n\n    return model\n\n\n# FIXME: https://github.com/PyCQA/pylint/issues/5784\n_ = custom_vdom_constructor(_fragment)\n\n\n# Document metadata\nbase = make_vdom_constructor(\"base\")\nhead = make_vdom_constructor(\"head\")\nlink = make_vdom_constructor(\"link\")\nmeta = make_vdom_constructor(\"meta\")\nstyle = make_vdom_constructor(\"style\")\ntitle = make_vdom_constructor(\"title\")\n\n# Content sectioning\naddress = make_vdom_constructor(\"address\")\narticle = make_vdom_constructor(\"article\")\naside = make_vdom_constructor(\"aside\")\nfooter = make_vdom_constructor(\"footer\")\nheader = make_vdom_constructor(\"header\")\nh1 = make_vdom_constructor(\"h1\")\nh2 = make_vdom_constructor(\"h2\")\nh3 = make_vdom_constructor(\"h3\")\nh4 = make_vdom_constructor(\"h4\")\nh5 = make_vdom_constructor(\"h5\")\nh6 = make_vdom_constructor(\"h6\")\nmain = make_vdom_constructor(\"main\")\nnav = make_vdom_constructor(\"nav\")\nsection = make_vdom_constructor(\"section\")\n\n# Text content\nblockquote = make_vdom_constructor(\"blockquote\")\ndd = make_vdom_constructor(\"dd\")\ndiv = make_vdom_constructor(\"div\")\ndl = make_vdom_constructor(\"dl\")\ndt = make_vdom_constructor(\"dt\")\nfigcaption = make_vdom_constructor(\"figcaption\")\nfigure = make_vdom_constructor(\"figure\")\nhr = make_vdom_constructor(\"hr\", allow_children=False)\nli = make_vdom_constructor(\"li\")\nol = make_vdom_constructor(\"ol\")\np = make_vdom_constructor(\"p\")\npre = make_vdom_constructor(\"pre\")\nul = make_vdom_constructor(\"ul\")\n\n# Inline text semantics\na = make_vdom_constructor(\"a\")\nabbr = make_vdom_constructor(\"abbr\")\nb = make_vdom_constructor(\"b\")\nbdi = make_vdom_constructor(\"bdi\")\nbdo = make_vdom_constructor(\"bdo\")\nbr = make_vdom_constructor(\"br\", allow_children=False)\ncite = make_vdom_constructor(\"cite\")\ncode = make_vdom_constructor(\"code\")\ndata = make_vdom_constructor(\"data\")\nem = make_vdom_constructor(\"em\")\ni = make_vdom_constructor(\"i\")\nkbd = make_vdom_constructor(\"kbd\")\nmark = make_vdom_constructor(\"mark\")\nq = make_vdom_constructor(\"q\")\nrp = make_vdom_constructor(\"rp\")\nrt = make_vdom_constructor(\"rt\")\nruby = make_vdom_constructor(\"ruby\")\ns = make_vdom_constructor(\"s\")\nsamp = make_vdom_constructor(\"samp\")\nsmall = make_vdom_constructor(\"small\")\nspan = make_vdom_constructor(\"span\")\nstrong = make_vdom_constructor(\"strong\")\nsub = make_vdom_constructor(\"sub\")\nsup = make_vdom_constructor(\"sup\")\ntime = make_vdom_constructor(\"time\")\nu = make_vdom_constructor(\"u\")\nvar = make_vdom_constructor(\"var\")\nwbr = make_vdom_constructor(\"wbr\")\n\n# Image and video\narea = make_vdom_constructor(\"area\", allow_children=False)\naudio = make_vdom_constructor(\"audio\")\n...\n", "needle_token_start": 739, "needle_token_end": 847, "code_context_ntokens": 16057, "output": ["```python\ndef sync_function() -> _EffectCleanFunc | None:\n    task = asyncio.create_task(async_function())\n\n    def clean_future() -> None:\n        if not task.cancel():\n            try:\n                clean = task.result()\n            except asyncio.CancelledError:\n                pass\n            else:\n                if clean is not None:\n                    clean()\n\n    return clean_future\n```"]}
{"repo": "reactive-python/reactpy", "name": "custom_vdom_constructor", "language": "python", "path": "src/py/reactpy/reactpy/core/vdom.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To transform a given function into a specialized constructor that creates virtual DOM (VDOM) dictionaries, ensuring that the function adheres to a specific interface for handling attributes, children, keys, and event handlers.\n2. **Input**: The input is a function that accepts four parameters: attributes (a dictionary), children (a list), a unique key (string or None), and event handlers (a dictionary).\n3. **Output**: The output is a new function that conforms to the VDOM dictionary constructor interface, capable of creating structured VDOM dictionaries based on the provided inputs.\n4. **Procedure**: The procedure involves:\n   - Wrapping the given function with another function that preprocesses the input by separating attributes and children, extracting the unique key, and segregating event handlers.\n   - The original function is then called with these processed inputs to generate a VDOM dictionary.\n   - The wrapper ensures that the function conforms to the expected interface for creating VDOM elements.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/core/events.py\nfrom __future__ import annotations\n\nimport asyncio\n...\n# Path: src/py/reactpy/reactpy/core/vdom.py\nfrom __future__ import annotations\n\nimport json\nfrom collections.abc import Mapping, Sequence\nfrom functools import wraps\nfrom typing import Any, Protocol, cast, overload\n\nfrom fastjsonschema import compile as compile_json_schema\n\nfrom reactpy._warnings import warn\nfrom reactpy.config import REACTPY_CHECK_JSON_ATTRS, REACTPY_DEBUG_MODE\nfrom reactpy.core._f_back import f_module_name\nfrom reactpy.core.events import EventHandler, to_event_handler_function\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomDictConstructor,\n    VdomJson,\n)\n\nVDOM_JSON_SCHEMA = {\n    \"$schema\": \"http://json-schema.org/draft-07/schema\",\n    \"$ref\": \"#/definitions/element\",\n    \"definitions\": {\n        \"element\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"tagName\": {\"type\": \"string\"},\n                \"key\": {\"type\": [\"string\", \"number\", \"null\"]},\n                \"error\": {\"type\": \"string\"},\n                \"children\": {\"$ref\": \"#/definitions/elementChildren\"},\n                \"attributes\": {\"type\": \"object\"},\n                \"eventHandlers\": {\"$ref\": \"#/definitions/elementEventHandlers\"},\n                \"importSource\": {\"$ref\": \"#/definitions/importSource\"},\n            },\n            # The 'tagName' is required because its presence is a useful indicator of\n            # whether a dictionary describes a VDOM model or not.\n            \"required\": [\"tagName\"],\n            \"dependentSchemas\": {\n                # When 'error' is given, the 'tagName' should be empty.\n                \"error\": {\"properties\": {\"tagName\": {\"maxLength\": 0}}}\n            },\n        },\n        \"elementChildren\": {\n            \"type\": \"array\",\n            \"items\": {\"$ref\": \"#/definitions/elementOrString\"},\n        },\n        \"elementEventHandlers\": {\n            \"type\": \"object\",\n            \"patternProperties\": {\n                \".*\": {\"$ref\": \"#/definitions/eventHandler\"},\n            },\n        },\n        \"eventHandler\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"target\": {\"type\": \"string\"},\n                \"preventDefault\": {\"type\": \"boolean\"},\n                \"stopPropagation\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"target\"],\n        },\n        \"importSource\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"source\": {\"type\": \"string\"},\n                \"sourceType\": {\"enum\": [\"URL\", \"NAME\"]},\n                \"fallback\": {\n                    \"type\": [\"object\", \"string\", \"null\"],\n                    \"if\": {\"not\": {\"type\": \"null\"}},\n                    \"then\": {\"$ref\": \"#/definitions/elementOrString\"},\n                },\n                \"unmountBeforeUpdate\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"source\"],\n        },\n        \"elementOrString\": {\n            \"type\": [\"object\", \"string\"],\n            \"if\": {\"type\": \"object\"},\n            \"then\": {\"$ref\": \"#/definitions/element\"},\n        },\n    },\n}\n\"\"\"JSON Schema describing serialized VDOM - see :ref:`VDOM` for more info\"\"\"\n\n\n# we can't add a docstring to this because Sphinx doesn't know how to find its source\n_COMPILED_VDOM_VALIDATOR = compile_json_schema(VDOM_JSON_SCHEMA)\n\n\ndef validate_vdom_json(value: Any) -> VdomJson:\n    \"\"\"Validate serialized VDOM - see :attr:`VDOM_JSON_SCHEMA` for more info\"\"\"\n    _COMPILED_VDOM_VALIDATOR(value)\n    return cast(VdomJson, value)\n\n\ndef is_vdom(value: Any) -> bool:\n    \"\"\"Return whether a value is a :class:`VdomDict`\n\n    This employs a very simple heuristic - something is VDOM if:\n\n    1. It is a ``dict`` instance\n    2. It contains the key ``\"tagName\"``\n    3. The value of the key ``\"tagName\"`` is a string\n\n    .. note::\n\n        Performing an ``isinstance(value, VdomDict)`` check is too restrictive since the\n        user would be forced to import ``VdomDict`` every time they needed to declare a\n        VDOM element. Giving the user more flexibility, at the cost of this check's\n        accuracy, is worth it.\n    \"\"\"\n    return (\n        isinstance(value, dict)\n        and \"tagName\" in value\n        and isinstance(value[\"tagName\"], str)\n    )\n\n\n@overload\ndef vdom(tag: str, *children: VdomChildren) -> VdomDict: ...\n\n\n@overload\ndef vdom(tag: str, attributes: VdomAttributes, *children: VdomChildren) -> VdomDict: ...\n\n\ndef vdom(\n    tag: str,\n    *attributes_and_children: Any,\n    **kwargs: Any,\n) -> VdomDict:\n    \"\"\"A helper function for creating VDOM elements.\n\n    Parameters:\n        tag:\n            The type of element (e.g. 'div', 'h1', 'img')\n        attributes_and_children:\n            An optional attribute mapping followed by any number of children or\n            iterables of children. The attribute mapping **must** precede the children,\n            or children which will be merged into their respective parts of the model.\n        key:\n            A string indicating the identity of a particular element. This is significant\n            to preserve event handlers across updates - without a key, a re-render would\n            cause these handlers to be deleted, but with a key, they would be redirected\n            to any newly defined handlers.\n        event_handlers:\n            Maps event types to coroutines that are responsible for handling those events.\n        import_source:\n            (subject to change) specifies javascript that, when evaluated returns a\n            React component.\n    \"\"\"\n    if kwargs:  # nocov\n        if \"key\" in kwargs:\n            if attributes_and_children:\n                maybe_attributes, *children = attributes_and_children\n                if _is_attributes(maybe_attributes):\n                    attributes_and_children = (\n                        {**maybe_attributes, \"key\": kwargs.pop(\"key\")},\n                        *children,\n                    )\n                else:\n                    attributes_and_children = (\n                        {\"key\": kwargs.pop(\"key\")},\n                        maybe_attributes,\n                        *children,\n                    )\n            else:\n                attributes_and_children = ({\"key\": kwargs.pop(\"key\")},)\n            warn(\n                \"An element's 'key' must be declared in an attribute dict instead \"\n                \"of as a keyword argument. This will error in a future version.\",\n                DeprecationWarning,\n            )\n\n        if kwargs:\n            msg = f\"Extra keyword arguments {kwargs}\"\n            raise ValueError(msg)\n\n    model: VdomDict = {\"tagName\": tag}\n\n    if not attributes_and_children:\n        return model\n\n    attributes, children = separate_attributes_and_children(attributes_and_children)\n    key = attributes.pop(\"key\", None)\n    attributes, event_handlers = separate_attributes_and_event_handlers(attributes)\n\n    if attributes:\n        if REACTPY_CHECK_JSON_ATTRS.current:\n            json.dumps(attributes)\n        model[\"attributes\"] = attributes\n\n    if children:\n        model[\"children\"] = children\n\n    if key is not None:\n        model[\"key\"] = key\n\n    if event_handlers:\n        model[\"eventHandlers\"] = event_handlers\n\n    return model\n\n\ndef make_vdom_constructor(\n    tag: str, allow_children: bool = True, import_source: ImportSourceDict | None = None\n) -> VdomDictConstructor:\n    \"\"\"Return a constructor for VDOM dictionaries with the given tag name.\n\n    The resulting callable will have the same interface as :func:`vdom` but without its\n    first ``tag`` argument.\n    \"\"\"\n\n    def constructor(*attributes_and_children: Any, **kwargs: Any) -> VdomDict:\n        model = vdom(tag, *attributes_and_children, **kwargs)\n        if not allow_children and \"children\" in model:\n            msg = f\"{tag!r} nodes cannot have children.\"\n            raise TypeError(msg)\n        if import_source:\n            model[\"importSource\"] = import_source\n        return model\n\n    # replicate common function attributes\n    constructor.__name__ = tag\n    constructor.__doc__ = (\n        \"Return a new \"\n        f\"`<{tag}> <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/{tag}>`__ \"\n        \"element represented by a :class:`VdomDict`.\"\n    )\n\n    module_name = f_module_name(1)\n    if module_name:\n        constructor.__module__ = module_name\n        constructor.__qualname__ = f\"{module_name}.{tag}\"\n\n    return cast(VdomDictConstructor, constructor)\n\n\n\ndef custom_vdom_constructor(func: _CustomVdomDictConstructor) -> VdomDictConstructor:\n    \"\"\"Cast function to VdomDictConstructor\"\"\"\n\n    @wraps(func)\n    def wrapper(*attributes_and_children: Any) -> VdomDict:\n        attributes, children = separate_attributes_and_children(attributes_and_children)\n        key = attributes.pop(\"key\", None)\n        attributes, event_handlers = separate_attributes_and_event_handlers(attributes)\n        return func(attributes, children, key, event_handlers)\n\n    return cast(VdomDictConstructor, wrapper)\n\n\ndef separate_attributes_and_children(\n    values: Sequence[Any],\n) -> tuple[dict[str, Any], list[Any]]:\n    if not values:\n        return {}, []\n\n    attributes: dict[str, Any]\n    children_or_iterables: Sequence[Any]\n    if _is_attributes(values[0]):\n        attributes, *children_or_iterables = values\n    else:\n        attributes = {}\n        children_or_iterables = values\n\n    children: list[Any] = []\n    for child in children_or_iterables:\n        if _is_single_child(child):\n            children.append(child)\n        else:\n            children.extend(child)\n\n    return attributes, children\n\n\ndef separate_attributes_and_event_handlers(\n    attributes: Mapping[str, Any]\n) -> tuple[dict[str, Any], EventHandlerDict]:\n    separated_attributes = {}\n    separated_event_handlers: dict[str, EventHandlerType] = {}\n\n    for k, v in attributes.items():\n        handler: EventHandlerType\n\n        if callable(v):\n            handler = EventHandler(to_event_handler_function(v))\n        elif (\n            # isinstance check on protocols is slow - use function attr pre-check as a\n            # quick filter before actually performing slow EventHandlerType type check\n            hasattr(v, \"function\")\n            and isinstance(v, EventHandlerType)\n        ):\n            handler = v\n        else:\n            separated_attributes[k] = v\n            continue\n\n        separated_event_handlers[k] = handler\n\n    return separated_attributes, dict(separated_event_handlers.items())\n\n\ndef _is_attributes(value: Any) -> bool:\n    return isinstance(value, Mapping) and \"tagName\" not in value\n\n\ndef _is_single_child(value: Any) -> bool:\n    if isinstance(value, (str, Mapping)) or not hasattr(value, \"__iter__\"):\n        return True\n    if REACTPY_DEBUG_MODE.current:\n        _validate_child_key_integrity(value)\n    return False\n\n\ndef _validate_child_key_integrity(value: Any) -> None:\n    if hasattr(value, \"__iter__\") and not hasattr(value, \"__len__\"):\n        warn(\n            f\"Did not verify key-path integrity of children in generator {value} \"\n            \"- pass a sequence (i.e. list of finite length) in order to verify\"\n        )\n    else:\n        for child in value:\n            if isinstance(child, ComponentType) and child.key is None:\n                warn(f\"Key not specified for child in list {child}\", UserWarning)\n            elif isinstance(child, Mapping) and \"key\" not in child:\n                # remove 'children' to reduce log spam\n                child_copy = {**child, \"children\": _EllipsisRepr()}\n                warn(f\"Key not specified for child in list {child_copy}\", UserWarning)\n\n\nclass _CustomVdomDictConstructor(Protocol):\n    def __call__(\n        self,\n        attributes: VdomAttributes,\n        children: Sequence[VdomChild],\n        key: Key | None,\n        event_handlers: EventHandlerDict,\n    ) -> VdomDict: ...\n\n\nclass _EllipsisRepr:\n    def __repr__(self) -> str:\n        return \"...\"\n\n# Path: src/py/reactpy/reactpy/utils.py\nfrom __future__ import annotations\n\nimport re\nfrom collections.abc import Iterable\nfrom itertools import chain\nfrom typing import Any, Callable, Generic, TypeVar, cast\n\nfrom lxml import etree\nfrom lxml.html import fromstring, tostring\n\nfrom reactpy.core.types import VdomDict\nfrom reactpy.core.vdom import vdom\n\n_RefValue = TypeVar(\"_RefValue\")\n_ModelTransform = Callable[[VdomDict], Any]\n_UNDEFINED: Any = object()\n\n\nclass Ref(Generic[_RefValue]):\n    \"\"\"Hold a reference to a value\n\n    This is used in imperative code to mutate the state of this object in order to\n    incur side effects. Generally refs should be avoided if possible, but sometimes\n    they are required.\n\n    Notes:\n        You can compare the contents for two ``Ref`` objects using the ``==`` operator.\n    \"\"\"\n\n    __slots__ = (\"current\",)\n\n    def __init__(self, initial_value: _RefValue = _UNDEFINED) -> None:\n        if initial_value is not _UNDEFINED:\n            self.current = initial_value\n            \"\"\"The present value\"\"\"\n\n    def set_current(self, new: _RefValue) -> _RefValue:\n        \"\"\"Set the current value and return what is now the old value\n\n        This is nice to use in ``lambda`` functions.\n        \"\"\"\n        old = self.current\n        self.current = new\n        return old\n\n    def __eq__(self, other: Any) -> bool:\n        try:\n            return isinstance(other, Ref) and (other.current == self.current)\n        except AttributeError:\n            # attribute error occurs for uninitialized refs\n            return False\n\n    def __repr__(self) -> str:\n        try:\n            current = repr(self.current)\n        except AttributeError:\n            # attribute error occurs for uninitialized refs\n            current = \"<undefined>\"\n        return f\"{type(self).__name__}({current})\"\n\n\ndef vdom_to_html(vdom: VdomDict) -> str:\n    \"\"\"Convert a VDOM dictionary into an HTML string\n\n    Only the following keys are translated to HTML:\n\n    - ``tagName``\n    - ``attributes``\n    - ``children`` (must be strings or more VDOM dicts)\n\n    Parameters:\n        vdom: The VdomDict element to convert to HTML\n    \"\"\"\n    temp_root = etree.Element(\"__temp__\")\n    _add_vdom_to_etree(temp_root, vdom)\n    html = cast(bytes, tostring(temp_root)).decode()\n    # strip out temp root <__temp__> element\n    return html[10:-11]\n\n\ndef html_to_vdom(\n    html: str, *transforms: _ModelTransform, strict: bool = True\n) -> VdomDict:\n    \"\"\"Transform HTML into a DOM model. Unique keys can be provided to HTML elements\n    using a ``key=...`` attribute within your HTML tag.\n\n    Parameters:\n        html:\n            The raw HTML as a string\n        transforms:\n            Functions of the form ``transform(old) -> new`` where ``old`` is a VDOM\n            dictionary which will be replaced by ``new``. For example, you could use a\n            transform function to add highlighting to a ``<code/>`` block.\n        strict:\n            If ``True``, raise an exception if the HTML does not perfectly follow HTML5\n            syntax.\n    \"\"\"\n    if not isinstance(html, str):  # nocov\n        msg = f\"Expected html to be a string, not {type(html).__name__}\"\n        raise TypeError(msg)\n\n    # If the user provided a string, convert it to a list of lxml.etree nodes\n    try:\n        root_node: etree._Element = fromstring(\n            html.strip(),\n            parser=etree.HTMLParser(\n                remove_comments=True,\n                remove_pis=True,\n                remove_blank_text=True,\n                recover=not strict,\n            ),\n        )\n    except etree.XMLSyntaxError as e:\n        if not strict:\n            raise e  # nocov\n        msg = \"An error has occurred while parsing the HTML.\\n\\nThis HTML may be malformatted, or may not perfectly adhere to HTML5.\\nIf you believe the exception above was due to something intentional, you can disable the strict parameter on html_to_vdom().\\nOtherwise, repair your broken HTML and try again.\"\n        raise HTMLParseError(msg) from e\n\n    return _etree_to_vdom(root_node, transforms)\n\n\nclass HTMLParseError(etree.LxmlSyntaxError):  # type: ignore[misc]\n    \"\"\"Raised when an HTML document cannot be parsed using strict parsing.\"\"\"\n\n\ndef _etree_to_vdom(\n    node: etree._Element, transforms: Iterable[_ModelTransform]\n) -> VdomDict:\n    \"\"\"Transform an lxml etree node into a DOM model\n\n    Parameters:\n        node:\n            The ``lxml.etree._Element`` node\n        transforms:\n            Functions of the form ``transform(old) -> new`` where ``old`` is a VDOM\n            dictionary which will be replaced by ``new``. For example, you could use a\n            transform function to add highlighting to a ``<code/>`` block.\n    \"\"\"\n    if not isinstance(node, etree._Element):  # nocov\n        msg = f\"Expected node to be a etree._Element, not {type(node).__name__}\"\n        raise TypeError(msg)\n\n    # Recursively call _etree_to_vdom() on all children\n    children = _generate_vdom_children(node, transforms)\n\n    # Convert the lxml node to a VDOM dict\n    el = vdom(node.tag, dict(node.items()), *children)\n\n    # Perform any necessary mutations on the VDOM attributes to meet VDOM spec\n    _mutate_vdom(el)\n\n    # Apply any provided transforms.\n    for transform in transforms:\n        el = transform(el)\n\n    return el\n\n\ndef _add_vdom_to_etree(parent: etree._Element, vdom: VdomDict | dict[str, Any]) -> None:\n    try:\n        tag = vdom[\"tagName\"]\n    except KeyError as e:\n        msg = f\"Expected a VDOM dict, not {vdom}\"\n        raise TypeError(msg) from e\n    else:\n        vdom = cast(VdomDict, vdom)\n\n    if tag:\n        element = etree.SubElement(parent, tag)\n        element.attrib.update(\n            _vdom_attr_to_html_str(k, v) for k, v in vdom.get(\"attributes\", {}).items()\n        )\n    else:\n        element = parent\n\n    for c in vdom.get(\"children\", []):\n        if isinstance(c, dict):\n            _add_vdom_to_etree(element, c)\n        else:\n            \"\"\"\n            LXML handles string children by storing them under `text` and `tail`\n            attributes of Element objects. The `text` attribute, if present, effectively\n            becomes that element's first child. Then the `tail` attribute, if present,\n            becomes a sibling that follows that element. For example, consider the\n            following HTML:\n\n                <p><a>hello</a>world</p>\n\n            In this code sample, \"hello\" is the `text` attribute of the `<a>` element\n            and \"world\" is the `tail` attribute of that same `<a>` element. It's for\n            this reason that, depending on whether the element being constructed has\n            non-string a child element, we need to assign a `text` vs `tail` attribute\n            to that element or the last non-string child respectively.\n            \"\"\"\n            if len(element):\n                last_child = element[-1]\n                last_child.tail = f\"{last_child.tail or ''}{c}\"\n            else:\n                element.text = f\"{element.text or ''}{c}\"\n\n\ndef _mutate_vdom(vdom: VdomDict) -> None:\n    \"\"\"Performs any necessary mutations on the VDOM attributes to meet VDOM spec.\n\n    Currently, this function only transforms the ``style`` attribute into a dictionary whose keys are\n    camelCase so as to be renderable by React.\n\n    This function may be extended in the future.\n    \"\"\"\n    # Determine if the style attribute needs to be converted to a dict\n    if (\n        \"attributes\" in vdom\n        and \"style\" in vdom[\"attributes\"]\n        and isinstance(vdom[\"attributes\"][\"style\"], str)\n    ):\n        # Convince type checker that it's safe to mutate attributes\n        assert isinstance(vdom[\"attributes\"], dict)  # noqa: S101\n\n        # Convert style attribute from str -> dict with camelCase keys\n        vdom[\"attributes\"][\"style\"] = {\n            key.strip().replace(\"-\", \"_\"): value.strip()\n            for key, value in (\n                part.split(\":\", 1)\n                for part in vdom[\"attributes\"][\"style\"].split(\";\")\n                if \":\" in part\n            )\n        }\n\n\ndef _generate_vdom_children(\n    node: etree._Element, transforms: Iterable[_ModelTransform]\n) -> list[VdomDict | str]:\n    \"\"\"Generates a list of VDOM children from an lxml node.\n\n    Inserts inner text and/or tail text in between VDOM children, if necessary.\n    \"\"\"\n    return (  # Get the inner text of the current node\n        [node.text] if node.text else []\n    ) + list(\n        chain(\n            *(\n                # Recursively convert each child node to VDOM\n                [_etree_to_vdom(child, transforms)]\n                # Insert the tail text between each child node\n                + ([child.tail] if child.tail else [])\n                for child in node.iterchildren(None)\n            )\n        )\n    )\n\n\ndef del_html_head_body_transform(vdom: VdomDict) -> VdomDict:\n    \"\"\"Transform intended for use with `html_to_vdom`.\n\n    Removes `<html>`, `<head>`, and `<body>` while preserving their children.\n\n    Parameters:\n        vdom:\n            The VDOM dictionary to transform.\n    \"\"\"\n    if vdom[\"tagName\"] in {\"html\", \"body\", \"head\"}:\n        return {\"tagName\": \"\", \"children\": vdom[\"children\"]}\n    return vdom\n\n\ndef _vdom_attr_to_html_str(key: str, value: Any) -> tuple[str, str]:\n    if key == \"style\":\n        if isinstance(value, dict):\n            value = \";\".join(\n                # We lower only to normalize - CSS is case-insensitive:\n                # https://www.w3.org/TR/css-fonts-3/#font-family-casing\n                f\"{_CAMEL_CASE_SUB_PATTERN.sub('-', k).lower()}:{v}\"\n                for k, v in value.items()\n            )\n    elif (\n        # camel to data-* attributes\n        key.startswith(\"data_\")\n        # camel to aria-* attributes\n        or key.startswith(\"aria_\")\n        # handle special cases\n        or key in DASHED_HTML_ATTRS\n    ):\n        key = key.replace(\"_\", \"-\")\n    elif (\n        # camel to data-* attributes\n        key.startswith(\"data\")\n        # camel to aria-* attributes\n        or key.startswith(\"aria\")\n        # handle special cases\n        or key in DASHED_HTML_ATTRS\n    ):\n        key = _CAMEL_CASE_SUB_PATTERN.sub(\"-\", key)\n\n    if callable(value):  # nocov\n        raise TypeError(f\"Cannot convert callable attribute {key}={value} to HTML\")\n\n    # Again, we lower the attribute name only to normalize - HTML is case-insensitive:\n    # http://w3c.github.io/html-reference/documents.html#case-insensitivity\n    return key.lower(), str(value)\n\n\n# see list of HTML attributes with dashes in them:\n# https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes#attribute_list\nDASHED_HTML_ATTRS = {\"accept_charset\", \"acceptCharset\", \"http_equiv\", \"httpEquiv\"}\n\n# Pattern for delimitting camelCase names (e.g. camelCase to camel-case)\n_CAMEL_CASE_SUB_PATTERN = re.compile(r\"(?<!^)(?=[A-Z])\")\n\n# Path: src/py/reactpy/reactpy/core/hooks.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import Coroutine, Sequence\nfrom logging import getLogger\nfrom types import FunctionType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Generic,\n    Protocol,\n    TypeVar,\n    cast,\n    overload,\n)\n\nfrom typing_extensions import TypeAlias\n\nfrom reactpy.config import REACTPY_DEBUG_MODE\nfrom reactpy.core._life_cycle_hook import current_hook\nfrom reactpy.core.types import Context, Key, State, VdomDict\nfrom reactpy.utils import Ref\n\nif not TYPE_CHECKING:\n    # make flake8 think that this variable exists\n    ellipsis = type(...)\n\n\n__all__ = [\n    \"use_state\",\n    \"use_effect\",\n    \"use_reducer\",\n    \"use_callback\",\n    \"use_ref\",\n    \"use_memo\",\n]\n\nlogger = getLogger(__name__)\n\n_Type = TypeVar(\"_Type\")\n\n\n@overload\ndef use_state(initial_value: Callable[[], _Type]) -> State[_Type]: ...\n\n\n@overload\ndef use_state(initial_value: _Type) -> State[_Type]: ...\n\n\ndef use_state(initial_value: _Type | Callable[[], _Type]) -> State[_Type]:\n    \"\"\"See the full :ref:`Use State` docs for details\n\n    Parameters:\n        initial_value:\n            Defines the initial value of the state. A callable (accepting no arguments)\n            can be used as a constructor function to avoid re-creating the initial value\n            on each render.\n\n    Returns:\n        A tuple containing the current state and a function to update it.\n    \"\"\"\n    current_state = _use_const(lambda: _CurrentState(initial_value))\n    return State(current_state.value, current_state.dispatch)\n\n\nclass _CurrentState(Generic[_Type]):\n    __slots__ = \"value\", \"dispatch\"\n\n    def __init__(\n        self,\n        initial_value: _Type | Callable[[], _Type],\n    ) -> None:\n        if callable(initial_value):\n            self.value = initial_value()\n        else:\n            self.value = initial_value\n\n        hook = current_hook()\n\n        def dispatch(new: _Type | Callable[[_Type], _Type]) -> None:\n            if callable(new):\n                next_value = new(self.value)\n            else:\n                next_value = new\n            if not strictly_equal(next_value, self.value):\n                self.value = next_value\n                hook.schedule_render()\n\n        self.dispatch = dispatch\n\n\n_EffectCleanFunc: TypeAlias = \"Callable[[], None]\"\n_SyncEffectFunc: TypeAlias = \"Callable[[], _EffectCleanFunc | None]\"\n_AsyncEffectFunc: TypeAlias = (\n    \"Callable[[], Coroutine[None, None, _EffectCleanFunc | None]]\"\n)\n_EffectApplyFunc: TypeAlias = \"_SyncEffectFunc | _AsyncEffectFunc\"\n\n\n@overload\ndef use_effect(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None]: ...\n\n\n@overload\ndef use_effect(\n    function: _EffectApplyFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None: ...\n\n\ndef use_effect(\n    function: _EffectApplyFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None] | None:\n    \"\"\"See the full :ref:`Use Effect` docs for details\n\n    Parameters:\n        function:\n            Applies the effect and can return a clean-up function\n        dependencies:\n            Dependencies for the effect. The effect will only trigger if the identity\n            of any value in the given sequence changes (i.e. their :func:`id` is\n            different). By default these are inferred based on local variables that are\n            referenced by the given function.\n\n    Returns:\n        If not function is provided, a decorator. Otherwise ``None``.\n    \"\"\"\n    hook = current_hook()\n\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n    last_clean_callback: Ref[_EffectCleanFunc | None] = use_ref(None)\n\n    def add_effect(function: _EffectApplyFunc) -> None:\n        if not asyncio.iscoroutinefunction(function):\n            sync_function = cast(_SyncEffectFunc, function)\n        else:\n            async_function = cast(_AsyncEffectFunc, function)\n\n            def sync_function() -> _EffectCleanFunc | None:\n                task = asyncio.create_task(async_function())\n\n                def clean_future() -> None:\n                    if not task.cancel():\n                        try:\n                            clean = task.result()\n                        except asyncio.CancelledError:\n                            pass\n                        else:\n                            if clean is not None:\n                                clean()\n\n                return clean_future\n\n        async def effect(stop: asyncio.Event) -> None:\n            if last_clean_callback.current is not None:\n                last_clean_callback.current()\n                last_clean_callback.current = None\n            clean = last_clean_callback.current = sync_function()\n            await stop.wait()\n            if clean is not None:\n                clean()\n\n        return memoize(lambda: hook.add_effect(effect))\n\n    if function is not None:\n        add_effect(function)\n        return None\n    else:\n        return add_effect\n\n\ndef use_debug_value(\n    message: Any | Callable[[], Any],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None:\n    \"\"\"Log debug information when the given message changes.\n\n    .. note::\n        This hook only logs if :data:`~reactpy.config.REACTPY_DEBUG_MODE` is active.\n\n    Unlike other hooks, a message is considered to have changed if the old and new\n    values are ``!=``. Because this comparison is performed on every render of the\n    component, it may be worth considering the performance cost in some situations.\n\n    Parameters:\n        message:\n            The value to log or a memoized function for generating the value.\n        dependencies:\n            Dependencies for the memoized function. The message will only be recomputed\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n    \"\"\"\n    old: Ref[Any] = _use_const(lambda: Ref(object()))\n    memo_func = message if callable(message) else lambda: message\n    new = use_memo(memo_func, dependencies)\n\n    if REACTPY_DEBUG_MODE.current and old.current != new:\n        old.current = new\n        logger.debug(f\"{current_hook().component} {new}\")\n\n\ndef create_context(default_value: _Type) -> Context[_Type]:\n    \"\"\"Return a new context type for use in :func:`use_context`\"\"\"\n\n    def context(\n        *children: Any,\n        value: _Type = default_value,\n        key: Key | None = None,\n    ) -> _ContextProvider[_Type]:\n        return _ContextProvider(\n            *children,\n            value=value,\n            key=key,\n            type=context,\n        )\n\n    context.__qualname__ = \"context\"\n\n    return context\n\n\ndef use_context(context: Context[_Type]) -> _Type:\n    \"\"\"Get the current value for the given context type.\n\n    See the full :ref:`Use Context` docs for more information.\n    \"\"\"\n    hook = current_hook()\n    provider = hook.get_context_provider(context)\n\n    if provider is None:\n        # same assertions but with normal exceptions\n        if not isinstance(context, FunctionType):\n            raise TypeError(f\"{context} is not a Context\")  # nocov\n        if context.__kwdefaults__ is None:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        if \"value\" not in context.__kwdefaults__:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        return cast(_Type, context.__kwdefaults__[\"value\"])\n\n    return provider.value\n\n\nclass _ContextProvider(Generic[_Type]):\n    def __init__(\n        self,\n        *children: Any,\n        value: _Type,\n        key: Key | None,\n        type: Context[_Type],\n    ) -> None:\n        self.children = children\n        self.key = key\n        self.type = type\n        self.value = value\n\n    def render(self) -> VdomDict:\n        current_hook().set_context_provider(self)\n        return {\"tagName\": \"\", \"children\": self.children}\n\n    def __repr__(self) -> str:\n        return f\"ContextProvider({self.type})\"\n\n\n_ActionType = TypeVar(\"_ActionType\")\n\n\ndef use_reducer(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    initial_value: _Type,\n) -> tuple[_Type, Callable[[_ActionType], None]]:\n    \"\"\"See the full :ref:`Use Reducer` docs for details\n\n    Parameters:\n        reducer:\n            A function which applies an action to the current state in order to\n            produce the next state.\n        initial_value:\n            The initial state value (same as for :func:`use_state`)\n\n    Returns:\n        A tuple containing the current state and a function to change it with an action\n    \"\"\"\n    state, set_state = use_state(initial_value)\n    return state, _use_const(lambda: _create_dispatcher(reducer, set_state))\n\n\ndef _create_dispatcher(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    set_state: Callable[[Callable[[_Type], _Type]], None],\n) -> Callable[[_ActionType], None]:\n    def dispatch(action: _ActionType) -> None:\n        set_state(lambda last_state: reducer(last_state, action))\n\n    return dispatch\n\n\n_CallbackFunc = TypeVar(\"_CallbackFunc\", bound=Callable[..., Any])\n\n\n@overload\ndef use_callback(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_CallbackFunc], _CallbackFunc]: ...\n\n\n@overload\ndef use_callback(\n    function: _CallbackFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc: ...\n\n\ndef use_callback(\n    function: _CallbackFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc | Callable[[_CallbackFunc], _CallbackFunc]:\n    \"\"\"See the full :ref:`Use Callback` docs for details\n\n    Parameters:\n        function:\n            The function whose identity will be preserved\n        dependencies:\n            Dependencies of the callback. The identity the ``function`` will be updated\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current function\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n\n    def setup(function: _CallbackFunc) -> _CallbackFunc:\n        return memoize(lambda: function)\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _LambdaCaller(Protocol):\n    \"\"\"MyPy doesn't know how to deal with TypeVars only used in function return\"\"\"\n\n    def __call__(self, func: Callable[[], _Type]) -> _Type: ...\n\n\n@overload\ndef use_memo(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _LambdaCaller: ...\n\n\n@overload\ndef use_memo(\n    function: Callable[[], _Type],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type: ...\n\n\ndef use_memo(\n    function: Callable[[], _Type] | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type | Callable[[Callable[[], _Type]], _Type]:\n    \"\"\"See the full :ref:`Use Memo` docs for details\n\n    Parameters:\n        function:\n            The function to be memoized.\n        dependencies:\n            Dependencies for the memoized function. The memo will only be recomputed if\n            the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current state\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n\n    memo: _Memo[_Type] = _use_const(_Memo)\n\n    if memo.empty():\n        # we need to initialize on the first run\n        changed = True\n        memo.deps = () if dependencies is None else dependencies\n    elif dependencies is None:\n        changed = True\n        memo.deps = ()\n    elif (\n        len(memo.deps) != len(dependencies)\n        # if deps are same length check identity for each item\n        or not all(\n            strictly_equal(current, new)\n            for current, new in zip(memo.deps, dependencies)\n        )\n    ):\n        memo.deps = dependencies\n        changed = True\n    else:\n        changed = False\n\n    setup: Callable[[Callable[[], _Type]], _Type]\n\n    if changed:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            current_value = memo.value = function()\n            return current_value\n\n    else:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            return memo.value\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _Memo(Generic[_Type]):\n    \"\"\"Simple object for storing memoization data\"\"\"\n\n    __slots__ = \"value\", \"deps\"\n\n    value: _Type\n    deps: Sequence[Any]\n\n    def empty(self) -> bool:\n        try:\n            self.value  # noqa: B018\n        except AttributeError:\n            return True\n        else:\n            return False\n\n\ndef use_ref(initial_value: _Type) -> Ref[_Type]:\n    \"\"\"See the full :ref:`Use State` docs for details\n\n    Parameters:\n        initial_value: The value initially assigned to the reference.\n\n    Returns:\n        A :class:`Ref` object.\n    \"\"\"\n    return _use_const(lambda: Ref(initial_value))\n\n\ndef _use_const(function: Callable[[], _Type]) -> _Type:\n    return current_hook().use_state(function)\n\n\ndef _try_to_infer_closure_values(\n    func: Callable[..., Any] | None,\n    values: Sequence[Any] | ellipsis | None,\n) -> Sequence[Any] | None:\n    if values is ...:\n        if isinstance(func, FunctionType):\n            return (\n                [cell.cell_contents for cell in func.__closure__]\n                if func.__closure__\n                else []\n            )\n        else:\n            return None\n    else:\n        return values\n\n\ndef strictly_equal(x: Any, y: Any) -> bool:\n    \"\"\"Check if two values are identical or, for a limited set or types, equal.\n\n    Only the following types are checked for equality rather than identity:\n\n    - ``int``\n    - ``float``\n    - ``complex``\n    - ``str``\n    - ``bytes``\n    - ``bytearray``\n    - ``memoryview``\n    \"\"\"\n    return x is y or (type(x) in _NUMERIC_TEXT_BINARY_TYPES and x == y)\n\n\n_NUMERIC_TEXT_BINARY_TYPES = {\n    # numeric\n    int,\n    float,\n    complex,\n    # text\n    str,\n    # binary types\n    bytes,\n    bytearray,\n    memoryview,\n}\n\n# Path: src/py/reactpy/reactpy/backend/hooks.py\nfrom __future__ import annotations\n\nfrom collections.abc import MutableMapping\nfrom typing import Any\n\nfrom reactpy.backend.types import Connection, Location\nfrom reactpy.core.hooks import create_context, use_context\nfrom reactpy.core.types import Context\n\n# backend implementations should establish this context at the root of an app\nConnectionContext: Context[Connection[Any] | None] = create_context(None)\n\n\ndef use_connection() -> Connection[Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`.\"\"\"\n    conn = use_context(ConnectionContext)\n    if conn is None:  # nocov\n        msg = \"No backend established a connection.\"\n        raise RuntimeError(msg)\n    return conn\n\n\ndef use_scope() -> MutableMapping[str, Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s scope.\"\"\"\n    return use_connection().scope\n\n\ndef use_location() -> Location:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s location.\"\"\"\n    return use_connection().location\n\n# Path: src/py/reactpy/reactpy/core/component.py\nfrom __future__ import annotations\n\nimport inspect\nfrom functools import wraps\nfrom typing import Any, Callable\n\nfrom reactpy.core.types import ComponentType, VdomDict\n\n\ndef component(\n    function: Callable[..., ComponentType | VdomDict | str | None]\n) -> Callable[..., Component]:\n    \"\"\"A decorator for defining a new component.\n\n    Parameters:\n        function: The component's :meth:`reactpy.core.proto.ComponentType.render` function.\n    \"\"\"\n    sig = inspect.signature(function)\n\n    if \"key\" in sig.parameters and sig.parameters[\"key\"].kind in (\n        inspect.Parameter.KEYWORD_ONLY,\n        inspect.Parameter.POSITIONAL_OR_KEYWORD,\n    ):\n        msg = f\"Component render function {function} uses reserved parameter 'key'\"\n        raise TypeError(msg)\n\n    @wraps(function)\n    def constructor(*args: Any, key: Any | None = None, **kwargs: Any) -> Component:\n        return Component(function, key, args, kwargs, sig)\n\n    return constructor\n\n\nclass Component:\n    \"\"\"An object for rending component models.\"\"\"\n\n    __slots__ = \"__weakref__\", \"_func\", \"_args\", \"_kwargs\", \"_sig\", \"key\", \"type\"\n\n    def __init__(\n        self,\n        function: Callable[..., ComponentType | VdomDict | str | None],\n        key: Any | None,\n        args: tuple[Any, ...],\n        kwargs: dict[str, Any],\n        sig: inspect.Signature,\n    ) -> None:\n        self.key = key\n        self.type = function\n        self._args = args\n        self._kwargs = kwargs\n        self._sig = sig\n\n    def render(self) -> ComponentType | VdomDict | str | None:\n        return self.type(*self._args, **self._kwargs)\n\n    def __repr__(self) -> str:\n        try:\n            args = self._sig.bind(*self._args, **self._kwargs).arguments\n        except TypeError:\n            return f\"{self.type.__name__}(...)\"\n        else:\n            items = \", \".join(f\"{k}={v!r}\" for k, v in args.items())\n            if items:\n                return f\"{self.type.__name__}({id(self):02x}, {items})\"\n            else:\n                return f\"{self.type.__name__}({id(self):02x})\"\n\n# Path: src/py/reactpy/reactpy/types.py\n\"\"\"Exports common types from:\n\n- :mod:`reactpy.core.types`\n- :mod:`reactpy.backend.types`\n\"\"\"\n\nfrom reactpy.backend.types import BackendType, Connection, Location\nfrom reactpy.core.component import Component\nfrom reactpy.core.types import (\n    ComponentConstructor,\n    ComponentType,\n    Context,\n    EventHandlerDict,\n    EventHandlerFunc,\n    EventHandlerMapping,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    LayoutType,\n    RootComponentConstructor,\n    State,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomJson,\n)\n\n__all__ = [\n    \"BackendType\",\n    \"Component\",\n    \"ComponentConstructor\",\n    \"ComponentType\",\n    \"Connection\",\n    \"Context\",\n    \"EventHandlerDict\",\n    \"EventHandlerFunc\",\n    \"EventHandlerMapping\",\n    \"EventHandlerType\",\n    \"ImportSourceDict\",\n    \"Key\",\n    \"LayoutType\",\n    \"Location\",\n    \"RootComponentConstructor\",\n    \"State\",\n    \"VdomAttributes\",\n    \"VdomChild\",\n    \"VdomChildren\",\n    \"VdomDict\",\n    \"VdomJson\",\n]\n\n# Path: src/py/reactpy/reactpy/backend/utils.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nimport socket\nimport sys\nfrom collections.abc import Iterator\nfrom contextlib import closing\nfrom importlib import import_module\nfrom typing import Any\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.types import RootComponentConstructor\n\nlogger = logging.getLogger(__name__)\n\nSUPPORTED_BACKENDS = (\n    \"fastapi\",\n    \"sanic\",\n    \"tornado\",\n    \"flask\",\n    \"starlette\",\n)\n\n\ndef run(\n    component: RootComponentConstructor,\n    host: str = \"127.0.0.1\",\n    port: int | None = None,\n    implementation: BackendType[Any] | None = None,\n) -> None:\n    \"\"\"Run a component with a development server\"\"\"\n    logger.warning(_DEVELOPMENT_RUN_FUNC_WARNING)\n\n    implementation = implementation or import_module(\"reactpy.backend.default\")\n    app = implementation.create_development_app()\n    implementation.configure(app, component)\n    port = port or find_available_port(host)\n    app_cls = type(app)\n\n    logger.info(\n        \"ReactPy is running with '%s.%s' at http://%s:%s\",\n        app_cls.__module__,\n        app_cls.__name__,\n        host,\n        port,\n    )\n    asyncio.run(implementation.serve_development_app(app, host, port))\n\n\ndef find_available_port(host: str, port_min: int = 8000, port_max: int = 9000) -> int:\n    \"\"\"Get a port that's available for the given host and port range\"\"\"\n    for port in range(port_min, port_max):\n        with closing(socket.socket()) as sock:\n            try:\n                if sys.platform in (\"linux\", \"darwin\"):\n                    # Fixes bug on Unix-like systems where every time you restart the\n                    # server you'll get a different port on Linux. This cannot be set\n                    # on Windows otherwise address will always be reused.\n                    # Ref: https://stackoverflow.com/a/19247688/3159288\n                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n                sock.bind((host, port))\n            except OSError:\n                pass\n            else:\n                return port\n    msg = f\"Host {host!r} has no available port in range {port_max}-{port_max}\"\n    raise RuntimeError(msg)\n\n\ndef all_implementations() -> Iterator[BackendType[Any]]:\n    \"\"\"Yield all available server implementations\"\"\"\n    for name in SUPPORTED_BACKENDS:\n        try:\n            import_module(name)\n        except ImportError:  # nocov\n            logger.debug(\"Failed to import %s\", name, exc_info=True)\n            continue\n\n        reactpy_backend_name = f\"{__name__.rsplit('.', 1)[0]}.{name}\"\n        yield import_module(reactpy_backend_name)\n\n\n_DEVELOPMENT_RUN_FUNC_WARNING = \"\"\"\\\nThe `run()` function is only intended for testing during development! To run \\\nin production, refer to the docs on how to use reactpy.backend.*.configure.\\\n\"\"\"\n\n# Path: src/py/reactpy/reactpy/core/layout.py\nfrom __future__ import annotations\n\nimport abc\nfrom asyncio import (\n    FIRST_COMPLETED,\n    CancelledError,\n    Queue,\n    Task,\n    create_task,\n    get_running_loop,\n    wait,\n)\nfrom collections import Counter\nfrom collections.abc import Sequence\nfrom contextlib import AsyncExitStack\nfrom logging import getLogger\nfrom typing import (\n    Any,\n    Callable,\n    Generic,\n    NamedTuple,\n    NewType,\n    TypeVar,\n    cast,\n)\nfrom uuid import uuid4\nfrom weakref import ref as weakref\n\nfrom anyio import Semaphore\nfrom typing_extensions import TypeAlias\n\nfrom reactpy.config import (\n    REACTPY_ASYNC_RENDERING,\n    REACTPY_CHECK_VDOM_SPEC,\n    REACTPY_DEBUG_MODE,\n)\nfrom reactpy.core._life_cycle_hook import LifeCycleHook\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    Key,\n    LayoutEventMessage,\n    LayoutUpdateMessage,\n    VdomChild,\n    VdomDict,\n    VdomJson,\n)\nfrom reactpy.core.vdom import validate_vdom_json\nfrom reactpy.utils import Ref\n\nlogger = getLogger(__name__)\n\n\nclass Layout:\n    \"\"\"Responsible for \"rendering\" components. That is, turning them into VDOM.\"\"\"\n\n    __slots__: tuple[str, ...] = (\n        \"root\",\n        \"_event_handlers\",\n        \"_rendering_queue\",\n        \"_render_tasks\",\n        \"_render_tasks_ready\",\n        \"_root_life_cycle_state_id\",\n        \"_model_states_by_life_cycle_state_id\",\n    )\n\n    if not hasattr(abc.ABC, \"__weakref__\"):  # nocov\n        __slots__ += (\"__weakref__\",)\n\n    def __init__(self, root: ComponentType) -> None:\n        super().__init__()\n        if not isinstance(root, ComponentType):\n            msg = f\"Expected a ComponentType, not {type(root)!r}.\"\n            raise TypeError(msg)\n        self.root = root\n\n    async def __aenter__(self) -> Layout:\n        # create attributes here to avoid access before entering context manager\n        self._event_handlers: EventHandlerDict = {}\n        self._render_tasks: set[Task[LayoutUpdateMessage]] = set()\n        self._render_tasks_ready: Semaphore = Semaphore(0)\n\n        self._rendering_queue: _ThreadSafeQueue[_LifeCycleStateId] = _ThreadSafeQueue()\n        root_model_state = _new_root_model_state(self.root, self._schedule_render_task)\n\n        self._root_life_cycle_state_id = root_id = root_model_state.life_cycle_state.id\n        self._model_states_by_life_cycle_state_id = {root_id: root_model_state}\n        self._schedule_render_task(root_id)\n\n        return self\n\n    async def __aexit__(self, *exc: Any) -> None:\n        root_csid = self._root_life_cycle_state_id\n        root_model_state = self._model_states_by_life_cycle_state_id[root_csid]\n\n        for t in self._render_tasks:\n            t.cancel()\n            try:\n                await t\n            except CancelledError:\n                pass\n\n        await self._unmount_model_states([root_model_state])\n\n        # delete attributes here to avoid access after exiting context manager\n        del self._event_handlers\n        del self._rendering_queue\n        del self._root_life_cycle_state_id\n        del self._model_states_by_life_cycle_state_id\n\n    async def deliver(self, event: LayoutEventMessage) -> None:\n        \"\"\"Dispatch an event to the targeted handler\"\"\"\n        # It is possible for an element in the frontend to produce an event\n        # associated with a backend model that has been deleted. We only handle\n        # events if the element and the handler exist in the backend. Otherwise\n        # we just ignore the event.\n        handler = self._event_handlers.get(event[\"target\"])\n\n        if handler is not None:\n            try:\n                await handler.function(event[\"data\"])\n            except Exception:\n                logger.exception(f\"Failed to execute event handler {handler}\")\n        else:\n            logger.info(\n                f\"Ignored event - handler {event['target']!r} \"\n                \"does not exist or its component unmounted\"\n            )\n\n    async def render(self) -> LayoutUpdateMessage:\n        if REACTPY_ASYNC_RENDERING.current:\n            return await self._concurrent_render()\n        else:  # nocov\n            return await self._serial_render()\n\n    async def _serial_render(self) -> LayoutUpdateMessage:  # nocov\n        \"\"\"Await the next available render. This will block until a component is updated\"\"\"\n        while True:\n            model_state_id = await self._rendering_queue.get()\n            try:\n                model_state = self._model_states_by_life_cycle_state_id[model_state_id]\n            except KeyError:\n                logger.debug(\n                    \"Did not render component with model state ID \"\n                    f\"{model_state_id!r} - component already unmounted\"\n                )\n            else:\n                return await self._create_layout_update(model_state)\n\n    async def _concurrent_render(self) -> LayoutUpdateMessage:\n        \"\"\"Await the next available render. This will block until a component is updated\"\"\"\n        await self._render_tasks_ready.acquire()\n        done, _ = await wait(self._render_tasks, return_when=FIRST_COMPLETED)\n        update_task: Task[LayoutUpdateMessage] = done.pop()\n        self._render_tasks.remove(update_task)\n        return update_task.result()\n\n    async def _create_layout_update(\n        self, old_state: _ModelState\n    ) -> LayoutUpdateMessage:\n        new_state = _copy_component_model_state(old_state)\n        component = new_state.life_cycle_state.component\n\n        async with AsyncExitStack() as exit_stack:\n            await self._render_component(exit_stack, old_state, new_state, component)\n\n        if REACTPY_CHECK_VDOM_SPEC.current:\n            validate_vdom_json(new_state.model.current)\n\n        return {\n            \"type\": \"layout-update\",\n            \"path\": new_state.patch_path,\n            \"model\": new_state.model.current,\n        }\n\n    async def _render_component(\n        self,\n        exit_stack: AsyncExitStack,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n        component: ComponentType,\n    ) -> None:\n        life_cycle_state = new_state.life_cycle_state\n        life_cycle_hook = life_cycle_state.hook\n\n        self._model_states_by_life_cycle_state_id[life_cycle_state.id] = new_state\n\n        await life_cycle_hook.affect_component_will_render(component)\n        exit_stack.push_async_callback(life_cycle_hook.affect_layout_did_render)\n        try:\n            raw_model = component.render()\n            # wrap the model in a fragment (i.e. tagName=\"\") to ensure components have\n            # a separate node in the model state tree. This could be removed if this\n            # components are given a node in the tree some other way\n            wrapper_model: VdomDict = {\"tagName\": \"\", \"children\": [raw_model]}\n            await self._render_model(exit_stack, old_state, new_state, wrapper_model)\n        except Exception as error:\n            logger.exception(f\"Failed to render {component}\")\n            new_state.model.current = {\n                \"tagName\": \"\",\n                \"error\": (\n                    f\"{type(error).__name__}: {error}\"\n                    if REACTPY_DEBUG_MODE.current\n                    else \"\"\n                ),\n            }\n        finally:\n            await life_cycle_hook.affect_component_did_render()\n\n        try:\n            parent = new_state.parent\n        except AttributeError:\n            pass  # only happens for root component\n        else:\n            key, index = new_state.key, new_state.index\n            parent.children_by_key[key] = new_state\n            # need to add this model to parent's children without mutating parent model\n            old_parent_model = parent.model.current\n            old_parent_children = old_parent_model[\"children\"]\n            parent.model.current = {\n                **old_parent_model,\n                \"children\": [\n                    *old_parent_children[:index],\n                    new_state.model.current,\n                    *old_parent_children[index + 1 :],\n                ],\n            }\n\n    async def _render_model(\n        self,\n        exit_stack: AsyncExitStack,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n        raw_model: Any,\n    ) -> None:\n        try:\n            new_state.model.current = {\"tagName\": raw_model[\"tagName\"]}\n        except Exception as e:  # nocov\n            msg = f\"Expected a VDOM element dict, not {raw_model}\"\n            raise ValueError(msg) from e\n        if \"key\" in raw_model:\n            new_state.key = new_state.model.current[\"key\"] = raw_model[\"key\"]\n        if \"importSource\" in raw_model:\n            new_state.model.current[\"importSource\"] = raw_model[\"importSource\"]\n        self._render_model_attributes(old_state, new_state, raw_model)\n        await self._render_model_children(\n            exit_stack, old_state, new_state, raw_model.get(\"children\", [])\n        )\n\n    def _render_model_attributes(\n        self,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n        raw_model: dict[str, Any],\n    ) -> None:\n        # extract event handlers from 'eventHandlers' and 'attributes'\n        handlers_by_event: EventHandlerDict = raw_model.get(\"eventHandlers\", {})\n\n        if \"attributes\" in raw_model:\n            attrs = raw_model[\"attributes\"].copy()\n            new_state.model.current[\"attributes\"] = attrs\n\n        if old_state is None:\n            self._render_model_event_handlers_without_old_state(\n                new_state, handlers_by_event\n            )\n            return None\n\n        for old_event in set(old_state.targets_by_event).difference(handlers_by_event):\n            old_target = old_state.targets_by_event[old_event]\n            del self._event_handlers[old_target]\n\n        if not handlers_by_event:\n            return None\n\n        model_event_handlers = new_state.model.current[\"eventHandlers\"] = {}\n        for event, handler in handlers_by_event.items():\n            if event in old_state.targets_by_event:\n                target = old_state.targets_by_event[event]\n            else:\n                target = uuid4().hex if handler.target is None else handler.target\n            new_state.targets_by_event[event] = target\n            self._event_handlers[target] = handler\n            model_event_handlers[event] = {\n                \"target\": target,\n                \"preventDefault\": handler.prevent_default,\n                \"stopPropagation\": handler.stop_propagation,\n            }\n\n        return None\n\n    def _render_model_event_handlers_without_old_state(\n        self,\n        new_state: _ModelState,\n        handlers_by_event: EventHandlerDict,\n    ) -> None:\n        if not handlers_by_event:\n            return None\n\n        model_event_handlers = new_state.model.current[\"eventHandlers\"] = {}\n        for event, handler in handlers_by_event.items():\n            target = uuid4().hex if handler.target is None else handler.target\n            new_state.targets_by_event[event] = target\n            self._event_handlers[target] = handler\n            model_event_handlers[event] = {\n                \"target\": target,\n                \"preventDefault\": handler.prevent_default,\n                \"stopPropagation\": handler.stop_propagation,\n            }\n\n        return None\n\n    async def _render_model_children(\n        self,\n        exit_stack: AsyncExitStack,\n        old_state: _ModelState | None,\n        new_state: _ModelState,\n...\n", "needle_token_start": 2344, "needle_token_end": 2490, "code_context_ntokens": 16058, "output": ["```python\ndef custom_vdom_constructor(func: _CustomVdomDictConstructor) -> VdomDictConstructor:\n    \"\"\"Cast function to VdomDictConstructor\"\"\"\n\n    @wraps(func)\n    def wrapper(*attributes_and_children: Any) -> VdomDict:\n        attributes, children = separate_attributes_and_children(attributes_and_children)\n        key = attributes.pop(\"key\", None)\n        attributes, event_handlers = separate_attributes_and_event_handlers(attributes)\n        return func(attributes, children, key, event_handlers)\n\n    return cast(VdomDictConstructor, wrapper)\n```"]}
{"repo": "reactive-python/reactpy", "name": "image", "language": "python", "path": "src/py/reactpy/reactpy/widgets.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create an HTML image element from a provided string or bytes, encoding the data into base64 format suitable for embedding directly into web pages.\n2. **Input**: The function accepts three parameters: a string specifying the image format (e.g., \"png\", \"jpeg\"), the image data either as a string or bytes, and an optional dictionary of additional HTML attributes for the image element.\n3. **Output**: Returns a dictionary representing a virtual DOM element for an image, which includes the tag name and attributes such as the source URL, which is constructed using the base64-encoded image data.\n4. **Procedure**: The function first checks and adjusts the image format if necessary (specifically for SVG). It then determines whether the image data is in string or byte format, encodes this data into base64, and constructs a data URL. This URL is used as the source attribute in the returned virtual DOM dictionary, which represents the image element.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/testing/backend.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nfrom contextlib import AsyncExitStack, suppress\nfrom types import TracebackType\nfrom typing import Any, Callable\nfrom urllib.parse import urlencode, urlunparse\n\nfrom reactpy.backend import default as default_server\nfrom reactpy.backend.types import BackendType\nfrom reactpy.backend.utils import find_available_port\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.core.component import component\nfrom reactpy.core.hooks import use_callback, use_effect, use_state\nfrom reactpy.core.types import ComponentConstructor\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    capture_reactpy_logs,\n    list_logged_exceptions,\n)\nfrom reactpy.utils import Ref\n\n\nclass BackendFixture:\n    \"\"\"A test fixture for running a server and imperatively displaying views\n\n    This fixture is typically used alongside async web drivers like ``playwight``.\n\n    Example:\n        .. code-block::\n\n            async with BackendFixture() as server:\n                server.mount(MyComponent)\n    \"\"\"\n\n    _records: list[logging.LogRecord]\n    _server_future: asyncio.Task[Any]\n    _exit_stack = AsyncExitStack()\n\n    def __init__(\n        self,\n        host: str = \"127.0.0.1\",\n        port: int | None = None,\n        app: Any | None = None,\n        implementation: BackendType[Any] | None = None,\n        options: Any | None = None,\n        timeout: float | None = None,\n    ) -> None:\n        self.host = host\n        self.port = port or find_available_port(host)\n        self.mount, self._root_component = _hotswap()\n        self.timeout = (\n            REACTPY_TESTING_DEFAULT_TIMEOUT.current if timeout is None else timeout\n        )\n\n        if app is not None and implementation is None:\n            msg = \"If an application instance its corresponding server implementation must be provided too.\"\n            raise ValueError(msg)\n\n        self._app = app\n        self.implementation = implementation or default_server\n        self._options = options\n\n    @property\n    def log_records(self) -> list[logging.LogRecord]:\n        \"\"\"A list of captured log records\"\"\"\n        return self._records\n\n    def url(self, path: str = \"\", query: Any | None = None) -> str:\n        \"\"\"Return a URL string pointing to the host and point of the server\n\n        Args:\n            path: the path to a resource on the server\n            query: a dictionary or list of query parameters\n        \"\"\"\n        return urlunparse(\n            [\n                \"http\",\n                f\"{self.host}:{self.port}\",\n                path,\n                \"\",\n                urlencode(query or ()),\n                \"\",\n            ]\n        )\n\n    def list_logged_exceptions(\n        self,\n        pattern: str = \"\",\n        types: type[Any] | tuple[type[Any], ...] = Exception,\n        log_level: int = logging.ERROR,\n        del_log_records: bool = True,\n    ) -> list[BaseException]:\n        \"\"\"Return a list of logged exception matching the given criteria\n\n...\n# Path: src/py/reactpy/reactpy/testing/common.py\nfrom __future__ import annotations\n\nimport asyncio\nimport inspect\nimport shutil\nimport time\nfrom collections.abc import Awaitable\nfrom functools import wraps\nfrom typing import Any, Callable, Generic, TypeVar, cast\nfrom uuid import uuid4\nfrom weakref import ref\n\nfrom typing_extensions import ParamSpec\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT, REACTPY_WEB_MODULES_DIR\nfrom reactpy.core._life_cycle_hook import LifeCycleHook, current_hook\nfrom reactpy.core.events import EventHandler, to_event_handler_function\n\n\ndef clear_reactpy_web_modules_dir() -> None:\n    \"\"\"Clear the directory where ReactPy stores registered web modules\"\"\"\n    for path in REACTPY_WEB_MODULES_DIR.current.iterdir():\n        shutil.rmtree(path) if path.is_dir() else path.unlink()\n\n\n_P = ParamSpec(\"_P\")\n_R = TypeVar(\"_R\")\n\n\n_DEFAULT_POLL_DELAY = 0.1\n\n\nclass poll(Generic[_R]):  # noqa: N801\n    \"\"\"Wait until the result of an sync or async function meets some condition\"\"\"\n\n    def __init__(\n        self,\n        function: Callable[_P, Awaitable[_R] | _R],\n        *args: _P.args,\n        **kwargs: _P.kwargs,\n    ) -> None:\n        coro: Callable[_P, Awaitable[_R]]\n        if not inspect.iscoroutinefunction(function):\n\n            async def coro(*args: _P.args, **kwargs: _P.kwargs) -> _R:\n                return cast(_R, function(*args, **kwargs))\n\n        else:\n            coro = cast(Callable[_P, Awaitable[_R]], function)\n        self._func = coro\n        self._args = args\n        self._kwargs = kwargs\n\n    async def until(\n        self,\n        condition: Callable[[_R], bool],\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n        description: str = \"condition to be true\",\n    ) -> None:\n        \"\"\"Check that the coroutines result meets a condition within the timeout\"\"\"\n        started_at = time.time()\n        while True:\n            await asyncio.sleep(delay)\n            result = await self._func(*self._args, **self._kwargs)\n            if condition(result):\n                break\n            elif (time.time() - started_at) > timeout:  # nocov\n                msg = f\"Expected {description} after {timeout} seconds - last value was {result!r}\"\n                raise asyncio.TimeoutError(msg)\n\n    async def until_is(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is identical to the given value\"\"\"\n        return await self.until(\n            lambda left: left is right,\n            timeout,\n            delay,\n            f\"value to be identical to {right!r}\",\n        )\n\n    async def until_equals(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is equal to the given value\"\"\"\n        return await self.until(\n            lambda left: left == right,\n            timeout,\n            delay,\n            f\"value to equal {right!r}\",\n        )\n\n\nclass HookCatcher:\n    \"\"\"Utility for capturing a LifeCycleHook from a component\n\n    Example:\n        .. code-block::\n\n            hooks = HookCatcher(index_by_kwarg=\"thing\")\n\n            @reactpy.component\n            @hooks.capture\n            def MyComponent(thing):\n                ...\n\n            ...  # render the component\n\n            # grab the last render of where MyComponent(thing='something')\n            hooks.index[\"something\"]\n            # or grab the hook from the component's last render\n            hooks.latest\n\n        After the first render of ``MyComponent`` the ``HookCatcher`` will have\n        captured the component's ``LifeCycleHook``.\n    \"\"\"\n\n    latest: LifeCycleHook\n\n    def __init__(self, index_by_kwarg: str | None = None):\n        self.index_by_kwarg = index_by_kwarg\n        self.index: dict[Any, LifeCycleHook] = {}\n\n    def capture(self, render_function: Callable[..., Any]) -> Callable[..., Any]:\n        \"\"\"Decorator for capturing a ``LifeCycleHook`` on each render of a component\"\"\"\n\n        # The render function holds a reference to `self` and, via the `LifeCycleHook`,\n        # the component. Some tests check whether components are garbage collected, thus\n        # we must use a `ref` here to ensure these checks pass once the catcher itself\n        # has been collected.\n        self_ref = ref(self)\n\n        @wraps(render_function)\n        def wrapper(*args: Any, **kwargs: Any) -> Any:\n            self = self_ref()\n            if self is None:\n                raise RuntimeError(\"Hook catcher has been garbage collected\")\n\n            hook = current_hook()\n            if self.index_by_kwarg is not None:\n                self.index[kwargs[self.index_by_kwarg]] = hook\n            self.latest = hook\n            return render_function(*args, **kwargs)\n\n        return wrapper\n\n\nclass StaticEventHandler:\n    \"\"\"Utility for capturing the target of one event handler\n\n    Example:\n        .. code-block::\n\n            static_handler = StaticEventHandler()\n\n            @reactpy.component\n            def MyComponent():\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handler.use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # gives the target ID for onClick where from the last render of MyComponent\n            static_handlers.target\n\n        If you need to capture event handlers from different instances of a component\n        the you should create multiple ``StaticEventHandler`` instances.\n\n        .. code-block::\n\n            static_handlers_by_key = {\n                \"first\": StaticEventHandler(),\n                \"second\": StaticEventHandler(),\n            }\n\n            @reactpy.component\n            def Parent():\n                return reactpy.html.div(Child(key=\"first\"), Child(key=\"second\"))\n\n            @reactpy.component\n            def Child(key):\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handlers_by_key[key].use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # grab the individual targets for each instance above\n            first_target = static_handlers_by_key[\"first\"].target\n            second_target = static_handlers_by_key[\"second\"].target\n    \"\"\"\n\n    def __init__(self) -> None:\n        self.target = uuid4().hex\n\n    def use(\n        self,\n        function: Callable[..., Any],\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n    ) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function),\n            stop_propagation,\n            prevent_default,\n            self.target,\n        )\n\n# Path: src/py/reactpy/reactpy/testing/display.py\nfrom __future__ import annotations\n\nfrom contextlib import AsyncExitStack\nfrom types import TracebackType\nfrom typing import Any\n\nfrom playwright.async_api import (\n    Browser,\n    BrowserContext,\n    ElementHandle,\n    Page,\n    async_playwright,\n)\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.types import RootComponentConstructor\n\n\nclass DisplayFixture:\n    \"\"\"A fixture for running web-based tests using ``playwright``\"\"\"\n\n    _exit_stack: AsyncExitStack\n\n    def __init__(\n        self,\n        backend: BackendFixture | None = None,\n        driver: Browser | BrowserContext | Page | None = None,\n        url_prefix: str = \"\",\n    ) -> None:\n        if backend is not None:\n            self.backend = backend\n        if driver is not None:\n            if isinstance(driver, Page):\n                self.page = driver\n            else:\n                self._browser = driver\n        self.url_prefix = url_prefix\n\n    async def show(\n        self,\n        component: RootComponentConstructor,\n    ) -> None:\n        self.backend.mount(component)\n        await self.goto(\"/\")\n        await self.root_element()  # check that root element is attached\n\n    async def goto(\n        self, path: str, query: Any | None = None, add_url_prefix: bool = True\n    ) -> None:\n        await self.page.goto(\n            self.backend.url(\n                f\"{self.url_prefix}{path}\" if add_url_prefix else path, query\n            )\n        )\n\n    async def root_element(self) -> ElementHandle:\n        element = await self.page.wait_for_selector(\"#app\", state=\"attached\")\n        if element is None:  # nocov\n            msg = \"Root element not attached\"\n            raise RuntimeError(msg)\n        return element\n\n    async def __aenter__(self) -> DisplayFixture:\n        es = self._exit_stack = AsyncExitStack()\n\n        browser: Browser | BrowserContext\n        if not hasattr(self, \"page\"):\n            if not hasattr(self, \"_browser\"):\n                pw = await es.enter_async_context(async_playwright())\n                browser = await pw.chromium.launch()\n            else:\n                browser = self._browser\n            self.page = await browser.new_page()\n\n        self.page.set_default_timeout(REACTPY_TESTING_DEFAULT_TIMEOUT.current * 1000)\n\n        if not hasattr(self, \"backend\"):\n            self.backend = BackendFixture()\n            await es.enter_async_context(self.backend)\n\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        self.backend.mount(None)\n        await self._exit_stack.aclose()\n\n# Path: src/py/reactpy/reactpy/testing/__init__.py\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.testing.common import (\n    HookCatcher,\n    StaticEventHandler,\n    clear_reactpy_web_modules_dir,\n    poll,\n)\nfrom reactpy.testing.display import DisplayFixture\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    assert_reactpy_did_log,\n    assert_reactpy_did_not_log,\n    capture_reactpy_logs,\n)\n\n__all__ = [\n    \"assert_reactpy_did_not_log\",\n    \"assert_reactpy_did_log\",\n    \"capture_reactpy_logs\",\n    \"clear_reactpy_web_modules_dir\",\n    \"DisplayFixture\",\n    \"HookCatcher\",\n    \"LogAssertionError\",\n    \"poll\",\n    \"BackendFixture\",\n    \"StaticEventHandler\",\n]\n\n# Path: src/py/reactpy/reactpy/widgets.py\nfrom __future__ import annotations\n\nfrom base64 import b64encode\nfrom collections.abc import Sequence\nfrom typing import TYPE_CHECKING, Any, Callable, Protocol, TypeVar\n\nimport reactpy\nfrom reactpy import html\nfrom reactpy._warnings import warn\nfrom reactpy.core.types import ComponentConstructor, VdomDict\n\n\n\ndef image(\n    format: str,\n    value: str | bytes = \"\",\n    attributes: dict[str, Any] | None = None,\n) -> VdomDict:\n    \"\"\"Utility for constructing an image from a string or bytes\n\n    The source value will automatically be encoded to base64\n    \"\"\"\n    if format == \"svg\":\n        format = \"svg+xml\"  # noqa: A001\n\n    if isinstance(value, str):\n        bytes_value = value.encode()\n    else:\n        bytes_value = value\n\n    base64_value = b64encode(bytes_value).decode()\n    src = f\"data:image/{format};base64,{base64_value}\"\n\n    return {\"tagName\": \"img\", \"attributes\": {\"src\": src, **(attributes or {})}}\n\n\n_Value = TypeVar(\"_Value\")\n\n\ndef use_linked_inputs(\n    attributes: Sequence[dict[str, Any]],\n    on_change: Callable[[_Value], None] = lambda value: None,\n    cast: _CastFunc[_Value] = lambda value: value,\n    initial_value: str = \"\",\n    ignore_empty: bool = True,\n) -> list[VdomDict]:\n    \"\"\"Return a list of linked inputs equal to the number of given attributes.\n\n    Parameters:\n        attributes:\n            That attributes of each returned input element. If the number of generated\n            inputs is variable, you may need to assign each one a\n            :ref:`key <Organizing Items With Keys>` by including a ``\"key\"`` in each\n            attribute dictionary.\n        on_change:\n            A callback which is triggered when any input is changed. This callback need\n            not update the 'value' field in the attributes of the inputs since that is\n            handled automatically.\n        cast:\n            Cast the 'value' of changed inputs that is passed to ``on_change``.\n        initial_value:\n            Initialize the 'value' field of the inputs.\n        ignore_empty:\n            Do not trigger ``on_change`` if the 'value' is an empty string.\n    \"\"\"\n    value, set_value = reactpy.hooks.use_state(initial_value)\n\n    def sync_inputs(event: dict[str, Any]) -> None:\n        new_value = event[\"target\"][\"value\"]\n        set_value(new_value)\n        if not new_value and ignore_empty:\n            return None\n        on_change(cast(new_value))\n\n    inputs: list[VdomDict] = []\n    for attrs in attributes:\n        inputs.append(html.input({**attrs, \"on_change\": sync_inputs, \"value\": value}))\n\n    return inputs\n\n\n_CastTo_co = TypeVar(\"_CastTo_co\", covariant=True)\n\n\nclass _CastFunc(Protocol[_CastTo_co]):\n    def __call__(self, value: str) -> _CastTo_co: ...\n\n\nif TYPE_CHECKING:\n    from reactpy.testing.backend import _MountFunc\n\n\ndef hotswap(\n    update_on_change: bool = False,\n) -> tuple[_MountFunc, ComponentConstructor]:  # nocov\n    warn(\n        \"The 'hotswap' function is deprecated and will be removed in a future release\",\n        DeprecationWarning,\n        stacklevel=2,\n    )\n    from reactpy.testing.backend import _hotswap\n\n    return _hotswap(update_on_change)\n\n# Path: src/py/reactpy/reactpy/__init__.py\nfrom reactpy import backend, config, html, logging, sample, svg, types, web, widgets\nfrom reactpy.backend.hooks import use_connection, use_location, use_scope\nfrom reactpy.backend.utils import run\nfrom reactpy.core import hooks\nfrom reactpy.core.component import component\nfrom reactpy.core.events import event\nfrom reactpy.core.hooks import (\n    create_context,\n    use_callback,\n    use_context,\n    use_debug_value,\n    use_effect,\n    use_memo,\n    use_reducer,\n    use_ref,\n    use_state,\n)\nfrom reactpy.core.layout import Layout\nfrom reactpy.core.vdom import vdom\nfrom reactpy.utils import Ref, html_to_vdom, vdom_to_html\n\n__author__ = \"The Reactive Python Team\"\n__version__ = \"1.0.2\"  # DO NOT MODIFY\n\n__all__ = [\n    \"backend\",\n    \"component\",\n    \"config\",\n    \"create_context\",\n    \"event\",\n    \"hooks\",\n    \"html_to_vdom\",\n    \"html\",\n    \"Layout\",\n    \"logging\",\n    \"Ref\",\n    \"run\",\n    \"sample\",\n    \"Stop\",\n    \"svg\",\n    \"types\",\n    \"use_callback\",\n    \"use_connection\",\n    \"use_context\",\n    \"use_debug_value\",\n    \"use_effect\",\n    \"use_location\",\n    \"use_memo\",\n    \"use_reducer\",\n    \"use_ref\",\n    \"use_scope\",\n    \"use_state\",\n    \"vdom_to_html\",\n    \"vdom\",\n    \"web\",\n    \"widgets\",\n]\n\n# Path: src/py/reactpy/reactpy/_console/__init__.py\n\n# Path: src/py/reactpy/reactpy/_console/ast_utils.py\nfrom __future__ import annotations\n\nimport ast\nfrom collections.abc import Iterator, Sequence\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom textwrap import indent\nfrom tokenize import COMMENT as COMMENT_TOKEN\nfrom tokenize import generate_tokens\nfrom typing import Any\n\nimport click\n\nfrom reactpy import html\n\n\ndef rewrite_changed_nodes(\n    file: Path,\n    source: str,\n    tree: ast.AST,\n    changed: list[ChangedNode],\n) -> str:\n    ast.fix_missing_locations(tree)\n\n    lines = source.split(\"\\n\")\n\n    # find closest parent nodes that should be re-written\n    nodes_to_unparse: list[ast.AST] = []\n    for change in changed:\n        node_lineage = [change.node, *change.parents]\n        for i in range(len(node_lineage) - 1):\n            current_node, next_node = node_lineage[i : i + 2]\n            if (\n                not hasattr(next_node, \"lineno\")\n                or next_node.lineno < change.node.lineno\n                or isinstance(next_node, (ast.ClassDef, ast.FunctionDef))\n            ):\n                nodes_to_unparse.append(current_node)\n                break\n        else:  # nocov\n            msg = \"Failed to change code\"\n            raise RuntimeError(msg)\n\n    # check if an nodes to rewrite contain each other, pick outermost nodes\n    current_outermost_node, *sorted_nodes_to_unparse = sorted(\n        nodes_to_unparse, key=lambda n: n.lineno\n    )\n    outermost_nodes_to_unparse = [current_outermost_node]\n    for node in sorted_nodes_to_unparse:\n        if (\n            not current_outermost_node.end_lineno\n            or node.lineno > current_outermost_node.end_lineno\n        ):\n            current_outermost_node = node\n            outermost_nodes_to_unparse.append(node)\n\n    moved_comment_lines_from_end: list[int] = []\n    # now actually rewrite these nodes (in reverse to avoid changes earlier in file)\n    for node in reversed(outermost_nodes_to_unparse):\n        # make a best effort to preserve any comments that we're going to overwrite\n        comments = _find_comments(lines[node.lineno - 1 : node.end_lineno])\n\n        # there may be some content just before and after the content we're re-writing\n        before_replacement = lines[node.lineno - 1][: node.col_offset].lstrip()\n\n        after_replacement = (\n            lines[node.end_lineno - 1][node.end_col_offset :].strip()\n            if node.end_lineno is not None and node.end_col_offset is not None\n            else \"\"\n        )\n\n        replacement = indent(\n            before_replacement\n            + \"\\n\".join([*comments, ast.unparse(node)])\n            + after_replacement,\n            \" \" * (node.col_offset - len(before_replacement)),\n        )\n\n        lines[node.lineno - 1 : node.end_lineno or node.lineno] = [replacement]\n\n        if comments:\n            moved_comment_lines_from_end.append(len(lines) - node.lineno)\n\n    for lineno_from_end in sorted(set(moved_comment_lines_from_end)):\n        click.echo(f\"Moved comments to {file}:{len(lines) - lineno_from_end}\")\n\n    return \"\\n\".join(lines)\n\n\n@dataclass\nclass ChangedNode:\n    node: ast.AST\n    parents: Sequence[ast.AST]\n\n\ndef find_element_constructor_usages(\n    tree: ast.AST, add_props: bool = False\n) -> Iterator[ElementConstructorInfo]:\n    changed: list[Sequence[ast.AST]] = []\n    for parents, node in _walk_with_parent(tree):\n        if not (isinstance(node, ast.Call)):\n            continue\n\n        func = node.func\n        if isinstance(func, ast.Attribute) and (\n            (isinstance(func.value, ast.Name) and func.value.id == \"html\")\n            or (isinstance(func.value, ast.Attribute) and func.value.attr == \"html\")\n        ):\n            name = func.attr\n        elif isinstance(func, ast.Name):\n            name = func.id\n        else:\n            continue\n\n        maybe_attr_dict_node: Any | None = None\n\n        if name == \"vdom\":\n            if len(node.args) == 0:\n                continue\n            elif len(node.args) == 1:\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.append(maybe_attr_dict_node)\n                else:\n                    continue\n            elif isinstance(node.args[1], (ast.Constant, ast.JoinedStr)):\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.insert(1, maybe_attr_dict_node)\n                else:\n                    continue\n            elif len(node.args) >= 2:  # noqa: PLR2004\n                maybe_attr_dict_node = node.args[1]\n        elif hasattr(html, name):\n            if len(node.args) == 0:\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.append(maybe_attr_dict_node)\n                else:\n                    continue\n            elif isinstance(node.args[0], (ast.Constant, ast.JoinedStr)):\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.insert(0, maybe_attr_dict_node)\n                else:\n                    continue\n            else:\n                maybe_attr_dict_node = node.args[0]\n\n        if not maybe_attr_dict_node:\n            continue\n\n        if isinstance(maybe_attr_dict_node, ast.Dict) or (\n            isinstance(maybe_attr_dict_node, ast.Call)\n            and isinstance(maybe_attr_dict_node.func, ast.Name)\n            and maybe_attr_dict_node.func.id == \"dict\"\n            and isinstance(maybe_attr_dict_node.func.ctx, ast.Load)\n        ):\n            yield ElementConstructorInfo(node, maybe_attr_dict_node, parents)\n\n    return changed\n\n\n@dataclass\nclass ElementConstructorInfo:\n    call: ast.Call\n    props: ast.Dict | ast.Call\n    parents: Sequence[ast.AST]\n\n\ndef _find_comments(lines: list[str]) -> list[str]:\n    iter_lines = iter(lines)\n    return [\n        token\n        for token_type, token, _, _, _ in generate_tokens(lambda: next(iter_lines))\n        if token_type == COMMENT_TOKEN\n    ]\n\n\ndef _walk_with_parent(\n    node: ast.AST, parents: tuple[ast.AST, ...] = ()\n) -> Iterator[tuple[tuple[ast.AST, ...], ast.AST]]:\n    parents = (node, *parents)\n    for child in ast.iter_child_nodes(node):\n        yield parents, child\n        yield from _walk_with_parent(child, parents)\n\n# Path: src/py/reactpy/reactpy/_console/rewrite_camel_case_props.py\nfrom __future__ import annotations\n\nimport ast\nimport re\nimport sys\nfrom copy import copy\nfrom keyword import kwlist\nfrom pathlib import Path\nfrom typing import Callable\n\nimport click\n\nfrom reactpy._console.ast_utils import (\n    ChangedNode,\n    find_element_constructor_usages,\n    rewrite_changed_nodes,\n)\n\nCAMEL_CASE_SUB_PATTERN = re.compile(r\"(?<!^)(?=[A-Z])\")\n\n\n@click.command()\n@click.argument(\"paths\", nargs=-1, type=click.Path(exists=True))\ndef rewrite_camel_case_props(paths: list[str]) -> None:\n    \"\"\"Rewrite camelCase props to snake_case\"\"\"\n    if sys.version_info < (3, 9):  # nocov\n        msg = \"This command requires Python>=3.9\"\n        raise RuntimeError(msg)\n\n    for p in map(Path, paths):\n        for f in [p] if p.is_file() else p.rglob(\"*.py\"):\n            result = generate_rewrite(file=f, source=f.read_text(encoding=\"utf-8\"))\n            if result is not None:\n                f.write_text(result)\n\n\ndef generate_rewrite(file: Path, source: str) -> str | None:\n    tree = ast.parse(source)\n\n    changed = find_nodes_to_change(tree)\n    if not changed:\n        return None\n\n    new = rewrite_changed_nodes(file, source, tree, changed)\n    return new\n\n\ndef find_nodes_to_change(tree: ast.AST) -> list[ChangedNode]:\n    changed: list[ChangedNode] = []\n    for el_info in find_element_constructor_usages(tree):\n        if _rewrite_props(el_info.props, _construct_prop_item):\n            changed.append(ChangedNode(el_info.call, el_info.parents))\n    return changed\n\n\ndef conv_attr_name(name: str) -> str:\n    new_name = CAMEL_CASE_SUB_PATTERN.sub(\"_\", name).lower()\n    return f\"{new_name}_\" if new_name in kwlist else new_name\n\n\ndef _construct_prop_item(key: str, value: ast.expr) -> tuple[str, ast.expr]:\n    if key == \"style\" and isinstance(value, (ast.Dict, ast.Call)):\n        new_value = copy(value)\n        if _rewrite_props(\n            new_value,\n            lambda k, v: (\n                (k, v)\n                # avoid infinite recursion\n                if k == \"style\"\n                else _construct_prop_item(k, v)\n            ),\n        ):\n            value = new_value\n    else:\n        key = conv_attr_name(key)\n    return key, value\n\n\ndef _rewrite_props(\n    props_node: ast.Dict | ast.Call,\n    constructor: Callable[[str, ast.expr], tuple[str, ast.expr]],\n) -> bool:\n    if isinstance(props_node, ast.Dict):\n        did_change = False\n        keys: list[ast.expr | None] = []\n        values: list[ast.expr] = []\n        for k, v in zip(props_node.keys, props_node.values):\n            if isinstance(k, ast.Constant) and isinstance(k.value, str):\n                k_value, new_v = constructor(k.value, v)\n                if k_value != k.value or new_v is not v:\n                    did_change = True\n                k = ast.Constant(value=k_value)\n                v = new_v\n            keys.append(k)\n            values.append(v)\n        if not did_change:\n            return False\n        props_node.keys = keys\n        props_node.values = values\n    else:\n        did_change = False\n        keywords: list[ast.keyword] = []\n        for kw in props_node.keywords:\n            if kw.arg is not None:\n                kw_arg, kw_value = constructor(kw.arg, kw.value)\n                if kw_arg != kw.arg or kw_value is not kw.value:\n                    did_change = True\n                kw = ast.keyword(arg=kw_arg, value=kw_value)\n            keywords.append(kw)\n        if not did_change:\n            return False\n        props_node.keywords = keywords\n    return True\n\n# Path: src/py/reactpy/reactpy/_console/rewrite_keys.py\nfrom __future__ import annotations\n\nimport ast\nimport sys\nfrom pathlib import Path\n\nimport click\n\nfrom reactpy import html\nfrom reactpy._console.ast_utils import (\n    ChangedNode,\n    find_element_constructor_usages,\n    rewrite_changed_nodes,\n)\n\n\n@click.command()\n@click.argument(\"paths\", nargs=-1, type=click.Path(exists=True))\ndef rewrite_keys(paths: list[str]) -> None:\n    \"\"\"Rewrite files under the given paths using the new html element API.\n\n    The old API required users to pass a dictionary of attributes to html element\n    constructor functions. For example:\n\n    >>> html.div({\"className\": \"x\"}, \"y\")\n    {\"tagName\": \"div\", \"attributes\": {\"className\": \"x\"}, \"children\": [\"y\"]}\n\n    The latest API though allows for attributes to be passed as snake_cased keyword\n    arguments instead. The above example would be rewritten as:\n\n    >>> html.div(\"y\", class_name=\"x\")\n    {\"tagName\": \"div\", \"attributes\": {\"class_name\": \"x\"}, \"children\": [\"y\"]}\n\n    All snake_case attributes are converted to camelCase by the client where necessary.\n\n    ----- Notes -----\n\n    While this command does it's best to preserve as much of the original code as\n    possible, there are inevitably some limitations in doing this. As a result, we\n    recommend running your code formatter like Black against your code after executing\n    this command.\n\n    Additionally, We are unable to preserve the location of comments that lie within any\n    rewritten code. This command will place the comments in the code it plans to rewrite\n    just above its changes. As such it requires manual intervention to put those\n    comments back in their original location.\n    \"\"\"\n    if sys.version_info < (3, 9):  # nocov\n        msg = \"This command requires Python>=3.9\"\n        raise RuntimeError(msg)\n\n    for p in map(Path, paths):\n        for f in [p] if p.is_file() else p.rglob(\"*.py\"):\n            result = generate_rewrite(file=f, source=f.read_text(encoding=\"utf-8\"))\n            if result is not None:\n                f.write_text(result)\n\n\ndef generate_rewrite(file: Path, source: str) -> str | None:\n    tree = ast.parse(source)\n\n    changed = find_nodes_to_change(tree)\n    if not changed:\n        log_could_not_rewrite(file, tree)\n        return None\n\n    new = rewrite_changed_nodes(file, source, tree, changed)\n    log_could_not_rewrite(file, ast.parse(new))\n\n    return new\n\n\ndef find_nodes_to_change(tree: ast.AST) -> list[ChangedNode]:\n    changed: list[ChangedNode] = []\n    for el_info in find_element_constructor_usages(tree, add_props=True):\n        for kw in list(el_info.call.keywords):\n            if kw.arg == \"key\":\n                break\n        else:\n            continue\n\n        if isinstance(el_info.props, ast.Dict):\n            el_info.props.keys.append(ast.Constant(\"key\"))\n            el_info.props.values.append(kw.value)\n        else:\n            el_info.props.keywords.append(ast.keyword(arg=\"key\", value=kw.value))\n\n        el_info.call.keywords.remove(kw)\n        changed.append(ChangedNode(el_info.call, el_info.parents))\n\n    return changed\n\n\ndef log_could_not_rewrite(file: Path, tree: ast.AST) -> None:\n    for node in ast.walk(tree):\n        if not (isinstance(node, ast.Call) and node.keywords):\n            continue\n\n        func = node.func\n        if isinstance(func, ast.Attribute):\n            name = func.attr\n        elif isinstance(func, ast.Name):\n            name = func.id\n        else:\n            continue\n\n        if (\n            name == \"vdom\"\n            or hasattr(html, name)\n            and any(kw.arg == \"key\" for kw in node.keywords)\n        ):\n            click.echo(f\"Unable to rewrite usage at {file}:{node.lineno}\")\n\n# Path: src/py/reactpy/reactpy/__main__.py\nimport click\n\nimport reactpy\nfrom reactpy._console.rewrite_camel_case_props import rewrite_camel_case_props\nfrom reactpy._console.rewrite_keys import rewrite_keys\n\n\n@click.group()\n@click.version_option(reactpy.__version__, prog_name=reactpy.__name__)\ndef app() -> None:\n    pass\n\n\napp.add_command(rewrite_keys)\napp.add_command(rewrite_camel_case_props)\n\n\nif __name__ == \"__main__\":\n    app()\n\n# Path: src/py/reactpy/reactpy/backend/_common.py\nfrom __future__ import annotations\n\nimport asyncio\nimport os\nfrom collections.abc import Awaitable, Sequence\nfrom dataclasses import dataclass\nfrom pathlib import Path, PurePosixPath\nfrom typing import TYPE_CHECKING, Any, cast\n\nfrom reactpy import __file__ as _reactpy_file_path\nfrom reactpy import html\nfrom reactpy.config import REACTPY_WEB_MODULES_DIR\nfrom reactpy.core.types import VdomDict\nfrom reactpy.utils import vdom_to_html\n\nif TYPE_CHECKING:\n    import uvicorn\n    from asgiref.typing import ASGIApplication\n\nPATH_PREFIX = PurePosixPath(\"/_reactpy\")\nMODULES_PATH = PATH_PREFIX / \"modules\"\nASSETS_PATH = PATH_PREFIX / \"assets\"\nSTREAM_PATH = PATH_PREFIX / \"stream\"\nCLIENT_BUILD_DIR = Path(_reactpy_file_path).parent / \"_static\" / \"app\" / \"dist\"\n\n\nasync def serve_with_uvicorn(\n    app: ASGIApplication | Any,\n    host: str,\n    port: int,\n    started: asyncio.Event | None,\n) -> None:\n    \"\"\"Run a development server for an ASGI application\"\"\"\n    import uvicorn\n\n    server = uvicorn.Server(\n        uvicorn.Config(\n            app,\n            host=host,\n            port=port,\n            loop=\"asyncio\",\n        )\n    )\n    server.config.setup_event_loop()\n    coros: list[Awaitable[Any]] = [server.serve()]\n\n    # If a started event is provided, then use it signal based on `server.started`\n    if started:\n        coros.append(_check_if_started(server, started))\n\n    try:\n        await asyncio.gather(*coros)\n    finally:\n        # Since we aren't using the uvicorn's `run()` API, we can't guarantee uvicorn's\n        # order of operations. So we need to make sure `shutdown()` always has an initialized\n        # list of `self.servers` to use.\n        if not hasattr(server, \"servers\"):  # nocov\n            server.servers = []\n        await asyncio.wait_for(server.shutdown(), timeout=3)\n\n\nasync def _check_if_started(server: uvicorn.Server, started: asyncio.Event) -> None:\n    while not server.started:\n        await asyncio.sleep(0.2)\n    started.set()\n\n\ndef safe_client_build_dir_path(path: str) -> Path:\n    \"\"\"Prevent path traversal out of :data:`CLIENT_BUILD_DIR`\"\"\"\n    return traversal_safe_path(\n        CLIENT_BUILD_DIR, *(\"index.html\" if path in {\"\", \"/\"} else path).split(\"/\")\n    )\n\n\ndef safe_web_modules_dir_path(path: str) -> Path:\n    \"\"\"Prevent path traversal out of :data:`reactpy.config.REACTPY_WEB_MODULES_DIR`\"\"\"\n    return traversal_safe_path(REACTPY_WEB_MODULES_DIR.current, *path.split(\"/\"))\n\n\ndef traversal_safe_path(root: str | Path, *unsafe: str | Path) -> Path:\n    \"\"\"Raise a ``ValueError`` if the ``unsafe`` path resolves outside the root dir.\"\"\"\n    root = os.path.abspath(root)\n\n    # Resolve relative paths but not symlinks - symlinks should be ok since their\n    # presence and where they point is under the control of the developer.\n    path = os.path.abspath(os.path.join(root, *unsafe))\n\n    if os.path.commonprefix([root, path]) != root:\n        # If the common prefix is not root directory we resolved outside the root dir\n        msg = \"Unsafe path\"\n        raise ValueError(msg)\n\n    return Path(path)\n\n\ndef read_client_index_html(options: CommonOptions) -> str:\n    return (\n        (CLIENT_BUILD_DIR / \"index.html\")\n        .read_text()\n        .format(__head__=vdom_head_elements_to_html(options.head))\n    )\n\n\ndef vdom_head_elements_to_html(head: Sequence[VdomDict] | VdomDict | str) -> str:\n    if isinstance(head, str):\n        return head\n    elif isinstance(head, dict):\n        if head.get(\"tagName\") == \"head\":\n            head = cast(VdomDict, {**head, \"tagName\": \"\"})\n        return vdom_to_html(head)\n    else:\n        return vdom_to_html(html._(*head))\n\n\n@dataclass\nclass CommonOptions:\n    \"\"\"Options for ReactPy's built-in backed server implementations\"\"\"\n\n    head: Sequence[VdomDict] | VdomDict | str = (\n        html.title(\"ReactPy\"),\n        html.link(\n            {\n                \"rel\": \"icon\",\n                \"href\": \"/_reactpy/assets/reactpy-logo.ico\",\n                \"type\": \"image/x-icon\",\n            }\n        ),\n    )\n    \"\"\"Add elements to the ``<head>`` of the application.\n\n    For example, this can be used to customize the title of the page, link extra\n    scripts, or load stylesheets.\n    \"\"\"\n\n    url_prefix: str = \"\"\n    \"\"\"The URL prefix where ReactPy resources will be served from\"\"\"\n\n    serve_index_route: bool = True\n    \"\"\"Automatically generate and serve the index route (``/``)\"\"\"\n\n    def __post_init__(self) -> None:\n        if self.url_prefix and not self.url_prefix.startswith(\"/\"):\n            msg = \"Expected 'url_prefix' to start with '/'\"\n            raise ValueError(msg)\n\n# Path: src/py/reactpy/reactpy/core/serve.py\nfrom __future__ import annotations\n\nfrom collections.abc import Awaitable\nfrom logging import getLogger\nfrom typing import Callable\nfrom warnings import warn\n\nfrom anyio import create_task_group\nfrom anyio.abc import TaskGroup\n\nfrom reactpy.config import REACTPY_DEBUG_MODE\nfrom reactpy.core.types import LayoutEventMessage, LayoutType, LayoutUpdateMessage\n\nlogger = getLogger(__name__)\n\n\nSendCoroutine = Callable[[LayoutUpdateMessage], Awaitable[None]]\n\"\"\"Send model patches given by a dispatcher\"\"\"\n\nRecvCoroutine = Callable[[], Awaitable[LayoutEventMessage]]\n\"\"\"Called by a dispatcher to return a :class:`reactpy.core.layout.LayoutEventMessage`\n\nThe event will then trigger an :class:`reactpy.core.proto.EventHandlerType` in a layout.\n\"\"\"\n\n\nclass Stop(BaseException):\n    \"\"\"Deprecated\n\n    Stop serving changes and events\n\n    Raising this error will tell dispatchers to gracefully exit. Typically this is\n    called by code running inside a layout to tell it to stop rendering.\n    \"\"\"\n\n\nasync def serve_layout(\n    layout: LayoutType[LayoutUpdateMessage, LayoutEventMessage],\n    send: SendCoroutine,\n    recv: RecvCoroutine,\n) -> None:\n    \"\"\"Run a dispatch loop for a single view instance\"\"\"\n    async with layout:\n        try:\n            async with create_task_group() as task_group:\n                task_group.start_soon(_single_outgoing_loop, layout, send)\n                task_group.start_soon(_single_incoming_loop, task_group, layout, recv)\n        except Stop:  # nocov\n            warn(\n                \"The Stop exception is deprecated and will be removed in a future version\",\n                UserWarning,\n                stacklevel=1,\n            )\n            logger.info(f\"Stopped serving {layout}\")\n\n\nasync def _single_outgoing_loop(\n    layout: LayoutType[LayoutUpdateMessage, LayoutEventMessage], send: SendCoroutine\n) -> None:\n    while True:\n        update = await layout.render()\n        try:\n            await send(update)\n        except Exception:  # nocov\n            if not REACTPY_DEBUG_MODE.current:\n                msg = (\n                    \"Failed to send update. More info may be available \"\n                    \"if you enabling debug mode by setting \"\n                    \"`reactpy.config.REACTPY_DEBUG_MODE.current = True`.\"\n                )\n                logger.error(msg)\n            raise\n\n\nasync def _single_incoming_loop(\n    task_group: TaskGroup,\n    layout: LayoutType[LayoutUpdateMessage, LayoutEventMessage],\n    recv: RecvCoroutine,\n) -> None:\n    while True:\n        # We need to fire and forget here so that we avoid waiting on the completion\n        # of this event handler before receiving and running the next one.\n        task_group.start_soon(layout.deliver, await recv())\n\n# Path: src/py/reactpy/reactpy/backend/starlette.py\nfrom __future__ import annotations\n\nimport asyncio\nimport json\nimport logging\nfrom collections.abc import Awaitable\nfrom dataclasses import dataclass\nfrom typing import Any, Callable\n\nfrom exceptiongroup import BaseExceptionGroup\nfrom starlette.applications import Starlette\nfrom starlette.middleware.cors import CORSMiddleware\nfrom starlette.requests import Request\nfrom starlette.responses import HTMLResponse\nfrom starlette.staticfiles import StaticFiles\nfrom starlette.websockets import WebSocket, WebSocketDisconnect\n\nfrom reactpy.backend._common import (\n    ASSETS_PATH,\n    CLIENT_BUILD_DIR,\n    MODULES_PATH,\n    STREAM_PATH,\n    CommonOptions,\n    read_client_index_html,\n    serve_with_uvicorn,\n)\nfrom reactpy.backend.hooks import ConnectionContext\nfrom reactpy.backend.hooks import use_connection as _use_connection\nfrom reactpy.backend.types import Connection, Location\nfrom reactpy.config import REACTPY_WEB_MODULES_DIR\nfrom reactpy.core.layout import Layout\nfrom reactpy.core.serve import RecvCoroutine, SendCoroutine, serve_layout\nfrom reactpy.core.types import RootComponentConstructor\n\nlogger = logging.getLogger(__name__)\n\n\n# BackendType.Options\n@dataclass\nclass Options(CommonOptions):\n    \"\"\"Render server config for :func:`reactpy.backend.starlette.configure`\"\"\"\n\n    cors: bool | dict[str, Any] = False\n    \"\"\"Enable or configure Cross Origin Resource Sharing (CORS)\n\n    For more information see docs for ``starlette.middleware.cors.CORSMiddleware``\n    \"\"\"\n\n\n# BackendType.configure\ndef configure(\n    app: Starlette,\n    component: RootComponentConstructor,\n    options: Options | None = None,\n) -> None:\n    \"\"\"Configure the necessary ReactPy routes on the given app.\n\n    Parameters:\n        app: An application instance\n        component: A component constructor\n        options: Options for configuring server behavior\n    \"\"\"\n    options = options or Options()\n\n    # this route should take priority so set up it up first\n    _setup_single_view_dispatcher_route(options, app, component)\n\n    _setup_common_routes(options, app)\n\n\n# BackendType.create_development_app\ndef create_development_app() -> Starlette:\n    \"\"\"Return a :class:`Starlette` app instance in debug mode\"\"\"\n    return Starlette(debug=True)\n\n\n# BackendType.serve_development_app\nasync def serve_development_app(\n    app: Starlette,\n    host: str,\n    port: int,\n    started: asyncio.Event | None = None,\n) -> None:\n    \"\"\"Run a development server for starlette\"\"\"\n    await serve_with_uvicorn(app, host, port, started)\n\n\ndef use_websocket() -> WebSocket:\n    \"\"\"Get the current WebSocket object\"\"\"\n    return use_connection().carrier\n\n\ndef use_connection() -> Connection[WebSocket]:\n    conn = _use_connection()\n    if not isinstance(conn.carrier, WebSocket):  # nocov\n        msg = f\"Connection has unexpected carrier {conn.carrier}. Are you running with a Flask server?\"\n        raise TypeError(msg)\n    return conn\n\n\ndef _setup_common_routes(options: Options, app: Starlette) -> None:\n    cors_options = options.cors\n    if cors_options:  # nocov\n        cors_params = (\n            cors_options if isinstance(cors_options, dict) else {\"allow_origins\": [\"*\"]}\n        )\n        app.add_middleware(CORSMiddleware, **cors_params)\n\n    # This really should be added to the APIRouter, but there's a bug in Starlette\n    # BUG: https://github.com/tiangolo/fastapi/issues/1469\n    url_prefix = options.url_prefix\n\n    app.mount(\n        str(MODULES_PATH),\n        StaticFiles(directory=REACTPY_WEB_MODULES_DIR.current, check_dir=False),\n    )\n    app.mount(\n        str(ASSETS_PATH),\n        StaticFiles(directory=CLIENT_BUILD_DIR / \"assets\", check_dir=False),\n    )\n    # register this last so it takes least priority\n    index_route = _make_index_route(options)\n\n    if options.serve_index_route:\n        app.add_route(f\"{url_prefix}/\", index_route)\n        app.add_route(url_prefix + \"/{path:path}\", index_route)\n\n\ndef _make_index_route(options: Options) -> Callable[[Request], Awaitable[HTMLResponse]]:\n    index_html = read_client_index_html(options)\n\n    async def serve_index(request: Request) -> HTMLResponse:\n        return HTMLResponse(index_html)\n\n    return serve_index\n\n\ndef _setup_single_view_dispatcher_route(\n    options: Options, app: Starlette, component: RootComponentConstructor\n) -> None:\n    async def model_stream(socket: WebSocket) -> None:\n        await socket.accept()\n        send, recv = _make_send_recv_callbacks(socket)\n\n        pathname = \"/\" + socket.scope[\"path_params\"].get(\"path\", \"\")\n        pathname = pathname[len(options.url_prefix) :] or \"/\"\n        search = socket.scope[\"query_string\"].decode()\n\n        try:\n            await serve_layout(\n                Layout(\n                    ConnectionContext(\n                        component(),\n                        value=Connection(\n                            scope=socket.scope,\n                            location=Location(pathname, f\"?{search}\" if search else \"\"),\n                            carrier=socket,\n                        ),\n                    )\n                ),\n                send,\n                recv,\n            )\n        except BaseExceptionGroup as egroup:\n            for e in egroup.exceptions:\n                if isinstance(e, WebSocketDisconnect):\n                    logger.info(f\"WebSocket disconnect: {e.code}\")\n                    break\n            else:  # nocov\n                raise\n\n    app.add_websocket_route(str(STREAM_PATH), model_stream)\n    app.add_websocket_route(f\"{STREAM_PATH}/{{path:path}}\", model_stream)\n\n\ndef _make_send_recv_callbacks(\n    socket: WebSocket,\n) -> tuple[SendCoroutine, RecvCoroutine]:\n    async def sock_send(value: Any) -> None:\n        await socket.send_text(json.dumps(value))\n\n    async def sock_recv() -> Any:\n        return json.loads(await socket.receive_text())\n\n    return sock_send, sock_recv\n\n# Path: src/py/reactpy/reactpy/backend/fastapi.py\nfrom __future__ import annotations\n\nfrom fastapi import FastAPI\n\nfrom reactpy.backend import starlette\n\n# BackendType.Options\nOptions = starlette.Options\n\n# BackendType.configure\nconfigure = starlette.configure\n\n\n# BackendType.create_development_app\ndef create_development_app() -> FastAPI:\n    \"\"\"Create a development ``FastAPI`` application instance.\"\"\"\n    return FastAPI(debug=True)\n\n\n# BackendType.serve_development_app\nserve_development_app = starlette.serve_development_app\n\nuse_connection = starlette.use_connection\n\nuse_websocket = starlette.use_websocket\n\n# Path: src/py/reactpy/reactpy/backend/flask.py\nfrom __future__ import annotations\n\nimport asyncio\nimport json\nimport logging\nimport os\nfrom asyncio import Queue as AsyncQueue\nfrom dataclasses import dataclass\nfrom queue import Queue as ThreadQueue\nfrom threading import Event as ThreadEvent\nfrom threading import Thread\nfrom typing import Any, Callable, NamedTuple, NoReturn, cast\n\nfrom flask import (\n    Blueprint,\n    Flask,\n    Request,\n    copy_current_request_context,\n    request,\n    send_file,\n)\nfrom flask_cors import CORS\nfrom flask_sock import Sock\nfrom simple_websocket import Server as WebSocket\nfrom werkzeug.serving import BaseWSGIServer, make_server\n\nimport reactpy\nfrom reactpy.backend._common import (\n    ASSETS_PATH,\n    MODULES_PATH,\n    PATH_PREFIX,\n    STREAM_PATH,\n    CommonOptions,\n    read_client_index_html,\n    safe_client_build_dir_path,\n    safe_web_modules_dir_path,\n)\nfrom reactpy.backend.hooks import ConnectionContext\nfrom reactpy.backend.hooks import use_connection as _use_connection\nfrom reactpy.backend.types import Connection, Location\nfrom reactpy.core.serve import serve_layout\nfrom reactpy.core.types import ComponentType, RootComponentConstructor\nfrom reactpy.utils import Ref\n\nlogger = logging.getLogger(__name__)\n\n\n# BackendType.Options\n@dataclass\nclass Options(CommonOptions):\n    \"\"\"Render server config for :func:`reactpy.backend.flask.configure`\"\"\"\n\n    cors: bool | dict[str, Any] = False\n    \"\"\"Enable or configure Cross Origin Resource Sharing (CORS)\n\n    For more information see docs for ``flask_cors.CORS``\n    \"\"\"\n\n\n# BackendType.configure\ndef configure(\n    app: Flask, component: RootComponentConstructor, options: Options | None = None\n) -> None:\n    \"\"\"Configure the necessary ReactPy routes on the given app.\n\n    Parameters:\n        app: An application instance\n        component: A component constructor\n        options: Options for configuring server behavior\n    \"\"\"\n    options = options or Options()\n\n    api_bp = Blueprint(f\"reactpy_api_{id(app)}\", __name__, url_prefix=str(PATH_PREFIX))\n    spa_bp = Blueprint(\n        f\"reactpy_spa_{id(app)}\", __name__, url_prefix=options.url_prefix\n    )\n\n    _setup_single_view_dispatcher_route(api_bp, options, component)\n    _setup_common_routes(api_bp, spa_bp, options)\n\n    app.register_blueprint(api_bp)\n    app.register_blueprint(spa_bp)\n\n\n# BackendType.create_development_app\ndef create_development_app() -> Flask:\n    \"\"\"Create an application instance for development purposes\"\"\"\n    os.environ[\"FLASK_DEBUG\"] = \"true\"\n    return Flask(__name__)\n\n\n# BackendType.serve_development_app\nasync def serve_development_app(\n    app: Flask,\n    host: str,\n    port: int,\n    started: asyncio.Event | None = None,\n) -> None:\n    \"\"\"Run a development server for FastAPI\"\"\"\n    loop = asyncio.get_running_loop()\n    stopped = asyncio.Event()\n\n    server: Ref[BaseWSGIServer] = Ref()\n\n    def run_server() -> None:\n        server.current = make_server(host, port, app, threaded=True)\n        if started:\n            loop.call_soon_threadsafe(started.set)\n        try:\n            server.current.serve_forever()  # type: ignore\n        finally:\n            loop.call_soon_threadsafe(stopped.set)\n\n    thread = Thread(target=run_server, daemon=True)\n    thread.start()\n\n    if started:\n        await started.wait()\n\n    try:\n        await stopped.wait()\n    finally:\n        # we may have exited because this task was cancelled\n        server.current.shutdown()\n        # the thread should eventually join\n        thread.join(timeout=3)\n        # just double check it happened\n        if thread.is_alive():  # nocov\n            msg = \"Failed to shutdown server.\"\n            raise RuntimeError(msg)\n\n\ndef use_websocket() -> WebSocket:\n    \"\"\"A handle to the current websocket\"\"\"\n    return use_connection().carrier.websocket\n\n\ndef use_request() -> Request:\n    \"\"\"Get the current ``Request``\"\"\"\n    return use_connection().carrier.request\n\n\ndef use_connection() -> Connection[_FlaskCarrier]:\n    \"\"\"Get the current :class:`Connection`\"\"\"\n    conn = _use_connection()\n    if not isinstance(conn.carrier, _FlaskCarrier):  # nocov\n        msg = f\"Connection has unexpected carrier {conn.carrier}. Are you running with a Flask server?\"\n        raise TypeError(msg)\n    return conn\n\n\ndef _setup_common_routes(\n    api_blueprint: Blueprint,\n    spa_blueprint: Blueprint,\n    options: Options,\n) -> None:\n    cors_options = options.cors\n    if cors_options:  # nocov\n        cors_params = cors_options if isinstance(cors_options, dict) else {}\n        CORS(api_blueprint, **cors_params)\n\n    @api_blueprint.route(f\"/{ASSETS_PATH.name}/<path:path>\")\n    def send_assets_dir(path: str = \"\") -> Any:\n        return send_file(safe_client_build_dir_path(f\"assets/{path}\"))\n\n    @api_blueprint.route(f\"/{MODULES_PATH.name}/<path:path>\")\n    def send_modules_dir(path: str = \"\") -> Any:\n        return send_file(safe_web_modules_dir_path(path), mimetype=\"text/javascript\")\n\n    index_html = read_client_index_html(options)\n\n    if options.serve_index_route:\n\n        @spa_blueprint.route(\"/\")\n        @spa_blueprint.route(\"/<path:_>\")\n        def send_client_dir(_: str = \"\") -> Any:\n            return index_html\n\n\ndef _setup_single_view_dispatcher_route(\n    api_blueprint: Blueprint, options: Options, constructor: RootComponentConstructor\n) -> None:\n    sock = Sock(api_blueprint)\n\n    def model_stream(ws: WebSocket, path: str = \"\") -> None:\n        def send(value: Any) -> None:\n            ws.send(json.dumps(value))\n\n        def recv() -> Any:\n            return json.loads(ws.receive())\n\n        _dispatch_in_thread(\n            ws,\n            # remove any url prefix from path\n            path[len(options.url_prefix) :],\n            constructor(),\n            send,\n            recv,\n        )\n\n    sock.route(STREAM_PATH.name, endpoint=\"without_path\")(model_stream)\n    sock.route(f\"{STREAM_PATH.name}/<path:path>\", endpoint=\"with_path\")(model_stream)\n\n\ndef _dispatch_in_thread(\n    websocket: WebSocket,\n    path: str,\n    component: ComponentType,\n    send: Callable[[Any], None],\n    recv: Callable[[], Any | None],\n) -> NoReturn:\n    dispatch_thread_info_created = ThreadEvent()\n    dispatch_thread_info_ref: reactpy.Ref[_DispatcherThreadInfo | None] = reactpy.Ref(\n        None\n    )\n\n    @copy_current_request_context\n    def run_dispatcher() -> None:\n        loop = asyncio.new_event_loop()\n        asyncio.set_event_loop(loop)\n\n        thread_send_queue: ThreadQueue[Any] = ThreadQueue()\n        async_recv_queue: AsyncQueue[Any] = AsyncQueue()\n\n        async def send_coro(value: Any) -> None:\n            thread_send_queue.put(value)\n\n        async def main() -> None:\n            search = request.query_string.decode()\n            await serve_layout(\n                reactpy.Layout(\n                    ConnectionContext(\n                        component,\n                        value=Connection(\n                            scope=request.environ,\n                            location=Location(\n                                pathname=f\"/{path}\",\n                                search=f\"?{search}\" if search else \"\",\n                            ),\n                            carrier=_FlaskCarrier(request, websocket),\n                        ),\n                    ),\n                ),\n                send_coro,\n                async_recv_queue.get,\n            )\n\n        main_future = asyncio.ensure_future(main(), loop=loop)\n\n        dispatch_thread_info_ref.current = _DispatcherThreadInfo(\n            dispatch_loop=loop,\n            dispatch_future=main_future,\n            thread_send_queue=thread_send_queue,\n            async_recv_queue=async_recv_queue,\n        )\n        dispatch_thread_info_created.set()\n\n        loop.run_until_complete(main_future)\n\n...\n", "needle_token_start": 3907, "needle_token_end": 4106, "code_context_ntokens": 16025, "output": ["```python\ndef image(\n    format: str,\n    value: str | bytes = \"\",\n    attributes: dict[str, Any] | None = None,\n) -> VdomDict:\n    \"\"\"Utility for constructing an image from a string or bytes\n\n    The source value will automatically be encoded to base64\n    \"\"\"\n    if format == \"svg\":\n        format = \"svg+xml\"  # noqa: A001\n\n    if isinstance(value, str):\n        bytes_value = value.encode()\n    else:\n        bytes_value = value\n\n    base64_value = b64encode(bytes_value).decode()\n    src = f\"data:image/{format};base64,{base64_value}\"\n\n    return {\"tagName\": \"img\", \"attributes\": {\"src\": src, **(attributes or {})}}\n```"]}
{"repo": "reactive-python/reactpy", "name": "use", "language": "python", "path": "src/py/reactpy/reactpy/testing/common.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To create an event handler that is associated with a unique identifier, which can be configured to stop propagation or prevent default actions during event handling.\n2. **Input**: A function that defines the behavior of the event handler, and optional boolean flags to control event propagation and default actions.\n3. **Output**: An event handler object that encapsulates the user-defined function, event control behaviors, and a unique identifier.\n4. **Procedure**: The method generates a unique identifier for the event handler upon object instantiation. When called, it wraps the provided function with additional functionalities based on the specified flags for stopping propagation and preventing default actions, and associates it with the unique identifier.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/backend/default.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom logging import getLogger\nfrom sys import exc_info\nfrom typing import Any, NoReturn\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.backend.utils import SUPPORTED_BACKENDS, all_implementations\n...\n# Path: src/py/reactpy/reactpy/testing/logs.py\nfrom __future__ import annotations\n\nimport logging\nimport re\nfrom collections.abc import Iterator\nfrom contextlib import contextmanager\nfrom traceback import format_exception\nfrom typing import Any, NoReturn\n\nfrom reactpy.logging import ROOT_LOGGER\n\n\nclass LogAssertionError(AssertionError):\n    \"\"\"An assertion error raised in relation to log messages.\"\"\"\n\n\n@contextmanager\ndef assert_reactpy_did_log(\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> Iterator[None]:\n    \"\"\"Assert that ReactPy produced a log matching the described message or error.\n\n    Args:\n        match_message: Must match a logged message.\n        error_type: Checks the type of logged exceptions.\n        match_error: Must match an error message.\n    \"\"\"\n    message_pattern = re.compile(match_message)\n    error_pattern = re.compile(match_error)\n\n    with capture_reactpy_logs() as log_records:\n        try:\n            yield None\n        except Exception:\n            raise\n        else:\n            for record in list(log_records):\n                if (\n                    # record message matches\n                    message_pattern.findall(record.getMessage())\n                    # error type matches\n                    and (\n                        error_type is None\n                        or (\n                            record.exc_info is not None\n                            and record.exc_info[0] is not None\n                            and issubclass(record.exc_info[0], error_type)\n                        )\n                    )\n                    # error message pattern matches\n                    and (\n                        not match_error\n                        or (\n                            record.exc_info is not None\n                            and error_pattern.findall(\n                                \"\".join(format_exception(*record.exc_info))\n                            )\n                        )\n                    )\n                ):\n                    break\n            else:  # nocov\n                _raise_log_message_error(\n                    \"Could not find a log record matching the given\",\n                    match_message,\n                    error_type,\n                    match_error,\n                )\n\n\n@contextmanager\ndef assert_reactpy_did_not_log(\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> Iterator[None]:\n    \"\"\"Assert the inverse of :func:`assert_reactpy_logged`\"\"\"\n    try:\n        with assert_reactpy_did_log(match_message, error_type, match_error):\n            yield None\n    except LogAssertionError:\n        pass\n    else:\n        _raise_log_message_error(\n            \"Did find a log record matching the given\",\n            match_message,\n            error_type,\n            match_error,\n        )\n\n\ndef list_logged_exceptions(\n    log_records: list[logging.LogRecord],\n    pattern: str = \"\",\n    types: type[Any] | tuple[type[Any], ...] = Exception,\n    log_level: int = logging.ERROR,\n    del_log_records: bool = True,\n) -> list[BaseException]:\n    \"\"\"Return a list of logged exception matching the given criteria\n\n    Args:\n        log_level: The level of log to check\n        exclude_exc_types: Any exception types to ignore\n        del_log_records: Whether to delete the log records for yielded exceptions\n    \"\"\"\n    found: list[BaseException] = []\n    compiled_pattern = re.compile(pattern)\n    for index, record in enumerate(log_records):\n        if record.levelno >= log_level and record.exc_info:\n            error = record.exc_info[1]\n            if (\n                error is not None\n                and isinstance(error, types)\n                and compiled_pattern.search(str(error))\n            ):\n                if del_log_records:\n                    del log_records[index - len(found)]\n                found.append(error)\n    return found\n\n\n@contextmanager\ndef capture_reactpy_logs() -> Iterator[list[logging.LogRecord]]:\n    \"\"\"Capture logs from ReactPy\n\n    Any logs produced in this context are cleared afterwards\n    \"\"\"\n    original_level = ROOT_LOGGER.level\n    ROOT_LOGGER.setLevel(logging.DEBUG)\n    try:\n        if _LOG_RECORD_CAPTOR in ROOT_LOGGER.handlers:\n            start_index = len(_LOG_RECORD_CAPTOR.records)\n            try:\n                yield _LOG_RECORD_CAPTOR.records\n            finally:\n                end_index = len(_LOG_RECORD_CAPTOR.records)\n                _LOG_RECORD_CAPTOR.records[start_index:end_index] = []\n            return None\n\n        ROOT_LOGGER.addHandler(_LOG_RECORD_CAPTOR)\n        try:\n            yield _LOG_RECORD_CAPTOR.records\n        finally:\n            ROOT_LOGGER.removeHandler(_LOG_RECORD_CAPTOR)\n            _LOG_RECORD_CAPTOR.records.clear()\n    finally:\n        ROOT_LOGGER.setLevel(original_level)\n\n\nclass _LogRecordCaptor(logging.NullHandler):\n    def __init__(self) -> None:\n        self.records: list[logging.LogRecord] = []\n        super().__init__()\n\n    def handle(self, record: logging.LogRecord) -> bool:\n        self.records.append(record)\n        return True\n\n\n_LOG_RECORD_CAPTOR = _LogRecordCaptor()\n\n\ndef _raise_log_message_error(\n    prefix: str,\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> NoReturn:\n    conditions = []\n    if match_message:\n        conditions.append(f\"log message pattern {match_message!r}\")\n    if error_type:\n        conditions.append(f\"exception type {error_type}\")\n    if match_error:\n        conditions.append(f\"error message pattern {match_error!r}\")\n    raise LogAssertionError(prefix + \" \" + \" and \".join(conditions))\n\n# Path: src/py/reactpy/reactpy/testing/backend.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nfrom contextlib import AsyncExitStack, suppress\nfrom types import TracebackType\nfrom typing import Any, Callable\nfrom urllib.parse import urlencode, urlunparse\n\nfrom reactpy.backend import default as default_server\nfrom reactpy.backend.types import BackendType\nfrom reactpy.backend.utils import find_available_port\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.core.component import component\nfrom reactpy.core.hooks import use_callback, use_effect, use_state\nfrom reactpy.core.types import ComponentConstructor\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    capture_reactpy_logs,\n    list_logged_exceptions,\n)\nfrom reactpy.utils import Ref\n\n\nclass BackendFixture:\n    \"\"\"A test fixture for running a server and imperatively displaying views\n\n    This fixture is typically used alongside async web drivers like ``playwight``.\n\n    Example:\n        .. code-block::\n\n            async with BackendFixture() as server:\n                server.mount(MyComponent)\n    \"\"\"\n\n    _records: list[logging.LogRecord]\n    _server_future: asyncio.Task[Any]\n    _exit_stack = AsyncExitStack()\n\n    def __init__(\n        self,\n        host: str = \"127.0.0.1\",\n        port: int | None = None,\n        app: Any | None = None,\n        implementation: BackendType[Any] | None = None,\n        options: Any | None = None,\n        timeout: float | None = None,\n    ) -> None:\n        self.host = host\n        self.port = port or find_available_port(host)\n        self.mount, self._root_component = _hotswap()\n        self.timeout = (\n            REACTPY_TESTING_DEFAULT_TIMEOUT.current if timeout is None else timeout\n        )\n\n        if app is not None and implementation is None:\n            msg = \"If an application instance its corresponding server implementation must be provided too.\"\n            raise ValueError(msg)\n\n        self._app = app\n        self.implementation = implementation or default_server\n        self._options = options\n\n    @property\n    def log_records(self) -> list[logging.LogRecord]:\n        \"\"\"A list of captured log records\"\"\"\n        return self._records\n\n    def url(self, path: str = \"\", query: Any | None = None) -> str:\n        \"\"\"Return a URL string pointing to the host and point of the server\n\n        Args:\n            path: the path to a resource on the server\n            query: a dictionary or list of query parameters\n        \"\"\"\n        return urlunparse(\n            [\n                \"http\",\n                f\"{self.host}:{self.port}\",\n                path,\n                \"\",\n                urlencode(query or ()),\n                \"\",\n            ]\n        )\n\n    def list_logged_exceptions(\n        self,\n        pattern: str = \"\",\n        types: type[Any] | tuple[type[Any], ...] = Exception,\n        log_level: int = logging.ERROR,\n        del_log_records: bool = True,\n    ) -> list[BaseException]:\n        \"\"\"Return a list of logged exception matching the given criteria\n\n        Args:\n            log_level: The level of log to check\n            exclude_exc_types: Any exception types to ignore\n            del_log_records: Whether to delete the log records for yielded exceptions\n        \"\"\"\n        return list_logged_exceptions(\n            self.log_records,\n            pattern,\n            types,\n            log_level,\n            del_log_records,\n        )\n\n    async def __aenter__(self) -> BackendFixture:\n        self._exit_stack = AsyncExitStack()\n        self._records = self._exit_stack.enter_context(capture_reactpy_logs())\n\n        app = self._app or self.implementation.create_development_app()\n        self.implementation.configure(app, self._root_component, self._options)\n\n        started = asyncio.Event()\n        server_future = asyncio.create_task(\n            self.implementation.serve_development_app(\n                app, self.host, self.port, started\n            )\n        )\n\n        async def stop_server() -> None:\n            server_future.cancel()\n            with suppress(asyncio.CancelledError):\n                await asyncio.wait_for(server_future, timeout=self.timeout)\n\n        self._exit_stack.push_async_callback(stop_server)\n\n        try:\n            await asyncio.wait_for(started.wait(), timeout=self.timeout)\n        except Exception:  # nocov\n            # see if we can await the future for a more helpful error\n            await asyncio.wait_for(server_future, timeout=self.timeout)\n            raise\n\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        await self._exit_stack.aclose()\n\n        self.mount(None)  # reset the view\n\n        logged_errors = self.list_logged_exceptions(del_log_records=False)\n        if logged_errors:  # nocov\n            msg = \"Unexpected logged exception\"\n            raise LogAssertionError(msg) from logged_errors[0]\n\n\n_MountFunc = Callable[[\"Callable[[], Any] | None\"], None]\n\n\ndef _hotswap(update_on_change: bool = False) -> tuple[_MountFunc, ComponentConstructor]:\n    \"\"\"Swap out components from a layout on the fly.\n\n    Since you can't change the component functions used to create a layout\n    in an imperative manner, you can use ``hotswap`` to do this so\n    long as you set things up ahead of time.\n\n    Parameters:\n        update_on_change: Whether or not all views of the layout should be updated on a swap.\n\n    Example:\n        .. code-block:: python\n\n            import reactpy\n\n            show, root = reactpy.hotswap()\n            PerClientStateServer(root).run_in_thread(\"localhost\", 8765)\n\n            @reactpy.component\n            def DivOne(self):\n                return {\"tagName\": \"div\", \"children\": [1]}\n\n            show(DivOne)\n\n            # displaying the output now will show DivOne\n\n            @reactpy.component\n            def DivTwo(self):\n                return {\"tagName\": \"div\", \"children\": [2]}\n\n            show(DivTwo)\n\n            # displaying the output now will show DivTwo\n    \"\"\"\n    constructor_ref: Ref[Callable[[], Any]] = Ref(lambda: None)\n\n    if update_on_change:\n        set_constructor_callbacks: set[Callable[[Callable[[], Any]], None]] = set()\n\n        @component\n        def HotSwap() -> Any:\n            # new displays will adopt the latest constructor and arguments\n            constructor, _set_constructor = use_state(lambda: constructor_ref.current)\n            set_constructor = use_callback(lambda new: _set_constructor(lambda _: new))\n\n            def add_callback() -> Callable[[], None]:\n                set_constructor_callbacks.add(set_constructor)\n                return lambda: set_constructor_callbacks.remove(set_constructor)\n\n            use_effect(add_callback)\n\n            return constructor()\n\n        def swap(constructor: Callable[[], Any] | None) -> None:\n            constructor = constructor_ref.current = constructor or (lambda: None)\n\n            for set_constructor in set_constructor_callbacks:\n                set_constructor(constructor)\n\n    else:\n\n        @component\n        def HotSwap() -> Any:\n            return constructor_ref.current()\n\n        def swap(constructor: Callable[[], Any] | None) -> None:\n            constructor_ref.current = constructor or (lambda: None)\n\n    return swap, HotSwap\n\n# Path: src/py/reactpy/reactpy/testing/common.py\nfrom __future__ import annotations\n\nimport asyncio\nimport inspect\nimport shutil\nimport time\nfrom collections.abc import Awaitable\nfrom functools import wraps\nfrom typing import Any, Callable, Generic, TypeVar, cast\nfrom uuid import uuid4\nfrom weakref import ref\n\nfrom typing_extensions import ParamSpec\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT, REACTPY_WEB_MODULES_DIR\nfrom reactpy.core._life_cycle_hook import LifeCycleHook, current_hook\nfrom reactpy.core.events import EventHandler, to_event_handler_function\n\n\ndef clear_reactpy_web_modules_dir() -> None:\n    \"\"\"Clear the directory where ReactPy stores registered web modules\"\"\"\n    for path in REACTPY_WEB_MODULES_DIR.current.iterdir():\n        shutil.rmtree(path) if path.is_dir() else path.unlink()\n\n\n_P = ParamSpec(\"_P\")\n_R = TypeVar(\"_R\")\n\n\n_DEFAULT_POLL_DELAY = 0.1\n\n\nclass poll(Generic[_R]):  # noqa: N801\n    \"\"\"Wait until the result of an sync or async function meets some condition\"\"\"\n\n    def __init__(\n        self,\n        function: Callable[_P, Awaitable[_R] | _R],\n        *args: _P.args,\n        **kwargs: _P.kwargs,\n    ) -> None:\n        coro: Callable[_P, Awaitable[_R]]\n        if not inspect.iscoroutinefunction(function):\n\n            async def coro(*args: _P.args, **kwargs: _P.kwargs) -> _R:\n                return cast(_R, function(*args, **kwargs))\n\n        else:\n            coro = cast(Callable[_P, Awaitable[_R]], function)\n        self._func = coro\n        self._args = args\n        self._kwargs = kwargs\n\n    async def until(\n        self,\n        condition: Callable[[_R], bool],\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n        description: str = \"condition to be true\",\n    ) -> None:\n        \"\"\"Check that the coroutines result meets a condition within the timeout\"\"\"\n        started_at = time.time()\n        while True:\n            await asyncio.sleep(delay)\n            result = await self._func(*self._args, **self._kwargs)\n            if condition(result):\n                break\n            elif (time.time() - started_at) > timeout:  # nocov\n                msg = f\"Expected {description} after {timeout} seconds - last value was {result!r}\"\n                raise asyncio.TimeoutError(msg)\n\n    async def until_is(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is identical to the given value\"\"\"\n        return await self.until(\n            lambda left: left is right,\n            timeout,\n            delay,\n            f\"value to be identical to {right!r}\",\n        )\n\n    async def until_equals(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is equal to the given value\"\"\"\n        return await self.until(\n            lambda left: left == right,\n            timeout,\n            delay,\n            f\"value to equal {right!r}\",\n        )\n\n\nclass HookCatcher:\n    \"\"\"Utility for capturing a LifeCycleHook from a component\n\n    Example:\n        .. code-block::\n\n            hooks = HookCatcher(index_by_kwarg=\"thing\")\n\n            @reactpy.component\n            @hooks.capture\n            def MyComponent(thing):\n                ...\n\n            ...  # render the component\n\n            # grab the last render of where MyComponent(thing='something')\n            hooks.index[\"something\"]\n            # or grab the hook from the component's last render\n            hooks.latest\n\n        After the first render of ``MyComponent`` the ``HookCatcher`` will have\n        captured the component's ``LifeCycleHook``.\n    \"\"\"\n\n    latest: LifeCycleHook\n\n    def __init__(self, index_by_kwarg: str | None = None):\n        self.index_by_kwarg = index_by_kwarg\n        self.index: dict[Any, LifeCycleHook] = {}\n\n    def capture(self, render_function: Callable[..., Any]) -> Callable[..., Any]:\n        \"\"\"Decorator for capturing a ``LifeCycleHook`` on each render of a component\"\"\"\n\n        # The render function holds a reference to `self` and, via the `LifeCycleHook`,\n        # the component. Some tests check whether components are garbage collected, thus\n        # we must use a `ref` here to ensure these checks pass once the catcher itself\n        # has been collected.\n        self_ref = ref(self)\n\n        @wraps(render_function)\n        def wrapper(*args: Any, **kwargs: Any) -> Any:\n            self = self_ref()\n            if self is None:\n                raise RuntimeError(\"Hook catcher has been garbage collected\")\n\n            hook = current_hook()\n            if self.index_by_kwarg is not None:\n                self.index[kwargs[self.index_by_kwarg]] = hook\n            self.latest = hook\n            return render_function(*args, **kwargs)\n\n        return wrapper\n\n\nclass StaticEventHandler:\n    \"\"\"Utility for capturing the target of one event handler\n\n    Example:\n        .. code-block::\n\n            static_handler = StaticEventHandler()\n\n            @reactpy.component\n            def MyComponent():\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handler.use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # gives the target ID for onClick where from the last render of MyComponent\n            static_handlers.target\n\n        If you need to capture event handlers from different instances of a component\n        the you should create multiple ``StaticEventHandler`` instances.\n\n        .. code-block::\n\n            static_handlers_by_key = {\n                \"first\": StaticEventHandler(),\n                \"second\": StaticEventHandler(),\n            }\n\n            @reactpy.component\n            def Parent():\n                return reactpy.html.div(Child(key=\"first\"), Child(key=\"second\"))\n\n            @reactpy.component\n            def Child(key):\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handlers_by_key[key].use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # grab the individual targets for each instance above\n            first_target = static_handlers_by_key[\"first\"].target\n            second_target = static_handlers_by_key[\"second\"].target\n    \"\"\"\n\n    def __init__(self) -> None:\n        self.target = uuid4().hex\n\n    \ndef use(\n        self,\n        function: Callable[..., Any],\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n    ) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function),\n            stop_propagation,\n            prevent_default,\n            self.target,\n        )\n\n# Path: src/py/reactpy/reactpy/testing/display.py\nfrom __future__ import annotations\n\nfrom contextlib import AsyncExitStack\nfrom types import TracebackType\nfrom typing import Any\n\nfrom playwright.async_api import (\n    Browser,\n    BrowserContext,\n    ElementHandle,\n    Page,\n    async_playwright,\n)\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.types import RootComponentConstructor\n\n\nclass DisplayFixture:\n    \"\"\"A fixture for running web-based tests using ``playwright``\"\"\"\n\n    _exit_stack: AsyncExitStack\n\n    def __init__(\n        self,\n        backend: BackendFixture | None = None,\n        driver: Browser | BrowserContext | Page | None = None,\n        url_prefix: str = \"\",\n    ) -> None:\n        if backend is not None:\n            self.backend = backend\n        if driver is not None:\n            if isinstance(driver, Page):\n                self.page = driver\n            else:\n                self._browser = driver\n        self.url_prefix = url_prefix\n\n    async def show(\n        self,\n        component: RootComponentConstructor,\n    ) -> None:\n        self.backend.mount(component)\n        await self.goto(\"/\")\n        await self.root_element()  # check that root element is attached\n\n    async def goto(\n        self, path: str, query: Any | None = None, add_url_prefix: bool = True\n    ) -> None:\n        await self.page.goto(\n            self.backend.url(\n                f\"{self.url_prefix}{path}\" if add_url_prefix else path, query\n            )\n        )\n\n    async def root_element(self) -> ElementHandle:\n        element = await self.page.wait_for_selector(\"#app\", state=\"attached\")\n        if element is None:  # nocov\n            msg = \"Root element not attached\"\n            raise RuntimeError(msg)\n        return element\n\n    async def __aenter__(self) -> DisplayFixture:\n        es = self._exit_stack = AsyncExitStack()\n\n        browser: Browser | BrowserContext\n        if not hasattr(self, \"page\"):\n            if not hasattr(self, \"_browser\"):\n                pw = await es.enter_async_context(async_playwright())\n                browser = await pw.chromium.launch()\n            else:\n                browser = self._browser\n            self.page = await browser.new_page()\n\n        self.page.set_default_timeout(REACTPY_TESTING_DEFAULT_TIMEOUT.current * 1000)\n\n        if not hasattr(self, \"backend\"):\n            self.backend = BackendFixture()\n            await es.enter_async_context(self.backend)\n\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        self.backend.mount(None)\n        await self._exit_stack.aclose()\n\n# Path: src/py/reactpy/reactpy/testing/__init__.py\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.testing.common import (\n    HookCatcher,\n    StaticEventHandler,\n    clear_reactpy_web_modules_dir,\n    poll,\n)\nfrom reactpy.testing.display import DisplayFixture\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    assert_reactpy_did_log,\n    assert_reactpy_did_not_log,\n    capture_reactpy_logs,\n)\n\n__all__ = [\n    \"assert_reactpy_did_not_log\",\n    \"assert_reactpy_did_log\",\n    \"capture_reactpy_logs\",\n    \"clear_reactpy_web_modules_dir\",\n    \"DisplayFixture\",\n    \"HookCatcher\",\n    \"LogAssertionError\",\n    \"poll\",\n    \"BackendFixture\",\n    \"StaticEventHandler\",\n]\n\n# Path: src/py/reactpy/reactpy/widgets.py\nfrom __future__ import annotations\n\nfrom base64 import b64encode\nfrom collections.abc import Sequence\nfrom typing import TYPE_CHECKING, Any, Callable, Protocol, TypeVar\n\nimport reactpy\nfrom reactpy import html\nfrom reactpy._warnings import warn\nfrom reactpy.core.types import ComponentConstructor, VdomDict\n\n\ndef image(\n    format: str,\n    value: str | bytes = \"\",\n    attributes: dict[str, Any] | None = None,\n) -> VdomDict:\n    \"\"\"Utility for constructing an image from a string or bytes\n\n    The source value will automatically be encoded to base64\n    \"\"\"\n    if format == \"svg\":\n        format = \"svg+xml\"  # noqa: A001\n\n    if isinstance(value, str):\n        bytes_value = value.encode()\n    else:\n        bytes_value = value\n\n    base64_value = b64encode(bytes_value).decode()\n    src = f\"data:image/{format};base64,{base64_value}\"\n\n    return {\"tagName\": \"img\", \"attributes\": {\"src\": src, **(attributes or {})}}\n\n\n_Value = TypeVar(\"_Value\")\n\n\ndef use_linked_inputs(\n    attributes: Sequence[dict[str, Any]],\n    on_change: Callable[[_Value], None] = lambda value: None,\n    cast: _CastFunc[_Value] = lambda value: value,\n    initial_value: str = \"\",\n    ignore_empty: bool = True,\n) -> list[VdomDict]:\n    \"\"\"Return a list of linked inputs equal to the number of given attributes.\n\n    Parameters:\n        attributes:\n            That attributes of each returned input element. If the number of generated\n            inputs is variable, you may need to assign each one a\n            :ref:`key <Organizing Items With Keys>` by including a ``\"key\"`` in each\n            attribute dictionary.\n        on_change:\n            A callback which is triggered when any input is changed. This callback need\n            not update the 'value' field in the attributes of the inputs since that is\n            handled automatically.\n        cast:\n            Cast the 'value' of changed inputs that is passed to ``on_change``.\n        initial_value:\n            Initialize the 'value' field of the inputs.\n        ignore_empty:\n            Do not trigger ``on_change`` if the 'value' is an empty string.\n    \"\"\"\n    value, set_value = reactpy.hooks.use_state(initial_value)\n\n    def sync_inputs(event: dict[str, Any]) -> None:\n        new_value = event[\"target\"][\"value\"]\n        set_value(new_value)\n        if not new_value and ignore_empty:\n            return None\n        on_change(cast(new_value))\n\n    inputs: list[VdomDict] = []\n    for attrs in attributes:\n        inputs.append(html.input({**attrs, \"on_change\": sync_inputs, \"value\": value}))\n\n    return inputs\n\n\n_CastTo_co = TypeVar(\"_CastTo_co\", covariant=True)\n\n\nclass _CastFunc(Protocol[_CastTo_co]):\n    def __call__(self, value: str) -> _CastTo_co: ...\n\n\nif TYPE_CHECKING:\n    from reactpy.testing.backend import _MountFunc\n\n\ndef hotswap(\n    update_on_change: bool = False,\n) -> tuple[_MountFunc, ComponentConstructor]:  # nocov\n    warn(\n        \"The 'hotswap' function is deprecated and will be removed in a future release\",\n        DeprecationWarning,\n        stacklevel=2,\n    )\n    from reactpy.testing.backend import _hotswap\n\n    return _hotswap(update_on_change)\n\n# Path: src/py/reactpy/reactpy/__init__.py\nfrom reactpy import backend, config, html, logging, sample, svg, types, web, widgets\nfrom reactpy.backend.hooks import use_connection, use_location, use_scope\nfrom reactpy.backend.utils import run\nfrom reactpy.core import hooks\nfrom reactpy.core.component import component\nfrom reactpy.core.events import event\nfrom reactpy.core.hooks import (\n    create_context,\n    use_callback,\n    use_context,\n    use_debug_value,\n    use_effect,\n    use_memo,\n    use_reducer,\n    use_ref,\n    use_state,\n)\nfrom reactpy.core.layout import Layout\nfrom reactpy.core.vdom import vdom\nfrom reactpy.utils import Ref, html_to_vdom, vdom_to_html\n\n__author__ = \"The Reactive Python Team\"\n__version__ = \"1.0.2\"  # DO NOT MODIFY\n\n__all__ = [\n    \"backend\",\n    \"component\",\n    \"config\",\n    \"create_context\",\n    \"event\",\n    \"hooks\",\n    \"html_to_vdom\",\n    \"html\",\n    \"Layout\",\n    \"logging\",\n    \"Ref\",\n    \"run\",\n    \"sample\",\n    \"Stop\",\n    \"svg\",\n    \"types\",\n    \"use_callback\",\n    \"use_connection\",\n    \"use_context\",\n    \"use_debug_value\",\n    \"use_effect\",\n    \"use_location\",\n    \"use_memo\",\n    \"use_reducer\",\n    \"use_ref\",\n    \"use_scope\",\n    \"use_state\",\n    \"vdom_to_html\",\n    \"vdom\",\n    \"web\",\n    \"widgets\",\n]\n\n# Path: src/py/reactpy/reactpy/_console/__init__.py\n\n# Path: src/py/reactpy/reactpy/_console/ast_utils.py\nfrom __future__ import annotations\n\nimport ast\nfrom collections.abc import Iterator, Sequence\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom textwrap import indent\nfrom tokenize import COMMENT as COMMENT_TOKEN\nfrom tokenize import generate_tokens\nfrom typing import Any\n\nimport click\n\nfrom reactpy import html\n\n\ndef rewrite_changed_nodes(\n    file: Path,\n    source: str,\n    tree: ast.AST,\n    changed: list[ChangedNode],\n) -> str:\n    ast.fix_missing_locations(tree)\n\n    lines = source.split(\"\\n\")\n\n    # find closest parent nodes that should be re-written\n    nodes_to_unparse: list[ast.AST] = []\n    for change in changed:\n        node_lineage = [change.node, *change.parents]\n        for i in range(len(node_lineage) - 1):\n            current_node, next_node = node_lineage[i : i + 2]\n            if (\n                not hasattr(next_node, \"lineno\")\n                or next_node.lineno < change.node.lineno\n                or isinstance(next_node, (ast.ClassDef, ast.FunctionDef))\n            ):\n                nodes_to_unparse.append(current_node)\n                break\n        else:  # nocov\n            msg = \"Failed to change code\"\n            raise RuntimeError(msg)\n\n    # check if an nodes to rewrite contain each other, pick outermost nodes\n    current_outermost_node, *sorted_nodes_to_unparse = sorted(\n        nodes_to_unparse, key=lambda n: n.lineno\n    )\n    outermost_nodes_to_unparse = [current_outermost_node]\n    for node in sorted_nodes_to_unparse:\n        if (\n            not current_outermost_node.end_lineno\n            or node.lineno > current_outermost_node.end_lineno\n        ):\n            current_outermost_node = node\n            outermost_nodes_to_unparse.append(node)\n\n    moved_comment_lines_from_end: list[int] = []\n    # now actually rewrite these nodes (in reverse to avoid changes earlier in file)\n    for node in reversed(outermost_nodes_to_unparse):\n        # make a best effort to preserve any comments that we're going to overwrite\n        comments = _find_comments(lines[node.lineno - 1 : node.end_lineno])\n\n        # there may be some content just before and after the content we're re-writing\n        before_replacement = lines[node.lineno - 1][: node.col_offset].lstrip()\n\n        after_replacement = (\n            lines[node.end_lineno - 1][node.end_col_offset :].strip()\n            if node.end_lineno is not None and node.end_col_offset is not None\n            else \"\"\n        )\n\n        replacement = indent(\n            before_replacement\n            + \"\\n\".join([*comments, ast.unparse(node)])\n            + after_replacement,\n            \" \" * (node.col_offset - len(before_replacement)),\n        )\n\n        lines[node.lineno - 1 : node.end_lineno or node.lineno] = [replacement]\n\n        if comments:\n            moved_comment_lines_from_end.append(len(lines) - node.lineno)\n\n    for lineno_from_end in sorted(set(moved_comment_lines_from_end)):\n        click.echo(f\"Moved comments to {file}:{len(lines) - lineno_from_end}\")\n\n    return \"\\n\".join(lines)\n\n\n@dataclass\nclass ChangedNode:\n    node: ast.AST\n    parents: Sequence[ast.AST]\n\n\ndef find_element_constructor_usages(\n    tree: ast.AST, add_props: bool = False\n) -> Iterator[ElementConstructorInfo]:\n    changed: list[Sequence[ast.AST]] = []\n    for parents, node in _walk_with_parent(tree):\n        if not (isinstance(node, ast.Call)):\n            continue\n\n        func = node.func\n        if isinstance(func, ast.Attribute) and (\n            (isinstance(func.value, ast.Name) and func.value.id == \"html\")\n            or (isinstance(func.value, ast.Attribute) and func.value.attr == \"html\")\n        ):\n            name = func.attr\n        elif isinstance(func, ast.Name):\n            name = func.id\n        else:\n            continue\n\n        maybe_attr_dict_node: Any | None = None\n\n        if name == \"vdom\":\n            if len(node.args) == 0:\n                continue\n            elif len(node.args) == 1:\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.append(maybe_attr_dict_node)\n                else:\n                    continue\n            elif isinstance(node.args[1], (ast.Constant, ast.JoinedStr)):\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.insert(1, maybe_attr_dict_node)\n                else:\n                    continue\n            elif len(node.args) >= 2:  # noqa: PLR2004\n                maybe_attr_dict_node = node.args[1]\n        elif hasattr(html, name):\n            if len(node.args) == 0:\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.append(maybe_attr_dict_node)\n                else:\n                    continue\n            elif isinstance(node.args[0], (ast.Constant, ast.JoinedStr)):\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.insert(0, maybe_attr_dict_node)\n                else:\n                    continue\n            else:\n                maybe_attr_dict_node = node.args[0]\n\n        if not maybe_attr_dict_node:\n            continue\n\n        if isinstance(maybe_attr_dict_node, ast.Dict) or (\n            isinstance(maybe_attr_dict_node, ast.Call)\n            and isinstance(maybe_attr_dict_node.func, ast.Name)\n            and maybe_attr_dict_node.func.id == \"dict\"\n            and isinstance(maybe_attr_dict_node.func.ctx, ast.Load)\n        ):\n            yield ElementConstructorInfo(node, maybe_attr_dict_node, parents)\n\n    return changed\n\n\n@dataclass\nclass ElementConstructorInfo:\n    call: ast.Call\n    props: ast.Dict | ast.Call\n    parents: Sequence[ast.AST]\n\n\ndef _find_comments(lines: list[str]) -> list[str]:\n    iter_lines = iter(lines)\n    return [\n        token\n        for token_type, token, _, _, _ in generate_tokens(lambda: next(iter_lines))\n        if token_type == COMMENT_TOKEN\n    ]\n\n\ndef _walk_with_parent(\n    node: ast.AST, parents: tuple[ast.AST, ...] = ()\n) -> Iterator[tuple[tuple[ast.AST, ...], ast.AST]]:\n    parents = (node, *parents)\n    for child in ast.iter_child_nodes(node):\n        yield parents, child\n        yield from _walk_with_parent(child, parents)\n\n# Path: src/py/reactpy/reactpy/_console/rewrite_camel_case_props.py\nfrom __future__ import annotations\n\nimport ast\nimport re\nimport sys\nfrom copy import copy\nfrom keyword import kwlist\nfrom pathlib import Path\nfrom typing import Callable\n\nimport click\n\nfrom reactpy._console.ast_utils import (\n    ChangedNode,\n    find_element_constructor_usages,\n    rewrite_changed_nodes,\n)\n\nCAMEL_CASE_SUB_PATTERN = re.compile(r\"(?<!^)(?=[A-Z])\")\n\n\n@click.command()\n@click.argument(\"paths\", nargs=-1, type=click.Path(exists=True))\ndef rewrite_camel_case_props(paths: list[str]) -> None:\n    \"\"\"Rewrite camelCase props to snake_case\"\"\"\n    if sys.version_info < (3, 9):  # nocov\n        msg = \"This command requires Python>=3.9\"\n        raise RuntimeError(msg)\n\n    for p in map(Path, paths):\n        for f in [p] if p.is_file() else p.rglob(\"*.py\"):\n            result = generate_rewrite(file=f, source=f.read_text(encoding=\"utf-8\"))\n            if result is not None:\n                f.write_text(result)\n\n\ndef generate_rewrite(file: Path, source: str) -> str | None:\n    tree = ast.parse(source)\n\n    changed = find_nodes_to_change(tree)\n    if not changed:\n        return None\n\n    new = rewrite_changed_nodes(file, source, tree, changed)\n    return new\n\n\ndef find_nodes_to_change(tree: ast.AST) -> list[ChangedNode]:\n    changed: list[ChangedNode] = []\n    for el_info in find_element_constructor_usages(tree):\n        if _rewrite_props(el_info.props, _construct_prop_item):\n            changed.append(ChangedNode(el_info.call, el_info.parents))\n    return changed\n\n\ndef conv_attr_name(name: str) -> str:\n    new_name = CAMEL_CASE_SUB_PATTERN.sub(\"_\", name).lower()\n    return f\"{new_name}_\" if new_name in kwlist else new_name\n\n\ndef _construct_prop_item(key: str, value: ast.expr) -> tuple[str, ast.expr]:\n    if key == \"style\" and isinstance(value, (ast.Dict, ast.Call)):\n        new_value = copy(value)\n        if _rewrite_props(\n            new_value,\n            lambda k, v: (\n                (k, v)\n                # avoid infinite recursion\n                if k == \"style\"\n                else _construct_prop_item(k, v)\n            ),\n        ):\n            value = new_value\n    else:\n        key = conv_attr_name(key)\n    return key, value\n\n\ndef _rewrite_props(\n    props_node: ast.Dict | ast.Call,\n    constructor: Callable[[str, ast.expr], tuple[str, ast.expr]],\n) -> bool:\n    if isinstance(props_node, ast.Dict):\n        did_change = False\n        keys: list[ast.expr | None] = []\n        values: list[ast.expr] = []\n        for k, v in zip(props_node.keys, props_node.values):\n            if isinstance(k, ast.Constant) and isinstance(k.value, str):\n                k_value, new_v = constructor(k.value, v)\n                if k_value != k.value or new_v is not v:\n                    did_change = True\n                k = ast.Constant(value=k_value)\n                v = new_v\n            keys.append(k)\n            values.append(v)\n        if not did_change:\n            return False\n        props_node.keys = keys\n        props_node.values = values\n    else:\n        did_change = False\n        keywords: list[ast.keyword] = []\n        for kw in props_node.keywords:\n            if kw.arg is not None:\n                kw_arg, kw_value = constructor(kw.arg, kw.value)\n                if kw_arg != kw.arg or kw_value is not kw.value:\n                    did_change = True\n                kw = ast.keyword(arg=kw_arg, value=kw_value)\n            keywords.append(kw)\n        if not did_change:\n            return False\n        props_node.keywords = keywords\n    return True\n\n# Path: src/py/reactpy/reactpy/_console/rewrite_keys.py\nfrom __future__ import annotations\n\nimport ast\nimport sys\nfrom pathlib import Path\n\nimport click\n\nfrom reactpy import html\nfrom reactpy._console.ast_utils import (\n    ChangedNode,\n    find_element_constructor_usages,\n    rewrite_changed_nodes,\n)\n\n\n@click.command()\n@click.argument(\"paths\", nargs=-1, type=click.Path(exists=True))\ndef rewrite_keys(paths: list[str]) -> None:\n    \"\"\"Rewrite files under the given paths using the new html element API.\n\n    The old API required users to pass a dictionary of attributes to html element\n    constructor functions. For example:\n\n    >>> html.div({\"className\": \"x\"}, \"y\")\n    {\"tagName\": \"div\", \"attributes\": {\"className\": \"x\"}, \"children\": [\"y\"]}\n\n    The latest API though allows for attributes to be passed as snake_cased keyword\n    arguments instead. The above example would be rewritten as:\n\n    >>> html.div(\"y\", class_name=\"x\")\n    {\"tagName\": \"div\", \"attributes\": {\"class_name\": \"x\"}, \"children\": [\"y\"]}\n\n    All snake_case attributes are converted to camelCase by the client where necessary.\n\n    ----- Notes -----\n\n    While this command does it's best to preserve as much of the original code as\n    possible, there are inevitably some limitations in doing this. As a result, we\n    recommend running your code formatter like Black against your code after executing\n    this command.\n\n    Additionally, We are unable to preserve the location of comments that lie within any\n    rewritten code. This command will place the comments in the code it plans to rewrite\n    just above its changes. As such it requires manual intervention to put those\n    comments back in their original location.\n    \"\"\"\n    if sys.version_info < (3, 9):  # nocov\n        msg = \"This command requires Python>=3.9\"\n        raise RuntimeError(msg)\n\n    for p in map(Path, paths):\n        for f in [p] if p.is_file() else p.rglob(\"*.py\"):\n            result = generate_rewrite(file=f, source=f.read_text(encoding=\"utf-8\"))\n            if result is not None:\n                f.write_text(result)\n\n\ndef generate_rewrite(file: Path, source: str) -> str | None:\n    tree = ast.parse(source)\n\n    changed = find_nodes_to_change(tree)\n    if not changed:\n        log_could_not_rewrite(file, tree)\n        return None\n\n    new = rewrite_changed_nodes(file, source, tree, changed)\n    log_could_not_rewrite(file, ast.parse(new))\n\n    return new\n\n\ndef find_nodes_to_change(tree: ast.AST) -> list[ChangedNode]:\n    changed: list[ChangedNode] = []\n    for el_info in find_element_constructor_usages(tree, add_props=True):\n        for kw in list(el_info.call.keywords):\n            if kw.arg == \"key\":\n                break\n        else:\n            continue\n\n        if isinstance(el_info.props, ast.Dict):\n            el_info.props.keys.append(ast.Constant(\"key\"))\n            el_info.props.values.append(kw.value)\n        else:\n            el_info.props.keywords.append(ast.keyword(arg=\"key\", value=kw.value))\n\n        el_info.call.keywords.remove(kw)\n        changed.append(ChangedNode(el_info.call, el_info.parents))\n\n    return changed\n\n\ndef log_could_not_rewrite(file: Path, tree: ast.AST) -> None:\n    for node in ast.walk(tree):\n        if not (isinstance(node, ast.Call) and node.keywords):\n            continue\n\n        func = node.func\n        if isinstance(func, ast.Attribute):\n            name = func.attr\n        elif isinstance(func, ast.Name):\n            name = func.id\n        else:\n            continue\n\n        if (\n            name == \"vdom\"\n            or hasattr(html, name)\n            and any(kw.arg == \"key\" for kw in node.keywords)\n        ):\n            click.echo(f\"Unable to rewrite usage at {file}:{node.lineno}\")\n\n# Path: src/py/reactpy/reactpy/__main__.py\nimport click\n\nimport reactpy\nfrom reactpy._console.rewrite_camel_case_props import rewrite_camel_case_props\nfrom reactpy._console.rewrite_keys import rewrite_keys\n\n\n@click.group()\n@click.version_option(reactpy.__version__, prog_name=reactpy.__name__)\ndef app() -> None:\n    pass\n\n\napp.add_command(rewrite_keys)\napp.add_command(rewrite_camel_case_props)\n\n\nif __name__ == \"__main__\":\n    app()\n\n# Path: src/py/reactpy/reactpy/backend/_common.py\nfrom __future__ import annotations\n\nimport asyncio\nimport os\nfrom collections.abc import Awaitable, Sequence\nfrom dataclasses import dataclass\nfrom pathlib import Path, PurePosixPath\nfrom typing import TYPE_CHECKING, Any, cast\n\nfrom reactpy import __file__ as _reactpy_file_path\nfrom reactpy import html\nfrom reactpy.config import REACTPY_WEB_MODULES_DIR\nfrom reactpy.core.types import VdomDict\nfrom reactpy.utils import vdom_to_html\n\nif TYPE_CHECKING:\n    import uvicorn\n    from asgiref.typing import ASGIApplication\n\nPATH_PREFIX = PurePosixPath(\"/_reactpy\")\nMODULES_PATH = PATH_PREFIX / \"modules\"\nASSETS_PATH = PATH_PREFIX / \"assets\"\nSTREAM_PATH = PATH_PREFIX / \"stream\"\nCLIENT_BUILD_DIR = Path(_reactpy_file_path).parent / \"_static\" / \"app\" / \"dist\"\n\n\nasync def serve_with_uvicorn(\n    app: ASGIApplication | Any,\n    host: str,\n    port: int,\n    started: asyncio.Event | None,\n) -> None:\n    \"\"\"Run a development server for an ASGI application\"\"\"\n    import uvicorn\n\n    server = uvicorn.Server(\n        uvicorn.Config(\n            app,\n            host=host,\n            port=port,\n            loop=\"asyncio\",\n        )\n    )\n    server.config.setup_event_loop()\n    coros: list[Awaitable[Any]] = [server.serve()]\n\n    # If a started event is provided, then use it signal based on `server.started`\n    if started:\n        coros.append(_check_if_started(server, started))\n\n    try:\n        await asyncio.gather(*coros)\n    finally:\n        # Since we aren't using the uvicorn's `run()` API, we can't guarantee uvicorn's\n        # order of operations. So we need to make sure `shutdown()` always has an initialized\n        # list of `self.servers` to use.\n        if not hasattr(server, \"servers\"):  # nocov\n            server.servers = []\n        await asyncio.wait_for(server.shutdown(), timeout=3)\n\n\nasync def _check_if_started(server: uvicorn.Server, started: asyncio.Event) -> None:\n    while not server.started:\n        await asyncio.sleep(0.2)\n    started.set()\n\n\ndef safe_client_build_dir_path(path: str) -> Path:\n    \"\"\"Prevent path traversal out of :data:`CLIENT_BUILD_DIR`\"\"\"\n    return traversal_safe_path(\n        CLIENT_BUILD_DIR, *(\"index.html\" if path in {\"\", \"/\"} else path).split(\"/\")\n    )\n\n\ndef safe_web_modules_dir_path(path: str) -> Path:\n    \"\"\"Prevent path traversal out of :data:`reactpy.config.REACTPY_WEB_MODULES_DIR`\"\"\"\n    return traversal_safe_path(REACTPY_WEB_MODULES_DIR.current, *path.split(\"/\"))\n\n\ndef traversal_safe_path(root: str | Path, *unsafe: str | Path) -> Path:\n    \"\"\"Raise a ``ValueError`` if the ``unsafe`` path resolves outside the root dir.\"\"\"\n    root = os.path.abspath(root)\n\n    # Resolve relative paths but not symlinks - symlinks should be ok since their\n    # presence and where they point is under the control of the developer.\n    path = os.path.abspath(os.path.join(root, *unsafe))\n\n    if os.path.commonprefix([root, path]) != root:\n        # If the common prefix is not root directory we resolved outside the root dir\n        msg = \"Unsafe path\"\n        raise ValueError(msg)\n\n    return Path(path)\n\n\ndef read_client_index_html(options: CommonOptions) -> str:\n    return (\n        (CLIENT_BUILD_DIR / \"index.html\")\n        .read_text()\n        .format(__head__=vdom_head_elements_to_html(options.head))\n    )\n\n\ndef vdom_head_elements_to_html(head: Sequence[VdomDict] | VdomDict | str) -> str:\n    if isinstance(head, str):\n        return head\n    elif isinstance(head, dict):\n        if head.get(\"tagName\") == \"head\":\n            head = cast(VdomDict, {**head, \"tagName\": \"\"})\n        return vdom_to_html(head)\n    else:\n        return vdom_to_html(html._(*head))\n\n\n@dataclass\nclass CommonOptions:\n    \"\"\"Options for ReactPy's built-in backed server implementations\"\"\"\n\n    head: Sequence[VdomDict] | VdomDict | str = (\n        html.title(\"ReactPy\"),\n        html.link(\n            {\n                \"rel\": \"icon\",\n                \"href\": \"/_reactpy/assets/reactpy-logo.ico\",\n                \"type\": \"image/x-icon\",\n            }\n        ),\n    )\n    \"\"\"Add elements to the ``<head>`` of the application.\n\n    For example, this can be used to customize the title of the page, link extra\n    scripts, or load stylesheets.\n    \"\"\"\n\n    url_prefix: str = \"\"\n    \"\"\"The URL prefix where ReactPy resources will be served from\"\"\"\n\n    serve_index_route: bool = True\n    \"\"\"Automatically generate and serve the index route (``/``)\"\"\"\n\n    def __post_init__(self) -> None:\n        if self.url_prefix and not self.url_prefix.startswith(\"/\"):\n            msg = \"Expected 'url_prefix' to start with '/'\"\n            raise ValueError(msg)\n\n# Path: src/py/reactpy/reactpy/core/serve.py\nfrom __future__ import annotations\n\nfrom collections.abc import Awaitable\nfrom logging import getLogger\nfrom typing import Callable\nfrom warnings import warn\n\nfrom anyio import create_task_group\nfrom anyio.abc import TaskGroup\n\nfrom reactpy.config import REACTPY_DEBUG_MODE\nfrom reactpy.core.types import LayoutEventMessage, LayoutType, LayoutUpdateMessage\n\nlogger = getLogger(__name__)\n\n\nSendCoroutine = Callable[[LayoutUpdateMessage], Awaitable[None]]\n\"\"\"Send model patches given by a dispatcher\"\"\"\n\nRecvCoroutine = Callable[[], Awaitable[LayoutEventMessage]]\n\"\"\"Called by a dispatcher to return a :class:`reactpy.core.layout.LayoutEventMessage`\n\nThe event will then trigger an :class:`reactpy.core.proto.EventHandlerType` in a layout.\n\"\"\"\n\n\nclass Stop(BaseException):\n    \"\"\"Deprecated\n\n    Stop serving changes and events\n\n    Raising this error will tell dispatchers to gracefully exit. Typically this is\n    called by code running inside a layout to tell it to stop rendering.\n    \"\"\"\n\n\nasync def serve_layout(\n    layout: LayoutType[LayoutUpdateMessage, LayoutEventMessage],\n    send: SendCoroutine,\n    recv: RecvCoroutine,\n) -> None:\n    \"\"\"Run a dispatch loop for a single view instance\"\"\"\n    async with layout:\n        try:\n            async with create_task_group() as task_group:\n                task_group.start_soon(_single_outgoing_loop, layout, send)\n                task_group.start_soon(_single_incoming_loop, task_group, layout, recv)\n        except Stop:  # nocov\n            warn(\n                \"The Stop exception is deprecated and will be removed in a future version\",\n                UserWarning,\n                stacklevel=1,\n            )\n            logger.info(f\"Stopped serving {layout}\")\n\n\nasync def _single_outgoing_loop(\n    layout: LayoutType[LayoutUpdateMessage, LayoutEventMessage], send: SendCoroutine\n) -> None:\n    while True:\n        update = await layout.render()\n        try:\n            await send(update)\n        except Exception:  # nocov\n            if not REACTPY_DEBUG_MODE.current:\n                msg = (\n                    \"Failed to send update. More info may be available \"\n                    \"if you enabling debug mode by setting \"\n                    \"`reactpy.config.REACTPY_DEBUG_MODE.current = True`.\"\n                )\n                logger.error(msg)\n            raise\n\n\nasync def _single_incoming_loop(\n    task_group: TaskGroup,\n    layout: LayoutType[LayoutUpdateMessage, LayoutEventMessage],\n    recv: RecvCoroutine,\n) -> None:\n    while True:\n        # We need to fire and forget here so that we avoid waiting on the completion\n        # of this event handler before receiving and running the next one.\n        task_group.start_soon(layout.deliver, await recv())\n\n# Path: src/py/reactpy/reactpy/backend/starlette.py\nfrom __future__ import annotations\n\nimport asyncio\nimport json\nimport logging\nfrom collections.abc import Awaitable\nfrom dataclasses import dataclass\nfrom typing import Any, Callable\n\nfrom exceptiongroup import BaseExceptionGroup\nfrom starlette.applications import Starlette\nfrom starlette.middleware.cors import CORSMiddleware\nfrom starlette.requests import Request\nfrom starlette.responses import HTMLResponse\nfrom starlette.staticfiles import StaticFiles\nfrom starlette.websockets import WebSocket, WebSocketDisconnect\n\nfrom reactpy.backend._common import (\n    ASSETS_PATH,\n    CLIENT_BUILD_DIR,\n    MODULES_PATH,\n    STREAM_PATH,\n    CommonOptions,\n    read_client_index_html,\n    serve_with_uvicorn,\n)\nfrom reactpy.backend.hooks import ConnectionContext\nfrom reactpy.backend.hooks import use_connection as _use_connection\nfrom reactpy.backend.types import Connection, Location\nfrom reactpy.config import REACTPY_WEB_MODULES_DIR\nfrom reactpy.core.layout import Layout\nfrom reactpy.core.serve import RecvCoroutine, SendCoroutine, serve_layout\nfrom reactpy.core.types import RootComponentConstructor\n\nlogger = logging.getLogger(__name__)\n\n\n# BackendType.Options\n@dataclass\nclass Options(CommonOptions):\n    \"\"\"Render server config for :func:`reactpy.backend.starlette.configure`\"\"\"\n\n    cors: bool | dict[str, Any] = False\n    \"\"\"Enable or configure Cross Origin Resource Sharing (CORS)\n\n    For more information see docs for ``starlette.middleware.cors.CORSMiddleware``\n    \"\"\"\n\n\n# BackendType.configure\ndef configure(\n    app: Starlette,\n    component: RootComponentConstructor,\n    options: Options | None = None,\n) -> None:\n    \"\"\"Configure the necessary ReactPy routes on the given app.\n\n    Parameters:\n        app: An application instance\n        component: A component constructor\n        options: Options for configuring server behavior\n    \"\"\"\n    options = options or Options()\n\n    # this route should take priority so set up it up first\n    _setup_single_view_dispatcher_route(options, app, component)\n\n    _setup_common_routes(options, app)\n\n\n# BackendType.create_development_app\ndef create_development_app() -> Starlette:\n    \"\"\"Return a :class:`Starlette` app instance in debug mode\"\"\"\n    return Starlette(debug=True)\n\n\n# BackendType.serve_development_app\nasync def serve_development_app(\n    app: Starlette,\n    host: str,\n    port: int,\n    started: asyncio.Event | None = None,\n) -> None:\n    \"\"\"Run a development server for starlette\"\"\"\n    await serve_with_uvicorn(app, host, port, started)\n\n\ndef use_websocket() -> WebSocket:\n    \"\"\"Get the current WebSocket object\"\"\"\n    return use_connection().carrier\n\n\ndef use_connection() -> Connection[WebSocket]:\n    conn = _use_connection()\n    if not isinstance(conn.carrier, WebSocket):  # nocov\n        msg = f\"Connection has unexpected carrier {conn.carrier}. Are you running with a Flask server?\"\n        raise TypeError(msg)\n    return conn\n\n\ndef _setup_common_routes(options: Options, app: Starlette) -> None:\n    cors_options = options.cors\n    if cors_options:  # nocov\n        cors_params = (\n            cors_options if isinstance(cors_options, dict) else {\"allow_origins\": [\"*\"]}\n        )\n        app.add_middleware(CORSMiddleware, **cors_params)\n\n    # This really should be added to the APIRouter, but there's a bug in Starlette\n    # BUG: https://github.com/tiangolo/fastapi/issues/1469\n    url_prefix = options.url_prefix\n\n    app.mount(\n        str(MODULES_PATH),\n        StaticFiles(directory=REACTPY_WEB_MODULES_DIR.current, check_dir=False),\n    )\n    app.mount(\n        str(ASSETS_PATH),\n        StaticFiles(directory=CLIENT_BUILD_DIR / \"assets\", check_dir=False),\n    )\n    # register this last so it takes least priority\n    index_route = _make_index_route(options)\n\n    if options.serve_index_route:\n        app.add_route(f\"{url_prefix}/\", index_route)\n        app.add_route(url_prefix + \"/{path:path}\", index_route)\n\n\ndef _make_index_route(options: Options) -> Callable[[Request], Awaitable[HTMLResponse]]:\n    index_html = read_client_index_html(options)\n\n    async def serve_index(request: Request) -> HTMLResponse:\n        return HTMLResponse(index_html)\n\n    return serve_index\n\n\ndef _setup_single_view_dispatcher_route(\n    options: Options, app: Starlette, component: RootComponentConstructor\n) -> None:\n    async def model_stream(socket: WebSocket) -> None:\n        await socket.accept()\n        send, recv = _make_send_recv_callbacks(socket)\n\n        pathname = \"/\" + socket.scope[\"path_params\"].get(\"path\", \"\")\n...\n", "needle_token_start": 5565, "needle_token_end": 5652, "code_context_ntokens": 16019, "output": ["```python\ndef use(\n        self,\n        function: Callable[..., Any],\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n    ) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function),\n            stop_propagation,\n            prevent_default,\n            self.target,\n        )\n```"]}
{"repo": "reactive-python/reactpy", "name": "_equal_files", "language": "python", "path": "src/py/reactpy/reactpy/web/module.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To determine if two specified files are identical.\n2. **Input**: Two file paths.\n3. **Output**: A boolean value indicating whether the files are the same.\n4. **Procedure**: The function first resolves the paths of the input files. If either file is a symbolic link, it checks if the resolved paths are identical. If not, it performs a deep comparison of the files' contents to determine if they are the same.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/html.py\n\"\"\"\n\n**Fragment**\n\n- :func:`_`\n\n**Document metadata**\n\n- :func:`base`\n- :func:`head`\n- :func:`link`\n- :func:`meta`\n- :func:`style`\n- :func:`title`\n\n**Content sectioning**\n\n- :func:`address`\n- :func:`article`\n- :func:`aside`\n- :func:`footer`\n- :func:`header`\n- :func:`h1`\n- :func:`h2`\n- :func:`h3`\n- :func:`h4`\n- :func:`h5`\n- :func:`h6`\n- :func:`main`\n- :func:`nav`\n- :func:`section`\n\n**Text content**\n\n- :func:`blockquote`\n- :func:`dd`\n- :func:`div`\n- :func:`dl`\n- :func:`dt`\n- :func:`figcaption`\n- :func:`figure`\n- :func:`hr`\n- :func:`li`\n- :func:`ol`\n- :func:`p`\n- :func:`pre`\n- :func:`ul`\n\n**Inline text semantics**\n\n- :func:`a`\n- :func:`abbr`\n- :func:`b`\n- :func:`bdi`\n- :func:`bdo`\n- :func:`br`\n- :func:`cite`\n- :func:`code`\n- :func:`data`\n- :func:`em`\n- :func:`i`\n- :func:`kbd`\n- :func:`mark`\n- :func:`q`\n- :func:`rp`\n- :func:`rt`\n- :func:`ruby`\n- :func:`s`\n- :func:`samp`\n- :func:`small`\n- :func:`span`\n- :func:`strong`\n- :func:`sub`\n- :func:`sup`\n- :func:`time`\n- :func:`u`\n- :func:`var`\n- :func:`wbr`\n\n**Image and video**\n\n- :func:`area`\n- :func:`audio`\n- :func:`img`\n- :func:`map`\n- :func:`track`\n- :func:`video`\n\n**Embedded content**\n\n- :func:`embed`\n- :func:`iframe`\n- :func:`object`\n- :func:`param`\n- :func:`picture`\n- :func:`portal`\n- :func:`source`\n\n**SVG and MathML**\n\n- :func:`svg`\n- :func:`math`\n\n**Scripting**\n\n- :func:`canvas`\n- :func:`noscript`\n- :func:`script`\n\n**Demarcating edits**\n\n- :func:`del_`\n- :func:`ins`\n\n**Table content**\n\n- :func:`caption`\n- :func:`col`\n- :func:`colgroup`\n- :func:`table`\n- :func:`tbody`\n- :func:`td`\n- :func:`tfoot`\n- :func:`th`\n- :func:`thead`\n- :func:`tr`\n\n**Forms**\n\n- :func:`button`\n- :func:`fieldset`\n- :func:`form`\n- :func:`input`\n- :func:`label`\n- :func:`legend`\n- :func:`meter`\n- :func:`option`\n- :func:`output`\n- :func:`progress`\n- :func:`select`\n- :func:`textarea`\n\n**Interactive elements**\n\n- :func:`details`\n- :func:`dialog`\n- :func:`menu`\n- :func:`menuitem`\n- :func:`summary`\n\n**Web components**\n\n- :func:`slot`\n- :func:`template`\n\n.. autofunction:: _\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom collections.abc import Sequence\n\n...\n# Path: src/py/reactpy/reactpy/logging.py\nimport logging\nimport sys\nfrom logging.config import dictConfig\n\nfrom reactpy.config import REACTPY_DEBUG_MODE\n\ndictConfig(\n    {\n        \"version\": 1,\n        \"disable_existing_loggers\": False,\n        \"loggers\": {\n            \"reactpy\": {\"handlers\": [\"console\"]},\n        },\n        \"handlers\": {\n            \"console\": {\n                \"class\": \"logging.StreamHandler\",\n                \"formatter\": \"generic\",\n                \"stream\": sys.stdout,\n            }\n        },\n        \"formatters\": {\n            \"generic\": {\n                \"format\": \"%(asctime)s | %(log_color)s%(levelname)s%(reset)s | %(message)s\",\n                \"datefmt\": r\"%Y-%m-%dT%H:%M:%S%z\",\n                \"class\": \"colorlog.ColoredFormatter\",\n            }\n        },\n    }\n)\n\n\nROOT_LOGGER = logging.getLogger(\"reactpy\")\n\"\"\"ReactPy's root logger instance\"\"\"\n\n\n@REACTPY_DEBUG_MODE.subscribe\ndef _set_debug_level(debug: bool) -> None:\n    if debug:\n        ROOT_LOGGER.setLevel(\"DEBUG\")\n        ROOT_LOGGER.debug(\"ReactPy is in debug mode\")\n    else:\n        ROOT_LOGGER.setLevel(\"INFO\")\n\n# Path: src/py/reactpy/reactpy/sample.py\nfrom __future__ import annotations\n\nfrom reactpy import html\nfrom reactpy.core.component import component\nfrom reactpy.core.types import VdomDict\n\n\n@component\ndef SampleApp() -> VdomDict:\n    return html.div(\n        {\"id\": \"sample\", \"style\": {\"padding\": \"15px\"}},\n        html.h1(\"Sample Application\"),\n        html.p(\n            \"This is a basic application made with ReactPy. Click \",\n            html.a(\n                {\"href\": \"https://pypi.org/project/reactpy/\", \"target\": \"_blank\"},\n                \"here\",\n            ),\n            \" to learn more.\",\n        ),\n    )\n\n# Path: src/py/reactpy/reactpy/svg.py\nfrom reactpy.core.vdom import make_vdom_constructor\n\n__all__ = (\n    \"a\",\n    \"animate\",\n    \"animate_motion\",\n    \"animate_transform\",\n    \"circle\",\n    \"clip_path\",\n    \"defs\",\n    \"desc\",\n    \"discard\",\n    \"ellipse\",\n    \"fe_blend\",\n    \"fe_color_matrix\",\n    \"fe_component_transfer\",\n    \"fe_composite\",\n    \"fe_convolve_matrix\",\n    \"fe_diffuse_lighting\",\n    \"fe_displacement_map\",\n    \"fe_distant_light\",\n    \"fe_drop_shadow\",\n    \"fe_flood\",\n    \"fe_func_a\",\n    \"fe_func_b\",\n    \"fe_func_g\",\n    \"fe_func_r\",\n    \"fe_gaussian_blur\",\n    \"fe_image\",\n    \"fe_merge\",\n    \"fe_merge_node\",\n    \"fe_morphology\",\n    \"fe_offset\",\n    \"fe_point_light\",\n    \"fe_specular_lighting\",\n    \"fe_spot_light\",\n    \"fe_tile\",\n    \"fe_turbulence\",\n    \"filter\",\n    \"foreign_object\",\n    \"g\",\n    \"hatch\",\n    \"hatchpath\",\n    \"image\",\n    \"line\",\n    \"linear_gradient\",\n    \"marker\",\n    \"mask\",\n    \"metadata\",\n    \"mpath\",\n    \"path\",\n    \"pattern\",\n    \"polygon\",\n    \"polyline\",\n    \"radial_gradient\",\n    \"rect\",\n    \"script\",\n    \"set\",\n    \"stop\",\n    \"style\",\n    \"svg\",\n    \"switch\",\n    \"symbol\",\n    \"text\",\n    \"text_path\",\n    \"title\",\n    \"tspan\",\n    \"use\",\n    \"view\",\n)\n\na = make_vdom_constructor(\"a\")\nanimate = make_vdom_constructor(\"animate\", allow_children=False)\nanimate_motion = make_vdom_constructor(\"animateMotion\", allow_children=False)\nanimate_transform = make_vdom_constructor(\"animateTransform\", allow_children=False)\ncircle = make_vdom_constructor(\"circle\", allow_children=False)\nclip_path = make_vdom_constructor(\"clipPath\")\ndefs = make_vdom_constructor(\"defs\")\ndesc = make_vdom_constructor(\"desc\", allow_children=False)\ndiscard = make_vdom_constructor(\"discard\", allow_children=False)\nellipse = make_vdom_constructor(\"ellipse\", allow_children=False)\nfe_blend = make_vdom_constructor(\"feBlend\", allow_children=False)\nfe_color_matrix = make_vdom_constructor(\"feColorMatrix\", allow_children=False)\nfe_component_transfer = make_vdom_constructor(\n    \"feComponentTransfer\", allow_children=False\n)\nfe_composite = make_vdom_constructor(\"feComposite\", allow_children=False)\nfe_convolve_matrix = make_vdom_constructor(\"feConvolveMatrix\", allow_children=False)\nfe_diffuse_lighting = make_vdom_constructor(\"feDiffuseLighting\", allow_children=False)\nfe_displacement_map = make_vdom_constructor(\"feDisplacementMap\", allow_children=False)\nfe_distant_light = make_vdom_constructor(\"feDistantLight\", allow_children=False)\nfe_drop_shadow = make_vdom_constructor(\"feDropShadow\", allow_children=False)\nfe_flood = make_vdom_constructor(\"feFlood\", allow_children=False)\nfe_func_a = make_vdom_constructor(\"feFuncA\", allow_children=False)\nfe_func_b = make_vdom_constructor(\"feFuncB\", allow_children=False)\nfe_func_g = make_vdom_constructor(\"feFuncG\", allow_children=False)\nfe_func_r = make_vdom_constructor(\"feFuncR\", allow_children=False)\nfe_gaussian_blur = make_vdom_constructor(\"feGaussianBlur\", allow_children=False)\nfe_image = make_vdom_constructor(\"feImage\", allow_children=False)\nfe_merge = make_vdom_constructor(\"feMerge\", allow_children=False)\nfe_merge_node = make_vdom_constructor(\"feMergeNode\", allow_children=False)\nfe_morphology = make_vdom_constructor(\"feMorphology\", allow_children=False)\nfe_offset = make_vdom_constructor(\"feOffset\", allow_children=False)\nfe_point_light = make_vdom_constructor(\"fePointLight\", allow_children=False)\nfe_specular_lighting = make_vdom_constructor(\"feSpecularLighting\", allow_children=False)\nfe_spot_light = make_vdom_constructor(\"feSpotLight\", allow_children=False)\nfe_tile = make_vdom_constructor(\"feTile\", allow_children=False)\nfe_turbulence = make_vdom_constructor(\"feTurbulence\", allow_children=False)\nfilter = make_vdom_constructor(\"filter\", allow_children=False)  # noqa: A001\nforeign_object = make_vdom_constructor(\"foreignObject\", allow_children=False)\ng = make_vdom_constructor(\"g\")\nhatch = make_vdom_constructor(\"hatch\", allow_children=False)\nhatchpath = make_vdom_constructor(\"hatchpath\", allow_children=False)\nimage = make_vdom_constructor(\"image\", allow_children=False)\nline = make_vdom_constructor(\"line\", allow_children=False)\nlinear_gradient = make_vdom_constructor(\"linearGradient\", allow_children=False)\nmarker = make_vdom_constructor(\"marker\")\nmask = make_vdom_constructor(\"mask\")\nmetadata = make_vdom_constructor(\"metadata\", allow_children=False)\nmpath = make_vdom_constructor(\"mpath\", allow_children=False)\npath = make_vdom_constructor(\"path\", allow_children=False)\npattern = make_vdom_constructor(\"pattern\")\npolygon = make_vdom_constructor(\"polygon\", allow_children=False)\npolyline = make_vdom_constructor(\"polyline\", allow_children=False)\nradial_gradient = make_vdom_constructor(\"radialGradient\", allow_children=False)\nrect = make_vdom_constructor(\"rect\", allow_children=False)\nscript = make_vdom_constructor(\"script\", allow_children=False)\nset = make_vdom_constructor(\"set\", allow_children=False)  # noqa: A001\nstop = make_vdom_constructor(\"stop\", allow_children=False)\nstyle = make_vdom_constructor(\"style\", allow_children=False)\nsvg = make_vdom_constructor(\"svg\")\nswitch = make_vdom_constructor(\"switch\")\nsymbol = make_vdom_constructor(\"symbol\")\ntext = make_vdom_constructor(\"text\", allow_children=False)\ntext_path = make_vdom_constructor(\"textPath\", allow_children=False)\ntitle = make_vdom_constructor(\"title\", allow_children=False)\ntspan = make_vdom_constructor(\"tspan\", allow_children=False)\nuse = make_vdom_constructor(\"use\", allow_children=False)\nview = make_vdom_constructor(\"view\", allow_children=False)\n\n# Path: src/py/reactpy/reactpy/web/utils.py\nimport logging\nimport re\nfrom pathlib import Path, PurePosixPath\nfrom urllib.parse import urlparse, urlunparse\n\nimport requests\n\nlogger = logging.getLogger(__name__)\n\n\ndef module_name_suffix(name: str) -> str:\n    if name.startswith(\"@\"):\n        name = name[1:]\n    head, _, tail = name.partition(\"@\")  # handle version identifier\n    version, _, tail = tail.partition(\"/\")  # get section after version\n    return PurePosixPath(tail or head).suffix or \".js\"\n\n\ndef resolve_module_exports_from_file(\n    file: Path,\n    max_depth: int,\n    is_re_export: bool = False,\n) -> set[str]:\n    if max_depth == 0:\n        logger.warning(f\"Did not resolve all exports for {file} - max depth reached\")\n        return set()\n    elif not file.exists():\n        logger.warning(f\"Did not resolve exports for unknown file {file}\")\n        return set()\n\n    export_names, references = resolve_module_exports_from_source(\n        file.read_text(encoding=\"utf-8\"), exclude_default=is_re_export\n    )\n\n    for ref in references:\n        if urlparse(ref).scheme:  # is an absolute URL\n            export_names.update(\n                resolve_module_exports_from_url(ref, max_depth - 1, is_re_export=True)\n            )\n        else:\n            path = file.parent.joinpath(*ref.split(\"/\"))\n            export_names.update(\n                resolve_module_exports_from_file(path, max_depth - 1, is_re_export=True)\n            )\n\n    return export_names\n\n\ndef resolve_module_exports_from_url(\n    url: str,\n    max_depth: int,\n    is_re_export: bool = False,\n) -> set[str]:\n    if max_depth == 0:\n        logger.warning(f\"Did not resolve all exports for {url} - max depth reached\")\n        return set()\n\n    try:\n        text = requests.get(url, timeout=5).text\n    except requests.exceptions.ConnectionError as error:\n        reason = \"\" if error is None else \" - {error.errno}\"\n        logger.warning(\"Did not resolve exports for url \" + url + reason)\n        return set()\n\n    export_names, references = resolve_module_exports_from_source(\n        text, exclude_default=is_re_export\n    )\n\n    for ref in references:\n        url = _resolve_relative_url(url, ref)\n        export_names.update(\n            resolve_module_exports_from_url(url, max_depth - 1, is_re_export=True)\n        )\n\n    return export_names\n\n\ndef resolve_module_exports_from_source(\n    content: str, exclude_default: bool\n) -> tuple[set[str], set[str]]:\n    names: set[str] = set()\n    references: set[str] = set()\n\n    if _JS_DEFAULT_EXPORT_PATTERN.search(content):\n        names.add(\"default\")\n\n    # Exporting functions and classes\n    names.update(_JS_FUNC_OR_CLS_EXPORT_PATTERN.findall(content))\n\n    for export in _JS_GENERAL_EXPORT_PATTERN.findall(content):\n        export = export.rstrip(\";\").strip()\n        # Exporting individual features\n        if export.startswith(\"let \"):\n            names.update(let.split(\"=\", 1)[0] for let in export[4:].split(\",\"))\n        # Renaming exports and export list\n        elif export.startswith(\"{\") and export.endswith(\"}\"):\n            names.update(\n                item.split(\" as \", 1)[-1] for item in export.strip(\"{}\").split(\",\")\n            )\n        # Exporting destructured assignments with renaming\n        elif export.startswith(\"const \"):\n            names.update(\n                item.split(\":\", 1)[0]\n                for item in export[6:].split(\"=\", 1)[0].strip(\"{}\").split(\",\")\n            )\n        # Default exports\n        elif export.startswith(\"default \"):\n            names.add(\"default\")\n        # Aggregating modules\n        elif export.startswith(\"* as \"):\n            names.add(export[5:].split(\" from \", 1)[0])\n        elif export.startswith(\"* \"):\n            references.add(export[2:].split(\"from \", 1)[-1].strip(\"'\\\"\"))\n        elif export.startswith(\"{\") and \" from \" in export:\n            names.update(\n                item.split(\" as \", 1)[-1]\n                for item in export.split(\" from \")[0].strip(\"{}\").split(\",\")\n            )\n        elif not (export.startswith(\"function \") or export.startswith(\"class \")):\n            logger.warning(f\"Unknown export type {export!r}\")\n\n    names = {n.strip() for n in names}\n    references = {r.strip() for r in references}\n\n    if exclude_default and \"default\" in names:\n        names.remove(\"default\")\n\n    return names, references\n\n\ndef _resolve_relative_url(base_url: str, rel_url: str) -> str:\n    if not rel_url.startswith(\".\"):\n        if rel_url.startswith(\"/\"):\n            # copy scheme and hostname from base_url\n            return urlunparse(urlparse(base_url)[:2] + urlparse(rel_url)[2:])\n        else:\n            return rel_url\n\n    base_url = base_url.rsplit(\"/\", 1)[0]\n\n    if rel_url.startswith(\"./\"):\n        return base_url + rel_url[1:]\n\n    while rel_url.startswith(\"../\"):\n        base_url = base_url.rsplit(\"/\", 1)[0]\n        rel_url = rel_url[3:]\n\n    return f\"{base_url}/{rel_url}\"\n\n\n_JS_DEFAULT_EXPORT_PATTERN = re.compile(\n    r\";?\\s*export\\s+default\\s\",\n)\n_JS_FUNC_OR_CLS_EXPORT_PATTERN = re.compile(\n    r\";?\\s*export\\s+(?:function|class)\\s+([a-zA-Z_$][0-9a-zA-Z_$]*)\"\n)\n_JS_GENERAL_EXPORT_PATTERN = re.compile(\n    r\"(?:^|;|})\\s*export(?=\\s+|{)(.*?)(?=;|$)\", re.MULTILINE\n)\n\n# Path: src/py/reactpy/reactpy/web/module.py\nfrom __future__ import annotations\n\nimport filecmp\nimport logging\nimport shutil\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom string import Template\nfrom typing import Any, NewType, overload\nfrom urllib.parse import urlparse\n\nfrom reactpy._warnings import warn\nfrom reactpy.config import REACTPY_DEBUG_MODE, REACTPY_WEB_MODULES_DIR\nfrom reactpy.core.types import ImportSourceDict, VdomDictConstructor\nfrom reactpy.core.vdom import make_vdom_constructor\nfrom reactpy.web.utils import (\n    module_name_suffix,\n    resolve_module_exports_from_file,\n    resolve_module_exports_from_url,\n)\n\nlogger = logging.getLogger(__name__)\n\nSourceType = NewType(\"SourceType\", str)\n\nNAME_SOURCE = SourceType(\"NAME\")\n\"\"\"A named source - usually a Javascript package name\"\"\"\n\nURL_SOURCE = SourceType(\"URL\")\n\"\"\"A source loaded from a URL, usually a CDN\"\"\"\n\n\ndef module_from_url(\n    url: str,\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n) -> WebModule:\n    \"\"\"Load a :class:`WebModule` from a :data:`URL_SOURCE`\n\n    Parameters:\n        url:\n            Where the javascript module will be loaded from which conforms to the\n            interface for :ref:`Custom Javascript Components`\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n    \"\"\"\n    return WebModule(\n        source=url,\n        source_type=URL_SOURCE,\n        default_fallback=fallback,\n        file=None,\n        export_names=(\n            resolve_module_exports_from_url(url, resolve_exports_depth)\n            if (\n                resolve_exports\n                if resolve_exports is not None\n                else REACTPY_DEBUG_MODE.current\n            )\n            else None\n        ),\n        unmount_before_update=unmount_before_update,\n    )\n\n\n_FROM_TEMPLATE_DIR = \"__from_template__\"\n\n\ndef module_from_template(\n    template: str,\n    package: str,\n    cdn: str = \"https://esm.sh\",\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n) -> WebModule:\n    \"\"\"Create a :class:`WebModule` from a framework template\n\n    This is useful for experimenting with component libraries that do not already\n    support ReactPy's :ref:`Custom Javascript Component` interface.\n\n    .. warning::\n\n        This approach is not recommended for use in a production setting because the\n        framework templates may use unpinned dependencies that could change without\n        warning. It's best to author a module adhering to the\n        :ref:`Custom Javascript Component` interface instead.\n\n    **Templates**\n\n    - ``react``: for modules exporting React components\n\n    Parameters:\n        template:\n            The name of the framework template to use with the given ``package``.\n        package:\n            The name of a package to load. May include a file extension (defaults to\n            ``.js`` if not given)\n        cdn:\n            Where the package should be loaded from. The CDN must distribute ESM modules\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n    \"\"\"\n    warn(\n        \"module_from_template() is deprecated due to instability - use the Javascript \"\n        \"Components API instead. This function will be removed in a future release.\",\n        DeprecationWarning,\n    )\n    template_name, _, template_version = template.partition(\"@\")\n    template_version = \"@\" + template_version if template_version else \"\"\n\n    # We do this since the package may be any valid URL path. Thus we may need to strip\n    # object parameters or query information so we save the resulting template under the\n    # correct file name.\n    package_name = urlparse(package).path\n\n    # downstream code assumes no trailing slash\n    cdn = cdn.rstrip(\"/\")\n\n    template_file_name = template_name + module_name_suffix(package_name)\n\n    template_file = Path(__file__).parent / \"templates\" / template_file_name\n    if not template_file.exists():\n        msg = f\"No template for {template_file_name!r} exists\"\n        raise ValueError(msg)\n\n    variables = {\"PACKAGE\": package, \"CDN\": cdn, \"VERSION\": template_version}\n    content = Template(template_file.read_text(encoding=\"utf-8\")).substitute(variables)\n\n    return module_from_string(\n        _FROM_TEMPLATE_DIR + \"/\" + package_name,\n        content,\n        fallback,\n        resolve_exports,\n        resolve_exports_depth,\n        unmount_before_update=unmount_before_update,\n    )\n\n\ndef module_from_file(\n    name: str,\n    file: str | Path,\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n    symlink: bool = False,\n) -> WebModule:\n    \"\"\"Load a :class:`WebModule` from a given ``file``\n\n    Parameters:\n        name:\n            The name of the package\n        file:\n            The file from which the content of the web module will be created.\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n        symlink:\n            Whether the web module should be saved as a symlink to the given ``file``.\n    \"\"\"\n    name += module_name_suffix(name)\n\n    source_file = Path(file).resolve()\n    target_file = _web_module_path(name)\n    if not source_file.exists():\n        msg = f\"Source file does not exist: {source_file}\"\n        raise FileNotFoundError(msg)\n\n    if not target_file.exists():\n        _copy_file(target_file, source_file, symlink)\n    elif not _equal_files(source_file, target_file):\n        logger.info(\n            f\"Existing web module {name!r} will \"\n            f\"be replaced with {target_file.resolve()}\"\n        )\n        target_file.unlink()\n        _copy_file(target_file, source_file, symlink)\n\n    return WebModule(\n        source=name,\n        source_type=NAME_SOURCE,\n        default_fallback=fallback,\n        file=target_file,\n        export_names=(\n            resolve_module_exports_from_file(source_file, resolve_exports_depth)\n            if (\n                resolve_exports\n                if resolve_exports is not None\n                else REACTPY_DEBUG_MODE.current\n            )\n            else None\n        ),\n        unmount_before_update=unmount_before_update,\n    )\n\n\n\ndef _equal_files(f1: Path, f2: Path) -> bool:\n    f1 = f1.resolve()\n    f2 = f2.resolve()\n    return (\n        (f1.is_symlink() or f2.is_symlink()) and (f1.resolve() == f2.resolve())\n    ) or filecmp.cmp(str(f1), str(f2), shallow=False)\n\n\ndef _copy_file(target: Path, source: Path, symlink: bool) -> None:\n    target.parent.mkdir(parents=True, exist_ok=True)\n    if symlink:\n        target.symlink_to(source)\n    else:\n        shutil.copy(source, target)\n\n\ndef module_from_string(\n    name: str,\n    content: str,\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n) -> WebModule:\n    \"\"\"Load a :class:`WebModule` whose ``content`` comes from a string.\n\n    Parameters:\n        name:\n            The name of the package\n        content:\n            The contents of the web module\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n    \"\"\"\n    name += module_name_suffix(name)\n\n    target_file = _web_module_path(name)\n\n    if target_file.exists() and target_file.read_text(encoding=\"utf-8\") != content:\n        logger.info(\n            f\"Existing web module {name!r} will \"\n            f\"be replaced with {target_file.resolve()}\"\n        )\n        target_file.unlink()\n\n    target_file.parent.mkdir(parents=True, exist_ok=True)\n    target_file.write_text(content)\n\n    return WebModule(\n        source=name,\n        source_type=NAME_SOURCE,\n        default_fallback=fallback,\n        file=target_file,\n        export_names=(\n            resolve_module_exports_from_file(target_file, resolve_exports_depth)\n            if (\n                resolve_exports\n                if resolve_exports is not None\n                else REACTPY_DEBUG_MODE.current\n            )\n            else None\n        ),\n        unmount_before_update=unmount_before_update,\n    )\n\n\n@dataclass(frozen=True)\nclass WebModule:\n    source: str\n    source_type: SourceType\n    default_fallback: Any | None\n    export_names: set[str] | None\n    file: Path | None\n    unmount_before_update: bool\n\n\n@overload\ndef export(\n    web_module: WebModule,\n    export_names: str,\n    fallback: Any | None = ...,\n    allow_children: bool = ...,\n) -> VdomDictConstructor: ...\n\n\n@overload\ndef export(\n    web_module: WebModule,\n    export_names: list[str] | tuple[str, ...],\n    fallback: Any | None = ...,\n    allow_children: bool = ...,\n) -> list[VdomDictConstructor]: ...\n\n\ndef export(\n    web_module: WebModule,\n    export_names: str | list[str] | tuple[str, ...],\n    fallback: Any | None = None,\n    allow_children: bool = True,\n) -> VdomDictConstructor | list[VdomDictConstructor]:\n    \"\"\"Return one or more VDOM constructors from a :class:`WebModule`\n\n    Parameters:\n        export_names:\n            One or more names to export. If given as a string, a single component\n            will be returned. If a list is given, then a list of components will be\n            returned.\n        fallback:\n            What to temporarily display while the module is being loaded.\n        allow_children:\n            Whether or not these components can have children.\n    \"\"\"\n    if isinstance(export_names, str):\n        if (\n            web_module.export_names is not None\n            and export_names not in web_module.export_names\n        ):\n            msg = f\"{web_module.source!r} does not export {export_names!r}\"\n            raise ValueError(msg)\n        return _make_export(web_module, export_names, fallback, allow_children)\n    else:\n        if web_module.export_names is not None:\n            missing = sorted(set(export_names).difference(web_module.export_names))\n            if missing:\n                msg = f\"{web_module.source!r} does not export {missing!r}\"\n                raise ValueError(msg)\n        return [\n            _make_export(web_module, name, fallback, allow_children)\n            for name in export_names\n        ]\n\n\ndef _make_export(\n    web_module: WebModule,\n    name: str,\n    fallback: Any | None,\n    allow_children: bool,\n) -> VdomDictConstructor:\n    return make_vdom_constructor(\n        name,\n        allow_children=allow_children,\n        import_source=ImportSourceDict(\n            source=web_module.source,\n            sourceType=web_module.source_type,\n            fallback=(fallback or web_module.default_fallback),\n            unmountBeforeUpdate=web_module.unmount_before_update,\n        ),\n    )\n\n\ndef _web_module_path(name: str) -> Path:\n    directory = REACTPY_WEB_MODULES_DIR.current\n    path = directory.joinpath(*name.split(\"/\"))\n    return path.with_suffix(path.suffix)\n\n# Path: src/py/reactpy/reactpy/web/__init__.py\nfrom reactpy.web.module import (\n    export,\n    module_from_file,\n    module_from_string,\n    module_from_template,\n    module_from_url,\n)\n\n__all__ = [\n    \"module_from_file\",\n    \"module_from_string\",\n    \"module_from_template\",\n    \"module_from_url\",\n    \"export\",\n]\n\n# Path: src/py/reactpy/reactpy/backend/default.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom logging import getLogger\nfrom sys import exc_info\nfrom typing import Any, NoReturn\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.backend.utils import SUPPORTED_BACKENDS, all_implementations\nfrom reactpy.types import RootComponentConstructor\n\nlogger = getLogger(__name__)\n_DEFAULT_IMPLEMENTATION: BackendType[Any] | None = None\n\n\n# BackendType.Options\nclass Options:  # nocov\n    \"\"\"Configuration options that can be provided to the backend.\n    This definition should not be used/instantiated. It exists only for\n    type hinting purposes.\"\"\"\n\n    def __init__(self, *args: Any, **kwds: Any) -> NoReturn:\n        msg = \"Default implementation has no options.\"\n        raise ValueError(msg)\n\n\n# BackendType.configure\ndef configure(\n    app: Any, component: RootComponentConstructor, options: None = None\n) -> None:\n    \"\"\"Configure the given app instance to display the given component\"\"\"\n    if options is not None:  # nocov\n        msg = \"Default implementation cannot be configured with options\"\n        raise ValueError(msg)\n    return _default_implementation().configure(app, component)\n\n\n# BackendType.create_development_app\ndef create_development_app() -> Any:\n    \"\"\"Create an application instance for development purposes\"\"\"\n    return _default_implementation().create_development_app()\n\n\n# BackendType.serve_development_app\nasync def serve_development_app(\n    app: Any,\n    host: str,\n    port: int,\n    started: asyncio.Event | None = None,\n) -> None:\n    \"\"\"Run an application using a development server\"\"\"\n    return await _default_implementation().serve_development_app(\n        app, host, port, started\n    )\n\n\ndef _default_implementation() -> BackendType[Any]:\n    \"\"\"Get the first available server implementation\"\"\"\n    global _DEFAULT_IMPLEMENTATION  # noqa: PLW0603\n\n    if _DEFAULT_IMPLEMENTATION is not None:\n        return _DEFAULT_IMPLEMENTATION\n\n    try:\n        implementation = next(all_implementations())\n    except StopIteration:  # nocov\n        logger.debug(\"Backend implementation import failed\", exc_info=exc_info())\n        supported_backends = \", \".join(SUPPORTED_BACKENDS)\n        msg = (\n            \"It seems you haven't installed a backend. To resolve this issue, \"\n            \"you can install a backend by running:\\n\\n\"\n            '\\033[1mpip install \"reactpy[starlette]\"\\033[0m\\n\\n'\n            f\"Other supported backends include: {supported_backends}.\"\n        )\n        raise RuntimeError(msg) from None\n    else:\n        _DEFAULT_IMPLEMENTATION = implementation\n        return implementation\n\n# Path: src/py/reactpy/reactpy/testing/logs.py\nfrom __future__ import annotations\n\nimport logging\nimport re\nfrom collections.abc import Iterator\nfrom contextlib import contextmanager\nfrom traceback import format_exception\nfrom typing import Any, NoReturn\n\nfrom reactpy.logging import ROOT_LOGGER\n\n\nclass LogAssertionError(AssertionError):\n    \"\"\"An assertion error raised in relation to log messages.\"\"\"\n\n\n@contextmanager\ndef assert_reactpy_did_log(\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> Iterator[None]:\n    \"\"\"Assert that ReactPy produced a log matching the described message or error.\n\n    Args:\n        match_message: Must match a logged message.\n        error_type: Checks the type of logged exceptions.\n        match_error: Must match an error message.\n    \"\"\"\n    message_pattern = re.compile(match_message)\n    error_pattern = re.compile(match_error)\n\n    with capture_reactpy_logs() as log_records:\n        try:\n            yield None\n        except Exception:\n            raise\n        else:\n            for record in list(log_records):\n                if (\n                    # record message matches\n                    message_pattern.findall(record.getMessage())\n                    # error type matches\n                    and (\n                        error_type is None\n                        or (\n                            record.exc_info is not None\n                            and record.exc_info[0] is not None\n                            and issubclass(record.exc_info[0], error_type)\n                        )\n                    )\n                    # error message pattern matches\n                    and (\n                        not match_error\n                        or (\n                            record.exc_info is not None\n                            and error_pattern.findall(\n                                \"\".join(format_exception(*record.exc_info))\n                            )\n                        )\n                    )\n                ):\n                    break\n            else:  # nocov\n                _raise_log_message_error(\n                    \"Could not find a log record matching the given\",\n                    match_message,\n                    error_type,\n                    match_error,\n                )\n\n\n@contextmanager\ndef assert_reactpy_did_not_log(\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> Iterator[None]:\n    \"\"\"Assert the inverse of :func:`assert_reactpy_logged`\"\"\"\n    try:\n        with assert_reactpy_did_log(match_message, error_type, match_error):\n            yield None\n    except LogAssertionError:\n        pass\n    else:\n        _raise_log_message_error(\n            \"Did find a log record matching the given\",\n            match_message,\n            error_type,\n            match_error,\n        )\n\n\ndef list_logged_exceptions(\n    log_records: list[logging.LogRecord],\n    pattern: str = \"\",\n    types: type[Any] | tuple[type[Any], ...] = Exception,\n    log_level: int = logging.ERROR,\n    del_log_records: bool = True,\n) -> list[BaseException]:\n    \"\"\"Return a list of logged exception matching the given criteria\n\n    Args:\n        log_level: The level of log to check\n        exclude_exc_types: Any exception types to ignore\n        del_log_records: Whether to delete the log records for yielded exceptions\n    \"\"\"\n    found: list[BaseException] = []\n    compiled_pattern = re.compile(pattern)\n    for index, record in enumerate(log_records):\n        if record.levelno >= log_level and record.exc_info:\n            error = record.exc_info[1]\n            if (\n                error is not None\n                and isinstance(error, types)\n                and compiled_pattern.search(str(error))\n            ):\n                if del_log_records:\n                    del log_records[index - len(found)]\n                found.append(error)\n    return found\n\n\n@contextmanager\ndef capture_reactpy_logs() -> Iterator[list[logging.LogRecord]]:\n    \"\"\"Capture logs from ReactPy\n\n    Any logs produced in this context are cleared afterwards\n    \"\"\"\n    original_level = ROOT_LOGGER.level\n    ROOT_LOGGER.setLevel(logging.DEBUG)\n    try:\n        if _LOG_RECORD_CAPTOR in ROOT_LOGGER.handlers:\n            start_index = len(_LOG_RECORD_CAPTOR.records)\n            try:\n                yield _LOG_RECORD_CAPTOR.records\n            finally:\n                end_index = len(_LOG_RECORD_CAPTOR.records)\n                _LOG_RECORD_CAPTOR.records[start_index:end_index] = []\n            return None\n\n        ROOT_LOGGER.addHandler(_LOG_RECORD_CAPTOR)\n        try:\n            yield _LOG_RECORD_CAPTOR.records\n        finally:\n            ROOT_LOGGER.removeHandler(_LOG_RECORD_CAPTOR)\n            _LOG_RECORD_CAPTOR.records.clear()\n    finally:\n        ROOT_LOGGER.setLevel(original_level)\n\n\nclass _LogRecordCaptor(logging.NullHandler):\n    def __init__(self) -> None:\n        self.records: list[logging.LogRecord] = []\n        super().__init__()\n\n    def handle(self, record: logging.LogRecord) -> bool:\n        self.records.append(record)\n        return True\n\n\n_LOG_RECORD_CAPTOR = _LogRecordCaptor()\n\n\ndef _raise_log_message_error(\n    prefix: str,\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> NoReturn:\n    conditions = []\n    if match_message:\n        conditions.append(f\"log message pattern {match_message!r}\")\n    if error_type:\n        conditions.append(f\"exception type {error_type}\")\n    if match_error:\n        conditions.append(f\"error message pattern {match_error!r}\")\n    raise LogAssertionError(prefix + \" \" + \" and \".join(conditions))\n\n# Path: src/py/reactpy/reactpy/testing/backend.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nfrom contextlib import AsyncExitStack, suppress\nfrom types import TracebackType\nfrom typing import Any, Callable\nfrom urllib.parse import urlencode, urlunparse\n\nfrom reactpy.backend import default as default_server\nfrom reactpy.backend.types import BackendType\nfrom reactpy.backend.utils import find_available_port\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.core.component import component\nfrom reactpy.core.hooks import use_callback, use_effect, use_state\nfrom reactpy.core.types import ComponentConstructor\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    capture_reactpy_logs,\n    list_logged_exceptions,\n)\nfrom reactpy.utils import Ref\n\n\nclass BackendFixture:\n    \"\"\"A test fixture for running a server and imperatively displaying views\n\n    This fixture is typically used alongside async web drivers like ``playwight``.\n\n    Example:\n        .. code-block::\n\n            async with BackendFixture() as server:\n                server.mount(MyComponent)\n    \"\"\"\n\n    _records: list[logging.LogRecord]\n    _server_future: asyncio.Task[Any]\n    _exit_stack = AsyncExitStack()\n\n    def __init__(\n        self,\n        host: str = \"127.0.0.1\",\n        port: int | None = None,\n        app: Any | None = None,\n        implementation: BackendType[Any] | None = None,\n        options: Any | None = None,\n        timeout: float | None = None,\n    ) -> None:\n        self.host = host\n        self.port = port or find_available_port(host)\n        self.mount, self._root_component = _hotswap()\n        self.timeout = (\n            REACTPY_TESTING_DEFAULT_TIMEOUT.current if timeout is None else timeout\n        )\n\n        if app is not None and implementation is None:\n            msg = \"If an application instance its corresponding server implementation must be provided too.\"\n            raise ValueError(msg)\n\n        self._app = app\n        self.implementation = implementation or default_server\n        self._options = options\n\n    @property\n    def log_records(self) -> list[logging.LogRecord]:\n        \"\"\"A list of captured log records\"\"\"\n        return self._records\n\n    def url(self, path: str = \"\", query: Any | None = None) -> str:\n        \"\"\"Return a URL string pointing to the host and point of the server\n\n        Args:\n            path: the path to a resource on the server\n            query: a dictionary or list of query parameters\n        \"\"\"\n        return urlunparse(\n            [\n                \"http\",\n                f\"{self.host}:{self.port}\",\n                path,\n                \"\",\n                urlencode(query or ()),\n                \"\",\n            ]\n        )\n\n    def list_logged_exceptions(\n        self,\n        pattern: str = \"\",\n        types: type[Any] | tuple[type[Any], ...] = Exception,\n        log_level: int = logging.ERROR,\n        del_log_records: bool = True,\n    ) -> list[BaseException]:\n        \"\"\"Return a list of logged exception matching the given criteria\n\n        Args:\n            log_level: The level of log to check\n            exclude_exc_types: Any exception types to ignore\n            del_log_records: Whether to delete the log records for yielded exceptions\n        \"\"\"\n        return list_logged_exceptions(\n            self.log_records,\n            pattern,\n            types,\n            log_level,\n            del_log_records,\n        )\n\n    async def __aenter__(self) -> BackendFixture:\n        self._exit_stack = AsyncExitStack()\n        self._records = self._exit_stack.enter_context(capture_reactpy_logs())\n\n        app = self._app or self.implementation.create_development_app()\n        self.implementation.configure(app, self._root_component, self._options)\n\n        started = asyncio.Event()\n        server_future = asyncio.create_task(\n            self.implementation.serve_development_app(\n                app, self.host, self.port, started\n            )\n        )\n\n        async def stop_server() -> None:\n            server_future.cancel()\n            with suppress(asyncio.CancelledError):\n                await asyncio.wait_for(server_future, timeout=self.timeout)\n\n        self._exit_stack.push_async_callback(stop_server)\n\n        try:\n            await asyncio.wait_for(started.wait(), timeout=self.timeout)\n        except Exception:  # nocov\n            # see if we can await the future for a more helpful error\n            await asyncio.wait_for(server_future, timeout=self.timeout)\n            raise\n\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        await self._exit_stack.aclose()\n\n        self.mount(None)  # reset the view\n\n        logged_errors = self.list_logged_exceptions(del_log_records=False)\n        if logged_errors:  # nocov\n            msg = \"Unexpected logged exception\"\n            raise LogAssertionError(msg) from logged_errors[0]\n\n\n_MountFunc = Callable[[\"Callable[[], Any] | None\"], None]\n\n\ndef _hotswap(update_on_change: bool = False) -> tuple[_MountFunc, ComponentConstructor]:\n    \"\"\"Swap out components from a layout on the fly.\n\n    Since you can't change the component functions used to create a layout\n    in an imperative manner, you can use ``hotswap`` to do this so\n    long as you set things up ahead of time.\n\n    Parameters:\n        update_on_change: Whether or not all views of the layout should be updated on a swap.\n\n    Example:\n        .. code-block:: python\n\n            import reactpy\n\n            show, root = reactpy.hotswap()\n            PerClientStateServer(root).run_in_thread(\"localhost\", 8765)\n\n            @reactpy.component\n            def DivOne(self):\n                return {\"tagName\": \"div\", \"children\": [1]}\n\n            show(DivOne)\n\n            # displaying the output now will show DivOne\n\n            @reactpy.component\n            def DivTwo(self):\n                return {\"tagName\": \"div\", \"children\": [2]}\n\n            show(DivTwo)\n\n            # displaying the output now will show DivTwo\n    \"\"\"\n    constructor_ref: Ref[Callable[[], Any]] = Ref(lambda: None)\n\n    if update_on_change:\n        set_constructor_callbacks: set[Callable[[Callable[[], Any]], None]] = set()\n\n        @component\n        def HotSwap() -> Any:\n            # new displays will adopt the latest constructor and arguments\n            constructor, _set_constructor = use_state(lambda: constructor_ref.current)\n            set_constructor = use_callback(lambda new: _set_constructor(lambda _: new))\n\n            def add_callback() -> Callable[[], None]:\n                set_constructor_callbacks.add(set_constructor)\n                return lambda: set_constructor_callbacks.remove(set_constructor)\n\n            use_effect(add_callback)\n\n            return constructor()\n\n        def swap(constructor: Callable[[], Any] | None) -> None:\n            constructor = constructor_ref.current = constructor or (lambda: None)\n\n            for set_constructor in set_constructor_callbacks:\n                set_constructor(constructor)\n\n    else:\n\n        @component\n        def HotSwap() -> Any:\n            return constructor_ref.current()\n\n        def swap(constructor: Callable[[], Any] | None) -> None:\n            constructor_ref.current = constructor or (lambda: None)\n\n    return swap, HotSwap\n\n# Path: src/py/reactpy/reactpy/testing/common.py\nfrom __future__ import annotations\n\nimport asyncio\nimport inspect\nimport shutil\nimport time\nfrom collections.abc import Awaitable\nfrom functools import wraps\nfrom typing import Any, Callable, Generic, TypeVar, cast\nfrom uuid import uuid4\nfrom weakref import ref\n\nfrom typing_extensions import ParamSpec\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT, REACTPY_WEB_MODULES_DIR\nfrom reactpy.core._life_cycle_hook import LifeCycleHook, current_hook\nfrom reactpy.core.events import EventHandler, to_event_handler_function\n\n\ndef clear_reactpy_web_modules_dir() -> None:\n    \"\"\"Clear the directory where ReactPy stores registered web modules\"\"\"\n    for path in REACTPY_WEB_MODULES_DIR.current.iterdir():\n        shutil.rmtree(path) if path.is_dir() else path.unlink()\n\n\n_P = ParamSpec(\"_P\")\n_R = TypeVar(\"_R\")\n\n\n_DEFAULT_POLL_DELAY = 0.1\n\n\nclass poll(Generic[_R]):  # noqa: N801\n    \"\"\"Wait until the result of an sync or async function meets some condition\"\"\"\n\n    def __init__(\n        self,\n        function: Callable[_P, Awaitable[_R] | _R],\n        *args: _P.args,\n        **kwargs: _P.kwargs,\n    ) -> None:\n        coro: Callable[_P, Awaitable[_R]]\n        if not inspect.iscoroutinefunction(function):\n\n            async def coro(*args: _P.args, **kwargs: _P.kwargs) -> _R:\n                return cast(_R, function(*args, **kwargs))\n\n        else:\n            coro = cast(Callable[_P, Awaitable[_R]], function)\n        self._func = coro\n        self._args = args\n        self._kwargs = kwargs\n\n    async def until(\n        self,\n        condition: Callable[[_R], bool],\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n        description: str = \"condition to be true\",\n    ) -> None:\n        \"\"\"Check that the coroutines result meets a condition within the timeout\"\"\"\n        started_at = time.time()\n        while True:\n            await asyncio.sleep(delay)\n            result = await self._func(*self._args, **self._kwargs)\n            if condition(result):\n                break\n            elif (time.time() - started_at) > timeout:  # nocov\n                msg = f\"Expected {description} after {timeout} seconds - last value was {result!r}\"\n                raise asyncio.TimeoutError(msg)\n\n    async def until_is(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is identical to the given value\"\"\"\n        return await self.until(\n            lambda left: left is right,\n            timeout,\n            delay,\n            f\"value to be identical to {right!r}\",\n        )\n\n    async def until_equals(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is equal to the given value\"\"\"\n        return await self.until(\n            lambda left: left == right,\n            timeout,\n            delay,\n            f\"value to equal {right!r}\",\n        )\n\n\nclass HookCatcher:\n    \"\"\"Utility for capturing a LifeCycleHook from a component\n\n    Example:\n        .. code-block::\n\n            hooks = HookCatcher(index_by_kwarg=\"thing\")\n\n            @reactpy.component\n            @hooks.capture\n            def MyComponent(thing):\n                ...\n\n            ...  # render the component\n\n            # grab the last render of where MyComponent(thing='something')\n            hooks.index[\"something\"]\n            # or grab the hook from the component's last render\n            hooks.latest\n\n        After the first render of ``MyComponent`` the ``HookCatcher`` will have\n        captured the component's ``LifeCycleHook``.\n    \"\"\"\n\n    latest: LifeCycleHook\n\n    def __init__(self, index_by_kwarg: str | None = None):\n        self.index_by_kwarg = index_by_kwarg\n        self.index: dict[Any, LifeCycleHook] = {}\n\n    def capture(self, render_function: Callable[..., Any]) -> Callable[..., Any]:\n        \"\"\"Decorator for capturing a ``LifeCycleHook`` on each render of a component\"\"\"\n\n        # The render function holds a reference to `self` and, via the `LifeCycleHook`,\n        # the component. Some tests check whether components are garbage collected, thus\n        # we must use a `ref` here to ensure these checks pass once the catcher itself\n        # has been collected.\n        self_ref = ref(self)\n\n        @wraps(render_function)\n        def wrapper(*args: Any, **kwargs: Any) -> Any:\n            self = self_ref()\n            if self is None:\n                raise RuntimeError(\"Hook catcher has been garbage collected\")\n\n            hook = current_hook()\n            if self.index_by_kwarg is not None:\n                self.index[kwargs[self.index_by_kwarg]] = hook\n            self.latest = hook\n            return render_function(*args, **kwargs)\n\n        return wrapper\n\n\nclass StaticEventHandler:\n    \"\"\"Utility for capturing the target of one event handler\n\n    Example:\n        .. code-block::\n\n            static_handler = StaticEventHandler()\n\n            @reactpy.component\n            def MyComponent():\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handler.use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # gives the target ID for onClick where from the last render of MyComponent\n            static_handlers.target\n\n        If you need to capture event handlers from different instances of a component\n        the you should create multiple ``StaticEventHandler`` instances.\n\n        .. code-block::\n\n            static_handlers_by_key = {\n                \"first\": StaticEventHandler(),\n                \"second\": StaticEventHandler(),\n            }\n\n            @reactpy.component\n            def Parent():\n                return reactpy.html.div(Child(key=\"first\"), Child(key=\"second\"))\n\n            @reactpy.component\n            def Child(key):\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handlers_by_key[key].use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # grab the individual targets for each instance above\n            first_target = static_handlers_by_key[\"first\"].target\n            second_target = static_handlers_by_key[\"second\"].target\n    \"\"\"\n\n    def __init__(self) -> None:\n        self.target = uuid4().hex\n\n    def use(\n        self,\n        function: Callable[..., Any],\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n    ) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function),\n            stop_propagation,\n            prevent_default,\n            self.target,\n        )\n\n# Path: src/py/reactpy/reactpy/testing/display.py\nfrom __future__ import annotations\n\nfrom contextlib import AsyncExitStack\nfrom types import TracebackType\nfrom typing import Any\n\nfrom playwright.async_api import (\n    Browser,\n    BrowserContext,\n    ElementHandle,\n    Page,\n    async_playwright,\n)\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.types import RootComponentConstructor\n\n\nclass DisplayFixture:\n    \"\"\"A fixture for running web-based tests using ``playwright``\"\"\"\n\n    _exit_stack: AsyncExitStack\n\n    def __init__(\n        self,\n        backend: BackendFixture | None = None,\n        driver: Browser | BrowserContext | Page | None = None,\n        url_prefix: str = \"\",\n    ) -> None:\n        if backend is not None:\n            self.backend = backend\n        if driver is not None:\n            if isinstance(driver, Page):\n                self.page = driver\n            else:\n                self._browser = driver\n        self.url_prefix = url_prefix\n\n    async def show(\n        self,\n        component: RootComponentConstructor,\n    ) -> None:\n        self.backend.mount(component)\n        await self.goto(\"/\")\n        await self.root_element()  # check that root element is attached\n\n    async def goto(\n        self, path: str, query: Any | None = None, add_url_prefix: bool = True\n    ) -> None:\n        await self.page.goto(\n            self.backend.url(\n                f\"{self.url_prefix}{path}\" if add_url_prefix else path, query\n            )\n        )\n\n    async def root_element(self) -> ElementHandle:\n        element = await self.page.wait_for_selector(\"#app\", state=\"attached\")\n        if element is None:  # nocov\n            msg = \"Root element not attached\"\n            raise RuntimeError(msg)\n        return element\n\n    async def __aenter__(self) -> DisplayFixture:\n        es = self._exit_stack = AsyncExitStack()\n\n        browser: Browser | BrowserContext\n        if not hasattr(self, \"page\"):\n            if not hasattr(self, \"_browser\"):\n                pw = await es.enter_async_context(async_playwright())\n                browser = await pw.chromium.launch()\n            else:\n                browser = self._browser\n            self.page = await browser.new_page()\n\n        self.page.set_default_timeout(REACTPY_TESTING_DEFAULT_TIMEOUT.current * 1000)\n\n        if not hasattr(self, \"backend\"):\n            self.backend = BackendFixture()\n            await es.enter_async_context(self.backend)\n\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        self.backend.mount(None)\n        await self._exit_stack.aclose()\n\n# Path: src/py/reactpy/reactpy/testing/__init__.py\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.testing.common import (\n    HookCatcher,\n    StaticEventHandler,\n    clear_reactpy_web_modules_dir,\n    poll,\n)\nfrom reactpy.testing.display import DisplayFixture\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    assert_reactpy_did_log,\n    assert_reactpy_did_not_log,\n    capture_reactpy_logs,\n)\n\n__all__ = [\n...\n", "needle_token_start": 7214, "needle_token_end": 7316, "code_context_ntokens": 16089, "output": ["```python\ndef _equal_files(f1: Path, f2: Path) -> bool:\n    f1 = f1.resolve()\n    f2 = f2.resolve()\n    return (\n        (f1.is_symlink() or f2.is_symlink()) and (f1.resolve() == f2.resolve())\n    ) or filecmp.cmp(str(f1), str(f2), shallow=False)\n```"]}
{"repo": "reactive-python/reactpy", "name": "stop_server", "language": "python", "path": "src/py/reactpy/reactpy/testing/backend.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to terminate a running asynchronous server task gracefully within a specified timeout period.\n2. **Input**: There are no direct inputs as it operates within its defined context and uses the class's attributes.\n3. **Output**: There is no direct output returned from the function; however, it ensures the server task is cancelled and awaited for proper cleanup.\n4. **Procedure**: The function first sends a cancellation request to the server task. It then waits for the task to acknowledge the cancellation and complete any necessary cleanup operations, enforcing a timeout limit to prevent indefinite waiting.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/web/utils.py\nimport logging\nimport re\nfrom pathlib import Path, PurePosixPath\nfrom urllib.parse import urlparse, urlunparse\n\nimport requests\n\nlogger = logging.getLogger(__name__)\n\n\ndef module_name_suffix(name: str) -> str:\n    if name.startswith(\"@\"):\n        name = name[1:]\n    head, _, tail = name.partition(\"@\")  # handle version identifier\n    version, _, tail = tail.partition(\"/\")  # get section after version\n    return PurePosixPath(tail or head).suffix or \".js\"\n\n\ndef resolve_module_exports_from_file(\n    file: Path,\n    max_depth: int,\n    is_re_export: bool = False,\n) -> set[str]:\n    if max_depth == 0:\n        logger.warning(f\"Did not resolve all exports for {file} - max depth reached\")\n        return set()\n    elif not file.exists():\n        logger.warning(f\"Did not resolve exports for unknown file {file}\")\n        return set()\n\n    export_names, references = resolve_module_exports_from_source(\n        file.read_text(encoding=\"utf-8\"), exclude_default=is_re_export\n    )\n\n    for ref in references:\n        if urlparse(ref).scheme:  # is an absolute URL\n            export_names.update(\n                resolve_module_exports_from_url(ref, max_depth - 1, is_re_export=True)\n            )\n        else:\n            path = file.parent.joinpath(*ref.split(\"/\"))\n            export_names.update(\n                resolve_module_exports_from_file(path, max_depth - 1, is_re_export=True)\n            )\n\n    return export_names\n\n\ndef resolve_module_exports_from_url(\n    url: str,\n    max_depth: int,\n    is_re_export: bool = False,\n) -> set[str]:\n    if max_depth == 0:\n        logger.warning(f\"Did not resolve all exports for {url} - max depth reached\")\n        return set()\n\n    try:\n        text = requests.get(url, timeout=5).text\n    except requests.exceptions.ConnectionError as error:\n        reason = \"\" if error is None else \" - {error.errno}\"\n        logger.warning(\"Did not resolve exports for url \" + url + reason)\n        return set()\n\n    export_names, references = resolve_module_exports_from_source(\n        text, exclude_default=is_re_export\n    )\n\n    for ref in references:\n        url = _resolve_relative_url(url, ref)\n        export_names.update(\n            resolve_module_exports_from_url(url, max_depth - 1, is_re_export=True)\n        )\n\n    return export_names\n\n\ndef resolve_module_exports_from_source(\n    content: str, exclude_default: bool\n) -> tuple[set[str], set[str]]:\n    names: set[str] = set()\n    references: set[str] = set()\n\n    if _JS_DEFAULT_EXPORT_PATTERN.search(content):\n        names.add(\"default\")\n\n    # Exporting functions and classes\n    names.update(_JS_FUNC_OR_CLS_EXPORT_PATTERN.findall(content))\n\n    for export in _JS_GENERAL_EXPORT_PATTERN.findall(content):\n        export = export.rstrip(\";\").strip()\n        # Exporting individual features\n        if export.startswith(\"let \"):\n            names.update(let.split(\"=\", 1)[0] for let in export[4:].split(\",\"))\n        # Renaming exports and export list\n        elif export.startswith(\"{\") and export.endswith(\"}\"):\n            names.update(\n                item.split(\" as \", 1)[-1] for item in export.strip(\"{}\").split(\",\")\n            )\n        # Exporting destructured assignments with renaming\n        elif export.startswith(\"const \"):\n            names.update(\n                item.split(\":\", 1)[0]\n                for item in export[6:].split(\"=\", 1)[0].strip(\"{}\").split(\",\")\n            )\n        # Default exports\n        elif export.startswith(\"default \"):\n            names.add(\"default\")\n        # Aggregating modules\n        elif export.startswith(\"* as \"):\n            names.add(export[5:].split(\" from \", 1)[0])\n        elif export.startswith(\"* \"):\n            references.add(export[2:].split(\"from \", 1)[-1].strip(\"'\\\"\"))\n        elif export.startswith(\"{\") and \" from \" in export:\n            names.update(\n                item.split(\" as \", 1)[-1]\n                for item in export.split(\" from \")[0].strip(\"{}\").split(\",\")\n            )\n        elif not (export.startswith(\"function \") or export.startswith(\"class \")):\n            logger.warning(f\"Unknown export type {export!r}\")\n\n    names = {n.strip() for n in names}\n    references = {r.strip() for r in references}\n\n    if exclude_default and \"default\" in names:\n        names.remove(\"default\")\n\n    return names, references\n\n\ndef _resolve_relative_url(base_url: str, rel_url: str) -> str:\n    if not rel_url.startswith(\".\"):\n        if rel_url.startswith(\"/\"):\n            # copy scheme and hostname from base_url\n            return urlunparse(urlparse(base_url)[:2] + urlparse(rel_url)[2:])\n        else:\n            return rel_url\n\n    base_url = base_url.rsplit(\"/\", 1)[0]\n\n    if rel_url.startswith(\"./\"):\n        return base_url + rel_url[1:]\n\n    while rel_url.startswith(\"../\"):\n        base_url = base_url.rsplit(\"/\", 1)[0]\n        rel_url = rel_url[3:]\n\n    return f\"{base_url}/{rel_url}\"\n\n\n_JS_DEFAULT_EXPORT_PATTERN = re.compile(\n    r\";?\\s*export\\s+default\\s\",\n)\n_JS_FUNC_OR_CLS_EXPORT_PATTERN = re.compile(\n...\n# Path: src/py/reactpy/reactpy/web/module.py\nfrom __future__ import annotations\n\nimport filecmp\nimport logging\nimport shutil\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom string import Template\nfrom typing import Any, NewType, overload\nfrom urllib.parse import urlparse\n\nfrom reactpy._warnings import warn\nfrom reactpy.config import REACTPY_DEBUG_MODE, REACTPY_WEB_MODULES_DIR\nfrom reactpy.core.types import ImportSourceDict, VdomDictConstructor\nfrom reactpy.core.vdom import make_vdom_constructor\nfrom reactpy.web.utils import (\n    module_name_suffix,\n    resolve_module_exports_from_file,\n    resolve_module_exports_from_url,\n)\n\nlogger = logging.getLogger(__name__)\n\nSourceType = NewType(\"SourceType\", str)\n\nNAME_SOURCE = SourceType(\"NAME\")\n\"\"\"A named source - usually a Javascript package name\"\"\"\n\nURL_SOURCE = SourceType(\"URL\")\n\"\"\"A source loaded from a URL, usually a CDN\"\"\"\n\n\ndef module_from_url(\n    url: str,\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n) -> WebModule:\n    \"\"\"Load a :class:`WebModule` from a :data:`URL_SOURCE`\n\n    Parameters:\n        url:\n            Where the javascript module will be loaded from which conforms to the\n            interface for :ref:`Custom Javascript Components`\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n    \"\"\"\n    return WebModule(\n        source=url,\n        source_type=URL_SOURCE,\n        default_fallback=fallback,\n        file=None,\n        export_names=(\n            resolve_module_exports_from_url(url, resolve_exports_depth)\n            if (\n                resolve_exports\n                if resolve_exports is not None\n                else REACTPY_DEBUG_MODE.current\n            )\n            else None\n        ),\n        unmount_before_update=unmount_before_update,\n    )\n\n\n_FROM_TEMPLATE_DIR = \"__from_template__\"\n\n\ndef module_from_template(\n    template: str,\n    package: str,\n    cdn: str = \"https://esm.sh\",\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n) -> WebModule:\n    \"\"\"Create a :class:`WebModule` from a framework template\n\n    This is useful for experimenting with component libraries that do not already\n    support ReactPy's :ref:`Custom Javascript Component` interface.\n\n    .. warning::\n\n        This approach is not recommended for use in a production setting because the\n        framework templates may use unpinned dependencies that could change without\n        warning. It's best to author a module adhering to the\n        :ref:`Custom Javascript Component` interface instead.\n\n    **Templates**\n\n    - ``react``: for modules exporting React components\n\n    Parameters:\n        template:\n            The name of the framework template to use with the given ``package``.\n        package:\n            The name of a package to load. May include a file extension (defaults to\n            ``.js`` if not given)\n        cdn:\n            Where the package should be loaded from. The CDN must distribute ESM modules\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n    \"\"\"\n    warn(\n        \"module_from_template() is deprecated due to instability - use the Javascript \"\n        \"Components API instead. This function will be removed in a future release.\",\n        DeprecationWarning,\n    )\n    template_name, _, template_version = template.partition(\"@\")\n    template_version = \"@\" + template_version if template_version else \"\"\n\n    # We do this since the package may be any valid URL path. Thus we may need to strip\n    # object parameters or query information so we save the resulting template under the\n    # correct file name.\n    package_name = urlparse(package).path\n\n    # downstream code assumes no trailing slash\n    cdn = cdn.rstrip(\"/\")\n\n    template_file_name = template_name + module_name_suffix(package_name)\n\n    template_file = Path(__file__).parent / \"templates\" / template_file_name\n    if not template_file.exists():\n        msg = f\"No template for {template_file_name!r} exists\"\n        raise ValueError(msg)\n\n    variables = {\"PACKAGE\": package, \"CDN\": cdn, \"VERSION\": template_version}\n    content = Template(template_file.read_text(encoding=\"utf-8\")).substitute(variables)\n\n    return module_from_string(\n        _FROM_TEMPLATE_DIR + \"/\" + package_name,\n        content,\n        fallback,\n        resolve_exports,\n        resolve_exports_depth,\n        unmount_before_update=unmount_before_update,\n    )\n\n\ndef module_from_file(\n    name: str,\n    file: str | Path,\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n    symlink: bool = False,\n) -> WebModule:\n    \"\"\"Load a :class:`WebModule` from a given ``file``\n\n    Parameters:\n        name:\n            The name of the package\n        file:\n            The file from which the content of the web module will be created.\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n        symlink:\n            Whether the web module should be saved as a symlink to the given ``file``.\n    \"\"\"\n    name += module_name_suffix(name)\n\n    source_file = Path(file).resolve()\n    target_file = _web_module_path(name)\n    if not source_file.exists():\n        msg = f\"Source file does not exist: {source_file}\"\n        raise FileNotFoundError(msg)\n\n    if not target_file.exists():\n        _copy_file(target_file, source_file, symlink)\n    elif not _equal_files(source_file, target_file):\n        logger.info(\n            f\"Existing web module {name!r} will \"\n            f\"be replaced with {target_file.resolve()}\"\n        )\n        target_file.unlink()\n        _copy_file(target_file, source_file, symlink)\n\n    return WebModule(\n        source=name,\n        source_type=NAME_SOURCE,\n        default_fallback=fallback,\n        file=target_file,\n        export_names=(\n            resolve_module_exports_from_file(source_file, resolve_exports_depth)\n            if (\n                resolve_exports\n                if resolve_exports is not None\n                else REACTPY_DEBUG_MODE.current\n            )\n            else None\n        ),\n        unmount_before_update=unmount_before_update,\n    )\n\n\ndef _equal_files(f1: Path, f2: Path) -> bool:\n    f1 = f1.resolve()\n    f2 = f2.resolve()\n    return (\n        (f1.is_symlink() or f2.is_symlink()) and (f1.resolve() == f2.resolve())\n    ) or filecmp.cmp(str(f1), str(f2), shallow=False)\n\n\ndef _copy_file(target: Path, source: Path, symlink: bool) -> None:\n    target.parent.mkdir(parents=True, exist_ok=True)\n    if symlink:\n        target.symlink_to(source)\n    else:\n        shutil.copy(source, target)\n\n\ndef module_from_string(\n    name: str,\n    content: str,\n    fallback: Any | None = None,\n    resolve_exports: bool | None = None,\n    resolve_exports_depth: int = 5,\n    unmount_before_update: bool = False,\n) -> WebModule:\n    \"\"\"Load a :class:`WebModule` whose ``content`` comes from a string.\n\n    Parameters:\n        name:\n            The name of the package\n        content:\n            The contents of the web module\n        fallback:\n            What to temporarily display while the module is being loaded.\n        resolve_imports:\n            Whether to try and find all the named exports of this module.\n        resolve_exports_depth:\n            How deeply to search for those exports.\n        unmount_before_update:\n            Cause the component to be unmounted before each update. This option should\n            only be used if the imported package fails to re-render when props change.\n            Using this option has negative performance consequences since all DOM\n            elements must be changed on each render. See :issue:`461` for more info.\n    \"\"\"\n    name += module_name_suffix(name)\n\n    target_file = _web_module_path(name)\n\n    if target_file.exists() and target_file.read_text(encoding=\"utf-8\") != content:\n        logger.info(\n            f\"Existing web module {name!r} will \"\n            f\"be replaced with {target_file.resolve()}\"\n        )\n        target_file.unlink()\n\n    target_file.parent.mkdir(parents=True, exist_ok=True)\n    target_file.write_text(content)\n\n    return WebModule(\n        source=name,\n        source_type=NAME_SOURCE,\n        default_fallback=fallback,\n        file=target_file,\n        export_names=(\n            resolve_module_exports_from_file(target_file, resolve_exports_depth)\n            if (\n                resolve_exports\n                if resolve_exports is not None\n                else REACTPY_DEBUG_MODE.current\n            )\n            else None\n        ),\n        unmount_before_update=unmount_before_update,\n    )\n\n\n@dataclass(frozen=True)\nclass WebModule:\n    source: str\n    source_type: SourceType\n    default_fallback: Any | None\n    export_names: set[str] | None\n    file: Path | None\n    unmount_before_update: bool\n\n\n@overload\ndef export(\n    web_module: WebModule,\n    export_names: str,\n    fallback: Any | None = ...,\n    allow_children: bool = ...,\n) -> VdomDictConstructor: ...\n\n\n@overload\ndef export(\n    web_module: WebModule,\n    export_names: list[str] | tuple[str, ...],\n    fallback: Any | None = ...,\n    allow_children: bool = ...,\n) -> list[VdomDictConstructor]: ...\n\n\ndef export(\n    web_module: WebModule,\n    export_names: str | list[str] | tuple[str, ...],\n    fallback: Any | None = None,\n    allow_children: bool = True,\n) -> VdomDictConstructor | list[VdomDictConstructor]:\n    \"\"\"Return one or more VDOM constructors from a :class:`WebModule`\n\n    Parameters:\n        export_names:\n            One or more names to export. If given as a string, a single component\n            will be returned. If a list is given, then a list of components will be\n            returned.\n        fallback:\n            What to temporarily display while the module is being loaded.\n        allow_children:\n            Whether or not these components can have children.\n    \"\"\"\n    if isinstance(export_names, str):\n        if (\n            web_module.export_names is not None\n            and export_names not in web_module.export_names\n        ):\n            msg = f\"{web_module.source!r} does not export {export_names!r}\"\n            raise ValueError(msg)\n        return _make_export(web_module, export_names, fallback, allow_children)\n    else:\n        if web_module.export_names is not None:\n            missing = sorted(set(export_names).difference(web_module.export_names))\n            if missing:\n                msg = f\"{web_module.source!r} does not export {missing!r}\"\n                raise ValueError(msg)\n        return [\n            _make_export(web_module, name, fallback, allow_children)\n            for name in export_names\n        ]\n\n\ndef _make_export(\n    web_module: WebModule,\n    name: str,\n    fallback: Any | None,\n    allow_children: bool,\n) -> VdomDictConstructor:\n    return make_vdom_constructor(\n        name,\n        allow_children=allow_children,\n        import_source=ImportSourceDict(\n            source=web_module.source,\n            sourceType=web_module.source_type,\n            fallback=(fallback or web_module.default_fallback),\n            unmountBeforeUpdate=web_module.unmount_before_update,\n        ),\n    )\n\n\ndef _web_module_path(name: str) -> Path:\n    directory = REACTPY_WEB_MODULES_DIR.current\n    path = directory.joinpath(*name.split(\"/\"))\n    return path.with_suffix(path.suffix)\n\n# Path: src/py/reactpy/reactpy/web/__init__.py\nfrom reactpy.web.module import (\n    export,\n    module_from_file,\n    module_from_string,\n    module_from_template,\n    module_from_url,\n)\n\n__all__ = [\n    \"module_from_file\",\n    \"module_from_string\",\n    \"module_from_template\",\n    \"module_from_url\",\n    \"export\",\n]\n\n# Path: src/py/reactpy/reactpy/backend/default.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom logging import getLogger\nfrom sys import exc_info\nfrom typing import Any, NoReturn\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.backend.utils import SUPPORTED_BACKENDS, all_implementations\nfrom reactpy.types import RootComponentConstructor\n\nlogger = getLogger(__name__)\n_DEFAULT_IMPLEMENTATION: BackendType[Any] | None = None\n\n\n# BackendType.Options\nclass Options:  # nocov\n    \"\"\"Configuration options that can be provided to the backend.\n    This definition should not be used/instantiated. It exists only for\n    type hinting purposes.\"\"\"\n\n    def __init__(self, *args: Any, **kwds: Any) -> NoReturn:\n        msg = \"Default implementation has no options.\"\n        raise ValueError(msg)\n\n\n# BackendType.configure\ndef configure(\n    app: Any, component: RootComponentConstructor, options: None = None\n) -> None:\n    \"\"\"Configure the given app instance to display the given component\"\"\"\n    if options is not None:  # nocov\n        msg = \"Default implementation cannot be configured with options\"\n        raise ValueError(msg)\n    return _default_implementation().configure(app, component)\n\n\n# BackendType.create_development_app\ndef create_development_app() -> Any:\n    \"\"\"Create an application instance for development purposes\"\"\"\n    return _default_implementation().create_development_app()\n\n\n# BackendType.serve_development_app\nasync def serve_development_app(\n    app: Any,\n    host: str,\n    port: int,\n    started: asyncio.Event | None = None,\n) -> None:\n    \"\"\"Run an application using a development server\"\"\"\n    return await _default_implementation().serve_development_app(\n        app, host, port, started\n    )\n\n\ndef _default_implementation() -> BackendType[Any]:\n    \"\"\"Get the first available server implementation\"\"\"\n    global _DEFAULT_IMPLEMENTATION  # noqa: PLW0603\n\n    if _DEFAULT_IMPLEMENTATION is not None:\n        return _DEFAULT_IMPLEMENTATION\n\n    try:\n        implementation = next(all_implementations())\n    except StopIteration:  # nocov\n        logger.debug(\"Backend implementation import failed\", exc_info=exc_info())\n        supported_backends = \", \".join(SUPPORTED_BACKENDS)\n        msg = (\n            \"It seems you haven't installed a backend. To resolve this issue, \"\n            \"you can install a backend by running:\\n\\n\"\n            '\\033[1mpip install \"reactpy[starlette]\"\\033[0m\\n\\n'\n            f\"Other supported backends include: {supported_backends}.\"\n        )\n        raise RuntimeError(msg) from None\n    else:\n        _DEFAULT_IMPLEMENTATION = implementation\n        return implementation\n\n# Path: src/py/reactpy/reactpy/testing/logs.py\nfrom __future__ import annotations\n\nimport logging\nimport re\nfrom collections.abc import Iterator\nfrom contextlib import contextmanager\nfrom traceback import format_exception\nfrom typing import Any, NoReturn\n\nfrom reactpy.logging import ROOT_LOGGER\n\n\nclass LogAssertionError(AssertionError):\n    \"\"\"An assertion error raised in relation to log messages.\"\"\"\n\n\n@contextmanager\ndef assert_reactpy_did_log(\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> Iterator[None]:\n    \"\"\"Assert that ReactPy produced a log matching the described message or error.\n\n    Args:\n        match_message: Must match a logged message.\n        error_type: Checks the type of logged exceptions.\n        match_error: Must match an error message.\n    \"\"\"\n    message_pattern = re.compile(match_message)\n    error_pattern = re.compile(match_error)\n\n    with capture_reactpy_logs() as log_records:\n        try:\n            yield None\n        except Exception:\n            raise\n        else:\n            for record in list(log_records):\n                if (\n                    # record message matches\n                    message_pattern.findall(record.getMessage())\n                    # error type matches\n                    and (\n                        error_type is None\n                        or (\n                            record.exc_info is not None\n                            and record.exc_info[0] is not None\n                            and issubclass(record.exc_info[0], error_type)\n                        )\n                    )\n                    # error message pattern matches\n                    and (\n                        not match_error\n                        or (\n                            record.exc_info is not None\n                            and error_pattern.findall(\n                                \"\".join(format_exception(*record.exc_info))\n                            )\n                        )\n                    )\n                ):\n                    break\n            else:  # nocov\n                _raise_log_message_error(\n                    \"Could not find a log record matching the given\",\n                    match_message,\n                    error_type,\n                    match_error,\n                )\n\n\n@contextmanager\ndef assert_reactpy_did_not_log(\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> Iterator[None]:\n    \"\"\"Assert the inverse of :func:`assert_reactpy_logged`\"\"\"\n    try:\n        with assert_reactpy_did_log(match_message, error_type, match_error):\n            yield None\n    except LogAssertionError:\n        pass\n    else:\n        _raise_log_message_error(\n            \"Did find a log record matching the given\",\n            match_message,\n            error_type,\n            match_error,\n        )\n\n\ndef list_logged_exceptions(\n    log_records: list[logging.LogRecord],\n    pattern: str = \"\",\n    types: type[Any] | tuple[type[Any], ...] = Exception,\n    log_level: int = logging.ERROR,\n    del_log_records: bool = True,\n) -> list[BaseException]:\n    \"\"\"Return a list of logged exception matching the given criteria\n\n    Args:\n        log_level: The level of log to check\n        exclude_exc_types: Any exception types to ignore\n        del_log_records: Whether to delete the log records for yielded exceptions\n    \"\"\"\n    found: list[BaseException] = []\n    compiled_pattern = re.compile(pattern)\n    for index, record in enumerate(log_records):\n        if record.levelno >= log_level and record.exc_info:\n            error = record.exc_info[1]\n            if (\n                error is not None\n                and isinstance(error, types)\n                and compiled_pattern.search(str(error))\n            ):\n                if del_log_records:\n                    del log_records[index - len(found)]\n                found.append(error)\n    return found\n\n\n@contextmanager\ndef capture_reactpy_logs() -> Iterator[list[logging.LogRecord]]:\n    \"\"\"Capture logs from ReactPy\n\n    Any logs produced in this context are cleared afterwards\n    \"\"\"\n    original_level = ROOT_LOGGER.level\n    ROOT_LOGGER.setLevel(logging.DEBUG)\n    try:\n        if _LOG_RECORD_CAPTOR in ROOT_LOGGER.handlers:\n            start_index = len(_LOG_RECORD_CAPTOR.records)\n            try:\n                yield _LOG_RECORD_CAPTOR.records\n            finally:\n                end_index = len(_LOG_RECORD_CAPTOR.records)\n                _LOG_RECORD_CAPTOR.records[start_index:end_index] = []\n            return None\n\n        ROOT_LOGGER.addHandler(_LOG_RECORD_CAPTOR)\n        try:\n            yield _LOG_RECORD_CAPTOR.records\n        finally:\n            ROOT_LOGGER.removeHandler(_LOG_RECORD_CAPTOR)\n            _LOG_RECORD_CAPTOR.records.clear()\n    finally:\n        ROOT_LOGGER.setLevel(original_level)\n\n\nclass _LogRecordCaptor(logging.NullHandler):\n    def __init__(self) -> None:\n        self.records: list[logging.LogRecord] = []\n        super().__init__()\n\n    def handle(self, record: logging.LogRecord) -> bool:\n        self.records.append(record)\n        return True\n\n\n_LOG_RECORD_CAPTOR = _LogRecordCaptor()\n\n\ndef _raise_log_message_error(\n    prefix: str,\n    match_message: str = \"\",\n    error_type: type[Exception] | None = None,\n    match_error: str = \"\",\n) -> NoReturn:\n    conditions = []\n    if match_message:\n        conditions.append(f\"log message pattern {match_message!r}\")\n    if error_type:\n        conditions.append(f\"exception type {error_type}\")\n    if match_error:\n        conditions.append(f\"error message pattern {match_error!r}\")\n    raise LogAssertionError(prefix + \" \" + \" and \".join(conditions))\n\n# Path: src/py/reactpy/reactpy/testing/backend.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nfrom contextlib import AsyncExitStack, suppress\nfrom types import TracebackType\nfrom typing import Any, Callable\nfrom urllib.parse import urlencode, urlunparse\n\nfrom reactpy.backend import default as default_server\nfrom reactpy.backend.types import BackendType\nfrom reactpy.backend.utils import find_available_port\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.core.component import component\nfrom reactpy.core.hooks import use_callback, use_effect, use_state\nfrom reactpy.core.types import ComponentConstructor\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    capture_reactpy_logs,\n    list_logged_exceptions,\n)\nfrom reactpy.utils import Ref\n\n\nclass BackendFixture:\n    \"\"\"A test fixture for running a server and imperatively displaying views\n\n    This fixture is typically used alongside async web drivers like ``playwight``.\n\n    Example:\n        .. code-block::\n\n            async with BackendFixture() as server:\n                server.mount(MyComponent)\n    \"\"\"\n\n    _records: list[logging.LogRecord]\n    _server_future: asyncio.Task[Any]\n    _exit_stack = AsyncExitStack()\n\n    def __init__(\n        self,\n        host: str = \"127.0.0.1\",\n        port: int | None = None,\n        app: Any | None = None,\n        implementation: BackendType[Any] | None = None,\n        options: Any | None = None,\n        timeout: float | None = None,\n    ) -> None:\n        self.host = host\n        self.port = port or find_available_port(host)\n        self.mount, self._root_component = _hotswap()\n        self.timeout = (\n            REACTPY_TESTING_DEFAULT_TIMEOUT.current if timeout is None else timeout\n        )\n\n        if app is not None and implementation is None:\n            msg = \"If an application instance its corresponding server implementation must be provided too.\"\n            raise ValueError(msg)\n\n        self._app = app\n        self.implementation = implementation or default_server\n        self._options = options\n\n    @property\n    def log_records(self) -> list[logging.LogRecord]:\n        \"\"\"A list of captured log records\"\"\"\n        return self._records\n\n    def url(self, path: str = \"\", query: Any | None = None) -> str:\n        \"\"\"Return a URL string pointing to the host and point of the server\n\n        Args:\n            path: the path to a resource on the server\n            query: a dictionary or list of query parameters\n        \"\"\"\n        return urlunparse(\n            [\n                \"http\",\n                f\"{self.host}:{self.port}\",\n                path,\n                \"\",\n                urlencode(query or ()),\n                \"\",\n            ]\n        )\n\n    def list_logged_exceptions(\n        self,\n        pattern: str = \"\",\n        types: type[Any] | tuple[type[Any], ...] = Exception,\n        log_level: int = logging.ERROR,\n        del_log_records: bool = True,\n    ) -> list[BaseException]:\n        \"\"\"Return a list of logged exception matching the given criteria\n\n        Args:\n            log_level: The level of log to check\n            exclude_exc_types: Any exception types to ignore\n            del_log_records: Whether to delete the log records for yielded exceptions\n        \"\"\"\n        return list_logged_exceptions(\n            self.log_records,\n            pattern,\n            types,\n            log_level,\n            del_log_records,\n        )\n\n    async def __aenter__(self) -> BackendFixture:\n        self._exit_stack = AsyncExitStack()\n        self._records = self._exit_stack.enter_context(capture_reactpy_logs())\n\n        app = self._app or self.implementation.create_development_app()\n        self.implementation.configure(app, self._root_component, self._options)\n\n        started = asyncio.Event()\n        server_future = asyncio.create_task(\n            self.implementation.serve_development_app(\n                app, self.host, self.port, started\n            )\n        )\n\n        \nasync def stop_server() -> None:\n            server_future.cancel()\n            with suppress(asyncio.CancelledError):\n                await asyncio.wait_for(server_future, timeout=self.timeout)\n\n        self._exit_stack.push_async_callback(stop_server)\n\n        try:\n            await asyncio.wait_for(started.wait(), timeout=self.timeout)\n        except Exception:  # nocov\n            # see if we can await the future for a more helpful error\n            await asyncio.wait_for(server_future, timeout=self.timeout)\n            raise\n\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        await self._exit_stack.aclose()\n\n        self.mount(None)  # reset the view\n\n        logged_errors = self.list_logged_exceptions(del_log_records=False)\n        if logged_errors:  # nocov\n            msg = \"Unexpected logged exception\"\n            raise LogAssertionError(msg) from logged_errors[0]\n\n\n_MountFunc = Callable[[\"Callable[[], Any] | None\"], None]\n\n\ndef _hotswap(update_on_change: bool = False) -> tuple[_MountFunc, ComponentConstructor]:\n    \"\"\"Swap out components from a layout on the fly.\n\n    Since you can't change the component functions used to create a layout\n    in an imperative manner, you can use ``hotswap`` to do this so\n    long as you set things up ahead of time.\n\n    Parameters:\n        update_on_change: Whether or not all views of the layout should be updated on a swap.\n\n    Example:\n        .. code-block:: python\n\n            import reactpy\n\n            show, root = reactpy.hotswap()\n            PerClientStateServer(root).run_in_thread(\"localhost\", 8765)\n\n            @reactpy.component\n            def DivOne(self):\n                return {\"tagName\": \"div\", \"children\": [1]}\n\n            show(DivOne)\n\n            # displaying the output now will show DivOne\n\n            @reactpy.component\n            def DivTwo(self):\n                return {\"tagName\": \"div\", \"children\": [2]}\n\n            show(DivTwo)\n\n            # displaying the output now will show DivTwo\n    \"\"\"\n    constructor_ref: Ref[Callable[[], Any]] = Ref(lambda: None)\n\n    if update_on_change:\n        set_constructor_callbacks: set[Callable[[Callable[[], Any]], None]] = set()\n\n        @component\n        def HotSwap() -> Any:\n            # new displays will adopt the latest constructor and arguments\n            constructor, _set_constructor = use_state(lambda: constructor_ref.current)\n            set_constructor = use_callback(lambda new: _set_constructor(lambda _: new))\n\n            def add_callback() -> Callable[[], None]:\n                set_constructor_callbacks.add(set_constructor)\n                return lambda: set_constructor_callbacks.remove(set_constructor)\n\n            use_effect(add_callback)\n\n            return constructor()\n\n        def swap(constructor: Callable[[], Any] | None) -> None:\n            constructor = constructor_ref.current = constructor or (lambda: None)\n\n            for set_constructor in set_constructor_callbacks:\n                set_constructor(constructor)\n\n    else:\n\n        @component\n        def HotSwap() -> Any:\n            return constructor_ref.current()\n\n        def swap(constructor: Callable[[], Any] | None) -> None:\n            constructor_ref.current = constructor or (lambda: None)\n\n    return swap, HotSwap\n\n# Path: src/py/reactpy/reactpy/testing/common.py\nfrom __future__ import annotations\n\nimport asyncio\nimport inspect\nimport shutil\nimport time\nfrom collections.abc import Awaitable\nfrom functools import wraps\nfrom typing import Any, Callable, Generic, TypeVar, cast\nfrom uuid import uuid4\nfrom weakref import ref\n\nfrom typing_extensions import ParamSpec\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT, REACTPY_WEB_MODULES_DIR\nfrom reactpy.core._life_cycle_hook import LifeCycleHook, current_hook\nfrom reactpy.core.events import EventHandler, to_event_handler_function\n\n\ndef clear_reactpy_web_modules_dir() -> None:\n    \"\"\"Clear the directory where ReactPy stores registered web modules\"\"\"\n    for path in REACTPY_WEB_MODULES_DIR.current.iterdir():\n        shutil.rmtree(path) if path.is_dir() else path.unlink()\n\n\n_P = ParamSpec(\"_P\")\n_R = TypeVar(\"_R\")\n\n\n_DEFAULT_POLL_DELAY = 0.1\n\n\nclass poll(Generic[_R]):  # noqa: N801\n    \"\"\"Wait until the result of an sync or async function meets some condition\"\"\"\n\n    def __init__(\n        self,\n        function: Callable[_P, Awaitable[_R] | _R],\n        *args: _P.args,\n        **kwargs: _P.kwargs,\n    ) -> None:\n        coro: Callable[_P, Awaitable[_R]]\n        if not inspect.iscoroutinefunction(function):\n\n            async def coro(*args: _P.args, **kwargs: _P.kwargs) -> _R:\n                return cast(_R, function(*args, **kwargs))\n\n        else:\n            coro = cast(Callable[_P, Awaitable[_R]], function)\n        self._func = coro\n        self._args = args\n        self._kwargs = kwargs\n\n    async def until(\n        self,\n        condition: Callable[[_R], bool],\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n        description: str = \"condition to be true\",\n    ) -> None:\n        \"\"\"Check that the coroutines result meets a condition within the timeout\"\"\"\n        started_at = time.time()\n        while True:\n            await asyncio.sleep(delay)\n            result = await self._func(*self._args, **self._kwargs)\n            if condition(result):\n                break\n            elif (time.time() - started_at) > timeout:  # nocov\n                msg = f\"Expected {description} after {timeout} seconds - last value was {result!r}\"\n                raise asyncio.TimeoutError(msg)\n\n    async def until_is(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is identical to the given value\"\"\"\n        return await self.until(\n            lambda left: left is right,\n            timeout,\n            delay,\n            f\"value to be identical to {right!r}\",\n        )\n\n    async def until_equals(\n        self,\n        right: _R,\n        timeout: float = REACTPY_TESTING_DEFAULT_TIMEOUT.current,\n        delay: float = _DEFAULT_POLL_DELAY,\n    ) -> None:\n        \"\"\"Wait until the result is equal to the given value\"\"\"\n        return await self.until(\n            lambda left: left == right,\n            timeout,\n            delay,\n            f\"value to equal {right!r}\",\n        )\n\n\nclass HookCatcher:\n    \"\"\"Utility for capturing a LifeCycleHook from a component\n\n    Example:\n        .. code-block::\n\n            hooks = HookCatcher(index_by_kwarg=\"thing\")\n\n            @reactpy.component\n            @hooks.capture\n            def MyComponent(thing):\n                ...\n\n            ...  # render the component\n\n            # grab the last render of where MyComponent(thing='something')\n            hooks.index[\"something\"]\n            # or grab the hook from the component's last render\n            hooks.latest\n\n        After the first render of ``MyComponent`` the ``HookCatcher`` will have\n        captured the component's ``LifeCycleHook``.\n    \"\"\"\n\n    latest: LifeCycleHook\n\n    def __init__(self, index_by_kwarg: str | None = None):\n        self.index_by_kwarg = index_by_kwarg\n        self.index: dict[Any, LifeCycleHook] = {}\n\n    def capture(self, render_function: Callable[..., Any]) -> Callable[..., Any]:\n        \"\"\"Decorator for capturing a ``LifeCycleHook`` on each render of a component\"\"\"\n\n        # The render function holds a reference to `self` and, via the `LifeCycleHook`,\n        # the component. Some tests check whether components are garbage collected, thus\n        # we must use a `ref` here to ensure these checks pass once the catcher itself\n        # has been collected.\n        self_ref = ref(self)\n\n        @wraps(render_function)\n        def wrapper(*args: Any, **kwargs: Any) -> Any:\n            self = self_ref()\n            if self is None:\n                raise RuntimeError(\"Hook catcher has been garbage collected\")\n\n            hook = current_hook()\n            if self.index_by_kwarg is not None:\n                self.index[kwargs[self.index_by_kwarg]] = hook\n            self.latest = hook\n            return render_function(*args, **kwargs)\n\n        return wrapper\n\n\nclass StaticEventHandler:\n    \"\"\"Utility for capturing the target of one event handler\n\n    Example:\n        .. code-block::\n\n            static_handler = StaticEventHandler()\n\n            @reactpy.component\n            def MyComponent():\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handler.use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # gives the target ID for onClick where from the last render of MyComponent\n            static_handlers.target\n\n        If you need to capture event handlers from different instances of a component\n        the you should create multiple ``StaticEventHandler`` instances.\n\n        .. code-block::\n\n            static_handlers_by_key = {\n                \"first\": StaticEventHandler(),\n                \"second\": StaticEventHandler(),\n            }\n\n            @reactpy.component\n            def Parent():\n                return reactpy.html.div(Child(key=\"first\"), Child(key=\"second\"))\n\n            @reactpy.component\n            def Child(key):\n                state, set_state = reactpy.hooks.use_state(0)\n                handler = static_handlers_by_key[key].use(lambda event: set_state(state + 1))\n                return reactpy.html.button({\"onClick\": handler}, \"Click me!\")\n\n            # grab the individual targets for each instance above\n            first_target = static_handlers_by_key[\"first\"].target\n            second_target = static_handlers_by_key[\"second\"].target\n    \"\"\"\n\n    def __init__(self) -> None:\n        self.target = uuid4().hex\n\n    def use(\n        self,\n        function: Callable[..., Any],\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n    ) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function),\n            stop_propagation,\n            prevent_default,\n            self.target,\n        )\n\n# Path: src/py/reactpy/reactpy/testing/display.py\nfrom __future__ import annotations\n\nfrom contextlib import AsyncExitStack\nfrom types import TracebackType\nfrom typing import Any\n\nfrom playwright.async_api import (\n    Browser,\n    BrowserContext,\n    ElementHandle,\n    Page,\n    async_playwright,\n)\n\nfrom reactpy.config import REACTPY_TESTING_DEFAULT_TIMEOUT\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.types import RootComponentConstructor\n\n\nclass DisplayFixture:\n    \"\"\"A fixture for running web-based tests using ``playwright``\"\"\"\n\n    _exit_stack: AsyncExitStack\n\n    def __init__(\n        self,\n        backend: BackendFixture | None = None,\n        driver: Browser | BrowserContext | Page | None = None,\n        url_prefix: str = \"\",\n    ) -> None:\n        if backend is not None:\n            self.backend = backend\n        if driver is not None:\n            if isinstance(driver, Page):\n                self.page = driver\n            else:\n                self._browser = driver\n        self.url_prefix = url_prefix\n\n    async def show(\n        self,\n        component: RootComponentConstructor,\n    ) -> None:\n        self.backend.mount(component)\n        await self.goto(\"/\")\n        await self.root_element()  # check that root element is attached\n\n    async def goto(\n        self, path: str, query: Any | None = None, add_url_prefix: bool = True\n    ) -> None:\n        await self.page.goto(\n            self.backend.url(\n                f\"{self.url_prefix}{path}\" if add_url_prefix else path, query\n            )\n        )\n\n    async def root_element(self) -> ElementHandle:\n        element = await self.page.wait_for_selector(\"#app\", state=\"attached\")\n        if element is None:  # nocov\n            msg = \"Root element not attached\"\n            raise RuntimeError(msg)\n        return element\n\n    async def __aenter__(self) -> DisplayFixture:\n        es = self._exit_stack = AsyncExitStack()\n\n        browser: Browser | BrowserContext\n        if not hasattr(self, \"page\"):\n            if not hasattr(self, \"_browser\"):\n                pw = await es.enter_async_context(async_playwright())\n                browser = await pw.chromium.launch()\n            else:\n                browser = self._browser\n            self.page = await browser.new_page()\n\n        self.page.set_default_timeout(REACTPY_TESTING_DEFAULT_TIMEOUT.current * 1000)\n\n        if not hasattr(self, \"backend\"):\n            self.backend = BackendFixture()\n            await es.enter_async_context(self.backend)\n\n        return self\n\n    async def __aexit__(\n        self,\n        exc_type: type[BaseException] | None,\n        exc_value: BaseException | None,\n        traceback: TracebackType | None,\n    ) -> None:\n        self.backend.mount(None)\n        await self._exit_stack.aclose()\n\n# Path: src/py/reactpy/reactpy/testing/__init__.py\nfrom reactpy.testing.backend import BackendFixture\nfrom reactpy.testing.common import (\n    HookCatcher,\n    StaticEventHandler,\n    clear_reactpy_web_modules_dir,\n    poll,\n)\nfrom reactpy.testing.display import DisplayFixture\nfrom reactpy.testing.logs import (\n    LogAssertionError,\n    assert_reactpy_did_log,\n    assert_reactpy_did_not_log,\n    capture_reactpy_logs,\n)\n\n__all__ = [\n    \"assert_reactpy_did_not_log\",\n    \"assert_reactpy_did_log\",\n    \"capture_reactpy_logs\",\n    \"clear_reactpy_web_modules_dir\",\n    \"DisplayFixture\",\n    \"HookCatcher\",\n    \"LogAssertionError\",\n    \"poll\",\n    \"BackendFixture\",\n    \"StaticEventHandler\",\n]\n\n# Path: src/py/reactpy/reactpy/widgets.py\nfrom __future__ import annotations\n\nfrom base64 import b64encode\nfrom collections.abc import Sequence\nfrom typing import TYPE_CHECKING, Any, Callable, Protocol, TypeVar\n\nimport reactpy\nfrom reactpy import html\nfrom reactpy._warnings import warn\nfrom reactpy.core.types import ComponentConstructor, VdomDict\n\n\ndef image(\n    format: str,\n    value: str | bytes = \"\",\n    attributes: dict[str, Any] | None = None,\n) -> VdomDict:\n    \"\"\"Utility for constructing an image from a string or bytes\n\n    The source value will automatically be encoded to base64\n    \"\"\"\n    if format == \"svg\":\n        format = \"svg+xml\"  # noqa: A001\n\n    if isinstance(value, str):\n        bytes_value = value.encode()\n    else:\n        bytes_value = value\n\n    base64_value = b64encode(bytes_value).decode()\n    src = f\"data:image/{format};base64,{base64_value}\"\n\n    return {\"tagName\": \"img\", \"attributes\": {\"src\": src, **(attributes or {})}}\n\n\n_Value = TypeVar(\"_Value\")\n\n\ndef use_linked_inputs(\n    attributes: Sequence[dict[str, Any]],\n    on_change: Callable[[_Value], None] = lambda value: None,\n    cast: _CastFunc[_Value] = lambda value: value,\n    initial_value: str = \"\",\n    ignore_empty: bool = True,\n) -> list[VdomDict]:\n    \"\"\"Return a list of linked inputs equal to the number of given attributes.\n\n    Parameters:\n        attributes:\n            That attributes of each returned input element. If the number of generated\n            inputs is variable, you may need to assign each one a\n            :ref:`key <Organizing Items With Keys>` by including a ``\"key\"`` in each\n            attribute dictionary.\n        on_change:\n            A callback which is triggered when any input is changed. This callback need\n            not update the 'value' field in the attributes of the inputs since that is\n            handled automatically.\n        cast:\n            Cast the 'value' of changed inputs that is passed to ``on_change``.\n        initial_value:\n            Initialize the 'value' field of the inputs.\n        ignore_empty:\n            Do not trigger ``on_change`` if the 'value' is an empty string.\n    \"\"\"\n    value, set_value = reactpy.hooks.use_state(initial_value)\n\n    def sync_inputs(event: dict[str, Any]) -> None:\n        new_value = event[\"target\"][\"value\"]\n        set_value(new_value)\n        if not new_value and ignore_empty:\n            return None\n        on_change(cast(new_value))\n\n    inputs: list[VdomDict] = []\n    for attrs in attributes:\n        inputs.append(html.input({**attrs, \"on_change\": sync_inputs, \"value\": value}))\n\n    return inputs\n\n\n_CastTo_co = TypeVar(\"_CastTo_co\", covariant=True)\n\n\nclass _CastFunc(Protocol[_CastTo_co]):\n    def __call__(self, value: str) -> _CastTo_co: ...\n\n\nif TYPE_CHECKING:\n    from reactpy.testing.backend import _MountFunc\n\n\ndef hotswap(\n    update_on_change: bool = False,\n) -> tuple[_MountFunc, ComponentConstructor]:  # nocov\n    warn(\n        \"The 'hotswap' function is deprecated and will be removed in a future release\",\n        DeprecationWarning,\n        stacklevel=2,\n    )\n    from reactpy.testing.backend import _hotswap\n\n    return _hotswap(update_on_change)\n\n# Path: src/py/reactpy/reactpy/__init__.py\nfrom reactpy import backend, config, html, logging, sample, svg, types, web, widgets\nfrom reactpy.backend.hooks import use_connection, use_location, use_scope\nfrom reactpy.backend.utils import run\nfrom reactpy.core import hooks\nfrom reactpy.core.component import component\nfrom reactpy.core.events import event\nfrom reactpy.core.hooks import (\n    create_context,\n    use_callback,\n    use_context,\n    use_debug_value,\n    use_effect,\n    use_memo,\n    use_reducer,\n    use_ref,\n    use_state,\n)\nfrom reactpy.core.layout import Layout\nfrom reactpy.core.vdom import vdom\nfrom reactpy.utils import Ref, html_to_vdom, vdom_to_html\n\n__author__ = \"The Reactive Python Team\"\n__version__ = \"1.0.2\"  # DO NOT MODIFY\n\n__all__ = [\n    \"backend\",\n    \"component\",\n    \"config\",\n    \"create_context\",\n    \"event\",\n    \"hooks\",\n    \"html_to_vdom\",\n    \"html\",\n    \"Layout\",\n    \"logging\",\n    \"Ref\",\n    \"run\",\n    \"sample\",\n    \"Stop\",\n    \"svg\",\n    \"types\",\n    \"use_callback\",\n    \"use_connection\",\n    \"use_context\",\n    \"use_debug_value\",\n    \"use_effect\",\n    \"use_location\",\n    \"use_memo\",\n    \"use_reducer\",\n    \"use_ref\",\n    \"use_scope\",\n    \"use_state\",\n    \"vdom_to_html\",\n    \"vdom\",\n    \"web\",\n    \"widgets\",\n]\n\n# Path: src/py/reactpy/reactpy/_console/__init__.py\n\n# Path: src/py/reactpy/reactpy/_console/ast_utils.py\nfrom __future__ import annotations\n\nimport ast\nfrom collections.abc import Iterator, Sequence\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom textwrap import indent\nfrom tokenize import COMMENT as COMMENT_TOKEN\nfrom tokenize import generate_tokens\nfrom typing import Any\n\nimport click\n\nfrom reactpy import html\n\n\ndef rewrite_changed_nodes(\n    file: Path,\n    source: str,\n    tree: ast.AST,\n    changed: list[ChangedNode],\n) -> str:\n    ast.fix_missing_locations(tree)\n\n    lines = source.split(\"\\n\")\n\n    # find closest parent nodes that should be re-written\n    nodes_to_unparse: list[ast.AST] = []\n    for change in changed:\n        node_lineage = [change.node, *change.parents]\n        for i in range(len(node_lineage) - 1):\n            current_node, next_node = node_lineage[i : i + 2]\n            if (\n                not hasattr(next_node, \"lineno\")\n                or next_node.lineno < change.node.lineno\n                or isinstance(next_node, (ast.ClassDef, ast.FunctionDef))\n            ):\n                nodes_to_unparse.append(current_node)\n                break\n        else:  # nocov\n            msg = \"Failed to change code\"\n            raise RuntimeError(msg)\n\n    # check if an nodes to rewrite contain each other, pick outermost nodes\n    current_outermost_node, *sorted_nodes_to_unparse = sorted(\n        nodes_to_unparse, key=lambda n: n.lineno\n    )\n    outermost_nodes_to_unparse = [current_outermost_node]\n    for node in sorted_nodes_to_unparse:\n        if (\n            not current_outermost_node.end_lineno\n            or node.lineno > current_outermost_node.end_lineno\n        ):\n            current_outermost_node = node\n            outermost_nodes_to_unparse.append(node)\n\n    moved_comment_lines_from_end: list[int] = []\n    # now actually rewrite these nodes (in reverse to avoid changes earlier in file)\n    for node in reversed(outermost_nodes_to_unparse):\n        # make a best effort to preserve any comments that we're going to overwrite\n        comments = _find_comments(lines[node.lineno - 1 : node.end_lineno])\n\n        # there may be some content just before and after the content we're re-writing\n        before_replacement = lines[node.lineno - 1][: node.col_offset].lstrip()\n\n        after_replacement = (\n            lines[node.end_lineno - 1][node.end_col_offset :].strip()\n            if node.end_lineno is not None and node.end_col_offset is not None\n            else \"\"\n        )\n\n        replacement = indent(\n            before_replacement\n            + \"\\n\".join([*comments, ast.unparse(node)])\n            + after_replacement,\n            \" \" * (node.col_offset - len(before_replacement)),\n        )\n\n        lines[node.lineno - 1 : node.end_lineno or node.lineno] = [replacement]\n\n        if comments:\n            moved_comment_lines_from_end.append(len(lines) - node.lineno)\n\n    for lineno_from_end in sorted(set(moved_comment_lines_from_end)):\n        click.echo(f\"Moved comments to {file}:{len(lines) - lineno_from_end}\")\n\n    return \"\\n\".join(lines)\n\n\n@dataclass\nclass ChangedNode:\n    node: ast.AST\n    parents: Sequence[ast.AST]\n\n\ndef find_element_constructor_usages(\n    tree: ast.AST, add_props: bool = False\n) -> Iterator[ElementConstructorInfo]:\n    changed: list[Sequence[ast.AST]] = []\n    for parents, node in _walk_with_parent(tree):\n        if not (isinstance(node, ast.Call)):\n            continue\n\n        func = node.func\n        if isinstance(func, ast.Attribute) and (\n            (isinstance(func.value, ast.Name) and func.value.id == \"html\")\n            or (isinstance(func.value, ast.Attribute) and func.value.attr == \"html\")\n        ):\n            name = func.attr\n        elif isinstance(func, ast.Name):\n            name = func.id\n        else:\n            continue\n\n        maybe_attr_dict_node: Any | None = None\n\n        if name == \"vdom\":\n            if len(node.args) == 0:\n                continue\n            elif len(node.args) == 1:\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.append(maybe_attr_dict_node)\n                else:\n                    continue\n            elif isinstance(node.args[1], (ast.Constant, ast.JoinedStr)):\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.insert(1, maybe_attr_dict_node)\n                else:\n                    continue\n            elif len(node.args) >= 2:  # noqa: PLR2004\n                maybe_attr_dict_node = node.args[1]\n        elif hasattr(html, name):\n            if len(node.args) == 0:\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.append(maybe_attr_dict_node)\n                else:\n                    continue\n            elif isinstance(node.args[0], (ast.Constant, ast.JoinedStr)):\n                maybe_attr_dict_node = ast.Dict(keys=[], values=[])\n                if add_props:\n                    node.args.insert(0, maybe_attr_dict_node)\n                else:\n                    continue\n            else:\n                maybe_attr_dict_node = node.args[0]\n\n        if not maybe_attr_dict_node:\n            continue\n\n        if isinstance(maybe_attr_dict_node, ast.Dict) or (\n            isinstance(maybe_attr_dict_node, ast.Call)\n            and isinstance(maybe_attr_dict_node.func, ast.Name)\n            and maybe_attr_dict_node.func.id == \"dict\"\n            and isinstance(maybe_attr_dict_node.func.ctx, ast.Load)\n        ):\n            yield ElementConstructorInfo(node, maybe_attr_dict_node, parents)\n\n    return changed\n\n\n@dataclass\nclass ElementConstructorInfo:\n    call: ast.Call\n    props: ast.Dict | ast.Call\n    parents: Sequence[ast.AST]\n\n\ndef _find_comments(lines: list[str]) -> list[str]:\n    iter_lines = iter(lines)\n    return [\n        token\n        for token_type, token, _, _, _ in generate_tokens(lambda: next(iter_lines))\n        if token_type == COMMENT_TOKEN\n    ]\n\n\ndef _walk_with_parent(\n    node: ast.AST, parents: tuple[ast.AST, ...] = ()\n) -> Iterator[tuple[tuple[ast.AST, ...], ast.AST]]:\n    parents = (node, *parents)\n    for child in ast.iter_child_nodes(node):\n        yield parents, child\n...\n", "needle_token_start": 8800, "needle_token_end": 8853, "code_context_ntokens": 16034, "output": ["```python\nasync def stop_server() -> None:\n    server_future.cancel()\n    with suppress(asyncio.CancelledError):\n        await asyncio.wait_for(server_future, timeout=self.timeout)\n```"]}
{"repo": "reactive-python/reactpy", "name": "create_context", "language": "python", "path": "src/py/reactpy/reactpy/core/hooks.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a new context type that can be utilized within a specific context management function to share data across different parts of an application without having to pass props down manually through every level of the component tree.\n2. **Input**: A single parameter representing the default value for the context.\n3. **Output**: Returns a context object that can be used to provide and consume values in different parts of the application.\n4. **Procedure**: The function defines an inner function that acts as a context provider, capable of accepting children components and an optional value to override the default. This inner function is then returned as the context object, which can be used in conjunction with a context consumer function to access the provided value in descendant components.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/core/_life_cycle_hook.py\nfrom __future__ import annotations\n\nimport logging\nfrom asyncio import Event, Task, create_task, gather\nfrom typing import Any, Callable, Protocol, TypeVar\n\nfrom anyio import Semaphore\n...\n# Path: src/py/reactpy/reactpy/core/_f_back.py\nfrom __future__ import annotations\n\nimport inspect\nfrom types import FrameType\n\n\ndef f_module_name(index: int = 0) -> str:\n    frame = f_back(index + 1)\n    if frame is None:\n        return \"\"  # nocov\n    name = frame.f_globals.get(\"__name__\", \"\")\n    if not isinstance(name, str):\n        raise TypeError(\"Expected module name to be a string\")  # nocov\n    return name\n\n\ndef f_back(index: int = 0) -> FrameType | None:\n    frame = inspect.currentframe()\n    while frame is not None:\n        if index < 0:\n            return frame\n        frame = frame.f_back\n        index -= 1\n    return None  # nocov\n\n# Path: src/py/reactpy/reactpy/core/events.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import Sequence\nfrom typing import Any, Callable, Literal, overload\n\nfrom anyio import create_task_group\n\nfrom reactpy.core.types import EventHandlerFunc, EventHandlerType\n\n\n@overload\ndef event(\n    function: Callable[..., Any],\n    *,\n    stop_propagation: bool = ...,\n    prevent_default: bool = ...,\n) -> EventHandler: ...\n\n\n@overload\ndef event(\n    function: Literal[None] = ...,\n    *,\n    stop_propagation: bool = ...,\n    prevent_default: bool = ...,\n) -> Callable[[Callable[..., Any]], EventHandler]: ...\n\n\ndef event(\n    function: Callable[..., Any] | None = None,\n    *,\n    stop_propagation: bool = False,\n    prevent_default: bool = False,\n) -> EventHandler | Callable[[Callable[..., Any]], EventHandler]:\n    \"\"\"A decorator for constructing an :class:`EventHandler`.\n\n    While you're always free to add callbacks by assigning them to an element's attributes\n\n    .. code-block:: python\n\n        element = reactpy.html.button({\"onClick\": my_callback})\n\n    You may want the ability to prevent the default action associated with the event\n    from taking place, or stopping the event from propagating up the DOM. This decorator\n    allows you to add that functionality to your callbacks.\n\n    .. code-block:: python\n\n        @event(stop_propagation=True, prevent_default=True)\n        def my_callback(*data):\n            ...\n\n        element = reactpy.html.button({\"onClick\": my_callback})\n\n    Parameters:\n        function:\n            A function or coroutine responsible for handling the event.\n        stop_propagation:\n            Block the event from propagating further up the DOM.\n        prevent_default:\n            Stops the default actional associate with the event from taking place.\n    \"\"\"\n\n    def setup(function: Callable[..., Any]) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function, positional_args=True),\n            stop_propagation,\n            prevent_default,\n        )\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass EventHandler:\n    \"\"\"Turn a function or coroutine into an event handler\n\n    Parameters:\n        function:\n            The function or coroutine which handles the event.\n        stop_propagation:\n            Block the event from propagating further up the DOM.\n        prevent_default:\n            Stops the default action associate with the event from taking place.\n        target:\n            A unique identifier for this event handler (auto-generated by default)\n    \"\"\"\n\n    __slots__ = (\n        \"__weakref__\",\n        \"function\",\n        \"prevent_default\",\n        \"stop_propagation\",\n        \"target\",\n    )\n\n    def __init__(\n        self,\n        function: EventHandlerFunc,\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n        target: str | None = None,\n    ) -> None:\n        self.function = to_event_handler_function(function, positional_args=False)\n        self.prevent_default = prevent_default\n        self.stop_propagation = stop_propagation\n        self.target = target\n\n    def __eq__(self, other: Any) -> bool:\n        undefined = object()\n        for attr in (\n            \"function\",\n            \"prevent_default\",\n            \"stop_propagation\",\n            \"target\",\n        ):\n            if not attr.startswith(\"_\"):\n                if not getattr(other, attr, undefined) == getattr(self, attr):\n                    return False\n        return True\n\n    def __repr__(self) -> str:\n        public_names = [name for name in self.__slots__ if not name.startswith(\"_\")]\n        items = \", \".join([f\"{n}={getattr(self, n)!r}\" for n in public_names])\n        return f\"{type(self).__name__}({items})\"\n\n\ndef to_event_handler_function(\n    function: Callable[..., Any],\n    positional_args: bool = True,\n) -> EventHandlerFunc:\n    \"\"\"Make a :data:`~reactpy.core.proto.EventHandlerFunc` from a function or coroutine\n\n    Parameters:\n        function:\n            A function or coroutine accepting a number of positional arguments.\n        positional_args:\n            Whether to pass the event parameters a positional args or as a list.\n    \"\"\"\n    if positional_args:\n        if asyncio.iscoroutinefunction(function):\n\n            async def wrapper(data: Sequence[Any]) -> None:\n                await function(*data)\n\n        else:\n\n            async def wrapper(data: Sequence[Any]) -> None:\n                function(*data)\n\n        return wrapper\n    elif not asyncio.iscoroutinefunction(function):\n\n        async def wrapper(data: Sequence[Any]) -> None:\n            function(data)\n\n        return wrapper\n    else:\n        return function\n\n\ndef merge_event_handlers(\n    event_handlers: Sequence[EventHandlerType],\n) -> EventHandlerType:\n    \"\"\"Merge multiple event handlers into one\n\n    Raises a ValueError if any handlers have conflicting\n    :attr:`~reactpy.core.proto.EventHandlerType.stop_propagation` or\n    :attr:`~reactpy.core.proto.EventHandlerType.prevent_default` attributes.\n    \"\"\"\n    if not event_handlers:\n        msg = \"No event handlers to merge\"\n        raise ValueError(msg)\n    elif len(event_handlers) == 1:\n        return event_handlers[0]\n\n    first_handler = event_handlers[0]\n\n    stop_propagation = first_handler.stop_propagation\n    prevent_default = first_handler.prevent_default\n    target = first_handler.target\n\n    for handler in event_handlers:\n        if (\n            handler.stop_propagation != stop_propagation\n            or handler.prevent_default != prevent_default\n            or handler.target != target\n        ):\n            msg = \"Cannot merge handlers - 'stop_propagation', 'prevent_default' or 'target' mismatch.\"\n            raise ValueError(msg)\n\n    return EventHandler(\n        merge_event_handler_funcs([h.function for h in event_handlers]),\n        stop_propagation,\n        prevent_default,\n        target,\n    )\n\n\ndef merge_event_handler_funcs(\n    functions: Sequence[EventHandlerFunc],\n) -> EventHandlerFunc:\n    \"\"\"Make one event handler function from many\"\"\"\n    if not functions:\n        msg = \"No event handler functions to merge\"\n        raise ValueError(msg)\n    elif len(functions) == 1:\n        return functions[0]\n\n    async def await_all_event_handlers(data: Sequence[Any]) -> None:\n        async with create_task_group() as group:\n            for func in functions:\n                group.start_soon(func, data)\n\n    return await_all_event_handlers\n\n# Path: src/py/reactpy/reactpy/core/vdom.py\nfrom __future__ import annotations\n\nimport json\nfrom collections.abc import Mapping, Sequence\nfrom functools import wraps\nfrom typing import Any, Protocol, cast, overload\n\nfrom fastjsonschema import compile as compile_json_schema\n\nfrom reactpy._warnings import warn\nfrom reactpy.config import REACTPY_CHECK_JSON_ATTRS, REACTPY_DEBUG_MODE\nfrom reactpy.core._f_back import f_module_name\nfrom reactpy.core.events import EventHandler, to_event_handler_function\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomDictConstructor,\n    VdomJson,\n)\n\nVDOM_JSON_SCHEMA = {\n    \"$schema\": \"http://json-schema.org/draft-07/schema\",\n    \"$ref\": \"#/definitions/element\",\n    \"definitions\": {\n        \"element\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"tagName\": {\"type\": \"string\"},\n                \"key\": {\"type\": [\"string\", \"number\", \"null\"]},\n                \"error\": {\"type\": \"string\"},\n                \"children\": {\"$ref\": \"#/definitions/elementChildren\"},\n                \"attributes\": {\"type\": \"object\"},\n                \"eventHandlers\": {\"$ref\": \"#/definitions/elementEventHandlers\"},\n                \"importSource\": {\"$ref\": \"#/definitions/importSource\"},\n            },\n            # The 'tagName' is required because its presence is a useful indicator of\n            # whether a dictionary describes a VDOM model or not.\n            \"required\": [\"tagName\"],\n            \"dependentSchemas\": {\n                # When 'error' is given, the 'tagName' should be empty.\n                \"error\": {\"properties\": {\"tagName\": {\"maxLength\": 0}}}\n            },\n        },\n        \"elementChildren\": {\n            \"type\": \"array\",\n            \"items\": {\"$ref\": \"#/definitions/elementOrString\"},\n        },\n        \"elementEventHandlers\": {\n            \"type\": \"object\",\n            \"patternProperties\": {\n                \".*\": {\"$ref\": \"#/definitions/eventHandler\"},\n            },\n        },\n        \"eventHandler\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"target\": {\"type\": \"string\"},\n                \"preventDefault\": {\"type\": \"boolean\"},\n                \"stopPropagation\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"target\"],\n        },\n        \"importSource\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"source\": {\"type\": \"string\"},\n                \"sourceType\": {\"enum\": [\"URL\", \"NAME\"]},\n                \"fallback\": {\n                    \"type\": [\"object\", \"string\", \"null\"],\n                    \"if\": {\"not\": {\"type\": \"null\"}},\n                    \"then\": {\"$ref\": \"#/definitions/elementOrString\"},\n                },\n                \"unmountBeforeUpdate\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"source\"],\n        },\n        \"elementOrString\": {\n            \"type\": [\"object\", \"string\"],\n            \"if\": {\"type\": \"object\"},\n            \"then\": {\"$ref\": \"#/definitions/element\"},\n        },\n    },\n}\n\"\"\"JSON Schema describing serialized VDOM - see :ref:`VDOM` for more info\"\"\"\n\n\n# we can't add a docstring to this because Sphinx doesn't know how to find its source\n_COMPILED_VDOM_VALIDATOR = compile_json_schema(VDOM_JSON_SCHEMA)\n\n\ndef validate_vdom_json(value: Any) -> VdomJson:\n    \"\"\"Validate serialized VDOM - see :attr:`VDOM_JSON_SCHEMA` for more info\"\"\"\n    _COMPILED_VDOM_VALIDATOR(value)\n    return cast(VdomJson, value)\n\n\ndef is_vdom(value: Any) -> bool:\n    \"\"\"Return whether a value is a :class:`VdomDict`\n\n    This employs a very simple heuristic - something is VDOM if:\n\n    1. It is a ``dict`` instance\n    2. It contains the key ``\"tagName\"``\n    3. The value of the key ``\"tagName\"`` is a string\n\n    .. note::\n\n        Performing an ``isinstance(value, VdomDict)`` check is too restrictive since the\n        user would be forced to import ``VdomDict`` every time they needed to declare a\n        VDOM element. Giving the user more flexibility, at the cost of this check's\n        accuracy, is worth it.\n    \"\"\"\n    return (\n        isinstance(value, dict)\n        and \"tagName\" in value\n        and isinstance(value[\"tagName\"], str)\n    )\n\n\n@overload\ndef vdom(tag: str, *children: VdomChildren) -> VdomDict: ...\n\n\n@overload\ndef vdom(tag: str, attributes: VdomAttributes, *children: VdomChildren) -> VdomDict: ...\n\n\ndef vdom(\n    tag: str,\n    *attributes_and_children: Any,\n    **kwargs: Any,\n) -> VdomDict:\n    \"\"\"A helper function for creating VDOM elements.\n\n    Parameters:\n        tag:\n            The type of element (e.g. 'div', 'h1', 'img')\n        attributes_and_children:\n            An optional attribute mapping followed by any number of children or\n            iterables of children. The attribute mapping **must** precede the children,\n            or children which will be merged into their respective parts of the model.\n        key:\n            A string indicating the identity of a particular element. This is significant\n            to preserve event handlers across updates - without a key, a re-render would\n            cause these handlers to be deleted, but with a key, they would be redirected\n            to any newly defined handlers.\n        event_handlers:\n            Maps event types to coroutines that are responsible for handling those events.\n        import_source:\n            (subject to change) specifies javascript that, when evaluated returns a\n            React component.\n    \"\"\"\n    if kwargs:  # nocov\n        if \"key\" in kwargs:\n            if attributes_and_children:\n                maybe_attributes, *children = attributes_and_children\n                if _is_attributes(maybe_attributes):\n                    attributes_and_children = (\n                        {**maybe_attributes, \"key\": kwargs.pop(\"key\")},\n                        *children,\n                    )\n                else:\n                    attributes_and_children = (\n                        {\"key\": kwargs.pop(\"key\")},\n                        maybe_attributes,\n                        *children,\n                    )\n            else:\n                attributes_and_children = ({\"key\": kwargs.pop(\"key\")},)\n            warn(\n                \"An element's 'key' must be declared in an attribute dict instead \"\n                \"of as a keyword argument. This will error in a future version.\",\n                DeprecationWarning,\n            )\n\n        if kwargs:\n            msg = f\"Extra keyword arguments {kwargs}\"\n            raise ValueError(msg)\n\n    model: VdomDict = {\"tagName\": tag}\n\n    if not attributes_and_children:\n        return model\n\n    attributes, children = separate_attributes_and_children(attributes_and_children)\n    key = attributes.pop(\"key\", None)\n    attributes, event_handlers = separate_attributes_and_event_handlers(attributes)\n\n    if attributes:\n        if REACTPY_CHECK_JSON_ATTRS.current:\n            json.dumps(attributes)\n        model[\"attributes\"] = attributes\n\n    if children:\n        model[\"children\"] = children\n\n    if key is not None:\n        model[\"key\"] = key\n\n    if event_handlers:\n        model[\"eventHandlers\"] = event_handlers\n\n    return model\n\n\ndef make_vdom_constructor(\n    tag: str, allow_children: bool = True, import_source: ImportSourceDict | None = None\n) -> VdomDictConstructor:\n    \"\"\"Return a constructor for VDOM dictionaries with the given tag name.\n\n    The resulting callable will have the same interface as :func:`vdom` but without its\n    first ``tag`` argument.\n    \"\"\"\n\n    def constructor(*attributes_and_children: Any, **kwargs: Any) -> VdomDict:\n        model = vdom(tag, *attributes_and_children, **kwargs)\n        if not allow_children and \"children\" in model:\n            msg = f\"{tag!r} nodes cannot have children.\"\n            raise TypeError(msg)\n        if import_source:\n            model[\"importSource\"] = import_source\n        return model\n\n    # replicate common function attributes\n    constructor.__name__ = tag\n    constructor.__doc__ = (\n        \"Return a new \"\n        f\"`<{tag}> <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/{tag}>`__ \"\n        \"element represented by a :class:`VdomDict`.\"\n    )\n\n    module_name = f_module_name(1)\n    if module_name:\n        constructor.__module__ = module_name\n        constructor.__qualname__ = f\"{module_name}.{tag}\"\n\n    return cast(VdomDictConstructor, constructor)\n\n\ndef custom_vdom_constructor(func: _CustomVdomDictConstructor) -> VdomDictConstructor:\n    \"\"\"Cast function to VdomDictConstructor\"\"\"\n\n    @wraps(func)\n    def wrapper(*attributes_and_children: Any) -> VdomDict:\n        attributes, children = separate_attributes_and_children(attributes_and_children)\n        key = attributes.pop(\"key\", None)\n        attributes, event_handlers = separate_attributes_and_event_handlers(attributes)\n        return func(attributes, children, key, event_handlers)\n\n    return cast(VdomDictConstructor, wrapper)\n\n\ndef separate_attributes_and_children(\n    values: Sequence[Any],\n) -> tuple[dict[str, Any], list[Any]]:\n    if not values:\n        return {}, []\n\n    attributes: dict[str, Any]\n    children_or_iterables: Sequence[Any]\n    if _is_attributes(values[0]):\n        attributes, *children_or_iterables = values\n    else:\n        attributes = {}\n        children_or_iterables = values\n\n    children: list[Any] = []\n    for child in children_or_iterables:\n        if _is_single_child(child):\n            children.append(child)\n        else:\n            children.extend(child)\n\n    return attributes, children\n\n\ndef separate_attributes_and_event_handlers(\n    attributes: Mapping[str, Any]\n) -> tuple[dict[str, Any], EventHandlerDict]:\n    separated_attributes = {}\n    separated_event_handlers: dict[str, EventHandlerType] = {}\n\n    for k, v in attributes.items():\n        handler: EventHandlerType\n\n        if callable(v):\n            handler = EventHandler(to_event_handler_function(v))\n        elif (\n            # isinstance check on protocols is slow - use function attr pre-check as a\n            # quick filter before actually performing slow EventHandlerType type check\n            hasattr(v, \"function\")\n            and isinstance(v, EventHandlerType)\n        ):\n            handler = v\n        else:\n            separated_attributes[k] = v\n            continue\n\n        separated_event_handlers[k] = handler\n\n    return separated_attributes, dict(separated_event_handlers.items())\n\n\ndef _is_attributes(value: Any) -> bool:\n    return isinstance(value, Mapping) and \"tagName\" not in value\n\n\ndef _is_single_child(value: Any) -> bool:\n    if isinstance(value, (str, Mapping)) or not hasattr(value, \"__iter__\"):\n        return True\n    if REACTPY_DEBUG_MODE.current:\n        _validate_child_key_integrity(value)\n    return False\n\n\ndef _validate_child_key_integrity(value: Any) -> None:\n    if hasattr(value, \"__iter__\") and not hasattr(value, \"__len__\"):\n        warn(\n            f\"Did not verify key-path integrity of children in generator {value} \"\n            \"- pass a sequence (i.e. list of finite length) in order to verify\"\n        )\n    else:\n        for child in value:\n            if isinstance(child, ComponentType) and child.key is None:\n                warn(f\"Key not specified for child in list {child}\", UserWarning)\n            elif isinstance(child, Mapping) and \"key\" not in child:\n                # remove 'children' to reduce log spam\n                child_copy = {**child, \"children\": _EllipsisRepr()}\n                warn(f\"Key not specified for child in list {child_copy}\", UserWarning)\n\n\nclass _CustomVdomDictConstructor(Protocol):\n    def __call__(\n        self,\n        attributes: VdomAttributes,\n        children: Sequence[VdomChild],\n        key: Key | None,\n        event_handlers: EventHandlerDict,\n    ) -> VdomDict: ...\n\n\nclass _EllipsisRepr:\n    def __repr__(self) -> str:\n        return \"...\"\n\n# Path: src/py/reactpy/reactpy/utils.py\nfrom __future__ import annotations\n\nimport re\nfrom collections.abc import Iterable\nfrom itertools import chain\nfrom typing import Any, Callable, Generic, TypeVar, cast\n\nfrom lxml import etree\nfrom lxml.html import fromstring, tostring\n\nfrom reactpy.core.types import VdomDict\nfrom reactpy.core.vdom import vdom\n\n_RefValue = TypeVar(\"_RefValue\")\n_ModelTransform = Callable[[VdomDict], Any]\n_UNDEFINED: Any = object()\n\n\nclass Ref(Generic[_RefValue]):\n    \"\"\"Hold a reference to a value\n\n    This is used in imperative code to mutate the state of this object in order to\n    incur side effects. Generally refs should be avoided if possible, but sometimes\n    they are required.\n\n    Notes:\n        You can compare the contents for two ``Ref`` objects using the ``==`` operator.\n    \"\"\"\n\n    __slots__ = (\"current\",)\n\n    def __init__(self, initial_value: _RefValue = _UNDEFINED) -> None:\n        if initial_value is not _UNDEFINED:\n            self.current = initial_value\n            \"\"\"The present value\"\"\"\n\n    def set_current(self, new: _RefValue) -> _RefValue:\n        \"\"\"Set the current value and return what is now the old value\n\n        This is nice to use in ``lambda`` functions.\n        \"\"\"\n        old = self.current\n        self.current = new\n        return old\n\n    def __eq__(self, other: Any) -> bool:\n        try:\n            return isinstance(other, Ref) and (other.current == self.current)\n        except AttributeError:\n            # attribute error occurs for uninitialized refs\n            return False\n\n    def __repr__(self) -> str:\n        try:\n            current = repr(self.current)\n        except AttributeError:\n            # attribute error occurs for uninitialized refs\n            current = \"<undefined>\"\n        return f\"{type(self).__name__}({current})\"\n\n\ndef vdom_to_html(vdom: VdomDict) -> str:\n    \"\"\"Convert a VDOM dictionary into an HTML string\n\n    Only the following keys are translated to HTML:\n\n    - ``tagName``\n    - ``attributes``\n    - ``children`` (must be strings or more VDOM dicts)\n\n    Parameters:\n        vdom: The VdomDict element to convert to HTML\n    \"\"\"\n    temp_root = etree.Element(\"__temp__\")\n    _add_vdom_to_etree(temp_root, vdom)\n    html = cast(bytes, tostring(temp_root)).decode()\n    # strip out temp root <__temp__> element\n    return html[10:-11]\n\n\ndef html_to_vdom(\n    html: str, *transforms: _ModelTransform, strict: bool = True\n) -> VdomDict:\n    \"\"\"Transform HTML into a DOM model. Unique keys can be provided to HTML elements\n    using a ``key=...`` attribute within your HTML tag.\n\n    Parameters:\n        html:\n            The raw HTML as a string\n        transforms:\n            Functions of the form ``transform(old) -> new`` where ``old`` is a VDOM\n            dictionary which will be replaced by ``new``. For example, you could use a\n            transform function to add highlighting to a ``<code/>`` block.\n        strict:\n            If ``True``, raise an exception if the HTML does not perfectly follow HTML5\n            syntax.\n    \"\"\"\n    if not isinstance(html, str):  # nocov\n        msg = f\"Expected html to be a string, not {type(html).__name__}\"\n        raise TypeError(msg)\n\n    # If the user provided a string, convert it to a list of lxml.etree nodes\n    try:\n        root_node: etree._Element = fromstring(\n            html.strip(),\n            parser=etree.HTMLParser(\n                remove_comments=True,\n                remove_pis=True,\n                remove_blank_text=True,\n                recover=not strict,\n            ),\n        )\n    except etree.XMLSyntaxError as e:\n        if not strict:\n            raise e  # nocov\n        msg = \"An error has occurred while parsing the HTML.\\n\\nThis HTML may be malformatted, or may not perfectly adhere to HTML5.\\nIf you believe the exception above was due to something intentional, you can disable the strict parameter on html_to_vdom().\\nOtherwise, repair your broken HTML and try again.\"\n        raise HTMLParseError(msg) from e\n\n    return _etree_to_vdom(root_node, transforms)\n\n\nclass HTMLParseError(etree.LxmlSyntaxError):  # type: ignore[misc]\n    \"\"\"Raised when an HTML document cannot be parsed using strict parsing.\"\"\"\n\n\ndef _etree_to_vdom(\n    node: etree._Element, transforms: Iterable[_ModelTransform]\n) -> VdomDict:\n    \"\"\"Transform an lxml etree node into a DOM model\n\n    Parameters:\n        node:\n            The ``lxml.etree._Element`` node\n        transforms:\n            Functions of the form ``transform(old) -> new`` where ``old`` is a VDOM\n            dictionary which will be replaced by ``new``. For example, you could use a\n            transform function to add highlighting to a ``<code/>`` block.\n    \"\"\"\n    if not isinstance(node, etree._Element):  # nocov\n        msg = f\"Expected node to be a etree._Element, not {type(node).__name__}\"\n        raise TypeError(msg)\n\n    # Recursively call _etree_to_vdom() on all children\n    children = _generate_vdom_children(node, transforms)\n\n    # Convert the lxml node to a VDOM dict\n    el = vdom(node.tag, dict(node.items()), *children)\n\n    # Perform any necessary mutations on the VDOM attributes to meet VDOM spec\n    _mutate_vdom(el)\n\n    # Apply any provided transforms.\n    for transform in transforms:\n        el = transform(el)\n\n    return el\n\n\ndef _add_vdom_to_etree(parent: etree._Element, vdom: VdomDict | dict[str, Any]) -> None:\n    try:\n        tag = vdom[\"tagName\"]\n    except KeyError as e:\n        msg = f\"Expected a VDOM dict, not {vdom}\"\n        raise TypeError(msg) from e\n    else:\n        vdom = cast(VdomDict, vdom)\n\n    if tag:\n        element = etree.SubElement(parent, tag)\n        element.attrib.update(\n            _vdom_attr_to_html_str(k, v) for k, v in vdom.get(\"attributes\", {}).items()\n        )\n    else:\n        element = parent\n\n    for c in vdom.get(\"children\", []):\n        if isinstance(c, dict):\n            _add_vdom_to_etree(element, c)\n        else:\n            \"\"\"\n            LXML handles string children by storing them under `text` and `tail`\n            attributes of Element objects. The `text` attribute, if present, effectively\n            becomes that element's first child. Then the `tail` attribute, if present,\n            becomes a sibling that follows that element. For example, consider the\n            following HTML:\n\n                <p><a>hello</a>world</p>\n\n            In this code sample, \"hello\" is the `text` attribute of the `<a>` element\n            and \"world\" is the `tail` attribute of that same `<a>` element. It's for\n            this reason that, depending on whether the element being constructed has\n            non-string a child element, we need to assign a `text` vs `tail` attribute\n            to that element or the last non-string child respectively.\n            \"\"\"\n            if len(element):\n                last_child = element[-1]\n                last_child.tail = f\"{last_child.tail or ''}{c}\"\n            else:\n                element.text = f\"{element.text or ''}{c}\"\n\n\ndef _mutate_vdom(vdom: VdomDict) -> None:\n    \"\"\"Performs any necessary mutations on the VDOM attributes to meet VDOM spec.\n\n    Currently, this function only transforms the ``style`` attribute into a dictionary whose keys are\n    camelCase so as to be renderable by React.\n\n    This function may be extended in the future.\n    \"\"\"\n    # Determine if the style attribute needs to be converted to a dict\n    if (\n        \"attributes\" in vdom\n        and \"style\" in vdom[\"attributes\"]\n        and isinstance(vdom[\"attributes\"][\"style\"], str)\n    ):\n        # Convince type checker that it's safe to mutate attributes\n        assert isinstance(vdom[\"attributes\"], dict)  # noqa: S101\n\n        # Convert style attribute from str -> dict with camelCase keys\n        vdom[\"attributes\"][\"style\"] = {\n            key.strip().replace(\"-\", \"_\"): value.strip()\n            for key, value in (\n                part.split(\":\", 1)\n                for part in vdom[\"attributes\"][\"style\"].split(\";\")\n                if \":\" in part\n            )\n        }\n\n\ndef _generate_vdom_children(\n    node: etree._Element, transforms: Iterable[_ModelTransform]\n) -> list[VdomDict | str]:\n    \"\"\"Generates a list of VDOM children from an lxml node.\n\n    Inserts inner text and/or tail text in between VDOM children, if necessary.\n    \"\"\"\n    return (  # Get the inner text of the current node\n        [node.text] if node.text else []\n    ) + list(\n        chain(\n            *(\n                # Recursively convert each child node to VDOM\n                [_etree_to_vdom(child, transforms)]\n                # Insert the tail text between each child node\n                + ([child.tail] if child.tail else [])\n                for child in node.iterchildren(None)\n            )\n        )\n    )\n\n\ndef del_html_head_body_transform(vdom: VdomDict) -> VdomDict:\n    \"\"\"Transform intended for use with `html_to_vdom`.\n\n    Removes `<html>`, `<head>`, and `<body>` while preserving their children.\n\n    Parameters:\n        vdom:\n            The VDOM dictionary to transform.\n    \"\"\"\n    if vdom[\"tagName\"] in {\"html\", \"body\", \"head\"}:\n        return {\"tagName\": \"\", \"children\": vdom[\"children\"]}\n    return vdom\n\n\ndef _vdom_attr_to_html_str(key: str, value: Any) -> tuple[str, str]:\n    if key == \"style\":\n        if isinstance(value, dict):\n            value = \";\".join(\n                # We lower only to normalize - CSS is case-insensitive:\n                # https://www.w3.org/TR/css-fonts-3/#font-family-casing\n                f\"{_CAMEL_CASE_SUB_PATTERN.sub('-', k).lower()}:{v}\"\n                for k, v in value.items()\n            )\n    elif (\n        # camel to data-* attributes\n        key.startswith(\"data_\")\n        # camel to aria-* attributes\n        or key.startswith(\"aria_\")\n        # handle special cases\n        or key in DASHED_HTML_ATTRS\n    ):\n        key = key.replace(\"_\", \"-\")\n    elif (\n        # camel to data-* attributes\n        key.startswith(\"data\")\n        # camel to aria-* attributes\n        or key.startswith(\"aria\")\n        # handle special cases\n        or key in DASHED_HTML_ATTRS\n    ):\n        key = _CAMEL_CASE_SUB_PATTERN.sub(\"-\", key)\n\n    if callable(value):  # nocov\n        raise TypeError(f\"Cannot convert callable attribute {key}={value} to HTML\")\n\n    # Again, we lower the attribute name only to normalize - HTML is case-insensitive:\n    # http://w3c.github.io/html-reference/documents.html#case-insensitivity\n    return key.lower(), str(value)\n\n\n# see list of HTML attributes with dashes in them:\n# https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes#attribute_list\nDASHED_HTML_ATTRS = {\"accept_charset\", \"acceptCharset\", \"http_equiv\", \"httpEquiv\"}\n\n# Pattern for delimitting camelCase names (e.g. camelCase to camel-case)\n_CAMEL_CASE_SUB_PATTERN = re.compile(r\"(?<!^)(?=[A-Z])\")\n\n# Path: src/py/reactpy/reactpy/core/hooks.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import Coroutine, Sequence\nfrom logging import getLogger\nfrom types import FunctionType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Generic,\n    Protocol,\n    TypeVar,\n    cast,\n    overload,\n)\n\nfrom typing_extensions import TypeAlias\n\nfrom reactpy.config import REACTPY_DEBUG_MODE\nfrom reactpy.core._life_cycle_hook import current_hook\nfrom reactpy.core.types import Context, Key, State, VdomDict\nfrom reactpy.utils import Ref\n\nif not TYPE_CHECKING:\n    # make flake8 think that this variable exists\n    ellipsis = type(...)\n\n\n__all__ = [\n    \"use_state\",\n    \"use_effect\",\n    \"use_reducer\",\n    \"use_callback\",\n    \"use_ref\",\n    \"use_memo\",\n]\n\nlogger = getLogger(__name__)\n\n_Type = TypeVar(\"_Type\")\n\n\n@overload\ndef use_state(initial_value: Callable[[], _Type]) -> State[_Type]: ...\n\n\n@overload\ndef use_state(initial_value: _Type) -> State[_Type]: ...\n\n\ndef use_state(initial_value: _Type | Callable[[], _Type]) -> State[_Type]:\n    \"\"\"See the full :ref:`Use State` docs for details\n\n    Parameters:\n        initial_value:\n            Defines the initial value of the state. A callable (accepting no arguments)\n            can be used as a constructor function to avoid re-creating the initial value\n            on each render.\n\n    Returns:\n        A tuple containing the current state and a function to update it.\n    \"\"\"\n    current_state = _use_const(lambda: _CurrentState(initial_value))\n    return State(current_state.value, current_state.dispatch)\n\n\nclass _CurrentState(Generic[_Type]):\n    __slots__ = \"value\", \"dispatch\"\n\n    def __init__(\n        self,\n        initial_value: _Type | Callable[[], _Type],\n    ) -> None:\n        if callable(initial_value):\n            self.value = initial_value()\n        else:\n            self.value = initial_value\n\n        hook = current_hook()\n\n        def dispatch(new: _Type | Callable[[_Type], _Type]) -> None:\n            if callable(new):\n                next_value = new(self.value)\n            else:\n                next_value = new\n            if not strictly_equal(next_value, self.value):\n                self.value = next_value\n                hook.schedule_render()\n\n        self.dispatch = dispatch\n\n\n_EffectCleanFunc: TypeAlias = \"Callable[[], None]\"\n_SyncEffectFunc: TypeAlias = \"Callable[[], _EffectCleanFunc | None]\"\n_AsyncEffectFunc: TypeAlias = (\n    \"Callable[[], Coroutine[None, None, _EffectCleanFunc | None]]\"\n)\n_EffectApplyFunc: TypeAlias = \"_SyncEffectFunc | _AsyncEffectFunc\"\n\n\n@overload\ndef use_effect(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None]: ...\n\n\n@overload\ndef use_effect(\n    function: _EffectApplyFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None: ...\n\n\ndef use_effect(\n    function: _EffectApplyFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None] | None:\n    \"\"\"See the full :ref:`Use Effect` docs for details\n\n    Parameters:\n        function:\n            Applies the effect and can return a clean-up function\n        dependencies:\n            Dependencies for the effect. The effect will only trigger if the identity\n            of any value in the given sequence changes (i.e. their :func:`id` is\n            different). By default these are inferred based on local variables that are\n            referenced by the given function.\n\n    Returns:\n        If not function is provided, a decorator. Otherwise ``None``.\n    \"\"\"\n    hook = current_hook()\n\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n    last_clean_callback: Ref[_EffectCleanFunc | None] = use_ref(None)\n\n    def add_effect(function: _EffectApplyFunc) -> None:\n        if not asyncio.iscoroutinefunction(function):\n            sync_function = cast(_SyncEffectFunc, function)\n        else:\n            async_function = cast(_AsyncEffectFunc, function)\n\n            def sync_function() -> _EffectCleanFunc | None:\n                task = asyncio.create_task(async_function())\n\n                def clean_future() -> None:\n                    if not task.cancel():\n                        try:\n                            clean = task.result()\n                        except asyncio.CancelledError:\n                            pass\n                        else:\n                            if clean is not None:\n                                clean()\n\n                return clean_future\n\n        async def effect(stop: asyncio.Event) -> None:\n            if last_clean_callback.current is not None:\n                last_clean_callback.current()\n                last_clean_callback.current = None\n            clean = last_clean_callback.current = sync_function()\n            await stop.wait()\n            if clean is not None:\n                clean()\n\n        return memoize(lambda: hook.add_effect(effect))\n\n    if function is not None:\n        add_effect(function)\n        return None\n    else:\n        return add_effect\n\n\ndef use_debug_value(\n    message: Any | Callable[[], Any],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None:\n    \"\"\"Log debug information when the given message changes.\n\n    .. note::\n        This hook only logs if :data:`~reactpy.config.REACTPY_DEBUG_MODE` is active.\n\n    Unlike other hooks, a message is considered to have changed if the old and new\n    values are ``!=``. Because this comparison is performed on every render of the\n    component, it may be worth considering the performance cost in some situations.\n\n    Parameters:\n        message:\n            The value to log or a memoized function for generating the value.\n        dependencies:\n            Dependencies for the memoized function. The message will only be recomputed\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n    \"\"\"\n    old: Ref[Any] = _use_const(lambda: Ref(object()))\n    memo_func = message if callable(message) else lambda: message\n    new = use_memo(memo_func, dependencies)\n\n    if REACTPY_DEBUG_MODE.current and old.current != new:\n        old.current = new\n        logger.debug(f\"{current_hook().component} {new}\")\n\n\n\ndef create_context(default_value: _Type) -> Context[_Type]:\n    \"\"\"Return a new context type for use in :func:`use_context`\"\"\"\n\n    def context(\n        *children: Any,\n        value: _Type = default_value,\n        key: Key | None = None,\n    ) -> _ContextProvider[_Type]:\n        return _ContextProvider(\n            *children,\n            value=value,\n            key=key,\n            type=context,\n        )\n\n    context.__qualname__ = \"context\"\n\n    return context\n\n\ndef use_context(context: Context[_Type]) -> _Type:\n    \"\"\"Get the current value for the given context type.\n\n    See the full :ref:`Use Context` docs for more information.\n    \"\"\"\n    hook = current_hook()\n    provider = hook.get_context_provider(context)\n\n    if provider is None:\n        # same assertions but with normal exceptions\n        if not isinstance(context, FunctionType):\n            raise TypeError(f\"{context} is not a Context\")  # nocov\n        if context.__kwdefaults__ is None:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        if \"value\" not in context.__kwdefaults__:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        return cast(_Type, context.__kwdefaults__[\"value\"])\n\n    return provider.value\n\n\nclass _ContextProvider(Generic[_Type]):\n    def __init__(\n        self,\n        *children: Any,\n        value: _Type,\n        key: Key | None,\n        type: Context[_Type],\n    ) -> None:\n        self.children = children\n        self.key = key\n        self.type = type\n        self.value = value\n\n    def render(self) -> VdomDict:\n        current_hook().set_context_provider(self)\n        return {\"tagName\": \"\", \"children\": self.children}\n\n    def __repr__(self) -> str:\n        return f\"ContextProvider({self.type})\"\n\n\n_ActionType = TypeVar(\"_ActionType\")\n\n\ndef use_reducer(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    initial_value: _Type,\n) -> tuple[_Type, Callable[[_ActionType], None]]:\n    \"\"\"See the full :ref:`Use Reducer` docs for details\n\n    Parameters:\n        reducer:\n            A function which applies an action to the current state in order to\n            produce the next state.\n        initial_value:\n            The initial state value (same as for :func:`use_state`)\n\n    Returns:\n        A tuple containing the current state and a function to change it with an action\n    \"\"\"\n    state, set_state = use_state(initial_value)\n    return state, _use_const(lambda: _create_dispatcher(reducer, set_state))\n\n\ndef _create_dispatcher(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    set_state: Callable[[Callable[[_Type], _Type]], None],\n) -> Callable[[_ActionType], None]:\n    def dispatch(action: _ActionType) -> None:\n        set_state(lambda last_state: reducer(last_state, action))\n\n    return dispatch\n\n\n_CallbackFunc = TypeVar(\"_CallbackFunc\", bound=Callable[..., Any])\n\n\n@overload\ndef use_callback(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_CallbackFunc], _CallbackFunc]: ...\n\n\n@overload\ndef use_callback(\n    function: _CallbackFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc: ...\n\n\ndef use_callback(\n    function: _CallbackFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc | Callable[[_CallbackFunc], _CallbackFunc]:\n    \"\"\"See the full :ref:`Use Callback` docs for details\n\n    Parameters:\n        function:\n            The function whose identity will be preserved\n        dependencies:\n            Dependencies of the callback. The identity the ``function`` will be updated\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current function\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n\n    def setup(function: _CallbackFunc) -> _CallbackFunc:\n        return memoize(lambda: function)\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _LambdaCaller(Protocol):\n    \"\"\"MyPy doesn't know how to deal with TypeVars only used in function return\"\"\"\n\n    def __call__(self, func: Callable[[], _Type]) -> _Type: ...\n\n\n@overload\ndef use_memo(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _LambdaCaller: ...\n\n\n@overload\ndef use_memo(\n    function: Callable[[], _Type],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type: ...\n\n\ndef use_memo(\n    function: Callable[[], _Type] | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type | Callable[[Callable[[], _Type]], _Type]:\n    \"\"\"See the full :ref:`Use Memo` docs for details\n\n    Parameters:\n        function:\n            The function to be memoized.\n        dependencies:\n            Dependencies for the memoized function. The memo will only be recomputed if\n            the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current state\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n\n    memo: _Memo[_Type] = _use_const(_Memo)\n\n    if memo.empty():\n        # we need to initialize on the first run\n        changed = True\n        memo.deps = () if dependencies is None else dependencies\n    elif dependencies is None:\n        changed = True\n        memo.deps = ()\n    elif (\n        len(memo.deps) != len(dependencies)\n        # if deps are same length check identity for each item\n        or not all(\n            strictly_equal(current, new)\n            for current, new in zip(memo.deps, dependencies)\n        )\n    ):\n        memo.deps = dependencies\n        changed = True\n    else:\n        changed = False\n\n    setup: Callable[[Callable[[], _Type]], _Type]\n\n    if changed:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            current_value = memo.value = function()\n            return current_value\n\n    else:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            return memo.value\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _Memo(Generic[_Type]):\n    \"\"\"Simple object for storing memoization data\"\"\"\n\n    __slots__ = \"value\", \"deps\"\n\n    value: _Type\n    deps: Sequence[Any]\n\n    def empty(self) -> bool:\n        try:\n            self.value  # noqa: B018\n        except AttributeError:\n            return True\n        else:\n            return False\n\n\ndef use_ref(initial_value: _Type) -> Ref[_Type]:\n    \"\"\"See the full :ref:`Use State` docs for details\n\n    Parameters:\n        initial_value: The value initially assigned to the reference.\n\n    Returns:\n        A :class:`Ref` object.\n    \"\"\"\n    return _use_const(lambda: Ref(initial_value))\n\n\ndef _use_const(function: Callable[[], _Type]) -> _Type:\n    return current_hook().use_state(function)\n\n\ndef _try_to_infer_closure_values(\n    func: Callable[..., Any] | None,\n    values: Sequence[Any] | ellipsis | None,\n) -> Sequence[Any] | None:\n    if values is ...:\n        if isinstance(func, FunctionType):\n            return (\n                [cell.cell_contents for cell in func.__closure__]\n                if func.__closure__\n                else []\n            )\n        else:\n            return None\n    else:\n        return values\n\n\ndef strictly_equal(x: Any, y: Any) -> bool:\n    \"\"\"Check if two values are identical or, for a limited set or types, equal.\n\n    Only the following types are checked for equality rather than identity:\n\n    - ``int``\n    - ``float``\n    - ``complex``\n    - ``str``\n    - ``bytes``\n    - ``bytearray``\n    - ``memoryview``\n    \"\"\"\n    return x is y or (type(x) in _NUMERIC_TEXT_BINARY_TYPES and x == y)\n\n\n_NUMERIC_TEXT_BINARY_TYPES = {\n    # numeric\n    int,\n    float,\n    complex,\n    # text\n    str,\n    # binary types\n    bytes,\n    bytearray,\n    memoryview,\n}\n\n# Path: src/py/reactpy/reactpy/backend/hooks.py\nfrom __future__ import annotations\n\nfrom collections.abc import MutableMapping\nfrom typing import Any\n\nfrom reactpy.backend.types import Connection, Location\nfrom reactpy.core.hooks import create_context, use_context\nfrom reactpy.core.types import Context\n\n# backend implementations should establish this context at the root of an app\nConnectionContext: Context[Connection[Any] | None] = create_context(None)\n\n\ndef use_connection() -> Connection[Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`.\"\"\"\n    conn = use_context(ConnectionContext)\n    if conn is None:  # nocov\n        msg = \"No backend established a connection.\"\n        raise RuntimeError(msg)\n    return conn\n\n\ndef use_scope() -> MutableMapping[str, Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s scope.\"\"\"\n    return use_connection().scope\n\n\ndef use_location() -> Location:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s location.\"\"\"\n    return use_connection().location\n\n# Path: src/py/reactpy/reactpy/core/component.py\nfrom __future__ import annotations\n\nimport inspect\nfrom functools import wraps\nfrom typing import Any, Callable\n\nfrom reactpy.core.types import ComponentType, VdomDict\n\n\ndef component(\n    function: Callable[..., ComponentType | VdomDict | str | None]\n) -> Callable[..., Component]:\n    \"\"\"A decorator for defining a new component.\n\n    Parameters:\n        function: The component's :meth:`reactpy.core.proto.ComponentType.render` function.\n    \"\"\"\n    sig = inspect.signature(function)\n\n    if \"key\" in sig.parameters and sig.parameters[\"key\"].kind in (\n        inspect.Parameter.KEYWORD_ONLY,\n        inspect.Parameter.POSITIONAL_OR_KEYWORD,\n    ):\n        msg = f\"Component render function {function} uses reserved parameter 'key'\"\n        raise TypeError(msg)\n\n    @wraps(function)\n    def constructor(*args: Any, key: Any | None = None, **kwargs: Any) -> Component:\n        return Component(function, key, args, kwargs, sig)\n\n    return constructor\n\n\nclass Component:\n    \"\"\"An object for rending component models.\"\"\"\n\n    __slots__ = \"__weakref__\", \"_func\", \"_args\", \"_kwargs\", \"_sig\", \"key\", \"type\"\n\n    def __init__(\n        self,\n        function: Callable[..., ComponentType | VdomDict | str | None],\n        key: Any | None,\n        args: tuple[Any, ...],\n        kwargs: dict[str, Any],\n        sig: inspect.Signature,\n    ) -> None:\n        self.key = key\n        self.type = function\n        self._args = args\n        self._kwargs = kwargs\n        self._sig = sig\n\n    def render(self) -> ComponentType | VdomDict | str | None:\n        return self.type(*self._args, **self._kwargs)\n\n    def __repr__(self) -> str:\n        try:\n            args = self._sig.bind(*self._args, **self._kwargs).arguments\n        except TypeError:\n            return f\"{self.type.__name__}(...)\"\n        else:\n            items = \", \".join(f\"{k}={v!r}\" for k, v in args.items())\n            if items:\n                return f\"{self.type.__name__}({id(self):02x}, {items})\"\n            else:\n                return f\"{self.type.__name__}({id(self):02x})\"\n\n# Path: src/py/reactpy/reactpy/types.py\n\"\"\"Exports common types from:\n\n- :mod:`reactpy.core.types`\n- :mod:`reactpy.backend.types`\n\"\"\"\n\nfrom reactpy.backend.types import BackendType, Connection, Location\nfrom reactpy.core.component import Component\nfrom reactpy.core.types import (\n    ComponentConstructor,\n    ComponentType,\n    Context,\n    EventHandlerDict,\n    EventHandlerFunc,\n    EventHandlerMapping,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    LayoutType,\n    RootComponentConstructor,\n    State,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomJson,\n)\n\n__all__ = [\n    \"BackendType\",\n    \"Component\",\n    \"ComponentConstructor\",\n    \"ComponentType\",\n    \"Connection\",\n    \"Context\",\n    \"EventHandlerDict\",\n    \"EventHandlerFunc\",\n    \"EventHandlerMapping\",\n    \"EventHandlerType\",\n    \"ImportSourceDict\",\n    \"Key\",\n    \"LayoutType\",\n    \"Location\",\n    \"RootComponentConstructor\",\n    \"State\",\n    \"VdomAttributes\",\n    \"VdomChild\",\n    \"VdomChildren\",\n    \"VdomDict\",\n    \"VdomJson\",\n]\n\n# Path: src/py/reactpy/reactpy/backend/utils.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nimport socket\nimport sys\nfrom collections.abc import Iterator\nfrom contextlib import closing\nfrom importlib import import_module\nfrom typing import Any\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.types import RootComponentConstructor\n\nlogger = logging.getLogger(__name__)\n\nSUPPORTED_BACKENDS = (\n    \"fastapi\",\n    \"sanic\",\n    \"tornado\",\n    \"flask\",\n    \"starlette\",\n)\n\n\ndef run(\n    component: RootComponentConstructor,\n    host: str = \"127.0.0.1\",\n    port: int | None = None,\n    implementation: BackendType[Any] | None = None,\n) -> None:\n    \"\"\"Run a component with a development server\"\"\"\n    logger.warning(_DEVELOPMENT_RUN_FUNC_WARNING)\n\n    implementation = implementation or import_module(\"reactpy.backend.default\")\n    app = implementation.create_development_app()\n    implementation.configure(app, component)\n    port = port or find_available_port(host)\n    app_cls = type(app)\n\n    logger.info(\n        \"ReactPy is running with '%s.%s' at http://%s:%s\",\n        app_cls.__module__,\n        app_cls.__name__,\n        host,\n        port,\n    )\n    asyncio.run(implementation.serve_development_app(app, host, port))\n\n\ndef find_available_port(host: str, port_min: int = 8000, port_max: int = 9000) -> int:\n    \"\"\"Get a port that's available for the given host and port range\"\"\"\n    for port in range(port_min, port_max):\n        with closing(socket.socket()) as sock:\n            try:\n                if sys.platform in (\"linux\", \"darwin\"):\n                    # Fixes bug on Unix-like systems where every time you restart the\n                    # server you'll get a different port on Linux. This cannot be set\n                    # on Windows otherwise address will always be reused.\n                    # Ref: https://stackoverflow.com/a/19247688/3159288\n                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n                sock.bind((host, port))\n            except OSError:\n                pass\n            else:\n                return port\n    msg = f\"Host {host!r} has no available port in range {port_max}-{port_max}\"\n    raise RuntimeError(msg)\n\n\ndef all_implementations() -> Iterator[BackendType[Any]]:\n    \"\"\"Yield all available server implementations\"\"\"\n    for name in SUPPORTED_BACKENDS:\n        try:\n            import_module(name)\n        except ImportError:  # nocov\n            logger.debug(\"Failed to import %s\", name, exc_info=True)\n            continue\n\n        reactpy_backend_name = f\"{__name__.rsplit('.', 1)[0]}.{name}\"\n        yield import_module(reactpy_backend_name)\n\n\n_DEVELOPMENT_RUN_FUNC_WARNING = \"\"\"\\\nThe `run()` function is only intended for testing during development! To run \\\nin production, refer to the docs on how to use reactpy.backend.*.configure.\\\n\"\"\"\n\n# Path: src/py/reactpy/reactpy/core/layout.py\nfrom __future__ import annotations\n\nimport abc\nfrom asyncio import (\n    FIRST_COMPLETED,\n    CancelledError,\n    Queue,\n    Task,\n    create_task,\n    get_running_loop,\n    wait,\n)\nfrom collections import Counter\nfrom collections.abc import Sequence\nfrom contextlib import AsyncExitStack\nfrom logging import getLogger\nfrom typing import (\n    Any,\n    Callable,\n    Generic,\n    NamedTuple,\n    NewType,\n    TypeVar,\n    cast,\n)\nfrom uuid import uuid4\nfrom weakref import ref as weakref\n\nfrom anyio import Semaphore\nfrom typing_extensions import TypeAlias\n\nfrom reactpy.config import (\n    REACTPY_ASYNC_RENDERING,\n    REACTPY_CHECK_VDOM_SPEC,\n    REACTPY_DEBUG_MODE,\n)\nfrom reactpy.core._life_cycle_hook import LifeCycleHook\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    Key,\n    LayoutEventMessage,\n    LayoutUpdateMessage,\n    VdomChild,\n    VdomDict,\n    VdomJson,\n)\nfrom reactpy.core.vdom import validate_vdom_json\nfrom reactpy.utils import Ref\n\nlogger = getLogger(__name__)\n\n\nclass Layout:\n    \"\"\"Responsible for \"rendering\" components. That is, turning them into VDOM.\"\"\"\n\n    __slots__: tuple[str, ...] = (\n        \"root\",\n        \"_event_handlers\",\n        \"_rendering_queue\",\n        \"_render_tasks\",\n        \"_render_tasks_ready\",\n        \"_root_life_cycle_state_id\",\n        \"_model_states_by_life_cycle_state_id\",\n    )\n\n    if not hasattr(abc.ABC, \"__weakref__\"):  # nocov\n        __slots__ += (\"__weakref__\",)\n\n    def __init__(self, root: ComponentType) -> None:\n        super().__init__()\n        if not isinstance(root, ComponentType):\n            msg = f\"Expected a ComponentType, not {type(root)!r}.\"\n            raise TypeError(msg)\n        self.root = root\n\n    async def __aenter__(self) -> Layout:\n        # create attributes here to avoid access before entering context manager\n        self._event_handlers: EventHandlerDict = {}\n        self._render_tasks: set[Task[LayoutUpdateMessage]] = set()\n        self._render_tasks_ready: Semaphore = Semaphore(0)\n\n        self._rendering_queue: _ThreadSafeQueue[_LifeCycleStateId] = _ThreadSafeQueue()\n        root_model_state = _new_root_model_state(self.root, self._schedule_render_task)\n\n        self._root_life_cycle_state_id = root_id = root_model_state.life_cycle_state.id\n        self._model_states_by_life_cycle_state_id = {root_id: root_model_state}\n        self._schedule_render_task(root_id)\n\n        return self\n\n    async def __aexit__(self, *exc: Any) -> None:\n        root_csid = self._root_life_cycle_state_id\n        root_model_state = self._model_states_by_life_cycle_state_id[root_csid]\n\n        for t in self._render_tasks:\n            t.cancel()\n            try:\n                await t\n            except CancelledError:\n                pass\n\n        await self._unmount_model_states([root_model_state])\n\n        # delete attributes here to avoid access after exiting context manager\n        del self._event_handlers\n        del self._rendering_queue\n        del self._root_life_cycle_state_id\n        del self._model_states_by_life_cycle_state_id\n\n    async def deliver(self, event: LayoutEventMessage) -> None:\n        \"\"\"Dispatch an event to the targeted handler\"\"\"\n        # It is possible for an element in the frontend to produce an event\n        # associated with a backend model that has been deleted. We only handle\n        # events if the element and the handler exist in the backend. Otherwise\n        # we just ignore the event.\n        handler = self._event_handlers.get(event[\"target\"])\n\n        if handler is not None:\n            try:\n                await handler.function(event[\"data\"])\n            except Exception:\n                logger.exception(f\"Failed to execute event handler {handler}\")\n        else:\n            logger.info(\n...\n", "needle_token_start": 10360, "needle_token_end": 10491, "code_context_ntokens": 16025, "output": ["```python\ndef create_context(default_value: _Type) -> Context[_Type]:\n    \"\"\"Return a new context type for use in :func:`use_context`\"\"\"\n\n    def context(\n        *children: Any,\n        value: _Type = default_value,\n        key: Key | None = None,\n    ) -> _ContextProvider[_Type]:\n        return _ContextProvider(\n            *children,\n            value=value,\n            key=key,\n            type=context,\n        )\n\n    context.__qualname__ = \"context\"\n\n    return context\n```"]}
{"repo": "reactive-python/reactpy", "name": "set_context_provider", "language": "python", "path": "src/py/reactpy/reactpy/core/_life_cycle_hook.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To assign a context provider to the hook, enabling it to supply state to any child components that request a context of the same type.\n2. **Input**: A context provider, which includes a specific context type and the associated state management logic.\n3. **Output**: There is no direct output; the function modifies the internal state of the hook by updating the context providers map.\n4. **Procedure**: The function takes a context provider as input and maps its context type to the provider itself within a dictionary that tracks context providers. This allows child components to access shared state through the context system.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/backend/__init__.py\nimport mimetypes\nfrom logging import getLogger\n\n_logger = getLogger(__name__)\n\n# Fix for missing mime types due to OS corruption/misconfiguration\n# Example: https://github.com/encode/starlette/issues/829\nif not mimetypes.inited:\n    mimetypes.init()\nfor extension, mime_type in {\n    \".js\": \"application/javascript\",\n    \".css\": \"text/css\",\n    \".json\": \"application/json\",\n}.items():\n    if not mimetypes.types_map.get(extension):  # pragma: no cover\n        _logger.warning(\n            \"Mime type '%s = %s' is missing. Please research how to \"\n            \"fix missing mime types on your operating system.\",\n            extension,\n            mime_type,\n        )\n        mimetypes.add_type(mime_type, extension)\n\n# Path: src/py/reactpy/reactpy/core/__init__.py\n\n# Path: src/py/reactpy/reactpy/core/types.py\nfrom __future__ import annotations\n\nimport sys\nfrom collections import namedtuple\nfrom collections.abc import Mapping, Sequence\nfrom types import TracebackType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Generic,\n    Literal,\n    NamedTuple,\n    Protocol,\n    TypeVar,\n    overload,\n    runtime_checkable,\n)\n\nfrom typing_extensions import TypeAlias, TypedDict\n\n_Type = TypeVar(\"_Type\")\n\n\nif TYPE_CHECKING or sys.version_info < (3, 9) or sys.version_info >= (3, 11):\n\n    class State(NamedTuple, Generic[_Type]):\n        value: _Type\n        set_value: Callable[[_Type | Callable[[_Type], _Type]], None]\n\nelse:  # nocov\n    State = namedtuple(\"State\", (\"value\", \"set_value\"))\n\n\nComponentConstructor = Callable[..., \"ComponentType\"]\n\"\"\"Simple function returning a new component\"\"\"\n\nRootComponentConstructor = Callable[[], \"ComponentType\"]\n\"\"\"The root component should be constructed by a function accepting no arguments.\"\"\"\n\n\nKey: TypeAlias = \"str | int\"\n\n\n_OwnType = TypeVar(\"_OwnType\")\n\n\n@runtime_checkable\nclass ComponentType(Protocol):\n    \"\"\"The expected interface for all component-like objects\"\"\"\n\n    key: Key | None\n    \"\"\"An identifier which is unique amongst a component's immediate siblings\"\"\"\n\n    type: Any\n    \"\"\"The function or class defining the behavior of this component\n\n    This is used to see if two component instances share the same definition.\n    \"\"\"\n\n    def render(self) -> VdomDict | ComponentType | str | None:\n        \"\"\"Render the component's view model.\"\"\"\n\n\n_Render_co = TypeVar(\"_Render_co\", covariant=True)\n_Event_contra = TypeVar(\"_Event_contra\", contravariant=True)\n\n\n@runtime_checkable\nclass LayoutType(Protocol[_Render_co, _Event_contra]):\n    \"\"\"Renders and delivers, updates to views and events to handlers, respectively\"\"\"\n\n    async def render(self) -> _Render_co:\n        \"\"\"Render an update to a view\"\"\"\n\n    async def deliver(self, event: _Event_contra) -> None:\n        \"\"\"Relay an event to its respective handler\"\"\"\n\n    async def __aenter__(self) -> LayoutType[_Render_co, _Event_contra]:\n        \"\"\"Prepare the layout for its first render\"\"\"\n\n    async def __aexit__(\n        self,\n        exc_type: type[Exception],\n        exc_value: Exception,\n        traceback: TracebackType,\n    ) -> bool | None:\n        \"\"\"Clean up the view after its final render\"\"\"\n\n\nVdomAttributes = Mapping[str, Any]\n\"\"\"Describes the attributes of a :class:`VdomDict`\"\"\"\n\nVdomChild: TypeAlias = \"ComponentType | VdomDict | str | None | Any\"\n\"\"\"A single child element of a :class:`VdomDict`\"\"\"\n\nVdomChildren: TypeAlias = \"Sequence[VdomChild] | VdomChild\"\n\"\"\"Describes a series of :class:`VdomChild` elements\"\"\"\n\n\nclass _VdomDictOptional(TypedDict, total=False):\n    key: Key | None\n    children: Sequence[ComponentType | VdomChild]\n    attributes: VdomAttributes\n    eventHandlers: EventHandlerDict\n    importSource: ImportSourceDict\n\n\nclass _VdomDictRequired(TypedDict, total=True):\n    tagName: str\n\n\nclass VdomDict(_VdomDictRequired, _VdomDictOptional):\n    \"\"\"A :ref:`VDOM` dictionary\"\"\"\n\n\nclass ImportSourceDict(TypedDict):\n    source: str\n    fallback: Any\n    sourceType: str\n    unmountBeforeUpdate: bool\n\n\nclass _OptionalVdomJson(TypedDict, total=False):\n    key: Key\n    error: str\n    children: list[Any]\n    attributes: dict[str, Any]\n    eventHandlers: dict[str, _JsonEventTarget]\n    importSource: _JsonImportSource\n\n\nclass _RequiredVdomJson(TypedDict, total=True):\n    tagName: str\n\n\nclass VdomJson(_RequiredVdomJson, _OptionalVdomJson):\n    \"\"\"A JSON serializable form of :class:`VdomDict` matching the :data:`VDOM_JSON_SCHEMA`\"\"\"\n\n\nclass _JsonEventTarget(TypedDict):\n    target: str\n    preventDefault: bool\n    stopPropagation: bool\n\n\nclass _JsonImportSource(TypedDict):\n    source: str\n    fallback: Any\n\n\nEventHandlerMapping = Mapping[str, \"EventHandlerType\"]\n\"\"\"A generic mapping between event names to their handlers\"\"\"\n\nEventHandlerDict: TypeAlias = \"dict[str, EventHandlerType]\"\n\"\"\"A dict mapping between event names to their handlers\"\"\"\n\n\nclass EventHandlerFunc(Protocol):\n    \"\"\"A coroutine which can handle event data\"\"\"\n\n    async def __call__(self, data: Sequence[Any]) -> None: ...\n\n\n@runtime_checkable\nclass EventHandlerType(Protocol):\n    \"\"\"Defines a handler for some event\"\"\"\n\n    prevent_default: bool\n    \"\"\"Whether to block the event from propagating further up the DOM\"\"\"\n\n    stop_propagation: bool\n    \"\"\"Stops the default action associate with the event from taking place.\"\"\"\n\n    function: EventHandlerFunc\n    \"\"\"A coroutine which can respond to an event and its data\"\"\"\n\n    target: str | None\n    \"\"\"Typically left as ``None`` except when a static target is useful.\n\n    When testing, it may be useful to specify a static target ID so events can be\n    triggered programmatically.\n\n    .. note::\n\n        When ``None``, it is left to a :class:`LayoutType` to auto generate a unique ID.\n    \"\"\"\n\n\nclass VdomDictConstructor(Protocol):\n    \"\"\"Standard function for constructing a :class:`VdomDict`\"\"\"\n\n    @overload\n    def __call__(\n        self, attributes: VdomAttributes, *children: VdomChildren\n    ) -> VdomDict: ...\n\n    @overload\n    def __call__(self, *children: VdomChildren) -> VdomDict: ...\n\n    @overload\n    def __call__(\n        self, *attributes_and_children: VdomAttributes | VdomChildren\n    ) -> VdomDict: ...\n\n\nclass LayoutUpdateMessage(TypedDict):\n    \"\"\"A message describing an update to a layout\"\"\"\n\n    type: Literal[\"layout-update\"]\n    \"\"\"The type of message\"\"\"\n    path: str\n    \"\"\"JSON Pointer path to the model element being updated\"\"\"\n    model: VdomJson\n    \"\"\"The model to assign at the given JSON Pointer path\"\"\"\n\n\nclass LayoutEventMessage(TypedDict):\n    \"\"\"Message describing an event originating from an element in the layout\"\"\"\n\n    type: Literal[\"layout-event\"]\n    \"\"\"The type of message\"\"\"\n    target: str\n    \"\"\"The ID of the event handler.\"\"\"\n    data: Sequence[Any]\n    \"\"\"A list of event data passed to the event handler.\"\"\"\n\n\nclass Context(Protocol[_Type]):\n    \"\"\"Returns a :class:`ContextProvider` component\"\"\"\n\n    def __call__(\n        self,\n        *children: Any,\n        value: _Type = ...,\n        key: Key | None = ...,\n    ) -> ContextProviderType[_Type]: ...\n\n\nclass ContextProviderType(ComponentType, Protocol[_Type]):\n    \"\"\"A component which provides a context value to its children\"\"\"\n\n    type: Context[_Type]\n    \"\"\"The context type\"\"\"\n\n    @property\n    def value(self) -> _Type:\n        \"Current context value\"\n\n# Path: src/py/reactpy/reactpy/backend/types.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import MutableMapping\nfrom dataclasses import dataclass\nfrom typing import Any, Callable, Generic, Protocol, TypeVar, runtime_checkable\n\nfrom reactpy.core.types import RootComponentConstructor\n\n_App = TypeVar(\"_App\")\n\n\n@runtime_checkable\nclass BackendType(Protocol[_App]):\n    \"\"\"Common interface for built-in web server/framework integrations\"\"\"\n\n    Options: Callable[..., Any]\n    \"\"\"A constructor for options passed to :meth:`BackendType.configure`\"\"\"\n\n    def configure(\n        self,\n        app: _App,\n        component: RootComponentConstructor,\n        options: Any | None = None,\n    ) -> None:\n        \"\"\"Configure the given app instance to display the given component\"\"\"\n\n    def create_development_app(self) -> _App:\n        \"\"\"Create an application instance for development purposes\"\"\"\n\n    async def serve_development_app(\n        self,\n        app: _App,\n        host: str,\n        port: int,\n        started: asyncio.Event | None = None,\n    ) -> None:\n        \"\"\"Run an application using a development server\"\"\"\n\n\n_Carrier = TypeVar(\"_Carrier\")\n\n\n@dataclass\nclass Connection(Generic[_Carrier]):\n    \"\"\"Represents a connection with a client\"\"\"\n\n    scope: MutableMapping[str, Any]\n    \"\"\"An ASGI scope or WSGI environment dictionary\"\"\"\n\n    location: Location\n    \"\"\"The current location (URL)\"\"\"\n\n    carrier: _Carrier\n    \"\"\"How the connection is mediated. For example, a request or websocket.\n\n    This typically depends on the backend implementation.\n    \"\"\"\n\n\n@dataclass\nclass Location:\n    \"\"\"Represents the current location (URL)\n\n    Analogous to, but not necessarily identical to, the client-side\n    ``document.location`` object.\n    \"\"\"\n\n    pathname: str\n    \"\"\"the path of the URL for the location\"\"\"\n\n    search: str\n    \"\"\"A search or query string - a '?' followed by the parameters of the URL.\n\n    If there are no search parameters this should be an empty string\n    \"\"\"\n\n# Path: src/py/reactpy/reactpy/_warnings.py\nfrom collections.abc import Iterator\nfrom functools import wraps\nfrom inspect import currentframe\nfrom types import FrameType\nfrom typing import TYPE_CHECKING, Any\nfrom warnings import warn as _warn\n\n\n@wraps(_warn)\ndef warn(*args: Any, **kwargs: Any) -> Any:\n    # warn at call site outside of ReactPy\n    _warn(*args, stacklevel=_frame_depth_in_module() + 1, **kwargs)  # type: ignore\n\n\nif TYPE_CHECKING:\n    warn = _warn  # noqa: F811\n\n\ndef _frame_depth_in_module() -> int:\n    depth = 0\n    for frame in _iter_frames(2):\n        module_name = frame.f_globals.get(\"__name__\")\n        if not module_name or not module_name.startswith(\"reactpy.\"):\n            break\n        depth += 1\n    return depth\n\n\ndef _iter_frames(index: int = 1) -> Iterator[FrameType]:\n    frame = currentframe()\n    while frame is not None:\n        if index == 0:\n            yield frame\n        else:\n            index -= 1\n        frame = frame.f_back\n\n# Path: src/py/reactpy/reactpy/_option.py\nfrom __future__ import annotations\n\nimport os\nfrom logging import getLogger\nfrom typing import Any, Callable, Generic, TypeVar, cast\n\nfrom reactpy._warnings import warn\n\n_O = TypeVar(\"_O\")\nlogger = getLogger(__name__)\nUNDEFINED = cast(Any, object())\n\n\nclass Option(Generic[_O]):\n    \"\"\"An option that can be set using an environment variable of the same name\"\"\"\n\n    def __init__(\n        self,\n        name: str,\n        default: _O = UNDEFINED,\n        mutable: bool = True,\n        parent: Option[_O] | None = None,\n        validator: Callable[[Any], _O] = lambda x: cast(_O, x),\n    ) -> None:\n        self._name = name\n        self._mutable = mutable\n        self._validator = validator\n        self._subscribers: list[Callable[[_O], None]] = []\n\n        if name in os.environ:\n            self._current = validator(os.environ[name])\n\n        if parent is not None:\n            if not (parent.mutable and self.mutable):\n                raise TypeError(\"Parent and child options must be mutable\")\n            self._default = parent.default\n            parent.subscribe(self.set_current)\n        elif default is not UNDEFINED:\n            self._default = default\n        else:\n            raise TypeError(\"Must specify either a default or a parent option\")\n\n        logger.debug(f\"{self._name}={self.current}\")\n\n    @property\n    def name(self) -> str:\n        \"\"\"The name of this option (used to load environment variables)\"\"\"\n        return self._name\n\n    @property\n    def mutable(self) -> bool:\n        \"\"\"Whether this option can be modified after being loaded\"\"\"\n        return self._mutable\n\n    @property\n    def default(self) -> _O:\n        \"\"\"This option's default value\"\"\"\n        return self._default\n\n    @property\n    def current(self) -> _O:\n        try:\n            return self._current\n        except AttributeError:\n            return self._default\n\n    @current.setter\n    def current(self, new: _O) -> None:\n        self.set_current(new)\n\n    @current.deleter\n    def current(self) -> None:\n        self.unset()\n\n    def subscribe(self, handler: Callable[[_O], None]) -> Callable[[_O], None]:\n        \"\"\"Register a callback that will be triggered when this option changes\"\"\"\n        if not self.mutable:\n            msg = \"Immutable options cannot be subscribed to.\"\n            raise TypeError(msg)\n        self._subscribers.append(handler)\n        handler(self.current)\n        return handler\n\n    def is_set(self) -> bool:\n        \"\"\"Whether this option has a value other than its default.\"\"\"\n        return hasattr(self, \"_current\")\n\n    def set_current(self, new: Any) -> None:\n        \"\"\"Set the value of this option\n\n        Raises a ``TypeError`` if this option is not :attr:`Option.mutable`.\n        \"\"\"\n        old = self.current\n        if new is old:\n            return None\n\n        if not self._mutable:\n            msg = f\"{self} cannot be modified after initial load\"\n            raise TypeError(msg)\n\n        try:\n            new = self._current = self._validator(new)\n        except ValueError as error:\n            raise ValueError(f\"Invalid value for {self._name}: {new!r}\") from error\n\n        logger.debug(f\"{self._name}={self._current}\")\n        if new != old:\n            for sub_func in self._subscribers:\n                sub_func(new)\n\n    def set_default(self, new: _O) -> _O:\n        \"\"\"Set the value of this option if not :meth:`Option.is_set`\n\n        Returns the current value (a la :meth:`dict.set_default`)\n        \"\"\"\n        if not self.is_set():\n            self.set_current(new)\n        return self._current\n\n    def reload(self) -> None:\n        \"\"\"Reload this option from its environment variable\"\"\"\n        self.set_current(os.environ.get(self._name, self._default))\n\n    def unset(self) -> None:\n        \"\"\"Remove the current value, the default will be used until it is set again.\"\"\"\n        if not self._mutable:\n            msg = f\"{self} cannot be modified after initial load\"\n            raise TypeError(msg)\n        old = self.current\n        if hasattr(self, \"_current\"):\n            delattr(self, \"_current\")\n        if self.current != old:\n            for sub_func in self._subscribers:\n                sub_func(self.current)\n\n    def __repr__(self) -> str:\n        return f\"Option({self._name}={self.current!r})\"\n\n\nclass DeprecatedOption(Option[_O]):\n    \"\"\"An option that will warn when it is accessed\"\"\"\n\n    def __init__(self, *args: Any, message: str, **kwargs: Any) -> None:\n        super().__init__(*args, **kwargs)\n        self._deprecation_message = message\n\n    @Option.current.getter  # type: ignore\n    def current(self) -> _O:\n        try:\n            # we access the current value during init to debug log it\n            # no need to warn unless it's actually used. since this attr\n            # is only set after super().__init__ is called, we can check\n            # for it to determine if it's being accessed by a user.\n            msg = self._deprecation_message\n        except AttributeError:\n            pass\n        else:\n            warn(msg, DeprecationWarning)\n        return super().current\n\n# Path: src/py/reactpy/reactpy/config.py\n\"\"\"\nReactPy provides a series of configuration options that can be set using environment\nvariables or, for those which allow it, a programmatic interface.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom tempfile import TemporaryDirectory\n\nfrom reactpy._option import Option\n\nTRUE_VALUES = {\"true\", \"1\"}\nFALSE_VALUES = {\"false\", \"0\"}\n\n\ndef boolean(value: str | bool | int) -> bool:\n    if isinstance(value, bool):\n        return value\n    elif isinstance(value, int):\n        return bool(value)\n    elif not isinstance(value, str):\n        raise TypeError(f\"Expected str or bool, got {type(value).__name__}\")\n\n    if value.lower() in TRUE_VALUES:\n        return True\n    elif value.lower() in FALSE_VALUES:\n        return False\n    else:\n        raise ValueError(\n            f\"Invalid boolean value {value!r} - expected \"\n            f\"one of {list(TRUE_VALUES | FALSE_VALUES)}\"\n        )\n\n\nREACTPY_DEBUG_MODE = Option(\n    \"REACTPY_DEBUG_MODE\", default=False, validator=boolean, mutable=True\n)\n\"\"\"Get extra logs and validation checks at the cost of performance.\n\nThis will enable the following:\n\n- :data:`REACTPY_CHECK_VDOM_SPEC`\n- :data:`REACTPY_CHECK_JSON_ATTRS`\n\"\"\"\n\nREACTPY_CHECK_VDOM_SPEC = Option(\"REACTPY_CHECK_VDOM_SPEC\", parent=REACTPY_DEBUG_MODE)\n\"\"\"Checks which ensure VDOM is rendered to spec\n\nFor more info on the VDOM spec, see here: :ref:`VDOM JSON Schema`\n\"\"\"\n\nREACTPY_CHECK_JSON_ATTRS = Option(\"REACTPY_CHECK_JSON_ATTRS\", parent=REACTPY_DEBUG_MODE)\n\"\"\"Checks that all VDOM attributes are JSON serializable\n\nThe VDOM spec is not able to enforce this on its own since attributes could anything.\n\"\"\"\n\n# Because these web modules will be linked dynamically at runtime this can be temporary.\n# Assigning to a variable here ensures that the directory is not deleted until the end\n# of the program.\n_DEFAULT_WEB_MODULES_DIR = TemporaryDirectory()\n\nREACTPY_WEB_MODULES_DIR = Option(\n    \"REACTPY_WEB_MODULES_DIR\",\n    default=Path(_DEFAULT_WEB_MODULES_DIR.name),\n    validator=Path,\n)\n\"\"\"The location ReactPy will use to store its client application\n\nThis directory **MUST** be treated as a black box. Downstream applications **MUST NOT**\nassume anything about the structure of this directory see :mod:`reactpy.web.module` for a\nset of publicly available APIs for working with the client.\n\"\"\"\n\nREACTPY_TESTING_DEFAULT_TIMEOUT = Option(\n    \"REACTPY_TESTING_DEFAULT_TIMEOUT\",\n    5.0,\n    mutable=False,\n    validator=float,\n)\n\"\"\"A default timeout for testing utilities in ReactPy\"\"\"\n\nREACTPY_ASYNC_RENDERING = Option(\n    \"REACTPY_CONCURRENT_RENDERING\",\n    default=False,\n    mutable=True,\n    validator=boolean,\n)\n\"\"\"Whether to render components concurrently. This is currently an experimental feature.\"\"\"\n\n# Path: src/py/reactpy/reactpy/core/_thread_local.py\nfrom threading import Thread, current_thread\nfrom typing import Callable, Generic, TypeVar\nfrom weakref import WeakKeyDictionary\n\n_StateType = TypeVar(\"_StateType\")\n\n\nclass ThreadLocal(Generic[_StateType]):\n    \"\"\"Utility for managing per-thread state information\"\"\"\n\n    def __init__(self, default: Callable[[], _StateType]):\n        self._default = default\n        self._state: WeakKeyDictionary[Thread, _StateType] = WeakKeyDictionary()\n\n    def get(self) -> _StateType:\n        thread = current_thread()\n        if thread not in self._state:\n            state = self._state[thread] = self._default()\n        else:\n            state = self._state[thread]\n        return state\n\n# Path: src/py/reactpy/reactpy/core/_life_cycle_hook.py\nfrom __future__ import annotations\n\nimport logging\nfrom asyncio import Event, Task, create_task, gather\nfrom typing import Any, Callable, Protocol, TypeVar\n\nfrom anyio import Semaphore\n\nfrom reactpy.core._thread_local import ThreadLocal\nfrom reactpy.core.types import ComponentType, Context, ContextProviderType\n\nT = TypeVar(\"T\")\n\n\nclass EffectFunc(Protocol):\n    async def __call__(self, stop: Event) -> None: ...\n\n\nlogger = logging.getLogger(__name__)\n\n_HOOK_STATE: ThreadLocal[list[LifeCycleHook]] = ThreadLocal(list)\n\n\ndef current_hook() -> LifeCycleHook:\n    \"\"\"Get the current :class:`LifeCycleHook`\"\"\"\n    hook_stack = _HOOK_STATE.get()\n    if not hook_stack:\n        msg = \"No life cycle hook is active. Are you rendering in a layout?\"\n        raise RuntimeError(msg)\n    return hook_stack[-1]\n\n\nclass LifeCycleHook:\n    \"\"\"An object which manages the \"life cycle\" of a layout component.\n\n    The \"life cycle\" of a component is the set of events which occur from the time\n    a component is first rendered until it is removed from the layout. The life cycle\n    is ultimately driven by the layout itself, but components can \"hook\" into those\n    events to perform actions. Components gain access to their own life cycle hook\n    by calling :func:`current_hook`. They can then perform actions such as:\n\n    1. Adding state via :meth:`use_state`\n    2. Adding effects via :meth:`add_effect`\n    3. Setting or getting context providers via\n       :meth:`LifeCycleHook.set_context_provider` and\n       :meth:`get_context_provider` respectively.\n\n    Components can request access to their own life cycle events and state through hooks\n    while :class:`~reactpy.core.proto.LayoutType` objects drive drive the life cycle\n    forward by triggering events and rendering view changes.\n\n    Example:\n\n        If removed from the complexities of a layout, a very simplified full life cycle\n        for a single component with no child components would look a bit like this:\n\n        .. testcode::\n\n            from reactpy.core._life_cycle_hook import LifeCycleHook\n            from reactpy.core.hooks import current_hook\n\n            # this function will come from a layout implementation\n            schedule_render = lambda: ...\n\n            # --- start life cycle ---\n\n            hook = LifeCycleHook(schedule_render)\n\n            # --- start render cycle ---\n\n            component = ...\n            await hook.affect_component_will_render(component)\n            try:\n                # render the component\n                ...\n\n                # the component may access the current hook\n                assert current_hook() is hook\n\n                # and save state or add effects\n                current_hook().use_state(lambda: ...)\n\n                async def my_effect(stop_event):\n                    ...\n\n                current_hook().add_effect(my_effect)\n            finally:\n                await hook.affect_component_did_render()\n\n            # This should only be called after the full set of changes associated with a\n            # given render have been completed.\n            await hook.affect_layout_did_render()\n\n            # Typically an event occurs and a new render is scheduled, thus beginning\n            # the render cycle anew.\n            hook.schedule_render()\n\n\n            # --- end render cycle ---\n\n            hook.affect_component_will_unmount()\n            del hook\n\n            # --- end render cycle ---\n    \"\"\"\n\n    __slots__ = (\n        \"__weakref__\",\n        \"_context_providers\",\n        \"_current_state_index\",\n        \"_effect_funcs\",\n        \"_effect_stops\",\n        \"_effect_tasks\",\n        \"_render_access\",\n        \"_rendered_atleast_once\",\n        \"_schedule_render_callback\",\n        \"_scheduled_render\",\n        \"_state\",\n        \"component\",\n    )\n\n    component: ComponentType\n\n    def __init__(\n        self,\n        schedule_render: Callable[[], None],\n    ) -> None:\n        self._context_providers: dict[Context[Any], ContextProviderType[Any]] = {}\n        self._schedule_render_callback = schedule_render\n        self._scheduled_render = False\n        self._rendered_atleast_once = False\n        self._current_state_index = 0\n        self._state: tuple[Any, ...] = ()\n        self._effect_funcs: list[EffectFunc] = []\n        self._effect_tasks: list[Task[None]] = []\n        self._effect_stops: list[Event] = []\n        self._render_access = Semaphore(1)  # ensure only one render at a time\n\n    def schedule_render(self) -> None:\n        if self._scheduled_render:\n            return None\n        try:\n            self._schedule_render_callback()\n        except Exception:\n            msg = f\"Failed to schedule render via {self._schedule_render_callback}\"\n            logger.exception(msg)\n        else:\n            self._scheduled_render = True\n\n    def use_state(self, function: Callable[[], T]) -> T:\n        \"\"\"Add state to this hook\n\n        If this hook has not yet rendered, the state is appended to the state tuple.\n        Otherwise, the state is retrieved from the tuple. This allows state to be\n        preserved across renders.\n        \"\"\"\n        if not self._rendered_atleast_once:\n            # since we're not initialized yet we're just appending state\n            result = function()\n            self._state += (result,)\n        else:\n            # once finalized we iterate over each succesively used piece of state\n            result = self._state[self._current_state_index]\n        self._current_state_index += 1\n        return result\n\n    def add_effect(self, effect_func: EffectFunc) -> None:\n        \"\"\"Add an effect to this hook\n\n        A task to run the effect is created when the component is done rendering.\n        When the component will be unmounted, the event passed to the effect is\n        triggered and the task is awaited. The effect should eventually halt after\n        the event is triggered.\n        \"\"\"\n        self._effect_funcs.append(effect_func)\n\n    \ndef set_context_provider(self, provider: ContextProviderType[Any]) -> None:\n        \"\"\"Set a context provider for this hook\n\n        The context provider will be used to provide state to any child components\n        of this hook's component which request a context provider of the same type.\n        \"\"\"\n        self._context_providers[provider.type] = provider\n\n    def get_context_provider(\n        self, context: Context[T]\n    ) -> ContextProviderType[T] | None:\n        \"\"\"Get a context provider for this hook of the given type\n\n        The context provider will have been set by a parent component. If no provider\n        is found, ``None`` is returned.\n        \"\"\"\n        return self._context_providers.get(context)\n\n    async def affect_component_will_render(self, component: ComponentType) -> None:\n        \"\"\"The component is about to render\"\"\"\n        await self._render_access.acquire()\n        self._scheduled_render = False\n        self.component = component\n        self.set_current()\n\n    async def affect_component_did_render(self) -> None:\n        \"\"\"The component completed a render\"\"\"\n        self.unset_current()\n        self._rendered_atleast_once = True\n        self._current_state_index = 0\n        self._render_access.release()\n        del self.component\n\n    async def affect_layout_did_render(self) -> None:\n        \"\"\"The layout completed a render\"\"\"\n        stop = Event()\n        self._effect_stops.append(stop)\n        self._effect_tasks.extend(create_task(e(stop)) for e in self._effect_funcs)\n        self._effect_funcs.clear()\n\n    async def affect_component_will_unmount(self) -> None:\n        \"\"\"The component is about to be removed from the layout\"\"\"\n        for stop in self._effect_stops:\n            stop.set()\n        self._effect_stops.clear()\n        try:\n            await gather(*self._effect_tasks)\n        except Exception:\n            logger.exception(\"Error in effect\")\n        finally:\n            self._effect_tasks.clear()\n\n    def set_current(self) -> None:\n        \"\"\"Set this hook as the active hook in this thread\n\n        This method is called by a layout before entering the render method\n        of this hook's associated component.\n        \"\"\"\n        hook_stack = _HOOK_STATE.get()\n        if hook_stack:\n            parent = hook_stack[-1]\n            self._context_providers.update(parent._context_providers)\n        hook_stack.append(self)\n\n    def unset_current(self) -> None:\n        \"\"\"Unset this hook as the active hook in this thread\"\"\"\n        if _HOOK_STATE.get().pop() is not self:\n            raise RuntimeError(\"Hook stack is in an invalid state\")  # nocov\n\n# Path: src/py/reactpy/reactpy/core/_f_back.py\nfrom __future__ import annotations\n\nimport inspect\nfrom types import FrameType\n\n\ndef f_module_name(index: int = 0) -> str:\n    frame = f_back(index + 1)\n    if frame is None:\n        return \"\"  # nocov\n    name = frame.f_globals.get(\"__name__\", \"\")\n    if not isinstance(name, str):\n        raise TypeError(\"Expected module name to be a string\")  # nocov\n    return name\n\n\ndef f_back(index: int = 0) -> FrameType | None:\n    frame = inspect.currentframe()\n    while frame is not None:\n        if index < 0:\n            return frame\n        frame = frame.f_back\n        index -= 1\n    return None  # nocov\n\n# Path: src/py/reactpy/reactpy/core/events.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import Sequence\nfrom typing import Any, Callable, Literal, overload\n\nfrom anyio import create_task_group\n\nfrom reactpy.core.types import EventHandlerFunc, EventHandlerType\n\n\n@overload\ndef event(\n    function: Callable[..., Any],\n    *,\n    stop_propagation: bool = ...,\n    prevent_default: bool = ...,\n) -> EventHandler: ...\n\n\n@overload\ndef event(\n    function: Literal[None] = ...,\n    *,\n    stop_propagation: bool = ...,\n    prevent_default: bool = ...,\n) -> Callable[[Callable[..., Any]], EventHandler]: ...\n\n\ndef event(\n    function: Callable[..., Any] | None = None,\n    *,\n    stop_propagation: bool = False,\n    prevent_default: bool = False,\n) -> EventHandler | Callable[[Callable[..., Any]], EventHandler]:\n    \"\"\"A decorator for constructing an :class:`EventHandler`.\n\n    While you're always free to add callbacks by assigning them to an element's attributes\n\n    .. code-block:: python\n\n        element = reactpy.html.button({\"onClick\": my_callback})\n\n    You may want the ability to prevent the default action associated with the event\n    from taking place, or stopping the event from propagating up the DOM. This decorator\n    allows you to add that functionality to your callbacks.\n\n    .. code-block:: python\n\n        @event(stop_propagation=True, prevent_default=True)\n        def my_callback(*data):\n            ...\n\n        element = reactpy.html.button({\"onClick\": my_callback})\n\n    Parameters:\n        function:\n            A function or coroutine responsible for handling the event.\n        stop_propagation:\n            Block the event from propagating further up the DOM.\n        prevent_default:\n            Stops the default actional associate with the event from taking place.\n    \"\"\"\n\n    def setup(function: Callable[..., Any]) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function, positional_args=True),\n            stop_propagation,\n            prevent_default,\n        )\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass EventHandler:\n    \"\"\"Turn a function or coroutine into an event handler\n\n    Parameters:\n        function:\n            The function or coroutine which handles the event.\n        stop_propagation:\n            Block the event from propagating further up the DOM.\n        prevent_default:\n            Stops the default action associate with the event from taking place.\n        target:\n            A unique identifier for this event handler (auto-generated by default)\n    \"\"\"\n\n    __slots__ = (\n        \"__weakref__\",\n        \"function\",\n        \"prevent_default\",\n        \"stop_propagation\",\n        \"target\",\n    )\n\n    def __init__(\n        self,\n        function: EventHandlerFunc,\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n        target: str | None = None,\n    ) -> None:\n        self.function = to_event_handler_function(function, positional_args=False)\n        self.prevent_default = prevent_default\n        self.stop_propagation = stop_propagation\n        self.target = target\n\n    def __eq__(self, other: Any) -> bool:\n        undefined = object()\n        for attr in (\n            \"function\",\n            \"prevent_default\",\n            \"stop_propagation\",\n            \"target\",\n        ):\n            if not attr.startswith(\"_\"):\n                if not getattr(other, attr, undefined) == getattr(self, attr):\n                    return False\n        return True\n\n    def __repr__(self) -> str:\n        public_names = [name for name in self.__slots__ if not name.startswith(\"_\")]\n        items = \", \".join([f\"{n}={getattr(self, n)!r}\" for n in public_names])\n        return f\"{type(self).__name__}({items})\"\n\n\ndef to_event_handler_function(\n    function: Callable[..., Any],\n    positional_args: bool = True,\n) -> EventHandlerFunc:\n    \"\"\"Make a :data:`~reactpy.core.proto.EventHandlerFunc` from a function or coroutine\n\n    Parameters:\n        function:\n            A function or coroutine accepting a number of positional arguments.\n        positional_args:\n            Whether to pass the event parameters a positional args or as a list.\n    \"\"\"\n    if positional_args:\n        if asyncio.iscoroutinefunction(function):\n\n            async def wrapper(data: Sequence[Any]) -> None:\n                await function(*data)\n\n        else:\n\n            async def wrapper(data: Sequence[Any]) -> None:\n                function(*data)\n\n        return wrapper\n    elif not asyncio.iscoroutinefunction(function):\n\n        async def wrapper(data: Sequence[Any]) -> None:\n            function(data)\n\n        return wrapper\n    else:\n        return function\n\n\ndef merge_event_handlers(\n    event_handlers: Sequence[EventHandlerType],\n) -> EventHandlerType:\n    \"\"\"Merge multiple event handlers into one\n\n    Raises a ValueError if any handlers have conflicting\n    :attr:`~reactpy.core.proto.EventHandlerType.stop_propagation` or\n    :attr:`~reactpy.core.proto.EventHandlerType.prevent_default` attributes.\n    \"\"\"\n    if not event_handlers:\n        msg = \"No event handlers to merge\"\n        raise ValueError(msg)\n    elif len(event_handlers) == 1:\n        return event_handlers[0]\n\n    first_handler = event_handlers[0]\n\n    stop_propagation = first_handler.stop_propagation\n    prevent_default = first_handler.prevent_default\n    target = first_handler.target\n\n    for handler in event_handlers:\n        if (\n            handler.stop_propagation != stop_propagation\n            or handler.prevent_default != prevent_default\n            or handler.target != target\n        ):\n            msg = \"Cannot merge handlers - 'stop_propagation', 'prevent_default' or 'target' mismatch.\"\n            raise ValueError(msg)\n\n    return EventHandler(\n        merge_event_handler_funcs([h.function for h in event_handlers]),\n        stop_propagation,\n        prevent_default,\n        target,\n    )\n\n\ndef merge_event_handler_funcs(\n    functions: Sequence[EventHandlerFunc],\n) -> EventHandlerFunc:\n    \"\"\"Make one event handler function from many\"\"\"\n    if not functions:\n        msg = \"No event handler functions to merge\"\n        raise ValueError(msg)\n    elif len(functions) == 1:\n        return functions[0]\n\n    async def await_all_event_handlers(data: Sequence[Any]) -> None:\n        async with create_task_group() as group:\n            for func in functions:\n                group.start_soon(func, data)\n\n    return await_all_event_handlers\n\n# Path: src/py/reactpy/reactpy/core/vdom.py\nfrom __future__ import annotations\n\nimport json\nfrom collections.abc import Mapping, Sequence\nfrom functools import wraps\nfrom typing import Any, Protocol, cast, overload\n\nfrom fastjsonschema import compile as compile_json_schema\n\nfrom reactpy._warnings import warn\nfrom reactpy.config import REACTPY_CHECK_JSON_ATTRS, REACTPY_DEBUG_MODE\nfrom reactpy.core._f_back import f_module_name\nfrom reactpy.core.events import EventHandler, to_event_handler_function\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomDictConstructor,\n    VdomJson,\n)\n\nVDOM_JSON_SCHEMA = {\n    \"$schema\": \"http://json-schema.org/draft-07/schema\",\n    \"$ref\": \"#/definitions/element\",\n    \"definitions\": {\n        \"element\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"tagName\": {\"type\": \"string\"},\n                \"key\": {\"type\": [\"string\", \"number\", \"null\"]},\n                \"error\": {\"type\": \"string\"},\n                \"children\": {\"$ref\": \"#/definitions/elementChildren\"},\n                \"attributes\": {\"type\": \"object\"},\n                \"eventHandlers\": {\"$ref\": \"#/definitions/elementEventHandlers\"},\n                \"importSource\": {\"$ref\": \"#/definitions/importSource\"},\n            },\n            # The 'tagName' is required because its presence is a useful indicator of\n            # whether a dictionary describes a VDOM model or not.\n            \"required\": [\"tagName\"],\n            \"dependentSchemas\": {\n                # When 'error' is given, the 'tagName' should be empty.\n                \"error\": {\"properties\": {\"tagName\": {\"maxLength\": 0}}}\n            },\n        },\n        \"elementChildren\": {\n            \"type\": \"array\",\n            \"items\": {\"$ref\": \"#/definitions/elementOrString\"},\n        },\n        \"elementEventHandlers\": {\n            \"type\": \"object\",\n            \"patternProperties\": {\n                \".*\": {\"$ref\": \"#/definitions/eventHandler\"},\n            },\n        },\n        \"eventHandler\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"target\": {\"type\": \"string\"},\n                \"preventDefault\": {\"type\": \"boolean\"},\n                \"stopPropagation\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"target\"],\n        },\n        \"importSource\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"source\": {\"type\": \"string\"},\n                \"sourceType\": {\"enum\": [\"URL\", \"NAME\"]},\n                \"fallback\": {\n                    \"type\": [\"object\", \"string\", \"null\"],\n                    \"if\": {\"not\": {\"type\": \"null\"}},\n                    \"then\": {\"$ref\": \"#/definitions/elementOrString\"},\n                },\n                \"unmountBeforeUpdate\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"source\"],\n        },\n        \"elementOrString\": {\n            \"type\": [\"object\", \"string\"],\n            \"if\": {\"type\": \"object\"},\n            \"then\": {\"$ref\": \"#/definitions/element\"},\n        },\n    },\n}\n\"\"\"JSON Schema describing serialized VDOM - see :ref:`VDOM` for more info\"\"\"\n\n\n# we can't add a docstring to this because Sphinx doesn't know how to find its source\n_COMPILED_VDOM_VALIDATOR = compile_json_schema(VDOM_JSON_SCHEMA)\n\n\ndef validate_vdom_json(value: Any) -> VdomJson:\n    \"\"\"Validate serialized VDOM - see :attr:`VDOM_JSON_SCHEMA` for more info\"\"\"\n    _COMPILED_VDOM_VALIDATOR(value)\n    return cast(VdomJson, value)\n\n\ndef is_vdom(value: Any) -> bool:\n    \"\"\"Return whether a value is a :class:`VdomDict`\n\n    This employs a very simple heuristic - something is VDOM if:\n\n    1. It is a ``dict`` instance\n    2. It contains the key ``\"tagName\"``\n    3. The value of the key ``\"tagName\"`` is a string\n\n    .. note::\n\n        Performing an ``isinstance(value, VdomDict)`` check is too restrictive since the\n        user would be forced to import ``VdomDict`` every time they needed to declare a\n        VDOM element. Giving the user more flexibility, at the cost of this check's\n        accuracy, is worth it.\n    \"\"\"\n    return (\n        isinstance(value, dict)\n        and \"tagName\" in value\n        and isinstance(value[\"tagName\"], str)\n    )\n\n\n@overload\ndef vdom(tag: str, *children: VdomChildren) -> VdomDict: ...\n\n\n@overload\ndef vdom(tag: str, attributes: VdomAttributes, *children: VdomChildren) -> VdomDict: ...\n\n\ndef vdom(\n    tag: str,\n    *attributes_and_children: Any,\n    **kwargs: Any,\n) -> VdomDict:\n...\n", "needle_token_start": 7172, "needle_token_end": 7254, "code_context_ntokens": 11219, "output": ["```python\ndef set_context_provider(self, provider: ContextProviderType[Any]) -> None:\n    \"\"\"Set a context provider for this hook\n\n    The context provider will be used to provide state to any child components\n    of this hook's component which request a context provider of the same type.\n    \"\"\"\n    self._context_providers[provider.type] = provider\n```"]}
{"repo": "reactive-python/reactpy", "name": "boolean", "language": "python", "path": "src/py/reactpy/reactpy/config.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To convert various data types into a boolean value based on predefined true and false representations.\n2. **Input**: Accepts a string, boolean, or integer.\n3. **Output**: Returns a boolean value.\n4. **Procedure**: The function first checks the data type of the input. If it's already a boolean, it returns the value directly. If it's an integer, it converts it to a boolean where zero is `False` and any non-zero value is `True`. If the input is a string, the function converts it to lowercase and checks against predefined sets of values that represent true and false. If the string matches a true value, it returns `True`; if it matches a false value, it returns `False`. If the string does not match any predefined values, it raises a `ValueError`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/backend/__init__.py\nimport mimetypes\nfrom logging import getLogger\n\n_logger = getLogger(__name__)\n\n# Fix for missing mime types due to OS corruption/misconfiguration\n# Example: https://github.com/encode/starlette/issues/829\nif not mimetypes.inited:\n    mimetypes.init()\nfor extension, mime_type in {\n    \".js\": \"application/javascript\",\n    \".css\": \"text/css\",\n    \".json\": \"application/json\",\n}.items():\n    if not mimetypes.types_map.get(extension):  # pragma: no cover\n        _logger.warning(\n            \"Mime type '%s = %s' is missing. Please research how to \"\n            \"fix missing mime types on your operating system.\",\n            extension,\n            mime_type,\n        )\n        mimetypes.add_type(mime_type, extension)\n\n# Path: src/py/reactpy/reactpy/core/__init__.py\n\n# Path: src/py/reactpy/reactpy/core/types.py\nfrom __future__ import annotations\n\nimport sys\nfrom collections import namedtuple\nfrom collections.abc import Mapping, Sequence\nfrom types import TracebackType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Generic,\n    Literal,\n    NamedTuple,\n    Protocol,\n    TypeVar,\n    overload,\n    runtime_checkable,\n)\n\nfrom typing_extensions import TypeAlias, TypedDict\n\n_Type = TypeVar(\"_Type\")\n\n\nif TYPE_CHECKING or sys.version_info < (3, 9) or sys.version_info >= (3, 11):\n\n    class State(NamedTuple, Generic[_Type]):\n        value: _Type\n        set_value: Callable[[_Type | Callable[[_Type], _Type]], None]\n\nelse:  # nocov\n    State = namedtuple(\"State\", (\"value\", \"set_value\"))\n\n\nComponentConstructor = Callable[..., \"ComponentType\"]\n\"\"\"Simple function returning a new component\"\"\"\n\nRootComponentConstructor = Callable[[], \"ComponentType\"]\n\"\"\"The root component should be constructed by a function accepting no arguments.\"\"\"\n\n\nKey: TypeAlias = \"str | int\"\n\n\n_OwnType = TypeVar(\"_OwnType\")\n\n\n@runtime_checkable\nclass ComponentType(Protocol):\n    \"\"\"The expected interface for all component-like objects\"\"\"\n\n    key: Key | None\n    \"\"\"An identifier which is unique amongst a component's immediate siblings\"\"\"\n\n    type: Any\n    \"\"\"The function or class defining the behavior of this component\n\n    This is used to see if two component instances share the same definition.\n    \"\"\"\n\n    def render(self) -> VdomDict | ComponentType | str | None:\n        \"\"\"Render the component's view model.\"\"\"\n\n\n_Render_co = TypeVar(\"_Render_co\", covariant=True)\n_Event_contra = TypeVar(\"_Event_contra\", contravariant=True)\n\n\n@runtime_checkable\nclass LayoutType(Protocol[_Render_co, _Event_contra]):\n    \"\"\"Renders and delivers, updates to views and events to handlers, respectively\"\"\"\n\n    async def render(self) -> _Render_co:\n        \"\"\"Render an update to a view\"\"\"\n\n    async def deliver(self, event: _Event_contra) -> None:\n        \"\"\"Relay an event to its respective handler\"\"\"\n\n    async def __aenter__(self) -> LayoutType[_Render_co, _Event_contra]:\n        \"\"\"Prepare the layout for its first render\"\"\"\n\n    async def __aexit__(\n        self,\n        exc_type: type[Exception],\n        exc_value: Exception,\n        traceback: TracebackType,\n    ) -> bool | None:\n        \"\"\"Clean up the view after its final render\"\"\"\n\n\nVdomAttributes = Mapping[str, Any]\n\"\"\"Describes the attributes of a :class:`VdomDict`\"\"\"\n\nVdomChild: TypeAlias = \"ComponentType | VdomDict | str | None | Any\"\n\"\"\"A single child element of a :class:`VdomDict`\"\"\"\n\nVdomChildren: TypeAlias = \"Sequence[VdomChild] | VdomChild\"\n\"\"\"Describes a series of :class:`VdomChild` elements\"\"\"\n\n\nclass _VdomDictOptional(TypedDict, total=False):\n    key: Key | None\n    children: Sequence[ComponentType | VdomChild]\n    attributes: VdomAttributes\n    eventHandlers: EventHandlerDict\n    importSource: ImportSourceDict\n\n\nclass _VdomDictRequired(TypedDict, total=True):\n    tagName: str\n\n\nclass VdomDict(_VdomDictRequired, _VdomDictOptional):\n    \"\"\"A :ref:`VDOM` dictionary\"\"\"\n\n\nclass ImportSourceDict(TypedDict):\n    source: str\n    fallback: Any\n    sourceType: str\n    unmountBeforeUpdate: bool\n\n\nclass _OptionalVdomJson(TypedDict, total=False):\n    key: Key\n    error: str\n    children: list[Any]\n    attributes: dict[str, Any]\n    eventHandlers: dict[str, _JsonEventTarget]\n    importSource: _JsonImportSource\n\n\nclass _RequiredVdomJson(TypedDict, total=True):\n    tagName: str\n\n\nclass VdomJson(_RequiredVdomJson, _OptionalVdomJson):\n    \"\"\"A JSON serializable form of :class:`VdomDict` matching the :data:`VDOM_JSON_SCHEMA`\"\"\"\n\n\nclass _JsonEventTarget(TypedDict):\n    target: str\n    preventDefault: bool\n    stopPropagation: bool\n\n\nclass _JsonImportSource(TypedDict):\n    source: str\n    fallback: Any\n\n\nEventHandlerMapping = Mapping[str, \"EventHandlerType\"]\n\"\"\"A generic mapping between event names to their handlers\"\"\"\n\nEventHandlerDict: TypeAlias = \"dict[str, EventHandlerType]\"\n\"\"\"A dict mapping between event names to their handlers\"\"\"\n\n\nclass EventHandlerFunc(Protocol):\n    \"\"\"A coroutine which can handle event data\"\"\"\n\n    async def __call__(self, data: Sequence[Any]) -> None: ...\n\n\n@runtime_checkable\nclass EventHandlerType(Protocol):\n    \"\"\"Defines a handler for some event\"\"\"\n\n    prevent_default: bool\n    \"\"\"Whether to block the event from propagating further up the DOM\"\"\"\n\n    stop_propagation: bool\n    \"\"\"Stops the default action associate with the event from taking place.\"\"\"\n\n    function: EventHandlerFunc\n    \"\"\"A coroutine which can respond to an event and its data\"\"\"\n\n    target: str | None\n    \"\"\"Typically left as ``None`` except when a static target is useful.\n\n    When testing, it may be useful to specify a static target ID so events can be\n    triggered programmatically.\n\n    .. note::\n\n        When ``None``, it is left to a :class:`LayoutType` to auto generate a unique ID.\n    \"\"\"\n\n\nclass VdomDictConstructor(Protocol):\n    \"\"\"Standard function for constructing a :class:`VdomDict`\"\"\"\n\n    @overload\n    def __call__(\n        self, attributes: VdomAttributes, *children: VdomChildren\n    ) -> VdomDict: ...\n\n    @overload\n    def __call__(self, *children: VdomChildren) -> VdomDict: ...\n\n    @overload\n    def __call__(\n        self, *attributes_and_children: VdomAttributes | VdomChildren\n    ) -> VdomDict: ...\n\n\nclass LayoutUpdateMessage(TypedDict):\n    \"\"\"A message describing an update to a layout\"\"\"\n\n    type: Literal[\"layout-update\"]\n    \"\"\"The type of message\"\"\"\n    path: str\n    \"\"\"JSON Pointer path to the model element being updated\"\"\"\n    model: VdomJson\n    \"\"\"The model to assign at the given JSON Pointer path\"\"\"\n\n\nclass LayoutEventMessage(TypedDict):\n    \"\"\"Message describing an event originating from an element in the layout\"\"\"\n\n    type: Literal[\"layout-event\"]\n    \"\"\"The type of message\"\"\"\n    target: str\n    \"\"\"The ID of the event handler.\"\"\"\n    data: Sequence[Any]\n    \"\"\"A list of event data passed to the event handler.\"\"\"\n\n\nclass Context(Protocol[_Type]):\n    \"\"\"Returns a :class:`ContextProvider` component\"\"\"\n\n    def __call__(\n        self,\n        *children: Any,\n        value: _Type = ...,\n        key: Key | None = ...,\n    ) -> ContextProviderType[_Type]: ...\n\n\nclass ContextProviderType(ComponentType, Protocol[_Type]):\n    \"\"\"A component which provides a context value to its children\"\"\"\n\n    type: Context[_Type]\n    \"\"\"The context type\"\"\"\n\n    @property\n    def value(self) -> _Type:\n        \"Current context value\"\n\n# Path: src/py/reactpy/reactpy/backend/types.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import MutableMapping\nfrom dataclasses import dataclass\nfrom typing import Any, Callable, Generic, Protocol, TypeVar, runtime_checkable\n\nfrom reactpy.core.types import RootComponentConstructor\n\n_App = TypeVar(\"_App\")\n\n\n@runtime_checkable\nclass BackendType(Protocol[_App]):\n    \"\"\"Common interface for built-in web server/framework integrations\"\"\"\n\n    Options: Callable[..., Any]\n    \"\"\"A constructor for options passed to :meth:`BackendType.configure`\"\"\"\n\n    def configure(\n        self,\n        app: _App,\n        component: RootComponentConstructor,\n        options: Any | None = None,\n    ) -> None:\n        \"\"\"Configure the given app instance to display the given component\"\"\"\n\n    def create_development_app(self) -> _App:\n        \"\"\"Create an application instance for development purposes\"\"\"\n\n    async def serve_development_app(\n        self,\n        app: _App,\n        host: str,\n        port: int,\n        started: asyncio.Event | None = None,\n    ) -> None:\n        \"\"\"Run an application using a development server\"\"\"\n\n\n_Carrier = TypeVar(\"_Carrier\")\n\n\n@dataclass\nclass Connection(Generic[_Carrier]):\n    \"\"\"Represents a connection with a client\"\"\"\n\n    scope: MutableMapping[str, Any]\n    \"\"\"An ASGI scope or WSGI environment dictionary\"\"\"\n\n    location: Location\n    \"\"\"The current location (URL)\"\"\"\n\n    carrier: _Carrier\n    \"\"\"How the connection is mediated. For example, a request or websocket.\n\n    This typically depends on the backend implementation.\n    \"\"\"\n\n\n@dataclass\nclass Location:\n    \"\"\"Represents the current location (URL)\n\n    Analogous to, but not necessarily identical to, the client-side\n    ``document.location`` object.\n    \"\"\"\n\n    pathname: str\n    \"\"\"the path of the URL for the location\"\"\"\n\n    search: str\n    \"\"\"A search or query string - a '?' followed by the parameters of the URL.\n\n    If there are no search parameters this should be an empty string\n    \"\"\"\n\n# Path: src/py/reactpy/reactpy/_warnings.py\nfrom collections.abc import Iterator\nfrom functools import wraps\nfrom inspect import currentframe\nfrom types import FrameType\nfrom typing import TYPE_CHECKING, Any\nfrom warnings import warn as _warn\n\n\n@wraps(_warn)\ndef warn(*args: Any, **kwargs: Any) -> Any:\n    # warn at call site outside of ReactPy\n    _warn(*args, stacklevel=_frame_depth_in_module() + 1, **kwargs)  # type: ignore\n\n\nif TYPE_CHECKING:\n    warn = _warn  # noqa: F811\n\n\ndef _frame_depth_in_module() -> int:\n    depth = 0\n    for frame in _iter_frames(2):\n        module_name = frame.f_globals.get(\"__name__\")\n        if not module_name or not module_name.startswith(\"reactpy.\"):\n            break\n        depth += 1\n    return depth\n\n\ndef _iter_frames(index: int = 1) -> Iterator[FrameType]:\n    frame = currentframe()\n    while frame is not None:\n        if index == 0:\n            yield frame\n        else:\n            index -= 1\n        frame = frame.f_back\n\n# Path: src/py/reactpy/reactpy/_option.py\nfrom __future__ import annotations\n\nimport os\nfrom logging import getLogger\nfrom typing import Any, Callable, Generic, TypeVar, cast\n\nfrom reactpy._warnings import warn\n\n_O = TypeVar(\"_O\")\nlogger = getLogger(__name__)\nUNDEFINED = cast(Any, object())\n\n\nclass Option(Generic[_O]):\n    \"\"\"An option that can be set using an environment variable of the same name\"\"\"\n\n    def __init__(\n        self,\n        name: str,\n        default: _O = UNDEFINED,\n        mutable: bool = True,\n        parent: Option[_O] | None = None,\n        validator: Callable[[Any], _O] = lambda x: cast(_O, x),\n    ) -> None:\n        self._name = name\n        self._mutable = mutable\n        self._validator = validator\n        self._subscribers: list[Callable[[_O], None]] = []\n\n        if name in os.environ:\n            self._current = validator(os.environ[name])\n\n        if parent is not None:\n            if not (parent.mutable and self.mutable):\n                raise TypeError(\"Parent and child options must be mutable\")\n            self._default = parent.default\n            parent.subscribe(self.set_current)\n        elif default is not UNDEFINED:\n            self._default = default\n        else:\n            raise TypeError(\"Must specify either a default or a parent option\")\n\n        logger.debug(f\"{self._name}={self.current}\")\n\n    @property\n    def name(self) -> str:\n        \"\"\"The name of this option (used to load environment variables)\"\"\"\n        return self._name\n\n    @property\n    def mutable(self) -> bool:\n        \"\"\"Whether this option can be modified after being loaded\"\"\"\n        return self._mutable\n\n    @property\n    def default(self) -> _O:\n        \"\"\"This option's default value\"\"\"\n        return self._default\n\n    @property\n    def current(self) -> _O:\n        try:\n            return self._current\n        except AttributeError:\n            return self._default\n\n    @current.setter\n    def current(self, new: _O) -> None:\n        self.set_current(new)\n\n    @current.deleter\n    def current(self) -> None:\n        self.unset()\n\n    def subscribe(self, handler: Callable[[_O], None]) -> Callable[[_O], None]:\n        \"\"\"Register a callback that will be triggered when this option changes\"\"\"\n        if not self.mutable:\n            msg = \"Immutable options cannot be subscribed to.\"\n            raise TypeError(msg)\n        self._subscribers.append(handler)\n        handler(self.current)\n        return handler\n\n    def is_set(self) -> bool:\n        \"\"\"Whether this option has a value other than its default.\"\"\"\n        return hasattr(self, \"_current\")\n\n    def set_current(self, new: Any) -> None:\n        \"\"\"Set the value of this option\n\n        Raises a ``TypeError`` if this option is not :attr:`Option.mutable`.\n        \"\"\"\n        old = self.current\n        if new is old:\n            return None\n\n        if not self._mutable:\n            msg = f\"{self} cannot be modified after initial load\"\n            raise TypeError(msg)\n\n        try:\n            new = self._current = self._validator(new)\n        except ValueError as error:\n            raise ValueError(f\"Invalid value for {self._name}: {new!r}\") from error\n\n        logger.debug(f\"{self._name}={self._current}\")\n        if new != old:\n            for sub_func in self._subscribers:\n                sub_func(new)\n\n    def set_default(self, new: _O) -> _O:\n        \"\"\"Set the value of this option if not :meth:`Option.is_set`\n\n        Returns the current value (a la :meth:`dict.set_default`)\n        \"\"\"\n        if not self.is_set():\n            self.set_current(new)\n        return self._current\n\n    def reload(self) -> None:\n        \"\"\"Reload this option from its environment variable\"\"\"\n        self.set_current(os.environ.get(self._name, self._default))\n\n    def unset(self) -> None:\n        \"\"\"Remove the current value, the default will be used until it is set again.\"\"\"\n        if not self._mutable:\n            msg = f\"{self} cannot be modified after initial load\"\n            raise TypeError(msg)\n        old = self.current\n        if hasattr(self, \"_current\"):\n            delattr(self, \"_current\")\n        if self.current != old:\n            for sub_func in self._subscribers:\n                sub_func(self.current)\n\n    def __repr__(self) -> str:\n        return f\"Option({self._name}={self.current!r})\"\n\n\nclass DeprecatedOption(Option[_O]):\n    \"\"\"An option that will warn when it is accessed\"\"\"\n\n    def __init__(self, *args: Any, message: str, **kwargs: Any) -> None:\n        super().__init__(*args, **kwargs)\n        self._deprecation_message = message\n\n    @Option.current.getter  # type: ignore\n    def current(self) -> _O:\n        try:\n            # we access the current value during init to debug log it\n            # no need to warn unless it's actually used. since this attr\n            # is only set after super().__init__ is called, we can check\n            # for it to determine if it's being accessed by a user.\n            msg = self._deprecation_message\n        except AttributeError:\n            pass\n        else:\n            warn(msg, DeprecationWarning)\n        return super().current\n\n# Path: src/py/reactpy/reactpy/config.py\n\"\"\"\nReactPy provides a series of configuration options that can be set using environment\nvariables or, for those which allow it, a programmatic interface.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom pathlib import Path\nfrom tempfile import TemporaryDirectory\n\nfrom reactpy._option import Option\n\nTRUE_VALUES = {\"true\", \"1\"}\nFALSE_VALUES = {\"false\", \"0\"}\n\n\n\ndef boolean(value: str | bool | int) -> bool:\n    if isinstance(value, bool):\n        return value\n    elif isinstance(value, int):\n        return bool(value)\n    elif not isinstance(value, str):\n        raise TypeError(f\"Expected str or bool, got {type(value).__name__}\")\n\n    if value.lower() in TRUE_VALUES:\n        return True\n    elif value.lower() in FALSE_VALUES:\n        return False\n    else:\n        raise ValueError(\n            f\"Invalid boolean value {value!r} - expected \"\n            f\"one of {list(TRUE_VALUES | FALSE_VALUES)}\"\n        )\n\n\nREACTPY_DEBUG_MODE = Option(\n    \"REACTPY_DEBUG_MODE\", default=False, validator=boolean, mutable=True\n)\n\"\"\"Get extra logs and validation checks at the cost of performance.\n\nThis will enable the following:\n\n- :data:`REACTPY_CHECK_VDOM_SPEC`\n- :data:`REACTPY_CHECK_JSON_ATTRS`\n\"\"\"\n\nREACTPY_CHECK_VDOM_SPEC = Option(\"REACTPY_CHECK_VDOM_SPEC\", parent=REACTPY_DEBUG_MODE)\n\"\"\"Checks which ensure VDOM is rendered to spec\n\nFor more info on the VDOM spec, see here: :ref:`VDOM JSON Schema`\n\"\"\"\n\nREACTPY_CHECK_JSON_ATTRS = Option(\"REACTPY_CHECK_JSON_ATTRS\", parent=REACTPY_DEBUG_MODE)\n\"\"\"Checks that all VDOM attributes are JSON serializable\n\nThe VDOM spec is not able to enforce this on its own since attributes could anything.\n\"\"\"\n\n# Because these web modules will be linked dynamically at runtime this can be temporary.\n# Assigning to a variable here ensures that the directory is not deleted until the end\n# of the program.\n_DEFAULT_WEB_MODULES_DIR = TemporaryDirectory()\n\nREACTPY_WEB_MODULES_DIR = Option(\n    \"REACTPY_WEB_MODULES_DIR\",\n    default=Path(_DEFAULT_WEB_MODULES_DIR.name),\n    validator=Path,\n)\n\"\"\"The location ReactPy will use to store its client application\n\nThis directory **MUST** be treated as a black box. Downstream applications **MUST NOT**\nassume anything about the structure of this directory see :mod:`reactpy.web.module` for a\nset of publicly available APIs for working with the client.\n\"\"\"\n\nREACTPY_TESTING_DEFAULT_TIMEOUT = Option(\n    \"REACTPY_TESTING_DEFAULT_TIMEOUT\",\n    5.0,\n    mutable=False,\n    validator=float,\n)\n\"\"\"A default timeout for testing utilities in ReactPy\"\"\"\n\nREACTPY_ASYNC_RENDERING = Option(\n    \"REACTPY_CONCURRENT_RENDERING\",\n    default=False,\n    mutable=True,\n    validator=boolean,\n)\n\"\"\"Whether to render components concurrently. This is currently an experimental feature.\"\"\"\n\n# Path: src/py/reactpy/reactpy/core/_thread_local.py\nfrom threading import Thread, current_thread\nfrom typing import Callable, Generic, TypeVar\nfrom weakref import WeakKeyDictionary\n\n_StateType = TypeVar(\"_StateType\")\n\n\nclass ThreadLocal(Generic[_StateType]):\n    \"\"\"Utility for managing per-thread state information\"\"\"\n\n    def __init__(self, default: Callable[[], _StateType]):\n        self._default = default\n        self._state: WeakKeyDictionary[Thread, _StateType] = WeakKeyDictionary()\n\n    def get(self) -> _StateType:\n        thread = current_thread()\n        if thread not in self._state:\n            state = self._state[thread] = self._default()\n        else:\n            state = self._state[thread]\n        return state\n\n# Path: src/py/reactpy/reactpy/core/_life_cycle_hook.py\nfrom __future__ import annotations\n\nimport logging\nfrom asyncio import Event, Task, create_task, gather\nfrom typing import Any, Callable, Protocol, TypeVar\n\nfrom anyio import Semaphore\n\nfrom reactpy.core._thread_local import ThreadLocal\nfrom reactpy.core.types import ComponentType, Context, ContextProviderType\n\nT = TypeVar(\"T\")\n\n\nclass EffectFunc(Protocol):\n    async def __call__(self, stop: Event) -> None: ...\n\n\nlogger = logging.getLogger(__name__)\n\n_HOOK_STATE: ThreadLocal[list[LifeCycleHook]] = ThreadLocal(list)\n\n\ndef current_hook() -> LifeCycleHook:\n    \"\"\"Get the current :class:`LifeCycleHook`\"\"\"\n    hook_stack = _HOOK_STATE.get()\n    if not hook_stack:\n        msg = \"No life cycle hook is active. Are you rendering in a layout?\"\n        raise RuntimeError(msg)\n    return hook_stack[-1]\n\n\nclass LifeCycleHook:\n    \"\"\"An object which manages the \"life cycle\" of a layout component.\n\n    The \"life cycle\" of a component is the set of events which occur from the time\n    a component is first rendered until it is removed from the layout. The life cycle\n    is ultimately driven by the layout itself, but components can \"hook\" into those\n    events to perform actions. Components gain access to their own life cycle hook\n    by calling :func:`current_hook`. They can then perform actions such as:\n\n    1. Adding state via :meth:`use_state`\n    2. Adding effects via :meth:`add_effect`\n    3. Setting or getting context providers via\n       :meth:`LifeCycleHook.set_context_provider` and\n       :meth:`get_context_provider` respectively.\n\n    Components can request access to their own life cycle events and state through hooks\n    while :class:`~reactpy.core.proto.LayoutType` objects drive drive the life cycle\n    forward by triggering events and rendering view changes.\n\n    Example:\n\n        If removed from the complexities of a layout, a very simplified full life cycle\n        for a single component with no child components would look a bit like this:\n\n        .. testcode::\n\n            from reactpy.core._life_cycle_hook import LifeCycleHook\n            from reactpy.core.hooks import current_hook\n\n            # this function will come from a layout implementation\n            schedule_render = lambda: ...\n\n            # --- start life cycle ---\n\n            hook = LifeCycleHook(schedule_render)\n\n            # --- start render cycle ---\n\n            component = ...\n            await hook.affect_component_will_render(component)\n            try:\n                # render the component\n                ...\n\n                # the component may access the current hook\n                assert current_hook() is hook\n\n                # and save state or add effects\n                current_hook().use_state(lambda: ...)\n\n                async def my_effect(stop_event):\n                    ...\n\n                current_hook().add_effect(my_effect)\n            finally:\n                await hook.affect_component_did_render()\n\n            # This should only be called after the full set of changes associated with a\n            # given render have been completed.\n            await hook.affect_layout_did_render()\n\n            # Typically an event occurs and a new render is scheduled, thus beginning\n            # the render cycle anew.\n            hook.schedule_render()\n\n\n            # --- end render cycle ---\n\n            hook.affect_component_will_unmount()\n            del hook\n\n            # --- end render cycle ---\n    \"\"\"\n\n    __slots__ = (\n        \"__weakref__\",\n        \"_context_providers\",\n        \"_current_state_index\",\n        \"_effect_funcs\",\n        \"_effect_stops\",\n        \"_effect_tasks\",\n        \"_render_access\",\n        \"_rendered_atleast_once\",\n        \"_schedule_render_callback\",\n        \"_scheduled_render\",\n        \"_state\",\n        \"component\",\n    )\n\n    component: ComponentType\n\n    def __init__(\n        self,\n        schedule_render: Callable[[], None],\n    ) -> None:\n        self._context_providers: dict[Context[Any], ContextProviderType[Any]] = {}\n        self._schedule_render_callback = schedule_render\n        self._scheduled_render = False\n        self._rendered_atleast_once = False\n        self._current_state_index = 0\n        self._state: tuple[Any, ...] = ()\n        self._effect_funcs: list[EffectFunc] = []\n        self._effect_tasks: list[Task[None]] = []\n        self._effect_stops: list[Event] = []\n        self._render_access = Semaphore(1)  # ensure only one render at a time\n\n    def schedule_render(self) -> None:\n        if self._scheduled_render:\n            return None\n        try:\n            self._schedule_render_callback()\n        except Exception:\n            msg = f\"Failed to schedule render via {self._schedule_render_callback}\"\n            logger.exception(msg)\n        else:\n            self._scheduled_render = True\n\n    def use_state(self, function: Callable[[], T]) -> T:\n        \"\"\"Add state to this hook\n\n        If this hook has not yet rendered, the state is appended to the state tuple.\n        Otherwise, the state is retrieved from the tuple. This allows state to be\n        preserved across renders.\n        \"\"\"\n        if not self._rendered_atleast_once:\n            # since we're not initialized yet we're just appending state\n            result = function()\n            self._state += (result,)\n        else:\n            # once finalized we iterate over each succesively used piece of state\n            result = self._state[self._current_state_index]\n        self._current_state_index += 1\n        return result\n\n    def add_effect(self, effect_func: EffectFunc) -> None:\n        \"\"\"Add an effect to this hook\n\n...\n", "needle_token_start": 4624, "needle_token_end": 4786, "code_context_ntokens": 7097, "output": ["```python\ndef boolean(value: str | bool | int) -> bool:\n    if isinstance(value, bool):\n        return value\n    elif isinstance(value, int):\n        return bool(value)\n    elif not isinstance(value, str):\n        raise TypeError(f\"Expected str or bool, got {type(value).__name__}\")\n\n    if value.lower() in TRUE_VALUES:\n        return True\n    elif value.lower() in FALSE_VALUES:\n        return False\n    else:\n        raise ValueError(\n            f\"Invalid boolean value {value!r} - expected \"\n            f\"one of {list(TRUE_VALUES | FALSE_VALUES)}\"\n        )\n```"]}
{"repo": "reactive-python/reactpy", "name": "all_implementations", "language": "python", "path": "src/py/reactpy/reactpy/backend/utils.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to iterate through and provide all available server backend implementations supported by a Python library.\n2. **Input**: There are no inputs required for this function.\n3. **Output**: The function yields each successfully imported server backend implementation module.\n4. **Procedure**: The function iterates over a predefined list of backend names. For each name, it attempts to import the corresponding module. If the import is successful, the function then constructs the full module name for the backend specific to the library and yields the imported module. If the import fails, it logs the failure and continues to the next backend.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/py/reactpy/reactpy/core/_life_cycle_hook.py\nfrom __future__ import annotations\n\nimport logging\nfrom asyncio import Event, Task, create_task, gather\nfrom typing import Any, Callable, Protocol, TypeVar\n\nfrom anyio import Semaphore\n\nfrom reactpy.core._thread_local import ThreadLocal\nfrom reactpy.core.types import ComponentType, Context, ContextProviderType\n\nT = TypeVar(\"T\")\n\n\nclass EffectFunc(Protocol):\n    async def __call__(self, stop: Event) -> None: ...\n\n\nlogger = logging.getLogger(__name__)\n\n_HOOK_STATE: ThreadLocal[list[LifeCycleHook]] = ThreadLocal(list)\n\n\ndef current_hook() -> LifeCycleHook:\n    \"\"\"Get the current :class:`LifeCycleHook`\"\"\"\n    hook_stack = _HOOK_STATE.get()\n    if not hook_stack:\n        msg = \"No life cycle hook is active. Are you rendering in a layout?\"\n        raise RuntimeError(msg)\n    return hook_stack[-1]\n\n\nclass LifeCycleHook:\n    \"\"\"An object which manages the \"life cycle\" of a layout component.\n\n    The \"life cycle\" of a component is the set of events which occur from the time\n    a component is first rendered until it is removed from the layout. The life cycle\n    is ultimately driven by the layout itself, but components can \"hook\" into those\n    events to perform actions. Components gain access to their own life cycle hook\n    by calling :func:`current_hook`. They can then perform actions such as:\n\n    1. Adding state via :meth:`use_state`\n    2. Adding effects via :meth:`add_effect`\n    3. Setting or getting context providers via\n       :meth:`LifeCycleHook.set_context_provider` and\n       :meth:`get_context_provider` respectively.\n\n    Components can request access to their own life cycle events and state through hooks\n    while :class:`~reactpy.core.proto.LayoutType` objects drive drive the life cycle\n    forward by triggering events and rendering view changes.\n\n    Example:\n\n        If removed from the complexities of a layout, a very simplified full life cycle\n...\n# Path: src/py/reactpy/reactpy/core/_f_back.py\nfrom __future__ import annotations\n\nimport inspect\nfrom types import FrameType\n\n\ndef f_module_name(index: int = 0) -> str:\n    frame = f_back(index + 1)\n    if frame is None:\n        return \"\"  # nocov\n    name = frame.f_globals.get(\"__name__\", \"\")\n    if not isinstance(name, str):\n        raise TypeError(\"Expected module name to be a string\")  # nocov\n    return name\n\n\ndef f_back(index: int = 0) -> FrameType | None:\n    frame = inspect.currentframe()\n    while frame is not None:\n        if index < 0:\n            return frame\n        frame = frame.f_back\n        index -= 1\n    return None  # nocov\n\n# Path: src/py/reactpy/reactpy/core/events.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import Sequence\nfrom typing import Any, Callable, Literal, overload\n\nfrom anyio import create_task_group\n\nfrom reactpy.core.types import EventHandlerFunc, EventHandlerType\n\n\n@overload\ndef event(\n    function: Callable[..., Any],\n    *,\n    stop_propagation: bool = ...,\n    prevent_default: bool = ...,\n) -> EventHandler: ...\n\n\n@overload\ndef event(\n    function: Literal[None] = ...,\n    *,\n    stop_propagation: bool = ...,\n    prevent_default: bool = ...,\n) -> Callable[[Callable[..., Any]], EventHandler]: ...\n\n\ndef event(\n    function: Callable[..., Any] | None = None,\n    *,\n    stop_propagation: bool = False,\n    prevent_default: bool = False,\n) -> EventHandler | Callable[[Callable[..., Any]], EventHandler]:\n    \"\"\"A decorator for constructing an :class:`EventHandler`.\n\n    While you're always free to add callbacks by assigning them to an element's attributes\n\n    .. code-block:: python\n\n        element = reactpy.html.button({\"onClick\": my_callback})\n\n    You may want the ability to prevent the default action associated with the event\n    from taking place, or stopping the event from propagating up the DOM. This decorator\n    allows you to add that functionality to your callbacks.\n\n    .. code-block:: python\n\n        @event(stop_propagation=True, prevent_default=True)\n        def my_callback(*data):\n            ...\n\n        element = reactpy.html.button({\"onClick\": my_callback})\n\n    Parameters:\n        function:\n            A function or coroutine responsible for handling the event.\n        stop_propagation:\n            Block the event from propagating further up the DOM.\n        prevent_default:\n            Stops the default actional associate with the event from taking place.\n    \"\"\"\n\n    def setup(function: Callable[..., Any]) -> EventHandler:\n        return EventHandler(\n            to_event_handler_function(function, positional_args=True),\n            stop_propagation,\n            prevent_default,\n        )\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass EventHandler:\n    \"\"\"Turn a function or coroutine into an event handler\n\n    Parameters:\n        function:\n            The function or coroutine which handles the event.\n        stop_propagation:\n            Block the event from propagating further up the DOM.\n        prevent_default:\n            Stops the default action associate with the event from taking place.\n        target:\n            A unique identifier for this event handler (auto-generated by default)\n    \"\"\"\n\n    __slots__ = (\n        \"__weakref__\",\n        \"function\",\n        \"prevent_default\",\n        \"stop_propagation\",\n        \"target\",\n    )\n\n    def __init__(\n        self,\n        function: EventHandlerFunc,\n        stop_propagation: bool = False,\n        prevent_default: bool = False,\n        target: str | None = None,\n    ) -> None:\n        self.function = to_event_handler_function(function, positional_args=False)\n        self.prevent_default = prevent_default\n        self.stop_propagation = stop_propagation\n        self.target = target\n\n    def __eq__(self, other: Any) -> bool:\n        undefined = object()\n        for attr in (\n            \"function\",\n            \"prevent_default\",\n            \"stop_propagation\",\n            \"target\",\n        ):\n            if not attr.startswith(\"_\"):\n                if not getattr(other, attr, undefined) == getattr(self, attr):\n                    return False\n        return True\n\n    def __repr__(self) -> str:\n        public_names = [name for name in self.__slots__ if not name.startswith(\"_\")]\n        items = \", \".join([f\"{n}={getattr(self, n)!r}\" for n in public_names])\n        return f\"{type(self).__name__}({items})\"\n\n\ndef to_event_handler_function(\n    function: Callable[..., Any],\n    positional_args: bool = True,\n) -> EventHandlerFunc:\n    \"\"\"Make a :data:`~reactpy.core.proto.EventHandlerFunc` from a function or coroutine\n\n    Parameters:\n        function:\n            A function or coroutine accepting a number of positional arguments.\n        positional_args:\n            Whether to pass the event parameters a positional args or as a list.\n    \"\"\"\n    if positional_args:\n        if asyncio.iscoroutinefunction(function):\n\n            async def wrapper(data: Sequence[Any]) -> None:\n                await function(*data)\n\n        else:\n\n            async def wrapper(data: Sequence[Any]) -> None:\n                function(*data)\n\n        return wrapper\n    elif not asyncio.iscoroutinefunction(function):\n\n        async def wrapper(data: Sequence[Any]) -> None:\n            function(data)\n\n        return wrapper\n    else:\n        return function\n\n\ndef merge_event_handlers(\n    event_handlers: Sequence[EventHandlerType],\n) -> EventHandlerType:\n    \"\"\"Merge multiple event handlers into one\n\n    Raises a ValueError if any handlers have conflicting\n    :attr:`~reactpy.core.proto.EventHandlerType.stop_propagation` or\n    :attr:`~reactpy.core.proto.EventHandlerType.prevent_default` attributes.\n    \"\"\"\n    if not event_handlers:\n        msg = \"No event handlers to merge\"\n        raise ValueError(msg)\n    elif len(event_handlers) == 1:\n        return event_handlers[0]\n\n    first_handler = event_handlers[0]\n\n    stop_propagation = first_handler.stop_propagation\n    prevent_default = first_handler.prevent_default\n    target = first_handler.target\n\n    for handler in event_handlers:\n        if (\n            handler.stop_propagation != stop_propagation\n            or handler.prevent_default != prevent_default\n            or handler.target != target\n        ):\n            msg = \"Cannot merge handlers - 'stop_propagation', 'prevent_default' or 'target' mismatch.\"\n            raise ValueError(msg)\n\n    return EventHandler(\n        merge_event_handler_funcs([h.function for h in event_handlers]),\n        stop_propagation,\n        prevent_default,\n        target,\n    )\n\n\ndef merge_event_handler_funcs(\n    functions: Sequence[EventHandlerFunc],\n) -> EventHandlerFunc:\n    \"\"\"Make one event handler function from many\"\"\"\n    if not functions:\n        msg = \"No event handler functions to merge\"\n        raise ValueError(msg)\n    elif len(functions) == 1:\n        return functions[0]\n\n    async def await_all_event_handlers(data: Sequence[Any]) -> None:\n        async with create_task_group() as group:\n            for func in functions:\n                group.start_soon(func, data)\n\n    return await_all_event_handlers\n\n# Path: src/py/reactpy/reactpy/core/vdom.py\nfrom __future__ import annotations\n\nimport json\nfrom collections.abc import Mapping, Sequence\nfrom functools import wraps\nfrom typing import Any, Protocol, cast, overload\n\nfrom fastjsonschema import compile as compile_json_schema\n\nfrom reactpy._warnings import warn\nfrom reactpy.config import REACTPY_CHECK_JSON_ATTRS, REACTPY_DEBUG_MODE\nfrom reactpy.core._f_back import f_module_name\nfrom reactpy.core.events import EventHandler, to_event_handler_function\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomDictConstructor,\n    VdomJson,\n)\n\nVDOM_JSON_SCHEMA = {\n    \"$schema\": \"http://json-schema.org/draft-07/schema\",\n    \"$ref\": \"#/definitions/element\",\n    \"definitions\": {\n        \"element\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"tagName\": {\"type\": \"string\"},\n                \"key\": {\"type\": [\"string\", \"number\", \"null\"]},\n                \"error\": {\"type\": \"string\"},\n                \"children\": {\"$ref\": \"#/definitions/elementChildren\"},\n                \"attributes\": {\"type\": \"object\"},\n                \"eventHandlers\": {\"$ref\": \"#/definitions/elementEventHandlers\"},\n                \"importSource\": {\"$ref\": \"#/definitions/importSource\"},\n            },\n            # The 'tagName' is required because its presence is a useful indicator of\n            # whether a dictionary describes a VDOM model or not.\n            \"required\": [\"tagName\"],\n            \"dependentSchemas\": {\n                # When 'error' is given, the 'tagName' should be empty.\n                \"error\": {\"properties\": {\"tagName\": {\"maxLength\": 0}}}\n            },\n        },\n        \"elementChildren\": {\n            \"type\": \"array\",\n            \"items\": {\"$ref\": \"#/definitions/elementOrString\"},\n        },\n        \"elementEventHandlers\": {\n            \"type\": \"object\",\n            \"patternProperties\": {\n                \".*\": {\"$ref\": \"#/definitions/eventHandler\"},\n            },\n        },\n        \"eventHandler\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"target\": {\"type\": \"string\"},\n                \"preventDefault\": {\"type\": \"boolean\"},\n                \"stopPropagation\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"target\"],\n        },\n        \"importSource\": {\n            \"type\": \"object\",\n            \"properties\": {\n                \"source\": {\"type\": \"string\"},\n                \"sourceType\": {\"enum\": [\"URL\", \"NAME\"]},\n                \"fallback\": {\n                    \"type\": [\"object\", \"string\", \"null\"],\n                    \"if\": {\"not\": {\"type\": \"null\"}},\n                    \"then\": {\"$ref\": \"#/definitions/elementOrString\"},\n                },\n                \"unmountBeforeUpdate\": {\"type\": \"boolean\"},\n            },\n            \"required\": [\"source\"],\n        },\n        \"elementOrString\": {\n            \"type\": [\"object\", \"string\"],\n            \"if\": {\"type\": \"object\"},\n            \"then\": {\"$ref\": \"#/definitions/element\"},\n        },\n    },\n}\n\"\"\"JSON Schema describing serialized VDOM - see :ref:`VDOM` for more info\"\"\"\n\n\n# we can't add a docstring to this because Sphinx doesn't know how to find its source\n_COMPILED_VDOM_VALIDATOR = compile_json_schema(VDOM_JSON_SCHEMA)\n\n\ndef validate_vdom_json(value: Any) -> VdomJson:\n    \"\"\"Validate serialized VDOM - see :attr:`VDOM_JSON_SCHEMA` for more info\"\"\"\n    _COMPILED_VDOM_VALIDATOR(value)\n    return cast(VdomJson, value)\n\n\ndef is_vdom(value: Any) -> bool:\n    \"\"\"Return whether a value is a :class:`VdomDict`\n\n    This employs a very simple heuristic - something is VDOM if:\n\n    1. It is a ``dict`` instance\n    2. It contains the key ``\"tagName\"``\n    3. The value of the key ``\"tagName\"`` is a string\n\n    .. note::\n\n        Performing an ``isinstance(value, VdomDict)`` check is too restrictive since the\n        user would be forced to import ``VdomDict`` every time they needed to declare a\n        VDOM element. Giving the user more flexibility, at the cost of this check's\n        accuracy, is worth it.\n    \"\"\"\n    return (\n        isinstance(value, dict)\n        and \"tagName\" in value\n        and isinstance(value[\"tagName\"], str)\n    )\n\n\n@overload\ndef vdom(tag: str, *children: VdomChildren) -> VdomDict: ...\n\n\n@overload\ndef vdom(tag: str, attributes: VdomAttributes, *children: VdomChildren) -> VdomDict: ...\n\n\ndef vdom(\n    tag: str,\n    *attributes_and_children: Any,\n    **kwargs: Any,\n) -> VdomDict:\n    \"\"\"A helper function for creating VDOM elements.\n\n    Parameters:\n        tag:\n            The type of element (e.g. 'div', 'h1', 'img')\n        attributes_and_children:\n            An optional attribute mapping followed by any number of children or\n            iterables of children. The attribute mapping **must** precede the children,\n            or children which will be merged into their respective parts of the model.\n        key:\n            A string indicating the identity of a particular element. This is significant\n            to preserve event handlers across updates - without a key, a re-render would\n            cause these handlers to be deleted, but with a key, they would be redirected\n            to any newly defined handlers.\n        event_handlers:\n            Maps event types to coroutines that are responsible for handling those events.\n        import_source:\n            (subject to change) specifies javascript that, when evaluated returns a\n            React component.\n    \"\"\"\n    if kwargs:  # nocov\n        if \"key\" in kwargs:\n            if attributes_and_children:\n                maybe_attributes, *children = attributes_and_children\n                if _is_attributes(maybe_attributes):\n                    attributes_and_children = (\n                        {**maybe_attributes, \"key\": kwargs.pop(\"key\")},\n                        *children,\n                    )\n                else:\n                    attributes_and_children = (\n                        {\"key\": kwargs.pop(\"key\")},\n                        maybe_attributes,\n                        *children,\n                    )\n            else:\n                attributes_and_children = ({\"key\": kwargs.pop(\"key\")},)\n            warn(\n                \"An element's 'key' must be declared in an attribute dict instead \"\n                \"of as a keyword argument. This will error in a future version.\",\n                DeprecationWarning,\n            )\n\n        if kwargs:\n            msg = f\"Extra keyword arguments {kwargs}\"\n            raise ValueError(msg)\n\n    model: VdomDict = {\"tagName\": tag}\n\n    if not attributes_and_children:\n        return model\n\n    attributes, children = separate_attributes_and_children(attributes_and_children)\n    key = attributes.pop(\"key\", None)\n    attributes, event_handlers = separate_attributes_and_event_handlers(attributes)\n\n    if attributes:\n        if REACTPY_CHECK_JSON_ATTRS.current:\n            json.dumps(attributes)\n        model[\"attributes\"] = attributes\n\n    if children:\n        model[\"children\"] = children\n\n    if key is not None:\n        model[\"key\"] = key\n\n    if event_handlers:\n        model[\"eventHandlers\"] = event_handlers\n\n    return model\n\n\ndef make_vdom_constructor(\n    tag: str, allow_children: bool = True, import_source: ImportSourceDict | None = None\n) -> VdomDictConstructor:\n    \"\"\"Return a constructor for VDOM dictionaries with the given tag name.\n\n    The resulting callable will have the same interface as :func:`vdom` but without its\n    first ``tag`` argument.\n    \"\"\"\n\n    def constructor(*attributes_and_children: Any, **kwargs: Any) -> VdomDict:\n        model = vdom(tag, *attributes_and_children, **kwargs)\n        if not allow_children and \"children\" in model:\n            msg = f\"{tag!r} nodes cannot have children.\"\n            raise TypeError(msg)\n        if import_source:\n            model[\"importSource\"] = import_source\n        return model\n\n    # replicate common function attributes\n    constructor.__name__ = tag\n    constructor.__doc__ = (\n        \"Return a new \"\n        f\"`<{tag}> <https://developer.mozilla.org/en-US/docs/Web/HTML/Element/{tag}>`__ \"\n        \"element represented by a :class:`VdomDict`.\"\n    )\n\n    module_name = f_module_name(1)\n    if module_name:\n        constructor.__module__ = module_name\n        constructor.__qualname__ = f\"{module_name}.{tag}\"\n\n    return cast(VdomDictConstructor, constructor)\n\n\ndef custom_vdom_constructor(func: _CustomVdomDictConstructor) -> VdomDictConstructor:\n    \"\"\"Cast function to VdomDictConstructor\"\"\"\n\n    @wraps(func)\n    def wrapper(*attributes_and_children: Any) -> VdomDict:\n        attributes, children = separate_attributes_and_children(attributes_and_children)\n        key = attributes.pop(\"key\", None)\n        attributes, event_handlers = separate_attributes_and_event_handlers(attributes)\n        return func(attributes, children, key, event_handlers)\n\n    return cast(VdomDictConstructor, wrapper)\n\n\ndef separate_attributes_and_children(\n    values: Sequence[Any],\n) -> tuple[dict[str, Any], list[Any]]:\n    if not values:\n        return {}, []\n\n    attributes: dict[str, Any]\n    children_or_iterables: Sequence[Any]\n    if _is_attributes(values[0]):\n        attributes, *children_or_iterables = values\n    else:\n        attributes = {}\n        children_or_iterables = values\n\n    children: list[Any] = []\n    for child in children_or_iterables:\n        if _is_single_child(child):\n            children.append(child)\n        else:\n            children.extend(child)\n\n    return attributes, children\n\n\ndef separate_attributes_and_event_handlers(\n    attributes: Mapping[str, Any]\n) -> tuple[dict[str, Any], EventHandlerDict]:\n    separated_attributes = {}\n    separated_event_handlers: dict[str, EventHandlerType] = {}\n\n    for k, v in attributes.items():\n        handler: EventHandlerType\n\n        if callable(v):\n            handler = EventHandler(to_event_handler_function(v))\n        elif (\n            # isinstance check on protocols is slow - use function attr pre-check as a\n            # quick filter before actually performing slow EventHandlerType type check\n            hasattr(v, \"function\")\n            and isinstance(v, EventHandlerType)\n        ):\n            handler = v\n        else:\n            separated_attributes[k] = v\n            continue\n\n        separated_event_handlers[k] = handler\n\n    return separated_attributes, dict(separated_event_handlers.items())\n\n\ndef _is_attributes(value: Any) -> bool:\n    return isinstance(value, Mapping) and \"tagName\" not in value\n\n\ndef _is_single_child(value: Any) -> bool:\n    if isinstance(value, (str, Mapping)) or not hasattr(value, \"__iter__\"):\n        return True\n    if REACTPY_DEBUG_MODE.current:\n        _validate_child_key_integrity(value)\n    return False\n\n\ndef _validate_child_key_integrity(value: Any) -> None:\n    if hasattr(value, \"__iter__\") and not hasattr(value, \"__len__\"):\n        warn(\n            f\"Did not verify key-path integrity of children in generator {value} \"\n            \"- pass a sequence (i.e. list of finite length) in order to verify\"\n        )\n    else:\n        for child in value:\n            if isinstance(child, ComponentType) and child.key is None:\n                warn(f\"Key not specified for child in list {child}\", UserWarning)\n            elif isinstance(child, Mapping) and \"key\" not in child:\n                # remove 'children' to reduce log spam\n                child_copy = {**child, \"children\": _EllipsisRepr()}\n                warn(f\"Key not specified for child in list {child_copy}\", UserWarning)\n\n\nclass _CustomVdomDictConstructor(Protocol):\n    def __call__(\n        self,\n        attributes: VdomAttributes,\n        children: Sequence[VdomChild],\n        key: Key | None,\n        event_handlers: EventHandlerDict,\n    ) -> VdomDict: ...\n\n\nclass _EllipsisRepr:\n    def __repr__(self) -> str:\n        return \"...\"\n\n# Path: src/py/reactpy/reactpy/utils.py\nfrom __future__ import annotations\n\nimport re\nfrom collections.abc import Iterable\nfrom itertools import chain\nfrom typing import Any, Callable, Generic, TypeVar, cast\n\nfrom lxml import etree\nfrom lxml.html import fromstring, tostring\n\nfrom reactpy.core.types import VdomDict\nfrom reactpy.core.vdom import vdom\n\n_RefValue = TypeVar(\"_RefValue\")\n_ModelTransform = Callable[[VdomDict], Any]\n_UNDEFINED: Any = object()\n\n\nclass Ref(Generic[_RefValue]):\n    \"\"\"Hold a reference to a value\n\n    This is used in imperative code to mutate the state of this object in order to\n    incur side effects. Generally refs should be avoided if possible, but sometimes\n    they are required.\n\n    Notes:\n        You can compare the contents for two ``Ref`` objects using the ``==`` operator.\n    \"\"\"\n\n    __slots__ = (\"current\",)\n\n    def __init__(self, initial_value: _RefValue = _UNDEFINED) -> None:\n        if initial_value is not _UNDEFINED:\n            self.current = initial_value\n            \"\"\"The present value\"\"\"\n\n    def set_current(self, new: _RefValue) -> _RefValue:\n        \"\"\"Set the current value and return what is now the old value\n\n        This is nice to use in ``lambda`` functions.\n        \"\"\"\n        old = self.current\n        self.current = new\n        return old\n\n    def __eq__(self, other: Any) -> bool:\n        try:\n            return isinstance(other, Ref) and (other.current == self.current)\n        except AttributeError:\n            # attribute error occurs for uninitialized refs\n            return False\n\n    def __repr__(self) -> str:\n        try:\n            current = repr(self.current)\n        except AttributeError:\n            # attribute error occurs for uninitialized refs\n            current = \"<undefined>\"\n        return f\"{type(self).__name__}({current})\"\n\n\ndef vdom_to_html(vdom: VdomDict) -> str:\n    \"\"\"Convert a VDOM dictionary into an HTML string\n\n    Only the following keys are translated to HTML:\n\n    - ``tagName``\n    - ``attributes``\n    - ``children`` (must be strings or more VDOM dicts)\n\n    Parameters:\n        vdom: The VdomDict element to convert to HTML\n    \"\"\"\n    temp_root = etree.Element(\"__temp__\")\n    _add_vdom_to_etree(temp_root, vdom)\n    html = cast(bytes, tostring(temp_root)).decode()\n    # strip out temp root <__temp__> element\n    return html[10:-11]\n\n\ndef html_to_vdom(\n    html: str, *transforms: _ModelTransform, strict: bool = True\n) -> VdomDict:\n    \"\"\"Transform HTML into a DOM model. Unique keys can be provided to HTML elements\n    using a ``key=...`` attribute within your HTML tag.\n\n    Parameters:\n        html:\n            The raw HTML as a string\n        transforms:\n            Functions of the form ``transform(old) -> new`` where ``old`` is a VDOM\n            dictionary which will be replaced by ``new``. For example, you could use a\n            transform function to add highlighting to a ``<code/>`` block.\n        strict:\n            If ``True``, raise an exception if the HTML does not perfectly follow HTML5\n            syntax.\n    \"\"\"\n    if not isinstance(html, str):  # nocov\n        msg = f\"Expected html to be a string, not {type(html).__name__}\"\n        raise TypeError(msg)\n\n    # If the user provided a string, convert it to a list of lxml.etree nodes\n    try:\n        root_node: etree._Element = fromstring(\n            html.strip(),\n            parser=etree.HTMLParser(\n                remove_comments=True,\n                remove_pis=True,\n                remove_blank_text=True,\n                recover=not strict,\n            ),\n        )\n    except etree.XMLSyntaxError as e:\n        if not strict:\n            raise e  # nocov\n        msg = \"An error has occurred while parsing the HTML.\\n\\nThis HTML may be malformatted, or may not perfectly adhere to HTML5.\\nIf you believe the exception above was due to something intentional, you can disable the strict parameter on html_to_vdom().\\nOtherwise, repair your broken HTML and try again.\"\n        raise HTMLParseError(msg) from e\n\n    return _etree_to_vdom(root_node, transforms)\n\n\nclass HTMLParseError(etree.LxmlSyntaxError):  # type: ignore[misc]\n    \"\"\"Raised when an HTML document cannot be parsed using strict parsing.\"\"\"\n\n\ndef _etree_to_vdom(\n    node: etree._Element, transforms: Iterable[_ModelTransform]\n) -> VdomDict:\n    \"\"\"Transform an lxml etree node into a DOM model\n\n    Parameters:\n        node:\n            The ``lxml.etree._Element`` node\n        transforms:\n            Functions of the form ``transform(old) -> new`` where ``old`` is a VDOM\n            dictionary which will be replaced by ``new``. For example, you could use a\n            transform function to add highlighting to a ``<code/>`` block.\n    \"\"\"\n    if not isinstance(node, etree._Element):  # nocov\n        msg = f\"Expected node to be a etree._Element, not {type(node).__name__}\"\n        raise TypeError(msg)\n\n    # Recursively call _etree_to_vdom() on all children\n    children = _generate_vdom_children(node, transforms)\n\n    # Convert the lxml node to a VDOM dict\n    el = vdom(node.tag, dict(node.items()), *children)\n\n    # Perform any necessary mutations on the VDOM attributes to meet VDOM spec\n    _mutate_vdom(el)\n\n    # Apply any provided transforms.\n    for transform in transforms:\n        el = transform(el)\n\n    return el\n\n\ndef _add_vdom_to_etree(parent: etree._Element, vdom: VdomDict | dict[str, Any]) -> None:\n    try:\n        tag = vdom[\"tagName\"]\n    except KeyError as e:\n        msg = f\"Expected a VDOM dict, not {vdom}\"\n        raise TypeError(msg) from e\n    else:\n        vdom = cast(VdomDict, vdom)\n\n    if tag:\n        element = etree.SubElement(parent, tag)\n        element.attrib.update(\n            _vdom_attr_to_html_str(k, v) for k, v in vdom.get(\"attributes\", {}).items()\n        )\n    else:\n        element = parent\n\n    for c in vdom.get(\"children\", []):\n        if isinstance(c, dict):\n            _add_vdom_to_etree(element, c)\n        else:\n            \"\"\"\n            LXML handles string children by storing them under `text` and `tail`\n            attributes of Element objects. The `text` attribute, if present, effectively\n            becomes that element's first child. Then the `tail` attribute, if present,\n            becomes a sibling that follows that element. For example, consider the\n            following HTML:\n\n                <p><a>hello</a>world</p>\n\n            In this code sample, \"hello\" is the `text` attribute of the `<a>` element\n            and \"world\" is the `tail` attribute of that same `<a>` element. It's for\n            this reason that, depending on whether the element being constructed has\n            non-string a child element, we need to assign a `text` vs `tail` attribute\n            to that element or the last non-string child respectively.\n            \"\"\"\n            if len(element):\n                last_child = element[-1]\n                last_child.tail = f\"{last_child.tail or ''}{c}\"\n            else:\n                element.text = f\"{element.text or ''}{c}\"\n\n\ndef _mutate_vdom(vdom: VdomDict) -> None:\n    \"\"\"Performs any necessary mutations on the VDOM attributes to meet VDOM spec.\n\n    Currently, this function only transforms the ``style`` attribute into a dictionary whose keys are\n    camelCase so as to be renderable by React.\n\n    This function may be extended in the future.\n    \"\"\"\n    # Determine if the style attribute needs to be converted to a dict\n    if (\n        \"attributes\" in vdom\n        and \"style\" in vdom[\"attributes\"]\n        and isinstance(vdom[\"attributes\"][\"style\"], str)\n    ):\n        # Convince type checker that it's safe to mutate attributes\n        assert isinstance(vdom[\"attributes\"], dict)  # noqa: S101\n\n        # Convert style attribute from str -> dict with camelCase keys\n        vdom[\"attributes\"][\"style\"] = {\n            key.strip().replace(\"-\", \"_\"): value.strip()\n            for key, value in (\n                part.split(\":\", 1)\n                for part in vdom[\"attributes\"][\"style\"].split(\";\")\n                if \":\" in part\n            )\n        }\n\n\ndef _generate_vdom_children(\n    node: etree._Element, transforms: Iterable[_ModelTransform]\n) -> list[VdomDict | str]:\n    \"\"\"Generates a list of VDOM children from an lxml node.\n\n    Inserts inner text and/or tail text in between VDOM children, if necessary.\n    \"\"\"\n    return (  # Get the inner text of the current node\n        [node.text] if node.text else []\n    ) + list(\n        chain(\n            *(\n                # Recursively convert each child node to VDOM\n                [_etree_to_vdom(child, transforms)]\n                # Insert the tail text between each child node\n                + ([child.tail] if child.tail else [])\n                for child in node.iterchildren(None)\n            )\n        )\n    )\n\n\ndef del_html_head_body_transform(vdom: VdomDict) -> VdomDict:\n    \"\"\"Transform intended for use with `html_to_vdom`.\n\n    Removes `<html>`, `<head>`, and `<body>` while preserving their children.\n\n    Parameters:\n        vdom:\n            The VDOM dictionary to transform.\n    \"\"\"\n    if vdom[\"tagName\"] in {\"html\", \"body\", \"head\"}:\n        return {\"tagName\": \"\", \"children\": vdom[\"children\"]}\n    return vdom\n\n\ndef _vdom_attr_to_html_str(key: str, value: Any) -> tuple[str, str]:\n    if key == \"style\":\n        if isinstance(value, dict):\n            value = \";\".join(\n                # We lower only to normalize - CSS is case-insensitive:\n                # https://www.w3.org/TR/css-fonts-3/#font-family-casing\n                f\"{_CAMEL_CASE_SUB_PATTERN.sub('-', k).lower()}:{v}\"\n                for k, v in value.items()\n            )\n    elif (\n        # camel to data-* attributes\n        key.startswith(\"data_\")\n        # camel to aria-* attributes\n        or key.startswith(\"aria_\")\n        # handle special cases\n        or key in DASHED_HTML_ATTRS\n    ):\n        key = key.replace(\"_\", \"-\")\n    elif (\n        # camel to data-* attributes\n        key.startswith(\"data\")\n        # camel to aria-* attributes\n        or key.startswith(\"aria\")\n        # handle special cases\n        or key in DASHED_HTML_ATTRS\n    ):\n        key = _CAMEL_CASE_SUB_PATTERN.sub(\"-\", key)\n\n    if callable(value):  # nocov\n        raise TypeError(f\"Cannot convert callable attribute {key}={value} to HTML\")\n\n    # Again, we lower the attribute name only to normalize - HTML is case-insensitive:\n    # http://w3c.github.io/html-reference/documents.html#case-insensitivity\n    return key.lower(), str(value)\n\n\n# see list of HTML attributes with dashes in them:\n# https://developer.mozilla.org/en-US/docs/Web/HTML/Attributes#attribute_list\nDASHED_HTML_ATTRS = {\"accept_charset\", \"acceptCharset\", \"http_equiv\", \"httpEquiv\"}\n\n# Pattern for delimitting camelCase names (e.g. camelCase to camel-case)\n_CAMEL_CASE_SUB_PATTERN = re.compile(r\"(?<!^)(?=[A-Z])\")\n\n# Path: src/py/reactpy/reactpy/core/hooks.py\nfrom __future__ import annotations\n\nimport asyncio\nfrom collections.abc import Coroutine, Sequence\nfrom logging import getLogger\nfrom types import FunctionType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Generic,\n    Protocol,\n    TypeVar,\n    cast,\n    overload,\n)\n\nfrom typing_extensions import TypeAlias\n\nfrom reactpy.config import REACTPY_DEBUG_MODE\nfrom reactpy.core._life_cycle_hook import current_hook\nfrom reactpy.core.types import Context, Key, State, VdomDict\nfrom reactpy.utils import Ref\n\nif not TYPE_CHECKING:\n    # make flake8 think that this variable exists\n    ellipsis = type(...)\n\n\n__all__ = [\n    \"use_state\",\n    \"use_effect\",\n    \"use_reducer\",\n    \"use_callback\",\n    \"use_ref\",\n    \"use_memo\",\n]\n\nlogger = getLogger(__name__)\n\n_Type = TypeVar(\"_Type\")\n\n\n@overload\ndef use_state(initial_value: Callable[[], _Type]) -> State[_Type]: ...\n\n\n@overload\ndef use_state(initial_value: _Type) -> State[_Type]: ...\n\n\ndef use_state(initial_value: _Type | Callable[[], _Type]) -> State[_Type]:\n    \"\"\"See the full :ref:`Use State` docs for details\n\n    Parameters:\n        initial_value:\n            Defines the initial value of the state. A callable (accepting no arguments)\n            can be used as a constructor function to avoid re-creating the initial value\n            on each render.\n\n    Returns:\n        A tuple containing the current state and a function to update it.\n    \"\"\"\n    current_state = _use_const(lambda: _CurrentState(initial_value))\n    return State(current_state.value, current_state.dispatch)\n\n\nclass _CurrentState(Generic[_Type]):\n    __slots__ = \"value\", \"dispatch\"\n\n    def __init__(\n        self,\n        initial_value: _Type | Callable[[], _Type],\n    ) -> None:\n        if callable(initial_value):\n            self.value = initial_value()\n        else:\n            self.value = initial_value\n\n        hook = current_hook()\n\n        def dispatch(new: _Type | Callable[[_Type], _Type]) -> None:\n            if callable(new):\n                next_value = new(self.value)\n            else:\n                next_value = new\n            if not strictly_equal(next_value, self.value):\n                self.value = next_value\n                hook.schedule_render()\n\n        self.dispatch = dispatch\n\n\n_EffectCleanFunc: TypeAlias = \"Callable[[], None]\"\n_SyncEffectFunc: TypeAlias = \"Callable[[], _EffectCleanFunc | None]\"\n_AsyncEffectFunc: TypeAlias = (\n    \"Callable[[], Coroutine[None, None, _EffectCleanFunc | None]]\"\n)\n_EffectApplyFunc: TypeAlias = \"_SyncEffectFunc | _AsyncEffectFunc\"\n\n\n@overload\ndef use_effect(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None]: ...\n\n\n@overload\ndef use_effect(\n    function: _EffectApplyFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None: ...\n\n\ndef use_effect(\n    function: _EffectApplyFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_EffectApplyFunc], None] | None:\n    \"\"\"See the full :ref:`Use Effect` docs for details\n\n    Parameters:\n        function:\n            Applies the effect and can return a clean-up function\n        dependencies:\n            Dependencies for the effect. The effect will only trigger if the identity\n            of any value in the given sequence changes (i.e. their :func:`id` is\n            different). By default these are inferred based on local variables that are\n            referenced by the given function.\n\n    Returns:\n        If not function is provided, a decorator. Otherwise ``None``.\n    \"\"\"\n    hook = current_hook()\n\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n    last_clean_callback: Ref[_EffectCleanFunc | None] = use_ref(None)\n\n    def add_effect(function: _EffectApplyFunc) -> None:\n        if not asyncio.iscoroutinefunction(function):\n            sync_function = cast(_SyncEffectFunc, function)\n        else:\n            async_function = cast(_AsyncEffectFunc, function)\n\n            def sync_function() -> _EffectCleanFunc | None:\n                task = asyncio.create_task(async_function())\n\n                def clean_future() -> None:\n                    if not task.cancel():\n                        try:\n                            clean = task.result()\n                        except asyncio.CancelledError:\n                            pass\n                        else:\n                            if clean is not None:\n                                clean()\n\n                return clean_future\n\n        async def effect(stop: asyncio.Event) -> None:\n            if last_clean_callback.current is not None:\n                last_clean_callback.current()\n                last_clean_callback.current = None\n            clean = last_clean_callback.current = sync_function()\n            await stop.wait()\n            if clean is not None:\n                clean()\n\n        return memoize(lambda: hook.add_effect(effect))\n\n    if function is not None:\n        add_effect(function)\n        return None\n    else:\n        return add_effect\n\n\ndef use_debug_value(\n    message: Any | Callable[[], Any],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> None:\n    \"\"\"Log debug information when the given message changes.\n\n    .. note::\n        This hook only logs if :data:`~reactpy.config.REACTPY_DEBUG_MODE` is active.\n\n    Unlike other hooks, a message is considered to have changed if the old and new\n    values are ``!=``. Because this comparison is performed on every render of the\n    component, it may be worth considering the performance cost in some situations.\n\n    Parameters:\n        message:\n            The value to log or a memoized function for generating the value.\n        dependencies:\n            Dependencies for the memoized function. The message will only be recomputed\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n    \"\"\"\n    old: Ref[Any] = _use_const(lambda: Ref(object()))\n    memo_func = message if callable(message) else lambda: message\n    new = use_memo(memo_func, dependencies)\n\n    if REACTPY_DEBUG_MODE.current and old.current != new:\n        old.current = new\n        logger.debug(f\"{current_hook().component} {new}\")\n\n\ndef create_context(default_value: _Type) -> Context[_Type]:\n    \"\"\"Return a new context type for use in :func:`use_context`\"\"\"\n\n    def context(\n        *children: Any,\n        value: _Type = default_value,\n        key: Key | None = None,\n    ) -> _ContextProvider[_Type]:\n        return _ContextProvider(\n            *children,\n            value=value,\n            key=key,\n            type=context,\n        )\n\n    context.__qualname__ = \"context\"\n\n    return context\n\n\ndef use_context(context: Context[_Type]) -> _Type:\n    \"\"\"Get the current value for the given context type.\n\n    See the full :ref:`Use Context` docs for more information.\n    \"\"\"\n    hook = current_hook()\n    provider = hook.get_context_provider(context)\n\n    if provider is None:\n        # same assertions but with normal exceptions\n        if not isinstance(context, FunctionType):\n            raise TypeError(f\"{context} is not a Context\")  # nocov\n        if context.__kwdefaults__ is None:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        if \"value\" not in context.__kwdefaults__:\n            raise TypeError(f\"{context} has no 'value' kwarg\")  # nocov\n        return cast(_Type, context.__kwdefaults__[\"value\"])\n\n    return provider.value\n\n\nclass _ContextProvider(Generic[_Type]):\n    def __init__(\n        self,\n        *children: Any,\n        value: _Type,\n        key: Key | None,\n        type: Context[_Type],\n    ) -> None:\n        self.children = children\n        self.key = key\n        self.type = type\n        self.value = value\n\n    def render(self) -> VdomDict:\n        current_hook().set_context_provider(self)\n        return {\"tagName\": \"\", \"children\": self.children}\n\n    def __repr__(self) -> str:\n        return f\"ContextProvider({self.type})\"\n\n\n_ActionType = TypeVar(\"_ActionType\")\n\n\ndef use_reducer(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    initial_value: _Type,\n) -> tuple[_Type, Callable[[_ActionType], None]]:\n    \"\"\"See the full :ref:`Use Reducer` docs for details\n\n    Parameters:\n        reducer:\n            A function which applies an action to the current state in order to\n            produce the next state.\n        initial_value:\n            The initial state value (same as for :func:`use_state`)\n\n    Returns:\n        A tuple containing the current state and a function to change it with an action\n    \"\"\"\n    state, set_state = use_state(initial_value)\n    return state, _use_const(lambda: _create_dispatcher(reducer, set_state))\n\n\ndef _create_dispatcher(\n    reducer: Callable[[_Type, _ActionType], _Type],\n    set_state: Callable[[Callable[[_Type], _Type]], None],\n) -> Callable[[_ActionType], None]:\n    def dispatch(action: _ActionType) -> None:\n        set_state(lambda last_state: reducer(last_state, action))\n\n    return dispatch\n\n\n_CallbackFunc = TypeVar(\"_CallbackFunc\", bound=Callable[..., Any])\n\n\n@overload\ndef use_callback(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> Callable[[_CallbackFunc], _CallbackFunc]: ...\n\n\n@overload\ndef use_callback(\n    function: _CallbackFunc,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc: ...\n\n\ndef use_callback(\n    function: _CallbackFunc | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _CallbackFunc | Callable[[_CallbackFunc], _CallbackFunc]:\n    \"\"\"See the full :ref:`Use Callback` docs for details\n\n    Parameters:\n        function:\n            The function whose identity will be preserved\n        dependencies:\n            Dependencies of the callback. The identity the ``function`` will be updated\n            if the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current function\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n    memoize = use_memo(dependencies=dependencies)\n\n    def setup(function: _CallbackFunc) -> _CallbackFunc:\n        return memoize(lambda: function)\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _LambdaCaller(Protocol):\n    \"\"\"MyPy doesn't know how to deal with TypeVars only used in function return\"\"\"\n\n    def __call__(self, func: Callable[[], _Type]) -> _Type: ...\n\n\n@overload\ndef use_memo(\n    function: None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _LambdaCaller: ...\n\n\n@overload\ndef use_memo(\n    function: Callable[[], _Type],\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type: ...\n\n\ndef use_memo(\n    function: Callable[[], _Type] | None = None,\n    dependencies: Sequence[Any] | ellipsis | None = ...,\n) -> _Type | Callable[[Callable[[], _Type]], _Type]:\n    \"\"\"See the full :ref:`Use Memo` docs for details\n\n    Parameters:\n        function:\n            The function to be memoized.\n        dependencies:\n            Dependencies for the memoized function. The memo will only be recomputed if\n            the identity of any value in the given sequence changes (i.e. their\n            :func:`id` is different). By default these are inferred based on local\n            variables that are referenced by the given function.\n\n    Returns:\n        The current state\n    \"\"\"\n    dependencies = _try_to_infer_closure_values(function, dependencies)\n\n    memo: _Memo[_Type] = _use_const(_Memo)\n\n    if memo.empty():\n        # we need to initialize on the first run\n        changed = True\n        memo.deps = () if dependencies is None else dependencies\n    elif dependencies is None:\n        changed = True\n        memo.deps = ()\n    elif (\n        len(memo.deps) != len(dependencies)\n        # if deps are same length check identity for each item\n        or not all(\n            strictly_equal(current, new)\n            for current, new in zip(memo.deps, dependencies)\n        )\n    ):\n        memo.deps = dependencies\n        changed = True\n    else:\n        changed = False\n\n    setup: Callable[[Callable[[], _Type]], _Type]\n\n    if changed:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            current_value = memo.value = function()\n            return current_value\n\n    else:\n\n        def setup(function: Callable[[], _Type]) -> _Type:\n            return memo.value\n\n    if function is not None:\n        return setup(function)\n    else:\n        return setup\n\n\nclass _Memo(Generic[_Type]):\n    \"\"\"Simple object for storing memoization data\"\"\"\n\n    __slots__ = \"value\", \"deps\"\n\n    value: _Type\n    deps: Sequence[Any]\n\n    def empty(self) -> bool:\n        try:\n            self.value  # noqa: B018\n        except AttributeError:\n            return True\n        else:\n            return False\n\n\ndef use_ref(initial_value: _Type) -> Ref[_Type]:\n    \"\"\"See the full :ref:`Use State` docs for details\n\n    Parameters:\n        initial_value: The value initially assigned to the reference.\n\n    Returns:\n        A :class:`Ref` object.\n    \"\"\"\n    return _use_const(lambda: Ref(initial_value))\n\n\ndef _use_const(function: Callable[[], _Type]) -> _Type:\n    return current_hook().use_state(function)\n\n\ndef _try_to_infer_closure_values(\n    func: Callable[..., Any] | None,\n    values: Sequence[Any] | ellipsis | None,\n) -> Sequence[Any] | None:\n    if values is ...:\n        if isinstance(func, FunctionType):\n            return (\n                [cell.cell_contents for cell in func.__closure__]\n                if func.__closure__\n                else []\n            )\n        else:\n            return None\n    else:\n        return values\n\n\ndef strictly_equal(x: Any, y: Any) -> bool:\n    \"\"\"Check if two values are identical or, for a limited set or types, equal.\n\n    Only the following types are checked for equality rather than identity:\n\n    - ``int``\n    - ``float``\n    - ``complex``\n    - ``str``\n    - ``bytes``\n    - ``bytearray``\n    - ``memoryview``\n    \"\"\"\n    return x is y or (type(x) in _NUMERIC_TEXT_BINARY_TYPES and x == y)\n\n\n_NUMERIC_TEXT_BINARY_TYPES = {\n    # numeric\n    int,\n    float,\n    complex,\n    # text\n    str,\n    # binary types\n    bytes,\n    bytearray,\n    memoryview,\n}\n\n# Path: src/py/reactpy/reactpy/backend/hooks.py\nfrom __future__ import annotations\n\nfrom collections.abc import MutableMapping\nfrom typing import Any\n\nfrom reactpy.backend.types import Connection, Location\nfrom reactpy.core.hooks import create_context, use_context\nfrom reactpy.core.types import Context\n\n# backend implementations should establish this context at the root of an app\nConnectionContext: Context[Connection[Any] | None] = create_context(None)\n\n\ndef use_connection() -> Connection[Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`.\"\"\"\n    conn = use_context(ConnectionContext)\n    if conn is None:  # nocov\n        msg = \"No backend established a connection.\"\n        raise RuntimeError(msg)\n    return conn\n\n\ndef use_scope() -> MutableMapping[str, Any]:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s scope.\"\"\"\n    return use_connection().scope\n\n\ndef use_location() -> Location:\n    \"\"\"Get the current :class:`~reactpy.backend.types.Connection`'s location.\"\"\"\n    return use_connection().location\n\n# Path: src/py/reactpy/reactpy/core/component.py\nfrom __future__ import annotations\n\nimport inspect\nfrom functools import wraps\nfrom typing import Any, Callable\n\nfrom reactpy.core.types import ComponentType, VdomDict\n\n\ndef component(\n    function: Callable[..., ComponentType | VdomDict | str | None]\n) -> Callable[..., Component]:\n    \"\"\"A decorator for defining a new component.\n\n    Parameters:\n        function: The component's :meth:`reactpy.core.proto.ComponentType.render` function.\n    \"\"\"\n    sig = inspect.signature(function)\n\n    if \"key\" in sig.parameters and sig.parameters[\"key\"].kind in (\n        inspect.Parameter.KEYWORD_ONLY,\n        inspect.Parameter.POSITIONAL_OR_KEYWORD,\n    ):\n        msg = f\"Component render function {function} uses reserved parameter 'key'\"\n        raise TypeError(msg)\n\n    @wraps(function)\n    def constructor(*args: Any, key: Any | None = None, **kwargs: Any) -> Component:\n        return Component(function, key, args, kwargs, sig)\n\n    return constructor\n\n\nclass Component:\n    \"\"\"An object for rending component models.\"\"\"\n\n    __slots__ = \"__weakref__\", \"_func\", \"_args\", \"_kwargs\", \"_sig\", \"key\", \"type\"\n\n    def __init__(\n        self,\n        function: Callable[..., ComponentType | VdomDict | str | None],\n        key: Any | None,\n        args: tuple[Any, ...],\n        kwargs: dict[str, Any],\n        sig: inspect.Signature,\n    ) -> None:\n        self.key = key\n        self.type = function\n        self._args = args\n        self._kwargs = kwargs\n        self._sig = sig\n\n    def render(self) -> ComponentType | VdomDict | str | None:\n        return self.type(*self._args, **self._kwargs)\n\n    def __repr__(self) -> str:\n        try:\n            args = self._sig.bind(*self._args, **self._kwargs).arguments\n        except TypeError:\n            return f\"{self.type.__name__}(...)\"\n        else:\n            items = \", \".join(f\"{k}={v!r}\" for k, v in args.items())\n            if items:\n                return f\"{self.type.__name__}({id(self):02x}, {items})\"\n            else:\n                return f\"{self.type.__name__}({id(self):02x})\"\n\n# Path: src/py/reactpy/reactpy/types.py\n\"\"\"Exports common types from:\n\n- :mod:`reactpy.core.types`\n- :mod:`reactpy.backend.types`\n\"\"\"\n\nfrom reactpy.backend.types import BackendType, Connection, Location\nfrom reactpy.core.component import Component\nfrom reactpy.core.types import (\n    ComponentConstructor,\n    ComponentType,\n    Context,\n    EventHandlerDict,\n    EventHandlerFunc,\n    EventHandlerMapping,\n    EventHandlerType,\n    ImportSourceDict,\n    Key,\n    LayoutType,\n    RootComponentConstructor,\n    State,\n    VdomAttributes,\n    VdomChild,\n    VdomChildren,\n    VdomDict,\n    VdomJson,\n)\n\n__all__ = [\n    \"BackendType\",\n    \"Component\",\n    \"ComponentConstructor\",\n    \"ComponentType\",\n    \"Connection\",\n    \"Context\",\n    \"EventHandlerDict\",\n    \"EventHandlerFunc\",\n    \"EventHandlerMapping\",\n    \"EventHandlerType\",\n    \"ImportSourceDict\",\n    \"Key\",\n    \"LayoutType\",\n    \"Location\",\n    \"RootComponentConstructor\",\n    \"State\",\n    \"VdomAttributes\",\n    \"VdomChild\",\n    \"VdomChildren\",\n    \"VdomDict\",\n    \"VdomJson\",\n]\n\n# Path: src/py/reactpy/reactpy/backend/utils.py\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nimport socket\nimport sys\nfrom collections.abc import Iterator\nfrom contextlib import closing\nfrom importlib import import_module\nfrom typing import Any\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.types import RootComponentConstructor\n\nlogger = logging.getLogger(__name__)\n\nSUPPORTED_BACKENDS = (\n    \"fastapi\",\n    \"sanic\",\n    \"tornado\",\n    \"flask\",\n    \"starlette\",\n)\n\n\ndef run(\n    component: RootComponentConstructor,\n    host: str = \"127.0.0.1\",\n    port: int | None = None,\n    implementation: BackendType[Any] | None = None,\n) -> None:\n    \"\"\"Run a component with a development server\"\"\"\n    logger.warning(_DEVELOPMENT_RUN_FUNC_WARNING)\n\n    implementation = implementation or import_module(\"reactpy.backend.default\")\n    app = implementation.create_development_app()\n    implementation.configure(app, component)\n    port = port or find_available_port(host)\n    app_cls = type(app)\n\n    logger.info(\n        \"ReactPy is running with '%s.%s' at http://%s:%s\",\n        app_cls.__module__,\n        app_cls.__name__,\n        host,\n        port,\n    )\n    asyncio.run(implementation.serve_development_app(app, host, port))\n\n\ndef find_available_port(host: str, port_min: int = 8000, port_max: int = 9000) -> int:\n    \"\"\"Get a port that's available for the given host and port range\"\"\"\n    for port in range(port_min, port_max):\n        with closing(socket.socket()) as sock:\n            try:\n                if sys.platform in (\"linux\", \"darwin\"):\n                    # Fixes bug on Unix-like systems where every time you restart the\n                    # server you'll get a different port on Linux. This cannot be set\n                    # on Windows otherwise address will always be reused.\n                    # Ref: https://stackoverflow.com/a/19247688/3159288\n                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n                sock.bind((host, port))\n            except OSError:\n                pass\n            else:\n                return port\n    msg = f\"Host {host!r} has no available port in range {port_max}-{port_max}\"\n    raise RuntimeError(msg)\n\n\n\ndef all_implementations() -> Iterator[BackendType[Any]]:\n    \"\"\"Yield all available server implementations\"\"\"\n    for name in SUPPORTED_BACKENDS:\n        try:\n            import_module(name)\n        except ImportError:  # nocov\n            logger.debug(\"Failed to import %s\", name, exc_info=True)\n            continue\n\n        reactpy_backend_name = f\"{__name__.rsplit('.', 1)[0]}.{name}\"\n        yield import_module(reactpy_backend_name)\n\n\n_DEVELOPMENT_RUN_FUNC_WARNING = \"\"\"\\\nThe `run()` function is only intended for testing during development! To run \\\nin production, refer to the docs on how to use reactpy.backend.*.configure.\\\n\"\"\"\n\n# Path: src/py/reactpy/reactpy/core/layout.py\nfrom __future__ import annotations\n\nimport abc\nfrom asyncio import (\n    FIRST_COMPLETED,\n    CancelledError,\n    Queue,\n    Task,\n    create_task,\n    get_running_loop,\n    wait,\n)\nfrom collections import Counter\nfrom collections.abc import Sequence\nfrom contextlib import AsyncExitStack\nfrom logging import getLogger\nfrom typing import (\n    Any,\n    Callable,\n    Generic,\n    NamedTuple,\n    NewType,\n    TypeVar,\n    cast,\n)\nfrom uuid import uuid4\nfrom weakref import ref as weakref\n\nfrom anyio import Semaphore\nfrom typing_extensions import TypeAlias\n\nfrom reactpy.config import (\n    REACTPY_ASYNC_RENDERING,\n    REACTPY_CHECK_VDOM_SPEC,\n    REACTPY_DEBUG_MODE,\n)\nfrom reactpy.core._life_cycle_hook import LifeCycleHook\nfrom reactpy.core.types import (\n    ComponentType,\n    EventHandlerDict,\n    Key,\n    LayoutEventMessage,\n    LayoutUpdateMessage,\n    VdomChild,\n    VdomDict,\n    VdomJson,\n)\nfrom reactpy.core.vdom import validate_vdom_json\nfrom reactpy.utils import Ref\n\nlogger = getLogger(__name__)\n\n\nclass Layout:\n    \"\"\"Responsible for \"rendering\" components. That is, turning them into VDOM.\"\"\"\n\n    __slots__: tuple[str, ...] = (\n        \"root\",\n        \"_event_handlers\",\n        \"_rendering_queue\",\n        \"_render_tasks\",\n        \"_render_tasks_ready\",\n        \"_root_life_cycle_state_id\",\n        \"_model_states_by_life_cycle_state_id\",\n    )\n\n    if not hasattr(abc.ABC, \"__weakref__\"):  # nocov\n        __slots__ += (\"__weakref__\",)\n\n    def __init__(self, root: ComponentType) -> None:\n        super().__init__()\n        if not isinstance(root, ComponentType):\n            msg = f\"Expected a ComponentType, not {type(root)!r}.\"\n            raise TypeError(msg)\n        self.root = root\n\n    async def __aenter__(self) -> Layout:\n        # create attributes here to avoid access before entering context manager\n        self._event_handlers: EventHandlerDict = {}\n        self._render_tasks: set[Task[LayoutUpdateMessage]] = set()\n        self._render_tasks_ready: Semaphore = Semaphore(0)\n\n...\n", "needle_token_start": 15147, "needle_token_end": 15280, "code_context_ntokens": 16005, "output": ["```python\nfrom __future__ import annotations\n\nimport asyncio\nimport logging\nimport socket\nimport sys\nfrom collections.abc import Iterator\nfrom contextlib import closing\nfrom importlib import import_module\nfrom typing import Any\n\nfrom reactpy.backend.types import BackendType\nfrom reactpy.types import RootComponentConstructor\n\nlogger = logging.getLogger(__name__)\n\nSUPPORTED_BACKENDS = (\n    \"fastapi\",\n    \"sanic\",\n    \"tornado\",\n    \"flask\",\n    \"starlette\",\n)\n\n\ndef run(\n    component: RootComponentConstructor,\n    host: str = \"127.0.0.1\",\n    port: int | None = None,\n    implementation: BackendType[Any] | None = None,\n) -> None:\n    \"\"\"Run a component with a development server\"\"\"\n    logger.warning(_DEVELOPMENT_RUN_FUNC_WARNING)\n\n    implementation = implementation or import_module(\"reactpy.backend.default\")\n    app = implementation.create_development_app()\n    implementation.configure(app, component)\n    port = port or find_available_port(host)\n    app_cls = type(app)\n\n    logger.info(\n        \"ReactPy is running with '%s.%s' at http://%s:%s\",\n        app_cls.__module__,\n        app_cls.__name__,\n        host,\n        port,\n    )\n    asyncio.run(implementation.serve_development_app(app, host, port))\n\n\ndef find_available_port(host: str, port_min: int = 8000, port_max: int = 9000) -> int:\n    \"\"\"Get a port that's available for the given host and port range\"\"\"\n    for port in range(port_min, port_max):\n        with closing(socket.socket()) as sock:\n            try:\n                if sys.platform in (\"linux\", \"darwin\"):\n                    # Fixes bug on Unix-like systems where every time you restart the\n                    # server you'll get a different port on Linux. This cannot be set\n                    # on Windows otherwise address will always be reused.\n                    # Ref: https://stackoverflow.com/a/19247688/3159288\n                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n                sock.bind((host, port))\n            except OSError:\n                pass\n            else:\n                return port\n    msg = f\"Host {host!r} has no available port in range {port_max}-{port_max}\"\n    raise RuntimeError(msg)\n\n\n\ndef all_implementations() -> Iterator[BackendType[Any]]:\n    \"\"\"Yield all available server implementations\"\"\"\n    for name in SUPPORTED_BACKENDS:\n        try:\n            import_module(name)\n        except ImportError:  # nocov\n            logger.debug(\"Failed to import %s\", name, exc_info=True)\n            continue\n\n        reactpy_backend_name = f\"{__name__.rsplit('.', 1)[0]}.{name}\"\n        yield import_module(reactpy_backend_name)\n\n\n_DEVELOPMENT_RUN_FUNC_WARNING = \"\"\"\\\nThe `run()` function is only intended for testing during development! To run \\\nin production, refer to the docs on how to use reactpy.backend.*.configure.\\\n\"\"\"\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "register", "language": "python", "path": "src/marshmallow/class_registry.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to add a serializer class to a centralized registry, allowing for string-based lookup of these classes, which is particularly useful in scenarios involving nested serialization.\n2. **Input**: The function takes two parameters: a string representing the class name and the class itself.\n3. **Output**: There is no return value; the function operates by modifying a global registry.\n4. **Procedure**: The function first determines the module where the class is defined and constructs a full path identifier for the class. It then checks if the class is already registered under the simple class name without causing duplication in the same module. It adds the class to the registry under both the simple class name and the full path. If the full path is already registered, it updates the existing entry to ensure the class is correctly referenced.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/marshmallow/base.py\n\"\"\"Abstract base classes.\n\nThese are necessary to avoid circular imports between schema.py and fields.py.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nfrom abc import ABC, abstractmethod\n\n\nclass FieldABC(ABC):\n    \"\"\"Abstract base class from which all Field classes inherit.\"\"\"\n\n    parent = None\n    name = None\n    root = None\n\n    @abstractmethod\n    def serialize(self, attr, obj, accessor=None):\n        pass\n\n    @abstractmethod\n    def deserialize(self, value):\n        pass\n\n    @abstractmethod\n    def _serialize(self, value, attr, obj, **kwargs):\n        pass\n\n    @abstractmethod\n    def _deserialize(self, value, attr, data, **kwargs):\n        pass\n\n\nclass SchemaABC(ABC):\n    \"\"\"Abstract base class from which all Schemas inherit.\"\"\"\n\n    @abstractmethod\n    def dump(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def dumps(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def load(self, data, *, many: bool | None = None, partial=None, unknown=None):\n        pass\n\n    @abstractmethod\n    def loads(\n        self,\n        json_data,\n        *,\n        many: bool | None = None,\n...\n# Path: src/marshmallow/class_registry.py\n\"\"\"A registry of :class:`Schema <marshmallow.Schema>` classes. This allows for string\nlookup of schemas, which may be used with\nclass:`fields.Nested <marshmallow.fields.Nested>`.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\nfrom marshmallow.exceptions import RegistryError\n\nif typing.TYPE_CHECKING:\n    from marshmallow import Schema\n\n    SchemaType = typing.Type[Schema]\n\n# {\n#   <class_name>: <list of class objects>\n#   <module_path_to_class>: <list of class objects>\n# }\n_registry = {}  # type: dict[str, list[SchemaType]]\n\n\n\ndef register(classname: str, cls: SchemaType) -> None:\n    \"\"\"Add a class to the registry of serializer classes. When a class is\n    registered, an entry for both its classname and its full, module-qualified\n    path are added to the registry.\n\n    Example: ::\n\n        class MyClass:\n            pass\n\n        register('MyClass', MyClass)\n        # Registry:\n        # {\n        #   'MyClass': [path.to.MyClass],\n        #   'path.to.MyClass': [path.to.MyClass],\n        # }\n\n    \"\"\"\n    # Module where the class is located\n    module = cls.__module__\n    # Full module path to the class\n    # e.g. user.schemas.UserSchema\n    fullpath = \".\".join([module, classname])\n    # If the class is already registered; need to check if the entries are\n    # in the same module as cls to avoid having multiple instances of the same\n    # class in the registry\n    if classname in _registry and not any(\n        each.__module__ == module for each in _registry[classname]\n    ):\n        _registry[classname].append(cls)\n    elif classname not in _registry:\n        _registry[classname] = [cls]\n\n    # Also register the full path\n    if fullpath not in _registry:\n        _registry.setdefault(fullpath, []).append(cls)\n    else:\n        # If fullpath does exist, replace existing entry\n        _registry[fullpath] = [cls]\n    return None\n\n\ndef get_class(classname: str, all: bool = False) -> list[SchemaType] | SchemaType:\n    \"\"\"Retrieve a class from the registry.\n\n    :raises: marshmallow.exceptions.RegistryError if the class cannot be found\n        or if there are multiple entries for the given class name.\n    \"\"\"\n    try:\n        classes = _registry[classname]\n    except KeyError as error:\n        raise RegistryError(\n            f\"Class with name {classname!r} was not found. You may need \"\n            \"to import the class.\"\n        ) from error\n    if len(classes) > 1:\n        if all:\n            return _registry[classname]\n        raise RegistryError(\n            f\"Multiple classes with name {classname!r} \"\n            \"were found. Please use the full, \"\n            \"module-qualified path.\"\n        )\n    else:\n        return _registry[classname][0]\n\n# Path: src/marshmallow/error_store.py\n\"\"\"Utilities for storing collections of error messages.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\n\nfrom marshmallow.exceptions import SCHEMA\n\n\nclass ErrorStore:\n    def __init__(self):\n        #: Dictionary of errors stored during serialization\n        self.errors = {}\n\n    def store_error(self, messages, field_name=SCHEMA, index=None):\n        # field error  -> store/merge error messages under field name key\n        # schema error -> if string or list, store/merge under _schema key\n        #              -> if dict, store/merge with other top-level keys\n        if field_name != SCHEMA or not isinstance(messages, dict):\n            messages = {field_name: messages}\n        if index is not None:\n            messages = {index: messages}\n        self.errors = merge_errors(self.errors, messages)\n\n\ndef merge_errors(errors1, errors2):\n    \"\"\"Deeply merge two error messages.\n\n    The format of ``errors1`` and ``errors2`` matches the ``message``\n    parameter of :exc:`marshmallow.exceptions.ValidationError`.\n    \"\"\"\n    if not errors1:\n        return errors2\n    if not errors2:\n        return errors1\n    if isinstance(errors1, list):\n        if isinstance(errors2, list):\n            return errors1 + errors2\n        if isinstance(errors2, dict):\n            return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n        return errors1 + [errors2]\n    if isinstance(errors1, dict):\n        if isinstance(errors2, list):\n            return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n        if isinstance(errors2, dict):\n            errors = dict(errors1)\n            for key, val in errors2.items():\n                if key in errors:\n                    errors[key] = merge_errors(errors[key], val)\n                else:\n                    errors[key] = val\n            return errors\n        return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n    if isinstance(errors2, list):\n        return [errors1] + errors2\n    if isinstance(errors2, dict):\n        return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n    return [errors1, errors2]\n\n# Path: src/marshmallow/orderedset.py\n# OrderedSet\n# Copyright (c) 2009 Raymond Hettinger\n#\n# Permission is hereby granted, free of charge, to any person\n# obtaining a copy of this software and associated documentation files\n# (the \"Software\"), to deal in the Software without restriction,\n# including without limitation the rights to use, copy, modify, merge,\n# publish, distribute, sublicense, and/or sell copies of the Software,\n# and to permit persons to whom the Software is furnished to do so,\n# subject to the following conditions:\n#\n#     The above copyright notice and this permission notice shall be\n#     included in all copies or substantial portions of the Software.\n#\n#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES\n#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\n#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\n#     OTHER DEALINGS IN THE SOFTWARE.\nfrom collections.abc import MutableSet\n\n\nclass OrderedSet(MutableSet):\n    def __init__(self, iterable=None):\n        self.end = end = []\n        end += [None, end, end]  # sentinel node for doubly linked list\n        self.map = {}  # key --> [key, prev, next]\n        if iterable is not None:\n            self |= iterable\n\n    def __len__(self):\n        return len(self.map)\n\n    def __contains__(self, key):\n        return key in self.map\n\n    def add(self, key):\n        if key not in self.map:\n            end = self.end\n            curr = end[1]\n            curr[2] = end[1] = self.map[key] = [key, curr, end]\n\n    def discard(self, key):\n        if key in self.map:\n            key, prev, next = self.map.pop(key)\n            prev[2] = next\n            next[1] = prev\n\n    def __iter__(self):\n        end = self.end\n        curr = end[2]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[2]\n\n    def __reversed__(self):\n        end = self.end\n        curr = end[1]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[1]\n\n    def pop(self, last=True):\n        if not self:\n            raise KeyError(\"set is empty\")\n        key = self.end[1][0] if last else self.end[2][0]\n        self.discard(key)\n        return key\n\n    def __repr__(self):\n        if not self:\n            return f\"{self.__class__.__name__}()\"\n        return f\"{self.__class__.__name__}({list(self)!r})\"\n\n    def __eq__(self, other):\n        if isinstance(other, OrderedSet):\n            return len(self) == len(other) and list(self) == list(other)\n        return set(self) == set(other)\n\n\nif __name__ == \"__main__\":\n    s = OrderedSet(\"abracadaba\")\n    t = OrderedSet(\"simsalabim\")\n    print(s | t)\n    print(s & t)\n    print(s - t)\n\n# Path: src/marshmallow/types.py\n\"\"\"Type aliases.\n\n.. warning::\n\n    This module is provisional. Types may be modified, added, and removed between minor releases.\n\"\"\"\nimport typing\n\nStrSequenceOrSet = typing.Union[typing.Sequence[str], typing.AbstractSet[str]]\nTag = typing.Union[str, typing.Tuple[str, bool]]\nValidator = typing.Callable[[typing.Any], typing.Any]\n\n# Path: src/marshmallow/warnings.py\nclass RemovedInMarshmallow4Warning(DeprecationWarning):\n    pass\n\n# Path: src/marshmallow/utils.py\n\"\"\"Utility methods for marshmallow.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport datetime as dt\nimport functools\nimport inspect\nimport json\nimport re\nimport typing\nimport warnings\nfrom collections.abc import Mapping\nfrom email.utils import format_datetime, parsedate_to_datetime\nfrom pprint import pprint as py_pprint\n\nfrom marshmallow.base import FieldABC\nfrom marshmallow.exceptions import FieldInstanceResolutionError\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\nEXCLUDE = \"exclude\"\nINCLUDE = \"include\"\nRAISE = \"raise\"\n_UNKNOWN_VALUES = {EXCLUDE, INCLUDE, RAISE}\n\n\nclass _Missing:\n    def __bool__(self):\n        return False\n\n    def __copy__(self):\n        return self\n\n    def __deepcopy__(self, _):\n        return self\n\n    def __repr__(self):\n        return \"<marshmallow.missing>\"\n\n\n# Singleton value that indicates that a field's value is missing from input\n# dict passed to :meth:`Schema.load`. If the field's value is not required,\n# it's ``default`` value is used.\nmissing = _Missing()\n\n\ndef is_generator(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a generator\"\"\"\n    return inspect.isgeneratorfunction(obj) or inspect.isgenerator(obj)\n\n\ndef is_iterable_but_not_string(obj) -> bool:\n    \"\"\"Return True if ``obj`` is an iterable object that isn't a string.\"\"\"\n    return (hasattr(obj, \"__iter__\") and not hasattr(obj, \"strip\")) or is_generator(obj)\n\n\ndef is_collection(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a collection type, e.g list, tuple, queryset.\"\"\"\n    return is_iterable_but_not_string(obj) and not isinstance(obj, Mapping)\n\n\ndef is_instance_or_subclass(val, class_) -> bool:\n    \"\"\"Return True if ``val`` is either a subclass or instance of ``class_``.\"\"\"\n    try:\n        return issubclass(val, class_)\n    except TypeError:\n        return isinstance(val, class_)\n\n\ndef is_keyed_tuple(obj) -> bool:\n    \"\"\"Return True if ``obj`` has keyed tuple behavior, such as\n    namedtuples or SQLAlchemy's KeyedTuples.\n    \"\"\"\n    return isinstance(obj, tuple) and hasattr(obj, \"_fields\")\n\n\ndef pprint(obj, *args, **kwargs) -> None:\n    \"\"\"Pretty-printing function that can pretty-print OrderedDicts\n    like regular dictionaries. Useful for printing the output of\n    :meth:`marshmallow.Schema.dump`.\n\n    .. deprecated:: 3.7.0\n        marshmallow.pprint will be removed in marshmallow 4.\n    \"\"\"\n    warnings.warn(\n        \"marshmallow's pprint function is deprecated and will be removed in marshmallow 4.\",\n        RemovedInMarshmallow4Warning,\n        stacklevel=2,\n    )\n    if isinstance(obj, collections.OrderedDict):\n        print(json.dumps(obj, *args, **kwargs))\n    else:\n        py_pprint(obj, *args, **kwargs)\n\n\n# https://stackoverflow.com/a/27596917\ndef is_aware(datetime: dt.datetime) -> bool:\n    return (\n        datetime.tzinfo is not None and datetime.tzinfo.utcoffset(datetime) is not None\n    )\n\n\ndef from_rfc(datestring: str) -> dt.datetime:\n    \"\"\"Parse a RFC822-formatted datetime string and return a datetime object.\n\n    https://stackoverflow.com/questions/885015/how-to-parse-a-rfc-2822-date-time-into-a-python-datetime  # noqa: B950\n    \"\"\"\n    return parsedate_to_datetime(datestring)\n\n\ndef rfcformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the RFC822-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return format_datetime(datetime)\n\n\n# Hat tip to Django for ISO8601 deserialization functions\n\n_iso8601_datetime_re = re.compile(\n    r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})\"\n    r\"[T ](?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n    r\"(?P<tzinfo>Z|[+-]\\d{2}(?::?\\d{2})?)?$\"\n)\n\n_iso8601_date_re = re.compile(r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})$\")\n\n_iso8601_time_re = re.compile(\n    r\"(?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n)\n\n\ndef get_fixed_timezone(offset: int | float | dt.timedelta) -> dt.timezone:\n    \"\"\"Return a tzinfo instance with a fixed offset from UTC.\"\"\"\n    if isinstance(offset, dt.timedelta):\n        offset = offset.total_seconds() // 60\n    sign = \"-\" if offset < 0 else \"+\"\n    hhmm = \"%02d%02d\" % divmod(abs(offset), 60)\n    name = sign + hhmm\n    return dt.timezone(dt.timedelta(minutes=offset), name)\n\n\ndef from_iso_datetime(value):\n    \"\"\"Parse a string and return a datetime.datetime.\n\n    This function supports time zone offsets. When the input contains one,\n    the output uses a timezone with a fixed offset from UTC.\n    \"\"\"\n    match = _iso8601_datetime_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted datetime string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    tzinfo = kw.pop(\"tzinfo\")\n    if tzinfo == \"Z\":\n        tzinfo = dt.timezone.utc\n    elif tzinfo is not None:\n        offset_mins = int(tzinfo[-2:]) if len(tzinfo) > 3 else 0\n        offset = 60 * int(tzinfo[1:3]) + offset_mins\n        if tzinfo[0] == \"-\":\n            offset = -offset\n        tzinfo = get_fixed_timezone(offset)\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    kw[\"tzinfo\"] = tzinfo\n    return dt.datetime(**kw)\n\n\ndef from_iso_time(value):\n    \"\"\"Parse a string and return a datetime.time.\n\n    This function doesn't support time zone offsets.\n    \"\"\"\n    match = _iso8601_time_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted time string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    return dt.time(**kw)\n\n\ndef from_iso_date(value):\n    \"\"\"Parse a string and return a datetime.date.\"\"\"\n    match = _iso8601_date_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted date string\")\n    kw = {k: int(v) for k, v in match.groupdict().items()}\n    return dt.date(**kw)\n\n\ndef from_timestamp(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    if value < 0:\n        raise ValueError(\"Not a valid POSIX timestamp\")\n\n    # Load a timestamp with utc as timezone to prevent using system timezone.\n    # Then set timezone to None, to let the Field handle adding timezone info.\n    try:\n        return dt.datetime.fromtimestamp(value, tz=dt.timezone.utc).replace(tzinfo=None)\n    except OverflowError as exc:\n        raise ValueError(\"Timestamp is too large\") from exc\n    except OSError as exc:\n        raise ValueError(\"Error converting value to datetime\") from exc\n\n\ndef from_timestamp_ms(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    return from_timestamp(value / 1000)\n\n\ndef timestamp(\n    value: dt.datetime,\n) -> float:\n    if not is_aware(value):\n        # When a date is naive, use UTC as zone info to prevent using system timezone.\n        value = value.replace(tzinfo=dt.timezone.utc)\n    return value.timestamp()\n\n\ndef timestamp_ms(value: dt.datetime) -> float:\n    return timestamp(value) * 1000\n\n\ndef isoformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the ISO8601-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return datetime.isoformat()\n\n\ndef to_iso_time(time: dt.time) -> str:\n    return dt.time.isoformat(time)\n\n\ndef to_iso_date(date: dt.date) -> str:\n    return dt.date.isoformat(date)\n\n\ndef ensure_text_type(val: str | bytes) -> str:\n    if isinstance(val, bytes):\n        val = val.decode(\"utf-8\")\n    return str(val)\n\n\ndef pluck(dictlist: list[dict[str, typing.Any]], key: str):\n    \"\"\"Extracts a list of dictionary values from a list of dictionaries.\n    ::\n\n        >>> dlist = [{'id': 1, 'name': 'foo'}, {'id': 2, 'name': 'bar'}]\n        >>> pluck(dlist, 'id')\n        [1, 2]\n    \"\"\"\n    return [d[key] for d in dictlist]\n\n\n# Various utilities for pulling keyed values from objects\n\n\ndef get_value(obj, key: int | str, default=missing):\n    \"\"\"Helper for pulling a keyed value off various types of objects. Fields use\n    this method by default to access attributes of the source object. For object `x`\n    and attribute `i`, this method first tries to access `x[i]`, and then falls back to\n    `x.i` if an exception is raised.\n\n    .. warning::\n        If an object `x` does not raise an exception when `x[i]` does not exist,\n        `get_value` will never check the value `x.i`. Consider overriding\n        `marshmallow.fields.Field.get_value` in this case.\n    \"\"\"\n    if not isinstance(key, int) and \".\" in key:\n        return _get_value_for_keys(obj, key.split(\".\"), default)\n    else:\n        return _get_value_for_key(obj, key, default)\n\n\ndef _get_value_for_keys(obj, keys, default):\n    if len(keys) == 1:\n        return _get_value_for_key(obj, keys[0], default)\n    else:\n        return _get_value_for_keys(\n            _get_value_for_key(obj, keys[0], default), keys[1:], default\n        )\n\n\ndef _get_value_for_key(obj, key, default):\n    if not hasattr(obj, \"__getitem__\"):\n        return getattr(obj, key, default)\n\n    try:\n        return obj[key]\n    except (KeyError, IndexError, TypeError, AttributeError):\n        return getattr(obj, key, default)\n\n\ndef set_value(dct: dict[str, typing.Any], key: str, value: typing.Any):\n    \"\"\"Set a value in a dict. If `key` contains a '.', it is assumed\n    be a path (i.e. dot-delimited string) to the value's location.\n\n    ::\n\n        >>> d = {}\n        >>> set_value(d, 'foo.bar', 42)\n        >>> d\n        {'foo': {'bar': 42}}\n    \"\"\"\n    if \".\" in key:\n        head, rest = key.split(\".\", 1)\n        target = dct.setdefault(head, {})\n        if not isinstance(target, dict):\n            raise ValueError(\n                f\"Cannot set {key} in {head} \" f\"due to existing value: {target}\"\n            )\n        set_value(target, rest, value)\n    else:\n        dct[key] = value\n\n\ndef callable_or_raise(obj):\n    \"\"\"Check that an object is callable, else raise a :exc:`TypeError`.\"\"\"\n    if not callable(obj):\n        raise TypeError(f\"Object {obj!r} is not callable.\")\n    return obj\n\n\ndef _signature(func: typing.Callable) -> list[str]:\n    return list(inspect.signature(func).parameters.keys())\n\n\ndef get_func_args(func: typing.Callable) -> list[str]:\n    \"\"\"Given a callable, return a list of argument names. Handles\n    `functools.partial` objects and class-based callables.\n\n    .. versionchanged:: 3.0.0a1\n        Do not return bound arguments, eg. ``self``.\n    \"\"\"\n    if inspect.isfunction(func) or inspect.ismethod(func):\n        return _signature(func)\n    if isinstance(func, functools.partial):\n        return _signature(func.func)\n    # Callable class\n    return _signature(func)\n\n\ndef resolve_field_instance(cls_or_instance):\n    \"\"\"Return a Schema instance from a Schema class or instance.\n\n    :param type|Schema cls_or_instance: Marshmallow Schema class or instance.\n    \"\"\"\n    if isinstance(cls_or_instance, type):\n        if not issubclass(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance()\n    else:\n        if not isinstance(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance\n\n\ndef timedelta_to_microseconds(value: dt.timedelta) -> int:\n    \"\"\"Compute the total microseconds of a timedelta\n\n    https://github.com/python/cpython/blob/bb3e0c240bc60fe08d332ff5955d54197f79751c/Lib/datetime.py#L665-L667  # noqa: B950\n    \"\"\"\n    return (value.days * (24 * 3600) + value.seconds) * 1000000 + value.microseconds\n\n\ndef validate_unknown_parameter_value(obj: typing.Any) -> str:\n    if obj not in _UNKNOWN_VALUES:\n        raise ValueError(\n            f\"Object {obj!r} is not a valid value for the 'unknown' parameter\"\n        )\n    return obj\n\n# Path: src/marshmallow/schema.py\n\"\"\"The :class:`Schema` class, including its metaclass and options (class Meta).\"\"\"\nfrom __future__ import annotations\n\nimport copy\nimport datetime as dt\nimport decimal\nimport inspect\nimport json\nimport typing\nimport uuid\nimport warnings\nfrom abc import ABCMeta\nfrom collections import OrderedDict, defaultdict\nfrom collections.abc import Mapping\nfrom functools import lru_cache\n\nfrom marshmallow import base, class_registry, types\nfrom marshmallow import fields as ma_fields\nfrom marshmallow.decorators import (\n    POST_DUMP,\n    POST_LOAD,\n    PRE_DUMP,\n    PRE_LOAD,\n    VALIDATES,\n    VALIDATES_SCHEMA,\n)\nfrom marshmallow.error_store import ErrorStore\nfrom marshmallow.exceptions import StringNotCollectionError, ValidationError\nfrom marshmallow.orderedset import OrderedSet\nfrom marshmallow.utils import (\n    EXCLUDE,\n    INCLUDE,\n    RAISE,\n    get_value,\n    is_collection,\n    is_instance_or_subclass,\n    missing,\n    set_value,\n    validate_unknown_parameter_value,\n)\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\n_T = typing.TypeVar(\"_T\")\n\n\ndef _get_fields(attrs):\n    \"\"\"Get fields from a class\n\n    :param attrs: Mapping of class attributes\n    \"\"\"\n    return [\n        (field_name, field_value)\n        for field_name, field_value in attrs.items()\n        if is_instance_or_subclass(field_value, base.FieldABC)\n    ]\n\n\n# This function allows Schemas to inherit from non-Schema classes and ensures\n#   inheritance according to the MRO\ndef _get_fields_by_mro(klass):\n    \"\"\"Collect fields from a class, following its method resolution order. The\n    class itself is excluded from the search; only its parents are checked. Get\n    fields from ``_declared_fields`` if available, else use ``__dict__``.\n\n    :param type klass: Class whose fields to retrieve\n    \"\"\"\n    mro = inspect.getmro(klass)\n    # Loop over mro in reverse to maintain correct order of fields\n    return sum(\n        (\n            _get_fields(\n                getattr(base, \"_declared_fields\", base.__dict__),\n            )\n            for base in mro[:0:-1]\n        ),\n        [],\n    )\n\n\nclass SchemaMeta(ABCMeta):\n    \"\"\"Metaclass for the Schema class. Binds the declared fields to\n    a ``_declared_fields`` attribute, which is a dictionary mapping attribute\n    names to field objects. Also sets the ``opts`` class attribute, which is\n    the Schema class's ``class Meta`` options.\n    \"\"\"\n\n    def __new__(mcs, name, bases, attrs):\n        meta = attrs.get(\"Meta\")\n        ordered = getattr(meta, \"ordered\", False)\n        if not ordered:\n            # Inherit 'ordered' option\n            # Warning: We loop through bases instead of MRO because we don't\n            # yet have access to the class object\n            # (i.e. can't call super before we have fields)\n            for base_ in bases:\n                if hasattr(base_, \"Meta\") and hasattr(base_.Meta, \"ordered\"):\n                    ordered = base_.Meta.ordered\n                    break\n            else:\n                ordered = False\n        cls_fields = _get_fields(attrs)\n        # Remove fields from list of class attributes to avoid shadowing\n        # Schema attributes/methods in case of name conflict\n        for field_name, _ in cls_fields:\n            del attrs[field_name]\n        klass = super().__new__(mcs, name, bases, attrs)\n        inherited_fields = _get_fields_by_mro(klass)\n\n        meta = klass.Meta\n        # Set klass.opts in __new__ rather than __init__ so that it is accessible in\n        # get_declared_fields\n        klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered)\n        # Add fields specified in the `include` class Meta option\n        cls_fields += list(klass.opts.include.items())\n\n        # Assign _declared_fields on class\n        klass._declared_fields = mcs.get_declared_fields(\n            klass=klass,\n            cls_fields=cls_fields,\n            inherited_fields=inherited_fields,\n            dict_cls=dict,\n        )\n        return klass\n\n    @classmethod\n    def get_declared_fields(\n        mcs,\n        klass: type,\n        cls_fields: list,\n        inherited_fields: list,\n        dict_cls: type = dict,\n    ):\n        \"\"\"Returns a dictionary of field_name => `Field` pairs declared on the class.\n        This is exposed mainly so that plugins can add additional fields, e.g. fields\n        computed from class Meta options.\n\n        :param klass: The class object.\n        :param cls_fields: The fields declared on the class, including those added\n            by the ``include`` class Meta option.\n        :param inherited_fields: Inherited fields.\n        :param dict_cls: dict-like class to use for dict output Default to ``dict``.\n        \"\"\"\n        return dict_cls(inherited_fields + cls_fields)\n\n    def __init__(cls, name, bases, attrs):\n        super().__init__(name, bases, attrs)\n        if name and cls.opts.register:\n            class_registry.register(name, cls)\n        cls._hooks = cls.resolve_hooks()\n\n    def resolve_hooks(cls) -> dict[types.Tag, list[str]]:\n        \"\"\"Add in the decorated processors\n\n        By doing this after constructing the class, we let standard inheritance\n        do all the hard work.\n        \"\"\"\n        mro = inspect.getmro(cls)\n\n        hooks = defaultdict(list)  # type: typing.Dict[types.Tag, typing.List[str]]\n\n        for attr_name in dir(cls):\n            # Need to look up the actual descriptor, not whatever might be\n            # bound to the class. This needs to come from the __dict__ of the\n            # declaring class.\n            for parent in mro:\n                try:\n                    attr = parent.__dict__[attr_name]\n                except KeyError:\n                    continue\n                else:\n                    break\n            else:\n                # In case we didn't find the attribute and didn't break above.\n                # We should never hit this - it's just here for completeness\n                # to exclude the possibility of attr being undefined.\n                continue\n\n            try:\n                hook_config = attr.__marshmallow_hook__\n            except AttributeError:\n                pass\n            else:\n                for key in hook_config.keys():\n                    # Use name here so we can get the bound method later, in\n                    # case the processor was a descriptor or something.\n                    hooks[key].append(attr_name)\n\n        return hooks\n\n\nclass SchemaOpts:\n    \"\"\"class Meta options for the :class:`Schema`. Defines defaults.\"\"\"\n\n    def __init__(self, meta, ordered: bool = False):\n        self.fields = getattr(meta, \"fields\", ())\n        if not isinstance(self.fields, (list, tuple)):\n            raise ValueError(\"`fields` option must be a list or tuple.\")\n        self.additional = getattr(meta, \"additional\", ())\n        if not isinstance(self.additional, (list, tuple)):\n            raise ValueError(\"`additional` option must be a list or tuple.\")\n        if self.fields and self.additional:\n            raise ValueError(\n                \"Cannot set both `fields` and `additional` options\"\n                \" for the same Schema.\"\n            )\n        self.exclude = getattr(meta, \"exclude\", ())\n        if not isinstance(self.exclude, (list, tuple)):\n            raise ValueError(\"`exclude` must be a list or tuple.\")\n        self.dateformat = getattr(meta, \"dateformat\", None)\n        self.datetimeformat = getattr(meta, \"datetimeformat\", None)\n        self.timeformat = getattr(meta, \"timeformat\", None)\n        if hasattr(meta, \"json_module\"):\n            warnings.warn(\n                \"The json_module class Meta option is deprecated. Use render_module instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            render_module = getattr(meta, \"json_module\", json)\n        else:\n            render_module = json\n        self.render_module = getattr(meta, \"render_module\", render_module)\n        self.ordered = getattr(meta, \"ordered\", ordered)\n        self.index_errors = getattr(meta, \"index_errors\", True)\n        self.include = getattr(meta, \"include\", {})\n        self.load_only = getattr(meta, \"load_only\", ())\n        self.dump_only = getattr(meta, \"dump_only\", ())\n        self.unknown = validate_unknown_parameter_value(getattr(meta, \"unknown\", RAISE))\n        self.register = getattr(meta, \"register\", True)\n\n\nclass Schema(base.SchemaABC, metaclass=SchemaMeta):\n    \"\"\"Base schema class with which to define custom schemas.\n\n    Example usage:\n\n    .. code-block:: python\n\n        import datetime as dt\n        from dataclasses import dataclass\n\n        from marshmallow import Schema, fields\n\n\n        @dataclass\n        class Album:\n            title: str\n            release_date: dt.date\n\n\n        class AlbumSchema(Schema):\n            title = fields.Str()\n            release_date = fields.Date()\n\n\n        album = Album(\"Beggars Banquet\", dt.date(1968, 12, 6))\n        schema = AlbumSchema()\n        data = schema.dump(album)\n        data  # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'}\n\n    :param only: Whitelist of the declared fields to select when\n        instantiating the Schema. If None, all fields are used. Nested fields\n        can be represented with dot delimiters.\n    :param exclude: Blacklist of the declared fields to exclude\n        when instantiating the Schema. If a field appears in both `only` and\n        `exclude`, it is not used. Nested fields can be represented with dot\n        delimiters.\n    :param many: Should be set to `True` if ``obj`` is a collection\n        so that the object will be serialized to a list.\n    :param context: Optional context passed to :class:`fields.Method` and\n        :class:`fields.Function` fields.\n    :param load_only: Fields to skip during serialization (write-only fields)\n    :param dump_only: Fields to skip during deserialization (read-only fields)\n    :param partial: Whether to ignore missing fields and not require\n        any fields declared. Propagates down to ``Nested`` fields as well. If\n        its value is an iterable, only missing fields listed in that iterable\n        will be ignored. Use dot delimiters to specify nested fields.\n    :param unknown: Whether to exclude, include, or raise an error for unknown\n        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n\n    .. versionchanged:: 3.0.0\n        `prefix` parameter removed.\n\n    .. versionchanged:: 2.0.0\n        `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of\n        `marshmallow.decorators.validates_schema`,\n        `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`.\n        `__accessor__` and `__error_handler__` are deprecated. Implement the\n        `handle_error` and `get_attribute` methods instead.\n    \"\"\"\n\n    TYPE_MAPPING = {\n        str: ma_fields.String,\n        bytes: ma_fields.String,\n        dt.datetime: ma_fields.DateTime,\n        float: ma_fields.Float,\n        bool: ma_fields.Boolean,\n        tuple: ma_fields.Raw,\n        list: ma_fields.Raw,\n        set: ma_fields.Raw,\n        int: ma_fields.Integer,\n        uuid.UUID: ma_fields.UUID,\n        dt.time: ma_fields.Time,\n        dt.date: ma_fields.Date,\n        dt.timedelta: ma_fields.TimeDelta,\n        decimal.Decimal: ma_fields.Decimal,\n    }  # type: typing.Dict[type, typing.Type[ma_fields.Field]]\n    #: Overrides for default schema-level error messages\n    error_messages = {}  # type: typing.Dict[str, str]\n\n    _default_error_messages = {\n        \"type\": \"Invalid input type.\",\n        \"unknown\": \"Unknown field.\",\n    }  # type: typing.Dict[str, str]\n\n    OPTIONS_CLASS = SchemaOpts  # type: type\n\n    set_class = OrderedSet\n\n    # These get set by SchemaMeta\n    opts = None  # type: SchemaOpts\n    _declared_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n    _hooks = {}  # type: typing.Dict[types.Tag, typing.List[str]]\n\n    class Meta:\n        \"\"\"Options object for a Schema.\n\n        Example usage: ::\n\n            class Meta:\n                fields = (\"id\", \"email\", \"date_created\")\n                exclude = (\"password\", \"secret_attribute\")\n\n        Available options:\n\n        - ``fields``: Tuple or list of fields to include in the serialized result.\n        - ``additional``: Tuple or list of fields to include *in addition* to the\n            explicitly declared fields. ``additional`` and ``fields`` are\n            mutually-exclusive options.\n        - ``include``: Dictionary of additional fields to include in the schema. It is\n            usually better to define fields as class variables, but you may need to\n            use this option, e.g., if your fields are Python keywords. May be an\n            `OrderedDict`.\n        - ``exclude``: Tuple or list of fields to exclude in the serialized result.\n            Nested fields can be represented with dot delimiters.\n        - ``dateformat``: Default format for `Date <fields.Date>` fields.\n        - ``datetimeformat``: Default format for `DateTime <fields.DateTime>` fields.\n        - ``timeformat``: Default format for `Time <fields.Time>` fields.\n        - ``render_module``: Module to use for `loads <Schema.loads>` and `dumps <Schema.dumps>`.\n            Defaults to `json` from the standard library.\n        - ``ordered``: If `True`, output of `Schema.dump` will be a `collections.OrderedDict`.\n        - ``index_errors``: If `True`, errors dictionaries will include the index\n            of invalid items in a collection.\n        - ``load_only``: Tuple or list of fields to exclude from serialized results.\n        - ``dump_only``: Tuple or list of fields to exclude from deserialization\n        - ``unknown``: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        - ``register``: Whether to register the `Schema` with marshmallow's internal\n            class registry. Must be `True` if you intend to refer to this `Schema`\n            by class name in `Nested` fields. Only set this to `False` when memory\n            usage is critical. Defaults to `True`.\n        \"\"\"\n\n    def __init__(\n        self,\n        *,\n        only: types.StrSequenceOrSet | None = None,\n        exclude: types.StrSequenceOrSet = (),\n        many: bool = False,\n        context: dict | None = None,\n        load_only: types.StrSequenceOrSet = (),\n        dump_only: types.StrSequenceOrSet = (),\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        # Raise error if only or exclude is passed as string, not list of strings\n        if only is not None and not is_collection(only):\n            raise StringNotCollectionError('\"only\" should be a list of strings')\n        if not is_collection(exclude):\n            raise StringNotCollectionError('\"exclude\" should be a list of strings')\n        # copy declared fields from metaclass\n        self.declared_fields = copy.deepcopy(self._declared_fields)\n        self.many = many\n        self.only = only\n        self.exclude: set[typing.Any] | typing.MutableSet[typing.Any] = set(\n            self.opts.exclude\n        ) | set(exclude)\n        self.ordered = self.opts.ordered\n        self.load_only = set(load_only) or set(self.opts.load_only)\n        self.dump_only = set(dump_only) or set(self.opts.dump_only)\n        self.partial = partial\n        self.unknown = (\n            self.opts.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        self.context = context or {}\n        self._normalize_nested_options()\n        #: Dictionary mapping field_names -> :class:`Field` objects\n        self.fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.load_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.dump_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self._init_fields()\n        messages = {}\n        messages.update(self._default_error_messages)\n        for cls in reversed(self.__class__.__mro__):\n            messages.update(getattr(cls, \"error_messages\", {}))\n        messages.update(self.error_messages or {})\n        self.error_messages = messages\n\n    def __repr__(self) -> str:\n        return f\"<{self.__class__.__name__}(many={self.many})>\"\n\n    @property\n    def dict_class(self) -> type:\n        return OrderedDict if self.ordered else dict\n\n    @classmethod\n    def from_dict(\n        cls,\n        fields: dict[str, ma_fields.Field | type],\n        *,\n        name: str = \"GeneratedSchema\",\n    ) -> type:\n        \"\"\"Generate a `Schema` class given a dictionary of fields.\n\n        .. code-block:: python\n\n            from marshmallow import Schema, fields\n\n            PersonSchema = Schema.from_dict({\"name\": fields.Str()})\n            print(PersonSchema().load({\"name\": \"David\"}))  # => {'name': 'David'}\n\n        Generated schemas are not added to the class registry and therefore cannot\n        be referred to by name in `Nested` fields.\n\n        :param dict fields: Dictionary mapping field names to field instances.\n        :param str name: Optional name for the class, which will appear in\n            the ``repr`` for the class.\n\n        .. versionadded:: 3.0.0\n        \"\"\"\n        attrs = fields.copy()\n        attrs[\"Meta\"] = type(\n            \"GeneratedMeta\", (getattr(cls, \"Meta\", object),), {\"register\": False}\n        )\n        schema_cls = type(name, (cls,), attrs)\n        return schema_cls\n\n    ##### Override-able methods #####\n\n    def handle_error(\n        self, error: ValidationError, data: typing.Any, *, many: bool, **kwargs\n    ):\n        \"\"\"Custom error handler function for the schema.\n\n        :param error: The `ValidationError` raised during (de)serialization.\n        :param data: The original input data.\n        :param many: Value of ``many`` on dump or load.\n        :param partial: Value of ``partial`` on load.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0rc9\n            Receives `many` and `partial` (on deserialization) as keyword arguments.\n        \"\"\"\n        pass\n\n    def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any):\n        \"\"\"Defines how to pull values from an object to serialize.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0a1\n            Changed position of ``obj`` and ``attr``.\n        \"\"\"\n        return get_value(obj, attr, default)\n\n    ##### Serialization/Deserialization API #####\n\n    @staticmethod\n    def _call_and_store(getter_func, data, *, field_name, error_store, index=None):\n        \"\"\"Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.\n\n        :param callable getter_func: Function for getting the serialized/deserialized\n            value from ``data``.\n        :param data: The data passed to ``getter_func``.\n        :param str field_name: Field name.\n        :param int index: Index of the item being validated, if validating a collection,\n            otherwise `None`.\n        \"\"\"\n        try:\n            value = getter_func(data)\n        except ValidationError as error:\n            error_store.store_error(error.messages, field_name, index=index)\n            # When a Nested field fails validation, the marshalled data is stored\n            # on the ValidationError's valid_data attribute\n            return error.valid_data or missing\n        return value\n\n    def _serialize(self, obj: _T | typing.Iterable[_T], *, many: bool = False):\n        \"\"\"Serialize ``obj``.\n\n        :param obj: The object(s) to serialize.\n        :param bool many: `True` if ``data`` should be serialized as a collection.\n        :return: A dictionary of the serialized data\n\n        .. versionchanged:: 1.0.0\n            Renamed from ``marshal``.\n        \"\"\"\n        if many and obj is not None:\n            return [\n                self._serialize(d, many=False)\n                for d in typing.cast(typing.Iterable[_T], obj)\n            ]\n        ret = self.dict_class()\n        for attr_name, field_obj in self.dump_fields.items():\n            value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)\n            if value is missing:\n                continue\n            key = field_obj.data_key if field_obj.data_key is not None else attr_name\n            ret[key] = value\n        return ret\n\n    def dump(self, obj: typing.Any, *, many: bool | None = None):\n        \"\"\"Serialize an object to native Python data types according to this\n        Schema's fields.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: Serialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        .. versionchanged:: 3.0.0rc9\n            Validation no longer occurs upon serialization.\n        \"\"\"\n        many = self.many if many is None else bool(many)\n        if self._has_processors(PRE_DUMP):\n            processed_obj = self._invoke_dump_processors(\n                PRE_DUMP, obj, many=many, original_data=obj\n            )\n        else:\n            processed_obj = obj\n\n        result = self._serialize(processed_obj, many=many)\n\n        if self._has_processors(POST_DUMP):\n            result = self._invoke_dump_processors(\n                POST_DUMP, result, many=many, original_data=obj\n            )\n\n        return result\n\n    def dumps(self, obj: typing.Any, *args, many: bool | None = None, **kwargs):\n        \"\"\"Same as :meth:`dump`, except return a JSON-encoded string.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: A ``json`` string\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        \"\"\"\n        serialized = self.dump(obj, many=many)\n        return self.opts.render_module.dumps(serialized, *args, **kwargs)\n\n    def _deserialize(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        error_store: ErrorStore,\n        many: bool = False,\n        partial=None,\n        unknown=RAISE,\n        index=None,\n    ) -> _T | list[_T]:\n        \"\"\"Deserialize ``data``.\n\n        :param dict data: The data to deserialize.\n        :param ErrorStore error_store: Structure to store errors.\n        :param bool many: `True` if ``data`` should be deserialized as a collection.\n        :param bool|tuple partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        :param int index: Index of the item being serialized (for storing errors) if\n            serializing a collection, otherwise `None`.\n        :return: A dictionary of the deserialized data.\n        \"\"\"\n        index_errors = self.opts.index_errors\n        index = index if index_errors else None\n        if many:\n            if not is_collection(data):\n                error_store.store_error([self.error_messages[\"type\"]], index=index)\n                ret_l = []  # type: typing.List[_T]\n            else:\n                ret_l = [\n                    typing.cast(\n                        _T,\n                        self._deserialize(\n                            typing.cast(typing.Mapping[str, typing.Any], d),\n                            error_store=error_store,\n                            many=False,\n                            partial=partial,\n                            unknown=unknown,\n                            index=idx,\n                        ),\n                    )\n                    for idx, d in enumerate(data)\n                ]\n            return ret_l\n        ret_d = self.dict_class()\n        # Check data is a dict\n        if not isinstance(data, Mapping):\n            error_store.store_error([self.error_messages[\"type\"]], index=index)\n        else:\n            partial_is_collection = is_collection(partial)\n            for attr_name, field_obj in self.load_fields.items():\n                field_name = (\n                    field_obj.data_key if field_obj.data_key is not None else attr_name\n                )\n                raw_value = data.get(field_name, missing)\n                if raw_value is missing:\n                    # Ignore missing field if we're allowed to.\n                    if partial is True or (\n                        partial_is_collection and attr_name in partial\n                    ):\n                        continue\n                d_kwargs = {}\n                # Allow partial loading of nested schemas.\n                if partial_is_collection:\n                    prefix = field_name + \".\"\n                    len_prefix = len(prefix)\n                    sub_partial = [\n                        f[len_prefix:] for f in partial if f.startswith(prefix)\n                    ]\n                    d_kwargs[\"partial\"] = sub_partial\n                elif partial is not None:\n                    d_kwargs[\"partial\"] = partial\n\n                def getter(\n                    val, field_obj=field_obj, field_name=field_name, d_kwargs=d_kwargs\n                ):\n                    return field_obj.deserialize(\n                        val,\n                        field_name,\n                        data,\n                        **d_kwargs,\n                    )\n\n                value = self._call_and_store(\n                    getter_func=getter,\n                    data=raw_value,\n                    field_name=field_name,\n                    error_store=error_store,\n                    index=index,\n                )\n                if value is not missing:\n                    key = field_obj.attribute or attr_name\n                    set_value(ret_d, key, value)\n            if unknown != EXCLUDE:\n                fields = {\n                    field_obj.data_key if field_obj.data_key is not None else field_name\n                    for field_name, field_obj in self.load_fields.items()\n                }\n                for key in set(data) - fields:\n                    value = data[key]\n                    if unknown == INCLUDE:\n                        ret_d[key] = value\n                    elif unknown == RAISE:\n                        error_store.store_error(\n                            [self.error_messages[\"unknown\"]],\n                            key,\n                            (index if index_errors else None),\n                        )\n        return ret_d\n\n    def load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        \"\"\"Deserialize a data structure to an object defined by this Schema's fields.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        return self._do_load(\n            data, many=many, partial=partial, unknown=unknown, postprocess=True\n        )\n\n    def loads(\n        self,\n        json_data: str,\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        **kwargs,\n    ):\n        \"\"\"Same as :meth:`load`, except it takes a JSON string as input.\n\n        :param json_data: A JSON string of the data to deserialize.\n        :param many: Whether to deserialize `obj` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        data = self.opts.render_module.loads(json_data, **kwargs)\n        return self.load(data, many=many, partial=partial, unknown=unknown)\n\n    def _run_validator(\n        self,\n        validator_func,\n        output,\n        *,\n        original_data,\n        error_store,\n        many,\n        partial,\n        pass_original,\n        index=None,\n    ):\n        try:\n            if pass_original:  # Pass original, raw data (before unmarshalling)\n                validator_func(output, original_data, partial=partial, many=many)\n            else:\n                validator_func(output, partial=partial, many=many)\n        except ValidationError as err:\n            error_store.store_error(err.messages, err.field_name, index=index)\n\n    def validate(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n    ) -> dict[str, list[str]]:\n        \"\"\"Validate `data` against the schema, returning a dictionary of\n        validation errors.\n\n        :param data: The data to validate.\n        :param many: Whether to validate `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :return: A dictionary of validation errors.\n\n        .. versionadded:: 1.1.0\n        \"\"\"\n        try:\n            self._do_load(data, many=many, partial=partial, postprocess=False)\n        except ValidationError as exc:\n            return typing.cast(typing.Dict[str, typing.List[str]], exc.messages)\n        return {}\n\n    ##### Private Helpers #####\n\n    def _do_load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        postprocess: bool = True,\n    ):\n        \"\"\"Deserialize `data`, returning the deserialized result.\n        This method is private API.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to validate required fields. If its\n            value is an iterable, only fields listed in that iterable will be\n            ignored will be allowed missing. If `True`, all fields will be allowed missing.\n            If `None`, the value for `self.partial` is used.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :param postprocess: Whether to run post_load methods..\n        :return: Deserialized data\n        \"\"\"\n...\n", "needle_token_start": 603, "needle_token_end": 985, "code_context_ntokens": 16130, "output": ["```python\ndef register(classname: str, cls: SchemaType) -> None:\n    \"\"\"Add a class to the registry of serializer classes. When a class is\n    registered, an entry for both its classname and its full, module-qualified\n    path are added to the registry.\n\n    Example: ::\n\n        class MyClass:\n            pass\n\n        register('MyClass', MyClass)\n        # Registry:\n        # {\n        #   'MyClass': [path.to.MyClass],\n        #   'path.to.MyClass': [path.to.MyClass],\n        # }\n\n    \"\"\"\n    # Module where the class is located\n    module = cls.__module__\n    # Full module path to the class\n    # e.g. user.schemas.UserSchema\n    fullpath = \".\".join([module, classname])\n    # If the class is already registered; need to check if the entries are\n    # in the same module as cls to avoid having multiple instances of the same\n    # class in the registry\n    if classname in _registry and not any(\n        each.__module__ == module for each in _registry[classname]\n    ):\n        _registry[classname].append(cls)\n    elif classname not in _registry:\n        _registry[classname] = [cls]\n\n    # Also register the full path\n    if fullpath not in _registry:\n        _registry.setdefault(fullpath, []).append(cls)\n    else:\n        # If fullpath does exist, replace existing entry\n        _registry[fullpath] = [cls]\n    return None\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "store_error", "language": "python", "path": "src/marshmallow/error_store.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to accumulate error messages related to data serialization, categorizing them by field name or schema, and optionally by an index if provided.\n2. **Input**: The function accepts error messages (either as strings, lists, or dictionaries), a field name identifier (defaulting to a schema-wide error identifier if not specified), and an optional index to specify a particular element in a collection.\n3. **Output**: There is no direct output returned from the function; instead, it modifies an internal dictionary to include the new error messages, organizing them according to the provided field name and index.\n4. **Procedure**: The function first checks if the error message should be categorized under a specific field or as a general schema error. If an index is provided, the error message is further nested under this index. The function then merges these newly formatted error messages into an existing dictionary of errors, ensuring that all related errors are grouped together appropriately.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/marshmallow/decorators.py\n\"\"\"Decorators for registering schema pre-processing and post-processing methods.\nThese should be imported from the top-level `marshmallow` module.\n\nMethods decorated with\n`pre_load <marshmallow.decorators.pre_load>`, `post_load <marshmallow.decorators.post_load>`,\n`pre_dump <marshmallow.decorators.pre_dump>`, `post_dump <marshmallow.decorators.post_dump>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``many`` as a keyword argument. In addition, `pre_load <marshmallow.decorators.pre_load>`,\n`post_load <marshmallow.decorators.post_load>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``partial``. If you don't need these arguments, add ``**kwargs`` to your method\nsignature.\n\n\nExample: ::\n\n    from marshmallow import (\n        Schema, pre_load, pre_dump, post_load, validates_schema,\n        validates, fields, ValidationError\n    )\n\n    class UserSchema(Schema):\n\n        email = fields.Str(required=True)\n...\n# Path: src/marshmallow/exceptions.py\n\"\"\"Exception classes for marshmallow-related errors.\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\n# Key used for schema-level validation errors\nSCHEMA = \"_schema\"\n\n\nclass MarshmallowError(Exception):\n    \"\"\"Base class for all marshmallow-related errors.\"\"\"\n\n\nclass ValidationError(MarshmallowError):\n    \"\"\"Raised when validation fails on a field or schema.\n\n    Validators and custom fields should raise this exception.\n\n    :param message: An error message, list of error messages, or dict of\n        error messages. If a dict, the keys are subitems and the values are error messages.\n    :param field_name: Field name to store the error on.\n        If `None`, the error is stored as schema-level error.\n    :param data: Raw input data.\n    :param valid_data: Valid (de)serialized data.\n    \"\"\"\n\n    def __init__(\n        self,\n        message: str | list | dict,\n        field_name: str = SCHEMA,\n        data: typing.Mapping[str, typing.Any]\n        | typing.Iterable[typing.Mapping[str, typing.Any]]\n        | None = None,\n        valid_data: list[dict[str, typing.Any]] | dict[str, typing.Any] | None = None,\n        **kwargs,\n    ):\n        self.messages = [message] if isinstance(message, (str, bytes)) else message\n        self.field_name = field_name\n        self.data = data\n        self.valid_data = valid_data\n        self.kwargs = kwargs\n        super().__init__(message)\n\n    def normalized_messages(self):\n        if self.field_name == SCHEMA and isinstance(self.messages, dict):\n            return self.messages\n        return {self.field_name: self.messages}\n\n    @property\n    def messages_dict(self) -> dict[str, typing.Any]:\n        if not isinstance(self.messages, dict):\n            raise TypeError(\n                \"cannot access 'messages_dict' when 'messages' is of type \"\n                + type(self.messages).__name__\n            )\n        return self.messages\n\n\nclass RegistryError(NameError):\n    \"\"\"Raised when an invalid operation is performed on the serializer\n    class registry.\n    \"\"\"\n\n\nclass StringNotCollectionError(MarshmallowError, TypeError):\n    \"\"\"Raised when a string is passed when a list of strings is expected.\"\"\"\n\n\nclass FieldInstanceResolutionError(MarshmallowError, TypeError):\n    \"\"\"Raised when schema to instantiate is neither a Schema class nor an instance.\"\"\"\n\n# Path: src/marshmallow/base.py\n\"\"\"Abstract base classes.\n\nThese are necessary to avoid circular imports between schema.py and fields.py.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nfrom abc import ABC, abstractmethod\n\n\nclass FieldABC(ABC):\n    \"\"\"Abstract base class from which all Field classes inherit.\"\"\"\n\n    parent = None\n    name = None\n    root = None\n\n    @abstractmethod\n    def serialize(self, attr, obj, accessor=None):\n        pass\n\n    @abstractmethod\n    def deserialize(self, value):\n        pass\n\n    @abstractmethod\n    def _serialize(self, value, attr, obj, **kwargs):\n        pass\n\n    @abstractmethod\n    def _deserialize(self, value, attr, data, **kwargs):\n        pass\n\n\nclass SchemaABC(ABC):\n    \"\"\"Abstract base class from which all Schemas inherit.\"\"\"\n\n    @abstractmethod\n    def dump(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def dumps(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def load(self, data, *, many: bool | None = None, partial=None, unknown=None):\n        pass\n\n    @abstractmethod\n    def loads(\n        self,\n        json_data,\n        *,\n        many: bool | None = None,\n        partial=None,\n        unknown=None,\n        **kwargs,\n    ):\n        pass\n\n# Path: src/marshmallow/class_registry.py\n\"\"\"A registry of :class:`Schema <marshmallow.Schema>` classes. This allows for string\nlookup of schemas, which may be used with\nclass:`fields.Nested <marshmallow.fields.Nested>`.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\nfrom marshmallow.exceptions import RegistryError\n\nif typing.TYPE_CHECKING:\n    from marshmallow import Schema\n\n    SchemaType = typing.Type[Schema]\n\n# {\n#   <class_name>: <list of class objects>\n#   <module_path_to_class>: <list of class objects>\n# }\n_registry = {}  # type: dict[str, list[SchemaType]]\n\n\ndef register(classname: str, cls: SchemaType) -> None:\n    \"\"\"Add a class to the registry of serializer classes. When a class is\n    registered, an entry for both its classname and its full, module-qualified\n    path are added to the registry.\n\n    Example: ::\n\n        class MyClass:\n            pass\n\n        register('MyClass', MyClass)\n        # Registry:\n        # {\n        #   'MyClass': [path.to.MyClass],\n        #   'path.to.MyClass': [path.to.MyClass],\n        # }\n\n    \"\"\"\n    # Module where the class is located\n    module = cls.__module__\n    # Full module path to the class\n    # e.g. user.schemas.UserSchema\n    fullpath = \".\".join([module, classname])\n    # If the class is already registered; need to check if the entries are\n    # in the same module as cls to avoid having multiple instances of the same\n    # class in the registry\n    if classname in _registry and not any(\n        each.__module__ == module for each in _registry[classname]\n    ):\n        _registry[classname].append(cls)\n    elif classname not in _registry:\n        _registry[classname] = [cls]\n\n    # Also register the full path\n    if fullpath not in _registry:\n        _registry.setdefault(fullpath, []).append(cls)\n    else:\n        # If fullpath does exist, replace existing entry\n        _registry[fullpath] = [cls]\n    return None\n\n\ndef get_class(classname: str, all: bool = False) -> list[SchemaType] | SchemaType:\n    \"\"\"Retrieve a class from the registry.\n\n    :raises: marshmallow.exceptions.RegistryError if the class cannot be found\n        or if there are multiple entries for the given class name.\n    \"\"\"\n    try:\n        classes = _registry[classname]\n    except KeyError as error:\n        raise RegistryError(\n            f\"Class with name {classname!r} was not found. You may need \"\n            \"to import the class.\"\n        ) from error\n    if len(classes) > 1:\n        if all:\n            return _registry[classname]\n        raise RegistryError(\n            f\"Multiple classes with name {classname!r} \"\n            \"were found. Please use the full, \"\n            \"module-qualified path.\"\n        )\n    else:\n        return _registry[classname][0]\n\n# Path: src/marshmallow/error_store.py\n\"\"\"Utilities for storing collections of error messages.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\n\nfrom marshmallow.exceptions import SCHEMA\n\n\nclass ErrorStore:\n    def __init__(self):\n        #: Dictionary of errors stored during serialization\n        self.errors = {}\n\n    \ndef store_error(self, messages, field_name=SCHEMA, index=None):\n        # field error  -> store/merge error messages under field name key\n        # schema error -> if string or list, store/merge under _schema key\n        #              -> if dict, store/merge with other top-level keys\n        if field_name != SCHEMA or not isinstance(messages, dict):\n            messages = {field_name: messages}\n        if index is not None:\n            messages = {index: messages}\n        self.errors = merge_errors(self.errors, messages)\n\n\ndef merge_errors(errors1, errors2):\n    \"\"\"Deeply merge two error messages.\n\n    The format of ``errors1`` and ``errors2`` matches the ``message``\n    parameter of :exc:`marshmallow.exceptions.ValidationError`.\n    \"\"\"\n    if not errors1:\n        return errors2\n    if not errors2:\n        return errors1\n    if isinstance(errors1, list):\n        if isinstance(errors2, list):\n            return errors1 + errors2\n        if isinstance(errors2, dict):\n            return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n        return errors1 + [errors2]\n    if isinstance(errors1, dict):\n        if isinstance(errors2, list):\n            return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n        if isinstance(errors2, dict):\n            errors = dict(errors1)\n            for key, val in errors2.items():\n                if key in errors:\n                    errors[key] = merge_errors(errors[key], val)\n                else:\n                    errors[key] = val\n            return errors\n        return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n    if isinstance(errors2, list):\n        return [errors1] + errors2\n    if isinstance(errors2, dict):\n        return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n    return [errors1, errors2]\n\n# Path: src/marshmallow/orderedset.py\n# OrderedSet\n# Copyright (c) 2009 Raymond Hettinger\n#\n# Permission is hereby granted, free of charge, to any person\n# obtaining a copy of this software and associated documentation files\n# (the \"Software\"), to deal in the Software without restriction,\n# including without limitation the rights to use, copy, modify, merge,\n# publish, distribute, sublicense, and/or sell copies of the Software,\n# and to permit persons to whom the Software is furnished to do so,\n# subject to the following conditions:\n#\n#     The above copyright notice and this permission notice shall be\n#     included in all copies or substantial portions of the Software.\n#\n#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES\n#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\n#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\n#     OTHER DEALINGS IN THE SOFTWARE.\nfrom collections.abc import MutableSet\n\n\nclass OrderedSet(MutableSet):\n    def __init__(self, iterable=None):\n        self.end = end = []\n        end += [None, end, end]  # sentinel node for doubly linked list\n        self.map = {}  # key --> [key, prev, next]\n        if iterable is not None:\n            self |= iterable\n\n    def __len__(self):\n        return len(self.map)\n\n    def __contains__(self, key):\n        return key in self.map\n\n    def add(self, key):\n        if key not in self.map:\n            end = self.end\n            curr = end[1]\n            curr[2] = end[1] = self.map[key] = [key, curr, end]\n\n    def discard(self, key):\n        if key in self.map:\n            key, prev, next = self.map.pop(key)\n            prev[2] = next\n            next[1] = prev\n\n    def __iter__(self):\n        end = self.end\n        curr = end[2]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[2]\n\n    def __reversed__(self):\n        end = self.end\n        curr = end[1]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[1]\n\n    def pop(self, last=True):\n        if not self:\n            raise KeyError(\"set is empty\")\n        key = self.end[1][0] if last else self.end[2][0]\n        self.discard(key)\n        return key\n\n    def __repr__(self):\n        if not self:\n            return f\"{self.__class__.__name__}()\"\n        return f\"{self.__class__.__name__}({list(self)!r})\"\n\n    def __eq__(self, other):\n        if isinstance(other, OrderedSet):\n            return len(self) == len(other) and list(self) == list(other)\n        return set(self) == set(other)\n\n\nif __name__ == \"__main__\":\n    s = OrderedSet(\"abracadaba\")\n    t = OrderedSet(\"simsalabim\")\n    print(s | t)\n    print(s & t)\n    print(s - t)\n\n# Path: src/marshmallow/types.py\n\"\"\"Type aliases.\n\n.. warning::\n\n    This module is provisional. Types may be modified, added, and removed between minor releases.\n\"\"\"\nimport typing\n\nStrSequenceOrSet = typing.Union[typing.Sequence[str], typing.AbstractSet[str]]\nTag = typing.Union[str, typing.Tuple[str, bool]]\nValidator = typing.Callable[[typing.Any], typing.Any]\n\n# Path: src/marshmallow/warnings.py\nclass RemovedInMarshmallow4Warning(DeprecationWarning):\n    pass\n\n# Path: src/marshmallow/utils.py\n\"\"\"Utility methods for marshmallow.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport datetime as dt\nimport functools\nimport inspect\nimport json\nimport re\nimport typing\nimport warnings\nfrom collections.abc import Mapping\nfrom email.utils import format_datetime, parsedate_to_datetime\nfrom pprint import pprint as py_pprint\n\nfrom marshmallow.base import FieldABC\nfrom marshmallow.exceptions import FieldInstanceResolutionError\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\nEXCLUDE = \"exclude\"\nINCLUDE = \"include\"\nRAISE = \"raise\"\n_UNKNOWN_VALUES = {EXCLUDE, INCLUDE, RAISE}\n\n\nclass _Missing:\n    def __bool__(self):\n        return False\n\n    def __copy__(self):\n        return self\n\n    def __deepcopy__(self, _):\n        return self\n\n    def __repr__(self):\n        return \"<marshmallow.missing>\"\n\n\n# Singleton value that indicates that a field's value is missing from input\n# dict passed to :meth:`Schema.load`. If the field's value is not required,\n# it's ``default`` value is used.\nmissing = _Missing()\n\n\ndef is_generator(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a generator\"\"\"\n    return inspect.isgeneratorfunction(obj) or inspect.isgenerator(obj)\n\n\ndef is_iterable_but_not_string(obj) -> bool:\n    \"\"\"Return True if ``obj`` is an iterable object that isn't a string.\"\"\"\n    return (hasattr(obj, \"__iter__\") and not hasattr(obj, \"strip\")) or is_generator(obj)\n\n\ndef is_collection(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a collection type, e.g list, tuple, queryset.\"\"\"\n    return is_iterable_but_not_string(obj) and not isinstance(obj, Mapping)\n\n\ndef is_instance_or_subclass(val, class_) -> bool:\n    \"\"\"Return True if ``val`` is either a subclass or instance of ``class_``.\"\"\"\n    try:\n        return issubclass(val, class_)\n    except TypeError:\n        return isinstance(val, class_)\n\n\ndef is_keyed_tuple(obj) -> bool:\n    \"\"\"Return True if ``obj`` has keyed tuple behavior, such as\n    namedtuples or SQLAlchemy's KeyedTuples.\n    \"\"\"\n    return isinstance(obj, tuple) and hasattr(obj, \"_fields\")\n\n\ndef pprint(obj, *args, **kwargs) -> None:\n    \"\"\"Pretty-printing function that can pretty-print OrderedDicts\n    like regular dictionaries. Useful for printing the output of\n    :meth:`marshmallow.Schema.dump`.\n\n    .. deprecated:: 3.7.0\n        marshmallow.pprint will be removed in marshmallow 4.\n    \"\"\"\n    warnings.warn(\n        \"marshmallow's pprint function is deprecated and will be removed in marshmallow 4.\",\n        RemovedInMarshmallow4Warning,\n        stacklevel=2,\n    )\n    if isinstance(obj, collections.OrderedDict):\n        print(json.dumps(obj, *args, **kwargs))\n    else:\n        py_pprint(obj, *args, **kwargs)\n\n\n# https://stackoverflow.com/a/27596917\ndef is_aware(datetime: dt.datetime) -> bool:\n    return (\n        datetime.tzinfo is not None and datetime.tzinfo.utcoffset(datetime) is not None\n    )\n\n\ndef from_rfc(datestring: str) -> dt.datetime:\n    \"\"\"Parse a RFC822-formatted datetime string and return a datetime object.\n\n    https://stackoverflow.com/questions/885015/how-to-parse-a-rfc-2822-date-time-into-a-python-datetime  # noqa: B950\n    \"\"\"\n    return parsedate_to_datetime(datestring)\n\n\ndef rfcformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the RFC822-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return format_datetime(datetime)\n\n\n# Hat tip to Django for ISO8601 deserialization functions\n\n_iso8601_datetime_re = re.compile(\n    r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})\"\n    r\"[T ](?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n    r\"(?P<tzinfo>Z|[+-]\\d{2}(?::?\\d{2})?)?$\"\n)\n\n_iso8601_date_re = re.compile(r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})$\")\n\n_iso8601_time_re = re.compile(\n    r\"(?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n)\n\n\ndef get_fixed_timezone(offset: int | float | dt.timedelta) -> dt.timezone:\n    \"\"\"Return a tzinfo instance with a fixed offset from UTC.\"\"\"\n    if isinstance(offset, dt.timedelta):\n        offset = offset.total_seconds() // 60\n    sign = \"-\" if offset < 0 else \"+\"\n    hhmm = \"%02d%02d\" % divmod(abs(offset), 60)\n    name = sign + hhmm\n    return dt.timezone(dt.timedelta(minutes=offset), name)\n\n\ndef from_iso_datetime(value):\n    \"\"\"Parse a string and return a datetime.datetime.\n\n    This function supports time zone offsets. When the input contains one,\n    the output uses a timezone with a fixed offset from UTC.\n    \"\"\"\n    match = _iso8601_datetime_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted datetime string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    tzinfo = kw.pop(\"tzinfo\")\n    if tzinfo == \"Z\":\n        tzinfo = dt.timezone.utc\n    elif tzinfo is not None:\n        offset_mins = int(tzinfo[-2:]) if len(tzinfo) > 3 else 0\n        offset = 60 * int(tzinfo[1:3]) + offset_mins\n        if tzinfo[0] == \"-\":\n            offset = -offset\n        tzinfo = get_fixed_timezone(offset)\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    kw[\"tzinfo\"] = tzinfo\n    return dt.datetime(**kw)\n\n\ndef from_iso_time(value):\n    \"\"\"Parse a string and return a datetime.time.\n\n    This function doesn't support time zone offsets.\n    \"\"\"\n    match = _iso8601_time_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted time string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    return dt.time(**kw)\n\n\ndef from_iso_date(value):\n    \"\"\"Parse a string and return a datetime.date.\"\"\"\n    match = _iso8601_date_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted date string\")\n    kw = {k: int(v) for k, v in match.groupdict().items()}\n    return dt.date(**kw)\n\n\ndef from_timestamp(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    if value < 0:\n        raise ValueError(\"Not a valid POSIX timestamp\")\n\n    # Load a timestamp with utc as timezone to prevent using system timezone.\n    # Then set timezone to None, to let the Field handle adding timezone info.\n    try:\n        return dt.datetime.fromtimestamp(value, tz=dt.timezone.utc).replace(tzinfo=None)\n    except OverflowError as exc:\n        raise ValueError(\"Timestamp is too large\") from exc\n    except OSError as exc:\n        raise ValueError(\"Error converting value to datetime\") from exc\n\n\ndef from_timestamp_ms(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    return from_timestamp(value / 1000)\n\n\ndef timestamp(\n    value: dt.datetime,\n) -> float:\n    if not is_aware(value):\n        # When a date is naive, use UTC as zone info to prevent using system timezone.\n        value = value.replace(tzinfo=dt.timezone.utc)\n    return value.timestamp()\n\n\ndef timestamp_ms(value: dt.datetime) -> float:\n    return timestamp(value) * 1000\n\n\ndef isoformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the ISO8601-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return datetime.isoformat()\n\n\ndef to_iso_time(time: dt.time) -> str:\n    return dt.time.isoformat(time)\n\n\ndef to_iso_date(date: dt.date) -> str:\n    return dt.date.isoformat(date)\n\n\ndef ensure_text_type(val: str | bytes) -> str:\n    if isinstance(val, bytes):\n        val = val.decode(\"utf-8\")\n    return str(val)\n\n\ndef pluck(dictlist: list[dict[str, typing.Any]], key: str):\n    \"\"\"Extracts a list of dictionary values from a list of dictionaries.\n    ::\n\n        >>> dlist = [{'id': 1, 'name': 'foo'}, {'id': 2, 'name': 'bar'}]\n        >>> pluck(dlist, 'id')\n        [1, 2]\n    \"\"\"\n    return [d[key] for d in dictlist]\n\n\n# Various utilities for pulling keyed values from objects\n\n\ndef get_value(obj, key: int | str, default=missing):\n    \"\"\"Helper for pulling a keyed value off various types of objects. Fields use\n    this method by default to access attributes of the source object. For object `x`\n    and attribute `i`, this method first tries to access `x[i]`, and then falls back to\n    `x.i` if an exception is raised.\n\n    .. warning::\n        If an object `x` does not raise an exception when `x[i]` does not exist,\n        `get_value` will never check the value `x.i`. Consider overriding\n        `marshmallow.fields.Field.get_value` in this case.\n    \"\"\"\n    if not isinstance(key, int) and \".\" in key:\n        return _get_value_for_keys(obj, key.split(\".\"), default)\n    else:\n        return _get_value_for_key(obj, key, default)\n\n\ndef _get_value_for_keys(obj, keys, default):\n    if len(keys) == 1:\n        return _get_value_for_key(obj, keys[0], default)\n    else:\n        return _get_value_for_keys(\n            _get_value_for_key(obj, keys[0], default), keys[1:], default\n        )\n\n\ndef _get_value_for_key(obj, key, default):\n    if not hasattr(obj, \"__getitem__\"):\n        return getattr(obj, key, default)\n\n    try:\n        return obj[key]\n    except (KeyError, IndexError, TypeError, AttributeError):\n        return getattr(obj, key, default)\n\n\ndef set_value(dct: dict[str, typing.Any], key: str, value: typing.Any):\n    \"\"\"Set a value in a dict. If `key` contains a '.', it is assumed\n    be a path (i.e. dot-delimited string) to the value's location.\n\n    ::\n\n        >>> d = {}\n        >>> set_value(d, 'foo.bar', 42)\n        >>> d\n        {'foo': {'bar': 42}}\n    \"\"\"\n    if \".\" in key:\n        head, rest = key.split(\".\", 1)\n        target = dct.setdefault(head, {})\n        if not isinstance(target, dict):\n            raise ValueError(\n                f\"Cannot set {key} in {head} \" f\"due to existing value: {target}\"\n            )\n        set_value(target, rest, value)\n    else:\n        dct[key] = value\n\n\ndef callable_or_raise(obj):\n    \"\"\"Check that an object is callable, else raise a :exc:`TypeError`.\"\"\"\n    if not callable(obj):\n        raise TypeError(f\"Object {obj!r} is not callable.\")\n    return obj\n\n\ndef _signature(func: typing.Callable) -> list[str]:\n    return list(inspect.signature(func).parameters.keys())\n\n\ndef get_func_args(func: typing.Callable) -> list[str]:\n    \"\"\"Given a callable, return a list of argument names. Handles\n    `functools.partial` objects and class-based callables.\n\n    .. versionchanged:: 3.0.0a1\n        Do not return bound arguments, eg. ``self``.\n    \"\"\"\n    if inspect.isfunction(func) or inspect.ismethod(func):\n        return _signature(func)\n    if isinstance(func, functools.partial):\n        return _signature(func.func)\n    # Callable class\n    return _signature(func)\n\n\ndef resolve_field_instance(cls_or_instance):\n    \"\"\"Return a Schema instance from a Schema class or instance.\n\n    :param type|Schema cls_or_instance: Marshmallow Schema class or instance.\n    \"\"\"\n    if isinstance(cls_or_instance, type):\n        if not issubclass(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance()\n    else:\n        if not isinstance(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance\n\n\ndef timedelta_to_microseconds(value: dt.timedelta) -> int:\n    \"\"\"Compute the total microseconds of a timedelta\n\n    https://github.com/python/cpython/blob/bb3e0c240bc60fe08d332ff5955d54197f79751c/Lib/datetime.py#L665-L667  # noqa: B950\n    \"\"\"\n    return (value.days * (24 * 3600) + value.seconds) * 1000000 + value.microseconds\n\n\ndef validate_unknown_parameter_value(obj: typing.Any) -> str:\n    if obj not in _UNKNOWN_VALUES:\n        raise ValueError(\n            f\"Object {obj!r} is not a valid value for the 'unknown' parameter\"\n        )\n    return obj\n\n# Path: src/marshmallow/schema.py\n\"\"\"The :class:`Schema` class, including its metaclass and options (class Meta).\"\"\"\nfrom __future__ import annotations\n\nimport copy\nimport datetime as dt\nimport decimal\nimport inspect\nimport json\nimport typing\nimport uuid\nimport warnings\nfrom abc import ABCMeta\nfrom collections import OrderedDict, defaultdict\nfrom collections.abc import Mapping\nfrom functools import lru_cache\n\nfrom marshmallow import base, class_registry, types\nfrom marshmallow import fields as ma_fields\nfrom marshmallow.decorators import (\n    POST_DUMP,\n    POST_LOAD,\n    PRE_DUMP,\n    PRE_LOAD,\n    VALIDATES,\n    VALIDATES_SCHEMA,\n)\nfrom marshmallow.error_store import ErrorStore\nfrom marshmallow.exceptions import StringNotCollectionError, ValidationError\nfrom marshmallow.orderedset import OrderedSet\nfrom marshmallow.utils import (\n    EXCLUDE,\n    INCLUDE,\n    RAISE,\n    get_value,\n    is_collection,\n    is_instance_or_subclass,\n    missing,\n    set_value,\n    validate_unknown_parameter_value,\n)\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\n_T = typing.TypeVar(\"_T\")\n\n\ndef _get_fields(attrs):\n    \"\"\"Get fields from a class\n\n    :param attrs: Mapping of class attributes\n    \"\"\"\n    return [\n        (field_name, field_value)\n        for field_name, field_value in attrs.items()\n        if is_instance_or_subclass(field_value, base.FieldABC)\n    ]\n\n\n# This function allows Schemas to inherit from non-Schema classes and ensures\n#   inheritance according to the MRO\ndef _get_fields_by_mro(klass):\n    \"\"\"Collect fields from a class, following its method resolution order. The\n    class itself is excluded from the search; only its parents are checked. Get\n    fields from ``_declared_fields`` if available, else use ``__dict__``.\n\n    :param type klass: Class whose fields to retrieve\n    \"\"\"\n    mro = inspect.getmro(klass)\n    # Loop over mro in reverse to maintain correct order of fields\n    return sum(\n        (\n            _get_fields(\n                getattr(base, \"_declared_fields\", base.__dict__),\n            )\n            for base in mro[:0:-1]\n        ),\n        [],\n    )\n\n\nclass SchemaMeta(ABCMeta):\n    \"\"\"Metaclass for the Schema class. Binds the declared fields to\n    a ``_declared_fields`` attribute, which is a dictionary mapping attribute\n    names to field objects. Also sets the ``opts`` class attribute, which is\n    the Schema class's ``class Meta`` options.\n    \"\"\"\n\n    def __new__(mcs, name, bases, attrs):\n        meta = attrs.get(\"Meta\")\n        ordered = getattr(meta, \"ordered\", False)\n        if not ordered:\n            # Inherit 'ordered' option\n            # Warning: We loop through bases instead of MRO because we don't\n            # yet have access to the class object\n            # (i.e. can't call super before we have fields)\n            for base_ in bases:\n                if hasattr(base_, \"Meta\") and hasattr(base_.Meta, \"ordered\"):\n                    ordered = base_.Meta.ordered\n                    break\n            else:\n                ordered = False\n        cls_fields = _get_fields(attrs)\n        # Remove fields from list of class attributes to avoid shadowing\n        # Schema attributes/methods in case of name conflict\n        for field_name, _ in cls_fields:\n            del attrs[field_name]\n        klass = super().__new__(mcs, name, bases, attrs)\n        inherited_fields = _get_fields_by_mro(klass)\n\n        meta = klass.Meta\n        # Set klass.opts in __new__ rather than __init__ so that it is accessible in\n        # get_declared_fields\n        klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered)\n        # Add fields specified in the `include` class Meta option\n        cls_fields += list(klass.opts.include.items())\n\n        # Assign _declared_fields on class\n        klass._declared_fields = mcs.get_declared_fields(\n            klass=klass,\n            cls_fields=cls_fields,\n            inherited_fields=inherited_fields,\n            dict_cls=dict,\n        )\n        return klass\n\n    @classmethod\n    def get_declared_fields(\n        mcs,\n        klass: type,\n        cls_fields: list,\n        inherited_fields: list,\n        dict_cls: type = dict,\n    ):\n        \"\"\"Returns a dictionary of field_name => `Field` pairs declared on the class.\n        This is exposed mainly so that plugins can add additional fields, e.g. fields\n        computed from class Meta options.\n\n        :param klass: The class object.\n        :param cls_fields: The fields declared on the class, including those added\n            by the ``include`` class Meta option.\n        :param inherited_fields: Inherited fields.\n        :param dict_cls: dict-like class to use for dict output Default to ``dict``.\n        \"\"\"\n        return dict_cls(inherited_fields + cls_fields)\n\n    def __init__(cls, name, bases, attrs):\n        super().__init__(name, bases, attrs)\n        if name and cls.opts.register:\n            class_registry.register(name, cls)\n        cls._hooks = cls.resolve_hooks()\n\n    def resolve_hooks(cls) -> dict[types.Tag, list[str]]:\n        \"\"\"Add in the decorated processors\n\n        By doing this after constructing the class, we let standard inheritance\n        do all the hard work.\n        \"\"\"\n        mro = inspect.getmro(cls)\n\n        hooks = defaultdict(list)  # type: typing.Dict[types.Tag, typing.List[str]]\n\n        for attr_name in dir(cls):\n            # Need to look up the actual descriptor, not whatever might be\n            # bound to the class. This needs to come from the __dict__ of the\n            # declaring class.\n            for parent in mro:\n                try:\n                    attr = parent.__dict__[attr_name]\n                except KeyError:\n                    continue\n                else:\n                    break\n            else:\n                # In case we didn't find the attribute and didn't break above.\n                # We should never hit this - it's just here for completeness\n                # to exclude the possibility of attr being undefined.\n                continue\n\n            try:\n                hook_config = attr.__marshmallow_hook__\n            except AttributeError:\n                pass\n            else:\n                for key in hook_config.keys():\n                    # Use name here so we can get the bound method later, in\n                    # case the processor was a descriptor or something.\n                    hooks[key].append(attr_name)\n\n        return hooks\n\n\nclass SchemaOpts:\n    \"\"\"class Meta options for the :class:`Schema`. Defines defaults.\"\"\"\n\n    def __init__(self, meta, ordered: bool = False):\n        self.fields = getattr(meta, \"fields\", ())\n        if not isinstance(self.fields, (list, tuple)):\n            raise ValueError(\"`fields` option must be a list or tuple.\")\n        self.additional = getattr(meta, \"additional\", ())\n        if not isinstance(self.additional, (list, tuple)):\n            raise ValueError(\"`additional` option must be a list or tuple.\")\n        if self.fields and self.additional:\n            raise ValueError(\n                \"Cannot set both `fields` and `additional` options\"\n                \" for the same Schema.\"\n            )\n        self.exclude = getattr(meta, \"exclude\", ())\n        if not isinstance(self.exclude, (list, tuple)):\n            raise ValueError(\"`exclude` must be a list or tuple.\")\n        self.dateformat = getattr(meta, \"dateformat\", None)\n        self.datetimeformat = getattr(meta, \"datetimeformat\", None)\n        self.timeformat = getattr(meta, \"timeformat\", None)\n        if hasattr(meta, \"json_module\"):\n            warnings.warn(\n                \"The json_module class Meta option is deprecated. Use render_module instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            render_module = getattr(meta, \"json_module\", json)\n        else:\n            render_module = json\n        self.render_module = getattr(meta, \"render_module\", render_module)\n        self.ordered = getattr(meta, \"ordered\", ordered)\n        self.index_errors = getattr(meta, \"index_errors\", True)\n        self.include = getattr(meta, \"include\", {})\n        self.load_only = getattr(meta, \"load_only\", ())\n        self.dump_only = getattr(meta, \"dump_only\", ())\n        self.unknown = validate_unknown_parameter_value(getattr(meta, \"unknown\", RAISE))\n        self.register = getattr(meta, \"register\", True)\n\n\nclass Schema(base.SchemaABC, metaclass=SchemaMeta):\n    \"\"\"Base schema class with which to define custom schemas.\n\n    Example usage:\n\n    .. code-block:: python\n\n        import datetime as dt\n        from dataclasses import dataclass\n\n        from marshmallow import Schema, fields\n\n\n        @dataclass\n        class Album:\n            title: str\n            release_date: dt.date\n\n\n        class AlbumSchema(Schema):\n            title = fields.Str()\n            release_date = fields.Date()\n\n\n        album = Album(\"Beggars Banquet\", dt.date(1968, 12, 6))\n        schema = AlbumSchema()\n        data = schema.dump(album)\n        data  # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'}\n\n    :param only: Whitelist of the declared fields to select when\n        instantiating the Schema. If None, all fields are used. Nested fields\n        can be represented with dot delimiters.\n    :param exclude: Blacklist of the declared fields to exclude\n        when instantiating the Schema. If a field appears in both `only` and\n        `exclude`, it is not used. Nested fields can be represented with dot\n        delimiters.\n    :param many: Should be set to `True` if ``obj`` is a collection\n        so that the object will be serialized to a list.\n    :param context: Optional context passed to :class:`fields.Method` and\n        :class:`fields.Function` fields.\n    :param load_only: Fields to skip during serialization (write-only fields)\n    :param dump_only: Fields to skip during deserialization (read-only fields)\n    :param partial: Whether to ignore missing fields and not require\n        any fields declared. Propagates down to ``Nested`` fields as well. If\n        its value is an iterable, only missing fields listed in that iterable\n        will be ignored. Use dot delimiters to specify nested fields.\n    :param unknown: Whether to exclude, include, or raise an error for unknown\n        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n\n    .. versionchanged:: 3.0.0\n        `prefix` parameter removed.\n\n    .. versionchanged:: 2.0.0\n        `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of\n        `marshmallow.decorators.validates_schema`,\n        `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`.\n        `__accessor__` and `__error_handler__` are deprecated. Implement the\n        `handle_error` and `get_attribute` methods instead.\n    \"\"\"\n\n    TYPE_MAPPING = {\n        str: ma_fields.String,\n        bytes: ma_fields.String,\n        dt.datetime: ma_fields.DateTime,\n        float: ma_fields.Float,\n        bool: ma_fields.Boolean,\n        tuple: ma_fields.Raw,\n        list: ma_fields.Raw,\n        set: ma_fields.Raw,\n        int: ma_fields.Integer,\n        uuid.UUID: ma_fields.UUID,\n        dt.time: ma_fields.Time,\n        dt.date: ma_fields.Date,\n        dt.timedelta: ma_fields.TimeDelta,\n        decimal.Decimal: ma_fields.Decimal,\n    }  # type: typing.Dict[type, typing.Type[ma_fields.Field]]\n    #: Overrides for default schema-level error messages\n    error_messages = {}  # type: typing.Dict[str, str]\n\n    _default_error_messages = {\n        \"type\": \"Invalid input type.\",\n        \"unknown\": \"Unknown field.\",\n    }  # type: typing.Dict[str, str]\n\n    OPTIONS_CLASS = SchemaOpts  # type: type\n\n    set_class = OrderedSet\n\n    # These get set by SchemaMeta\n    opts = None  # type: SchemaOpts\n    _declared_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n    _hooks = {}  # type: typing.Dict[types.Tag, typing.List[str]]\n\n    class Meta:\n        \"\"\"Options object for a Schema.\n\n        Example usage: ::\n\n            class Meta:\n                fields = (\"id\", \"email\", \"date_created\")\n                exclude = (\"password\", \"secret_attribute\")\n\n        Available options:\n\n        - ``fields``: Tuple or list of fields to include in the serialized result.\n        - ``additional``: Tuple or list of fields to include *in addition* to the\n            explicitly declared fields. ``additional`` and ``fields`` are\n            mutually-exclusive options.\n        - ``include``: Dictionary of additional fields to include in the schema. It is\n            usually better to define fields as class variables, but you may need to\n            use this option, e.g., if your fields are Python keywords. May be an\n            `OrderedDict`.\n        - ``exclude``: Tuple or list of fields to exclude in the serialized result.\n            Nested fields can be represented with dot delimiters.\n        - ``dateformat``: Default format for `Date <fields.Date>` fields.\n        - ``datetimeformat``: Default format for `DateTime <fields.DateTime>` fields.\n        - ``timeformat``: Default format for `Time <fields.Time>` fields.\n        - ``render_module``: Module to use for `loads <Schema.loads>` and `dumps <Schema.dumps>`.\n            Defaults to `json` from the standard library.\n        - ``ordered``: If `True`, output of `Schema.dump` will be a `collections.OrderedDict`.\n        - ``index_errors``: If `True`, errors dictionaries will include the index\n            of invalid items in a collection.\n        - ``load_only``: Tuple or list of fields to exclude from serialized results.\n        - ``dump_only``: Tuple or list of fields to exclude from deserialization\n        - ``unknown``: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        - ``register``: Whether to register the `Schema` with marshmallow's internal\n            class registry. Must be `True` if you intend to refer to this `Schema`\n            by class name in `Nested` fields. Only set this to `False` when memory\n            usage is critical. Defaults to `True`.\n        \"\"\"\n\n    def __init__(\n        self,\n        *,\n        only: types.StrSequenceOrSet | None = None,\n        exclude: types.StrSequenceOrSet = (),\n        many: bool = False,\n        context: dict | None = None,\n        load_only: types.StrSequenceOrSet = (),\n        dump_only: types.StrSequenceOrSet = (),\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        # Raise error if only or exclude is passed as string, not list of strings\n        if only is not None and not is_collection(only):\n            raise StringNotCollectionError('\"only\" should be a list of strings')\n        if not is_collection(exclude):\n            raise StringNotCollectionError('\"exclude\" should be a list of strings')\n        # copy declared fields from metaclass\n        self.declared_fields = copy.deepcopy(self._declared_fields)\n        self.many = many\n        self.only = only\n        self.exclude: set[typing.Any] | typing.MutableSet[typing.Any] = set(\n            self.opts.exclude\n        ) | set(exclude)\n        self.ordered = self.opts.ordered\n        self.load_only = set(load_only) or set(self.opts.load_only)\n        self.dump_only = set(dump_only) or set(self.opts.dump_only)\n        self.partial = partial\n        self.unknown = (\n            self.opts.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        self.context = context or {}\n        self._normalize_nested_options()\n        #: Dictionary mapping field_names -> :class:`Field` objects\n        self.fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.load_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.dump_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self._init_fields()\n        messages = {}\n        messages.update(self._default_error_messages)\n        for cls in reversed(self.__class__.__mro__):\n            messages.update(getattr(cls, \"error_messages\", {}))\n        messages.update(self.error_messages or {})\n        self.error_messages = messages\n\n    def __repr__(self) -> str:\n        return f\"<{self.__class__.__name__}(many={self.many})>\"\n\n    @property\n    def dict_class(self) -> type:\n        return OrderedDict if self.ordered else dict\n\n    @classmethod\n    def from_dict(\n        cls,\n        fields: dict[str, ma_fields.Field | type],\n        *,\n        name: str = \"GeneratedSchema\",\n    ) -> type:\n        \"\"\"Generate a `Schema` class given a dictionary of fields.\n\n        .. code-block:: python\n\n            from marshmallow import Schema, fields\n\n            PersonSchema = Schema.from_dict({\"name\": fields.Str()})\n            print(PersonSchema().load({\"name\": \"David\"}))  # => {'name': 'David'}\n\n        Generated schemas are not added to the class registry and therefore cannot\n        be referred to by name in `Nested` fields.\n\n        :param dict fields: Dictionary mapping field names to field instances.\n        :param str name: Optional name for the class, which will appear in\n            the ``repr`` for the class.\n\n        .. versionadded:: 3.0.0\n        \"\"\"\n        attrs = fields.copy()\n        attrs[\"Meta\"] = type(\n            \"GeneratedMeta\", (getattr(cls, \"Meta\", object),), {\"register\": False}\n        )\n        schema_cls = type(name, (cls,), attrs)\n        return schema_cls\n\n    ##### Override-able methods #####\n\n    def handle_error(\n        self, error: ValidationError, data: typing.Any, *, many: bool, **kwargs\n    ):\n        \"\"\"Custom error handler function for the schema.\n\n        :param error: The `ValidationError` raised during (de)serialization.\n        :param data: The original input data.\n        :param many: Value of ``many`` on dump or load.\n        :param partial: Value of ``partial`` on load.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0rc9\n            Receives `many` and `partial` (on deserialization) as keyword arguments.\n        \"\"\"\n        pass\n\n    def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any):\n        \"\"\"Defines how to pull values from an object to serialize.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0a1\n            Changed position of ``obj`` and ``attr``.\n        \"\"\"\n        return get_value(obj, attr, default)\n\n    ##### Serialization/Deserialization API #####\n\n    @staticmethod\n    def _call_and_store(getter_func, data, *, field_name, error_store, index=None):\n        \"\"\"Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.\n\n        :param callable getter_func: Function for getting the serialized/deserialized\n            value from ``data``.\n        :param data: The data passed to ``getter_func``.\n        :param str field_name: Field name.\n        :param int index: Index of the item being validated, if validating a collection,\n            otherwise `None`.\n        \"\"\"\n        try:\n            value = getter_func(data)\n        except ValidationError as error:\n            error_store.store_error(error.messages, field_name, index=index)\n            # When a Nested field fails validation, the marshalled data is stored\n            # on the ValidationError's valid_data attribute\n            return error.valid_data or missing\n        return value\n\n    def _serialize(self, obj: _T | typing.Iterable[_T], *, many: bool = False):\n        \"\"\"Serialize ``obj``.\n\n        :param obj: The object(s) to serialize.\n        :param bool many: `True` if ``data`` should be serialized as a collection.\n        :return: A dictionary of the serialized data\n\n        .. versionchanged:: 1.0.0\n            Renamed from ``marshal``.\n        \"\"\"\n        if many and obj is not None:\n            return [\n                self._serialize(d, many=False)\n                for d in typing.cast(typing.Iterable[_T], obj)\n            ]\n        ret = self.dict_class()\n        for attr_name, field_obj in self.dump_fields.items():\n            value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)\n            if value is missing:\n                continue\n            key = field_obj.data_key if field_obj.data_key is not None else attr_name\n            ret[key] = value\n        return ret\n\n    def dump(self, obj: typing.Any, *, many: bool | None = None):\n        \"\"\"Serialize an object to native Python data types according to this\n        Schema's fields.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: Serialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        .. versionchanged:: 3.0.0rc9\n            Validation no longer occurs upon serialization.\n        \"\"\"\n        many = self.many if many is None else bool(many)\n        if self._has_processors(PRE_DUMP):\n            processed_obj = self._invoke_dump_processors(\n                PRE_DUMP, obj, many=many, original_data=obj\n            )\n        else:\n            processed_obj = obj\n\n        result = self._serialize(processed_obj, many=many)\n\n        if self._has_processors(POST_DUMP):\n            result = self._invoke_dump_processors(\n                POST_DUMP, result, many=many, original_data=obj\n            )\n\n        return result\n\n    def dumps(self, obj: typing.Any, *args, many: bool | None = None, **kwargs):\n        \"\"\"Same as :meth:`dump`, except return a JSON-encoded string.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: A ``json`` string\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        \"\"\"\n        serialized = self.dump(obj, many=many)\n        return self.opts.render_module.dumps(serialized, *args, **kwargs)\n\n    def _deserialize(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        error_store: ErrorStore,\n        many: bool = False,\n        partial=None,\n        unknown=RAISE,\n        index=None,\n    ) -> _T | list[_T]:\n        \"\"\"Deserialize ``data``.\n\n        :param dict data: The data to deserialize.\n        :param ErrorStore error_store: Structure to store errors.\n        :param bool many: `True` if ``data`` should be deserialized as a collection.\n        :param bool|tuple partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        :param int index: Index of the item being serialized (for storing errors) if\n            serializing a collection, otherwise `None`.\n        :return: A dictionary of the deserialized data.\n        \"\"\"\n        index_errors = self.opts.index_errors\n        index = index if index_errors else None\n        if many:\n            if not is_collection(data):\n                error_store.store_error([self.error_messages[\"type\"]], index=index)\n                ret_l = []  # type: typing.List[_T]\n            else:\n                ret_l = [\n                    typing.cast(\n                        _T,\n                        self._deserialize(\n                            typing.cast(typing.Mapping[str, typing.Any], d),\n                            error_store=error_store,\n                            many=False,\n                            partial=partial,\n                            unknown=unknown,\n                            index=idx,\n                        ),\n                    )\n                    for idx, d in enumerate(data)\n                ]\n            return ret_l\n        ret_d = self.dict_class()\n        # Check data is a dict\n        if not isinstance(data, Mapping):\n            error_store.store_error([self.error_messages[\"type\"]], index=index)\n        else:\n            partial_is_collection = is_collection(partial)\n            for attr_name, field_obj in self.load_fields.items():\n                field_name = (\n                    field_obj.data_key if field_obj.data_key is not None else attr_name\n                )\n                raw_value = data.get(field_name, missing)\n                if raw_value is missing:\n                    # Ignore missing field if we're allowed to.\n                    if partial is True or (\n                        partial_is_collection and attr_name in partial\n                    ):\n                        continue\n                d_kwargs = {}\n                # Allow partial loading of nested schemas.\n                if partial_is_collection:\n                    prefix = field_name + \".\"\n                    len_prefix = len(prefix)\n                    sub_partial = [\n                        f[len_prefix:] for f in partial if f.startswith(prefix)\n                    ]\n                    d_kwargs[\"partial\"] = sub_partial\n                elif partial is not None:\n                    d_kwargs[\"partial\"] = partial\n\n                def getter(\n                    val, field_obj=field_obj, field_name=field_name, d_kwargs=d_kwargs\n                ):\n                    return field_obj.deserialize(\n                        val,\n                        field_name,\n                        data,\n                        **d_kwargs,\n                    )\n\n                value = self._call_and_store(\n                    getter_func=getter,\n                    data=raw_value,\n                    field_name=field_name,\n                    error_store=error_store,\n                    index=index,\n                )\n                if value is not missing:\n                    key = field_obj.attribute or attr_name\n                    set_value(ret_d, key, value)\n            if unknown != EXCLUDE:\n                fields = {\n                    field_obj.data_key if field_obj.data_key is not None else field_name\n                    for field_name, field_obj in self.load_fields.items()\n                }\n                for key in set(data) - fields:\n                    value = data[key]\n                    if unknown == INCLUDE:\n                        ret_d[key] = value\n                    elif unknown == RAISE:\n                        error_store.store_error(\n                            [self.error_messages[\"unknown\"]],\n                            key,\n                            (index if index_errors else None),\n                        )\n        return ret_d\n\n    def load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        \"\"\"Deserialize a data structure to an object defined by this Schema's fields.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        return self._do_load(\n            data, many=many, partial=partial, unknown=unknown, postprocess=True\n        )\n\n    def loads(\n        self,\n        json_data: str,\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        **kwargs,\n    ):\n        \"\"\"Same as :meth:`load`, except it takes a JSON string as input.\n\n        :param json_data: A JSON string of the data to deserialize.\n        :param many: Whether to deserialize `obj` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n...\n", "needle_token_start": 2336, "needle_token_end": 2471, "code_context_ntokens": 16117, "output": ["```python\ndef store_error(self, messages, field_name=SCHEMA, index=None):\n    # field error  -> store/merge error messages under field name key\n    # schema error -> if string or list, store/merge under _schema key\n    #              -> if dict, store/merge with other top-level keys\n    if field_name != SCHEMA or not isinstance(messages, dict):\n        messages = {field_name: messages}\n    if index is not None:\n        messages = {index: messages}\n    self.errors = merge_errors(self.errors, messages)\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "dict_class", "language": "python", "path": "src/marshmallow/schema.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To determine the type of dictionary used in the serialization process based on the ordering preference.\n2. **Input**: No direct inputs; the decision is based on the internal state of the ordering flag.\n3. **Output**: Returns the class type of the dictionary to be used.\n4. **Procedure**: Checks if the ordering flag is set to true; if so, it returns an ordered dictionary type that maintains the order of insertion. Otherwise, it returns a standard dictionary type that does not guarantee order.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n    :param type klass: Class whose fields to retrieve\n    \"\"\"\n    mro = inspect.getmro(klass)\n    # Loop over mro in reverse to maintain correct order of fields\n    return sum(\n        (\n            _get_fields(\n                getattr(base, \"_declared_fields\", base.__dict__),\n            )\n            for base in mro[:0:-1]\n        ),\n        [],\n    )\n\n\nclass SchemaMeta(ABCMeta):\n    \"\"\"Metaclass for the Schema class. Binds the declared fields to\n    a ``_declared_fields`` attribute, which is a dictionary mapping attribute\n    names to field objects. Also sets the ``opts`` class attribute, which is\n    the Schema class's ``class Meta`` options.\n    \"\"\"\n\n    def __new__(mcs, name, bases, attrs):\n        meta = attrs.get(\"Meta\")\n        ordered = getattr(meta, \"ordered\", False)\n        if not ordered:\n            # Inherit 'ordered' option\n            # Warning: We loop through bases instead of MRO because we don't\n            # yet have access to the class object\n            # (i.e. can't call super before we have fields)\n            for base_ in bases:\n                if hasattr(base_, \"Meta\") and hasattr(base_.Meta, \"ordered\"):\n                    ordered = base_.Meta.ordered\n                    break\n            else:\n                ordered = False\n        cls_fields = _get_fields(attrs)\n        # Remove fields from list of class attributes to avoid shadowing\n        # Schema attributes/methods in case of name conflict\n        for field_name, _ in cls_fields:\n            del attrs[field_name]\n        klass = super().__new__(mcs, name, bases, attrs)\n        inherited_fields = _get_fields_by_mro(klass)\n\n        meta = klass.Meta\n        # Set klass.opts in __new__ rather than __init__ so that it is accessible in\n        # get_declared_fields\n        klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered)\n        # Add fields specified in the `include` class Meta option\n        cls_fields += list(klass.opts.include.items())\n\n        # Assign _declared_fields on class\n        klass._declared_fields = mcs.get_declared_fields(\n            klass=klass,\n            cls_fields=cls_fields,\n            inherited_fields=inherited_fields,\n            dict_cls=dict,\n        )\n        return klass\n\n    @classmethod\n    def get_declared_fields(\n        mcs,\n        klass: type,\n        cls_fields: list,\n        inherited_fields: list,\n        dict_cls: type = dict,\n    ):\n        \"\"\"Returns a dictionary of field_name => `Field` pairs declared on the class.\n        This is exposed mainly so that plugins can add additional fields, e.g. fields\n        computed from class Meta options.\n\n        :param klass: The class object.\n        :param cls_fields: The fields declared on the class, including those added\n            by the ``include`` class Meta option.\n        :param inherited_fields: Inherited fields.\n        :param dict_cls: dict-like class to use for dict output Default to ``dict``.\n        \"\"\"\n        return dict_cls(inherited_fields + cls_fields)\n\n    def __init__(cls, name, bases, attrs):\n        super().__init__(name, bases, attrs)\n        if name and cls.opts.register:\n            class_registry.register(name, cls)\n        cls._hooks = cls.resolve_hooks()\n\n    def resolve_hooks(cls) -> dict[types.Tag, list[str]]:\n        \"\"\"Add in the decorated processors\n\n        By doing this after constructing the class, we let standard inheritance\n        do all the hard work.\n        \"\"\"\n        mro = inspect.getmro(cls)\n\n        hooks = defaultdict(list)  # type: typing.Dict[types.Tag, typing.List[str]]\n\n        for attr_name in dir(cls):\n            # Need to look up the actual descriptor, not whatever might be\n            # bound to the class. This needs to come from the __dict__ of the\n            # declaring class.\n            for parent in mro:\n                try:\n                    attr = parent.__dict__[attr_name]\n                except KeyError:\n                    continue\n                else:\n                    break\n            else:\n                # In case we didn't find the attribute and didn't break above.\n                # We should never hit this - it's just here for completeness\n                # to exclude the possibility of attr being undefined.\n                continue\n\n            try:\n                hook_config = attr.__marshmallow_hook__\n            except AttributeError:\n                pass\n            else:\n                for key in hook_config.keys():\n                    # Use name here so we can get the bound method later, in\n                    # case the processor was a descriptor or something.\n                    hooks[key].append(attr_name)\n\n        return hooks\n\n\nclass SchemaOpts:\n    \"\"\"class Meta options for the :class:`Schema`. Defines defaults.\"\"\"\n\n    def __init__(self, meta, ordered: bool = False):\n        self.fields = getattr(meta, \"fields\", ())\n        if not isinstance(self.fields, (list, tuple)):\n            raise ValueError(\"`fields` option must be a list or tuple.\")\n        self.additional = getattr(meta, \"additional\", ())\n        if not isinstance(self.additional, (list, tuple)):\n            raise ValueError(\"`additional` option must be a list or tuple.\")\n        if self.fields and self.additional:\n            raise ValueError(\n                \"Cannot set both `fields` and `additional` options\"\n                \" for the same Schema.\"\n            )\n        self.exclude = getattr(meta, \"exclude\", ())\n        if not isinstance(self.exclude, (list, tuple)):\n            raise ValueError(\"`exclude` must be a list or tuple.\")\n        self.dateformat = getattr(meta, \"dateformat\", None)\n        self.datetimeformat = getattr(meta, \"datetimeformat\", None)\n        self.timeformat = getattr(meta, \"timeformat\", None)\n        if hasattr(meta, \"json_module\"):\n            warnings.warn(\n                \"The json_module class Meta option is deprecated. Use render_module instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            render_module = getattr(meta, \"json_module\", json)\n        else:\n            render_module = json\n        self.render_module = getattr(meta, \"render_module\", render_module)\n        self.ordered = getattr(meta, \"ordered\", ordered)\n        self.index_errors = getattr(meta, \"index_errors\", True)\n        self.include = getattr(meta, \"include\", {})\n        self.load_only = getattr(meta, \"load_only\", ())\n        self.dump_only = getattr(meta, \"dump_only\", ())\n        self.unknown = validate_unknown_parameter_value(getattr(meta, \"unknown\", RAISE))\n        self.register = getattr(meta, \"register\", True)\n\n\nclass Schema(base.SchemaABC, metaclass=SchemaMeta):\n    \"\"\"Base schema class with which to define custom schemas.\n\n    Example usage:\n\n    .. code-block:: python\n\n        import datetime as dt\n        from dataclasses import dataclass\n\n        from marshmallow import Schema, fields\n\n\n        @dataclass\n        class Album:\n            title: str\n            release_date: dt.date\n\n\n        class AlbumSchema(Schema):\n            title = fields.Str()\n            release_date = fields.Date()\n\n\n        album = Album(\"Beggars Banquet\", dt.date(1968, 12, 6))\n        schema = AlbumSchema()\n        data = schema.dump(album)\n        data  # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'}\n\n    :param only: Whitelist of the declared fields to select when\n        instantiating the Schema. If None, all fields are used. Nested fields\n        can be represented with dot delimiters.\n    :param exclude: Blacklist of the declared fields to exclude\n        when instantiating the Schema. If a field appears in both `only` and\n        `exclude`, it is not used. Nested fields can be represented with dot\n        delimiters.\n    :param many: Should be set to `True` if ``obj`` is a collection\n        so that the object will be serialized to a list.\n    :param context: Optional context passed to :class:`fields.Method` and\n        :class:`fields.Function` fields.\n    :param load_only: Fields to skip during serialization (write-only fields)\n    :param dump_only: Fields to skip during deserialization (read-only fields)\n    :param partial: Whether to ignore missing fields and not require\n        any fields declared. Propagates down to ``Nested`` fields as well. If\n        its value is an iterable, only missing fields listed in that iterable\n        will be ignored. Use dot delimiters to specify nested fields.\n    :param unknown: Whether to exclude, include, or raise an error for unknown\n        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n\n    .. versionchanged:: 3.0.0\n        `prefix` parameter removed.\n\n    .. versionchanged:: 2.0.0\n        `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of\n        `marshmallow.decorators.validates_schema`,\n        `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`.\n        `__accessor__` and `__error_handler__` are deprecated. Implement the\n        `handle_error` and `get_attribute` methods instead.\n    \"\"\"\n\n    TYPE_MAPPING = {\n        str: ma_fields.String,\n        bytes: ma_fields.String,\n        dt.datetime: ma_fields.DateTime,\n        float: ma_fields.Float,\n        bool: ma_fields.Boolean,\n        tuple: ma_fields.Raw,\n        list: ma_fields.Raw,\n        set: ma_fields.Raw,\n        int: ma_fields.Integer,\n        uuid.UUID: ma_fields.UUID,\n        dt.time: ma_fields.Time,\n        dt.date: ma_fields.Date,\n        dt.timedelta: ma_fields.TimeDelta,\n        decimal.Decimal: ma_fields.Decimal,\n    }  # type: typing.Dict[type, typing.Type[ma_fields.Field]]\n    #: Overrides for default schema-level error messages\n    error_messages = {}  # type: typing.Dict[str, str]\n\n    _default_error_messages = {\n        \"type\": \"Invalid input type.\",\n        \"unknown\": \"Unknown field.\",\n    }  # type: typing.Dict[str, str]\n\n    OPTIONS_CLASS = SchemaOpts  # type: type\n\n    set_class = OrderedSet\n\n    # These get set by SchemaMeta\n    opts = None  # type: SchemaOpts\n    _declared_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n    _hooks = {}  # type: typing.Dict[types.Tag, typing.List[str]]\n\n    class Meta:\n        \"\"\"Options object for a Schema.\n\n        Example usage: ::\n\n            class Meta:\n                fields = (\"id\", \"email\", \"date_created\")\n                exclude = (\"password\", \"secret_attribute\")\n\n        Available options:\n\n        - ``fields``: Tuple or list of fields to include in the serialized result.\n        - ``additional``: Tuple or list of fields to include *in addition* to the\n            explicitly declared fields. ``additional`` and ``fields`` are\n            mutually-exclusive options.\n        - ``include``: Dictionary of additional fields to include in the schema. It is\n            usually better to define fields as class variables, but you may need to\n            use this option, e.g., if your fields are Python keywords. May be an\n            `OrderedDict`.\n        - ``exclude``: Tuple or list of fields to exclude in the serialized result.\n            Nested fields can be represented with dot delimiters.\n        - ``dateformat``: Default format for `Date <fields.Date>` fields.\n        - ``datetimeformat``: Default format for `DateTime <fields.DateTime>` fields.\n        - ``timeformat``: Default format for `Time <fields.Time>` fields.\n        - ``render_module``: Module to use for `loads <Schema.loads>` and `dumps <Schema.dumps>`.\n            Defaults to `json` from the standard library.\n        - ``ordered``: If `True`, output of `Schema.dump` will be a `collections.OrderedDict`.\n        - ``index_errors``: If `True`, errors dictionaries will include the index\n            of invalid items in a collection.\n        - ``load_only``: Tuple or list of fields to exclude from serialized results.\n        - ``dump_only``: Tuple or list of fields to exclude from deserialization\n        - ``unknown``: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        - ``register``: Whether to register the `Schema` with marshmallow's internal\n            class registry. Must be `True` if you intend to refer to this `Schema`\n            by class name in `Nested` fields. Only set this to `False` when memory\n            usage is critical. Defaults to `True`.\n        \"\"\"\n\n    def __init__(\n        self,\n        *,\n        only: types.StrSequenceOrSet | None = None,\n        exclude: types.StrSequenceOrSet = (),\n        many: bool = False,\n        context: dict | None = None,\n        load_only: types.StrSequenceOrSet = (),\n        dump_only: types.StrSequenceOrSet = (),\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        # Raise error if only or exclude is passed as string, not list of strings\n        if only is not None and not is_collection(only):\n            raise StringNotCollectionError('\"only\" should be a list of strings')\n        if not is_collection(exclude):\n            raise StringNotCollectionError('\"exclude\" should be a list of strings')\n        # copy declared fields from metaclass\n        self.declared_fields = copy.deepcopy(self._declared_fields)\n        self.many = many\n        self.only = only\n        self.exclude: set[typing.Any] | typing.MutableSet[typing.Any] = set(\n            self.opts.exclude\n        ) | set(exclude)\n        self.ordered = self.opts.ordered\n        self.load_only = set(load_only) or set(self.opts.load_only)\n        self.dump_only = set(dump_only) or set(self.opts.dump_only)\n        self.partial = partial\n        self.unknown = (\n            self.opts.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        self.context = context or {}\n        self._normalize_nested_options()\n        #: Dictionary mapping field_names -> :class:`Field` objects\n        self.fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.load_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.dump_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self._init_fields()\n        messages = {}\n        messages.update(self._default_error_messages)\n        for cls in reversed(self.__class__.__mro__):\n            messages.update(getattr(cls, \"error_messages\", {}))\n        messages.update(self.error_messages or {})\n        self.error_messages = messages\n\n    def __repr__(self) -> str:\n        return f\"<{self.__class__.__name__}(many={self.many})>\"\n\n    @property\n    \ndef dict_class(self) -> type:\n        return OrderedDict if self.ordered else dict\n\n    @classmethod\n    def from_dict(\n        cls,\n        fields: dict[str, ma_fields.Field | type],\n        *,\n        name: str = \"GeneratedSchema\",\n    ) -> type:\n        \"\"\"Generate a `Schema` class given a dictionary of fields.\n\n        .. code-block:: python\n\n            from marshmallow import Schema, fields\n\n            PersonSchema = Schema.from_dict({\"name\": fields.Str()})\n            print(PersonSchema().load({\"name\": \"David\"}))  # => {'name': 'David'}\n\n        Generated schemas are not added to the class registry and therefore cannot\n        be referred to by name in `Nested` fields.\n\n        :param dict fields: Dictionary mapping field names to field instances.\n        :param str name: Optional name for the class, which will appear in\n            the ``repr`` for the class.\n\n        .. versionadded:: 3.0.0\n        \"\"\"\n        attrs = fields.copy()\n        attrs[\"Meta\"] = type(\n            \"GeneratedMeta\", (getattr(cls, \"Meta\", object),), {\"register\": False}\n        )\n        schema_cls = type(name, (cls,), attrs)\n        return schema_cls\n\n    ##### Override-able methods #####\n\n    def handle_error(\n        self, error: ValidationError, data: typing.Any, *, many: bool, **kwargs\n    ):\n        \"\"\"Custom error handler function for the schema.\n\n        :param error: The `ValidationError` raised during (de)serialization.\n        :param data: The original input data.\n        :param many: Value of ``many`` on dump or load.\n        :param partial: Value of ``partial`` on load.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0rc9\n            Receives `many` and `partial` (on deserialization) as keyword arguments.\n        \"\"\"\n        pass\n\n    def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any):\n        \"\"\"Defines how to pull values from an object to serialize.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0a1\n            Changed position of ``obj`` and ``attr``.\n        \"\"\"\n        return get_value(obj, attr, default)\n\n    ##### Serialization/Deserialization API #####\n\n    @staticmethod\n    def _call_and_store(getter_func, data, *, field_name, error_store, index=None):\n        \"\"\"Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.\n\n        :param callable getter_func: Function for getting the serialized/deserialized\n            value from ``data``.\n        :param data: The data passed to ``getter_func``.\n        :param str field_name: Field name.\n        :param int index: Index of the item being validated, if validating a collection,\n            otherwise `None`.\n        \"\"\"\n        try:\n            value = getter_func(data)\n        except ValidationError as error:\n            error_store.store_error(error.messages, field_name, index=index)\n            # When a Nested field fails validation, the marshalled data is stored\n            # on the ValidationError's valid_data attribute\n            return error.valid_data or missing\n        return value\n\n    def _serialize(self, obj: _T | typing.Iterable[_T], *, many: bool = False):\n        \"\"\"Serialize ``obj``.\n\n        :param obj: The object(s) to serialize.\n        :param bool many: `True` if ``data`` should be serialized as a collection.\n        :return: A dictionary of the serialized data\n\n        .. versionchanged:: 1.0.0\n            Renamed from ``marshal``.\n        \"\"\"\n        if many and obj is not None:\n            return [\n                self._serialize(d, many=False)\n                for d in typing.cast(typing.Iterable[_T], obj)\n            ]\n        ret = self.dict_class()\n        for attr_name, field_obj in self.dump_fields.items():\n            value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)\n            if value is missing:\n                continue\n            key = field_obj.data_key if field_obj.data_key is not None else attr_name\n            ret[key] = value\n        return ret\n\n    def dump(self, obj: typing.Any, *, many: bool | None = None):\n        \"\"\"Serialize an object to native Python data types according to this\n        Schema's fields.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: Serialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        .. versionchanged:: 3.0.0rc9\n            Validation no longer occurs upon serialization.\n        \"\"\"\n        many = self.many if many is None else bool(many)\n        if self._has_processors(PRE_DUMP):\n            processed_obj = self._invoke_dump_processors(\n                PRE_DUMP, obj, many=many, original_data=obj\n            )\n        else:\n            processed_obj = obj\n\n        result = self._serialize(processed_obj, many=many)\n\n        if self._has_processors(POST_DUMP):\n            result = self._invoke_dump_processors(\n                POST_DUMP, result, many=many, original_data=obj\n            )\n\n        return result\n\n    def dumps(self, obj: typing.Any, *args, many: bool | None = None, **kwargs):\n        \"\"\"Same as :meth:`dump`, except return a JSON-encoded string.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: A ``json`` string\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        \"\"\"\n        serialized = self.dump(obj, many=many)\n        return self.opts.render_module.dumps(serialized, *args, **kwargs)\n\n    def _deserialize(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        error_store: ErrorStore,\n        many: bool = False,\n        partial=None,\n        unknown=RAISE,\n        index=None,\n    ) -> _T | list[_T]:\n        \"\"\"Deserialize ``data``.\n\n        :param dict data: The data to deserialize.\n        :param ErrorStore error_store: Structure to store errors.\n        :param bool many: `True` if ``data`` should be deserialized as a collection.\n        :param bool|tuple partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        :param int index: Index of the item being serialized (for storing errors) if\n            serializing a collection, otherwise `None`.\n        :return: A dictionary of the deserialized data.\n        \"\"\"\n        index_errors = self.opts.index_errors\n        index = index if index_errors else None\n        if many:\n            if not is_collection(data):\n                error_store.store_error([self.error_messages[\"type\"]], index=index)\n                ret_l = []  # type: typing.List[_T]\n            else:\n                ret_l = [\n                    typing.cast(\n                        _T,\n                        self._deserialize(\n                            typing.cast(typing.Mapping[str, typing.Any], d),\n                            error_store=error_store,\n                            many=False,\n                            partial=partial,\n                            unknown=unknown,\n                            index=idx,\n                        ),\n                    )\n                    for idx, d in enumerate(data)\n                ]\n            return ret_l\n        ret_d = self.dict_class()\n        # Check data is a dict\n        if not isinstance(data, Mapping):\n            error_store.store_error([self.error_messages[\"type\"]], index=index)\n        else:\n            partial_is_collection = is_collection(partial)\n            for attr_name, field_obj in self.load_fields.items():\n                field_name = (\n                    field_obj.data_key if field_obj.data_key is not None else attr_name\n                )\n                raw_value = data.get(field_name, missing)\n                if raw_value is missing:\n                    # Ignore missing field if we're allowed to.\n                    if partial is True or (\n                        partial_is_collection and attr_name in partial\n                    ):\n                        continue\n                d_kwargs = {}\n                # Allow partial loading of nested schemas.\n                if partial_is_collection:\n                    prefix = field_name + \".\"\n                    len_prefix = len(prefix)\n                    sub_partial = [\n                        f[len_prefix:] for f in partial if f.startswith(prefix)\n                    ]\n                    d_kwargs[\"partial\"] = sub_partial\n                elif partial is not None:\n                    d_kwargs[\"partial\"] = partial\n\n                def getter(\n                    val, field_obj=field_obj, field_name=field_name, d_kwargs=d_kwargs\n                ):\n                    return field_obj.deserialize(\n                        val,\n                        field_name,\n                        data,\n                        **d_kwargs,\n                    )\n\n                value = self._call_and_store(\n                    getter_func=getter,\n                    data=raw_value,\n                    field_name=field_name,\n                    error_store=error_store,\n                    index=index,\n                )\n                if value is not missing:\n                    key = field_obj.attribute or attr_name\n                    set_value(ret_d, key, value)\n            if unknown != EXCLUDE:\n                fields = {\n                    field_obj.data_key if field_obj.data_key is not None else field_name\n                    for field_name, field_obj in self.load_fields.items()\n                }\n                for key in set(data) - fields:\n                    value = data[key]\n                    if unknown == INCLUDE:\n                        ret_d[key] = value\n                    elif unknown == RAISE:\n                        error_store.store_error(\n                            [self.error_messages[\"unknown\"]],\n                            key,\n                            (index if index_errors else None),\n                        )\n        return ret_d\n\n    def load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        \"\"\"Deserialize a data structure to an object defined by this Schema's fields.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        return self._do_load(\n            data, many=many, partial=partial, unknown=unknown, postprocess=True\n        )\n\n    def loads(\n        self,\n        json_data: str,\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        **kwargs,\n    ):\n        \"\"\"Same as :meth:`load`, except it takes a JSON string as input.\n\n        :param json_data: A JSON string of the data to deserialize.\n        :param many: Whether to deserialize `obj` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        data = self.opts.render_module.loads(json_data, **kwargs)\n        return self.load(data, many=many, partial=partial, unknown=unknown)\n\n    def _run_validator(\n        self,\n        validator_func,\n        output,\n        *,\n        original_data,\n        error_store,\n        many,\n        partial,\n        pass_original,\n        index=None,\n    ):\n        try:\n            if pass_original:  # Pass original, raw data (before unmarshalling)\n                validator_func(output, original_data, partial=partial, many=many)\n            else:\n                validator_func(output, partial=partial, many=many)\n        except ValidationError as err:\n            error_store.store_error(err.messages, err.field_name, index=index)\n\n    def validate(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n    ) -> dict[str, list[str]]:\n        \"\"\"Validate `data` against the schema, returning a dictionary of\n        validation errors.\n\n        :param data: The data to validate.\n        :param many: Whether to validate `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :return: A dictionary of validation errors.\n\n        .. versionadded:: 1.1.0\n        \"\"\"\n        try:\n            self._do_load(data, many=many, partial=partial, postprocess=False)\n        except ValidationError as exc:\n            return typing.cast(typing.Dict[str, typing.List[str]], exc.messages)\n        return {}\n\n    ##### Private Helpers #####\n\n    def _do_load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        postprocess: bool = True,\n    ):\n        \"\"\"Deserialize `data`, returning the deserialized result.\n        This method is private API.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to validate required fields. If its\n            value is an iterable, only fields listed in that iterable will be\n            ignored will be allowed missing. If `True`, all fields will be allowed missing.\n            If `None`, the value for `self.partial` is used.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :param postprocess: Whether to run post_load methods..\n        :return: Deserialized data\n        \"\"\"\n        error_store = ErrorStore()\n        errors = {}  # type: dict[str, list[str]]\n        many = self.many if many is None else bool(many)\n        unknown = (\n            self.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        if partial is None:\n            partial = self.partial\n        # Run preprocessors\n        if self._has_processors(PRE_LOAD):\n            try:\n                processed_data = self._invoke_load_processors(\n                    PRE_LOAD, data, many=many, original_data=data, partial=partial\n                )\n            except ValidationError as err:\n                errors = err.normalized_messages()\n                result = None  # type: list | dict | None\n        else:\n            processed_data = data\n        if not errors:\n            # Deserialize data\n            result = self._deserialize(\n                processed_data,\n                error_store=error_store,\n                many=many,\n                partial=partial,\n                unknown=unknown,\n            )\n            # Run field-level validation\n            self._invoke_field_validators(\n                error_store=error_store, data=result, many=many\n            )\n            # Run schema-level validation\n            if self._has_processors(VALIDATES_SCHEMA):\n                field_errors = bool(error_store.errors)\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=True,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=False,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n            errors = error_store.errors\n            # Run post processors\n            if not errors and postprocess and self._has_processors(POST_LOAD):\n                try:\n                    result = self._invoke_load_processors(\n                        POST_LOAD,\n                        result,\n                        many=many,\n                        original_data=data,\n                        partial=partial,\n                    )\n                except ValidationError as err:\n                    errors = err.normalized_messages()\n        if errors:\n            exc = ValidationError(errors, data=data, valid_data=result)\n            self.handle_error(exc, data, many=many, partial=partial)\n            raise exc\n\n        return result\n\n    def _normalize_nested_options(self) -> None:\n        \"\"\"Apply then flatten nested schema options.\n        This method is private API.\n        \"\"\"\n        if self.only is not None:\n            # Apply the only option to nested fields.\n            self.__apply_nested_option(\"only\", self.only, \"intersection\")\n            # Remove the child field names from the only option.\n            self.only = self.set_class([field.split(\".\", 1)[0] for field in self.only])\n        if self.exclude:\n            # Apply the exclude option to nested fields.\n            self.__apply_nested_option(\"exclude\", self.exclude, \"union\")\n            # Remove the parent field names from the exclude option.\n            self.exclude = self.set_class(\n                [field for field in self.exclude if \".\" not in field]\n            )\n\n    def __apply_nested_option(self, option_name, field_names, set_operation) -> None:\n        \"\"\"Apply nested options to nested fields\"\"\"\n        # Split nested field names on the first dot.\n        nested_fields = [name.split(\".\", 1) for name in field_names if \".\" in name]\n        # Partition the nested field names by parent field.\n        nested_options = defaultdict(list)  # type: defaultdict\n        for parent, nested_names in nested_fields:\n            nested_options[parent].append(nested_names)\n        # Apply the nested field options.\n        for key, options in iter(nested_options.items()):\n            new_options = self.set_class(options)\n            original_options = getattr(self.declared_fields[key], option_name, ())\n            if original_options:\n                if set_operation == \"union\":\n                    new_options |= self.set_class(original_options)\n                if set_operation == \"intersection\":\n                    new_options &= self.set_class(original_options)\n            setattr(self.declared_fields[key], option_name, new_options)\n\n    def _init_fields(self) -> None:\n        \"\"\"Update self.fields, self.load_fields, and self.dump_fields based on schema options.\n        This method is private API.\n        \"\"\"\n        if self.opts.fields:\n            available_field_names = self.set_class(self.opts.fields)\n        else:\n            available_field_names = self.set_class(self.declared_fields.keys())\n            if self.opts.additional:\n                available_field_names |= self.set_class(self.opts.additional)\n\n        invalid_fields = self.set_class()\n\n        if self.only is not None:\n            # Return only fields specified in only option\n            field_names: typing.AbstractSet[typing.Any] = self.set_class(self.only)\n\n            invalid_fields |= field_names - available_field_names\n        else:\n            field_names = available_field_names\n\n        # If \"exclude\" option or param is specified, remove those fields.\n        if self.exclude:\n            # Note that this isn't available_field_names, since we want to\n            # apply \"only\" for the actual calculation.\n            field_names = field_names - self.exclude\n            invalid_fields |= self.exclude - available_field_names\n\n        if invalid_fields:\n            message = f\"Invalid fields for {self}: {invalid_fields}.\"\n            raise ValueError(message)\n\n        fields_dict = self.dict_class()\n        for field_name in field_names:\n            field_obj = self.declared_fields.get(field_name, ma_fields.Inferred())\n            self._bind_field(field_name, field_obj)\n            fields_dict[field_name] = field_obj\n\n        load_fields, dump_fields = self.dict_class(), self.dict_class()\n        for field_name, field_obj in fields_dict.items():\n            if not field_obj.dump_only:\n                load_fields[field_name] = field_obj\n            if not field_obj.load_only:\n                dump_fields[field_name] = field_obj\n\n        dump_data_keys = [\n            field_obj.data_key if field_obj.data_key is not None else name\n            for name, field_obj in dump_fields.items()\n        ]\n        if len(dump_data_keys) != len(set(dump_data_keys)):\n            data_keys_duplicates = {\n                x for x in dump_data_keys if dump_data_keys.count(x) > 1\n            }\n            raise ValueError(\n                \"The data_key argument for one or more fields collides \"\n                \"with another field's name or data_key argument. \"\n                \"Check the following field names and \"\n                f\"data_key arguments: {list(data_keys_duplicates)}\"\n            )\n        load_attributes = [obj.attribute or name for name, obj in load_fields.items()]\n        if len(load_attributes) != len(set(load_attributes)):\n            attributes_duplicates = {\n                x for x in load_attributes if load_attributes.count(x) > 1\n            }\n            raise ValueError(\n                \"The attribute argument for one or more fields collides \"\n                \"with another field's name or attribute argument. \"\n                \"Check the following field names and \"\n                f\"attribute arguments: {list(attributes_duplicates)}\"\n            )\n\n        self.fields = fields_dict\n        self.dump_fields = dump_fields\n        self.load_fields = load_fields\n\n    def on_bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None:\n        \"\"\"Hook to modify a field when it is bound to the `Schema`.\n\n        No-op by default.\n        \"\"\"\n        return None\n\n    def _bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None:\n        \"\"\"Bind field to the schema, setting any necessary attributes on the\n        field (e.g. parent and name).\n\n        Also set field load_only and dump_only values if field_name was\n        specified in ``class Meta``.\n        \"\"\"\n        if field_name in self.load_only:\n            field_obj.load_only = True\n        if field_name in self.dump_only:\n            field_obj.dump_only = True\n        try:\n            field_obj._bind_to_schema(field_name, self)\n        except TypeError as error:\n            # Field declared as a class, not an instance. Ignore type checking because\n            # we handle unsupported arg types, i.e. this is dead code from\n            # the type checker's perspective.\n            if isinstance(field_obj, type) and issubclass(field_obj, base.FieldABC):\n                msg = (\n                    f'Field for \"{field_name}\" must be declared as a '\n                    \"Field instance, not a class. \"\n                    f'Did you mean \"fields.{field_obj.__name__}()\"?'  # type: ignore\n                )\n                raise TypeError(msg) from error\n            raise error\n        self.on_bind_field(field_name, field_obj)\n\n    @lru_cache(maxsize=8)  # noqa (https://github.com/PyCQA/flake8-bugbear/issues/310)\n    def _has_processors(self, tag) -> bool:\n        return bool(self._hooks[(tag, True)] or self._hooks[(tag, False)])\n\n    def _invoke_dump_processors(\n        self, tag: str, data, *, many: bool, original_data=None\n    ):\n        # The pass_many post-dump processors may do things like add an envelope, so\n        # invoke those after invoking the non-pass_many processors which will expect\n        # to get a list of items.\n        data = self._invoke_processors(\n            tag, pass_many=False, data=data, many=many, original_data=original_data\n        )\n        data = self._invoke_processors(\n            tag, pass_many=True, data=data, many=many, original_data=original_data\n        )\n        return data\n\n    def _invoke_load_processors(\n        self,\n        tag: str,\n        data,\n        *,\n        many: bool,\n        original_data,\n        partial: bool | types.StrSequenceOrSet | None,\n    ):\n        # This has to invert the order of the dump processors, so run the pass_many\n        # processors first.\n        data = self._invoke_processors(\n            tag,\n            pass_many=True,\n            data=data,\n            many=many,\n            original_data=original_data,\n            partial=partial,\n        )\n        data = self._invoke_processors(\n            tag,\n            pass_many=False,\n            data=data,\n            many=many,\n            original_data=original_data,\n            partial=partial,\n        )\n        return data\n\n    def _invoke_field_validators(self, *, error_store: ErrorStore, data, many: bool):\n        for attr_name in self._hooks[VALIDATES]:\n            validator = getattr(self, attr_name)\n            validator_kwargs = validator.__marshmallow_hook__[VALIDATES]\n            field_name = validator_kwargs[\"field_name\"]\n\n            try:\n                field_obj = self.fields[field_name]\n            except KeyError as error:\n                if field_name in self.declared_fields:\n                    continue\n                raise ValueError(f'\"{field_name}\" field does not exist.') from error\n\n            data_key = (\n                field_obj.data_key if field_obj.data_key is not None else field_name\n            )\n            if many:\n                for idx, item in enumerate(data):\n                    try:\n                        value = item[field_obj.attribute or field_name]\n                    except KeyError:\n                        pass\n                    else:\n                        validated_value = self._call_and_store(\n                            getter_func=validator,\n                            data=value,\n                            field_name=data_key,\n                            error_store=error_store,\n                            index=(idx if self.opts.index_errors else None),\n                        )\n                        if validated_value is missing:\n                            data[idx].pop(field_name, None)\n            else:\n                try:\n                    value = data[field_obj.attribute or field_name]\n                except KeyError:\n                    pass\n                else:\n                    validated_value = self._call_and_store(\n                        getter_func=validator,\n                        data=value,\n                        field_name=data_key,\n                        error_store=error_store,\n                    )\n                    if validated_value is missing:\n                        data.pop(field_name, None)\n\n    def _invoke_schema_validators(\n        self,\n        *,\n        error_store: ErrorStore,\n        pass_many: bool,\n        data,\n        original_data,\n        many: bool,\n        partial: bool | types.StrSequenceOrSet | None,\n        field_errors: bool = False,\n    ):\n        for attr_name in self._hooks[(VALIDATES_SCHEMA, pass_many)]:\n            validator = getattr(self, attr_name)\n            validator_kwargs = validator.__marshmallow_hook__[\n                (VALIDATES_SCHEMA, pass_many)\n            ]\n            if field_errors and validator_kwargs[\"skip_on_field_errors\"]:\n                continue\n            pass_original = validator_kwargs.get(\"pass_original\", False)\n\n            if many and not pass_many:\n                for idx, (item, orig) in enumerate(zip(data, original_data)):\n                    self._run_validator(\n                        validator,\n                        item,\n                        original_data=orig,\n                        error_store=error_store,\n                        many=many,\n                        partial=partial,\n                        index=idx,\n                        pass_original=pass_original,\n                    )\n            else:\n                self._run_validator(\n                    validator,\n                    data,\n                    original_data=original_data,\n                    error_store=error_store,\n                    many=many,\n                    pass_original=pass_original,\n                    partial=partial,\n                )\n\n    def _invoke_processors(\n        self,\n        tag: str,\n        *,\n        pass_many: bool,\n        data,\n        many: bool,\n        original_data=None,\n        **kwargs,\n    ):\n        key = (tag, pass_many)\n        for attr_name in self._hooks[key]:\n            # This will be a bound method.\n            processor = getattr(self, attr_name)\n\n            processor_kwargs = processor.__marshmallow_hook__[key]\n            pass_original = processor_kwargs.get(\"pass_original\", False)\n\n            if many and not pass_many:\n                if pass_original:\n                    data = [\n                        processor(item, original, many=many, **kwargs)\n                        for item, original in zip(data, original_data)\n                    ]\n                else:\n                    data = [processor(item, many=many, **kwargs) for item in data]\n            else:\n                if pass_original:\n                    data = processor(data, original_data, many=many, **kwargs)\n                else:\n                    data = processor(data, many=many, **kwargs)\n        return data\n\n\nBaseSchema = Schema  # for backwards compatibility\n\n# Path: src/marshmallow/validate.py\n\"\"\"Validation classes for various types of data.\"\"\"\nfrom __future__ import annotations\n\nimport re\nimport typing\nfrom abc import ABC, abstractmethod\nfrom itertools import zip_longest\nfrom operator import attrgetter\n\nfrom marshmallow import types\nfrom marshmallow.exceptions import ValidationError\n\n_T = typing.TypeVar(\"_T\")\n\n\nclass Validator(ABC):\n    \"\"\"Abstract base class for validators.\n\n    .. note::\n        This class does not provide any validation behavior. It is only used to\n        add a useful `__repr__` implementation for validators.\n    \"\"\"\n\n    error = None  # type: str | None\n\n    def __repr__(self) -> str:\n        args = self._repr_args()\n        args = f\"{args}, \" if args else \"\"\n\n        return f\"<{self.__class__.__name__}({args}error={self.error!r})>\"\n\n    def _repr_args(self) -> str:\n        \"\"\"A string representation of the args passed to this validator. Used by\n        `__repr__`.\n        \"\"\"\n        return \"\"\n\n    @abstractmethod\n    def __call__(self, value: typing.Any) -> typing.Any:\n        ...\n\n\nclass And(Validator):\n    \"\"\"Compose multiple validators and combine their error messages.\n\n    Example: ::\n\n        from marshmallow import validate, ValidationError\n\n        def is_even(value):\n            if value % 2 != 0:\n                raise ValidationError(\"Not an even value.\")\n\n        validator = validate.And(validate.Range(min=0), is_even)\n        validator(-1)\n        # ValidationError: ['Must be greater than or equal to 0.', 'Not an even value.']\n\n    :param validators: Validators to combine.\n    :param error: Error message to use when a validator returns ``False``.\n    \"\"\"\n\n    default_error_message = \"Invalid value.\"\n\n    def __init__(self, *validators: types.Validator, error: str | None = None):\n        self.validators = tuple(validators)\n        self.error = error or self.default_error_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"validators={self.validators!r}\"\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        errors = []\n        kwargs = {}\n        for validator in self.validators:\n            try:\n                r = validator(value)\n                if not isinstance(validator, Validator) and r is False:\n                    raise ValidationError(self.error)\n            except ValidationError as err:\n                kwargs.update(err.kwargs)\n                if isinstance(err.messages, dict):\n                    errors.append(err.messages)\n                else:\n                    # FIXME : Get rid of cast\n                    errors.extend(typing.cast(list, err.messages))\n        if errors:\n            raise ValidationError(errors, **kwargs)\n        return value\n\n\nclass URL(Validator):\n    \"\"\"Validate a URL.\n\n    :param relative: Whether to allow relative URLs.\n    :param absolute: Whether to allow absolute URLs.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`.\n    :param schemes: Valid schemes. By default, ``http``, ``https``,\n        ``ftp``, and ``ftps`` are allowed.\n    :param require_tld: Whether to reject non-FQDN hostnames.\n    \"\"\"\n\n    class RegexMemoizer:\n        def __init__(self):\n            self._memoized = {}\n\n        def _regex_generator(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            hostname_variants = [\n                # a normal domain name, expressed in [A-Z0-9] chars with hyphens allowed only in the middle\n                # note that the regex will be compiled with IGNORECASE, so these are upper and lowercase chars\n                (\n                    r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n                    r\"(?:[A-Z]{2,6}\\.?|[A-Z0-9-]{2,}\\.?)\"\n                ),\n                # or the special string 'localhost'\n                r\"localhost\",\n                # or IPv4\n                r\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",\n                # or IPv6\n                r\"\\[[A-F0-9]*:[A-F0-9:]+\\]\",\n            ]\n            if not require_tld:\n                # allow dotless hostnames\n                hostname_variants.append(r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.?)\")\n\n            absolute_part = \"\".join(\n                (\n                    # scheme (e.g. 'https://', 'ftp://', etc)\n                    # this is validated separately against allowed schemes, so in the regex\n                    # we simply want to capture its existence\n                    r\"(?:[a-z0-9\\.\\-\\+]*)://\",\n                    # userinfo, for URLs encoding authentication\n                    # e.g. 'ftp://foo:bar@ftp.example.org/'\n                    r\"(?:(?:[a-z0-9\\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?\",\n                    # netloc, the hostname/domain part of the URL plus the optional port\n                    r\"(?:\",\n                    \"|\".join(hostname_variants),\n                    r\")\",\n                    r\"(?::\\d+)?\",\n                )\n            )\n            relative_part = r\"(?:/?|[/?]\\S+)\\Z\"\n\n            if relative:\n                if absolute:\n                    parts: tuple[str, ...] = (\n                        r\"^(\",\n                        absolute_part,\n                        r\")?\",\n                        relative_part,\n                    )\n                else:\n                    parts = (r\"^\", relative_part)\n            else:\n                parts = (r\"^\", absolute_part, relative_part)\n\n            return re.compile(\"\".join(parts), re.IGNORECASE)\n\n        def __call__(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            key = (relative, absolute, require_tld)\n            if key not in self._memoized:\n                self._memoized[key] = self._regex_generator(\n                    relative, absolute, require_tld\n                )\n\n            return self._memoized[key]\n\n    _regex = RegexMemoizer()\n\n    default_message = \"Not a valid URL.\"\n    default_schemes = {\"http\", \"https\", \"ftp\", \"ftps\"}\n\n    def __init__(\n        self,\n        *,\n        relative: bool = False,\n        absolute: bool = True,\n        schemes: types.StrSequenceOrSet | None = None,\n        require_tld: bool = True,\n        error: str | None = None,\n    ):\n        if not relative and not absolute:\n            raise ValueError(\n                \"URL validation cannot set both relative and absolute to False.\"\n            )\n        self.relative = relative\n        self.absolute = absolute\n        self.error = error or self.default_message  # type: str\n        self.schemes = schemes or self.default_schemes\n        self.require_tld = require_tld\n\n    def _repr_args(self) -> str:\n        return f\"relative={self.relative!r}, absolute={self.absolute!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(input=value)\n\n    def __call__(self, value: str) -> str:\n        message = self._format_error(value)\n        if not value:\n            raise ValidationError(message)\n\n        # Check first if the scheme is valid\n        if \"://\" in value:\n            scheme = value.split(\"://\")[0].lower()\n            if scheme not in self.schemes:\n                raise ValidationError(message)\n\n        regex = self._regex(self.relative, self.absolute, self.require_tld)\n\n        if not regex.search(value):\n            raise ValidationError(message)\n\n        return value\n\n\nclass Email(Validator):\n    \"\"\"Validate an email address.\n\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated with `{input}`.\n    \"\"\"\n\n    USER_REGEX = re.compile(\n        r\"(^[-!#$%&'*+/=?^`{}|~\\w]+(\\.[-!#$%&'*+/=?^`{}|~\\w]+)*\\Z\"  # dot-atom\n        # quoted-string\n        r'|^\"([\\001-\\010\\013\\014\\016-\\037!#-\\[\\]-\\177]'\n        r'|\\\\[\\001-\\011\\013\\014\\016-\\177])*\"\\Z)',\n        re.IGNORECASE | re.UNICODE,\n    )\n\n    DOMAIN_REGEX = re.compile(\n        # domain\n        r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n        r\"(?:[A-Z]{2,6}|[A-Z0-9-]{2,})\\Z\"\n        # literal form, ipv4 address (SMTP 4.1.3)\n        r\"|^\\[(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)\"\n        r\"(\\.(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)){3}\\]\\Z\",\n        re.IGNORECASE | re.UNICODE,\n    )\n\n    DOMAIN_WHITELIST = (\"localhost\",)\n\n    default_message = \"Not a valid email address.\"\n\n    def __init__(self, *, error: str | None = None):\n        self.error = error or self.default_message  # type: str\n\n    def _format_error(self, value: str) -> str:\n        return self.error.format(input=value)\n\n    def __call__(self, value: str) -> str:\n        message = self._format_error(value)\n\n        if not value or \"@\" not in value:\n            raise ValidationError(message)\n\n        user_part, domain_part = value.rsplit(\"@\", 1)\n\n        if not self.USER_REGEX.match(user_part):\n            raise ValidationError(message)\n\n        if domain_part not in self.DOMAIN_WHITELIST:\n            if not self.DOMAIN_REGEX.match(domain_part):\n                try:\n                    domain_part = domain_part.encode(\"idna\").decode(\"ascii\")\n                except UnicodeError:\n                    pass\n                else:\n                    if self.DOMAIN_REGEX.match(domain_part):\n                        return value\n                raise ValidationError(message)\n\n        return value\n\n\nclass Range(Validator):\n    \"\"\"Validator which succeeds if the value passed to it is within the specified\n    range. If ``min`` is not specified, or is specified as `None`,\n    no lower bound exists. If ``max`` is not specified, or is specified as `None`,\n    no upper bound exists. The inclusivity of the bounds (if they exist) is configurable.\n    If ``min_inclusive`` is not specified, or is specified as `True`, then\n    the ``min`` bound is included in the range. If ``max_inclusive`` is not specified,\n    or is specified as `True`, then the ``max`` bound is included in the range.\n\n    :param min: The minimum value (lower bound). If not provided, minimum\n        value will not be checked.\n    :param max: The maximum value (upper bound). If not provided, maximum\n        value will not be checked.\n    :param min_inclusive: Whether the `min` bound is included in the range.\n    :param max_inclusive: Whether the `max` bound is included in the range.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`, `{min}` and `{max}`.\n    \"\"\"\n\n    message_min = \"Must be {min_op} {{min}}.\"\n    message_max = \"Must be {max_op} {{max}}.\"\n    message_all = \"Must be {min_op} {{min}} and {max_op} {{max}}.\"\n\n    message_gte = \"greater than or equal to\"\n    message_gt = \"greater than\"\n    message_lte = \"less than or equal to\"\n    message_lt = \"less than\"\n\n    def __init__(\n        self,\n        min=None,\n        max=None,\n        *,\n        min_inclusive: bool = True,\n        max_inclusive: bool = True,\n        error: str | None = None,\n    ):\n        self.min = min\n        self.max = max\n        self.error = error\n        self.min_inclusive = min_inclusive\n        self.max_inclusive = max_inclusive\n\n        # interpolate messages based on bound inclusivity\n        self.message_min = self.message_min.format(\n            min_op=self.message_gte if self.min_inclusive else self.message_gt\n        )\n        self.message_max = self.message_max.format(\n            max_op=self.message_lte if self.max_inclusive else self.message_lt\n        )\n        self.message_all = self.message_all.format(\n            min_op=self.message_gte if self.min_inclusive else self.message_gt,\n            max_op=self.message_lte if self.max_inclusive else self.message_lt,\n        )\n\n    def _repr_args(self) -> str:\n...\n", "needle_token_start": 4033, "needle_token_end": 4055, "code_context_ntokens": 16179, "output": ["```python\ndef dict_class(self) -> type:\n    return OrderedDict if self.ordered else dict\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "options", "language": "python", "path": "src/marshmallow/validate.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a sequence of pairs, each consisting of a string representation of a choice and its corresponding label. This is particularly useful for populating user interface elements like dropdown menus in forms.\n2. **Input**: Accepts either a callable function or a string. The callable should take one argument and return a value that represents a choice. If a string is provided, it is used to specify the name of an attribute of the choice objects.\n3. **Output**: Produces an iterable of tuples, where each tuple contains two elements: a string representation of a choice (obtained via the provided callable or attribute name) and a label for that choice.\n4. **Procedure**: The function first determines how to obtain the value representation of each choice, either by using a provided callable or by fetching the specified attribute of the choice. It then pairs each choice with its corresponding label (or a default empty string if no label is provided) and returns these pairs as an iterable.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n    :param relative: Whether to allow relative URLs.\n    :param absolute: Whether to allow absolute URLs.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`.\n    :param schemes: Valid schemes. By default, ``http``, ``https``,\n        ``ftp``, and ``ftps`` are allowed.\n    :param require_tld: Whether to reject non-FQDN hostnames.\n    \"\"\"\n\n    class RegexMemoizer:\n        def __init__(self):\n            self._memoized = {}\n\n        def _regex_generator(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            hostname_variants = [\n                # a normal domain name, expressed in [A-Z0-9] chars with hyphens allowed only in the middle\n                # note that the regex will be compiled with IGNORECASE, so these are upper and lowercase chars\n                (\n                    r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n                    r\"(?:[A-Z]{2,6}\\.?|[A-Z0-9-]{2,}\\.?)\"\n                ),\n                # or the special string 'localhost'\n                r\"localhost\",\n                # or IPv4\n                r\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",\n                # or IPv6\n                r\"\\[[A-F0-9]*:[A-F0-9:]+\\]\",\n            ]\n            if not require_tld:\n                # allow dotless hostnames\n                hostname_variants.append(r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.?)\")\n\n            absolute_part = \"\".join(\n                (\n                    # scheme (e.g. 'https://', 'ftp://', etc)\n                    # this is validated separately against allowed schemes, so in the regex\n                    # we simply want to capture its existence\n                    r\"(?:[a-z0-9\\.\\-\\+]*)://\",\n                    # userinfo, for URLs encoding authentication\n                    # e.g. 'ftp://foo:bar@ftp.example.org/'\n                    r\"(?:(?:[a-z0-9\\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?\",\n                    # netloc, the hostname/domain part of the URL plus the optional port\n                    r\"(?:\",\n                    \"|\".join(hostname_variants),\n                    r\")\",\n                    r\"(?::\\d+)?\",\n                )\n            )\n            relative_part = r\"(?:/?|[/?]\\S+)\\Z\"\n\n            if relative:\n                if absolute:\n                    parts: tuple[str, ...] = (\n                        r\"^(\",\n                        absolute_part,\n                        r\")?\",\n                        relative_part,\n                    )\n                else:\n                    parts = (r\"^\", relative_part)\n            else:\n                parts = (r\"^\", absolute_part, relative_part)\n\n            return re.compile(\"\".join(parts), re.IGNORECASE)\n\n        def __call__(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            key = (relative, absolute, require_tld)\n            if key not in self._memoized:\n                self._memoized[key] = self._regex_generator(\n                    relative, absolute, require_tld\n                )\n\n            return self._memoized[key]\n\n    _regex = RegexMemoizer()\n\n    default_message = \"Not a valid URL.\"\n    default_schemes = {\"http\", \"https\", \"ftp\", \"ftps\"}\n\n    def __init__(\n        self,\n        *,\n        relative: bool = False,\n        absolute: bool = True,\n        schemes: types.StrSequenceOrSet | None = None,\n        require_tld: bool = True,\n        error: str | None = None,\n    ):\n        if not relative and not absolute:\n            raise ValueError(\n                \"URL validation cannot set both relative and absolute to False.\"\n            )\n        self.relative = relative\n        self.absolute = absolute\n        self.error = error or self.default_message  # type: str\n        self.schemes = schemes or self.default_schemes\n        self.require_tld = require_tld\n\n    def _repr_args(self) -> str:\n        return f\"relative={self.relative!r}, absolute={self.absolute!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(input=value)\n\n    def __call__(self, value: str) -> str:\n        message = self._format_error(value)\n        if not value:\n            raise ValidationError(message)\n\n        # Check first if the scheme is valid\n        if \"://\" in value:\n            scheme = value.split(\"://\")[0].lower()\n            if scheme not in self.schemes:\n                raise ValidationError(message)\n\n        regex = self._regex(self.relative, self.absolute, self.require_tld)\n\n        if not regex.search(value):\n            raise ValidationError(message)\n\n        return value\n\n\nclass Email(Validator):\n    \"\"\"Validate an email address.\n\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated with `{input}`.\n    \"\"\"\n\n    USER_REGEX = re.compile(\n        r\"(^[-!#$%&'*+/=?^`{}|~\\w]+(\\.[-!#$%&'*+/=?^`{}|~\\w]+)*\\Z\"  # dot-atom\n        # quoted-string\n        r'|^\"([\\001-\\010\\013\\014\\016-\\037!#-\\[\\]-\\177]'\n        r'|\\\\[\\001-\\011\\013\\014\\016-\\177])*\"\\Z)',\n        re.IGNORECASE | re.UNICODE,\n    )\n\n    DOMAIN_REGEX = re.compile(\n        # domain\n        r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n        r\"(?:[A-Z]{2,6}|[A-Z0-9-]{2,})\\Z\"\n        # literal form, ipv4 address (SMTP 4.1.3)\n        r\"|^\\[(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)\"\n        r\"(\\.(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)){3}\\]\\Z\",\n        re.IGNORECASE | re.UNICODE,\n    )\n\n    DOMAIN_WHITELIST = (\"localhost\",)\n\n    default_message = \"Not a valid email address.\"\n\n    def __init__(self, *, error: str | None = None):\n        self.error = error or self.default_message  # type: str\n\n    def _format_error(self, value: str) -> str:\n        return self.error.format(input=value)\n\n    def __call__(self, value: str) -> str:\n        message = self._format_error(value)\n\n        if not value or \"@\" not in value:\n            raise ValidationError(message)\n\n        user_part, domain_part = value.rsplit(\"@\", 1)\n\n        if not self.USER_REGEX.match(user_part):\n            raise ValidationError(message)\n\n        if domain_part not in self.DOMAIN_WHITELIST:\n            if not self.DOMAIN_REGEX.match(domain_part):\n                try:\n                    domain_part = domain_part.encode(\"idna\").decode(\"ascii\")\n                except UnicodeError:\n                    pass\n                else:\n                    if self.DOMAIN_REGEX.match(domain_part):\n                        return value\n                raise ValidationError(message)\n\n        return value\n\n\nclass Range(Validator):\n    \"\"\"Validator which succeeds if the value passed to it is within the specified\n    range. If ``min`` is not specified, or is specified as `None`,\n    no lower bound exists. If ``max`` is not specified, or is specified as `None`,\n    no upper bound exists. The inclusivity of the bounds (if they exist) is configurable.\n    If ``min_inclusive`` is not specified, or is specified as `True`, then\n    the ``min`` bound is included in the range. If ``max_inclusive`` is not specified,\n    or is specified as `True`, then the ``max`` bound is included in the range.\n\n    :param min: The minimum value (lower bound). If not provided, minimum\n        value will not be checked.\n    :param max: The maximum value (upper bound). If not provided, maximum\n        value will not be checked.\n    :param min_inclusive: Whether the `min` bound is included in the range.\n    :param max_inclusive: Whether the `max` bound is included in the range.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`, `{min}` and `{max}`.\n    \"\"\"\n\n    message_min = \"Must be {min_op} {{min}}.\"\n    message_max = \"Must be {max_op} {{max}}.\"\n    message_all = \"Must be {min_op} {{min}} and {max_op} {{max}}.\"\n\n    message_gte = \"greater than or equal to\"\n    message_gt = \"greater than\"\n    message_lte = \"less than or equal to\"\n    message_lt = \"less than\"\n\n    def __init__(\n        self,\n        min=None,\n        max=None,\n        *,\n        min_inclusive: bool = True,\n        max_inclusive: bool = True,\n        error: str | None = None,\n    ):\n        self.min = min\n        self.max = max\n        self.error = error\n        self.min_inclusive = min_inclusive\n        self.max_inclusive = max_inclusive\n\n        # interpolate messages based on bound inclusivity\n        self.message_min = self.message_min.format(\n            min_op=self.message_gte if self.min_inclusive else self.message_gt\n        )\n        self.message_max = self.message_max.format(\n            max_op=self.message_lte if self.max_inclusive else self.message_lt\n        )\n        self.message_all = self.message_all.format(\n            min_op=self.message_gte if self.min_inclusive else self.message_gt,\n            max_op=self.message_lte if self.max_inclusive else self.message_lt,\n        )\n\n    def _repr_args(self) -> str:\n        return \"min={!r}, max={!r}, min_inclusive={!r}, max_inclusive={!r}\".format(\n            self.min, self.max, self.min_inclusive, self.max_inclusive\n        )\n\n    def _format_error(self, value: _T, message: str) -> str:\n        return (self.error or message).format(input=value, min=self.min, max=self.max)\n\n    def __call__(self, value: _T) -> _T:\n        if self.min is not None and (\n            value < self.min if self.min_inclusive else value <= self.min\n        ):\n            message = self.message_min if self.max is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        if self.max is not None and (\n            value > self.max if self.max_inclusive else value >= self.max\n        ):\n            message = self.message_max if self.min is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        return value\n\n\nclass Length(Validator):\n    \"\"\"Validator which succeeds if the value passed to it has a\n    length between a minimum and maximum. Uses len(), so it\n    can work for strings, lists, or anything with length.\n\n    :param min: The minimum length. If not provided, minimum length\n        will not be checked.\n    :param max: The maximum length. If not provided, maximum length\n        will not be checked.\n    :param equal: The exact length. If provided, maximum and minimum\n        length will not be checked.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`, `{min}` and `{max}`.\n    \"\"\"\n\n    message_min = \"Shorter than minimum length {min}.\"\n    message_max = \"Longer than maximum length {max}.\"\n    message_all = \"Length must be between {min} and {max}.\"\n    message_equal = \"Length must be {equal}.\"\n\n    def __init__(\n        self,\n        min: int | None = None,\n        max: int | None = None,\n        *,\n        equal: int | None = None,\n        error: str | None = None,\n    ):\n        if equal is not None and any([min, max]):\n            raise ValueError(\n                \"The `equal` parameter was provided, maximum or \"\n                \"minimum parameter must not be provided.\"\n            )\n\n        self.min = min\n        self.max = max\n        self.error = error\n        self.equal = equal\n\n    def _repr_args(self) -> str:\n        return f\"min={self.min!r}, max={self.max!r}, equal={self.equal!r}\"\n\n    def _format_error(self, value: typing.Sized, message: str) -> str:\n        return (self.error or message).format(\n            input=value, min=self.min, max=self.max, equal=self.equal\n        )\n\n    def __call__(self, value: typing.Sized) -> typing.Sized:\n        length = len(value)\n\n        if self.equal is not None:\n            if length != self.equal:\n                raise ValidationError(self._format_error(value, self.message_equal))\n            return value\n\n        if self.min is not None and length < self.min:\n            message = self.message_min if self.max is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        if self.max is not None and length > self.max:\n            message = self.message_max if self.min is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        return value\n\n\nclass Equal(Validator):\n    \"\"\"Validator which succeeds if the ``value`` passed to it is\n    equal to ``comparable``.\n\n    :param comparable: The object to compare to.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}` and `{other}`.\n    \"\"\"\n\n    default_message = \"Must be equal to {other}.\"\n\n    def __init__(self, comparable, *, error: str | None = None):\n        self.comparable = comparable\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"comparable={self.comparable!r}\"\n\n    def _format_error(self, value: _T) -> str:\n        return self.error.format(input=value, other=self.comparable)\n\n    def __call__(self, value: _T) -> _T:\n        if value != self.comparable:\n            raise ValidationError(self._format_error(value))\n        return value\n\n\nclass Regexp(Validator):\n    \"\"\"Validator which succeeds if the ``value`` matches ``regex``.\n\n    .. note::\n\n        Uses `re.match`, which searches for a match at the beginning of a string.\n\n    :param regex: The regular expression string to use. Can also be a compiled\n        regular expression pattern.\n    :param flags: The regexp flags to use, for example re.IGNORECASE. Ignored\n        if ``regex`` is not a string.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}` and `{regex}`.\n    \"\"\"\n\n    default_message = \"String does not match expected pattern.\"\n\n    def __init__(\n        self,\n        regex: str | bytes | typing.Pattern,\n        flags: int = 0,\n        *,\n        error: str | None = None,\n    ):\n        self.regex = (\n            re.compile(regex, flags) if isinstance(regex, (str, bytes)) else regex\n        )\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"regex={self.regex!r}\"\n\n    def _format_error(self, value: str | bytes) -> str:\n        return self.error.format(input=value, regex=self.regex.pattern)\n\n    @typing.overload\n    def __call__(self, value: str) -> str:\n        ...\n\n    @typing.overload\n    def __call__(self, value: bytes) -> bytes:\n        ...\n\n    def __call__(self, value):\n        if self.regex.match(value) is None:\n            raise ValidationError(self._format_error(value))\n\n        return value\n\n\nclass Predicate(Validator):\n    \"\"\"Call the specified ``method`` of the ``value`` object. The\n    validator succeeds if the invoked method returns an object that\n    evaluates to True in a Boolean context. Any additional keyword\n    argument will be passed to the method.\n\n    :param method: The name of the method to invoke.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}` and `{method}`.\n    :param kwargs: Additional keyword arguments to pass to the method.\n    \"\"\"\n\n    default_message = \"Invalid input.\"\n\n    def __init__(self, method: str, *, error: str | None = None, **kwargs):\n        self.method = method\n        self.error = error or self.default_message  # type: str\n        self.kwargs = kwargs\n\n    def _repr_args(self) -> str:\n        return f\"method={self.method!r}, kwargs={self.kwargs!r}\"\n\n    def _format_error(self, value: typing.Any) -> str:\n        return self.error.format(input=value, method=self.method)\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        method = getattr(value, self.method)\n\n        if not method(**self.kwargs):\n            raise ValidationError(self._format_error(value))\n\n        return value\n\n\nclass NoneOf(Validator):\n    \"\"\"Validator which fails if ``value`` is a member of ``iterable``.\n\n    :param iterable: A sequence of invalid values.\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated using `{input}` and `{values}`.\n    \"\"\"\n\n    default_message = \"Invalid input.\"\n\n    def __init__(self, iterable: typing.Iterable, *, error: str | None = None):\n        self.iterable = iterable\n        self.values_text = \", \".join(str(each) for each in self.iterable)\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"iterable={self.iterable!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(input=value, values=self.values_text)\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        try:\n            if value in self.iterable:\n                raise ValidationError(self._format_error(value))\n        except TypeError:\n            pass\n\n        return value\n\n\nclass OneOf(Validator):\n    \"\"\"Validator which succeeds if ``value`` is a member of ``choices``.\n\n    :param choices: A sequence of valid values.\n    :param labels: Optional sequence of labels to pair with the choices.\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated with `{input}`, `{choices}` and `{labels}`.\n    \"\"\"\n\n    default_message = \"Must be one of: {choices}.\"\n\n    def __init__(\n        self,\n        choices: typing.Iterable,\n        labels: typing.Iterable[str] | None = None,\n        *,\n        error: str | None = None,\n    ):\n        self.choices = choices\n        self.choices_text = \", \".join(str(choice) for choice in self.choices)\n        self.labels = labels if labels is not None else []\n        self.labels_text = \", \".join(str(label) for label in self.labels)\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"choices={self.choices!r}, labels={self.labels!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(\n            input=value, choices=self.choices_text, labels=self.labels_text\n        )\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        try:\n            if value not in self.choices:\n                raise ValidationError(self._format_error(value))\n        except TypeError as error:\n            raise ValidationError(self._format_error(value)) from error\n\n        return value\n\n    \ndef options(\n        self,\n        valuegetter: str | typing.Callable[[typing.Any], typing.Any] = str,\n    ) -> typing.Iterable[tuple[typing.Any, str]]:\n        \"\"\"Return a generator over the (value, label) pairs, where value\n        is a string associated with each choice. This convenience method\n        is useful to populate, for instance, a form select field.\n\n        :param valuegetter: Can be a callable or a string. In the former case, it must\n            be a one-argument callable which returns the value of a\n            choice. In the latter case, the string specifies the name\n            of an attribute of the choice objects. Defaults to `str()`\n            or `str()`.\n        \"\"\"\n        valuegetter = valuegetter if callable(valuegetter) else attrgetter(valuegetter)\n        pairs = zip_longest(self.choices, self.labels, fillvalue=\"\")\n\n        return ((valuegetter(choice), label) for choice, label in pairs)\n\n\nclass ContainsOnly(OneOf):\n    \"\"\"Validator which succeeds if ``value`` is a sequence and each element\n    in the sequence is also in the sequence passed as ``choices``. Empty input\n    is considered valid.\n\n    :param iterable choices: Same as :class:`OneOf`.\n    :param iterable labels: Same as :class:`OneOf`.\n    :param str error: Same as :class:`OneOf`.\n\n    .. versionchanged:: 3.0.0b2\n        Duplicate values are considered valid.\n    .. versionchanged:: 3.0.0b2\n        Empty input is considered valid. Use `validate.Length(min=1) <marshmallow.validate.Length>`\n        to validate against empty inputs.\n    \"\"\"\n\n    default_message = \"One or more of the choices you made was not in: {choices}.\"\n\n    def _format_error(self, value) -> str:\n        value_text = \", \".join(str(val) for val in value)\n        return super()._format_error(value_text)\n\n    def __call__(self, value: typing.Sequence[_T]) -> typing.Sequence[_T]:\n        # We can't use set.issubset because does not handle unhashable types\n        for val in value:\n            if val not in self.choices:\n                raise ValidationError(self._format_error(value))\n        return value\n\n\nclass ContainsNoneOf(NoneOf):\n    \"\"\"Validator which fails if ``value`` is a sequence and any element\n    in the sequence is a member of the sequence passed as ``iterable``. Empty input\n    is considered valid.\n\n    :param iterable iterable: Same as :class:`NoneOf`.\n    :param str error: Same as :class:`NoneOf`.\n\n    .. versionadded:: 3.6.0\n    \"\"\"\n\n    default_message = \"One or more of the choices you made was in: {values}.\"\n\n    def _format_error(self, value) -> str:\n        value_text = \", \".join(str(val) for val in value)\n        return super()._format_error(value_text)\n\n    def __call__(self, value: typing.Sequence[_T]) -> typing.Sequence[_T]:\n        for val in value:\n            if val in self.iterable:\n                raise ValidationError(self._format_error(value))\n        return value\n\n# Path: src/marshmallow/fields.py\n\"\"\"Field classes for various types of data.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport copy\nimport datetime as dt\nimport decimal\nimport ipaddress\nimport math\nimport numbers\nimport typing\nimport uuid\nimport warnings\nfrom collections.abc import Mapping as _Mapping\nfrom enum import Enum as EnumType\n\nfrom marshmallow import class_registry, types, utils, validate\nfrom marshmallow.base import FieldABC, SchemaABC\nfrom marshmallow.exceptions import (\n    FieldInstanceResolutionError,\n    StringNotCollectionError,\n    ValidationError,\n)\nfrom marshmallow.utils import (\n    is_aware,\n    is_collection,\n    resolve_field_instance,\n)\nfrom marshmallow.utils import (\n    missing as missing_,\n)\nfrom marshmallow.validate import And, Length\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\n__all__ = [\n    \"Field\",\n    \"Raw\",\n    \"Nested\",\n    \"Mapping\",\n    \"Dict\",\n    \"List\",\n    \"Tuple\",\n    \"String\",\n    \"UUID\",\n    \"Number\",\n    \"Integer\",\n    \"Decimal\",\n    \"Boolean\",\n    \"Float\",\n    \"DateTime\",\n    \"NaiveDateTime\",\n    \"AwareDateTime\",\n    \"Time\",\n    \"Date\",\n    \"TimeDelta\",\n    \"Url\",\n    \"URL\",\n    \"Email\",\n    \"IP\",\n    \"IPv4\",\n    \"IPv6\",\n    \"IPInterface\",\n    \"IPv4Interface\",\n    \"IPv6Interface\",\n    \"Enum\",\n    \"Method\",\n    \"Function\",\n    \"Str\",\n    \"Bool\",\n    \"Int\",\n    \"Constant\",\n    \"Pluck\",\n]\n\n_T = typing.TypeVar(\"_T\")\n\n\nclass Field(FieldABC):\n    \"\"\"Basic field from which other fields should extend. It applies no\n    formatting by default, and should only be used in cases where\n    data does not need to be formatted before being serialized or deserialized.\n    On error, the name of the field will be returned.\n\n    :param dump_default: If set, this value will be used during serialization if the\n        input value is missing. If not set, the field will be excluded from the\n        serialized output if the input value is missing. May be a value or a callable.\n    :param load_default: Default deserialization value for the field if the field is not\n        found in the input data. May be a value or a callable.\n    :param data_key: The name of the dict key in the external representation, i.e.\n        the input of `load` and the output of `dump`.\n        If `None`, the key will match the name of the field.\n    :param attribute: The name of the key/attribute in the internal representation, i.e.\n        the output of `load` and the input of `dump`.\n        If `None`, the key/attribute will match the name of the field.\n        Note: This should only be used for very specific use cases such as\n        outputting multiple fields for a single attribute, or using keys/attributes\n        that are invalid variable names, unsuitable for field names. In most cases,\n        you should use ``data_key`` instead.\n    :param validate: Validator or collection of validators that are called\n        during deserialization. Validator takes a field's input value as\n        its only parameter and returns a boolean.\n        If it returns `False`, an :exc:`ValidationError` is raised.\n    :param required: Raise a :exc:`ValidationError` if the field value\n        is not supplied during deserialization.\n    :param allow_none: Set this to `True` if `None` should be considered a valid value during\n        validation/deserialization. If ``load_default=None`` and ``allow_none`` is unset,\n        will default to ``True``. Otherwise, the default is ``False``.\n    :param load_only: If `True` skip this field during serialization, otherwise\n        its value will be present in the serialized data.\n    :param dump_only: If `True` skip this field during deserialization, otherwise\n        its value will be present in the deserialized object. In the context of an\n        HTTP API, this effectively marks the field as \"read-only\".\n    :param dict error_messages: Overrides for `Field.default_error_messages`.\n    :param metadata: Extra information to be stored as field metadata.\n\n    .. versionchanged:: 2.0.0\n        Removed `error` parameter. Use ``error_messages`` instead.\n\n    .. versionchanged:: 2.0.0\n        Added `allow_none` parameter, which makes validation/deserialization of `None`\n        consistent across fields.\n\n    .. versionchanged:: 2.0.0\n        Added `load_only` and `dump_only` parameters, which allow field skipping\n        during the (de)serialization process.\n\n    .. versionchanged:: 2.0.0\n        Added `missing` parameter, which indicates the value for a field if the field\n        is not found during deserialization.\n\n    .. versionchanged:: 2.0.0\n        ``default`` value is only used if explicitly set. Otherwise, missing values\n        inputs are excluded from serialized output.\n\n    .. versionchanged:: 3.0.0b8\n        Add ``data_key`` parameter for the specifying the key in the input and\n        output data. This parameter replaced both ``load_from`` and ``dump_to``.\n    \"\"\"\n\n    # Some fields, such as Method fields and Function fields, are not expected\n    #  to exist as attributes on the objects to serialize. Set this to False\n    #  for those fields\n    _CHECK_ATTRIBUTE = True\n\n    #: Default error messages for various kinds of errors. The keys in this dictionary\n    #: are passed to `Field.make_error`. The values are error messages passed to\n    #: :exc:`marshmallow.exceptions.ValidationError`.\n    default_error_messages = {\n        \"required\": \"Missing data for required field.\",\n        \"null\": \"Field may not be null.\",\n        \"validator_failed\": \"Invalid value.\",\n    }\n\n    def __init__(\n        self,\n        *,\n        load_default: typing.Any = missing_,\n        missing: typing.Any = missing_,\n        dump_default: typing.Any = missing_,\n        default: typing.Any = missing_,\n        data_key: str | None = None,\n        attribute: str | None = None,\n        validate: (\n            None\n            | typing.Callable[[typing.Any], typing.Any]\n            | typing.Iterable[typing.Callable[[typing.Any], typing.Any]]\n        ) = None,\n        required: bool = False,\n        allow_none: bool | None = None,\n        load_only: bool = False,\n        dump_only: bool = False,\n        error_messages: dict[str, str] | None = None,\n        metadata: typing.Mapping[str, typing.Any] | None = None,\n        **additional_metadata,\n    ) -> None:\n        # handle deprecated `default` and `missing` parameters\n        if default is not missing_:\n            warnings.warn(\n                \"The 'default' argument to fields is deprecated. \"\n                \"Use 'dump_default' instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            if dump_default is missing_:\n                dump_default = default\n        if missing is not missing_:\n            warnings.warn(\n                \"The 'missing' argument to fields is deprecated. \"\n                \"Use 'load_default' instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            if load_default is missing_:\n                load_default = missing\n        self.dump_default = dump_default\n        self.load_default = load_default\n\n        self.attribute = attribute\n        self.data_key = data_key\n        self.validate = validate\n        if validate is None:\n            self.validators = []\n        elif callable(validate):\n            self.validators = [validate]\n        elif utils.is_iterable_but_not_string(validate):\n            self.validators = list(validate)\n        else:\n            raise ValueError(\n                \"The 'validate' parameter must be a callable \"\n                \"or a collection of callables.\"\n            )\n\n        # If allow_none is None and load_default is None\n        # None should be considered valid by default\n        self.allow_none = load_default is None if allow_none is None else allow_none\n        self.load_only = load_only\n        self.dump_only = dump_only\n        if required is True and load_default is not missing_:\n            raise ValueError(\"'load_default' must not be set for required fields.\")\n        self.required = required\n\n        metadata = metadata or {}\n        self.metadata = {**metadata, **additional_metadata}\n        if additional_metadata:\n            warnings.warn(\n                \"Passing field metadata as keyword arguments is deprecated. Use the \"\n                \"explicit `metadata=...` argument instead. \"\n                f\"Additional metadata: {additional_metadata}\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n\n        # Collect default error message from self and parent classes\n        messages = {}  # type: dict[str, str]\n        for cls in reversed(self.__class__.__mro__):\n            messages.update(getattr(cls, \"default_error_messages\", {}))\n        messages.update(error_messages or {})\n        self.error_messages = messages\n\n    def __repr__(self) -> str:\n        return (\n            f\"<fields.{self.__class__.__name__}(dump_default={self.dump_default!r}, \"\n            f\"attribute={self.attribute!r}, \"\n            f\"validate={self.validate}, required={self.required}, \"\n            f\"load_only={self.load_only}, dump_only={self.dump_only}, \"\n            f\"load_default={self.load_default}, allow_none={self.allow_none}, \"\n            f\"error_messages={self.error_messages})>\"\n        )\n\n    def __deepcopy__(self, memo):\n        return copy.copy(self)\n\n    def get_value(self, obj, attr, accessor=None, default=missing_):\n        \"\"\"Return the value for a given key from an object.\n\n        :param object obj: The object to get the value from.\n        :param str attr: The attribute/key in `obj` to get the value from.\n        :param callable accessor: A callable used to retrieve the value of `attr` from\n            the object `obj`. Defaults to `marshmallow.utils.get_value`.\n        \"\"\"\n        accessor_func = accessor or utils.get_value\n        check_key = attr if self.attribute is None else self.attribute\n        return accessor_func(obj, check_key, default)\n\n    def _validate(self, value):\n        \"\"\"Perform validation on ``value``. Raise a :exc:`ValidationError` if validation\n        does not succeed.\n        \"\"\"\n        self._validate_all(value)\n\n    @property\n    def _validate_all(self):\n        return And(*self.validators, error=self.error_messages[\"validator_failed\"])\n\n    def make_error(self, key: str, **kwargs) -> ValidationError:\n        \"\"\"Helper method to make a `ValidationError` with an error message\n        from ``self.error_messages``.\n        \"\"\"\n        try:\n            msg = self.error_messages[key]\n        except KeyError as error:\n            class_name = self.__class__.__name__\n            message = (\n                f\"ValidationError raised by `{class_name}`, but error key `{key}` does \"\n                \"not exist in the `error_messages` dictionary.\"\n            )\n            raise AssertionError(message) from error\n        if isinstance(msg, (str, bytes)):\n            msg = msg.format(**kwargs)\n        return ValidationError(msg)\n\n    def fail(self, key: str, **kwargs):\n        \"\"\"Helper method that raises a `ValidationError` with an error message\n        from ``self.error_messages``.\n\n        .. deprecated:: 3.0.0\n            Use `make_error <marshmallow.fields.Field.make_error>` instead.\n        \"\"\"\n        warnings.warn(\n            f'`Field.fail` is deprecated. Use `raise self.make_error(\"{key}\", ...)` instead.',\n            RemovedInMarshmallow4Warning,\n            stacklevel=2,\n        )\n        raise self.make_error(key=key, **kwargs)\n\n    def _validate_missing(self, value):\n        \"\"\"Validate missing values. Raise a :exc:`ValidationError` if\n        `value` should be considered missing.\n        \"\"\"\n        if value is missing_ and self.required:\n            raise self.make_error(\"required\")\n        if value is None and not self.allow_none:\n            raise self.make_error(\"null\")\n\n    def serialize(\n        self,\n        attr: str,\n        obj: typing.Any,\n        accessor: typing.Callable[[typing.Any, str, typing.Any], typing.Any]\n        | None = None,\n        **kwargs,\n    ):\n        \"\"\"Pulls the value for the given key from the object, applies the\n        field's formatting and returns the result.\n\n        :param attr: The attribute/key to get from the object.\n        :param obj: The object to access the attribute/key from.\n        :param accessor: Function used to access values from ``obj``.\n        :param kwargs: Field-specific keyword arguments.\n        \"\"\"\n        if self._CHECK_ATTRIBUTE:\n            value = self.get_value(obj, attr, accessor=accessor)\n            if value is missing_:\n                default = self.dump_default\n                value = default() if callable(default) else default\n            if value is missing_:\n                return value\n        else:\n            value = None\n        return self._serialize(value, attr, obj, **kwargs)\n\n    def deserialize(\n        self,\n        value: typing.Any,\n        attr: str | None = None,\n        data: typing.Mapping[str, typing.Any] | None = None,\n        **kwargs,\n    ):\n        \"\"\"Deserialize ``value``.\n\n        :param value: The value to deserialize.\n        :param attr: The attribute/key in `data` to deserialize.\n        :param data: The raw input data passed to `Schema.load`.\n        :param kwargs: Field-specific keyword arguments.\n        :raise ValidationError: If an invalid value is passed or if a required value\n            is missing.\n        \"\"\"\n        # Validate required fields, deserialize, then validate\n        # deserialized value\n        self._validate_missing(value)\n        if value is missing_:\n            _miss = self.load_default\n            return _miss() if callable(_miss) else _miss\n        if self.allow_none and value is None:\n            return None\n        output = self._deserialize(value, attr, data, **kwargs)\n        self._validate(output)\n        return output\n\n    # Methods for concrete classes to override.\n\n    def _bind_to_schema(self, field_name, schema):\n        \"\"\"Update field with values from its parent schema. Called by\n        :meth:`Schema._bind_field <marshmallow.Schema._bind_field>`.\n\n        :param str field_name: Field name set in schema.\n        :param Schema|Field schema: Parent object.\n        \"\"\"\n        self.parent = self.parent or schema\n        self.name = self.name or field_name\n        self.root = self.root or (\n            self.parent.root if isinstance(self.parent, FieldABC) else self.parent\n        )\n\n    def _serialize(\n        self, value: typing.Any, attr: str | None, obj: typing.Any, **kwargs\n    ):\n        \"\"\"Serializes ``value`` to a basic Python datatype. Noop by default.\n        Concrete :class:`Field` classes should implement this method.\n\n        Example: ::\n\n            class TitleCase(Field):\n                def _serialize(self, value, attr, obj, **kwargs):\n                    if not value:\n                        return ''\n                    return str(value).title()\n\n        :param value: The value to be serialized.\n        :param str attr: The attribute or key on the object to be serialized.\n        :param object obj: The object the value was pulled from.\n        :param dict kwargs: Field-specific keyword arguments.\n        :return: The serialized value\n        \"\"\"\n        return value\n\n    def _deserialize(\n        self,\n        value: typing.Any,\n        attr: str | None,\n        data: typing.Mapping[str, typing.Any] | None,\n        **kwargs,\n    ):\n        \"\"\"Deserialize value. Concrete :class:`Field` classes should implement this method.\n\n        :param value: The value to be deserialized.\n        :param attr: The attribute/key in `data` to be deserialized.\n        :param data: The raw input data passed to the `Schema.load`.\n        :param kwargs: Field-specific keyword arguments.\n        :raise ValidationError: In case of formatting or validation failure.\n        :return: The deserialized value.\n\n        .. versionchanged:: 2.0.0\n            Added ``attr`` and ``data`` parameters.\n\n        .. versionchanged:: 3.0.0\n            Added ``**kwargs`` to signature.\n        \"\"\"\n        return value\n\n    # Properties\n\n    @property\n    def context(self):\n        \"\"\"The context dictionary for the parent :class:`Schema`.\"\"\"\n        return self.parent.context\n\n    # the default and missing properties are provided for compatibility and\n    # emit warnings when they are accessed and set\n    @property\n    def default(self):\n        warnings.warn(\n            \"The 'default' attribute of fields is deprecated. \"\n            \"Use 'dump_default' instead.\",\n            RemovedInMarshmallow4Warning,\n            stacklevel=2,\n        )\n        return self.dump_default\n\n    @default.setter\n    def default(self, value):\n        warnings.warn(\n            \"The 'default' attribute of fields is deprecated. \"\n            \"Use 'dump_default' instead.\",\n            RemovedInMarshmallow4Warning,\n            stacklevel=2,\n        )\n        self.dump_default = value\n\n    @property\n    def missing(self):\n        warnings.warn(\n            \"The 'missing' attribute of fields is deprecated. \"\n            \"Use 'load_default' instead.\",\n            RemovedInMarshmallow4Warning,\n            stacklevel=2,\n        )\n        return self.load_default\n\n    @missing.setter\n    def missing(self, value):\n        warnings.warn(\n            \"The 'missing' attribute of fields is deprecated. \"\n            \"Use 'load_default' instead.\",\n            RemovedInMarshmallow4Warning,\n            stacklevel=2,\n        )\n        self.load_default = value\n\n\nclass Raw(Field):\n    \"\"\"Field that applies no formatting.\"\"\"\n\n\nclass Nested(Field):\n    \"\"\"Allows you to nest a :class:`Schema <marshmallow.Schema>`\n    inside a field.\n\n    Examples: ::\n\n        class ChildSchema(Schema):\n            id = fields.Str()\n            name = fields.Str()\n            # Use lambda functions when you need two-way nesting or self-nesting\n            parent = fields.Nested(lambda: ParentSchema(only=(\"id\",)), dump_only=True)\n            siblings = fields.List(fields.Nested(lambda: ChildSchema(only=(\"id\", \"name\"))))\n\n        class ParentSchema(Schema):\n            id = fields.Str()\n            children = fields.List(\n                fields.Nested(ChildSchema(only=(\"id\", \"parent\", \"siblings\")))\n            )\n            spouse = fields.Nested(lambda: ParentSchema(only=(\"id\",)))\n\n    When passing a `Schema <marshmallow.Schema>` instance as the first argument,\n    the instance's ``exclude``, ``only``, and ``many`` attributes will be respected.\n\n    Therefore, when passing the ``exclude``, ``only``, or ``many`` arguments to `fields.Nested`,\n    you should pass a `Schema <marshmallow.Schema>` class (not an instance) as the first argument.\n\n    ::\n\n        # Yes\n        author = fields.Nested(UserSchema, only=('id', 'name'))\n\n        # No\n        author = fields.Nested(UserSchema(), only=('id', 'name'))\n\n    :param nested: `Schema` instance, class, class name (string), dictionary, or callable that\n        returns a `Schema` or dictionary. Dictionaries are converted with `Schema.from_dict`.\n    :param exclude: A list or tuple of fields to exclude.\n    :param only: A list or tuple of fields to marshal. If `None`, all fields are marshalled.\n        This parameter takes precedence over ``exclude``.\n    :param many: Whether the field is a collection of objects.\n    :param unknown: Whether to exclude, include, or raise an error for unknown\n        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n    :param kwargs: The same keyword arguments that :class:`Field` receives.\n    \"\"\"\n\n    #: Default error messages.\n    default_error_messages = {\"type\": \"Invalid type.\"}\n\n    def __init__(\n        self,\n        nested: SchemaABC\n        | type\n        | str\n        | dict[str, Field | type]\n        | typing.Callable[[], SchemaABC | type | dict[str, Field | type]],\n        *,\n        dump_default: typing.Any = missing_,\n        default: typing.Any = missing_,\n        only: types.StrSequenceOrSet | None = None,\n        exclude: types.StrSequenceOrSet = (),\n        many: bool = False,\n        unknown: str | None = None,\n        **kwargs,\n    ):\n        # Raise error if only or exclude is passed as string, not list of strings\n        if only is not None and not is_collection(only):\n            raise StringNotCollectionError('\"only\" should be a collection of strings.')\n        if not is_collection(exclude):\n            raise StringNotCollectionError(\n                '\"exclude\" should be a collection of strings.'\n            )\n        if nested == \"self\":\n            warnings.warn(\n                \"Passing 'self' to `Nested` is deprecated. \"\n                \"Use `Nested(lambda: MySchema(...))` instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n        self.nested = nested\n        self.only = only\n        self.exclude = exclude\n        self.many = many\n        self.unknown = unknown\n        self._schema = None  # Cached Schema instance\n        super().__init__(default=default, dump_default=dump_default, **kwargs)\n\n    @property\n    def schema(self):\n        \"\"\"The nested Schema object.\n\n        .. versionchanged:: 1.0.0\n            Renamed from `serializer` to `schema`.\n        \"\"\"\n        if not self._schema:\n            # Inherit context from parent.\n            context = getattr(self.parent, \"context\", {})\n            if callable(self.nested) and not isinstance(self.nested, type):\n                nested = self.nested()\n            else:\n                nested = self.nested\n            if isinstance(nested, dict):\n                # defer the import of `marshmallow.schema` to avoid circular imports\n                from marshmallow.schema import Schema\n\n                nested = Schema.from_dict(nested)\n\n            if isinstance(nested, SchemaABC):\n                self._schema = copy.copy(nested)\n                self._schema.context.update(context)\n                # Respect only and exclude passed from parent and re-initialize fields\n                set_class = self._schema.set_class\n                if self.only is not None:\n                    if self._schema.only is not None:\n                        original = self._schema.only\n                    else:  # only=None -> all fields\n                        original = self._schema.fields.keys()\n                    self._schema.only = set_class(self.only) & set_class(original)\n                if self.exclude:\n                    original = self._schema.exclude\n                    self._schema.exclude = set_class(self.exclude) | set_class(original)\n                self._schema._init_fields()\n            else:\n                if isinstance(nested, type) and issubclass(nested, SchemaABC):\n                    schema_class = nested\n                elif not isinstance(nested, (str, bytes)):\n                    raise ValueError(\n                        \"`Nested` fields must be passed a \"\n                        f\"`Schema`, not {nested.__class__}.\"\n                    )\n                elif nested == \"self\":\n                    schema_class = self.root.__class__\n                else:\n                    schema_class = class_registry.get_class(nested)\n                self._schema = schema_class(\n                    many=self.many,\n                    only=self.only,\n                    exclude=self.exclude,\n                    context=context,\n                    load_only=self._nested_normalized_option(\"load_only\"),\n                    dump_only=self._nested_normalized_option(\"dump_only\"),\n                )\n        return self._schema\n\n    def _nested_normalized_option(self, option_name: str) -> list[str]:\n        nested_field = \"%s.\" % self.name\n        return [\n            field.split(nested_field, 1)[1]\n            for field in getattr(self.root, option_name, set())\n            if field.startswith(nested_field)\n        ]\n\n    def _serialize(self, nested_obj, attr, obj, **kwargs):\n        # Load up the schema first. This allows a RegistryError to be raised\n        # if an invalid schema name was passed\n        schema = self.schema\n        if nested_obj is None:\n            return None\n        many = schema.many or self.many\n        return schema.dump(nested_obj, many=many)\n\n    def _test_collection(self, value):\n        many = self.schema.many or self.many\n        if many and not utils.is_collection(value):\n            raise self.make_error(\"type\", input=value, type=value.__class__.__name__)\n\n    def _load(self, value, data, partial=None):\n        try:\n            valid_data = self.schema.load(value, unknown=self.unknown, partial=partial)\n        except ValidationError as error:\n            raise ValidationError(\n                error.messages, valid_data=error.valid_data\n            ) from error\n        return valid_data\n\n    def _deserialize(self, value, attr, data, partial=None, **kwargs):\n        \"\"\"Same as :meth:`Field._deserialize` with additional ``partial`` argument.\n\n        :param bool|tuple partial: For nested schemas, the ``partial``\n            parameter passed to `Schema.load`.\n\n        .. versionchanged:: 3.0.0\n            Add ``partial`` parameter.\n        \"\"\"\n        self._test_collection(value)\n        return self._load(value, data, partial=partial)\n\n\nclass Pluck(Nested):\n    \"\"\"Allows you to replace nested data with one of the data's fields.\n\n    Example: ::\n\n        from marshmallow import Schema, fields\n\n        class ArtistSchema(Schema):\n            id = fields.Int()\n            name = fields.Str()\n\n        class AlbumSchema(Schema):\n            artist = fields.Pluck(ArtistSchema, 'id')\n\n\n        in_data = {'artist': 42}\n        loaded = AlbumSchema().load(in_data) # => {'artist': {'id': 42}}\n        dumped = AlbumSchema().dump(loaded)  # => {'artist': 42}\n\n    :param Schema nested: The Schema class or class name (string)\n        to nest, or ``\"self\"`` to nest the :class:`Schema` within itself.\n    :param str field_name: The key to pluck a value from.\n    :param kwargs: The same keyword arguments that :class:`Nested` receives.\n    \"\"\"\n\n    def __init__(\n        self,\n        nested: SchemaABC | type | str | typing.Callable[[], SchemaABC],\n        field_name: str,\n        **kwargs,\n    ):\n        super().__init__(nested, only=(field_name,), **kwargs)\n        self.field_name = field_name\n\n    @property\n    def _field_data_key(self):\n        only_field = self.schema.fields[self.field_name]\n        return only_field.data_key or self.field_name\n\n    def _serialize(self, nested_obj, attr, obj, **kwargs):\n        ret = super()._serialize(nested_obj, attr, obj, **kwargs)\n        if ret is None:\n            return None\n        if self.many:\n            return utils.pluck(ret, key=self._field_data_key)\n        return ret[self._field_data_key]\n\n    def _deserialize(self, value, attr, data, partial=None, **kwargs):\n        self._test_collection(value)\n        if self.many:\n            value = [{self._field_data_key: v} for v in value]\n        else:\n            value = {self._field_data_key: value}\n        return self._load(value, data, partial=partial)\n\n\nclass List(Field):\n    \"\"\"A list field, composed with another `Field` class or\n    instance.\n\n    Example: ::\n\n        numbers = fields.List(fields.Float())\n\n    :param cls_or_instance: A field class or instance.\n    :param kwargs: The same keyword arguments that :class:`Field` receives.\n\n    .. versionchanged:: 2.0.0\n        The ``allow_none`` parameter now applies to deserialization and\n        has the same semantics as the other fields.\n\n    .. versionchanged:: 3.0.0rc9\n        Does not serialize scalar values to single-item lists.\n    \"\"\"\n\n    #: Default error messages.\n    default_error_messages = {\"invalid\": \"Not a valid list.\"}\n\n    def __init__(self, cls_or_instance: Field | type, **kwargs):\n        super().__init__(**kwargs)\n        try:\n            self.inner = resolve_field_instance(cls_or_instance)\n        except FieldInstanceResolutionError as error:\n            raise ValueError(\n                \"The list elements must be a subclass or instance of \"\n                \"marshmallow.base.FieldABC.\"\n            ) from error\n        if isinstance(self.inner, Nested):\n            self.only = self.inner.only\n            self.exclude = self.inner.exclude\n\n    def _bind_to_schema(self, field_name, schema):\n        super()._bind_to_schema(field_name, schema)\n        self.inner = copy.deepcopy(self.inner)\n        self.inner._bind_to_schema(field_name, self)\n        if isinstance(self.inner, Nested):\n            self.inner.only = self.only\n            self.inner.exclude = self.exclude\n\n    def _serialize(self, value, attr, obj, **kwargs) -> list[typing.Any] | None:\n        if value is None:\n            return None\n        return [self.inner._serialize(each, attr, obj, **kwargs) for each in value]\n\n    def _deserialize(self, value, attr, data, **kwargs) -> list[typing.Any]:\n        if not utils.is_collection(value):\n            raise self.make_error(\"invalid\")\n\n        result = []\n        errors = {}\n        for idx, each in enumerate(value):\n            try:\n                result.append(self.inner.deserialize(each, **kwargs))\n            except ValidationError as error:\n                if error.valid_data is not None:\n                    result.append(error.valid_data)\n                errors.update({idx: error.messages})\n        if errors:\n            raise ValidationError(errors, valid_data=result)\n        return result\n\n\nclass Tuple(Field):\n    \"\"\"A tuple field, composed of a fixed number of other `Field` classes or\n    instances\n\n    Example: ::\n\n        row = Tuple((fields.String(), fields.Integer(), fields.Float()))\n\n    .. note::\n        Because of the structured nature of `collections.namedtuple` and\n        `typing.NamedTuple`, using a Schema within a Nested field for them is\n        more appropriate than using a `Tuple` field.\n\n    :param Iterable[Field] tuple_fields: An iterable of field classes or\n        instances.\n    :param kwargs: The same keyword arguments that :class:`Field` receives.\n\n    .. versionadded:: 3.0.0rc4\n    \"\"\"\n\n    #: Default error messages.\n    default_error_messages = {\"invalid\": \"Not a valid tuple.\"}\n\n    def __init__(self, tuple_fields, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        if not utils.is_collection(tuple_fields):\n            raise ValueError(\n                \"tuple_fields must be an iterable of Field classes or \" \"instances.\"\n            )\n\n        try:\n            self.tuple_fields = [\n                resolve_field_instance(cls_or_instance)\n                for cls_or_instance in tuple_fields\n            ]\n        except FieldInstanceResolutionError as error:\n            raise ValueError(\n                'Elements of \"tuple_fields\" must be subclasses or '\n                \"instances of marshmallow.base.FieldABC.\"\n            ) from error\n\n        self.validate_length = Length(equal=len(self.tuple_fields))\n\n    def _bind_to_schema(self, field_name, schema):\n        super()._bind_to_schema(field_name, schema)\n        new_tuple_fields = []\n        for field in self.tuple_fields:\n            field = copy.deepcopy(field)\n            field._bind_to_schema(field_name, self)\n            new_tuple_fields.append(field)\n\n        self.tuple_fields = new_tuple_fields\n\n    def _serialize(self, value, attr, obj, **kwargs) -> tuple | None:\n        if value is None:\n            return None\n\n        return tuple(\n            field._serialize(each, attr, obj, **kwargs)\n            for field, each in zip(self.tuple_fields, value)\n        )\n\n    def _deserialize(self, value, attr, data, **kwargs) -> tuple:\n        if not utils.is_collection(value):\n            raise self.make_error(\"invalid\")\n\n        self.validate_length(value)\n\n        result = []\n        errors = {}\n\n        for idx, (field, each) in enumerate(zip(self.tuple_fields, value)):\n            try:\n                result.append(field.deserialize(each, **kwargs))\n            except ValidationError as error:\n                if error.valid_data is not None:\n                    result.append(error.valid_data)\n                errors.update({idx: error.messages})\n        if errors:\n            raise ValidationError(errors, valid_data=result)\n\n        return tuple(result)\n\n\nclass String(Field):\n    \"\"\"A string field.\n\n    :param kwargs: The same keyword arguments that :class:`Field` receives.\n    \"\"\"\n\n    #: Default error messages.\n    default_error_messages = {\n        \"invalid\": \"Not a valid string.\",\n        \"invalid_utf8\": \"Not a valid utf-8 string.\",\n    }\n\n    def _serialize(self, value, attr, obj, **kwargs) -> str | None:\n        if value is None:\n            return None\n        return utils.ensure_text_type(value)\n\n    def _deserialize(self, value, attr, data, **kwargs) -> typing.Any:\n        if not isinstance(value, (str, bytes)):\n            raise self.make_error(\"invalid\")\n        try:\n            return utils.ensure_text_type(value)\n        except UnicodeDecodeError as error:\n            raise self.make_error(\"invalid_utf8\") from error\n\n\nclass UUID(String):\n    \"\"\"A UUID field.\"\"\"\n\n    #: Default error messages.\n    default_error_messages = {\"invalid_uuid\": \"Not a valid UUID.\"}\n\n    def _validated(self, value) -> uuid.UUID | None:\n        \"\"\"Format the value or raise a :exc:`ValidationError` if an error occurs.\"\"\"\n        if value is None:\n            return None\n        if isinstance(value, uuid.UUID):\n            return value\n        try:\n            if isinstance(value, bytes) and len(value) == 16:\n                return uuid.UUID(bytes=value)\n            return uuid.UUID(value)\n        except (ValueError, AttributeError, TypeError) as error:\n            raise self.make_error(\"invalid_uuid\") from error\n\n    def _deserialize(self, value, attr, data, **kwargs) -> uuid.UUID | None:\n        return self._validated(value)\n\n\nclass Number(Field):\n    \"\"\"Base class for number fields.\n\n    :param bool as_string: If `True`, format the serialized value as a string.\n    :param kwargs: The same keyword arguments that :class:`Field` receives.\n    \"\"\"\n\n    num_type = float  # type: typing.Type\n\n    #: Default error messages.\n    default_error_messages = {\n        \"invalid\": \"Not a valid number.\",\n        \"too_large\": \"Number too large.\",\n    }\n\n    def __init__(self, *, as_string: bool = False, **kwargs):\n        self.as_string = as_string\n        super().__init__(**kwargs)\n\n    def _format_num(self, value) -> typing.Any:\n        \"\"\"Return the number value for value, given this field's `num_type`.\"\"\"\n        return self.num_type(value)\n\n    def _validated(self, value) -> _T | None:\n        \"\"\"Format the value or raise a :exc:`ValidationError` if an error occurs.\"\"\"\n        if value is None:\n            return None\n        # (value is True or value is False) is ~5x faster than isinstance(value, bool)\n        if value is True or value is False:\n            raise self.make_error(\"invalid\", input=value)\n        try:\n            return self._format_num(value)\n        except (TypeError, ValueError) as error:\n            raise self.make_error(\"invalid\", input=value) from error\n        except OverflowError as error:\n            raise self.make_error(\"too_large\", input=value) from error\n\n    def _to_string(self, value) -> str:\n        return str(value)\n\n    def _serialize(self, value, attr, obj, **kwargs) -> str | _T | None:\n        \"\"\"Return a string if `self.as_string=True`, otherwise return this field's `num_type`.\"\"\"\n        if value is None:\n            return None\n        ret = self._format_num(value)  # type: _T\n        return self._to_string(ret) if self.as_string else ret\n\n    def _deserialize(self, value, attr, data, **kwargs) -> _T | None:\n        return self._validated(value)\n\n\nclass Integer(Number):\n    \"\"\"An integer field.\n\n    :param strict: If `True`, only integer types are valid.\n...\n", "needle_token_start": 5502, "needle_token_end": 5746, "code_context_ntokens": 16109, "output": ["```python\ndef options(\n        self,\n        valuegetter: str | typing.Callable[[typing.Any], typing.Any] = str,\n    ) -> typing.Iterable[tuple[typing.Any, str]]:\n        \"\"\"Return a generator over the (value, label) pairs, where value\n        is a string associated with each choice. This convenience method\n        is useful to populate, for instance, a form select field.\n\n        :param valuegetter: Can be a callable or a string. In the former case, it must\n            be a one-argument callable which returns the value of a\n            choice. In the latter case, the string specifies the name\n            of an attribute of the choice objects. Defaults to `str()`\n            or `str()`.\n        \"\"\"\n        valuegetter = valuegetter if callable(valuegetter) else attrgetter(valuegetter)\n        pairs = zip_longest(self.choices, self.labels, fillvalue=\"\")\n\n        return ((valuegetter(choice), label) for choice, label in pairs)\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "_invoke_field_validators", "language": "python", "path": "src/marshmallow/schema.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to execute field-specific validation methods on data, either for individual items or collections of items, and handle any resulting errors or data modifications.\n2. **Input**: The function takes an error handling object, the data to be validated (which can be a single item or a list of items), and a boolean indicating if the data represents multiple items.\n3. **Output**: There is no direct output from this function; however, it modifies the input data based on validation results and updates the error handling object with any validation errors encountered.\n4. **Procedure**: \n   - The function iterates over registered validation methods.\n   - For each validation method, it identifies the corresponding field in the data.\n   - If the field exists, the function retrieves the value from the data.\n   - The validation method is then called with this value.\n   - If the validation method indicates that the value is missing (using a specific missing marker), the value is removed from the data.\n   - Any errors raised during validation are captured and stored in the error handling object.\n   - This process is repeated for each item in the data if the input represents multiple items.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    ):\n        \"\"\"Custom error handler function for the schema.\n\n        :param error: The `ValidationError` raised during (de)serialization.\n        :param data: The original input data.\n        :param many: Value of ``many`` on dump or load.\n        :param partial: Value of ``partial`` on load.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0rc9\n            Receives `many` and `partial` (on deserialization) as keyword arguments.\n        \"\"\"\n        pass\n\n    def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any):\n        \"\"\"Defines how to pull values from an object to serialize.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0a1\n            Changed position of ``obj`` and ``attr``.\n        \"\"\"\n        return get_value(obj, attr, default)\n\n    ##### Serialization/Deserialization API #####\n\n    @staticmethod\n    def _call_and_store(getter_func, data, *, field_name, error_store, index=None):\n        \"\"\"Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.\n\n        :param callable getter_func: Function for getting the serialized/deserialized\n            value from ``data``.\n        :param data: The data passed to ``getter_func``.\n        :param str field_name: Field name.\n        :param int index: Index of the item being validated, if validating a collection,\n            otherwise `None`.\n        \"\"\"\n        try:\n            value = getter_func(data)\n        except ValidationError as error:\n            error_store.store_error(error.messages, field_name, index=index)\n            # When a Nested field fails validation, the marshalled data is stored\n            # on the ValidationError's valid_data attribute\n            return error.valid_data or missing\n        return value\n\n    def _serialize(self, obj: _T | typing.Iterable[_T], *, many: bool = False):\n        \"\"\"Serialize ``obj``.\n\n        :param obj: The object(s) to serialize.\n        :param bool many: `True` if ``data`` should be serialized as a collection.\n        :return: A dictionary of the serialized data\n\n        .. versionchanged:: 1.0.0\n            Renamed from ``marshal``.\n        \"\"\"\n        if many and obj is not None:\n            return [\n                self._serialize(d, many=False)\n                for d in typing.cast(typing.Iterable[_T], obj)\n            ]\n        ret = self.dict_class()\n        for attr_name, field_obj in self.dump_fields.items():\n            value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)\n            if value is missing:\n                continue\n            key = field_obj.data_key if field_obj.data_key is not None else attr_name\n            ret[key] = value\n        return ret\n\n    def dump(self, obj: typing.Any, *, many: bool | None = None):\n        \"\"\"Serialize an object to native Python data types according to this\n        Schema's fields.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: Serialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        .. versionchanged:: 3.0.0rc9\n            Validation no longer occurs upon serialization.\n        \"\"\"\n        many = self.many if many is None else bool(many)\n        if self._has_processors(PRE_DUMP):\n            processed_obj = self._invoke_dump_processors(\n                PRE_DUMP, obj, many=many, original_data=obj\n            )\n        else:\n            processed_obj = obj\n\n        result = self._serialize(processed_obj, many=many)\n\n        if self._has_processors(POST_DUMP):\n            result = self._invoke_dump_processors(\n                POST_DUMP, result, many=many, original_data=obj\n            )\n\n        return result\n\n    def dumps(self, obj: typing.Any, *args, many: bool | None = None, **kwargs):\n        \"\"\"Same as :meth:`dump`, except return a JSON-encoded string.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: A ``json`` string\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        \"\"\"\n        serialized = self.dump(obj, many=many)\n        return self.opts.render_module.dumps(serialized, *args, **kwargs)\n\n    def _deserialize(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        error_store: ErrorStore,\n        many: bool = False,\n        partial=None,\n        unknown=RAISE,\n        index=None,\n    ) -> _T | list[_T]:\n        \"\"\"Deserialize ``data``.\n\n        :param dict data: The data to deserialize.\n        :param ErrorStore error_store: Structure to store errors.\n        :param bool many: `True` if ``data`` should be deserialized as a collection.\n        :param bool|tuple partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        :param int index: Index of the item being serialized (for storing errors) if\n            serializing a collection, otherwise `None`.\n        :return: A dictionary of the deserialized data.\n        \"\"\"\n        index_errors = self.opts.index_errors\n        index = index if index_errors else None\n        if many:\n            if not is_collection(data):\n                error_store.store_error([self.error_messages[\"type\"]], index=index)\n                ret_l = []  # type: typing.List[_T]\n            else:\n                ret_l = [\n                    typing.cast(\n                        _T,\n                        self._deserialize(\n                            typing.cast(typing.Mapping[str, typing.Any], d),\n                            error_store=error_store,\n                            many=False,\n                            partial=partial,\n                            unknown=unknown,\n                            index=idx,\n                        ),\n                    )\n                    for idx, d in enumerate(data)\n                ]\n            return ret_l\n        ret_d = self.dict_class()\n        # Check data is a dict\n        if not isinstance(data, Mapping):\n            error_store.store_error([self.error_messages[\"type\"]], index=index)\n        else:\n            partial_is_collection = is_collection(partial)\n            for attr_name, field_obj in self.load_fields.items():\n                field_name = (\n                    field_obj.data_key if field_obj.data_key is not None else attr_name\n                )\n                raw_value = data.get(field_name, missing)\n                if raw_value is missing:\n                    # Ignore missing field if we're allowed to.\n                    if partial is True or (\n                        partial_is_collection and attr_name in partial\n                    ):\n                        continue\n                d_kwargs = {}\n                # Allow partial loading of nested schemas.\n                if partial_is_collection:\n                    prefix = field_name + \".\"\n                    len_prefix = len(prefix)\n                    sub_partial = [\n                        f[len_prefix:] for f in partial if f.startswith(prefix)\n                    ]\n                    d_kwargs[\"partial\"] = sub_partial\n                elif partial is not None:\n                    d_kwargs[\"partial\"] = partial\n\n                def getter(\n                    val, field_obj=field_obj, field_name=field_name, d_kwargs=d_kwargs\n                ):\n                    return field_obj.deserialize(\n                        val,\n                        field_name,\n                        data,\n                        **d_kwargs,\n                    )\n\n                value = self._call_and_store(\n                    getter_func=getter,\n                    data=raw_value,\n                    field_name=field_name,\n                    error_store=error_store,\n                    index=index,\n                )\n                if value is not missing:\n                    key = field_obj.attribute or attr_name\n                    set_value(ret_d, key, value)\n            if unknown != EXCLUDE:\n                fields = {\n                    field_obj.data_key if field_obj.data_key is not None else field_name\n                    for field_name, field_obj in self.load_fields.items()\n                }\n                for key in set(data) - fields:\n                    value = data[key]\n                    if unknown == INCLUDE:\n                        ret_d[key] = value\n                    elif unknown == RAISE:\n                        error_store.store_error(\n                            [self.error_messages[\"unknown\"]],\n                            key,\n                            (index if index_errors else None),\n                        )\n        return ret_d\n\n    def load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        \"\"\"Deserialize a data structure to an object defined by this Schema's fields.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        return self._do_load(\n            data, many=many, partial=partial, unknown=unknown, postprocess=True\n        )\n\n    def loads(\n        self,\n        json_data: str,\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        **kwargs,\n    ):\n        \"\"\"Same as :meth:`load`, except it takes a JSON string as input.\n\n        :param json_data: A JSON string of the data to deserialize.\n        :param many: Whether to deserialize `obj` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        data = self.opts.render_module.loads(json_data, **kwargs)\n        return self.load(data, many=many, partial=partial, unknown=unknown)\n\n    def _run_validator(\n        self,\n        validator_func,\n        output,\n        *,\n        original_data,\n        error_store,\n        many,\n        partial,\n        pass_original,\n        index=None,\n    ):\n        try:\n            if pass_original:  # Pass original, raw data (before unmarshalling)\n                validator_func(output, original_data, partial=partial, many=many)\n            else:\n                validator_func(output, partial=partial, many=many)\n        except ValidationError as err:\n            error_store.store_error(err.messages, err.field_name, index=index)\n\n    def validate(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n    ) -> dict[str, list[str]]:\n        \"\"\"Validate `data` against the schema, returning a dictionary of\n        validation errors.\n\n        :param data: The data to validate.\n        :param many: Whether to validate `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :return: A dictionary of validation errors.\n\n        .. versionadded:: 1.1.0\n        \"\"\"\n        try:\n            self._do_load(data, many=many, partial=partial, postprocess=False)\n        except ValidationError as exc:\n            return typing.cast(typing.Dict[str, typing.List[str]], exc.messages)\n        return {}\n\n    ##### Private Helpers #####\n\n    def _do_load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        postprocess: bool = True,\n    ):\n        \"\"\"Deserialize `data`, returning the deserialized result.\n        This method is private API.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to validate required fields. If its\n            value is an iterable, only fields listed in that iterable will be\n            ignored will be allowed missing. If `True`, all fields will be allowed missing.\n            If `None`, the value for `self.partial` is used.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :param postprocess: Whether to run post_load methods..\n        :return: Deserialized data\n        \"\"\"\n        error_store = ErrorStore()\n        errors = {}  # type: dict[str, list[str]]\n        many = self.many if many is None else bool(many)\n        unknown = (\n            self.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        if partial is None:\n            partial = self.partial\n        # Run preprocessors\n        if self._has_processors(PRE_LOAD):\n            try:\n                processed_data = self._invoke_load_processors(\n                    PRE_LOAD, data, many=many, original_data=data, partial=partial\n                )\n            except ValidationError as err:\n                errors = err.normalized_messages()\n                result = None  # type: list | dict | None\n        else:\n            processed_data = data\n        if not errors:\n            # Deserialize data\n            result = self._deserialize(\n                processed_data,\n                error_store=error_store,\n                many=many,\n                partial=partial,\n                unknown=unknown,\n            )\n            # Run field-level validation\n            self._invoke_field_validators(\n                error_store=error_store, data=result, many=many\n            )\n            # Run schema-level validation\n            if self._has_processors(VALIDATES_SCHEMA):\n                field_errors = bool(error_store.errors)\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=True,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=False,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n            errors = error_store.errors\n            # Run post processors\n            if not errors and postprocess and self._has_processors(POST_LOAD):\n                try:\n                    result = self._invoke_load_processors(\n                        POST_LOAD,\n                        result,\n                        many=many,\n                        original_data=data,\n                        partial=partial,\n                    )\n                except ValidationError as err:\n                    errors = err.normalized_messages()\n        if errors:\n            exc = ValidationError(errors, data=data, valid_data=result)\n            self.handle_error(exc, data, many=many, partial=partial)\n            raise exc\n\n        return result\n\n    def _normalize_nested_options(self) -> None:\n        \"\"\"Apply then flatten nested schema options.\n        This method is private API.\n        \"\"\"\n        if self.only is not None:\n            # Apply the only option to nested fields.\n            self.__apply_nested_option(\"only\", self.only, \"intersection\")\n            # Remove the child field names from the only option.\n            self.only = self.set_class([field.split(\".\", 1)[0] for field in self.only])\n        if self.exclude:\n            # Apply the exclude option to nested fields.\n            self.__apply_nested_option(\"exclude\", self.exclude, \"union\")\n            # Remove the parent field names from the exclude option.\n            self.exclude = self.set_class(\n                [field for field in self.exclude if \".\" not in field]\n            )\n\n    def __apply_nested_option(self, option_name, field_names, set_operation) -> None:\n        \"\"\"Apply nested options to nested fields\"\"\"\n        # Split nested field names on the first dot.\n        nested_fields = [name.split(\".\", 1) for name in field_names if \".\" in name]\n        # Partition the nested field names by parent field.\n        nested_options = defaultdict(list)  # type: defaultdict\n        for parent, nested_names in nested_fields:\n            nested_options[parent].append(nested_names)\n        # Apply the nested field options.\n        for key, options in iter(nested_options.items()):\n            new_options = self.set_class(options)\n            original_options = getattr(self.declared_fields[key], option_name, ())\n            if original_options:\n                if set_operation == \"union\":\n                    new_options |= self.set_class(original_options)\n                if set_operation == \"intersection\":\n                    new_options &= self.set_class(original_options)\n            setattr(self.declared_fields[key], option_name, new_options)\n\n    def _init_fields(self) -> None:\n        \"\"\"Update self.fields, self.load_fields, and self.dump_fields based on schema options.\n        This method is private API.\n        \"\"\"\n        if self.opts.fields:\n            available_field_names = self.set_class(self.opts.fields)\n        else:\n            available_field_names = self.set_class(self.declared_fields.keys())\n            if self.opts.additional:\n                available_field_names |= self.set_class(self.opts.additional)\n\n        invalid_fields = self.set_class()\n\n        if self.only is not None:\n            # Return only fields specified in only option\n            field_names: typing.AbstractSet[typing.Any] = self.set_class(self.only)\n\n            invalid_fields |= field_names - available_field_names\n        else:\n            field_names = available_field_names\n\n        # If \"exclude\" option or param is specified, remove those fields.\n        if self.exclude:\n            # Note that this isn't available_field_names, since we want to\n            # apply \"only\" for the actual calculation.\n            field_names = field_names - self.exclude\n            invalid_fields |= self.exclude - available_field_names\n\n        if invalid_fields:\n            message = f\"Invalid fields for {self}: {invalid_fields}.\"\n            raise ValueError(message)\n\n        fields_dict = self.dict_class()\n        for field_name in field_names:\n            field_obj = self.declared_fields.get(field_name, ma_fields.Inferred())\n            self._bind_field(field_name, field_obj)\n            fields_dict[field_name] = field_obj\n\n        load_fields, dump_fields = self.dict_class(), self.dict_class()\n        for field_name, field_obj in fields_dict.items():\n            if not field_obj.dump_only:\n                load_fields[field_name] = field_obj\n            if not field_obj.load_only:\n                dump_fields[field_name] = field_obj\n\n        dump_data_keys = [\n            field_obj.data_key if field_obj.data_key is not None else name\n            for name, field_obj in dump_fields.items()\n        ]\n        if len(dump_data_keys) != len(set(dump_data_keys)):\n            data_keys_duplicates = {\n                x for x in dump_data_keys if dump_data_keys.count(x) > 1\n            }\n            raise ValueError(\n                \"The data_key argument for one or more fields collides \"\n                \"with another field's name or data_key argument. \"\n                \"Check the following field names and \"\n                f\"data_key arguments: {list(data_keys_duplicates)}\"\n            )\n        load_attributes = [obj.attribute or name for name, obj in load_fields.items()]\n        if len(load_attributes) != len(set(load_attributes)):\n            attributes_duplicates = {\n                x for x in load_attributes if load_attributes.count(x) > 1\n            }\n            raise ValueError(\n                \"The attribute argument for one or more fields collides \"\n                \"with another field's name or attribute argument. \"\n                \"Check the following field names and \"\n                f\"attribute arguments: {list(attributes_duplicates)}\"\n            )\n\n        self.fields = fields_dict\n        self.dump_fields = dump_fields\n        self.load_fields = load_fields\n\n    def on_bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None:\n        \"\"\"Hook to modify a field when it is bound to the `Schema`.\n\n        No-op by default.\n        \"\"\"\n        return None\n\n    def _bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None:\n        \"\"\"Bind field to the schema, setting any necessary attributes on the\n        field (e.g. parent and name).\n\n        Also set field load_only and dump_only values if field_name was\n        specified in ``class Meta``.\n        \"\"\"\n        if field_name in self.load_only:\n            field_obj.load_only = True\n        if field_name in self.dump_only:\n            field_obj.dump_only = True\n        try:\n            field_obj._bind_to_schema(field_name, self)\n        except TypeError as error:\n            # Field declared as a class, not an instance. Ignore type checking because\n            # we handle unsupported arg types, i.e. this is dead code from\n            # the type checker's perspective.\n            if isinstance(field_obj, type) and issubclass(field_obj, base.FieldABC):\n                msg = (\n                    f'Field for \"{field_name}\" must be declared as a '\n                    \"Field instance, not a class. \"\n                    f'Did you mean \"fields.{field_obj.__name__}()\"?'  # type: ignore\n                )\n                raise TypeError(msg) from error\n            raise error\n        self.on_bind_field(field_name, field_obj)\n\n    @lru_cache(maxsize=8)  # noqa (https://github.com/PyCQA/flake8-bugbear/issues/310)\n    def _has_processors(self, tag) -> bool:\n        return bool(self._hooks[(tag, True)] or self._hooks[(tag, False)])\n\n    def _invoke_dump_processors(\n        self, tag: str, data, *, many: bool, original_data=None\n    ):\n        # The pass_many post-dump processors may do things like add an envelope, so\n        # invoke those after invoking the non-pass_many processors which will expect\n        # to get a list of items.\n        data = self._invoke_processors(\n            tag, pass_many=False, data=data, many=many, original_data=original_data\n        )\n        data = self._invoke_processors(\n            tag, pass_many=True, data=data, many=many, original_data=original_data\n        )\n        return data\n\n    def _invoke_load_processors(\n        self,\n        tag: str,\n        data,\n        *,\n        many: bool,\n        original_data,\n        partial: bool | types.StrSequenceOrSet | None,\n    ):\n        # This has to invert the order of the dump processors, so run the pass_many\n        # processors first.\n        data = self._invoke_processors(\n            tag,\n            pass_many=True,\n            data=data,\n            many=many,\n            original_data=original_data,\n            partial=partial,\n        )\n        data = self._invoke_processors(\n            tag,\n            pass_many=False,\n            data=data,\n            many=many,\n            original_data=original_data,\n            partial=partial,\n        )\n        return data\n\n    \ndef _invoke_field_validators(self, *, error_store: ErrorStore, data, many: bool):\n        for attr_name in self._hooks[VALIDATES]:\n            validator = getattr(self, attr_name)\n            validator_kwargs = validator.__marshmallow_hook__[VALIDATES]\n            field_name = validator_kwargs[\"field_name\"]\n\n            try:\n                field_obj = self.fields[field_name]\n            except KeyError as error:\n                if field_name in self.declared_fields:\n                    continue\n                raise ValueError(f'\"{field_name}\" field does not exist.') from error\n\n            data_key = (\n                field_obj.data_key if field_obj.data_key is not None else field_name\n            )\n            if many:\n                for idx, item in enumerate(data):\n                    try:\n                        value = item[field_obj.attribute or field_name]\n                    except KeyError:\n                        pass\n                    else:\n                        validated_value = self._call_and_store(\n                            getter_func=validator,\n                            data=value,\n                            field_name=data_key,\n                            error_store=error_store,\n                            index=(idx if self.opts.index_errors else None),\n                        )\n                        if validated_value is missing:\n                            data[idx].pop(field_name, None)\n            else:\n                try:\n                    value = data[field_obj.attribute or field_name]\n                except KeyError:\n                    pass\n                else:\n                    validated_value = self._call_and_store(\n                        getter_func=validator,\n                        data=value,\n                        field_name=data_key,\n                        error_store=error_store,\n                    )\n                    if validated_value is missing:\n                        data.pop(field_name, None)\n\n    def _invoke_schema_validators(\n        self,\n        *,\n        error_store: ErrorStore,\n        pass_many: bool,\n        data,\n        original_data,\n        many: bool,\n        partial: bool | types.StrSequenceOrSet | None,\n        field_errors: bool = False,\n    ):\n        for attr_name in self._hooks[(VALIDATES_SCHEMA, pass_many)]:\n            validator = getattr(self, attr_name)\n            validator_kwargs = validator.__marshmallow_hook__[\n                (VALIDATES_SCHEMA, pass_many)\n            ]\n            if field_errors and validator_kwargs[\"skip_on_field_errors\"]:\n                continue\n            pass_original = validator_kwargs.get(\"pass_original\", False)\n\n            if many and not pass_many:\n                for idx, (item, orig) in enumerate(zip(data, original_data)):\n                    self._run_validator(\n                        validator,\n                        item,\n                        original_data=orig,\n                        error_store=error_store,\n                        many=many,\n                        partial=partial,\n                        index=idx,\n                        pass_original=pass_original,\n                    )\n            else:\n                self._run_validator(\n                    validator,\n                    data,\n                    original_data=original_data,\n                    error_store=error_store,\n                    many=many,\n                    pass_original=pass_original,\n                    partial=partial,\n                )\n\n    def _invoke_processors(\n        self,\n        tag: str,\n        *,\n        pass_many: bool,\n        data,\n        many: bool,\n        original_data=None,\n        **kwargs,\n    ):\n        key = (tag, pass_many)\n        for attr_name in self._hooks[key]:\n            # This will be a bound method.\n            processor = getattr(self, attr_name)\n\n            processor_kwargs = processor.__marshmallow_hook__[key]\n            pass_original = processor_kwargs.get(\"pass_original\", False)\n\n            if many and not pass_many:\n                if pass_original:\n                    data = [\n                        processor(item, original, many=many, **kwargs)\n                        for item, original in zip(data, original_data)\n                    ]\n                else:\n                    data = [processor(item, many=many, **kwargs) for item in data]\n            else:\n                if pass_original:\n                    data = processor(data, original_data, many=many, **kwargs)\n                else:\n                    data = processor(data, many=many, **kwargs)\n        return data\n\n\nBaseSchema = Schema  # for backwards compatibility\n\n# Path: src/marshmallow/validate.py\n\"\"\"Validation classes for various types of data.\"\"\"\nfrom __future__ import annotations\n\nimport re\nimport typing\nfrom abc import ABC, abstractmethod\nfrom itertools import zip_longest\nfrom operator import attrgetter\n\nfrom marshmallow import types\nfrom marshmallow.exceptions import ValidationError\n\n_T = typing.TypeVar(\"_T\")\n\n\nclass Validator(ABC):\n    \"\"\"Abstract base class for validators.\n\n    .. note::\n        This class does not provide any validation behavior. It is only used to\n        add a useful `__repr__` implementation for validators.\n    \"\"\"\n\n    error = None  # type: str | None\n\n    def __repr__(self) -> str:\n        args = self._repr_args()\n        args = f\"{args}, \" if args else \"\"\n\n        return f\"<{self.__class__.__name__}({args}error={self.error!r})>\"\n\n    def _repr_args(self) -> str:\n        \"\"\"A string representation of the args passed to this validator. Used by\n        `__repr__`.\n        \"\"\"\n        return \"\"\n\n    @abstractmethod\n    def __call__(self, value: typing.Any) -> typing.Any:\n        ...\n\n\nclass And(Validator):\n    \"\"\"Compose multiple validators and combine their error messages.\n\n    Example: ::\n\n        from marshmallow import validate, ValidationError\n\n        def is_even(value):\n            if value % 2 != 0:\n                raise ValidationError(\"Not an even value.\")\n\n        validator = validate.And(validate.Range(min=0), is_even)\n        validator(-1)\n        # ValidationError: ['Must be greater than or equal to 0.', 'Not an even value.']\n\n    :param validators: Validators to combine.\n    :param error: Error message to use when a validator returns ``False``.\n    \"\"\"\n\n    default_error_message = \"Invalid value.\"\n\n    def __init__(self, *validators: types.Validator, error: str | None = None):\n        self.validators = tuple(validators)\n        self.error = error or self.default_error_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"validators={self.validators!r}\"\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        errors = []\n        kwargs = {}\n        for validator in self.validators:\n            try:\n                r = validator(value)\n                if not isinstance(validator, Validator) and r is False:\n                    raise ValidationError(self.error)\n            except ValidationError as err:\n                kwargs.update(err.kwargs)\n                if isinstance(err.messages, dict):\n                    errors.append(err.messages)\n                else:\n                    # FIXME : Get rid of cast\n                    errors.extend(typing.cast(list, err.messages))\n        if errors:\n            raise ValidationError(errors, **kwargs)\n        return value\n\n\nclass URL(Validator):\n    \"\"\"Validate a URL.\n\n    :param relative: Whether to allow relative URLs.\n    :param absolute: Whether to allow absolute URLs.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`.\n    :param schemes: Valid schemes. By default, ``http``, ``https``,\n        ``ftp``, and ``ftps`` are allowed.\n    :param require_tld: Whether to reject non-FQDN hostnames.\n    \"\"\"\n\n    class RegexMemoizer:\n        def __init__(self):\n            self._memoized = {}\n\n        def _regex_generator(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            hostname_variants = [\n                # a normal domain name, expressed in [A-Z0-9] chars with hyphens allowed only in the middle\n                # note that the regex will be compiled with IGNORECASE, so these are upper and lowercase chars\n                (\n                    r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n                    r\"(?:[A-Z]{2,6}\\.?|[A-Z0-9-]{2,}\\.?)\"\n                ),\n                # or the special string 'localhost'\n                r\"localhost\",\n                # or IPv4\n                r\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",\n                # or IPv6\n                r\"\\[[A-F0-9]*:[A-F0-9:]+\\]\",\n            ]\n            if not require_tld:\n                # allow dotless hostnames\n                hostname_variants.append(r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.?)\")\n\n            absolute_part = \"\".join(\n                (\n                    # scheme (e.g. 'https://', 'ftp://', etc)\n                    # this is validated separately against allowed schemes, so in the regex\n                    # we simply want to capture its existence\n                    r\"(?:[a-z0-9\\.\\-\\+]*)://\",\n                    # userinfo, for URLs encoding authentication\n                    # e.g. 'ftp://foo:bar@ftp.example.org/'\n                    r\"(?:(?:[a-z0-9\\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?\",\n                    # netloc, the hostname/domain part of the URL plus the optional port\n                    r\"(?:\",\n                    \"|\".join(hostname_variants),\n                    r\")\",\n                    r\"(?::\\d+)?\",\n                )\n            )\n            relative_part = r\"(?:/?|[/?]\\S+)\\Z\"\n\n            if relative:\n                if absolute:\n                    parts: tuple[str, ...] = (\n                        r\"^(\",\n                        absolute_part,\n                        r\")?\",\n                        relative_part,\n                    )\n                else:\n                    parts = (r\"^\", relative_part)\n            else:\n                parts = (r\"^\", absolute_part, relative_part)\n\n            return re.compile(\"\".join(parts), re.IGNORECASE)\n\n        def __call__(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            key = (relative, absolute, require_tld)\n            if key not in self._memoized:\n                self._memoized[key] = self._regex_generator(\n                    relative, absolute, require_tld\n                )\n\n            return self._memoized[key]\n\n    _regex = RegexMemoizer()\n\n    default_message = \"Not a valid URL.\"\n    default_schemes = {\"http\", \"https\", \"ftp\", \"ftps\"}\n\n    def __init__(\n        self,\n        *,\n        relative: bool = False,\n        absolute: bool = True,\n        schemes: types.StrSequenceOrSet | None = None,\n        require_tld: bool = True,\n        error: str | None = None,\n    ):\n        if not relative and not absolute:\n            raise ValueError(\n                \"URL validation cannot set both relative and absolute to False.\"\n            )\n        self.relative = relative\n        self.absolute = absolute\n        self.error = error or self.default_message  # type: str\n        self.schemes = schemes or self.default_schemes\n        self.require_tld = require_tld\n\n    def _repr_args(self) -> str:\n        return f\"relative={self.relative!r}, absolute={self.absolute!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(input=value)\n\n    def __call__(self, value: str) -> str:\n        message = self._format_error(value)\n        if not value:\n            raise ValidationError(message)\n\n        # Check first if the scheme is valid\n        if \"://\" in value:\n            scheme = value.split(\"://\")[0].lower()\n            if scheme not in self.schemes:\n                raise ValidationError(message)\n\n        regex = self._regex(self.relative, self.absolute, self.require_tld)\n\n        if not regex.search(value):\n            raise ValidationError(message)\n\n        return value\n\n\nclass Email(Validator):\n    \"\"\"Validate an email address.\n\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated with `{input}`.\n    \"\"\"\n\n    USER_REGEX = re.compile(\n        r\"(^[-!#$%&'*+/=?^`{}|~\\w]+(\\.[-!#$%&'*+/=?^`{}|~\\w]+)*\\Z\"  # dot-atom\n        # quoted-string\n        r'|^\"([\\001-\\010\\013\\014\\016-\\037!#-\\[\\]-\\177]'\n        r'|\\\\[\\001-\\011\\013\\014\\016-\\177])*\"\\Z)',\n        re.IGNORECASE | re.UNICODE,\n    )\n\n    DOMAIN_REGEX = re.compile(\n        # domain\n        r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n        r\"(?:[A-Z]{2,6}|[A-Z0-9-]{2,})\\Z\"\n        # literal form, ipv4 address (SMTP 4.1.3)\n        r\"|^\\[(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)\"\n        r\"(\\.(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)){3}\\]\\Z\",\n        re.IGNORECASE | re.UNICODE,\n    )\n\n    DOMAIN_WHITELIST = (\"localhost\",)\n\n    default_message = \"Not a valid email address.\"\n\n    def __init__(self, *, error: str | None = None):\n        self.error = error or self.default_message  # type: str\n\n    def _format_error(self, value: str) -> str:\n        return self.error.format(input=value)\n\n    def __call__(self, value: str) -> str:\n        message = self._format_error(value)\n\n        if not value or \"@\" not in value:\n            raise ValidationError(message)\n\n        user_part, domain_part = value.rsplit(\"@\", 1)\n\n        if not self.USER_REGEX.match(user_part):\n            raise ValidationError(message)\n\n        if domain_part not in self.DOMAIN_WHITELIST:\n            if not self.DOMAIN_REGEX.match(domain_part):\n                try:\n                    domain_part = domain_part.encode(\"idna\").decode(\"ascii\")\n                except UnicodeError:\n                    pass\n                else:\n                    if self.DOMAIN_REGEX.match(domain_part):\n                        return value\n                raise ValidationError(message)\n\n        return value\n\n\nclass Range(Validator):\n    \"\"\"Validator which succeeds if the value passed to it is within the specified\n    range. If ``min`` is not specified, or is specified as `None`,\n    no lower bound exists. If ``max`` is not specified, or is specified as `None`,\n    no upper bound exists. The inclusivity of the bounds (if they exist) is configurable.\n    If ``min_inclusive`` is not specified, or is specified as `True`, then\n    the ``min`` bound is included in the range. If ``max_inclusive`` is not specified,\n    or is specified as `True`, then the ``max`` bound is included in the range.\n\n    :param min: The minimum value (lower bound). If not provided, minimum\n        value will not be checked.\n    :param max: The maximum value (upper bound). If not provided, maximum\n        value will not be checked.\n    :param min_inclusive: Whether the `min` bound is included in the range.\n    :param max_inclusive: Whether the `max` bound is included in the range.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`, `{min}` and `{max}`.\n    \"\"\"\n\n    message_min = \"Must be {min_op} {{min}}.\"\n    message_max = \"Must be {max_op} {{max}}.\"\n    message_all = \"Must be {min_op} {{min}} and {max_op} {{max}}.\"\n\n    message_gte = \"greater than or equal to\"\n    message_gt = \"greater than\"\n    message_lte = \"less than or equal to\"\n    message_lt = \"less than\"\n\n    def __init__(\n        self,\n        min=None,\n        max=None,\n        *,\n        min_inclusive: bool = True,\n        max_inclusive: bool = True,\n        error: str | None = None,\n    ):\n        self.min = min\n        self.max = max\n        self.error = error\n        self.min_inclusive = min_inclusive\n        self.max_inclusive = max_inclusive\n\n        # interpolate messages based on bound inclusivity\n        self.message_min = self.message_min.format(\n            min_op=self.message_gte if self.min_inclusive else self.message_gt\n        )\n        self.message_max = self.message_max.format(\n            max_op=self.message_lte if self.max_inclusive else self.message_lt\n        )\n        self.message_all = self.message_all.format(\n            min_op=self.message_gte if self.min_inclusive else self.message_gt,\n            max_op=self.message_lte if self.max_inclusive else self.message_lt,\n        )\n\n    def _repr_args(self) -> str:\n        return \"min={!r}, max={!r}, min_inclusive={!r}, max_inclusive={!r}\".format(\n            self.min, self.max, self.min_inclusive, self.max_inclusive\n        )\n\n    def _format_error(self, value: _T, message: str) -> str:\n        return (self.error or message).format(input=value, min=self.min, max=self.max)\n\n    def __call__(self, value: _T) -> _T:\n        if self.min is not None and (\n            value < self.min if self.min_inclusive else value <= self.min\n        ):\n            message = self.message_min if self.max is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        if self.max is not None and (\n            value > self.max if self.max_inclusive else value >= self.max\n        ):\n            message = self.message_max if self.min is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        return value\n\n\nclass Length(Validator):\n    \"\"\"Validator which succeeds if the value passed to it has a\n    length between a minimum and maximum. Uses len(), so it\n    can work for strings, lists, or anything with length.\n\n    :param min: The minimum length. If not provided, minimum length\n        will not be checked.\n    :param max: The maximum length. If not provided, maximum length\n        will not be checked.\n    :param equal: The exact length. If provided, maximum and minimum\n        length will not be checked.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`, `{min}` and `{max}`.\n    \"\"\"\n\n    message_min = \"Shorter than minimum length {min}.\"\n    message_max = \"Longer than maximum length {max}.\"\n    message_all = \"Length must be between {min} and {max}.\"\n    message_equal = \"Length must be {equal}.\"\n\n    def __init__(\n        self,\n        min: int | None = None,\n        max: int | None = None,\n        *,\n        equal: int | None = None,\n        error: str | None = None,\n    ):\n        if equal is not None and any([min, max]):\n            raise ValueError(\n                \"The `equal` parameter was provided, maximum or \"\n                \"minimum parameter must not be provided.\"\n            )\n\n        self.min = min\n        self.max = max\n        self.error = error\n        self.equal = equal\n\n    def _repr_args(self) -> str:\n        return f\"min={self.min!r}, max={self.max!r}, equal={self.equal!r}\"\n\n    def _format_error(self, value: typing.Sized, message: str) -> str:\n        return (self.error or message).format(\n            input=value, min=self.min, max=self.max, equal=self.equal\n        )\n\n    def __call__(self, value: typing.Sized) -> typing.Sized:\n        length = len(value)\n\n        if self.equal is not None:\n            if length != self.equal:\n                raise ValidationError(self._format_error(value, self.message_equal))\n            return value\n\n        if self.min is not None and length < self.min:\n            message = self.message_min if self.max is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        if self.max is not None and length > self.max:\n            message = self.message_max if self.min is None else self.message_all\n            raise ValidationError(self._format_error(value, message))\n\n        return value\n\n\nclass Equal(Validator):\n    \"\"\"Validator which succeeds if the ``value`` passed to it is\n    equal to ``comparable``.\n\n    :param comparable: The object to compare to.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}` and `{other}`.\n    \"\"\"\n\n    default_message = \"Must be equal to {other}.\"\n\n    def __init__(self, comparable, *, error: str | None = None):\n        self.comparable = comparable\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"comparable={self.comparable!r}\"\n\n    def _format_error(self, value: _T) -> str:\n        return self.error.format(input=value, other=self.comparable)\n\n    def __call__(self, value: _T) -> _T:\n        if value != self.comparable:\n            raise ValidationError(self._format_error(value))\n        return value\n\n\nclass Regexp(Validator):\n    \"\"\"Validator which succeeds if the ``value`` matches ``regex``.\n\n    .. note::\n\n        Uses `re.match`, which searches for a match at the beginning of a string.\n\n    :param regex: The regular expression string to use. Can also be a compiled\n        regular expression pattern.\n    :param flags: The regexp flags to use, for example re.IGNORECASE. Ignored\n        if ``regex`` is not a string.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}` and `{regex}`.\n    \"\"\"\n\n    default_message = \"String does not match expected pattern.\"\n\n    def __init__(\n        self,\n        regex: str | bytes | typing.Pattern,\n        flags: int = 0,\n        *,\n        error: str | None = None,\n    ):\n        self.regex = (\n            re.compile(regex, flags) if isinstance(regex, (str, bytes)) else regex\n        )\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"regex={self.regex!r}\"\n\n    def _format_error(self, value: str | bytes) -> str:\n        return self.error.format(input=value, regex=self.regex.pattern)\n\n    @typing.overload\n    def __call__(self, value: str) -> str:\n        ...\n\n    @typing.overload\n    def __call__(self, value: bytes) -> bytes:\n        ...\n\n    def __call__(self, value):\n        if self.regex.match(value) is None:\n            raise ValidationError(self._format_error(value))\n\n        return value\n\n\nclass Predicate(Validator):\n    \"\"\"Call the specified ``method`` of the ``value`` object. The\n    validator succeeds if the invoked method returns an object that\n    evaluates to True in a Boolean context. Any additional keyword\n    argument will be passed to the method.\n\n    :param method: The name of the method to invoke.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}` and `{method}`.\n    :param kwargs: Additional keyword arguments to pass to the method.\n    \"\"\"\n\n    default_message = \"Invalid input.\"\n\n    def __init__(self, method: str, *, error: str | None = None, **kwargs):\n        self.method = method\n        self.error = error or self.default_message  # type: str\n        self.kwargs = kwargs\n\n    def _repr_args(self) -> str:\n        return f\"method={self.method!r}, kwargs={self.kwargs!r}\"\n\n    def _format_error(self, value: typing.Any) -> str:\n        return self.error.format(input=value, method=self.method)\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        method = getattr(value, self.method)\n\n        if not method(**self.kwargs):\n            raise ValidationError(self._format_error(value))\n\n        return value\n\n\nclass NoneOf(Validator):\n    \"\"\"Validator which fails if ``value`` is a member of ``iterable``.\n\n    :param iterable: A sequence of invalid values.\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated using `{input}` and `{values}`.\n    \"\"\"\n\n    default_message = \"Invalid input.\"\n\n    def __init__(self, iterable: typing.Iterable, *, error: str | None = None):\n        self.iterable = iterable\n        self.values_text = \", \".join(str(each) for each in self.iterable)\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"iterable={self.iterable!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(input=value, values=self.values_text)\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        try:\n            if value in self.iterable:\n                raise ValidationError(self._format_error(value))\n        except TypeError:\n            pass\n\n        return value\n\n\nclass OneOf(Validator):\n    \"\"\"Validator which succeeds if ``value`` is a member of ``choices``.\n\n    :param choices: A sequence of valid values.\n    :param labels: Optional sequence of labels to pair with the choices.\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated with `{input}`, `{choices}` and `{labels}`.\n    \"\"\"\n\n    default_message = \"Must be one of: {choices}.\"\n\n    def __init__(\n        self,\n        choices: typing.Iterable,\n        labels: typing.Iterable[str] | None = None,\n        *,\n        error: str | None = None,\n    ):\n        self.choices = choices\n        self.choices_text = \", \".join(str(choice) for choice in self.choices)\n        self.labels = labels if labels is not None else []\n        self.labels_text = \", \".join(str(label) for label in self.labels)\n        self.error = error or self.default_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"choices={self.choices!r}, labels={self.labels!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(\n            input=value, choices=self.choices_text, labels=self.labels_text\n        )\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        try:\n            if value not in self.choices:\n                raise ValidationError(self._format_error(value))\n        except TypeError as error:\n            raise ValidationError(self._format_error(value)) from error\n\n        return value\n\n    def options(\n        self,\n        valuegetter: str | typing.Callable[[typing.Any], typing.Any] = str,\n    ) -> typing.Iterable[tuple[typing.Any, str]]:\n        \"\"\"Return a generator over the (value, label) pairs, where value\n        is a string associated with each choice. This convenience method\n        is useful to populate, for instance, a form select field.\n\n        :param valuegetter: Can be a callable or a string. In the former case, it must\n            be a one-argument callable which returns the value of a\n            choice. In the latter case, the string specifies the name\n            of an attribute of the choice objects. Defaults to `str()`\n            or `str()`.\n        \"\"\"\n        valuegetter = valuegetter if callable(valuegetter) else attrgetter(valuegetter)\n        pairs = zip_longest(self.choices, self.labels, fillvalue=\"\")\n\n        return ((valuegetter(choice), label) for choice, label in pairs)\n\n\nclass ContainsOnly(OneOf):\n    \"\"\"Validator which succeeds if ``value`` is a sequence and each element\n    in the sequence is also in the sequence passed as ``choices``. Empty input\n    is considered valid.\n\n    :param iterable choices: Same as :class:`OneOf`.\n    :param iterable labels: Same as :class:`OneOf`.\n    :param str error: Same as :class:`OneOf`.\n\n    .. versionchanged:: 3.0.0b2\n        Duplicate values are considered valid.\n    .. versionchanged:: 3.0.0b2\n        Empty input is considered valid. Use `validate.Length(min=1) <marshmallow.validate.Length>`\n        to validate against empty inputs.\n    \"\"\"\n\n    default_message = \"One or more of the choices you made was not in: {choices}.\"\n\n    def _format_error(self, value) -> str:\n        value_text = \", \".join(str(val) for val in value)\n        return super()._format_error(value_text)\n\n    def __call__(self, value: typing.Sequence[_T]) -> typing.Sequence[_T]:\n        # We can't use set.issubset because does not handle unhashable types\n        for val in value:\n            if val not in self.choices:\n                raise ValidationError(self._format_error(value))\n        return value\n\n\nclass ContainsNoneOf(NoneOf):\n    \"\"\"Validator which fails if ``value`` is a sequence and any element\n    in the sequence is a member of the sequence passed as ``iterable``. Empty input\n    is considered valid.\n\n    :param iterable iterable: Same as :class:`NoneOf`.\n    :param str error: Same as :class:`NoneOf`.\n\n    .. versionadded:: 3.6.0\n    \"\"\"\n\n    default_message = \"One or more of the choices you made was in: {values}.\"\n\n    def _format_error(self, value) -> str:\n        value_text = \", \".join(str(val) for val in value)\n        return super()._format_error(value_text)\n\n    def __call__(self, value: typing.Sequence[_T]) -> typing.Sequence[_T]:\n        for val in value:\n            if val in self.iterable:\n                raise ValidationError(self._format_error(value))\n        return value\n\n# Path: src/marshmallow/fields.py\n\"\"\"Field classes for various types of data.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport copy\nimport datetime as dt\nimport decimal\nimport ipaddress\nimport math\nimport numbers\nimport typing\nimport uuid\nimport warnings\nfrom collections.abc import Mapping as _Mapping\nfrom enum import Enum as EnumType\n\nfrom marshmallow import class_registry, types, utils, validate\nfrom marshmallow.base import FieldABC, SchemaABC\nfrom marshmallow.exceptions import (\n    FieldInstanceResolutionError,\n    StringNotCollectionError,\n    ValidationError,\n)\nfrom marshmallow.utils import (\n    is_aware,\n    is_collection,\n    resolve_field_instance,\n)\nfrom marshmallow.utils import (\n    missing as missing_,\n)\nfrom marshmallow.validate import And, Length\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\n__all__ = [\n    \"Field\",\n    \"Raw\",\n    \"Nested\",\n    \"Mapping\",\n    \"Dict\",\n    \"List\",\n    \"Tuple\",\n    \"String\",\n    \"UUID\",\n    \"Number\",\n    \"Integer\",\n    \"Decimal\",\n    \"Boolean\",\n    \"Float\",\n    \"DateTime\",\n    \"NaiveDateTime\",\n    \"AwareDateTime\",\n    \"Time\",\n    \"Date\",\n    \"TimeDelta\",\n    \"Url\",\n    \"URL\",\n    \"Email\",\n    \"IP\",\n    \"IPv4\",\n    \"IPv6\",\n    \"IPInterface\",\n    \"IPv4Interface\",\n    \"IPv6Interface\",\n    \"Enum\",\n    \"Method\",\n    \"Function\",\n    \"Str\",\n    \"Bool\",\n    \"Int\",\n    \"Constant\",\n    \"Pluck\",\n]\n\n_T = typing.TypeVar(\"_T\")\n\n\nclass Field(FieldABC):\n    \"\"\"Basic field from which other fields should extend. It applies no\n    formatting by default, and should only be used in cases where\n    data does not need to be formatted before being serialized or deserialized.\n    On error, the name of the field will be returned.\n\n    :param dump_default: If set, this value will be used during serialization if the\n        input value is missing. If not set, the field will be excluded from the\n        serialized output if the input value is missing. May be a value or a callable.\n    :param load_default: Default deserialization value for the field if the field is not\n        found in the input data. May be a value or a callable.\n    :param data_key: The name of the dict key in the external representation, i.e.\n        the input of `load` and the output of `dump`.\n        If `None`, the key will match the name of the field.\n    :param attribute: The name of the key/attribute in the internal representation, i.e.\n        the output of `load` and the input of `dump`.\n        If `None`, the key/attribute will match the name of the field.\n        Note: This should only be used for very specific use cases such as\n        outputting multiple fields for a single attribute, or using keys/attributes\n        that are invalid variable names, unsuitable for field names. In most cases,\n        you should use ``data_key`` instead.\n    :param validate: Validator or collection of validators that are called\n...\n", "needle_token_start": 7062, "needle_token_end": 7537, "code_context_ntokens": 16143, "output": ["```python\ndef _invoke_field_validators(self, *, error_store: ErrorStore, data, many: bool):\n    for attr_name in self._hooks[VALIDATES]:\n        validator = getattr(self, attr_name)\n        validator_kwargs = validator.__marshmallow_hook__[VALIDATES]\n        field_name = validator_kwargs[\"field_name\"]\n\n        try:\n            field_obj = self.fields[field_name]\n        except KeyError as error:\n            if field_name in self.declared_fields:\n                continue\n            raise ValueError(f'\"{field_name}\" field does not exist.') from error\n\n        data_key = (\n            field_obj.data_key if field_obj.data_key is not None else field_name\n        )\n        if many:\n            for idx, item in enumerate(data):\n                try:\n                    value = item[field_obj.attribute or field_name]\n                except KeyError:\n                    pass\n                else:\n                    validated_value = self._call_and_store(\n                        getter_func=validator,\n                        data=value,\n                        field_name=data_key,\n                        error_store=error_store,\n                        index=(idx if self.opts.index_errors else None),\n                    )\n                    if validated_value is missing:\n                        data[idx].pop(field_name, None)\n        else:\n            try:\n                value = data[field_obj.attribute or field_name]\n            except KeyError:\n                pass\n            else:\n                validated_value = self._call_and_store(\n                    getter_func=validator,\n                    data=value,\n                    field_name=data_key,\n                    error_store=error_store,\n                )\n                if validated_value is missing:\n                    data.pop(field_name, None)\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "_run_validator", "language": "python", "path": "src/marshmallow/schema.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to execute a validation function on deserialized data, handling any validation errors that occur.\n2. **Input**: It accepts a validation function, the deserialized data to validate, the original data structure, an error storage mechanism, and flags indicating whether the data represents multiple items, whether partial data is allowed, and whether the original data should be passed to the validation function.\n3. **Output**: There is no direct output from this function; however, it updates the error storage with any validation errors encountered.\n4. **Procedure**: The function first determines whether to pass the original data structure along with the deserialized data to the validation function based on a specific flag. It then executes the validation function. If the validation function raises a validation error, this function captures the error and updates the error storage with the error details, including the specific field and index if applicable.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/marshmallow/utils.py\n\"\"\"Utility methods for marshmallow.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport datetime as dt\nimport functools\nimport inspect\nimport json\nimport re\nimport typing\nimport warnings\nfrom collections.abc import Mapping\nfrom email.utils import format_datetime, parsedate_to_datetime\nfrom pprint import pprint as py_pprint\n\nfrom marshmallow.base import FieldABC\nfrom marshmallow.exceptions import FieldInstanceResolutionError\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\nEXCLUDE = \"exclude\"\nINCLUDE = \"include\"\nRAISE = \"raise\"\n_UNKNOWN_VALUES = {EXCLUDE, INCLUDE, RAISE}\n\n\nclass _Missing:\n    def __bool__(self):\n        return False\n\n    def __copy__(self):\n        return self\n\n    def __deepcopy__(self, _):\n        return self\n\n    def __repr__(self):\n        return \"<marshmallow.missing>\"\n\n\n# Singleton value that indicates that a field's value is missing from input\n# dict passed to :meth:`Schema.load`. If the field's value is not required,\n# it's ``default`` value is used.\nmissing = _Missing()\n\n\ndef is_generator(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a generator\"\"\"\n    return inspect.isgeneratorfunction(obj) or inspect.isgenerator(obj)\n\n\ndef is_iterable_but_not_string(obj) -> bool:\n    \"\"\"Return True if ``obj`` is an iterable object that isn't a string.\"\"\"\n    return (hasattr(obj, \"__iter__\") and not hasattr(obj, \"strip\")) or is_generator(obj)\n\n\ndef is_collection(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a collection type, e.g list, tuple, queryset.\"\"\"\n    return is_iterable_but_not_string(obj) and not isinstance(obj, Mapping)\n\n\ndef is_instance_or_subclass(val, class_) -> bool:\n...\n# Path: src/marshmallow/schema.py\n\"\"\"The :class:`Schema` class, including its metaclass and options (class Meta).\"\"\"\nfrom __future__ import annotations\n\nimport copy\nimport datetime as dt\nimport decimal\nimport inspect\nimport json\nimport typing\nimport uuid\nimport warnings\nfrom abc import ABCMeta\nfrom collections import OrderedDict, defaultdict\nfrom collections.abc import Mapping\nfrom functools import lru_cache\n\nfrom marshmallow import base, class_registry, types\nfrom marshmallow import fields as ma_fields\nfrom marshmallow.decorators import (\n    POST_DUMP,\n    POST_LOAD,\n    PRE_DUMP,\n    PRE_LOAD,\n    VALIDATES,\n    VALIDATES_SCHEMA,\n)\nfrom marshmallow.error_store import ErrorStore\nfrom marshmallow.exceptions import StringNotCollectionError, ValidationError\nfrom marshmallow.orderedset import OrderedSet\nfrom marshmallow.utils import (\n    EXCLUDE,\n    INCLUDE,\n    RAISE,\n    get_value,\n    is_collection,\n    is_instance_or_subclass,\n    missing,\n    set_value,\n    validate_unknown_parameter_value,\n)\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\n_T = typing.TypeVar(\"_T\")\n\n\ndef _get_fields(attrs):\n    \"\"\"Get fields from a class\n\n    :param attrs: Mapping of class attributes\n    \"\"\"\n    return [\n        (field_name, field_value)\n        for field_name, field_value in attrs.items()\n        if is_instance_or_subclass(field_value, base.FieldABC)\n    ]\n\n\n# This function allows Schemas to inherit from non-Schema classes and ensures\n#   inheritance according to the MRO\ndef _get_fields_by_mro(klass):\n    \"\"\"Collect fields from a class, following its method resolution order. The\n    class itself is excluded from the search; only its parents are checked. Get\n    fields from ``_declared_fields`` if available, else use ``__dict__``.\n\n    :param type klass: Class whose fields to retrieve\n    \"\"\"\n    mro = inspect.getmro(klass)\n    # Loop over mro in reverse to maintain correct order of fields\n    return sum(\n        (\n            _get_fields(\n                getattr(base, \"_declared_fields\", base.__dict__),\n            )\n            for base in mro[:0:-1]\n        ),\n        [],\n    )\n\n\nclass SchemaMeta(ABCMeta):\n    \"\"\"Metaclass for the Schema class. Binds the declared fields to\n    a ``_declared_fields`` attribute, which is a dictionary mapping attribute\n    names to field objects. Also sets the ``opts`` class attribute, which is\n    the Schema class's ``class Meta`` options.\n    \"\"\"\n\n    def __new__(mcs, name, bases, attrs):\n        meta = attrs.get(\"Meta\")\n        ordered = getattr(meta, \"ordered\", False)\n        if not ordered:\n            # Inherit 'ordered' option\n            # Warning: We loop through bases instead of MRO because we don't\n            # yet have access to the class object\n            # (i.e. can't call super before we have fields)\n            for base_ in bases:\n                if hasattr(base_, \"Meta\") and hasattr(base_.Meta, \"ordered\"):\n                    ordered = base_.Meta.ordered\n                    break\n            else:\n                ordered = False\n        cls_fields = _get_fields(attrs)\n        # Remove fields from list of class attributes to avoid shadowing\n        # Schema attributes/methods in case of name conflict\n        for field_name, _ in cls_fields:\n            del attrs[field_name]\n        klass = super().__new__(mcs, name, bases, attrs)\n        inherited_fields = _get_fields_by_mro(klass)\n\n        meta = klass.Meta\n        # Set klass.opts in __new__ rather than __init__ so that it is accessible in\n        # get_declared_fields\n        klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered)\n        # Add fields specified in the `include` class Meta option\n        cls_fields += list(klass.opts.include.items())\n\n        # Assign _declared_fields on class\n        klass._declared_fields = mcs.get_declared_fields(\n            klass=klass,\n            cls_fields=cls_fields,\n            inherited_fields=inherited_fields,\n            dict_cls=dict,\n        )\n        return klass\n\n    @classmethod\n    def get_declared_fields(\n        mcs,\n        klass: type,\n        cls_fields: list,\n        inherited_fields: list,\n        dict_cls: type = dict,\n    ):\n        \"\"\"Returns a dictionary of field_name => `Field` pairs declared on the class.\n        This is exposed mainly so that plugins can add additional fields, e.g. fields\n        computed from class Meta options.\n\n        :param klass: The class object.\n        :param cls_fields: The fields declared on the class, including those added\n            by the ``include`` class Meta option.\n        :param inherited_fields: Inherited fields.\n        :param dict_cls: dict-like class to use for dict output Default to ``dict``.\n        \"\"\"\n        return dict_cls(inherited_fields + cls_fields)\n\n    def __init__(cls, name, bases, attrs):\n        super().__init__(name, bases, attrs)\n        if name and cls.opts.register:\n            class_registry.register(name, cls)\n        cls._hooks = cls.resolve_hooks()\n\n    def resolve_hooks(cls) -> dict[types.Tag, list[str]]:\n        \"\"\"Add in the decorated processors\n\n        By doing this after constructing the class, we let standard inheritance\n        do all the hard work.\n        \"\"\"\n        mro = inspect.getmro(cls)\n\n        hooks = defaultdict(list)  # type: typing.Dict[types.Tag, typing.List[str]]\n\n        for attr_name in dir(cls):\n            # Need to look up the actual descriptor, not whatever might be\n            # bound to the class. This needs to come from the __dict__ of the\n            # declaring class.\n            for parent in mro:\n                try:\n                    attr = parent.__dict__[attr_name]\n                except KeyError:\n                    continue\n                else:\n                    break\n            else:\n                # In case we didn't find the attribute and didn't break above.\n                # We should never hit this - it's just here for completeness\n                # to exclude the possibility of attr being undefined.\n                continue\n\n            try:\n                hook_config = attr.__marshmallow_hook__\n            except AttributeError:\n                pass\n            else:\n                for key in hook_config.keys():\n                    # Use name here so we can get the bound method later, in\n                    # case the processor was a descriptor or something.\n                    hooks[key].append(attr_name)\n\n        return hooks\n\n\nclass SchemaOpts:\n    \"\"\"class Meta options for the :class:`Schema`. Defines defaults.\"\"\"\n\n    def __init__(self, meta, ordered: bool = False):\n        self.fields = getattr(meta, \"fields\", ())\n        if not isinstance(self.fields, (list, tuple)):\n            raise ValueError(\"`fields` option must be a list or tuple.\")\n        self.additional = getattr(meta, \"additional\", ())\n        if not isinstance(self.additional, (list, tuple)):\n            raise ValueError(\"`additional` option must be a list or tuple.\")\n        if self.fields and self.additional:\n            raise ValueError(\n                \"Cannot set both `fields` and `additional` options\"\n                \" for the same Schema.\"\n            )\n        self.exclude = getattr(meta, \"exclude\", ())\n        if not isinstance(self.exclude, (list, tuple)):\n            raise ValueError(\"`exclude` must be a list or tuple.\")\n        self.dateformat = getattr(meta, \"dateformat\", None)\n        self.datetimeformat = getattr(meta, \"datetimeformat\", None)\n        self.timeformat = getattr(meta, \"timeformat\", None)\n        if hasattr(meta, \"json_module\"):\n            warnings.warn(\n                \"The json_module class Meta option is deprecated. Use render_module instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            render_module = getattr(meta, \"json_module\", json)\n        else:\n            render_module = json\n        self.render_module = getattr(meta, \"render_module\", render_module)\n        self.ordered = getattr(meta, \"ordered\", ordered)\n        self.index_errors = getattr(meta, \"index_errors\", True)\n        self.include = getattr(meta, \"include\", {})\n        self.load_only = getattr(meta, \"load_only\", ())\n        self.dump_only = getattr(meta, \"dump_only\", ())\n        self.unknown = validate_unknown_parameter_value(getattr(meta, \"unknown\", RAISE))\n        self.register = getattr(meta, \"register\", True)\n\n\nclass Schema(base.SchemaABC, metaclass=SchemaMeta):\n    \"\"\"Base schema class with which to define custom schemas.\n\n    Example usage:\n\n    .. code-block:: python\n\n        import datetime as dt\n        from dataclasses import dataclass\n\n        from marshmallow import Schema, fields\n\n\n        @dataclass\n        class Album:\n            title: str\n            release_date: dt.date\n\n\n        class AlbumSchema(Schema):\n            title = fields.Str()\n            release_date = fields.Date()\n\n\n        album = Album(\"Beggars Banquet\", dt.date(1968, 12, 6))\n        schema = AlbumSchema()\n        data = schema.dump(album)\n        data  # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'}\n\n    :param only: Whitelist of the declared fields to select when\n        instantiating the Schema. If None, all fields are used. Nested fields\n        can be represented with dot delimiters.\n    :param exclude: Blacklist of the declared fields to exclude\n        when instantiating the Schema. If a field appears in both `only` and\n        `exclude`, it is not used. Nested fields can be represented with dot\n        delimiters.\n    :param many: Should be set to `True` if ``obj`` is a collection\n        so that the object will be serialized to a list.\n    :param context: Optional context passed to :class:`fields.Method` and\n        :class:`fields.Function` fields.\n    :param load_only: Fields to skip during serialization (write-only fields)\n    :param dump_only: Fields to skip during deserialization (read-only fields)\n    :param partial: Whether to ignore missing fields and not require\n        any fields declared. Propagates down to ``Nested`` fields as well. If\n        its value is an iterable, only missing fields listed in that iterable\n        will be ignored. Use dot delimiters to specify nested fields.\n    :param unknown: Whether to exclude, include, or raise an error for unknown\n        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n\n    .. versionchanged:: 3.0.0\n        `prefix` parameter removed.\n\n    .. versionchanged:: 2.0.0\n        `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of\n        `marshmallow.decorators.validates_schema`,\n        `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`.\n        `__accessor__` and `__error_handler__` are deprecated. Implement the\n        `handle_error` and `get_attribute` methods instead.\n    \"\"\"\n\n    TYPE_MAPPING = {\n        str: ma_fields.String,\n        bytes: ma_fields.String,\n        dt.datetime: ma_fields.DateTime,\n        float: ma_fields.Float,\n        bool: ma_fields.Boolean,\n        tuple: ma_fields.Raw,\n        list: ma_fields.Raw,\n        set: ma_fields.Raw,\n        int: ma_fields.Integer,\n        uuid.UUID: ma_fields.UUID,\n        dt.time: ma_fields.Time,\n        dt.date: ma_fields.Date,\n        dt.timedelta: ma_fields.TimeDelta,\n        decimal.Decimal: ma_fields.Decimal,\n    }  # type: typing.Dict[type, typing.Type[ma_fields.Field]]\n    #: Overrides for default schema-level error messages\n    error_messages = {}  # type: typing.Dict[str, str]\n\n    _default_error_messages = {\n        \"type\": \"Invalid input type.\",\n        \"unknown\": \"Unknown field.\",\n    }  # type: typing.Dict[str, str]\n\n    OPTIONS_CLASS = SchemaOpts  # type: type\n\n    set_class = OrderedSet\n\n    # These get set by SchemaMeta\n    opts = None  # type: SchemaOpts\n    _declared_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n    _hooks = {}  # type: typing.Dict[types.Tag, typing.List[str]]\n\n    class Meta:\n        \"\"\"Options object for a Schema.\n\n        Example usage: ::\n\n            class Meta:\n                fields = (\"id\", \"email\", \"date_created\")\n                exclude = (\"password\", \"secret_attribute\")\n\n        Available options:\n\n        - ``fields``: Tuple or list of fields to include in the serialized result.\n        - ``additional``: Tuple or list of fields to include *in addition* to the\n            explicitly declared fields. ``additional`` and ``fields`` are\n            mutually-exclusive options.\n        - ``include``: Dictionary of additional fields to include in the schema. It is\n            usually better to define fields as class variables, but you may need to\n            use this option, e.g., if your fields are Python keywords. May be an\n            `OrderedDict`.\n        - ``exclude``: Tuple or list of fields to exclude in the serialized result.\n            Nested fields can be represented with dot delimiters.\n        - ``dateformat``: Default format for `Date <fields.Date>` fields.\n        - ``datetimeformat``: Default format for `DateTime <fields.DateTime>` fields.\n        - ``timeformat``: Default format for `Time <fields.Time>` fields.\n        - ``render_module``: Module to use for `loads <Schema.loads>` and `dumps <Schema.dumps>`.\n            Defaults to `json` from the standard library.\n        - ``ordered``: If `True`, output of `Schema.dump` will be a `collections.OrderedDict`.\n        - ``index_errors``: If `True`, errors dictionaries will include the index\n            of invalid items in a collection.\n        - ``load_only``: Tuple or list of fields to exclude from serialized results.\n        - ``dump_only``: Tuple or list of fields to exclude from deserialization\n        - ``unknown``: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        - ``register``: Whether to register the `Schema` with marshmallow's internal\n            class registry. Must be `True` if you intend to refer to this `Schema`\n            by class name in `Nested` fields. Only set this to `False` when memory\n            usage is critical. Defaults to `True`.\n        \"\"\"\n\n    def __init__(\n        self,\n        *,\n        only: types.StrSequenceOrSet | None = None,\n        exclude: types.StrSequenceOrSet = (),\n        many: bool = False,\n        context: dict | None = None,\n        load_only: types.StrSequenceOrSet = (),\n        dump_only: types.StrSequenceOrSet = (),\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        # Raise error if only or exclude is passed as string, not list of strings\n        if only is not None and not is_collection(only):\n            raise StringNotCollectionError('\"only\" should be a list of strings')\n        if not is_collection(exclude):\n            raise StringNotCollectionError('\"exclude\" should be a list of strings')\n        # copy declared fields from metaclass\n        self.declared_fields = copy.deepcopy(self._declared_fields)\n        self.many = many\n        self.only = only\n        self.exclude: set[typing.Any] | typing.MutableSet[typing.Any] = set(\n            self.opts.exclude\n        ) | set(exclude)\n        self.ordered = self.opts.ordered\n        self.load_only = set(load_only) or set(self.opts.load_only)\n        self.dump_only = set(dump_only) or set(self.opts.dump_only)\n        self.partial = partial\n        self.unknown = (\n            self.opts.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        self.context = context or {}\n        self._normalize_nested_options()\n        #: Dictionary mapping field_names -> :class:`Field` objects\n        self.fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.load_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.dump_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self._init_fields()\n        messages = {}\n        messages.update(self._default_error_messages)\n        for cls in reversed(self.__class__.__mro__):\n            messages.update(getattr(cls, \"error_messages\", {}))\n        messages.update(self.error_messages or {})\n        self.error_messages = messages\n\n    def __repr__(self) -> str:\n        return f\"<{self.__class__.__name__}(many={self.many})>\"\n\n    @property\n    def dict_class(self) -> type:\n        return OrderedDict if self.ordered else dict\n\n    @classmethod\n    def from_dict(\n        cls,\n        fields: dict[str, ma_fields.Field | type],\n        *,\n        name: str = \"GeneratedSchema\",\n    ) -> type:\n        \"\"\"Generate a `Schema` class given a dictionary of fields.\n\n        .. code-block:: python\n\n            from marshmallow import Schema, fields\n\n            PersonSchema = Schema.from_dict({\"name\": fields.Str()})\n            print(PersonSchema().load({\"name\": \"David\"}))  # => {'name': 'David'}\n\n        Generated schemas are not added to the class registry and therefore cannot\n        be referred to by name in `Nested` fields.\n\n        :param dict fields: Dictionary mapping field names to field instances.\n        :param str name: Optional name for the class, which will appear in\n            the ``repr`` for the class.\n\n        .. versionadded:: 3.0.0\n        \"\"\"\n        attrs = fields.copy()\n        attrs[\"Meta\"] = type(\n            \"GeneratedMeta\", (getattr(cls, \"Meta\", object),), {\"register\": False}\n        )\n        schema_cls = type(name, (cls,), attrs)\n        return schema_cls\n\n    ##### Override-able methods #####\n\n    def handle_error(\n        self, error: ValidationError, data: typing.Any, *, many: bool, **kwargs\n    ):\n        \"\"\"Custom error handler function for the schema.\n\n        :param error: The `ValidationError` raised during (de)serialization.\n        :param data: The original input data.\n        :param many: Value of ``many`` on dump or load.\n        :param partial: Value of ``partial`` on load.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0rc9\n            Receives `many` and `partial` (on deserialization) as keyword arguments.\n        \"\"\"\n        pass\n\n    def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any):\n        \"\"\"Defines how to pull values from an object to serialize.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0a1\n            Changed position of ``obj`` and ``attr``.\n        \"\"\"\n        return get_value(obj, attr, default)\n\n    ##### Serialization/Deserialization API #####\n\n    @staticmethod\n    def _call_and_store(getter_func, data, *, field_name, error_store, index=None):\n        \"\"\"Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.\n\n        :param callable getter_func: Function for getting the serialized/deserialized\n            value from ``data``.\n        :param data: The data passed to ``getter_func``.\n        :param str field_name: Field name.\n        :param int index: Index of the item being validated, if validating a collection,\n            otherwise `None`.\n        \"\"\"\n        try:\n            value = getter_func(data)\n        except ValidationError as error:\n            error_store.store_error(error.messages, field_name, index=index)\n            # When a Nested field fails validation, the marshalled data is stored\n            # on the ValidationError's valid_data attribute\n            return error.valid_data or missing\n        return value\n\n    def _serialize(self, obj: _T | typing.Iterable[_T], *, many: bool = False):\n        \"\"\"Serialize ``obj``.\n\n        :param obj: The object(s) to serialize.\n        :param bool many: `True` if ``data`` should be serialized as a collection.\n        :return: A dictionary of the serialized data\n\n        .. versionchanged:: 1.0.0\n            Renamed from ``marshal``.\n        \"\"\"\n        if many and obj is not None:\n            return [\n                self._serialize(d, many=False)\n                for d in typing.cast(typing.Iterable[_T], obj)\n            ]\n        ret = self.dict_class()\n        for attr_name, field_obj in self.dump_fields.items():\n            value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)\n            if value is missing:\n                continue\n            key = field_obj.data_key if field_obj.data_key is not None else attr_name\n            ret[key] = value\n        return ret\n\n    def dump(self, obj: typing.Any, *, many: bool | None = None):\n        \"\"\"Serialize an object to native Python data types according to this\n        Schema's fields.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: Serialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        .. versionchanged:: 3.0.0rc9\n            Validation no longer occurs upon serialization.\n        \"\"\"\n        many = self.many if many is None else bool(many)\n        if self._has_processors(PRE_DUMP):\n            processed_obj = self._invoke_dump_processors(\n                PRE_DUMP, obj, many=many, original_data=obj\n            )\n        else:\n            processed_obj = obj\n\n        result = self._serialize(processed_obj, many=many)\n\n        if self._has_processors(POST_DUMP):\n            result = self._invoke_dump_processors(\n                POST_DUMP, result, many=many, original_data=obj\n            )\n\n        return result\n\n    def dumps(self, obj: typing.Any, *args, many: bool | None = None, **kwargs):\n        \"\"\"Same as :meth:`dump`, except return a JSON-encoded string.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: A ``json`` string\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        \"\"\"\n        serialized = self.dump(obj, many=many)\n        return self.opts.render_module.dumps(serialized, *args, **kwargs)\n\n    def _deserialize(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        error_store: ErrorStore,\n        many: bool = False,\n        partial=None,\n        unknown=RAISE,\n        index=None,\n    ) -> _T | list[_T]:\n        \"\"\"Deserialize ``data``.\n\n        :param dict data: The data to deserialize.\n        :param ErrorStore error_store: Structure to store errors.\n        :param bool many: `True` if ``data`` should be deserialized as a collection.\n        :param bool|tuple partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        :param int index: Index of the item being serialized (for storing errors) if\n            serializing a collection, otherwise `None`.\n        :return: A dictionary of the deserialized data.\n        \"\"\"\n        index_errors = self.opts.index_errors\n        index = index if index_errors else None\n        if many:\n            if not is_collection(data):\n                error_store.store_error([self.error_messages[\"type\"]], index=index)\n                ret_l = []  # type: typing.List[_T]\n            else:\n                ret_l = [\n                    typing.cast(\n                        _T,\n                        self._deserialize(\n                            typing.cast(typing.Mapping[str, typing.Any], d),\n                            error_store=error_store,\n                            many=False,\n                            partial=partial,\n                            unknown=unknown,\n                            index=idx,\n                        ),\n                    )\n                    for idx, d in enumerate(data)\n                ]\n            return ret_l\n        ret_d = self.dict_class()\n        # Check data is a dict\n        if not isinstance(data, Mapping):\n            error_store.store_error([self.error_messages[\"type\"]], index=index)\n        else:\n            partial_is_collection = is_collection(partial)\n            for attr_name, field_obj in self.load_fields.items():\n                field_name = (\n                    field_obj.data_key if field_obj.data_key is not None else attr_name\n                )\n                raw_value = data.get(field_name, missing)\n                if raw_value is missing:\n                    # Ignore missing field if we're allowed to.\n                    if partial is True or (\n                        partial_is_collection and attr_name in partial\n                    ):\n                        continue\n                d_kwargs = {}\n                # Allow partial loading of nested schemas.\n                if partial_is_collection:\n                    prefix = field_name + \".\"\n                    len_prefix = len(prefix)\n                    sub_partial = [\n                        f[len_prefix:] for f in partial if f.startswith(prefix)\n                    ]\n                    d_kwargs[\"partial\"] = sub_partial\n                elif partial is not None:\n                    d_kwargs[\"partial\"] = partial\n\n                def getter(\n                    val, field_obj=field_obj, field_name=field_name, d_kwargs=d_kwargs\n                ):\n                    return field_obj.deserialize(\n                        val,\n                        field_name,\n                        data,\n                        **d_kwargs,\n                    )\n\n                value = self._call_and_store(\n                    getter_func=getter,\n                    data=raw_value,\n                    field_name=field_name,\n                    error_store=error_store,\n                    index=index,\n                )\n                if value is not missing:\n                    key = field_obj.attribute or attr_name\n                    set_value(ret_d, key, value)\n            if unknown != EXCLUDE:\n                fields = {\n                    field_obj.data_key if field_obj.data_key is not None else field_name\n                    for field_name, field_obj in self.load_fields.items()\n                }\n                for key in set(data) - fields:\n                    value = data[key]\n                    if unknown == INCLUDE:\n                        ret_d[key] = value\n                    elif unknown == RAISE:\n                        error_store.store_error(\n                            [self.error_messages[\"unknown\"]],\n                            key,\n                            (index if index_errors else None),\n                        )\n        return ret_d\n\n    def load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        \"\"\"Deserialize a data structure to an object defined by this Schema's fields.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        return self._do_load(\n            data, many=many, partial=partial, unknown=unknown, postprocess=True\n        )\n\n    def loads(\n        self,\n        json_data: str,\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        **kwargs,\n    ):\n        \"\"\"Same as :meth:`load`, except it takes a JSON string as input.\n\n        :param json_data: A JSON string of the data to deserialize.\n        :param many: Whether to deserialize `obj` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        data = self.opts.render_module.loads(json_data, **kwargs)\n        return self.load(data, many=many, partial=partial, unknown=unknown)\n\n    \ndef _run_validator(\n        self,\n        validator_func,\n        output,\n        *,\n        original_data,\n        error_store,\n        many,\n        partial,\n        pass_original,\n        index=None,\n    ):\n        try:\n            if pass_original:  # Pass original, raw data (before unmarshalling)\n                validator_func(output, original_data, partial=partial, many=many)\n            else:\n                validator_func(output, partial=partial, many=many)\n        except ValidationError as err:\n            error_store.store_error(err.messages, err.field_name, index=index)\n\n    def validate(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n    ) -> dict[str, list[str]]:\n        \"\"\"Validate `data` against the schema, returning a dictionary of\n        validation errors.\n\n        :param data: The data to validate.\n        :param many: Whether to validate `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :return: A dictionary of validation errors.\n\n        .. versionadded:: 1.1.0\n        \"\"\"\n        try:\n            self._do_load(data, many=many, partial=partial, postprocess=False)\n        except ValidationError as exc:\n            return typing.cast(typing.Dict[str, typing.List[str]], exc.messages)\n        return {}\n\n    ##### Private Helpers #####\n\n    def _do_load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        postprocess: bool = True,\n    ):\n        \"\"\"Deserialize `data`, returning the deserialized result.\n        This method is private API.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to validate required fields. If its\n            value is an iterable, only fields listed in that iterable will be\n            ignored will be allowed missing. If `True`, all fields will be allowed missing.\n            If `None`, the value for `self.partial` is used.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :param postprocess: Whether to run post_load methods..\n        :return: Deserialized data\n        \"\"\"\n        error_store = ErrorStore()\n        errors = {}  # type: dict[str, list[str]]\n        many = self.many if many is None else bool(many)\n        unknown = (\n            self.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        if partial is None:\n            partial = self.partial\n        # Run preprocessors\n        if self._has_processors(PRE_LOAD):\n            try:\n                processed_data = self._invoke_load_processors(\n                    PRE_LOAD, data, many=many, original_data=data, partial=partial\n                )\n            except ValidationError as err:\n                errors = err.normalized_messages()\n                result = None  # type: list | dict | None\n        else:\n            processed_data = data\n        if not errors:\n            # Deserialize data\n            result = self._deserialize(\n                processed_data,\n                error_store=error_store,\n                many=many,\n                partial=partial,\n                unknown=unknown,\n            )\n            # Run field-level validation\n            self._invoke_field_validators(\n                error_store=error_store, data=result, many=many\n            )\n            # Run schema-level validation\n            if self._has_processors(VALIDATES_SCHEMA):\n                field_errors = bool(error_store.errors)\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=True,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=False,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n            errors = error_store.errors\n            # Run post processors\n            if not errors and postprocess and self._has_processors(POST_LOAD):\n                try:\n                    result = self._invoke_load_processors(\n                        POST_LOAD,\n                        result,\n                        many=many,\n                        original_data=data,\n                        partial=partial,\n                    )\n                except ValidationError as err:\n                    errors = err.normalized_messages()\n        if errors:\n            exc = ValidationError(errors, data=data, valid_data=result)\n            self.handle_error(exc, data, many=many, partial=partial)\n            raise exc\n\n        return result\n\n    def _normalize_nested_options(self) -> None:\n        \"\"\"Apply then flatten nested schema options.\n        This method is private API.\n        \"\"\"\n        if self.only is not None:\n            # Apply the only option to nested fields.\n            self.__apply_nested_option(\"only\", self.only, \"intersection\")\n            # Remove the child field names from the only option.\n            self.only = self.set_class([field.split(\".\", 1)[0] for field in self.only])\n        if self.exclude:\n            # Apply the exclude option to nested fields.\n            self.__apply_nested_option(\"exclude\", self.exclude, \"union\")\n            # Remove the parent field names from the exclude option.\n            self.exclude = self.set_class(\n                [field for field in self.exclude if \".\" not in field]\n            )\n\n    def __apply_nested_option(self, option_name, field_names, set_operation) -> None:\n        \"\"\"Apply nested options to nested fields\"\"\"\n        # Split nested field names on the first dot.\n        nested_fields = [name.split(\".\", 1) for name in field_names if \".\" in name]\n        # Partition the nested field names by parent field.\n        nested_options = defaultdict(list)  # type: defaultdict\n        for parent, nested_names in nested_fields:\n            nested_options[parent].append(nested_names)\n        # Apply the nested field options.\n        for key, options in iter(nested_options.items()):\n            new_options = self.set_class(options)\n            original_options = getattr(self.declared_fields[key], option_name, ())\n            if original_options:\n                if set_operation == \"union\":\n                    new_options |= self.set_class(original_options)\n                if set_operation == \"intersection\":\n                    new_options &= self.set_class(original_options)\n            setattr(self.declared_fields[key], option_name, new_options)\n\n    def _init_fields(self) -> None:\n        \"\"\"Update self.fields, self.load_fields, and self.dump_fields based on schema options.\n        This method is private API.\n        \"\"\"\n        if self.opts.fields:\n            available_field_names = self.set_class(self.opts.fields)\n        else:\n            available_field_names = self.set_class(self.declared_fields.keys())\n            if self.opts.additional:\n                available_field_names |= self.set_class(self.opts.additional)\n\n        invalid_fields = self.set_class()\n\n        if self.only is not None:\n            # Return only fields specified in only option\n            field_names: typing.AbstractSet[typing.Any] = self.set_class(self.only)\n\n            invalid_fields |= field_names - available_field_names\n        else:\n            field_names = available_field_names\n\n        # If \"exclude\" option or param is specified, remove those fields.\n        if self.exclude:\n            # Note that this isn't available_field_names, since we want to\n            # apply \"only\" for the actual calculation.\n            field_names = field_names - self.exclude\n            invalid_fields |= self.exclude - available_field_names\n\n        if invalid_fields:\n            message = f\"Invalid fields for {self}: {invalid_fields}.\"\n            raise ValueError(message)\n\n        fields_dict = self.dict_class()\n        for field_name in field_names:\n            field_obj = self.declared_fields.get(field_name, ma_fields.Inferred())\n            self._bind_field(field_name, field_obj)\n            fields_dict[field_name] = field_obj\n\n        load_fields, dump_fields = self.dict_class(), self.dict_class()\n        for field_name, field_obj in fields_dict.items():\n            if not field_obj.dump_only:\n                load_fields[field_name] = field_obj\n            if not field_obj.load_only:\n                dump_fields[field_name] = field_obj\n\n        dump_data_keys = [\n            field_obj.data_key if field_obj.data_key is not None else name\n            for name, field_obj in dump_fields.items()\n        ]\n        if len(dump_data_keys) != len(set(dump_data_keys)):\n            data_keys_duplicates = {\n                x for x in dump_data_keys if dump_data_keys.count(x) > 1\n            }\n            raise ValueError(\n                \"The data_key argument for one or more fields collides \"\n                \"with another field's name or data_key argument. \"\n                \"Check the following field names and \"\n                f\"data_key arguments: {list(data_keys_duplicates)}\"\n            )\n        load_attributes = [obj.attribute or name for name, obj in load_fields.items()]\n        if len(load_attributes) != len(set(load_attributes)):\n            attributes_duplicates = {\n                x for x in load_attributes if load_attributes.count(x) > 1\n            }\n            raise ValueError(\n                \"The attribute argument for one or more fields collides \"\n                \"with another field's name or attribute argument. \"\n                \"Check the following field names and \"\n                f\"attribute arguments: {list(attributes_duplicates)}\"\n            )\n\n        self.fields = fields_dict\n        self.dump_fields = dump_fields\n        self.load_fields = load_fields\n\n    def on_bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None:\n        \"\"\"Hook to modify a field when it is bound to the `Schema`.\n\n        No-op by default.\n        \"\"\"\n        return None\n\n    def _bind_field(self, field_name: str, field_obj: ma_fields.Field) -> None:\n        \"\"\"Bind field to the schema, setting any necessary attributes on the\n        field (e.g. parent and name).\n\n        Also set field load_only and dump_only values if field_name was\n        specified in ``class Meta``.\n        \"\"\"\n        if field_name in self.load_only:\n            field_obj.load_only = True\n        if field_name in self.dump_only:\n            field_obj.dump_only = True\n        try:\n            field_obj._bind_to_schema(field_name, self)\n        except TypeError as error:\n            # Field declared as a class, not an instance. Ignore type checking because\n            # we handle unsupported arg types, i.e. this is dead code from\n            # the type checker's perspective.\n            if isinstance(field_obj, type) and issubclass(field_obj, base.FieldABC):\n                msg = (\n                    f'Field for \"{field_name}\" must be declared as a '\n                    \"Field instance, not a class. \"\n                    f'Did you mean \"fields.{field_obj.__name__}()\"?'  # type: ignore\n                )\n                raise TypeError(msg) from error\n            raise error\n        self.on_bind_field(field_name, field_obj)\n\n    @lru_cache(maxsize=8)  # noqa (https://github.com/PyCQA/flake8-bugbear/issues/310)\n    def _has_processors(self, tag) -> bool:\n        return bool(self._hooks[(tag, True)] or self._hooks[(tag, False)])\n\n    def _invoke_dump_processors(\n        self, tag: str, data, *, many: bool, original_data=None\n    ):\n        # The pass_many post-dump processors may do things like add an envelope, so\n        # invoke those after invoking the non-pass_many processors which will expect\n        # to get a list of items.\n        data = self._invoke_processors(\n            tag, pass_many=False, data=data, many=many, original_data=original_data\n        )\n        data = self._invoke_processors(\n            tag, pass_many=True, data=data, many=many, original_data=original_data\n        )\n        return data\n\n    def _invoke_load_processors(\n        self,\n        tag: str,\n        data,\n        *,\n        many: bool,\n        original_data,\n        partial: bool | types.StrSequenceOrSet | None,\n    ):\n        # This has to invert the order of the dump processors, so run the pass_many\n        # processors first.\n        data = self._invoke_processors(\n            tag,\n            pass_many=True,\n            data=data,\n            many=many,\n            original_data=original_data,\n            partial=partial,\n        )\n        data = self._invoke_processors(\n            tag,\n            pass_many=False,\n            data=data,\n            many=many,\n            original_data=original_data,\n            partial=partial,\n        )\n        return data\n\n    def _invoke_field_validators(self, *, error_store: ErrorStore, data, many: bool):\n        for attr_name in self._hooks[VALIDATES]:\n            validator = getattr(self, attr_name)\n            validator_kwargs = validator.__marshmallow_hook__[VALIDATES]\n            field_name = validator_kwargs[\"field_name\"]\n\n            try:\n                field_obj = self.fields[field_name]\n            except KeyError as error:\n                if field_name in self.declared_fields:\n                    continue\n                raise ValueError(f'\"{field_name}\" field does not exist.') from error\n\n            data_key = (\n                field_obj.data_key if field_obj.data_key is not None else field_name\n            )\n            if many:\n                for idx, item in enumerate(data):\n                    try:\n                        value = item[field_obj.attribute or field_name]\n                    except KeyError:\n                        pass\n                    else:\n                        validated_value = self._call_and_store(\n                            getter_func=validator,\n                            data=value,\n                            field_name=data_key,\n                            error_store=error_store,\n                            index=(idx if self.opts.index_errors else None),\n                        )\n                        if validated_value is missing:\n                            data[idx].pop(field_name, None)\n            else:\n                try:\n                    value = data[field_obj.attribute or field_name]\n                except KeyError:\n                    pass\n                else:\n                    validated_value = self._call_and_store(\n                        getter_func=validator,\n                        data=value,\n                        field_name=data_key,\n                        error_store=error_store,\n                    )\n                    if validated_value is missing:\n                        data.pop(field_name, None)\n\n    def _invoke_schema_validators(\n        self,\n        *,\n        error_store: ErrorStore,\n        pass_many: bool,\n        data,\n        original_data,\n        many: bool,\n        partial: bool | types.StrSequenceOrSet | None,\n        field_errors: bool = False,\n    ):\n        for attr_name in self._hooks[(VALIDATES_SCHEMA, pass_many)]:\n            validator = getattr(self, attr_name)\n            validator_kwargs = validator.__marshmallow_hook__[\n                (VALIDATES_SCHEMA, pass_many)\n            ]\n            if field_errors and validator_kwargs[\"skip_on_field_errors\"]:\n                continue\n            pass_original = validator_kwargs.get(\"pass_original\", False)\n\n            if many and not pass_many:\n                for idx, (item, orig) in enumerate(zip(data, original_data)):\n                    self._run_validator(\n                        validator,\n                        item,\n                        original_data=orig,\n                        error_store=error_store,\n                        many=many,\n                        partial=partial,\n                        index=idx,\n                        pass_original=pass_original,\n                    )\n            else:\n                self._run_validator(\n                    validator,\n                    data,\n                    original_data=original_data,\n                    error_store=error_store,\n                    many=many,\n                    pass_original=pass_original,\n                    partial=partial,\n                )\n\n    def _invoke_processors(\n        self,\n        tag: str,\n        *,\n        pass_many: bool,\n        data,\n        many: bool,\n        original_data=None,\n        **kwargs,\n    ):\n        key = (tag, pass_many)\n        for attr_name in self._hooks[key]:\n            # This will be a bound method.\n            processor = getattr(self, attr_name)\n\n            processor_kwargs = processor.__marshmallow_hook__[key]\n            pass_original = processor_kwargs.get(\"pass_original\", False)\n\n            if many and not pass_many:\n                if pass_original:\n                    data = [\n                        processor(item, original, many=many, **kwargs)\n                        for item, original in zip(data, original_data)\n                    ]\n                else:\n                    data = [processor(item, many=many, **kwargs) for item in data]\n            else:\n                if pass_original:\n                    data = processor(data, original_data, many=many, **kwargs)\n                else:\n                    data = processor(data, many=many, **kwargs)\n        return data\n\n\nBaseSchema = Schema  # for backwards compatibility\n\n# Path: src/marshmallow/validate.py\n\"\"\"Validation classes for various types of data.\"\"\"\nfrom __future__ import annotations\n\nimport re\nimport typing\nfrom abc import ABC, abstractmethod\nfrom itertools import zip_longest\nfrom operator import attrgetter\n\nfrom marshmallow import types\nfrom marshmallow.exceptions import ValidationError\n\n_T = typing.TypeVar(\"_T\")\n\n\nclass Validator(ABC):\n    \"\"\"Abstract base class for validators.\n\n    .. note::\n        This class does not provide any validation behavior. It is only used to\n        add a useful `__repr__` implementation for validators.\n    \"\"\"\n\n    error = None  # type: str | None\n\n    def __repr__(self) -> str:\n        args = self._repr_args()\n        args = f\"{args}, \" if args else \"\"\n\n        return f\"<{self.__class__.__name__}({args}error={self.error!r})>\"\n\n    def _repr_args(self) -> str:\n        \"\"\"A string representation of the args passed to this validator. Used by\n        `__repr__`.\n        \"\"\"\n        return \"\"\n\n    @abstractmethod\n    def __call__(self, value: typing.Any) -> typing.Any:\n        ...\n\n\nclass And(Validator):\n    \"\"\"Compose multiple validators and combine their error messages.\n\n    Example: ::\n\n        from marshmallow import validate, ValidationError\n\n        def is_even(value):\n            if value % 2 != 0:\n                raise ValidationError(\"Not an even value.\")\n\n        validator = validate.And(validate.Range(min=0), is_even)\n        validator(-1)\n        # ValidationError: ['Must be greater than or equal to 0.', 'Not an even value.']\n\n    :param validators: Validators to combine.\n    :param error: Error message to use when a validator returns ``False``.\n    \"\"\"\n\n    default_error_message = \"Invalid value.\"\n\n    def __init__(self, *validators: types.Validator, error: str | None = None):\n        self.validators = tuple(validators)\n        self.error = error or self.default_error_message  # type: str\n\n    def _repr_args(self) -> str:\n        return f\"validators={self.validators!r}\"\n\n    def __call__(self, value: typing.Any) -> typing.Any:\n        errors = []\n        kwargs = {}\n        for validator in self.validators:\n            try:\n                r = validator(value)\n                if not isinstance(validator, Validator) and r is False:\n                    raise ValidationError(self.error)\n            except ValidationError as err:\n                kwargs.update(err.kwargs)\n                if isinstance(err.messages, dict):\n                    errors.append(err.messages)\n                else:\n                    # FIXME : Get rid of cast\n                    errors.extend(typing.cast(list, err.messages))\n        if errors:\n            raise ValidationError(errors, **kwargs)\n        return value\n\n\nclass URL(Validator):\n    \"\"\"Validate a URL.\n\n    :param relative: Whether to allow relative URLs.\n    :param absolute: Whether to allow absolute URLs.\n    :param error: Error message to raise in case of a validation error.\n        Can be interpolated with `{input}`.\n    :param schemes: Valid schemes. By default, ``http``, ``https``,\n        ``ftp``, and ``ftps`` are allowed.\n    :param require_tld: Whether to reject non-FQDN hostnames.\n    \"\"\"\n\n    class RegexMemoizer:\n        def __init__(self):\n            self._memoized = {}\n\n        def _regex_generator(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            hostname_variants = [\n                # a normal domain name, expressed in [A-Z0-9] chars with hyphens allowed only in the middle\n                # note that the regex will be compiled with IGNORECASE, so these are upper and lowercase chars\n                (\n                    r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n                    r\"(?:[A-Z]{2,6}\\.?|[A-Z0-9-]{2,}\\.?)\"\n                ),\n                # or the special string 'localhost'\n                r\"localhost\",\n                # or IPv4\n                r\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",\n                # or IPv6\n                r\"\\[[A-F0-9]*:[A-F0-9:]+\\]\",\n            ]\n            if not require_tld:\n                # allow dotless hostnames\n                hostname_variants.append(r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.?)\")\n\n            absolute_part = \"\".join(\n                (\n                    # scheme (e.g. 'https://', 'ftp://', etc)\n                    # this is validated separately against allowed schemes, so in the regex\n                    # we simply want to capture its existence\n                    r\"(?:[a-z0-9\\.\\-\\+]*)://\",\n                    # userinfo, for URLs encoding authentication\n                    # e.g. 'ftp://foo:bar@ftp.example.org/'\n                    r\"(?:(?:[a-z0-9\\-._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?\",\n                    # netloc, the hostname/domain part of the URL plus the optional port\n                    r\"(?:\",\n                    \"|\".join(hostname_variants),\n                    r\")\",\n                    r\"(?::\\d+)?\",\n                )\n            )\n            relative_part = r\"(?:/?|[/?]\\S+)\\Z\"\n\n            if relative:\n                if absolute:\n                    parts: tuple[str, ...] = (\n                        r\"^(\",\n                        absolute_part,\n                        r\")?\",\n                        relative_part,\n                    )\n                else:\n                    parts = (r\"^\", relative_part)\n            else:\n                parts = (r\"^\", absolute_part, relative_part)\n\n            return re.compile(\"\".join(parts), re.IGNORECASE)\n\n        def __call__(\n            self, relative: bool, absolute: bool, require_tld: bool\n        ) -> typing.Pattern:\n            key = (relative, absolute, require_tld)\n            if key not in self._memoized:\n                self._memoized[key] = self._regex_generator(\n                    relative, absolute, require_tld\n                )\n\n            return self._memoized[key]\n\n    _regex = RegexMemoizer()\n\n    default_message = \"Not a valid URL.\"\n    default_schemes = {\"http\", \"https\", \"ftp\", \"ftps\"}\n\n    def __init__(\n        self,\n        *,\n        relative: bool = False,\n        absolute: bool = True,\n        schemes: types.StrSequenceOrSet | None = None,\n        require_tld: bool = True,\n        error: str | None = None,\n    ):\n        if not relative and not absolute:\n            raise ValueError(\n                \"URL validation cannot set both relative and absolute to False.\"\n            )\n        self.relative = relative\n        self.absolute = absolute\n        self.error = error or self.default_message  # type: str\n        self.schemes = schemes or self.default_schemes\n        self.require_tld = require_tld\n\n    def _repr_args(self) -> str:\n        return f\"relative={self.relative!r}, absolute={self.absolute!r}\"\n\n    def _format_error(self, value) -> str:\n        return self.error.format(input=value)\n\n    def __call__(self, value: str) -> str:\n        message = self._format_error(value)\n        if not value:\n            raise ValidationError(message)\n\n        # Check first if the scheme is valid\n        if \"://\" in value:\n            scheme = value.split(\"://\")[0].lower()\n            if scheme not in self.schemes:\n                raise ValidationError(message)\n\n        regex = self._regex(self.relative, self.absolute, self.require_tld)\n\n        if not regex.search(value):\n            raise ValidationError(message)\n\n        return value\n\n\nclass Email(Validator):\n    \"\"\"Validate an email address.\n\n    :param error: Error message to raise in case of a validation error. Can be\n        interpolated with `{input}`.\n    \"\"\"\n\n    USER_REGEX = re.compile(\n        r\"(^[-!#$%&'*+/=?^`{}|~\\w]+(\\.[-!#$%&'*+/=?^`{}|~\\w]+)*\\Z\"  # dot-atom\n        # quoted-string\n        r'|^\"([\\001-\\010\\013\\014\\016-\\037!#-\\[\\]-\\177]'\n        r'|\\\\[\\001-\\011\\013\\014\\016-\\177])*\"\\Z)',\n        re.IGNORECASE | re.UNICODE,\n    )\n\n    DOMAIN_REGEX = re.compile(\n        # domain\n        r\"(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\\.)+\"\n        r\"(?:[A-Z]{2,6}|[A-Z0-9-]{2,})\\Z\"\n        # literal form, ipv4 address (SMTP 4.1.3)\n        r\"|^\\[(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)\"\n...\n", "needle_token_start": 8818, "needle_token_end": 8979, "code_context_ntokens": 16170, "output": ["```python\ndef _run_validator(\n        self,\n        validator_func,\n        output,\n        *,\n        original_data,\n        error_store,\n        many,\n        partial,\n        pass_original,\n        index=None,\n    ):\n        try:\n            if pass_original:  # Pass original, raw data (before unmarshalling)\n                validator_func(output, original_data, partial=partial, many=many)\n            else:\n                validator_func(output, partial=partial, many=many)\n        except ValidationError as err:\n            error_store.store_error(err.messages, err.field_name, index=index)\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "from_iso_time", "language": "python", "path": "src/marshmallow/utils.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To convert a string representing a time in ISO8601 format into a time object.\n2. **Input**: A string formatted according to ISO8601 standards, representing a time without timezone information.\n3. **Output**: A time object representing the time specified in the input string.\n4. **Procedure**: The function first matches the input string against a regular expression designed to parse ISO8601 time strings. If the string does not match the pattern, it raises a ValueError indicating the string is not valid. If the string matches, it extracts components such as hour, minute, second, and microsecond from the string. These extracted values are then converted into integers (with microseconds adjusted for precision) and used to construct a time object, which is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/marshmallow/decorators.py\n\"\"\"Decorators for registering schema pre-processing and post-processing methods.\nThese should be imported from the top-level `marshmallow` module.\n\nMethods decorated with\n`pre_load <marshmallow.decorators.pre_load>`, `post_load <marshmallow.decorators.post_load>`,\n`pre_dump <marshmallow.decorators.pre_dump>`, `post_dump <marshmallow.decorators.post_dump>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``many`` as a keyword argument. In addition, `pre_load <marshmallow.decorators.pre_load>`,\n`post_load <marshmallow.decorators.post_load>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``partial``. If you don't need these arguments, add ``**kwargs`` to your method\nsignature.\n\n\nExample: ::\n\n    from marshmallow import (\n        Schema, pre_load, pre_dump, post_load, validates_schema,\n        validates, fields, ValidationError\n    )\n\n    class UserSchema(Schema):\n\n        email = fields.Str(required=True)\n        age = fields.Integer(required=True)\n\n        @post_load\n        def lowerstrip_email(self, item, many, **kwargs):\n            item['email'] = item['email'].lower().strip()\n            return item\n\n        @pre_load(pass_many=True)\n        def remove_envelope(self, data, many, **kwargs):\n            namespace = 'results' if many else 'result'\n            return data[namespace]\n\n        @post_dump(pass_many=True)\n        def add_envelope(self, data, many, **kwargs):\n            namespace = 'results' if many else 'result'\n            return {namespace: data}\n\n        @validates_schema\n        def validate_email(self, data, **kwargs):\n            if len(data['email']) < 3:\n                raise ValidationError('Email must be more than 3 characters', 'email')\n\n        @validates('age')\n        def validate_age(self, data, **kwargs):\n            if data < 14:\n                raise ValidationError('Too young!')\n\n.. note::\n    These decorators only work with instance methods. Class and static\n    methods are not supported.\n\n.. warning::\n    The invocation order of decorated methods of the same type is not guaranteed.\n    If you need to guarantee order of different processing steps, you should put\n    them in the same processing method.\n\"\"\"\nfrom __future__ import annotations\n\nimport functools\nfrom typing import Any, Callable, cast\n\nPRE_DUMP = \"pre_dump\"\nPOST_DUMP = \"post_dump\"\nPRE_LOAD = \"pre_load\"\nPOST_LOAD = \"post_load\"\nVALIDATES = \"validates\"\nVALIDATES_SCHEMA = \"validates_schema\"\n\n\nclass MarshmallowHook:\n    __marshmallow_hook__: dict[tuple[str, bool] | str, Any] | None = None\n\n\ndef validates(field_name: str) -> Callable[..., Any]:\n    \"\"\"Register a field validator.\n\n    :param str field_name: Name of the field that the method validates.\n    \"\"\"\n    return set_hook(None, VALIDATES, field_name=field_name)\n\n\ndef validates_schema(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n    skip_on_field_errors: bool = True,\n) -> Callable[..., Any]:\n    \"\"\"Register a schema-level validator.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.validate` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before unmarshalling) will be passed as\n    an additional argument to the method.\n\n    If ``skip_on_field_errors=True``, this validation method will be skipped whenever\n    validation errors have been detected when validating fields.\n\n    .. versionchanged:: 3.0.0b1\n        ``skip_on_field_errors`` defaults to `True`.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(\n        fn,\n        (VALIDATES_SCHEMA, pass_many),\n        pass_original=pass_original,\n        skip_on_field_errors=skip_on_field_errors,\n    )\n\n\ndef pre_dump(\n    fn: Callable[..., Any] | None = None, pass_many: bool = False\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke before serializing an object. The method\n    receives the object to be serialized and returns the processed object.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.dump` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    .. versionchanged:: 3.0.0\n        ``many`` is always passed as a keyword arguments to the decorated method.\n    \"\"\"\n    return set_hook(fn, (PRE_DUMP, pass_many))\n\n\ndef post_dump(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke after serializing an object. The method\n    receives the serialized object and returns the processed object.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.dump` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before serializing) will be passed as\n    an additional argument to the method.\n\n    .. versionchanged:: 3.0.0\n        ``many`` is always passed as a keyword arguments to the decorated method.\n    \"\"\"\n    return set_hook(fn, (POST_DUMP, pass_many), pass_original=pass_original)\n\n\ndef pre_load(\n    fn: Callable[..., Any] | None = None, pass_many: bool = False\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke before deserializing an object. The method\n    receives the data to be deserialized and returns the processed data.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.load` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(fn, (PRE_LOAD, pass_many))\n\n\ndef post_load(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke after deserializing an object. The method\n    receives the deserialized data and returns the processed data.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.load` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before deserializing) will be passed as\n    an additional argument to the method.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(fn, (POST_LOAD, pass_many), pass_original=pass_original)\n\n\ndef set_hook(\n    fn: Callable[..., Any] | None, key: tuple[str, bool] | str, **kwargs: Any\n) -> Callable[..., Any]:\n    \"\"\"Mark decorated function as a hook to be picked up later.\n    You should not need to use this method directly.\n\n    .. note::\n        Currently only works with functions and instance methods. Class and\n        static methods are not supported.\n\n    :return: Decorated function if supplied, else this decorator with its args\n        bound.\n    \"\"\"\n    # Allow using this as either a decorator or a decorator factory.\n    if fn is None:\n        return functools.partial(set_hook, key=key, **kwargs)\n\n    # Set a __marshmallow_hook__ attribute instead of wrapping in some class,\n    # because I still want this to end up as a normal (unbound) method.\n    function = cast(MarshmallowHook, fn)\n    try:\n        hook_config = function.__marshmallow_hook__\n    except AttributeError:\n        function.__marshmallow_hook__ = hook_config = {}\n    # Also save the kwargs for the tagged function on\n    # __marshmallow_hook__, keyed by (<tag>, <pass_many>)\n    if hook_config is not None:\n        hook_config[key] = kwargs\n\n    return fn\n\n# Path: src/marshmallow/exceptions.py\n\"\"\"Exception classes for marshmallow-related errors.\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\n# Key used for schema-level validation errors\nSCHEMA = \"_schema\"\n\n\nclass MarshmallowError(Exception):\n    \"\"\"Base class for all marshmallow-related errors.\"\"\"\n\n\nclass ValidationError(MarshmallowError):\n    \"\"\"Raised when validation fails on a field or schema.\n\n    Validators and custom fields should raise this exception.\n\n    :param message: An error message, list of error messages, or dict of\n        error messages. If a dict, the keys are subitems and the values are error messages.\n    :param field_name: Field name to store the error on.\n        If `None`, the error is stored as schema-level error.\n    :param data: Raw input data.\n    :param valid_data: Valid (de)serialized data.\n    \"\"\"\n\n    def __init__(\n        self,\n        message: str | list | dict,\n        field_name: str = SCHEMA,\n        data: typing.Mapping[str, typing.Any]\n        | typing.Iterable[typing.Mapping[str, typing.Any]]\n        | None = None,\n        valid_data: list[dict[str, typing.Any]] | dict[str, typing.Any] | None = None,\n        **kwargs,\n    ):\n        self.messages = [message] if isinstance(message, (str, bytes)) else message\n        self.field_name = field_name\n        self.data = data\n        self.valid_data = valid_data\n        self.kwargs = kwargs\n        super().__init__(message)\n\n    def normalized_messages(self):\n        if self.field_name == SCHEMA and isinstance(self.messages, dict):\n            return self.messages\n        return {self.field_name: self.messages}\n\n    @property\n    def messages_dict(self) -> dict[str, typing.Any]:\n        if not isinstance(self.messages, dict):\n            raise TypeError(\n                \"cannot access 'messages_dict' when 'messages' is of type \"\n                + type(self.messages).__name__\n            )\n        return self.messages\n\n\nclass RegistryError(NameError):\n    \"\"\"Raised when an invalid operation is performed on the serializer\n    class registry.\n    \"\"\"\n\n\nclass StringNotCollectionError(MarshmallowError, TypeError):\n    \"\"\"Raised when a string is passed when a list of strings is expected.\"\"\"\n\n\nclass FieldInstanceResolutionError(MarshmallowError, TypeError):\n    \"\"\"Raised when schema to instantiate is neither a Schema class nor an instance.\"\"\"\n\n# Path: src/marshmallow/base.py\n\"\"\"Abstract base classes.\n\nThese are necessary to avoid circular imports between schema.py and fields.py.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nfrom abc import ABC, abstractmethod\n\n\nclass FieldABC(ABC):\n    \"\"\"Abstract base class from which all Field classes inherit.\"\"\"\n\n    parent = None\n    name = None\n    root = None\n\n    @abstractmethod\n    def serialize(self, attr, obj, accessor=None):\n        pass\n\n    @abstractmethod\n    def deserialize(self, value):\n        pass\n\n    @abstractmethod\n    def _serialize(self, value, attr, obj, **kwargs):\n        pass\n\n    @abstractmethod\n    def _deserialize(self, value, attr, data, **kwargs):\n        pass\n\n\nclass SchemaABC(ABC):\n    \"\"\"Abstract base class from which all Schemas inherit.\"\"\"\n\n    @abstractmethod\n    def dump(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def dumps(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def load(self, data, *, many: bool | None = None, partial=None, unknown=None):\n        pass\n\n    @abstractmethod\n    def loads(\n        self,\n        json_data,\n        *,\n        many: bool | None = None,\n        partial=None,\n        unknown=None,\n        **kwargs,\n    ):\n        pass\n\n# Path: src/marshmallow/class_registry.py\n\"\"\"A registry of :class:`Schema <marshmallow.Schema>` classes. This allows for string\nlookup of schemas, which may be used with\nclass:`fields.Nested <marshmallow.fields.Nested>`.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\nfrom marshmallow.exceptions import RegistryError\n\nif typing.TYPE_CHECKING:\n    from marshmallow import Schema\n\n    SchemaType = typing.Type[Schema]\n\n# {\n#   <class_name>: <list of class objects>\n#   <module_path_to_class>: <list of class objects>\n# }\n_registry = {}  # type: dict[str, list[SchemaType]]\n\n\ndef register(classname: str, cls: SchemaType) -> None:\n    \"\"\"Add a class to the registry of serializer classes. When a class is\n    registered, an entry for both its classname and its full, module-qualified\n    path are added to the registry.\n\n    Example: ::\n\n        class MyClass:\n            pass\n\n        register('MyClass', MyClass)\n        # Registry:\n        # {\n        #   'MyClass': [path.to.MyClass],\n        #   'path.to.MyClass': [path.to.MyClass],\n        # }\n\n    \"\"\"\n    # Module where the class is located\n    module = cls.__module__\n    # Full module path to the class\n    # e.g. user.schemas.UserSchema\n    fullpath = \".\".join([module, classname])\n    # If the class is already registered; need to check if the entries are\n    # in the same module as cls to avoid having multiple instances of the same\n    # class in the registry\n    if classname in _registry and not any(\n        each.__module__ == module for each in _registry[classname]\n    ):\n        _registry[classname].append(cls)\n    elif classname not in _registry:\n        _registry[classname] = [cls]\n\n    # Also register the full path\n    if fullpath not in _registry:\n        _registry.setdefault(fullpath, []).append(cls)\n    else:\n        # If fullpath does exist, replace existing entry\n        _registry[fullpath] = [cls]\n    return None\n\n\ndef get_class(classname: str, all: bool = False) -> list[SchemaType] | SchemaType:\n    \"\"\"Retrieve a class from the registry.\n\n    :raises: marshmallow.exceptions.RegistryError if the class cannot be found\n        or if there are multiple entries for the given class name.\n    \"\"\"\n    try:\n        classes = _registry[classname]\n    except KeyError as error:\n        raise RegistryError(\n            f\"Class with name {classname!r} was not found. You may need \"\n            \"to import the class.\"\n        ) from error\n    if len(classes) > 1:\n        if all:\n            return _registry[classname]\n        raise RegistryError(\n            f\"Multiple classes with name {classname!r} \"\n            \"were found. Please use the full, \"\n            \"module-qualified path.\"\n        )\n    else:\n        return _registry[classname][0]\n\n# Path: src/marshmallow/error_store.py\n\"\"\"Utilities for storing collections of error messages.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\n\nfrom marshmallow.exceptions import SCHEMA\n\n\nclass ErrorStore:\n    def __init__(self):\n        #: Dictionary of errors stored during serialization\n        self.errors = {}\n\n    def store_error(self, messages, field_name=SCHEMA, index=None):\n        # field error  -> store/merge error messages under field name key\n        # schema error -> if string or list, store/merge under _schema key\n        #              -> if dict, store/merge with other top-level keys\n        if field_name != SCHEMA or not isinstance(messages, dict):\n            messages = {field_name: messages}\n        if index is not None:\n            messages = {index: messages}\n        self.errors = merge_errors(self.errors, messages)\n\n\ndef merge_errors(errors1, errors2):\n    \"\"\"Deeply merge two error messages.\n\n    The format of ``errors1`` and ``errors2`` matches the ``message``\n    parameter of :exc:`marshmallow.exceptions.ValidationError`.\n    \"\"\"\n    if not errors1:\n        return errors2\n    if not errors2:\n        return errors1\n    if isinstance(errors1, list):\n        if isinstance(errors2, list):\n            return errors1 + errors2\n        if isinstance(errors2, dict):\n            return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n        return errors1 + [errors2]\n    if isinstance(errors1, dict):\n        if isinstance(errors2, list):\n            return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n        if isinstance(errors2, dict):\n            errors = dict(errors1)\n            for key, val in errors2.items():\n                if key in errors:\n                    errors[key] = merge_errors(errors[key], val)\n                else:\n                    errors[key] = val\n            return errors\n        return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n    if isinstance(errors2, list):\n        return [errors1] + errors2\n    if isinstance(errors2, dict):\n        return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n    return [errors1, errors2]\n\n# Path: src/marshmallow/orderedset.py\n# OrderedSet\n# Copyright (c) 2009 Raymond Hettinger\n#\n# Permission is hereby granted, free of charge, to any person\n# obtaining a copy of this software and associated documentation files\n# (the \"Software\"), to deal in the Software without restriction,\n# including without limitation the rights to use, copy, modify, merge,\n# publish, distribute, sublicense, and/or sell copies of the Software,\n# and to permit persons to whom the Software is furnished to do so,\n# subject to the following conditions:\n#\n#     The above copyright notice and this permission notice shall be\n#     included in all copies or substantial portions of the Software.\n#\n#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES\n#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\n#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\n#     OTHER DEALINGS IN THE SOFTWARE.\nfrom collections.abc import MutableSet\n\n\nclass OrderedSet(MutableSet):\n    def __init__(self, iterable=None):\n        self.end = end = []\n        end += [None, end, end]  # sentinel node for doubly linked list\n        self.map = {}  # key --> [key, prev, next]\n        if iterable is not None:\n            self |= iterable\n\n    def __len__(self):\n        return len(self.map)\n\n    def __contains__(self, key):\n        return key in self.map\n\n    def add(self, key):\n        if key not in self.map:\n            end = self.end\n            curr = end[1]\n            curr[2] = end[1] = self.map[key] = [key, curr, end]\n\n    def discard(self, key):\n        if key in self.map:\n            key, prev, next = self.map.pop(key)\n            prev[2] = next\n            next[1] = prev\n\n    def __iter__(self):\n        end = self.end\n        curr = end[2]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[2]\n\n    def __reversed__(self):\n        end = self.end\n        curr = end[1]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[1]\n\n    def pop(self, last=True):\n        if not self:\n            raise KeyError(\"set is empty\")\n        key = self.end[1][0] if last else self.end[2][0]\n        self.discard(key)\n        return key\n\n    def __repr__(self):\n        if not self:\n            return f\"{self.__class__.__name__}()\"\n        return f\"{self.__class__.__name__}({list(self)!r})\"\n\n    def __eq__(self, other):\n        if isinstance(other, OrderedSet):\n            return len(self) == len(other) and list(self) == list(other)\n        return set(self) == set(other)\n\n\nif __name__ == \"__main__\":\n    s = OrderedSet(\"abracadaba\")\n    t = OrderedSet(\"simsalabim\")\n    print(s | t)\n    print(s & t)\n    print(s - t)\n\n# Path: src/marshmallow/types.py\n\"\"\"Type aliases.\n\n.. warning::\n\n    This module is provisional. Types may be modified, added, and removed between minor releases.\n\"\"\"\nimport typing\n\nStrSequenceOrSet = typing.Union[typing.Sequence[str], typing.AbstractSet[str]]\nTag = typing.Union[str, typing.Tuple[str, bool]]\nValidator = typing.Callable[[typing.Any], typing.Any]\n\n# Path: src/marshmallow/warnings.py\nclass RemovedInMarshmallow4Warning(DeprecationWarning):\n    pass\n\n# Path: src/marshmallow/utils.py\n\"\"\"Utility methods for marshmallow.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport datetime as dt\nimport functools\nimport inspect\nimport json\nimport re\nimport typing\nimport warnings\nfrom collections.abc import Mapping\nfrom email.utils import format_datetime, parsedate_to_datetime\nfrom pprint import pprint as py_pprint\n\nfrom marshmallow.base import FieldABC\nfrom marshmallow.exceptions import FieldInstanceResolutionError\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\nEXCLUDE = \"exclude\"\nINCLUDE = \"include\"\nRAISE = \"raise\"\n_UNKNOWN_VALUES = {EXCLUDE, INCLUDE, RAISE}\n\n\nclass _Missing:\n    def __bool__(self):\n        return False\n\n    def __copy__(self):\n        return self\n\n    def __deepcopy__(self, _):\n        return self\n\n    def __repr__(self):\n        return \"<marshmallow.missing>\"\n\n\n# Singleton value that indicates that a field's value is missing from input\n# dict passed to :meth:`Schema.load`. If the field's value is not required,\n# it's ``default`` value is used.\nmissing = _Missing()\n\n\ndef is_generator(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a generator\"\"\"\n    return inspect.isgeneratorfunction(obj) or inspect.isgenerator(obj)\n\n\ndef is_iterable_but_not_string(obj) -> bool:\n    \"\"\"Return True if ``obj`` is an iterable object that isn't a string.\"\"\"\n    return (hasattr(obj, \"__iter__\") and not hasattr(obj, \"strip\")) or is_generator(obj)\n\n\ndef is_collection(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a collection type, e.g list, tuple, queryset.\"\"\"\n    return is_iterable_but_not_string(obj) and not isinstance(obj, Mapping)\n\n\ndef is_instance_or_subclass(val, class_) -> bool:\n    \"\"\"Return True if ``val`` is either a subclass or instance of ``class_``.\"\"\"\n    try:\n        return issubclass(val, class_)\n    except TypeError:\n        return isinstance(val, class_)\n\n\ndef is_keyed_tuple(obj) -> bool:\n    \"\"\"Return True if ``obj`` has keyed tuple behavior, such as\n    namedtuples or SQLAlchemy's KeyedTuples.\n    \"\"\"\n    return isinstance(obj, tuple) and hasattr(obj, \"_fields\")\n\n\ndef pprint(obj, *args, **kwargs) -> None:\n    \"\"\"Pretty-printing function that can pretty-print OrderedDicts\n    like regular dictionaries. Useful for printing the output of\n    :meth:`marshmallow.Schema.dump`.\n\n    .. deprecated:: 3.7.0\n        marshmallow.pprint will be removed in marshmallow 4.\n    \"\"\"\n    warnings.warn(\n        \"marshmallow's pprint function is deprecated and will be removed in marshmallow 4.\",\n        RemovedInMarshmallow4Warning,\n        stacklevel=2,\n    )\n    if isinstance(obj, collections.OrderedDict):\n        print(json.dumps(obj, *args, **kwargs))\n    else:\n        py_pprint(obj, *args, **kwargs)\n\n\n# https://stackoverflow.com/a/27596917\ndef is_aware(datetime: dt.datetime) -> bool:\n    return (\n        datetime.tzinfo is not None and datetime.tzinfo.utcoffset(datetime) is not None\n    )\n\n\ndef from_rfc(datestring: str) -> dt.datetime:\n    \"\"\"Parse a RFC822-formatted datetime string and return a datetime object.\n\n    https://stackoverflow.com/questions/885015/how-to-parse-a-rfc-2822-date-time-into-a-python-datetime  # noqa: B950\n    \"\"\"\n    return parsedate_to_datetime(datestring)\n\n\ndef rfcformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the RFC822-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return format_datetime(datetime)\n\n\n# Hat tip to Django for ISO8601 deserialization functions\n\n_iso8601_datetime_re = re.compile(\n    r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})\"\n    r\"[T ](?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n    r\"(?P<tzinfo>Z|[+-]\\d{2}(?::?\\d{2})?)?$\"\n)\n\n_iso8601_date_re = re.compile(r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})$\")\n\n_iso8601_time_re = re.compile(\n    r\"(?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n)\n\n\ndef get_fixed_timezone(offset: int | float | dt.timedelta) -> dt.timezone:\n    \"\"\"Return a tzinfo instance with a fixed offset from UTC.\"\"\"\n    if isinstance(offset, dt.timedelta):\n        offset = offset.total_seconds() // 60\n    sign = \"-\" if offset < 0 else \"+\"\n    hhmm = \"%02d%02d\" % divmod(abs(offset), 60)\n    name = sign + hhmm\n    return dt.timezone(dt.timedelta(minutes=offset), name)\n\n\ndef from_iso_datetime(value):\n    \"\"\"Parse a string and return a datetime.datetime.\n\n    This function supports time zone offsets. When the input contains one,\n    the output uses a timezone with a fixed offset from UTC.\n    \"\"\"\n    match = _iso8601_datetime_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted datetime string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    tzinfo = kw.pop(\"tzinfo\")\n    if tzinfo == \"Z\":\n        tzinfo = dt.timezone.utc\n    elif tzinfo is not None:\n        offset_mins = int(tzinfo[-2:]) if len(tzinfo) > 3 else 0\n        offset = 60 * int(tzinfo[1:3]) + offset_mins\n        if tzinfo[0] == \"-\":\n            offset = -offset\n        tzinfo = get_fixed_timezone(offset)\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    kw[\"tzinfo\"] = tzinfo\n    return dt.datetime(**kw)\n\n\n\ndef from_iso_time(value):\n    \"\"\"Parse a string and return a datetime.time.\n\n    This function doesn't support time zone offsets.\n    \"\"\"\n    match = _iso8601_time_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted time string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    return dt.time(**kw)\n\n\ndef from_iso_date(value):\n    \"\"\"Parse a string and return a datetime.date.\"\"\"\n    match = _iso8601_date_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted date string\")\n    kw = {k: int(v) for k, v in match.groupdict().items()}\n    return dt.date(**kw)\n\n\ndef from_timestamp(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    if value < 0:\n        raise ValueError(\"Not a valid POSIX timestamp\")\n\n    # Load a timestamp with utc as timezone to prevent using system timezone.\n    # Then set timezone to None, to let the Field handle adding timezone info.\n    try:\n        return dt.datetime.fromtimestamp(value, tz=dt.timezone.utc).replace(tzinfo=None)\n    except OverflowError as exc:\n        raise ValueError(\"Timestamp is too large\") from exc\n    except OSError as exc:\n        raise ValueError(\"Error converting value to datetime\") from exc\n\n\ndef from_timestamp_ms(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    return from_timestamp(value / 1000)\n\n\ndef timestamp(\n    value: dt.datetime,\n) -> float:\n    if not is_aware(value):\n        # When a date is naive, use UTC as zone info to prevent using system timezone.\n        value = value.replace(tzinfo=dt.timezone.utc)\n    return value.timestamp()\n\n\ndef timestamp_ms(value: dt.datetime) -> float:\n    return timestamp(value) * 1000\n\n\ndef isoformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the ISO8601-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return datetime.isoformat()\n\n\ndef to_iso_time(time: dt.time) -> str:\n    return dt.time.isoformat(time)\n\n\ndef to_iso_date(date: dt.date) -> str:\n    return dt.date.isoformat(date)\n\n\ndef ensure_text_type(val: str | bytes) -> str:\n    if isinstance(val, bytes):\n        val = val.decode(\"utf-8\")\n    return str(val)\n\n\ndef pluck(dictlist: list[dict[str, typing.Any]], key: str):\n    \"\"\"Extracts a list of dictionary values from a list of dictionaries.\n    ::\n\n        >>> dlist = [{'id': 1, 'name': 'foo'}, {'id': 2, 'name': 'bar'}]\n        >>> pluck(dlist, 'id')\n        [1, 2]\n    \"\"\"\n    return [d[key] for d in dictlist]\n\n\n# Various utilities for pulling keyed values from objects\n\n\ndef get_value(obj, key: int | str, default=missing):\n    \"\"\"Helper for pulling a keyed value off various types of objects. Fields use\n    this method by default to access attributes of the source object. For object `x`\n    and attribute `i`, this method first tries to access `x[i]`, and then falls back to\n    `x.i` if an exception is raised.\n\n    .. warning::\n        If an object `x` does not raise an exception when `x[i]` does not exist,\n        `get_value` will never check the value `x.i`. Consider overriding\n        `marshmallow.fields.Field.get_value` in this case.\n    \"\"\"\n    if not isinstance(key, int) and \".\" in key:\n        return _get_value_for_keys(obj, key.split(\".\"), default)\n    else:\n        return _get_value_for_key(obj, key, default)\n\n\ndef _get_value_for_keys(obj, keys, default):\n    if len(keys) == 1:\n        return _get_value_for_key(obj, keys[0], default)\n    else:\n        return _get_value_for_keys(\n            _get_value_for_key(obj, keys[0], default), keys[1:], default\n        )\n\n\ndef _get_value_for_key(obj, key, default):\n    if not hasattr(obj, \"__getitem__\"):\n        return getattr(obj, key, default)\n\n    try:\n        return obj[key]\n    except (KeyError, IndexError, TypeError, AttributeError):\n        return getattr(obj, key, default)\n\n\ndef set_value(dct: dict[str, typing.Any], key: str, value: typing.Any):\n    \"\"\"Set a value in a dict. If `key` contains a '.', it is assumed\n    be a path (i.e. dot-delimited string) to the value's location.\n\n    ::\n\n        >>> d = {}\n        >>> set_value(d, 'foo.bar', 42)\n        >>> d\n        {'foo': {'bar': 42}}\n    \"\"\"\n    if \".\" in key:\n        head, rest = key.split(\".\", 1)\n        target = dct.setdefault(head, {})\n        if not isinstance(target, dict):\n            raise ValueError(\n                f\"Cannot set {key} in {head} \" f\"due to existing value: {target}\"\n            )\n        set_value(target, rest, value)\n    else:\n        dct[key] = value\n\n\ndef callable_or_raise(obj):\n    \"\"\"Check that an object is callable, else raise a :exc:`TypeError`.\"\"\"\n    if not callable(obj):\n        raise TypeError(f\"Object {obj!r} is not callable.\")\n    return obj\n\n\ndef _signature(func: typing.Callable) -> list[str]:\n    return list(inspect.signature(func).parameters.keys())\n\n\ndef get_func_args(func: typing.Callable) -> list[str]:\n    \"\"\"Given a callable, return a list of argument names. Handles\n    `functools.partial` objects and class-based callables.\n\n    .. versionchanged:: 3.0.0a1\n        Do not return bound arguments, eg. ``self``.\n    \"\"\"\n    if inspect.isfunction(func) or inspect.ismethod(func):\n        return _signature(func)\n    if isinstance(func, functools.partial):\n        return _signature(func.func)\n    # Callable class\n    return _signature(func)\n\n\ndef resolve_field_instance(cls_or_instance):\n    \"\"\"Return a Schema instance from a Schema class or instance.\n\n    :param type|Schema cls_or_instance: Marshmallow Schema class or instance.\n    \"\"\"\n    if isinstance(cls_or_instance, type):\n        if not issubclass(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance()\n    else:\n        if not isinstance(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance\n\n\ndef timedelta_to_microseconds(value: dt.timedelta) -> int:\n    \"\"\"Compute the total microseconds of a timedelta\n\n    https://github.com/python/cpython/blob/bb3e0c240bc60fe08d332ff5955d54197f79751c/Lib/datetime.py#L665-L667  # noqa: B950\n    \"\"\"\n    return (value.days * (24 * 3600) + value.seconds) * 1000000 + value.microseconds\n\n\ndef validate_unknown_parameter_value(obj: typing.Any) -> str:\n    if obj not in _UNKNOWN_VALUES:\n        raise ValueError(\n            f\"Object {obj!r} is not a valid value for the 'unknown' parameter\"\n        )\n    return obj\n\n# Path: src/marshmallow/schema.py\n\"\"\"The :class:`Schema` class, including its metaclass and options (class Meta).\"\"\"\nfrom __future__ import annotations\n\nimport copy\nimport datetime as dt\nimport decimal\nimport inspect\nimport json\nimport typing\nimport uuid\nimport warnings\nfrom abc import ABCMeta\nfrom collections import OrderedDict, defaultdict\nfrom collections.abc import Mapping\nfrom functools import lru_cache\n\nfrom marshmallow import base, class_registry, types\nfrom marshmallow import fields as ma_fields\nfrom marshmallow.decorators import (\n    POST_DUMP,\n    POST_LOAD,\n    PRE_DUMP,\n    PRE_LOAD,\n    VALIDATES,\n    VALIDATES_SCHEMA,\n)\nfrom marshmallow.error_store import ErrorStore\nfrom marshmallow.exceptions import StringNotCollectionError, ValidationError\nfrom marshmallow.orderedset import OrderedSet\nfrom marshmallow.utils import (\n    EXCLUDE,\n    INCLUDE,\n    RAISE,\n    get_value,\n    is_collection,\n    is_instance_or_subclass,\n    missing,\n    set_value,\n    validate_unknown_parameter_value,\n)\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\n_T = typing.TypeVar(\"_T\")\n\n\ndef _get_fields(attrs):\n    \"\"\"Get fields from a class\n\n    :param attrs: Mapping of class attributes\n    \"\"\"\n    return [\n        (field_name, field_value)\n        for field_name, field_value in attrs.items()\n        if is_instance_or_subclass(field_value, base.FieldABC)\n    ]\n\n\n# This function allows Schemas to inherit from non-Schema classes and ensures\n#   inheritance according to the MRO\ndef _get_fields_by_mro(klass):\n    \"\"\"Collect fields from a class, following its method resolution order. The\n    class itself is excluded from the search; only its parents are checked. Get\n    fields from ``_declared_fields`` if available, else use ``__dict__``.\n\n    :param type klass: Class whose fields to retrieve\n    \"\"\"\n    mro = inspect.getmro(klass)\n    # Loop over mro in reverse to maintain correct order of fields\n    return sum(\n        (\n            _get_fields(\n                getattr(base, \"_declared_fields\", base.__dict__),\n            )\n            for base in mro[:0:-1]\n        ),\n        [],\n    )\n\n\nclass SchemaMeta(ABCMeta):\n    \"\"\"Metaclass for the Schema class. Binds the declared fields to\n    a ``_declared_fields`` attribute, which is a dictionary mapping attribute\n    names to field objects. Also sets the ``opts`` class attribute, which is\n    the Schema class's ``class Meta`` options.\n    \"\"\"\n\n    def __new__(mcs, name, bases, attrs):\n        meta = attrs.get(\"Meta\")\n        ordered = getattr(meta, \"ordered\", False)\n        if not ordered:\n            # Inherit 'ordered' option\n            # Warning: We loop through bases instead of MRO because we don't\n            # yet have access to the class object\n            # (i.e. can't call super before we have fields)\n            for base_ in bases:\n                if hasattr(base_, \"Meta\") and hasattr(base_.Meta, \"ordered\"):\n                    ordered = base_.Meta.ordered\n                    break\n            else:\n                ordered = False\n        cls_fields = _get_fields(attrs)\n        # Remove fields from list of class attributes to avoid shadowing\n        # Schema attributes/methods in case of name conflict\n        for field_name, _ in cls_fields:\n            del attrs[field_name]\n        klass = super().__new__(mcs, name, bases, attrs)\n        inherited_fields = _get_fields_by_mro(klass)\n\n        meta = klass.Meta\n        # Set klass.opts in __new__ rather than __init__ so that it is accessible in\n        # get_declared_fields\n        klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered)\n        # Add fields specified in the `include` class Meta option\n        cls_fields += list(klass.opts.include.items())\n\n        # Assign _declared_fields on class\n        klass._declared_fields = mcs.get_declared_fields(\n            klass=klass,\n            cls_fields=cls_fields,\n            inherited_fields=inherited_fields,\n            dict_cls=dict,\n        )\n        return klass\n\n    @classmethod\n    def get_declared_fields(\n        mcs,\n        klass: type,\n        cls_fields: list,\n        inherited_fields: list,\n        dict_cls: type = dict,\n    ):\n        \"\"\"Returns a dictionary of field_name => `Field` pairs declared on the class.\n        This is exposed mainly so that plugins can add additional fields, e.g. fields\n        computed from class Meta options.\n\n        :param klass: The class object.\n        :param cls_fields: The fields declared on the class, including those added\n            by the ``include`` class Meta option.\n        :param inherited_fields: Inherited fields.\n        :param dict_cls: dict-like class to use for dict output Default to ``dict``.\n        \"\"\"\n        return dict_cls(inherited_fields + cls_fields)\n\n    def __init__(cls, name, bases, attrs):\n        super().__init__(name, bases, attrs)\n        if name and cls.opts.register:\n            class_registry.register(name, cls)\n        cls._hooks = cls.resolve_hooks()\n\n    def resolve_hooks(cls) -> dict[types.Tag, list[str]]:\n        \"\"\"Add in the decorated processors\n\n        By doing this after constructing the class, we let standard inheritance\n        do all the hard work.\n        \"\"\"\n        mro = inspect.getmro(cls)\n\n        hooks = defaultdict(list)  # type: typing.Dict[types.Tag, typing.List[str]]\n\n        for attr_name in dir(cls):\n            # Need to look up the actual descriptor, not whatever might be\n            # bound to the class. This needs to come from the __dict__ of the\n            # declaring class.\n            for parent in mro:\n                try:\n                    attr = parent.__dict__[attr_name]\n                except KeyError:\n                    continue\n                else:\n                    break\n            else:\n                # In case we didn't find the attribute and didn't break above.\n                # We should never hit this - it's just here for completeness\n                # to exclude the possibility of attr being undefined.\n                continue\n\n            try:\n                hook_config = attr.__marshmallow_hook__\n            except AttributeError:\n                pass\n            else:\n                for key in hook_config.keys():\n                    # Use name here so we can get the bound method later, in\n                    # case the processor was a descriptor or something.\n                    hooks[key].append(attr_name)\n\n        return hooks\n\n\nclass SchemaOpts:\n    \"\"\"class Meta options for the :class:`Schema`. Defines defaults.\"\"\"\n\n    def __init__(self, meta, ordered: bool = False):\n        self.fields = getattr(meta, \"fields\", ())\n        if not isinstance(self.fields, (list, tuple)):\n            raise ValueError(\"`fields` option must be a list or tuple.\")\n        self.additional = getattr(meta, \"additional\", ())\n        if not isinstance(self.additional, (list, tuple)):\n            raise ValueError(\"`additional` option must be a list or tuple.\")\n        if self.fields and self.additional:\n            raise ValueError(\n                \"Cannot set both `fields` and `additional` options\"\n                \" for the same Schema.\"\n            )\n        self.exclude = getattr(meta, \"exclude\", ())\n        if not isinstance(self.exclude, (list, tuple)):\n            raise ValueError(\"`exclude` must be a list or tuple.\")\n        self.dateformat = getattr(meta, \"dateformat\", None)\n        self.datetimeformat = getattr(meta, \"datetimeformat\", None)\n        self.timeformat = getattr(meta, \"timeformat\", None)\n        if hasattr(meta, \"json_module\"):\n            warnings.warn(\n                \"The json_module class Meta option is deprecated. Use render_module instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            render_module = getattr(meta, \"json_module\", json)\n        else:\n            render_module = json\n        self.render_module = getattr(meta, \"render_module\", render_module)\n        self.ordered = getattr(meta, \"ordered\", ordered)\n        self.index_errors = getattr(meta, \"index_errors\", True)\n        self.include = getattr(meta, \"include\", {})\n        self.load_only = getattr(meta, \"load_only\", ())\n        self.dump_only = getattr(meta, \"dump_only\", ())\n        self.unknown = validate_unknown_parameter_value(getattr(meta, \"unknown\", RAISE))\n        self.register = getattr(meta, \"register\", True)\n\n\nclass Schema(base.SchemaABC, metaclass=SchemaMeta):\n    \"\"\"Base schema class with which to define custom schemas.\n\n    Example usage:\n\n    .. code-block:: python\n\n        import datetime as dt\n        from dataclasses import dataclass\n\n        from marshmallow import Schema, fields\n\n\n        @dataclass\n        class Album:\n            title: str\n            release_date: dt.date\n\n\n        class AlbumSchema(Schema):\n            title = fields.Str()\n            release_date = fields.Date()\n\n\n        album = Album(\"Beggars Banquet\", dt.date(1968, 12, 6))\n        schema = AlbumSchema()\n        data = schema.dump(album)\n        data  # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'}\n\n    :param only: Whitelist of the declared fields to select when\n        instantiating the Schema. If None, all fields are used. Nested fields\n        can be represented with dot delimiters.\n    :param exclude: Blacklist of the declared fields to exclude\n        when instantiating the Schema. If a field appears in both `only` and\n        `exclude`, it is not used. Nested fields can be represented with dot\n        delimiters.\n    :param many: Should be set to `True` if ``obj`` is a collection\n        so that the object will be serialized to a list.\n    :param context: Optional context passed to :class:`fields.Method` and\n        :class:`fields.Function` fields.\n    :param load_only: Fields to skip during serialization (write-only fields)\n    :param dump_only: Fields to skip during deserialization (read-only fields)\n    :param partial: Whether to ignore missing fields and not require\n        any fields declared. Propagates down to ``Nested`` fields as well. If\n        its value is an iterable, only missing fields listed in that iterable\n        will be ignored. Use dot delimiters to specify nested fields.\n    :param unknown: Whether to exclude, include, or raise an error for unknown\n        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n\n    .. versionchanged:: 3.0.0\n        `prefix` parameter removed.\n\n    .. versionchanged:: 2.0.0\n        `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of\n        `marshmallow.decorators.validates_schema`,\n        `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`.\n        `__accessor__` and `__error_handler__` are deprecated. Implement the\n        `handle_error` and `get_attribute` methods instead.\n    \"\"\"\n\n    TYPE_MAPPING = {\n        str: ma_fields.String,\n        bytes: ma_fields.String,\n        dt.datetime: ma_fields.DateTime,\n        float: ma_fields.Float,\n        bool: ma_fields.Boolean,\n        tuple: ma_fields.Raw,\n        list: ma_fields.Raw,\n        set: ma_fields.Raw,\n        int: ma_fields.Integer,\n        uuid.UUID: ma_fields.UUID,\n        dt.time: ma_fields.Time,\n        dt.date: ma_fields.Date,\n        dt.timedelta: ma_fields.TimeDelta,\n        decimal.Decimal: ma_fields.Decimal,\n    }  # type: typing.Dict[type, typing.Type[ma_fields.Field]]\n    #: Overrides for default schema-level error messages\n    error_messages = {}  # type: typing.Dict[str, str]\n\n    _default_error_messages = {\n        \"type\": \"Invalid input type.\",\n        \"unknown\": \"Unknown field.\",\n    }  # type: typing.Dict[str, str]\n\n    OPTIONS_CLASS = SchemaOpts  # type: type\n\n    set_class = OrderedSet\n\n    # These get set by SchemaMeta\n    opts = None  # type: SchemaOpts\n    _declared_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n    _hooks = {}  # type: typing.Dict[types.Tag, typing.List[str]]\n\n    class Meta:\n        \"\"\"Options object for a Schema.\n\n        Example usage: ::\n\n            class Meta:\n                fields = (\"id\", \"email\", \"date_created\")\n                exclude = (\"password\", \"secret_attribute\")\n\n        Available options:\n\n        - ``fields``: Tuple or list of fields to include in the serialized result.\n        - ``additional``: Tuple or list of fields to include *in addition* to the\n            explicitly declared fields. ``additional`` and ``fields`` are\n            mutually-exclusive options.\n        - ``include``: Dictionary of additional fields to include in the schema. It is\n            usually better to define fields as class variables, but you may need to\n            use this option, e.g., if your fields are Python keywords. May be an\n...\n", "needle_token_start": 7998, "needle_token_end": 8157, "code_context_ntokens": 13707, "output": ["```python\ndef from_iso_time(value):\n    \"\"\"Parse a string and return a datetime.time.\n\n    This function doesn't support time zone offsets.\n    \"\"\"\n    match = _iso8601_time_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted time string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    return dt.time(**kw)\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "__len__", "language": "python", "path": "src/marshmallow/orderedset.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To determine the number of unique elements stored in a custom set-like data structure.\n2. **Input**: No explicit input is required from the user when this method is called.\n3. **Output**: Returns an integer representing the count of unique elements.\n4. **Procedure**: This method calculates the total number of keys in an internal dictionary that maps each unique element to its position in a doubly linked list, effectively counting the elements in the data structure.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/marshmallow/decorators.py\n\"\"\"Decorators for registering schema pre-processing and post-processing methods.\nThese should be imported from the top-level `marshmallow` module.\n\nMethods decorated with\n`pre_load <marshmallow.decorators.pre_load>`, `post_load <marshmallow.decorators.post_load>`,\n`pre_dump <marshmallow.decorators.pre_dump>`, `post_dump <marshmallow.decorators.post_dump>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``many`` as a keyword argument. In addition, `pre_load <marshmallow.decorators.pre_load>`,\n`post_load <marshmallow.decorators.post_load>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``partial``. If you don't need these arguments, add ``**kwargs`` to your method\nsignature.\n\n\nExample: ::\n\n    from marshmallow import (\n        Schema, pre_load, pre_dump, post_load, validates_schema,\n        validates, fields, ValidationError\n    )\n\n    class UserSchema(Schema):\n\n        email = fields.Str(required=True)\n        age = fields.Integer(required=True)\n\n        @post_load\n        def lowerstrip_email(self, item, many, **kwargs):\n            item['email'] = item['email'].lower().strip()\n            return item\n\n        @pre_load(pass_many=True)\n        def remove_envelope(self, data, many, **kwargs):\n            namespace = 'results' if many else 'result'\n            return data[namespace]\n\n        @post_dump(pass_many=True)\n        def add_envelope(self, data, many, **kwargs):\n            namespace = 'results' if many else 'result'\n            return {namespace: data}\n\n        @validates_schema\n        def validate_email(self, data, **kwargs):\n            if len(data['email']) < 3:\n                raise ValidationError('Email must be more than 3 characters', 'email')\n\n        @validates('age')\n        def validate_age(self, data, **kwargs):\n            if data < 14:\n                raise ValidationError('Too young!')\n\n.. note::\n    These decorators only work with instance methods. Class and static\n    methods are not supported.\n\n.. warning::\n    The invocation order of decorated methods of the same type is not guaranteed.\n    If you need to guarantee order of different processing steps, you should put\n    them in the same processing method.\n\"\"\"\nfrom __future__ import annotations\n\nimport functools\nfrom typing import Any, Callable, cast\n\nPRE_DUMP = \"pre_dump\"\nPOST_DUMP = \"post_dump\"\nPRE_LOAD = \"pre_load\"\nPOST_LOAD = \"post_load\"\nVALIDATES = \"validates\"\nVALIDATES_SCHEMA = \"validates_schema\"\n\n\nclass MarshmallowHook:\n    __marshmallow_hook__: dict[tuple[str, bool] | str, Any] | None = None\n\n\ndef validates(field_name: str) -> Callable[..., Any]:\n    \"\"\"Register a field validator.\n\n    :param str field_name: Name of the field that the method validates.\n    \"\"\"\n    return set_hook(None, VALIDATES, field_name=field_name)\n\n\ndef validates_schema(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n    skip_on_field_errors: bool = True,\n) -> Callable[..., Any]:\n    \"\"\"Register a schema-level validator.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.validate` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before unmarshalling) will be passed as\n    an additional argument to the method.\n\n    If ``skip_on_field_errors=True``, this validation method will be skipped whenever\n    validation errors have been detected when validating fields.\n\n    .. versionchanged:: 3.0.0b1\n        ``skip_on_field_errors`` defaults to `True`.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(\n        fn,\n        (VALIDATES_SCHEMA, pass_many),\n        pass_original=pass_original,\n        skip_on_field_errors=skip_on_field_errors,\n    )\n\n\ndef pre_dump(\n    fn: Callable[..., Any] | None = None, pass_many: bool = False\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke before serializing an object. The method\n    receives the object to be serialized and returns the processed object.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.dump` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    .. versionchanged:: 3.0.0\n        ``many`` is always passed as a keyword arguments to the decorated method.\n    \"\"\"\n    return set_hook(fn, (PRE_DUMP, pass_many))\n\n\ndef post_dump(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke after serializing an object. The method\n    receives the serialized object and returns the processed object.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.dump` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before serializing) will be passed as\n    an additional argument to the method.\n\n    .. versionchanged:: 3.0.0\n        ``many`` is always passed as a keyword arguments to the decorated method.\n    \"\"\"\n    return set_hook(fn, (POST_DUMP, pass_many), pass_original=pass_original)\n\n\ndef pre_load(\n    fn: Callable[..., Any] | None = None, pass_many: bool = False\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke before deserializing an object. The method\n    receives the data to be deserialized and returns the processed data.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.load` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(fn, (PRE_LOAD, pass_many))\n\n\ndef post_load(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke after deserializing an object. The method\n    receives the deserialized data and returns the processed data.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.load` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before deserializing) will be passed as\n    an additional argument to the method.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(fn, (POST_LOAD, pass_many), pass_original=pass_original)\n\n\ndef set_hook(\n    fn: Callable[..., Any] | None, key: tuple[str, bool] | str, **kwargs: Any\n) -> Callable[..., Any]:\n    \"\"\"Mark decorated function as a hook to be picked up later.\n    You should not need to use this method directly.\n\n    .. note::\n        Currently only works with functions and instance methods. Class and\n        static methods are not supported.\n\n    :return: Decorated function if supplied, else this decorator with its args\n        bound.\n    \"\"\"\n    # Allow using this as either a decorator or a decorator factory.\n    if fn is None:\n        return functools.partial(set_hook, key=key, **kwargs)\n\n    # Set a __marshmallow_hook__ attribute instead of wrapping in some class,\n    # because I still want this to end up as a normal (unbound) method.\n    function = cast(MarshmallowHook, fn)\n    try:\n        hook_config = function.__marshmallow_hook__\n    except AttributeError:\n        function.__marshmallow_hook__ = hook_config = {}\n    # Also save the kwargs for the tagged function on\n    # __marshmallow_hook__, keyed by (<tag>, <pass_many>)\n    if hook_config is not None:\n        hook_config[key] = kwargs\n\n    return fn\n\n# Path: src/marshmallow/exceptions.py\n\"\"\"Exception classes for marshmallow-related errors.\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\n# Key used for schema-level validation errors\nSCHEMA = \"_schema\"\n\n\nclass MarshmallowError(Exception):\n    \"\"\"Base class for all marshmallow-related errors.\"\"\"\n\n\nclass ValidationError(MarshmallowError):\n    \"\"\"Raised when validation fails on a field or schema.\n\n    Validators and custom fields should raise this exception.\n\n    :param message: An error message, list of error messages, or dict of\n        error messages. If a dict, the keys are subitems and the values are error messages.\n    :param field_name: Field name to store the error on.\n        If `None`, the error is stored as schema-level error.\n    :param data: Raw input data.\n    :param valid_data: Valid (de)serialized data.\n    \"\"\"\n\n    def __init__(\n        self,\n        message: str | list | dict,\n        field_name: str = SCHEMA,\n        data: typing.Mapping[str, typing.Any]\n        | typing.Iterable[typing.Mapping[str, typing.Any]]\n        | None = None,\n        valid_data: list[dict[str, typing.Any]] | dict[str, typing.Any] | None = None,\n        **kwargs,\n    ):\n        self.messages = [message] if isinstance(message, (str, bytes)) else message\n        self.field_name = field_name\n        self.data = data\n        self.valid_data = valid_data\n        self.kwargs = kwargs\n        super().__init__(message)\n\n    def normalized_messages(self):\n        if self.field_name == SCHEMA and isinstance(self.messages, dict):\n            return self.messages\n        return {self.field_name: self.messages}\n\n    @property\n    def messages_dict(self) -> dict[str, typing.Any]:\n        if not isinstance(self.messages, dict):\n            raise TypeError(\n                \"cannot access 'messages_dict' when 'messages' is of type \"\n                + type(self.messages).__name__\n            )\n        return self.messages\n\n\nclass RegistryError(NameError):\n    \"\"\"Raised when an invalid operation is performed on the serializer\n    class registry.\n    \"\"\"\n\n\nclass StringNotCollectionError(MarshmallowError, TypeError):\n    \"\"\"Raised when a string is passed when a list of strings is expected.\"\"\"\n\n\nclass FieldInstanceResolutionError(MarshmallowError, TypeError):\n    \"\"\"Raised when schema to instantiate is neither a Schema class nor an instance.\"\"\"\n\n# Path: src/marshmallow/base.py\n\"\"\"Abstract base classes.\n\nThese are necessary to avoid circular imports between schema.py and fields.py.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nfrom abc import ABC, abstractmethod\n\n\nclass FieldABC(ABC):\n    \"\"\"Abstract base class from which all Field classes inherit.\"\"\"\n\n    parent = None\n    name = None\n    root = None\n\n    @abstractmethod\n    def serialize(self, attr, obj, accessor=None):\n        pass\n\n    @abstractmethod\n    def deserialize(self, value):\n        pass\n\n    @abstractmethod\n    def _serialize(self, value, attr, obj, **kwargs):\n        pass\n\n    @abstractmethod\n    def _deserialize(self, value, attr, data, **kwargs):\n        pass\n\n\nclass SchemaABC(ABC):\n    \"\"\"Abstract base class from which all Schemas inherit.\"\"\"\n\n    @abstractmethod\n    def dump(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def dumps(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def load(self, data, *, many: bool | None = None, partial=None, unknown=None):\n        pass\n\n    @abstractmethod\n    def loads(\n        self,\n        json_data,\n        *,\n        many: bool | None = None,\n        partial=None,\n        unknown=None,\n        **kwargs,\n    ):\n        pass\n\n# Path: src/marshmallow/class_registry.py\n\"\"\"A registry of :class:`Schema <marshmallow.Schema>` classes. This allows for string\nlookup of schemas, which may be used with\nclass:`fields.Nested <marshmallow.fields.Nested>`.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\nfrom marshmallow.exceptions import RegistryError\n\nif typing.TYPE_CHECKING:\n    from marshmallow import Schema\n\n    SchemaType = typing.Type[Schema]\n\n# {\n#   <class_name>: <list of class objects>\n#   <module_path_to_class>: <list of class objects>\n# }\n_registry = {}  # type: dict[str, list[SchemaType]]\n\n\ndef register(classname: str, cls: SchemaType) -> None:\n    \"\"\"Add a class to the registry of serializer classes. When a class is\n    registered, an entry for both its classname and its full, module-qualified\n    path are added to the registry.\n\n    Example: ::\n\n        class MyClass:\n            pass\n\n        register('MyClass', MyClass)\n        # Registry:\n        # {\n        #   'MyClass': [path.to.MyClass],\n        #   'path.to.MyClass': [path.to.MyClass],\n        # }\n\n    \"\"\"\n    # Module where the class is located\n    module = cls.__module__\n    # Full module path to the class\n    # e.g. user.schemas.UserSchema\n    fullpath = \".\".join([module, classname])\n    # If the class is already registered; need to check if the entries are\n    # in the same module as cls to avoid having multiple instances of the same\n    # class in the registry\n    if classname in _registry and not any(\n        each.__module__ == module for each in _registry[classname]\n    ):\n        _registry[classname].append(cls)\n    elif classname not in _registry:\n        _registry[classname] = [cls]\n\n    # Also register the full path\n    if fullpath not in _registry:\n        _registry.setdefault(fullpath, []).append(cls)\n    else:\n        # If fullpath does exist, replace existing entry\n        _registry[fullpath] = [cls]\n    return None\n\n\ndef get_class(classname: str, all: bool = False) -> list[SchemaType] | SchemaType:\n    \"\"\"Retrieve a class from the registry.\n\n    :raises: marshmallow.exceptions.RegistryError if the class cannot be found\n        or if there are multiple entries for the given class name.\n    \"\"\"\n    try:\n        classes = _registry[classname]\n    except KeyError as error:\n        raise RegistryError(\n            f\"Class with name {classname!r} was not found. You may need \"\n            \"to import the class.\"\n        ) from error\n    if len(classes) > 1:\n        if all:\n            return _registry[classname]\n        raise RegistryError(\n            f\"Multiple classes with name {classname!r} \"\n            \"were found. Please use the full, \"\n            \"module-qualified path.\"\n        )\n    else:\n        return _registry[classname][0]\n\n# Path: src/marshmallow/error_store.py\n\"\"\"Utilities for storing collections of error messages.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\n\nfrom marshmallow.exceptions import SCHEMA\n\n\nclass ErrorStore:\n    def __init__(self):\n        #: Dictionary of errors stored during serialization\n        self.errors = {}\n\n    def store_error(self, messages, field_name=SCHEMA, index=None):\n        # field error  -> store/merge error messages under field name key\n        # schema error -> if string or list, store/merge under _schema key\n        #              -> if dict, store/merge with other top-level keys\n        if field_name != SCHEMA or not isinstance(messages, dict):\n            messages = {field_name: messages}\n        if index is not None:\n            messages = {index: messages}\n        self.errors = merge_errors(self.errors, messages)\n\n\ndef merge_errors(errors1, errors2):\n    \"\"\"Deeply merge two error messages.\n\n    The format of ``errors1`` and ``errors2`` matches the ``message``\n    parameter of :exc:`marshmallow.exceptions.ValidationError`.\n    \"\"\"\n    if not errors1:\n        return errors2\n    if not errors2:\n        return errors1\n    if isinstance(errors1, list):\n        if isinstance(errors2, list):\n            return errors1 + errors2\n        if isinstance(errors2, dict):\n            return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n        return errors1 + [errors2]\n    if isinstance(errors1, dict):\n        if isinstance(errors2, list):\n            return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n        if isinstance(errors2, dict):\n            errors = dict(errors1)\n            for key, val in errors2.items():\n                if key in errors:\n                    errors[key] = merge_errors(errors[key], val)\n                else:\n                    errors[key] = val\n            return errors\n        return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n    if isinstance(errors2, list):\n        return [errors1] + errors2\n    if isinstance(errors2, dict):\n        return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n    return [errors1, errors2]\n\n# Path: src/marshmallow/orderedset.py\n# OrderedSet\n# Copyright (c) 2009 Raymond Hettinger\n#\n# Permission is hereby granted, free of charge, to any person\n# obtaining a copy of this software and associated documentation files\n# (the \"Software\"), to deal in the Software without restriction,\n# including without limitation the rights to use, copy, modify, merge,\n# publish, distribute, sublicense, and/or sell copies of the Software,\n# and to permit persons to whom the Software is furnished to do so,\n# subject to the following conditions:\n#\n#     The above copyright notice and this permission notice shall be\n#     included in all copies or substantial portions of the Software.\n#\n#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES\n#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\n#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\n#     OTHER DEALINGS IN THE SOFTWARE.\nfrom collections.abc import MutableSet\n\n\nclass OrderedSet(MutableSet):\n    def __init__(self, iterable=None):\n        self.end = end = []\n        end += [None, end, end]  # sentinel node for doubly linked list\n        self.map = {}  # key --> [key, prev, next]\n        if iterable is not None:\n            self |= iterable\n\n    \ndef __len__(self):\n        return len(self.map)\n\n    def __contains__(self, key):\n        return key in self.map\n\n    def add(self, key):\n        if key not in self.map:\n            end = self.end\n            curr = end[1]\n            curr[2] = end[1] = self.map[key] = [key, curr, end]\n\n    def discard(self, key):\n        if key in self.map:\n            key, prev, next = self.map.pop(key)\n            prev[2] = next\n            next[1] = prev\n\n    def __iter__(self):\n        end = self.end\n        curr = end[2]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[2]\n\n    def __reversed__(self):\n        end = self.end\n        curr = end[1]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[1]\n\n    def pop(self, last=True):\n        if not self:\n            raise KeyError(\"set is empty\")\n        key = self.end[1][0] if last else self.end[2][0]\n        self.discard(key)\n        return key\n\n    def __repr__(self):\n        if not self:\n            return f\"{self.__class__.__name__}()\"\n        return f\"{self.__class__.__name__}({list(self)!r})\"\n\n    def __eq__(self, other):\n        if isinstance(other, OrderedSet):\n            return len(self) == len(other) and list(self) == list(other)\n        return set(self) == set(other)\n\n\nif __name__ == \"__main__\":\n    s = OrderedSet(\"abracadaba\")\n    t = OrderedSet(\"simsalabim\")\n    print(s | t)\n    print(s & t)\n    print(s - t)\n\n# Path: src/marshmallow/types.py\n\"\"\"Type aliases.\n\n.. warning::\n\n    This module is provisional. Types may be modified, added, and removed between minor releases.\n\"\"\"\nimport typing\n\nStrSequenceOrSet = typing.Union[typing.Sequence[str], typing.AbstractSet[str]]\nTag = typing.Union[str, typing.Tuple[str, bool]]\nValidator = typing.Callable[[typing.Any], typing.Any]\n\n# Path: src/marshmallow/warnings.py\nclass RemovedInMarshmallow4Warning(DeprecationWarning):\n    pass\n\n# Path: src/marshmallow/utils.py\n\"\"\"Utility methods for marshmallow.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport datetime as dt\nimport functools\nimport inspect\nimport json\nimport re\nimport typing\nimport warnings\nfrom collections.abc import Mapping\nfrom email.utils import format_datetime, parsedate_to_datetime\nfrom pprint import pprint as py_pprint\n\nfrom marshmallow.base import FieldABC\nfrom marshmallow.exceptions import FieldInstanceResolutionError\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\nEXCLUDE = \"exclude\"\nINCLUDE = \"include\"\nRAISE = \"raise\"\n_UNKNOWN_VALUES = {EXCLUDE, INCLUDE, RAISE}\n\n\nclass _Missing:\n    def __bool__(self):\n        return False\n\n    def __copy__(self):\n        return self\n\n    def __deepcopy__(self, _):\n        return self\n\n    def __repr__(self):\n        return \"<marshmallow.missing>\"\n\n\n# Singleton value that indicates that a field's value is missing from input\n# dict passed to :meth:`Schema.load`. If the field's value is not required,\n# it's ``default`` value is used.\nmissing = _Missing()\n\n\ndef is_generator(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a generator\"\"\"\n    return inspect.isgeneratorfunction(obj) or inspect.isgenerator(obj)\n\n\ndef is_iterable_but_not_string(obj) -> bool:\n    \"\"\"Return True if ``obj`` is an iterable object that isn't a string.\"\"\"\n    return (hasattr(obj, \"__iter__\") and not hasattr(obj, \"strip\")) or is_generator(obj)\n\n\ndef is_collection(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a collection type, e.g list, tuple, queryset.\"\"\"\n    return is_iterable_but_not_string(obj) and not isinstance(obj, Mapping)\n\n\ndef is_instance_or_subclass(val, class_) -> bool:\n    \"\"\"Return True if ``val`` is either a subclass or instance of ``class_``.\"\"\"\n    try:\n        return issubclass(val, class_)\n    except TypeError:\n        return isinstance(val, class_)\n\n\ndef is_keyed_tuple(obj) -> bool:\n    \"\"\"Return True if ``obj`` has keyed tuple behavior, such as\n    namedtuples or SQLAlchemy's KeyedTuples.\n    \"\"\"\n    return isinstance(obj, tuple) and hasattr(obj, \"_fields\")\n\n\ndef pprint(obj, *args, **kwargs) -> None:\n    \"\"\"Pretty-printing function that can pretty-print OrderedDicts\n    like regular dictionaries. Useful for printing the output of\n    :meth:`marshmallow.Schema.dump`.\n\n    .. deprecated:: 3.7.0\n        marshmallow.pprint will be removed in marshmallow 4.\n    \"\"\"\n    warnings.warn(\n        \"marshmallow's pprint function is deprecated and will be removed in marshmallow 4.\",\n        RemovedInMarshmallow4Warning,\n        stacklevel=2,\n    )\n    if isinstance(obj, collections.OrderedDict):\n        print(json.dumps(obj, *args, **kwargs))\n    else:\n        py_pprint(obj, *args, **kwargs)\n\n\n# https://stackoverflow.com/a/27596917\ndef is_aware(datetime: dt.datetime) -> bool:\n    return (\n        datetime.tzinfo is not None and datetime.tzinfo.utcoffset(datetime) is not None\n    )\n\n\ndef from_rfc(datestring: str) -> dt.datetime:\n    \"\"\"Parse a RFC822-formatted datetime string and return a datetime object.\n\n    https://stackoverflow.com/questions/885015/how-to-parse-a-rfc-2822-date-time-into-a-python-datetime  # noqa: B950\n    \"\"\"\n    return parsedate_to_datetime(datestring)\n\n\ndef rfcformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the RFC822-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return format_datetime(datetime)\n\n\n# Hat tip to Django for ISO8601 deserialization functions\n\n_iso8601_datetime_re = re.compile(\n    r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})\"\n    r\"[T ](?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n    r\"(?P<tzinfo>Z|[+-]\\d{2}(?::?\\d{2})?)?$\"\n)\n\n_iso8601_date_re = re.compile(r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})$\")\n\n_iso8601_time_re = re.compile(\n    r\"(?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n)\n\n\ndef get_fixed_timezone(offset: int | float | dt.timedelta) -> dt.timezone:\n    \"\"\"Return a tzinfo instance with a fixed offset from UTC.\"\"\"\n    if isinstance(offset, dt.timedelta):\n        offset = offset.total_seconds() // 60\n    sign = \"-\" if offset < 0 else \"+\"\n    hhmm = \"%02d%02d\" % divmod(abs(offset), 60)\n    name = sign + hhmm\n    return dt.timezone(dt.timedelta(minutes=offset), name)\n\n\ndef from_iso_datetime(value):\n    \"\"\"Parse a string and return a datetime.datetime.\n\n    This function supports time zone offsets. When the input contains one,\n    the output uses a timezone with a fixed offset from UTC.\n    \"\"\"\n    match = _iso8601_datetime_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted datetime string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    tzinfo = kw.pop(\"tzinfo\")\n    if tzinfo == \"Z\":\n        tzinfo = dt.timezone.utc\n    elif tzinfo is not None:\n        offset_mins = int(tzinfo[-2:]) if len(tzinfo) > 3 else 0\n        offset = 60 * int(tzinfo[1:3]) + offset_mins\n        if tzinfo[0] == \"-\":\n            offset = -offset\n        tzinfo = get_fixed_timezone(offset)\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    kw[\"tzinfo\"] = tzinfo\n    return dt.datetime(**kw)\n\n\ndef from_iso_time(value):\n    \"\"\"Parse a string and return a datetime.time.\n\n    This function doesn't support time zone offsets.\n    \"\"\"\n    match = _iso8601_time_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted time string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    return dt.time(**kw)\n\n\ndef from_iso_date(value):\n    \"\"\"Parse a string and return a datetime.date.\"\"\"\n    match = _iso8601_date_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted date string\")\n    kw = {k: int(v) for k, v in match.groupdict().items()}\n    return dt.date(**kw)\n\n\ndef from_timestamp(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    if value < 0:\n        raise ValueError(\"Not a valid POSIX timestamp\")\n\n    # Load a timestamp with utc as timezone to prevent using system timezone.\n    # Then set timezone to None, to let the Field handle adding timezone info.\n    try:\n        return dt.datetime.fromtimestamp(value, tz=dt.timezone.utc).replace(tzinfo=None)\n    except OverflowError as exc:\n        raise ValueError(\"Timestamp is too large\") from exc\n    except OSError as exc:\n        raise ValueError(\"Error converting value to datetime\") from exc\n\n\ndef from_timestamp_ms(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    return from_timestamp(value / 1000)\n\n\ndef timestamp(\n    value: dt.datetime,\n) -> float:\n    if not is_aware(value):\n        # When a date is naive, use UTC as zone info to prevent using system timezone.\n        value = value.replace(tzinfo=dt.timezone.utc)\n    return value.timestamp()\n\n\ndef timestamp_ms(value: dt.datetime) -> float:\n    return timestamp(value) * 1000\n\n\ndef isoformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the ISO8601-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return datetime.isoformat()\n\n\ndef to_iso_time(time: dt.time) -> str:\n    return dt.time.isoformat(time)\n\n\ndef to_iso_date(date: dt.date) -> str:\n    return dt.date.isoformat(date)\n\n\ndef ensure_text_type(val: str | bytes) -> str:\n    if isinstance(val, bytes):\n        val = val.decode(\"utf-8\")\n    return str(val)\n\n\ndef pluck(dictlist: list[dict[str, typing.Any]], key: str):\n    \"\"\"Extracts a list of dictionary values from a list of dictionaries.\n    ::\n\n        >>> dlist = [{'id': 1, 'name': 'foo'}, {'id': 2, 'name': 'bar'}]\n        >>> pluck(dlist, 'id')\n        [1, 2]\n    \"\"\"\n    return [d[key] for d in dictlist]\n\n\n# Various utilities for pulling keyed values from objects\n\n\ndef get_value(obj, key: int | str, default=missing):\n    \"\"\"Helper for pulling a keyed value off various types of objects. Fields use\n    this method by default to access attributes of the source object. For object `x`\n    and attribute `i`, this method first tries to access `x[i]`, and then falls back to\n    `x.i` if an exception is raised.\n\n    .. warning::\n        If an object `x` does not raise an exception when `x[i]` does not exist,\n        `get_value` will never check the value `x.i`. Consider overriding\n        `marshmallow.fields.Field.get_value` in this case.\n    \"\"\"\n    if not isinstance(key, int) and \".\" in key:\n        return _get_value_for_keys(obj, key.split(\".\"), default)\n    else:\n        return _get_value_for_key(obj, key, default)\n\n\ndef _get_value_for_keys(obj, keys, default):\n    if len(keys) == 1:\n        return _get_value_for_key(obj, keys[0], default)\n    else:\n        return _get_value_for_keys(\n            _get_value_for_key(obj, keys[0], default), keys[1:], default\n        )\n\n\ndef _get_value_for_key(obj, key, default):\n    if not hasattr(obj, \"__getitem__\"):\n        return getattr(obj, key, default)\n\n    try:\n        return obj[key]\n    except (KeyError, IndexError, TypeError, AttributeError):\n        return getattr(obj, key, default)\n\n\ndef set_value(dct: dict[str, typing.Any], key: str, value: typing.Any):\n    \"\"\"Set a value in a dict. If `key` contains a '.', it is assumed\n    be a path (i.e. dot-delimited string) to the value's location.\n\n    ::\n\n        >>> d = {}\n        >>> set_value(d, 'foo.bar', 42)\n        >>> d\n        {'foo': {'bar': 42}}\n    \"\"\"\n    if \".\" in key:\n        head, rest = key.split(\".\", 1)\n        target = dct.setdefault(head, {})\n        if not isinstance(target, dict):\n            raise ValueError(\n                f\"Cannot set {key} in {head} \" f\"due to existing value: {target}\"\n            )\n...\n", "needle_token_start": 5474, "needle_token_end": 5489, "code_context_ntokens": 9485, "output": ["```python\nclass OrderedSet(MutableSet):\n    def __init__(self, iterable=None):\n        self.end = end = []\n        end += [None, end, end]  # sentinel node for doubly linked list\n        self.map = {}  # key --> [key, prev, next]\n        if iterable is not None:\n            self |= iterable\n\n    def __len__(self):\n        return len(self.map)\n\n    def __contains__(self, key):\n        return key in self.map\n\n    def add(self, key):\n        if key not in self.map:\n            end = self.end\n            curr = end[1]\n            curr[2] = end[1] = self.map[key] = [key, curr, end]\n\n    def discard(self, key):\n        if key in self.map:\n            key, prev, next = self.map.pop(key)\n            prev[2] = next\n            next[1] = prev\n\n    def __iter__(self):\n        end = self.end\n        curr = end[2]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[2]\n\n    def __reversed__(self):\n        end = self.end\n        curr = end[1]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[1]\n\n    def pop(self, last=True):\n        if not self:\n            raise KeyError(\"set is empty\")\n        key = self.end[1][0] if last else self.end[2][0]\n        self.discard(key)\n        return key\n\n    def __repr__(self):\n        if not self:\n            return f\"{self.__class__.__name__}()\"\n        return f\"{self.__class__.__name__}({list(self)!r})\"\n\n    def __eq__(self, other):\n        if isinstance(other, OrderedSet):\n            return len(self) == len(other) and list(self) == list(other)\n        return set(self) == set(other)\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "normalized_messages", "language": "python", "path": "src/marshmallow/exceptions.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To format error messages in a consistent dictionary structure, facilitating easier error handling and display.\n2. **Input**: This method does not take external inputs as it operates on the instance's attributes.\n3. **Output**: A dictionary where keys represent field names and values are the corresponding error messages. If the error is schema-level, the output will be the error message dictionary directly.\n4. **Procedure**: The method checks if the error is a schema-level error and if the error messages are already in dictionary form. If both conditions are met, it returns the error messages as is. Otherwise, it constructs a new dictionary with the field name as the key and the error messages as the value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/marshmallow/decorators.py\n\"\"\"Decorators for registering schema pre-processing and post-processing methods.\nThese should be imported from the top-level `marshmallow` module.\n\nMethods decorated with\n`pre_load <marshmallow.decorators.pre_load>`, `post_load <marshmallow.decorators.post_load>`,\n`pre_dump <marshmallow.decorators.pre_dump>`, `post_dump <marshmallow.decorators.post_dump>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``many`` as a keyword argument. In addition, `pre_load <marshmallow.decorators.pre_load>`,\n`post_load <marshmallow.decorators.post_load>`,\nand `validates_schema <marshmallow.decorators.validates_schema>` receive\n``partial``. If you don't need these arguments, add ``**kwargs`` to your method\nsignature.\n\n\nExample: ::\n\n    from marshmallow import (\n        Schema, pre_load, pre_dump, post_load, validates_schema,\n        validates, fields, ValidationError\n    )\n\n    class UserSchema(Schema):\n\n        email = fields.Str(required=True)\n        age = fields.Integer(required=True)\n\n        @post_load\n        def lowerstrip_email(self, item, many, **kwargs):\n            item['email'] = item['email'].lower().strip()\n            return item\n\n        @pre_load(pass_many=True)\n        def remove_envelope(self, data, many, **kwargs):\n            namespace = 'results' if many else 'result'\n            return data[namespace]\n\n        @post_dump(pass_many=True)\n        def add_envelope(self, data, many, **kwargs):\n            namespace = 'results' if many else 'result'\n            return {namespace: data}\n\n        @validates_schema\n        def validate_email(self, data, **kwargs):\n            if len(data['email']) < 3:\n                raise ValidationError('Email must be more than 3 characters', 'email')\n\n        @validates('age')\n        def validate_age(self, data, **kwargs):\n            if data < 14:\n                raise ValidationError('Too young!')\n\n.. note::\n    These decorators only work with instance methods. Class and static\n    methods are not supported.\n\n.. warning::\n    The invocation order of decorated methods of the same type is not guaranteed.\n    If you need to guarantee order of different processing steps, you should put\n    them in the same processing method.\n\"\"\"\nfrom __future__ import annotations\n\nimport functools\nfrom typing import Any, Callable, cast\n\nPRE_DUMP = \"pre_dump\"\nPOST_DUMP = \"post_dump\"\nPRE_LOAD = \"pre_load\"\nPOST_LOAD = \"post_load\"\nVALIDATES = \"validates\"\nVALIDATES_SCHEMA = \"validates_schema\"\n\n\nclass MarshmallowHook:\n    __marshmallow_hook__: dict[tuple[str, bool] | str, Any] | None = None\n\n\ndef validates(field_name: str) -> Callable[..., Any]:\n    \"\"\"Register a field validator.\n\n    :param str field_name: Name of the field that the method validates.\n    \"\"\"\n    return set_hook(None, VALIDATES, field_name=field_name)\n\n\ndef validates_schema(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n    skip_on_field_errors: bool = True,\n) -> Callable[..., Any]:\n    \"\"\"Register a schema-level validator.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.validate` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before unmarshalling) will be passed as\n    an additional argument to the method.\n\n    If ``skip_on_field_errors=True``, this validation method will be skipped whenever\n    validation errors have been detected when validating fields.\n\n    .. versionchanged:: 3.0.0b1\n        ``skip_on_field_errors`` defaults to `True`.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(\n        fn,\n        (VALIDATES_SCHEMA, pass_many),\n        pass_original=pass_original,\n        skip_on_field_errors=skip_on_field_errors,\n    )\n\n\ndef pre_dump(\n    fn: Callable[..., Any] | None = None, pass_many: bool = False\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke before serializing an object. The method\n    receives the object to be serialized and returns the processed object.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.dump` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    .. versionchanged:: 3.0.0\n        ``many`` is always passed as a keyword arguments to the decorated method.\n    \"\"\"\n    return set_hook(fn, (PRE_DUMP, pass_many))\n\n\ndef post_dump(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke after serializing an object. The method\n    receives the serialized object and returns the processed object.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.dump` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before serializing) will be passed as\n    an additional argument to the method.\n\n    .. versionchanged:: 3.0.0\n        ``many`` is always passed as a keyword arguments to the decorated method.\n    \"\"\"\n    return set_hook(fn, (POST_DUMP, pass_many), pass_original=pass_original)\n\n\ndef pre_load(\n    fn: Callable[..., Any] | None = None, pass_many: bool = False\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke before deserializing an object. The method\n    receives the data to be deserialized and returns the processed data.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.load` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(fn, (PRE_LOAD, pass_many))\n\n\ndef post_load(\n    fn: Callable[..., Any] | None = None,\n    pass_many: bool = False,\n    pass_original: bool = False,\n) -> Callable[..., Any]:\n    \"\"\"Register a method to invoke after deserializing an object. The method\n    receives the deserialized data and returns the processed data.\n\n    By default it receives a single object at a time, transparently handling the ``many``\n    argument passed to the `Schema`'s :func:`~marshmallow.Schema.load` call.\n    If ``pass_many=True``, the raw data (which may be a collection) is passed.\n\n    If ``pass_original=True``, the original data (before deserializing) will be passed as\n    an additional argument to the method.\n\n    .. versionchanged:: 3.0.0\n        ``partial`` and ``many`` are always passed as keyword arguments to\n        the decorated method.\n    \"\"\"\n    return set_hook(fn, (POST_LOAD, pass_many), pass_original=pass_original)\n\n\ndef set_hook(\n    fn: Callable[..., Any] | None, key: tuple[str, bool] | str, **kwargs: Any\n) -> Callable[..., Any]:\n    \"\"\"Mark decorated function as a hook to be picked up later.\n    You should not need to use this method directly.\n\n    .. note::\n        Currently only works with functions and instance methods. Class and\n        static methods are not supported.\n\n    :return: Decorated function if supplied, else this decorator with its args\n        bound.\n    \"\"\"\n    # Allow using this as either a decorator or a decorator factory.\n    if fn is None:\n        return functools.partial(set_hook, key=key, **kwargs)\n\n    # Set a __marshmallow_hook__ attribute instead of wrapping in some class,\n    # because I still want this to end up as a normal (unbound) method.\n    function = cast(MarshmallowHook, fn)\n    try:\n        hook_config = function.__marshmallow_hook__\n    except AttributeError:\n        function.__marshmallow_hook__ = hook_config = {}\n    # Also save the kwargs for the tagged function on\n    # __marshmallow_hook__, keyed by (<tag>, <pass_many>)\n    if hook_config is not None:\n        hook_config[key] = kwargs\n\n    return fn\n\n# Path: src/marshmallow/exceptions.py\n\"\"\"Exception classes for marshmallow-related errors.\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\n# Key used for schema-level validation errors\nSCHEMA = \"_schema\"\n\n\nclass MarshmallowError(Exception):\n    \"\"\"Base class for all marshmallow-related errors.\"\"\"\n\n\nclass ValidationError(MarshmallowError):\n    \"\"\"Raised when validation fails on a field or schema.\n\n    Validators and custom fields should raise this exception.\n\n    :param message: An error message, list of error messages, or dict of\n        error messages. If a dict, the keys are subitems and the values are error messages.\n    :param field_name: Field name to store the error on.\n        If `None`, the error is stored as schema-level error.\n    :param data: Raw input data.\n    :param valid_data: Valid (de)serialized data.\n    \"\"\"\n\n    def __init__(\n        self,\n        message: str | list | dict,\n        field_name: str = SCHEMA,\n        data: typing.Mapping[str, typing.Any]\n        | typing.Iterable[typing.Mapping[str, typing.Any]]\n        | None = None,\n        valid_data: list[dict[str, typing.Any]] | dict[str, typing.Any] | None = None,\n        **kwargs,\n    ):\n        self.messages = [message] if isinstance(message, (str, bytes)) else message\n        self.field_name = field_name\n        self.data = data\n        self.valid_data = valid_data\n        self.kwargs = kwargs\n        super().__init__(message)\n\n    \ndef normalized_messages(self):\n        if self.field_name == SCHEMA and isinstance(self.messages, dict):\n            return self.messages\n        return {self.field_name: self.messages}\n\n    @property\n    def messages_dict(self) -> dict[str, typing.Any]:\n        if not isinstance(self.messages, dict):\n            raise TypeError(\n                \"cannot access 'messages_dict' when 'messages' is of type \"\n                + type(self.messages).__name__\n            )\n        return self.messages\n\n\nclass RegistryError(NameError):\n    \"\"\"Raised when an invalid operation is performed on the serializer\n    class registry.\n    \"\"\"\n\n\nclass StringNotCollectionError(MarshmallowError, TypeError):\n    \"\"\"Raised when a string is passed when a list of strings is expected.\"\"\"\n\n\nclass FieldInstanceResolutionError(MarshmallowError, TypeError):\n    \"\"\"Raised when schema to instantiate is neither a Schema class nor an instance.\"\"\"\n\n# Path: src/marshmallow/base.py\n\"\"\"Abstract base classes.\n\nThese are necessary to avoid circular imports between schema.py and fields.py.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nfrom abc import ABC, abstractmethod\n\n\nclass FieldABC(ABC):\n    \"\"\"Abstract base class from which all Field classes inherit.\"\"\"\n\n    parent = None\n    name = None\n    root = None\n\n    @abstractmethod\n    def serialize(self, attr, obj, accessor=None):\n        pass\n\n    @abstractmethod\n    def deserialize(self, value):\n        pass\n\n    @abstractmethod\n    def _serialize(self, value, attr, obj, **kwargs):\n        pass\n\n    @abstractmethod\n    def _deserialize(self, value, attr, data, **kwargs):\n        pass\n\n\nclass SchemaABC(ABC):\n    \"\"\"Abstract base class from which all Schemas inherit.\"\"\"\n\n    @abstractmethod\n    def dump(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def dumps(self, obj, *, many: bool | None = None):\n        pass\n\n    @abstractmethod\n    def load(self, data, *, many: bool | None = None, partial=None, unknown=None):\n        pass\n\n    @abstractmethod\n    def loads(\n        self,\n        json_data,\n        *,\n        many: bool | None = None,\n        partial=None,\n        unknown=None,\n        **kwargs,\n    ):\n        pass\n\n# Path: src/marshmallow/class_registry.py\n\"\"\"A registry of :class:`Schema <marshmallow.Schema>` classes. This allows for string\nlookup of schemas, which may be used with\nclass:`fields.Nested <marshmallow.fields.Nested>`.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\nfrom __future__ import annotations\n\nimport typing\n\nfrom marshmallow.exceptions import RegistryError\n\nif typing.TYPE_CHECKING:\n    from marshmallow import Schema\n\n    SchemaType = typing.Type[Schema]\n\n# {\n#   <class_name>: <list of class objects>\n#   <module_path_to_class>: <list of class objects>\n# }\n_registry = {}  # type: dict[str, list[SchemaType]]\n\n\ndef register(classname: str, cls: SchemaType) -> None:\n    \"\"\"Add a class to the registry of serializer classes. When a class is\n    registered, an entry for both its classname and its full, module-qualified\n    path are added to the registry.\n\n    Example: ::\n\n        class MyClass:\n            pass\n\n        register('MyClass', MyClass)\n        # Registry:\n        # {\n        #   'MyClass': [path.to.MyClass],\n        #   'path.to.MyClass': [path.to.MyClass],\n        # }\n\n    \"\"\"\n    # Module where the class is located\n    module = cls.__module__\n    # Full module path to the class\n    # e.g. user.schemas.UserSchema\n    fullpath = \".\".join([module, classname])\n    # If the class is already registered; need to check if the entries are\n    # in the same module as cls to avoid having multiple instances of the same\n    # class in the registry\n    if classname in _registry and not any(\n        each.__module__ == module for each in _registry[classname]\n    ):\n        _registry[classname].append(cls)\n    elif classname not in _registry:\n        _registry[classname] = [cls]\n\n    # Also register the full path\n    if fullpath not in _registry:\n        _registry.setdefault(fullpath, []).append(cls)\n    else:\n        # If fullpath does exist, replace existing entry\n        _registry[fullpath] = [cls]\n    return None\n\n\ndef get_class(classname: str, all: bool = False) -> list[SchemaType] | SchemaType:\n    \"\"\"Retrieve a class from the registry.\n\n    :raises: marshmallow.exceptions.RegistryError if the class cannot be found\n        or if there are multiple entries for the given class name.\n    \"\"\"\n    try:\n        classes = _registry[classname]\n    except KeyError as error:\n        raise RegistryError(\n            f\"Class with name {classname!r} was not found. You may need \"\n            \"to import the class.\"\n        ) from error\n    if len(classes) > 1:\n        if all:\n            return _registry[classname]\n        raise RegistryError(\n            f\"Multiple classes with name {classname!r} \"\n            \"were found. Please use the full, \"\n            \"module-qualified path.\"\n        )\n    else:\n        return _registry[classname][0]\n\n# Path: src/marshmallow/error_store.py\n\"\"\"Utilities for storing collections of error messages.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\n\nfrom marshmallow.exceptions import SCHEMA\n\n\nclass ErrorStore:\n    def __init__(self):\n        #: Dictionary of errors stored during serialization\n        self.errors = {}\n\n    def store_error(self, messages, field_name=SCHEMA, index=None):\n        # field error  -> store/merge error messages under field name key\n        # schema error -> if string or list, store/merge under _schema key\n        #              -> if dict, store/merge with other top-level keys\n        if field_name != SCHEMA or not isinstance(messages, dict):\n            messages = {field_name: messages}\n        if index is not None:\n            messages = {index: messages}\n        self.errors = merge_errors(self.errors, messages)\n\n\ndef merge_errors(errors1, errors2):\n    \"\"\"Deeply merge two error messages.\n\n    The format of ``errors1`` and ``errors2`` matches the ``message``\n    parameter of :exc:`marshmallow.exceptions.ValidationError`.\n    \"\"\"\n    if not errors1:\n        return errors2\n    if not errors2:\n        return errors1\n    if isinstance(errors1, list):\n        if isinstance(errors2, list):\n            return errors1 + errors2\n        if isinstance(errors2, dict):\n            return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n        return errors1 + [errors2]\n    if isinstance(errors1, dict):\n        if isinstance(errors2, list):\n            return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n        if isinstance(errors2, dict):\n            errors = dict(errors1)\n            for key, val in errors2.items():\n                if key in errors:\n                    errors[key] = merge_errors(errors[key], val)\n                else:\n                    errors[key] = val\n            return errors\n        return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n    if isinstance(errors2, list):\n        return [errors1] + errors2\n    if isinstance(errors2, dict):\n        return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n    return [errors1, errors2]\n\n# Path: src/marshmallow/orderedset.py\n# OrderedSet\n# Copyright (c) 2009 Raymond Hettinger\n#\n# Permission is hereby granted, free of charge, to any person\n# obtaining a copy of this software and associated documentation files\n# (the \"Software\"), to deal in the Software without restriction,\n# including without limitation the rights to use, copy, modify, merge,\n# publish, distribute, sublicense, and/or sell copies of the Software,\n# and to permit persons to whom the Software is furnished to do so,\n# subject to the following conditions:\n#\n#     The above copyright notice and this permission notice shall be\n#     included in all copies or substantial portions of the Software.\n#\n#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES\n#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\n...\n", "needle_token_start": 2849, "needle_token_end": 2899, "code_context_ntokens": 5279, "output": ["```python\nclass ErrorStore:\n    def __init__(self):\n        #: Dictionary of errors stored during serialization\n        self.errors = {}\n\n    def store_error(self, messages, field_name=SCHEMA, index=None):\n        # field error  -> store/merge error messages under field name key\n        # schema error -> if string or list, store/merge under _schema key\n        #              -> if dict, store/merge with other top-level keys\n        if field_name != SCHEMA or not isinstance(messages, dict):\n            messages = {field_name: messages}\n        if index is not None:\n            messages = {index: messages}\n        self.errors = merge_errors(self.errors, messages)\n\n\ndef merge_errors(errors1, errors2):\n    \"\"\"Deeply merge two error messages.\n\n    The format of ``errors1`` and ``errors2`` matches the ``message``\n    parameter of :exc:`marshmallow.exceptions.ValidationError`.\n    \"\"\"\n    if not errors1:\n        return errors2\n    if not errors2:\n        return errors1\n    if isinstance(errors1, list):\n        if isinstance(errors2, list):\n            return errors1 + errors2\n        if isinstance(errors2, dict):\n            return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n        return errors1 + [errors2]\n    if isinstance(errors1, dict):\n        if isinstance(errors2, list):\n            return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n        if isinstance(errors2, dict):\n            errors = dict(errors1)\n            for key, val in errors2.items():\n                if key in errors:\n                    errors[key] = merge_errors(errors[key], val)\n                else:\n                    errors[key] = val\n            return errors\n        return dict(errors1, **{SCHEMA: merge_errors(errors1.get(SCHEMA), errors2)})\n    if isinstance(errors2, list):\n        return [errors1] + errors2\n    if isinstance(errors2, dict):\n        return dict(errors2, **{SCHEMA: merge_errors(errors1, errors2.get(SCHEMA))})\n    return [errors1, errors2]\n```"]}
{"repo": "marshmallow-code/marshmallow", "name": "_normalize_nested_options", "language": "python", "path": "src/marshmallow/schema.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To adjust and simplify the settings related to field inclusion and exclusion for nested schemas within a data processing context.\n2. **Input**: The function does not take parameters directly; it operates on the internal state of an object that includes settings for which fields to include or exclude.\n3. **Output**: There is no direct output; the function modifies the internal state of the object.\n4. **Procedure**: \n   - If there is a specification on which fields to include, this specification is applied recursively to nested fields, and then the specification is simplified to only include the top-level field names.\n   - If there is a specification on which fields to exclude, this specification is also applied recursively to nested fields. Subsequently, the specification is adjusted to exclude any top-level field names that do not contain nested specifications.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: src/marshmallow/error_store.py\n\"\"\"Utilities for storing collections of error messages.\n\n.. warning::\n\n    This module is treated as private API.\n    Users should not need to use this module directly.\n\"\"\"\n\nfrom marshmallow.exceptions import SCHEMA\n\n\nclass ErrorStore:\n    def __init__(self):\n        #: Dictionary of errors stored during serialization\n        self.errors = {}\n\n    def store_error(self, messages, field_name=SCHEMA, index=None):\n        # field error  -> store/merge error messages under field name key\n        # schema error -> if string or list, store/merge under _schema key\n        #              -> if dict, store/merge with other top-level keys\n        if field_name != SCHEMA or not isinstance(messages, dict):\n            messages = {field_name: messages}\n        if index is not None:\n            messages = {index: messages}\n        self.errors = merge_errors(self.errors, messages)\n\n\ndef merge_errors(errors1, errors2):\n    \"\"\"Deeply merge two error messages.\n\n    The format of ``errors1`` and ``errors2`` matches the ``message``\n    parameter of :exc:`marshmallow.exceptions.ValidationError`.\n    \"\"\"\n    if not errors1:\n...\n# Path: src/marshmallow/orderedset.py\n# OrderedSet\n# Copyright (c) 2009 Raymond Hettinger\n#\n# Permission is hereby granted, free of charge, to any person\n# obtaining a copy of this software and associated documentation files\n# (the \"Software\"), to deal in the Software without restriction,\n# including without limitation the rights to use, copy, modify, merge,\n# publish, distribute, sublicense, and/or sell copies of the Software,\n# and to permit persons to whom the Software is furnished to do so,\n# subject to the following conditions:\n#\n#     The above copyright notice and this permission notice shall be\n#     included in all copies or substantial portions of the Software.\n#\n#     THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\n#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES\n#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\n#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT\n#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR\n#     OTHER DEALINGS IN THE SOFTWARE.\nfrom collections.abc import MutableSet\n\n\nclass OrderedSet(MutableSet):\n    def __init__(self, iterable=None):\n        self.end = end = []\n        end += [None, end, end]  # sentinel node for doubly linked list\n        self.map = {}  # key --> [key, prev, next]\n        if iterable is not None:\n            self |= iterable\n\n    def __len__(self):\n        return len(self.map)\n\n    def __contains__(self, key):\n        return key in self.map\n\n    def add(self, key):\n        if key not in self.map:\n            end = self.end\n            curr = end[1]\n            curr[2] = end[1] = self.map[key] = [key, curr, end]\n\n    def discard(self, key):\n        if key in self.map:\n            key, prev, next = self.map.pop(key)\n            prev[2] = next\n            next[1] = prev\n\n    def __iter__(self):\n        end = self.end\n        curr = end[2]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[2]\n\n    def __reversed__(self):\n        end = self.end\n        curr = end[1]\n        while curr is not end:\n            yield curr[0]\n            curr = curr[1]\n\n    def pop(self, last=True):\n        if not self:\n            raise KeyError(\"set is empty\")\n        key = self.end[1][0] if last else self.end[2][0]\n        self.discard(key)\n        return key\n\n    def __repr__(self):\n        if not self:\n            return f\"{self.__class__.__name__}()\"\n        return f\"{self.__class__.__name__}({list(self)!r})\"\n\n    def __eq__(self, other):\n        if isinstance(other, OrderedSet):\n            return len(self) == len(other) and list(self) == list(other)\n        return set(self) == set(other)\n\n\nif __name__ == \"__main__\":\n    s = OrderedSet(\"abracadaba\")\n    t = OrderedSet(\"simsalabim\")\n    print(s | t)\n    print(s & t)\n    print(s - t)\n\n# Path: src/marshmallow/types.py\n\"\"\"Type aliases.\n\n.. warning::\n\n    This module is provisional. Types may be modified, added, and removed between minor releases.\n\"\"\"\nimport typing\n\nStrSequenceOrSet = typing.Union[typing.Sequence[str], typing.AbstractSet[str]]\nTag = typing.Union[str, typing.Tuple[str, bool]]\nValidator = typing.Callable[[typing.Any], typing.Any]\n\n# Path: src/marshmallow/warnings.py\nclass RemovedInMarshmallow4Warning(DeprecationWarning):\n    pass\n\n# Path: src/marshmallow/utils.py\n\"\"\"Utility methods for marshmallow.\"\"\"\nfrom __future__ import annotations\n\nimport collections\nimport datetime as dt\nimport functools\nimport inspect\nimport json\nimport re\nimport typing\nimport warnings\nfrom collections.abc import Mapping\nfrom email.utils import format_datetime, parsedate_to_datetime\nfrom pprint import pprint as py_pprint\n\nfrom marshmallow.base import FieldABC\nfrom marshmallow.exceptions import FieldInstanceResolutionError\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\nEXCLUDE = \"exclude\"\nINCLUDE = \"include\"\nRAISE = \"raise\"\n_UNKNOWN_VALUES = {EXCLUDE, INCLUDE, RAISE}\n\n\nclass _Missing:\n    def __bool__(self):\n        return False\n\n    def __copy__(self):\n        return self\n\n    def __deepcopy__(self, _):\n        return self\n\n    def __repr__(self):\n        return \"<marshmallow.missing>\"\n\n\n# Singleton value that indicates that a field's value is missing from input\n# dict passed to :meth:`Schema.load`. If the field's value is not required,\n# it's ``default`` value is used.\nmissing = _Missing()\n\n\ndef is_generator(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a generator\"\"\"\n    return inspect.isgeneratorfunction(obj) or inspect.isgenerator(obj)\n\n\ndef is_iterable_but_not_string(obj) -> bool:\n    \"\"\"Return True if ``obj`` is an iterable object that isn't a string.\"\"\"\n    return (hasattr(obj, \"__iter__\") and not hasattr(obj, \"strip\")) or is_generator(obj)\n\n\ndef is_collection(obj) -> bool:\n    \"\"\"Return True if ``obj`` is a collection type, e.g list, tuple, queryset.\"\"\"\n    return is_iterable_but_not_string(obj) and not isinstance(obj, Mapping)\n\n\ndef is_instance_or_subclass(val, class_) -> bool:\n    \"\"\"Return True if ``val`` is either a subclass or instance of ``class_``.\"\"\"\n    try:\n        return issubclass(val, class_)\n    except TypeError:\n        return isinstance(val, class_)\n\n\ndef is_keyed_tuple(obj) -> bool:\n    \"\"\"Return True if ``obj`` has keyed tuple behavior, such as\n    namedtuples or SQLAlchemy's KeyedTuples.\n    \"\"\"\n    return isinstance(obj, tuple) and hasattr(obj, \"_fields\")\n\n\ndef pprint(obj, *args, **kwargs) -> None:\n    \"\"\"Pretty-printing function that can pretty-print OrderedDicts\n    like regular dictionaries. Useful for printing the output of\n    :meth:`marshmallow.Schema.dump`.\n\n    .. deprecated:: 3.7.0\n        marshmallow.pprint will be removed in marshmallow 4.\n    \"\"\"\n    warnings.warn(\n        \"marshmallow's pprint function is deprecated and will be removed in marshmallow 4.\",\n        RemovedInMarshmallow4Warning,\n        stacklevel=2,\n    )\n    if isinstance(obj, collections.OrderedDict):\n        print(json.dumps(obj, *args, **kwargs))\n    else:\n        py_pprint(obj, *args, **kwargs)\n\n\n# https://stackoverflow.com/a/27596917\ndef is_aware(datetime: dt.datetime) -> bool:\n    return (\n        datetime.tzinfo is not None and datetime.tzinfo.utcoffset(datetime) is not None\n    )\n\n\ndef from_rfc(datestring: str) -> dt.datetime:\n    \"\"\"Parse a RFC822-formatted datetime string and return a datetime object.\n\n    https://stackoverflow.com/questions/885015/how-to-parse-a-rfc-2822-date-time-into-a-python-datetime  # noqa: B950\n    \"\"\"\n    return parsedate_to_datetime(datestring)\n\n\ndef rfcformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the RFC822-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return format_datetime(datetime)\n\n\n# Hat tip to Django for ISO8601 deserialization functions\n\n_iso8601_datetime_re = re.compile(\n    r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})\"\n    r\"[T ](?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n    r\"(?P<tzinfo>Z|[+-]\\d{2}(?::?\\d{2})?)?$\"\n)\n\n_iso8601_date_re = re.compile(r\"(?P<year>\\d{4})-(?P<month>\\d{1,2})-(?P<day>\\d{1,2})$\")\n\n_iso8601_time_re = re.compile(\n    r\"(?P<hour>\\d{1,2}):(?P<minute>\\d{1,2})\"\n    r\"(?::(?P<second>\\d{1,2})(?:\\.(?P<microsecond>\\d{1,6})\\d{0,6})?)?\"\n)\n\n\ndef get_fixed_timezone(offset: int | float | dt.timedelta) -> dt.timezone:\n    \"\"\"Return a tzinfo instance with a fixed offset from UTC.\"\"\"\n    if isinstance(offset, dt.timedelta):\n        offset = offset.total_seconds() // 60\n    sign = \"-\" if offset < 0 else \"+\"\n    hhmm = \"%02d%02d\" % divmod(abs(offset), 60)\n    name = sign + hhmm\n    return dt.timezone(dt.timedelta(minutes=offset), name)\n\n\ndef from_iso_datetime(value):\n    \"\"\"Parse a string and return a datetime.datetime.\n\n    This function supports time zone offsets. When the input contains one,\n    the output uses a timezone with a fixed offset from UTC.\n    \"\"\"\n    match = _iso8601_datetime_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted datetime string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    tzinfo = kw.pop(\"tzinfo\")\n    if tzinfo == \"Z\":\n        tzinfo = dt.timezone.utc\n    elif tzinfo is not None:\n        offset_mins = int(tzinfo[-2:]) if len(tzinfo) > 3 else 0\n        offset = 60 * int(tzinfo[1:3]) + offset_mins\n        if tzinfo[0] == \"-\":\n            offset = -offset\n        tzinfo = get_fixed_timezone(offset)\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    kw[\"tzinfo\"] = tzinfo\n    return dt.datetime(**kw)\n\n\ndef from_iso_time(value):\n    \"\"\"Parse a string and return a datetime.time.\n\n    This function doesn't support time zone offsets.\n    \"\"\"\n    match = _iso8601_time_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted time string\")\n    kw = match.groupdict()\n    kw[\"microsecond\"] = kw[\"microsecond\"] and kw[\"microsecond\"].ljust(6, \"0\")\n    kw = {k: int(v) for k, v in kw.items() if v is not None}\n    return dt.time(**kw)\n\n\ndef from_iso_date(value):\n    \"\"\"Parse a string and return a datetime.date.\"\"\"\n    match = _iso8601_date_re.match(value)\n    if not match:\n        raise ValueError(\"Not a valid ISO8601-formatted date string\")\n    kw = {k: int(v) for k, v in match.groupdict().items()}\n    return dt.date(**kw)\n\n\ndef from_timestamp(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    if value < 0:\n        raise ValueError(\"Not a valid POSIX timestamp\")\n\n    # Load a timestamp with utc as timezone to prevent using system timezone.\n    # Then set timezone to None, to let the Field handle adding timezone info.\n    try:\n        return dt.datetime.fromtimestamp(value, tz=dt.timezone.utc).replace(tzinfo=None)\n    except OverflowError as exc:\n        raise ValueError(\"Timestamp is too large\") from exc\n    except OSError as exc:\n        raise ValueError(\"Error converting value to datetime\") from exc\n\n\ndef from_timestamp_ms(value: typing.Any) -> dt.datetime:\n    value = float(value)\n    return from_timestamp(value / 1000)\n\n\ndef timestamp(\n    value: dt.datetime,\n) -> float:\n    if not is_aware(value):\n        # When a date is naive, use UTC as zone info to prevent using system timezone.\n        value = value.replace(tzinfo=dt.timezone.utc)\n    return value.timestamp()\n\n\ndef timestamp_ms(value: dt.datetime) -> float:\n    return timestamp(value) * 1000\n\n\ndef isoformat(datetime: dt.datetime) -> str:\n    \"\"\"Return the ISO8601-formatted representation of a datetime object.\n\n    :param datetime datetime: The datetime.\n    \"\"\"\n    return datetime.isoformat()\n\n\ndef to_iso_time(time: dt.time) -> str:\n    return dt.time.isoformat(time)\n\n\ndef to_iso_date(date: dt.date) -> str:\n    return dt.date.isoformat(date)\n\n\ndef ensure_text_type(val: str | bytes) -> str:\n    if isinstance(val, bytes):\n        val = val.decode(\"utf-8\")\n    return str(val)\n\n\ndef pluck(dictlist: list[dict[str, typing.Any]], key: str):\n    \"\"\"Extracts a list of dictionary values from a list of dictionaries.\n    ::\n\n        >>> dlist = [{'id': 1, 'name': 'foo'}, {'id': 2, 'name': 'bar'}]\n        >>> pluck(dlist, 'id')\n        [1, 2]\n    \"\"\"\n    return [d[key] for d in dictlist]\n\n\n# Various utilities for pulling keyed values from objects\n\n\ndef get_value(obj, key: int | str, default=missing):\n    \"\"\"Helper for pulling a keyed value off various types of objects. Fields use\n    this method by default to access attributes of the source object. For object `x`\n    and attribute `i`, this method first tries to access `x[i]`, and then falls back to\n    `x.i` if an exception is raised.\n\n    .. warning::\n        If an object `x` does not raise an exception when `x[i]` does not exist,\n        `get_value` will never check the value `x.i`. Consider overriding\n        `marshmallow.fields.Field.get_value` in this case.\n    \"\"\"\n    if not isinstance(key, int) and \".\" in key:\n        return _get_value_for_keys(obj, key.split(\".\"), default)\n    else:\n        return _get_value_for_key(obj, key, default)\n\n\ndef _get_value_for_keys(obj, keys, default):\n    if len(keys) == 1:\n        return _get_value_for_key(obj, keys[0], default)\n    else:\n        return _get_value_for_keys(\n            _get_value_for_key(obj, keys[0], default), keys[1:], default\n        )\n\n\ndef _get_value_for_key(obj, key, default):\n    if not hasattr(obj, \"__getitem__\"):\n        return getattr(obj, key, default)\n\n    try:\n        return obj[key]\n    except (KeyError, IndexError, TypeError, AttributeError):\n        return getattr(obj, key, default)\n\n\ndef set_value(dct: dict[str, typing.Any], key: str, value: typing.Any):\n    \"\"\"Set a value in a dict. If `key` contains a '.', it is assumed\n    be a path (i.e. dot-delimited string) to the value's location.\n\n    ::\n\n        >>> d = {}\n        >>> set_value(d, 'foo.bar', 42)\n        >>> d\n        {'foo': {'bar': 42}}\n    \"\"\"\n    if \".\" in key:\n        head, rest = key.split(\".\", 1)\n        target = dct.setdefault(head, {})\n        if not isinstance(target, dict):\n            raise ValueError(\n                f\"Cannot set {key} in {head} \" f\"due to existing value: {target}\"\n            )\n        set_value(target, rest, value)\n    else:\n        dct[key] = value\n\n\ndef callable_or_raise(obj):\n    \"\"\"Check that an object is callable, else raise a :exc:`TypeError`.\"\"\"\n    if not callable(obj):\n        raise TypeError(f\"Object {obj!r} is not callable.\")\n    return obj\n\n\ndef _signature(func: typing.Callable) -> list[str]:\n    return list(inspect.signature(func).parameters.keys())\n\n\ndef get_func_args(func: typing.Callable) -> list[str]:\n    \"\"\"Given a callable, return a list of argument names. Handles\n    `functools.partial` objects and class-based callables.\n\n    .. versionchanged:: 3.0.0a1\n        Do not return bound arguments, eg. ``self``.\n    \"\"\"\n    if inspect.isfunction(func) or inspect.ismethod(func):\n        return _signature(func)\n    if isinstance(func, functools.partial):\n        return _signature(func.func)\n    # Callable class\n    return _signature(func)\n\n\ndef resolve_field_instance(cls_or_instance):\n    \"\"\"Return a Schema instance from a Schema class or instance.\n\n    :param type|Schema cls_or_instance: Marshmallow Schema class or instance.\n    \"\"\"\n    if isinstance(cls_or_instance, type):\n        if not issubclass(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance()\n    else:\n        if not isinstance(cls_or_instance, FieldABC):\n            raise FieldInstanceResolutionError\n        return cls_or_instance\n\n\ndef timedelta_to_microseconds(value: dt.timedelta) -> int:\n    \"\"\"Compute the total microseconds of a timedelta\n\n    https://github.com/python/cpython/blob/bb3e0c240bc60fe08d332ff5955d54197f79751c/Lib/datetime.py#L665-L667  # noqa: B950\n    \"\"\"\n    return (value.days * (24 * 3600) + value.seconds) * 1000000 + value.microseconds\n\n\ndef validate_unknown_parameter_value(obj: typing.Any) -> str:\n    if obj not in _UNKNOWN_VALUES:\n        raise ValueError(\n            f\"Object {obj!r} is not a valid value for the 'unknown' parameter\"\n        )\n    return obj\n\n# Path: src/marshmallow/schema.py\n\"\"\"The :class:`Schema` class, including its metaclass and options (class Meta).\"\"\"\nfrom __future__ import annotations\n\nimport copy\nimport datetime as dt\nimport decimal\nimport inspect\nimport json\nimport typing\nimport uuid\nimport warnings\nfrom abc import ABCMeta\nfrom collections import OrderedDict, defaultdict\nfrom collections.abc import Mapping\nfrom functools import lru_cache\n\nfrom marshmallow import base, class_registry, types\nfrom marshmallow import fields as ma_fields\nfrom marshmallow.decorators import (\n    POST_DUMP,\n    POST_LOAD,\n    PRE_DUMP,\n    PRE_LOAD,\n    VALIDATES,\n    VALIDATES_SCHEMA,\n)\nfrom marshmallow.error_store import ErrorStore\nfrom marshmallow.exceptions import StringNotCollectionError, ValidationError\nfrom marshmallow.orderedset import OrderedSet\nfrom marshmallow.utils import (\n    EXCLUDE,\n    INCLUDE,\n    RAISE,\n    get_value,\n    is_collection,\n    is_instance_or_subclass,\n    missing,\n    set_value,\n    validate_unknown_parameter_value,\n)\nfrom marshmallow.warnings import RemovedInMarshmallow4Warning\n\n_T = typing.TypeVar(\"_T\")\n\n\ndef _get_fields(attrs):\n    \"\"\"Get fields from a class\n\n    :param attrs: Mapping of class attributes\n    \"\"\"\n    return [\n        (field_name, field_value)\n        for field_name, field_value in attrs.items()\n        if is_instance_or_subclass(field_value, base.FieldABC)\n    ]\n\n\n# This function allows Schemas to inherit from non-Schema classes and ensures\n#   inheritance according to the MRO\ndef _get_fields_by_mro(klass):\n    \"\"\"Collect fields from a class, following its method resolution order. The\n    class itself is excluded from the search; only its parents are checked. Get\n    fields from ``_declared_fields`` if available, else use ``__dict__``.\n\n    :param type klass: Class whose fields to retrieve\n    \"\"\"\n    mro = inspect.getmro(klass)\n    # Loop over mro in reverse to maintain correct order of fields\n    return sum(\n        (\n            _get_fields(\n                getattr(base, \"_declared_fields\", base.__dict__),\n            )\n            for base in mro[:0:-1]\n        ),\n        [],\n    )\n\n\nclass SchemaMeta(ABCMeta):\n    \"\"\"Metaclass for the Schema class. Binds the declared fields to\n    a ``_declared_fields`` attribute, which is a dictionary mapping attribute\n    names to field objects. Also sets the ``opts`` class attribute, which is\n    the Schema class's ``class Meta`` options.\n    \"\"\"\n\n    def __new__(mcs, name, bases, attrs):\n        meta = attrs.get(\"Meta\")\n        ordered = getattr(meta, \"ordered\", False)\n        if not ordered:\n            # Inherit 'ordered' option\n            # Warning: We loop through bases instead of MRO because we don't\n            # yet have access to the class object\n            # (i.e. can't call super before we have fields)\n            for base_ in bases:\n                if hasattr(base_, \"Meta\") and hasattr(base_.Meta, \"ordered\"):\n                    ordered = base_.Meta.ordered\n                    break\n            else:\n                ordered = False\n        cls_fields = _get_fields(attrs)\n        # Remove fields from list of class attributes to avoid shadowing\n        # Schema attributes/methods in case of name conflict\n        for field_name, _ in cls_fields:\n            del attrs[field_name]\n        klass = super().__new__(mcs, name, bases, attrs)\n        inherited_fields = _get_fields_by_mro(klass)\n\n        meta = klass.Meta\n        # Set klass.opts in __new__ rather than __init__ so that it is accessible in\n        # get_declared_fields\n        klass.opts = klass.OPTIONS_CLASS(meta, ordered=ordered)\n        # Add fields specified in the `include` class Meta option\n        cls_fields += list(klass.opts.include.items())\n\n        # Assign _declared_fields on class\n        klass._declared_fields = mcs.get_declared_fields(\n            klass=klass,\n            cls_fields=cls_fields,\n            inherited_fields=inherited_fields,\n            dict_cls=dict,\n        )\n        return klass\n\n    @classmethod\n    def get_declared_fields(\n        mcs,\n        klass: type,\n        cls_fields: list,\n        inherited_fields: list,\n        dict_cls: type = dict,\n    ):\n        \"\"\"Returns a dictionary of field_name => `Field` pairs declared on the class.\n        This is exposed mainly so that plugins can add additional fields, e.g. fields\n        computed from class Meta options.\n\n        :param klass: The class object.\n        :param cls_fields: The fields declared on the class, including those added\n            by the ``include`` class Meta option.\n        :param inherited_fields: Inherited fields.\n        :param dict_cls: dict-like class to use for dict output Default to ``dict``.\n        \"\"\"\n        return dict_cls(inherited_fields + cls_fields)\n\n    def __init__(cls, name, bases, attrs):\n        super().__init__(name, bases, attrs)\n        if name and cls.opts.register:\n            class_registry.register(name, cls)\n        cls._hooks = cls.resolve_hooks()\n\n    def resolve_hooks(cls) -> dict[types.Tag, list[str]]:\n        \"\"\"Add in the decorated processors\n\n        By doing this after constructing the class, we let standard inheritance\n        do all the hard work.\n        \"\"\"\n        mro = inspect.getmro(cls)\n\n        hooks = defaultdict(list)  # type: typing.Dict[types.Tag, typing.List[str]]\n\n        for attr_name in dir(cls):\n            # Need to look up the actual descriptor, not whatever might be\n            # bound to the class. This needs to come from the __dict__ of the\n            # declaring class.\n            for parent in mro:\n                try:\n                    attr = parent.__dict__[attr_name]\n                except KeyError:\n                    continue\n                else:\n                    break\n            else:\n                # In case we didn't find the attribute and didn't break above.\n                # We should never hit this - it's just here for completeness\n                # to exclude the possibility of attr being undefined.\n                continue\n\n            try:\n                hook_config = attr.__marshmallow_hook__\n            except AttributeError:\n                pass\n            else:\n                for key in hook_config.keys():\n                    # Use name here so we can get the bound method later, in\n                    # case the processor was a descriptor or something.\n                    hooks[key].append(attr_name)\n\n        return hooks\n\n\nclass SchemaOpts:\n    \"\"\"class Meta options for the :class:`Schema`. Defines defaults.\"\"\"\n\n    def __init__(self, meta, ordered: bool = False):\n        self.fields = getattr(meta, \"fields\", ())\n        if not isinstance(self.fields, (list, tuple)):\n            raise ValueError(\"`fields` option must be a list or tuple.\")\n        self.additional = getattr(meta, \"additional\", ())\n        if not isinstance(self.additional, (list, tuple)):\n            raise ValueError(\"`additional` option must be a list or tuple.\")\n        if self.fields and self.additional:\n            raise ValueError(\n                \"Cannot set both `fields` and `additional` options\"\n                \" for the same Schema.\"\n            )\n        self.exclude = getattr(meta, \"exclude\", ())\n        if not isinstance(self.exclude, (list, tuple)):\n            raise ValueError(\"`exclude` must be a list or tuple.\")\n        self.dateformat = getattr(meta, \"dateformat\", None)\n        self.datetimeformat = getattr(meta, \"datetimeformat\", None)\n        self.timeformat = getattr(meta, \"timeformat\", None)\n        if hasattr(meta, \"json_module\"):\n            warnings.warn(\n                \"The json_module class Meta option is deprecated. Use render_module instead.\",\n                RemovedInMarshmallow4Warning,\n                stacklevel=2,\n            )\n            render_module = getattr(meta, \"json_module\", json)\n        else:\n            render_module = json\n        self.render_module = getattr(meta, \"render_module\", render_module)\n        self.ordered = getattr(meta, \"ordered\", ordered)\n        self.index_errors = getattr(meta, \"index_errors\", True)\n        self.include = getattr(meta, \"include\", {})\n        self.load_only = getattr(meta, \"load_only\", ())\n        self.dump_only = getattr(meta, \"dump_only\", ())\n        self.unknown = validate_unknown_parameter_value(getattr(meta, \"unknown\", RAISE))\n        self.register = getattr(meta, \"register\", True)\n\n\nclass Schema(base.SchemaABC, metaclass=SchemaMeta):\n    \"\"\"Base schema class with which to define custom schemas.\n\n    Example usage:\n\n    .. code-block:: python\n\n        import datetime as dt\n        from dataclasses import dataclass\n\n        from marshmallow import Schema, fields\n\n\n        @dataclass\n        class Album:\n            title: str\n            release_date: dt.date\n\n\n        class AlbumSchema(Schema):\n            title = fields.Str()\n            release_date = fields.Date()\n\n\n        album = Album(\"Beggars Banquet\", dt.date(1968, 12, 6))\n        schema = AlbumSchema()\n        data = schema.dump(album)\n        data  # {'release_date': '1968-12-06', 'title': 'Beggars Banquet'}\n\n    :param only: Whitelist of the declared fields to select when\n        instantiating the Schema. If None, all fields are used. Nested fields\n        can be represented with dot delimiters.\n    :param exclude: Blacklist of the declared fields to exclude\n        when instantiating the Schema. If a field appears in both `only` and\n        `exclude`, it is not used. Nested fields can be represented with dot\n        delimiters.\n    :param many: Should be set to `True` if ``obj`` is a collection\n        so that the object will be serialized to a list.\n    :param context: Optional context passed to :class:`fields.Method` and\n        :class:`fields.Function` fields.\n    :param load_only: Fields to skip during serialization (write-only fields)\n    :param dump_only: Fields to skip during deserialization (read-only fields)\n    :param partial: Whether to ignore missing fields and not require\n        any fields declared. Propagates down to ``Nested`` fields as well. If\n        its value is an iterable, only missing fields listed in that iterable\n        will be ignored. Use dot delimiters to specify nested fields.\n    :param unknown: Whether to exclude, include, or raise an error for unknown\n        fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n\n    .. versionchanged:: 3.0.0\n        `prefix` parameter removed.\n\n    .. versionchanged:: 2.0.0\n        `__validators__`, `__preprocessors__`, and `__data_handlers__` are removed in favor of\n        `marshmallow.decorators.validates_schema`,\n        `marshmallow.decorators.pre_load` and `marshmallow.decorators.post_dump`.\n        `__accessor__` and `__error_handler__` are deprecated. Implement the\n        `handle_error` and `get_attribute` methods instead.\n    \"\"\"\n\n    TYPE_MAPPING = {\n        str: ma_fields.String,\n        bytes: ma_fields.String,\n        dt.datetime: ma_fields.DateTime,\n        float: ma_fields.Float,\n        bool: ma_fields.Boolean,\n        tuple: ma_fields.Raw,\n        list: ma_fields.Raw,\n        set: ma_fields.Raw,\n        int: ma_fields.Integer,\n        uuid.UUID: ma_fields.UUID,\n        dt.time: ma_fields.Time,\n        dt.date: ma_fields.Date,\n        dt.timedelta: ma_fields.TimeDelta,\n        decimal.Decimal: ma_fields.Decimal,\n    }  # type: typing.Dict[type, typing.Type[ma_fields.Field]]\n    #: Overrides for default schema-level error messages\n    error_messages = {}  # type: typing.Dict[str, str]\n\n    _default_error_messages = {\n        \"type\": \"Invalid input type.\",\n        \"unknown\": \"Unknown field.\",\n    }  # type: typing.Dict[str, str]\n\n    OPTIONS_CLASS = SchemaOpts  # type: type\n\n    set_class = OrderedSet\n\n    # These get set by SchemaMeta\n    opts = None  # type: SchemaOpts\n    _declared_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n    _hooks = {}  # type: typing.Dict[types.Tag, typing.List[str]]\n\n    class Meta:\n        \"\"\"Options object for a Schema.\n\n        Example usage: ::\n\n            class Meta:\n                fields = (\"id\", \"email\", \"date_created\")\n                exclude = (\"password\", \"secret_attribute\")\n\n        Available options:\n\n        - ``fields``: Tuple or list of fields to include in the serialized result.\n        - ``additional``: Tuple or list of fields to include *in addition* to the\n            explicitly declared fields. ``additional`` and ``fields`` are\n            mutually-exclusive options.\n        - ``include``: Dictionary of additional fields to include in the schema. It is\n            usually better to define fields as class variables, but you may need to\n            use this option, e.g., if your fields are Python keywords. May be an\n            `OrderedDict`.\n        - ``exclude``: Tuple or list of fields to exclude in the serialized result.\n            Nested fields can be represented with dot delimiters.\n        - ``dateformat``: Default format for `Date <fields.Date>` fields.\n        - ``datetimeformat``: Default format for `DateTime <fields.DateTime>` fields.\n        - ``timeformat``: Default format for `Time <fields.Time>` fields.\n        - ``render_module``: Module to use for `loads <Schema.loads>` and `dumps <Schema.dumps>`.\n            Defaults to `json` from the standard library.\n        - ``ordered``: If `True`, output of `Schema.dump` will be a `collections.OrderedDict`.\n        - ``index_errors``: If `True`, errors dictionaries will include the index\n            of invalid items in a collection.\n        - ``load_only``: Tuple or list of fields to exclude from serialized results.\n        - ``dump_only``: Tuple or list of fields to exclude from deserialization\n        - ``unknown``: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        - ``register``: Whether to register the `Schema` with marshmallow's internal\n            class registry. Must be `True` if you intend to refer to this `Schema`\n            by class name in `Nested` fields. Only set this to `False` when memory\n            usage is critical. Defaults to `True`.\n        \"\"\"\n\n    def __init__(\n        self,\n        *,\n        only: types.StrSequenceOrSet | None = None,\n        exclude: types.StrSequenceOrSet = (),\n        many: bool = False,\n        context: dict | None = None,\n        load_only: types.StrSequenceOrSet = (),\n        dump_only: types.StrSequenceOrSet = (),\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        # Raise error if only or exclude is passed as string, not list of strings\n        if only is not None and not is_collection(only):\n            raise StringNotCollectionError('\"only\" should be a list of strings')\n        if not is_collection(exclude):\n            raise StringNotCollectionError('\"exclude\" should be a list of strings')\n        # copy declared fields from metaclass\n        self.declared_fields = copy.deepcopy(self._declared_fields)\n        self.many = many\n        self.only = only\n        self.exclude: set[typing.Any] | typing.MutableSet[typing.Any] = set(\n            self.opts.exclude\n        ) | set(exclude)\n        self.ordered = self.opts.ordered\n        self.load_only = set(load_only) or set(self.opts.load_only)\n        self.dump_only = set(dump_only) or set(self.opts.dump_only)\n        self.partial = partial\n        self.unknown = (\n            self.opts.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        self.context = context or {}\n        self._normalize_nested_options()\n        #: Dictionary mapping field_names -> :class:`Field` objects\n        self.fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.load_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self.dump_fields = {}  # type: typing.Dict[str, ma_fields.Field]\n        self._init_fields()\n        messages = {}\n        messages.update(self._default_error_messages)\n        for cls in reversed(self.__class__.__mro__):\n            messages.update(getattr(cls, \"error_messages\", {}))\n        messages.update(self.error_messages or {})\n        self.error_messages = messages\n\n    def __repr__(self) -> str:\n        return f\"<{self.__class__.__name__}(many={self.many})>\"\n\n    @property\n    def dict_class(self) -> type:\n        return OrderedDict if self.ordered else dict\n\n    @classmethod\n    def from_dict(\n        cls,\n        fields: dict[str, ma_fields.Field | type],\n        *,\n        name: str = \"GeneratedSchema\",\n    ) -> type:\n        \"\"\"Generate a `Schema` class given a dictionary of fields.\n\n        .. code-block:: python\n\n            from marshmallow import Schema, fields\n\n            PersonSchema = Schema.from_dict({\"name\": fields.Str()})\n            print(PersonSchema().load({\"name\": \"David\"}))  # => {'name': 'David'}\n\n        Generated schemas are not added to the class registry and therefore cannot\n        be referred to by name in `Nested` fields.\n\n        :param dict fields: Dictionary mapping field names to field instances.\n        :param str name: Optional name for the class, which will appear in\n            the ``repr`` for the class.\n\n        .. versionadded:: 3.0.0\n        \"\"\"\n        attrs = fields.copy()\n        attrs[\"Meta\"] = type(\n            \"GeneratedMeta\", (getattr(cls, \"Meta\", object),), {\"register\": False}\n        )\n        schema_cls = type(name, (cls,), attrs)\n        return schema_cls\n\n    ##### Override-able methods #####\n\n    def handle_error(\n        self, error: ValidationError, data: typing.Any, *, many: bool, **kwargs\n    ):\n        \"\"\"Custom error handler function for the schema.\n\n        :param error: The `ValidationError` raised during (de)serialization.\n        :param data: The original input data.\n        :param many: Value of ``many`` on dump or load.\n        :param partial: Value of ``partial`` on load.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0rc9\n            Receives `many` and `partial` (on deserialization) as keyword arguments.\n        \"\"\"\n        pass\n\n    def get_attribute(self, obj: typing.Any, attr: str, default: typing.Any):\n        \"\"\"Defines how to pull values from an object to serialize.\n\n        .. versionadded:: 2.0.0\n\n        .. versionchanged:: 3.0.0a1\n            Changed position of ``obj`` and ``attr``.\n        \"\"\"\n        return get_value(obj, attr, default)\n\n    ##### Serialization/Deserialization API #####\n\n    @staticmethod\n    def _call_and_store(getter_func, data, *, field_name, error_store, index=None):\n        \"\"\"Call ``getter_func`` with ``data`` as its argument, and store any `ValidationErrors`.\n\n        :param callable getter_func: Function for getting the serialized/deserialized\n            value from ``data``.\n        :param data: The data passed to ``getter_func``.\n        :param str field_name: Field name.\n        :param int index: Index of the item being validated, if validating a collection,\n            otherwise `None`.\n        \"\"\"\n        try:\n            value = getter_func(data)\n        except ValidationError as error:\n            error_store.store_error(error.messages, field_name, index=index)\n            # When a Nested field fails validation, the marshalled data is stored\n            # on the ValidationError's valid_data attribute\n            return error.valid_data or missing\n        return value\n\n    def _serialize(self, obj: _T | typing.Iterable[_T], *, many: bool = False):\n        \"\"\"Serialize ``obj``.\n\n        :param obj: The object(s) to serialize.\n        :param bool many: `True` if ``data`` should be serialized as a collection.\n        :return: A dictionary of the serialized data\n\n        .. versionchanged:: 1.0.0\n            Renamed from ``marshal``.\n        \"\"\"\n        if many and obj is not None:\n            return [\n                self._serialize(d, many=False)\n                for d in typing.cast(typing.Iterable[_T], obj)\n            ]\n        ret = self.dict_class()\n        for attr_name, field_obj in self.dump_fields.items():\n            value = field_obj.serialize(attr_name, obj, accessor=self.get_attribute)\n            if value is missing:\n                continue\n            key = field_obj.data_key if field_obj.data_key is not None else attr_name\n            ret[key] = value\n        return ret\n\n    def dump(self, obj: typing.Any, *, many: bool | None = None):\n        \"\"\"Serialize an object to native Python data types according to this\n        Schema's fields.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: Serialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        .. versionchanged:: 3.0.0rc9\n            Validation no longer occurs upon serialization.\n        \"\"\"\n        many = self.many if many is None else bool(many)\n        if self._has_processors(PRE_DUMP):\n            processed_obj = self._invoke_dump_processors(\n                PRE_DUMP, obj, many=many, original_data=obj\n            )\n        else:\n            processed_obj = obj\n\n        result = self._serialize(processed_obj, many=many)\n\n        if self._has_processors(POST_DUMP):\n            result = self._invoke_dump_processors(\n                POST_DUMP, result, many=many, original_data=obj\n            )\n\n        return result\n\n    def dumps(self, obj: typing.Any, *args, many: bool | None = None, **kwargs):\n        \"\"\"Same as :meth:`dump`, except return a JSON-encoded string.\n\n        :param obj: The object to serialize.\n        :param many: Whether to serialize `obj` as a collection. If `None`, the value\n            for `self.many` is used.\n        :return: A ``json`` string\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the serialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if ``obj`` is invalid.\n        \"\"\"\n        serialized = self.dump(obj, many=many)\n        return self.opts.render_module.dumps(serialized, *args, **kwargs)\n\n    def _deserialize(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        error_store: ErrorStore,\n        many: bool = False,\n        partial=None,\n        unknown=RAISE,\n        index=None,\n    ) -> _T | list[_T]:\n        \"\"\"Deserialize ``data``.\n\n        :param dict data: The data to deserialize.\n        :param ErrorStore error_store: Structure to store errors.\n        :param bool many: `True` if ``data`` should be deserialized as a collection.\n        :param bool|tuple partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n        :param int index: Index of the item being serialized (for storing errors) if\n            serializing a collection, otherwise `None`.\n        :return: A dictionary of the deserialized data.\n        \"\"\"\n        index_errors = self.opts.index_errors\n        index = index if index_errors else None\n        if many:\n            if not is_collection(data):\n                error_store.store_error([self.error_messages[\"type\"]], index=index)\n                ret_l = []  # type: typing.List[_T]\n            else:\n                ret_l = [\n                    typing.cast(\n                        _T,\n                        self._deserialize(\n                            typing.cast(typing.Mapping[str, typing.Any], d),\n                            error_store=error_store,\n                            many=False,\n                            partial=partial,\n                            unknown=unknown,\n                            index=idx,\n                        ),\n                    )\n                    for idx, d in enumerate(data)\n                ]\n            return ret_l\n        ret_d = self.dict_class()\n        # Check data is a dict\n        if not isinstance(data, Mapping):\n            error_store.store_error([self.error_messages[\"type\"]], index=index)\n        else:\n            partial_is_collection = is_collection(partial)\n            for attr_name, field_obj in self.load_fields.items():\n                field_name = (\n                    field_obj.data_key if field_obj.data_key is not None else attr_name\n                )\n                raw_value = data.get(field_name, missing)\n                if raw_value is missing:\n                    # Ignore missing field if we're allowed to.\n                    if partial is True or (\n                        partial_is_collection and attr_name in partial\n                    ):\n                        continue\n                d_kwargs = {}\n                # Allow partial loading of nested schemas.\n                if partial_is_collection:\n                    prefix = field_name + \".\"\n                    len_prefix = len(prefix)\n                    sub_partial = [\n                        f[len_prefix:] for f in partial if f.startswith(prefix)\n                    ]\n                    d_kwargs[\"partial\"] = sub_partial\n                elif partial is not None:\n                    d_kwargs[\"partial\"] = partial\n\n                def getter(\n                    val, field_obj=field_obj, field_name=field_name, d_kwargs=d_kwargs\n                ):\n                    return field_obj.deserialize(\n                        val,\n                        field_name,\n                        data,\n                        **d_kwargs,\n                    )\n\n                value = self._call_and_store(\n                    getter_func=getter,\n                    data=raw_value,\n                    field_name=field_name,\n                    error_store=error_store,\n                    index=index,\n                )\n                if value is not missing:\n                    key = field_obj.attribute or attr_name\n                    set_value(ret_d, key, value)\n            if unknown != EXCLUDE:\n                fields = {\n                    field_obj.data_key if field_obj.data_key is not None else field_name\n                    for field_name, field_obj in self.load_fields.items()\n                }\n                for key in set(data) - fields:\n                    value = data[key]\n                    if unknown == INCLUDE:\n                        ret_d[key] = value\n                    elif unknown == RAISE:\n                        error_store.store_error(\n                            [self.error_messages[\"unknown\"]],\n                            key,\n                            (index if index_errors else None),\n                        )\n        return ret_d\n\n    def load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n    ):\n        \"\"\"Deserialize a data structure to an object defined by this Schema's fields.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        return self._do_load(\n            data, many=many, partial=partial, unknown=unknown, postprocess=True\n        )\n\n    def loads(\n        self,\n        json_data: str,\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        **kwargs,\n    ):\n        \"\"\"Same as :meth:`load`, except it takes a JSON string as input.\n\n        :param json_data: A JSON string of the data to deserialize.\n        :param many: Whether to deserialize `obj` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :return: Deserialized data\n\n        .. versionadded:: 1.0.0\n        .. versionchanged:: 3.0.0b7\n            This method returns the deserialized data rather than a ``(data, errors)`` duple.\n            A :exc:`ValidationError <marshmallow.exceptions.ValidationError>` is raised\n            if invalid data are passed.\n        \"\"\"\n        data = self.opts.render_module.loads(json_data, **kwargs)\n        return self.load(data, many=many, partial=partial, unknown=unknown)\n\n    def _run_validator(\n        self,\n        validator_func,\n        output,\n        *,\n        original_data,\n        error_store,\n        many,\n        partial,\n        pass_original,\n        index=None,\n    ):\n        try:\n            if pass_original:  # Pass original, raw data (before unmarshalling)\n                validator_func(output, original_data, partial=partial, many=many)\n            else:\n                validator_func(output, partial=partial, many=many)\n        except ValidationError as err:\n            error_store.store_error(err.messages, err.field_name, index=index)\n\n    def validate(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n    ) -> dict[str, list[str]]:\n        \"\"\"Validate `data` against the schema, returning a dictionary of\n        validation errors.\n\n        :param data: The data to validate.\n        :param many: Whether to validate `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to ignore missing fields and not require\n            any fields declared. Propagates down to ``Nested`` fields as well. If\n            its value is an iterable, only missing fields listed in that iterable\n            will be ignored. Use dot delimiters to specify nested fields.\n        :return: A dictionary of validation errors.\n\n        .. versionadded:: 1.1.0\n        \"\"\"\n        try:\n            self._do_load(data, many=many, partial=partial, postprocess=False)\n        except ValidationError as exc:\n            return typing.cast(typing.Dict[str, typing.List[str]], exc.messages)\n        return {}\n\n    ##### Private Helpers #####\n\n    def _do_load(\n        self,\n        data: (\n            typing.Mapping[str, typing.Any]\n            | typing.Iterable[typing.Mapping[str, typing.Any]]\n        ),\n        *,\n        many: bool | None = None,\n        partial: bool | types.StrSequenceOrSet | None = None,\n        unknown: str | None = None,\n        postprocess: bool = True,\n    ):\n        \"\"\"Deserialize `data`, returning the deserialized result.\n        This method is private API.\n\n        :param data: The data to deserialize.\n        :param many: Whether to deserialize `data` as a collection. If `None`, the\n            value for `self.many` is used.\n        :param partial: Whether to validate required fields. If its\n            value is an iterable, only fields listed in that iterable will be\n            ignored will be allowed missing. If `True`, all fields will be allowed missing.\n            If `None`, the value for `self.partial` is used.\n        :param unknown: Whether to exclude, include, or raise an error for unknown\n            fields in the data. Use `EXCLUDE`, `INCLUDE` or `RAISE`.\n            If `None`, the value for `self.unknown` is used.\n        :param postprocess: Whether to run post_load methods..\n        :return: Deserialized data\n        \"\"\"\n        error_store = ErrorStore()\n        errors = {}  # type: dict[str, list[str]]\n        many = self.many if many is None else bool(many)\n        unknown = (\n            self.unknown\n            if unknown is None\n            else validate_unknown_parameter_value(unknown)\n        )\n        if partial is None:\n            partial = self.partial\n        # Run preprocessors\n        if self._has_processors(PRE_LOAD):\n            try:\n                processed_data = self._invoke_load_processors(\n                    PRE_LOAD, data, many=many, original_data=data, partial=partial\n                )\n            except ValidationError as err:\n                errors = err.normalized_messages()\n                result = None  # type: list | dict | None\n        else:\n            processed_data = data\n        if not errors:\n            # Deserialize data\n            result = self._deserialize(\n                processed_data,\n                error_store=error_store,\n                many=many,\n                partial=partial,\n                unknown=unknown,\n            )\n            # Run field-level validation\n            self._invoke_field_validators(\n                error_store=error_store, data=result, many=many\n            )\n            # Run schema-level validation\n            if self._has_processors(VALIDATES_SCHEMA):\n                field_errors = bool(error_store.errors)\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=True,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n                self._invoke_schema_validators(\n                    error_store=error_store,\n                    pass_many=False,\n                    data=result,\n                    original_data=data,\n                    many=many,\n                    partial=partial,\n                    field_errors=field_errors,\n                )\n            errors = error_store.errors\n            # Run post processors\n            if not errors and postprocess and self._has_processors(POST_LOAD):\n                try:\n                    result = self._invoke_load_processors(\n                        POST_LOAD,\n                        result,\n                        many=many,\n                        original_data=data,\n                        partial=partial,\n                    )\n                except ValidationError as err:\n                    errors = err.normalized_messages()\n        if errors:\n            exc = ValidationError(errors, data=data, valid_data=result)\n            self.handle_error(exc, data, many=many, partial=partial)\n            raise exc\n\n        return result\n\n    \ndef _normalize_nested_options(self) -> None:\n        \"\"\"Apply then flatten nested schema options.\n        This method is private API.\n        \"\"\"\n        if self.only is not None:\n            # Apply the only option to nested fields.\n            self.__apply_nested_option(\"only\", self.only, \"intersection\")\n            # Remove the child field names from the only option.\n            self.only = self.set_class([field.split(\".\", 1)[0] for field in self.only])\n        if self.exclude:\n            # Apply the exclude option to nested fields.\n            self.__apply_nested_option(\"exclude\", self.exclude, \"union\")\n            # Remove the parent field names from the exclude option.\n            self.exclude = self.set_class(\n                [field for field in self.exclude if \".\" not in field]\n            )\n\n    def __apply_nested_option(self, option_name, field_names, set_operation) -> None:\n        \"\"\"Apply nested options to nested fields\"\"\"\n        # Split nested field names on the first dot.\n        nested_fields = [name.split(\".\", 1) for name in field_names if \".\" in name]\n        # Partition the nested field names by parent field.\n        nested_options = defaultdict(list)  # type: defaultdict\n        for parent, nested_names in nested_fields:\n            nested_options[parent].append(nested_names)\n        # Apply the nested field options.\n        for key, options in iter(nested_options.items()):\n            new_options = self.set_class(options)\n            original_options = getattr(self.declared_fields[key], option_name, ())\n            if original_options:\n                if set_operation == \"union\":\n                    new_options |= self.set_class(original_options)\n                if set_operation == \"intersection\":\n                    new_options &= self.set_class(original_options)\n            setattr(self.declared_fields[key], option_name, new_options)\n\n    def _init_fields(self) -> None:\n        \"\"\"Update self.fields, self.load_fields, and self.dump_fields based on schema options.\n        This method is private API.\n        \"\"\"\n        if self.opts.fields:\n            available_field_names = self.set_class(self.opts.fields)\n        else:\n            available_field_names = self.set_class(self.declared_fields.keys())\n            if self.opts.additional:\n                available_field_names |= self.set_class(self.opts.additional)\n\n        invalid_fields = self.set_class()\n\n        if self.only is not None:\n            # Return only fields specified in only option\n            field_names: typing.AbstractSet[typing.Any] = self.set_class(self.only)\n\n            invalid_fields |= field_names - available_field_names\n        else:\n            field_names = available_field_names\n\n        # If \"exclude\" option or param is specified, remove those fields.\n        if self.exclude:\n            # Note that this isn't available_field_names, since we want to\n            # apply \"only\" for the actual calculation.\n            field_names = field_names - self.exclude\n            invalid_fields |= self.exclude - available_field_names\n\n        if invalid_fields:\n            message = f\"Invalid fields for {self}: {invalid_fields}.\"\n            raise ValueError(message)\n\n        fields_dict = self.dict_class()\n        for field_name in field_names:\n            field_obj = self.declared_fields.get(field_name, ma_fields.Inferred())\n...\n", "needle_token_start": 15237, "needle_token_end": 15446, "code_context_ntokens": 16143, "output": ["```python\ndef _normalize_nested_options(self) -> None:\n    \"\"\"Apply then flatten nested schema options.\n    This method is private API.\n    \"\"\"\n    if self.only is not None:\n        # Apply the only option to nested fields.\n        self.__apply_nested_option(\"only\", self.only, \"intersection\")\n        # Remove the child field names from the only option.\n        self.only = self.set_class([field.split(\".\", 1)[0] for field in self.only])\n    if self.exclude:\n        # Apply the exclude option to nested fields.\n        self.__apply_nested_option(\"exclude\", self.exclude, \"union\")\n        # Remove the parent field names from the exclude option.\n        self.exclude = self.set_class(\n            [field for field in self.exclude if \".\" not in field]\n        )\n\ndef __apply_nested_option(self, option_name, field_names, set_operation) -> None:\n    \"\"\"Apply nested options to nested fields\"\"\"\n    # Split nested field names on the first dot.\n    nested_fields = [name.split(\".\", 1) for name in field_names if \".\" in name]\n    # Partition the nested field names by parent field.\n    nested_options = defaultdict(list)  # type: defaultdict\n    for parent, nested_names in nested_fields:\n        nested_options[parent].append(nested_names)\n    # Apply the nested field options.\n    for key, options in iter(nested_options.items()):\n        new_options = self.set_class(options)\n        original_options = getattr(self.declared_fields[key], option_name, ())\n        if original_options:\n            if set_operation == \"union\":\n                new_options |= self.set_class(original_options)\n            if set_operation == \"intersection\":\n                new_options &= self.set_class(original_options)\n        setattr(self.declared_fields[key], option_name, new_options)\n```"]}
{"repo": "ethereum/web3.py", "name": "test_eth_get_logs_with_logs_none_topic_args", "language": "python", "path": "web3/_utils/module_testing/eth_module.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To verify that querying blockchain logs with a filter containing only `None` values for topics results in no logs being returned.\n2. **Input**: A filter parameter set with the starting block number and an array of topics filled entirely with `None`.\n3. **Output**: An empty list of logs, confirming that no entries match the given filter criteria.\n4. **Procedure**: The function sets up a filter to search from the genesis block and specifies three `None` values in the topics array. It then queries the blockchain logs using this filter and checks that the result is an empty list, indicating no logs were found that match the filter.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    ) -> None:\n        # Test with block range\n\n        # the range includes the block where the log resides in\n        filter_params: FilterParams = {\n            \"fromBlock\": block_with_txn_with_log[\"number\"],\n            \"toBlock\": block_with_txn_with_log[\"number\"],\n        }\n        result = w3.eth.get_logs(filter_params)\n        assert_contains_log(\n            result, block_with_txn_with_log, emitter_contract_address, txn_hash_with_log\n        )\n\n        # specify only `from_block`. by default `to_block` should be 'latest'\n        filter_params = {\n            \"fromBlock\": BlockNumber(0),\n        }\n        result = w3.eth.get_logs(filter_params)\n        assert_contains_log(\n            result, block_with_txn_with_log, emitter_contract_address, txn_hash_with_log\n        )\n\n        # Test with `address`\n\n        # filter with emitter_contract.address\n        filter_params = {\n            \"fromBlock\": BlockNumber(0),\n            \"address\": emitter_contract_address,\n        }\n\n    def test_eth_get_logs_with_logs_topic_args(\n        self,\n        w3: \"Web3\",\n        block_with_txn_with_log: BlockData,\n        emitter_contract_address: ChecksumAddress,\n        txn_hash_with_log: HexStr,\n    ) -> None:\n        # Test with None event sig\n\n        filter_params: FilterParams = {\n            \"fromBlock\": BlockNumber(0),\n            \"topics\": [\n                None,\n                HexStr(\n                    \"0x000000000000000000000000000000000000000000000000000000000000d431\"\n                ),\n            ],\n        }\n\n        result = w3.eth.get_logs(filter_params)\n        assert_contains_log(\n            result, block_with_txn_with_log, emitter_contract_address, txn_hash_with_log\n        )\n\n        # Test with None indexed arg\n        filter_params = {\n            \"fromBlock\": BlockNumber(0),\n            \"topics\": [\n                HexStr(\n                    \"0x057bc32826fbe161da1c110afcdcae7c109a8b69149f727fc37a603c60ef94ca\"\n                ),\n                None,\n            ],\n        }\n        result = w3.eth.get_logs(filter_params)\n        assert_contains_log(\n            result, block_with_txn_with_log, emitter_contract_address, txn_hash_with_log\n        )\n\n    def \ntest_eth_get_logs_with_logs_none_topic_args(self, w3: \"Web3\") -> None:\n        # Test with None overflowing\n        filter_params: FilterParams = {\n            \"fromBlock\": BlockNumber(0),\n            \"topics\": [None, None, None],\n        }\n\n        result = w3.eth.get_logs(filter_params)\n        assert len(result) == 0\n\n    def test_eth_call_old_contract_state(\n        self, w3: \"Web3\", math_contract: \"Contract\", unlocked_account: ChecksumAddress\n    ) -> None:\n        start_block = w3.eth.get_block(\"latest\")\n        block_num = start_block[\"number\"]\n        block_hash = start_block[\"hash\"]\n\n        math_contract.functions.incrementCounter().transact({\"from\": unlocked_account})\n\n        # This isn't an incredibly convincing test since we can't mine, and\n        # the default resolved block is latest, So if block_identifier was ignored\n        # we would get the same result. For now, we mostly depend on core tests.\n        # Ideas to improve this test:\n        #  - Enable on-demand mining in more clients\n        #  - Increment the math contract in all of the fixtures, and check the\n        #    value in an old block\n        block_hash_call_result = math_contract.functions.counter().call(\n            block_identifier=block_hash\n        )\n        block_num_call_result = math_contract.functions.counter().call(\n            block_identifier=block_num\n        )\n        latest_call_result = math_contract.functions.counter().call(\n            block_identifier=\"latest\"\n        )\n        default_call_result = math_contract.functions.counter().call()\n        pending_call_result = math_contract.functions.counter().call(\n            block_identifier=\"pending\"\n        )\n\n        assert block_hash_call_result == 0\n        assert block_num_call_result == 0\n        assert latest_call_result == 0\n        assert default_call_result == 0\n\n        if pending_call_result != 1:\n            raise AssertionError(\n                f\"pending call result was {pending_call_result} instead of 1\"\n            )\n\n    def test_eth_uninstall_filter(self, w3: \"Web3\") -> None:\n        filter = w3.eth.filter({})\n        assert is_string(filter.filter_id)\n\n        success = w3.eth.uninstall_filter(filter.filter_id)\n        assert success is True\n\n        failure = w3.eth.uninstall_filter(filter.filter_id)\n        assert failure is False\n\n    def test_eth_get_raw_transaction(self, w3: \"Web3\", mined_txn_hash: HexStr) -> None:\n        raw_transaction = w3.eth.get_raw_transaction(mined_txn_hash)\n        assert is_bytes(raw_transaction)\n\n    def test_eth_get_raw_transaction_raises_error(self, w3: \"Web3\") -> None:\n        with pytest.raises(\n            TransactionNotFound, match=f\"Transaction with hash: '{UNKNOWN_HASH}'\"\n        ):\n            w3.eth.get_raw_transaction(UNKNOWN_HASH)\n\n    def test_eth_get_raw_transaction_by_block(\n        self,\n        w3: \"Web3\",\n        unlocked_account_dual_type: ChecksumAddress,\n        block_with_txn: BlockData,\n    ) -> None:\n        # eth_getRawTransactionByBlockNumberAndIndex: block identifier\n        # send a txn to make sure pending block has at least one txn\n        w3.eth.send_transaction(\n            {\n                \"from\": unlocked_account_dual_type,\n                \"to\": unlocked_account_dual_type,\n                \"value\": Wei(1),\n            }\n        )\n        last_pending_txn_index = len(w3.eth.get_block(\"pending\")[\"transactions\"]) - 1\n        raw_transaction = w3.eth.get_raw_transaction_by_block(\n            \"pending\", last_pending_txn_index\n        )\n        assert is_bytes(raw_transaction)\n\n        # eth_getRawTransactionByBlockNumberAndIndex: block number\n        block_with_txn_number = block_with_txn[\"number\"]\n        assert is_integer(block_with_txn_number)\n        raw_transaction = w3.eth.get_raw_transaction_by_block(block_with_txn_number, 0)\n        assert is_bytes(raw_transaction)\n\n        # eth_getRawTransactionByBlockHashAndIndex: block hash\n        block_with_txn_hash = block_with_txn[\"hash\"]\n        assert is_bytes(block_with_txn_hash)\n        raw_transaction = w3.eth.get_raw_transaction_by_block(block_with_txn_hash, 0)\n        assert is_bytes(raw_transaction)\n\n    @pytest.mark.parametrize(\"unknown_block_num_or_hash\", (1234567899999, UNKNOWN_HASH))\n    def test_eth_get_raw_transaction_by_block_raises_error(\n        self, w3: \"Web3\", unknown_block_num_or_hash: Union[int, HexBytes]\n    ) -> None:\n        with pytest.raises(\n            TransactionNotFound,\n            match=(\n                f\"Transaction index: 0 on block id: \"\n                f\"{to_hex_if_integer(unknown_block_num_or_hash)!r} not found.\"\n            ),\n        ):\n            w3.eth.get_raw_transaction_by_block(unknown_block_num_or_hash, 0)\n\n    def test_eth_get_raw_transaction_by_block_raises_error_block_identifier(\n        self, w3: \"Web3\"\n    ) -> None:\n        unknown_identifier = \"unknown\"\n        with pytest.raises(\n            ValueError,\n            match=(\n                \"Value did not match any of the recognized block identifiers: \"\n                f\"{unknown_identifier}\"\n            ),\n        ):\n            w3.eth.get_raw_transaction_by_block(unknown_identifier, 0)  # type: ignore\n\n    def test_default_account(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        # check defaults to empty\n        default_account = w3.eth.default_account\n        assert default_account is empty\n\n        # check setter\n        w3.eth.default_account = unlocked_account_dual_type\n        default_account = w3.eth.default_account\n        assert default_account == unlocked_account_dual_type\n\n        # reset to default\n        w3.eth.default_account = empty\n\n    def test_default_block(\n        self,\n        w3: \"Web3\",\n    ) -> None:\n        # check defaults to 'latest'\n        default_block = w3.eth.default_block\n        assert default_block == \"latest\"\n\n        # check setter\n        w3.eth.default_block = BlockNumber(12345)\n        default_block = w3.eth.default_block\n        assert default_block == BlockNumber(12345)\n\n        # reset to default\n        w3.eth.default_block = \"latest\"\n\n# Path: web3/_utils/module_testing/go_ethereum_admin_module.py\nimport pytest\nfrom typing import (\n    TYPE_CHECKING,\n    List,\n)\n\nfrom web3.datastructures import (\n    AttributeDict,\n)\nfrom web3.types import (\n    EnodeURI,\n)\n\nif TYPE_CHECKING:\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\n\nclass GoEthereumAdminModuleTest:\n    def test_add_peer(self, w3: \"Web3\") -> None:\n        result = w3.geth.admin.add_peer(\n            EnodeURI(\n                \"enode://f1a6b0bdbf014355587c3018454d070ac57801f05d3b39fe85da574f002a32e929f683d72aa5a8318382e4d3c7a05c9b91687b0d997a39619fb8a6e7ad88e512@1.1.1.1:30303\"  # noqa: E501\n            ),\n        )\n        assert result is True\n\n    def test_admin_datadir(self, w3: \"Web3\", datadir: str) -> None:\n        result = w3.geth.admin.datadir()\n        assert result == datadir\n\n    def test_admin_node_info(self, w3: \"Web3\") -> None:\n        result = w3.geth.admin.node_info()\n        expected = AttributeDict(\n            {\n                \"id\": \"\",\n                \"name\": \"\",\n                \"enode\": \"\",\n                \"ip\": \"\",\n                \"ports\": AttributeDict({}),\n                \"listenAddr\": \"\",\n                \"protocols\": AttributeDict({}),\n            }\n        )\n        # Test that result gives at least the keys that are listed in `expected`\n        assert not set(expected.keys()).difference(result.keys())\n\n    def test_admin_peers(self, w3: \"Web3\") -> None:\n        enode = w3.geth.admin.node_info()[\"enode\"]\n        w3.geth.admin.add_peer(enode)\n        result = w3.geth.admin.peers()\n        assert len(result) == 1\n\n    def test_admin_start_stop_http(self, w3: \"Web3\") -> None:\n        stop = w3.geth.admin.stop_http()\n        assert stop is True\n\n        start = w3.geth.admin.start_http()\n        assert start is True\n\n    def test_admin_start_stop_ws(self, w3: \"Web3\") -> None:\n        stop = w3.geth.admin.stop_ws()\n        assert stop is True\n\n        start = w3.geth.admin.start_ws()\n        assert start is True\n\n\nclass GoEthereumAsyncAdminModuleTest:\n    @pytest.mark.asyncio\n    async def test_async_datadir(self, async_w3: \"AsyncWeb3\") -> None:\n        datadir = await async_w3.geth.admin.datadir()\n        assert isinstance(datadir, str)\n\n    @pytest.mark.asyncio\n    async def test_async_node_info(self, async_w3: \"AsyncWeb3\") -> None:\n        node_info = await async_w3.geth.admin.node_info()\n        assert \"Geth\" in node_info[\"name\"]\n\n    @pytest.mark.asyncio\n    async def test_async_nodes(self, async_w3: \"AsyncWeb3\") -> None:\n        nodes = await async_w3.geth.admin.peers()\n        assert isinstance(nodes, List)\n\n    @pytest.mark.asyncio\n    async def test_admin_peers(self, async_w3: \"AsyncWeb3\") -> None:\n        node_info = await async_w3.geth.admin.node_info()\n        await async_w3.geth.admin.add_peer(node_info[\"enode\"])\n        result = await async_w3.geth.admin.peers()\n        assert len(result) == 1\n\n    @pytest.mark.asyncio\n    async def test_admin_start_stop_http(self, async_w3: \"AsyncWeb3\") -> None:\n        stop = await async_w3.geth.admin.stop_http()\n        assert stop is True\n\n        start = await async_w3.geth.admin.start_http()\n        assert start is True\n\n    @pytest.mark.asyncio\n    async def test_admin_start_stop_ws(self, async_w3: \"AsyncWeb3\") -> None:\n        stop = await async_w3.geth.admin.stop_ws()\n        assert stop is True\n\n        start = await async_w3.geth.admin.start_ws()\n        assert start is True\n\n# Path: web3/_utils/module_testing/go_ethereum_personal_module.py\nimport json\nimport pytest\nfrom typing import (\n    TYPE_CHECKING,\n    cast,\n)\n\nfrom eth_typing import (\n    ChecksumAddress,\n)\nfrom eth_utils import (\n    is_checksum_address,\n    is_list_like,\n    is_same_address,\n    is_string,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3 import (\n    constants,\n)\nfrom web3.datastructures import (\n    AttributeDict,\n)\nfrom web3.types import (\n    TxParams,\n    Wei,\n)\n\nif TYPE_CHECKING:\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\nPRIVATE_KEY_HEX = \"0x56ebb41875ceedd42e395f730e03b5c44989393c9f0484ee6bc05f933673458f\"\nSECOND_PRIVATE_KEY_HEX = (\n    \"0x56ebb41875ceedd42e395f730e03b5c44989393c9f0484ee6bc05f9336712345\"\n)\nTHIRD_PRIVATE_KEY_HEX = (\n    \"0x56ebb41875ceedd42e395f730e03b5c44989393c9f0484ee6bc05f9336754321\"\n)\nPASSWORD = \"web3-testing\"\nADDRESS = \"0x844B417c0C58B02c2224306047B9fb0D3264fE8c\"\nSECOND_ADDRESS = \"0xB96b6B21053e67BA59907E252D990C71742c41B8\"\n\n\nPRIVATE_KEY_FOR_UNLOCK = (\n    \"0x392f63a79b1ff8774845f3fa69de4a13800a59e7083f5187f1558f0797ad0f01\"\n)\nACCOUNT_FOR_UNLOCK = \"0x12efDc31B1a8FA1A1e756DFD8A1601055C971E13\"\n\n\nclass GoEthereumPersonalModuleTest:\n    def test_personal_import_raw_key(self, w3: \"Web3\") -> None:\n        actual = w3.geth.personal.import_raw_key(PRIVATE_KEY_HEX, PASSWORD)\n        assert actual == ADDRESS\n\n    def test_personal_list_accounts(self, w3: \"Web3\") -> None:\n        accounts = w3.geth.personal.list_accounts()\n        assert is_list_like(accounts)\n        assert len(accounts) > 0\n        assert all((is_checksum_address(item) for item in accounts))\n\n    def test_personal_list_wallets(self, w3: \"Web3\") -> None:\n        wallets = w3.geth.personal.list_wallets()\n        assert is_list_like(wallets)\n        assert len(wallets) > 0\n        assert is_checksum_address(wallets[0][\"accounts\"][0][\"address\"])\n        assert is_string(wallets[0][\"accounts\"][0][\"url\"])\n        assert is_string(wallets[0][\"status\"])\n        assert is_string(wallets[0][\"url\"])\n\n    def test_personal_lock_account(\n        self, w3: \"Web3\", unlockable_account_dual_type: ChecksumAddress\n    ) -> None:\n        # TODO: how do we test this better?\n        w3.geth.personal.lock_account(unlockable_account_dual_type)\n\n    def test_personal_unlock_account_success(\n        self,\n        w3: \"Web3\",\n        unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        result = w3.geth.personal.unlock_account(\n            unlockable_account_dual_type, unlockable_account_pw\n        )\n        assert result is True\n\n    def test_personal_unlock_account_failure(\n        self, w3: \"Web3\", unlockable_account_dual_type: ChecksumAddress\n    ) -> None:\n        with pytest.raises(ValueError):\n            w3.geth.personal.unlock_account(\n                unlockable_account_dual_type, \"bad-password\"\n            )\n\n    def test_personal_new_account(self, w3: \"Web3\") -> None:\n        new_account = w3.geth.personal.new_account(PASSWORD)\n        assert is_checksum_address(new_account)\n\n    def test_personal_send_transaction(\n        self,\n        w3: \"Web3\",\n        unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        assert (\n            w3.eth.get_balance(unlockable_account_dual_type) > constants.WEI_PER_ETHER\n        )\n        txn_params: TxParams = {\n            \"from\": unlockable_account_dual_type,\n            \"to\": unlockable_account_dual_type,\n            \"gas\": 21000,\n            \"value\": Wei(1),\n            \"gasPrice\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.geth.personal.send_transaction(txn_params, unlockable_account_pw)\n        assert txn_hash\n        transaction = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(\n            transaction[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            transaction[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert transaction[\"gas\"] == txn_params[\"gas\"]\n        assert transaction[\"value\"] == txn_params[\"value\"]\n        assert transaction[\"gasPrice\"] == txn_params[\"gasPrice\"]\n\n    def test_personal_sign_and_ecrecover(\n        self,\n        w3: \"Web3\",\n        unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        message = \"test-web3-geth-personal-sign\"\n        signature = w3.geth.personal.sign(\n            message, unlockable_account_dual_type, unlockable_account_pw\n        )\n        signer = w3.geth.personal.ec_recover(message, signature)\n        assert is_same_address(signer, unlockable_account_dual_type)\n\n    @pytest.mark.xfail(\n        reason=\"personal_sign_typed_data JSON RPC call has not been released in geth\"\n    )\n    def test_personal_sign_typed_data(\n        self,\n        w3: \"Web3\",\n        unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        typed_message = \"\"\"\n            {\n                \"types\": {\n                    \"EIP712Domain\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"version\", \"type\": \"string\"},\n                        {\"name\": \"chainId\", \"type\": \"uint256\"},\n                        {\"name\": \"verifyingContract\", \"type\": \"address\"}\n                    ],\n                    \"Person\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"wallet\", \"type\": \"address\"}\n                    ],\n                    \"Mail\": [\n                        {\"name\": \"from\", \"type\": \"Person\"},\n                        {\"name\": \"to\", \"type\": \"Person\"},\n                        {\"name\": \"contents\", \"type\": \"string\"}\n                    ]\n                },\n                \"primaryType\": \"Mail\",\n                \"domain\": {\n                    \"name\": \"Ether Mail\",\n                    \"version\": \"1\",\n                    \"chainId\": \"0x01\",\n                    \"verifyingContract\": \"0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC\"\n                },\n                \"message\": {\n                    \"from\": {\n                        \"name\": \"Cow\",\n                        \"wallet\": \"0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826\"\n                    },\n                    \"to\": {\n                        \"name\": \"Bob\",\n                        \"wallet\": \"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB\"\n                    },\n                    \"contents\": \"Hello, Bob!\"\n                }\n            }\n        \"\"\"\n        signature = HexBytes(\n            w3.geth.personal.sign_typed_data(\n                json.loads(typed_message),\n                unlockable_account_dual_type,\n                unlockable_account_pw,\n            )\n        )\n\n        expected_signature = HexBytes(\n            \"0xc8b56aaeefd10ab4005c2455daf28d9082af661ac347cd\"\n            \"b612d5b5e11f339f2055be831bf57a6e6cb5f6d93448fa35\"\n            \"c1bd56fe1d745ffa101e74697108668c401c\"\n        )\n        assert signature == expected_signature\n        assert len(signature) == 32 + 32 + 1\n\n\nclass GoEthereumAsyncPersonalModuleTest:\n    @pytest.mark.asyncio\n    async def test_async_sign_and_ec_recover(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        message = \"This is a test\"\n        signature = await async_w3.geth.personal.sign(\n            message, async_unlockable_account_dual_type, unlockable_account_pw\n        )\n        address = await async_w3.geth.personal.ec_recover(message, signature)\n        assert is_same_address(async_unlockable_account_dual_type, address)\n\n    @pytest.mark.asyncio\n    async def test_async_import_key(self, async_w3: \"AsyncWeb3\") -> None:\n        address = await async_w3.geth.personal.import_raw_key(\n            THIRD_PRIVATE_KEY_HEX, \"Testing\"\n        )\n        assert address is not None\n\n    @pytest.mark.asyncio\n    async def test_async_list_accounts(self, async_w3: \"AsyncWeb3\") -> None:\n        accounts = await async_w3.geth.personal.list_accounts()\n        assert len(accounts) > 0\n\n    @pytest.mark.asyncio\n    async def test_async_list_wallets(self, async_w3: \"AsyncWeb3\") -> None:\n        wallets = await async_w3.geth.personal.list_wallets()\n        assert isinstance(wallets[0], AttributeDict)\n\n    @pytest.mark.asyncio\n    async def test_async_new_account(self, async_w3: \"AsyncWeb3\") -> None:\n        passphrase = \"Create New Account\"\n        account = await async_w3.geth.personal.new_account(passphrase)\n        assert is_checksum_address(account)\n\n    @pytest.mark.asyncio\n    async def test_async_unlock_lock_account(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        unlocked = await async_w3.geth.personal.unlock_account(\n            async_unlockable_account_dual_type, unlockable_account_pw\n        )\n        assert unlocked is True\n        locked = await async_w3.geth.personal.lock_account(\n            async_unlockable_account_dual_type\n        )\n        assert locked is True\n\n    @pytest.mark.asyncio\n    async def test_async_send_transaction(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        tx_params = TxParams()\n        tx_params[\"to\"] = async_unlockable_account_dual_type\n        tx_params[\"from\"] = async_unlockable_account_dual_type\n        tx_params[\"value\"] = Wei(123)\n        response = await async_w3.geth.personal.send_transaction(\n            tx_params, unlockable_account_pw\n        )\n        assert response is not None\n\n    @pytest.mark.xfail(\n        reason=\"personal_signTypedData JSON RPC call has not been released in geth\"\n    )\n    @pytest.mark.asyncio\n    async def test_async_sign_typed_data(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlockable_account_dual_type: ChecksumAddress,\n        unlockable_account_pw: str,\n    ) -> None:\n        message = {\"message\": \"This is a test\"}\n        signature = await async_w3.geth.personal.sign_typed_data(\n            message, async_unlockable_account_dual_type, unlockable_account_pw\n        )\n        address = await async_w3.geth.personal.ec_recover(\n            json.dumps(message), signature\n        )\n        assert is_same_address(async_unlockable_account_dual_type, address)\n\n# Path: web3/_utils/module_testing/go_ethereum_txpool_module.py\nimport pytest\n\nfrom web3 import (\n    AsyncWeb3,\n    Web3,\n)\n\n\nclass GoEthereumAsyncTxPoolModuleTest:\n    @pytest.mark.asyncio\n    async def test_async_geth_txpool_inspect(self, async_w3: \"AsyncWeb3\") -> None:\n        test_data = await async_w3.geth.txpool.inspect()\n        assert \"pending\" in test_data\n\n    @pytest.mark.asyncio\n    async def test_async_geth_txpool_content(self, async_w3: \"AsyncWeb3\") -> None:\n        test_data = await async_w3.geth.txpool.content()\n        assert \"pending\" in test_data\n\n    @pytest.mark.asyncio\n    async def test_async_geth_txpool_status(self, async_w3: \"AsyncWeb3\") -> None:\n        test_data = await async_w3.geth.txpool.status()\n        assert \"pending\" in test_data\n\n\nclass GoEthereumTxPoolModuleTest:\n    def test_geth_txpool_inspect(self, w3: \"Web3\") -> None:\n        test_data = w3.geth.txpool.inspect()\n        assert \"pending\" in test_data\n\n    def test_geth_txpool_content(self, w3: \"Web3\") -> None:\n        test_data = w3.geth.txpool.content()\n        assert \"pending\" in test_data\n\n    def test_geth_txpool_status(self, w3: \"Web3\") -> None:\n        test_data = w3.geth.txpool.status()\n        assert \"pending\" in test_data\n\n# Path: web3/_utils/module_testing/net_module.py\nimport pytest\nfrom typing import (\n    TYPE_CHECKING,\n)\n\nfrom eth_utils import (\n    is_boolean,\n    is_integer,\n    is_string,\n)\n\nif TYPE_CHECKING:\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\n\nclass NetModuleTest:\n    def test_net_version(self, w3: \"Web3\") -> None:\n        version = w3.net.version\n\n        assert is_string(version)\n        assert version.isdigit()\n\n    def test_net_listening(self, w3: \"Web3\") -> None:\n        listening = w3.net.listening\n\n        assert is_boolean(listening)\n\n    def test_net_peer_count(self, w3: \"Web3\") -> None:\n        peer_count = w3.net.peer_count\n\n        assert is_integer(peer_count)\n\n\nclass AsyncNetModuleTest:\n    @pytest.mark.asyncio\n    async def test_net_version(self, async_w3: \"AsyncWeb3\") -> None:\n        version = await async_w3.net.version\n\n        assert is_string(version)\n        assert version.isdigit()\n\n    @pytest.mark.asyncio\n    async def test_net_listening(self, async_w3: \"AsyncWeb3\") -> None:\n        listening = await async_w3.net.listening\n\n        assert is_boolean(listening)\n\n    @pytest.mark.asyncio\n    async def test_net_peer_count(self, async_w3: \"AsyncWeb3\") -> None:\n        peer_count = await async_w3.net.peer_count\n\n        assert is_integer(peer_count)\n\n# Path: web3/_utils/module_testing/web3_module.py\nimport pytest\nfrom typing import (\n    Any,\n    NoReturn,\n    Sequence,\n    Union,\n)\n\nfrom eth_typing import (\n    ChecksumAddress,\n    HexAddress,\n    HexStr,\n    TypeStr,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3 import (\n    AsyncWeb3,\n    Web3,\n)\nfrom web3._utils.ens import (\n    ens_addresses,\n)\nfrom web3.exceptions import (\n    InvalidAddress,\n)\n\n\nclass Web3ModuleTest:\n    def test_web3_client_version(self, w3: Web3) -> None:\n        client_version = w3.client_version\n        self._check_web3_client_version(client_version)\n\n    def _check_web3_client_version(self, client_version: str) -> NoReturn:\n        raise NotImplementedError(\"Must be implemented by subclasses\")\n\n    # Contract that calculated test values can be found at\n    # https://kovan.etherscan.io/address/0xb9be06f5b99372cf9afbccadbbb9954ccaf7f4bb#code\n    @pytest.mark.parametrize(\n        \"types,values,expected\",\n        (\n            (\n                [\"bool\"],\n                [True],\n                HexBytes(\n                    \"0x5fe7f977e71dba2ea1a68e21057beebb9be2ac30c6410aa38d4f3fbe41dcffd2\"\n                ),\n            ),\n            (\n                [\"uint8\", \"uint8\", \"uint8\"],\n                [97, 98, 99],\n                HexBytes(\n                    \"0x4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45\"\n                ),\n            ),\n            (\n                [\"uint248\"],\n                [30],\n                HexBytes(\n                    \"0x30f95d210785601eb33ae4d53d405b26f920e765dff87cca8e9a4aec99f82671\"\n                ),\n            ),\n            (\n                [\"bool\", \"uint16\"],\n                [True, 299],\n                HexBytes(\n                    \"0xed18599ccd80ee9fae9a28b0e34a5573c3233d7468f808fd659bc171cf0b43bd\"\n                ),\n            ),\n            (\n                [\"int256\"],\n                [-10],\n                HexBytes(\n                    \"0xd6fb717f7e270a360f5093ce6a7a3752183e89c9a9afe5c0cb54b458a304d3d5\"\n                ),\n            ),\n            (\n                [\"int256\"],\n                [10],\n                HexBytes(\n                    \"0xc65a7bb8d6351c1cf70c95a316cc6a92839c986682d98bc35f958f4883f9d2a8\"\n                ),\n            ),\n            (\n                [\"int8\", \"uint8\"],\n                [-10, 18],\n                HexBytes(\n                    \"0x5c6ab1e634c08d9c0f4df4d789e8727943ef010dd7ca8e3c89de197a26d148be\"\n                ),\n            ),\n            (\n                [\"address\"],\n                [\"0x49eddd3769c0712032808d86597b84ac5c2f5614\"],\n                InvalidAddress,\n            ),\n            (\n                [\"address\"],\n                [\"0x49EdDD3769c0712032808D86597B84ac5c2F5614\"],\n                HexBytes(\n                    \"0x2ff37b5607484cd4eecf6d13292e22bd6e5401eaffcc07e279583bc742c68882\"\n                ),\n            ),\n            (\n                [\"bytes2\"],\n                [\"0x5402\"],\n                HexBytes(\n                    \"0x4ed9171bda52fca71ab28e7f452bd6eacc3e5a568a47e0fa53b503159a9b8910\"\n                ),\n            ),\n            (\n                [\"bytes3\"],\n                [\"0x5402\"],\n                HexBytes(\n                    \"0x4ed9171bda52fca71ab28e7f452bd6eacc3e5a568a47e0fa53b503159a9b8910\"\n                ),\n            ),\n            (\n                [\"bytes\"],\n                [\n                    \"0x636865636b6c6f6e6762797465737472696e676167\"\n                    \"61696e7374736f6c6964697479736861336861736866756e6374696f6e\"\n                ],\n                HexBytes(\n                    \"0xd78a84d65721b67e4011b10c99dafdedcdcd7cb30153064f773e210b4762e22f\"\n                ),\n            ),\n            (\n                [\"string\"],\n                [\"testing a string!\"],\n                HexBytes(\n                    \"0xe8c275c0b4070a5ec6cfcb83f0ba394b30ddd283de785d43f2eabfb04bd96747\"\n                ),\n            ),\n            (\n                [\"string\", \"bool\", \"uint16\", \"bytes2\", \"address\"],\n                [\n                    \"testing a string!\",\n                    False,\n                    299,\n                    \"0x5402\",\n                    \"0x49eddd3769c0712032808d86597b84ac5c2f5614\",\n                ],\n                InvalidAddress,\n            ),\n            (\n                [\"string\", \"bool\", \"uint16\", \"bytes2\", \"address\"],\n                [\n                    \"testing a string!\",\n                    False,\n                    299,\n                    \"0x5402\",\n                    \"0x49EdDD3769c0712032808D86597B84ac5c2F5614\",\n                ],\n                HexBytes(\n                    \"0x8cc6eabb25b842715e8ca39e2524ed946759aa37bfb7d4b81829cf5a7e266103\"\n                ),\n            ),\n            (\n                [\"bool[2][]\"],\n                [[[True, False], [False, True]]],\n                HexBytes(\n                    \"0x1eef261f2eb51a8c736d52be3f91ff79e78a9ec5df2b7f50d0c6f98ed1e2bc06\"\n                ),\n            ),\n            (\n                [\"bool[]\"],\n                [[True, False, True]],\n                HexBytes(\n                    \"0x5c6090c0461491a2941743bda5c3658bf1ea53bbd3edcde54e16205e18b45792\"\n                ),\n            ),\n            (\n                [\"uint24[]\"],\n                [[1, 0, 1]],\n                HexBytes(\n                    \"0x5c6090c0461491a2941743bda5c3658bf1ea53bbd3edcde54e16205e18b45792\"\n                ),\n            ),\n            (\n                [\"uint8[2]\"],\n                [[8, 9]],\n                HexBytes(\n                    \"0xc7694af312c4f286114180fd0ba6a52461fcee8a381636770b19a343af92538a\"\n                ),\n            ),\n            (\n                [\"uint256[2]\"],\n                [[8, 9]],\n                HexBytes(\n                    \"0xc7694af312c4f286114180fd0ba6a52461fcee8a381636770b19a343af92538a\"\n                ),\n            ),\n            (\n                [\"uint8[]\"],\n                [[8]],\n                HexBytes(\n                    \"0xf3f7a9fe364faab93b216da50a3214154f22a0a2b415b23a84c8169e8b636ee3\"\n                ),\n            ),\n            (\n                [\"address[]\"],\n                [\n                    [\n                        \"0x49EdDD3769c0712032808D86597B84ac5c2F5614\",\n                        \"0xA6b759bBbf4B59D24acf7E06e79f3a5D104fdCE5\",\n                    ]\n                ],\n                HexBytes(\n                    \"0xb98565c0c26a962fd54d93b0ed6fb9296e03e9da29d2281ed3e3473109ef7dde\"\n                ),\n            ),\n            (\n                [\"address[]\"],\n                [\n                    [\n                        \"0x49EdDD3769c0712032808D86597B84ac5c2F5614\",\n                        \"0xa6b759bbbf4b59d24acf7e06e79f3a5d104fdce5\",\n                    ]\n                ],\n                InvalidAddress,\n            ),\n        ),\n    )\n    @pytest.mark.parametrize(\n        \"w3\",\n        (\n            Web3,\n            AsyncWeb3,\n        ),\n    )\n    def test_solidity_keccak(\n        self,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        types: Sequence[TypeStr],\n        values: Sequence[Any],\n        expected: HexBytes,\n    ) -> None:\n        if isinstance(expected, type) and issubclass(expected, Exception):\n            with pytest.raises(expected):\n                w3.solidity_keccak(types, values)\n            return\n\n        actual = w3.solidity_keccak(types, values)\n        assert actual == expected\n\n    @pytest.mark.parametrize(\n        \"types, values, expected\",\n        (\n            (\n                [\"address\"],\n                [\"one.eth\"],\n                HexBytes(\n                    \"0x2ff37b5607484cd4eecf6d13292e22bd6e5401eaffcc07e279583bc742c68882\"\n                ),\n            ),\n            (\n                [\"address[]\"],\n                [[\"one.eth\", \"two.eth\"]],\n                HexBytes(\n                    \"0xb98565c0c26a962fd54d93b0ed6fb9296e03e9da29d2281ed3e3473109ef7dde\"\n                ),\n            ),\n        ),\n    )\n    @pytest.mark.parametrize(\n        \"w3\",\n        (\n            Web3(),\n            AsyncWeb3(),\n        ),\n    )\n    def test_solidity_keccak_ens(\n        self,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        types: Sequence[TypeStr],\n        values: Sequence[str],\n        expected: HexBytes,\n    ) -> None:\n        with ens_addresses(\n            w3,\n            {\n                \"one.eth\": ChecksumAddress(\n                    HexAddress(HexStr(\"0x49EdDD3769c0712032808D86597B84ac5c2F5614\"))\n                ),\n                \"two.eth\": ChecksumAddress(\n                    HexAddress(HexStr(\"0xA6b759bBbf4B59D24acf7E06e79f3a5D104fdCE5\"))\n                ),\n            },\n        ):\n            # when called as class method, any name lookup attempt will fail\n            with pytest.raises(InvalidAddress):\n                Web3.solidity_keccak(types, values)\n\n            # when called as instance method, ens lookups can succeed\n            actual = w3.solidity_keccak(types, values)\n            assert actual == expected\n\n    @pytest.mark.parametrize(\n        \"types,values\",\n        (\n            ([\"address\"], [\"0xA6b759bBbf4B59D24acf7E06e79f3a5D104fdCE5\", True]),\n            ([\"address\", \"bool\"], [\"0xA6b759bBbf4B59D24acf7E06e79f3a5D104fdCE5\"]),\n            ([], [\"0xA6b759bBbf4B59D24acf7E06e79f3a5D104fdCE5\"]),\n        ),\n    )\n    def test_solidity_keccak_same_number_of_types_and_values(\n        self, w3: \"Web3\", types: Sequence[TypeStr], values: Sequence[Any]\n    ) -> None:\n        with pytest.raises(ValueError):\n            w3.solidity_keccak(types, values)\n\n    def test_is_connected(self, w3: \"Web3\") -> None:\n        assert w3.is_connected()\n\n# Path: web3/_utils/module_testing/__init__.py\nfrom .eth_module import (\n    AsyncEthModuleTest,\n    EthModuleTest,\n)\nfrom .go_ethereum_admin_module import (\n    GoEthereumAdminModuleTest,\n)\nfrom .go_ethereum_personal_module import (\n    GoEthereumPersonalModuleTest,\n)\nfrom .go_ethereum_txpool_module import (\n    GoEthereumAsyncTxPoolModuleTest,\n    GoEthereumTxPoolModuleTest,\n)\nfrom .net_module import (\n    AsyncNetModuleTest,\n    NetModuleTest,\n)\nfrom .web3_module import (\n    Web3ModuleTest,\n)\n\n# Path: web3/_utils/module_testing/persistent_connection_provider.py\nimport asyncio\nimport pytest\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Tuple,\n    cast,\n)\n\nfrom eth_utils import (\n    is_hexstr,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3.datastructures import (\n    AttributeDict,\n)\nfrom web3.middleware import (\n    ExtraDataToPOAMiddleware,\n)\nfrom web3.types import (\n    FormattedEthSubscriptionResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (\n        AsyncWeb3,\n    )\n\n\nclass PersistentConnectionProviderTest:\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\n        \"subscription_params,ws_subscription_response,expected_formatted_result\",\n        (\n            (\n                (\"newHeads\",),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": {\n                            \"number\": \"0x539\",\n                            \"hash\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                            \"parentHash\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                            \"sha3Uncles\": \"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",  # noqa: E501\n                            \"logsBloom\": \"0x00\",\n                            \"transactionsRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"stateRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"receiptsRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"miner\": \"0x0000000000000000000000000000000000000000\",\n                            \"difficulty\": \"0x0\",\n                            \"extraData\": \"0x496c6c756d696e61746520446d6f63726174697a6520447374726962757465\",  # noqa: E501\n                            \"gasLimit\": \"0x1c9c380\",\n                            \"gasUsed\": \"0xd1ce44\",\n                            \"timestamp\": \"0x539\",\n                            \"baseFeePerGas\": \"0x26f93fef9\",\n                            \"withdrawalsRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"nonce\": \"0x0000000000000000\",\n                            \"mixHash\": \"0x73e9e036ec894047f29954571d4b6d9e8717de7304269c263cbf150caa4e0768\",  # noqa: E501\n                        },\n                    },\n                },\n                AttributeDict(\n                    {\n                        \"number\": 1337,\n                        \"hash\": HexBytes(\n                            \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"  # noqa: E501\n                        ),\n                        \"parentHash\": HexBytes(\n                            \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"  # noqa: E501\n                        ),\n                        \"sha3Uncles\": HexBytes(\n                            \"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\"  # noqa: E501\n                        ),\n                        \"logsBloom\": HexBytes(\"0x00\"),\n                        \"transactionsRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"stateRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"receiptsRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"miner\": \"0x0000000000000000000000000000000000000000\",\n                        \"difficulty\": 0,\n                        \"extraData\": HexBytes(\n                            \"0x496c6c756d696e61746520446d6f63726174697a6520447374726962757465\"  # noqa: E501\n                        ),\n                        \"gasLimit\": 30000000,\n                        \"gasUsed\": 13749828,\n                        \"timestamp\": 1337,\n                        \"baseFeePerGas\": 10461904633,\n                        \"withdrawalsRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"nonce\": HexBytes(\"0x0000000000000000\"),\n                        \"mixHash\": HexBytes(\n                            \"0x73e9e036ec894047f29954571d4b6d9e8717de7304269c263cbf150caa4e0768\"  # noqa: E501\n                        ),\n                    }\n                ),\n            ),\n            (\n                (\"newPendingTransactions\", True),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": {\n                            \"blockHash\": None,\n                            \"blockNumber\": None,\n                            \"from\": \"0x0000000000000000000000000000000000000000\",\n                            \"gas\": \"0xf2f4\",\n                            \"gasPrice\": \"0x29035f36f\",\n                            \"maxFeePerGas\": \"0x29035f36f\",\n                            \"maxPriorityFeePerGas\": \"0x3b9aca00\",\n                            \"hash\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                            \"input\": \"0x00\",\n                            \"nonce\": \"0x2013\",\n                            \"to\": \"0x0000000000000000000000000000000000000000\",\n                            \"transactionIndex\": None,\n                            \"value\": \"0x0\",\n                            \"type\": \"0x2\",\n                            \"accessList\": [],\n                            \"chainId\": \"0x1\",\n                            \"v\": \"0x1\",\n                            \"r\": \"0x3c144a7c00ed3118d55445cd5be2ae4620ca377f7c685e9c5f3687671d4dece1\",  # noqa: E501\n                            \"s\": \"0x284de67cbf75fec8a9edb368dee3a37cf6faba87f0af4413b2f869ebfa87d002\",  # noqa: E501\n                            \"yParity\": \"0x1\",\n                        },\n                    },\n                },\n                AttributeDict(\n                    {\n                        \"blockHash\": None,\n                        \"blockNumber\": None,\n                        \"from\": \"0x0000000000000000000000000000000000000000\",\n                        \"gas\": 62196,\n                        \"gasPrice\": 11009389423,\n                        \"maxFeePerGas\": 11009389423,\n                        \"maxPriorityFeePerGas\": 1000000000,\n                        \"hash\": HexBytes(\n                            \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"  # noqa: E501\n                        ),\n                        \"input\": HexBytes(\"0x00\"),\n                        \"nonce\": 8211,\n                        \"to\": \"0x0000000000000000000000000000000000000000\",\n                        \"transactionIndex\": None,\n                        \"value\": 0,\n                        \"type\": 2,\n                        \"accessList\": [],\n                        \"chainId\": 1,\n                        \"v\": 1,\n                        \"r\": HexBytes(\n                            \"0x3c144a7c00ed3118d55445cd5be2ae4620ca377f7c685e9c5f3687671d4dece1\"  # noqa: E501\n                        ),\n                        \"s\": HexBytes(\n                            \"0x284de67cbf75fec8a9edb368dee3a37cf6faba87f0af4413b2f869ebfa87d002\"  # noqa: E501\n                        ),\n                        \"yParity\": 1,\n                    }\n                ),\n            ),\n            (\n                (\"newPendingTransactions\", False),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n...\n", "needle_token_start": 747, "needle_token_end": 853, "code_context_ntokens": 16244, "output": ["```python\ndef test_eth_get_logs_with_logs_none_topic_args(self, w3: \"Web3\") -> None:\n    # Test with None overflowing\n    filter_params: FilterParams = {\n        \"fromBlock\": BlockNumber(0),\n        \"topics\": [None, None, None],\n    }\n\n    result = w3.eth.get_logs(filter_params)\n    assert len(result) == 0\n```"]}
{"repo": "ethereum/web3.py", "name": "_get_response_for_request_id", "language": "python", "path": "web3/providers/persistent/persistent.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the response associated with a specific request identifier from a cache, with a mechanism to handle timeouts if the response is not available within a specified duration.\n2. **Input**: A unique identifier for the request and an optional timeout duration.\n3. **Output**: The response corresponding to the request identifier if available; otherwise, an exception is raised if the response is not found within the timeout period.\n4. **Procedure**: The function checks a cache for the response corresponding to the given request identifier. It continuously yields control back to the event loop to allow other operations to proceed while waiting. If the response is found within the allotted time, it is returned. If not, an exception indicating a timeout is raised.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: web3/providers/persistent/request_processor.py\nimport asyncio\nfrom copy import (\n    copy,\n)\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Dict,\n    Optional,\n    Tuple,\n)\n\nfrom web3._utils.caching import (\n    RequestInformation,\n    generate_cache_key,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\nfrom web3.utils import (\n    SimpleCache,\n)\n\nif TYPE_CHECKING:\n    from web3.providers.persistent import (\n        PersistentConnectionProvider,\n    )\n\n\nclass RequestProcessor:\n    _subscription_queue_synced_with_ws_stream: bool = False\n\n    def __init__(\n        self,\n        provider: \"PersistentConnectionProvider\",\n        subscription_response_queue_size: int = 500,\n    ) -> None:\n        self._provider = provider\n\n        self._request_information_cache: SimpleCache = SimpleCache(500)\n        self._request_response_cache: SimpleCache = SimpleCache(500)\n        self._subscription_response_queue: asyncio.Queue[RPCResponse] = asyncio.Queue(\n            maxsize=subscription_response_queue_size\n        )\n\n    @property\n    def active_subscriptions(self) -> Dict[str, Any]:\n        return {\n            value.subscription_id: {\"params\": value.params}\n            for key, value in self._request_information_cache.items()\n            if value.method == \"eth_subscribe\"\n        }\n\n    # request information cache\n\n    def cache_request_information(\n        self,\n        method: RPCEndpoint,\n        params: Any,\n        response_formatters: Tuple[Callable[..., Any], ...],\n    ) -> Optional[str]:\n        cached_requests_key = generate_cache_key((method, params))\n        if cached_requests_key in self._provider._request_cache._data:\n            cached_response = self._provider._request_cache._data[cached_requests_key]\n            cached_response_id = cached_response.get(\"id\")\n            cache_key = generate_cache_key(cached_response_id)\n            if cache_key in self._request_information_cache:\n                self._provider.logger.debug(\n                    \"This is a cached request, not caching request info because it is \"\n                    f\"not unique:\\n    method={method},\\n    params={params}\"\n                )\n                return None\n\n        # copy the request counter and find the next request id without incrementing\n        # since this is done when / if the request is successfully sent\n        request_id = next(copy(self._provider.request_counter))\n        cache_key = generate_cache_key(request_id)\n\n        self._bump_cache_if_key_present(cache_key, request_id)\n\n        request_info = RequestInformation(\n            method,\n            params,\n            response_formatters,\n        )\n        self._provider.logger.debug(\n            f\"Caching request info:\\n    request_id={request_id},\\n\"\n            f\"    cache_key={cache_key},\\n    request_info={request_info.__dict__}\"\n        )\n        self._request_information_cache.cache(\n            cache_key,\n            request_info,\n        )\n        return cache_key\n\n    def _bump_cache_if_key_present(self, cache_key: str, request_id: int) -> None:\n        \"\"\"\n        If the cache key is present in the cache, bump the cache key and request id\n        by one to make room for the new request. This behavior is necessary when a\n        request is made but inner requests, say to `eth_estimateGas` if the `gas` is\n        missing, are made before the original request is sent.\n        \"\"\"\n        if cache_key in self._request_information_cache:\n            original_request_info = self._request_information_cache.get_cache_entry(\n                cache_key\n            )\n            bump = generate_cache_key(request_id + 1)\n\n            # recursively bump the cache if the new key is also present\n            self._bump_cache_if_key_present(bump, request_id + 1)\n\n            self._provider.logger.debug(\n                \"Caching internal request. Bumping original request in cache:\\n\"\n                f\"    request_id=[{request_id}] -> [{request_id + 1}],\\n\"\n                f\"    cache_key=[{cache_key}] -> [{bump}],\\n\"\n                f\"    request_info={original_request_info.__dict__}\"\n            )\n            self._request_information_cache.cache(bump, original_request_info)\n\n    def pop_cached_request_information(\n        self, cache_key: str\n    ) -> Optional[RequestInformation]:\n        request_info = self._request_information_cache.pop(cache_key)\n        if request_info is not None:\n            self._provider.logger.debug(\n                \"Request info popped from cache:\\n\"\n                f\"    cache_key={cache_key},\\n    request_info={request_info.__dict__}\"\n            )\n        return request_info\n\n    def get_request_information_for_response(\n        self,\n        response: RPCResponse,\n    ) -> RequestInformation:\n        if \"method\" in response and response[\"method\"] == \"eth_subscription\":\n            if \"params\" not in response:\n                raise ValueError(\"Subscription response must have params field\")\n            if \"subscription\" not in response[\"params\"]:\n                raise ValueError(\n                    \"Subscription response params must have subscription field\"\n                )\n\n            # retrieve the request info from the cache using the subscription id\n            cache_key = generate_cache_key(response[\"params\"][\"subscription\"])\n            request_info = (\n                # don't pop the request info from the cache, since we need to keep it\n                # to process future subscription responses\n                # i.e. subscription request information remains in the cache\n                self._request_information_cache.get_cache_entry(cache_key)\n            )\n\n        else:\n            # retrieve the request info from the cache using the request id\n            cache_key = generate_cache_key(response[\"id\"])\n            if response in self._provider._request_cache._data.values():\n                request_info = (\n                    # don't pop the request info from the cache, since we need to keep\n                    # it to process future responses\n                    # i.e. request information remains in the cache\n                    self._request_information_cache.get_cache_entry(cache_key)\n                )\n            else:\n                request_info = (\n...\n# Path: web3/providers/persistent/persistent.py\nfrom abc import (\n    ABC,\n)\nimport asyncio\nimport logging\nfrom typing import (\n    Optional,\n)\n\nfrom web3._utils.caching import (\n    generate_cache_key,\n)\nfrom web3.exceptions import (\n    TimeExhausted,\n)\nfrom web3.providers.async_base import (\n    AsyncJSONBaseProvider,\n)\nfrom web3.providers.persistent.request_processor import (\n    RequestProcessor,\n)\nfrom web3.types import (\n    RPCId,\n    RPCResponse,\n)\n\nDEFAULT_PERSISTENT_CONNECTION_TIMEOUT = 30.0\n\n\nclass PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):\n    logger = logging.getLogger(\"web3.providers.PersistentConnectionProvider\")\n    has_persistent_connection = True\n    endpoint_uri: Optional[str] = None\n\n    _request_processor: RequestProcessor\n    _message_listener_task: Optional[\"asyncio.Task[None]\"] = None\n    _listen_event: asyncio.Event = asyncio.Event()\n\n    def __init__(\n        self,\n        request_timeout: float = DEFAULT_PERSISTENT_CONNECTION_TIMEOUT,\n        subscription_response_queue_size: int = 500,\n        silence_listener_task_exceptions: bool = False,\n    ) -> None:\n        super().__init__()\n        self._request_processor = RequestProcessor(\n            self,\n            subscription_response_queue_size=subscription_response_queue_size,\n        )\n        self.request_timeout = request_timeout\n        self.silence_listener_task_exceptions = silence_listener_task_exceptions\n\n    async def connect(self) -> None:\n        raise NotImplementedError(\"Must be implemented by subclasses\")\n\n    async def disconnect(self) -> None:\n        raise NotImplementedError(\"Must be implemented by subclasses\")\n\n    async def _message_listener(self) -> None:\n        raise NotImplementedError(\"Must be implemented by subclasses\")\n\n    \nasync def _get_response_for_request_id(\n        self, request_id: RPCId, timeout: Optional[float] = None\n    ) -> RPCResponse:\n        if timeout is None:\n            timeout = self.request_timeout\n\n        async def _match_response_id_to_request_id() -> RPCResponse:\n            request_cache_key = generate_cache_key(request_id)\n\n            while True:\n                # sleep(0) here seems to be the most efficient way to yield control\n                # back to the event loop while waiting for the response to be in the\n                # queue.\n                await asyncio.sleep(0)\n\n                if request_cache_key in self._request_processor._request_response_cache:\n                    self.logger.debug(\n                        f\"Popping response for id {request_id} from cache.\"\n                    )\n                    popped_response = self._request_processor.pop_raw_response(\n                        cache_key=request_cache_key,\n                    )\n                    return popped_response\n\n        try:\n            # Add the request timeout around the while loop that checks the request\n            # cache and tried to recv(). If the request is neither in the cache, nor\n            # received within the request_timeout, raise ``TimeExhausted``.\n            return await asyncio.wait_for(_match_response_id_to_request_id(), timeout)\n        except asyncio.TimeoutError:\n            raise TimeExhausted(\n                f\"Timed out waiting for response with request id `{request_id}` after \"\n                f\"{self.request_timeout} second(s). This may be due to the provider \"\n                \"not returning a response with the same id that was sent in the \"\n                \"request or an exception raised during the request was caught and \"\n                \"allowed to continue.\"\n            )\n\n# Path: web3/method.py\nimport functools\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Dict,\n    Generic,\n    List,\n    Optional,\n    Sequence,\n    Tuple,\n    Type,\n    TypeVar,\n    Union,\n)\nimport warnings\n\nfrom eth_utils.curried import (\n    to_tuple,\n)\nfrom eth_utils.toolz import (\n    pipe,\n)\n\nfrom web3._utils.method_formatters import (\n    get_error_formatters,\n    get_null_result_formatters,\n    get_request_formatters,\n    get_result_formatters,\n)\nfrom web3._utils.rpc_abi import (\n    RPC,\n)\nfrom web3.exceptions import (\n    Web3ValidationError,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    TReturn,\n)\n\nif TYPE_CHECKING:\n    from web3 import Web3  # noqa: F401\n    from web3.module import Module  # noqa: F401\n\n\nMunger = Callable[..., Any]\n\n\n@to_tuple\ndef _apply_request_formatters(\n    params: Any, request_formatters: Dict[RPCEndpoint, Callable[..., TReturn]]\n) -> Tuple[Any, ...]:\n    if request_formatters:\n        formatted_params = pipe(params, request_formatters)\n        return formatted_params\n    return params\n\n\ndef _set_mungers(\n    mungers: Optional[Sequence[Munger]], is_property: bool\n) -> Sequence[Any]:\n    if is_property and mungers:\n        raise Web3ValidationError(\"Mungers cannot be used with a property.\")\n\n    return (\n        mungers\n        if mungers\n        else [default_munger] if is_property else [default_root_munger]\n    )\n\n\ndef default_munger(_module: \"Module\", *args: Any, **kwargs: Any) -> Tuple[()]:\n    if args or kwargs:\n        raise Web3ValidationError(\"Parameters cannot be passed to a property.\")\n    return ()\n\n\ndef default_root_munger(_module: \"Module\", *args: Any) -> List[Any]:\n    return [*args]\n\n\nTFunc = TypeVar(\"TFunc\", bound=Callable[..., Any])\n\n\nclass Method(Generic[TFunc]):\n    \"\"\"Method object for web3 module methods\n\n    Calls to the Method go through these steps:\n\n    1. input munging - includes normalization, parameter checking, early parameter\n    formatting. Any processing on the input parameters that need to happen before\n    json_rpc method string selection occurs.\n\n            A note about mungers: The first (root) munger should reflect the desired\n        api function arguments. In other words, if the api function wants to\n        behave as: `get_balance(account, block_identifier=None)`, the root munger\n        should accept these same arguments, with the addition of the module as\n        the first argument e.g.:\n\n        ```\n        def get_balance_root_munger(module, account, block_identifier=None):\n            if block_identifier is None:\n                block_identifier = DEFAULT_BLOCK\n            return module, [account, block_identifier]\n        ```\n\n        all mungers should return an argument list.\n\n        if no munger is provided, a default munger expecting no method arguments\n        will be used.\n\n    2. method selection - The json_rpc_method argument can be method string or a\n    function that returns a method string. If a callable is provided the processed\n    method inputs are passed to the method selection function, and the returned\n    method string is used.\n\n    3. request and response formatters are set - formatters are retrieved\n    using the json rpc method string.\n\n    4. After the parameter processing from steps 1-3 the request is made using\n    the calling function returned by the module attribute ``retrieve_caller_fn``\n    and the response formatters are applied to the output.\n    \"\"\"\n\n    def __init__(\n        self,\n        json_rpc_method: Optional[RPCEndpoint] = None,\n        mungers: Optional[Sequence[Munger]] = None,\n        request_formatters: Optional[Callable[..., TReturn]] = None,\n        result_formatters: Optional[Callable[..., TReturn]] = None,\n        null_result_formatters: Optional[Callable[..., TReturn]] = None,\n        method_choice_depends_on_args: Optional[Callable[..., RPCEndpoint]] = None,\n        is_property: bool = False,\n    ):\n        self.json_rpc_method = json_rpc_method\n        self.mungers = _set_mungers(mungers, is_property)\n        self.request_formatters = request_formatters or get_request_formatters\n        self.result_formatters = result_formatters or get_result_formatters\n        self.null_result_formatters = (\n            null_result_formatters or get_null_result_formatters\n        )\n        self.method_choice_depends_on_args = method_choice_depends_on_args\n        self.is_property = is_property\n\n    def __get__(\n        self, obj: Optional[\"Module\"] = None, obj_type: Optional[Type[\"Module\"]] = None\n    ) -> TFunc:\n        if obj is None:\n            raise TypeError(\n                \"Direct calls to methods are not supported. \"\n                \"Methods must be called from an module instance, \"\n                \"usually attached to a web3 instance.\"\n            )\n        return obj.retrieve_caller_fn(self)\n\n    @property\n    def method_selector_fn(\n        self,\n    ) -> Callable[..., Union[RPCEndpoint, Callable[..., RPCEndpoint]]]:\n        \"\"\"Gets the method selector from the config.\"\"\"\n        if callable(self.json_rpc_method):\n            return self.json_rpc_method\n        elif isinstance(self.json_rpc_method, (str,)):\n            return lambda *_: self.json_rpc_method\n        raise ValueError(\n            \"``json_rpc_method`` config invalid.  May be a string or function\"\n        )\n\n    def input_munger(self, module: \"Module\", args: Any, kwargs: Any) -> List[Any]:\n        # This function takes the input parameters and munges them.\n        # See the test_process_params test in ``tests/core/method-class/test_method.py``\n        # for an example with multiple mungers.\n        return functools.reduce(\n            lambda args, munger: munger(module, *args, **kwargs), self.mungers, args\n        )\n\n    def process_params(self, module: \"Module\", *args: Any, **kwargs: Any) -> Tuple[\n        Tuple[Union[RPCEndpoint, Callable[..., RPCEndpoint]], Tuple[Any, ...]],\n        Tuple[\n            Union[TReturn, Dict[str, Callable[..., Any]]],\n            Callable[..., Any],\n            Union[TReturn, Callable[..., Any]],\n        ],\n    ]:\n        params = self.input_munger(module, args, kwargs)\n\n        if self.method_choice_depends_on_args:\n            # If the method choice depends on the args that get passed in,\n            # the first parameter determines which method needs to be called\n            self.json_rpc_method = self.method_choice_depends_on_args(value=params[0])\n\n            pending_or_latest_filter_methods = [\n                RPC.eth_newPendingTransactionFilter,\n                RPC.eth_newBlockFilter,\n            ]\n            if self.json_rpc_method in pending_or_latest_filter_methods:\n                # For pending or latest filter methods, use params to determine\n                # which method to call, but don't pass them through with the request\n                params = []\n\n        method = self.method_selector_fn()\n        response_formatters = (\n            self.result_formatters(method, module),\n            get_error_formatters(method),\n            self.null_result_formatters(method),\n        )\n        request = (\n            method,\n            _apply_request_formatters(params, self.request_formatters(method)),\n        )\n        return request, response_formatters\n\n\nclass DeprecatedMethod:\n    def __init__(\n        self, method: Method[Callable[..., Any]], old_name: str, new_name: str\n    ) -> None:\n        self.method = method\n        self.old_name = old_name\n        self.new_name = new_name\n\n    def __get__(\n        self, obj: Optional[\"Module\"] = None, obj_type: Optional[Type[\"Module\"]] = None\n    ) -> Any:\n        warnings.warn(\n            f\"{self.old_name} is deprecated in favor of {self.new_name}\",\n            category=DeprecationWarning,\n        )\n        return self.method.__get__(obj, obj_type)\n\n# Path: web3/module.py\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Coroutine,\n    Dict,\n    Optional,\n    TypeVar,\n    Union,\n    cast,\n)\n\nfrom eth_abi.codec import (\n    ABICodec,\n)\nfrom eth_utils.toolz import (\n    curry,\n    pipe,\n)\n\nfrom web3._utils.filters import (\n    AsyncLogFilter,\n    LogFilter,\n    _UseExistingFilter,\n)\nfrom web3.method import (\n    Method,\n)\nfrom web3.providers.persistent import (\n    PersistentConnectionProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\n\n@curry\ndef apply_result_formatters(\n    result_formatters: Callable[..., Any], result: RPCResponse\n) -> RPCResponse:\n    if result_formatters:\n        formatted_result = pipe(result, result_formatters)\n        return formatted_result\n    else:\n        return result\n\n\nTReturn = TypeVar(\"TReturn\")\n\n\n@curry\ndef retrieve_blocking_method_call_fn(\n    w3: \"Web3\", module: \"Module\", method: Method[Callable[..., TReturn]]\n) -> Callable[..., Union[TReturn, LogFilter]]:\n    def caller(*args: Any, **kwargs: Any) -> Union[TReturn, LogFilter]:\n        try:\n            (method_str, params), response_formatters = method.process_params(\n                module, *args, **kwargs\n            )\n        except _UseExistingFilter as err:\n            return LogFilter(eth_module=module, filter_id=err.filter_id)\n\n        (\n            result_formatters,\n            error_formatters,\n            null_result_formatters,\n        ) = response_formatters\n        result = w3.manager.request_blocking(\n            method_str, params, error_formatters, null_result_formatters\n        )\n        return apply_result_formatters(result_formatters, result)\n\n    return caller\n\n\n@curry\ndef retrieve_async_method_call_fn(\n    async_w3: \"AsyncWeb3\", module: \"Module\", method: Method[Callable[..., Any]]\n) -> Callable[..., Coroutine[Any, Any, Optional[Union[RPCResponse, AsyncLogFilter]]]]:\n    async def caller(*args: Any, **kwargs: Any) -> Union[RPCResponse, AsyncLogFilter]:\n        try:\n            (method_str, params), response_formatters = method.process_params(\n                module, *args, **kwargs\n            )\n        except _UseExistingFilter as err:\n            return AsyncLogFilter(eth_module=module, filter_id=err.filter_id)\n\n        if isinstance(async_w3.provider, PersistentConnectionProvider):\n            # TODO: The typing does not seem to be correct for response_formatters.\n            #   For now, keep the expected typing but ignore it here.\n            provider = async_w3.provider\n            cache_key = provider._request_processor.cache_request_information(\n                cast(RPCEndpoint, method_str), params, response_formatters  # type: ignore # noqa: E501\n            )\n            try:\n                method_str = cast(RPCEndpoint, method_str)\n                return await async_w3.manager.send(method_str, params)\n            except Exception as e:\n                if (\n                    cache_key is not None\n                    and cache_key\n                    in provider._request_processor._request_information_cache\n                ):\n                    provider._request_processor.pop_cached_request_information(\n                        cache_key\n                    )\n                raise e\n        else:\n            (\n                result_formatters,\n                error_formatters,\n                null_result_formatters,\n            ) = response_formatters\n\n            result = await async_w3.manager.coro_request(\n                method_str, params, error_formatters, null_result_formatters\n            )\n            return apply_result_formatters(result_formatters, result)\n\n    return caller\n\n\n#  Module should no longer have access to the full web3 api.\n#  Only the calling functions need access to the request methods.\n#  Any \"re-entrant\" shenanigans can go in the middlewares, which do\n#  have web3 access.\nclass Module:\n    is_async = False\n\n    def __init__(self, w3: Union[\"AsyncWeb3\", \"Web3\"]) -> None:\n        if self.is_async:\n            self.retrieve_caller_fn = retrieve_async_method_call_fn(w3, self)\n        else:\n            self.retrieve_caller_fn = retrieve_blocking_method_call_fn(w3, self)\n        self.w3 = w3\n\n    @property\n    def codec(self) -> ABICodec:\n        # use codec set on the Web3 instance\n        return self.w3.codec\n\n    def attach_methods(\n        self,\n        methods: Dict[str, Method[Callable[..., Any]]],\n    ) -> None:\n        for method_name, method_class in methods.items():\n            klass = (\n                method_class.__get__(obj=self)()\n                if method_class.is_property\n                else method_class.__get__(obj=self)\n            )\n            setattr(self, method_name, klass)\n\n# Path: web3/manager.py\nimport asyncio\nimport logging\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    AsyncGenerator,\n    Callable,\n    List,\n    Optional,\n    Sequence,\n    Tuple,\n    Union,\n    cast,\n)\n\nfrom eth_utils.toolz import (\n    pipe,\n)\nfrom hexbytes import (\n    HexBytes,\n)\nfrom websockets.exceptions import (\n    ConnectionClosedOK,\n)\n\nfrom web3._utils.caching import (\n    generate_cache_key,\n)\nfrom web3._utils.compat import (\n    Self,\n)\nfrom web3.datastructures import (\n    NamedElementOnion,\n)\nfrom web3.exceptions import (\n    BadResponseFormat,\n    MethodUnavailable,\n    ProviderConnectionError,\n)\nfrom web3.middleware import (\n    AttributeDictMiddleware,\n    BufferedGasEstimateMiddleware,\n    ENSNameToAddressMiddleware,\n    GasPriceStrategyMiddleware,\n    ValidationMiddleware,\n)\nfrom web3.middleware.base import (\n    Middleware,\n    MiddlewareOnion,\n)\nfrom web3.module import (\n    apply_result_formatters,\n)\nfrom web3.providers import (\n    AutoProvider,\n    PersistentConnectionProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n    from web3.middleware.base import (  # noqa: F401\n        Web3Middleware,\n    )\n    from web3.providers import (  # noqa: F401\n        AsyncBaseProvider,\n        BaseProvider,\n    )\n    from web3.providers.persistent.request_processor import (  # noqa: F401\n        RequestProcessor,\n    )\n\n\nNULL_RESPONSES = [None, HexBytes(\"0x\"), \"0x\"]\nMETHOD_NOT_FOUND = -32601\n\n\ndef _raise_bad_response_format(response: RPCResponse, error: str = \"\") -> None:\n    message = \"The response was in an unexpected format and unable to be parsed.\"\n    raw_response = f\"The raw response is: {response}\"\n\n    if error is not None and error != \"\":\n        message = f\"{message} {error}. {raw_response}\"\n    else:\n        message = f\"{message} {raw_response}\"\n\n    raise BadResponseFormat(message)\n\n\ndef apply_error_formatters(\n    error_formatters: Callable[..., Any],\n    response: RPCResponse,\n) -> RPCResponse:\n    if error_formatters:\n        formatted_resp = pipe(response, error_formatters)\n        return formatted_resp\n    else:\n        return response\n\n\ndef apply_null_result_formatters(\n    null_result_formatters: Callable[..., Any],\n    response: RPCResponse,\n    params: Optional[Any] = None,\n) -> RPCResponse:\n    if null_result_formatters:\n        formatted_resp = pipe(params, null_result_formatters)\n        return formatted_resp\n    else:\n        return response\n\n\nclass RequestManager:\n    logger = logging.getLogger(\"web3.manager.RequestManager\")\n\n    middleware_onion: Union[\"MiddlewareOnion\", NamedElementOnion[None, None]]\n\n    def __init__(\n        self,\n        w3: Union[\"AsyncWeb3\", \"Web3\"],\n        provider: Optional[Union[\"BaseProvider\", \"AsyncBaseProvider\"]] = None,\n        middlewares: Optional[Sequence[Tuple[Middleware, str]]] = None,\n    ) -> None:\n        self.w3 = w3\n\n        if provider is None:\n            self.provider = AutoProvider()\n        else:\n            self.provider = provider\n\n        if middlewares is None:\n            middlewares = self.get_default_middlewares()\n\n        self.middleware_onion = NamedElementOnion(middlewares)\n\n        if isinstance(provider, PersistentConnectionProvider):\n            # set up the request processor to be able to properly process ordered\n            # responses from the persistent connection as FIFO\n            provider = cast(PersistentConnectionProvider, self.provider)\n            self._request_processor: RequestProcessor = provider._request_processor\n\n    w3: Union[\"AsyncWeb3\", \"Web3\"] = None\n    _provider = None\n\n    @property\n    def provider(self) -> Union[\"BaseProvider\", \"AsyncBaseProvider\"]:\n        return self._provider\n\n    @provider.setter\n    def provider(self, provider: Union[\"BaseProvider\", \"AsyncBaseProvider\"]) -> None:\n        self._provider = provider\n\n    @staticmethod\n    def get_default_middlewares() -> List[Tuple[Middleware, str]]:\n        \"\"\"\n        List the default middlewares for the request manager.\n        Documentation should remain in sync with these defaults.\n        \"\"\"\n        return [\n            (GasPriceStrategyMiddleware, \"gas_price_strategy\"),\n            (ENSNameToAddressMiddleware, \"ens_name_to_address\"),\n            (AttributeDictMiddleware, \"attrdict\"),\n            (ValidationMiddleware, \"validation\"),\n            (BufferedGasEstimateMiddleware, \"gas_estimate\"),\n        ]\n\n    #\n    # Provider requests and response\n    #\n    def _make_request(\n        self, method: Union[RPCEndpoint, Callable[..., RPCEndpoint]], params: Any\n    ) -> RPCResponse:\n        provider = cast(\"BaseProvider\", self.provider)\n        request_func = provider.request_func(\n            cast(\"Web3\", self.w3), cast(\"MiddlewareOnion\", self.middleware_onion)\n        )\n        self.logger.debug(f\"Making request. Method: {method}\")\n        return request_func(method, params)\n\n    async def _coro_make_request(\n        self, method: Union[RPCEndpoint, Callable[..., RPCEndpoint]], params: Any\n    ) -> RPCResponse:\n        provider = cast(\"AsyncBaseProvider\", self.provider)\n        request_func = await provider.request_func(\n            cast(\"AsyncWeb3\", self.w3), cast(\"MiddlewareOnion\", self.middleware_onion)\n        )\n        self.logger.debug(f\"Making request. Method: {method}\")\n        return await request_func(method, params)\n\n    #\n    # formatted_response parses and validates JSON-RPC responses for expected\n    # properties (result or an error) with the expected types.\n    #\n    # Required properties are not strictly enforced to further determine which\n    # exception to raise for specific cases.\n    #\n    # See also: https://www.jsonrpc.org/specification\n    #\n    @staticmethod\n    def formatted_response(\n        response: RPCResponse,\n        params: Any,\n        error_formatters: Optional[Callable[..., Any]] = None,\n        null_result_formatters: Optional[Callable[..., Any]] = None,\n    ) -> Any:\n        # jsonrpc is not enforced (as per the spec) but if present, it must be 2.0\n        if \"jsonrpc\" in response and response[\"jsonrpc\"] != \"2.0\":\n            _raise_bad_response_format(\n                response, 'The \"jsonrpc\" field must be present with a value of \"2.0\"'\n            )\n\n        # id is not enforced (as per the spec) but if present, it must be a\n        # string or integer\n        # TODO: v7 - enforce id per the spec\n        if \"id\" in response:\n            response_id = response[\"id\"]\n            # id is always None for errors\n            if response_id is None and \"error\" not in response:\n                _raise_bad_response_format(\n                    response, '\"id\" must be None when an error is present'\n                )\n            elif not isinstance(response_id, (str, int, type(None))):\n                _raise_bad_response_format(response, '\"id\" must be a string or integer')\n\n        # Response may not include both \"error\" and \"result\"\n        if \"error\" in response and \"result\" in response:\n            _raise_bad_response_format(\n                response, 'Response cannot include both \"error\" and \"result\"'\n            )\n\n        # Format and validate errors\n        elif \"error\" in response:\n            error = response.get(\"error\")\n            # Raise the error when the value is a string\n            if error is None or isinstance(error, str):\n                raise ValueError(error)\n\n            # Errors must include an integer code\n            code = error.get(\"code\")\n            if not isinstance(code, int):\n                _raise_bad_response_format(response, \"error['code'] must be an integer\")\n            elif code == METHOD_NOT_FOUND:\n                raise MethodUnavailable(error)\n\n            # Errors must include a message\n            if not isinstance(error.get(\"message\"), str):\n                _raise_bad_response_format(\n                    response, \"error['message'] must be a string\"\n                )\n\n            apply_error_formatters(error_formatters, response)\n\n            raise ValueError(error)\n\n        # Format and validate results\n        elif \"result\" in response:\n            # Null values for result should apply null_result_formatters\n            # Skip when result not present in the response (fallback to False)\n            if response.get(\"result\", False) in NULL_RESPONSES:\n                apply_null_result_formatters(null_result_formatters, response, params)\n            return response.get(\"result\")\n\n        # Response from eth_subscription includes response[\"params\"][\"result\"]\n        elif (\n            response.get(\"method\") == \"eth_subscription\"\n            and response.get(\"params\") is not None\n            and response[\"params\"].get(\"subscription\") is not None\n            and response[\"params\"].get(\"result\") is not None\n        ):\n            return {\n                \"subscription\": response[\"params\"][\"subscription\"],\n                \"result\": response[\"params\"][\"result\"],\n            }\n\n        # Any other response type raises BadResponseFormat\n        else:\n            _raise_bad_response_format(response)\n\n    def request_blocking(\n        self,\n        method: Union[RPCEndpoint, Callable[..., RPCEndpoint]],\n        params: Any,\n        error_formatters: Optional[Callable[..., Any]] = None,\n        null_result_formatters: Optional[Callable[..., Any]] = None,\n    ) -> Any:\n        \"\"\"\n        Make a synchronous request using the provider\n        \"\"\"\n        response = self._make_request(method, params)\n        return self.formatted_response(\n            response, params, error_formatters, null_result_formatters\n        )\n\n    async def coro_request(\n        self,\n        method: Union[RPCEndpoint, Callable[..., RPCEndpoint]],\n        params: Any,\n        error_formatters: Optional[Callable[..., Any]] = None,\n        null_result_formatters: Optional[Callable[..., Any]] = None,\n    ) -> Any:\n        \"\"\"\n        Coroutine for making a request using the provider\n        \"\"\"\n        response = await self._coro_make_request(method, params)\n        return self.formatted_response(\n            response, params, error_formatters, null_result_formatters\n        )\n\n    # -- persistent connection -- #\n\n    async def send(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        provider = cast(PersistentConnectionProvider, self._provider)\n        request_func = await provider.request_func(\n            cast(\"AsyncWeb3\", self.w3), cast(\"MiddlewareOnion\", self.middleware_onion)\n        )\n        self.logger.debug(\n            \"Making request to open socket connection - \"\n            f\"uri: {provider.endpoint_uri}, method: {method}\"\n        )\n        response = await request_func(method, params)\n        return await self._process_response(response)\n\n    def _persistent_message_stream(self) -> \"_AsyncPersistentMessageStream\":\n        return _AsyncPersistentMessageStream(self)\n\n    async def _get_next_message(self) -> Any:\n        return await self._message_stream().__anext__()\n\n    async def _message_stream(self) -> AsyncGenerator[RPCResponse, None]:\n        if not isinstance(self._provider, PersistentConnectionProvider):\n            raise TypeError(\n                \"Only providers that maintain an open, persistent connection \"\n                \"can listen to streams.\"\n            )\n\n        if self._provider._message_listener_task is None:\n            raise ProviderConnectionError(\n                \"No listener found for persistent connection.\"\n            )\n\n        while True:\n            # sleep(0) here seems to be the most efficient way to yield control\n            # back to the event loop while waiting for the response in the queue.\n            await asyncio.sleep(0)\n\n            response = self._request_processor.pop_raw_response(subscription=True)\n            if (\n                response is not None\n                and response.get(\"params\", {}).get(\"subscription\")\n                in self._request_processor.active_subscriptions\n            ):\n                # if response is an active subscription response, process it\n                yield await self._process_response(response)\n\n    async def _process_response(self, response: RPCResponse) -> RPCResponse:\n        provider = cast(PersistentConnectionProvider, self._provider)\n        request_info = self._request_processor.get_request_information_for_response(\n            response\n        )\n\n        if request_info is None:\n            self.logger.debug(\"No cache key found for response, returning raw response\")\n            return response\n        else:\n            if request_info.method == \"eth_subscribe\" and \"result\" in response.keys():\n                # if response for the initial eth_subscribe request, which returns the\n                # subscription id\n                subscription_id = response[\"result\"]\n                cache_key = generate_cache_key(subscription_id)\n                if cache_key not in self._request_processor._request_information_cache:\n                    # cache by subscription id in order to process each response for the\n                    # subscription as it comes in\n                    request_info.subscription_id = subscription_id\n                    provider.logger.debug(\n                        \"Caching eth_subscription info:\\n    \"\n                        f\"cache_key={cache_key},\\n    \"\n                        f\"request_info={request_info.__dict__}\"\n                    )\n                    self._request_processor._request_information_cache.cache(\n                        cache_key, request_info\n                    )\n\n            # pipe response back through middleware response processors\n            if len(request_info.middleware_response_processors) > 0:\n                response = pipe(response, *request_info.middleware_response_processors)\n\n            (\n                result_formatters,\n                error_formatters,\n                null_formatters,\n            ) = request_info.response_formatters\n            partly_formatted_response = self.formatted_response(\n                response,\n                request_info.params,\n                error_formatters,\n                null_formatters,\n            )\n            return apply_result_formatters(result_formatters, partly_formatted_response)\n\n\nclass _AsyncPersistentMessageStream:\n    \"\"\"\n    Async generator for pulling subscription responses from the request processor\n    subscription queue. This abstraction is necessary to define the `__aiter__()`\n    method required for use with \"async for\" loops.\n    \"\"\"\n\n    def __init__(self, manager: RequestManager, *args: Any, **kwargs: Any) -> None:\n        self.manager = manager\n        self.provider: PersistentConnectionProvider = cast(\n            PersistentConnectionProvider, manager._provider\n        )\n        super().__init__(*args, **kwargs)\n\n    def __aiter__(self) -> Self:\n        return self\n\n    async def __anext__(self) -> RPCResponse:\n        try:\n            return await self.manager._get_next_message()\n        except ConnectionClosedOK:\n            raise StopAsyncIteration\n\n# Path: web3/providers/persistent/persistent_connection.py\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n)\n\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n    )\n    from web3.manager import (  # noqa: F401\n        _AsyncPersistentMessageStream,\n    )\n\n\nclass PersistentConnection:\n    \"\"\"\n    A class that houses the public API for interacting with the persistent connection\n    via a `AsyncWeb3` instance instantiated with a `PersistentConnectionProvider` class.\n    \"\"\"\n\n    def __init__(self, w3: \"AsyncWeb3\"):\n        self._manager = w3.manager\n\n    # -- public methods -- #\n    @property\n    def subscriptions(self) -> Dict[str, Any]:\n        return self._manager._request_processor.active_subscriptions\n\n    async def send(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        return await self._manager.send(method, params)\n\n    async def recv(self) -> Any:\n        return await self._manager._get_next_message()\n\n    def process_subscriptions(self) -> \"_AsyncPersistentMessageStream\":\n        return self._manager._persistent_message_stream()\n\n# Path: web3/providers/persistent/websocket.py\nimport asyncio\nimport json\nimport logging\nimport os\nfrom typing import (\n    Any,\n    Dict,\n    Optional,\n    Union,\n)\n\nfrom eth_typing import (\n    URI,\n)\nfrom toolz import (\n    merge,\n)\nfrom websockets import (\n    WebSocketClientProtocol,\n)\nfrom websockets.client import (\n    connect,\n)\nfrom websockets.exceptions import (\n    WebSocketException,\n)\n\nfrom web3._utils.caching import (\n    async_handle_request_caching,\n)\nfrom web3.exceptions import (\n    ProviderConnectionError,\n    Web3ValidationError,\n)\nfrom web3.providers.persistent import (\n    PersistentConnectionProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nDEFAULT_PING_INTERVAL = 30  # 30 seconds\nDEFAULT_PING_TIMEOUT = 300  # 5 minutes\n\nVALID_WEBSOCKET_URI_PREFIXES = {\"ws://\", \"wss://\"}\nRESTRICTED_WEBSOCKET_KWARGS = {\"uri\", \"loop\"}\nDEFAULT_WEBSOCKET_KWARGS = {\n    # set how long to wait between pings from the server\n    \"ping_interval\": DEFAULT_PING_INTERVAL,\n    # set how long to wait without a pong response before closing the connection\n    \"ping_timeout\": DEFAULT_PING_TIMEOUT,\n}\n\n\ndef get_default_endpoint() -> URI:\n    return URI(os.environ.get(\"WEB3_WS_PROVIDER_URI\", \"ws://127.0.0.1:8546\"))\n\n\nclass WebSocketProvider(PersistentConnectionProvider):\n    logger = logging.getLogger(\"web3.providers.WebSocketProvider\")\n    is_async: bool = True\n\n    _max_connection_retries: int = 5\n    _ws: Optional[WebSocketClientProtocol] = None\n\n    def __init__(\n        self,\n        endpoint_uri: Optional[Union[URI, str]] = None,\n        websocket_kwargs: Optional[Dict[str, Any]] = None,\n        # `PersistentConnectionProvider` kwargs can be passed through\n        **kwargs: Any,\n    ) -> None:\n        self.endpoint_uri = URI(endpoint_uri)\n        if self.endpoint_uri is None:\n            self.endpoint_uri = get_default_endpoint()\n\n        if not any(\n            self.endpoint_uri.startswith(prefix)\n            for prefix in VALID_WEBSOCKET_URI_PREFIXES\n        ):\n            raise Web3ValidationError(\n                \"WebSocket endpoint uri must begin with 'ws://' or 'wss://': \"\n                f\"{self.endpoint_uri}\"\n            )\n\n        if websocket_kwargs is not None:\n            found_restricted_keys = set(websocket_kwargs).intersection(\n                RESTRICTED_WEBSOCKET_KWARGS\n            )\n            if found_restricted_keys:\n                raise Web3ValidationError(\n                    \"Found restricted keys for websocket_kwargs: \"\n                    f\"{found_restricted_keys}.\"\n                )\n\n        self.websocket_kwargs = merge(DEFAULT_WEBSOCKET_KWARGS, websocket_kwargs or {})\n\n        super().__init__(**kwargs)\n\n    def __str__(self) -> str:\n        return f\"WebSocket connection: {self.endpoint_uri}\"\n\n    async def is_connected(self, show_traceback: bool = False) -> bool:\n        if not self._ws:\n            return False\n\n        try:\n            await self._ws.pong()\n            return True\n\n        except WebSocketException as e:\n            if show_traceback:\n                raise ProviderConnectionError(\n                    f\"Error connecting to endpoint: '{self.endpoint_uri}'\"\n                ) from e\n            return False\n\n    async def connect(self) -> None:\n        _connection_attempts = 0\n        _backoff_rate_change = 1.75\n        _backoff_time = 1.75\n\n        while _connection_attempts != self._max_connection_retries:\n            try:\n                _connection_attempts += 1\n                self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)\n                self._message_listener_task = asyncio.create_task(\n                    self._message_listener()\n                )\n                break\n            except WebSocketException as e:\n                if _connection_attempts == self._max_connection_retries:\n                    raise ProviderConnectionError(\n                        f\"Could not connect to endpoint: {self.endpoint_uri}. \"\n                        f\"Retries exceeded max of {self._max_connection_retries}.\"\n                    ) from e\n                self.logger.info(\n                    f\"Could not connect to endpoint: {self.endpoint_uri}. Retrying in \"\n                    f\"{round(_backoff_time, 1)} seconds.\",\n                    exc_info=True,\n                )\n                await asyncio.sleep(_backoff_time)\n                _backoff_time *= _backoff_rate_change\n\n    async def disconnect(self) -> None:\n        if self._ws is not None and not self._ws.closed:\n            await self._ws.close()\n            self._ws = None\n            self.logger.debug(\n                f'Successfully disconnected from endpoint: \"{self.endpoint_uri}'\n            )\n\n        try:\n            self._message_listener_task.cancel()\n            await self._message_listener_task\n        except (asyncio.CancelledError, StopAsyncIteration):\n            pass\n        self._request_processor.clear_caches()\n\n    @async_handle_request_caching\n    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        request_data = self.encode_rpc_request(method, params)\n\n        if self._ws is None:\n            raise ProviderConnectionError(\n                \"Connection to websocket has not been initiated for the provider.\"\n            )\n\n        await asyncio.wait_for(\n            self._ws.send(request_data), timeout=self.request_timeout\n        )\n\n        current_request_id = json.loads(request_data)[\"id\"]\n        response = await self._get_response_for_request_id(current_request_id)\n\n        return response\n\n    async def _message_listener(self) -> None:\n        self.logger.info(\n            \"WebSocket listener background task started. Storing all messages in \"\n            \"appropriate request processor queues / caches to be processed.\"\n        )\n        while True:\n            # the use of sleep(0) seems to be the most efficient way to yield control\n            # back to the event loop to share the loop with other tasks.\n            await asyncio.sleep(0)\n\n            try:\n                async for raw_message in self._ws:\n                    await asyncio.sleep(0)\n\n                    response = json.loads(raw_message)\n                    subscription = response.get(\"method\") == \"eth_subscription\"\n                    await self._request_processor.cache_raw_response(\n                        response, subscription=subscription\n                    )\n            except Exception as e:\n                if not self.silence_listener_task_exceptions:\n                    loop = asyncio.get_event_loop()\n                    for task in asyncio.all_tasks(loop=loop):\n                        task.cancel()\n                    raise e\n\n                self.logger.error(\n                    \"Exception caught in listener, error logging and keeping \"\n                    \"listener background task alive.\"\n                    f\"\\n    error={e.__class__.__name__}: {e}\"\n                )\n\n# Path: web3/providers/persistent/__init__.py\nfrom .persistent import (\n    PersistentConnectionProvider,\n)\nfrom .persistent_connection import (\n    PersistentConnection,\n)\nfrom .request_processor import (\n    RequestProcessor,\n)\nfrom .async_ipc import (\n    AsyncIPCProvider,\n)\nfrom .websocket import (\n    WebSocketProvider,\n)\n\n# Path: web3/providers/async_base.py\nimport asyncio\nimport itertools\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Coroutine,\n    Optional,\n    Set,\n    Tuple,\n    cast,\n)\n\nfrom eth_utils import (\n    is_text,\n    to_bytes,\n    to_text,\n)\n\nfrom web3._utils.caching import (\n    async_handle_request_caching,\n)\nfrom web3._utils.encoding import (\n    FriendlyJsonSerde,\n    Web3JsonEncoder,\n)\nfrom web3.exceptions import (\n    ProviderConnectionError,\n)\nfrom web3.middleware import (\n    async_combine_middlewares,\n)\nfrom web3.middleware.base import (\n    Middleware,\n    MiddlewareOnion,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\nfrom web3.utils import (\n    SimpleCache,\n)\n\nif TYPE_CHECKING:\n    from websockets import (\n        WebSocketClientProtocol,\n    )\n\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        WebSocketProvider,\n    )\n    from web3.providers.persistent import (  # noqa: F401\n        RequestProcessor,\n    )\n\n\nCACHEABLE_REQUESTS = cast(\n    Set[RPCEndpoint],\n    (\n        \"eth_chainId\",\n        \"eth_getBlockByHash\",\n        \"eth_getBlockTransactionCountByHash\",\n        \"eth_getRawTransactionByHash\",\n        \"eth_getTransactionByBlockHashAndIndex\",\n        \"eth_getTransactionByHash\",\n        \"eth_getUncleByBlockHashAndIndex\",\n        \"eth_getUncleCountByBlockHash\",\n        \"net_version\",\n        \"web3_clientVersion\",\n    ),\n)\n\n\nclass AsyncBaseProvider:\n    _request_func_cache: Tuple[\n        Tuple[Middleware, ...], Callable[..., Coroutine[Any, Any, RPCResponse]]\n    ] = (None, None)\n\n    is_async = True\n    has_persistent_connection = False\n    global_ccip_read_enabled: bool = True\n    ccip_read_max_redirects: int = 4\n\n    # request caching\n    cache_allowed_requests: bool = False\n    cacheable_requests: Set[RPCEndpoint] = CACHEABLE_REQUESTS\n    _request_cache: SimpleCache\n    _request_cache_lock: asyncio.Lock = asyncio.Lock()\n\n    def __init__(self) -> None:\n        self._request_cache = SimpleCache(1000)\n\n    async def request_func(\n        self, async_w3: \"AsyncWeb3\", middleware_onion: MiddlewareOnion\n    ) -> Callable[..., Coroutine[Any, Any, RPCResponse]]:\n        middlewares: Tuple[Middleware, ...] = middleware_onion.as_tuple_of_middlewares()\n\n        cache_key = self._request_func_cache[0]\n        if cache_key != middlewares:\n            self._request_func_cache = (\n                middlewares,\n                await async_combine_middlewares(\n                    middlewares=middlewares,\n                    async_w3=async_w3,\n                    provider_request_fn=self.make_request,\n                ),\n            )\n        return self._request_func_cache[-1]\n\n    @async_handle_request_caching\n    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        raise NotImplementedError(\"Providers must implement this method\")\n\n    async def is_connected(self, show_traceback: bool = False) -> bool:\n        raise NotImplementedError(\"Providers must implement this method\")\n\n    # -- persistent connection providers -- #\n\n    _request_processor: \"RequestProcessor\"\n    _message_listener_task: \"asyncio.Task[None]\"\n    _listen_event: \"asyncio.Event\"\n\n    async def connect(self) -> None:\n        raise NotImplementedError(\n            \"Persistent connection providers must implement this method\"\n        )\n\n    async def disconnect(self) -> None:\n        raise NotImplementedError(\n            \"Persistent connection providers must implement this method\"\n        )\n\n    # WebSocket typing\n    _ws: \"WebSocketClientProtocol\"\n\n    # IPC typing\n    _reader: Optional[asyncio.StreamReader]\n    _writer: Optional[asyncio.StreamWriter]\n\n\nclass AsyncJSONBaseProvider(AsyncBaseProvider):\n    def __init__(self) -> None:\n        super().__init__()\n        self.request_counter = itertools.count()\n\n    def encode_rpc_request(self, method: RPCEndpoint, params: Any) -> bytes:\n        request_id = next(self.request_counter)\n        rpc_dict = {\n            \"jsonrpc\": \"2.0\",\n            \"method\": method,\n            \"params\": params or [],\n            \"id\": request_id,\n        }\n        encoded = FriendlyJsonSerde().json_encode(rpc_dict, cls=Web3JsonEncoder)\n        return to_bytes(text=encoded)\n\n    def decode_rpc_response(self, raw_response: bytes) -> RPCResponse:\n        text_response = str(\n            to_text(raw_response) if not is_text(raw_response) else raw_response\n        )\n        return cast(RPCResponse, FriendlyJsonSerde().json_decode(text_response))\n\n    async def is_connected(self, show_traceback: bool = False) -> bool:\n        try:\n            response = await self.make_request(RPCEndpoint(\"web3_clientVersion\"), [])\n        except (OSError, ProviderConnectionError) as e:\n            if show_traceback:\n                raise ProviderConnectionError(\n                    f\"Problem connecting to provider with error: {type(e)}: {e}\"\n                )\n            return False\n\n        if \"error\" in response:\n            if show_traceback:\n                raise ProviderConnectionError(\n                    f\"Error received from provider: {response}\"\n                )\n            return False\n\n        if response.get(\"jsonrpc\") == \"2.0\":\n            return True\n        else:\n            if show_traceback:\n                raise ProviderConnectionError(f\"Bad jsonrpc version: {response}\")\n            return False\n\n# Path: web3/providers/auto.py\nimport os\nfrom typing import (\n    Any,\n    Callable,\n    Dict,\n    Optional,\n    Sequence,\n    Tuple,\n    Type,\n    Union,\n)\nfrom urllib.parse import (\n    urlparse,\n)\n\nfrom eth_typing import (\n    URI,\n)\n\nfrom web3.exceptions import (\n    CannotHandleRequest,\n)\nfrom web3.providers import (\n    BaseProvider,\n    HTTPProvider,\n    IPCProvider,\n    LegacyWebSocketProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nHTTP_SCHEMES = {\"http\", \"https\"}\nWS_SCHEMES = {\"ws\", \"wss\"}\n\n\ndef load_provider_from_environment() -> BaseProvider:\n    uri_string = URI(os.environ.get(\"WEB3_PROVIDER_URI\", \"\"))\n    if not uri_string:\n        return None\n\n    return load_provider_from_uri(uri_string)\n\n\ndef load_provider_from_uri(\n    uri_string: URI, headers: Optional[Dict[str, Tuple[str, str]]] = None\n) -> BaseProvider:\n    uri = urlparse(uri_string)\n    if uri.scheme == \"file\":\n        return IPCProvider(uri.path)\n    elif uri.scheme in HTTP_SCHEMES:\n        return HTTPProvider(uri_string, headers)\n    elif uri.scheme in WS_SCHEMES:\n        return LegacyWebSocketProvider(uri_string)\n    else:\n        raise NotImplementedError(\n            \"Web3 does not know how to connect to scheme \"\n            f\"{uri.scheme!r} in {uri_string!r}\"\n        )\n\n\nclass AutoProvider(BaseProvider):\n    default_providers = (\n        load_provider_from_environment,\n        IPCProvider,\n        HTTPProvider,\n        LegacyWebSocketProvider,\n    )\n    _active_provider = None\n\n    def __init__(\n        self,\n        potential_providers: Optional[\n            Sequence[Union[Callable[..., BaseProvider], Type[BaseProvider]]]\n        ] = None,\n    ) -> None:\n        \"\"\"\n        :param iterable potential_providers: ordered series of provider classes\n            to attempt with\n\n        AutoProvider will initialize each potential provider (without arguments),\n        in an attempt to find an active node. The list will default to\n        :attribute:`default_providers`.\n        \"\"\"\n        if potential_providers:\n            self._potential_providers = potential_providers\n        else:\n            self._potential_providers = self.default_providers\n\n    def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        try:\n            return self._proxy_request(method, params)\n        except OSError:\n            return self._proxy_request(method, params, use_cache=False)\n\n    def is_connected(self, show_traceback: bool = False) -> bool:\n        provider = self._get_active_provider(use_cache=True)\n        return provider is not None and provider.is_connected(show_traceback)\n\n    def _proxy_request(\n        self, method: RPCEndpoint, params: Any, use_cache: bool = True\n    ) -> RPCResponse:\n        provider = self._get_active_provider(use_cache)\n        if provider is None:\n            raise CannotHandleRequest(\n                \"Could not discover provider while making request: \"\n                f\"method:{method}\\nparams:{params}\\n\"\n            )\n\n        return provider.make_request(method, params)\n\n    def _get_active_provider(self, use_cache: bool) -> Optional[BaseProvider]:\n        if use_cache and self._active_provider is not None:\n            return self._active_provider\n\n        for Provider in self._potential_providers:\n            provider = Provider()\n            if provider is not None and provider.is_connected():\n                self._active_provider = provider\n                return provider\n\n        return None\n\n# Path: web3/providers/legacy_websocket.py\nimport asyncio\nimport json\nimport logging\nimport os\nfrom threading import (\n    Thread,\n)\nfrom types import (\n    TracebackType,\n)\nfrom typing import (\n    Any,\n    Optional,\n    Type,\n    Union,\n)\n\nfrom eth_typing import (\n    URI,\n)\nfrom websockets.client import (\n    connect,\n)\nfrom websockets.legacy.client import (\n    WebSocketClientProtocol,\n)\n\nfrom web3.exceptions import (\n    Web3ValidationError,\n)\nfrom web3.providers.base import (\n    JSONBaseProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nRESTRICTED_WEBSOCKET_KWARGS = {\"uri\", \"loop\"}\nDEFAULT_WEBSOCKET_TIMEOUT = 30\n\n\ndef _start_event_loop(loop: asyncio.AbstractEventLoop) -> None:\n    asyncio.set_event_loop(loop)\n    loop.run_forever()\n    loop.close()\n\n\ndef _get_threaded_loop() -> asyncio.AbstractEventLoop:\n    new_loop = asyncio.new_event_loop()\n    thread_loop = Thread(target=_start_event_loop, args=(new_loop,), daemon=True)\n    thread_loop.start()\n    return new_loop\n\n\ndef get_default_endpoint() -> URI:\n    return URI(os.environ.get(\"WEB3_WS_PROVIDER_URI\", \"ws://127.0.0.1:8546\"))\n\n\nclass PersistentWebSocket:\n    def __init__(self, endpoint_uri: URI, websocket_kwargs: Any) -> None:\n        self.ws: Optional[WebSocketClientProtocol] = None\n        self.endpoint_uri = endpoint_uri\n        self.websocket_kwargs = websocket_kwargs\n\n    async def __aenter__(self) -> WebSocketClientProtocol:\n        if self.ws is None:\n            self.ws = await connect(uri=self.endpoint_uri, **self.websocket_kwargs)\n        return self.ws\n\n    async def __aexit__(\n        self,\n        exc_type: Type[BaseException],\n        exc_val: BaseException,\n        exc_tb: TracebackType,\n    ) -> None:\n        if exc_val is not None:\n            try:\n                await self.ws.close()\n            except Exception:\n                pass\n            self.ws = None\n\n\nclass LegacyWebSocketProvider(JSONBaseProvider):\n    logger = logging.getLogger(\"web3.providers.WebSocketProvider\")\n    _loop = None\n\n    def __init__(\n        self,\n        endpoint_uri: Optional[Union[URI, str]] = None,\n        websocket_kwargs: Optional[Any] = None,\n...\n", "needle_token_start": 2195, "needle_token_end": 2635, "code_context_ntokens": 16115, "output": ["```python\nasync def _get_response_for_request_id(\n        self, request_id: RPCId, timeout: Optional[float] = None\n    ) -> RPCResponse:\n        if timeout is None:\n            timeout = self.request_timeout\n\n        async def _match_response_id_to_request_id() -> RPCResponse:\n            request_cache_key = generate_cache_key(request_id)\n\n            while True:\n                # sleep(0) here seems to be the most efficient way to yield control\n                # back to the event loop while waiting for the response to be in the\n                # queue.\n                await asyncio.sleep(0)\n\n                if request_cache_key in self._request_processor._request_response_cache:\n                    self.logger.debug(\n                        f\"Popping response for id {request_id} from cache.\"\n                    )\n                    popped_response = self._request_processor.pop_raw_response(\n                        cache_key=request_cache_key,\n                    )\n                    return popped_response\n\n        try:\n            # Add the request timeout around the while loop that checks the request\n            # cache and tried to recv(). If the request is neither in the cache, nor\n            # received within the request_timeout, raise ``TimeExhausted``.\n            return await asyncio.wait_for(_match_response_id_to_request_id(), timeout)\n        except asyncio.TimeoutError:\n            raise TimeExhausted(\n                f\"Timed out waiting for response with request id `{request_id}` after \"\n                f\"{self.request_timeout} second(s). This may be due to the provider \"\n                \"not returning a response with the same id that was sent in the \"\n                \"request or an exception raised during the request was caught and \"\n                \"allowed to continue.\"\n            )\n```"]}
{"repo": "ethereum/web3.py", "name": "get_default_middlewares", "language": "python", "path": "web3/manager.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To provide a list of default middleware components that are essential for processing and managing blockchain requests.\n2. **Input**: None, as this method does not require any parameters to be passed.\n3. **Output**: A list of tuples, where each tuple contains a middleware component and its corresponding identifier as a string.\n4. **Procedure**: The method statically defines and returns a list of middleware components that are commonly used to handle various aspects of blockchain interactions, such as gas price strategy, name resolution, data formatting, input validation, and gas estimation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: web3/method.py\nimport functools\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Dict,\n    Generic,\n    List,\n    Optional,\n    Sequence,\n    Tuple,\n    Type,\n    TypeVar,\n    Union,\n)\nimport warnings\n\nfrom eth_utils.curried import (\n    to_tuple,\n)\nfrom eth_utils.toolz import (\n    pipe,\n)\n\nfrom web3._utils.method_formatters import (\n    get_error_formatters,\n    get_null_result_formatters,\n    get_request_formatters,\n    get_result_formatters,\n)\nfrom web3._utils.rpc_abi import (\n    RPC,\n)\nfrom web3.exceptions import (\n    Web3ValidationError,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    TReturn,\n)\n\nif TYPE_CHECKING:\n    from web3 import Web3  # noqa: F401\n    from web3.module import Module  # noqa: F401\n\n\nMunger = Callable[..., Any]\n\n\n@to_tuple\ndef _apply_request_formatters(\n    params: Any, request_formatters: Dict[RPCEndpoint, Callable[..., TReturn]]\n) -> Tuple[Any, ...]:\n    if request_formatters:\n        formatted_params = pipe(params, request_formatters)\n        return formatted_params\n    return params\n\n\ndef _set_mungers(\n    mungers: Optional[Sequence[Munger]], is_property: bool\n) -> Sequence[Any]:\n    if is_property and mungers:\n        raise Web3ValidationError(\"Mungers cannot be used with a property.\")\n\n    return (\n        mungers\n        if mungers\n        else [default_munger] if is_property else [default_root_munger]\n    )\n\n\ndef default_munger(_module: \"Module\", *args: Any, **kwargs: Any) -> Tuple[()]:\n    if args or kwargs:\n        raise Web3ValidationError(\"Parameters cannot be passed to a property.\")\n    return ()\n\n\ndef default_root_munger(_module: \"Module\", *args: Any) -> List[Any]:\n    return [*args]\n\n\nTFunc = TypeVar(\"TFunc\", bound=Callable[..., Any])\n\n\nclass Method(Generic[TFunc]):\n    \"\"\"Method object for web3 module methods\n\n    Calls to the Method go through these steps:\n\n    1. input munging - includes normalization, parameter checking, early parameter\n    formatting. Any processing on the input parameters that need to happen before\n    json_rpc method string selection occurs.\n\n            A note about mungers: The first (root) munger should reflect the desired\n        api function arguments. In other words, if the api function wants to\n        behave as: `get_balance(account, block_identifier=None)`, the root munger\n        should accept these same arguments, with the addition of the module as\n        the first argument e.g.:\n\n        ```\n        def get_balance_root_munger(module, account, block_identifier=None):\n            if block_identifier is None:\n                block_identifier = DEFAULT_BLOCK\n            return module, [account, block_identifier]\n        ```\n\n        all mungers should return an argument list.\n\n        if no munger is provided, a default munger expecting no method arguments\n        will be used.\n\n    2. method selection - The json_rpc_method argument can be method string or a\n    function that returns a method string. If a callable is provided the processed\n    method inputs are passed to the method selection function, and the returned\n    method string is used.\n\n    3. request and response formatters are set - formatters are retrieved\n    using the json rpc method string.\n\n    4. After the parameter processing from steps 1-3 the request is made using\n    the calling function returned by the module attribute ``retrieve_caller_fn``\n    and the response formatters are applied to the output.\n    \"\"\"\n\n    def __init__(\n        self,\n        json_rpc_method: Optional[RPCEndpoint] = None,\n        mungers: Optional[Sequence[Munger]] = None,\n        request_formatters: Optional[Callable[..., TReturn]] = None,\n        result_formatters: Optional[Callable[..., TReturn]] = None,\n        null_result_formatters: Optional[Callable[..., TReturn]] = None,\n        method_choice_depends_on_args: Optional[Callable[..., RPCEndpoint]] = None,\n        is_property: bool = False,\n    ):\n        self.json_rpc_method = json_rpc_method\n        self.mungers = _set_mungers(mungers, is_property)\n        self.request_formatters = request_formatters or get_request_formatters\n        self.result_formatters = result_formatters or get_result_formatters\n        self.null_result_formatters = (\n            null_result_formatters or get_null_result_formatters\n        )\n        self.method_choice_depends_on_args = method_choice_depends_on_args\n        self.is_property = is_property\n\n    def __get__(\n...\n# Path: web3/module.py\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Coroutine,\n    Dict,\n    Optional,\n    TypeVar,\n    Union,\n    cast,\n)\n\nfrom eth_abi.codec import (\n    ABICodec,\n)\nfrom eth_utils.toolz import (\n    curry,\n    pipe,\n)\n\nfrom web3._utils.filters import (\n    AsyncLogFilter,\n    LogFilter,\n    _UseExistingFilter,\n)\nfrom web3.method import (\n    Method,\n)\nfrom web3.providers.persistent import (\n    PersistentConnectionProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\n\n@curry\ndef apply_result_formatters(\n    result_formatters: Callable[..., Any], result: RPCResponse\n) -> RPCResponse:\n    if result_formatters:\n        formatted_result = pipe(result, result_formatters)\n        return formatted_result\n    else:\n        return result\n\n\nTReturn = TypeVar(\"TReturn\")\n\n\n@curry\ndef retrieve_blocking_method_call_fn(\n    w3: \"Web3\", module: \"Module\", method: Method[Callable[..., TReturn]]\n) -> Callable[..., Union[TReturn, LogFilter]]:\n    def caller(*args: Any, **kwargs: Any) -> Union[TReturn, LogFilter]:\n        try:\n            (method_str, params), response_formatters = method.process_params(\n                module, *args, **kwargs\n            )\n        except _UseExistingFilter as err:\n            return LogFilter(eth_module=module, filter_id=err.filter_id)\n\n        (\n            result_formatters,\n            error_formatters,\n            null_result_formatters,\n        ) = response_formatters\n        result = w3.manager.request_blocking(\n            method_str, params, error_formatters, null_result_formatters\n        )\n        return apply_result_formatters(result_formatters, result)\n\n    return caller\n\n\n@curry\ndef retrieve_async_method_call_fn(\n    async_w3: \"AsyncWeb3\", module: \"Module\", method: Method[Callable[..., Any]]\n) -> Callable[..., Coroutine[Any, Any, Optional[Union[RPCResponse, AsyncLogFilter]]]]:\n    async def caller(*args: Any, **kwargs: Any) -> Union[RPCResponse, AsyncLogFilter]:\n        try:\n            (method_str, params), response_formatters = method.process_params(\n                module, *args, **kwargs\n            )\n        except _UseExistingFilter as err:\n            return AsyncLogFilter(eth_module=module, filter_id=err.filter_id)\n\n        if isinstance(async_w3.provider, PersistentConnectionProvider):\n            # TODO: The typing does not seem to be correct for response_formatters.\n            #   For now, keep the expected typing but ignore it here.\n            provider = async_w3.provider\n            cache_key = provider._request_processor.cache_request_information(\n                cast(RPCEndpoint, method_str), params, response_formatters  # type: ignore # noqa: E501\n            )\n            try:\n                method_str = cast(RPCEndpoint, method_str)\n                return await async_w3.manager.send(method_str, params)\n            except Exception as e:\n                if (\n                    cache_key is not None\n                    and cache_key\n                    in provider._request_processor._request_information_cache\n                ):\n                    provider._request_processor.pop_cached_request_information(\n                        cache_key\n                    )\n                raise e\n        else:\n            (\n                result_formatters,\n                error_formatters,\n                null_result_formatters,\n            ) = response_formatters\n\n            result = await async_w3.manager.coro_request(\n                method_str, params, error_formatters, null_result_formatters\n            )\n            return apply_result_formatters(result_formatters, result)\n\n    return caller\n\n\n#  Module should no longer have access to the full web3 api.\n#  Only the calling functions need access to the request methods.\n#  Any \"re-entrant\" shenanigans can go in the middlewares, which do\n#  have web3 access.\nclass Module:\n    is_async = False\n\n    def __init__(self, w3: Union[\"AsyncWeb3\", \"Web3\"]) -> None:\n        if self.is_async:\n            self.retrieve_caller_fn = retrieve_async_method_call_fn(w3, self)\n        else:\n            self.retrieve_caller_fn = retrieve_blocking_method_call_fn(w3, self)\n        self.w3 = w3\n\n    @property\n    def codec(self) -> ABICodec:\n        # use codec set on the Web3 instance\n        return self.w3.codec\n\n    def attach_methods(\n        self,\n        methods: Dict[str, Method[Callable[..., Any]]],\n    ) -> None:\n        for method_name, method_class in methods.items():\n            klass = (\n                method_class.__get__(obj=self)()\n                if method_class.is_property\n                else method_class.__get__(obj=self)\n            )\n            setattr(self, method_name, klass)\n\n# Path: web3/manager.py\nimport asyncio\nimport logging\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    AsyncGenerator,\n    Callable,\n    List,\n    Optional,\n    Sequence,\n    Tuple,\n    Union,\n    cast,\n)\n\nfrom eth_utils.toolz import (\n    pipe,\n)\nfrom hexbytes import (\n    HexBytes,\n)\nfrom websockets.exceptions import (\n    ConnectionClosedOK,\n)\n\nfrom web3._utils.caching import (\n    generate_cache_key,\n)\nfrom web3._utils.compat import (\n    Self,\n)\nfrom web3.datastructures import (\n    NamedElementOnion,\n)\nfrom web3.exceptions import (\n    BadResponseFormat,\n    MethodUnavailable,\n    ProviderConnectionError,\n)\nfrom web3.middleware import (\n    AttributeDictMiddleware,\n    BufferedGasEstimateMiddleware,\n    ENSNameToAddressMiddleware,\n    GasPriceStrategyMiddleware,\n    ValidationMiddleware,\n)\nfrom web3.middleware.base import (\n    Middleware,\n    MiddlewareOnion,\n)\nfrom web3.module import (\n    apply_result_formatters,\n)\nfrom web3.providers import (\n    AutoProvider,\n    PersistentConnectionProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n    from web3.middleware.base import (  # noqa: F401\n        Web3Middleware,\n    )\n    from web3.providers import (  # noqa: F401\n        AsyncBaseProvider,\n        BaseProvider,\n    )\n    from web3.providers.persistent.request_processor import (  # noqa: F401\n        RequestProcessor,\n    )\n\n\nNULL_RESPONSES = [None, HexBytes(\"0x\"), \"0x\"]\nMETHOD_NOT_FOUND = -32601\n\n\ndef _raise_bad_response_format(response: RPCResponse, error: str = \"\") -> None:\n    message = \"The response was in an unexpected format and unable to be parsed.\"\n    raw_response = f\"The raw response is: {response}\"\n\n    if error is not None and error != \"\":\n        message = f\"{message} {error}. {raw_response}\"\n    else:\n        message = f\"{message} {raw_response}\"\n\n    raise BadResponseFormat(message)\n\n\ndef apply_error_formatters(\n    error_formatters: Callable[..., Any],\n    response: RPCResponse,\n) -> RPCResponse:\n    if error_formatters:\n        formatted_resp = pipe(response, error_formatters)\n        return formatted_resp\n    else:\n        return response\n\n\ndef apply_null_result_formatters(\n    null_result_formatters: Callable[..., Any],\n    response: RPCResponse,\n    params: Optional[Any] = None,\n) -> RPCResponse:\n    if null_result_formatters:\n        formatted_resp = pipe(params, null_result_formatters)\n        return formatted_resp\n    else:\n        return response\n\n\nclass RequestManager:\n    logger = logging.getLogger(\"web3.manager.RequestManager\")\n\n    middleware_onion: Union[\"MiddlewareOnion\", NamedElementOnion[None, None]]\n\n    def __init__(\n        self,\n        w3: Union[\"AsyncWeb3\", \"Web3\"],\n        provider: Optional[Union[\"BaseProvider\", \"AsyncBaseProvider\"]] = None,\n        middlewares: Optional[Sequence[Tuple[Middleware, str]]] = None,\n    ) -> None:\n        self.w3 = w3\n\n        if provider is None:\n            self.provider = AutoProvider()\n        else:\n            self.provider = provider\n\n        if middlewares is None:\n            middlewares = self.get_default_middlewares()\n\n        self.middleware_onion = NamedElementOnion(middlewares)\n\n        if isinstance(provider, PersistentConnectionProvider):\n            # set up the request processor to be able to properly process ordered\n            # responses from the persistent connection as FIFO\n            provider = cast(PersistentConnectionProvider, self.provider)\n            self._request_processor: RequestProcessor = provider._request_processor\n\n    w3: Union[\"AsyncWeb3\", \"Web3\"] = None\n    _provider = None\n\n    @property\n    def provider(self) -> Union[\"BaseProvider\", \"AsyncBaseProvider\"]:\n        return self._provider\n\n    @provider.setter\n    def provider(self, provider: Union[\"BaseProvider\", \"AsyncBaseProvider\"]) -> None:\n        self._provider = provider\n\n    @staticmethod\n    \ndef get_default_middlewares() -> List[Tuple[Middleware, str]]:\n        \"\"\"\n        List the default middlewares for the request manager.\n        Documentation should remain in sync with these defaults.\n        \"\"\"\n        return [\n            (GasPriceStrategyMiddleware, \"gas_price_strategy\"),\n            (ENSNameToAddressMiddleware, \"ens_name_to_address\"),\n            (AttributeDictMiddleware, \"attrdict\"),\n            (ValidationMiddleware, \"validation\"),\n            (BufferedGasEstimateMiddleware, \"gas_estimate\"),\n        ]\n\n    #\n    # Provider requests and response\n    #\n    def _make_request(\n        self, method: Union[RPCEndpoint, Callable[..., RPCEndpoint]], params: Any\n    ) -> RPCResponse:\n        provider = cast(\"BaseProvider\", self.provider)\n        request_func = provider.request_func(\n            cast(\"Web3\", self.w3), cast(\"MiddlewareOnion\", self.middleware_onion)\n        )\n        self.logger.debug(f\"Making request. Method: {method}\")\n        return request_func(method, params)\n\n    async def _coro_make_request(\n        self, method: Union[RPCEndpoint, Callable[..., RPCEndpoint]], params: Any\n    ) -> RPCResponse:\n        provider = cast(\"AsyncBaseProvider\", self.provider)\n        request_func = await provider.request_func(\n            cast(\"AsyncWeb3\", self.w3), cast(\"MiddlewareOnion\", self.middleware_onion)\n        )\n        self.logger.debug(f\"Making request. Method: {method}\")\n        return await request_func(method, params)\n\n    #\n    # formatted_response parses and validates JSON-RPC responses for expected\n    # properties (result or an error) with the expected types.\n    #\n    # Required properties are not strictly enforced to further determine which\n    # exception to raise for specific cases.\n    #\n    # See also: https://www.jsonrpc.org/specification\n    #\n    @staticmethod\n    def formatted_response(\n        response: RPCResponse,\n        params: Any,\n        error_formatters: Optional[Callable[..., Any]] = None,\n        null_result_formatters: Optional[Callable[..., Any]] = None,\n    ) -> Any:\n        # jsonrpc is not enforced (as per the spec) but if present, it must be 2.0\n        if \"jsonrpc\" in response and response[\"jsonrpc\"] != \"2.0\":\n            _raise_bad_response_format(\n                response, 'The \"jsonrpc\" field must be present with a value of \"2.0\"'\n            )\n\n        # id is not enforced (as per the spec) but if present, it must be a\n        # string or integer\n        # TODO: v7 - enforce id per the spec\n        if \"id\" in response:\n            response_id = response[\"id\"]\n            # id is always None for errors\n            if response_id is None and \"error\" not in response:\n                _raise_bad_response_format(\n                    response, '\"id\" must be None when an error is present'\n                )\n            elif not isinstance(response_id, (str, int, type(None))):\n                _raise_bad_response_format(response, '\"id\" must be a string or integer')\n\n        # Response may not include both \"error\" and \"result\"\n        if \"error\" in response and \"result\" in response:\n            _raise_bad_response_format(\n                response, 'Response cannot include both \"error\" and \"result\"'\n            )\n\n        # Format and validate errors\n        elif \"error\" in response:\n            error = response.get(\"error\")\n            # Raise the error when the value is a string\n            if error is None or isinstance(error, str):\n                raise ValueError(error)\n\n            # Errors must include an integer code\n            code = error.get(\"code\")\n            if not isinstance(code, int):\n                _raise_bad_response_format(response, \"error['code'] must be an integer\")\n            elif code == METHOD_NOT_FOUND:\n                raise MethodUnavailable(error)\n\n            # Errors must include a message\n            if not isinstance(error.get(\"message\"), str):\n                _raise_bad_response_format(\n                    response, \"error['message'] must be a string\"\n                )\n\n            apply_error_formatters(error_formatters, response)\n\n            raise ValueError(error)\n\n        # Format and validate results\n        elif \"result\" in response:\n            # Null values for result should apply null_result_formatters\n            # Skip when result not present in the response (fallback to False)\n            if response.get(\"result\", False) in NULL_RESPONSES:\n                apply_null_result_formatters(null_result_formatters, response, params)\n            return response.get(\"result\")\n\n        # Response from eth_subscription includes response[\"params\"][\"result\"]\n        elif (\n            response.get(\"method\") == \"eth_subscription\"\n            and response.get(\"params\") is not None\n            and response[\"params\"].get(\"subscription\") is not None\n            and response[\"params\"].get(\"result\") is not None\n        ):\n            return {\n                \"subscription\": response[\"params\"][\"subscription\"],\n                \"result\": response[\"params\"][\"result\"],\n            }\n\n        # Any other response type raises BadResponseFormat\n        else:\n            _raise_bad_response_format(response)\n\n    def request_blocking(\n        self,\n        method: Union[RPCEndpoint, Callable[..., RPCEndpoint]],\n        params: Any,\n        error_formatters: Optional[Callable[..., Any]] = None,\n        null_result_formatters: Optional[Callable[..., Any]] = None,\n    ) -> Any:\n        \"\"\"\n        Make a synchronous request using the provider\n        \"\"\"\n        response = self._make_request(method, params)\n        return self.formatted_response(\n            response, params, error_formatters, null_result_formatters\n        )\n\n    async def coro_request(\n        self,\n        method: Union[RPCEndpoint, Callable[..., RPCEndpoint]],\n        params: Any,\n        error_formatters: Optional[Callable[..., Any]] = None,\n        null_result_formatters: Optional[Callable[..., Any]] = None,\n    ) -> Any:\n        \"\"\"\n        Coroutine for making a request using the provider\n        \"\"\"\n        response = await self._coro_make_request(method, params)\n        return self.formatted_response(\n            response, params, error_formatters, null_result_formatters\n        )\n\n    # -- persistent connection -- #\n\n    async def send(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        provider = cast(PersistentConnectionProvider, self._provider)\n        request_func = await provider.request_func(\n            cast(\"AsyncWeb3\", self.w3), cast(\"MiddlewareOnion\", self.middleware_onion)\n        )\n        self.logger.debug(\n            \"Making request to open socket connection - \"\n            f\"uri: {provider.endpoint_uri}, method: {method}\"\n        )\n        response = await request_func(method, params)\n        return await self._process_response(response)\n\n    def _persistent_message_stream(self) -> \"_AsyncPersistentMessageStream\":\n        return _AsyncPersistentMessageStream(self)\n\n    async def _get_next_message(self) -> Any:\n        return await self._message_stream().__anext__()\n\n    async def _message_stream(self) -> AsyncGenerator[RPCResponse, None]:\n        if not isinstance(self._provider, PersistentConnectionProvider):\n            raise TypeError(\n                \"Only providers that maintain an open, persistent connection \"\n                \"can listen to streams.\"\n            )\n\n        if self._provider._message_listener_task is None:\n            raise ProviderConnectionError(\n                \"No listener found for persistent connection.\"\n            )\n\n        while True:\n            # sleep(0) here seems to be the most efficient way to yield control\n            # back to the event loop while waiting for the response in the queue.\n            await asyncio.sleep(0)\n\n            response = self._request_processor.pop_raw_response(subscription=True)\n            if (\n                response is not None\n                and response.get(\"params\", {}).get(\"subscription\")\n                in self._request_processor.active_subscriptions\n            ):\n                # if response is an active subscription response, process it\n                yield await self._process_response(response)\n\n    async def _process_response(self, response: RPCResponse) -> RPCResponse:\n        provider = cast(PersistentConnectionProvider, self._provider)\n        request_info = self._request_processor.get_request_information_for_response(\n            response\n        )\n\n        if request_info is None:\n            self.logger.debug(\"No cache key found for response, returning raw response\")\n            return response\n        else:\n            if request_info.method == \"eth_subscribe\" and \"result\" in response.keys():\n                # if response for the initial eth_subscribe request, which returns the\n                # subscription id\n                subscription_id = response[\"result\"]\n                cache_key = generate_cache_key(subscription_id)\n                if cache_key not in self._request_processor._request_information_cache:\n                    # cache by subscription id in order to process each response for the\n                    # subscription as it comes in\n                    request_info.subscription_id = subscription_id\n                    provider.logger.debug(\n                        \"Caching eth_subscription info:\\n    \"\n                        f\"cache_key={cache_key},\\n    \"\n                        f\"request_info={request_info.__dict__}\"\n                    )\n                    self._request_processor._request_information_cache.cache(\n                        cache_key, request_info\n                    )\n\n            # pipe response back through middleware response processors\n            if len(request_info.middleware_response_processors) > 0:\n                response = pipe(response, *request_info.middleware_response_processors)\n\n            (\n                result_formatters,\n                error_formatters,\n                null_formatters,\n            ) = request_info.response_formatters\n            partly_formatted_response = self.formatted_response(\n                response,\n                request_info.params,\n                error_formatters,\n                null_formatters,\n            )\n            return apply_result_formatters(result_formatters, partly_formatted_response)\n\n\nclass _AsyncPersistentMessageStream:\n    \"\"\"\n    Async generator for pulling subscription responses from the request processor\n    subscription queue. This abstraction is necessary to define the `__aiter__()`\n    method required for use with \"async for\" loops.\n    \"\"\"\n\n    def __init__(self, manager: RequestManager, *args: Any, **kwargs: Any) -> None:\n        self.manager = manager\n        self.provider: PersistentConnectionProvider = cast(\n            PersistentConnectionProvider, manager._provider\n        )\n        super().__init__(*args, **kwargs)\n\n    def __aiter__(self) -> Self:\n        return self\n\n    async def __anext__(self) -> RPCResponse:\n        try:\n            return await self.manager._get_next_message()\n        except ConnectionClosedOK:\n            raise StopAsyncIteration\n\n# Path: web3/providers/persistent/persistent_connection.py\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n)\n\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n    )\n    from web3.manager import (  # noqa: F401\n        _AsyncPersistentMessageStream,\n    )\n\n\nclass PersistentConnection:\n    \"\"\"\n    A class that houses the public API for interacting with the persistent connection\n    via a `AsyncWeb3` instance instantiated with a `PersistentConnectionProvider` class.\n    \"\"\"\n\n    def __init__(self, w3: \"AsyncWeb3\"):\n        self._manager = w3.manager\n\n    # -- public methods -- #\n    @property\n    def subscriptions(self) -> Dict[str, Any]:\n        return self._manager._request_processor.active_subscriptions\n\n    async def send(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        return await self._manager.send(method, params)\n\n    async def recv(self) -> Any:\n        return await self._manager._get_next_message()\n\n    def process_subscriptions(self) -> \"_AsyncPersistentMessageStream\":\n        return self._manager._persistent_message_stream()\n\n# Path: web3/providers/persistent/websocket.py\nimport asyncio\nimport json\nimport logging\nimport os\nfrom typing import (\n    Any,\n    Dict,\n    Optional,\n    Union,\n)\n\nfrom eth_typing import (\n    URI,\n)\nfrom toolz import (\n    merge,\n)\nfrom websockets import (\n    WebSocketClientProtocol,\n)\nfrom websockets.client import (\n    connect,\n)\nfrom websockets.exceptions import (\n    WebSocketException,\n)\n\nfrom web3._utils.caching import (\n    async_handle_request_caching,\n)\nfrom web3.exceptions import (\n    ProviderConnectionError,\n    Web3ValidationError,\n)\nfrom web3.providers.persistent import (\n    PersistentConnectionProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nDEFAULT_PING_INTERVAL = 30  # 30 seconds\nDEFAULT_PING_TIMEOUT = 300  # 5 minutes\n\nVALID_WEBSOCKET_URI_PREFIXES = {\"ws://\", \"wss://\"}\nRESTRICTED_WEBSOCKET_KWARGS = {\"uri\", \"loop\"}\nDEFAULT_WEBSOCKET_KWARGS = {\n    # set how long to wait between pings from the server\n    \"ping_interval\": DEFAULT_PING_INTERVAL,\n    # set how long to wait without a pong response before closing the connection\n    \"ping_timeout\": DEFAULT_PING_TIMEOUT,\n}\n\n\ndef get_default_endpoint() -> URI:\n    return URI(os.environ.get(\"WEB3_WS_PROVIDER_URI\", \"ws://127.0.0.1:8546\"))\n\n\nclass WebSocketProvider(PersistentConnectionProvider):\n    logger = logging.getLogger(\"web3.providers.WebSocketProvider\")\n    is_async: bool = True\n\n    _max_connection_retries: int = 5\n    _ws: Optional[WebSocketClientProtocol] = None\n\n    def __init__(\n        self,\n        endpoint_uri: Optional[Union[URI, str]] = None,\n        websocket_kwargs: Optional[Dict[str, Any]] = None,\n        # `PersistentConnectionProvider` kwargs can be passed through\n        **kwargs: Any,\n    ) -> None:\n        self.endpoint_uri = URI(endpoint_uri)\n        if self.endpoint_uri is None:\n            self.endpoint_uri = get_default_endpoint()\n\n        if not any(\n            self.endpoint_uri.startswith(prefix)\n            for prefix in VALID_WEBSOCKET_URI_PREFIXES\n        ):\n            raise Web3ValidationError(\n                \"WebSocket endpoint uri must begin with 'ws://' or 'wss://': \"\n                f\"{self.endpoint_uri}\"\n            )\n\n        if websocket_kwargs is not None:\n            found_restricted_keys = set(websocket_kwargs).intersection(\n                RESTRICTED_WEBSOCKET_KWARGS\n            )\n            if found_restricted_keys:\n                raise Web3ValidationError(\n                    \"Found restricted keys for websocket_kwargs: \"\n                    f\"{found_restricted_keys}.\"\n                )\n\n        self.websocket_kwargs = merge(DEFAULT_WEBSOCKET_KWARGS, websocket_kwargs or {})\n\n        super().__init__(**kwargs)\n\n    def __str__(self) -> str:\n        return f\"WebSocket connection: {self.endpoint_uri}\"\n\n    async def is_connected(self, show_traceback: bool = False) -> bool:\n        if not self._ws:\n            return False\n\n        try:\n            await self._ws.pong()\n            return True\n\n        except WebSocketException as e:\n            if show_traceback:\n                raise ProviderConnectionError(\n                    f\"Error connecting to endpoint: '{self.endpoint_uri}'\"\n                ) from e\n            return False\n\n    async def connect(self) -> None:\n        _connection_attempts = 0\n        _backoff_rate_change = 1.75\n        _backoff_time = 1.75\n\n        while _connection_attempts != self._max_connection_retries:\n            try:\n                _connection_attempts += 1\n                self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)\n                self._message_listener_task = asyncio.create_task(\n                    self._message_listener()\n                )\n                break\n            except WebSocketException as e:\n                if _connection_attempts == self._max_connection_retries:\n                    raise ProviderConnectionError(\n                        f\"Could not connect to endpoint: {self.endpoint_uri}. \"\n                        f\"Retries exceeded max of {self._max_connection_retries}.\"\n                    ) from e\n                self.logger.info(\n                    f\"Could not connect to endpoint: {self.endpoint_uri}. Retrying in \"\n                    f\"{round(_backoff_time, 1)} seconds.\",\n                    exc_info=True,\n                )\n                await asyncio.sleep(_backoff_time)\n                _backoff_time *= _backoff_rate_change\n\n    async def disconnect(self) -> None:\n        if self._ws is not None and not self._ws.closed:\n            await self._ws.close()\n            self._ws = None\n            self.logger.debug(\n                f'Successfully disconnected from endpoint: \"{self.endpoint_uri}'\n            )\n\n        try:\n            self._message_listener_task.cancel()\n            await self._message_listener_task\n        except (asyncio.CancelledError, StopAsyncIteration):\n            pass\n        self._request_processor.clear_caches()\n\n    @async_handle_request_caching\n    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        request_data = self.encode_rpc_request(method, params)\n\n        if self._ws is None:\n            raise ProviderConnectionError(\n                \"Connection to websocket has not been initiated for the provider.\"\n            )\n\n        await asyncio.wait_for(\n            self._ws.send(request_data), timeout=self.request_timeout\n        )\n\n        current_request_id = json.loads(request_data)[\"id\"]\n        response = await self._get_response_for_request_id(current_request_id)\n\n        return response\n\n    async def _message_listener(self) -> None:\n        self.logger.info(\n            \"WebSocket listener background task started. Storing all messages in \"\n            \"appropriate request processor queues / caches to be processed.\"\n        )\n        while True:\n            # the use of sleep(0) seems to be the most efficient way to yield control\n            # back to the event loop to share the loop with other tasks.\n            await asyncio.sleep(0)\n\n            try:\n                async for raw_message in self._ws:\n                    await asyncio.sleep(0)\n\n                    response = json.loads(raw_message)\n                    subscription = response.get(\"method\") == \"eth_subscription\"\n                    await self._request_processor.cache_raw_response(\n                        response, subscription=subscription\n                    )\n            except Exception as e:\n                if not self.silence_listener_task_exceptions:\n                    loop = asyncio.get_event_loop()\n                    for task in asyncio.all_tasks(loop=loop):\n                        task.cancel()\n                    raise e\n\n                self.logger.error(\n                    \"Exception caught in listener, error logging and keeping \"\n                    \"listener background task alive.\"\n                    f\"\\n    error={e.__class__.__name__}: {e}\"\n                )\n\n# Path: web3/providers/persistent/__init__.py\nfrom .persistent import (\n    PersistentConnectionProvider,\n)\nfrom .persistent_connection import (\n    PersistentConnection,\n)\nfrom .request_processor import (\n    RequestProcessor,\n)\nfrom .async_ipc import (\n    AsyncIPCProvider,\n)\nfrom .websocket import (\n    WebSocketProvider,\n)\n\n# Path: web3/providers/async_base.py\nimport asyncio\nimport itertools\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Coroutine,\n    Optional,\n    Set,\n    Tuple,\n    cast,\n)\n\nfrom eth_utils import (\n    is_text,\n    to_bytes,\n    to_text,\n)\n\nfrom web3._utils.caching import (\n    async_handle_request_caching,\n)\nfrom web3._utils.encoding import (\n    FriendlyJsonSerde,\n    Web3JsonEncoder,\n)\nfrom web3.exceptions import (\n    ProviderConnectionError,\n)\nfrom web3.middleware import (\n    async_combine_middlewares,\n)\nfrom web3.middleware.base import (\n    Middleware,\n    MiddlewareOnion,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\nfrom web3.utils import (\n    SimpleCache,\n)\n\nif TYPE_CHECKING:\n    from websockets import (\n        WebSocketClientProtocol,\n    )\n\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        WebSocketProvider,\n    )\n    from web3.providers.persistent import (  # noqa: F401\n        RequestProcessor,\n    )\n\n\nCACHEABLE_REQUESTS = cast(\n    Set[RPCEndpoint],\n    (\n        \"eth_chainId\",\n        \"eth_getBlockByHash\",\n        \"eth_getBlockTransactionCountByHash\",\n        \"eth_getRawTransactionByHash\",\n        \"eth_getTransactionByBlockHashAndIndex\",\n        \"eth_getTransactionByHash\",\n        \"eth_getUncleByBlockHashAndIndex\",\n        \"eth_getUncleCountByBlockHash\",\n        \"net_version\",\n        \"web3_clientVersion\",\n    ),\n)\n\n\nclass AsyncBaseProvider:\n    _request_func_cache: Tuple[\n        Tuple[Middleware, ...], Callable[..., Coroutine[Any, Any, RPCResponse]]\n    ] = (None, None)\n\n    is_async = True\n    has_persistent_connection = False\n    global_ccip_read_enabled: bool = True\n    ccip_read_max_redirects: int = 4\n\n    # request caching\n    cache_allowed_requests: bool = False\n    cacheable_requests: Set[RPCEndpoint] = CACHEABLE_REQUESTS\n    _request_cache: SimpleCache\n    _request_cache_lock: asyncio.Lock = asyncio.Lock()\n\n    def __init__(self) -> None:\n        self._request_cache = SimpleCache(1000)\n\n    async def request_func(\n        self, async_w3: \"AsyncWeb3\", middleware_onion: MiddlewareOnion\n    ) -> Callable[..., Coroutine[Any, Any, RPCResponse]]:\n        middlewares: Tuple[Middleware, ...] = middleware_onion.as_tuple_of_middlewares()\n\n        cache_key = self._request_func_cache[0]\n        if cache_key != middlewares:\n            self._request_func_cache = (\n                middlewares,\n                await async_combine_middlewares(\n                    middlewares=middlewares,\n                    async_w3=async_w3,\n                    provider_request_fn=self.make_request,\n                ),\n            )\n        return self._request_func_cache[-1]\n\n    @async_handle_request_caching\n    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        raise NotImplementedError(\"Providers must implement this method\")\n\n    async def is_connected(self, show_traceback: bool = False) -> bool:\n        raise NotImplementedError(\"Providers must implement this method\")\n\n    # -- persistent connection providers -- #\n\n    _request_processor: \"RequestProcessor\"\n    _message_listener_task: \"asyncio.Task[None]\"\n    _listen_event: \"asyncio.Event\"\n\n    async def connect(self) -> None:\n        raise NotImplementedError(\n            \"Persistent connection providers must implement this method\"\n        )\n\n    async def disconnect(self) -> None:\n        raise NotImplementedError(\n            \"Persistent connection providers must implement this method\"\n        )\n\n    # WebSocket typing\n    _ws: \"WebSocketClientProtocol\"\n\n    # IPC typing\n    _reader: Optional[asyncio.StreamReader]\n    _writer: Optional[asyncio.StreamWriter]\n\n\nclass AsyncJSONBaseProvider(AsyncBaseProvider):\n    def __init__(self) -> None:\n        super().__init__()\n        self.request_counter = itertools.count()\n\n    def encode_rpc_request(self, method: RPCEndpoint, params: Any) -> bytes:\n        request_id = next(self.request_counter)\n        rpc_dict = {\n            \"jsonrpc\": \"2.0\",\n            \"method\": method,\n            \"params\": params or [],\n            \"id\": request_id,\n        }\n        encoded = FriendlyJsonSerde().json_encode(rpc_dict, cls=Web3JsonEncoder)\n        return to_bytes(text=encoded)\n\n    def decode_rpc_response(self, raw_response: bytes) -> RPCResponse:\n        text_response = str(\n            to_text(raw_response) if not is_text(raw_response) else raw_response\n        )\n        return cast(RPCResponse, FriendlyJsonSerde().json_decode(text_response))\n\n    async def is_connected(self, show_traceback: bool = False) -> bool:\n        try:\n            response = await self.make_request(RPCEndpoint(\"web3_clientVersion\"), [])\n        except (OSError, ProviderConnectionError) as e:\n            if show_traceback:\n                raise ProviderConnectionError(\n                    f\"Problem connecting to provider with error: {type(e)}: {e}\"\n                )\n            return False\n\n        if \"error\" in response:\n            if show_traceback:\n                raise ProviderConnectionError(\n                    f\"Error received from provider: {response}\"\n                )\n            return False\n\n        if response.get(\"jsonrpc\") == \"2.0\":\n            return True\n        else:\n            if show_traceback:\n                raise ProviderConnectionError(f\"Bad jsonrpc version: {response}\")\n            return False\n\n# Path: web3/providers/auto.py\nimport os\nfrom typing import (\n    Any,\n    Callable,\n    Dict,\n    Optional,\n    Sequence,\n    Tuple,\n    Type,\n    Union,\n)\nfrom urllib.parse import (\n    urlparse,\n)\n\nfrom eth_typing import (\n    URI,\n)\n\nfrom web3.exceptions import (\n    CannotHandleRequest,\n)\nfrom web3.providers import (\n    BaseProvider,\n    HTTPProvider,\n    IPCProvider,\n    LegacyWebSocketProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nHTTP_SCHEMES = {\"http\", \"https\"}\nWS_SCHEMES = {\"ws\", \"wss\"}\n\n\ndef load_provider_from_environment() -> BaseProvider:\n    uri_string = URI(os.environ.get(\"WEB3_PROVIDER_URI\", \"\"))\n    if not uri_string:\n        return None\n\n    return load_provider_from_uri(uri_string)\n\n\ndef load_provider_from_uri(\n    uri_string: URI, headers: Optional[Dict[str, Tuple[str, str]]] = None\n) -> BaseProvider:\n    uri = urlparse(uri_string)\n    if uri.scheme == \"file\":\n        return IPCProvider(uri.path)\n    elif uri.scheme in HTTP_SCHEMES:\n        return HTTPProvider(uri_string, headers)\n    elif uri.scheme in WS_SCHEMES:\n        return LegacyWebSocketProvider(uri_string)\n    else:\n        raise NotImplementedError(\n            \"Web3 does not know how to connect to scheme \"\n            f\"{uri.scheme!r} in {uri_string!r}\"\n        )\n\n\nclass AutoProvider(BaseProvider):\n    default_providers = (\n        load_provider_from_environment,\n        IPCProvider,\n        HTTPProvider,\n        LegacyWebSocketProvider,\n    )\n    _active_provider = None\n\n    def __init__(\n        self,\n        potential_providers: Optional[\n            Sequence[Union[Callable[..., BaseProvider], Type[BaseProvider]]]\n        ] = None,\n    ) -> None:\n        \"\"\"\n        :param iterable potential_providers: ordered series of provider classes\n            to attempt with\n\n        AutoProvider will initialize each potential provider (without arguments),\n        in an attempt to find an active node. The list will default to\n        :attribute:`default_providers`.\n        \"\"\"\n        if potential_providers:\n            self._potential_providers = potential_providers\n        else:\n            self._potential_providers = self.default_providers\n\n    def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        try:\n            return self._proxy_request(method, params)\n        except OSError:\n            return self._proxy_request(method, params, use_cache=False)\n\n    def is_connected(self, show_traceback: bool = False) -> bool:\n        provider = self._get_active_provider(use_cache=True)\n        return provider is not None and provider.is_connected(show_traceback)\n\n    def _proxy_request(\n        self, method: RPCEndpoint, params: Any, use_cache: bool = True\n    ) -> RPCResponse:\n        provider = self._get_active_provider(use_cache)\n        if provider is None:\n            raise CannotHandleRequest(\n                \"Could not discover provider while making request: \"\n                f\"method:{method}\\nparams:{params}\\n\"\n            )\n\n        return provider.make_request(method, params)\n\n    def _get_active_provider(self, use_cache: bool) -> Optional[BaseProvider]:\n        if use_cache and self._active_provider is not None:\n            return self._active_provider\n\n        for Provider in self._potential_providers:\n            provider = Provider()\n            if provider is not None and provider.is_connected():\n                self._active_provider = provider\n                return provider\n\n        return None\n\n# Path: web3/providers/legacy_websocket.py\nimport asyncio\nimport json\nimport logging\nimport os\nfrom threading import (\n    Thread,\n)\nfrom types import (\n    TracebackType,\n)\nfrom typing import (\n    Any,\n    Optional,\n    Type,\n    Union,\n)\n\nfrom eth_typing import (\n    URI,\n)\nfrom websockets.client import (\n    connect,\n)\nfrom websockets.legacy.client import (\n    WebSocketClientProtocol,\n)\n\nfrom web3.exceptions import (\n    Web3ValidationError,\n)\nfrom web3.providers.base import (\n    JSONBaseProvider,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nRESTRICTED_WEBSOCKET_KWARGS = {\"uri\", \"loop\"}\nDEFAULT_WEBSOCKET_TIMEOUT = 30\n\n\ndef _start_event_loop(loop: asyncio.AbstractEventLoop) -> None:\n    asyncio.set_event_loop(loop)\n    loop.run_forever()\n    loop.close()\n\n\ndef _get_threaded_loop() -> asyncio.AbstractEventLoop:\n    new_loop = asyncio.new_event_loop()\n    thread_loop = Thread(target=_start_event_loop, args=(new_loop,), daemon=True)\n    thread_loop.start()\n    return new_loop\n\n\ndef get_default_endpoint() -> URI:\n    return URI(os.environ.get(\"WEB3_WS_PROVIDER_URI\", \"ws://127.0.0.1:8546\"))\n\n\nclass PersistentWebSocket:\n    def __init__(self, endpoint_uri: URI, websocket_kwargs: Any) -> None:\n        self.ws: Optional[WebSocketClientProtocol] = None\n        self.endpoint_uri = endpoint_uri\n        self.websocket_kwargs = websocket_kwargs\n\n    async def __aenter__(self) -> WebSocketClientProtocol:\n        if self.ws is None:\n            self.ws = await connect(uri=self.endpoint_uri, **self.websocket_kwargs)\n        return self.ws\n\n    async def __aexit__(\n        self,\n        exc_type: Type[BaseException],\n        exc_val: BaseException,\n        exc_tb: TracebackType,\n    ) -> None:\n        if exc_val is not None:\n            try:\n                await self.ws.close()\n            except Exception:\n                pass\n            self.ws = None\n\n\nclass LegacyWebSocketProvider(JSONBaseProvider):\n    logger = logging.getLogger(\"web3.providers.WebSocketProvider\")\n    _loop = None\n\n    def __init__(\n        self,\n        endpoint_uri: Optional[Union[URI, str]] = None,\n        websocket_kwargs: Optional[Any] = None,\n        websocket_timeout: int = DEFAULT_WEBSOCKET_TIMEOUT,\n    ) -> None:\n        self.endpoint_uri = URI(endpoint_uri)\n        self.websocket_timeout = websocket_timeout\n        if self.endpoint_uri is None:\n            self.endpoint_uri = get_default_endpoint()\n        if LegacyWebSocketProvider._loop is None:\n            LegacyWebSocketProvider._loop = _get_threaded_loop()\n        if websocket_kwargs is None:\n            websocket_kwargs = {}\n        else:\n            found_restricted_keys = set(websocket_kwargs).intersection(\n                RESTRICTED_WEBSOCKET_KWARGS\n            )\n            if found_restricted_keys:\n                raise Web3ValidationError(\n                    f\"{RESTRICTED_WEBSOCKET_KWARGS} are not allowed \"\n                    f\"in websocket_kwargs, found: {found_restricted_keys}\"\n                )\n        self.conn = PersistentWebSocket(self.endpoint_uri, websocket_kwargs)\n        super().__init__()\n\n    def __str__(self) -> str:\n        return f\"WS connection {self.endpoint_uri}\"\n\n    async def coro_make_request(self, request_data: bytes) -> RPCResponse:\n        async with self.conn as conn:\n            await asyncio.wait_for(\n                conn.send(request_data), timeout=self.websocket_timeout\n            )\n            return json.loads(\n                await asyncio.wait_for(conn.recv(), timeout=self.websocket_timeout)\n            )\n\n    def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        self.logger.debug(\n            f\"Making request WebSocket. URI: {self.endpoint_uri}, \" f\"Method: {method}\"\n        )\n        request_data = self.encode_rpc_request(method, params)\n        future = asyncio.run_coroutine_threadsafe(\n            self.coro_make_request(request_data), LegacyWebSocketProvider._loop\n        )\n        return future.result()\n\n# Path: web3/_utils/http.py\ndef construct_user_agent(class_type: type) -> str:\n    from web3 import (\n        __version__ as web3_version,\n    )\n\n    return f\"web3.py/{web3_version}/{class_type.__module__}.{class_type.__qualname__}\"\n\n# Path: web3/providers/rpc/utils.py\nfrom typing import (\n    Sequence,\n    Type,\n)\n\nfrom pydantic import (\n    BaseModel,\n)\n\nfrom web3.types import (\n    RPCEndpoint,\n)\n\nREQUEST_RETRY_ALLOWLIST = [\n    \"admin\",\n    \"net\",\n    \"txpool\",\n    \"testing\",\n    \"evm\",\n    \"eth_protocolVersion\",\n    \"eth_syncing\",\n    \"eth_coinbase\",\n    \"eth_mining\",\n    \"eth_hashrate\",\n    \"eth_chainId\",\n    \"eth_gasPrice\",\n    \"eth_accounts\",\n    \"eth_blockNumber\",\n    \"eth_getBalance\",\n    \"eth_getStorageAt\",\n    \"eth_getProof\",\n    \"eth_getCode\",\n    \"eth_getBlockByNumber\",\n    \"eth_getBlockByHash\",\n    \"eth_getBlockTransactionCountByNumber\",\n    \"eth_getBlockTransactionCountByHash\",\n    \"eth_getUncleCountByBlockNumber\",\n    \"eth_getUncleCountByBlockHash\",\n    \"eth_getTransactionByHash\",\n    \"eth_getTransactionByBlockHashAndIndex\",\n    \"eth_getTransactionByBlockNumberAndIndex\",\n    \"eth_getTransactionReceipt\",\n    \"eth_getTransactionCount\",\n    \"eth_getRawTransactionByHash\",\n    \"eth_call\",\n    \"eth_estimateGas\",\n    \"eth_createAccessList\",\n    \"eth_maxPriorityFeePerGas\",\n    \"eth_newBlockFilter\",\n    \"eth_newPendingTransactionFilter\",\n    \"eth_newFilter\",\n    \"eth_getFilterChanges\",\n    \"eth_getFilterLogs\",\n    \"eth_getLogs\",\n    \"eth_uninstallFilter\",\n    \"eth_getCompilers\",\n    \"eth_getWork\",\n    \"eth_sign\",\n    \"eth_signTypedData\",\n    \"eth_sendRawTransaction\",\n    \"personal_importRawKey\",\n    \"personal_newAccount\",\n    \"personal_listAccounts\",\n    \"personal_listWallets\",\n    \"personal_lockAccount\",\n    \"personal_unlockAccount\",\n    \"personal_ecRecover\",\n    \"personal_sign\",\n    \"personal_signTypedData\",\n]\n\n\ndef check_if_retry_on_failure(\n    method: RPCEndpoint,\n    allowlist: Sequence[str] = None,\n) -> bool:\n    if allowlist is None:\n        allowlist = REQUEST_RETRY_ALLOWLIST\n\n    if method in allowlist or method.split(\"_\")[0] in allowlist:\n        return True\n    else:\n        return False\n\n\nclass ExceptionRetryConfiguration(BaseModel):\n    errors: Sequence[Type[BaseException]]\n    retries: int\n    backoff_factor: float\n    method_allowlist: Sequence[str]\n\n    def __init__(\n        self,\n        errors: Sequence[Type[BaseException]] = None,\n        retries: int = 5,\n        backoff_factor: float = 0.5,\n        method_allowlist: Sequence[str] = None,\n    ):\n        super().__init__(\n            errors=errors,\n            retries=retries,\n            backoff_factor=backoff_factor,\n            method_allowlist=method_allowlist or REQUEST_RETRY_ALLOWLIST,\n        )\n\n# Path: web3/providers/rpc/async_rpc.py\nimport asyncio\nimport logging\nfrom typing import (\n    Any,\n    Dict,\n    Iterable,\n    Optional,\n    Tuple,\n    Union,\n)\n\nfrom aiohttp import (\n    ClientError,\n    ClientSession,\n)\nfrom eth_typing import (\n    URI,\n)\nfrom eth_utils import (\n    to_dict,\n)\n\nfrom web3._utils.http import (\n    construct_user_agent,\n)\nfrom web3._utils.request import (\n    async_cache_and_return_session as _async_cache_and_return_session,\n    async_make_post_request,\n    get_default_http_endpoint,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nfrom ..._utils.caching import (\n    async_handle_request_caching,\n)\nfrom ..async_base import (\n    AsyncJSONBaseProvider,\n)\nfrom .utils import (\n    ExceptionRetryConfiguration,\n    check_if_retry_on_failure,\n)\n\n\nclass AsyncHTTPProvider(AsyncJSONBaseProvider):\n    logger = logging.getLogger(\"web3.providers.AsyncHTTPProvider\")\n    endpoint_uri = None\n    _request_kwargs = None\n\n    def __init__(\n        self,\n        endpoint_uri: Optional[Union[URI, str]] = None,\n        request_kwargs: Optional[Any] = None,\n        exception_retry_configuration: Optional[\n            ExceptionRetryConfiguration\n        ] = ExceptionRetryConfiguration(errors=(ClientError, TimeoutError)),\n    ) -> None:\n        if endpoint_uri is None:\n            self.endpoint_uri = get_default_http_endpoint()\n        else:\n            self.endpoint_uri = URI(endpoint_uri)\n\n        self._request_kwargs = request_kwargs or {}\n        self.exception_retry_configuration = exception_retry_configuration\n\n        super().__init__()\n\n    async def cache_async_session(self, session: ClientSession) -> ClientSession:\n        return await _async_cache_and_return_session(self.endpoint_uri, session)\n\n    def __str__(self) -> str:\n        return f\"RPC connection {self.endpoint_uri}\"\n\n    @to_dict\n    def get_request_kwargs(self) -> Iterable[Tuple[str, Any]]:\n        if \"headers\" not in self._request_kwargs:\n            yield \"headers\", self.get_request_headers()\n        for key, value in self._request_kwargs.items():\n            yield key, value\n\n    def get_request_headers(self) -> Dict[str, str]:\n        return {\n            \"Content-Type\": \"application/json\",\n            \"User-Agent\": construct_user_agent(type(self)),\n        }\n\n    async def _make_request(self, method: RPCEndpoint, request_data: bytes) -> bytes:\n        \"\"\"\n        If exception_retry_configuration is set, retry on failure; otherwise, make\n        the request without retrying.\n        \"\"\"\n        if (\n            self.exception_retry_configuration is not None\n            and check_if_retry_on_failure(\n                method, self.exception_retry_configuration.method_allowlist\n            )\n        ):\n            for i in range(self.exception_retry_configuration.retries):\n                try:\n                    return await async_make_post_request(\n                        self.endpoint_uri, request_data, **self.get_request_kwargs()\n                    )\n                except tuple(self.exception_retry_configuration.errors):\n                    if i < self.exception_retry_configuration.retries - 1:\n                        await asyncio.sleep(\n                            self.exception_retry_configuration.backoff_factor\n                        )\n                        continue\n                    else:\n                        raise\n            return None\n        else:\n            return await async_make_post_request(\n                self.endpoint_uri, request_data, **self.get_request_kwargs()\n            )\n\n    @async_handle_request_caching\n    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:\n        self.logger.debug(\n            f\"Making request HTTP. URI: {self.endpoint_uri}, Method: {method}\"\n        )\n        request_data = self.encode_rpc_request(method, params)\n        raw_response = await self._make_request(method, request_data)\n        response = self.decode_rpc_response(raw_response)\n        self.logger.debug(\n            f\"Getting response HTTP. URI: {self.endpoint_uri}, \"\n            f\"Method: {method}, Response: {response}\"\n        )\n        return response\n\n# Path: web3/providers/rpc/rpc.py\nimport logging\nimport time\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Iterable,\n    Optional,\n    Tuple,\n    Union,\n)\n\nfrom eth_typing import (\n    URI,\n)\nfrom eth_utils import (\n    to_dict,\n)\nimport requests\n\nfrom web3._utils.http import (\n    construct_user_agent,\n)\nfrom web3._utils.request import (\n    cache_and_return_session,\n    get_default_http_endpoint,\n    make_post_request,\n)\nfrom web3.types import (\n    RPCEndpoint,\n    RPCResponse,\n)\n\nfrom ..._utils.caching import (\n    handle_request_caching,\n)\nfrom ..base import (\n    JSONBaseProvider,\n)\nfrom .utils import (\n    ExceptionRetryConfiguration,\n    check_if_retry_on_failure,\n)\n\nif TYPE_CHECKING:\n    from web3.middleware.base import (  # noqa: F401\n        Middleware,\n    )\n\n\nclass HTTPProvider(JSONBaseProvider):\n    logger = logging.getLogger(\"web3.providers.HTTPProvider\")\n    endpoint_uri = None\n\n    _request_args = None\n    _request_kwargs = None\n\n    exception_retry_configuration: Optional[ExceptionRetryConfiguration] = None\n\n    def __init__(\n        self,\n        endpoint_uri: Optional[Union[URI, str]] = None,\n        request_kwargs: Optional[Any] = None,\n        session: Optional[Any] = None,\n        exception_retry_configuration: Optional[ExceptionRetryConfiguration] = (\n            ExceptionRetryConfiguration(\n                errors=(\n                    ConnectionError,\n                    requests.HTTPError,\n                    requests.Timeout,\n                )\n            )\n        ),\n    ) -> None:\n        if endpoint_uri is None:\n            self.endpoint_uri = get_default_http_endpoint()\n        else:\n            self.endpoint_uri = URI(endpoint_uri)\n\n        self._request_kwargs = request_kwargs or {}\n        self.exception_retry_configuration = exception_retry_configuration\n\n        if session:\n            cache_and_return_session(self.endpoint_uri, session)\n\n        super().__init__()\n\n    def __str__(self) -> str:\n        return f\"RPC connection {self.endpoint_uri}\"\n\n    @to_dict\n    def get_request_kwargs(self) -> Iterable[Tuple[str, Any]]:\n        if \"headers\" not in self._request_kwargs:\n            yield \"headers\", self.get_request_headers()\n        for key, value in self._request_kwargs.items():\n            yield key, value\n\n    def get_request_headers(self) -> Dict[str, str]:\n        return {\n            \"Content-Type\": \"application/json\",\n            \"User-Agent\": construct_user_agent(type(self)),\n        }\n\n    def _make_request(self, method: RPCEndpoint, request_data: bytes) -> bytes:\n        \"\"\"\n        If exception_retry_configuration is set, retry on failure; otherwise, make\n        the request without retrying.\n        \"\"\"\n        if (\n            self.exception_retry_configuration is not None\n            and check_if_retry_on_failure(\n                method, self.exception_retry_configuration.method_allowlist\n            )\n        ):\n            for i in range(self.exception_retry_configuration.retries):\n                try:\n                    return make_post_request(\n...\n", "needle_token_start": 3935, "needle_token_end": 4072, "code_context_ntokens": 16086, "output": ["```python\n@staticmethod\ndef get_default_middlewares() -> List[Tuple[Middleware, str]]:\n    \"\"\"\n    List the default middlewares for the request manager.\n    Documentation should remain in sync with these defaults.\n    \"\"\"\n    return [\n        (GasPriceStrategyMiddleware, \"gas_price_strategy\"),\n        (ENSNameToAddressMiddleware, \"ens_name_to_address\"),\n        (AttributeDictMiddleware, \"attrdict\"),\n        (ValidationMiddleware, \"validation\"),\n        (BufferedGasEstimateMiddleware, \"gas_estimate\"),\n    ]\n```"]}
{"repo": "ethereum/web3.py", "name": "_get_transaction", "language": "python", "path": "web3/contract/base_contract.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to prepare transaction parameters for deploying a smart contract, ensuring all necessary data is included and formatted correctly.\n2. **Input**: An optional dictionary containing transaction parameters.\n3. **Output**: A dictionary of transaction parameters, including default values and encoded contract data necessary for deployment.\n4. **Procedure**: \n   - Initializes a transaction dictionary either from the provided input or as an empty dictionary if no input is provided.\n   - Validates and removes any forbidden keys from the transaction dictionary.\n   - Sets a default sender address if it's not specified and a default account exists.\n   - Adds encoded contract data to the transaction dictionary.\n   - Returns the prepared transaction dictionary.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                self.w3.eth.default_account,  # type: ignore\n            )\n\n        if \"to\" not in call_transaction:\n            if isinstance(self, type):\n                raise ValueError(\n                    \"When using `Contract.[methodtype].[method].call()` from\"\n                    \" a contract factory you \"\n                    \"must provide a `to` address with the transaction\"\n                )\n            else:\n                raise ValueError(\n                    \"Please ensure that this contract instance has an address.\"\n                )\n\n        return call_transaction\n\n    def _transact(self, transaction: Optional[TxParams] = None) -> TxParams:\n        if transaction is None:\n            transact_transaction: TxParams = {}\n        else:\n            transact_transaction = cast(TxParams, dict(**transaction))\n\n        if \"data\" in transact_transaction:\n            raise ValueError(\"Cannot set 'data' field in transact transaction\")\n\n        if self.address is not None:\n            transact_transaction.setdefault(\"to\", self.address)\n        if self.w3.eth.default_account is not empty:\n            # type ignored b/c check prevents an empty default_account\n            transact_transaction.setdefault(\n                \"from\", self.w3.eth.default_account  # type: ignore\n            )\n\n        if \"to\" not in transact_transaction:\n            if isinstance(self, type):\n                raise ValueError(\n                    \"When using `Contract.transact` from a contract factory you \"\n                    \"must provide a `to` address with the transaction\"\n                )\n            else:\n                raise ValueError(\n                    \"Please ensure that this contract instance has an address.\"\n                )\n        return transact_transaction\n\n    def _estimate_gas(self, transaction: Optional[TxParams] = None) -> TxParams:\n        if transaction is None:\n            estimate_gas_transaction: TxParams = {}\n        else:\n            estimate_gas_transaction = cast(TxParams, dict(**transaction))\n\n        if \"data\" in estimate_gas_transaction:\n            raise ValueError(\"Cannot set 'data' field in estimate_gas transaction\")\n        if \"to\" in estimate_gas_transaction:\n            raise ValueError(\"Cannot set to in estimate_gas transaction\")\n\n        if self.address:\n            estimate_gas_transaction.setdefault(\"to\", self.address)\n        if self.w3.eth.default_account is not empty:\n            # type ignored b/c check prevents an empty default_account\n            estimate_gas_transaction.setdefault(\n                \"from\", self.w3.eth.default_account  # type: ignore\n            )\n\n        if \"to\" not in estimate_gas_transaction:\n            if isinstance(self, type):\n                raise ValueError(\n                    \"When using `Contract.estimate_gas` from a contract factory \"\n                    \"you must provide a `to` address with the transaction\"\n                )\n            else:\n                raise ValueError(\n                    \"Please ensure that this contract instance has an address.\"\n                )\n        return estimate_gas_transaction\n\n    def _build_transaction(self, transaction: Optional[TxParams] = None) -> TxParams:\n        if transaction is None:\n            built_transaction: TxParams = {}\n        else:\n            built_transaction = cast(TxParams, dict(**transaction))\n\n        if \"data\" in built_transaction:\n            raise ValueError(\"Cannot set 'data' field in build transaction\")\n\n        if not self.address and \"to\" not in built_transaction:\n            raise ValueError(\n                \"When using `ContractFunction.build_transaction` from a contract \"\n                \"factory you must provide a `to` address with the transaction\"\n            )\n        if self.address and \"to\" in built_transaction:\n            raise ValueError(\"Cannot set 'to' field in contract call build transaction\")\n\n        if self.address:\n            built_transaction.setdefault(\"to\", self.address)\n\n        if \"to\" not in built_transaction:\n            raise ValueError(\n                \"Please ensure that this contract instance has an address.\"\n            )\n\n        return built_transaction\n\n    @combomethod\n    def _encode_transaction_data(cls) -> HexStr:\n        return add_0x_prefix(encode_abi(cls.w3, cls.abi, cls.arguments, cls.selector))\n\n    _return_data_normalizers: Optional[Tuple[Callable[..., Any], ...]] = tuple()\n\n    def __repr__(self) -> str:\n        if self.abi:\n            _repr = f\"<Function {abi_to_signature(self.abi)}\"\n            if self.arguments is not None:\n                _repr += f\" bound to {self.arguments!r}\"\n            return _repr + \">\"\n        return f\"<Function {self.fn_name}>\"\n\n    @classmethod\n    def factory(\n        cls, class_name: str, **kwargs: Any\n    ) -> Union[\"ContractFunction\", \"AsyncContractFunction\"]:\n        return PropertyCheckingFactory(class_name, (cls,), kwargs)(kwargs.get(\"abi\"))\n\n\nclass BaseContractFunctions:\n    \"\"\"Class containing contract function objects\"\"\"\n\n    def __init__(\n        self,\n        abi: ABI,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        contract_function_class: Union[\n            Type[\"ContractFunction\"], Type[\"AsyncContractFunction\"]\n        ],\n        address: Optional[ChecksumAddress] = None,\n        decode_tuples: Optional[bool] = False,\n    ) -> None:\n        self.abi = abi\n        self.w3 = w3\n        self.address = address\n\n        if self.abi:\n            self._functions = filter_by_type(\"function\", self.abi)\n            for func in self._functions:\n                setattr(\n                    self,\n                    func[\"name\"],\n                    contract_function_class.factory(\n                        func[\"name\"],\n                        w3=self.w3,\n                        contract_abi=self.abi,\n                        address=self.address,\n                        decode_tuples=decode_tuples,\n                        function_identifier=func[\"name\"],\n                    ),\n                )\n\n    def __iter__(self) -> Iterable[\"ABIFunction\"]:\n        if not hasattr(self, \"_functions\") or not self._functions:\n            return\n\n        for func in self._functions:\n            yield self[func[\"name\"]]\n\n    def __getitem__(self, function_name: str) -> ABIFunction:\n        return getattr(self, function_name)\n\n    def __hasattr__(self, function_name: str) -> bool:\n        try:\n            return function_name in self.__dict__[\"_functions\"]\n        except ABIFunctionNotFound:\n            return False\n\n\nclass BaseContract:\n    \"\"\"Base class for Contract proxy classes.\n\n    First you need to create your Contract classes using\n    :meth:`web3.eth.Eth.contract` that takes compiled Solidity contract\n    ABI definitions as input.  The created class object will be a subclass of\n    this base class.\n\n    After you have your Contract proxy class created you can interact with\n    smart contracts\n\n    * Create a Contract proxy object for an existing deployed smart contract by\n      its address using :meth:`__init__`\n\n    * Deploy a new smart contract using :py:meth:`Contract.constructor.transact()`\n    \"\"\"\n\n    # set during class construction\n    w3: Union[\"Web3\", \"AsyncWeb3\"] = None\n\n    # instance level properties\n    address: ChecksumAddress = None\n\n    # class properties (overridable at instance level)\n    abi: ABI = None\n\n    asm = None\n    ast = None\n\n    bytecode = None\n    bytecode_runtime = None\n    clone_bin = None\n\n    decode_tuples = None\n    dev_doc = None\n    interface = None\n    metadata = None\n    opcodes = None\n    src_map = None\n    src_map_runtime = None\n    user_doc = None\n\n    #  Public API\n    #\n    @combomethod\n    def encodeABI(\n        cls,\n        fn_name: str,\n        args: Optional[Any] = None,\n        kwargs: Optional[Any] = None,\n        data: Optional[HexStr] = None,\n    ) -> HexStr:\n        \"\"\"\n        Encodes the arguments using the Ethereum ABI for the contract function\n        that matches the given name and arguments..\n\n        :param data: defaults to function selector\n        \"\"\"\n        fn_abi, fn_selector, fn_arguments = get_function_info(\n            fn_name,\n            cls.w3.codec,\n            contract_abi=cls.abi,\n            args=args,\n            kwargs=kwargs,\n        )\n\n        if data is None:\n            data = fn_selector\n\n        return encode_abi(cls.w3, fn_abi, fn_arguments, data)\n\n    @combomethod\n    def all_functions(\n        self,\n    ) -> \"BaseContractFunction\":\n        return self.find_functions_by_identifier(\n            self.abi, self.w3, self.address, lambda _: True\n        )\n\n    @combomethod\n    def get_function_by_signature(self, signature: str) -> \"BaseContractFunction\":\n        if \" \" in signature:\n            raise ValueError(\n                \"Function signature should not contain any spaces. \"\n                f\"Found spaces in input: {signature}\"\n            )\n\n        def callable_check(fn_abi: ABIFunction) -> bool:\n            return abi_to_signature(fn_abi) == signature\n\n        fns = self.find_functions_by_identifier(\n            self.abi, self.w3, self.address, callable_check\n        )\n        return self.get_function_by_identifier(fns, \"signature\")\n\n    @combomethod\n    def find_functions_by_name(self, fn_name: str) -> \"BaseContractFunction\":\n        def callable_check(fn_abi: ABIFunction) -> bool:\n            return fn_abi[\"name\"] == fn_name\n\n        return self.find_functions_by_identifier(\n            self.abi, self.w3, self.address, callable_check\n        )\n\n    @combomethod\n    def get_function_by_name(self, fn_name: str) -> \"BaseContractFunction\":\n        fns = self.find_functions_by_name(fn_name)\n        return self.get_function_by_identifier(fns, \"name\")\n\n    @combomethod\n    def get_function_by_selector(\n        self, selector: Union[bytes, int, HexStr]\n    ) -> \"BaseContractFunction\":\n        def callable_check(fn_abi: ABIFunction) -> bool:\n            # typed dict cannot be used w/ a normal Dict\n            # https://github.com/python/mypy/issues/4976\n            return encode_hex(function_abi_to_4byte_selector(fn_abi)) == to_4byte_hex(selector)  # type: ignore # noqa: E501\n\n        fns = self.find_functions_by_identifier(\n            self.abi, self.w3, self.address, callable_check\n        )\n        return self.get_function_by_identifier(fns, \"selector\")\n\n    @combomethod\n    def decode_function_input(\n        self, data: HexStr\n    ) -> Tuple[\"BaseContractFunction\", Dict[str, Any]]:\n        # type ignored b/c expects data arg to be HexBytes\n        data = HexBytes(data)  # type: ignore\n        func = self.get_function_by_selector(data[:4])\n        arguments = decode_transaction_data(\n            func.abi, data, normalizers=BASE_RETURN_NORMALIZERS\n        )\n        return func, arguments\n\n    @combomethod\n    def find_functions_by_args(self, *args: Any) -> \"BaseContractFunction\":\n        def callable_check(fn_abi: ABIFunction) -> bool:\n            return check_if_arguments_can_be_encoded(\n                fn_abi, self.w3.codec, args=args, kwargs={}\n            )\n\n        return self.find_functions_by_identifier(\n            self.abi, self.w3, self.address, callable_check\n        )\n\n    @combomethod\n    def get_function_by_args(self, *args: Any) -> \"BaseContractFunction\":\n        fns = self.find_functions_by_args(*args)\n        return self.get_function_by_identifier(fns, \"args\")\n\n    #\n    # Private Helpers\n    #\n    _return_data_normalizers: Tuple[Callable[..., Any], ...] = tuple()\n\n    @classmethod\n    def _prepare_transaction(\n        cls,\n        fn_name: str,\n        fn_args: Optional[Any] = None,\n        fn_kwargs: Optional[Any] = None,\n        transaction: Optional[TxParams] = None,\n    ) -> TxParams:\n        return prepare_transaction(\n            cls.address,\n            cls.w3,\n            fn_identifier=fn_name,\n            contract_abi=cls.abi,\n            transaction=transaction,\n            fn_args=fn_args,\n            fn_kwargs=fn_kwargs,\n        )\n\n    @classmethod\n    def _find_matching_fn_abi(\n        cls,\n        fn_identifier: Optional[str] = None,\n        args: Optional[Any] = None,\n        kwargs: Optional[Any] = None,\n    ) -> ABIFunction:\n        return find_matching_fn_abi(\n            cls.abi, cls.w3.codec, fn_identifier=fn_identifier, args=args, kwargs=kwargs\n        )\n\n    @classmethod\n    def _find_matching_event_abi(\n        cls,\n        event_name: Optional[str] = None,\n        argument_names: Optional[Sequence[str]] = None,\n    ) -> ABIEvent:\n        return find_matching_event_abi(\n            abi=cls.abi, event_name=event_name, argument_names=argument_names\n        )\n\n    @combomethod\n    def _encode_constructor_data(\n        cls, args: Optional[Any] = None, kwargs: Optional[Any] = None\n    ) -> HexStr:\n        constructor_abi = get_constructor_abi(cls.abi)\n\n        if constructor_abi:\n            if args is None:\n                args = tuple()\n            if kwargs is None:\n                kwargs = {}\n\n            arguments = merge_args_and_kwargs(constructor_abi, args, kwargs)\n\n            deploy_data = add_0x_prefix(\n                encode_abi(cls.w3, constructor_abi, arguments, data=cls.bytecode)\n            )\n        else:\n            if args is not None or kwargs is not None:\n                msg = \"Constructor args were provided, but no constructor function was provided.\"  # noqa: E501\n                raise TypeError(msg)\n\n            deploy_data = to_hex(cls.bytecode)\n\n        return deploy_data\n\n    @combomethod\n    def find_functions_by_identifier(\n        cls,\n        contract_abi: ABI,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        address: ChecksumAddress,\n        callable_check: Callable[..., Any],\n    ) -> List[Any]:\n        raise NotImplementedError(\n            \"This method should be implemented in the inherited class\"\n        )\n\n    @combomethod\n    def get_function_by_identifier(\n        cls, fns: Sequence[\"BaseContractFunction\"], identifier: str\n    ) -> \"BaseContractFunction\":\n        raise NotImplementedError(\n            \"This method should be implemented in the inherited class\"\n        )\n\n    @staticmethod\n    def get_fallback_function(\n        abi: ABI,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        function_type: Type[\"BaseContractFunction\"],\n        address: Optional[ChecksumAddress] = None,\n    ) -> \"BaseContractFunction\":\n        if abi and fallback_func_abi_exists(abi):\n            return function_type.factory(\n                \"fallback\",\n                w3=w3,\n                contract_abi=abi,\n                address=address,\n                function_identifier=FallbackFn,\n            )()\n\n        return cast(function_type, NonExistentFallbackFunction())  # type: ignore\n\n    @staticmethod\n    def get_receive_function(\n        abi: ABI,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        function_type: Type[\"BaseContractFunction\"],\n        address: Optional[ChecksumAddress] = None,\n    ) -> \"BaseContractFunction\":\n        if abi and receive_func_abi_exists(abi):\n            return function_type.factory(\n                \"receive\",\n                w3=w3,\n                contract_abi=abi,\n                address=address,\n                function_identifier=ReceiveFn,\n            )()\n\n        return cast(function_type, NonExistentReceiveFunction())  # type: ignore\n\n\nclass BaseContractCaller:\n    \"\"\"\n    An alternative Contract API.\n\n    This call:\n\n    > contract.caller({'from': eth.accounts[1], 'gas': 100000, ...}).add(2, 3)\n    is equivalent to this call in the classic contract:\n    > contract.functions.add(2, 3).call({'from': eth.accounts[1], 'gas': 100000, ...})\n\n    Other options for invoking this class include:\n\n    > contract.caller.add(2, 3)\n\n    or\n\n    > contract.caller().add(2, 3)\n\n    or\n\n    > contract.caller(transaction={'from': eth.accounts[1], 'gas': 100000, ...}).add(2, 3)  # noqa: E501\n    \"\"\"\n\n    # mypy types\n    _functions: List[Union[ABIFunction, ABIEvent]]\n\n    def __init__(\n        self,\n        abi: ABI,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        address: ChecksumAddress,\n        decode_tuples: Optional[bool] = False,\n    ) -> None:\n        self.w3 = w3\n        self.address = address\n        self.abi = abi\n        self.decode_tuples = decode_tuples\n        self._functions = []\n\n    def __getattr__(self, function_name: str) -> Any:\n        if self.abi is None:\n            raise NoABIFound(\n                \"There is no ABI found for this contract.\",\n            )\n        elif not self._functions or len(self._functions) == 0:\n            raise NoABIFunctionsFound(\n                \"The ABI for this contract contains no function definitions. \",\n                \"Are you sure you provided the correct contract ABI?\",\n            )\n        elif function_name not in {fn[\"name\"] for fn in self._functions}:\n            functions_available = \", \".join([fn[\"name\"] for fn in self._functions])\n            raise ABIFunctionNotFound(\n                f\"The function '{function_name}' was not found in this contract's ABI.\",\n                \" Here is a list of all of the function names found: \",\n                f\"{functions_available}. \",\n                \"Did you mean to call one of those functions?\",\n            )\n        else:\n            return super().__getattribute__(function_name)\n\n    def __hasattr__(self, event_name: str) -> bool:\n        try:\n            return event_name in self.__dict__[\"_events\"]\n        except ABIFunctionNotFound:\n            return False\n\n    @staticmethod\n    def call_function(\n        fn: TContractFn,\n        *args: Any,\n        transaction: Optional[TxParams] = None,\n        block_identifier: BlockIdentifier = None,\n        ccip_read_enabled: Optional[bool] = None,\n        **kwargs: Any,\n    ) -> Any:\n        if transaction is None:\n            transaction = {}\n        return fn(*args, **kwargs).call(\n            transaction=transaction,\n            block_identifier=block_identifier,\n            ccip_read_enabled=ccip_read_enabled,\n        )\n\n\nclass BaseContractConstructor:\n    \"\"\"\n    Class for contract constructor API.\n    \"\"\"\n\n    def __init__(\n        self,\n        w3: Union[\"Web3\", \"AsyncWeb3\"],\n        abi: ABI,\n        bytecode: HexStr,\n        *args: Any,\n        **kwargs: Any,\n    ) -> None:\n        self.w3 = w3\n        self.abi = abi\n        self.bytecode = bytecode\n        self.data_in_transaction = self._encode_data_in_transaction(*args, **kwargs)\n\n    @combomethod\n    def _encode_data_in_transaction(self, *args: Any, **kwargs: Any) -> HexStr:\n        constructor_abi = get_constructor_abi(self.abi)\n\n        if constructor_abi:\n            if not args:\n                args = tuple()\n            if not kwargs:\n                kwargs = {}\n\n            arguments = merge_args_and_kwargs(constructor_abi, args, kwargs)\n            data = add_0x_prefix(\n                encode_abi(self.w3, constructor_abi, arguments, data=self.bytecode)\n            )\n        else:\n            data = to_hex(self.bytecode)\n\n        return data\n\n    @combomethod\n    def _estimate_gas(self, transaction: Optional[TxParams] = None) -> TxParams:\n        if transaction is None:\n            estimate_gas_transaction: TxParams = {}\n        else:\n            estimate_gas_transaction = cast(TxParams, dict(**transaction))\n            self.check_forbidden_keys_in_transaction(\n                estimate_gas_transaction, [\"data\", \"to\"]\n            )\n\n        if self.w3.eth.default_account is not empty:\n            # type ignored b/c check prevents an empty default_account\n            estimate_gas_transaction.setdefault(\n                \"from\", self.w3.eth.default_account  # type: ignore\n            )\n\n        estimate_gas_transaction[\"data\"] = self.data_in_transaction\n\n        return estimate_gas_transaction\n\n    \ndef _get_transaction(self, transaction: Optional[TxParams] = None) -> TxParams:\n        if transaction is None:\n            transact_transaction: TxParams = {}\n        else:\n            transact_transaction = cast(TxParams, dict(**transaction))\n            self.check_forbidden_keys_in_transaction(\n                transact_transaction, [\"data\", \"to\"]\n            )\n\n        if self.w3.eth.default_account is not empty:\n            # type ignored b/c check prevents an empty default_account\n            transact_transaction.setdefault(\n                \"from\", self.w3.eth.default_account  # type: ignore\n            )\n\n        transact_transaction[\"data\"] = self.data_in_transaction\n\n        return transact_transaction\n\n    @combomethod\n    def _build_transaction(self, transaction: Optional[TxParams] = None) -> TxParams:\n        built_transaction = self._get_transaction(transaction)\n        built_transaction[\"to\"] = Address(b\"\")\n        return built_transaction\n\n    @staticmethod\n    def check_forbidden_keys_in_transaction(\n        transaction: TxParams, forbidden_keys: Optional[Collection[str]] = None\n    ) -> None:\n        keys_found = transaction.keys() & forbidden_keys\n        if keys_found:\n            raise ValueError(\n                f\"Cannot set '{', '.join(keys_found)}' field(s) in transaction\"\n            )\n\n\nclass NonExistentFallbackFunction:\n    @staticmethod\n    def _raise_exception() -> NoReturn:\n        raise FallbackNotFound(\"No fallback function was found in the contract ABI.\")\n\n    def __getattr__(self, attr: Any) -> Callable[[], None]:\n        return self._raise_exception\n\n\nclass NonExistentReceiveFunction:\n    @staticmethod\n    def _raise_exception() -> NoReturn:\n        raise FallbackNotFound(\"No receive function was found in the contract ABI.\")\n\n    def __getattr__(self, attr: Any) -> Callable[[], None]:\n        return self._raise_exception\n\n# Path: web3/contract/utils.py\nimport itertools\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    List,\n    Optional,\n    Sequence,\n    Tuple,\n    Type,\n    Union,\n)\n\nfrom eth_abi.exceptions import (\n    DecodingError,\n)\nfrom eth_typing import (\n    ChecksumAddress,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3._utils.abi import (\n    filter_by_type,\n    get_abi_output_types,\n    map_abi_data,\n    named_tree,\n    recursive_dict_to_namedtuple,\n)\nfrom web3._utils.async_transactions import (\n    async_fill_transaction_defaults,\n)\nfrom web3._utils.contracts import (\n    find_matching_fn_abi,\n    prepare_transaction,\n)\nfrom web3._utils.normalizers import (\n    BASE_RETURN_NORMALIZERS,\n)\nfrom web3._utils.transactions import (\n    fill_transaction_defaults,\n)\nfrom web3.exceptions import (\n    BadFunctionCallOutput,\n)\nfrom web3.types import (\n    ABI,\n    ABIFunction,\n    BlockIdentifier,\n    FunctionIdentifier,\n    StateOverride,\n    TContractFn,\n    TxParams,\n)\n\nif TYPE_CHECKING:\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\nACCEPTABLE_EMPTY_STRINGS = [\"0x\", b\"0x\", \"\", b\"\"]\n\n\ndef call_contract_function(\n    w3: \"Web3\",\n    address: ChecksumAddress,\n    normalizers: Tuple[Callable[..., Any], ...],\n    function_identifier: FunctionIdentifier,\n    transaction: TxParams,\n    block_id: Optional[BlockIdentifier] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    state_override: Optional[StateOverride] = None,\n    ccip_read_enabled: Optional[bool] = None,\n    decode_tuples: Optional[bool] = False,\n    *args: Any,\n    **kwargs: Any,\n) -> Any:\n    \"\"\"\n    Helper function for interacting with a contract function using the\n    `eth_call` API.\n    \"\"\"\n    call_transaction = prepare_transaction(\n        address,\n        w3,\n        fn_identifier=function_identifier,\n        contract_abi=contract_abi,\n        fn_abi=fn_abi,\n        transaction=transaction,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    return_data = w3.eth.call(\n        call_transaction,\n        block_identifier=block_id,\n        state_override=state_override,\n        ccip_read_enabled=ccip_read_enabled,\n    )\n\n    if fn_abi is None:\n        fn_abi = find_matching_fn_abi(\n            contract_abi, w3.codec, function_identifier, args, kwargs\n        )\n\n    output_types = get_abi_output_types(fn_abi)\n\n    try:\n        output_data = w3.codec.decode(output_types, return_data)\n    except DecodingError as e:\n        # Provide a more helpful error message than the one provided by\n        # eth-abi-utils\n        is_missing_code_error = (\n            return_data in ACCEPTABLE_EMPTY_STRINGS\n            and w3.eth.get_code(address) in ACCEPTABLE_EMPTY_STRINGS\n        )\n        if is_missing_code_error:\n            msg = (\n                \"Could not transact with/call contract function, is contract \"\n                \"deployed correctly and chain synced?\"\n            )\n        else:\n            msg = (\n                f\"Could not decode contract function call to {function_identifier} \"\n                f\"with return data: {str(return_data)}, output_types: {output_types}\"\n            )\n        raise BadFunctionCallOutput(msg) from e\n\n    _normalizers = itertools.chain(\n        BASE_RETURN_NORMALIZERS,\n        normalizers,\n    )\n    normalized_data = map_abi_data(_normalizers, output_types, output_data)\n\n    if decode_tuples:\n        decoded = named_tree(fn_abi[\"outputs\"], normalized_data)\n        normalized_data = recursive_dict_to_namedtuple(decoded)\n\n    if len(normalized_data) == 1:\n        return normalized_data[0]\n    else:\n        return normalized_data\n\n\ndef transact_with_contract_function(\n    address: ChecksumAddress,\n    w3: \"Web3\",\n    function_name: Optional[FunctionIdentifier] = None,\n    transaction: Optional[TxParams] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    *args: Any,\n    **kwargs: Any,\n) -> HexBytes:\n    \"\"\"\n    Helper function for interacting with a contract function by sending a\n    transaction.\n    \"\"\"\n    transact_transaction = prepare_transaction(\n        address,\n        w3,\n        fn_identifier=function_name,\n        contract_abi=contract_abi,\n        transaction=transaction,\n        fn_abi=fn_abi,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    txn_hash = w3.eth.send_transaction(transact_transaction)\n    return txn_hash\n\n\ndef estimate_gas_for_function(\n    address: ChecksumAddress,\n    w3: \"Web3\",\n    fn_identifier: Optional[FunctionIdentifier] = None,\n    transaction: Optional[TxParams] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    block_identifier: Optional[BlockIdentifier] = None,\n    state_override: Optional[StateOverride] = None,\n    *args: Any,\n    **kwargs: Any,\n) -> int:\n    \"\"\"Estimates gas cost a function call would take.\n\n    Don't call this directly, instead use :meth:`Contract.estimate_gas`\n    on your contract instance.\n    \"\"\"\n    estimate_transaction = prepare_transaction(\n        address,\n        w3,\n        fn_identifier=fn_identifier,\n        contract_abi=contract_abi,\n        fn_abi=fn_abi,\n        transaction=transaction,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    return w3.eth.estimate_gas(estimate_transaction, block_identifier, state_override)\n\n\ndef build_transaction_for_function(\n    address: ChecksumAddress,\n    w3: \"Web3\",\n    function_name: Optional[FunctionIdentifier] = None,\n    transaction: Optional[TxParams] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    *args: Any,\n    **kwargs: Any,\n) -> TxParams:\n    \"\"\"Builds a dictionary with the fields required to make the given transaction\n\n    Don't call this directly, instead use :meth:`Contract.build_transaction`\n    on your contract instance.\n    \"\"\"\n    prepared_transaction = prepare_transaction(\n        address,\n        w3,\n        fn_identifier=function_name,\n        contract_abi=contract_abi,\n        fn_abi=fn_abi,\n        transaction=transaction,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    prepared_transaction = fill_transaction_defaults(w3, prepared_transaction)\n\n    return prepared_transaction\n\n\ndef find_functions_by_identifier(\n    contract_abi: ABI,\n    w3: Union[\"Web3\", \"AsyncWeb3\"],\n    address: ChecksumAddress,\n    callable_check: Callable[..., Any],\n    function_type: Type[TContractFn],\n) -> List[TContractFn]:\n    fns_abi = filter_by_type(\"function\", contract_abi)\n    return [\n        function_type.factory(\n            fn_abi[\"name\"],\n            w3=w3,\n            contract_abi=contract_abi,\n            address=address,\n            function_identifier=fn_abi[\"name\"],\n            abi=fn_abi,\n        )\n        for fn_abi in fns_abi\n        if callable_check(fn_abi)\n    ]\n\n\ndef get_function_by_identifier(\n    fns: Sequence[TContractFn], identifier: str\n) -> TContractFn:\n    if len(fns) > 1:\n        raise ValueError(\n            f\"Found multiple functions with matching {identifier}. \" f\"Found: {fns!r}\"\n        )\n    elif len(fns) == 0:\n        raise ValueError(f\"Could not find any function with matching {identifier}\")\n    return fns[0]\n\n\n# --- async --- #\n\n\nasync def async_call_contract_function(\n    async_w3: \"AsyncWeb3\",\n    address: ChecksumAddress,\n    normalizers: Tuple[Callable[..., Any], ...],\n    function_identifier: FunctionIdentifier,\n    transaction: TxParams,\n    block_id: Optional[BlockIdentifier] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    state_override: Optional[StateOverride] = None,\n    ccip_read_enabled: Optional[bool] = None,\n    decode_tuples: Optional[bool] = False,\n    *args: Any,\n    **kwargs: Any,\n) -> Any:\n    \"\"\"\n    Helper function for interacting with a contract function using the\n    `eth_call` API.\n    \"\"\"\n    call_transaction = prepare_transaction(\n        address,\n        async_w3,\n        fn_identifier=function_identifier,\n        contract_abi=contract_abi,\n        fn_abi=fn_abi,\n        transaction=transaction,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    return_data = await async_w3.eth.call(\n        call_transaction,\n        block_identifier=block_id,\n        state_override=state_override,\n        ccip_read_enabled=ccip_read_enabled,\n    )\n\n    if fn_abi is None:\n        fn_abi = find_matching_fn_abi(\n            contract_abi, async_w3.codec, function_identifier, args, kwargs\n        )\n\n    output_types = get_abi_output_types(fn_abi)\n\n    try:\n        output_data = async_w3.codec.decode(output_types, return_data)\n    except DecodingError as e:\n        # Provide a more helpful error message than the one provided by\n        # eth-abi-utils\n        is_missing_code_error = (\n            return_data in ACCEPTABLE_EMPTY_STRINGS\n            and await async_w3.eth.get_code(address) in ACCEPTABLE_EMPTY_STRINGS\n        )\n        if is_missing_code_error:\n            msg = (\n                \"Could not transact with/call contract function, is contract \"\n                \"deployed correctly and chain synced?\"\n            )\n        else:\n            msg = (\n                f\"Could not decode contract function call to {function_identifier} \"\n                f\"with return data: {str(return_data)}, output_types: {output_types}\"\n            )\n        raise BadFunctionCallOutput(msg) from e\n\n    _normalizers = itertools.chain(\n        BASE_RETURN_NORMALIZERS,\n        normalizers,\n    )\n    normalized_data = map_abi_data(_normalizers, output_types, output_data)\n\n    if decode_tuples:\n        decoded = named_tree(fn_abi[\"outputs\"], normalized_data)\n        normalized_data = recursive_dict_to_namedtuple(decoded)\n\n    if len(normalized_data) == 1:\n        return normalized_data[0]\n    else:\n        return normalized_data\n\n\nasync def async_transact_with_contract_function(\n    address: ChecksumAddress,\n    async_w3: \"AsyncWeb3\",\n    function_name: Optional[FunctionIdentifier] = None,\n    transaction: Optional[TxParams] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    *args: Any,\n    **kwargs: Any,\n) -> HexBytes:\n    \"\"\"\n    Helper function for interacting with a contract function by sending a\n    transaction.\n    \"\"\"\n    transact_transaction = prepare_transaction(\n        address,\n        async_w3,\n        fn_identifier=function_name,\n        contract_abi=contract_abi,\n        transaction=transaction,\n        fn_abi=fn_abi,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    txn_hash = await async_w3.eth.send_transaction(transact_transaction)\n    return txn_hash\n\n\nasync def async_estimate_gas_for_function(\n    address: ChecksumAddress,\n    async_w3: \"AsyncWeb3\",\n    fn_identifier: Optional[FunctionIdentifier] = None,\n    transaction: Optional[TxParams] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    block_identifier: Optional[BlockIdentifier] = None,\n    state_override: Optional[StateOverride] = None,\n    *args: Any,\n    **kwargs: Any,\n) -> int:\n    \"\"\"Estimates gas cost a function call would take.\n\n    Don't call this directly, instead use :meth:`Contract.estimate_gas`\n    on your contract instance.\n    \"\"\"\n    estimate_transaction = prepare_transaction(\n        address,\n        async_w3,\n        fn_identifier=fn_identifier,\n        contract_abi=contract_abi,\n        fn_abi=fn_abi,\n        transaction=transaction,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    return await async_w3.eth.estimate_gas(\n        estimate_transaction, block_identifier, state_override\n    )\n\n\nasync def async_build_transaction_for_function(\n    address: ChecksumAddress,\n    async_w3: \"AsyncWeb3\",\n    function_name: Optional[FunctionIdentifier] = None,\n    transaction: Optional[TxParams] = None,\n    contract_abi: Optional[ABI] = None,\n    fn_abi: Optional[ABIFunction] = None,\n    *args: Any,\n    **kwargs: Any,\n) -> TxParams:\n    \"\"\"Builds a dictionary with the fields required to make the given transaction\n\n    Don't call this directly, instead use :meth:`Contract.build_transaction`\n    on your contract instance.\n    \"\"\"\n    prepared_transaction = prepare_transaction(\n        address,\n        async_w3,\n        fn_identifier=function_name,\n        contract_abi=contract_abi,\n        fn_abi=fn_abi,\n        transaction=transaction,\n        fn_args=args,\n        fn_kwargs=kwargs,\n    )\n\n    return await async_fill_transaction_defaults(async_w3, prepared_transaction)\n\n# Path: web3/contract/contract.py\nimport copy\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Dict,\n    Iterable,\n    List,\n    Optional,\n    Sequence,\n    Type,\n    cast,\n)\n\nfrom eth_typing import (\n    ChecksumAddress,\n)\nfrom eth_utils import (\n    combomethod,\n)\nfrom eth_utils.toolz import (\n    partial,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3._utils.abi import (\n    fallback_func_abi_exists,\n    filter_by_type,\n    receive_func_abi_exists,\n)\nfrom web3._utils.compat import (\n    Self,\n)\nfrom web3._utils.contracts import (\n    parse_block_identifier,\n)\nfrom web3._utils.datatypes import (\n    PropertyCheckingFactory,\n)\nfrom web3._utils.events import (\n    EventFilterBuilder,\n    get_event_data,\n)\nfrom web3._utils.filters import (\n    LogFilter,\n)\nfrom web3._utils.function_identifiers import (\n    FallbackFn,\n    ReceiveFn,\n)\nfrom web3._utils.normalizers import (\n    normalize_abi,\n    normalize_address,\n    normalize_bytecode,\n)\nfrom web3._utils.transactions import (\n    fill_transaction_defaults,\n)\nfrom web3.contract.base_contract import (\n    BaseContract,\n    BaseContractCaller,\n    BaseContractConstructor,\n    BaseContractEvent,\n    BaseContractEvents,\n    BaseContractFunction,\n    BaseContractFunctions,\n    NonExistentFallbackFunction,\n    NonExistentReceiveFunction,\n)\nfrom web3.contract.utils import (\n    build_transaction_for_function,\n    call_contract_function,\n    estimate_gas_for_function,\n    find_functions_by_identifier,\n    get_function_by_identifier,\n    transact_with_contract_function,\n)\nfrom web3.exceptions import (\n    ABIFunctionNotFound,\n    NoABIFound,\n    NoABIFunctionsFound,\n    Web3ValidationError,\n)\nfrom web3.types import (\n    ABI,\n    BlockIdentifier,\n    EventData,\n    StateOverride,\n    TxParams,\n)\nfrom web3.utils import (\n    get_abi_input_names,\n)\n\nif TYPE_CHECKING:\n    from ens import ENS  # noqa: F401\n    from web3 import Web3  # noqa: F401\n\n\nclass ContractEvent(BaseContractEvent):\n    # mypy types\n    w3: \"Web3\"\n\n    @combomethod\n    def get_logs(\n        self,\n        argument_filters: Optional[Dict[str, Any]] = None,\n        fromBlock: Optional[BlockIdentifier] = None,\n        toBlock: Optional[BlockIdentifier] = None,\n        block_hash: Optional[HexBytes] = None,\n    ) -> Iterable[EventData]:\n        \"\"\"Get events for this contract instance using eth_getLogs API.\n\n        This is a stateless method, as opposed to create_filter.\n        It can be safely called against nodes which do not provide\n        eth_newFilter API, like Infura nodes.\n\n        If there are many events,\n        like ``Transfer`` events for a popular token,\n        the Ethereum node might be overloaded and timeout\n        on the underlying JSON-RPC call.\n\n        Example - how to get all ERC-20 token transactions\n        for the latest 10 blocks:\n\n        .. code-block:: python\n\n            from = max(mycontract.web3.eth.block_number - 10, 1)\n            to = mycontract.web3.eth.block_number\n\n            events = mycontract.events.Transfer.get_logs(fromBlock=from, toBlock=to)\n\n            for e in events:\n                print(e[\"args\"][\"from\"],\n                    e[\"args\"][\"to\"],\n                    e[\"args\"][\"value\"])\n\n        The returned processed log values will look like:\n\n        .. code-block:: python\n\n            (\n                AttributeDict({\n                 'args': AttributeDict({}),\n                 'event': 'LogNoArguments',\n                 'logIndex': 0,\n                 'transactionIndex': 0,\n                 'transactionHash': HexBytes('...'),\n                 'address': '0xF2E246BB76DF876Cef8b38ae84130F4F55De395b',\n                 'blockHash': HexBytes('...'),\n                 'blockNumber': 3\n                }),\n                AttributeDict(...),\n                ...\n            )\n\n        See also: :func:`web3.middleware.filter.LocalFilterMiddleware`.\n\n        :param argument_filters: Filter by argument values. Indexed arguments are\n          filtered by the node while non-indexed arguments are filtered by the library.\n        :param fromBlock: block number or \"latest\", defaults to \"latest\"\n        :param toBlock: block number or \"latest\". Defaults to \"latest\"\n        :param block_hash: block hash. block_hash cannot be set at the\n          same time as fromBlock or toBlock\n        :yield: Tuple of :class:`AttributeDict` instances\n        \"\"\"\n        event_abi = self._get_event_abi()\n\n        # validate ``argument_filters`` if present\n        if argument_filters is not None:\n            event_arg_names = get_abi_input_names(event_abi)\n            if not all(arg in event_arg_names for arg in argument_filters.keys()):\n                raise Web3ValidationError(\n                    \"When filtering by argument names, all argument names must be \"\n                    \"present in the contract's event ABI.\"\n                )\n\n        _filter_params = self._get_event_filter_params(\n            event_abi, argument_filters, fromBlock, toBlock, block_hash\n        )\n        # call JSON-RPC API\n        logs = self.w3.eth.get_logs(_filter_params)\n\n        # convert raw binary data to Python proxy objects as described by ABI:\n        all_event_logs = tuple(\n            get_event_data(self.w3.codec, event_abi, entry) for entry in logs\n        )\n        filtered_logs = self._process_get_logs_argument_filters(\n            event_abi,\n            all_event_logs,\n            argument_filters,\n        )\n        sorted_logs = sorted(filtered_logs, key=lambda e: e[\"logIndex\"])\n        sorted_logs = sorted(sorted_logs, key=lambda e: e[\"blockNumber\"])\n        return sorted_logs\n\n    @combomethod\n    def create_filter(\n        self,\n        *,  # PEP 3102\n        argument_filters: Optional[Dict[str, Any]] = None,\n        fromBlock: Optional[BlockIdentifier] = None,\n        toBlock: BlockIdentifier = \"latest\",\n        address: Optional[ChecksumAddress] = None,\n        topics: Optional[Sequence[Any]] = None,\n    ) -> LogFilter:\n        \"\"\"\n        Create filter object that tracks logs emitted by this contract event.\n        \"\"\"\n        filter_builder = EventFilterBuilder(self._get_event_abi(), self.w3.codec)\n        self._set_up_filter_builder(\n            argument_filters,\n            fromBlock,\n            toBlock,\n            address,\n            topics,\n            filter_builder,\n        )\n        log_filter = filter_builder.deploy(self.w3)\n        log_filter.log_entry_formatter = get_event_data(\n            self.w3.codec, self._get_event_abi()\n        )\n        log_filter.builder = filter_builder\n\n        return log_filter\n\n    @combomethod\n    def build_filter(self) -> EventFilterBuilder:\n        builder = EventFilterBuilder(\n            self._get_event_abi(),\n            self.w3.codec,\n            formatter=get_event_data(self.w3.codec, self._get_event_abi()),\n        )\n        builder.address = self.address\n        return builder\n\n\nclass ContractEvents(BaseContractEvents):\n    def __init__(\n        self, abi: ABI, w3: \"Web3\", address: Optional[ChecksumAddress] = None\n    ) -> None:\n        super().__init__(abi, w3, ContractEvent, address)\n\n\nclass ContractFunction(BaseContractFunction):\n    # mypy types\n    w3: \"Web3\"\n\n    def __call__(self, *args: Any, **kwargs: Any) -> \"ContractFunction\":\n        clone = copy.copy(self)\n        if args is None:\n            clone.args = tuple()\n        else:\n            clone.args = args\n\n        if kwargs is None:\n            clone.kwargs = {}\n        else:\n            clone.kwargs = kwargs\n        clone._set_function_info()\n        return clone\n\n    @classmethod\n    def factory(cls, class_name: str, **kwargs: Any) -> Self:\n        return PropertyCheckingFactory(class_name, (cls,), kwargs)(kwargs.get(\"abi\"))\n\n    def call(\n        self,\n        transaction: Optional[TxParams] = None,\n        block_identifier: BlockIdentifier = None,\n        state_override: Optional[StateOverride] = None,\n        ccip_read_enabled: Optional[bool] = None,\n    ) -> Any:\n        \"\"\"\n        Execute a contract function call using the `eth_call` interface.\n\n        This method prepares a ``Caller`` object that exposes the contract\n        functions and public variables as callable Python functions.\n\n        Reading a public ``owner`` address variable example:\n\n        .. code-block:: python\n\n            ContractFactory = w3.eth.contract(\n                abi=wallet_contract_definition[\"abi\"]\n            )\n\n            # Not a real contract address\n            contract = ContractFactory(\"0x2f70d3d26829e412A602E83FE8EeBF80255AEeA5\")\n\n            # Read \"owner\" public variable\n            addr = contract.functions.owner().call()\n\n        :param transaction: Dictionary of transaction info for web3 interface\n        :param block_identifier: TODO\n        :param state_override TODO\n        :param ccip_read_enabled TODO\n        :return: ``Caller`` object that has contract public functions\n            and variables exposed as Python methods\n        \"\"\"\n        call_transaction = self._get_call_txparams(transaction)\n\n        block_id = parse_block_identifier(self.w3, block_identifier)\n\n        return call_contract_function(\n            self.w3,\n            self.address,\n            self._return_data_normalizers,\n            self.function_identifier,\n            call_transaction,\n            block_id,\n            self.contract_abi,\n            self.abi,\n            state_override,\n            ccip_read_enabled,\n            self.decode_tuples,\n            *self.args,\n            **self.kwargs,\n        )\n\n    def transact(self, transaction: Optional[TxParams] = None) -> HexBytes:\n        setup_transaction = self._transact(transaction)\n        return transact_with_contract_function(\n            self.address,\n            self.w3,\n            self.function_identifier,\n            setup_transaction,\n            self.contract_abi,\n            self.abi,\n            *self.args,\n            **self.kwargs,\n        )\n\n    def estimate_gas(\n        self,\n        transaction: Optional[TxParams] = None,\n        block_identifier: Optional[BlockIdentifier] = None,\n        state_override: Optional[StateOverride] = None,\n    ) -> int:\n        setup_transaction = self._estimate_gas(transaction)\n        return estimate_gas_for_function(\n            self.address,\n            self.w3,\n            self.function_identifier,\n            setup_transaction,\n            self.contract_abi,\n            self.abi,\n            block_identifier,\n            state_override,\n            *self.args,\n            **self.kwargs,\n        )\n\n    def build_transaction(self, transaction: Optional[TxParams] = None) -> TxParams:\n        built_transaction = self._build_transaction(transaction)\n        return build_transaction_for_function(\n            self.address,\n            self.w3,\n            self.function_identifier,\n            built_transaction,\n            self.contract_abi,\n            self.abi,\n            *self.args,\n            **self.kwargs,\n        )\n\n    @staticmethod\n    def get_fallback_function(\n        abi: ABI,\n        w3: \"Web3\",\n        address: Optional[ChecksumAddress] = None,\n    ) -> \"ContractFunction\":\n        if abi and fallback_func_abi_exists(abi):\n            return ContractFunction.factory(\n                \"fallback\",\n                w3=w3,\n                contract_abi=abi,\n                address=address,\n                function_identifier=FallbackFn,\n            )()\n        return cast(ContractFunction, NonExistentFallbackFunction())\n\n    @staticmethod\n    def get_receive_function(\n        abi: ABI,\n        w3: \"Web3\",\n        address: Optional[ChecksumAddress] = None,\n    ) -> \"ContractFunction\":\n        if abi and receive_func_abi_exists(abi):\n            return ContractFunction.factory(\n                \"receive\",\n                w3=w3,\n                contract_abi=abi,\n                address=address,\n                function_identifier=ReceiveFn,\n            )()\n        return cast(ContractFunction, NonExistentReceiveFunction())\n\n\nclass ContractFunctions(BaseContractFunctions):\n    def __init__(\n        self,\n        abi: ABI,\n        w3: \"Web3\",\n        address: Optional[ChecksumAddress] = None,\n        decode_tuples: Optional[bool] = False,\n    ) -> None:\n        super().__init__(abi, w3, ContractFunction, address, decode_tuples)\n\n    def __getattr__(self, function_name: str) -> \"ContractFunction\":\n        if self.abi is None:\n            raise NoABIFound(\n                \"There is no ABI found for this contract.\",\n            )\n        if \"_functions\" not in self.__dict__:\n            raise NoABIFunctionsFound(\n                \"The abi for this contract contains no function definitions. \",\n                \"Are you sure you provided the correct contract abi?\",\n            )\n        elif function_name not in self.__dict__[\"_functions\"]:\n            raise ABIFunctionNotFound(\n                f\"The function '{function_name}' was not found in this contract's abi.\",\n                \" Are you sure you provided the correct contract abi?\",\n            )\n        else:\n            return super().__getattribute__(function_name)\n\n\nclass Contract(BaseContract):\n    # mypy types\n    w3: \"Web3\"\n    functions: ContractFunctions = None\n    caller: \"ContractCaller\" = None\n\n    # Instance of :class:`ContractEvents` presenting available Event ABIs\n    events: ContractEvents = None\n\n    def __init__(self, address: Optional[ChecksumAddress] = None) -> None:\n        \"\"\"Create a new smart contract proxy object.\n        :param address: Contract address as 0x hex string\"\"\"\n        _w3 = self.w3\n        if _w3 is None:\n            raise AttributeError(\n                \"The `Contract` class has not been initialized.  Please use the \"\n                \"`web3.contract` interface to create your contract class.\"\n            )\n\n        if address:\n            self.address = normalize_address(cast(\"ENS\", _w3.ens), address)\n\n        if not self.address:\n            raise TypeError(\n                \"The address argument is required to instantiate a contract.\"\n            )\n\n        self.functions = ContractFunctions(\n            self.abi, _w3, self.address, decode_tuples=self.decode_tuples\n        )\n        self.caller = ContractCaller(\n            self.abi, _w3, self.address, decode_tuples=self.decode_tuples\n        )\n        self.events = ContractEvents(self.abi, _w3, self.address)\n        self.fallback = Contract.get_fallback_function(\n            self.abi,\n            _w3,\n            ContractFunction,\n            self.address,\n        )\n        self.receive = Contract.get_receive_function(\n            self.abi,\n            _w3,\n            ContractFunction,\n            self.address,\n        )\n\n    @classmethod\n    def factory(\n        cls, w3: \"Web3\", class_name: Optional[str] = None, **kwargs: Any\n    ) -> Type[Self]:\n        kwargs[\"w3\"] = w3\n\n        normalizers = {\n            \"abi\": normalize_abi,\n            \"address\": partial(normalize_address, w3.ens),\n            \"bytecode\": normalize_bytecode,\n            \"bytecode_runtime\": normalize_bytecode,\n        }\n\n        contract = cast(\n            Type[Self],\n            PropertyCheckingFactory(\n                class_name or cls.__name__,\n                (cls,),\n                kwargs,\n                normalizers=normalizers,\n            ),\n        )\n        contract.functions = ContractFunctions(\n            contract.abi, contract.w3, decode_tuples=contract.decode_tuples\n        )\n        contract.caller = ContractCaller(\n            contract.abi,\n            contract.w3,\n            contract.address,\n            decode_tuples=contract.decode_tuples,\n        )\n        contract.events = ContractEvents(contract.abi, contract.w3)\n        contract.fallback = Contract.get_fallback_function(\n            contract.abi,\n            contract.w3,\n            ContractFunction,\n        )\n        contract.receive = Contract.get_receive_function(\n            contract.abi,\n            contract.w3,\n            ContractFunction,\n        )\n\n        return contract\n\n    @classmethod\n    def constructor(cls, *args: Any, **kwargs: Any) -> \"ContractConstructor\":\n        \"\"\"\n        :param args: The contract constructor arguments as positional arguments\n        :param kwargs: The contract constructor arguments as keyword arguments\n        :return: a contract constructor object\n        \"\"\"\n        if cls.bytecode is None:\n            raise ValueError(\n                \"Cannot call constructor on a contract that does not have \"\n                \"'bytecode' associated with it\"\n            )\n\n        return ContractConstructor(cls.w3, cls.abi, cls.bytecode, *args, **kwargs)\n\n    @combomethod\n    def find_functions_by_identifier(\n        cls,\n        contract_abi: ABI,\n        w3: \"Web3\",\n        address: ChecksumAddress,\n        callable_check: Callable[..., Any],\n    ) -> List[\"ContractFunction\"]:\n        return cast(\n            List[\"ContractFunction\"],\n            find_functions_by_identifier(\n                contract_abi, w3, address, callable_check, ContractFunction\n            ),\n        )\n\n    @combomethod\n    def get_function_by_identifier(\n        cls, fns: Sequence[\"ContractFunction\"], identifier: str\n    ) -> \"ContractFunction\":\n        return get_function_by_identifier(fns, identifier)\n\n\nclass ContractCaller(BaseContractCaller):\n    # mypy types\n    w3: \"Web3\"\n\n    def __init__(\n        self,\n        abi: ABI,\n        w3: \"Web3\",\n        address: ChecksumAddress,\n        transaction: Optional[TxParams] = None,\n        block_identifier: BlockIdentifier = None,\n        ccip_read_enabled: Optional[bool] = None,\n        decode_tuples: Optional[bool] = False,\n    ) -> None:\n        super().__init__(abi, w3, address, decode_tuples=decode_tuples)\n\n        if self.abi:\n            if transaction is None:\n                transaction = {}\n\n            self._functions = filter_by_type(\"function\", self.abi)\n            for func in self._functions:\n                fn = ContractFunction.factory(\n                    func[\"name\"],\n                    w3=w3,\n                    contract_abi=self.abi,\n                    address=self.address,\n                    function_identifier=func[\"name\"],\n                    decode_tuples=decode_tuples,\n                )\n\n                caller_method = partial(\n                    self.call_function,\n                    fn,\n                    transaction=transaction,\n                    block_identifier=block_identifier,\n                    ccip_read_enabled=ccip_read_enabled,\n                )\n\n                setattr(self, func[\"name\"], caller_method)\n\n    def __call__(\n        self,\n        transaction: Optional[TxParams] = None,\n        block_identifier: BlockIdentifier = None,\n        ccip_read_enabled: Optional[bool] = None,\n    ) -> \"ContractCaller\":\n        if transaction is None:\n            transaction = {}\n\n        return type(self)(\n            self.abi,\n            self.w3,\n            self.address,\n            transaction=transaction,\n            block_identifier=block_identifier,\n            ccip_read_enabled=ccip_read_enabled,\n            decode_tuples=self.decode_tuples,\n        )\n\n\nclass ContractConstructor(BaseContractConstructor):\n    # mypy types\n    w3: \"Web3\"\n\n    @combomethod\n    def transact(self, transaction: Optional[TxParams] = None) -> HexBytes:\n        return self.w3.eth.send_transaction(self._get_transaction(transaction))\n\n    @combomethod\n    def build_transaction(self, transaction: Optional[TxParams] = None) -> TxParams:\n        \"\"\"\n        Build the transaction dictionary without sending\n        \"\"\"\n        built_transaction = self._build_transaction(transaction)\n        return fill_transaction_defaults(self.w3, built_transaction)\n\n    @combomethod\n    def estimate_gas(\n        self,\n        transaction: Optional[TxParams] = None,\n        block_identifier: Optional[BlockIdentifier] = None,\n    ) -> int:\n        transaction = self._estimate_gas(transaction)\n\n        return self.w3.eth.estimate_gas(transaction, block_identifier=block_identifier)\n\n# Path: web3/types.py\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Coroutine,\n    Dict,\n    List,\n    NewType,\n    Optional,\n    Sequence,\n    Type,\n    TypeVar,\n    Union,\n)\n\nfrom eth_typing import (\n    Address,\n    BlockNumber,\n    ChecksumAddress,\n    Hash32,\n    HexStr,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3._utils.compat import (\n    Literal,\n    NotRequired,\n    TypedDict,\n)\nfrom web3._utils.function_identifiers import (\n    FallbackFn,\n    ReceiveFn,\n)\n\nif TYPE_CHECKING:\n    from web3.contract.async_contract import AsyncContractFunction  # noqa: F401\n    from web3.contract.contract import ContractFunction  # noqa: F401\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\n\nTReturn = TypeVar(\"TReturn\")\nTParams = TypeVar(\"TParams\")\nTValue = TypeVar(\"TValue\")\n\nBlockParams = Literal[\"latest\", \"earliest\", \"pending\", \"safe\", \"finalized\"]\nBlockIdentifier = Union[BlockParams, BlockNumber, Hash32, HexStr, HexBytes, int]\nLatestBlockParam = Literal[\"latest\"]\n\nFunctionIdentifier = Union[str, Type[FallbackFn], Type[ReceiveFn]]\n\n# bytes, hexbytes, or hexstr representing a 32 byte hash\n_Hash32 = Union[Hash32, HexBytes, HexStr]\nEnodeURI = NewType(\"EnodeURI\", str)\nENS = NewType(\"ENS\", str)\nNonce = NewType(\"Nonce\", int)\nRPCEndpoint = NewType(\"RPCEndpoint\", str)\nTimestamp = NewType(\"Timestamp\", int)\nWei = NewType(\"Wei\", int)\nGwei = NewType(\"Gwei\", int)\nFormatters = Dict[RPCEndpoint, Callable[..., Any]]\n\n\nclass AccessListEntry(TypedDict):\n    address: HexStr\n    storageKeys: Sequence[HexStr]\n\n\nAccessList = NewType(\"AccessList\", Sequence[AccessListEntry])\n\n\n# todo: move these to eth_typing once web3 is type hinted\nclass ABIEventParams(TypedDict, total=False):\n    indexed: bool\n    name: str\n    type: str\n\n\n...\n", "needle_token_start": 5536, "needle_token_end": 5725, "code_context_ntokens": 16114, "output": ["```python\ndef _get_transaction(self, transaction: Optional[TxParams] = None) -> TxParams:\n    if transaction is None:\n        transact_transaction: TxParams = {}\n    else:\n        transact_transaction = cast(TxParams, dict(**transaction))\n        self.check_forbidden_keys_in_transaction(\n            transact_transaction, [\"data\", \"to\"]\n        )\n\n    if self.w3.eth.default_account is not empty:\n        # type ignored b/c check prevents an empty default_account\n        transact_transaction.setdefault(\n            \"from\", self.w3.eth.default_account  # type: ignore\n        )\n\n    transact_transaction[\"data\"] = self.data_in_transaction\n\n    return transact_transaction\n```"]}
{"repo": "ethereum/web3.py", "name": "test_async_eth_sign_transaction_legacy", "language": "python", "path": "web3/_utils/module_testing/eth_module.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To verify the correct signing and recovery of a transaction using legacy parameters in an asynchronous environment.\n2. **Input**: Transaction parameters including sender, recipient, value, gas, gas price, and nonce.\n3. **Output**: The signed transaction details and confirmation that the recovered signatory matches the original sender.\n4. **Procedure**: \n   - Define the transaction parameters.\n   - Sign the transaction asynchronously.\n   - Recover the signatory from the signed transaction.\n   - Validate that the recovered signatory matches the original sender and that all transaction parameters are correctly reflected in the signed transaction.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: web3/_utils/module_testing/utils.py\nfrom asyncio import (\n    iscoroutinefunction,\n)\nimport copy\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Union,\n    cast,\n)\n\nfrom toolz import (\n    merge,\n)\n\nif TYPE_CHECKING:\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n    from web3._utils.compat import (  # noqa: F401\n        Self,\n    )\n    from web3.types import (  # noqa: F401\n        AsyncMakeRequestFn,\n        MakeRequestFn,\n        RPCEndpoint,\n        RPCResponse,\n    )\n\n\nclass RequestMocker:\n    \"\"\"\n    Context manager to mock requests made by a web3 instance. This is meant to be used\n    via a ``request_mocker`` fixture defined within the appropriate context.\n\n    Example:\n\n        def test_my_w3(w3, request_mocker):\n            assert w3.eth.block_number == 0\n\n            with request_mocker(w3, mock_results={\"eth_blockNumber\": \"0x1\"}):\n                assert w3.eth.block_number == 1\n\n            assert w3.eth.block_number == 0\n\n    Example with async and a mocked response object:\n\n        async def test_my_w3(async_w3, request_mocker):\n            def _iter_responses():\n                    while True:\n                        yield {\"error\": {\"message\": \"transaction indexing in progress\"}}\n                        yield {\"error\": {\"message\": \"transaction indexing in progress\"}}\n                        yield {\"result\": {\"status\": \"0x1\"}}\n\n            iter_responses = _iter_responses()\n\n            async with request_mocker(\n                async_w3,\n                mock_responses={\n                    \"eth_getTransactionReceipt\": lambda *_: next(iter_responses)\n                },\n            ):\n                # assert that the first two error responses are handled and the result\n                # is eventually returned when present\n                assert await w3.eth.get_transaction_receipt(\"0x1\") == \"0x1\"\n\n\n    - ``mock_results`` is a dict mapping method names to the desired \"result\" object of\n        the RPC response.\n    - ``mock_errors`` is a dict mapping method names to the desired\n        \"error\" object of the RPC response.\n    -``mock_responses`` is a dict mapping method names to the entire RPC response\n        object. This can be useful if you wish to return an iterator which returns\n        different responses on each call to the method.\n\n    If a method name is not present in any of the dicts above, the request is made as\n    usual.\n    \"\"\"\n\n    def __init__(\n        self,\n        w3: Union[\"AsyncWeb3\", \"Web3\"],\n        mock_results: Dict[Union[\"RPCEndpoint\", str], Any] = None,\n        mock_errors: Dict[Union[\"RPCEndpoint\", str], Any] = None,\n        mock_responses: Dict[Union[\"RPCEndpoint\", str], Any] = None,\n    ):\n        self.w3 = w3\n...\n# Path: web3/_utils/module_testing/eth_module.py\nimport json\nimport math\nimport pytest\nfrom random import (\n    randint,\n)\nimport re\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    List,\n    Type,\n    Union,\n    cast,\n)\n\nimport eth_abi as abi\nfrom eth_typing import (\n    BlockNumber,\n    ChecksumAddress,\n    HexAddress,\n    HexStr,\n)\nfrom eth_utils import (\n    is_boolean,\n    is_bytes,\n    is_checksum_address,\n    is_dict,\n    is_integer,\n    is_list_like,\n    is_same_address,\n    is_string,\n    remove_0x_prefix,\n    to_bytes,\n)\nfrom eth_utils.toolz import (\n    assoc,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3._utils.empty import (\n    empty,\n)\nfrom web3._utils.ens import (\n    ens_addresses,\n)\nfrom web3._utils.error_formatters_utils import (\n    PANIC_ERROR_CODES,\n)\nfrom web3._utils.fee_utils import (\n    PRIORITY_FEE_MIN,\n)\nfrom web3._utils.method_formatters import (\n    to_hex_if_integer,\n)\nfrom web3._utils.module_testing.module_testing_utils import (\n    assert_contains_log,\n    async_mock_offchain_lookup_request_response,\n    flaky_geth_dev_mining,\n    mock_offchain_lookup_request_response,\n)\nfrom web3._utils.module_testing.utils import (\n    RequestMocker,\n)\nfrom web3._utils.type_conversion import (\n    to_hex_if_bytes,\n)\nfrom web3.exceptions import (\n    BlockNotFound,\n    ContractCustomError,\n    ContractLogicError,\n    ContractPanicError,\n    InvalidAddress,\n    InvalidTransaction,\n    MultipleFailedRequests,\n    NameNotFound,\n    OffchainLookup,\n    TimeExhausted,\n    TooManyRequests,\n    TransactionNotFound,\n    TransactionTypeMismatch,\n    Web3ValidationError,\n)\nfrom web3.middleware import (\n    ExtraDataToPOAMiddleware,\n)\nfrom web3.types import (\n    ENS,\n    BlockData,\n    FilterParams,\n    Nonce,\n    RPCEndpoint,\n    StateOverrideParams,\n    SyncStatus,\n    TxData,\n    TxParams,\n    Wei,\n)\n\nUNKNOWN_ADDRESS = ChecksumAddress(\n    HexAddress(HexStr(\"0xdEADBEeF00000000000000000000000000000000\"))\n)\n\nUNKNOWN_HASH = HexStr(\n    \"0xdeadbeef00000000000000000000000000000000000000000000000000000000\"\n)\n# \"test offchain lookup\" as an abi-encoded string\nOFFCHAIN_LOOKUP_TEST_DATA = \"0x0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001474657374206f6666636861696e206c6f6f6b7570000000000000000000000000\"  # noqa: E501\nOFFCHAIN_LOOKUP_4BYTE_DATA = \"0x556f1830\"\nOFFCHAIN_LOOKUP_RETURN_DATA = \"00000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000001c0da96d05a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002400000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000002c68747470733a2f2f776562332e70792f676174657761792f7b73656e6465727d2f7b646174617d2e6a736f6e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002568747470733a2f2f776562332e70792f676174657761792f7b73656e6465727d2e6a736f6e00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001474657374206f6666636861696e206c6f6f6b757000000000000000000000000000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001474657374206f6666636861696e206c6f6f6b7570000000000000000000000000\"  # noqa: E501\n# \"web3py\" as an abi-encoded string\nWEB3PY_AS_HEXBYTES = \"0x000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000067765623370790000000000000000000000000000000000000000000000000000\"  # noqa: E501\n\nRLP_ACCESS_LIST = [\n    (\n        \"0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae\",\n        (\n            \"0x0000000000000000000000000000000000000000000000000000000000000003\",\n            \"0x0000000000000000000000000000000000000000000000000000000000000007\",\n        ),\n    ),\n    (\"0xbb9bc244d798123fde783fcc1c72d3bb8c189413\", ()),\n]\n\nRPC_ACCESS_LIST = [\n    {\n        \"address\": \"0xde0b295669a9fd93d5f28d9ec85e40f4cb697bae\",\n        \"storageKeys\": (\n            \"0x0000000000000000000000000000000000000000000000000000000000000003\",\n            \"0x0000000000000000000000000000000000000000000000000000000000000007\",\n        ),\n    },\n    {\"address\": \"0xbb9bc244d798123fde783fcc1c72d3bb8c189413\", \"storageKeys\": ()},\n]\n\nif TYPE_CHECKING:\n    from _pytest.monkeypatch import MonkeyPatch  # noqa: F401\n\n    from web3.contract import Contract  # noqa: F401\n    from web3.main import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n\n\ndef abi_encoded_offchain_lookup_contract_address(\n    w3: Union[\"Web3\", \"AsyncWeb3\"], offchain_lookup_contract: \"Contract\"\n) -> HexAddress:\n    return HexAddress(\n        remove_0x_prefix(\n            w3.to_hex(\n                abi.encode(\n                    [\"address\"],\n                    [to_bytes(hexstr=offchain_lookup_contract.address)],\n                )\n            )\n        )\n    )\n\n\nclass AsyncEthModuleTest:\n    @pytest.mark.asyncio\n    async def test_eth_gas_price(self, async_w3: \"AsyncWeb3\") -> None:\n        gas_price = await async_w3.eth.gas_price\n\n        assert gas_price > 0\n\n    @pytest.mark.asyncio\n    async def test_is_connected(self, async_w3: \"AsyncWeb3\") -> None:\n        is_connected = await async_w3.is_connected()\n        assert is_connected is True\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_legacy(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": await async_w3.eth.gas_price,\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"gasPrice\"] == txn_params[\"gasPrice\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_modify_transaction_legacy(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": async_w3.to_wei(\n                1, \"gwei\"\n            ),  # must be greater than base_fee post London\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = await async_w3.eth.modify_transaction(\n            txn_hash, gasPrice=(cast(int, txn_params[\"gasPrice\"]) * 2), value=2\n        )\n        modified_txn = await async_w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert modified_txn[\"gasPrice\"] == cast(int, txn_params[\"gasPrice\"]) * 2\n\n    @pytest.mark.asyncio\n    async def test_eth_modify_transaction(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n            \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = await async_w3.eth.modify_transaction(\n            txn_hash,\n            value=2,\n            maxPriorityFeePerGas=(cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2),\n            maxFeePerGas=(cast(Wei, txn_params[\"maxFeePerGas\"]) * 2),\n        )\n        modified_txn = await async_w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert (\n            modified_txn[\"maxPriorityFeePerGas\"]\n            == cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2\n        )\n        assert modified_txn[\"maxFeePerGas\"] == cast(Wei, txn_params[\"maxFeePerGas\"]) * 2\n\n    @pytest.mark.asyncio\n    async def test_async_eth_sign_transaction(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n            \"nonce\": Nonce(0),\n        }\n        result = await async_w3.eth.sign_transaction(txn_params)\n        signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n        assert async_unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n        assert (\n            result[\"tx\"][\"maxPriorityFeePerGas\"] == txn_params[\"maxPriorityFeePerGas\"]\n        )\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_sign_typed_data(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n        async_skip_if_testrpc: Callable[[\"AsyncWeb3\"], None],\n    ) -> None:\n        validJSONMessage = \"\"\"\n            {\n                \"types\": {\n                    \"EIP712Domain\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"version\", \"type\": \"string\"},\n                        {\"name\": \"chainId\", \"type\": \"uint256\"},\n                        {\"name\": \"verifyingContract\", \"type\": \"address\"}\n                    ],\n                    \"Person\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"wallet\", \"type\": \"address\"}\n                    ],\n                    \"Mail\": [\n                        {\"name\": \"from\", \"type\": \"Person\"},\n                        {\"name\": \"to\", \"type\": \"Person\"},\n                        {\"name\": \"contents\", \"type\": \"string\"}\n                    ]\n                },\n                \"primaryType\": \"Mail\",\n                \"domain\": {\n                    \"name\": \"Ether Mail\",\n                    \"version\": \"1\",\n                    \"chainId\": \"0x01\",\n                    \"verifyingContract\": \"0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC\"\n                },\n                \"message\": {\n                    \"from\": {\n                        \"name\": \"Cow\",\n                        \"wallet\": \"0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826\"\n                    },\n                    \"to\": {\n                        \"name\": \"Bob\",\n                        \"wallet\": \"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB\"\n                    },\n                    \"contents\": \"Hello, Bob!\"\n                }\n            }\n        \"\"\"\n        async_skip_if_testrpc(async_w3)\n        signature = HexBytes(\n            await async_w3.eth.sign_typed_data(\n                async_unlocked_account_dual_type, json.loads(validJSONMessage)\n            )\n        )\n        assert len(signature) == 32 + 32 + 1\n\n    @pytest.mark.asyncio\n    async def test_invalid_eth_sign_typed_data(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n        async_skip_if_testrpc: Callable[[\"AsyncWeb3\"], None],\n    ) -> None:\n        async_skip_if_testrpc(async_w3)\n        invalid_typed_message = \"\"\"\n            {\n                \"types\": {\n                    \"EIP712Domain\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"version\", \"type\": \"string\"},\n                        {\"name\": \"chainId\", \"type\": \"uint256\"},\n                        {\"name\": \"verifyingContract\", \"type\": \"address\"}\n                    ],\n                    \"Person\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"wallet\", \"type\": \"address\"}\n                    ],\n                    \"Mail\": [\n                        {\"name\": \"from\", \"type\": \"Person\"},\n                        {\"name\": \"to\", \"type\": \"Person[2]\"},\n                        {\"name\": \"contents\", \"type\": \"string\"}\n                    ]\n                },\n                \"primaryType\": \"Mail\",\n                \"domain\": {\n                    \"name\": \"Ether Mail\",\n                    \"version\": \"1\",\n                    \"chainId\": \"0x01\",\n                    \"verifyingContract\": \"0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC\"\n                },\n                \"message\": {\n                    \"from\": {\n                        \"name\": \"Cow\",\n                        \"wallet\": \"0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826\"\n                    },\n                    \"to\": [{\n                        \"name\": \"Bob\",\n                        \"wallet\": \"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB\"\n                    }],\n                    \"contents\": \"Hello, Bob!\"\n                }\n            }\n        \"\"\"\n        with pytest.raises(\n            ValueError,\n            match=r\".*Expected 2 items for array type Person\\[2\\], got 1 items.*\",\n        ):\n            await async_w3.eth.sign_typed_data(\n                async_unlocked_account_dual_type, json.loads(invalid_typed_message)\n            )\n\n    @pytest.mark.asyncio\n    \nasync def test_async_eth_sign_transaction_legacy(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": await async_w3.eth.gas_price,\n            \"nonce\": Nonce(0),\n        }\n        result = await async_w3.eth.sign_transaction(txn_params)\n        signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n        assert async_unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"gasPrice\"] == txn_params[\"gasPrice\"]\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    async def test_async_eth_sign_transaction_hex_fees(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": hex(async_w3.to_wei(2, \"gwei\")),\n            \"maxPriorityFeePerGas\": hex(async_w3.to_wei(1, \"gwei\")),\n            \"nonce\": Nonce(0),\n        }\n        result = await async_w3.eth.sign_transaction(txn_params)\n        signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n        assert async_unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"maxFeePerGas\"] == int(str(txn_params[\"maxFeePerGas\"]), 16)\n        assert result[\"tx\"][\"maxPriorityFeePerGas\"] == int(\n            str(txn_params[\"maxPriorityFeePerGas\"]), 16\n        )\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    @pytest.mark.xfail(\n        reason=\"async name_to_address_middleware has not been implemented yet\"\n    )\n    async def test_async_eth_sign_transaction_ens_names(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        with ens_addresses(async_w3, {\"unlocked-account.eth\": async_unlocked_account}):\n            txn_params: TxParams = {\n                \"from\": \"unlocked-account.eth\",\n                \"to\": \"unlocked-account.eth\",\n                \"value\": Wei(1),\n                \"gas\": 21000,\n                \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n                \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n                \"nonce\": Nonce(0),\n            }\n            result = await async_w3.eth.sign_transaction(txn_params)\n            signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n            assert async_unlocked_account == signatory_account\n            assert result[\"tx\"][\"to\"] == async_unlocked_account\n            assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n            assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n            assert result[\"tx\"][\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n            assert (\n                result[\"tx\"][\"maxPriorityFeePerGas\"]\n                == txn_params[\"maxPriorityFeePerGas\"]\n            )\n            assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": async_w3.to_wei(3, \"gwei\"),\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n        assert txn[\"maxPriorityFeePerGas\"] == txn_params[\"maxPriorityFeePerGas\"]\n        assert txn[\"gasPrice\"] == txn_params[\"maxFeePerGas\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_default_fees(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert is_integer(txn[\"maxPriorityFeePerGas\"])\n        assert is_integer(txn[\"maxFeePerGas\"])\n        assert txn[\"gasPrice\"] == txn[\"maxFeePerGas\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_hex_fees(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": hex(250 * 10**9),\n            \"maxPriorityFeePerGas\": hex(2 * 10**9),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"maxFeePerGas\"] == 250 * 10**9\n        assert txn[\"maxPriorityFeePerGas\"] == 2 * 10**9\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_no_gas(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"maxFeePerGas\": Wei(250 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 121000  # 21000 + buffer\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_with_gas_price(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": Wei(1),\n            \"maxFeePerGas\": Wei(250 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        with pytest.raises(TransactionTypeMismatch):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_no_priority_fee(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(250 * 10**9),\n        }\n        with pytest.raises(\n            InvalidTransaction, match=\"maxPriorityFeePerGas must be defined\"\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_no_max_fee(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        maxPriorityFeePerGas = async_w3.to_wei(2, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": maxPriorityFeePerGas,\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n\n        block = await async_w3.eth.get_block(\"latest\")\n        assert txn[\"maxFeePerGas\"] == maxPriorityFeePerGas + 2 * block[\"baseFeePerGas\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_max_fee_less_than_tip(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        with pytest.raises(\n            InvalidTransaction, match=\"maxFeePerGas must be >= maxPriorityFeePerGas\"\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_validation_middleware_chain_id_mismatch(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        wrong_chain_id = 1234567890\n        actual_chain_id = await async_w3.eth.chain_id\n\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n            \"chainId\": wrong_chain_id,\n        }\n        with pytest.raises(\n            Web3ValidationError,\n            match=f\"The transaction declared chain ID {wrong_chain_id}, \"\n            f\"but the connected node is on {actual_chain_id}\",\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_ExtraDataToPOAMiddleware(\n        self, async_w3: \"AsyncWeb3\", request_mocker: Type[RequestMocker]\n    ) -> None:\n        async_w3.middleware_onion.inject(ExtraDataToPOAMiddleware, \"poa\", layer=0)\n        extra_data = f\"0x{'ff' * 33}\"\n\n        async with request_mocker(\n            async_w3,\n            mock_results={\"eth_getBlockByNumber\": {\"extraData\": extra_data}},\n        ):\n            block = await async_w3.eth.get_block(\"latest\")\n\n        assert \"extraData\" not in block\n        assert block[\"proofOfAuthorityData\"] == to_bytes(hexstr=extra_data)\n\n        # clean up\n        async_w3.middleware_onion.remove(\"poa\")\n\n    @pytest.mark.asyncio\n    async def test_eth_send_raw_transaction(self, async_w3: \"AsyncWeb3\") -> None:\n        # private key 0x3c2ab4e8f17a7dea191b8c991522660126d681039509dc3bb31af7c9bdb63518\n        # This is an unfunded account, but the transaction has a 0 gas price, so is\n        # valid. It never needs to be mined, we just want the transaction hash back\n        # to confirm.\n        # tx = {'to': '0x0000000000000000000000000000000000000000', 'value': 0,  'nonce': 1, 'gas': 21000, 'gasPrice': 0, 'chainId': 131277322940537}  # noqa: E501\n        # NOTE: nonce=1 to make txn unique from the non-async version of this test\n        raw_txn = HexBytes(\n            \"0xf8650180825208940000000000000000000000000000000000000000808086eecac466e115a0ffdd42d7dee4ac85427468bc616812e49432e285e4e8f5cd9381163ac3b28108a04ec6b0d89ecbd5e89b0399f336ad50f283fafd70e86593250bf5a2adfb93d17e\"  # noqa: E501\n        )\n        expected_hash = HexStr(\n            \"0x52b0ff9cb472f25872fa8ec6a62fa59454fc2ae7901cfcc6cc89d096f49b8fc1\"\n        )\n        txn_hash = await async_w3.eth.send_raw_transaction(raw_txn)\n        assert txn_hash == async_w3.to_bytes(hexstr=expected_hash)\n\n    @pytest.mark.asyncio\n    async def test_GasPriceStrategyMiddleware(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        two_gwei_in_wei = async_w3.to_wei(2, \"gwei\")\n\n        def gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return two_gwei_in_wei\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert txn[\"gasPrice\"] == two_gwei_in_wei\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    async def test_gas_price_strategy_middleware_hex_value(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        two_gwei_in_wei = async_w3.to_wei(2, \"gwei\")\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> str:\n            return hex(two_gwei_in_wei)\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)  # type: ignore\n\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert txn[\"gasPrice\"] == two_gwei_in_wei\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\n        \"max_fee\", (1000000000, None), ids=[\"with_max_fee\", \"without_max_fee\"]\n    )\n    async def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n        max_fee: Wei,\n    ) -> None:\n        max_priority_fee = async_w3.to_wei(1, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": max_priority_fee,\n        }\n        if max_fee is not None:\n            txn_params = assoc(txn_params, \"maxFeePerGas\", max_fee)\n\n        def gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return async_w3.to_wei(2, \"gwei\")\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        latest_block = await async_w3.eth.get_block(\"latest\")\n        assert (\n            txn[\"maxFeePerGas\"] == max_fee\n            if max_fee is not None\n            else 2 * latest_block[\"baseFeePerGas\"] + max_priority_fee\n        )\n        assert txn[\"maxPriorityFeePerGas\"] == max_priority_fee\n        assert txn[\"gasPrice\"] == txn[\"maxFeePerGas\"]\n\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    async def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn_no_tip(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1000000000),\n        }\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> Wei:\n            return async_w3.to_wei(2, \"gwei\")\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        with pytest.raises(\n            InvalidTransaction, match=\"maxPriorityFeePerGas must be defined\"\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    async def test_eth_estimate_gas(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        gas_estimate = await async_w3.eth.estimate_gas(\n            {\n                \"from\": async_unlocked_account_dual_type,\n                \"to\": async_unlocked_account_dual_type,\n                \"value\": Wei(1),\n            }\n        )\n        assert is_integer(gas_estimate)\n        assert gas_estimate > 0\n\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\n        \"params\",\n        (\n            {\n                \"nonce\": 1,  # int\n                \"balance\": 1,  # int\n                \"code\": HexStr(\"0x\"),  # HexStr\n                # with state\n                \"state\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n            {\n                \"nonce\": HexStr(\"0x1\"),  # HexStr\n                \"balance\": HexStr(\"0x1\"),  # HexStr\n                \"code\": b\"\\x00\",  # bytes\n                # with stateDiff\n                \"stateDiff\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n        ),\n    )\n    async def test_eth_estimate_gas_with_override_param_type_check(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_math_contract: \"Contract\",\n        params: StateOverrideParams,\n    ) -> None:\n        txn_params: TxParams = {\"from\": await async_w3.eth.coinbase}\n\n        # assert does not raise\n        await async_w3.eth.estimate_gas(\n            txn_params, None, {async_math_contract.address: params}\n        )\n\n    @pytest.mark.asyncio\n    async def test_eth_fee_history(self, async_w3: \"AsyncWeb3\") -> None:\n        fee_history = await async_w3.eth.fee_history(1, \"latest\", [50])\n        assert is_list_like(fee_history[\"baseFeePerGas\"])\n        assert is_list_like(fee_history[\"gasUsedRatio\"])\n        assert is_integer(fee_history[\"oldestBlock\"])\n        assert fee_history[\"oldestBlock\"] >= 0\n        assert is_list_like(fee_history[\"reward\"])\n        if len(fee_history[\"reward\"]) > 0:\n            assert is_list_like(fee_history[\"reward\"][0])\n\n    @pytest.mark.asyncio\n    async def test_eth_fee_history_with_integer(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        fee_history = await async_w3.eth.fee_history(\n            1, async_empty_block[\"number\"], [50]\n        )\n        assert is_list_like(fee_history[\"baseFeePerGas\"])\n        assert is_list_like(fee_history[\"gasUsedRatio\"])\n        assert is_integer(fee_history[\"oldestBlock\"])\n        assert fee_history[\"oldestBlock\"] >= 0\n        assert is_list_like(fee_history[\"reward\"])\n        if len(fee_history[\"reward\"]) > 0:\n            assert is_list_like(fee_history[\"reward\"][0])\n\n    @pytest.mark.asyncio\n    async def test_eth_fee_history_no_reward_percentiles(\n        self, async_w3: \"AsyncWeb3\"\n    ) -> None:\n        fee_history = await async_w3.eth.fee_history(1, \"latest\")\n        assert is_list_like(fee_history[\"baseFeePerGas\"])\n        assert is_list_like(fee_history[\"gasUsedRatio\"])\n        assert is_integer(fee_history[\"oldestBlock\"])\n        assert fee_history[\"oldestBlock\"] >= 0\n\n    @pytest.mark.asyncio\n    async def test_eth_max_priority_fee(self, async_w3: \"AsyncWeb3\") -> None:\n        max_priority_fee = await async_w3.eth.max_priority_fee\n        assert is_integer(max_priority_fee)\n\n    @pytest.mark.asyncio\n    async def test_eth_max_priority_fee_with_fee_history_calculation(\n        self, async_w3: \"AsyncWeb3\", request_mocker: Type[RequestMocker]\n    ) -> None:\n        async with request_mocker(\n            async_w3,\n            mock_errors={RPCEndpoint(\"eth_maxPriorityFeePerGas\"): {}},\n            mock_results={RPCEndpoint(\"eth_feeHistory\"): {\"reward\": [[0]]}},\n        ):\n            with pytest.warns(\n                UserWarning,\n                match=(\n                    \"There was an issue with the method eth_maxPriorityFeePerGas. \"\n                    \"Calculating using eth_feeHistory.\"\n                ),\n            ):\n                priority_fee = await async_w3.eth.max_priority_fee\n                assert is_integer(priority_fee)\n                assert priority_fee == PRIORITY_FEE_MIN\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByHash(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(async_empty_block[\"hash\"])\n        assert block[\"hash\"] == async_empty_block[\"hash\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByHash_not_found(self, async_w3: \"AsyncWeb3\") -> None:\n        with pytest.raises(BlockNotFound):\n            await async_w3.eth.get_block(UNKNOWN_HASH)\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByHash_pending(self, async_w3: \"AsyncWeb3\") -> None:\n        block = await async_w3.eth.get_block(\"pending\")\n        assert block[\"hash\"] is None\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_with_integer(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(async_empty_block[\"number\"])\n        assert block[\"number\"] == async_empty_block[\"number\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_latest(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        current_block_number = await async_w3.eth.block_number\n        block = await async_w3.eth.get_block(\"latest\")\n        assert block[\"number\"] == current_block_number\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_not_found(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        with pytest.raises(BlockNotFound):\n            await async_w3.eth.get_block(BlockNumber(12345))\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_pending(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        current_block_number = await async_w3.eth.block_number\n        block = await async_w3.eth.get_block(\"pending\")\n        assert block[\"number\"] == current_block_number + 1\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_earliest(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        genesis_block = await async_w3.eth.get_block(BlockNumber(0))\n        block = await async_w3.eth.get_block(\"earliest\")\n        assert block[\"number\"] == 0\n        assert block[\"hash\"] == genesis_block[\"hash\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_safe(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(\"safe\")\n        assert block is not None\n        assert isinstance(block[\"number\"], int)\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_finalized(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(\"finalized\")\n        assert block is not None\n        assert isinstance(block[\"number\"], int)\n\n    @pytest.mark.asyncio\n    async def test_eth_get_block_by_number_full_transactions(\n        self, async_w3: \"AsyncWeb3\", async_block_with_txn: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(async_block_with_txn[\"number\"], True)\n        transaction = cast(TxData, block[\"transactions\"][0])\n        assert transaction[\"hash\"] == async_block_with_txn[\"transactions\"][0]\n\n    @pytest.mark.asyncio\n    async def test_eth_get_raw_transaction(\n        self, async_w3: \"AsyncWeb3\", mined_txn_hash: HexStr\n    ) -> None:\n        raw_transaction = await async_w3.eth.get_raw_transaction(mined_txn_hash)\n        assert is_bytes(raw_transaction)\n\n    @pytest.mark.asyncio\n    async def test_eth_get_raw_transaction_raises_error(\n        self, async_w3: \"AsyncWeb3\"\n    ) -> None:\n        with pytest.raises(\n            TransactionNotFound, match=f\"Transaction with hash: '{UNKNOWN_HASH}'\"\n        ):\n            await async_w3.eth.get_raw_transaction(UNKNOWN_HASH)\n\n    @pytest.mark.asyncio\n    async def test_eth_get_raw_transaction_by_block(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_block_with_txn: BlockData,\n        async_unlocked_account_dual_type: ChecksumAddress,\n    ) -> None:\n        # eth_getRawTransactionByBlockNumberAndIndex: block identifier\n...\n", "needle_token_start": 7175, "needle_token_end": 7463, "code_context_ntokens": 16254, "output": ["```python\n@pytest.mark.asyncio\nasync def test_async_eth_sign_transaction_legacy(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": await async_w3.eth.gas_price,\n            \"nonce\": Nonce(0),\n        }\n        result = await async_w3.eth.sign_transaction(txn_params)\n        signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n        assert async_unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"gasPrice\"] == txn_params[\"gasPrice\"]\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n```"]}
{"repo": "ethereum/web3.py", "name": "data_tree_map", "language": "python", "path": "web3/_utils/abi.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to apply a given transformation function to each element within a structured data tree, where each element is associated with a specific data type.\n2. **Input**: The function takes two parameters: a transformation function that operates on a data type and its associated data, and a data tree where each element is a tuple of a data type and its corresponding data.\n3. **Output**: The output is a modified data tree where each element has been transformed by the provided function, maintaining the structure of the original data tree.\n4. **Procedure**: The function iterates over the data tree, applying the transformation function to each element. If an element is a tuple containing a data type and data (recognized as a specific data type structure), the transformation function is applied to it. The function is applied recursively to ensure all nested elements are processed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: web3/contract/async_contract.py\nimport copy\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Awaitable,\n    Callable,\n    Dict,\n    Iterable,\n    List,\n    Optional,\n    Sequence,\n    Type,\n    cast,\n)\n\nfrom eth_typing import (\n    ChecksumAddress,\n)\nfrom eth_utils import (\n    combomethod,\n)\nfrom eth_utils.toolz import (\n    partial,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3._utils.abi import (\n    fallback_func_abi_exists,\n    filter_by_type,\n    receive_func_abi_exists,\n)\nfrom web3._utils.async_transactions import (\n    async_fill_transaction_defaults,\n)\nfrom web3._utils.compat import (\n    Self,\n)\nfrom web3._utils.contracts import (\n    async_parse_block_identifier,\n)\nfrom web3._utils.datatypes import (\n    PropertyCheckingFactory,\n)\nfrom web3._utils.events import (\n    AsyncEventFilterBuilder,\n    get_event_data,\n)\nfrom web3._utils.filters import (\n    AsyncLogFilter,\n)\nfrom web3._utils.function_identifiers import (\n    FallbackFn,\n    ReceiveFn,\n)\nfrom web3._utils.normalizers import (\n    normalize_abi,\n    normalize_address_no_ens,\n    normalize_bytecode,\n)\nfrom web3.contract.base_contract import (\n    BaseContract,\n    BaseContractCaller,\n    BaseContractConstructor,\n    BaseContractEvent,\n    BaseContractEvents,\n    BaseContractFunction,\n    BaseContractFunctions,\n    NonExistentFallbackFunction,\n    NonExistentReceiveFunction,\n)\nfrom web3.contract.utils import (\n    async_build_transaction_for_function,\n    async_call_contract_function,\n    async_estimate_gas_for_function,\n...\n# Path: web3/contract/__init__.py\nfrom web3.contract.async_contract import (\n    AsyncContract,\n    AsyncContractCaller,\n)\nfrom web3.contract.contract import (\n    Contract,\n    ContractCaller,\n    ContractConstructor,\n)\n\n# Path: web3/_utils/ens.py\nfrom contextlib import (\n    contextmanager,\n)\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Iterator,\n    Union,\n    cast,\n)\n\nfrom eth_typing import (\n    ChecksumAddress,\n)\nfrom eth_utils import (\n    is_0x_prefixed,\n    is_hex,\n    is_hex_address,\n    to_checksum_address,\n)\n\nfrom ens import (\n    ENS,\n    AsyncENS,\n)\nfrom web3.exceptions import (\n    NameNotFound,\n)\n\nif TYPE_CHECKING:\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n        Web3,\n    )\n    from web3.contract import (  # noqa: F401\n        Contract,\n    )\n\n\ndef is_ens_name(value: Any) -> bool:\n    if not isinstance(value, str):\n        return False\n    elif is_hex_address(value):\n        return False\n    elif is_0x_prefixed(value) and is_hex(value):\n        return False\n    else:\n        return ENS.is_valid_name(value)\n\n\ndef validate_name_has_address(ens: ENS, name: str) -> ChecksumAddress:\n    addr = ens.address(name)\n    if addr:\n        return to_checksum_address(addr)\n    else:\n        raise NameNotFound(f\"Could not find address for name {name!r}\")\n\n\nclass StaticENS:\n    def __init__(self, name_addr_pairs: Dict[str, ChecksumAddress]) -> None:\n        self.registry = dict(name_addr_pairs)\n\n    def address(self, name: str) -> ChecksumAddress:\n        return self.registry.get(name, None)\n\n\n@contextmanager\ndef ens_addresses(\n    w3: Union[\"Web3\", \"AsyncWeb3\"], name_addr_pairs: Dict[str, ChecksumAddress]\n) -> Iterator[None]:\n    original_ens = w3.ens\n    if w3.provider.is_async:\n        w3.ens = cast(AsyncENS, StaticENS(name_addr_pairs))\n    else:\n        w3.ens = cast(ENS, StaticENS(name_addr_pairs))\n    yield\n    w3.ens = original_ens\n\n\n@contextmanager\ndef contract_ens_addresses(\n    contract: \"Contract\", name_addr_pairs: Dict[str, ChecksumAddress]\n) -> Iterator[None]:\n    \"\"\"\n    Use this context manager to temporarily resolve name/address pairs\n    supplied as the argument. For example:\n\n    with contract_ens_addresses(mycontract, [('resolve-as-1s.eth', '0x111...111')]):\n        # any contract call or transaction in here would only resolve the above ENS pair\n    \"\"\"\n    with ens_addresses(contract.w3, name_addr_pairs):\n        yield\n\n\n# --- async --- #\n\n\nasync def async_validate_name_has_address(\n    async_ens: AsyncENS, name: str\n) -> ChecksumAddress:\n    addr = await async_ens.address(name)\n    if not addr:\n        raise NameNotFound(f\"Could not find address for name {name!r}\")\n    return addr\n\n# Path: web3/_utils/abi.py\nimport binascii\nfrom collections import (\n    abc,\n    namedtuple,\n)\nimport copy\nimport itertools\nimport re\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Callable,\n    Collection,\n    Coroutine,\n    Dict,\n    Iterable,\n    List,\n    Mapping,\n    Optional,\n    Sequence,\n    Tuple,\n    Type,\n    Union,\n    cast,\n)\n\nfrom eth_abi import (\n    codec,\n    decoding,\n    encoding,\n)\nfrom eth_abi.base import (\n    parse_type_str,\n)\nfrom eth_abi.exceptions import (\n    ValueOutOfBounds,\n)\nfrom eth_abi.grammar import (\n    ABIType,\n    BasicType,\n    TupleType,\n    parse,\n)\nfrom eth_abi.registry import (\n    ABIRegistry,\n    BaseEquals,\n    registry as default_registry,\n)\nfrom eth_typing import (\n    HexStr,\n    TypeStr,\n)\nfrom eth_utils import (\n    decode_hex,\n    is_bytes,\n    is_list_like,\n    is_string,\n    is_text,\n    to_text,\n    to_tuple,\n)\nfrom eth_utils.abi import (\n    collapse_if_tuple,\n)\nfrom eth_utils.toolz import (\n    curry,\n    partial,\n    pipe,\n)\n\nfrom web3._utils.decorators import (\n    reject_recursive_repeats,\n)\nfrom web3._utils.ens import (\n    is_ens_name,\n)\nfrom web3._utils.formatters import (\n    recursive_map,\n)\nfrom web3.exceptions import (\n    FallbackNotFound,\n    MismatchedABI,\n)\nfrom web3.types import (\n    ABI,\n    ABIEvent,\n    ABIEventParams,\n    ABIFunction,\n    ABIFunctionParams,\n    TReturn,\n)\nfrom web3.utils import (  # public utils module\n    get_abi_input_names,\n)\n\nif TYPE_CHECKING:\n    from web3 import (  # noqa: F401\n        AsyncWeb3,\n    )\n\n\ndef filter_by_type(_type: str, contract_abi: ABI) -> List[Union[ABIFunction, ABIEvent]]:\n    return [abi for abi in contract_abi if abi[\"type\"] == _type]\n\n\ndef filter_by_name(name: str, contract_abi: ABI) -> List[Union[ABIFunction, ABIEvent]]:\n    return [\n        abi\n        for abi in contract_abi\n        if (\n            abi[\"type\"] not in (\"fallback\", \"constructor\", \"receive\")\n            and abi[\"name\"] == name\n        )\n    ]\n\n\ndef get_abi_input_types(abi: ABIFunction) -> List[str]:\n    if \"inputs\" not in abi and (abi[\"type\"] == \"fallback\" or abi[\"type\"] == \"receive\"):\n        return []\n    else:\n        return [collapse_if_tuple(cast(Dict[str, Any], arg)) for arg in abi[\"inputs\"]]\n\n\ndef get_abi_output_types(abi: ABIFunction) -> List[str]:\n    if abi[\"type\"] == \"fallback\":\n        return []\n    else:\n        return [collapse_if_tuple(cast(Dict[str, Any], arg)) for arg in abi[\"outputs\"]]\n\n\ndef get_receive_func_abi(contract_abi: ABI) -> ABIFunction:\n    receive_abis = filter_by_type(\"receive\", contract_abi)\n    if receive_abis:\n        return cast(ABIFunction, receive_abis[0])\n    else:\n        raise FallbackNotFound(\"No receive function was found in the contract ABI.\")\n\n\ndef get_fallback_func_abi(contract_abi: ABI) -> ABIFunction:\n    fallback_abis = filter_by_type(\"fallback\", contract_abi)\n    if fallback_abis:\n        return cast(ABIFunction, fallback_abis[0])\n    else:\n        raise FallbackNotFound(\"No fallback function was found in the contract ABI.\")\n\n\ndef fallback_func_abi_exists(contract_abi: ABI) -> List[Union[ABIFunction, ABIEvent]]:\n    return filter_by_type(\"fallback\", contract_abi)\n\n\ndef receive_func_abi_exists(contract_abi: ABI) -> List[Union[ABIFunction, ABIEvent]]:\n    return filter_by_type(\"receive\", contract_abi)\n\n\ndef get_indexed_event_inputs(event_abi: ABIEvent) -> List[ABIEventParams]:\n    return [arg for arg in event_abi[\"inputs\"] if arg[\"indexed\"] is True]\n\n\ndef exclude_indexed_event_inputs(event_abi: ABIEvent) -> List[ABIEventParams]:\n    return [arg for arg in event_abi[\"inputs\"] if arg[\"indexed\"] is False]\n\n\ndef get_normalized_abi_arg_type(abi_arg: ABIEventParams) -> str:\n    \"\"\"\n    Return the normalized type for the abi argument provided.\n    In order to account for tuple argument types, this abstraction\n    makes use of `collapse_if_tuple()` to collapse the appropriate component\n    types within a tuple type, if present.\n    \"\"\"\n    return collapse_if_tuple(dict(abi_arg))\n\n\ndef filter_by_argument_count(\n    num_arguments: int, contract_abi: ABI\n) -> List[Union[ABIFunction, ABIEvent]]:\n    return [abi for abi in contract_abi if len(abi[\"inputs\"]) == num_arguments]\n\n\ndef filter_by_argument_name(\n    argument_names: Collection[str], contract_abi: ABI\n) -> List[Union[ABIFunction, ABIEvent]]:\n    return [\n        abi\n        for abi in contract_abi\n        if set(argument_names).intersection(get_abi_input_names(abi))\n        == set(argument_names)\n    ]\n\n\nclass AddressEncoder(encoding.AddressEncoder):\n    @classmethod\n    def validate_value(cls, value: Any) -> None:\n        if is_ens_name(value):\n            return\n\n        super().validate_value(value)\n\n\nclass AcceptsHexStrEncoder(encoding.BaseEncoder):\n    subencoder_cls: Type[encoding.BaseEncoder] = None\n    is_strict: bool = None\n    is_big_endian: bool = False\n    data_byte_size: int = None\n    value_bit_size: int = None\n\n    def __init__(\n        self,\n        subencoder: encoding.BaseEncoder,\n        **kwargs: Dict[str, Any],\n    ) -> None:\n        super().__init__(**kwargs)\n        self.subencoder = subencoder\n        self.is_dynamic = subencoder.is_dynamic\n\n    @classmethod\n    def from_type_str(\n        cls, abi_type: TypeStr, registry: ABIRegistry\n    ) -> \"AcceptsHexStrEncoder\":\n        subencoder_cls = cls.get_subencoder_class()\n        # cast b/c expects BaseCoder but `from_type_string`\n        # restricted to BaseEncoder subclasses\n        subencoder = cast(\n            encoding.BaseEncoder, subencoder_cls.from_type_str(abi_type, registry)\n        )\n        return cls(subencoder)\n\n    @classmethod\n    def get_subencoder_class(cls) -> Type[encoding.BaseEncoder]:\n        if cls.subencoder_cls is None:\n            raise AttributeError(f\"No subencoder class is set. {cls.__name__}\")\n        return cls.subencoder_cls\n\n    def validate_value(self, value: Any) -> None:\n        normalized_value = self.validate_and_normalize(value)\n        self.subencoder.validate_value(normalized_value)\n\n    def encode(self, value: Any) -> bytes:\n        normalized_value = self.validate_and_normalize(value)\n        return self.subencoder.encode(normalized_value)\n\n    def validate_and_normalize(self, value: Any) -> HexStr:\n        if not is_bytes(value) and not is_text(value):\n            self.invalidate_value(value)\n\n        raw_value = value\n        if is_text(value):\n            try:\n                value = decode_hex(value)\n            except binascii.Error:\n                self.invalidate_value(\n                    value,\n                    msg=f\"{value} is an invalid hex string\",\n                )\n            else:\n                if raw_value[:2] != \"0x\" and self.is_strict:\n                    self.invalidate_value(\n                        raw_value, msg=\"hex string must be prefixed with 0x\"\n                    )\n\n        if self.is_strict and self.data_byte_size is not None:\n            if len(value) > self.data_byte_size:\n                self.invalidate_value(\n                    value,\n                    exc=ValueOutOfBounds,\n                    msg=f\"exceeds total byte size for bytes{self.data_byte_size} \"\n                    \"encoding\",\n                )\n            elif len(value) < self.data_byte_size:\n                self.invalidate_value(\n                    value,\n                    exc=ValueOutOfBounds,\n                    msg=f\"less than total byte size for bytes{self.data_byte_size} \"\n                    \"encoding\",\n                )\n\n        return value\n\n\nclass BytesEncoder(AcceptsHexStrEncoder):\n    subencoder_cls = encoding.BytesEncoder\n    is_strict = False\n\n\nclass ExactLengthBytesEncoder(BytesEncoder):\n    is_strict = True\n\n    def validate(self) -> None:\n        super().validate()\n        if self.value_bit_size is None:\n            raise ValueError(\"`value_bit_size` may not be none\")\n        if self.data_byte_size is None:\n            raise ValueError(\"`data_byte_size` may not be none\")\n        if self.is_big_endian is None:\n            raise ValueError(\"`is_big_endian` may not be none\")\n\n        if self.value_bit_size % 8 != 0:\n            raise ValueError(\n                f\"Invalid value bit size: {self.value_bit_size}. \"\n                \"Must be a multiple of 8\"\n            )\n\n        if self.value_bit_size > self.data_byte_size * 8:\n            raise ValueError(\"Value byte size exceeds data size\")\n\n    @parse_type_str(\"bytes\")\n    def from_type_str(\n        cls, abi_type: BasicType, registry: ABIRegistry\n    ) -> \"ExactLengthBytesEncoder\":\n        subencoder_cls = cls.get_subencoder_class()\n        subencoder = subencoder_cls.from_type_str(abi_type.to_type_str(), registry)\n        # type ignored b/c @parse_type_str decorator turns it into a classmethod,\n        # so mypy thinks cls(...) is a call to __call__, but actually calls __init__\n        return cls(  # type: ignore\n            subencoder,\n            value_bit_size=abi_type.sub * 8,\n            data_byte_size=abi_type.sub,\n        )\n\n\nclass ByteStringEncoder(AcceptsHexStrEncoder):\n    subencoder_cls = encoding.ByteStringEncoder\n    is_strict = False\n\n\nclass StrictByteStringEncoder(AcceptsHexStrEncoder):\n    subencoder_cls = encoding.ByteStringEncoder\n    is_strict = True\n\n\nclass TextStringEncoder(encoding.TextStringEncoder):\n    @classmethod\n    def validate_value(cls, value: Any) -> None:\n        if is_bytes(value):\n            try:\n                value = to_text(value)\n            except UnicodeDecodeError:\n                cls.invalidate_value(\n                    value,\n                    msg=\"not decodable as unicode string\",\n                )\n\n        super().validate_value(value)\n\n\ndef filter_by_encodability(\n    abi_codec: codec.ABIEncoder,\n    args: Sequence[Any],\n    kwargs: Dict[str, Any],\n    contract_abi: ABI,\n) -> List[ABIFunction]:\n    return [\n        cast(ABIFunction, function_abi)\n        for function_abi in contract_abi\n        if check_if_arguments_can_be_encoded(\n            cast(ABIFunction, function_abi), abi_codec, args, kwargs\n        )\n    ]\n\n\ndef check_if_arguments_can_be_encoded(\n    function_abi: ABIFunction,\n    abi_codec: codec.ABIEncoder,\n    args: Sequence[Any],\n    kwargs: Dict[str, Any],\n) -> bool:\n    try:\n        arguments = merge_args_and_kwargs(function_abi, args, kwargs)\n    except TypeError:\n        return False\n\n    if len(function_abi.get(\"inputs\", [])) != len(arguments):\n        return False\n\n    try:\n        types, aligned_args = get_aligned_abi_inputs(function_abi, arguments)\n    except TypeError:\n        return False\n\n    return all(\n        abi_codec.is_encodable(_type, arg) for _type, arg in zip(types, aligned_args)\n    )\n\n\ndef merge_args_and_kwargs(\n    function_abi: ABIFunction, args: Sequence[Any], kwargs: Dict[str, Any]\n) -> Tuple[Any, ...]:\n    \"\"\"\n    Takes a list of positional args (``args``) and a dict of keyword args\n    (``kwargs``) defining values to be passed to a call to the contract function\n    described by ``function_abi``.  Checks to ensure that the correct number of\n    args were given, no duplicate args were given, and no unknown args were\n    given.  Returns a list of argument values aligned to the order of inputs\n    defined in ``function_abi``.\n    \"\"\"\n    # Ensure the function is being applied to the correct number of args\n    if len(args) + len(kwargs) != len(function_abi.get(\"inputs\", [])):\n        raise TypeError(\n            f\"Incorrect argument count. Expected '{len(function_abi['inputs'])}'\"\n            f\". Got '{len(args) + len(kwargs)}'\"\n        )\n\n    # If no keyword args were given, we don't need to align them\n    if not kwargs:\n        return cast(Tuple[Any, ...], args)\n\n    kwarg_names = set(kwargs.keys())\n    sorted_arg_names = tuple(arg_abi[\"name\"] for arg_abi in function_abi[\"inputs\"])\n    args_as_kwargs = dict(zip(sorted_arg_names, args))\n\n    # Check for duplicate args\n    duplicate_args = kwarg_names.intersection(args_as_kwargs.keys())\n    if duplicate_args:\n        raise TypeError(\n            f\"{function_abi.get('name')}() got multiple values for argument(s) \"\n            f\"'{', '.join(duplicate_args)}'\"\n        )\n\n    # Check for unknown args\n    unknown_args = kwarg_names.difference(sorted_arg_names)\n    if unknown_args:\n        if function_abi.get(\"name\"):\n            raise TypeError(\n                f\"{function_abi.get('name')}() got unexpected keyword argument(s)\"\n                f\" '{', '.join(unknown_args)}'\"\n            )\n        raise TypeError(\n            f\"Type: '{function_abi.get('type')}' got unexpected keyword argument(s)\"\n            f\" '{', '.join(unknown_args)}'\"\n        )\n\n    # Sort args according to their position in the ABI and unzip them from their\n    # names\n    sorted_args = tuple(\n        zip(\n            *sorted(\n                itertools.chain(kwargs.items(), args_as_kwargs.items()),\n                key=lambda kv: sorted_arg_names.index(kv[0]),\n            )\n        )\n    )\n\n    if sorted_args:\n        return sorted_args[1]\n    else:\n        return tuple()\n\n\nTUPLE_TYPE_STR_RE = re.compile(r\"^(tuple)((\\[([1-9]\\d*\\b)?])*)??$\")\n\n\ndef get_tuple_type_str_parts(s: str) -> Optional[Tuple[str, Optional[str]]]:\n    \"\"\"\n    Takes a JSON ABI type string.  For tuple type strings, returns the separated\n    prefix and array dimension parts.  For all other strings, returns ``None``.\n    \"\"\"\n    match = TUPLE_TYPE_STR_RE.match(s)\n\n    if match is not None:\n        tuple_prefix = match.group(1)\n        tuple_dims = match.group(2)\n\n        return tuple_prefix, tuple_dims\n\n    return None\n\n\ndef _align_abi_input(arg_abi: ABIFunctionParams, arg: Any) -> Tuple[Any, ...]:\n    \"\"\"\n    Aligns the values of any mapping at any level of nesting in ``arg``\n    according to the layout of the corresponding abi spec.\n    \"\"\"\n    tuple_parts = get_tuple_type_str_parts(arg_abi[\"type\"])\n\n    if tuple_parts is None:\n        # Arg is non-tuple.  Just return value.\n        return arg\n\n    tuple_prefix, tuple_dims = tuple_parts\n    if tuple_dims is None:\n        # Arg is non-list tuple.  Each sub arg in `arg` will be aligned\n        # according to its corresponding abi.\n        sub_abis = arg_abi[\"components\"]\n    else:\n        num_dims = tuple_dims.count(\"[\")\n\n        # Arg is list tuple.  A non-list version of its abi will be used to\n        # align each element in `arg`.\n        new_abi = copy.copy(arg_abi)\n        new_abi[\"type\"] = tuple_prefix + \"[]\" * (num_dims - 1)\n\n        sub_abis = itertools.repeat(new_abi)  # type: ignore\n\n    if isinstance(arg, abc.Mapping):\n        # Arg is mapping.  Align values according to abi order.\n        aligned_arg = tuple(arg[abi[\"name\"]] for abi in sub_abis)\n    else:\n        aligned_arg = arg\n\n    if not is_list_like(aligned_arg):\n        raise TypeError(\n            f'Expected non-string sequence for \"{arg_abi.get(\"type\")}\" '\n            f\"component type: got {aligned_arg}\"\n        )\n\n    # convert NamedTuple to regular tuple\n    typing = tuple if isinstance(aligned_arg, tuple) else type(aligned_arg)\n\n    return typing(\n        _align_abi_input(sub_abi, sub_arg)\n        for sub_abi, sub_arg in zip(sub_abis, aligned_arg)\n    )\n\n\ndef get_aligned_abi_inputs(\n    abi: ABIFunction, args: Union[Tuple[Any, ...], Mapping[Any, Any]]\n) -> Tuple[Tuple[Any, ...], Tuple[Any, ...]]:\n    \"\"\"\n    Takes a function ABI (``abi``) and a sequence or mapping of args (``args``).\n    Returns a list of type strings for the function's inputs and a list of\n    arguments which have been aligned to the layout of those types.  The args\n    contained in ``args`` may contain nested mappings or sequences corresponding\n    to tuple-encoded values in ``abi``.\n    \"\"\"\n    input_abis = abi.get(\"inputs\", [])\n\n    if isinstance(args, abc.Mapping):\n        # `args` is mapping.  Align values according to abi order.\n        args = tuple(args[abi[\"name\"]] for abi in input_abis)\n\n    return (\n        # typed dict cannot be used w/ a normal Dict\n        # https://github.com/python/mypy/issues/4976\n        tuple(collapse_if_tuple(abi) for abi in input_abis),  # type: ignore\n        type(args)(_align_abi_input(abi, arg) for abi, arg in zip(input_abis, args)),\n    )\n\n\ndef get_constructor_abi(contract_abi: ABI) -> ABIFunction:\n    candidates = [abi for abi in contract_abi if abi[\"type\"] == \"constructor\"]\n    if len(candidates) == 1:\n        return candidates[0]\n    elif len(candidates) == 0:\n        return None\n    elif len(candidates) > 1:\n        raise ValueError(\"Found multiple constructors.\")\n    return None\n\n\nDYNAMIC_TYPES = [\"bytes\", \"string\"]\n\nINT_SIZES = range(8, 257, 8)\nBYTES_SIZES = range(1, 33)\nUINT_TYPES = [f\"uint{i}\" for i in INT_SIZES]\nINT_TYPES = [f\"int{i}\" for i in INT_SIZES]\nBYTES_TYPES = [f\"bytes{i}\" for i in BYTES_SIZES] + [\"bytes32.byte\"]\n\nSTATIC_TYPES = list(\n    itertools.chain(\n        [\"address\", \"bool\"],\n        UINT_TYPES,\n        INT_TYPES,\n        BYTES_TYPES,\n    )\n)\n\nBASE_TYPE_REGEX = \"|\".join(\n    (_type + \"(?![a-z0-9])\" for _type in itertools.chain(STATIC_TYPES, DYNAMIC_TYPES))\n)\n\nSUB_TYPE_REGEX = r\"\\[\" \"[0-9]*\" r\"\\]\"\n\nTYPE_REGEX = (\"^\" \"(?:{base_type})\" \"(?:(?:{sub_type})*)?\" \"$\").format(\n    base_type=BASE_TYPE_REGEX,\n    sub_type=SUB_TYPE_REGEX,\n)\n\n\ndef is_recognized_type(abi_type: TypeStr) -> bool:\n    return bool(re.match(TYPE_REGEX, abi_type))\n\n\ndef is_bool_type(abi_type: TypeStr) -> bool:\n    return abi_type == \"bool\"\n\n\ndef is_uint_type(abi_type: TypeStr) -> bool:\n    return abi_type in UINT_TYPES\n\n\ndef is_int_type(abi_type: TypeStr) -> bool:\n    return abi_type in INT_TYPES\n\n\ndef is_address_type(abi_type: TypeStr) -> bool:\n    return abi_type == \"address\"\n\n\ndef is_bytes_type(abi_type: TypeStr) -> bool:\n    return abi_type in BYTES_TYPES + [\"bytes\"]\n\n\ndef is_string_type(abi_type: TypeStr) -> bool:\n    return abi_type == \"string\"\n\n\n@curry\ndef is_length(target_length: int, value: abc.Sized) -> bool:\n    return len(value) == target_length\n\n\ndef size_of_type(abi_type: TypeStr) -> int:\n    \"\"\"\n    Returns size in bits of abi_type\n    \"\"\"\n    if \"string\" in abi_type:\n        return None\n    if \"byte\" in abi_type:\n        return None\n    if \"[\" in abi_type:\n        return None\n    if abi_type == \"bool\":\n        return 8\n    if abi_type == \"address\":\n        return 160\n    return int(re.sub(r\"\\D\", \"\", abi_type))\n\n\nEND_BRACKETS_OF_ARRAY_TYPE_REGEX = r\"\\[[^]]*\\]$\"\n\n\ndef sub_type_of_array_type(abi_type: TypeStr) -> str:\n    if not is_array_type(abi_type):\n        raise ValueError(f\"Cannot parse subtype of nonarray abi-type: {abi_type}\")\n\n    return re.sub(END_BRACKETS_OF_ARRAY_TYPE_REGEX, \"\", abi_type, 1)\n\n\ndef length_of_array_type(abi_type: TypeStr) -> int:\n    if not is_array_type(abi_type):\n        raise ValueError(f\"Cannot parse length of nonarray abi-type: {abi_type}\")\n\n    inner_brackets = (\n        re.search(END_BRACKETS_OF_ARRAY_TYPE_REGEX, abi_type).group(0).strip(\"[]\")\n    )\n    if not inner_brackets:\n        return None\n    else:\n        return int(inner_brackets)\n\n\nARRAY_REGEX = (\"^\" \"[a-zA-Z0-9_]+\" \"({sub_type})+\" \"$\").format(sub_type=SUB_TYPE_REGEX)\n\n\ndef is_array_type(abi_type: TypeStr) -> bool:\n    return bool(re.match(ARRAY_REGEX, abi_type))\n\n\nNAME_REGEX = \"[a-zA-Z_]\" \"[a-zA-Z0-9_]*\"\n\n\nENUM_REGEX = (\"^\" \"{lib_name}\" r\"\\.\" \"{enum_name}\" \"$\").format(\n    lib_name=NAME_REGEX, enum_name=NAME_REGEX\n)\n\n\ndef is_probably_enum(abi_type: TypeStr) -> bool:\n    return bool(re.match(ENUM_REGEX, abi_type))\n\n\n@to_tuple\ndef normalize_event_input_types(\n    abi_args: Collection[Union[ABIFunction, ABIEvent]]\n) -> Iterable[Union[ABIFunction, ABIEvent, Dict[TypeStr, Any]]]:\n    for arg in abi_args:\n        if is_recognized_type(arg[\"type\"]):\n            yield arg\n        elif is_probably_enum(arg[\"type\"]):\n            yield {k: \"uint8\" if k == \"type\" else v for k, v in arg.items()}\n        else:\n            yield arg\n\n\ndef abi_to_signature(abi: Union[ABIFunction, ABIEvent]) -> str:\n    function_signature = \"{fn_name}({fn_input_types})\".format(\n        fn_name=abi[\"name\"],\n        fn_input_types=\",\".join(\n            collapse_if_tuple(dict(arg))\n            for arg in normalize_event_input_types(abi.get(\"inputs\", []))\n        ),\n    )\n    return function_signature\n\n\n########################################################\n#\n#  Conditionally modifying data, tagged with ABI Types\n#\n########################################################\n\n\n@curry\ndef map_abi_data(\n    normalizers: Sequence[Callable[[TypeStr, Any], Tuple[TypeStr, Any]]],\n    types: Sequence[TypeStr],\n    data: Sequence[Any],\n) -> Any:\n    \"\"\"\n    This function will apply normalizers to your data, in the\n    context of the relevant types. Each normalizer is in the format:\n\n    def normalizer(datatype, data):\n        # Conditionally modify data\n        return (datatype, data)\n\n    Where datatype is a valid ABI type string, like \"uint\".\n\n    In case of an array, like \"bool[2]\", normalizer will receive `data`\n    as an iterable of typed data, like `[(\"bool\", True), (\"bool\", False)]`.\n\n    Internals\n    ---\n\n    This is accomplished by:\n\n    1. Decorating the data tree with types\n    2. Recursively mapping each of the normalizers to the data\n    3. Stripping the types back out of the tree\n    \"\"\"\n    pipeline = itertools.chain(\n        [abi_data_tree(types)],\n        map(data_tree_map, normalizers),\n        [partial(recursive_map, strip_abi_type)],\n    )\n\n    return pipe(data, *pipeline)\n\n\n@curry\ndef abi_data_tree(types: Sequence[TypeStr], data: Sequence[Any]) -> List[Any]:\n    \"\"\"\n    Decorate the data tree with pairs of (type, data). The pair tuple is actually an\n    ABITypedData, but can be accessed as a tuple.\n\n    As an example:\n\n    >>> abi_data_tree(types=[\"bool[2]\", \"uint\"], data=[[True, False], 0])\n    [(\"bool[2]\", [(\"bool\", True), (\"bool\", False)]), (\"uint256\", 0)]\n    \"\"\"\n    return [\n        abi_sub_tree(data_type, data_value)\n        for data_type, data_value in zip(types, data)\n    ]\n\n\n@curry\n\ndef data_tree_map(\n    func: Callable[[TypeStr, Any], Tuple[TypeStr, Any]], data_tree: Any\n) -> \"ABITypedData\":\n    \"\"\"\n    Map func to every ABITypedData element in the tree. func will\n    receive two args: abi_type, and data\n    \"\"\"\n\n    def map_to_typed_data(elements: Any) -> \"ABITypedData\":\n        if isinstance(elements, ABITypedData) and elements.abi_type is not None:\n            return ABITypedData(func(*elements))\n        else:\n            return elements\n\n    return recursive_map(map_to_typed_data, data_tree)\n\n\nclass ABITypedData(namedtuple(\"ABITypedData\", \"abi_type, data\")):\n    \"\"\"\n    This class marks data as having a certain ABI-type.\n\n    >>> a1 = ABITypedData(['address', addr1])\n    >>> a2 = ABITypedData(['address', addr2])\n    >>> addrs = ABITypedData(['address[]', [a1, a2]])\n\n    You can access the fields using tuple() interface, or with\n    attributes:\n\n    >>> assert a1.abi_type == a1[0]\n    >>> assert a1.data == a1[1]\n\n    Unlike a typical `namedtuple`, you initialize with a single\n    positional argument that is iterable, to match the init\n    interface of all other relevant collections.\n    \"\"\"\n\n    def __new__(cls, iterable: Iterable[Any]) -> \"ABITypedData\":\n        return super().__new__(cls, *iterable)\n\n\ndef abi_sub_tree(\n    type_str_or_abi_type: Optional[Union[TypeStr, ABIType]], data_value: Any\n) -> ABITypedData:\n    if type_str_or_abi_type is None:\n        return ABITypedData([None, data_value])\n\n    if isinstance(type_str_or_abi_type, TypeStr):\n        abi_type = parse(type_str_or_abi_type)\n    else:\n        abi_type = type_str_or_abi_type\n\n    # In the two special cases below, we rebuild the given data structures with\n    # annotated items\n    if abi_type.is_array:\n        # If type is array, determine item type and annotate all\n        # items in iterable with that type\n        item_type_str = abi_type.item_type.to_type_str()\n        value_to_annotate = [\n            abi_sub_tree(item_type_str, item_value) for item_value in data_value\n        ]\n    elif isinstance(abi_type, TupleType):\n        # Otherwise, if type is tuple, determine component types and annotate\n        # tuple components in iterable respectively with those types\n        value_to_annotate = type(data_value)(\n            abi_sub_tree(comp_type.to_type_str(), comp_value)\n            for comp_type, comp_value in zip(abi_type.components, data_value)\n        )\n    else:\n        value_to_annotate = data_value\n\n    return ABITypedData(\n        [\n            abi_type.to_type_str(),\n            value_to_annotate,\n        ]\n    )\n\n\ndef strip_abi_type(elements: Any) -> Any:\n    if isinstance(elements, ABITypedData):\n        return elements.data\n    else:\n        return elements\n\n\ndef build_non_strict_registry() -> ABIRegistry:\n    # We make a copy here just to make sure that eth-abi's default registry is not\n    # affected by our custom encoder subclasses\n    registry = default_registry.copy()\n\n    registry.unregister(\"address\")\n    registry.unregister(\"bytes<M>\")\n    registry.unregister(\"bytes\")\n    registry.unregister(\"string\")\n\n    registry.register(\n        BaseEquals(\"address\"),\n        AddressEncoder,\n        decoding.AddressDecoder,\n        label=\"address\",\n    )\n    registry.register(\n        BaseEquals(\"bytes\", with_sub=True),\n        BytesEncoder,\n        decoding.BytesDecoder,\n        label=\"bytes<M>\",\n    )\n    registry.register(\n        BaseEquals(\"bytes\", with_sub=False),\n        ByteStringEncoder,\n        decoding.ByteStringDecoder,\n        label=\"bytes\",\n    )\n    registry.register(\n        BaseEquals(\"string\"),\n        TextStringEncoder,\n        decoding.StringDecoder,\n        label=\"string\",\n    )\n    return registry\n\n\ndef build_strict_registry() -> ABIRegistry:\n    registry = default_registry.copy()\n\n    registry.unregister(\"address\")\n    registry.unregister(\"bytes<M>\")\n    registry.unregister(\"bytes\")\n    registry.unregister(\"string\")\n\n    registry.register(\n        BaseEquals(\"address\"),\n        AddressEncoder,\n        decoding.AddressDecoder,\n        label=\"address\",\n    )\n    registry.register(\n        BaseEquals(\"bytes\", with_sub=True),\n        ExactLengthBytesEncoder,\n        decoding.BytesDecoder,\n        label=\"bytes<M>\",\n    )\n    registry.register(\n        BaseEquals(\"bytes\", with_sub=False),\n        StrictByteStringEncoder,\n        decoding.ByteStringDecoder,\n        label=\"bytes\",\n    )\n    registry.register(\n        BaseEquals(\"string\"),\n        encoding.TextStringEncoder,\n        decoding.StringDecoder,\n        label=\"string\",\n    )\n    return registry\n\n\ndef named_tree(\n    abi: Iterable[Union[ABIFunctionParams, ABIFunction, ABIEvent, Dict[TypeStr, Any]]],\n    data: Iterable[Tuple[Any, ...]],\n) -> Dict[str, Any]:\n    \"\"\"\n    Convert function inputs/outputs or event data tuple to dict with names from ABI.\n    \"\"\"\n    names = [item[\"name\"] for item in abi]\n    items = [_named_subtree(*item) for item in zip(abi, data)]\n\n    return dict(zip(names, items))\n\n\ndef _named_subtree(\n    abi: Union[ABIFunctionParams, ABIFunction, ABIEvent, Dict[TypeStr, Any]],\n    data: Tuple[Any, ...],\n) -> Union[Dict[str, Any], Tuple[Any, ...], List[Any]]:\n    abi_type = parse(collapse_if_tuple(dict(abi)))\n\n    if abi_type.is_array:\n        item_type = abi_type.item_type.to_type_str()\n        item_abi = {**abi, \"type\": item_type, \"name\": \"\"}\n        items = [_named_subtree(item_abi, item) for item in data]\n        return items\n\n    elif isinstance(abi_type, TupleType):\n        abi = cast(ABIFunctionParams, abi)\n        names = [item[\"name\"] for item in abi[\"components\"]]\n        items = [_named_subtree(*item) for item in zip(abi[\"components\"], data)]\n\n        if len(names) == len(data):\n            return dict(zip(names, items))\n        else:\n            raise MismatchedABI(\n                f\"ABI fields {names} has length {len(names)} but received \"\n                f\"data {data} with length {len(data)}\"\n            )\n\n    return data\n\n\ndef recursive_dict_to_namedtuple(data: Dict[str, Any]) -> Tuple[Any, ...]:\n    def _dict_to_namedtuple(\n        value: Union[Dict[str, Any], List[Any]]\n    ) -> Union[Tuple[Any, ...], List[Any]]:\n        if not isinstance(value, dict):\n            return value\n\n        keys, values = zip(*value.items()) if value else ((), ())\n        return abi_decoded_namedtuple_factory(keys)(values)\n\n    return recursive_map(_dict_to_namedtuple, data)\n\n\ndef abi_decoded_namedtuple_factory(\n    fields: Tuple[Any, ...]\n) -> Callable[..., Tuple[Any, ...]]:\n    class ABIDecodedNamedTuple(namedtuple(\"ABIDecodedNamedTuple\", fields, rename=True)):  # type: ignore # noqa: E501\n        def __new__(self, args: Any) -> \"ABIDecodedNamedTuple\":\n            return super().__new__(self, *args)\n\n    return ABIDecodedNamedTuple\n\n\n# -- async -- #\n\n\nasync def async_data_tree_map(\n    async_w3: \"AsyncWeb3\",\n    func: Callable[\n        [\"AsyncWeb3\", TypeStr, Any], Coroutine[Any, Any, Tuple[TypeStr, Any]]\n    ],\n    data_tree: Any,\n) -> \"ABITypedData\":\n    \"\"\"\n    Map an awaitable method to every ABITypedData element in the tree.\n\n    The awaitable method should receive three positional args:\n        async_w3, abi_type, and data\n    \"\"\"\n\n    async def async_map_to_typed_data(elements: Any) -> \"ABITypedData\":\n        if isinstance(elements, ABITypedData) and elements.abi_type is not None:\n            formatted = await func(async_w3, *elements)\n            return ABITypedData(formatted)\n        else:\n            return elements\n\n    return await async_recursive_map(async_w3, async_map_to_typed_data, data_tree)\n\n\n@reject_recursive_repeats\nasync def async_recursive_map(\n    async_w3: \"AsyncWeb3\",\n    func: Callable[[Any], Coroutine[Any, Any, TReturn]],\n    data: Any,\n) -> TReturn:\n    \"\"\"\n    Apply an awaitable method to data and any collection items inside data\n    (using async_map_collection).\n\n    Define the awaitable method so that it only applies to the type of value that you\n    want it to apply to.\n    \"\"\"\n\n    async def async_recurse(item: Any) -> TReturn:\n        return await async_recursive_map(async_w3, func, item)\n\n    items_mapped = await async_map_if_collection(async_recurse, data)\n    return await func(items_mapped)\n\n\nasync def async_map_if_collection(\n    func: Callable[[Any], Coroutine[Any, Any, Any]], value: Any\n) -> Any:\n    \"\"\"\n    Apply an awaitable method to each element of a collection or value of a dictionary.\n    If the value is not a collection, return it unmodified.\n    \"\"\"\n\n    datatype = type(value)\n    if isinstance(value, Mapping):\n        return datatype({key: await func(val) for key, val in value.values()})\n    if is_string(value):\n        return value\n    elif isinstance(value, Iterable):\n        return datatype([await func(item) for item in value])\n    else:\n        return value\n\n# Path: web3/_utils/module.py\nimport inspect\nfrom io import (\n    UnsupportedOperation,\n)\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    List,\n    Optional,\n    Sequence,\n    Union,\n)\n\nfrom web3.exceptions import (\n    Web3ValidationError,\n)\nfrom web3.module import (\n    Module,\n)\n\nif TYPE_CHECKING:\n    from web3.main import BaseWeb3  # noqa: F401\n\n\ndef _validate_init_params_and_return_if_found(module_class: Any) -> List[str]:\n    init_params_raw = list(inspect.signature(module_class.__init__).parameters)\n    module_init_params = [\n        param for param in init_params_raw if param not in [\"self\", \"args\", \"kwargs\"]\n    ]\n\n    if len(module_init_params) > 1:\n        raise UnsupportedOperation(\n            \"A module class may accept a single `Web3` instance as the first \"\n            \"argument of its __init__() method. More than one argument found for \"\n            f\"{module_class.__name__}: {module_init_params}\"\n        )\n\n    return module_init_params\n\n\ndef attach_modules(\n    parent_module: Union[\"BaseWeb3\", \"Module\"],\n    module_definitions: Dict[str, Any],\n    w3: Optional[Union[\"BaseWeb3\", \"Module\"]] = None,\n) -> None:\n    for module_name, module_info in module_definitions.items():\n        module_info_is_list_like = isinstance(module_info, Sequence)\n\n        module_class = module_info[0] if module_info_is_list_like else module_info\n\n        if hasattr(parent_module, module_name):\n            raise AttributeError(\n                f\"Cannot set {parent_module} module named '{module_name}'. \"\n                \" The web3 object already has an attribute with that name\"\n            )\n\n        # The parent module is the ``Web3`` instance on first run of the loop and w3 is\n        # None. Thus, set w3 to the parent_module. The import needs to happen locally\n        # due to circular import issues.\n        if w3 is None:\n            from web3 import (\n                AsyncWeb3,\n                Web3,\n            )\n\n            if isinstance(parent_module, Web3) or isinstance(parent_module, AsyncWeb3):\n                w3 = parent_module\n\n        module_init_params = _validate_init_params_and_return_if_found(module_class)\n        if len(module_init_params) == 1:\n            # Modules that need access to the ``Web3`` instance may accept the\n            # instance as the first arg in their ``__init__()`` method. This is the\n            # case for any module that inherits from ``web3.module.Module``.\n            # e.g. def __init__(self, w3):\n            setattr(parent_module, module_name, module_class(w3))\n        else:\n            # Modules need not take in a ``Web3`` instance in\n            # their ``__init__()`` if not needed\n            setattr(parent_module, module_name, module_class())\n\n        if module_info_is_list_like:\n            if len(module_info) == 2:\n                submodule_definitions = module_info[1]\n                module = getattr(parent_module, module_name)\n                attach_modules(module, submodule_definitions, w3)\n            elif len(module_info) != 1:\n                raise Web3ValidationError(\n                    \"Module definitions can only have 1 or 2 elements.\"\n                )\n\n# Path: web3/geth.py\nfrom typing import (\n    Any,\n    Awaitable,\n    Callable,\n    Dict,\n    List,\n    Optional,\n    Tuple,\n)\n\nfrom eth_typing.encoding import (\n    HexStr,\n)\nfrom eth_typing.evm import (\n    ChecksumAddress,\n)\nfrom hexbytes.main import (\n    HexBytes,\n)\n\nfrom web3._utils.compat import (\n    Protocol,\n)\nfrom web3._utils.rpc_abi import (\n    RPC,\n)\nfrom web3.method import (\n    Method,\n    default_root_munger,\n)\nfrom web3.module import (\n    Module,\n)\nfrom web3.types import (\n    EnodeURI,\n    GethWallet,\n    NodeInfo,\n    Peer,\n    TxParams,\n    TxPoolContent,\n    TxPoolInspect,\n    TxPoolStatus,\n)\n\n\nclass UnlockAccountWrapper(Protocol):\n    def __call__(\n        self,\n        account: ChecksumAddress,\n        passphrase: str,\n        duration: Optional[int] = None,\n    ) -> bool:\n        pass\n\n\nclass GethPersonal(Module):\n    \"\"\"\n    https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-personal\n    \"\"\"\n\n    is_async = False\n\n    ec_recover: Method[Callable[[str, HexStr], ChecksumAddress]] = Method(\n        RPC.personal_ecRecover,\n        mungers=[default_root_munger],\n    )\n\n    import_raw_key: Method[Callable[[str, str], ChecksumAddress]] = Method(\n        RPC.personal_importRawKey,\n        mungers=[default_root_munger],\n    )\n\n    list_accounts: Method[Callable[[], List[ChecksumAddress]]] = Method(\n        RPC.personal_listAccounts,\n        is_property=True,\n    )\n\n    list_wallets: Method[Callable[[], List[GethWallet]]] = Method(\n        RPC.personal_listWallets,\n        is_property=True,\n    )\n\n    send_transaction: Method[Callable[[TxParams, str], HexBytes]] = Method(\n        RPC.personal_sendTransaction,\n        mungers=[default_root_munger],\n    )\n\n    sign: Method[Callable[[str, ChecksumAddress, Optional[str]], HexStr]] = Method(\n        RPC.personal_sign,\n        mungers=[default_root_munger],\n    )\n\n    sign_typed_data: Method[\n        Callable[[Dict[str, Any], ChecksumAddress, str], HexStr]\n    ] = Method(\n        RPC.personal_signTypedData,\n        mungers=[default_root_munger],\n    )\n\n    new_account: Method[Callable[[str], ChecksumAddress]] = Method(\n        RPC.personal_newAccount,\n        mungers=[default_root_munger],\n    )\n\n    lock_account: Method[Callable[[ChecksumAddress], bool]] = Method(\n        RPC.personal_lockAccount,\n        mungers=[default_root_munger],\n    )\n\n    unlock_account: Method[UnlockAccountWrapper] = Method(\n        RPC.personal_unlockAccount,\n        mungers=[default_root_munger],\n    )\n\n\nclass GethTxPool(Module):\n    \"\"\"\n    https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-txpool\n    \"\"\"\n\n    is_async = False\n\n    content: Method[Callable[[], TxPoolContent]] = Method(\n        RPC.txpool_content,\n        is_property=True,\n    )\n\n    inspect: Method[Callable[[], TxPoolInspect]] = Method(\n        RPC.txpool_inspect,\n        is_property=True,\n    )\n\n    status: Method[Callable[[], TxPoolStatus]] = Method(\n        RPC.txpool_status,\n        is_property=True,\n    )\n\n\nclass ServerConnection(Protocol):\n    def __call__(\n        self,\n        host: str = \"localhost\",\n        port: int = 8546,\n        cors: str = \"\",\n        apis: str = \"eth,net,web3\",\n    ) -> bool:\n        pass\n\n\ndef admin_start_params_munger(\n    _module: Module,\n    host: str = \"localhost\",\n    port: int = 8546,\n    cors: str = \"\",\n    apis: str = \"eth,net,web3\",\n) -> Tuple[str, int, str, str]:\n    return (host, port, cors, apis)\n\n\nclass GethAdmin(Module):\n    \"\"\"\n    https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-admin\n    \"\"\"\n\n    is_async = False\n\n    add_peer: Method[Callable[[EnodeURI], bool]] = Method(\n        RPC.admin_addPeer,\n        mungers=[default_root_munger],\n    )\n\n    datadir: Method[Callable[[], str]] = Method(\n        RPC.admin_datadir,\n        is_property=True,\n    )\n\n    node_info: Method[Callable[[], NodeInfo]] = Method(\n        RPC.admin_nodeInfo,\n        is_property=True,\n    )\n\n    peers: Method[Callable[[], List[Peer]]] = Method(\n        RPC.admin_peers,\n        is_property=True,\n    )\n\n    start_http: Method[ServerConnection] = Method(\n        RPC.admin_startHTTP,\n        mungers=[admin_start_params_munger],\n    )\n\n    start_ws: Method[ServerConnection] = Method(\n        RPC.admin_startWS,\n        mungers=[admin_start_params_munger],\n    )\n\n    stop_http: Method[Callable[[], bool]] = Method(\n        RPC.admin_stopHTTP,\n        is_property=True,\n    )\n\n    stop_ws: Method[Callable[[], bool]] = Method(\n        RPC.admin_stopWS,\n        is_property=True,\n    )\n\n\nclass Geth(Module):\n    personal: GethPersonal\n    admin: GethAdmin\n    txpool: GethTxPool\n\n\n# --- async --- #\n\n\nclass AsyncGethTxPool(Module):\n    \"\"\"\n    https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-txpool\n    \"\"\"\n\n    is_async = True\n\n    _content: Method[Callable[[], Awaitable[TxPoolContent]]] = Method(\n        RPC.txpool_content,\n        is_property=True,\n    )\n\n    async def content(self) -> TxPoolContent:\n        return await self._content()\n\n    _inspect: Method[Callable[[], Awaitable[TxPoolInspect]]] = Method(\n        RPC.txpool_inspect,\n        is_property=True,\n    )\n\n    async def inspect(self) -> TxPoolInspect:\n        return await self._inspect()\n\n    _status: Method[Callable[[], Awaitable[TxPoolStatus]]] = Method(\n        RPC.txpool_status,\n        is_property=True,\n    )\n\n    async def status(self) -> TxPoolStatus:\n        return await self._status()\n\n\nclass AsyncGethAdmin(Module):\n    \"\"\"\n    https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-admin\n    \"\"\"\n\n    is_async = True\n\n    _add_peer: Method[Callable[[EnodeURI], Awaitable[bool]]] = Method(\n        RPC.admin_addPeer,\n        mungers=[default_root_munger],\n    )\n\n    async def add_peer(self, node_url: EnodeURI) -> bool:\n        return await self._add_peer(node_url)\n\n    _datadir: Method[Callable[[], Awaitable[str]]] = Method(\n        RPC.admin_datadir,\n        is_property=True,\n    )\n\n    async def datadir(self) -> str:\n        return await self._datadir()\n\n    _node_info: Method[Callable[[], Awaitable[NodeInfo]]] = Method(\n        RPC.admin_nodeInfo,\n        is_property=True,\n    )\n\n    async def node_info(self) -> NodeInfo:\n        return await self._node_info()\n\n    _peers: Method[Callable[[], Awaitable[List[Peer]]]] = Method(\n        RPC.admin_peers,\n        is_property=True,\n    )\n\n    async def peers(self) -> List[Peer]:\n        return await self._peers()\n\n    # start_http and stop_http\n\n    _start_http: Method[Callable[[str, int, str, str], Awaitable[bool]]] = Method(\n        RPC.admin_startHTTP,\n        mungers=[admin_start_params_munger],\n    )\n\n    _stop_http: Method[Callable[[], Awaitable[bool]]] = Method(\n        RPC.admin_stopHTTP,\n        is_property=True,\n    )\n\n    async def start_http(\n        self,\n        host: str = \"localhost\",\n        port: int = 8546,\n        cors: str = \"\",\n        apis: str = \"eth,net,web3\",\n    ) -> bool:\n        return await self._start_http(host, port, cors, apis)\n\n    async def stop_http(self) -> bool:\n        return await self._stop_http()\n\n    # start_ws and stop_ws\n\n    _start_ws: Method[Callable[[str, int, str, str], Awaitable[bool]]] = Method(\n        RPC.admin_startWS,\n        mungers=[admin_start_params_munger],\n    )\n\n    _stop_ws: Method[Callable[[], Awaitable[bool]]] = Method(\n        RPC.admin_stopWS,\n        is_property=True,\n    )\n\n    async def start_ws(\n        self,\n        host: str = \"localhost\",\n        port: int = 8546,\n        cors: str = \"\",\n        apis: str = \"eth,net,web3\",\n    ) -> bool:\n        return await self._start_ws(host, port, cors, apis)\n\n    async def stop_ws(self) -> bool:\n        return await self._stop_ws()\n\n\nclass AsyncGethPersonal(Module):\n    \"\"\"\n    https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-personal\n    \"\"\"\n\n    is_async = True\n\n    # ec_recover\n\n    _ec_recover: Method[Callable[[str, HexStr], Awaitable[ChecksumAddress]]] = Method(\n        RPC.personal_ecRecover,\n        mungers=[default_root_munger],\n    )\n\n    async def ec_recover(self, message: str, signature: HexStr) -> ChecksumAddress:\n        return await self._ec_recover(message, signature)\n\n    # import_raw_key\n\n    _import_raw_key: Method[Callable[[str, str], Awaitable[ChecksumAddress]]] = Method(\n        RPC.personal_importRawKey,\n        mungers=[default_root_munger],\n    )\n\n    async def import_raw_key(\n        self, private_key: str, passphrase: str\n    ) -> ChecksumAddress:\n        return await self._import_raw_key(private_key, passphrase)\n\n    # list_accounts and list_wallets\n\n    _list_accounts: Method[Callable[[], Awaitable[List[ChecksumAddress]]]] = Method(\n        RPC.personal_listAccounts,\n        is_property=True,\n    )\n\n    _list_wallets: Method[Callable[[], Awaitable[List[GethWallet]]]] = Method(\n        RPC.personal_listWallets,\n        is_property=True,\n    )\n\n    async def list_accounts(self) -> List[ChecksumAddress]:\n        return await self._list_accounts()\n\n    async def list_wallets(self) -> List[GethWallet]:\n        return await self._list_wallets()\n\n    # send_transaction\n\n    _send_transaction: Method[Callable[[TxParams, str], Awaitable[HexBytes]]] = Method(\n        RPC.personal_sendTransaction,\n        mungers=[default_root_munger],\n    )\n\n    async def send_transaction(\n        self, transaction: TxParams, passphrase: str\n    ) -> HexBytes:\n        return await self._send_transaction(transaction, passphrase)\n\n    # sign and sign_typed_data\n\n    _sign: Method[\n        Callable[[str, ChecksumAddress, Optional[str]], Awaitable[HexStr]]\n    ] = Method(\n        RPC.personal_sign,\n        mungers=[default_root_munger],\n    )\n\n    _sign_typed_data: Method[\n        Callable[[Dict[str, Any], ChecksumAddress, str], Awaitable[HexStr]]\n    ] = Method(\n        RPC.personal_signTypedData,\n        mungers=[default_root_munger],\n    )\n\n    async def sign(\n        self, message: str, account: ChecksumAddress, passphrase: str\n    ) -> HexStr:\n        return await self._sign(message, account, passphrase)\n\n    async def sign_typed_data(\n        self, message: Dict[str, Any], account: ChecksumAddress, passphrase: str\n    ) -> HexStr:\n        return await self._sign_typed_data(message, account, passphrase)\n\n    # new_account, lock_account, and unlock_account\n...\n", "needle_token_start": 8725, "needle_token_end": 8895, "code_context_ntokens": 16020, "output": ["```python\n@curry\ndef map_abi_data(\n    normalizers: Sequence[Callable[[TypeStr, Any], Tuple[TypeStr, Any]]],\n    types: Sequence[TypeStr],\n    data: Sequence[Any],\n) -> Any:\n    \"\"\"\n    This function will apply normalizers to your data, in the\n    context of the relevant types. Each normalizer is in the format:\n\n    def normalizer(datatype, data):\n        # Conditionally modify data\n        return (datatype, data)\n\n    Where datatype is a valid ABI type string, like \"uint\".\n\n    In case of an array, like \"bool[2]\", normalizer will receive `data`\n    as an iterable of typed data, like `[(\"bool\", True), (\"bool\", False)]`.\n\n    Internals\n    ---\n\n    This is accomplished by:\n\n    1. Decorating the data tree with types\n    2. Recursively mapping each of the normalizers to the data\n    3. Stripping the types back out of the tree\n    \"\"\"\n    pipeline = itertools.chain(\n        [abi_data_tree(types)],\n        map(data_tree_map, normalizers),\n        [partial(recursive_map, strip_abi_type)],\n    )\n\n    return pipe(data, *pipeline)\n```"]}
{"repo": "ethereum/web3.py", "name": "test_gas_price_strategy_hex_value", "language": "python", "path": "web3/_utils/module_testing/eth_module.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To verify that the system correctly handles a gas price strategy function returning a hexadecimal string value for the gas price when sending a transaction.\n2. **Input**: The input includes a transaction parameter dictionary without a specified gas price but with a custom gas price strategy set to return a hexadecimal string.\n3. **Output**: The output is the validation that the transaction uses the gas price as defined by the hexadecimal string returned from the gas price strategy.\n4. **Procedure**: \n   - Define a transaction parameter dictionary without a specified gas price.\n   - Set a custom gas price strategy that returns a hexadecimal string representing the gas price.\n   - Send a transaction using these parameters.\n   - Retrieve and verify that the transaction's gas price matches the expected value converted from the hexadecimal string.\n   - Reset the gas price strategy to ensure no side effects for other tests.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        assert len(accounts) != 0\n        assert all((is_checksum_address(account) for account in accounts))\n        assert w3.eth.coinbase in accounts\n\n    def test_eth_block_number(self, w3: \"Web3\") -> None:\n        block_number = w3.eth.block_number\n        assert is_integer(block_number)\n        assert block_number >= 0\n\n    def test_eth_get_block_number(self, w3: \"Web3\") -> None:\n        block_number = w3.eth.get_block_number()\n        assert is_integer(block_number)\n        assert block_number >= 0\n\n    def test_eth_get_balance(self, w3: \"Web3\") -> None:\n        coinbase = w3.eth.coinbase\n\n        with pytest.raises(InvalidAddress):\n            w3.eth.get_balance(ChecksumAddress(HexAddress(HexStr(coinbase.lower()))))\n\n        balance = w3.eth.get_balance(coinbase)\n\n        assert is_integer(balance)\n        assert balance >= 0\n\n    def test_eth_get_balance_with_block_identifier(self, w3: \"Web3\") -> None:\n        miner_address = w3.eth.get_block(1)[\"miner\"]\n        balance_post_genesis = w3.eth.get_balance(miner_address, 1)\n        later_balance = w3.eth.get_balance(miner_address, \"latest\")\n\n        assert is_integer(balance_post_genesis)\n        assert is_integer(later_balance)\n        assert later_balance > balance_post_genesis\n\n    @pytest.mark.parametrize(\n        \"address, expect_success\",\n        [(\"test-address.eth\", True), (\"not-an-address.eth\", False)],\n    )\n    def test_eth_get_balance_with_ens_name(\n        self, w3: \"Web3\", address: ChecksumAddress, expect_success: bool\n    ) -> None:\n        with ens_addresses(w3, {\"test-address.eth\": w3.eth.accounts[0]}):\n            if expect_success:\n                balance = w3.eth.get_balance(address)\n                assert is_integer(balance)\n                assert balance >= 0\n            else:\n                with pytest.raises(NameNotFound):\n                    w3.eth.get_balance(address)\n\n    def test_eth_get_storage_at(self, w3: \"Web3\", storage_contract: \"Contract\") -> None:\n        storage_contract_address = storage_contract.address\n\n        slot_0 = w3.eth.get_storage_at(storage_contract_address, 0)\n        assert slot_0 == HexBytes(f\"0x{'00' * 32}\")\n\n        slot_1 = w3.eth.get_storage_at(storage_contract_address, 1)\n        assert slot_1 == HexBytes(f\"0x{'00' * 31}01\")\n\n        slot_2 = w3.eth.get_storage_at(storage_contract_address, 2)\n        assert slot_2 == HexBytes(f\"0x{'00' * 31}02\")\n\n        slot_3 = w3.eth.get_storage_at(storage_contract_address, 3)\n        assert slot_3 == HexBytes(\n            \"0x746872656500000000000000000000000000000000000000000000000000000a\"\n        )\n        assert bytes(slot_3[:5]) == b\"three\"\n\n        slot_4 = w3.eth.get_storage_at(storage_contract_address, 4)\n        assert slot_4 == HexBytes(\n            \"0x666f757200000000000000000000000000000000000000000000000000000008\"\n        )\n        assert bytes(slot_4[:4]) == b\"four\"\n\n    def test_eth_get_storage_at_ens_name(\n        self, w3: \"Web3\", storage_contract: \"Contract\"\n    ) -> None:\n        with ens_addresses(w3, {\"storage.eth\": storage_contract.address}):\n            storage = w3.eth.get_storage_at(ENS(\"storage.eth\"), 1)\n            assert storage == HexBytes(f\"0x{'00' * 31}01\")\n\n    def test_eth_get_storage_at_invalid_address(self, w3: \"Web3\") -> None:\n        coinbase = w3.eth.coinbase\n        with pytest.raises(InvalidAddress):\n            w3.eth.get_storage_at(\n                ChecksumAddress(HexAddress(HexStr(coinbase.lower()))), 0\n            )\n\n    def test_eth_get_transaction_count(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        transaction_count = w3.eth.get_transaction_count(unlocked_account_dual_type)\n        assert is_integer(transaction_count)\n        assert transaction_count >= 0\n\n    def test_eth_get_transaction_count_ens_name(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        with ens_addresses(\n            w3, {\"unlocked-acct-dual-type.eth\": unlocked_account_dual_type}\n        ):\n            transaction_count = w3.eth.get_transaction_count(\n                ENS(\"unlocked-acct-dual-type.eth\")\n            )\n            assert is_integer(transaction_count)\n            assert transaction_count >= 0\n\n    def test_eth_get_transaction_count_invalid_address(self, w3: \"Web3\") -> None:\n        coinbase = w3.eth.coinbase\n        with pytest.raises(InvalidAddress):\n            w3.eth.get_transaction_count(\n                ChecksumAddress(HexAddress(HexStr(coinbase.lower())))\n            )\n\n    def test_eth_getBlockTransactionCountByHash_empty_block(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        transaction_count = w3.eth.get_block_transaction_count(empty_block[\"hash\"])\n\n        assert is_integer(transaction_count)\n        assert transaction_count == 0\n\n    def test_eth_getBlockTransactionCountByNumber_empty_block(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        transaction_count = w3.eth.get_block_transaction_count(empty_block[\"number\"])\n\n        assert is_integer(transaction_count)\n        assert transaction_count == 0\n\n    def test_eth_getBlockTransactionCountByHash_block_with_txn(\n        self, w3: \"Web3\", block_with_txn: BlockData\n    ) -> None:\n        transaction_count = w3.eth.get_block_transaction_count(block_with_txn[\"hash\"])\n\n        assert is_integer(transaction_count)\n        assert transaction_count >= 1\n\n    def test_eth_getBlockTransactionCountByNumber_block_with_txn(\n        self, w3: \"Web3\", block_with_txn: BlockData\n    ) -> None:\n        transaction_count = w3.eth.get_block_transaction_count(block_with_txn[\"number\"])\n\n        assert is_integer(transaction_count)\n        assert transaction_count >= 1\n\n    def test_eth_getUncleCountByBlockHash(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        uncle_count = w3.eth.get_uncle_count(empty_block[\"hash\"])\n\n        assert is_integer(uncle_count)\n        assert uncle_count == 0\n\n    def test_eth_getUncleCountByBlockNumber(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        uncle_count = w3.eth.get_uncle_count(empty_block[\"number\"])\n\n        assert is_integer(uncle_count)\n        assert uncle_count == 0\n\n    def test_eth_get_code(\n        self, w3: \"Web3\", math_contract_address: ChecksumAddress\n    ) -> None:\n        code = w3.eth.get_code(math_contract_address)\n        assert isinstance(code, HexBytes)\n        assert len(code) > 0\n\n    def test_eth_get_code_ens_address(\n        self, w3: \"Web3\", math_contract_address: ChecksumAddress\n    ) -> None:\n        with ens_addresses(w3, {\"mathcontract.eth\": math_contract_address}):\n            code = w3.eth.get_code(ENS(\"mathcontract.eth\"))\n            assert isinstance(code, HexBytes)\n            assert len(code) > 0\n\n    def test_eth_get_code_invalid_address(\n        self, w3: \"Web3\", math_contract: \"Contract\"\n    ) -> None:\n        with pytest.raises(InvalidAddress):\n            w3.eth.get_code(\n                ChecksumAddress(HexAddress(HexStr(math_contract.address.lower())))\n            )\n\n    def test_eth_get_code_with_block_identifier(\n        self, w3: \"Web3\", emitter_contract: \"Contract\"\n    ) -> None:\n        code = w3.eth.get_code(\n            emitter_contract.address, block_identifier=w3.eth.block_number\n        )\n        assert isinstance(code, HexBytes)\n        assert len(code) > 0\n\n    def test_eth_create_access_list(\n        self,\n        w3: \"Web3\",\n        unlocked_account_dual_type: ChecksumAddress,\n        math_contract: \"Contract\",\n    ) -> None:\n        # Initialize transaction for gas estimation\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n\n        txn = math_contract.functions.incrementCounter(1).build_transaction(txn_params)\n\n        # create access list using data from transaction\n        response = w3.eth.create_access_list(\n            {\n                \"from\": unlocked_account_dual_type,\n                \"to\": math_contract.address,\n                \"data\": txn[\"data\"],\n            }\n        )\n\n        assert is_dict(response)\n        access_list = response[\"accessList\"]\n        assert len(access_list) > 0\n        assert access_list[0][\"address\"] is not None\n        assert is_checksum_address(access_list[0][\"address\"])\n        assert len(access_list[0][\"storageKeys\"][0]) == 32\n        assert int(response[\"gasUsed\"]) >= 0\n\n    def test_eth_sign(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        signature = w3.eth.sign(\n            unlocked_account_dual_type, text=\"Message t\u00f6 sign. Longer than hash!\"\n        )\n        assert is_bytes(signature)\n        assert len(signature) == 32 + 32 + 1\n\n        # test other formats\n        hexsign = w3.eth.sign(\n            unlocked_account_dual_type,\n            hexstr=HexStr(\n                \"0x4d6573736167652074c3b6207369676e2e204c6f6e676572207468616e206861736821\"  # noqa: E501\n            ),\n        )\n        assert hexsign == signature\n\n        intsign = w3.eth.sign(\n            unlocked_account_dual_type,\n            0x4D6573736167652074C3B6207369676E2E204C6F6E676572207468616E206861736821,\n        )\n        assert intsign == signature\n\n        bytessign = w3.eth.sign(\n            unlocked_account_dual_type, b\"Message t\\xc3\\xb6 sign. Longer than hash!\"\n        )\n        assert bytessign == signature\n\n        new_signature = w3.eth.sign(\n            unlocked_account_dual_type, text=\"different message is different\"\n        )\n        assert new_signature != signature\n\n    def test_eth_sign_ens_names(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        with ens_addresses(w3, {\"unlocked-acct.eth\": unlocked_account_dual_type}):\n            signature = w3.eth.sign(\n                \"unlocked-acct.eth\", text=\"Message t\u00f6 sign. Longer than hash!\"\n            )\n            assert is_bytes(signature)\n            assert len(signature) == 32 + 32 + 1\n\n    def test_eth_sign_typed_data(\n        self,\n        w3: \"Web3\",\n        unlocked_account_dual_type: ChecksumAddress,\n        skip_if_testrpc: Callable[[\"Web3\"], None],\n    ) -> None:\n        validJSONMessage = \"\"\"\n            {\n                \"types\": {\n                    \"EIP712Domain\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"version\", \"type\": \"string\"},\n                        {\"name\": \"chainId\", \"type\": \"uint256\"},\n                        {\"name\": \"verifyingContract\", \"type\": \"address\"}\n                    ],\n                    \"Person\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"wallet\", \"type\": \"address\"}\n                    ],\n                    \"Mail\": [\n                        {\"name\": \"from\", \"type\": \"Person\"},\n                        {\"name\": \"to\", \"type\": \"Person\"},\n                        {\"name\": \"contents\", \"type\": \"string\"}\n                    ]\n                },\n                \"primaryType\": \"Mail\",\n                \"domain\": {\n                    \"name\": \"Ether Mail\",\n                    \"version\": \"1\",\n                    \"chainId\": \"0x01\",\n                    \"verifyingContract\": \"0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC\"\n                },\n                \"message\": {\n                    \"from\": {\n                        \"name\": \"Cow\",\n                        \"wallet\": \"0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826\"\n                    },\n                    \"to\": {\n                        \"name\": \"Bob\",\n                        \"wallet\": \"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB\"\n                    },\n                    \"contents\": \"Hello, Bob!\"\n                }\n            }\n        \"\"\"\n        skip_if_testrpc(w3)\n        signature = HexBytes(\n            w3.eth.sign_typed_data(\n                unlocked_account_dual_type, json.loads(validJSONMessage)\n            )\n        )\n        assert len(signature) == 32 + 32 + 1\n\n    def test_invalid_eth_sign_typed_data(\n        self,\n        w3: \"Web3\",\n        unlocked_account_dual_type: ChecksumAddress,\n        skip_if_testrpc: Callable[[\"Web3\"], None],\n    ) -> None:\n        skip_if_testrpc(w3)\n        invalid_typed_message = \"\"\"\n            {\n                \"types\": {\n                    \"EIP712Domain\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"version\", \"type\": \"string\"},\n                        {\"name\": \"chainId\", \"type\": \"uint256\"},\n                        {\"name\": \"verifyingContract\", \"type\": \"address\"}\n                    ],\n                    \"Person\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"wallet\", \"type\": \"address\"}\n                    ],\n                    \"Mail\": [\n                        {\"name\": \"from\", \"type\": \"Person\"},\n                        {\"name\": \"to\", \"type\": \"Person[2]\"},\n                        {\"name\": \"contents\", \"type\": \"string\"}\n                    ]\n                },\n                \"primaryType\": \"Mail\",\n                \"domain\": {\n                    \"name\": \"Ether Mail\",\n                    \"version\": \"1\",\n                    \"chainId\": \"0x01\",\n                    \"verifyingContract\": \"0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC\"\n                },\n                \"message\": {\n                    \"from\": {\n                        \"name\": \"Cow\",\n                        \"wallet\": \"0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826\"\n                    },\n                    \"to\": [{\n                        \"name\": \"Bob\",\n                        \"wallet\": \"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB\"\n                    }],\n                    \"contents\": \"Hello, Bob!\"\n                }\n            }\n        \"\"\"\n        with pytest.raises(\n            ValueError,\n            match=r\".*Expected 2 items for array type Person\\[2\\], got 1 items.*\",\n        ):\n            w3.eth.sign_typed_data(\n                unlocked_account_dual_type, json.loads(invalid_typed_message)\n            )\n\n    def test_eth_sign_transaction_legacy(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.eth.gas_price,\n            \"nonce\": Nonce(0),\n        }\n        result = w3.eth.sign_transaction(txn_params)\n        signatory_account = w3.eth.account.recover_transaction(result[\"raw\"])\n        assert unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"gasPrice\"] == txn_params[\"gasPrice\"]\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    def test_eth_sign_transaction(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n            \"nonce\": Nonce(0),\n        }\n        result = w3.eth.sign_transaction(txn_params)\n        signatory_account = w3.eth.account.recover_transaction(result[\"raw\"])\n        assert unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n        assert (\n            result[\"tx\"][\"maxPriorityFeePerGas\"] == txn_params[\"maxPriorityFeePerGas\"]\n        )\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    def test_eth_sign_transaction_hex_fees(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": hex(w3.to_wei(2, \"gwei\")),\n            \"maxPriorityFeePerGas\": hex(w3.to_wei(1, \"gwei\")),\n            \"nonce\": Nonce(0),\n        }\n        result = w3.eth.sign_transaction(txn_params)\n        signatory_account = w3.eth.account.recover_transaction(result[\"raw\"])\n        assert unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"maxFeePerGas\"] == int(str(txn_params[\"maxFeePerGas\"]), 16)\n        assert result[\"tx\"][\"maxPriorityFeePerGas\"] == int(\n            str(txn_params[\"maxPriorityFeePerGas\"]), 16\n        )\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    def test_eth_sign_transaction_ens_names(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        with ens_addresses(w3, {\"unlocked-account.eth\": unlocked_account}):\n            txn_params: TxParams = {\n                \"from\": \"unlocked-account.eth\",\n                \"to\": \"unlocked-account.eth\",\n                \"value\": Wei(1),\n                \"gas\": 21000,\n                \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n                \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n                \"nonce\": Nonce(0),\n            }\n            result = w3.eth.sign_transaction(txn_params)\n            signatory_account = w3.eth.account.recover_transaction(result[\"raw\"])\n            assert unlocked_account == signatory_account\n            assert result[\"tx\"][\"to\"] == unlocked_account\n            assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n            assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n            assert result[\"tx\"][\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n            assert (\n                result[\"tx\"][\"maxPriorityFeePerGas\"]\n                == txn_params[\"maxPriorityFeePerGas\"]\n            )\n            assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    def test_eth_send_transaction_addr_checksum_required(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        non_checksum_addr = unlocked_account.lower()\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n\n        with pytest.raises(InvalidAddress):\n            invalid_params = cast(\n                TxParams, dict(txn_params, **{\"from\": non_checksum_addr})\n            )\n            w3.eth.send_transaction(invalid_params)\n\n        with pytest.raises(InvalidAddress):\n            invalid_params = cast(\n                TxParams, dict(txn_params, **{\"to\": non_checksum_addr})\n            )\n            w3.eth.send_transaction(invalid_params)\n\n    def test_eth_send_transaction_legacy(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(\n                1, \"gwei\"\n            ),  # post-london needs to be more than the base fee\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"gasPrice\"] == txn_params[\"gasPrice\"]\n\n    def test_eth_send_transaction(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(3, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n        assert txn[\"maxPriorityFeePerGas\"] == txn_params[\"maxPriorityFeePerGas\"]\n        assert txn[\"gasPrice\"] == txn_params[\"maxFeePerGas\"]\n\n    def test_eth_send_transaction_with_nonce(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        max_priority_fee_per_gas = w3.to_wei(1.234, \"gwei\")\n        max_fee_per_gas = Wei(\n            w3.eth.get_block(\"latest\")[\"baseFeePerGas\"] + max_priority_fee_per_gas\n        )\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": max_fee_per_gas,\n            \"maxPriorityFeePerGas\": max_priority_fee_per_gas,\n            \"nonce\": Nonce(w3.eth.get_transaction_count(unlocked_account, \"pending\")),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n        assert txn[\"maxPriorityFeePerGas\"] == txn_params[\"maxPriorityFeePerGas\"]\n        assert txn[\"nonce\"] == txn_params[\"nonce\"]\n        assert is_integer(txn[\"gasPrice\"])\n        assert is_integer(txn_params[\"maxFeePerGas\"])\n\n    def test_eth_send_transaction_default_fees(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert is_integer(txn[\"maxPriorityFeePerGas\"])\n        assert is_integer(txn[\"maxFeePerGas\"])\n        assert txn[\"gasPrice\"] == txn[\"maxFeePerGas\"]\n\n    def test_eth_send_transaction_hex_fees(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": hex(250 * 10**9),\n            \"maxPriorityFeePerGas\": hex(2 * 10**9),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"maxFeePerGas\"] == 250 * 10**9\n        assert txn[\"maxPriorityFeePerGas\"] == 2 * 10**9\n\n    def test_eth_send_transaction_no_gas(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"maxFeePerGas\": Wei(250 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 121000  # 21000 + buffer\n\n    def test_eth_send_transaction_with_gas_price(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": Wei(1),\n            \"maxFeePerGas\": Wei(250 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        with pytest.raises(TransactionTypeMismatch):\n            w3.eth.send_transaction(txn_params)\n\n    def test_eth_send_transaction_no_priority_fee(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(250 * 10**9),\n        }\n        with pytest.raises(\n            InvalidTransaction, match=\"maxPriorityFeePerGas must be defined\"\n        ):\n            w3.eth.send_transaction(txn_params)\n\n    def test_eth_send_transaction_no_max_fee(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        max_priority_fee_per_gas = w3.to_wei(2, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": max_priority_fee_per_gas,\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert is_integer(txn[\"maxPriorityFeePerGas\"])\n        assert txn[\"maxPriorityFeePerGas\"] == max_priority_fee_per_gas\n        assert is_integer(txn[\"maxFeePerGas\"])\n\n    def test_eth_send_transaction_max_fee_less_than_tip(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        with pytest.raises(\n            InvalidTransaction, match=\"maxFeePerGas must be >= maxPriorityFeePerGas\"\n        ):\n            w3.eth.send_transaction(txn_params)\n\n    def test_validation_middleware_chain_id_mismatch(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        wrong_chain_id = 1234567890\n        actual_chain_id = w3.eth.chain_id\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": Wei(21000),\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n            \"chainId\": wrong_chain_id,\n        }\n        with pytest.raises(\n            Web3ValidationError,\n            match=f\"The transaction declared chain ID {wrong_chain_id}, \"\n            f\"but the connected node is on {actual_chain_id}\",\n        ):\n            w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.parametrize(\n        \"max_fee\", (1000000000, None), ids=[\"with_max_fee\", \"without_max_fee\"]\n    )\n    def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress, max_fee: Wei\n    ) -> None:\n        max_priority_fee = w3.to_wei(1, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": max_priority_fee,\n        }\n        if max_fee is not None:\n            txn_params = assoc(txn_params, \"maxFeePerGas\", max_fee)\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> Wei:\n            return w3.to_wei(2, \"gwei\")\n\n        w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        latest_block = w3.eth.get_block(\"latest\")\n        assert (\n            txn[\"maxFeePerGas\"] == max_fee\n            if max_fee is not None\n            else 2 * latest_block[\"baseFeePerGas\"] + max_priority_fee\n        )\n        assert txn[\"maxPriorityFeePerGas\"] == max_priority_fee\n        assert txn[\"gasPrice\"] == txn[\"maxFeePerGas\"]\n\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn_no_tip(\n        self,\n        w3: \"Web3\",\n        unlocked_account_dual_type: ChecksumAddress,\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1000000000),\n        }\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> Wei:\n            return w3.to_wei(2, \"gwei\")\n\n        w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        with pytest.raises(\n            InvalidTransaction, match=\"maxPriorityFeePerGas must be defined\"\n        ):\n            w3.eth.send_transaction(txn_params)\n\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    def \ntest_gas_price_strategy_hex_value(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        two_gwei_in_wei = w3.to_wei(2, \"gwei\")\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> str:\n            return hex(two_gwei_in_wei)\n\n        w3.eth.set_gas_price_strategy(gas_price_strategy)  # type: ignore\n\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert txn[\"gasPrice\"] == two_gwei_in_wei\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_legacy(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(\n                1, \"gwei\"\n            ),  # must be greater than base_fee post London\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params[\"gasPrice\"] = w3.to_wei(2, \"gwei\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n\n        assert is_same_address(\n            replace_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            replace_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert replace_txn[\"value\"] == 1\n        assert replace_txn[\"gas\"] == 21000\n        assert replace_txn[\"gasPrice\"] == txn_params[\"gasPrice\"]\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        two_gwei_in_wei = w3.to_wei(2, \"gwei\")\n        three_gwei_in_wei = w3.to_wei(3, \"gwei\")\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": two_gwei_in_wei,\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params[\"maxFeePerGas\"] = three_gwei_in_wei\n        txn_params[\"maxPriorityFeePerGas\"] = two_gwei_in_wei\n\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n\n        assert is_same_address(\n            replace_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            replace_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert replace_txn[\"value\"] == 1\n        assert replace_txn[\"gas\"] == 21000\n        assert replace_txn[\"maxFeePerGas\"] == three_gwei_in_wei\n        assert replace_txn[\"maxPriorityFeePerGas\"] == two_gwei_in_wei\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_underpriced(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(3, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        one_gwei_in_wei = w3.to_wei(1, \"gwei\")\n        txn_params[\"maxFeePerGas\"] = one_gwei_in_wei\n        txn_params[\"maxPriorityFeePerGas\"] = one_gwei_in_wei\n\n        with pytest.raises(ValueError, match=\"replacement transaction underpriced\"):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_non_existing_transaction(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(3, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        with pytest.raises(TransactionNotFound):\n            w3.eth.replace_transaction(\n                HexStr(\n                    \"0x98e8cc09b311583c5079fa600f6c2a3bea8611af168c52e4b60b5b243a441997\"\n                ),\n                txn_params,\n            )\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_already_mined(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        w3.eth.wait_for_transaction_receipt(txn_hash, timeout=10)\n\n        txn_params[\"maxFeePerGas\"] = w3.to_wei(3, \"gwei\")\n        txn_params[\"maxPriorityFeePerGas\"] = w3.to_wei(2, \"gwei\")\n        with pytest.raises(ValueError, match=\"Supplied transaction with hash\"):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_incorrect_nonce(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        txn_params[\"maxFeePerGas\"] = w3.to_wei(3, \"gwei\")\n        txn_params[\"maxPriorityFeePerGas\"] = w3.to_wei(2, \"gwei\")\n        txn_params[\"nonce\"] = Nonce(txn[\"nonce\"] + 1)\n        with pytest.raises(ValueError):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_too_low(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params[\"gasPrice\"] = w3.to_wei(1, \"gwei\")\n        with pytest.raises(ValueError):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_defaulting_minimum(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        gas_price = w3.to_wei(1, \"gwei\")\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": gas_price,\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params.pop(\"gasPrice\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n\n        assert replace_txn[\"gasPrice\"] == math.ceil(\n            gas_price * 1.125\n        )  # minimum gas price\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_defaulting_strategy_higher(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        two_gwei_in_wei = w3.to_wei(2, \"gwei\")\n\n        def higher_gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return two_gwei_in_wei\n\n        w3.eth.set_gas_price_strategy(higher_gas_price_strategy)\n\n        txn_params.pop(\"gasPrice\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n        assert (\n            replace_txn[\"gasPrice\"] == two_gwei_in_wei\n        )  # Strategy provides higher gas price\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_defaulting_strategy_lower(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        gas_price = w3.to_wei(2, \"gwei\")\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": gas_price,\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        def lower_gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return w3.to_wei(1, \"gwei\")\n\n        w3.eth.set_gas_price_strategy(lower_gas_price_strategy)\n\n        txn_params.pop(\"gasPrice\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n        # Strategy provides lower gas price - minimum preferred\n        assert replace_txn[\"gasPrice\"] == math.ceil(gas_price * 1.125)\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    def test_eth_modify_transaction_legacy(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(\n                1, \"gwei\"\n            ),  # must be greater than base_fee post London\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = w3.eth.modify_transaction(\n            txn_hash, gasPrice=(cast(int, txn_params[\"gasPrice\"]) * 2), value=2\n        )\n        modified_txn = w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert modified_txn[\"gasPrice\"] == cast(int, txn_params[\"gasPrice\"]) * 2\n\n    def test_eth_modify_transaction(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = w3.eth.modify_transaction(\n            txn_hash,\n            value=2,\n            maxPriorityFeePerGas=(cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2),\n            maxFeePerGas=(cast(Wei, txn_params[\"maxFeePerGas\"]) * 2),\n        )\n        modified_txn = w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert (\n            modified_txn[\"maxPriorityFeePerGas\"]\n            == cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2\n        )\n        assert modified_txn[\"maxFeePerGas\"] == cast(Wei, txn_params[\"maxFeePerGas\"]) * 2\n\n    def test_eth_send_raw_transaction(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        signed_tx = w3.eth.account.sign_transaction(\n            {\n                \"to\": \"0x0000000000000000000000000000000000000000\",\n                \"value\": 0,\n                \"nonce\": w3.eth.get_transaction_count(unlocked_account),\n                \"gas\": 21000,\n                \"maxFeePerGas\": 1000000000,\n                \"maxPriorityFeePerGas\": 1000000000,\n                \"chainId\": 131277322940537,\n            },\n            # unlocked_account private key:\n            \"0x392f63a79b1ff8774845f3fa69de4a13800a59e7083f5187f1558f0797ad0f01\",\n        )\n        txn_hash = w3.eth.send_raw_transaction(signed_tx.rawTransaction)\n        assert txn_hash == signed_tx.hash\n\n    def test_eth_call(self, w3: \"Web3\", math_contract: \"Contract\") -> None:\n        coinbase = w3.eth.coinbase\n        txn_params = math_contract._prepare_transaction(\n            fn_name=\"add\",\n            fn_args=(7, 11),\n            transaction={\"from\": coinbase, \"to\": math_contract.address},\n        )\n        call_result = w3.eth.call(txn_params)\n        assert is_string(call_result)\n        (result,) = w3.codec.decode([\"uint256\"], call_result)\n        assert result == 18\n\n    def test_eth_call_with_override_code(\n        self, w3: \"Web3\", revert_contract: \"Contract\"\n    ) -> None:\n        coinbase = w3.eth.coinbase\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"normalFunction\",\n            transaction={\"from\": coinbase, \"to\": revert_contract.address},\n        )\n        call_result = w3.eth.call(txn_params)\n        (result,) = w3.codec.decode([\"bool\"], call_result)\n        assert result is True\n\n        # override runtime bytecode: `normalFunction` returns `false`\n        override_code = HexStr(\n            \"0x6080604052348015600f57600080fd5b5060043610603c5760003560e01c8063185c38a4146041578063c06a97cb146049578063d67e4b84146051575b600080fd5b60476071565b005b604f60df565b005b605760e4565b604051808215151515815260200191505060405180910390f35b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252601b8152602001807f46756e6374696f6e20686173206265656e2072657665727465642e000000000081525060200191505060405180910390fd5b600080fd5b60008090509056fea2646970667358221220bb71e9e9a2e271cd0fbe833524a3ea67df95f25ea13aef5b0a761fa52b538f1064736f6c63430006010033\"  # noqa: E501\n        )\n        call_result = w3.eth.call(\n            txn_params, \"latest\", {revert_contract.address: {\"code\": override_code}}\n        )\n        (result,) = w3.codec.decode([\"bool\"], call_result)\n        assert result is False\n\n        # test bytes\n\n        bytes_call_result = w3.eth.call(\n            txn_params,\n            \"latest\",\n...\n", "needle_token_start": 10417, "needle_token_end": 10707, "code_context_ntokens": 16239, "output": ["```python\ndef test_gas_price_strategy_hex_value(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        two_gwei_in_wei = w3.to_wei(2, \"gwei\")\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> str:\n            return hex(two_gwei_in_wei)\n\n        w3.eth.set_gas_price_strategy(gas_price_strategy)  # type: ignore\n\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert txn[\"gasPrice\"] == two_gwei_in_wei\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n```"]}
{"repo": "ethereum/web3.py", "name": "test_eth_estimate_gas_revert_with_msg", "language": "python", "path": "web3/_utils/module_testing/eth_module.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To verify that the system correctly identifies and handles a transaction that is expected to revert due to a specific error message in a smart contract function.\n2. **Input**: A transaction object that includes details such as the sender's address and the target contract address.\n3. **Output**: The test is expected to raise an exception indicating that the transaction has been reverted along with a specific error message.\n4. **Procedure**: The test prepares a transaction intended to call a function known to revert. It then attempts to estimate the gas for this transaction, expecting to catch a specific error that indicates the function's execution has been reverted with a predefined message.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        max_priority_fee_per_gas = w3.to_wei(2, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": max_priority_fee_per_gas,\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert is_integer(txn[\"maxPriorityFeePerGas\"])\n        assert txn[\"maxPriorityFeePerGas\"] == max_priority_fee_per_gas\n        assert is_integer(txn[\"maxFeePerGas\"])\n\n    def test_eth_send_transaction_max_fee_less_than_tip(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        with pytest.raises(\n            InvalidTransaction, match=\"maxFeePerGas must be >= maxPriorityFeePerGas\"\n        ):\n            w3.eth.send_transaction(txn_params)\n\n    def test_validation_middleware_chain_id_mismatch(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        wrong_chain_id = 1234567890\n        actual_chain_id = w3.eth.chain_id\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": Wei(21000),\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n            \"chainId\": wrong_chain_id,\n        }\n        with pytest.raises(\n            Web3ValidationError,\n            match=f\"The transaction declared chain ID {wrong_chain_id}, \"\n            f\"but the connected node is on {actual_chain_id}\",\n        ):\n            w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.parametrize(\n        \"max_fee\", (1000000000, None), ids=[\"with_max_fee\", \"without_max_fee\"]\n    )\n    def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress, max_fee: Wei\n    ) -> None:\n        max_priority_fee = w3.to_wei(1, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": max_priority_fee,\n        }\n        if max_fee is not None:\n            txn_params = assoc(txn_params, \"maxFeePerGas\", max_fee)\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> Wei:\n            return w3.to_wei(2, \"gwei\")\n\n        w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        latest_block = w3.eth.get_block(\"latest\")\n        assert (\n            txn[\"maxFeePerGas\"] == max_fee\n            if max_fee is not None\n            else 2 * latest_block[\"baseFeePerGas\"] + max_priority_fee\n        )\n        assert txn[\"maxPriorityFeePerGas\"] == max_priority_fee\n        assert txn[\"gasPrice\"] == txn[\"maxFeePerGas\"]\n\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn_no_tip(\n        self,\n        w3: \"Web3\",\n        unlocked_account_dual_type: ChecksumAddress,\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1000000000),\n        }\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> Wei:\n            return w3.to_wei(2, \"gwei\")\n\n        w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        with pytest.raises(\n            InvalidTransaction, match=\"maxPriorityFeePerGas must be defined\"\n        ):\n            w3.eth.send_transaction(txn_params)\n\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    def test_gas_price_strategy_hex_value(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        two_gwei_in_wei = w3.to_wei(2, \"gwei\")\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> str:\n            return hex(two_gwei_in_wei)\n\n        w3.eth.set_gas_price_strategy(gas_price_strategy)  # type: ignore\n\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        assert txn[\"gasPrice\"] == two_gwei_in_wei\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_legacy(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(\n                1, \"gwei\"\n            ),  # must be greater than base_fee post London\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params[\"gasPrice\"] = w3.to_wei(2, \"gwei\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n\n        assert is_same_address(\n            replace_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            replace_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert replace_txn[\"value\"] == 1\n        assert replace_txn[\"gas\"] == 21000\n        assert replace_txn[\"gasPrice\"] == txn_params[\"gasPrice\"]\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        two_gwei_in_wei = w3.to_wei(2, \"gwei\")\n        three_gwei_in_wei = w3.to_wei(3, \"gwei\")\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": two_gwei_in_wei,\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params[\"maxFeePerGas\"] = three_gwei_in_wei\n        txn_params[\"maxPriorityFeePerGas\"] = two_gwei_in_wei\n\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n\n        assert is_same_address(\n            replace_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            replace_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert replace_txn[\"value\"] == 1\n        assert replace_txn[\"gas\"] == 21000\n        assert replace_txn[\"maxFeePerGas\"] == three_gwei_in_wei\n        assert replace_txn[\"maxPriorityFeePerGas\"] == two_gwei_in_wei\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_underpriced(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(3, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        one_gwei_in_wei = w3.to_wei(1, \"gwei\")\n        txn_params[\"maxFeePerGas\"] = one_gwei_in_wei\n        txn_params[\"maxPriorityFeePerGas\"] = one_gwei_in_wei\n\n        with pytest.raises(ValueError, match=\"replacement transaction underpriced\"):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_non_existing_transaction(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(3, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        with pytest.raises(TransactionNotFound):\n            w3.eth.replace_transaction(\n                HexStr(\n                    \"0x98e8cc09b311583c5079fa600f6c2a3bea8611af168c52e4b60b5b243a441997\"\n                ),\n                txn_params,\n            )\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_already_mined(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        w3.eth.wait_for_transaction_receipt(txn_hash, timeout=10)\n\n        txn_params[\"maxFeePerGas\"] = w3.to_wei(3, \"gwei\")\n        txn_params[\"maxPriorityFeePerGas\"] = w3.to_wei(2, \"gwei\")\n        with pytest.raises(ValueError, match=\"Supplied transaction with hash\"):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_incorrect_nonce(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n        txn = w3.eth.get_transaction(txn_hash)\n\n        txn_params[\"maxFeePerGas\"] = w3.to_wei(3, \"gwei\")\n        txn_params[\"maxPriorityFeePerGas\"] = w3.to_wei(2, \"gwei\")\n        txn_params[\"nonce\"] = Nonce(txn[\"nonce\"] + 1)\n        with pytest.raises(ValueError):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_too_low(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account_dual_type,\n            \"to\": unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params[\"gasPrice\"] = w3.to_wei(1, \"gwei\")\n        with pytest.raises(ValueError):\n            w3.eth.replace_transaction(txn_hash, txn_params)\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_defaulting_minimum(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        gas_price = w3.to_wei(1, \"gwei\")\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": gas_price,\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        txn_params.pop(\"gasPrice\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n\n        assert replace_txn[\"gasPrice\"] == math.ceil(\n            gas_price * 1.125\n        )  # minimum gas price\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_defaulting_strategy_higher(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        two_gwei_in_wei = w3.to_wei(2, \"gwei\")\n\n        def higher_gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return two_gwei_in_wei\n\n        w3.eth.set_gas_price_strategy(higher_gas_price_strategy)\n\n        txn_params.pop(\"gasPrice\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n        assert (\n            replace_txn[\"gasPrice\"] == two_gwei_in_wei\n        )  # Strategy provides higher gas price\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @flaky_geth_dev_mining\n    def test_eth_replace_transaction_gas_price_defaulting_strategy_lower(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        gas_price = w3.to_wei(2, \"gwei\")\n\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": gas_price,\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        def lower_gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return w3.to_wei(1, \"gwei\")\n\n        w3.eth.set_gas_price_strategy(lower_gas_price_strategy)\n\n        txn_params.pop(\"gasPrice\")\n        replace_txn_hash = w3.eth.replace_transaction(txn_hash, txn_params)\n        replace_txn = w3.eth.get_transaction(replace_txn_hash)\n        # Strategy provides lower gas price - minimum preferred\n        assert replace_txn[\"gasPrice\"] == math.ceil(gas_price * 1.125)\n        w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    def test_eth_modify_transaction_legacy(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": w3.to_wei(\n                1, \"gwei\"\n            ),  # must be greater than base_fee post London\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = w3.eth.modify_transaction(\n            txn_hash, gasPrice=(cast(int, txn_params[\"gasPrice\"]) * 2), value=2\n        )\n        modified_txn = w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert modified_txn[\"gasPrice\"] == cast(int, txn_params[\"gasPrice\"]) * 2\n\n    def test_eth_modify_transaction(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": unlocked_account,\n            \"to\": unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n            \"maxFeePerGas\": w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = w3.eth.modify_transaction(\n            txn_hash,\n            value=2,\n            maxPriorityFeePerGas=(cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2),\n            maxFeePerGas=(cast(Wei, txn_params[\"maxFeePerGas\"]) * 2),\n        )\n        modified_txn = w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert (\n            modified_txn[\"maxPriorityFeePerGas\"]\n            == cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2\n        )\n        assert modified_txn[\"maxFeePerGas\"] == cast(Wei, txn_params[\"maxFeePerGas\"]) * 2\n\n    def test_eth_send_raw_transaction(\n        self, w3: \"Web3\", unlocked_account: ChecksumAddress\n    ) -> None:\n        signed_tx = w3.eth.account.sign_transaction(\n            {\n                \"to\": \"0x0000000000000000000000000000000000000000\",\n                \"value\": 0,\n                \"nonce\": w3.eth.get_transaction_count(unlocked_account),\n                \"gas\": 21000,\n                \"maxFeePerGas\": 1000000000,\n                \"maxPriorityFeePerGas\": 1000000000,\n                \"chainId\": 131277322940537,\n            },\n            # unlocked_account private key:\n            \"0x392f63a79b1ff8774845f3fa69de4a13800a59e7083f5187f1558f0797ad0f01\",\n        )\n        txn_hash = w3.eth.send_raw_transaction(signed_tx.rawTransaction)\n        assert txn_hash == signed_tx.hash\n\n    def test_eth_call(self, w3: \"Web3\", math_contract: \"Contract\") -> None:\n        coinbase = w3.eth.coinbase\n        txn_params = math_contract._prepare_transaction(\n            fn_name=\"add\",\n            fn_args=(7, 11),\n            transaction={\"from\": coinbase, \"to\": math_contract.address},\n        )\n        call_result = w3.eth.call(txn_params)\n        assert is_string(call_result)\n        (result,) = w3.codec.decode([\"uint256\"], call_result)\n        assert result == 18\n\n    def test_eth_call_with_override_code(\n        self, w3: \"Web3\", revert_contract: \"Contract\"\n    ) -> None:\n        coinbase = w3.eth.coinbase\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"normalFunction\",\n            transaction={\"from\": coinbase, \"to\": revert_contract.address},\n        )\n        call_result = w3.eth.call(txn_params)\n        (result,) = w3.codec.decode([\"bool\"], call_result)\n        assert result is True\n\n        # override runtime bytecode: `normalFunction` returns `false`\n        override_code = HexStr(\n            \"0x6080604052348015600f57600080fd5b5060043610603c5760003560e01c8063185c38a4146041578063c06a97cb146049578063d67e4b84146051575b600080fd5b60476071565b005b604f60df565b005b605760e4565b604051808215151515815260200191505060405180910390f35b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252601b8152602001807f46756e6374696f6e20686173206265656e2072657665727465642e000000000081525060200191505060405180910390fd5b600080fd5b60008090509056fea2646970667358221220bb71e9e9a2e271cd0fbe833524a3ea67df95f25ea13aef5b0a761fa52b538f1064736f6c63430006010033\"  # noqa: E501\n        )\n        call_result = w3.eth.call(\n            txn_params, \"latest\", {revert_contract.address: {\"code\": override_code}}\n        )\n        (result,) = w3.codec.decode([\"bool\"], call_result)\n        assert result is False\n\n        # test bytes\n\n        bytes_call_result = w3.eth.call(\n            txn_params,\n            \"latest\",\n            {revert_contract.address: {\"code\": to_bytes(hexstr=override_code)}},\n        )\n        (bytes_result,) = w3.codec.decode([\"bool\"], bytes_call_result)\n        assert bytes_result is False\n\n    @pytest.mark.parametrize(\n        \"params\",\n        (\n            {\n                \"nonce\": 1,  # int\n                \"balance\": 1,  # int\n                \"code\": HexStr(\"0x\"),  # HexStr\n                # with state\n                \"state\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n            {\n                \"nonce\": HexStr(\"0x1\"),  # HexStr\n                \"balance\": HexStr(\"0x1\"),  # HexStr\n                \"code\": b\"\\x00\",  # bytes\n                # with stateDiff\n                \"stateDiff\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n        ),\n    )\n    def test_eth_call_with_override_param_type_check(\n        self,\n        w3: \"Web3\",\n        math_contract: \"Contract\",\n        params: StateOverrideParams,\n    ) -> None:\n        txn_params: TxParams = {\"from\": w3.eth.coinbase}\n\n        # assert does not raise\n        w3.eth.call(txn_params, \"latest\", {math_contract.address: params})\n\n    def test_eth_call_with_0_result(\n        self, w3: \"Web3\", math_contract: \"Contract\"\n    ) -> None:\n        coinbase = w3.eth.coinbase\n        txn_params = math_contract._prepare_transaction(\n            fn_name=\"add\",\n            fn_args=(0, 0),\n            transaction={\"from\": coinbase, \"to\": math_contract.address},\n        )\n        call_result = w3.eth.call(txn_params)\n        assert is_string(call_result)\n        (result,) = w3.codec.decode([\"uint256\"], call_result)\n        assert result == 0\n\n    def test_eth_call_revert_with_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"revertWithMessage\",\n            transaction={\n                \"from\": unlocked_account,\n                \"to\": revert_contract.address,\n            },\n        )\n        data = \"0x08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001b46756e6374696f6e20686173206265656e2072657665727465642e0000000000\"  # noqa: E501\n        with pytest.raises(\n            ContractLogicError, match=\"execution reverted: Function has been reverted\"\n        ) as excinfo:\n            w3.eth.call(txn_params)\n        assert excinfo.value.data == data\n\n    def test_eth_call_revert_without_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        with pytest.raises(ContractLogicError, match=\"execution reverted\"):\n            txn_params = revert_contract._prepare_transaction(\n                fn_name=\"revertWithoutMessage\",\n                transaction={\n                    \"from\": unlocked_account,\n                    \"to\": revert_contract.address,\n                },\n            )\n            w3.eth.call(txn_params)\n\n    def test_eth_call_custom_error_revert_with_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        data = revert_contract.encodeABI(\n            fn_name=\"UnauthorizedWithMessage\", args=[\"You are not authorized\"]\n        )\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"customErrorWithMessage\",\n            transaction={\n                \"from\": unlocked_account,\n                \"to\": revert_contract.address,\n            },\n        )\n        with pytest.raises(ContractCustomError, match=data) as excinfo:\n            w3.eth.call(txn_params)\n        assert excinfo.value.data == data\n\n    def test_eth_call_custom_error_revert_without_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        data = revert_contract.encodeABI(fn_name=\"Unauthorized\")\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"customErrorWithoutMessage\",\n            transaction={\n                \"from\": unlocked_account,\n                \"to\": revert_contract.address,\n            },\n        )\n        with pytest.raises(ContractCustomError, match=data) as excinfo:\n            w3.eth.call(txn_params)\n        assert excinfo.value.data == data\n\n    @pytest.mark.parametrize(\n        \"panic_error,params\",\n        (\n            (\"01\", []),\n            (\"11\", []),\n            (\"12\", [0]),\n            (\"21\", [-1]),\n            (\"22\", []),\n            (\"31\", []),\n            (\"32\", []),\n            (\"41\", []),\n            (\"51\", []),\n        ),\n    )\n    def test_contract_panic_errors(\n        self,\n        w3: \"Web3\",\n        panic_errors_contract: \"Contract\",\n        panic_error: str,\n        params: List[Any],\n    ) -> None:\n        method = getattr(\n            panic_errors_contract.functions,\n            f\"errorCode{panic_error}\",\n        )\n        error_msg = PANIC_ERROR_CODES[panic_error]\n\n        with pytest.raises(ContractPanicError, match=re.escape(error_msg)):\n            method(*params).call()\n\n    def test_eth_call_offchain_lookup(\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n        monkeypatch: \"MonkeyPatch\",\n    ) -> None:\n        normalized_contract_address = to_hex_if_bytes(\n            offchain_lookup_contract.address\n        ).lower()\n        mock_offchain_lookup_request_response(\n            monkeypatch,\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}/{OFFCHAIN_LOOKUP_TEST_DATA}.json\",  # noqa: E501\n            mocked_json_data=WEB3PY_AS_HEXBYTES,\n        )\n        response = offchain_lookup_contract.functions.testOffchainLookup(\n            OFFCHAIN_LOOKUP_TEST_DATA\n        ).call()\n        assert w3.codec.decode([\"string\"], response)[0] == \"web3py\"\n\n    def test_eth_call_offchain_lookup_raises_when_ccip_read_is_disabled(\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n    ) -> None:\n        return_data = (\n            OFFCHAIN_LOOKUP_4BYTE_DATA\n            + abi_encoded_offchain_lookup_contract_address(w3, offchain_lookup_contract)\n            + OFFCHAIN_LOOKUP_RETURN_DATA\n        )\n        # test ContractFunction call\n        with pytest.raises(OffchainLookup) as e:\n            offchain_lookup_contract.functions.testOffchainLookup(\n                OFFCHAIN_LOOKUP_TEST_DATA\n            ).call(ccip_read_enabled=False)\n        assert e.value.data == return_data\n\n        # test ContractCaller call\n        with pytest.raises(OffchainLookup) as excinfo:\n            offchain_lookup_contract.caller(ccip_read_enabled=False).testOffchainLookup(\n                OFFCHAIN_LOOKUP_TEST_DATA\n            )\n        assert excinfo.value.data == return_data\n\n        # test global flag on the provider\n        w3.provider.global_ccip_read_enabled = False\n\n        with pytest.raises(OffchainLookup) as exc_info:\n            offchain_lookup_contract.functions.testOffchainLookup(\n                OFFCHAIN_LOOKUP_TEST_DATA\n            ).call()\n        assert exc_info.value.data == return_data\n\n        w3.provider.global_ccip_read_enabled = True  # cleanup\n\n    def test_eth_call_offchain_lookup_call_flag_overrides_provider_flag(\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n        monkeypatch: \"MonkeyPatch\",\n    ) -> None:\n        normalized_contract_address = to_hex_if_bytes(\n            offchain_lookup_contract.address\n        ).lower()\n        mock_offchain_lookup_request_response(\n            monkeypatch,\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}/{OFFCHAIN_LOOKUP_TEST_DATA}.json\",  # noqa: E501\n            mocked_json_data=WEB3PY_AS_HEXBYTES,\n        )\n\n        w3.provider.global_ccip_read_enabled = False\n\n        response = offchain_lookup_contract.functions.testOffchainLookup(\n            OFFCHAIN_LOOKUP_TEST_DATA\n        ).call(ccip_read_enabled=True)\n        assert w3.codec.decode([\"string\"], response)[0] == \"web3py\"\n\n        w3.provider.global_ccip_read_enabled = True  # cleanup\n\n    @pytest.mark.parametrize(\"max_redirects\", range(-1, 4))\n    def test_eth_call_offchain_lookup_raises_if_max_redirects_is_less_than_4(\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n        max_redirects: int,\n    ) -> None:\n        default_max_redirects = w3.provider.ccip_read_max_redirects\n\n        w3.provider.ccip_read_max_redirects = max_redirects\n        with pytest.raises(ValueError, match=\"at least 4\"):\n            offchain_lookup_contract.functions.testOffchainLookup(\n                OFFCHAIN_LOOKUP_TEST_DATA\n            ).call()\n\n        w3.provider.ccip_read_max_redirects = default_max_redirects  # cleanup\n\n    def test_eth_call_offchain_lookup_raises_for_improperly_formatted_rest_request_response(  # noqa: E501\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n        monkeypatch: \"MonkeyPatch\",\n    ) -> None:\n        normalized_contract_address = to_hex_if_bytes(\n            offchain_lookup_contract.address\n        ).lower()\n        mock_offchain_lookup_request_response(\n            monkeypatch,\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}/{OFFCHAIN_LOOKUP_TEST_DATA}.json\",  # noqa: E501\n            mocked_json_data=WEB3PY_AS_HEXBYTES,\n            json_data_field=\"not_data\",\n        )\n        with pytest.raises(Web3ValidationError, match=\"missing 'data' field\"):\n            offchain_lookup_contract.functions.testOffchainLookup(\n                OFFCHAIN_LOOKUP_TEST_DATA\n            ).call()\n\n    @pytest.mark.parametrize(\"status_code_non_4xx_error\", [100, 300, 500, 600])\n    def test_eth_call_offchain_lookup_tries_next_url_for_non_4xx_error_status_and_tests_POST(  # noqa: E501\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n        monkeypatch: \"MonkeyPatch\",\n        status_code_non_4xx_error: int,\n    ) -> None:\n        normalized_contract_address = to_hex_if_bytes(\n            offchain_lookup_contract.address\n        ).lower()\n\n        # The next url in our test contract doesn't contain '{data}', triggering\n        # the POST request logic. The idea here is to return a bad status for\n        # the first url (GET) and a success status from the second call (POST)\n        # to test both that we move on to the next url with non 4xx status and\n        # that the POST logic is also working as expected.\n        mock_offchain_lookup_request_response(\n            monkeypatch,\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}/{OFFCHAIN_LOOKUP_TEST_DATA}.json\",  # noqa: E501\n            mocked_status_code=status_code_non_4xx_error,\n            mocked_json_data=WEB3PY_AS_HEXBYTES,\n        )\n        mock_offchain_lookup_request_response(\n            monkeypatch,\n            http_method=\"POST\",\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}.json\",  # noqa: E501\n            mocked_status_code=200,\n            mocked_json_data=WEB3PY_AS_HEXBYTES,\n            sender=normalized_contract_address,\n            calldata=OFFCHAIN_LOOKUP_TEST_DATA,\n        )\n        response = offchain_lookup_contract.functions.testOffchainLookup(\n            OFFCHAIN_LOOKUP_TEST_DATA\n        ).call()\n        assert w3.codec.decode([\"string\"], response)[0] == \"web3py\"\n\n    def test_eth_call_offchain_lookup_calls_raise_for_status_for_4xx_status_code(\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n        monkeypatch: \"MonkeyPatch\",\n    ) -> None:\n        normalized_contract_address = to_hex_if_bytes(\n            offchain_lookup_contract.address\n        ).lower()\n        mock_offchain_lookup_request_response(\n            monkeypatch,\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}/{OFFCHAIN_LOOKUP_TEST_DATA}.json\",  # noqa: E501\n            mocked_status_code=randint(400, 499),\n            mocked_json_data=WEB3PY_AS_HEXBYTES,\n        )\n        with pytest.raises(Exception, match=\"called raise_for_status\\\\(\\\\)\"):\n            offchain_lookup_contract.functions.testOffchainLookup(\n                OFFCHAIN_LOOKUP_TEST_DATA\n            ).call()\n\n    def test_eth_call_offchain_lookup_raises_when_all_supplied_urls_fail(\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n    ) -> None:\n        # GET and POST requests should fail since responses are not mocked\n        with pytest.raises(\n            MultipleFailedRequests, match=\"Offchain lookup failed for supplied urls\"\n        ):\n            offchain_lookup_contract.functions.testOffchainLookup(\n                OFFCHAIN_LOOKUP_TEST_DATA\n            ).call()\n\n    def test_eth_call_continuous_offchain_lookup_raises_with_too_many_requests(\n        self,\n        w3: \"Web3\",\n        offchain_lookup_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n        monkeypatch: \"MonkeyPatch\",\n    ) -> None:\n        normalized_contract_address = to_hex_if_bytes(\n            offchain_lookup_contract.address\n        ).lower()\n        mock_offchain_lookup_request_response(\n            monkeypatch,\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}/0x.json\",  # noqa: E501\n        )\n        with pytest.raises(TooManyRequests, match=\"Too many CCIP read redirects\"):\n            offchain_lookup_contract.caller().continuousOffchainLookup()\n\n    def \ntest_eth_estimate_gas_revert_with_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        with pytest.raises(\n            ContractLogicError, match=\"execution reverted: Function has been reverted\"\n        ):\n            txn_params = revert_contract._prepare_transaction(\n                fn_name=\"revertWithMessage\",\n                transaction={\n                    \"from\": unlocked_account,\n                    \"to\": revert_contract.address,\n                },\n            )\n            w3.eth.estimate_gas(txn_params)\n\n    def test_eth_estimate_gas_revert_without_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        with pytest.raises(ContractLogicError, match=\"execution reverted\"):\n            txn_params = revert_contract._prepare_transaction(\n                fn_name=\"revertWithoutMessage\",\n                transaction={\n                    \"from\": unlocked_account,\n                    \"to\": revert_contract.address,\n                },\n            )\n            w3.eth.estimate_gas(txn_params)\n\n    def test_eth_estimate_gas_custom_error_revert_with_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        data = revert_contract.encodeABI(\n            fn_name=\"UnauthorizedWithMessage\", args=[\"You are not authorized\"]\n        )\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"customErrorWithMessage\",\n            transaction={\n                \"from\": unlocked_account,\n                \"to\": revert_contract.address,\n            },\n        )\n        with pytest.raises(ContractCustomError, match=data) as excinfo:\n            w3.eth.estimate_gas(txn_params)\n        assert excinfo.value.data == data\n\n    def test_eth_estimate_gas_custom_error_revert_without_msg(\n        self,\n        w3: \"Web3\",\n        revert_contract: \"Contract\",\n        unlocked_account: ChecksumAddress,\n    ) -> None:\n        data = revert_contract.encodeABI(fn_name=\"Unauthorized\")\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"customErrorWithoutMessage\",\n            transaction={\n                \"from\": unlocked_account,\n                \"to\": revert_contract.address,\n            },\n        )\n        with pytest.raises(ContractCustomError, match=data) as excinfo:\n            w3.eth.estimate_gas(txn_params)\n        assert excinfo.value.data == data\n\n    def test_eth_estimate_gas(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        gas_estimate = w3.eth.estimate_gas(\n            {\n                \"from\": unlocked_account_dual_type,\n                \"to\": unlocked_account_dual_type,\n                \"value\": Wei(1),\n            }\n        )\n        assert is_integer(gas_estimate)\n        assert gas_estimate > 0\n\n    def test_eth_estimate_gas_with_block(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        gas_estimate = w3.eth.estimate_gas(\n            {\n                \"from\": unlocked_account_dual_type,\n                \"to\": unlocked_account_dual_type,\n                \"value\": Wei(1),\n            },\n            \"latest\",\n        )\n        assert is_integer(gas_estimate)\n        assert gas_estimate > 0\n\n    @pytest.mark.parametrize(\n        \"params\",\n        (\n            {\n                \"nonce\": 1,  # int\n                \"balance\": 1,  # int\n                \"code\": HexStr(\"0x\"),  # HexStr\n                # with state\n                \"state\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n            {\n                \"nonce\": HexStr(\"0x1\"),  # HexStr\n                \"balance\": HexStr(\"0x1\"),  # HexStr\n                \"code\": b\"\\x00\",  # bytes\n                # with stateDiff\n                \"stateDiff\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n        ),\n    )\n    def test_eth_estimate_gas_with_override_param_type_check(\n        self,\n        w3: \"Web3\",\n        math_contract: \"Contract\",\n        params: StateOverrideParams,\n    ) -> None:\n        txn_params: TxParams = {\"from\": w3.eth.coinbase}\n\n        # assert does not raise\n        w3.eth.estimate_gas(txn_params, None, {math_contract.address: params})\n\n    def test_eth_getBlockByHash(self, w3: \"Web3\", empty_block: BlockData) -> None:\n        block = w3.eth.get_block(empty_block[\"hash\"])\n        assert block[\"hash\"] == empty_block[\"hash\"]\n        assert block[\"receiptsRoot\"] == empty_block[\"receiptsRoot\"]\n        assert block[\"logsBloom\"] == empty_block[\"logsBloom\"]\n\n    def test_eth_getBlockByHash_not_found(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        with pytest.raises(BlockNotFound):\n            w3.eth.get_block(UNKNOWN_HASH)\n\n    def test_eth_getBlockByHash_pending(self, w3: \"Web3\") -> None:\n        block = w3.eth.get_block(\"pending\")\n        assert block[\"hash\"] is None\n\n    def test_eth_getBlockByNumber_with_integer(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        block = w3.eth.get_block(empty_block[\"number\"])\n        assert block[\"number\"] == empty_block[\"number\"]\n\n    def test_eth_getBlockByNumber_latest(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        current_block_number = w3.eth.block_number\n        block = w3.eth.get_block(\"latest\")\n        assert block[\"number\"] == current_block_number\n\n    def test_eth_getBlockByNumber_not_found(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        with pytest.raises(BlockNotFound):\n            w3.eth.get_block(BlockNumber(12345))\n\n    def test_eth_getBlockByNumber_pending(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        current_block_number = w3.eth.block_number\n        block = w3.eth.get_block(\"pending\")\n        assert block[\"number\"] == current_block_number + 1\n\n    def test_eth_getBlockByNumber_earliest(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        genesis_block = w3.eth.get_block(BlockNumber(0))\n        block = w3.eth.get_block(\"earliest\")\n        assert block[\"number\"] == 0\n        assert block[\"hash\"] == genesis_block[\"hash\"]\n\n    def test_eth_getBlockByNumber_safe(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        block = w3.eth.get_block(\"safe\")\n        assert block is not None\n        assert isinstance(block[\"number\"], int)\n\n    def test_eth_getBlockByNumber_finalized(\n        self, w3: \"Web3\", empty_block: BlockData\n    ) -> None:\n        block = w3.eth.get_block(\"finalized\")\n        assert block is not None\n        assert isinstance(block[\"number\"], int)\n\n    def test_eth_getBlockByNumber_full_transactions(\n        self, w3: \"Web3\", block_with_txn: BlockData\n    ) -> None:\n        block = w3.eth.get_block(block_with_txn[\"number\"], True)\n        transaction = block[\"transactions\"][0]\n        assert transaction[\"hash\"] == block_with_txn[\"transactions\"][0]  # type: ignore\n\n    def test_eth_getTransactionByHash(self, w3: \"Web3\", mined_txn_hash: HexStr) -> None:\n        transaction = w3.eth.get_transaction(mined_txn_hash)\n        assert is_dict(transaction)\n        assert transaction[\"hash\"] == HexBytes(mined_txn_hash)\n\n    def test_eth_getTransactionByHash_contract_creation(\n        self, w3: \"Web3\", math_contract_deploy_txn_hash: HexStr\n    ) -> None:\n        transaction = w3.eth.get_transaction(math_contract_deploy_txn_hash)\n        assert is_dict(transaction)\n        assert transaction[\"to\"] is None, f\"to field is {transaction['to']!r}\"\n\n    def test_eth_getTransactionByBlockHashAndIndex(\n        self, w3: \"Web3\", block_with_txn: BlockData, mined_txn_hash: HexStr\n    ) -> None:\n        transaction = w3.eth.get_transaction_by_block(block_with_txn[\"hash\"], 0)\n        assert is_dict(transaction)\n        assert transaction[\"hash\"] == HexBytes(mined_txn_hash)\n\n    def test_eth_getTransactionByBlockNumberAndIndex(\n        self, w3: \"Web3\", block_with_txn: BlockData, mined_txn_hash: HexStr\n    ) -> None:\n        transaction = w3.eth.get_transaction_by_block(block_with_txn[\"number\"], 0)\n        assert is_dict(transaction)\n        assert transaction[\"hash\"] == HexBytes(mined_txn_hash)\n\n    def test_eth_get_transaction_receipt_mined(\n        self, w3: \"Web3\", block_with_txn: BlockData, mined_txn_hash: HexStr\n    ) -> None:\n        receipt = w3.eth.get_transaction_receipt(mined_txn_hash)\n        assert is_dict(receipt)\n        assert receipt[\"blockNumber\"] == block_with_txn[\"number\"]\n        assert receipt[\"blockHash\"] == block_with_txn[\"hash\"]\n        assert receipt[\"transactionIndex\"] == 0\n        assert receipt[\"transactionHash\"] == HexBytes(mined_txn_hash)\n        assert is_checksum_address(receipt[\"to\"])\n        assert receipt[\"from\"] is not None\n        assert is_checksum_address(receipt[\"from\"])\n\n        effective_gas_price = receipt[\"effectiveGasPrice\"]\n        assert isinstance(effective_gas_price, int)\n        assert effective_gas_price > 0\n\n    def test_eth_get_transaction_receipt_unmined(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_hash = w3.eth.send_transaction(\n            {\n                \"from\": unlocked_account_dual_type,\n                \"to\": unlocked_account_dual_type,\n                \"value\": Wei(1),\n                \"gas\": 21000,\n                \"maxFeePerGas\": w3.to_wei(3, \"gwei\"),\n                \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n            }\n        )\n        with pytest.raises(TransactionNotFound):\n            w3.eth.get_transaction_receipt(txn_hash)\n\n    def test_eth_get_transaction_receipt_with_log_entry(\n        self,\n        w3: \"Web3\",\n        block_with_txn_with_log: BlockData,\n        emitter_contract: \"Contract\",\n        txn_hash_with_log: HexStr,\n    ) -> None:\n        receipt = w3.eth.get_transaction_receipt(txn_hash_with_log)\n        assert is_dict(receipt)\n        assert receipt[\"blockNumber\"] == block_with_txn_with_log[\"number\"]\n        assert receipt[\"blockHash\"] == block_with_txn_with_log[\"hash\"]\n        assert receipt[\"transactionIndex\"] == 0\n        assert receipt[\"transactionHash\"] == HexBytes(txn_hash_with_log)\n\n        assert len(receipt[\"logs\"]) == 1\n        log_entry = receipt[\"logs\"][0]\n\n        assert log_entry[\"blockNumber\"] == block_with_txn_with_log[\"number\"]\n        assert log_entry[\"blockHash\"] == block_with_txn_with_log[\"hash\"]\n        assert log_entry[\"logIndex\"] == 0\n        assert is_same_address(log_entry[\"address\"], emitter_contract.address)\n        assert log_entry[\"transactionIndex\"] == 0\n        assert log_entry[\"transactionHash\"] == HexBytes(txn_hash_with_log)\n\n    def test_eth_wait_for_transaction_receipt_mined(\n        self, w3: \"Web3\", block_with_txn: BlockData, mined_txn_hash: HexStr\n    ) -> None:\n        receipt = w3.eth.wait_for_transaction_receipt(mined_txn_hash)\n        assert is_dict(receipt)\n        assert receipt[\"blockNumber\"] == block_with_txn[\"number\"]\n        assert receipt[\"blockHash\"] == block_with_txn[\"hash\"]\n        assert receipt[\"transactionIndex\"] == 0\n        assert receipt[\"transactionHash\"] == HexBytes(mined_txn_hash)\n        assert is_checksum_address(receipt[\"to\"])\n        assert receipt[\"from\"] is not None\n        assert is_checksum_address(receipt[\"from\"])\n\n        effective_gas_price = receipt[\"effectiveGasPrice\"]\n        assert isinstance(effective_gas_price, int)\n        assert effective_gas_price > 0\n\n    def test_eth_wait_for_transaction_receipt_unmined(\n        self, w3: \"Web3\", unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_hash = w3.eth.send_transaction(\n            {\n                \"from\": unlocked_account_dual_type,\n                \"to\": unlocked_account_dual_type,\n                \"value\": Wei(1),\n                \"gas\": 21000,\n                \"maxFeePerGas\": w3.to_wei(3, \"gwei\"),\n                \"maxPriorityFeePerGas\": w3.to_wei(1, \"gwei\"),\n            }\n        )\n\n        timeout = 0.01\n        with pytest.raises(TimeExhausted) as exc_info:\n            w3.eth.wait_for_transaction_receipt(txn_hash, timeout=timeout)\n\n        assert (_ in str(exc_info) for _ in [repr(txn_hash), timeout])\n\n    def test_eth_wait_for_transaction_receipt_with_log_entry(\n        self,\n        w3: \"Web3\",\n        block_with_txn_with_log: BlockData,\n        emitter_contract: \"Contract\",\n        txn_hash_with_log: HexStr,\n    ) -> None:\n        receipt = w3.eth.wait_for_transaction_receipt(txn_hash_with_log)\n        assert is_dict(receipt)\n        assert receipt[\"blockNumber\"] == block_with_txn_with_log[\"number\"]\n        assert receipt[\"blockHash\"] == block_with_txn_with_log[\"hash\"]\n...\n", "needle_token_start": 12107, "needle_token_end": 12276, "code_context_ntokens": 16253, "output": ["```python\ndef test_eth_estimate_gas_revert_with_msg(\n    self, w3: \"Web3\", revert_contract: \"Contract\", unlocked_account: ChecksumAddress\n) -> None:\n    with pytest.raises(\n        ContractLogicError, match=\"execution reverted: Function has been reverted\"\n    ):\n        txn_params = revert_contract._prepare_transaction(\n            fn_name=\"revertWithMessage\",\n            transaction={\n                \"from\": unlocked_account,\n                \"to\": revert_contract.address,\n            },\n        )\n        w3.eth.estimate_gas(txn_params)\n```"]}
{"repo": "ethereum/web3.py", "name": "_make_get_request", "language": "python", "path": "web3/beacon/beacon.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to fetch data from a specified server endpoint and return the response in a structured format.\n2. **Input**: It takes a string representing the specific endpoint of a server from which data needs to be retrieved.\n3. **Output**: It returns a dictionary containing the data fetched from the server endpoint.\n4. **Procedure**: The function constructs the full URL by appending the endpoint to the base server URL. It then sends an HTTP GET request to this URL and waits for the response. The response is processed and returned as a JSON-formatted dictionary.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: web3/_utils/module_testing/web3_module.py\nimport pytest\nfrom typing import (\n    Any,\n    NoReturn,\n    Sequence,\n    Union,\n)\n\nfrom eth_typing import (\n    ChecksumAddress,\n    HexAddress,\n    HexStr,\n    TypeStr,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3 import (\n    AsyncWeb3,\n    Web3,\n)\nfrom web3._utils.ens import (\n    ens_addresses,\n)\nfrom web3.exceptions import (\n    InvalidAddress,\n)\n\n\nclass Web3ModuleTest:\n    def test_web3_client_version(self, w3: Web3) -> None:\n        client_version = w3.client_version\n        self._check_web3_client_version(client_version)\n\n    def _check_web3_client_version(self, client_version: str) -> NoReturn:\n        raise NotImplementedError(\"Must be implemented by subclasses\")\n\n    # Contract that calculated test values can be found at\n    # https://kovan.etherscan.io/address/0xb9be06f5b99372cf9afbccadbbb9954ccaf7f4bb#code\n    @pytest.mark.parametrize(\n        \"types,values,expected\",\n        (\n            (\n                [\"bool\"],\n                [True],\n                HexBytes(\n                    \"0x5fe7f977e71dba2ea1a68e21057beebb9be2ac30c6410aa38d4f3fbe41dcffd2\"\n                ),\n            ),\n            (\n                [\"uint8\", \"uint8\", \"uint8\"],\n                [97, 98, 99],\n                HexBytes(\n                    \"0x4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45\"\n                ),\n            ),\n            (\n                [\"uint248\"],\n                [30],\n                HexBytes(\n                    \"0x30f95d210785601eb33ae4d53d405b26f920e765dff87cca8e9a4aec99f82671\"\n                ),\n            ),\n            (\n                [\"bool\", \"uint16\"],\n                [True, 299],\n                HexBytes(\n                    \"0xed18599ccd80ee9fae9a28b0e34a5573c3233d7468f808fd659bc171cf0b43bd\"\n                ),\n            ),\n            (\n                [\"int256\"],\n                [-10],\n                HexBytes(\n                    \"0xd6fb717f7e270a360f5093ce6a7a3752183e89c9a9afe5c0cb54b458a304d3d5\"\n                ),\n            ),\n            (\n                [\"int256\"],\n                [10],\n                HexBytes(\n                    \"0xc65a7bb8d6351c1cf70c95a316cc6a92839c986682d98bc35f958f4883f9d2a8\"\n                ),\n            ),\n            (\n                [\"int8\", \"uint8\"],\n                [-10, 18],\n                HexBytes(\n                    \"0x5c6ab1e634c08d9c0f4df4d789e8727943ef010dd7ca8e3c89de197a26d148be\"\n                ),\n            ),\n            (\n                [\"address\"],\n                [\"0x49eddd3769c0712032808d86597b84ac5c2f5614\"],\n                InvalidAddress,\n            ),\n            (\n                [\"address\"],\n                [\"0x49EdDD3769c0712032808D86597B84ac5c2F5614\"],\n                HexBytes(\n                    \"0x2ff37b5607484cd4eecf6d13292e22bd6e5401eaffcc07e279583bc742c68882\"\n                ),\n            ),\n            (\n                [\"bytes2\"],\n                [\"0x5402\"],\n                HexBytes(\n                    \"0x4ed9171bda52fca71ab28e7f452bd6eacc3e5a568a47e0fa53b503159a9b8910\"\n                ),\n            ),\n            (\n                [\"bytes3\"],\n                [\"0x5402\"],\n                HexBytes(\n                    \"0x4ed9171bda52fca71ab28e7f452bd6eacc3e5a568a47e0fa53b503159a9b8910\"\n                ),\n            ),\n            (\n                [\"bytes\"],\n                [\n                    \"0x636865636b6c6f6e6762797465737472696e676167\"\n                    \"61696e7374736f6c6964697479736861336861736866756e6374696f6e\"\n                ],\n                HexBytes(\n                    \"0xd78a84d65721b67e4011b10c99dafdedcdcd7cb30153064f773e210b4762e22f\"\n                ),\n            ),\n            (\n                [\"string\"],\n                [\"testing a string!\"],\n                HexBytes(\n                    \"0xe8c275c0b4070a5ec6cfcb83f0ba394b30ddd283de785d43f2eabfb04bd96747\"\n                ),\n            ),\n            (\n                [\"string\", \"bool\", \"uint16\", \"bytes2\", \"address\"],\n                [\n                    \"testing a string!\",\n                    False,\n                    299,\n                    \"0x5402\",\n                    \"0x49eddd3769c0712032808d86597b84ac5c2f5614\",\n                ],\n                InvalidAddress,\n            ),\n            (\n                [\"string\", \"bool\", \"uint16\", \"bytes2\", \"address\"],\n                [\n                    \"testing a string!\",\n                    False,\n                    299,\n                    \"0x5402\",\n                    \"0x49EdDD3769c0712032808D86597B84ac5c2F5614\",\n                ],\n                HexBytes(\n                    \"0x8cc6eabb25b842715e8ca39e2524ed946759aa37bfb7d4b81829cf5a7e266103\"\n                ),\n            ),\n            (\n                [\"bool[2][]\"],\n                [[[True, False], [False, True]]],\n                HexBytes(\n                    \"0x1eef261f2eb51a8c736d52be3f91ff79e78a9ec5df2b7f50d0c6f98ed1e2bc06\"\n                ),\n            ),\n            (\n                [\"bool[]\"],\n                [[True, False, True]],\n                HexBytes(\n                    \"0x5c6090c0461491a2941743bda5c3658bf1ea53bbd3edcde54e16205e18b45792\"\n                ),\n            ),\n            (\n                [\"uint24[]\"],\n                [[1, 0, 1]],\n                HexBytes(\n                    \"0x5c6090c0461491a2941743bda5c3658bf1ea53bbd3edcde54e16205e18b45792\"\n                ),\n            ),\n            (\n                [\"uint8[2]\"],\n                [[8, 9]],\n                HexBytes(\n                    \"0xc7694af312c4f286114180fd0ba6a52461fcee8a381636770b19a343af92538a\"\n                ),\n            ),\n            (\n                [\"uint256[2]\"],\n                [[8, 9]],\n                HexBytes(\n                    \"0xc7694af312c4f286114180fd0ba6a52461fcee8a381636770b19a343af92538a\"\n                ),\n            ),\n            (\n                [\"uint8[]\"],\n                [[8]],\n                HexBytes(\n                    \"0xf3f7a9fe364faab93b216da50a3214154f22a0a2b415b23a84c8169e8b636ee3\"\n                ),\n            ),\n            (\n                [\"address[]\"],\n                [\n                    [\n                        \"0x49EdDD3769c0712032808D86597B84ac5c2F5614\",\n                        \"0xA6b759bBbf4B59D24acf7E06e79f3a5D104fdCE5\",\n                    ]\n                ],\n                HexBytes(\n...\n# Path: web3/_utils/module_testing/__init__.py\nfrom .eth_module import (\n    AsyncEthModuleTest,\n    EthModuleTest,\n)\nfrom .go_ethereum_admin_module import (\n    GoEthereumAdminModuleTest,\n)\nfrom .go_ethereum_personal_module import (\n    GoEthereumPersonalModuleTest,\n)\nfrom .go_ethereum_txpool_module import (\n    GoEthereumAsyncTxPoolModuleTest,\n    GoEthereumTxPoolModuleTest,\n)\nfrom .net_module import (\n    AsyncNetModuleTest,\n    NetModuleTest,\n)\nfrom .web3_module import (\n    Web3ModuleTest,\n)\n\n# Path: web3/_utils/module_testing/persistent_connection_provider.py\nimport asyncio\nimport pytest\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    Tuple,\n    cast,\n)\n\nfrom eth_utils import (\n    is_hexstr,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3.datastructures import (\n    AttributeDict,\n)\nfrom web3.middleware import (\n    ExtraDataToPOAMiddleware,\n)\nfrom web3.types import (\n    FormattedEthSubscriptionResponse,\n)\n\nif TYPE_CHECKING:\n    from web3.main import (\n        AsyncWeb3,\n    )\n\n\nclass PersistentConnectionProviderTest:\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\n        \"subscription_params,ws_subscription_response,expected_formatted_result\",\n        (\n            (\n                (\"newHeads\",),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": {\n                            \"number\": \"0x539\",\n                            \"hash\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                            \"parentHash\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                            \"sha3Uncles\": \"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\",  # noqa: E501\n                            \"logsBloom\": \"0x00\",\n                            \"transactionsRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"stateRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"receiptsRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"miner\": \"0x0000000000000000000000000000000000000000\",\n                            \"difficulty\": \"0x0\",\n                            \"extraData\": \"0x496c6c756d696e61746520446d6f63726174697a6520447374726962757465\",  # noqa: E501\n                            \"gasLimit\": \"0x1c9c380\",\n                            \"gasUsed\": \"0xd1ce44\",\n                            \"timestamp\": \"0x539\",\n                            \"baseFeePerGas\": \"0x26f93fef9\",\n                            \"withdrawalsRoot\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"nonce\": \"0x0000000000000000\",\n                            \"mixHash\": \"0x73e9e036ec894047f29954571d4b6d9e8717de7304269c263cbf150caa4e0768\",  # noqa: E501\n                        },\n                    },\n                },\n                AttributeDict(\n                    {\n                        \"number\": 1337,\n                        \"hash\": HexBytes(\n                            \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"  # noqa: E501\n                        ),\n                        \"parentHash\": HexBytes(\n                            \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"  # noqa: E501\n                        ),\n                        \"sha3Uncles\": HexBytes(\n                            \"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347\"  # noqa: E501\n                        ),\n                        \"logsBloom\": HexBytes(\"0x00\"),\n                        \"transactionsRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"stateRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"receiptsRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"miner\": \"0x0000000000000000000000000000000000000000\",\n                        \"difficulty\": 0,\n                        \"extraData\": HexBytes(\n                            \"0x496c6c756d696e61746520446d6f63726174697a6520447374726962757465\"  # noqa: E501\n                        ),\n                        \"gasLimit\": 30000000,\n                        \"gasUsed\": 13749828,\n                        \"timestamp\": 1337,\n                        \"baseFeePerGas\": 10461904633,\n                        \"withdrawalsRoot\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"nonce\": HexBytes(\"0x0000000000000000\"),\n                        \"mixHash\": HexBytes(\n                            \"0x73e9e036ec894047f29954571d4b6d9e8717de7304269c263cbf150caa4e0768\"  # noqa: E501\n                        ),\n                    }\n                ),\n            ),\n            (\n                (\"newPendingTransactions\", True),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": {\n                            \"blockHash\": None,\n                            \"blockNumber\": None,\n                            \"from\": \"0x0000000000000000000000000000000000000000\",\n                            \"gas\": \"0xf2f4\",\n                            \"gasPrice\": \"0x29035f36f\",\n                            \"maxFeePerGas\": \"0x29035f36f\",\n                            \"maxPriorityFeePerGas\": \"0x3b9aca00\",\n                            \"hash\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                            \"input\": \"0x00\",\n                            \"nonce\": \"0x2013\",\n                            \"to\": \"0x0000000000000000000000000000000000000000\",\n                            \"transactionIndex\": None,\n                            \"value\": \"0x0\",\n                            \"type\": \"0x2\",\n                            \"accessList\": [],\n                            \"chainId\": \"0x1\",\n                            \"v\": \"0x1\",\n                            \"r\": \"0x3c144a7c00ed3118d55445cd5be2ae4620ca377f7c685e9c5f3687671d4dece1\",  # noqa: E501\n                            \"s\": \"0x284de67cbf75fec8a9edb368dee3a37cf6faba87f0af4413b2f869ebfa87d002\",  # noqa: E501\n                            \"yParity\": \"0x1\",\n                        },\n                    },\n                },\n                AttributeDict(\n                    {\n                        \"blockHash\": None,\n                        \"blockNumber\": None,\n                        \"from\": \"0x0000000000000000000000000000000000000000\",\n                        \"gas\": 62196,\n                        \"gasPrice\": 11009389423,\n                        \"maxFeePerGas\": 11009389423,\n                        \"maxPriorityFeePerGas\": 1000000000,\n                        \"hash\": HexBytes(\n                            \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"  # noqa: E501\n                        ),\n                        \"input\": HexBytes(\"0x00\"),\n                        \"nonce\": 8211,\n                        \"to\": \"0x0000000000000000000000000000000000000000\",\n                        \"transactionIndex\": None,\n                        \"value\": 0,\n                        \"type\": 2,\n                        \"accessList\": [],\n                        \"chainId\": 1,\n                        \"v\": 1,\n                        \"r\": HexBytes(\n                            \"0x3c144a7c00ed3118d55445cd5be2ae4620ca377f7c685e9c5f3687671d4dece1\"  # noqa: E501\n                        ),\n                        \"s\": HexBytes(\n                            \"0x284de67cbf75fec8a9edb368dee3a37cf6faba87f0af4413b2f869ebfa87d002\"  # noqa: E501\n                        ),\n                        \"yParity\": 1,\n                    }\n                ),\n            ),\n            (\n                (\"newPendingTransactions\", False),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                    },\n                },\n                HexBytes(\n                    \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"\n                ),\n            ),\n            (\n                (\"logs\", {\"address\": \"0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2\"}),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": {\n                            \"removed\": False,\n                            \"logIndex\": \"0x0\",\n                            \"transactionIndex\": \"0x0\",\n                            \"transactionHash\": \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\",  # noqa: E501\n                            \"blockHash\": \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\",  # noqa: E501\n                            \"blockNumber\": \"0x539\",\n                            \"address\": \"0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2\",\n                            \"data\": \"0x00\",\n                            \"topics\": [\n                                \"0xe1fffdd4923d04f559f4d29e8bfc6cda04eb5b0d3c460751c2402c5c5cc9105c\",  # noqa: E501\n                                \"0x00000000000000000000000016250d5630b4cf539739df2c5dacb4c659f2482d\",  # noqa: E501\n                            ],\n                        },\n                    },\n                },\n                AttributeDict(\n                    {\n                        \"removed\": False,\n                        \"logIndex\": 0,\n                        \"transactionIndex\": 0,\n                        \"transactionHash\": HexBytes(\n                            \"0x56260fe8298aff6d360e3a68fa855693f25dcb2708d8a7e509e8519b265d3988\"  # noqa: E501\n                        ),\n                        \"blockHash\": HexBytes(\n                            \"0xb46b85928f2c2264c2bf7ad5c6d6985664f1527e744193ef990cc0d3da5afc5e\"  # noqa: E501\n                        ),\n                        \"blockNumber\": 1337,\n                        \"address\": \"0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2\",\n                        \"data\": HexBytes(\"0x00\"),\n                        \"topics\": [\n                            HexBytes(\n                                \"0xe1fffdd4923d04f559f4d29e8bfc6cda04eb5b0d3c460751c2402c5c5cc9105c\"  # noqa: E501\n                            ),\n                            HexBytes(\n                                \"0x00000000000000000000000016250d5630b4cf539739df2c5dacb4c659f2482d\"  # noqa: E501\n                            ),\n                        ],\n                    }\n                ),\n            ),\n            (\n                (\"syncing\",),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": False,\n                    },\n                },\n                False,\n            ),\n            (\n                (\"syncing\",),\n                {\n                    \"jsonrpc\": \"2.0\",\n                    \"method\": \"eth_subscription\",\n                    \"params\": {\n                        \"subscription\": \"THIS_WILL_BE_REPLACED_IN_THE_TEST\",\n                        \"result\": {\n                            \"isSyncing\": True,\n                            \"startingBlock\": \"0x0\",\n                            \"currentBlock\": \"0x4346fe\",\n                            \"highestBlock\": \"0x434806\",\n                        },\n                    },\n                },\n                AttributeDict(\n                    {\n                        \"isSyncing\": True,\n                        \"startingBlock\": 0,\n                        \"currentBlock\": 4409086,\n                        \"highestBlock\": 4409350,\n                    }\n                ),\n            ),\n        ),\n        ids=[\n            \"newHeads\",\n            \"newPendingTransactions-FullTxs\",\n            \"newPendingTransactions-TxHashes\",\n            \"logs\",\n            \"syncing-False\",\n            \"syncing-True\",\n        ],\n    )\n    async def test_async_eth_subscribe_mocked(\n        self,\n        async_w3: \"AsyncWeb3\",\n        subscription_params: Tuple[Any, ...],\n        ws_subscription_response: Dict[str, Any],\n        expected_formatted_result: Any,\n    ) -> None:\n        sub_id = await async_w3.eth.subscribe(*subscription_params)\n        assert is_hexstr(sub_id)\n\n        # stub out the subscription id so we know how to process the response\n        ws_subscription_response[\"params\"][\"subscription\"] = sub_id\n\n        # add the response to the subscription response cache as if it came from the\n        # websocket connection\n        await async_w3.provider._request_processor.cache_raw_response(\n            ws_subscription_response, subscription=True\n        )\n\n        async for msg in async_w3.socket.process_subscriptions():\n            response = cast(FormattedEthSubscriptionResponse, msg)\n            assert response[\"subscription\"] == sub_id\n            assert response[\"result\"] == expected_formatted_result\n\n            # only testing one message, so break here\n            break\n\n    @pytest.mark.asyncio\n    async def test_async_extradata_poa_middleware_on_eth_subscription(\n        self,\n        async_w3: \"AsyncWeb3\",\n    ) -> None:\n        async_w3.middleware_onion.inject(\n            ExtraDataToPOAMiddleware, \"poa_middleware\", layer=0\n        )\n\n        sub_id = await async_w3.eth.subscribe(\"newHeads\")\n        assert is_hexstr(sub_id)\n\n        # add the response to the subscription response cache as if it came from the\n        # websocket connection\n        await async_w3.provider._request_processor.cache_raw_response(\n            {\n                \"jsonrpc\": \"2.0\",\n                \"method\": \"eth_subscription\",\n                \"params\": {\n                    \"subscription\": sub_id,\n                    \"result\": {\n                        \"extraData\": f\"0x{'00' * 100}\",\n                    },\n                },\n            },\n            subscription=True,\n        )\n\n        async for msg in async_w3.socket.process_subscriptions():\n            response = cast(FormattedEthSubscriptionResponse, msg)\n            assert response.keys() == {\"subscription\", \"result\"}\n            assert response[\"subscription\"] == sub_id\n            assert response[\"result\"][\"proofOfAuthorityData\"] == HexBytes(  # type: ignore  # noqa: E501\n                f\"0x{'00' * 100}\"\n            )\n\n            # only testing one message, so break here\n            break\n\n        # clean up\n        async_w3.middleware_onion.remove(\"poa_middleware\")\n\n    @pytest.mark.asyncio\n    async def test_asyncio_gather_for_multiple_requests_matches_the_responses(\n        self,\n        async_w3: \"AsyncWeb3\",\n    ) -> None:\n        (\n            latest,\n            chain_id,\n            block_num,\n            chain_id2,\n            pending,\n            chain_id3,\n        ) = await asyncio.gather(\n            async_w3.eth.get_block(\"latest\"),\n            async_w3.eth.chain_id,\n            async_w3.eth.block_number,\n            async_w3.eth.chain_id,\n            async_w3.eth.get_block(\"pending\"),\n            async_w3.eth.chain_id,\n        )\n\n        # assert attrdict middleware was applied appropriately\n        assert isinstance(latest, AttributeDict)\n        assert isinstance(pending, AttributeDict)\n\n        # assert block values\n        some_block_keys = [\n            \"number\",\n            \"hash\",\n            \"parentHash\",\n            \"transactionsRoot\",\n            \"stateRoot\",\n            \"receiptsRoot\",\n            \"size\",\n            \"gasLimit\",\n            \"gasUsed\",\n            \"timestamp\",\n            \"transactions\",\n            \"baseFeePerGas\",\n        ]\n        assert all(k in latest.keys() for k in some_block_keys)\n        assert all(k in pending.keys() for k in some_block_keys)\n\n        assert isinstance(block_num, int)\n        assert latest[\"number\"] == block_num\n\n        assert isinstance(chain_id, int)\n        assert isinstance(chain_id2, int)\n        assert isinstance(chain_id3, int)\n\n# Path: web3/auto/__init__.py\nfrom web3 import (\n    Web3,\n)\n\nw3 = Web3()\n\n# Path: web3/auto/gethdev.py\nfrom web3 import (\n    AsyncIPCProvider,\n    AsyncWeb3,\n    IPCProvider,\n    Web3,\n)\nfrom web3.middleware import (\n    ExtraDataToPOAMiddleware,\n)\nfrom web3.providers.ipc import (\n    get_dev_ipc_path,\n)\n\nw3 = Web3(IPCProvider(get_dev_ipc_path()))\nw3.middleware_onion.inject(ExtraDataToPOAMiddleware, layer=0)\n\nasync_w3 = AsyncWeb3(AsyncIPCProvider(get_dev_ipc_path()))\nasync_w3.middleware_onion.inject(ExtraDataToPOAMiddleware, layer=0)\n\n# Path: web3/beacon/api_endpoints.py\n# [ BEACON endpoints ]\n\nGET_GENESIS = \"/eth/v1/beacon/genesis\"\n\n# states\nGET_HASH_ROOT = \"/eth/v1/beacon/states/{0}/root\"\nGET_FORK_DATA = \"/eth/v1/beacon/states/{0}/fork\"\nGET_FINALITY_CHECKPOINT = \"/eth/v1/beacon/states/{0}/finality_checkpoints\"\nGET_VALIDATORS = \"/eth/v1/beacon/states/{0}/validators\"\nGET_VALIDATOR = \"/eth/v1/beacon/states/{0}/validators/{1}\"\nGET_VALIDATOR_BALANCES = \"/eth/v1/beacon/states/{0}/validator_balances\"\nGET_EPOCH_COMMITTEES = \"/eth/v1/beacon/states/{0}/committees\"\nGET_EPOCH_SYNC_COMMITTEES = \"/eth/v1/beacon/states/{0}/sync_committees\"\nGET_EPOCH_RANDAO = \"/eth/v1/beacon/states/{0}/randao\"\n\n# headers\nGET_BLOCK_HEADERS = \"/eth/v1/beacon/headers\"\nGET_BLOCK_HEADER = \"/eth/v1/beacon/headers/{0}\"\n\n# blocks\nGET_BLOCK = \"/eth/v2/beacon/blocks/{0}\"\nGET_BLOCK_ROOT = \"/eth/v1/beacon/blocks/{0}/root\"\nGET_BLOCK_ATTESTATIONS = \"/eth/v1/beacon/blocks/{0}/attestations\"\nGET_BLINDED_BLOCKS = \"/eth/v1/beacon/blinded_blocks/{0}\"\n\n# rewards\nGET_REWARDS = \"/eth/v1/beacon/rewards/blocks/{0}\"\n\n# light client\nGET_LIGHT_CLIENT_BOOTSTRAP_STRUCTURE = \"/eth/v1/beacon/light_client/bootstrap/{0}\"\nGET_LIGHT_CLIENT_UPDATES = \"/eth/v1/beacon/light_client/updates\"\nGET_LIGHT_CLIENT_FINALITY_UPDATE = \"/eth/v1/beacon/light_client/finality_update\"\nGET_LIGHT_CLIENT_OPTIMISTIC_UPDATE = \"/eth/v1/beacon/light_client/optimistic_update\"\n\n# pool\nGET_ATTESTATIONS = \"/eth/v1/beacon/pool/attestations\"\nGET_ATTESTER_SLASHINGS = \"/eth/v1/beacon/pool/attester_slashings\"\nGET_PROPOSER_SLASHINGS = \"/eth/v1/beacon/pool/proposer_slashings\"\nGET_VOLUNTARY_EXITS = \"/eth/v1/beacon/pool/voluntary_exits\"\nGET_BLS_TO_EXECUTION_CHANGES = \"/eth/v1/beacon/pool/bls_to_execution_changes\"\n\n\n# [ CONFIG endpoints ]\n\nGET_FORK_SCHEDULE = \"/eth/v1/config/fork_schedule\"\nGET_SPEC = \"/eth/v1/config/spec\"\nGET_DEPOSIT_CONTRACT = \"/eth/v1/config/deposit_contract\"\n\n# [ DEBUG endpoints ]\n\nGET_BEACON_STATE = \"/eth/v1/debug/beacon/states/{0}\"\nGET_BEACON_HEADS = \"/eth/v1/debug/beacon/heads\"\n\n# [ NODE endpoints ]\n\nGET_NODE_IDENTITY = \"/eth/v1/node/identity\"\nGET_PEERS = \"/eth/v1/node/peers\"\nGET_PEER = \"/eth/v1/node/peers/{0}\"\nGET_HEALTH = \"/eth/v1/node/health\"\nGET_VERSION = \"/eth/v1/node/version\"\nGET_SYNCING = \"/eth/v1/node/syncing\"\n\n# Path: web3/beacon/async_beacon.py\nfrom typing import (\n    Any,\n    Dict,\n)\n\nfrom eth_typing import (\n    URI,\n    HexStr,\n)\n\nfrom web3._utils.request import (\n    async_get_response_from_get_request,\n    async_json_make_get_request,\n)\nfrom web3.beacon.api_endpoints import (\n    GET_ATTESTATIONS,\n    GET_ATTESTER_SLASHINGS,\n    GET_BEACON_HEADS,\n    GET_BEACON_STATE,\n    GET_BLINDED_BLOCKS,\n    GET_BLOCK,\n    GET_BLOCK_ATTESTATIONS,\n    GET_BLOCK_HEADER,\n    GET_BLOCK_HEADERS,\n    GET_BLOCK_ROOT,\n    GET_BLS_TO_EXECUTION_CHANGES,\n    GET_DEPOSIT_CONTRACT,\n    GET_EPOCH_COMMITTEES,\n    GET_EPOCH_RANDAO,\n    GET_EPOCH_SYNC_COMMITTEES,\n    GET_FINALITY_CHECKPOINT,\n    GET_FORK_DATA,\n    GET_FORK_SCHEDULE,\n    GET_GENESIS,\n    GET_HASH_ROOT,\n    GET_HEALTH,\n    GET_LIGHT_CLIENT_BOOTSTRAP_STRUCTURE,\n    GET_LIGHT_CLIENT_FINALITY_UPDATE,\n    GET_LIGHT_CLIENT_OPTIMISTIC_UPDATE,\n    GET_LIGHT_CLIENT_UPDATES,\n    GET_NODE_IDENTITY,\n    GET_PEER,\n    GET_PEERS,\n    GET_PROPOSER_SLASHINGS,\n    GET_REWARDS,\n    GET_SPEC,\n    GET_SYNCING,\n    GET_VALIDATOR,\n    GET_VALIDATOR_BALANCES,\n    GET_VALIDATORS,\n    GET_VERSION,\n    GET_VOLUNTARY_EXITS,\n)\n\n\nclass AsyncBeacon:\n    is_async = True\n\n    def __init__(\n        self,\n        base_url: str,\n        request_timeout: float = 10.0,\n    ) -> None:\n        self.base_url = base_url\n        self.request_timeout = request_timeout\n\n    async def _async_make_get_request(self, endpoint_uri: str) -> Dict[str, Any]:\n        uri = URI(self.base_url + endpoint_uri)\n        return await async_json_make_get_request(uri, timeout=self.request_timeout)\n\n    # [ BEACON endpoints ]\n\n    # states\n\n    async def get_genesis(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_GENESIS)\n\n    async def get_hash_root(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_HASH_ROOT.format(state_id))\n\n    async def get_fork_data(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_FORK_DATA.format(state_id))\n\n    async def get_finality_checkpoint(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(\n            GET_FINALITY_CHECKPOINT.format(state_id)\n        )\n\n    async def get_validators(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_VALIDATORS.format(state_id))\n\n    async def get_validator(\n        self, validator_id: str, state_id: str = \"head\"\n    ) -> Dict[str, Any]:\n        return await self._async_make_get_request(\n            GET_VALIDATOR.format(state_id, validator_id)\n        )\n\n    async def get_validator_balances(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(\n            GET_VALIDATOR_BALANCES.format(state_id)\n        )\n\n    async def get_epoch_committees(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_EPOCH_COMMITTEES.format(state_id))\n\n    async def get_epoch_sync_committees(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(\n            GET_EPOCH_SYNC_COMMITTEES.format(state_id)\n        )\n\n    async def get_epoch_randao(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_EPOCH_RANDAO.format(state_id))\n\n    # headers\n\n    async def get_block_headers(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BLOCK_HEADERS)\n\n    async def get_block_header(self, block_id: str) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BLOCK_HEADER.format(block_id))\n\n    # block\n\n    async def get_block(self, block_id: str) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BLOCK.format(block_id))\n\n    async def get_block_root(self, block_id: str) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BLOCK_ROOT.format(block_id))\n\n    async def get_block_attestations(self, block_id: str) -> Dict[str, Any]:\n        return await self._async_make_get_request(\n            GET_BLOCK_ATTESTATIONS.format(block_id)\n        )\n\n    async def get_blinded_blocks(self, block_id: str) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BLINDED_BLOCKS.format(block_id))\n\n    # rewards\n\n    async def get_rewards(self, block_id: str) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_REWARDS.format(block_id))\n\n    # light client (untested but follows spec)\n\n    async def get_light_client_bootstrap_structure(\n        self, block_root: HexStr\n    ) -> Dict[str, Any]:\n        return await self._async_make_get_request(\n            GET_LIGHT_CLIENT_BOOTSTRAP_STRUCTURE.format(block_root)\n        )\n\n    async def get_light_client_updates(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_LIGHT_CLIENT_UPDATES)\n\n    async def get_light_client_finality_update(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_LIGHT_CLIENT_FINALITY_UPDATE)\n\n    async def get_light_client_optimistic_update(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_LIGHT_CLIENT_OPTIMISTIC_UPDATE)\n\n    # pool\n\n    async def get_attestations(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_ATTESTATIONS)\n\n    async def get_attester_slashings(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_ATTESTER_SLASHINGS)\n\n    async def get_proposer_slashings(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_PROPOSER_SLASHINGS)\n\n    async def get_voluntary_exits(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_VOLUNTARY_EXITS)\n\n    async def get_bls_to_execution_changes(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BLS_TO_EXECUTION_CHANGES)\n\n    # [ CONFIG endpoints ]\n\n    async def get_fork_schedule(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_FORK_SCHEDULE)\n\n    async def get_spec(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_SPEC)\n\n    async def get_deposit_contract(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_DEPOSIT_CONTRACT)\n\n    # [ DEBUG endpoints ]\n\n    async def get_beacon_state(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BEACON_STATE.format(state_id))\n\n    async def get_beacon_heads(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_BEACON_HEADS)\n\n    # [ NODE endpoints ]\n\n    async def get_node_identity(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_NODE_IDENTITY)\n\n    async def get_peers(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_PEERS)\n\n    async def get_peer(self, peer_id: str) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_PEER.format(peer_id))\n\n    async def get_health(self) -> int:\n        url = URI(self.base_url + GET_HEALTH)\n        response = await async_get_response_from_get_request(url)\n        return response.status\n\n    async def get_version(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_VERSION)\n\n    async def get_syncing(self) -> Dict[str, Any]:\n        return await self._async_make_get_request(GET_SYNCING)\n\n# Path: web3/beacon/beacon.py\nfrom typing import (\n    Any,\n    Dict,\n)\n\nfrom eth_typing import (\n    URI,\n    HexStr,\n)\n\nfrom web3._utils.request import (\n    get_response_from_get_request,\n    json_make_get_request,\n)\nfrom web3.beacon.api_endpoints import (\n    GET_ATTESTATIONS,\n    GET_ATTESTER_SLASHINGS,\n    GET_BEACON_HEADS,\n    GET_BEACON_STATE,\n    GET_BLINDED_BLOCKS,\n    GET_BLOCK,\n    GET_BLOCK_ATTESTATIONS,\n    GET_BLOCK_HEADER,\n    GET_BLOCK_HEADERS,\n    GET_BLOCK_ROOT,\n    GET_BLS_TO_EXECUTION_CHANGES,\n    GET_DEPOSIT_CONTRACT,\n    GET_EPOCH_COMMITTEES,\n    GET_EPOCH_RANDAO,\n    GET_EPOCH_SYNC_COMMITTEES,\n    GET_FINALITY_CHECKPOINT,\n    GET_FORK_DATA,\n    GET_FORK_SCHEDULE,\n    GET_GENESIS,\n    GET_HASH_ROOT,\n    GET_HEALTH,\n    GET_LIGHT_CLIENT_BOOTSTRAP_STRUCTURE,\n    GET_LIGHT_CLIENT_FINALITY_UPDATE,\n    GET_LIGHT_CLIENT_OPTIMISTIC_UPDATE,\n    GET_LIGHT_CLIENT_UPDATES,\n    GET_NODE_IDENTITY,\n    GET_PEER,\n    GET_PEERS,\n    GET_PROPOSER_SLASHINGS,\n    GET_REWARDS,\n    GET_SPEC,\n    GET_SYNCING,\n    GET_VALIDATOR,\n    GET_VALIDATOR_BALANCES,\n    GET_VALIDATORS,\n    GET_VERSION,\n    GET_VOLUNTARY_EXITS,\n)\n\n\nclass Beacon:\n    def __init__(\n        self,\n        base_url: str,\n        request_timeout: float = 10.0,\n    ) -> None:\n        self.base_url = base_url\n        self.request_timeout = request_timeout\n\n    \ndef _make_get_request(self, endpoint_url: str) -> Dict[str, Any]:\n        uri = URI(self.base_url + endpoint_url)\n        return json_make_get_request(uri, timeout=self.request_timeout)\n\n    # [ BEACON endpoints ]\n\n    # states\n\n    def get_genesis(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_GENESIS)\n\n    def get_hash_root(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_HASH_ROOT.format(state_id))\n\n    def get_fork_data(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_FORK_DATA.format(state_id))\n\n    def get_finality_checkpoint(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_FINALITY_CHECKPOINT.format(state_id))\n\n    def get_validators(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_VALIDATORS.format(state_id))\n\n    def get_validator(\n        self, validator_id: str, state_id: str = \"head\"\n    ) -> Dict[str, Any]:\n        return self._make_get_request(GET_VALIDATOR.format(state_id, validator_id))\n\n    def get_validator_balances(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_VALIDATOR_BALANCES.format(state_id))\n\n    def get_epoch_committees(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_EPOCH_COMMITTEES.format(state_id))\n\n    def get_epoch_sync_committees(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_EPOCH_SYNC_COMMITTEES.format(state_id))\n\n    def get_epoch_randao(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_EPOCH_RANDAO.format(state_id))\n\n    # headers\n\n    def get_block_headers(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_HEADERS)\n\n    def get_block_header(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_HEADER.format(block_id))\n\n    # blocks\n\n    def get_block(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK.format(block_id))\n\n    def get_block_root(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_ROOT.format(block_id))\n\n    def get_block_attestations(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_ATTESTATIONS.format(block_id))\n\n    def get_blinded_blocks(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLINDED_BLOCKS.format(block_id))\n\n    # rewards\n\n    def get_rewards(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_REWARDS.format(block_id))\n\n    # light client (untested but follows spec)\n\n    def get_light_client_bootstrap_structure(\n        self, block_root: HexStr\n    ) -> Dict[str, Any]:\n        return self._make_get_request(\n            GET_LIGHT_CLIENT_BOOTSTRAP_STRUCTURE.format(block_root)\n        )\n\n    def get_light_client_updates(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_LIGHT_CLIENT_UPDATES)\n\n    def get_light_client_finality_update(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_LIGHT_CLIENT_FINALITY_UPDATE)\n\n    def get_light_client_optimistic_update(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_LIGHT_CLIENT_OPTIMISTIC_UPDATE)\n\n    # pool\n\n    def get_attestations(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_ATTESTATIONS)\n\n    def get_attester_slashings(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_ATTESTER_SLASHINGS)\n\n    def get_proposer_slashings(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_PROPOSER_SLASHINGS)\n\n    def get_voluntary_exits(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_VOLUNTARY_EXITS)\n\n    def get_bls_to_execution_changes(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLS_TO_EXECUTION_CHANGES)\n\n    # [ CONFIG endpoints ]\n\n    def get_fork_schedule(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_FORK_SCHEDULE)\n\n    def get_spec(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_SPEC)\n\n    def get_deposit_contract(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_DEPOSIT_CONTRACT)\n\n    # [ DEBUG endpoints ]\n\n    def get_beacon_state(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_BEACON_STATE.format(state_id))\n\n    def get_beacon_heads(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_BEACON_HEADS)\n\n    # [ NODE endpoints ]\n\n    def get_node_identity(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_NODE_IDENTITY)\n\n    def get_peers(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_PEERS)\n\n    def get_peer(self, peer_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_PEER.format(peer_id))\n\n    def get_health(self) -> int:\n        url = URI(self.base_url + GET_HEALTH)\n        response = get_response_from_get_request(url)\n        return response.status_code\n\n    def get_version(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_VERSION)\n\n    def get_syncing(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_SYNCING)\n\n# Path: web3/beacon/__init__.py\nfrom .async_beacon import AsyncBeacon\nfrom .beacon import Beacon\n\n# Path: web3/gas_strategies/rpc.py\nfrom typing import (\n    Optional,\n)\n\nfrom web3 import (\n    Web3,\n)\nfrom web3.types import (\n    TxParams,\n    Wei,\n)\n\n\ndef rpc_gas_price_strategy(\n    w3: Web3, transaction_params: Optional[TxParams] = None\n) -> Wei:\n    \"\"\"\n    A simple gas price strategy deriving it's value from the eth_gasPrice JSON-RPC call.\n    \"\"\"\n    return w3.eth.gas_price\n\n# Path: web3/gas_strategies/time_based.py\nimport collections\nimport math\nimport operator\nfrom typing import (\n    Iterable,\n    Sequence,\n    Tuple,\n)\n\nfrom eth_typing import (\n    ChecksumAddress,\n)\nfrom eth_utils import (\n    to_tuple,\n)\nfrom eth_utils.toolz import (\n    curry,\n    groupby,\n    sliding_window,\n)\nfrom hexbytes import (\n    HexBytes,\n)\n\nfrom web3 import (\n    Web3,\n)\nfrom web3._utils.math import (\n    percentile,\n)\nfrom web3.exceptions import (\n    InsufficientData,\n    Web3ValidationError,\n)\nfrom web3.types import (\n    BlockNumber,\n    GasPriceStrategy,\n    TxParams,\n    Wei,\n)\n\nMinerData = collections.namedtuple(\n    \"MinerData\", [\"miner\", \"num_blocks\", \"min_gas_price\", \"low_percentile_gas_price\"]\n)\n...\n", "needle_token_start": 13728, "needle_token_end": 13789, "code_context_ntokens": 16145, "output": ["```python\nfrom typing import (\n    Any,\n    Dict,\n)\n\nfrom eth_typing import (\n    URI,\n    HexStr,\n)\n\nfrom web3._utils.request import (\n    get_response_from_get_request,\n    json_make_get_request,\n)\nfrom web3.beacon.api_endpoints import (\n    GET_ATTESTATIONS,\n    GET_ATTESTER_SLASHINGS,\n    GET_BEACON_HEADS,\n    GET_BEACON_STATE,\n    GET_BLINDED_BLOCKS,\n    GET_BLOCK,\n    GET_BLOCK_ATTESTATIONS,\n    GET_BLOCK_HEADER,\n    GET_BLOCK_HEADERS,\n    GET_BLOCK_ROOT,\n    GET_BLS_TO_EXECUTION_CHANGES,\n    GET_DEPOSIT_CONTRACT,\n    GET_EPOCH_COMMITTEES,\n    GET_EPOCH_RANDAO,\n    GET_EPOCH_SYNC_COMMITTEES,\n    GET_FINALITY_CHECKPOINT,\n    GET_FORK_DATA,\n    GET_FORK_SCHEDULE,\n    GET_GENESIS,\n    GET_HASH_ROOT,\n    GET_HEALTH,\n    GET_LIGHT_CLIENT_BOOTSTRAP_STRUCTURE,\n    GET_LIGHT_CLIENT_FINALITY_UPDATE,\n    GET_LIGHT_CLIENT_OPTIMISTIC_UPDATE,\n    GET_LIGHT_CLIENT_UPDATES,\n    GET_NODE_IDENTITY,\n    GET_PEER,\n    GET_PEERS,\n    GET_PROPOSER_SLASHINGS,\n    GET_REWARDS,\n    GET_SPEC,\n    GET_SYNCING,\n    GET_VALIDATOR,\n    GET_VALIDATOR_BALANCES,\n    GET_VALIDATORS,\n    GET_VERSION,\n    GET_VOLUNTARY_EXITS,\n)\n\n\nclass Beacon:\n    def __init__(\n        self,\n        base_url: str,\n        request_timeout: float = 10.0,\n    ) -> None:\n        self.base_url = base_url\n        self.request_timeout = request_timeout\n\n    def _make_get_request(self, endpoint_url: str) -> Dict[str, Any]:\n        uri = URI(self.base_url + endpoint_url)\n        return json_make_get_request(uri, timeout=self.request_timeout)\n\n    # [ BEACON endpoints ]\n\n    # states\n\n    def get_genesis(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_GENESIS)\n\n    def get_hash_root(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_HASH_ROOT.format(state_id))\n\n    def get_fork_data(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_FORK_DATA.format(state_id))\n\n    def get_finality_checkpoint(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_FINALITY_CHECKPOINT.format(state_id))\n\n    def get_validators(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_VALIDATORS.format(state_id))\n\n    def get_validator(\n        self, validator_id: str, state_id: str = \"head\"\n    ) -> Dict[str, Any]:\n        return self._make_get_request(GET_VALIDATOR.format(state_id, validator_id))\n\n    def get_validator_balances(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_VALIDATOR_BALANCES.format(state_id))\n\n    def get_epoch_committees(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_EPOCH_COMMITTEES.format(state_id))\n\n    def get_epoch_sync_committees(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_EPOCH_SYNC_COMMITTEES.format(state_id))\n\n    def get_epoch_randao(self, state_id: str = \"head\") -> Dict[str, Any]:\n        return self._make_get_request(GET_EPOCH_RANDAO.format(state_id))\n\n    # headers\n\n    def get_block_headers(self) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_HEADERS)\n\n    def get_block_header(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_HEADER.format(block_id))\n\n    # blocks\n\n    def get_block(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK.format(block_id))\n\n    def get_block_root(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_ROOT.format(block_id))\n\n    def get_block_attestations(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLOCK_ATTESTATIONS.format(block_id))\n\n    def get_blinded_blocks(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_BLINDED_BLOCKS.format(block_id))\n\n    # rewards\n\n    def get_rewards(self, block_id: str) -> Dict[str, Any]:\n        return self._make_get_request(GET_REWARDS.format(block_id))\n\n    # light client (untested but follows spec)\n\n    def get_light_client"]}
{"repo": "ethereum/web3.py", "name": "test_eth_call_revert_custom_error_with_msg", "language": "python", "path": "web3/_utils/module_testing/eth_module.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To verify that a specific Ethereum smart contract function correctly raises a custom error with a predefined message when called under unauthorized conditions.\n2. **Input**: The test function inputs include a transaction parameter set specifying the caller and the contract address, and the encoded ABI of the unauthorized action.\n3. **Output**: The expected output is the raising of a custom error that matches the encoded unauthorized action message.\n4. **Procedure**: The test initiates by encoding the unauthorized action and its message using the contract's ABI. It then prepares a transaction to call the custom error function from an unauthorized account. The test asserts that this transaction call raises the expected custom error with the specific message encoded earlier.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            ),  # must be greater than base_fee post London\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = await async_w3.eth.modify_transaction(\n            txn_hash, gasPrice=(cast(int, txn_params[\"gasPrice\"]) * 2), value=2\n        )\n        modified_txn = await async_w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert modified_txn[\"gasPrice\"] == cast(int, txn_params[\"gasPrice\"]) * 2\n\n    @pytest.mark.asyncio\n    async def test_eth_modify_transaction(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n            \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n\n        modified_txn_hash = await async_w3.eth.modify_transaction(\n            txn_hash,\n            value=2,\n            maxPriorityFeePerGas=(cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2),\n            maxFeePerGas=(cast(Wei, txn_params[\"maxFeePerGas\"]) * 2),\n        )\n        modified_txn = await async_w3.eth.get_transaction(modified_txn_hash)\n\n        assert is_same_address(\n            modified_txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"])\n        )\n        assert is_same_address(\n            modified_txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"])\n        )\n        assert modified_txn[\"value\"] == 2\n        assert modified_txn[\"gas\"] == 21000\n        assert (\n            modified_txn[\"maxPriorityFeePerGas\"]\n            == cast(Wei, txn_params[\"maxPriorityFeePerGas\"]) * 2\n        )\n        assert modified_txn[\"maxFeePerGas\"] == cast(Wei, txn_params[\"maxFeePerGas\"]) * 2\n\n    @pytest.mark.asyncio\n    async def test_async_eth_sign_transaction(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n            \"nonce\": Nonce(0),\n        }\n        result = await async_w3.eth.sign_transaction(txn_params)\n        signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n        assert async_unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n        assert (\n            result[\"tx\"][\"maxPriorityFeePerGas\"] == txn_params[\"maxPriorityFeePerGas\"]\n        )\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_sign_typed_data(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n        async_skip_if_testrpc: Callable[[\"AsyncWeb3\"], None],\n    ) -> None:\n        validJSONMessage = \"\"\"\n            {\n                \"types\": {\n                    \"EIP712Domain\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"version\", \"type\": \"string\"},\n                        {\"name\": \"chainId\", \"type\": \"uint256\"},\n                        {\"name\": \"verifyingContract\", \"type\": \"address\"}\n                    ],\n                    \"Person\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"wallet\", \"type\": \"address\"}\n                    ],\n                    \"Mail\": [\n                        {\"name\": \"from\", \"type\": \"Person\"},\n                        {\"name\": \"to\", \"type\": \"Person\"},\n                        {\"name\": \"contents\", \"type\": \"string\"}\n                    ]\n                },\n                \"primaryType\": \"Mail\",\n                \"domain\": {\n                    \"name\": \"Ether Mail\",\n                    \"version\": \"1\",\n                    \"chainId\": \"0x01\",\n                    \"verifyingContract\": \"0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC\"\n                },\n                \"message\": {\n                    \"from\": {\n                        \"name\": \"Cow\",\n                        \"wallet\": \"0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826\"\n                    },\n                    \"to\": {\n                        \"name\": \"Bob\",\n                        \"wallet\": \"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB\"\n                    },\n                    \"contents\": \"Hello, Bob!\"\n                }\n            }\n        \"\"\"\n        async_skip_if_testrpc(async_w3)\n        signature = HexBytes(\n            await async_w3.eth.sign_typed_data(\n                async_unlocked_account_dual_type, json.loads(validJSONMessage)\n            )\n        )\n        assert len(signature) == 32 + 32 + 1\n\n    @pytest.mark.asyncio\n    async def test_invalid_eth_sign_typed_data(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n        async_skip_if_testrpc: Callable[[\"AsyncWeb3\"], None],\n    ) -> None:\n        async_skip_if_testrpc(async_w3)\n        invalid_typed_message = \"\"\"\n            {\n                \"types\": {\n                    \"EIP712Domain\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"version\", \"type\": \"string\"},\n                        {\"name\": \"chainId\", \"type\": \"uint256\"},\n                        {\"name\": \"verifyingContract\", \"type\": \"address\"}\n                    ],\n                    \"Person\": [\n                        {\"name\": \"name\", \"type\": \"string\"},\n                        {\"name\": \"wallet\", \"type\": \"address\"}\n                    ],\n                    \"Mail\": [\n                        {\"name\": \"from\", \"type\": \"Person\"},\n                        {\"name\": \"to\", \"type\": \"Person[2]\"},\n                        {\"name\": \"contents\", \"type\": \"string\"}\n                    ]\n                },\n                \"primaryType\": \"Mail\",\n                \"domain\": {\n                    \"name\": \"Ether Mail\",\n                    \"version\": \"1\",\n                    \"chainId\": \"0x01\",\n                    \"verifyingContract\": \"0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC\"\n                },\n                \"message\": {\n                    \"from\": {\n                        \"name\": \"Cow\",\n                        \"wallet\": \"0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826\"\n                    },\n                    \"to\": [{\n                        \"name\": \"Bob\",\n                        \"wallet\": \"0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB\"\n                    }],\n                    \"contents\": \"Hello, Bob!\"\n                }\n            }\n        \"\"\"\n        with pytest.raises(\n            ValueError,\n            match=r\".*Expected 2 items for array type Person\\[2\\], got 1 items.*\",\n        ):\n            await async_w3.eth.sign_typed_data(\n                async_unlocked_account_dual_type, json.loads(invalid_typed_message)\n            )\n\n    @pytest.mark.asyncio\n    async def test_async_eth_sign_transaction_legacy(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": await async_w3.eth.gas_price,\n            \"nonce\": Nonce(0),\n        }\n        result = await async_w3.eth.sign_transaction(txn_params)\n        signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n        assert async_unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"gasPrice\"] == txn_params[\"gasPrice\"]\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    async def test_async_eth_sign_transaction_hex_fees(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account,\n            \"to\": async_unlocked_account,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": hex(async_w3.to_wei(2, \"gwei\")),\n            \"maxPriorityFeePerGas\": hex(async_w3.to_wei(1, \"gwei\")),\n            \"nonce\": Nonce(0),\n        }\n        result = await async_w3.eth.sign_transaction(txn_params)\n        signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n        assert async_unlocked_account == signatory_account\n        assert result[\"tx\"][\"to\"] == txn_params[\"to\"]\n        assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n        assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n        assert result[\"tx\"][\"maxFeePerGas\"] == int(str(txn_params[\"maxFeePerGas\"]), 16)\n        assert result[\"tx\"][\"maxPriorityFeePerGas\"] == int(\n            str(txn_params[\"maxPriorityFeePerGas\"]), 16\n        )\n        assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    @pytest.mark.xfail(\n        reason=\"async name_to_address_middleware has not been implemented yet\"\n    )\n    async def test_async_eth_sign_transaction_ens_names(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account: ChecksumAddress\n    ) -> None:\n        with ens_addresses(async_w3, {\"unlocked-account.eth\": async_unlocked_account}):\n            txn_params: TxParams = {\n                \"from\": \"unlocked-account.eth\",\n                \"to\": \"unlocked-account.eth\",\n                \"value\": Wei(1),\n                \"gas\": 21000,\n                \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n                \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n                \"nonce\": Nonce(0),\n            }\n            result = await async_w3.eth.sign_transaction(txn_params)\n            signatory_account = async_w3.eth.account.recover_transaction(result[\"raw\"])\n            assert async_unlocked_account == signatory_account\n            assert result[\"tx\"][\"to\"] == async_unlocked_account\n            assert result[\"tx\"][\"value\"] == txn_params[\"value\"]\n            assert result[\"tx\"][\"gas\"] == txn_params[\"gas\"]\n            assert result[\"tx\"][\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n            assert (\n                result[\"tx\"][\"maxPriorityFeePerGas\"]\n                == txn_params[\"maxPriorityFeePerGas\"]\n            )\n            assert result[\"tx\"][\"nonce\"] == txn_params[\"nonce\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": async_w3.to_wei(3, \"gwei\"),\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"maxFeePerGas\"] == txn_params[\"maxFeePerGas\"]\n        assert txn[\"maxPriorityFeePerGas\"] == txn_params[\"maxPriorityFeePerGas\"]\n        assert txn[\"gasPrice\"] == txn_params[\"maxFeePerGas\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_default_fees(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert is_integer(txn[\"maxPriorityFeePerGas\"])\n        assert is_integer(txn[\"maxFeePerGas\"])\n        assert txn[\"gasPrice\"] == txn[\"maxFeePerGas\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_hex_fees(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": hex(250 * 10**9),\n            \"maxPriorityFeePerGas\": hex(2 * 10**9),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n        assert txn[\"maxFeePerGas\"] == 250 * 10**9\n        assert txn[\"maxPriorityFeePerGas\"] == 2 * 10**9\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_no_gas(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"maxFeePerGas\": Wei(250 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 121000  # 21000 + buffer\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_with_gas_price(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"gasPrice\": Wei(1),\n            \"maxFeePerGas\": Wei(250 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        with pytest.raises(TransactionTypeMismatch):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_no_priority_fee(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(250 * 10**9),\n        }\n        with pytest.raises(\n            InvalidTransaction, match=\"maxPriorityFeePerGas must be defined\"\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_no_max_fee(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        maxPriorityFeePerGas = async_w3.to_wei(2, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": maxPriorityFeePerGas,\n        }\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert is_same_address(txn[\"from\"], cast(ChecksumAddress, txn_params[\"from\"]))\n        assert is_same_address(txn[\"to\"], cast(ChecksumAddress, txn_params[\"to\"]))\n        assert txn[\"value\"] == 1\n        assert txn[\"gas\"] == 21000\n\n        block = await async_w3.eth.get_block(\"latest\")\n        assert txn[\"maxFeePerGas\"] == maxPriorityFeePerGas + 2 * block[\"baseFeePerGas\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_send_transaction_max_fee_less_than_tip(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1 * 10**9),\n            \"maxPriorityFeePerGas\": Wei(2 * 10**9),\n        }\n        with pytest.raises(\n            InvalidTransaction, match=\"maxFeePerGas must be >= maxPriorityFeePerGas\"\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_validation_middleware_chain_id_mismatch(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        wrong_chain_id = 1234567890\n        actual_chain_id = await async_w3.eth.chain_id\n\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": async_w3.to_wei(2, \"gwei\"),\n            \"maxPriorityFeePerGas\": async_w3.to_wei(1, \"gwei\"),\n            \"chainId\": wrong_chain_id,\n        }\n        with pytest.raises(\n            Web3ValidationError,\n            match=f\"The transaction declared chain ID {wrong_chain_id}, \"\n            f\"but the connected node is on {actual_chain_id}\",\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_ExtraDataToPOAMiddleware(\n        self, async_w3: \"AsyncWeb3\", request_mocker: Type[RequestMocker]\n    ) -> None:\n        async_w3.middleware_onion.inject(ExtraDataToPOAMiddleware, \"poa\", layer=0)\n        extra_data = f\"0x{'ff' * 33}\"\n\n        async with request_mocker(\n            async_w3,\n            mock_results={\"eth_getBlockByNumber\": {\"extraData\": extra_data}},\n        ):\n            block = await async_w3.eth.get_block(\"latest\")\n\n        assert \"extraData\" not in block\n        assert block[\"proofOfAuthorityData\"] == to_bytes(hexstr=extra_data)\n\n        # clean up\n        async_w3.middleware_onion.remove(\"poa\")\n\n    @pytest.mark.asyncio\n    async def test_eth_send_raw_transaction(self, async_w3: \"AsyncWeb3\") -> None:\n        # private key 0x3c2ab4e8f17a7dea191b8c991522660126d681039509dc3bb31af7c9bdb63518\n        # This is an unfunded account, but the transaction has a 0 gas price, so is\n        # valid. It never needs to be mined, we just want the transaction hash back\n        # to confirm.\n        # tx = {'to': '0x0000000000000000000000000000000000000000', 'value': 0,  'nonce': 1, 'gas': 21000, 'gasPrice': 0, 'chainId': 131277322940537}  # noqa: E501\n        # NOTE: nonce=1 to make txn unique from the non-async version of this test\n        raw_txn = HexBytes(\n            \"0xf8650180825208940000000000000000000000000000000000000000808086eecac466e115a0ffdd42d7dee4ac85427468bc616812e49432e285e4e8f5cd9381163ac3b28108a04ec6b0d89ecbd5e89b0399f336ad50f283fafd70e86593250bf5a2adfb93d17e\"  # noqa: E501\n        )\n        expected_hash = HexStr(\n            \"0x52b0ff9cb472f25872fa8ec6a62fa59454fc2ae7901cfcc6cc89d096f49b8fc1\"\n        )\n        txn_hash = await async_w3.eth.send_raw_transaction(raw_txn)\n        assert txn_hash == async_w3.to_bytes(hexstr=expected_hash)\n\n    @pytest.mark.asyncio\n    async def test_GasPriceStrategyMiddleware(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        two_gwei_in_wei = async_w3.to_wei(2, \"gwei\")\n\n        def gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return two_gwei_in_wei\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert txn[\"gasPrice\"] == two_gwei_in_wei\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    async def test_gas_price_strategy_middleware_hex_value(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        two_gwei_in_wei = async_w3.to_wei(2, \"gwei\")\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> str:\n            return hex(two_gwei_in_wei)\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)  # type: ignore\n\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        assert txn[\"gasPrice\"] == two_gwei_in_wei\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\n        \"max_fee\", (1000000000, None), ids=[\"with_max_fee\", \"without_max_fee\"]\n    )\n    async def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n        max_fee: Wei,\n    ) -> None:\n        max_priority_fee = async_w3.to_wei(1, \"gwei\")\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxPriorityFeePerGas\": max_priority_fee,\n        }\n        if max_fee is not None:\n            txn_params = assoc(txn_params, \"maxFeePerGas\", max_fee)\n\n        def gas_price_strategy(w3: \"Web3\", txn: TxParams) -> Wei:\n            return async_w3.to_wei(2, \"gwei\")\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        txn_hash = await async_w3.eth.send_transaction(txn_params)\n        txn = await async_w3.eth.get_transaction(txn_hash)\n\n        latest_block = await async_w3.eth.get_block(\"latest\")\n        assert (\n            txn[\"maxFeePerGas\"] == max_fee\n            if max_fee is not None\n            else 2 * latest_block[\"baseFeePerGas\"] + max_priority_fee\n        )\n        assert txn[\"maxPriorityFeePerGas\"] == max_priority_fee\n        assert txn[\"gasPrice\"] == txn[\"maxFeePerGas\"]\n\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    async def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn_no_tip(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n    ) -> None:\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"to\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n            \"maxFeePerGas\": Wei(1000000000),\n        }\n\n        def gas_price_strategy(_w3: \"Web3\", _txn: TxParams) -> Wei:\n            return async_w3.to_wei(2, \"gwei\")\n\n        async_w3.eth.set_gas_price_strategy(gas_price_strategy)\n\n        with pytest.raises(\n            InvalidTransaction, match=\"maxPriorityFeePerGas must be defined\"\n        ):\n            await async_w3.eth.send_transaction(txn_params)\n\n        async_w3.eth.set_gas_price_strategy(None)  # reset strategy\n\n    @pytest.mark.asyncio\n    async def test_eth_estimate_gas(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        gas_estimate = await async_w3.eth.estimate_gas(\n            {\n                \"from\": async_unlocked_account_dual_type,\n                \"to\": async_unlocked_account_dual_type,\n                \"value\": Wei(1),\n            }\n        )\n        assert is_integer(gas_estimate)\n        assert gas_estimate > 0\n\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\n        \"params\",\n        (\n            {\n                \"nonce\": 1,  # int\n                \"balance\": 1,  # int\n                \"code\": HexStr(\"0x\"),  # HexStr\n                # with state\n                \"state\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n            {\n                \"nonce\": HexStr(\"0x1\"),  # HexStr\n                \"balance\": HexStr(\"0x1\"),  # HexStr\n                \"code\": b\"\\x00\",  # bytes\n                # with stateDiff\n                \"stateDiff\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n        ),\n    )\n    async def test_eth_estimate_gas_with_override_param_type_check(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_math_contract: \"Contract\",\n        params: StateOverrideParams,\n    ) -> None:\n        txn_params: TxParams = {\"from\": await async_w3.eth.coinbase}\n\n        # assert does not raise\n        await async_w3.eth.estimate_gas(\n            txn_params, None, {async_math_contract.address: params}\n        )\n\n    @pytest.mark.asyncio\n    async def test_eth_fee_history(self, async_w3: \"AsyncWeb3\") -> None:\n        fee_history = await async_w3.eth.fee_history(1, \"latest\", [50])\n        assert is_list_like(fee_history[\"baseFeePerGas\"])\n        assert is_list_like(fee_history[\"gasUsedRatio\"])\n        assert is_integer(fee_history[\"oldestBlock\"])\n        assert fee_history[\"oldestBlock\"] >= 0\n        assert is_list_like(fee_history[\"reward\"])\n        if len(fee_history[\"reward\"]) > 0:\n            assert is_list_like(fee_history[\"reward\"][0])\n\n    @pytest.mark.asyncio\n    async def test_eth_fee_history_with_integer(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        fee_history = await async_w3.eth.fee_history(\n            1, async_empty_block[\"number\"], [50]\n        )\n        assert is_list_like(fee_history[\"baseFeePerGas\"])\n        assert is_list_like(fee_history[\"gasUsedRatio\"])\n        assert is_integer(fee_history[\"oldestBlock\"])\n        assert fee_history[\"oldestBlock\"] >= 0\n        assert is_list_like(fee_history[\"reward\"])\n        if len(fee_history[\"reward\"]) > 0:\n            assert is_list_like(fee_history[\"reward\"][0])\n\n    @pytest.mark.asyncio\n    async def test_eth_fee_history_no_reward_percentiles(\n        self, async_w3: \"AsyncWeb3\"\n    ) -> None:\n        fee_history = await async_w3.eth.fee_history(1, \"latest\")\n        assert is_list_like(fee_history[\"baseFeePerGas\"])\n        assert is_list_like(fee_history[\"gasUsedRatio\"])\n        assert is_integer(fee_history[\"oldestBlock\"])\n        assert fee_history[\"oldestBlock\"] >= 0\n\n    @pytest.mark.asyncio\n    async def test_eth_max_priority_fee(self, async_w3: \"AsyncWeb3\") -> None:\n        max_priority_fee = await async_w3.eth.max_priority_fee\n        assert is_integer(max_priority_fee)\n\n    @pytest.mark.asyncio\n    async def test_eth_max_priority_fee_with_fee_history_calculation(\n        self, async_w3: \"AsyncWeb3\", request_mocker: Type[RequestMocker]\n    ) -> None:\n        async with request_mocker(\n            async_w3,\n            mock_errors={RPCEndpoint(\"eth_maxPriorityFeePerGas\"): {}},\n            mock_results={RPCEndpoint(\"eth_feeHistory\"): {\"reward\": [[0]]}},\n        ):\n            with pytest.warns(\n                UserWarning,\n                match=(\n                    \"There was an issue with the method eth_maxPriorityFeePerGas. \"\n                    \"Calculating using eth_feeHistory.\"\n                ),\n            ):\n                priority_fee = await async_w3.eth.max_priority_fee\n                assert is_integer(priority_fee)\n                assert priority_fee == PRIORITY_FEE_MIN\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByHash(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(async_empty_block[\"hash\"])\n        assert block[\"hash\"] == async_empty_block[\"hash\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByHash_not_found(self, async_w3: \"AsyncWeb3\") -> None:\n        with pytest.raises(BlockNotFound):\n            await async_w3.eth.get_block(UNKNOWN_HASH)\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByHash_pending(self, async_w3: \"AsyncWeb3\") -> None:\n        block = await async_w3.eth.get_block(\"pending\")\n        assert block[\"hash\"] is None\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_with_integer(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(async_empty_block[\"number\"])\n        assert block[\"number\"] == async_empty_block[\"number\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_latest(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        current_block_number = await async_w3.eth.block_number\n        block = await async_w3.eth.get_block(\"latest\")\n        assert block[\"number\"] == current_block_number\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_not_found(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        with pytest.raises(BlockNotFound):\n            await async_w3.eth.get_block(BlockNumber(12345))\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_pending(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        current_block_number = await async_w3.eth.block_number\n        block = await async_w3.eth.get_block(\"pending\")\n        assert block[\"number\"] == current_block_number + 1\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_earliest(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        genesis_block = await async_w3.eth.get_block(BlockNumber(0))\n        block = await async_w3.eth.get_block(\"earliest\")\n        assert block[\"number\"] == 0\n        assert block[\"hash\"] == genesis_block[\"hash\"]\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_safe(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(\"safe\")\n        assert block is not None\n        assert isinstance(block[\"number\"], int)\n\n    @pytest.mark.asyncio\n    async def test_eth_getBlockByNumber_finalized(\n        self, async_w3: \"AsyncWeb3\", async_empty_block: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(\"finalized\")\n        assert block is not None\n        assert isinstance(block[\"number\"], int)\n\n    @pytest.mark.asyncio\n    async def test_eth_get_block_by_number_full_transactions(\n        self, async_w3: \"AsyncWeb3\", async_block_with_txn: BlockData\n    ) -> None:\n        block = await async_w3.eth.get_block(async_block_with_txn[\"number\"], True)\n        transaction = cast(TxData, block[\"transactions\"][0])\n        assert transaction[\"hash\"] == async_block_with_txn[\"transactions\"][0]\n\n    @pytest.mark.asyncio\n    async def test_eth_get_raw_transaction(\n        self, async_w3: \"AsyncWeb3\", mined_txn_hash: HexStr\n    ) -> None:\n        raw_transaction = await async_w3.eth.get_raw_transaction(mined_txn_hash)\n        assert is_bytes(raw_transaction)\n\n    @pytest.mark.asyncio\n    async def test_eth_get_raw_transaction_raises_error(\n        self, async_w3: \"AsyncWeb3\"\n    ) -> None:\n        with pytest.raises(\n            TransactionNotFound, match=f\"Transaction with hash: '{UNKNOWN_HASH}'\"\n        ):\n            await async_w3.eth.get_raw_transaction(UNKNOWN_HASH)\n\n    @pytest.mark.asyncio\n    async def test_eth_get_raw_transaction_by_block(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_block_with_txn: BlockData,\n        async_unlocked_account_dual_type: ChecksumAddress,\n    ) -> None:\n        # eth_getRawTransactionByBlockNumberAndIndex: block identifier\n        # send a txn to make sure pending block has at least one txn\n        await async_w3.eth.send_transaction(\n            {\n                \"from\": async_unlocked_account_dual_type,\n                \"to\": async_unlocked_account_dual_type,\n                \"value\": Wei(1),\n            }\n        )\n        pending_block = await async_w3.eth.get_block(\"pending\")\n        last_pending_txn_index = len(pending_block[\"transactions\"]) - 1\n        raw_txn = await async_w3.eth.get_raw_transaction_by_block(\n            \"pending\", last_pending_txn_index\n        )\n        assert is_bytes(raw_txn)\n\n        # eth_getRawTransactionByBlockNumberAndIndex: block number\n        async_block_with_txn_number = async_block_with_txn[\"number\"]\n        raw_transaction = await async_w3.eth.get_raw_transaction_by_block(\n            async_block_with_txn_number, 0\n        )\n        assert is_bytes(raw_transaction)\n\n        # eth_getRawTransactionByBlockHashAndIndex: block hash\n        async_block_with_txn_hash = async_block_with_txn[\"hash\"]\n        raw_transaction = await async_w3.eth.get_raw_transaction_by_block(\n            async_block_with_txn_hash, 0\n        )\n        assert is_bytes(raw_transaction)\n\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\"unknown_block_num_or_hash\", (1234567899999, UNKNOWN_HASH))\n    async def test_eth_get_raw_transaction_by_block_raises_error(\n        self, async_w3: \"AsyncWeb3\", unknown_block_num_or_hash: Union[int, HexBytes]\n    ) -> None:\n        with pytest.raises(\n            TransactionNotFound,\n            match=(\n                f\"Transaction index: 0 on block id: \"\n                f\"{to_hex_if_integer(unknown_block_num_or_hash)!r} \"\n                f\"not found.\"\n            ),\n        ):\n            await async_w3.eth.get_raw_transaction_by_block(\n                unknown_block_num_or_hash, 0\n            )\n\n    @pytest.mark.asyncio\n    async def test_eth_get_raw_transaction_by_block_raises_error_block_identifier(\n        self, async_w3: \"AsyncWeb3\"\n    ) -> None:\n        unknown_identifier = \"unknown\"\n        with pytest.raises(\n            ValueError,\n            match=(\n                \"Value did not match any of the recognized block identifiers: \"\n                f\"{unknown_identifier}\"\n            ),\n        ):\n            await async_w3.eth.get_raw_transaction_by_block(\n                unknown_identifier, 0  # type: ignore\n            )\n\n    @pytest.mark.asyncio\n    async def test_eth_get_balance(self, async_w3: \"AsyncWeb3\") -> None:\n        coinbase = await async_w3.eth.coinbase\n\n        with pytest.raises(InvalidAddress):\n            await async_w3.eth.get_balance(\n                ChecksumAddress(HexAddress(HexStr(coinbase.lower())))\n            )\n\n        balance = await async_w3.eth.get_balance(coinbase)\n\n        assert is_integer(balance)\n        assert balance >= 0\n\n    @pytest.mark.asyncio\n    async def test_eth_get_code(\n        self, async_w3: \"AsyncWeb3\", async_math_contract_address: ChecksumAddress\n    ) -> None:\n        code = await async_w3.eth.get_code(async_math_contract_address)\n        assert isinstance(code, HexBytes)\n        assert len(code) > 0\n\n    @pytest.mark.asyncio\n    async def test_eth_get_code_invalid_address(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_math_contract: \"Contract\",\n    ) -> None:\n        with pytest.raises(InvalidAddress):\n            await async_w3.eth.get_code(\n                ChecksumAddress(HexAddress(HexStr(async_math_contract.address.lower())))\n            )\n\n    @pytest.mark.asyncio\n    async def test_eth_get_code_with_block_identifier(\n        self, async_w3: \"AsyncWeb3\", async_emitter_contract: \"Contract\"\n    ) -> None:\n        block_id = await async_w3.eth.block_number\n        code = await async_w3.eth.get_code(async_emitter_contract.address, block_id)\n        assert isinstance(code, HexBytes)\n        assert len(code) > 0\n\n    @pytest.mark.asyncio\n    async def test_eth_create_access_list(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_unlocked_account_dual_type: ChecksumAddress,\n        async_math_contract: \"Contract\",\n    ) -> None:\n        # Initialize transaction for gas estimation\n        txn_params: TxParams = {\n            \"from\": async_unlocked_account_dual_type,\n            \"value\": Wei(1),\n            \"gas\": 21000,\n        }\n        txn = async_math_contract._prepare_transaction(\n            fn_name=\"incrementCounter\",\n            fn_args=[1],\n            transaction=txn_params,\n        )\n\n        # create access list using data from transaction\n        response = await async_w3.eth.create_access_list(\n            {\n                \"from\": async_unlocked_account_dual_type,\n                \"to\": async_math_contract.address,\n                \"data\": txn[\"data\"],\n            }\n        )\n\n        assert is_dict(response)\n        access_list = response[\"accessList\"]\n        assert len(access_list) > 0\n        assert access_list[0][\"address\"] is not None\n        assert is_checksum_address(access_list[0][\"address\"])\n        assert len(access_list[0][\"storageKeys\"][0]) == 32\n        assert int(response[\"gasUsed\"]) >= 0\n\n    @pytest.mark.asyncio\n    async def test_eth_get_transaction_count(\n        self, async_w3: \"AsyncWeb3\", async_unlocked_account_dual_type: ChecksumAddress\n    ) -> None:\n        transaction_count = await async_w3.eth.get_transaction_count(\n            async_unlocked_account_dual_type\n        )\n        assert is_integer(transaction_count)\n        assert transaction_count >= 0\n\n    @pytest.mark.asyncio\n    async def test_eth_call(\n        self, async_w3: \"AsyncWeb3\", async_math_contract: \"Contract\"\n    ) -> None:\n        coinbase = await async_w3.eth.coinbase\n        txn_params = async_math_contract._prepare_transaction(\n            fn_name=\"add\",\n            fn_args=(7, 11),\n            transaction={\"from\": coinbase, \"to\": async_math_contract.address},\n        )\n        call_result = await async_w3.eth.call(txn_params)\n        assert is_string(call_result)\n        (result,) = async_w3.codec.decode([\"uint256\"], call_result)\n        assert result == 18\n\n    @pytest.mark.asyncio\n    async def test_eth_call_with_override_code(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_revert_contract: \"Contract\",\n    ) -> None:\n        coinbase = await async_w3.eth.coinbase\n        txn_params = async_revert_contract._prepare_transaction(\n            fn_name=\"normalFunction\",\n            transaction={\"from\": coinbase, \"to\": async_revert_contract.address},\n        )\n        call_result = await async_w3.eth.call(txn_params)\n        (result,) = async_w3.codec.decode([\"bool\"], call_result)\n        assert result is True\n\n        # override runtime bytecode: `normalFunction` returns `false`\n        override_code = HexStr(\n            \"0x6080604052348015600f57600080fd5b5060043610603c5760003560e01c8063185c38a4146041578063c06a97cb146049578063d67e4b84146051575b600080fd5b60476071565b005b604f60df565b005b605760e4565b604051808215151515815260200191505060405180910390f35b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252601b8152602001807f46756e6374696f6e20686173206265656e2072657665727465642e000000000081525060200191505060405180910390fd5b600080fd5b60008090509056fea2646970667358221220bb71e9e9a2e271cd0fbe833524a3ea67df95f25ea13aef5b0a761fa52b538f1064736f6c63430006010033\"  # noqa: E501\n        )\n        call_result = await async_w3.eth.call(\n            txn_params,\n            \"latest\",\n            {async_revert_contract.address: {\"code\": override_code}},\n        )\n        (result,) = async_w3.codec.decode([\"bool\"], call_result)\n        assert result is False\n\n        # test bytes\n\n        bytes_call_result = await async_w3.eth.call(\n            txn_params,\n            \"latest\",\n            {async_revert_contract.address: {\"code\": to_bytes(hexstr=override_code)}},\n        )\n        (bytes_result,) = async_w3.codec.decode([\"bool\"], bytes_call_result)\n        assert bytes_result is False\n\n    @pytest.mark.asyncio\n    @pytest.mark.parametrize(\n        \"params\",\n        (\n            {\n                \"nonce\": 1,  # int\n                \"balance\": 1,  # int\n                \"code\": HexStr(\"0x\"),  # HexStr\n                # with state\n                \"state\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n            {\n                \"nonce\": HexStr(\"0x1\"),  # HexStr\n                \"balance\": HexStr(\"0x1\"),  # HexStr\n                \"code\": b\"\\x00\",  # bytes\n                # with stateDiff\n                \"stateDiff\": {HexStr(f\"0x{'00' * 32}\"): HexStr(f\"0x{'00' * 32}\")},\n            },\n        ),\n    )\n    async def test_eth_call_with_override_param_type_check(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_math_contract: \"Contract\",\n        params: StateOverrideParams,\n    ) -> None:\n        coinbase = await async_w3.eth.coinbase\n        txn_params: TxParams = {\"from\": coinbase}\n\n        # assert does not raise\n        await async_w3.eth.call(\n            txn_params, \"latest\", {async_math_contract.address: params}\n        )\n\n    @pytest.mark.asyncio\n    async def test_eth_call_with_0_result(\n        self, async_w3: \"AsyncWeb3\", async_math_contract: \"Contract\"\n    ) -> None:\n        coinbase = await async_w3.eth.coinbase\n        txn_params = async_math_contract._prepare_transaction(\n            fn_name=\"add\",\n            fn_args=(0, 0),\n            transaction={\"from\": coinbase, \"to\": async_math_contract.address},\n        )\n        call_result = await async_w3.eth.call(txn_params)\n        assert is_string(call_result)\n        (result,) = async_w3.codec.decode([\"uint256\"], call_result)\n        assert result == 0\n\n    @pytest.mark.asyncio\n    async def test_eth_call_revert_with_msg(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_revert_contract: \"Contract\",\n        async_unlocked_account: ChecksumAddress,\n    ) -> None:\n        txn_params = async_revert_contract._prepare_transaction(\n            fn_name=\"revertWithMessage\",\n            transaction={\n                \"from\": async_unlocked_account,\n                \"to\": async_revert_contract.address,\n            },\n        )\n        with pytest.raises(\n            ContractLogicError, match=\"execution reverted: Function has been reverted\"\n        ):\n            await async_w3.eth.call(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_eth_call_revert_without_msg(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_revert_contract: \"Contract\",\n        async_unlocked_account: ChecksumAddress,\n    ) -> None:\n        with pytest.raises(ContractLogicError, match=\"execution reverted\"):\n            txn_params = async_revert_contract._prepare_transaction(\n                fn_name=\"revertWithoutMessage\",\n                transaction={\n                    \"from\": async_unlocked_account,\n                    \"to\": async_revert_contract.address,\n                },\n            )\n            await async_w3.eth.call(txn_params)\n\n    @pytest.mark.asyncio\n    \nasync def test_eth_call_revert_custom_error_with_msg(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_revert_contract: \"Contract\",\n        async_unlocked_account: ChecksumAddress,\n    ) -> None:\n        data = async_revert_contract.encodeABI(\n            fn_name=\"UnauthorizedWithMessage\", args=[\"You are not authorized\"]\n        )\n        txn_params = async_revert_contract._prepare_transaction(\n            fn_name=\"customErrorWithMessage\",\n            transaction={\n                \"from\": async_unlocked_account,\n                \"to\": async_revert_contract.address,\n            },\n        )\n        with pytest.raises(ContractCustomError, match=data):\n            await async_w3.eth.call(txn_params)\n\n    @pytest.mark.asyncio\n    async def test_eth_call_revert_custom_error_without_msg(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_revert_contract: \"Contract\",\n        async_unlocked_account: ChecksumAddress,\n    ) -> None:\n        data = async_revert_contract.encodeABI(fn_name=\"Unauthorized\")\n        txn_params = async_revert_contract._prepare_transaction(\n            fn_name=\"customErrorWithoutMessage\",\n            transaction={\n                \"from\": async_unlocked_account,\n                \"to\": async_revert_contract.address,\n            },\n        )\n        with pytest.raises(ContractCustomError, match=data):\n            await async_w3.eth.call(txn_params)\n\n    @pytest.mark.parametrize(\n        \"panic_error,params\",\n        (\n            (\"01\", []),\n            (\"11\", []),\n            (\"12\", [0]),\n            (\"21\", [-1]),\n            (\"22\", []),\n            (\"31\", []),\n            (\"32\", []),\n            (\"41\", []),\n            (\"51\", []),\n        ),\n    )\n    @pytest.mark.asyncio\n    async def test_contract_panic_errors(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_panic_errors_contract: \"Contract\",\n        panic_error: str,\n        params: List[Any],\n    ) -> None:\n        method = getattr(\n            async_panic_errors_contract.functions,\n            f\"errorCode{panic_error}\",\n        )\n        error_msg = PANIC_ERROR_CODES[panic_error]\n\n        with pytest.raises(ContractPanicError, match=re.escape(error_msg)):\n            await method(*params).call()\n\n    @pytest.mark.asyncio\n    async def test_eth_call_offchain_lookup(\n        self,\n        async_w3: \"AsyncWeb3\",\n        async_offchain_lookup_contract: \"Contract\",\n        async_unlocked_account: ChecksumAddress,\n        monkeypatch: \"MonkeyPatch\",\n    ) -> None:\n        normalized_contract_address = to_hex_if_bytes(\n            async_offchain_lookup_contract.address\n        ).lower()\n\n        async_mock_offchain_lookup_request_response(\n            monkeypatch,\n            mocked_request_url=f\"https://web3.py/gateway/{normalized_contract_address}/{OFFCHAIN_LOOKUP_TEST_DATA}.json\",  # noqa: E501\n...\n", "needle_token_start": 15358, "needle_token_end": 15568, "code_context_ntokens": 16260, "output": ["```python\n@pytest.mark.asyncio\nasync def test_eth_call_revert_custom_error_with_msg(\n    self,\n    async_w3: \"AsyncWeb3\",\n    async_revert_contract: \"Contract\",\n    async_unlocked_account: ChecksumAddress,\n) -> None:\n    data = async_revert_contract.encodeABI(\n        fn_name=\"UnauthorizedWithMessage\", args=[\"You are not authorized\"]\n    )\n    txn_params = async_revert_contract._prepare_transaction(\n        fn_name=\"customErrorWithMessage\",\n        transaction={\n            \"from\": async_unlocked_account,\n            \"to\": async_revert_contract.address,\n        },\n    )\n    with pytest.raises(ContractCustomError, match=data):\n        await async_w3.eth.call(txn_params)\n```"]}
{"repo": "Ciphey/Ciphey", "name": "chars_to_byte", "language": "python", "path": "ciphey/basemods/Decoders/base69.py", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: Converts a pair of characters from a specific encoding into a single byte value.\n2. **Input**: A string of two characters.\n3. **Output**: An integer representing the byte value derived from the input characters.\n4. **Procedure**: The function calculates the byte value by finding the positions of each character in a predefined character set. The position of the second character is multiplied by the length of the character set (69 in this case) and added to the position of the first character to produce the final byte value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/basemods/Decoders/base62.py\nfrom typing import Dict, Optional\n\nimport base62\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base62(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base62 decoding\n        \"\"\"\n        try:\n...\n# Path: ciphey/basemods/Decoders/base69.py\n# Translated to Python and adapted for Ciphey from the JS original at https://github.com/pshihn/base69\n\n\nimport re\nfrom math import ceil\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, WordList, registry\n\n\n@registry.register\nclass Base69(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base69 decoding\n        \"\"\"\n        # Remove whitespace\n        try:\n            ctext = re.sub(r\"\\s+\", \"\", ctext, flags=re.UNICODE)\n            extra_bytes = 0\n            clen = len(ctext)\n\n            if ctext[:-1] == \"=\":\n                extra_bytes = int(ctext[clen - 2])\n\n            CHUNK_COUNT = ceil(clen / 16)\n            result = [0 for _ in range(CHUNK_COUNT * 7 - extra_bytes)]\n\n            for i in range(CHUNK_COUNT):\n                chunk_string = ctext[i * 16 : (i + 1) * 16]\n                if extra_bytes and (i == CHUNK_COUNT - 1):\n                    insert = self.decode_chunk(chunk_string)\n                    for n, elem in enumerate(insert[0 : 7 - extra_bytes]):\n                        result[n + i * 7] = elem\n                else:\n                    insert = self.decode_chunk(chunk_string)\n                    for n, elem in enumerate(insert):\n                        result[n + i * 7] = elem % 256\n            return bytearray(result).decode().strip(\"\\x00\")\n        except Exception:\n            return None\n\n    def decode_chunk(self, s: str):\n        padded_bytes = s.endswith(\"=\")\n\n        decoded = [0 for _ in range(8)]\n        for i in range(8):\n            decoded[i] = (\n                0\n                if i == 7 and padded_bytes\n                else self.chars_to_byte(s[i * 2 : i * 2 + 2])\n            )\n\n        result = [0 for _ in range(7)]\n        for i in range(7):\n            t1 = decoded[i] << (i + 1)\n            t2 = decoded[i + 1] >> (7 - i - 1)\n            result[i] = t1 | t2\n        return result\n\n    \ndef chars_to_byte(self, s: str):\n        return (69 * self.CHARS.index(s[1])) + (self.CHARS.index(s[0]))\n\n    @staticmethod\n    def priority() -> float:\n        # If this becomes lower or equal to the reverse, it breaks.\n        # So I'll set it to 0.2 for now since it is very fast anyways.\n        return 0.2\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.CHARS = config.get_resource(self._params()[\"dict\"], WordList)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The charset used for the decoder.\",\n                req=False,\n                default=\"cipheydists::list::base69\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base69\"\n\n# Path: ciphey/basemods/Decoders/base91.py\nfrom typing import Dict, Optional\n\nimport base91\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base91(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base91 decoding\n        \"\"\"\n        try:\n            return base91.decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base91\"\n\n# Path: ciphey/basemods/Decoders/bases.py\nimport base64\nimport types\nfrom typing import Any, Callable, Optional\n\nimport logging\nfrom rich.logging import RichHandler\nimport re\n\nfrom ciphey.common import id_lambda\nfrom ciphey.iface import Decoder, registry\n\n\ndef _dispatch(self: Any, ctext: str, func: Callable[[str], bytes]) -> Optional[bytes]:\n    logging.debug(f\"Attempting {self.getTarget()}\")\n\n    try:\n        # remove all whitespace\n        ctext = re.sub(r\"\\s+\", \"\", ctext, re.UNICODE)\n        result = func(ctext)\n        logging.info(f\"{self.getTarget()} successful, returning {result}\")\n        return result\n    except ValueError:\n        logging.debug(f\"Failed to decode {self.getTarget()}\")\n        return None\n\n\n_bases = {\n    \"base16\": (base64.b16decode, 0.4),\n    \"base32\": (base64.b32decode, 0.01),\n    \"base64\": (base64.b64decode, 0.4),\n    \"base85\": (base64.b85decode, 0.01),\n    \"ascii85\": (base64.a85decode, 0.1),\n}\n\n\ndef gen_class(name, decoder, priority, ns):\n    ns[\"_get_func\"] = id_lambda(decoder)\n    ns[\"decode\"] = lambda self, ctext: _dispatch(self, ctext, self._get_func())\n    ns[\"getParams\"] = id_lambda(None)\n    ns[\"getTarget\"] = id_lambda(name)\n    ns[\"priority\"] = id_lambda(priority)\n    ns[\"__init__\"] = lambda self, config: super(type(self), self).__init__(config)\n\n\nfor name, (decoder, priority) in _bases.items():\n    t = types.new_class(\n        name,\n        (Decoder[str],),\n        exec_body=lambda x: gen_class(name, decoder, priority, x),\n    )\n\n    registry.register(t)\n\n# Path: ciphey/basemods/Decoders/baudot.py\nimport re\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Baudot(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        result = \"\"\n        switch_to_digit_map = 0\n        if re.search(\"^[01]{5}$\", ctext.split()[0]):\n            for i in ctext.split():\n                if i == \"11011\":\n                    switch_to_digit_map = 1\n                if i == \"11111\":\n                    switch_to_digit_map = 0\n                if switch_to_digit_map == 1:\n                    result += self.BAUDOT_DICT[\"+\" + i]\n                if switch_to_digit_map == 0:\n                    result += self.BAUDOT_DICT[i]\n            return result\n        else:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BAUDOT_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The baudot alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::baudot\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"baudot\"\n\n# Path: ciphey/basemods/Decoders/binary.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Binary(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        try:\n            ctext = re.sub(r\"[^\\S \\n]\", \" \", ctext, flags=re.UNICODE)\n            ctext = ctext.replace(\"\\n\", \" \")\n\n            existing_split = self.try_split(ctext.split(\" \"))\n            if existing_split is not None:\n                return existing_split\n\n            # Now we try our own grouping\n\n            # Remove final bit of whitespace\n            ctext = ctext.replace(\" \", \"\")\n            # Split into bytes, and test\n            return self.try_split([ctext[i : i + 8] for i in range(0, len(ctext), 8)])\n        # Catch bad octal chars\n        except ValueError:\n            return None\n\n    def try_split(self, split_text: List[str]):\n        ret = []\n\n        for i in split_text:\n            if len(i) == 0:\n                continue\n            val = int(i, 2)\n            if val > 255 or val < 0:\n                return None\n            ret.append(val)\n\n        if len(ret) != 0:\n            ret = bytes(ret)\n            logging.info(f\"binary successful, returning {ret.__repr__()}\")\n            return ret\n\n    @staticmethod\n    def priority() -> float:\n        return 0.3\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"binary\"\n\n# Path: ciphey/basemods/Decoders/braille.py\nimport re\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\nimport logging\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Braille(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Braille decoding\n        \"\"\"\n        logging.debug(\"Attempting Braille\")\n        ctext_decoded = \"\"\n        braille_matches = 0\n        for symbol in self.BRAILLE_DICT_INV.values():\n            if symbol in ctext:\n                braille_matches += 1\n            else:\n                continue\n        if braille_matches == 0:\n            logging.debug(\"Failed to decode Braille due to invalid characters\")\n            return None\n\n        for pattern, value in self.BRAILLE_DICT.items():\n            ctext = re.sub(pattern, value, ctext)\n\n        wordArr = []\n        for word in ctext.split(\" \"):\n            # If two commas are in front of a word, uppercase the word and remove the comma\n            if word[:2].find(\",,\") != -1:\n                wordArr.append(word.replace(\",,\", \"\").upper())\n            else:\n                wordArr.append(word)\n\n        result = []\n        for word in wordArr:\n            # If one comma is in front of a word, capitalize the word and remove the comma\n            if word[0].find(\",\") != -1:\n                result.append(word.replace(\",\", \"\").capitalize())\n            else:\n                result.append(word)\n        ctext_decoded = \" \".join(result)\n        logging.info(f\"Braille successful, returning '{ctext_decoded}'\")\n        return ctext_decoded\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BRAILLE_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.BRAILLE_DICT_INV = {v: k for k, v in self.BRAILLE_DICT.items()}\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The Braille dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::braille\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"braille\"\n\n# Path: ciphey/basemods/Decoders/brainfuck.py\nimport re\nimport time\nfrom typing import Dict, Optional, Tuple\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, WordList, registry\n\n\n@registry.register\nclass Brainfuck(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Takes a ciphertext and treats it as a Brainfuck program,\n        interpreting it and saving the output as a string to return.\n\n        Brainfuck is a very simple, Turing-complete esoteric language.\n        Below is a simplified interpreter that attempts to check whether a\n        given ciphertext is a brainfuck program that would output a string.\n\n        A program that can be \"decoded\" like this is one that:\n            * Does not require user input (\",\" instruction)\n            * Includes at least one putchar instruction (\".\")\n            * Does not contain anything but the main 7 instructions,\n                (excluding \",\") and whitespace\n\n        Details:\n            * This implementation wraps the memory pointer for \">\" and \"<\"\n            * It is time-limited to 60 seconds, to prevent hangups\n            * The program starts with 100 memory cells, chosen arbitrarily\n        \"\"\"\n\n        logging.debug(\"Attempting brainfuck\")\n\n        result = \"\"\n        memory = [0] * 100\n        codeptr, memptr = 0, 0  # Instruction pointer and stack pointer\n        timelimit = 60  # The timeout in seconds\n\n        bracemap, isbf = self.bracemap_and_check(ctext)\n\n        # If it doesn't appear to be valid brainfuck code\n        if not isbf:\n            logging.debug(\"Failed to interpret brainfuck due to invalid characters\")\n            return None\n\n        # Get start time\n        start = time.time()\n\n        while codeptr < len(ctext):\n\n            current = time.time()\n\n            # Return none if we've been running for over a minute\n            if current - start > timelimit:\n                logging.debug(\"Failed to interpret brainfuck due to timing out\")\n                return None\n\n            cmd = ctext[codeptr]\n\n            if cmd == \"+\":\n                if memory[memptr] < 255:\n                    memory[memptr] = memory[memptr] + 1\n                else:\n                    memory[memptr] = 0\n\n            elif cmd == \"-\":\n                if memory[memptr] > 0:\n                    memory[memptr] = memory[memptr] - 1\n                else:\n                    memory[memptr] = 255\n\n            elif cmd == \">\":\n                if memptr == len(memory) - 1:\n                    memory.append(0)\n                memptr += 1\n\n            elif cmd == \"<\":\n                if memptr == 0:\n                    memptr = len(memory) - 1\n                else:\n                    memptr -= 1\n\n            # If we're at the beginning of the loop and the memory is 0, exit the loop\n            elif cmd == \"[\" and memory[memptr] == 0:\n                codeptr = bracemap[codeptr]\n\n            # If we're at the end of the loop and the memory is >0, jmp to the beginning of the loop\n            elif cmd == \"]\" and memory[memptr]:\n                codeptr = bracemap[codeptr]\n\n            # Store the output as a string instead of printing it out\n            elif cmd == \".\":\n                result += chr(memory[memptr])\n\n            codeptr += 1\n\n        logging.info(f\"Brainfuck successful, returning '{result}'\")\n        return result\n\n    def bracemap_and_check(self, program: str) -> Tuple[Optional[Dict], bool]:\n        \"\"\"\n        Create a bracemap of brackets in the program, to compute jmps.\n        Maps open -> close brackets as well as close -> open brackets.\n\n        Also returns True if the program is valid Brainfuck code. If False, we\n        won't even try to run it.\n        \"\"\"\n\n        open_stack = []\n        bracemap = dict()\n        legal_instructions = {\"+\", \"-\", \">\", \"<\", \"[\", \"]\", \".\"}\n        legal_count = 0\n\n        # If the program actually outputs anything (contains \".\")\n        prints = False\n\n        for idx, instruction in enumerate(program):\n            # If instruction is brainfuck (without input) or whitespace, it counts\n            if instruction in legal_instructions or re.match(r\"\\s\", instruction):\n                legal_count += 1\n\n            if not prints and instruction == \".\":\n                # If there are no \".\" instructions then this program will not output anything\n                prints = True\n\n            elif instruction == \"[\":\n                open_stack.append(idx)\n\n            elif instruction == \"]\":\n                try:\n                    opbracket = open_stack.pop()\n                    bracemap[opbracket] = idx\n                    bracemap[idx] = opbracket\n                except IndexError:\n                    # Mismatched braces, not a valid program\n                    # Closing braces > opening braces\n                    return (None, False)\n\n        # 1. All characters are instructions or whitespace\n        # 2. There are no extra open braces\n        # 3. There is at least one character to be \"printed\"\n        # (result is >=1 in length)\n        is_brainfuck = legal_count == len(program) and len(open_stack) == 0 and prints\n\n        return bracemap, is_brainfuck\n\n    @staticmethod\n    def priority() -> float:\n        # Not uncommon, but not very common either. It's also slow.\n        return 0.08\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.ALPHABET = config.get_resource(self._params()[\"dict\"], WordList)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"Brainfuck alphabet (default English)\",\n                req=False,\n                default=\"cipheydists::list::englishAlphabet\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"brainfuck\"\n\n# Path: ciphey/basemods/Decoders/decimal.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Decimal(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Decimal decoding\n        \"\"\"\n        logging.debug(\"Attempting decimal\")\n        ctext_converted = []\n        ctext_split = re.split(r\"[ ,;:\\-\\n]\", ctext)\n        delimiters = set(sorted(re.sub(r\"[^ ,;:\\-\\n]\", \"\", ctext)))\n        ctext_num = re.sub(r\"[,;:\\-\\s]\", \"\", ctext)\n        ctext_decoded = \"\"\n        if ctext_num.isnumeric() is False:\n            logging.debug(\"Failed to decode decimal due to non numeric character(s)\")\n            return None\n        try:\n            for i in ctext_split:\n                val = int(i)\n                if val > 255 or val < 0:\n                    logging.debug(\n                        f\"Failed to decode decimal due to invalid number '{val}'\"\n                    )\n                    return None\n                ctext_converted.append(chr(val))\n            ctext_decoded = \"\".join(ctext_converted)\n            logging.info(\n                f\"Decimal successful, returning '{ctext_decoded}' with delimiter(s) {delimiters}\"\n            )\n            return ctext_decoded\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"decimal\"\n\n# Path: ciphey/basemods/Decoders/dna.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Dna(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs DNA decoding\n        \"\"\"\n        logging.debug(\"Attempting DNA decoder\")\n        ctext_decoded = \"\"\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext)\n        ctext = \" \".join(ctext[i : i + 3] for i in range(0, len(ctext), 3))\n        ctext_split = ctext.split(\" \")\n        dna_keys = self.DNA_DICT.keys()\n\n        for i in ctext_split:\n            if i in dna_keys:\n                ctext_decoded += self.DNA_DICT[i]\n            else:\n                return None\n        logging.info(f\"DNA successful, returning '{ctext_decoded}'\")\n        return ctext_decoded\n\n    @staticmethod\n    def priority() -> float:\n        return 0.2\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.DNA_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The DNA alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::dna\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"dna\"\n\n# Path: ciphey/basemods/Decoders/dtmf.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Dtmf(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs DTMF decoding\n        \"\"\"\n        logging.debug(\"Attempting DTMF decoder\")\n        ctext_decoded = \"\"\n        ctext = re.sub(r\"[,;:\\-\\/\\s]\", \"\", ctext)\n        ctext = \" \".join(ctext[i : i + 7] for i in range(0, len(ctext), 7))\n        ctext_split = ctext.split(\" \")\n        dtmf_keys = self.DTMF_DICT.keys()\n\n        for i in ctext_split:\n            if i in dtmf_keys:\n                ctext_decoded += self.DTMF_DICT[i]\n            else:\n                return None\n        logging.info(f\"DTMF successful, returning '{ctext_decoded}'\")\n        return ctext_decoded\n\n    @staticmethod\n    def priority() -> float:\n        return 0.2\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.DTMF_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The DTMF alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::dtmf\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"dtmf\"\n\n# Path: ciphey/basemods/Decoders/galactic.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Galactic(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Takes a string written in the 'Standard Galactic Alphabet'\n        (aka Minecraft Enchanting Table Symbols) and translates it to ASCII text.\n        \"\"\"\n        logging.debug(\"Attempting Standard Galactic Alphabet decoder\")\n\n        # To avoid complications, only move forward with the decoding if we can\n        # reasonably assume that the input string is written in the galactic alphabet\n        galactic_matches = 0\n        for symbol in self.GALACTIC_DICT.keys():\n            # These symbols are assumed to be frequent enough in regular\n            # text to be skipped when counting the matches. All others are counted.\n            if symbol in ctext and symbol not in [\"!\", \"|\"]:\n                galactic_matches += 1\n            else:\n                continue\n        if galactic_matches == 0:\n            logging.debug(\n                \"No matching galactic alphabet letters found. Skipping galactic decoder\"\n            )\n            return None\n        logging.debug(f\"{galactic_matches} galactic alphabet letters found. \")\n\n        result = \"\"\n        # Take out the problematic characters consisting of multiple symbols\n        ctext = (\n            ctext.replace(\"||\", \"|\")\n            .replace(\"/\", \"\")\n            .replace(\"\u00a1\", \"\")\n            .replace(\" \u0323 \", \"\")\n            .replace(\"\u0307\", \"x\")\n        )\n        logging.debug(f\"Modified string is {ctext}\")\n\n        for letter in ctext:\n            if letter in self.GALACTIC_DICT.keys():\n                # Match every letter of the input to its galactic counterpoint\n                result += self.GALACTIC_DICT[letter]\n            else:\n                # If the current character is not in the defined alphabet,\n                # just accept it as-is (useful for numbers, punctuation, etc.)\n                result += letter\n        # Remove the trailing space (appearing as a leading space)\n        # from the x that results from the diacritic replacement\n        result = result.replace(\"x \", \"x\")\n        logging.debug(f\"Decoded string is {result}\")\n        return result\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.01\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.GALACTIC_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The galactic alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::galactic\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"galactic\"\n\n# Path: ciphey/basemods/Decoders/gzip.py\nimport zlib\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Gzip(Decoder[bytes]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Gzip decoding\n        \"\"\"\n        try:\n            return zlib.decompress(ctext, 16 + zlib.MAX_WBITS).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"gzip\"\n\n# Path: ciphey/basemods/Decoders/hexadecimal.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Hexadecimal(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Hexadecimal decoding\n        \"\"\"\n        ctext_decoded = \"\"\n        try:\n            ctext_decoded = bytearray.fromhex(ctext).decode(\"utf-8\")\n            return ctext_decoded\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.015\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"hexadecimal\"\n\n# Path: ciphey/basemods/Decoders/leetspeak.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Leetspeak(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        for src, dst in self.translate.items():\n            ctext = ctext.replace(src, dst)\n        return ctext\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.translate = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The leetspeak dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::leet\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"leetspeak\"\n\n# Path: ciphey/basemods/Decoders/morse_code.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Morse_code(Decoder[str]):\n    # A priority list for char/word boundaries\n    BOUNDARIES = {\" \": 1, \"/\": 2, \"\\n\": 3}\n    PURGE = {ord(c): None for c in BOUNDARIES.keys()}\n    MAX_PRIORITY = 3\n    ALLOWED = {\".\", \"-\", \" \", \"/\", \"\\n\"}\n    MORSE_CODE_DICT: Dict[str, str]\n    MORSE_CODE_DICT_INV: Dict[str, str]\n\n    def decode(self, ctext: T) -> Optional[U]:\n        logging.debug(\"Attempting Morse code decoder\")\n\n        char_boundary = word_boundary = None\n\n        char_boundary = word_boundary = None\n        char_priority = word_priority = 0\n        # Custom loop allows early break\n        for i in ctext:\n            i_priority = self.BOUNDARIES.get(i)\n            if i_priority is None:\n                if i in self.ALLOWED:\n                    continue\n                logging.debug(f\"Non-morse char '{i}' found\")\n                return None\n\n            if i_priority <= char_priority or i == char_boundary or i == word_boundary:\n                continue\n            # Default to having a char boundary over a word boundary\n            if (\n                i_priority > word_priority\n                and word_boundary is None\n                and char_boundary is not None\n            ):\n                word_priority = i_priority\n                word_boundary = i\n                continue\n            char_priority = i_priority\n            char_boundary = i\n\n        logging.debug(\n            f\"Char boundary is unicode {ord(char_boundary)}, and word boundary is unicode {ord(word_boundary) if word_boundary is not None else None}\"\n        )\n\n        result = \"\"\n\n        for word in ctext.split(word_boundary) if word_boundary else [ctext]:\n            logging.debug(f\"Attempting to decode word {word}\")\n            for char in word.split(char_boundary):\n                char = char.translate(self.PURGE)\n                if len(char) == 0:\n                    continue\n                try:\n                    m = self.MORSE_CODE_DICT_INV[char]\n                except KeyError:\n                    logging.debug(f\"Invalid codeword '{char}' found\")\n                    return None\n                result = result + m\n            # after every word add a space\n            result = result + \" \"\n        if len(result) == 0:\n            logging.debug(\"Morse code failed to match\")\n            return None\n        # Remove trailing space\n        result = result[:-1]\n        logging.info(f\"Morse code successful, returning {result}\")\n        return result.strip().upper()\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.MORSE_CODE_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.MORSE_CODE_DICT_INV = {v: k for k, v in self.MORSE_CODE_DICT.items()}\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The morse code dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::morse\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"morse_code\"\n\n# Path: ciphey/basemods/Decoders/multi_tap.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Multi_tap(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        result = \"\"\n        for x in ctext.split():\n            if x == self.SPACE_DIGIT:  # Check if it's a space\n                result += \" \"\n            elif not Multi_tap.valid_code_part(x):\n                return None\n            else:\n                result += self.decode_num_to_char(x)\n\n        return result\n\n    @staticmethod\n    def valid_code_part(code: str) -> bool:\n        if not code.isdigit():\n            return False\n\n        # if not all the digits are the same\n        if not Multi_tap.is_all_dup(code):\n            return False\n\n        if int(code[0]) not in range(2, 10):\n            return False\n\n        if len(code) > 4:\n            return False\n\n        return True\n\n    @staticmethod\n    def decode_num_to_char(number: str) -> str:\n        index = Multi_tap.calculate_index(number)\n        return Multi_tap.number_index_to_char(index)\n\n    @staticmethod\n    def is_all_dup(code):\n        return len(set(code)) == 1\n\n    @staticmethod\n    def calculate_index(number: str) -> int:\n        first_number_as_int = int(number[0])\n\n        number_index = Multi_tap.get_index_from_first_digit(first_number_as_int)\n\n        # Add to index the number of the char : \"22\" -> index += 1\n        num_rest_numbers = len(number) - 1\n        number_index += num_rest_numbers\n\n        return number_index\n\n    @staticmethod\n    def number_index_to_char(index_number: int) -> str:\n        start_ascii_value = ord(\"A\")\n        return chr(start_ascii_value + index_number)\n\n    @staticmethod\n    def get_index_from_first_digit(first_digit: int) -> int:\n        number_index = 0\n        if first_digit >= 8:  # s have 4 chars\n            number_index += 1\n\n        first_digit -= 2  # start in 200\n\n        number_index += first_digit * 3  # jump 3 every time\n\n        return number_index\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.SPACE_DIGIT = \"0\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"multi_tap\"\n\n# Path: ciphey/basemods/Decoders/octal.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Octal(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Octal decoding\n        \"\"\"\n        str_converted = []\n        octal_seq = ctext.split(\" \")\n        if len(octal_seq) == 1:\n            # Concatted octal must be formed of octal triplets\n            if len(ctext) % 3 != 0:\n                return None\n            octal_seq = [ctext[i : i + 3] for i in range(0, len(ctext), 3)]\n            logging.debug(f\"Trying chunked octal {octal_seq}\")\n        try:\n            for octal_char in octal_seq:\n                if len(octal_char) > 3:\n                    logging.debug(\"Octal subseq too long\")\n                    return None\n                n = int(octal_char, 8)\n                if (\n                    n < 0\n                ):  # n cannot be greater than 255, as we checked that with the earlier length check\n                    logging.debug(f\"Non octal char {octal_char}\")\n                    return None\n                str_converted.append(n)\n\n            return bytes(str_converted)\n        # Catch bad octal chars\n        except ValueError:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.025\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"octal\"\n\n# Path: ciphey/basemods/Decoders/reverse.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Reverse(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        return ctext[::-1]\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"reverse\"\n\n# Path: ciphey/basemods/Decoders/tap_code.py\n# by https://github.com/RustyDucky and https://github.com/lukasgabriel\n\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Tap_code(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Tap code decoding\n        \"\"\"\n        try:\n            result = \"\"\n            combinations = ctext.split(\" \")\n            for fragment in combinations:\n                result += self.TABLE.get(fragment)\n            return result\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.06\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.TABLE = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The table of letters used for the tap code interpretation.\",\n                req=False,\n                default=\"cipheydists::translate::tap_code\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"tap_code\"\n\n# Path: ciphey/basemods/Decoders/unicode.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Utf8(Decoder[bytes]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs UTF-8 decoding\n        \"\"\"\n        logging.debug(\"Attempting UTF-8 decoder\")\n        result = \"\"\n        try:\n            result = ctext.decode(\"utf-8\")\n            if result != ctext:\n                logging.info(f\"UTF-8 successful, returning '{result}'\")\n                return result\n            else:\n                return None\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.9\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"utf8\"\n\n# Path: ciphey/basemods/Decoders/url.py\nfrom typing import Dict, Optional\nfrom urllib.parse import unquote_plus\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Url(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs URL decoding\n        \"\"\"\n        logging.debug(\"Attempting URL\")\n        result = \"\"\n        try:\n            result = unquote_plus(ctext, errors=\"strict\")\n            if result != ctext:\n                logging.info(f\"URL successful, returning '{result}'\")\n                return result\n            else:\n                return None\n        except Exception:\n            logging.debug(\"Failed to decode URL\")\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"url\"\n\n# Path: ciphey/basemods/Decoders/uuencode.py\nfrom binascii import a2b_uu\nfrom codecs import decode\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Uuencode(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        UUEncode (Unix to Unix Encoding) is a symmetric encryption\n        based on conversion of binary data (split into 6-bit blocks) into ASCII characters.\n\n        This function decodes the input string 'ctext' if it has been encoded using 'uuencoder'\n        It will return None otherwise\n        \"\"\"\n        logging.debug(\"Attempting UUencode\")\n        result = \"\"\n        try:\n            # UUencoded messages may begin with prefix \"begin\" and end with suffix \"end\"\n            # In that case, we use the codecs module in Python\n            ctext_strip = ctext.strip()\n            if ctext_strip.startswith(\"begin\") and ctext_strip.endswith(\"end\"):\n                result = decode(bytes(ctext, \"utf-8\"), \"uu\").decode()\n            else:\n                # If there isn't a \"being\" prefix and \"end\" suffix, we use the binascii module instead\n                # It is possible that the ctext has multiple lines, so convert each line and append\n                ctext_split = list(filter(None, ctext.splitlines()))\n                for _, value in enumerate(ctext_split):\n                    result += a2b_uu(value).decode(\"utf-8\")\n            logging.info(f\"UUencode successful, returning '{result}'\")\n            return result\n        except Exception:\n            logging.debug(\"Failed to decode UUencode\")\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"uuencode\"\n\n# Path: ciphey/basemods/Decoders/__init__.py\nfrom . import (\n    a1z26,\n    atbash,\n    base58_bitcoin,\n    base58_ripple,\n    base62,\n    base69,\n    base91,\n    bases,\n    baudot,\n    binary,\n    braille,\n    brainfuck,\n    decimal,\n    dna,\n    dtmf,\n    galactic,\n    gzip,\n    hexadecimal,\n    leetspeak,\n    morse_code,\n    multi_tap,\n    octal,\n    reverse,\n    tap_code,\n    unicode,\n    url,\n    uuencode,\n)\n\n# Path: ciphey/basemods/Resources/cipheydists.py\nfrom functools import lru_cache\nfrom typing import Any, Dict, Optional, Set\n\nimport cipheydists\nimport logging\n\nfrom ciphey.iface import (\n    Config,\n    Distribution,\n    ParamSpec,\n    ResourceLoader,\n    Translation,\n    WordList,\n    registry,\n)\n\n\n@registry.register_multi(WordList, Distribution, Translation)\nclass CipheyDists(ResourceLoader):\n    # _wordlists: Set[str] = frozenset({\"english\", \"english1000\", \"englishStopWords\"})\n    # _brandons: Set[str] = frozenset({\"english\"})\n    # _dists: Set[str] = frozenset({\"twist\"})\n    # _translates: Set[str] = frozenset({\"morse\"})\n    _getters = {\n        \"list\": cipheydists.get_list,\n        \"dist\": cipheydists.get_dist,\n        \"brandon\": cipheydists.get_brandon,\n        \"translate\": cipheydists.get_translate,\n    }\n\n    def whatResources(self) -> Optional[Set[str]]:\n        pass\n\n    @lru_cache()\n    def getResource(self, name: str) -> Any:\n        logging.debug(f\"Loading cipheydists resource {name}\")\n        prefix, name = name.split(\"::\", 1)\n        return self._getters[prefix](name)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n# Path: ciphey/basemods/Resources/files.py\nimport csv\nimport json\nfrom functools import lru_cache\nfrom typing import Dict, Generic, Optional, Set\n\nfrom ciphey.iface import (\n    Config,\n    Distribution,\n    ParamSpec,\n    ResourceLoader,\n    T,\n    WordList,\n    registry,\n)\n\n\n# We can use a generic resource loader here, as we can instantiate it later\n@registry.register_multi(WordList, Distribution)\nclass Json(ResourceLoader):\n    def whatResources(self) -> T:\n        return self._names\n\n    @lru_cache()\n    def getResource(self, name: str) -> T:\n        prefix, name = name.split(\"::\", 1)\n        return {\"wordlist\": (lambda js: {js}), \"dist\": (lambda js: js)}[prefix](\n            json.load(open(self._paths[int(name) - 1]))\n        )\n\n    @staticmethod\n    def getName() -> str:\n        return \"json\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\"path\": ParamSpec(req=True, desc=\"The path to a JSON file\", list=True)}\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self._paths = self._params()[\"path\"]\n        self._names = set(range(1, len(self._paths)))\n\n\n# We can use a generic resource loader here, as we can instantiate it later\n@registry.register_multi(WordList, Distribution)\nclass Csv(Generic[T], ResourceLoader[T]):\n    def whatResources(self) -> Set[str]:\n        return self._names\n\n    @lru_cache()\n    def getResource(self, name: str) -> T:\n        prefix, name = name.split(\"::\", 1)\n        return {\n            \"wordlist\": (lambda reader: {i[0] for i in reader}),\n            \"dist\": (lambda reader: {i[0]: float(i[1]) for i in reader}),\n        }[prefix](csv.reader(open(self._paths[int(name) - 1])))\n\n    @staticmethod\n    def getName() -> str:\n        return \"csv\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\"path\": ParamSpec(req=True, desc=\"The path to a CSV file\", list=True)}\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self._paths = self._params()[\"path\"]\n        self._names = set(range(1, len(self._paths)))\n\n# Path: ciphey/basemods/Resources/__init__.py\nfrom . import cipheydists, files\n\n# Path: ciphey/basemods/Searchers/ausearch.py\nimport bisect\nimport distutils\nimport math\nfrom copy import copy\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nfrom typing import Any, Dict, Generic, List, Optional, TypeVar, Union\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import (\n    Checker,\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    Decoder,\n    ParamSpec,\n    Searcher,\n    SearchLevel,\n    SearchResult,\n    T,\n    registry,\n)\n\n\"\"\"\nWe are using a tree structure here, because that makes searching and tracing back easier\nAs such, when we encounter another possible parent, we remove that edge\n\"\"\"\n\n\nclass DuplicateNode(Exception):\n    pass\n\n\n@dataclass\nclass AuSearchSuccessful(Exception):\n    target: \"Node\"\n    info: str\n\n\n@dataclass\nclass Node:\n    # The root has no parent edge\n    level: SearchLevel\n    parent: Optional[\"Edge\"] = None\n    depth: int = 0\n\n    @staticmethod\n    def decoding(\n        config: Config, route: Union[Cracker, Decoder], result: Any, source: \"Node\"\n    ) -> \"Node\":\n        if not config.cache.mark_ctext(result):\n            raise DuplicateNode()\n\n        checker: Checker = config.objs[\"checker\"]\n        ret = Node(\n            parent=None,\n            level=SearchLevel(\n                name=type(route).__name__.lower(), result=CrackResult(value=result)\n            ),\n            depth=source.depth + 1,\n        )\n        edge = Edge(source=source, route=route, dest=ret)\n        ret.parent = edge\n        check_res = checker(result)\n        if check_res is not None:\n            raise AuSearchSuccessful(target=ret, info=check_res)\n        return ret\n\n    @staticmethod\n    def cracker(config: Config, edge_template: \"Edge\", result: CrackResult) -> \"Node\":\n        if not config.cache.mark_ctext(result.value):\n            raise DuplicateNode()\n\n        checker: Checker = config.objs[\"checker\"]\n        # Edges do not directly contain containers, so this is fine\n        edge = copy(edge_template)\n        ret = Node(\n            parent=edge,\n            level=SearchLevel(name=type(edge.route).__name__.lower(), result=result),\n            depth=edge.source.depth + 1,\n        )\n        edge.dest = ret\n        check_res = checker(result.value)\n        if check_res is not None:\n            raise AuSearchSuccessful(target=ret, info=check_res)\n        return ret\n\n    @staticmethod\n    def root(config: Config, ctext: Any):\n        if not config.cache.mark_ctext(ctext):\n            raise DuplicateNode()\n\n        return Node(parent=None, level=SearchLevel.input(ctext))\n\n    def get_path(self):\n        if self.parent is None:\n            return [self.level]\n        return self.parent.source.get_path() + [self.level]\n\n\n@dataclass\nclass AusearchEdge:\n    # TODO: This is just CrackInfo with failure probability added...\n    success_probability: float\n    failure_probability: float\n    success_time: float\n    failure_time: float\n\n    def __init__(self, success_probability, success_time, failure_time):\n        self.success_probability = success_probability\n        self.failure_probability = 1.0 - success_probability\n        self.success_time = success_time\n        self.failure_time = failure_time\n\n\n@dataclass\nclass AusearchResult:\n    weight: float\n    index: int\n\n\ndef calculate_score(info: CrackInfo):\n    return info.success_likelihood / \\\n        (info.success_runtime * info.success_likelihood + info.failure_runtime * (1-info.success_likelihood))\n\n\n@dataclass\nclass Edge:\n    source: Node\n    route: Union[Cracker, Decoder]\n    dest: Optional[Node] = None\n    # Info is not filled in for Decoders\n    score: Optional[float] = None\n\n\nPriorityType = TypeVar(\"PriorityType\")\n\n\nclass PriorityWorkQueue(Generic[PriorityType, T]):\n    _sorted_priorities: List[PriorityType]\n    _queues: Dict[Any, List[T]]\n\n    def add_work(self, priority: PriorityType, work: List[T]) -> None:\n        logging.debug(f\"\"\"Adding work at depth {priority}\"\"\")\n\n        idx = bisect.bisect_left(self._sorted_priorities, priority)\n        if (\n            idx == len(self._sorted_priorities)\n            or self._sorted_priorities[idx] != priority\n        ):\n            self._sorted_priorities.insert(idx, priority)\n        self._queues.setdefault(priority, []).extend(work)\n\n    def get_work(self) -> T:\n        best_priority = self._sorted_priorities[0]\n        target = self._queues[best_priority]\n        ret = target.pop(0)\n        if len(target) == 0:\n            self._sorted_priorities.pop()\n        return ret\n\n    def get_work_chunk(self) -> List[T]:\n        \"\"\"Returns the best work for now\"\"\"\n        if len(self._sorted_priorities) == 0:\n            return []\n        best_priority = self._sorted_priorities.pop(0)\n        return self._queues.pop(best_priority)\n\n    def empty(self):\n        return len(self._sorted_priorities) == 0\n\n    def __init__(self):\n        self._sorted_priorities = []\n        self._queues = {}\n\n\n@registry.register\nclass AuSearch(Searcher):\n    # Deeper paths get done later\n    work: PriorityWorkQueue[int, Edge]\n\n    @staticmethod\n    def get_crackers_for(t: type):\n        return registry[Cracker[t]]\n\n    @lru_cache()  # To save extra sorting\n    def get_decoders_for(self, t: type):\n        ret = registry[Decoder[t]]\n        ret.sort(key=lambda x: x.priority(), reverse=True)\n        return ret\n\n    # def expand(self, edge: Edge) -> List[Edge]:\n    #     \"\"\"Evaluates the destination of the given, and adds its child edges to the pool\"\"\"\n    #     edge.dest = Node(parent=edge, level=edge.route(edge.source.level.result.value))\n\n    def expand_crackers(self, node: Node) -> None:\n        if node.depth >= self.max_cipher_depth:\n            return\n\n        res = node.level.result.value\n        additional_work = []\n\n        for i in self.get_crackers_for(type(res)):\n            inst = self._config()(i)\n            info = inst.getInfo(res)\n            if info.success_likelihood < self.p_threshold:\n                continue\n            additional_work.append(\n                Edge(source=node, route=inst, score=calculate_score(inst.getInfo(res)))\n            )\n\n        priority = min(node.depth, self.priority_cap)\n        if self.invert_priority:\n            priority = -priority\n\n        self.work.add_work(priority, additional_work)\n\n    def expand_decodings(self, node: Node) -> None:\n        val = node.level.result.value\n\n        for decoder in self.get_decoders_for(type(val)):\n            inst = self._config()(decoder)\n            res = inst(val)\n            if res is None:\n                continue\n            try:\n                new_node = Node.decoding(\n                    config=self._config(), route=inst, result=res, source=node\n                )\n            except DuplicateNode:\n                continue\n\n            logging.debug(\"Nesting encodings\")\n            self.recursive_expand(new_node, False)\n\n    def recursive_expand(self, node: Node, nested: bool = True) -> None:\n        if node.depth >= self.max_depth:\n            return\n\n        logging.debug(f\"Expanding depth {node.depth}\")\n\n        self.expand_decodings(node)\n\n        # Doing this last allows us to catch simple nested encodings faster\n        if not nested or self.enable_nested:\n            self.expand_crackers(node)\n\n    def search(self, ctext: Any) -> Optional[SearchResult]:\n        logging.debug(\n            f\"\"\"Beginning AuSearch with {\"inverted\" if self.invert_priority else \"normal\"} priority\"\"\"\n        )\n\n        try:\n            root = Node.root(self._config(), ctext)\n        except DuplicateNode:\n            return None\n\n        check_res = self._config().objs[\"checker\"](ctext)\n        if check_res is not None:\n            return SearchResult(check_res=check_res, path=[root.level])\n\n        try:\n            self.recursive_expand(root, False)\n\n            while True:\n                if self.work.empty():\n                    break\n                # Get the highest level result\n                chunk = self.work.get_work_chunk()\n                chunk.sort(key=lambda i: i.score)\n                # Work through all of this level's results\n                while len(chunk) != 0:\n                    logging.debug(f\"{len(chunk)} remaining on this level\")\n                    # TODO Cyclic uses some tricky C++ here\n                    # I know because it's sorted the one at the back (the anti-weight)\n                    # is the most likely\n\n                    edge: Edge = chunk.pop(-1)\n\n                    # Expand the node\n                    res = edge.route(edge.source.level.result.value)\n                    if res is None:\n                        continue\n                    for i in res:\n                        try:\n                            node = Node.cracker(\n...\n", "needle_token_start": 770, "needle_token_end": 814, "code_context_ntokens": 15997, "output": ["```python\ndef chars_to_byte(self, s: str):\n    return (69 * self.CHARS.index(s[1])) + (self.CHARS.index(s[0]))\n```"]}
{"repo": "Ciphey/Ciphey", "name": "__ge__", "language": "python", "path": "ciphey/iface/_modules.py", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To determine if the priority of the current instance is greater than or equal to another instance.\n2. **Input**: Another instance of the same class.\n3. **Output**: A boolean value indicating whether the current instance's priority is greater than or equal to the other instance's priority.\n4. **Procedure**: The method retrieves the priority value of the current instance and compares it with the priority value of the other instance using a greater than or equal to comparison.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/iface/_fwd.py\nregistry = None\nconfig = type(None)\n\n# Path: ciphey/iface/_modules.py\nfrom abc import ABC, abstractmethod\nfrom typing import Any, Dict, Generic, List, NamedTuple, Optional, Set, Type, TypeVar\n\nfrom rich import box\nfrom rich.console import Console\nfrom rich.markup import escape\nfrom rich.table import Table\n\nfrom ._fwd import config as Config\n\nT = TypeVar(\"T\")\nU = TypeVar(\"U\")\n\nconsole = Console()\n\n\nclass ParamSpec(NamedTuple):\n    \"\"\"\n    Attributes:\n        req         Whether this argument is required\n        desc        A description of what this argument does\n        default     The default value for this argument. Ignored if req == True or configPath is not None\n        config_ref  The path to the config that should be the default value\n        list        Whether this parameter is in the form of a list, and can therefore be specified more than once\n        visible     Whether the user can tweak this via the command line\n    \"\"\"\n\n    req: bool\n    desc: str\n    default: Optional[Any] = None\n    list: bool = False\n    config_ref: Optional[List[str]] = None\n    visible: bool = True\n\n\nclass ConfigurableModule(ABC):\n    @staticmethod\n    @abstractmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        \"\"\"\n        Returns a dictionary of `argument name: argument specification`\n        \"\"\"\n        pass\n\n    def _checkParams(self):\n        \"\"\"\n        Fills the given params dict with default values where arguments are not given,\n        using None as the default value for default values\n        \"\"\"\n\n        params = self._params()\n        config = self._config()\n\n        for key, value in self.getParams().items():\n            # If we already have it, then we don't need to do anything\n            if key in params:\n                continue\n            # If we don't have it, but it's required, then fail\n            if value.req:\n                raise KeyError(\n                    f\"Missing required param {key} for {type(self).__name__.lower()}\"\n                )\n            # If it's a reference by default, fill that in\n            if value.config_ref is not None:\n                tmp = getattr(config, value.config_ref[0])\n                params[key] = (\n                    tmp[value.config_ref[1:]] if len(value.config_ref) > 1 else tmp\n                )\n            # Otherwise, put in the default value (if it exists)\n            elif value.default is not None:\n                params[key] = value.default\n\n    def _params(self):\n        return self._params_obj\n\n    def _config(self):\n        return self._config_obj\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        self._config_obj = config\n        if self.getParams() is not None:\n            self._params_obj = config.params.setdefault(type(self).__name__.lower(), {})\n            self._checkParams()\n\n\nclass Targeted(ABC):\n    @staticmethod\n    @abstractmethod\n    def getTarget() -> str:\n        \"\"\"Should return the target that this object attacks/decodes\"\"\"\n        pass\n\n\nclass PolymorphicChecker(ConfigurableModule):\n    @abstractmethod\n    def check(self, text) -> Optional[str]:\n        \"\"\"Should return some description (or an empty string) on success, otherwise return None\"\"\"\n        pass\n\n    @abstractmethod\n    def getExpectedRuntime(self, text) -> float:\n        pass\n\n    def __call__(self, *args):\n        return self.check(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass Checker(Generic[T], ConfigurableModule):\n    @abstractmethod\n    def check(self, text: T) -> Optional[str]:\n        \"\"\"Should return some description (or an empty string) on success, otherwise return None\"\"\"\n        pass\n\n    @abstractmethod\n    def getExpectedRuntime(self, text: T) -> float:\n        pass\n\n    def __call__(self, *args):\n        return self.check(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @classmethod\n    def convert(cls, expected: Set[type]):\n        class PolyWrapperClass(PolymorphicChecker):\n            @staticmethod\n            def getParams() -> Optional[Dict[str, ParamSpec]]:\n                return cls.getParams()\n\n            def check(self, text) -> Optional[str]:\n                \"\"\"Should return some description (or an empty string) on success, otherwise return None\"\"\"\n                if type(text) not in expected:\n                    return None\n                else:\n                    return self._base.check(text)\n\n            def getExpectedRuntime(self, text) -> float:\n                if type(text) not in expected:\n                    return 0\n                else:\n                    return self._base.getExpectedRuntime(text)\n\n            def __init__(self, config: Config):\n                super().__init__(config)\n                # This is easier than inheritance\n                self._base = cls(config)\n\n        PolyWrapperClass.__name__ = cls.__name__\n\n        return PolyWrapperClass\n\n\n# class Detector(Generic[T], ConfigurableModule, KnownUtility, Targeted):\n#     @abstractmethod\n#     def scoreLikelihood(self, ctext: T) -> Dict[str, float]:\n#         \"\"\"Should return a dictionary of (cipher_name: score)\"\"\"\n#         pass\n#\n#     def __call__(self, *args): return self.scoreLikelihood(*args)\n#\n#     @abstractmethod\n#     def __init__(self, config: Config): super().__init__(config)\n\n\nclass Decoder(Generic[T], ConfigurableModule, Targeted):\n    \"\"\"Represents the undoing of some encoding into a different (or the same) type\"\"\"\n\n    @abstractmethod\n    def decode(self, ctext: T) -> Optional[U]:\n        pass\n\n    @staticmethod\n    @abstractmethod\n    def priority() -> float:\n        \"\"\"What proportion of decodings are this?\"\"\"\n        pass\n\n    def __call__(self, *args):\n        return self.decode(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass DecoderComparer:\n    value: Type[Decoder]\n\n    def __le__(self, other: \"DecoderComparer\"):\n        return self.value.priority() <= other.value.priority()\n\n    \ndef __ge__(self, other: \"DecoderComparer\"):\n        return self.value.priority() >= other.value.priority()\n\n    def __lt__(self, other: \"DecoderComparer\"):\n        return self.value.priority() < other.value.priority() and self != other\n\n    def __gt__(self, other: \"DecoderComparer\"):\n        return self.value.priority() > other.value.priority() and self != other\n\n    def __init__(self, value: Type[Decoder]):\n        self.value = value\n\n    def __repr__(self):\n        return f\"<DecoderComparer {self.value}:{self.value.priority()}>\"\n\n\nclass CrackResult(NamedTuple):\n    # TODO consider using Generic[T] again for value's type once\n    # https://bugs.python.org/issue36517 is resolved\n    value: Any\n    key_info: Optional[str] = None\n    misc_info: Optional[str] = None\n\n\nclass CrackInfo(NamedTuple):\n    success_likelihood: float\n    success_runtime: float\n    failure_runtime: float\n\n\nclass Cracker(Generic[T], ConfigurableModule, Targeted):\n    @abstractmethod\n    def getInfo(self, ctext: T) -> CrackInfo:\n        \"\"\"Should return some informed guesses on resource consumption when run on `ctext`\"\"\"\n        pass\n\n    @abstractmethod\n    def attemptCrack(self, ctext: T) -> List[CrackResult]:\n        \"\"\"\n        This should attempt to crack the cipher `target`, and return a list of candidate solutions\n        \"\"\"\n        # FIXME: Actually CrackResult[T], but python complains\n        pass\n\n    def __call__(self, *args):\n        return self.attemptCrack(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass ResourceLoader(Generic[T], ConfigurableModule):\n    @abstractmethod\n    def whatResources(self) -> Optional[Set[str]]:\n        \"\"\"\n        Return a set of the names of instances T you can provide.\n        The names SHOULD be unique amongst ResourceLoaders of the same type\n\n        These names will be exposed as f\"{self.__name__}::{name}\", use split_resource_name to recover this\n\n        If you cannot reasonably determine what resources you provide, return None instead\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def getResource(self, name: str) -> T:\n        \"\"\"\n        Returns the requested distribution\n\n        The behavior is undefined if `name not in self.what_resources()`\n        \"\"\"\n        pass\n\n    def __call__(self, *args):\n        return self.getResource(*args)\n\n    def __getitem__(self, *args):\n        return self.getResource(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass SearchLevel(NamedTuple):\n    name: str\n    result: CrackResult\n\n    @staticmethod\n    def input(ctext: Any):\n        return SearchLevel(name=\"input\", result=CrackResult(ctext))\n\n\nclass SearchResult(NamedTuple):\n    path: List[SearchLevel]\n    check_res: str\n\n\nclass Searcher(ConfigurableModule):\n    \"\"\"A very basic interface for code that plans out how to crack the ciphertext\"\"\"\n\n    @abstractmethod\n    def search(self, ctext: Any) -> Optional[SearchResult]:\n        \"\"\"Returns the path to the correct ciphertext\"\"\"\n        pass\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\ndef pretty_search_results(res: SearchResult, display_intermediate: bool = False) -> str:\n    # TODO what is display_intermediate\n    ret: str = \"\"\n    table = Table(show_header=False, box=box.ROUNDED, safe_box=False)\n    # Only print the checker if we need to. Normal people don't know what\n    # \"quadgrams\", \"brandon\", \"json checker\" is.\n    # We print the checker if its regex or another language, so long as it starts with:\n    # \"The\" like \"The plaintext is a Uniform Resource Locator (URL).\"\n    if len(res.check_res) != 0 and (\"The\" == res.check_res[0:3] or \"Passed\" == res.check_res[0:6]):\n        ret += f\"{res.check_res}\\n\"\n\n    def add_one():\n        out = \"\"\n        if i.name == \"utf8\":\n            out += f\"   [#808080]{i.name}[/#808080]\\n\"\n        else:\n            out += f\"   {i.name}\"\n        already_broken = False\n        if i.result.key_info is not None:\n            out += f\":\\n    Key: {i.result.key_info}\\n\"\n            already_broken = True\n        if i.result.misc_info is not None:\n            if not already_broken:\n                out += \":\\n\"\n            out += f\"    Misc: {i.result.misc_info}\\n\"\n            already_broken = True\n        if display_intermediate:\n            if not already_broken:\n                out += \":\\n\"\n            out += f'    Value: \"{i.result.value}\"\\n'\n            already_broken = True\n        if not already_broken:\n            out += \"\\n\"\n        return out, already_broken\n\n    # Skip the 'input' and print in order\n    already_broken = False\n    out = \"\"\n    for i in res.path[1:]:\n        output, already_broken = add_one()\n        out += output\n\n    if out:\n        if len(out.split(\"\\n\")) > 1:\n            ret += \"Formats used:\\n\"\n        else:\n            ret += \"Format used:\\n\"\n        ret += out\n\n    # Remove trailing newline\n    ret = ret[:-1]\n\n    # If we didn't show intermediate steps, then print the final result\n    if already_broken:\n        ret += f\"\"\"\\nPlaintext: [bold green]\"{escape(res.path[-1].result.value)}\"[bold green]\"\"\"\n    else:\n        ret += f\"\"\"Plaintext: [bold green]\"{escape(res.path[-1].result.value)}\"[bold green]\"\"\"\n\n    table.add_row(ret)\n    return table\n\n\n# Some common collection types\nDistribution = Dict[str, float]\nTranslation = Dict[str, str]\nWordList = Set[str]\n\n# Path: ciphey/iface/_config.py\nimport datetime\nimport os\nimport pydoc\nfrom typing import Any, Callable, Dict, List, Optional, Type, Union\n\nimport appdirs\nimport yaml\nimport logging\nfrom rich.logging import RichHandler\n\nfrom . import _fwd\nfrom ._modules import PolymorphicChecker, ResourceLoader, Searcher\n\n\nclass Cache:\n    \"\"\"Used to track state between levels of recursion to stop infinite loops, and to optimise repeating actions\"\"\"\n\n    def __init__(self):\n        self._cache: Dict[Any, Dict[str, Any]] = {}\n\n    def mark_ctext(self, ctext: Any) -> bool:\n        if (isinstance(ctext, str) or isinstance(ctext, bytes)) and len(ctext) < 4:\n            logging.debug(f\"Candidate {ctext.__repr__()} too short!\")\n            return False\n\n        if ctext in self._cache:\n            logging.debug(f\"Deduped {ctext.__repr__()}\")\n            return False\n\n        logging.debug(f\"New ctext {ctext.__repr__()}\")\n\n        self._cache[ctext] = {}\n        return True\n\n    def get_or_update(self, ctext: Any, keyname: str, get_value: Callable[[], Any]):\n        # Should have been marked first\n        target = self._cache[ctext]\n        res = target.get(keyname)\n        if res is not None:\n            return res\n\n        val = get_value()\n        target[keyname] = val\n        return val\n\n    def try_get(self, ctext: Any, keyname: str):\n        return self._cache[ctext].get(keyname)\n\n\ndef split_resource_name(full_name: str) -> (str, str):\n    return full_name.split(\"::\", 1)\n\n\nclass Config:\n    def __init__(self):\n        self.verbosity: int = 0\n        self.searcher: str = \"ausearch\"\n        self.params: Dict[str, Dict[str, Union[str, List[str]]]] = {}\n        self.format: str = \"str\"\n        self.modules: List[str] = []\n        self.checker: str = \"ezcheck\"\n        self.default_dist: str = \"cipheydists::dist::english\"\n        self.timeout: Optional[int] = None\n        self._inst: Dict[type, Any] = {}\n        self.objs: Dict[str, Any] = {}\n        self.cache: Cache = Cache()\n\n    @staticmethod\n    def get_default_dir() -> str:\n        return appdirs.user_config_dir(\"ciphey\")\n\n    def merge_dict(self, config_file: Optional[Dict[str, Any]]):\n        if config_file is None:\n            return\n        for a, b in config_file.items():\n            self.update(a, b)\n\n    def load_file(\n        self,\n        path: str = os.path.join(get_default_dir.__func__(), \"config.yml\"),\n        create=False,\n    ):\n        try:\n            with open(path, \"r+\") as file:\n                return self.merge_dict(yaml.safe_load(file))\n        except FileNotFoundError:\n            if create:\n                open(path, \"w+\")\n\n    def instantiate(self, t: type) -> Any:\n        \"\"\"\n        Used to enable caching of a instantiated type after the configuration has settled\n        \"\"\"\n        # We cannot use set default as that would construct it again, and throw away the result\n        res = self._inst.get(t)\n        if res is not None:\n            return res\n        ret = t(self)\n        self._inst[t] = ret\n        return ret\n\n    def __call__(self, t: type) -> Any:\n        return self.instantiate(t)\n\n    def update(self, attrname: str, value: Optional[Any]):\n        if value is not None:\n            setattr(self, attrname, value)\n\n    def update_param(self, owner: str, name: str, value: Optional[Any]):\n        if value is None:\n            return\n\n        target = self.params.setdefault(owner, {})\n\n        if _fwd.registry.get_named(owner).getParams()[name].list:\n            target.setdefault(name, []).append(value)\n        else:\n            target[name] = value\n\n    def update_format(self, value: Optional[str]):\n        if value is not None:\n            self.format = value\n\n    def load_objs(self):\n        # Basic type conversion\n        if self.timeout is not None:\n            self.objs[\"timeout\"] = datetime.timedelta(seconds=int(self.timeout))\n        self.objs[\"format\"] = pydoc.locate(self.format)\n\n        # Checkers do not depend on any other config object\n        logging.debug(f\"Registry is {_fwd.registry._reg[PolymorphicChecker]}\")\n        self.objs[\"checker\"] = self(\n            _fwd.registry.get_named(self.checker, PolymorphicChecker)\n        )\n        # Searchers only depend on checkers\n        self.objs[\"searcher\"] = self(_fwd.registry.get_named(self.searcher, Searcher))\n\n    def update_log_level(self, verbosity: Optional[int]):\n        if verbosity is None:\n            return\n        self.verbosity = verbosity\n\n        if verbosity == 0:\n            self.verbosity = logging.WARNING\n        elif verbosity == 1:\n            self.verbosity = logging.INFO\n        elif verbosity >= 2:\n            self.verbosity = logging.DEBUG\n        else:\n            logging.disable(logging.CRITICAL)\n            return\n\n        # https://rich.readthedocs.io/en/latest/logging.html for more on RichHandler\n        logging.basicConfig(\n            level=self.verbosity,\n            datefmt=\"[%X]\",\n            handlers=[RichHandler(markup=True, rich_tracebacks=True)],\n        )\n        logging.debug(f\"Verbosity set to level {verbosity}\")\n\n    def load_modules(self):\n        import importlib.util\n\n        for i in self.modules:\n            spec = importlib.util.spec_from_file_location(\"ciphey.module_load_site\", i)\n            mod = importlib.util.module_from_spec(spec)\n            spec.loader.exec_module(mod)\n\n        logging.info(f\"Loaded modules {_fwd.registry.get_all_names()}\")\n\n    def complete_config(self) -> \"Config\":\n        \"\"\"This does all the loading for the config, and then returns itself\"\"\"\n        self.load_modules()\n        self.load_objs()\n        self.update_log_level(self.verbosity)\n        return self\n\n    def get_resource(self, res_name: str, t: Optional[Type] = None) -> Any:\n        logging.debug(f\"Loading resource {res_name} of type {t}\")\n\n        # FIXME: Actually returns obj of type `t`, but python is bad\n        loader, name = split_resource_name(res_name)\n        if t is None:\n            return self(_fwd.registry.get_named(loader, ResourceLoader))(name)\n        else:\n            return self(_fwd.registry.get_named(loader, ResourceLoader[t]))(name)\n\n    # Setter methods for cleaner library API\n    def set_verbosity(self, i):\n        self.update_log_level(i)\n        return self\n\n    def set_spinner(self, spinner):\n        self.objs[\"spinner\"] = spinner\n\n    def pause_spinner_handle(self):\n        spinner = self.objs.get(\"spinner\")\n\n        class PausedSpinner:\n            def __enter__(self):\n                if spinner is not None:\n                    spinner.stop()\n\n            def __exit__(self, exc_type, exc_val, exc_tb):\n                if spinner is not None:\n                    spinner.start()\n\n        return PausedSpinner()\n\n    @staticmethod\n    def library_default():\n        \"\"\"The default config for use in a library\"\"\"\n        return Config().set_verbosity(-1)\n\n    def __str__(self):\n        return str(\n            {\n                \"verbosity\": self.verbosity,\n                \"searcher\": self.searcher,\n                \"params\": self.params,\n                \"format\": self.format,\n                \"modules\": self.modules,\n                \"checker\": self.checker,\n                \"default_dist\": self.default_dist,\n                \"timeout\": self.timeout,\n            }\n        )\n\n\n_fwd.config = Config\n\n# Path: ciphey/iface/_registry.py\nfrom typing import Any, Dict, List, Optional, Set, Tuple, Type, Union\n\ntry:\n    from typing import get_args, get_origin\nexcept ImportError:\n    from typing_inspect import get_origin, get_args\n\nfrom . import _fwd\nfrom ._modules import *\n\n\nclass Registry:\n    # I was planning on using __init_subclass__, but that is incompatible with dynamic type creation when we have\n    # generic keys\n\n    RegElem = Union[List[Type], Dict[Type, \"RegElem\"]]\n\n    _reg: Dict[Type, RegElem] = {}\n    _names: Dict[str, Tuple[Type, Set[Type]]] = {}\n    _targets: Dict[str, Dict[Type, List[Type]]] = {}\n    _modules = {Checker, Cracker, Decoder, ResourceLoader, Searcher, PolymorphicChecker}\n\n    def _register_one(self, input_type, module_base, module_args):\n        if len(module_args) == 0:\n            self._reg.setdefault(module_base, []).append(input_type)\n            return\n\n        target_reg = self._reg.setdefault(module_base, {})\n        # Seek to the given type\n        for subtype in module_args[0:-1]:\n            target_reg = target_reg.setdefault(subtype, {})\n        target_reg.setdefault(module_args[-1], []).append(input_type)\n\n    def _real_register(self, input_type: type, *args) -> Type:\n        name = input_type.__name__.lower()\n        name_target = self._names[name] = (input_type, set())\n\n        if issubclass(input_type, Targeted):\n            target = input_type.getTarget()\n        else:\n            target = None\n\n        if issubclass(input_type, Searcher):\n            module_type = module_base = Searcher\n            module_args = ()\n        else:\n            module_type: Optional[Type] = None\n            module_base = None\n\n            # Work out what module type this is\n            if len(args) == 0 and hasattr(input_type, \"__orig_bases__\"):\n                for i in input_type.__orig_bases__:\n                    if module_type is not None:\n                        raise TypeError(\n                            f\"Type derived from multiple registrable base classes {i} and {module_type}\"\n                        )\n                    module_base = get_origin(i)\n                    if module_base not in self._modules:\n                        continue\n                    module_type = i\n            else:\n                for i in self._modules:\n                    if not issubclass(input_type, i):\n                        continue\n                    if module_type is not None:\n                        raise TypeError(\n                            f\"Type derived from multiple registrable base classes {i} and {module_type}\"\n                        )\n                    module_type = i\n            if module_type is None:\n                raise TypeError(\"No registrable base class\")\n\n            # Replace input type with polymorphic checker if required\n            if issubclass(input_type, Checker):\n                if len(args) == 0:\n                    arg = [\n                        get_args(i)\n                        for i in input_type.__orig_bases__\n                        if get_origin(i) == Checker\n                    ][0]\n                    if len(arg) != 1:\n                        raise TypeError(\"No argument for Checker\")\n                    input_type = input_type.convert({arg[0]})\n                else:\n                    input_type = input_type.convert(set(args))\n                self._register_one(input_type, PolymorphicChecker, [])\n                # Refresh the names with the new type\n                name_target = self._names[name] = (input_type, {PolymorphicChecker})\n\n            # Now handle the difference between register and register_multi\n            if len(args) == 0:\n                if module_type is PolymorphicChecker:\n                    module_base = PolymorphicChecker\n                elif module_base is None:\n                    raise TypeError(\"No type argument given\")\n                self._register_one(input_type, module_base, get_args(module_type))\n                name_target[1].add(module_base)\n            else:\n                if module_base is not None:\n                    raise TypeError(f\"Redundant type argument for {module_type}\")\n                module_base = module_type\n                for module_args in args:\n                    # Correct missing brackets\n                    if not isinstance(module_args, tuple):\n                        module_args = (module_args,)\n\n                    self._register_one(input_type, module_base, module_args)\n                    name_target[1].add(module_type[module_args])\n\n        name_target[1].add(module_type)\n\n        if target is not None and issubclass(module_base, Targeted):\n            self._targets.setdefault(target, {}).setdefault(module_type, []).append(\n                input_type\n            )\n\n        return input_type\n\n    def register(self, input_type):\n        return self._real_register(input_type)\n\n    def register_multi(self, *x):\n        return lambda input_type: self._real_register(input_type, *x)\n\n    def __getitem__(self, i: type) -> Optional[Any]:\n        target_type = get_origin(i)\n        # Check if this is a non-generic type, and return the whole dict if it is\n        if target_type is None:\n            return self._reg[i]\n\n        target_subtypes = get_args(i)\n        target_list = self._reg.setdefault(target_type, {})\n        for subtype in target_subtypes:\n            target_list = target_list.setdefault(subtype, {})\n        return target_list\n\n    def get_named(self, name: str, type_constraint: Type = None) -> Any:\n        ret = self._names[name.lower()]\n        if type_constraint and type_constraint not in ret[1]:\n            raise TypeError(f\"Type mismatch: wanted {type_constraint}, got {ret[1]}\")\n        return ret[0]\n\n    def get_targeted(\n        self, target: str, type_constraint: Type = None\n    ) -> Optional[Union[Dict[Type, Set[Type]], Set[Type]]]:\n        x = self._targets.get(target)\n        if x is None or type_constraint is None:\n            return x\n        return x.get(type_constraint)\n\n    def get_all_names(self) -> List[str]:\n        return list(self._names.keys())\n\n    def __str__(self):\n        return f\"ciphey.iface.Registry {{_reg: {self._reg}, _names: {self._names}, _targets: {self._targets}}}\"\n\n\n_fwd.registry = Registry()\n\n# Path: ciphey/iface/__init__.py\nfrom ._config import Config\n\nfrom ._modules import (\n    Checker,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    Decoder,\n    DecoderComparer,\n    Distribution,\n    ParamSpec,\n    PolymorphicChecker,\n    ResourceLoader,\n    Searcher,\n    SearchLevel,\n    SearchResult,\n    T,\n    Translation,\n    U,\n    WordList,\n    pretty_search_results,\n)\nfrom ._registry import get_args, get_origin\n\nfrom ._fwd import registry\n\n# Path: ciphey/basemods/Checkers/any.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, ParamSpec, PolymorphicChecker, registry\n\n\n@registry.register\nclass Any(PolymorphicChecker):\n    \"\"\"Should only be used for debugging, frankly\"\"\"\n\n    def getExpectedRuntime(self, text) -> float:\n        return 0  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    def check(self, text: str) -> Optional[str]:\n        return \"\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/mathsHelper.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to provide helper functions for mathematics\n(oh, not entirely mathematics either. Some NLP stuff and sorting dicts. It's just a helper class\n)\n\"\"\"\n\nfrom collections import OrderedDict\nfrom string import punctuation\nfrom typing import Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\n\nclass mathsHelper:\n    \"\"\"Class to provide helper functions for mathematics and other small things\"\"\"\n\n    def __init__(self):\n        # ETAOIN is the most popular letters in order\n        self.ETAOIN = \"ETAOINSHRDLCUMWFGYPBVKJXQZ\"\n        self.LETTERS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n\n    @staticmethod\n    def gcd(a, b) -> int:\n        \"\"\"Greatest common divisor.\n\n        The Greatest Common Divisor of a and b using Euclid's Algorithm.\n\n        Args:\n            a -> num 1\n            b -> num 2\n\n        Returns:\n            Returns  GCD(a, b)\n\n        \"\"\"\n        # Return\n        while a != 0:\n            a, b = b % a, a\n        return b\n\n    @staticmethod\n    def mod_inv(a: int, m: int) -> Optional[int]:\n        \"\"\"\n        Returns the modular inverse of a mod m, or None if it does not exist.\n\n        The modular inverse of a is the number a_inv that satisfies the equation\n        a_inv * a mod m === 1 mod m\n\n        Note: This is a naive implementation, and runtime may be improved in several ways.\n        For instance by checking if m is prime to perform a different calculation,\n        or by using the extended euclidean algorithm.\n        \"\"\"\n        for i in range(1, m):\n            if (m * i + 1) % a == 0:\n                return (m * i + 1) // a\n        return None\n\n    @staticmethod\n    def percentage(part: float, whole: float) -> float:\n        \"\"\"Returns percentage.\n\n        Just a normal algorithm to return the percent.\n\n        Args:\n            part -> part of the whole number\n            whole -> the whole number\n\n        Returns:\n            Returns the percentage of part to whole.\n\n        \"\"\"\n        if part <= 0 or whole <= 0:\n            return 0\n        # works with percentages\n        return 100 * float(part) / float(whole)\n\n    def sort_prob_table(self, prob_table: dict) -> dict:\n        \"\"\"Sorts the probability table.\n\n        Sorts a dictionary of dictionaries (and all the sub-dictionaries).\n\n        Args:\n            prob_table -> The probability table returned by the neural network to sort.\n\n        Returns:\n            Returns the prob_table, but sorted.\n\n        \"\"\"\n        # for each object: prob table in dictionary\n        max_overall: int = 0\n        max_dict_pair: dict = {}\n        highest_key = None\n        empty_dict: dict = {}\n        # sorts the prob table before we find max, and converts it to order dicts\n        for key, value in prob_table.items():\n            prob_table[key] = self.new_sort(value)\n            prob_table[key] = dict(prob_table[key])\n\n        # gets maximum key then sets it to the front\n        counter_max: int = 0\n        counter_prob: int = len(prob_table)\n        while counter_max < counter_prob:\n            max_overall = 0\n            highest_key = None\n            logging.debug(\n                f\"Running while loop in sort_prob_table, counterMax is {counter_max}\"\n            )\n            for key, value in prob_table.items():\n                logging.debug(f\"Sorting {key}\")\n                maxLocal = 0\n                # for each item in that table\n                for key2, value2 in value.items():\n                    logging.debug(\n                        f\"Running key2 {key2}, value2 {value2} for loop for {value.items()}\"\n                    )\n                    maxLocal = maxLocal + value2\n                    logging.debug(\n                        f\"MaxLocal is {maxLocal} and maxOverall is {max_overall}\"\n                    )\n                    if maxLocal > max_overall:\n                        logging.debug(f\"New max local found {maxLocal}\")\n                        # because the dict doesn't reset\n                        max_dict_pair = {}\n                        max_overall = maxLocal\n                        # so eventually, we get the maximum dict pairing?\n                        max_dict_pair[key] = value\n                        highest_key = key\n                        logging.debug(f\"Highest key is {highest_key}\")\n                # removes the highest key from the prob table\n            logging.debug(\n                f\"Prob table is {prob_table} and highest key is {highest_key}\"\n            )\n            logging.debug(f\"Removing {prob_table[highest_key]}\")\n            del prob_table[highest_key]\n            logging.debug(f\"Prob table after deletion is {prob_table}\")\n            counter_max += 1\n            empty_dict = {**empty_dict, **max_dict_pair}\n\n        # returns the max dict (at the start) with the prob table\n        # this way, it should always work on most likely first.\n        logging.debug(\n            f\"The prob table is {prob_table} and the maxDictPair is {max_dict_pair}\"\n        )\n        logging.debug(f\"The new sorted prob table is {empty_dict}\")\n        return empty_dict\n\n    @staticmethod\n    def new_sort(new_dict: dict) -> dict:\n        \"\"\"Uses OrderedDict to sort a dictionary.\n\n        I think it's faster than my implementation.\n\n        Args:\n            new_dict -> the dictionary to sort\n\n        Returns:\n            Returns the dict, but sorted.\n\n        \"\"\"\n        # (f\"d is {d}\")\n        logging.debug(f\"The old dictionary before new_sort() is {new_dict}\")\n        sorted_i = OrderedDict(\n            sorted(new_dict.items(), key=lambda x: x[1], reverse=True)\n        )\n        logging.debug(f\"The dictionary after new_sort() is {sorted_i}\")\n        # sortedI = sort_dictionary(x)\n        return sorted_i\n\n    @staticmethod\n    def is_ascii(s: str) -> bool:\n        \"\"\"Returns the boolean value if is_ascii is an ascii char.\n\n        Does what it says on the tree. Stolen from\n        https://stackoverflow.com/questions/196345/how-to-check-if-a-string-in-python-is-in-ascii\n\n        Args:\n            s -> the char to check.\n\n        Returns:\n            Returns the boolean of the char.\n\n        \"\"\"\n\n        return bool(lambda s: len(s) == len(s.encode()))\n\n    @staticmethod\n    def strip_punctuation(text: str) -> str:\n        \"\"\"Strips punctuation from a given string.\n\n        Uses string.punctuation.\n\n        Args:\n            text -> the text to strip punctuation from.\n\n        Returns:\n            Returns string without punctuation.\n        \"\"\"\n        text: str = (str(text).translate(str.maketrans(\"\", \"\", punctuation))).strip(\n            \"\\n\"\n        )\n        return text\n\n# Path: ciphey/basemods/Checkers/brandon.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to determine whether something is English or not.\n1. Calculate the Chi Squared score of a sentence\n2. If the score is significantly lower than the average score, it _might_ be English\n    2.1. If the score _might_ be English, then take the text and compare it to the sorted dictionary\n    in O(n log n) time.\n    It creates a percentage of \"How much of this text is in the dictionary?\"\n    The dictionary contains:\n        * 20,000 most common US words\n        * 10,000 most common UK words (there's no repetition between the two)\n        * The top 10,000 passwords\n    If the word \"Looks like\" English (chi-squared) and if it contains English words, we can conclude it is\n    very likely English. The alternative is doing the dictionary thing but with an entire 479k word dictionary (slower)\n    2.2. If the score is not English, but we haven't tested enough to create an average, then test it against\n     the dictionary\n\nThings to optimise:\n* We only run the dictionary if it's 20% smaller than the average for chi squared\n* We consider it \"English\" if 45% of the text matches the dictionary\n* We run the dictionary if there is less than 10 total chisquared test\n\nHow to add a language:\n* Download your desired dictionary. Try to make it the most popular words, for example. Place this file into this\n folder with languagename.txt\nAs an example, this comes built in with english.txt\nFind the statistical frequency of each letter in that language.\nFor English, we have:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n}\nIn chisquared.py\nTo add your language, do:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n    \"German\": [0.0973]\n}\nIn alphabetical order\nAnd you're.... Done! Make sure the name of the two match up\n\"\"\"\nimport sys\nfrom math import ceil\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nsys.path.append(\"..\")\ntry:\n    import mathsHelper as mh\nexcept ModuleNotFoundError:\n    import ciphey.mathsHelper as mh\n\n\n@registry.register\nclass Brandon(Checker[str]):\n    \"\"\"\n    Class designed to confirm whether something is **language** based on how many words of **language** appears\n    Call confirmLanguage(text, language)\n    * text: the text you want to confirm\n    * language: the language you want to confirm\n\n    Find out what language it is by using chisquared.py, the highest chisquared score is the language\n    languageThreshold = 45\n    if a string is 45% **language** words, then it's confirmed to be english\n    \"\"\"\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually work this out\n        # TODO its 0.2 seconds on average\n        return 1e-4  # 100 \u00b5s\n\n    wordlist: set\n\n    def clean_text(self, text: str) -> set:\n        \"\"\"Cleans the text ready to be checked\n\n        Strips punctuation, makes it lower case, turns it into a set separated by spaces, removes duplicate words\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            text -> the text as a list, now cleaned\n\n        \"\"\"\n        # makes the text unique words and readable\n        text = text.lower()\n        text = self.mh.strip_punctuation(text)\n        text = text.split(\" \")\n        text = filter(lambda x: len(x) > 2, text)\n        text = set(text)\n        return text\n\n    def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        \"\"\"Given text determine if it passes checker\n\n        The checker uses the variable passed to it. I.E. Stopwords list, 1k words, dictionary\n\n        Args:\n            text -> The text to check\n            threshold -> at what point do we return True? The percentage of text that is in var before we return True\n            text_length -> the length of the text\n            var -> the variable we are checking against. Stopwords list, 1k words list, dictionary list.\n        Returns:\n            boolean -> True for it passes the test, False for it fails the test.\"\"\"\n        if text is None:\n            logging.debug(\"Checker's text is None, so returning False\")\n            return False\n        if var is None:\n            logging.debug(\"Checker's input var is None, so returning False\")\n            return False\n\n        percent = ceil(text_length * threshold)\n        logging.debug(f\"Checker's chunks are size {percent}\")\n        meet_threshold = 0\n        location = 0\n        end = percent\n\n        if text_length <= 0:\n            return False\n\n        while location <= text_length:\n            # chunks the text, so only gets THRESHOLD chunks of text at a time\n            text = list(text)\n            to_analyse = text[location:end]\n            logging.debug(f\"To analyse is {to_analyse}\")\n            for word in to_analyse:\n                # if word is a stopword, + 1 to the counter\n                if word in var:\n                    logging.debug(\n                        f\"{word} is in var, which means I am +=1 to the meet_threshold which is {meet_threshold}\"\n                    )\n                    meet_threshold += 1\n                meet_threshold_percent = meet_threshold / text_length\n                if meet_threshold_percent >= threshold:\n                    logging.debug(\n                        f\"Returning true since the percentage is {meet_threshold / text_length} and the threshold is {threshold}\"\n                    )\n                    # if we meet the threshold, return True\n                    # otherwise, go over again until we do\n                    # We do this in the for loop because if we're at 24% and THRESHOLD is 25\n                    # we don't want to wait THRESHOLD to return true, we want to return True ASAP\n                    return True\n            location = end\n            end = end + percent\n        logging.debug(\n            f\"The language proportion {meet_threshold_percent} is under the threshold {threshold}\"\n        )\n        return False\n\n    def __init__(self, config: Config):\n        # Suppresses warning\n        super().__init__(config)\n        self.mh = mh.mathsHelper()\n\n        phases = config.get_resource(self._params()[\"phases\"])\n\n        self.thresholds_phase1 = phases[\"1\"]\n        self.thresholds_phase2 = phases[\"2\"]\n        self.top1000Words = config.get_resource(self._params().get(\"top1000\"))\n        self.wordlist = config.get_resource(self._params()[\"wordlist\"])\n        self.stopwords = config.get_resource(self._params().get(\"stopwords\"))\n\n        self.len_phase1 = len(self.thresholds_phase1)\n        self.len_phase2 = len(self.thresholds_phase2)\n\n    def check(self, text: str) -> Optional[str]:\n        \"\"\"Checks to see if the text is in English\n\n        Performs a decryption, but mainly parses the internal data packet and prints useful information.\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            bool -> True if the text is English, False otherwise.\n\n        \"\"\"\n        logging.debug(f'In Language Checker with \"{text}\"')\n        text = self.clean_text(text)\n        logging.debug(f'Text split to \"{text}\"')\n        if text == \"\":\n            logging.debug(\"Returning None from Brandon as the text cleaned is none.\")\n            return None\n\n        length_text = len(text)\n\n        what_to_use = {}\n\n        # this code decides what checker / threshold to use\n        # if text is over or equal to maximum size, just use the maximum possible checker\n        what_to_use = self.calculateWhatChecker(\n            length_text, self.thresholds_phase1.keys()\n        )\n        logging.debug(self.thresholds_phase1)\n        what_to_use = self.thresholds_phase1[str(what_to_use)]\n        # def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        if \"check\" in what_to_use:\n            # perform check 1k words\n            result = self.checker(\n                text, what_to_use[\"check\"], length_text, self.top1000Words\n            )\n        elif \"stop\" in what_to_use:\n            # perform stopwords\n            result = self.checker(\n                text, what_to_use[\"stop\"], length_text, self.stopwords\n            )\n        elif \"dict\" in what_to_use:\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n            # If result is None, no point doing it again in phase2\n            if not result:\n                return None\n        else:\n            logging.info(f\"It is neither stop or check, but instead {what_to_use}\")\n\n        # return False if phase 1 fails\n        if not result:\n            return None\n        else:\n            what_to_use = self.calculateWhatChecker(\n                length_text, self.thresholds_phase2.keys()\n            )\n            what_to_use = self.thresholds_phase2[str(what_to_use)]\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n        return \"\" if result else None\n\n    def calculateWhatChecker(self, length_text, key):\n        \"\"\"Calculates what threshold / checker to use\n\n        If the length of the text is over the maximum sentence length, use the last checker / threshold\n        Otherwise, traverse the keys backwards until we find a key range that does not fit.\n        So we traverse backwards and see if the sentence length is between current - 1 and current\n        In this way, we find the absolute lowest checker / percentage threshold.\n        We traverse backwards because if the text is longer than the max sentence length, we already know.\n        In total, the keys are only 5 items long or so. It is not expensive to move backwards, nor is it expensive to move forwards.\n\n        Args:\n            length_text -> The length of the text\n            key -> What key we want to use. I.E. Phase1 keys, Phase2 keys.\n        Returns:\n            what_to_use -> the key of the lowest checker.\"\"\"\n\n        _keys = list(key)\n        _keys = list(map(int, _keys))\n        if length_text >= int(_keys[-1]):\n            what_to_use = list(key)[_keys.index(_keys[-1])]\n        else:\n            # this algorithm finds the smallest possible fit for the text\n            for counter, i in reversed(list(enumerate(_keys))):\n                #  [0, 110, 150]\n                if i <= length_text:\n                    what_to_use = i\n        return what_to_use\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"top1000\": ParamSpec(\n                desc=\"A wordlist of the top 1000 words\",\n                req=False,\n                default=\"cipheydists::list::english1000\",\n            ),\n            \"wordlist\": ParamSpec(\n                desc=\"A wordlist of all the words\",\n                req=False,\n                default=\"cipheydists::list::english\",\n            ),\n            \"stopwords\": ParamSpec(\n                desc=\"A wordlist of StopWords\",\n                req=False,\n                default=\"cipheydists::list::englishStopWords\",\n            ),\n            \"threshold\": ParamSpec(\n                desc=\"The minimum proportion (between 0 and 1) that must be in the dictionary\",\n                req=False,\n                default=0.45,\n            ),\n            \"phases\": ParamSpec(\n                desc=\"Language-specific phase thresholds\",\n                req=False,\n                default=\"cipheydists::brandon::english\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/format.py\nimport json\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass JsonChecker(Checker[str]):\n\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying json checker\")\n\n        # https://github.com/Ciphey/Ciphey/issues/389\n        if text.isdigit():\n            return None\n\n        try:\n            json.loads(text)\n            return \"\"\n        except ValueError:\n            return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 1e-7 * len(text)  # From benchmarks I found online\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/human.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, registry\nfrom rich.console import Console\nfrom rich.markup import escape\n\nconsole = Console()\n\n\n@registry.register\nclass HumanChecker(Checker[str]):\n\n    \"\"\"\n    Uses the person's decision to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: str) -> Optional[str]:\n        with self._config().pause_spinner_handle():\n            response = console.input(\n                f\"Possible plaintext: [blue bold]{escape(ctext.__repr__())}[/blue bold] ([green]y[/green]/[red]N[/red]): \"\n            )\n        if response == \"y\":\n            return \"\"\n        elif response in (\"n\", \"\"):\n            return None\n        else:\n            return self.check(ctext)\n\n    def getExpectedRuntime(self, text: str) -> float:\n        return 1  # About a second\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Checkers/quadgrams.py\nimport logging\nimport re\nfrom math import log10\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, Translation, registry\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Quadgrams(Checker[str]):\n\n    \"\"\"\n    Uses Quadgrams to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying Quadgrams checker\")\n        # Capitalize and remove everything that's not a letter\n        ctext = re.sub(\"[^A-Z]\", \"\", ctext.upper())\n        quadgrams = self.QUADGRAMS_DICT\n        quadgrams_sum = sum(quadgrams.values())\n        score = 0\n        for key in quadgrams.keys():\n            quadgrams[key] = float(quadgrams[key]) / quadgrams_sum\n        floor = log10(0.01 / quadgrams_sum)\n        for i in range(len(ctext) - 4 + 1):\n            # Get all quadgrams from ctext and check if they're in the dict\n            # If yes then add the score of those quadgrams to the total score\n            if ctext[i : i + 4] in quadgrams:\n                score += quadgrams[ctext[i : i + 4]]\n            else:\n                score += floor\n        if len(ctext) > 0:\n            score = score / len(ctext)\n        logging.info(f\"Quadgrams is {score}\")\n        # The default threshold was found to work the best from lots of testing\n        if score > self.threshold:\n            return \"\"\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The quadgrams dictionary to use\",\n                req=False,\n                default=\"cipheydists::dist::quadgrams\",\n            ),\n            \"score\": ParamSpec(\n                desc=\"The score threshold to use\",\n                req=False,\n                default=0.00011,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.QUADGRAMS_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.threshold = float(self._params()[\"score\"])\n\n# Path: ciphey/basemods/Checkers/regex.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Regex(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = list(map(re.compile, self._params()[\"regex\"]))\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"Passed with regex {regex}. Want to contribute to Ciphey? Submit your regex here to allow Ciphey to automatically get this next time https://github.com/bee-san/pyWhat/wiki/Adding-your-own-Regex\\n\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"regex\": ParamSpec(\n                req=True,\n                desc=\"The regex that must be matched (in a substring)\",\n                list=True,\n            )\n        }\n\n\n@registry.register\nclass RegexList(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = []\n        for i in self._params()[\"resource\"]:\n            self.regexes += [re.compile(regex) for regex in config.get_resource(i)]\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"passed with regex {regex}\"\n\n...\n", "needle_token_start": 1653, "needle_token_end": 1687, "code_context_ntokens": 15333, "output": ["```python\ndef __ge__(self, other: \"DecoderComparer\"):\n    return self.value.priority() >= other.value.priority()\n```"]}
{"repo": "Ciphey/Ciphey", "name": "alpha", "language": "python", "path": "ciphey/basemods/Crackers/hash.py", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle a specific type of cryptographic hash cracking, integrating with a broader decryption framework.\n2. **Input**: The function accepts two parameters: a ciphertext (string) and a hash type (string).\n3. **Output**: The function returns `None`, indicating either a placeholder implementation or that the output is handled through side effects not visible in the provided snippet.\n4. **Procedure**: Currently, the function does not implement any operational code, as indicated by the direct return of `None`. This suggests it might be a stub or template for future development or integration.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/basemods/Searchers/ausearch.py\nimport bisect\nimport distutils\nimport math\nfrom copy import copy\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nfrom typing import Any, Dict, Generic, List, Optional, TypeVar, Union\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import (\n    Checker,\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    Decoder,\n...\n# Path: ciphey/basemods/Searchers/__init__.py\nfrom . import ausearch\n\n# Path: ciphey/basemods/__init__.py\nfrom . import Checkers, Crackers, Decoders, Resources, Searchers\n\n# Path: ciphey/ciphey.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\nhttps://github.com/Ciphey\nhttps://github.com/Ciphey/Ciphey/wiki\n\nThe cycle goes:\nmain -> argparsing (if needed) -> call_encryption -> new Ciphey object -> decrypt() -> produceProbTable ->\none_level_of_decryption -> decrypt_normal\n\"\"\"\nimport os\nimport warnings\nfrom typing import Any, Optional, Union\n\nimport click\nfrom appdirs import AppDirs\nimport logging\nfrom rich.logging import RichHandler\nfrom rich.console import Console\n\nfrom . import iface\n\nwarnings.filterwarnings(\"ignore\")\n\nconsole = Console()\n\n\ndef decrypt(config: iface.Config, ctext: Any) -> Union[str, bytes]:\n    \"\"\"A simple alias for searching a ctext and makes the answer pretty\"\"\"\n    res: Optional[iface.SearchResult] = config.objs[\"searcher\"].search(ctext)\n    if res is None:\n        return \"Failed to crack\"\n    if config.verbosity < 0:\n        return res.path[-1].result.value\n    else:\n        return iface.pretty_search_results(res)\n\n\ndef get_name(ctx, param, value):\n    # reads from stdin if value was not supplied\n    if not value and not click.get_text_stream(\"stdin\").isatty():\n        click.get_text_stream(\"stdin\").read().strip()\n        return click.get_text_stream(\"stdin\").read().strip()\n    else:\n        return value\n\n\ndef print_help(ctx):\n    # prints help menu\n    # if no arguments are passed\n    click.echo(ctx.get_help())\n    ctx.exit()\n\n\n@click.command()\n@click.option(\n    \"-t\",\n    \"--text\",\n    help=\"The ciphertext you want to decrypt.\",\n    type=str,\n)\n@click.option(\n    \"-q\", \"--quiet\", help=\"Decrease verbosity\", type=int, count=True, default=None\n)\n@click.option(\n    \"-g\",\n    \"--greppable\",\n    help=\"Only print the answer (useful for grep)\",\n    type=bool,\n    is_flag=True,\n    default=None,\n)\n@click.option(\"-v\", \"--verbose\", count=True, type=int)\n@click.option(\"-C\", \"--checker\", help=\"Use the given checker\", default=None)\n@click.option(\n    \"-c\",\n    \"--config\",\n    help=\"Uses the given config file. Defaults to appdirs.user_config_dir('ciphey', 'ciphey')/'config.yml'\",\n)\n@click.option(\"-w\", \"--wordlist\", help=\"Uses the given wordlist\")\n@click.option(\n    \"-p\",\n    \"--param\",\n    help=\"Passes a parameter to the language checker\",\n    multiple=True,\n)\n@click.option(\n    \"-l\",\n    \"--list-params\",\n    help=\"List the parameters of the selected module\",\n    type=bool,\n)\n@click.option(\n    \"--searcher\",\n    help=\"Select the searching algorithm to use\",\n)\n# HARLAN TODO XXX\n# I switched this to a boolean flag system\n# https://click.palletsprojects.com/en/7.x/options/#boolean-flags\n# True for bytes input, False for str\n@click.option(\n    \"-b\",\n    \"--bytes\",\n    help=\"Forces ciphey to use binary mode for the input\",\n    is_flag=True,\n    default=None,\n)\n@click.option(\n    \"--default-dist\",\n    help=\"Sets the default character/byte distribution\",\n    type=str,\n    default=None,\n)\n@click.option(\n    \"-m\",\n    \"--module\",\n    help=\"Adds a module from the given path\",\n    type=click.Path(),\n    multiple=True,\n)\n@click.option(\n    \"-A\",\n    \"--appdirs\",\n    help=\"Print the location of where Ciphey wants the settings file to be\",\n    type=bool,\n    is_flag=True,\n)\n@click.option(\"-f\", \"--file\", type=click.File(\"rb\"), required=False)\n@click.argument(\"text_stdin\", callback=get_name, required=False)\ndef main(**kwargs):\n    \"\"\"Ciphey - Automated Decryption Tool\n\n    Documentation:\n    https://github.com/Ciphey/Ciphey/wiki\\n\n    Discord (support here, we're online most of the day):\n    http://discord.skerritt.blog\\n\n    GitHub:\n    https://github.com/ciphey/ciphey\\n\n\n    Ciphey is an automated decryption tool using smart artificial intelligence and natural language processing. Input encrypted text, get the decrypted text back.\n\n    Examples:\\n\n        Basic Usage: ciphey -t \"aGVsbG8gbXkgbmFtZSBpcyBiZWU=\"\n\n    \"\"\"\n\n    \"\"\"Function to deal with arguments. Either calls with args or not. Makes Pytest work.\n\n    It gets the arguments in the function definition using locals()\n    if withArgs is True, that means this is being called with command line args\n    so go to arg_parsing() to get those args\n    we then update locals() with the new command line args and remove \"withArgs\"\n    This function then calls call_encryption(**result) which passes our dict of args\n    to the function as its own arguments using dict unpacking.\n        Returns:\n            The output of the decryption.\n    \"\"\"\n\n    # if user wants to know where appdirs is\n    # print and exit\n    if \"appdirs\" in kwargs and kwargs[\"appdirs\"]:\n        dirs = AppDirs(\"Ciphey\", \"Ciphey\")\n        path_to_config = dirs.user_config_dir\n        print(\n            f\"The settings.yml file should be at {os.path.join(path_to_config, 'settings.yml')}\"\n        )\n        return None\n\n    # Now we create the config object\n    config = iface.Config()\n\n    # Load the settings file into the config\n    load_msg: str\n    cfg_arg = kwargs[\"config\"]\n    if cfg_arg is None:\n        # Make sure that the config dir actually exists\n        os.makedirs(iface.Config.get_default_dir(), exist_ok=True)\n        config.load_file(create=True)\n        load_msg = f\"Opened config file at {os.path.join(iface.Config.get_default_dir(), 'config.yml')}\"\n    else:\n        config.load_file(cfg_arg)\n        load_msg = f\"Opened config file at {cfg_arg}\"\n\n    # Load the verbosity, so that we can start logging\n    verbosity = kwargs[\"verbose\"]\n    quiet = kwargs[\"quiet\"]\n    if verbosity is None:\n        if quiet is not None:\n            verbosity = -quiet\n    elif quiet is not None:\n        verbosity -= quiet\n    if kwargs[\"greppable\"] is not None:\n        verbosity -= 999\n    # Use the existing value as a base\n    config.verbosity += verbosity\n    config.update_log_level(config.verbosity)\n    logging.info(load_msg)\n    logging.debug(f\"Got cmdline args {kwargs}\")\n\n    # Now we load the modules\n    module_arg = kwargs[\"module\"]\n    if module_arg is not None:\n        config.modules += list(module_arg)\n\n    # We need to load formats BEFORE we instantiate objects\n    if kwargs[\"bytes\"] is not None:\n        config.update_format(\"bytes\")\n\n    # Next, load the objects\n    params = kwargs[\"param\"]\n    if params is not None:\n        for i in params:\n            key, value = i.split(\"=\", 1)\n            parent, name = key.split(\".\", 1)\n            config.update_param(parent, name, value)\n    config.update(\"checker\", kwargs[\"checker\"])\n    config.update(\"searcher\", kwargs[\"searcher\"])\n    config.update(\"default_dist\", kwargs[\"default_dist\"])\n\n    config.complete_config()\n\n    logging.debug(f\"Command line opts: {kwargs}\")\n    logging.debug(f\"Config finalised: {config}\")\n\n    # Finally, we load the plaintext\n    if kwargs[\"text\"] is None:\n        if kwargs[\"file\"] is not None:\n            kwargs[\"text\"] = kwargs[\"file\"].read()\n        elif kwargs[\"text_stdin\"] is not None:\n            kwargs[\"text\"] = kwargs[\"text_stdin\"]\n        else:\n            # else print help menu\n            print(\"[bold red]Error. No inputs were given to Ciphey. [bold red]\")\n\n            @click.pass_context\n            def all_procedure(ctx):\n                print_help(ctx)\n\n            all_procedure()\n\n            return None\n\n    if issubclass(config.objs[\"format\"], type(kwargs[\"text\"])):\n        pass\n    elif config.objs[\"format\"] == str and isinstance(kwargs[\"text\"], bytes):\n        kwargs[\"text\"] = kwargs[\"text\"].decode(\"utf-8\")\n    elif config.objs[\"format\"] == bytes and isinstance(kwargs[\"text\"], str):\n        kwargs[\"text\"] = kwargs[\"text\"].encode(\"utf-8\")\n    else:\n        raise TypeError(f\"Cannot load type {config.format} from {type(kwargs['text'])}\")\n\n    result: Optional[str]\n\n    # if debug or quiet mode is on, run without spinner\n    if config.verbosity != 0:\n        result = decrypt(config, kwargs[\"text\"])\n    else:\n        # else, run with spinner if verbosity is 0\n        with console.status(\"[bold green]Thinking...\", spinner=\"moon\") as status:\n            config.set_spinner(status)\n            result = decrypt(config, kwargs[\"text\"])\n    if result is None:\n        result = \"Could not find any solutions.\"\n\n    console.print(result)\n\n# Path: ciphey/__init__.py\nfrom . import basemods, common, iface\nfrom .ciphey import decrypt\n\n# Path: ciphey/__main__.py\n#! /usr/bin/env python3\n\n\"\"\"\nCiphey: https://github.com/Ciphey/Ciphey\n\"\"\"\n\nimport platform\nimport sys\n\nif __name__ == \"__main__\":\n    major = sys.version_info[0]\n    minor = sys.version_info[1]\n\n    python_version = (\n        str(sys.version_info[0])\n        + \".\"\n        + str(sys.version_info[1])\n        + \".\"\n        + str(sys.version_info[2])\n    )\n\n    if major != 3 or minor < 6:\n        print(\n            f\"Ciphey requires Python 3.6+, you are using {python_version}. Please install a higher Python version. https://www.python.org/downloads/\"\n        )\n        print(\n            \"Alternatively, visit our Discord and use the Ciphey bot in #bots http://discord.skerritt.blog\"\n        )\n        sys.exit(1)\n    if platform.system() == \"Windows\":\n        if minor > 8:\n            print(\n                \"Ciphey does not currently support Python 3.9 on Windows. Please use the Discord bot at http://discord.skerritt.blog\"\n            )\n            sys.exit(1)\n\n        if sys.maxsize > 2 ** 32 is False:\n            print(\n                \"You are using Python 32 bit and Windows, Ciphey does not support this. Please upgrade to Python 64-bit here https://www.python.org/downloads/\"\n            )\n            sys.exit(1)\n    from .ciphey import main\n\n    main()\n\n# Path: ciphey/basemods/Checkers/entropy.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Entropy(Checker[str]):\n\n    \"\"\"\n    Uses entropy to determine plaintext\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying entropy checker\")\n        pass\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        # Uses benchmark from Discord\n        return 2e-7 * len(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/gtest.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass GTestChecker(Checker[str]):\n\n    \"\"\"\n    G-test of fitness, similar to Chi squared.\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying entropy checker\")\n        pass\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 4e-7 * len(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Crackers/hash.py\n\"\"\"\nThis is Hashbuster but slightly modified to work with Ciphey.\nWhy reinvent the wheel?\nChanges (that I can remember)\n* timeout set, as hashbuster took AGES before timeout was set.\nhttps://github.com/s0md3v/Hash-Buster\n\"\"\"\n\nimport re\nfrom typing import Dict, List, Optional\n\nimport requests\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, T, registry\n\nthread_count = 4\n\n\n\ndef alpha(ctext, hashtype):\n    return None\n\n\ndef beta(ctext, hashtype):\n    try:\n        response = requests.get(\n            \"https://hashtoolkit.com/reverse-hash/?hash=\", ctext, timeout=5\n        ).text\n    except requests.exceptions.ReadTimeout as e:\n        logging.info(f\"Beta failed timeout {e}\")\n    match = re.search(r'/generate-hash/?text=.*?\"', response)\n    if match:\n        return match.group(1)\n    return None\n\n\ndef gamma(ctext, hashtype):\n    try:\n        response = requests.get(\n            \"https://www.nitrxgen.net/md5db/\" + ctext, timeout=5\n        ).text\n    except requests.exceptions.ReadTimeout as e:\n        logging.info(f\"Gamma failed with {e}\")\n    if response:\n        return response\n    else:\n        return None\n\n\ndef delta(ctext, hashtype):\n    return None\n\n\ndef theta(ctext, hashtype):\n    try:\n        response = requests.get(\n            \"https://md5decrypt.net/Api/api.php?hash=%s&hash_type=%s&email=deanna_abshire@proxymail.eu&code=1152464b80a61728\"\n            % (ctext, hashtype),\n            timeout=5,\n        ).text\n    except requests.exceptions.ReadTimeout as e:\n        logging.info(f\"Gamma failed with {e}\")\n    if len(response) != 0:\n        return response\n    else:\n        return None\n\n\nmd5 = [gamma, alpha, beta, theta, delta]\nsha1 = [alpha, beta, theta, delta]\nsha256 = [alpha, beta, theta]\nsha384 = [alpha, beta, theta]\nsha512 = [alpha, beta, theta]\n\n\nresult = {}\n\n\ndef crack(ctext):\n    raise \"Error Crack is called\"\n\n\ndef threaded(ctext):\n    resp = crack(ctext)\n    if resp:\n        print(ctext + \" : \" + resp)\n        result[ctext] = resp\n\n\n@registry.register\nclass HashBuster(Cracker[str]):\n    @staticmethod\n    def getTarget() -> str:\n        return \"hash\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def getInfo(self, ctext: T) -> CrackInfo:\n        # TODO calculate these properly\n        return CrackInfo(\n            success_likelihood=0.5,\n            success_runtime=5,\n            failure_runtime=5,\n        )\n\n    def attemptCrack(self, ctext: T) -> List[CrackResult]:\n        logging.info(\"Starting to crack hashes\")\n        result = False\n\n        candidates = []\n        if len(ctext) == 32:\n            for api in md5:\n                r = api(ctext, \"md5\")\n                if result is not None or r is not None:\n                    logging.debug(\"MD5 returns True {r}\")\n                    candidates.append(result, \"MD5\")\n        elif len(ctext) == 40:\n            for api in sha1:\n                r = api(ctext, \"sha1\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha1 returns true\")\n                    candidates.append(result, \"SHA1\")\n        elif len(ctext) == 64:\n            for api in sha256:\n                r = api(ctext, \"sha256\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha256 returns true\")\n                    candidates.append(result, \"SHA256\")\n        elif len(ctext) == 96:\n            for api in sha384:\n                r = api(ctext, \"sha384\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha384 returns true\")\n                    candidates.append(result, \"SHA384\")\n        elif len(ctext) == 128:\n            for api in sha512:\n                r = api(ctext, \"sha512\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha512 returns true\")\n                    candidates.append(result, \"SHA512\")\n\n        # TODO what the fuck is this code?\n        logging.debug(f\"Hash buster returning {result}\")\n        # TODO add to 5.1 make this return multiple possible candidates\n        return [CrackResult(value=candidates[0][0], misc_info=candidates[1][1])]\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Decoders/base58_flickr.py\nfrom typing import Dict, Optional\n\nimport base58\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base58_flickr(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base58 (Flickr) decoding\n        \"\"\"\n        FLICKR_ALPHABET = b\"123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ\"\n        try:\n            return base58.b58decode(ctext, alphabet=FLICKR_ALPHABET).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base58_flickr\"\n\n# Path: ciphey/basemods/Decoders/base64_url.py\nimport base64\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base64_url(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base64 URL decoding\n        \"\"\"\n        ctext_padding = ctext + \"=\" * (4 - len(ctext) % 4)\n        try:\n            return base64.urlsafe_b64decode(ctext_padding).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base64_url\"\n\n# Path: ciphey/basemods/Decoders/base65536.py\nfrom typing import Dict, Optional\n\nimport base65536\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base65536(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base65536 decoding\n        \"\"\"\n        try:\n            return base65536.decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base65536\"\n\n# Path: ciphey/basemods/Decoders/z85.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\nfrom zmq.utils import z85\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Z85(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Z85 decoding\n        \"\"\"\n        ctext_len = len(ctext)\n        if ctext_len % 5:\n            logging.debug(\n                f\"Failed to decode Z85 because length must be a multiple of 5, not '{ctext_len}'\"\n            )\n            return None\n        try:\n            return z85.decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"z85\"\n\n# Path: ciphey/basemods/Searchers/perfection.py\nfrom typing import Dict, Optional, Set\n\nfrom ciphey.iface import Config, ParamSpec, registry\n\nfrom .ausearch import AuSearch, Node\n\n\n@registry.register\nclass Perfection(AuSearch):\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def findBestNode(self, nodes: Set[Node]) -> Node:\n        return next(iter(nodes))\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Searchers/astar.py\nimport cipheycore\n\n\nclass Node:\n    \"\"\"\n    A node has a value associated with it\n    Calculated from the heuristic\n    \"\"\"\n\n    def __init__(\n        self,\n        config,\n        h: float = None,\n        edges: (any, float) = None,\n        ctext: str = None,\n    ):\n        self.weight = h\n        # Edges is a list of other nodes it can connect to\n        self.edges = edges\n        self.ctext = ctext\n        self.h = h\n        self.path = []\n        self.information_content = config.cache.get_or_update(\n            self.text,\n            \"cipheycore::info_content\",\n            lambda: cipheycore.info_content(self.ctext),\n        )\n\n    def __le__(self, node2):\n        # if self is less than other\n        return self.x <= node2.x\n\n    def __lt__(self, node2):\n        return self.x < node2.x\n\n    def append_edge(self, edge):\n        self.edges.append(edge)\n\n    def get_edges(self):\n        return self.edges\n\n\nclass Graph:\n    # example of adjacency list (or rather map)\n    # adjacency_list = {\n    # 'A': [('B', 1), ('C', 3), ('D', 7)],\n    # 'B': [('D', 5)],\n    # 'C': [('D', 12)]\n    # }\n\n    def __init__(self, adjacency_list):\n        \"\"\"\n        adjacency list: basically the graph\n        \"\"\"\n        self.adjacency_list = adjacency_list\n        self.original_input = cipheycore.info_content(input)\n\n    def get_neighbors(self, v):\n        try:\n            return self.adjacency_list[v]\n        except KeyError:\n            # If we have exhausted the adjacency list\n            return []\n\n    # heuristic function with equal values for all nodes\n    def heuristic(self, n: Node):\n        return n.info_content / self.original_input\n\n    def a_star_algorithm(self, start_node: Node, stop_node: Node):\n        # TODO store the graph as an attribute\n        # open_list is a list of nodes which have been visited, but who's neighbors\n        # haven't all been inspected, starts off with the start node\n        # closed_list is a list of nodes which have been visited\n        # and who's neighbors have been inspected\n        open_list = set([start_node])\n        closed_list = set([])\n\n        # g contains current distances from start_node to all other nodes\n        # the default value (if it's not found in the map) is +infinity\n        g = {}\n\n        g[start_node] = 0\n\n        # parents contains an adjacency map of all nodes\n        parents = {}\n        parents[start_node] = start_node\n\n        while len(open_list) > 0:\n            print(f\"The open list is {open_list}\")\n            n = None\n\n            # find a node with the lowest value of f() - evaluation function\n            for v in open_list:\n                # TODO if v == decoder, run the decoder\n                print(f\"The for loop node v is {v}\")\n                if n is None or g[v] + self.h(v) < g[n] + self.h(n):\n                    n = v\n                    print(f\"The value of n is {n}\")\n\n            if n is None:\n                print(\"Path does not exist!\")\n                return None\n\n            # if the current node is the stop_node\n            # then we begin reconstructin the path from it to the start_node\n            # NOTE Uncomment this for an exit condition\n            # TODO Make it exit if decrypter returns True\n            # TODO We need to append the decryption methods to each node\n            # So when we reconstruct the path we can reconstruct the decryptions\n            # used\n            if n == stop_node:\n                print(\"n is the stop node, we are stopping!\")\n                reconst_path = []\n\n                while parents[n] != n:\n                    reconst_path.append(n)\n                    n = parents[n]\n\n                reconst_path.append(start_node)\n\n                reconst_path.reverse()\n\n                print(\"Path found: {}\".format(reconst_path))\n                return reconst_path\n\n            print(n)\n            for (m, weight) in self.get_neighbors(n):\n                print(f\"And the iteration is ({m}, {weight})\")\n                # if the current node isn't in both open_list and closed_list\n                # add it to open_list and note n as it's parent\n                if m not in open_list and m not in closed_list:\n                    open_list.add(m)\n                    parents[m] = n\n                    g[m] = g[n] + weight\n\n                # otherwise, check if it's quicker to first visit n, then m\n                # and if it is, update parent data and g data\n                # and if the node was in the closed_list, move it to open_list\n                else:\n                    if g[m] > g[n] + weight:\n                        g[m] = g[n] + weight\n                        parents[m] = n\n\n                        if m in closed_list:\n                            closed_list.remove(m)\n                            open_list.add(m)\n\n            # remove n from the open_list, and add it to closed_list\n            # because all of his neighbors were inspected\n            # open_list.remove(node)\n            # closed_list.add(node)\n\n            open_list.remove(n)\n            closed_list.add(n)\n            print(\"\\n\")\n\n        print(\"Path does not exist!\")\n        return None\n\n\nadjacency_list = {\n    \"A\": [(\"B\", 1), (\"C\", 3), (\"D\", 7)],\n    \"B\": [(\"D\", 5)],\n    \"C\": [(\"D\", 12)],\n}\nA = Node(1)\nB = Node(7)\nC = Node(9)\nD = Node(16)\n\nA.edges = [(B, 1), (C, 3), (D, 7)]\nB.edges = [(D, 5)]\nC.edges = [(D, 12)]\n\n# TODO use a dictionary comprehension to make this\nadjacency_list = {\n    A: A.edges,\n    B: B.edges,\n    C: C.edges,\n}\ngraph1 = Graph(adjacency_list)\ngraph1.a_star_algorithm(A, D)\n\n\"\"\"\nMaybe after it\n\"\"\"\n\n# Path: ciphey/basemods/Searchers/imperfection.py\nimport heapq\n\n\nclass Imperfection:\n    \"\"\"The graph is a Node: [List of nodes]\n    Where each item in the list of nodes can also have a node with a list of nodes\n\n    The result is that we can keep track of edges, while also keeping it small\n\n    To calculate current, we push the entire graph to A*\n\n    And it calculates the next node to choose, as well as increasing the size\n    of the graph with values\n\n    We're using a heap, meaning the element at [0] is always the smallest element\n\n    So we choose that and return it.\n\n\n    The current A* implementation has an end, we simply do not let it end as LC will make it\n    end far before it reaches Searcher again.\n\n    Current is the start position, so if we say we always start at the start of the graph it'll\n    go through the entire graph\n\n    graph = {\n            Node: [\n                {Node :\n                {\n                    node\n                    }\n                }\n                ]\n            }\n\n    For encodings we just do them straight out\n\n    The last value of parents from abstract\n    \"\"\"\n\n    \"\"\"\n\n   graph = {'A': ['B', 'C'],\n             'B': ['C', 'D'],\n             'C': ['D'],\n             'D': ['C'],\n             'E': ['F'],\n             'F': ['C']}\"\"\"\n\n    def __init__(self):\n        None\n\n    def findBestNode(self, nodes):\n        \"\"\"Finds the best decryption module\"\"\"\n        return next(iter(nodes))\n\n    # def aStar(self, graph, current, end):\n    #     \"\"\"The A* search algorithm\n\n    #     We're using heaps to find the minimum element (the one that will be the next current)\n    #     Heaps are like sets with O(1) lookup time, but maintain the lowest element as [0]\n    #     Sets insert in O(1), heaps in O(log N).\n\n    #     https://stackoverflow.com/questions/4159331/python-speed-up-an-a-star-pathfinding-algorithm\n\n    #     Current appears to be the list of all new tiles we can reach from current location\n\n    #     End is the end node, that won't actually run bc LC will make it return before it hits aSTar function\n    #     so tbh I'll just make it infinite unless something else forces a return\n\n    #     The graph is the actual data structure used. According to StackOverflow, it looks like this:\n\n    #     graph = {'A': ['B', 'C'],\n    #              'B': ['C', 'D'],\n    #              'C': ['D'],\n    #              'D': ['C'],\n    #              'E': ['F'],\n    #              'F': ['C']}\n\n    #     \"\"\"\n\n    #     # Runs decodings first\n\n    #     openSet = set()\n    #     openHeap = []\n    #     closedSet = set()\n\n    #     def retracePath(c):\n    #         # Retraces a path back to the start\n    #         path = [c]\n    #         while c.parent is not None:\n    #             c = c.parent\n    #             path.append(c)\n    #         path.reverse()\n    #         return path\n\n    #     # Adds the current location (start) to the heap and set\n    #     openSet.add(current)\n    #     openHeap.append((0, current))\n\n    #     # while openSet contains items\n    #     while openSet:\n    #         # TODO change openSet  to a heap?\n    #         # gets the 2nd element from the first element of the heap\n    #         # so the heap is (0, current)\n    #         # which means we pop current\n    #         # this makes me think that current isn't the first?\n    #         current = heapq.heappop(openHeap)[1]\n    #         # We don't actually want to end, so I'm commenting this:\n    #         # XXX\n    #         if current == end:\n    #             return retracePath(current)\n    #         # Removes it from todo and into done i think\n    #         # closedSet appears to be the set of things we have done\n    #         openSet.remove(current)\n    #         closedSet.add(current)\n\n    #         \"\"\"\n    #         Okay so our graph looks like this:\n    #         graph = {\n    #                 Node: [\n    #                     {Node :\n    #                     {\n    #                         node\n    #                         }\n    #                     }\n    #                     ]\n    #                 }\n    #         graph[current] **SHOULD** be the list of nodes which contains dictionaries of nodes\n\n    #         \"\"\"\n    #         for tile in graph[current]:\n    #             # ClosedSet appears to be the list of visited nodes\n    #             # TODO place this as a class attribute\n    #             if tile not in closedSet:\n    #                 # This is the heuristic\n    #                 # TODO expected_time/probability + k * heuristic, for some experimentally determined value of k\n    #                 tile.H = (abs(end.x - tile.x) + abs(end.y - tile.y)) * 10\n\n    #                 # if tile is not in the openSet, add it and then pop it from the heap\n    #                 if tile not in openSet:\n    #                     openSet.add(tile)\n    #                     heapq.heappush(openHeap, (tile.H, tile))\n    #                 #  I have no idea where this code is called lol\n    #                 tile.parent = current\n\n    #     # This returns Nothing\n    #     # I need to modify it so it finds the best item from Current\n    #     # So basically, return item 0 of openHeap\n    #     # return openHeap[0]\n    #     # Since the [0] item is always minimum\n    #     return []\n    def aStar(self, graph, current, end):\n        print(f\"The graph is {graph}\\nCurrent is {current}\\n and End is {end}\")\n        openSet = set()\n        openHeap = []\n        closedSet = set()\n\n        def retracePath(c):\n            print(\"Calling retrace path\")\n            path = [c]\n            while c.parent is not None:\n                c = c.parent\n                path.append(c)\n            path.reverse()\n            return path\n\n        print(\"\\n\")\n\n        openSet.add(current)\n        openHeap.append((0, current))\n        while openSet:\n            print(f\"Openset is {openSet}\")\n            print(f\"OpenHeap is {openHeap}\")\n            print(f\"ClosedSet is {closedSet}\")\n            print(f\"Current is {current}\")\n            print(f\"I am popping {openHeap} with the first element\")\n            current = heapq.heappop(openHeap)[1]\n            print(f\"Current is now {current}\")\n            print(f\"Graph current is {graph[current]}\")\n            if current == end:\n                return retracePath(current)\n            openSet.remove(current)\n            closedSet.add(current)\n            for tile in graph[current]:\n                if tile not in closedSet:\n                    tile.H = (abs(end.x - tile.x) + abs(end.y - tile.y)) * 10\n                    tile.H = 1\n                    if tile not in openSet:\n                        openSet.add(tile)\n                        heapq.heappush(openHeap, (tile.H, tile))\n                    tile.parent = current\n            print(\"\\n\")\n        return []\n\n\nclass Node:\n    \"\"\"\n    A node has a value associated with it\n    Calculated from the heuristic\n    \"\"\"\n\n    def __init__(self, h):\n        self.h = h\n        self.x = self.h\n        self.y = 0.6\n\n    def __le__(self, node2):\n        # if self is less than other\n        return self.x <= node2.x\n\n    def __lt__(self, node2):\n        return self.x < node2.x\n\n\nif __name__ == \"__main__\":\n    obj = Imperfection()\n    graph = {\n        \"A\": [\"B\", \"C\"],\n        \"B\": [\"C\", \"D\"],\n        \"C\": [\"D\"],\n        \"D\": [\"C\"],\n        \"E\": [\"F\"],\n        \"F\": [\"C\"],\n    }\n    # Makes the graph\n    y = Node(0.5)\n    x = Node(0.3)\n    p = Node(0.7)\n    q = Node(0.9)\n    graph = {y: [x, p], p: q}\n\n    print(obj.aStar(graph, y, q))\n\n", "needle_token_start": 4015, "needle_token_end": 4029, "code_context_ntokens": 10573, "output": ["```python\ndef alpha(ctext, hashtype):\n    return None\n```"]}
{"repo": "Ciphey/Ciphey", "name": "strip_punctuation", "language": "python", "path": "ciphey/mathsHelper.py", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To remove all punctuation characters from a provided string, making it cleaner for text processing or analysis.\n2. **Input**: A single string which may contain various characters including punctuation.\n3. **Output**: A modified version of the input string with all punctuation characters removed.\n4. **Procedure**: The function utilizes a translation table to replace each punctuation character with an empty string, effectively removing them from the input string. It also trims newline characters from the ends of the string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/iface/_config.py\nimport datetime\nimport os\nimport pydoc\nfrom typing import Any, Callable, Dict, List, Optional, Type, Union\n\nimport appdirs\nimport yaml\nimport logging\nfrom rich.logging import RichHandler\n\nfrom . import _fwd\nfrom ._modules import PolymorphicChecker, ResourceLoader, Searcher\n\n\nclass Cache:\n    \"\"\"Used to track state between levels of recursion to stop infinite loops, and to optimise repeating actions\"\"\"\n\n    def __init__(self):\n        self._cache: Dict[Any, Dict[str, Any]] = {}\n\n    def mark_ctext(self, ctext: Any) -> bool:\n        if (isinstance(ctext, str) or isinstance(ctext, bytes)) and len(ctext) < 4:\n            logging.debug(f\"Candidate {ctext.__repr__()} too short!\")\n            return False\n\n        if ctext in self._cache:\n            logging.debug(f\"Deduped {ctext.__repr__()}\")\n            return False\n\n        logging.debug(f\"New ctext {ctext.__repr__()}\")\n\n        self._cache[ctext] = {}\n        return True\n\n    def get_or_update(self, ctext: Any, keyname: str, get_value: Callable[[], Any]):\n        # Should have been marked first\n        target = self._cache[ctext]\n        res = target.get(keyname)\n        if res is not None:\n            return res\n\n        val = get_value()\n        target[keyname] = val\n        return val\n\n    def try_get(self, ctext: Any, keyname: str):\n        return self._cache[ctext].get(keyname)\n\n\ndef split_resource_name(full_name: str) -> (str, str):\n    return full_name.split(\"::\", 1)\n\n\nclass Config:\n    def __init__(self):\n        self.verbosity: int = 0\n        self.searcher: str = \"ausearch\"\n        self.params: Dict[str, Dict[str, Union[str, List[str]]]] = {}\n        self.format: str = \"str\"\n        self.modules: List[str] = []\n        self.checker: str = \"ezcheck\"\n        self.default_dist: str = \"cipheydists::dist::english\"\n        self.timeout: Optional[int] = None\n        self._inst: Dict[type, Any] = {}\n        self.objs: Dict[str, Any] = {}\n        self.cache: Cache = Cache()\n\n    @staticmethod\n    def get_default_dir() -> str:\n        return appdirs.user_config_dir(\"ciphey\")\n\n    def merge_dict(self, config_file: Optional[Dict[str, Any]]):\n        if config_file is None:\n            return\n        for a, b in config_file.items():\n            self.update(a, b)\n\n    def load_file(\n        self,\n        path: str = os.path.join(get_default_dir.__func__(), \"config.yml\"),\n        create=False,\n    ):\n        try:\n            with open(path, \"r+\") as file:\n                return self.merge_dict(yaml.safe_load(file))\n        except FileNotFoundError:\n            if create:\n                open(path, \"w+\")\n\n    def instantiate(self, t: type) -> Any:\n        \"\"\"\n        Used to enable caching of a instantiated type after the configuration has settled\n        \"\"\"\n        # We cannot use set default as that would construct it again, and throw away the result\n        res = self._inst.get(t)\n        if res is not None:\n            return res\n        ret = t(self)\n        self._inst[t] = ret\n        return ret\n\n    def __call__(self, t: type) -> Any:\n        return self.instantiate(t)\n\n    def update(self, attrname: str, value: Optional[Any]):\n        if value is not None:\n            setattr(self, attrname, value)\n\n    def update_param(self, owner: str, name: str, value: Optional[Any]):\n        if value is None:\n            return\n\n        target = self.params.setdefault(owner, {})\n\n        if _fwd.registry.get_named(owner).getParams()[name].list:\n            target.setdefault(name, []).append(value)\n        else:\n            target[name] = value\n\n    def update_format(self, value: Optional[str]):\n        if value is not None:\n            self.format = value\n\n    def load_objs(self):\n        # Basic type conversion\n        if self.timeout is not None:\n            self.objs[\"timeout\"] = datetime.timedelta(seconds=int(self.timeout))\n        self.objs[\"format\"] = pydoc.locate(self.format)\n\n        # Checkers do not depend on any other config object\n        logging.debug(f\"Registry is {_fwd.registry._reg[PolymorphicChecker]}\")\n        self.objs[\"checker\"] = self(\n            _fwd.registry.get_named(self.checker, PolymorphicChecker)\n        )\n        # Searchers only depend on checkers\n        self.objs[\"searcher\"] = self(_fwd.registry.get_named(self.searcher, Searcher))\n\n    def update_log_level(self, verbosity: Optional[int]):\n        if verbosity is None:\n            return\n        self.verbosity = verbosity\n\n        if verbosity == 0:\n            self.verbosity = logging.WARNING\n        elif verbosity == 1:\n            self.verbosity = logging.INFO\n        elif verbosity >= 2:\n            self.verbosity = logging.DEBUG\n        else:\n            logging.disable(logging.CRITICAL)\n            return\n\n...\n# Path: ciphey/iface/_registry.py\nfrom typing import Any, Dict, List, Optional, Set, Tuple, Type, Union\n\ntry:\n    from typing import get_args, get_origin\nexcept ImportError:\n    from typing_inspect import get_origin, get_args\n\nfrom . import _fwd\nfrom ._modules import *\n\n\nclass Registry:\n    # I was planning on using __init_subclass__, but that is incompatible with dynamic type creation when we have\n    # generic keys\n\n    RegElem = Union[List[Type], Dict[Type, \"RegElem\"]]\n\n    _reg: Dict[Type, RegElem] = {}\n    _names: Dict[str, Tuple[Type, Set[Type]]] = {}\n    _targets: Dict[str, Dict[Type, List[Type]]] = {}\n    _modules = {Checker, Cracker, Decoder, ResourceLoader, Searcher, PolymorphicChecker}\n\n    def _register_one(self, input_type, module_base, module_args):\n        if len(module_args) == 0:\n            self._reg.setdefault(module_base, []).append(input_type)\n            return\n\n        target_reg = self._reg.setdefault(module_base, {})\n        # Seek to the given type\n        for subtype in module_args[0:-1]:\n            target_reg = target_reg.setdefault(subtype, {})\n        target_reg.setdefault(module_args[-1], []).append(input_type)\n\n    def _real_register(self, input_type: type, *args) -> Type:\n        name = input_type.__name__.lower()\n        name_target = self._names[name] = (input_type, set())\n\n        if issubclass(input_type, Targeted):\n            target = input_type.getTarget()\n        else:\n            target = None\n\n        if issubclass(input_type, Searcher):\n            module_type = module_base = Searcher\n            module_args = ()\n        else:\n            module_type: Optional[Type] = None\n            module_base = None\n\n            # Work out what module type this is\n            if len(args) == 0 and hasattr(input_type, \"__orig_bases__\"):\n                for i in input_type.__orig_bases__:\n                    if module_type is not None:\n                        raise TypeError(\n                            f\"Type derived from multiple registrable base classes {i} and {module_type}\"\n                        )\n                    module_base = get_origin(i)\n                    if module_base not in self._modules:\n                        continue\n                    module_type = i\n            else:\n                for i in self._modules:\n                    if not issubclass(input_type, i):\n                        continue\n                    if module_type is not None:\n                        raise TypeError(\n                            f\"Type derived from multiple registrable base classes {i} and {module_type}\"\n                        )\n                    module_type = i\n            if module_type is None:\n                raise TypeError(\"No registrable base class\")\n\n            # Replace input type with polymorphic checker if required\n            if issubclass(input_type, Checker):\n                if len(args) == 0:\n                    arg = [\n                        get_args(i)\n                        for i in input_type.__orig_bases__\n                        if get_origin(i) == Checker\n                    ][0]\n                    if len(arg) != 1:\n                        raise TypeError(\"No argument for Checker\")\n                    input_type = input_type.convert({arg[0]})\n                else:\n                    input_type = input_type.convert(set(args))\n                self._register_one(input_type, PolymorphicChecker, [])\n                # Refresh the names with the new type\n                name_target = self._names[name] = (input_type, {PolymorphicChecker})\n\n            # Now handle the difference between register and register_multi\n            if len(args) == 0:\n                if module_type is PolymorphicChecker:\n                    module_base = PolymorphicChecker\n                elif module_base is None:\n                    raise TypeError(\"No type argument given\")\n                self._register_one(input_type, module_base, get_args(module_type))\n                name_target[1].add(module_base)\n            else:\n                if module_base is not None:\n                    raise TypeError(f\"Redundant type argument for {module_type}\")\n                module_base = module_type\n                for module_args in args:\n                    # Correct missing brackets\n                    if not isinstance(module_args, tuple):\n                        module_args = (module_args,)\n\n                    self._register_one(input_type, module_base, module_args)\n                    name_target[1].add(module_type[module_args])\n\n        name_target[1].add(module_type)\n\n        if target is not None and issubclass(module_base, Targeted):\n            self._targets.setdefault(target, {}).setdefault(module_type, []).append(\n                input_type\n            )\n\n        return input_type\n\n    def register(self, input_type):\n        return self._real_register(input_type)\n\n    def register_multi(self, *x):\n        return lambda input_type: self._real_register(input_type, *x)\n\n    def __getitem__(self, i: type) -> Optional[Any]:\n        target_type = get_origin(i)\n        # Check if this is a non-generic type, and return the whole dict if it is\n        if target_type is None:\n            return self._reg[i]\n\n        target_subtypes = get_args(i)\n        target_list = self._reg.setdefault(target_type, {})\n        for subtype in target_subtypes:\n            target_list = target_list.setdefault(subtype, {})\n        return target_list\n\n    def get_named(self, name: str, type_constraint: Type = None) -> Any:\n        ret = self._names[name.lower()]\n        if type_constraint and type_constraint not in ret[1]:\n            raise TypeError(f\"Type mismatch: wanted {type_constraint}, got {ret[1]}\")\n        return ret[0]\n\n    def get_targeted(\n        self, target: str, type_constraint: Type = None\n    ) -> Optional[Union[Dict[Type, Set[Type]], Set[Type]]]:\n        x = self._targets.get(target)\n        if x is None or type_constraint is None:\n            return x\n        return x.get(type_constraint)\n\n    def get_all_names(self) -> List[str]:\n        return list(self._names.keys())\n\n    def __str__(self):\n        return f\"ciphey.iface.Registry {{_reg: {self._reg}, _names: {self._names}, _targets: {self._targets}}}\"\n\n\n_fwd.registry = Registry()\n\n# Path: ciphey/iface/__init__.py\nfrom ._config import Config\n\nfrom ._modules import (\n    Checker,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    Decoder,\n    DecoderComparer,\n    Distribution,\n    ParamSpec,\n    PolymorphicChecker,\n    ResourceLoader,\n    Searcher,\n    SearchLevel,\n    SearchResult,\n    T,\n    Translation,\n    U,\n    WordList,\n    pretty_search_results,\n)\nfrom ._registry import get_args, get_origin\n\nfrom ._fwd import registry\n\n# Path: ciphey/basemods/Checkers/any.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, ParamSpec, PolymorphicChecker, registry\n\n\n@registry.register\nclass Any(PolymorphicChecker):\n    \"\"\"Should only be used for debugging, frankly\"\"\"\n\n    def getExpectedRuntime(self, text) -> float:\n        return 0  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    def check(self, text: str) -> Optional[str]:\n        return \"\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/mathsHelper.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to provide helper functions for mathematics\n(oh, not entirely mathematics either. Some NLP stuff and sorting dicts. It's just a helper class\n)\n\"\"\"\n\nfrom collections import OrderedDict\nfrom string import punctuation\nfrom typing import Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\n\nclass mathsHelper:\n    \"\"\"Class to provide helper functions for mathematics and other small things\"\"\"\n\n    def __init__(self):\n        # ETAOIN is the most popular letters in order\n        self.ETAOIN = \"ETAOINSHRDLCUMWFGYPBVKJXQZ\"\n        self.LETTERS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n\n    @staticmethod\n    def gcd(a, b) -> int:\n        \"\"\"Greatest common divisor.\n\n        The Greatest Common Divisor of a and b using Euclid's Algorithm.\n\n        Args:\n            a -> num 1\n            b -> num 2\n\n        Returns:\n            Returns  GCD(a, b)\n\n        \"\"\"\n        # Return\n        while a != 0:\n            a, b = b % a, a\n        return b\n\n    @staticmethod\n    def mod_inv(a: int, m: int) -> Optional[int]:\n        \"\"\"\n        Returns the modular inverse of a mod m, or None if it does not exist.\n\n        The modular inverse of a is the number a_inv that satisfies the equation\n        a_inv * a mod m === 1 mod m\n\n        Note: This is a naive implementation, and runtime may be improved in several ways.\n        For instance by checking if m is prime to perform a different calculation,\n        or by using the extended euclidean algorithm.\n        \"\"\"\n        for i in range(1, m):\n            if (m * i + 1) % a == 0:\n                return (m * i + 1) // a\n        return None\n\n    @staticmethod\n    def percentage(part: float, whole: float) -> float:\n        \"\"\"Returns percentage.\n\n        Just a normal algorithm to return the percent.\n\n        Args:\n            part -> part of the whole number\n            whole -> the whole number\n\n        Returns:\n            Returns the percentage of part to whole.\n\n        \"\"\"\n        if part <= 0 or whole <= 0:\n            return 0\n        # works with percentages\n        return 100 * float(part) / float(whole)\n\n    def sort_prob_table(self, prob_table: dict) -> dict:\n        \"\"\"Sorts the probability table.\n\n        Sorts a dictionary of dictionaries (and all the sub-dictionaries).\n\n        Args:\n            prob_table -> The probability table returned by the neural network to sort.\n\n        Returns:\n            Returns the prob_table, but sorted.\n\n        \"\"\"\n        # for each object: prob table in dictionary\n        max_overall: int = 0\n        max_dict_pair: dict = {}\n        highest_key = None\n        empty_dict: dict = {}\n        # sorts the prob table before we find max, and converts it to order dicts\n        for key, value in prob_table.items():\n            prob_table[key] = self.new_sort(value)\n            prob_table[key] = dict(prob_table[key])\n\n        # gets maximum key then sets it to the front\n        counter_max: int = 0\n        counter_prob: int = len(prob_table)\n        while counter_max < counter_prob:\n            max_overall = 0\n            highest_key = None\n            logging.debug(\n                f\"Running while loop in sort_prob_table, counterMax is {counter_max}\"\n            )\n            for key, value in prob_table.items():\n                logging.debug(f\"Sorting {key}\")\n                maxLocal = 0\n                # for each item in that table\n                for key2, value2 in value.items():\n                    logging.debug(\n                        f\"Running key2 {key2}, value2 {value2} for loop for {value.items()}\"\n                    )\n                    maxLocal = maxLocal + value2\n                    logging.debug(\n                        f\"MaxLocal is {maxLocal} and maxOverall is {max_overall}\"\n                    )\n                    if maxLocal > max_overall:\n                        logging.debug(f\"New max local found {maxLocal}\")\n                        # because the dict doesn't reset\n                        max_dict_pair = {}\n                        max_overall = maxLocal\n                        # so eventually, we get the maximum dict pairing?\n                        max_dict_pair[key] = value\n                        highest_key = key\n                        logging.debug(f\"Highest key is {highest_key}\")\n                # removes the highest key from the prob table\n            logging.debug(\n                f\"Prob table is {prob_table} and highest key is {highest_key}\"\n            )\n            logging.debug(f\"Removing {prob_table[highest_key]}\")\n            del prob_table[highest_key]\n            logging.debug(f\"Prob table after deletion is {prob_table}\")\n            counter_max += 1\n            empty_dict = {**empty_dict, **max_dict_pair}\n\n        # returns the max dict (at the start) with the prob table\n        # this way, it should always work on most likely first.\n        logging.debug(\n            f\"The prob table is {prob_table} and the maxDictPair is {max_dict_pair}\"\n        )\n        logging.debug(f\"The new sorted prob table is {empty_dict}\")\n        return empty_dict\n\n    @staticmethod\n    def new_sort(new_dict: dict) -> dict:\n        \"\"\"Uses OrderedDict to sort a dictionary.\n\n        I think it's faster than my implementation.\n\n        Args:\n            new_dict -> the dictionary to sort\n\n        Returns:\n            Returns the dict, but sorted.\n\n        \"\"\"\n        # (f\"d is {d}\")\n        logging.debug(f\"The old dictionary before new_sort() is {new_dict}\")\n        sorted_i = OrderedDict(\n            sorted(new_dict.items(), key=lambda x: x[1], reverse=True)\n        )\n        logging.debug(f\"The dictionary after new_sort() is {sorted_i}\")\n        # sortedI = sort_dictionary(x)\n        return sorted_i\n\n    @staticmethod\n    def is_ascii(s: str) -> bool:\n        \"\"\"Returns the boolean value if is_ascii is an ascii char.\n\n        Does what it says on the tree. Stolen from\n        https://stackoverflow.com/questions/196345/how-to-check-if-a-string-in-python-is-in-ascii\n\n        Args:\n            s -> the char to check.\n\n        Returns:\n            Returns the boolean of the char.\n\n        \"\"\"\n\n        return bool(lambda s: len(s) == len(s.encode()))\n\n    @staticmethod\n    def strip_punctuation(text: str) -> str:\n        \"\"\"Strips punctuation from a given string.\n\n        Uses string.punctuation.\n\n        Args:\n            text -> the text to strip punctuation from.\n\n        Returns:\n            Returns string without punctuation.\n        \"\"\"\n        text: str = (str(text).translate(str.maketrans(\"\", \"\", punctuation))).\nstrip(\n            \"\\n\"\n        )\n        return text\n\n# Path: ciphey/basemods/Checkers/brandon.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to determine whether something is English or not.\n1. Calculate the Chi Squared score of a sentence\n2. If the score is significantly lower than the average score, it _might_ be English\n    2.1. If the score _might_ be English, then take the text and compare it to the sorted dictionary\n    in O(n log n) time.\n    It creates a percentage of \"How much of this text is in the dictionary?\"\n    The dictionary contains:\n        * 20,000 most common US words\n        * 10,000 most common UK words (there's no repetition between the two)\n        * The top 10,000 passwords\n    If the word \"Looks like\" English (chi-squared) and if it contains English words, we can conclude it is\n    very likely English. The alternative is doing the dictionary thing but with an entire 479k word dictionary (slower)\n    2.2. If the score is not English, but we haven't tested enough to create an average, then test it against\n     the dictionary\n\nThings to optimise:\n* We only run the dictionary if it's 20% smaller than the average for chi squared\n* We consider it \"English\" if 45% of the text matches the dictionary\n* We run the dictionary if there is less than 10 total chisquared test\n\nHow to add a language:\n* Download your desired dictionary. Try to make it the most popular words, for example. Place this file into this\n folder with languagename.txt\nAs an example, this comes built in with english.txt\nFind the statistical frequency of each letter in that language.\nFor English, we have:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n}\nIn chisquared.py\nTo add your language, do:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n    \"German\": [0.0973]\n}\nIn alphabetical order\nAnd you're.... Done! Make sure the name of the two match up\n\"\"\"\nimport sys\nfrom math import ceil\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nsys.path.append(\"..\")\ntry:\n    import mathsHelper as mh\nexcept ModuleNotFoundError:\n    import ciphey.mathsHelper as mh\n\n\n@registry.register\nclass Brandon(Checker[str]):\n    \"\"\"\n    Class designed to confirm whether something is **language** based on how many words of **language** appears\n    Call confirmLanguage(text, language)\n    * text: the text you want to confirm\n    * language: the language you want to confirm\n\n    Find out what language it is by using chisquared.py, the highest chisquared score is the language\n    languageThreshold = 45\n    if a string is 45% **language** words, then it's confirmed to be english\n    \"\"\"\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually work this out\n        # TODO its 0.2 seconds on average\n        return 1e-4  # 100 \u00b5s\n\n    wordlist: set\n\n    def clean_text(self, text: str) -> set:\n        \"\"\"Cleans the text ready to be checked\n\n        Strips punctuation, makes it lower case, turns it into a set separated by spaces, removes duplicate words\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            text -> the text as a list, now cleaned\n\n        \"\"\"\n        # makes the text unique words and readable\n        text = text.lower()\n        text = self.mh.strip_punctuation(text)\n        text = text.split(\" \")\n        text = filter(lambda x: len(x) > 2, text)\n        text = set(text)\n        return text\n\n    def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        \"\"\"Given text determine if it passes checker\n\n        The checker uses the variable passed to it. I.E. Stopwords list, 1k words, dictionary\n\n        Args:\n            text -> The text to check\n            threshold -> at what point do we return True? The percentage of text that is in var before we return True\n            text_length -> the length of the text\n            var -> the variable we are checking against. Stopwords list, 1k words list, dictionary list.\n        Returns:\n            boolean -> True for it passes the test, False for it fails the test.\"\"\"\n        if text is None:\n            logging.debug(\"Checker's text is None, so returning False\")\n            return False\n        if var is None:\n            logging.debug(\"Checker's input var is None, so returning False\")\n            return False\n\n        percent = ceil(text_length * threshold)\n        logging.debug(f\"Checker's chunks are size {percent}\")\n        meet_threshold = 0\n        location = 0\n        end = percent\n\n        if text_length <= 0:\n            return False\n\n        while location <= text_length:\n            # chunks the text, so only gets THRESHOLD chunks of text at a time\n            text = list(text)\n            to_analyse = text[location:end]\n            logging.debug(f\"To analyse is {to_analyse}\")\n            for word in to_analyse:\n                # if word is a stopword, + 1 to the counter\n                if word in var:\n                    logging.debug(\n                        f\"{word} is in var, which means I am +=1 to the meet_threshold which is {meet_threshold}\"\n                    )\n                    meet_threshold += 1\n                meet_threshold_percent = meet_threshold / text_length\n                if meet_threshold_percent >= threshold:\n                    logging.debug(\n                        f\"Returning true since the percentage is {meet_threshold / text_length} and the threshold is {threshold}\"\n                    )\n                    # if we meet the threshold, return True\n                    # otherwise, go over again until we do\n                    # We do this in the for loop because if we're at 24% and THRESHOLD is 25\n                    # we don't want to wait THRESHOLD to return true, we want to return True ASAP\n                    return True\n            location = end\n            end = end + percent\n        logging.debug(\n            f\"The language proportion {meet_threshold_percent} is under the threshold {threshold}\"\n        )\n        return False\n\n    def __init__(self, config: Config):\n        # Suppresses warning\n        super().__init__(config)\n        self.mh = mh.mathsHelper()\n\n        phases = config.get_resource(self._params()[\"phases\"])\n\n        self.thresholds_phase1 = phases[\"1\"]\n        self.thresholds_phase2 = phases[\"2\"]\n        self.top1000Words = config.get_resource(self._params().get(\"top1000\"))\n        self.wordlist = config.get_resource(self._params()[\"wordlist\"])\n        self.stopwords = config.get_resource(self._params().get(\"stopwords\"))\n\n        self.len_phase1 = len(self.thresholds_phase1)\n        self.len_phase2 = len(self.thresholds_phase2)\n\n    def check(self, text: str) -> Optional[str]:\n        \"\"\"Checks to see if the text is in English\n\n        Performs a decryption, but mainly parses the internal data packet and prints useful information.\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            bool -> True if the text is English, False otherwise.\n\n        \"\"\"\n        logging.debug(f'In Language Checker with \"{text}\"')\n        text = self.clean_text(text)\n        logging.debug(f'Text split to \"{text}\"')\n        if text == \"\":\n            logging.debug(\"Returning None from Brandon as the text cleaned is none.\")\n            return None\n\n        length_text = len(text)\n\n        what_to_use = {}\n\n        # this code decides what checker / threshold to use\n        # if text is over or equal to maximum size, just use the maximum possible checker\n        what_to_use = self.calculateWhatChecker(\n            length_text, self.thresholds_phase1.keys()\n        )\n        logging.debug(self.thresholds_phase1)\n        what_to_use = self.thresholds_phase1[str(what_to_use)]\n        # def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        if \"check\" in what_to_use:\n            # perform check 1k words\n            result = self.checker(\n                text, what_to_use[\"check\"], length_text, self.top1000Words\n            )\n        elif \"stop\" in what_to_use:\n            # perform stopwords\n            result = self.checker(\n                text, what_to_use[\"stop\"], length_text, self.stopwords\n            )\n        elif \"dict\" in what_to_use:\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n            # If result is None, no point doing it again in phase2\n            if not result:\n                return None\n        else:\n            logging.info(f\"It is neither stop or check, but instead {what_to_use}\")\n\n        # return False if phase 1 fails\n        if not result:\n            return None\n        else:\n            what_to_use = self.calculateWhatChecker(\n                length_text, self.thresholds_phase2.keys()\n            )\n            what_to_use = self.thresholds_phase2[str(what_to_use)]\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n        return \"\" if result else None\n\n    def calculateWhatChecker(self, length_text, key):\n        \"\"\"Calculates what threshold / checker to use\n\n        If the length of the text is over the maximum sentence length, use the last checker / threshold\n        Otherwise, traverse the keys backwards until we find a key range that does not fit.\n        So we traverse backwards and see if the sentence length is between current - 1 and current\n        In this way, we find the absolute lowest checker / percentage threshold.\n        We traverse backwards because if the text is longer than the max sentence length, we already know.\n        In total, the keys are only 5 items long or so. It is not expensive to move backwards, nor is it expensive to move forwards.\n\n        Args:\n            length_text -> The length of the text\n            key -> What key we want to use. I.E. Phase1 keys, Phase2 keys.\n        Returns:\n            what_to_use -> the key of the lowest checker.\"\"\"\n\n        _keys = list(key)\n        _keys = list(map(int, _keys))\n        if length_text >= int(_keys[-1]):\n            what_to_use = list(key)[_keys.index(_keys[-1])]\n        else:\n            # this algorithm finds the smallest possible fit for the text\n            for counter, i in reversed(list(enumerate(_keys))):\n                #  [0, 110, 150]\n                if i <= length_text:\n                    what_to_use = i\n        return what_to_use\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"top1000\": ParamSpec(\n                desc=\"A wordlist of the top 1000 words\",\n                req=False,\n                default=\"cipheydists::list::english1000\",\n            ),\n            \"wordlist\": ParamSpec(\n                desc=\"A wordlist of all the words\",\n                req=False,\n                default=\"cipheydists::list::english\",\n            ),\n            \"stopwords\": ParamSpec(\n                desc=\"A wordlist of StopWords\",\n                req=False,\n                default=\"cipheydists::list::englishStopWords\",\n            ),\n            \"threshold\": ParamSpec(\n                desc=\"The minimum proportion (between 0 and 1) that must be in the dictionary\",\n                req=False,\n                default=0.45,\n            ),\n            \"phases\": ParamSpec(\n                desc=\"Language-specific phase thresholds\",\n                req=False,\n                default=\"cipheydists::brandon::english\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/format.py\nimport json\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass JsonChecker(Checker[str]):\n\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying json checker\")\n\n        # https://github.com/Ciphey/Ciphey/issues/389\n        if text.isdigit():\n            return None\n\n        try:\n            json.loads(text)\n            return \"\"\n        except ValueError:\n            return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 1e-7 * len(text)  # From benchmarks I found online\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/human.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, registry\nfrom rich.console import Console\nfrom rich.markup import escape\n\nconsole = Console()\n\n\n@registry.register\nclass HumanChecker(Checker[str]):\n\n    \"\"\"\n    Uses the person's decision to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: str) -> Optional[str]:\n        with self._config().pause_spinner_handle():\n            response = console.input(\n                f\"Possible plaintext: [blue bold]{escape(ctext.__repr__())}[/blue bold] ([green]y[/green]/[red]N[/red]): \"\n            )\n        if response == \"y\":\n            return \"\"\n        elif response in (\"n\", \"\"):\n            return None\n        else:\n            return self.check(ctext)\n\n    def getExpectedRuntime(self, text: str) -> float:\n        return 1  # About a second\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Checkers/quadgrams.py\nimport logging\nimport re\nfrom math import log10\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, Translation, registry\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Quadgrams(Checker[str]):\n\n    \"\"\"\n    Uses Quadgrams to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying Quadgrams checker\")\n        # Capitalize and remove everything that's not a letter\n        ctext = re.sub(\"[^A-Z]\", \"\", ctext.upper())\n        quadgrams = self.QUADGRAMS_DICT\n        quadgrams_sum = sum(quadgrams.values())\n        score = 0\n        for key in quadgrams.keys():\n            quadgrams[key] = float(quadgrams[key]) / quadgrams_sum\n        floor = log10(0.01 / quadgrams_sum)\n        for i in range(len(ctext) - 4 + 1):\n            # Get all quadgrams from ctext and check if they're in the dict\n            # If yes then add the score of those quadgrams to the total score\n            if ctext[i : i + 4] in quadgrams:\n                score += quadgrams[ctext[i : i + 4]]\n            else:\n                score += floor\n        if len(ctext) > 0:\n            score = score / len(ctext)\n        logging.info(f\"Quadgrams is {score}\")\n        # The default threshold was found to work the best from lots of testing\n        if score > self.threshold:\n            return \"\"\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The quadgrams dictionary to use\",\n                req=False,\n                default=\"cipheydists::dist::quadgrams\",\n            ),\n            \"score\": ParamSpec(\n                desc=\"The score threshold to use\",\n                req=False,\n                default=0.00011,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.QUADGRAMS_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.threshold = float(self._params()[\"score\"])\n\n# Path: ciphey/basemods/Checkers/regex.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Regex(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = list(map(re.compile, self._params()[\"regex\"]))\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"Passed with regex {regex}. Want to contribute to Ciphey? Submit your regex here to allow Ciphey to automatically get this next time https://github.com/bee-san/pyWhat/wiki/Adding-your-own-Regex\\n\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"regex\": ParamSpec(\n                req=True,\n                desc=\"The regex that must be matched (in a substring)\",\n                list=True,\n            )\n        }\n\n\n@registry.register\nclass RegexList(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = []\n        for i in self._params()[\"resource\"]:\n            self.regexes += [re.compile(regex) for regex in config.get_resource(i)]\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"passed with regex {regex}\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"resource\": ParamSpec(\n                req=True,\n                desc=\"A list of regexes that could be matched\",\n                list=True,\n            )\n        }\n\n# Path: ciphey/basemods/Checkers/what.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\nimport logging\nfrom rich.logging import RichHandler\nfrom pywhat import identifier\nfrom rich.console import Console\n\nconsole = Console()\n\n\n@registry.register\nclass What(Checker[str]):\n\n    \"\"\"\n    Uses PyWhat to determine plaintext with regexes\n    https://github.com/bee-san/pyWhat\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying PyWhat checker\")\n        returned_regexes = self.id.identify(ctext)\n        if returned_regexes[\"Regexes\"]:\n            matched_regex = returned_regexes[\"Regexes\"]['text'][0][\"Regex Pattern\"]\n\n            ret = f'The plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            human = (\n                f'\\nI think the plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            )\n\n            if \"Description\" in matched_regex and matched_regex[\"Description\"]:\n                s = matched_regex[\"Description\"]\n                # lowercases first letter so it doesn't look weird\n                s = f\", which is {s[0].lower() + s[1:]}\\n\"\n                ret += s\n                human += s\n\n            # if URL is attached, include that too.\n            if \"URL\" in matched_regex and matched_regex[\"URL\"]:\n                link = matched_regex[\"URL\"] + ctext.replace(\" \", \"\")\n                ret += f\"\\nClick here to view in browser [#CAE4F1][link={link}]{link}[/link][/#CAE4F1]\\n\"\n\n            # If greppable mode is on, don't print this\n            if self.config.verbosity >= 0:\n                # Print with full stop\n                console.print(human)\n            return ret\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.config = config\n        self.id = identifier.Identifier()\n\n# Path: ciphey/basemods/Checkers/ezcheck.py\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nfrom .brandon import Brandon\nfrom .format import JsonChecker\nfrom .human import HumanChecker\nfrom .quadgrams import Quadgrams\nfrom .regex import RegexList\nfrom .what import What\n\n\n@registry.register\nclass EzCheck(Checker[str]):\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns, followed by a human check\n    \"\"\"\n\n    def check(self, text: str) -> Optional[str]:\n        for checker in self.checkers:\n            res = checker.check(text)\n            if (\n                res is not None\n                and (self.decider is None or self.decider.check(text)) is not None\n            ):\n                return res\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        return sum(\n            i.getExpectedRuntime(text) for i in self.checkers\n        ) + self.decider.getExpectedRuntime(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        self.checkers: List[Checker[str]] = []\n        # Disable human checker for automated systems\n        if config.verbosity >= 0:\n            self.decider = config(HumanChecker)\n        else:\n            self.decider = None\n\n        # We need to modify the config for each of the objects\n\n        # First PyWhat, as it's the fastest\n        self.checkers.append(config(What))\n\n        # Next, the json checker\n        self.checkers.append(config(JsonChecker))\n\n        # Second to last, the quadgrams checker\n        self.checkers.append(config(Quadgrams))\n\n        # Finally, the Brandon checker, as it is the slowest\n        self.checkers.append(config(Brandon))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/quorum.py\nfrom typing import Dict, Generic, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, _registry\n\n\nclass Quorum(Generic[T], Checker[T]):\n    def check(self, text: T) -> Optional[str]:\n        left = self._params().k\n        results = []\n        for checker in self.checkers:\n            results.append(checker.check(text))\n            if results[-1] is None:\n                continue\n            left -= 1\n            # Early return check\n            if left == 0:\n                return str(results)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        if self._params().k is None:\n            k = len(self._params()[\"checker\"])\n        # These checks need to be separate, to make sure that we do not have zero members\n        if self._params().k == 0 or self._params().k > len(self._params()[\"checker\"]):\n            raise IndexError(\n                \"k must be between 0 and the number of checkers (inclusive)\"\n            )\n\n        self.checkers = []\n        for i in self._params()[\"checker\"]:\n            # This enforces type consistency\n            self.checkers.append(_registry.get_named(i, Checker[T]))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"checker\": ParamSpec(\n                req=True, desc=\"The checkers to be used for analysis\", list=True\n            ),\n            \"k\": ParamSpec(\n                req=False,\n                desc=\"The minimum quorum size. Defaults to the number of checkers\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/__init__.py\nfrom . import any, brandon, ezcheck, format, human, quadgrams, quorum, regex, what\n\n# Path: ciphey/common.py\n\"\"\"Some useful adapters\"\"\"\nfrom typing import Any\n\n\ndef id_lambda(value: Any):\n    \"\"\"\n    A function used in dynamic class generation that abstracts away a constant return value (like in getName)\n    \"\"\"\n    return lambda *args: value\n\n\ndef fix_case(target: str, base: str) -> str:\n    \"\"\"Returns the lower-case string target with the case of base\"\"\"\n    ret = \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n    return \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n\n# Path: ciphey/basemods/Crackers/affine.py\n# Community\n# by https://github.com/Ozzyz\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\nfrom ciphey.mathsHelper import mathsHelper\n\n\n@registry.register\nclass Affine(Cracker[str]):\n    \"\"\"\n    Each character in the Affine Cipher is encoded with the rule E(x) = (ax + b) mod m\n    m is the size of the alphabet, while a and b are the keys in the cipher. a must be coprime to b.\n    The Caesar cipher is a specific case of the Affine Cipher, with a=1 and b being the shift of the cipher.\n    Decryption is performed by D(x) = a_inv (x - b) mod m where a_inv is the modular multiplicative inverse of a mod m.\n\n    In this version of the Affine Cipher, we do not allow alphabets with several instances of the same letter in different cases.\n    For instance, the alphabet 'ABCdef123' is allowed, but 'AaBbCc' is not.\n    \"\"\"\n\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"affine\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Brute forces all the possible combinations of a and b to attempt to crack the cipher.\n        \"\"\"\n        logging.debug(\"Attempting affine\")\n        candidates = []\n\n        # a and b are coprime if gcd(a,b) is 1.\n        possible_a = [\n            a\n            for a in range(1, self.alphabet_length)\n            if mathsHelper.gcd(a, self.alphabet_length) == 1\n        ]\n        logging.info(\n            f\"Trying Affine Cracker with {len(possible_a)} a-values and {self.alphabet_length} b-values\"\n        )\n\n        for a in possible_a:\n            a_inv = mathsHelper.mod_inv(a, self.alphabet_length)\n            # If there is no inverse, we cannot decrypt the text\n            if a_inv is None:\n                continue\n            for b in range(self.alphabet_length):\n                # Pass in lowered text. This means that we expect alphabets to not contain both 'a' and 'A'.\n                translated = self.decrypt(ctext.lower(), a_inv, b, self.alphabet_length)\n\n                candidate_probability = self.plaintext_probability(translated)\n                if candidate_probability > self.plaintext_prob_threshold:\n                    candidates.append(\n                        CrackResult(\n                            value=fix_case(translated, ctext), key_info=f\"a={a}, b={b}\"\n                        )\n                    )\n        logging.info(f\"Affine Cipher returned {len(candidates)} candidates\")\n        return candidates\n\n    def plaintext_probability(self, translated: str) -> float:\n        \"\"\"\n        Analyses the translated text and applies the chi squared test to see if it is a probable plaintext candidate\n        Returns the probability of the chi-squared test.\n        \"\"\"\n        analysis = cipheycore.analyse_string(translated)\n        return cipheycore.chisq_test(analysis, self.expected)\n\n    def decrypt(self, text: str, a_inv: int, b: int, m: int) -> str:\n        \"\"\"\n        Each letter is decrypted at D(x) = a_inv (x - b) mod m where x is the char\n        We treat the char value as its index in the alphabet, so if\n        the alphabet is 'abcd....' and the char is 'b', it has the value 1.\n        \"\"\"\n        return \"\".join([self.decryptChar(char, a_inv, b, m) for char in text])\n\n    def decryptChar(self, char: str, a_inv: int, b: int, m: int) -> str:\n\n        # We lower the alphabet since both ctext and alphabet need to be in the same case in order\n        # to perform the shifts. The translated text will have fixed case after the translation anyways.\n        # This is only necessary if the specified alphabet is uppercase.\n        alphabet = [x.lower() for x in self.group]\n\n        # Preserve characters that are not in alphabet\n        if char not in alphabet:\n            return char\n        char_idx = alphabet.index(char)\n        decrypted_char_idx = (a_inv * (char_idx - b)) % m\n        return alphabet[decrypted_char_idx]\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.alphabet_length = len(self.group)\n        self.cache = config.cache\n        self.plaintext_prob_threshold = 0.01\n\n# Path: ciphey/basemods/Crackers/ascii_shift.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Ascii_shift(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"ascii_shift\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ASCII shift cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ASCII shift returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ASCII shift cipher alphabet\",\n                req=False,\n                default=\"\"\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/baconian.py\nimport re\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\nimport logging\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Baconian(Cracker[str]):\n...\n", "needle_token_start": 5601, "needle_token_end": 5616, "code_context_ntokens": 16082, "output": ["```python\n@staticmethod\ndef strip_punctuation(text: str) -> str:\n    \"\"\"Strips punctuation from a given string.\n\n    Uses string.punctuation.\n\n    Args:\n        text -> the text to strip punctuation from.\n\n    Returns:\n        Returns string without punctuation.\n    \"\"\"\n    text: str = (str(text).translate(str.maketrans(\"\", \"\", punctuation))).\nstrip(\n            \"\\n\"\n        )\n    return text\n```"]}
{"repo": "Ciphey/Ciphey", "name": "crackOne", "language": "python", "path": "ciphey/basemods/Crackers/vigenere.py", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to decrypt a cipher text that has been encrypted using a specific polyalphabetic substitution cipher method. It identifies potential decryption keys based on statistical analysis and returns a list of possible plaintexts.\n2. **Input**: The function takes three parameters: the cipher text as a string, a statistical analysis result of the cipher text, and the original cipher text for case correction purposes.\n3. **Output**: Outputs a list of decryption results, each containing the decrypted text, information about the decryption key used, and additional statistical information related to the decryption process.\n4. **Procedure**: The function performs the following steps:\n   - It calls a decryption analysis function to get potential decryption keys based on the provided analysis.\n   - Limits the number of potential keys to a predefined maximum.\n   - Decrypts the cipher text using each of the potential keys.\n   - Corrects the case of the decrypted text to match the original cipher text.\n   - Compiles the decryption results, including the decrypted text, key information, and statistical data, into a list.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/basemods/Crackers/ascii_shift.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Ascii_shift(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"ascii_shift\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ASCII shift cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ASCII shift returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ASCII shift cipher alphabet\",\n                req=False,\n                default=\"\"\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n...\n# Path: ciphey/basemods/Crackers/baconian.py\nimport re\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\nimport logging\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Baconian(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"baconian\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to decode both variants of the Baconian cipher.\n        \"\"\"\n        logging.debug(\"Attempting Baconian cracker\")\n        candidates = []\n        result = []\n        ctext_decoded = \"\"\n        ctext_decoded2 = \"\"\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext only contains A and B\n        if bool(re.search(r\"[^AB]\", ctext)) is True:\n            logging.debug(\"Failed to crack baconian due to non baconian character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 5\n        ctext_len = len(ctext)\n        if ctext_len % 5:\n            logging.debug(\n                f\"Failed to decode Baconian because length must be a multiple of 5, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 5\n        ctext = \" \".join(ctext[i : i + 5] for i in range(0, len(ctext), 5))\n        ctext_split = ctext.split(\" \")\n        baconian_keys = self.BACONIAN_DICT.keys()\n\n        # Decode I=J and U=V variant\n        for i in ctext_split:\n            if i in baconian_keys:\n                ctext_decoded += self.BACONIAN_DICT[i]\n\n        # Decode variant that assigns each letter a unique code\n        for i in ctext_split:\n            if \"+\" + i in baconian_keys:\n                ctext_decoded2 += self.BACONIAN_DICT[\"+\" + i]\n\n        candidates.append(ctext_decoded)\n        candidates.append(ctext_decoded2)\n        for candidate in candidates:\n            if candidate != \"\":\n                if candidate == candidates[0]:\n                    result.append(CrackResult(value=candidate, key_info=\"I=J & U=V\"))\n                else:\n                    result.append(CrackResult(value=candidate))\n        logging.debug(f\"Baconian cracker - Returning results: {result}\")\n        return result\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"dict\": ParamSpec(\n                desc=\"The Baconian alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::baconian\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BACONIAN_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n\n# Path: ciphey/basemods/Crackers/caesar.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Caesar(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"caesar\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying caesar cipher on {ctext}\")\n        # Convert it to lower case\n        #\n        # TODO: handle different alphabets\n        if self.lower:\n            message = ctext.lower()\n        else:\n            message = ctext\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"Caesar returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(message, candidate.key, self.group)\n            candidates.append(\n                CrackResult(value=fix_case(translated, ctext), key_info=candidate.key)\n            )\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the caesar cipher alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n            \"lower\": ParamSpec(\n                desc=\"Whether or not the ciphertext should be converted to lowercase first\",\n                req=False,\n                default=True,\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.lower: Union[str, bool] = self._params()[\"lower\"]\n        if not isinstance(self.lower, bool):\n            self.lower = util.strtobool(self.lower)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/rot47.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Rot47(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"rot47\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ROT47 cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ROT47 returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ROT47 cipher alphabet\",\n                req=False,\n                default=\"\"\"!\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/soundex.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\n\n\n@registry.register\nclass Soundex(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"soundex\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to crack Soundex by generating all possible combinations.\n        \"\"\"\n        logging.debug(\"Attempting Soundex cracker\")\n        word_list = []\n        sentences = []\n        result = []\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext contains only A-Z and 0-9\n        if bool(re.search(r\"[^A-Z0-9]\", ctext)) is True:\n            logging.debug(\"Failed to crack soundex due to non soundex character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 4\n        ctext_len = len(ctext)\n        if ctext_len % 4:\n            logging.debug(\n                f\"Failed to decode Soundex because length must be a multiple of 4, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 4\n        ctext = \" \".join(ctext[i : i + 4] for i in range(0, len(ctext), 4))\n        ctext_split = ctext.split(\" \")\n        soundex_keys = self.SOUNDEX_DICT.keys()\n\n        # Find all words that correspond to each given soundex code\n        for code in ctext_split:\n            if code in soundex_keys:\n                word_list.append(self.SOUNDEX_DICT[code])\n\n        logging.info(f\"Possible words for given encoded text: {word_list}\")\n\n        # Find all possible sentences\n        self.getSentenceCombo(\n            word_list,\n            sentences,\n            self.frequency_dict,\n            self.sentence_freq,\n            self.word_freq,\n        )\n\n        sorted_sentences = self.sortlistwithdict(sentences, self.frequency_dict)\n\n        for sentence in sorted_sentences:\n            result.append(CrackResult(value=sentence))\n\n        logging.debug(f\"Soundex cracker - Returning results: {result}\")\n        return result\n\n    def sortlistwithdict(self, listtosort, hashes):\n        \"\"\"\n        This function uses the sum of ranks (based on frequency) of each word in each\n        sentence and sorts them according to it.\n        \"\"\"\n        return sorted(listtosort, key=lambda x: hashes[x])\n\n    def getSentenceCombo(\n        self, A, sentences, frequency_dict, sentence_freq, word_freq, result=\"\", n=0\n    ):\n        \"\"\"\n        This function uses recursion to generate a list of sentences from all possible\n        words for a given set of soundex codes.\n        \"\"\"\n        logging.debug(\"Creating all possible sentences from Soundex\")\n        if n == len(A):\n            sentences.append(result[1:])\n            for word in result[1:].split():\n                # Adding the rank of each word to find out the sentence's net frequency\n                if word in word_freq:\n                    sentence_freq += word_freq.index(word)\n                # If the word isn't in the frequency list then it's a very uncommon word\n                # so we add a large number (5000)\n                else:\n                    sentence_freq += 5000\n            frequency_dict[result[1:]] = sentence_freq\n            sentence_freq = 0\n            return\n\n        for word in A[n]:\n            out = result + \" \" + word\n            self.getSentenceCombo(\n                A, sentences, frequency_dict, sentence_freq, word_freq, out, n + 1\n            )\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The Soundex dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::soundex\",\n            ),\n            \"freq\": ParamSpec(\n                desc=\"The word frequency dictionary to use\",\n                req=False,\n                default=\"cipheydists::list::English5000Freq\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.SOUNDEX_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.word_freq = config.get_resource(self._params()[\"freq\"], Translation)\n        self.frequency_dict = {}\n        self.sentence_freq = 0\n\n# Path: ciphey/basemods/Crackers/vigenere.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Vigenere(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        if self.keysize is not None:\n            analysis = self.cache.get_or_update(\n                ctext,\n                f\"vigenere::{self.keysize}\",\n                lambda: cipheycore.analyse_string(\n                    ctext.lower(), self.keysize, self.group\n                ),\n            )\n\n            val = cipheycore.vigenere_detect(analysis, self.expected)\n\n            logging.info(f\"Vigenere has likelihood {val}\")\n\n            return CrackInfo(\n                success_likelihood=val,\n                # TODO: actually calculate runtimes\n                success_runtime=1e-3,\n                failure_runtime=1e-2,\n            )\n\n        likely_lens = self.cache.get_or_update(\n            ctext,\n            \"vigenere::likely_lens\",\n            lambda: cipheycore.vigenere_likely_key_lens(\n                ctext.lower(), self.expected, self.group, self.detect_p_value\n            ),\n        )\n\n        # Filter out the lens that make no sense\n        likely_lens = [i for i in likely_lens if i.len <= self.max_key_length]\n\n        for keysize in likely_lens:\n            # Store the analysis\n            analysis = self.cache.get_or_update(\n                ctext, f\"vigenere::{keysize.len}\", lambda: keysize.tab\n            )\n        if len(likely_lens) == 0:\n            return CrackInfo(\n                success_likelihood=0,\n                # TODO: actually calculate runtimes\n                success_runtime=2e-3,\n                failure_runtime=2e-2,\n            )\n\n        logging.info(\n            f\"Vigenere has likelihood {likely_lens[0].p_value} with lens {[i.len for i in likely_lens]}\"\n        )\n\n        return CrackInfo(\n            success_likelihood=likely_lens[0].p_value,\n            # TODO: actually calculate runtimes\n            success_runtime=2e-4,\n            failure_runtime=2e-4,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"vigenere\"\n\n    def crackOne(\n        self, ctext: str, analysis: cipheycore.windowed_analysis_res, real_ctext: str\n    ) -> List[CrackResult]:\n        possible_keys = cipheycore.vigenere_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n        if len(possible_keys) > self.clamp:\n            possible_keys = possible_keys[: self.clamp]\n  \n      logging.debug(\n            f\"Vigenere crack got keys: {[[i for i in candidate.key] for candidate in possible_keys]}\"\n        )\n        return [\n            CrackResult(\n                value=fix_case(\n                    cipheycore.vigenere_decrypt(ctext, candidate.key, self.group),\n                    real_ctext,\n                ),\n                key_info=\"\".join([self.group[i] for i in candidate.key]),\n                misc_info=f\"p-value was {candidate.p_value}\",\n            )\n            for candidate in possible_keys[: min(len(possible_keys), 10)]\n        ]\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(\"Trying vigenere cipher\")\n        # Convert it to lower case\n        if self.lower:\n            message = ctext.lower()\n        else:\n            message = ctext\n\n        # Analysis must be done here, where we know the case for the cache\n        if self.keysize is not None:\n            return self.crackOne(\n                message,\n                self.cache.get_or_update(\n                    ctext,\n                    f\"vigenere::{self.keysize}\",\n                    lambda: cipheycore.analyse_string(\n                        message, self.keysize, self.group\n                    ),\n                ),\n                ctext,\n            )\n\n        arrs = []\n        likely_lens = self.cache.get_or_update(\n            ctext,\n            \"vigenere::likely_lens\",\n            lambda: cipheycore.vigenere_likely_key_lens(\n                message, self.expected, self.group\n            ),\n        )\n        possible_lens = [i for i in likely_lens]\n        possible_lens.sort(key=lambda i: i.p_value)\n        logging.debug(f\"Got possible lengths {[i.len for i in likely_lens]}\")\n        # TODO: work out length\n        for i in possible_lens:\n            arrs.extend(\n                self.crackOne(\n                    message,\n                    self.cache.get_or_update(\n                        ctext,\n                        f\"vigenere::{i.len}\",\n                        lambda: cipheycore.analyse_string(message, i.len, self.group),\n                    ),\n                    ctext,\n                )\n            )\n\n        logging.info(f\"Vigenere returned {len(arrs)} candidates\")\n        return arrs\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the caesar cipher alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n            \"lower\": ParamSpec(\n                desc=\"Whether or not the ciphertext should be converted to lowercase first\",\n                req=False,\n                default=True,\n            ),\n            \"keysize\": ParamSpec(\n                desc=\"A key size that should be used. If not given, will attempt to work it out\",\n                req=False,\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for windowed frequency analysis\",\n                req=False,\n                default=0.5,\n            ),\n            \"detect_p_value\": ParamSpec(\n                desc=\"The p-value to use for the detection of Vigenere length\",\n                req=False,\n                default=0.01,\n            ),\n            \"clamp\": ParamSpec(\n                desc=\"The maximum number of candidates that can be returned per key len\",\n                req=False,\n                default=10,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.lower: Union[str, bool] = self._params()[\"lower\"]\n        if not isinstance(self.lower, bool):\n            self.lower = util.strtobool(self.lower)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.keysize = self._params().get(\"keysize\")\n        if self.keysize is not None:\n            self.keysize = int(self.keysize)\n        self.p_value = float(self._params()[\"p_value\"])\n        self.detect_p_value = float(self._params()[\"detect_p_value\"])\n        self.clamp = int(self._params()[\"clamp\"])\n        self.max_key_length = 16\n\n# Path: ciphey/basemods/Crackers/xandy.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Xandy(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def binary_to_ascii(variant):\n        # Convert the binary string to an integer with base 2\n        binary_int = int(variant, 2)\n        byte_number = binary_int.bit_length() + 7 // 8\n\n        # Convert the resulting int to a bytearray and then decode it to ASCII text\n        binary_array = binary_int.to_bytes(byte_number, \"big\")\n        try:\n            ascii_text = binary_array.decode()\n            logging.debug(f\"Found possible solution: {ascii_text[:32]}\")\n            return ascii_text\n        except UnicodeDecodeError as e:\n            logging.debug(f\"Failed to crack X-Y due to a UnicodeDecodeError: {e}\")\n            return \"\"\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"xandy\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Checks an input if it only consists of two or three different letters.\n        If this is the case, it attempts to regard those letters as\n        0 and 1 (with the third characters as an optional delimiter) and then\n        converts it to ASCII text.\n        \"\"\"\n        logging.debug(\"Attempting X-Y replacement\")\n        variants = []\n        candidates = []\n        result = []\n\n        # Convert the ctext to all-lowercase and regex-match & replace all whitespace\n        ctext = re.sub(r\"\\s+\", \"\", ctext.lower(), flags=re.UNICODE)\n\n        # cset contains every unique value in the ctext\n        cset = list(set(list(ctext)))\n        cset_len = len(cset)\n\n        if not 1 < cset_len < 4:\n            # We only consider inputs with two or three unique values\n            logging.debug(\n                \"Failed to crack X-Y due to not containing two or three unique values\"\n            )\n            return None\n\n        logging.debug(f\"String contains {cset_len} unique values: {cset}\")\n\n        # In case of three unique values, we regard the least frequent character as the delimiter\n        if cset_len == 3:\n            # Count each unique character in the set to determine the least frequent one\n            counting_list = []\n            for char in cset:\n                counting_list.append(ctext.count(char))\n            val, index = min((val, index) for (index, val) in enumerate(counting_list))\n            delimiter = cset[index]\n            logging.debug(\n                f\"{delimiter} occurs {val} times and is the probable delimiter\"\n            )\n            # Remove the delimiter from the ctext and compute new cset\n            ctext = ctext.replace(delimiter, \"\")\n            cset = list(set(list(ctext)))\n\n        # Form both variants of the substitution\n        for i in range(2):\n            if i:\n                variants.append(ctext.replace(cset[0], \"1\").replace(cset[1], \"0\"))\n            else:\n                variants.append(ctext.replace(cset[0], \"0\").replace(cset[1], \"1\"))\n\n        # Apply function to both variants and strip stray NULL characters\n        for variant in variants:\n            candidates.append(self.binary_to_ascii(variant).strip(\"\\x00\"))\n        for i, candidate in enumerate(candidates):\n            if candidate != \"\":\n                keyinfo = f\"{cset[0]} -> {i} & {cset[1]} -> {str(int(not i))}\"\n                result.append(CrackResult(value=candidate, key_info=keyinfo))\n                logging.debug(f\"X-Y cracker - Returning results: {result}\")\n                return result\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            )\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n\n# Path: ciphey/basemods/Crackers/xortool.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: bee-san\n\"\"\"\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom xortool_ciphey import tool_main\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass XorTool(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            # TODO: actually calculate runtimes\n            success_runtime=1e-8,\n            failure_runtime=1e-8,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"xortool\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.debug(\"Trying xortool cipher\")\n        # TODO handle different charsets\n        # TODO allow more config over xortool\n\n        logging.debug(f\"{ctext}\")\n\n        # https://github.com/Ciphey/xortool/discussions/4\n        # for docs on this function\n        try:\n            result = tool_main.api(str.encode(ctext))\n        except:\n            logging.debug(\"Xor failed.\")\n            return\n\n        result = CrackResult(value=result[1][\"Dexored\"], key_info=result[0][\"keys\"])\n\n        return [result]\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            ),\n        }\n\n    @staticmethod\n    def score_utility() -> float:\n        return 1.5\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = self._params()[\"p_value\"]\n\n# Path: ciphey/basemods/Crackers/__init__.py\nfrom . import (\n    affine,\n    ascii_shift,\n    baconian,\n    caesar,\n    rot47,\n    soundex,\n    vigenere,\n    xandy,\n    xortool,\n)\n\n# Path: ciphey/basemods/Decoders/a1z26.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass A1z26(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs A1Z26 decoding\n        \"\"\"\n        logging.debug(\"Attempting A1Z26\")\n        ctext_converted = []\n        ctext_split = re.split(r\"[ ,;:\\-\\n]\", ctext)\n        delimiters = set(sorted(re.sub(r\"[^ ,;:\\-\\n]\", \"\", ctext)))\n        ctext_num = re.sub(r\"[,;:\\-\\s]\", \"\", ctext)\n        ctext_decoded = \"\"\n        if ctext_num.isnumeric() is False:\n            logging.debug(\"Failed to decode A1Z26 due to non numeric character(s)\")\n            return None\n        try:\n            for i in ctext_split:\n                val = int(i)\n                if val > 26 or val < 1:\n                    logging.debug(\n                        f\"Failed to decode A1Z26 due to invalid number '{val}'\"\n                    )\n                    return None\n                val2 = int(i) + 96\n                ctext_converted.append(chr(val2))\n            ctext_decoded = \"\".join(ctext_converted)\n            logging.info(\n                f\"A1Z26 successful, returning '{ctext_decoded}' with delimiter(s) {delimiters}\"\n            )\n            return ctext_decoded\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"a1z26\"\n\n# Path: ciphey/basemods/Decoders/atbash.py\nfrom typing import Dict, Optional\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, WordList, registry\n\n\n@registry.register\nclass Atbash(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Takes an encoded string and attempts to decode it according to the Atbash cipher.\n\n        The Atbash cipher is a very simple substitution cipher without a key.\n        It operates by replacing every letter in the input by its 'counterpoint'\n        in the alphabet. Example: A -> Z, B -> Y, ... , M -> N and vice versa.\n        \"\"\"\n\n        result = \"\"\n        atbash_dict = {self.ALPHABET[i]: self.ALPHABET[::-1][i] for i in range(26)}\n\n        for letter in ctext.lower():\n            if letter in atbash_dict.keys():\n                # Match every letter of the input to its atbash counterpoint\n                result += atbash_dict[letter]\n            else:\n                # If the current character is not in the defined alphabet,\n                # just accept it as-is (useful for numbers, punctuation, etc.)\n                result += letter\n        return fix_case(result, ctext)\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.1\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.ALPHABET = config.get_resource(self._params()[\"dict\"], WordList)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The alphabet used for the atbash operation.\",\n                req=False,\n                default=\"cipheydists::list::englishAlphabet\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"atbash\"\n\n# Path: ciphey/basemods/Decoders/base58_bitcoin.py\nfrom typing import Dict, Optional\n\nimport base58\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base58_bitcoin(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base58 (Bitcoin) decoding\n        \"\"\"\n        try:\n            return base58.b58decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base58_bitcoin\"\n\n# Path: ciphey/basemods/Decoders/base58_ripple.py\nfrom typing import Dict, Optional\n\nimport base58\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base58_ripple(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base58 (Ripple) decoding\n        \"\"\"\n        try:\n            return base58.b58decode(ctext, alphabet=base58.RIPPLE_ALPHABET).decode(\n                \"utf-8\"\n            )\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base58_ripple\"\n\n# Path: ciphey/basemods/Decoders/base62.py\nfrom typing import Dict, Optional\n\nimport base62\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base62(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base62 decoding\n        \"\"\"\n        try:\n            return base62.decodebytes(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base62\"\n\n# Path: ciphey/basemods/Decoders/base69.py\n# Translated to Python and adapted for Ciphey from the JS original at https://github.com/pshihn/base69\n\n\nimport re\nfrom math import ceil\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, WordList, registry\n\n\n@registry.register\nclass Base69(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base69 decoding\n        \"\"\"\n        # Remove whitespace\n        try:\n            ctext = re.sub(r\"\\s+\", \"\", ctext, flags=re.UNICODE)\n            extra_bytes = 0\n            clen = len(ctext)\n\n            if ctext[:-1] == \"=\":\n                extra_bytes = int(ctext[clen - 2])\n\n            CHUNK_COUNT = ceil(clen / 16)\n            result = [0 for _ in range(CHUNK_COUNT * 7 - extra_bytes)]\n\n            for i in range(CHUNK_COUNT):\n                chunk_string = ctext[i * 16 : (i + 1) * 16]\n                if extra_bytes and (i == CHUNK_COUNT - 1):\n                    insert = self.decode_chunk(chunk_string)\n                    for n, elem in enumerate(insert[0 : 7 - extra_bytes]):\n                        result[n + i * 7] = elem\n                else:\n                    insert = self.decode_chunk(chunk_string)\n                    for n, elem in enumerate(insert):\n                        result[n + i * 7] = elem % 256\n            return bytearray(result).decode().strip(\"\\x00\")\n        except Exception:\n            return None\n\n    def decode_chunk(self, s: str):\n        padded_bytes = s.endswith(\"=\")\n\n        decoded = [0 for _ in range(8)]\n        for i in range(8):\n            decoded[i] = (\n                0\n                if i == 7 and padded_bytes\n                else self.chars_to_byte(s[i * 2 : i * 2 + 2])\n            )\n\n        result = [0 for _ in range(7)]\n        for i in range(7):\n            t1 = decoded[i] << (i + 1)\n            t2 = decoded[i + 1] >> (7 - i - 1)\n            result[i] = t1 | t2\n        return result\n\n    def chars_to_byte(self, s: str):\n        return (69 * self.CHARS.index(s[1])) + (self.CHARS.index(s[0]))\n\n    @staticmethod\n    def priority() -> float:\n        # If this becomes lower or equal to the reverse, it breaks.\n        # So I'll set it to 0.2 for now since it is very fast anyways.\n        return 0.2\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.CHARS = config.get_resource(self._params()[\"dict\"], WordList)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The charset used for the decoder.\",\n                req=False,\n                default=\"cipheydists::list::base69\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base69\"\n\n# Path: ciphey/basemods/Decoders/base91.py\nfrom typing import Dict, Optional\n\nimport base91\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base91(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base91 decoding\n        \"\"\"\n        try:\n            return base91.decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base91\"\n\n# Path: ciphey/basemods/Decoders/bases.py\nimport base64\nimport types\nfrom typing import Any, Callable, Optional\n\nimport logging\nfrom rich.logging import RichHandler\nimport re\n\nfrom ciphey.common import id_lambda\nfrom ciphey.iface import Decoder, registry\n\n\ndef _dispatch(self: Any, ctext: str, func: Callable[[str], bytes]) -> Optional[bytes]:\n    logging.debug(f\"Attempting {self.getTarget()}\")\n\n    try:\n        # remove all whitespace\n        ctext = re.sub(r\"\\s+\", \"\", ctext, re.UNICODE)\n        result = func(ctext)\n        logging.info(f\"{self.getTarget()} successful, returning {result}\")\n        return result\n    except ValueError:\n        logging.debug(f\"Failed to decode {self.getTarget()}\")\n        return None\n\n\n_bases = {\n    \"base16\": (base64.b16decode, 0.4),\n    \"base32\": (base64.b32decode, 0.01),\n    \"base64\": (base64.b64decode, 0.4),\n    \"base85\": (base64.b85decode, 0.01),\n    \"ascii85\": (base64.a85decode, 0.1),\n}\n\n\ndef gen_class(name, decoder, priority, ns):\n    ns[\"_get_func\"] = id_lambda(decoder)\n    ns[\"decode\"] = lambda self, ctext: _dispatch(self, ctext, self._get_func())\n    ns[\"getParams\"] = id_lambda(None)\n    ns[\"getTarget\"] = id_lambda(name)\n    ns[\"priority\"] = id_lambda(priority)\n    ns[\"__init__\"] = lambda self, config: super(type(self), self).__init__(config)\n\n\nfor name, (decoder, priority) in _bases.items():\n    t = types.new_class(\n        name,\n        (Decoder[str],),\n        exec_body=lambda x: gen_class(name, decoder, priority, x),\n    )\n\n    registry.register(t)\n\n# Path: ciphey/basemods/Decoders/baudot.py\nimport re\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Baudot(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        result = \"\"\n        switch_to_digit_map = 0\n        if re.search(\"^[01]{5}$\", ctext.split()[0]):\n            for i in ctext.split():\n                if i == \"11011\":\n                    switch_to_digit_map = 1\n                if i == \"11111\":\n                    switch_to_digit_map = 0\n                if switch_to_digit_map == 1:\n                    result += self.BAUDOT_DICT[\"+\" + i]\n                if switch_to_digit_map == 0:\n                    result += self.BAUDOT_DICT[i]\n            return result\n        else:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BAUDOT_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The baudot alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::baudot\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"baudot\"\n\n# Path: ciphey/basemods/Decoders/binary.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Binary(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        try:\n            ctext = re.sub(r\"[^\\S \\n]\", \" \", ctext, flags=re.UNICODE)\n            ctext = ctext.replace(\"\\n\", \" \")\n\n            existing_split = self.try_split(ctext.split(\" \"))\n            if existing_split is not None:\n                return existing_split\n\n            # Now we try our own grouping\n\n            # Remove final bit of whitespace\n            ctext = ctext.replace(\" \", \"\")\n            # Split into bytes, and test\n            return self.try_split([ctext[i : i + 8] for i in range(0, len(ctext), 8)])\n        # Catch bad octal chars\n        except ValueError:\n            return None\n\n    def try_split(self, split_text: List[str]):\n        ret = []\n\n        for i in split_text:\n            if len(i) == 0:\n                continue\n            val = int(i, 2)\n            if val > 255 or val < 0:\n                return None\n            ret.append(val)\n\n        if len(ret) != 0:\n            ret = bytes(ret)\n            logging.info(f\"binary successful, returning {ret.__repr__()}\")\n            return ret\n\n    @staticmethod\n    def priority() -> float:\n        return 0.3\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"binary\"\n\n# Path: ciphey/basemods/Decoders/braille.py\nimport re\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\nimport logging\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Braille(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Braille decoding\n        \"\"\"\n        logging.debug(\"Attempting Braille\")\n        ctext_decoded = \"\"\n        braille_matches = 0\n        for symbol in self.BRAILLE_DICT_INV.values():\n            if symbol in ctext:\n                braille_matches += 1\n            else:\n                continue\n        if braille_matches == 0:\n            logging.debug(\"Failed to decode Braille due to invalid characters\")\n            return None\n\n        for pattern, value in self.BRAILLE_DICT.items():\n            ctext = re.sub(pattern, value, ctext)\n\n        wordArr = []\n        for word in ctext.split(\" \"):\n            # If two commas are in front of a word, uppercase the word and remove the comma\n            if word[:2].find(\",,\") != -1:\n                wordArr.append(word.replace(\",,\", \"\").upper())\n            else:\n                wordArr.append(word)\n\n        result = []\n        for word in wordArr:\n            # If one comma is in front of a word, capitalize the word and remove the comma\n            if word[0].find(\",\") != -1:\n                result.append(word.replace(\",\", \"\").capitalize())\n            else:\n                result.append(word)\n        ctext_decoded = \" \".join(result)\n        logging.info(f\"Braille successful, returning '{ctext_decoded}'\")\n        return ctext_decoded\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BRAILLE_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.BRAILLE_DICT_INV = {v: k for k, v in self.BRAILLE_DICT.items()}\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The Braille dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::braille\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"braille\"\n\n# Path: ciphey/basemods/Decoders/brainfuck.py\nimport re\nimport time\nfrom typing import Dict, Optional, Tuple\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, WordList, registry\n\n\n@registry.register\nclass Brainfuck(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Takes a ciphertext and treats it as a Brainfuck program,\n        interpreting it and saving the output as a string to return.\n\n        Brainfuck is a very simple, Turing-complete esoteric language.\n        Below is a simplified interpreter that attempts to check whether a\n        given ciphertext is a brainfuck program that would output a string.\n\n        A program that can be \"decoded\" like this is one that:\n            * Does not require user input (\",\" instruction)\n            * Includes at least one putchar instruction (\".\")\n            * Does not contain anything but the main 7 instructions,\n                (excluding \",\") and whitespace\n\n        Details:\n            * This implementation wraps the memory pointer for \">\" and \"<\"\n            * It is time-limited to 60 seconds, to prevent hangups\n            * The program starts with 100 memory cells, chosen arbitrarily\n        \"\"\"\n\n        logging.debug(\"Attempting brainfuck\")\n\n        result = \"\"\n        memory = [0] * 100\n        codeptr, memptr = 0, 0  # Instruction pointer and stack pointer\n        timelimit = 60  # The timeout in seconds\n\n        bracemap, isbf = self.bracemap_and_check(ctext)\n\n        # If it doesn't appear to be valid brainfuck code\n        if not isbf:\n            logging.debug(\"Failed to interpret brainfuck due to invalid characters\")\n            return None\n...\n", "needle_token_start": 7123, "needle_token_end": 7385, "code_context_ntokens": 16094, "output": ["```python\ndef crackOne(self, ctext: str, analysis: cipheycore.windowed_analysis_res, real_ctext: str) -> List[CrackResult]:\n    possible_keys = cipheycore.vigenere_crack(\n        analysis, self.expected, self.group, self.p_value\n    )\n    if len(possible_keys) > self.clamp:\n        possible_keys = possible_keys[: self.clamp]\n  \n    logging.debug(\n        f\"Vigenere crack got keys: {[[i for i in candidate.key] for candidate in possible_keys]}\"\n    )\n    return [\n        CrackResult(\n            value=fix_case(\n                cipheycore.vigenere_decrypt(ctext, candidate.key, self.group),\n                real_ctext,\n            ),\n            key_info=\"\".join([self.group[i] for i in candidate.key]),\n            misc_info=f\"p-value was {candidate.p_value}\",\n        )\n        for candidate in possible_keys[: min(len(possible_keys), 10)]\n    ]\n```"]}
{"repo": "Ciphey/Ciphey", "name": "new_sort", "language": "python", "path": "ciphey/mathsHelper.py", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To organize the elements of a dictionary in descending order based on their values.\n2. **Input**: A dictionary where keys are associated with numerical values.\n3. **Output**: A dictionary sorted in descending order of the values, maintaining the order of keys with equal values as in the original dictionary.\n4. **Procedure**: The function takes a dictionary, sorts it based on the values in descending order using a lambda function as the sorting key, and returns the sorted dictionary as an ordered dictionary to preserve the order of insertion.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/iface/_modules.py\nfrom abc import ABC, abstractmethod\nfrom typing import Any, Dict, Generic, List, NamedTuple, Optional, Set, Type, TypeVar\n\nfrom rich import box\nfrom rich.console import Console\nfrom rich.markup import escape\nfrom rich.table import Table\n\nfrom ._fwd import config as Config\n\nT = TypeVar(\"T\")\nU = TypeVar(\"U\")\n\nconsole = Console()\n\n\nclass ParamSpec(NamedTuple):\n    \"\"\"\n    Attributes:\n        req         Whether this argument is required\n        desc        A description of what this argument does\n        default     The default value for this argument. Ignored if req == True or configPath is not None\n        config_ref  The path to the config that should be the default value\n        list        Whether this parameter is in the form of a list, and can therefore be specified more than once\n        visible     Whether the user can tweak this via the command line\n    \"\"\"\n\n    req: bool\n    desc: str\n    default: Optional[Any] = None\n    list: bool = False\n    config_ref: Optional[List[str]] = None\n    visible: bool = True\n\n\nclass ConfigurableModule(ABC):\n    @staticmethod\n    @abstractmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        \"\"\"\n        Returns a dictionary of `argument name: argument specification`\n        \"\"\"\n        pass\n\n    def _checkParams(self):\n        \"\"\"\n        Fills the given params dict with default values where arguments are not given,\n        using None as the default value for default values\n        \"\"\"\n\n        params = self._params()\n        config = self._config()\n\n        for key, value in self.getParams().items():\n            # If we already have it, then we don't need to do anything\n            if key in params:\n                continue\n            # If we don't have it, but it's required, then fail\n            if value.req:\n                raise KeyError(\n                    f\"Missing required param {key} for {type(self).__name__.lower()}\"\n                )\n            # If it's a reference by default, fill that in\n            if value.config_ref is not None:\n                tmp = getattr(config, value.config_ref[0])\n                params[key] = (\n                    tmp[value.config_ref[1:]] if len(value.config_ref) > 1 else tmp\n                )\n            # Otherwise, put in the default value (if it exists)\n            elif value.default is not None:\n                params[key] = value.default\n\n    def _params(self):\n        return self._params_obj\n\n    def _config(self):\n        return self._config_obj\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        self._config_obj = config\n        if self.getParams() is not None:\n            self._params_obj = config.params.setdefault(type(self).__name__.lower(), {})\n            self._checkParams()\n\n\nclass Targeted(ABC):\n    @staticmethod\n    @abstractmethod\n    def getTarget() -> str:\n        \"\"\"Should return the target that this object attacks/decodes\"\"\"\n        pass\n\n\nclass PolymorphicChecker(ConfigurableModule):\n    @abstractmethod\n    def check(self, text) -> Optional[str]:\n        \"\"\"Should return some description (or an empty string) on success, otherwise return None\"\"\"\n        pass\n\n    @abstractmethod\n    def getExpectedRuntime(self, text) -> float:\n        pass\n\n    def __call__(self, *args):\n        return self.check(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass Checker(Generic[T], ConfigurableModule):\n    @abstractmethod\n    def check(self, text: T) -> Optional[str]:\n        \"\"\"Should return some description (or an empty string) on success, otherwise return None\"\"\"\n        pass\n\n    @abstractmethod\n    def getExpectedRuntime(self, text: T) -> float:\n        pass\n\n    def __call__(self, *args):\n        return self.check(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @classmethod\n    def convert(cls, expected: Set[type]):\n        class PolyWrapperClass(PolymorphicChecker):\n            @staticmethod\n            def getParams() -> Optional[Dict[str, ParamSpec]]:\n                return cls.getParams()\n\n            def check(self, text) -> Optional[str]:\n                \"\"\"Should return some description (or an empty string) on success, otherwise return None\"\"\"\n                if type(text) not in expected:\n                    return None\n                else:\n                    return self._base.check(text)\n\n            def getExpectedRuntime(self, text) -> float:\n                if type(text) not in expected:\n                    return 0\n                else:\n                    return self._base.getExpectedRuntime(text)\n\n            def __init__(self, config: Config):\n                super().__init__(config)\n                # This is easier than inheritance\n                self._base = cls(config)\n\n        PolyWrapperClass.__name__ = cls.__name__\n\n        return PolyWrapperClass\n\n\n# class Detector(Generic[T], ConfigurableModule, KnownUtility, Targeted):\n#     @abstractmethod\n#     def scoreLikelihood(self, ctext: T) -> Dict[str, float]:\n#         \"\"\"Should return a dictionary of (cipher_name: score)\"\"\"\n#         pass\n#\n#     def __call__(self, *args): return self.scoreLikelihood(*args)\n#\n#     @abstractmethod\n#     def __init__(self, config: Config): super().__init__(config)\n\n\nclass Decoder(Generic[T], ConfigurableModule, Targeted):\n    \"\"\"Represents the undoing of some encoding into a different (or the same) type\"\"\"\n\n    @abstractmethod\n    def decode(self, ctext: T) -> Optional[U]:\n        pass\n\n    @staticmethod\n    @abstractmethod\n    def priority() -> float:\n        \"\"\"What proportion of decodings are this?\"\"\"\n        pass\n\n    def __call__(self, *args):\n        return self.decode(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass DecoderComparer:\n    value: Type[Decoder]\n\n    def __le__(self, other: \"DecoderComparer\"):\n        return self.value.priority() <= other.value.priority()\n\n    def __ge__(self, other: \"DecoderComparer\"):\n        return self.value.priority() >= other.value.priority()\n\n    def __lt__(self, other: \"DecoderComparer\"):\n        return self.value.priority() < other.value.priority() and self != other\n\n    def __gt__(self, other: \"DecoderComparer\"):\n        return self.value.priority() > other.value.priority() and self != other\n\n    def __init__(self, value: Type[Decoder]):\n        self.value = value\n\n    def __repr__(self):\n        return f\"<DecoderComparer {self.value}:{self.value.priority()}>\"\n\n\nclass CrackResult(NamedTuple):\n    # TODO consider using Generic[T] again for value's type once\n    # https://bugs.python.org/issue36517 is resolved\n    value: Any\n    key_info: Optional[str] = None\n    misc_info: Optional[str] = None\n\n\nclass CrackInfo(NamedTuple):\n    success_likelihood: float\n    success_runtime: float\n    failure_runtime: float\n\n\nclass Cracker(Generic[T], ConfigurableModule, Targeted):\n    @abstractmethod\n    def getInfo(self, ctext: T) -> CrackInfo:\n        \"\"\"Should return some informed guesses on resource consumption when run on `ctext`\"\"\"\n        pass\n\n    @abstractmethod\n    def attemptCrack(self, ctext: T) -> List[CrackResult]:\n        \"\"\"\n        This should attempt to crack the cipher `target`, and return a list of candidate solutions\n        \"\"\"\n        # FIXME: Actually CrackResult[T], but python complains\n        pass\n\n    def __call__(self, *args):\n        return self.attemptCrack(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass ResourceLoader(Generic[T], ConfigurableModule):\n    @abstractmethod\n    def whatResources(self) -> Optional[Set[str]]:\n        \"\"\"\n        Return a set of the names of instances T you can provide.\n        The names SHOULD be unique amongst ResourceLoaders of the same type\n\n        These names will be exposed as f\"{self.__name__}::{name}\", use split_resource_name to recover this\n\n        If you cannot reasonably determine what resources you provide, return None instead\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def getResource(self, name: str) -> T:\n        \"\"\"\n        Returns the requested distribution\n\n        The behavior is undefined if `name not in self.what_resources()`\n        \"\"\"\n        pass\n\n    def __call__(self, *args):\n        return self.getResource(*args)\n\n    def __getitem__(self, *args):\n        return self.getResource(*args)\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\nclass SearchLevel(NamedTuple):\n    name: str\n    result: CrackResult\n\n    @staticmethod\n    def input(ctext: Any):\n        return SearchLevel(name=\"input\", result=CrackResult(ctext))\n\n\nclass SearchResult(NamedTuple):\n    path: List[SearchLevel]\n    check_res: str\n\n\nclass Searcher(ConfigurableModule):\n    \"\"\"A very basic interface for code that plans out how to crack the ciphertext\"\"\"\n\n    @abstractmethod\n    def search(self, ctext: Any) -> Optional[SearchResult]:\n        \"\"\"Returns the path to the correct ciphertext\"\"\"\n        pass\n\n    @abstractmethod\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n\ndef pretty_search_results(res: SearchResult, display_intermediate: bool = False) -> str:\n    # TODO what is display_intermediate\n    ret: str = \"\"\n    table = Table(show_header=False, box=box.ROUNDED, safe_box=False)\n    # Only print the checker if we need to. Normal people don't know what\n    # \"quadgrams\", \"brandon\", \"json checker\" is.\n    # We print the checker if its regex or another language, so long as it starts with:\n    # \"The\" like \"The plaintext is a Uniform Resource Locator (URL).\"\n...\n# Path: ciphey/iface/_config.py\nimport datetime\nimport os\nimport pydoc\nfrom typing import Any, Callable, Dict, List, Optional, Type, Union\n\nimport appdirs\nimport yaml\nimport logging\nfrom rich.logging import RichHandler\n\nfrom . import _fwd\nfrom ._modules import PolymorphicChecker, ResourceLoader, Searcher\n\n\nclass Cache:\n    \"\"\"Used to track state between levels of recursion to stop infinite loops, and to optimise repeating actions\"\"\"\n\n    def __init__(self):\n        self._cache: Dict[Any, Dict[str, Any]] = {}\n\n    def mark_ctext(self, ctext: Any) -> bool:\n        if (isinstance(ctext, str) or isinstance(ctext, bytes)) and len(ctext) < 4:\n            logging.debug(f\"Candidate {ctext.__repr__()} too short!\")\n            return False\n\n        if ctext in self._cache:\n            logging.debug(f\"Deduped {ctext.__repr__()}\")\n            return False\n\n        logging.debug(f\"New ctext {ctext.__repr__()}\")\n\n        self._cache[ctext] = {}\n        return True\n\n    def get_or_update(self, ctext: Any, keyname: str, get_value: Callable[[], Any]):\n        # Should have been marked first\n        target = self._cache[ctext]\n        res = target.get(keyname)\n        if res is not None:\n            return res\n\n        val = get_value()\n        target[keyname] = val\n        return val\n\n    def try_get(self, ctext: Any, keyname: str):\n        return self._cache[ctext].get(keyname)\n\n\ndef split_resource_name(full_name: str) -> (str, str):\n    return full_name.split(\"::\", 1)\n\n\nclass Config:\n    def __init__(self):\n        self.verbosity: int = 0\n        self.searcher: str = \"ausearch\"\n        self.params: Dict[str, Dict[str, Union[str, List[str]]]] = {}\n        self.format: str = \"str\"\n        self.modules: List[str] = []\n        self.checker: str = \"ezcheck\"\n        self.default_dist: str = \"cipheydists::dist::english\"\n        self.timeout: Optional[int] = None\n        self._inst: Dict[type, Any] = {}\n        self.objs: Dict[str, Any] = {}\n        self.cache: Cache = Cache()\n\n    @staticmethod\n    def get_default_dir() -> str:\n        return appdirs.user_config_dir(\"ciphey\")\n\n    def merge_dict(self, config_file: Optional[Dict[str, Any]]):\n        if config_file is None:\n            return\n        for a, b in config_file.items():\n            self.update(a, b)\n\n    def load_file(\n        self,\n        path: str = os.path.join(get_default_dir.__func__(), \"config.yml\"),\n        create=False,\n    ):\n        try:\n            with open(path, \"r+\") as file:\n                return self.merge_dict(yaml.safe_load(file))\n        except FileNotFoundError:\n            if create:\n                open(path, \"w+\")\n\n    def instantiate(self, t: type) -> Any:\n        \"\"\"\n        Used to enable caching of a instantiated type after the configuration has settled\n        \"\"\"\n        # We cannot use set default as that would construct it again, and throw away the result\n        res = self._inst.get(t)\n        if res is not None:\n            return res\n        ret = t(self)\n        self._inst[t] = ret\n        return ret\n\n    def __call__(self, t: type) -> Any:\n        return self.instantiate(t)\n\n    def update(self, attrname: str, value: Optional[Any]):\n        if value is not None:\n            setattr(self, attrname, value)\n\n    def update_param(self, owner: str, name: str, value: Optional[Any]):\n        if value is None:\n            return\n\n        target = self.params.setdefault(owner, {})\n\n        if _fwd.registry.get_named(owner).getParams()[name].list:\n            target.setdefault(name, []).append(value)\n        else:\n            target[name] = value\n\n    def update_format(self, value: Optional[str]):\n        if value is not None:\n            self.format = value\n\n    def load_objs(self):\n        # Basic type conversion\n        if self.timeout is not None:\n            self.objs[\"timeout\"] = datetime.timedelta(seconds=int(self.timeout))\n        self.objs[\"format\"] = pydoc.locate(self.format)\n\n        # Checkers do not depend on any other config object\n        logging.debug(f\"Registry is {_fwd.registry._reg[PolymorphicChecker]}\")\n        self.objs[\"checker\"] = self(\n            _fwd.registry.get_named(self.checker, PolymorphicChecker)\n        )\n        # Searchers only depend on checkers\n        self.objs[\"searcher\"] = self(_fwd.registry.get_named(self.searcher, Searcher))\n\n    def update_log_level(self, verbosity: Optional[int]):\n        if verbosity is None:\n            return\n        self.verbosity = verbosity\n\n        if verbosity == 0:\n            self.verbosity = logging.WARNING\n        elif verbosity == 1:\n            self.verbosity = logging.INFO\n        elif verbosity >= 2:\n            self.verbosity = logging.DEBUG\n        else:\n            logging.disable(logging.CRITICAL)\n            return\n\n        # https://rich.readthedocs.io/en/latest/logging.html for more on RichHandler\n        logging.basicConfig(\n            level=self.verbosity,\n            datefmt=\"[%X]\",\n            handlers=[RichHandler(markup=True, rich_tracebacks=True)],\n        )\n        logging.debug(f\"Verbosity set to level {verbosity}\")\n\n    def load_modules(self):\n        import importlib.util\n\n        for i in self.modules:\n            spec = importlib.util.spec_from_file_location(\"ciphey.module_load_site\", i)\n            mod = importlib.util.module_from_spec(spec)\n            spec.loader.exec_module(mod)\n\n        logging.info(f\"Loaded modules {_fwd.registry.get_all_names()}\")\n\n    def complete_config(self) -> \"Config\":\n        \"\"\"This does all the loading for the config, and then returns itself\"\"\"\n        self.load_modules()\n        self.load_objs()\n        self.update_log_level(self.verbosity)\n        return self\n\n    def get_resource(self, res_name: str, t: Optional[Type] = None) -> Any:\n        logging.debug(f\"Loading resource {res_name} of type {t}\")\n\n        # FIXME: Actually returns obj of type `t`, but python is bad\n        loader, name = split_resource_name(res_name)\n        if t is None:\n            return self(_fwd.registry.get_named(loader, ResourceLoader))(name)\n        else:\n            return self(_fwd.registry.get_named(loader, ResourceLoader[t]))(name)\n\n    # Setter methods for cleaner library API\n    def set_verbosity(self, i):\n        self.update_log_level(i)\n        return self\n\n    def set_spinner(self, spinner):\n        self.objs[\"spinner\"] = spinner\n\n    def pause_spinner_handle(self):\n        spinner = self.objs.get(\"spinner\")\n\n        class PausedSpinner:\n            def __enter__(self):\n                if spinner is not None:\n                    spinner.stop()\n\n            def __exit__(self, exc_type, exc_val, exc_tb):\n                if spinner is not None:\n                    spinner.start()\n\n        return PausedSpinner()\n\n    @staticmethod\n    def library_default():\n        \"\"\"The default config for use in a library\"\"\"\n        return Config().set_verbosity(-1)\n\n    def __str__(self):\n        return str(\n            {\n                \"verbosity\": self.verbosity,\n                \"searcher\": self.searcher,\n                \"params\": self.params,\n                \"format\": self.format,\n                \"modules\": self.modules,\n                \"checker\": self.checker,\n                \"default_dist\": self.default_dist,\n                \"timeout\": self.timeout,\n            }\n        )\n\n\n_fwd.config = Config\n\n# Path: ciphey/iface/_registry.py\nfrom typing import Any, Dict, List, Optional, Set, Tuple, Type, Union\n\ntry:\n    from typing import get_args, get_origin\nexcept ImportError:\n    from typing_inspect import get_origin, get_args\n\nfrom . import _fwd\nfrom ._modules import *\n\n\nclass Registry:\n    # I was planning on using __init_subclass__, but that is incompatible with dynamic type creation when we have\n    # generic keys\n\n    RegElem = Union[List[Type], Dict[Type, \"RegElem\"]]\n\n    _reg: Dict[Type, RegElem] = {}\n    _names: Dict[str, Tuple[Type, Set[Type]]] = {}\n    _targets: Dict[str, Dict[Type, List[Type]]] = {}\n    _modules = {Checker, Cracker, Decoder, ResourceLoader, Searcher, PolymorphicChecker}\n\n    def _register_one(self, input_type, module_base, module_args):\n        if len(module_args) == 0:\n            self._reg.setdefault(module_base, []).append(input_type)\n            return\n\n        target_reg = self._reg.setdefault(module_base, {})\n        # Seek to the given type\n        for subtype in module_args[0:-1]:\n            target_reg = target_reg.setdefault(subtype, {})\n        target_reg.setdefault(module_args[-1], []).append(input_type)\n\n    def _real_register(self, input_type: type, *args) -> Type:\n        name = input_type.__name__.lower()\n        name_target = self._names[name] = (input_type, set())\n\n        if issubclass(input_type, Targeted):\n            target = input_type.getTarget()\n        else:\n            target = None\n\n        if issubclass(input_type, Searcher):\n            module_type = module_base = Searcher\n            module_args = ()\n        else:\n            module_type: Optional[Type] = None\n            module_base = None\n\n            # Work out what module type this is\n            if len(args) == 0 and hasattr(input_type, \"__orig_bases__\"):\n                for i in input_type.__orig_bases__:\n                    if module_type is not None:\n                        raise TypeError(\n                            f\"Type derived from multiple registrable base classes {i} and {module_type}\"\n                        )\n                    module_base = get_origin(i)\n                    if module_base not in self._modules:\n                        continue\n                    module_type = i\n            else:\n                for i in self._modules:\n                    if not issubclass(input_type, i):\n                        continue\n                    if module_type is not None:\n                        raise TypeError(\n                            f\"Type derived from multiple registrable base classes {i} and {module_type}\"\n                        )\n                    module_type = i\n            if module_type is None:\n                raise TypeError(\"No registrable base class\")\n\n            # Replace input type with polymorphic checker if required\n            if issubclass(input_type, Checker):\n                if len(args) == 0:\n                    arg = [\n                        get_args(i)\n                        for i in input_type.__orig_bases__\n                        if get_origin(i) == Checker\n                    ][0]\n                    if len(arg) != 1:\n                        raise TypeError(\"No argument for Checker\")\n                    input_type = input_type.convert({arg[0]})\n                else:\n                    input_type = input_type.convert(set(args))\n                self._register_one(input_type, PolymorphicChecker, [])\n                # Refresh the names with the new type\n                name_target = self._names[name] = (input_type, {PolymorphicChecker})\n\n            # Now handle the difference between register and register_multi\n            if len(args) == 0:\n                if module_type is PolymorphicChecker:\n                    module_base = PolymorphicChecker\n                elif module_base is None:\n                    raise TypeError(\"No type argument given\")\n                self._register_one(input_type, module_base, get_args(module_type))\n                name_target[1].add(module_base)\n            else:\n                if module_base is not None:\n                    raise TypeError(f\"Redundant type argument for {module_type}\")\n                module_base = module_type\n                for module_args in args:\n                    # Correct missing brackets\n                    if not isinstance(module_args, tuple):\n                        module_args = (module_args,)\n\n                    self._register_one(input_type, module_base, module_args)\n                    name_target[1].add(module_type[module_args])\n\n        name_target[1].add(module_type)\n\n        if target is not None and issubclass(module_base, Targeted):\n            self._targets.setdefault(target, {}).setdefault(module_type, []).append(\n                input_type\n            )\n\n        return input_type\n\n    def register(self, input_type):\n        return self._real_register(input_type)\n\n    def register_multi(self, *x):\n        return lambda input_type: self._real_register(input_type, *x)\n\n    def __getitem__(self, i: type) -> Optional[Any]:\n        target_type = get_origin(i)\n        # Check if this is a non-generic type, and return the whole dict if it is\n        if target_type is None:\n            return self._reg[i]\n\n        target_subtypes = get_args(i)\n        target_list = self._reg.setdefault(target_type, {})\n        for subtype in target_subtypes:\n            target_list = target_list.setdefault(subtype, {})\n        return target_list\n\n    def get_named(self, name: str, type_constraint: Type = None) -> Any:\n        ret = self._names[name.lower()]\n        if type_constraint and type_constraint not in ret[1]:\n            raise TypeError(f\"Type mismatch: wanted {type_constraint}, got {ret[1]}\")\n        return ret[0]\n\n    def get_targeted(\n        self, target: str, type_constraint: Type = None\n    ) -> Optional[Union[Dict[Type, Set[Type]], Set[Type]]]:\n        x = self._targets.get(target)\n        if x is None or type_constraint is None:\n            return x\n        return x.get(type_constraint)\n\n    def get_all_names(self) -> List[str]:\n        return list(self._names.keys())\n\n    def __str__(self):\n        return f\"ciphey.iface.Registry {{_reg: {self._reg}, _names: {self._names}, _targets: {self._targets}}}\"\n\n\n_fwd.registry = Registry()\n\n# Path: ciphey/iface/__init__.py\nfrom ._config import Config\n\nfrom ._modules import (\n    Checker,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    Decoder,\n    DecoderComparer,\n    Distribution,\n    ParamSpec,\n    PolymorphicChecker,\n    ResourceLoader,\n    Searcher,\n    SearchLevel,\n    SearchResult,\n    T,\n    Translation,\n    U,\n    WordList,\n    pretty_search_results,\n)\nfrom ._registry import get_args, get_origin\n\nfrom ._fwd import registry\n\n# Path: ciphey/basemods/Checkers/any.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, ParamSpec, PolymorphicChecker, registry\n\n\n@registry.register\nclass Any(PolymorphicChecker):\n    \"\"\"Should only be used for debugging, frankly\"\"\"\n\n    def getExpectedRuntime(self, text) -> float:\n        return 0  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    def check(self, text: str) -> Optional[str]:\n        return \"\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/mathsHelper.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to provide helper functions for mathematics\n(oh, not entirely mathematics either. Some NLP stuff and sorting dicts. It's just a helper class\n)\n\"\"\"\n\nfrom collections import OrderedDict\nfrom string import punctuation\nfrom typing import Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\n\nclass mathsHelper:\n    \"\"\"Class to provide helper functions for mathematics and other small things\"\"\"\n\n    def __init__(self):\n        # ETAOIN is the most popular letters in order\n        self.ETAOIN = \"ETAOINSHRDLCUMWFGYPBVKJXQZ\"\n        self.LETTERS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n\n    @staticmethod\n    def gcd(a, b) -> int:\n        \"\"\"Greatest common divisor.\n\n        The Greatest Common Divisor of a and b using Euclid's Algorithm.\n\n        Args:\n            a -> num 1\n            b -> num 2\n\n        Returns:\n            Returns  GCD(a, b)\n\n        \"\"\"\n        # Return\n        while a != 0:\n            a, b = b % a, a\n        return b\n\n    @staticmethod\n    def mod_inv(a: int, m: int) -> Optional[int]:\n        \"\"\"\n        Returns the modular inverse of a mod m, or None if it does not exist.\n\n        The modular inverse of a is the number a_inv that satisfies the equation\n        a_inv * a mod m === 1 mod m\n\n        Note: This is a naive implementation, and runtime may be improved in several ways.\n        For instance by checking if m is prime to perform a different calculation,\n        or by using the extended euclidean algorithm.\n        \"\"\"\n        for i in range(1, m):\n            if (m * i + 1) % a == 0:\n                return (m * i + 1) // a\n        return None\n\n    @staticmethod\n    def percentage(part: float, whole: float) -> float:\n        \"\"\"Returns percentage.\n\n        Just a normal algorithm to return the percent.\n\n        Args:\n            part -> part of the whole number\n            whole -> the whole number\n\n        Returns:\n            Returns the percentage of part to whole.\n\n        \"\"\"\n        if part <= 0 or whole <= 0:\n            return 0\n        # works with percentages\n        return 100 * float(part) / float(whole)\n\n    def sort_prob_table(self, prob_table: dict) -> dict:\n        \"\"\"Sorts the probability table.\n\n        Sorts a dictionary of dictionaries (and all the sub-dictionaries).\n\n        Args:\n            prob_table -> The probability table returned by the neural network to sort.\n\n        Returns:\n            Returns the prob_table, but sorted.\n\n        \"\"\"\n        # for each object: prob table in dictionary\n        max_overall: int = 0\n        max_dict_pair: dict = {}\n        highest_key = None\n        empty_dict: dict = {}\n        # sorts the prob table before we find max, and converts it to order dicts\n        for key, value in prob_table.items():\n            prob_table[key] = self.new_sort(value)\n            prob_table[key] = dict(prob_table[key])\n\n        # gets maximum key then sets it to the front\n        counter_max: int = 0\n        counter_prob: int = len(prob_table)\n        while counter_max < counter_prob:\n            max_overall = 0\n            highest_key = None\n            logging.debug(\n                f\"Running while loop in sort_prob_table, counterMax is {counter_max}\"\n            )\n            for key, value in prob_table.items():\n                logging.debug(f\"Sorting {key}\")\n                maxLocal = 0\n                # for each item in that table\n                for key2, value2 in value.items():\n                    logging.debug(\n                        f\"Running key2 {key2}, value2 {value2} for loop for {value.items()}\"\n                    )\n                    maxLocal = maxLocal + value2\n                    logging.debug(\n                        f\"MaxLocal is {maxLocal} and maxOverall is {max_overall}\"\n                    )\n                    if maxLocal > max_overall:\n                        logging.debug(f\"New max local found {maxLocal}\")\n                        # because the dict doesn't reset\n                        max_dict_pair = {}\n                        max_overall = maxLocal\n                        # so eventually, we get the maximum dict pairing?\n                        max_dict_pair[key] = value\n                        highest_key = key\n                        logging.debug(f\"Highest key is {highest_key}\")\n                # removes the highest key from the prob table\n            logging.debug(\n                f\"Prob table is {prob_table} and highest key is {highest_key}\"\n            )\n            logging.debug(f\"Removing {prob_table[highest_key]}\")\n            del prob_table[highest_key]\n            logging.debug(f\"Prob table after deletion is {prob_table}\")\n            counter_max += 1\n            empty_dict = {**empty_dict, **max_dict_pair}\n\n        # returns the max dict (at the start) with the prob table\n        # this way, it should always work on most likely first.\n        logging.debug(\n            f\"The prob table is {prob_table} and the maxDictPair is {max_dict_pair}\"\n        )\n        logging.debug(f\"The new sorted prob table is {empty_dict}\")\n        return empty_dict\n\n    @staticmethod\n    def new_sort(new_dict: dict) -> dict:\n        \"\"\"Uses OrderedDict to sort a dictionary.\n\n        I think it's faster than my implementation.\n\n        Args:\n            new_dict -> the dictionary to sort\n\n        Returns:\n            Returns the dict, but sorted.\n\n        \"\"\"\n        # (f\"d is {d}\")\n        logging.debug(f\"The old dictionary before new\n_sort() is {new_dict}\")\n        sorted_i = OrderedDict(\n            sorted(new_dict.items(), key=lambda x: x[1], reverse=True)\n        )\n        logging.debug(f\"The dictionary after new_sort() is {sorted_i}\")\n        # sortedI = sort_dictionary(x)\n        return sorted_i\n\n    @staticmethod\n    def is_ascii(s: str) -> bool:\n        \"\"\"Returns the boolean value if is_ascii is an ascii char.\n\n        Does what it says on the tree. Stolen from\n        https://stackoverflow.com/questions/196345/how-to-check-if-a-string-in-python-is-in-ascii\n\n        Args:\n            s -> the char to check.\n\n        Returns:\n            Returns the boolean of the char.\n\n        \"\"\"\n\n        return bool(lambda s: len(s) == len(s.encode()))\n\n    @staticmethod\n    def strip_punctuation(text: str) -> str:\n        \"\"\"Strips punctuation from a given string.\n\n        Uses string.punctuation.\n\n        Args:\n            text -> the text to strip punctuation from.\n\n        Returns:\n            Returns string without punctuation.\n        \"\"\"\n        text: str = (str(text).translate(str.maketrans(\"\", \"\", punctuation))).strip(\n            \"\\n\"\n        )\n        return text\n\n# Path: ciphey/basemods/Checkers/brandon.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to determine whether something is English or not.\n1. Calculate the Chi Squared score of a sentence\n2. If the score is significantly lower than the average score, it _might_ be English\n    2.1. If the score _might_ be English, then take the text and compare it to the sorted dictionary\n    in O(n log n) time.\n    It creates a percentage of \"How much of this text is in the dictionary?\"\n    The dictionary contains:\n        * 20,000 most common US words\n        * 10,000 most common UK words (there's no repetition between the two)\n        * The top 10,000 passwords\n    If the word \"Looks like\" English (chi-squared) and if it contains English words, we can conclude it is\n    very likely English. The alternative is doing the dictionary thing but with an entire 479k word dictionary (slower)\n    2.2. If the score is not English, but we haven't tested enough to create an average, then test it against\n     the dictionary\n\nThings to optimise:\n* We only run the dictionary if it's 20% smaller than the average for chi squared\n* We consider it \"English\" if 45% of the text matches the dictionary\n* We run the dictionary if there is less than 10 total chisquared test\n\nHow to add a language:\n* Download your desired dictionary. Try to make it the most popular words, for example. Place this file into this\n folder with languagename.txt\nAs an example, this comes built in with english.txt\nFind the statistical frequency of each letter in that language.\nFor English, we have:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n}\nIn chisquared.py\nTo add your language, do:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n    \"German\": [0.0973]\n}\nIn alphabetical order\nAnd you're.... Done! Make sure the name of the two match up\n\"\"\"\nimport sys\nfrom math import ceil\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nsys.path.append(\"..\")\ntry:\n    import mathsHelper as mh\nexcept ModuleNotFoundError:\n    import ciphey.mathsHelper as mh\n\n\n@registry.register\nclass Brandon(Checker[str]):\n    \"\"\"\n    Class designed to confirm whether something is **language** based on how many words of **language** appears\n    Call confirmLanguage(text, language)\n    * text: the text you want to confirm\n    * language: the language you want to confirm\n\n    Find out what language it is by using chisquared.py, the highest chisquared score is the language\n    languageThreshold = 45\n    if a string is 45% **language** words, then it's confirmed to be english\n    \"\"\"\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually work this out\n        # TODO its 0.2 seconds on average\n        return 1e-4  # 100 \u00b5s\n\n    wordlist: set\n\n    def clean_text(self, text: str) -> set:\n        \"\"\"Cleans the text ready to be checked\n\n        Strips punctuation, makes it lower case, turns it into a set separated by spaces, removes duplicate words\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            text -> the text as a list, now cleaned\n\n        \"\"\"\n        # makes the text unique words and readable\n        text = text.lower()\n        text = self.mh.strip_punctuation(text)\n        text = text.split(\" \")\n        text = filter(lambda x: len(x) > 2, text)\n        text = set(text)\n        return text\n\n    def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        \"\"\"Given text determine if it passes checker\n\n        The checker uses the variable passed to it. I.E. Stopwords list, 1k words, dictionary\n\n        Args:\n            text -> The text to check\n            threshold -> at what point do we return True? The percentage of text that is in var before we return True\n            text_length -> the length of the text\n            var -> the variable we are checking against. Stopwords list, 1k words list, dictionary list.\n        Returns:\n            boolean -> True for it passes the test, False for it fails the test.\"\"\"\n        if text is None:\n            logging.debug(\"Checker's text is None, so returning False\")\n            return False\n        if var is None:\n            logging.debug(\"Checker's input var is None, so returning False\")\n            return False\n\n        percent = ceil(text_length * threshold)\n        logging.debug(f\"Checker's chunks are size {percent}\")\n        meet_threshold = 0\n        location = 0\n        end = percent\n\n        if text_length <= 0:\n            return False\n\n        while location <= text_length:\n            # chunks the text, so only gets THRESHOLD chunks of text at a time\n            text = list(text)\n            to_analyse = text[location:end]\n            logging.debug(f\"To analyse is {to_analyse}\")\n            for word in to_analyse:\n                # if word is a stopword, + 1 to the counter\n                if word in var:\n                    logging.debug(\n                        f\"{word} is in var, which means I am +=1 to the meet_threshold which is {meet_threshold}\"\n                    )\n                    meet_threshold += 1\n                meet_threshold_percent = meet_threshold / text_length\n                if meet_threshold_percent >= threshold:\n                    logging.debug(\n                        f\"Returning true since the percentage is {meet_threshold / text_length} and the threshold is {threshold}\"\n                    )\n                    # if we meet the threshold, return True\n                    # otherwise, go over again until we do\n                    # We do this in the for loop because if we're at 24% and THRESHOLD is 25\n                    # we don't want to wait THRESHOLD to return true, we want to return True ASAP\n                    return True\n            location = end\n            end = end + percent\n        logging.debug(\n            f\"The language proportion {meet_threshold_percent} is under the threshold {threshold}\"\n        )\n        return False\n\n    def __init__(self, config: Config):\n        # Suppresses warning\n        super().__init__(config)\n        self.mh = mh.mathsHelper()\n\n        phases = config.get_resource(self._params()[\"phases\"])\n\n        self.thresholds_phase1 = phases[\"1\"]\n        self.thresholds_phase2 = phases[\"2\"]\n        self.top1000Words = config.get_resource(self._params().get(\"top1000\"))\n        self.wordlist = config.get_resource(self._params()[\"wordlist\"])\n        self.stopwords = config.get_resource(self._params().get(\"stopwords\"))\n\n        self.len_phase1 = len(self.thresholds_phase1)\n        self.len_phase2 = len(self.thresholds_phase2)\n\n    def check(self, text: str) -> Optional[str]:\n        \"\"\"Checks to see if the text is in English\n\n        Performs a decryption, but mainly parses the internal data packet and prints useful information.\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            bool -> True if the text is English, False otherwise.\n\n        \"\"\"\n        logging.debug(f'In Language Checker with \"{text}\"')\n        text = self.clean_text(text)\n        logging.debug(f'Text split to \"{text}\"')\n        if text == \"\":\n            logging.debug(\"Returning None from Brandon as the text cleaned is none.\")\n            return None\n\n        length_text = len(text)\n\n        what_to_use = {}\n\n        # this code decides what checker / threshold to use\n        # if text is over or equal to maximum size, just use the maximum possible checker\n        what_to_use = self.calculateWhatChecker(\n            length_text, self.thresholds_phase1.keys()\n        )\n        logging.debug(self.thresholds_phase1)\n        what_to_use = self.thresholds_phase1[str(what_to_use)]\n        # def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        if \"check\" in what_to_use:\n            # perform check 1k words\n            result = self.checker(\n                text, what_to_use[\"check\"], length_text, self.top1000Words\n            )\n        elif \"stop\" in what_to_use:\n            # perform stopwords\n            result = self.checker(\n                text, what_to_use[\"stop\"], length_text, self.stopwords\n            )\n        elif \"dict\" in what_to_use:\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n            # If result is None, no point doing it again in phase2\n            if not result:\n                return None\n        else:\n            logging.info(f\"It is neither stop or check, but instead {what_to_use}\")\n\n        # return False if phase 1 fails\n        if not result:\n            return None\n        else:\n            what_to_use = self.calculateWhatChecker(\n                length_text, self.thresholds_phase2.keys()\n            )\n            what_to_use = self.thresholds_phase2[str(what_to_use)]\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n        return \"\" if result else None\n\n    def calculateWhatChecker(self, length_text, key):\n        \"\"\"Calculates what threshold / checker to use\n\n        If the length of the text is over the maximum sentence length, use the last checker / threshold\n        Otherwise, traverse the keys backwards until we find a key range that does not fit.\n        So we traverse backwards and see if the sentence length is between current - 1 and current\n        In this way, we find the absolute lowest checker / percentage threshold.\n        We traverse backwards because if the text is longer than the max sentence length, we already know.\n        In total, the keys are only 5 items long or so. It is not expensive to move backwards, nor is it expensive to move forwards.\n\n        Args:\n            length_text -> The length of the text\n            key -> What key we want to use. I.E. Phase1 keys, Phase2 keys.\n        Returns:\n            what_to_use -> the key of the lowest checker.\"\"\"\n\n        _keys = list(key)\n        _keys = list(map(int, _keys))\n        if length_text >= int(_keys[-1]):\n            what_to_use = list(key)[_keys.index(_keys[-1])]\n        else:\n            # this algorithm finds the smallest possible fit for the text\n            for counter, i in reversed(list(enumerate(_keys))):\n                #  [0, 110, 150]\n                if i <= length_text:\n                    what_to_use = i\n        return what_to_use\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"top1000\": ParamSpec(\n                desc=\"A wordlist of the top 1000 words\",\n                req=False,\n                default=\"cipheydists::list::english1000\",\n            ),\n            \"wordlist\": ParamSpec(\n                desc=\"A wordlist of all the words\",\n                req=False,\n                default=\"cipheydists::list::english\",\n            ),\n            \"stopwords\": ParamSpec(\n                desc=\"A wordlist of StopWords\",\n                req=False,\n                default=\"cipheydists::list::englishStopWords\",\n            ),\n            \"threshold\": ParamSpec(\n                desc=\"The minimum proportion (between 0 and 1) that must be in the dictionary\",\n                req=False,\n                default=0.45,\n            ),\n            \"phases\": ParamSpec(\n                desc=\"Language-specific phase thresholds\",\n                req=False,\n                default=\"cipheydists::brandon::english\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/format.py\nimport json\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass JsonChecker(Checker[str]):\n\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying json checker\")\n\n        # https://github.com/Ciphey/Ciphey/issues/389\n        if text.isdigit():\n            return None\n\n        try:\n            json.loads(text)\n            return \"\"\n        except ValueError:\n            return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 1e-7 * len(text)  # From benchmarks I found online\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/human.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, registry\nfrom rich.console import Console\nfrom rich.markup import escape\n\nconsole = Console()\n\n\n@registry.register\nclass HumanChecker(Checker[str]):\n\n    \"\"\"\n    Uses the person's decision to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: str) -> Optional[str]:\n        with self._config().pause_spinner_handle():\n            response = console.input(\n                f\"Possible plaintext: [blue bold]{escape(ctext.__repr__())}[/blue bold] ([green]y[/green]/[red]N[/red]): \"\n            )\n        if response == \"y\":\n            return \"\"\n        elif response in (\"n\", \"\"):\n            return None\n        else:\n            return self.check(ctext)\n\n    def getExpectedRuntime(self, text: str) -> float:\n        return 1  # About a second\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Checkers/quadgrams.py\nimport logging\nimport re\nfrom math import log10\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, Translation, registry\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Quadgrams(Checker[str]):\n\n    \"\"\"\n    Uses Quadgrams to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying Quadgrams checker\")\n        # Capitalize and remove everything that's not a letter\n        ctext = re.sub(\"[^A-Z]\", \"\", ctext.upper())\n        quadgrams = self.QUADGRAMS_DICT\n        quadgrams_sum = sum(quadgrams.values())\n        score = 0\n        for key in quadgrams.keys():\n            quadgrams[key] = float(quadgrams[key]) / quadgrams_sum\n        floor = log10(0.01 / quadgrams_sum)\n        for i in range(len(ctext) - 4 + 1):\n            # Get all quadgrams from ctext and check if they're in the dict\n            # If yes then add the score of those quadgrams to the total score\n            if ctext[i : i + 4] in quadgrams:\n                score += quadgrams[ctext[i : i + 4]]\n            else:\n                score += floor\n        if len(ctext) > 0:\n            score = score / len(ctext)\n        logging.info(f\"Quadgrams is {score}\")\n        # The default threshold was found to work the best from lots of testing\n        if score > self.threshold:\n            return \"\"\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The quadgrams dictionary to use\",\n                req=False,\n                default=\"cipheydists::dist::quadgrams\",\n            ),\n            \"score\": ParamSpec(\n                desc=\"The score threshold to use\",\n                req=False,\n                default=0.00011,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.QUADGRAMS_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.threshold = float(self._params()[\"score\"])\n\n# Path: ciphey/basemods/Checkers/regex.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Regex(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = list(map(re.compile, self._params()[\"regex\"]))\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"Passed with regex {regex}. Want to contribute to Ciphey? Submit your regex here to allow Ciphey to automatically get this next time https://github.com/bee-san/pyWhat/wiki/Adding-your-own-Regex\\n\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"regex\": ParamSpec(\n                req=True,\n                desc=\"The regex that must be matched (in a substring)\",\n                list=True,\n            )\n        }\n\n\n@registry.register\nclass RegexList(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = []\n        for i in self._params()[\"resource\"]:\n            self.regexes += [re.compile(regex) for regex in config.get_resource(i)]\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"passed with regex {regex}\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"resource\": ParamSpec(\n                req=True,\n                desc=\"A list of regexes that could be matched\",\n                list=True,\n            )\n        }\n\n# Path: ciphey/basemods/Checkers/what.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\nimport logging\nfrom rich.logging import RichHandler\nfrom pywhat import identifier\nfrom rich.console import Console\n\nconsole = Console()\n\n\n@registry.register\nclass What(Checker[str]):\n\n    \"\"\"\n    Uses PyWhat to determine plaintext with regexes\n    https://github.com/bee-san/pyWhat\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying PyWhat checker\")\n        returned_regexes = self.id.identify(ctext)\n        if returned_regexes[\"Regexes\"]:\n            matched_regex = returned_regexes[\"Regexes\"]['text'][0][\"Regex Pattern\"]\n\n            ret = f'The plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            human = (\n                f'\\nI think the plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            )\n\n            if \"Description\" in matched_regex and matched_regex[\"Description\"]:\n                s = matched_regex[\"Description\"]\n                # lowercases first letter so it doesn't look weird\n                s = f\", which is {s[0].lower() + s[1:]}\\n\"\n                ret += s\n                human += s\n\n            # if URL is attached, include that too.\n            if \"URL\" in matched_regex and matched_regex[\"URL\"]:\n                link = matched_regex[\"URL\"] + ctext.replace(\" \", \"\")\n                ret += f\"\\nClick here to view in browser [#CAE4F1][link={link}]{link}[/link][/#CAE4F1]\\n\"\n\n            # If greppable mode is on, don't print this\n            if self.config.verbosity >= 0:\n                # Print with full stop\n                console.print(human)\n            return ret\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.config = config\n        self.id = identifier.Identifier()\n\n# Path: ciphey/basemods/Checkers/ezcheck.py\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nfrom .brandon import Brandon\nfrom .format import JsonChecker\nfrom .human import HumanChecker\nfrom .quadgrams import Quadgrams\nfrom .regex import RegexList\nfrom .what import What\n\n\n@registry.register\nclass EzCheck(Checker[str]):\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns, followed by a human check\n    \"\"\"\n\n    def check(self, text: str) -> Optional[str]:\n        for checker in self.checkers:\n            res = checker.check(text)\n            if (\n                res is not None\n                and (self.decider is None or self.decider.check(text)) is not None\n            ):\n                return res\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        return sum(\n            i.getExpectedRuntime(text) for i in self.checkers\n        ) + self.decider.getExpectedRuntime(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        self.checkers: List[Checker[str]] = []\n        # Disable human checker for automated systems\n        if config.verbosity >= 0:\n            self.decider = config(HumanChecker)\n        else:\n            self.decider = None\n\n        # We need to modify the config for each of the objects\n\n        # First PyWhat, as it's the fastest\n        self.checkers.append(config(What))\n\n        # Next, the json checker\n        self.checkers.append(config(JsonChecker))\n\n        # Second to last, the quadgrams checker\n        self.checkers.append(config(Quadgrams))\n\n        # Finally, the Brandon checker, as it is the slowest\n        self.checkers.append(config(Brandon))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/quorum.py\nfrom typing import Dict, Generic, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, _registry\n\n\nclass Quorum(Generic[T], Checker[T]):\n    def check(self, text: T) -> Optional[str]:\n        left = self._params().k\n        results = []\n        for checker in self.checkers:\n            results.append(checker.check(text))\n...\n", "needle_token_start": 8705, "needle_token_end": 8905, "code_context_ntokens": 16045, "output": ["```python\nfrom collections import OrderedDict\n\ndef sort_prob_table(prob_table: dict) -> dict:\n    \"\"\"Sorts the probability table.\n\n    Sorts a dictionary of dictionaries (and all the sub-dictionaries).\n\n    Args:\n        prob_table -> The probability table returned by the neural network to sort.\n\n    Returns:\n        Returns the prob_table, but sorted.\n\n    \"\"\"\n    # for each object: prob table in dictionary\n    max_overall: int = 0\n    max_dict_pair: dict = {}\n    highest_key = None\n    empty_dict: dict = {}\n    # sorts the prob table before we find max, and converts it to order dicts\n    for key, value in prob_table.items():\n        prob_table[key] = self.new_sort(value)\n        prob_table[key] = dict(prob_table[key])\n\n    # gets maximum key then sets it to the front\n    counter_max: int = 0\n    counter_prob: int = len(prob_table)\n    while counter_max < counter_prob:\n        max_overall = 0\n        highest_key = None\n        logging.debug(\n            f\"Running while loop in sort_prob_table, counterMax is {counter_max}\"\n        )\n        for key, value in prob_table.items():\n            logging.debug(f\"Sorting {key}\")\n            maxLocal = 0\n            # for each item in that table\n            for key2, value2 in value.items():\n                logging.debug(\n                    f\"Running key2 {key2}, value2 {value2} for loop for {value.items()}\"\n                )\n                maxLocal = maxLocal + value2\n                logging.debug(\n                    f\"MaxLocal is {maxLocal} and maxOverall is {max_overall}\"\n                )\n                if maxLocal > max_overall:\n                    logging.debug(f\"New max local found {maxLocal}\")\n                    # because the dict doesn't reset\n                    max_dict_pair = {}\n                    max_overall = maxLocal\n                    # so eventually, we get the maximum dict pairing?\n                    max_dict_pair[key] = value\n                    highest_key = key\n                    logging.debug(f\"Highest key is {highest_key}\")\n            # removes the highest key from the prob table\n        logging.debug(\n            f\"Prob table is {prob_table} and highest key is {highest_key}\"\n        )\n        logging.debug(f\"Removing {prob_table[highest_key]}\")\n        del prob_table[highest_key]\n        logging.debug(f\"Prob table after deletion is {prob_table}\")\n        counter_max += 1\n        empty_dict = {**empty_dict, **max_dict_pair}\n\n    # returns the max dict (at the start) with the prob table\n    # this way, it should always work on most likely first.\n    logging.debug(\n        f\"The prob table is {prob_table} and the maxDictPair is {max_dict_pair}\"\n    )\n    logging.debug(f\"The new sorted prob table is {empty_dict}\")\n    return empty_dict\n```"]}
{"repo": "Ciphey/Ciphey", "name": "get_name", "language": "python", "path": "ciphey/ciphey.py", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve a string value, either from a command-line argument or from standard input if the argument is not provided.\n2. **Input**: It takes three parameters: a context object, a parameter object, and a value which is potentially a string.\n3. **Output**: Returns a string that is either the input value provided or read from standard input.\n4. **Procedure**: The function first checks if a value is provided. If not, it checks if the standard input is not a terminal (i.e., there is data being piped or redirected into the program). It then reads this input, strips any surrounding whitespace, and returns it. If a value is provided, or if there is no data on standard input, it simply returns the provided value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/basemods/Decoders/galactic.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Galactic(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Takes a string written in the 'Standard Galactic Alphabet'\n        (aka Minecraft Enchanting Table Symbols) and translates it to ASCII text.\n        \"\"\"\n        logging.debug(\"Attempting Standard Galactic Alphabet decoder\")\n\n        # To avoid complications, only move forward with the decoding if we can\n        # reasonably assume that the input string is written in the galactic alphabet\n        galactic_matches = 0\n        for symbol in self.GALACTIC_DICT.keys():\n            # These symbols are assumed to be frequent enough in regular\n            # text to be skipped when counting the matches. All others are counted.\n            if symbol in ctext and symbol not in [\"!\", \"|\"]:\n                galactic_matches += 1\n            else:\n...\n# Path: ciphey/basemods/Decoders/gzip.py\nimport zlib\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Gzip(Decoder[bytes]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Gzip decoding\n        \"\"\"\n        try:\n            return zlib.decompress(ctext, 16 + zlib.MAX_WBITS).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"gzip\"\n\n# Path: ciphey/basemods/Decoders/hexadecimal.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Hexadecimal(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Hexadecimal decoding\n        \"\"\"\n        ctext_decoded = \"\"\n        try:\n            ctext_decoded = bytearray.fromhex(ctext).decode(\"utf-8\")\n            return ctext_decoded\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.015\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"hexadecimal\"\n\n# Path: ciphey/basemods/Decoders/leetspeak.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Leetspeak(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        for src, dst in self.translate.items():\n            ctext = ctext.replace(src, dst)\n        return ctext\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.translate = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The leetspeak dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::leet\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"leetspeak\"\n\n# Path: ciphey/basemods/Decoders/morse_code.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Morse_code(Decoder[str]):\n    # A priority list for char/word boundaries\n    BOUNDARIES = {\" \": 1, \"/\": 2, \"\\n\": 3}\n    PURGE = {ord(c): None for c in BOUNDARIES.keys()}\n    MAX_PRIORITY = 3\n    ALLOWED = {\".\", \"-\", \" \", \"/\", \"\\n\"}\n    MORSE_CODE_DICT: Dict[str, str]\n    MORSE_CODE_DICT_INV: Dict[str, str]\n\n    def decode(self, ctext: T) -> Optional[U]:\n        logging.debug(\"Attempting Morse code decoder\")\n\n        char_boundary = word_boundary = None\n\n        char_boundary = word_boundary = None\n        char_priority = word_priority = 0\n        # Custom loop allows early break\n        for i in ctext:\n            i_priority = self.BOUNDARIES.get(i)\n            if i_priority is None:\n                if i in self.ALLOWED:\n                    continue\n                logging.debug(f\"Non-morse char '{i}' found\")\n                return None\n\n            if i_priority <= char_priority or i == char_boundary or i == word_boundary:\n                continue\n            # Default to having a char boundary over a word boundary\n            if (\n                i_priority > word_priority\n                and word_boundary is None\n                and char_boundary is not None\n            ):\n                word_priority = i_priority\n                word_boundary = i\n                continue\n            char_priority = i_priority\n            char_boundary = i\n\n        logging.debug(\n            f\"Char boundary is unicode {ord(char_boundary)}, and word boundary is unicode {ord(word_boundary) if word_boundary is not None else None}\"\n        )\n\n        result = \"\"\n\n        for word in ctext.split(word_boundary) if word_boundary else [ctext]:\n            logging.debug(f\"Attempting to decode word {word}\")\n            for char in word.split(char_boundary):\n                char = char.translate(self.PURGE)\n                if len(char) == 0:\n                    continue\n                try:\n                    m = self.MORSE_CODE_DICT_INV[char]\n                except KeyError:\n                    logging.debug(f\"Invalid codeword '{char}' found\")\n                    return None\n                result = result + m\n            # after every word add a space\n            result = result + \" \"\n        if len(result) == 0:\n            logging.debug(\"Morse code failed to match\")\n            return None\n        # Remove trailing space\n        result = result[:-1]\n        logging.info(f\"Morse code successful, returning {result}\")\n        return result.strip().upper()\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.MORSE_CODE_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.MORSE_CODE_DICT_INV = {v: k for k, v in self.MORSE_CODE_DICT.items()}\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The morse code dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::morse\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"morse_code\"\n\n# Path: ciphey/basemods/Decoders/multi_tap.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Multi_tap(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        result = \"\"\n        for x in ctext.split():\n            if x == self.SPACE_DIGIT:  # Check if it's a space\n                result += \" \"\n            elif not Multi_tap.valid_code_part(x):\n                return None\n            else:\n                result += self.decode_num_to_char(x)\n\n        return result\n\n    @staticmethod\n    def valid_code_part(code: str) -> bool:\n        if not code.isdigit():\n            return False\n\n        # if not all the digits are the same\n        if not Multi_tap.is_all_dup(code):\n            return False\n\n        if int(code[0]) not in range(2, 10):\n            return False\n\n        if len(code) > 4:\n            return False\n\n        return True\n\n    @staticmethod\n    def decode_num_to_char(number: str) -> str:\n        index = Multi_tap.calculate_index(number)\n        return Multi_tap.number_index_to_char(index)\n\n    @staticmethod\n    def is_all_dup(code):\n        return len(set(code)) == 1\n\n    @staticmethod\n    def calculate_index(number: str) -> int:\n        first_number_as_int = int(number[0])\n\n        number_index = Multi_tap.get_index_from_first_digit(first_number_as_int)\n\n        # Add to index the number of the char : \"22\" -> index += 1\n        num_rest_numbers = len(number) - 1\n        number_index += num_rest_numbers\n\n        return number_index\n\n    @staticmethod\n    def number_index_to_char(index_number: int) -> str:\n        start_ascii_value = ord(\"A\")\n        return chr(start_ascii_value + index_number)\n\n    @staticmethod\n    def get_index_from_first_digit(first_digit: int) -> int:\n        number_index = 0\n        if first_digit >= 8:  # s have 4 chars\n            number_index += 1\n\n        first_digit -= 2  # start in 200\n\n        number_index += first_digit * 3  # jump 3 every time\n\n        return number_index\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.SPACE_DIGIT = \"0\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"multi_tap\"\n\n# Path: ciphey/basemods/Decoders/octal.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Octal(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Octal decoding\n        \"\"\"\n        str_converted = []\n        octal_seq = ctext.split(\" \")\n        if len(octal_seq) == 1:\n            # Concatted octal must be formed of octal triplets\n            if len(ctext) % 3 != 0:\n                return None\n            octal_seq = [ctext[i : i + 3] for i in range(0, len(ctext), 3)]\n            logging.debug(f\"Trying chunked octal {octal_seq}\")\n        try:\n            for octal_char in octal_seq:\n                if len(octal_char) > 3:\n                    logging.debug(\"Octal subseq too long\")\n                    return None\n                n = int(octal_char, 8)\n                if (\n                    n < 0\n                ):  # n cannot be greater than 255, as we checked that with the earlier length check\n                    logging.debug(f\"Non octal char {octal_char}\")\n                    return None\n                str_converted.append(n)\n\n            return bytes(str_converted)\n        # Catch bad octal chars\n        except ValueError:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.025\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"octal\"\n\n# Path: ciphey/basemods/Decoders/reverse.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Reverse(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        return ctext[::-1]\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"reverse\"\n\n# Path: ciphey/basemods/Decoders/tap_code.py\n# by https://github.com/RustyDucky and https://github.com/lukasgabriel\n\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, Translation, U, registry\n\n\n@registry.register\nclass Tap_code(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Tap code decoding\n        \"\"\"\n        try:\n            result = \"\"\n            combinations = ctext.split(\" \")\n            for fragment in combinations:\n                result += self.TABLE.get(fragment)\n            return result\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.06\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.TABLE = config.get_resource(self._params()[\"dict\"], Translation)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The table of letters used for the tap code interpretation.\",\n                req=False,\n                default=\"cipheydists::translate::tap_code\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"tap_code\"\n\n# Path: ciphey/basemods/Decoders/unicode.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Utf8(Decoder[bytes]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs UTF-8 decoding\n        \"\"\"\n        logging.debug(\"Attempting UTF-8 decoder\")\n        result = \"\"\n        try:\n            result = ctext.decode(\"utf-8\")\n            if result != ctext:\n                logging.info(f\"UTF-8 successful, returning '{result}'\")\n                return result\n            else:\n                return None\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.9\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"utf8\"\n\n# Path: ciphey/basemods/Decoders/url.py\nfrom typing import Dict, Optional\nfrom urllib.parse import unquote_plus\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Url(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs URL decoding\n        \"\"\"\n        logging.debug(\"Attempting URL\")\n        result = \"\"\n        try:\n            result = unquote_plus(ctext, errors=\"strict\")\n            if result != ctext:\n                logging.info(f\"URL successful, returning '{result}'\")\n                return result\n            else:\n                return None\n        except Exception:\n            logging.debug(\"Failed to decode URL\")\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"url\"\n\n# Path: ciphey/basemods/Decoders/uuencode.py\nfrom binascii import a2b_uu\nfrom codecs import decode\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Uuencode(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        UUEncode (Unix to Unix Encoding) is a symmetric encryption\n        based on conversion of binary data (split into 6-bit blocks) into ASCII characters.\n\n        This function decodes the input string 'ctext' if it has been encoded using 'uuencoder'\n        It will return None otherwise\n        \"\"\"\n        logging.debug(\"Attempting UUencode\")\n        result = \"\"\n        try:\n            # UUencoded messages may begin with prefix \"begin\" and end with suffix \"end\"\n            # In that case, we use the codecs module in Python\n            ctext_strip = ctext.strip()\n            if ctext_strip.startswith(\"begin\") and ctext_strip.endswith(\"end\"):\n                result = decode(bytes(ctext, \"utf-8\"), \"uu\").decode()\n            else:\n                # If there isn't a \"being\" prefix and \"end\" suffix, we use the binascii module instead\n                # It is possible that the ctext has multiple lines, so convert each line and append\n                ctext_split = list(filter(None, ctext.splitlines()))\n                for _, value in enumerate(ctext_split):\n                    result += a2b_uu(value).decode(\"utf-8\")\n            logging.info(f\"UUencode successful, returning '{result}'\")\n            return result\n        except Exception:\n            logging.debug(\"Failed to decode UUencode\")\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"uuencode\"\n\n# Path: ciphey/basemods/Decoders/__init__.py\nfrom . import (\n    a1z26,\n    atbash,\n    base58_bitcoin,\n    base58_ripple,\n    base62,\n    base69,\n    base91,\n    bases,\n    baudot,\n    binary,\n    braille,\n    brainfuck,\n    decimal,\n    dna,\n    dtmf,\n    galactic,\n    gzip,\n    hexadecimal,\n    leetspeak,\n    morse_code,\n    multi_tap,\n    octal,\n    reverse,\n    tap_code,\n    unicode,\n    url,\n    uuencode,\n)\n\n# Path: ciphey/basemods/Resources/cipheydists.py\nfrom functools import lru_cache\nfrom typing import Any, Dict, Optional, Set\n\nimport cipheydists\nimport logging\n\nfrom ciphey.iface import (\n    Config,\n    Distribution,\n    ParamSpec,\n    ResourceLoader,\n    Translation,\n    WordList,\n    registry,\n)\n\n\n@registry.register_multi(WordList, Distribution, Translation)\nclass CipheyDists(ResourceLoader):\n    # _wordlists: Set[str] = frozenset({\"english\", \"english1000\", \"englishStopWords\"})\n    # _brandons: Set[str] = frozenset({\"english\"})\n    # _dists: Set[str] = frozenset({\"twist\"})\n    # _translates: Set[str] = frozenset({\"morse\"})\n    _getters = {\n        \"list\": cipheydists.get_list,\n        \"dist\": cipheydists.get_dist,\n        \"brandon\": cipheydists.get_brandon,\n        \"translate\": cipheydists.get_translate,\n    }\n\n    def whatResources(self) -> Optional[Set[str]]:\n        pass\n\n    @lru_cache()\n    def getResource(self, name: str) -> Any:\n        logging.debug(f\"Loading cipheydists resource {name}\")\n        prefix, name = name.split(\"::\", 1)\n        return self._getters[prefix](name)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n# Path: ciphey/basemods/Resources/files.py\nimport csv\nimport json\nfrom functools import lru_cache\nfrom typing import Dict, Generic, Optional, Set\n\nfrom ciphey.iface import (\n    Config,\n    Distribution,\n    ParamSpec,\n    ResourceLoader,\n    T,\n    WordList,\n    registry,\n)\n\n\n# We can use a generic resource loader here, as we can instantiate it later\n@registry.register_multi(WordList, Distribution)\nclass Json(ResourceLoader):\n    def whatResources(self) -> T:\n        return self._names\n\n    @lru_cache()\n    def getResource(self, name: str) -> T:\n        prefix, name = name.split(\"::\", 1)\n        return {\"wordlist\": (lambda js: {js}), \"dist\": (lambda js: js)}[prefix](\n            json.load(open(self._paths[int(name) - 1]))\n        )\n\n    @staticmethod\n    def getName() -> str:\n        return \"json\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\"path\": ParamSpec(req=True, desc=\"The path to a JSON file\", list=True)}\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self._paths = self._params()[\"path\"]\n        self._names = set(range(1, len(self._paths)))\n\n\n# We can use a generic resource loader here, as we can instantiate it later\n@registry.register_multi(WordList, Distribution)\nclass Csv(Generic[T], ResourceLoader[T]):\n    def whatResources(self) -> Set[str]:\n        return self._names\n\n    @lru_cache()\n    def getResource(self, name: str) -> T:\n        prefix, name = name.split(\"::\", 1)\n        return {\n            \"wordlist\": (lambda reader: {i[0] for i in reader}),\n            \"dist\": (lambda reader: {i[0]: float(i[1]) for i in reader}),\n        }[prefix](csv.reader(open(self._paths[int(name) - 1])))\n\n    @staticmethod\n    def getName() -> str:\n        return \"csv\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\"path\": ParamSpec(req=True, desc=\"The path to a CSV file\", list=True)}\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self._paths = self._params()[\"path\"]\n        self._names = set(range(1, len(self._paths)))\n\n# Path: ciphey/basemods/Resources/__init__.py\nfrom . import cipheydists, files\n\n# Path: ciphey/basemods/Searchers/ausearch.py\nimport bisect\nimport distutils\nimport math\nfrom copy import copy\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nfrom typing import Any, Dict, Generic, List, Optional, TypeVar, Union\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import (\n    Checker,\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    Decoder,\n    ParamSpec,\n    Searcher,\n    SearchLevel,\n    SearchResult,\n    T,\n    registry,\n)\n\n\"\"\"\nWe are using a tree structure here, because that makes searching and tracing back easier\nAs such, when we encounter another possible parent, we remove that edge\n\"\"\"\n\n\nclass DuplicateNode(Exception):\n    pass\n\n\n@dataclass\nclass AuSearchSuccessful(Exception):\n    target: \"Node\"\n    info: str\n\n\n@dataclass\nclass Node:\n    # The root has no parent edge\n    level: SearchLevel\n    parent: Optional[\"Edge\"] = None\n    depth: int = 0\n\n    @staticmethod\n    def decoding(\n        config: Config, route: Union[Cracker, Decoder], result: Any, source: \"Node\"\n    ) -> \"Node\":\n        if not config.cache.mark_ctext(result):\n            raise DuplicateNode()\n\n        checker: Checker = config.objs[\"checker\"]\n        ret = Node(\n            parent=None,\n            level=SearchLevel(\n                name=type(route).__name__.lower(), result=CrackResult(value=result)\n            ),\n            depth=source.depth + 1,\n        )\n        edge = Edge(source=source, route=route, dest=ret)\n        ret.parent = edge\n        check_res = checker(result)\n        if check_res is not None:\n            raise AuSearchSuccessful(target=ret, info=check_res)\n        return ret\n\n    @staticmethod\n    def cracker(config: Config, edge_template: \"Edge\", result: CrackResult) -> \"Node\":\n        if not config.cache.mark_ctext(result.value):\n            raise DuplicateNode()\n\n        checker: Checker = config.objs[\"checker\"]\n        # Edges do not directly contain containers, so this is fine\n        edge = copy(edge_template)\n        ret = Node(\n            parent=edge,\n            level=SearchLevel(name=type(edge.route).__name__.lower(), result=result),\n            depth=edge.source.depth + 1,\n        )\n        edge.dest = ret\n        check_res = checker(result.value)\n        if check_res is not None:\n            raise AuSearchSuccessful(target=ret, info=check_res)\n        return ret\n\n    @staticmethod\n    def root(config: Config, ctext: Any):\n        if not config.cache.mark_ctext(ctext):\n            raise DuplicateNode()\n\n        return Node(parent=None, level=SearchLevel.input(ctext))\n\n    def get_path(self):\n        if self.parent is None:\n            return [self.level]\n        return self.parent.source.get_path() + [self.level]\n\n\n@dataclass\nclass AusearchEdge:\n    # TODO: This is just CrackInfo with failure probability added...\n    success_probability: float\n    failure_probability: float\n    success_time: float\n    failure_time: float\n\n    def __init__(self, success_probability, success_time, failure_time):\n        self.success_probability = success_probability\n        self.failure_probability = 1.0 - success_probability\n        self.success_time = success_time\n        self.failure_time = failure_time\n\n\n@dataclass\nclass AusearchResult:\n    weight: float\n    index: int\n\n\ndef calculate_score(info: CrackInfo):\n    return info.success_likelihood / \\\n        (info.success_runtime * info.success_likelihood + info.failure_runtime * (1-info.success_likelihood))\n\n\n@dataclass\nclass Edge:\n    source: Node\n    route: Union[Cracker, Decoder]\n    dest: Optional[Node] = None\n    # Info is not filled in for Decoders\n    score: Optional[float] = None\n\n\nPriorityType = TypeVar(\"PriorityType\")\n\n\nclass PriorityWorkQueue(Generic[PriorityType, T]):\n    _sorted_priorities: List[PriorityType]\n    _queues: Dict[Any, List[T]]\n\n    def add_work(self, priority: PriorityType, work: List[T]) -> None:\n        logging.debug(f\"\"\"Adding work at depth {priority}\"\"\")\n\n        idx = bisect.bisect_left(self._sorted_priorities, priority)\n        if (\n            idx == len(self._sorted_priorities)\n            or self._sorted_priorities[idx] != priority\n        ):\n            self._sorted_priorities.insert(idx, priority)\n        self._queues.setdefault(priority, []).extend(work)\n\n    def get_work(self) -> T:\n        best_priority = self._sorted_priorities[0]\n        target = self._queues[best_priority]\n        ret = target.pop(0)\n        if len(target) == 0:\n            self._sorted_priorities.pop()\n        return ret\n\n    def get_work_chunk(self) -> List[T]:\n        \"\"\"Returns the best work for now\"\"\"\n        if len(self._sorted_priorities) == 0:\n            return []\n        best_priority = self._sorted_priorities.pop(0)\n        return self._queues.pop(best_priority)\n\n    def empty(self):\n        return len(self._sorted_priorities) == 0\n\n    def __init__(self):\n        self._sorted_priorities = []\n        self._queues = {}\n\n\n@registry.register\nclass AuSearch(Searcher):\n    # Deeper paths get done later\n    work: PriorityWorkQueue[int, Edge]\n\n    @staticmethod\n    def get_crackers_for(t: type):\n        return registry[Cracker[t]]\n\n    @lru_cache()  # To save extra sorting\n    def get_decoders_for(self, t: type):\n        ret = registry[Decoder[t]]\n        ret.sort(key=lambda x: x.priority(), reverse=True)\n        return ret\n\n    # def expand(self, edge: Edge) -> List[Edge]:\n    #     \"\"\"Evaluates the destination of the given, and adds its child edges to the pool\"\"\"\n    #     edge.dest = Node(parent=edge, level=edge.route(edge.source.level.result.value))\n\n    def expand_crackers(self, node: Node) -> None:\n        if node.depth >= self.max_cipher_depth:\n            return\n\n        res = node.level.result.value\n        additional_work = []\n\n        for i in self.get_crackers_for(type(res)):\n            inst = self._config()(i)\n            info = inst.getInfo(res)\n            if info.success_likelihood < self.p_threshold:\n                continue\n            additional_work.append(\n                Edge(source=node, route=inst, score=calculate_score(inst.getInfo(res)))\n            )\n\n        priority = min(node.depth, self.priority_cap)\n        if self.invert_priority:\n            priority = -priority\n\n        self.work.add_work(priority, additional_work)\n\n    def expand_decodings(self, node: Node) -> None:\n        val = node.level.result.value\n\n        for decoder in self.get_decoders_for(type(val)):\n            inst = self._config()(decoder)\n            res = inst(val)\n            if res is None:\n                continue\n            try:\n                new_node = Node.decoding(\n                    config=self._config(), route=inst, result=res, source=node\n                )\n            except DuplicateNode:\n                continue\n\n            logging.debug(\"Nesting encodings\")\n            self.recursive_expand(new_node, False)\n\n    def recursive_expand(self, node: Node, nested: bool = True) -> None:\n        if node.depth >= self.max_depth:\n            return\n\n        logging.debug(f\"Expanding depth {node.depth}\")\n\n        self.expand_decodings(node)\n\n        # Doing this last allows us to catch simple nested encodings faster\n        if not nested or self.enable_nested:\n            self.expand_crackers(node)\n\n    def search(self, ctext: Any) -> Optional[SearchResult]:\n        logging.debug(\n            f\"\"\"Beginning AuSearch with {\"inverted\" if self.invert_priority else \"normal\"} priority\"\"\"\n        )\n\n        try:\n            root = Node.root(self._config(), ctext)\n        except DuplicateNode:\n            return None\n\n        check_res = self._config().objs[\"checker\"](ctext)\n        if check_res is not None:\n            return SearchResult(check_res=check_res, path=[root.level])\n\n        try:\n            self.recursive_expand(root, False)\n\n            while True:\n                if self.work.empty():\n                    break\n                # Get the highest level result\n                chunk = self.work.get_work_chunk()\n                chunk.sort(key=lambda i: i.score)\n                # Work through all of this level's results\n                while len(chunk) != 0:\n                    logging.debug(f\"{len(chunk)} remaining on this level\")\n                    # TODO Cyclic uses some tricky C++ here\n                    # I know because it's sorted the one at the back (the anti-weight)\n                    # is the most likely\n\n                    edge: Edge = chunk.pop(-1)\n\n                    # Expand the node\n                    res = edge.route(edge.source.level.result.value)\n                    if res is None:\n                        continue\n                    for i in res:\n                        try:\n                            node = Node.cracker(\n                                config=self._config(), edge_template=edge, result=i\n                            )\n                            self.recursive_expand(node)\n                        except DuplicateNode:\n                            continue\n\n        except AuSearchSuccessful as e:\n            logging.info(\"AuSearch succeeded\")\n            return SearchResult(path=e.target.get_path(), check_res=e.info)\n\n        logging.info(\"AuSearch failed\")\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self._checker: Checker = config.objs[\"checker\"]\n        self.work = PriorityWorkQueue()  # Has to be defined here because of sharing\n        self.invert_priority = bool(\n            distutils.util.strtobool(self._params()[\"invert_priority\"])\n        )\n        self.priority_cap = int(self._params()[\"priority_cap\"])\n        self.enable_nested = bool(\n            distutils.util.strtobool(self._params()[\"enable_nested\"])\n        )\n        self.max_cipher_depth = int(self._params()[\"max_cipher_depth\"])\n        if self.max_cipher_depth == 0:\n            self.max_cipher_depth = math.inf\n        self.max_depth = int(self._params()[\"max_depth\"])\n        if self.max_depth == 0:\n            self.max_depth = math.inf\n        self.p_threshold = float(self._params()[\"p_threshold\"])\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"enable_nested\": ParamSpec(\n                req=False,\n                desc=\"Enables nested ciphers. \"\n                \"Incredibly slow, and not guaranteed to terminate\",\n                default=\"False\",\n            ),\n            \"invert_priority\": ParamSpec(\n                req=False,\n                desc=\"Causes more complex encodings to be looked at first. \"\n                \"Good for deeply buried encodings.\",\n                default=\"False\",\n            ),\n            \"max_cipher_depth\": ParamSpec(\n                req=False,\n                desc=\"The depth at which we stop trying to crack ciphers. \"\n                \"Set to 0 to disable\",\n                default=\"0\",\n            ),\n            \"max_depth\": ParamSpec(\n                req=False,\n                desc=\"The depth at which we give up. \" \n                     \"Set to 0 to disable\",\n                default=\"0\",\n            ),\n            \"priority_cap\": ParamSpec(\n                req=False,\n                desc=\"Sets the maximum depth before we give up ordering items.\",\n                default=\"2\",\n            ),\n            \"p_threshold\": ParamSpec(\n                req=False,\n                desc=\"Will skip any crackers which have less than this likelihood of succeeding. \"\n                     \"Set to 0 to disable\",\n                default=\"0.01\",\n            ),\n        }\n\n# Path: ciphey/basemods/Searchers/__init__.py\nfrom . import ausearch\n\n# Path: ciphey/basemods/__init__.py\nfrom . import Checkers, Crackers, Decoders, Resources, Searchers\n\n# Path: ciphey/ciphey.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\nhttps://github.com/Ciphey\nhttps://github.com/Ciphey/Ciphey/wiki\n\nThe cycle goes:\nmain -> argparsing (if needed) -> call_encryption -> new Ciphey object -> decrypt() -> produceProbTable ->\none_level_of_decryption -> decrypt_normal\n\"\"\"\nimport os\nimport warnings\nfrom typing import Any, Optional, Union\n\nimport click\nfrom appdirs import AppDirs\nimport logging\nfrom rich.logging import RichHandler\nfrom rich.console import Console\n\nfrom . import iface\n\nwarnings.filterwarnings(\"ignore\")\n\nconsole = Console()\n\n\ndef decrypt(config: iface.Config, ctext: Any) -> Union[str, bytes]:\n    \"\"\"A simple alias for searching a ctext and makes the answer pretty\"\"\"\n    res: Optional[iface.SearchResult] = config.objs[\"searcher\"].search(ctext)\n    if res is None:\n        return \"Failed to crack\"\n    if config.verbosity < 0:\n        return res.path[-1].result.value\n    else:\n        return iface.pretty_search_results(res)\n\n\ndef get_name(ctx, param, value):\n    # reads from stdin if value was not supplied\n    if not value and not click.get_text_stream(\"stdin\").isatty():\n        click.get_text_stream(\"stdin\").read().strip()\n        return click.get_text_stream(\"stdin\").read().strip()\n    else:\n        return value\n\n\ndef print_help(ctx):\n    # prints help menu\n    # if no \narguments are passed\n    click.echo(ctx.get_help())\n    ctx.exit()\n\n\n@click.command()\n@click.option(\n    \"-t\",\n    \"--text\",\n    help=\"The ciphertext you want to decrypt.\",\n    type=str,\n)\n@click.option(\n    \"-q\", \"--quiet\", help=\"Decrease verbosity\", type=int, count=True, default=None\n)\n@click.option(\n    \"-g\",\n    \"--greppable\",\n    help=\"Only print the answer (useful for grep)\",\n    type=bool,\n    is_flag=True,\n    default=None,\n)\n@click.option(\"-v\", \"--verbose\", count=True, type=int)\n@click.option(\"-C\", \"--checker\", help=\"Use the given checker\", default=None)\n@click.option(\n    \"-c\",\n    \"--config\",\n    help=\"Uses the given config file. Defaults to appdirs.user_config_dir('ciphey', 'ciphey')/'config.yml'\",\n)\n@click.option(\"-w\", \"--wordlist\", help=\"Uses the given wordlist\")\n@click.option(\n    \"-p\",\n    \"--param\",\n    help=\"Passes a parameter to the language checker\",\n    multiple=True,\n)\n@click.option(\n    \"-l\",\n    \"--list-params\",\n    help=\"List the parameters of the selected module\",\n    type=bool,\n)\n@click.option(\n    \"--searcher\",\n    help=\"Select the searching algorithm to use\",\n)\n# HARLAN TODO XXX\n# I switched this to a boolean flag system\n# https://click.palletsprojects.com/en/7.x/options/#boolean-flags\n# True for bytes input, False for str\n@click.option(\n    \"-b\",\n    \"--bytes\",\n    help=\"Forces ciphey to use binary mode for the input\",\n    is_flag=True,\n    default=None,\n)\n@click.option(\n    \"--default-dist\",\n    help=\"Sets the default character/byte distribution\",\n    type=str,\n    default=None,\n)\n@click.option(\n    \"-m\",\n    \"--module\",\n    help=\"Adds a module from the given path\",\n    type=click.Path(),\n    multiple=True,\n)\n@click.option(\n    \"-A\",\n    \"--appdirs\",\n    help=\"Print the location of where Ciphey wants the settings file to be\",\n    type=bool,\n    is_flag=True,\n)\n@click.option(\"-f\", \"--file\", type=click.File(\"rb\"), required=False)\n@click.argument(\"text_stdin\", callback=get_name, required=False)\ndef main(**kwargs):\n    \"\"\"Ciphey - Automated Decryption Tool\n\n    Documentation:\n    https://github.com/Ciphey/Ciphey/wiki\\n\n    Discord (support here, we're online most of the day):\n    http://discord.skerritt.blog\\n\n    GitHub:\n    https://github.com/ciphey/ciphey\\n\n\n    Ciphey is an automated decryption tool using smart artificial intelligence and natural language processing. Input encrypted text, get the decrypted text back.\n\n    Examples:\\n\n        Basic Usage: ciphey -t \"aGVsbG8gbXkgbmFtZSBpcyBiZWU=\"\n\n    \"\"\"\n\n    \"\"\"Function to deal with arguments. Either calls with args or not. Makes Pytest work.\n\n    It gets the arguments in the function definition using locals()\n    if withArgs is True, that means this is being called with command line args\n    so go to arg_parsing() to get those args\n    we then update locals() with the new command line args and remove \"withArgs\"\n    This function then calls call_encryption(**result) which passes our dict of args\n    to the function as its own arguments using dict unpacking.\n        Returns:\n            The output of the decryption.\n    \"\"\"\n\n    # if user wants to know where appdirs is\n    # print and exit\n    if \"appdirs\" in kwargs and kwargs[\"appdirs\"]:\n        dirs = AppDirs(\"Ciphey\", \"Ciphey\")\n        path_to_config = dirs.user_config_dir\n        print(\n            f\"The settings.yml file should be at {os.path.join(path_to_config, 'settings.yml')}\"\n        )\n        return None\n\n    # Now we create the config object\n    config = iface.Config()\n\n    # Load the settings file into the config\n    load_msg: str\n    cfg_arg = kwargs[\"config\"]\n    if cfg_arg is None:\n        # Make sure that the config dir actually exists\n        os.makedirs(iface.Config.get_default_dir(), exist_ok=True)\n        config.load_file(create=True)\n        load_msg = f\"Opened config file at {os.path.join(iface.Config.get_default_dir(), 'config.yml')}\"\n    else:\n        config.load_file(cfg_arg)\n        load_msg = f\"Opened config file at {cfg_arg}\"\n\n    # Load the verbosity, so that we can start logging\n    verbosity = kwargs[\"verbose\"]\n    quiet = kwargs[\"quiet\"]\n    if verbosity is None:\n        if quiet is not None:\n            verbosity = -quiet\n    elif quiet is not None:\n        verbosity -= quiet\n    if kwargs[\"greppable\"] is not None:\n        verbosity -= 999\n    # Use the existing value as a base\n    config.verbosity += verbosity\n    config.update_log_level(config.verbosity)\n    logging.info(load_msg)\n    logging.debug(f\"Got cmdline args {kwargs}\")\n\n    # Now we load the modules\n    module_arg = kwargs[\"module\"]\n    if module_arg is not None:\n        config.modules += list(module_arg)\n\n    # We need to load formats BEFORE we instantiate objects\n    if kwargs[\"bytes\"] is not None:\n        config.update_format(\"bytes\")\n\n    # Next, load the objects\n    params = kwargs[\"param\"]\n    if params is not None:\n        for i in params:\n            key, value = i.split(\"=\", 1)\n            parent, name = key.split(\".\", 1)\n            config.update_param(parent, name, value)\n    config.update(\"checker\", kwargs[\"checker\"])\n    config.update(\"searcher\", kwargs[\"searcher\"])\n    config.update(\"default_dist\", kwargs[\"default_dist\"])\n\n    config.complete_config()\n\n    logging.debug(f\"Command line opts: {kwargs}\")\n    logging.debug(f\"Config finalised: {config}\")\n\n    # Finally, we load the plaintext\n    if kwargs[\"text\"] is None:\n        if kwargs[\"file\"] is not None:\n            kwargs[\"text\"] = kwargs[\"file\"].read()\n        elif kwargs[\"text_stdin\"] is not None:\n            kwargs[\"text\"] = kwargs[\"text_stdin\"]\n        else:\n            # else print help menu\n            print(\"[bold red]Error. No inputs were given to Ciphey. [bold red]\")\n\n            @click.pass_context\n            def all_procedure(ctx):\n                print_help(ctx)\n\n            all_procedure()\n\n            return None\n\n    if issubclass(config.objs[\"format\"], type(kwargs[\"text\"])):\n        pass\n    elif config.objs[\"format\"] == str and isinstance(kwargs[\"text\"], bytes):\n        kwargs[\"text\"] = kwargs[\"text\"].decode(\"utf-8\")\n    elif config.objs[\"format\"] == bytes and isinstance(kwargs[\"text\"], str):\n        kwargs[\"text\"] = kwargs[\"text\"].encode(\"utf-8\")\n    else:\n        raise TypeError(f\"Cannot load type {config.format} from {type(kwargs['text'])}\")\n\n    result: Optional[str]\n\n    # if debug or quiet mode is on, run without spinner\n    if config.verbosity != 0:\n        result = decrypt(config, kwargs[\"text\"])\n    else:\n        # else, run with spinner if verbosity is 0\n        with console.status(\"[bold green]Thinking...\", spinner=\"moon\") as status:\n            config.set_spinner(status)\n            result = decrypt(config, kwargs[\"text\"])\n    if result is None:\n        result = \"Could not find any solutions.\"\n\n    console.print(result)\n\n# Path: ciphey/__init__.py\nfrom . import basemods, common, iface\nfrom .ciphey import decrypt\n\n# Path: ciphey/__main__.py\n#! /usr/bin/env python3\n\n\"\"\"\nCiphey: https://github.com/Ciphey/Ciphey\n\"\"\"\n\nimport platform\nimport sys\n\nif __name__ == \"__main__\":\n    major = sys.version_info[0]\n    minor = sys.version_info[1]\n\n    python_version = (\n        str(sys.version_info[0])\n        + \".\"\n        + str(sys.version_info[1])\n        + \".\"\n        + str(sys.version_info[2])\n    )\n\n    if major != 3 or minor < 6:\n        print(\n            f\"Ciphey requires Python 3.6+, you are using {python_version}. Please install a higher Python version. https://www.python.org/downloads/\"\n        )\n        print(\n            \"Alternatively, visit our Discord and use the Ciphey bot in #bots http://discord.skerritt.blog\"\n        )\n        sys.exit(1)\n    if platform.system() == \"Windows\":\n        if minor > 8:\n            print(\n                \"Ciphey does not currently support Python 3.9 on Windows. Please use the Discord bot at http://discord.skerritt.blog\"\n            )\n            sys.exit(1)\n\n        if sys.maxsize > 2 ** 32 is False:\n            print(\n                \"You are using Python 32 bit and Windows, Ciphey does not support this. Please upgrade to Python 64-bit here https://www.python.org/downloads/\"\n            )\n            sys.exit(1)\n    from .ciphey import main\n\n    main()\n\n# Path: ciphey/basemods/Checkers/entropy.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Entropy(Checker[str]):\n\n    \"\"\"\n    Uses entropy to determine plaintext\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying entropy checker\")\n        pass\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        # Uses benchmark from Discord\n        return 2e-7 * len(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/gtest.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass GTestChecker(Checker[str]):\n\n    \"\"\"\n    G-test of fitness, similar to Chi squared.\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying entropy checker\")\n        pass\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 4e-7 * len(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Crackers/hash.py\n\"\"\"\nThis is Hashbuster but slightly modified to work with Ciphey.\nWhy reinvent the wheel?\nChanges (that I can remember)\n* timeout set, as hashbuster took AGES before timeout was set.\nhttps://github.com/s0md3v/Hash-Buster\n\"\"\"\n\nimport re\nfrom typing import Dict, List, Optional\n\nimport requests\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, T, registry\n\nthread_count = 4\n\n\ndef alpha(ctext, hashtype):\n    return None\n\n\ndef beta(ctext, hashtype):\n    try:\n        response = requests.get(\n            \"https://hashtoolkit.com/reverse-hash/?hash=\", ctext, timeout=5\n        ).text\n    except requests.exceptions.ReadTimeout as e:\n        logging.info(f\"Beta failed timeout {e}\")\n    match = re.search(r'/generate-hash/?text=.*?\"', response)\n    if match:\n        return match.group(1)\n    return None\n\n\ndef gamma(ctext, hashtype):\n    try:\n        response = requests.get(\n            \"https://www.nitrxgen.net/md5db/\" + ctext, timeout=5\n        ).text\n    except requests.exceptions.ReadTimeout as e:\n        logging.info(f\"Gamma failed with {e}\")\n    if response:\n        return response\n    else:\n        return None\n\n\ndef delta(ctext, hashtype):\n    return None\n\n\ndef theta(ctext, hashtype):\n    try:\n        response = requests.get(\n            \"https://md5decrypt.net/Api/api.php?hash=%s&hash_type=%s&email=deanna_abshire@proxymail.eu&code=1152464b80a61728\"\n            % (ctext, hashtype),\n            timeout=5,\n        ).text\n    except requests.exceptions.ReadTimeout as e:\n        logging.info(f\"Gamma failed with {e}\")\n    if len(response) != 0:\n        return response\n    else:\n        return None\n\n\nmd5 = [gamma, alpha, beta, theta, delta]\nsha1 = [alpha, beta, theta, delta]\nsha256 = [alpha, beta, theta]\nsha384 = [alpha, beta, theta]\nsha512 = [alpha, beta, theta]\n\n\nresult = {}\n\n\ndef crack(ctext):\n    raise \"Error Crack is called\"\n\n\ndef threaded(ctext):\n    resp = crack(ctext)\n    if resp:\n        print(ctext + \" : \" + resp)\n        result[ctext] = resp\n\n\n@registry.register\nclass HashBuster(Cracker[str]):\n    @staticmethod\n    def getTarget() -> str:\n        return \"hash\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def getInfo(self, ctext: T) -> CrackInfo:\n        # TODO calculate these properly\n        return CrackInfo(\n            success_likelihood=0.5,\n            success_runtime=5,\n            failure_runtime=5,\n        )\n\n    def attemptCrack(self, ctext: T) -> List[CrackResult]:\n        logging.info(\"Starting to crack hashes\")\n        result = False\n\n        candidates = []\n        if len(ctext) == 32:\n            for api in md5:\n                r = api(ctext, \"md5\")\n                if result is not None or r is not None:\n                    logging.debug(\"MD5 returns True {r}\")\n                    candidates.append(result, \"MD5\")\n        elif len(ctext) == 40:\n            for api in sha1:\n                r = api(ctext, \"sha1\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha1 returns true\")\n                    candidates.append(result, \"SHA1\")\n        elif len(ctext) == 64:\n            for api in sha256:\n                r = api(ctext, \"sha256\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha256 returns true\")\n                    candidates.append(result, \"SHA256\")\n        elif len(ctext) == 96:\n            for api in sha384:\n                r = api(ctext, \"sha384\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha384 returns true\")\n                    candidates.append(result, \"SHA384\")\n        elif len(ctext) == 128:\n            for api in sha512:\n                r = api(ctext, \"sha512\")\n                if result is not None and r is not None:\n                    logging.debug(\"sha512 returns true\")\n                    candidates.append(result, \"SHA512\")\n\n        # TODO what the fuck is this code?\n        logging.debug(f\"Hash buster returning {result}\")\n        # TODO add to 5.1 make this return multiple possible candidates\n        return [CrackResult(value=candidates[0][0], misc_info=candidates[1][1])]\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Decoders/base58_flickr.py\nfrom typing import Dict, Optional\n\nimport base58\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base58_flickr(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base58 (Flickr) decoding\n        \"\"\"\n        FLICKR_ALPHABET = b\"123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ\"\n        try:\n            return base58.b58decode(ctext, alphabet=FLICKR_ALPHABET).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base58_flickr\"\n\n# Path: ciphey/basemods/Decoders/base64_url.py\nimport base64\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base64_url(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base64 URL decoding\n        \"\"\"\n        ctext_padding = ctext + \"=\" * (4 - len(ctext) % 4)\n        try:\n            return base64.urlsafe_b64decode(ctext_padding).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base64_url\"\n\n# Path: ciphey/basemods/Decoders/base65536.py\nfrom typing import Dict, Optional\n\nimport base65536\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base65536(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base65536 decoding\n        \"\"\"\n        try:\n            return base65536.decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base65536\"\n\n# Path: ciphey/basemods/Decoders/z85.py\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\nfrom zmq.utils import z85\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Z85(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Z85 decoding\n        \"\"\"\n        ctext_len = len(ctext)\n        if ctext_len % 5:\n            logging.debug(\n                f\"Failed to decode Z85 because length must be a multiple of 5, not '{ctext_len}'\"\n            )\n            return None\n        try:\n            return z85.decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n...\n", "needle_token_start": 10368, "needle_token_end": 10477, "code_context_ntokens": 16038, "output": ["```python\ndef get_name(ctx, param, value):\n    # reads from stdin if value was not supplied\n    if not value and not click.get_text_stream(\"stdin\").isatty():\n        click.get_text_stream(\"stdin\").read().strip()\n        return click.get_text_stream(\"stdin\").read().strip()\n    else:\n        return value\n```"]}
{"repo": "Ciphey/Ciphey", "name": "binary_to_ascii", "language": "python", "path": "ciphey/basemods/Crackers/xandy.py", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert binary data encoded as a string into ASCII text.\n2. **Input**: A string representing binary data.\n3. **Output**: The ASCII representation of the input binary string if conversion is successful; otherwise, an empty string if there is a decoding error.\n4. **Procedure**: The function first converts the binary string to an integer using base 2. It then calculates the necessary number of bytes to represent this integer and converts the integer to a bytearray. This bytearray is then attempted to be decoded into ASCII text. If successful, the ASCII text is returned; if a Unicode decoding error occurs, an empty string is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/basemods/Checkers/regex.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Regex(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = list(map(re.compile, self._params()[\"regex\"]))\n...\n# Path: ciphey/basemods/Checkers/what.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\nimport logging\nfrom rich.logging import RichHandler\nfrom pywhat import identifier\nfrom rich.console import Console\n\nconsole = Console()\n\n\n@registry.register\nclass What(Checker[str]):\n\n    \"\"\"\n    Uses PyWhat to determine plaintext with regexes\n    https://github.com/bee-san/pyWhat\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying PyWhat checker\")\n        returned_regexes = self.id.identify(ctext)\n        if returned_regexes[\"Regexes\"]:\n            matched_regex = returned_regexes[\"Regexes\"]['text'][0][\"Regex Pattern\"]\n\n            ret = f'The plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            human = (\n                f'\\nI think the plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            )\n\n            if \"Description\" in matched_regex and matched_regex[\"Description\"]:\n                s = matched_regex[\"Description\"]\n                # lowercases first letter so it doesn't look weird\n                s = f\", which is {s[0].lower() + s[1:]}\\n\"\n                ret += s\n                human += s\n\n            # if URL is attached, include that too.\n            if \"URL\" in matched_regex and matched_regex[\"URL\"]:\n                link = matched_regex[\"URL\"] + ctext.replace(\" \", \"\")\n                ret += f\"\\nClick here to view in browser [#CAE4F1][link={link}]{link}[/link][/#CAE4F1]\\n\"\n\n            # If greppable mode is on, don't print this\n            if self.config.verbosity >= 0:\n                # Print with full stop\n                console.print(human)\n            return ret\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.config = config\n        self.id = identifier.Identifier()\n\n# Path: ciphey/basemods/Checkers/ezcheck.py\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nfrom .brandon import Brandon\nfrom .format import JsonChecker\nfrom .human import HumanChecker\nfrom .quadgrams import Quadgrams\nfrom .regex import RegexList\nfrom .what import What\n\n\n@registry.register\nclass EzCheck(Checker[str]):\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns, followed by a human check\n    \"\"\"\n\n    def check(self, text: str) -> Optional[str]:\n        for checker in self.checkers:\n            res = checker.check(text)\n            if (\n                res is not None\n                and (self.decider is None or self.decider.check(text)) is not None\n            ):\n                return res\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        return sum(\n            i.getExpectedRuntime(text) for i in self.checkers\n        ) + self.decider.getExpectedRuntime(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        self.checkers: List[Checker[str]] = []\n        # Disable human checker for automated systems\n        if config.verbosity >= 0:\n            self.decider = config(HumanChecker)\n        else:\n            self.decider = None\n\n        # We need to modify the config for each of the objects\n\n        # First PyWhat, as it's the fastest\n        self.checkers.append(config(What))\n\n        # Next, the json checker\n        self.checkers.append(config(JsonChecker))\n\n        # Second to last, the quadgrams checker\n        self.checkers.append(config(Quadgrams))\n\n        # Finally, the Brandon checker, as it is the slowest\n        self.checkers.append(config(Brandon))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/quorum.py\nfrom typing import Dict, Generic, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, _registry\n\n\nclass Quorum(Generic[T], Checker[T]):\n    def check(self, text: T) -> Optional[str]:\n        left = self._params().k\n        results = []\n        for checker in self.checkers:\n            results.append(checker.check(text))\n            if results[-1] is None:\n                continue\n            left -= 1\n            # Early return check\n            if left == 0:\n                return str(results)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        if self._params().k is None:\n            k = len(self._params()[\"checker\"])\n        # These checks need to be separate, to make sure that we do not have zero members\n        if self._params().k == 0 or self._params().k > len(self._params()[\"checker\"]):\n            raise IndexError(\n                \"k must be between 0 and the number of checkers (inclusive)\"\n            )\n\n        self.checkers = []\n        for i in self._params()[\"checker\"]:\n            # This enforces type consistency\n            self.checkers.append(_registry.get_named(i, Checker[T]))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"checker\": ParamSpec(\n                req=True, desc=\"The checkers to be used for analysis\", list=True\n            ),\n            \"k\": ParamSpec(\n                req=False,\n                desc=\"The minimum quorum size. Defaults to the number of checkers\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/__init__.py\nfrom . import any, brandon, ezcheck, format, human, quadgrams, quorum, regex, what\n\n# Path: ciphey/common.py\n\"\"\"Some useful adapters\"\"\"\nfrom typing import Any\n\n\ndef id_lambda(value: Any):\n    \"\"\"\n    A function used in dynamic class generation that abstracts away a constant return value (like in getName)\n    \"\"\"\n    return lambda *args: value\n\n\ndef fix_case(target: str, base: str) -> str:\n    \"\"\"Returns the lower-case string target with the case of base\"\"\"\n    ret = \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n    return \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n\n# Path: ciphey/basemods/Crackers/affine.py\n# Community\n# by https://github.com/Ozzyz\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\nfrom ciphey.mathsHelper import mathsHelper\n\n\n@registry.register\nclass Affine(Cracker[str]):\n    \"\"\"\n    Each character in the Affine Cipher is encoded with the rule E(x) = (ax + b) mod m\n    m is the size of the alphabet, while a and b are the keys in the cipher. a must be coprime to b.\n    The Caesar cipher is a specific case of the Affine Cipher, with a=1 and b being the shift of the cipher.\n    Decryption is performed by D(x) = a_inv (x - b) mod m where a_inv is the modular multiplicative inverse of a mod m.\n\n    In this version of the Affine Cipher, we do not allow alphabets with several instances of the same letter in different cases.\n    For instance, the alphabet 'ABCdef123' is allowed, but 'AaBbCc' is not.\n    \"\"\"\n\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"affine\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Brute forces all the possible combinations of a and b to attempt to crack the cipher.\n        \"\"\"\n        logging.debug(\"Attempting affine\")\n        candidates = []\n\n        # a and b are coprime if gcd(a,b) is 1.\n        possible_a = [\n            a\n            for a in range(1, self.alphabet_length)\n            if mathsHelper.gcd(a, self.alphabet_length) == 1\n        ]\n        logging.info(\n            f\"Trying Affine Cracker with {len(possible_a)} a-values and {self.alphabet_length} b-values\"\n        )\n\n        for a in possible_a:\n            a_inv = mathsHelper.mod_inv(a, self.alphabet_length)\n            # If there is no inverse, we cannot decrypt the text\n            if a_inv is None:\n                continue\n            for b in range(self.alphabet_length):\n                # Pass in lowered text. This means that we expect alphabets to not contain both 'a' and 'A'.\n                translated = self.decrypt(ctext.lower(), a_inv, b, self.alphabet_length)\n\n                candidate_probability = self.plaintext_probability(translated)\n                if candidate_probability > self.plaintext_prob_threshold:\n                    candidates.append(\n                        CrackResult(\n                            value=fix_case(translated, ctext), key_info=f\"a={a}, b={b}\"\n                        )\n                    )\n        logging.info(f\"Affine Cipher returned {len(candidates)} candidates\")\n        return candidates\n\n    def plaintext_probability(self, translated: str) -> float:\n        \"\"\"\n        Analyses the translated text and applies the chi squared test to see if it is a probable plaintext candidate\n        Returns the probability of the chi-squared test.\n        \"\"\"\n        analysis = cipheycore.analyse_string(translated)\n        return cipheycore.chisq_test(analysis, self.expected)\n\n    def decrypt(self, text: str, a_inv: int, b: int, m: int) -> str:\n        \"\"\"\n        Each letter is decrypted at D(x) = a_inv (x - b) mod m where x is the char\n        We treat the char value as its index in the alphabet, so if\n        the alphabet is 'abcd....' and the char is 'b', it has the value 1.\n        \"\"\"\n        return \"\".join([self.decryptChar(char, a_inv, b, m) for char in text])\n\n    def decryptChar(self, char: str, a_inv: int, b: int, m: int) -> str:\n\n        # We lower the alphabet since both ctext and alphabet need to be in the same case in order\n        # to perform the shifts. The translated text will have fixed case after the translation anyways.\n        # This is only necessary if the specified alphabet is uppercase.\n        alphabet = [x.lower() for x in self.group]\n\n        # Preserve characters that are not in alphabet\n        if char not in alphabet:\n            return char\n        char_idx = alphabet.index(char)\n        decrypted_char_idx = (a_inv * (char_idx - b)) % m\n        return alphabet[decrypted_char_idx]\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.alphabet_length = len(self.group)\n        self.cache = config.cache\n        self.plaintext_prob_threshold = 0.01\n\n# Path: ciphey/basemods/Crackers/ascii_shift.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Ascii_shift(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"ascii_shift\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ASCII shift cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ASCII shift returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ASCII shift cipher alphabet\",\n                req=False,\n                default=\"\"\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/baconian.py\nimport re\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\nimport logging\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Baconian(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"baconian\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to decode both variants of the Baconian cipher.\n        \"\"\"\n        logging.debug(\"Attempting Baconian cracker\")\n        candidates = []\n        result = []\n        ctext_decoded = \"\"\n        ctext_decoded2 = \"\"\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext only contains A and B\n        if bool(re.search(r\"[^AB]\", ctext)) is True:\n            logging.debug(\"Failed to crack baconian due to non baconian character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 5\n        ctext_len = len(ctext)\n        if ctext_len % 5:\n            logging.debug(\n                f\"Failed to decode Baconian because length must be a multiple of 5, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 5\n        ctext = \" \".join(ctext[i : i + 5] for i in range(0, len(ctext), 5))\n        ctext_split = ctext.split(\" \")\n        baconian_keys = self.BACONIAN_DICT.keys()\n\n        # Decode I=J and U=V variant\n        for i in ctext_split:\n            if i in baconian_keys:\n                ctext_decoded += self.BACONIAN_DICT[i]\n\n        # Decode variant that assigns each letter a unique code\n        for i in ctext_split:\n            if \"+\" + i in baconian_keys:\n                ctext_decoded2 += self.BACONIAN_DICT[\"+\" + i]\n\n        candidates.append(ctext_decoded)\n        candidates.append(ctext_decoded2)\n        for candidate in candidates:\n            if candidate != \"\":\n                if candidate == candidates[0]:\n                    result.append(CrackResult(value=candidate, key_info=\"I=J & U=V\"))\n                else:\n                    result.append(CrackResult(value=candidate))\n        logging.debug(f\"Baconian cracker - Returning results: {result}\")\n        return result\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"dict\": ParamSpec(\n                desc=\"The Baconian alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::baconian\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BACONIAN_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n\n# Path: ciphey/basemods/Crackers/caesar.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Caesar(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"caesar\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying caesar cipher on {ctext}\")\n        # Convert it to lower case\n        #\n        # TODO: handle different alphabets\n        if self.lower:\n            message = ctext.lower()\n        else:\n            message = ctext\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"Caesar returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(message, candidate.key, self.group)\n            candidates.append(\n                CrackResult(value=fix_case(translated, ctext), key_info=candidate.key)\n            )\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the caesar cipher alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n            \"lower\": ParamSpec(\n                desc=\"Whether or not the ciphertext should be converted to lowercase first\",\n                req=False,\n                default=True,\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.lower: Union[str, bool] = self._params()[\"lower\"]\n        if not isinstance(self.lower, bool):\n            self.lower = util.strtobool(self.lower)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/rot47.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Rot47(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"rot47\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ROT47 cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ROT47 returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ROT47 cipher alphabet\",\n                req=False,\n                default=\"\"\"!\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/soundex.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\n\n\n@registry.register\nclass Soundex(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"soundex\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to crack Soundex by generating all possible combinations.\n        \"\"\"\n        logging.debug(\"Attempting Soundex cracker\")\n        word_list = []\n        sentences = []\n        result = []\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext contains only A-Z and 0-9\n        if bool(re.search(r\"[^A-Z0-9]\", ctext)) is True:\n            logging.debug(\"Failed to crack soundex due to non soundex character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 4\n        ctext_len = len(ctext)\n        if ctext_len % 4:\n            logging.debug(\n                f\"Failed to decode Soundex because length must be a multiple of 4, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 4\n        ctext = \" \".join(ctext[i : i + 4] for i in range(0, len(ctext), 4))\n        ctext_split = ctext.split(\" \")\n        soundex_keys = self.SOUNDEX_DICT.keys()\n\n        # Find all words that correspond to each given soundex code\n        for code in ctext_split:\n            if code in soundex_keys:\n                word_list.append(self.SOUNDEX_DICT[code])\n\n        logging.info(f\"Possible words for given encoded text: {word_list}\")\n\n        # Find all possible sentences\n        self.getSentenceCombo(\n            word_list,\n            sentences,\n            self.frequency_dict,\n            self.sentence_freq,\n            self.word_freq,\n        )\n\n        sorted_sentences = self.sortlistwithdict(sentences, self.frequency_dict)\n\n        for sentence in sorted_sentences:\n            result.append(CrackResult(value=sentence))\n\n        logging.debug(f\"Soundex cracker - Returning results: {result}\")\n        return result\n\n    def sortlistwithdict(self, listtosort, hashes):\n        \"\"\"\n        This function uses the sum of ranks (based on frequency) of each word in each\n        sentence and sorts them according to it.\n        \"\"\"\n        return sorted(listtosort, key=lambda x: hashes[x])\n\n    def getSentenceCombo(\n        self, A, sentences, frequency_dict, sentence_freq, word_freq, result=\"\", n=0\n    ):\n        \"\"\"\n        This function uses recursion to generate a list of sentences from all possible\n        words for a given set of soundex codes.\n        \"\"\"\n        logging.debug(\"Creating all possible sentences from Soundex\")\n        if n == len(A):\n            sentences.append(result[1:])\n            for word in result[1:].split():\n                # Adding the rank of each word to find out the sentence's net frequency\n                if word in word_freq:\n                    sentence_freq += word_freq.index(word)\n                # If the word isn't in the frequency list then it's a very uncommon word\n                # so we add a large number (5000)\n                else:\n                    sentence_freq += 5000\n            frequency_dict[result[1:]] = sentence_freq\n            sentence_freq = 0\n            return\n\n        for word in A[n]:\n            out = result + \" \" + word\n            self.getSentenceCombo(\n                A, sentences, frequency_dict, sentence_freq, word_freq, out, n + 1\n            )\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The Soundex dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::soundex\",\n            ),\n            \"freq\": ParamSpec(\n                desc=\"The word frequency dictionary to use\",\n                req=False,\n                default=\"cipheydists::list::English5000Freq\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.SOUNDEX_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.word_freq = config.get_resource(self._params()[\"freq\"], Translation)\n        self.frequency_dict = {}\n        self.sentence_freq = 0\n\n# Path: ciphey/basemods/Crackers/vigenere.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Vigenere(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        if self.keysize is not None:\n            analysis = self.cache.get_or_update(\n                ctext,\n                f\"vigenere::{self.keysize}\",\n                lambda: cipheycore.analyse_string(\n                    ctext.lower(), self.keysize, self.group\n                ),\n            )\n\n            val = cipheycore.vigenere_detect(analysis, self.expected)\n\n            logging.info(f\"Vigenere has likelihood {val}\")\n\n            return CrackInfo(\n                success_likelihood=val,\n                # TODO: actually calculate runtimes\n                success_runtime=1e-3,\n                failure_runtime=1e-2,\n            )\n\n        likely_lens = self.cache.get_or_update(\n            ctext,\n            \"vigenere::likely_lens\",\n            lambda: cipheycore.vigenere_likely_key_lens(\n                ctext.lower(), self.expected, self.group, self.detect_p_value\n            ),\n        )\n\n        # Filter out the lens that make no sense\n        likely_lens = [i for i in likely_lens if i.len <= self.max_key_length]\n\n        for keysize in likely_lens:\n            # Store the analysis\n            analysis = self.cache.get_or_update(\n                ctext, f\"vigenere::{keysize.len}\", lambda: keysize.tab\n            )\n        if len(likely_lens) == 0:\n            return CrackInfo(\n                success_likelihood=0,\n                # TODO: actually calculate runtimes\n                success_runtime=2e-3,\n                failure_runtime=2e-2,\n            )\n\n        logging.info(\n            f\"Vigenere has likelihood {likely_lens[0].p_value} with lens {[i.len for i in likely_lens]}\"\n        )\n\n        return CrackInfo(\n            success_likelihood=likely_lens[0].p_value,\n            # TODO: actually calculate runtimes\n            success_runtime=2e-4,\n            failure_runtime=2e-4,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"vigenere\"\n\n    def crackOne(\n        self, ctext: str, analysis: cipheycore.windowed_analysis_res, real_ctext: str\n    ) -> List[CrackResult]:\n        possible_keys = cipheycore.vigenere_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n        if len(possible_keys) > self.clamp:\n            possible_keys = possible_keys[: self.clamp]\n        logging.debug(\n            f\"Vigenere crack got keys: {[[i for i in candidate.key] for candidate in possible_keys]}\"\n        )\n        return [\n            CrackResult(\n                value=fix_case(\n                    cipheycore.vigenere_decrypt(ctext, candidate.key, self.group),\n                    real_ctext,\n                ),\n                key_info=\"\".join([self.group[i] for i in candidate.key]),\n                misc_info=f\"p-value was {candidate.p_value}\",\n            )\n            for candidate in possible_keys[: min(len(possible_keys), 10)]\n        ]\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(\"Trying vigenere cipher\")\n        # Convert it to lower case\n        if self.lower:\n            message = ctext.lower()\n        else:\n            message = ctext\n\n        # Analysis must be done here, where we know the case for the cache\n        if self.keysize is not None:\n            return self.crackOne(\n                message,\n                self.cache.get_or_update(\n                    ctext,\n                    f\"vigenere::{self.keysize}\",\n                    lambda: cipheycore.analyse_string(\n                        message, self.keysize, self.group\n                    ),\n                ),\n                ctext,\n            )\n\n        arrs = []\n        likely_lens = self.cache.get_or_update(\n            ctext,\n            \"vigenere::likely_lens\",\n            lambda: cipheycore.vigenere_likely_key_lens(\n                message, self.expected, self.group\n            ),\n        )\n        possible_lens = [i for i in likely_lens]\n        possible_lens.sort(key=lambda i: i.p_value)\n        logging.debug(f\"Got possible lengths {[i.len for i in likely_lens]}\")\n        # TODO: work out length\n        for i in possible_lens:\n            arrs.extend(\n                self.crackOne(\n                    message,\n                    self.cache.get_or_update(\n                        ctext,\n                        f\"vigenere::{i.len}\",\n                        lambda: cipheycore.analyse_string(message, i.len, self.group),\n                    ),\n                    ctext,\n                )\n            )\n\n        logging.info(f\"Vigenere returned {len(arrs)} candidates\")\n        return arrs\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the caesar cipher alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n            \"lower\": ParamSpec(\n                desc=\"Whether or not the ciphertext should be converted to lowercase first\",\n                req=False,\n                default=True,\n            ),\n            \"keysize\": ParamSpec(\n                desc=\"A key size that should be used. If not given, will attempt to work it out\",\n                req=False,\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for windowed frequency analysis\",\n                req=False,\n                default=0.5,\n            ),\n            \"detect_p_value\": ParamSpec(\n                desc=\"The p-value to use for the detection of Vigenere length\",\n                req=False,\n                default=0.01,\n            ),\n            \"clamp\": ParamSpec(\n                desc=\"The maximum number of candidates that can be returned per key len\",\n                req=False,\n                default=10,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.lower: Union[str, bool] = self._params()[\"lower\"]\n        if not isinstance(self.lower, bool):\n            self.lower = util.strtobool(self.lower)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.keysize = self._params().get(\"keysize\")\n        if self.keysize is not None:\n            self.keysize = int(self.keysize)\n        self.p_value = float(self._params()[\"p_value\"])\n        self.detect_p_value = float(self._params()[\"detect_p_value\"])\n        self.clamp = int(self._params()[\"clamp\"])\n        self.max_key_length = 16\n\n# Path: ciphey/basemods/Crackers/xandy.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Xandy(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    \ndef binary_to_ascii(variant):\n        # Convert the binary string to an integer with base 2\n        binary_int = int(variant, 2)\n        byte_number = binary_int.bit_length() + 7 // 8\n\n        # Convert the resulting int to a bytearray and then decode it to ASCII text\n        binary_array = binary_int.to_bytes(byte_number, \"big\")\n        try:\n            ascii_text = binary_array.decode()\n            logging.debug(f\"Found possible solution: {ascii_text[:32]}\")\n            return ascii_text\n        except UnicodeDecodeError as e:\n            logging.debug(f\"Failed to crack X-Y due to a UnicodeDecodeError: {e}\")\n            return \"\"\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"xandy\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Checks an input if it only consists of two or three different letters.\n        If this is the case, it attempts to regard those letters as\n        0 and 1 (with the third characters as an optional delimiter) and then\n        converts it to ASCII text.\n        \"\"\"\n        logging.debug(\"Attempting X-Y replacement\")\n        variants = []\n        candidates = []\n        result = []\n\n        # Convert the ctext to all-lowercase and regex-match & replace all whitespace\n        ctext = re.sub(r\"\\s+\", \"\", ctext.lower(), flags=re.UNICODE)\n\n        # cset contains every unique value in the ctext\n        cset = list(set(list(ctext)))\n        cset_len = len(cset)\n\n        if not 1 < cset_len < 4:\n            # We only consider inputs with two or three unique values\n            logging.debug(\n                \"Failed to crack X-Y due to not containing two or three unique values\"\n            )\n            return None\n\n        logging.debug(f\"String contains {cset_len} unique values: {cset}\")\n\n        # In case of three unique values, we regard the least frequent character as the delimiter\n        if cset_len == 3:\n            # Count each unique character in the set to determine the least frequent one\n            counting_list = []\n            for char in cset:\n                counting_list.append(ctext.count(char))\n            val, index = min((val, index) for (index, val) in enumerate(counting_list))\n            delimiter = cset[index]\n            logging.debug(\n                f\"{delimiter} occurs {val} times and is the probable delimiter\"\n            )\n            # Remove the delimiter from the ctext and compute new cset\n            ctext = ctext.replace(delimiter, \"\")\n            cset = list(set(list(ctext)))\n\n        # Form both variants of the substitution\n        for i in range(2):\n            if i:\n                variants.append(ctext.replace(cset[0], \"1\").replace(cset[1], \"0\"))\n            else:\n                variants.append(ctext.replace(cset[0], \"0\").replace(cset[1], \"1\"))\n\n        # Apply function to both variants and strip stray NULL characters\n        for variant in variants:\n            candidates.append(self.binary_to_ascii(variant).strip(\"\\x00\"))\n        for i, candidate in enumerate(candidates):\n            if candidate != \"\":\n                keyinfo = f\"{cset[0]} -> {i} & {cset[1]} -> {str(int(not i))}\"\n                result.append(CrackResult(value=candidate, key_info=keyinfo))\n                logging.debug(f\"X-Y cracker - Returning results: {result}\")\n                return result\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            )\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n\n# Path: ciphey/basemods/Crackers/xortool.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: bee-san\n\"\"\"\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom xortool_ciphey import tool_main\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass XorTool(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            # TODO: actually calculate runtimes\n            success_runtime=1e-8,\n            failure_runtime=1e-8,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"xortool\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.debug(\"Trying xortool cipher\")\n        # TODO handle different charsets\n        # TODO allow more config over xortool\n\n        logging.debug(f\"{ctext}\")\n\n        # https://github.com/Ciphey/xortool/discussions/4\n        # for docs on this function\n        try:\n            result = tool_main.api(str.encode(ctext))\n        except:\n            logging.debug(\"Xor failed.\")\n            return\n\n        result = CrackResult(value=result[1][\"Dexored\"], key_info=result[0][\"keys\"])\n\n        return [result]\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            ),\n        }\n\n    @staticmethod\n    def score_utility() -> float:\n        return 1.5\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = self._params()[\"p_value\"]\n\n# Path: ciphey/basemods/Crackers/__init__.py\nfrom . import (\n    affine,\n    ascii_shift,\n    baconian,\n    caesar,\n    rot47,\n    soundex,\n    vigenere,\n    xandy,\n    xortool,\n)\n\n# Path: ciphey/basemods/Decoders/a1z26.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass A1z26(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs A1Z26 decoding\n        \"\"\"\n        logging.debug(\"Attempting A1Z26\")\n        ctext_converted = []\n        ctext_split = re.split(r\"[ ,;:\\-\\n]\", ctext)\n        delimiters = set(sorted(re.sub(r\"[^ ,;:\\-\\n]\", \"\", ctext)))\n        ctext_num = re.sub(r\"[,;:\\-\\s]\", \"\", ctext)\n        ctext_decoded = \"\"\n        if ctext_num.isnumeric() is False:\n            logging.debug(\"Failed to decode A1Z26 due to non numeric character(s)\")\n            return None\n        try:\n            for i in ctext_split:\n                val = int(i)\n                if val > 26 or val < 1:\n                    logging.debug(\n                        f\"Failed to decode A1Z26 due to invalid number '{val}'\"\n                    )\n                    return None\n                val2 = int(i) + 96\n                ctext_converted.append(chr(val2))\n            ctext_decoded = \"\".join(ctext_converted)\n            logging.info(\n                f\"A1Z26 successful, returning '{ctext_decoded}' with delimiter(s) {delimiters}\"\n            )\n            return ctext_decoded\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"a1z26\"\n\n# Path: ciphey/basemods/Decoders/atbash.py\nfrom typing import Dict, Optional\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, WordList, registry\n\n\n@registry.register\nclass Atbash(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Takes an encoded string and attempts to decode it according to the Atbash cipher.\n\n        The Atbash cipher is a very simple substitution cipher without a key.\n        It operates by replacing every letter in the input by its 'counterpoint'\n        in the alphabet. Example: A -> Z, B -> Y, ... , M -> N and vice versa.\n        \"\"\"\n\n        result = \"\"\n        atbash_dict = {self.ALPHABET[i]: self.ALPHABET[::-1][i] for i in range(26)}\n\n        for letter in ctext.lower():\n            if letter in atbash_dict.keys():\n                # Match every letter of the input to its atbash counterpoint\n                result += atbash_dict[letter]\n            else:\n                # If the current character is not in the defined alphabet,\n                # just accept it as-is (useful for numbers, punctuation, etc.)\n                result += letter\n        return fix_case(result, ctext)\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.1\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.ALPHABET = config.get_resource(self._params()[\"dict\"], WordList)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The alphabet used for the atbash operation.\",\n                req=False,\n                default=\"cipheydists::list::englishAlphabet\",\n            )\n        }\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"atbash\"\n\n# Path: ciphey/basemods/Decoders/base58_bitcoin.py\nfrom typing import Dict, Optional\n\nimport base58\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base58_bitcoin(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base58 (Bitcoin) decoding\n        \"\"\"\n        try:\n            return base58.b58decode(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base58_bitcoin\"\n\n# Path: ciphey/basemods/Decoders/base58_ripple.py\nfrom typing import Dict, Optional\n\nimport base58\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base58_ripple(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base58 (Ripple) decoding\n        \"\"\"\n        try:\n            return base58.b58decode(ctext, alphabet=base58.RIPPLE_ALPHABET).decode(\n                \"utf-8\"\n            )\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base58_ripple\"\n\n# Path: ciphey/basemods/Decoders/base62.py\nfrom typing import Dict, Optional\n\nimport base62\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, registry\n\n\n@registry.register\nclass Base62(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base62 decoding\n        \"\"\"\n        try:\n            return base62.decodebytes(ctext).decode(\"utf-8\")\n        except Exception:\n            return None\n\n    @staticmethod\n    def priority() -> float:\n        # Not expected to show up often, but also very fast to check.\n        return 0.05\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"base62\"\n\n# Path: ciphey/basemods/Decoders/base69.py\n# Translated to Python and adapted for Ciphey from the JS original at https://github.com/pshihn/base69\n\n\nimport re\nfrom math import ceil\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Config, Decoder, ParamSpec, T, U, WordList, registry\n\n\n@registry.register\nclass Base69(Decoder[str]):\n    def decode(self, ctext: T) -> Optional[U]:\n        \"\"\"\n        Performs Base69 decoding\n        \"\"\"\n        # Remove whitespace\n        try:\n            ctext = re.sub(r\"\\s+\", \"\", ctext, flags=re.UNICODE)\n            extra_bytes = 0\n            clen = len(ctext)\n\n            if ctext[:-1] == \"=\":\n                extra_bytes = int(ctext[clen - 2])\n\n            CHUNK_COUNT = ceil(clen / 16)\n            result = [0 for _ in range(CHUNK_COUNT * 7 - extra_bytes)]\n\n            for i in range(CHUNK_COUNT):\n...\n", "needle_token_start": 12001, "needle_token_end": 12183, "code_context_ntokens": 16098, "output": ["```python\ndef binary_to_ascii(variant):\n    # Convert the binary string to an integer with base 2\n    binary_int = int(variant, 2)\n    byte_number = binary_int.bit_length() + 7 // 8\n\n    # Convert the resulting int to a bytearray and then decode it to ASCII text\n    binary_array = binary_int.to_bytes(byte_number, \"big\")\n    try:\n        ascii_text = binary_array.decode()\n        logging.debug(f\"Found possible solution: {ascii_text[:32]}\")\n        return ascii_text\n    except UnicodeDecodeError as e:\n        logging.debug(f\"Failed to crack X-Y due to a UnicodeDecodeError: {e}\")\n        return \"\"\n```"]}
{"repo": "Ciphey/Ciphey", "name": "getSentenceCombo", "language": "python", "path": "ciphey/basemods/Crackers/soundex.py", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to recursively generate all possible sentences from a list of lists of words, each corresponding to a set of soundex codes, and calculate their frequency scores based on word usage.\n2. **Input**: The function takes a list of lists where each sublist contains words that match a soundex code, a list to store sentences, a dictionary to map sentences to their frequency scores, and two dictionaries for word and sentence frequencies. It also takes a string to accumulate words into sentences and an index to track recursion depth.\n3. **Output**: It does not return a value directly but populates the provided list with all possible sentence combinations and updates the dictionary with their corresponding frequency scores.\n4. **Procedure**: The function uses recursion to explore all combinations of words from the input list. For each word in the current sublist, it appends the word to the ongoing sentence and calls itself with the next index. Once the end of the list is reached, the complete sentence is stored, and its frequency score is calculated by summing the ranks of its words from the frequency dictionary. If a word is not found in the dictionary, a high penalty score is added to denote rarity.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/basemods/Checkers/brandon.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to determine whether something is English or not.\n1. Calculate the Chi Squared score of a sentence\n2. If the score is significantly lower than the average score, it _might_ be English\n    2.1. If the score _might_ be English, then take the text and compare it to the sorted dictionary\n    in O(n log n) time.\n    It creates a percentage of \"How much of this text is in the dictionary?\"\n    The dictionary contains:\n        * 20,000 most common US words\n        * 10,000 most common UK words (there's no repetition between the two)\n        * The top 10,000 passwords\n    If the word \"Looks like\" English (chi-squared) and if it contains English words, we can conclude it is\n    very likely English. The alternative is doing the dictionary thing but with an entire 479k word dictionary (slower)\n    2.2. If the score is not English, but we haven't tested enough to create an average, then test it against\n     the dictionary\n\nThings to optimise:\n* We only run the dictionary if it's 20% smaller than the average for chi squared\n* We consider it \"English\" if 45% of the text matches the dictionary\n* We run the dictionary if there is less than 10 total chisquared test\n\nHow to add a language:\n* Download your desired dictionary. Try to make it the most popular words, for example. Place this file into this\n folder with languagename.txt\nAs an example, this comes built in with english.txt\nFind the statistical frequency of each letter in that language.\nFor English, we have:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n}\nIn chisquared.py\nTo add your language, do:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n    \"German\": [0.0973]\n}\nIn alphabetical order\nAnd you're.... Done! Make sure the name of the two match up\n\"\"\"\nimport sys\nfrom math import ceil\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nsys.path.append(\"..\")\ntry:\n    import mathsHelper as mh\nexcept ModuleNotFoundError:\n    import ciphey.mathsHelper as mh\n\n\n@registry.register\nclass Brandon(Checker[str]):\n    \"\"\"\n    Class designed to confirm whether something is **language** based on how many words of **language** appears\n    Call confirmLanguage(text, language)\n    * text: the text you want to confirm\n    * language: the language you want to confirm\n\n    Find out what language it is by using chisquared.py, the highest chisquared score is the language\n    languageThreshold = 45\n    if a string is 45% **language** words, then it's confirmed to be english\n    \"\"\"\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually work this out\n        # TODO its 0.2 seconds on average\n        return 1e-4  # 100 \u00b5s\n\n    wordlist: set\n\n    def clean_text(self, text: str) -> set:\n        \"\"\"Cleans the text ready to be checked\n\n        Strips punctuation, makes it lower case, turns it into a set separated by spaces, removes duplicate words\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            text -> the text as a list, now cleaned\n\n        \"\"\"\n        # makes the text unique words and readable\n        text = text.lower()\n        text = self.mh.strip_punctuation(text)\n        text = text.split(\" \")\n        text = filter(lambda x: len(x) > 2, text)\n        text = set(text)\n        return text\n\n    def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        \"\"\"Given text determine if it passes checker\n\n        The checker uses the variable passed to it. I.E. Stopwords list, 1k words, dictionary\n\n        Args:\n            text -> The text to check\n            threshold -> at what point do we return True? The percentage of text that is in var before we return True\n            text_length -> the length of the text\n            var -> the variable we are checking against. Stopwords list, 1k words list, dictionary list.\n        Returns:\n            boolean -> True for it passes the test, False for it fails the test.\"\"\"\n        if text is None:\n            logging.debug(\"Checker's text is None, so returning False\")\n            return False\n        if var is None:\n            logging.debug(\"Checker's input var is None, so returning False\")\n            return False\n\n        percent = ceil(text_length * threshold)\n        logging.debug(f\"Checker's chunks are size {percent}\")\n        meet_threshold = 0\n        location = 0\n        end = percent\n\n        if text_length <= 0:\n            return False\n\n        while location <= text_length:\n            # chunks the text, so only gets THRESHOLD chunks of text at a time\n            text = list(text)\n            to_analyse = text[location:end]\n            logging.debug(f\"To analyse is {to_analyse}\")\n            for word in to_analyse:\n                # if word is a stopword, + 1 to the counter\n                if word in var:\n                    logging.debug(\n                        f\"{word} is in var, which means I am +=1 to the meet_threshold which is {meet_threshold}\"\n                    )\n                    meet_threshold += 1\n                meet_threshold_percent = meet_threshold / text_length\n                if meet_threshold_percent >= threshold:\n                    logging.debug(\n                        f\"Returning true since the percentage is {meet_threshold / text_length} and the threshold is {threshold}\"\n                    )\n                    # if we meet the threshold, return True\n                    # otherwise, go over again until we do\n                    # We do this in the for loop because if we're at 24% and THRESHOLD is 25\n                    # we don't want to wait THRESHOLD to return true, we want to return True ASAP\n                    return True\n            location = end\n            end = end + percent\n        logging.debug(\n            f\"The language proportion {meet_threshold_percent} is under the threshold {threshold}\"\n        )\n        return False\n\n    def __init__(self, config: Config):\n        # Suppresses warning\n        super().__init__(config)\n        self.mh = mh.mathsHelper()\n\n        phases = config.get_resource(self._params()[\"phases\"])\n\n        self.thresholds_phase1 = phases[\"1\"]\n        self.thresholds_phase2 = phases[\"2\"]\n        self.top1000Words = config.get_resource(self._params().get(\"top1000\"))\n        self.wordlist = config.get_resource(self._params()[\"wordlist\"])\n        self.stopwords = config.get_resource(self._params().get(\"stopwords\"))\n\n        self.len_phase1 = len(self.thresholds_phase1)\n        self.len_phase2 = len(self.thresholds_phase2)\n\n    def check(self, text: str) -> Optional[str]:\n        \"\"\"Checks to see if the text is in English\n\n        Performs a decryption, but mainly parses the internal data packet and prints useful information.\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            bool -> True if the text is English, False otherwise.\n\n        \"\"\"\n        logging.debug(f'In Language Checker with \"{text}\"')\n        text = self.clean_text(text)\n        logging.debug(f'Text split to \"{text}\"')\n        if text == \"\":\n            logging.debug(\"Returning None from Brandon as the text cleaned is none.\")\n            return None\n\n        length_text = len(text)\n\n        what_to_use = {}\n...\n# Path: ciphey/basemods/Checkers/format.py\nimport json\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass JsonChecker(Checker[str]):\n\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying json checker\")\n\n        # https://github.com/Ciphey/Ciphey/issues/389\n        if text.isdigit():\n            return None\n\n        try:\n            json.loads(text)\n            return \"\"\n        except ValueError:\n            return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 1e-7 * len(text)  # From benchmarks I found online\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/human.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, registry\nfrom rich.console import Console\nfrom rich.markup import escape\n\nconsole = Console()\n\n\n@registry.register\nclass HumanChecker(Checker[str]):\n\n    \"\"\"\n    Uses the person's decision to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: str) -> Optional[str]:\n        with self._config().pause_spinner_handle():\n            response = console.input(\n                f\"Possible plaintext: [blue bold]{escape(ctext.__repr__())}[/blue bold] ([green]y[/green]/[red]N[/red]): \"\n            )\n        if response == \"y\":\n            return \"\"\n        elif response in (\"n\", \"\"):\n            return None\n        else:\n            return self.check(ctext)\n\n    def getExpectedRuntime(self, text: str) -> float:\n        return 1  # About a second\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Checkers/quadgrams.py\nimport logging\nimport re\nfrom math import log10\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, Translation, registry\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Quadgrams(Checker[str]):\n\n    \"\"\"\n    Uses Quadgrams to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying Quadgrams checker\")\n        # Capitalize and remove everything that's not a letter\n        ctext = re.sub(\"[^A-Z]\", \"\", ctext.upper())\n        quadgrams = self.QUADGRAMS_DICT\n        quadgrams_sum = sum(quadgrams.values())\n        score = 0\n        for key in quadgrams.keys():\n            quadgrams[key] = float(quadgrams[key]) / quadgrams_sum\n        floor = log10(0.01 / quadgrams_sum)\n        for i in range(len(ctext) - 4 + 1):\n            # Get all quadgrams from ctext and check if they're in the dict\n            # If yes then add the score of those quadgrams to the total score\n            if ctext[i : i + 4] in quadgrams:\n                score += quadgrams[ctext[i : i + 4]]\n            else:\n                score += floor\n        if len(ctext) > 0:\n            score = score / len(ctext)\n        logging.info(f\"Quadgrams is {score}\")\n        # The default threshold was found to work the best from lots of testing\n        if score > self.threshold:\n            return \"\"\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The quadgrams dictionary to use\",\n                req=False,\n                default=\"cipheydists::dist::quadgrams\",\n            ),\n            \"score\": ParamSpec(\n                desc=\"The score threshold to use\",\n                req=False,\n                default=0.00011,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.QUADGRAMS_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.threshold = float(self._params()[\"score\"])\n\n# Path: ciphey/basemods/Checkers/regex.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Regex(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = list(map(re.compile, self._params()[\"regex\"]))\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"Passed with regex {regex}. Want to contribute to Ciphey? Submit your regex here to allow Ciphey to automatically get this next time https://github.com/bee-san/pyWhat/wiki/Adding-your-own-Regex\\n\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"regex\": ParamSpec(\n                req=True,\n                desc=\"The regex that must be matched (in a substring)\",\n                list=True,\n            )\n        }\n\n\n@registry.register\nclass RegexList(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = []\n        for i in self._params()[\"resource\"]:\n            self.regexes += [re.compile(regex) for regex in config.get_resource(i)]\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"passed with regex {regex}\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"resource\": ParamSpec(\n                req=True,\n                desc=\"A list of regexes that could be matched\",\n                list=True,\n            )\n        }\n\n# Path: ciphey/basemods/Checkers/what.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\nimport logging\nfrom rich.logging import RichHandler\nfrom pywhat import identifier\nfrom rich.console import Console\n\nconsole = Console()\n\n\n@registry.register\nclass What(Checker[str]):\n\n    \"\"\"\n    Uses PyWhat to determine plaintext with regexes\n    https://github.com/bee-san/pyWhat\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying PyWhat checker\")\n        returned_regexes = self.id.identify(ctext)\n        if returned_regexes[\"Regexes\"]:\n            matched_regex = returned_regexes[\"Regexes\"]['text'][0][\"Regex Pattern\"]\n\n            ret = f'The plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            human = (\n                f'\\nI think the plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            )\n\n            if \"Description\" in matched_regex and matched_regex[\"Description\"]:\n                s = matched_regex[\"Description\"]\n                # lowercases first letter so it doesn't look weird\n                s = f\", which is {s[0].lower() + s[1:]}\\n\"\n                ret += s\n                human += s\n\n            # if URL is attached, include that too.\n            if \"URL\" in matched_regex and matched_regex[\"URL\"]:\n                link = matched_regex[\"URL\"] + ctext.replace(\" \", \"\")\n                ret += f\"\\nClick here to view in browser [#CAE4F1][link={link}]{link}[/link][/#CAE4F1]\\n\"\n\n            # If greppable mode is on, don't print this\n            if self.config.verbosity >= 0:\n                # Print with full stop\n                console.print(human)\n            return ret\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.config = config\n        self.id = identifier.Identifier()\n\n# Path: ciphey/basemods/Checkers/ezcheck.py\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nfrom .brandon import Brandon\nfrom .format import JsonChecker\nfrom .human import HumanChecker\nfrom .quadgrams import Quadgrams\nfrom .regex import RegexList\nfrom .what import What\n\n\n@registry.register\nclass EzCheck(Checker[str]):\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns, followed by a human check\n    \"\"\"\n\n    def check(self, text: str) -> Optional[str]:\n        for checker in self.checkers:\n            res = checker.check(text)\n            if (\n                res is not None\n                and (self.decider is None or self.decider.check(text)) is not None\n            ):\n                return res\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        return sum(\n            i.getExpectedRuntime(text) for i in self.checkers\n        ) + self.decider.getExpectedRuntime(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        self.checkers: List[Checker[str]] = []\n        # Disable human checker for automated systems\n        if config.verbosity >= 0:\n            self.decider = config(HumanChecker)\n        else:\n            self.decider = None\n\n        # We need to modify the config for each of the objects\n\n        # First PyWhat, as it's the fastest\n        self.checkers.append(config(What))\n\n        # Next, the json checker\n        self.checkers.append(config(JsonChecker))\n\n        # Second to last, the quadgrams checker\n        self.checkers.append(config(Quadgrams))\n\n        # Finally, the Brandon checker, as it is the slowest\n        self.checkers.append(config(Brandon))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/quorum.py\nfrom typing import Dict, Generic, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, _registry\n\n\nclass Quorum(Generic[T], Checker[T]):\n    def check(self, text: T) -> Optional[str]:\n        left = self._params().k\n        results = []\n        for checker in self.checkers:\n            results.append(checker.check(text))\n            if results[-1] is None:\n                continue\n            left -= 1\n            # Early return check\n            if left == 0:\n                return str(results)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        if self._params().k is None:\n            k = len(self._params()[\"checker\"])\n        # These checks need to be separate, to make sure that we do not have zero members\n        if self._params().k == 0 or self._params().k > len(self._params()[\"checker\"]):\n            raise IndexError(\n                \"k must be between 0 and the number of checkers (inclusive)\"\n            )\n\n        self.checkers = []\n        for i in self._params()[\"checker\"]:\n            # This enforces type consistency\n            self.checkers.append(_registry.get_named(i, Checker[T]))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"checker\": ParamSpec(\n                req=True, desc=\"The checkers to be used for analysis\", list=True\n            ),\n            \"k\": ParamSpec(\n                req=False,\n                desc=\"The minimum quorum size. Defaults to the number of checkers\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/__init__.py\nfrom . import any, brandon, ezcheck, format, human, quadgrams, quorum, regex, what\n\n# Path: ciphey/common.py\n\"\"\"Some useful adapters\"\"\"\nfrom typing import Any\n\n\ndef id_lambda(value: Any):\n    \"\"\"\n    A function used in dynamic class generation that abstracts away a constant return value (like in getName)\n    \"\"\"\n    return lambda *args: value\n\n\ndef fix_case(target: str, base: str) -> str:\n    \"\"\"Returns the lower-case string target with the case of base\"\"\"\n    ret = \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n    return \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n\n# Path: ciphey/basemods/Crackers/affine.py\n# Community\n# by https://github.com/Ozzyz\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\nfrom ciphey.mathsHelper import mathsHelper\n\n\n@registry.register\nclass Affine(Cracker[str]):\n    \"\"\"\n    Each character in the Affine Cipher is encoded with the rule E(x) = (ax + b) mod m\n    m is the size of the alphabet, while a and b are the keys in the cipher. a must be coprime to b.\n    The Caesar cipher is a specific case of the Affine Cipher, with a=1 and b being the shift of the cipher.\n    Decryption is performed by D(x) = a_inv (x - b) mod m where a_inv is the modular multiplicative inverse of a mod m.\n\n    In this version of the Affine Cipher, we do not allow alphabets with several instances of the same letter in different cases.\n    For instance, the alphabet 'ABCdef123' is allowed, but 'AaBbCc' is not.\n    \"\"\"\n\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"affine\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Brute forces all the possible combinations of a and b to attempt to crack the cipher.\n        \"\"\"\n        logging.debug(\"Attempting affine\")\n        candidates = []\n\n        # a and b are coprime if gcd(a,b) is 1.\n        possible_a = [\n            a\n            for a in range(1, self.alphabet_length)\n            if mathsHelper.gcd(a, self.alphabet_length) == 1\n        ]\n        logging.info(\n            f\"Trying Affine Cracker with {len(possible_a)} a-values and {self.alphabet_length} b-values\"\n        )\n\n        for a in possible_a:\n            a_inv = mathsHelper.mod_inv(a, self.alphabet_length)\n            # If there is no inverse, we cannot decrypt the text\n            if a_inv is None:\n                continue\n            for b in range(self.alphabet_length):\n                # Pass in lowered text. This means that we expect alphabets to not contain both 'a' and 'A'.\n                translated = self.decrypt(ctext.lower(), a_inv, b, self.alphabet_length)\n\n                candidate_probability = self.plaintext_probability(translated)\n                if candidate_probability > self.plaintext_prob_threshold:\n                    candidates.append(\n                        CrackResult(\n                            value=fix_case(translated, ctext), key_info=f\"a={a}, b={b}\"\n                        )\n                    )\n        logging.info(f\"Affine Cipher returned {len(candidates)} candidates\")\n        return candidates\n\n    def plaintext_probability(self, translated: str) -> float:\n        \"\"\"\n        Analyses the translated text and applies the chi squared test to see if it is a probable plaintext candidate\n        Returns the probability of the chi-squared test.\n        \"\"\"\n        analysis = cipheycore.analyse_string(translated)\n        return cipheycore.chisq_test(analysis, self.expected)\n\n    def decrypt(self, text: str, a_inv: int, b: int, m: int) -> str:\n        \"\"\"\n        Each letter is decrypted at D(x) = a_inv (x - b) mod m where x is the char\n        We treat the char value as its index in the alphabet, so if\n        the alphabet is 'abcd....' and the char is 'b', it has the value 1.\n        \"\"\"\n        return \"\".join([self.decryptChar(char, a_inv, b, m) for char in text])\n\n    def decryptChar(self, char: str, a_inv: int, b: int, m: int) -> str:\n\n        # We lower the alphabet since both ctext and alphabet need to be in the same case in order\n        # to perform the shifts. The translated text will have fixed case after the translation anyways.\n        # This is only necessary if the specified alphabet is uppercase.\n        alphabet = [x.lower() for x in self.group]\n\n        # Preserve characters that are not in alphabet\n        if char not in alphabet:\n            return char\n        char_idx = alphabet.index(char)\n        decrypted_char_idx = (a_inv * (char_idx - b)) % m\n        return alphabet[decrypted_char_idx]\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.alphabet_length = len(self.group)\n        self.cache = config.cache\n        self.plaintext_prob_threshold = 0.01\n\n# Path: ciphey/basemods/Crackers/ascii_shift.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Ascii_shift(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"ascii_shift\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ASCII shift cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ASCII shift returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ASCII shift cipher alphabet\",\n                req=False,\n                default=\"\"\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/baconian.py\nimport re\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\nimport logging\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Baconian(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"baconian\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to decode both variants of the Baconian cipher.\n        \"\"\"\n        logging.debug(\"Attempting Baconian cracker\")\n        candidates = []\n        result = []\n        ctext_decoded = \"\"\n        ctext_decoded2 = \"\"\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext only contains A and B\n        if bool(re.search(r\"[^AB]\", ctext)) is True:\n            logging.debug(\"Failed to crack baconian due to non baconian character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 5\n        ctext_len = len(ctext)\n        if ctext_len % 5:\n            logging.debug(\n                f\"Failed to decode Baconian because length must be a multiple of 5, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 5\n        ctext = \" \".join(ctext[i : i + 5] for i in range(0, len(ctext), 5))\n        ctext_split = ctext.split(\" \")\n        baconian_keys = self.BACONIAN_DICT.keys()\n\n        # Decode I=J and U=V variant\n        for i in ctext_split:\n            if i in baconian_keys:\n                ctext_decoded += self.BACONIAN_DICT[i]\n\n        # Decode variant that assigns each letter a unique code\n        for i in ctext_split:\n            if \"+\" + i in baconian_keys:\n                ctext_decoded2 += self.BACONIAN_DICT[\"+\" + i]\n\n        candidates.append(ctext_decoded)\n        candidates.append(ctext_decoded2)\n        for candidate in candidates:\n            if candidate != \"\":\n                if candidate == candidates[0]:\n                    result.append(CrackResult(value=candidate, key_info=\"I=J & U=V\"))\n                else:\n                    result.append(CrackResult(value=candidate))\n        logging.debug(f\"Baconian cracker - Returning results: {result}\")\n        return result\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"dict\": ParamSpec(\n                desc=\"The Baconian alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::baconian\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BACONIAN_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n\n# Path: ciphey/basemods/Crackers/caesar.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Caesar(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"caesar\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying caesar cipher on {ctext}\")\n        # Convert it to lower case\n        #\n        # TODO: handle different alphabets\n        if self.lower:\n            message = ctext.lower()\n        else:\n            message = ctext\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"Caesar returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(message, candidate.key, self.group)\n            candidates.append(\n                CrackResult(value=fix_case(translated, ctext), key_info=candidate.key)\n            )\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the caesar cipher alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n            \"lower\": ParamSpec(\n                desc=\"Whether or not the ciphertext should be converted to lowercase first\",\n                req=False,\n                default=True,\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.lower: Union[str, bool] = self._params()[\"lower\"]\n        if not isinstance(self.lower, bool):\n            self.lower = util.strtobool(self.lower)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/rot47.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Rot47(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"rot47\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ROT47 cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ROT47 returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ROT47 cipher alphabet\",\n                req=False,\n                default=\"\"\"!\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/soundex.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\n\n\n@registry.register\nclass Soundex(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"soundex\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to crack Soundex by generating all possible combinations.\n        \"\"\"\n        logging.debug(\"Attempting Soundex cracker\")\n        word_list = []\n        sentences = []\n        result = []\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext contains only A-Z and 0-9\n        if bool(re.search(r\"[^A-Z0-9]\", ctext)) is True:\n            logging.debug(\"Failed to crack soundex due to non soundex character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 4\n        ctext_len = len(ctext)\n        if ctext_len % 4:\n            logging.debug(\n                f\"Failed to decode Soundex because length must be a multiple of 4, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 4\n        ctext = \" \".join(ctext[i : i + 4] for i in range(0, len(ctext), 4))\n        ctext_split = ctext.split(\" \")\n        soundex_keys = self.SOUNDEX_DICT.keys()\n\n        # Find all words that correspond to each given soundex code\n        for code in ctext_split:\n            if code in soundex_keys:\n                word_list.append(self.SOUNDEX_DICT[code])\n\n        logging.info(f\"Possible words for given encoded text: {word_list}\")\n\n        # Find all possible sentences\n        self.getSentenceCombo(\n            word_list,\n            sentences,\n            self.frequency_dict,\n            self.sentence_freq,\n            self.word_freq,\n        )\n\n        sorted_sentences = self.sortlistwithdict(sentences, self.frequency_dict)\n\n        for sentence in sorted_sentences:\n            result.append(CrackResult(value=sentence))\n\n        logging.debug(f\"Soundex cracker - Returning results: {result}\")\n        return result\n\n    def sortlistwithdict(self, listtosort, hashes):\n        \"\"\"\n        This function uses the sum of ranks (based on frequency) of each word in each\n        sentence and sorts them according to it.\n        \"\"\"\n        return sorted(listtosort, key=lambda x: hashes[x])\n\n    \ndef getSentenceCombo(\n        self, A, sentences, frequency_dict, sentence_freq, word_freq, result=\"\", n=0\n    ):\n        \"\"\"\n        This function uses recursion to generate a list of sentences from all possible\n        words for a given set of soundex codes.\n        \"\"\"\n        logging.debug(\"Creating all possible sentences from Soundex\")\n        if n == len(A):\n            sentences.append(result[1:])\n            for word in result[1:].split():\n                # Adding the rank of each word to find out the sentence's net frequency\n                if word in word_freq:\n                    sentence_freq += word_freq.index(word)\n                # If the word isn't in the frequency list then it's a very uncommon word\n                # so we add a large number (5000)\n                else:\n                    sentence_freq += 5000\n            frequency_dict[result[1:]] = sentence_freq\n            sentence_freq = 0\n            return\n\n        for word in A[n]:\n            out = result + \" \" + word\n            self.getSentenceCombo(\n                A, sentences, frequency_dict, sentence_freq, word_freq, out, n + 1\n            )\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The Soundex dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::soundex\",\n            ),\n            \"freq\": ParamSpec(\n                desc=\"The word frequency dictionary to use\",\n                req=False,\n                default=\"cipheydists::list::English5000Freq\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.SOUNDEX_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.word_freq = config.get_resource(self._params()[\"freq\"], Translation)\n        self.frequency_dict = {}\n        self.sentence_freq = 0\n\n# Path: ciphey/basemods/Crackers/vigenere.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Vigenere(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        if self.keysize is not None:\n            analysis = self.cache.get_or_update(\n                ctext,\n                f\"vigenere::{self.keysize}\",\n                lambda: cipheycore.analyse_string(\n                    ctext.lower(), self.keysize, self.group\n                ),\n            )\n\n            val = cipheycore.vigenere_detect(analysis, self.expected)\n\n            logging.info(f\"Vigenere has likelihood {val}\")\n\n            return CrackInfo(\n                success_likelihood=val,\n                # TODO: actually calculate runtimes\n                success_runtime=1e-3,\n                failure_runtime=1e-2,\n            )\n\n        likely_lens = self.cache.get_or_update(\n            ctext,\n            \"vigenere::likely_lens\",\n            lambda: cipheycore.vigenere_likely_key_lens(\n                ctext.lower(), self.expected, self.group, self.detect_p_value\n            ),\n        )\n\n        # Filter out the lens that make no sense\n        likely_lens = [i for i in likely_lens if i.len <= self.max_key_length]\n\n        for keysize in likely_lens:\n            # Store the analysis\n            analysis = self.cache.get_or_update(\n                ctext, f\"vigenere::{keysize.len}\", lambda: keysize.tab\n            )\n        if len(likely_lens) == 0:\n            return CrackInfo(\n                success_likelihood=0,\n                # TODO: actually calculate runtimes\n                success_runtime=2e-3,\n                failure_runtime=2e-2,\n            )\n\n        logging.info(\n            f\"Vigenere has likelihood {likely_lens[0].p_value} with lens {[i.len for i in likely_lens]}\"\n        )\n\n        return CrackInfo(\n            success_likelihood=likely_lens[0].p_value,\n            # TODO: actually calculate runtimes\n            success_runtime=2e-4,\n            failure_runtime=2e-4,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"vigenere\"\n\n    def crackOne(\n        self, ctext: str, analysis: cipheycore.windowed_analysis_res, real_ctext: str\n    ) -> List[CrackResult]:\n        possible_keys = cipheycore.vigenere_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n        if len(possible_keys) > self.clamp:\n            possible_keys = possible_keys[: self.clamp]\n        logging.debug(\n            f\"Vigenere crack got keys: {[[i for i in candidate.key] for candidate in possible_keys]}\"\n        )\n        return [\n            CrackResult(\n                value=fix_case(\n                    cipheycore.vigenere_decrypt(ctext, candidate.key, self.group),\n                    real_ctext,\n                ),\n                key_info=\"\".join([self.group[i] for i in candidate.key]),\n                misc_info=f\"p-value was {candidate.p_value}\",\n            )\n            for candidate in possible_keys[: min(len(possible_keys), 10)]\n        ]\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(\"Trying vigenere cipher\")\n        # Convert it to lower case\n        if self.lower:\n            message = ctext.lower()\n        else:\n            message = ctext\n\n        # Analysis must be done here, where we know the case for the cache\n        if self.keysize is not None:\n            return self.crackOne(\n                message,\n                self.cache.get_or_update(\n                    ctext,\n                    f\"vigenere::{self.keysize}\",\n                    lambda: cipheycore.analyse_string(\n                        message, self.keysize, self.group\n                    ),\n                ),\n                ctext,\n            )\n\n        arrs = []\n        likely_lens = self.cache.get_or_update(\n            ctext,\n            \"vigenere::likely_lens\",\n            lambda: cipheycore.vigenere_likely_key_lens(\n                message, self.expected, self.group\n            ),\n        )\n        possible_lens = [i for i in likely_lens]\n        possible_lens.sort(key=lambda i: i.p_value)\n        logging.debug(f\"Got possible lengths {[i.len for i in likely_lens]}\")\n        # TODO: work out length\n        for i in possible_lens:\n            arrs.extend(\n                self.crackOne(\n                    message,\n                    self.cache.get_or_update(\n                        ctext,\n                        f\"vigenere::{i.len}\",\n                        lambda: cipheycore.analyse_string(message, i.len, self.group),\n                    ),\n                    ctext,\n                )\n            )\n\n        logging.info(f\"Vigenere returned {len(arrs)} candidates\")\n        return arrs\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the caesar cipher alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n            \"lower\": ParamSpec(\n                desc=\"Whether or not the ciphertext should be converted to lowercase first\",\n                req=False,\n                default=True,\n            ),\n            \"keysize\": ParamSpec(\n                desc=\"A key size that should be used. If not given, will attempt to work it out\",\n                req=False,\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for windowed frequency analysis\",\n                req=False,\n                default=0.5,\n            ),\n            \"detect_p_value\": ParamSpec(\n                desc=\"The p-value to use for the detection of Vigenere length\",\n                req=False,\n                default=0.01,\n            ),\n            \"clamp\": ParamSpec(\n                desc=\"The maximum number of candidates that can be returned per key len\",\n                req=False,\n                default=10,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.lower: Union[str, bool] = self._params()[\"lower\"]\n        if not isinstance(self.lower, bool):\n...\n", "needle_token_start": 13535, "needle_token_end": 13841, "code_context_ntokens": 16111, "output": ["```python\ndef getSentenceCombo(\n        self, A, sentences, frequency_dict, sentence_freq, word_freq, result=\"\", n=0\n    ):\n        \"\"\"\n        This function uses recursion to generate a list of sentences from all possible\n        words for a given set of soundex codes.\n        \"\"\"\n        logging.debug(\"Creating all possible sentences from Soundex\")\n        if n == len(A):\n            sentences.append(result[1:])\n            for word in result[1:].split():\n                # Adding the rank of each word to find out the sentence's net frequency\n                if word in word_freq:\n                    sentence_freq += word_freq.index(word)\n                # If the word isn't in the frequency list then it's a very uncommon word\n                # so we add a large number (5000)\n                else:\n                    sentence_freq += 5000\n            frequency_dict[result[1:]] = sentence_freq\n            sentence_freq = 0\n            return\n\n        for word in A[n]:\n            out = result + \" \" + word\n            self.getSentenceCombo(\n                A, sentences, frequency_dict, sentence_freq, word_freq, out, n + 1\n            )\n```"]}
{"repo": "Ciphey/Ciphey", "name": "sortlistwithdict", "language": "python", "path": "ciphey/basemods/Crackers/soundex.py", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to organize a list of sentences based on the cumulative frequency of the words they contain, with the aim of prioritizing sentences that are more likely to be meaningful or relevant based on their word usage.\n2. **Input**: The function takes two parameters: a list of sentences and a dictionary where keys are words and values are their corresponding frequency ranks.\n3. **Output**: It returns a list of sentences sorted from the most likely to the least likely to be relevant, based on the sum of the frequency ranks of the words in each sentence.\n4. **Procedure**: The function sorts the list of sentences by calculating the sum of the frequency ranks for each word in a sentence (using the provided dictionary) and then ordering the sentences from the lowest sum (highest priority) to the highest sum (lowest priority).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "# Path: ciphey/mathsHelper.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to provide helper functions for mathematics\n(oh, not entirely mathematics either. Some NLP stuff and sorting dicts. It's just a helper class\n)\n\"\"\"\n\nfrom collections import OrderedDict\nfrom string import punctuation\nfrom typing import Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\n\nclass mathsHelper:\n    \"\"\"Class to provide helper functions for mathematics and other small things\"\"\"\n\n    def __init__(self):\n        # ETAOIN is the most popular letters in order\n        self.ETAOIN = \"ETAOINSHRDLCUMWFGYPBVKJXQZ\"\n        self.LETTERS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n\n    @staticmethod\n    def gcd(a, b) -> int:\n        \"\"\"Greatest common divisor.\n\n        The Greatest Common Divisor of a and b using Euclid's Algorithm.\n\n        Args:\n            a -> num 1\n            b -> num 2\n\n        Returns:\n            Returns  GCD(a, b)\n\n        \"\"\"\n        # Return\n        while a != 0:\n            a, b = b % a, a\n        return b\n\n    @staticmethod\n    def mod_inv(a: int, m: int) -> Optional[int]:\n        \"\"\"\n        Returns the modular inverse of a mod m, or None if it does not exist.\n\n        The modular inverse of a is the number a_inv that satisfies the equation\n        a_inv * a mod m === 1 mod m\n\n        Note: This is a naive implementation, and runtime may be improved in several ways.\n        For instance by checking if m is prime to perform a different calculation,\n        or by using the extended euclidean algorithm.\n        \"\"\"\n        for i in range(1, m):\n            if (m * i + 1) % a == 0:\n                return (m * i + 1) // a\n        return None\n\n    @staticmethod\n...\n# Path: ciphey/basemods/Checkers/brandon.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\nClass to determine whether something is English or not.\n1. Calculate the Chi Squared score of a sentence\n2. If the score is significantly lower than the average score, it _might_ be English\n    2.1. If the score _might_ be English, then take the text and compare it to the sorted dictionary\n    in O(n log n) time.\n    It creates a percentage of \"How much of this text is in the dictionary?\"\n    The dictionary contains:\n        * 20,000 most common US words\n        * 10,000 most common UK words (there's no repetition between the two)\n        * The top 10,000 passwords\n    If the word \"Looks like\" English (chi-squared) and if it contains English words, we can conclude it is\n    very likely English. The alternative is doing the dictionary thing but with an entire 479k word dictionary (slower)\n    2.2. If the score is not English, but we haven't tested enough to create an average, then test it against\n     the dictionary\n\nThings to optimise:\n* We only run the dictionary if it's 20% smaller than the average for chi squared\n* We consider it \"English\" if 45% of the text matches the dictionary\n* We run the dictionary if there is less than 10 total chisquared test\n\nHow to add a language:\n* Download your desired dictionary. Try to make it the most popular words, for example. Place this file into this\n folder with languagename.txt\nAs an example, this comes built in with english.txt\nFind the statistical frequency of each letter in that language.\nFor English, we have:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n}\nIn chisquared.py\nTo add your language, do:\nself.languages = {\n    \"English\":\n    [0.0855, 0.0160, 0.0316, 0.0387, 0.1210,0.0218, 0.0209, 0.0496, 0.0733, 0.0022,0.0081, 0.0421, 0.0253, 0.0717,\n    0.0747,0.0207, 0.0010, 0.0633, 0.0673, 0.0894,0.0268, 0.0106, 0.0183, 0.0019, 0.0172,0.0011]\n    \"German\": [0.0973]\n}\nIn alphabetical order\nAnd you're.... Done! Make sure the name of the two match up\n\"\"\"\nimport sys\nfrom math import ceil\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nsys.path.append(\"..\")\ntry:\n    import mathsHelper as mh\nexcept ModuleNotFoundError:\n    import ciphey.mathsHelper as mh\n\n\n@registry.register\nclass Brandon(Checker[str]):\n    \"\"\"\n    Class designed to confirm whether something is **language** based on how many words of **language** appears\n    Call confirmLanguage(text, language)\n    * text: the text you want to confirm\n    * language: the language you want to confirm\n\n    Find out what language it is by using chisquared.py, the highest chisquared score is the language\n    languageThreshold = 45\n    if a string is 45% **language** words, then it's confirmed to be english\n    \"\"\"\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually work this out\n        # TODO its 0.2 seconds on average\n        return 1e-4  # 100 \u00b5s\n\n    wordlist: set\n\n    def clean_text(self, text: str) -> set:\n        \"\"\"Cleans the text ready to be checked\n\n        Strips punctuation, makes it lower case, turns it into a set separated by spaces, removes duplicate words\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            text -> the text as a list, now cleaned\n\n        \"\"\"\n        # makes the text unique words and readable\n        text = text.lower()\n        text = self.mh.strip_punctuation(text)\n        text = text.split(\" \")\n        text = filter(lambda x: len(x) > 2, text)\n        text = set(text)\n        return text\n\n    def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        \"\"\"Given text determine if it passes checker\n\n        The checker uses the variable passed to it. I.E. Stopwords list, 1k words, dictionary\n\n        Args:\n            text -> The text to check\n            threshold -> at what point do we return True? The percentage of text that is in var before we return True\n            text_length -> the length of the text\n            var -> the variable we are checking against. Stopwords list, 1k words list, dictionary list.\n        Returns:\n            boolean -> True for it passes the test, False for it fails the test.\"\"\"\n        if text is None:\n            logging.debug(\"Checker's text is None, so returning False\")\n            return False\n        if var is None:\n            logging.debug(\"Checker's input var is None, so returning False\")\n            return False\n\n        percent = ceil(text_length * threshold)\n        logging.debug(f\"Checker's chunks are size {percent}\")\n        meet_threshold = 0\n        location = 0\n        end = percent\n\n        if text_length <= 0:\n            return False\n\n        while location <= text_length:\n            # chunks the text, so only gets THRESHOLD chunks of text at a time\n            text = list(text)\n            to_analyse = text[location:end]\n            logging.debug(f\"To analyse is {to_analyse}\")\n            for word in to_analyse:\n                # if word is a stopword, + 1 to the counter\n                if word in var:\n                    logging.debug(\n                        f\"{word} is in var, which means I am +=1 to the meet_threshold which is {meet_threshold}\"\n                    )\n                    meet_threshold += 1\n                meet_threshold_percent = meet_threshold / text_length\n                if meet_threshold_percent >= threshold:\n                    logging.debug(\n                        f\"Returning true since the percentage is {meet_threshold / text_length} and the threshold is {threshold}\"\n                    )\n                    # if we meet the threshold, return True\n                    # otherwise, go over again until we do\n                    # We do this in the for loop because if we're at 24% and THRESHOLD is 25\n                    # we don't want to wait THRESHOLD to return true, we want to return True ASAP\n                    return True\n            location = end\n            end = end + percent\n        logging.debug(\n            f\"The language proportion {meet_threshold_percent} is under the threshold {threshold}\"\n        )\n        return False\n\n    def __init__(self, config: Config):\n        # Suppresses warning\n        super().__init__(config)\n        self.mh = mh.mathsHelper()\n\n        phases = config.get_resource(self._params()[\"phases\"])\n\n        self.thresholds_phase1 = phases[\"1\"]\n        self.thresholds_phase2 = phases[\"2\"]\n        self.top1000Words = config.get_resource(self._params().get(\"top1000\"))\n        self.wordlist = config.get_resource(self._params()[\"wordlist\"])\n        self.stopwords = config.get_resource(self._params().get(\"stopwords\"))\n\n        self.len_phase1 = len(self.thresholds_phase1)\n        self.len_phase2 = len(self.thresholds_phase2)\n\n    def check(self, text: str) -> Optional[str]:\n        \"\"\"Checks to see if the text is in English\n\n        Performs a decryption, but mainly parses the internal data packet and prints useful information.\n\n        Args:\n            text -> The text we use to perform analysis on\n\n        Returns:\n            bool -> True if the text is English, False otherwise.\n\n        \"\"\"\n        logging.debug(f'In Language Checker with \"{text}\"')\n        text = self.clean_text(text)\n        logging.debug(f'Text split to \"{text}\"')\n        if text == \"\":\n            logging.debug(\"Returning None from Brandon as the text cleaned is none.\")\n            return None\n\n        length_text = len(text)\n\n        what_to_use = {}\n\n        # this code decides what checker / threshold to use\n        # if text is over or equal to maximum size, just use the maximum possible checker\n        what_to_use = self.calculateWhatChecker(\n            length_text, self.thresholds_phase1.keys()\n        )\n        logging.debug(self.thresholds_phase1)\n        what_to_use = self.thresholds_phase1[str(what_to_use)]\n        # def checker(self, text: str, threshold: float, text_length: int, var: set) -> bool:\n        if \"check\" in what_to_use:\n            # perform check 1k words\n            result = self.checker(\n                text, what_to_use[\"check\"], length_text, self.top1000Words\n            )\n        elif \"stop\" in what_to_use:\n            # perform stopwords\n            result = self.checker(\n                text, what_to_use[\"stop\"], length_text, self.stopwords\n            )\n        elif \"dict\" in what_to_use:\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n            # If result is None, no point doing it again in phase2\n            if not result:\n                return None\n        else:\n            logging.info(f\"It is neither stop or check, but instead {what_to_use}\")\n\n        # return False if phase 1 fails\n        if not result:\n            return None\n        else:\n            what_to_use = self.calculateWhatChecker(\n                length_text, self.thresholds_phase2.keys()\n            )\n            what_to_use = self.thresholds_phase2[str(what_to_use)]\n            result = self.checker(text, what_to_use[\"dict\"], length_text, self.wordlist)\n        return \"\" if result else None\n\n    def calculateWhatChecker(self, length_text, key):\n        \"\"\"Calculates what threshold / checker to use\n\n        If the length of the text is over the maximum sentence length, use the last checker / threshold\n        Otherwise, traverse the keys backwards until we find a key range that does not fit.\n        So we traverse backwards and see if the sentence length is between current - 1 and current\n        In this way, we find the absolute lowest checker / percentage threshold.\n        We traverse backwards because if the text is longer than the max sentence length, we already know.\n        In total, the keys are only 5 items long or so. It is not expensive to move backwards, nor is it expensive to move forwards.\n\n        Args:\n            length_text -> The length of the text\n            key -> What key we want to use. I.E. Phase1 keys, Phase2 keys.\n        Returns:\n            what_to_use -> the key of the lowest checker.\"\"\"\n\n        _keys = list(key)\n        _keys = list(map(int, _keys))\n        if length_text >= int(_keys[-1]):\n            what_to_use = list(key)[_keys.index(_keys[-1])]\n        else:\n            # this algorithm finds the smallest possible fit for the text\n            for counter, i in reversed(list(enumerate(_keys))):\n                #  [0, 110, 150]\n                if i <= length_text:\n                    what_to_use = i\n        return what_to_use\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"top1000\": ParamSpec(\n                desc=\"A wordlist of the top 1000 words\",\n                req=False,\n                default=\"cipheydists::list::english1000\",\n            ),\n            \"wordlist\": ParamSpec(\n                desc=\"A wordlist of all the words\",\n                req=False,\n                default=\"cipheydists::list::english\",\n            ),\n            \"stopwords\": ParamSpec(\n                desc=\"A wordlist of StopWords\",\n                req=False,\n                default=\"cipheydists::list::englishStopWords\",\n            ),\n            \"threshold\": ParamSpec(\n                desc=\"The minimum proportion (between 0 and 1) that must be in the dictionary\",\n                req=False,\n                default=0.45,\n            ),\n            \"phases\": ParamSpec(\n                desc=\"Language-specific phase thresholds\",\n                req=False,\n                default=\"cipheydists::brandon::english\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/format.py\nimport json\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass JsonChecker(Checker[str]):\n\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns\n    \"\"\"\n\n    def check(self, text: T) -> Optional[str]:\n        logging.debug(\"Trying json checker\")\n\n        # https://github.com/Ciphey/Ciphey/issues/389\n        if text.isdigit():\n            return None\n\n        try:\n            json.loads(text)\n            return \"\"\n        except ValueError:\n            return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 1e-7 * len(text)  # From benchmarks I found online\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/human.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, registry\nfrom rich.console import Console\nfrom rich.markup import escape\n\nconsole = Console()\n\n\n@registry.register\nclass HumanChecker(Checker[str]):\n\n    \"\"\"\n    Uses the person's decision to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: str) -> Optional[str]:\n        with self._config().pause_spinner_handle():\n            response = console.input(\n                f\"Possible plaintext: [blue bold]{escape(ctext.__repr__())}[/blue bold] ([green]y[/green]/[red]N[/red]): \"\n            )\n        if response == \"y\":\n            return \"\"\n        elif response in (\"n\", \"\"):\n            return None\n        else:\n            return self.check(ctext)\n\n    def getExpectedRuntime(self, text: str) -> float:\n        return 1  # About a second\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n# Path: ciphey/basemods/Checkers/quadgrams.py\nimport logging\nimport re\nfrom math import log10\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, Translation, registry\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Quadgrams(Checker[str]):\n\n    \"\"\"\n    Uses Quadgrams to determine plaintext\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying Quadgrams checker\")\n        # Capitalize and remove everything that's not a letter\n        ctext = re.sub(\"[^A-Z]\", \"\", ctext.upper())\n        quadgrams = self.QUADGRAMS_DICT\n        quadgrams_sum = sum(quadgrams.values())\n        score = 0\n        for key in quadgrams.keys():\n            quadgrams[key] = float(quadgrams[key]) / quadgrams_sum\n        floor = log10(0.01 / quadgrams_sum)\n        for i in range(len(ctext) - 4 + 1):\n            # Get all quadgrams from ctext and check if they're in the dict\n            # If yes then add the score of those quadgrams to the total score\n            if ctext[i : i + 4] in quadgrams:\n                score += quadgrams[ctext[i : i + 4]]\n            else:\n                score += floor\n        if len(ctext) > 0:\n            score = score / len(ctext)\n        logging.info(f\"Quadgrams is {score}\")\n        # The default threshold was found to work the best from lots of testing\n        if score > self.threshold:\n            return \"\"\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The quadgrams dictionary to use\",\n                req=False,\n                default=\"cipheydists::dist::quadgrams\",\n            ),\n            \"score\": ParamSpec(\n                desc=\"The score threshold to use\",\n                req=False,\n                default=0.00011,\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.QUADGRAMS_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.threshold = float(self._params()[\"score\"])\n\n# Path: ciphey/basemods/Checkers/regex.py\nimport re\nfrom typing import Dict, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\n\n@registry.register\nclass Regex(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = list(map(re.compile, self._params()[\"regex\"]))\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"Passed with regex {regex}. Want to contribute to Ciphey? Submit your regex here to allow Ciphey to automatically get this next time https://github.com/bee-san/pyWhat/wiki/Adding-your-own-Regex\\n\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"regex\": ParamSpec(\n                req=True,\n                desc=\"The regex that must be matched (in a substring)\",\n                list=True,\n            )\n        }\n\n\n@registry.register\nclass RegexList(Checker[str]):\n    def getExpectedRuntime(self, text: T) -> float:\n        return 1e-5  # TODO: actually calculate this\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.regexes = []\n        for i in self._params()[\"resource\"]:\n            self.regexes += [re.compile(regex) for regex in config.get_resource(i)]\n        logging.debug(f\"There are {len(self.regexes)} regexes\")\n\n    def check(self, text: str) -> Optional[str]:\n        for regex in self.regexes:\n            logging.debug(f\"Trying regex {regex} on {text}\")\n            res = regex.search(text)\n            logging.debug(f\"Results: {res}\")\n            if res:\n                return f\"passed with regex {regex}\"\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"resource\": ParamSpec(\n                req=True,\n                desc=\"A list of regexes that could be matched\",\n                list=True,\n            )\n        }\n\n# Path: ciphey/basemods/Checkers/what.py\nfrom typing import Dict, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\nimport logging\nfrom rich.logging import RichHandler\nfrom pywhat import identifier\nfrom rich.console import Console\n\nconsole = Console()\n\n\n@registry.register\nclass What(Checker[str]):\n\n    \"\"\"\n    Uses PyWhat to determine plaintext with regexes\n    https://github.com/bee-san/pyWhat\n    \"\"\"\n\n    def check(self, ctext: T) -> Optional[str]:\n        logging.debug(\"Trying PyWhat checker\")\n        returned_regexes = self.id.identify(ctext)\n        if returned_regexes[\"Regexes\"]:\n            matched_regex = returned_regexes[\"Regexes\"]['text'][0][\"Regex Pattern\"]\n\n            ret = f'The plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            human = (\n                f'\\nI think the plaintext is a [yellow]{matched_regex[\"Name\"]}[/yellow]'\n            )\n\n            if \"Description\" in matched_regex and matched_regex[\"Description\"]:\n                s = matched_regex[\"Description\"]\n                # lowercases first letter so it doesn't look weird\n                s = f\", which is {s[0].lower() + s[1:]}\\n\"\n                ret += s\n                human += s\n\n            # if URL is attached, include that too.\n            if \"URL\" in matched_regex and matched_regex[\"URL\"]:\n                link = matched_regex[\"URL\"] + ctext.replace(\" \", \"\")\n                ret += f\"\\nClick here to view in browser [#CAE4F1][link={link}]{link}[/link][/#CAE4F1]\\n\"\n\n            # If greppable mode is on, don't print this\n            if self.config.verbosity >= 0:\n                # Print with full stop\n                console.print(human)\n            return ret\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        # TODO: actually bench this\n        return 2e-7 * len(text)\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return None\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.config = config\n        self.id = identifier.Identifier()\n\n# Path: ciphey/basemods/Checkers/ezcheck.py\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, registry\n\nfrom .brandon import Brandon\nfrom .format import JsonChecker\nfrom .human import HumanChecker\nfrom .quadgrams import Quadgrams\nfrom .regex import RegexList\nfrom .what import What\n\n\n@registry.register\nclass EzCheck(Checker[str]):\n    \"\"\"\n    This object is effectively a prebuilt quorum (with requirement 1) of common patterns, followed by a human check\n    \"\"\"\n\n    def check(self, text: str) -> Optional[str]:\n        for checker in self.checkers:\n            res = checker.check(text)\n            if (\n                res is not None\n                and (self.decider is None or self.decider.check(text)) is not None\n            ):\n                return res\n        return None\n\n    def getExpectedRuntime(self, text: T) -> float:\n        return sum(\n            i.getExpectedRuntime(text) for i in self.checkers\n        ) + self.decider.getExpectedRuntime(text)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        self.checkers: List[Checker[str]] = []\n        # Disable human checker for automated systems\n        if config.verbosity >= 0:\n            self.decider = config(HumanChecker)\n        else:\n            self.decider = None\n\n        # We need to modify the config for each of the objects\n\n        # First PyWhat, as it's the fastest\n        self.checkers.append(config(What))\n\n        # Next, the json checker\n        self.checkers.append(config(JsonChecker))\n\n        # Second to last, the quadgrams checker\n        self.checkers.append(config(Quadgrams))\n\n        # Finally, the Brandon checker, as it is the slowest\n        self.checkers.append(config(Brandon))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        pass\n\n# Path: ciphey/basemods/Checkers/quorum.py\nfrom typing import Dict, Generic, Optional\n\nfrom ciphey.iface import Checker, Config, ParamSpec, T, _registry\n\n\nclass Quorum(Generic[T], Checker[T]):\n    def check(self, text: T) -> Optional[str]:\n        left = self._params().k\n        results = []\n        for checker in self.checkers:\n            results.append(checker.check(text))\n            if results[-1] is None:\n                continue\n            left -= 1\n            # Early return check\n            if left == 0:\n                return str(results)\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n\n        if self._params().k is None:\n            k = len(self._params()[\"checker\"])\n        # These checks need to be separate, to make sure that we do not have zero members\n        if self._params().k == 0 or self._params().k > len(self._params()[\"checker\"]):\n            raise IndexError(\n                \"k must be between 0 and the number of checkers (inclusive)\"\n            )\n\n        self.checkers = []\n        for i in self._params()[\"checker\"]:\n            # This enforces type consistency\n            self.checkers.append(_registry.get_named(i, Checker[T]))\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"checker\": ParamSpec(\n                req=True, desc=\"The checkers to be used for analysis\", list=True\n            ),\n            \"k\": ParamSpec(\n                req=False,\n                desc=\"The minimum quorum size. Defaults to the number of checkers\",\n            ),\n        }\n\n# Path: ciphey/basemods/Checkers/__init__.py\nfrom . import any, brandon, ezcheck, format, human, quadgrams, quorum, regex, what\n\n# Path: ciphey/common.py\n\"\"\"Some useful adapters\"\"\"\nfrom typing import Any\n\n\ndef id_lambda(value: Any):\n    \"\"\"\n    A function used in dynamic class generation that abstracts away a constant return value (like in getName)\n    \"\"\"\n    return lambda *args: value\n\n\ndef fix_case(target: str, base: str) -> str:\n    \"\"\"Returns the lower-case string target with the case of base\"\"\"\n    ret = \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n    return \"\".join(\n        [\n            target[i].upper() if base[i].isupper() else target[i]\n            for i in range(len(target))\n        ]\n    )\n\n# Path: ciphey/basemods/Crackers/affine.py\n# Community\n# by https://github.com/Ozzyz\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\nfrom ciphey.mathsHelper import mathsHelper\n\n\n@registry.register\nclass Affine(Cracker[str]):\n    \"\"\"\n    Each character in the Affine Cipher is encoded with the rule E(x) = (ax + b) mod m\n    m is the size of the alphabet, while a and b are the keys in the cipher. a must be coprime to b.\n    The Caesar cipher is a specific case of the Affine Cipher, with a=1 and b being the shift of the cipher.\n    Decryption is performed by D(x) = a_inv (x - b) mod m where a_inv is the modular multiplicative inverse of a mod m.\n\n    In this version of the Affine Cipher, we do not allow alphabets with several instances of the same letter in different cases.\n    For instance, the alphabet 'ABCdef123' is allowed, but 'AaBbCc' is not.\n    \"\"\"\n\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"affine\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Brute forces all the possible combinations of a and b to attempt to crack the cipher.\n        \"\"\"\n        logging.debug(\"Attempting affine\")\n        candidates = []\n\n        # a and b are coprime if gcd(a,b) is 1.\n        possible_a = [\n            a\n            for a in range(1, self.alphabet_length)\n            if mathsHelper.gcd(a, self.alphabet_length) == 1\n        ]\n        logging.info(\n            f\"Trying Affine Cracker with {len(possible_a)} a-values and {self.alphabet_length} b-values\"\n        )\n\n        for a in possible_a:\n            a_inv = mathsHelper.mod_inv(a, self.alphabet_length)\n            # If there is no inverse, we cannot decrypt the text\n            if a_inv is None:\n                continue\n            for b in range(self.alphabet_length):\n                # Pass in lowered text. This means that we expect alphabets to not contain both 'a' and 'A'.\n                translated = self.decrypt(ctext.lower(), a_inv, b, self.alphabet_length)\n\n                candidate_probability = self.plaintext_probability(translated)\n                if candidate_probability > self.plaintext_prob_threshold:\n                    candidates.append(\n                        CrackResult(\n                            value=fix_case(translated, ctext), key_info=f\"a={a}, b={b}\"\n                        )\n                    )\n        logging.info(f\"Affine Cipher returned {len(candidates)} candidates\")\n        return candidates\n\n    def plaintext_probability(self, translated: str) -> float:\n        \"\"\"\n        Analyses the translated text and applies the chi squared test to see if it is a probable plaintext candidate\n        Returns the probability of the chi-squared test.\n        \"\"\"\n        analysis = cipheycore.analyse_string(translated)\n        return cipheycore.chisq_test(analysis, self.expected)\n\n    def decrypt(self, text: str, a_inv: int, b: int, m: int) -> str:\n        \"\"\"\n        Each letter is decrypted at D(x) = a_inv (x - b) mod m where x is the char\n        We treat the char value as its index in the alphabet, so if\n        the alphabet is 'abcd....' and the char is 'b', it has the value 1.\n        \"\"\"\n        return \"\".join([self.decryptChar(char, a_inv, b, m) for char in text])\n\n    def decryptChar(self, char: str, a_inv: int, b: int, m: int) -> str:\n\n        # We lower the alphabet since both ctext and alphabet need to be in the same case in order\n        # to perform the shifts. The translated text will have fixed case after the translation anyways.\n        # This is only necessary if the specified alphabet is uppercase.\n        alphabet = [x.lower() for x in self.group]\n\n        # Preserve characters that are not in alphabet\n        if char not in alphabet:\n            return char\n        char_idx = alphabet.index(char)\n        decrypted_char_idx = (a_inv * (char_idx - b)) % m\n        return alphabet[decrypted_char_idx]\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.alphabet_length = len(self.group)\n        self.cache = config.cache\n        self.plaintext_prob_threshold = 0.01\n\n# Path: ciphey/basemods/Crackers/ascii_shift.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Ascii_shift(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"ascii_shift\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ASCII shift cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ASCII shift returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ASCII shift cipher alphabet\",\n                req=False,\n                default=\"\"\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/baconian.py\nimport re\nfrom typing import Dict, List, Optional\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\nimport logging\nfrom rich.logging import RichHandler\n\n\n@registry.register\nclass Baconian(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"baconian\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to decode both variants of the Baconian cipher.\n        \"\"\"\n        logging.debug(\"Attempting Baconian cracker\")\n        candidates = []\n        result = []\n        ctext_decoded = \"\"\n        ctext_decoded2 = \"\"\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext only contains A and B\n        if bool(re.search(r\"[^AB]\", ctext)) is True:\n            logging.debug(\"Failed to crack baconian due to non baconian character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 5\n        ctext_len = len(ctext)\n        if ctext_len % 5:\n            logging.debug(\n                f\"Failed to decode Baconian because length must be a multiple of 5, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 5\n        ctext = \" \".join(ctext[i : i + 5] for i in range(0, len(ctext), 5))\n        ctext_split = ctext.split(\" \")\n        baconian_keys = self.BACONIAN_DICT.keys()\n\n        # Decode I=J and U=V variant\n        for i in ctext_split:\n            if i in baconian_keys:\n                ctext_decoded += self.BACONIAN_DICT[i]\n\n        # Decode variant that assigns each letter a unique code\n        for i in ctext_split:\n            if \"+\" + i in baconian_keys:\n                ctext_decoded2 += self.BACONIAN_DICT[\"+\" + i]\n\n        candidates.append(ctext_decoded)\n        candidates.append(ctext_decoded2)\n        for candidate in candidates:\n            if candidate != \"\":\n                if candidate == candidates[0]:\n                    result.append(CrackResult(value=candidate, key_info=\"I=J & U=V\"))\n                else:\n                    result.append(CrackResult(value=candidate))\n        logging.debug(f\"Baconian cracker - Returning results: {result}\")\n        return result\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"dict\": ParamSpec(\n                desc=\"The Baconian alphabet dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::baconian\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.BACONIAN_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n\n# Path: ciphey/basemods/Crackers/caesar.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.common import fix_case\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Caesar(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"caesar\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying caesar cipher on {ctext}\")\n        # Convert it to lower case\n        #\n        # TODO: handle different alphabets\n        if self.lower:\n            message = ctext.lower()\n        else:\n            message = ctext\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"Caesar returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(message, candidate.key, self.group)\n            candidates.append(\n                CrackResult(value=fix_case(translated, ctext), key_info=candidate.key)\n            )\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the caesar cipher alphabet\",\n                req=False,\n                default=\"abcdefghijklmnopqrstuvwxyz\",\n            ),\n            \"lower\": ParamSpec(\n                desc=\"Whether or not the ciphertext should be converted to lowercase first\",\n                req=False,\n                default=True,\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.lower: Union[str, bool] = self._params()[\"lower\"]\n        if not isinstance(self.lower, bool):\n            self.lower = util.strtobool(self.lower)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/rot47.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\n\nfrom typing import Dict, List, Optional\n\nimport cipheycore\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import Config, Cracker, CrackInfo, CrackResult, ParamSpec, registry\n\n\n@registry.register\nclass Rot47(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n\n        return CrackInfo(\n            success_likelihood=cipheycore.caesar_detect(analysis, self.expected),\n            # TODO: actually calculate runtimes\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"rot47\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        logging.info(f\"Trying ROT47 cipher on {ctext}\")\n\n        logging.debug(\"Beginning cipheycore simple analysis\")\n\n        # Hand it off to the core\n        analysis = self.cache.get_or_update(\n            ctext,\n            \"cipheycore::simple_analysis\",\n            lambda: cipheycore.analyse_string(ctext),\n        )\n        logging.debug(\"Beginning cipheycore::caesar\")\n        possible_keys = cipheycore.caesar_crack(\n            analysis, self.expected, self.group, self.p_value\n        )\n\n        n_candidates = len(possible_keys)\n        logging.info(f\"ROT47 returned {n_candidates} candidates\")\n\n        if n_candidates == 0:\n            logging.debug(\"Filtering for better results\")\n            analysis = cipheycore.analyse_string(ctext, self.group)\n            possible_keys = cipheycore.caesar_crack(\n                analysis, self.expected, self.group, self.p_value\n            )\n\n        candidates = []\n\n        for candidate in possible_keys:\n            logging.debug(f\"Candidate {candidate.key} has prob {candidate.p_value}\")\n            translated = cipheycore.caesar_decrypt(ctext, candidate.key, self.group)\n            candidates.append(CrackResult(value=translated, key_info=candidate.key))\n\n        return candidates\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"expected\": ParamSpec(\n                desc=\"The expected distribution of the plaintext\",\n                req=False,\n                config_ref=[\"default_dist\"],\n            ),\n            \"group\": ParamSpec(\n                desc=\"An ordered sequence of chars that make up the ROT47 cipher alphabet\",\n                req=False,\n                default=\"\"\"!\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\"\"\",\n            ),\n            \"p_value\": ParamSpec(\n                desc=\"The p-value to use for standard frequency analysis\",\n                req=False,\n                default=0.01,\n            )\n            # TODO: add \"filter\" param\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.group = list(self._params()[\"group\"])\n        self.expected = config.get_resource(self._params()[\"expected\"])\n        self.cache = config.cache\n        self.p_value = float(self._params()[\"p_value\"])\n\n# Path: ciphey/basemods/Crackers/soundex.py\nimport re\nfrom typing import Dict, List, Optional\n\nimport logging\nfrom rich.logging import RichHandler\n\nfrom ciphey.iface import (\n    Config,\n    Cracker,\n    CrackInfo,\n    CrackResult,\n    ParamSpec,\n    Translation,\n    registry,\n)\n\n\n@registry.register\nclass Soundex(Cracker[str]):\n    def getInfo(self, ctext: str) -> CrackInfo:\n        return CrackInfo(\n            success_likelihood=0.1,\n            success_runtime=1e-5,\n            failure_runtime=1e-5,\n        )\n\n    @staticmethod\n    def getTarget() -> str:\n        return \"soundex\"\n\n    def attemptCrack(self, ctext: str) -> List[CrackResult]:\n        \"\"\"\n        Attempts to crack Soundex by generating all possible combinations.\n        \"\"\"\n        logging.debug(\"Attempting Soundex cracker\")\n        word_list = []\n        sentences = []\n        result = []\n\n        # Convert to uppercase and replace delimiters and whitespace with nothing\n        ctext = re.sub(r\"[,;:\\-\\s]\", \"\", ctext.upper())\n\n        # Make sure ctext contains only A-Z and 0-9\n        if bool(re.search(r\"[^A-Z0-9]\", ctext)) is True:\n            logging.debug(\"Failed to crack soundex due to non soundex character(s)\")\n            return None\n\n        # Make sure ctext is divisible by 4\n        ctext_len = len(ctext)\n        if ctext_len % 4:\n            logging.debug(\n                f\"Failed to decode Soundex because length must be a multiple of 4, not '{ctext_len}'\"\n            )\n            return None\n\n        # Split ctext into groups of 4\n        ctext = \" \".join(ctext[i : i + 4] for i in range(0, len(ctext), 4))\n        ctext_split = ctext.split(\" \")\n        soundex_keys = self.SOUNDEX_DICT.keys()\n\n        # Find all words that correspond to each given soundex code\n        for code in ctext_split:\n            if code in soundex_keys:\n                word_list.append(self.SOUNDEX_DICT[code])\n\n        logging.info(f\"Possible words for given encoded text: {word_list}\")\n\n        # Find all possible sentences\n        self.getSentenceCombo(\n            word_list,\n            sentences,\n            self.frequency_dict,\n            self.sentence_freq,\n            self.word_freq,\n        )\n\n        sorted_sentences = self.sortlistwithdict(sentences, self.frequency_dict)\n\n        for sentence in sorted_sentences:\n            result.append(CrackResult(value=sentence))\n\n        logging.debug(f\"Soundex cracker - Returning results: {result}\")\n        return result\n\n    \ndef sortlistwithdict(self, listtosort, hashes):\n        \"\"\"\n        This function uses the sum of ranks (based on frequency) of each word in each\n        sentence and sorts them according to it.\n        \"\"\"\n        return sorted(listtosort, key=lambda x: hashes[x])\n\n    def getSentenceCombo(\n        self, A, sentences, frequency_dict, sentence_freq, word_freq, result=\"\", n=0\n    ):\n        \"\"\"\n        This function uses recursion to generate a list of sentences from all possible\n        words for a given set of soundex codes.\n        \"\"\"\n        logging.debug(\"Creating all possible sentences from Soundex\")\n        if n == len(A):\n            sentences.append(result[1:])\n            for word in result[1:].split():\n                # Adding the rank of each word to find out the sentence's net frequency\n                if word in word_freq:\n                    sentence_freq += word_freq.index(word)\n                # If the word isn't in the frequency list then it's a very uncommon word\n                # so we add a large number (5000)\n                else:\n                    sentence_freq += 5000\n            frequency_dict[result[1:]] = sentence_freq\n            sentence_freq = 0\n            return\n\n        for word in A[n]:\n            out = result + \" \" + word\n            self.getSentenceCombo(\n                A, sentences, frequency_dict, sentence_freq, word_freq, out, n + 1\n            )\n\n    @staticmethod\n    def getParams() -> Optional[Dict[str, ParamSpec]]:\n        return {\n            \"dict\": ParamSpec(\n                desc=\"The Soundex dictionary to use\",\n                req=False,\n                default=\"cipheydists::translate::soundex\",\n            ),\n            \"freq\": ParamSpec(\n                desc=\"The word frequency dictionary to use\",\n                req=False,\n                default=\"cipheydists::list::English5000Freq\",\n            ),\n        }\n\n    def __init__(self, config: Config):\n        super().__init__(config)\n        self.SOUNDEX_DICT = config.get_resource(self._params()[\"dict\"], Translation)\n        self.word_freq = config.get_resource(self._params()[\"freq\"], Translation)\n        self.frequency_dict = {}\n        self.sentence_freq = 0\n\n# Path: ciphey/basemods/Crackers/vigenere.py\n\"\"\"\n \u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\n\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255d\u255a\u2588\u2588\u2557 \u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255d\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557   \u255a\u2588\u2588\u2588\u2588\u2554\u255d\n\u2588\u2588\u2551     \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u255d \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255d    \u255a\u2588\u2588\u2554\u255d\n\u255a\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557   \u2588\u2588\u2551\n\u00a9 Brandon Skerritt\nGithub: brandonskerritt\n\"\"\"\nfrom distutils import util\nfrom typing import Dict, List, Optional, Union\n...\n", "needle_token_start": 15257, "needle_token_end": 15328, "code_context_ntokens": 16105, "output": ["```python\ndef sortlistwithdict(self, listtosort, hashes):\n    \"\"\"\n    This function uses the sum of ranks (based on frequency) of each word in each\n    sentence and sorts them according to it.\n    \"\"\"\n    return sorted(listtosort, key=lambda x: hashes[x])\n```"]}
{"repo": "apache/logging-log4cxx", "name": "ConfiguratorWatchdog", "language": "cpp", "path": "src/main/cpp/optionconverter.cpp", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To monitor changes in a configuration file and reload the configuration if any changes are detected.\n2. **Input**: A configuration handler and the path to a configuration file.\n3. **Output**: There is no direct output; however, the effect is the reconfiguration of the system based on the updated file.\n4. **Procedure**: This component inherits from a file monitoring class, setting up a watch on the specified file. When a change is detected in the file, it uses the provided configuration handler to reapply the settings from the file.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/level.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n...\n// Path: src/main/cpp/optionconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <log4cxx/spi/loggerrepository.h>\n#include <log4cxx/appenderskeleton.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <algorithm>\n#include <ctype.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/exception.h>\n#include <stdlib.h>\n#include <log4cxx/helpers/properties.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/object.h>\n#include <log4cxx/helpers/class.h>\n#include <log4cxx/helpers/loader.h>\n#include <log4cxx/helpers/system.h>\n#include <log4cxx/propertyconfigurator.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/file.h>\n#include <log4cxx/xml/domconfigurator.h>\n#include <log4cxx/logmanager.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n#include <log4cxx/helpers/filewatchdog.h>\n\nnamespace LOG4CXX_NS\n{\n\nclass ConfiguratorWatchdog  : public helpers::FileWatchdog\n{\n\tspi::ConfiguratorPtr m_config;\n\tpublic:\n    \nConfiguratorWatchdog(const spi::ConfiguratorPtr& config, const File& filename)\n        : helpers::FileWatchdog(filename)\n        , m_config(config)\n    {\n    }\n\n    /**\n    Call PropertyConfigurator#doConfigure(const String& configFileName,\n    const spi::LoggerRepositoryPtr& hierarchy) with the\n    <code>filename</code> to reconfigure log4cxx.\n    */\n    void doOnChange() override\n    {\n        m_config->doConfigure(file(), LogManager::getLoggerRepository());\n    }\n};\n\n}\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\n\nLogString OptionConverter::convertSpecialChars(const LogString& s)\n{\n\tlogchar c;\n\tLogString sbuf;\n\n\tLogString::const_iterator i = s.begin();\n\n\twhile (i != s.end())\n\t{\n\t\tc = *i++;\n\n\t\tif (c == 0x5C /* '\\\\' */)\n\t\t{\n\t\t\tc =  *i++;\n\n\t\t\tswitch (c)\n\t\t\t{\n\t\t\t\tcase 0x6E: //'n'\n\t\t\t\t\tc = 0x0A;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x72: //'r'\n\t\t\t\t\tc = 0x0D;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x74: //'t'\n\t\t\t\t\tc = 0x09;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x66: //'f'\n\t\t\t\t\tc = 0x0C;\n\t\t\t\t\tbreak;\n\n\t\t\t\tdefault:\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tsbuf.append(1, c);\n\t}\n\n\treturn sbuf;\n}\n\n\nbool OptionConverter::toBoolean(const LogString& value, bool dEfault)\n{\n\tif (value.length() >= 4)\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(value.substr(0, 4),\n\t\t\t\tLOG4CXX_STR(\"TRUE\"), LOG4CXX_STR(\"true\")))\n\t\t{\n\t\t\treturn true;\n\t\t}\n\t}\n\n\tif (dEfault && value.length() >= 5)\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(value.substr(0, 5),\n\t\t\t\tLOG4CXX_STR(\"FALSE\"), LOG4CXX_STR(\"false\")))\n\t\t{\n\t\t\treturn false;\n\t\t}\n\t}\n\n\treturn dEfault;\n}\n\nint OptionConverter::toInt(const LogString& value, int dEfault)\n{\n\tLogString trimmed(StringHelper::trim(value));\n\n\tif (trimmed.empty())\n\t{\n\t\treturn dEfault;\n\t}\n\n\tLOG4CXX_ENCODE_CHAR(cvalue, trimmed);\n\n\treturn (int) atol(cvalue.c_str());\n}\n\nlong OptionConverter::toFileSize(const LogString& s, long dEfault)\n{\n\tif (s.empty())\n\t{\n\t\treturn dEfault;\n\t}\n\n\tsize_t index = s.find_first_of(LOG4CXX_STR(\"bB\"));\n\n\tif (index != LogString::npos && index > 0)\n\t{\n\t\tlong multiplier = 1;\n\t\tindex--;\n\n\t\tif (s[index] == 0x6B /* 'k' */ || s[index] == 0x4B /* 'K' */)\n\t\t{\n\t\t\tmultiplier = 1024;\n\t\t}\n\t\telse if (s[index] == 0x6D /* 'm' */ || s[index] == 0x4D /* 'M' */)\n\t\t{\n\t\t\tmultiplier = 1024 * 1024;\n\t\t}\n\t\telse if (s[index] == 0x67 /* 'g'*/ || s[index] == 0x47 /* 'G' */)\n\t\t{\n\t\t\tmultiplier = 1024 * 1024 * 1024;\n\t\t}\n\n\t\treturn toInt(s.substr(0, index), 1) * multiplier;\n\t}\n\n\treturn toInt(s, 1);\n}\n\nLogString OptionConverter::findAndSubst(const LogString& key, Properties& props)\n{\n\tLogString value(props.getProperty(key));\n\n\tif (value.empty())\n\t{\n\t\treturn value;\n\t}\n\n\ttry\n\t{\n\t\treturn substVars(value, props);\n\t}\n\tcatch (IllegalArgumentException& e)\n\t{\n\t\tLogLog::error(((LogString) LOG4CXX_STR(\"Bad option value [\"))\n\t\t\t+ value + LOG4CXX_STR(\"].\"), e);\n\t\treturn value;\n\t}\n}\n\nLogString OptionConverter::substVars(const LogString& val, Properties& props)\n{\n\tLogString sbuf;\n\tconst logchar delimStartArray[] = { 0x24, 0x7B, 0 };\n\tconst LogString delimStart(delimStartArray);\n\tconst logchar delimStop = 0x7D; // '}';\n\tconst size_t DELIM_START_LEN = 2;\n\tconst size_t DELIM_STOP_LEN = 1;\n\n\tsize_t i = 0;\n\n\twhile (true)\n\t{\n\t\tsize_t j = val.find(delimStart, i);\n\n\t\tif (j == val.npos)\n\t\t{\n\t\t\t// no more variables\n\t\t\tif (i == 0)\n\t\t\t{\n\t\t\t\t// this is a simple string\n\t\t\t\treturn val;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t// add the tail string which contails no variables and return the result.\n\t\t\t\tsbuf.append(val.substr(i, val.length() - i));\n\t\t\t\treturn sbuf;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tsbuf.append(val.substr(i, j - i));\n\t\t\tsize_t k = val.find(delimStop, j);\n\n\t\t\tif (k == val.npos)\n\t\t\t{\n\t\t\t\tLogString msg(1, (logchar) 0x22 /* '\\\"' */);\n\t\t\t\tmsg.append(val);\n\t\t\t\tmsg.append(LOG4CXX_STR(\"\\\" has no closing brace. Opening brace at position \"));\n\t\t\t\tPool p;\n\t\t\t\tStringHelper::toString(j, p, msg);\n\t\t\t\tmsg.append(1, (logchar) 0x2E /* '.' */);\n\t\t\t\tthrow IllegalArgumentException(msg);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tj += DELIM_START_LEN;\n\t\t\t\tLogString key = val.substr(j, k - j);\n\t\t\t\t// first try in System properties\n\t\t\t\tLogString replacement(getSystemProperty(key, LogString()));\n\n\t\t\t\t// then try props parameter\n\t\t\t\tif (replacement.empty())\n\t\t\t\t{\n\t\t\t\t\treplacement = props.getProperty(key);\n\t\t\t\t}\n\n\t\t\t\tif (!replacement.empty())\n\t\t\t\t{\n\t\t\t\t\t// Do variable substitution on the replacement string\n\t\t\t\t\t// such that we can solve \"Hello ${x2}\" as \"Hello p1\"\n\t\t\t\t\t// the where the properties are\n\t\t\t\t\t// x1=p1\n\t\t\t\t\t// x2=${x1}\n\t\t\t\t\tLogString recursiveReplacement = substVars(replacement, props);\n\t\t\t\t\tsbuf.append(recursiveReplacement);\n\t\t\t\t}\n\n\t\t\t\ti = k + DELIM_STOP_LEN;\n\t\t\t}\n\t\t}\n\t}\n}\n\nLogString OptionConverter::getSystemProperty(const LogString& key, const LogString& def)\n{\n\tif (!key.empty())\n\t{\n\t\tLogString value(System::getProperty(key));\n\n\t\tif (!value.empty())\n\t\t{\n\t\t\treturn value;\n\t\t}\n\t}\n\n\treturn def;\n}\n\nLevelPtr OptionConverter::toLevel(const LogString& value,\n\tconst LevelPtr& defaultValue)\n{\n\tsize_t hashIndex = value.find(LOG4CXX_STR(\"#\"));\n\n\tif (hashIndex == LogString::npos)\n\t{\n\t\tif (value.empty())\n\t\t{\n\t\t\treturn defaultValue;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tLogLog::debug(\n\t\t\t\t((LogString) LOG4CXX_STR(\"OptionConverter::toLevel: no class name specified, level=[\"))\n\t\t\t\t+ value\n\t\t\t\t+ LOG4CXX_STR(\"]\"));\n\t\t\t// no class name specified : use standard Level class\n\t\t\treturn Level::toLevelLS(value, defaultValue);\n\t\t}\n\t}\n\n\tLogString clazz = value.substr(hashIndex + 1);\n\tLogString levelName = value.substr(0, hashIndex);\n\tLogLog::debug(((LogString) LOG4CXX_STR(\"OptionConverter::toLevel: class=[\"))\n\t\t+ clazz + LOG4CXX_STR(\"], level=[\") + levelName + LOG4CXX_STR(\"]\"));\n\n\t// This is degenerate case but you never know.\n\tif (levelName.empty())\n\t{\n\t\treturn Level::toLevelLS(value, defaultValue);\n\t}\n\n\ttry\n\t{\n\t\tLevel::LevelClass& levelClass =\n\t\t\t(Level::LevelClass&)Loader::loadClass(clazz);\n\t\treturn levelClass.toLevel(levelName);\n\t}\n\tcatch (ClassNotFoundException&)\n\t{\n\t\tLogLog::warn(((LogString) LOG4CXX_STR(\"custom level class [\"))\n\t\t\t+ clazz + LOG4CXX_STR(\"] not found.\"));\n\t}\n\tcatch (Exception& oops)\n\t{\n\t\tLogLog::warn(\n\t\t\tLOG4CXX_STR(\"class [\") + clazz + LOG4CXX_STR(\"], level [\") + levelName +\n\t\t\tLOG4CXX_STR(\"] conversion) failed.\"), oops);\n\t}\n\tcatch (...)\n\t{\n\t\tLogLog::warn(\n\t\t\tLOG4CXX_STR(\"class [\") + clazz + LOG4CXX_STR(\"], level [\") + levelName +\n\t\t\tLOG4CXX_STR(\"] conversion) failed.\"));\n\t}\n\n\treturn defaultValue;\n}\n\n\nObjectPtr OptionConverter::instantiateByKey(Properties& props, const LogString& key,\n\tconst Class& superClass, const ObjectPtr& defaultValue)\n{\n\t// Get the value of the property in string form\n\tLogString className(findAndSubst(key, props));\n\n\tif (className.empty())\n\t{\n\t\tLogLog::error(\n\t\t\t((LogString) LOG4CXX_STR(\"Could not find value for key \")) + key);\n\t\treturn defaultValue;\n\t}\n\n\t// Trim className to avoid trailing spaces that cause problems.\n\treturn OptionConverter::instantiateByClassName(\n\t\t\tStringHelper::trim(className), superClass, defaultValue);\n}\n\nObjectPtr OptionConverter::instantiateByClassName(const LogString& className,\n\tconst Class& superClass, const ObjectPtr& defaultValue)\n{\n\tif (!className.empty())\n\t{\n\t\ttry\n\t\t{\n\t\t\tconst Class& classObj = Loader::loadClass(className);\n\t\t\tObjectPtr newObject =  ObjectPtr(classObj.newInstance());\n\n\t\t\tif (!newObject->instanceof(superClass))\n\t\t\t{\n\t\t\t\treturn defaultValue;\n\t\t\t}\n\n\t\t\treturn newObject;\n\t\t}\n\t\tcatch (Exception& e)\n\t\t{\n\t\t\tLogLog::error(LOG4CXX_STR(\"Could not instantiate class [\") +\n\t\t\t\tclassName + LOG4CXX_STR(\"].\"), e);\n\t\t}\n\t}\n\n\treturn defaultValue;\n}\n\nvoid OptionConverter::selectAndConfigure(const File& configFileName,\n\tconst LogString& _clazz, spi::LoggerRepositoryPtr hierarchy, int delay)\n{\n\tConfiguratorPtr configurator;\n\tLogString clazz = _clazz;\n\n\tLogString filename(configFileName.getPath());\n\n#if LOG4CXX_HAS_DOMCONFIGURATOR\n\tif (clazz.empty()\n\t\t&& filename.length() > 4\n\t\t&& StringHelper::equalsIgnoreCase(\n\t\t\tfilename.substr(filename.length() - 4),\n\t\t\tLOG4CXX_STR(\".XML\"), LOG4CXX_STR(\".xml\")))\n\t{\n\t\tclazz = LOG4CXX_NS::xml::DOMConfigurator::getStaticClass().toString();\n\t}\n#endif\n\n\tif (!clazz.empty())\n\t{\n\t\tLogLog::debug(LOG4CXX_STR(\"Preferred configurator class: \") + clazz);\n\t\tconst Class& clazzObj = Loader::loadClass(clazz);\n\t\tObjectPtr obj = ObjectPtr(clazzObj.newInstance());\n\t\tconfigurator = LOG4CXX_NS::cast<Configurator>(obj);\n\n\t\tif (configurator == 0)\n\t\t{\n\t\t\tLogLog::error(LOG4CXX_STR(\"Could not instantiate configurator [\")\n\t\t\t\t+ clazz + LOG4CXX_STR(\"].\"));\n\t\t\treturn;\n\t\t}\n\t}\n\telse\n\t{\n\t\tconfigurator = std::make_shared<PropertyConfigurator>();\n\t}\n\n\tif (0 < delay)\n\t{\n\t\tauto dog = new ConfiguratorWatchdog(configurator, configFileName);\n\t\tAPRInitializer::registerCleanup(dog);\n\t\tdog->setDelay(delay);\n\t\tdog->start();\n\t}\n\telse\n\t\tconfigurator->doConfigure(configFileName, hierarchy);\n}\n\n// Path: src/main/cpp/locale.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/helpers/locale.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct Locale::LocalePrivate\n{\n\tLocalePrivate(const LogString& language1)\n\t\t: language(language1)\n\t{\n\t}\n\n\tLocalePrivate(const LogString& language1, const LogString& country1)\n\t\t: language(language1), country(country1)\n\t{\n\t}\n\n\tLocalePrivate(const LogString& language1, const LogString& country1,\n\t\tconst LogString& variant1)\n\t\t: language(language1), country(country1), variant(variant1)\n\t{\n\t}\n\n\tconst LogString language;\n\tconst LogString country;\n\tconst LogString variant;\n};\n\nLocale::Locale(const LogString& language1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1))\n{\n}\n\nLocale::Locale(const LogString& language1, const LogString& country1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1, country1))\n{\n}\n\nLocale::Locale(const LogString& language1, const LogString& country1,\n\tconst LogString& variant1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1, country1, variant1))\n{\n}\n\nLocale::~Locale() {}\n\nconst LogString& Locale::getLanguage() const\n{\n\treturn m_priv->language;\n}\n\nconst LogString& Locale::getCountry() const\n{\n\treturn m_priv->country;\n}\n\nconst LogString& Locale::getVariant() const\n{\n\treturn m_priv->variant;\n}\n\n\n// Path: src/main/cpp/charsetencoder.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/exception.h>\n#include <apr_xlate.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <algorithm>\n\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n\n#include <log4cxx/private/log4cxx_private.h>\n#include <apr_portable.h>\n#include <mutex>\n\n#ifdef LOG4CXX_HAS_WCSTOMBS\n\t#include <stdlib.h>\n#endif\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(CharsetEncoder)\n\nnamespace LOG4CXX_NS\n{\n\nnamespace helpers\n{\n\n#if APR_HAS_XLATE\n/**\n* A character encoder implemented using apr_xlate.\n*/\nclass APRCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tAPRCharsetEncoder(const LogString& topage) : pool()\n\t\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\t\t\tconst char* frompage = \"WCHAR_T\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\tconst char* frompage = \"UTF-8\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\t\tconst char* frompage = \"UTF-16\";\n#endif\n\t\t\tstd::string tpage(Transcoder::encodeCharsetName(topage));\n\t\t\tapr_status_t stat = apr_xlate_open(&convset,\n\t\t\t\t\ttpage.c_str(),\n\t\t\t\t\tfrompage,\n\t\t\t\t\tpool.getAPRPool());\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tthrow IllegalArgumentException(topage);\n\t\t\t}\n\t\t}\n\n\t\tvirtual ~APRCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tapr_status_t stat;\n\t\t\tsize_t outbytes_left = out.remaining();\n\t\t\tsize_t initial_outbytes_left = outbytes_left;\n\t\t\tsize_t position = out.position();\n\n\t\t\tif (iter == in.end())\n\t\t\t{\n\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\tstat = apr_xlate_conv_buffer(convset, NULL, NULL,\n\t\t\t\t\t\tout.data() + position, &outbytes_left);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tLogString::size_type inOffset = (iter - in.begin());\n\t\t\t\tapr_size_t inbytes_left =\n\t\t\t\t\t(in.size() - inOffset) * sizeof(LogString::value_type);\n\t\t\t\tapr_size_t initial_inbytes_left = inbytes_left;\n\t\t\t\t{\n\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\tstat = apr_xlate_conv_buffer(convset,\n\t\t\t\t\t\t\t(const char*) (in.data() + inOffset),\n\t\t\t\t\t\t\t&inbytes_left,\n\t\t\t\t\t\t\tout.data() + position,\n\t\t\t\t\t\t\t&outbytes_left);\n\t\t\t\t}\n\t\t\t\titer += ((initial_inbytes_left - inbytes_left) / sizeof(LogString::value_type));\n\t\t\t}\n\n\t\t\tout.position(out.position() + (initial_outbytes_left - outbytes_left));\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tAPRCharsetEncoder(const APRCharsetEncoder&);\n\t\tAPRCharsetEncoder& operator=(const APRCharsetEncoder&);\n\t\tPool pool;\n\t\tstd::mutex mutex;\n\t\tapr_xlate_t* convset;\n};\n#endif\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS\n/**\n *  A character encoder implemented using wcstombs.\n*/\nclass WcstombsCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tWcstombsCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\t/**\n\t\t *   Converts a wchar_t to the default external multibyte encoding.\n\t\t */\n\t\tlog4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\tsize_t outbytes_left = out.remaining();\n\t\t\t\tsize_t position = out.position();\n\t\t\t\tstd::wstring::size_type inOffset = (iter - in.begin());\n\t\t\t\tenum { BUFSIZE = 256 };\n\t\t\t\twchar_t buf[BUFSIZE];\n\t\t\t\tsize_t chunkSize = BUFSIZE - 1;\n\n\t\t\t\tif (chunkSize * MB_LEN_MAX > outbytes_left)\n\t\t\t\t{\n\t\t\t\t\tchunkSize = outbytes_left / MB_LEN_MAX;\n\t\t\t\t}\n\n\t\t\t\tif (chunkSize > in.length() - inOffset)\n\t\t\t\t{\n\t\t\t\t\tchunkSize = in.length() - inOffset;\n\t\t\t\t}\n\n\t\t\t\tmemset(buf, 0, BUFSIZE * sizeof(wchar_t));\n\t\t\t\tmemcpy(buf,\n\t\t\t\t\tin.data() + inOffset,\n\t\t\t\t\tchunkSize * sizeof(wchar_t));\n\t\t\t\tsize_t converted = wcstombs(out.data() + position, buf, outbytes_left);\n\n\t\t\t\tif (converted == (size_t) -1)\n\t\t\t\t{\n\t\t\t\t\tstat = APR_BADARG;\n\n\t\t\t\t\t//\n\t\t\t\t\t//   if unconvertable character was encountered\n\t\t\t\t\t//       repeatedly halve source to get fragment that\n\t\t\t\t\t//       can be converted\n\t\t\t\t\tfor (chunkSize /= 2;\n\t\t\t\t\t\tchunkSize > 0;\n\t\t\t\t\t\tchunkSize /= 2)\n\t\t\t\t\t{\n\t\t\t\t\t\tbuf[chunkSize] = 0;\n\t\t\t\t\t\tconverted = wcstombs(out.data() + position, buf, outbytes_left);\n\n\t\t\t\t\t\tif (converted != (size_t) -1)\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\titer += chunkSize;\n\t\t\t\t\t\t\tout.position(out.position() + converted);\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\titer += chunkSize;\n\t\t\t\t\tout.position(out.position() + converted);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\n\tprivate:\n\t\tWcstombsCharsetEncoder(const WcstombsCharsetEncoder&);\n\t\tWcstombsCharsetEncoder& operator=(const WcstombsCharsetEncoder&);\n};\n#endif\n\n\n/**\n*   Encodes a LogString to US-ASCII.\n*/\nclass USASCIICharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tUSASCIICharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\twhile (out.remaining() > 0 && iter != in.end())\n\t\t\t\t{\n\t\t\t\t\tLogString::const_iterator prev(iter);\n\t\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\t\tif (sv <= 0x7F)\n\t\t\t\t\t{\n\t\t\t\t\t\tout.put((char) sv);\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\titer = prev;\n\t\t\t\t\t\tstat = APR_BADARG;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tUSASCIICharsetEncoder(const USASCIICharsetEncoder&);\n\t\tUSASCIICharsetEncoder& operator=(const USASCIICharsetEncoder&);\n};\n\n/**\n*   Converts a LogString to ISO-8859-1.\n*/\nclass ISOLatinCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tISOLatinCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\twhile (out.remaining() > 0 && iter != in.end())\n\t\t\t\t{\n\t\t\t\t\tLogString::const_iterator prev(iter);\n\t\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\t\tif (sv <= 0xFF)\n\t\t\t\t\t{\n\t\t\t\t\t\tout.put((char) sv);\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\titer = prev;\n\t\t\t\t\t\tstat = APR_BADARG;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tISOLatinCharsetEncoder(const ISOLatinCharsetEncoder&);\n\t\tISOLatinCharsetEncoder& operator=(const ISOLatinCharsetEncoder&);\n};\n\n/**\n*   Encodes a LogString to a byte array when the encodings are identical.\n*/\nclass TrivialCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tTrivialCharsetEncoder()\n\t\t{\n\t\t}\n\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\tsize_t requested = in.length() - (iter - in.begin());\n\n\t\t\t\tif (requested > out.remaining() / sizeof(logchar))\n\t\t\t\t{\n\t\t\t\t\trequested = out.remaining() / sizeof(logchar);\n\t\t\t\t}\n\n\t\t\t\tmemcpy(out.current(),\n\t\t\t\t\t(const char*) in.data() + (iter - in.begin()),\n\t\t\t\t\trequested * sizeof(logchar));\n\t\t\t\titer += requested;\n\t\t\t\tout.position(out.position() + requested * sizeof(logchar));\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tTrivialCharsetEncoder(const TrivialCharsetEncoder&);\n\t\tTrivialCharsetEncoder& operator=(const TrivialCharsetEncoder&);\n};\n\n#if LOG4CXX_LOGCHAR_IS_UTF8\ntypedef TrivialCharsetEncoder UTF8CharsetEncoder;\n#else\n/**\n *  Converts a LogString to UTF-8.\n */\nclass UTF8CharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tUTF8CharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\twhile (iter != in.end() && out.remaining() >= 8)\n\t\t\t{\n\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t}\n\n\t\t\t\tTranscoder::encodeUTF8(sv, out);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tUTF8CharsetEncoder(const UTF8CharsetEncoder&);\n\t\tUTF8CharsetEncoder& operator=(const UTF8CharsetEncoder&);\n};\n#endif\n\n/**\n *   Encodes a LogString to UTF16-BE.\n */\nclass UTF16BECharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tUTF16BECharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\twhile (iter != in.end() && out.remaining() >= 4)\n\t\t\t{\n\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t}\n\n\t\t\t\tTranscoder::encodeUTF16BE(sv, out);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tUTF16BECharsetEncoder(const UTF16BECharsetEncoder&);\n\t\tUTF16BECharsetEncoder& operator=(const UTF16BECharsetEncoder&);\n};\n\n/**\n *   Encodes a LogString to UTF16-LE.\n */\nclass UTF16LECharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tUTF16LECharsetEncoder()\n\t\t{\n\t\t}\n\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\twhile (iter != in.end() && out.remaining() >= 4)\n\t\t\t{\n\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t}\n\n\t\t\t\tTranscoder::encodeUTF16LE(sv, out);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\tprivate:\n\t\tUTF16LECharsetEncoder(const UTF16LECharsetEncoder&);\n\t\tUTF16LECharsetEncoder& operator=(const UTF16LECharsetEncoder&);\n};\n\n/**\n *    Charset encoder that uses current locale settings.\n */\nclass LocaleCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tLocaleCharsetEncoder() : state()\n\t\t{\n\t\t}\n\t\tlog4cxx_status_t encode\n\t\t\t( const LogString&           in\n\t\t\t, LogString::const_iterator& iter\n\t\t\t, ByteBuffer&                out\n\t\t\t) override\n\t\t{\n\t\t\tlog4cxx_status_t result = APR_SUCCESS;\n#if !LOG4CXX_CHARSET_EBCDIC\n\t\t\tchar* current = out.current();\n\t\t\tsize_t remain = out.remaining();\n\t\t\tif (std::mbsinit(&this->state)) // ByteBuffer not partially encoded?\n\t\t\t{\n\t\t\t\t// Copy single byte characters\n\t\t\t\tfor (;\n\t\t\t\t\titer != in.end() && ((unsigned int) *iter) < 0x80 && 0 < remain;\n\t\t\t\t\titer++, remain--, current++)\n\t\t\t\t{\n\t\t\t\t\t*current = *iter;\n\t\t\t\t}\n\t\t\t}\n#endif\n\t\t\t// Encode characters that may require multiple bytes\n\t\t\twhile (iter != in.end() && MB_CUR_MAX <= remain)\n\t\t\t{\n\t\t\t\tauto ch = Transcoder::decode(in, iter);\n\t\t\t\tauto n = std::wcrtomb(current, ch, &this->state);\n\t\t\t\tif (static_cast<std::size_t>(-1) == n) // not a valid wide character?\n\t\t\t\t{\n\t\t\t\t\tresult = APR_BADARG;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tremain -= n;\n\t\t\t\tcurrent += n;\n\t\t\t}\n\t\t\tout.position(current - out.data());\n\t\t\treturn result;\n\t\t}\n\n\tprivate:\n\t\tstd::mbstate_t state;\n};\n\n\n} // namespace helpers\n\n}  //namespace log4cxx\n\n\n\nCharsetEncoder::CharsetEncoder()\n{\n}\n\nCharsetEncoder::~CharsetEncoder()\n{\n}\n\nCharsetEncoderPtr CharsetEncoder::getDefaultEncoder()\n{\n\tstatic WideLife<CharsetEncoderPtr> encoder(createDefaultEncoder());\n\n\t//\n\t//  if invoked after static variable destruction\n\t//     (if logging is called in the destructor of a static object)\n\t//     then create a new decoder.\n\t//\n\tif (encoder.value() == 0)\n\t{\n\t\treturn CharsetEncoderPtr( createDefaultEncoder() );\n\t}\n\n\treturn encoder;\n}\n\nCharsetEncoder* CharsetEncoder::createDefaultEncoder()\n{\n#if LOG4CXX_CHARSET_UTF8\n\treturn new UTF8CharsetEncoder();\n#elif LOG4CXX_CHARSET_ISO88591\n\treturn new ISOLatinCharsetEncoder();\n#elif LOG4CXX_CHARSET_USASCII\n\treturn new USASCIICharsetEncoder();\n#elif LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS\n\treturn new WcstombsCharsetEncoder();\n#else\n\treturn new LocaleCharsetEncoder();\n#endif\n}\n\n\nCharsetEncoderPtr CharsetEncoder::getUTF8Encoder()\n{\n\treturn std::make_shared<UTF8CharsetEncoder>();\n}\n\n\n\nCharsetEncoderPtr CharsetEncoder::getEncoder(const LogString& charset)\n{\n\tif (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-8\"), LOG4CXX_STR(\"utf-8\"))\n\t\t|| StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP65001\"), LOG4CXX_STR(\"cp65001\")))\n\t{\n\t\treturn std::make_shared<UTF8CharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"C\"), LOG4CXX_STR(\"c\")) ||\n\t\tcharset == LOG4CXX_STR(\"646\") ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"US-ASCII\"), LOG4CXX_STR(\"us-ascii\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO646-US\"), LOG4CXX_STR(\"iso646-US\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ANSI_X3.4-1968\"), LOG4CXX_STR(\"ansi_x3.4-1968\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP20127\"), LOG4CXX_STR(\"cp20127\")))\n\t{\n\t\treturn std::make_shared<USASCIICharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-8859-1\"), LOG4CXX_STR(\"iso-8859-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-LATIN-1\"), LOG4CXX_STR(\"iso-latin-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP1252\"), LOG4CXX_STR(\"cp1252\")))\n\t{\n\t\treturn std::make_shared<ISOLatinCharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-16BE\"), LOG4CXX_STR(\"utf-16be\"))\n\t\t|| StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-16\"), LOG4CXX_STR(\"utf-16\"))\n\t\t|| StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP1200\"), LOG4CXX_STR(\"cp1200\")))\n\t{\n\t\treturn std::make_shared<UTF16BECharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-16LE\"), LOG4CXX_STR(\"utf-16le\")))\n\t{\n\t\treturn std::make_shared<UTF16LECharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"LOCALE\"), LOG4CXX_STR(\"locale\")))\n\t{\n\t\treturn std::make_shared<LocaleCharsetEncoder>();\n\t}\n\n#if APR_HAS_XLATE\n\treturn std::make_shared<APRCharsetEncoder>(charset);\n#else\n\tthrow IllegalArgumentException(charset);\n#endif\n}\n\n\nvoid CharsetEncoder::reset()\n{\n}\n\nvoid CharsetEncoder::flush(ByteBuffer& /* out */ )\n{\n}\n\n\nvoid CharsetEncoder::encode(CharsetEncoderPtr& enc,\n\tconst LogString& src,\n\tLogString::const_iterator& iter,\n\tByteBuffer& dst)\n{\n\tlog4cxx_status_t stat = enc->encode(src, iter, dst);\n\n\tif (stat != APR_SUCCESS && iter != src.end())\n\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR || LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\titer++;\n#elif LOG4CXX_LOGCHAR_IS_UTF8\n\n\t\t//  advance past this character and all continuation characters\n\t\twhile ((*(++iter) & 0xC0) == 0x80);\n\n#else\n#error logchar is unrecognized\n#endif\n\t\tdst.put(Transcoder::LOSSCHAR);\n\t}\n}\n\nbool CharsetEncoder::isTriviallyCopyable(const LogString& src, const CharsetEncoderPtr& enc)\n{\n\tbool result;\n#if !LOG4CXX_CHARSET_EBCDIC\n\tif (dynamic_cast<LocaleCharsetEncoder*>(enc.get()))\n\t{\n\t\tresult = src.end() == std::find_if(src.begin(), src.end()\n\t\t\t, [](const logchar& ch) -> bool { return 0x80 <= (unsigned int)ch; });\n\t}\n\telse\n#endif\n\t\tresult = !!dynamic_cast<TrivialCharsetEncoder*>(enc.get());\n\treturn result;\n}\n\n// Path: src/main/cpp/colorstartpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/colorstartpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(ColorStartPatternConverter)\n\n#define priv static_cast<ColorPatternConverterPrivate*>(m_priv.get())\n\nstatic LogString colorToANSISequence(const LogString& color, bool isForeground, Pool& pool){\n\tint numberToConvert = 0;\n\n\tif(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"BLACK\"), LOG4CXX_STR(\"black\"))){\n\t\tnumberToConvert = 30;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"RED\"), LOG4CXX_STR(\"red\"))){\n\t\tnumberToConvert = 31;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"GREEN\"), LOG4CXX_STR(\"green\"))){\n\t\tnumberToConvert = 32;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"YELLOW\"), LOG4CXX_STR(\"yellow\"))){\n\t\tnumberToConvert = 33;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"BLUE\"), LOG4CXX_STR(\"blue\"))){\n\t\tnumberToConvert = 34;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"MAGENTA\"), LOG4CXX_STR(\"magenta\"))){\n\t\tnumberToConvert = 35;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"CYAN\"), LOG4CXX_STR(\"cyan\"))){\n\t\tnumberToConvert = 36;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"WHITE\"), LOG4CXX_STR(\"white\"))){\n\t\tnumberToConvert = 37;\n\t}\n\n\tif( numberToConvert == 0 ){\n\t\treturn LOG4CXX_STR(\"\");\n\t}\n\tLogString ret;\n\tif( isForeground == false ){\n\t\tnumberToConvert += 10;\n\t}\n\tStringHelper::toString(numberToConvert, pool, ret);\n\treturn ret;\n}\n\nstatic LogString graphicsModeToANSISequence(const LogString& graphicsMode, Pool& pool){\n\tint numberToConvert = 0;\n\n\tif(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"BOLD\"), LOG4CXX_STR(\"bold\"))){\n\t\tnumberToConvert = 1;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"DIM\"), LOG4CXX_STR(\"dim\"))){\n\t\tnumberToConvert = 2;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"ITALIC\"), LOG4CXX_STR(\"italic\"))){\n\t\tnumberToConvert = 3;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"UNDERLINE\"), LOG4CXX_STR(\"underline\"))){\n\t\tnumberToConvert = 4;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"BLINKING\"), LOG4CXX_STR(\"blinking\"))){\n\t\tnumberToConvert = 5;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"INVERSE\"), LOG4CXX_STR(\"inverse\"))){\n\t\tnumberToConvert = 7;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"STRIKETHROUGH\"), LOG4CXX_STR(\"strikethrough\"))){\n\t\tnumberToConvert = 9;\n\t}\n\n\tif( numberToConvert == 0 ){\n\t\treturn LOG4CXX_STR(\"\");\n\t}\n\tLogString ret;\n\tStringHelper::toString(numberToConvert, pool, ret);\n\treturn ret;\n}\n\nstatic LogString convertSingleSequence(const LogString& sequence, Pool& pool){\n\tLogString strInParens;\n\tbool inParens = false;\n\tbool hasParens = false;\n\tsize_t x = 0;\n\n\tfor(x = 0; x < sequence.length(); x++){\n\t\tif( sequence[x] == '(' && !inParens ){\n\t\t\tinParens = true;\n\t\t\thasParens = true;\n\t\t\tcontinue;\n\t\t}else if( sequence[x] == '(' && inParens ){\n\t\t\t// Unbalanced parens - parse invalid\n\t\t\treturn LOG4CXX_STR(\"\");\n\t\t}\n\n\t\tif( sequence[x] == ')' && inParens ){\n\t\t\thasParens = true;\n\t\t\tinParens = false;\n\t\t\tbreak;\n\t\t}\n\n\t\tif( inParens ){\n\t\t\tstrInParens.push_back(sequence[x]);\n\t\t}\n\t}\n\n\tif( (x != (sequence.length() - 1) || inParens) && hasParens ){\n\t\t// Unbalanced parens, or more data in the string than we expected - parse invalid\n\t\treturn LOG4CXX_STR(\"\");\n\t}\n\n\tif(StringHelper::startsWith(sequence, LOG4CXX_STR(\"fg(\"))){\n\t\t// Parse foreground\n\t\treturn colorToANSISequence(strInParens, true, pool);\n\t}else if(StringHelper::startsWith(sequence, LOG4CXX_STR(\"bg(\"))){\n\t\treturn colorToANSISequence(strInParens, false, pool);\n\t}else{\n\t\treturn graphicsModeToANSISequence(sequence, pool);\n\t}\n}\n\nstruct ColorStartPatternConverter::ColorPatternConverterPrivate : public PatternConverterPrivate\n{\n\tColorPatternConverterPrivate( const LogString& name, const LogString& style ) :\n\t\tPatternConverterPrivate( name, style ){}\n\n\tLogString m_fatalColor;\n\tLogString m_errorColor;\n\tLogString m_warnColor;\n\tLogString m_infoColor;\n\tLogString m_debugColor;\n\tLogString m_traceColor;\n};\n\nColorStartPatternConverter::ColorStartPatternConverter() :\n\tLoggingEventPatternConverter(std::make_unique<ColorPatternConverterPrivate>(LOG4CXX_STR(\"Color Start\"),\n\t\tLOG4CXX_STR(\"colorStart\")))\n{\n}\n\nPatternConverterPtr ColorStartPatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> instance = std::make_shared<ColorStartPatternConverter>();\n\treturn instance;\n}\n\nvoid ColorStartPatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& p) const\n{\n\n\tLOG4CXX_NS::LevelPtr lvl = event->getLevel();\n\n\tswitch (lvl->toInt())\n\t{\n\t\tcase LOG4CXX_NS::Level::FATAL_INT:\n\t\t\ttoAppendTo.append(priv->m_fatalColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::ERROR_INT:\n\t\t\ttoAppendTo.append(priv->m_errorColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::WARN_INT:\n\t\t\ttoAppendTo.append(priv->m_warnColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::INFO_INT:\n\t\t\ttoAppendTo.append(priv->m_infoColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::DEBUG_INT:\n\t\t\ttoAppendTo.append(priv->m_debugColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::TRACE_INT:\n\t\t\ttoAppendTo.append(priv->m_traceColor);\n\t\t\tbreak;\n\n\t\tdefault:\n\t\t\tbreak;\n\t}\n}\n\nvoid ColorStartPatternConverter::setFatalColor(const LogString& color){\n\tparseColor(color, &(priv->m_fatalColor));\n}\n\nvoid ColorStartPatternConverter::setErrorColor(const LogString& color){\n\tparseColor(color, &(priv->m_errorColor));\n}\n\nvoid ColorStartPatternConverter::setWarnColor(const LogString& color){\n\tparseColor(color, &(priv->m_warnColor));\n}\n\nvoid ColorStartPatternConverter::setInfoColor(const LogString& color){\n\tparseColor(color, &(priv->m_infoColor));\n}\n\nvoid ColorStartPatternConverter::setDebugColor(const LogString& color){\n\tparseColor(color, &(priv->m_debugColor));\n}\n\nvoid ColorStartPatternConverter::setTraceColor(const LogString& color){\n\tparseColor(color, &(priv->m_traceColor));\n}\n\nvoid ColorStartPatternConverter::parseColor(const LogString& color, LogString* result){\n\tLogString lower = StringHelper::toLowerCase(color);\n\tPool pool;\n\n\t// If the color we are trying to parse is blank, clear our result\n\tif(StringHelper::trim(color).empty() ||\n\t\t\tStringHelper::equalsIgnoreCase(color,\n\t\t\t\t\t\t\t\t\t\t   LOG4CXX_STR(\"NONE\"),\n\t\t\t\t\t\t\t\t\t\t   LOG4CXX_STR(\"none\"))){\n\t\tresult->clear();\n\t\treturn;\n\t}\n\n\tif( StringHelper::startsWith(lower, LOG4CXX_STR(\"\\\\x1b\")) ){\n\t\tif( color[color.size() - 1] != 'm' ){\n\t\t\t// In order for this to be a valid ANSI escape sequence,\n\t\t\t// it must end with an 'm'.  If it does not, reject.\n\t\t\treturn;\n\t\t}\n\t\t// We start with an escape sequence, copy the data over after the escape byte\n\t\tresult->clear();\n\t\tresult->append(LOG4CXX_STR(\"\\x1b\"));\n\t\tfor( size_t x = 4; x < color.size(); x++ ){\n\t\t\tresult->push_back(color[x]);\n\t\t}\n\t}else{\n\t\t// We do not start with an escape sequence: try to parse color\n\t\t// Escape sequence information:\n\t\t// https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797\n\t\t// https://en.wikipedia.org/wiki/ANSI_escape_code\n\t\tresult->clear();\n\t\tresult->append(LOG4CXX_STR(\"\\x1b[\"));\n\t\tLogString tmp;\n\t\tfor( size_t x = 0; x < color.size(); x++ ){\n\t\t\tif(color[x] == '|' ){\n\t\t\t\tLogString toAppend = convertSingleSequence(tmp, pool);\n\t\t\t\ttmp.clear();\n\t\t\t\tif(!toAppend.empty()){\n\t\t\t\t\tresult->push_back(';');\n\t\t\t\t\tresult->append(toAppend);\n\t\t\t\t}\n\t\t\t}else{\n\t\t\t\ttmp.push_back(color[x]);\n\t\t\t}\n\t\t}\n\t\tLogString toAppend = convertSingleSequence(tmp, pool);\n\t\ttmp.clear();\n\t\tif(!toAppend.empty()){\n\t\t\tresult->push_back(';');\n\t\t\tresult->append(toAppend);\n\t\t}\n\t\tresult->append(LOG4CXX_STR(\"m\"));\n\t}\n}\n\n// Path: src/main/cpp/literalpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/literalpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\n...\n", "needle_token_start": 767, "needle_token_end": 816, "code_context_ntokens": 15188, "output": ["```cpp\nnamespace LOG4CXX_NS\n{\n\nclass ConfiguratorWatchdog  : public helpers::FileWatchdog\n{\n\tspi::ConfiguratorPtr m_config;\n\tpublic:\n    \nConfiguratorWatchdog(const spi::ConfiguratorPtr& config, const File& filename)\n        : helpers::FileWatchdog(filename)\n        , m_config(config)\n    {\n    }\n\n    /**\n    Call PropertyConfigurator#doConfigure(const String& configFileName,\n    const spi::LoggerRepositoryPtr& hierarchy) with the\n    <code>filename</code> to reconfigure log4cxx.\n    */\n    void doOnChange() override\n    {\n        m_config->doConfigure(file(), LogManager::getLoggerRepository());\n    }\n};\n\n}\n```"]}
{"repo": "apache/logging-log4cxx", "name": "TelnetAppenderPriv", "language": "cpp", "path": "src/main/cpp/telnetappender.cpp", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this component is to manage the logging output via a network connection, allowing multiple clients to view log messages in real-time through a telnet session.\n2. **Input**: It takes two primary inputs: a port number for the telnet server to listen on, and a maximum number of client connections that can simultaneously connect to the telnet server.\n3. **Output**: There is no direct output in the traditional sense, as this component primarily handles network connections and streams logging data to connected clients.\n4. **Procedure**: This component initializes a server listening on a specified port with a capacity for a specified maximum number of connections. It manages these connections, ensuring that each connected client receives real-time logging data. The component uses UTF-8 encoding for transmitting data to maintain consistency and support international character sets.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/fileappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/fileappender.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/pool.h>\n#include <log4cxx/helpers/fileoutputstream.h>\n#include <log4cxx/helpers/outputstreamwriter.h>\n#include <log4cxx/helpers/bufferedwriter.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/private/writerappender_priv.h>\n#include <log4cxx/private/fileappender_priv.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\nIMPLEMENT_LOG4CXX_OBJECT(FileAppender)\n\n#define _priv static_cast<FileAppenderPriv*>(m_priv.get())\n\nFileAppender::FileAppender() :\n\tWriterAppender (std::make_unique<FileAppenderPriv>())\n{\n}\n\nFileAppender::FileAppender\n\t( const LayoutPtr& layout1\n\t, const LogString& fileName1\n\t, bool append1\n\t, bool bufferedIO1\n\t, int bufferSize1\n\t)\n...\n// Path: src/main/cpp/transform.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/transform.h>\n#include <log4cxx/helpers/widelife.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\n\n\nvoid Transform::appendEscapingTags(\n\tLogString& buf, const LogString& input)\n{\n\t//Check if the string is zero length -- if so, return\n\t//what was sent in.\n\n\tif (input.length() == 0 )\n\t{\n\t\treturn;\n\t}\n\n\tlogchar specials[] = { 0x22 /* \" */, 0x26 /* & */, 0x3C /* < */, 0x3E /* > */, 0x00 };\n\tsize_t start = 0;\n\tsize_t special = input.find_first_of(specials, start);\n\n\twhile (special != LogString::npos)\n\t{\n\t\tif (special > start)\n\t\t{\n\t\t\tbuf.append(input, start, special - start);\n\t\t}\n\n\t\tswitch (input[special])\n\t\t{\n\t\t\tcase 0x22:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&quot;\"));\n\t\t\t\tbreak;\n\n\t\t\tcase 0x26:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&amp;\"));\n\t\t\t\tbreak;\n\n\t\t\tcase 0x3C:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&lt;\"));\n\t\t\t\tbreak;\n\n\t\t\tcase 0x3E:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&gt;\"));\n\t\t\t\tbreak;\n\n\t\t\tdefault:\n\t\t\t\tbuf.append(1, input[special]);\n\t\t\t\tbreak;\n\t\t}\n\n\t\tstart = special + 1;\n\n\t\tif (special < input.size())\n\t\t{\n\t\t\tspecial = input.find_first_of(specials, start);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tspecial = LogString::npos;\n\t\t}\n\t}\n\n\tif (start < input.size())\n\t{\n\t\tbuf.append(input, start, input.size() - start);\n\t}\n}\n\nvoid Transform::appendEscapingCDATA(\n\tLogString& buf, const LogString& input)\n{\n\tstatic const WideLife<LogString> CDATA_END(LOG4CXX_STR(\"]]>\"));\n\tstatic const WideLife<LogString> CDATA_EMBEDED_END(LOG4CXX_STR(\"]]>]]&gt;<![CDATA[\"));\n\n\tconst LogString::size_type CDATA_END_LEN = 3;\n\n\n\tif (input.length() == 0 )\n\t{\n\t\treturn;\n\t}\n\n\tLogString::size_type end = input.find(CDATA_END);\n\n\tif (end == LogString::npos)\n\t{\n\t\tbuf.append(input);\n\t\treturn;\n\t}\n\n\tLogString::size_type start = 0;\n\n\twhile (end != LogString::npos)\n\t{\n\t\tbuf.append(input, start, end - start);\n\t\tbuf.append(CDATA_EMBEDED_END);\n\t\tstart = end + CDATA_END_LEN;\n\n\t\tif (start < input.length())\n\t\t{\n\t\t\tend = input.find(CDATA_END, start);\n\t\t}\n\t\telse\n\t\t{\n\t\t\treturn;\n\t\t}\n\t}\n\n\tbuf.append(input, start, input.length() - start);\n}\n\n\n// Path: src/main/cpp/telnetappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/net/telnetappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/threadutility.h>\n#include <log4cxx/private/appenderskeleton_priv.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::net;\n\nIMPLEMENT_LOG4CXX_OBJECT(TelnetAppender)\n\nstruct TelnetAppender::TelnetAppenderPriv : public AppenderSkeletonPrivate\n{\n\t\nTelnetAppenderPriv( int port, int maxConnections ) : AppenderSkeletonPrivate(),\n\t\tport(port),\n\t\tconnections(maxConnections),\n\t\tencoding(LOG4CXX_STR(\"UTF-8\")),\n\t\tencoder(CharsetEncoder::getUTF8Encoder()),\n\t\tsh(),\n\t\tactiveConnections(0) {}\n\n\tint port;\n\tConnectionList connections;\n\tLogString encoding;\n\tLOG4CXX_NS::helpers::CharsetEncoderPtr encoder;\n\tstd::unique_ptr<helpers::ServerSocket> serverSocket;\n\tstd::thread sh;\n\tsize_t activeConnections;\n};\n\n#define _priv static_cast<TelnetAppenderPriv*>(m_priv.get())\n\n/** The default telnet server port */\nconst int TelnetAppender::DEFAULT_PORT = 23;\n\n/** The maximum number of concurrent connections */\nconst int TelnetAppender::MAX_CONNECTIONS = 20;\n\nTelnetAppender::TelnetAppender()\n\t: AppenderSkeleton (std::make_unique<TelnetAppenderPriv>(DEFAULT_PORT, MAX_CONNECTIONS))\n{\n}\n\nTelnetAppender::~TelnetAppender()\n{\n\tfinalize();\n}\n\nvoid TelnetAppender::activateOptions(Pool& /* p */)\n{\n\tif (_priv->serverSocket == NULL)\n\t{\n\t\t_priv->serverSocket = ServerSocket::create(_priv->port);\n\t\t_priv->serverSocket->setSoTimeout(1000);\n\t}\n\n\t_priv->sh = ThreadUtility::instance()->createThread( LOG4CXX_STR(\"TelnetAppender\"), &TelnetAppender::acceptConnections, this );\n}\n\nvoid TelnetAppender::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"PORT\"), LOG4CXX_STR(\"port\")))\n\t{\n\t\tsetPort(OptionConverter::toInt(value, DEFAULT_PORT));\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"ENCODING\"), LOG4CXX_STR(\"encoding\")))\n\t{\n\t\tsetEncoding(value);\n\t}\n\telse\n\t{\n\t\tAppenderSkeleton::setOption(option, value);\n\t}\n}\n\nLogString TelnetAppender::getEncoding() const\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\treturn _priv->encoding;\n}\n\nvoid TelnetAppender::setEncoding(const LogString& value)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t_priv->encoder = CharsetEncoder::getEncoder(value);\n\t_priv->encoding = value;\n}\n\n\nvoid TelnetAppender::close()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\tif (_priv->closed)\n\t{\n\t\treturn;\n\t}\n\n\t_priv->closed = true;\n\n\tSocketPtr nullSocket;\n\n\tfor (auto& item : _priv->connections)\n\t{\n\t\tif (item)\n\t\t{\n\t\t\titem->close();\n\t\t\titem = nullSocket;\n\t\t}\n\t}\n\n\tif (_priv->serverSocket != NULL)\n\t{\n\t\ttry\n\t\t{\n\t\t\t_priv->serverSocket->close();\n\t\t}\n\t\tcatch (Exception&)\n\t\t{\n\t\t}\n\t}\n\n\tif ( _priv->sh.joinable() )\n\t{\n\t\t_priv->sh.join();\n\t}\n\n\t_priv->activeConnections = 0;\n}\n\n\nvoid TelnetAppender::write(ByteBuffer& buf)\n{\n\tfor (auto& item :_priv->connections)\n\t{\n\t\tif (item)\n\t\t{\n\t\t\ttry\n\t\t\t{\n\t\t\t\tByteBuffer b(buf.current(), buf.remaining());\n\t\t\t\titem->write(b);\n\t\t\t}\n\t\t\tcatch (Exception&)\n\t\t\t{\n\t\t\t\t// The client has closed the connection, remove it from our list:\n\t\t\t\titem.reset();\n\t\t\t\t_priv->activeConnections--;\n\t\t\t}\n\t\t}\n\t}\n}\n\nvoid TelnetAppender::writeStatus(const SocketPtr& socket, const LogString& msg, Pool& p)\n{\n\tsize_t bytesSize = msg.size() * 2;\n\tchar* bytes = p.pstralloc(bytesSize);\n\n\tLogString::const_iterator msgIter(msg.begin());\n\tByteBuffer buf(bytes, bytesSize);\n\n\twhile (msgIter != msg.end())\n\t{\n\t\t_priv->encoder->encode(msg, msgIter, buf);\n\t\tbuf.flip();\n\t\tsocket->write(buf);\n\t\tbuf.clear();\n\t}\n}\n\nvoid TelnetAppender::append(const spi::LoggingEventPtr& event, Pool& p)\n{\n\tsize_t count = _priv->activeConnections;\n\n\tif (count > 0)\n\t{\n\t\tLogString msg;\n\t\t_priv->layout->format(msg, event, _priv->pool);\n\t\tmsg.append(LOG4CXX_STR(\"\\r\\n\"));\n\t\tsize_t bytesSize = msg.size() * 2;\n\t\tchar* bytes = p.pstralloc(bytesSize);\n\n\t\tLogString::const_iterator msgIter(msg.begin());\n\t\tByteBuffer buf(bytes, bytesSize);\n\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\t\twhile (msgIter != msg.end())\n\t\t{\n\t\t\tlog4cxx_status_t stat = _priv->encoder->encode(msg, msgIter, buf);\n\t\t\tbuf.flip();\n\t\t\twrite(buf);\n\t\t\tbuf.clear();\n\n\t\t\tif (CharsetEncoder::isError(stat))\n\t\t\t{\n\t\t\t\tLogString unrepresented(1, 0x3F /* '?' */);\n\t\t\t\tLogString::const_iterator unrepresentedIter(unrepresented.begin());\n\t\t\t\tstat = _priv->encoder->encode(unrepresented, unrepresentedIter, buf);\n\t\t\t\tbuf.flip();\n\t\t\t\twrite(buf);\n\t\t\t\tbuf.clear();\n\t\t\t\tmsgIter++;\n\t\t\t}\n\t\t}\n\t}\n}\n\nvoid TelnetAppender::acceptConnections()\n{\n\n\t// main loop; is left when This->closed is != 0 after an accept()\n\twhile (true)\n\t{\n\t\ttry\n\t\t{\n\t\t\tSocketPtr newClient = _priv->serverSocket->accept();\n\t\t\tbool done = _priv->closed;\n\n\t\t\tif (done)\n\t\t\t{\n\t\t\t\tPool p;\n\t\t\t\twriteStatus(newClient, LOG4CXX_STR(\"Log closed.\\r\\n\"), p);\n\t\t\t\tnewClient->close();\n\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tsize_t count = _priv->activeConnections;\n\n\t\t\tif (count >= _priv->connections.size())\n\t\t\t{\n\t\t\t\tPool p;\n\t\t\t\twriteStatus(newClient, LOG4CXX_STR(\"Too many connections.\\r\\n\"), p);\n\t\t\t\tnewClient->close();\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t//\n\t\t\t\t//   find unoccupied connection\n\t\t\t\t//\n\t\t\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\t\t\t\tfor (auto& item : _priv->connections)\n\t\t\t\t{\n\t\t\t\t\tif (!item)\n\t\t\t\t\t{\n\t\t\t\t\t\titem = newClient;\n\t\t\t\t\t\t_priv->activeConnections++;\n\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tPool p;\n\t\t\t\tLogString oss(LOG4CXX_STR(\"TelnetAppender v1.0 (\"));\n\t\t\t\tStringHelper::toString((int) count + 1, p, oss);\n\t\t\t\toss += LOG4CXX_STR(\" active connections)\\r\\n\\r\\n\");\n\t\t\t\twriteStatus(newClient, oss, p);\n\t\t\t}\n\t\t}\n\t\tcatch (InterruptedIOException&)\n\t\t{\n\t\t\tif (_priv->closed)\n\t\t\t{\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\tcatch (Exception& e)\n\t\t{\n\t\t\tif (!_priv->closed)\n\t\t\t{\n\t\t\t\tLogLog::error(LOG4CXX_STR(\"Encountered error while in SocketHandler loop.\"), e);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\n}\n\nint TelnetAppender::getPort() const\n{\n\treturn _priv->port;\n}\n\nvoid TelnetAppender::setPort(int port1)\n{\n\t_priv->port = port1;\n}\n\n// Path: src/main/cpp/outputdebugstringappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#if defined(_WIN32)\n#include <log4cxx/nt/outputdebugstringappender.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/private/appenderskeleton_priv.h>\n\n#include \"windows.h\"\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::nt;\n\nIMPLEMENT_LOG4CXX_OBJECT(OutputDebugStringAppender)\n\nOutputDebugStringAppender::OutputDebugStringAppender()\n{\n}\n\nvoid OutputDebugStringAppender::append(const spi::LoggingEventPtr& event, Pool& p)\n{\n\tLogString buf;\n\tthis->m_priv->layout->format(buf, event, p);\n#if LOG4CXX_WCHAR_T_API\n\tLOG4CXX_ENCODE_WCHAR(wstr, buf);\n\t::OutputDebugStringW(wstr.c_str());\n#else\n\tLOG4CXX_ENCODE_CHAR(str, buf);\n\t::OutputDebugStringA(str.c_str());\n#endif\n}\n\n#endif\n\n\n// Path: src/main/cpp/charsetdecoder.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#define NOMINMAX /* tell windows not to define min/max macros */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/charsetdecoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/pool.h>\n#include <log4cxx/helpers/loglog.h>\n#include <apr_xlate.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n#include <locale.h>\n#include <apr_portable.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(CharsetDecoder)\n\n\nnamespace LOG4CXX_NS\n{\nnamespace helpers\n{\n\n#if APR_HAS_XLATE\n/**\n *  Converts from an arbitrary encoding to LogString\n *    using apr_xlate.  Requires real iconv implementation,\n*    apr-iconv will crash in use.\n */\nclass APRCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\t/**\n\t\t *  Creates a new instance.\n\t\t *  @param frompage name of source encoding.\n\t\t */\n\t\tAPRCharsetDecoder(const LogString& frompage) : pool()\n\t\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\t\t\tconst char* topage = \"WCHAR_T\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\tconst char* topage = \"UTF-8\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\t\tconst char* topage = \"UTF-16\";\n#endif\n\t\t\tstd::string fpage(Transcoder::encodeCharsetName(frompage));\n\t\t\tapr_status_t stat = apr_xlate_open(&convset,\n\t\t\t\t\ttopage,\n\t\t\t\t\tfpage.c_str(),\n\t\t\t\t\tpool.getAPRPool());\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tthrow IllegalArgumentException(frompage);\n\t\t\t}\n\t\t}\n\n\t\t/**\n\t\t *  Destructor.\n\t\t */\n\t\tvirtual ~APRCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tenum { BUFSIZE = 256 };\n\t\t\tlogchar buf[BUFSIZE];\n\t\t\tconst apr_size_t initial_outbytes_left = BUFSIZE * sizeof(logchar);\n\t\t\tapr_status_t stat = APR_SUCCESS;\n\n\t\t\tif (in.remaining() == 0)\n\t\t\t{\n\t\t\t\tsize_t outbytes_left = initial_outbytes_left;\n\t\t\t\t{\n\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\tstat = apr_xlate_conv_buffer((apr_xlate_t*) convset,\n\t\t\t\t\t\t\tNULL, NULL, (char*) buf, &outbytes_left);\n\t\t\t\t}\n\t\t\t\tout.append(buf, (initial_outbytes_left - outbytes_left) / sizeof(logchar));\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\twhile (in.remaining() > 0 && stat == APR_SUCCESS)\n\t\t\t\t{\n\t\t\t\t\tsize_t inbytes_left = in.remaining();\n\t\t\t\t\tsize_t initial_inbytes_left = inbytes_left;\n\t\t\t\t\tsize_t pos = in.position();\n\t\t\t\t\tapr_size_t outbytes_left = initial_outbytes_left;\n\t\t\t\t\t{\n\t\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\t\tstat = apr_xlate_conv_buffer((apr_xlate_t*) convset,\n\t\t\t\t\t\t\t\tin.data() + pos,\n\t\t\t\t\t\t\t\t&inbytes_left,\n\t\t\t\t\t\t\t\t(char*) buf,\n\t\t\t\t\t\t\t\t&outbytes_left);\n\t\t\t\t\t}\n\t\t\t\t\tout.append(buf, (initial_outbytes_left - outbytes_left) / sizeof(logchar));\n\t\t\t\t\tin.position(pos + (initial_inbytes_left - inbytes_left));\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tAPRCharsetDecoder(const APRCharsetDecoder&);\n\t\tAPRCharsetDecoder& operator=(const APRCharsetDecoder&);\n\t\tLOG4CXX_NS::helpers::Pool pool;\n\t\tstd::mutex mutex;\n\t\tapr_xlate_t* convset;\n};\n\n#endif\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_MBSRTOWCS\n/**\n*    Converts from the default multi-byte string to\n*        LogString using mbstowcs.\n*\n*/\nclass MbstowcsCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tMbstowcsCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~MbstowcsCharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tinline log4cxx_status_t append(LogString& out, const wchar_t* buf)\n\t\t{\n\t\t\tout.append(buf);\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\t\t\tenum { BUFSIZE = 256 };\n\t\t\twchar_t wbuf[BUFSIZE];\n\t\t\tchar cbuf[BUFSIZE*4];\n\n\t\t\tmbstate_t mbstate;\n\t\t\tmemset(&mbstate, 0, sizeof(mbstate));\n\n\t\t\twhile (in.remaining() > 0)\n\t\t\t{\n\t\t\t\tconst char* src = in.current();\n\n\t\t\t\tif (*src == 0)\n\t\t\t\t{\n\t\t\t\t\tout.append(1, (logchar) 0);\n\t\t\t\t\tin.position(in.position() + 1);\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\tauto available = std::min(sizeof (cbuf) - 1, in.remaining());\n\t\t\t\t\tstrncpy(cbuf, src, available);\n\t\t\t\t\tcbuf[available] = 0;\n\t\t\t\t\tsrc = cbuf;\n\t\t\t\t\tsize_t wCharCount = mbsrtowcs(wbuf,\n\t\t\t\t\t\t\t&src,\n\t\t\t\t\t\t\tBUFSIZE - 1,\n\t\t\t\t\t\t\t&mbstate);\n\t\t\t\t\tauto converted = src - cbuf;\n\t\t\t\t\tin.position(in.position() + converted);\n\n\t\t\t\t\tif (wCharCount == (size_t) -1) // Illegal byte sequence?\n\t\t\t\t\t{\n\t\t\t\t\t\tLogString msg(LOG4CXX_STR(\"Illegal byte sequence at \"));\n\t\t\t\t\t\tmsg.append(std::to_wstring(in.position()));\n\t\t\t\t\t\tmsg.append(LOG4CXX_STR(\" of \"));\n\t\t\t\t\t\tmsg.append(std::to_wstring(in.limit()));\n\t\t\t\t\t\tLogLog::warn(msg);\n\t\t\t\t\t\tstat = APR_BADCH;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\twbuf[wCharCount] = 0;\n\t\t\t\t\t\tstat = append(out, wbuf);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\n\tprivate:\n\t\tMbstowcsCharsetDecoder(const MbstowcsCharsetDecoder&);\n\t\tMbstowcsCharsetDecoder& operator=(const MbstowcsCharsetDecoder&);\n};\n#endif\n\n\n/**\n*    Decoder used when the external and internal charsets\n*    are the same.\n*\n*/\nclass TrivialCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tTrivialCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~TrivialCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tsize_t remaining = in.remaining();\n\n\t\t\tif ( remaining > 0)\n\t\t\t{\n\t\t\t\tconst logchar* src = (const logchar*) (in.data() + in.position());\n\t\t\t\tsize_t count = remaining / sizeof(logchar);\n\t\t\t\tout.append(src, count);\n\t\t\t\tin.position(in.position() + remaining);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\n\n\tprivate:\n\t\tTrivialCharsetDecoder(const TrivialCharsetDecoder&);\n\t\tTrivialCharsetDecoder& operator=(const TrivialCharsetDecoder&);\n};\n\n\n#if LOG4CXX_LOGCHAR_IS_UTF8\ntypedef TrivialCharsetDecoder UTF8CharsetDecoder;\n#else\n/**\n*    Converts from UTF-8 to std::wstring\n*\n*/\nclass UTF8CharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tUTF8CharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~UTF8CharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tif (in.remaining() > 0)\n\t\t\t{\n\t\t\t\tstd::string tmp(in.current(), in.remaining());\n\t\t\t\tstd::string::const_iterator iter = tmp.begin();\n\n\t\t\t\twhile (iter != tmp.end())\n\t\t\t\t{\n\t\t\t\t\tunsigned int sv = Transcoder::decode(tmp, iter);\n\n\t\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t\t{\n\t\t\t\t\t\tsize_t offset = iter - tmp.begin();\n\t\t\t\t\t\tin.position(in.position() + offset);\n\t\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\tTranscoder::encode(sv, out);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tin.position(in.limit());\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tUTF8CharsetDecoder(const UTF8CharsetDecoder&);\n\t\tUTF8CharsetDecoder& operator=(const UTF8CharsetDecoder&);\n};\n#endif\n\n/**\n*    Converts from ISO-8859-1 to LogString.\n*\n*/\nclass ISOLatinCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tISOLatinCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~ISOLatinCharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tif (in.remaining() > 0)\n\t\t\t{\n\n\t\t\t\tconst unsigned char* src = (unsigned char*) in.current();\n\t\t\t\tconst unsigned char* srcEnd = src + in.remaining();\n\n\t\t\t\twhile (src < srcEnd)\n\t\t\t\t{\n\t\t\t\t\tunsigned int sv = *(src++);\n\t\t\t\t\tTranscoder::encode(sv, out);\n\t\t\t\t}\n\n\t\t\t\tin.position(in.limit());\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\n\n\tprivate:\n\t\tISOLatinCharsetDecoder(const ISOLatinCharsetDecoder&);\n\t\tISOLatinCharsetDecoder& operator=(const ISOLatinCharsetDecoder&);\n};\n\n\n/**\n*    Converts from US-ASCII to LogString.\n*\n*/\nclass USASCIICharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tUSASCIICharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~USASCIICharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (in.remaining() > 0)\n\t\t\t{\n\n\t\t\t\tconst unsigned char* src = (unsigned char*) in.current();\n\t\t\t\tconst unsigned char* srcEnd = src + in.remaining();\n\n\t\t\t\twhile (src < srcEnd)\n\t\t\t\t{\n\t\t\t\t\tunsigned char sv = *src;\n\n\t\t\t\t\tif (sv < 0x80)\n\t\t\t\t\t{\n\t\t\t\t\t\tsrc++;\n\t\t\t\t\t\tTranscoder::encode(sv, out);\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\tstat = APR_BADARG;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tin.position(src - (const unsigned char*) in.data());\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\n\tprivate:\n\t\tUSASCIICharsetDecoder(const USASCIICharsetDecoder&);\n\t\tUSASCIICharsetDecoder& operator=(const USASCIICharsetDecoder&);\n};\n\n/**\n *    Charset decoder that uses current locale settings.\n */\nclass LocaleCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tLocaleCharsetDecoder() : state()\n\t\t{\n\t\t}\n\t\tlog4cxx_status_t decode(ByteBuffer& in, LogString& out) override\n\t\t{\n\t\t\tlog4cxx_status_t result = APR_SUCCESS;\n\t\t\tconst char* p = in.current();\n\t\t\tsize_t i = in.position();\n\t\t\tsize_t remain = in.limit() - i;\n#if !LOG4CXX_CHARSET_EBCDIC\n\t\t\tif (std::mbsinit(&this->state)) // ByteBuffer not partially decoded?\n\t\t\t{\n\t\t\t\t// Copy single byte characters\n\t\t\t\tfor (; 0 < remain && ((unsigned int) *p) < 0x80; --remain, ++i, p++)\n\t\t\t\t{\n\t\t\t\t\tout.append(1, *p);\n\t\t\t\t}\n\t\t\t}\n#endif\n\t\t\t// Decode characters that may be represented by multiple bytes\n\t\t\twhile (0 < remain)\n\t\t\t{\n\t\t\t\twchar_t ch = 0;\n\t\t\t\tsize_t n = std::mbrtowc(&ch, p, remain, &this->state);\n\t\t\t\tif (0 == n) // NULL encountered?\n\t\t\t\t{\n\t\t\t\t\t++i;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tif (static_cast<std::size_t>(-1) == n) // decoding error?\n\t\t\t\t{\n\t\t\t\t\tresult = APR_BADARG;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tif (static_cast<std::size_t>(-2) == n) // incomplete sequence?\n\t\t\t\t{\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tTranscoder::encode(static_cast<unsigned int>(ch), out);\n\t\t\t\tremain -= n;\n\t\t\t\ti += n;\n\t\t\t\tp += n;\n\t\t\t}\n\t\t\tin.position(i);\n\t\t\treturn result;\n\t\t}\n\n\tprivate:\n\t\tstd::mbstate_t state;\n};\n\n\n\n} // namespace helpers\n\n}  //namespace log4cxx\n\n\nCharsetDecoder::CharsetDecoder()\n{\n}\n\n\nCharsetDecoder::~CharsetDecoder()\n{\n}\n\nCharsetDecoder* CharsetDecoder::createDefaultDecoder()\n{\n#if LOG4CXX_CHARSET_UTF8\n\treturn new UTF8CharsetDecoder();\n#elif LOG4CXX_CHARSET_ISO88591 || defined(_WIN32_WCE)\n\treturn new ISOLatinCharsetDecoder();\n#elif LOG4CXX_CHARSET_USASCII\n\treturn new USASCIICharsetDecoder();\n#elif LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_MBSRTOWCS\n\treturn new MbstowcsCharsetDecoder();\n#else\n\treturn new LocaleCharsetDecoder();\n#endif\n}\n\nCharsetDecoderPtr CharsetDecoder::getDefaultDecoder()\n{\n\tstatic WideLife<CharsetDecoderPtr> decoder(createDefaultDecoder());\n\n\t//\n\t//  if invoked after static variable destruction\n\t//     (if logging is called in the destructor of a static object)\n\t//     then create a new decoder.\n\t//\n\tif (decoder.value() == 0)\n\t{\n\t\treturn CharsetDecoderPtr( createDefaultDecoder() );\n\t}\n\n\treturn decoder;\n}\n\nCharsetDecoderPtr CharsetDecoder::getUTF8Decoder()\n{\n\tstatic WideLife<CharsetDecoderPtr> decoder(new UTF8CharsetDecoder());\n\n\t//\n\t//  if invoked after static variable destruction\n\t//     (if logging is called in the destructor of a static object)\n\t//     then create a new decoder.\n\t//\n\tif (decoder.value() == 0)\n\t{\n\t\treturn std::make_shared<UTF8CharsetDecoder>();\n\t}\n\n\treturn decoder;\n}\n\nCharsetDecoderPtr CharsetDecoder::getISOLatinDecoder()\n{\n\treturn std::make_shared<ISOLatinCharsetDecoder>();\n}\n\n\nCharsetDecoderPtr CharsetDecoder::getDecoder(const LogString& charset)\n{\n\tif (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-8\"), LOG4CXX_STR(\"utf-8\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF8\"), LOG4CXX_STR(\"utf8\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP65001\"), LOG4CXX_STR(\"cp65001\")))\n\t{\n\t\treturn std::make_shared<UTF8CharsetDecoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"C\"), LOG4CXX_STR(\"c\")) ||\n\t\tcharset == LOG4CXX_STR(\"646\") ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"US-ASCII\"), LOG4CXX_STR(\"us-ascii\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO646-US\"), LOG4CXX_STR(\"iso646-US\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ANSI_X3.4-1968\"), LOG4CXX_STR(\"ansi_x3.4-1968\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP20127\"), LOG4CXX_STR(\"cp20127\")))\n\t{\n\t\treturn std::make_shared<USASCIICharsetDecoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-8859-1\"), LOG4CXX_STR(\"iso-8859-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-LATIN-1\"), LOG4CXX_STR(\"iso-latin-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP1252\"), LOG4CXX_STR(\"cp1252\")))\n\t{\n\t\treturn std::make_shared<ISOLatinCharsetDecoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"LOCALE\"), LOG4CXX_STR(\"locale\")))\n\t{\n\t\treturn std::make_shared<LocaleCharsetDecoder>();\n\t}\n\n#if APR_HAS_XLATE\n\treturn std::make_shared<APRCharsetDecoder>(charset);\n#else\n\tthrow IllegalArgumentException(charset);\n#endif\n}\n\n\n\n\n\n\n\n// Path: src/main/cpp/throwableinformationpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/throwableinformationpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct ThrowableInformationPatternConverter::ThrowableInformationPatternConverterPrivate :\n\tpublic PatternConverterPrivate\n{\n\tThrowableInformationPatternConverterPrivate( const LogString& name, const LogString& style, bool shortReport ) :\n\t\tPatternConverterPrivate( name, style ),\n\t\tshortReport(shortReport) {}\n\n\t/**\n\t * If \"short\", only first line of throwable report will be formatted.\n\t */\n\tconst bool shortReport;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(ThrowableInformationPatternConverter)\n\nThrowableInformationPatternConverter::ThrowableInformationPatternConverter(bool shortReport1) :\n\tLoggingEventPatternConverter(\n\t\tstd::make_unique<ThrowableInformationPatternConverterPrivate>(\n\t\t\tLOG4CXX_STR(\"Throwable\"),\n\t\t\tLOG4CXX_STR(\"throwable\"),\n\t\t\tshortReport1))\n{\n}\n\nPatternConverterPtr ThrowableInformationPatternConverter::newInstance(\n\tconst std::vector<LogString>& options)\n{\n\tif (options.size() > 0 && options[0].compare(LOG4CXX_STR(\"short\")) == 0)\n\t{\n\t\tstatic WideLife<PatternConverterPtr> shortConverter = std::make_shared<ThrowableInformationPatternConverter>(true);\n\t\treturn shortConverter;\n\t}\n\n\tstatic WideLife<PatternConverterPtr> converter = std::make_shared<ThrowableInformationPatternConverter>(false);\n\treturn converter;\n}\n\nvoid ThrowableInformationPatternConverter::format(\n\tconst LoggingEventPtr& /* event */,\n\tLogString& /* toAppendTo */,\n\tPool& /* p */) const\n{\n}\n\n/**\n * This converter obviously handles throwables.\n * @return true.\n */\nbool ThrowableInformationPatternConverter::handlesThrowable() const\n{\n\treturn true;\n}\n\n// Path: src/main/cpp/htmllayout.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/htmllayout.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/transform.h>\n#include <log4cxx/helpers/iso8601dateformat.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/date.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\nstruct HTMLLayout::HTMLLayoutPrivate\n{\n\tHTMLLayoutPrivate()\n\t\t: locationInfo(false)\n\t\t, title(LOG4CXX_STR(\"Log4cxx Log Messages\"))\n\t\t, dateFormat()\n\t\t, expectedPatternLength(100)\n\t\t{}\n\n\t// Print no location info by default\n\tbool locationInfo; //= false\n\n\tLogString title;\n\n\thelpers::ISO8601DateFormat dateFormat;\n\n\t// Expected length of a formatted event excluding the message text\n\tsize_t expectedPatternLength;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(HTMLLayout)\n\n\nHTMLLayout::HTMLLayout()\n\t: m_priv(std::make_unique<HTMLLayoutPrivate>())\n{\n\tm_priv->dateFormat.setTimeZone(TimeZone::getGMT());\n\tm_priv->expectedPatternLength = getFormattedEventCharacterCount() * 2;\n}\n\nHTMLLayout::~HTMLLayout() {}\n\n\nvoid HTMLLayout::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\n\tif (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"TITLE\"), LOG4CXX_STR(\"title\")))\n\t{\n\t\tsetTitle(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"LOCATIONINFO\"), LOG4CXX_STR(\"locationinfo\")))\n\t{\n\t\tsetLocationInfo(OptionConverter::toBoolean(value, false));\n\t\tm_priv->expectedPatternLength = getFormattedEventCharacterCount() * 2;\n\t}\n}\n\nvoid HTMLLayout::format(LogString& output,\n\tconst spi::LoggingEventPtr& event,\n\tPool& p) const\n{\n\toutput.reserve(m_priv->expectedPatternLength + event->getMessage().size());\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<tr>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<td>\"));\n\n\tm_priv->dateFormat.format(output, event->getTimeStamp(), p);\n\n\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"\"));\n\tLogString threadName(event->getThreadName());\n\toutput.append(threadName);\n\toutput.append(LOG4CXX_STR(\" thread\\\">\"));\n\toutput.append(threadName);\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"Level\\\">\"));\n\n\tif (event->getLevel()->equals(Level::getDebug()))\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<font color=\\\"#339933\\\">\"));\n\t\toutput.append(event->getLevel()->toString());\n\t\toutput.append(LOG4CXX_STR(\"</font>\"));\n\t}\n\telse if (event->getLevel()->isGreaterOrEqual(Level::getWarn()))\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<font color=\\\"#993300\\\"><strong>\"));\n\t\toutput.append(event->getLevel()->toString());\n\t\toutput.append(LOG4CXX_STR(\"</strong></font>\"));\n\t}\n\telse\n\t{\n\t\toutput.append(event->getLevel()->toString());\n\t}\n\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"\"));\n\toutput.append(event->getLoggerName());\n\toutput.append(LOG4CXX_STR(\" logger\\\">\"));\n\tTransform::appendEscapingTags(output, event->getLoggerName());\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\tif (m_priv->locationInfo)\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<td>\"));\n\t\tconst LocationInfo& locInfo = event->getLocationInformation();\n\t\tLOG4CXX_DECODE_CHAR(fileName, locInfo.getFileName());\n\t\tTransform::appendEscapingTags(output, fileName);\n\t\toutput.append(1, (logchar) 0x3A /* ':' */);\n\t\tint line = event->getLocationInformation().getLineNumber();\n\n\t\tif (line != 0)\n\t\t{\n\t\t\tStringHelper::toString(line, p, output);\n\t\t}\n\n\t\toutput.append(LOG4CXX_STR(\"</td>\"));\n\t\toutput.append(LOG4CXX_EOL);\n\t}\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"Message\\\">\"));\n\tTransform::appendEscapingTags(output, event->getRenderedMessage());\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"</tr>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\tLogString ndcVal;\n\n\tif (event->getNDC(ndcVal))\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<tr><td bgcolor=\\\"#EEEEEE\\\" \"));\n\t\toutput.append(LOG4CXX_STR(\"style=\\\"font-size : xx-small;\\\" colspan=\\\"6\\\" \"));\n\t\toutput.append(LOG4CXX_STR(\"title=\\\"Nested Diagnostic Context\\\">\"));\n\t\toutput.append(LOG4CXX_STR(\"NDC: \"));\n\t\tTransform::appendEscapingTags(output, ndcVal);\n\t\toutput.append(LOG4CXX_STR(\"</td></tr>\"));\n\t\toutput.append(LOG4CXX_EOL);\n\t}\n}\n\nvoid HTMLLayout::appendHeader(LogString& output, Pool& p)\n{\n\toutput.append(LOG4CXX_STR(\"<!DOCTYPE HTML PUBLIC \"));\n\toutput.append(LOG4CXX_STR(\"\\\"-//W3C//DTD HTML 4.01 Transitional//EN\\\" \"));\n\toutput.append(LOG4CXX_STR(\"\\\"http://www.w3.org/TR/html4/loose.dtd\\\">\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<html>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<head>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<title>\"));\n\toutput.append(m_priv->title);\n\toutput.append(LOG4CXX_STR(\"</title>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<style type=\\\"text/css\\\">\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<!--\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"body, table {font-family: arial,sans-serif; font-size: x-small;}\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"th {background: #336699; color: #FFFFFF; text-align: left;}\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"-->\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"</style>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"</head>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<body bgcolor=\\\"#FFFFFF\\\" topmargin=\\\"6\\\" leftmargin=\\\"6\\\">\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<hr size=\\\"1\\\" noshade>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"Log session start time \"));\n\n\tm_priv->dateFormat.format(output, Date::currentTime(), p);\n\n\toutput.append(LOG4CXX_STR(\"<br>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<br>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<table cellspacing=\\\"0\\\" cellpadding=\\\"4\\\" border=\\\"1\\\" bordercolor=\\\"#224466\\\" width=\\\"100%\\\">\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<tr>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<th>Time</th>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<th>Thread</th>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<th>Level</th>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<th>Logger</th>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\tif (m_priv->locationInfo)\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<th>File:Line</th>\"));\n\t\toutput.append(LOG4CXX_EOL);\n\t}\n\n\toutput.append(LOG4CXX_STR(\"<th>Message</th>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"</tr>\"));\n\toutput.append(LOG4CXX_EOL);\n}\n\nvoid HTMLLayout::appendFooter(LogString& output, Pool& /* pool */ )\n{\n\toutput.append(LOG4CXX_STR(\"</table>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<br>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"</body></html>\"));\n}\n\nvoid HTMLLayout::setLocationInfo(bool locationInfoFlag)\n{\n\tm_priv->locationInfo = locationInfoFlag;\n}\n\nbool HTMLLayout::getLocationInfo() const\n{\n\treturn m_priv->locationInfo;\n}\n\nvoid HTMLLayout::setTitle(const LogString& title1)\n{\n\tm_priv->title.assign(title1);\n}\n\nconst LogString& HTMLLayout::getTitle() const\n{\n\treturn m_priv->title;\n}\n\nLogString HTMLLayout::getContentType() const\n{\n\treturn LOG4CXX_STR(\"text/html\");\n}\n\nbool HTMLLayout::ignoresThrowable() const\n{\n\treturn false;\n}\n\n// Path: src/main/cpp/logger.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n...\n", "needle_token_start": 2241, "needle_token_end": 2330, "code_context_ntokens": 15179, "output": ["```cpp\n// Path: src/main/cpp/telnetappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/net/telnetappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/threadutility.h>\n#include <log4cxx/private/appenderskeleton_priv.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::net;\n\nIMPLEMENT_LOG4CXX_OBJECT(TelnetAppender)\n\nstruct TelnetAppender::TelnetAppenderPriv : public AppenderSkeletonPrivate\n{\n\t\nTelnetAppenderPriv( int port, int maxConnections ) : AppenderSkeletonPrivate(),\n\t\tport(port),\n\t\tconnections(maxConnections),\n\t\tencoding(LOG4CXX_STR(\"UTF-8\")),\n\t\tencoder(CharsetEncoder::getUTF8Encoder()),\n\t\tsh(),\n\t\tactiveConnections(0) {}\n\n\tint port;\n\tConnectionList connections;\n\tLogString encoding;\n\tLOG4CXX_NS::helpers::CharsetEncoderPtr encoder;\n\tstd::unique_ptr<helpers::ServerSocket> serverSocket;\n\tstd::thread sh;\n\tsize_t activeConnections;\n};\n\n#define _priv static_cast<TelnetAppenderPriv*>(m_priv.get())\n\n/** The default telnet server port */\nconst int TelnetAppender::DEFAULT_PORT = 23;\n\n/** The maximum number of concurrent connections */\nconst int TelnetAppender::MAX_CONNECTIONS = 20;\n\nTelnetAppender::TelnetAppender()\n\t: AppenderSkeleton (std::make_unique<TelnetAppenderPriv>(DEFAULT_PORT, MAX_CONNECTIONS))\n{\n}\n\nTelnetAppender::~TelnetAppender()\n{\n\tfinalize();\n}\n\nvoid TelnetAppender::activateOptions(Pool& /* p */)\n{\n\tif (_priv->serverSocket == NULL)\n\t{\n\t\t_priv->serverSocket = ServerSocket::create(_priv->port);\n\t\t_priv->serverSocket->setSoTimeout(1000);\n\t}\n\n\t_priv->sh = ThreadUtility::instance()->createThread( LOG4CXX_STR(\"TelnetAppender\"), &TelnetAppender::acceptConnections, this );\n}\n\nvoid TelnetAppender::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"PORT\"), LOG4CXX_STR(\"port\")))\n\t{\n\t\tsetPort(OptionConverter::toInt(value, DEFAULT_PORT));\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"ENCODING\"), LOG4CXX_STR(\"encoding\")))\n\t{\n\t\tsetEncoding(value);\n\t}\n\telse\n\t{\n\t\tAppenderSkeleton::setOption(option, value);\n\t}\n}\n\nLogString TelnetAppender::getEncoding() const\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\treturn _priv->encoding;\n}\n\nvoid TelnetAppender::setEncoding(const LogString& value)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t_priv->encoder = CharsetEncoder::getEncoder(value);\n\t_priv->encoding = value;\n}\n\n\nvoid TelnetAppender::close()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\tif (_priv->closed)\n\t{\n\t\treturn;\n\t}\n\n\t_priv->closed = true;\n\n\tSocketPtr nullSocket;\n\n\tfor (auto& item : _priv->connections)\n\t{\n\t\tif (item)\n\t\t{\n\t\t\titem->close();\n\t\t\titem = nullSocket;\n\t\t}\n\t}\n\n\tif (_priv->serverSocket != NULL)\n\t{\n\t\ttry\n\t\t{\n\t\t\t_priv->serverSocket->close();\n\t\t}\n\t\tcatch (Exception&)\n\t\t{\n\t\t}\n\t}\n\n\tif ( _priv->sh.joinable() )\n\t{\n\t\t_priv->sh.join();\n\t}\n\n\t_priv->activeConnections = 0;\n}\n\n\nvoid TelnetAppender"]}
{"repo": "apache/logging-log4cxx", "name": "HierarchyPrivate", "language": "cpp", "path": "src/main/cpp/hierarchy.cpp", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To manage and maintain the state and configuration of a logging hierarchy, ensuring proper initialization and tracking of various status flags and settings related to logging.\n2. **Input**: No direct inputs as it initializes internal state when an instance of the containing class is created.\n3. **Output**: Does not produce a direct output but initializes and maintains internal state variables that affect the behavior of the logging system.\n4. **Procedure**: Initializes internal state with default values, including flags for configuration status and warnings, and sets a default logging level threshold. This setup is crucial for the correct functioning and configuration management of the logging system.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/rollingfileappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/rolling/rollingfileappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/rolling/rolloverdescription.h>\n#include <log4cxx/helpers/fileoutputstream.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/rolling/fixedwindowrollingpolicy.h>\n#include <log4cxx/rolling/timebasedrollingpolicy.h>\n#include <log4cxx/rolling/sizebasedtriggeringpolicy.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/private/fileappender_priv.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::rolling;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\nstruct RollingFileAppender::RollingFileAppenderPriv : public FileAppenderPriv\n{\n\tRollingFileAppenderPriv() :\n\t\tFileAppenderPriv(),\n\t\tfileLength(0) {}\n\n\t/**\n\t * Triggering policy.\n\t */\n\tTriggeringPolicyPtr triggeringPolicy;\n\n\t/**\n\t * Rolling policy.\n\t */\n\tRollingPolicyPtr rollingPolicy;\n\n\t/**\n\t * Length of current active log file.\n\t */\n\tsize_t fileLength;\n\n\t/**\n\t *  save the loggingevent\n\t */\n\tspi::LoggingEventPtr _event;\n};\n\n#define _priv static_cast<RollingFileAppenderPriv*>(m_priv.get())\n\nIMPLEMENT_LOG4CXX_OBJECT(RollingFileAppender)\n\n\n/**\n * Construct a new instance.\n */\nRollingFileAppender::RollingFileAppender() :\n\tFileAppender (std::make_unique<RollingFileAppenderPriv>())\n{\n}\n\nvoid RollingFileAppender::setOption(const LogString& option, const LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"MAXFILESIZE\"), LOG4CXX_STR(\"maxfilesize\"))\n\t\t|| StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"MAXIMUMFILESIZE\"), LOG4CXX_STR(\"maximumfilesize\")))\n\t{\n\t\tsetMaxFileSize(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"MAXBACKUPINDEX\"), LOG4CXX_STR(\"maxbackupindex\"))\n\t\t|| StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"MAXIMUMBACKUPINDEX\"), LOG4CXX_STR(\"maximumbackupindex\")))\n\t{\n\t\tsetMaxBackupIndex(StringHelper::toInt(value));\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"FILEDATEPATTERN\"), LOG4CXX_STR(\"filedatepattern\")))\n\t{\n\t\tsetDatePattern(value);\n\t}\n\telse\n\t{\n\t\tFileAppender::setOption(option, value);\n\t}\n}\n\nint RollingFileAppender::getMaxBackupIndex() const\n{\n\tint result = 1;\n\tif (auto fwrp = LOG4CXX_NS::cast<FixedWindowRollingPolicy>(_priv->rollingPolicy))\n\t\tresult = fwrp->getMaxIndex();\n\treturn result;\n}\n\nvoid RollingFileAppender::setMaxBackupIndex(int maxBackups)\n{\n\tauto fwrp = LOG4CXX_NS::cast<FixedWindowRollingPolicy>(_priv->rollingPolicy);\n\tif (!fwrp)\n\t{\n\t\tfwrp = std::make_shared<FixedWindowRollingPolicy>();\n\t\tfwrp->setFileNamePattern(getFile() + LOG4CXX_STR(\".%i\"));\n\t\t_priv->rollingPolicy = fwrp;\n\t}\n\tfwrp->setMaxIndex(maxBackups);\n}\n\nsize_t RollingFileAppender::getMaximumFileSize() const\n{\n\tsize_t result = 10 * 1024 * 1024;\n\tif (auto sbtp = LOG4CXX_NS::cast<SizeBasedTriggeringPolicy>(_priv->triggeringPolicy))\n\t\tresult = sbtp->getMaxFileSize();\n\treturn result;\n}\n\nvoid RollingFileAppender::setMaximumFileSize(size_t maxFileSize)\n{\n\tauto sbtp = LOG4CXX_NS::cast<SizeBasedTriggeringPolicy>(_priv->triggeringPolicy);\n\tif (!sbtp)\n\t{\n\t\tsbtp = std::make_shared<SizeBasedTriggeringPolicy>();\n\t\t_priv->triggeringPolicy = sbtp;\n\t}\n\tsbtp->setMaxFileSize(maxFileSize);\n}\n\nvoid RollingFileAppender::setMaxFileSize(const LogString& value)\n{\n\tsetMaximumFileSize(OptionConverter::toFileSize(value, long(getMaximumFileSize() + 1)));\n}\n\nLogString RollingFileAppender::makeFileNamePattern(const LogString& datePattern)\n{\n\tLogString result(getFile());\n\tbool inLiteral = false;\n\tbool inPattern = false;\n\n\tfor (size_t i = 0; i < datePattern.length(); i++)\n\t{\n\t\tif (datePattern[i] == 0x27 /* '\\'' */)\n\t\t{\n\t\t\tinLiteral = !inLiteral;\n\n\t\t\tif (inLiteral && inPattern)\n\t\t\t{\n\t\t\t\tresult.append(1, (logchar) 0x7D /* '}' */);\n\t\t\t\tinPattern = false;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (!inLiteral && !inPattern)\n\t\t\t{\n\t\t\t\tconst logchar dbrace[] = { 0x25, 0x64, 0x7B, 0 }; // \"%d{\"\n\t\t\t\tresult.append(dbrace);\n\t\t\t\tinPattern = true;\n\t\t\t}\n\n\t\t\tresult.append(1, datePattern[i]);\n\t\t}\n\t}\n\n\tif (inPattern)\n\t{\n\t\tresult.append(1, (logchar) 0x7D /* '}' */);\n\t}\n\treturn result;\n}\n\nvoid RollingFileAppender::setDatePattern(const LogString& newPattern)\n{\n\tauto tbrp = LOG4CXX_NS::cast<TimeBasedRollingPolicy>(_priv->rollingPolicy);\n\tif (!tbrp)\n\t{\n\t\ttbrp = std::make_shared<TimeBasedRollingPolicy>();\n\t\t_priv->rollingPolicy = tbrp;\n\t}\n\ttbrp->setFileNamePattern(makeFileNamePattern(newPattern));\n}\n\n/**\n * Prepare instance of use.\n */\nvoid RollingFileAppender::activateOptions(Pool& p)\n{\n\tif (!_priv->rollingPolicy)\n\t{\n\t\tLogLog::warn(LOG4CXX_STR(\"No rolling policy configured for the appender named [\")\n\t\t\t+ _priv->name + LOG4CXX_STR(\"].\"));\n\t\tauto fwrp = std::make_shared<FixedWindowRollingPolicy>();\n\t\tfwrp->setFileNamePattern(getFile() + LOG4CXX_STR(\".%i\"));\n\t\t_priv->rollingPolicy = fwrp;\n\t}\n\n\t//\n\t//  if no explicit triggering policy and rolling policy is both.\n\t//\n\tif (!_priv->triggeringPolicy)\n\t{\n\t\tTriggeringPolicyPtr trig = LOG4CXX_NS::cast<TriggeringPolicy>(_priv->rollingPolicy);\n\n\t\tif (trig != NULL)\n\t\t{\n\t\t\t_priv->triggeringPolicy = trig;\n\t\t}\n\t}\n\n\tif (!_priv->triggeringPolicy)\n\t{\n\t\tLogLog::warn(LOG4CXX_STR(\"No triggering policy configured for the appender named [\")\n\t\t\t+ _priv->name + LOG4CXX_STR(\"].\"));\n\t\t_priv->triggeringPolicy = std::make_shared<SizeBasedTriggeringPolicy>();\n\t}\n\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t_priv->triggeringPolicy->activateOptions(p);\n\t\t_priv->rollingPolicy->activateOptions(p);\n\n\t\ttry\n\t\t{\n\t\t\tRolloverDescriptionPtr rollover1 =\n\t\t\t\t_priv->rollingPolicy->initialize(getFile(), getAppend(), p);\n\n\t\t\tif (rollover1 != NULL)\n\t\t\t{\n\t\t\t\tActionPtr syncAction(rollover1->getSynchronous());\n\n\t\t\t\tif (syncAction != NULL)\n\t\t\t\t{\n\t\t\t\t\tsyncAction->execute(p);\n\t\t\t\t}\n\n\t\t\t\t_priv->fileName = rollover1->getActiveFileName();\n\t\t\t\t_priv->fileAppend = rollover1->getAppend();\n\n\t\t\t\t//\n\t\t\t\t//  async action not yet implemented\n\t\t\t\t//\n\t\t\t\tActionPtr asyncAction(rollover1->getAsynchronous());\n\n\t\t\t\tif (asyncAction != NULL)\n\t\t\t\t{\n\t\t\t\t\tasyncAction->execute(p);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tFile activeFile;\n\t\t\tactiveFile.setPath(getFile());\n\n\t\t\tif (getAppend())\n\t\t\t{\n\t\t\t\t_priv->fileLength = activeFile.length(p);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t_priv->fileLength = 0;\n\t\t\t}\n\n\t\t\tFileAppender::activateOptionsInternal(p);\n\t\t}\n\t\tcatch (std::exception&)\n\t\t{\n\t\t\tLogLog::warn(\n\t\t\t\tLogString(LOG4CXX_STR(\"Exception will initializing RollingFileAppender named \"))\n\t\t\t\t+ getName());\n\t\t}\n\t}\n}\n\n/**\n   Implements the usual roll over behaviour.\n\n   <p>If <code>MaxBackupIndex</code> is positive, then files\n   {<code>File.1</code>, ..., <code>File.MaxBackupIndex -1</code>}\n   are renamed to {<code>File.2</code>, ...,\n   <code>File.MaxBackupIndex</code>}. Moreover, <code>File</code> is\n   renamed <code>File.1</code> and closed. A new <code>File</code> is\n   created to receive further log output.\n\n   <p>If <code>MaxBackupIndex</code> is equal to zero, then the\n   <code>File</code> is truncated with no backup files created.\n\n * @return true if rollover performed.\n */\nbool RollingFileAppender::rollover(Pool& p)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\treturn rolloverInternal(p);\n}\n\nbool RollingFileAppender::rolloverInternal(Pool& p)\n{\n\t//\n\t//   can't roll without a policy\n\t//\n\tif (_priv->rollingPolicy != NULL)\n\t{\n\n\t\t{\n\t\t\t\ttry\n\t\t\t\t{\n\t\t\t\t\tRolloverDescriptionPtr rollover1(_priv->rollingPolicy->rollover(this->getFile(), this->getAppend(), p));\n\n\t\t\t\t\tif (rollover1 != NULL)\n\t\t\t\t\t{\n\t\t\t\t\t\tif (rollover1->getActiveFileName() == getFile())\n\t\t\t\t\t\t{\n...\n// Path: src/main/cpp/hierarchy.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <log4cxx/hierarchy.h>\n#include <log4cxx/defaultloggerfactory.h>\n#include <log4cxx/logger.h>\n#include <log4cxx/spi/hierarchyeventlistener.h>\n#include <log4cxx/level.h>\n#include <algorithm>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/appender.h>\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/stringhelper.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/spi/rootlogger.h>\n#include <mutex>\n#include \"assert.h\"\n\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\n\ntypedef std::map<LogString, LoggerPtr> LoggerMap;\ntypedef std::map<LogString, ProvisionNode> ProvisionNodeMap;\n\nstruct Hierarchy::HierarchyPrivate\n{\n\t\nHierarchyPrivate()\n\t\t: configured(false)\n\t\t, emittedNoAppenderWarning(false)\n\t\t, emittedNoResourceBundleWarning(false)\n\t\t, thresholdInt(Level::ALL_INT)\n\t{\n\t}\n\n\thelpers::Pool pool;\n\tmutable std::recursive_mutex mutex;\n\tmutable std::mutex configuredMutex;\n\tbool configured;\n\tbool emittedNoAppenderWarning;\n\tbool emittedNoResourceBundleWarning;\n\tint thresholdInt;\n\n\tspi::HierarchyEventListenerList listeners;\n\tLoggerPtr root;\n\tLevelPtr threshold;\n\tLoggerMap loggers;\n\tProvisionNodeMap provisionNodes;\n\n\tstd::vector<AppenderPtr> allAppenders;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(Hierarchy)\n\nHierarchy::Hierarchy() :\n\tm_priv(std::make_unique<HierarchyPrivate>())\n{\n}\n\nHierarchy::~Hierarchy()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\tfor (auto& item : m_priv->loggers)\n\t{\n\t\tif (auto& pLogger = item.second)\n\t\t{\n\t\t\tpLogger->removeHierarchy();\n\t\t\tpLogger->removeAllAppenders();\n\t\t}\n\t}\n\tif (m_priv->root)\n\t{\n\t\tm_priv->root->removeHierarchy();\n\t\tm_priv->root->removeAllAppenders();\n\t}\n}\n\nvoid Hierarchy::addHierarchyEventListener(const spi::HierarchyEventListenerPtr& listener)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n\tif (std::find(m_priv->listeners.begin(), m_priv->listeners.end(), listener) != m_priv->listeners.end())\n\t{\n\t\tLogLog::warn(LOG4CXX_STR(\"Ignoring attempt to add an existent listener.\"));\n\t}\n\telse\n\t{\n\t\tm_priv->listeners.push_back(listener);\n\t}\n}\n\nvoid Hierarchy::removeHierarchyEventListener(const spi::HierarchyEventListenerPtr& listener)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n    auto found = std::find(m_priv->listeners.begin(), m_priv->listeners.end(), listener);\n    if(found != m_priv->listeners.end()){\n        m_priv->listeners.erase(found);\n    }\n}\n\nvoid Hierarchy::clear()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\tm_priv->loggers.clear();\n}\n\nvoid Hierarchy::emitNoAppenderWarning(const Logger* logger)\n{\n\tbool emitWarning = false;\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\t\temitWarning = !m_priv->emittedNoAppenderWarning;\n\t\tm_priv->emittedNoAppenderWarning = true;\n\t}\n\n\t// No appender in hierarchy, warn user only once.\n\tif (emitWarning)\n\t{\n\t\tLogLog::warn(((LogString) LOG4CXX_STR(\"No appender could be found for logger (\"))\n\t\t\t+ logger->getName() + LOG4CXX_STR(\").\"));\n\t\tLogLog::warn(LOG4CXX_STR(\"Please initialize the log4cxx system properly.\"));\n\t}\n}\n\n\nLoggerPtr Hierarchy::exists(const LogString& name)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n\tLoggerPtr logger;\n\tLoggerMap::iterator it = m_priv->loggers.find(name);\n\n\tif (it != m_priv->loggers.end())\n\t{\n\t\tlogger = it->second;\n\t}\n\n\n\treturn logger;\n}\n\nvoid Hierarchy::setThreshold(const LevelPtr& l)\n{\n\tif (l != 0)\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\t\tsetThresholdInternal(l);\n\t}\n}\n\nvoid Hierarchy::setThreshold(const LogString& levelStr)\n{\n\tLevelPtr l(Level::toLevelLS(levelStr, 0));\n\n\tif (l != 0)\n\t{\n\t\tsetThreshold(l);\n\t}\n\telse\n\t{\n\t\tLogLog::warn(((LogString) LOG4CXX_STR(\"No level could be found named \\\"\"))\n\t\t\t+ levelStr + LOG4CXX_STR(\"\\\".\"));\n\t}\n}\n\nvoid Hierarchy::setThresholdInternal(const LevelPtr& l)\n{\n\tm_priv->thresholdInt = l->toInt();\n\tm_priv->threshold = l;\n\n\tif (m_priv->thresholdInt != Level::ALL_INT)\n\t{\n\t\tm_priv->configured = true;\n\t}\n}\n\nvoid Hierarchy::fireAddAppenderEvent(const Logger* logger, const Appender* appender)\n{\n\tsetConfigured(true);\n\tHierarchyEventListenerList clonedList;\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\t\tclonedList = m_priv->listeners;\n\t}\n\n\tfor (auto& item : clonedList)\n\t\titem->addAppenderEvent(logger, appender);\n}\n\nvoid Hierarchy::fireRemoveAppenderEvent(const Logger* logger, const Appender* appender)\n\n{\n\tHierarchyEventListenerList clonedList;\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\t\tclonedList = m_priv->listeners;\n\t}\n\tfor (auto& item : clonedList)\n\t\titem->removeAppenderEvent(logger, appender);\n}\n\nLevelPtr Hierarchy::getThreshold() const\n{\n\treturn m_priv->threshold ? m_priv->threshold : Level::getAll();\n}\n\nLoggerPtr Hierarchy::getLogger(const LogString& name)\n{\n\tstatic WideLife<spi::LoggerFactoryPtr> defaultFactory = std::make_shared<DefaultLoggerFactory>();\n\treturn getLogger(name, defaultFactory);\n}\n\nLoggerPtr Hierarchy::getLogger(const LogString& name,\n\tconst spi::LoggerFactoryPtr& factory)\n{\n\tauto root = getRootLogger();\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n\tLoggerMap::iterator it = m_priv->loggers.find(name);\n\tLoggerPtr result;\n\n\tif (it != m_priv->loggers.end())\n\t{\n\t\tresult = it->second;\n\t}\n\tif (!result && factory)\n\t{\n\t\tLoggerPtr logger(factory->makeNewLoggerInstance(m_priv->pool, name));\n\t\tlogger->setHierarchy(this);\n\t\tm_priv->loggers.insert(LoggerMap::value_type(name, logger));\n\n\t\tProvisionNodeMap::iterator it2 = m_priv->provisionNodes.find(name);\n\n\t\tif (it2 != m_priv->provisionNodes.end())\n\t\t{\n\t\t\tupdateChildren(it2->second, logger);\n\t\t\tm_priv->provisionNodes.erase(it2);\n\t\t}\n\n\t\tupdateParents(logger, root);\n\t\tresult = logger;\n\t}\n\treturn result;\n\n}\n\nLoggerList Hierarchy::getCurrentLoggers() const\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n\tLoggerList v;\n\tfor (auto& item : m_priv->loggers)\n\t{\n\t\tif (auto pLogger = item.second)\n\t\t\tv.push_back(pLogger);\n\t}\n\treturn v;\n}\n\nLoggerPtr Hierarchy::getRootLogger() const\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\tif (!m_priv->root)\n\t{\n\t\tm_priv->root = std::make_shared<RootLogger>(m_priv->pool, Level::getDebug());\n\t\tm_priv->root->setHierarchy(const_cast<Hierarchy*>(this));\n\t}\n\n\treturn m_priv->root;\n}\n\nbool Hierarchy::isDisabled(int level) const\n{\n\treturn m_priv->thresholdInt > level;\n}\n\nvoid Hierarchy::ensureIsConfigured(std::function<void()> configurator)\n{\n\tstd::unique_lock<std::mutex> lock(m_priv->configuredMutex);\n\tif (!m_priv->configured)\n\t{\n\t\tconfigurator();\n\t\tm_priv->configured = true;\n\t}\n}\n\nvoid Hierarchy::resetConfiguration()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n\tif (m_priv->root)\n\t{\n\t\tm_priv->root->setLevel(Level::getDebug());\n\t\tm_priv->root->setResourceBundle(0);\n\t}\n\tsetThresholdInternal(Level::getAll());\n\n\tshutdownInternal();\n\n\tfor (auto& item : m_priv->loggers)\n\t{\n\t\tif (auto pLogger = item.second)\n\t\t{\n\t\t\tpLogger->setLevel(0);\n\t\t\tpLogger->setAdditivity(true);\n\t\t\tpLogger->setResourceBundle(0);\n\t\t}\n\t}\n}\n\nvoid Hierarchy::shutdown()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n\tshutdownInternal();\n}\n\nvoid Hierarchy::shutdownInternal()\n{\n\tm_priv->configured = false;\n\n\t// begin by closing nested appenders\n\tif (m_priv->root)\n\t\tm_priv->root->closeNestedAppenders();\n\n\tfor (auto& item : m_priv->loggers)\n\t{\n\t\tif (auto pLogger = item.second)\n\t\t\tpLogger->closeNestedAppenders();\n\t}\n\n\t// then, remove all appenders\n\tif (m_priv->root)\n\t\tm_priv->root->removeAllAppenders();\n\n\tfor (auto& item : m_priv->loggers)\n\t{\n\t\tif (auto pLogger = item.second)\n\t\t\tpLogger->removeAllAppenders();\n\t}\n}\n\nvoid Hierarchy::updateParents(const LoggerPtr& logger, const LoggerPtr& root)\n{\n\tconst LogString name(logger->getName());\n\tsize_t length = name.size();\n\tbool parentFound = false;\n\n\n\t// if name = \"w.x.y.z\", loop through \"w.x.y\", \"w.x\" and \"w\", but not \"w.x.y.z\"\n\tfor (size_t i = name.find_last_of(0x2E /* '.' */, length - 1);\n\t\t(i != LogString::npos) && (i != 0);\n\t\ti = name.find_last_of(0x2E /* '.' */, i - 1))\n\t{\n\t\tLogString substr = name.substr(0, i);\n\n\t\tLoggerMap::iterator it = m_priv->loggers.find(substr);\n\n\t\tif (it != m_priv->loggers.end())\n\t\t{\n\t\t\tparentFound = true;\n\t\t\tlogger->setParent( it->second );\n\t\t\tbreak; // no need to update the ancestors of the closest ancestor\n\t\t}\n\t\telse\n\t\t{\n\t\t\tProvisionNodeMap::iterator it2 = m_priv->provisionNodes.find(substr);\n\n\t\t\tif (it2 != m_priv->provisionNodes.end())\n\t\t\t{\n\t\t\t\tit2->second.push_back(logger);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tProvisionNode node(1, logger);\n\t\t\t\tm_priv->provisionNodes.insert(\n\t\t\t\t\tProvisionNodeMap::value_type(substr, node));\n\t\t\t}\n\t\t}\n\t}\n\n\t// If we could not find any existing parents, then link with root.\n\tif (!parentFound)\n\t{\n\t\tlogger->setParent( root );\n\t}\n}\n\nvoid Hierarchy::updateChildren(ProvisionNode& pn, const LoggerPtr& logger)\n{\n\tfor (auto& l : pn)\n\t{\n\t\t// Unless this child already points to a correct (lower) parent,\n\t\t// make logger.parent point to l.parent and l.parent to logger.\n\t\tif (!StringHelper::startsWith(l->getParent()->getName(), logger->getName()))\n\t\t{\n\t\t\tlogger->setParent( l->getParent() );\n\t\t\tl->setParent( logger );\n\t\t}\n\t}\n    \n}\n\nvoid Hierarchy::updateChildren(const Logger* parent)\n{\n\tfor (auto& item : m_priv->loggers)\n\t{\n\t\tfor (auto l = item.second; l; l = l->getParent())\n\t\t{\n\t\t\tif (l->getParent().get() == parent)\n\t\t\t{\n\t\t\t\titem.second->updateThreshold();\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n}\n\nvoid Hierarchy::setConfigured(bool newValue)\n{\n\tstd::unique_lock<std::mutex> lock(m_priv->configuredMutex, std::try_to_lock);\n\tif (lock.owns_lock()) // Not being auto-configured?\n\t\tm_priv->configured = newValue;\n}\n\nbool Hierarchy::isConfigured()\n{\n\tstd::unique_lock<std::mutex> lock(m_priv->configuredMutex); // Blocks while auto-configuration is active\n\treturn m_priv->configured;\n}\n\nHierarchyPtr Hierarchy::create()\n{\n\tHierarchyPtr ret(new Hierarchy);\n\treturn ret;\n}\n\nvoid Hierarchy::clearAppenders()\n{\n\tm_priv->allAppenders.clear();\n}\n\nvoid Hierarchy::addAppender(AppenderPtr appender)\n{\n\tm_priv->allAppenders.push_back(appender);\n}\n\nbool Hierarchy::removeLogger(const LogString& name, bool ifNotUsed)\n{\n\tauto parentRefCount = [this](const LoggerPtr& child) -> int\n\t{\n\t\tint result = 0;\n\t\tfor (auto& node : m_priv->provisionNodes)\n\t\t{\n\t\t\tif (node.second.end() != std::find(node.second.begin(), node.second.end(), child))\n\t\t\t\t++result;\n\t\t}\n\t\treturn result;\n\t};\n\tbool result = false;\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\tauto it = m_priv->loggers.find(name);\n\tif (it == m_priv->loggers.end())\n\t\t;\n\telse if (ifNotUsed && 1 + parentRefCount(it->second) < it->second.use_count())\n\t\t;\n\telse\n\t{\n\t\tfor (auto& node : m_priv->provisionNodes)\n\t\t{\n\t\t\tfor (size_t i = node.second.size(); 0 < i; )\n\t\t\t{\n\t\t\t\tif (node.second[--i] == it->second)\n\t\t\t\t\tnode.second.erase(node.second.begin() + i);\n\t\t\t}\n\t\t}\n\t\tm_priv->loggers.erase(it);\n\t\tresult = true;\n\t}\n\treturn result;\n}\n\n// Path: src/main/cpp/fileoutputstream.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/fileoutputstream.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <apr_file_io.h>\n#include <log4cxx/helpers/transcoder.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct FileOutputStream::FileOutputStreamPrivate\n{\n\tFileOutputStreamPrivate() : fileptr(nullptr) {}\n\n\tPool pool;\n\tapr_file_t* fileptr;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(FileOutputStream)\n\nFileOutputStream::FileOutputStream(const LogString& filename,\n\tbool append) : m_priv(std::make_unique<FileOutputStreamPrivate>())\n{\n\tm_priv->fileptr = open(filename, append, m_priv->pool);\n}\n\nFileOutputStream::FileOutputStream(const logchar* filename,\n\tbool append) : m_priv(std::make_unique<FileOutputStreamPrivate>())\n{\n\tm_priv->fileptr = open(filename, append, m_priv->pool);\n}\n\napr_file_t* FileOutputStream::open(const LogString& filename,\n\tbool append, Pool& pool)\n{\n\tapr_fileperms_t perm = APR_OS_DEFAULT;\n\tapr_int32_t flags = APR_WRITE | APR_CREATE;\n\n\tif (append)\n\t{\n\t\tflags |= APR_APPEND;\n\t}\n\telse\n\t{\n\t\tflags |= APR_TRUNCATE;\n\t}\n\n\tFile fn;\n\tfn.setPath(filename);\n\tapr_file_t* fileptr = 0;\n\tapr_status_t stat = fn.open(&fileptr, flags, perm, pool);\n\n\tif (stat != APR_SUCCESS)\n\t{\n\t\tthrow IOException(stat);\n\t}\n\n\treturn fileptr;\n}\n\nFileOutputStream::~FileOutputStream()\n{\n\tif (m_priv->fileptr != NULL && !APRInitializer::isDestructed)\n\t{\n\t\tapr_file_close(m_priv->fileptr);\n\t}\n}\n\nvoid FileOutputStream::close(Pool& /* p */)\n{\n\tif (m_priv->fileptr != NULL)\n\t{\n\t\tapr_status_t stat = apr_file_close(m_priv->fileptr);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\tm_priv->fileptr = NULL;\n\t}\n}\n\nvoid FileOutputStream::flush(Pool& /* p */)\n{\n}\n\nvoid FileOutputStream::write(ByteBuffer& buf, Pool& /* p */ )\n{\n\tif (m_priv->fileptr == NULL)\n\t{\n\t\tthrow IOException(-1);\n\t}\n\n\tsize_t nbytes = buf.remaining();\n\tsize_t pos = buf.position();\n\tconst char* data = buf.data();\n\n\twhile (nbytes > 0)\n\t{\n\t\tapr_status_t stat = apr_file_write(\n\t\t\t\tm_priv->fileptr, data + pos, &nbytes);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\tpos += nbytes;\n\t\tbuf.position(pos);\n\t\tnbytes = buf.remaining();\n\t}\n}\n\napr_file_t* FileOutputStream::getFilePtr() const{\n\treturn m_priv->fileptr;\n}\n\n\n// Path: src/main/cpp/atexitregistry.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/private/atexitregistry.h>\n#include <mutex>\n#include <map>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nnamespace\n{\n\tstruct AtExitRegistryImpl : public AtExitRegistry\n\t{\n\t\t~AtExitRegistryImpl()\n\t\t{\n\t\t\tstd::lock_guard<std::recursive_mutex> lock(mutex);\n\t\t\twhile(!actions.empty())\n\t\t\t{\n\t\t\t\tstd::function<void()> action = std::move(actions.begin()->second);\n\t\t\t\tactions.erase(actions.begin());\n\t\t\t\taction();\n\t\t\t}\n\t\t}\n\n\t\tvoid add(void* key, std::function<void()> action)\n\t\t{\n\t\t\tstd::lock_guard<std::recursive_mutex> lock(mutex);\n\t\t\tactions.emplace(key, std::move(action));\n\t\t}\n\n\t\tvoid del(void* key)\n\t\t{\n\t\t\tstd::lock_guard<std::recursive_mutex> lock(mutex);\n\t\t\tactions.erase(key);\n\t\t}\n\n\tprivate:\n\t\tstd::recursive_mutex mutex;\n\t\tstd::map<void*, std::function<void()>> actions;\n\t} s_instance;\n}\n\nAtExitRegistry& AtExitRegistry::instance()\n{\n\treturn s_instance;\n}\n\nvoid AtExitRegistry::add(void* key, std::function<void()> action)\n{\n\treturn s_instance.add(key, std::move(action));\n}\n\nvoid AtExitRegistry::del(void* key)\n{\n\treturn s_instance.del(key);\n}\n\n\n// Path: src/main/cpp/cacheddateformat.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#define __STDC_CONSTANT_MACROS\n#define NOMINMAX /* tell wnidows not to define min/max macros */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/cacheddateformat.h>\n#include <log4cxx/helpers/pool.h>\n#include <limits>\n#include <log4cxx/helpers/exception.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::pattern;\n\nstruct CachedDateFormat::CachedDateFormatPriv\n{\n\tCachedDateFormatPriv(DateFormatPtr dateFormat, int expiration1) :\n\t\tformatter(dateFormat),\n\t\tmillisecondStart(0),\n\t\tslotBegin(std::numeric_limits<log4cxx_time_t>::min()),\n\t\tcache(50, 0x20),\n\t\texpiration(expiration1),\n\t\tpreviousTime(std::numeric_limits<log4cxx_time_t>::min())\n\t{}\n\n\t/**\n\t *   Wrapped formatter.\n\t */\n\tLOG4CXX_NS::helpers::DateFormatPtr formatter;\n\n\t/**\n\t *  Index of initial digit of millisecond pattern or\n\t *   UNRECOGNIZED_MILLISECONDS or NO_MILLISECONDS.\n\t */\n\tmutable int millisecondStart;\n\n\t/**\n\t *  Integral second preceding the previous convered Date.\n\t */\n\tmutable log4cxx_time_t slotBegin;\n\n\n\t/**\n\t *  Cache of previous conversion.\n\t */\n\tmutable LogString cache;\n\n\n\t/**\n\t *  Maximum validity period for the cache.\n\t *  Typically 1, use cache for duplicate requests only, or\n\t *  1000000, use cache for requests within the same integral second.\n\t */\n\tconst int expiration;\n\n\t/**\n\t *  Date requested in previous conversion.\n\t */\n\tmutable log4cxx_time_t previousTime;\n};\n\n\n/**\n*  Supported digit set.  If the wrapped DateFormat uses\n*  a different unit set, the millisecond pattern\n*  will not be recognized and duplicate requests\n*  will use the cache.\n*/\nconst logchar CachedDateFormat::digits[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0 };\n\n\n/**\n * First magic number (in microseconds) used to detect\n * the millisecond position.\n */\nconst int CachedDateFormat::magic1 = 654000;\n\n\n/**\n *  Expected representation of first magic number in milliseconds.\n */\nconst logchar CachedDateFormat::magicString1[] = { 0x36, 0x35, 0x34, 0 };\n\n\n/**\n * Second magic number (in microseconds) used to detect\n * the millisecond position.\n */\nconst int CachedDateFormat::magic2 = 987000;\n\n\n/**\n *  Expected representation of second magic number in milliseconds.\n */\nconst logchar CachedDateFormat::magicString2[] = { 0x39, 0x38, 0x37, 0};\n\n\n/**\n *  Expected representation of 0 milliseconds.\n */\nconst logchar CachedDateFormat::zeroString[] = { 0x30, 0x30, 0x30, 0 };\n\n/**\n *  Creates a new CachedDateFormat object.\n *  @param dateFormat Date format, may not be null.\n *  @param expiration maximum cached range in milliseconds.\n *    If the dateFormat is known to be incompatible with the\n *      caching algorithm, use a value of 0 to totally disable\n *      caching or 1 to only use cache for duplicate requests.\n */\nCachedDateFormat::CachedDateFormat(const DateFormatPtr& dateFormat,\n\tint expiration1) :\n\tm_priv(std::make_unique<CachedDateFormatPriv>(dateFormat, expiration1))\n{\n\tif (dateFormat == NULL)\n\t{\n\t\tthrow IllegalArgumentException(LOG4CXX_STR(\"dateFormat cannot be null\"));\n\t}\n\n\tif (expiration1 < 0)\n\t{\n\t\tthrow IllegalArgumentException(LOG4CXX_STR(\"expiration must be non-negative\"));\n\t}\n}\n\nCachedDateFormat::~CachedDateFormat() {}\n\n\n/**\n * Finds start of millisecond field in formatted time.\n * @param time long time, must be integral number of seconds\n * @param formatted String corresponding formatted string\n * @param formatter DateFormat date format\n * @return int position in string of first digit of milliseconds,\n *    -1 indicates no millisecond field, -2 indicates unrecognized\n *    field (likely RelativeTimeDateFormat)\n */\nint CachedDateFormat::findMillisecondStart(\n\tlog4cxx_time_t time, const LogString& formatted,\n\tconst DateFormatPtr& formatter,\n\tPool& pool)\n{\n\n\tlog4cxx_time_t slotBegin = (time / 1000000) * 1000000;\n\n\tif (slotBegin > time)\n\t{\n\t\tslotBegin -= 1000000;\n\t}\n\n\tint millis = (int) (time - slotBegin) / 1000;\n\n\t// the magic numbers are in microseconds\n\tint magic = magic1;\n\tLogString magicString(magicString1);\n\n\tif (millis == magic1 / 1000)\n\t{\n\t\tmagic = magic2;\n\t\tmagicString = magicString2;\n\t}\n\n\tLogString plusMagic;\n\tformatter->format(plusMagic, slotBegin + magic, pool);\n\n\t/**\n\t *   If the string lengths differ then\n\t *      we can't use the cache except for duplicate requests.\n\t */\n\tif (plusMagic.length() != formatted.length())\n\t{\n\t\treturn UNRECOGNIZED_MILLISECONDS;\n\t}\n\telse\n\t{\n\t\t// find first difference between values\n\t\tfor (LogString::size_type i = 0; i < formatted.length(); i++)\n\t\t{\n\t\t\tif (formatted[i] != plusMagic[i])\n\t\t\t{\n\t\t\t\t//\n\t\t\t\t//   determine the expected digits for the base time\n\t\t\t\tconst logchar abc[] = { 0x41, 0x42, 0x43, 0 };\n\t\t\t\tLogString formattedMillis(abc);\n\t\t\t\tmillisecondFormat(millis, formattedMillis, 0);\n\n\t\t\t\tLogString plusZero;\n\t\t\t\tformatter->format(plusZero, slotBegin, pool);\n\n\t\t\t\t// Test if the next 1..3 characters match the magic string, main problem is that magic\n\t\t\t\t// available millis in formatted can overlap. Therefore the current i is not always the\n\t\t\t\t// index of the first millis char, but may be already within the millis. Besides that\n\t\t\t\t// the millis can occur everywhere in formatted. See LOGCXX-420 and following.\n\t\t\t\tsize_t  magicLength     = magicString.length();\n\t\t\t\tsize_t  overlapping     = magicString.find(plusMagic[i]);\n\t\t\t\tint     possibleRetVal  = int(i - overlapping);\n\n\t\t\t\tif (plusZero.length() == formatted.length()\n\t\t\t\t\t&& regionMatches(magicString,       0, plusMagic,   possibleRetVal, magicLength)\n\t\t\t\t\t&& regionMatches(formattedMillis,   0, formatted,   possibleRetVal, magicLength)\n\t\t\t\t\t&& regionMatches(zeroString,        0, plusZero,    possibleRetVal, magicLength)\n\t\t\t\t\t// The following will and should fail for patterns with more than one SSS because\n\t\t\t\t\t// we only seem to be able to change one SSS in e.g. format and need to reformat the\n\t\t\t\t\t// whole string in other cases.\n\t\t\t\t\t&& (formatted.length() == possibleRetVal + magicLength\n\t\t\t\t\t\t|| plusZero.compare(possibleRetVal + magicLength,\n\t\t\t\t\t\t\tLogString::npos, plusMagic, possibleRetVal + magicLength, LogString::npos) == 0))\n\t\t\t\t{\n\t\t\t\t\treturn possibleRetVal;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\treturn UNRECOGNIZED_MILLISECONDS;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn NO_MILLISECONDS;\n}\n\n\n/**\n * Formats a millisecond count into a date/time string.\n *\n *  @param now Number of milliseconds after midnight 1 Jan 1970 GMT.\n *  @param sbuf the string buffer to write to\n */\nvoid CachedDateFormat::format(LogString& buf, log4cxx_time_t now, Pool& p) const\n{\n\n\t//\n\t// If the current requested time is identical to the previously\n\t//     requested time, then append the cache contents.\n\t//\n\tif (now == m_priv->previousTime)\n\t{\n\t\tbuf.append(m_priv->cache);\n\t\treturn;\n\t}\n\n\t//\n\t//   If millisecond pattern was not unrecognized\n\t//     (that is if it was found or milliseconds did not appear)\n\t//\n\tif (m_priv->millisecondStart != UNRECOGNIZED_MILLISECONDS)\n\t{\n\t\t//    Check if the cache is still valid.\n\t\t//    If the requested time is within the same integral second\n\t\t//       as the last request and a shorter expiration was not requested.\n\t\tif (now < m_priv->slotBegin + m_priv->expiration\n\t\t\t&& now >= m_priv->slotBegin\n\t\t\t&& now < m_priv->slotBegin + 1000000L)\n\t\t{\n\t\t\t//\n\t\t\t//    if there was a millisecond field then update it\n\t\t\t//\n\t\t\tif (m_priv->millisecondStart >= 0)\n\t\t\t{\n\t\t\t\tmillisecondFormat((int) ((now - m_priv->slotBegin) / 1000), m_priv->cache, m_priv->millisecondStart);\n\t\t\t}\n\n\t\t\t//\n\t\t\t//   update the previously requested time\n\t\t\t//      (the slot begin should be unchanged)\n\t\t\tm_priv->previousTime = now;\n\t\t\tbuf.append(m_priv->cache);\n\n\t\t\treturn;\n\t\t}\n\t}\n\n\t//\n\t//  could not use previous value.\n\t//    Call underlying formatter to format date.\n\tm_priv->cache.erase(m_priv->cache.begin(), m_priv->cache.end());\n\tm_priv->formatter->format(m_priv->cache, now, p);\n\tbuf.append(m_priv->cache);\n\tm_priv->previousTime = now;\n\tm_priv->slotBegin = (m_priv->previousTime / 1000000) * 1000000;\n\n\tif (m_priv->slotBegin > m_priv->previousTime)\n\t{\n\t\tm_priv->slotBegin -= 1000000;\n\t}\n\n\t//\n\t//    if the milliseconds field was previous found\n\t//       then reevaluate in case it moved.\n\t//\n\tif (m_priv->millisecondStart >= 0)\n\t{\n\t\tm_priv->millisecondStart = findMillisecondStart(now, m_priv->cache, m_priv->formatter, p);\n\t}\n}\n\n/**\n *   Formats a count of milliseconds (0-999) into a numeric representation.\n *   @param millis Millisecond count between 0 and 999.\n *   @buf String buffer, may not be null.\n *   @offset Starting position in buffer, the length of the\n *       buffer must be at least offset + 3.\n */\nvoid CachedDateFormat::millisecondFormat(int millis,\n\tLogString& buf,\n\tint offset)\n{\n\tbuf[offset] = digits[millis / 100];\n\tbuf[offset + 1] = digits[(millis / 10) % 10];\n\tbuf[offset + 2] = digits[millis  % 10];\n}\n\n/**\n * Set timezone.\n *\n * @remarks Setting the timezone using getCalendar().setTimeZone()\n * will likely cause caching to misbehave.\n * @param timeZone TimeZone new timezone\n */\nvoid CachedDateFormat::setTimeZone(const TimeZonePtr& timeZone)\n{\n\tm_priv->formatter->setTimeZone(timeZone);\n\tm_priv->previousTime = std::numeric_limits<log4cxx_time_t>::min();\n\tm_priv->slotBegin = std::numeric_limits<log4cxx_time_t>::min();\n}\n\n\n\nvoid CachedDateFormat::numberFormat(LogString& s, int n, Pool& p) const\n{\n\tm_priv->formatter->numberFormat(s, n, p);\n}\n\n\n/**\n * Gets maximum cache validity for the specified SimpleDateTime\n *    conversion pattern.\n *  @param pattern conversion pattern, may not be null.\n *  @returns Duration in microseconds from an integral second\n *      that the cache will return consistent results.\n */\nint CachedDateFormat::getMaximumCacheValidity(const LogString& pattern)\n{\n\t//\n\t//   If there are more \"S\" in the pattern than just one \"SSS\" then\n\t//      (for example, \"HH:mm:ss,SSS SSS\"), then set the expiration to\n\t//      one millisecond which should only perform duplicate request caching.\n\t//\n\tconst logchar S = 0x53;\n\tconst logchar SSS[] = { 0x53, 0x53, 0x53, 0 };\n\tsize_t firstS = pattern.find(S);\n\tsize_t len = pattern.length();\n\n\t//\n\t//   if there are no S's or\n\t//      three that start with the first S and no fourth S in the string\n\t//\n\tif (firstS == LogString::npos ||\n\t\t(len >= firstS + 3 && pattern.compare(firstS, 3, SSS) == 0\n\t\t\t&& (len == firstS + 3 ||\n\t\t\t\tpattern.find(S, firstS + 3) == LogString::npos)))\n\t{\n\t\treturn 1000000;\n\t}\n\n\treturn 1000;\n}\n\n\n/**\n* Tests if two string regions are equal.\n* @param target target string.\n* @param toffset character position in target to start comparison.\n* @param other other string.\n* @param ooffset character position in other to start comparison.\n* @param len length of region.\n* @return true if regions are equal.\n*/\nbool CachedDateFormat::regionMatches(\n\tconst LogString& target,\n\tsize_t toffset,\n\tconst LogString& other,\n\tsize_t ooffset,\n\tsize_t len)\n{\n\treturn target.compare(toffset, len, other, ooffset, len) == 0;\n}\n\n\n// Path: src/main/cpp/loader.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/loader.h>\n#include <log4cxx/appender.h>\n#include <log4cxx/spi/filter.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <log4cxx/spi/loggerrepository.h>\n#include <log4cxx/helpers/object.h>\n#include <log4cxx/spi/errorhandler.h>\n#include <log4cxx/filter/denyallfilter.h>\n#include <log4cxx/spi/repositoryselector.h>\n#include <log4cxx/spi/appenderattachable.h>\n#include <log4cxx/helpers/xml.h>\n#include <log4cxx/spi/triggeringeventevaluator.h>\n#include <fstream>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/fileinputstream.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::filter;\n\nIMPLEMENT_LOG4CXX_OBJECT(Object)\nIMPLEMENT_LOG4CXX_OBJECT(OptionHandler)\nIMPLEMENT_LOG4CXX_OBJECT(ErrorHandler)\nIMPLEMENT_LOG4CXX_OBJECT(Appender)\nIMPLEMENT_LOG4CXX_OBJECT(Filter)\nIMPLEMENT_LOG4CXX_OBJECT(AppenderAttachable)\nIMPLEMENT_LOG4CXX_OBJECT(LoggerFactory)\nIMPLEMENT_LOG4CXX_OBJECT(LoggerRepository)\nIMPLEMENT_LOG4CXX_OBJECT(DenyAllFilter)\nIMPLEMENT_LOG4CXX_OBJECT(RepositorySelector)\nIMPLEMENT_LOG4CXX_OBJECT(XMLDOMNode)\nIMPLEMENT_LOG4CXX_OBJECT(XMLDOMDocument)\nIMPLEMENT_LOG4CXX_OBJECT(XMLDOMElement)\nIMPLEMENT_LOG4CXX_OBJECT(XMLDOMNodeList)\nIMPLEMENT_LOG4CXX_OBJECT(TriggeringEventEvaluator)\n\nconst Class& Loader::loadClass(const LogString& clazz)\n{\n\treturn Class::forName(clazz);\n}\n\n\nInputStreamPtr Loader::getResourceAsStream(const LogString& name)\n{\n\n\ttry\n\t{\n\t\treturn std::make_shared<FileInputStream>(name);\n\t}\n\tcatch (const IOException&)\n\t{\n\t}\n\n\treturn 0;\n}\n\n// Path: src/main/cpp/logmanager.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logmanager.h>\n#include <log4cxx/defaultconfigurator.h>\n#include <log4cxx/spi/defaultrepositoryselector.h>\n#include <log4cxx/hierarchy.h>\n#include <log4cxx/spi/rootlogger.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <stdexcept>\n#include <log4cxx/level.h>\n#include <log4cxx/spi/loggerrepository.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/loglog.h>\n\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/file.h>\n#include <log4cxx/helpers/transcoder.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n\nusing namespace LOG4CXX_NS;\n...\n", "needle_token_start": 3790, "needle_token_end": 3847, "code_context_ntokens": 15213, "output": ["```cpp\n// Path: src/main/cpp/hierarchy.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <log4cxx/hierarchy.h>\n#include <log4cxx/defaultloggerfactory.h>\n#include <log4cxx/logger.h>\n#include <log4cxx/spi/hierarchyeventlistener.h>\n#include <log4cxx/level.h>\n#include <algorithm>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/appender.h>\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/stringhelper.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/spi/rootlogger.h>\n#include <mutex>\n#include \"assert.h\"\n\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\n\ntypedef std::map<LogString, LoggerPtr> LoggerMap;\ntypedef std::map<LogString, ProvisionNode> ProvisionNodeMap;\n\nstruct Hierarchy::HierarchyPrivate\n{\n\t\nHierarchyPrivate()\n\t\t: configured(false)\n\t\t, emittedNoAppenderWarning(false)\n\t\t, emittedNoResourceBundleWarning(false)\n\t\t, thresholdInt(Level::ALL_INT)\n\t{\n\t}\n\n\thelpers::Pool pool;\n\tmutable std::recursive_mutex mutex;\n\tmutable std::mutex configuredMutex;\n\tbool configured;\n\tbool emittedNoAppenderWarning;\n\tbool emittedNoResourceBundleWarning;\n\tint thresholdInt;\n\n\tspi::HierarchyEventListenerList listeners;\n\tLoggerPtr root;\n\tLevelPtr threshold;\n\tLoggerMap loggers;\n\tProvisionNodeMap provisionNodes;\n\n\tstd::vector<AppenderPtr> allAppenders;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(Hierarchy)\n\nHierarchy::Hierarchy() :\n\tm_priv(std::make_unique<HierarchyPrivate>())\n{\n}\n\nHierarchy::~Hierarchy()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\tfor (auto& item : m_priv->loggers)\n\t{\n\t\tif (auto& pLogger = item.second)\n\t\t{\n\t\t\tpLogger->removeHierarchy();\n\t\t\tpLogger->removeAllAppenders();\n\t\t}\n\t}\n\tif (m_priv->root)\n\t{\n\t\tm_priv->root->removeHierarchy();\n\t\tm_priv->root->removeAllAppenders();\n\t}\n}\n\nvoid Hierarchy::addHierarchyEventListener(const spi::HierarchyEventListenerPtr& listener)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n\tif (std::find(m_priv->listeners.begin(), m_priv->listeners.end(), listener) != m_priv->listeners.end())\n\t{\n\t\tLogLog::warn(LOG4CXX_STR(\"Ignoring attempt to add an existent listener.\"));\n\t}\n\telse\n\t{\n\t\tm_priv->listeners.push_back(listener);\n\t}\n}\n\nvoid Hierarchy::removeHierarchyEventListener(const spi::HierarchyEventListenerPtr& listener)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\n    auto found = std::find(m_priv->listeners.begin(), m_priv->listeners.end(), listener);\n    if(found != m_priv->listeners.end()){\n        m_priv->listeners.erase(found);\n    }\n}\n\nvoid Hierarchy::clear()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\tm_priv->loggers.clear();\n}\n\nvoid Hierarchy::emitNoAppenderWarning(const Logger* logger)\n{\n\tbool emitWarning = false;\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(m_priv->mutex);\n\t\temitWarning = !m_priv->emittedNoAppenderWarning;\n\t\tm_priv->emittedNoAppenderWarning = true;\n\t}\n\n\t// No appender in hierarchy, warn user only once.\n\tif (emitWarning)\n\t{\n\t\tLogLog::warn(((LogString) LOG4CXX_STR(\"No appender could be found for logger (\"))\n\t\t\t+ logger->getName() + LOG4CXX_STR(\").\"));\n\t\tLogLog::warn(LOG4CXX_STR(\"Please initialize the log4cxx system properly.\"));\n\t}\n}\n\n\nLoggerPtr Hierarchy::exists(const LogString& name)\n{\n\tstd::lock_guard"]}
{"repo": "apache/logging-log4cxx", "name": "decodeUTF16", "language": "cpp", "path": "src/main/cpp/transcoder.cpp", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function decodes a sequence of UTF-16 encoded characters into a Unicode scalar value, handling surrogate pairs correctly.\n2. **Input**: The function takes a string and an iterator pointing to the current character in the string.\n3. **Output**: It returns an unsigned integer representing the Unicode scalar value of the character or characters pointed to by the iterator. If the character is part of a valid surrogate pair, it returns the combined scalar value; otherwise, it returns the value of the single character or an error code for unrecognized sequences.\n4. **Procedure**: \n   - First, the function checks if the current character is a high-surrogate (in the range 0xD800 to 0xDBFF). If not, it simply returns the character's value and advances the iterator unless the character is 0xFFFF.\n   - If the character is a high-surrogate, the function checks the next character to see if it is a low-surrogate (in the range 0xDC00 to 0xDFFF). If so, it calculates the scalar value from the surrogate pair, advances the iterator past both characters, and returns the scalar value.\n   - If the sequence does not form a valid surrogate pair, the function returns an error code (0xFFFF) without advancing the iterator.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/resourcebundle.cpp\n/*\n...\n// Path: src/main/cpp/outputstreamwriter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/outputstreamwriter.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/stringhelper.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(OutputStreamWriter)\n\nstruct OutputStreamWriter::OutputStreamWriterPrivate{\n\tOutputStreamWriterPrivate(OutputStreamPtr& out1) : out(out1), enc(CharsetEncoder::getDefaultEncoder()){}\n\n\tOutputStreamWriterPrivate(OutputStreamPtr& out1,\n\t\t\t\t\t\t\t  CharsetEncoderPtr& enc1)\n\t\t: out(out1), enc(enc1){}\n\n\tOutputStreamPtr out;\n\tCharsetEncoderPtr enc;\n};\n\nOutputStreamWriter::OutputStreamWriter(OutputStreamPtr& out1)\n\t: m_priv(std::make_unique<OutputStreamWriterPrivate>(out1))\n{\n\tif (out1 == 0)\n\t{\n\t\tthrow NullPointerException(LOG4CXX_STR(\"out parameter may not be null.\"));\n\t}\n}\n\nOutputStreamWriter::OutputStreamWriter(OutputStreamPtr& out1,\n\tCharsetEncoderPtr& enc1)\n\t: m_priv(std::make_unique<OutputStreamWriterPrivate>(out1, enc1))\n{\n\tif (out1 == 0)\n\t{\n\t\tthrow NullPointerException(LOG4CXX_STR(\"out parameter may not be null.\"));\n\t}\n\n\tif (enc1 == 0)\n\t{\n\t\tthrow NullPointerException(LOG4CXX_STR(\"enc parameter may not be null.\"));\n\t}\n}\n\nOutputStreamWriter::~OutputStreamWriter()\n{\n}\n\nvoid OutputStreamWriter::close(Pool& p)\n{\n\tm_priv->out->close(p);\n}\n\nvoid OutputStreamWriter::flush(Pool& p)\n{\n\tm_priv->out->flush(p);\n}\n\nvoid OutputStreamWriter::write(const LogString& str, Pool& p)\n{\n\tif (str.empty())\n\t\treturn;\n\tif (CharsetEncoder::isTriviallyCopyable(str, m_priv->enc))\n\t{\n\t\tByteBuffer buf((char*)str.data(), str.size() * sizeof (logchar));\n\t\tm_priv->out->write(buf, p);\n\t}\n\telse\n\t{\n\t\tenum { BUFSIZE = 1024 };\n\t\tchar stackData[BUFSIZE];\n\t\tchar* rawbuf = stackData;\n\t\tsize_t bufSize = BUFSIZE;\n#ifdef LOG4CXX_MULTI_PROCESS\n\t\tstd::vector<char> heapData;\n\t\t// Ensure the logging event is a single write system call to keep events from each process separate\n\t\tif (bufSize < str.length() * 2)\n\t\t{\n\t\t\theapData.resize(bufSize = str.length() * 2);\n\t\t\trawbuf = heapData.data();\n\t\t}\n#endif\n\t\tByteBuffer buf(rawbuf, bufSize);\n\t\tm_priv->enc->reset();\n\t\tLogString::const_iterator iter = str.begin();\n\n\t\twhile (iter != str.end())\n\t\t{\n\t\t\tCharsetEncoder::encode(m_priv->enc, str, iter, buf);\n\t\t\tbuf.flip();\n\t\t\tm_priv->out->write(buf, p);\n\t\t\tbuf.clear();\n\t\t}\n\n\t\tCharsetEncoder::encode(m_priv->enc, str, iter, buf);\n\t\tm_priv->enc->flush(buf);\n\t\tbuf.flip();\n\t\tm_priv->out->write(buf, p);\n\t}\n}\n\nOutputStreamPtr OutputStreamWriter::getOutputStreamPtr() const\n{\n\treturn m_priv->out;\n}\n\n\n// Path: src/main/cpp/transcoder.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/pool.h>\n#include <stdlib.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/charsetdecoder.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/loglog.h>\n#include <vector>\n#include <cstring>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n\n#if LOG4CXX_CFSTRING_API\n\t#include <CoreFoundation/CFString.h>\n#endif\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\n\nvoid Transcoder::decodeUTF8(const std::string& src, LogString& dst)\n{\n\tstd::string::const_iterator iter = src.begin();\n\n\twhile (iter != src.end())\n\t{\n\t\tunsigned int sv = decode(src, iter);\n\n\t\tif (sv != 0xFFFF)\n\t\t{\n\t\t\tencode(sv, dst);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdst.append(1, LOSSCHAR);\n\t\t\titer++;\n\t\t}\n\t}\n}\n\nvoid Transcoder::encodeUTF8(const LogString& src, std::string& dst)\n{\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\tdst.append(src);\n#else\n\tLogString::const_iterator iter = src.begin();\n\n\twhile (iter != src.end())\n\t{\n\t\tunsigned int sv = decode(src, iter);\n\n\t\tif (sv != 0xFFFF)\n\t\t{\n\t\t\tencode(sv, dst);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdst.append(1, LOSSCHAR);\n\t\t\titer++;\n\t\t}\n\t}\n\n#endif\n}\n\nchar* Transcoder::encodeUTF8(const LogString& src, Pool& p)\n{\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\treturn p.pstrdup(src);\n#else\n\tstd::string tmp;\n\tencodeUTF8(src, tmp);\n\treturn p.pstrdup(tmp);\n#endif\n}\n\n\nvoid Transcoder::encodeUTF8(unsigned int sv, ByteBuffer& dst)\n{\n\tsize_t bytes = encodeUTF8(sv, dst.current());\n\tdst.position(dst.position() + bytes);\n}\n\n\nsize_t Transcoder::encodeUTF8(unsigned int ch, char* dst)\n{\n\tif (ch < 0x80)\n\t{\n\t\tdst[0] = (char) ch;\n\t\treturn 1;\n\t}\n\telse if (ch < 0x800)\n\t{\n\t\tdst[0] = (char) (0xC0 + (ch >> 6));\n\t\tdst[1] = (char) (0x80 + (ch & 0x3F));\n\t\treturn 2;\n\t}\n\telse if (ch < 0x10000)\n\t{\n\t\tdst[0] = (char) (0xE0 + (ch >> 12));\n\t\tdst[1] = (char) (0x80 + ((ch >> 6) & 0x3F));\n\t\tdst[2] = (char) (0x80 + (ch & 0x3F));\n\t\treturn 3;\n\t}\n\telse if (ch <= 0x10FFFF)\n\t{\n\t\tdst[0] = (char) (0xF0 + (ch >> 18));\n\t\tdst[1] = (char) (0x80 + ((ch >> 12) & 0x3F));\n\t\tdst[2] = (char) (0x80 + ((ch >> 6) & 0x3F));\n\t\tdst[3] = (char) (0x80 + (ch & 0x3F));\n\t\treturn 4;\n\t}\n\telse\n\t{\n\t\t//\n\t\t//  output UTF-8 encoding of 0xFFFF\n\t\t//\n\t\tdst[0] = (char) 0xEF;\n\t\tdst[1] = (char) 0xBF;\n\t\tdst[2] = (char) 0xBF;\n\t\treturn 3;\n\t}\n}\n\nvoid Transcoder::encodeUTF16BE(unsigned int sv, ByteBuffer& dst)\n{\n\tsize_t bytes = encodeUTF16BE(sv, dst.current());\n\tdst.position(dst.position() + bytes);\n}\n\n\nsize_t Transcoder::encodeUTF16BE(unsigned int ch, char* dst)\n{\n\tif (ch <= 0xFFFF)\n\t{\n\t\tdst[0] = (char) (ch >> 8);\n\t\tdst[1] = (char) (ch & 0xFF);\n\t\treturn 2;\n\t}\n\n\tif (ch <= 0x10FFFF)\n\t{\n\t\tunsigned char w = (unsigned char) ((ch >> 16) - 1);\n\t\tdst[0] = (char) (0xD8 + (w >> 2));\n\t\tdst[1] = (char) (((w & 0x03) << 6) + ((ch >> 10) & 0x3F));\n\t\tdst[2] = (char) (0xDC + ((ch & 0x30) >> 4));\n\t\tdst[3] = (char) (ch & 0xFF);\n\t\treturn 4;\n\t}\n\n\tdst[0] = dst[1] = (char) 0xFF;\n\treturn 2;\n}\n\nvoid Transcoder::encodeUTF16LE(unsigned int sv, ByteBuffer& dst)\n{\n\tsize_t bytes = encodeUTF16LE(sv, dst.current());\n\tdst.position(dst.position() + bytes);\n}\n\nsize_t Transcoder::encodeUTF16LE(unsigned int ch, char* dst)\n{\n\tif (ch <= 0xFFFF)\n\t{\n\t\tdst[1] = (char) (ch >> 8);\n\t\tdst[0] = (char) (ch & 0xFF);\n\t\treturn 2;\n\t}\n\n\tif (ch <= 0x10FFFF)\n\t{\n\t\tunsigned char w = (unsigned char) ((ch >> 16) - 1);\n\t\tdst[1] = (char) (0xD8 + (w >> 2));\n\t\tdst[0] = (char) (((w & 0x03) << 6) + ((ch >> 10) & 0x3F));\n\t\tdst[3] = (char) (0xDC + ((ch & 0x30) >> 4));\n\t\tdst[2] = (char) (ch & 0xFF);\n\t\treturn 4;\n\t}\n\n\tdst[0] = dst[1] = (char) 0xFF;\n\treturn 2;\n}\n\n\nunsigned int Transcoder::decode(const std::string& src,\n\tstd::string::const_iterator& iter)\n{\n\tstd::string::const_iterator start(iter);\n\tunsigned char ch1 = *(iter++);\n\n\tif (ch1 <= 0x7F)\n\t{\n\t\treturn ch1;\n\t}\n\n\t//\n\t//   should not have continuation character here\n\t//\n\tif ((ch1 & 0xC0) != 0x80 && iter != src.end())\n\t{\n\t\tunsigned char ch2 = *(iter++);\n\n\t\t//\n\t\t//   should be continuation\n\t\tif ((ch2 & 0xC0) != 0x80)\n\t\t{\n\t\t\titer = start;\n\t\t\treturn 0xFFFF;\n\t\t}\n\n\t\tif ((ch1 & 0xE0) == 0xC0)\n\t\t{\n\t\t\tif ((ch2 & 0xC0) == 0x80)\n\t\t\t{\n\t\t\t\tunsigned int rv = ((ch1 & 0x1F) << 6) + (ch2 & 0x3F);\n\n\t\t\t\tif (rv >= 0x80)\n\t\t\t\t{\n\t\t\t\t\treturn rv;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\titer = start;\n\t\t\treturn 0xFFFF;\n\t\t}\n\n\t\tif (iter != src.end())\n\t\t{\n\t\t\tunsigned char ch3 = *(iter++);\n\n\t\t\t//\n\t\t\t//   should be continuation\n\t\t\t//\n\t\t\tif ((ch3 & 0xC0) != 0x80)\n\t\t\t{\n\t\t\t\titer = start;\n\t\t\t\treturn 0xFFFF;\n\t\t\t}\n\n\t\t\tif ((ch1 & 0xF0) == 0xE0)\n\t\t\t{\n\t\t\t\tunsigned rv = ((ch1 & 0x0F) << 12)\n\t\t\t\t\t+ ((ch2 & 0x3F) << 6)\n\t\t\t\t\t+ (ch3 & 0x3F);\n\n\t\t\t\tif (rv <= 0x800)\n\t\t\t\t{\n\t\t\t\t\titer = start;\n\t\t\t\t\treturn 0xFFFF;\n\t\t\t\t}\n\n\t\t\t\treturn rv;\n\t\t\t}\n\n\t\t\tif (iter != src.end())\n\t\t\t{\n\t\t\t\tunsigned char ch4 = *(iter++);\n\n\t\t\t\tif ((ch4 & 0xC0) != 0x80)\n\t\t\t\t{\n\t\t\t\t\titer = start;\n\t\t\t\t\treturn 0xFFFF;\n\t\t\t\t}\n\n\t\t\t\tunsigned int rv = ((ch1 & 0x07) << 18)\n\t\t\t\t\t+ ((ch2 & 0x3F) << 12)\n\t\t\t\t\t+ ((ch3 & 0x3F) << 6)\n\t\t\t\t\t+ (ch4 & 0x3F);\n\n\t\t\t\tif (rv > 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn rv;\n\t\t\t\t}\n\n\t\t\t}\n\t\t}\n\t}\n\n\titer = start;\n\treturn 0xFFFF;\n}\n\n\nvoid Transcoder::encode(unsigned int sv, std::string& dst)\n{\n\tchar tmp[8];\n\tsize_t bytes = encodeUTF8(sv, tmp);\n\tdst.append(tmp, bytes);\n}\n\n\nvoid Transcoder::decode(const std::string& src, LogString& dst)\n{\n#if LOG4CXX_CHARSET_UTF8 && LOG4CXX_LOGCHAR_IS_UTF8\n\tdst.append(src);\n#else\n\tstatic CharsetDecoderPtr decoder(CharsetDecoder::getDefaultDecoder());\n\tdst.reserve(dst.size() + src.size());\n\tstd::string::const_iterator iter = src.begin();\n#if !LOG4CXX_CHARSET_EBCDIC\n\n\tfor (;\n\t\titer != src.end() && ((unsigned char) *iter) < 0x80;\n\t\titer++)\n\t{\n\t\tdst.append(1, *iter);\n\t}\n\n#endif\n\n\tif (iter != src.end())\n\t{\n\t\tsize_t offset = iter - src.begin();\n\t\tByteBuffer buf(const_cast<char*>(src.data() + offset), src.size() - offset);\n\n\t\twhile (buf.remaining() > 0)\n\t\t{\n\t\t\tlog4cxx_status_t stat = decoder->decode(buf, dst);\n\n\t\t\tif (CharsetDecoder::isError(stat))\n\t\t\t{\n\t\t\t\tdst.append(1, LOSSCHAR);\n\t\t\t\tbuf.position(buf.position() + 1);\n\t\t\t}\n\t\t}\n\n\t\tdecoder->decode(buf, dst);\n\t}\n\n#endif\n}\n\nchar* Transcoder::encode(const LogString& src, Pool& p)\n{\n#if LOG4CXX_CHARSET_UTF8 && LOG4CXX_LOGCHAR_IS_UTF8\n\treturn p.pstrdup(src);\n#else\n\tstd::string tmp;\n\tencode(src, tmp);\n\treturn p.pstrdup(tmp);\n#endif\n}\n\n\n\nvoid Transcoder::encode(const LogString& src, std::string& dst)\n{\n#if LOG4CXX_CHARSET_UTF8 && LOG4CXX_LOGCHAR_IS_UTF8\n\tdst.append(src);\n#else\n\tstatic CharsetEncoderPtr encoder(CharsetEncoder::getDefaultEncoder());\n\tdst.reserve(dst.size() + src.size());\n\tLogString::const_iterator iter = src.begin();\n#if !LOG4CXX_CHARSET_EBCDIC\n\n\tfor (;\n\t\titer != src.end() && ((unsigned int) *iter) < 0x80;\n\t\titer++)\n\t{\n\t\tdst.append(1, *iter);\n\t}\n\n#endif\n\n\tif (iter != src.end())\n\t{\n\t\tchar buf[BUFSIZE];\n\t\tByteBuffer out(buf, BUFSIZE);\n\n\t\twhile (iter != src.end())\n\t\t{\n\t\t\tlog4cxx_status_t stat = encoder->encode(src, iter, out);\n\t\t\tout.flip();\n\t\t\tdst.append(out.data(), out.limit());\n\t\t\tout.clear();\n\n\t\t\tif (CharsetEncoder::isError(stat))\n\t\t\t{\n\t\t\t\tdst.append(1, LOSSCHAR);\n\t\t\t\titer++;\n\t\t\t}\n\t\t}\n\n\t\tencoder->encode(src, iter, out);\n\t}\n\n#endif\n}\n\n\ntemplate<class String, class Iterator>\n\nstatic unsigned int decodeUTF16(const String& in, Iterator& iter)\n{\n\tunsigned int ch1 = *iter;\n\n\t//\n\t//   if not surrogate pair\n\t//\n\tif (ch1 < 0xD800 || ch1 > 0xDFFF)\n\t{\n\t\t//\n\t\t//  then advance iterator and return wchar_t value\n\t\t//\n\t\tif (ch1 != 0xFFFF)\n\t\t{\n\t\t\titer++;\n\t\t}\n\n\t\treturn ch1;\n\t}\n\telse if (ch1 < 0xDC00)\n\t{\n\t\t//\n\t\t//  started with high-surrogate value\n\t\t//     if there is an additional wchar_t\n\t\tIterator iter2 = iter + 1;\n\n\t\tif (iter2 != in.end())\n\t\t{\n\t\t\tunsigned int ch2 = *iter2;\n\n\t\t\t//\n\t\t\t//    if it is a matching low surrogate then\n\t\t\t//       advance the iterator and return the scalar value\n\t\t\tif (ch2 >= 0xDC00 && ch2 <= 0xDFFF)\n\t\t\t{\n\t\t\t\titer += 2;\n\t\t\t\treturn (ch1 - 0xD800) * 0x400 + (ch2 - 0xDC00) + 0x10000;\n\t\t\t}\n\t\t}\n\t}\n\n\t//\n\t//    unrecognized value, do not advance iterator\n\t//\n\treturn 0xFFFF;\n}\n\ntemplate<class String>\nstatic void encodeUTF16(unsigned int sv, String& dst)\n{\n\tif (sv < 0x10000)\n\t{\n\t\tdst.append(1, sv);\n\t}\n\telse\n\t{\n\t\tunsigned char u = (unsigned char) (sv >> 16);\n\t\tunsigned char w = (unsigned char) (u - 1);\n\t\tunsigned short hs = (0xD800 + ((w & 0xF) << 6) + ((sv & 0xFFFF) >> 10));\n\t\tunsigned short ls = (0xDC00 + (sv & 0x3FF));\n\t\tdst.append(1, hs);\n\t\tdst.append(1, ls);\n\t}\n}\n\n\n\n#if LOG4CXX_WCHAR_T_API || LOG4CXX_LOGCHAR_IS_WCHAR_T || defined(WIN32) || defined(_WIN32)\nvoid Transcoder::decode(const std::wstring& src, LogString& dst)\n{\n#if LOG4CXX_LOGCHAR_IS_WCHAR_T\n\tdst.append(src, len);\n#else\n\tstd::wstring::const_iterator i = src.begin();\n\n\twhile (i != src.end())\n\t{\n\t\tunsigned int cp = decode(src, i);\n\n\t\tif (cp != 0xFFFF)\n\t\t{\n\t\t\tencode(cp, dst);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdst.append(1, LOSSCHAR);\n\t\t\ti++;\n\t\t}\n\t}\n\n#endif\n}\n\nvoid Transcoder::encode(const LogString& src, std::wstring& dst)\n{\n#if LOG4CXX_LOGCHAR_IS_WCHAR_T\n\tdst.append(src);\n#else\n\n\tfor (LogString::const_iterator i = src.begin(); i != src.end();)\n\t{\n\t\tunsigned int cp = Transcoder::decode(src, i);\n\n\t\tif (cp != 0xFFFF)\n\t\t{\n\t\t\tencode(cp, dst);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdst.append(1, LOSSCHAR);\n\t\t\ti++;\n\t\t}\n\t}\n\n#endif\n}\n\nwchar_t* Transcoder::wencode(const LogString& src, Pool& p)\n{\n#if LOG4CXX_LOGCHAR_IS_WCHAR_T\n\tstd::wstring& tmp = src;\n#else\n\tstd::wstring tmp;\n\tencode(src, tmp);\n#endif\n\twchar_t* dst = (wchar_t*) p.palloc((tmp.length() + 1) * sizeof(wchar_t));\n\tdst[tmp.length()] = 0;\n\tstd::memcpy(dst, tmp.data(), tmp.length() * sizeof(wchar_t));\n\treturn dst;\n}\n\n\nunsigned int Transcoder::decode(const std::wstring& in,\n\tstd::wstring::const_iterator& iter)\n{\n#if defined(__STDC_ISO_10646__)\n\treturn *(iter++);\n#else\n\treturn decodeUTF16(in, iter);\n#endif\n}\n\n\nvoid Transcoder::encode(unsigned int sv, std::wstring& dst)\n{\n#if defined(__STDC_ISO_10646__)\n\tdst.append(1, sv);\n#else\n\n\tif (sizeof(wchar_t) == 4)\n\t{\n\t\tdst.append(1, sv);\n\t}\n\telse\n\t{\n\t\tencodeUTF16(sv, dst);\n\t}\n\n#endif\n}\n\n#endif\n\n\n\n#if LOG4CXX_UNICHAR_API || LOG4CXX_LOGCHAR_IS_UNICHAR\nvoid Transcoder::decode(const std::basic_string<UniChar>& src, LogString& dst)\n{\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\tdst.append(src);\n#else\n\n\tfor (std::basic_string<UniChar>::const_iterator i = src.begin();\n\t\ti != src.end();)\n\t{\n\t\tunsigned int cp = decode(src, i);\n\t\tencode(cp, dst);\n\t}\n\n#endif\n}\n\nvoid Transcoder::encode(const LogString& src, std::basic_string<UniChar>& dst)\n{\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\tdst.append(src);\n#else\n\n\tfor (LogString::const_iterator i = src.begin();\n\t\ti != src.end();)\n\t{\n\t\tunsigned int cp = decode(src, i);\n\t\tencode(cp, dst);\n\t}\n\n#endif\n}\n\nunsigned int Transcoder::decode(const std::basic_string<UniChar>& in,\n\tstd::basic_string<UniChar>::const_iterator& iter)\n{\n\treturn decodeUTF16(in, iter);\n}\n\nvoid Transcoder::encode(unsigned int sv, std::basic_string<UniChar>& dst)\n{\n\tencodeUTF16(sv, dst);\n}\n\n#endif\n\n#if LOG4CXX_CFSTRING_API\nvoid Transcoder::decode(const CFStringRef& src, LogString& dst)\n{\n\tauto length = CFStringGetLength(src);\n#if defined(_DEBUG)\n\tPool pool;\n\tLogString msg(LOG4CXX_STR(\"Transcoder::decodeCFString\"));\n\tmsg += LOG4CXX_STR(\" length \");\n\tStringHelper::toString((size_t)length, pool, msg);\n\tLogLog::debug(msg);\n#endif\n\n\tif (length > 0)\n\t{\n\t\tstd::vector<unsigned short> tmp(length);\n\t\tCFStringGetCharacters(src, CFRangeMake(0, length), &tmp[0]);\n\t\tfor (auto i = tmp.begin(); i != tmp.end(); )\n\t\t{\n\t\t\tunsigned int cp = decodeUTF16(tmp, i);\n\t\t\tencode(cp, dst);\n\t\t}\n\t}\n}\n\nCFStringRef Transcoder::encode(const LogString& src)\n{\n\tstd::basic_string<unsigned short> tmp;\n\tfor (auto ch : src)\n\t\tencodeUTF16(ch, tmp);\n\treturn CFStringCreateWithCharacters(kCFAllocatorDefault, tmp.data(), tmp.size());\n}\n#endif // #if LOG4CXX_CFSTRING_API\n\n\nlogchar Transcoder::decode(char val)\n{\n#if LOG4CXX_CHARSET_EBCDIC\n\tLogString dst;\n\tTranscoder::decode(std::string(1, val), dst);\n\treturn dst[0];\n#else\n\treturn val;\n#endif\n}\n\nLogString Transcoder::decode(const char* val)\n{\n#if LOG4CXX_LOGCHAR_IS_UTF8 && !LOG4CXX_CHARSET_EBCDIC\n\treturn val;\n#else\n\tLogString dst;\n\tTranscoder::decode(val, dst);\n\treturn dst;\n#endif\n}\n\n\nstd::string Transcoder::encodeCharsetName(const LogString& val)\n{\n\tchar asciiTable[] = { ' ', '!', '\"', '#', '$', '%', '&', '\\'', '(', ')', '*', '+', ',', '-', '.', '/',\n\t\t\t'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?',\n\t\t\t'@', 'A', 'B', 'C', 'D', 'E', 'F',  'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',\n\t\t\t'P', 'Q', 'R', 'S', 'T', 'U', 'V',  'W', 'X', 'Y', 'Z', '[', '\\\\', ']', '^', '_',\n\t\t\t'`', 'a', 'b', 'c', 'd', 'e', 'f',  'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',\n\t\t\t'p', 'q', 'r', 's', 't', 'u', 'v',  'w', 'x', 'y', 'z', '{', '|', '}', '~'\n\t\t};\n\tstd::string out;\n\n\tfor (auto& item : val)\n\t{\n\t\tif (item >= 0x20 && item < 0x7F)\n\t\t{\n\t\t\tout.append(1, asciiTable[item - 0x20]);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tout.append(1, LOSSCHAR);\n\t\t}\n\t}\n\n\treturn out;\n}\n\n// Path: src/main/cpp/linelocationpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/linelocationpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(LineLocationPatternConverter)\n\nLineLocationPatternConverter::LineLocationPatternConverter() :\n\tLoggingEventPatternConverter(LOG4CXX_STR(\"Line\"),\n\t\tLOG4CXX_STR(\"line\"))\n{\n}\n\nPatternConverterPtr LineLocationPatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> instance = std::make_shared<LineLocationPatternConverter>();\n\treturn instance;\n}\n\nvoid LineLocationPatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& p) const\n{\n\tStringHelper::toString(\n\t\tevent->getLocationInformation().getLineNumber(),\n\t\tp, toAppendTo);\n}\n\n// Path: src/main/cpp/integerpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/integerpatternconverter.h>\n#include <log4cxx/helpers/integer.h>\n#include <log4cxx/helpers/stringhelper.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(IntegerPatternConverter)\n\nIntegerPatternConverter::IntegerPatternConverter() :\n\tPatternConverter(LOG4CXX_STR(\"Integer\"),\n\t\tLOG4CXX_STR(\"integer\"))\n{\n}\n\nPatternConverterPtr IntegerPatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> instance = std::make_shared<IntegerPatternConverter>();\n\treturn instance;\n}\n\nvoid IntegerPatternConverter::format(\n\tconst ObjectPtr& obj,\n\tLogString& toAppendTo,\n\tPool& p) const\n{\n\tIntegerPtr i = LOG4CXX_NS::cast<Integer>(obj);\n\n\tif (i != NULL)\n\t{\n\t\tStringHelper::toString(i->intValue(), p, toAppendTo);\n\t}\n}\n\n// Path: src/main/cpp/triggeringpolicy.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/rolling/triggeringpolicy.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::rolling;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(TriggeringPolicy)\n\nTriggeringPolicy::~TriggeringPolicy()\n{\n}\n\n// Path: src/main/cpp/fallbackerrorhandler.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/appender.h>\n#include <log4cxx/logger.h>\n#include <log4cxx/varia/fallbackerrorhandler.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/hierarchy.h>\n#include <log4cxx/logmanager.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::varia;\n\nIMPLEMENT_LOG4CXX_OBJECT(FallbackErrorHandler)\n\nstruct FallbackErrorHandler::FallbackErrorHandlerPrivate\n{\n\tAppenderWeakPtr backup;\n\tAppenderWeakPtr primary;\n\tstd::vector<LoggerPtr> loggers;\n\tbool errorReported = false;\n};\n\nFallbackErrorHandler::FallbackErrorHandler()\n\t: m_priv(std::make_unique<FallbackErrorHandlerPrivate>())\n{\n}\n\nFallbackErrorHandler::~FallbackErrorHandler() {}\n\nvoid FallbackErrorHandler::setLogger(const LoggerPtr& logger)\n{\n\tLogLog::debug(((LogString) LOG4CXX_STR(\"FB: Adding logger [\"))\n\t\t+ logger->getName() + LOG4CXX_STR(\"].\"));\n\tm_priv->loggers.push_back(logger);\n}\n\nvoid FallbackErrorHandler::error(const LogString& message,\n\tconst std::exception& e,\n\tint errorCode) const\n{\n\terror(message, e, errorCode, 0);\n}\n\nvoid FallbackErrorHandler::error(const LogString& message,\n\tconst std::exception& e,\n\tint, const spi::LoggingEventPtr&) const\n{\n\tLogLog::debug(((LogString) LOG4CXX_STR(\"FB: The following error reported: \"))\n\t\t+  message, e);\n\tLogLog::debug(LOG4CXX_STR(\"FB: INITIATING FALLBACK PROCEDURE.\"));\n\n\tAppenderPtr primaryLocked = m_priv->primary.lock();\n\tAppenderPtr backupLocked = m_priv->backup.lock();\n\n\tif ( !primaryLocked || !backupLocked )\n\t{\n\t\treturn;\n\t}\n\n\tfor (LoggerPtr l : m_priv->loggers)\n\t{\n\t\tLogLog::debug(((LogString) LOG4CXX_STR(\"FB: Searching for [\"))\n\t\t\t+ primaryLocked->getName() + LOG4CXX_STR(\"] in logger [\")\n\t\t\t+ l->getName() + LOG4CXX_STR(\"].\"));\n\t\tLogLog::debug(((LogString) LOG4CXX_STR(\"FB: Replacing [\"))\n\t\t\t+ primaryLocked->getName() + LOG4CXX_STR(\"] by [\")\n\t\t\t+ backupLocked->getName() + LOG4CXX_STR(\"] in logger [\")\n\t\t\t+ l->getName() + LOG4CXX_STR(\"].\"));\n\t\tl->removeAppender(primaryLocked);\n\t\tLogLog::debug(((LogString) LOG4CXX_STR(\"FB: Adding appender [\"))\n\t\t\t+ backupLocked->getName() + LOG4CXX_STR(\"] to logger \")\n\t\t\t+ l->getName());\n\t\tl->addAppender(backupLocked);\n\t}\n\tm_priv->errorReported = true;\n}\n\nvoid FallbackErrorHandler::setAppender(const AppenderPtr& primary1)\n{\n\tLogLog::debug(((LogString) LOG4CXX_STR(\"FB: Setting primary appender to [\"))\n\t\t+ primary1->getName() + LOG4CXX_STR(\"].\"));\n\tm_priv->primary = primary1;\n}\n\nvoid FallbackErrorHandler::setBackupAppender(const AppenderPtr& backup1)\n{\n\tLogLog::debug(((LogString) LOG4CXX_STR(\"FB: Setting backup appender to [\"))\n\t\t+ backup1->getName() + LOG4CXX_STR(\"].\"));\n\tm_priv->backup = backup1;\n\n\t// Make sure that we keep a reference to the appender around, since otherwise\n\t// the appender would be lost if it has no loggers that use it.\n\tLoggerRepository* repository = LogManager::getRootLogger()->getLoggerRepository();\n\tHierarchy* hierarchy = dynamic_cast<Hierarchy*>(repository);\n\tif(hierarchy){\n\t\thierarchy->addAppender(backup1);\n\t}\n\n}\n\nvoid FallbackErrorHandler::activateOptions(Pool&)\n{\n}\n\nvoid FallbackErrorHandler::setOption(const LogString&, const LogString&)\n{\n}\n\nbool FallbackErrorHandler::errorReported() const\n{\n\treturn m_priv->errorReported;\n}\n\n// Path: src/main/cpp/ndc.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/ndc.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/threadspecificdata.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nNDC::NDC(const std::string& message)\n{\n\tpush(message);\n}\n\nNDC::~NDC()\n{\n\tpop();\n}\n\n\nLogString& NDC::getMessage(NDC::DiagnosticContext& ctx)\n{\n\treturn ctx.first;\n}\n\nLogString& NDC::getFullMessage(NDC::DiagnosticContext& ctx)\n{\n\treturn ctx.second;\n}\n\nvoid NDC::clear()\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\twhile (!stack.empty())\n\t\t{\n\t\t\tstack.pop();\n\t\t}\n\n\t\tdata->recycle();\n\t}\n}\n\nNDC::Stack* NDC::cloneStack()\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\treturn new Stack(stack);\n\t\t}\n\t}\n\n\treturn new Stack();\n}\n\nvoid NDC::inherit(NDC::Stack* stack)\n{\n\tif (stack != NULL)\n\t{\n\t\tThreadSpecificData::inherit(*stack);\n\t\tdelete stack;\n\t}\n}\n\n\nbool NDC::get(LogString& dest)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tdest.append(getFullMessage(stack.top()));\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\nint NDC::getDepth()\n{\n\tint size = 0;\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tsize = (int)data->getStack().size();\n\n\t\tif (size == 0)\n\t\t{\n\t\t\tdata->recycle();\n\t\t}\n\t}\n\n\treturn size;\n}\n\nLogString NDC::pop()\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tLogString value(getMessage(stack.top()));\n\t\t\tstack.pop();\n\t\t\tdata->recycle();\n\t\t\treturn value;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn LogString();\n}\n\nbool NDC::pop(std::string& dst)\n{\n\tbool retval = false;\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tTranscoder::encode(getMessage(stack.top()), dst);\n\t\t\tstack.pop();\n\t\t\tretval = true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn retval;\n}\n\nLogString NDC::peek()\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\treturn getMessage(stack.top());\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn LogString();\n}\n\nbool NDC::peek(std::string& dst)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tTranscoder::encode(getMessage(stack.top()), dst);\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\nvoid NDC::pushLS(const LogString& message)\n{\n\tThreadSpecificData::push(message);\n}\n\nvoid NDC::push(const std::string& message)\n{\n\tLOG4CXX_DECODE_CHAR(msg, message);\n\tpushLS(msg);\n}\n\nvoid NDC::remove()\n{\n\tclear();\n}\n\nbool NDC::empty()\n{\n\tbool empty = true;\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\t\tempty = stack.empty();\n\n\t\tif (empty)\n\t\t{\n\t\t\tdata->recycle();\n\t\t}\n\t}\n\n\treturn empty;\n}\n\n#if LOG4CXX_WCHAR_T_API\nNDC::NDC(const std::wstring& message)\n{\n\tpush(message);\n}\n\nvoid NDC::push(const std::wstring& message)\n{\n\tLOG4CXX_DECODE_WCHAR(msg, message);\n\tpushLS(msg);\n}\n\nbool NDC::pop(std::wstring& dst)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tTranscoder::encode(getMessage(stack.top()), dst);\n\t\t\tstack.pop();\n\t\t\tdata->recycle();\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\nbool NDC::peek(std::wstring& dst)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tTranscoder::encode(getMessage(stack.top()), dst);\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\n#endif\n\n\n#if LOG4CXX_UNICHAR_API\nNDC::NDC(const std::basic_string<UniChar>& message)\n{\n\tpush(message);\n}\n\nvoid NDC::push(const std::basic_string<UniChar>& message)\n{\n\tLOG4CXX_DECODE_UNICHAR(msg, message);\n\tpushLS(msg);\n}\n\nbool NDC::pop(std::basic_string<UniChar>& dst)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tTranscoder::encode(getMessage(stack.top()), dst);\n\t\t\tstack.pop();\n\t\t\tdata->recycle();\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\nbool NDC::peek(std::basic_string<UniChar>& dst)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tTranscoder::encode(getMessage(stack.top()), dst);\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\n#endif\n\n\n#if LOG4CXX_CFSTRING_API\nNDC::NDC(const CFStringRef& message)\n{\n\tpush(message);\n}\n\nvoid NDC::push(const CFStringRef& message)\n{\n\tLOG4CXX_DECODE_CFSTRING(msg, message);\n\tpushLS(msg);\n}\n\nbool NDC::pop(CFStringRef& dst)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tdst = Transcoder::encode(getMessage(stack.top()));\n\t\t\tstack.pop();\n\t\t\tdata->recycle();\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\nbool NDC::peek(CFStringRef& dst)\n{\n\tThreadSpecificData* data = ThreadSpecificData::getCurrentData();\n\n\tif (data != 0)\n\t{\n\t\tStack& stack = data->getStack();\n\n\t\tif (!stack.empty())\n\t\t{\n\t\t\tdst = Transcoder::encode(getMessage(stack.top()));\n\t\t\treturn true;\n\t\t}\n\n\t\tdata->recycle();\n\t}\n\n\treturn false;\n}\n\n#endif\n\n\n// Path: src/main/cpp/fileinputstream.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/fileinputstream.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <apr_file_io.h>\n#include <log4cxx/helpers/transcoder.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct FileInputStream::FileInputStreamPrivate\n{\n\tFileInputStreamPrivate() : fileptr(nullptr) {}\n\n\tPool pool;\n\tapr_file_t* fileptr;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(FileInputStream)\n\nFileInputStream::FileInputStream(const LogString& filename) :\n\tm_priv(std::make_unique<FileInputStreamPrivate>())\n{\n\topen(filename);\n}\n\nFileInputStream::FileInputStream(const logchar* filename) :\n\tm_priv(std::make_unique<FileInputStreamPrivate>())\n{\n\tLogString fn(filename);\n\topen(fn);\n}\n\n\nvoid FileInputStream::open(const LogString& filename)\n{\n\tapr_fileperms_t perm = APR_OS_DEFAULT;\n\tapr_int32_t flags = APR_READ;\n\tapr_status_t stat = File().setPath(filename).open(&m_priv->fileptr, flags, perm, m_priv->pool);\n\n\tif (stat != APR_SUCCESS)\n\t{\n\t\tthrow IOException(stat);\n\t}\n}\n\n\nFileInputStream::FileInputStream(const File& aFile) :\n\tm_priv(std::make_unique<FileInputStreamPrivate>())\n{\n\tapr_fileperms_t perm = APR_OS_DEFAULT;\n\tapr_int32_t flags = APR_READ;\n\tapr_status_t stat = aFile.open(&m_priv->fileptr, flags, perm, m_priv->pool);\n\n\tif (stat != APR_SUCCESS)\n\t{\n\t\tthrow IOException(stat);\n\t}\n}\n\n\nFileInputStream::~FileInputStream()\n{\n\tif (m_priv->fileptr != NULL && !APRInitializer::isDestructed)\n\t{\n\t\tapr_file_close(m_priv->fileptr);\n\t}\n}\n\n\nvoid FileInputStream::close()\n{\n\tapr_status_t stat = apr_file_close(m_priv->fileptr);\n\n\tif (stat == APR_SUCCESS)\n\t{\n\t\tm_priv->fileptr = NULL;\n\t}\n\telse\n\t{\n\t\tthrow IOException(stat);\n\t}\n}\n\n\nint FileInputStream::read(ByteBuffer& buf)\n{\n\tapr_size_t bytesRead = buf.remaining();\n\tapr_status_t stat = apr_file_read(m_priv->fileptr, buf.current(), &bytesRead);\n\tint retval = -1;\n\n\tif (!APR_STATUS_IS_EOF(stat))\n\t{\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\tbuf.position(buf.position() + bytesRead);\n\t\tretval = (int)bytesRead;\n\t}\n\n\treturn retval;\n}\n\n// Path: src/main/cpp/classnamepatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/classnamepatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(ClassNamePatternConverter)\n\nClassNamePatternConverter::ClassNamePatternConverter(\n\tconst std::vector<LogString>& options) :\n\tNamePatternConverter(LOG4CXX_STR(\"Class Name\"),\n\t\tLOG4CXX_STR(\"class name\"), options)\n{\n}\n\nPatternConverterPtr ClassNamePatternConverter::newInstance(\n\tconst std::vector<LogString>& options)\n{\n\tif (options.size() == 0)\n\t{\n\t\tstatic WideLife<PatternConverterPtr> def = std::make_shared<ClassNamePatternConverter>(options);\n\t\treturn def;\n\t}\n\n\treturn std::make_shared<ClassNamePatternConverter>(options);\n}\n\nvoid ClassNamePatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& /* p */) const\n{\n\tint initialLength = (int)toAppendTo.length();\n\tappend(toAppendTo, event->getLocationInformation().getClassName());\n\tabbreviate(initialLength, toAppendTo);\n}\n\n// Path: src/main/cpp/pool.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n...\n", "needle_token_start": 5095, "needle_token_end": 5486, "code_context_ntokens": 15141, "output": ["```cpp\ntemplate<class String, class Iterator>\nstatic unsigned int decodeUTF16(const String& in, Iterator& iter)\n{\n\tunsigned int ch1 = *iter;\n\n\t//\n\t//   if not surrogate pair\n\t//\n\tif (ch1 < 0xD800 || ch1 > 0xDFFF)\n\t{\n\t\t//\n\t\t//  then advance iterator and return wchar_t value\n\t\t//\n\t\tif (ch1 != 0xFFFF)\n\t\t{\n\t\t\titer++;\n\t\t}\n\n\t\treturn ch1;\n\t}\n\telse if (ch1 < 0xDC00)\n\t{\n\t\t//\n\t\t//  started with high-surrogate value\n\t\t//     if there is an additional wchar_t\n\t\tIterator iter2 = iter + 1;\n\n\t\tif (iter2 != in.end())\n\t\t{\n\t\t\tunsigned int ch2 = *iter2;\n\n\t\t\t//\n\t\t\t//    if it is a matching low surrogate then\n\t\t\t//       advance the iterator and return the scalar value\n\t\t\tif (ch2 >= 0xDC00 && ch2 <= 0xDFFF)\n\t\t\t{\n\t\t\t\titer += 2;\n\t\t\t\treturn (ch1 - 0xD800) * 0x400 + (ch2 - 0xDC00) + 0x10000;\n\t\t\t}\n\t\t}\n\t}\n\n\t//\n\t//    unrecognized value, do not advance iterator\n\t//\n\treturn 0xFFFF;\n}\n```"]}
{"repo": "apache/logging-log4cxx", "name": "catch", "language": "cpp", "path": "src/main/cpp/odbcappender.cpp", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this function is to handle exceptions that occur during the execution of a prepared SQL statement in a database logging context. It ensures that any SQL-related errors are caught and reported appropriately.\n2. **Input**: The input includes a prepared SQL statement and a logging event that contains the data to be logged.\n3. **Output**: There is no direct output from this function; however, it affects the program by logging error messages to an error handler if an exception occurs.\n4. **Procedure**: The procedure involves attempting to execute a prepared SQL statement with the provided logging event data. If the execution fails (indicated by a negative return value), an SQL exception is thrown. This exception is then caught, and an error message is logged through an error handler, specifying the nature of the failure and the error code associated with the SQL execution failure.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " * limitations under the License.\n */\n#include <log4cxx/db/odbcappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/patternlayout.h>\n#include <log4cxx/pattern/mdcpatternconverter.h>\n#include <apr_strings.h>\n#include <apr_time.h>\n#include <cmath> // std::pow\n\n#include <log4cxx/pattern/loggerpatternconverter.h>\n#include <log4cxx/pattern/classnamepatternconverter.h>\n#include <log4cxx/pattern/datepatternconverter.h>\n#include <log4cxx/pattern/filelocationpatternconverter.h>\n#include <log4cxx/pattern/fulllocationpatternconverter.h>\n#include <log4cxx/pattern/shortfilelocationpatternconverter.h>\n#include <log4cxx/pattern/linelocationpatternconverter.h>\n#include <log4cxx/pattern/messagepatternconverter.h>\n#include <log4cxx/pattern/methodlocationpatternconverter.h>\n#include <log4cxx/pattern/levelpatternconverter.h>\n#include <log4cxx/pattern/threadpatternconverter.h>\n#include <log4cxx/pattern/threadusernamepatternconverter.h>\n#include <log4cxx/pattern/ndcpatternconverter.h>\n\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n#if LOG4CXX_HAVE_ODBC\n\t#if defined(WIN32) || defined(_WIN32)\n\t\t#include <windows.h>\n\t#endif\n\t#include <sqlext.h>\n#else\n\ttypedef void* SQLHSTMT;\n#endif\n#include <log4cxx/private/odbcappender_priv.h>\n#if defined(min)\n\t#undef min\n#endif\n#include <cstring>\n#include <algorithm>\n\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::db;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::pattern;\n\nSQLException::SQLException(short fHandleType,\n\tvoid* hInput, const char* prolog,\n\tLOG4CXX_NS::helpers::Pool& p)\n\t: Exception(formatMessage(fHandleType, hInput, prolog, p))\n{\n}\n\n\nSQLException::SQLException(const char* msg)\n\t: Exception(msg)\n{\n}\n\nSQLException::SQLException(const SQLException& src)\n\t: Exception(src)\n{\n}\n\nconst char* SQLException::formatMessage(short fHandleType,\n\tvoid* hInput, const char* prolog, LOG4CXX_NS::helpers::Pool& p)\n{\n\tstd::string strReturn(prolog);\n\tstrReturn.append(\" - \");\n#if LOG4CXX_HAVE_ODBC\n\tSQLCHAR       SqlState[6];\n\tSQLCHAR       Msg[SQL_MAX_MESSAGE_LENGTH];\n\tSQLINTEGER    NativeError;\n\tSQLSMALLINT   i;\n\tSQLSMALLINT   MsgLen;\n\tSQLRETURN     rc2;\n\n\t// Get the status records.\n\ti = 1;\n\n\twhile ((rc2 = SQLGetDiagRecA(fHandleType, hInput, i, SqlState, &NativeError,\n\t\t\t\t\tMsg, sizeof(Msg), &MsgLen)) != SQL_NO_DATA)\n\t{\n\t\tstrReturn.append((char*) Msg);\n\t\ti++;\n\t}\n\n#else\n\tstrReturn.append(\"log4cxx built without ODBC support\");\n#endif\n\n\treturn apr_pstrdup((apr_pool_t*) p.getAPRPool(), strReturn.c_str());\n}\n\n\nIMPLEMENT_LOG4CXX_OBJECT(ODBCAppender)\n\n#define _priv static_cast<ODBCAppenderPriv*>(m_priv.get())\n\nODBCAppender::ODBCAppender()\n\t: AppenderSkeleton (std::make_unique<ODBCAppenderPriv>(\n#if LOG4CXX_EVENTS_AT_EXIT\n\t\t[this] {\n\t\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t\tif(_priv->closed)\n\t\t\t\treturn;\n\t\t\ttry\n\t\t\t{\n\t\t\t\tflushBuffer(_priv->pool);\n\t\t\t}\n\t\t\tcatch (SQLException& e)\n\t\t\t{\n\t\t\t\t_priv->errorHandler->error(LOG4CXX_STR(\"Error flushing connection\"),\n\t\t\t\t\te, ErrorCode::GENERIC_FAILURE);\n\t\t\t}\n\t\t}\n#endif\n\t\t\t\t\t\t\t\t))\n{\n}\n\nODBCAppender::~ODBCAppender()\n{\n\tfinalize();\n}\n\n#define RULES_PUT(spec, cls) \\\n\tspecs.insert(PatternMap::value_type(LogString(LOG4CXX_STR(spec)), cls ::newInstance))\n\nstatic PatternMap getFormatSpecifiers()\n{\n\tPatternMap specs;\n\tif (specs.empty())\n\t{\n\t\tRULES_PUT(\"logger\", LoggerPatternConverter);\n\t\tRULES_PUT(\"class\", ClassNamePatternConverter);\n\t\tRULES_PUT(\"time\", DatePatternConverter);\n\t\tRULES_PUT(\"shortfilename\", ShortFileLocationPatternConverter);\n\t\tRULES_PUT(\"fullfilename\", FileLocationPatternConverter);\n\t\tRULES_PUT(\"location\", FullLocationPatternConverter);\n\t\tRULES_PUT(\"line\", LineLocationPatternConverter);\n\t\tRULES_PUT(\"message\", MessagePatternConverter);\n\t\tRULES_PUT(\"method\", MethodLocationPatternConverter);\n\t\tRULES_PUT(\"level\", LevelPatternConverter);\n\t\tRULES_PUT(\"thread\", ThreadPatternConverter);\n\t\tRULES_PUT(\"threadname\", ThreadUsernamePatternConverter);\n\t\tRULES_PUT(\"mdc\", MDCPatternConverter);\n\t\tRULES_PUT(\"ndc\", NDCPatternConverter);\n\t}\n\treturn specs;\n}\n\nvoid ODBCAppender::setOption(const LogString& option, const LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"BUFFERSIZE\"), LOG4CXX_STR(\"buffersize\")))\n\t{\n\t\tsetBufferSize((size_t)OptionConverter::toInt(value, 1));\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"PASSWORD\"), LOG4CXX_STR(\"password\")))\n\t{\n\t\tsetPassword(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"SQL\"), LOG4CXX_STR(\"sql\")))\n\t{\n\t\tsetSql(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"URL\"), LOG4CXX_STR(\"url\"))\n\t\t|| StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"DSN\"), LOG4CXX_STR(\"dsn\"))\n\t\t|| StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"CONNECTIONSTRING\"), LOG4CXX_STR(\"connectionstring\"))  )\n\t{\n\t\tsetURL(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"USER\"), LOG4CXX_STR(\"user\")))\n\t{\n\t\tsetUser(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"COLUMNMAPPING\"), LOG4CXX_STR(\"columnmapping\")))\n\t{\n\t\t_priv->mappedName.push_back(value);\n\t}\n\telse\n\t{\n\t\tAppenderSkeleton::setOption(option, value);\n\t}\n}\n\n// Does ODBCAppender require a layout?\n\nbool ODBCAppender::requiresLayout() const\n{\n\treturn false;\n}\n\nvoid ODBCAppender::activateOptions(LOG4CXX_NS::helpers::Pool&)\n{\n#if !LOG4CXX_HAVE_ODBC\n\tLogLog::error(LOG4CXX_STR(\"Can not activate ODBCAppender unless compiled with ODBC support.\"));\n#else\n\tif (_priv->mappedName.empty())\n\t{\n\t\tLogLog::error(LOG4CXX_STR(\"ODBCAppender column mappings not defined, logging events will not be inserted\"));\n\t}\n\tauto specs = getFormatSpecifiers();\n\tfor (auto& name : _priv->mappedName)\n\t{\n\t\tauto lowerName = StringHelper::toLowerCase(name);\n\t\tauto pItem = specs.find(lowerName);\n\t\tif (specs.end() == pItem)\n\t\t{\n\t\t\tif (lowerName.size() < 5\n\t\t\t || lowerName.substr(0, 4) != LOG4CXX_STR(\"mdc{\"))\n\t\t\t\tLogLog::error(name + LOG4CXX_STR(\" is not a supported ColumnMapping value\"));\n\t\t\telse // A single MDC entry\n\t\t\t{\n\t\t\t\tauto index = lowerName.find(0x7D /* '}' */, 4);\n\t\t\t\tauto len = (lowerName.npos == index ? lowerName.size() : index) - 4;\n\t\t\t\tODBCAppenderPriv::DataBinding paramData{ 0, 0, 0, 0, 0 };\n\t\t\t\tparamData.converter = std::make_shared<MDCPatternConverter>(lowerName.substr(4, len));\n\t\t\t\t_priv->parameterValue.push_back(paramData);\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tODBCAppenderPriv::DataBinding paramData{ 0, 0, 0, 0, 0 };\n\t\t\tstd::vector<LogString> options;\n\t\t\tif (LOG4CXX_STR(\"time\") == pItem->first)\n\t\t\t\toptions.push_back(LOG4CXX_STR(\"yyyy-MM-dd HH:mm:ss.SSSSSS\"));\n\t\t\tparamData.converter = LOG4CXX_NS::cast<LoggingEventPatternConverter>((pItem->second)(options));\n\t\t\t_priv->parameterValue.push_back(paramData);\n\t\t}\n\t}\n#endif\n}\n\n\nvoid ODBCAppender::append(const spi::LoggingEventPtr& event, LOG4CXX_NS::helpers::Pool& p)\n{\n#if LOG4CXX_HAVE_ODBC\n\t_priv->buffer.push_back(event);\n\n\tif (_priv->buffer.size() >= _priv->bufferSize)\n\t{\n\t\tflushBuffer(p);\n\t}\n\n#endif\n}\n\nLogString ODBCAppender::getLogStatement(const spi::LoggingEventPtr& event, LOG4CXX_NS::helpers::Pool& p) const\n{\n    return LogString();\n}\n\nvoid ODBCAppender::execute(const LogString& sql, LOG4CXX_NS::helpers::Pool& p)\n{\n}\n\n/* The default behavior holds a single connection open until the appender\nis closed (typically when garbage collected).*/\nvoid ODBCAppender::closeConnection(ODBCAppender::SQLHDBC /* con */)\n{\n}\n\nODBCAppender::SQLHDBC ODBCAppender::getConnection(LOG4CXX_NS::helpers::Pool& p)\n{\n#if LOG4CXX_HAVE_ODBC\n\tSQLRETURN ret;\n\n\tif (_priv->env == SQL_NULL_HENV)\n\t{\n\t\tret = SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &_priv->env);\n\n\t\tif (ret < 0)\n\t\t{\n\t\t\tSQLException ex(SQL_HANDLE_ENV, _priv->env, \"Failed to allocate SQL handle\", p);\n\t\t\t_priv->env = SQL_NULL_HENV;\n\t\t\tthrow ex;\n\t\t}\n\n\t\tret = SQLSetEnvAttr(_priv->env, SQL_ATTR_ODBC_VERSION, (SQLPOINTER) SQL_OV_ODBC3, SQL_IS_INTEGER);\n\n\t\tif (ret < 0)\n\t\t{\n\t\t\tSQLException ex(SQL_HANDLE_ENV, _priv->env, \"Failed to set odbc version\", p);\n\t\t\tSQLFreeHandle(SQL_HANDLE_ENV, _priv->env);\n\t\t\t_priv->env = SQL_NULL_HENV;\n\t\t\tthrow ex;\n\t\t}\n\t}\n\n\tif (_priv->connection == SQL_NULL_HDBC)\n\t{\n\t\tret = SQLAllocHandle(SQL_HANDLE_DBC, _priv->env, &_priv->connection);\n\n\t\tif (ret < 0)\n\t\t{\n\t\t\tSQLException ex(SQL_HANDLE_DBC, _priv->connection, \"Failed to allocate sql handle\", p);\n\t\t\t_priv->connection = SQL_NULL_HDBC;\n\t\t\tthrow ex;\n\t\t}\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\t\tSQLWCHAR *wUser = nullptr, *wPwd = nullptr;\n\t\tif (!_priv->databaseUser.empty())\n\t\t\twUser = (SQLWCHAR*)_priv->databaseUser.c_str();\n\t\tif (!_priv->databasePassword.empty())\n\t\t\twPwd = (SQLWCHAR*)_priv->databasePassword.c_str();\n\t\tret = SQLConnectW(_priv->connection\n\t\t\t, (SQLWCHAR*)_priv->databaseURL.c_str(), SQL_NTS\n\t\t\t, wUser, SQL_NTS\n\t\t\t, wPwd, SQL_NTS\n\t\t\t);\n#elif LOG4CXX_LOGCHAR_IS_UTF8\n\t\tSQLCHAR *wUser = nullptr, *wPwd = nullptr;\n\t\tif (!_priv->databaseUser.empty())\n\t\t\twUser = (SQLCHAR*)_priv->databaseUser.c_str();\n\t\tif (!_priv->databasePassword.empty())\n\t\t\twPwd = (SQLCHAR*)_priv->databasePassword.c_str();\n\t\tret = SQLConnectA(_priv->connection\n\t\t\t, (SQLCHAR*)_priv->databaseURL.c_str(), SQL_NTS\n\t\t\t, wUser, SQL_NTS\n\t\t\t, wPwd, SQL_NTS\n\t\t\t);\n#else\n\t\tSQLWCHAR* wURL, *wUser = nullptr, *wPwd = nullptr;\n\t\tencode(&wURL, _priv->databaseURL, p);\n\t\tif (!_priv->databaseUser.empty())\n\t\t\tencode(&wUser, _priv->databaseUser, p);\n\t\tif (!_priv->databasePassword.empty())\n\t\t\tencode(&wPwd, _priv->databasePassword, p);\n\n\t\tret = SQLConnectW( _priv->connection\n\t\t\t, wURL, SQL_NTS\n\t\t\t, wUser, SQL_NTS\n\t\t\t, wPwd, SQL_NTS\n\t\t\t);\n#endif\n\n\t\tif (ret < 0)\n\t\t{\n\t\t\tSQLException ex(SQL_HANDLE_DBC, _priv->connection, \"Failed to connect to database\", p);\n\t\t\tSQLFreeHandle(SQL_HANDLE_DBC, _priv->connection);\n\t\t\t_priv->connection = SQL_NULL_HDBC;\n\t\t\tthrow ex;\n\t\t}\n\t}\n\n\treturn _priv->connection;\n#else\n\treturn 0;\n#endif\n}\n\nvoid ODBCAppender::close()\n{\n\tif (_priv->closed)\n\t{\n\t\treturn;\n\t}\n\n\tPool p;\n\n\ttry\n\t{\n\t\tflushBuffer(p);\n\t}\n\tcatch (SQLException& e)\n\t{\n\t\t_priv->errorHandler->error(LOG4CXX_STR(\"Error closing connection\"),\n\t\t\te, ErrorCode::GENERIC_FAILURE);\n\t}\n\n#if LOG4CXX_HAVE_ODBC\n\n\tif (_priv->connection != SQL_NULL_HDBC)\n\t{\n\t\tSQLDisconnect(_priv->connection);\n\t\tSQLFreeHandle(SQL_HANDLE_DBC, _priv->connection);\n\t}\n\n\tif (_priv->env != SQL_NULL_HENV)\n\t{\n\t\tSQLFreeHandle(SQL_HANDLE_ENV, _priv->env);\n\t}\n\n#endif\n\t_priv->closed = true;\n}\n\n#if LOG4CXX_HAVE_ODBC\nvoid ODBCAppender::ODBCAppenderPriv::setPreparedStatement(SQLHDBC con, Pool& p)\n{\n\tauto ret = SQLAllocHandle( SQL_HANDLE_STMT, con, &this->preparedStatement);\n\tif (ret < 0)\n\t{\n\t\tthrow SQLException( SQL_HANDLE_DBC, con, \"Failed to allocate statement handle.\", p);\n\t}\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\tret = SQLPrepareW(this->preparedStatement, (SQLWCHAR*)this->sqlStatement.c_str(), SQL_NTS);\n#elif LOG4CXX_LOGCHAR_IS_UTF8\n\tret = SQLPrepareA(this->preparedStatement, (SQLCHAR*)this->sqlStatement.c_str(), SQL_NTS);\n#else\n\tSQLWCHAR* wsql;\n\tencode(&wsql, this->sqlStatement, p);\n\tret = SQLPrepareW(this->preparedStatement, wsql, SQL_NTS);\n#endif\n\tif (ret < 0)\n\t{\n\t\tthrow SQLException(SQL_HANDLE_STMT, this->preparedStatement, \"Failed to prepare sql statement.\", p);\n\t}\n\n\tint parameterNumber = 0;\n\tfor (auto& item : this->parameterValue)\n\t{\n\t\t++parameterNumber;\n\t\tSQLSMALLINT  targetType;\n\t\tSQLULEN      targetMaxCharCount;\n\t\tSQLSMALLINT  decimalDigits;\n\t\tSQLSMALLINT  nullable;\n\t\tauto ret = SQLDescribeParam\n\t\t\t( this->preparedStatement\n\t\t\t, parameterNumber\n\t\t\t, &targetType\n\t\t\t, &targetMaxCharCount\n\t\t\t, &decimalDigits\n\t\t\t, &nullable\n\t\t\t);\n\t\tif (ret < 0)\n\t\t{\n\t\t\tthrow SQLException(SQL_HANDLE_STMT, this->preparedStatement, \"Failed to describe parameter\", p);\n\t\t}\n\t\tif (SQL_CHAR == targetType || SQL_VARCHAR == targetType || SQL_LONGVARCHAR == targetType)\n\t\t{\n\t\t\titem.paramType = SQL_C_CHAR;\n\t\t\titem.paramMaxCharCount = targetMaxCharCount;\n\t\t\titem.paramValueSize = (SQLINTEGER)(item.paramMaxCharCount) * sizeof(char) + sizeof(char);\n\t\t\titem.paramValue = (SQLPOINTER)p.palloc(item.paramValueSize + sizeof(char));\n\t\t}\n\t\telse if (SQL_WCHAR == targetType || SQL_WVARCHAR == targetType || SQL_WLONGVARCHAR == targetType)\n\t\t{\n\t\t\titem.paramType = SQL_C_WCHAR;\n\t\t\titem.paramMaxCharCount = targetMaxCharCount;\n\t\t\titem.paramValueSize = (SQLINTEGER)(targetMaxCharCount) * sizeof(wchar_t) + sizeof(wchar_t);\n\t\t\titem.paramValue = (SQLPOINTER)p.palloc(item.paramValueSize + sizeof(wchar_t));\n\t\t}\n\t\telse if (SQL_TYPE_TIMESTAMP == targetType || SQL_TYPE_DATE == targetType || SQL_TYPE_TIME == targetType\n\t\t\t|| SQL_DATETIME == targetType)\n\t\t{\n\t\t\titem.paramType = SQL_C_TYPE_TIMESTAMP;\n\t\t\titem.paramMaxCharCount = (0 <= decimalDigits) ? decimalDigits : 6;\n\t\t\titem.paramValueSize = sizeof(SQL_TIMESTAMP_STRUCT);\n\t\t\titem.paramValue = (SQLPOINTER)p.palloc(item.paramValueSize);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (SQL_INTEGER != targetType)\n\t\t\t{\n\t\t\t\tLogString msg(LOG4CXX_STR(\"Unexpected targetType (\"));\n\t\t\t\thelpers::StringHelper::toString(targetType, p, msg);\n\t\t\t\tmsg += LOG4CXX_STR(\") at parameter \");\n\t\t\t\thelpers::StringHelper::toString(parameterNumber, p, msg);\n\t\t\t\tmsg += LOG4CXX_STR(\" while preparing SQL\");\n\t\t\t\tLogLog::warn(msg);\n\t\t\t}\n\t\t\titem.paramMaxCharCount = 30;\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\titem.paramType = SQL_C_CHAR;\n\t\t\titem.paramValueSize = (SQLINTEGER)(item.paramMaxCharCount) * sizeof(char);\n\t\t\titem.paramValue = (SQLPOINTER)p.palloc(item.paramValueSize + sizeof(char));\n#else\n\t\t\titem.paramType = SQL_C_WCHAR;\n\t\t\titem.paramValueSize = (SQLINTEGER)(item.paramMaxCharCount) * sizeof(wchar_t);\n\t\t\titem.paramValue = (SQLPOINTER)p.palloc(item.paramValueSize + sizeof(wchar_t));\n#endif\n\t\t}\n\t\titem.strLen_or_Ind = SQL_NTS;\n\t\tret = SQLBindParameter\n\t\t\t( this->preparedStatement\n\t\t\t, parameterNumber\n\t\t\t, SQL_PARAM_INPUT\n\t\t\t, item.paramType  // ValueType\n\t\t\t, targetType\n\t\t\t, targetMaxCharCount\n\t\t\t, decimalDigits\n\t\t\t, item.paramValue\n\t\t\t, item.paramValueSize\n\t\t\t, &item.strLen_or_Ind\n\t\t\t);\n\t\tif (ret < 0)\n\t\t{\n\t\t\tthrow SQLException(SQL_HANDLE_STMT, this->preparedStatement, \"Failed to bind parameter\", p);\n\t\t}\n\t}\n}\n\nvoid ODBCAppender::ODBCAppenderPriv::setParameterValues(const spi::LoggingEventPtr& event, Pool& p)\n{\n\tfor (auto& item : this->parameterValue)\n\t{\n\t\tif (!item.paramValue || item.paramValueSize <= 0)\n\t\t\t;\n\t\telse if (SQL_C_WCHAR == item.paramType)\n\t\t{\n\t\t\tLogString sbuf;\n\t\t\titem.converter->format(event, sbuf, p);\n#if LOG4CXX_LOGCHAR_IS_WCHAR_T\n\t\t\tstd::wstring& tmp = sbuf;\n#else\n\t\t\tstd::wstring tmp;\n\t\t\tTranscoder::encode(sbuf, tmp);\n#endif\n\t\t\tauto dst = (wchar_t*)item.paramValue;\n\t\t\tauto charCount = std::min(size_t(item.paramMaxCharCount), tmp.size());\n\t\t\tauto copySize = std::min(size_t(item.paramValueSize - 1), charCount * sizeof(wchar_t));\n\t\t\tstd::memcpy(dst, tmp.data(), copySize);\n\t\t\tdst[copySize / sizeof(wchar_t)] = 0;\n\t\t}\n\t\telse if (SQL_C_CHAR == item.paramType)\n\t\t{\n\t\t\tLogString sbuf;\n\t\t\titem.converter->format(event, sbuf, p);\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\tstd::string& tmp = sbuf;\n#else\n\t\t\tstd::string tmp;\n\t\t\tTranscoder::encode(sbuf, tmp);\n#endif\n\t\t\tauto dst = (char*)item.paramValue;\n\t\t\tauto sz = std::min(size_t(item.paramMaxCharCount), tmp.size());\n\t\t\tauto copySize = std::min(size_t(item.paramValueSize - 1), sz * sizeof(char));\n\t\t\tstd::memcpy(dst, tmp.data(), copySize);\n\t\t\tdst[copySize] = 0;\n\t\t}\n\t\telse if (SQL_C_TYPE_TIMESTAMP == item.paramType)\n\t\t{\n\t\t\tapr_time_exp_t exploded;\n\t\t\tapr_status_t stat = this->timeZone->explode(&exploded, event->getTimeStamp());\n\t\t\tif (stat == APR_SUCCESS)\n\t\t\t{\n\t\t\t\tauto dst = (SQL_TIMESTAMP_STRUCT*)item.paramValue;\n\t\t\t\tdst->year = 1900 + exploded.tm_year;\n\t\t\t\tdst->month = 1 + exploded.tm_mon;\n\t\t\t\tdst->day = exploded.tm_mday;\n\t\t\t\tdst->hour = exploded.tm_hour;\n\t\t\t\tdst->minute = exploded.tm_min;\n\t\t\t\tdst->second = exploded.tm_sec;\n\t\t\t\t// Prevent '[ODBC SQL Server Driver]Datetime field overflow' by rounding to the target field precision\n\t\t\t\tint roundingExponent = 6 - (int)item.paramMaxCharCount;\n\t\t\t\tif (0 < roundingExponent)\n\t\t\t\t{\n\t\t\t\t\tint roundingDivisor = (int)std::pow(10, roundingExponent);\n\t\t\t\t\tdst->fraction = 1000 * roundingDivisor * ((exploded.tm_usec + roundingDivisor / 2) / roundingDivisor);\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t\tdst->fraction = 1000 * exploded.tm_usec;\n\t\t\t}\n\t\t}\n\t}\n}\n#endif\n\nvoid ODBCAppender::flushBuffer(Pool& p)\n{\n\tfor (auto& logEvent : _priv->buffer)\n\t{\n\t\tif (_priv->parameterValue.empty())\n\t\t\t_priv->errorHandler->error(LOG4CXX_STR(\"ODBCAppender column mappings not defined\"));\n#if LOG4CXX_HAVE_ODBC\n\t\telse try\n\t\t{\n\t\t\tif (0 == _priv->preparedStatement)\n\t\t\t\t_priv->setPreparedStatement(getConnection(p), p);\n\t\t\t_priv->setParameterValues(logEvent, p);\n\t\t\tauto ret = SQLExecute(_priv->preparedStatement);\n\t\t\tif (ret < 0)\n\t\t\t{\n\t\t\t\tthrow SQLException(SQL_HANDLE_STMT, _priv->preparedStatement, \"Failed to execute prepared statement\", p);\n\t\t\t}\n\t\t}\n\t\t\ncatch (SQLException& e)\n\t\t{\n\t\t\t_priv->errorHandler->error(LOG4CXX_STR(\"Failed to execute sql\"), e,\n\t\t\t\tErrorCode::FLUSH_FAILURE);\n\t\t}\n#endif\n\t}\n\n\t// clear the buffer of reported events\n\t_priv->buffer.clear();\n}\n\nvoid ODBCAppender::setSql(const LogString& s)\n{\n    _priv->sqlStatement = s;\n}\n\n#if LOG4CXX_WCHAR_T_API || LOG4CXX_LOGCHAR_IS_WCHAR_T || defined(WIN32) || defined(_WIN32)\nvoid ODBCAppender::encode(wchar_t** dest, const LogString& src, Pool& p)\n{\n\t*dest = Transcoder::wencode(src, p);\n}\n#endif\n\nvoid ODBCAppender::encode(unsigned short** dest,\n\tconst LogString& src, Pool& p)\n{\n\t//  worst case double number of characters from UTF-8 or wchar_t\n\t*dest = (unsigned short*)\n\t\tp.palloc((src.size() + 1) * 2 * sizeof(unsigned short));\n\tunsigned short* current = *dest;\n\n\tfor (LogString::const_iterator i = src.begin();\n\t\ti != src.end();)\n\t{\n\t\tunsigned int sv = Transcoder::decode(src, i);\n\n\t\tif (sv < 0x10000)\n\t\t{\n\t\t\t*current++ = (unsigned short) sv;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tunsigned char u = (unsigned char) (sv >> 16);\n\t\t\tunsigned char w = (unsigned char) (u - 1);\n\t\t\tunsigned short hs = (0xD800 + ((w & 0xF) << 6) + ((sv & 0xFFFF) >> 10));\n\t\t\tunsigned short ls = (0xDC00 + (sv & 0x3FF));\n\t\t\t*current++ = (unsigned short) hs;\n\t\t\t*current++ = (unsigned short) ls;\n\t\t}\n\t}\n\n\t*current = 0;\n}\n\nconst LogString& ODBCAppender::getSql() const\n{\n\treturn _priv->sqlStatement;\n}\n\nvoid ODBCAppender::setUser(const LogString& user)\n{\n\t_priv->databaseUser = user;\n}\n\nvoid ODBCAppender::setURL(const LogString& url)\n{\n\t_priv->databaseURL = url;\n}\n\nvoid ODBCAppender::setPassword(const LogString& password)\n{\n\t_priv->databasePassword = password;\n}\n\nvoid ODBCAppender::setBufferSize(size_t newBufferSize)\n{\n\t_priv->bufferSize = newBufferSize;\n}\n\nconst LogString& ODBCAppender::getUser() const\n{\n\treturn _priv->databaseUser;\n}\n\nconst LogString& ODBCAppender::getURL() const\n{\n\treturn _priv->databaseURL;\n}\n\nconst LogString& ODBCAppender::getPassword() const\n{\n\treturn _priv->databasePassword;\n}\n\nsize_t ODBCAppender::getBufferSize() const\n{\n\treturn _priv->bufferSize;\n}\n\n\n// Path: src/main/cpp/stringtokenizer.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/stringtokenizer.h>\n#include <log4cxx/helpers/exception.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct StringTokenizer::StringTokenizerPrivate{\n\tStringTokenizerPrivate(const LogString& str, const LogString& delim1) : src(str), delim(delim1), pos(0){}\n\tLogString src;\n\tLogString delim;\n\tsize_t pos;\n};\n\n\nStringTokenizer::StringTokenizer(const LogString& str, const LogString& delim1)\n\t: m_priv(std::make_unique<StringTokenizerPrivate>(str, delim1))\n{\n}\n\nStringTokenizer::~StringTokenizer()\n{\n}\n\nbool StringTokenizer::hasMoreTokens() const\n{\n\treturn (m_priv->pos != LogString::npos\n\t\t\t&& m_priv->src.find_first_not_of(m_priv->delim, m_priv->pos) != LogString::npos);\n}\n\nLogString StringTokenizer::nextToken()\n{\n\tif (m_priv->pos != LogString::npos)\n\t{\n\t\tsize_t nextPos = m_priv->src.find_first_not_of(m_priv->delim, m_priv->pos);\n\n\t\tif (nextPos != LogString::npos)\n\t\t{\n\t\t\tm_priv->pos = m_priv->src.find_first_of(m_priv->delim, nextPos);\n\n\t\t\tif (m_priv->pos == LogString::npos)\n\t\t\t{\n\t\t\t\treturn m_priv->src.substr(nextPos);\n\t\t\t}\n\n\t\t\treturn m_priv->src.substr(nextPos, m_priv->pos - nextPos);\n\t\t}\n\t}\n\n\tthrow NoSuchElementException();\n#if LOG4CXX_RETURN_AFTER_THROW\n\treturn LogString();\n#endif\n}\n\n// Path: src/main/cpp/resourcebundle.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/resourcebundle.h>\n#include <log4cxx/helpers/propertyresourcebundle.h>\n#include <log4cxx/helpers/loader.h>\n#include <log4cxx/helpers/pool.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/locale.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(ResourceBundle)\n\nResourceBundlePtr ResourceBundle::getBundle(const LogString& baseName,\n\tconst Locale& locale)\n{\n\tstd::vector<LogString> bundlesNames;\n\n\tif (!locale.getVariant().empty())\n\t{\n\t\tbundlesNames.push_back(baseName + LOG4CXX_STR(\"_\") +\n\t\t\tlocale.getLanguage() + LOG4CXX_STR(\"_\") +\n\t\t\tlocale.getCountry() + LOG4CXX_STR(\"_\") +\n\t\t\tlocale.getVariant());\n\t}\n\n\tif (!locale.getCountry().empty())\n\t{\n\t\tbundlesNames.push_back(baseName + LOG4CXX_STR(\"_\") +\n\t\t\tlocale.getLanguage() + LOG4CXX_STR(\"_\") +\n\t\t\tlocale.getCountry());\n\t}\n\n\tif (!locale.getLanguage().empty())\n\t{\n\t\tbundlesNames.push_back(baseName + LOG4CXX_STR(\"_\") +\n\t\t\tlocale.getLanguage());\n\t}\n\n\tbundlesNames.push_back(baseName);\n\n\tPropertyResourceBundlePtr resourceBundle, previous;\n\tfor (auto bundleName : bundlesNames)\n\t{\n\t\tPropertyResourceBundlePtr current;\n\n\t\t// Try loading a class which implements ResourceBundle\n\t\ttry\n\t\t{\n\t\t\tconst Class& classObj = Loader::loadClass(bundleName);\n\t\t\tObjectPtr obj = ObjectPtr(classObj.newInstance());\n\t\t\tcurrent = LOG4CXX_NS::cast<PropertyResourceBundle>(obj);\n\t\t}\n\t\tcatch (ClassNotFoundException&)\n\t\t{\n\t\t\tcurrent.reset();\n\t\t}\n\n\t\t// No class found, then try to create a PropertyResourceBundle from a file\n\t\tif (!current)\n\t\t{\n\t\t\tInputStreamPtr bundleStream =\n\t\t\t\tLoader::getResourceAsStream(\n\t\t\t\t\tbundleName + LOG4CXX_STR(\".properties\"));\n\n\t\t\tif (!bundleStream)\n\t\t\t{\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\ttry\n\t\t\t{\n\t\t\t\tcurrent = std::make_shared<PropertyResourceBundle>(bundleStream);\n\t\t\t}\n\t\t\tcatch (Exception&)\n\t\t\t{\n\t\t\t\tthrow;\n\t\t\t}\n\t\t}\n\n\t\t// Add the new resource bundle to the hierarchy\n\t\tif (!resourceBundle)\n\t\t{\n\t\t\tresourceBundle = current;\n\t\t\tprevious = current;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tprevious->setParent(current);\n\t\t\tprevious = current;\n\t\t}\n\t}\n\n\t// no resource bundle found at all, then throw exception\n\tif (!resourceBundle)\n\t{\n\t\tthrow MissingResourceException(\n\t\t\t((LogString) LOG4CXX_STR(\"Missing resource bundle \")) + baseName);\n\t}\n\n\treturn resourceBundle;\n}\n\n\n// Path: src/main/cpp/outputstreamwriter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/outputstreamwriter.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/stringhelper.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(OutputStreamWriter)\n\nstruct OutputStreamWriter::OutputStreamWriterPrivate{\n\tOutputStreamWriterPrivate(OutputStreamPtr& out1) : out(out1), enc(CharsetEncoder::getDefaultEncoder()){}\n\n\tOutputStreamWriterPrivate(OutputStreamPtr& out1,\n\t\t\t\t\t\t\t  CharsetEncoderPtr& enc1)\n\t\t: out(out1), enc(enc1){}\n\n\tOutputStreamPtr out;\n\tCharsetEncoderPtr enc;\n};\n\nOutputStreamWriter::OutputStreamWriter(OutputStreamPtr& out1)\n\t: m_priv(std::make_unique<OutputStreamWriterPrivate>(out1))\n{\n\tif (out1 == 0)\n\t{\n\t\tthrow NullPointerException(LOG4CXX_STR(\"out parameter may not be null.\"));\n\t}\n}\n\nOutputStreamWriter::OutputStreamWriter(OutputStreamPtr& out1,\n\tCharsetEncoderPtr& enc1)\n\t: m_priv(std::make_unique<OutputStreamWriterPrivate>(out1, enc1))\n{\n\tif (out1 == 0)\n\t{\n\t\tthrow NullPointerException(LOG4CXX_STR(\"out parameter may not be null.\"));\n\t}\n\n\tif (enc1 == 0)\n\t{\n\t\tthrow NullPointerException(LOG4CXX_STR(\"enc parameter may not be null.\"));\n\t}\n}\n\nOutputStreamWriter::~OutputStreamWriter()\n{\n}\n\nvoid OutputStreamWriter::close(Pool& p)\n{\n\tm_priv->out->close(p);\n}\n\nvoid OutputStreamWriter::flush(Pool& p)\n{\n\tm_priv->out->flush(p);\n}\n\nvoid OutputStreamWriter::write(const LogString& str, Pool& p)\n{\n\tif (str.empty())\n\t\treturn;\n\tif (CharsetEncoder::isTriviallyCopyable(str, m_priv->enc))\n\t{\n\t\tByteBuffer buf((char*)str.data(), str.size() * sizeof (logchar));\n\t\tm_priv->out->write(buf, p);\n\t}\n\telse\n\t{\n\t\tenum { BUFSIZE = 1024 };\n\t\tchar stackData[BUFSIZE];\n\t\tchar* rawbuf = stackData;\n\t\tsize_t bufSize = BUFSIZE;\n#ifdef LOG4CXX_MULTI_PROCESS\n\t\tstd::vector<char> heapData;\n\t\t// Ensure the logging event is a single write system call to keep events from each process separate\n\t\tif (bufSize < str.length() * 2)\n\t\t{\n\t\t\theapData.resize(bufSize = str.length() * 2);\n\t\t\trawbuf = heapData.data();\n\t\t}\n#endif\n\t\tByteBuffer buf(rawbuf, bufSize);\n\t\tm_priv->enc->reset();\n\t\tLogString::const_iterator iter = str.begin();\n\n\t\twhile (iter != str.end())\n\t\t{\n\t\t\tCharsetEncoder::encode(m_priv->enc, str, iter, buf);\n\t\t\tbuf.flip();\n\t\t\tm_priv->out->write(buf, p);\n\t\t\tbuf.clear();\n\t\t}\n\n\t\tCharsetEncoder::encode(m_priv->enc, str, iter, buf);\n\t\tm_priv->enc->flush(buf);\n\t\tbuf.flip();\n\t\tm_priv->out->write(buf, p);\n\t}\n}\n\nOutputStreamPtr OutputStreamWriter::getOutputStreamPtr() const\n{\n\treturn m_priv->out;\n}\n\n\n// Path: src/main/cpp/transcoder.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/pool.h>\n#include <stdlib.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/charsetdecoder.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/loglog.h>\n#include <vector>\n#include <cstring>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n\n#if LOG4CXX_CFSTRING_API\n\t#include <CoreFoundation/CFString.h>\n#endif\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\n\nvoid Transcoder::decodeUTF8(const std::string& src, LogString& dst)\n{\n\tstd::string::const_iterator iter = src.begin();\n\n\twhile (iter != src.end())\n\t{\n\t\tunsigned int sv = decode(src, iter);\n\n\t\tif (sv != 0xFFFF)\n\t\t{\n\t\t\tencode(sv, dst);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdst.append(1, LOSSCHAR);\n\t\t\titer++;\n\t\t}\n\t}\n}\n\nvoid Transcoder::encodeUTF8(const LogString& src, std::string& dst)\n{\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\tdst.append(src);\n#else\n\tLogString::const_iterator iter = src.begin();\n\n\twhile (iter != src.end())\n\t{\n\t\tunsigned int sv = decode(src, iter);\n\n\t\tif (sv != 0xFFFF)\n\t\t{\n\t\t\tencode(sv, dst);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tdst.append(1, LOSSCHAR);\n\t\t\titer++;\n\t\t}\n\t}\n\n#endif\n}\n\nchar* Transcoder::encodeUTF8(const LogString& src, Pool& p)\n{\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\treturn p.pstrdup(src);\n#else\n\tstd::string tmp;\n\tencodeUTF8(src, tmp);\n\treturn p.pstrdup(tmp);\n#endif\n}\n\n\nvoid Transcoder::encodeUTF8(unsigned int sv, ByteBuffer& dst)\n{\n\tsize_t bytes = encodeUTF8(sv, dst.current());\n\tdst.position(dst.position() + bytes);\n}\n\n\nsize_t Transcoder::encodeUTF8(unsigned int ch, char* dst)\n{\n\tif (ch < 0x80)\n\t{\n\t\tdst[0] = (char) ch;\n\t\treturn 1;\n\t}\n\telse if (ch < 0x800)\n\t{\n\t\tdst[0] = (char) (0xC0 + (ch >> 6));\n\t\tdst[1] = (char) (0x80 + (ch & 0x3F));\n\t\treturn 2;\n\t}\n\telse if (ch < 0x10000)\n\t{\n\t\tdst[0] = (char) (0xE0 + (ch >> 12));\n\t\tdst[1] = (char) (0x80 + ((ch >> 6) & 0x3F));\n\t\tdst[2] = (char) (0x80 + (ch & 0x3F));\n\t\treturn 3;\n\t}\n\telse if (ch <= 0x10FFFF)\n\t{\n\t\tdst[0] = (char) (0xF0 + (ch >> 18));\n\t\tdst[1] = (char) (0x80 + ((ch >> 12) & 0x3F));\n\t\tdst[2] = (char) (0x80 + ((ch >> 6) & 0x3F));\n\t\tdst[3] = (char) (0x80 + (ch & 0x3F));\n\t\treturn 4;\n\t}\n\telse\n\t{\n\t\t//\n\t\t//  output UTF-8 encoding of 0xFFFF\n\t\t//\n\t\tdst[0] = (char) 0xEF;\n\t\tdst[1] = (char) 0xBF;\n\t\tdst[2] = (char) 0xBF;\n\t\treturn 3;\n\t}\n}\n\nvoid Transcoder::encodeUTF16BE(unsigned int sv, ByteBuffer& dst)\n{\n\tsize_t bytes = encodeUTF16BE(sv, dst.current());\n\tdst.position(dst.position() + bytes);\n}\n\n\nsize_t Transcoder::encodeUTF16BE(unsigned int ch, char* dst)\n{\n\tif (ch <= 0xFFFF)\n\t{\n\t\tdst[0] = (char) (ch >> 8);\n\t\tdst[1] = (char) (ch & 0xFF);\n\t\treturn 2;\n\t}\n\n\tif (ch <= 0x10FFFF)\n\t{\n\t\tunsigned char w = (unsigned char) ((ch >> 16) - 1);\n\t\tdst[0] = (char) (0xD8 + (w >> 2));\n\t\tdst[1] = (char) (((w & 0x03) << 6) + ((ch >> 10) & 0x3F));\n\t\tdst[2] = (char) (0xDC + ((ch & 0x30) >> 4));\n\t\tdst[3] = (char) (ch & 0xFF);\n\t\treturn 4;\n\t}\n\n\tdst[0] = dst[1] = (char) 0xFF;\n\treturn 2;\n}\n\nvoid Transcoder::encodeUTF16LE(unsigned int sv, ByteBuffer& dst)\n{\n\tsize_t bytes = encodeUTF16LE(sv, dst.current());\n\tdst.position(dst.position() + bytes);\n}\n\nsize_t Transcoder::encodeUTF16LE(unsigned int ch, char* dst)\n{\n\tif (ch <= 0xFFFF)\n\t{\n\t\tdst[1] = (char) (ch >> 8);\n\t\tdst[0] = (char) (ch & 0xFF);\n\t\treturn 2;\n\t}\n\n\tif (ch <= 0x10FFFF)\n\t{\n\t\tunsigned char w = (unsigned char) ((ch >> 16) - 1);\n\t\tdst[1] = (char) (0xD8 + (w >> 2));\n\t\tdst[0] = (char) (((w & 0x03) << 6) + ((ch >> 10) & 0x3F));\n\t\tdst[3] = (char) (0xDC + ((ch & 0x30) >> 4));\n\t\tdst[2] = (char) (ch & 0xFF);\n\t\treturn 4;\n\t}\n\n\tdst[0] = dst[1] = (char) 0xFF;\n\treturn 2;\n}\n\n\nunsigned int Transcoder::decode(const std::string& src,\n\tstd::string::const_iterator& iter)\n{\n\tstd::string::const_iterator start(iter);\n\tunsigned char ch1 = *(iter++);\n\n\tif (ch1 <= 0x7F)\n\t{\n\t\treturn ch1;\n\t}\n\n\t//\n\t//   should not have continuation character here\n\t//\n\tif ((ch1 & 0xC0) != 0x80 && iter != src.end())\n\t{\n\t\tunsigned char ch2 = *(iter++);\n\n\t\t//\n\t\t//   should be continuation\n\t\tif ((ch2 & 0xC0) != 0x80)\n\t\t{\n\t\t\titer = start;\n\t\t\treturn 0xFFFF;\n\t\t}\n\n\t\tif ((ch1 & 0xE0) == 0xC0)\n\t\t{\n\t\t\tif ((ch2 & 0xC0) == 0x80)\n\t\t\t{\n\t\t\t\tunsigned int rv = ((ch1 & 0x1F) << 6) + (ch2 & 0x3F);\n\n\t\t\t\tif (rv >= 0x80)\n\t\t\t\t{\n\t\t\t\t\treturn rv;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\titer = start;\n\t\t\treturn 0xFFFF;\n\t\t}\n\n\t\tif (iter != src.end())\n\t\t{\n\t\t\tunsigned char ch3 = *(iter++);\n\n\t\t\t//\n\t\t\t//   should be continuation\n\t\t\t//\n\t\t\tif ((ch3 & 0xC0) != 0x80)\n\t\t\t{\n\t\t\t\titer = start;\n\t\t\t\treturn 0xFFFF;\n\t\t\t}\n\n\t\t\tif ((ch1 & 0xF0) == 0xE0)\n\t\t\t{\n\t\t\t\tunsigned rv = ((ch1 & 0x0F) << 12)\n\t\t\t\t\t+ ((ch2 & 0x3F) << 6)\n\t\t\t\t\t+ (ch3 & 0x3F);\n\n\t\t\t\tif (rv <= 0x800)\n\t\t\t\t{\n\t\t\t\t\titer = start;\n\t\t\t\t\treturn 0xFFFF;\n\t\t\t\t}\n\n\t\t\t\treturn rv;\n\t\t\t}\n\n\t\t\tif (iter != src.end())\n\t\t\t{\n\t\t\t\tunsigned char ch4 = *(iter++);\n\n\t\t\t\tif ((ch4 & 0xC0) != 0x80)\n\t\t\t\t{\n\t\t\t\t\titer = start;\n\t\t\t\t\treturn 0xFFFF;\n\t\t\t\t}\n\n\t\t\t\tunsigned int rv = ((ch1 & 0x07) << 18)\n\t\t\t\t\t+ ((ch2 & 0x3F) << 12)\n\t\t\t\t\t+ ((ch3 & 0x3F) << 6)\n\t\t\t\t\t+ (ch4 & 0x3F);\n\n\t\t\t\tif (rv > 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn rv;\n\t\t\t\t}\n\n\t\t\t}\n\t\t}\n\t}\n\n\titer = start;\n\treturn 0xFFFF;\n}\n\n\nvoid Transcoder::encode(unsigned int sv, std::string& dst)\n{\n\tchar tmp[8];\n\tsize_t bytes = encodeUTF8(sv, tmp);\n\tdst.append(tmp, bytes);\n}\n\n\nvoid Transcoder::decode(const std::string& src, LogString& dst)\n{\n#if LOG4CXX_CHARSET_UTF8 && LOG4CXX_LOGCHAR_IS_UTF8\n\tdst.append(src);\n#else\n\tstatic CharsetDecoderPtr decoder(CharsetDecoder::getDefaultDecoder());\n\tdst.reserve(dst.size() + src.size());\n\tstd::string::const_iterator iter = src.begin();\n#if !LOG4CXX_CHARSET_EBCDIC\n\n\tfor (;\n\t\titer != src.end() && ((unsigned char) *iter) < 0x80;\n\t\titer++)\n\t{\n\t\tdst.append(1, *iter);\n\t}\n\n#endif\n\n\tif (iter != src.end())\n\t{\n\t\tsize_t offset = iter - src.begin();\n\t\tByteBuffer buf(const_cast<char*>(src.data() + offset), src.size() - offset);\n\n\t\twhile (buf.remaining() > 0)\n\t\t{\n\t\t\tlog4cxx_status_t stat = decoder->decode(buf, dst);\n\n\t\t\tif (CharsetDecoder::isError(stat))\n\t\t\t{\n\t\t\t\tdst.append(1, LOSSCHAR);\n\t\t\t\tbuf.position(buf.position() + 1);\n\t\t\t}\n\t\t}\n\n\t\tdecoder->decode(buf, dst);\n\t}\n\n#endif\n}\n\nchar* Transcoder::encode(const LogString& src, Pool& p)\n{\n#if LOG4CXX_CHARSET_UTF8 && LOG4CXX_LOGCHAR_IS_UTF8\n\treturn p.pstrdup(src);\n#else\n\tstd::string tmp;\n\tencode(src, tmp);\n\treturn p.pstrdup(tmp);\n#endif\n}\n\n\n\nvoid Transcoder::encode(const LogString& src, std::string& dst)\n{\n#if LOG4CXX_CHARSET_UTF8 && LOG4CXX_LOGCHAR_IS_UTF8\n\tdst.append(src);\n#else\n\tstatic CharsetEncoderPtr encoder(CharsetEncoder::getDefaultEncoder());\n\tdst.reserve(dst.size() + src.size());\n\tLogString::const_iterator iter = src.begin();\n#if !LOG4CXX_CHARSET_EBCDIC\n\n\tfor (;\n\t\titer != src.end() && ((unsigned int) *iter) < 0x80;\n\t\titer++)\n\t{\n\t\tdst.append(1, *iter);\n\t}\n\n#endif\n\n\tif (iter != src.end())\n\t{\n\t\tchar buf[BUFSIZE];\n\t\tByteBuffer out(buf, BUFSIZE);\n\n\t\twhile (iter != src.end())\n\t\t{\n\t\t\tlog4cxx_status_t stat = encoder->encode(src, iter, out);\n\t\t\tout.flip();\n\t\t\tdst.append(out.data(), out.limit());\n\t\t\tout.clear();\n\n\t\t\tif (CharsetEncoder::isError(stat))\n\t\t\t{\n\t\t\t\tdst.append(1, LOSSCHAR);\n\t\t\t\titer++;\n\t\t\t}\n\t\t}\n\n\t\tencoder->encode(src, iter, out);\n\t}\n\n#endif\n}\n\n\ntemplate<class String, class Iterator>\nstatic unsigned int decodeUTF16(const String& in, Iterator& iter)\n{\n\tunsigned int ch1 = *iter;\n\n\t//\n\t//   if not surrogate pair\n\t//\n\tif (ch1 < 0xD800 || ch1 > 0xDFFF)\n\t{\n\t\t//\n\t\t//  then advance iterator and return wchar_t value\n\t\t//\n\t\tif (ch1 != 0xFFFF)\n\t\t{\n\t\t\titer++;\n\t\t}\n\n\t\treturn ch1;\n\t}\n\telse if (ch1 < 0xDC00)\n\t{\n\t\t//\n\t\t//  started with high-surrogate value\n\t\t//     if there is an additional wchar_t\n\t\tIterator iter2 = iter + 1;\n\n\t\tif (iter2 != in.end())\n\t\t{\n\t\t\tunsigned int ch2 = *iter2;\n\n\t\t\t//\n\t\t\t//    if it is a matching low surrogate then\n\t\t\t//       advance the iterator and return the scalar value\n\t\t\tif (ch2 >= 0xDC00 && ch2 <= 0xDFFF)\n\t\t\t{\n\t\t\t\titer += 2;\n\t\t\t\treturn (ch1 - 0xD800) * 0x400 + (ch2 - 0xDC00) + 0x10000;\n\t\t\t}\n\t\t}\n\t}\n\n\t//\n\t//    unrecognized value, do not advance iterator\n\t//\n\treturn 0xFFFF;\n}\n\ntemplate<class String>\nstatic void encodeUTF16(unsigned int sv, String& dst)\n{\n\tif (sv < 0x10000)\n\t{\n\t\tdst.append(1, sv);\n\t}\n\telse\n\t{\n\t\tunsigned char u = (unsigned char) (sv >> 16);\n\t\tunsigned char w = (unsigned char) (u - 1);\n\t\tunsigned short hs = (0xD800 + ((w & 0xF) << 6) + ((sv & 0xFFFF) >> 10));\n...\n", "needle_token_start": 6881, "needle_token_end": 6938, "code_context_ntokens": 15209, "output": ["```cpp\nvoid ODBCAppender::execute(const LogString& sql, LOG4CXX_NS::helpers::Pool& p)\n{\n#if LOG4CXX_HAVE_ODBC\n\tSQLRETURN ret;\n\tSQLHSTMT stmt;\n\n\tret = SQLAllocHandle(SQL_HANDLE_STMT, getConnection(p), &stmt);\n\n\tif (ret < 0)\n\t{\n\t\tSQLException ex(SQL_HANDLE_STMT, stmt, \"Failed to allocate statement handle.\", p);\n\t\tSQLFreeHandle(SQL_HANDLE_STMT, stmt);\n\t\tthrow ex;\n\t}\n\n\tret = SQLPrepareA(stmt, (SQLCHAR*)sql.c_str(), SQL_NTS);\n\n\tif (ret < 0)\n\t{\n\t\tSQLException ex(SQL_HANDLE_STMT, stmt, \"Failed to prepare SQL statement.\", p);\n\t\tSQLFreeHandle(SQL_HANDLE_STMT, stmt);\n\t\tthrow ex;\n\t}\n\n\tret = SQLExecute(stmt);\n\n\tif (ret < 0)\n\t{\n\t\tSQLException ex(SQL_HANDLE_STMT, stmt, \"Failed to execute prepared statement\", p);\n\t\tSQLFreeHandle(SQL_HANDLE_STMT, stmt);\n\t\tthrow ex;\n\t}\n\n\tSQLFreeHandle(SQL_HANDLE_STMT, stmt);\n#endif\n}\n```"]}
{"repo": "apache/logging-log4cxx", "name": "StrftimeDateFormatPrivate", "language": "cpp", "path": "src/main/cpp/strftimedateformat.cpp", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this component is to manage and format date and time strings according to specified time zones, primarily used in logging contexts.\n2. **Input**: The input typically includes a time zone identifier and a date-time value that needs formatting.\n3. **Output**: The output is a formatted date-time string adjusted according to the specified time zone.\n4. **Procedure**: The procedure involves initializing with a default time zone, parsing the time zone identifier if provided, calculating the time offset based on the parsed hours and minutes, and then formatting the date-time string according to the calculated time zone offset.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/xmllayout.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/xml/xmllayout.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/transform.h>\n#include <log4cxx/helpers/iso8601dateformat.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/ndc.h>\n\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::xml;\n\nstruct XMLLayout::XMLLayoutPrivate\n{\n\tXMLLayoutPrivate()\n\t\t: locationInfo(false)\n\t\t, properties(false)\n\t\t, expectedPatternLength(100)\n\t\t{}\n\n\t// Print no location info by default\n\tbool locationInfo; //= false\n\tbool properties; // = false\n\n\t// Expected length of a formatted event excluding the message text\n\tsize_t expectedPatternLength;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(XMLLayout)\n\nXMLLayout::XMLLayout()\n\t: m_priv(std::make_unique<XMLLayoutPrivate>())\n{\n\tm_priv->expectedPatternLength = getFormattedEventCharacterCount() * 2;\n}\n\nXMLLayout::~XMLLayout() {}\n\nvoid XMLLayout::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"LOCATIONINFO\"), LOG4CXX_STR(\"locationinfo\")))\n\t{\n\t\tsetLocationInfo(OptionConverter::toBoolean(value, false));\n\t\tm_priv->expectedPatternLength = getFormattedEventCharacterCount() * 2;\n\t}\n\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"PROPERTIES\"), LOG4CXX_STR(\"properties\")))\n\t{\n\t\tsetProperties(OptionConverter::toBoolean(value, false));\n\t\tm_priv->expectedPatternLength = getFormattedEventCharacterCount() * 2;\n\t}\n}\n\nvoid XMLLayout::format(LogString& output,\n\tconst spi::LoggingEventPtr& event,\n\tPool& p) const\n{\n\toutput.reserve(m_priv->expectedPatternLength + event->getMessage().size());\n\toutput.append(LOG4CXX_STR(\"<log4j:event logger=\\\"\"));\n\tTransform::appendEscapingTags(output, event->getLoggerName());\n\toutput.append(LOG4CXX_STR(\"\\\" timestamp=\\\"\"));\n\tStringHelper::toString(event->getTimeStamp() / 1000L, p, output);\n\toutput.append(LOG4CXX_STR(\"\\\" level=\\\"\"));\n\tTransform::appendEscapingTags(output, event->getLevel()->toString());\n\toutput.append(LOG4CXX_STR(\"\\\" thread=\\\"\"));\n\tTransform::appendEscapingTags(output, event->getThreadName());\n\toutput.append(LOG4CXX_STR(\"\\\">\"));\n\toutput.append(LOG4CXX_EOL);\n\n\toutput.append(LOG4CXX_STR(\"<log4j:message><![CDATA[\"));\n\t// Append the rendered message. Also make sure to escape any\n\t// existing CDATA sections.\n\tTransform::appendEscapingCDATA(output, event->getRenderedMessage());\n\toutput.append(LOG4CXX_STR(\"]]></log4j:message>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\tLogString ndc;\n\n\tif (event->getNDC(ndc))\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<log4j:NDC><![CDATA[\"));\n\t\tTransform::appendEscapingCDATA(output, ndc);\n\t\toutput.append(LOG4CXX_STR(\"]]></log4j:NDC>\"));\n\t\toutput.append(LOG4CXX_EOL);\n\t}\n\n\tif (m_priv->locationInfo)\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<log4j:locationInfo class=\\\"\"));\n\t\tconst LocationInfo& locInfo = event->getLocationInformation();\n\t\tLOG4CXX_DECODE_CHAR(className, locInfo.getClassName());\n\t\tTransform::appendEscapingTags(output, className);\n\t\toutput.append(LOG4CXX_STR(\"\\\" method=\\\"\"));\n\t\tLOG4CXX_DECODE_CHAR(method, locInfo.getMethodName());\n\t\tTransform::appendEscapingTags(output, method);\n\t\toutput.append(LOG4CXX_STR(\"\\\" file=\\\"\"));\n\t\tLOG4CXX_DECODE_CHAR(fileName, locInfo.getFileName());\n\t\tTransform::appendEscapingTags(output, fileName);\n\t\toutput.append(LOG4CXX_STR(\"\\\" line=\\\"\"));\n\t\tStringHelper::toString(locInfo.getLineNumber(), p, output);\n\t\toutput.append(LOG4CXX_STR(\"\\\"/>\"));\n\t\toutput.append(LOG4CXX_EOL);\n\t}\n\n\tif (m_priv->properties)\n\t{\n\t\tLoggingEvent::KeySet propertySet(event->getPropertyKeySet());\n\t\tLoggingEvent::KeySet keySet(event->getMDCKeySet());\n\n\t\tif (!(keySet.empty() && propertySet.empty()))\n\t\t{\n\t\t\toutput.append(LOG4CXX_STR(\"<log4j:properties>\"));\n\t\t\toutput.append(LOG4CXX_EOL);\n\n\t\t\tfor (auto key : keySet)\n\t\t\t{\n\t\t\t\tLogString value;\n\n\t\t\t\tif (event->getMDC(key, value))\n\t\t\t\t{\n\t\t\t\t\toutput.append(LOG4CXX_STR(\"<log4j:data name=\\\"\"));\n\t\t\t\t\tTransform::appendEscapingTags(output, key);\n\t\t\t\t\toutput.append(LOG4CXX_STR(\"\\\" value=\\\"\"));\n\t\t\t\t\tTransform::appendEscapingTags(output, value);\n\t\t\t\t\toutput.append(LOG4CXX_STR(\"\\\"/>\"));\n\t\t\t\t\toutput.append(LOG4CXX_EOL);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor (auto key : propertySet)\n\t\t\t{\n\t\t\t\tLogString value;\n\n\t\t\t\tif (event->getProperty(key, value))\n\t\t\t\t{\n\t\t\t\t\toutput.append(LOG4CXX_STR(\"<log4j:data name=\\\"\"));\n\t\t\t\t\tTransform::appendEscapingTags(output, key);\n\t\t\t\t\toutput.append(LOG4CXX_STR(\"\\\" value=\\\"\"));\n\t\t\t\t\tTransform::appendEscapingTags(output, value);\n\t\t\t\t\toutput.append(LOG4CXX_STR(\"\\\"/>\"));\n\t\t\t\t\toutput.append(LOG4CXX_EOL);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\toutput.append(LOG4CXX_STR(\"</log4j:properties>\"));\n\t\t\toutput.append(LOG4CXX_EOL);\n\t\t}\n...\n// Path: src/main/cpp/simplelayout.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/simplelayout.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/level.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::spi;\n\nIMPLEMENT_LOG4CXX_OBJECT(SimpleLayout)\n\n\n\nvoid SimpleLayout::format(LogString& output,\n\tconst spi::LoggingEventPtr& event,\n\tLOG4CXX_NS::helpers::Pool&) const\n{\n\toutput.append(event->getLevel()->toString());\n\toutput.append(LOG4CXX_STR(\" - \"));\n\toutput.append(event->getRenderedMessage());\n\toutput.append(LOG4CXX_EOL);\n}\n\n// Path: src/main/cpp/writerappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/writerappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/layout.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/private/appenderskeleton_priv.h>\n#include <log4cxx/private/writerappender_priv.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\n#define _priv static_cast<WriterAppenderPriv*>(m_priv.get())\n\nIMPLEMENT_LOG4CXX_OBJECT(WriterAppender)\n\nWriterAppender::WriterAppender() :\n\tAppenderSkeleton (std::make_unique<WriterAppenderPriv>())\n{\n}\n\nWriterAppender::WriterAppender(const LayoutPtr& layout1,\n\tLOG4CXX_NS::helpers::WriterPtr& writer1)\n\t: AppenderSkeleton (std::make_unique<WriterAppenderPriv>(layout1, writer1))\n{\n\tPool p;\n\tactivateOptions(p);\n}\n\nWriterAppender::WriterAppender(const LayoutPtr& layout1)\n\t: AppenderSkeleton (std::make_unique<WriterAppenderPriv>(layout1))\n{\n}\n\nWriterAppender::WriterAppender(std::unique_ptr<WriterAppenderPriv> priv)\n\t: AppenderSkeleton (std::move(priv))\n{\n\n}\n\nWriterAppender::~WriterAppender()\n{\n\tfinalize();\n}\n\nvoid WriterAppender::activateOptions(Pool& p)\n{\n\tint errors = 0;\n\n\tif (_priv->layout == 0)\n\t{\n\t\t_priv->errorHandler->error(\n\t\t\t((LogString) LOG4CXX_STR(\"No layout set for the appender named [\"))\n\t\t\t+ _priv->name + LOG4CXX_STR(\"].\"));\n\t\terrors++;\n\t}\n\n\tif (_priv->writer == 0)\n\t{\n\t\t_priv->errorHandler->error(\n\t\t\t((LogString) LOG4CXX_STR(\"No writer set for the appender named [\"))\n\t\t\t+ _priv->name + LOG4CXX_STR(\"].\"));\n\t\terrors++;\n\t}\n\n\tif (errors == 0)\n\t{\n\t\tAppenderSkeleton::activateOptions(p);\n\t}\n}\n\n\n\nvoid WriterAppender::append(const spi::LoggingEventPtr& event, Pool& pool1)\n{\n\n\tif (!checkEntryConditions())\n\t{\n\t\treturn;\n\t}\n\n\tsubAppend(event, pool1);\n}\n\n/**\n   This method determines if there is a sense in attempting to append.\n\n   <p>It checks whether there is a set output target and also if\n   there is a set layout. If these checks fail, then the boolean\n   value <code>false</code> is returned. */\nbool WriterAppender::checkEntryConditions() const\n{\n\tstatic bool warnedClosed = false;\n\tstatic bool warnedNoWriter = false;\n\tstatic bool warnedNoLayout = false;\n\n\tif (_priv->closed)\n\t{\n\t\tif (!warnedClosed)\n\t\t{\n\t\t\tLogLog::warn(LOG4CXX_STR(\"Not allowed to write to a closed appender.\"));\n\t\t\twarnedClosed = true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tif (_priv->writer == 0)\n\t{\n\t\tif (!warnedNoWriter)\n\t\t{\n\t\t\t_priv->errorHandler->error(\n\t\t\t\tLogString(LOG4CXX_STR(\"No output stream or file set for the appender named [\")) +\n\t\t\t\t_priv->name + LOG4CXX_STR(\"].\"));\n\t\t\twarnedNoWriter = true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tif (_priv->layout == 0)\n\t{\n\t\tif (!warnedNoLayout)\n\t\t{\n\t\t\t_priv->errorHandler->error(\n\t\t\t\tLogString(LOG4CXX_STR(\"No layout set for the appender named [\")) +\n\t\t\t\t_priv->name + LOG4CXX_STR(\"].\"));\n\t\t\twarnedNoLayout = true;\n\t\t}\n\t\treturn false;\n\t}\n\n\treturn true;\n}\n\n\n\n\n/**\n   Close this appender instance. The underlying stream or writer is\n   also closed.\n\n   <p>Closed appenders cannot be reused.\n\n   @see #setWriter\n   */\nvoid WriterAppender::close()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\tif (_priv->closed)\n\t{\n\t\treturn;\n\t}\n\n\t_priv->closed = true;\n\tcloseWriter();\n}\n\n/**\n * Close the underlying {@link java.io.Writer}.\n * */\nvoid WriterAppender::closeWriter()\n{\n\tif (_priv->writer != NULL)\n\t{\n\t\ttry\n\t\t{\n\t\t\t// before closing we have to output out layout's footer\n\t\t\t//\n\t\t\t//   Using the object's pool since this is a one-shot operation\n\t\t\t//    and pool is likely to be reclaimed soon when appender is destructed.\n\t\t\t//\n\t\t\twriteFooter(_priv->pool);\n\t\t\t_priv->writer->close(_priv->pool);\n\t\t\t_priv->writer = 0;\n\t\t}\n\t\tcatch (IOException& e)\n\t\t{\n\t\t\tLogLog::error(LogString(LOG4CXX_STR(\"Could not close writer for WriterAppender named \")) + _priv->name, e);\n\t\t}\n\t}\n\n}\n\n/**\n   Returns an OutputStreamWriter when passed an OutputStream.  The\n   encoding used will depend on the value of the\n   <code>encoding</code> property.  If the encoding value is\n   specified incorrectly the writer will be opened using the default\n   system encoding (an error message will be printed to the loglog.  */\nWriterPtr WriterAppender::createWriter(OutputStreamPtr& os)\n{\n\n\tLogString enc(getEncoding());\n\n\tCharsetEncoderPtr encoder;\n\n\tif (enc.empty())\n\t{\n\t\tencoder = CharsetEncoder::getDefaultEncoder();\n\t}\n\telse\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(enc,\n\t\t\t\tLOG4CXX_STR(\"utf-16\"), LOG4CXX_STR(\"UTF-16\")))\n\t\t{\n\t\t\tencoder = CharsetEncoder::getEncoder(LOG4CXX_STR(\"UTF-16BE\"));\n\t\t}\n\t\telse\n\t\t{\n\t\t\tencoder = CharsetEncoder::getEncoder(enc);\n\t\t}\n\n\t\tif (encoder == NULL)\n\t\t{\n\t\t\tencoder = CharsetEncoder::getDefaultEncoder();\n\t\t\tLogLog::warn(LOG4CXX_STR(\"Error initializing output writer.\"));\n\t\t\tLogLog::warn(LOG4CXX_STR(\"Unsupported encoding?\"));\n\t\t}\n\t}\n\n\treturn WriterPtr(new OutputStreamWriter(os, encoder));\n}\n\nLogString WriterAppender::getEncoding() const\n{\n\treturn _priv->encoding;\n}\n\nvoid WriterAppender::setEncoding(const LogString& enc)\n{\n\t_priv->encoding = enc;\n}\n\nvoid WriterAppender::subAppend(const spi::LoggingEventPtr& event, Pool& p)\n{\n\tLogString msg;\n\t_priv->layout->format(msg, event, p);\n\n\tif (_priv->writer != NULL)\n\t{\n\t\t_priv->writer->write(msg, p);\n\n\t\tif (_priv->immediateFlush)\n\t\t{\n\t\t\t_priv->writer->flush(p);\n\t\t}\n\t}\n}\n\n\nvoid WriterAppender::writeFooter(Pool& p)\n{\n\tif (_priv->layout != NULL)\n\t{\n\t\tLogString foot;\n\t\t_priv->layout->appendFooter(foot, p);\n\t\t_priv->writer->write(foot, p);\n\t}\n}\n\nvoid WriterAppender::writeHeader(Pool& p)\n{\n\tif (_priv->layout != NULL)\n\t{\n\t\tLogString header;\n\t\t_priv->layout->appendHeader(header, p);\n\t\t_priv->writer->write(header, p);\n\t}\n}\n\n\nvoid WriterAppender::setWriter(const WriterPtr& newWriter)\n{\n\tstd::unique_lock<std::recursive_mutex> lock(_priv->mutex);\n\tsetWriterInternal(newWriter);\n}\n\nvoid WriterAppender::setWriterInternal(const WriterPtr& newWriter)\n{\n\t_priv->writer = newWriter;\n}\n\nbool WriterAppender::requiresLayout() const\n{\n\treturn true;\n}\n\nvoid WriterAppender::setOption(const LogString& option, const LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"ENCODING\"), LOG4CXX_STR(\"encoding\")))\n\t{\n\t\tsetEncoding(value);\n\t}\n\telse\n\t{\n\t\tAppenderSkeleton::setOption(option, value);\n\t}\n}\n\n\nvoid WriterAppender::setImmediateFlush(bool value)\n{\n\t_priv->immediateFlush = value;\n}\n\nbool WriterAppender::getImmediateFlush() const\n{\n\treturn _priv->immediateFlush;\n}\n\nconst LOG4CXX_NS::helpers::WriterPtr WriterAppender::getWriter() const{\n\treturn _priv->writer;\n}\n\n// Path: src/main/cpp/timezone.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#define __STDC_CONSTANT_MACROS\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/timezone.h>\n#include <stdlib.h>\n\n#include <apr_time.h>\n#include <apr_pools.h>\n#include <apr_strings.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/pool.h>\n#include <log4cxx/logger.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT( TimeZone )\n\nnamespace LOG4CXX_NS\n{\nnamespace helpers\n{\nnamespace TimeZoneImpl\n{\n/** Time zone object that represents GMT. */\nclass GMTTimeZone : public TimeZone\n{\n\tpublic:\n\t\t/** Class factory. */\n\t\tstatic const TimeZonePtr& getInstance()\n\t\t{\n\t\t\tstatic WideLife<TimeZonePtr> tz = std::make_shared<GMTTimeZone>();\n\t\t\treturn tz;\n\t\t}\n\n\t\t/** Explode time to human readable form. */\n\t\tlog4cxx_status_t explode( apr_time_exp_t* result, log4cxx_time_t input ) const\n\t\t{\n\t\t\tapr_status_t stat;\n\n\t\t\t//  APR 1.1 and early mishandles microseconds on dates\n\t\t\t//   before 1970, APR bug 32520\n\t\t\tif (LOG4CXX_UNLIKELY(input < 0 && apr_time_usec(input) < 0))\n\t\t\t{\n\t\t\t\tapr_time_t floorTime = (apr_time_sec(input) - 1) * APR_USEC_PER_SEC;\n\t\t\t\tstat = apr_time_exp_gmt(result, floorTime);\n\t\t\t\tresult->tm_usec = (int) (input - floorTime);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tstat = apr_time_exp_gmt( result, input );\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\t\tGMTTimeZone() : TimeZone( LOG4CXX_STR(\"GMT\") )\n\t\t{\n\t\t}\n};\n\n\n\n/** Time zone object that represents GMT. */\nclass LocalTimeZone : public TimeZone\n{\n\tpublic:\n\t\t/** Class factory. */\n\t\tstatic const TimeZonePtr& getInstance()\n\t\t{\n\t\t\tstatic WideLife<TimeZonePtr> tz = std::make_shared<LocalTimeZone>();\n\t\t\treturn tz;\n\t\t}\n\n\t\t/** Explode time to human readable form. */\n\t\tlog4cxx_status_t explode( apr_time_exp_t* result, log4cxx_time_t input ) const\n\t\t{\n\t\t\tapr_status_t stat;\n\n\t\t\t//  APR 1.1 and early mishandles microseconds on dates\n\t\t\t//   before 1970, APR bug 32520\n\t\t\tif (LOG4CXX_UNLIKELY(input < 0 && apr_time_usec(input) < 0))\n\t\t\t{\n\t\t\t\tapr_time_t floorTime = (apr_time_sec(input) - 1) * APR_USEC_PER_SEC;\n\t\t\t\tstat = apr_time_exp_lt(result, floorTime);\n\t\t\t\tresult->tm_usec = (int) (input - floorTime);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tstat = apr_time_exp_lt( result, input );\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\t\tLocalTimeZone() : TimeZone( getTimeZoneName() )\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tstatic const LogString getTimeZoneName()\n\t\t{\n\t\t\tconst int MAX_TZ_LENGTH = 255;\n\t\t\tchar tzName[MAX_TZ_LENGTH];\n\t\t\tapr_size_t tzLength;\n\t\t\tapr_time_exp_t tm;\n\t\t\tapr_time_exp_lt(&tm, 0);\n\t\t\tapr_strftime(tzName, &tzLength, MAX_TZ_LENGTH, \"%Z\", &tm);\n\n\t\t\tif (tzLength == 0)\n\t\t\t{\n\t\t\t\tapr_strftime(tzName, &tzLength, MAX_TZ_LENGTH, \"%z\", &tm);\n\t\t\t}\n\n\t\t\ttzName[tzLength] = 0;\n\t\t\tLogString retval;\n\t\t\tLOG4CXX_NS::helpers::Transcoder::decode(tzName, retval);\n\t\t\treturn retval;\n\t\t}\n\n};\n\n\n\n/** Time zone object that represents a fixed offset from GMT. */\nclass FixedTimeZone : public TimeZone\n{\n\tpublic:\n\t\tFixedTimeZone( const LogString& name, apr_int32_t offset1 ) : TimeZone( name ), offset( offset1 )\n\t\t{\n\t\t}\n\n\t\t/** Explode time to human readable form. */\n\t\tlog4cxx_status_t explode( apr_time_exp_t* result, log4cxx_time_t input ) const\n\t\t{\n\t\t\tapr_status_t stat;\n\n\t\t\t//  APR 1.1 and early mishandles microseconds on dates\n\t\t\t//   before 1970, APR bug 32520\n\t\t\tif (LOG4CXX_UNLIKELY(input < 0 && apr_time_usec(input) < 0))\n\t\t\t{\n\t\t\t\tapr_time_t floorTime = (apr_time_sec(input) - 1) * APR_USEC_PER_SEC;\n\t\t\t\tstat = apr_time_exp_tz(result, floorTime, offset);\n\t\t\t\tresult->tm_usec = (int) (input - floorTime);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tstat = apr_time_exp_tz( result, input, offset );\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\tprivate:\n\t\tconst apr_int32_t offset;\n};\n\n}\n}\n}\n\n\n\nTimeZone::TimeZone( const LogString& id1 ) : id( id1 )\n{\n}\n\nTimeZone::~TimeZone()\n{\n}\n\nconst TimeZonePtr& TimeZone::getDefault()\n{\n\treturn LOG4CXX_NS::helpers::TimeZoneImpl::LocalTimeZone::getInstance();\n}\n\nconst TimeZonePtr& TimeZone::getGMT()\n{\n\treturn LOG4CXX_NS::helpers::TimeZoneImpl::GMTTimeZone::getInstance();\n}\n\nconst TimeZonePtr TimeZone::getTimeZone( const LogString& id )\n{\n\tconst logchar gmt[] = { 0x47, 0x4D, 0x54, 0 };\n\n\tif ( id == gmt )\n\t{\n\t\treturn LOG4CXX_NS::helpers::TimeZoneImpl::GMTTimeZone::getInstance();\n\t}\n\n\tif ( id.length() >= 5 && id.substr( 0, 3 ) == gmt )\n\t{\n\t\tint hours = 0;\n\t\tint minutes = 0;\n\t\tint sign = 1;\n\n\t\tif (id[3] == 0x2D /* '-' */)\n\t\t{\n\t\t\tsign = -1;\n\t\t}\n\n\t\tLogString off( id.substr( 4 ) );\n\n\t\tif ( id.length() >= 7 )\n\t\t{\n\t\t\tsize_t colonPos = off.find( 0x3A /* ':' */);\n\n\t\t\tif ( colonPos == LogString::npos )\n\t\t\t{\n\t\t\t\tminutes = StringHelper::toInt(off.substr(off.length() - 2));\n\t\t\t\thours = StringHelper::toInt(off.substr(0, off.length() - 2));\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tminutes = StringHelper::toInt(off.substr(colonPos + 1));\n\t\t\t\thours = StringHelper::toInt(off.substr(0, colonPos));\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\thours = StringHelper::toInt(off);\n\t\t}\n\n\t\tLogString s(gmt);\n\t\tPool p;\n\t\tLogString hh;\n\t\tStringHelper::toString(hours, p, hh);\n\n\t\tif (sign > 0)\n\t\t{\n\t\t\ts.append(1, (logchar) 0x2B /* '+' */);\n\t\t}\n\t\telse\n\t\t{\n\t\t\ts.append(1, (logchar) 0x2D /* '-' */);\n\t\t}\n\n\t\tif (hh.length() == 1)\n\t\t{\n\t\t\ts.append(1, (logchar) 0x30 /* '0' */);\n\t\t}\n\n\t\ts.append(hh);\n\t\ts.append(1, (logchar) 0x3A /*' :' */);\n\t\tLogString mm;\n\t\tStringHelper::toString(minutes, p, mm);\n\n\t\tif (mm.length() == 1)\n\t\t{\n\t\t\ts.append(1, (logchar) 0x30 /* '0' */);\n\t\t}\n\n\t\ts.append(mm);\n\t\tapr_int32_t offset = sign * (hours * 3600 + minutes * 60);\n\t\treturn std::make_shared<helpers::TimeZoneImpl::FixedTimeZone>( s, offset );\n\t}\n\n\tconst TimeZonePtr& ltz = getDefault();\n\n\tif ( ltz->getID() == id )\n\t{\n\t\treturn ltz;\n\t}\n\n\treturn getGMT();\n}\n\n\n// Path: src/main/cpp/strftimedateformat.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/strftimedateformat.h>\n\n#include <apr_time.h>\n#include <log4cxx/helpers/transcoder.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct StrftimeDateFormat::StrftimeDateFormatPrivate{\n\t\nStrftimeDateFormatPrivate() :\n\t\ttimeZone(TimeZone::getDefault())\n\t{}\n\n\t/**\n\t*    Time zone.\n\t*/\n\tTimeZonePtr timeZone;\n\tstd::string pattern;\n};\n\n\nStrftimeDateFormat::StrftimeDateFormat(const LogString& fmt)\n\t: m_priv(std::make_unique<StrftimeDateFormatPrivate>())\n{\n\tLOG4CXX_NS::helpers::Transcoder::encode(fmt, m_priv->pattern);\n}\n\nStrftimeDateFormat::~StrftimeDateFormat()\n{\n}\n\n\nvoid StrftimeDateFormat::format(LogString& s, log4cxx_time_t time, Pool& /* p */ ) const\n{\n\tapr_time_exp_t exploded;\n\tapr_status_t stat = m_priv->timeZone->explode(&exploded, time);\n\n\tif (stat == APR_SUCCESS)\n\t{\n\t\tconst apr_size_t bufSize = 255;\n\t\tchar buf[bufSize];\n\t\tapr_size_t bufLen;\n\t\tstat = apr_strftime(buf, &bufLen, bufSize, m_priv->pattern.c_str(), &exploded);\n\n\t\tif (stat == APR_SUCCESS)\n\t\t{\n\t\t\tLOG4CXX_NS::helpers::Transcoder::decode(std::string(buf, bufLen), s);\n\t\t}\n\t}\n}\n\nvoid StrftimeDateFormat::setTimeZone(const TimeZonePtr& zone)\n{\n\tm_priv->timeZone = zone;\n}\n\n\n\n// Path: src/main/cpp/hexdump.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/log4cxx.h>\n/* Prevent error C2491: 'std::numpunct<_Elem>::id': definition of dllimport static data member not allowed */\n#if defined(_MSC_VER) && (LOG4CXX_UNICHAR_API || LOG4CXX_LOGCHAR_IS_UNICHAR)\n#define __FORCE_INSTANCE\n#endif\n#include <log4cxx/hexdump.h>\n#include <log4cxx/log4cxx.h>\n#include <sstream>\n#include <ios>\n#include <iomanip>\n#include <cctype>\n\nusing namespace LOG4CXX_NS;\n\ntypedef std::basic_stringstream<logchar> LogStream;\n\nLogString LOG4CXX_NS::hexdump(const void* bytes, uint32_t len, HexdumpFlags flags){\n\tLogString ret;\n\tconst uint8_t* bytes_u8 = static_cast<const uint8_t*>(bytes);\n\tLogStream sstream;\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\tconst wchar_t fill_char = L'0';\n\tconst wchar_t space_fill_char = L' ';\n#else\n\tconst logchar fill_char = '0';\n\tconst logchar space_fill_char = ' ';\n#endif\n\n\tif(flags & HexdumpFlags::AddStartingNewline){\n\t\tsstream << LOG4CXX_EOL;\n\t}\n\n\tfor(uint32_t offset = 0; offset < len; offset += 16){\n\t\tif(offset != 0){\n\t\t\tsstream << LOG4CXX_EOL;\n\t\t}\n\n\t\t// Print out the offset\n\t\tsstream << std::hex << std::setw(8) << std::setfill(fill_char) << offset << std::resetiosflags(std::ios_base::fmtflags(0));\n\n\t\tsstream << std::setw(0) << LOG4CXX_STR(\"  \");\n\n\t\t// Print out the first 8 bytes\n\t\tfor(int byte = 0; byte < 8; byte++){\n\t\t\tif(offset + byte >= len){\n\t\t\t\tsstream << LOG4CXX_STR(\"  \");\n\t\t\t\tif(byte != 8){\n\t\t\t\t\tsstream << LOG4CXX_STR(\" \");\n\t\t\t\t}\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tsstream << std::hex << std::setw(2) << std::setfill(fill_char) << static_cast<int>(bytes_u8[offset + byte]) << std::resetiosflags(std::ios_base::fmtflags(0));\n\t\t\tsstream << std::setfill(space_fill_char);\n\t\t\tif(byte != 8){\n\t\t\t\tsstream << LOG4CXX_STR(\" \");\n\t\t\t}\n\t\t}\n\n\t\tsstream << LOG4CXX_STR(\" \");\n\n\t\t// Print out the last 8 bytes\n\t\tfor(int byte = 8; byte < 16; byte++){\n\t\t\tif(offset + byte >= len){\n\t\t\t\tsstream << LOG4CXX_STR(\"  \");\n\t\t\t\tif(byte != 15){\n\t\t\t\t\tsstream << LOG4CXX_STR(\" \");\n\t\t\t\t}\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tsstream << std::hex << std::setw(2) << std::setfill(fill_char) << static_cast<int>(bytes_u8[offset + byte]) << std::resetiosflags(std::ios_base::fmtflags(0));\n\t\t\tif(byte != 15){\n\t\t\t\tsstream << LOG4CXX_STR(\" \");\n\t\t\t}\n\t\t}\n\n\t\t// Print out the ASCII text\n\t\tsstream << LOG4CXX_STR(\"  |\");\n\t\tfor(int byte = 0; byte < 16; byte++){\n\t\t\tif(offset + byte >= len){\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tif(std::isprint(bytes_u8[offset + byte])){\n\t\t\t\tlogchar to_append = bytes_u8[offset + byte];\n\t\t\t\tsstream << to_append;\n\t\t\t}else{\n\t\t\t\tsstream << LOG4CXX_STR(\".\");\n\t\t\t}\n\t\t}\n\t\tsstream << LOG4CXX_STR(\"|\");\n\t}\n\n\tif(flags & HexdumpFlags::AddEndingNewline){\n\t\tsstream << LOG4CXX_EOL;\n\t}\n\n\treturn sstream.str();\n}\n\n// Path: src/main/cpp/inputstream.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/inputstream.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(InputStream)\n\nInputStream::InputStream()\n{\n}\n\nInputStream::~InputStream()\n{\n}\n\n// Path: src/main/cpp/writer.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/writer.h>\n#include <log4cxx/helpers/loglog.h>\n#include <stdexcept>\n\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(Writer)\n\nWriter::Writer()\n{\n}\n\nWriter::~Writer()\n{\n}\n\n// Path: src/main/cpp/multiprocessrollingfileappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <apr_portable.h>\n#include <apr_file_io.h>\n#include <apr_mmap.h>\n#ifndef MAX_FILE_LEN\n\t#define MAX_FILE_LEN 2048\n#endif\n#include <log4cxx/pattern/filedatepatternconverter.h>\n#include <log4cxx/helpers/date.h>\n\n#include <log4cxx/rolling/multiprocessrollingfileappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/rolling/rolloverdescription.h>\n#include <log4cxx/helpers/fileoutputstream.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/rolling/fixedwindowrollingpolicy.h>\n#include <log4cxx/rolling/manualtriggeringpolicy.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/private/fileappender_priv.h>\n#include <log4cxx/rolling/timebasedrollingpolicy.h>\n#include <log4cxx/private/boost-std-configuration.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::rolling;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\nstruct MultiprocessRollingFileAppender::MultiprocessRollingFileAppenderPriv : public FileAppenderPriv\n{\n\tMultiprocessRollingFileAppenderPriv() :\n\t\tFileAppenderPriv(),\n\t\tfileLength(0) {}\n\n\t/**\n\t * Triggering policy.\n\t */\n\tTriggeringPolicyPtr triggeringPolicy;\n\n\t/**\n\t * Rolling policy.\n\t */\n\tRollingPolicyPtr rollingPolicy;\n\n\t/**\n\t * Length of current active log file.\n\t */\n\tsize_t fileLength;\n\n\t/**\n\t *  save the loggingevent\n\t */\n\tspi::LoggingEventPtr _event;\n};\n\n#define _priv static_cast<MultiprocessRollingFileAppenderPriv*>(m_priv.get())\n\nIMPLEMENT_LOG4CXX_OBJECT(MultiprocessRollingFileAppender)\n\n\n/**\n * Construct a new instance.\n */\nMultiprocessRollingFileAppender::MultiprocessRollingFileAppender() :\n\tFileAppender (std::make_unique<MultiprocessRollingFileAppenderPriv>())\n{\n}\n\n/**\n * Prepare instance of use.\n */\nvoid MultiprocessRollingFileAppender::activateOptions(Pool& p)\n{\n\tif (_priv->rollingPolicy == NULL)\n\t{\n\t\tauto fwrp = std::make_shared<FixedWindowRollingPolicy>();\n\t\tfwrp->setFileNamePattern(getFile() + LOG4CXX_STR(\".%i\"));\n\t\t_priv->rollingPolicy = fwrp;\n\t}\n\n\t//\n\t//  if no explicit triggering policy and rolling policy is both.\n\t//\n\tif (_priv->triggeringPolicy == NULL)\n\t{\n\t\tTriggeringPolicyPtr trig = LOG4CXX_NS::cast<TriggeringPolicy>(_priv->rollingPolicy);\n\n\t\tif (trig != NULL)\n\t\t{\n\t\t\t_priv->triggeringPolicy = trig;\n\t\t}\n\t}\n\n\tif (_priv->triggeringPolicy == NULL)\n\t{\n\t\t_priv->triggeringPolicy = std::make_shared<ManualTriggeringPolicy>();\n\t}\n\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t_priv->triggeringPolicy->activateOptions(p);\n\t\t_priv->rollingPolicy->activateOptions(p);\n\n\t\ttry\n\t\t{\n\t\t\tRolloverDescriptionPtr rollover1 =\n\t\t\t\t_priv->rollingPolicy->initialize(getFile(), getAppend(), p);\n\n\t\t\tif (rollover1 != NULL)\n\t\t\t{\n\t\t\t\tActionPtr syncAction(rollover1->getSynchronous());\n\n\t\t\t\tif (syncAction != NULL)\n\t\t\t\t{\n\t\t\t\t\tsyncAction->execute(p);\n\t\t\t\t}\n\n\t\t\t\t_priv->fileName = rollover1->getActiveFileName();\n\t\t\t\t_priv->fileAppend = rollover1->getAppend();\n\n\t\t\t\t//\n\t\t\t\t//  async action not yet implemented\n\t\t\t\t//\n\t\t\t\tActionPtr asyncAction(rollover1->getAsynchronous());\n\n\t\t\t\tif (asyncAction != NULL)\n\t\t\t\t{\n\t\t\t\t\tasyncAction->execute(p);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tFile activeFile;\n\t\t\tactiveFile.setPath(getFile());\n\n\t\t\tif (getAppend())\n\t\t\t{\n\t\t\t\t_priv->fileLength = activeFile.length(p);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t_priv->fileLength = 0;\n\t\t\t}\n\n\t\t\tFileAppender::activateOptionsInternal(p);\n\t\t}\n\t\tcatch (std::exception&)\n\t\t{\n\t\t\tLogLog::warn(\n\t\t\t\tLogString(LOG4CXX_STR(\"Exception will initializing RollingFileAppender named \"))\n\t\t\t\t+ getName());\n\t\t}\n\t}\n}\n\nvoid MultiprocessRollingFileAppender::releaseFileLock(apr_file_t* lock_file)\n{\n\tif (lock_file)\n\t{\n\t\tapr_status_t stat = apr_file_unlock(lock_file);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tLogLog::warn(LOG4CXX_STR(\"flock: unlock failed\"));\n\t\t}\n\n\t\tapr_file_close(lock_file);\n\t\tlock_file = NULL;\n\t}\n}\n\n/**\n   Implements the usual roll over behaviour.\n\n   <p>If <code>MaxBackupIndex</code> is positive, then files\n   {<code>File.1</code>, ..., <code>File.MaxBackupIndex -1</code>}\n   are renamed to {<code>File.2</code>, ...,\n   <code>File.MaxBackupIndex</code>}. Moreover, <code>File</code> is\n   renamed <code>File.1</code> and closed. A new <code>File</code> is\n   created to receive further log output.\n\n   <p>If <code>MaxBackupIndex</code> is equal to zero, then the\n   <code>File</code> is truncated with no backup files created.\n\n * @return true if rollover performed.\n */\nbool MultiprocessRollingFileAppender::rollover(Pool& p)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\treturn rolloverInternal(p);\n}\n\nbool MultiprocessRollingFileAppender::rolloverInternal(Pool& p)\n{\n\t//\n\t//   can't roll without a policy\n\t//\n\tif (_priv->rollingPolicy != NULL)\n\t{\n\n\t\t{\n\t\t\tLogString fileName(getFile());\n\t\t\tRollingPolicyBasePtr basePolicy = LOG4CXX_NS::cast<RollingPolicyBase>(_priv->rollingPolicy);\n\t\t\tapr_time_t n = apr_time_now();\n\t\t\tObjectPtr obj = std::make_shared<Date>(n);\n\t\t\tLogString fileNamePattern;\n\n\t\t\tif (basePolicy)\n\t\t\t{\n\t\t\t\tif (basePolicy->getPatternConverterList().size())\n\t\t\t\t{\n\t\t\t\t\t(*(basePolicy->getPatternConverterList().begin()))->format(obj, fileNamePattern, p);\n\t\t\t\t\tfileName = std::string(fileNamePattern);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tbool bAlreadyRolled = true;\n\t\t\tchar szDirName[MAX_FILE_LEN] = {'\\0'};\n\t\t\tchar szBaseName[MAX_FILE_LEN] = {'\\0'};\n\t\t\tchar szUid[MAX_FILE_LEN] = {'\\0'};\n\t\t\tmemcpy(szDirName, fileName.c_str(), fileName.size() > MAX_FILE_LEN ? MAX_FILE_LEN : fileName.size());\n\t\t\tmemcpy(szBaseName, fileName.c_str(), fileName.size() > MAX_FILE_LEN ? MAX_FILE_LEN : fileName.size());\n\t\t\tapr_uid_t uid;\n\t\t\tapr_gid_t groupid;\n\t\t\tapr_status_t stat = apr_uid_current(&uid, &groupid, p.getAPRPool());\n\n\t\t\tif (stat == APR_SUCCESS)\n\t\t\t{\n#ifdef WIN32\n\t\t\t\tsnprintf(szUid, MAX_FILE_LEN, \"%p\", uid);\n#else\n\t\t\t\tsnprintf(szUid, MAX_FILE_LEN, \"%u\", (unsigned int)uid);\n#endif\n\t\t\t}\n\n\t\t\tLOG4CXX_NS::filesystem::path path = szDirName;\n\t\t\tconst auto lockname = path.parent_path() / (path.filename().string() + szUid + \".lock\");\n\t\t\tapr_file_t* lock_file;\n\t\t\tstat = apr_file_open(&lock_file, lockname.string().c_str(), APR_CREATE | APR_READ | APR_WRITE, APR_OS_DEFAULT, p.getAPRPool());\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tLogString err = LOG4CXX_STR(\"lockfile return error: open lockfile failed. \");\n\t\t\t\terr += (strerror(errno));\n\t\t\t\tLogLog::warn(err);\n\t\t\t\tbAlreadyRolled = false;\n\t\t\t\tlock_file = NULL;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tstat = apr_file_lock(lock_file, APR_FLOCK_EXCLUSIVE);\n\n\t\t\t\tif (stat != APR_SUCCESS)\n\t\t\t\t{\n\t\t\t\t\tLogString err = LOG4CXX_STR(\"apr_file_lock: lock failed. \");\n\t\t\t\t\terr += (strerror(errno));\n\t\t\t\t\tLogLog::warn(err);\n\t\t\t\t\tbAlreadyRolled = false;\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\tif (_priv->_event)\n\t\t\t\t\t{\n\t\t\t\t\t\t_priv->triggeringPolicy->isTriggeringEvent(this, _priv->_event, getFile(), getFileLength());\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (bAlreadyRolled)\n\t\t\t{\n\t\t\t\tapr_finfo_t finfo1, finfo2;\n\t\t\t\tapr_status_t st1, st2;\n\t\t\t\tconst WriterPtr writer = getWriter();\n\t\t\t\tconst FileOutputStreamPtr fos = LOG4CXX_NS::cast<FileOutputStream>( writer );\n\t\t\t\tif( !fos ){\n\t\t\t\t\tLogLog::error( LOG4CXX_STR(\"Can't cast writer to FileOutputStream\") );\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t\tapr_file_t* _fd = fos->getFilePtr();\n\t\t\t\tst1 = apr_file_info_get(&finfo1, APR_FINFO_IDENT, _fd);\n\n\t\t\t\tif (st1 != APR_SUCCESS)\n\t\t\t\t{\n\t\t\t\t\tLogLog::warn(LOG4CXX_STR(\"apr_file_info_get failed\"));\n\t\t\t\t}\n\n\t\t\t\tLogString fname = getFile();\n\t\t\t\tst2 = apr_stat(&finfo2, fname.c_str(), APR_FINFO_IDENT, p.getAPRPool());\n\n\t\t\t\tif (st2 != APR_SUCCESS)\n\t\t\t\t{\n\t\t\t\t\tLogLog::warn(LOG4CXX_STR(\"apr_stat failed.\"));\n\t\t\t\t}\n\n\t\t\t\tbAlreadyRolled = ((st1 == APR_SUCCESS) && (st2 == APR_SUCCESS)\n\t\t\t\t\t\t&& ((finfo1.device != finfo2.device) || (finfo1.inode != finfo2.inode)));\n\t\t\t}\n\n\t\t\tif (!bAlreadyRolled)\n\t\t\t{\n\n\t\t\t\ttry\n\t\t\t\t{\n\t\t\t\t\tRolloverDescriptionPtr rollover1(_priv->rollingPolicy->rollover(this->getFile(), this->getAppend(), p));\n\n\t\t\t\t\tif (rollover1 != NULL)\n\t\t\t\t\t{\n\t\t\t\t\t\tif (rollover1->getActiveFileName() == getFile())\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tcloseWriter();\n\n\t\t\t\t\t\t\tbool success = true;\n\n\t\t\t\t\t\t\tif (rollover1->getSynchronous() != NULL)\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tsuccess = false;\n\n\t\t\t\t\t\t\t\ttry\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tsuccess = rollover1->getSynchronous()->execute(p);\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tcatch (std::exception& ex)\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tLogLog::warn(LOG4CXX_STR(\"Exception on rollover\"));\n\t\t\t\t\t\t\t\t\tLogString exmsg;\n\t\t\t\t\t\t\t\t\tLOG4CXX_NS::helpers::Transcoder::decode(ex.what(), exmsg);\n\t\t\t\t\t\t\t\t\t_priv->errorHandler->error(exmsg, ex, 0);\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif (success)\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tif (rollover1->getAppend())\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t_priv->fileLength = File().setPath(rollover1->getActiveFileName()).length(p);\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\telse\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t_priv->fileLength = 0;\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t//\n\t\t\t\t\t\t\t\t//  async action not yet implemented\n\t\t\t\t\t\t\t\t//\n\t\t\t\t\t\t\t\tActionPtr asyncAction(rollover1->getAsynchronous());\n\n\t\t\t\t\t\t\t\tif (asyncAction != NULL)\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tasyncAction->execute(p);\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\tsetFileInternal(\n\t\t\t\t\t\t\t\t\trollover1->getActiveFileName(), rollover1->getAppend(),\n\t\t\t\t\t\t\t\t\t_priv->bufferedIO, _priv->bufferSize, p);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\telse\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tsetFileInternal(\n\t\t\t\t\t\t\t\t\trollover1->getActiveFileName(), true, _priv->bufferedIO, _priv->bufferSize, p);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\telse\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tcloseWriter();\n\t\t\t\t\t\t\tsetFileInternal(rollover1->getActiveFileName());\n\t\t\t\t\t\t\t// Call activateOptions to create any intermediate directories(if required)\n\t\t\t\t\t\t\tFileAppender::activateOptionsInternal(p);\n\t\t\t\t\t\t\tOutputStreamPtr os(new FileOutputStream(\n\t\t\t\t\t\t\t\t\trollover1->getActiveFileName(), rollover1->getAppend()));\n\t\t\t\t\t\t\tWriterPtr newWriter(createWriter(os));\n\t\t\t\t\t\t\tsetWriterInternal(newWriter);\n\n\t\t\t\t\t\t\tbool success = true;\n\n...\n", "needle_token_start": 8334, "needle_token_end": 8356, "code_context_ntokens": 15213, "output": ["```cpp\n// Path: src/main/cpp/timezone.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#define __STDC_CONSTANT_MACROS\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/timezone.h>\n#include <stdlib.h>\n\n#include <apr_time.h>\n#include <apr_pools.h>\n#include <apr_strings.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/pool.h>\n#include <log4cxx/logger.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT( TimeZone )\n\nnamespace LOG4CXX_NS\n{\nnamespace helpers\n{\nnamespace TimeZoneImpl\n{\n/** Time zone object that represents GMT. */\nclass GMTTimeZone : public TimeZone\n{\n\tpublic:\n\t\t/** Class factory. */\n\t\tstatic const TimeZonePtr& getInstance()\n\t\t{\n\t\t\tstatic WideLife<TimeZonePtr> tz = std::make_shared<GMTTimeZone>();\n\t\t\treturn tz;\n\t\t}\n\n\t\t/** Explode time to human readable form. */\n\t\tlog4cxx_status_t explode( apr_time_exp_t* result, log4cxx_time_t input ) const\n\t\t{\n\t\t\tapr_status_t stat;\n\n\t\t\t//  APR 1.1 and early mishandles microseconds on dates\n\t\t\t//   before 1970, APR bug 32520\n\t\t\tif (LOG4CXX_UNLIKELY(input < 0 && apr_time_usec(input) < 0))\n\t\t\t{\n\t\t\t\tapr_time_t floorTime = (apr_time_sec(input) - 1) * APR_USEC_PER_SEC;\n\t\t\t\tstat = apr_time_exp_gmt(result, floorTime);\n\t\t\t\tresult->tm_usec = (int) (input - floorTime);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tstat = apr_time_exp_gmt( result, input );\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\t\tGMTTimeZone() : TimeZone( LOG4CXX_STR(\"GMT\") )\n\t\t{\n\t\t}\n};\n\n\n\n/** Time zone object that represents GMT. */\nclass LocalTimeZone : public TimeZone\n{\n\tpublic:\n\t\t/** Class factory. */\n\t\tstatic const TimeZonePtr& getInstance()\n\t\t{\n\t\t\tstatic WideLife<TimeZonePtr> tz = std::make_shared<LocalTimeZone>();\n\t\t\treturn tz;\n\t\t}\n\n\t\t/** Explode time to human readable form. */\n\t\tlog4cxx_status_t explode( apr_time_exp_t* result, log4cxx_time_t input ) const\n\t\t{\n\t\t\tapr_status_t stat;\n\n\t\t\t//  APR 1.1 and early mishandles microseconds on dates\n\t\t\t//   before 1970, APR bug 32520\n\t\t\tif (LOG4CXX_UNLIKELY(input < 0 && apr_time_usec(input) < 0))\n\t\t\t{\n\t\t\t\tapr_time_t floorTime = (apr_time_sec(input) - 1) * APR_USEC_PER_SEC;\n\t\t\t\tstat = apr_time_exp_lt(result, floorTime);\n\t\t\t\tresult->tm_usec = (int) (input - floorTime);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tstat = apr_time_exp_lt( result, input );\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\t\tLocalTimeZone() : TimeZone( getTimeZoneName() )\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tstatic const LogString getTimeZoneName()\n\t\t{\n\t\t\tconst int MAX_TZ_LENGTH = 255;\n\t\t\tchar tzName[MAX_TZ_LENGTH];\n\t\t\tapr_size_t tzLength;\n\t\t\tapr_time_exp_t tm;\n\t\t\tapr_time_exp_lt(&tm, 0);\n\t\t\tapr_strftime(tzName, &tzLength, MAX_TZ_LENGTH, \"%Z\", &tm);\n\n\t\t\tif (tzLength == 0)\n\t\t\t{\n\t\t\t\tapr_strftime(tzName, &tzLength, MAX_TZ_LENGTH, \"%z\", &tm);\n\t\t\t}\n\n\t\t\ttzName[tzLength] = 0;\n\t\t\tLogString retval;\n\t\t\tLOG4CXX_NS::helpers::Transcoder::decode(t"]}
{"repo": "apache/logging-log4cxx", "name": "UTF8CharsetEncoder", "language": "cpp", "path": "src/main/cpp/charsetencoder.cpp", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a sequence of characters into UTF-8 encoded bytes, which is a widely used character encoding system that supports a large range of characters from various scripts.\n2. **Input**: It accepts a sequence of characters along with a position indicator to start the conversion, and a buffer to store the encoded bytes.\n3. **Output**: The result is a status indicating the success or failure of the encoding process, and the buffer is filled with the UTF-8 encoded bytes.\n4. **Procedure**: The function iterates over the input characters, converting each character into its corresponding UTF-8 encoded form. It checks if the character can be encoded within the remaining space of the buffer. If a character cannot be encoded (due to space constraints or encoding issues), the process stops, and an appropriate status is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/level.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT_WITH_CUSTOM_CLASS(Level, LevelClass)\n\nLevelPtr Level::getOff()\n{\n\tstatic WideLife<LevelPtr> offLevel = std::make_shared<Level>(Level::OFF_INT, LOG4CXX_STR(\"OFF\"), 0);\n\treturn offLevel;\n}\n\nLevelPtr Level::getFatal()\n{\n\tstatic WideLife<LevelPtr> fatalLevel = std::make_shared<Level>(Level::FATAL_INT, LOG4CXX_STR(\"FATAL\"), 0);\n\treturn fatalLevel;\n}\n\nLevelPtr Level::getError()\n{\n\tstatic WideLife<LevelPtr> errorLevel = std::make_shared<Level>(Level::ERROR_INT, LOG4CXX_STR(\"ERROR\"), 3);\n\treturn errorLevel;\n}\n\nLevelPtr Level::getWarn()\n{\n\tstatic WideLife<LevelPtr> warnLevel = std::make_shared<Level>(Level::WARN_INT, LOG4CXX_STR(\"WARN\"), 4);\n\treturn warnLevel;\n}\n\nLevelPtr Level::getInfo()\n{\n\tstatic WideLife<LevelPtr> infoLevel = std::make_shared<Level>(Level::INFO_INT, LOG4CXX_STR(\"INFO\"), 6);\n\treturn infoLevel;\n}\n\nLevelPtr Level::getDebug()\n{\n\tstatic WideLife<LevelPtr> debugLevel = std::make_shared<Level>(Level::DEBUG_INT, LOG4CXX_STR(\"DEBUG\"), 7);\n\treturn debugLevel;\n}\n\nLevelPtr Level::getTrace()\n{\n\tstatic WideLife<LevelPtr> traceLevel = std::make_shared<Level>(Level::TRACE_INT, LOG4CXX_STR(\"TRACE\"), 7);\n\treturn traceLevel;\n}\n\n\nLevelPtr Level::getAll()\n{\n\tstatic WideLife<LevelPtr> allLevel = std::make_shared<Level>(Level::ALL_INT, LOG4CXX_STR(\"ALL\"), 7);\n\treturn allLevel;\n}\n\n\n\nLevel::Level(int level1,\n\tconst LogString& name1, int syslogEquivalent1)\n\t: level(level1), name(name1), syslogEquivalent(syslogEquivalent1)\n{\n\tAPRInitializer::initialize();\n}\n\n\nLevelPtr Level::toLevelLS(const LogString& sArg)\n{\n\treturn toLevelLS(sArg, Level::getDebug());\n}\n\nLogString Level::toString() const\n{\n\treturn name;\n}\n\n\nLevelPtr Level::toLevel(int val)\n{\n\treturn toLevel(val, Level::getDebug());\n}\n\nconst Level::Data& Level::getData()\n{\n\tstatic Data data =\n\t\t{ getOff()\n\t\t, getFatal()\n\t\t, getError()\n\t\t, getWarn()\n\t\t, getInfo()\n\t\t, getDebug()\n\t\t, getTrace()\n\t\t, getAll()\n\t\t};\n\treturn data;\n}\n\n\nLevelPtr Level::toLevel(int val, const LevelPtr& defaultLevel)\n{\n\tswitch (val)\n\t{\n\t\tcase ALL_INT:\n\t\t\treturn getAll();\n\n\t\tcase DEBUG_INT:\n\t\t\treturn getDebug();\n\n\t\tcase TRACE_INT:\n\t\t\treturn getTrace();\n\n\t\tcase INFO_INT:\n\t\t\treturn getInfo();\n\n\t\tcase WARN_INT:\n\t\t\treturn getWarn();\n\n\t\tcase ERROR_INT:\n\t\t\treturn getError();\n\n\t\tcase FATAL_INT:\n\t\t\treturn getFatal();\n\n\t\tcase OFF_INT:\n\t\t\treturn getOff();\n\n\t\tdefault:\n\t\t\treturn defaultLevel;\n\t}\n}\n\nLevelPtr Level::toLevel(const std::string& sArg)\n{\n\treturn toLevel(sArg, Level::getDebug());\n}\n\nLevelPtr Level::toLevel(const std::string& sArg, const LevelPtr& defaultLevel)\n{\n...\n// Path: src/main/cpp/optionconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <log4cxx/spi/loggerrepository.h>\n#include <log4cxx/appenderskeleton.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <algorithm>\n#include <ctype.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/exception.h>\n#include <stdlib.h>\n#include <log4cxx/helpers/properties.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/object.h>\n#include <log4cxx/helpers/class.h>\n#include <log4cxx/helpers/loader.h>\n#include <log4cxx/helpers/system.h>\n#include <log4cxx/propertyconfigurator.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/file.h>\n#include <log4cxx/xml/domconfigurator.h>\n#include <log4cxx/logmanager.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n#include <log4cxx/helpers/filewatchdog.h>\n\nnamespace LOG4CXX_NS\n{\n\nclass ConfiguratorWatchdog  : public helpers::FileWatchdog\n{\n\tspi::ConfiguratorPtr m_config;\n\tpublic:\n    ConfiguratorWatchdog(const spi::ConfiguratorPtr& config, const File& filename)\n        : helpers::FileWatchdog(filename)\n        , m_config(config)\n    {\n    }\n\n    /**\n    Call PropertyConfigurator#doConfigure(const String& configFileName,\n    const spi::LoggerRepositoryPtr& hierarchy) with the\n    <code>filename</code> to reconfigure log4cxx.\n    */\n    void doOnChange() override\n    {\n        m_config->doConfigure(file(), LogManager::getLoggerRepository());\n    }\n};\n\n}\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\n\nLogString OptionConverter::convertSpecialChars(const LogString& s)\n{\n\tlogchar c;\n\tLogString sbuf;\n\n\tLogString::const_iterator i = s.begin();\n\n\twhile (i != s.end())\n\t{\n\t\tc = *i++;\n\n\t\tif (c == 0x5C /* '\\\\' */)\n\t\t{\n\t\t\tc =  *i++;\n\n\t\t\tswitch (c)\n\t\t\t{\n\t\t\t\tcase 0x6E: //'n'\n\t\t\t\t\tc = 0x0A;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x72: //'r'\n\t\t\t\t\tc = 0x0D;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x74: //'t'\n\t\t\t\t\tc = 0x09;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x66: //'f'\n\t\t\t\t\tc = 0x0C;\n\t\t\t\t\tbreak;\n\n\t\t\t\tdefault:\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tsbuf.append(1, c);\n\t}\n\n\treturn sbuf;\n}\n\n\nbool OptionConverter::toBoolean(const LogString& value, bool dEfault)\n{\n\tif (value.length() >= 4)\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(value.substr(0, 4),\n\t\t\t\tLOG4CXX_STR(\"TRUE\"), LOG4CXX_STR(\"true\")))\n\t\t{\n\t\t\treturn true;\n\t\t}\n\t}\n\n\tif (dEfault && value.length() >= 5)\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(value.substr(0, 5),\n\t\t\t\tLOG4CXX_STR(\"FALSE\"), LOG4CXX_STR(\"false\")))\n\t\t{\n\t\t\treturn false;\n\t\t}\n\t}\n\n\treturn dEfault;\n}\n\nint OptionConverter::toInt(const LogString& value, int dEfault)\n{\n\tLogString trimmed(StringHelper::trim(value));\n\n\tif (trimmed.empty())\n\t{\n\t\treturn dEfault;\n\t}\n\n\tLOG4CXX_ENCODE_CHAR(cvalue, trimmed);\n\n\treturn (int) atol(cvalue.c_str());\n}\n\nlong OptionConverter::toFileSize(const LogString& s, long dEfault)\n{\n\tif (s.empty())\n\t{\n\t\treturn dEfault;\n\t}\n\n\tsize_t index = s.find_first_of(LOG4CXX_STR(\"bB\"));\n\n\tif (index != LogString::npos && index > 0)\n\t{\n\t\tlong multiplier = 1;\n\t\tindex--;\n\n\t\tif (s[index] == 0x6B /* 'k' */ || s[index] == 0x4B /* 'K' */)\n\t\t{\n\t\t\tmultiplier = 1024;\n\t\t}\n\t\telse if (s[index] == 0x6D /* 'm' */ || s[index] == 0x4D /* 'M' */)\n\t\t{\n\t\t\tmultiplier = 1024 * 1024;\n\t\t}\n\t\telse if (s[index] == 0x67 /* 'g'*/ || s[index] == 0x47 /* 'G' */)\n\t\t{\n\t\t\tmultiplier = 1024 * 1024 * 1024;\n\t\t}\n\n\t\treturn toInt(s.substr(0, index), 1) * multiplier;\n\t}\n\n\treturn toInt(s, 1);\n}\n\nLogString OptionConverter::findAndSubst(const LogString& key, Properties& props)\n{\n\tLogString value(props.getProperty(key));\n\n\tif (value.empty())\n\t{\n\t\treturn value;\n\t}\n\n\ttry\n\t{\n\t\treturn substVars(value, props);\n\t}\n\tcatch (IllegalArgumentException& e)\n\t{\n\t\tLogLog::error(((LogString) LOG4CXX_STR(\"Bad option value [\"))\n\t\t\t+ value + LOG4CXX_STR(\"].\"), e);\n\t\treturn value;\n\t}\n}\n\nLogString OptionConverter::substVars(const LogString& val, Properties& props)\n{\n\tLogString sbuf;\n\tconst logchar delimStartArray[] = { 0x24, 0x7B, 0 };\n\tconst LogString delimStart(delimStartArray);\n\tconst logchar delimStop = 0x7D; // '}';\n\tconst size_t DELIM_START_LEN = 2;\n\tconst size_t DELIM_STOP_LEN = 1;\n\n\tsize_t i = 0;\n\n\twhile (true)\n\t{\n\t\tsize_t j = val.find(delimStart, i);\n\n\t\tif (j == val.npos)\n\t\t{\n\t\t\t// no more variables\n\t\t\tif (i == 0)\n\t\t\t{\n\t\t\t\t// this is a simple string\n\t\t\t\treturn val;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t// add the tail string which contails no variables and return the result.\n\t\t\t\tsbuf.append(val.substr(i, val.length() - i));\n\t\t\t\treturn sbuf;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tsbuf.append(val.substr(i, j - i));\n\t\t\tsize_t k = val.find(delimStop, j);\n\n\t\t\tif (k == val.npos)\n\t\t\t{\n\t\t\t\tLogString msg(1, (logchar) 0x22 /* '\\\"' */);\n\t\t\t\tmsg.append(val);\n\t\t\t\tmsg.append(LOG4CXX_STR(\"\\\" has no closing brace. Opening brace at position \"));\n\t\t\t\tPool p;\n\t\t\t\tStringHelper::toString(j, p, msg);\n\t\t\t\tmsg.append(1, (logchar) 0x2E /* '.' */);\n\t\t\t\tthrow IllegalArgumentException(msg);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tj += DELIM_START_LEN;\n\t\t\t\tLogString key = val.substr(j, k - j);\n\t\t\t\t// first try in System properties\n\t\t\t\tLogString replacement(getSystemProperty(key, LogString()));\n\n\t\t\t\t// then try props parameter\n\t\t\t\tif (replacement.empty())\n\t\t\t\t{\n\t\t\t\t\treplacement = props.getProperty(key);\n\t\t\t\t}\n\n\t\t\t\tif (!replacement.empty())\n\t\t\t\t{\n\t\t\t\t\t// Do variable substitution on the replacement string\n\t\t\t\t\t// such that we can solve \"Hello ${x2}\" as \"Hello p1\"\n\t\t\t\t\t// the where the properties are\n\t\t\t\t\t// x1=p1\n\t\t\t\t\t// x2=${x1}\n\t\t\t\t\tLogString recursiveReplacement = substVars(replacement, props);\n\t\t\t\t\tsbuf.append(recursiveReplacement);\n\t\t\t\t}\n\n\t\t\t\ti = k + DELIM_STOP_LEN;\n\t\t\t}\n\t\t}\n\t}\n}\n\nLogString OptionConverter::getSystemProperty(const LogString& key, const LogString& def)\n{\n\tif (!key.empty())\n\t{\n\t\tLogString value(System::getProperty(key));\n\n\t\tif (!value.empty())\n\t\t{\n\t\t\treturn value;\n\t\t}\n\t}\n\n\treturn def;\n}\n\nLevelPtr OptionConverter::toLevel(const LogString& value,\n\tconst LevelPtr& defaultValue)\n{\n\tsize_t hashIndex = value.find(LOG4CXX_STR(\"#\"));\n\n\tif (hashIndex == LogString::npos)\n\t{\n\t\tif (value.empty())\n\t\t{\n\t\t\treturn defaultValue;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tLogLog::debug(\n\t\t\t\t((LogString) LOG4CXX_STR(\"OptionConverter::toLevel: no class name specified, level=[\"))\n\t\t\t\t+ value\n\t\t\t\t+ LOG4CXX_STR(\"]\"));\n\t\t\t// no class name specified : use standard Level class\n\t\t\treturn Level::toLevelLS(value, defaultValue);\n\t\t}\n\t}\n\n\tLogString clazz = value.substr(hashIndex + 1);\n\tLogString levelName = value.substr(0, hashIndex);\n\tLogLog::debug(((LogString) LOG4CXX_STR(\"OptionConverter::toLevel: class=[\"))\n\t\t+ clazz + LOG4CXX_STR(\"], level=[\") + levelName + LOG4CXX_STR(\"]\"));\n\n\t// This is degenerate case but you never know.\n\tif (levelName.empty())\n\t{\n\t\treturn Level::toLevelLS(value, defaultValue);\n\t}\n\n\ttry\n\t{\n\t\tLevel::LevelClass& levelClass =\n\t\t\t(Level::LevelClass&)Loader::loadClass(clazz);\n\t\treturn levelClass.toLevel(levelName);\n\t}\n\tcatch (ClassNotFoundException&)\n\t{\n\t\tLogLog::warn(((LogString) LOG4CXX_STR(\"custom level class [\"))\n\t\t\t+ clazz + LOG4CXX_STR(\"] not found.\"));\n\t}\n\tcatch (Exception& oops)\n\t{\n\t\tLogLog::warn(\n\t\t\tLOG4CXX_STR(\"class [\") + clazz + LOG4CXX_STR(\"], level [\") + levelName +\n\t\t\tLOG4CXX_STR(\"] conversion) failed.\"), oops);\n\t}\n\tcatch (...)\n\t{\n\t\tLogLog::warn(\n\t\t\tLOG4CXX_STR(\"class [\") + clazz + LOG4CXX_STR(\"], level [\") + levelName +\n\t\t\tLOG4CXX_STR(\"] conversion) failed.\"));\n\t}\n\n\treturn defaultValue;\n}\n\n\nObjectPtr OptionConverter::instantiateByKey(Properties& props, const LogString& key,\n\tconst Class& superClass, const ObjectPtr& defaultValue)\n{\n\t// Get the value of the property in string form\n\tLogString className(findAndSubst(key, props));\n\n\tif (className.empty())\n\t{\n\t\tLogLog::error(\n\t\t\t((LogString) LOG4CXX_STR(\"Could not find value for key \")) + key);\n\t\treturn defaultValue;\n\t}\n\n\t// Trim className to avoid trailing spaces that cause problems.\n\treturn OptionConverter::instantiateByClassName(\n\t\t\tStringHelper::trim(className), superClass, defaultValue);\n}\n\nObjectPtr OptionConverter::instantiateByClassName(const LogString& className,\n\tconst Class& superClass, const ObjectPtr& defaultValue)\n{\n\tif (!className.empty())\n\t{\n\t\ttry\n\t\t{\n\t\t\tconst Class& classObj = Loader::loadClass(className);\n\t\t\tObjectPtr newObject =  ObjectPtr(classObj.newInstance());\n\n\t\t\tif (!newObject->instanceof(superClass))\n\t\t\t{\n\t\t\t\treturn defaultValue;\n\t\t\t}\n\n\t\t\treturn newObject;\n\t\t}\n\t\tcatch (Exception& e)\n\t\t{\n\t\t\tLogLog::error(LOG4CXX_STR(\"Could not instantiate class [\") +\n\t\t\t\tclassName + LOG4CXX_STR(\"].\"), e);\n\t\t}\n\t}\n\n\treturn defaultValue;\n}\n\nvoid OptionConverter::selectAndConfigure(const File& configFileName,\n\tconst LogString& _clazz, spi::LoggerRepositoryPtr hierarchy, int delay)\n{\n\tConfiguratorPtr configurator;\n\tLogString clazz = _clazz;\n\n\tLogString filename(configFileName.getPath());\n\n#if LOG4CXX_HAS_DOMCONFIGURATOR\n\tif (clazz.empty()\n\t\t&& filename.length() > 4\n\t\t&& StringHelper::equalsIgnoreCase(\n\t\t\tfilename.substr(filename.length() - 4),\n\t\t\tLOG4CXX_STR(\".XML\"), LOG4CXX_STR(\".xml\")))\n\t{\n\t\tclazz = LOG4CXX_NS::xml::DOMConfigurator::getStaticClass().toString();\n\t}\n#endif\n\n\tif (!clazz.empty())\n\t{\n\t\tLogLog::debug(LOG4CXX_STR(\"Preferred configurator class: \") + clazz);\n\t\tconst Class& clazzObj = Loader::loadClass(clazz);\n\t\tObjectPtr obj = ObjectPtr(clazzObj.newInstance());\n\t\tconfigurator = LOG4CXX_NS::cast<Configurator>(obj);\n\n\t\tif (configurator == 0)\n\t\t{\n\t\t\tLogLog::error(LOG4CXX_STR(\"Could not instantiate configurator [\")\n\t\t\t\t+ clazz + LOG4CXX_STR(\"].\"));\n\t\t\treturn;\n\t\t}\n\t}\n\telse\n\t{\n\t\tconfigurator = std::make_shared<PropertyConfigurator>();\n\t}\n\n\tif (0 < delay)\n\t{\n\t\tauto dog = new ConfiguratorWatchdog(configurator, configFileName);\n\t\tAPRInitializer::registerCleanup(dog);\n\t\tdog->setDelay(delay);\n\t\tdog->start();\n\t}\n\telse\n\t\tconfigurator->doConfigure(configFileName, hierarchy);\n}\n\n// Path: src/main/cpp/locale.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/helpers/locale.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct Locale::LocalePrivate\n{\n\tLocalePrivate(const LogString& language1)\n\t\t: language(language1)\n\t{\n\t}\n\n\tLocalePrivate(const LogString& language1, const LogString& country1)\n\t\t: language(language1), country(country1)\n\t{\n\t}\n\n\tLocalePrivate(const LogString& language1, const LogString& country1,\n\t\tconst LogString& variant1)\n\t\t: language(language1), country(country1), variant(variant1)\n\t{\n\t}\n\n\tconst LogString language;\n\tconst LogString country;\n\tconst LogString variant;\n};\n\nLocale::Locale(const LogString& language1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1))\n{\n}\n\nLocale::Locale(const LogString& language1, const LogString& country1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1, country1))\n{\n}\n\nLocale::Locale(const LogString& language1, const LogString& country1,\n\tconst LogString& variant1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1, country1, variant1))\n{\n}\n\nLocale::~Locale() {}\n\nconst LogString& Locale::getLanguage() const\n{\n\treturn m_priv->language;\n}\n\nconst LogString& Locale::getCountry() const\n{\n\treturn m_priv->country;\n}\n\nconst LogString& Locale::getVariant() const\n{\n\treturn m_priv->variant;\n}\n\n\n// Path: src/main/cpp/charsetencoder.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/exception.h>\n#include <apr_xlate.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <algorithm>\n\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n\n#include <log4cxx/private/log4cxx_private.h>\n#include <apr_portable.h>\n#include <mutex>\n\n#ifdef LOG4CXX_HAS_WCSTOMBS\n\t#include <stdlib.h>\n#endif\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(CharsetEncoder)\n\nnamespace LOG4CXX_NS\n{\n\nnamespace helpers\n{\n\n#if APR_HAS_XLATE\n/**\n* A character encoder implemented using apr_xlate.\n*/\nclass APRCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tAPRCharsetEncoder(const LogString& topage) : pool()\n\t\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\t\t\tconst char* frompage = \"WCHAR_T\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\tconst char* frompage = \"UTF-8\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\t\tconst char* frompage = \"UTF-16\";\n#endif\n\t\t\tstd::string tpage(Transcoder::encodeCharsetName(topage));\n\t\t\tapr_status_t stat = apr_xlate_open(&convset,\n\t\t\t\t\ttpage.c_str(),\n\t\t\t\t\tfrompage,\n\t\t\t\t\tpool.getAPRPool());\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tthrow IllegalArgumentException(topage);\n\t\t\t}\n\t\t}\n\n\t\tvirtual ~APRCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tapr_status_t stat;\n\t\t\tsize_t outbytes_left = out.remaining();\n\t\t\tsize_t initial_outbytes_left = outbytes_left;\n\t\t\tsize_t position = out.position();\n\n\t\t\tif (iter == in.end())\n\t\t\t{\n\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\tstat = apr_xlate_conv_buffer(convset, NULL, NULL,\n\t\t\t\t\t\tout.data() + position, &outbytes_left);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tLogString::size_type inOffset = (iter - in.begin());\n\t\t\t\tapr_size_t inbytes_left =\n\t\t\t\t\t(in.size() - inOffset) * sizeof(LogString::value_type);\n\t\t\t\tapr_size_t initial_inbytes_left = inbytes_left;\n\t\t\t\t{\n\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\tstat = apr_xlate_conv_buffer(convset,\n\t\t\t\t\t\t\t(const char*) (in.data() + inOffset),\n\t\t\t\t\t\t\t&inbytes_left,\n\t\t\t\t\t\t\tout.data() + position,\n\t\t\t\t\t\t\t&outbytes_left);\n\t\t\t\t}\n\t\t\t\titer += ((initial_inbytes_left - inbytes_left) / sizeof(LogString::value_type));\n\t\t\t}\n\n\t\t\tout.position(out.position() + (initial_outbytes_left - outbytes_left));\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tAPRCharsetEncoder(const APRCharsetEncoder&);\n\t\tAPRCharsetEncoder& operator=(const APRCharsetEncoder&);\n\t\tPool pool;\n\t\tstd::mutex mutex;\n\t\tapr_xlate_t* convset;\n};\n#endif\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS\n/**\n *  A character encoder implemented using wcstombs.\n*/\nclass WcstombsCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tWcstombsCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\t/**\n\t\t *   Converts a wchar_t to the default external multibyte encoding.\n\t\t */\n\t\tlog4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\tsize_t outbytes_left = out.remaining();\n\t\t\t\tsize_t position = out.position();\n\t\t\t\tstd::wstring::size_type inOffset = (iter - in.begin());\n\t\t\t\tenum { BUFSIZE = 256 };\n\t\t\t\twchar_t buf[BUFSIZE];\n\t\t\t\tsize_t chunkSize = BUFSIZE - 1;\n\n\t\t\t\tif (chunkSize * MB_LEN_MAX > outbytes_left)\n\t\t\t\t{\n\t\t\t\t\tchunkSize = outbytes_left / MB_LEN_MAX;\n\t\t\t\t}\n\n\t\t\t\tif (chunkSize > in.length() - inOffset)\n\t\t\t\t{\n\t\t\t\t\tchunkSize = in.length() - inOffset;\n\t\t\t\t}\n\n\t\t\t\tmemset(buf, 0, BUFSIZE * sizeof(wchar_t));\n\t\t\t\tmemcpy(buf,\n\t\t\t\t\tin.data() + inOffset,\n\t\t\t\t\tchunkSize * sizeof(wchar_t));\n\t\t\t\tsize_t converted = wcstombs(out.data() + position, buf, outbytes_left);\n\n\t\t\t\tif (converted == (size_t) -1)\n\t\t\t\t{\n\t\t\t\t\tstat = APR_BADARG;\n\n\t\t\t\t\t//\n\t\t\t\t\t//   if unconvertable character was encountered\n\t\t\t\t\t//       repeatedly halve source to get fragment that\n\t\t\t\t\t//       can be converted\n\t\t\t\t\tfor (chunkSize /= 2;\n\t\t\t\t\t\tchunkSize > 0;\n\t\t\t\t\t\tchunkSize /= 2)\n\t\t\t\t\t{\n\t\t\t\t\t\tbuf[chunkSize] = 0;\n\t\t\t\t\t\tconverted = wcstombs(out.data() + position, buf, outbytes_left);\n\n\t\t\t\t\t\tif (converted != (size_t) -1)\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\titer += chunkSize;\n\t\t\t\t\t\t\tout.position(out.position() + converted);\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\titer += chunkSize;\n\t\t\t\t\tout.position(out.position() + converted);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\n\tprivate:\n\t\tWcstombsCharsetEncoder(const WcstombsCharsetEncoder&);\n\t\tWcstombsCharsetEncoder& operator=(const WcstombsCharsetEncoder&);\n};\n#endif\n\n\n/**\n*   Encodes a LogString to US-ASCII.\n*/\nclass USASCIICharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tUSASCIICharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\twhile (out.remaining() > 0 && iter != in.end())\n\t\t\t\t{\n\t\t\t\t\tLogString::const_iterator prev(iter);\n\t\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\t\tif (sv <= 0x7F)\n\t\t\t\t\t{\n\t\t\t\t\t\tout.put((char) sv);\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\titer = prev;\n\t\t\t\t\t\tstat = APR_BADARG;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tUSASCIICharsetEncoder(const USASCIICharsetEncoder&);\n\t\tUSASCIICharsetEncoder& operator=(const USASCIICharsetEncoder&);\n};\n\n/**\n*   Converts a LogString to ISO-8859-1.\n*/\nclass ISOLatinCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tISOLatinCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\twhile (out.remaining() > 0 && iter != in.end())\n\t\t\t\t{\n\t\t\t\t\tLogString::const_iterator prev(iter);\n\t\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\t\tif (sv <= 0xFF)\n\t\t\t\t\t{\n\t\t\t\t\t\tout.put((char) sv);\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\titer = prev;\n\t\t\t\t\t\tstat = APR_BADARG;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tISOLatinCharsetEncoder(const ISOLatinCharsetEncoder&);\n\t\tISOLatinCharsetEncoder& operator=(const ISOLatinCharsetEncoder&);\n};\n\n/**\n*   Encodes a LogString to a byte array when the encodings are identical.\n*/\nclass TrivialCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tTrivialCharsetEncoder()\n\t\t{\n\t\t}\n\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tif (iter != in.end())\n\t\t\t{\n\t\t\t\tsize_t requested = in.length() - (iter - in.begin());\n\n\t\t\t\tif (requested > out.remaining() / sizeof(logchar))\n\t\t\t\t{\n\t\t\t\t\trequested = out.remaining() / sizeof(logchar);\n\t\t\t\t}\n\n\t\t\t\tmemcpy(out.current(),\n\t\t\t\t\t(const char*) in.data() + (iter - in.begin()),\n\t\t\t\t\trequested * sizeof(logchar));\n\t\t\t\titer += requested;\n\t\t\t\tout.position(out.position() + requested * sizeof(logchar));\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tTrivialCharsetEncoder(const TrivialCharsetEncoder&);\n\t\tTrivialCharsetEncoder& operator=(const TrivialCharsetEncoder&);\n};\n\n#if LOG4CXX_LOGCHAR_IS_UTF8\ntypedef TrivialCharsetEncoder UTF8CharsetEncoder;\n#else\n/**\n *  Converts a LogString to UTF-8.\n */\nclass UTF8CharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\t\nUTF8CharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\twhile (iter != in.end() && out.remaining() >= 8)\n\t\t\t{\n\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t}\n\n\t\t\t\tTranscoder::encodeUTF8(sv, out);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tUTF8CharsetEncoder(const UTF8CharsetEncoder&);\n\t\tUTF8CharsetEncoder& operator=(const UTF8CharsetEncoder&);\n};\n#endif\n\n/**\n *   Encodes a LogString to UTF16-BE.\n */\nclass UTF16BECharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tUTF16BECharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\twhile (iter != in.end() && out.remaining() >= 4)\n\t\t\t{\n\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t}\n\n\t\t\t\tTranscoder::encodeUTF16BE(sv, out);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tUTF16BECharsetEncoder(const UTF16BECharsetEncoder&);\n\t\tUTF16BECharsetEncoder& operator=(const UTF16BECharsetEncoder&);\n};\n\n/**\n *   Encodes a LogString to UTF16-LE.\n */\nclass UTF16LECharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tUTF16LECharsetEncoder()\n\t\t{\n\t\t}\n\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\twhile (iter != in.end() && out.remaining() >= 4)\n\t\t\t{\n\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t}\n\n\t\t\t\tTranscoder::encodeUTF16LE(sv, out);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\tprivate:\n\t\tUTF16LECharsetEncoder(const UTF16LECharsetEncoder&);\n\t\tUTF16LECharsetEncoder& operator=(const UTF16LECharsetEncoder&);\n};\n\n/**\n *    Charset encoder that uses current locale settings.\n */\nclass LocaleCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\tLocaleCharsetEncoder() : state()\n\t\t{\n\t\t}\n\t\tlog4cxx_status_t encode\n\t\t\t( const LogString&           in\n\t\t\t, LogString::const_iterator& iter\n\t\t\t, ByteBuffer&                out\n\t\t\t) override\n\t\t{\n\t\t\tlog4cxx_status_t result = APR_SUCCESS;\n#if !LOG4CXX_CHARSET_EBCDIC\n\t\t\tchar* current = out.current();\n\t\t\tsize_t remain = out.remaining();\n\t\t\tif (std::mbsinit(&this->state)) // ByteBuffer not partially encoded?\n\t\t\t{\n\t\t\t\t// Copy single byte characters\n\t\t\t\tfor (;\n\t\t\t\t\titer != in.end() && ((unsigned int) *iter) < 0x80 && 0 < remain;\n\t\t\t\t\titer++, remain--, current++)\n\t\t\t\t{\n\t\t\t\t\t*current = *iter;\n\t\t\t\t}\n\t\t\t}\n#endif\n\t\t\t// Encode characters that may require multiple bytes\n\t\t\twhile (iter != in.end() && MB_CUR_MAX <= remain)\n\t\t\t{\n\t\t\t\tauto ch = Transcoder::decode(in, iter);\n\t\t\t\tauto n = std::wcrtomb(current, ch, &this->state);\n\t\t\t\tif (static_cast<std::size_t>(-1) == n) // not a valid wide character?\n\t\t\t\t{\n\t\t\t\t\tresult = APR_BADARG;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tremain -= n;\n\t\t\t\tcurrent += n;\n\t\t\t}\n\t\t\tout.position(current - out.data());\n\t\t\treturn result;\n\t\t}\n\n\tprivate:\n\t\tstd::mbstate_t state;\n};\n\n\n} // namespace helpers\n\n}  //namespace log4cxx\n\n\n\nCharsetEncoder::CharsetEncoder()\n{\n}\n\nCharsetEncoder::~CharsetEncoder()\n{\n}\n\nCharsetEncoderPtr CharsetEncoder::getDefaultEncoder()\n{\n\tstatic WideLife<CharsetEncoderPtr> encoder(createDefaultEncoder());\n\n\t//\n\t//  if invoked after static variable destruction\n\t//     (if logging is called in the destructor of a static object)\n\t//     then create a new decoder.\n\t//\n\tif (encoder.value() == 0)\n\t{\n\t\treturn CharsetEncoderPtr( createDefaultEncoder() );\n\t}\n\n\treturn encoder;\n}\n\nCharsetEncoder* CharsetEncoder::createDefaultEncoder()\n{\n#if LOG4CXX_CHARSET_UTF8\n\treturn new UTF8CharsetEncoder();\n#elif LOG4CXX_CHARSET_ISO88591\n\treturn new ISOLatinCharsetEncoder();\n#elif LOG4CXX_CHARSET_USASCII\n\treturn new USASCIICharsetEncoder();\n#elif LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS\n\treturn new WcstombsCharsetEncoder();\n#else\n\treturn new LocaleCharsetEncoder();\n#endif\n}\n\n\nCharsetEncoderPtr CharsetEncoder::getUTF8Encoder()\n{\n\treturn std::make_shared<UTF8CharsetEncoder>();\n}\n\n\n\nCharsetEncoderPtr CharsetEncoder::getEncoder(const LogString& charset)\n{\n\tif (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-8\"), LOG4CXX_STR(\"utf-8\"))\n\t\t|| StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP65001\"), LOG4CXX_STR(\"cp65001\")))\n\t{\n\t\treturn std::make_shared<UTF8CharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"C\"), LOG4CXX_STR(\"c\")) ||\n\t\tcharset == LOG4CXX_STR(\"646\") ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"US-ASCII\"), LOG4CXX_STR(\"us-ascii\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO646-US\"), LOG4CXX_STR(\"iso646-US\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ANSI_X3.4-1968\"), LOG4CXX_STR(\"ansi_x3.4-1968\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP20127\"), LOG4CXX_STR(\"cp20127\")))\n\t{\n\t\treturn std::make_shared<USASCIICharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-8859-1\"), LOG4CXX_STR(\"iso-8859-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-LATIN-1\"), LOG4CXX_STR(\"iso-latin-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP1252\"), LOG4CXX_STR(\"cp1252\")))\n\t{\n\t\treturn std::make_shared<ISOLatinCharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-16BE\"), LOG4CXX_STR(\"utf-16be\"))\n\t\t|| StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-16\"), LOG4CXX_STR(\"utf-16\"))\n\t\t|| StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP1200\"), LOG4CXX_STR(\"cp1200\")))\n\t{\n\t\treturn std::make_shared<UTF16BECharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-16LE\"), LOG4CXX_STR(\"utf-16le\")))\n\t{\n\t\treturn std::make_shared<UTF16LECharsetEncoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"LOCALE\"), LOG4CXX_STR(\"locale\")))\n\t{\n\t\treturn std::make_shared<LocaleCharsetEncoder>();\n\t}\n\n#if APR_HAS_XLATE\n\treturn std::make_shared<APRCharsetEncoder>(charset);\n#else\n\tthrow IllegalArgumentException(charset);\n#endif\n}\n\n\nvoid CharsetEncoder::reset()\n{\n}\n\nvoid CharsetEncoder::flush(ByteBuffer& /* out */ )\n{\n}\n\n\nvoid CharsetEncoder::encode(CharsetEncoderPtr& enc,\n\tconst LogString& src,\n\tLogString::const_iterator& iter,\n\tByteBuffer& dst)\n{\n\tlog4cxx_status_t stat = enc->encode(src, iter, dst);\n\n\tif (stat != APR_SUCCESS && iter != src.end())\n\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR || LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\titer++;\n#elif LOG4CXX_LOGCHAR_IS_UTF8\n\n\t\t//  advance past this character and all continuation characters\n\t\twhile ((*(++iter) & 0xC0) == 0x80);\n\n#else\n#error logchar is unrecognized\n#endif\n\t\tdst.put(Transcoder::LOSSCHAR);\n\t}\n}\n\nbool CharsetEncoder::isTriviallyCopyable(const LogString& src, const CharsetEncoderPtr& enc)\n{\n\tbool result;\n#if !LOG4CXX_CHARSET_EBCDIC\n\tif (dynamic_cast<LocaleCharsetEncoder*>(enc.get()))\n\t{\n\t\tresult = src.end() == std::find_if(src.begin(), src.end()\n\t\t\t, [](const logchar& ch) -> bool { return 0x80 <= (unsigned int)ch; });\n\t}\n\telse\n#endif\n\t\tresult = !!dynamic_cast<TrivialCharsetEncoder*>(enc.get());\n\treturn result;\n}\n\n// Path: src/main/cpp/colorstartpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/colorstartpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(ColorStartPatternConverter)\n\n#define priv static_cast<ColorPatternConverterPrivate*>(m_priv.get())\n\nstatic LogString colorToANSISequence(const LogString& color, bool isForeground, Pool& pool){\n\tint numberToConvert = 0;\n\n\tif(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"BLACK\"), LOG4CXX_STR(\"black\"))){\n\t\tnumberToConvert = 30;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"RED\"), LOG4CXX_STR(\"red\"))){\n\t\tnumberToConvert = 31;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"GREEN\"), LOG4CXX_STR(\"green\"))){\n\t\tnumberToConvert = 32;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"YELLOW\"), LOG4CXX_STR(\"yellow\"))){\n\t\tnumberToConvert = 33;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"BLUE\"), LOG4CXX_STR(\"blue\"))){\n\t\tnumberToConvert = 34;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"MAGENTA\"), LOG4CXX_STR(\"magenta\"))){\n\t\tnumberToConvert = 35;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"CYAN\"), LOG4CXX_STR(\"cyan\"))){\n\t\tnumberToConvert = 36;\n\t}else if(StringHelper::equalsIgnoreCase(color, LOG4CXX_STR(\"WHITE\"), LOG4CXX_STR(\"white\"))){\n\t\tnumberToConvert = 37;\n\t}\n\n\tif( numberToConvert == 0 ){\n\t\treturn LOG4CXX_STR(\"\");\n\t}\n\tLogString ret;\n\tif( isForeground == false ){\n\t\tnumberToConvert += 10;\n\t}\n\tStringHelper::toString(numberToConvert, pool, ret);\n\treturn ret;\n}\n\nstatic LogString graphicsModeToANSISequence(const LogString& graphicsMode, Pool& pool){\n\tint numberToConvert = 0;\n\n\tif(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"BOLD\"), LOG4CXX_STR(\"bold\"))){\n\t\tnumberToConvert = 1;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"DIM\"), LOG4CXX_STR(\"dim\"))){\n\t\tnumberToConvert = 2;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"ITALIC\"), LOG4CXX_STR(\"italic\"))){\n\t\tnumberToConvert = 3;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"UNDERLINE\"), LOG4CXX_STR(\"underline\"))){\n\t\tnumberToConvert = 4;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"BLINKING\"), LOG4CXX_STR(\"blinking\"))){\n\t\tnumberToConvert = 5;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"INVERSE\"), LOG4CXX_STR(\"inverse\"))){\n\t\tnumberToConvert = 7;\n\t}else if(StringHelper::equalsIgnoreCase(graphicsMode, LOG4CXX_STR(\"STRIKETHROUGH\"), LOG4CXX_STR(\"strikethrough\"))){\n\t\tnumberToConvert = 9;\n\t}\n\n\tif( numberToConvert == 0 ){\n\t\treturn LOG4CXX_STR(\"\");\n\t}\n\tLogString ret;\n\tStringHelper::toString(numberToConvert, pool, ret);\n\treturn ret;\n}\n\nstatic LogString convertSingleSequence(const LogString& sequence, Pool& pool){\n\tLogString strInParens;\n\tbool inParens = false;\n\tbool hasParens = false;\n\tsize_t x = 0;\n\n\tfor(x = 0; x < sequence.length(); x++){\n\t\tif( sequence[x] == '(' && !inParens ){\n\t\t\tinParens = true;\n\t\t\thasParens = true;\n\t\t\tcontinue;\n\t\t}else if( sequence[x] == '(' && inParens ){\n\t\t\t// Unbalanced parens - parse invalid\n\t\t\treturn LOG4CXX_STR(\"\");\n\t\t}\n\n\t\tif( sequence[x] == ')' && inParens ){\n\t\t\thasParens = true;\n\t\t\tinParens = false;\n\t\t\tbreak;\n\t\t}\n\n\t\tif( inParens ){\n\t\t\tstrInParens.push_back(sequence[x]);\n\t\t}\n\t}\n\n\tif( (x != (sequence.length() - 1) || inParens) && hasParens ){\n\t\t// Unbalanced parens, or more data in the string than we expected - parse invalid\n\t\treturn LOG4CXX_STR(\"\");\n\t}\n\n\tif(StringHelper::startsWith(sequence, LOG4CXX_STR(\"fg(\"))){\n\t\t// Parse foreground\n\t\treturn colorToANSISequence(strInParens, true, pool);\n\t}else if(StringHelper::startsWith(sequence, LOG4CXX_STR(\"bg(\"))){\n\t\treturn colorToANSISequence(strInParens, false, pool);\n\t}else{\n\t\treturn graphicsModeToANSISequence(sequence, pool);\n\t}\n}\n\nstruct ColorStartPatternConverter::ColorPatternConverterPrivate : public PatternConverterPrivate\n{\n\tColorPatternConverterPrivate( const LogString& name, const LogString& style ) :\n\t\tPatternConverterPrivate( name, style ){}\n\n\tLogString m_fatalColor;\n\tLogString m_errorColor;\n\tLogString m_warnColor;\n\tLogString m_infoColor;\n\tLogString m_debugColor;\n\tLogString m_traceColor;\n};\n\nColorStartPatternConverter::ColorStartPatternConverter() :\n\tLoggingEventPatternConverter(std::make_unique<ColorPatternConverterPrivate>(LOG4CXX_STR(\"Color Start\"),\n\t\tLOG4CXX_STR(\"colorStart\")))\n{\n}\n\nPatternConverterPtr ColorStartPatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> instance = std::make_shared<ColorStartPatternConverter>();\n\treturn instance;\n}\n\nvoid ColorStartPatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& p) const\n{\n\n\tLOG4CXX_NS::LevelPtr lvl = event->getLevel();\n\n\tswitch (lvl->toInt())\n\t{\n\t\tcase LOG4CXX_NS::Level::FATAL_INT:\n\t\t\ttoAppendTo.append(priv->m_fatalColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::ERROR_INT:\n\t\t\ttoAppendTo.append(priv->m_errorColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::WARN_INT:\n\t\t\ttoAppendTo.append(priv->m_warnColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::INFO_INT:\n\t\t\ttoAppendTo.append(priv->m_infoColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::DEBUG_INT:\n\t\t\ttoAppendTo.append(priv->m_debugColor);\n\t\t\tbreak;\n\n\t\tcase LOG4CXX_NS::Level::TRACE_INT:\n\t\t\ttoAppendTo.append(priv->m_traceColor);\n\t\t\tbreak;\n\n\t\tdefault:\n...\n", "needle_token_start": 9822, "needle_token_end": 9837, "code_context_ntokens": 15168, "output": ["```cpp\n#if LOG4CXX_LOGCHAR_IS_UTF8\ntypedef TrivialCharsetEncoder UTF8CharsetEncoder;\n#else\n/**\n *  Converts a LogString to UTF-8.\n */\nclass UTF8CharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\t\nUTF8CharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\twhile (iter != in.end() && out.remaining() >= 8)\n\t\t\t{\n\t\t\t\tunsigned int sv = Transcoder::decode(in, iter);\n\n\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t{\n\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t}\n\n\t\t\t\tTranscoder::encodeUTF8(sv, out);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tUTF8CharsetEncoder(const UTF8CharsetEncoder&);\n\t\tUTF8CharsetEncoder& operator=(const UTF8CharsetEncoder&);\n};\n#endif\n```"]}
{"repo": "apache/logging-log4cxx", "name": "PropertiesPatternConverterPrivate", "language": "cpp", "path": "src/main/cpp/propertiespatternconverter.cpp", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The purpose is to handle the customization of logging output based on specific properties. It allows the configuration of log output to include specific property values dynamically.\n2. **Input**: The input includes a property name which specifies which property of the logging event should be included in the log output.\n3. **Output**: There is no direct output; however, it modifies the logging behavior by incorporating the specified property's value into the log output.\n4. **Procedure**: The procedure involves initializing with the name of the property to be used. When a logging event occurs, the specified property's value is retrieved and formatted according to the defined style, and then included in the log output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/systemerrwriter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n...\n// Path: src/main/cpp/threadusernamepatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/threadusernamepatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(ThreadUsernamePatternConverter)\n\nThreadUsernamePatternConverter::ThreadUsernamePatternConverter() :\n\tLoggingEventPatternConverter(LOG4CXX_STR(\"Thread Name\"),\n\t\tLOG4CXX_STR(\"Thread Name\"))\n{\n}\n\nPatternConverterPtr ThreadUsernamePatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> def = std::make_shared<ThreadUsernamePatternConverter>();\n\treturn def;\n}\n\nvoid ThreadUsernamePatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& /* p */) const\n{\n\ttoAppendTo.append(event->getThreadUserName());\n}\n\n\n// Path: src/main/cpp/file.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/file.h>\n#include <apr_file_io.h>\n#include <apr_file_info.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/pool.h>\n#include <assert.h>\n#include <log4cxx/helpers/exception.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct File::FilePrivate{\n\tFilePrivate() :\n\t\tautoDelete(false)\n\t{}\n\n\tFilePrivate(LogString path) :\n\t\tpath(path),\n\t\tautoDelete(false)\n\t{}\n\n\tFilePrivate(LogString path, bool autoDelete) :\n\t\tpath(path),\n\t\tautoDelete(autoDelete)\n\t{}\n\n\tLogString path;\n\tbool autoDelete;\n};\n\nFile::File() :\n\tm_priv(std::make_unique<FilePrivate>())\n{\n}\n\ntemplate<class S>\nstatic LogString decodeLS(const S* src)\n{\n\tLogString dst;\n\n\tif (src != 0)\n\t{\n\t\tTranscoder::decode(src, dst);\n\t}\n\n\treturn dst;\n}\n\ntemplate<class S>\nstatic LogString decodeLS(const std::basic_string<S>& src)\n{\n\tLogString dst;\n\tTranscoder::decode(src, dst);\n\treturn dst;\n}\n\n\nFile::File(const std::string& name1)\n\t: m_priv(std::make_unique<FilePrivate>(decodeLS(name1)))\n{\n}\n\nFile::File(const char* name1)\n\t: m_priv(std::make_unique<FilePrivate>(decodeLS(name1)))\n{\n}\n\n#if LOG4CXX_WCHAR_T_API\nFile::File(const std::wstring& name1)\n\t: m_priv(std::make_unique<FilePrivate>(decodeLS(name1)))\n{\n}\n\nFile::File(const wchar_t* name1)\n\t: m_priv(std::make_unique<FilePrivate>(decodeLS(name1)))\n{\n}\n#endif\n\n#if LOG4CXX_UNICHAR_API || LOG4CXX_LOGCHAR_IS_UNICHAR\nFile::File(const std::basic_string<UniChar>& name1)\n\t: m_priv(std::make_unique<FilePrivate>(decodeLS(name1)))\n{\n}\n\nFile::File(const UniChar* name1)\n\t: m_priv(std::make_unique<FilePrivate>(decodeLS(name1)))\n{\n}\n#endif\n\n#if LOG4CXX_CFSTRING_API\nFile::File(const CFStringRef& name1)\n\t: m_priv(std::make_unique<FilePrivate>(decodeLS(name1)))\n{\n}\n#endif\n\nFile::File(const File& src)\n\t: m_priv(std::make_unique<FilePrivate>(src.m_priv->path, src.m_priv->autoDelete))\n{\n}\n\nFile& File::operator=(const File& src)\n{\n\tif (this == &src)\n\t{\n\t\treturn *this;\n\t}\n\n\tm_priv->path.assign(src.m_priv->path);\n\tm_priv->autoDelete = src.m_priv->autoDelete;\n\n\treturn *this;\n}\n\n\nFile::~File()\n{\n\tif(m_priv->autoDelete){\n\t\tPool p;\n\t\tdeleteFile(p);\n\t}\n}\n\n\nLogString File::getPath() const\n{\n\treturn m_priv->path;\n}\n\nFile& File::setPath(const LogString& newName)\n{\n\tm_priv->path.assign(newName);\n\treturn *this;\n}\n\nLogString File::getName() const\n{\n\tconst logchar slashes[] = { 0x2F, 0x5C, 0 };\n\tsize_t lastSlash = m_priv->path.find_last_of(slashes);\n\n\tif (lastSlash != LogString::npos)\n\t{\n\t\treturn m_priv->path.substr(lastSlash + 1);\n\t}\n\n\treturn m_priv->path;\n}\n\nchar* File::getPath(Pool& p) const\n{\n\tint style = APR_FILEPATH_ENCODING_UNKNOWN;\n\tapr_filepath_encoding(&style, p.getAPRPool());\n\tchar* retval = NULL;\n\n\tif (style == APR_FILEPATH_ENCODING_UTF8)\n\t{\n\t\tretval = Transcoder::encodeUTF8(m_priv->path, p);\n\t}\n\telse\n\t{\n\t\tretval = Transcoder::encode(m_priv->path, p);\n\t}\n\n\treturn retval;\n}\n\nlog4cxx_status_t File::open(apr_file_t** file, int flags,\n\tint perm, Pool& p) const\n{\n\treturn apr_file_open(file, getPath(p), flags, perm, p.getAPRPool());\n}\n\n\n\nbool File::exists(Pool& p) const\n{\n\tapr_finfo_t finfo;\n\tapr_status_t rv = apr_stat(&finfo, getPath(p),\n\t\t\t0, p.getAPRPool());\n\treturn rv == APR_SUCCESS;\n}\n\nchar* File::convertBackSlashes(char* src)\n{\n\tfor (char* c = src; *c != 0; c++)\n\t{\n\t\tif (*c == '\\\\')\n\t\t{\n\t\t\t*c = '/';\n\t\t}\n\t}\n\n\treturn src;\n}\n\nbool File::deleteFile(Pool& p) const\n{\n\tapr_status_t rv = apr_file_remove(convertBackSlashes(getPath(p)),\n\t\t\tp.getAPRPool());\n\treturn rv == APR_SUCCESS;\n}\n\nbool File::renameTo(const File& dest, Pool& p) const\n{\n\tapr_status_t rv = apr_file_rename(convertBackSlashes(getPath(p)),\n\t\t\tconvertBackSlashes(dest.getPath(p)),\n\t\t\tp.getAPRPool());\n\treturn rv == APR_SUCCESS;\n}\n\n\nsize_t File::length(Pool& pool) const\n{\n\tapr_finfo_t finfo;\n\tapr_status_t rv = apr_stat(&finfo, getPath(pool),\n\t\t\tAPR_FINFO_SIZE, pool.getAPRPool());\n\n\tif (rv == APR_SUCCESS)\n\t{\n\t\treturn (size_t) finfo.size;\n\t}\n\n\treturn 0;\n}\n\n\nlog4cxx_time_t File::lastModified(Pool& pool) const\n{\n\tapr_finfo_t finfo;\n\tapr_status_t rv = apr_stat(&finfo, getPath(pool),\n\t\t\tAPR_FINFO_MTIME, pool.getAPRPool());\n\n\tif (rv == APR_SUCCESS)\n\t{\n\t\treturn finfo.mtime;\n\t}\n\n\treturn 0;\n}\n\n\nstd::vector<LogString> File::list(Pool& p) const\n{\n\tapr_dir_t* dir;\n\tapr_finfo_t entry;\n\tstd::vector<LogString> filenames;\n\n\tapr_status_t stat = apr_dir_open(&dir,\n\t\t\tconvertBackSlashes(getPath(p)),\n\t\t\tp.getAPRPool());\n\n\tif (stat == APR_SUCCESS)\n\t{\n\t\tint style = APR_FILEPATH_ENCODING_UNKNOWN;\n\t\tapr_filepath_encoding(&style, p.getAPRPool());\n\t\tstat = apr_dir_read(&entry, APR_FINFO_DIRENT, dir);\n\n\t\twhile (stat == APR_SUCCESS)\n\t\t{\n\t\t\tif (entry.name != NULL)\n\t\t\t{\n\t\t\t\tLogString filename;\n\n\t\t\t\tif (style == APR_FILEPATH_ENCODING_UTF8)\n\t\t\t\t{\n\t\t\t\t\tTranscoder::decodeUTF8(entry.name, filename);\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\tTranscoder::decode(entry.name, filename);\n\t\t\t\t}\n\n\t\t\t\tfilenames.push_back(filename);\n\t\t\t}\n\n\t\t\tstat = apr_dir_read(&entry, APR_FINFO_DIRENT, dir);\n\t\t}\n\n\t\tstat = apr_dir_close(dir);\n\t}\n\n\treturn filenames;\n}\n\nLogString File::getParent(Pool&) const\n{\n\tLogString::size_type slashPos = m_priv->path.rfind(LOG4CXX_STR('/'));\n\tLogString::size_type backPos = m_priv->path.rfind(LOG4CXX_STR('\\\\'));\n\n\tif (slashPos == LogString::npos)\n\t{\n\t\tslashPos = backPos;\n\t}\n\telse\n\t{\n\t\tif (backPos != LogString::npos && backPos > slashPos)\n\t\t{\n\t\t\tslashPos = backPos;\n\t\t}\n\t}\n\n\tLogString parent;\n\n\tif (slashPos != LogString::npos && slashPos > 0)\n\t{\n\t\tparent.assign(m_priv->path, 0, slashPos);\n\t}\n\n\treturn parent;\n}\n\nbool File::mkdirs(Pool& p) const\n{\n\tapr_status_t stat = apr_dir_make_recursive(convertBackSlashes(getPath(p)),\n\t\t\tAPR_OS_DEFAULT, p.getAPRPool());\n\treturn stat == APR_SUCCESS;\n}\n\nvoid File::setAutoDelete(bool autoDelete){\n\tm_priv->autoDelete = autoDelete;\n}\n\nbool File::getAutoDelete() const{\n\treturn m_priv->autoDelete;\n}\n\n// Path: src/main/cpp/systemoutwriter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/systemoutwriter.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <stdio.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(SystemOutWriter)\n\nSystemOutWriter::SystemOutWriter()\n{\n}\n\nSystemOutWriter::~SystemOutWriter()\n{\n}\n\nvoid SystemOutWriter::close(Pool& /* p */ )\n{\n}\n\nvoid SystemOutWriter::flush(Pool& /* p */ )\n{\n\tflush();\n}\n\nvoid SystemOutWriter::write(const LogString& str, Pool& /* p */ )\n{\n\twrite(str);\n}\n\nbool SystemOutWriter::isWide()\n{\n#if LOG4CXX_FORCE_WIDE_CONSOLE\n\treturn true;\n#elif LOG4CXX_FORCE_BYTE_CONSOLE || !LOG4CXX_HAS_FWIDE\n\treturn false;\n#else\n\treturn fwide(stdout, 0) > 0;\n#endif\n}\n\nvoid SystemOutWriter::write(const LogString& str)\n{\n#if LOG4CXX_WCHAR_T_API\n\n\tif (isWide())\n\t{\n\t\tLOG4CXX_ENCODE_WCHAR(msg, str);\n\t\tfputws(msg.c_str(), stdout);\n\t\treturn;\n\t}\n\n#endif\n\tLOG4CXX_ENCODE_CHAR(msg, str);\n\tfputs(msg.c_str(), stdout);\n}\n\nvoid SystemOutWriter::flush()\n{\n\tfflush(stdout);\n}\n\n// Path: src/main/cpp/gzcompressaction.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/rolling/gzcompressaction.h>\n#include <apr_thread_proc.h>\n#include <apr_strings.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/private/action_priv.h>\n#include <log4cxx/helpers/loglog.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::rolling;\nusing namespace LOG4CXX_NS::helpers;\n\n#define priv static_cast<GZCompressActionPrivate*>(m_priv.get())\n\nstruct GZCompressAction::GZCompressActionPrivate : public ActionPrivate\n{\n\tGZCompressActionPrivate( const File& toRename,\n\t\tconst File& renameTo,\n\t\tbool deleteSource):\n\t\tsource(toRename), destination(renameTo), deleteSource(deleteSource) {}\n\n\tconst File source;\n\tFile destination;\n\tbool deleteSource;\n\tbool throwIOExceptionOnForkFailure = true;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(GZCompressAction)\n\nGZCompressAction::GZCompressAction(const File& src,\n\tconst File& dest,\n\tbool del)\n\t: Action(std::make_unique<GZCompressActionPrivate>(\n\t\t\t  src, dest, del))\n{\n}\n\nGZCompressAction::~GZCompressAction() {}\n\nbool GZCompressAction::execute(LOG4CXX_NS::helpers::Pool& p) const\n{\n\tif (priv->source.exists(p))\n\t{\n\t\tapr_pool_t* aprpool = p.getAPRPool();\n\t\tapr_procattr_t* attr;\n\t\tapr_status_t stat = apr_procattr_create(&attr, aprpool);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\tstat = apr_procattr_io_set(attr, APR_NO_PIPE, APR_FULL_BLOCK, APR_FULL_BLOCK);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\tstat = apr_procattr_cmdtype_set(attr, APR_PROGRAM_PATH);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\t//\n\t\t//   set child process output to destination file\n\t\t//\n\t\tapr_file_t* child_out;\n\t\tapr_int32_t flags = APR_FOPEN_READ | APR_FOPEN_WRITE |\n\t\t\tAPR_FOPEN_CREATE | APR_FOPEN_TRUNCATE;\n\t\tstat = priv->destination.open(&child_out, flags, APR_OS_DEFAULT, p);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\tstat =  apr_procattr_child_out_set(attr, child_out, NULL);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\t//\n\t\t//   redirect the child's error stream to this processes' error stream\n\t\t//\n\t\tapr_file_t* child_err;\n\t\tstat = apr_file_open_stderr(&child_err, aprpool);\n\n\t\tif (stat == APR_SUCCESS)\n\t\t{\n\t\t\tstat =  apr_procattr_child_err_set(attr, child_err, NULL);\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tthrow IOException(stat);\n\t\t\t}\n\t\t}\n\n\t\tpriv->destination.setAutoDelete(true);\n\n\t\tconst char** args = (const char**)\n\t\t\tapr_palloc(aprpool, 4 * sizeof(*args));\n\t\tint i = 0;\n\t\targs[i++] = \"gzip\";\n\t\targs[i++] = \"-c\";\n\t\targs[i++] = Transcoder::encode(priv->source.getPath(), p);\n\t\targs[i++] = NULL;\n\n\t\tapr_proc_t pid;\n\t\tstat = apr_proc_create(&pid, \"gzip\", args, NULL, attr, aprpool);\n\n\t\tif (stat != APR_SUCCESS && priv->throwIOExceptionOnForkFailure)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}else if(stat != APR_SUCCESS && !priv->throwIOExceptionOnForkFailure)\n\t\t{\n\t\t\t/* If we fail here (to create the gzip child process),\n\t\t\t * skip the compression and consider the rotation to be\n\t\t\t * otherwise successful. The caller has already rotated\n\t\t\t * the log file (`source` here refers to the\n\t\t\t * uncompressed, rotated path, and `destination` the\n\t\t\t * same path with `.gz` appended). Remove the empty\n\t\t\t * destination file and leave source as-is.\n\t\t\t */\n\t\t\tLogLog::warn(LOG4CXX_STR(\"Failed to fork gzip during log rotation; leaving log file uncompressed\"));\n\t\t\tstat = apr_file_close(child_out);\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tLogLog::warn(LOG4CXX_STR(\"Failed to close abandoned .gz file; ignoring\"));\n\t\t\t}\n\t\t\treturn true;\n\t\t}\n\n\t\tapr_proc_wait(&pid, NULL, NULL, APR_WAIT);\n\t\tstat = apr_file_close(child_out);\n\n\t\tif (stat != APR_SUCCESS)\n\t\t{\n\t\t\tthrow IOException(stat);\n\t\t}\n\n\t\tpriv->destination.setAutoDelete(false);\n\n\t\tif (priv->deleteSource)\n\t\t{\n\t\t\tpriv->source.deleteFile(p);\n\t\t}\n\n\t\treturn true;\n\t}\n\n\treturn false;\n}\n\nvoid GZCompressAction::setThrowIOExceptionOnForkFailure(bool throwIO){\n\tpriv->throwIOExceptionOnForkFailure = throwIO;\n}\n\n\n// Path: src/main/cpp/relativetimepatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/relativetimepatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(RelativeTimePatternConverter)\n\nRelativeTimePatternConverter::RelativeTimePatternConverter() :\n\tLoggingEventPatternConverter(LOG4CXX_STR(\"Time\"),\n\t\tLOG4CXX_STR(\"time\"))\n{\n}\n\nPatternConverterPtr RelativeTimePatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> def = std::make_shared<RelativeTimePatternConverter>();\n\treturn def;\n}\n\nvoid RelativeTimePatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& p) const\n{\n\tlog4cxx_time_t delta = (event->getTimeStamp() - LoggingEvent::getStartTime()) / 1000;\n\tStringHelper::toString(delta, p, toAppendTo);\n}\n\n\n// Path: src/main/cpp/levelmatchfilter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/filter/levelmatchfilter.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/level.h>\n#include <log4cxx/private/filter_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::filter;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\n#define priv static_cast<LevelMatchFilterPrivate*>(m_priv.get())\n\nstruct LevelMatchFilter::LevelMatchFilterPrivate : public FilterPrivate\n{\n\tbool acceptOnMatch;\n\tLevelPtr levelToMatch;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(LevelMatchFilter)\n\nLevelMatchFilter::LevelMatchFilter()\n\t: Filter(std::make_unique<LevelMatchFilterPrivate>())\n{\n\tpriv->acceptOnMatch = true;\n}\n\nLevelMatchFilter::~LevelMatchFilter() {}\n\nvoid LevelMatchFilter::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\n\n\tif (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"LEVELTOMATCH\"), LOG4CXX_STR(\"leveltomatch\")))\n\t{\n\t\tsetLevelToMatch(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"ACCEPTONMATCH\"), LOG4CXX_STR(\"acceptonmatch\")))\n\t{\n\t\tpriv->acceptOnMatch = OptionConverter::toBoolean(value, priv->acceptOnMatch);\n\t}\n}\n\nvoid LevelMatchFilter::setLevelToMatch(const LogString& levelToMatch1)\n{\n\tpriv->levelToMatch = OptionConverter::toLevel(levelToMatch1, priv->levelToMatch);\n}\n\nLogString LevelMatchFilter::getLevelToMatch() const\n{\n\treturn priv->levelToMatch->toString();\n}\n\nFilter::FilterDecision LevelMatchFilter::decide(\n\tconst LOG4CXX_NS::spi::LoggingEventPtr& event) const\n{\n\tif (priv->levelToMatch != 0 && priv->levelToMatch->equals(event->getLevel()))\n\t{\n\t\tif (priv->acceptOnMatch)\n\t\t{\n\t\t\treturn Filter::ACCEPT;\n\t\t}\n\t\telse\n\t\t{\n\t\t\treturn Filter::DENY;\n\t\t}\n\t}\n\telse\n\t{\n\t\treturn Filter::NEUTRAL;\n\t}\n}\n\nvoid LevelMatchFilter::setAcceptOnMatch(bool acceptOnMatch1)\n{\n\tpriv->acceptOnMatch = acceptOnMatch1;\n}\n\nbool LevelMatchFilter::getAcceptOnMatch() const\n{\n\treturn priv->acceptOnMatch;\n}\n\n// Path: src/main/cpp/fulllocationpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/fulllocationpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(FullLocationPatternConverter)\n\nFullLocationPatternConverter::FullLocationPatternConverter() :\n\tLoggingEventPatternConverter(LOG4CXX_STR(\"Full Location\"),\n\t\tLOG4CXX_STR(\"fullLocation\"))\n{\n}\n\nPatternConverterPtr FullLocationPatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> instance = std::make_shared<FullLocationPatternConverter>();\n\treturn instance;\n}\n\nvoid FullLocationPatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& p) const\n{\n\tappend(toAppendTo, event->getLocationInformation().getFileName());\n\ttoAppendTo.append(1, (logchar) 0x28 /* '(' */);\n\tStringHelper::toString(\n\t\tevent->getLocationInformation().getLineNumber(),\n\t\tp, toAppendTo);\n\ttoAppendTo.append(1, (logchar) 0x29 /* ')' */);\n}\n\n// Path: src/main/cpp/layout.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/layout.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(Layout)\n\n\nLayout::~Layout() {}\n\nLogString Layout::getContentType() const\n{\n\treturn LOG4CXX_STR(\"text/plain\");\n}\n\nvoid Layout::appendHeader(LogString&, LOG4CXX_NS::helpers::Pool&) {}\n\nvoid Layout::appendFooter(LogString&, LOG4CXX_NS::helpers::Pool&) {}\n\n/**\n * The expected length of a formatted event excluding the message text\n */\nsize_t Layout::getFormattedEventCharacterCount() const\n{\n\tauto exampleEvent = std::make_shared<spi::LoggingEvent>\n\t\t( LOG4CXX_STR(\"example.logger\")\n\t\t, Level::getDebug()\n\t\t, LOG4CXX_LOCATION\n\t\t, LogString()\n\t\t);\n\tLogString text;\n\tPool pool;\n\tformat(text, exampleEvent, pool);\n\treturn text.size();\n}\n\n// Path: src/main/cpp/levelpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/levelpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/level.h>\n\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(LevelPatternConverter)\n\nLevelPatternConverter::LevelPatternConverter() :\n\tLoggingEventPatternConverter(LOG4CXX_STR(\"Level\"),\n\t\tLOG4CXX_STR(\"level\"))\n{\n}\n\nPatternConverterPtr LevelPatternConverter::newInstance(\n\tconst std::vector<LogString>& /* options */)\n{\n\tstatic WideLife<PatternConverterPtr> def = std::make_shared<LevelPatternConverter>();\n\treturn def;\n}\n\nvoid LevelPatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tLOG4CXX_NS::helpers::Pool& /* p */) const\n{\n\ttoAppendTo.append(event->getLevel()->toString());\n}\n\n\n/**\n * {@inheritDoc}\n */\nLogString LevelPatternConverter::getStyleClass(const ObjectPtr& obj) const\n{\n\tLoggingEventPtr e = LOG4CXX_NS::cast<LoggingEvent>(obj);\n\n\tif (e != NULL)\n\t{\n\t\tint lint = e->getLevel()->toInt();\n\n\t\tswitch (lint)\n\t\t{\n\t\t\tcase Level::TRACE_INT:\n\t\t\t\treturn LOG4CXX_STR(\"level trace\");\n\n\t\t\tcase Level::DEBUG_INT:\n\t\t\t\treturn LOG4CXX_STR(\"level debug\");\n\n\t\t\tcase Level::INFO_INT:\n\t\t\t\treturn LOG4CXX_STR(\"level info\");\n\n\t\t\tcase Level::WARN_INT:\n\t\t\t\treturn LOG4CXX_STR(\"level warn\");\n\n\t\t\tcase Level::ERROR_INT:\n\t\t\t\treturn LOG4CXX_STR(\"level error\");\n\n\t\t\tcase Level::FATAL_INT:\n\t\t\t\treturn LOG4CXX_STR(\"level fatal\");\n\n\t\t\tdefault:\n\t\t\t\treturn LogString(LOG4CXX_STR(\"level \")) + e->getLevel()->toString();\n\t\t}\n\t}\n\n\treturn LOG4CXX_STR(\"level\");\n}\n\n// Path: src/main/cpp/consoleappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/consoleappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/systemoutwriter.h>\n#include <log4cxx/helpers/systemerrwriter.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/layout.h>\n#include <log4cxx/private/appenderskeleton_priv.h>\n#include <log4cxx/private/writerappender_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct ConsoleAppender::ConsoleAppenderPriv : public WriterAppender::WriterAppenderPriv\n{\n\tConsoleAppenderPriv(LogString target) :\n\t\tWriterAppenderPriv(),\n\t\ttarget(target) {}\n\n\tLogString target;\n};\n\n#define _priv static_cast<ConsoleAppenderPriv*>(m_priv.get())\n\nIMPLEMENT_LOG4CXX_OBJECT(ConsoleAppender)\n\nConsoleAppender::ConsoleAppender()\n\t: WriterAppender (std::make_unique<ConsoleAppenderPriv>(getSystemOut()))\n{\n}\n\nConsoleAppender::ConsoleAppender(const LayoutPtr& layout)\n\t: WriterAppender (std::make_unique<ConsoleAppenderPriv>(getSystemOut()))\n{\n\tsetLayout(layout);\n\tPool p;\n\tsetWriter(std::make_shared<SystemOutWriter>());\n\tWriterAppender::activateOptions(p);\n}\n\nConsoleAppender::ConsoleAppender(const LayoutPtr& layout, const LogString& target)\n\t: WriterAppender (std::make_unique<ConsoleAppenderPriv>(target))\n{\n\tsetLayout(layout);\n\tsetTarget(target);\n\tPool p;\n\tConsoleAppender::activateOptions(p);\n}\n\nConsoleAppender::~ConsoleAppender()\n{\n\tfinalize();\n}\n\nconst LogString& ConsoleAppender::getSystemOut()\n{\n\tstatic const WideLife<LogString> name(LOG4CXX_STR(\"System.out\"));\n\treturn name;\n}\n\nconst LogString& ConsoleAppender::getSystemErr()\n{\n\tstatic const WideLife<LogString> name(LOG4CXX_STR(\"System.err\"));\n\treturn name;\n}\n\nvoid ConsoleAppender::setTarget(const LogString& value)\n{\n\tLogString v = StringHelper::trim(value);\n\n\tif (StringHelper::equalsIgnoreCase(v,\n\t\t\tLOG4CXX_STR(\"SYSTEM.OUT\"), LOG4CXX_STR(\"system.out\")))\n\t{\n\t\t_priv->target = getSystemOut();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(v,\n\t\t\tLOG4CXX_STR(\"SYSTEM.ERR\"), LOG4CXX_STR(\"system.err\")))\n\t{\n\t\t_priv->target = getSystemErr();\n\t}\n\telse\n\t{\n\t\ttargetWarn(value);\n\t}\n}\n\nLogString ConsoleAppender::getTarget() const\n{\n\treturn _priv->target;\n}\n\nvoid ConsoleAppender::targetWarn(const LogString& val)\n{\n\tLogLog::warn(((LogString) LOG4CXX_STR(\"[\"))\n\t\t+ val +  LOG4CXX_STR(\"] should be system.out or system.err.\"));\n\tLogLog::warn(LOG4CXX_STR(\"Using previously set target, System.out by default.\"));\n}\n\nvoid ConsoleAppender::activateOptions(Pool& p)\n{\n\tif (StringHelper::equalsIgnoreCase(_priv->target,\n\t\t\tLOG4CXX_STR(\"SYSTEM.OUT\"), LOG4CXX_STR(\"system.out\")))\n\t{\n\t\tWriterPtr writer1 = std::make_shared<SystemOutWriter>();\n\t\tsetWriter(writer1);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(_priv->target,\n\t\t\tLOG4CXX_STR(\"SYSTEM.ERR\"), LOG4CXX_STR(\"system.err\")))\n\t{\n\t\tWriterPtr writer1 = std::make_shared<SystemErrWriter>();\n\t\tsetWriter(writer1);\n\t}\n\n\tWriterAppender::activateOptions(p);\n}\n\nvoid ConsoleAppender::setOption(const LogString& option, const LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"TARGET\"), LOG4CXX_STR(\"target\")))\n\t{\n\t\tsetTarget(value);\n\t}\n\telse\n\t{\n\t\tWriterAppender::setOption(option, value);\n\t}\n}\n\n\n\n\n\n\n\n// Path: src/main/cpp/propertiespatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/propertiespatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\n#include <iterator>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\n#define priv static_cast<PropertiesPatternConverterPrivate*>(m_priv.get())\n\nstruct PropertiesPatternConverter::PropertiesPatternConverterPrivate : public PatternConverterPrivate\n{\n\t\nPropertiesPatternConverterPrivate( const LogString& name, const LogString& style, const LogString& propertyName ) :\n\t\tPatternConverterPrivate( name, style ),\n\t\toption(propertyName) {}\n\n\t/**\n\t * Name of property to output.\n\t */\n\tconst LogString option;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(PropertiesPatternConverter)\n\nPropertiesPatternConverter::PropertiesPatternConverter(const LogString& name1,\n\tconst LogString& propertyName) :\n\tLoggingEventPatternConverter(\n\t\tstd::make_unique<PropertiesPatternConverterPrivate>(name1, LOG4CXX_STR(\"property\"), propertyName))\n{\n}\n\nPatternConverterPtr PropertiesPatternConverter::newInstance(\n\tconst std::vector<LogString>& options)\n{\n\tif (options.size() == 0)\n\t{\n\t\tstatic WideLife<PatternConverterPtr> def = std::make_shared<PropertiesPatternConverter>(\n\t\t\t\tLOG4CXX_STR(\"Properties\"), LOG4CXX_STR(\"\"));\n\t\treturn def;\n\t}\n\n\tLogString converterName(LOG4CXX_STR(\"Property{\"));\n\tconverterName.append(options[0]);\n\tconverterName.append(LOG4CXX_STR(\"}\"));\n\treturn std::make_shared<PropertiesPatternConverter>(converterName, options[0]);\n}\n\nvoid PropertiesPatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& /* p */) const\n{\n\tif (priv->option.length() == 0)\n\t{\n\t\ttoAppendTo.append(1, (logchar) 0x7B /* '{' */);\n\n\t\tfor (auto const& item : event->getMDCKeySet())\n\t\t{\n\t\t\ttoAppendTo.append(1, (logchar) 0x7B /* '{' */);\n\t\t\ttoAppendTo.append(item);\n\t\t\ttoAppendTo.append(1, (logchar) 0x2C /* ',' */);\n\t\t\tevent->getMDC(item, toAppendTo);\n\t\t\ttoAppendTo.append(1, (logchar) 0x7D /* '}' */);\n\t\t}\n\n\t\ttoAppendTo.append(1, (logchar) 0x7D /* '}' */);\n\n\t}\n\telse\n\t{\n\t\tevent->getMDC(priv->option, toAppendTo);\n\t}\n}\n\n\n// Path: src/main/cpp/basicconfigurator.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/basicconfigurator.h>\n#include <log4cxx/patternlayout.h>\n#include <log4cxx/consoleappender.h>\n#include <log4cxx/logmanager.h>\n#include <log4cxx/logger.h>\n#include <log4cxx/helpers/widelife.h>\n\nusing namespace LOG4CXX_NS;\n\nvoid BasicConfigurator::configure(const LayoutPtr& layoutArg)\n{\n\tLogManager::getLoggerRepository()->setConfigured(true);\n\tauto layout = layoutArg;\n\tif (!layout)\n\t{\n\t\tstatic const helpers::WideLife<LogString> TTCC_CONVERSION_PATTERN(LOG4CXX_STR(\"%r [%t] %p %c %x - %m%n\"));\n\t\tlayout = std::make_shared<PatternLayout>(TTCC_CONVERSION_PATTERN);\n\t}\n\tauto appender = std::make_shared<ConsoleAppender>(layout);\n\tLogger::getRootLogger()->addAppender(appender);\n}\n\nvoid BasicConfigurator::configure(const AppenderPtr& appender)\n{\n\tLoggerPtr root = Logger::getRootLogger();\n\troot->addAppender(appender);\n}\n\nvoid BasicConfigurator::resetConfiguration()\n{\n\tLogManager::resetConfiguration();\n}\n\n// Path: src/main/cpp/threadutility.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include \"log4cxx/helpers/threadutility.h\"\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include \"log4cxx/private/log4cxx_private.h\"\n#include \"log4cxx/helpers/loglog.h\"\n#include \"log4cxx/helpers/transcoder.h\"\n\n#include <signal.h>\n#include <mutex>\n\n#if WIN32\n\t#include <windows.h>\n\t#include <processthreadsapi.h>\n#endif\n\nnamespace LOG4CXX_NS\n{\nnamespace helpers\n{\n\nstruct ThreadUtility::priv_data\n{\n\tpriv_data()\n\t{\n\t\tstart_pre = nullptr;\n\t\tstarted = nullptr;\n\t\tstart_post = nullptr;\n\t}\n\n\tThreadStartPre start_pre;\n\tThreadStarted started;\n\tThreadStartPost start_post;\n};\n\n#if LOG4CXX_HAS_PTHREAD_SIGMASK\n\tstatic thread_local sigset_t old_mask;\n\tstatic thread_local bool sigmask_valid;\n#endif\n\nThreadUtility::ThreadUtility() :\n\tm_priv( new priv_data() )\n{\n\t// Block signals by default.\n\tconfigureFuncs( std::bind( &ThreadUtility::preThreadBlockSignals, this ),\n\t\tnullptr,\n\t\tstd::bind( &ThreadUtility::postThreadUnblockSignals, this ) );\n}\n\nThreadUtility::~ThreadUtility() {}\n\nThreadUtility* ThreadUtility::instance()\n{\n\tstatic WideLife<ThreadUtility> instance;\n\treturn &instance.value();\n}\n\nvoid ThreadUtility::configure( ThreadConfigurationType type )\n{\n\tauto utility = instance();\n\n\tif ( type == ThreadConfigurationType::NoConfiguration )\n\t{\n\t\tutility->configureFuncs( nullptr, nullptr, nullptr );\n\t}\n\telse if ( type == ThreadConfigurationType::NameThreadOnly )\n\t{\n\t\tutility->configureFuncs( nullptr,\n\t\t\tstd::bind( &ThreadUtility::threadStartedNameThread, utility,\n\t\t\t\tstd::placeholders::_1,\n\t\t\t\tstd::placeholders::_2,\n\t\t\t\tstd::placeholders::_3 ),\n\t\t\tnullptr );\n\t}\n\telse if ( type == ThreadConfigurationType::BlockSignalsOnly )\n\t{\n\t\tutility->configureFuncs( std::bind( &ThreadUtility::preThreadBlockSignals, utility ),\n\t\t\tnullptr,\n\t\t\tstd::bind( &ThreadUtility::postThreadUnblockSignals, utility ) );\n\t}\n\telse if ( type == ThreadConfigurationType::BlockSignalsAndNameThread )\n\t{\n\t\tutility->configureFuncs( std::bind( &ThreadUtility::preThreadBlockSignals, utility ),\n\t\t\tstd::bind( &ThreadUtility::threadStartedNameThread, utility,\n\t\t\t\tstd::placeholders::_1,\n\t\t\t\tstd::placeholders::_2,\n\t\t\t\tstd::placeholders::_3 ),\n\t\t\tstd::bind( &ThreadUtility::postThreadUnblockSignals, utility ) );\n\t}\n}\n\nvoid ThreadUtility::configureFuncs( ThreadStartPre pre_start,\n\tThreadStarted started,\n\tThreadStartPost post_start )\n{\n\tm_priv->start_pre = pre_start;\n\tm_priv->started = started;\n\tm_priv->start_post = post_start;\n}\n\nvoid ThreadUtility::preThreadBlockSignals()\n{\n#if LOG4CXX_HAS_PTHREAD_SIGMASK\n\tsigset_t set;\n\tsigfillset(&set);\n\n\tif ( pthread_sigmask(SIG_SETMASK, &set, &old_mask) < 0 )\n\t{\n\t\tLOGLOG_ERROR( LOG4CXX_STR(\"Unable to set thread sigmask\") );\n\t\tsigmask_valid = false;\n\t}\n\telse\n\t{\n\t\tsigmask_valid = true;\n\t}\n\n#endif /* LOG4CXX_HAS_PTHREAD_SIGMASK */\n}\n\nvoid ThreadUtility::threadStartedNameThread(LogString threadName,\n\tstd::thread::id /*threadId*/,\n\tstd::thread::native_handle_type nativeHandle)\n{\n#if LOG4CXX_HAS_PTHREAD_SETNAME\n\tLOG4CXX_ENCODE_CHAR(sthreadName, threadName);\n\tif (pthread_setname_np(static_cast<pthread_t>(nativeHandle), sthreadName.c_str()) < 0) {\n\t\tLOGLOG_ERROR(LOG4CXX_STR(\"unable to set thread name\"));\n\t}\n#elif WIN32\n\ttypedef HRESULT (WINAPI *TSetThreadDescription)(HANDLE, PCWSTR);\n\tstatic struct initialiser\n\t{\n\t\tHMODULE hKernelBase;\n\t\tTSetThreadDescription SetThreadDescription;\n\t\tinitialiser()\n\t\t\t: hKernelBase(GetModuleHandleA(\"KernelBase.dll\"))\n\t\t\t, SetThreadDescription(nullptr)\n\t\t{\n\t\t\tif (hKernelBase)\n\t\t\t\tSetThreadDescription = reinterpret_cast<TSetThreadDescription>(GetProcAddress(hKernelBase, \"SetThreadDescription\"));\n\t\t}\n\t} win32Func;\n\tif (win32Func.SetThreadDescription)\n\t{\n\t\tLOG4CXX_ENCODE_WCHAR(wthreadName, threadName);\n\t\tif(FAILED(win32Func.SetThreadDescription(static_cast<HANDLE>(nativeHandle), wthreadName.c_str())))\n\t\t\tLOGLOG_ERROR( LOG4CXX_STR(\"unable to set thread name\") );\n\t}\n#endif\n}\n\nvoid ThreadUtility::postThreadUnblockSignals()\n{\n#if LOG4CXX_HAS_PTHREAD_SIGMASK\n\n\t// Only restore the signal mask if we were able to set it in the first place.\n\tif ( sigmask_valid )\n\t{\n\t\tif ( pthread_sigmask(SIG_SETMASK, &old_mask, nullptr) < 0 )\n\t\t{\n\t\t\tLOGLOG_ERROR( LOG4CXX_STR(\"Unable to set thread sigmask\") );\n\t\t}\n\t}\n\n#endif /* LOG4CXX_HAS_PTHREAD_SIGMASK */\n}\n\n\nThreadStartPre ThreadUtility::preStartFunction()\n{\n\treturn m_priv->start_pre;\n}\n\nThreadStarted ThreadUtility::threadStartedFunction()\n{\n\treturn m_priv->started;\n}\n\nThreadStartPost ThreadUtility::postStartFunction()\n{\n\treturn m_priv->start_post;\n}\n\n} //namespace helpers\n} //namespace log4cxx\n\n// Path: src/main/cpp/inetaddress.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/inetaddress.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/pool.h>\n\n#include \"apr_network_io.h\"\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(InetAddress)\n\nstruct InetAddress::InetAddressPrivate{\n\n\tLogString ipAddrString;\n\tLogString hostNameString;\n};\n\nUnknownHostException::UnknownHostException(const LogString& msg1)\n\t: Exception(msg1)\n{\n}\n\nUnknownHostException::UnknownHostException(const UnknownHostException& src)\n\t: Exception(src)\n{\n}\n\nUnknownHostException& UnknownHostException::operator=(const UnknownHostException& src)\n{\n\tException::operator=(src);\n\treturn *this;\n}\n\n\nInetAddress::InetAddress(const LogString& hostName, const LogString& hostAddr)\n\t: m_priv(std::make_unique<InetAddressPrivate>())\n{\n\tm_priv->ipAddrString = hostAddr;\n\tm_priv->hostNameString = hostName;\n}\n\nInetAddress::~InetAddress(){}\n\n/** Determines all the IP addresses of a host, given the host's name.\n*/\nstd::vector<InetAddressPtr> InetAddress::getAllByName(const LogString& host)\n{\n\tLOG4CXX_ENCODE_CHAR(encodedHost, host);\n\n\t// retrieve information about the given host\n\tPool addrPool;\n\n\tapr_sockaddr_t* address = 0;\n\tapr_status_t status =\n\t\tapr_sockaddr_info_get(&address, encodedHost.c_str(),\n...\n", "needle_token_start": 11412, "needle_token_end": 11457, "code_context_ntokens": 15257, "output": ["```cpp\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/propertiespatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\n#include <iterator>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\n#define priv static_cast<PropertiesPatternConverterPrivate*>(m_priv.get())\n\nstruct PropertiesPatternConverter::PropertiesPatternConverterPrivate : public PatternConverterPrivate\n{\n\t\nPropertiesPatternConverterPrivate( const LogString& name, const LogString& style, const LogString& propertyName ) :\n\t\tPatternConverterPrivate( name, style ),\n\t\toption(propertyName) {}\n\n\t/**\n\t * Name of property to output.\n\t */\n\tconst LogString option;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(PropertiesPatternConverter)\n\nPropertiesPatternConverter::PropertiesPatternConverter(const LogString& name1,\n\tconst LogString& propertyName) :\n\tLoggingEventPatternConverter(\n\t\tstd::make_unique<PropertiesPatternConverterPrivate>(name1, LOG4CXX_STR(\"property\"), propertyName))\n{\n}\n\nPatternConverterPtr PropertiesPatternConverter::newInstance(\n\tconst std::vector<LogString>& options)\n{\n\tif (options.size() == 0)\n\t{\n\t\tstatic WideLife<PatternConverterPtr> def = std::make_shared<PropertiesPatternConverter>(\n\t\t\t\tLOG4CXX_STR(\"Properties\"), LOG4CXX_STR(\"\"));\n\t\treturn def;\n\t}\n\n\tLogString converterName(LOG4CXX_STR(\"Property{\"));\n\tconverterName.append(options[0]);\n\tconverterName.append(LOG4CXX_STR(\"}\"));\n\treturn std::make_shared<PropertiesPatternConverter>(converterName, options[0]);\n}\n\nvoid PropertiesPatternConverter::format(\n\tconst LoggingEventPtr& event,\n\tLogString& toAppendTo,\n\tPool& /* p */) const\n{\n\tif (priv->option.length() == 0)\n\t{\n\t\ttoAppendTo.append(1, (logchar) 0x7B /* '{' */);\n\n\t\tfor (auto const& item : event->getMDCKeySet())\n\t\t{\n\t\t\ttoAppendTo.append(1, (logchar) 0x7B /* '{' */);\n\t\t\ttoAppendTo.append(item);\n\t\t\ttoAppendTo.append(1, (logchar) 0x2C /* ',' */);\n\t\t\tevent->getMDC(item, toAppendTo);\n\t\t\ttoAppendTo.append(1, (logchar) 0x7D /* '}' */);\n\t\t}\n\n\t\ttoAppendTo.append(1, (logchar) 0x7D /* '}' */);\n\n\t}\n\telse\n\t{\n\t\tevent->getMDC(priv->option, toAppendTo);\n\t}\n}\n```"]}
{"repo": "apache/logging-log4cxx", "name": "ThrowableInformationPatternConverterPrivate", "language": "cpp", "path": "src/main/cpp/throwableinformationpatternconverter.cpp", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To configure and handle the conversion of throwable information from logging events into a formatted string, with an option for abbreviated content.\n2. **Input**: Takes a name and style for formatting, along with a boolean to determine if the throwable information should be reported in a shortened form.\n3. **Output**: Does not directly output data; instead, it sets up the internal configuration for how throwable information should be formatted when converted.\n4. **Procedure**: Initializes the internal settings based on the provided parameters, setting up how throwable information will be processed and formatted during logging operations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/fileappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/fileappender.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/pool.h>\n#include <log4cxx/helpers/fileoutputstream.h>\n#include <log4cxx/helpers/outputstreamwriter.h>\n#include <log4cxx/helpers/bufferedwriter.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/private/writerappender_priv.h>\n#include <log4cxx/private/fileappender_priv.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\nIMPLEMENT_LOG4CXX_OBJECT(FileAppender)\n\n#define _priv static_cast<FileAppenderPriv*>(m_priv.get())\n\nFileAppender::FileAppender() :\n\tWriterAppender (std::make_unique<FileAppenderPriv>())\n{\n}\n\nFileAppender::FileAppender\n\t( const LayoutPtr& layout1\n\t, const LogString& fileName1\n\t, bool append1\n\t, bool bufferedIO1\n\t, int bufferSize1\n\t)\n\t: WriterAppender(std::make_unique<FileAppenderPriv>(layout1, fileName1, append1, bufferedIO1, bufferSize1))\n{\n\tPool p;\n\tactivateOptions(p);\n}\n\nFileAppender::FileAppender\n\t( const LayoutPtr& layout1\n\t, const LogString& fileName1\n\t, bool append1\n\t)\n\t: WriterAppender(std::make_unique<FileAppenderPriv>(layout1, fileName1, append1, false))\n{\n\tPool p;\n\tactivateOptions(p);\n}\n\nFileAppender::FileAppender(const LayoutPtr& layout1, const LogString& fileName1)\n\t: WriterAppender(std::make_unique<FileAppenderPriv>(layout1, fileName1))\n{\n\tPool p;\n\tactivateOptions(p);\n}\n\nFileAppender::FileAppender(std::unique_ptr<FileAppenderPriv> priv)\n\t: WriterAppender (std::move(priv))\n{\n}\n\nFileAppender::~FileAppender()\n{\n\tfinalize();\n}\n\nvoid FileAppender::setAppend(bool fileAppend1)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t_priv->fileAppend = fileAppend1;\n}\n\nvoid FileAppender::setFile(const LogString& file)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\tsetFileInternal(file);\n}\n\nvoid FileAppender::setFileInternal(const LogString& file)\n{\n\t_priv->fileName = file;\n}\n\nvoid FileAppender::setBufferedIO(bool bufferedIO1)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t_priv->bufferedIO = bufferedIO1;\n\n\tif (bufferedIO1)\n\t{\n\t\tsetImmediateFlush(false);\n\t}\n}\n\nvoid FileAppender::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"FILE\"), LOG4CXX_STR(\"file\"))\n\t\t|| StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"FILENAME\"), LOG4CXX_STR(\"filename\")))\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t_priv->fileName = stripDuplicateBackslashes(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"APPEND\"), LOG4CXX_STR(\"append\")))\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t_priv->fileAppend = OptionConverter::toBoolean(value, true);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"BUFFEREDIO\"), LOG4CXX_STR(\"bufferedio\")))\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t_priv->bufferedIO = OptionConverter::toBoolean(value, false);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"IMMEDIATEFLUSH\"), LOG4CXX_STR(\"immediateflush\")))\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t_priv->bufferedIO = !OptionConverter::toBoolean(value, false);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"BUFFERSIZE\"), LOG4CXX_STR(\"buffersize\")))\n\t{\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t\t_priv->bufferSize = OptionConverter::toFileSize(value, 8 * 1024);\n\t}\n\telse\n\t{\n\t\tWriterAppender::setOption(option, value);\n\t}\n}\n\nvoid FileAppender::activateOptions(Pool& p)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\tactivateOptionsInternal(p);\n}\n\nvoid FileAppender::activateOptionsInternal(Pool& p)\n{\n\tint errors = 0;\n\n\tif (!_priv->fileName.empty())\n\t{\n\t\ttry\n\t\t{\n\t\t\tsetFileInternal(_priv->fileName, _priv->fileAppend, _priv->bufferedIO, _priv->bufferSize, p);\n\t\t}\n\t\tcatch (IOException& e)\n\t\t{\n\t\t\terrors++;\n\t\t\tLogString msg(LOG4CXX_STR(\"setFile(\"));\n\t\t\tmsg.append(_priv->fileName);\n\t\t\tmsg.append(1, (logchar) 0x2C /* ',' */);\n\t\t\tStringHelper::toString(_priv->fileAppend, msg);\n\t\t\tmsg.append(LOG4CXX_STR(\") call failed.\"));\n\t\t\t_priv->errorHandler->error(msg, e, ErrorCode::FILE_OPEN_FAILURE);\n\t\t}\n\t}\n\telse\n\t{\n\t\terrors++;\n\t\tLogLog::error(LogString(LOG4CXX_STR(\"File option not set for appender [\"))\n\t\t\t+  _priv->name + LOG4CXX_STR(\"].\"));\n\t\tLogLog::warn(LOG4CXX_STR(\"Are you using FileAppender instead of ConsoleAppender?\"));\n\t}\n\n\tif (errors == 0)\n\t{\n\t\tWriterAppender::activateOptions(p);\n\t}\n}\n\n\n/**\n * Replaces double backslashes (except the leading doubles of UNC's)\n * with single backslashes for compatibility with existing path\n * specifications that were working around use of\n * OptionConverter::convertSpecialChars in XML configuration files.\n *\n * @param src source string\n * @return modified string\n *\n *\n */\n...\n// Path: src/main/cpp/transform.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/transform.h>\n#include <log4cxx/helpers/widelife.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\n\n\nvoid Transform::appendEscapingTags(\n\tLogString& buf, const LogString& input)\n{\n\t//Check if the string is zero length -- if so, return\n\t//what was sent in.\n\n\tif (input.length() == 0 )\n\t{\n\t\treturn;\n\t}\n\n\tlogchar specials[] = { 0x22 /* \" */, 0x26 /* & */, 0x3C /* < */, 0x3E /* > */, 0x00 };\n\tsize_t start = 0;\n\tsize_t special = input.find_first_of(specials, start);\n\n\twhile (special != LogString::npos)\n\t{\n\t\tif (special > start)\n\t\t{\n\t\t\tbuf.append(input, start, special - start);\n\t\t}\n\n\t\tswitch (input[special])\n\t\t{\n\t\t\tcase 0x22:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&quot;\"));\n\t\t\t\tbreak;\n\n\t\t\tcase 0x26:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&amp;\"));\n\t\t\t\tbreak;\n\n\t\t\tcase 0x3C:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&lt;\"));\n\t\t\t\tbreak;\n\n\t\t\tcase 0x3E:\n\t\t\t\tbuf.append(LOG4CXX_STR(\"&gt;\"));\n\t\t\t\tbreak;\n\n\t\t\tdefault:\n\t\t\t\tbuf.append(1, input[special]);\n\t\t\t\tbreak;\n\t\t}\n\n\t\tstart = special + 1;\n\n\t\tif (special < input.size())\n\t\t{\n\t\t\tspecial = input.find_first_of(specials, start);\n\t\t}\n\t\telse\n\t\t{\n\t\t\tspecial = LogString::npos;\n\t\t}\n\t}\n\n\tif (start < input.size())\n\t{\n\t\tbuf.append(input, start, input.size() - start);\n\t}\n}\n\nvoid Transform::appendEscapingCDATA(\n\tLogString& buf, const LogString& input)\n{\n\tstatic const WideLife<LogString> CDATA_END(LOG4CXX_STR(\"]]>\"));\n\tstatic const WideLife<LogString> CDATA_EMBEDED_END(LOG4CXX_STR(\"]]>]]&gt;<![CDATA[\"));\n\n\tconst LogString::size_type CDATA_END_LEN = 3;\n\n\n\tif (input.length() == 0 )\n\t{\n\t\treturn;\n\t}\n\n\tLogString::size_type end = input.find(CDATA_END);\n\n\tif (end == LogString::npos)\n\t{\n\t\tbuf.append(input);\n\t\treturn;\n\t}\n\n\tLogString::size_type start = 0;\n\n\twhile (end != LogString::npos)\n\t{\n\t\tbuf.append(input, start, end - start);\n\t\tbuf.append(CDATA_EMBEDED_END);\n\t\tstart = end + CDATA_END_LEN;\n\n\t\tif (start < input.length())\n\t\t{\n\t\t\tend = input.find(CDATA_END, start);\n\t\t}\n\t\telse\n\t\t{\n\t\t\treturn;\n\t\t}\n\t}\n\n\tbuf.append(input, start, input.length() - start);\n}\n\n\n// Path: src/main/cpp/telnetappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/net/telnetappender.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/threadutility.h>\n#include <log4cxx/private/appenderskeleton_priv.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::net;\n\nIMPLEMENT_LOG4CXX_OBJECT(TelnetAppender)\n\nstruct TelnetAppender::TelnetAppenderPriv : public AppenderSkeletonPrivate\n{\n\tTelnetAppenderPriv( int port, int maxConnections ) : AppenderSkeletonPrivate(),\n\t\tport(port),\n\t\tconnections(maxConnections),\n\t\tencoding(LOG4CXX_STR(\"UTF-8\")),\n\t\tencoder(CharsetEncoder::getUTF8Encoder()),\n\t\tsh(),\n\t\tactiveConnections(0) {}\n\n\tint port;\n\tConnectionList connections;\n\tLogString encoding;\n\tLOG4CXX_NS::helpers::CharsetEncoderPtr encoder;\n\tstd::unique_ptr<helpers::ServerSocket> serverSocket;\n\tstd::thread sh;\n\tsize_t activeConnections;\n};\n\n#define _priv static_cast<TelnetAppenderPriv*>(m_priv.get())\n\n/** The default telnet server port */\nconst int TelnetAppender::DEFAULT_PORT = 23;\n\n/** The maximum number of concurrent connections */\nconst int TelnetAppender::MAX_CONNECTIONS = 20;\n\nTelnetAppender::TelnetAppender()\n\t: AppenderSkeleton (std::make_unique<TelnetAppenderPriv>(DEFAULT_PORT, MAX_CONNECTIONS))\n{\n}\n\nTelnetAppender::~TelnetAppender()\n{\n\tfinalize();\n}\n\nvoid TelnetAppender::activateOptions(Pool& /* p */)\n{\n\tif (_priv->serverSocket == NULL)\n\t{\n\t\t_priv->serverSocket = ServerSocket::create(_priv->port);\n\t\t_priv->serverSocket->setSoTimeout(1000);\n\t}\n\n\t_priv->sh = ThreadUtility::instance()->createThread( LOG4CXX_STR(\"TelnetAppender\"), &TelnetAppender::acceptConnections, this );\n}\n\nvoid TelnetAppender::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\tif (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"PORT\"), LOG4CXX_STR(\"port\")))\n\t{\n\t\tsetPort(OptionConverter::toInt(value, DEFAULT_PORT));\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option, LOG4CXX_STR(\"ENCODING\"), LOG4CXX_STR(\"encoding\")))\n\t{\n\t\tsetEncoding(value);\n\t}\n\telse\n\t{\n\t\tAppenderSkeleton::setOption(option, value);\n\t}\n}\n\nLogString TelnetAppender::getEncoding() const\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\treturn _priv->encoding;\n}\n\nvoid TelnetAppender::setEncoding(const LogString& value)\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\t_priv->encoder = CharsetEncoder::getEncoder(value);\n\t_priv->encoding = value;\n}\n\n\nvoid TelnetAppender::close()\n{\n\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\tif (_priv->closed)\n\t{\n\t\treturn;\n\t}\n\n\t_priv->closed = true;\n\n\tSocketPtr nullSocket;\n\n\tfor (auto& item : _priv->connections)\n\t{\n\t\tif (item)\n\t\t{\n\t\t\titem->close();\n\t\t\titem = nullSocket;\n\t\t}\n\t}\n\n\tif (_priv->serverSocket != NULL)\n\t{\n\t\ttry\n\t\t{\n\t\t\t_priv->serverSocket->close();\n\t\t}\n\t\tcatch (Exception&)\n\t\t{\n\t\t}\n\t}\n\n\tif ( _priv->sh.joinable() )\n\t{\n\t\t_priv->sh.join();\n\t}\n\n\t_priv->activeConnections = 0;\n}\n\n\nvoid TelnetAppender::write(ByteBuffer& buf)\n{\n\tfor (auto& item :_priv->connections)\n\t{\n\t\tif (item)\n\t\t{\n\t\t\ttry\n\t\t\t{\n\t\t\t\tByteBuffer b(buf.current(), buf.remaining());\n\t\t\t\titem->write(b);\n\t\t\t}\n\t\t\tcatch (Exception&)\n\t\t\t{\n\t\t\t\t// The client has closed the connection, remove it from our list:\n\t\t\t\titem.reset();\n\t\t\t\t_priv->activeConnections--;\n\t\t\t}\n\t\t}\n\t}\n}\n\nvoid TelnetAppender::writeStatus(const SocketPtr& socket, const LogString& msg, Pool& p)\n{\n\tsize_t bytesSize = msg.size() * 2;\n\tchar* bytes = p.pstralloc(bytesSize);\n\n\tLogString::const_iterator msgIter(msg.begin());\n\tByteBuffer buf(bytes, bytesSize);\n\n\twhile (msgIter != msg.end())\n\t{\n\t\t_priv->encoder->encode(msg, msgIter, buf);\n\t\tbuf.flip();\n\t\tsocket->write(buf);\n\t\tbuf.clear();\n\t}\n}\n\nvoid TelnetAppender::append(const spi::LoggingEventPtr& event, Pool& p)\n{\n\tsize_t count = _priv->activeConnections;\n\n\tif (count > 0)\n\t{\n\t\tLogString msg;\n\t\t_priv->layout->format(msg, event, _priv->pool);\n\t\tmsg.append(LOG4CXX_STR(\"\\r\\n\"));\n\t\tsize_t bytesSize = msg.size() * 2;\n\t\tchar* bytes = p.pstralloc(bytesSize);\n\n\t\tLogString::const_iterator msgIter(msg.begin());\n\t\tByteBuffer buf(bytes, bytesSize);\n\n\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\t\twhile (msgIter != msg.end())\n\t\t{\n\t\t\tlog4cxx_status_t stat = _priv->encoder->encode(msg, msgIter, buf);\n\t\t\tbuf.flip();\n\t\t\twrite(buf);\n\t\t\tbuf.clear();\n\n\t\t\tif (CharsetEncoder::isError(stat))\n\t\t\t{\n\t\t\t\tLogString unrepresented(1, 0x3F /* '?' */);\n\t\t\t\tLogString::const_iterator unrepresentedIter(unrepresented.begin());\n\t\t\t\tstat = _priv->encoder->encode(unrepresented, unrepresentedIter, buf);\n\t\t\t\tbuf.flip();\n\t\t\t\twrite(buf);\n\t\t\t\tbuf.clear();\n\t\t\t\tmsgIter++;\n\t\t\t}\n\t\t}\n\t}\n}\n\nvoid TelnetAppender::acceptConnections()\n{\n\n\t// main loop; is left when This->closed is != 0 after an accept()\n\twhile (true)\n\t{\n\t\ttry\n\t\t{\n\t\t\tSocketPtr newClient = _priv->serverSocket->accept();\n\t\t\tbool done = _priv->closed;\n\n\t\t\tif (done)\n\t\t\t{\n\t\t\t\tPool p;\n\t\t\t\twriteStatus(newClient, LOG4CXX_STR(\"Log closed.\\r\\n\"), p);\n\t\t\t\tnewClient->close();\n\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tsize_t count = _priv->activeConnections;\n\n\t\t\tif (count >= _priv->connections.size())\n\t\t\t{\n\t\t\t\tPool p;\n\t\t\t\twriteStatus(newClient, LOG4CXX_STR(\"Too many connections.\\r\\n\"), p);\n\t\t\t\tnewClient->close();\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t//\n\t\t\t\t//   find unoccupied connection\n\t\t\t\t//\n\t\t\t\tstd::lock_guard<std::recursive_mutex> lock(_priv->mutex);\n\n\t\t\t\tfor (auto& item : _priv->connections)\n\t\t\t\t{\n\t\t\t\t\tif (!item)\n\t\t\t\t\t{\n\t\t\t\t\t\titem = newClient;\n\t\t\t\t\t\t_priv->activeConnections++;\n\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tPool p;\n\t\t\t\tLogString oss(LOG4CXX_STR(\"TelnetAppender v1.0 (\"));\n\t\t\t\tStringHelper::toString((int) count + 1, p, oss);\n\t\t\t\toss += LOG4CXX_STR(\" active connections)\\r\\n\\r\\n\");\n\t\t\t\twriteStatus(newClient, oss, p);\n\t\t\t}\n\t\t}\n\t\tcatch (InterruptedIOException&)\n\t\t{\n\t\t\tif (_priv->closed)\n\t\t\t{\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t\tcatch (Exception& e)\n\t\t{\n\t\t\tif (!_priv->closed)\n\t\t\t{\n\t\t\t\tLogLog::error(LOG4CXX_STR(\"Encountered error while in SocketHandler loop.\"), e);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\n}\n\nint TelnetAppender::getPort() const\n{\n\treturn _priv->port;\n}\n\nvoid TelnetAppender::setPort(int port1)\n{\n\t_priv->port = port1;\n}\n\n// Path: src/main/cpp/outputdebugstringappender.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#if defined(_WIN32)\n#include <log4cxx/nt/outputdebugstringappender.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/private/appenderskeleton_priv.h>\n\n#include \"windows.h\"\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::nt;\n\nIMPLEMENT_LOG4CXX_OBJECT(OutputDebugStringAppender)\n\nOutputDebugStringAppender::OutputDebugStringAppender()\n{\n}\n\nvoid OutputDebugStringAppender::append(const spi::LoggingEventPtr& event, Pool& p)\n{\n\tLogString buf;\n\tthis->m_priv->layout->format(buf, event, p);\n#if LOG4CXX_WCHAR_T_API\n\tLOG4CXX_ENCODE_WCHAR(wstr, buf);\n\t::OutputDebugStringW(wstr.c_str());\n#else\n\tLOG4CXX_ENCODE_CHAR(str, buf);\n\t::OutputDebugStringA(str.c_str());\n#endif\n}\n\n#endif\n\n\n// Path: src/main/cpp/charsetdecoder.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#define NOMINMAX /* tell windows not to define min/max macros */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/charsetdecoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/exception.h>\n#include <log4cxx/helpers/pool.h>\n#include <log4cxx/helpers/loglog.h>\n#include <apr_xlate.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n#include <locale.h>\n#include <apr_portable.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(CharsetDecoder)\n\n\nnamespace LOG4CXX_NS\n{\nnamespace helpers\n{\n\n#if APR_HAS_XLATE\n/**\n *  Converts from an arbitrary encoding to LogString\n *    using apr_xlate.  Requires real iconv implementation,\n*    apr-iconv will crash in use.\n */\nclass APRCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\t/**\n\t\t *  Creates a new instance.\n\t\t *  @param frompage name of source encoding.\n\t\t */\n\t\tAPRCharsetDecoder(const LogString& frompage) : pool()\n\t\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\t\t\tconst char* topage = \"WCHAR_T\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\tconst char* topage = \"UTF-8\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\t\tconst char* topage = \"UTF-16\";\n#endif\n\t\t\tstd::string fpage(Transcoder::encodeCharsetName(frompage));\n\t\t\tapr_status_t stat = apr_xlate_open(&convset,\n\t\t\t\t\ttopage,\n\t\t\t\t\tfpage.c_str(),\n\t\t\t\t\tpool.getAPRPool());\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tthrow IllegalArgumentException(frompage);\n\t\t\t}\n\t\t}\n\n\t\t/**\n\t\t *  Destructor.\n\t\t */\n\t\tvirtual ~APRCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tenum { BUFSIZE = 256 };\n\t\t\tlogchar buf[BUFSIZE];\n\t\t\tconst apr_size_t initial_outbytes_left = BUFSIZE * sizeof(logchar);\n\t\t\tapr_status_t stat = APR_SUCCESS;\n\n\t\t\tif (in.remaining() == 0)\n\t\t\t{\n\t\t\t\tsize_t outbytes_left = initial_outbytes_left;\n\t\t\t\t{\n\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\tstat = apr_xlate_conv_buffer((apr_xlate_t*) convset,\n\t\t\t\t\t\t\tNULL, NULL, (char*) buf, &outbytes_left);\n\t\t\t\t}\n\t\t\t\tout.append(buf, (initial_outbytes_left - outbytes_left) / sizeof(logchar));\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\twhile (in.remaining() > 0 && stat == APR_SUCCESS)\n\t\t\t\t{\n\t\t\t\t\tsize_t inbytes_left = in.remaining();\n\t\t\t\t\tsize_t initial_inbytes_left = inbytes_left;\n\t\t\t\t\tsize_t pos = in.position();\n\t\t\t\t\tapr_size_t outbytes_left = initial_outbytes_left;\n\t\t\t\t\t{\n\t\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\t\tstat = apr_xlate_conv_buffer((apr_xlate_t*) convset,\n\t\t\t\t\t\t\t\tin.data() + pos,\n\t\t\t\t\t\t\t\t&inbytes_left,\n\t\t\t\t\t\t\t\t(char*) buf,\n\t\t\t\t\t\t\t\t&outbytes_left);\n\t\t\t\t\t}\n\t\t\t\t\tout.append(buf, (initial_outbytes_left - outbytes_left) / sizeof(logchar));\n\t\t\t\t\tin.position(pos + (initial_inbytes_left - inbytes_left));\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tAPRCharsetDecoder(const APRCharsetDecoder&);\n\t\tAPRCharsetDecoder& operator=(const APRCharsetDecoder&);\n\t\tLOG4CXX_NS::helpers::Pool pool;\n\t\tstd::mutex mutex;\n\t\tapr_xlate_t* convset;\n};\n\n#endif\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_MBSRTOWCS\n/**\n*    Converts from the default multi-byte string to\n*        LogString using mbstowcs.\n*\n*/\nclass MbstowcsCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tMbstowcsCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~MbstowcsCharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tinline log4cxx_status_t append(LogString& out, const wchar_t* buf)\n\t\t{\n\t\t\tout.append(buf);\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\t\t\tenum { BUFSIZE = 256 };\n\t\t\twchar_t wbuf[BUFSIZE];\n\t\t\tchar cbuf[BUFSIZE*4];\n\n\t\t\tmbstate_t mbstate;\n\t\t\tmemset(&mbstate, 0, sizeof(mbstate));\n\n\t\t\twhile (in.remaining() > 0)\n\t\t\t{\n\t\t\t\tconst char* src = in.current();\n\n\t\t\t\tif (*src == 0)\n\t\t\t\t{\n\t\t\t\t\tout.append(1, (logchar) 0);\n\t\t\t\t\tin.position(in.position() + 1);\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\tauto available = std::min(sizeof (cbuf) - 1, in.remaining());\n\t\t\t\t\tstrncpy(cbuf, src, available);\n\t\t\t\t\tcbuf[available] = 0;\n\t\t\t\t\tsrc = cbuf;\n\t\t\t\t\tsize_t wCharCount = mbsrtowcs(wbuf,\n\t\t\t\t\t\t\t&src,\n\t\t\t\t\t\t\tBUFSIZE - 1,\n\t\t\t\t\t\t\t&mbstate);\n\t\t\t\t\tauto converted = src - cbuf;\n\t\t\t\t\tin.position(in.position() + converted);\n\n\t\t\t\t\tif (wCharCount == (size_t) -1) // Illegal byte sequence?\n\t\t\t\t\t{\n\t\t\t\t\t\tLogString msg(LOG4CXX_STR(\"Illegal byte sequence at \"));\n\t\t\t\t\t\tmsg.append(std::to_wstring(in.position()));\n\t\t\t\t\t\tmsg.append(LOG4CXX_STR(\" of \"));\n\t\t\t\t\t\tmsg.append(std::to_wstring(in.limit()));\n\t\t\t\t\t\tLogLog::warn(msg);\n\t\t\t\t\t\tstat = APR_BADCH;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\twbuf[wCharCount] = 0;\n\t\t\t\t\t\tstat = append(out, wbuf);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\n\tprivate:\n\t\tMbstowcsCharsetDecoder(const MbstowcsCharsetDecoder&);\n\t\tMbstowcsCharsetDecoder& operator=(const MbstowcsCharsetDecoder&);\n};\n#endif\n\n\n/**\n*    Decoder used when the external and internal charsets\n*    are the same.\n*\n*/\nclass TrivialCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tTrivialCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~TrivialCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tsize_t remaining = in.remaining();\n\n\t\t\tif ( remaining > 0)\n\t\t\t{\n\t\t\t\tconst logchar* src = (const logchar*) (in.data() + in.position());\n\t\t\t\tsize_t count = remaining / sizeof(logchar);\n\t\t\t\tout.append(src, count);\n\t\t\t\tin.position(in.position() + remaining);\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\n\n\tprivate:\n\t\tTrivialCharsetDecoder(const TrivialCharsetDecoder&);\n\t\tTrivialCharsetDecoder& operator=(const TrivialCharsetDecoder&);\n};\n\n\n#if LOG4CXX_LOGCHAR_IS_UTF8\ntypedef TrivialCharsetDecoder UTF8CharsetDecoder;\n#else\n/**\n*    Converts from UTF-8 to std::wstring\n*\n*/\nclass UTF8CharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tUTF8CharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~UTF8CharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tif (in.remaining() > 0)\n\t\t\t{\n\t\t\t\tstd::string tmp(in.current(), in.remaining());\n\t\t\t\tstd::string::const_iterator iter = tmp.begin();\n\n\t\t\t\twhile (iter != tmp.end())\n\t\t\t\t{\n\t\t\t\t\tunsigned int sv = Transcoder::decode(tmp, iter);\n\n\t\t\t\t\tif (sv == 0xFFFF)\n\t\t\t\t\t{\n\t\t\t\t\t\tsize_t offset = iter - tmp.begin();\n\t\t\t\t\t\tin.position(in.position() + offset);\n\t\t\t\t\t\treturn APR_BADARG;\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\tTranscoder::encode(sv, out);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tin.position(in.limit());\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\tprivate:\n\t\tUTF8CharsetDecoder(const UTF8CharsetDecoder&);\n\t\tUTF8CharsetDecoder& operator=(const UTF8CharsetDecoder&);\n};\n#endif\n\n/**\n*    Converts from ISO-8859-1 to LogString.\n*\n*/\nclass ISOLatinCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tISOLatinCharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~ISOLatinCharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tif (in.remaining() > 0)\n\t\t\t{\n\n\t\t\t\tconst unsigned char* src = (unsigned char*) in.current();\n\t\t\t\tconst unsigned char* srcEnd = src + in.remaining();\n\n\t\t\t\twhile (src < srcEnd)\n\t\t\t\t{\n\t\t\t\t\tunsigned int sv = *(src++);\n\t\t\t\t\tTranscoder::encode(sv, out);\n\t\t\t\t}\n\n\t\t\t\tin.position(in.limit());\n\t\t\t}\n\n\t\t\treturn APR_SUCCESS;\n\t\t}\n\n\n\n\tprivate:\n\t\tISOLatinCharsetDecoder(const ISOLatinCharsetDecoder&);\n\t\tISOLatinCharsetDecoder& operator=(const ISOLatinCharsetDecoder&);\n};\n\n\n/**\n*    Converts from US-ASCII to LogString.\n*\n*/\nclass USASCIICharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tUSASCIICharsetDecoder()\n\t\t{\n\t\t}\n\n\t\tvirtual ~USASCIICharsetDecoder()\n\t\t{\n\t\t}\n\n\tprivate:\n\n\t\tvirtual log4cxx_status_t decode(ByteBuffer& in,\n\t\t\tLogString& out)\n\t\t{\n\t\t\tlog4cxx_status_t stat = APR_SUCCESS;\n\n\t\t\tif (in.remaining() > 0)\n\t\t\t{\n\n\t\t\t\tconst unsigned char* src = (unsigned char*) in.current();\n\t\t\t\tconst unsigned char* srcEnd = src + in.remaining();\n\n\t\t\t\twhile (src < srcEnd)\n\t\t\t\t{\n\t\t\t\t\tunsigned char sv = *src;\n\n\t\t\t\t\tif (sv < 0x80)\n\t\t\t\t\t{\n\t\t\t\t\t\tsrc++;\n\t\t\t\t\t\tTranscoder::encode(sv, out);\n\t\t\t\t\t}\n\t\t\t\t\telse\n\t\t\t\t\t{\n\t\t\t\t\t\tstat = APR_BADARG;\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tin.position(src - (const unsigned char*) in.data());\n\t\t\t}\n\n\t\t\treturn stat;\n\t\t}\n\n\n\n\tprivate:\n\t\tUSASCIICharsetDecoder(const USASCIICharsetDecoder&);\n\t\tUSASCIICharsetDecoder& operator=(const USASCIICharsetDecoder&);\n};\n\n/**\n *    Charset decoder that uses current locale settings.\n */\nclass LocaleCharsetDecoder : public CharsetDecoder\n{\n\tpublic:\n\t\tLocaleCharsetDecoder() : state()\n\t\t{\n\t\t}\n\t\tlog4cxx_status_t decode(ByteBuffer& in, LogString& out) override\n\t\t{\n\t\t\tlog4cxx_status_t result = APR_SUCCESS;\n\t\t\tconst char* p = in.current();\n\t\t\tsize_t i = in.position();\n\t\t\tsize_t remain = in.limit() - i;\n#if !LOG4CXX_CHARSET_EBCDIC\n\t\t\tif (std::mbsinit(&this->state)) // ByteBuffer not partially decoded?\n\t\t\t{\n\t\t\t\t// Copy single byte characters\n\t\t\t\tfor (; 0 < remain && ((unsigned int) *p) < 0x80; --remain, ++i, p++)\n\t\t\t\t{\n\t\t\t\t\tout.append(1, *p);\n\t\t\t\t}\n\t\t\t}\n#endif\n\t\t\t// Decode characters that may be represented by multiple bytes\n\t\t\twhile (0 < remain)\n\t\t\t{\n\t\t\t\twchar_t ch = 0;\n\t\t\t\tsize_t n = std::mbrtowc(&ch, p, remain, &this->state);\n\t\t\t\tif (0 == n) // NULL encountered?\n\t\t\t\t{\n\t\t\t\t\t++i;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tif (static_cast<std::size_t>(-1) == n) // decoding error?\n\t\t\t\t{\n\t\t\t\t\tresult = APR_BADARG;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tif (static_cast<std::size_t>(-2) == n) // incomplete sequence?\n\t\t\t\t{\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tTranscoder::encode(static_cast<unsigned int>(ch), out);\n\t\t\t\tremain -= n;\n\t\t\t\ti += n;\n\t\t\t\tp += n;\n\t\t\t}\n\t\t\tin.position(i);\n\t\t\treturn result;\n\t\t}\n\n\tprivate:\n\t\tstd::mbstate_t state;\n};\n\n\n\n} // namespace helpers\n\n}  //namespace log4cxx\n\n\nCharsetDecoder::CharsetDecoder()\n{\n}\n\n\nCharsetDecoder::~CharsetDecoder()\n{\n}\n\nCharsetDecoder* CharsetDecoder::createDefaultDecoder()\n{\n#if LOG4CXX_CHARSET_UTF8\n\treturn new UTF8CharsetDecoder();\n#elif LOG4CXX_CHARSET_ISO88591 || defined(_WIN32_WCE)\n\treturn new ISOLatinCharsetDecoder();\n#elif LOG4CXX_CHARSET_USASCII\n\treturn new USASCIICharsetDecoder();\n#elif LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_MBSRTOWCS\n\treturn new MbstowcsCharsetDecoder();\n#else\n\treturn new LocaleCharsetDecoder();\n#endif\n}\n\nCharsetDecoderPtr CharsetDecoder::getDefaultDecoder()\n{\n\tstatic WideLife<CharsetDecoderPtr> decoder(createDefaultDecoder());\n\n\t//\n\t//  if invoked after static variable destruction\n\t//     (if logging is called in the destructor of a static object)\n\t//     then create a new decoder.\n\t//\n\tif (decoder.value() == 0)\n\t{\n\t\treturn CharsetDecoderPtr( createDefaultDecoder() );\n\t}\n\n\treturn decoder;\n}\n\nCharsetDecoderPtr CharsetDecoder::getUTF8Decoder()\n{\n\tstatic WideLife<CharsetDecoderPtr> decoder(new UTF8CharsetDecoder());\n\n\t//\n\t//  if invoked after static variable destruction\n\t//     (if logging is called in the destructor of a static object)\n\t//     then create a new decoder.\n\t//\n\tif (decoder.value() == 0)\n\t{\n\t\treturn std::make_shared<UTF8CharsetDecoder>();\n\t}\n\n\treturn decoder;\n}\n\nCharsetDecoderPtr CharsetDecoder::getISOLatinDecoder()\n{\n\treturn std::make_shared<ISOLatinCharsetDecoder>();\n}\n\n\nCharsetDecoderPtr CharsetDecoder::getDecoder(const LogString& charset)\n{\n\tif (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF-8\"), LOG4CXX_STR(\"utf-8\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"UTF8\"), LOG4CXX_STR(\"utf8\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP65001\"), LOG4CXX_STR(\"cp65001\")))\n\t{\n\t\treturn std::make_shared<UTF8CharsetDecoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"C\"), LOG4CXX_STR(\"c\")) ||\n\t\tcharset == LOG4CXX_STR(\"646\") ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"US-ASCII\"), LOG4CXX_STR(\"us-ascii\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO646-US\"), LOG4CXX_STR(\"iso646-US\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ANSI_X3.4-1968\"), LOG4CXX_STR(\"ansi_x3.4-1968\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP20127\"), LOG4CXX_STR(\"cp20127\")))\n\t{\n\t\treturn std::make_shared<USASCIICharsetDecoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-8859-1\"), LOG4CXX_STR(\"iso-8859-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"ISO-LATIN-1\"), LOG4CXX_STR(\"iso-latin-1\")) ||\n\t\tStringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"CP1252\"), LOG4CXX_STR(\"cp1252\")))\n\t{\n\t\treturn std::make_shared<ISOLatinCharsetDecoder>();\n\t}\n\telse if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR(\"LOCALE\"), LOG4CXX_STR(\"locale\")))\n\t{\n\t\treturn std::make_shared<LocaleCharsetDecoder>();\n\t}\n\n#if APR_HAS_XLATE\n\treturn std::make_shared<APRCharsetDecoder>(charset);\n#else\n\tthrow IllegalArgumentException(charset);\n#endif\n}\n\n\n\n\n\n\n\n// Path: src/main/cpp/throwableinformationpatternconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/throwableinformationpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct ThrowableInformationPatternConverter::ThrowableInformationPatternConverterPrivate :\n\tpublic PatternConverterPrivate\n{\n\t\nThrowableInformationPatternConverterPrivate( const LogString& name, const LogString& style, bool shortReport ) :\n\t\tPatternConverterPrivate( name, style ),\n\t\tshortReport(shortReport) {}\n\n\t/**\n\t * If \"short\", only first line of throwable report will be formatted.\n\t */\n\tconst bool shortReport;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(ThrowableInformationPatternConverter)\n\nThrowableInformationPatternConverter::ThrowableInformationPatternConverter(bool shortReport1) :\n\tLoggingEventPatternConverter(\n\t\tstd::make_unique<ThrowableInformationPatternConverterPrivate>(\n\t\t\tLOG4CXX_STR(\"Throwable\"),\n\t\t\tLOG4CXX_STR(\"throwable\"),\n\t\t\tshortReport1))\n{\n}\n\nPatternConverterPtr ThrowableInformationPatternConverter::newInstance(\n\tconst std::vector<LogString>& options)\n{\n\tif (options.size() > 0 && options[0].compare(LOG4CXX_STR(\"short\")) == 0)\n\t{\n\t\tstatic WideLife<PatternConverterPtr> shortConverter = std::make_shared<ThrowableInformationPatternConverter>(true);\n\t\treturn shortConverter;\n\t}\n\n\tstatic WideLife<PatternConverterPtr> converter = std::make_shared<ThrowableInformationPatternConverter>(false);\n\treturn converter;\n}\n\nvoid ThrowableInformationPatternConverter::format(\n\tconst LoggingEventPtr& /* event */,\n\tLogString& /* toAppendTo */,\n\tPool& /* p */) const\n{\n}\n\n/**\n * This converter obviously handles throwables.\n * @return true.\n */\nbool ThrowableInformationPatternConverter::handlesThrowable() const\n{\n\treturn true;\n}\n\n// Path: src/main/cpp/htmllayout.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/htmllayout.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/transform.h>\n#include <log4cxx/helpers/iso8601dateformat.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/date.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\nstruct HTMLLayout::HTMLLayoutPrivate\n{\n\tHTMLLayoutPrivate()\n\t\t: locationInfo(false)\n\t\t, title(LOG4CXX_STR(\"Log4cxx Log Messages\"))\n\t\t, dateFormat()\n\t\t, expectedPatternLength(100)\n\t\t{}\n\n\t// Print no location info by default\n\tbool locationInfo; //= false\n\n\tLogString title;\n\n\thelpers::ISO8601DateFormat dateFormat;\n\n\t// Expected length of a formatted event excluding the message text\n\tsize_t expectedPatternLength;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(HTMLLayout)\n\n\nHTMLLayout::HTMLLayout()\n\t: m_priv(std::make_unique<HTMLLayoutPrivate>())\n{\n\tm_priv->dateFormat.setTimeZone(TimeZone::getGMT());\n\tm_priv->expectedPatternLength = getFormattedEventCharacterCount() * 2;\n}\n\nHTMLLayout::~HTMLLayout() {}\n\n\nvoid HTMLLayout::setOption(const LogString& option,\n\tconst LogString& value)\n{\n\n\tif (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"TITLE\"), LOG4CXX_STR(\"title\")))\n\t{\n\t\tsetTitle(value);\n\t}\n\telse if (StringHelper::equalsIgnoreCase(option,\n\t\t\tLOG4CXX_STR(\"LOCATIONINFO\"), LOG4CXX_STR(\"locationinfo\")))\n\t{\n\t\tsetLocationInfo(OptionConverter::toBoolean(value, false));\n\t\tm_priv->expectedPatternLength = getFormattedEventCharacterCount() * 2;\n\t}\n}\n\nvoid HTMLLayout::format(LogString& output,\n\tconst spi::LoggingEventPtr& event,\n\tPool& p) const\n{\n\toutput.reserve(m_priv->expectedPatternLength + event->getMessage().size());\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<tr>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"<td>\"));\n\n\tm_priv->dateFormat.format(output, event->getTimeStamp(), p);\n\n\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"\"));\n\tLogString threadName(event->getThreadName());\n\toutput.append(threadName);\n\toutput.append(LOG4CXX_STR(\" thread\\\">\"));\n\toutput.append(threadName);\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"Level\\\">\"));\n\n\tif (event->getLevel()->equals(Level::getDebug()))\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<font color=\\\"#339933\\\">\"));\n\t\toutput.append(event->getLevel()->toString());\n\t\toutput.append(LOG4CXX_STR(\"</font>\"));\n\t}\n\telse if (event->getLevel()->isGreaterOrEqual(Level::getWarn()))\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<font color=\\\"#993300\\\"><strong>\"));\n\t\toutput.append(event->getLevel()->toString());\n\t\toutput.append(LOG4CXX_STR(\"</strong></font>\"));\n\t}\n\telse\n\t{\n\t\toutput.append(event->getLevel()->toString());\n\t}\n\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"\"));\n\toutput.append(event->getLoggerName());\n\toutput.append(LOG4CXX_STR(\" logger\\\">\"));\n\tTransform::appendEscapingTags(output, event->getLoggerName());\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\tif (m_priv->locationInfo)\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<td>\"));\n\t\tconst LocationInfo& locInfo = event->getLocationInformation();\n\t\tLOG4CXX_DECODE_CHAR(fileName, locInfo.getFileName());\n\t\tTransform::appendEscapingTags(output, fileName);\n\t\toutput.append(1, (logchar) 0x3A /* ':' */);\n\t\tint line = event->getLocationInformation().getLineNumber();\n\n\t\tif (line != 0)\n\t\t{\n\t\t\tStringHelper::toString(line, p, output);\n\t\t}\n\n\t\toutput.append(LOG4CXX_STR(\"</td>\"));\n\t\toutput.append(LOG4CXX_EOL);\n\t}\n\n\toutput.append(LOG4CXX_STR(\"<td title=\\\"Message\\\">\"));\n\tTransform::appendEscapingTags(output, event->getRenderedMessage());\n\toutput.append(LOG4CXX_STR(\"</td>\"));\n\toutput.append(LOG4CXX_EOL);\n\toutput.append(LOG4CXX_STR(\"</tr>\"));\n\toutput.append(LOG4CXX_EOL);\n\n\tLogString ndcVal;\n\n\tif (event->getNDC(ndcVal))\n\t{\n\t\toutput.append(LOG4CXX_STR(\"<tr><td bgcolor=\\\"#EEEEEE\\\" \"));\n\t\toutput.append(LOG4CXX_STR(\"style=\\\"font-size : xx-small;\\\" colspan=\\\"6\\\" \"));\n\t\toutput.append(LOG4CXX_STR(\"title=\\\"Nested Diagnostic Context\\\">\"));\n\t\toutput.append(LOG4CXX_STR(\"NDC: \"));\n...\n", "needle_token_start": 12850, "needle_token_end": 12896, "code_context_ntokens": 15142, "output": ["```cpp\n#include <log4cxx/logstring.h>\n#include <log4cxx/pattern/throwableinformationpatternconverter.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/spi/location/locationinfo.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/private/patternconverter_priv.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::pattern;\nusing namespace LOG4CXX_NS::spi;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct ThrowableInformationPatternConverter::ThrowableInformationPatternConverterPrivate :\n\tpublic PatternConverterPrivate\n{\n\t\nThrowableInformationPatternConverterPrivate( const LogString& name, const LogString& style, bool shortReport ) :\n\t\tPatternConverterPrivate( name, style ),\n\t\tshortReport(shortReport) {}\n\n\t/**\n\t * If \"short\", only first line of throwable report will be formatted.\n\t */\n\tconst bool shortReport;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(ThrowableInformationPatternConverter)\n\nThrowableInformationPatternConverter::ThrowableInformationPatternConverter(bool shortReport1) :\n\tLoggingEventPatternConverter(\n\t\tstd::make_unique<ThrowableInformationPatternConverterPrivate>(\n\t\t\tLOG4CXX_STR(\"Throwable\"),\n\t\t\tLOG4CXX_STR(\"throwable\"),\n\t\t\tshortReport1))\n{\n}\n\nPatternConverterPtr ThrowableInformationPatternConverter::newInstance(\n\tconst std::vector<LogString>& options)\n{\n\tif (options.size() > 0 && options[0].compare(LOG4CXX_STR(\"short\")) == 0)\n\t{\n\t\tstatic WideLife<PatternConverterPtr> shortConverter = std::make_shared<ThrowableInformationPatternConverter>(true);\n\t\treturn shortConverter;\n\t}\n\n\tstatic WideLife<PatternConverterPtr> converter = std::make_shared<ThrowableInformationPatternConverter>(false);\n\treturn converter;\n}\n\nvoid ThrowableInformationPatternConverter::format(\n\tconst LoggingEventPtr& /* event */,\n\tLogString& /* toAppendTo */,\n\tPool& /* p */) const\n{\n}\n\n/**\n * This converter obviously handles throwables.\n * @return true.\n */\nbool ThrowableInformationPatternConverter::handlesThrowable() const\n{\n\treturn true;\n}\n```"]}
{"repo": "apache/logging-log4cxx", "name": "APRCharsetEncoder", "language": "cpp", "path": "src/main/cpp/charsetencoder.cpp", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To convert characters from a native encoding format to a specified target encoding format using the Apache Portable Runtime (APR) library's translation facilities.\n2. **Input**: A string representing the target encoding format (e.g., \"UTF-8\", \"ISO-8859-1\").\n3. **Output**: There is no direct output from the constructor itself; however, it sets up an environment for subsequent character encoding operations, throwing an exception if the setup fails.\n4. **Procedure**: The procedure involves initializing a memory pool, determining the native character encoding format based on compile-time macros, and then attempting to create a translation handle that can convert from the native format to the specified target format. If the translation setup is successful, the environment is ready for encoding operations; otherwise, an exception is thrown indicating a problem with the specified target encoding.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/cpp/logger.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/logger.h>\n#include <log4cxx/spi/loggingevent.h>\n#include <log4cxx/logmanager.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <log4cxx/appender.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/hierarchy.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/helpers/appenderattachableimpl.h>\n#include <log4cxx/helpers/exception.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/private/log4cxx_private.h>\n#include <log4cxx/helpers/aprinitializer.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\nstruct Logger::LoggerPrivate\n{\n\tLoggerPrivate(Pool& p, const LogString& name1):\n\t\tname(name1),\n\t\trepositoryRaw(0),\n\t\taai(p),\n\t\tadditive(true),\n\t\tlevelData(Level::getData()) {}\n\n\t/**\n\tThe name of this logger.\n\t*/\n\tLogString name;\n\n\t/**\n\tThe assigned level of this logger.  The\n\t<code>level</code> variable need not be assigned a value in\n\twhich case it is inherited form the hierarchy.  */\n\tLevelPtr level;\n\n\t/**\n\tThe parent of this logger. All loggers have at least one\n\tancestor which is the root logger. */\n\tLoggerPtr parent;\n\n\t/** The resourceBundle for localized messages.\n\n\t@see setResourceBundle, getResourceBundle\n\t*/\n\thelpers::ResourceBundlePtr resourceBundle;\n\n\n\t// Loggers need to know what Hierarchy they are in\n\tLOG4CXX_NS::spi::LoggerRepository* repositoryRaw;\n\n\thelpers::AppenderAttachableImpl aai;\n\n\t/** Additivity is set to true by default, that is children inherit\n\t        the appenders of their ancestors by default. If this variable is\n\t        set to <code>false</code> then the appenders found in the\n\t        ancestors of this logger are not used. However, the children\n\t        of this logger will inherit its appenders, unless the children\n\t        have their additivity flag set to <code>false</code> too. See\n\t        the user manual for more details. */\n\tbool additive;\n\n\tconst Level::Data& levelData;\n};\n\nIMPLEMENT_LOG4CXX_OBJECT(Logger)\n\nLogger::Logger(Pool& p, const LogString& name1)\n\t: m_priv(std::make_unique<LoggerPrivate>(p, name1))\n\t, m_threshold(0)\n{\n}\n\nLogger::~Logger()\n{\n}\n\nvoid Logger::addAppender(const AppenderPtr newAppender)\n{\n\tm_priv->aai.addAppender(newAppender);\n\tif (auto rep = getHierarchy())\n\t{\n\t\trep->fireAddAppenderEvent(this, newAppender.get());\n\t}\n}\n\nvoid Logger::reconfigure( const std::vector<AppenderPtr>& appenders, bool additive1 )\n{\n\tm_priv->additive = additive1;\n\n\tm_priv->aai.removeAllAppenders();\n\n\tfor (auto const& item : appenders)\n\t{\n\t\tm_priv->aai.addAppender(item);\n\n\t\tif (auto rep = getHierarchy())\n\t\t{\n\t\t\trep->fireAddAppenderEvent(this, item.get());\n\t\t}\n\t}\n}\n\nvoid Logger::callAppenders(const spi::LoggingEventPtr& event, Pool& p) const\n{\n\tint writes = 0;\n\n\tfor (const Logger* logger = this;\n\t\tlogger != 0;\n\t\tlogger = logger->m_priv->parent.get())\n\t{\n\t\twrites += logger->m_priv->aai.appendLoopOnAppenders(event, p);\n\n\t\tif (!logger->m_priv->additive)\n\t\t{\n\t\t\tbreak;\n\t\t}\n\t}\n\n\tauto rep = getHierarchy();\n\n\tif (writes == 0 && rep)\n\t{\n\t\trep->emitNoAppenderWarning(const_cast<Logger*>(this));\n\t}\n}\n\nvoid Logger::closeNestedAppenders()\n{\n\tfor (auto& item : getAllAppenders())\n\t{\n\t\titem->close();\n\t}\n}\n\nvoid Logger::addEvent(const LevelPtr& level, std::string&& message, const LocationInfo& location) const\n{\n\tif (!getHierarchy()) // Has removeHierarchy() been called?\n\t\treturn;\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\tauto event = std::make_shared<LoggingEvent>(m_priv->name, level, location, std::move(message));\n#else\n\tLOG4CXX_DECODE_CHAR(msg, message);\n\tauto event = std::make_shared<LoggingEvent>(m_priv->name, level, location, std::move(msg));\n#endif\n\tPool p;\n\tcallAppenders(event, p);\n}\n\nvoid Logger::addInfoEvent(std::string&& message, const LocationInfo& location) const\n{\n\taddEvent(m_priv->levelData.Info, std::move(message), location);\n}\n\nvoid Logger::addDebugEvent(std::string&& message, const LocationInfo& location) const\n{\n\taddEvent(m_priv->levelData.Debug, std::move(message), location);\n}\n\nvoid Logger::addTraceEvent(std::string&& message, const LocationInfo& location) const\n{\n\taddEvent(m_priv->levelData.Trace, std::move(message), location);\n}\n\nvoid Logger::forcedLog(const LevelPtr& level, const std::string& message,\n\tconst LocationInfo& location) const\n{\n\tif (!getHierarchy()) // Has removeHierarchy() been called?\n\t\treturn;\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\tauto event = std::make_shared<LoggingEvent>(m_priv->name, level, message, location);\n#else\n\tLOG4CXX_DECODE_CHAR(msg, message);\n\tauto event = std::make_shared<LoggingEvent>(m_priv->name, level, location, std::move(msg));\n#endif\n\tPool p;\n\tcallAppenders(event, p);\n}\n\nvoid Logger::forcedLog(const LevelPtr& level1, const std::string& message) const\n{\n\tforcedLog(level1, message, LocationInfo::getLocationUnavailable());\n}\n\nvoid Logger::addEventLS(const LevelPtr& level, LogString&& message, const LocationInfo& location) const\n{\n\tif (!getHierarchy()) // Has removeHierarchy() been called?\n\t\treturn;\n\tauto event = std::make_shared<LoggingEvent>(m_priv->name, level, location, std::move(message));\n\tPool p;\n\tcallAppenders(event, p);\n}\n\nvoid Logger::forcedLogLS(const LevelPtr& level1, const LogString& message,\n\tconst LocationInfo& location) const\n{\n\tif (!getHierarchy()) // Has removeHierarchy() been called?\n\t\treturn;\n\tauto event = std::make_shared<LoggingEvent>(m_priv->name, level1, message, location);\n\tPool p;\n\tcallAppenders(event, p);\n}\n\n\nbool Logger::getAdditivity() const\n{\n\treturn m_priv->additive;\n}\n\nAppenderList Logger::getAllAppenders() const\n{\n\treturn m_priv->aai.getAllAppenders();\n}\n\nAppenderPtr Logger::getAppender(const LogString& name1) const\n{\n\treturn m_priv->aai.getAppender(name1);\n}\n\nconst LevelPtr& Logger::getEffectiveLevel() const\n{\n\tfor (const Logger* l = this; l != 0; l = l->m_priv->parent.get())\n\t{\n\t\tif (l->m_priv->level != 0)\n\t\t{\n\t\t\treturn l->m_priv->level;\n\t\t}\n\t}\n\n\tthrow NullPointerException(LOG4CXX_STR(\"No level specified for logger or ancestors.\"));\n#if LOG4CXX_RETURN_AFTER_THROW\n\treturn m_priv->level;\n#endif\n}\n\nLoggerRepository* Logger::getLoggerRepository() const\n{\n\treturn m_priv->repositoryRaw;\n}\n\nLoggerRepository* Logger::getHierarchy() const\n{\n\treturn m_priv->repositoryRaw;\n}\n\nResourceBundlePtr Logger::getResourceBundle() const\n{\n\tfor (const Logger* l = this; l != 0; l = l->m_priv->parent.get())\n\t{\n\t\tif (l->m_priv->resourceBundle != 0)\n\t\t{\n\t\t\treturn l->m_priv->resourceBundle;\n\t\t}\n\t}\n\n\t// It might be the case that there is no resource bundle\n\treturn 0;\n}\n\n\nLogString Logger::getResourceBundleString(const LogString& key) const\n{\n\tResourceBundlePtr rb = getResourceBundle();\n\n\t// This is one of the rare cases where we can use logging in order\n\t// to report errors from within log4j.\n\tif (rb == 0)\n\t{\n\t\treturn LogString();\n\t}\n\telse\n\t{\n\t\ttry\n\t\t{\n\t\t\treturn rb->getString(key);\n\t\t}\n\t\tcatch (MissingResourceException&)\n\t\t{\n\t\t\tlogLS(Level::getError(), LOG4CXX_STR(\"No resource is associated with key \\\"\") +\n\t\t\t\tkey + LOG4CXX_STR(\"\\\".\"), LocationInfo::getLocationUnavailable());\n\n\t\t\treturn LogString();\n\t\t}\n\t}\n}\n\n\nLoggerPtr Logger::getParent() const\n{\n\treturn m_priv->parent;\n}\n\nconst LevelPtr& Logger::getLevel() const\n{\n\treturn m_priv->level;\n}\n\nbool Logger::isAttached(const AppenderPtr appender) const\n{\n\treturn m_priv->aai.isAttached(appender);\n}\n\nbool Logger::isTraceEnabled() const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(Level::TRACE_INT))\n\t{\n\t\treturn false;\n\t}\n\n\treturn getEffectiveLevel()->toInt() <= Level::TRACE_INT;\n}\n\nbool Logger::isDebugEnabled() const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(Level::DEBUG_INT))\n\t{\n\t\treturn false;\n\t}\n\n\treturn getEffectiveLevel()->toInt() <= Level::DEBUG_INT;\n}\n\nbool Logger::isEnabledFor(const LevelPtr& level1) const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(level1->toInt()))\n\t{\n\t\treturn false;\n\t}\n\n\treturn level1->isGreaterOrEqual(getEffectiveLevel());\n}\n\n\nbool Logger::isInfoEnabled() const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(Level::INFO_INT))\n\t{\n\t\treturn false;\n\t}\n\n\treturn getEffectiveLevel()->toInt() <= Level::INFO_INT;\n}\n\nbool Logger::isErrorEnabled() const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(Level::ERROR_INT))\n\t{\n\t\treturn false;\n\t}\n\n\treturn getEffectiveLevel()->toInt() <= Level::ERROR_INT;\n}\n\nbool Logger::isWarnEnabled() const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(Level::WARN_INT))\n\t{\n\t\treturn false;\n\t}\n\n\treturn getEffectiveLevel()->toInt() <= Level::WARN_INT;\n}\n\nbool Logger::isFatalEnabled() const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(Level::FATAL_INT))\n\t{\n\t\treturn false;\n\t}\n\n\treturn getEffectiveLevel()->toInt() <= Level::FATAL_INT;\n}\n\n/*void Logger::l7dlog(const LevelPtr& level, const String& key,\n                        const char* file, int line)\n{\n\tauto rep = getHierarchy();\n\n        if (!rep || rep->isDisabled(level->level))\n        {\n                return;\n        }\n\n        if (level->isGreaterOrEqual(getEffectiveLevel()))\n        {\n                String msg = getResourceBundleString(key);\n\n                // if message corresponding to 'key' could not be found in the\n                // resource bundle, then default to 'key'.\n                if (msg.empty())\n                {\n                        msg = key;\n                }\n\n                forcedLog(FQCN, level, msg, file, line);\n        }\n}*/\n\n\n\nvoid Logger::l7dlog(const LevelPtr& level1, const LogString& key,\n\tconst LocationInfo& location, const std::vector<LogString>& params) const\n{\n\tauto rep = getHierarchy();\n\n\tif (!rep || rep->isDisabled(level1->toInt()))\n\t{\n\t\treturn;\n\t}\n\n\tif (level1->isGreaterOrEqual(getEffectiveLevel()))\n\t{\n\t\tLogString pattern = getResourceBundleString(key);\n\t\tLogString msg;\n\n\t\tif (pattern.empty())\n\t\t{\n\t\t\tmsg = key;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tmsg = StringHelper::format(pattern, params);\n\t\t}\n\n\t\taddEventLS(level1, std::move(msg), location);\n\t}\n}\n\nvoid Logger::l7dlog(const LevelPtr& level1, const std::string& key,\n\tconst LocationInfo& location) const\n{\n\tLOG4CXX_DECODE_CHAR(lkey, key);\n\n\tstd::vector<LogString> values(0);\n\tl7dlog(level1, lkey, location, values);\n}\n\nvoid Logger::l7dlog(const LevelPtr& level1, const std::string& key,\n\tconst LocationInfo& location, const std::string& val1) const\n{\n\tLOG4CXX_DECODE_CHAR(lkey, key);\n...\n// Path: src/main/cpp/aprdatagramsocket.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/private/aprdatagramsocket.h>\n#include <log4cxx/private/datagramsocket_priv.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <apr_network_io.h>\n\nnamespace LOG4CXX_NS\n{\nnamespace helpers\n{\n\n#define _priv static_cast<APRDatagramSocketPriv*>(m_priv.get())\n\nstruct APRDatagramSocket::APRDatagramSocketPriv : public DatagramSocketPriv {\n\tAPRDatagramSocketPriv() :\n\t\tDatagramSocketPriv(),\n\t\tsocket(nullptr)\n\t{}\n\n\tAPRDatagramSocketPriv(int port) :\n\t\tDatagramSocketPriv(port),\n\t\tsocket(nullptr)\n\t{}\n\n\tAPRDatagramSocketPriv(int port, InetAddressPtr localAddress) :\n\t\tDatagramSocketPriv(port, localAddress),\n\t\tsocket(nullptr)\n\t{}\n\n\t/** The APR socket */\n\tapr_socket_t* socket;\n\n\t/** The memory pool for the socket */\n\tPool socketPool;\n};\n\nAPRDatagramSocket::APRDatagramSocket() :\n\tDatagramSocket(std::make_unique<APRDatagramSocketPriv>()){\n\tinit();\n}\n\nAPRDatagramSocket::APRDatagramSocket(int port) :\n\tDatagramSocket(std::make_unique<APRDatagramSocketPriv>(port)){\n\tinit();\n}\n\nAPRDatagramSocket::APRDatagramSocket(int port, InetAddressPtr laddr) :\n\tDatagramSocket(std::make_unique<APRDatagramSocketPriv>(port, laddr)){\n\tinit();\n}\n\nvoid APRDatagramSocket::init()\n{\n\tapr_socket_t* newSocket;\n\tapr_status_t status =\n\t\tapr_socket_create(&newSocket, APR_INET, SOCK_DGRAM,\n\t\t\tAPR_PROTO_UDP, _priv->socketPool.getAPRPool());\n\t_priv->socket = newSocket;\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow SocketException(status);\n\t}\n}\n\nvoid APRDatagramSocket::receive(DatagramPacketPtr& p)\n{\n\tPool addrPool;\n\n\t// Create the address from which to receive the datagram packet\n\tLOG4CXX_ENCODE_CHAR(hostAddr, p->getAddress()->getHostAddress());\n\tapr_sockaddr_t* addr;\n\tapr_status_t status =\n\t\tapr_sockaddr_info_get(&addr, hostAddr.c_str(), APR_INET,\n\t\t\tp->getPort(), 0, addrPool.getAPRPool());\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow SocketException(status);\n\t}\n\n\t// receive the datagram packet\n\tapr_size_t len = p->getLength();\n\tstatus = apr_socket_recvfrom(addr, _priv->socket, 0,\n\t\t\t(char*)p->getData(), &len);\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow IOException(status);\n\t}\n}\n\nvoid APRDatagramSocket::send(DatagramPacketPtr& p)\n{\n\tPool addrPool;\n\n\t// create the adress to which to send the datagram packet\n\tLOG4CXX_ENCODE_CHAR(hostAddr, p->getAddress()->getHostAddress());\n\tapr_sockaddr_t* addr;\n\tapr_status_t status =\n\t\tapr_sockaddr_info_get(&addr, hostAddr.c_str(), APR_INET, p->getPort(),\n\t\t\t0, addrPool.getAPRPool());\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow SocketException(status);\n\t}\n\n\t// send the datagram packet\n\tapr_size_t len = p->getLength();\n\tstatus = apr_socket_sendto(_priv->socket, addr, 0,\n\t\t\t(char*)p->getData(), &len);\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow IOException(status);\n\t}\n}\n\nvoid APRDatagramSocket::close(){\n\tif (_priv->socket != 0)\n\t{\n\t\tapr_status_t status = apr_socket_close(_priv->socket);\n\n\t\tif (status != APR_SUCCESS)\n\t\t{\n\t\t\tthrow SocketException(status);\n\t\t}\n\n\t\t_priv->socket = 0;\n\t\t_priv->localPort = 0;\n\t}\n}\n\nvoid APRDatagramSocket::bind(int localPort1, InetAddressPtr localAddress1)\n{\n\tPool addrPool;\n\n\t// Create server socket address (including port number)\n\tLOG4CXX_ENCODE_CHAR(hostAddr, localAddress1->getHostAddress());\n\tapr_sockaddr_t* server_addr;\n\tapr_status_t status =\n\t\tapr_sockaddr_info_get(&server_addr, hostAddr.c_str(), APR_INET,\n\t\t\tlocalPort1, 0, addrPool.getAPRPool());\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow BindException(status);\n\t}\n\n\t// bind the socket to the address\n\tstatus = apr_socket_bind(_priv->socket, server_addr);\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow BindException(status);\n\t}\n\n\tm_priv->localPort = localPort1;\n\tm_priv->localAddress = localAddress1;\n}\n\n\nvoid APRDatagramSocket::connect(InetAddressPtr address1, int port1)\n{\n\tm_priv->address = address1;\n\tm_priv->port = port1;\n\n\tPool addrPool;\n\n\t// create socket address\n\tLOG4CXX_ENCODE_CHAR(hostAddr, address1->getHostAddress());\n\tapr_sockaddr_t* client_addr;\n\tapr_status_t status =\n\t\tapr_sockaddr_info_get(&client_addr, hostAddr.c_str(), APR_INET,\n\t\t\tm_priv->port, 0, addrPool.getAPRPool());\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow ConnectException(status);\n\t}\n\n\t// connect the socket\n\tstatus = apr_socket_connect(_priv->socket, client_addr);\n\n\tif (status != APR_SUCCESS)\n\t{\n\t\tthrow ConnectException(status);\n\t}\n}\n\n\nbool APRDatagramSocket::isClosed() const\n{\n\treturn _priv->socket != nullptr;\n}\n\n} //namespace helpers\n} //namespace log4cxx\n\n// Path: src/main/cpp/classregistration.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/classregistration.h>\n#include <log4cxx/helpers/class.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nClassRegistration::ClassRegistration(ClassAccessor accessor)\n{\n\tClass::registerClass((*accessor)());\n}\n\n\n\n// Path: src/main/cpp/level.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n#include <mutex>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT_WITH_CUSTOM_CLASS(Level, LevelClass)\n\nLevelPtr Level::getOff()\n{\n\tstatic WideLife<LevelPtr> offLevel = std::make_shared<Level>(Level::OFF_INT, LOG4CXX_STR(\"OFF\"), 0);\n\treturn offLevel;\n}\n\nLevelPtr Level::getFatal()\n{\n\tstatic WideLife<LevelPtr> fatalLevel = std::make_shared<Level>(Level::FATAL_INT, LOG4CXX_STR(\"FATAL\"), 0);\n\treturn fatalLevel;\n}\n\nLevelPtr Level::getError()\n{\n\tstatic WideLife<LevelPtr> errorLevel = std::make_shared<Level>(Level::ERROR_INT, LOG4CXX_STR(\"ERROR\"), 3);\n\treturn errorLevel;\n}\n\nLevelPtr Level::getWarn()\n{\n\tstatic WideLife<LevelPtr> warnLevel = std::make_shared<Level>(Level::WARN_INT, LOG4CXX_STR(\"WARN\"), 4);\n\treturn warnLevel;\n}\n\nLevelPtr Level::getInfo()\n{\n\tstatic WideLife<LevelPtr> infoLevel = std::make_shared<Level>(Level::INFO_INT, LOG4CXX_STR(\"INFO\"), 6);\n\treturn infoLevel;\n}\n\nLevelPtr Level::getDebug()\n{\n\tstatic WideLife<LevelPtr> debugLevel = std::make_shared<Level>(Level::DEBUG_INT, LOG4CXX_STR(\"DEBUG\"), 7);\n\treturn debugLevel;\n}\n\nLevelPtr Level::getTrace()\n{\n\tstatic WideLife<LevelPtr> traceLevel = std::make_shared<Level>(Level::TRACE_INT, LOG4CXX_STR(\"TRACE\"), 7);\n\treturn traceLevel;\n}\n\n\nLevelPtr Level::getAll()\n{\n\tstatic WideLife<LevelPtr> allLevel = std::make_shared<Level>(Level::ALL_INT, LOG4CXX_STR(\"ALL\"), 7);\n\treturn allLevel;\n}\n\n\n\nLevel::Level(int level1,\n\tconst LogString& name1, int syslogEquivalent1)\n\t: level(level1), name(name1), syslogEquivalent(syslogEquivalent1)\n{\n\tAPRInitializer::initialize();\n}\n\n\nLevelPtr Level::toLevelLS(const LogString& sArg)\n{\n\treturn toLevelLS(sArg, Level::getDebug());\n}\n\nLogString Level::toString() const\n{\n\treturn name;\n}\n\n\nLevelPtr Level::toLevel(int val)\n{\n\treturn toLevel(val, Level::getDebug());\n}\n\nconst Level::Data& Level::getData()\n{\n\tstatic Data data =\n\t\t{ getOff()\n\t\t, getFatal()\n\t\t, getError()\n\t\t, getWarn()\n\t\t, getInfo()\n\t\t, getDebug()\n\t\t, getTrace()\n\t\t, getAll()\n\t\t};\n\treturn data;\n}\n\n\nLevelPtr Level::toLevel(int val, const LevelPtr& defaultLevel)\n{\n\tswitch (val)\n\t{\n\t\tcase ALL_INT:\n\t\t\treturn getAll();\n\n\t\tcase DEBUG_INT:\n\t\t\treturn getDebug();\n\n\t\tcase TRACE_INT:\n\t\t\treturn getTrace();\n\n\t\tcase INFO_INT:\n\t\t\treturn getInfo();\n\n\t\tcase WARN_INT:\n\t\t\treturn getWarn();\n\n\t\tcase ERROR_INT:\n\t\t\treturn getError();\n\n\t\tcase FATAL_INT:\n\t\t\treturn getFatal();\n\n\t\tcase OFF_INT:\n\t\t\treturn getOff();\n\n\t\tdefault:\n\t\t\treturn defaultLevel;\n\t}\n}\n\nLevelPtr Level::toLevel(const std::string& sArg)\n{\n\treturn toLevel(sArg, Level::getDebug());\n}\n\nLevelPtr Level::toLevel(const std::string& sArg, const LevelPtr& defaultLevel)\n{\n\tLOG4CXX_DECODE_CHAR(s, sArg);\n\treturn toLevelLS(s, defaultLevel);\n}\n\nvoid Level::toString(std::string& dst) const\n{\n\tTranscoder::encode(name, dst);\n}\n\n#if LOG4CXX_WCHAR_T_API\nLevelPtr Level::toLevel(const std::wstring& sArg)\n{\n\treturn toLevel(sArg, Level::getDebug());\n}\n\nLevelPtr Level::toLevel(const std::wstring& sArg, const LevelPtr& defaultLevel)\n{\n\tLOG4CXX_DECODE_WCHAR(s, sArg);\n\treturn toLevelLS(s, defaultLevel);\n}\n\nvoid Level::toString(std::wstring& dst) const\n{\n\tTranscoder::encode(name, dst);\n}\n\n#endif\n\n#if LOG4CXX_UNICHAR_API || LOG4CXX_LOGCHAR_IS_UNICHAR\nLevelPtr Level::toLevel(const std::basic_string<UniChar>& sArg)\n{\n\treturn toLevel(sArg, Level::getDebug());\n}\n\nLevelPtr Level::toLevel(const std::basic_string<UniChar>& sArg, const LevelPtr& defaultLevel)\n{\n\tLOG4CXX_DECODE_UNICHAR(s, sArg);\n\treturn toLevelLS(s, defaultLevel);\n}\n\nvoid Level::toString(std::basic_string<UniChar>& dst) const\n{\n\tTranscoder::encode(name, dst);\n}\n\n#endif\n\n#if LOG4CXX_CFSTRING_API\nLevelPtr Level::toLevel(const CFStringRef& sArg)\n{\n\treturn toLevel(sArg, Level::getDebug());\n}\n\nLevelPtr Level::toLevel(const CFStringRef& sArg, const LevelPtr& defaultLevel)\n{\n\tLogString s;\n\tTranscoder::decode(sArg, s);\n\treturn toLevelLS(s, defaultLevel);\n}\n\nvoid Level::toString(CFStringRef& dst) const\n{\n\tdst = Transcoder::encode(name);\n}\n#endif\n\n\nLevelPtr Level::toLevelLS(const LogString& sArg, const LevelPtr& defaultLevel)\n{\n\tconst LogString trimmed(StringHelper::trim(sArg));\n\tconst size_t len = trimmed.length();\n\n\tif (len == 4)\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"INFO\"), LOG4CXX_STR(\"info\")))\n\t\t{\n\t\t\treturn getInfo();\n\t\t}\n\n\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"WARN\"), LOG4CXX_STR(\"warn\")))\n\t\t{\n\t\t\treturn getWarn();\n\t\t}\n\t}\n\telse\n\t{\n\t\tif (len == 5)\n\t\t{\n\t\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"DEBUG\"), LOG4CXX_STR(\"debug\")))\n\t\t\t{\n\t\t\t\treturn getDebug();\n\t\t\t}\n\n\t\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"TRACE\"), LOG4CXX_STR(\"trace\")))\n\t\t\t{\n\t\t\t\treturn getTrace();\n\t\t\t}\n\n\t\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"ERROR\"), LOG4CXX_STR(\"error\")))\n\t\t\t{\n\t\t\t\treturn getError();\n\t\t\t}\n\n\t\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"FATAL\"), LOG4CXX_STR(\"fatal\")))\n\t\t\t{\n\t\t\t\treturn getFatal();\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tif (len == 3)\n\t\t\t{\n\t\t\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"OFF\"), LOG4CXX_STR(\"off\")))\n\t\t\t\t{\n\t\t\t\t\treturn getOff();\n\t\t\t\t}\n\n\t\t\t\tif (StringHelper::equalsIgnoreCase(trimmed, LOG4CXX_STR(\"ALL\"), LOG4CXX_STR(\"all\")))\n\t\t\t\t{\n\t\t\t\t\treturn getAll();\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn defaultLevel;\n}\n\n\nbool Level::equals(const LevelPtr& level1) const\n{\n\treturn level1 && this->level == level1->level;\n}\n\nbool Level::isGreaterOrEqual(const LevelPtr& level1) const\n{\n\treturn level1 && this->level >= level1->level;\n}\n\n\n// Path: src/main/cpp/optionconverter.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/logstring.h>\n#include <log4cxx/spi/loggerfactory.h>\n#include <log4cxx/spi/loggerrepository.h>\n#include <log4cxx/appenderskeleton.h>\n#include <log4cxx/helpers/optionconverter.h>\n#include <algorithm>\n#include <ctype.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/exception.h>\n#include <stdlib.h>\n#include <log4cxx/helpers/properties.h>\n#include <log4cxx/helpers/loglog.h>\n#include <log4cxx/level.h>\n#include <log4cxx/helpers/object.h>\n#include <log4cxx/helpers/class.h>\n#include <log4cxx/helpers/loader.h>\n#include <log4cxx/helpers/system.h>\n#include <log4cxx/propertyconfigurator.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <log4cxx/file.h>\n#include <log4cxx/xml/domconfigurator.h>\n#include <log4cxx/logmanager.h>\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n#include <log4cxx/helpers/aprinitializer.h>\n#include <log4cxx/helpers/filewatchdog.h>\n\nnamespace LOG4CXX_NS\n{\n\nclass ConfiguratorWatchdog  : public helpers::FileWatchdog\n{\n\tspi::ConfiguratorPtr m_config;\n\tpublic:\n    ConfiguratorWatchdog(const spi::ConfiguratorPtr& config, const File& filename)\n        : helpers::FileWatchdog(filename)\n        , m_config(config)\n    {\n    }\n\n    /**\n    Call PropertyConfigurator#doConfigure(const String& configFileName,\n    const spi::LoggerRepositoryPtr& hierarchy) with the\n    <code>filename</code> to reconfigure log4cxx.\n    */\n    void doOnChange() override\n    {\n        m_config->doConfigure(file(), LogManager::getLoggerRepository());\n    }\n};\n\n}\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\nusing namespace LOG4CXX_NS::spi;\n\n\nLogString OptionConverter::convertSpecialChars(const LogString& s)\n{\n\tlogchar c;\n\tLogString sbuf;\n\n\tLogString::const_iterator i = s.begin();\n\n\twhile (i != s.end())\n\t{\n\t\tc = *i++;\n\n\t\tif (c == 0x5C /* '\\\\' */)\n\t\t{\n\t\t\tc =  *i++;\n\n\t\t\tswitch (c)\n\t\t\t{\n\t\t\t\tcase 0x6E: //'n'\n\t\t\t\t\tc = 0x0A;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x72: //'r'\n\t\t\t\t\tc = 0x0D;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x74: //'t'\n\t\t\t\t\tc = 0x09;\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase 0x66: //'f'\n\t\t\t\t\tc = 0x0C;\n\t\t\t\t\tbreak;\n\n\t\t\t\tdefault:\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\n\t\tsbuf.append(1, c);\n\t}\n\n\treturn sbuf;\n}\n\n\nbool OptionConverter::toBoolean(const LogString& value, bool dEfault)\n{\n\tif (value.length() >= 4)\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(value.substr(0, 4),\n\t\t\t\tLOG4CXX_STR(\"TRUE\"), LOG4CXX_STR(\"true\")))\n\t\t{\n\t\t\treturn true;\n\t\t}\n\t}\n\n\tif (dEfault && value.length() >= 5)\n\t{\n\t\tif (StringHelper::equalsIgnoreCase(value.substr(0, 5),\n\t\t\t\tLOG4CXX_STR(\"FALSE\"), LOG4CXX_STR(\"false\")))\n\t\t{\n\t\t\treturn false;\n\t\t}\n\t}\n\n\treturn dEfault;\n}\n\nint OptionConverter::toInt(const LogString& value, int dEfault)\n{\n\tLogString trimmed(StringHelper::trim(value));\n\n\tif (trimmed.empty())\n\t{\n\t\treturn dEfault;\n\t}\n\n\tLOG4CXX_ENCODE_CHAR(cvalue, trimmed);\n\n\treturn (int) atol(cvalue.c_str());\n}\n\nlong OptionConverter::toFileSize(const LogString& s, long dEfault)\n{\n\tif (s.empty())\n\t{\n\t\treturn dEfault;\n\t}\n\n\tsize_t index = s.find_first_of(LOG4CXX_STR(\"bB\"));\n\n\tif (index != LogString::npos && index > 0)\n\t{\n\t\tlong multiplier = 1;\n\t\tindex--;\n\n\t\tif (s[index] == 0x6B /* 'k' */ || s[index] == 0x4B /* 'K' */)\n\t\t{\n\t\t\tmultiplier = 1024;\n\t\t}\n\t\telse if (s[index] == 0x6D /* 'm' */ || s[index] == 0x4D /* 'M' */)\n\t\t{\n\t\t\tmultiplier = 1024 * 1024;\n\t\t}\n\t\telse if (s[index] == 0x67 /* 'g'*/ || s[index] == 0x47 /* 'G' */)\n\t\t{\n\t\t\tmultiplier = 1024 * 1024 * 1024;\n\t\t}\n\n\t\treturn toInt(s.substr(0, index), 1) * multiplier;\n\t}\n\n\treturn toInt(s, 1);\n}\n\nLogString OptionConverter::findAndSubst(const LogString& key, Properties& props)\n{\n\tLogString value(props.getProperty(key));\n\n\tif (value.empty())\n\t{\n\t\treturn value;\n\t}\n\n\ttry\n\t{\n\t\treturn substVars(value, props);\n\t}\n\tcatch (IllegalArgumentException& e)\n\t{\n\t\tLogLog::error(((LogString) LOG4CXX_STR(\"Bad option value [\"))\n\t\t\t+ value + LOG4CXX_STR(\"].\"), e);\n\t\treturn value;\n\t}\n}\n\nLogString OptionConverter::substVars(const LogString& val, Properties& props)\n{\n\tLogString sbuf;\n\tconst logchar delimStartArray[] = { 0x24, 0x7B, 0 };\n\tconst LogString delimStart(delimStartArray);\n\tconst logchar delimStop = 0x7D; // '}';\n\tconst size_t DELIM_START_LEN = 2;\n\tconst size_t DELIM_STOP_LEN = 1;\n\n\tsize_t i = 0;\n\n\twhile (true)\n\t{\n\t\tsize_t j = val.find(delimStart, i);\n\n\t\tif (j == val.npos)\n\t\t{\n\t\t\t// no more variables\n\t\t\tif (i == 0)\n\t\t\t{\n\t\t\t\t// this is a simple string\n\t\t\t\treturn val;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t// add the tail string which contails no variables and return the result.\n\t\t\t\tsbuf.append(val.substr(i, val.length() - i));\n\t\t\t\treturn sbuf;\n\t\t\t}\n\t\t}\n\t\telse\n\t\t{\n\t\t\tsbuf.append(val.substr(i, j - i));\n\t\t\tsize_t k = val.find(delimStop, j);\n\n\t\t\tif (k == val.npos)\n\t\t\t{\n\t\t\t\tLogString msg(1, (logchar) 0x22 /* '\\\"' */);\n\t\t\t\tmsg.append(val);\n\t\t\t\tmsg.append(LOG4CXX_STR(\"\\\" has no closing brace. Opening brace at position \"));\n\t\t\t\tPool p;\n\t\t\t\tStringHelper::toString(j, p, msg);\n\t\t\t\tmsg.append(1, (logchar) 0x2E /* '.' */);\n\t\t\t\tthrow IllegalArgumentException(msg);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tj += DELIM_START_LEN;\n\t\t\t\tLogString key = val.substr(j, k - j);\n\t\t\t\t// first try in System properties\n\t\t\t\tLogString replacement(getSystemProperty(key, LogString()));\n\n\t\t\t\t// then try props parameter\n\t\t\t\tif (replacement.empty())\n\t\t\t\t{\n\t\t\t\t\treplacement = props.getProperty(key);\n\t\t\t\t}\n\n\t\t\t\tif (!replacement.empty())\n\t\t\t\t{\n\t\t\t\t\t// Do variable substitution on the replacement string\n\t\t\t\t\t// such that we can solve \"Hello ${x2}\" as \"Hello p1\"\n\t\t\t\t\t// the where the properties are\n\t\t\t\t\t// x1=p1\n\t\t\t\t\t// x2=${x1}\n\t\t\t\t\tLogString recursiveReplacement = substVars(replacement, props);\n\t\t\t\t\tsbuf.append(recursiveReplacement);\n\t\t\t\t}\n\n\t\t\t\ti = k + DELIM_STOP_LEN;\n\t\t\t}\n\t\t}\n\t}\n}\n\nLogString OptionConverter::getSystemProperty(const LogString& key, const LogString& def)\n{\n\tif (!key.empty())\n\t{\n\t\tLogString value(System::getProperty(key));\n\n\t\tif (!value.empty())\n\t\t{\n\t\t\treturn value;\n\t\t}\n\t}\n\n\treturn def;\n}\n\nLevelPtr OptionConverter::toLevel(const LogString& value,\n\tconst LevelPtr& defaultValue)\n{\n\tsize_t hashIndex = value.find(LOG4CXX_STR(\"#\"));\n\n\tif (hashIndex == LogString::npos)\n\t{\n\t\tif (value.empty())\n\t\t{\n\t\t\treturn defaultValue;\n\t\t}\n\t\telse\n\t\t{\n\t\t\tLogLog::debug(\n\t\t\t\t((LogString) LOG4CXX_STR(\"OptionConverter::toLevel: no class name specified, level=[\"))\n\t\t\t\t+ value\n\t\t\t\t+ LOG4CXX_STR(\"]\"));\n\t\t\t// no class name specified : use standard Level class\n\t\t\treturn Level::toLevelLS(value, defaultValue);\n\t\t}\n\t}\n\n\tLogString clazz = value.substr(hashIndex + 1);\n\tLogString levelName = value.substr(0, hashIndex);\n\tLogLog::debug(((LogString) LOG4CXX_STR(\"OptionConverter::toLevel: class=[\"))\n\t\t+ clazz + LOG4CXX_STR(\"], level=[\") + levelName + LOG4CXX_STR(\"]\"));\n\n\t// This is degenerate case but you never know.\n\tif (levelName.empty())\n\t{\n\t\treturn Level::toLevelLS(value, defaultValue);\n\t}\n\n\ttry\n\t{\n\t\tLevel::LevelClass& levelClass =\n\t\t\t(Level::LevelClass&)Loader::loadClass(clazz);\n\t\treturn levelClass.toLevel(levelName);\n\t}\n\tcatch (ClassNotFoundException&)\n\t{\n\t\tLogLog::warn(((LogString) LOG4CXX_STR(\"custom level class [\"))\n\t\t\t+ clazz + LOG4CXX_STR(\"] not found.\"));\n\t}\n\tcatch (Exception& oops)\n\t{\n\t\tLogLog::warn(\n\t\t\tLOG4CXX_STR(\"class [\") + clazz + LOG4CXX_STR(\"], level [\") + levelName +\n\t\t\tLOG4CXX_STR(\"] conversion) failed.\"), oops);\n\t}\n\tcatch (...)\n\t{\n\t\tLogLog::warn(\n\t\t\tLOG4CXX_STR(\"class [\") + clazz + LOG4CXX_STR(\"], level [\") + levelName +\n\t\t\tLOG4CXX_STR(\"] conversion) failed.\"));\n\t}\n\n\treturn defaultValue;\n}\n\n\nObjectPtr OptionConverter::instantiateByKey(Properties& props, const LogString& key,\n\tconst Class& superClass, const ObjectPtr& defaultValue)\n{\n\t// Get the value of the property in string form\n\tLogString className(findAndSubst(key, props));\n\n\tif (className.empty())\n\t{\n\t\tLogLog::error(\n\t\t\t((LogString) LOG4CXX_STR(\"Could not find value for key \")) + key);\n\t\treturn defaultValue;\n\t}\n\n\t// Trim className to avoid trailing spaces that cause problems.\n\treturn OptionConverter::instantiateByClassName(\n\t\t\tStringHelper::trim(className), superClass, defaultValue);\n}\n\nObjectPtr OptionConverter::instantiateByClassName(const LogString& className,\n\tconst Class& superClass, const ObjectPtr& defaultValue)\n{\n\tif (!className.empty())\n\t{\n\t\ttry\n\t\t{\n\t\t\tconst Class& classObj = Loader::loadClass(className);\n\t\t\tObjectPtr newObject =  ObjectPtr(classObj.newInstance());\n\n\t\t\tif (!newObject->instanceof(superClass))\n\t\t\t{\n\t\t\t\treturn defaultValue;\n\t\t\t}\n\n\t\t\treturn newObject;\n\t\t}\n\t\tcatch (Exception& e)\n\t\t{\n\t\t\tLogLog::error(LOG4CXX_STR(\"Could not instantiate class [\") +\n\t\t\t\tclassName + LOG4CXX_STR(\"].\"), e);\n\t\t}\n\t}\n\n\treturn defaultValue;\n}\n\nvoid OptionConverter::selectAndConfigure(const File& configFileName,\n\tconst LogString& _clazz, spi::LoggerRepositoryPtr hierarchy, int delay)\n{\n\tConfiguratorPtr configurator;\n\tLogString clazz = _clazz;\n\n\tLogString filename(configFileName.getPath());\n\n#if LOG4CXX_HAS_DOMCONFIGURATOR\n\tif (clazz.empty()\n\t\t&& filename.length() > 4\n\t\t&& StringHelper::equalsIgnoreCase(\n\t\t\tfilename.substr(filename.length() - 4),\n\t\t\tLOG4CXX_STR(\".XML\"), LOG4CXX_STR(\".xml\")))\n\t{\n\t\tclazz = LOG4CXX_NS::xml::DOMConfigurator::getStaticClass().toString();\n\t}\n#endif\n\n\tif (!clazz.empty())\n\t{\n\t\tLogLog::debug(LOG4CXX_STR(\"Preferred configurator class: \") + clazz);\n\t\tconst Class& clazzObj = Loader::loadClass(clazz);\n\t\tObjectPtr obj = ObjectPtr(clazzObj.newInstance());\n\t\tconfigurator = LOG4CXX_NS::cast<Configurator>(obj);\n\n\t\tif (configurator == 0)\n\t\t{\n\t\t\tLogLog::error(LOG4CXX_STR(\"Could not instantiate configurator [\")\n\t\t\t\t+ clazz + LOG4CXX_STR(\"].\"));\n\t\t\treturn;\n\t\t}\n\t}\n\telse\n\t{\n\t\tconfigurator = std::make_shared<PropertyConfigurator>();\n\t}\n\n\tif (0 < delay)\n\t{\n\t\tauto dog = new ConfiguratorWatchdog(configurator, configFileName);\n\t\tAPRInitializer::registerCleanup(dog);\n\t\tdog->setDelay(delay);\n\t\tdog->start();\n\t}\n\telse\n\t\tconfigurator->doConfigure(configFileName, hierarchy);\n}\n\n// Path: src/main/cpp/locale.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <log4cxx/helpers/locale.h>\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nstruct Locale::LocalePrivate\n{\n\tLocalePrivate(const LogString& language1)\n\t\t: language(language1)\n\t{\n\t}\n\n\tLocalePrivate(const LogString& language1, const LogString& country1)\n\t\t: language(language1), country(country1)\n\t{\n\t}\n\n\tLocalePrivate(const LogString& language1, const LogString& country1,\n\t\tconst LogString& variant1)\n\t\t: language(language1), country(country1), variant(variant1)\n\t{\n\t}\n\n\tconst LogString language;\n\tconst LogString country;\n\tconst LogString variant;\n};\n\nLocale::Locale(const LogString& language1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1))\n{\n}\n\nLocale::Locale(const LogString& language1, const LogString& country1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1, country1))\n{\n}\n\nLocale::Locale(const LogString& language1, const LogString& country1,\n\tconst LogString& variant1)\n\t: m_priv(std::make_unique<LocalePrivate>(language1, country1, variant1))\n{\n}\n\nLocale::~Locale() {}\n\nconst LogString& Locale::getLanguage() const\n{\n\treturn m_priv->language;\n}\n\nconst LogString& Locale::getCountry() const\n{\n\treturn m_priv->country;\n}\n\nconst LogString& Locale::getVariant() const\n{\n\treturn m_priv->variant;\n}\n\n\n// Path: src/main/cpp/charsetencoder.cpp\n/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/exception.h>\n#include <apr_xlate.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <algorithm>\n\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n\n#include <log4cxx/private/log4cxx_private.h>\n#include <apr_portable.h>\n#include <mutex>\n\n#ifdef LOG4CXX_HAS_WCSTOMBS\n\t#include <stdlib.h>\n#endif\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(CharsetEncoder)\n\nnamespace LOG4CXX_NS\n{\n\nnamespace helpers\n{\n\n#if APR_HAS_XLATE\n/**\n* A character encoder implemented using apr_xlate.\n*/\nclass APRCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\t\nAPRCharsetEncoder(const LogString& topage) : pool()\n\t\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\t\t\tconst char* frompage = \"WCHAR_T\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\tconst char* frompage = \"UTF-8\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\t\tconst char* frompage = \"UTF-16\";\n#endif\n\t\t\tstd::string tpage(Transcoder::encodeCharsetName(topage));\n\t\t\tapr_status_t stat = apr_xlate_open(&convset,\n\t\t\t\t\ttpage.c_str(),\n\t\t\t\t\tfrompage,\n\t\t\t\t\tpool.getAPRPool());\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tthrow IllegalArgumentException(topage);\n\t\t\t}\n\t\t}\n\n\t\tvirtual ~APRCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tapr_status_t stat;\n\t\t\tsize_t outbytes_left = out.remaining();\n\t\t\tsize_t initial_outbytes_left = outbytes_left;\n\t\t\tsize_t position = out.position();\n\n\t\t\tif (iter == in.end())\n\t\t\t{\n\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\tstat = apr_xlate_conv_buffer(convset, NULL, NULL,\n\t\t\t\t\t\tout.data() + position, &outbytes_left);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tLogString::size_type inOffset = (iter - in.begin());\n\t\t\t\tapr_size_t inbytes_left =\n\t\t\t\t\t(in.size() - inOffset) * sizeof(LogString::value_type);\n\t\t\t\tapr_size_t initial_inbytes_left = inbytes_left;\n\t\t\t\t{\n\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\tstat = apr_xlate_conv_buffer(convset,\n\t\t\t\t\t\t\t(const char*) (in.data() + inOffset),\n\t\t\t\t\t\t\t&inbytes_left,\n\t\t\t\t\t\t\tout.data() + position,\n\t\t\t\t\t\t\t&outbytes_left);\n\t\t\t\t}\n\t\t\t\titer += ((initial_inbytes_left - inbytes_left) / sizeof(LogString::value_type));\n\t\t\t}\n\n\t\t\tout.position(out.position() + (initial_outbytes_left - outbytes_left));\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tAPRCharsetEncoder(const APRCharsetEncoder&);\n\t\tAPRCharsetEncoder& operator=(const APRCharsetEncoder&);\n\t\tPool pool;\n\t\tstd::mutex mutex;\n\t\tapr_xlate_t* convset;\n};\n#endif\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS\n/**\n *  A character encoder implemented using wcstombs.\n*/\n...\n", "needle_token_start": 14257, "needle_token_end": 14503, "code_context_ntokens": 15137, "output": ["```cpp\n#include <log4cxx/logstring.h>\n#include <log4cxx/helpers/charsetencoder.h>\n#include <log4cxx/helpers/bytebuffer.h>\n#include <log4cxx/helpers/exception.h>\n#include <apr_xlate.h>\n#include <log4cxx/helpers/stringhelper.h>\n#include <log4cxx/helpers/transcoder.h>\n#include <algorithm>\n\n#if !defined(LOG4CXX)\n\t#define LOG4CXX 1\n#endif\n\n#include <log4cxx/private/log4cxx_private.h>\n#include <apr_portable.h>\n#include <mutex>\n\n#ifdef LOG4CXX_HAS_WCSTOMBS\n\t#include <stdlib.h>\n#endif\n\nusing namespace LOG4CXX_NS;\nusing namespace LOG4CXX_NS::helpers;\n\nIMPLEMENT_LOG4CXX_OBJECT(CharsetEncoder)\n\nnamespace LOG4CXX_NS\n{\n\nnamespace helpers\n{\n\n#if APR_HAS_XLATE\n/**\n* A character encoder implemented using apr_xlate.\n*/\nclass APRCharsetEncoder : public CharsetEncoder\n{\n\tpublic:\n\t\t\nAPRCharsetEncoder(const LogString& topage) : pool()\n\t\t{\n#if LOG4CXX_LOGCHAR_IS_WCHAR\n\t\t\tconst char* frompage = \"WCHAR_T\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UTF8\n\t\t\tconst char* frompage = \"UTF-8\";\n#endif\n#if LOG4CXX_LOGCHAR_IS_UNICHAR\n\t\t\tconst char* frompage = \"UTF-16\";\n#endif\n\t\t\tstd::string tpage(Transcoder::encodeCharsetName(topage));\n\t\t\tapr_status_t stat = apr_xlate_open(&convset,\n\t\t\t\t\ttpage.c_str(),\n\t\t\t\t\tfrompage,\n\t\t\t\t\tpool.getAPRPool());\n\n\t\t\tif (stat != APR_SUCCESS)\n\t\t\t{\n\t\t\t\tthrow IllegalArgumentException(topage);\n\t\t\t}\n\t\t}\n\n\t\tvirtual ~APRCharsetEncoder()\n\t\t{\n\t\t}\n\n\t\tvirtual log4cxx_status_t encode(const LogString& in,\n\t\t\tLogString::const_iterator& iter,\n\t\t\tByteBuffer& out)\n\t\t{\n\t\t\tapr_status_t stat;\n\t\t\tsize_t outbytes_left = out.remaining();\n\t\t\tsize_t initial_outbytes_left = outbytes_left;\n\t\t\tsize_t position = out.position();\n\n\t\t\tif (iter == in.end())\n\t\t\t{\n\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\tstat = apr_xlate_conv_buffer(convset, NULL, NULL,\n\t\t\t\t\t\tout.data() + position, &outbytes_left);\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tLogString::size_type inOffset = (iter - in.begin());\n\t\t\t\tapr_size_t inbytes_left =\n\t\t\t\t\t(in.size() - inOffset) * sizeof(LogString::value_type);\n\t\t\t\tapr_size_t initial_inbytes_left = inbytes_left;\n\t\t\t\t{\n\t\t\t\t\tstd::unique_lock<std::mutex> lock(mutex);\n\t\t\t\t\tstat = apr_xlate_conv_buffer(convset,\n\t\t\t\t\t\t\t(const char*) (in.data() + inOffset),\n\t\t\t\t\t\t\t&inbytes_left,\n\t\t\t\t\t\t\tout.data() + position,\n\t\t\t\t\t\t\t&outbytes_left);\n\t\t\t\t}\n\t\t\t\titer += ((initial_inbytes_left - inbytes_left) / sizeof(LogString::value_type));\n\t\t\t}\n\n\t\t\tout.position(out.position() + (initial_outbytes_left - outbytes_left));\n\t\t\treturn stat;\n\t\t}\n\n\tprivate:\n\t\tAPRCharsetEncoder(const APRCharsetEncoder&);\n\t\tAPRCharsetEncoder& operator=(const APRCharsetEncoder&);\n\t\tPool pool;\n\t\tstd::mutex mutex;\n\t\tapr_xlate_t* convset;\n};\n#endif\n\n#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_WCSTOMBS\n/**\n *  A character encoder implemented using wcstombs.\n*/\n...\n```"]}
{"repo": "skypjack/uvw", "name": "try_read", "language": "cpp", "path": "src/uvw/util.h", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To attempt reading data using a provided function, handling potential buffer size issues dynamically.\n2. **Input**: A function that reads data into a buffer, additional arguments for the function, and a buffer size.\n3. **Output**: A string containing the data read, or an empty string if the read fails.\n4. **Procedure**: \n   - Initially, a fixed-size buffer is used to attempt reading data.\n   - If the buffer is too small (indicated by a specific error code), a larger buffer is dynamically allocated based on the required size.\n   - The function is called again with the new buffer. If successful, the data is converted to a string; otherwise, an empty string is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " */\nstruct uts_name {\n    uts_name(std::shared_ptr<uv_utsname_t> init);\n\n    /**\n     * @brief Gets the operating system name (like \"Linux\").\n     * @return The operating system name.\n     */\n    std::string sysname() const noexcept;\n\n    /**\n     * @brief Gets the operating system release (like \"2.6.28\").\n     * @return The operating system release.\n     */\n    std::string release() const noexcept;\n\n    /**\n     * @brief Gets the operating system version.\n     * @return The operating system version\n     */\n    std::string version() const noexcept;\n\n    /**\n     * @brief Gets the hardware identifier.\n     * @return The hardware identifier.\n     */\n    std::string machine() const noexcept;\n\nprivate:\n    std::shared_ptr<uv_utsname_t> uname;\n};\n\n/**\n * @brief The IPv4 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv4 {};\n\n/**\n * @brief The IPv6 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv6 {};\n\n/**\n * @brief Address representation.\n */\nstruct socket_address {\n    std::string ip;    /*!< Either an IPv4 or an IPv6. */\n    unsigned int port; /*!< A valid service identifier. */\n};\n\n/**\n * \\brief CPU information.\n */\nstruct cpu_info {\n    using cpu_time = decltype(uv_cpu_info_t::cpu_times);\n\n    std::string model; /*!< The model of the CPU. */\n    int speed;         /*!< The frequency of the CPU. */\n\n    /**\n     * @brief CPU times.\n     *\n     * It is built up of the following data members: `user`, `nice`, `sys`,\n     * `idle`, `irq`, all of them having type `uint64_t`.\n     */\n    cpu_time times;\n};\n\n/**\n * \\brief Interface address.\n */\nstruct interface_address {\n    std::string name;       /*!< The name of the interface (as an example _eth0_). */\n    char physical[6];       /*!< The physical address. */\n    bool internal;          /*!< True if it is an internal interface (as an example _loopback_), false otherwise. */\n    socket_address address; /*!< The address of the given interface. */\n    socket_address netmask; /*!< The netmask of the given interface. */\n};\n\nnamespace details {\n\nstatic constexpr std::size_t DEFAULT_SIZE = 128;\n\ntemplate<typename F, typename... Args>\n\nstd::string try_read(F &&f, Args &&...args) noexcept {\n    std::size_t size = DEFAULT_SIZE;\n    char buf[DEFAULT_SIZE];\n    std::string str{};\n    auto err = std::forward<F>(f)(args..., buf, &size);\n\n    if(UV_ENOBUFS == err) {\n        std::unique_ptr<char[]> data{new char[size]};\n        err = std::forward<F>(f)(args..., data.get(), &size);\n\n        if(0 == err) {\n            str = data.get();\n        }\n    } else if(0 == err) {\n        str.assign(buf, size);\n    }\n\n    return str;\n}\n\nvoid common_alloc_callback(uv_handle_t *, std::size_t suggested, uv_buf_t *buf);\n\ntemplate<typename Type, auto Alloc>\nvoid common_alloc_callback(uv_handle_t *handle, std::size_t suggested, uv_buf_t *buf) {\n    auto [alloc, size] = Alloc(*static_cast<const Type *>(handle->data), suggested);\n    *buf = uv_buf_init(alloc, static_cast<unsigned int>(size));\n}\n\nsockaddr ip_addr(const char *addr, unsigned int port);\nsocket_address sock_addr(const sockaddr_in &addr);\nsocket_address sock_addr(const sockaddr_in6 &addr);\nsocket_address sock_addr(const sockaddr &addr);\nsocket_address sock_addr(const sockaddr_storage &storage);\n\n} // namespace details\n\n/**\n * @brief Miscellaneous utilities.\n *\n * Miscellaneous functions that don\u2019t really belong to any other class.\n */\nstruct utilities {\n    using malloc_func_type = void *(*)(size_t);\n    using realloc_func_type = void *(*)(void *, size_t);\n    using calloc_func_type = void *(*)(size_t, size_t);\n    using free_func_type = void (*)(void *);\n\n    /**\n     * @brief OS dedicated utilities.\n     */\n    struct os {\n        /**\n         * @brief Returns the current process id.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_getpid)\n         * for further details.\n         *\n         * @return The current process id.\n         */\n        static pid_type pid() noexcept;\n\n        /**\n         * @brief Returns the parent process id.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_getppid)\n         * for further details.\n         *\n         * @return The parent process id.\n         */\n        static pid_type ppid() noexcept;\n\n        /**\n         * @brief Gets the current user's home directory.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_homedir)\n         * for further details.\n         *\n         * @return The current user's home directory, an empty string in case of\n         * errors.\n         */\n        static std::string homedir() noexcept;\n\n        /**\n         * @brief Gets the temp directory.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_tmpdir)\n         * for further details.\n         *\n         * @return The temp directory, an empty string in case of errors.\n         */\n        static std::string tmpdir() noexcept;\n\n        /**\n         * @brief Retrieves an environment variable.\n         * @param name The name of the variable to be retrieved.\n         * @return The value of the environment variable, an empty string in\n         * case of errors.\n         */\n        static std::string env(const std::string &name) noexcept;\n\n        /**\n         * @brief Creates, updates or deletes an environment variable.\n         * @param name The name of the variable to be updated.\n         * @param value The value to be used for the variable (an empty string\n         * to unset it).\n         * @return True in case of success, false otherwise.\n         */\n        static bool env(const std::string &name, const std::string &value) noexcept;\n\n        /**\n         * @brief Retrieves all environment variables and iterates them.\n         *\n         * Environment variables are passed one at a time to the callback in the\n         * form of `std::string_view`s.<br/>\n         * The signature of the function call operator must be such that it\n         * accepts two parameters, the name and the value of the i-th variable.\n         *\n         * @tparam Func Type of a function object to which to pass environment\n         * variables.\n         * @param func A function object to which to pass environment variables.\n         * @return True in case of success, false otherwise.\n         */\n        template<typename Func>\n        static std::enable_if_t<std::is_invocable_v<Func, std::string_view, std::string_view>, bool>\n        env(Func func) noexcept {\n            uv_env_item_t *items = nullptr;\n            int count{};\n\n            const bool ret = (uv_os_environ(&items, &count) == 0);\n\n            if(ret) {\n                for(int pos = 0; pos < count; ++pos) {\n                    func(std::string_view{items[pos].name}, std::string_view{items[pos].value});\n                }\n\n                uv_os_free_environ(items, count);\n            }\n\n            return ret;\n        }\n\n        /**\n         * @brief Returns the hostname.\n         * @return The hostname, an empty string in case of errors.\n         */\n        static std::string hostname() noexcept;\n\n        /**\n         * @brief Gets name and information about the current kernel.\n         *\n         * This function can be used to get name and information about the\n         * current kernel. The populated data includes the operating system\n         * name, release, version, and machine.\n         *\n         * @return Name and information about the current kernel.\n         */\n        static uts_name uname() noexcept;\n\n        /**\n         * @brief Gets a subset of the password file entry.\n         *\n         * This function can be used to get the subset of the password file\n         * entry for the current effective uid (not the real uid).\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_get_passwd)\n         * for further details.\n         *\n         * @return The accessible subset of the password file entry.\n         */\n        static passwd_info passwd() noexcept;\n\n        /**\n         * @brief Retrieves the scheduling priority of a process.\n         *\n         * The returned value is between -20 (high priority) and 19 (low priority).\n         * A value that is out of range is returned in case of errors.\n         *\n         * @note\n         * On Windows, the result won't equal necessarily the exact value of the\n         * priority because of a mapping to a Windows priority class.\n         *\n         * @param pid A valid process id.\n         * @return The scheduling priority of the process.\n         */\n        static int priority(pid_type pid);\n\n        /**\n         * @brief Sets the scheduling priority of a process.\n         *\n         * The returned value range is between -20 (high priority) and 19 (low\n         * priority).\n         *\n         * @note\n         * On Windows, the priority is mapped to a Windows priority class. When\n         * retrieving the process priority, the result won't equal necessarily the\n         * exact value of the priority.\n         *\n         * @param pid A valid process id.\n         * @param prio The scheduling priority to set to the process.\n         * @return True in case of success, false otherwise.\n         */\n        static bool priority(pid_type pid, int prio);\n    };\n\n    /**\n     * @brief Gets the type of the handle given a category.\n     * @param category A properly initialized handle category.\n     * @return The actual type of the handle as defined by handle_type\n     */\n    static handle_type guess_handle(handle_category category) noexcept;\n\n    /**\n     * @brief Gets the type of the stream to be used with the given descriptor.\n     *\n     * Returns the type of stream that should be used with a given file\n     * descriptor.<br/>\n     * Usually this will be used during initialization to guess the type of the\n     * stdio streams.\n     *\n     * @param file A valid descriptor.\n     * @return One of the following types:\n     *\n     * * `handle_type::UNKNOWN`\n     * * `handle_type::PIPE`\n     * * `handle_type::TCP`\n     * * `handle_type::TTY`\n     * * `handle_type::UDP`\n     * * `handle_type::FILE`\n     */\n    static handle_type guess_handle(file_handle file) noexcept;\n\n    /** @brief Gets information about the CPUs on the system.\n     *\n     * This function can be used to query the underlying system and get a set of\n     * descriptors of all the available CPUs.\n     *\n     * @return A set of descriptors of all the available CPUs.\n     */\n    static std::vector<cpu_info> cpu() noexcept;\n\n    /**\n     * @brief Gets a set of descriptors of all the available interfaces.\n     *\n     * This function can be used to query the underlying system and get a set of\n     * descriptors of all the available interfaces, either internal or not.\n     *\n     * @return A set of descriptors of all the available interfaces.\n     */\n    static std::vector<interface_address> interface_addresses() noexcept;\n\n    /**\n     * @brief IPv6-capable implementation of\n     * [if_indextoname](https://linux.die.net/man/3/if_indextoname).\n     *\n     * Mapping between network interface names and indexes.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_if_indextoname)\n     * for further details.\n     *\n     * @param index Network interface index.\n     * @return Network interface name.\n     */\n    static std::string index_to_name(unsigned int index) noexcept;\n\n    /**\n     * @brief Retrieves a network interface identifier.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_if_indextoiid)\n     * for further details.\n     *\n     * @param index Network interface index.\n     * @return Network interface identifier.\n     */\n    static std::string index_to_iid(unsigned int index) noexcept;\n\n    /**\n     * @brief Override the use of some standard library\u2019s functions.\n     *\n     * Override the use of the standard library\u2019s memory allocation\n     * functions.<br/>\n     * This method must be invoked before any other `uvw` function is called or\n     * after all resources have been freed and thus the underlying library\n     * doesn\u2019t reference any allocated memory chunk.\n     *\n     * If any of the function pointers is _null_, the invokation will fail.\n     *\n     * @note\n     * There is no protection against changing the allocator multiple times. If\n     * the user changes it they are responsible for making sure the allocator is\n     * changed while no memory was allocated with the previous allocator, or\n     * that they are compatible.\n     *\n     * @param malloc_func Replacement function for _malloc_.\n     * @param realloc_func Replacement function for _realloc_.\n     * @param calloc_func Replacement function for _calloc_.\n     * @param free_func Replacement function for _free_.\n     * @return True in case of success, false otherwise.\n     */\n    static bool replace_allocator(malloc_func_type malloc_func, realloc_func_type realloc_func, calloc_func_type calloc_func, free_func_type free_func) noexcept;\n\n    /**\n     * @brief Gets the load average.\n     * @return `[0,0,0]` on Windows (not available), the load average otherwise.\n     */\n    static std::array<double, 3> load_average() noexcept;\n\n    /**\n     * @brief Store the program arguments.\n     *\n     * Required for getting / setting the process title.\n     *\n     * @return Arguments that haven't been consumed internally.\n     */\n    static char **setup_args(int argc, char **argv);\n\n    /**\n     * @brief Gets the title of the current process.\n     * @return The process title.\n     */\n    static std::string process_title();\n\n    /**\n     * @brief Sets the current process title.\n     * @param title The process title to be set.\n     * @return True in case of success, false otherwise.\n     */\n    static bool process_title(const std::string &title);\n\n    /**\n     * @brief Gets memory information (in bytes).\n     * @return Memory information.\n     */\n    static uint64_t total_memory() noexcept;\n\n    /**\n     * @brief Gets the amount of memory available to the process (in bytes).\n     *\n     * Gets the amount of memory available to the process based on limits\n     * imposed by the OS. If there is no such constraint, or the constraint is\n     * unknown, `0` is returned.<br/>\n     * Note that it is not unusual for this value to be less than or greater\n     * than `totalMemory`.\n     *\n     * @return Amount of memory available to the process.\n     */\n    static uint64_t constrained_memory() noexcept;\n\n    /**\n     * @brief Gets the amount of free memory still available to the process.\n     * @return Amount of free memory still available to the process (in bytes).\n     */\n    static uint64_t available_memory() noexcept;\n\n    /**\n     * @brief Gets the current system uptime.\n     * @return The current system uptime or 0 in case of errors.\n     */\n    static double uptime() noexcept;\n\n    /**\n     * @brief Gets the resource usage measures for the current process.\n     * @return Resource usage measures, zeroes-filled object in case of errors.\n     */\n    static resource_usage rusage() noexcept;\n\n    /**\n     * @brief Gets the current system time from a high-resolution clock source.\n     * @param source Clock source, either real-time or monotonic.\n     * @return Current system time from the given high-resolution clock source.\n     */\n    static timespec64 gettime(clock_id source) noexcept;\n\n    /**\n     * @brief Gets the current high-resolution real time.\n     *\n     * The time is expressed in nanoseconds. It is relative to an arbitrary time\n     * in the past. It is not related to the time of the day and therefore not\n     * subject to clock drift. The primary use is for measuring performance\n     * between interval.\n     *\n     * @return The current high-resolution real time.\n     */\n    static uint64_t hrtime() noexcept;\n\n    /**\n     * @brief Gets the executable path.\n     * @return The executable path, an empty string in case of errors.\n     */\n    static std::string path() noexcept;\n\n    /**\n     * @brief Gets the current working directory.\n     * @return The current working directory, an empty string in case of errors.\n     */\n    static std::string cwd() noexcept;\n\n    /**\n     * @brief Changes the current working directory.\n     * @param dir The working directory to be set.\n     * @return True in case of success, false otherwise.\n     */\n    static bool chdir(const std::string &dir) noexcept;\n\n    /**\n     * @brief Cross-platform implementation of\n     * [`gettimeofday`](https://linux.die.net/man/2/gettimeofday)\n     * @return The current time.\n     */\n    static timeval64 time_of_day() noexcept;\n\n    /**\n     * @brief Causes the calling thread to sleep for a while.\n     * @param msec Number of milliseconds to sleep.\n     */\n    static void sleep(unsigned int msec) noexcept;\n\n    /**\n     * @brief Returns an estimate of the amount of parallelism a program should\n     * use (always a non-zero value).\n     * @return Estimate of the amount of parallelism a program should use.\n     */\n    static unsigned int available_parallelism() noexcept;\n};\n\n/**\n * @brief Helper type for visitors.\n * @tparam Func Types of function objects.\n */\ntemplate<class... Func>\nstruct overloaded: Func... {\n    using Func::operator()...;\n};\n\n/**\n * @brief Deduction guide.\n * @tparam Func Types of function objects.\n */\ntemplate<class... Func>\noverloaded(Func...) -> overloaded<Func...>;\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"util.cpp\"\n#endif\n\n#endif // UVW_UTIL_INCLUDE_H\n\n// Path: src/uvw/loop.cpp\n#ifdef UVW_AS_LIB\n#    include \"loop.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE loop::loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept\n    : uv_loop{std::move(ptr)} {}\n\nUVW_INLINE std::shared_ptr<loop> loop::create() {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{new uv_loop_t, [](uv_loop_t *l) { delete l; }};\n    auto curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n\n    if(uv_loop_init(curr->uv_loop.get())) {\n        curr = nullptr;\n    }\n\n    return curr;\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::create(uv_loop_t *res) {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{res, [](uv_loop_t *) {}};\n    return std::shared_ptr<loop>{new loop{std::move(ptr)}};\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::get_default() {\n    static std::weak_ptr<loop> ref;\n    std::shared_ptr<loop> curr;\n\n    if(ref.expired()) {\n        auto def = uv_default_loop();\n\n        if(def) {\n            auto ptr = std::unique_ptr<uv_loop_t, deleter>(def, [](uv_loop_t *) {});\n            curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n        }\n\n        ref = curr;\n    } else {\n        curr = ref.lock();\n    }\n\n    return curr;\n}\n\nUVW_INLINE loop::~loop() noexcept {\n    if(uv_loop) {\n        close();\n    }\n}\n\nUVW_INLINE int loop::close() {\n    int ret = 0;\n\n    if(uv_loop) {\n        ret = uv_loop_close(uv_loop.get());\n        uv_loop.reset();\n    }\n\n    return ret;\n}\n\nUVW_INLINE int loop::run(run_mode mode) noexcept {\n    return uv_run(uv_loop.get(), static_cast<uv_run_mode>(mode));\n}\n\nUVW_INLINE bool loop::alive() const noexcept {\n    return !!uv_loop_alive(uv_loop.get());\n}\n\nUVW_INLINE void loop::stop() noexcept {\n    uv_stop(uv_loop.get());\n}\n\nUVW_INLINE int loop::descriptor() const noexcept {\n    return uv_backend_fd(uv_loop.get());\n}\n\nUVW_INLINE std::pair<bool, loop::time> loop::timeout() const noexcept {\n    auto to = uv_backend_timeout(uv_loop.get());\n    return std::make_pair(to == -1, time{to});\n}\n\nUVW_INLINE loop::time loop::idle_time() const noexcept {\n    return time{uv_metrics_idle_time(uv_loop.get())};\n}\n\nUVW_INLINE metrics_type loop::metrics() const noexcept {\n    metrics_type res{};\n    uv_metrics_info(uv_loop.get(), &res);\n    return res;\n}\n\nUVW_INLINE loop::time loop::now() const noexcept {\n    return time{uv_now(uv_loop.get())};\n}\n\nUVW_INLINE void loop::update() const noexcept {\n    return uv_update_time(uv_loop.get());\n}\n\nUVW_INLINE int loop::fork() noexcept {\n    return uv_loop_fork(uv_loop.get());\n}\n\nUVW_INLINE void loop::data(std::shared_ptr<void> ud) {\n    user_data = std::move(ud);\n}\n\nUVW_INLINE const uv_loop_t *loop::raw() const noexcept {\n    return uv_loop.get();\n}\n\nUVW_INLINE uv_loop_t *loop::raw() noexcept {\n    return const_cast<uv_loop_t *>(const_cast<const loop *>(this)->raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/loop.h\n#ifndef UVW_LOOP_INCLUDE_H\n#define UVW_LOOP_INCLUDE_H\n\n#ifdef _WIN32\n#    include <ciso646>\n#endif\n\n#include <chrono>\n#include <functional>\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"util.h\"\n\nnamespace uvw {\n\nclass async_handle;\nclass check_handle;\nclass fs_event_handle;\nclass fs_poll_handle;\nclass idle_handle;\nclass pipe_handle;\nclass poll_handle;\nclass prepare_handle;\nclass process_handle;\nclass signal_handle;\nclass tcp_handle;\nclass timer_handle;\nclass tty_handle;\nclass udp_handle;\n\nnamespace details {\n\nenum class uvw_loop_option : std::underlying_type_t<uv_loop_option> {\n    BLOCK_SIGNAL = UV_LOOP_BLOCK_SIGNAL,\n    IDLE_TIME = UV_METRICS_IDLE_TIME\n};\n\nenum class uvw_run_mode : std::underlying_type_t<uv_run_mode> {\n    DEFAULT = UV_RUN_DEFAULT,\n    ONCE = UV_RUN_ONCE,\n    NOWAIT = UV_RUN_NOWAIT\n};\n\n} // namespace details\n\nusing metrics_type = uv_metrics_t; /*!< Library equivalent for uv_metrics_t. */\n\n/**\n * @brief The loop class.\n *\n * The event loop is the central part of `uvw`'s functionalities, as well as\n * `libuv`'s ones.<br/>\n * It takes care of polling for I/O and scheduling callbacks to be run based on\n * different sources of events.\n */\nclass loop final: public emitter<loop>, public std::enable_shared_from_this<loop> {\n    using deleter = void (*)(uv_loop_t *);\n\n    template<typename, typename, typename...>\n    friend class resource;\n\n    class uv_token {\n        friend class loop;\n        explicit uv_token(int) {}\n    };\n\n    template<typename Type>\n    auto init(int, Type &value) -> decltype(value.init()) {\n        return value.init();\n    }\n\n    template<typename Type>\n    int init(char, Type &) {\n        return 0;\n    }\n\n    loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept;\n\npublic:\n    using token = uv_token;\n    using time = std::chrono::duration<uint64_t, std::milli>;\n    using option = details::uvw_loop_option;\n    using run_mode = details::uvw_run_mode;\n\n    /**\n     * @brief Initializes a new loop instance.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create();\n\n    /**\n     * @brief Initializes a new loop instance from an existing resource.\n     *\n     * The lifetime of the resource must exceed that of the instance to which\n     * it's associated. Management of the memory associated with the resource is\n     * in charge of the user.\n     *\n     * @param res A valid pointer to a correctly initialized resource.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create(uv_loop_t *res);\n\n    /**\n     * @brief Gets the initialized default loop.\n     *\n     * It may return an empty pointer in case of failure.<br>\n     * This function is just a convenient way for having a global loop\n     * throughout an application, the default loop is in no way different than\n     * the ones initialized with `create()`.<br>\n     * As such, the default loop can be closed with `close()` so the resources\n     * associated with it are freed (even if it is not strictly necessary).\n     *\n     * @return The initialized default loop.\n     */\n    static std::shared_ptr<loop> get_default();\n\n    loop(const loop &) = delete;\n    loop(loop &&other) = delete;\n\n    loop &operator=(const loop &) = delete;\n    loop &operator=(loop &&other) = delete;\n\n    ~loop() noexcept;\n\n    /**\n     * @brief Sets additional loop options.\n     *\n     * You should normally call this before the first call to uv_run() unless\n     * mentioned otherwise.<br/>\n     * Supported options:\n     *\n     * * `loop::option::BLOCK_SIGNAL`: Block a signal when polling for new\n     * events. A second argument is required and it is the signal number.\n     * * `loop::option::IDLE_TIME`: Accumulate the amount of idle time the event\n     * loop spends in the event provider. This option is necessary to use\n     * `idle_time()`.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_configure)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    template<typename... Args>\n    int configure(option flag, Args &&...args) {\n        return uv_loop_configure(uv_loop.get(), static_cast<uv_loop_option>(flag), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Creates resources of any type.\n     *\n     * This should be used as a default method to create resources.<br/>\n     * The arguments are the ones required for the specific resource.\n     *\n     * Use it as `loop->resource<uvw::timer_handle>()`.\n     *\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> resource(Args &&...args) {\n        auto ptr = uninitialized_resource<R>(std::forward<Args>(args)...);\n        return (init(0, *ptr) == 0) ? ptr : nullptr;\n    }\n\n    /**\n     * @brief Creates uninitialized resources of any type.\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> uninitialized_resource(Args &&...args) {\n        return std::make_shared<R>(token{0}, shared_from_this(), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Releases all internal loop resources.\n     *\n     * Call this function only when the loop has finished executing and all open\n     * handles and requests have been closed, or the loop will error.\n     *\n     * @return Underlying return value.\n     */\n    int close();\n\n    /**\n     * @brief Runs the event loop.\n     *\n     * Available modes are:\n     *\n     * * `loop::run_mode::DEFAULT`: Runs the event loop until there are no more\n     * active and referenced handles or requests.\n     * * `loop::run_mode::ONCE`: Poll for i/o once. Note that this function\n     * blocks if there are no pending callbacks.\n     * * `loop::run_mode::NOWAIT`: Poll for i/o once but don\u2019t block if there\n     * are no pending callbacks.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_run)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int run(run_mode mode = run_mode::DEFAULT) noexcept;\n\n    /**\n     * @brief Checks if there are active resources.\n     * @return True if there are active resources in the loop.\n     */\n    bool alive() const noexcept;\n\n    /**\n     * @brief Stops the event loop.\n     *\n     * It causes `run()` to end as soon as possible.<br/>\n     * This will happen not sooner than the next loop iteration.<br/>\n     * If this function was called before blocking for I/O, the loop won\u2019t block\n     * for I/O on this iteration.\n     */\n    void stop() noexcept;\n\n    /**\n     * @brief Get backend file descriptor.\n     *\n     * Only kqueue, epoll and event ports are supported.<br/>\n     * This can be used in conjunction with `run(loop::run_mode::NOWAIT)` to\n     * poll in one thread and run the event loop\u2019s callbacks in another.\n     *\n     * @return The backend file descriptor.\n     */\n    int descriptor() const noexcept;\n\n    /**\n     * @brief Gets the poll timeout.\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of valid timeout, false otherwise.\n     * * Milliseconds (`std::chrono::duration<uint64_t, std::milli>`).\n     */\n    std::pair<bool, time> timeout() const noexcept;\n\n    /**\n     * @brief Returns the amount of time the event loop has been idle. The call\n     * is thread safe.\n     * @return The accumulated time spent idle.\n     */\n    time idle_time() const noexcept;\n\n    /**\n     * @brief Tracks various internal operations of the event loop.\n     * @return Event loop metrics.\n     */\n    metrics_type metrics() const noexcept;\n\n    /**\n     * @brief Returns the current timestamp in milliseconds.\n     *\n     * The timestamp is cached at the start of the event loop tick.<br/>\n     * The timestamp increases monotonically from some arbitrary point in\n     * time.<br/>\n     * Don\u2019t make assumptions about the starting point, you will only get\n     * disappointed.\n     *\n     * @return The current timestamp in milliseconds (actual type is\n     * `std::chrono::duration<uint64_t, std::milli>`).\n     */\n    time now() const noexcept;\n\n    /**\n     * @brief Updates the event loop\u2019s concept of _now_.\n     *\n     * The current time is cached at the start of the event loop tick in order\n     * to reduce the number of time-related system calls.<br/>\n     * You won\u2019t normally need to call this function unless you have callbacks\n     * that block the event loop for longer periods of time, where _longer_ is\n     * somewhat subjective but probably on the order of a millisecond or more.\n     */\n    void update() const noexcept;\n\n    /**\n     * @brief Walks the list of handles.\n     *\n     * The callback is invoked once for each handle that is still active.\n     *\n     * @param callback A function to invoke once for each active handle.\n     */\n    template<typename Func>\n    void walk(Func callback) {\n        auto func = [](uv_handle_t *hndl, void *callback_func) {\n            if(hndl->data) {\n                auto &cb = *static_cast<Func *>(callback_func);\n\n                switch(utilities::guess_handle(handle_category{hndl->type})) {\n                case handle_type::ASYNC:\n                    cb(*static_cast<async_handle *>(hndl->data));\n                    break;\n                case handle_type::CHECK:\n                    cb(*static_cast<check_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_EVENT:\n                    cb(*static_cast<fs_event_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_POLL:\n                    cb(*static_cast<fs_poll_handle *>(hndl->data));\n                    break;\n                case handle_type::IDLE:\n                    cb(*static_cast<idle_handle *>(hndl->data));\n                    break;\n                case handle_type::PIPE:\n                    cb(*static_cast<pipe_handle *>(hndl->data));\n                    break;\n                case handle_type::POLL:\n                    cb(*static_cast<poll_handle *>(hndl->data));\n                    break;\n                case handle_type::PREPARE:\n                    cb(*static_cast<prepare_handle *>(hndl->data));\n                    break;\n                case handle_type::PROCESS:\n                    cb(*static_cast<process_handle *>(hndl->data));\n                    break;\n                case handle_type::SIGNAL:\n                    cb(*static_cast<signal_handle *>(hndl->data));\n                    break;\n                case handle_type::TCP:\n                    cb(*static_cast<tcp_handle *>(hndl->data));\n                    break;\n                case handle_type::TIMER:\n                    cb(*static_cast<timer_handle *>(hndl->data));\n                    break;\n                case handle_type::TTY:\n                    cb(*static_cast<tty_handle *>(hndl->data));\n                    break;\n                case handle_type::UDP:\n                    cb(*static_cast<udp_handle *>(hndl->data));\n                    break;\n                default:\n                    // this handle isn't managed by uvw, let it be...\n                    break;\n                }\n            }\n        };\n\n        uv_walk(uv_loop.get(), func, &callback);\n    }\n\n    /**\n     * @brief Reinitialize any kernel state necessary in the child process after\n     * a fork(2) system call.\n     *\n     * Previously started watchers will continue to be started in the child\n     * process.\n     *\n     * It is necessary to explicitly call this function on every event loop\n     * created in the parent process that you plan to continue to use in the\n     * child, including the default loop (even if you don\u2019t continue to use it\n     * in the parent). This function must be called before calling any API\n     * function using the loop in the child. Failure to do so will result in\n     * undefined behaviour, possibly including duplicate events delivered to\n     * both parent and child or aborting the child process.\n     *\n     * When possible, it is preferred to create a new loop in the child process\n     * instead of reusing a loop created in the parent. New loops created in the\n     * child process after the fork should not use this function.\n     *\n     * Note that this function is not implemented on Windows.<br/>\n     * Note also that this function is experimental in `libuv`. It may contain\n     * bugs, and is subject to change or removal. API and ABI stability is not\n     * guaranteed.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_fork)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int fork() noexcept;\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param ud User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> ud);\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const uv_loop_t *raw() const noexcept;\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    uv_loop_t *raw() noexcept;\n\nprivate:\n    std::unique_ptr<uv_loop_t, deleter> uv_loop;\n    std::shared_ptr<void> user_data{nullptr};\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"loop.cpp\"\n#endif\n\n#endif // UVW_LOOP_INCLUDE_H\n\n// Path: src/uvw/uv_type.hpp\n#ifndef UVW_UV_TYPE_INCLUDE_H\n#define UVW_UV_TYPE_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include \"config.h\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/**\n * @brief Wrapper class for underlying types.\n *\n * It acts mainly as a wrapper around data structures of the underlying library.\n */\ntemplate<typename U>\nstruct uv_type {\n    explicit uv_type(loop::token, std::shared_ptr<loop> ref) noexcept\n        : owner{std::move(ref)}, resource{} {}\n\n    uv_type(const uv_type &) = delete;\n    uv_type(uv_type &&) = delete;\n\n    uv_type &operator=(const uv_type &) = delete;\n    uv_type &operator=(uv_type &&) = delete;\n\n    /**\n     * @brief Gets the loop from which the resource was originated.\n     * @return A reference to a loop instance.\n     */\n    loop &parent() const noexcept {\n        return *owner;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const U *raw() const noexcept {\n        return &resource;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    U *raw() noexcept {\n        return &resource;\n    }\n\nprotected:\n    ~uv_type() = default;\n\nprivate:\n    std::shared_ptr<loop> owner;\n    U resource;\n};\n\n} // namespace uvw\n\n#endif // UVW_UV_TYPE_INCLUDE_H\n\n// Path: src/uvw/resource.hpp\n#ifndef UVW_RESOURCE_INCLUDE_H\n#define UVW_RESOURCE_INCLUDE_H\n\n#include <memory>\n#include <utility>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Common class for almost all the resources available in `uvw`.\n *\n * This is the base class for handles and requests.\n */\ntemplate<typename T, typename U, typename... E>\nclass resource: public uv_type<U>, public emitter<T, E...>, public std::enable_shared_from_this<T> {\nprotected:\n    int leak_if(int err) noexcept {\n        if(err == 0) {\n            self_ptr = this->shared_from_this();\n        }\n\n        return err;\n    }\n\n    void self_reset() noexcept {\n        self_ptr.reset();\n    }\n\n    bool has_self() const noexcept {\n        return static_cast<bool>(self_ptr);\n    }\n\npublic:\n    explicit resource(loop::token token, std::shared_ptr<loop> ref)\n        : uv_type<U>{token, std::move(ref)} {\n        this->raw()->data = this;\n    }\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param udata User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> udata) {\n        user_data = std::move(udata);\n    }\n\nprivate:\n    std::shared_ptr<void> user_data{nullptr};\n    std::shared_ptr<void> self_ptr{nullptr};\n};\n\n} // namespace uvw\n\n#endif // UVW_RESOURCE_INCLUDE_H\n\n// Path: src/uvw/handle.hpp\n#ifndef UVW_HANDLE_INCLUDE_H\n#define UVW_HANDLE_INCLUDE_H\n\n#include <cstddef>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n#include \"util.h\"\n\nnamespace uvw {\n\n/*! @brief Close event. */\nstruct close_event {};\n\n/**\n * @brief Handle base class.\n *\n * Base type for all `uvw` handle types.\n */\ntemplate<typename T, typename U, typename... E>\nclass handle: public resource<T, U, close_event, E...> {\nprotected:\n    static void close_callback(uv_handle_t *hndl) {\n        handle<T, U, E...> &ref = *(static_cast<T *>(hndl->data));\n        [[maybe_unused]] auto ptr = ref.shared_from_this();\n        ref.self_reset();\n        ref.publish(close_event{});\n    }\n\n    uv_handle_t *as_uv_handle() {\n        return reinterpret_cast<uv_handle_t *>(this->raw());\n    }\n\n    const uv_handle_t *as_uv_handle() const {\n        return reinterpret_cast<const uv_handle_t *>(this->raw());\n    }\n\npublic:\n    using resource<T, U, close_event, E...>::resource;\n\n    /**\n     * @brief Gets the category of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, an opaque value that identifies the\n     * category of the handle is made available to the users.\n     *\n     * @return The actual category of the handle.\n     */\n    handle_category category() const noexcept {\n        return handle_category{as_uv_handle()->type};\n    }\n\n    /**\n     * @brief Gets the type of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, the type of the underlying handle as\n     * specified by handle_type is made available to the users.\n     *\n     * @return The actual type of the handle.\n     */\n    handle_type type() const noexcept {\n        return utilities::guess_handle(category());\n    }\n\n    /**\n     * @brief Checks if the handle is active.\n     *\n     * What _active_ means depends on the type of handle:\n     *\n     * * An async_handle handle is always active and cannot be deactivated,\n     * except by closing it with uv_close().\n     * * A pipe, tcp, udp, etc. handle - basically any handle that deals with\n     * I/O - is active when it is doing something that involves I/O, like\n     * reading, writing, connecting, accepting new connections, etc.\n     * * A check, idle, timer, etc. handle is active when it has been started\n     * with a call to `start()`.\n     *\n     * Rule of thumb: if a handle of type `foo_handle` has a `start()` member\n     * method, then it\u2019s active from the moment that method is called. Likewise,\n     * `stop()` deactivates the handle again.\n     *\n     * @return True if the handle is active, false otherwise.\n     */\n    bool active() const noexcept {\n        return !!uv_is_active(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if a handle is closing or closed.\n     *\n     * This function should only be used between the initialization of the\n     * handle and the arrival of the close callback.\n     *\n     * @return True if the handle is closing or closed, false otherwise.\n     */\n    bool closing() const noexcept {\n        return !!uv_is_closing(as_uv_handle());\n    }\n\n    /**\n     * @brief Request handle to be closed.\n     *\n     * This **must** be called on each handle before memory is released.<br/>\n     * In-progress requests are cancelled and this can result in errors.\n     *\n     * The handle will emit a close event when finished.\n     */\n    void close() noexcept {\n        if(!closing()) {\n            uv_close(as_uv_handle(), &handle<T, U, E...>::close_callback);\n        }\n    }\n\n    /**\n     * @brief Reference the given handle.\n     *\n     * References are idempotent, that is, if a handle is already referenced\n     * calling this function again will have no effect.\n     */\n    void reference() noexcept {\n        uv_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Unreference the given handle.\n     *\n     * References are idempotent, that is, if a handle is not referenced calling\n     * this function again will have no effect.\n     */\n    void unreference() noexcept {\n        uv_unref(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if the given handle referenced.\n     * @return True if the handle referenced, false otherwise.\n     */\n    bool referenced() const noexcept {\n        return !!uv_has_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Returns the size of the underlying handle type.\n     * @return The size of the underlying handle type.\n     */\n    std::size_t size() const noexcept {\n        return uv_handle_size(as_uv_handle()->type);\n    }\n\n    /**\n     * @brief Gets the size of the send buffer used for the socket.\n     *\n     * Gets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipeand udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the send buffer, the underlying return value in case\n     * of errors.\n     */\n    int send_buffer_size() {\n        int value = 0;\n        auto err = uv_send_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the send buffer used for the socket.\n     *\n     * Sets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int send_buffer_size(int value) {\n        return uv_send_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the size of the receive buffer used for the socket.\n     *\n     * Gets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the receive buffer, the underlying return value in\n     * case of errors.\n     */\n    int recv_buffer_size() {\n        int value = 0;\n        auto err = uv_recv_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the receive buffer used for the socket.\n     *\n     * Sets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int recv_buffer_size(int value) {\n        return uv_recv_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the platform dependent file descriptor equivalent.\n     *\n     * Supported handles:\n     *\n     * * tcp_handle\n     * * pipe_handle\n     * * tty_handle\n     * * udp_handle\n     * * poll_handle\n     *\n     * If invoked on a different handle, one that doesn\u2019t have an attached file\n     * descriptor yet or one which was closed, an invalid value is returned.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/handle.html#c.uv_fileno)\n     * for further details.\n     *\n     * @return The file descriptor attached to the hande or a negative value in\n     * case of errors.\n     */\n    os_file_descriptor fd() const {\n        uv_os_fd_t fd;\n        uv_fileno(as_uv_handle(), &fd);\n        return fd;\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_HANDLE_INCLUDE_H\n\n// Path: src/uvw/prepare.h\n#ifndef UVW_PREPARE_INCLUDE_H\n#define UVW_PREPARE_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Prepare event. */\nstruct prepare_event {};\n\n/**\n * @brief The prepare handle.\n *\n * Prepare handles will emit a prepare event once per loop iteration, right\n * before polling for I/O.\n *\n * To create a `prepare_handle` through a `loop`, no arguments are required.\n */\nclass prepare_handle final: public handle<prepare_handle, uv_prepare_t, prepare_event> {\n    static void start_callback(uv_prepare_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * A prepare event will be emitted once per loop iteration, right before\n     * polling for I/O.\n     *\n     * The handle will start emitting prepare events when needed.\n     *\n     * @return Underlying return value.\n     */\n    int start();\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"prepare.cpp\"\n#endif\n\n#endif // UVW_PREPARE_INCLUDE_H\n\n// Path: src/uvw/prepare.cpp\n#ifdef UVW_AS_LIB\n#    include \"prepare.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void prepare_handle::start_callback(uv_prepare_t *hndl) {\n    prepare_handle &prepare = *(static_cast<prepare_handle *>(hndl->data));\n    prepare.publish(prepare_event{});\n}\n\nUVW_INLINE int prepare_handle::init() {\n    return leak_if(uv_prepare_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int prepare_handle::start() {\n    return uv_prepare_start(raw(), &start_callback);\n}\n\nUVW_INLINE int prepare_handle::stop() {\n    return uv_prepare_stop(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/request.hpp\n#ifndef UVW_REQUEST_INCLUDE_H\n#define UVW_REQUEST_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Request base class.\n *\n * Base type for all `uvw` request types.\n */\ntemplate<typename T, typename U, typename... E>\nclass request: public resource<T, U, E...> {\nprotected:\n    static auto reserve(U *req) {\n        auto ptr = static_cast<T *>(req->data)->shared_from_this();\n        ptr->self_reset();\n        return ptr;\n    }\n\npublic:\n    using resource<T, U, E...>::resource;\n\n    /**\n     * @brief Cancels a pending request.\n     *\n     * This method fails if the request is executing or has finished\n     * executing.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/request.html#c.uv_cancel)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int cancel() {\n        return uv_cancel(reinterpret_cast<uv_req_t *>(this->raw()));\n    }\n\n    /**\n     * @brief Returns the size of the underlying request type.\n     * @return The size of the underlying request type.\n     */\n    std::size_t size() const noexcept {\n        return uv_req_size(reinterpret_cast<const uv_req_t *>(this->raw())->type);\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_REQUEST_INCLUDE_H\n\n// Path: src/uvw/stream.h\n#ifndef UVW_STREAM_INCLUDE_H\n#define UVW_STREAM_INCLUDE_H\n\n#include <algorithm>\n#include <cstddef>\n#include <iterator>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"handle.hpp\"\n#include \"loop.h\"\n#include \"request.hpp\"\n\nnamespace uvw {\n\n/*! @brief Connect event. */\nstruct connect_event {};\n\n/*! @brief End event. */\nstruct end_event {};\n\n/*! @brief Listen event. */\nstruct listen_event {};\n\n/*! @brief Shutdown event. */\nstruct shutdown_event {};\n\n/*! @brief Write event. */\nstruct write_event {};\n\n/*! @brief Data event. */\nstruct data_event {\n    explicit data_event(std::unique_ptr<char[]> buf, std::size_t len) noexcept;\n\n    std::unique_ptr<char[]> data; /*!< A bunch of data read on the stream. */\n    std::size_t length;           /*!< The amount of data read on the stream. */\n};\n\nnamespace details {\n\nclass connect_req final: public request<connect_req, uv_connect_t, connect_event> {\n    static void connect_callback(uv_connect_t *req, int status);\n\npublic:\n    using request::request;\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback);\n        return this->leak_if(0);\n    }\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<!std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        return this->leak_if(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback));\n    }\n};\n\nclass shutdown_req final: public request<shutdown_req, uv_shutdown_t, shutdown_event> {\n    static void shoutdown_callback(uv_shutdown_t *req, int status);\n\npublic:\n    using request::request;\n\n    int shutdown(uv_stream_t *hndl);\n};\n\ntemplate<typename Deleter>\nclass write_req final: public request<write_req<Deleter>, uv_write_t, write_event> {\n...\n", "needle_token_start": 692, "needle_token_end": 871, "code_context_ntokens": 16064, "output": ["```cpp\ntemplate<typename F, typename... Args>\nstd::string try_read(F &&f, Args &&...args) noexcept {\n    std::size_t size = DEFAULT_SIZE;\n    char buf[DEFAULT_SIZE];\n    std::string str{};\n    auto err = std::forward<F>(f)(args..., buf, &size);\n\n    if(UV_ENOBUFS == err) {\n        std::unique_ptr<char[]> data{new char[size]};\n        err = std::forward<F>(f)(args..., data.get(), &size);\n\n        if(0 == err) {\n            str = data.get();\n        }\n    } else if(0 == err) {\n        str.assign(buf, size);\n    }\n\n    return str;\n}\n```"]}
{"repo": "skypjack/uvw", "name": "set", "language": "cpp", "path": "src/uvw/thread.h", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to store a variable in thread-local storage, ensuring that the variable is accessible only to the thread that set it.\n2. **Input**: The function takes a pointer to the variable that needs to be stored in the thread-local storage.\n3. **Output**: There is no return value; the function operates by side effect, storing the provided variable in a thread-specific location.\n4. **Procedure**: The function casts the provided variable pointer to a generic type and uses an underlying library function to associate this pointer with a thread-specific key, effectively storing the variable in the thread's local storage.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/check.cpp\n#ifdef UVW_AS_LIB\n#    include \"check.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void check_handle::start_callback(uv_check_t *hndl) {\n    check_handle &check = *(static_cast<check_handle *>(hndl->data));\n    check.publish(check_event{});\n}\n...\n// Path: src/uvw/enum.hpp\n#ifndef UVW_ENUM_INCLUDE_HPP\n#define UVW_ENUM_INCLUDE_HPP\n\n#include <type_traits>\n#include \"config.h\"\n\n/**\n * @brief Operator available for enums for which bitmask support is enabled.\n * @tparam Type Enum class type.\n * @param lhs The first value to use.\n * @param rhs The second value to use.\n * @return The result of invoking the operator on the underlying types of the\n * two values provided.\n */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator|(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) | static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator&(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) & static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator^(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) ^ static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/**\n * @brief Operator available for enums for which bitmask support is enabled.\n * @tparam Type Enum class type.\n * @param value The value to use.\n * @return The result of invoking the operator on the underlying types of the\n * value provided.\n */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator~(const Type value) noexcept {\n    return static_cast<Type>(~static_cast<std::underlying_type_t<Type>>(value));\n}\n\n/*! @copydoc operator~ */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM, bool{})>\noperator!(const Type value) noexcept {\n    return !static_cast<std::underlying_type_t<Type>>(value);\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator|=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs | rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator&=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs & rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator^=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs ^ rhs));\n}\n\n#endif\n\n// Path: src/uvw/thread.h\n#ifndef UVW_THREAD_INCLUDE_H\n#define UVW_THREAD_INCLUDE_H\n\n#include <cstddef>\n#include <cstdint>\n#include <memory>\n#include <string>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"enum.hpp\"\n#include \"loop.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_thread_create_flags : std::underlying_type_t<uv_thread_create_flags> {\n    THREAD_NO_FLAGS = UV_THREAD_NO_FLAGS,\n    THREAD_HAS_STACK_SIZE = UV_THREAD_HAS_STACK_SIZE\n};\n\n}\n\nclass thread;\nclass thread_local_storage;\nclass once;\nclass mutex;\nclass rwlock;\nclass semaphore;\nclass condition;\nclass barrier;\n\n/**\n * @brief The thread wrapper.\n *\n * To create a `thread` through a `loop`, arguments follow:\n *\n * * A callback invoked to initialize thread execution. The type must be such\n * that it can be assigned to an `std::function<void(std::shared_ptr<void>)>`.\n * * An optional payload the type of which is `std::shared_ptr<void>`.\n */\nclass thread final: public uv_type<uv_thread_t> {\n    using internal_task = std::function<void(std::shared_ptr<void>)>;\n\n    static void create_callback(void *arg);\n\npublic:\n    using create_flags = details::uvw_thread_create_flags;\n    using task = internal_task;\n    using type = uv_thread_t;\n\n    explicit thread(loop::token token, std::shared_ptr<loop> ref, task t, std::shared_ptr<void> d = nullptr) noexcept;\n\n    /**\n     * @brief Obtains the identifier of the calling thread.\n     * @return The identifier of the calling thread.\n     */\n    static type self() noexcept;\n\n    /**\n     * @brief Gets the CPU number on which the calling thread is running.\n     * @return The CPU number on which the calling thread is running.\n     */\n    static int getcpu() noexcept;\n\n    /**\n     * @brief Compares thread by means of their identifiers.\n     * @param tl A valid instance of a thread.\n     * @param tr A valid instance of a thread.\n     * @return True if the two threads are the same thread, false otherwise.\n     */\n    static bool equal(const thread &tl, const thread &tr) noexcept;\n\n    ~thread() noexcept;\n\n    /**\n     * @brief Creates a new thread.\n     * @return True in case of success, false otherwise.\n     */\n    bool run() noexcept;\n\n    /**\n     * @brief Creates a new thread.\n     *\n     * Available flags are:\n     *\n     * * `thread::create_flags::THREAD_NO_FLAGS`: no flags set.\n     * * `thread::create_flags::THREAD_HAS_STACK_SIZE`: if set, `stack` specifies a\n     * stack size for the new thread. 0 indicates that the default value should\n     * be used (it behaves as if the flag was not set). Other values will be\n     * rounded up to the nearest page boundary.\n     *\n     * @return True in case of success, false otherwise.\n     */\n    bool run(create_flags opts, std::size_t stack = {}) noexcept;\n\n    /**\n     * @brief Joins with a terminated thread.\n     * @return True in case of success, false otherwise.\n     */\n    bool join() noexcept;\n\nprivate:\n    std::shared_ptr<void> data;\n    task func;\n};\n\n/**\n * @brief The thread local storage wrapper.\n *\n * A storage area that can only be accessed by one thread. The variable can be\n * seen as a global variable that is only visible to a particular thread and not\n * the whole program.\n */\nclass thread_local_storage final: public uv_type<uv_key_t> {\npublic:\n    explicit thread_local_storage(loop::token token, std::shared_ptr<loop> ref) noexcept;\n\n    ~thread_local_storage() noexcept;\n\n    /**\n     * @brief Gets the value of a given variable.\n     * @tparam T Type to which to cast the opaque storage area.\n     * @return A pointer to the given variable.\n     */\n    template<typename T>\n    T *get() noexcept {\n        return static_cast<T *>(uv_key_get(uv_type::raw()));\n    }\n\n    /**\n     * @brief Sets the value of a given variable.\n     * @tparam T Type of the variable to store aside.\n     * @param value A valid pointer to the variable to store\n     */\n    template<typename T>\n    \nvoid set(T *value) noexcept {\n        return uv_key_set(uv_type::raw(), value);\n    }\n};\n\n/**\n * @brief The once wrapper.\n *\n * Runs a function once and only once. Concurrent calls to `once` will block all\n * callers except one (it\u2019s unspecified which one).\n */\nclass once final: public uv_type<uv_once_t> {\n    static uv_once_t *guard() noexcept;\n\npublic:\n    using uv_type::uv_type;\n\n    /**\n     * @brief Runs a function once and only once.\n     *\n     * The callback must be such that it's convertible to `void(*)(void)`. Free\n     * functions and non-capturing lambdas are both viable solutions.\n     *\n     * @tparam F Type of the callback.\n     * @param f A valid callback function.\n     */\n    template<typename F>\n    static void run(F &&f) noexcept {\n        using callback_type = void (*)(void);\n        static_assert(std::is_convertible_v<F, callback_type>);\n        callback_type cb = f;\n        uv_once(guard(), cb);\n    }\n};\n\n/**\n * @brief The mutex wrapper.\n *\n * To create a `mutex` through a `loop`, arguments follow:\n *\n * * An option boolean that specifies if the mutex is a recursive one. The\n * default value is false, the mutex isn't recursive.\n */\nclass mutex final: public uv_type<uv_mutex_t> {\n    friend class condition;\n\npublic:\n    explicit mutex(loop::token token, std::shared_ptr<loop> ref, bool recursive = false) noexcept;\n\n    ~mutex() noexcept;\n\n    /**\n     * @brief Locks the mutex.\n     */\n    void lock() noexcept;\n\n    /**\n     * @brief Tries to lock the mutex.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_lock() noexcept;\n\n    /**\n     * @brief Unlocks the mutex.\n     */\n    void unlock() noexcept;\n};\n\n/**\n * @brief The rwlock wrapper.\n */\nclass rwlock final: public uv_type<uv_rwlock_t> {\npublic:\n    explicit rwlock(loop::token token, std::shared_ptr<loop> ref) noexcept;\n\n    ~rwlock() noexcept;\n\n    /**\n     * @brief Locks a read-write lock object for reading.\n     */\n    void rdlock() noexcept;\n\n    /**\n     * @brief Tries to lock a read-write lock object for reading.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_rdlock() noexcept;\n\n    /**\n     * @brief Unlocks a read-write lock object previously locked for reading.\n     */\n    void rdunlock() noexcept;\n\n    /**\n     * @brief Locks a read-write lock object for writing.\n     */\n    void wrlock() noexcept;\n\n    /**\n     * @brief Tries to lock a read-write lock object for writing.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_wrlock() noexcept;\n\n    /**\n     * @brief Unlocks a read-write lock object previously locked for writing.\n     */\n    void wrunlock() noexcept;\n};\n\n/**\n * @brief The semaphore wrapper.\n *\n * To create a `semaphore` through a `loop`, arguments follow:\n *\n * * An unsigned integer that specifies the initial value for the semaphore.\n */\nclass semaphore final: public uv_type<uv_sem_t> {\npublic:\n    explicit semaphore(loop::token token, std::shared_ptr<loop> ref, unsigned int value) noexcept;\n\n    ~semaphore() noexcept;\n\n    /**\n     * @brief Unlocks a semaphore.\n     */\n    void post() noexcept;\n\n    /**\n     * @brief Locks a semaphore.\n     */\n    void wait() noexcept;\n\n    /**\n     * @brief Tries to lock a semaphore.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_wait() noexcept;\n};\n\n/**\n * @brief The condition wrapper.\n */\nclass condition final: public uv_type<uv_cond_t> {\npublic:\n    explicit condition(loop::token token, std::shared_ptr<loop> ref) noexcept;\n\n    ~condition() noexcept;\n\n    /**\n     * @brief Signals a condition.\n     *\n     * This function shall unblock at least one of the threads that are blocked\n     * on the specified condition variable (if any threads are blocked on it).\n     */\n    void signal() noexcept;\n\n    /**\n     * @brief Broadcasts a condition.\n     *\n     * This function shall unblock threads blocked on a condition variable.\n     */\n    void broadcast() noexcept;\n\n    /**\n     * @brief Waits on a condition.\n     *\n     * These function atomically releases the mutex and causes the calling\n     * thread to block on the condition variable.\n     *\n     * @param mtx A mutex locked by the calling thread, otherwise expect\n     * undefined behavior.\n     */\n    void wait(mutex &mtx) noexcept;\n\n    /**\n     * @brief Waits on a condition.\n     *\n     * These function atomically releases the mutex and causes the calling\n     * thread to block on the condition variable.<br/>\n     * The functions returns with an error if the absolute time specified passes\n     * (that is, system time equals or exceeds it) before the condition is\n     * signaled or broadcasted, or if the absolute time specified has already\n     * been passed at the time of the call.\n     *\n     * @param mtx A mutex locked by the calling thread, otherwise expect\n     * undefined behavior.\n     * @param timeout The maximum time to wait before to return.\n     * @return True in case of success, false otherwise.\n     */\n    bool timed_wait(mutex &mtx, uint64_t timeout) noexcept;\n};\n\n/**\n * @brief The barrier wrapper.\n *\n * To create a `barrier` through a `loop`, arguments follow:\n *\n * * An unsigned integer that specifies the number of threads that must call\n * `wait` before any of them successfully return from the call. The value\n * specified must be greater than zero.\n */\nclass barrier final: public uv_type<uv_barrier_t> {\npublic:\n    explicit barrier(loop::token token, std::shared_ptr<loop> ref, unsigned int count) noexcept;\n\n    ~barrier() noexcept;\n\n    /**\n     * @brief Synchronizes at a barrier.\n     * @return True in case of success, false otherwise.\n     */\n    bool wait() noexcept;\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"thread.cpp\"\n#endif\n\n#endif // UVW_THREAD_INCLUDE_H\n\n// Path: src/uvw/thread.cpp\n#ifdef UVW_AS_LIB\n#    include \"thread.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE thread::thread(loop::token token, std::shared_ptr<loop> ref, task t, std::shared_ptr<void> d) noexcept\n    : uv_type{token, std::move(ref)},\n      data{std::move(d)},\n      func{std::move(t)} {}\n\nUVW_INLINE void thread::create_callback(void *arg) {\n    thread &curr = *(static_cast<thread *>(arg));\n    curr.func(curr.data);\n}\n\nUVW_INLINE thread::type thread::self() noexcept {\n    return uv_thread_self();\n}\n\nUVW_INLINE int thread::getcpu() noexcept {\n    return uv_thread_getcpu();\n}\n\nUVW_INLINE bool thread::equal(const thread &tl, const thread &tr) noexcept {\n    return !(0 == uv_thread_equal(tl.raw(), tr.raw()));\n}\n\nUVW_INLINE thread::~thread() noexcept {\n    join();\n}\n\nUVW_INLINE bool thread::run() noexcept {\n    return (0 == uv_thread_create(raw(), &create_callback, this));\n}\n\nUVW_INLINE bool thread::run(create_flags opts, std::size_t stack) noexcept {\n    uv_thread_options_t params{static_cast<unsigned int>(opts), stack};\n    return (0 == uv_thread_create_ex(raw(), &params, &create_callback, this));\n}\n\nUVW_INLINE bool thread::join() noexcept {\n    return (0 == uv_thread_join(raw()));\n}\n\nUVW_INLINE thread_local_storage::thread_local_storage(loop::token token, std::shared_ptr<loop> ref) noexcept\n    : uv_type{token, std::move(ref)} {\n    uv_key_create(uv_type::raw());\n}\n\nUVW_INLINE thread_local_storage::~thread_local_storage() noexcept {\n    uv_key_delete(uv_type::raw());\n}\n\nUVW_INLINE uv_once_t *once::guard() noexcept {\n    static uv_once_t once = UV_ONCE_INIT;\n    return &once;\n}\n\nUVW_INLINE mutex::mutex(loop::token token, std::shared_ptr<loop> ref, bool recursive) noexcept\n    : uv_type{token, std::move(ref)} {\n    if(recursive) {\n        uv_mutex_init_recursive(raw());\n    } else {\n        uv_mutex_init(raw());\n    }\n}\n\nUVW_INLINE mutex::~mutex() noexcept {\n    uv_mutex_destroy(raw());\n}\n\nUVW_INLINE void mutex::lock() noexcept {\n    uv_mutex_lock(raw());\n}\n\nUVW_INLINE bool mutex::try_lock() noexcept {\n    return (0 == uv_mutex_trylock(raw()));\n}\n\nUVW_INLINE void mutex::unlock() noexcept {\n    uv_mutex_unlock(raw());\n}\n\nUVW_INLINE rwlock::rwlock(loop::token token, std::shared_ptr<loop> ref) noexcept\n    : uv_type{token, std::move(ref)} {\n    uv_rwlock_init(raw());\n}\n\nUVW_INLINE rwlock::~rwlock() noexcept {\n    uv_rwlock_destroy(raw());\n}\n\nUVW_INLINE void rwlock::rdlock() noexcept {\n    uv_rwlock_rdlock(raw());\n}\n\nUVW_INLINE bool rwlock::try_rdlock() noexcept {\n    return (0 == uv_rwlock_tryrdlock(raw()));\n}\n\nUVW_INLINE void rwlock::rdunlock() noexcept {\n    uv_rwlock_rdunlock(raw());\n}\n\nUVW_INLINE void rwlock::wrlock() noexcept {\n    uv_rwlock_wrlock(raw());\n}\n\nUVW_INLINE bool rwlock::try_wrlock() noexcept {\n    return (0 == uv_rwlock_trywrlock(raw()));\n}\n\nUVW_INLINE void rwlock::wrunlock() noexcept {\n    uv_rwlock_wrunlock(raw());\n}\n\nUVW_INLINE semaphore::semaphore(loop::token token, std::shared_ptr<loop> ref, unsigned int value) noexcept\n    : uv_type{token, std::move(ref)} {\n    uv_sem_init(raw(), value);\n}\n\nUVW_INLINE semaphore::~semaphore() noexcept {\n    uv_sem_destroy(raw());\n}\n\nUVW_INLINE void semaphore::post() noexcept {\n    uv_sem_post(raw());\n}\n\nUVW_INLINE void semaphore::wait() noexcept {\n    uv_sem_wait(raw());\n}\n\nUVW_INLINE bool semaphore::try_wait() noexcept {\n    return (0 == uv_sem_trywait(raw()));\n}\n\nUVW_INLINE condition::condition(loop::token token, std::shared_ptr<loop> ref) noexcept\n    : uv_type{token, std::move(ref)} {\n    uv_cond_init(raw());\n}\n\nUVW_INLINE condition::~condition() noexcept {\n    uv_cond_destroy(raw());\n}\n\nUVW_INLINE void condition::signal() noexcept {\n    uv_cond_signal(raw());\n}\n\nUVW_INLINE void condition::broadcast() noexcept {\n    uv_cond_broadcast(raw());\n}\n\nUVW_INLINE void condition::wait(mutex &mtx) noexcept {\n    uv_cond_wait(raw(), mtx.raw());\n}\n\nUVW_INLINE bool condition::timed_wait(mutex &mtx, uint64_t timeout) noexcept {\n    return (0 == uv_cond_timedwait(raw(), mtx.raw(), timeout));\n}\n\nUVW_INLINE barrier::barrier(loop::token token, std::shared_ptr<loop> ref, unsigned int count) noexcept\n    : uv_type{token, std::move(ref)} {\n    uv_barrier_init(raw(), count);\n}\n\nUVW_INLINE barrier::~barrier() noexcept {\n    uv_barrier_destroy(raw());\n}\n\nUVW_INLINE bool barrier::wait() noexcept {\n    return (0 == uv_barrier_wait(raw()));\n}\n\n} // namespace uvw\n\n// Path: src/uvw/async.h\n#ifndef UVW_ASYNC_INCLUDE_H\n#define UVW_ASYNC_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Async event. */\nstruct async_event {};\n\n/**\n * @brief The async handle.\n *\n * Async handles allow the user to _wakeup_ the event loop and get an event\n * emitted from another thread.\n *\n * To create an `async_handle` through a `loop`, no arguments are required.\n */\nclass async_handle final: public handle<async_handle, uv_async_t, async_event> {\n    static void send_callback(uv_async_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     *\n     * Unlike other handle initialization functions, it immediately starts the\n     * handle.\n     *\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Wakeups the event loop and emits the async event.\n     *\n     * It\u2019s safe to call this function from any thread.<br/>\n     * An async event is emitted on the loop thread.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/async.html#c.uv_async_send)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int send();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"async.cpp\"\n#endif\n\n#endif // UVW_ASYNC_INCLUDE_H\n\n// Path: src/uvw/async.cpp\n#ifdef UVW_AS_LIB\n#    include \"async.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void async_handle::send_callback(uv_async_t *hndl) {\n    async_handle &async = *(static_cast<async_handle *>(hndl->data));\n    async.publish(async_event{});\n}\n\nUVW_INLINE int async_handle::init() {\n    return leak_if(uv_async_init(parent().raw(), raw(), &send_callback));\n}\n\nUVW_INLINE int async_handle::send() {\n    return uv_async_send(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/fs.h\n#ifndef UVW_FS_INCLUDE_H\n#define UVW_FS_INCLUDE_H\n\n#include <chrono>\n#include <memory>\n#include <string>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"enum.hpp\"\n#include \"loop.h\"\n#include \"request.hpp\"\n#include \"util.h\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_fs_type : std::underlying_type_t<uv_fs_type> {\n    UNKNOWN = UV_FS_UNKNOWN,\n    CUSTOM = UV_FS_CUSTOM,\n    OPEN = UV_FS_OPEN,\n    CLOSE = UV_FS_CLOSE,\n    READ = UV_FS_READ,\n    WRITE = UV_FS_WRITE,\n    SENDFILE = UV_FS_SENDFILE,\n    STAT = UV_FS_STAT,\n    LSTAT = UV_FS_LSTAT,\n    FSTAT = UV_FS_FSTAT,\n    FTRUNCATE = UV_FS_FTRUNCATE,\n    UTIME = UV_FS_UTIME,\n    FUTIME = UV_FS_FUTIME,\n    ACCESS = UV_FS_ACCESS,\n    CHMOD = UV_FS_CHMOD,\n    FCHMOD = UV_FS_FCHMOD,\n    FSYNC = UV_FS_FSYNC,\n    FDATASYNC = UV_FS_FDATASYNC,\n    UNLINK = UV_FS_UNLINK,\n    RMDIR = UV_FS_RMDIR,\n    MKDIR = UV_FS_MKDIR,\n    MKDTEMP = UV_FS_MKDTEMP,\n    RENAME = UV_FS_RENAME,\n    SCANDIR = UV_FS_SCANDIR,\n    LINK = UV_FS_LINK,\n    SYMLINK = UV_FS_SYMLINK,\n    READLINK = UV_FS_READLINK,\n    CHOWN = UV_FS_CHOWN,\n    FCHOWN = UV_FS_FCHOWN,\n    REALPATH = UV_FS_REALPATH,\n    COPYFILE = UV_FS_COPYFILE,\n    LCHOWN = UV_FS_LCHOWN,\n    OPENDIR = UV_FS_OPENDIR,\n    READDIR = UV_FS_READDIR,\n    CLOSEDIR = UV_FS_CLOSEDIR,\n    STATFS = UV_FS_STATFS,\n    MKSTEMP = UV_FS_MKSTEMP,\n    LUTIME = UV_FS_LUTIME\n};\n\nenum class uvw_dirent_type_t : std::underlying_type_t<uv_dirent_type_t> {\n    UNKNOWN = UV_DIRENT_UNKNOWN,\n    FILE = UV_DIRENT_FILE,\n    DIR = UV_DIRENT_DIR,\n    LINK = UV_DIRENT_LINK,\n    FIFO = UV_DIRENT_FIFO,\n    SOCKET = UV_DIRENT_SOCKET,\n    CHAR = UV_DIRENT_CHAR,\n    BLOCK = UV_DIRENT_BLOCK\n};\n\nenum class uvw_file_open_flags : int {\n    APPEND = UV_FS_O_APPEND,\n    CREAT = UV_FS_O_CREAT,\n    DIRECT = UV_FS_O_DIRECT,\n    DIRECTORY = UV_FS_O_DIRECTORY,\n    DSYNC = UV_FS_O_DSYNC,\n    EXCL = UV_FS_O_EXCL,\n    EXLOCK = UV_FS_O_EXLOCK,\n    FILEMAP = UV_FS_O_FILEMAP,\n    NOATIME = UV_FS_O_NOATIME,\n    NOCTTY = UV_FS_O_NOCTTY,\n    NOFOLLOW = UV_FS_O_NOFOLLOW,\n    NONBLOCK = UV_FS_O_NONBLOCK,\n    RANDOM = UV_FS_O_RANDOM,\n    RDONLY = UV_FS_O_RDONLY,\n    RDWR = UV_FS_O_RDWR,\n    SEQUENTIAL = UV_FS_O_SEQUENTIAL,\n    SHORT_LIVED = UV_FS_O_SHORT_LIVED,\n    SYMLINK = UV_FS_O_SYMLINK,\n    SYNC = UV_FS_O_SYNC,\n    TEMPORARY = UV_FS_O_TEMPORARY,\n    TRUNC = UV_FS_O_TRUNC,\n    WRONLY = UV_FS_O_WRONLY,\n    _UVW_ENUM = 0\n};\n\nenum class uvw_copy_file_flags : int {\n    EXCL = UV_FS_COPYFILE_EXCL,\n    FICLONE = UV_FS_COPYFILE_FICLONE,\n    FICLONE_FORCE = UV_FS_COPYFILE_FICLONE_FORCE,\n    _UVW_ENUM = 0\n};\n\nenum class uvw_symlink_flags : int {\n    DIR = UV_FS_SYMLINK_DIR,\n    JUNCTION = UV_FS_SYMLINK_JUNCTION,\n    _UVW_ENUM = 0\n};\n\n} // namespace details\n\n/**\n * @brief Common fs event.\n *\n * Available types are:\n *\n * * `fs_request::fs_type::UNKNOWN`\n * * `fs_request::fs_type::CUSTOM`\n * * `fs_request::fs_type::OPEN`\n * * `fs_request::fs_type::CLOSE`\n * * `fs_request::fs_type::READ`\n * * `fs_request::fs_type::WRITE`\n * * `fs_request::fs_type::SENDFILE`\n * * `fs_request::fs_type::STAT`\n * * `fs_request::fs_type::LSTAT`\n * * `fs_request::fs_type::FSTAT`\n * * `fs_request::fs_type::FTRUNCATE`\n * * `fs_request::fs_type::UTIME`\n * * `fs_request::fs_type::FUTIME`\n * * `fs_request::fs_type::ACCESS`\n * * `fs_request::fs_type::CHMOD`\n * * `fs_request::fs_type::FCHMOD`\n * * `fs_request::fs_type::FSYNC`\n * * `fs_request::fs_type::FDATASYNC`\n * * `fs_request::fs_type::UNLINK`\n * * `fs_request::fs_type::RMDIR`\n * * `fs_request::fs_type::MKDIR`\n * * `fs_request::fs_type::MKDTEMP`\n * * `fs_request::fs_type::RENAME`\n * * `fs_request::fs_type::SCANDIR`\n * * `fs_request::fs_type::LINK`\n * * `fs_request::fs_type::SYMLINK`\n * * `fs_request::fs_type::READLINK`\n * * `fs_request::fs_type::CHOWN`\n * * `fs_request::fs_type::FCHOWN`\n * * `fs_request::fs_type::REALPATH`\n * * `fs_request::fs_type::COPYFILE`\n * * `fs_request::fs_type::LCHOWN`\n * * `fs_request::fs_type::OPENDIR`\n * * `fs_request::fs_type::READDIR`\n * * `fs_request::fs_type::CLOSEDIR`\n * * `fs_request::fs_type::STATFS`\n * * `fs_request::fs_type::MKSTEMP`\n * * `fs_request::fs_type::LUTIME`\n *\n * See the official\n * [documentation](http://docs.libuv.org/en/v1.x/fs.html#c.uv_fs_type)\n * for further details.\n */\nstruct fs_event {\n    using fs_type = details::uvw_fs_type;\n    using entry_type = details::uvw_dirent_type_t;\n\n    fs_event(const uv_fs_t &req, std::unique_ptr<const char[]> data)\n        : fs_event{req} {\n        read.data = std::move(data);\n    }\n\n    fs_event(const uv_fs_t &req)\n        : type{req.fs_type},\n          path{req.path},\n          result{static_cast<std::size_t>(req.result)} {\n        switch(type) {\n        case fs_type::STAT:\n        case fs_type::LSTAT:\n        case fs_type::FSTAT:\n            stat = *static_cast<file_info *>(req.ptr);\n            break;\n        case fs_type::READLINK:\n            readlink.data = static_cast<char *>(req.ptr);\n            break;\n        case fs_type::READDIR:\n            dirent.name = static_cast<uv_dir_t *>(req.ptr)->dirents[0].name;\n            dirent.type = static_cast<entry_type>(static_cast<uv_dir_t *>(req.ptr)->dirents[0].type);\n            dirent.eos = !req.result;\n            break;\n        case fs_type::STATFS:\n            statfs = *static_cast<fs_info *>(req.ptr);\n            break;\n        default:\n            // nothing to do here\n            break;\n        }\n    }\n\n    fs_type type;       /*!< Actual event type. */\n    const char *path;   /*!< The path affecting the request. */\n    std::size_t result; /*!< Result value for the specific type. */\n\n    struct {\n        std::unique_ptr<const char[]> data; /*!< A bunch of data read from the given path. */\n    } read;\n\n    struct {\n        const char *data; /*!< The content of a symbolic link. */\n    } readlink;\n\n    file_info stat; /*!< An initialized instance of file_info. */\n    fs_info statfs; /*!< An initialized instance of fs_info. */\n\n    struct {\n        const char *name; /*!< The name of the last entry. */\n        entry_type type;  /*!< The entry type. */\n        bool eos;         /*!< True if there a no more entries to read. */\n    } dirent;\n};\n\n/**\n * @brief Base class for fs/file request.\n *\n * Not directly instantiable, should not be used by the users of the library.\n */\ntemplate<typename T>\nclass fs_request: public request<T, uv_fs_t, fs_event> {\nprotected:\n    static void fs_request_callback(uv_fs_t *req) {\n        if(auto ptr = request<T, uv_fs_t, fs_event>::reserve(req); req->result < 0) {\n            ptr->publish(error_event{req->result});\n        } else {\n            ptr->publish(fs_event{*req});\n        }\n    }\n\npublic:\n    using time = std::chrono::duration<double>;\n    using fs_type = details::uvw_fs_type;\n    using entry_type = details::uvw_dirent_type_t;\n\n    using request<T, uv_fs_t, fs_event>::request;\n};\n\n/**\n * @brief The file request.\n *\n * Cross-platform sync and async filesystem operations.<br/>\n * All file operations are run on the threadpool.\n *\n * To create a `file_req` through a `loop`, no arguments are required.\n *\n * See the official\n * [documentation](http://docs.libuv.org/en/v1.x/fs.html)\n * for further details.\n */\nclass file_req final: public fs_request<file_req> {\n    static constexpr uv_file BAD_FD = -1;\n\n    static void fs_open_callback(uv_fs_t *req);\n    static void fs_close_callback(uv_fs_t *req);\n    static void fs_read_callback(uv_fs_t *req);\n\npublic:\n    using file_open_flags = details::uvw_file_open_flags;\n\n    using fs_request::fs_request;\n\n    ~file_req() noexcept;\n\n    /**\n     * @brief Async [close](http://linux.die.net/man/2/close).\n     *\n     * Emit a `fs_event` event when completed.\n     */\n    void close();\n\n    /**\n     * @brief Sync [close](http://linux.die.net/man/2/close).\n     * @return True in case of success, false otherwise.\n     */\n    bool close_sync();\n\n    /**\n     * @brief Async [open](http://linux.die.net/man/2/open).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * Available flags are:\n     *\n     * * `file_req::file_open_flags::APPEND`\n     * * `file_req::file_open_flags::CREAT`\n     * * `file_req::file_open_flags::DIRECT`\n     * * `file_req::file_open_flags::DIRECTORY`\n     * * `file_req::file_open_flags::DSYNC`\n     * * `file_req::file_open_flags::EXCL`\n     * * `file_req::file_open_flags::EXLOCK`\n     * * `file_req::file_open_flags::FILEMAP`\n     * * `file_req::file_open_flags::NOATIME`\n     * * `file_req::file_open_flags::NOCTTY`\n     * * `file_req::file_open_flags::NOFOLLOW`\n     * * `file_req::file_open_flags::NONBLOCK`\n     * * `file_req::file_open_flags::RANDOM`\n     * * `file_req::file_open_flags::RDONLY`\n     * * `file_req::file_open_flags::RDWR`\n     * * `file_req::file_open_flags::SEQUENTIAL`\n     * * `file_req::file_open_flags::SHORT_LIVED`\n     * * `file_req::file_open_flags::SYMLINK`\n     * * `file_req::file_open_flags::SYNC`\n     * * `file_req::file_open_flags::TEMPORARY`\n     * * `file_req::file_open_flags::TRUNC`\n     * * `file_req::file_open_flags::WRONLY`\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/fs.html#file-open-constants)\n     * for further details.\n     *\n     * @param path A valid path name for a file.\n     * @param flags Flags made out of underlying constants.\n     * @param mode Mode, as described in the official documentation.\n     */\n    void open(const std::string &path, file_open_flags flags, int mode);\n\n    /**\n     * @brief Sync [open](http://linux.die.net/man/2/open).\n     *\n     * Available flags are:\n     *\n     * * `file_req::file_open_flags::APPEND`\n     * * `file_req::file_open_flags::CREAT`\n     * * `file_req::file_open_flags::DIRECT`\n     * * `file_req::file_open_flags::DIRECTORY`\n     * * `file_req::file_open_flags::DSYNC`\n     * * `file_req::file_open_flags::EXCL`\n     * * `file_req::file_open_flags::EXLOCK`\n     * * `file_req::file_open_flags::FILEMAP`\n     * * `file_req::file_open_flags::NOATIME`\n     * * `file_req::file_open_flags::NOCTTY`\n     * * `file_req::file_open_flags::NOFOLLOW`\n     * * `file_req::file_open_flags::NONBLOCK`\n     * * `file_req::file_open_flags::RANDOM`\n     * * `file_req::file_open_flags::RDONLY`\n     * * `file_req::file_open_flags::RDWR`\n     * * `file_req::file_open_flags::SEQUENTIAL`\n     * * `file_req::file_open_flags::SHORT_LIVED`\n     * * `file_req::file_open_flags::SYMLINK`\n     * * `file_req::file_open_flags::SYNC`\n     * * `file_req::file_open_flags::TEMPORARY`\n     * * `file_req::file_open_flags::TRUNC`\n     * * `file_req::file_open_flags::WRONLY`\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/fs.html#file-open-constants)\n     * for further details.\n     *\n     * @param path A valid path name for a file.\n     * @param flags Flags made out of underlying constants.\n     * @param mode Mode, as described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool open_sync(const std::string &path, file_open_flags flags, int mode);\n\n    /**\n     * @brief Async [read](http://linux.die.net/man/2/preadv).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param offset Offset, as described in the official documentation.\n     * @param len Length, as described in the official documentation.\n     */\n    void read(int64_t offset, unsigned int len);\n\n    /**\n     * @brief Sync [read](http://linux.die.net/man/2/preadv).\n     *\n     * @param offset Offset, as described in the official documentation.\n     * @param len Length, as described in the official documentation.\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * A `std::pair` composed as it follows:\n     *   * A bunch of data read from the given path.\n     *   * The amount of data read from the given path.\n     */\n    std::pair<bool, std::pair<std::unique_ptr<const char[]>, std::size_t>> read_sync(int64_t offset, unsigned int len);\n\n    /**\n     * @brief Async [write](http://linux.die.net/man/2/pwritev).\n     *\n     * The request takes the ownership of the data and it is in charge of delete\n     * them.\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param buf The data to be written.\n     * @param len The lenght of the submitted data.\n     * @param offset Offset, as described in the official documentation.\n     */\n    void write(std::unique_ptr<char[]> buf, unsigned int len, int64_t offset);\n\n    /**\n     * @brief Async [write](http://linux.die.net/man/2/pwritev).\n     *\n     * The request doesn't take the ownership of the data. Be sure that their\n     * lifetime overcome the one of the request.\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param buf The data to be written.\n     * @param len The lenght of the submitted data.\n     * @param offset Offset, as described in the official documentation.\n     */\n    void write(char *buf, unsigned int len, int64_t offset);\n\n    /**\n     * @brief Sync [write](http://linux.die.net/man/2/pwritev).\n     *\n     * @param buf The data to be written.\n     * @param len The lenght of the submitted data.\n     * @param offset Offset, as described in the official documentation.\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * The amount of data written to the given path.\n     */\n    std::pair<bool, std::size_t> write_sync(std::unique_ptr<char[]> buf, unsigned int len, int64_t offset);\n\n    /**\n     * @brief Async [fstat](http://linux.die.net/man/2/fstat).\n     *\n     * Emit a `fs_event` event when completed.\n     */\n    void stat();\n\n    /**\n     * @brief Sync [fstat](http://linux.die.net/man/2/fstat).\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * An initialized instance of file_info.\n     */\n    std::pair<bool, file_info> stat_sync();\n\n    /**\n     * @brief Async [fsync](http://linux.die.net/man/2/fsync).\n     *\n     * Emit a `fs_event` event when completed.\n     */\n    void sync();\n\n    /**\n     * @brief Sync [fsync](http://linux.die.net/man/2/fsync).\n     * @return True in case of success, false otherwise.\n     */\n    bool sync_sync();\n\n    /**\n     * @brief Async [fdatasync](http://linux.die.net/man/2/fdatasync).\n     *\n     * Emit a `fs_event` event when completed.\n     */\n    void datasync();\n\n    /**\n     * @brief Sync [fdatasync](http://linux.die.net/man/2/fdatasync).\n     * @return True in case of success, false otherwise.\n     */\n    bool datasync_sync();\n\n    /**\n     * @brief Async [ftruncate](http://linux.die.net/man/2/ftruncate).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param offset Offset, as described in the official documentation.\n     */\n    void truncate(int64_t offset);\n\n    /**\n     * @brief Sync [ftruncate](http://linux.die.net/man/2/ftruncate).\n     * @param offset Offset, as described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool truncate_sync(int64_t offset);\n\n    /**\n     * @brief Async [sendfile](http://linux.die.net/man/2/sendfile).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param out A valid instance of file_handle.\n     * @param offset Offset, as described in the official documentation.\n     * @param length Length, as described in the official documentation.\n     */\n    void sendfile(file_handle out, int64_t offset, std::size_t length);\n\n    /**\n     * @brief Sync [sendfile](http://linux.die.net/man/2/sendfile).\n     *\n     * @param out A valid instance of file_handle.\n     * @param offset Offset, as described in the official documentation.\n     * @param length Length, as described in the official documentation.\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * The amount of data transferred.\n     */\n    std::pair<bool, std::size_t> sendfile_sync(file_handle out, int64_t offset, std::size_t length);\n\n    /**\n     * @brief Async [fchmod](http://linux.die.net/man/2/fchmod).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param mode Mode, as described in the official documentation.\n     */\n    void chmod(int mode);\n\n    /**\n     * @brief Sync [fchmod](http://linux.die.net/man/2/fchmod).\n     * @param mode Mode, as described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool chmod_sync(int mode);\n\n    /**\n     * @brief Async [futime](http://linux.die.net/man/3/futimes).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param atime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     * @param mtime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     */\n    void futime(time atime, time mtime);\n\n    /**\n     * @brief Sync [futime](http://linux.die.net/man/3/futimes).\n     * @param atime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     * @param mtime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool futime_sync(time atime, time mtime);\n\n    /**\n     * @brief Async [fchown](http://linux.die.net/man/2/fchown).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param uid UID, as described in the official documentation.\n     * @param gid GID, as described in the official documentation.\n     */\n    void chown(uid_type uid, gid_type gid);\n\n    /**\n     * @brief Sync [fchown](http://linux.die.net/man/2/fchown).\n     * @param uid UID, as described in the official documentation.\n     * @param gid GID, as described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool chown_sync(uid_type uid, gid_type gid);\n\n    /**\n     * @brief Cast operator to file_handle.\n     *\n     * Cast operator to an internal representation of the underlying file\n     * handle.\n     *\n     * @return A valid instance of file_handle (the descriptor can be invalid).\n     */\n    operator file_handle() const noexcept;\n\nprivate:\n    std::unique_ptr<char[]> current{nullptr};\n    uv_buf_t buffer{};\n    uv_file file{BAD_FD};\n};\n\n/**\n * @brief The fs request.\n *\n * Cross-platform sync and async filesystem operations.<br/>\n * All file operations are run on the threadpool.\n *\n * To create a `fs_req` through a `loop`, no arguments are required.\n *\n * See the official\n * [documentation](http://docs.libuv.org/en/v1.x/fs.html)\n * for further details.\n */\nclass fs_req final: public fs_request<fs_req> {\npublic:\n    using copy_file_flags = details::uvw_copy_file_flags;\n    using symlink_flags = details::uvw_symlink_flags;\n\n    using fs_request::fs_request;\n\n    ~fs_req() noexcept;\n\n    /**\n     * @brief Async [unlink](http://linux.die.net/man/2/unlink).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param path Path, as described in the official documentation.\n     */\n    void unlink(const std::string &path);\n\n    /**\n     * @brief Sync [unlink](http://linux.die.net/man/2/unlink).\n     * @param path Path, as described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool unlink_sync(const std::string &path);\n\n    /**\n     * @brief Async [mkdir](http://linux.die.net/man/2/mkdir).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param path Path, as described in the official documentation.\n     * @param mode Mode, as described in the official documentation.\n     */\n    void mkdir(const std::string &path, int mode);\n\n    /**\n     * @brief Sync [mkdir](http://linux.die.net/man/2/mkdir).\n     * @param path Path, as described in the official documentation.\n     * @param mode Mode, as described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool mkdir_sync(const std::string &path, int mode);\n\n    /**\n     * @brief Async [mktemp](http://linux.die.net/man/3/mkdtemp).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param tpl Template, as described in the official documentation.\n     */\n    void mkdtemp(const std::string &tpl);\n\n    /**\n     * @brief Sync [mktemp](http://linux.die.net/man/3/mkdtemp).\n     *\n     * @param tpl Template, as described in the official documentation.\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * The actual path of the newly created directory.\n     */\n    std::pair<bool, const char *> mkdtemp_sync(const std::string &tpl);\n\n    /**\n     * @brief Async [mkstemp](https://linux.die.net/man/3/mkstemp).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param tpl Template, as described in the official documentation.\n     */\n    void mkstemp(const std::string &tpl);\n\n    /**\n     * @brief Sync [mkstemp](https://linux.die.net/man/3/mkstemp).\n     *\n     * Returns a composed value where:\n     *\n     * * The first parameter indicates the created file path.\n     * * The second parameter is the file descriptor as an integer.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/fs.html#c.uv_fs_mkstemp)\n     * for further details.\n     *\n     * @param tpl Template, as described in the official documentation.\n     *\n     * @return A pair where:\n\n     * * The first parameter is a boolean value that is true in case of success,\n     * false otherwise.\n     * * The second parameter is a composed value (see above).\n     */\n    std::pair<bool, std::pair<std::string, std::size_t>> mkstemp_sync(const std::string &tpl);\n\n    /**\n     * @brief Async [lutime](http://linux.die.net/man/3/lutimes).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param path Path, as described in the official documentation.\n     * @param atime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     * @param mtime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     */\n    void lutime(const std::string &path, time atime, time mtime);\n\n    /**\n     * @brief Sync [lutime](http://linux.die.net/man/3/lutimes).\n     * @param path Path, as described in the official documentation.\n     * @param atime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     * @param mtime `std::chrono::duration<double>`, having the same meaning as\n     * described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool lutime_sync(const std::string &path, time atime, time mtime);\n\n    /**\n     * @brief Async [rmdir](http://linux.die.net/man/2/rmdir).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param path Path, as described in the official documentation.\n     */\n    void rmdir(const std::string &path);\n\n    /**\n     * @brief Sync [rmdir](http://linux.die.net/man/2/rmdir).\n     * @param path Path, as described in the official documentation.\n     * @return True in case of success, false otherwise.\n     */\n    bool rmdir_sync(const std::string &path);\n\n    /**\n     * @brief Async [scandir](http://linux.die.net/man/3/scandir).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param path Path, as described in the official documentation.\n     * @param flags Flags, as described in the official documentation.\n     */\n    void scandir(const std::string &path, int flags);\n\n    /**\n     * @brief Sync [scandir](http://linux.die.net/man/3/scandir).\n     *\n     * @param path Path, as described in the official documentation.\n     * @param flags Flags, as described in the official documentation.\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * The number of directory entries selected.\n     */\n    std::pair<bool, std::size_t> scandir_sync(const std::string &path, int flags);\n\n    /**\n     * @brief Gets entries populated with the next directory entry data.\n     *\n     * Returns a composed value where:\n     *\n     * * The first parameter indicates the entry type (see below).\n     * * The second parameter is a string that contains the actual value.\n     *\n     * Available entry types are:\n     *\n     * * `fs_req::entry_type::UNKNOWN`\n     * * `fs_req::entry_type::FILE`\n     * * `fs_req::entry_type::DIR`\n     * * `fs_req::entry_type::LINK`\n     * * `fs_req::entry_type::FIFO`\n     * * `fs_req::entry_type::SOCKET`\n     * * `fs_req::entry_type::CHAR`\n     * * `fs_req::entry_type::BLOCK`\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/fs.html#c.uv_dirent_t)\n     * for further details.\n     *\n     * @return A pair where:\n     *\n     * * The first parameter is a boolean value that indicates if the current\n     * entry is still valid.\n     * * The second parameter is a composed value (see above).\n     */\n    std::pair<bool, std::pair<entry_type, const char *>> scandir_next();\n\n    /**\n     * @brief Async [stat](http://linux.die.net/man/2/stat).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param path Path, as described in the official documentation.\n     */\n    void stat(const std::string &path);\n\n    /**\n     * @brief Sync [stat](http://linux.die.net/man/2/stat).\n     *\n     * @param path Path, as described in the official documentation.\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * An initialized instance of file_info.\n     */\n    std::pair<bool, file_info> stat_sync(const std::string &path);\n\n    /**\n     * @brief Async [lstat](http://linux.die.net/man/2/lstat).\n     *\n     * Emit a `fs_event` event when completed.\n     *\n     * @param path Path, as described in the official documentation.\n     */\n    void lstat(const std::string &path);\n\n    /**\n     * @brief Sync [lstat](http://linux.die.net/man/2/lstat).\n     *\n     * @param path Path, as described in the official documentation.\n     *\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of success, false otherwise.\n     * * An initialized instance of file_info.\n     */\n...\n", "needle_token_start": 2380, "needle_token_end": 2411, "code_context_ntokens": 16040, "output": ["```cpp\ntemplate<typename T>\nvoid set(T *value) noexcept {\n    return uv_key_set(uv_type::raw(), value);\n}\n```"]}
{"repo": "skypjack/uvw", "name": "uv_token", "language": "cpp", "path": "src/uvw/loop.h", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this entity is to act as a control mechanism within a class, ensuring that certain operations are restricted to be performed only through the class itself or its friends, thereby encapsulating specific functionalities.\n2. **Input**: It takes an integer as an input, which is used solely to differentiate its constructor and does not carry semantic meaning beyond controlling access.\n3. **Output**: This entity does not produce an output as it is not a function but a constructor for creating instances with restricted access.\n4. **Procedure**: The procedure involves instantiating this entity by passing an integer, which effectively serves as a key or token, allowing the creation of an instance. This instance is then used internally within the class to manage or guard certain functionalities that are sensitive or should be restricted in scope.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/util.h\n#ifndef UVW_UTIL_INCLUDE_H\n#define UVW_UTIL_INCLUDE_H\n\n#include <array>\n#include <cstddef>\n#include <cstdint>\n#include <memory>\n#include <string>\n#include <string_view>\n#include <type_traits>\n#include <utility>\n#include <vector>\n#include <uv.h>\n#include \"config.h\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_handle_type : std::underlying_type_t<uv_handle_type> {\n    UNKNOWN = UV_UNKNOWN_HANDLE,\n    ASYNC = UV_ASYNC,\n    CHECK = UV_CHECK,\n    FS_EVENT = UV_FS_EVENT,\n    FS_POLL = UV_FS_POLL,\n    HANDLE = UV_HANDLE,\n    IDLE = UV_IDLE,\n    PIPE = UV_NAMED_PIPE,\n    POLL = UV_POLL,\n    PREPARE = UV_PREPARE,\n    PROCESS = UV_PROCESS,\n    STREAM = UV_STREAM,\n    TCP = UV_TCP,\n    TIMER = UV_TIMER,\n    TTY = UV_TTY,\n    UDP = UV_UDP,\n    SIGNAL = UV_SIGNAL,\n    FILE = UV_FILE\n};\n\nenum class uvw_clock_id : std::underlying_type_t<uv_clock_id> {\n    MONOTONIC = UV_CLOCK_MONOTONIC,\n    REALTIME = UV_CLOCK_REALTIME\n};\n\ntemplate<typename T>\nstruct uv_type_wrapper {\n    using Type = T;\n\n    constexpr uv_type_wrapper()\n        : value{} {}\n\n    constexpr uv_type_wrapper(Type val)\n        : value{val} {}\n\n    constexpr operator Type() const noexcept {\n        return value;\n    }\n\n    bool operator==(uv_type_wrapper other) const noexcept {\n        return value == other.value;\n    }\n\nprivate:\n    const Type value;\n};\n\ntemplate<typename T>\nbool operator==(uv_type_wrapper<T> lhs, uv_type_wrapper<T> rhs) {\n    return !(lhs == rhs);\n}\n\n} // namespace details\n\n/**\n * @brief Windows size representation.\n */\nstruct win_size {\n    int width;  /*!< The _width_ of the given window. */\n    int height; /*!< The _height_ of the given window. */\n};\n\nusing handle_type = details::uvw_handle_type;                     /*!< The type of a handle. */\nusing handle_category = details::uv_type_wrapper<uv_handle_type>; /*!< Utility class that wraps an internal handle type. */\nusing file_handle = details::uv_type_wrapper<uv_file>;            /*!< Utility class that wraps an internal file handle. */\nusing os_socket_handle = details::uv_type_wrapper<uv_os_sock_t>;  /*!< Utility class that wraps an os socket handle. */\nusing os_file_descriptor = details::uv_type_wrapper<uv_os_fd_t>;  /*!< Utility class that wraps an os file descriptor. */\nusing pid_type = details::uv_type_wrapper<uv_pid_t>;              /*!< Utility class that wraps a cross platform representation of a pid. */\nusing clock_id = details::uvw_clock_id;                           /*!< Utility class that wraps a clock source. */\n\nconstexpr file_handle std_in{0};  /*!< Placeholder for stdin descriptor. */\nconstexpr file_handle std_out{1}; /*!< Placeholder for stdout descriptor. */\nconstexpr file_handle std_err{2}; /*!< Placeholder for stderr descriptor. */\n\nusing time_spec = uv_timespec_t; /*!< Library equivalent for uv_timespec_t. */\nusing file_info = uv_stat_t;     /*!< Library equivalent for uv_stat_t. */\nusing fs_info = uv_statfs_t;     /*!< Library equivalent for uv_statfs_t. */\nusing uid_type = uv_uid_t;       /*!< Library equivalent for uv_uid_t. */\nusing gid_type = uv_gid_t;       /*!< Library equivalent for uv_gid_t. */\n\nusing timeval = uv_timeval_t;       /*!< Library equivalent for uv_timeval_t. */\nusing timeval64 = uv_timeval64_t;   /*!< Library equivalent for uv_timeval64_t. */\nusing timespec64 = uv_timespec64_t; /*!< Library equivalent for uv_timespec64_t. */\nusing resource_usage = uv_rusage_t; /*!< Library equivalent for uv_rusage_t. */\n\n/**\n * @brief Utility class.\n *\n * This class can be used to query the subset of the password file entry for the\n * current effective uid (not the real uid).\n *\n * \\sa utilities::passwd\n */\nstruct passwd_info {\n    passwd_info(std::shared_ptr<uv_passwd_t> pwd);\n\n    /**\n     * @brief Gets the username.\n     * @return The username of the current effective uid (not the real uid).\n     */\n    std::string username() const noexcept;\n\n    /**\n     * @brief Gets the uid.\n     * @return The current effective uid (not the real uid).\n     */\n    decltype(uv_passwd_t::uid) uid() const noexcept;\n\n    /**\n     * @brief Gets the gid.\n     * @return The gid of the current effective uid (not the real uid).\n     */\n    decltype(uv_passwd_t::gid) gid() const noexcept;\n\n    /**\n     * @brief Gets the shell.\n     * @return The shell of the current effective uid (not the real uid).\n     */\n    std::string shell() const noexcept;\n\n    /**\n     * @brief Gets the homedir.\n     * @return The homedir of the current effective uid (not the real uid).\n     */\n    std::string homedir() const noexcept;\n\n    /**\n     * @brief Checks if the instance contains valid data.\n     * @return True if data are all valid, false otherwise.\n     */\n    operator bool() const noexcept;\n\nprivate:\n    std::shared_ptr<uv_passwd_t> value;\n};\n\n/**\n * @brief Utility class.\n *\n * This class can be used to get name and information about the current kernel.\n * The populated data includes the operating system name, release, version, and\n * machine.\n *\n * \\sa utilities::uname\n */\nstruct uts_name {\n    uts_name(std::shared_ptr<uv_utsname_t> init);\n\n    /**\n     * @brief Gets the operating system name (like \"Linux\").\n     * @return The operating system name.\n     */\n    std::string sysname() const noexcept;\n\n    /**\n     * @brief Gets the operating system release (like \"2.6.28\").\n     * @return The operating system release.\n     */\n    std::string release() const noexcept;\n\n    /**\n     * @brief Gets the operating system version.\n     * @return The operating system version\n     */\n    std::string version() const noexcept;\n\n    /**\n     * @brief Gets the hardware identifier.\n     * @return The hardware identifier.\n     */\n    std::string machine() const noexcept;\n\nprivate:\n    std::shared_ptr<uv_utsname_t> uname;\n};\n\n/**\n * @brief The IPv4 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv4 {};\n\n/**\n * @brief The IPv6 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv6 {};\n\n/**\n * @brief Address representation.\n */\nstruct socket_address {\n    std::string ip;    /*!< Either an IPv4 or an IPv6. */\n    unsigned int port; /*!< A valid service identifier. */\n};\n\n/**\n * \\brief CPU information.\n */\nstruct cpu_info {\n    using cpu_time = decltype(uv_cpu_info_t::cpu_times);\n\n    std::string model; /*!< The model of the CPU. */\n    int speed;         /*!< The frequency of the CPU. */\n\n    /**\n     * @brief CPU times.\n...\n// Path: src/uvw/loop.cpp\n#ifdef UVW_AS_LIB\n#    include \"loop.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE loop::loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept\n    : uv_loop{std::move(ptr)} {}\n\nUVW_INLINE std::shared_ptr<loop> loop::create() {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{new uv_loop_t, [](uv_loop_t *l) { delete l; }};\n    auto curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n\n    if(uv_loop_init(curr->uv_loop.get())) {\n        curr = nullptr;\n    }\n\n    return curr;\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::create(uv_loop_t *res) {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{res, [](uv_loop_t *) {}};\n    return std::shared_ptr<loop>{new loop{std::move(ptr)}};\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::get_default() {\n    static std::weak_ptr<loop> ref;\n    std::shared_ptr<loop> curr;\n\n    if(ref.expired()) {\n        auto def = uv_default_loop();\n\n        if(def) {\n            auto ptr = std::unique_ptr<uv_loop_t, deleter>(def, [](uv_loop_t *) {});\n            curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n        }\n\n        ref = curr;\n    } else {\n        curr = ref.lock();\n    }\n\n    return curr;\n}\n\nUVW_INLINE loop::~loop() noexcept {\n    if(uv_loop) {\n        close();\n    }\n}\n\nUVW_INLINE int loop::close() {\n    int ret = 0;\n\n    if(uv_loop) {\n        ret = uv_loop_close(uv_loop.get());\n        uv_loop.reset();\n    }\n\n    return ret;\n}\n\nUVW_INLINE int loop::run(run_mode mode) noexcept {\n    return uv_run(uv_loop.get(), static_cast<uv_run_mode>(mode));\n}\n\nUVW_INLINE bool loop::alive() const noexcept {\n    return !!uv_loop_alive(uv_loop.get());\n}\n\nUVW_INLINE void loop::stop() noexcept {\n    uv_stop(uv_loop.get());\n}\n\nUVW_INLINE int loop::descriptor() const noexcept {\n    return uv_backend_fd(uv_loop.get());\n}\n\nUVW_INLINE std::pair<bool, loop::time> loop::timeout() const noexcept {\n    auto to = uv_backend_timeout(uv_loop.get());\n    return std::make_pair(to == -1, time{to});\n}\n\nUVW_INLINE loop::time loop::idle_time() const noexcept {\n    return time{uv_metrics_idle_time(uv_loop.get())};\n}\n\nUVW_INLINE metrics_type loop::metrics() const noexcept {\n    metrics_type res{};\n    uv_metrics_info(uv_loop.get(), &res);\n    return res;\n}\n\nUVW_INLINE loop::time loop::now() const noexcept {\n    return time{uv_now(uv_loop.get())};\n}\n\nUVW_INLINE void loop::update() const noexcept {\n    return uv_update_time(uv_loop.get());\n}\n\nUVW_INLINE int loop::fork() noexcept {\n    return uv_loop_fork(uv_loop.get());\n}\n\nUVW_INLINE void loop::data(std::shared_ptr<void> ud) {\n    user_data = std::move(ud);\n}\n\nUVW_INLINE const uv_loop_t *loop::raw() const noexcept {\n    return uv_loop.get();\n}\n\nUVW_INLINE uv_loop_t *loop::raw() noexcept {\n    return const_cast<uv_loop_t *>(const_cast<const loop *>(this)->raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/loop.h\n#ifndef UVW_LOOP_INCLUDE_H\n#define UVW_LOOP_INCLUDE_H\n\n#ifdef _WIN32\n#    include <ciso646>\n#endif\n\n#include <chrono>\n#include <functional>\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"util.h\"\n\nnamespace uvw {\n\nclass async_handle;\nclass check_handle;\nclass fs_event_handle;\nclass fs_poll_handle;\nclass idle_handle;\nclass pipe_handle;\nclass poll_handle;\nclass prepare_handle;\nclass process_handle;\nclass signal_handle;\nclass tcp_handle;\nclass timer_handle;\nclass tty_handle;\nclass udp_handle;\n\nnamespace details {\n\nenum class uvw_loop_option : std::underlying_type_t<uv_loop_option> {\n    BLOCK_SIGNAL = UV_LOOP_BLOCK_SIGNAL,\n    IDLE_TIME = UV_METRICS_IDLE_TIME\n};\n\nenum class uvw_run_mode : std::underlying_type_t<uv_run_mode> {\n    DEFAULT = UV_RUN_DEFAULT,\n    ONCE = UV_RUN_ONCE,\n    NOWAIT = UV_RUN_NOWAIT\n};\n\n} // namespace details\n\nusing metrics_type = uv_metrics_t; /*!< Library equivalent for uv_metrics_t. */\n\n/**\n * @brief The loop class.\n *\n * The event loop is the central part of `uvw`'s functionalities, as well as\n * `libuv`'s ones.<br/>\n * It takes care of polling for I/O and scheduling callbacks to be run based on\n * different sources of events.\n */\nclass loop final: public emitter<loop>, public std::enable_shared_from_this<loop> {\n    using deleter = void (*)(uv_loop_t *);\n\n    template<typename, typename, typename...>\n    friend class resource;\n\n    class uv_token {\n        friend class loop;\n        \nexplicit uv_token(int) {}\n    };\n\n    template<typename Type>\n    auto init(int, Type &value) -> decltype(value.init()) {\n        return value.init();\n    }\n\n    template<typename Type>\n    int init(char, Type &) {\n        return 0;\n    }\n\n    loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept;\n\npublic:\n    using token = uv_token;\n    using time = std::chrono::duration<uint64_t, std::milli>;\n    using option = details::uvw_loop_option;\n    using run_mode = details::uvw_run_mode;\n\n    /**\n     * @brief Initializes a new loop instance.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create();\n\n    /**\n     * @brief Initializes a new loop instance from an existing resource.\n     *\n     * The lifetime of the resource must exceed that of the instance to which\n     * it's associated. Management of the memory associated with the resource is\n     * in charge of the user.\n     *\n     * @param res A valid pointer to a correctly initialized resource.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create(uv_loop_t *res);\n\n    /**\n     * @brief Gets the initialized default loop.\n     *\n     * It may return an empty pointer in case of failure.<br>\n     * This function is just a convenient way for having a global loop\n     * throughout an application, the default loop is in no way different than\n     * the ones initialized with `create()`.<br>\n     * As such, the default loop can be closed with `close()` so the resources\n     * associated with it are freed (even if it is not strictly necessary).\n     *\n     * @return The initialized default loop.\n     */\n    static std::shared_ptr<loop> get_default();\n\n    loop(const loop &) = delete;\n    loop(loop &&other) = delete;\n\n    loop &operator=(const loop &) = delete;\n    loop &operator=(loop &&other) = delete;\n\n    ~loop() noexcept;\n\n    /**\n     * @brief Sets additional loop options.\n     *\n     * You should normally call this before the first call to uv_run() unless\n     * mentioned otherwise.<br/>\n     * Supported options:\n     *\n     * * `loop::option::BLOCK_SIGNAL`: Block a signal when polling for new\n     * events. A second argument is required and it is the signal number.\n     * * `loop::option::IDLE_TIME`: Accumulate the amount of idle time the event\n     * loop spends in the event provider. This option is necessary to use\n     * `idle_time()`.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_configure)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    template<typename... Args>\n    int configure(option flag, Args &&...args) {\n        return uv_loop_configure(uv_loop.get(), static_cast<uv_loop_option>(flag), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Creates resources of any type.\n     *\n     * This should be used as a default method to create resources.<br/>\n     * The arguments are the ones required for the specific resource.\n     *\n     * Use it as `loop->resource<uvw::timer_handle>()`.\n     *\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> resource(Args &&...args) {\n        auto ptr = uninitialized_resource<R>(std::forward<Args>(args)...);\n        return (init(0, *ptr) == 0) ? ptr : nullptr;\n    }\n\n    /**\n     * @brief Creates uninitialized resources of any type.\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> uninitialized_resource(Args &&...args) {\n        return std::make_shared<R>(token{0}, shared_from_this(), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Releases all internal loop resources.\n     *\n     * Call this function only when the loop has finished executing and all open\n     * handles and requests have been closed, or the loop will error.\n     *\n     * @return Underlying return value.\n     */\n    int close();\n\n    /**\n     * @brief Runs the event loop.\n     *\n     * Available modes are:\n     *\n     * * `loop::run_mode::DEFAULT`: Runs the event loop until there are no more\n     * active and referenced handles or requests.\n     * * `loop::run_mode::ONCE`: Poll for i/o once. Note that this function\n     * blocks if there are no pending callbacks.\n     * * `loop::run_mode::NOWAIT`: Poll for i/o once but don\u2019t block if there\n     * are no pending callbacks.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_run)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int run(run_mode mode = run_mode::DEFAULT) noexcept;\n\n    /**\n     * @brief Checks if there are active resources.\n     * @return True if there are active resources in the loop.\n     */\n    bool alive() const noexcept;\n\n    /**\n     * @brief Stops the event loop.\n     *\n     * It causes `run()` to end as soon as possible.<br/>\n     * This will happen not sooner than the next loop iteration.<br/>\n     * If this function was called before blocking for I/O, the loop won\u2019t block\n     * for I/O on this iteration.\n     */\n    void stop() noexcept;\n\n    /**\n     * @brief Get backend file descriptor.\n     *\n     * Only kqueue, epoll and event ports are supported.<br/>\n     * This can be used in conjunction with `run(loop::run_mode::NOWAIT)` to\n     * poll in one thread and run the event loop\u2019s callbacks in another.\n     *\n     * @return The backend file descriptor.\n     */\n    int descriptor() const noexcept;\n\n    /**\n     * @brief Gets the poll timeout.\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of valid timeout, false otherwise.\n     * * Milliseconds (`std::chrono::duration<uint64_t, std::milli>`).\n     */\n    std::pair<bool, time> timeout() const noexcept;\n\n    /**\n     * @brief Returns the amount of time the event loop has been idle. The call\n     * is thread safe.\n     * @return The accumulated time spent idle.\n     */\n    time idle_time() const noexcept;\n\n    /**\n     * @brief Tracks various internal operations of the event loop.\n     * @return Event loop metrics.\n     */\n    metrics_type metrics() const noexcept;\n\n    /**\n     * @brief Returns the current timestamp in milliseconds.\n     *\n     * The timestamp is cached at the start of the event loop tick.<br/>\n     * The timestamp increases monotonically from some arbitrary point in\n     * time.<br/>\n     * Don\u2019t make assumptions about the starting point, you will only get\n     * disappointed.\n     *\n     * @return The current timestamp in milliseconds (actual type is\n     * `std::chrono::duration<uint64_t, std::milli>`).\n     */\n    time now() const noexcept;\n\n    /**\n     * @brief Updates the event loop\u2019s concept of _now_.\n     *\n     * The current time is cached at the start of the event loop tick in order\n     * to reduce the number of time-related system calls.<br/>\n     * You won\u2019t normally need to call this function unless you have callbacks\n     * that block the event loop for longer periods of time, where _longer_ is\n     * somewhat subjective but probably on the order of a millisecond or more.\n     */\n    void update() const noexcept;\n\n    /**\n     * @brief Walks the list of handles.\n     *\n     * The callback is invoked once for each handle that is still active.\n     *\n     * @param callback A function to invoke once for each active handle.\n     */\n    template<typename Func>\n    void walk(Func callback) {\n        auto func = [](uv_handle_t *hndl, void *callback_func) {\n            if(hndl->data) {\n                auto &cb = *static_cast<Func *>(callback_func);\n\n                switch(utilities::guess_handle(handle_category{hndl->type})) {\n                case handle_type::ASYNC:\n                    cb(*static_cast<async_handle *>(hndl->data));\n                    break;\n                case handle_type::CHECK:\n                    cb(*static_cast<check_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_EVENT:\n                    cb(*static_cast<fs_event_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_POLL:\n                    cb(*static_cast<fs_poll_handle *>(hndl->data));\n                    break;\n                case handle_type::IDLE:\n                    cb(*static_cast<idle_handle *>(hndl->data));\n                    break;\n                case handle_type::PIPE:\n                    cb(*static_cast<pipe_handle *>(hndl->data));\n                    break;\n                case handle_type::POLL:\n                    cb(*static_cast<poll_handle *>(hndl->data));\n                    break;\n                case handle_type::PREPARE:\n                    cb(*static_cast<prepare_handle *>(hndl->data));\n                    break;\n                case handle_type::PROCESS:\n                    cb(*static_cast<process_handle *>(hndl->data));\n                    break;\n                case handle_type::SIGNAL:\n                    cb(*static_cast<signal_handle *>(hndl->data));\n                    break;\n                case handle_type::TCP:\n                    cb(*static_cast<tcp_handle *>(hndl->data));\n                    break;\n                case handle_type::TIMER:\n                    cb(*static_cast<timer_handle *>(hndl->data));\n                    break;\n                case handle_type::TTY:\n                    cb(*static_cast<tty_handle *>(hndl->data));\n                    break;\n                case handle_type::UDP:\n                    cb(*static_cast<udp_handle *>(hndl->data));\n                    break;\n                default:\n                    // this handle isn't managed by uvw, let it be...\n                    break;\n                }\n            }\n        };\n\n        uv_walk(uv_loop.get(), func, &callback);\n    }\n\n    /**\n     * @brief Reinitialize any kernel state necessary in the child process after\n     * a fork(2) system call.\n     *\n     * Previously started watchers will continue to be started in the child\n     * process.\n     *\n     * It is necessary to explicitly call this function on every event loop\n     * created in the parent process that you plan to continue to use in the\n     * child, including the default loop (even if you don\u2019t continue to use it\n     * in the parent). This function must be called before calling any API\n     * function using the loop in the child. Failure to do so will result in\n     * undefined behaviour, possibly including duplicate events delivered to\n     * both parent and child or aborting the child process.\n     *\n     * When possible, it is preferred to create a new loop in the child process\n     * instead of reusing a loop created in the parent. New loops created in the\n     * child process after the fork should not use this function.\n     *\n     * Note that this function is not implemented on Windows.<br/>\n     * Note also that this function is experimental in `libuv`. It may contain\n     * bugs, and is subject to change or removal. API and ABI stability is not\n     * guaranteed.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_fork)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int fork() noexcept;\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param ud User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> ud);\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const uv_loop_t *raw() const noexcept;\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    uv_loop_t *raw() noexcept;\n\nprivate:\n    std::unique_ptr<uv_loop_t, deleter> uv_loop;\n    std::shared_ptr<void> user_data{nullptr};\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"loop.cpp\"\n#endif\n\n#endif // UVW_LOOP_INCLUDE_H\n\n// Path: src/uvw/uv_type.hpp\n#ifndef UVW_UV_TYPE_INCLUDE_H\n#define UVW_UV_TYPE_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include \"config.h\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/**\n * @brief Wrapper class for underlying types.\n *\n * It acts mainly as a wrapper around data structures of the underlying library.\n */\ntemplate<typename U>\nstruct uv_type {\n    explicit uv_type(loop::token, std::shared_ptr<loop> ref) noexcept\n        : owner{std::move(ref)}, resource{} {}\n\n    uv_type(const uv_type &) = delete;\n    uv_type(uv_type &&) = delete;\n\n    uv_type &operator=(const uv_type &) = delete;\n    uv_type &operator=(uv_type &&) = delete;\n\n    /**\n     * @brief Gets the loop from which the resource was originated.\n     * @return A reference to a loop instance.\n     */\n    loop &parent() const noexcept {\n        return *owner;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const U *raw() const noexcept {\n        return &resource;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    U *raw() noexcept {\n        return &resource;\n    }\n\nprotected:\n    ~uv_type() = default;\n\nprivate:\n    std::shared_ptr<loop> owner;\n    U resource;\n};\n\n} // namespace uvw\n\n#endif // UVW_UV_TYPE_INCLUDE_H\n\n// Path: src/uvw/resource.hpp\n#ifndef UVW_RESOURCE_INCLUDE_H\n#define UVW_RESOURCE_INCLUDE_H\n\n#include <memory>\n#include <utility>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Common class for almost all the resources available in `uvw`.\n *\n * This is the base class for handles and requests.\n */\ntemplate<typename T, typename U, typename... E>\nclass resource: public uv_type<U>, public emitter<T, E...>, public std::enable_shared_from_this<T> {\nprotected:\n    int leak_if(int err) noexcept {\n        if(err == 0) {\n            self_ptr = this->shared_from_this();\n        }\n\n        return err;\n    }\n\n    void self_reset() noexcept {\n        self_ptr.reset();\n    }\n\n    bool has_self() const noexcept {\n        return static_cast<bool>(self_ptr);\n    }\n\npublic:\n    explicit resource(loop::token token, std::shared_ptr<loop> ref)\n        : uv_type<U>{token, std::move(ref)} {\n        this->raw()->data = this;\n    }\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param udata User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> udata) {\n        user_data = std::move(udata);\n    }\n\nprivate:\n    std::shared_ptr<void> user_data{nullptr};\n    std::shared_ptr<void> self_ptr{nullptr};\n};\n\n} // namespace uvw\n\n#endif // UVW_RESOURCE_INCLUDE_H\n\n// Path: src/uvw/handle.hpp\n#ifndef UVW_HANDLE_INCLUDE_H\n#define UVW_HANDLE_INCLUDE_H\n\n#include <cstddef>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n#include \"util.h\"\n\nnamespace uvw {\n\n/*! @brief Close event. */\nstruct close_event {};\n\n/**\n * @brief Handle base class.\n *\n * Base type for all `uvw` handle types.\n */\ntemplate<typename T, typename U, typename... E>\nclass handle: public resource<T, U, close_event, E...> {\nprotected:\n    static void close_callback(uv_handle_t *hndl) {\n        handle<T, U, E...> &ref = *(static_cast<T *>(hndl->data));\n        [[maybe_unused]] auto ptr = ref.shared_from_this();\n        ref.self_reset();\n        ref.publish(close_event{});\n    }\n\n    uv_handle_t *as_uv_handle() {\n        return reinterpret_cast<uv_handle_t *>(this->raw());\n    }\n\n    const uv_handle_t *as_uv_handle() const {\n        return reinterpret_cast<const uv_handle_t *>(this->raw());\n    }\n\npublic:\n    using resource<T, U, close_event, E...>::resource;\n\n    /**\n     * @brief Gets the category of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, an opaque value that identifies the\n     * category of the handle is made available to the users.\n     *\n     * @return The actual category of the handle.\n     */\n    handle_category category() const noexcept {\n        return handle_category{as_uv_handle()->type};\n    }\n\n    /**\n     * @brief Gets the type of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, the type of the underlying handle as\n     * specified by handle_type is made available to the users.\n     *\n     * @return The actual type of the handle.\n     */\n    handle_type type() const noexcept {\n        return utilities::guess_handle(category());\n    }\n\n    /**\n     * @brief Checks if the handle is active.\n     *\n     * What _active_ means depends on the type of handle:\n     *\n     * * An async_handle handle is always active and cannot be deactivated,\n     * except by closing it with uv_close().\n     * * A pipe, tcp, udp, etc. handle - basically any handle that deals with\n     * I/O - is active when it is doing something that involves I/O, like\n     * reading, writing, connecting, accepting new connections, etc.\n     * * A check, idle, timer, etc. handle is active when it has been started\n     * with a call to `start()`.\n     *\n     * Rule of thumb: if a handle of type `foo_handle` has a `start()` member\n     * method, then it\u2019s active from the moment that method is called. Likewise,\n     * `stop()` deactivates the handle again.\n     *\n     * @return True if the handle is active, false otherwise.\n     */\n    bool active() const noexcept {\n        return !!uv_is_active(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if a handle is closing or closed.\n     *\n     * This function should only be used between the initialization of the\n     * handle and the arrival of the close callback.\n     *\n     * @return True if the handle is closing or closed, false otherwise.\n     */\n    bool closing() const noexcept {\n        return !!uv_is_closing(as_uv_handle());\n    }\n\n    /**\n     * @brief Request handle to be closed.\n     *\n     * This **must** be called on each handle before memory is released.<br/>\n     * In-progress requests are cancelled and this can result in errors.\n     *\n     * The handle will emit a close event when finished.\n     */\n    void close() noexcept {\n        if(!closing()) {\n            uv_close(as_uv_handle(), &handle<T, U, E...>::close_callback);\n        }\n    }\n\n    /**\n     * @brief Reference the given handle.\n     *\n     * References are idempotent, that is, if a handle is already referenced\n     * calling this function again will have no effect.\n     */\n    void reference() noexcept {\n        uv_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Unreference the given handle.\n     *\n     * References are idempotent, that is, if a handle is not referenced calling\n     * this function again will have no effect.\n     */\n    void unreference() noexcept {\n        uv_unref(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if the given handle referenced.\n     * @return True if the handle referenced, false otherwise.\n     */\n    bool referenced() const noexcept {\n        return !!uv_has_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Returns the size of the underlying handle type.\n     * @return The size of the underlying handle type.\n     */\n    std::size_t size() const noexcept {\n        return uv_handle_size(as_uv_handle()->type);\n    }\n\n    /**\n     * @brief Gets the size of the send buffer used for the socket.\n     *\n     * Gets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipeand udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the send buffer, the underlying return value in case\n     * of errors.\n     */\n    int send_buffer_size() {\n        int value = 0;\n        auto err = uv_send_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the send buffer used for the socket.\n     *\n     * Sets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int send_buffer_size(int value) {\n        return uv_send_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the size of the receive buffer used for the socket.\n     *\n     * Gets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the receive buffer, the underlying return value in\n     * case of errors.\n     */\n    int recv_buffer_size() {\n        int value = 0;\n        auto err = uv_recv_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the receive buffer used for the socket.\n     *\n     * Sets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int recv_buffer_size(int value) {\n        return uv_recv_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the platform dependent file descriptor equivalent.\n     *\n     * Supported handles:\n     *\n     * * tcp_handle\n     * * pipe_handle\n     * * tty_handle\n     * * udp_handle\n     * * poll_handle\n     *\n     * If invoked on a different handle, one that doesn\u2019t have an attached file\n     * descriptor yet or one which was closed, an invalid value is returned.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/handle.html#c.uv_fileno)\n     * for further details.\n     *\n     * @return The file descriptor attached to the hande or a negative value in\n     * case of errors.\n     */\n    os_file_descriptor fd() const {\n        uv_os_fd_t fd;\n        uv_fileno(as_uv_handle(), &fd);\n        return fd;\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_HANDLE_INCLUDE_H\n\n// Path: src/uvw/prepare.h\n#ifndef UVW_PREPARE_INCLUDE_H\n#define UVW_PREPARE_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Prepare event. */\nstruct prepare_event {};\n\n/**\n * @brief The prepare handle.\n *\n * Prepare handles will emit a prepare event once per loop iteration, right\n * before polling for I/O.\n *\n * To create a `prepare_handle` through a `loop`, no arguments are required.\n */\nclass prepare_handle final: public handle<prepare_handle, uv_prepare_t, prepare_event> {\n    static void start_callback(uv_prepare_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * A prepare event will be emitted once per loop iteration, right before\n     * polling for I/O.\n     *\n     * The handle will start emitting prepare events when needed.\n     *\n     * @return Underlying return value.\n     */\n    int start();\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"prepare.cpp\"\n#endif\n\n#endif // UVW_PREPARE_INCLUDE_H\n\n// Path: src/uvw/prepare.cpp\n#ifdef UVW_AS_LIB\n#    include \"prepare.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void prepare_handle::start_callback(uv_prepare_t *hndl) {\n    prepare_handle &prepare = *(static_cast<prepare_handle *>(hndl->data));\n    prepare.publish(prepare_event{});\n}\n\nUVW_INLINE int prepare_handle::init() {\n    return leak_if(uv_prepare_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int prepare_handle::start() {\n    return uv_prepare_start(raw(), &start_callback);\n}\n\nUVW_INLINE int prepare_handle::stop() {\n    return uv_prepare_stop(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/request.hpp\n#ifndef UVW_REQUEST_INCLUDE_H\n#define UVW_REQUEST_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Request base class.\n *\n * Base type for all `uvw` request types.\n */\ntemplate<typename T, typename U, typename... E>\nclass request: public resource<T, U, E...> {\nprotected:\n    static auto reserve(U *req) {\n        auto ptr = static_cast<T *>(req->data)->shared_from_this();\n        ptr->self_reset();\n        return ptr;\n    }\n\npublic:\n    using resource<T, U, E...>::resource;\n\n    /**\n     * @brief Cancels a pending request.\n     *\n     * This method fails if the request is executing or has finished\n     * executing.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/request.html#c.uv_cancel)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int cancel() {\n        return uv_cancel(reinterpret_cast<uv_req_t *>(this->raw()));\n    }\n\n    /**\n     * @brief Returns the size of the underlying request type.\n     * @return The size of the underlying request type.\n     */\n    std::size_t size() const noexcept {\n        return uv_req_size(reinterpret_cast<const uv_req_t *>(this->raw())->type);\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_REQUEST_INCLUDE_H\n\n// Path: src/uvw/stream.h\n#ifndef UVW_STREAM_INCLUDE_H\n#define UVW_STREAM_INCLUDE_H\n\n#include <algorithm>\n#include <cstddef>\n#include <iterator>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"handle.hpp\"\n#include \"loop.h\"\n#include \"request.hpp\"\n\nnamespace uvw {\n\n/*! @brief Connect event. */\nstruct connect_event {};\n\n/*! @brief End event. */\nstruct end_event {};\n\n/*! @brief Listen event. */\nstruct listen_event {};\n\n/*! @brief Shutdown event. */\nstruct shutdown_event {};\n\n/*! @brief Write event. */\nstruct write_event {};\n\n/*! @brief Data event. */\nstruct data_event {\n    explicit data_event(std::unique_ptr<char[]> buf, std::size_t len) noexcept;\n\n    std::unique_ptr<char[]> data; /*!< A bunch of data read on the stream. */\n    std::size_t length;           /*!< The amount of data read on the stream. */\n};\n\nnamespace details {\n\nclass connect_req final: public request<connect_req, uv_connect_t, connect_event> {\n    static void connect_callback(uv_connect_t *req, int status);\n\npublic:\n    using request::request;\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback);\n        return this->leak_if(0);\n    }\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<!std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        return this->leak_if(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback));\n    }\n};\n\nclass shutdown_req final: public request<shutdown_req, uv_shutdown_t, shutdown_event> {\n    static void shoutdown_callback(uv_shutdown_t *req, int status);\n\npublic:\n    using request::request;\n\n    int shutdown(uv_stream_t *hndl);\n};\n\ntemplate<typename Deleter>\nclass write_req final: public request<write_req<Deleter>, uv_write_t, write_event> {\n    static void write_callback(uv_write_t *req, int status) {\n        if(auto ptr = request<write_req<Deleter>, uv_write_t, write_event>::reserve(req); status) {\n            ptr->publish(error_event{status});\n        } else {\n            ptr->publish(write_event{});\n        }\n    }\n\npublic:\n    write_req(loop::token token, std::shared_ptr<loop> parent, std::unique_ptr<char[], Deleter> dt, unsigned int len)\n        : request<write_req<Deleter>, uv_write_t, write_event>{token, std::move(parent)},\n          data{std::move(dt)},\n          buf{uv_buf_init(data.get(), len)} {}\n\n    int write(uv_stream_t *hndl) {\n        return this->leak_if(uv_write(this->raw(), hndl, &buf, 1, &write_callback));\n    }\n\n    int write(uv_stream_t *hndl, uv_stream_t *send) {\n        return this->leak_if(uv_write2(this->raw(), hndl, &buf, 1, send, &write_callback));\n    }\n\nprivate:\n    std::unique_ptr<char[], Deleter> data;\n    uv_buf_t buf;\n};\n\n} // namespace details\n\n/**\n * @brief The stream handle.\n *\n * Stream handles provide an abstraction of a duplex communication channel.\n * The stream handle is an intermediate type, `uvw` provides three stream\n * implementations: tcp, pipe and tty handles.\n */\ntemplate<typename T, typename U, typename... E>\nclass stream_handle: public handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...> {\n    using base = handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...>;\n\n    template<typename, typename, typename...>\n    friend class stream_handle;\n\n    static constexpr unsigned int DEFAULT_BACKLOG = 128;\n\n    static void read_callback(uv_stream_t *hndl, ssize_t nread, const uv_buf_t *buf) {\n        T &ref = *(static_cast<T *>(hndl->data));\n        // data will be destroyed no matter of what the value of nread is\n        std::unique_ptr<char[]> data{buf->base};\n\n        // nread == 0 is ignored (see http://docs.libuv.org/en/v1.x/stream.html)\n        // equivalent to EAGAIN/EWOULDBLOCK, it shouldn't be treated as an error\n        // for we don't have data to emit though, it's fine to suppress it\n\n        if(nread == UV_EOF) {\n            // end of stream\n            ref.publish(end_event{});\n        } else if(nread > 0) {\n            // data available\n            ref.publish(data_event{std::move(data), static_cast<std::size_t>(nread)});\n        } else if(nread < 0) {\n            // transmission error\n            ref.publish(error_event(nread));\n        }\n    }\n\n    static void listen_callback(uv_stream_t *hndl, int status) {\n        if(T &ref = *(static_cast<T *>(hndl->data)); status) {\n            ref.publish(error_event{status});\n        } else {\n            ref.publish(listen_event{});\n        }\n    }\n\n    uv_stream_t *as_uv_stream() {\n        return reinterpret_cast<uv_stream_t *>(this->raw());\n    }\n\n    const uv_stream_t *as_uv_stream() const {\n        return reinterpret_cast<const uv_stream_t *>(this->raw());\n    }\n\npublic:\n#ifdef _MSC_VER\n    stream_handle(loop::token token, std::shared_ptr<loop> ref)\n        : base{token, std::move(ref)} {}\n#else\n    using base::base;\n#endif\n\n    /**\n     * @brief Shutdowns the outgoing (write) side of a duplex stream.\n     *\n     * It waits for pending write requests to complete. The handle should refer\n     * to a initialized stream.<br/>\n     * A shutdown event will be emitted after shutdown is complete.\n     *\n     * @return Underlying return value.\n     */\n    int shutdown() {\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        auto shutdown = this->parent().template resource<details::shutdown_req>();\n        shutdown->template on<error_event>(listener);\n        shutdown->template on<shutdown_event>(listener);\n\n        return shutdown->shutdown(as_uv_stream());\n    }\n\n    /**\n     * @brief Starts listening for incoming connections.\n     *\n     * When a new incoming connection is received, a listen event is\n     * emitted.\n     *\n     * @param backlog Indicates the number of connections the kernel might\n     * queue, same as listen(2).\n     *\n     * @return Underlying return value.\n     */\n    int listen(int backlog = DEFAULT_BACKLOG) {\n        return uv_listen(as_uv_stream(), backlog, &listen_callback);\n    }\n\n    /**\n     * @brief Accepts incoming connections.\n     *\n     * This call is used in conjunction with `listen()` to accept incoming\n     * connections. Call this function after receiving a listen event to accept\n     * the connection. Before calling this function, the submitted handle must\n     * be initialized.\n     *\n     * When the listen event is emitted it is guaranteed that this function will\n     * complete successfully the first time. If you attempt to use it more than\n     * once, it may fail.<br/>\n     * It is suggested to only call this function once per listen event.\n     *\n     * @note\n     * Both the handles must be running on the same loop.\n     *\n     * @param ref An initialized handle to be used to accept the connection.\n     * @return Underlying return value.\n     */\n    template<typename S>\n    int accept(S &ref) {\n        return uv_accept(as_uv_stream(), ref.as_uv_stream());\n    }\n\n    /**\n     * @brief Starts reading data from an incoming stream.\n     *\n     * A data event will be emitted several times until there is no more data to\n     * read or `stop()` is called.<br/>\n     * An end event will be emitted when there is no more data to read.\n     *\n     * @return Underlying return value.\n     */\n    int read() {\n        return uv_read_start(as_uv_stream(), &details::common_alloc_callback, &read_callback);\n    }\n\n    /**\n     * @brief Starts reading data from an incoming stream.\n     * @sa read\n     * @tparam Alloc Custom allocation function.\n     * @return Underlying return value.\n     */\n    template<auto Alloc>\n    int read() {\n        return uv_read_start(as_uv_stream(), &details::common_alloc_callback<T, Alloc>, &read_callback);\n    }\n\n    /**\n     * @brief Stops reading data from the stream.\n     *\n     * This function is idempotent and may be safely called on a stopped stream.\n     *\n     * @return Underlying return value.\n     */\n    int stop() {\n        return uv_read_stop(as_uv_stream());\n    }\n\n    /**\n     * @brief Writes data to the stream.\n     *\n     * Data are written in order. The handle takes the ownership of the data and\n     * it is in charge of delete them.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename Deleter>\n    int write(std::unique_ptr<char[], Deleter> data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<Deleter>>(std::move(data), len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream());\n    }\n\n    /**\n     * @brief Writes data to the stream.\n     *\n     * Data are written in order. The handle doesn't take the ownership of the\n     * data. Be sure that their lifetime overcome the one of the request.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int write(char *data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<void (*)(char *)>>(std::unique_ptr<char[], void (*)(char *)>{data, [](char *) {}}, len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream());\n    }\n\n    /**\n     * @brief Extended write function for sending handles over a pipe handle.\n     *\n     * The pipe must be initialized with `ipc == true`.\n     *\n     * `send` must be a tcp or pipe handle, which is a server or a connection\n     * (listening or connected state). Bound sockets or pipes will be assumed to\n     * be servers.\n     *\n     * The handle takes the ownership of the data and it is in charge of delete\n     * them.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param send The handle over which to write data.\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename S, typename Deleter>\n    int write(S &send, std::unique_ptr<char[], Deleter> data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<Deleter>>(std::move(data), len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream(), send.as_uv_stream());\n    }\n\n    /**\n     * @brief Extended write function for sending handles over a pipe handle.\n     *\n...\n", "needle_token_start": 3974, "needle_token_end": 3983, "code_context_ntokens": 16074, "output": ["```cpp\nclass uv_token {\n    friend class loop;\n    \nexplicit uv_token(int) {}\n};\n```"]}
{"repo": "skypjack/uvw", "name": "write_req", "language": "cpp", "path": "src/uvw/stream.h", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this function is to handle the asynchronous writing of data to a stream, managing memory and notifying upon completion or error.\n2. **Input**: It takes a token for event loop association, a shared pointer to the event loop, a unique pointer to the data buffer managed by a custom deleter, and the length of the data to be written.\n3. **Output**: It does not return a value directly but triggers events upon completion. If the write operation is successful, a success event is published; if there is an error, an error event with the status code is published.\n4. **Procedure**: The function initializes a write request with the provided data and length. It then sets up a callback to handle the result of the write operation. When the write operation completes, the callback checks the status. If there is an error, it publishes an error event; otherwise, it publishes a success event indicating the write operation was successful.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/uv_type.hpp\n#ifndef UVW_UV_TYPE_INCLUDE_H\n#define UVW_UV_TYPE_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include \"config.h\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/**\n * @brief Wrapper class for underlying types.\n *\n * It acts mainly as a wrapper around data structures of the underlying library.\n */\ntemplate<typename U>\nstruct uv_type {\n    explicit uv_type(loop::token, std::shared_ptr<loop> ref) noexcept\n        : owner{std::move(ref)}, resource{} {}\n\n    uv_type(const uv_type &) = delete;\n    uv_type(uv_type &&) = delete;\n\n    uv_type &operator=(const uv_type &) = delete;\n    uv_type &operator=(uv_type &&) = delete;\n\n    /**\n     * @brief Gets the loop from which the resource was originated.\n     * @return A reference to a loop instance.\n     */\n    loop &parent() const noexcept {\n        return *owner;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const U *raw() const noexcept {\n        return &resource;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    U *raw() noexcept {\n        return &resource;\n    }\n\nprotected:\n    ~uv_type() = default;\n\nprivate:\n    std::shared_ptr<loop> owner;\n    U resource;\n};\n\n} // namespace uvw\n...\n// Path: src/uvw/resource.hpp\n#ifndef UVW_RESOURCE_INCLUDE_H\n#define UVW_RESOURCE_INCLUDE_H\n\n#include <memory>\n#include <utility>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Common class for almost all the resources available in `uvw`.\n *\n * This is the base class for handles and requests.\n */\ntemplate<typename T, typename U, typename... E>\nclass resource: public uv_type<U>, public emitter<T, E...>, public std::enable_shared_from_this<T> {\nprotected:\n    int leak_if(int err) noexcept {\n        if(err == 0) {\n            self_ptr = this->shared_from_this();\n        }\n\n        return err;\n    }\n\n    void self_reset() noexcept {\n        self_ptr.reset();\n    }\n\n    bool has_self() const noexcept {\n        return static_cast<bool>(self_ptr);\n    }\n\npublic:\n    explicit resource(loop::token token, std::shared_ptr<loop> ref)\n        : uv_type<U>{token, std::move(ref)} {\n        this->raw()->data = this;\n    }\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param udata User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> udata) {\n        user_data = std::move(udata);\n    }\n\nprivate:\n    std::shared_ptr<void> user_data{nullptr};\n    std::shared_ptr<void> self_ptr{nullptr};\n};\n\n} // namespace uvw\n\n#endif // UVW_RESOURCE_INCLUDE_H\n\n// Path: src/uvw/handle.hpp\n#ifndef UVW_HANDLE_INCLUDE_H\n#define UVW_HANDLE_INCLUDE_H\n\n#include <cstddef>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n#include \"util.h\"\n\nnamespace uvw {\n\n/*! @brief Close event. */\nstruct close_event {};\n\n/**\n * @brief Handle base class.\n *\n * Base type for all `uvw` handle types.\n */\ntemplate<typename T, typename U, typename... E>\nclass handle: public resource<T, U, close_event, E...> {\nprotected:\n    static void close_callback(uv_handle_t *hndl) {\n        handle<T, U, E...> &ref = *(static_cast<T *>(hndl->data));\n        [[maybe_unused]] auto ptr = ref.shared_from_this();\n        ref.self_reset();\n        ref.publish(close_event{});\n    }\n\n    uv_handle_t *as_uv_handle() {\n        return reinterpret_cast<uv_handle_t *>(this->raw());\n    }\n\n    const uv_handle_t *as_uv_handle() const {\n        return reinterpret_cast<const uv_handle_t *>(this->raw());\n    }\n\npublic:\n    using resource<T, U, close_event, E...>::resource;\n\n    /**\n     * @brief Gets the category of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, an opaque value that identifies the\n     * category of the handle is made available to the users.\n     *\n     * @return The actual category of the handle.\n     */\n    handle_category category() const noexcept {\n        return handle_category{as_uv_handle()->type};\n    }\n\n    /**\n     * @brief Gets the type of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, the type of the underlying handle as\n     * specified by handle_type is made available to the users.\n     *\n     * @return The actual type of the handle.\n     */\n    handle_type type() const noexcept {\n        return utilities::guess_handle(category());\n    }\n\n    /**\n     * @brief Checks if the handle is active.\n     *\n     * What _active_ means depends on the type of handle:\n     *\n     * * An async_handle handle is always active and cannot be deactivated,\n     * except by closing it with uv_close().\n     * * A pipe, tcp, udp, etc. handle - basically any handle that deals with\n     * I/O - is active when it is doing something that involves I/O, like\n     * reading, writing, connecting, accepting new connections, etc.\n     * * A check, idle, timer, etc. handle is active when it has been started\n     * with a call to `start()`.\n     *\n     * Rule of thumb: if a handle of type `foo_handle` has a `start()` member\n     * method, then it\u2019s active from the moment that method is called. Likewise,\n     * `stop()` deactivates the handle again.\n     *\n     * @return True if the handle is active, false otherwise.\n     */\n    bool active() const noexcept {\n        return !!uv_is_active(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if a handle is closing or closed.\n     *\n     * This function should only be used between the initialization of the\n     * handle and the arrival of the close callback.\n     *\n     * @return True if the handle is closing or closed, false otherwise.\n     */\n    bool closing() const noexcept {\n        return !!uv_is_closing(as_uv_handle());\n    }\n\n    /**\n     * @brief Request handle to be closed.\n     *\n     * This **must** be called on each handle before memory is released.<br/>\n     * In-progress requests are cancelled and this can result in errors.\n     *\n     * The handle will emit a close event when finished.\n     */\n    void close() noexcept {\n        if(!closing()) {\n            uv_close(as_uv_handle(), &handle<T, U, E...>::close_callback);\n        }\n    }\n\n    /**\n     * @brief Reference the given handle.\n     *\n     * References are idempotent, that is, if a handle is already referenced\n     * calling this function again will have no effect.\n     */\n    void reference() noexcept {\n        uv_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Unreference the given handle.\n     *\n     * References are idempotent, that is, if a handle is not referenced calling\n     * this function again will have no effect.\n     */\n    void unreference() noexcept {\n        uv_unref(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if the given handle referenced.\n     * @return True if the handle referenced, false otherwise.\n     */\n    bool referenced() const noexcept {\n        return !!uv_has_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Returns the size of the underlying handle type.\n     * @return The size of the underlying handle type.\n     */\n    std::size_t size() const noexcept {\n        return uv_handle_size(as_uv_handle()->type);\n    }\n\n    /**\n     * @brief Gets the size of the send buffer used for the socket.\n     *\n     * Gets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipeand udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the send buffer, the underlying return value in case\n     * of errors.\n     */\n    int send_buffer_size() {\n        int value = 0;\n        auto err = uv_send_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the send buffer used for the socket.\n     *\n     * Sets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int send_buffer_size(int value) {\n        return uv_send_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the size of the receive buffer used for the socket.\n     *\n     * Gets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the receive buffer, the underlying return value in\n     * case of errors.\n     */\n    int recv_buffer_size() {\n        int value = 0;\n        auto err = uv_recv_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the receive buffer used for the socket.\n     *\n     * Sets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int recv_buffer_size(int value) {\n        return uv_recv_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the platform dependent file descriptor equivalent.\n     *\n     * Supported handles:\n     *\n     * * tcp_handle\n     * * pipe_handle\n     * * tty_handle\n     * * udp_handle\n     * * poll_handle\n     *\n     * If invoked on a different handle, one that doesn\u2019t have an attached file\n     * descriptor yet or one which was closed, an invalid value is returned.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/handle.html#c.uv_fileno)\n     * for further details.\n     *\n     * @return The file descriptor attached to the hande or a negative value in\n     * case of errors.\n     */\n    os_file_descriptor fd() const {\n        uv_os_fd_t fd;\n        uv_fileno(as_uv_handle(), &fd);\n        return fd;\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_HANDLE_INCLUDE_H\n\n// Path: src/uvw/prepare.h\n#ifndef UVW_PREPARE_INCLUDE_H\n#define UVW_PREPARE_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Prepare event. */\nstruct prepare_event {};\n\n/**\n * @brief The prepare handle.\n *\n * Prepare handles will emit a prepare event once per loop iteration, right\n * before polling for I/O.\n *\n * To create a `prepare_handle` through a `loop`, no arguments are required.\n */\nclass prepare_handle final: public handle<prepare_handle, uv_prepare_t, prepare_event> {\n    static void start_callback(uv_prepare_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * A prepare event will be emitted once per loop iteration, right before\n     * polling for I/O.\n     *\n     * The handle will start emitting prepare events when needed.\n     *\n     * @return Underlying return value.\n     */\n    int start();\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"prepare.cpp\"\n#endif\n\n#endif // UVW_PREPARE_INCLUDE_H\n\n// Path: src/uvw/prepare.cpp\n#ifdef UVW_AS_LIB\n#    include \"prepare.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void prepare_handle::start_callback(uv_prepare_t *hndl) {\n    prepare_handle &prepare = *(static_cast<prepare_handle *>(hndl->data));\n    prepare.publish(prepare_event{});\n}\n\nUVW_INLINE int prepare_handle::init() {\n    return leak_if(uv_prepare_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int prepare_handle::start() {\n    return uv_prepare_start(raw(), &start_callback);\n}\n\nUVW_INLINE int prepare_handle::stop() {\n    return uv_prepare_stop(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/request.hpp\n#ifndef UVW_REQUEST_INCLUDE_H\n#define UVW_REQUEST_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Request base class.\n *\n * Base type for all `uvw` request types.\n */\ntemplate<typename T, typename U, typename... E>\nclass request: public resource<T, U, E...> {\nprotected:\n    static auto reserve(U *req) {\n        auto ptr = static_cast<T *>(req->data)->shared_from_this();\n        ptr->self_reset();\n        return ptr;\n    }\n\npublic:\n    using resource<T, U, E...>::resource;\n\n    /**\n     * @brief Cancels a pending request.\n     *\n     * This method fails if the request is executing or has finished\n     * executing.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/request.html#c.uv_cancel)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int cancel() {\n        return uv_cancel(reinterpret_cast<uv_req_t *>(this->raw()));\n    }\n\n    /**\n     * @brief Returns the size of the underlying request type.\n     * @return The size of the underlying request type.\n     */\n    std::size_t size() const noexcept {\n        return uv_req_size(reinterpret_cast<const uv_req_t *>(this->raw())->type);\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_REQUEST_INCLUDE_H\n\n// Path: src/uvw/stream.h\n#ifndef UVW_STREAM_INCLUDE_H\n#define UVW_STREAM_INCLUDE_H\n\n#include <algorithm>\n#include <cstddef>\n#include <iterator>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"handle.hpp\"\n#include \"loop.h\"\n#include \"request.hpp\"\n\nnamespace uvw {\n\n/*! @brief Connect event. */\nstruct connect_event {};\n\n/*! @brief End event. */\nstruct end_event {};\n\n/*! @brief Listen event. */\nstruct listen_event {};\n\n/*! @brief Shutdown event. */\nstruct shutdown_event {};\n\n/*! @brief Write event. */\nstruct write_event {};\n\n/*! @brief Data event. */\nstruct data_event {\n    explicit data_event(std::unique_ptr<char[]> buf, std::size_t len) noexcept;\n\n    std::unique_ptr<char[]> data; /*!< A bunch of data read on the stream. */\n    std::size_t length;           /*!< The amount of data read on the stream. */\n};\n\nnamespace details {\n\nclass connect_req final: public request<connect_req, uv_connect_t, connect_event> {\n    static void connect_callback(uv_connect_t *req, int status);\n\npublic:\n    using request::request;\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback);\n        return this->leak_if(0);\n    }\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<!std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        return this->leak_if(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback));\n    }\n};\n\nclass shutdown_req final: public request<shutdown_req, uv_shutdown_t, shutdown_event> {\n    static void shoutdown_callback(uv_shutdown_t *req, int status);\n\npublic:\n    using request::request;\n\n    int shutdown(uv_stream_t *hndl);\n};\n\ntemplate<typename Deleter>\nclass write_req final: public request<write_req<Deleter>, uv_write_t, write_event> {\n    static void write_callback(uv_write_t *req, int status) {\n        if(auto ptr = request<write_req<Deleter>, uv_write_t, write_event>::reserve(req); status) {\n            ptr->publish(error_event{status});\n        } else {\n            ptr->publish(write_event{});\n        }\n    }\n\npublic:\n    \nwrite_req(loop::token token, std::shared_ptr<loop> parent, std::unique_ptr<char[], Deleter> dt, unsigned int len)\n        : request<write_req<Deleter>, uv_write_t, write_event>{token, std::move(parent)},\n          data{std::move(dt)},\n          buf{uv_buf_init(data.get(), len)} {}\n\n    int write(uv_stream_t *hndl) {\n        return this->leak_if(uv_write(this->raw(), hndl, &buf, 1, &write_callback));\n    }\n\n    int write(uv_stream_t *hndl, uv_stream_t *send) {\n        return this->leak_if(uv_write2(this->raw(), hndl, &buf, 1, send, &write_callback));\n    }\n\nprivate:\n    std::unique_ptr<char[], Deleter> data;\n    uv_buf_t buf;\n};\n\n} // namespace details\n\n/**\n * @brief The stream handle.\n *\n * Stream handles provide an abstraction of a duplex communication channel.\n * The stream handle is an intermediate type, `uvw` provides three stream\n * implementations: tcp, pipe and tty handles.\n */\ntemplate<typename T, typename U, typename... E>\nclass stream_handle: public handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...> {\n    using base = handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...>;\n\n    template<typename, typename, typename...>\n    friend class stream_handle;\n\n    static constexpr unsigned int DEFAULT_BACKLOG = 128;\n\n    static void read_callback(uv_stream_t *hndl, ssize_t nread, const uv_buf_t *buf) {\n        T &ref = *(static_cast<T *>(hndl->data));\n        // data will be destroyed no matter of what the value of nread is\n        std::unique_ptr<char[]> data{buf->base};\n\n        // nread == 0 is ignored (see http://docs.libuv.org/en/v1.x/stream.html)\n        // equivalent to EAGAIN/EWOULDBLOCK, it shouldn't be treated as an error\n        // for we don't have data to emit though, it's fine to suppress it\n\n        if(nread == UV_EOF) {\n            // end of stream\n            ref.publish(end_event{});\n        } else if(nread > 0) {\n            // data available\n            ref.publish(data_event{std::move(data), static_cast<std::size_t>(nread)});\n        } else if(nread < 0) {\n            // transmission error\n            ref.publish(error_event(nread));\n        }\n    }\n\n    static void listen_callback(uv_stream_t *hndl, int status) {\n        if(T &ref = *(static_cast<T *>(hndl->data)); status) {\n            ref.publish(error_event{status});\n        } else {\n            ref.publish(listen_event{});\n        }\n    }\n\n    uv_stream_t *as_uv_stream() {\n        return reinterpret_cast<uv_stream_t *>(this->raw());\n    }\n\n    const uv_stream_t *as_uv_stream() const {\n        return reinterpret_cast<const uv_stream_t *>(this->raw());\n    }\n\npublic:\n#ifdef _MSC_VER\n    stream_handle(loop::token token, std::shared_ptr<loop> ref)\n        : base{token, std::move(ref)} {}\n#else\n    using base::base;\n#endif\n\n    /**\n     * @brief Shutdowns the outgoing (write) side of a duplex stream.\n     *\n     * It waits for pending write requests to complete. The handle should refer\n     * to a initialized stream.<br/>\n     * A shutdown event will be emitted after shutdown is complete.\n     *\n     * @return Underlying return value.\n     */\n    int shutdown() {\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        auto shutdown = this->parent().template resource<details::shutdown_req>();\n        shutdown->template on<error_event>(listener);\n        shutdown->template on<shutdown_event>(listener);\n\n        return shutdown->shutdown(as_uv_stream());\n    }\n\n    /**\n     * @brief Starts listening for incoming connections.\n     *\n     * When a new incoming connection is received, a listen event is\n     * emitted.\n     *\n     * @param backlog Indicates the number of connections the kernel might\n     * queue, same as listen(2).\n     *\n     * @return Underlying return value.\n     */\n    int listen(int backlog = DEFAULT_BACKLOG) {\n        return uv_listen(as_uv_stream(), backlog, &listen_callback);\n    }\n\n    /**\n     * @brief Accepts incoming connections.\n     *\n     * This call is used in conjunction with `listen()` to accept incoming\n     * connections. Call this function after receiving a listen event to accept\n     * the connection. Before calling this function, the submitted handle must\n     * be initialized.\n     *\n     * When the listen event is emitted it is guaranteed that this function will\n     * complete successfully the first time. If you attempt to use it more than\n     * once, it may fail.<br/>\n     * It is suggested to only call this function once per listen event.\n     *\n     * @note\n     * Both the handles must be running on the same loop.\n     *\n     * @param ref An initialized handle to be used to accept the connection.\n     * @return Underlying return value.\n     */\n    template<typename S>\n    int accept(S &ref) {\n        return uv_accept(as_uv_stream(), ref.as_uv_stream());\n    }\n\n    /**\n     * @brief Starts reading data from an incoming stream.\n     *\n     * A data event will be emitted several times until there is no more data to\n     * read or `stop()` is called.<br/>\n     * An end event will be emitted when there is no more data to read.\n     *\n     * @return Underlying return value.\n     */\n    int read() {\n        return uv_read_start(as_uv_stream(), &details::common_alloc_callback, &read_callback);\n    }\n\n    /**\n     * @brief Starts reading data from an incoming stream.\n     * @sa read\n     * @tparam Alloc Custom allocation function.\n     * @return Underlying return value.\n     */\n    template<auto Alloc>\n    int read() {\n        return uv_read_start(as_uv_stream(), &details::common_alloc_callback<T, Alloc>, &read_callback);\n    }\n\n    /**\n     * @brief Stops reading data from the stream.\n     *\n     * This function is idempotent and may be safely called on a stopped stream.\n     *\n     * @return Underlying return value.\n     */\n    int stop() {\n        return uv_read_stop(as_uv_stream());\n    }\n\n    /**\n     * @brief Writes data to the stream.\n     *\n     * Data are written in order. The handle takes the ownership of the data and\n     * it is in charge of delete them.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename Deleter>\n    int write(std::unique_ptr<char[], Deleter> data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<Deleter>>(std::move(data), len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream());\n    }\n\n    /**\n     * @brief Writes data to the stream.\n     *\n     * Data are written in order. The handle doesn't take the ownership of the\n     * data. Be sure that their lifetime overcome the one of the request.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int write(char *data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<void (*)(char *)>>(std::unique_ptr<char[], void (*)(char *)>{data, [](char *) {}}, len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream());\n    }\n\n    /**\n     * @brief Extended write function for sending handles over a pipe handle.\n     *\n     * The pipe must be initialized with `ipc == true`.\n     *\n     * `send` must be a tcp or pipe handle, which is a server or a connection\n     * (listening or connected state). Bound sockets or pipes will be assumed to\n     * be servers.\n     *\n     * The handle takes the ownership of the data and it is in charge of delete\n     * them.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param send The handle over which to write data.\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename S, typename Deleter>\n    int write(S &send, std::unique_ptr<char[], Deleter> data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<Deleter>>(std::move(data), len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream(), send.as_uv_stream());\n    }\n\n    /**\n     * @brief Extended write function for sending handles over a pipe handle.\n     *\n     * The pipe must be initialized with `ipc == true`.\n     *\n     * `send` must be a tcp or pipe handle, which is a server or a connection\n     * (listening or connected state). Bound sockets or pipes will be assumed to\n     * be servers.\n     *\n     * The handle doesn't take the ownership of the data. Be sure that their\n     * lifetime overcome the one of the request.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param send The handle over which to write data.\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename S>\n    int write(S &send, char *data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<void (*)(char *)>>(std::unique_ptr<char[], void (*)(char *)>{data, [](char *) {}}, len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream(), send.as_uv_stream());\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `write()`, but won\u2019t queue a write request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_write(std::unique_ptr<char[]> data, unsigned int len) {\n        uv_buf_t bufs[] = {uv_buf_init(data.get(), len)};\n        return uv_try_write(as_uv_stream(), bufs, 1);\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `try_write` for sending handles over a pipe.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @param send A valid handle suitable for the purpose.\n     * @return Underlying return value.\n     */\n    template<typename V, typename W>\n    int try_write(std::unique_ptr<char[]> data, unsigned int len, stream_handle<V, W> &send) {\n        uv_buf_t bufs[] = {uv_buf_init(data.get(), len)};\n        return uv_try_write2(as_uv_stream(), bufs, 1, send.raw());\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `write()`, but won\u2019t queue a write request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_write(char *data, unsigned int len) {\n        uv_buf_t bufs[] = {uv_buf_init(data, len)};\n        return uv_try_write(as_uv_stream(), bufs, 1);\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `try_write` for sending handles over a pipe.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @param send A valid handle suitable for the purpose.\n     * @return Underlying return value.\n     */\n    template<typename V, typename W>\n    int try_write(char *data, unsigned int len, stream_handle<V, W> &send) {\n        uv_buf_t bufs[] = {uv_buf_init(data, len)};\n        return uv_try_write2(as_uv_stream(), bufs, 1, send.raw());\n    }\n\n    /**\n     * @brief Checks if the stream is readable.\n     * @return True if the stream is readable, false otherwise.\n     */\n    bool readable() const noexcept {\n        return (uv_is_readable(as_uv_stream()) == 1);\n    }\n\n    /**\n     * @brief Checks if the stream is writable.\n     * @return True if the stream is writable, false otherwise.\n     */\n    bool writable() const noexcept {\n        return (uv_is_writable(as_uv_stream()) == 1);\n    }\n\n    /**\n     * @brief Enables or disables blocking mode for a stream.\n     *\n     * When blocking mode is enabled all writes complete synchronously. The\n     * interface remains unchanged otherwise, e.g. completion or failure of the\n     * operation will still be reported through events which are emitted\n     * asynchronously.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/stream.html#c.uv_stream_set_blocking)\n     * for further details.\n     *\n     * @param enable True to enable blocking mode, false otherwise.\n     * @return True in case of success, false otherwise.\n     */\n    bool blocking(bool enable = false) {\n        return (0 == uv_stream_set_blocking(as_uv_stream(), enable));\n    }\n\n    /**\n     * @brief Gets the amount of queued bytes waiting to be sent.\n     * @return Amount of queued bytes waiting to be sent.\n     */\n    size_t write_queue_size() const noexcept {\n        return uv_stream_get_write_queue_size(as_uv_stream());\n    }\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"stream.cpp\"\n#endif\n\n#endif // UVW_STREAM_INCLUDE_H\n\n// Path: src/uvw/stream.cpp\n#ifdef UVW_AS_LIB\n#    include \"stream.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE data_event::data_event(std::unique_ptr<char[]> buf, std::size_t len) noexcept\n    : data{std::move(buf)},\n      length{len} {}\n\nUVW_INLINE void details::connect_req::connect_callback(uv_connect_t *req, int status) {\n    if(auto ptr = reserve(req); status) {\n        ptr->publish(error_event{status});\n    } else {\n        ptr->publish(connect_event{});\n    }\n}\n\nUVW_INLINE void details::shutdown_req::shoutdown_callback(uv_shutdown_t *req, int status) {\n    if(auto ptr = reserve(req); status) {\n        ptr->publish(error_event{status});\n    } else {\n        ptr->publish(shutdown_event{});\n    }\n}\n\nUVW_INLINE int details::shutdown_req::shutdown(uv_stream_t *hndl) {\n    return this->leak_if(uv_shutdown(raw(), hndl, &shoutdown_callback));\n}\n\n} // namespace uvw\n\n// Path: src/uvw/check.h\n#ifndef UVW_CHECK_INCLUDE_H\n#define UVW_CHECK_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Check event. */\nstruct check_event {};\n\n/**\n * @brief The check handle.\n *\n * Check handles will emit a check event once per loop iteration, right after\n * polling for I/O.\n *\n * To create a `check_handle` through a `loop`, no arguments are required.\n */\nclass check_handle final: public handle<check_handle, uv_check_t, check_event> {\n    static void start_callback(uv_check_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * A check event will be emitted once per loop iteration, right after\n     * polling for I/O.\n     *\n     * @return Underlying return value.\n     */\n    int start();\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"check.cpp\"\n#endif\n\n#endif // UVW_CHECK_INCLUDE_H\n\n// Path: src/uvw/check.cpp\n#ifdef UVW_AS_LIB\n#    include \"check.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void check_handle::start_callback(uv_check_t *hndl) {\n    check_handle &check = *(static_cast<check_handle *>(hndl->data));\n    check.publish(check_event{});\n}\n\nUVW_INLINE int check_handle::init() {\n    return leak_if(uv_check_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int check_handle::start() {\n    return uv_check_start(raw(), &start_callback);\n}\n\nUVW_INLINE int check_handle::stop() {\n    return uv_check_stop(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/enum.hpp\n#ifndef UVW_ENUM_INCLUDE_HPP\n#define UVW_ENUM_INCLUDE_HPP\n\n#include <type_traits>\n#include \"config.h\"\n\n/**\n * @brief Operator available for enums for which bitmask support is enabled.\n * @tparam Type Enum class type.\n * @param lhs The first value to use.\n * @param rhs The second value to use.\n * @return The result of invoking the operator on the underlying types of the\n * two values provided.\n */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator|(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) | static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator&(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) & static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator^(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) ^ static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/**\n * @brief Operator available for enums for which bitmask support is enabled.\n * @tparam Type Enum class type.\n * @param value The value to use.\n * @return The result of invoking the operator on the underlying types of the\n * value provided.\n */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator~(const Type value) noexcept {\n    return static_cast<Type>(~static_cast<std::underlying_type_t<Type>>(value));\n}\n\n/*! @copydoc operator~ */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM, bool{})>\noperator!(const Type value) noexcept {\n    return !static_cast<std::underlying_type_t<Type>>(value);\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator|=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs | rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator&=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs & rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator^=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs ^ rhs));\n}\n\n#endif\n\n// Path: src/uvw/thread.h\n#ifndef UVW_THREAD_INCLUDE_H\n#define UVW_THREAD_INCLUDE_H\n\n#include <cstddef>\n#include <cstdint>\n#include <memory>\n#include <string>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"enum.hpp\"\n#include \"loop.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_thread_create_flags : std::underlying_type_t<uv_thread_create_flags> {\n    THREAD_NO_FLAGS = UV_THREAD_NO_FLAGS,\n    THREAD_HAS_STACK_SIZE = UV_THREAD_HAS_STACK_SIZE\n};\n\n}\n\nclass thread;\nclass thread_local_storage;\nclass once;\nclass mutex;\nclass rwlock;\nclass semaphore;\nclass condition;\nclass barrier;\n\n/**\n * @brief The thread wrapper.\n *\n * To create a `thread` through a `loop`, arguments follow:\n *\n * * A callback invoked to initialize thread execution. The type must be such\n * that it can be assigned to an `std::function<void(std::shared_ptr<void>)>`.\n * * An optional payload the type of which is `std::shared_ptr<void>`.\n */\nclass thread final: public uv_type<uv_thread_t> {\n    using internal_task = std::function<void(std::shared_ptr<void>)>;\n\n    static void create_callback(void *arg);\n\npublic:\n    using create_flags = details::uvw_thread_create_flags;\n    using task = internal_task;\n    using type = uv_thread_t;\n\n    explicit thread(loop::token token, std::shared_ptr<loop> ref, task t, std::shared_ptr<void> d = nullptr) noexcept;\n\n    /**\n     * @brief Obtains the identifier of the calling thread.\n     * @return The identifier of the calling thread.\n     */\n    static type self() noexcept;\n\n    /**\n     * @brief Gets the CPU number on which the calling thread is running.\n     * @return The CPU number on which the calling thread is running.\n     */\n    static int getcpu() noexcept;\n\n    /**\n     * @brief Compares thread by means of their identifiers.\n     * @param tl A valid instance of a thread.\n     * @param tr A valid instance of a thread.\n     * @return True if the two threads are the same thread, false otherwise.\n     */\n    static bool equal(const thread &tl, const thread &tr) noexcept;\n\n    ~thread() noexcept;\n\n    /**\n     * @brief Creates a new thread.\n     * @return True in case of success, false otherwise.\n     */\n    bool run() noexcept;\n\n    /**\n     * @brief Creates a new thread.\n     *\n     * Available flags are:\n     *\n     * * `thread::create_flags::THREAD_NO_FLAGS`: no flags set.\n     * * `thread::create_flags::THREAD_HAS_STACK_SIZE`: if set, `stack` specifies a\n     * stack size for the new thread. 0 indicates that the default value should\n     * be used (it behaves as if the flag was not set). Other values will be\n     * rounded up to the nearest page boundary.\n     *\n     * @return True in case of success, false otherwise.\n     */\n    bool run(create_flags opts, std::size_t stack = {}) noexcept;\n\n    /**\n     * @brief Joins with a terminated thread.\n     * @return True in case of success, false otherwise.\n     */\n    bool join() noexcept;\n\nprivate:\n    std::shared_ptr<void> data;\n    task func;\n};\n\n/**\n * @brief The thread local storage wrapper.\n *\n * A storage area that can only be accessed by one thread. The variable can be\n * seen as a global variable that is only visible to a particular thread and not\n * the whole program.\n */\nclass thread_local_storage final: public uv_type<uv_key_t> {\npublic:\n    explicit thread_local_storage(loop::token token, std::shared_ptr<loop> ref) noexcept;\n\n    ~thread_local_storage() noexcept;\n\n    /**\n     * @brief Gets the value of a given variable.\n     * @tparam T Type to which to cast the opaque storage area.\n     * @return A pointer to the given variable.\n     */\n    template<typename T>\n    T *get() noexcept {\n        return static_cast<T *>(uv_key_get(uv_type::raw()));\n    }\n\n    /**\n     * @brief Sets the value of a given variable.\n     * @tparam T Type of the variable to store aside.\n     * @param value A valid pointer to the variable to store\n     */\n    template<typename T>\n    void set(T *value) noexcept {\n        return uv_key_set(uv_type::raw(), value);\n    }\n};\n\n/**\n * @brief The once wrapper.\n *\n * Runs a function once and only once. Concurrent calls to `once` will block all\n * callers except one (it\u2019s unspecified which one).\n */\nclass once final: public uv_type<uv_once_t> {\n    static uv_once_t *guard() noexcept;\n\npublic:\n    using uv_type::uv_type;\n\n    /**\n     * @brief Runs a function once and only once.\n     *\n     * The callback must be such that it's convertible to `void(*)(void)`. Free\n     * functions and non-capturing lambdas are both viable solutions.\n     *\n     * @tparam F Type of the callback.\n     * @param f A valid callback function.\n     */\n    template<typename F>\n    static void run(F &&f) noexcept {\n        using callback_type = void (*)(void);\n        static_assert(std::is_convertible_v<F, callback_type>);\n        callback_type cb = f;\n        uv_once(guard(), cb);\n    }\n};\n\n/**\n * @brief The mutex wrapper.\n *\n * To create a `mutex` through a `loop`, arguments follow:\n *\n * * An option boolean that specifies if the mutex is a recursive one. The\n * default value is false, the mutex isn't recursive.\n */\nclass mutex final: public uv_type<uv_mutex_t> {\n    friend class condition;\n\npublic:\n    explicit mutex(loop::token token, std::shared_ptr<loop> ref, bool recursive = false) noexcept;\n\n    ~mutex() noexcept;\n\n    /**\n     * @brief Locks the mutex.\n     */\n    void lock() noexcept;\n\n    /**\n     * @brief Tries to lock the mutex.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_lock() noexcept;\n\n    /**\n     * @brief Unlocks the mutex.\n     */\n    void unlock() noexcept;\n};\n\n/**\n * @brief The rwlock wrapper.\n */\nclass rwlock final: public uv_type<uv_rwlock_t> {\npublic:\n    explicit rwlock(loop::token token, std::shared_ptr<loop> ref) noexcept;\n\n    ~rwlock() noexcept;\n\n    /**\n     * @brief Locks a read-write lock object for reading.\n     */\n    void rdlock() noexcept;\n\n    /**\n     * @brief Tries to lock a read-write lock object for reading.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_rdlock() noexcept;\n\n    /**\n     * @brief Unlocks a read-write lock object previously locked for reading.\n     */\n    void rdunlock() noexcept;\n\n    /**\n     * @brief Locks a read-write lock object for writing.\n     */\n    void wrlock() noexcept;\n\n    /**\n     * @brief Tries to lock a read-write lock object for writing.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_wrlock() noexcept;\n\n    /**\n     * @brief Unlocks a read-write lock object previously locked for writing.\n     */\n    void wrunlock() noexcept;\n};\n\n/**\n * @brief The semaphore wrapper.\n *\n * To create a `semaphore` through a `loop`, arguments follow:\n *\n * * An unsigned integer that specifies the initial value for the semaphore.\n */\nclass semaphore final: public uv_type<uv_sem_t> {\npublic:\n    explicit semaphore(loop::token token, std::shared_ptr<loop> ref, unsigned int value) noexcept;\n\n    ~semaphore() noexcept;\n\n    /**\n     * @brief Unlocks a semaphore.\n     */\n    void post() noexcept;\n\n    /**\n     * @brief Locks a semaphore.\n     */\n    void wait() noexcept;\n\n    /**\n     * @brief Tries to lock a semaphore.\n     * @return True in case of success, false otherwise.\n     */\n    bool try_wait() noexcept;\n};\n\n/**\n * @brief The condition wrapper.\n */\nclass condition final: public uv_type<uv_cond_t> {\npublic:\n    explicit condition(loop::token token, std::shared_ptr<loop> ref) noexcept;\n\n    ~condition() noexcept;\n\n    /**\n     * @brief Signals a condition.\n     *\n     * This function shall unblock at least one of the threads that are blocked\n     * on the specified condition variable (if any threads are blocked on it).\n     */\n    void signal() noexcept;\n\n    /**\n     * @brief Broadcasts a condition.\n     *\n     * This function shall unblock threads blocked on a condition variable.\n     */\n    void broadcast() noexcept;\n\n    /**\n     * @brief Waits on a condition.\n     *\n     * These function atomically releases the mutex and causes the calling\n     * thread to block on the condition variable.\n     *\n     * @param mtx A mutex locked by the calling thread, otherwise expect\n     * undefined behavior.\n     */\n    void wait(mutex &mtx) noexcept;\n\n    /**\n     * @brief Waits on a condition.\n     *\n     * These function atomically releases the mutex and causes the calling\n     * thread to block on the condition variable.<br/>\n     * The functions returns with an error if the absolute time specified passes\n     * (that is, system time equals or exceeds it) before the condition is\n     * signaled or broadcasted, or if the absolute time specified has already\n     * been passed at the time of the call.\n     *\n     * @param mtx A mutex locked by the calling thread, otherwise expect\n     * undefined behavior.\n     * @param timeout The maximum time to wait before to return.\n     * @return True in case of success, false otherwise.\n     */\n    bool timed_wait(mutex &mtx, uint64_t timeout) noexcept;\n};\n\n/**\n * @brief The barrier wrapper.\n *\n * To create a `barrier` through a `loop`, arguments follow:\n *\n * * An unsigned integer that specifies the number of threads that must call\n * `wait` before any of them successfully return from the call. The value\n * specified must be greater than zero.\n */\nclass barrier final: public uv_type<uv_barrier_t> {\npublic:\n    explicit barrier(loop::token token, std::shared_ptr<loop> ref, unsigned int count) noexcept;\n\n    ~barrier() noexcept;\n\n    /**\n     * @brief Synchronizes at a barrier.\n     * @return True in case of success, false otherwise.\n     */\n    bool wait() noexcept;\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"thread.cpp\"\n#endif\n\n#endif // UVW_THREAD_INCLUDE_H\n\n// Path: src/uvw/thread.cpp\n#ifdef UVW_AS_LIB\n#    include \"thread.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE thread::thread(loop::token token, std::shared_ptr<loop> ref, task t, std::shared_ptr<void> d) noexcept\n    : uv_type{token, std::move(ref)},\n      data{std::move(d)},\n      func{std::move(t)} {}\n\nUVW_INLINE void thread::create_callback(void *arg) {\n    thread &curr = *(static_cast<thread *>(arg));\n    curr.func(curr.data);\n}\n\nUVW_INLINE thread::type thread::self() noexcept {\n    return uv_thread_self();\n}\n\nUVW_INLINE int thread::getcpu() noexcept {\n    return uv_thread_getcpu();\n}\n\nUVW_INLINE bool thread::equal(const thread &tl, const thread &tr) noexcept {\n    return !(0 == uv_thread_equal(tl.raw(), tr.raw()));\n}\n\nUVW_INLINE thread::~thread() noexcept {\n    join();\n}\n\nUVW_INLINE bool thread::run() noexcept {\n    return (0 == uv_thread_create(raw(), &create_callback, this));\n}\n\nUVW_INLINE bool thread::run(create_flags opts, std::size_t stack) noexcept {\n    uv_thread_options_t params{static_cast<unsigned int>(opts), stack};\n    return (0 == uv_thread_create_ex(raw(), &params, &create_callback, this));\n}\n\nUVW_INLINE bool thread::join() noexcept {\n    return (0 == uv_thread_join(raw()));\n}\n\nUVW_INLINE thread_local_storage::thread_local_storage(loop::token token, std::shared_ptr<loop> ref) noexcept\n    : uv_type{token, std::move(ref)} {\n    uv_key_create(uv_type::raw());\n}\n\nUVW_INLINE thread_local_storage::~thread_local_storage() noexcept {\n    uv_key_delete(uv_type::raw());\n}\n\nUVW_INLINE uv_once_t *once::guard() noexcept {\n    static uv_once_t once = UV_ONCE_INIT;\n    return &once;\n}\n\nUVW_INLINE mutex::mutex(loop::token token, std::shared_ptr<loop> ref, bool recursive) noexcept\n    : uv_type{token, std::move(ref)} {\n    if(recursive) {\n        uv_mutex_init_recursive(raw());\n    } else {\n        uv_mutex_init(raw());\n    }\n}\n\nUVW_INLINE mutex::~mutex() noexcept {\n    uv_mutex_destroy(raw());\n}\n\nUVW_INLINE void mutex::lock() noexcept {\n    uv_mutex_lock(raw());\n}\n\nUVW_INLINE bool mutex::try_lock() noexcept {\n    return (0 == uv_mutex_trylock(raw()));\n}\n\nUVW_INLINE void mutex::unlock() noexcept {\n    uv_mutex_unlock(raw());\n}\n\nUVW_INLINE rwlock::rwlock(loop::token token, std::shared_ptr<loop> ref) noexcept\n    : uv_type{token, std::move(ref)} {\n    uv_rwlock_init(raw());\n}\n\nUVW_INLINE rwlock::~rwlock() noexcept {\n    uv_rwlock_destroy(raw());\n}\n\nUVW_INLINE void rwlock::rdlock() noexcept {\n    uv_rwlock_rdlock(raw());\n}\n\nUVW_INLINE bool rwlock::try_rdlock() noexcept {\n    return (0 == uv_rwlock_tryrdlock(raw()));\n}\n\nUVW_INLINE void rwlock::rdunlock() noexcept {\n    uv_rwlock_rdunlock(raw());\n}\n\nUVW_INLINE void rwlock::wrlock() noexcept {\n    uv_rwlock_wrlock(raw());\n}\n\nUVW_INLINE bool rwlock::try_wrlock() noexcept {\n...\n", "needle_token_start": 5567, "needle_token_end": 5664, "code_context_ntokens": 16023, "output": ["```cpp\ntemplate<typename Deleter>\nclass write_req final: public request<write_req<Deleter>, uv_write_t, write_event> {\n    static void write_callback(uv_write_t *req, int status) {\n        if(auto ptr = request<write_req<Deleter>, uv_write_t, write_event>::reserve(req); status) {\n            ptr->publish(error_event{status});\n        } else {\n            ptr->publish(write_event{});\n        }\n    }\n\npublic:\n    write_req(loop::token token, std::shared_ptr<loop> parent, std::unique_ptr<char[], Deleter> dt, unsigned int len)\n        : request<write_req<Deleter>, uv_write_t, write_event>{token, std::move(parent)},\n          data{std::move(dt)},\n          buf{uv_buf_init(data.get(), len)} {}\n\n    int write(uv_stream_t *hndl) {\n        return this->leak_if(uv_write(this->raw(), hndl, &buf, 1, &write_callback));\n    }\n\n    int write(uv_stream_t *hndl, uv_stream_t *send) {\n        return this->leak_if(uv_write2(this->raw(), hndl, &buf, 1, send, &write_callback));\n    }\n\nprivate:\n    std::unique_ptr<char[], Deleter> data;\n    uv_buf_t buf;\n};\n```"]}
{"repo": "skypjack/uvw", "name": "recv", "language": "cpp", "path": "src/uvw/udp.h", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to prepare the UDP socket for receiving data. If the socket is not already bound, it automatically binds to an all-interfaces IPv4 address with a random port.\n2. **Input**: There are no parameters required for this function.\n3. **Output**: Returns an integer value that represents the success or failure of the operation based on the underlying system call.\n4. **Procedure**: The function checks if the socket is bound; if not, it binds the socket to a default address and port. It then sets up the socket to start receiving data, which will trigger a data event upon data arrival.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/tty.h\n#ifndef UVW_TTY_INCLUDE_H\n#define UVW_TTY_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <uv.h>\n#include \"config.h\"\n#include \"stream.h\"\n#include \"util.h\"\n\nnamespace uvw {\n\nnamespace details {\n\nstruct reset_mode_memo {\n    ~reset_mode_memo();\n};\n\nenum class uvw_tty_mode_t : std::underlying_type_t<uv_tty_mode_t> {\n    NORMAL = UV_TTY_MODE_NORMAL,\n    RAW = UV_TTY_MODE_RAW,\n    IO = UV_TTY_MODE_IO\n};\n\nenum class uvw_tty_vtermstate_t : std::underlying_type_t<uv_tty_vtermstate_t> {\n    SUPPORTED = UV_TTY_SUPPORTED,\n    UNSUPPORTED = UV_TTY_UNSUPPORTED\n};\n\n} // namespace details\n\n/**\n * @brief The tty handle.\n *\n * TTY handles represent a stream for the console.\n *\n * To create a `tty_handle` through a `loop`, arguments follow:\n *\n * * A valid file_handle. Usually the file descriptor will be:\n *     * `uvw::std_in` or `0` for `stdin`\n *     * `uvw::std_out` or `1` for `stdout`\n *     * `uvw::std_err` or `2` for `stderr`\n * * A boolean value that specifies the plan on calling `read()` with this\n * stream. Remember that `stdin` is readable, `stdout` is not.\n *\n * See the official\n * [documentation](http://docs.libuv.org/en/v1.x/tty.html#c.uv_tty_init)\n * for further details.\n */\nclass tty_handle final: public stream_handle<tty_handle, uv_tty_t> {\n    static std::shared_ptr<details::reset_mode_memo> mode_memo_handler();\n\npublic:\n    using tty_mode = details::uvw_tty_mode_t;\n    using tty_vtermstate = details::uvw_tty_vtermstate_t;\n\n    explicit tty_handle(loop::token token, std::shared_ptr<loop> ref, file_handle desc, bool readable);\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Sets the TTY using the specified terminal mode.\n     *\n     * Available modes are:\n     *\n     * * `TTY::tty_mode::NORMAL`\n     * * `TTY::tty_mode::RAW`\n     * * `TTY::tty_mode::IO`\n     *\n     * See the official\n...\n// Path: src/uvw/tty.cpp\n#ifdef UVW_AS_LIB\n#    include \"tty.h\"\n#endif\n\n#include <utility>\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE details::reset_mode_memo::~reset_mode_memo() {\n    uv_tty_reset_mode();\n}\n\nUVW_INLINE tty_handle::tty_handle(loop::token token, std::shared_ptr<loop> ref, file_handle desc, bool readable)\n    : stream_handle{token, std::move(ref)},\n      memo{mode_memo_handler()},\n      fd{desc},\n      rw{readable} {}\n\nUVW_INLINE std::shared_ptr<details::reset_mode_memo> tty_handle::mode_memo_handler() {\n    static std::weak_ptr<details::reset_mode_memo> weak;\n    auto shared = weak.lock();\n    if(!shared) { weak = shared = std::make_shared<details::reset_mode_memo>(); }\n    return shared;\n}\n\nUVW_INLINE int tty_handle::init() {\n    return leak_if(uv_tty_init(parent().raw(), raw(), fd, rw));\n}\n\nUVW_INLINE bool tty_handle::mode(tty_handle::tty_mode m) {\n    return (0 == uv_tty_set_mode(raw(), static_cast<uv_tty_mode_t>(m)));\n}\n\nUVW_INLINE bool tty_handle::reset_mode() noexcept {\n    return (0 == uv_tty_reset_mode());\n}\n\nUVW_INLINE win_size tty_handle::get_win_size() {\n    win_size size;\n\n    if(0 != uv_tty_get_winsize(raw(), &size.width, &size.height)) {\n        size.width = -1;\n        size.height = -1;\n    }\n\n    return size;\n}\n\nUVW_INLINE void tty_handle::vterm_state(tty_handle::tty_vtermstate s) const noexcept {\n    switch(s) {\n    case tty_vtermstate::SUPPORTED:\n        uv_tty_set_vterm_state(uv_tty_vtermstate_t::UV_TTY_SUPPORTED);\n        break;\n    case tty_vtermstate::UNSUPPORTED:\n        uv_tty_set_vterm_state(uv_tty_vtermstate_t::UV_TTY_UNSUPPORTED);\n        break;\n    }\n}\n\nUVW_INLINE tty_handle::tty_vtermstate tty_handle::vterm_state() const noexcept {\n    uv_tty_vtermstate_t state;\n    uv_tty_get_vterm_state(&state);\n    return tty_vtermstate{state};\n}\n\n} // namespace uvw\n\n// Path: src/uvw/udp.h\n#ifndef UVW_UDP_INCLUDE_H\n#define UVW_UDP_INCLUDE_H\n\n#include <cstddef>\n#include <memory>\n#include <string>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"enum.hpp\"\n#include \"handle.hpp\"\n#include \"request.hpp\"\n#include \"util.h\"\n\nnamespace uvw {\n\n/*! @brief Send event. */\nstruct send_event {};\n\n/*! @brief UDP data event. */\nstruct udp_data_event {\n    explicit udp_data_event(socket_address sndr, std::unique_ptr<char[]> buf, std::size_t len, bool part) noexcept;\n\n    std::unique_ptr<char[]> data; /*!< A bunch of data read on the stream. */\n    std::size_t length;           /*!< The amount of data read on the stream. */\n    socket_address sender;        /*!< A valid instance of socket_address. */\n    bool partial;                 /*!< True if the message was truncated, false otherwise. */\n};\n\nnamespace details {\n\nenum class uvw_udp_flags : std::underlying_type_t<uv_udp_flags> {\n    IPV6ONLY = UV_UDP_IPV6ONLY,\n    UDP_PARTIAL = UV_UDP_PARTIAL,\n    REUSEADDR = UV_UDP_REUSEADDR,\n    UDP_MMSG_CHUNK = UV_UDP_MMSG_CHUNK,\n    UDP_MMSG_FREE = UV_UDP_MMSG_FREE,\n    UDP_LINUX_RECVERR = UV_UDP_LINUX_RECVERR,\n    UDP_RECVMMSG = UV_UDP_RECVMMSG,\n    _UVW_ENUM = 0\n};\n\nenum class uvw_membership : std::underlying_type_t<uv_membership> {\n    LEAVE_GROUP = UV_LEAVE_GROUP,\n    JOIN_GROUP = UV_JOIN_GROUP\n};\n\nclass send_req final: public request<send_req, uv_udp_send_t, send_event> {\n    static void udp_send_callback(uv_udp_send_t *req, int status);\n\npublic:\n    using deleter = void (*)(char *);\n\n    send_req(loop::token token, std::shared_ptr<loop> parent, std::unique_ptr<char[], deleter> dt, unsigned int len);\n\n    int send(uv_udp_t *hndl, const struct sockaddr *addr);\n\nprivate:\n    std::unique_ptr<char[], deleter> data;\n    uv_buf_t buf;\n};\n\n} // namespace details\n\n/**\n * @brief The UDP handle.\n *\n * UDP handles encapsulate UDP communication for both clients and servers.<br/>\n * By default, _ipv4_ is used as a template parameter. The handle already\n * supports _IPv6_ out-of-the-box by using `uvw::ipv6`.\n *\n * To create an `udp_handle` through a `loop`, arguments follow:\n *\n * * An optional integer value that indicates optional flags used to initialize\n * the socket.\n *\n * See the official\n * [documentation](http://docs.libuv.org/en/v1.x/udp.html#c.uv_udp_init_ex)\n * for further details.\n */\nclass udp_handle final: public handle<udp_handle, uv_udp_t, send_event, udp_data_event> {\n    static void recv_callback(uv_udp_t *hndl, ssize_t nread, const uv_buf_t *buf, const sockaddr *addr, unsigned flags);\n\npublic:\n    using membership = details::uvw_membership;\n    using udp_flags = details::uvw_udp_flags;\n    using ipv4 = uvw::ipv4;\n    using ipv6 = uvw::ipv6;\n\n    explicit udp_handle(loop::token token, std::shared_ptr<loop> ref, unsigned int f = {});\n\n    /**\n     * @brief Initializes the handle. The actual socket is created lazily.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Opens an existing file descriptor or SOCKET as a UDP handle.\n     *\n     * The passed file descriptor or SOCKET is not checked for its type, but\n     * it\u2019s required that it represents a valid datagram socket.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/udp.html#c.uv_udp_open)\n     * for further details.\n     *\n     * @param socket A valid socket handle (either a file descriptor or a\n     * SOCKET).\n     *\n     * @return Underlying return value.\n     */\n    int open(os_socket_handle socket);\n\n    /**\n     * @brief Associates the handle to a remote address and port (either IPv4 or\n     * IPv6).\n     *\n     * Every message sent by this handle is automatically sent to the given\n     * destination.<br/>\n     * Trying to call this function on an already connected handle isn't\n     * allowed.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @return Underlying return value.\n     */\n    int connect(const sockaddr &addr);\n\n    /**\n     * @brief Associates the handle to a remote address and port (either IPv4 or\n     * IPv6).\n     *\n     * Every message sent by this handle is automatically sent to the given\n     * destination.<br/>\n     * Trying to call this function on an already connected handle isn't\n     * allowed.\n     *\n     * @param ip The address to which to bind.\n     * @param port The port to which to bind.\n     * @return Underlying return value.\n     */\n    int connect(const std::string &ip, unsigned int port);\n\n    /**\n     * @brief Associates the handle to a remote address and port (either IPv4 or\n     * IPv6).\n     *\n     * Every message sent by this handle is automatically sent to the given\n     * destination.<br/>\n     * Trying to call this function on an already connected handle isn't\n     * allowed.\n     *\n     * @param addr A valid instance of socket_address.\n     * @return Underlying return value.\n     */\n    int connect(socket_address addr);\n\n    /**\n     * @brief Disconnects the handle.\n     *\n     * Trying to disconnect a handle that is not connected isn't allowed.\n     *\n     * @return Underlying return value.\n     */\n    int disconnect();\n\n    /**\n     * @brief Gets the remote address to which the handle is connected, if any.\n     * @return A valid instance of socket_address, an empty one in case of\n     * errors.\n     */\n    socket_address peer() const noexcept;\n\n    /**\n     * @brief Binds the UDP handle to an IP address and port.\n     *\n     * Available flags are:\n     *\n     * * `udp_handle::udp_flags::IPV6ONLY`\n     * * `udp_handle::udp_flags::UDP_PARTIAL`\n     * * `udp_handle::udp_flags::REUSEADDR`\n     * * `udp_handle::udp_flags::UDP_MMSG_CHUNK`\n     * * `udp_handle::udp_flags::UDP_MMSG_FREE`\n     * * `udp_handle::udp_flags::UDP_LINUX_RECVERR`\n     * * `udp_handle::udp_flags::UDP_RECVMMSG`\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/udp.html#c.uv_udp_flags)\n     * for further details.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @param opts Optional additional flags.\n     * @return Underlying return value.\n     */\n    int bind(const sockaddr &addr, udp_flags opts = udp_flags::_UVW_ENUM);\n\n    /**\n     * @brief Binds the UDP handle to an IP address and port.\n     *\n     * Available flags are:\n     *\n     * * `udp_handle::udp_flags::IPV6ONLY`\n     * * `udp_handle::udp_flags::UDP_PARTIAL`\n     * * `udp_handle::udp_flags::REUSEADDR`\n     * * `udp_handle::udp_flags::UDP_MMSG_CHUNK`\n     * * `udp_handle::udp_flags::UDP_MMSG_FREE`\n     * * `udp_handle::udp_flags::UDP_LINUX_RECVERR`\n     * * `udp_handle::udp_flags::UDP_RECVMMSG`\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/udp.html#c.uv_udp_flags)\n     * for further details.\n     *\n     * @param ip The IP address to which to bind.\n     * @param port The port to which to bind.\n     * @param opts Optional additional flags.\n     * @return Underlying return value.\n     */\n    int bind(const std::string &ip, unsigned int port, udp_flags opts = udp_flags::_UVW_ENUM);\n\n    /**\n     * @brief Binds the UDP handle to an IP address and port.\n     *\n     * Available flags are:\n     *\n     * * `udp_handle::udp_flags::IPV6ONLY`\n     * * `udp_handle::udp_flags::UDP_PARTIAL`\n     * * `udp_handle::udp_flags::REUSEADDR`\n     * * `udp_handle::udp_flags::UDP_MMSG_CHUNK`\n     * * `udp_handle::udp_flags::UDP_MMSG_FREE`\n     * * `udp_handle::udp_flags::UDP_LINUX_RECVERR`\n     * * `udp_handle::udp_flags::UDP_RECVMMSG`\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/udp.html#c.uv_udp_flags)\n     * for further details.\n     *\n     * @param addr A valid instance of socket_address.\n     * @param opts Optional additional flags.\n     * @return Underlying return value.\n     */\n    int bind(socket_address addr, udp_flags opts = udp_flags::_UVW_ENUM);\n\n    /**\n     * @brief Get the local IP and port of the UDP handle.\n     * @return A valid instance of socket_address, an empty one in case of\n     * errors.\n     */\n    socket_address sock() const noexcept;\n\n    /**\n     * @brief Sets membership for a multicast address.\n     *\n     * Available values for `ms` are:\n     *\n     * * `udp_handle::membership::LEAVE_GROUP`\n     * * `udp_handle::membership::JOIN_GROUP`\n     *\n     * @param multicast Multicast address to set membership for.\n     * @param iface Interface address.\n     * @param ms Action to be performed.\n     * @return True in case of success, false otherwise.\n     */\n    bool multicast_membership(const std::string &multicast, const std::string &iface, membership ms);\n\n    /**\n     * @brief Sets IP multicast loop flag.\n     *\n     * This makes multicast packets loop back to local sockets.\n     *\n     * @param enable True to enable multicast loop, false otherwise.\n     * @return True in case of success, false otherwise.\n     */\n    bool multicast_loop(bool enable = true);\n\n    /**\n     * @brief Sets the multicast ttl.\n     * @param val A value in the range `[1, 255]`.\n     * @return True in case of success, false otherwise.\n     */\n    bool multicast_ttl(int val);\n\n    /**\n     * @brief Sets the multicast interface to send or receive data on.\n     * @param iface Interface address.\n     * @return True in case of success, false otherwise.\n     */\n    bool multicast_interface(const std::string &iface);\n\n    /**\n     * @brief Sets broadcast on or off.\n     * @param enable True to set broadcast on, false otherwise.\n     * @return True in case of success, false otherwise.\n     */\n    bool broadcast(bool enable = false);\n\n    /**\n     * @brief Sets the time to live.\n     * @param val A value in the range `[1, 255]`.\n     * @return True in case of success, false otherwise.\n     */\n    bool ttl(int val);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Note that if the socket has not previously been bound with `bind()`, it\n     * will be bound to `0.0.0.0` (the _all interfaces_ IPv4 address) and a\n     * random port number.\n     *\n     * The handle takes the ownership of the data and it is in charge of delete\n     * them.\n     *\n     * A send event will be emitted when the data have been sent.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int send(const sockaddr &addr, std::unique_ptr<char[]> data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Note that if the socket has not previously been bound with `bind()`, it\n     * will be bound to `0.0.0.0` (the _all interfaces_ IPv4 address) and a\n     * random port number.\n     *\n     * The handle takes the ownership of the data and it is in charge of delete\n     * them.\n     *\n     * A send event will be emitted when the data have been sent.\n     *\n     * @param ip The address to which to send data.\n     * @param port The port to which to send data.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int send(const std::string &ip, unsigned int port, std::unique_ptr<char[]> data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Note that if the socket has not previously been bound with `bind()`, it\n     * will be bound to `0.0.0.0` (the _all interfaces_ IPv4 address) and a\n     * random port number.\n     *\n     * The handle takes the ownership of the data and it is in charge of delete\n     * them.\n     *\n     * A send event will be emitted when the data have been sent.\n     *\n     * @param addr A valid instance of socket_address.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int send(socket_address addr, std::unique_ptr<char[]> data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Note that if the socket has not previously been bound with `bind()`, it\n     * will be bound to `0.0.0.0` (the _all interfaces_ IPv4 address) and a\n     * random port number.\n     *\n     * The handle doesn't take the ownership of the data. Be sure that their\n     * lifetime overcome the one of the request.\n     *\n     * A send event will be emitted when the data have been sent.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int send(const sockaddr &addr, char *data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Note that if the socket has not previously been bound with `bind()`, it\n     * will be bound to `0.0.0.0` (the _all interfaces_ IPv4 address) and a\n     * random port number.\n     *\n     * The handle doesn't take the ownership of the data. Be sure that their\n     * lifetime overcome the one of the request.\n     *\n     * A send event will be emitted when the data have been sent.\n     *\n     * @param ip The address to which to send data.\n     * @param port The port to which to send data.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int send(const std::string &ip, unsigned int port, char *data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Note that if the socket has not previously been bound with `bind()`, it\n     * will be bound to `0.0.0.0` (the _all interfaces_ IPv4 address) and a\n     * random port number.\n     *\n     * The handle doesn't take the ownership of the data. Be sure that their\n     * lifetime overcome the one of the request.\n     *\n     * A send event will be emitted when the data have been sent.\n     *\n     * @param addr A valid instance of socket_address.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int send(socket_address addr, char *data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Same as `send()`, but it won\u2019t queue a send request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_send(const sockaddr &addr, std::unique_ptr<char[]> data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Same as `send()`, but it won\u2019t queue a send request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param ip The address to which to send data.\n     * @param port The port to which to send data.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_send(const std::string &ip, unsigned int port, std::unique_ptr<char[]> data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Same as `send()`, but it won\u2019t queue a send request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param addr A valid instance of socket_address.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_send(socket_address addr, std::unique_ptr<char[]> data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Same as `send()`, but it won\u2019t queue a send request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_send(const sockaddr &addr, char *data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Same as `send()`, but it won\u2019t queue a send request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param ip The address to which to send data.\n     * @param port The port to which to send data.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_send(const std::string &ip, unsigned int port, char *data, unsigned int len);\n\n    /**\n     * @brief Sends data over the UDP socket.\n     *\n     * Same as `send()`, but it won\u2019t queue a send request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param addr A valid instance of socket_address.\n     * @param data The data to be sent.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_send(socket_address addr, char *data, unsigned int len);\n\n    /**\n     * @brief Prepares for receiving data.\n     *\n     * Note that if the socket has not previously been bound with `bind()`, it\n     * is bound to `0.0.0.0` (the _all interfaces_ IPv4 address) and a random\n     * port number.\n     *\n     * An UDP data event will be emitted when the handle receives data.\n     *\n     * @return Underlying return value.\n     */\n    int recv();\n\n    /**\n     * @brief Prepares for receiving data.\n     * @sa recv\n     * @tparam Alloc Custom allocation function.\n     * @return Underlying return value.\n     */\n    template<auto Alloc>\n    int recv() {\n        retur\nn uv_udp_recv_start(raw(), &details::common_alloc_callback<udp_handle, Alloc>, &recv_callback);\n    }\n\n    /**\n     * @brief Stops listening for incoming datagrams.\n     * @return Underlying return value.\n     */\n    int stop();\n\n    /**\n     * @brief Gets the number of bytes queued for sending.\n     *\n     * It strictly shows how much information is currently queued.\n     *\n     * @return Number of bytes queued for sending.\n     */\n    size_t send_queue_size() const noexcept;\n\n    /**\n     * @brief Number of send requests currently in the queue awaiting to be\n     * processed.\n     * @return Number of send requests currently in the queue.\n     */\n    size_t send_queue_count() const noexcept;\n\nprivate:\n    enum {\n        DEFAULT,\n        FLAGS\n    } tag{DEFAULT};\n\n    unsigned int flags{};\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"udp.cpp\"\n#endif\n\n#endif // UVW_UDP_INCLUDE_H\n\n// Path: src/uvw/udp.cpp\n#ifdef UVW_AS_LIB\n#    include \"udp.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE udp_data_event::udp_data_event(socket_address sndr, std::unique_ptr<char[]> buf, std::size_t len, bool part) noexcept\n    : data{std::move(buf)},\n      length{len},\n      sender{std::move(sndr)},\n      partial{part} {}\n\nUVW_INLINE void details::send_req::udp_send_callback(uv_udp_send_t *req, int status) {\n    if(auto ptr = reserve(req); status) {\n        ptr->publish(error_event{status});\n    } else {\n        ptr->publish(send_event{});\n    }\n}\n\nUVW_INLINE details::send_req::send_req(loop::token token, std::shared_ptr<loop> parent, std::unique_ptr<char[], deleter> dt, unsigned int len)\n    : request{token, std::move(parent)},\n      data{std::move(dt)},\n      buf{uv_buf_init(data.get(), len)} {}\n\nUVW_INLINE int details::send_req::send(uv_udp_t *hndl, const struct sockaddr *addr) {\n    return this->leak_if(uv_udp_send(raw(), hndl, &buf, 1, addr, &udp_send_callback));\n}\n\nUVW_INLINE void udp_handle::recv_callback(uv_udp_t *hndl, ssize_t nread, const uv_buf_t *buf, const sockaddr *addr, unsigned flags) {\n    udp_handle &udp = *(static_cast<udp_handle *>(hndl->data));\n    // data will be destroyed no matter of what the value of nread is\n    std::unique_ptr<char[]> data{buf->base};\n\n    if(nread > 0) {\n        // data available (can be truncated)\n        udp.publish(udp_data_event{details::sock_addr(*addr), std::move(data), static_cast<std::size_t>(nread), !(0 == (flags & UV_UDP_PARTIAL))});\n    } else if(nread == 0 && addr == nullptr) {\n        // no more data to be read, doing nothing is fine\n    } else if(nread == 0 && addr != nullptr) {\n        // empty udp packet\n        udp.publish(udp_data_event{details::sock_addr(*addr), std::move(data), static_cast<std::size_t>(nread), false});\n    } else {\n        // transmission error\n        udp.publish(error_event(nread));\n    }\n}\n\nUVW_INLINE udp_handle::udp_handle(loop::token token, std::shared_ptr<loop> ref, unsigned int f)\n    : handle{token, std::move(ref)}, tag{FLAGS}, flags{f} {}\n\nUVW_INLINE int udp_handle::init() {\n    if(tag == FLAGS) {\n        return leak_if(uv_udp_init_ex(parent().raw(), raw(), flags));\n    } else {\n        return leak_if(uv_udp_init(parent().raw(), raw()));\n    }\n}\n\nUVW_INLINE int udp_handle::open(os_socket_handle socket) {\n    return uv_udp_open(raw(), socket);\n}\n\nUVW_INLINE int udp_handle::connect(const sockaddr &addr) {\n    return uv_udp_connect(raw(), &addr);\n}\n\nUVW_INLINE int udp_handle::connect(const std::string &ip, unsigned int port) {\n    return connect(details::ip_addr(ip.data(), port));\n}\n\nUVW_INLINE int udp_handle::connect(socket_address addr) {\n    return connect(addr.ip, addr.port);\n}\n\nUVW_INLINE int udp_handle::disconnect() {\n    return uv_udp_connect(raw(), nullptr);\n}\n\nUVW_INLINE socket_address udp_handle::peer() const noexcept {\n    sockaddr_storage storage;\n    int len = sizeof(sockaddr_storage);\n    uv_udp_getpeername(raw(), reinterpret_cast<sockaddr *>(&storage), &len);\n    return details::sock_addr(storage);\n}\n\nUVW_INLINE int udp_handle::bind(const sockaddr &addr, udp_handle::udp_flags opts) {\n    return uv_udp_bind(raw(), &addr, static_cast<uv_udp_flags>(opts));\n}\n\nUVW_INLINE int udp_handle::bind(const std::string &ip, unsigned int port, udp_flags opts) {\n    return bind(details::ip_addr(ip.data(), port), opts);\n}\n\nUVW_INLINE int udp_handle::bind(socket_address addr, udp_flags opts) {\n    return bind(addr.ip, addr.port, opts);\n}\n\nUVW_INLINE socket_address udp_handle::sock() const noexcept {\n    sockaddr_storage storage;\n    int len = sizeof(sockaddr_storage);\n    uv_udp_getsockname(raw(), reinterpret_cast<sockaddr *>(&storage), &len);\n    return details::sock_addr(storage);\n}\n\nUVW_INLINE bool udp_handle::multicast_membership(const std::string &multicast, const std::string &iface, membership ms) {\n    return (0 == uv_udp_set_membership(raw(), multicast.data(), iface.data(), static_cast<uv_membership>(ms)));\n}\n\nUVW_INLINE bool udp_handle::multicast_loop(bool enable) {\n    return (0 == uv_udp_set_multicast_loop(raw(), enable));\n}\n\nUVW_INLINE bool udp_handle::multicast_ttl(int val) {\n    return (0 == uv_udp_set_multicast_ttl(raw(), val > 255 ? 255 : val));\n}\n\nUVW_INLINE bool udp_handle::multicast_interface(const std::string &iface) {\n    return (0 == uv_udp_set_multicast_interface(raw(), iface.data()));\n}\n\nUVW_INLINE bool udp_handle::broadcast(bool enable) {\n    return (0 == uv_udp_set_broadcast(raw(), enable));\n}\n\nUVW_INLINE bool udp_handle::ttl(int val) {\n    return (0 == uv_udp_set_ttl(raw(), val > 255 ? 255 : val));\n}\n\nUVW_INLINE int udp_handle::send(const sockaddr &addr, std::unique_ptr<char[]> data, unsigned int len) {\n    auto req = parent().resource<details::send_req>(std::unique_ptr<char[], details::send_req::deleter>{data.release(), [](char *ptr) { delete[] ptr; }}, len);\n\n    auto listener = [ptr = shared_from_this()](const auto &event, const auto &) {\n        ptr->publish(event);\n    };\n\n    req->on<error_event>(listener);\n    req->on<send_event>(listener);\n\n    return req->send(raw(), &addr);\n}\n\nUVW_INLINE int udp_handle::send(const std::string &ip, unsigned int port, std::unique_ptr<char[]> data, unsigned int len) {\n    return send(details::ip_addr(ip.data(), port), std::move(data), len);\n}\n\nUVW_INLINE int udp_handle::send(socket_address addr, std::unique_ptr<char[]> data, unsigned int len) {\n    return send(addr.ip, addr.port, std::move(data), len);\n}\n\nUVW_INLINE int udp_handle::send(const sockaddr &addr, char *data, unsigned int len) {\n    auto req = parent().resource<details::send_req>(std::unique_ptr<char[], details::send_req::deleter>{data, [](char *) {}}, len);\n\n    auto listener = [ptr = shared_from_this()](const auto &event, const auto &) {\n        ptr->publish(event);\n    };\n\n    req->on<error_event>(listener);\n    req->on<send_event>(listener);\n\n    return req->send(raw(), &addr);\n}\n\nUVW_INLINE int udp_handle::send(const std::string &ip, unsigned int port, char *data, unsigned int len) {\n    return send(details::ip_addr(ip.data(), port), data, len);\n}\n\nUVW_INLINE int udp_handle::send(socket_address addr, char *data, unsigned int len) {\n    return send(addr.ip, addr.port, data, len);\n}\n\nUVW_INLINE int udp_handle::try_send(const sockaddr &addr, std::unique_ptr<char[]> data, unsigned int len) {\n    uv_buf_t bufs[] = {uv_buf_init(data.get(), len)};\n    return uv_udp_try_send(raw(), bufs, 1, &addr);\n}\n\nUVW_INLINE int udp_handle::try_send(const std::string &ip, unsigned int port, std::unique_ptr<char[]> data, unsigned int len) {\n    return try_send(details::ip_addr(ip.data(), port), std::move(data), len);\n}\n\nUVW_INLINE int udp_handle::try_send(socket_address addr, std::unique_ptr<char[]> data, unsigned int len) {\n    return try_send(addr.ip, addr.port, std::move(data), len);\n}\n\nUVW_INLINE int udp_handle::try_send(const sockaddr &addr, char *data, unsigned int len) {\n    uv_buf_t bufs[] = {uv_buf_init(data, len)};\n    return uv_udp_try_send(raw(), bufs, 1, &addr);\n}\n\nUVW_INLINE int udp_handle::try_send(const std::string &ip, unsigned int port, char *data, unsigned int len) {\n    return try_send(details::ip_addr(ip.data(), port), data, len);\n}\n\nUVW_INLINE int udp_handle::try_send(socket_address addr, char *data, unsigned int len) {\n    return try_send(addr.ip, addr.port, data, len);\n}\n\nUVW_INLINE int udp_handle::recv() {\n    return uv_udp_recv_start(raw(), &details::common_alloc_callback, &recv_callback);\n}\n\nUVW_INLINE int udp_handle::stop() {\n    return uv_udp_recv_stop(raw());\n}\n\nUVW_INLINE size_t udp_handle::send_queue_size() const noexcept {\n    return uv_udp_get_send_queue_size(raw());\n}\n\nUVW_INLINE size_t udp_handle::send_queue_count() const noexcept {\n    return uv_udp_get_send_queue_count(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/signal.h\n#ifndef UVW_SIGNAL_INCLUDE_H\n#define UVW_SIGNAL_INCLUDE_H\n\n#include <uv.h>\n#include \"config.h\"\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Signal event. */\nstruct signal_event {\n    explicit signal_event(int sig) noexcept;\n\n    int signum; /*!< The signal being monitored by this handle. */\n};\n\n/**\n * @brief The signal handle.\n *\n * Signal handles implement Unix style signal handling on a per-event loop\n * bases.<br/>\n * Reception of some signals is emulated on Windows.\n *\n * To create a `signal_handle` through a `loop`, no arguments are required.\n *\n * See the official\n * [documentation](http://docs.libuv.org/en/v1.x/signal.html)\n * for further details.\n */\nclass signal_handle final: public handle<signal_handle, uv_signal_t, signal_event> {\n    static void start_callback(uv_signal_t *hndl, int signum);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * The handle will start emitting signal events when needed.\n     *\n     * @param signum The signal to be monitored.\n     * @return Underlying return value.\n     */\n    int start(int signum);\n\n    /**\n     * @brief Starts the handle.\n     *\n     * Same functionality as signal_handle::start but the signal handler is\n     * reset the moment the signal is received.\n     *\n     * @param signum The signal to be monitored.\n     * @return Underlying return value.\n     */\n    int one_shot(int signum);\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n\n    /**\n     * @brief Gets the signal being monitored.\n     * @return The signal being monitored.\n     */\n    int signal() const noexcept;\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"signal.cpp\"\n#endif\n\n#endif // UVW_SIGNAL_INCLUDE_H\n\n// Path: src/uvw/signal.cpp\n#ifdef UVW_AS_LIB\n#    include \"signal.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE signal_event::signal_event(int sig) noexcept\n    : signum{sig} {}\n\nUVW_INLINE void signal_handle::start_callback(uv_signal_t *hndl, int signum) {\n    signal_handle &signal = *(static_cast<signal_handle *>(hndl->data));\n    signal.publish(signal_event{signum});\n}\n\nUVW_INLINE int signal_handle::init() {\n    return leak_if(uv_signal_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int signal_handle::start(int signum) {\n    return uv_signal_start(raw(), &start_callback, signum);\n}\n\nUVW_INLINE int signal_handle::one_shot(int signum) {\n    return uv_signal_start_oneshot(raw(), &start_callback, signum);\n}\n\nUVW_INLINE int signal_handle::stop() {\n    return uv_signal_stop(raw());\n}\n\nUVW_INLINE int signal_handle::signal() const noexcept {\n    return raw()->signum;\n}\n\n} // namespace uvw\n\n// Path: src/uvw/tcp.h\n#ifndef UVW_TCP_INCLUDE_H\n#define UVW_TCP_INCLUDE_H\n\n#include <chrono>\n#include <memory>\n#include <string>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"enum.hpp\"\n#include \"request.hpp\"\n#include \"stream.h\"\n#include \"util.h\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_tcp_flags : std::underlying_type_t<uv_tcp_flags> {\n    IPV6ONLY = UV_TCP_IPV6ONLY,\n    _UVW_ENUM = 0\n};\n\n}\n\n/**\n * @brief The TCP handle.\n *\n * TCP handles are used to represent both TCP streams and servers.<br/>\n * By default, _ipv4_ is used as a template parameter. The handle already\n * supports _IPv6_ out-of-the-box by using `uvw::ipv6`.\n *\n * To create a `tcp_handle` through a `loop`, arguments follow:\n *\n * * An optional integer value that indicates the flags used to initialize\n * the socket.\n *\n * See the official\n * [documentation](http://docs.libuv.org/en/v1.x/tcp.html#c.uv_tcp_init_ex)\n * for further details.\n */\nclass tcp_handle final: public stream_handle<tcp_handle, uv_tcp_t> {\npublic:\n    using time = std::chrono::duration<unsigned int>;\n    using tcp_flags = details::uvw_tcp_flags;\n    using ipv4 = uvw::ipv4;\n    using ipv6 = uvw::ipv6;\n\n    explicit tcp_handle(loop::token token, std::shared_ptr<loop> ref, unsigned int f = {});\n\n    /**\n     * @brief Initializes the handle. No socket is created as of yet.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Opens an existing file descriptor or SOCKET as a TCP handle.\n     *\n     * The passed file descriptor or SOCKET is not checked for its type, but\n     * it\u2019s required that it represents a valid stream socket.\n     *\n     * @param socket A valid socket handle (either a file descriptor or a\n     * SOCKET).\n     *\n     * @return Underlying return value.\n     */\n    int open(os_socket_handle socket);\n\n    /**\n     * @brief Enables/Disables Nagle\u2019s algorithm.\n     * @param value True to enable it, false otherwise.\n     * @return True in case of success, false otherwise.\n     */\n    bool no_delay(bool value = false);\n\n    /**\n     * @brief Enables/Disables TCP keep-alive.\n     * @param enable True to enable it, false otherwise.\n     * @param val Initial delay in seconds (use\n     * `std::chrono::duration<unsigned int>`).\n     * @return True in case of success, false otherwise.\n     */\n    bool keep_alive(bool enable = false, time val = time{0});\n\n    /**\n     * @brief Enables/Disables simultaneous asynchronous accept requests.\n     *\n     * Enables/Disables simultaneous asynchronous accept requests that are\n     * queued by the operating system when listening for new TCP\n     * connections.<br/>\n     * This setting is used to tune a TCP server for the desired performance.\n     * Having simultaneous accepts can significantly improve the rate of\n     * accepting connections (which is why it is enabled by default) but may\n     * lead to uneven load distribution in multi-process setups.\n     *\n     * @param enable True to enable it, false otherwise.\n     * @return True in case of success, false otherwise.\n     */\n    bool simultaneous_accepts(bool enable = true);\n\n    /**\n     * @brief Binds the handle to an address and port.\n     *\n     * A successful call to this function does not guarantee that the call to\n     * `listen()` or `connect()` will work properly.\n     *\n     * Available flags are:\n     *\n     * * `tcp_handle::tcp_flags::IPV6ONLY`: it disables dual-stack support and\n     * only IPv6 is used.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @param opts Optional additional flags.\n     * @return Underlying return value.\n     */\n    int bind(const sockaddr &addr, tcp_flags opts = tcp_flags::_UVW_ENUM);\n\n    /**\n     * @brief Binds the handle to an address and port.\n     *\n     * A successful call to this function does not guarantee that the call to\n     * `listen()` or `connect()` will work properly.\n     *\n     * Available flags are:\n     *\n     * * `tcp_handle::tcp_flags::IPV6ONLY`: it disables dual-stack support and\n     * only IPv6 is used.\n     *\n     * @param ip The address to which to bind.\n     * @param port The port to which to bind.\n     * @param opts Optional additional flags.\n     * @return Underlying return value.\n     */\n    int bind(const std::string &ip, unsigned int port, tcp_flags opts = tcp_flags::_UVW_ENUM);\n\n    /**\n     * @brief Binds the handle to an address and port.\n     *\n     * A successful call to this function does not guarantee that the call to\n     * `listen()` or `connect()` will work properly.\n     *\n     * Available flags are:\n     *\n     * * `tcp_handle::tcp_flags::IPV6ONLY`: it disables dual-stack support and\n     * only IPv6 is used.\n     *\n     * @param addr A valid instance of socket_address.\n     * @param opts Optional additional flags.\n     * @return Underlying return value.\n     */\n    int bind(socket_address addr, tcp_flags opts = tcp_flags::_UVW_ENUM);\n\n    /**\n     * @brief Gets the current address to which the handle is bound.\n     * @return A valid instance of socket_address, an empty one in case of\n     * errors.\n     */\n    socket_address sock() const noexcept;\n\n    /**\n     * @brief Gets the address of the peer connected to the handle.\n     * @return A valid instance of socket_address, an empty one in case of\n     * errors.\n     */\n    socket_address peer() const noexcept;\n\n    /**\n     * @brief Establishes an IPv4 or IPv6 TCP connection.\n     *\n     * On Windows if the addr is initialized to point to an unspecified address\n     * (`0.0.0.0` or `::`) it will be changed to point to localhost. This is\n     * done to match the behavior of Linux systems.\n     *\n     * A connect event is emitted when the connection has been established.\n     *\n     * @param addr Initialized `sockaddr_in` or `sockaddr_in6` data structure.\n     * @return Underlying return value.\n     */\n    int connect(const sockaddr &addr);\n\n    /**\n     * @brief Establishes an IPv4 or IPv6 TCP connection.\n     *\n     * A connect event is emitted when the connection has been established.\n     *\n     * @param ip The address to which to bind.\n     * @param port The port to which to bind.\n     * @return Underlying return value.\n     */\n    int connect(const std::string &ip, unsigned int port);\n\n    /**\n     * @brief Establishes an IPv4 or IPv6 TCP connection.\n     *\n     * A connect event is emitted when the connection has been established.\n     *\n     * @param addr A valid instance of socket_address.\n     * @return Underlying return value.\n     */\n    int connect(socket_address addr);\n\n    /**\n     * @brief Resets a TCP connection by sending a RST packet.\n     *\n     * This is accomplished by setting the `SO_LINGER` socket option with a\n     * linger interval of zero and then calling `close`.<br/>\n     * Due to some platform inconsistencies, mixing of `shutdown` and\n     * `close_reset` calls is not allowed.\n     *\n     * A close event is emitted when the connection has been reset.\n     *\n     * @return Underlying return value.\n     */\n    int close_reset();\n\nprivate:\n    enum {\n        DEFAULT,\n        FLAGS\n    } tag;\n\n    unsigned int flags;\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"tcp.cpp\"\n#endif\n\n#endif // UVW_TCP_INCLUDE_H\n\n// Path: src/uvw/tcp.cpp\n#ifdef UVW_AS_LIB\n#    include \"tcp.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE tcp_handle::tcp_handle(loop::token token, std::shared_ptr<loop> ref, unsigned int f)\n    : stream_handle{token, std::move(ref)}, tag{f ? FLAGS : DEFAULT}, flags{f} {}\n\nUVW_INLINE int tcp_handle::init() {\n    if(tag == FLAGS) {\n        return leak_if(uv_tcp_init_ex(parent().raw(), raw(), flags));\n    } else {\n        return leak_if(uv_tcp_init(parent().raw(), raw()));\n    }\n}\n\nUVW_INLINE int tcp_handle::open(os_socket_handle socket) {\n    return uv_tcp_open(raw(), socket);\n}\n\nUVW_INLINE bool tcp_handle::no_delay(bool value) {\n    return (0 == uv_tcp_nodelay(raw(), value));\n}\n\nUVW_INLINE bool tcp_handle::keep_alive(bool enable, tcp_handle::time val) {\n    return (0 == uv_tcp_keepalive(raw(), enable, val.count()));\n}\n\nUVW_INLINE bool tcp_handle::simultaneous_accepts(bool enable) {\n    return (0 == uv_tcp_simultaneous_accepts(raw(), enable));\n}\n\nUVW_INLINE int tcp_handle::bind(const sockaddr &addr, tcp_flags opts) {\n    return uv_tcp_bind(raw(), &addr, static_cast<uv_tcp_flags>(opts));\n}\n\nUVW_INLINE int tcp_handle::bind(const std::string &ip, unsigned int port, tcp_flags opts) {\n    return bind(details::ip_addr(ip.data(), port), opts);\n}\n\nUVW_INLINE int tcp_handle::bind(socket_address addr, tcp_flags opts) {\n    return bind(addr.ip, addr.port, opts);\n}\n\nUVW_INLINE socket_address tcp_handle::sock() const noexcept {\n    sockaddr_storage storage;\n    int len = sizeof(sockaddr_storage);\n    uv_tcp_getsockname(raw(), reinterpret_cast<sockaddr *>(&storage), &len);\n    return details::sock_addr(storage);\n}\n\nUVW_INLINE socket_address tcp_handle::peer() const noexcept {\n    sockaddr_storage storage;\n    int len = sizeof(sockaddr_storage);\n    uv_tcp_getpeername(raw(), reinterpret_cast<sockaddr *>(&storage), &len);\n    return details::sock_addr(storage);\n}\n\nUVW_INLINE int tcp_handle::connect(const std::string &ip, unsigned int port) {\n    return connect(details::ip_addr(ip.data(), port));\n}\n\nUVW_INLINE int tcp_handle::connect(socket_address addr) {\n    return connect(addr.ip, addr.port);\n}\n\nUVW_INLINE int tcp_handle::connect(const sockaddr &addr) {\n    auto listener = [ptr = shared_from_this()](const auto &event, const auto &) {\n        ptr->publish(event);\n    };\n\n    auto req = parent().resource<details::connect_req>();\n    req->on<error_event>(listener);\n    req->on<connect_event>(listener);\n\n    return req->connect(&uv_tcp_connect, raw(), &addr);\n}\n\nUVW_INLINE int tcp_handle::close_reset() {\n    return uv_tcp_close_reset(raw(), &this->close_callback);\n}\n\n} // namespace uvw\n\n// Path: src/uvw/process.h\n#ifndef UVW_PROCESS_INCLUDE_H\n#define UVW_PROCESS_INCLUDE_H\n\n#include <memory>\n#include <string>\n#include <utility>\n#include <vector>\n#include <uv.h>\n#include \"config.h\"\n#include \"enum.hpp\"\n#include \"handle.hpp\"\n#include \"loop.h\"\n#include \"stream.h\"\n#include \"util.h\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_process_flags : std::underlying_type_t<uv_process_flags> {\n    SETUID = UV_PROCESS_SETUID,\n    SETGID = UV_PROCESS_SETGID,\n    WINDOWS_VERBATIM_ARGUMENTS = UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS,\n    DETACHED = UV_PROCESS_DETACHED,\n    WINDOWS_HIDE = UV_PROCESS_WINDOWS_HIDE,\n    WINDOWS_HIDE_CONSOLE = UV_PROCESS_WINDOWS_HIDE_CONSOLE,\n    WINDOWS_HIDE_GUI = UV_PROCESS_WINDOWS_HIDE_GUI,\n    _UVW_ENUM = 0\n};\n\nenum class uvw_stdio_flags : std::underlying_type_t<uv_stdio_flags> {\n    IGNORE_STREAM = UV_IGNORE,\n    CREATE_PIPE = UV_CREATE_PIPE,\n    INHERIT_FD = UV_INHERIT_FD,\n    INHERIT_STREAM = UV_INHERIT_STREAM,\n    READABLE_PIPE = UV_READABLE_PIPE,\n    WRITABLE_PIPE = UV_WRITABLE_PIPE,\n    OVERLAPPED_PIPE = UV_OVERLAPPED_PIPE,\n    _UVW_ENUM = 0\n};\n\n} // namespace details\n\n/*! @brief Exit event. */\nstruct exit_event {\n    explicit exit_event(int64_t code, int sig) noexcept;\n\n    int64_t status; /*!< The exit status. */\n    int signal;     /*!< The signal that caused the process to terminate, if any. */\n};\n\n/**\n * @brief The process handle.\n *\n * Process handles will spawn a new process and allow the user to control it and\n * establish communication channels with it using streams.\n */\nclass process_handle final: public handle<process_handle, uv_process_t, exit_event> {\n    static void exit_callback(uv_process_t *hndl, int64_t exit_status, int term_signal);\n\npublic:\n    using process_flags = details::uvw_process_flags;\n    using stdio_flags = details::uvw_stdio_flags;\n\n    process_handle(loop::token token, std::shared_ptr<loop> ref);\n\n    /**\n     * @brief Disables inheritance for file descriptors/handles.\n     *\n     * Disables inheritance for file descriptors/handles that this process\n     * inherited from its parent. The effect is that child processes spawned by\n     * this process don\u2019t accidentally inherit these handles.<br/>\n     * It is recommended to call this function as early in your program as\n     * possible, before the inherited file descriptors can be closed or\n     * duplicated.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/process.html#c.uv_disable_stdio_inheritance)\n     * for further details.\n     */\n    static void disable_stdio_inheritance() noexcept;\n\n    /**\n     * @brief kill Sends the specified signal to the given PID.\n     * @param pid A valid process id.\n     * @param signum A valid signal identifier.\n     * @return True in case of success, false otherwise.\n     */\n    static bool kill(int pid, int signum) noexcept;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief spawn Starts the process.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/process.html)\n     * for further details.\n     *\n     * @param file Path pointing to the program to be executed.\n     * @param args Command line arguments.\n     * @param env Optional environment for the new process.\n     * @return Underlying return value.\n     */\n    int spawn(const char *file, char **args, char **env = nullptr);\n\n    /**\n     * @brief Sends the specified signal to the internal process handle.\n     * @param signum A valid signal identifier.\n     * @return Underlying return value.\n     */\n    int kill(int signum);\n\n    /**\n     * @brief Gets the PID of the spawned process.\n     *\n     * It\u2019s set after calling `spawn()`.\n     *\n     * @return The PID of the spawned process.\n     */\n    int pid() noexcept;\n\n    /**\n     * @brief Sets the current working directory for the subprocess.\n     * @param path The working directory to be used when `spawn()` is invoked.\n     * @return A reference to this process handle.\n     */\n    process_handle &cwd(const std::string &path) noexcept;\n\n    /**\n     * @brief Sets flags that control how `spawn()` behaves.\n     *\n     * Available flags are:\n     *\n     * * `process_handle::process_flags::SETUID`\n...\n", "needle_token_start": 7238, "needle_token_end": 7289, "code_context_ntokens": 16083, "output": ["```cpp\nUVW_INLINE int udp_handle::recv() {\n    return uv_udp_recv_start(raw(), &details::common_alloc_callback, &recv_callback);\n}\n```"]}
{"repo": "skypjack/uvw", "name": "configure", "language": "cpp", "path": "src/uvw/loop.h", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To set additional options for an event loop, typically before it starts running, to modify its behavior based on specific needs such as blocking signals or tracking idle time.\n2. **Input**: A flag representing the option to be set and any necessary arguments that the specific option might require.\n3. **Output**: An integer indicating the success or failure of setting the option.\n4. **Procedure**: The method takes a flag and arguments, then applies these settings to the event loop using an underlying library call. The settings must be applied before the event loop starts processing events unless otherwise specified.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/util.h\n#ifndef UVW_UTIL_INCLUDE_H\n#define UVW_UTIL_INCLUDE_H\n\n#include <array>\n#include <cstddef>\n#include <cstdint>\n#include <memory>\n#include <string>\n#include <string_view>\n#include <type_traits>\n#include <utility>\n#include <vector>\n#include <uv.h>\n#include \"config.h\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_handle_type : std::underlying_type_t<uv_handle_type> {\n    UNKNOWN = UV_UNKNOWN_HANDLE,\n    ASYNC = UV_ASYNC,\n    CHECK = UV_CHECK,\n    FS_EVENT = UV_FS_EVENT,\n    FS_POLL = UV_FS_POLL,\n    HANDLE = UV_HANDLE,\n    IDLE = UV_IDLE,\n    PIPE = UV_NAMED_PIPE,\n    POLL = UV_POLL,\n    PREPARE = UV_PREPARE,\n    PROCESS = UV_PROCESS,\n    STREAM = UV_STREAM,\n    TCP = UV_TCP,\n    TIMER = UV_TIMER,\n    TTY = UV_TTY,\n    UDP = UV_UDP,\n    SIGNAL = UV_SIGNAL,\n    FILE = UV_FILE\n};\n\nenum class uvw_clock_id : std::underlying_type_t<uv_clock_id> {\n    MONOTONIC = UV_CLOCK_MONOTONIC,\n    REALTIME = UV_CLOCK_REALTIME\n};\n\ntemplate<typename T>\nstruct uv_type_wrapper {\n    using Type = T;\n\n    constexpr uv_type_wrapper()\n        : value{} {}\n\n    constexpr uv_type_wrapper(Type val)\n        : value{val} {}\n\n    constexpr operator Type() const noexcept {\n        return value;\n    }\n\n    bool operator==(uv_type_wrapper other) const noexcept {\n        return value == other.value;\n    }\n\nprivate:\n    const Type value;\n};\n\ntemplate<typename T>\nbool operator==(uv_type_wrapper<T> lhs, uv_type_wrapper<T> rhs) {\n    return !(lhs == rhs);\n}\n\n} // namespace details\n\n/**\n * @brief Windows size representation.\n */\nstruct win_size {\n    int width;  /*!< The _width_ of the given window. */\n    int height; /*!< The _height_ of the given window. */\n};\n\nusing handle_type = details::uvw_handle_type;                     /*!< The type of a handle. */\nusing handle_category = details::uv_type_wrapper<uv_handle_type>; /*!< Utility class that wraps an internal handle type. */\nusing file_handle = details::uv_type_wrapper<uv_file>;            /*!< Utility class that wraps an internal file handle. */\nusing os_socket_handle = details::uv_type_wrapper<uv_os_sock_t>;  /*!< Utility class that wraps an os socket handle. */\nusing os_file_descriptor = details::uv_type_wrapper<uv_os_fd_t>;  /*!< Utility class that wraps an os file descriptor. */\nusing pid_type = details::uv_type_wrapper<uv_pid_t>;              /*!< Utility class that wraps a cross platform representation of a pid. */\nusing clock_id = details::uvw_clock_id;                           /*!< Utility class that wraps a clock source. */\n\nconstexpr file_handle std_in{0};  /*!< Placeholder for stdin descriptor. */\nconstexpr file_handle std_out{1}; /*!< Placeholder for stdout descriptor. */\nconstexpr file_handle std_err{2}; /*!< Placeholder for stderr descriptor. */\n\nusing time_spec = uv_timespec_t; /*!< Library equivalent for uv_timespec_t. */\nusing file_info = uv_stat_t;     /*!< Library equivalent for uv_stat_t. */\nusing fs_info = uv_statfs_t;     /*!< Library equivalent for uv_statfs_t. */\nusing uid_type = uv_uid_t;       /*!< Library equivalent for uv_uid_t. */\nusing gid_type = uv_gid_t;       /*!< Library equivalent for uv_gid_t. */\n\nusing timeval = uv_timeval_t;       /*!< Library equivalent for uv_timeval_t. */\nusing timeval64 = uv_timeval64_t;   /*!< Library equivalent for uv_timeval64_t. */\nusing timespec64 = uv_timespec64_t; /*!< Library equivalent for uv_timespec64_t. */\nusing resource_usage = uv_rusage_t; /*!< Library equivalent for uv_rusage_t. */\n\n/**\n * @brief Utility class.\n *\n * This class can be used to query the subset of the password file entry for the\n * current effective uid (not the real uid).\n *\n * \\sa utilities::passwd\n */\nstruct passwd_info {\n    passwd_info(std::shared_ptr<uv_passwd_t> pwd);\n\n    /**\n     * @brief Gets the username.\n     * @return The username of the current effective uid (not the real uid).\n     */\n    std::string username() const noexcept;\n\n    /**\n     * @brief Gets the uid.\n     * @return The current effective uid (not the real uid).\n     */\n    decltype(uv_passwd_t::uid) uid() const noexcept;\n\n    /**\n     * @brief Gets the gid.\n     * @return The gid of the current effective uid (not the real uid).\n     */\n    decltype(uv_passwd_t::gid) gid() const noexcept;\n\n    /**\n     * @brief Gets the shell.\n     * @return The shell of the current effective uid (not the real uid).\n     */\n    std::string shell() const noexcept;\n\n    /**\n     * @brief Gets the homedir.\n     * @return The homedir of the current effective uid (not the real uid).\n     */\n    std::string homedir() const noexcept;\n\n    /**\n     * @brief Checks if the instance contains valid data.\n     * @return True if data are all valid, false otherwise.\n     */\n    operator bool() const noexcept;\n\nprivate:\n    std::shared_ptr<uv_passwd_t> value;\n};\n\n/**\n * @brief Utility class.\n *\n * This class can be used to get name and information about the current kernel.\n * The populated data includes the operating system name, release, version, and\n * machine.\n *\n * \\sa utilities::uname\n */\nstruct uts_name {\n    uts_name(std::shared_ptr<uv_utsname_t> init);\n\n    /**\n     * @brief Gets the operating system name (like \"Linux\").\n     * @return The operating system name.\n     */\n    std::string sysname() const noexcept;\n\n    /**\n     * @brief Gets the operating system release (like \"2.6.28\").\n     * @return The operating system release.\n     */\n    std::string release() const noexcept;\n\n    /**\n     * @brief Gets the operating system version.\n     * @return The operating system version\n     */\n    std::string version() const noexcept;\n\n    /**\n     * @brief Gets the hardware identifier.\n     * @return The hardware identifier.\n     */\n    std::string machine() const noexcept;\n\nprivate:\n    std::shared_ptr<uv_utsname_t> uname;\n};\n\n/**\n * @brief The IPv4 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv4 {};\n\n/**\n * @brief The IPv6 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv6 {};\n\n/**\n * @brief Address representation.\n */\nstruct socket_address {\n    std::string ip;    /*!< Either an IPv4 or an IPv6. */\n    unsigned int port; /*!< A valid service identifier. */\n};\n\n/**\n * \\brief CPU information.\n */\nstruct cpu_info {\n    using cpu_time = decltype(uv_cpu_info_t::cpu_times);\n\n    std::string model; /*!< The model of the CPU. */\n    int speed;         /*!< The frequency of the CPU. */\n\n    /**\n     * @brief CPU times.\n     *\n     * It is built up of the following data members: `user`, `nice`, `sys`,\n     * `idle`, `irq`, all of them having type `uint64_t`.\n     */\n    cpu_time times;\n};\n\n/**\n * \\brief Interface address.\n */\nstruct interface_address {\n    std::string name;       /*!< The name of the interface (as an example _eth0_). */\n    char physical[6];       /*!< The physical address. */\n    bool internal;          /*!< True if it is an internal interface (as an example _loopback_), false otherwise. */\n    socket_address address; /*!< The address of the given interface. */\n    socket_address netmask; /*!< The netmask of the given interface. */\n};\n\nnamespace details {\n\nstatic constexpr std::size_t DEFAULT_SIZE = 128;\n\ntemplate<typename F, typename... Args>\nstd::string try_read(F &&f, Args &&...args) noexcept {\n    std::size_t size = DEFAULT_SIZE;\n    char buf[DEFAULT_SIZE];\n    std::string str{};\n    auto err = std::forward<F>(f)(args..., buf, &size);\n\n    if(UV_ENOBUFS == err) {\n        std::unique_ptr<char[]> data{new char[size]};\n        err = std::forward<F>(f)(args..., data.get(), &size);\n\n        if(0 == err) {\n            str = data.get();\n        }\n    } else if(0 == err) {\n        str.assign(buf, size);\n    }\n\n    return str;\n}\n\nvoid common_alloc_callback(uv_handle_t *, std::size_t suggested, uv_buf_t *buf);\n\ntemplate<typename Type, auto Alloc>\nvoid common_alloc_callback(uv_handle_t *handle, std::size_t suggested, uv_buf_t *buf) {\n    auto [alloc, size] = Alloc(*static_cast<const Type *>(handle->data), suggested);\n    *buf = uv_buf_init(alloc, static_cast<unsigned int>(size));\n}\n\nsockaddr ip_addr(const char *addr, unsigned int port);\nsocket_address sock_addr(const sockaddr_in &addr);\nsocket_address sock_addr(const sockaddr_in6 &addr);\nsocket_address sock_addr(const sockaddr &addr);\nsocket_address sock_addr(const sockaddr_storage &storage);\n\n} // namespace details\n\n/**\n * @brief Miscellaneous utilities.\n *\n * Miscellaneous functions that don\u2019t really belong to any other class.\n */\nstruct utilities {\n    using malloc_func_type = void *(*)(size_t);\n    using realloc_func_type = void *(*)(void *, size_t);\n    using calloc_func_type = void *(*)(size_t, size_t);\n    using free_func_type = void (*)(void *);\n\n    /**\n     * @brief OS dedicated utilities.\n     */\n    struct os {\n        /**\n         * @brief Returns the current process id.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_getpid)\n         * for further details.\n         *\n         * @return The current process id.\n         */\n        static pid_type pid() noexcept;\n\n        /**\n         * @brief Returns the parent process id.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_getppid)\n         * for further details.\n         *\n         * @return The parent process id.\n         */\n        static pid_type ppid() noexcept;\n\n        /**\n         * @brief Gets the current user's home directory.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_homedir)\n         * for further details.\n         *\n         * @return The current user's home directory, an empty string in case of\n         * errors.\n         */\n        static std::string homedir() noexcept;\n\n        /**\n         * @brief Gets the temp directory.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_tmpdir)\n         * for further details.\n         *\n         * @return The temp directory, an empty string in case of errors.\n         */\n        static std::string tmpdir() noexcept;\n\n        /**\n         * @brief Retrieves an environment variable.\n         * @param name The name of the variable to be retrieved.\n         * @return The value of the environment variable, an empty string in\n         * case of errors.\n         */\n        static std::string env(const std::string &name) noexcept;\n\n        /**\n         * @brief Creates, updates or deletes an environment variable.\n         * @param name The name of the variable to be updated.\n         * @param value The value to be used for the variable (an empty string\n         * to unset it).\n         * @return True in case of success, false otherwise.\n         */\n        static bool env(const std::string &name, const std::string &value) noexcept;\n\n        /**\n         * @brief Retrieves all environment variables and iterates them.\n         *\n         * Environment variables are passed one at a time to the callback in the\n         * form of `std::string_view`s.<br/>\n         * The signature of the function call operator must be such that it\n         * accepts two parameters, the name and the value of the i-th variable.\n         *\n         * @tparam Func Type of a function object to which to pass environment\n         * variables.\n         * @param func A function object to which to pass environment variables.\n         * @return True in case of success, false otherwise.\n         */\n        template<typename Func>\n        static std::enable_if_t<std::is_invocable_v<Func, std::string_view, std::string_view>, bool>\n        env(Func func) noexcept {\n            uv_env_item_t *items = nullptr;\n            int count{};\n\n            const bool ret = (uv_os_environ(&items, &count) == 0);\n\n            if(ret) {\n                for(int pos = 0; pos < count; ++pos) {\n                    func(std::string_view{items[pos].name}, std::string_view{items[pos].value});\n                }\n\n                uv_os_free_environ(items, count);\n            }\n\n            return ret;\n        }\n\n        /**\n         * @brief Returns the hostname.\n         * @return The hostname, an empty string in case of errors.\n         */\n        static std::string hostname() noexcept;\n\n        /**\n         * @brief Gets name and information about the current kernel.\n         *\n         * This function can be used to get name and information about the\n         * current kernel. The populated data includes the operating system\n         * name, release, version, and machine.\n         *\n         * @return Name and information about the current kernel.\n         */\n        static uts_name uname() noexcept;\n\n        /**\n         * @brief Gets a subset of the password file entry.\n         *\n         * This function can be used to get the subset of the password file\n         * entry for the current effective uid (not the real uid).\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_get_passwd)\n         * for further details.\n         *\n         * @return The accessible subset of the password file entry.\n         */\n        static passwd_info passwd() noexcept;\n\n        /**\n         * @brief Retrieves the scheduling priority of a process.\n         *\n         * The returned value is between -20 (high priority) and 19 (low priority).\n         * A value that is out of range is returned in case of errors.\n         *\n         * @note\n         * On Windows, the result won't equal necessarily the exact value of the\n         * priority because of a mapping to a Windows priority class.\n         *\n         * @param pid A valid process id.\n         * @return The scheduling priority of the process.\n         */\n        static int priority(pid_type pid);\n\n        /**\n         * @brief Sets the scheduling priority of a process.\n         *\n         * The returned value range is between -20 (high priority) and 19 (low\n         * priority).\n         *\n         * @note\n         * On Windows, the priority is mapped to a Windows priority class. When\n         * retrieving the process priority, the result won't equal necessarily the\n         * exact value of the priority.\n         *\n         * @param pid A valid process id.\n         * @param prio The scheduling priority to set to the process.\n         * @return True in case of success, false otherwise.\n         */\n        static bool priority(pid_type pid, int prio);\n    };\n\n    /**\n     * @brief Gets the type of the handle given a category.\n     * @param category A properly initialized handle category.\n     * @return The actual type of the handle as defined by handle_type\n     */\n    static handle_type guess_handle(handle_category category) noexcept;\n\n    /**\n     * @brief Gets the type of the stream to be used with the given descriptor.\n     *\n     * Returns the type of stream that should be used with a given file\n     * descriptor.<br/>\n     * Usually this will be used during initialization to guess the type of the\n     * stdio streams.\n     *\n     * @param file A valid descriptor.\n     * @return One of the following types:\n     *\n     * * `handle_type::UNKNOWN`\n     * * `handle_type::PIPE`\n     * * `handle_type::TCP`\n     * * `handle_type::TTY`\n     * * `handle_type::UDP`\n     * * `handle_type::FILE`\n     */\n    static handle_type guess_handle(file_handle file) noexcept;\n\n    /** @brief Gets information about the CPUs on the system.\n     *\n     * This function can be used to query the underlying system and get a set of\n     * descriptors of all the available CPUs.\n     *\n     * @return A set of descriptors of all the available CPUs.\n     */\n    static std::vector<cpu_info> cpu() noexcept;\n\n    /**\n     * @brief Gets a set of descriptors of all the available interfaces.\n     *\n     * This function can be used to query the underlying system and get a set of\n     * descriptors of all the available interfaces, either internal or not.\n     *\n     * @return A set of descriptors of all the available interfaces.\n     */\n    static std::vector<interface_address> interface_addresses() noexcept;\n\n    /**\n     * @brief IPv6-capable implementation of\n     * [if_indextoname](https://linux.die.net/man/3/if_indextoname).\n     *\n     * Mapping between network interface names and indexes.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_if_indextoname)\n     * for further details.\n     *\n     * @param index Network interface index.\n     * @return Network interface name.\n     */\n    static std::string index_to_name(unsigned int index) noexcept;\n\n    /**\n     * @brief Retrieves a network interface identifier.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_if_indextoiid)\n     * for further details.\n     *\n     * @param index Network interface index.\n     * @return Network interface identifier.\n     */\n    static std::string index_to_iid(unsigned int index) noexcept;\n\n    /**\n     * @brief Override the use of some standard library\u2019s functions.\n     *\n     * Override the use of the standard library\u2019s memory allocation\n     * functions.<br/>\n     * This method must be invoked before any other `uvw` function is called or\n     * after all resources have been freed and thus the underlying library\n     * doesn\u2019t reference any allocated memory chunk.\n     *\n     * If any of the function pointers is _null_, the invokation will fail.\n     *\n     * @note\n     * There is no protection against changing the allocator multiple times. If\n     * the user changes it they are responsible for making sure the allocator is\n     * changed while no memory was allocated with the previous allocator, or\n     * that they are compatible.\n     *\n     * @param malloc_func Replacement function for _malloc_.\n     * @param realloc_func Replacement function for _realloc_.\n     * @param calloc_func Replacement function for _calloc_.\n     * @param free_func Replacement function for _free_.\n     * @return True in case of success, false otherwise.\n     */\n    static bool replace_allocator(malloc_func_type malloc_func, realloc_func_type realloc_func, calloc_func_type calloc_func, free_func_type free_func) noexcept;\n\n    /**\n     * @brief Gets the load average.\n     * @return `[0,0,0]` on Windows (not available), the load average otherwise.\n     */\n    static std::array<double, 3> load_average() noexcept;\n\n    /**\n     * @brief Store the program arguments.\n     *\n     * Required for getting / setting the process title.\n     *\n     * @return Arguments that haven't been consumed internally.\n     */\n    static char **setup_args(int argc, char **argv);\n\n    /**\n     * @brief Gets the title of the current process.\n     * @return The process title.\n     */\n    static std::string process_title();\n\n    /**\n     * @brief Sets the current process title.\n     * @param title The process title to be set.\n     * @return True in case of success, false otherwise.\n     */\n    static bool process_title(const std::string &title);\n\n    /**\n     * @brief Gets memory information (in bytes).\n     * @return Memory information.\n     */\n    static uint64_t total_memory() noexcept;\n\n    /**\n     * @brief Gets the amount of memory available to the process (in bytes).\n     *\n     * Gets the amount of memory available to the process based on limits\n     * imposed by the OS. If there is no such constraint, or the constraint is\n     * unknown, `0` is returned.<br/>\n     * Note that it is not unusual for this value to be less than or greater\n     * than `totalMemory`.\n     *\n     * @return Amount of memory available to the process.\n     */\n    static uint64_t constrained_memory() noexcept;\n\n    /**\n     * @brief Gets the amount of free memory still available to the process.\n     * @return Amount of free memory still available to the process (in bytes).\n     */\n    static uint64_t available_memory() noexcept;\n\n    /**\n     * @brief Gets the current system uptime.\n     * @return The current system uptime or 0 in case of errors.\n     */\n    static double uptime() noexcept;\n\n    /**\n     * @brief Gets the resource usage measures for the current process.\n     * @return Resource usage measures, zeroes-filled object in case of errors.\n     */\n    static resource_usage rusage() noexcept;\n\n    /**\n     * @brief Gets the current system time from a high-resolution clock source.\n     * @param source Clock source, either real-time or monotonic.\n     * @return Current system time from the given high-resolution clock source.\n     */\n    static timespec64 gettime(clock_id source) noexcept;\n\n    /**\n     * @brief Gets the current high-resolution real time.\n     *\n     * The time is expressed in nanoseconds. It is relative to an arbitrary time\n...\n// Path: src/uvw/loop.cpp\n#ifdef UVW_AS_LIB\n#    include \"loop.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE loop::loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept\n    : uv_loop{std::move(ptr)} {}\n\nUVW_INLINE std::shared_ptr<loop> loop::create() {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{new uv_loop_t, [](uv_loop_t *l) { delete l; }};\n    auto curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n\n    if(uv_loop_init(curr->uv_loop.get())) {\n        curr = nullptr;\n    }\n\n    return curr;\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::create(uv_loop_t *res) {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{res, [](uv_loop_t *) {}};\n    return std::shared_ptr<loop>{new loop{std::move(ptr)}};\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::get_default() {\n    static std::weak_ptr<loop> ref;\n    std::shared_ptr<loop> curr;\n\n    if(ref.expired()) {\n        auto def = uv_default_loop();\n\n        if(def) {\n            auto ptr = std::unique_ptr<uv_loop_t, deleter>(def, [](uv_loop_t *) {});\n            curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n        }\n\n        ref = curr;\n    } else {\n        curr = ref.lock();\n    }\n\n    return curr;\n}\n\nUVW_INLINE loop::~loop() noexcept {\n    if(uv_loop) {\n        close();\n    }\n}\n\nUVW_INLINE int loop::close() {\n    int ret = 0;\n\n    if(uv_loop) {\n        ret = uv_loop_close(uv_loop.get());\n        uv_loop.reset();\n    }\n\n    return ret;\n}\n\nUVW_INLINE int loop::run(run_mode mode) noexcept {\n    return uv_run(uv_loop.get(), static_cast<uv_run_mode>(mode));\n}\n\nUVW_INLINE bool loop::alive() const noexcept {\n    return !!uv_loop_alive(uv_loop.get());\n}\n\nUVW_INLINE void loop::stop() noexcept {\n    uv_stop(uv_loop.get());\n}\n\nUVW_INLINE int loop::descriptor() const noexcept {\n    return uv_backend_fd(uv_loop.get());\n}\n\nUVW_INLINE std::pair<bool, loop::time> loop::timeout() const noexcept {\n    auto to = uv_backend_timeout(uv_loop.get());\n    return std::make_pair(to == -1, time{to});\n}\n\nUVW_INLINE loop::time loop::idle_time() const noexcept {\n    return time{uv_metrics_idle_time(uv_loop.get())};\n}\n\nUVW_INLINE metrics_type loop::metrics() const noexcept {\n    metrics_type res{};\n    uv_metrics_info(uv_loop.get(), &res);\n    return res;\n}\n\nUVW_INLINE loop::time loop::now() const noexcept {\n    return time{uv_now(uv_loop.get())};\n}\n\nUVW_INLINE void loop::update() const noexcept {\n    return uv_update_time(uv_loop.get());\n}\n\nUVW_INLINE int loop::fork() noexcept {\n    return uv_loop_fork(uv_loop.get());\n}\n\nUVW_INLINE void loop::data(std::shared_ptr<void> ud) {\n    user_data = std::move(ud);\n}\n\nUVW_INLINE const uv_loop_t *loop::raw() const noexcept {\n    return uv_loop.get();\n}\n\nUVW_INLINE uv_loop_t *loop::raw() noexcept {\n    return const_cast<uv_loop_t *>(const_cast<const loop *>(this)->raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/loop.h\n#ifndef UVW_LOOP_INCLUDE_H\n#define UVW_LOOP_INCLUDE_H\n\n#ifdef _WIN32\n#    include <ciso646>\n#endif\n\n#include <chrono>\n#include <functional>\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"util.h\"\n\nnamespace uvw {\n\nclass async_handle;\nclass check_handle;\nclass fs_event_handle;\nclass fs_poll_handle;\nclass idle_handle;\nclass pipe_handle;\nclass poll_handle;\nclass prepare_handle;\nclass process_handle;\nclass signal_handle;\nclass tcp_handle;\nclass timer_handle;\nclass tty_handle;\nclass udp_handle;\n\nnamespace details {\n\nenum class uvw_loop_option : std::underlying_type_t<uv_loop_option> {\n    BLOCK_SIGNAL = UV_LOOP_BLOCK_SIGNAL,\n    IDLE_TIME = UV_METRICS_IDLE_TIME\n};\n\nenum class uvw_run_mode : std::underlying_type_t<uv_run_mode> {\n    DEFAULT = UV_RUN_DEFAULT,\n    ONCE = UV_RUN_ONCE,\n    NOWAIT = UV_RUN_NOWAIT\n};\n\n} // namespace details\n\nusing metrics_type = uv_metrics_t; /*!< Library equivalent for uv_metrics_t. */\n\n/**\n * @brief The loop class.\n *\n * The event loop is the central part of `uvw`'s functionalities, as well as\n * `libuv`'s ones.<br/>\n * It takes care of polling for I/O and scheduling callbacks to be run based on\n * different sources of events.\n */\nclass loop final: public emitter<loop>, public std::enable_shared_from_this<loop> {\n    using deleter = void (*)(uv_loop_t *);\n\n    template<typename, typename, typename...>\n    friend class resource;\n\n    class uv_token {\n        friend class loop;\n        explicit uv_token(int) {}\n    };\n\n    template<typename Type>\n    auto init(int, Type &value) -> decltype(value.init()) {\n        return value.init();\n    }\n\n    template<typename Type>\n    int init(char, Type &) {\n        return 0;\n    }\n\n    loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept;\n\npublic:\n    using token = uv_token;\n    using time = std::chrono::duration<uint64_t, std::milli>;\n    using option = details::uvw_loop_option;\n    using run_mode = details::uvw_run_mode;\n\n    /**\n     * @brief Initializes a new loop instance.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create();\n\n    /**\n     * @brief Initializes a new loop instance from an existing resource.\n     *\n     * The lifetime of the resource must exceed that of the instance to which\n     * it's associated. Management of the memory associated with the resource is\n     * in charge of the user.\n     *\n     * @param res A valid pointer to a correctly initialized resource.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create(uv_loop_t *res);\n\n    /**\n     * @brief Gets the initialized default loop.\n     *\n     * It may return an empty pointer in case of failure.<br>\n     * This function is just a convenient way for having a global loop\n     * throughout an application, the default loop is in no way different than\n     * the ones initialized with `create()`.<br>\n     * As such, the default loop can be closed with `close()` so the resources\n     * associated with it are freed (even if it is not strictly necessary).\n     *\n     * @return The initialized default loop.\n     */\n    static std::shared_ptr<loop> get_default();\n\n    loop(const loop &) = delete;\n    loop(loop &&other) = delete;\n\n    loop &operator=(const loop &) = delete;\n    loop &operator=(loop &&other) = delete;\n\n    ~loop() noexcept;\n\n    /**\n     * @brief Sets additional loop options.\n     *\n     * You should normally call this before the first call to uv_run() unless\n     * mentioned otherwise.<br/>\n     * Supported options:\n     *\n     * * `loop::option::BLOCK_SIGNAL`: Block a signal when polling for new\n     * events. A second argument is required and it is the signal number.\n     * * `loop::option::IDLE_TIME`: Accumulate the amount of idle time the event\n     * loop spends in the event provider. This option is necessary to use\n     * `idle_time()`.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_configure)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    template<typename... Args>\n    \nint configure(option flag, Args &&...args) {\n        return uv_loop_configure(uv_loop.get(), static_cast<uv_loop_option>(flag), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Creates resources of any type.\n     *\n     * This should be used as a default method to create resources.<br/>\n     * The arguments are the ones required for the specific resource.\n     *\n     * Use it as `loop->resource<uvw::timer_handle>()`.\n     *\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> resource(Args &&...args) {\n        auto ptr = uninitialized_resource<R>(std::forward<Args>(args)...);\n        return (init(0, *ptr) == 0) ? ptr : nullptr;\n    }\n\n    /**\n     * @brief Creates uninitialized resources of any type.\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> uninitialized_resource(Args &&...args) {\n        return std::make_shared<R>(token{0}, shared_from_this(), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Releases all internal loop resources.\n     *\n     * Call this function only when the loop has finished executing and all open\n     * handles and requests have been closed, or the loop will error.\n     *\n     * @return Underlying return value.\n     */\n    int close();\n\n    /**\n     * @brief Runs the event loop.\n     *\n     * Available modes are:\n     *\n     * * `loop::run_mode::DEFAULT`: Runs the event loop until there are no more\n     * active and referenced handles or requests.\n     * * `loop::run_mode::ONCE`: Poll for i/o once. Note that this function\n     * blocks if there are no pending callbacks.\n     * * `loop::run_mode::NOWAIT`: Poll for i/o once but don\u2019t block if there\n     * are no pending callbacks.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_run)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int run(run_mode mode = run_mode::DEFAULT) noexcept;\n\n    /**\n     * @brief Checks if there are active resources.\n     * @return True if there are active resources in the loop.\n     */\n    bool alive() const noexcept;\n\n    /**\n     * @brief Stops the event loop.\n     *\n     * It causes `run()` to end as soon as possible.<br/>\n     * This will happen not sooner than the next loop iteration.<br/>\n     * If this function was called before blocking for I/O, the loop won\u2019t block\n     * for I/O on this iteration.\n     */\n    void stop() noexcept;\n\n    /**\n     * @brief Get backend file descriptor.\n     *\n     * Only kqueue, epoll and event ports are supported.<br/>\n     * This can be used in conjunction with `run(loop::run_mode::NOWAIT)` to\n     * poll in one thread and run the event loop\u2019s callbacks in another.\n     *\n     * @return The backend file descriptor.\n     */\n    int descriptor() const noexcept;\n\n    /**\n     * @brief Gets the poll timeout.\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of valid timeout, false otherwise.\n     * * Milliseconds (`std::chrono::duration<uint64_t, std::milli>`).\n     */\n    std::pair<bool, time> timeout() const noexcept;\n\n    /**\n     * @brief Returns the amount of time the event loop has been idle. The call\n     * is thread safe.\n     * @return The accumulated time spent idle.\n     */\n    time idle_time() const noexcept;\n\n    /**\n     * @brief Tracks various internal operations of the event loop.\n     * @return Event loop metrics.\n     */\n    metrics_type metrics() const noexcept;\n\n    /**\n     * @brief Returns the current timestamp in milliseconds.\n     *\n     * The timestamp is cached at the start of the event loop tick.<br/>\n     * The timestamp increases monotonically from some arbitrary point in\n     * time.<br/>\n     * Don\u2019t make assumptions about the starting point, you will only get\n     * disappointed.\n     *\n     * @return The current timestamp in milliseconds (actual type is\n     * `std::chrono::duration<uint64_t, std::milli>`).\n     */\n    time now() const noexcept;\n\n    /**\n     * @brief Updates the event loop\u2019s concept of _now_.\n     *\n     * The current time is cached at the start of the event loop tick in order\n     * to reduce the number of time-related system calls.<br/>\n     * You won\u2019t normally need to call this function unless you have callbacks\n     * that block the event loop for longer periods of time, where _longer_ is\n     * somewhat subjective but probably on the order of a millisecond or more.\n     */\n    void update() const noexcept;\n\n    /**\n     * @brief Walks the list of handles.\n     *\n     * The callback is invoked once for each handle that is still active.\n     *\n     * @param callback A function to invoke once for each active handle.\n     */\n    template<typename Func>\n    void walk(Func callback) {\n        auto func = [](uv_handle_t *hndl, void *callback_func) {\n            if(hndl->data) {\n                auto &cb = *static_cast<Func *>(callback_func);\n\n                switch(utilities::guess_handle(handle_category{hndl->type})) {\n                case handle_type::ASYNC:\n                    cb(*static_cast<async_handle *>(hndl->data));\n                    break;\n                case handle_type::CHECK:\n                    cb(*static_cast<check_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_EVENT:\n                    cb(*static_cast<fs_event_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_POLL:\n                    cb(*static_cast<fs_poll_handle *>(hndl->data));\n                    break;\n                case handle_type::IDLE:\n                    cb(*static_cast<idle_handle *>(hndl->data));\n                    break;\n                case handle_type::PIPE:\n                    cb(*static_cast<pipe_handle *>(hndl->data));\n                    break;\n                case handle_type::POLL:\n                    cb(*static_cast<poll_handle *>(hndl->data));\n                    break;\n                case handle_type::PREPARE:\n                    cb(*static_cast<prepare_handle *>(hndl->data));\n                    break;\n                case handle_type::PROCESS:\n                    cb(*static_cast<process_handle *>(hndl->data));\n                    break;\n                case handle_type::SIGNAL:\n                    cb(*static_cast<signal_handle *>(hndl->data));\n                    break;\n                case handle_type::TCP:\n                    cb(*static_cast<tcp_handle *>(hndl->data));\n                    break;\n                case handle_type::TIMER:\n                    cb(*static_cast<timer_handle *>(hndl->data));\n                    break;\n                case handle_type::TTY:\n                    cb(*static_cast<tty_handle *>(hndl->data));\n                    break;\n                case handle_type::UDP:\n                    cb(*static_cast<udp_handle *>(hndl->data));\n                    break;\n                default:\n                    // this handle isn't managed by uvw, let it be...\n                    break;\n                }\n            }\n        };\n\n        uv_walk(uv_loop.get(), func, &callback);\n    }\n\n    /**\n     * @brief Reinitialize any kernel state necessary in the child process after\n     * a fork(2) system call.\n     *\n     * Previously started watchers will continue to be started in the child\n     * process.\n     *\n     * It is necessary to explicitly call this function on every event loop\n     * created in the parent process that you plan to continue to use in the\n     * child, including the default loop (even if you don\u2019t continue to use it\n     * in the parent). This function must be called before calling any API\n     * function using the loop in the child. Failure to do so will result in\n     * undefined behaviour, possibly including duplicate events delivered to\n     * both parent and child or aborting the child process.\n     *\n     * When possible, it is preferred to create a new loop in the child process\n     * instead of reusing a loop created in the parent. New loops created in the\n     * child process after the fork should not use this function.\n     *\n     * Note that this function is not implemented on Windows.<br/>\n     * Note also that this function is experimental in `libuv`. It may contain\n     * bugs, and is subject to change or removal. API and ABI stability is not\n     * guaranteed.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_fork)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int fork() noexcept;\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param ud User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> ud);\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const uv_loop_t *raw() const noexcept;\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    uv_loop_t *raw() noexcept;\n\nprivate:\n    std::unique_ptr<uv_loop_t, deleter> uv_loop;\n    std::shared_ptr<void> user_data{nullptr};\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"loop.cpp\"\n#endif\n\n#endif // UVW_LOOP_INCLUDE_H\n\n// Path: src/uvw/uv_type.hpp\n#ifndef UVW_UV_TYPE_INCLUDE_H\n#define UVW_UV_TYPE_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include \"config.h\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/**\n * @brief Wrapper class for underlying types.\n *\n * It acts mainly as a wrapper around data structures of the underlying library.\n */\ntemplate<typename U>\nstruct uv_type {\n    explicit uv_type(loop::token, std::shared_ptr<loop> ref) noexcept\n        : owner{std::move(ref)}, resource{} {}\n\n    uv_type(const uv_type &) = delete;\n    uv_type(uv_type &&) = delete;\n\n    uv_type &operator=(const uv_type &) = delete;\n    uv_type &operator=(uv_type &&) = delete;\n\n    /**\n     * @brief Gets the loop from which the resource was originated.\n     * @return A reference to a loop instance.\n     */\n    loop &parent() const noexcept {\n        return *owner;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const U *raw() const noexcept {\n        return &resource;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    U *raw() noexcept {\n        return &resource;\n    }\n\nprotected:\n    ~uv_type() = default;\n\nprivate:\n    std::shared_ptr<loop> owner;\n    U resource;\n};\n\n} // namespace uvw\n\n#endif // UVW_UV_TYPE_INCLUDE_H\n\n// Path: src/uvw/resource.hpp\n#ifndef UVW_RESOURCE_INCLUDE_H\n#define UVW_RESOURCE_INCLUDE_H\n\n#include <memory>\n#include <utility>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Common class for almost all the resources available in `uvw`.\n *\n * This is the base class for handles and requests.\n */\ntemplate<typename T, typename U, typename... E>\nclass resource: public uv_type<U>, public emitter<T, E...>, public std::enable_shared_from_this<T> {\nprotected:\n    int leak_if(int err) noexcept {\n        if(err == 0) {\n            self_ptr = this->shared_from_this();\n        }\n\n        return err;\n    }\n\n    void self_reset() noexcept {\n        self_ptr.reset();\n    }\n\n    bool has_self() const noexcept {\n        return static_cast<bool>(self_ptr);\n    }\n\npublic:\n    explicit resource(loop::token token, std::shared_ptr<loop> ref)\n        : uv_type<U>{token, std::move(ref)} {\n        this->raw()->data = this;\n    }\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param udata User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> udata) {\n        user_data = std::move(udata);\n    }\n\nprivate:\n    std::shared_ptr<void> user_data{nullptr};\n    std::shared_ptr<void> self_ptr{nullptr};\n};\n\n} // namespace uvw\n\n#endif // UVW_RESOURCE_INCLUDE_H\n\n// Path: src/uvw/handle.hpp\n#ifndef UVW_HANDLE_INCLUDE_H\n#define UVW_HANDLE_INCLUDE_H\n\n#include <cstddef>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n#include \"util.h\"\n\nnamespace uvw {\n\n/*! @brief Close event. */\nstruct close_event {};\n\n/**\n * @brief Handle base class.\n *\n * Base type for all `uvw` handle types.\n */\ntemplate<typename T, typename U, typename... E>\nclass handle: public resource<T, U, close_event, E...> {\nprotected:\n    static void close_callback(uv_handle_t *hndl) {\n        handle<T, U, E...> &ref = *(static_cast<T *>(hndl->data));\n        [[maybe_unused]] auto ptr = ref.shared_from_this();\n        ref.self_reset();\n        ref.publish(close_event{});\n    }\n\n    uv_handle_t *as_uv_handle() {\n        return reinterpret_cast<uv_handle_t *>(this->raw());\n    }\n\n    const uv_handle_t *as_uv_handle() const {\n        return reinterpret_cast<const uv_handle_t *>(this->raw());\n    }\n\npublic:\n    using resource<T, U, close_event, E...>::resource;\n\n    /**\n     * @brief Gets the category of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, an opaque value that identifies the\n     * category of the handle is made available to the users.\n     *\n     * @return The actual category of the handle.\n     */\n    handle_category category() const noexcept {\n        return handle_category{as_uv_handle()->type};\n    }\n\n    /**\n     * @brief Gets the type of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, the type of the underlying handle as\n     * specified by handle_type is made available to the users.\n     *\n     * @return The actual type of the handle.\n     */\n    handle_type type() const noexcept {\n        return utilities::guess_handle(category());\n    }\n\n    /**\n     * @brief Checks if the handle is active.\n     *\n     * What _active_ means depends on the type of handle:\n     *\n     * * An async_handle handle is always active and cannot be deactivated,\n     * except by closing it with uv_close().\n     * * A pipe, tcp, udp, etc. handle - basically any handle that deals with\n     * I/O - is active when it is doing something that involves I/O, like\n     * reading, writing, connecting, accepting new connections, etc.\n     * * A check, idle, timer, etc. handle is active when it has been started\n     * with a call to `start()`.\n     *\n     * Rule of thumb: if a handle of type `foo_handle` has a `start()` member\n     * method, then it\u2019s active from the moment that method is called. Likewise,\n     * `stop()` deactivates the handle again.\n     *\n     * @return True if the handle is active, false otherwise.\n     */\n    bool active() const noexcept {\n        return !!uv_is_active(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if a handle is closing or closed.\n     *\n     * This function should only be used between the initialization of the\n     * handle and the arrival of the close callback.\n     *\n     * @return True if the handle is closing or closed, false otherwise.\n     */\n    bool closing() const noexcept {\n        return !!uv_is_closing(as_uv_handle());\n    }\n\n    /**\n     * @brief Request handle to be closed.\n     *\n     * This **must** be called on each handle before memory is released.<br/>\n     * In-progress requests are cancelled and this can result in errors.\n     *\n     * The handle will emit a close event when finished.\n     */\n    void close() noexcept {\n        if(!closing()) {\n            uv_close(as_uv_handle(), &handle<T, U, E...>::close_callback);\n        }\n    }\n\n    /**\n     * @brief Reference the given handle.\n     *\n     * References are idempotent, that is, if a handle is already referenced\n     * calling this function again will have no effect.\n     */\n    void reference() noexcept {\n        uv_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Unreference the given handle.\n     *\n     * References are idempotent, that is, if a handle is not referenced calling\n     * this function again will have no effect.\n     */\n    void unreference() noexcept {\n        uv_unref(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if the given handle referenced.\n     * @return True if the handle referenced, false otherwise.\n     */\n    bool referenced() const noexcept {\n        return !!uv_has_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Returns the size of the underlying handle type.\n     * @return The size of the underlying handle type.\n     */\n    std::size_t size() const noexcept {\n        return uv_handle_size(as_uv_handle()->type);\n    }\n\n    /**\n     * @brief Gets the size of the send buffer used for the socket.\n     *\n     * Gets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipeand udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the send buffer, the underlying return value in case\n     * of errors.\n     */\n    int send_buffer_size() {\n        int value = 0;\n        auto err = uv_send_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the send buffer used for the socket.\n     *\n     * Sets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int send_buffer_size(int value) {\n        return uv_send_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the size of the receive buffer used for the socket.\n     *\n     * Gets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the receive buffer, the underlying return value in\n     * case of errors.\n     */\n    int recv_buffer_size() {\n        int value = 0;\n        auto err = uv_recv_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the receive buffer used for the socket.\n     *\n     * Sets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int recv_buffer_size(int value) {\n        return uv_recv_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the platform dependent file descriptor equivalent.\n     *\n     * Supported handles:\n     *\n     * * tcp_handle\n     * * pipe_handle\n     * * tty_handle\n     * * udp_handle\n     * * poll_handle\n     *\n     * If invoked on a different handle, one that doesn\u2019t have an attached file\n     * descriptor yet or one which was closed, an invalid value is returned.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/handle.html#c.uv_fileno)\n     * for further details.\n     *\n     * @return The file descriptor attached to the hande or a negative value in\n     * case of errors.\n     */\n    os_file_descriptor fd() const {\n        uv_os_fd_t fd;\n        uv_fileno(as_uv_handle(), &fd);\n        return fd;\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_HANDLE_INCLUDE_H\n\n// Path: src/uvw/prepare.h\n#ifndef UVW_PREPARE_INCLUDE_H\n#define UVW_PREPARE_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Prepare event. */\nstruct prepare_event {};\n\n/**\n * @brief The prepare handle.\n *\n * Prepare handles will emit a prepare event once per loop iteration, right\n * before polling for I/O.\n *\n * To create a `prepare_handle` through a `loop`, no arguments are required.\n */\nclass prepare_handle final: public handle<prepare_handle, uv_prepare_t, prepare_event> {\n    static void start_callback(uv_prepare_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * A prepare event will be emitted once per loop iteration, right before\n     * polling for I/O.\n     *\n     * The handle will start emitting prepare events when needed.\n     *\n     * @return Underlying return value.\n     */\n    int start();\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"prepare.cpp\"\n#endif\n\n#endif // UVW_PREPARE_INCLUDE_H\n\n// Path: src/uvw/prepare.cpp\n#ifdef UVW_AS_LIB\n#    include \"prepare.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void prepare_handle::start_callback(uv_prepare_t *hndl) {\n    prepare_handle &prepare = *(static_cast<prepare_handle *>(hndl->data));\n    prepare.publish(prepare_event{});\n}\n\nUVW_INLINE int prepare_handle::init() {\n    return leak_if(uv_prepare_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int prepare_handle::start() {\n    return uv_prepare_start(raw(), &start_callback);\n}\n\nUVW_INLINE int prepare_handle::stop() {\n    return uv_prepare_stop(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/request.hpp\n#ifndef UVW_REQUEST_INCLUDE_H\n...\n", "needle_token_start": 8790, "needle_token_end": 8844, "code_context_ntokens": 16077, "output": ["```cpp\ntemplate<typename... Args>\nint configure(option flag, Args &&...args) {\n    return uv_loop_configure(uv_loop.get(), static_cast<uv_loop_option>(flag), std::forward<Args>(args)...);\n}\n```"]}
{"repo": "skypjack/uvw", "name": "stream_handle", "language": "cpp", "path": "src/uvw/stream.h", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: Provides an abstraction layer for duplex communication channels, facilitating data transmission and reception across different types of stream-based connections such as TCP, pipes, and TTY.\n2. **Input**: Receives data through callbacks that handle various stream events like connection establishment, data reception, and stream closure.\n3. **Output**: Emits events corresponding to the stream's state changes and data flow, such as connection success, incoming data, or errors.\n4. **Procedure**: Initializes with a loop reference and sets up callbacks to handle stream events. It processes incoming data, handles connection lifecycle events, and manages errors and stream termination through a series of predefined callbacks.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/loop.h\n#ifndef UVW_LOOP_INCLUDE_H\n#define UVW_LOOP_INCLUDE_H\n\n#ifdef _WIN32\n#    include <ciso646>\n#endif\n\n#include <chrono>\n#include <functional>\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"util.h\"\n\nnamespace uvw {\n\nclass async_handle;\nclass check_handle;\nclass fs_event_handle;\nclass fs_poll_handle;\nclass idle_handle;\nclass pipe_handle;\nclass poll_handle;\nclass prepare_handle;\nclass process_handle;\nclass signal_handle;\nclass tcp_handle;\nclass timer_handle;\nclass tty_handle;\nclass udp_handle;\n\nnamespace details {\n\nenum class uvw_loop_option : std::underlying_type_t<uv_loop_option> {\n    BLOCK_SIGNAL = UV_LOOP_BLOCK_SIGNAL,\n    IDLE_TIME = UV_METRICS_IDLE_TIME\n};\n\nenum class uvw_run_mode : std::underlying_type_t<uv_run_mode> {\n    DEFAULT = UV_RUN_DEFAULT,\n    ONCE = UV_RUN_ONCE,\n    NOWAIT = UV_RUN_NOWAIT\n};\n\n} // namespace details\n\nusing metrics_type = uv_metrics_t; /*!< Library equivalent for uv_metrics_t. */\n\n/**\n * @brief The loop class.\n *\n * The event loop is the central part of `uvw`'s functionalities, as well as\n * `libuv`'s ones.<br/>\n * It takes care of polling for I/O and scheduling callbacks to be run based on\n * different sources of events.\n */\nclass loop final: public emitter<loop>, public std::enable_shared_from_this<loop> {\n    using deleter = void (*)(uv_loop_t *);\n\n    template<typename, typename, typename...>\n    friend class resource;\n\n    class uv_token {\n        friend class loop;\n        explicit uv_token(int) {}\n    };\n\n    template<typename Type>\n    auto init(int, Type &value) -> decltype(value.init()) {\n        return value.init();\n    }\n\n    template<typename Type>\n    int init(char, Type &) {\n        return 0;\n    }\n\n    loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept;\n\npublic:\n    using token = uv_token;\n    using time = std::chrono::duration<uint64_t, std::milli>;\n    using option = details::uvw_loop_option;\n    using run_mode = details::uvw_run_mode;\n\n    /**\n     * @brief Initializes a new loop instance.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create();\n\n    /**\n     * @brief Initializes a new loop instance from an existing resource.\n     *\n     * The lifetime of the resource must exceed that of the instance to which\n     * it's associated. Management of the memory associated with the resource is\n     * in charge of the user.\n     *\n     * @param res A valid pointer to a correctly initialized resource.\n     * @return A pointer to the newly created loop.\n     */\n    static std::shared_ptr<loop> create(uv_loop_t *res);\n\n    /**\n     * @brief Gets the initialized default loop.\n     *\n     * It may return an empty pointer in case of failure.<br>\n     * This function is just a convenient way for having a global loop\n     * throughout an application, the default loop is in no way different than\n     * the ones initialized with `create()`.<br>\n     * As such, the default loop can be closed with `close()` so the resources\n     * associated with it are freed (even if it is not strictly necessary).\n     *\n     * @return The initialized default loop.\n     */\n    static std::shared_ptr<loop> get_default();\n\n    loop(const loop &) = delete;\n    loop(loop &&other) = delete;\n\n    loop &operator=(const loop &) = delete;\n    loop &operator=(loop &&other) = delete;\n\n    ~loop() noexcept;\n\n    /**\n     * @brief Sets additional loop options.\n     *\n     * You should normally call this before the first call to uv_run() unless\n     * mentioned otherwise.<br/>\n     * Supported options:\n     *\n     * * `loop::option::BLOCK_SIGNAL`: Block a signal when polling for new\n     * events. A second argument is required and it is the signal number.\n     * * `loop::option::IDLE_TIME`: Accumulate the amount of idle time the event\n     * loop spends in the event provider. This option is necessary to use\n     * `idle_time()`.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_configure)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    template<typename... Args>\n    int configure(option flag, Args &&...args) {\n        return uv_loop_configure(uv_loop.get(), static_cast<uv_loop_option>(flag), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Creates resources of any type.\n     *\n     * This should be used as a default method to create resources.<br/>\n     * The arguments are the ones required for the specific resource.\n     *\n     * Use it as `loop->resource<uvw::timer_handle>()`.\n     *\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> resource(Args &&...args) {\n        auto ptr = uninitialized_resource<R>(std::forward<Args>(args)...);\n        return (init(0, *ptr) == 0) ? ptr : nullptr;\n    }\n\n    /**\n     * @brief Creates uninitialized resources of any type.\n     * @return A pointer to the newly created resource.\n     */\n    template<typename R, typename... Args>\n    std::shared_ptr<R> uninitialized_resource(Args &&...args) {\n        return std::make_shared<R>(token{0}, shared_from_this(), std::forward<Args>(args)...);\n    }\n\n    /**\n     * @brief Releases all internal loop resources.\n     *\n     * Call this function only when the loop has finished executing and all open\n     * handles and requests have been closed, or the loop will error.\n     *\n     * @return Underlying return value.\n     */\n    int close();\n\n    /**\n     * @brief Runs the event loop.\n     *\n     * Available modes are:\n     *\n     * * `loop::run_mode::DEFAULT`: Runs the event loop until there are no more\n     * active and referenced handles or requests.\n     * * `loop::run_mode::ONCE`: Poll for i/o once. Note that this function\n     * blocks if there are no pending callbacks.\n     * * `loop::run_mode::NOWAIT`: Poll for i/o once but don\u2019t block if there\n     * are no pending callbacks.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_run)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int run(run_mode mode = run_mode::DEFAULT) noexcept;\n\n    /**\n     * @brief Checks if there are active resources.\n     * @return True if there are active resources in the loop.\n     */\n    bool alive() const noexcept;\n\n    /**\n     * @brief Stops the event loop.\n     *\n     * It causes `run()` to end as soon as possible.<br/>\n     * This will happen not sooner than the next loop iteration.<br/>\n     * If this function was called before blocking for I/O, the loop won\u2019t block\n     * for I/O on this iteration.\n     */\n    void stop() noexcept;\n\n    /**\n     * @brief Get backend file descriptor.\n     *\n     * Only kqueue, epoll and event ports are supported.<br/>\n     * This can be used in conjunction with `run(loop::run_mode::NOWAIT)` to\n     * poll in one thread and run the event loop\u2019s callbacks in another.\n     *\n     * @return The backend file descriptor.\n     */\n    int descriptor() const noexcept;\n\n    /**\n     * @brief Gets the poll timeout.\n     * @return A `std::pair` composed as it follows:\n     * * A boolean value that is true in case of valid timeout, false otherwise.\n     * * Milliseconds (`std::chrono::duration<uint64_t, std::milli>`).\n     */\n    std::pair<bool, time> timeout() const noexcept;\n\n    /**\n     * @brief Returns the amount of time the event loop has been idle. The call\n     * is thread safe.\n     * @return The accumulated time spent idle.\n     */\n    time idle_time() const noexcept;\n\n    /**\n     * @brief Tracks various internal operations of the event loop.\n     * @return Event loop metrics.\n     */\n    metrics_type metrics() const noexcept;\n\n    /**\n     * @brief Returns the current timestamp in milliseconds.\n     *\n     * The timestamp is cached at the start of the event loop tick.<br/>\n     * The timestamp increases monotonically from some arbitrary point in\n     * time.<br/>\n     * Don\u2019t make assumptions about the starting point, you will only get\n     * disappointed.\n     *\n     * @return The current timestamp in milliseconds (actual type is\n     * `std::chrono::duration<uint64_t, std::milli>`).\n     */\n    time now() const noexcept;\n\n    /**\n     * @brief Updates the event loop\u2019s concept of _now_.\n     *\n     * The current time is cached at the start of the event loop tick in order\n     * to reduce the number of time-related system calls.<br/>\n     * You won\u2019t normally need to call this function unless you have callbacks\n     * that block the event loop for longer periods of time, where _longer_ is\n     * somewhat subjective but probably on the order of a millisecond or more.\n     */\n    void update() const noexcept;\n\n    /**\n     * @brief Walks the list of handles.\n     *\n     * The callback is invoked once for each handle that is still active.\n     *\n     * @param callback A function to invoke once for each active handle.\n     */\n    template<typename Func>\n    void walk(Func callback) {\n        auto func = [](uv_handle_t *hndl, void *callback_func) {\n            if(hndl->data) {\n                auto &cb = *static_cast<Func *>(callback_func);\n\n                switch(utilities::guess_handle(handle_category{hndl->type})) {\n                case handle_type::ASYNC:\n                    cb(*static_cast<async_handle *>(hndl->data));\n                    break;\n                case handle_type::CHECK:\n                    cb(*static_cast<check_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_EVENT:\n                    cb(*static_cast<fs_event_handle *>(hndl->data));\n                    break;\n                case handle_type::FS_POLL:\n                    cb(*static_cast<fs_poll_handle *>(hndl->data));\n                    break;\n                case handle_type::IDLE:\n                    cb(*static_cast<idle_handle *>(hndl->data));\n                    break;\n                case handle_type::PIPE:\n                    cb(*static_cast<pipe_handle *>(hndl->data));\n                    break;\n                case handle_type::POLL:\n                    cb(*static_cast<poll_handle *>(hndl->data));\n                    break;\n                case handle_type::PREPARE:\n                    cb(*static_cast<prepare_handle *>(hndl->data));\n                    break;\n                case handle_type::PROCESS:\n                    cb(*static_cast<process_handle *>(hndl->data));\n                    break;\n                case handle_type::SIGNAL:\n                    cb(*static_cast<signal_handle *>(hndl->data));\n                    break;\n                case handle_type::TCP:\n                    cb(*static_cast<tcp_handle *>(hndl->data));\n                    break;\n                case handle_type::TIMER:\n                    cb(*static_cast<timer_handle *>(hndl->data));\n                    break;\n                case handle_type::TTY:\n                    cb(*static_cast<tty_handle *>(hndl->data));\n                    break;\n                case handle_type::UDP:\n                    cb(*static_cast<udp_handle *>(hndl->data));\n                    break;\n                default:\n                    // this handle isn't managed by uvw, let it be...\n                    break;\n                }\n            }\n        };\n\n        uv_walk(uv_loop.get(), func, &callback);\n    }\n\n    /**\n     * @brief Reinitialize any kernel state necessary in the child process after\n     * a fork(2) system call.\n     *\n     * Previously started watchers will continue to be started in the child\n     * process.\n     *\n     * It is necessary to explicitly call this function on every event loop\n     * created in the parent process that you plan to continue to use in the\n     * child, including the default loop (even if you don\u2019t continue to use it\n     * in the parent). This function must be called before calling any API\n     * function using the loop in the child. Failure to do so will result in\n     * undefined behaviour, possibly including duplicate events delivered to\n     * both parent and child or aborting the child process.\n     *\n     * When possible, it is preferred to create a new loop in the child process\n     * instead of reusing a loop created in the parent. New loops created in the\n     * child process after the fork should not use this function.\n     *\n     * Note that this function is not implemented on Windows.<br/>\n     * Note also that this function is experimental in `libuv`. It may contain\n     * bugs, and is subject to change or removal. API and ABI stability is not\n     * guaranteed.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/loop.html#c.uv_loop_fork)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int fork() noexcept;\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param ud User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> ud);\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n...\n// Path: src/uvw/uv_type.hpp\n#ifndef UVW_UV_TYPE_INCLUDE_H\n#define UVW_UV_TYPE_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include \"config.h\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/**\n * @brief Wrapper class for underlying types.\n *\n * It acts mainly as a wrapper around data structures of the underlying library.\n */\ntemplate<typename U>\nstruct uv_type {\n    explicit uv_type(loop::token, std::shared_ptr<loop> ref) noexcept\n        : owner{std::move(ref)}, resource{} {}\n\n    uv_type(const uv_type &) = delete;\n    uv_type(uv_type &&) = delete;\n\n    uv_type &operator=(const uv_type &) = delete;\n    uv_type &operator=(uv_type &&) = delete;\n\n    /**\n     * @brief Gets the loop from which the resource was originated.\n     * @return A reference to a loop instance.\n     */\n    loop &parent() const noexcept {\n        return *owner;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    const U *raw() const noexcept {\n        return &resource;\n    }\n\n    /**\n     * @brief Gets the underlying raw data structure.\n     *\n     * This function should not be used, unless you know exactly what you are\n     * doing and what are the risks.<br/>\n     * Going raw is dangerous, mainly because the lifetime management of a loop,\n     * a handle or a request is in charge to the library itself and users should\n     * not work around it.\n     *\n     * @warning\n     * Use this function at your own risk, but do not expect any support in case\n     * of bugs.\n     *\n     * @return The underlying raw data structure.\n     */\n    U *raw() noexcept {\n        return &resource;\n    }\n\nprotected:\n    ~uv_type() = default;\n\nprivate:\n    std::shared_ptr<loop> owner;\n    U resource;\n};\n\n} // namespace uvw\n\n#endif // UVW_UV_TYPE_INCLUDE_H\n\n// Path: src/uvw/resource.hpp\n#ifndef UVW_RESOURCE_INCLUDE_H\n#define UVW_RESOURCE_INCLUDE_H\n\n#include <memory>\n#include <utility>\n#include \"config.h\"\n#include \"emitter.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Common class for almost all the resources available in `uvw`.\n *\n * This is the base class for handles and requests.\n */\ntemplate<typename T, typename U, typename... E>\nclass resource: public uv_type<U>, public emitter<T, E...>, public std::enable_shared_from_this<T> {\nprotected:\n    int leak_if(int err) noexcept {\n        if(err == 0) {\n            self_ptr = this->shared_from_this();\n        }\n\n        return err;\n    }\n\n    void self_reset() noexcept {\n        self_ptr.reset();\n    }\n\n    bool has_self() const noexcept {\n        return static_cast<bool>(self_ptr);\n    }\n\npublic:\n    explicit resource(loop::token token, std::shared_ptr<loop> ref)\n        : uv_type<U>{token, std::move(ref)} {\n        this->raw()->data = this;\n    }\n\n    /**\n     * @brief Gets user-defined data. `uvw` won't use this field in any case.\n     * @return User-defined data if any, an invalid pointer otherwise.\n     */\n    template<typename R = void>\n    std::shared_ptr<R> data() const {\n        return std::static_pointer_cast<R>(user_data);\n    }\n\n    /**\n     * @brief Sets arbitrary data. `uvw` won't use this field in any case.\n     * @param udata User-defined arbitrary data.\n     */\n    void data(std::shared_ptr<void> udata) {\n        user_data = std::move(udata);\n    }\n\nprivate:\n    std::shared_ptr<void> user_data{nullptr};\n    std::shared_ptr<void> self_ptr{nullptr};\n};\n\n} // namespace uvw\n\n#endif // UVW_RESOURCE_INCLUDE_H\n\n// Path: src/uvw/handle.hpp\n#ifndef UVW_HANDLE_INCLUDE_H\n#define UVW_HANDLE_INCLUDE_H\n\n#include <cstddef>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n#include \"util.h\"\n\nnamespace uvw {\n\n/*! @brief Close event. */\nstruct close_event {};\n\n/**\n * @brief Handle base class.\n *\n * Base type for all `uvw` handle types.\n */\ntemplate<typename T, typename U, typename... E>\nclass handle: public resource<T, U, close_event, E...> {\nprotected:\n    static void close_callback(uv_handle_t *hndl) {\n        handle<T, U, E...> &ref = *(static_cast<T *>(hndl->data));\n        [[maybe_unused]] auto ptr = ref.shared_from_this();\n        ref.self_reset();\n        ref.publish(close_event{});\n    }\n\n    uv_handle_t *as_uv_handle() {\n        return reinterpret_cast<uv_handle_t *>(this->raw());\n    }\n\n    const uv_handle_t *as_uv_handle() const {\n        return reinterpret_cast<const uv_handle_t *>(this->raw());\n    }\n\npublic:\n    using resource<T, U, close_event, E...>::resource;\n\n    /**\n     * @brief Gets the category of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, an opaque value that identifies the\n     * category of the handle is made available to the users.\n     *\n     * @return The actual category of the handle.\n     */\n    handle_category category() const noexcept {\n        return handle_category{as_uv_handle()->type};\n    }\n\n    /**\n     * @brief Gets the type of the handle.\n     *\n     * A base handle offers no functionality to promote it to the actual handle\n     * type. By means of this function, the type of the underlying handle as\n     * specified by handle_type is made available to the users.\n     *\n     * @return The actual type of the handle.\n     */\n    handle_type type() const noexcept {\n        return utilities::guess_handle(category());\n    }\n\n    /**\n     * @brief Checks if the handle is active.\n     *\n     * What _active_ means depends on the type of handle:\n     *\n     * * An async_handle handle is always active and cannot be deactivated,\n     * except by closing it with uv_close().\n     * * A pipe, tcp, udp, etc. handle - basically any handle that deals with\n     * I/O - is active when it is doing something that involves I/O, like\n     * reading, writing, connecting, accepting new connections, etc.\n     * * A check, idle, timer, etc. handle is active when it has been started\n     * with a call to `start()`.\n     *\n     * Rule of thumb: if a handle of type `foo_handle` has a `start()` member\n     * method, then it\u2019s active from the moment that method is called. Likewise,\n     * `stop()` deactivates the handle again.\n     *\n     * @return True if the handle is active, false otherwise.\n     */\n    bool active() const noexcept {\n        return !!uv_is_active(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if a handle is closing or closed.\n     *\n     * This function should only be used between the initialization of the\n     * handle and the arrival of the close callback.\n     *\n     * @return True if the handle is closing or closed, false otherwise.\n     */\n    bool closing() const noexcept {\n        return !!uv_is_closing(as_uv_handle());\n    }\n\n    /**\n     * @brief Request handle to be closed.\n     *\n     * This **must** be called on each handle before memory is released.<br/>\n     * In-progress requests are cancelled and this can result in errors.\n     *\n     * The handle will emit a close event when finished.\n     */\n    void close() noexcept {\n        if(!closing()) {\n            uv_close(as_uv_handle(), &handle<T, U, E...>::close_callback);\n        }\n    }\n\n    /**\n     * @brief Reference the given handle.\n     *\n     * References are idempotent, that is, if a handle is already referenced\n     * calling this function again will have no effect.\n     */\n    void reference() noexcept {\n        uv_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Unreference the given handle.\n     *\n     * References are idempotent, that is, if a handle is not referenced calling\n     * this function again will have no effect.\n     */\n    void unreference() noexcept {\n        uv_unref(as_uv_handle());\n    }\n\n    /**\n     * @brief Checks if the given handle referenced.\n     * @return True if the handle referenced, false otherwise.\n     */\n    bool referenced() const noexcept {\n        return !!uv_has_ref(as_uv_handle());\n    }\n\n    /**\n     * @brief Returns the size of the underlying handle type.\n     * @return The size of the underlying handle type.\n     */\n    std::size_t size() const noexcept {\n        return uv_handle_size(as_uv_handle()->type);\n    }\n\n    /**\n     * @brief Gets the size of the send buffer used for the socket.\n     *\n     * Gets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipeand udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the send buffer, the underlying return value in case\n     * of errors.\n     */\n    int send_buffer_size() {\n        int value = 0;\n        auto err = uv_send_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the send buffer used for the socket.\n     *\n     * Sets the size of the send buffer that the operating system uses for the\n     * socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int send_buffer_size(int value) {\n        return uv_send_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the size of the receive buffer used for the socket.\n     *\n     * Gets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will return double the size of the original set value.\n     *\n     * @return The size of the receive buffer, the underlying return value in\n     * case of errors.\n     */\n    int recv_buffer_size() {\n        int value = 0;\n        auto err = uv_recv_buffer_size(as_uv_handle(), &value);\n        return err ? err : value;\n    }\n\n    /**\n     * @brief Sets the size of the receive buffer used for the socket.\n     *\n     * Sets the size of the receive buffer that the operating system uses for\n     * the socket.<br/>\n     * This function works for tcp, pipe and udp handles on Unix and for tcp and\n     * udp handles on Windows.<br/>\n     * Note that Linux will set double the size.\n     *\n     * @return Underlying return value.\n     */\n    int recv_buffer_size(int value) {\n        return uv_recv_buffer_size(as_uv_handle(), &value);\n    }\n\n    /**\n     * @brief Gets the platform dependent file descriptor equivalent.\n     *\n     * Supported handles:\n     *\n     * * tcp_handle\n     * * pipe_handle\n     * * tty_handle\n     * * udp_handle\n     * * poll_handle\n     *\n     * If invoked on a different handle, one that doesn\u2019t have an attached file\n     * descriptor yet or one which was closed, an invalid value is returned.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/handle.html#c.uv_fileno)\n     * for further details.\n     *\n     * @return The file descriptor attached to the hande or a negative value in\n     * case of errors.\n     */\n    os_file_descriptor fd() const {\n        uv_os_fd_t fd;\n        uv_fileno(as_uv_handle(), &fd);\n        return fd;\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_HANDLE_INCLUDE_H\n\n// Path: src/uvw/prepare.h\n#ifndef UVW_PREPARE_INCLUDE_H\n#define UVW_PREPARE_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Prepare event. */\nstruct prepare_event {};\n\n/**\n * @brief The prepare handle.\n *\n * Prepare handles will emit a prepare event once per loop iteration, right\n * before polling for I/O.\n *\n * To create a `prepare_handle` through a `loop`, no arguments are required.\n */\nclass prepare_handle final: public handle<prepare_handle, uv_prepare_t, prepare_event> {\n    static void start_callback(uv_prepare_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * A prepare event will be emitted once per loop iteration, right before\n     * polling for I/O.\n     *\n     * The handle will start emitting prepare events when needed.\n     *\n     * @return Underlying return value.\n     */\n    int start();\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"prepare.cpp\"\n#endif\n\n#endif // UVW_PREPARE_INCLUDE_H\n\n// Path: src/uvw/prepare.cpp\n#ifdef UVW_AS_LIB\n#    include \"prepare.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void prepare_handle::start_callback(uv_prepare_t *hndl) {\n    prepare_handle &prepare = *(static_cast<prepare_handle *>(hndl->data));\n    prepare.publish(prepare_event{});\n}\n\nUVW_INLINE int prepare_handle::init() {\n    return leak_if(uv_prepare_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int prepare_handle::start() {\n    return uv_prepare_start(raw(), &start_callback);\n}\n\nUVW_INLINE int prepare_handle::stop() {\n    return uv_prepare_stop(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/request.hpp\n#ifndef UVW_REQUEST_INCLUDE_H\n#define UVW_REQUEST_INCLUDE_H\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"resource.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Request base class.\n *\n * Base type for all `uvw` request types.\n */\ntemplate<typename T, typename U, typename... E>\nclass request: public resource<T, U, E...> {\nprotected:\n    static auto reserve(U *req) {\n        auto ptr = static_cast<T *>(req->data)->shared_from_this();\n        ptr->self_reset();\n        return ptr;\n    }\n\npublic:\n    using resource<T, U, E...>::resource;\n\n    /**\n     * @brief Cancels a pending request.\n     *\n     * This method fails if the request is executing or has finished\n     * executing.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/request.html#c.uv_cancel)\n     * for further details.\n     *\n     * @return Underlying return value.\n     */\n    int cancel() {\n        return uv_cancel(reinterpret_cast<uv_req_t *>(this->raw()));\n    }\n\n    /**\n     * @brief Returns the size of the underlying request type.\n     * @return The size of the underlying request type.\n     */\n    std::size_t size() const noexcept {\n        return uv_req_size(reinterpret_cast<const uv_req_t *>(this->raw())->type);\n    }\n};\n\n} // namespace uvw\n\n#endif // UVW_REQUEST_INCLUDE_H\n\n// Path: src/uvw/stream.h\n#ifndef UVW_STREAM_INCLUDE_H\n#define UVW_STREAM_INCLUDE_H\n\n#include <algorithm>\n#include <cstddef>\n#include <iterator>\n#include <memory>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"handle.hpp\"\n#include \"loop.h\"\n#include \"request.hpp\"\n\nnamespace uvw {\n\n/*! @brief Connect event. */\nstruct connect_event {};\n\n/*! @brief End event. */\nstruct end_event {};\n\n/*! @brief Listen event. */\nstruct listen_event {};\n\n/*! @brief Shutdown event. */\nstruct shutdown_event {};\n\n/*! @brief Write event. */\nstruct write_event {};\n\n/*! @brief Data event. */\nstruct data_event {\n    explicit data_event(std::unique_ptr<char[]> buf, std::size_t len) noexcept;\n\n    std::unique_ptr<char[]> data; /*!< A bunch of data read on the stream. */\n    std::size_t length;           /*!< The amount of data read on the stream. */\n};\n\nnamespace details {\n\nclass connect_req final: public request<connect_req, uv_connect_t, connect_event> {\n    static void connect_callback(uv_connect_t *req, int status);\n\npublic:\n    using request::request;\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback);\n        return this->leak_if(0);\n    }\n\n    template<typename F, typename... Args>\n    auto connect(F &&f, Args &&...args) -> std::enable_if_t<!std::is_same_v<decltype(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback)), void>, int> {\n        return this->leak_if(std::forward<F>(f)(raw(), std::forward<Args>(args)..., &connect_callback));\n    }\n};\n\nclass shutdown_req final: public request<shutdown_req, uv_shutdown_t, shutdown_event> {\n    static void shoutdown_callback(uv_shutdown_t *req, int status);\n\npublic:\n    using request::request;\n\n    int shutdown(uv_stream_t *hndl);\n};\n\ntemplate<typename Deleter>\nclass write_req final: public request<write_req<Deleter>, uv_write_t, write_event> {\n    static void write_callback(uv_write_t *req, int status) {\n        if(auto ptr = request<write_req<Deleter>, uv_write_t, write_event>::reserve(req); status) {\n            ptr->publish(error_event{status});\n        } else {\n            ptr->publish(write_event{});\n        }\n    }\n\npublic:\n    write_req(loop::token token, std::shared_ptr<loop> parent, std::unique_ptr<char[], Deleter> dt, unsigned int len)\n        : request<write_req<Deleter>, uv_write_t, write_event>{token, std::move(parent)},\n          data{std::move(dt)},\n          buf{uv_buf_init(data.get(), len)} {}\n\n    int write(uv_stream_t *hndl) {\n        return this->leak_if(uv_write(this->raw(), hndl, &buf, 1, &write_callback));\n    }\n\n    int write(uv_stream_t *hndl, uv_stream_t *send) {\n        return this->leak_if(uv_write2(this->raw(), hndl, &buf, 1, send, &write_callback));\n    }\n\nprivate:\n    std::unique_ptr<char[], Deleter> data;\n    uv_buf_t buf;\n};\n\n} // namespace details\n\n/**\n * @brief The stream handle.\n *\n * Stream handles provide an abstraction of a duplex communication channel.\n * The stream handle is an intermediate type, `uvw` provides three stream\n * implementations: tcp, pipe and tty handles.\n */\ntemplate<typename T, typename U, typename... E>\nclass stream_handle: public handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...> {\n    using base = handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...>;\n\n    template<typename, typename, typename...>\n    friend class stream_handle;\n\n    static constexpr unsigned int DEFAULT_BACKLOG = 128;\n\n    static void read_callback(uv_stream_t *hndl, ssize_t nread, const uv_buf_t *buf) {\n        T &ref = *(static_cast<T *>(hndl->data));\n        // data will be destroyed no matter of what the value of nread is\n        std::unique_ptr<char[]> data{buf->base};\n\n        // nread == 0 is ignored (see http://docs.libuv.org/en/v1.x/stream.html)\n        // equivalent to EAGAIN/EWOULDBLOCK, it shouldn't be treated as an error\n        // for we don't have data to emit though, it's fine to suppress it\n\n        if(nread == UV_EOF) {\n            // end of stream\n            ref.publish(end_event{});\n        } else if(nread > 0) {\n            // data available\n            ref.publish(data_event{std::move(data), static_cast<std::size_t>(nread)});\n        } else if(nread < 0) {\n            // transmission error\n            ref.publish(error_event(nread));\n        }\n    }\n\n    static void listen_callback(uv_stream_t *hndl, int status) {\n        if(T &ref = *(static_cast<T *>(hndl->data)); status) {\n            ref.publish(error_event{status});\n        } else {\n            ref.publish(listen_event{});\n        }\n    }\n\n    uv_stream_t *as_uv_stream() {\n        return reinterpret_cast<uv_stream_t *>(this->raw());\n    }\n\n    const uv_stream_t *as_uv_stream() const {\n        return reinterpret_cast<const uv_stream_t *>(this->raw());\n    }\n\npublic:\n#ifdef _MSC_VER\n    \nstream_handle(loop::token token, std::shared_ptr<loop> ref)\n        : base{token, std::move(ref)} {}\n#else\n    using base::base;\n#endif\n\n    /**\n     * @brief Shutdowns the outgoing (write) side of a duplex stream.\n     *\n     * It waits for pending write requests to complete. The handle should refer\n     * to a initialized stream.<br/>\n     * A shutdown event will be emitted after shutdown is complete.\n     *\n     * @return Underlying return value.\n     */\n    int shutdown() {\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        auto shutdown = this->parent().template resource<details::shutdown_req>();\n        shutdown->template on<error_event>(listener);\n        shutdown->template on<shutdown_event>(listener);\n\n        return shutdown->shutdown(as_uv_stream());\n    }\n\n    /**\n     * @brief Starts listening for incoming connections.\n     *\n     * When a new incoming connection is received, a listen event is\n     * emitted.\n     *\n     * @param backlog Indicates the number of connections the kernel might\n     * queue, same as listen(2).\n     *\n     * @return Underlying return value.\n     */\n    int listen(int backlog = DEFAULT_BACKLOG) {\n        return uv_listen(as_uv_stream(), backlog, &listen_callback);\n    }\n\n    /**\n     * @brief Accepts incoming connections.\n     *\n     * This call is used in conjunction with `listen()` to accept incoming\n     * connections. Call this function after receiving a listen event to accept\n     * the connection. Before calling this function, the submitted handle must\n     * be initialized.\n     *\n     * When the listen event is emitted it is guaranteed that this function will\n     * complete successfully the first time. If you attempt to use it more than\n     * once, it may fail.<br/>\n     * It is suggested to only call this function once per listen event.\n     *\n     * @note\n     * Both the handles must be running on the same loop.\n     *\n     * @param ref An initialized handle to be used to accept the connection.\n     * @return Underlying return value.\n     */\n    template<typename S>\n    int accept(S &ref) {\n        return uv_accept(as_uv_stream(), ref.as_uv_stream());\n    }\n\n    /**\n     * @brief Starts reading data from an incoming stream.\n     *\n     * A data event will be emitted several times until there is no more data to\n     * read or `stop()` is called.<br/>\n     * An end event will be emitted when there is no more data to read.\n     *\n     * @return Underlying return value.\n     */\n    int read() {\n        return uv_read_start(as_uv_stream(), &details::common_alloc_callback, &read_callback);\n    }\n\n    /**\n     * @brief Starts reading data from an incoming stream.\n     * @sa read\n     * @tparam Alloc Custom allocation function.\n     * @return Underlying return value.\n     */\n    template<auto Alloc>\n    int read() {\n        return uv_read_start(as_uv_stream(), &details::common_alloc_callback<T, Alloc>, &read_callback);\n    }\n\n    /**\n     * @brief Stops reading data from the stream.\n     *\n     * This function is idempotent and may be safely called on a stopped stream.\n     *\n     * @return Underlying return value.\n     */\n    int stop() {\n        return uv_read_stop(as_uv_stream());\n    }\n\n    /**\n     * @brief Writes data to the stream.\n     *\n     * Data are written in order. The handle takes the ownership of the data and\n     * it is in charge of delete them.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename Deleter>\n    int write(std::unique_ptr<char[], Deleter> data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<Deleter>>(std::move(data), len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream());\n    }\n\n    /**\n     * @brief Writes data to the stream.\n     *\n     * Data are written in order. The handle doesn't take the ownership of the\n     * data. Be sure that their lifetime overcome the one of the request.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int write(char *data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<void (*)(char *)>>(std::unique_ptr<char[], void (*)(char *)>{data, [](char *) {}}, len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream());\n    }\n\n    /**\n     * @brief Extended write function for sending handles over a pipe handle.\n     *\n     * The pipe must be initialized with `ipc == true`.\n     *\n     * `send` must be a tcp or pipe handle, which is a server or a connection\n     * (listening or connected state). Bound sockets or pipes will be assumed to\n     * be servers.\n     *\n     * The handle takes the ownership of the data and it is in charge of delete\n     * them.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param send The handle over which to write data.\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename S, typename Deleter>\n    int write(S &send, std::unique_ptr<char[], Deleter> data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<Deleter>>(std::move(data), len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream(), send.as_uv_stream());\n    }\n\n    /**\n     * @brief Extended write function for sending handles over a pipe handle.\n     *\n     * The pipe must be initialized with `ipc == true`.\n     *\n     * `send` must be a tcp or pipe handle, which is a server or a connection\n     * (listening or connected state). Bound sockets or pipes will be assumed to\n     * be servers.\n     *\n     * The handle doesn't take the ownership of the data. Be sure that their\n     * lifetime overcome the one of the request.\n     *\n     * A write event will be emitted when the data have been written.\n     *\n     * @param send The handle over which to write data.\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    template<typename S>\n    int write(S &send, char *data, unsigned int len) {\n        auto req = this->parent().template resource<details::write_req<void (*)(char *)>>(std::unique_ptr<char[], void (*)(char *)>{data, [](char *) {}}, len);\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        req->template on<error_event>(listener);\n        req->template on<write_event>(listener);\n\n        return req->write(as_uv_stream(), send.as_uv_stream());\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `write()`, but won\u2019t queue a write request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_write(std::unique_ptr<char[]> data, unsigned int len) {\n        uv_buf_t bufs[] = {uv_buf_init(data.get(), len)};\n        return uv_try_write(as_uv_stream(), bufs, 1);\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `try_write` for sending handles over a pipe.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @param send A valid handle suitable for the purpose.\n     * @return Underlying return value.\n     */\n    template<typename V, typename W>\n    int try_write(std::unique_ptr<char[]> data, unsigned int len, stream_handle<V, W> &send) {\n        uv_buf_t bufs[] = {uv_buf_init(data.get(), len)};\n        return uv_try_write2(as_uv_stream(), bufs, 1, send.raw());\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `write()`, but won\u2019t queue a write request if it can\u2019t be\n     * completed immediately.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @return Underlying return value.\n     */\n    int try_write(char *data, unsigned int len) {\n        uv_buf_t bufs[] = {uv_buf_init(data, len)};\n        return uv_try_write(as_uv_stream(), bufs, 1);\n    }\n\n    /**\n     * @brief Queues a write request if it can be completed immediately.\n     *\n     * Same as `try_write` for sending handles over a pipe.\n     *\n     * @param data The data to be written to the stream.\n     * @param len The lenght of the submitted data.\n     * @param send A valid handle suitable for the purpose.\n     * @return Underlying return value.\n     */\n    template<typename V, typename W>\n    int try_write(char *data, unsigned int len, stream_handle<V, W> &send) {\n        uv_buf_t bufs[] = {uv_buf_init(data, len)};\n        return uv_try_write2(as_uv_stream(), bufs, 1, send.raw());\n    }\n\n    /**\n     * @brief Checks if the stream is readable.\n     * @return True if the stream is readable, false otherwise.\n     */\n    bool readable() const noexcept {\n        return (uv_is_readable(as_uv_stream()) == 1);\n    }\n\n    /**\n     * @brief Checks if the stream is writable.\n     * @return True if the stream is writable, false otherwise.\n     */\n    bool writable() const noexcept {\n        return (uv_is_writable(as_uv_stream()) == 1);\n    }\n\n    /**\n     * @brief Enables or disables blocking mode for a stream.\n     *\n     * When blocking mode is enabled all writes complete synchronously. The\n     * interface remains unchanged otherwise, e.g. completion or failure of the\n     * operation will still be reported through events which are emitted\n     * asynchronously.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/stream.html#c.uv_stream_set_blocking)\n     * for further details.\n     *\n     * @param enable True to enable blocking mode, false otherwise.\n     * @return True in case of success, false otherwise.\n     */\n    bool blocking(bool enable = false) {\n        return (0 == uv_stream_set_blocking(as_uv_stream(), enable));\n    }\n\n    /**\n     * @brief Gets the amount of queued bytes waiting to be sent.\n     * @return Amount of queued bytes waiting to be sent.\n     */\n    size_t write_queue_size() const noexcept {\n        return uv_stream_get_write_queue_size(as_uv_stream());\n    }\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"stream.cpp\"\n#endif\n\n#endif // UVW_STREAM_INCLUDE_H\n\n// Path: src/uvw/stream.cpp\n#ifdef UVW_AS_LIB\n#    include \"stream.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE data_event::data_event(std::unique_ptr<char[]> buf, std::size_t len) noexcept\n    : data{std::move(buf)},\n      length{len} {}\n\nUVW_INLINE void details::connect_req::connect_callback(uv_connect_t *req, int status) {\n    if(auto ptr = reserve(req); status) {\n        ptr->publish(error_event{status});\n    } else {\n        ptr->publish(connect_event{});\n    }\n}\n\nUVW_INLINE void details::shutdown_req::shoutdown_callback(uv_shutdown_t *req, int status) {\n    if(auto ptr = reserve(req); status) {\n        ptr->publish(error_event{status});\n    } else {\n        ptr->publish(shutdown_event{});\n    }\n}\n\nUVW_INLINE int details::shutdown_req::shutdown(uv_stream_t *hndl) {\n    return this->leak_if(uv_shutdown(raw(), hndl, &shoutdown_callback));\n}\n\n} // namespace uvw\n\n// Path: src/uvw/check.h\n#ifndef UVW_CHECK_INCLUDE_H\n#define UVW_CHECK_INCLUDE_H\n\n#include <uv.h>\n#include \"handle.hpp\"\n#include \"loop.h\"\n\nnamespace uvw {\n\n/*! @brief Check event. */\nstruct check_event {};\n\n/**\n * @brief The check handle.\n *\n * Check handles will emit a check event once per loop iteration, right after\n * polling for I/O.\n *\n * To create a `check_handle` through a `loop`, no arguments are required.\n */\nclass check_handle final: public handle<check_handle, uv_check_t, check_event> {\n    static void start_callback(uv_check_t *hndl);\n\npublic:\n    using handle::handle;\n\n    /**\n     * @brief Initializes the handle.\n     * @return Underlying return value.\n     */\n    int init();\n\n    /**\n     * @brief Starts the handle.\n     *\n     * A check event will be emitted once per loop iteration, right after\n     * polling for I/O.\n     *\n     * @return Underlying return value.\n     */\n    int start();\n\n    /**\n     * @brief Stops the handle.\n     * @return Underlying return value.\n     */\n    int stop();\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"check.cpp\"\n#endif\n\n#endif // UVW_CHECK_INCLUDE_H\n\n// Path: src/uvw/check.cpp\n#ifdef UVW_AS_LIB\n#    include \"check.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE void check_handle::start_callback(uv_check_t *hndl) {\n    check_handle &check = *(static_cast<check_handle *>(hndl->data));\n    check.publish(check_event{});\n}\n\nUVW_INLINE int check_handle::init() {\n    return leak_if(uv_check_init(parent().raw(), raw()));\n}\n\nUVW_INLINE int check_handle::start() {\n    return uv_check_start(raw(), &start_callback);\n}\n\nUVW_INLINE int check_handle::stop() {\n    return uv_check_stop(raw());\n}\n\n} // namespace uvw\n\n// Path: src/uvw/enum.hpp\n#ifndef UVW_ENUM_INCLUDE_HPP\n#define UVW_ENUM_INCLUDE_HPP\n\n#include <type_traits>\n#include \"config.h\"\n\n/**\n * @brief Operator available for enums for which bitmask support is enabled.\n * @tparam Type Enum class type.\n * @param lhs The first value to use.\n * @param rhs The second value to use.\n * @return The result of invoking the operator on the underlying types of the\n * two values provided.\n */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator|(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) | static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator&(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) & static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator^(const Type lhs, const Type rhs) noexcept {\n    return static_cast<Type>(static_cast<std::underlying_type_t<Type>>(lhs) ^ static_cast<std::underlying_type_t<Type>>(rhs));\n}\n\n/**\n * @brief Operator available for enums for which bitmask support is enabled.\n * @tparam Type Enum class type.\n * @param value The value to use.\n * @return The result of invoking the operator on the underlying types of the\n * value provided.\n */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM)>\noperator~(const Type value) noexcept {\n    return static_cast<Type>(~static_cast<std::underlying_type_t<Type>>(value));\n}\n\n/*! @copydoc operator~ */\ntemplate<typename Type>\n[[nodiscard]] constexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM, bool{})>\noperator!(const Type value) noexcept {\n    return !static_cast<std::underlying_type_t<Type>>(value);\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator|=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs | rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator&=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs & rhs));\n}\n\n/*! @copydoc operator| */\ntemplate<typename Type>\nconstexpr std::enable_if_t<std::is_enum_v<Type>, decltype(Type::_UVW_ENUM) &>\noperator^=(Type &lhs, const Type rhs) noexcept {\n    return (lhs = (lhs ^ rhs));\n}\n\n#endif\n\n// Path: src/uvw/thread.h\n#ifndef UVW_THREAD_INCLUDE_H\n#define UVW_THREAD_INCLUDE_H\n\n#include <cstddef>\n#include <cstdint>\n#include <memory>\n#include <string>\n#include <type_traits>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"enum.hpp\"\n#include \"loop.h\"\n#include \"uv_type.hpp\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_thread_create_flags : std::underlying_type_t<uv_thread_create_flags> {\n    THREAD_NO_FLAGS = UV_THREAD_NO_FLAGS,\n    THREAD_HAS_STACK_SIZE = UV_THREAD_HAS_STACK_SIZE\n};\n\n}\n\nclass thread;\nclass thread_local_storage;\nclass once;\nclass mutex;\nclass rwlock;\nclass semaphore;\nclass condition;\nclass barrier;\n\n/**\n...\n", "needle_token_start": 10440, "needle_token_end": 10473, "code_context_ntokens": 16088, "output": ["```cpp\ntemplate<typename T, typename U, typename... E>\nclass stream_handle: public handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...> {\n    using base = handle<T, U, listen_event, end_event, connect_event, shutdown_event, data_event, write_event, E...>;\n\n    template<typename, typename, typename...>\n    friend class stream_handle;\n\n    static constexpr unsigned int DEFAULT_BACKLOG = 128;\n\n    static void read_callback(uv_stream_t *hndl, ssize_t nread, const uv_buf_t *buf) {\n        T &ref = *(static_cast<T *>(hndl->data));\n        // data will be destroyed no matter of what the value of nread is\n        std::unique_ptr<char[]> data{buf->base};\n\n        // nread == 0 is ignored (see http://docs.libuv.org/en/v1.x/stream.html)\n        // equivalent to EAGAIN/EWOULDBLOCK, it shouldn't be treated as an error\n        // for we don't have data to emit though, it's fine to suppress it\n\n        if(nread == UV_EOF) {\n            // end of stream\n            ref.publish(end_event{});\n        } else if(nread > 0) {\n            // data available\n            ref.publish(data_event{std::move(data), static_cast<std::size_t>(nread)});\n        } else if(nread < 0) {\n            // transmission error\n            ref.publish(error_event(nread));\n        }\n    }\n\n    static void listen_callback(uv_stream_t *hndl, int status) {\n        if(T &ref = *(static_cast<T *>(hndl->data)); status) {\n            ref.publish(error_event{status});\n        } else {\n            ref.publish(listen_event{});\n        }\n    }\n\n    uv_stream_t *as_uv_stream() {\n        return reinterpret_cast<uv_stream_t *>(this->raw());\n    }\n\n    const uv_stream_t *as_uv_stream() const {\n        return reinterpret_cast<const uv_stream_t *>(this->raw());\n    }\n\npublic:\n#ifdef _MSC_VER\n    stream_handle(loop::token token, std::shared_ptr<loop> ref)\n        : base{token, std::move(ref)} {}\n#else\n    using base::base;\n#endif\n\n    /**\n     * @brief Shutdowns the outgoing (write) side of a duplex stream.\n     *\n     * It waits for pending write requests to complete. The handle should refer\n     * to a initialized stream.<br/>\n     * A shutdown event will be emitted after shutdown is complete.\n     *\n     * @return Underlying return value.\n     */\n    int shutdown() {\n        auto listener = [ptr = this->shared_from_this()](const auto &event, const auto &) {\n            ptr->publish(event);\n        };\n\n        auto shutdown = this->parent().template resource<details::shutdown_req>();\n        shutdown->template on<error_event>(listener);\n        shutdown->template on<shutdown_event>(listener);\n\n        return shutdown->shutdown(as_uv_stream());\n    }\n\n    /**\n     * @brief Starts listening for incoming connections.\n     *\n     * When a new incoming connection is received, a listen event is\n     * emitted.\n     *\n     * @param backlog Indicates the number of connections the kernel might\n     * queue, same as listen(2).\n     *\n     * @return Underlying return value.\n     */\n    int listen(int backlog = DEFAULT_BACKLOG) {\n        return uv_listen(as_uv_stream(), backlog, &listen_callback);\n    }\n\n    /**\n     * @brief Accepts incoming connections.\n     *\n     * This call is used in conjunction with `listen()` to accept incoming\n     * connections. Call this function after receiving a listen event to accept\n     * the connection. Before calling this function, the submitted handle must\n     * be initialized.\n     *\n     * When the listen event is emitted it is guaranteed that this function will\n     * complete successfully the first time. If you attempt to use it more than\n     * once, it may fail.<br/>\n     * It is suggested to only call this function once per listen event.\n     *\n     * @note\n     * Both the handles must be running on the same loop.\n     *\n     * @param ref An initialized handle to be used to accept the connection.\n     * @return Underlying return value.\n     */\n    template<typename S>\n    int accept(S &ref) {\n        return uv_accept(as_uv_stream(), ref.as_uv_stream());\n    }\n\n    /**\n     * @brief Starts reading data from an incoming stream.\n     *\n     * A data event will be emitted several times until there is no more data to\n     * read or `stop()` is called.<br/>\n     * An end event will be emitted when there is no more data to read.\n     *\n     * @return Underlying return value.\n     */\n    int read() {\n       "]}
{"repo": "skypjack/uvw", "name": "env", "language": "cpp", "path": "src/uvw/util.h", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To manage environment variables by either retrieving, setting, or iterating through them.\n2. **Input**: For retrieval and setting, the name of the environment variable is required. For setting, an additional value is needed which can be an empty string to unset the variable. For iteration, a function object is required that can handle the name and value of each environment variable.\n3. **Output**: For retrieval, it returns the value of the environment variable or an empty string if not found or on error. For setting, it returns a boolean indicating success or failure. For iteration, it returns a boolean indicating success or failure of the iteration process.\n4. **Procedure**: Depending on the operation:\n   - For retrieval, the function checks the system for the specified environment variable and returns its value.\n   - For setting, the function updates or deletes the specified environment variable based on the value provided.\n   - For iteration, the function retrieves all environment variables and passes each one to the provided function object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/config.h\n#ifndef UVW_CONFIG_H\n#define UVW_CONFIG_H\n\n#ifndef UVW_AS_LIB\n#    define UVW_INLINE inline\n#else\n#    define UVW_INLINE\n#endif\n\n#endif\n\n// Path: src/uvw/type_info.hpp\n#ifndef UVW_TYPE_INFO_INCLUDE_HPP\n#define UVW_TYPE_INFO_INCLUDE_HPP\n\n#include <cstdint>\n#include \"config.h\"\n\nnamespace uvw {\n\n/**\n * @cond TURN_OFF_DOXYGEN\n * Internal details not to be documented.\n */\n\nnamespace internal {\n\n// Fowler-Noll-Vo hash function v. 1a - the good\n[[nodiscard]] static constexpr std::uint32_t fnv1a(const char *curr) noexcept {\n    constexpr std::uint32_t offset = 2166136261;\n    constexpr std::uint32_t prime = 16777619;\n    auto value = offset;\n\n    while(*curr != 0) {\n        auto curr_val_int = static_cast<std::uint32_t>(*(curr++));\n        value = (value ^ curr_val_int) * prime;\n    }\n\n    return value;\n}\n\n[[nodiscard]] static inline std::uint32_t counter() noexcept {\n    static std::uint32_t cnt{};\n    return cnt++;\n}\n\ntemplate<typename Type>\n[[nodiscard]] static std::uint32_t fake() noexcept {\n    static std::uint32_t local = counter();\n    return local;\n}\n\n} // namespace internal\n\n/**\n * Internal details not to be documented.\n * @endcond\n */\n\n/**\n * @brief Returns a numerical identifier for a given type.\n * @tparam Type The type for which to return the numerical identifier.\n * @return The numerical identifier of the give type.\n */\ntemplate<typename Type>\n[[nodiscard]] static constexpr std::uint32_t type() noexcept {\n#if defined __clang__ || defined __GNUC__\n    return internal::fnv1a(__PRETTY_FUNCTION__);\n#elif defined _MSC_VER\n    return internal::fnv1a(__FUNCSIG__);\n#else\n    return internal::fake();\n#endif\n}\n\n} // namespace uvw\n\n#endif // UVW_TYPE_INFO_INCLUDE_HPP\n\n// Path: src/uvw/emitter.cpp\n#ifdef UVW_AS_LIB\n#    include \"emitter.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE int error_event::translate(int sys) noexcept {\n    return uv_translate_sys_error(sys);\n}\n\nUVW_INLINE const char *error_event::what() const noexcept {\n    return uv_strerror(ec);\n}\n\nUVW_INLINE const char *error_event::name() const noexcept {\n    return uv_err_name(ec);\n}\n\nUVW_INLINE int error_event::code() const noexcept {\n    return ec;\n}\n\nUVW_INLINE error_event::operator bool() const noexcept {\n    return ec < 0;\n}\n\n} // namespace uvw\n\n// Path: src/uvw/emitter.h\n#ifndef UVW_EMITTER_INCLUDE_H\n#define UVW_EMITTER_INCLUDE_H\n\n#include <cstddef>\n#include <cstdint>\n#include <functional>\n#include <list>\n#include <memory>\n#include <type_traits>\n#include <unordered_map>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"type_info.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Error event.\n *\n * Custom wrapper around error constants of `libuv`.\n */\nstruct error_event {\n    template<typename Type, typename = std::enable_if_t<std::is_integral_v<Type>>>\n    explicit error_event(Type val) noexcept\n        : ec{static_cast<int>(val)} {}\n\n    /**\n     * @brief Returns the `libuv` error code equivalent to the given platform dependent error code.\n     *\n     * It returns:\n     * * POSIX error codes on Unix (the ones stored in errno).\n     * * Win32 error codes on Windows (those returned by GetLastError() or WSAGetLastError()).\n     *\n     * If `sys` is already a `libuv` error code, it is simply returned.\n     *\n     * @param sys A platform dependent error code.\n     * @return The `libuv` error code equivalent to the given platform dependent error code.\n     */\n    static int translate(int sys) noexcept;\n\n    /**\n     * @brief Returns the error message for the given error code.\n     *\n     * Leaks a few bytes of memory when you call it with an unknown error code.\n     *\n     * @return The error message for the given error code.\n     */\n    const char *what() const noexcept;\n\n    /**\n     * @brief Returns the error name for the given error code.\n     *\n     * Leaks a few bytes of memory when you call it with an unknown error code.\n     *\n     * @return The error name for the given error code.\n     */\n    const char *name() const noexcept;\n\n    /**\n     * @brief Gets the underlying error code, that is an error constant of `libuv`.\n     * @return The underlying error code.\n     */\n    int code() const noexcept;\n\n    /**\n     * @brief Checks if the event contains a valid error code.\n     * @return True in case of success, false otherwise.\n     */\n    explicit operator bool() const noexcept;\n\nprivate:\n    const int ec;\n};\n\n/**\n * @brief Event emitter base class.\n *\n * Almost everything in `uvw` is an event emitter.<br/>\n * This is the base class from which resources and loops inherit.\n */\ntemplate<typename Elem, typename... Event>\nclass emitter {\npublic:\n    template<typename Type>\n    using listener_t = std::function<void(Type &, Elem &)>;\n\nprivate:\n    template<typename Type>\n    const auto &handler() const noexcept {\n        return std::get<listener_t<Type>>(handlers);\n    }\n\n    template<typename Type>\n    auto &handler() noexcept {\n        return std::get<listener_t<Type>>(handlers);\n    }\n\nprotected:\n    template<typename Type>\n    void publish(Type event) {\n        if(auto &listener = handler<Type>(); listener) {\n            listener(event, *static_cast<Elem *>(this));\n        }\n    }\n\npublic:\n    virtual ~emitter() noexcept {\n        static_assert(std::is_base_of_v<emitter<Elem, Event...>, Elem>);\n    }\n\n    /**\n     * @brief Registers a long-lived listener with the event emitter.\n     *\n     * This method is used to register a listener with the emitter.<br/>\n     * A listener is usually defined as a callable object assignable to a\n     * `std::function<void(const Event &, Elem &)`, where `Event` is the type of\n     * the event and `Elem` is the type of the resource.\n     *\n     * @param f A valid listener to be registered.\n     */\n    template<typename Type>\n    void on(listener_t<Type> f) {\n        handler<Type>() = std::move(f);\n    }\n\n    /*! @brief Disconnects the listener for the given event type. */\n    template<typename Type>\n    void reset() noexcept {\n        handler<Type>() = nullptr;\n    }\n\n    /*! @brief Disconnects all listeners. */\n    void reset() noexcept {\n        reset<error_event>();\n        (reset<Event>(), ...);\n    }\n\n    /**\n     * @brief Checks if there is a listener registered for the specific event.\n     * @return True if there is a listener registered for the specific event,\n     * false otherwise.\n     */\n    template<typename Type>\n    bool has() const noexcept {\n        return static_cast<bool>(handler<Type>());\n    }\n\nprivate:\n    std::tuple<listener_t<error_event>, listener_t<Event>...> handlers{};\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"emitter.cpp\"\n#endif\n\n#endif // UVW_EMITTER_INCLUDE_H\n\n// Path: src/uvw/util.cpp\n#ifdef UVW_AS_LIB\n#    include \"util.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE passwd_info::passwd_info(std::shared_ptr<uv_passwd_t> pwd)\n    : value{pwd} {}\n\nUVW_INLINE std::string passwd_info::username() const noexcept {\n    return ((value && value->username) ? value->username : \"\");\n}\n\nUVW_INLINE decltype(uv_passwd_t::uid) passwd_info::uid() const noexcept {\n    return (value ? value->uid : decltype(uv_passwd_t::uid){});\n}\n\nUVW_INLINE decltype(uv_passwd_t::gid) passwd_info::gid() const noexcept {\n    return (value ? value->gid : decltype(uv_passwd_t::gid){});\n}\n\nUVW_INLINE std::string passwd_info::shell() const noexcept {\n    return ((value && value->shell) ? value->shell : \"\");\n}\n\nUVW_INLINE std::string passwd_info::homedir() const noexcept {\n    return ((value && value->homedir) ? value->homedir : \"\");\n}\n\nUVW_INLINE passwd_info::operator bool() const noexcept {\n    return static_cast<bool>(value);\n}\n\nUVW_INLINE uts_name::uts_name(std::shared_ptr<uv_utsname_t> init)\n    : uname{init} {}\n\nUVW_INLINE std::string uts_name::sysname() const noexcept {\n    return uname ? uname->sysname : \"\";\n}\n\nUVW_INLINE std::string uts_name::release() const noexcept {\n    return uname ? uname->release : \"\";\n}\n\nUVW_INLINE std::string uts_name::version() const noexcept {\n    return uname ? uname->version : \"\";\n}\n\nUVW_INLINE std::string uts_name::machine() const noexcept {\n    return uname ? uname->machine : \"\";\n}\n\nnamespace details {\n\nUVW_INLINE void common_alloc_callback(uv_handle_t *, std::size_t suggested, uv_buf_t *buf) {\n    auto size = static_cast<unsigned int>(suggested);\n    *buf = uv_buf_init(new char[size], size);\n}\n\nUVW_INLINE sockaddr ip_addr(const char *addr, unsigned int port) {\n    // explicitly cast to avoid `-Wsign-conversion` warnings\n    // libuv internally just casts to an `unsigned short` anyway\n    auto signed_port = static_cast<int>(port);\n    if(sockaddr_in addr_in; uv_ip4_addr(addr, signed_port, &addr_in) == 0) {\n        return reinterpret_cast<const sockaddr &>(addr_in);\n    } else if(sockaddr_in6 addr_in6; uv_ip6_addr(addr, signed_port, &addr_in6) == 0) {\n        return reinterpret_cast<const sockaddr &>(addr_in6);\n    }\n\n    return {};\n}\n\nUVW_INLINE socket_address sock_addr(const sockaddr_in &addr) {\n    if(char name[details::DEFAULT_SIZE]; uv_ip4_name(&addr, name, details::DEFAULT_SIZE) == 0) {\n        return socket_address{std::string{name}, ntohs(addr.sin_port)};\n    }\n\n    return socket_address{};\n}\n\nUVW_INLINE socket_address sock_addr(const sockaddr_in6 &addr) {\n    if(char name[details::DEFAULT_SIZE]; uv_ip6_name(&addr, name, details::DEFAULT_SIZE) == 0) {\n        return socket_address{std::string{name}, ntohs(addr.sin6_port)};\n    }\n\n    return socket_address{};\n}\n\nUVW_INLINE socket_address sock_addr(const sockaddr &addr) {\n    if(addr.sa_family == AF_INET) {\n        return sock_addr(reinterpret_cast<const sockaddr_in &>(addr));\n    } else if(addr.sa_family == AF_INET6) {\n        return sock_addr(reinterpret_cast<const sockaddr_in6 &>(addr));\n    }\n\n    return socket_address{};\n}\n\nUVW_INLINE socket_address sock_addr(const sockaddr_storage &storage) {\n    if(storage.ss_family == AF_INET) {\n        return sock_addr(reinterpret_cast<const sockaddr_in &>(storage));\n    } else if(storage.ss_family == AF_INET6) {\n        return sock_addr(reinterpret_cast<const sockaddr_in6 &>(storage));\n    }\n\n    return socket_address{};\n}\n\n} // namespace details\n\nUVW_INLINE pid_type utilities::os::pid() noexcept {\n    return uv_os_getpid();\n}\n\nUVW_INLINE pid_type utilities::os::ppid() noexcept {\n    return uv_os_getppid();\n}\n\nUVW_INLINE std::string utilities::os::homedir() noexcept {\n    return details::try_read(&uv_os_homedir);\n}\n\nUVW_INLINE std::string utilities::os::tmpdir() noexcept {\n    return details::try_read(&uv_os_tmpdir);\n}\n\nUVW_INLINE std::string utilities::os::env(const std::string &name) noexcept {\n    return details::try_read(&uv_os_getenv, name.c_str());\n}\n\nUVW_INLINE bool utilities::os::env(const std::string &name, const std::string &value) noexcept {\n    return (0 == (value.empty() ? uv_os_unsetenv(name.c_str()) : uv_os_setenv(name.c_str(), value.c_str())));\n}\n\nUVW_INLINE std::string utilities::os::hostname() noexcept {\n    return details::try_read(&uv_os_gethostname);\n}\n\nUVW_INLINE uts_name utilities::os::uname() noexcept {\n    auto ptr = std::make_shared<uv_utsname_t>();\n    uv_os_uname(ptr.get());\n    return ptr;\n}\n\nUVW_INLINE passwd_info utilities::os::passwd() noexcept {\n    auto deleter = [](uv_passwd_t *passwd) {\n        uv_os_free_passwd(passwd);\n        delete passwd;\n    };\n\n    std::shared_ptr<uv_passwd_t> ptr{new uv_passwd_t, std::move(deleter)};\n    uv_os_get_passwd(ptr.get());\n    return ptr;\n}\n\nUVW_INLINE int utilities::os::priority(pid_type pid) {\n    int prio = 0;\n\n    if(uv_os_getpriority(pid, &prio)) {\n        prio = UV_PRIORITY_LOW + 1;\n    }\n\n    return prio;\n}\n\nUVW_INLINE bool utilities::os::priority(pid_type pid, int prio) {\n    return 0 == uv_os_setpriority(pid, prio);\n}\n\nUVW_INLINE handle_type utilities::guess_handle(handle_category category) noexcept {\n    switch(category) {\n    case UV_ASYNC:\n        return handle_type::ASYNC;\n    case UV_CHECK:\n        return handle_type::CHECK;\n    case UV_FS_EVENT:\n        return handle_type::FS_EVENT;\n    case UV_FS_POLL:\n        return handle_type::FS_POLL;\n    case UV_HANDLE:\n        return handle_type::HANDLE;\n    case UV_IDLE:\n        return handle_type::IDLE;\n    case UV_NAMED_PIPE:\n        return handle_type::PIPE;\n    case UV_POLL:\n        return handle_type::POLL;\n    case UV_PREPARE:\n        return handle_type::PREPARE;\n    case UV_PROCESS:\n        return handle_type::PROCESS;\n    case UV_STREAM:\n        return handle_type::STREAM;\n    case UV_TCP:\n        return handle_type::TCP;\n    case UV_TIMER:\n        return handle_type::TIMER;\n    case UV_TTY:\n        return handle_type::TTY;\n    case UV_UDP:\n        return handle_type::UDP;\n    case UV_SIGNAL:\n        return handle_type::SIGNAL;\n    case UV_FILE:\n        return handle_type::FILE;\n    default:\n        return handle_type::UNKNOWN;\n    }\n}\n\nUVW_INLINE handle_type utilities::guess_handle(file_handle file) noexcept {\n    handle_category category = uv_guess_handle(file);\n    return guess_handle(category);\n}\n\nUVW_INLINE std::vector<cpu_info> utilities::cpu() noexcept {\n    std::vector<cpu_info> cpuinfos;\n\n    uv_cpu_info_t *infos;\n    int count;\n\n    if(0 == uv_cpu_info(&infos, &count)) {\n        for(int next = 0; next < count; ++next) {\n            cpuinfos.push_back({infos[next].model, infos[next].speed, infos[next].cpu_times});\n        }\n\n        uv_free_cpu_info(infos, count);\n    }\n\n    return cpuinfos;\n}\n\nUVW_INLINE std::vector<interface_address> utilities::interface_addresses() noexcept {\n    std::vector<interface_address> interfaces;\n\n    uv_interface_address_t *ifaces{nullptr};\n    int count{0};\n\n    if(0 == uv_interface_addresses(&ifaces, &count)) {\n        for(int next = 0; next < count; ++next) {\n            interface_address iface_addr;\n\n            iface_addr.name = ifaces[next].name;\n            std::copy(ifaces[next].phys_addr, (ifaces[next].phys_addr + 6), iface_addr.physical);\n            iface_addr.internal = ifaces[next].is_internal == 0 ? false : true;\n\n            if(ifaces[next].address.address4.sin_family == AF_INET) {\n                iface_addr.address = details::sock_addr(ifaces[next].address.address4);\n                iface_addr.netmask = details::sock_addr(ifaces[next].netmask.netmask4);\n            } else if(ifaces[next].address.address4.sin_family == AF_INET6) {\n                iface_addr.address = details::sock_addr(ifaces[next].address.address6);\n                iface_addr.netmask = details::sock_addr(ifaces[next].netmask.netmask6);\n            }\n\n            interfaces.push_back(std::move(iface_addr));\n        }\n\n        uv_free_interface_addresses(ifaces, count);\n    }\n\n    return interfaces;\n}\n\nUVW_INLINE std::string utilities::index_to_name(unsigned int index) noexcept {\n    return details::try_read(&uv_if_indextoname, index);\n}\n\nUVW_INLINE std::string utilities::index_to_iid(unsigned int index) noexcept {\n    return details::try_read(&uv_if_indextoiid, index);\n}\n\nUVW_INLINE bool utilities::replace_allocator(malloc_func_type malloc_func, realloc_func_type realloc_func, calloc_func_type calloc_func, free_func_type free_func) noexcept {\n    return (0 == uv_replace_allocator(malloc_func, realloc_func, calloc_func, free_func));\n}\n\nUVW_INLINE std::array<double, 3> utilities::load_average() noexcept {\n    std::array<double, 3> avg;\n    uv_loadavg(avg.data());\n    return avg;\n}\n\nUVW_INLINE char **utilities::setup_args(int argc, char **argv) {\n    return uv_setup_args(argc, argv);\n}\n\nUVW_INLINE std::string utilities::process_title() {\n    std::size_t size = details::DEFAULT_SIZE;\n    char buf[details::DEFAULT_SIZE];\n    std::string str{};\n\n    if(0 == uv_get_process_title(buf, size)) {\n        str.assign(buf, size);\n    }\n\n    return str;\n}\n\nUVW_INLINE bool utilities::process_title(const std::string &title) {\n    return (0 == uv_set_process_title(title.c_str()));\n}\n\nUVW_INLINE uint64_t utilities::total_memory() noexcept {\n    return uv_get_total_memory();\n}\n\nUVW_INLINE uint64_t utilities::constrained_memory() noexcept {\n    return uv_get_constrained_memory();\n}\n\nUVW_INLINE uint64_t utilities::available_memory() noexcept {\n    return uv_get_available_memory();\n}\n\nUVW_INLINE double utilities::uptime() noexcept {\n    double ret;\n\n    if(0 != uv_uptime(&ret)) {\n        ret = 0;\n    }\n\n    return ret;\n}\n\nUVW_INLINE resource_usage utilities::rusage() noexcept {\n    resource_usage ru;\n    auto err = uv_getrusage(&ru);\n    return err ? resource_usage{} : ru;\n}\n\nUVW_INLINE timespec64 utilities::gettime(clock_id source) noexcept {\n    timespec64 ts;\n    auto err = uv_clock_gettime(static_cast<uv_clock_id>(source), &ts);\n    return err ? timespec64{} : ts;\n}\n\nUVW_INLINE uint64_t utilities::hrtime() noexcept {\n    return uv_hrtime();\n}\n\nUVW_INLINE std::string utilities::path() noexcept {\n    return details::try_read(&uv_exepath);\n}\n\nUVW_INLINE std::string utilities::cwd() noexcept {\n    return details::try_read(&uv_cwd);\n}\n\nUVW_INLINE bool utilities::chdir(const std::string &dir) noexcept {\n    return (0 == uv_chdir(dir.data()));\n}\n\nUVW_INLINE timeval64 utilities::time_of_day() noexcept {\n    uv_timeval64_t ret;\n    uv_gettimeofday(&ret);\n    return ret;\n}\n\nUVW_INLINE void utilities::sleep(unsigned int msec) noexcept {\n    uv_sleep(msec);\n}\n\nUVW_INLINE unsigned int utilities::available_parallelism() noexcept {\n    return uv_available_parallelism();\n}\n\n} // namespace uvw\n\n// Path: src/uvw/util.h\n#ifndef UVW_UTIL_INCLUDE_H\n#define UVW_UTIL_INCLUDE_H\n\n#include <array>\n#include <cstddef>\n#include <cstdint>\n#include <memory>\n#include <string>\n#include <string_view>\n#include <type_traits>\n#include <utility>\n#include <vector>\n#include <uv.h>\n#include \"config.h\"\n\nnamespace uvw {\n\nnamespace details {\n\nenum class uvw_handle_type : std::underlying_type_t<uv_handle_type> {\n    UNKNOWN = UV_UNKNOWN_HANDLE,\n    ASYNC = UV_ASYNC,\n    CHECK = UV_CHECK,\n    FS_EVENT = UV_FS_EVENT,\n    FS_POLL = UV_FS_POLL,\n    HANDLE = UV_HANDLE,\n    IDLE = UV_IDLE,\n    PIPE = UV_NAMED_PIPE,\n    POLL = UV_POLL,\n    PREPARE = UV_PREPARE,\n    PROCESS = UV_PROCESS,\n    STREAM = UV_STREAM,\n    TCP = UV_TCP,\n    TIMER = UV_TIMER,\n    TTY = UV_TTY,\n    UDP = UV_UDP,\n    SIGNAL = UV_SIGNAL,\n    FILE = UV_FILE\n};\n\nenum class uvw_clock_id : std::underlying_type_t<uv_clock_id> {\n    MONOTONIC = UV_CLOCK_MONOTONIC,\n    REALTIME = UV_CLOCK_REALTIME\n};\n\ntemplate<typename T>\nstruct uv_type_wrapper {\n    using Type = T;\n\n    constexpr uv_type_wrapper()\n        : value{} {}\n\n    constexpr uv_type_wrapper(Type val)\n        : value{val} {}\n\n    constexpr operator Type() const noexcept {\n        return value;\n    }\n\n    bool operator==(uv_type_wrapper other) const noexcept {\n        return value == other.value;\n    }\n\nprivate:\n    const Type value;\n};\n\ntemplate<typename T>\nbool operator==(uv_type_wrapper<T> lhs, uv_type_wrapper<T> rhs) {\n    return !(lhs == rhs);\n}\n\n} // namespace details\n\n/**\n * @brief Windows size representation.\n */\nstruct win_size {\n    int width;  /*!< The _width_ of the given window. */\n    int height; /*!< The _height_ of the given window. */\n};\n\nusing handle_type = details::uvw_handle_type;                     /*!< The type of a handle. */\nusing handle_category = details::uv_type_wrapper<uv_handle_type>; /*!< Utility class that wraps an internal handle type. */\nusing file_handle = details::uv_type_wrapper<uv_file>;            /*!< Utility class that wraps an internal file handle. */\nusing os_socket_handle = details::uv_type_wrapper<uv_os_sock_t>;  /*!< Utility class that wraps an os socket handle. */\nusing os_file_descriptor = details::uv_type_wrapper<uv_os_fd_t>;  /*!< Utility class that wraps an os file descriptor. */\nusing pid_type = details::uv_type_wrapper<uv_pid_t>;              /*!< Utility class that wraps a cross platform representation of a pid. */\nusing clock_id = details::uvw_clock_id;                           /*!< Utility class that wraps a clock source. */\n\nconstexpr file_handle std_in{0};  /*!< Placeholder for stdin descriptor. */\nconstexpr file_handle std_out{1}; /*!< Placeholder for stdout descriptor. */\nconstexpr file_handle std_err{2}; /*!< Placeholder for stderr descriptor. */\n\nusing time_spec = uv_timespec_t; /*!< Library equivalent for uv_timespec_t. */\nusing file_info = uv_stat_t;     /*!< Library equivalent for uv_stat_t. */\nusing fs_info = uv_statfs_t;     /*!< Library equivalent for uv_statfs_t. */\nusing uid_type = uv_uid_t;       /*!< Library equivalent for uv_uid_t. */\nusing gid_type = uv_gid_t;       /*!< Library equivalent for uv_gid_t. */\n\nusing timeval = uv_timeval_t;       /*!< Library equivalent for uv_timeval_t. */\nusing timeval64 = uv_timeval64_t;   /*!< Library equivalent for uv_timeval64_t. */\nusing timespec64 = uv_timespec64_t; /*!< Library equivalent for uv_timespec64_t. */\nusing resource_usage = uv_rusage_t; /*!< Library equivalent for uv_rusage_t. */\n\n/**\n * @brief Utility class.\n *\n * This class can be used to query the subset of the password file entry for the\n * current effective uid (not the real uid).\n *\n * \\sa utilities::passwd\n */\nstruct passwd_info {\n    passwd_info(std::shared_ptr<uv_passwd_t> pwd);\n\n    /**\n     * @brief Gets the username.\n     * @return The username of the current effective uid (not the real uid).\n     */\n    std::string username() const noexcept;\n\n    /**\n     * @brief Gets the uid.\n     * @return The current effective uid (not the real uid).\n     */\n    decltype(uv_passwd_t::uid) uid() const noexcept;\n\n    /**\n     * @brief Gets the gid.\n     * @return The gid of the current effective uid (not the real uid).\n     */\n    decltype(uv_passwd_t::gid) gid() const noexcept;\n\n    /**\n     * @brief Gets the shell.\n     * @return The shell of the current effective uid (not the real uid).\n     */\n    std::string shell() const noexcept;\n\n    /**\n     * @brief Gets the homedir.\n     * @return The homedir of the current effective uid (not the real uid).\n     */\n    std::string homedir() const noexcept;\n\n    /**\n     * @brief Checks if the instance contains valid data.\n     * @return True if data are all valid, false otherwise.\n     */\n    operator bool() const noexcept;\n\nprivate:\n    std::shared_ptr<uv_passwd_t> value;\n};\n\n/**\n * @brief Utility class.\n *\n * This class can be used to get name and information about the current kernel.\n * The populated data includes the operating system name, release, version, and\n * machine.\n *\n * \\sa utilities::uname\n */\nstruct uts_name {\n    uts_name(std::shared_ptr<uv_utsname_t> init);\n\n    /**\n     * @brief Gets the operating system name (like \"Linux\").\n     * @return The operating system name.\n     */\n    std::string sysname() const noexcept;\n\n    /**\n     * @brief Gets the operating system release (like \"2.6.28\").\n     * @return The operating system release.\n     */\n    std::string release() const noexcept;\n\n    /**\n     * @brief Gets the operating system version.\n     * @return The operating system version\n     */\n    std::string version() const noexcept;\n\n    /**\n     * @brief Gets the hardware identifier.\n     * @return The hardware identifier.\n     */\n    std::string machine() const noexcept;\n\nprivate:\n    std::shared_ptr<uv_utsname_t> uname;\n};\n\n/**\n * @brief The IPv4 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv4 {};\n\n/**\n * @brief The IPv6 tag.\n *\n * To be used as template parameter to switch between IPv4 and IPv6.\n */\nstruct ipv6 {};\n\n/**\n * @brief Address representation.\n */\nstruct socket_address {\n    std::string ip;    /*!< Either an IPv4 or an IPv6. */\n    unsigned int port; /*!< A valid service identifier. */\n};\n\n/**\n * \\brief CPU information.\n */\nstruct cpu_info {\n    using cpu_time = decltype(uv_cpu_info_t::cpu_times);\n\n    std::string model; /*!< The model of the CPU. */\n    int speed;         /*!< The frequency of the CPU. */\n\n    /**\n     * @brief CPU times.\n     *\n     * It is built up of the following data members: `user`, `nice`, `sys`,\n     * `idle`, `irq`, all of them having type `uint64_t`.\n     */\n    cpu_time times;\n};\n\n/**\n * \\brief Interface address.\n */\nstruct interface_address {\n    std::string name;       /*!< The name of the interface (as an example _eth0_). */\n    char physical[6];       /*!< The physical address. */\n    bool internal;          /*!< True if it is an internal interface (as an example _loopback_), false otherwise. */\n    socket_address address; /*!< The address of the given interface. */\n    socket_address netmask; /*!< The netmask of the given interface. */\n};\n\nnamespace details {\n\nstatic constexpr std::size_t DEFAULT_SIZE = 128;\n\ntemplate<typename F, typename... Args>\nstd::string try_read(F &&f, Args &&...args) noexcept {\n    std::size_t size = DEFAULT_SIZE;\n    char buf[DEFAULT_SIZE];\n    std::string str{};\n    auto err = std::forward<F>(f)(args..., buf, &size);\n\n    if(UV_ENOBUFS == err) {\n        std::unique_ptr<char[]> data{new char[size]};\n        err = std::forward<F>(f)(args..., data.get(), &size);\n\n        if(0 == err) {\n            str = data.get();\n        }\n    } else if(0 == err) {\n        str.assign(buf, size);\n    }\n\n    return str;\n}\n\nvoid common_alloc_callback(uv_handle_t *, std::size_t suggested, uv_buf_t *buf);\n\ntemplate<typename Type, auto Alloc>\nvoid common_alloc_callback(uv_handle_t *handle, std::size_t suggested, uv_buf_t *buf) {\n    auto [alloc, size] = Alloc(*static_cast<const Type *>(handle->data), suggested);\n    *buf = uv_buf_init(alloc, static_cast<unsigned int>(size));\n}\n\nsockaddr ip_addr(const char *addr, unsigned int port);\nsocket_address sock_addr(const sockaddr_in &addr);\nsocket_address sock_addr(const sockaddr_in6 &addr);\nsocket_address sock_addr(const sockaddr &addr);\nsocket_address sock_addr(const sockaddr_storage &storage);\n\n} // namespace details\n\n/**\n * @brief Miscellaneous utilities.\n *\n * Miscellaneous functions that don\u2019t really belong to any other class.\n */\nstruct utilities {\n    using malloc_func_type = void *(*)(size_t);\n    using realloc_func_type = void *(*)(void *, size_t);\n    using calloc_func_type = void *(*)(size_t, size_t);\n    using free_func_type = void (*)(void *);\n\n    /**\n     * @brief OS dedicated utilities.\n     */\n    struct os {\n        /**\n         * @brief Returns the current process id.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_getpid)\n         * for further details.\n         *\n         * @return The current process id.\n         */\n        static pid_type pid() noexcept;\n\n        /**\n         * @brief Returns the parent process id.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_getppid)\n         * for further details.\n         *\n         * @return The parent process id.\n         */\n        static pid_type ppid() noexcept;\n\n        /**\n         * @brief Gets the current user's home directory.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_homedir)\n         * for further details.\n         *\n         * @return The current user's home directory, an empty string in case of\n         * errors.\n         */\n        static std::string homedir() noexcept;\n\n        /**\n         * @brief Gets the temp directory.\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_tmpdir)\n         * for further details.\n         *\n         * @return The temp directory, an empty string in case of errors.\n         */\n        static std::string tmpdir() noexcept;\n\n        /**\n         * @brief Retrieves an environment variable.\n         * @param name The name of the variable to be retrieved.\n         * @return The value of the environment variable, an empty string in\n         * case of errors.\n         */\n        static std::string env(const std::string &name) noexcept;\n\n        /**\n         * @brief Creates, updates or deletes an environment variable.\n         * @param name The name of the variable to be updated.\n         * @param value The value to be used for the variable (an empty string\n         * to unset it).\n         * @return True in case of success, false otherwise.\n         */\n        static bool env(const std::string &name, const std::string &value) noexcept;\n\n        /**\n         * @brief Retrieves all environment variables and iterates them.\n         *\n         * Environment variables are passed one at a time to the callback in the\n         * form of `std::string_view`s.<br/>\n         * The signature of the function call operator must be such that it\n         * accepts two parameters, the name and the value of the i-th variable.\n         *\n         * @tparam Func Type of a function object to which to pass environment\n         * variables.\n         * @param func A function object to which to pass environment variables.\n         * @return True in case of success, false otherwise.\n         */\n        template<typename Func>\n        st\natic std::enable_if_t<std::is_invocable_v<Func, std::string_view, std::string_view>, bool>\n        env(Func func) noexcept {\n            uv_env_item_t *items = nullptr;\n            int count{};\n\n            const bool ret = (uv_os_environ(&items, &count) == 0);\n\n            if(ret) {\n                for(int pos = 0; pos < count; ++pos) {\n                    func(std::string_view{items[pos].name}, std::string_view{items[pos].value});\n                }\n\n                uv_os_free_environ(items, count);\n            }\n\n            return ret;\n        }\n\n        /**\n         * @brief Returns the hostname.\n         * @return The hostname, an empty string in case of errors.\n         */\n        static std::string hostname() noexcept;\n\n        /**\n         * @brief Gets name and information about the current kernel.\n         *\n         * This function can be used to get name and information about the\n         * current kernel. The populated data includes the operating system\n         * name, release, version, and machine.\n         *\n         * @return Name and information about the current kernel.\n         */\n        static uts_name uname() noexcept;\n\n        /**\n         * @brief Gets a subset of the password file entry.\n         *\n         * This function can be used to get the subset of the password file\n         * entry for the current effective uid (not the real uid).\n         *\n         * See the official\n         * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_os_get_passwd)\n         * for further details.\n         *\n         * @return The accessible subset of the password file entry.\n         */\n        static passwd_info passwd() noexcept;\n\n        /**\n         * @brief Retrieves the scheduling priority of a process.\n         *\n         * The returned value is between -20 (high priority) and 19 (low priority).\n         * A value that is out of range is returned in case of errors.\n         *\n         * @note\n         * On Windows, the result won't equal necessarily the exact value of the\n         * priority because of a mapping to a Windows priority class.\n         *\n         * @param pid A valid process id.\n         * @return The scheduling priority of the process.\n         */\n        static int priority(pid_type pid);\n\n        /**\n         * @brief Sets the scheduling priority of a process.\n         *\n         * The returned value range is between -20 (high priority) and 19 (low\n         * priority).\n         *\n         * @note\n         * On Windows, the priority is mapped to a Windows priority class. When\n         * retrieving the process priority, the result won't equal necessarily the\n         * exact value of the priority.\n         *\n         * @param pid A valid process id.\n         * @param prio The scheduling priority to set to the process.\n         * @return True in case of success, false otherwise.\n         */\n        static bool priority(pid_type pid, int prio);\n    };\n\n    /**\n     * @brief Gets the type of the handle given a category.\n     * @param category A properly initialized handle category.\n     * @return The actual type of the handle as defined by handle_type\n     */\n    static handle_type guess_handle(handle_category category) noexcept;\n\n    /**\n     * @brief Gets the type of the stream to be used with the given descriptor.\n     *\n     * Returns the type of stream that should be used with a given file\n     * descriptor.<br/>\n     * Usually this will be used during initialization to guess the type of the\n     * stdio streams.\n     *\n     * @param file A valid descriptor.\n     * @return One of the following types:\n     *\n     * * `handle_type::UNKNOWN`\n     * * `handle_type::PIPE`\n     * * `handle_type::TCP`\n     * * `handle_type::TTY`\n     * * `handle_type::UDP`\n     * * `handle_type::FILE`\n     */\n    static handle_type guess_handle(file_handle file) noexcept;\n\n    /** @brief Gets information about the CPUs on the system.\n     *\n     * This function can be used to query the underlying system and get a set of\n     * descriptors of all the available CPUs.\n     *\n     * @return A set of descriptors of all the available CPUs.\n     */\n    static std::vector<cpu_info> cpu() noexcept;\n\n    /**\n     * @brief Gets a set of descriptors of all the available interfaces.\n     *\n     * This function can be used to query the underlying system and get a set of\n     * descriptors of all the available interfaces, either internal or not.\n     *\n     * @return A set of descriptors of all the available interfaces.\n     */\n    static std::vector<interface_address> interface_addresses() noexcept;\n\n    /**\n     * @brief IPv6-capable implementation of\n     * [if_indextoname](https://linux.die.net/man/3/if_indextoname).\n     *\n     * Mapping between network interface names and indexes.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_if_indextoname)\n     * for further details.\n     *\n     * @param index Network interface index.\n     * @return Network interface name.\n     */\n    static std::string index_to_name(unsigned int index) noexcept;\n\n    /**\n     * @brief Retrieves a network interface identifier.\n     *\n     * See the official\n     * [documentation](http://docs.libuv.org/en/v1.x/misc.html#c.uv_if_indextoiid)\n     * for further details.\n     *\n     * @param index Network interface index.\n     * @return Network interface identifier.\n     */\n    static std::string index_to_iid(unsigned int index) noexcept;\n\n    /**\n     * @brief Override the use of some standard library\u2019s functions.\n     *\n     * Override the use of the standard library\u2019s memory allocation\n     * functions.<br/>\n     * This method must be invoked before any other `uvw` function is called or\n     * after all resources have been freed and thus the underlying library\n     * doesn\u2019t reference any allocated memory chunk.\n     *\n     * If any of the function pointers is _null_, the invokation will fail.\n     *\n     * @note\n     * There is no protection against changing the allocator multiple times. If\n     * the user changes it they are responsible for making sure the allocator is\n     * changed while no memory was allocated with the previous allocator, or\n     * that they are compatible.\n     *\n     * @param malloc_func Replacement function for _malloc_.\n     * @param realloc_func Replacement function for _realloc_.\n     * @param calloc_func Replacement function for _calloc_.\n     * @param free_func Replacement function for _free_.\n     * @return True in case of success, false otherwise.\n     */\n    static bool replace_allocator(malloc_func_type malloc_func, realloc_func_type realloc_func, calloc_func_type calloc_func, free_func_type free_func) noexcept;\n\n    /**\n     * @brief Gets the load average.\n     * @return `[0,0,0]` on Windows (not available), the load average otherwise.\n     */\n    static std::array<double, 3> load_average() noexcept;\n\n    /**\n     * @brief Store the program arguments.\n     *\n     * Required for getting / setting the process title.\n     *\n     * @return Arguments that haven't been consumed internally.\n     */\n    static char **setup_args(int argc, char **argv);\n\n    /**\n     * @brief Gets the title of the current process.\n     * @return The process title.\n     */\n    static std::string process_title();\n\n    /**\n     * @brief Sets the current process title.\n     * @param title The process title to be set.\n     * @return True in case of success, false otherwise.\n     */\n    static bool process_title(const std::string &title);\n\n    /**\n     * @brief Gets memory information (in bytes).\n     * @return Memory information.\n     */\n    static uint64_t total_memory() noexcept;\n\n    /**\n     * @brief Gets the amount of memory available to the process (in bytes).\n     *\n     * Gets the amount of memory available to the process based on limits\n     * imposed by the OS. If there is no such constraint, or the constraint is\n     * unknown, `0` is returned.<br/>\n     * Note that it is not unusual for this value to be less than or greater\n     * than `totalMemory`.\n     *\n     * @return Amount of memory available to the process.\n     */\n    static uint64_t constrained_memory() noexcept;\n\n    /**\n     * @brief Gets the amount of free memory still available to the process.\n     * @return Amount of free memory still available to the process (in bytes).\n     */\n    static uint64_t available_memory() noexcept;\n\n    /**\n     * @brief Gets the current system uptime.\n     * @return The current system uptime or 0 in case of errors.\n     */\n    static double uptime() noexcept;\n\n    /**\n     * @brief Gets the resource usage measures for the current process.\n     * @return Resource usage measures, zeroes-filled object in case of errors.\n     */\n    static resource_usage rusage() noexcept;\n\n    /**\n     * @brief Gets the current system time from a high-resolution clock source.\n     * @param source Clock source, either real-time or monotonic.\n     * @return Current system time from the given high-resolution clock source.\n     */\n    static timespec64 gettime(clock_id source) noexcept;\n\n    /**\n     * @brief Gets the current high-resolution real time.\n     *\n     * The time is expressed in nanoseconds. It is relative to an arbitrary time\n     * in the past. It is not related to the time of the day and therefore not\n     * subject to clock drift. The primary use is for measuring performance\n     * between interval.\n     *\n     * @return The current high-resolution real time.\n     */\n    static uint64_t hrtime() noexcept;\n\n    /**\n     * @brief Gets the executable path.\n     * @return The executable path, an empty string in case of errors.\n     */\n    static std::string path() noexcept;\n\n    /**\n     * @brief Gets the current working directory.\n     * @return The current working directory, an empty string in case of errors.\n     */\n    static std::string cwd() noexcept;\n\n    /**\n     * @brief Changes the current working directory.\n     * @param dir The working directory to be set.\n     * @return True in case of success, false otherwise.\n     */\n    static bool chdir(const std::string &dir) noexcept;\n\n    /**\n     * @brief Cross-platform implementation of\n     * [`gettimeofday`](https://linux.die.net/man/2/gettimeofday)\n     * @return The current time.\n     */\n    static timeval64 time_of_day() noexcept;\n\n    /**\n     * @brief Causes the calling thread to sleep for a while.\n     * @param msec Number of milliseconds to sleep.\n     */\n    static void sleep(unsigned int msec) noexcept;\n\n    /**\n     * @brief Returns an estimate of the amount of parallelism a program should\n     * use (always a non-zero value).\n     * @return Estimate of the amount of parallelism a program should use.\n     */\n    static unsigned int available_parallelism() noexcept;\n};\n\n/**\n * @brief Helper type for visitors.\n * @tparam Func Types of function objects.\n */\ntemplate<class... Func>\nstruct overloaded: Func... {\n    using Func::operator()...;\n};\n\n/**\n * @brief Deduction guide.\n * @tparam Func Types of function objects.\n */\ntemplate<class... Func>\noverloaded(Func...) -> overloaded<Func...>;\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"util.cpp\"\n#endif\n\n#endif // UVW_UTIL_INCLUDE_H\n\n// Path: src/uvw/loop.cpp\n#ifdef UVW_AS_LIB\n#    include \"loop.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE loop::loop(std::unique_ptr<uv_loop_t, deleter> ptr) noexcept\n    : uv_loop{std::move(ptr)} {}\n\nUVW_INLINE std::shared_ptr<loop> loop::create() {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{new uv_loop_t, [](uv_loop_t *l) { delete l; }};\n    auto curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n\n    if(uv_loop_init(curr->uv_loop.get())) {\n        curr = nullptr;\n    }\n\n    return curr;\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::create(uv_loop_t *res) {\n    auto ptr = std::unique_ptr<uv_loop_t, deleter>{res, [](uv_loop_t *) {}};\n    return std::shared_ptr<loop>{new loop{std::move(ptr)}};\n}\n\nUVW_INLINE std::shared_ptr<loop> loop::get_default() {\n    static std::weak_ptr<loop> ref;\n    std::shared_ptr<loop> curr;\n\n    if(ref.expired()) {\n        auto def = uv_default_loop();\n\n        if(def) {\n            auto ptr = std::unique_ptr<uv_loop_t, deleter>(def, [](uv_loop_t *) {});\n            curr = std::shared_ptr<loop>{new loop{std::move(ptr)}};\n        }\n\n        ref = curr;\n    } else {\n        curr = ref.lock();\n    }\n\n    return curr;\n}\n\nUVW_INLINE loop::~loop() noexcept {\n    if(uv_loop) {\n        close();\n    }\n}\n\nUVW_INLINE int loop::close() {\n    int ret = 0;\n\n    if(uv_loop) {\n        ret = uv_loop_close(uv_loop.get());\n        uv_loop.reset();\n    }\n\n    return ret;\n}\n\nUVW_INLINE int loop::run(run_mode mode) noexcept {\n    return uv_run(uv_loop.get(), static_cast<uv_run_mode>(mode));\n}\n\nUVW_INLINE bool loop::alive() const noexcept {\n    return !!uv_loop_alive(uv_loop.get());\n}\n\nUVW_INLINE void loop::stop() noexcept {\n    uv_stop(uv_loop.get());\n}\n\nUVW_INLINE int loop::descriptor() const noexcept {\n    return uv_backend_fd(uv_loop.get());\n}\n\nUVW_INLINE std::pair<bool, loop::time> loop::timeout() const noexcept {\n    auto to = uv_backend_timeout(uv_loop.get());\n    return std::make_pair(to == -1, time{to});\n}\n\nUVW_INLINE loop::time loop::idle_time() const noexcept {\n    return time{uv_metrics_idle_time(uv_loop.get())};\n}\n\nUVW_INLINE metrics_type loop::metrics() const noexcept {\n    metrics_type res{};\n    uv_metrics_info(uv_loop.get(), &res);\n    return res;\n}\n\nUVW_INLINE loop::time loop::now() const noexcept {\n    return time{uv_now(uv_loop.get())};\n}\n\nUVW_INLINE void loop::update() const noexcept {\n    return uv_update_time(uv_loop.get());\n}\n\nUVW_INLINE int loop::fork() noexcept {\n...\n", "needle_token_start": 9978, "needle_token_end": 10157, "code_context_ntokens": 14097, "output": ["```cpp\n/**\n * @brief Retrieves an environment variable.\n * @param name The name of the variable to be retrieved.\n * @return The value of the environment variable, an empty string in case of errors.\n */\nstatic std::string env(const std::string &name) noexcept {\n    return details::try_read(&uv_os_getenv, name.c_str());\n}\n\n/**\n * @brief Creates, updates or deletes an environment variable.\n * @param name The name of the variable to be updated.\n * @param value The value to be used for the variable (an empty string to unset it).\n * @return True in case of success, false otherwise.\n */\nstatic bool env(const std::string &name, const std::string &value) noexcept {\n    return (0 == (value.empty() ? uv_os_unsetenv(name.c_str()) : uv_os_setenv(name.c_str(), value.c_str())));\n}\n\n/**\n * @brief Retrieves all environment variables and iterates them.\n *\n * Environment variables are passed one at a time to the callback in the form of `std::string_view`s.<br/>\n * The signature of the function call operator must be such that it\n * accepts two parameters, the name and the value of the i-th variable.\n *\n * @tparam Func Type of a function object to which to pass environment variables.\n * @param func A function object to which to pass environment variables.\n * @return True in case of success, false otherwise.\n */\ntemplate<typename Func>\nstatic std::enable_if_t<std::is_invocable_v<Func, std::string_view, std::string_view>, bool>\nenv(Func func) noexcept {\n    uv_env_item_t *items = nullptr;\n    int count{};\n\n    const bool ret = (uv_os_environ(&items, &count) == 0);\n\n    if(ret) {\n        for(int pos = 0; pos < count; ++pos) {\n            func(std::string_view{items[pos].name}, std::string_view{items[pos].value});\n        }\n\n        uv_os_free_environ(items, count);\n    }\n\n    return ret;\n}\n```"]}
{"repo": "skypjack/uvw", "name": "fnv1a", "language": "cpp", "path": "src/uvw/type_info.hpp", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to compute a hash value for a string using a specific algorithm that is efficient and has a low collision rate, making it suitable for hash table implementations and other applications where a unique identifier for strings is needed.\n2. **Input**: It accepts a pointer to a constant character array (C-style string).\n3. **Output**: It returns a 32-bit unsigned integer representing the computed hash value of the input string.\n4. **Procedure**: The function initializes a hash value with a predefined constant. It then iterates through each character of the input string, combining the current hash value and the character's integer value using an exclusive OR operation, followed by multiplication with a prime number. This process repeats until the end of the string is reached, and the final hash value is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/config.h\n#ifndef UVW_CONFIG_H\n#define UVW_CONFIG_H\n\n#ifndef UVW_AS_LIB\n#    define UVW_INLINE inline\n#else\n#    define UVW_INLINE\n#endif\n\n#endif\n\n// Path: src/uvw/type_info.hpp\n#ifndef UVW_TYPE_INFO_INCLUDE_HPP\n#define UVW_TYPE_INFO_INCLUDE_HPP\n\n#include <cstdint>\n#include \"config.h\"\n\nnamespace uvw {\n\n/**\n * @cond TURN_OFF_DOXYGEN\n * Internal details not to be documented.\n */\n\nnamespace internal {\n\n// Fowler-Noll-Vo hash function v. 1a - the good\n\n[[nodiscard]] static constexpr std::uint32_t fnv1a(const char *curr) noexcept {\n    constexpr std::uint32_t offset = 2166136261;\n    constexpr std::uint32_t prime = 16777619;\n    auto value = offset;\n\n    while(*curr != 0) {\n        auto curr_val_int = static_cast<std::uint32_t>(*(curr++));\n        value = (value ^ curr_val_int) * prime;\n    }\n\n    return value;\n}\n\n[[nodiscard]] static inline std::uint32_t counter() noexcept {\n    static std::uint32_t cnt{};\n    return cnt++;\n}\n\ntemplate<typename Type>\n[[nodiscard]] static std::uint32_t fake() noexcept {\n    static std::uint32_t local = counter();\n    return local;\n}\n\n} // namespace internal\n\n/**\n * Internal details not to be documented.\n * @endcond\n */\n\n/**\n * @brief Returns a numerical identifier for a given type.\n * @tparam Type The type for which to return the numerical identifier.\n * @return The numerical identifier of the give type.\n */\ntemplate<typename Type>\n[[nodiscard]] static constexpr std::uint32_t type() noexcept {\n#if defined __clang__ || defined __GNUC__\n    return internal::fnv1a(__PRETTY_FUNCTION__);\n#elif defined _MSC_VER\n    return internal::fnv1a(__FUNCSIG__);\n#else\n    return internal::fake();\n#endif\n}\n\n} // namespace uvw\n\n#endif // UVW_TYPE_INFO_INCLUDE_HPP\n\n// Path: src/uvw/emitter.cpp\n#ifdef UVW_AS_LIB\n#    include \"emitter.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE int error_event::translate(int sys) noexcept {\n    return uv_translate_sys_error(sys);\n}\n\nUVW_INLINE const char *error_event::what() const noexcept {\n    return uv_strerror(ec);\n}\n\nUVW_INLINE const char *error_event::name() const noexcept {\n    return uv_err_name(ec);\n}\n\nUVW_INLINE int error_event::code() const noexcept {\n    return ec;\n}\n\nUVW_INLINE error_event::operator bool() const noexcept {\n    return ec < 0;\n}\n\n} // namespace uvw\n\n// Path: src/uvw/emitter.h\n#ifndef UVW_EMITTER_INCLUDE_H\n#define UVW_EMITTER_INCLUDE_H\n\n#include <cstddef>\n#include <cstdint>\n#include <functional>\n#include <list>\n#include <memory>\n#include <type_traits>\n#include <unordered_map>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"type_info.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Error event.\n *\n * Custom wrapper around error constants of `libuv`.\n */\nstruct error_event {\n    template<typename Type, typename = std::enable_if_t<std::is_integral_v<Type>>>\n    explicit error_event(Type val) noexcept\n        : ec{static_cast<int>(val)} {}\n\n    /**\n     * @brief Returns the `libuv` error code equivalent to the given platform dependent error code.\n     *\n     * It returns:\n     * * POSIX error codes on Unix (the ones stored in errno).\n     * * Win32 error codes on Windows (those returned by GetLastError() or WSAGetLastError()).\n     *\n     * If `sys` is already a `libuv` error code, it is simply returned.\n     *\n     * @param sys A platform dependent error code.\n     * @return The `libuv` error code equivalent to the given platform dependent error code.\n     */\n    static int translate(int sys) noexcept;\n\n    /**\n     * @brief Returns the error message for the given error code.\n     *\n     * Leaks a few bytes of memory when you call it with an unknown error code.\n     *\n     * @return The error message for the given error code.\n     */\n    const char *what() const noexcept;\n\n    /**\n     * @brief Returns the error name for the given error code.\n     *\n     * Leaks a few bytes of memory when you call it with an unknown error code.\n     *\n     * @return The error name for the given error code.\n     */\n    const char *name() const noexcept;\n\n    /**\n     * @brief Gets the underlying error code, that is an error constant of `libuv`.\n     * @return The underlying error code.\n     */\n    int code() const noexcept;\n\n    /**\n     * @brief Checks if the event contains a valid error code.\n     * @return True in case of success, false otherwise.\n     */\n    explicit operator bool() const noexcept;\n\nprivate:\n    const int ec;\n};\n\n/**\n * @brief Event emitter base class.\n *\n * Almost everything in `uvw` is an event emitter.<br/>\n * This is the base class from which resources and loops inherit.\n */\ntemplate<typename Elem, typename... Event>\nclass emitter {\npublic:\n    template<typename Type>\n    using listener_t = std::function<void(Type &, Elem &)>;\n\nprivate:\n    template<typename Type>\n    const auto &handler() const noexcept {\n        return std::get<listener_t<Type>>(handlers);\n    }\n\n    template<typename Type>\n    auto &handler() noexcept {\n        return std::get<listener_t<Type>>(handlers);\n    }\n\nprotected:\n    template<typename Type>\n    void publish(Type event) {\n        if(auto &listener = handler<Type>(); listener) {\n            listener(event, *static_cast<Elem *>(this));\n        }\n    }\n\npublic:\n    virtual ~emitter() noexcept {\n        static_assert(std::is_base_of_v<emitter<Elem, Event...>, Elem>);\n    }\n\n    /**\n     * @brief Registers a long-lived listener with the event emitter.\n     *\n     * This method is used to register a listener with the emitter.<br/>\n     * A listener is usually defined as a callable object assignable to a\n     * `std::function<void(const Event &, Elem &)`, where `Event` is the type of\n     * the event and `Elem` is the type of the resource.\n     *\n     * @param f A valid listener to be registered.\n     */\n    template<typename Type>\n    void on(listener_t<Type> f) {\n        handler<Type>() = std::move(f);\n    }\n\n    /*! @brief Disconnects the listener for the given event type. */\n    template<typename Type>\n    void reset() noexcept {\n        handler<Type>() = nullptr;\n    }\n\n    /*! @brief Disconnects all listeners. */\n    void reset() noexcept {\n        reset<error_event>();\n        (reset<Event>(), ...);\n    }\n\n    /**\n     * @brief Checks if there is a listener registered for the specific event.\n     * @return True if there is a listener registered for the specific event,\n     * false otherwise.\n     */\n    template<typename Type>\n    bool has() const noexcept {\n        return static_cast<bool>(handler<Type>());\n    }\n\nprivate:\n    std::tuple<listener_t<error_event>, listener_t<Event>...> handlers{};\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"emitter.cpp\"\n#endif\n\n#endif // UVW_EMITTER_INCLUDE_H\n\n// Path: src/uvw/util.cpp\n#ifdef UVW_AS_LIB\n#    include \"util.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE passwd_info::passwd_info(std::shared_ptr<uv_passwd_t> pwd)\n    : value{pwd} {}\n\nUVW_INLINE std::string passwd_info::username() const noexcept {\n    return ((value && value->username) ? value->username : \"\");\n}\n\nUVW_INLINE decltype(uv_passwd_t::uid) passwd_info::uid() const noexcept {\n    return (value ? value->uid : decltype(uv_passwd_t::uid){});\n}\n\nUVW_INLINE decltype(uv_passwd_t::gid) passwd_info::gid() const noexcept {\n    return (value ? value->gid : decltype(uv_passwd_t::gid){});\n}\n\nUVW_INLINE std::string passwd_info::shell() const noexcept {\n    return ((value && value->shell) ? value->shell : \"\");\n}\n\nUVW_INLINE std::string passwd_info::homedir() const noexcept {\n    return ((value && value->homedir) ? value->homedir : \"\");\n}\n\nUVW_INLINE passwd_info::operator bool() const noexcept {\n    return static_cast<bool>(value);\n}\n\nUVW_INLINE uts_name::uts_name(std::shared_ptr<uv_utsname_t> init)\n    : uname{init} {}\n\nUVW_INLINE std::string uts_name::sysname() const noexcept {\n...\n", "needle_token_start": 211, "needle_token_end": 358, "code_context_ntokens": 2670, "output": ["```cpp\n[[nodiscard]] static constexpr std::uint32_t fnv1a(const char *curr) noexcept {\n    constexpr std::uint32_t offset = 2166136261;\n    constexpr std::uint32_t prime = 16777619;\n    auto value = offset;\n\n    while(*curr != 0) {\n        auto curr_val_int = static_cast<std::uint32_t>(*(curr++));\n        value = (value ^ curr_val_int) * prime;\n    }\n\n    return value;\n}\n```"]}
{"repo": "skypjack/uvw", "name": "publish", "language": "cpp", "path": "src/uvw/emitter.h", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to notify registered listeners about specific events occurring within an object, allowing for event-driven responses.\n2. **Input**: The function takes a single parameter representing an event of a generic type.\n3. **Output**: There is no direct output from the function; however, it triggers side effects by invoking event listeners that may perform actions based on the event.\n4. **Procedure**: The function first retrieves the event listener associated with the event's type. If a listener is registered, it invokes this listener, passing the event and a reference to the object from which the event originated.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/uvw/config.h\n#ifndef UVW_CONFIG_H\n#define UVW_CONFIG_H\n\n#ifndef UVW_AS_LIB\n#    define UVW_INLINE inline\n#else\n#    define UVW_INLINE\n#endif\n\n#endif\n\n// Path: src/uvw/type_info.hpp\n#ifndef UVW_TYPE_INFO_INCLUDE_HPP\n#define UVW_TYPE_INFO_INCLUDE_HPP\n\n#include <cstdint>\n#include \"config.h\"\n\nnamespace uvw {\n\n/**\n * @cond TURN_OFF_DOXYGEN\n * Internal details not to be documented.\n */\n\nnamespace internal {\n\n// Fowler-Noll-Vo hash function v. 1a - the good\n[[nodiscard]] static constexpr std::uint32_t fnv1a(const char *curr) noexcept {\n    constexpr std::uint32_t offset = 2166136261;\n    constexpr std::uint32_t prime = 16777619;\n    auto value = offset;\n\n    while(*curr != 0) {\n        auto curr_val_int = static_cast<std::uint32_t>(*(curr++));\n        value = (value ^ curr_val_int) * prime;\n    }\n\n    return value;\n}\n\n[[nodiscard]] static inline std::uint32_t counter() noexcept {\n    static std::uint32_t cnt{};\n    return cnt++;\n}\n\ntemplate<typename Type>\n[[nodiscard]] static std::uint32_t fake() noexcept {\n    static std::uint32_t local = counter();\n    return local;\n}\n\n} // namespace internal\n\n/**\n * Internal details not to be documented.\n * @endcond\n */\n\n/**\n * @brief Returns a numerical identifier for a given type.\n * @tparam Type The type for which to return the numerical identifier.\n * @return The numerical identifier of the give type.\n */\ntemplate<typename Type>\n[[nodiscard]] static constexpr std::uint32_t type() noexcept {\n#if defined __clang__ || defined __GNUC__\n    return internal::fnv1a(__PRETTY_FUNCTION__);\n#elif defined _MSC_VER\n    return internal::fnv1a(__FUNCSIG__);\n#else\n    return internal::fake();\n#endif\n}\n\n} // namespace uvw\n\n#endif // UVW_TYPE_INFO_INCLUDE_HPP\n\n// Path: src/uvw/emitter.cpp\n#ifdef UVW_AS_LIB\n#    include \"emitter.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE int error_event::translate(int sys) noexcept {\n    return uv_translate_sys_error(sys);\n}\n\nUVW_INLINE const char *error_event::what() const noexcept {\n    return uv_strerror(ec);\n}\n\nUVW_INLINE const char *error_event::name() const noexcept {\n    return uv_err_name(ec);\n}\n\nUVW_INLINE int error_event::code() const noexcept {\n    return ec;\n}\n\nUVW_INLINE error_event::operator bool() const noexcept {\n    return ec < 0;\n}\n\n} // namespace uvw\n\n// Path: src/uvw/emitter.h\n#ifndef UVW_EMITTER_INCLUDE_H\n#define UVW_EMITTER_INCLUDE_H\n\n#include <cstddef>\n#include <cstdint>\n#include <functional>\n#include <list>\n#include <memory>\n#include <type_traits>\n#include <unordered_map>\n#include <utility>\n#include <uv.h>\n#include \"config.h\"\n#include \"type_info.hpp\"\n\nnamespace uvw {\n\n/**\n * @brief Error event.\n *\n * Custom wrapper around error constants of `libuv`.\n */\nstruct error_event {\n    template<typename Type, typename = std::enable_if_t<std::is_integral_v<Type>>>\n    explicit error_event(Type val) noexcept\n        : ec{static_cast<int>(val)} {}\n\n    /**\n     * @brief Returns the `libuv` error code equivalent to the given platform dependent error code.\n     *\n     * It returns:\n     * * POSIX error codes on Unix (the ones stored in errno).\n     * * Win32 error codes on Windows (those returned by GetLastError() or WSAGetLastError()).\n     *\n     * If `sys` is already a `libuv` error code, it is simply returned.\n     *\n     * @param sys A platform dependent error code.\n     * @return The `libuv` error code equivalent to the given platform dependent error code.\n     */\n    static int translate(int sys) noexcept;\n\n    /**\n     * @brief Returns the error message for the given error code.\n     *\n     * Leaks a few bytes of memory when you call it with an unknown error code.\n     *\n     * @return The error message for the given error code.\n     */\n    const char *what() const noexcept;\n\n    /**\n     * @brief Returns the error name for the given error code.\n     *\n     * Leaks a few bytes of memory when you call it with an unknown error code.\n     *\n     * @return The error name for the given error code.\n     */\n    const char *name() const noexcept;\n\n    /**\n     * @brief Gets the underlying error code, that is an error constant of `libuv`.\n     * @return The underlying error code.\n     */\n    int code() const noexcept;\n\n    /**\n     * @brief Checks if the event contains a valid error code.\n     * @return True in case of success, false otherwise.\n     */\n    explicit operator bool() const noexcept;\n\nprivate:\n    const int ec;\n};\n\n/**\n * @brief Event emitter base class.\n *\n * Almost everything in `uvw` is an event emitter.<br/>\n * This is the base class from which resources and loops inherit.\n */\ntemplate<typename Elem, typename... Event>\nclass emitter {\npublic:\n    template<typename Type>\n    using listener_t = std::function<void(Type &, Elem &)>;\n\nprivate:\n    template<typename Type>\n    const auto &handler() const noexcept {\n        return std::get<listener_t<Type>>(handlers);\n    }\n\n    template<typename Type>\n    auto &handler() noexcept {\n        return std::get<listener_t<Type>>(handlers);\n    }\n\nprotected:\n    template<typename Type>\n    \nvoid publish(Type event) {\n        if(auto &listener = handler<Type>(); listener) {\n            listener(event, *static_cast<Elem *>(this));\n        }\n    }\n\npublic:\n    virtual ~emitter() noexcept {\n        static_assert(std::is_base_of_v<emitter<Elem, Event...>, Elem>);\n    }\n\n    /**\n     * @brief Registers a long-lived listener with the event emitter.\n     *\n     * This method is used to register a listener with the emitter.<br/>\n     * A listener is usually defined as a callable object assignable to a\n     * `std::function<void(const Event &, Elem &)`, where `Event` is the type of\n     * the event and `Elem` is the type of the resource.\n     *\n     * @param f A valid listener to be registered.\n     */\n    template<typename Type>\n    void on(listener_t<Type> f) {\n        handler<Type>() = std::move(f);\n    }\n\n    /*! @brief Disconnects the listener for the given event type. */\n    template<typename Type>\n    void reset() noexcept {\n        handler<Type>() = nullptr;\n    }\n\n    /*! @brief Disconnects all listeners. */\n    void reset() noexcept {\n        reset<error_event>();\n        (reset<Event>(), ...);\n    }\n\n    /**\n     * @brief Checks if there is a listener registered for the specific event.\n     * @return True if there is a listener registered for the specific event,\n     * false otherwise.\n     */\n    template<typename Type>\n    bool has() const noexcept {\n        return static_cast<bool>(handler<Type>());\n    }\n\nprivate:\n    std::tuple<listener_t<error_event>, listener_t<Event>...> handlers{};\n};\n\n} // namespace uvw\n\n#ifndef UVW_AS_LIB\n#    include \"emitter.cpp\"\n#endif\n\n#endif // UVW_EMITTER_INCLUDE_H\n\n// Path: src/uvw/util.cpp\n#ifdef UVW_AS_LIB\n#    include \"util.h\"\n#endif\n\n#include \"config.h\"\n\nnamespace uvw {\n\nUVW_INLINE passwd_info::passwd_info(std::shared_ptr<uv_passwd_t> pwd)\n    : value{pwd} {}\n\nUVW_INLINE std::string passwd_info::username() const noexcept {\n    return ((value && value->username) ? value->username : \"\");\n}\n\nUVW_INLINE decltype(uv_passwd_t::uid) passwd_info::uid() const noexcept {\n    return (value ? value->uid : decltype(uv_passwd_t::uid){});\n}\n\nUVW_INLINE decltype(uv_passwd_t::gid) passwd_info::gid() const noexcept {\n    return (value ? value->gid : decltype(uv_passwd_t::gid){});\n}\n\nUVW_INLINE std::string passwd_info::shell() const noexcept {\n    return ((value && value->shell) ? value->shell : \"\");\n}\n\n...\n", "needle_token_start": 1716, "needle_token_end": 1761, "code_context_ntokens": 2527, "output": ["```cpp\ntemplate<typename Type>\nvoid publish(Type event) {\n    if(auto &listener = handler<Type>(); listener) {\n        listener(event, *static_cast<Elem *>(this));\n    }\n}\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "CompateStringsCaseInsensitive", "language": "cpp", "path": "clickhouse/types/type_parser.cpp", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to compare two strings in a case-insensitive manner, primarily to determine their equality or order without considering the case of the characters.\n2. **Input**: It takes two string-like objects as parameters.\n3. **Output**: Returns an integer: 0 if both strings are equal (ignoring case), a positive value if the first string is lexicographically greater, and a negative value if the second string is greater.\n4. **Procedure**: The function first checks if the sizes of the two strings differ and returns accordingly. If the sizes are the same, it performs a case-insensitive comparison using platform-specific functions to compare the strings for a specified length equal to their common size.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "#include \"clickhouse/base/platform.h\" // for _win_\n\n#include <algorithm>\n#include <cmath>\n#include <map>\n#include <mutex>\n#include <unordered_map>\n\n#if defined _win_\n#include <string.h>\n#else\n#include <strings.h>\n#endif\n\n\nnamespace clickhouse {\n\nbool TypeAst::operator==(const TypeAst & other) const {\n    return meta == other.meta\n        && code == other.code\n        && name == other.name\n        && value == other.value\n        && std::equal(elements.begin(), elements.end(), other.elements.begin(), other.elements.end());\n}\n\nstatic const std::unordered_map<std::string, Type::Code> kTypeCode = {\n    { \"Void\",        Type::Void },\n    { \"Int8\",        Type::Int8 },\n    { \"Int16\",       Type::Int16 },\n    { \"Int32\",       Type::Int32 },\n    { \"Int64\",       Type::Int64 },\n    { \"Bool\",        Type::UInt8 },\n    { \"UInt8\",       Type::UInt8 },\n    { \"UInt16\",      Type::UInt16 },\n    { \"UInt32\",      Type::UInt32 },\n    { \"UInt64\",      Type::UInt64 },\n    { \"Float32\",     Type::Float32 },\n    { \"Float64\",     Type::Float64 },\n    { \"String\",      Type::String },\n    { \"FixedString\", Type::FixedString },\n    { \"DateTime\",    Type::DateTime },\n    { \"DateTime64\",  Type::DateTime64 },\n    { \"Date\",        Type::Date },\n    { \"Date32\",      Type::Date32 },\n    { \"Array\",       Type::Array },\n    { \"Nullable\",    Type::Nullable },\n    { \"Tuple\",       Type::Tuple },\n    { \"Enum8\",       Type::Enum8 },\n    { \"Enum16\",      Type::Enum16 },\n    { \"UUID\",        Type::UUID },\n    { \"IPv4\",        Type::IPv4 },\n    { \"IPv6\",        Type::IPv6 },\n    { \"Int128\",      Type::Int128 },\n//    { \"UInt128\",      Type::UInt128 },\n    { \"Decimal\",     Type::Decimal },\n    { \"Decimal32\",   Type::Decimal32 },\n    { \"Decimal64\",   Type::Decimal64 },\n    { \"Decimal128\",  Type::Decimal128 },\n    { \"LowCardinality\", Type::LowCardinality },\n    { \"Map\",         Type::Map },\n    { \"Point\",       Type::Point },\n    { \"Ring\",        Type::Ring },\n    { \"Polygon\",     Type::Polygon },\n    { \"MultiPolygon\", Type::MultiPolygon },\n};\n\ntemplate <typename L, typename R>\n\ninline int CompateStringsCaseInsensitive(const L& left, const R& right) {\n    int64_t size_diff = left.size() - right.size();\n    if (size_diff != 0)\n        return size_diff > 0 ? 1 : -1;\n\n#if defined _win_\n    return _strnicmp(left.data(), right.data(), left.size());\n#else\n    return strncasecmp(left.data(), right.data(), left.size());\n#endif\n}\n\nstatic Type::Code GetTypeCode(const std::string& name) {\n    auto it = kTypeCode.find(name);\n    if (it != kTypeCode.end()) {\n        return it->second;\n    }\n\n    return Type::Void;\n}\n\nstatic TypeAst::Meta GetTypeMeta(const StringView& name) {\n    if (name == \"Array\") {\n        return TypeAst::Array;\n    }\n\n    if (name == \"Null\") {\n        return TypeAst::Null;\n    }\n\n    if (name == \"Nullable\") {\n        return TypeAst::Nullable;\n    }\n\n    if (name == \"Tuple\") {\n        return TypeAst::Tuple;\n    }\n\n    if (name == \"Enum8\" || name == \"Enum16\") {\n        return TypeAst::Enum;\n    }\n\n    if (name == \"LowCardinality\") {\n        return TypeAst::LowCardinality;\n    }\n\n    if (name == \"SimpleAggregateFunction\") {\n        return TypeAst::SimpleAggregateFunction;\n    }\n\n    if (name == \"Map\") {\n        return TypeAst::Map;\n    }\n\n    return TypeAst::Terminal;\n}\n\nbool ValidateAST(const TypeAst& ast) {\n    // Void terminal that is not actually \"void\" produced when unknown type is encountered.\n    if (ast.meta == TypeAst::Terminal\n            && ast.code == Type::Void\n            && CompateStringsCaseInsensitive(ast.name, std::string_view(\"void\")) != 0)\n        //throw UnimplementedError(\"Unsupported type: \" + ast.name);\n        return false;\n\n    return true;\n}\n\n\nTypeParser::TypeParser(const StringView& name)\n    : cur_(name.data())\n    , end_(name.data() + name.size())\n    , type_(nullptr)\n{\n}\n\nTypeParser::~TypeParser() = default;\n\nbool TypeParser::Parse(TypeAst* type) {\n    type_ = type;\n    open_elements_.push(type_);\n\n    size_t processed_tokens = 0;\n    do {\n        const Token & token = NextToken();\n        switch (token.type) {\n            case Token::QuotedString:\n            {\n                type_->meta = TypeAst::Terminal;\n                if (token.value.length() < 1)\n                    type_->value_string = {};\n                else\n                    type_->value_string = token.value.substr(1, token.value.length() - 2).to_string();\n                type_->code = Type::String;\n                break;\n            }\n            case Token::Name:\n                type_->meta = GetTypeMeta(token.value);\n                type_->name = token.value.to_string();\n                type_->code = GetTypeCode(type_->name);\n                break;\n            case Token::Number:\n                type_->meta = TypeAst::Number;\n                type_->value = std::stol(token.value.to_string());\n                break;\n            case Token::String:\n                type_->meta = TypeAst::String;\n                type_->value_string = std::string(token.value);\n                break;\n            case Token::LPar:\n                type_->elements.emplace_back(TypeAst());\n                open_elements_.push(type_);\n                type_ = &type_->elements.back();\n                break;\n            case Token::RPar:\n                type_ = open_elements_.top();\n                open_elements_.pop();\n                break;\n            case Token::Assign:\n            case Token::Comma:\n                type_ = open_elements_.top();\n                open_elements_.pop();\n                type_->elements.emplace_back(TypeAst());\n                open_elements_.push(type_);\n                type_ = &type_->elements.back();\n                break;\n            case Token::EOS:\n            {\n                // Ubalanced braces, brackets, etc is an error.\n                if (open_elements_.size() != 1)\n                    return false;\n\n                // Empty input string, no tokens produced\n                if (processed_tokens == 0)\n                    return false;\n\n                return ValidateAST(*type);\n            }\n            case Token::Invalid:\n                return false;\n        }\n        ++processed_tokens;\n    } while (true);\n}\n\nTypeParser::Token TypeParser::NextToken() {\n    for (; cur_ < end_; ++cur_) {\n        switch (*cur_) {\n            case ' ':\n            case '\\n':\n            case '\\t':\n            case '\\0':\n                continue;\n            case '=':\n                return Token{Token::Assign, StringView(cur_++, 1)};\n            case '(':\n                return Token{Token::LPar, StringView(cur_++, 1)};\n            case ')':\n                return Token{Token::RPar, StringView(cur_++, 1)};\n            case ',':\n                return Token{Token::Comma, StringView(cur_++, 1)};\n            case '\\'':\n            {\n                const auto end_quote_length = 1;\n                const StringView end_quote{cur_, end_quote_length};\n                // Fast forward to the closing quote.\n                const auto start = cur_++;\n                for (; cur_ < end_ - end_quote_length; ++cur_) {\n                    // TODO (nemkov): handle escaping ?\n                    if (end_quote == StringView{cur_, end_quote_length}) {\n                        cur_ += end_quote_length;\n\n                        return Token{Token::QuotedString, StringView{start, cur_}};\n                    }\n                }\n                return Token{Token::QuotedString, StringView(cur_++, 1)};\n            }\n\n            default: {\n                const char* st = cur_;\n\n                if (*cur_ == '\\'') {\n                    for (st = ++cur_; cur_ < end_; ++cur_) {\n                        if (*cur_ == '\\'') {\n                            return Token{Token::String, StringView(st, cur_++ - st)};\n                        }\n                    }\n\n                    return Token{Token::Invalid, StringView()};\n                }\n\n                if (isalpha(*cur_) || *cur_ == '_') {\n                    for (; cur_ < end_; ++cur_) {\n                        if (!isalpha(*cur_) && !isdigit(*cur_) && *cur_ != '_') {\n                            break;\n                        }\n                    }\n\n                    return Token{Token::Name, StringView(st, cur_)};\n                }\n\n                if (isdigit(*cur_) || *cur_ == '-') {\n                    for (++cur_; cur_ < end_; ++cur_) {\n                        if (!isdigit(*cur_)) {\n                            break;\n                        }\n                    }\n\n                    return Token{Token::Number, StringView(st, cur_)};\n                }\n\n                return Token{Token::Invalid, StringView()};\n            }\n        }\n    }\n\n    return Token{Token::EOS, StringView()};\n}\n\n\nconst TypeAst* ParseTypeName(const std::string& type_name) {\n    // Cache for type_name.\n    // Usually we won't have too many type names in the cache, so do not try to\n    // limit cache size.\n    static std::map<std::string, TypeAst> ast_cache;\n    static std::mutex lock;\n\n    std::lock_guard<std::mutex> guard(lock);\n    auto it = ast_cache.find(type_name);\n    if (it != ast_cache.end()) {\n        return &it->second;\n    }\n\n    auto& ast = ast_cache[type_name];\n    if (TypeParser(type_name).Parse(&ast)) {\n        return &ast;\n    }\n    ast_cache.erase(type_name);\n    return nullptr;\n}\n\n}\n\n// Path: clickhouse/version.h\n#pragma once\n\n#define CLICKHOUSE_CPP_VERSION_MAJOR 2\n#define CLICKHOUSE_CPP_VERSION_MINOR 5\n#define CLICKHOUSE_CPP_VERSION_PATCH 1\n\n#define CLICKHOUSE_CPP_VERSION_BUILD 0\n\n// Expecting each version component to be less than 99\n#define CLICKHOUSE_CPP_VERSION \\\n    CLICKHOUSE_CPP_VERSION_MAJOR *   100 * 100 * 100 \\\n    + CLICKHOUSE_CPP_VERSION_MINOR * 100 * 100 \\\n    + CLICKHOUSE_CPP_VERSION_PATCH * 100 \\\n    + CLICKHOUSE_CPP_VERSION_BUILD\n\n// Path: clickhouse/error_codes.h\n#pragma once\n\nnamespace clickhouse {\n\n// based on https://github.com/ClickHouse/ClickHouse/blob/54ae88a859507722821624476c3818152f944055/src/Common/ErrorCodes.cpp\n// (master on 28 Feb 2024)\nenum ErrorCodes {\n    OK = 0,\n    UNSUPPORTED_METHOD = 1,\n    UNSUPPORTED_PARAMETER = 2,\n    UNEXPECTED_END_OF_FILE = 3,\n    EXPECTED_END_OF_FILE = 4,\n    CANNOT_PARSE_TEXT = 6,\n    INCORRECT_NUMBER_OF_COLUMNS = 7,\n    THERE_IS_NO_COLUMN = 8,\n    SIZES_OF_COLUMNS_DOESNT_MATCH = 9,\n    NOT_FOUND_COLUMN_IN_BLOCK = 10,\n    POSITION_OUT_OF_BOUND = 11,\n    PARAMETER_OUT_OF_BOUND = 12,\n    SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH = 13,\n    DUPLICATE_COLUMN = 15,\n    NO_SUCH_COLUMN_IN_TABLE = 16,\n    SIZE_OF_FIXED_STRING_DOESNT_MATCH = 19,\n    NUMBER_OF_COLUMNS_DOESNT_MATCH = 20,\n    CANNOT_READ_FROM_ISTREAM = 23,\n    CANNOT_WRITE_TO_OSTREAM = 24,\n    CANNOT_PARSE_ESCAPE_SEQUENCE = 25,\n    CANNOT_PARSE_QUOTED_STRING = 26,\n    CANNOT_PARSE_INPUT_ASSERTION_FAILED = 27,\n    CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER = 28,\n    ATTEMPT_TO_READ_AFTER_EOF = 32,\n    CANNOT_READ_ALL_DATA = 33,\n    TOO_MANY_ARGUMENTS_FOR_FUNCTION = 34,\n    TOO_FEW_ARGUMENTS_FOR_FUNCTION = 35,\n    BAD_ARGUMENTS = 36,\n    UNKNOWN_ELEMENT_IN_AST = 37,\n    CANNOT_PARSE_DATE = 38,\n    TOO_LARGE_SIZE_COMPRESSED = 39,\n    CHECKSUM_DOESNT_MATCH = 40,\n    CANNOT_PARSE_DATETIME = 41,\n    NUMBER_OF_ARGUMENTS_DOESNT_MATCH = 42,\n    ILLEGAL_TYPE_OF_ARGUMENT = 43,\n    ILLEGAL_COLUMN = 44,\n    UNKNOWN_FUNCTION = 46,\n    UNKNOWN_IDENTIFIER = 47,\n    NOT_IMPLEMENTED = 48,\n    LOGICAL_ERROR = 49,\n    UNKNOWN_TYPE = 50,\n    EMPTY_LIST_OF_COLUMNS_QUERIED = 51,\n    COLUMN_QUERIED_MORE_THAN_ONCE = 52,\n    TYPE_MISMATCH = 53,\n    STORAGE_REQUIRES_PARAMETER = 55,\n    UNKNOWN_STORAGE = 56,\n    TABLE_ALREADY_EXISTS = 57,\n    TABLE_METADATA_ALREADY_EXISTS = 58,\n    ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER = 59,\n    UNKNOWN_TABLE = 60,\n    SYNTAX_ERROR = 62,\n    UNKNOWN_AGGREGATE_FUNCTION = 63,\n    CANNOT_GET_SIZE_OF_FIELD = 68,\n    ARGUMENT_OUT_OF_BOUND = 69,\n    CANNOT_CONVERT_TYPE = 70,\n    CANNOT_WRITE_AFTER_END_OF_BUFFER = 71,\n    CANNOT_PARSE_NUMBER = 72,\n    UNKNOWN_FORMAT = 73,\n    CANNOT_READ_FROM_FILE_DESCRIPTOR = 74,\n    CANNOT_WRITE_TO_FILE_DESCRIPTOR = 75,\n    CANNOT_OPEN_FILE = 76,\n    CANNOT_CLOSE_FILE = 77,\n    UNKNOWN_TYPE_OF_QUERY = 78,\n    INCORRECT_FILE_NAME = 79,\n    INCORRECT_QUERY = 80,\n    UNKNOWN_DATABASE = 81,\n    DATABASE_ALREADY_EXISTS = 82,\n    DIRECTORY_DOESNT_EXIST = 83,\n    DIRECTORY_ALREADY_EXISTS = 84,\n    FORMAT_IS_NOT_SUITABLE_FOR_INPUT = 85,\n    RECEIVED_ERROR_FROM_REMOTE_IO_SERVER = 86,\n    CANNOT_SEEK_THROUGH_FILE = 87,\n    CANNOT_TRUNCATE_FILE = 88,\n    UNKNOWN_COMPRESSION_METHOD = 89,\n    EMPTY_LIST_OF_COLUMNS_PASSED = 90,\n    SIZES_OF_MARKS_FILES_ARE_INCONSISTENT = 91,\n    EMPTY_DATA_PASSED = 92,\n    UNKNOWN_AGGREGATED_DATA_VARIANT = 93,\n    CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS = 94,\n    CANNOT_READ_FROM_SOCKET = 95,\n    CANNOT_WRITE_TO_SOCKET = 96,\n    UNKNOWN_PACKET_FROM_CLIENT = 99,\n    UNKNOWN_PACKET_FROM_SERVER = 100,\n    UNEXPECTED_PACKET_FROM_CLIENT = 101,\n    UNEXPECTED_PACKET_FROM_SERVER = 102,\n    TOO_SMALL_BUFFER_SIZE = 104,\n    FILE_DOESNT_EXIST = 107,\n    NO_DATA_TO_INSERT = 108,\n    CANNOT_BLOCK_SIGNAL = 109,\n    CANNOT_UNBLOCK_SIGNAL = 110,\n    CANNOT_MANIPULATE_SIGSET = 111,\n    CANNOT_WAIT_FOR_SIGNAL = 112,\n    THERE_IS_NO_SESSION = 113,\n    CANNOT_CLOCK_GETTIME = 114,\n    UNKNOWN_SETTING = 115,\n    THERE_IS_NO_DEFAULT_VALUE = 116,\n    INCORRECT_DATA = 117,\n    ENGINE_REQUIRED = 119,\n    CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE = 120,\n    UNSUPPORTED_JOIN_KEYS = 121,\n    INCOMPATIBLE_COLUMNS = 122,\n    UNKNOWN_TYPE_OF_AST_NODE = 123,\n    INCORRECT_ELEMENT_OF_SET = 124,\n    INCORRECT_RESULT_OF_SCALAR_SUBQUERY = 125,\n    ILLEGAL_INDEX = 127,\n    TOO_LARGE_ARRAY_SIZE = 128,\n    FUNCTION_IS_SPECIAL = 129,\n    CANNOT_READ_ARRAY_FROM_TEXT = 130,\n    TOO_LARGE_STRING_SIZE = 131,\n    AGGREGATE_FUNCTION_DOESNT_ALLOW_PARAMETERS = 133,\n    PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS = 134,\n    ZERO_ARRAY_OR_TUPLE_INDEX = 135,\n    UNKNOWN_ELEMENT_IN_CONFIG = 137,\n    EXCESSIVE_ELEMENT_IN_CONFIG = 138,\n    NO_ELEMENTS_IN_CONFIG = 139,\n    SAMPLING_NOT_SUPPORTED = 141,\n    NOT_FOUND_NODE = 142,\n    UNKNOWN_OVERFLOW_MODE = 145,\n    UNKNOWN_DIRECTION_OF_SORTING = 152,\n    ILLEGAL_DIVISION = 153,\n    DICTIONARIES_WAS_NOT_LOADED = 156,\n    TOO_MANY_ROWS = 158,\n    TIMEOUT_EXCEEDED = 159,\n    TOO_SLOW = 160,\n    TOO_MANY_COLUMNS = 161,\n    TOO_DEEP_SUBQUERIES = 162,\n    READONLY = 164,\n    TOO_MANY_TEMPORARY_COLUMNS = 165,\n    TOO_MANY_TEMPORARY_NON_CONST_COLUMNS = 166,\n    TOO_DEEP_AST = 167,\n    TOO_BIG_AST = 168,\n    BAD_TYPE_OF_FIELD = 169,\n    BAD_GET = 170,\n    CANNOT_CREATE_DIRECTORY = 172,\n    CANNOT_ALLOCATE_MEMORY = 173,\n    CYCLIC_ALIASES = 174,\n    MULTIPLE_EXPRESSIONS_FOR_ALIAS = 179,\n    THERE_IS_NO_PROFILE = 180,\n    ILLEGAL_FINAL = 181,\n    ILLEGAL_PREWHERE = 182,\n    UNEXPECTED_EXPRESSION = 183,\n    ILLEGAL_AGGREGATION = 184,\n    UNSUPPORTED_COLLATION_LOCALE = 186,\n    COLLATION_COMPARISON_FAILED = 187,\n    SIZES_OF_ARRAYS_DONT_MATCH = 190,\n    SET_SIZE_LIMIT_EXCEEDED = 191,\n    UNKNOWN_USER = 192,\n    WRONG_PASSWORD = 193,\n    REQUIRED_PASSWORD = 194,\n    IP_ADDRESS_NOT_ALLOWED = 195,\n    UNKNOWN_ADDRESS_PATTERN_TYPE = 196,\n    DNS_ERROR = 198,\n    UNKNOWN_QUOTA = 199,\n    QUOTA_EXCEEDED = 201,\n    TOO_MANY_SIMULTANEOUS_QUERIES = 202,\n    NO_FREE_CONNECTION = 203,\n    CANNOT_FSYNC = 204,\n    ALIAS_REQUIRED = 206,\n    AMBIGUOUS_IDENTIFIER = 207,\n    EMPTY_NESTED_TABLE = 208,\n    SOCKET_TIMEOUT = 209,\n    NETWORK_ERROR = 210,\n    EMPTY_QUERY = 211,\n    UNKNOWN_LOAD_BALANCING = 212,\n    UNKNOWN_TOTALS_MODE = 213,\n    CANNOT_STATVFS = 214,\n    NOT_AN_AGGREGATE = 215,\n    QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING = 216,\n    CLIENT_HAS_CONNECTED_TO_WRONG_PORT = 217,\n    TABLE_IS_DROPPED = 218,\n    DATABASE_NOT_EMPTY = 219,\n    DUPLICATE_INTERSERVER_IO_ENDPOINT = 220,\n    NO_SUCH_INTERSERVER_IO_ENDPOINT = 221,\n    UNEXPECTED_AST_STRUCTURE = 223,\n    REPLICA_IS_ALREADY_ACTIVE = 224,\n    NO_ZOOKEEPER = 225,\n    NO_FILE_IN_DATA_PART = 226,\n    UNEXPECTED_FILE_IN_DATA_PART = 227,\n    BAD_SIZE_OF_FILE_IN_DATA_PART = 228,\n    QUERY_IS_TOO_LARGE = 229,\n    NOT_FOUND_EXPECTED_DATA_PART = 230,\n    TOO_MANY_UNEXPECTED_DATA_PARTS = 231,\n    NO_SUCH_DATA_PART = 232,\n    BAD_DATA_PART_NAME = 233,\n    NO_REPLICA_HAS_PART = 234,\n    DUPLICATE_DATA_PART = 235,\n    ABORTED = 236,\n    NO_REPLICA_NAME_GIVEN = 237,\n    FORMAT_VERSION_TOO_OLD = 238,\n    CANNOT_MUNMAP = 239,\n    CANNOT_MREMAP = 240,\n    MEMORY_LIMIT_EXCEEDED = 241,\n    TABLE_IS_READ_ONLY = 242,\n    NOT_ENOUGH_SPACE = 243,\n    UNEXPECTED_ZOOKEEPER_ERROR = 244,\n    CORRUPTED_DATA = 246,\n    INVALID_PARTITION_VALUE = 248,\n    NO_SUCH_REPLICA = 251,\n    TOO_MANY_PARTS = 252,\n    REPLICA_ALREADY_EXISTS = 253,\n    NO_ACTIVE_REPLICAS = 254,\n    TOO_MANY_RETRIES_TO_FETCH_PARTS = 255,\n    PARTITION_ALREADY_EXISTS = 256,\n    PARTITION_DOESNT_EXIST = 257,\n    UNION_ALL_RESULT_STRUCTURES_MISMATCH = 258,\n    CLIENT_OUTPUT_FORMAT_SPECIFIED = 260,\n    UNKNOWN_BLOCK_INFO_FIELD = 261,\n    BAD_COLLATION = 262,\n    CANNOT_COMPILE_CODE = 263,\n    INCOMPATIBLE_TYPE_OF_JOIN = 264,\n    NO_AVAILABLE_REPLICA = 265,\n    MISMATCH_REPLICAS_DATA_SOURCES = 266,\n    INFINITE_LOOP = 269,\n    CANNOT_COMPRESS = 270,\n    CANNOT_DECOMPRESS = 271,\n    CANNOT_IO_SUBMIT = 272,\n    CANNOT_IO_GETEVENTS = 273,\n    AIO_READ_ERROR = 274,\n    AIO_WRITE_ERROR = 275,\n    INDEX_NOT_USED = 277,\n    ALL_CONNECTION_TRIES_FAILED = 279,\n    NO_AVAILABLE_DATA = 280,\n    DICTIONARY_IS_EMPTY = 281,\n    INCORRECT_INDEX = 282,\n    UNKNOWN_DISTRIBUTED_PRODUCT_MODE = 283,\n    WRONG_GLOBAL_SUBQUERY = 284,\n    TOO_FEW_LIVE_REPLICAS = 285,\n    UNSATISFIED_QUORUM_FOR_PREVIOUS_WRITE = 286,\n    UNKNOWN_FORMAT_VERSION = 287,\n    DISTRIBUTED_IN_JOIN_SUBQUERY_DENIED = 288,\n    REPLICA_IS_NOT_IN_QUORUM = 289,\n    LIMIT_EXCEEDED = 290,\n    DATABASE_ACCESS_DENIED = 291,\n    MONGODB_CANNOT_AUTHENTICATE = 293,\n    RECEIVED_EMPTY_DATA = 295,\n    SHARD_HAS_NO_CONNECTIONS = 297,\n    CANNOT_PIPE = 298,\n    CANNOT_FORK = 299,\n    CANNOT_DLSYM = 300,\n    CANNOT_CREATE_CHILD_PROCESS = 301,\n    CHILD_WAS_NOT_EXITED_NORMALLY = 302,\n    CANNOT_SELECT = 303,\n    CANNOT_WAITPID = 304,\n    TABLE_WAS_NOT_DROPPED = 305,\n    TOO_DEEP_RECURSION = 306,\n    TOO_MANY_BYTES = 307,\n    UNEXPECTED_NODE_IN_ZOOKEEPER = 308,\n    FUNCTION_CANNOT_HAVE_PARAMETERS = 309,\n    INVALID_CONFIG_PARAMETER = 318,\n    UNKNOWN_STATUS_OF_INSERT = 319,\n    VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE = 321,\n    UNKNOWN_DATABASE_ENGINE = 336,\n    UNFINISHED = 341,\n    METADATA_MISMATCH = 342,\n    SUPPORT_IS_DISABLED = 344,\n    TABLE_DIFFERS_TOO_MUCH = 345,\n    CANNOT_CONVERT_CHARSET = 346,\n    CANNOT_LOAD_CONFIG = 347,\n    CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN = 349,\n    AMBIGUOUS_COLUMN_NAME = 352,\n    INDEX_OF_POSITIONAL_ARGUMENT_IS_OUT_OF_RANGE = 353,\n    ZLIB_INFLATE_FAILED = 354,\n    ZLIB_DEFLATE_FAILED = 355,\n    INTO_OUTFILE_NOT_ALLOWED = 358,\n    TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT = 359,\n    CANNOT_CREATE_CHARSET_CONVERTER = 360,\n    SEEK_POSITION_OUT_OF_BOUND = 361,\n    CURRENT_WRITE_BUFFER_IS_EXHAUSTED = 362,\n    CANNOT_CREATE_IO_BUFFER = 363,\n    RECEIVED_ERROR_TOO_MANY_REQUESTS = 364,\n    SIZES_OF_NESTED_COLUMNS_ARE_INCONSISTENT = 366,\n    ALL_REPLICAS_ARE_STALE = 369,\n    DATA_TYPE_CANNOT_BE_USED_IN_TABLES = 370,\n    INCONSISTENT_CLUSTER_DEFINITION = 371,\n    SESSION_NOT_FOUND = 372,\n    SESSION_IS_LOCKED = 373,\n    INVALID_SESSION_TIMEOUT = 374,\n    CANNOT_DLOPEN = 375,\n    CANNOT_PARSE_UUID = 376,\n    ILLEGAL_SYNTAX_FOR_DATA_TYPE = 377,\n    DATA_TYPE_CANNOT_HAVE_ARGUMENTS = 378,\n    CANNOT_KILL = 380,\n    HTTP_LENGTH_REQUIRED = 381,\n    CANNOT_LOAD_CATBOOST_MODEL = 382,\n    CANNOT_APPLY_CATBOOST_MODEL = 383,\n    PART_IS_TEMPORARILY_LOCKED = 384,\n    MULTIPLE_STREAMS_REQUIRED = 385,\n    NO_COMMON_TYPE = 386,\n    DICTIONARY_ALREADY_EXISTS = 387,\n    CANNOT_ASSIGN_OPTIMIZE = 388,\n    INSERT_WAS_DEDUPLICATED = 389,\n    CANNOT_GET_CREATE_TABLE_QUERY = 390,\n    EXTERNAL_LIBRARY_ERROR = 391,\n    QUERY_IS_PROHIBITED = 392,\n    THERE_IS_NO_QUERY = 393,\n    QUERY_WAS_CANCELLED = 394,\n    FUNCTION_THROW_IF_VALUE_IS_NON_ZERO = 395,\n    TOO_MANY_ROWS_OR_BYTES = 396,\n    QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW = 397,\n    UNKNOWN_MUTATION_COMMAND = 398,\n    FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT = 399,\n    CANNOT_STAT = 400,\n    FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME = 401,\n    CANNOT_IOSETUP = 402,\n    INVALID_JOIN_ON_EXPRESSION = 403,\n    BAD_ODBC_CONNECTION_STRING = 404,\n    TOP_AND_LIMIT_TOGETHER = 406,\n    DECIMAL_OVERFLOW = 407,\n    BAD_REQUEST_PARAMETER = 408,\n    EXTERNAL_SERVER_IS_NOT_RESPONDING = 410,\n    PTHREAD_ERROR = 411,\n    NETLINK_ERROR = 412,\n    CANNOT_SET_SIGNAL_HANDLER = 413,\n    ALL_REPLICAS_LOST = 415,\n    REPLICA_STATUS_CHANGED = 416,\n    EXPECTED_ALL_OR_ANY = 417,\n    UNKNOWN_JOIN = 418,\n    MULTIPLE_ASSIGNMENTS_TO_COLUMN = 419,\n    CANNOT_UPDATE_COLUMN = 420,\n    CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421,\n    UNSUPPORTED_URI_SCHEME = 422,\n    CANNOT_GETTIMEOFDAY = 423,\n    CANNOT_LINK = 424,\n    SYSTEM_ERROR = 425,\n    CANNOT_COMPILE_REGEXP = 427,\n    FAILED_TO_GETPWUID = 429,\n    MISMATCHING_USERS_FOR_PROCESS_AND_DATA = 430,\n    ILLEGAL_SYNTAX_FOR_CODEC_TYPE = 431,\n    UNKNOWN_CODEC = 432,\n    ILLEGAL_CODEC_PARAMETER = 433,\n    CANNOT_PARSE_PROTOBUF_SCHEMA = 434,\n    NO_COLUMN_SERIALIZED_TO_REQUIRED_PROTOBUF_FIELD = 435,\n    PROTOBUF_BAD_CAST = 436,\n    PROTOBUF_FIELD_NOT_REPEATED = 437,\n    DATA_TYPE_CANNOT_BE_PROMOTED = 438,\n    CANNOT_SCHEDULE_TASK = 439,\n    INVALID_LIMIT_EXPRESSION = 440,\n    CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING = 441,\n    BAD_DATABASE_FOR_TEMPORARY_TABLE = 442,\n    NO_COLUMNS_SERIALIZED_TO_PROTOBUF_FIELDS = 443,\n    UNKNOWN_PROTOBUF_FORMAT = 444,\n    CANNOT_MPROTECT = 445,\n    FUNCTION_NOT_ALLOWED = 446,\n    HYPERSCAN_CANNOT_SCAN_TEXT = 447,\n    BROTLI_READ_FAILED = 448,\n    BROTLI_WRITE_FAILED = 449,\n    BAD_TTL_EXPRESSION = 450,\n    BAD_TTL_FILE = 451,\n    SETTING_CONSTRAINT_VIOLATION = 452,\n    MYSQL_CLIENT_INSUFFICIENT_CAPABILITIES = 453,\n    OPENSSL_ERROR = 454,\n    SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY = 455,\n    UNKNOWN_QUERY_PARAMETER = 456,\n    BAD_QUERY_PARAMETER = 457,\n    CANNOT_UNLINK = 458,\n    CANNOT_SET_THREAD_PRIORITY = 459,\n    CANNOT_CREATE_TIMER = 460,\n    CANNOT_SET_TIMER_PERIOD = 461,\n    CANNOT_FCNTL = 463,\n    CANNOT_PARSE_ELF = 464,\n    CANNOT_PARSE_DWARF = 465,\n    INSECURE_PATH = 466,\n    CANNOT_PARSE_BOOL = 467,\n    CANNOT_PTHREAD_ATTR = 468,\n    VIOLATED_CONSTRAINT = 469,\n    QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW = 470,\n    INVALID_SETTING_VALUE = 471,\n    READONLY_SETTING = 472,\n    DEADLOCK_AVOIDED = 473,\n    INVALID_TEMPLATE_FORMAT = 474,\n    INVALID_WITH_FILL_EXPRESSION = 475,\n    WITH_TIES_WITHOUT_ORDER_BY = 476,\n    INVALID_USAGE_OF_INPUT = 477,\n    UNKNOWN_POLICY = 478,\n    UNKNOWN_DISK = 479,\n    UNKNOWN_PROTOCOL = 480,\n    PATH_ACCESS_DENIED = 481,\n    DICTIONARY_ACCESS_DENIED = 482,\n    TOO_MANY_REDIRECTS = 483,\n    INTERNAL_REDIS_ERROR = 484,\n    CANNOT_GET_CREATE_DICTIONARY_QUERY = 487,\n    INCORRECT_DICTIONARY_DEFINITION = 489,\n    CANNOT_FORMAT_DATETIME = 490,\n    UNACCEPTABLE_URL = 491,\n    ACCESS_ENTITY_NOT_FOUND = 492,\n    ACCESS_ENTITY_ALREADY_EXISTS = 493,\n    ACCESS_STORAGE_READONLY = 495,\n    QUOTA_REQUIRES_CLIENT_KEY = 496,\n    ACCESS_DENIED = 497,\n    LIMIT_BY_WITH_TIES_IS_NOT_SUPPORTED = 498,\n    S3_ERROR = 499,\n    AZURE_BLOB_STORAGE_ERROR = 500,\n    CANNOT_CREATE_DATABASE = 501,\n    CANNOT_SIGQUEUE = 502,\n    AGGREGATE_FUNCTION_THROW = 503,\n    FILE_ALREADY_EXISTS = 504,\n    UNABLE_TO_SKIP_UNUSED_SHARDS = 507,\n    UNKNOWN_ACCESS_TYPE = 508,\n    INVALID_GRANT = 509,\n    CACHE_DICTIONARY_UPDATE_FAIL = 510,\n    UNKNOWN_ROLE = 511,\n    SET_NON_GRANTED_ROLE = 512,\n    UNKNOWN_PART_TYPE = 513,\n    ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND = 514,\n    INCORRECT_ACCESS_ENTITY_DEFINITION = 515,\n    AUTHENTICATION_FAILED = 516,\n    CANNOT_ASSIGN_ALTER = 517,\n    CANNOT_COMMIT_OFFSET = 518,\n    NO_REMOTE_SHARD_AVAILABLE = 519,\n    CANNOT_DETACH_DICTIONARY_AS_TABLE = 520,\n    ATOMIC_RENAME_FAIL = 521,\n    UNKNOWN_ROW_POLICY = 523,\n    ALTER_OF_COLUMN_IS_FORBIDDEN = 524,\n    INCORRECT_DISK_INDEX = 525,\n    NO_SUITABLE_FUNCTION_IMPLEMENTATION = 527,\n    CASSANDRA_INTERNAL_ERROR = 528,\n    NOT_A_LEADER = 529,\n    CANNOT_CONNECT_RABBITMQ = 530,\n    CANNOT_FSTAT = 531,\n    LDAP_ERROR = 532,\n    UNKNOWN_RAID_TYPE = 535,\n    CANNOT_RESTORE_FROM_FIELD_DUMP = 536,\n    ILLEGAL_MYSQL_VARIABLE = 537,\n    MYSQL_SYNTAX_ERROR = 538,\n    CANNOT_BIND_RABBITMQ_EXCHANGE = 539,\n    CANNOT_DECLARE_RABBITMQ_EXCHANGE = 540,\n    CANNOT_CREATE_RABBITMQ_QUEUE_BINDING = 541,\n    CANNOT_REMOVE_RABBITMQ_EXCHANGE = 542,\n    UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL = 543,\n    ROW_AND_ROWS_TOGETHER = 544,\n    FIRST_AND_NEXT_TOGETHER = 545,\n    NO_ROW_DELIMITER = 546,\n    INVALID_RAID_TYPE = 547,\n    UNKNOWN_VOLUME = 548,\n    DATA_TYPE_CANNOT_BE_USED_IN_KEY = 549,\n    UNRECOGNIZED_ARGUMENTS = 552,\n    LZMA_STREAM_ENCODER_FAILED = 553,\n    LZMA_STREAM_DECODER_FAILED = 554,\n    ROCKSDB_ERROR = 555,\n    SYNC_MYSQL_USER_ACCESS_ERROR = 556,\n    UNKNOWN_UNION = 557,\n    EXPECTED_ALL_OR_DISTINCT = 558,\n    INVALID_GRPC_QUERY_INFO = 559,\n    ZSTD_ENCODER_FAILED = 560,\n    ZSTD_DECODER_FAILED = 561,\n    TLD_LIST_NOT_FOUND = 562,\n    CANNOT_READ_MAP_FROM_TEXT = 563,\n    INTERSERVER_SCHEME_DOESNT_MATCH = 564,\n    TOO_MANY_PARTITIONS = 565,\n    CANNOT_RMDIR = 566,\n    DUPLICATED_PART_UUIDS = 567,\n    RAFT_ERROR = 568,\n    MULTIPLE_COLUMNS_SERIALIZED_TO_SAME_PROTOBUF_FIELD = 569,\n    DATA_TYPE_INCOMPATIBLE_WITH_PROTOBUF_FIELD = 570,\n    DATABASE_REPLICATION_FAILED = 571,\n    TOO_MANY_QUERY_PLAN_OPTIMIZATIONS = 572,\n    EPOLL_ERROR = 573,\n    DISTRIBUTED_TOO_MANY_PENDING_BYTES = 574,\n    UNKNOWN_SNAPSHOT = 575,\n    KERBEROS_ERROR = 576,\n    INVALID_SHARD_ID = 577,\n    INVALID_FORMAT_INSERT_QUERY_WITH_DATA = 578,\n    INCORRECT_PART_TYPE = 579,\n    CANNOT_SET_ROUNDING_MODE = 580,\n    TOO_LARGE_DISTRIBUTED_DEPTH = 581,\n    NO_SUCH_PROJECTION_IN_TABLE = 582,\n    ILLEGAL_PROJECTION = 583,\n    PROJECTION_NOT_USED = 584,\n    CANNOT_PARSE_YAML = 585,\n    CANNOT_CREATE_FILE = 586,\n    CONCURRENT_ACCESS_NOT_SUPPORTED = 587,\n    DISTRIBUTED_BROKEN_BATCH_INFO = 588,\n    DISTRIBUTED_BROKEN_BATCH_FILES = 589,\n    CANNOT_SYSCONF = 590,\n    SQLITE_ENGINE_ERROR = 591,\n    DATA_ENCRYPTION_ERROR = 592,\n    ZERO_COPY_REPLICATION_ERROR = 593,\n    BZIP2_STREAM_DECODER_FAILED = 594,\n    BZIP2_STREAM_ENCODER_FAILED = 595,\n    INTERSECT_OR_EXCEPT_RESULT_STRUCTURES_MISMATCH = 596,\n    NO_SUCH_ERROR_CODE = 597,\n    BACKUP_ALREADY_EXISTS = 598,\n    BACKUP_NOT_FOUND = 599,\n    BACKUP_VERSION_NOT_SUPPORTED = 600,\n    BACKUP_DAMAGED = 601,\n    NO_BASE_BACKUP = 602,\n    WRONG_BASE_BACKUP = 603,\n    BACKUP_ENTRY_ALREADY_EXISTS = 604,\n    BACKUP_ENTRY_NOT_FOUND = 605,\n    BACKUP_IS_EMPTY = 606,\n    CANNOT_RESTORE_DATABASE = 607,\n    CANNOT_RESTORE_TABLE = 608,\n    FUNCTION_ALREADY_EXISTS = 609,\n    CANNOT_DROP_FUNCTION = 610,\n    CANNOT_CREATE_RECURSIVE_FUNCTION = 611,\n    POSTGRESQL_CONNECTION_FAILURE = 614,\n    CANNOT_ADVISE = 615,\n    UNKNOWN_READ_METHOD = 616,\n    LZ4_ENCODER_FAILED = 617,\n    LZ4_DECODER_FAILED = 618,\n    POSTGRESQL_REPLICATION_INTERNAL_ERROR = 619,\n    QUERY_NOT_ALLOWED = 620,\n    CANNOT_NORMALIZE_STRING = 621,\n    CANNOT_PARSE_CAPN_PROTO_SCHEMA = 622,\n    CAPN_PROTO_BAD_CAST = 623,\n    BAD_FILE_TYPE = 624,\n    IO_SETUP_ERROR = 625,\n    CANNOT_SKIP_UNKNOWN_FIELD = 626,\n    BACKUP_ENGINE_NOT_FOUND = 627,\n    OFFSET_FETCH_WITHOUT_ORDER_BY = 628,\n    HTTP_RANGE_NOT_SATISFIABLE = 629,\n    HAVE_DEPENDENT_OBJECTS = 630,\n    UNKNOWN_FILE_SIZE = 631,\n    UNEXPECTED_DATA_AFTER_PARSED_VALUE = 632,\n    QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW = 633,\n    MONGODB_ERROR = 634,\n    CANNOT_POLL = 635,\n    CANNOT_EXTRACT_TABLE_STRUCTURE = 636,\n    INVALID_TABLE_OVERRIDE = 637,\n    SNAPPY_UNCOMPRESS_FAILED = 638,\n    SNAPPY_COMPRESS_FAILED = 639,\n    NO_HIVEMETASTORE = 640,\n    CANNOT_APPEND_TO_FILE = 641,\n    CANNOT_PACK_ARCHIVE = 642,\n    CANNOT_UNPACK_ARCHIVE = 643,\n    NUMBER_OF_DIMENSIONS_MISMATCHED = 645,\n    CANNOT_BACKUP_TABLE = 647,\n    WRONG_DDL_RENAMING_SETTINGS = 648,\n    INVALID_TRANSACTION = 649,\n    SERIALIZATION_ERROR = 650,\n    CAPN_PROTO_BAD_TYPE = 651,\n    ONLY_NULLS_WHILE_READING_SCHEMA = 652,\n    CANNOT_PARSE_BACKUP_SETTINGS = 653,\n    WRONG_BACKUP_SETTINGS = 654,\n    FAILED_TO_SYNC_BACKUP_OR_RESTORE = 655,\n    UNKNOWN_STATUS_OF_TRANSACTION = 659,\n    HDFS_ERROR = 660,\n    CANNOT_SEND_SIGNAL = 661,\n    FS_METADATA_ERROR = 662,\n    INCONSISTENT_METADATA_FOR_BACKUP = 663,\n    ACCESS_STORAGE_DOESNT_ALLOW_BACKUP = 664,\n    CANNOT_CONNECT_NATS = 665,\n    NOT_INITIALIZED = 667,\n    INVALID_STATE = 668,\n    NAMED_COLLECTION_DOESNT_EXIST = 669,\n    NAMED_COLLECTION_ALREADY_EXISTS = 670,\n    NAMED_COLLECTION_IS_IMMUTABLE = 671,\n    INVALID_SCHEDULER_NODE = 672,\n    RESOURCE_ACCESS_DENIED = 673,\n    RESOURCE_NOT_FOUND = 674,\n    CANNOT_PARSE_IPV4 = 675,\n    CANNOT_PARSE_IPV6 = 676,\n    THREAD_WAS_CANCELED = 677,\n    IO_URING_INIT_FAILED = 678,\n    IO_URING_SUBMIT_ERROR = 679,\n    MIXED_ACCESS_PARAMETER_TYPES = 690,\n    UNKNOWN_ELEMENT_OF_ENUM = 691,\n    TOO_MANY_MUTATIONS = 692,\n    AWS_ERROR = 693,\n    ASYNC_LOAD_CYCLE = 694,\n    ASYNC_LOAD_FAILED = 695,\n    ASYNC_LOAD_CANCELED = 696,\n    CANNOT_RESTORE_TO_NONENCRYPTED_DISK = 697,\n    INVALID_REDIS_STORAGE_TYPE = 698,\n    INVALID_REDIS_TABLE_STRUCTURE = 699,\n    USER_SESSION_LIMIT_EXCEEDED = 700,\n    CLUSTER_DOESNT_EXIST = 701,\n    CLIENT_INFO_DOES_NOT_MATCH = 702,\n    INVALID_IDENTIFIER = 703,\n    QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS = 704,\n    // TABLE_NOT_EMPTY = 705,\n    // LIBSSH_ERROR = 706,\n    // GCP_ERROR = 707,\n    // ILLEGAL_STATISTIC = 708,\n    // CANNOT_GET_REPLICATED_DATABASE_SNAPSHOT = 709,\n    // FAULT_INJECTED = 710,\n    // FILECACHE_ACCESS_DENIED = 711,\n    // TOO_MANY_MATERIALIZED_VIEWS = 712,\n    // UNEXPECTED_CLUSTER = 714,\n    // CANNOT_DETECT_FORMAT = 715,\n    // CANNOT_FORGET_PARTITION = 716,\n    KEEPER_EXCEPTION = 999,\n    POCO_EXCEPTION = 1000,\n    STD_EXCEPTION = 1001,\n    UNKNOWN_EXCEPTION = 1002,\n};\n\n}\n\n// Path: clickhouse/base/string_utils.h\n#pragma once\n\n#include \"string_view.h\"\n\n#include <sstream>\n#include <string>\n\nnamespace clickhouse {\n\ntemplate <typename T>\n[[deprecated(\"Not used by clickhosue-cpp itself, and will be removed in next major release (3.0) \")]]\ninline T FromString(const std::string& s) {\n   std::istringstream iss(s);\n   T result;\n   iss >> result;\n   return result;\n}\n\ntemplate <typename T>\n[[deprecated(\"Not used by clickhosue-cpp itself, and will be removed in next major release (3.0) \")]]\ninline T FromString(const StringView& s) {\n   std::istringstream iss((std::string(s)));\n   T result;\n   iss >> result;\n   return result;\n}\n\n}\n\n", "needle_token_start": 747, "needle_token_end": 871, "code_context_ntokens": 14467, "output": ["```cpp\ntemplate <typename L, typename R>\ninline int CompateStringsCaseInsensitive(const L& left, const R& right) {\n    int64_t size_diff = left.size() - right.size();\n    if (size_diff != 0)\n        return size_diff > 0 ? 1 : -1;\n\n#if defined _win_\n    return _strnicmp(left.data(), right.data(), left.size());\n#else\n    return strncasecmp(left.data(), right.data(), left.size());\n#endif\n}\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "ContainerToString", "language": "cpp", "path": "clickhouse/columns/itemview.cpp", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a container of elements into a single string representation, with elements separated by a specified delimiter.\n2. **Input**: It takes a container (like a vector, list, etc.) and an optional string delimiter (default is \", \").\n3. **Output**: It returns a string that concatenates all the elements of the container, separated by the specified delimiter.\n4. **Procedure**: The function iterates through each element of the container, appending each element to a string stream. If the current element is not the last in the container, the delimiter is appended after the element. The loop avoids incrementing the iterator inside the loop body to handle the delimiter logic correctly.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/columns/lowcardinality.cpp\n#include \"lowcardinality.h\"\n\n#include \"string.h\"\n#include \"nullable.h\"\n#include \"../base/wire_format.h\"\n\n#include <city.h>\n\n#include <functional>\n#include <string_view>\n#include <type_traits>\n\n#include <cassert>\n\nnamespace {\nusing namespace clickhouse;\n\nenum KeySerializationVersion {\n    SharedDictionariesWithAdditionalKeys = 1,\n};\n\nenum IndexType {\n    UInt8 = 0,\n    UInt16,\n    UInt32,\n    UInt64,\n};\n\nconstexpr uint64_t IndexTypeMask = 0b11111111;\n\nenum IndexFlag {\n    /// Need to read dictionary if it wasn't.\n    NeedGlobalDictionaryBit = 1u << 8u,\n    /// Need to read additional keys. Additional keys are stored before indexes as value N and N keys after them.\n    HasAdditionalKeysBit = 1u << 9u,\n    /// Need to update dictionary. It means that previous granule has different dictionary.\n    NeedUpdateDictionary = 1u << 10u\n};\n\nColumnRef createIndexColumn(IndexType type) {\n    switch (type) {\n        case IndexType::UInt8:\n            return std::make_shared<ColumnUInt8>();\n        case IndexType::UInt16:\n            return std::make_shared<ColumnUInt16>();\n        case IndexType::UInt32:\n            return std::make_shared<ColumnUInt32>();\n        case IndexType::UInt64:\n            return std::make_shared<ColumnUInt64>();\n    }\n\n    throw ValidationError(\"Invalid LowCardinality index type value: \" + std::to_string(static_cast<uint64_t>(type)));\n}\n\nIndexType indexTypeFromIndexColumn(const Column & index_column) {\n    switch (index_column.Type()->GetCode()) {\n        case Type::UInt8:\n            return IndexType::UInt8;\n        case Type::UInt16:\n            return IndexType::UInt16;\n        case Type::UInt32:\n            return IndexType::UInt32;\n        case Type::UInt64:\n            return IndexType::UInt64;\n        default:\n            throw ValidationError(\"Invalid index column type for LowCardinality column:\" + index_column.Type()->GetName());\n    }\n}\n\ntemplate <typename ResultColumnType, typename ColumnType>\ninline const ResultColumnType & column_down_cast(const ColumnType & c) {\n    return dynamic_cast<const ResultColumnType &>(c);\n}\n\ntemplate <typename ResultColumnType, typename ColumnType>\ninline ResultColumnType & column_down_cast(ColumnType & c) {\n    return dynamic_cast<ResultColumnType &>(c);\n}\n\n// std::visit-ish function to avoid including <variant> header, which is not present in older version of XCode.\ntemplate <typename Vizitor, typename ColumnType>\ninline auto VisitIndexColumn(Vizitor && vizitor, ColumnType && col) {\n    switch (col.Type()->GetCode()) {\n        case Type::UInt8:\n            return vizitor(column_down_cast<ColumnUInt8>(col));\n        case Type::UInt16:\n            return vizitor(column_down_cast<ColumnUInt16>(col));\n        case Type::UInt32:\n            return vizitor(column_down_cast<ColumnUInt32>(col));\n        case Type::UInt64:\n            return vizitor(column_down_cast<ColumnUInt64>(col));\n        default:\n            throw ValidationError(\"Invalid index column type \" + col.GetType().GetName());\n    }\n}\n\n// A special NULL-item, which is expected at pos(0) in dictionary,\n// note that we distinguish empty string from NULL-value.\ninline auto GetNullItemForDictionary(const ColumnRef dictionary) {\n    if (auto n = dictionary->As<ColumnNullable>()) {\n        return ItemView {};\n    } else {\n        return ItemView{dictionary->Type()->GetCode(), std::string_view{}};\n    }\n}\n\n// A special default item, which is expected at pos(0) in dictionary,\n// note that we distinguish empty string from NULL-value.\ninline ItemView GetDefaultItemForDictionary(const ColumnRef dictionary) {\n    if (auto n = dictionary->As<ColumnNullable>()) {\n        return GetDefaultItemForDictionary(n->Nested());\n    } else {\n        return ItemView{dictionary->Type()->GetCode(), std::string_view{}};\n    }\n}\n\nvoid AppendToDictionary(Column& dictionary, const ItemView & item);\n\ninline void AppendNullableToDictionary(ColumnNullable& nullable, const ItemView & item) {\n    auto nested = nullable.Nested();\n\n    const bool isNullValue = item.type == Type::Void;\n\n    if (isNullValue) {\n        AppendToDictionary(*nested, GetNullItemForDictionary(nested));\n    } else {\n        const auto nestedType = nested->GetType().GetCode();\n        if (nestedType != item.type) {\n            throw ValidationError(\"Invalid value. Type expected: \" + nested->GetType().GetName());\n        }\n\n        AppendToDictionary(*nested, item);\n    }\n\n    nullable.Append(isNullValue);\n}\n\ninline void AppendToDictionary(Column& dictionary, const ItemView & item) {\n    switch (dictionary.GetType().GetCode()) {\n        case Type::FixedString:\n            column_down_cast<ColumnFixedString>(dictionary).Append(item.get<std::string_view>());\n            return;\n        case Type::String:\n            column_down_cast<ColumnString>(dictionary).Append(item.get<std::string_view>());\n            return;\n        case Type::Nullable:\n            AppendNullableToDictionary(column_down_cast<ColumnNullable>(dictionary), item);\n            return;\n        default:\n            throw ValidationError(\"Unexpected dictionary column type: \" + dictionary.GetType().GetName());\n    }\n}\n\n}\n\nnamespace clickhouse {\nColumnLowCardinality::ColumnLowCardinality(ColumnRef dictionary_column)\n    : Column(Type::CreateLowCardinality(dictionary_column->Type())),\n      dictionary_column_(dictionary_column->CloneEmpty()), // safe way to get an column of the same type.\n      index_column_(std::make_shared<ColumnUInt32>())\n{\n    Setup(dictionary_column);\n}\n\nColumnLowCardinality::ColumnLowCardinality(std::shared_ptr<ColumnNullable> dictionary_column)\n    : Column(Type::CreateLowCardinality(dictionary_column->Type())),\n      dictionary_column_(dictionary_column->CloneEmpty()), // safe way to get an column of the same type.\n      index_column_(std::make_shared<ColumnUInt32>())\n{\n    AppendNullItem();\n    Setup(dictionary_column);\n}\n\nColumnLowCardinality::~ColumnLowCardinality()\n{}\n\nvoid ColumnLowCardinality::Reserve(size_t new_cap) {\n    dictionary_column_->Reserve(new_cap);\n    index_column_->Reserve(new_cap);\n}\n\nvoid ColumnLowCardinality::Setup(ColumnRef dictionary_column) {\n    AppendDefaultItem();\n\n    if (dictionary_column->Size() != 0) {\n        // Add values, updating index_column_ and unique_items_map_.\n\n        // TODO: it would be possible to eliminate copying\n        // by adding InsertUnsafe(pos, ItemView) method to a Column\n        // (to insert null-item at pos 0),\n        // but that is too much work for now.\n        for (size_t i = 0; i < dictionary_column->Size(); ++i) {\n            AppendUnsafe(dictionary_column->GetItem(i));\n        }\n    }\n}\n\nstd::uint64_t ColumnLowCardinality::getDictionaryIndex(std::uint64_t item_index) const {\n    return VisitIndexColumn([item_index](const auto & arg) -> std::uint64_t {\n        return arg[item_index];\n    }, *index_column_);\n}\n\nvoid ColumnLowCardinality::appendIndex(std::uint64_t item_index) {\n    // TODO (nemkov): handle case when index should go from UInt8 to UInt16, etc.\n    VisitIndexColumn([item_index](auto & arg) {\n        arg.Append(static_cast<typename std::decay_t<decltype(arg)>::DataType>(item_index));\n    }, *index_column_);\n}\n\nvoid ColumnLowCardinality::removeLastIndex() {\n    VisitIndexColumn([](auto & arg) {\n        arg.Erase(arg.Size() - 1);\n    }, *index_column_);\n}\n\ndetails::LowCardinalityHashKey ColumnLowCardinality::computeHashKey(const ItemView & item) {\n    static const auto hasher = std::hash<ItemView::DataType>{};\n    if (item.type == Type::Void) {\n        // to distinguish NULL of ColumnNullable and empty string.\n        return {0u, 0u};\n    }\n\n...\n// Path: clickhouse/columns/itemview.cpp\n#include \"../columns/itemview.h\"\n\n#include <algorithm>\n#include <sstream>\n\nnamespace {\n\ntemplate <typename Container>\n\nstd::string ContainerToString(Container container, const char * separator = \", \") {\n    std::stringstream sstr;\n    const auto end = std::end(container);\n    for (auto i = std::begin(container); i != end; /*intentionally no ++i*/) {\n        const auto & elem = *i;\n        sstr << elem;\n\n        if (++i != end) {\n            sstr << separator;\n        }\n    }\n\n    return sstr.str();\n}\n\n}\n\nnamespace clickhouse {\n\nvoid ItemView::ValidateData(Type::Code type, DataType data) {\n\n    auto AssertSize = [type, &data](std::initializer_list<int> allowed_sizes) -> void {\n        const auto end = std::end(allowed_sizes);\n        if (std::find(std::begin(allowed_sizes), end, static_cast<int>(data.size())) == end) {\n            throw AssertionError(std::string(\"ItemView value size mismatch for \")\n                    + Type::TypeName(type)\n                    + \" expected: \" + ContainerToString(allowed_sizes, \" or \")\n                    + \", got: \" + std::to_string(data.size()));\n        }\n    };\n\n    switch (type) {\n        case Type::Code::Void:\n            return AssertSize({0});\n\n        case Type::Code::Int8:\n        case Type::Code::UInt8:\n        case Type::Code::Enum8:\n            return AssertSize({1});\n\n        case Type::Code::Int16:\n        case Type::Code::UInt16:\n        case Type::Code::Date:\n        case Type::Code::Enum16:\n            return AssertSize({2});\n\n        case Type::Code::Int32:\n        case Type::Code::UInt32:\n        case Type::Code::Float32:\n        case Type::Code::DateTime:\n        case Type::Code::Date32:\n        case Type::Code::IPv4:\n        case Type::Code::Decimal32:\n            return AssertSize({4});\n\n        case Type::Code::Int64:\n        case Type::Code::UInt64:\n        case Type::Code::Float64:\n        case Type::Code::DateTime64:\n        case Type::Code::Decimal64:\n            return AssertSize({8});\n\n        case Type::Code::String:\n        case Type::Code::FixedString:\n            // value can be of any size\n            return;\n\n        case Type::Code::Array:\n        case Type::Code::Nullable:\n        case Type::Code::Tuple:\n        case Type::Code::LowCardinality:\n        case Type::Code::Map:\n            throw AssertionError(\"Unsupported type in ItemView: \" + std::string(Type::TypeName(type)));\n\n        case Type::Code::IPv6:\n        case Type::Code::UUID:\n        case Type::Code::Int128:\n        case Type::Code::Decimal128:\n            return AssertSize({16});\n\n        case Type::Code::Decimal:\n            // Could be either Decimal32, Decimal64 or Decimal128\n            return AssertSize({4, 8, 16});\n\n        default:\n            throw UnimplementedError(\"Unknown type code:\" + std::to_string(static_cast<int>(type)));\n    }\n}\n\n}\n\n// Path: clickhouse/columns/numeric.cpp\n#include \"numeric.h\"\n#include \"utils.h\"\n\n#include \"../base/wire_format.h\"\n\nnamespace clickhouse {\n\ntemplate <typename T>\nColumnVector<T>::ColumnVector()\n    : Column(Type::CreateSimple<T>())\n{\n}\n\ntemplate <typename T>\nColumnVector<T>::ColumnVector(const std::vector<T> & data)\n    : Column(Type::CreateSimple<T>())\n    , data_(data)\n{\n}\n\ntemplate <typename T>\nColumnVector<T>::ColumnVector(std::vector<T> && data)\n    : Column(Type::CreateSimple<T>())\n    , data_(std::move(data))\n{\n}\n\ntemplate <typename T>\nvoid ColumnVector<T>::Append(const T& value) {\n    data_.push_back(value);\n}\n\ntemplate <typename T>\nvoid ColumnVector<T>::Erase(size_t pos, size_t count) {\n    const auto begin = std::min(pos, data_.size());\n    const auto last = begin + std::min(data_.size() - begin, count);\n\n    data_.erase(data_.begin() + begin, data_.begin() + last);\n}\n\ntemplate <typename T>\nstd::vector<T>& ColumnVector<T>::GetWritableData() {\n    return data_;\n}\n\ntemplate <typename T>\nvoid ColumnVector<T>::Reserve(size_t new_cap) {\n    data_.reserve(new_cap);\n}\n\ntemplate <typename T>\nsize_t ColumnVector<T>::Capacity() const {\n    return data_.capacity();\n}\n\ntemplate <typename T>\nvoid ColumnVector<T>::Clear() {\n    data_.clear();\n}\n\ntemplate <typename T>\nconst T& ColumnVector<T>::At(size_t n) const {\n    return data_.at(n);\n}\n\ntemplate <typename T>\nvoid ColumnVector<T>::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnVector<T>>()) {\n        data_.insert(data_.end(), col->data_.begin(), col->data_.end());\n    }\n}\n\ntemplate <typename T>\nbool ColumnVector<T>::LoadBody(InputStream* input, size_t rows) {\n    data_.resize(rows);\n\n    return WireFormat::ReadBytes(*input, data_.data(), data_.size() * sizeof(T));\n}\n\ntemplate <typename T>\nvoid ColumnVector<T>::SaveBody(OutputStream* output) {\n    WireFormat::WriteBytes(*output, data_.data(), data_.size() * sizeof(T));\n}\n\ntemplate <typename T>\nsize_t ColumnVector<T>::Size() const {\n    return data_.size();\n}\n\ntemplate <typename T>\nColumnRef ColumnVector<T>::Slice(size_t begin, size_t len) const {\n    return std::make_shared<ColumnVector<T>>(SliceVector(data_, begin, len));\n}\n\ntemplate <typename T>\nColumnRef ColumnVector<T>::CloneEmpty() const {\n    return std::make_shared<ColumnVector<T>>();\n}\n\ntemplate <typename T>\nvoid ColumnVector<T>::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnVector<T> &>(other);\n    data_.swap(col.data_);\n}\n\ntemplate <typename T>\nItemView ColumnVector<T>::GetItem(size_t index) const  {\n    return ItemView{type_->GetCode(), data_[index]};\n}\n\ntemplate class ColumnVector<int8_t>;\ntemplate class ColumnVector<int16_t>;\ntemplate class ColumnVector<int32_t>;\ntemplate class ColumnVector<int64_t>;\n\ntemplate class ColumnVector<uint8_t>;\ntemplate class ColumnVector<uint16_t>;\ntemplate class ColumnVector<uint32_t>;\ntemplate class ColumnVector<uint64_t>;\ntemplate class ColumnVector<Int128>;\n\ntemplate class ColumnVector<float>;\ntemplate class ColumnVector<double>;\n\n}\n\n// Path: clickhouse/columns/decimal.cpp\n#include \"decimal.h\"\n\nnamespace\n{\nusing namespace clickhouse;\n\n#ifdef ABSL_HAVE_INTRINSIC_INT128\ntemplate <typename T>\ninline bool addOverflow(const Int128 & l, const T & r, Int128 * result)\n{\n    __int128 res;\n    const auto ret_value = __builtin_add_overflow(static_cast<__int128>(l), static_cast<__int128>(r), &res);\n\n    *result = res;\n    return ret_value;\n}\n\ntemplate <typename T>\ninline bool mulOverflow(const Int128 & l, const T & r, Int128 * result)\n{\n    __int128 res;\n    const auto ret_value = __builtin_mul_overflow(static_cast<__int128>(l), static_cast<__int128>(r), &res);\n\n    *result = res;\n    return ret_value;\n}\n\n#else\ntemplate <typename T>\ninline bool getSignBit(const T & v)\n{\n    return v < static_cast<T>(0);\n}\n\ninline bool getSignBit(const Int128 & v)\n{\n//    static constexpr Int128 zero {};\n//    return v < zero;\n\n    // Sign of the whole absl::int128 value is determined by sign of higher 64 bits.\n    return absl::Int128High64(v) < 0;\n}\n\ninline bool addOverflow(const Int128 & l, const Int128 & r, Int128 * result)\n{\n    //    *result = l + r;\n    //    const auto result_sign = getSignBit(*result);\n    //    return l_sign == r_sign && l_sign != result_sign;\n\n    // Based on code from:\n    // https://wiki.sei.cmu.edu/confluence/display/c/INT32-C.+Ensure+that+operations+on+signed+integers+do+not+result+in+overflow#INT32C.Ensurethatoperationsonsignedintegersdonotresultinoverflow-CompliantSolution\n    const auto r_positive = !getSignBit(r);\n\n    if ((r_positive && (l > (std::numeric_limits<Int128>::max() - r))) ||\n        (!r_positive && (l < (std::numeric_limits<Int128>::min() - r)))) {\n        return true;\n    }\n    *result = l + r;\n\n    return false;\n}\n\ntemplate <typename T>\ninline bool mulOverflow(const Int128 & l, const T & r, Int128 * result)\n{\n    // Based on code from:\n    // https://wiki.sei.cmu.edu/confluence/display/c/INT32-C.+Ensure+that+operations+on+signed+integers+do+not+result+in+overflow#INT32C.Ensurethatoperationsonsignedintegersdonotresultinoverflow-CompliantSolution.3\n    const auto l_positive = !getSignBit(l);\n    const auto r_positive = !getSignBit(r);\n\n    if (l_positive) {\n        if (r_positive) {\n            if (r != 0 && l > (std::numeric_limits<Int128>::max() / r)) {\n                return true;\n            }\n        } else {\n            if (l != 0 && r < (std::numeric_limits<Int128>::min() / l)) {\n                return true;\n            }\n        }\n    } else {\n        if (r_positive) {\n            if (r != 0 && l < (std::numeric_limits<Int128>::min() / r)) {\n                return true;\n            }\n        } else {\n            if (l != 0 && (r < (std::numeric_limits<Int128>::max() / l))) {\n                return true;\n            }\n        }\n    }\n\n    *result = l * r;\n    return false;\n}\n#endif\n\n}\n\nnamespace clickhouse {\n\nColumnDecimal::ColumnDecimal(size_t precision, size_t scale)\n    : Column(Type::CreateDecimal(precision, scale))\n{\n    if (precision <= 9) {\n        data_ = std::make_shared<ColumnInt32>();\n    } else if (precision <= 18) {\n        data_ = std::make_shared<ColumnInt64>();\n    } else {\n        data_ = std::make_shared<ColumnInt128>();\n    }\n}\n\nColumnDecimal::ColumnDecimal(TypeRef type, ColumnRef data)\n    : Column(type),\n      data_(data)\n{\n}\n\nvoid ColumnDecimal::Append(const Int128& value) {\n    if (data_->Type()->GetCode() == Type::Int32) {\n        data_->As<ColumnInt32>()->Append(static_cast<ColumnInt32::DataType>(value));\n    } else if (data_->Type()->GetCode() == Type::Int64) {\n        data_->As<ColumnInt64>()->Append(static_cast<ColumnInt64::DataType>(value));\n    } else {\n        data_->As<ColumnInt128>()->Append(static_cast<ColumnInt128::DataType>(value));\n    }\n}\n\nvoid ColumnDecimal::Append(const std::string& value) {\n    Int128 int_value = 0;\n    auto c = value.begin();\n    auto end = value.end();\n    bool sign = true;\n    bool has_dot = false;\n\n    size_t zeros = 0;\n\n    while (c != end) {\n        if (*c == '-') {\n            sign = false;\n            if (c != value.begin()) {\n                break;\n            }\n        } else if (*c == '.' && !has_dot) {\n            size_t distance = std::distance(c, end) - 1;\n            auto scale = type_->As<DecimalType>()->GetScale();\n\n            if (distance <= scale) {\n                zeros = scale - distance;\n            } else {\n                std::advance(end, scale - distance);\n            }\n\n            has_dot = true;\n        } else if (*c >= '0' && *c <= '9') {\n            if (mulOverflow(int_value, 10, &int_value) ||\n                addOverflow(int_value, *c - '0', &int_value)) {\n                throw AssertionError(\"value is too big for 128-bit integer\");\n            }\n        } else {\n            throw ValidationError(std::string(\"unexpected symbol '\") + (*c) + \"' in decimal value\");\n        }\n        ++c;\n    }\n\n    if (c != end) {\n        throw ValidationError(\"unexpected symbol '-' in decimal value\");\n    }\n\n    while (zeros) {\n        if (mulOverflow(int_value, 10, &int_value)) {\n            throw AssertionError(\"value is too big for 128-bit integer\");\n        }\n        --zeros;\n    }\n\n    Append(sign ? int_value : -int_value);\n}\n\nInt128 ColumnDecimal::At(size_t i) const {\n    switch (data_->Type()->GetCode()) {\n        case Type::Int32:\n            return static_cast<Int128>(data_->As<ColumnInt32>()->At(i));\n        case Type::Int64:\n            return static_cast<Int128>(data_->As<ColumnInt64>()->At(i));\n        case Type::Int128:\n            return data_->As<ColumnInt128>()->At(i);\n        default:\n            throw ValidationError(\"Invalid data_ column type in ColumnDecimal\");\n    }\n}\n\nvoid ColumnDecimal::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\nvoid ColumnDecimal::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnDecimal>()) {\n        data_->Append(col->data_);\n    }\n}\n\nbool ColumnDecimal::LoadBody(InputStream * input, size_t rows) {\n    return data_->LoadBody(input, rows);\n}\n\nvoid ColumnDecimal::SaveBody(OutputStream* output) {\n    data_->SaveBody(output);\n}\n\nvoid ColumnDecimal::Clear() {\n    data_->Clear();\n}\n\nsize_t ColumnDecimal::Size() const {\n    return data_->Size();\n}\n\nColumnRef ColumnDecimal::Slice(size_t begin, size_t len) const {\n    // coundn't use std::make_shared since this c-tor is private\n    return ColumnRef{new ColumnDecimal(type_, data_->Slice(begin, len))};\n}\n\nColumnRef ColumnDecimal::CloneEmpty() const {\n    // coundn't use std::make_shared since this c-tor is private\n    return ColumnRef{new ColumnDecimal(type_, data_->CloneEmpty())};\n}\n\nvoid ColumnDecimal::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnDecimal &>(other);\n    data_.swap(col.data_);\n}\n\nItemView ColumnDecimal::GetItem(size_t index) const {\n    return ItemView{GetType().GetCode(), data_->GetItem(index)};\n}\n\nsize_t ColumnDecimal::GetScale() const\n{\n    return type_->As<DecimalType>()->GetScale();\n}\n\nsize_t ColumnDecimal::GetPrecision() const\n{\n    return type_->As<DecimalType>()->GetPrecision();\n}\n\n}\n\n// Path: clickhouse/columns/string.cpp\n#include \"string.h\"\n#include \"utils.h\"\n\n#include \"../base/wire_format.h\"\n\nnamespace {\n\nconstexpr size_t DEFAULT_BLOCK_SIZE = 4096;\n\ntemplate <typename Container>\nsize_t ComputeTotalSize(const Container & strings, size_t begin = 0, size_t len = -1) {\n    size_t result = 0;\n    if (begin < strings.size()) {\n        len = std::min(len, strings.size() - begin);\n\n        for (size_t i = begin; i < begin + len; ++i)\n            result += strings[i].size();\n    }\n\n    return result;\n}\n\n}\n\nnamespace clickhouse {\n\nColumnFixedString::ColumnFixedString(size_t n)\n    : Column(Type::CreateString(n))\n    , string_size_(n)\n{\n}\n\nvoid ColumnFixedString::Reserve(size_t new_cap) {\n    data_.reserve(string_size_ * new_cap);\n}\n\nvoid ColumnFixedString::Append(std::string_view str) {\n    if (str.size() > string_size_) {\n        throw ValidationError(\"Expected string of length not greater than \"\n                                 + std::to_string(string_size_) + \" bytes, received \"\n                                 + std::to_string(str.size()) + \" bytes.\");\n    }\n\n    if (data_.capacity() - data_.size() < str.size()) {\n        // round up to the next block size\n        const auto new_size = (((data_.size() + string_size_) / DEFAULT_BLOCK_SIZE) + 1) * DEFAULT_BLOCK_SIZE;\n        data_.reserve(new_size);\n    }\n\n    data_.insert(data_.size(), str);\n    // Pad up to string_size_ with zeroes.\n    if (str.size() < string_size_) {\n        const auto padding_size = string_size_ - str.size();\n        data_.resize(data_.size() + padding_size, char(0));\n    }\n}\n\nvoid ColumnFixedString::Clear() {\n    data_.clear();\n}\n\nstd::string_view ColumnFixedString::At(size_t n) const {\n    const auto pos = n * string_size_;\n    return std::string_view(&data_.at(pos), string_size_);\n}\n\nsize_t ColumnFixedString::FixedSize() const {\n       return string_size_;\n}\n\nvoid ColumnFixedString::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnFixedString>()) {\n        if (string_size_ == col->string_size_) {\n            data_.insert(data_.end(), col->data_.begin(), col->data_.end());\n        }\n    }\n}\n\nbool ColumnFixedString::LoadBody(InputStream * input, size_t rows) {\n    data_.resize(string_size_ * rows);\n    if (!WireFormat::ReadBytes(*input, &data_[0], data_.size())) {\n        return false;\n    }\n\n    return true;\n}\n\nvoid ColumnFixedString::SaveBody(OutputStream* output) {\n    WireFormat::WriteBytes(*output, data_.data(), data_.size());\n}\n\nsize_t ColumnFixedString::Size() const {\n    return data_.size() / string_size_;\n}\n\nColumnRef ColumnFixedString::Slice(size_t begin, size_t len) const {\n    auto result = std::make_shared<ColumnFixedString>(string_size_);\n\n    if (begin < Size()) {\n        const auto b = begin * string_size_;\n        const auto l = len * string_size_;\n        result->data_ = data_.substr(b, std::min(data_.size() - b, l));\n    }\n\n    return result;\n}\n\nColumnRef ColumnFixedString::CloneEmpty() const {\n    return std::make_shared<ColumnFixedString>(string_size_);\n}\n\nvoid ColumnFixedString::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnFixedString &>(other);\n    std::swap(string_size_, col.string_size_);\n    data_.swap(col.data_);\n}\n\nItemView ColumnFixedString::GetItem(size_t index) const {\n    return ItemView{Type::FixedString, this->At(index)};\n}\n\nstruct ColumnString::Block\n{\n    using CharT = typename std::string::value_type;\n\n    explicit Block(size_t starting_capacity)\n        : size(0),\n        capacity(starting_capacity),\n        data_(new CharT[capacity])\n    {}\n\n    inline auto GetAvailable() const {\n        return capacity - size;\n    }\n\n    std::string_view AppendUnsafe(std::string_view str) {\n        const auto pos = &data_[size];\n\n        memcpy(pos, str.data(), str.size());\n        size += str.size();\n\n        return std::string_view(pos, str.size());\n    }\n\n    auto GetCurrentWritePos() {\n        return &data_[size];\n    }\n\n    std::string_view ConsumeTailAsStringViewUnsafe(size_t len) {\n        const auto start = &data_[size];\n        size += len;\n        return std::string_view(start, len);\n    }\n\n    size_t size;\n    const size_t capacity;\n    std::unique_ptr<CharT[]> data_;\n};\n\nColumnString::ColumnString()\n    : Column(Type::CreateString())\n{\n}\n\nColumnString::ColumnString(size_t element_count)\n    : Column(Type::CreateString())\n{\n    items_.reserve(element_count);\n    // 16 is arbitrary number, assumption that string values are about ~256 bytes long.\n    blocks_.reserve(std::max<size_t>(1, element_count / 16));\n}\n\nColumnString::ColumnString(const std::vector<std::string>& data)\n    : ColumnString()\n{\n    items_.reserve(data.size());\n    blocks_.emplace_back(ComputeTotalSize(data));\n\n    for (const auto & s : data) {\n        AppendUnsafe(s);\n    }\n}\n\nColumnString::ColumnString(std::vector<std::string>&& data)\n    : ColumnString()\n{\n    items_.reserve(data.size());\n\n    for (auto&& d : data) {\n        append_data_.emplace_back(std::move(d));\n        auto& last_data = append_data_.back();\n        items_.emplace_back(std::string_view{ last_data.data(),last_data.length() });\n    }\n}\n\nColumnString::~ColumnString()\n{}\n\nvoid ColumnString::Reserve(size_t new_cap) {\n    items_.reserve(new_cap);\n    // 16 is arbitrary number, assumption that string values are about ~256 bytes long.\n    blocks_.reserve(std::max<size_t>(1, new_cap / 16));\n}\n\nvoid ColumnString::Append(std::string_view str) {\n    if (blocks_.size() == 0 || blocks_.back().GetAvailable() < str.length()) {\n        blocks_.emplace_back(std::max(DEFAULT_BLOCK_SIZE, str.size()));\n    }\n\n    items_.emplace_back(blocks_.back().AppendUnsafe(str));\n}\n\nvoid ColumnString::Append(const char* str) {\n    Append(std::string_view(str, strlen(str)));\n}\n\nvoid ColumnString::Append(std::string&& steal_value) {\n    append_data_.emplace_back(std::move(steal_value));\n    auto& last_data = append_data_.back();\n    items_.emplace_back(std::string_view{ last_data.data(),last_data.length() });\n}\n\nvoid ColumnString::AppendNoManagedLifetime(std::string_view str) {\n    items_.emplace_back(str);\n}\n\nvoid ColumnString::AppendUnsafe(std::string_view str) {\n    items_.emplace_back(blocks_.back().AppendUnsafe(str));\n}\n\nvoid ColumnString::Clear() {\n    items_.clear();\n    blocks_.clear();\n    append_data_.clear();\n}\n\nstd::string_view ColumnString::At(size_t n) const {\n    return items_.at(n);\n}\n\nvoid ColumnString::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnString>()) {\n        const auto total_size = ComputeTotalSize(col->items_);\n\n        // TODO: fill up existing block with some items and then add a new one for the rest of items\n        if (blocks_.size() == 0 || blocks_.back().GetAvailable() < total_size)\n            blocks_.emplace_back(std::max(DEFAULT_BLOCK_SIZE, total_size));\n\n        // Intentionally not doing items_.reserve() since that cripples performance.\n        for (size_t i = 0; i < column->Size(); ++i) {\n            this->AppendUnsafe((*col)[i]);\n        }\n    }\n}\n\nbool ColumnString::LoadBody(InputStream* input, size_t rows) {\n    if (rows == 0) {\n        items_.clear();\n        blocks_.clear();\n\n        return true;\n    }\n\n    decltype(items_) new_items;\n    decltype(blocks_) new_blocks;\n\n    new_items.reserve(rows);\n\n    // Suboptimzal if the first row string is >DEFAULT_BLOCK_SIZE, but that must be a very rare case.\n    Block * block = &new_blocks.emplace_back(DEFAULT_BLOCK_SIZE);\n\n    for (size_t i = 0; i < rows; ++i) {\n        uint64_t len;\n        if (!WireFormat::ReadUInt64(*input, &len))\n            return false;\n\n        if (len > block->GetAvailable())\n            block = &new_blocks.emplace_back(std::max<size_t>(DEFAULT_BLOCK_SIZE, len));\n\n        if (!WireFormat::ReadBytes(*input, block->GetCurrentWritePos(), len))\n            return false;\n\n        new_items.emplace_back(block->ConsumeTailAsStringViewUnsafe(len));\n    }\n\n    items_.swap(new_items);\n    blocks_.swap(new_blocks);\n\n    return true;\n}\n\nvoid ColumnString::SaveBody(OutputStream* output) {\n    for (const auto & item : items_) {\n        WireFormat::WriteString(*output, item);\n    }\n}\n\nsize_t ColumnString::Size() const {\n    return items_.size();\n}\n\nColumnRef ColumnString::Slice(size_t begin, size_t len) const {\n    auto result = std::make_shared<ColumnString>();\n\n    if (begin < items_.size()) {\n        len = std::min(len, items_.size() - begin);\n        result->items_.reserve(len);\n\n        result->blocks_.emplace_back(ComputeTotalSize(items_, begin, len));\n        for (size_t i = begin; i < begin + len; ++i) {\n            result->Append(items_[i]);\n        }\n    }\n\n    return result;\n}\n\nColumnRef ColumnString::CloneEmpty() const {\n    return std::make_shared<ColumnString>();\n}\n\nvoid ColumnString::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnString &>(other);\n    items_.swap(col.items_);\n    blocks_.swap(col.blocks_);\n    append_data_.swap(col.append_data_);\n}\n\nItemView ColumnString::GetItem(size_t index) const {\n    return ItemView{Type::String, this->At(index)};\n}\n\n}\n\n// Path: clickhouse/columns/ip4.cpp\n#include \"ip4.h\"\n\n#include \"../base/socket.h\" // for platform-specific IPv4-related functions\n#include <stdexcept>\n\nnamespace clickhouse {\n\nColumnIPv4::ColumnIPv4()\n    : Column(Type::CreateIPv4())\n    , data_(std::make_shared<ColumnUInt32>())\n{\n}\n\nColumnIPv4::ColumnIPv4(ColumnRef data)\n    : Column(Type::CreateIPv4())\n    , data_(data ? data->As<ColumnUInt32>() : nullptr)\n{\n    if (!data_)\n        throw ValidationError(\"Expecting ColumnUInt32, got \" + (data ? data->GetType().GetName() : \"null\"));\n}\n\nColumnIPv4::ColumnIPv4(std::vector<uint32_t>&& data)\n    : Column(Type::CreateIPv4())\n{\n    for (auto& addr : data) {\n        addr = htonl(addr);\n    }\n    data_ = std::make_shared<ColumnUInt32>(std::move(data));\n}\n\nvoid ColumnIPv4::Append(const std::string& str) {\n    uint32_t address;\n    if (inet_pton(AF_INET, str.c_str(), &address) != 1)\n        throw ValidationError(\"invalid IPv4 format, ip: \" + str);\n    data_->Append(htonl(address));\n}\n\nvoid ColumnIPv4::Append(uint32_t ip) {\n    data_->Append(htonl(ip));\n}\n\nvoid ColumnIPv4::Append(in_addr ip) {\n    data_->Append(htonl(ip.s_addr));\n}\n\nvoid ColumnIPv4::Clear() {\n    data_->Clear();\n}\n\nin_addr ColumnIPv4::At(size_t n) const {\n    in_addr addr;\n    addr.s_addr = ntohl(data_->At(n));\n    return addr;\n}\n\nin_addr ColumnIPv4::operator [] (size_t n) const {\n    in_addr addr;\n    addr.s_addr = ntohl(data_->operator[](n));\n    return addr;\n}\n\nstd::string ColumnIPv4::AsString(size_t n) const {\n    const auto& addr = this->At(n);\n\n    char buf[INET_ADDRSTRLEN];\n    const char* ip_str = inet_ntop(AF_INET, &addr, buf, INET_ADDRSTRLEN);\n\n    if (ip_str == nullptr) {\n        throw std::system_error(\n                std::error_code(errno, std::generic_category()),\n                \"Invalid IPv4 data\");\n    }\n\n    return ip_str;\n}\n\nvoid ColumnIPv4::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\nvoid ColumnIPv4::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnIPv4>()) {\n        data_->Append(col->data_);\n    }\n}\n\nbool ColumnIPv4::LoadBody(InputStream * input, size_t rows) {\n    return data_->LoadBody(input, rows);\n}\n\nvoid ColumnIPv4::SaveBody(OutputStream* output) {\n    data_->SaveBody(output);\n}\n\nsize_t ColumnIPv4::Size() const {\n    return data_->Size();\n}\n\nColumnRef ColumnIPv4::Slice(size_t begin, size_t len) const {\n    return std::make_shared<ColumnIPv4>(data_->Slice(begin, len));\n}\n\nColumnRef ColumnIPv4::CloneEmpty() const {\n    return std::make_shared<ColumnIPv4>(data_->CloneEmpty());\n}\n\nvoid ColumnIPv4::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnIPv4 &>(other);\n    data_.swap(col.data_);\n}\n\nItemView ColumnIPv4::GetItem(size_t index) const {\n    return ItemView(Type::IPv4, data_->GetItem(index));\n}\n\n}\n\n// Path: clickhouse/columns/tuple.cpp\n#include \"tuple.h\"\n\nnamespace clickhouse {\n\nstatic std::vector<TypeRef> CollectTypes(const std::vector<ColumnRef>& columns) {\n    std::vector<TypeRef> types;\n    for (const auto& col : columns) {\n        types.push_back(col->Type());\n    }\n    return types;\n}\n\nColumnTuple::ColumnTuple(const std::vector<ColumnRef>& columns)\n    : Column(Type::CreateTuple(CollectTypes(columns)))\n    , columns_(columns)\n{\n}\n\nsize_t ColumnTuple::TupleSize() const {\n    return columns_.size();\n}\n\nvoid ColumnTuple::Reserve(size_t new_cap) {\n    for (auto& column : columns_) {\n        column->Reserve(new_cap);\n    }  \n}\n\nvoid ColumnTuple::Append(ColumnRef column) {\n    if (!this->Type()->IsEqual(column->Type())) {\n        throw ValidationError(\n            \"can't append column of type \" + column->Type()->GetName() + \" \"\n            \"to column type \" + this->Type()->GetName());\n    }\n    const auto & source_tuple_column = column->As<ColumnTuple>();\n    for (size_t ci = 0; ci < columns_.size(); ++ci) {\n        columns_[ci]->Append((*source_tuple_column)[ci]);\n    }\n}\nsize_t ColumnTuple::Size() const {\n    return columns_.empty() ? 0 : columns_[0]->Size();\n}\n\nColumnRef ColumnTuple::Slice(size_t begin, size_t len) const {\n    std::vector<ColumnRef> sliced_columns;\n    sliced_columns.reserve(columns_.size());\n    for(const auto &column : columns_) {\n        sliced_columns.push_back(column->Slice(begin, len));\n    }\n\n    return std::make_shared<ColumnTuple>(sliced_columns);\n}\n\nColumnRef ColumnTuple::CloneEmpty() const {\n    std::vector<ColumnRef> result_columns;\n    result_columns.reserve(columns_.size());\n\n    for(const auto &column : columns_) {\n        result_columns.push_back(column->CloneEmpty());\n    }\n\n    return std::make_shared<ColumnTuple>(result_columns);\n}\n\nbool ColumnTuple::LoadPrefix(InputStream* input, size_t rows) {\n    for (auto ci = columns_.begin(); ci != columns_.end(); ++ci) {\n        if (!(*ci)->LoadPrefix(input, rows)) {\n            return false;\n        }\n    }\n\n    return true;\n}\n\nbool ColumnTuple::LoadBody(InputStream* input, size_t rows) {\n    for (auto ci = columns_.begin(); ci != columns_.end(); ++ci) {\n        if (!(*ci)->LoadBody(input, rows)) {\n            return false;\n        }\n    }\n\n    return true;\n}\n\nvoid ColumnTuple::SavePrefix(OutputStream* output) {\n    for (auto & column : columns_) {\n        column->SavePrefix(output);\n    }\n}\n\nvoid ColumnTuple::SaveBody(OutputStream* output) {\n    for (auto & column : columns_) {\n        column->SaveBody(output);\n    }\n}\n\nvoid ColumnTuple::Clear() {\n    columns_.clear();\n}\n\nvoid ColumnTuple::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnTuple &>(other);\n    columns_.swap(col.columns_);\n}\n\n}\n\n// Path: clickhouse/columns/column.cpp\n#include \"column.h\"\n\nnamespace clickhouse {\n\nbool Column::LoadPrefix(InputStream*, size_t) {\n    /// does nothing by default\n    return true;\n}\n\nbool Column::Load(InputStream* input, size_t rows) {\n    return LoadPrefix(input, rows) && LoadBody(input, rows);\n}\n\nvoid Column::SavePrefix(OutputStream*) {\n    /// does nothing by default\n}\n\n/// Saves column data to output stream.\nvoid Column::Save(OutputStream* output) {\n    SavePrefix(output);\n    SaveBody(output);\n}\n\n}\n\n// Path: clickhouse/base/sslsocket.cpp\n#include \"sslsocket.h\"\n#include \"../client.h\"\n#include \"../exceptions.h\"\n\n#include <stdexcept>\n\n#include <openssl/ssl.h>\n#include <openssl/x509v3.h>\n#include <openssl/err.h>\n#include <openssl/asn1.h>\n\n\nnamespace {\n\nstd::string getCertificateInfo(X509* cert)\n{\n    if (!cert)\n        return \"No certificate\";\n\n    std::unique_ptr<BIO, decltype(&BIO_free)> mem_bio(BIO_new(BIO_s_mem()), &BIO_free);\n    X509_print(mem_bio.get(), cert);\n\n    char * data = nullptr;\n    auto len = BIO_get_mem_data(mem_bio.get(), &data);\n    if (len < 0)\n        return \"Can't get certificate info due to BIO error \" + std::to_string(len);\n\n    return std::string(data, len);\n}\n\nvoid throwSSLError(SSL * ssl, int error, const char * /*location*/, const char * /*statement*/, const std::string prefix = \"OpenSSL error: \") {\n    const auto detail_error = ERR_get_error();\n    auto reason = ERR_reason_error_string(detail_error);\n    reason = reason ? reason : \"Unknown SSL error\";\n\n    std::string reason_str = reason;\n    if (ssl) {\n        // Print certificate only if handshake isn't completed \n        if (auto ssl_session = SSL_get_session(ssl); ssl_session && SSL_get_state(ssl) != TLS_ST_OK)\n            reason_str += \"\\nServer certificate: \" + getCertificateInfo(SSL_SESSION_get0_peer(ssl_session));\n    }\n\n//    std::cerr << \"!!! SSL error at \" << location\n//              << \"\\n\\tcaused by \" << statement\n//              << \"\\n\\t: \"<< reason_str << \"(\" << error << \")\"\n//              << \"\\n\\t last err: \" << ERR_peek_last_error()\n//              << std::endl;\n\n    throw clickhouse::OpenSSLError(prefix + std::to_string(error) + \" : \" + reason_str);\n}\n\nvoid configureSSL(const clickhouse::SSLParams::ConfigurationType & configuration, SSL * ssl, SSL_CTX * context = nullptr) {\n    std::unique_ptr<SSL_CONF_CTX, decltype(&SSL_CONF_CTX_free)> conf_ctx_holder(SSL_CONF_CTX_new(), SSL_CONF_CTX_free);\n    auto conf_ctx = conf_ctx_holder.get();\n\n    // To make both cmdline and flag file commands start with no prefix.\n    SSL_CONF_CTX_set1_prefix(conf_ctx, \"\");\n    // Allow all set of client commands, also turn on proper error reporting to reuse throwSSLError().\n    SSL_CONF_CTX_set_flags(conf_ctx, SSL_CONF_FLAG_CMDLINE | SSL_CONF_FLAG_FILE | SSL_CONF_FLAG_CLIENT | SSL_CONF_FLAG_SHOW_ERRORS | SSL_CONF_FLAG_CERTIFICATE );\n    if (ssl)\n        SSL_CONF_CTX_set_ssl(conf_ctx, ssl);\n    else if (context)\n        SSL_CONF_CTX_set_ssl_ctx(conf_ctx, context);\n\n    for (const auto & kv : configuration) {\n        const int err = SSL_CONF_cmd(conf_ctx, kv.first.c_str(), (kv.second ? kv.second->c_str() : nullptr));\n        // From the documentation:\n        //  2 - both key and value used\n        //  1 - only key used\n        //  0 - error during processing\n        // -2 - key not recodnized\n        // -3 - missing value\n        const bool value_present = !!kv.second;\n        if (err == 2 || (err == 1 && !value_present))\n            continue;\n        else if (err == 0)\n            throwSSLError(ssl, SSL_ERROR_NONE, nullptr, nullptr, \"Failed to configure OpenSSL with command '\" + kv.first + \"' \");\n        else if (err == 1 && value_present)\n            throw clickhouse::OpenSSLError(\"Failed to configure OpenSSL: command '\" + kv.first + \"' needs no value\");\n        else if (err == -2)\n            throw clickhouse::OpenSSLError(\"Failed to configure OpenSSL: unknown command '\" + kv.first + \"'\");\n        else if (err == -3)\n            throw clickhouse::OpenSSLError(\"Failed to configure OpenSSL: command '\" + kv.first + \"' requires a value\");\n        else\n            throw clickhouse::OpenSSLError(\"Failed to configure OpenSSL: command '\" + kv.first + \"' unknown error: \" + std::to_string(err));\n    }\n}\n\n#define STRINGIFY_HELPER(x) #x\n#define STRINGIFY(x) STRINGIFY_HELPER(x)\n#define LOCATION __FILE__  \":\" STRINGIFY(__LINE__)\n\nstruct SSLInitializer {\n    SSLInitializer() {\n        SSL_library_init();\n        SSLeay_add_ssl_algorithms();\n        SSL_load_error_strings();\n    }\n};\n\nSSL_CTX * prepareSSLContext(const clickhouse::SSLParams & context_params) {\n    static const SSLInitializer ssl_initializer;\n\n    const SSL_METHOD *method = TLS_client_method();\n    std::unique_ptr<SSL_CTX, decltype(&SSL_CTX_free)> ctx(SSL_CTX_new(method), &SSL_CTX_free);\n\n    if (!ctx)\n        throw clickhouse::OpenSSLError(\"Failed to initialize SSL context\");\n\n#define HANDLE_SSL_CTX_ERROR(statement) do { \\\n    if (const auto ret_code = (statement); !ret_code) \\\n        throwSSLError(nullptr, static_cast<int>(ERR_peek_error()), LOCATION, #statement); \\\n} while(false);\n\n    if (context_params.use_default_ca_locations)\n        HANDLE_SSL_CTX_ERROR(SSL_CTX_set_default_verify_paths(ctx.get()));\n    if (!context_params.path_to_ca_directory.empty())\n        HANDLE_SSL_CTX_ERROR(\n            SSL_CTX_load_verify_locations(\n                ctx.get(),\n                nullptr,\n                context_params.path_to_ca_directory.c_str())\n        );\n\n    for (const auto & f : context_params.path_to_ca_files)\n        HANDLE_SSL_CTX_ERROR(SSL_CTX_load_verify_locations(ctx.get(), f.c_str(), nullptr));\n\n    if (context_params.context_options != -1)\n        SSL_CTX_set_options(ctx.get(), context_params.context_options);\n    if (context_params.min_protocol_version != -1)\n        HANDLE_SSL_CTX_ERROR(\n            SSL_CTX_set_min_proto_version(ctx.get(), context_params.min_protocol_version));\n    if (context_params.max_protocol_version != -1)\n        HANDLE_SSL_CTX_ERROR(\n            SSL_CTX_set_max_proto_version(ctx.get(), context_params.max_protocol_version));\n\n    return ctx.release();\n#undef HANDLE_SSL_CTX_ERROR\n}\n\nauto convertConfiguration(const decltype(clickhouse::ClientOptions::SSLOptions::configuration) & configuration)\n{\n    auto result = decltype(clickhouse::SSLParams::configuration){};\n    for (const auto & cv : configuration)\n        result.push_back({cv.command, cv.value});\n\n    return result;\n}\n\nclickhouse::SSLParams GetSSLParams(const clickhouse::ClientOptions& opts) {\n    const auto& ssl_options = *opts.ssl_options;\n    return clickhouse::SSLParams{\n            ssl_options.path_to_ca_files,\n            ssl_options.path_to_ca_directory,\n            ssl_options.use_default_ca_locations,\n            ssl_options.context_options,\n            ssl_options.min_protocol_version,\n            ssl_options.max_protocol_version,\n            ssl_options.use_sni,\n            ssl_options.skip_verification,\n            ssl_options.host_flags,\n            convertConfiguration(ssl_options.configuration)\n    };\n}\n\n}\n\nnamespace clickhouse {\n\nSSLContext::SSLContext(SSL_CTX & context)\n    : context_(&context, &SSL_CTX_free)\n{\n    SSL_CTX_up_ref(context_.get());\n}\n\nSSLContext::SSLContext(const SSLParams & context_params)\n    : context_(prepareSSLContext(context_params), &SSL_CTX_free)\n{\n}\n\nSSL_CTX * SSLContext::getContext() {\n    return context_.get();\n}\n\n// Allows caller to use returned value of `statement` if there was no error, throws exception otherwise.\n#define HANDLE_SSL_ERROR(SSL_PTR, statement) [&] { \\\n    if (const auto ret_code = (statement); ret_code <= 0) { \\\n        throwSSLError(SSL_PTR, SSL_get_error(SSL_PTR, static_cast<int>(ret_code)), LOCATION, #statement); \\\n        return static_cast<std::decay_t<decltype(ret_code)>>(0); \\\n    } \\\n    else \\\n        return ret_code; \\\n} ()\n\n/* // debug macro for tracing SSL state\n#define LOG_SSL_STATE() std::cerr << \"!!!!\" << LOCATION << \" @\" << __FUNCTION__ \\\n    << \"\\t\" << SSL_get_version(ssl_) << \" state: \"  << SSL_state_string_long(ssl_) \\\n    << \"\\n\\t handshake state: \" << SSL_get_state(ssl_) \\\n    << std::endl\n*/\nSSLSocket::SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params,\n                     const SSLParams & ssl_params, SSLContext& context)\n    : Socket(addr, timeout_params)\n    , ssl_(SSL_new(context.getContext()), &SSL_free)\n{\n    auto ssl = ssl_.get();\n    if (!ssl)\n        throw clickhouse::OpenSSLError(\"Failed to create SSL instance\");\n\n    std::unique_ptr<ASN1_OCTET_STRING, decltype(&ASN1_OCTET_STRING_free)> ip_addr(a2i_IPADDRESS(addr.Host().c_str()), &ASN1_OCTET_STRING_free);\n\n    HANDLE_SSL_ERROR(ssl, SSL_set_fd(ssl, static_cast<int>(handle_)));\n    if (ssl_params.use_SNI)\n        HANDLE_SSL_ERROR(ssl, SSL_set_tlsext_host_name(ssl, addr.Host().c_str()));\n\n    if (ssl_params.host_flags != -1)\n        SSL_set_hostflags(ssl, ssl_params.host_flags);\n    HANDLE_SSL_ERROR(ssl, SSL_set1_host(ssl, addr.Host().c_str()));\n\n    // DO NOT use SSL_set_verify(ssl, SSL_VERIFY_PEER, nullptr), since\n    // we check verification result later, and that provides better error message.\n\n    if (ssl_params.configuration.size() > 0)\n        configureSSL(ssl_params.configuration, ssl);\n\n    SSL_set_connect_state(ssl);\n    HANDLE_SSL_ERROR(ssl, SSL_connect(ssl));\n    HANDLE_SSL_ERROR(ssl, SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY));\n\n    if (const auto verify_result = SSL_get_verify_result(ssl); !ssl_params.skip_verification && verify_result != X509_V_OK) {\n        auto error_message = X509_verify_cert_error_string(verify_result);\n        throw clickhouse::OpenSSLError(\"Failed to verify SSL connection, X509_v error: \"\n                + std::to_string(verify_result)\n                + \" \" + error_message\n                + \"\\nServer certificate: \" + getCertificateInfo(SSL_get_peer_certificate(ssl)));\n    }\n\n    // Host name verification is done by OpenSSL itself, however if we are connecting to an ip-address,\n    // no verification is made, so we have to do it manually.\n    // Just in case if this is ever required, leave it here commented out.\n//    if (ip_addr) {\n//        // if hostname is actually an IP address\n//        HANDLE_SSL_ERROR(ssl, X509_check_ip(\n//                SSL_get_peer_certificate(ssl),\n//                ASN1_STRING_get0_data(ip_addr.get()),\n//                ASN1_STRING_length(ip_addr.get()),\n//                0));\n//    }\n}\n\nvoid SSLSocket::validateParams(const SSLParams & ssl_params) {\n    // We need either SSL or SSL_CTX to properly validate configuration, so create a temporary one.\n    std::unique_ptr<SSL_CTX, decltype(&SSL_CTX_free)> ctx(SSL_CTX_new(TLS_client_method()), &SSL_CTX_free);\n    configureSSL(ssl_params.configuration, nullptr, ctx.get());\n}\n\n\nSSLSocketFactory::SSLSocketFactory(const ClientOptions& opts)\n    : NonSecureSocketFactory()\n    , ssl_params_(GetSSLParams(opts)) {\n    if (opts.ssl_options->ssl_context) {\n        ssl_context_ = std::make_unique<SSLContext>(*opts.ssl_options->ssl_context);\n    } else {\n        ssl_context_ = std::make_unique<SSLContext>(ssl_params_);\n    }\n}\n\nSSLSocketFactory::~SSLSocketFactory() = default;\n\nstd::unique_ptr<Socket> SSLSocketFactory::doConnect(const NetworkAddress& address, const ClientOptions& opts) {\n    SocketTimeoutParams timeout_params { opts.connection_connect_timeout, opts.connection_recv_timeout, opts.connection_send_timeout };\n    return std::make_unique<SSLSocket>(address, timeout_params, ssl_params_, *ssl_context_);\n}\n\nstd::unique_ptr<InputStream> SSLSocket::makeInputStream() const {\n    return std::make_unique<SSLSocketInput>(ssl_.get());\n}\n\nstd::unique_ptr<OutputStream> SSLSocket::makeOutputStream() const {\n    return std::make_unique<SSLSocketOutput>(ssl_.get());\n}\n\nSSLSocketInput::SSLSocketInput(SSL *ssl)\n    : ssl_(ssl)\n{}\n\nsize_t SSLSocketInput::DoRead(void* buf, size_t len) {\n    size_t actually_read;\n    HANDLE_SSL_ERROR(ssl_, SSL_read_ex(ssl_, buf, len, &actually_read));\n    return actually_read;\n}\n\nSSLSocketOutput::SSLSocketOutput(SSL *ssl)\n    : ssl_(ssl)\n{}\n\nsize_t SSLSocketOutput::DoWrite(const void* data, size_t len) {\n    if (len > std::numeric_limits<int>::max())\n        // FIXME(vnemkov): We should do multiple `SSL_write`s in this case.\n        throw AssertionError(\"Failed to write too big chunk at once \"\n                + std::to_string(len) + \" > \" + std::to_string(std::numeric_limits<int>::max()));\n    return static_cast<size_t>(HANDLE_SSL_ERROR(ssl_, SSL_write(ssl_, data, static_cast<int>(len))));\n}\n\n#undef HANDLE_SSL_ERROR\n\n}\n\n// Path: clickhouse/base/singleton.h\n#pragma once\n\nnamespace clickhouse {\n\ntemplate <typename T>\nT* Singleton() {\n    static T instance;\n    return &instance;\n}\n\n}\n\n// Path: clickhouse/base/socket.cpp\n#include \"socket.h\"\n#include \"singleton.h\"\n#include \"../client.h\"\n\n#include <assert.h>\n#include <stdexcept>\n#include <system_error>\n#include <unordered_set>\n#include <memory.h>\n#include <thread>\n\n#if !defined(_win_)\n#   include <errno.h>\n#   include <fcntl.h>\n#   include <netdb.h>\n#   include <netinet/tcp.h>\n#   include <signal.h>\n#   include <unistd.h>\n#endif\n\nnamespace clickhouse {\n\n#if defined(_win_)\nchar const* windowsErrorCategory::name() const noexcept {\n    return \"WindowsSocketError\";\n}\n\nstd::string windowsErrorCategory::message(int c) const {\n    char error[UINT8_MAX];\n    auto len = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, nullptr, static_cast<DWORD>(c), 0, error, sizeof(error), nullptr);\n    if (len == 0) {\n        return \"unknown\";\n    }\n    while (len && (error[len - 1] == '\\r' || error[len - 1] == '\\n')) {\n...\n", "needle_token_start": 2356, "needle_token_end": 2473, "code_context_ntokens": 16054, "output": ["```cpp\ntemplate <typename Container>\nstd::string ContainerToString(Container container, const char * separator = \", \") {\n    std::stringstream sstr;\n    const auto end = std::end(container);\n    for (auto i = std::begin(container); i != end; /*intentionally no ++i*/) {\n        const auto & elem = *i;\n        sstr << elem;\n\n        if (++i != end) {\n            sstr << separator;\n        }\n    }\n\n    return sstr.str();\n}\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "MapValueView", "language": "cpp", "path": "clickhouse/columns/map.h", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: Provides a read-only view of a single row in a map-type column, allowing access to elements in a map structure with linear complexity.\n2. **Input**: Receives a view of an array representing a single map entry, which includes both keys and values.\n3. **Output**: Enables access to the elements of the map as pairs of keys and values.\n4. **Procedure**: Initializes with an array view of a map entry and facilitates the retrieval of elements as key-value pairs, though each access operation has a linear time complexity due to the underlying data structure.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/columns/nullable.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n\n#include <optional>\n\nnamespace clickhouse {\n\n/**\n * Represents column of Nullable(T).\n */\nclass ColumnNullable : public Column {\npublic:\n    ColumnNullable(ColumnRef nested, ColumnRef nulls);\n\n    /// Appends one null flag to the end of the column\n    void Append(bool isnull);\n\n    /// Returns null flag at given row number.\n    bool IsNull(size_t n) const;\n\n    /// Returns nested column.\n    ColumnRef Nested() const;\n\n    /// Returns nulls column.\n    ColumnRef Nulls() const;\n\n...\n// Path: clickhouse/columns/lowcardinality.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n#include \"nullable.h\"\n\n#include <functional>\n#include <string>\n#include <unordered_map>\n#include <utility>\n\nnamespace clickhouse {\n\ntemplate <typename NestedColumnType>\nclass ColumnLowCardinalityT;\n\nnamespace details {\n\n/** LowCardinalityHashKey used as key in unique items hashmap to abstract away key value\n * (type of which depends on dictionary column) and to reduce likelehood of collisions.\n *\n * In order to dramatically reduce collision rate, we use 2 different hashes from 2 different hash functions.\n * First hash is used in hashtable (to calculate item position).\n * Second one is used as part of key value and accessed via `operator==()` upon collision resolution/detection.\n */\nusing LowCardinalityHashKey = std::pair<std::uint64_t, std::uint64_t>;\n\nstruct LowCardinalityHashKeyHash {\n    inline std::size_t operator()(const LowCardinalityHashKey &hash_key) const noexcept {\n        return hash_key.first;\n    }\n};\n\n}\n\n/*\n * LC column contains an \"invisible\" default item at the beginning of the collection. [default, ...]\n * If the nested type is Nullable, it contains a null-item at the beginning and a default item at the second position. [null, default, ...]\n * Null map is not serialized in LC columns. Instead, nulls are tracked by having an index of 0.\n * */\nclass ColumnLowCardinality : public Column {\npublic:\n    using UniqueItems = std::unordered_map<details::LowCardinalityHashKey, size_t /*dictionary index*/, details::LowCardinalityHashKeyHash>;\n\n    template <typename T>\n    friend class ColumnLowCardinalityT;\n\nprivate:\n    // IMPLEMENTATION NOTE: ColumnLowCardinalityT takes reference to underlying dictionary column object,\n    // so make sure to NOT change address of the dictionary object (with reset(), swap()) or with anything else.\n    ColumnRef dictionary_column_;\n    ColumnRef index_column_;\n    UniqueItems unique_items_map_;\n\npublic:\n    ColumnLowCardinality(ColumnLowCardinality&& col) = default;\n    // c-tor makes a deep copy of the dictionary_column.\n    explicit ColumnLowCardinality(ColumnRef dictionary_column);\n    explicit ColumnLowCardinality(std::shared_ptr<ColumnNullable> dictionary_column);\n\n    template <typename T>\n    explicit ColumnLowCardinality(std::shared_ptr<ColumnNullableT<T>> dictionary_column)\n        : ColumnLowCardinality(dictionary_column->template As<ColumnNullable>())\n    {}\n\n    ~ColumnLowCardinality();\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends another LowCardinality column to the end of this one, updating dictionary.\n    void Append(ColumnRef /*column*/) override;\n\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data.\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of current column, with compacted dictionary\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n    ItemView GetItem(size_t index) const override;\n\n    size_t GetDictionarySize() const;\n    TypeRef GetNestedType() const;\n\nprotected:\n    std::uint64_t getDictionaryIndex(std::uint64_t item_index) const;\n    void appendIndex(std::uint64_t item_index);\n    void removeLastIndex();\n    ColumnRef GetDictionary();\n\n    void AppendUnsafe(const ItemView &);\n\nprivate:\n    void Setup(ColumnRef dictionary_column);\n    void AppendNullItem();\n    void AppendDefaultItem();\n\npublic:\n    static details::LowCardinalityHashKey computeHashKey(const ItemView &);\n};\n\n/** Type-aware wrapper that provides simple convenience interface for accessing/appending individual items.\n */\ntemplate <typename DictionaryColumnType>\nclass ColumnLowCardinalityT : public ColumnLowCardinality {\n\n    DictionaryColumnType& typed_dictionary_;\n    const Type::Code type_;\n\npublic:\n    using WrappedColumnType = DictionaryColumnType;\n    // Type this column takes as argument of Append and returns with At() and operator[]\n    using ValueType = typename DictionaryColumnType::ValueType;\n\n    explicit ColumnLowCardinalityT(ColumnLowCardinality&& col)\n        : ColumnLowCardinality(std::move(col))\n        ,  typed_dictionary_(dynamic_cast<DictionaryColumnType &>(*GetDictionary()))\n        ,  type_(GetTypeCode(typed_dictionary_))\n    {\n    }\n\n    template <typename ...Args>\n    explicit ColumnLowCardinalityT(Args &&... args)\n        : ColumnLowCardinalityT(std::make_shared<DictionaryColumnType>(std::forward<Args>(args)...))\n    {}\n\n    // Create LC<T> column from existing T-column, making a deep copy of all contents.\n    explicit ColumnLowCardinalityT(std::shared_ptr<DictionaryColumnType> dictionary_col)\n        : ColumnLowCardinality(dictionary_col)\n        , typed_dictionary_(dynamic_cast<DictionaryColumnType &>(*GetDictionary()))\n        , type_(GetTypeCode(typed_dictionary_))\n    {}\n\n    /// Extended interface to simplify reading/adding individual items.\n\n    /// Returns element at given row number.\n    inline ValueType At(size_t n) const {\n        return typed_dictionary_.At(getDictionaryIndex(n));\n    }\n\n    /// Returns element at given row number.\n    inline ValueType operator [] (size_t n) const {\n        return typed_dictionary_[getDictionaryIndex(n)];\n    }\n\n    // so the non-virtual Append below doesn't shadow Append() from base class when compiled with older compilers.\n    using ColumnLowCardinality::Append;\n\n    inline void Append(const ValueType & value) {\n        if constexpr (IsNullable<WrappedColumnType>) {\n            if (value.has_value()) {\n                AppendUnsafe(ItemView{type_, *value});\n            } else {\n                AppendUnsafe(ItemView{});\n            }\n        } else {\n            AppendUnsafe(ItemView{type_, value});\n        }\n    }\n\n    template <typename T>\n    inline void AppendMany(const T& container) {\n        for (const auto & item : container) {\n            Append(item);\n        }\n    }\n\n    /** Create a ColumnLowCardinalityT from a ColumnLowCardinality, without copying data and offsets, but by\n     * 'stealing' those from `col`.\n     *\n     *  Ownership of column internals is transferred to returned object, original (argument) object\n     *  MUST NOT BE USED IN ANY WAY, it is only safe to dispose it.\n     *\n     *  Throws an exception if `col` is of wrong type, it is safe to use original col in this case.\n     *  This is a static method to make such conversion verbose.\n     */\n    static auto Wrap(ColumnLowCardinality&& col) {\n        return std::make_shared<ColumnLowCardinalityT<WrappedColumnType>>(std::move(col));\n    }\n\n    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnLowCardinality&&>(col))); }\n\n    // Helper to simplify integration with other APIs\n    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnLowCardinality>())); }\n\n    ColumnRef Slice(size_t begin, size_t size) const override {\n        return Wrap(ColumnLowCardinality::Slice(begin, size));\n    }\n\n    ColumnRef CloneEmpty() const override { return Wrap(ColumnLowCardinality::CloneEmpty()); }\n\nprivate:\n\n    template <typename T>\n    static auto GetTypeCode(T& column) {\n        if constexpr (IsNullable<T>) {\n            return GetTypeCode(*column.Nested()->template AsStrict<typename T::NestedColumnType>());\n        } else {\n            return column.Type()->GetCode();\n        }\n    }\n};\n\n}\n\n// Path: clickhouse/base/projected_iterator.h\n#pragma once\n\n#include <iterator>\n#include <type_traits>\n#include <utility>\n\nnamespace clickhouse {\n\ntemplate <typename UnaryFunction, typename Iterator, typename Reference = decltype(std::declval<UnaryFunction>()(std::declval<Iterator>())),\n          typename Value = std::decay_t<Reference>>\nclass ProjectedIterator {\npublic:\n    using value_type        = Value;\n    using reference         = Reference;\n    using pointer           = Reference;\n    using difference_type   = typename std::iterator_traits<Iterator>::difference_type;\n    using iterator_category = typename std::iterator_traits<Iterator>::iterator_category;\n\n    ProjectedIterator() = default;\n\n    inline ProjectedIterator(Iterator const& iterator, UnaryFunction functor)\n        : iterator_(iterator)\n        , functor_(std::move(functor)) {\n    }\n\n    inline UnaryFunction functor() const { return functor; }\n\n    inline Iterator const& base() const { return iterator_; }\n\n    inline reference operator*() const { return functor_(iterator_); }\n\n    inline ProjectedIterator& operator++() {\n        ++iterator_;\n        return *this;\n    }\n\n    inline ProjectedIterator& operator--() {\n        --iterator_;\n        return *this;\n    }\n\n    inline bool operator==(const ProjectedIterator& other) const {\n        return this->iterator_ == other.iterator_;\n    }\n\n    inline bool operator!=(const ProjectedIterator& other) const {\n        return !(*this == other);\n    }\n\nprivate:\n    Iterator iterator_;\n    UnaryFunction functor_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/map.h\n#pragma once\n\n#include \"../base/projected_iterator.h\"\n#include \"array.h\"\n#include \"column.h\"\n#include \"tuple.h\"\n\n#include <functional>\n#include <map>\n\nnamespace clickhouse {\n\ntemplate <typename KeyColumnType, typename ValueColumnType>\nclass ColumnMapT;\n\n/**\n * Represents column of Map(K, V).\n */\nclass ColumnMap : public Column {\npublic:\n    /** Create a map of given type, with actual values and offsets.\n     *\n     *  Both `data` and `offsets` are used (and modified) internally bye ColumnArray.\n     *  Users are strongly advised against modifying contents of `data` or `offsets` afterwards.\n     */\n    explicit ColumnMap(ColumnRef data);\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column prefix from input stream.\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t, size_t) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column&) override;\n\n    /// Converts map at pos n to column.\n    /// Type of row is tuple {key, value}.\n    ColumnRef GetAsColumn(size_t n) const;\n\nprotected:\n    template <typename K, typename V>\n    friend class ColumnMapT;\n\n    ColumnMap(ColumnMap&& map);\n\nprivate:\n    std::shared_ptr<ColumnArray> data_;\n};\n\ntemplate <typename K, typename V>\nclass ColumnMapT : public ColumnMap {\npublic:\n    using KeyColumnType   = K;\n    using ValueColumnType = V;\n    using Key             = std::decay_t<decltype(std::declval<KeyColumnType>().At(0))>;\n    using Value           = std::decay_t<decltype(std::declval<ValueColumnType>().At(0))>;\n    using TupleColumnType = ColumnTupleT<KeyColumnType, ValueColumnType>;\n    using ArrayColumnType = ColumnArrayT<TupleColumnType>;\n\n    ColumnMapT(ColumnRef data)\n        : ColumnMap(data), typed_data_(data->AsStrict<ColumnArrayT<TupleColumnType>>()) {}\n\n    ColumnMapT(std::shared_ptr<KeyColumnType> keys, std::shared_ptr<ValueColumnType> values)\n        : ColumnMap(std::make_shared<ArrayColumnType>(std::make_shared<TupleColumnType>(\n              std::make_tuple(std::move(keys), std::move(values))))),\n          typed_data_(data_->template As<ArrayColumnType>()) {}\n\n    ColumnRef Slice(size_t begin, size_t len) const override {\n        return std::make_shared<ColumnMapT<K, V>>(typed_data_->Slice(begin, len));\n    }\n\n    ColumnRef CloneEmpty() const override {\n        return std::make_shared<ColumnMapT<K, V>>(typed_data_->CloneEmpty());\n    }\n\n    void Swap(Column& other) override {\n        auto& col = dynamic_cast<ColumnMapT<K, V>&>(other);\n        col.typed_data_.swap(typed_data_);\n        ColumnMap::Swap(other);\n    }\n\n    /// A single (row) value of the Map-column i.e. read-only map.\n    /// It has a linear time complexity to access items\n    /// Because data base type has same structure\n    /// \"This lookup works now with a linear complexity.\"\n    /// https://clickhouse.com/docs/en/sql-reference/data-types/map\n    /// Convert it to a suitable container required to access more than one element\n\n    class MapValueView {\n        const typename ArrayColumnType::ArrayValueView data_;\n\n    public:\n        using ValueType = std::pair<Key, Value>;\n\n        \nMapValueView(typename ArrayColumnType::ArrayValueView data) : data_(std::move(data)) {}\n\n        inline auto operator[](const Key& key) const { return (*Find(key)).second; }\n\n        inline auto At(const Key& key) const {\n            auto it = Find(key);\n            if (it == end()) throw ValidationError(\"ColumnMap value key not found\");\n            return (*it).second;\n        }\n\n        class Iterator {\n            typename ArrayColumnType::ArrayValueView::Iterator data_iterator_;\n\n        public:\n            Iterator() = default;\n\n            Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator)\n                : data_iterator_(data_iterator) {}\n\n            using ValueType = std::pair<Key, Value>;\n            using difference_type = size_t;\n            using value_type = ValueType;\n            using pointer = void;\n            using reference = ValueType&;\n            using iterator_category = std::forward_iterator_tag;\n\n            inline auto operator*() const {\n                auto tuple = *data_iterator_;\n                return ValueType{std::get<0>(tuple), std::get<1>(tuple)};\n            }\n\n            inline Iterator& operator++() {\n                ++data_iterator_;\n                return *this;\n            }\n\n            inline bool operator==(const Iterator& other) const {\n                return this->data_iterator_ == other.data_iterator_;\n            }\n\n            inline bool operator!=(const Iterator& other) const { return !(*this == other); }\n        };\n\n        // minimalistic stl-like container interface, hence the lowercase\n        inline Iterator begin() const { return Iterator{data_.begin()}; }\n\n        inline Iterator cbegin() const { return Iterator{data_.cbegin()}; }\n\n        inline Iterator end() const { return Iterator{data_.end()}; }\n\n        inline Iterator cend() const { return Iterator{data_.cend()}; }\n\n        inline size_t size() const { return data_.size(); }\n\n        // It is ugly to have both size() and Size(), but it is for compatitability with both STL\n        // and rest of the clickhouse-cpp.\n        inline size_t Size() const { return data_.Size(); }\n\n        inline size_t Count(const Key& key) const {\n            size_t result = 0;\n            for (auto item : data_) {\n                if (std::get<0>(item) == key) {\n                    ++result;\n                }\n            }\n            return result;\n        }\n\n        inline Iterator Find(const Key& key) const {\n            for (auto it = data_.begin(); it != data_.end(); ++it) {\n                if (std::get<0>(*it) == key) {\n                    return Iterator{it};\n                }\n            }\n            return end();\n        }\n\n        inline bool operator==(const MapValueView& other) const {\n            if (size() != other.size()) {\n                return false;\n            }\n            const auto make_index = [](const auto& data) {\n                std::vector<size_t> result{data.Size()};\n                std::generate(result.begin(), result.end(), [i = 0] () mutable {return i++;});\n                std::sort(result.begin(), result.end(), [&data](size_t l, size_t r) {return data[l] < data[r];});\n                return result;\n            };\n            const auto index = make_index(data_);\n            for (const auto& val : other.data_) {\n                if (!std::binary_search(index.begin(), index.end(), val,\n                        [&data = data_](const auto& l, size_t r) {return l < data[r];})) {\n                    return false;\n                }\n            }\n            return true;\n        }\n\n        inline bool operator!=(const MapValueView& other) const { return !(*this == other); }\n    };\n\n    inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; }\n\n    inline auto operator[](size_t index) const { return At(index); }\n\n    using ColumnMap::Append;\n\n    inline void Append(const MapValueView& value) { typed_data_->Append(value.data_); }\n\n    inline void Append(const std::vector<std::tuple<Key, Value>>& tuples) {\n        typed_data_->Append(tuples.begin(), tuples.end());\n    }\n\n    template <typename T>\n    inline void Append(const T& value) {\n        using BaseIter = decltype(value.begin());\n        using KeyOfT = decltype(std::declval<BaseIter>()->first);\n        using ValOfT = decltype(std::declval<BaseIter>()->second);\n        using Functor = std::function<std::tuple<KeyOfT, ValOfT>(const BaseIter&)>;\n        using Iterator = ProjectedIterator<Functor, BaseIter>;\n\n        Functor functor = [](const BaseIter& i) {\n            return std::make_tuple(std::cref(i->first), std::cref(i->second));\n        };\n\n        typed_data_->Append(Iterator{value.begin(), functor}, Iterator{value.end(), functor});\n    }\n\n    static auto Wrap(ColumnMap&& col) {\n        auto data = ArrayColumnType::Wrap(std::move(col.data_));\n        return std::make_shared<ColumnMapT<K, V>>(std::move(data));\n    }\n\n    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnMap&&>(col))); }\n\n    // Helper to simplify integration with other APIs\n    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnMap>())); }\n\nprivate:\n    std::shared_ptr<ArrayColumnType> typed_data_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/uuid.h\n#pragma once\n\n#include \"../base/uuid.h\"\n#include \"column.h\"\n#include \"numeric.h\"\n\nnamespace clickhouse {\n\n\n/**\n * Represents a UUID column.\n */\nclass ColumnUUID : public Column {\npublic:\n    ColumnUUID();\n\n    explicit ColumnUUID(ColumnRef data);\n\n    /// Appends one element to the end of column.\n    void Append(const UUID& value);\n\n    /// Returns element at given row number.\n    const UUID At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline const UUID operator [] (size_t n) const { return At(n); }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\n    ItemView GetItem(size_t) const override;\n\nprivate:\n    std::shared_ptr<ColumnUInt64> data_;\n};\n\n}\n\n// Path: clickhouse/client.h\n#pragma once\n\n#include \"query.h\"\n#include \"exceptions.h\"\n\n#include \"columns/array.h\"\n#include \"columns/date.h\"\n#include \"columns/decimal.h\"\n#include \"columns/enum.h\"\n#include \"columns/geo.h\"\n#include \"columns/ip4.h\"\n#include \"columns/ip6.h\"\n#include \"columns/lowcardinality.h\"\n#include \"columns/nullable.h\"\n#include \"columns/numeric.h\"\n#include \"columns/map.h\"\n#include \"columns/string.h\"\n#include \"columns/tuple.h\"\n#include \"columns/uuid.h\"\n\n#include <chrono>\n#include <cstdint>\n#include <memory>\n#include <ostream>\n#include <string>\n#include <optional>\n\ntypedef struct ssl_ctx_st SSL_CTX;\n\nnamespace clickhouse {\n\nstruct ServerInfo {\n    std::string name;\n    std::string timezone;\n    std::string display_name;\n    uint64_t    version_major;\n    uint64_t    version_minor;\n    uint64_t    version_patch;\n    uint64_t    revision;\n};\n\n/// Methods of block compression.\nenum class CompressionMethod {\n    None    = -1,\n    LZ4     =  1,\n};\n\nstruct Endpoint {\n    std::string host;\n    uint16_t port = 9000;\n    inline bool operator==(const Endpoint& right) const {\n        return host == right.host && port == right.port;\n    }\n};\n\nenum class EndpointsIterationAlgorithm {\n    RoundRobin = 0,\n};\n\nstruct ClientOptions {\n    // Setter goes first, so it is possible to apply 'deprecated' annotation safely.\n#define DECLARE_FIELD(name, type, setter, default_value) \\\n    inline auto & setter(const type& value) { \\\n        name = value; \\\n        return *this; \\\n    } \\\n    type name = default_value\n\n    /// Hostname of the server.\n    DECLARE_FIELD(host, std::string, SetHost, std::string());\n    /// Service port.\n    DECLARE_FIELD(port, uint16_t, SetPort, 9000);\n\n    /** Set endpoints (host+port), only one is used.\n     * Client tries to connect to those endpoints one by one, on the round-robin basis:\n     * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(),\n     * the first one to establish connection is used for the rest of the session.\n     * If port isn't specified, default(9000) value will be used.\n     */\n    DECLARE_FIELD(endpoints, std::vector<Endpoint>, SetEndpoints, {});\n\n    /// Default database.\n    DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, \"default\");\n    /// User name.\n    DECLARE_FIELD(user, std::string, SetUser, \"default\");\n    /// Access password.\n    DECLARE_FIELD(password, std::string, SetPassword, std::string());\n\n    /// By default all exceptions received during query execution will be\n    /// passed to OnException handler.  Set rethrow_exceptions to true to\n    /// enable throwing exceptions with standard c++ exception mechanism.\n    DECLARE_FIELD(rethrow_exceptions, bool, SetRethrowException, true);\n\n    /// Ping server every time before execute any query.\n    DECLARE_FIELD(ping_before_query, bool, SetPingBeforeQuery, false);\n    /// Count of retry to send request to server.\n    DECLARE_FIELD(send_retries, unsigned int, SetSendRetries, 1);\n    /// Amount of time to wait before next retry.\n    DECLARE_FIELD(retry_timeout, std::chrono::seconds, SetRetryTimeout, std::chrono::seconds(5));\n\n    /// Compression method.\n    DECLARE_FIELD(compression_method, CompressionMethod, SetCompressionMethod, CompressionMethod::None);\n\n    /// TCP Keep alive options\n    DECLARE_FIELD(tcp_keepalive, bool, TcpKeepAlive, false);\n    DECLARE_FIELD(tcp_keepalive_idle, std::chrono::seconds, SetTcpKeepAliveIdle, std::chrono::seconds(60));\n    DECLARE_FIELD(tcp_keepalive_intvl, std::chrono::seconds, SetTcpKeepAliveInterval, std::chrono::seconds(5));\n    DECLARE_FIELD(tcp_keepalive_cnt, unsigned int, SetTcpKeepAliveCount, 3);\n\n    // TCP options\n    DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true);\n\n    /// Connection socket connect timeout. If the timeout is negative then the connect operation will never timeout.\n    DECLARE_FIELD(connection_connect_timeout, std::chrono::milliseconds, SetConnectionConnectTimeout, std::chrono::seconds(5));\n\n    /// Connection socket timeout. If the timeout is set to zero then the operation will never timeout.\n    DECLARE_FIELD(connection_recv_timeout, std::chrono::milliseconds, SetConnectionRecvTimeout, std::chrono::milliseconds(0));\n    DECLARE_FIELD(connection_send_timeout, std::chrono::milliseconds, SetConnectionSendTimeout, std::chrono::milliseconds(0));\n\n    /** It helps to ease migration of the old codebases, which can't afford to switch\n    * to using ColumnLowCardinalityT or ColumnLowCardinality directly,\n    * but still want to benefit from smaller on-wire LowCardinality bandwidth footprint.\n    *\n    * @see LowCardinalitySerializationAdaptor, CreateColumnByType\n    */\n    [[deprecated(\"Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) \")]]\n    DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, false);\n\n    /** Set max size data to compress if compression enabled.\n     *\n     *  Allows choosing tradeoff between RAM\\CPU:\n     *  - Lower value reduces RAM usage, but slightly increases CPU usage.\n     *  - Higher value increases RAM usage but slightly decreases CPU usage.\n     */\n    DECLARE_FIELD(max_compression_chunk_size, unsigned int, SetMaxCompressionChunkSize, 65535);\n\n    struct SSLOptions {\n        /** There are two ways to configure an SSL connection:\n         *  - provide a pre-configured SSL_CTX, which is not modified and not owned by the Client.\n         *  - provide a set of options and allow the Client to create and configure SSL_CTX by itself.\n         */\n\n        /** Pre-configured SSL-context for SSL-connection.\n         *  If NOT null client DONES NOT take ownership of context and it must be valid for client lifetime.\n         *  If null client initlaizes OpenSSL and creates his own context, initializes it using\n         *  other options, like path_to_ca_files, path_to_ca_directory, use_default_ca_locations, etc.\n         *\n         *  Either way context is used to create an SSL-connection, which is then configured with\n         *  whatever was provided as `configuration`, `host_flags`, `skip_verification` and `use_sni`.\n         */\n        SSL_CTX * ssl_context = nullptr;\n        auto & SetExternalSSLContext(SSL_CTX * new_ssl_context) {\n            ssl_context = new_ssl_context;\n            return *this;\n        }\n\n        /** Means to validate the server-supplied certificate against trusted Certificate Authority (CA).\n         *  If no CAs are configured, the server's identity can't be validated, and the Client would err.\n         *  See https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_default_verify_paths.html\n        */\n        /// Load default CA certificates from default locations.\n        DECLARE_FIELD(use_default_ca_locations, bool, SetUseDefaultCALocations, true);\n        /// Path to the CA files to verify server certificate, may be empty.\n        DECLARE_FIELD(path_to_ca_files, std::vector<std::string>, SetPathToCAFiles, {});\n        /// Path to the directory with CA files used to validate server certificate, may be empty.\n        DECLARE_FIELD(path_to_ca_directory, std::string, SetPathToCADirectory, \"\");\n\n        /** Min and max protocol versions to use, set with SSL_CTX_set_min_proto_version and SSL_CTX_set_max_proto_version\n         *  for details see https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_min_proto_version.html\n         */\n        DECLARE_FIELD(min_protocol_version, int, SetMinProtocolVersion, DEFAULT_VALUE);\n        DECLARE_FIELD(max_protocol_version, int, SetMaxProtocolVersion, DEFAULT_VALUE);\n\n        /** Options to be set with SSL_CTX_set_options,\n         * for details see https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_options.html\n        */\n        DECLARE_FIELD(context_options, int, SetContextOptions, DEFAULT_VALUE);\n\n        /** Use SNI at ClientHello\n         */\n        DECLARE_FIELD(use_sni, bool, SetUseSNI, true);\n\n        /** Skip SSL session verification (server's certificate, etc).\n         *\n         *  WARNING: settig to true will bypass all SSL session checks, which\n         *  is dangerous, but can be used against self-signed certificates, e.g. for testing purposes.\n         */\n        DECLARE_FIELD(skip_verification, bool, SetSkipVerification, false);\n\n        /** Mode of verifying host ssl certificate against name of the host, set with SSL_set_hostflags.\n         *  For details see https://www.openssl.org/docs/man1.1.1/man3/SSL_set_hostflags.html\n         */\n        DECLARE_FIELD(host_flags, int, SetHostVerifyFlags, DEFAULT_VALUE);\n\n        struct CommandAndValue {\n            std::string command;\n            std::optional<std::string> value = std::nullopt;\n        };\n        /** Extra configuration options, set with SSL_CONF_cmd.\n         *  For deatils see https://www.openssl.org/docs/man1.1.1/man3/SSL_CONF_cmd.html\n         *\n         *  Takes multiple pairs of command-value strings, all commands are supported,\n         *  and prefix is empty.\n         *  i.e. pass `sigalgs` or `SignatureAlgorithms` instead of `-sigalgs`.\n         *\n         *  Rewrites any other options/flags if set in other ways.\n         */\n        DECLARE_FIELD(configuration, std::vector<CommandAndValue>, SetConfiguration, {});\n\n        static const int DEFAULT_VALUE = -1;\n    };\n\n    // By default SSL is turned off.\n    std::optional<SSLOptions> ssl_options = std::nullopt;\n\n    // Will throw an exception if client was built without SSL support.\n    ClientOptions& SetSSLOptions(SSLOptions options);\n\n#undef DECLARE_FIELD\n};\n\nstd::ostream& operator<<(std::ostream& os, const ClientOptions& options);\nstd::ostream& operator<<(std::ostream& os, const Endpoint& options);\n\nclass SocketFactory;\n\n/**\n *\n */\nclass Client {\npublic:\n     Client(const ClientOptions& opts);\n     Client(const ClientOptions& opts,\n            std::unique_ptr<SocketFactory> socket_factory);\n    ~Client();\n\n    /// Intends for execute arbitrary queries.\n    void Execute(const Query& query);\n\n    /// Intends for execute select queries.  Data will be returned with\n    /// one or more call of \\p cb.\n    void Select(const std::string& query, SelectCallback cb);\n    void Select(const std::string& query, const std::string& query_id, SelectCallback cb);\n\n    /// Executes a select query which can be canceled by returning false from\n    /// the data handler function \\p cb.\n    void SelectCancelable(const std::string& query, SelectCancelableCallback cb);\n    void SelectCancelable(const std::string& query, const std::string& query_id, SelectCancelableCallback cb);\n\n    /// Alias for Execute.\n    void Select(const Query& query);\n\n    /// Intends for insert block of data into a table \\p table_name.\n    void Insert(const std::string& table_name, const Block& block);\n    void Insert(const std::string& table_name, const std::string& query_id, const Block& block);\n\n    /// Ping server for aliveness.\n    void Ping();\n\n    /// Reset connection with initial params.\n    void ResetConnection();\n\n    const ServerInfo& GetServerInfo() const;\n\n    /// Get current connected endpoint.\n    /// In case when client is not connected to any endpoint, nullopt will returned.\n    const std::optional<Endpoint>& GetCurrentEndpoint() const;\n\n    // Try to connect to different endpoints one by one only one time. If it doesn't work, throw an exception.\n    void ResetConnectionEndpoint();\n\n    struct Version\n    {\n        uint16_t major;\n        uint16_t minor;\n        uint16_t patch;\n        uint16_t build;\n        const char * extra;\n    };\n\n    static Version GetVersion();\n\nprivate:\n    const ClientOptions options_;\n\n    class Impl;\n    std::unique_ptr<Impl> impl_;\n};\n\n}\n\n// Path: clickhouse/protocol.h\n#pragma once\n\nnamespace clickhouse {\n\n    /// Types of packets received from server\n    namespace ServerCodes {\n        enum {\n            Hello                = 0,    /// Name, version, revision.\n            Data                 = 1,    /// `Block` of data, may be compressed.\n            Exception            = 2,    /// Exception that occurred on server side during query execution.\n            Progress             = 3,    /// Query execcution progress: rows and bytes read.\n            Pong                 = 4,    /// response to Ping sent by client.\n            EndOfStream          = 5,    /// All packets were sent.\n            ProfileInfo          = 6,    /// Profiling data\n            Totals               = 7,    /// Block of totals, may be compressed.\n            Extremes             = 8,    /// Block of mins and maxs, may be compressed.\n            TablesStatusResponse = 9,    /// Response to TableStatus.\n            Log                  = 10,   /// Query execution log.\n            TableColumns         = 11,   /// Columns' description for default values calculation\n            PartUUIDs            = 12,   /// List of unique parts ids.\n            ReadTaskRequest      = 13,   /// String (UUID) describes a request for which next task is needed\n                                         /// This is such an inverted logic, where server sends requests\n                                         /// And client returns back response\n            ProfileEvents        = 14,   /// Packet with profile events from server.\n        };\n    }\n\n    /// Types of packets sent by client.\n    namespace ClientCodes {\n        enum {\n            Hello       = 0,    /// Name, version, default database name.\n            Query       = 1,    /** Query id, query settings, query processing stage,\n                                  * compression status, and query text (no INSERT data).\n                                  */\n            Data        = 2,    /// Data `Block` (e.g. INSERT data), may be compressed.\n            Cancel      = 3,    /// Cancel query.\n            Ping        = 4,    /// Check server connection.\n        };\n    }\n\n    /// Should we compress `Block`s of data\n    namespace CompressionState {\n        enum {\n            Disable     = 0,\n            Enable      = 1,\n        };\n    }\n\n    namespace Stages {\n        enum {\n            Complete    = 2,\n        };\n    }\n}\n\n// Path: clickhouse/base/input.h\n#pragma once\n\n#include <cstddef>\n#include <cstdint>\n#include <vector>\n#include <memory>\n\nnamespace clickhouse {\n\nclass InputStream {\npublic:\n    virtual ~InputStream() noexcept (false)\n    { }\n\n    /// Reads one byte from the stream.\n    inline bool ReadByte(uint8_t* byte) {\n        return DoRead(byte, sizeof(uint8_t)) == sizeof(uint8_t);\n    }\n\n    /// Reads some data from the stream.\n    inline size_t Read(void* buf, size_t len) {\n        return DoRead(buf, len);\n    }\n\n    // Skips a number of bytes.  Returns false if an underlying read error occurs.\n    virtual bool Skip(size_t bytes) = 0;\n\nprotected:\n    virtual size_t DoRead(void* buf, size_t len) = 0;\n};\n\n\nclass ZeroCopyInput : public InputStream {\npublic:\n    inline size_t Next(const void** buf, size_t len) {\n        return DoNext(buf, len);\n    }\n\n    bool Skip(size_t bytes) override;\n\nprotected:\n    virtual size_t DoNext(const void** ptr, size_t len) = 0;\n\n    size_t DoRead(void* buf, size_t len) override;\n};\n\n\n/**\n * A ZeroCopyInput stream backed by an in-memory array of bytes.\n */\nclass ArrayInput : public ZeroCopyInput {\npublic:\n     ArrayInput() noexcept;\n     ArrayInput(const void* buf, size_t len) noexcept;\n    ~ArrayInput() override;\n\n    /// Number of bytes available in the stream.\n    inline size_t Avail() const noexcept {\n        return len_;\n    }\n\n    /// Current read position in the memory block used by this stream.\n    inline const uint8_t* Data() const noexcept {\n        return data_;\n    }\n\n    /// Whether there is more data in the stream.\n    inline bool Exhausted() const noexcept {\n        return !Avail();\n    }\n\n    inline void Reset(const void* buf, size_t len) noexcept {\n        data_ = static_cast<const uint8_t*>(buf);\n        len_ = len;\n    }\n\nprivate:\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    const uint8_t* data_;\n    size_t len_;\n};\n\n\nclass BufferedInput : public ZeroCopyInput {\npublic:\n    BufferedInput(std::unique_ptr<InputStream> source, size_t buflen = 8192);\n    ~BufferedInput() override;\n\n    void Reset();\n\nprotected:\n    size_t DoRead(void* buf, size_t len) override;\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    std::unique_ptr<InputStream> const source_;\n    ArrayInput array_input_;\n    std::vector<uint8_t> buffer_;\n};\n\n}\n\n// Path: clickhouse/base/buffer.h\n#pragma once\n\n#include <cstdint>\n#include <vector>\n\nnamespace clickhouse {\n\nusing Buffer = std::vector<uint8_t>;\n\n}\n\n// Path: clickhouse/base/output.h\n#pragma once\n\n#include \"buffer.h\"\n\n#include <cstddef>\n#include <cstdint>\n#include <vector>\n#include <memory.h>\n#include <memory>\n\nnamespace clickhouse {\n\nclass OutputStream {\npublic:\n    virtual ~OutputStream()\n    { }\n\n    inline void Flush() {\n        DoFlush();\n    }\n\n    inline size_t Write(const void* data, size_t len) {\n        return DoWrite(data, len);\n    }\n\nprotected:\n    virtual void DoFlush() { }\n\n    virtual size_t DoWrite(const void* data, size_t len) = 0;\n};\n\n\nclass ZeroCopyOutput : public OutputStream {\npublic:\n    inline size_t Next(void** data, size_t size) {\n        return DoNext(data, size);\n    }\n\nprotected:\n    // Obtains a buffer into which data can be written.  Any data written\n    // into this buffer will eventually (maybe instantly, maybe later on)\n    // be written to the output.\n    virtual size_t DoNext(void** data, size_t len) = 0;\n\n    size_t DoWrite(const void* data, size_t len) override;\n};\n\n\n/**\n * A ZeroCopyOutput stream backed by an in-memory array of bytes.\n */\nclass ArrayOutput : public ZeroCopyOutput {\npublic:\n     ArrayOutput(void* buf, size_t len);\n    ~ArrayOutput() override;\n\n    /// Number of bytes available in the stream.\n    inline size_t Avail() const noexcept {\n        return end_ - buf_;\n    }\n\n    /// Current write position in the memory block used by this stream.\n    inline const uint8_t* Data() const noexcept {\n        return buf_;\n    }\n\n    /// Whether there is more space in the stream.\n    inline bool Exhausted() const noexcept {\n        return !Avail();\n    }\n\n    /// Initializes this stream with a new memory block.\n    inline void Reset(void* buf, size_t len) noexcept {\n        buf_ = static_cast<uint8_t*>(buf);\n        end_ = buf_ + len;\n        buffer_size_ = len;\n    }\n\n    /// Number of bytes written to the buffer.\n    inline size_t Size() const noexcept {\n        return buffer_size_ - Avail();\n    }\n\nprotected:\n    size_t DoNext(void** data, size_t len) override;\n\nprivate:\n    uint8_t* buf_;\n    uint8_t* end_;\n    size_t buffer_size_;\n};\n\n\n/**\n * A ZeroCopyOutput stream backed by a vector.\n *\n * Doesn't Flush() in destructor, client must ensure to do it manually at some point.\n */\nclass BufferOutput : public ZeroCopyOutput {\npublic:\n     BufferOutput(Buffer* buf);\n    ~BufferOutput() override;\n\nprotected:\n    size_t DoNext(void** data, size_t len) override;\n\nprivate:\n    Buffer* buf_;\n    size_t pos_;\n};\n\n/** BufferedOutput writes data to internal buffer first.\n *\n *  Any data goes to underlying stream only if internal buffer is full\n *  or when client invokes Flush() on this.\n *\n * Doesn't Flush() in destructor, client must ensure to do it manually at some point.\n */\nclass BufferedOutput : public ZeroCopyOutput {\npublic:\n    explicit BufferedOutput(std::unique_ptr<OutputStream> destination, size_t buflen = 8192);\n    ~BufferedOutput() override;\n\n    void Reset();\n\nprotected:\n    void DoFlush() override;\n    size_t DoNext(void** data, size_t len) override;\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    std::unique_ptr<OutputStream> const destination_;\n    Buffer buffer_;\n    ArrayOutput array_output_;\n};\n\ntemplate <typename T>\nvoid WriteUnaligned(void* buf, const T& value) {\n    memcpy(buf, &value, sizeof(value));\n}\n\n}\n\n// Path: clickhouse/base/compressed.h\n#pragma once\n\n#include \"input.h\"\n#include \"output.h\"\n#include \"buffer.h\"\n\nnamespace clickhouse {\n\nclass CompressedInput : public ZeroCopyInput {\npublic:\n    explicit CompressedInput(InputStream* input);\n    ~CompressedInput() override;\n\nprotected:\n    size_t DoNext(const void** ptr, size_t len) override;\n\n    bool Decompress();\n\nprivate:\n    InputStream* const input_;\n\n    Buffer data_;\n    ArrayInput mem_;\n};\n\nclass CompressedOutput : public OutputStream {\npublic:\n    explicit CompressedOutput(OutputStream * destination, size_t max_compressed_chunk_size = 0);\n    ~CompressedOutput() override;\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n    void DoFlush() override;\n\nprivate:\n    void Compress(const void * data, size_t len);\n    void PreallocateCompressBuffer(size_t input_size);\n\nprivate:\n    OutputStream * destination_;\n    const size_t max_compressed_chunk_size_;\n    Buffer compressed_buffer_;\n};\n\n}\n\n// Path: clickhouse/base/platform.h\n#pragma once\n\n#if defined(__linux__)\n#   define _linux_\n#elif defined(_WIN64)\n#   define _win64_\n#   define _win32_\n#elif defined(__WIN32__) || defined(_WIN32)\n#   define _win32_\n#elif defined(__APPLE__)\n#   define _darwin_\n#endif\n\n#if defined(_win32_) || defined(_win64_)\n#   define _win_\n#   if !defined(_WIN32_WINNT) || (_WIN32_WINNT < 0x0600)\n#      undef _WIN32_WINNT\n#      define _WIN32_WINNT 0x0600 // The WSAPoll function is defined on Windows Vista and later.\n#   endif\n#   define WIN32_LEAN_AND_MEAN 1  // don't include too much header automatically\n#endif\n\n#if defined(_linux_) || defined (_darwin_)\n#   define _unix_\n#endif\n\n#if defined(_MSC_VER)\n#   undef NOMINMAX\n#   define NOMINMAX\n#   include <basetsd.h>\n#   define ssize_t SSIZE_T\n#   define HAVE_SSIZE_T 1\n#endif\n\n// Path: clickhouse/base/endpoints_iterator.h\n#pragma once\n\n#include \"clickhouse/client.h\"\n#include <vector>\n\nnamespace clickhouse {\n\nstruct ClientOptions;\n\n/**\n * Base class for iterating through endpoints.\n*/\nclass EndpointsIteratorBase\n{\n public:\n   virtual ~EndpointsIteratorBase() = default;\n\n   virtual Endpoint Next() = 0;\n};\n\nclass RoundRobinEndpointsIterator : public EndpointsIteratorBase\n{\n public:\n    explicit RoundRobinEndpointsIterator(const std::vector<Endpoint>& opts);\n    Endpoint Next() override;\n\n    ~RoundRobinEndpointsIterator() override;\n\n private:\n    const std::vector<Endpoint>& endpoints;\n    size_t current_index;\n};\n\n}\n\n// Path: clickhouse/base/socket.h\n#pragma once\n\n#include \"platform.h\"\n#include \"input.h\"\n#include \"output.h\"\n#include \"endpoints_iterator.h\"\n\n#include <cstddef>\n#include <string>\n#include <chrono>\n\n#if defined(_win_)\n#   include <winsock2.h>\n#   include <ws2tcpip.h>\n#else\n#   include <arpa/inet.h>\n#   include <sys/types.h>\n#   include <sys/socket.h>\n#   include <poll.h>\n\n#   if !defined(SOCKET)\n#       define SOCKET int\n#   endif\n#endif\n\n#include <memory>\n#include <system_error>\n\nstruct addrinfo;\n\nnamespace clickhouse {\n\nstruct ClientOptions;\n\n/** Address of a host to establish connection to.\n *\n */\nclass NetworkAddress {\npublic:\n    explicit NetworkAddress(const std::string& host,\n                            const std::string& port = \"0\");\n    ~NetworkAddress();\n\n    const struct addrinfo* Info() const;\n    const std::string & Host() const;\n\nprivate:\n    const std::string host_;\n    struct addrinfo* info_;\n};\n\n#if defined(_win_)\n\nclass windowsErrorCategory : public std::error_category {\npublic:\n    char const* name() const noexcept override final;\n    std::string message(int c) const override final;\n\n    static windowsErrorCategory const& category();\n};\n\n#endif\n\n#if defined(_unix_)\n\nclass getaddrinfoErrorCategory : public std::error_category {\npublic:\n    char const* name() const noexcept override final;\n    std::string message(int c) const override final;\n\n    static getaddrinfoErrorCategory const& category();\n};\n\n#endif\n\n\nclass SocketBase {\npublic:\n    virtual ~SocketBase();\n\n    virtual std::unique_ptr<InputStream> makeInputStream() const = 0;\n    virtual std::unique_ptr<OutputStream> makeOutputStream() const = 0;\n};\n\n\nclass SocketFactory {\npublic:\n    virtual ~SocketFactory();\n\n    // TODO: move connection-related options to ConnectionOptions structure.\n\n    virtual std::unique_ptr<SocketBase> connect(const ClientOptions& opts, const Endpoint& endpoint) = 0;\n\n    virtual void sleepFor(const std::chrono::milliseconds& duration);\n};\n\n\nstruct SocketTimeoutParams {\n    std::chrono::milliseconds connect_timeout{ 5000 };\n    std::chrono::milliseconds recv_timeout{ 0 };\n    std::chrono::milliseconds send_timeout{ 0 };\n};\n\nclass Socket : public SocketBase {\npublic:\n    Socket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params);\n    Socket(const NetworkAddress& addr);\n    Socket(Socket&& other) noexcept;\n    Socket& operator=(Socket&& other) noexcept;\n\n    ~Socket() override;\n\n    /// @params idle the time (in seconds) the connection needs to remain\n    ///         idle before TCP starts sending keepalive probes.\n    /// @params intvl the time (in seconds) between individual keepalive probes.\n    /// @params cnt the maximum number of keepalive probes TCP should send\n    ///         before dropping the connection.\n    void SetTcpKeepAlive(int idle, int intvl, int cnt) noexcept;\n\n    /// @params nodelay whether to enable TCP_NODELAY\n    void SetTcpNoDelay(bool nodelay) noexcept;\n\n    std::unique_ptr<InputStream> makeInputStream() const override;\n    std::unique_ptr<OutputStream> makeOutputStream() const override;\n\nprotected:\n    Socket(const Socket&) = delete;\n    Socket& operator = (const Socket&) = delete;\n    void Close();\n\n    SOCKET handle_;\n};\n\n\nclass NonSecureSocketFactory : public SocketFactory {\npublic:\n    ~NonSecureSocketFactory() override;\n\n    std::unique_ptr<SocketBase> connect(const ClientOptions& opts, const Endpoint& endpoint) override;\n\nprotected:\n    virtual std::unique_ptr<Socket> doConnect(const NetworkAddress& address, const ClientOptions& opts);\n\n    void setSocketOptions(Socket& socket, const ClientOptions& opts);\n};\n\n\nclass SocketInput : public InputStream {\npublic:\n    explicit SocketInput(SOCKET s);\n    ~SocketInput();\n\nprotected:\n    bool Skip(size_t bytes) override;\n    size_t DoRead(void* buf, size_t len) override;\n\nprivate:\n    SOCKET s_;\n};\n\nclass SocketOutput : public OutputStream {\npublic:\n    explicit SocketOutput(SOCKET s);\n    ~SocketOutput();\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    SOCKET s_;\n};\n\nstatic struct NetrworkInitializer {\n    NetrworkInitializer();\n} gNetrworkInitializer;\n\n}\n\n// Path: clickhouse/base/wire_format.h\n#pragma once\n\n#include <string>\n#include <cstdint>\n\nnamespace clickhouse {\n\nclass InputStream;\nclass OutputStream;\n\nclass WireFormat {\npublic:\n    template <typename T>\n    static bool ReadFixed(InputStream& input, T* value);\n    static bool ReadString(InputStream& input, std::string* value);\n    static bool SkipString(InputStream& input);\n    static bool ReadBytes(InputStream& input, void* buf, size_t len);\n    static bool ReadUInt64(InputStream& input, uint64_t* value);\n    static bool ReadVarint64(InputStream& output, uint64_t* value);\n\n    template <typename T>\n    static void WriteFixed(OutputStream& output, const T& value);\n    static void WriteBytes(OutputStream& output, const void* buf, size_t len);\n    static void WriteString(OutputStream& output, std::string_view value);\n    static void WriteUInt64(OutputStream& output, const uint64_t value);\n    static void WriteVarint64(OutputStream& output, uint64_t value);\n\nprivate:\n    static bool ReadAll(InputStream& input, void* buf, size_t len);\n    static void WriteAll(OutputStream& output, const void* buf, size_t len);\n};\n\ntemplate <typename T>\ninline bool WireFormat::ReadFixed(InputStream& input, T* value) {\n    return ReadAll(input, value, sizeof(T));\n}\n\ninline bool WireFormat::ReadString(InputStream& input, std::string* value) {\n    uint64_t len = 0;\n    if (ReadVarint64(input, &len)) {\n        if (len > 0x00FFFFFFULL) {\n            return false;\n        }\n        value->resize((size_t)len);\n        return ReadAll(input, value->data(), (size_t)len);\n    }\n\n    return false;\n}\n\ninline bool WireFormat::ReadBytes(InputStream& input, void* buf, size_t len) {\n    return ReadAll(input, buf, len);\n}\n\ninline bool WireFormat::ReadUInt64(InputStream& input, uint64_t* value) {\n    return ReadVarint64(input, value);\n}\n\ntemplate <typename T>\ninline void WireFormat::WriteFixed(OutputStream& output, const T& value) {\n    WriteAll(output, &value, sizeof(T));\n}\n\ninline void WireFormat::WriteBytes(OutputStream& output, const void* buf, size_t len) {\n    WriteAll(output, buf, len);\n}\n\ninline void WireFormat::WriteString(OutputStream& output, std::string_view value) {\n    WriteVarint64(output, value.size());\n    WriteAll(output, value.data(), value.size());\n}\n\ninline void WireFormat::WriteUInt64(OutputStream& output, const uint64_t value) {\n    WriteVarint64(output, value);\n}\n\n}\n\n// Path: clickhouse/columns/factory.h\n#pragma once\n\n#include \"column.h\"\n\nnamespace clickhouse {\n\nstruct CreateColumnByTypeSettings\n{\n    bool low_cardinality_as_wrapped_column = false;\n};\n\nColumnRef CreateColumnByType(const std::string& type_name, CreateColumnByTypeSettings settings = {});\n\n}\n\n// Path: clickhouse/base/sslsocket.h\n#pragma once\n\n#include \"socket.h\"\n\n#include <memory>\n#include <optional>\n#include <vector>\n\ntypedef struct ssl_ctx_st SSL_CTX;\ntypedef struct ssl_st SSL;\n\nnamespace clickhouse {\n\nstruct SSLParams\n{\n    std::vector<std::string> path_to_ca_files;\n    std::string path_to_ca_directory;\n    bool use_default_ca_locations;\n    int context_options;\n    int min_protocol_version;\n    int max_protocol_version;\n    bool use_SNI;\n    bool skip_verification;\n    int host_flags;\n    using ConfigurationType = std::vector<std::pair<std::string, std::optional<std::string>>>;\n    ConfigurationType configuration;\n};\n\nclass SSLContext\n{\npublic:\n    explicit SSLContext(SSL_CTX & context);\n    explicit SSLContext(const SSLParams & context_params);\n    ~SSLContext() = default;\n\n    SSLContext(const SSLContext &) = delete;\n    SSLContext& operator=(const SSLContext &) = delete;\n    SSLContext(SSLContext &&) = delete;\n    SSLContext& operator=(SSLContext &) = delete;\n\nprivate:\n    friend class SSLSocket;\n    SSL_CTX * getContext();\n\nprivate:\n    std::unique_ptr<SSL_CTX, void (*)(SSL_CTX*)> context_;\n};\n\nclass SSLSocket : public Socket {\npublic:\n    explicit SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params,\n                       const SSLParams& ssl_params, SSLContext& context);\n\n    SSLSocket(SSLSocket &&) = default;\n    ~SSLSocket() override = default;\n\n    SSLSocket(const SSLSocket & ) = delete;\n    SSLSocket& operator=(const SSLSocket & ) = delete;\n\n    std::unique_ptr<InputStream> makeInputStream() const override;\n    std::unique_ptr<OutputStream> makeOutputStream() const override;\n\n    static void validateParams(const SSLParams & ssl_params);\nprivate:\n    std::unique_ptr<SSL, void (*)(SSL *s)> ssl_;\n};\n\nclass SSLSocketFactory : public NonSecureSocketFactory {\npublic:\n    explicit SSLSocketFactory(const ClientOptions& opts);\n    ~SSLSocketFactory() override;\n\nprotected:\n    std::unique_ptr<Socket> doConnect(const NetworkAddress& address, const ClientOptions& opts) override;\n\nprivate:\n    const SSLParams ssl_params_;\n    std::unique_ptr<SSLContext> ssl_context_;\n};\n\nclass SSLSocketInput : public InputStream {\npublic:\n    explicit SSLSocketInput(SSL *ssl);\n    ~SSLSocketInput() = default;\n\n    bool Skip(size_t /*bytes*/) override {\n        return false;\n    }\n\nprotected:\n    size_t DoRead(void* buf, size_t len) override;\n\nprivate:\n    // Not owning\n    SSL *ssl_;\n};\n\nclass SSLSocketOutput : public OutputStream {\npublic:\n    explicit SSLSocketOutput(SSL *ssl);\n    ~SSLSocketOutput() = default;\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    // Not owning\n    SSL *ssl_;\n};\n\n}\n\n// Path: clickhouse/client.cpp\n#include \"client.h\"\n#include \"clickhouse/version.h\"\n#include \"protocol.h\"\n\n#include \"base/compressed.h\"\n#include \"base/socket.h\"\n#include \"base/wire_format.h\"\n\n#include \"columns/factory.h\"\n\n#include <assert.h>\n#include <system_error>\n#include <vector>\n#include <sstream>\n\n#if defined(WITH_OPENSSL)\n#include \"base/sslsocket.h\"\n#endif\n\n#define DBMS_NAME                                       \"ClickHouse\"\n\n#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES         50264\n#define DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS   51554\n#define DBMS_MIN_REVISION_WITH_BLOCK_INFO               51903\n#define DBMS_MIN_REVISION_WITH_CLIENT_INFO              54032\n#define DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE          54058\n#define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060\n//#define DBMS_MIN_REVISION_WITH_TABLES_STATUS            54226\n#define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337\n#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME      54372\n#define DBMS_MIN_REVISION_WITH_VERSION_PATCH            54401\n...\n", "needle_token_start": 3969, "needle_token_end": 3993, "code_context_ntokens": 15941, "output": ["```cpp\nclass MapValueView {\n    const typename ArrayColumnType::ArrayValueView data_;\n\npublic:\n    using ValueType = std::pair<Key, Value>;\n\n    MapValueView(typename ArrayColumnType::ArrayValueView data) : data_(std::move(data)) {}\n\n    inline auto operator[](const Key& key) const { return (*Find(key)).second; }\n\n    inline auto At(const Key& key) const {\n        auto it = Find(key);\n        if (it == end()) throw ValidationError(\"ColumnMap value key not found\");\n        return (*it).second;\n    }\n\n    class Iterator {\n        typename ArrayColumnType::ArrayValueView::Iterator data_iterator_;\n\n    public:\n        Iterator() = default;\n\n        Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator)\n            : data_iterator_(data_iterator) {}\n\n        using ValueType = std::pair<Key, Value>;\n        using difference_type = size_t;\n        using value_type = ValueType;\n        using pointer = void;\n        using reference = ValueType&;\n        using iterator_category = std::forward_iterator_tag;\n\n        inline auto operator*() const {\n            auto tuple = *data_iterator_;\n            return ValueType{std::get<0>(tuple), std::get<1>(tuple)};\n        }\n\n        inline Iterator& operator++() {\n            ++data_iterator_;\n            return *this;\n        }\n\n        inline bool operator==(const Iterator& other) const {\n            return this->data_iterator_ == other.data_iterator_;\n        }\n\n        inline bool operator!=(const Iterator& other) const { return !(*this == other); }\n    };\n\n    // minimalistic stl-like container interface, hence the lowercase\n    inline Iterator begin() const { return Iterator{data_.begin()}; }\n\n    inline Iterator cbegin() const { return Iterator{data_.cbegin()}; }\n\n    inline Iterator end() const { return Iterator{data_.end()}; }\n\n    inline Iterator cend() const { return Iterator{data_.cend()}; }\n\n    inline size_t size() const { return data_.size(); }\n\n    // It is ugly to have both size() and Size(), but it is for compatitability with both STL\n    // and rest of the clickhouse-cpp.\n    inline size_t Size() const { return data_.Size(); }\n\n    inline size_t Count(const Key& key) const {\n        size_t result = 0;\n        for (auto item : data_) {\n            if (std::get<0>(item) == key) {\n                ++result;\n            }\n        }\n        return result;\n    }\n\n    inline Iterator Find(const Key& key) const {\n        for (auto it = data_.begin(); it != data_.end(); ++it) {\n            if (std::get<0>(*it) == key) {\n                return Iterator{it};\n            }\n        }\n        return end();\n    }\n\n    inline bool operator==(const MapValueView& other) const {\n        if (size() != other.size()) {\n            return false;\n        }\n        const auto make_index = [](const auto& data) {\n            std::vector<size_t> result{data.Size()};\n            std::generate(result.begin(), result.end(), [i = 0] () mutable {return i++;});\n            std::sort(result.begin(), result.end(), [&data = data_](size_t l, size_t r) {return data[l] < data[r];});\n            return result;\n        };\n        const auto index = make_index(data_);\n        for (const auto& val : other.data_) {\n            if (!std::binary_search(index.begin(), index.end(), val,\n                    [&data = data_](const auto& l, size_t r) {return l < data[r];})) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    inline bool operator!=(const MapValueView& other) const { return !(*this == other); }\n};\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "ServerException", "language": "cpp", "path": "clickhouse/exceptions.h", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To encapsulate and handle exceptions originating from the server, providing a structured way to manage and report errors that occur during server interactions.\n2. **Input**: Receives a unique pointer to an exception structure containing details such as error code, name, display text, and stack trace.\n3. **Output**: Does not return a value but initializes an object that inherits from a standard runtime error class, enriched with server-specific error information.\n4. **Procedure**: Constructs an object by taking a unique pointer to an exception structure, transferring ownership of this structure to the newly created object to maintain error details.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/types/types.h\n#pragma once\n\n#include \"absl/numeric/int128.h\"\n\n#include <atomic>\n#include <map>\n#include <memory>\n#include <string>\n#include <vector>\n#include <stdexcept>\n\nnamespace clickhouse {\n\nusing Int128 = absl::int128;\nusing Int64 = int64_t;\n\nusing TypeRef = std::shared_ptr<class Type>;\n\nclass Type {\npublic:\n    enum Code {\n        Void = 0,\n        Int8,\n        Int16,\n        Int32,\n        Int64,\n        UInt8,\n        UInt16,\n        UInt32,\n        UInt64,\n        Float32,\n        Float64,\n        String,\n        FixedString,\n        DateTime,\n        Date,\n        Array,\n        Nullable,\n        Tuple,\n        Enum8,\n        Enum16,\n        UUID,\n        IPv4,\n        IPv6,\n        Int128,\n        Decimal,\n        Decimal32,\n        Decimal64,\n        Decimal128,\n        LowCardinality,\n        DateTime64,\n        Date32,\n        Map,\n        Point,\n        Ring,\n        Polygon,\n        MultiPolygon\n    };\n\n    using EnumItem = std::pair<std::string /* name */, int16_t /* value */>;\n\nprotected:\n    Type(const Code code);\n\npublic:\n    template <typename Derived>\n    auto* As() {\n        return static_cast<Derived*>(this);\n    }\n\n    template <typename Derived>\n    const auto* As() const {\n        return static_cast<const Derived*>(this);\n    }\n\n    /// Type's code.\n    Code GetCode() const { return code_; }\n\n    /// String representation of the type.\n    std::string GetName() const;\n\n    /// Is given type same as current one.\n    bool IsEqual(const Type& other) const {\n        // Types are equal only if both code_ and type_unique_id_ are equal.\n        return this == &other\n                // GetTypeUniqueId() is relatively heavy, so avoid calling it when comparing obviously different types.\n                || (this->GetCode() == other.GetCode() && this->GetTypeUniqueId() == other.GetTypeUniqueId());\n    }\n\n    bool IsEqual(const TypeRef& other) const { return IsEqual(*other); }\n\n    /// Simple name, doesn't depend on parameters and\\or nested types, caller MUST NOT free returned value.\n    static const char* TypeName(Code);\n\npublic:\n    static TypeRef CreateArray(TypeRef item_type);\n\n    static TypeRef CreateDate();\n\n    static TypeRef CreateDate32();\n\n    static TypeRef CreateDateTime(std::string timezone = std::string());\n\n    static TypeRef CreateDateTime64(size_t precision, std::string timezone = std::string());\n\n    static TypeRef CreateDecimal(size_t precision, size_t scale);\n\n    static TypeRef CreateIPv4();\n\n    static TypeRef CreateIPv6();\n\n    static TypeRef CreateNothing();\n\n    static TypeRef CreateNullable(TypeRef nested_type);\n\n    template <typename T>\n    static TypeRef CreateSimple();\n\n    static TypeRef CreateString();\n\n    static TypeRef CreateString(size_t n);\n\n    static TypeRef CreateTuple(const std::vector<TypeRef>& item_types);\n\n    static TypeRef CreateEnum8(const std::vector<EnumItem>& enum_items);\n\n    static TypeRef CreateEnum16(const std::vector<EnumItem>& enum_items);\n\n    static TypeRef CreateUUID();\n\n    static TypeRef CreateLowCardinality(TypeRef item_type);\n\n    static TypeRef CreateMap(TypeRef key_type, TypeRef value_type);\n\n    static TypeRef CreatePoint();\n\n    static TypeRef CreateRing();\n\n    static TypeRef CreatePolygon();\n\n    static TypeRef CreateMultiPolygon();\n\nprivate:\n    uint64_t GetTypeUniqueId() const;\n\n    const Code code_;\n    mutable std::atomic<uint64_t> type_unique_id_;\n};\n\ninline bool operator==(const Type & left, const Type & right) {\n    if (&left == &right)\n        return true;\n    if (typeid(left) == typeid(right))\n        return left.IsEqual(right);\n    return false;\n}\n\ninline bool operator==(const TypeRef & left, const TypeRef & right) {\n    return *left == *right;\n}\n\nclass ArrayType : public Type {\npublic:\n    explicit ArrayType(TypeRef item_type);\n\n    std::string GetName() const { return std::string(\"Array(\") + item_type_->GetName() + \")\"; }\n\n    /// Type of array's elements.\n    inline TypeRef GetItemType() const { return item_type_; }\n\nprivate:\n    TypeRef item_type_;\n};\n\nclass DecimalType : public Type {\npublic:\n    DecimalType(size_t precision, size_t scale);\n\n    std::string GetName() const;\n    friend class EnumType;\n    friend class DateTimeType;\n\n    inline size_t GetScale() const { return scale_; }\n    inline size_t GetPrecision() const { return precision_; }\n\nprivate:\n    const size_t precision_, scale_;\n};\n\nnamespace details\n{\nclass TypeWithTimeZoneMixin\n{\npublic:\n    TypeWithTimeZoneMixin(std::string timezone);\n\n    /// Timezone associated with a data column.\n    const std::string & Timezone() const;\n\nprivate:\n    std::string timezone_;\n};\n}\n\nclass DateTimeType : public Type, public details::TypeWithTimeZoneMixin {\npublic:\n    explicit DateTimeType(std::string timezone);\n\n    std::string GetName() const;\n};\n\nclass DateTime64Type: public Type, public details::TypeWithTimeZoneMixin {\npublic:\n    explicit DateTime64Type(size_t precision, std::string timezone_);\n\n    std::string GetName() const;\n\n    inline size_t GetPrecision() const { return precision_; }\nprivate:\n    size_t precision_;\n};\n\nclass EnumType : public Type {\npublic:\n    EnumType(Type::Code type, const std::vector<EnumItem>& items);\n\n    std::string GetName() const;\n\n    /// Methods to work with enum types.\n    std::string_view GetEnumName(int16_t value) const;\n    int16_t GetEnumValue(const std::string& name) const;\n    bool HasEnumName(const std::string& name) const;\n    bool HasEnumValue(int16_t value) const;\n\nprivate:\n    using ValueToNameType     = std::map<int16_t, std::string_view>;\n    using NameToValueType     = std::map<std::string, int16_t>;\n    using ValueToNameIterator = ValueToNameType::const_iterator;\n\n    ValueToNameType value_to_name_;\n    NameToValueType name_to_value_;\n\npublic:\n    ValueToNameIterator BeginValueToName() const;\n    ValueToNameIterator EndValueToName() const;\n};\n\nclass FixedStringType : public Type {\npublic:\n    explicit FixedStringType(size_t n);\n\n    std::string GetName() const { return std::string(\"FixedString(\") + std::to_string(size_) + \")\"; }\n\n    inline size_t GetSize() const { return size_; }\n\nprivate:\n    size_t size_;\n};\n\nclass NullableType : public Type {\npublic:\n    explicit NullableType(TypeRef nested_type);\n\n    std::string GetName() const { return std::string(\"Nullable(\") + nested_type_->GetName() + \")\"; }\n\n    /// Type of nested nullable element.\n    TypeRef GetNestedType() const { return nested_type_; }\n\nprivate:\n    TypeRef nested_type_;\n};\n\nclass TupleType : public Type {\npublic:\n    explicit TupleType(const std::vector<TypeRef>& item_types);\n\n    std::string GetName() const;\n\n    /// Type of nested Tuple element type.\n    std::vector<TypeRef> GetTupleType() const { return item_types_; }\n\nprivate:\n    std::vector<TypeRef> item_types_;\n};\n\nclass LowCardinalityType : public Type {\npublic:\n    explicit LowCardinalityType(TypeRef nested_type);\n    ~LowCardinalityType();\n\n    std::string GetName() const { return std::string(\"LowCardinality(\") + nested_type_->GetName() + \")\"; }\n\n    /// Type of nested nullable element.\n    TypeRef GetNestedType() const { return nested_type_; }\n\nprivate:\n    TypeRef nested_type_;\n};\n\nclass MapType : public Type {\npublic:\n    explicit MapType(TypeRef key_type, TypeRef value_type);\n\n    std::string GetName() const;\n\n    /// Type of keys.\n    TypeRef GetKeyType() const { return key_type_; }\n\n    /// Type of values.\n    TypeRef GetValueType() const { return value_type_; }\n\nprivate:\n    TypeRef key_type_;\n    TypeRef value_type_;\n};\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int8_t>() {\n    return TypeRef(new Type(Int8));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int16_t>() {\n    return TypeRef(new Type(Int16));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int32_t>() {\n    return TypeRef(new Type(Int32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int64_t>() {\n    return TypeRef(new Type(Int64));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<Int128>() {\n    return TypeRef(new Type(Int128));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint8_t>() {\n    return TypeRef(new Type(UInt8));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint16_t>() {\n    return TypeRef(new Type(UInt16));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint32_t>() {\n    return TypeRef(new Type(UInt32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint64_t>() {\n    return TypeRef(new Type(UInt64));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<float>() {\n    return TypeRef(new Type(Float32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<double>() {\n    return TypeRef(new Type(Float64));\n}\n\n}  // namespace clickhouse\n\n// Path: clickhouse/server_exception.h\n#pragma once\n\n#include <string>\n#include <memory>\n\nnamespace clickhouse {\nstruct Exception {\n    int code = 0;\n    std::string name;\n    std::string display_text;\n    std::string stack_trace;\n    /// Pointer to nested exception.\n    std::unique_ptr<Exception> nested;\n};\n\n}\n\n// Path: clickhouse/exceptions.h\n#pragma once\n\n#include \"server_exception.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nclass Error : public std::runtime_error {\n    using std::runtime_error::runtime_error;\n};\n\n// Caused by any user-related code, like invalid column types or arguments passed to any method.\nclass ValidationError : public Error {\n    using Error::Error;\n};\n\n// Buffers+IO errors, failure to serialize/deserialize, checksum mismatches, etc.\nclass ProtocolError : public Error {\n    using Error::Error;\n};\n\nclass UnimplementedError : public Error {\n    using Error::Error;\n};\n\n// Internal validation error.\nclass AssertionError : public Error {\n    using Error::Error;\n};\n\nclass OpenSSLError : public Error {\n    using Error::Error;\n};\n\nclass LZ4Error : public Error {\n    using Error::Error;\n};\n\n// Exception received from server.\nclass ServerException : public Error {\npublic:\n    \nServerException(std::unique_ptr<Exception> e)\n        : Error(std::string())\n        , exception_(std::move(e))\n    {\n    }\n\n    int GetCode() const {\n        return exception_->code;\n    }\n\n    const Exception& GetException() const {\n        return *exception_;\n    }\n\n    const char* what() const noexcept override {\n        return exception_->display_text.c_str();\n    }\n\nprivate:\n    std::unique_ptr<Exception> exception_;\n};\nusing ServerError = ServerException;\n\n}\n\n// Path: clickhouse/columns/itemview.h\n#pragma once\n\n#include \"../types/types.h\"\n#include \"../exceptions.h\"\n\n#include <string_view>\n#include <stdexcept>\n#include <type_traits>\n\nnamespace clickhouse {\n\n/** ItemView is a view on a data stored in Column, safe-ish interface for reading values from Column.\n *\n * Data is not owned (hence the name View) and will be invalidated on column update, load\n * or destruction (basically on calling any non-const method of Column).\n * `type` reflects what is stored in `data` and can be almost any value-type\n * (except Nullable, Array, Tuple, LowCardinality).\n *\n */\nstruct ItemView {\n    using DataType = std::string_view;\n\n    const Type::Code type;\n    const DataType data;\n\nprivate:\n    template <typename T>\n    inline auto ConvertToStorageValue(const T& t) {\n        if constexpr (std::is_same_v<std::string_view, T> || std::is_same_v<std::string, T>) {\n            return std::string_view{t};\n        } else if constexpr (std::is_fundamental_v<T> || std::is_same_v<Int128, std::decay_t<T>>) {\n            return std::string_view{reinterpret_cast<const char*>(&t), sizeof(T)};\n        } else {\n            static_assert(!std::is_same_v<T, T>, \"Unknown type, which can't be stored in ItemView\");\n            return;\n        }\n    }\n\npublic:\n    ItemView(Type::Code type, DataType data)\n        : type(type),\n          data(data)\n    {\n        ValidateData(type, data);\n    }\n\n    ItemView(Type::Code type, ItemView other)\n        : type(type),\n          data(other.data)\n    {\n        ValidateData(type, data);\n    }\n\n    explicit ItemView()\n        : ItemView(Type::Void, std::string_view{})\n    {}\n\n    template <typename T>\n    explicit ItemView(Type::Code type, const T & value)\n        : ItemView(type, ConvertToStorageValue(value))\n    {}\n\n    template <typename T>\n    auto get() const {\n        using ValueType = std::remove_cv_t<std::decay_t<T>>;\n        if constexpr (std::is_same_v<std::string_view, ValueType> || std::is_same_v<std::string, ValueType>) {\n            return data;\n        } else if constexpr (std::is_fundamental_v<ValueType> || std::is_same_v<Int128, ValueType>) {\n            if (sizeof(ValueType) == data.size()) {\n                return *reinterpret_cast<const T*>(data.data());\n            } else {\n                throw AssertionError(\"Incompatitable value type and size. Requested size: \"\n                        + std::to_string(sizeof(ValueType)) + \" stored size: \" + std::to_string(data.size()));\n            }\n        }\n    }\n\n    inline std::string_view AsBinaryData() const {\n        return data;\n    }\n\n    // Validate that value matches type, will throw an exception if validation fails.\n    static void ValidateData(Type::Code type, DataType data);\n};\n\n}\n\n// Path: clickhouse/columns/column.h\n#pragma once\n\n#include \"../types/types.h\"\n#include \"../columns/itemview.h\"\n#include \"../exceptions.h\"\n\n#include <memory>\n#include <stdexcept>\n\nnamespace clickhouse {\n\nclass InputStream;\nclass OutputStream;\n\nusing ColumnRef = std::shared_ptr<class Column>;\n\n/**\n * An abstract base of all columns classes.\n */\nclass Column : public std::enable_shared_from_this<Column> {\npublic:\n    explicit inline Column(TypeRef type) : type_(type) {}\n\n    virtual ~Column() {}\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<T> As() {\n        return std::dynamic_pointer_cast<T>(shared_from_this());\n    }\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<const T> As() const {\n        return std::dynamic_pointer_cast<const T>(shared_from_this());\n    }\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<T> AsStrict() {\n        auto result = std::dynamic_pointer_cast<T>(shared_from_this());\n        if (!result) {\n            throw ValidationError(\"Can't cast from \" + type_->GetName());\n        }\n        return result;\n    }\n\n    /// Get type object of the column.\n    inline TypeRef Type() const { return type_; }\n    inline const class Type& GetType() const { return *type_; }\n\n    /// Appends content of given column to the end of current one.\n    virtual void Append(ColumnRef column) = 0;\n\n    /// Increase the capacity of the column for large block insertion.\n    virtual void Reserve(size_t new_cap) = 0;\n\n    /// Template method to load column data from input stream. It'll call LoadPrefix and LoadBody.\n    /// Should be called only once from the client. Derived classes should not call it.\n    bool Load(InputStream* input, size_t rows);\n\n    /// Loads column prefix from input stream.\n    virtual bool LoadPrefix(InputStream* input, size_t rows);\n\n    /// Loads column data from input stream.\n    virtual bool LoadBody(InputStream* input, size_t rows) = 0;\n\n    /// Saves column prefix to output stream. Column types with prefixes must implement it.\n    virtual void SavePrefix(OutputStream* output);\n\n    /// Saves column body to output stream.\n    virtual void SaveBody(OutputStream* output) = 0;\n\n    /// Template method to save to output stream. It'll call SavePrefix and SaveBody respectively\n    /// Should be called only once from the client. Derived classes should not call it.\n    /// Save is split in Prefix and Body because some data types require prefixes and specific serialization order.\n    /// For instance, Array(LowCardinality(X)) requires LowCardinality.key_version bytes to come before Array.offsets\n    void Save(OutputStream* output);\n\n    /// Clear column data .\n    virtual void Clear() = 0;\n\n    /// Returns count of rows in the column.\n    virtual size_t Size() const = 0;\n\n    /// Makes slice of the current column.\n    virtual ColumnRef Slice(size_t begin, size_t len) const = 0;\n\n    virtual ColumnRef CloneEmpty() const = 0;\n\n    virtual void Swap(Column&) = 0;\n\n    /// Get a view on raw item data if it is supported by column, will throw an exception if index is out of range.\n    /// Please note that view is invalidated once column items are added or deleted, column is loaded from strean or destroyed.\n    virtual ItemView GetItem(size_t) const {\n        throw UnimplementedError(\"GetItem() is not supported for column of \" + type_->GetName());\n    }\n\n    friend void swap(Column& left, Column& right) {\n        left.Swap(right);\n    }\n\nprotected:\n    TypeRef type_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/block.h\n#pragma once\n\n#include \"columns/column.h\"\n\nnamespace clickhouse {\n\nstruct BlockInfo {\n    uint8_t is_overflows = 0;\n    int32_t bucket_num = -1;\n};\n\nclass Block {\npublic:\n    /// Allow to iterate over block's columns.\n    class Iterator {\n    public:\n        Iterator(const Block& block);\n\n        /// Name of column.\n        const std::string& Name() const;\n\n        /// Type of column.\n        TypeRef Type() const;\n\n        /// Reference to column object.\n        ColumnRef Column() const;\n\n        /// Move to next column, returns false if next call to IsValid() would return false;\n        bool Next();\n\n        /// Is the iterator still valid.\n        bool IsValid() const;\n\n        size_t ColumnIndex() const {\n            return idx_;\n        }\n\n        Iterator& operator*() { return *this; }\n        const Iterator& operator*() const { return *this; }\n\n        bool operator==(const Iterator & other) const {\n            return &block_ == &other.block_ && idx_ == other.idx_;\n        }\n        bool operator!=(const Iterator & other) const {\n            return !(*this == other);\n        }\n\n        Iterator& operator++() {\n            this->Next();\n            return *this;\n        }\n\n    private:\n        friend class Block;\n        struct ConstructAtEndTag {};\n        Iterator(const Block& block, ConstructAtEndTag at_end);\n        Iterator() = delete;\n\n        const Block& block_;\n        size_t idx_;\n    };\n\npublic:\n     Block();\n     Block(size_t cols, size_t rows);\n    ~Block();\n\n    /// Append named column to the block.\n    void AppendColumn(const std::string& name, const ColumnRef& col);\n\n    /// Count of columns in the block.\n    size_t GetColumnCount() const;\n\n    const BlockInfo& Info() const;\n\n    /// Set block info\n    void SetInfo(BlockInfo info);\n\n    /// Count of rows in the block.\n    size_t GetRowCount() const;\n\n    size_t RefreshRowCount();\n\n    const std::string& GetColumnName(size_t idx) const {\n        return columns_.at(idx).name;\n    }\n\n    /// Reference to column by index in the block.\n    ColumnRef operator [] (size_t idx) const;\n\n    Iterator begin() const;\n    Iterator end() const;\n    Iterator cbegin() const { return begin(); }\n    Iterator cend() const { return end(); }\n\nprivate:\n    struct ColumnItem {\n        std::string name;\n        ColumnRef   column;\n    };\n\n    BlockInfo info_;\n    std::vector<ColumnItem> columns_;\n    /// Count of rows in the block.\n    size_t rows_;\n};\n\n}\n\n// Path: clickhouse/block.cpp\n#include \"block.h\"\n\n#include \"exceptions.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nBlock::Iterator::Iterator(const Block& block)\n    : block_(block)\n    , idx_(0)\n{\n}\n\nBlock::Iterator::Iterator(const Block& block, Block::Iterator::ConstructAtEndTag /*at_end*/)\n    : block_(block)\n    , idx_(block.GetColumnCount())\n{}\n\nconst std::string& Block::Iterator::Name() const {\n    return block_.columns_[idx_].name;\n}\n\nTypeRef Block::Iterator::Type() const {\n    return block_.columns_[idx_].column->Type();\n}\n\nColumnRef Block::Iterator::Column() const {\n    return block_.columns_[idx_].column;\n}\n\nbool Block::Iterator::Next() {\n    ++idx_;\n    return IsValid();\n}\n\nbool Block::Iterator::IsValid() const {\n    return idx_ < block_.columns_.size();\n}\n\n\nBlock::Block()\n    : rows_(0)\n{\n}\n\nBlock::Block(size_t cols, size_t rows)\n    : rows_(rows)\n{\n    columns_.reserve(cols);\n}\n\nBlock::~Block() = default;\n\nvoid Block::AppendColumn(const std::string& name, const ColumnRef& col) {\n    if (columns_.empty()) {\n        rows_ = col->Size();\n    } else if (col->Size() != rows_) {\n        throw ValidationError(\"all columns in block must have same count of rows. Name: [\"+name+\"], rows: [\"+std::to_string(rows_)+\"], columns: [\" + std::to_string(col->Size())+\"]\");\n    }\n\n    columns_.push_back(ColumnItem{name, col});\n}\n\n/// Count of columns in the block.\nsize_t Block::GetColumnCount() const {\n    return columns_.size();\n}\n\nconst BlockInfo& Block::Info() const {\n    return info_;\n}\n\n/// Set block info\nvoid Block::SetInfo(BlockInfo info) {\n    info_ = std::move(info);\n}\n\n/// Count of rows in the block.\nsize_t Block::GetRowCount() const {\n    return rows_;\n}\n\nsize_t Block::RefreshRowCount()\n{\n    size_t rows = 0UL;\n\n    for (size_t idx = 0UL; idx < columns_.size(); ++idx)\n    {\n       const std::string& name = columns_[idx].name;\n       const ColumnRef& col = columns_[idx].column;\n\n       if (idx == 0UL)\n           rows = col->Size();\n       else if (rows != col->Size())\n           throw ValidationError(\"all columns in block must have same count of rows. Name: [\"+name+\"], rows: [\"+std::to_string(rows)+\"], columns: [\" + std::to_string(col->Size())+\"]\");\n    }\n\n    rows_ = rows;\n    return rows_;\n}\n\nColumnRef Block::operator [] (size_t idx) const {\n    if (idx < columns_.size()) {\n        return columns_[idx].column;\n    }\n\n    throw std::out_of_range(\"column index is out of range. Index: [\"+std::to_string(idx)+\"], columns: [\" + std::to_string(columns_.size())+\"]\");\n}\n\nBlock::Iterator Block::begin() const {\n    return Iterator(*this);\n}\n\nBlock::Iterator Block::end() const {\n    return Iterator(*this, Iterator::ConstructAtEndTag{});\n}\n\n}\n\n// Path: clickhouse/base/uuid.h\n#pragma once\n\n#include <cstdint>\n#include <utility>\n\nnamespace clickhouse {\n\nusing UInt128 = std::pair<uint64_t, uint64_t>;\n\nusing UUID = UInt128;\n\n}\n\n// Path: clickhouse/base/open_telemetry.h\n#pragma once\n\n#include \"uuid.h\"\n\n#include <string>\n\nnamespace clickhouse::open_telemetry {\n\n/// See https://www.w3.org/TR/trace-context/ for trace_flags definition\nenum TraceFlags : uint8_t {\n    TRACE_FLAG_NONE    = 0,\n    TRACE_FLAG_SAMPLED = 1,\n};\n\n/// The runtime info we need to create new OpenTelemetry spans.\nstruct TracingContext {\n    UUID trace_id{};\n    uint64_t span_id = 0;\n    std::string tracestate;\n    uint8_t trace_flags = TRACE_FLAG_NONE;\n};\n\n}  // namespace clickhouse::open_telemetry\n\n// Path: clickhouse/query.h\n#pragma once\n\n#include \"block.h\"\n#include \"server_exception.h\"\n\n#include \"base/open_telemetry.h\"\n\n#include <cstdint>\n#include <functional>\n#include <memory>\n#include <optional>\n#include <string>\n#include <unordered_map>\n\nnamespace clickhouse {\n\nstruct QuerySettingsField {\n    enum Flags : uint64_t\n    {\n        IMPORTANT = 0x01,\n        CUSTOM = 0x02,\n        OBSOLETE = 0x04,\n    };\n    std::string value;\n    uint64_t flags{0};\n};\n\nusing QuerySettings = std::unordered_map<std::string, QuerySettingsField>;\n\nstruct Profile {\n    uint64_t rows = 0;\n    uint64_t blocks = 0;\n    uint64_t bytes = 0;\n    uint64_t rows_before_limit = 0;\n    bool applied_limit = false;\n    bool calculated_rows_before_limit = false;\n};\n\n\nstruct Progress {\n    uint64_t rows = 0;\n    uint64_t bytes = 0;\n    uint64_t total_rows = 0;\n    uint64_t written_rows = 0;\n    uint64_t written_bytes = 0;\n};\n\n\nclass QueryEvents {\npublic:\n    virtual ~QueryEvents()\n    { }\n\n    /// Some data was received.\n    virtual void OnData(const Block& block) = 0;\n    virtual bool OnDataCancelable(const Block& block) = 0;\n\n    virtual void OnServerException(const Exception& e) = 0;\n\n    virtual void OnProfile(const Profile& profile) = 0;\n\n    virtual void OnProgress(const Progress& progress) = 0;\n\n    /** Handle query execution logs provided by server.\n     *  Amount of logs regulated by `send_logs_level` setting.\n     *  By-default only `fatal` log events are sent to the client side.\n     */\n    virtual void OnServerLog(const Block& block) = 0;\n\n    /// Handle query execution profile events.\n    virtual void OnProfileEvents(const Block& block) = 0;\n\n    virtual void OnFinish() = 0;\n};\n\n\nusing ExceptionCallback        = std::function<void(const Exception& e)>;\nusing ProgressCallback         = std::function<void(const Progress& progress)>;\nusing SelectCallback           = std::function<void(const Block& block)>;\nusing SelectCancelableCallback = std::function<bool(const Block& block)>;\nusing SelectServerLogCallback  = std::function<bool(const Block& block)>;\nusing ProfileEventsCallback    = std::function<bool(const Block& block)>;\nusing ProfileCallbak           = std::function<void(const Profile& profile)>;\n\n\nclass Query : public QueryEvents {\npublic:\n     Query();\n     Query(const char* query, const char* query_id = nullptr);\n     Query(const std::string& query, const std::string& query_id = default_query_id);\n    ~Query() override;\n\n    ///\n    inline const std::string& GetText() const {\n        return query_;\n    }\n\n    inline const std::string& GetQueryID() const {\n        return query_id_;\n    }\n\n    inline const QuerySettings& GetQuerySettings() const {\n        return query_settings_;\n    }\n\n    /// Set per query settings\n    inline Query& SetQuerySettings(QuerySettings query_settings) {\n        query_settings_ = std::move(query_settings);\n        return *this;\n    }\n\n    /// Set per query setting\n    inline Query& SetSetting(const std::string& key, const QuerySettingsField& value) {\n        query_settings_[key] = value;\n        return *this;\n    }\n\n    inline const std::optional<open_telemetry::TracingContext>& GetTracingContext() const {\n        return tracing_context_;\n    }\n\n    /// Set tracing context for open telemetry signals\n    inline Query& SetTracingContext(open_telemetry::TracingContext tracing_context) {\n        tracing_context_ = std::move(tracing_context);\n        return *this;\n    }\n\n    /// Set handler for receiving result data.\n    inline Query& OnData(SelectCallback cb) {\n        select_cb_ = std::move(cb);\n        return *this;\n    }\n\n    inline Query& OnDataCancelable(SelectCancelableCallback cb) {\n        select_cancelable_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving server's exception.\n    inline Query& OnException(ExceptionCallback cb) {\n        exception_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving a progress of query execution.\n    inline Query& OnProgress(ProgressCallback cb) {\n        progress_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving a server log of query exceution.\n    inline Query& OnServerLog(SelectServerLogCallback cb) {\n        select_server_log_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving profile events.\n    inline Query& OnProfileEvents(ProfileEventsCallback cb) {\n        profile_events_callback_cb_ = std::move(cb);\n        return *this;\n    }\n\n    inline Query& OnProfile(ProfileCallbak cb) {\n        profile_callback_cb_ = std::move(cb);\n        return *this;\n    }\n\n    static const std::string default_query_id;\n\nprivate:\n    void OnData(const Block& block) override {\n        if (select_cb_) {\n            select_cb_(block);\n        }\n    }\n\n    bool OnDataCancelable(const Block& block) override {\n        if (select_cancelable_cb_) {\n            return select_cancelable_cb_(block);\n        } else {\n            return true;\n        }\n    }\n\n    void OnServerException(const Exception& e) override {\n        if (exception_cb_) {\n            exception_cb_(e);\n        }\n    }\n\n    void OnProfile(const Profile& profile) override {\n        if (profile_callback_cb_)\n            profile_callback_cb_(profile);\n    }\n\n    void OnProgress(const Progress& progress) override {\n        if (progress_cb_) {\n            progress_cb_(progress);\n        }\n    }\n\n    void OnServerLog(const Block& block) override {\n        if (select_server_log_cb_) {\n            select_server_log_cb_(block);\n        }\n    }\n\n    void OnProfileEvents(const Block& block) override {\n        if (profile_events_callback_cb_) {\n            profile_events_callback_cb_(block);\n        }\n    }\n\n    void OnFinish() override {\n    }\n\nprivate:\n    const std::string query_;\n    const std::string query_id_;\n    std::optional<open_telemetry::TracingContext> tracing_context_;\n    QuerySettings query_settings_;\n    ExceptionCallback exception_cb_;\n    ProgressCallback progress_cb_;\n    SelectCallback select_cb_;\n    SelectCancelableCallback select_cancelable_cb_;\n    SelectServerLogCallback select_server_log_cb_;\n    ProfileEventsCallback profile_events_callback_cb_;\n    ProfileCallbak profile_callback_cb_;\n};\n\n}\n\n// Path: clickhouse/columns/numeric.h\n#pragma once\n\n#include \"column.h\"\n#include \"absl/numeric/int128.h\"\n\nnamespace clickhouse {\n\n/**\n * Represents various numeric columns.\n */\ntemplate <typename T>\nclass ColumnVector : public Column {\npublic:\n    using DataType = T;\n    using ValueType = T;\n\n    ColumnVector();\n\n    explicit ColumnVector(const std::vector<T>& data);\n    explicit ColumnVector(std::vector<T> && data);\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends one element to the end of column.\n    void Append(const T& value);\n\n    /// Returns element at given row number.\n    const T& At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline const T& operator [] (size_t n) const { return At(n); }\n\n    void Erase(size_t pos, size_t count = 1);\n\n    /// Get Raw Vector Contents\n    std::vector<T>& GetWritableData();\n\n    /// Returns the capacity of the column\n    size_t Capacity() const;\n\npublic:\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\n    ItemView GetItem(size_t index) const override;\n\nprivate:\n    std::vector<T> data_;\n};\n\nusing Int128 = absl::int128;\nusing Int64 = int64_t;\n\nusing ColumnUInt8   = ColumnVector<uint8_t>;\nusing ColumnUInt16  = ColumnVector<uint16_t>;\nusing ColumnUInt32  = ColumnVector<uint32_t>;\nusing ColumnUInt64  = ColumnVector<uint64_t>;\n\nusing ColumnInt8    = ColumnVector<int8_t>;\nusing ColumnInt16   = ColumnVector<int16_t>;\nusing ColumnInt32   = ColumnVector<int32_t>;\nusing ColumnInt64   = ColumnVector<int64_t>;\nusing ColumnInt128  = ColumnVector<Int128>;\n\nusing ColumnFloat32 = ColumnVector<float>;\nusing ColumnFloat64 = ColumnVector<double>;\n\n}\n\n// Path: clickhouse/columns/utils.h\n#pragma once\n\n#include <algorithm>\n#include <vector>\n#include <memory>\n\nnamespace clickhouse {\n\ntemplate <typename T>\nstd::vector<T> SliceVector(const std::vector<T>& vec, size_t begin, size_t len) {\n    std::vector<T> result;\n\n    if (begin < vec.size()) {\n        len = std::min(len, vec.size() - begin);\n        result.assign(vec.begin() + begin, vec.begin() + (begin + len));\n    }\n\n    return result;\n}\n\ntemplate <typename T>\nstruct HasWrapMethod {\nprivate:\n    static int detect(...);\n    template <typename U>\n    static decltype(U::Wrap(std::move(std::declval<ColumnRef>()))) detect(const U&);\n\npublic:\n    static constexpr bool value = !std::is_same<int, decltype(detect(std::declval<T>()))>::value;\n};\n\ntemplate <typename T>\ninline std::shared_ptr<T> WrapColumn(ColumnRef&& column) {\n    if constexpr (HasWrapMethod<T>::value) {\n        return T::Wrap(std::move(column));\n    } else {\n        return column->template AsStrict<T>();\n    }\n}\n\n}\n\n// Path: clickhouse/columns/array.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n#include \"utils.h\"\n\n#include <memory>\n\nnamespace clickhouse {\n\ntemplate <typename NestedColumnType>\nclass ColumnArrayT;\n\n/**\n * Represents column of Array(T).\n */\nclass ColumnArray : public Column {\npublic:\n    using ValueType = ColumnRef;\n\n    /** Create an array of given type.\n     *\n     *  `data` is used internally (and modified) by ColumnArray.\n     *  Users are strongly advised against supplying non-empty columns and/or modifying\n     *  contents of `data` afterwards.\n     */\n    explicit ColumnArray(ColumnRef data);\n\n    /** Create an array of given type, with actual values and offsets.\n     *\n     *  Both `data` and `offsets` are used (and modified) internally bye ColumnArray.\n     *  Users are strongly advised against modifying contents of `data` or `offsets` afterwards.\n     */\n    ColumnArray(ColumnRef data, std::shared_ptr<ColumnUInt64> offsets);\n\n    /// Converts input column to array and appends as one row to the current column.\n    void AppendAsColumn(ColumnRef array);\n\n    /// Converts array at pos n to column.\n    /// Type of element of result column same as type of array element.\n    ColumnRef GetAsColumn(size_t n) const;\n\n    /// Shorthand to get a column casted to a proper type.\n    template <typename T>\n    auto GetAsColumnTyped(size_t n) const {\n        return GetAsColumn(n)->AsStrict<T>();\n    }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column prefix from input stream.\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t, size_t) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column&) override;\n\n    void OffsetsIncrease(size_t);\n\nprotected:\n    template<typename T> friend class ColumnArrayT;\n\n    ColumnArray(ColumnArray&& array);\n\n    size_t GetOffset(size_t n) const;\n    size_t GetSize(size_t n) const;\n    ColumnRef GetData();\n    void AddOffset(size_t n);\n    void Reset();\n\nprivate:\n    ColumnRef data_;\n    std::shared_ptr<ColumnUInt64> offsets_;\n};\n\ntemplate <typename ColumnType>\nclass ColumnArrayT : public ColumnArray {\npublic:\n    class ArrayValueView;\n    using ValueType = ArrayValueView;\n    using NestedColumnType = ColumnType;\n\n    explicit ColumnArrayT(std::shared_ptr<NestedColumnType> data)\n        : ColumnArray(data)\n        , typed_nested_data_(data)\n    {}\n\n    ColumnArrayT(std::shared_ptr<NestedColumnType> data, std::shared_ptr<ColumnUInt64> offsets)\n        : ColumnArray(data, offsets)\n        , typed_nested_data_(data)\n    {}\n\n    template <typename ...Args>\n    explicit ColumnArrayT(Args &&... args)\n        : ColumnArrayT(std::make_shared<NestedColumnType>(std::forward<Args>(args)...))\n    {}\n\n    /** Create a ColumnArrayT from a ColumnArray, without copying data and offsets, but by 'stealing' those from `col`.\n     *\n     *  Ownership of column internals is transferred to returned object, original (argument) object\n     *  MUST NOT BE USED IN ANY WAY, it is only safe to dispose it.\n     *\n     *  Throws an exception if `col` is of wrong type, it is safe to use original col in this case.\n     *  This is a static method to make such conversion verbose.\n     */\n    static auto Wrap(ColumnArray&& col) {\n        auto nested_data = WrapColumn<NestedColumnType>(col.GetData());\n        return std::make_shared<ColumnArrayT<NestedColumnType>>(nested_data, col.offsets_);\n    }\n\n    static auto Wrap(Column&& col) {\n        return Wrap(std::move(dynamic_cast<ColumnArray&&>(col)));\n    }\n\n    // Helper to simplify integration with other APIs\n    static auto Wrap(ColumnRef&& col) {\n        return Wrap(std::move(*col->AsStrict<ColumnArray>()));\n    }\n\n    /// A single (row) value of the Array-column, i.e. readonly array of items.\n    class ArrayValueView {\n        const std::shared_ptr<NestedColumnType> typed_nested_data_;\n        const size_t offset_;\n        const size_t size_;\n\n    public:\n        using ValueType = std::decay_t<decltype(std::declval<NestedColumnType>().At(0))>;\n\n        ArrayValueView(std::shared_ptr<NestedColumnType> data, size_t offset = 0, size_t size = std::numeric_limits<size_t>::max())\n            : typed_nested_data_(data)\n            , offset_(offset)\n            , size_(std::min(typed_nested_data_->Size() - offset, size))\n        {}\n\n        inline auto operator[](size_t index) const {\n            return (*typed_nested_data_)[offset_ + index];\n        }\n\n        inline auto At(size_t index) const {\n            if (index >= size_)\n                throw ValidationError(\"ColumnArray value index out of bounds: \"\n                        + std::to_string(index) + \", max is \" + std::to_string(size_));\n            return typed_nested_data_->At(offset_ + index);\n        }\n\n        class Iterator {\n            const std::shared_ptr<NestedColumnType> typed_nested_data_;\n            const size_t offset_;\n            const size_t size_;\n            size_t index_;\n        public:\n            Iterator() = default;\n\n            Iterator(std::shared_ptr<NestedColumnType> typed_nested_data, size_t offset, size_t size, size_t index)\n                : typed_nested_data_(typed_nested_data)\n                , offset_(offset)\n                , size_(size)\n                , index_(index)\n            {}\n\n            using ValueType = typename ArrayValueView::ValueType;\n\n            inline auto operator*() const {\n                return typed_nested_data_->At(offset_ + index_);\n            }\n\n            inline Iterator& operator++() {\n                ++index_;\n                return *this;\n            }\n\n            inline bool operator==(const Iterator& other) const {\n                return this->typed_nested_data_ == other.typed_nested_data_\n                        && this->offset_ == other.offset_\n                        && this->size_ == other.size_\n                        && this->index_ == other.index_;\n            }\n\n            inline bool operator!=(const Iterator& other) const {\n                return !(*this == other);\n            }\n        };\n\n        // minimalistic stl-like container interface, hence the lowercase\n        inline Iterator begin() const {\n            return Iterator{typed_nested_data_, offset_, size_, 0};\n        }\n\n        inline Iterator cbegin() const {\n            return Iterator{typed_nested_data_, offset_, size_, 0};\n        }\n\n        inline Iterator end() const {\n            return Iterator{typed_nested_data_, offset_, size_, size_};\n        }\n\n        inline Iterator cend() const {\n            return Iterator{typed_nested_data_, offset_, size_, size_};\n        }\n\n        inline size_t size() const {\n            return size_;\n        }\n\n        // It is ugly to have both size() and Size(), but it is for compatitability with both STL and rest of the clickhouse-cpp.\n        inline size_t Size() const {\n            return size_;\n        }\n\n        inline bool operator==(const ArrayValueView& other) const {\n            if (size() != other.size()) {\n                return false;\n            }\n            for (size_t i = 0; i < size_; ++i) {\n                if ((*this)[i] != other[i]) {\n                    return false;\n                }\n            }\n            return true;\n        }\n\n        inline bool operator!=(const ArrayValueView& other) const {\n            return !(*this == other);\n        }\n    };\n\n    inline auto At(size_t index) const {\n        if (index >= Size())\n            throw ValidationError(\"ColumnArray row index out of bounds: \"\n                    + std::to_string(index) + \", max is \" + std::to_string(Size()));\n\n        return ArrayValueView{typed_nested_data_, GetOffset(index), GetSize(index)};\n    }\n\n    inline auto operator[](size_t index) const {\n        return ArrayValueView{typed_nested_data_, GetOffset(index), GetSize(index)};\n    }\n\n    using ColumnArray::Append;\n\n    template <typename Container>\n    inline void Append(Container&& container) {\n        using container_type = decltype(container);\n        if constexpr (std::is_lvalue_reference_v<container_type> || \n            std::is_const_v<std::remove_reference_t<container_type>>) {\n            Append(std::begin(container), std::end(container));\n        }\n        else {\n            Append(std::make_move_iterator(std::begin(container)),\n                std::make_move_iterator(std::end(container)));\n        }\n    }\n\n    template <typename ValueType>\n    inline void Append(const std::initializer_list<ValueType>& container) {\n        Append(std::begin(container), std::end(container));\n    }\n\n    template <typename Begin, typename End>\n    inline void Append(Begin begin, End end) {\n        auto & nested_data = *typed_nested_data_;\n        size_t counter = 0;\n\n        while (begin != end) {\n            nested_data.Append(*begin);\n            ++begin;\n            ++counter;\n        }\n\n        // Even if there are 0 items, increase counter, creating empty array item.\n        AddOffset(counter);\n    }\n\n    ColumnRef Slice(size_t begin, size_t size) const override {\n        return Wrap(ColumnArray::Slice(begin, size));\n    }\n\n    ColumnRef CloneEmpty() const override {\n        return Wrap(ColumnArray::CloneEmpty());\n    }\n\n    void Swap(Column& other) override {\n        auto & col = dynamic_cast<ColumnArrayT<NestedColumnType> &>(other);\n        typed_nested_data_.swap(col.typed_nested_data_);\n        ColumnArray::Swap(other);\n    }\n\nprivate:\n    /// Helper to allow wrapping a \"typeless\" ColumnArray\n    ColumnArrayT(ColumnArray&& array, std::shared_ptr<NestedColumnType> nested_data)\n        : ColumnArray(std::move(array))\n        , typed_nested_data_(std::move(nested_data))\n    {}\n\n\nprivate:\n    std::shared_ptr<NestedColumnType> typed_nested_data_;\n};\n\n}\n\n// Path: clickhouse/columns/decimal.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n\nnamespace clickhouse {\n\n/**\n * Represents a column of decimal type.\n */\nclass ColumnDecimal : public Column {\npublic:\n    using ValueType = Int128;\n\n    ColumnDecimal(size_t precision, size_t scale);\n\n    void Append(const Int128& value);\n    void Append(const std::string& value);\n\n    Int128 At(size_t i) const;\n    inline auto operator[](size_t i) const { return At(i); }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n    void Append(ColumnRef column) override;\n    bool LoadBody(InputStream* input, size_t rows) override;\n    void SaveBody(OutputStream* output) override;\n    void Clear() override;\n    size_t Size() const override;\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n    ItemView GetItem(size_t index) const override;\n\n    size_t GetScale() const;\n    size_t GetPrecision() const;\n\nprivate:\n    /// Depending on a precision it can be one of:\n    ///  - ColumnInt32\n    ///  - ColumnInt64\n    ///  - ColumnInt128\n    ColumnRef data_;\n\n    explicit ColumnDecimal(TypeRef type, ColumnRef data);\n};\n\n}\n\n// Path: clickhouse/columns/date.h\n#pragma once\n\n#include \"decimal.h\"\n#include \"numeric.h\"\n\n#include <ctime>\n\nnamespace clickhouse {\n\n/** */\nclass ColumnDate : public Column {\npublic:\n    using ValueType = std::time_t;\n\n    ColumnDate();\n    explicit ColumnDate(std::vector<uint16_t>&& data);\n\n    /// Appends one element to the end of column.\n    /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving.\n    void Append(const std::time_t& value);\n...\n", "needle_token_start": 3074, "needle_token_end": 3113, "code_context_ntokens": 13439, "output": ["```cpp\nclass ServerException : public Error {\npublic:\n    ServerException(std::unique_ptr<Exception> e)\n        : Error(std::string())\n        , exception_(std::move(e))\n    {\n    }\n\n    int GetCode() const {\n        return exception_->code;\n    }\n\n    const Exception& GetException() const {\n        return *exception_;\n    }\n\n    const char* what() const noexcept override {\n        return exception_->display_text.c_str();\n    }\n\nprivate:\n    std::unique_ptr<Exception> exception_;\n};\nusing ServerError = ServerException;\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "AppendMany", "language": "cpp", "path": "clickhouse/columns/lowcardinality.h", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to add multiple elements to a low cardinality column in a batch, enhancing efficiency when processing collections of data.\n2. **Input**: Accepts a container (such as a list or vector) containing elements of a specific type that matches the column's data type.\n3. **Output**: There is no direct output returned by the function; however, it modifies the state of the column by appending each element from the input container.\n4. **Procedure**: Iterates over each element in the provided container, and for each element, it utilizes another function to safely append the element to the column, handling type-specific considerations such as nullability.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/columns/tuple.h\n#pragma once\n\n#include \"column.h\"\n#include \"utils.h\"\n\n#include <vector>\n\nnamespace clickhouse {\n\n/**\n * Represents column of Tuple([T]).\n */\nclass ColumnTuple : public Column {\npublic:\n    ColumnTuple(const std::vector<ColumnRef>& columns);\n\n    /// Returns count of columns in the tuple.\n    size_t TupleSize() const;\n\n    inline ColumnRef operator [] (size_t n) const {\n        return columns_[n];\n    }\n\n    inline ColumnRef At(size_t n) const {\n        return columns_[n];\n    }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column prefix from input stream.\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t, size_t) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\nprivate:\n    std::vector<ColumnRef> columns_;\n};\n\ntemplate <typename... Columns>\nclass ColumnTupleT : public ColumnTuple {\npublic:\n    using TupleOfColumns = std::tuple<std::shared_ptr<Columns>...>;\n\n    using ValueType = std::tuple<std::decay_t<decltype(std::declval<Columns>().At(0))>...>;\n\n    ColumnTupleT(std::tuple<std::shared_ptr<Columns>...> columns)\n        : ColumnTuple(TupleToVector(columns)), typed_columns_(std::move(columns)) {}\n\n    ColumnTupleT(std::vector<ColumnRef> columns)\n        : ColumnTuple(columns), typed_columns_(VectorToTuple(std::move(columns))) {}\n\n    ColumnTupleT(const std::initializer_list<ColumnRef> columns)\n        : ColumnTuple(columns), typed_columns_(VectorToTuple(std::move(columns))) {}\n\n    inline ValueType At(size_t index) const { return GetTupleOfValues(index); }\n\n    inline ValueType operator[](size_t index) const { return GetTupleOfValues(index); }\n\n    using ColumnTuple::Append;\n\n    template <typename... T>\n    inline void Append(std::tuple<T...> value) {\n        AppendTuple(std::move(value));\n    }\n\n    /** Create a ColumnTupleT from a ColumnTuple, without copying data and offsets, but by\n     * 'stealing' those from `col`.\n     *\n     *  Ownership of column internals is transferred to returned object, original (argument) object\n     *  MUST NOT BE USED IN ANY WAY, it is only safe to dispose it.\n     *\n     *  Throws an exception if `col` is of wrong type, it is safe to use original col in this case.\n     *  This is a static method to make such conversion verbose.\n     */\n    static auto Wrap(ColumnTuple&& col) {\n        if (col.TupleSize() != std::tuple_size_v<TupleOfColumns>) {\n            throw ValidationError(\"Can't wrap from \" + col.GetType().GetName());\n        }\n        return std::make_shared<ColumnTupleT<Columns...>>(VectorToTuple(std::move(col)));\n    }\n\n    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnTuple&&>(col))); }\n\n...\n// Path: clickhouse/columns/geo.h\n#pragma once\n\n#include \"array.h\"\n#include \"column.h\"\n#include \"numeric.h\"\n#include \"tuple.h\"\n\nnamespace clickhouse {\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nclass ColumnGeo : public Column {\npublic:\n    using ValueType = typename NestedColumnType::ValueType;\n\n    ColumnGeo();\n\n    explicit ColumnGeo(ColumnRef data);\n\n    /// Appends one element to the end of column.\n    template <typename T = ValueType>\n    void Append(const T& value) {\n        data_->Append(value);\n    }\n\n    /// Returns element at given row number.\n    const ValueType At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline const ValueType operator[](size_t n) const { return At(n); }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\nprivate:\n    std::shared_ptr<NestedColumnType> data_;\n};\n\n// /**\n//  * Represents a Point column.\n//  */\nusing ColumnPoint = ColumnGeo<ColumnTupleT<ColumnFloat64, ColumnFloat64>, Type::Code::Point>;\n\n/**\n * Represents a Ring column.\n */\nusing ColumnRing = ColumnGeo<ColumnArrayT<ColumnPoint>, Type::Code::Ring>;\n\n/**\n * Represents a Polygon column.\n */\nusing ColumnPolygon = ColumnGeo<ColumnArrayT<ColumnRing>, Type::Code::Polygon>;\n\n/**\n * Represents a MultiPolygon column.\n */\nusing ColumnMultiPolygon = ColumnGeo<ColumnArrayT<ColumnPolygon>, Type::Code::MultiPolygon>;\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/ip4.h\n#pragma once\n\n#include \"numeric.h\"\n\nstruct in_addr;\n\nnamespace clickhouse {\n\nclass ColumnIPv4 : public Column {\npublic:\n    using DataType = in_addr;\n    using ValueType = in_addr;\n\n    ColumnIPv4();\n    /** Takes ownership of the data, expects ColumnUInt32.\n     * Modifying memory pointed by `data` from outside is UB.\n     *\n     * TODO: deprecate and remove as it is too dangerous and error-prone.\n     */\n    explicit ColumnIPv4(ColumnRef data);\n\n    explicit ColumnIPv4(std::vector<uint32_t>&& data);\n\n    /// Appends one element to the column.\n    void Append(const std::string& ip);\n\n    /// @params ip numeric value with host byte order.\n    void Append(uint32_t ip);\n\n    ///\n    void Append(in_addr ip);\n\n    /// Returns element at given row number.\n    in_addr At(size_t n) const;\n\n    /// Returns element at given row number.\n    in_addr operator [] (size_t n) const;\n\n    std::string AsString(size_t n) const;\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\n    ItemView GetItem(size_t index) const override;\n\nprivate:\n    std::shared_ptr<ColumnUInt32> data_;\n};\n\n}\n\n// Path: clickhouse/columns/string.h\n#pragma once\n\n#include \"column.h\"\n\n#include <string>\n#include <string_view>\n#include <utility>\n#include <vector>\n#include <deque>\n\nnamespace clickhouse {\n\n/**\n * Represents column of fixed-length strings.\n */\nclass ColumnFixedString : public Column {\npublic:\n    using ValueType = std::string_view;\n\n    explicit ColumnFixedString(size_t n);\n\n    template <typename Values>\n    ColumnFixedString(size_t n, const Values & values)\n        : ColumnFixedString(n)\n    {\n        for (const auto & v : values)\n            Append(v);\n    }\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t) override;\n\n    /// Appends one element to the column.\n    void Append(std::string_view str);\n\n    /// Returns element at given row number.\n    std::string_view At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline std::string_view operator [] (size_t n) const { return At(n); }\n\n    /// Returns the max size of the fixed string\n    size_t FixedSize() const;\n\npublic:\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\n    ItemView GetItem(size_t) const override;\n\nprivate:\n    size_t string_size_;\n    std::string data_;\n};\n\n/**\n * Represents column of variable-length strings.\n */\nclass ColumnString : public Column {\npublic:\n    // Type this column takes as argument of Append and returns with At() and operator[]\n    using ValueType = std::string_view;\n\n    ColumnString();\n    ~ColumnString();\n\n    explicit ColumnString(size_t element_count);\n    explicit ColumnString(const std::vector<std::string> & data);\n    explicit ColumnString(std::vector<std::string>&& data);\n    ColumnString& operator=(const ColumnString&) = delete;\n    ColumnString(const ColumnString&) = delete;\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends one element to the column.\n    void Append(std::string_view str);\n\n    /// Appends one element to the column.\n    void Append(const char* str);\n\n    /// Appends one element to the column.\n    void Append(std::string&& steal_value);\n\n    /// Appends one element to the column.\n    /// If str lifetime is managed elsewhere and guaranteed to outlive the Block sent to the server\n    void AppendNoManagedLifetime(std::string_view str);\n\n    /// Returns element at given row number.\n    std::string_view At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline std::string_view operator [] (size_t n) const { return At(n); }\n\npublic:\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n    ItemView GetItem(size_t) const override;\n\nprivate:\n    void AppendUnsafe(std::string_view);\n\nprivate:\n    struct Block;\n\n    std::vector<std::string_view> items_;\n    std::vector<Block> blocks_;\n    std::deque<std::string> append_data_;\n};\n\n}\n\n// Path: clickhouse/columns/ip6.h\n#pragma once\n\n#include \"string.h\"\n#include <memory>\n\nstruct in6_addr;\n\nnamespace clickhouse {\n\nclass ColumnIPv6 : public Column {\npublic:\n    using DataType = in6_addr;\n    using ValueType = in6_addr;\n\n    ColumnIPv6();\n    /** Takes ownership of the data, expects ColumnFixedString.\n     * Modifying memory pointed by `data` from outside is UB.\n     *\n     * TODO: deprecate and remove as it is too dangerous and error-prone.\n     */\n    explicit ColumnIPv6(ColumnRef data);\n\n    /// Appends one element to the column.\n    void Append(const std::string_view& str);\n\n    void Append(const in6_addr* addr);\n    void Append(const in6_addr& addr);\n\n    /// Returns element at given row number.\n    in6_addr At(size_t n) const;\n\n    /// Returns element at given row number.\n    in6_addr operator [] (size_t n) const;\n\n    std::string AsString(size_t n) const;\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n    ItemView GetItem(size_t index) const override;\n\nprivate:\n    std::shared_ptr<ColumnFixedString> data_;\n};\n\n}\n\n// Path: clickhouse/columns/nullable.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n\n#include <optional>\n\nnamespace clickhouse {\n\n/**\n * Represents column of Nullable(T).\n */\nclass ColumnNullable : public Column {\npublic:\n    ColumnNullable(ColumnRef nested, ColumnRef nulls);\n\n    /// Appends one null flag to the end of the column\n    void Append(bool isnull);\n\n    /// Returns null flag at given row number.\n    bool IsNull(size_t n) const;\n\n    /// Returns nested column.\n    ColumnRef Nested() const;\n\n    /// Returns nulls column.\n    ColumnRef Nulls() const;\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column prefix from input stream.\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column&) override;\n\n    ItemView GetItem(size_t) const override;\n\nprivate:\n    ColumnRef nested_;\n    std::shared_ptr<ColumnUInt8> nulls_;\n};\n\ntemplate <typename ColumnType>\nclass ColumnNullableT : public ColumnNullable {\npublic:\n    using NestedColumnType = ColumnType;\n    using ValueType = std::optional<std::decay_t<decltype(std::declval<NestedColumnType>().At(0))>>;\n\n    ColumnNullableT(std::shared_ptr<NestedColumnType> data, std::shared_ptr<ColumnUInt8> nulls)\n        : ColumnNullable(data, nulls)\n        , typed_nested_data_(data)\n    {}\n\n    explicit ColumnNullableT(std::shared_ptr<NestedColumnType> data)\n        : ColumnNullableT(data, FillNulls(data->Size()))\n    {}\n\n    template <typename ...Args>\n    explicit ColumnNullableT(Args &&... args)\n        : ColumnNullableT(std::make_shared<NestedColumnType>(std::forward<Args>(args)...))\n    {}\n\n    inline ValueType At(size_t index) const {\n        return IsNull(index) ? ValueType{} : ValueType{typed_nested_data_->At(index)};\n    }\n\n    inline ValueType operator[](size_t index) const { return At(index); }\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override {\n        ColumnNullable::Append(std::move(column));\n    }\n\n    inline void Append(ValueType value) {\n        ColumnNullable::Append(!value.has_value());\n        if (value.has_value()) {\n            typed_nested_data_->Append(std::move(*value));\n        } else {\n            typed_nested_data_->Append(typename ValueType::value_type{});\n        }\n    }\n\n    /** Create a ColumnNullableT from a ColumnNullable, without copying data and offsets, but by\n     * 'stealing' those from `col`.\n     *\n     *  Ownership of column internals is transferred to returned object, original (argument) object\n     *  MUST NOT BE USED IN ANY WAY, it is only safe to dispose it.\n     *\n     *  Throws an exception if `col` is of wrong type, it is safe to use original col in this case.\n     *  This is a static method to make such conversion verbose.\n     */\n    static auto Wrap(ColumnNullable&& col) {\n        return std::make_shared<ColumnNullableT<NestedColumnType>>(\n            col.Nested()->AsStrict<NestedColumnType>(),\n            col.Nulls()->AsStrict<ColumnUInt8>()) ;\n    }\n\n    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnNullable&&>(col))); }\n\n    // Helper to simplify integration with other APIs\n    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnNullable>())); }\n\n    ColumnRef Slice(size_t begin, size_t size) const override {\n        return Wrap(ColumnNullable::Slice(begin, size));\n    }\n\n    ColumnRef CloneEmpty() const override { return Wrap(ColumnNullable::CloneEmpty()); }\n\n    void Swap(Column& other) override {\n        auto& col = dynamic_cast<ColumnNullableT<NestedColumnType>&>(other);\n        typed_nested_data_.swap(col.typed_nested_data_);\n        ColumnNullable::Swap(other);\n    }\n\nprivate:\n    static inline auto FillNulls(size_t n){\n        auto result = std::make_shared<ColumnUInt8>();\n        for (size_t i = 0; i < n; ++i) {\n            result->Append(0);\n        }\n        return result;\n    }\n\n    std::shared_ptr<NestedColumnType> typed_nested_data_;\n};\n\ntemplate <typename T>\nconstexpr bool IsNullable = std::is_base_of_v<ColumnNullable, T>;\n\n}\n\n// Path: clickhouse/columns/lowcardinality.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n#include \"nullable.h\"\n\n#include <functional>\n#include <string>\n#include <unordered_map>\n#include <utility>\n\nnamespace clickhouse {\n\ntemplate <typename NestedColumnType>\nclass ColumnLowCardinalityT;\n\nnamespace details {\n\n/** LowCardinalityHashKey used as key in unique items hashmap to abstract away key value\n * (type of which depends on dictionary column) and to reduce likelehood of collisions.\n *\n * In order to dramatically reduce collision rate, we use 2 different hashes from 2 different hash functions.\n * First hash is used in hashtable (to calculate item position).\n * Second one is used as part of key value and accessed via `operator==()` upon collision resolution/detection.\n */\nusing LowCardinalityHashKey = std::pair<std::uint64_t, std::uint64_t>;\n\nstruct LowCardinalityHashKeyHash {\n    inline std::size_t operator()(const LowCardinalityHashKey &hash_key) const noexcept {\n        return hash_key.first;\n    }\n};\n\n}\n\n/*\n * LC column contains an \"invisible\" default item at the beginning of the collection. [default, ...]\n * If the nested type is Nullable, it contains a null-item at the beginning and a default item at the second position. [null, default, ...]\n * Null map is not serialized in LC columns. Instead, nulls are tracked by having an index of 0.\n * */\nclass ColumnLowCardinality : public Column {\npublic:\n    using UniqueItems = std::unordered_map<details::LowCardinalityHashKey, size_t /*dictionary index*/, details::LowCardinalityHashKeyHash>;\n\n    template <typename T>\n    friend class ColumnLowCardinalityT;\n\nprivate:\n    // IMPLEMENTATION NOTE: ColumnLowCardinalityT takes reference to underlying dictionary column object,\n    // so make sure to NOT change address of the dictionary object (with reset(), swap()) or with anything else.\n    ColumnRef dictionary_column_;\n    ColumnRef index_column_;\n    UniqueItems unique_items_map_;\n\npublic:\n    ColumnLowCardinality(ColumnLowCardinality&& col) = default;\n    // c-tor makes a deep copy of the dictionary_column.\n    explicit ColumnLowCardinality(ColumnRef dictionary_column);\n    explicit ColumnLowCardinality(std::shared_ptr<ColumnNullable> dictionary_column);\n\n    template <typename T>\n    explicit ColumnLowCardinality(std::shared_ptr<ColumnNullableT<T>> dictionary_column)\n        : ColumnLowCardinality(dictionary_column->template As<ColumnNullable>())\n    {}\n\n    ~ColumnLowCardinality();\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends another LowCardinality column to the end of this one, updating dictionary.\n    void Append(ColumnRef /*column*/) override;\n\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data.\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of current column, with compacted dictionary\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n    ItemView GetItem(size_t index) const override;\n\n    size_t GetDictionarySize() const;\n    TypeRef GetNestedType() const;\n\nprotected:\n    std::uint64_t getDictionaryIndex(std::uint64_t item_index) const;\n    void appendIndex(std::uint64_t item_index);\n    void removeLastIndex();\n    ColumnRef GetDictionary();\n\n    void AppendUnsafe(const ItemView &);\n\nprivate:\n    void Setup(ColumnRef dictionary_column);\n    void AppendNullItem();\n    void AppendDefaultItem();\n\npublic:\n    static details::LowCardinalityHashKey computeHashKey(const ItemView &);\n};\n\n/** Type-aware wrapper that provides simple convenience interface for accessing/appending individual items.\n */\ntemplate <typename DictionaryColumnType>\nclass ColumnLowCardinalityT : public ColumnLowCardinality {\n\n    DictionaryColumnType& typed_dictionary_;\n    const Type::Code type_;\n\npublic:\n    using WrappedColumnType = DictionaryColumnType;\n    // Type this column takes as argument of Append and returns with At() and operator[]\n    using ValueType = typename DictionaryColumnType::ValueType;\n\n    explicit ColumnLowCardinalityT(ColumnLowCardinality&& col)\n        : ColumnLowCardinality(std::move(col))\n        ,  typed_dictionary_(dynamic_cast<DictionaryColumnType &>(*GetDictionary()))\n        ,  type_(GetTypeCode(typed_dictionary_))\n    {\n    }\n\n    template <typename ...Args>\n    explicit ColumnLowCardinalityT(Args &&... args)\n        : ColumnLowCardinalityT(std::make_shared<DictionaryColumnType>(std::forward<Args>(args)...))\n    {}\n\n    // Create LC<T> column from existing T-column, making a deep copy of all contents.\n    explicit ColumnLowCardinalityT(std::shared_ptr<DictionaryColumnType> dictionary_col)\n        : ColumnLowCardinality(dictionary_col)\n        , typed_dictionary_(dynamic_cast<DictionaryColumnType &>(*GetDictionary()))\n        , type_(GetTypeCode(typed_dictionary_))\n    {}\n\n    /// Extended interface to simplify reading/adding individual items.\n\n    /// Returns element at given row number.\n    inline ValueType At(size_t n) const {\n        return typed_dictionary_.At(getDictionaryIndex(n));\n    }\n\n    /// Returns element at given row number.\n    inline ValueType operator [] (size_t n) const {\n        return typed_dictionary_[getDictionaryIndex(n)];\n    }\n\n    // so the non-virtual Append below doesn't shadow Append() from base class when compiled with older compilers.\n    using ColumnLowCardinality::Append;\n\n    inline void Append(const ValueType & value) {\n        if constexpr (IsNullable<WrappedColumnType>) {\n            if (value.has_value()) {\n                AppendUnsafe(ItemView{type_, *value});\n            } else {\n                AppendUnsafe(ItemView{});\n            }\n        } else {\n            AppendUnsafe(ItemView{type_, value});\n        }\n    }\n\n    template <typename T>\n    \ninline void AppendMany(const T& container) {\n        for (const auto & item : container) {\n            Append(item);\n        }\n    }\n\n    /** Create a ColumnLowCardinalityT from a ColumnLowCardinality, without copying data and offsets, but by\n     * 'stealing' those from `col`.\n     *\n     *  Ownership of column internals is transferred to returned object, original (argument) object\n     *  MUST NOT BE USED IN ANY WAY, it is only safe to dispose it.\n     *\n     *  Throws an exception if `col` is of wrong type, it is safe to use original col in this case.\n     *  This is a static method to make such conversion verbose.\n     */\n    static auto Wrap(ColumnLowCardinality&& col) {\n        return std::make_shared<ColumnLowCardinalityT<WrappedColumnType>>(std::move(col));\n    }\n\n    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnLowCardinality&&>(col))); }\n\n    // Helper to simplify integration with other APIs\n    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnLowCardinality>())); }\n\n    ColumnRef Slice(size_t begin, size_t size) const override {\n        return Wrap(ColumnLowCardinality::Slice(begin, size));\n    }\n\n    ColumnRef CloneEmpty() const override { return Wrap(ColumnLowCardinality::CloneEmpty()); }\n\nprivate:\n\n    template <typename T>\n    static auto GetTypeCode(T& column) {\n        if constexpr (IsNullable<T>) {\n            return GetTypeCode(*column.Nested()->template AsStrict<typename T::NestedColumnType>());\n        } else {\n            return column.Type()->GetCode();\n        }\n    }\n};\n\n}\n\n// Path: clickhouse/base/projected_iterator.h\n#pragma once\n\n#include <iterator>\n#include <type_traits>\n#include <utility>\n\nnamespace clickhouse {\n\ntemplate <typename UnaryFunction, typename Iterator, typename Reference = decltype(std::declval<UnaryFunction>()(std::declval<Iterator>())),\n          typename Value = std::decay_t<Reference>>\nclass ProjectedIterator {\npublic:\n    using value_type        = Value;\n    using reference         = Reference;\n    using pointer           = Reference;\n    using difference_type   = typename std::iterator_traits<Iterator>::difference_type;\n    using iterator_category = typename std::iterator_traits<Iterator>::iterator_category;\n\n    ProjectedIterator() = default;\n\n    inline ProjectedIterator(Iterator const& iterator, UnaryFunction functor)\n        : iterator_(iterator)\n        , functor_(std::move(functor)) {\n    }\n\n    inline UnaryFunction functor() const { return functor; }\n\n    inline Iterator const& base() const { return iterator_; }\n\n    inline reference operator*() const { return functor_(iterator_); }\n\n    inline ProjectedIterator& operator++() {\n        ++iterator_;\n        return *this;\n    }\n\n    inline ProjectedIterator& operator--() {\n        --iterator_;\n        return *this;\n    }\n\n    inline bool operator==(const ProjectedIterator& other) const {\n        return this->iterator_ == other.iterator_;\n    }\n\n    inline bool operator!=(const ProjectedIterator& other) const {\n        return !(*this == other);\n    }\n\nprivate:\n    Iterator iterator_;\n    UnaryFunction functor_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/map.h\n#pragma once\n\n#include \"../base/projected_iterator.h\"\n#include \"array.h\"\n#include \"column.h\"\n#include \"tuple.h\"\n\n#include <functional>\n#include <map>\n\nnamespace clickhouse {\n\ntemplate <typename KeyColumnType, typename ValueColumnType>\nclass ColumnMapT;\n\n/**\n * Represents column of Map(K, V).\n */\nclass ColumnMap : public Column {\npublic:\n    /** Create a map of given type, with actual values and offsets.\n     *\n     *  Both `data` and `offsets` are used (and modified) internally bye ColumnArray.\n     *  Users are strongly advised against modifying contents of `data` or `offsets` afterwards.\n     */\n    explicit ColumnMap(ColumnRef data);\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column prefix from input stream.\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t, size_t) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column&) override;\n\n    /// Converts map at pos n to column.\n    /// Type of row is tuple {key, value}.\n    ColumnRef GetAsColumn(size_t n) const;\n\nprotected:\n    template <typename K, typename V>\n    friend class ColumnMapT;\n\n    ColumnMap(ColumnMap&& map);\n\nprivate:\n    std::shared_ptr<ColumnArray> data_;\n};\n\ntemplate <typename K, typename V>\nclass ColumnMapT : public ColumnMap {\npublic:\n    using KeyColumnType   = K;\n    using ValueColumnType = V;\n    using Key             = std::decay_t<decltype(std::declval<KeyColumnType>().At(0))>;\n    using Value           = std::decay_t<decltype(std::declval<ValueColumnType>().At(0))>;\n    using TupleColumnType = ColumnTupleT<KeyColumnType, ValueColumnType>;\n    using ArrayColumnType = ColumnArrayT<TupleColumnType>;\n\n    ColumnMapT(ColumnRef data)\n        : ColumnMap(data), typed_data_(data->AsStrict<ColumnArrayT<TupleColumnType>>()) {}\n\n    ColumnMapT(std::shared_ptr<KeyColumnType> keys, std::shared_ptr<ValueColumnType> values)\n        : ColumnMap(std::make_shared<ArrayColumnType>(std::make_shared<TupleColumnType>(\n              std::make_tuple(std::move(keys), std::move(values))))),\n          typed_data_(data_->template As<ArrayColumnType>()) {}\n\n    ColumnRef Slice(size_t begin, size_t len) const override {\n        return std::make_shared<ColumnMapT<K, V>>(typed_data_->Slice(begin, len));\n    }\n\n    ColumnRef CloneEmpty() const override {\n        return std::make_shared<ColumnMapT<K, V>>(typed_data_->CloneEmpty());\n    }\n\n    void Swap(Column& other) override {\n        auto& col = dynamic_cast<ColumnMapT<K, V>&>(other);\n        col.typed_data_.swap(typed_data_);\n        ColumnMap::Swap(other);\n    }\n\n    /// A single (row) value of the Map-column i.e. read-only map.\n    /// It has a linear time complexity to access items\n    /// Because data base type has same structure\n    /// \"This lookup works now with a linear complexity.\"\n    /// https://clickhouse.com/docs/en/sql-reference/data-types/map\n    /// Convert it to a suitable container required to access more than one element\n\n    class MapValueView {\n        const typename ArrayColumnType::ArrayValueView data_;\n\n    public:\n        using ValueType = std::pair<Key, Value>;\n\n        MapValueView(typename ArrayColumnType::ArrayValueView data) : data_(std::move(data)) {}\n\n        inline auto operator[](const Key& key) const { return (*Find(key)).second; }\n\n        inline auto At(const Key& key) const {\n            auto it = Find(key);\n            if (it == end()) throw ValidationError(\"ColumnMap value key not found\");\n            return (*it).second;\n        }\n\n        class Iterator {\n            typename ArrayColumnType::ArrayValueView::Iterator data_iterator_;\n\n        public:\n            Iterator() = default;\n\n            Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator)\n                : data_iterator_(data_iterator) {}\n\n            using ValueType = std::pair<Key, Value>;\n            using difference_type = size_t;\n            using value_type = ValueType;\n            using pointer = void;\n            using reference = ValueType&;\n            using iterator_category = std::forward_iterator_tag;\n\n            inline auto operator*() const {\n                auto tuple = *data_iterator_;\n                return ValueType{std::get<0>(tuple), std::get<1>(tuple)};\n            }\n\n            inline Iterator& operator++() {\n                ++data_iterator_;\n                return *this;\n            }\n\n            inline bool operator==(const Iterator& other) const {\n                return this->data_iterator_ == other.data_iterator_;\n            }\n\n            inline bool operator!=(const Iterator& other) const { return !(*this == other); }\n        };\n\n        // minimalistic stl-like container interface, hence the lowercase\n        inline Iterator begin() const { return Iterator{data_.begin()}; }\n\n        inline Iterator cbegin() const { return Iterator{data_.cbegin()}; }\n\n        inline Iterator end() const { return Iterator{data_.end()}; }\n\n        inline Iterator cend() const { return Iterator{data_.cend()}; }\n\n        inline size_t size() const { return data_.size(); }\n\n        // It is ugly to have both size() and Size(), but it is for compatitability with both STL\n        // and rest of the clickhouse-cpp.\n        inline size_t Size() const { return data_.Size(); }\n\n        inline size_t Count(const Key& key) const {\n            size_t result = 0;\n            for (auto item : data_) {\n                if (std::get<0>(item) == key) {\n                    ++result;\n                }\n            }\n            return result;\n        }\n\n        inline Iterator Find(const Key& key) const {\n            for (auto it = data_.begin(); it != data_.end(); ++it) {\n                if (std::get<0>(*it) == key) {\n                    return Iterator{it};\n                }\n            }\n            return end();\n        }\n\n        inline bool operator==(const MapValueView& other) const {\n            if (size() != other.size()) {\n                return false;\n            }\n            const auto make_index = [](const auto& data) {\n                std::vector<size_t> result{data.Size()};\n                std::generate(result.begin(), result.end(), [i = 0] () mutable {return i++;});\n                std::sort(result.begin(), result.end(), [&data](size_t l, size_t r) {return data[l] < data[r];});\n                return result;\n            };\n            const auto index = make_index(data_);\n            for (const auto& val : other.data_) {\n                if (!std::binary_search(index.begin(), index.end(), val,\n                        [&data = data_](const auto& l, size_t r) {return l < data[r];})) {\n                    return false;\n                }\n            }\n            return true;\n        }\n\n        inline bool operator!=(const MapValueView& other) const { return !(*this == other); }\n    };\n\n    inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; }\n\n    inline auto operator[](size_t index) const { return At(index); }\n\n    using ColumnMap::Append;\n\n    inline void Append(const MapValueView& value) { typed_data_->Append(value.data_); }\n\n    inline void Append(const std::vector<std::tuple<Key, Value>>& tuples) {\n        typed_data_->Append(tuples.begin(), tuples.end());\n    }\n\n    template <typename T>\n    inline void Append(const T& value) {\n        using BaseIter = decltype(value.begin());\n        using KeyOfT = decltype(std::declval<BaseIter>()->first);\n        using ValOfT = decltype(std::declval<BaseIter>()->second);\n        using Functor = std::function<std::tuple<KeyOfT, ValOfT>(const BaseIter&)>;\n        using Iterator = ProjectedIterator<Functor, BaseIter>;\n\n        Functor functor = [](const BaseIter& i) {\n            return std::make_tuple(std::cref(i->first), std::cref(i->second));\n        };\n\n        typed_data_->Append(Iterator{value.begin(), functor}, Iterator{value.end(), functor});\n    }\n\n    static auto Wrap(ColumnMap&& col) {\n        auto data = ArrayColumnType::Wrap(std::move(col.data_));\n        return std::make_shared<ColumnMapT<K, V>>(std::move(data));\n    }\n\n    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnMap&&>(col))); }\n\n    // Helper to simplify integration with other APIs\n    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnMap>())); }\n\nprivate:\n    std::shared_ptr<ArrayColumnType> typed_data_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/uuid.h\n#pragma once\n\n#include \"../base/uuid.h\"\n#include \"column.h\"\n#include \"numeric.h\"\n\nnamespace clickhouse {\n\n\n/**\n * Represents a UUID column.\n */\nclass ColumnUUID : public Column {\npublic:\n    ColumnUUID();\n\n    explicit ColumnUUID(ColumnRef data);\n\n    /// Appends one element to the end of column.\n    void Append(const UUID& value);\n\n    /// Returns element at given row number.\n    const UUID At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline const UUID operator [] (size_t n) const { return At(n); }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\n    ItemView GetItem(size_t) const override;\n\nprivate:\n    std::shared_ptr<ColumnUInt64> data_;\n};\n\n}\n\n// Path: clickhouse/client.h\n#pragma once\n\n#include \"query.h\"\n#include \"exceptions.h\"\n\n#include \"columns/array.h\"\n#include \"columns/date.h\"\n#include \"columns/decimal.h\"\n#include \"columns/enum.h\"\n#include \"columns/geo.h\"\n#include \"columns/ip4.h\"\n#include \"columns/ip6.h\"\n#include \"columns/lowcardinality.h\"\n#include \"columns/nullable.h\"\n#include \"columns/numeric.h\"\n#include \"columns/map.h\"\n#include \"columns/string.h\"\n#include \"columns/tuple.h\"\n#include \"columns/uuid.h\"\n\n#include <chrono>\n#include <cstdint>\n#include <memory>\n#include <ostream>\n#include <string>\n#include <optional>\n\ntypedef struct ssl_ctx_st SSL_CTX;\n\nnamespace clickhouse {\n\nstruct ServerInfo {\n    std::string name;\n    std::string timezone;\n    std::string display_name;\n    uint64_t    version_major;\n    uint64_t    version_minor;\n    uint64_t    version_patch;\n    uint64_t    revision;\n};\n\n/// Methods of block compression.\nenum class CompressionMethod {\n    None    = -1,\n    LZ4     =  1,\n};\n\nstruct Endpoint {\n    std::string host;\n    uint16_t port = 9000;\n    inline bool operator==(const Endpoint& right) const {\n        return host == right.host && port == right.port;\n    }\n};\n\nenum class EndpointsIterationAlgorithm {\n    RoundRobin = 0,\n};\n\nstruct ClientOptions {\n    // Setter goes first, so it is possible to apply 'deprecated' annotation safely.\n#define DECLARE_FIELD(name, type, setter, default_value) \\\n    inline auto & setter(const type& value) { \\\n        name = value; \\\n        return *this; \\\n    } \\\n    type name = default_value\n\n    /// Hostname of the server.\n    DECLARE_FIELD(host, std::string, SetHost, std::string());\n    /// Service port.\n    DECLARE_FIELD(port, uint16_t, SetPort, 9000);\n\n    /** Set endpoints (host+port), only one is used.\n     * Client tries to connect to those endpoints one by one, on the round-robin basis:\n     * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(),\n     * the first one to establish connection is used for the rest of the session.\n     * If port isn't specified, default(9000) value will be used.\n     */\n    DECLARE_FIELD(endpoints, std::vector<Endpoint>, SetEndpoints, {});\n\n    /// Default database.\n    DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, \"default\");\n    /// User name.\n    DECLARE_FIELD(user, std::string, SetUser, \"default\");\n    /// Access password.\n    DECLARE_FIELD(password, std::string, SetPassword, std::string());\n\n    /// By default all exceptions received during query execution will be\n    /// passed to OnException handler.  Set rethrow_exceptions to true to\n    /// enable throwing exceptions with standard c++ exception mechanism.\n    DECLARE_FIELD(rethrow_exceptions, bool, SetRethrowException, true);\n\n    /// Ping server every time before execute any query.\n    DECLARE_FIELD(ping_before_query, bool, SetPingBeforeQuery, false);\n    /// Count of retry to send request to server.\n    DECLARE_FIELD(send_retries, unsigned int, SetSendRetries, 1);\n    /// Amount of time to wait before next retry.\n    DECLARE_FIELD(retry_timeout, std::chrono::seconds, SetRetryTimeout, std::chrono::seconds(5));\n\n    /// Compression method.\n    DECLARE_FIELD(compression_method, CompressionMethod, SetCompressionMethod, CompressionMethod::None);\n\n    /// TCP Keep alive options\n    DECLARE_FIELD(tcp_keepalive, bool, TcpKeepAlive, false);\n    DECLARE_FIELD(tcp_keepalive_idle, std::chrono::seconds, SetTcpKeepAliveIdle, std::chrono::seconds(60));\n    DECLARE_FIELD(tcp_keepalive_intvl, std::chrono::seconds, SetTcpKeepAliveInterval, std::chrono::seconds(5));\n    DECLARE_FIELD(tcp_keepalive_cnt, unsigned int, SetTcpKeepAliveCount, 3);\n\n    // TCP options\n    DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true);\n\n    /// Connection socket connect timeout. If the timeout is negative then the connect operation will never timeout.\n    DECLARE_FIELD(connection_connect_timeout, std::chrono::milliseconds, SetConnectionConnectTimeout, std::chrono::seconds(5));\n\n    /// Connection socket timeout. If the timeout is set to zero then the operation will never timeout.\n    DECLARE_FIELD(connection_recv_timeout, std::chrono::milliseconds, SetConnectionRecvTimeout, std::chrono::milliseconds(0));\n    DECLARE_FIELD(connection_send_timeout, std::chrono::milliseconds, SetConnectionSendTimeout, std::chrono::milliseconds(0));\n\n    /** It helps to ease migration of the old codebases, which can't afford to switch\n    * to using ColumnLowCardinalityT or ColumnLowCardinality directly,\n    * but still want to benefit from smaller on-wire LowCardinality bandwidth footprint.\n    *\n    * @see LowCardinalitySerializationAdaptor, CreateColumnByType\n    */\n    [[deprecated(\"Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) \")]]\n    DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, false);\n\n    /** Set max size data to compress if compression enabled.\n     *\n     *  Allows choosing tradeoff between RAM\\CPU:\n     *  - Lower value reduces RAM usage, but slightly increases CPU usage.\n     *  - Higher value increases RAM usage but slightly decreases CPU usage.\n     */\n    DECLARE_FIELD(max_compression_chunk_size, unsigned int, SetMaxCompressionChunkSize, 65535);\n\n    struct SSLOptions {\n        /** There are two ways to configure an SSL connection:\n         *  - provide a pre-configured SSL_CTX, which is not modified and not owned by the Client.\n         *  - provide a set of options and allow the Client to create and configure SSL_CTX by itself.\n         */\n\n        /** Pre-configured SSL-context for SSL-connection.\n         *  If NOT null client DONES NOT take ownership of context and it must be valid for client lifetime.\n         *  If null client initlaizes OpenSSL and creates his own context, initializes it using\n         *  other options, like path_to_ca_files, path_to_ca_directory, use_default_ca_locations, etc.\n         *\n         *  Either way context is used to create an SSL-connection, which is then configured with\n         *  whatever was provided as `configuration`, `host_flags`, `skip_verification` and `use_sni`.\n         */\n        SSL_CTX * ssl_context = nullptr;\n        auto & SetExternalSSLContext(SSL_CTX * new_ssl_context) {\n            ssl_context = new_ssl_context;\n            return *this;\n        }\n\n        /** Means to validate the server-supplied certificate against trusted Certificate Authority (CA).\n         *  If no CAs are configured, the server's identity can't be validated, and the Client would err.\n         *  See https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_default_verify_paths.html\n        */\n        /// Load default CA certificates from default locations.\n        DECLARE_FIELD(use_default_ca_locations, bool, SetUseDefaultCALocations, true);\n        /// Path to the CA files to verify server certificate, may be empty.\n        DECLARE_FIELD(path_to_ca_files, std::vector<std::string>, SetPathToCAFiles, {});\n        /// Path to the directory with CA files used to validate server certificate, may be empty.\n        DECLARE_FIELD(path_to_ca_directory, std::string, SetPathToCADirectory, \"\");\n\n        /** Min and max protocol versions to use, set with SSL_CTX_set_min_proto_version and SSL_CTX_set_max_proto_version\n         *  for details see https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_min_proto_version.html\n         */\n        DECLARE_FIELD(min_protocol_version, int, SetMinProtocolVersion, DEFAULT_VALUE);\n        DECLARE_FIELD(max_protocol_version, int, SetMaxProtocolVersion, DEFAULT_VALUE);\n\n        /** Options to be set with SSL_CTX_set_options,\n         * for details see https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_options.html\n        */\n        DECLARE_FIELD(context_options, int, SetContextOptions, DEFAULT_VALUE);\n\n        /** Use SNI at ClientHello\n         */\n        DECLARE_FIELD(use_sni, bool, SetUseSNI, true);\n\n        /** Skip SSL session verification (server's certificate, etc).\n         *\n         *  WARNING: settig to true will bypass all SSL session checks, which\n         *  is dangerous, but can be used against self-signed certificates, e.g. for testing purposes.\n         */\n        DECLARE_FIELD(skip_verification, bool, SetSkipVerification, false);\n\n        /** Mode of verifying host ssl certificate against name of the host, set with SSL_set_hostflags.\n         *  For details see https://www.openssl.org/docs/man1.1.1/man3/SSL_set_hostflags.html\n         */\n        DECLARE_FIELD(host_flags, int, SetHostVerifyFlags, DEFAULT_VALUE);\n\n        struct CommandAndValue {\n            std::string command;\n            std::optional<std::string> value = std::nullopt;\n        };\n        /** Extra configuration options, set with SSL_CONF_cmd.\n         *  For deatils see https://www.openssl.org/docs/man1.1.1/man3/SSL_CONF_cmd.html\n         *\n         *  Takes multiple pairs of command-value strings, all commands are supported,\n         *  and prefix is empty.\n         *  i.e. pass `sigalgs` or `SignatureAlgorithms` instead of `-sigalgs`.\n         *\n         *  Rewrites any other options/flags if set in other ways.\n         */\n        DECLARE_FIELD(configuration, std::vector<CommandAndValue>, SetConfiguration, {});\n\n        static const int DEFAULT_VALUE = -1;\n    };\n\n    // By default SSL is turned off.\n    std::optional<SSLOptions> ssl_options = std::nullopt;\n\n    // Will throw an exception if client was built without SSL support.\n    ClientOptions& SetSSLOptions(SSLOptions options);\n\n#undef DECLARE_FIELD\n};\n\nstd::ostream& operator<<(std::ostream& os, const ClientOptions& options);\nstd::ostream& operator<<(std::ostream& os, const Endpoint& options);\n\nclass SocketFactory;\n\n/**\n *\n */\nclass Client {\npublic:\n     Client(const ClientOptions& opts);\n     Client(const ClientOptions& opts,\n            std::unique_ptr<SocketFactory> socket_factory);\n    ~Client();\n\n    /// Intends for execute arbitrary queries.\n    void Execute(const Query& query);\n\n    /// Intends for execute select queries.  Data will be returned with\n    /// one or more call of \\p cb.\n    void Select(const std::string& query, SelectCallback cb);\n    void Select(const std::string& query, const std::string& query_id, SelectCallback cb);\n\n    /// Executes a select query which can be canceled by returning false from\n    /// the data handler function \\p cb.\n    void SelectCancelable(const std::string& query, SelectCancelableCallback cb);\n    void SelectCancelable(const std::string& query, const std::string& query_id, SelectCancelableCallback cb);\n\n    /// Alias for Execute.\n    void Select(const Query& query);\n\n    /// Intends for insert block of data into a table \\p table_name.\n    void Insert(const std::string& table_name, const Block& block);\n    void Insert(const std::string& table_name, const std::string& query_id, const Block& block);\n\n    /// Ping server for aliveness.\n    void Ping();\n\n    /// Reset connection with initial params.\n    void ResetConnection();\n\n    const ServerInfo& GetServerInfo() const;\n\n    /// Get current connected endpoint.\n    /// In case when client is not connected to any endpoint, nullopt will returned.\n    const std::optional<Endpoint>& GetCurrentEndpoint() const;\n\n    // Try to connect to different endpoints one by one only one time. If it doesn't work, throw an exception.\n    void ResetConnectionEndpoint();\n\n    struct Version\n    {\n        uint16_t major;\n        uint16_t minor;\n        uint16_t patch;\n        uint16_t build;\n        const char * extra;\n    };\n\n    static Version GetVersion();\n\nprivate:\n    const ClientOptions options_;\n\n    class Impl;\n    std::unique_ptr<Impl> impl_;\n};\n\n}\n\n// Path: clickhouse/protocol.h\n#pragma once\n\nnamespace clickhouse {\n\n    /// Types of packets received from server\n    namespace ServerCodes {\n        enum {\n            Hello                = 0,    /// Name, version, revision.\n            Data                 = 1,    /// `Block` of data, may be compressed.\n            Exception            = 2,    /// Exception that occurred on server side during query execution.\n            Progress             = 3,    /// Query execcution progress: rows and bytes read.\n            Pong                 = 4,    /// response to Ping sent by client.\n            EndOfStream          = 5,    /// All packets were sent.\n            ProfileInfo          = 6,    /// Profiling data\n            Totals               = 7,    /// Block of totals, may be compressed.\n            Extremes             = 8,    /// Block of mins and maxs, may be compressed.\n            TablesStatusResponse = 9,    /// Response to TableStatus.\n            Log                  = 10,   /// Query execution log.\n            TableColumns         = 11,   /// Columns' description for default values calculation\n            PartUUIDs            = 12,   /// List of unique parts ids.\n            ReadTaskRequest      = 13,   /// String (UUID) describes a request for which next task is needed\n                                         /// This is such an inverted logic, where server sends requests\n                                         /// And client returns back response\n            ProfileEvents        = 14,   /// Packet with profile events from server.\n        };\n    }\n\n    /// Types of packets sent by client.\n    namespace ClientCodes {\n        enum {\n            Hello       = 0,    /// Name, version, default database name.\n            Query       = 1,    /** Query id, query settings, query processing stage,\n                                  * compression status, and query text (no INSERT data).\n                                  */\n            Data        = 2,    /// Data `Block` (e.g. INSERT data), may be compressed.\n            Cancel      = 3,    /// Cancel query.\n            Ping        = 4,    /// Check server connection.\n        };\n    }\n\n    /// Should we compress `Block`s of data\n    namespace CompressionState {\n        enum {\n            Disable     = 0,\n            Enable      = 1,\n        };\n    }\n\n    namespace Stages {\n        enum {\n            Complete    = 2,\n        };\n    }\n}\n\n// Path: clickhouse/base/input.h\n#pragma once\n\n#include <cstddef>\n#include <cstdint>\n#include <vector>\n#include <memory>\n\nnamespace clickhouse {\n\nclass InputStream {\npublic:\n    virtual ~InputStream() noexcept (false)\n    { }\n\n    /// Reads one byte from the stream.\n    inline bool ReadByte(uint8_t* byte) {\n        return DoRead(byte, sizeof(uint8_t)) == sizeof(uint8_t);\n    }\n\n    /// Reads some data from the stream.\n    inline size_t Read(void* buf, size_t len) {\n        return DoRead(buf, len);\n    }\n\n    // Skips a number of bytes.  Returns false if an underlying read error occurs.\n    virtual bool Skip(size_t bytes) = 0;\n\nprotected:\n    virtual size_t DoRead(void* buf, size_t len) = 0;\n};\n\n\nclass ZeroCopyInput : public InputStream {\npublic:\n    inline size_t Next(const void** buf, size_t len) {\n        return DoNext(buf, len);\n    }\n\n    bool Skip(size_t bytes) override;\n\nprotected:\n    virtual size_t DoNext(const void** ptr, size_t len) = 0;\n\n    size_t DoRead(void* buf, size_t len) override;\n};\n\n\n/**\n * A ZeroCopyInput stream backed by an in-memory array of bytes.\n */\nclass ArrayInput : public ZeroCopyInput {\npublic:\n     ArrayInput() noexcept;\n     ArrayInput(const void* buf, size_t len) noexcept;\n    ~ArrayInput() override;\n\n    /// Number of bytes available in the stream.\n    inline size_t Avail() const noexcept {\n        return len_;\n    }\n\n    /// Current read position in the memory block used by this stream.\n    inline const uint8_t* Data() const noexcept {\n        return data_;\n    }\n\n    /// Whether there is more data in the stream.\n    inline bool Exhausted() const noexcept {\n        return !Avail();\n    }\n\n    inline void Reset(const void* buf, size_t len) noexcept {\n        data_ = static_cast<const uint8_t*>(buf);\n        len_ = len;\n    }\n\nprivate:\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    const uint8_t* data_;\n    size_t len_;\n};\n\n\nclass BufferedInput : public ZeroCopyInput {\npublic:\n    BufferedInput(std::unique_ptr<InputStream> source, size_t buflen = 8192);\n    ~BufferedInput() override;\n\n    void Reset();\n\nprotected:\n    size_t DoRead(void* buf, size_t len) override;\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    std::unique_ptr<InputStream> const source_;\n    ArrayInput array_input_;\n    std::vector<uint8_t> buffer_;\n};\n\n}\n\n// Path: clickhouse/base/buffer.h\n#pragma once\n\n#include <cstdint>\n#include <vector>\n\nnamespace clickhouse {\n\nusing Buffer = std::vector<uint8_t>;\n\n}\n\n// Path: clickhouse/base/output.h\n#pragma once\n\n...\n", "needle_token_start": 7112, "needle_token_end": 7147, "code_context_ntokens": 15933, "output": ["```cpp\ntemplate <typename T>\ninline void AppendMany(const T& container) {\n    for (const auto & item : container) {\n        Append(item);\n    }\n}\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "GetEndpointsIterator", "language": "cpp", "path": "clickhouse/client.cpp", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To generate an iterator that cycles through a list of network connection points in a round-robin fashion.\n2. **Input**: A configuration object containing a list of network endpoints.\n3. **Output**: A unique pointer to an iterator base class, specifically initialized to iterate over the provided endpoints in a round-robin manner.\n4. **Procedure**: The function first checks if the list of endpoints in the input configuration is empty. If it is, an exception is thrown indicating the absence of endpoints. If not empty, the function creates and returns a unique pointer to a round-robin iterator initialized with the list of endpoints.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/client.h\n#pragma once\n\n#include \"query.h\"\n#include \"exceptions.h\"\n\n#include \"columns/array.h\"\n#include \"columns/date.h\"\n#include \"columns/decimal.h\"\n#include \"columns/enum.h\"\n#include \"columns/geo.h\"\n#include \"columns/ip4.h\"\n#include \"columns/ip6.h\"\n#include \"columns/lowcardinality.h\"\n#include \"columns/nullable.h\"\n#include \"columns/numeric.h\"\n#include \"columns/map.h\"\n#include \"columns/string.h\"\n#include \"columns/tuple.h\"\n#include \"columns/uuid.h\"\n\n#include <chrono>\n#include <cstdint>\n#include <memory>\n#include <ostream>\n#include <string>\n#include <optional>\n\ntypedef struct ssl_ctx_st SSL_CTX;\n\nnamespace clickhouse {\n\nstruct ServerInfo {\n    std::string name;\n    std::string timezone;\n    std::string display_name;\n    uint64_t    version_major;\n    uint64_t    version_minor;\n    uint64_t    version_patch;\n    uint64_t    revision;\n};\n\n/// Methods of block compression.\nenum class CompressionMethod {\n    None    = -1,\n    LZ4     =  1,\n};\n\nstruct Endpoint {\n    std::string host;\n    uint16_t port = 9000;\n    inline bool operator==(const Endpoint& right) const {\n        return host == right.host && port == right.port;\n    }\n};\n\nenum class EndpointsIterationAlgorithm {\n    RoundRobin = 0,\n};\n\nstruct ClientOptions {\n    // Setter goes first, so it is possible to apply 'deprecated' annotation safely.\n#define DECLARE_FIELD(name, type, setter, default_value) \\\n    inline auto & setter(const type& value) { \\\n        name = value; \\\n        return *this; \\\n    } \\\n    type name = default_value\n\n    /// Hostname of the server.\n    DECLARE_FIELD(host, std::string, SetHost, std::string());\n    /// Service port.\n    DECLARE_FIELD(port, uint16_t, SetPort, 9000);\n\n    /** Set endpoints (host+port), only one is used.\n     * Client tries to connect to those endpoints one by one, on the round-robin basis:\n     * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(),\n     * the first one to establish connection is used for the rest of the session.\n     * If port isn't specified, default(9000) value will be used.\n     */\n    DECLARE_FIELD(endpoints, std::vector<Endpoint>, SetEndpoints, {});\n\n    /// Default database.\n    DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, \"default\");\n    /// User name.\n    DECLARE_FIELD(user, std::string, SetUser, \"default\");\n    /// Access password.\n    DECLARE_FIELD(password, std::string, SetPassword, std::string());\n\n    /// By default all exceptions received during query execution will be\n    /// passed to OnException handler.  Set rethrow_exceptions to true to\n    /// enable throwing exceptions with standard c++ exception mechanism.\n...\n// Path: clickhouse/protocol.h\n#pragma once\n\nnamespace clickhouse {\n\n    /// Types of packets received from server\n    namespace ServerCodes {\n        enum {\n            Hello                = 0,    /// Name, version, revision.\n            Data                 = 1,    /// `Block` of data, may be compressed.\n            Exception            = 2,    /// Exception that occurred on server side during query execution.\n            Progress             = 3,    /// Query execcution progress: rows and bytes read.\n            Pong                 = 4,    /// response to Ping sent by client.\n            EndOfStream          = 5,    /// All packets were sent.\n            ProfileInfo          = 6,    /// Profiling data\n            Totals               = 7,    /// Block of totals, may be compressed.\n            Extremes             = 8,    /// Block of mins and maxs, may be compressed.\n            TablesStatusResponse = 9,    /// Response to TableStatus.\n            Log                  = 10,   /// Query execution log.\n            TableColumns         = 11,   /// Columns' description for default values calculation\n            PartUUIDs            = 12,   /// List of unique parts ids.\n            ReadTaskRequest      = 13,   /// String (UUID) describes a request for which next task is needed\n                                         /// This is such an inverted logic, where server sends requests\n                                         /// And client returns back response\n            ProfileEvents        = 14,   /// Packet with profile events from server.\n        };\n    }\n\n    /// Types of packets sent by client.\n    namespace ClientCodes {\n        enum {\n            Hello       = 0,    /// Name, version, default database name.\n            Query       = 1,    /** Query id, query settings, query processing stage,\n                                  * compression status, and query text (no INSERT data).\n                                  */\n            Data        = 2,    /// Data `Block` (e.g. INSERT data), may be compressed.\n            Cancel      = 3,    /// Cancel query.\n            Ping        = 4,    /// Check server connection.\n        };\n    }\n\n    /// Should we compress `Block`s of data\n    namespace CompressionState {\n        enum {\n            Disable     = 0,\n            Enable      = 1,\n        };\n    }\n\n    namespace Stages {\n        enum {\n            Complete    = 2,\n        };\n    }\n}\n\n// Path: clickhouse/base/input.h\n#pragma once\n\n#include <cstddef>\n#include <cstdint>\n#include <vector>\n#include <memory>\n\nnamespace clickhouse {\n\nclass InputStream {\npublic:\n    virtual ~InputStream() noexcept (false)\n    { }\n\n    /// Reads one byte from the stream.\n    inline bool ReadByte(uint8_t* byte) {\n        return DoRead(byte, sizeof(uint8_t)) == sizeof(uint8_t);\n    }\n\n    /// Reads some data from the stream.\n    inline size_t Read(void* buf, size_t len) {\n        return DoRead(buf, len);\n    }\n\n    // Skips a number of bytes.  Returns false if an underlying read error occurs.\n    virtual bool Skip(size_t bytes) = 0;\n\nprotected:\n    virtual size_t DoRead(void* buf, size_t len) = 0;\n};\n\n\nclass ZeroCopyInput : public InputStream {\npublic:\n    inline size_t Next(const void** buf, size_t len) {\n        return DoNext(buf, len);\n    }\n\n    bool Skip(size_t bytes) override;\n\nprotected:\n    virtual size_t DoNext(const void** ptr, size_t len) = 0;\n\n    size_t DoRead(void* buf, size_t len) override;\n};\n\n\n/**\n * A ZeroCopyInput stream backed by an in-memory array of bytes.\n */\nclass ArrayInput : public ZeroCopyInput {\npublic:\n     ArrayInput() noexcept;\n     ArrayInput(const void* buf, size_t len) noexcept;\n    ~ArrayInput() override;\n\n    /// Number of bytes available in the stream.\n    inline size_t Avail() const noexcept {\n        return len_;\n    }\n\n    /// Current read position in the memory block used by this stream.\n    inline const uint8_t* Data() const noexcept {\n        return data_;\n    }\n\n    /// Whether there is more data in the stream.\n    inline bool Exhausted() const noexcept {\n        return !Avail();\n    }\n\n    inline void Reset(const void* buf, size_t len) noexcept {\n        data_ = static_cast<const uint8_t*>(buf);\n        len_ = len;\n    }\n\nprivate:\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    const uint8_t* data_;\n    size_t len_;\n};\n\n\nclass BufferedInput : public ZeroCopyInput {\npublic:\n    BufferedInput(std::unique_ptr<InputStream> source, size_t buflen = 8192);\n    ~BufferedInput() override;\n\n    void Reset();\n\nprotected:\n    size_t DoRead(void* buf, size_t len) override;\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    std::unique_ptr<InputStream> const source_;\n    ArrayInput array_input_;\n    std::vector<uint8_t> buffer_;\n};\n\n}\n\n// Path: clickhouse/base/buffer.h\n#pragma once\n\n#include <cstdint>\n#include <vector>\n\nnamespace clickhouse {\n\nusing Buffer = std::vector<uint8_t>;\n\n}\n\n// Path: clickhouse/base/output.h\n#pragma once\n\n#include \"buffer.h\"\n\n#include <cstddef>\n#include <cstdint>\n#include <vector>\n#include <memory.h>\n#include <memory>\n\nnamespace clickhouse {\n\nclass OutputStream {\npublic:\n    virtual ~OutputStream()\n    { }\n\n    inline void Flush() {\n        DoFlush();\n    }\n\n    inline size_t Write(const void* data, size_t len) {\n        return DoWrite(data, len);\n    }\n\nprotected:\n    virtual void DoFlush() { }\n\n    virtual size_t DoWrite(const void* data, size_t len) = 0;\n};\n\n\nclass ZeroCopyOutput : public OutputStream {\npublic:\n    inline size_t Next(void** data, size_t size) {\n        return DoNext(data, size);\n    }\n\nprotected:\n    // Obtains a buffer into which data can be written.  Any data written\n    // into this buffer will eventually (maybe instantly, maybe later on)\n    // be written to the output.\n    virtual size_t DoNext(void** data, size_t len) = 0;\n\n    size_t DoWrite(const void* data, size_t len) override;\n};\n\n\n/**\n * A ZeroCopyOutput stream backed by an in-memory array of bytes.\n */\nclass ArrayOutput : public ZeroCopyOutput {\npublic:\n     ArrayOutput(void* buf, size_t len);\n    ~ArrayOutput() override;\n\n    /// Number of bytes available in the stream.\n    inline size_t Avail() const noexcept {\n        return end_ - buf_;\n    }\n\n    /// Current write position in the memory block used by this stream.\n    inline const uint8_t* Data() const noexcept {\n        return buf_;\n    }\n\n    /// Whether there is more space in the stream.\n    inline bool Exhausted() const noexcept {\n        return !Avail();\n    }\n\n    /// Initializes this stream with a new memory block.\n    inline void Reset(void* buf, size_t len) noexcept {\n        buf_ = static_cast<uint8_t*>(buf);\n        end_ = buf_ + len;\n        buffer_size_ = len;\n    }\n\n    /// Number of bytes written to the buffer.\n    inline size_t Size() const noexcept {\n        return buffer_size_ - Avail();\n    }\n\nprotected:\n    size_t DoNext(void** data, size_t len) override;\n\nprivate:\n    uint8_t* buf_;\n    uint8_t* end_;\n    size_t buffer_size_;\n};\n\n\n/**\n * A ZeroCopyOutput stream backed by a vector.\n *\n * Doesn't Flush() in destructor, client must ensure to do it manually at some point.\n */\nclass BufferOutput : public ZeroCopyOutput {\npublic:\n     BufferOutput(Buffer* buf);\n    ~BufferOutput() override;\n\nprotected:\n    size_t DoNext(void** data, size_t len) override;\n\nprivate:\n    Buffer* buf_;\n    size_t pos_;\n};\n\n/** BufferedOutput writes data to internal buffer first.\n *\n *  Any data goes to underlying stream only if internal buffer is full\n *  or when client invokes Flush() on this.\n *\n * Doesn't Flush() in destructor, client must ensure to do it manually at some point.\n */\nclass BufferedOutput : public ZeroCopyOutput {\npublic:\n    explicit BufferedOutput(std::unique_ptr<OutputStream> destination, size_t buflen = 8192);\n    ~BufferedOutput() override;\n\n    void Reset();\n\nprotected:\n    void DoFlush() override;\n    size_t DoNext(void** data, size_t len) override;\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    std::unique_ptr<OutputStream> const destination_;\n    Buffer buffer_;\n    ArrayOutput array_output_;\n};\n\ntemplate <typename T>\nvoid WriteUnaligned(void* buf, const T& value) {\n    memcpy(buf, &value, sizeof(value));\n}\n\n}\n\n// Path: clickhouse/base/compressed.h\n#pragma once\n\n#include \"input.h\"\n#include \"output.h\"\n#include \"buffer.h\"\n\nnamespace clickhouse {\n\nclass CompressedInput : public ZeroCopyInput {\npublic:\n    explicit CompressedInput(InputStream* input);\n    ~CompressedInput() override;\n\nprotected:\n    size_t DoNext(const void** ptr, size_t len) override;\n\n    bool Decompress();\n\nprivate:\n    InputStream* const input_;\n\n    Buffer data_;\n    ArrayInput mem_;\n};\n\nclass CompressedOutput : public OutputStream {\npublic:\n    explicit CompressedOutput(OutputStream * destination, size_t max_compressed_chunk_size = 0);\n    ~CompressedOutput() override;\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n    void DoFlush() override;\n\nprivate:\n    void Compress(const void * data, size_t len);\n    void PreallocateCompressBuffer(size_t input_size);\n\nprivate:\n    OutputStream * destination_;\n    const size_t max_compressed_chunk_size_;\n    Buffer compressed_buffer_;\n};\n\n}\n\n// Path: clickhouse/base/platform.h\n#pragma once\n\n#if defined(__linux__)\n#   define _linux_\n#elif defined(_WIN64)\n#   define _win64_\n#   define _win32_\n#elif defined(__WIN32__) || defined(_WIN32)\n#   define _win32_\n#elif defined(__APPLE__)\n#   define _darwin_\n#endif\n\n#if defined(_win32_) || defined(_win64_)\n#   define _win_\n#   if !defined(_WIN32_WINNT) || (_WIN32_WINNT < 0x0600)\n#      undef _WIN32_WINNT\n#      define _WIN32_WINNT 0x0600 // The WSAPoll function is defined on Windows Vista and later.\n#   endif\n#   define WIN32_LEAN_AND_MEAN 1  // don't include too much header automatically\n#endif\n\n#if defined(_linux_) || defined (_darwin_)\n#   define _unix_\n#endif\n\n#if defined(_MSC_VER)\n#   undef NOMINMAX\n#   define NOMINMAX\n#   include <basetsd.h>\n#   define ssize_t SSIZE_T\n#   define HAVE_SSIZE_T 1\n#endif\n\n// Path: clickhouse/base/endpoints_iterator.h\n#pragma once\n\n#include \"clickhouse/client.h\"\n#include <vector>\n\nnamespace clickhouse {\n\nstruct ClientOptions;\n\n/**\n * Base class for iterating through endpoints.\n*/\nclass EndpointsIteratorBase\n{\n public:\n   virtual ~EndpointsIteratorBase() = default;\n\n   virtual Endpoint Next() = 0;\n};\n\nclass RoundRobinEndpointsIterator : public EndpointsIteratorBase\n{\n public:\n    explicit RoundRobinEndpointsIterator(const std::vector<Endpoint>& opts);\n    Endpoint Next() override;\n\n    ~RoundRobinEndpointsIterator() override;\n\n private:\n    const std::vector<Endpoint>& endpoints;\n    size_t current_index;\n};\n\n}\n\n// Path: clickhouse/base/socket.h\n#pragma once\n\n#include \"platform.h\"\n#include \"input.h\"\n#include \"output.h\"\n#include \"endpoints_iterator.h\"\n\n#include <cstddef>\n#include <string>\n#include <chrono>\n\n#if defined(_win_)\n#   include <winsock2.h>\n#   include <ws2tcpip.h>\n#else\n#   include <arpa/inet.h>\n#   include <sys/types.h>\n#   include <sys/socket.h>\n#   include <poll.h>\n\n#   if !defined(SOCKET)\n#       define SOCKET int\n#   endif\n#endif\n\n#include <memory>\n#include <system_error>\n\nstruct addrinfo;\n\nnamespace clickhouse {\n\nstruct ClientOptions;\n\n/** Address of a host to establish connection to.\n *\n */\nclass NetworkAddress {\npublic:\n    explicit NetworkAddress(const std::string& host,\n                            const std::string& port = \"0\");\n    ~NetworkAddress();\n\n    const struct addrinfo* Info() const;\n    const std::string & Host() const;\n\nprivate:\n    const std::string host_;\n    struct addrinfo* info_;\n};\n\n#if defined(_win_)\n\nclass windowsErrorCategory : public std::error_category {\npublic:\n    char const* name() const noexcept override final;\n    std::string message(int c) const override final;\n\n    static windowsErrorCategory const& category();\n};\n\n#endif\n\n#if defined(_unix_)\n\nclass getaddrinfoErrorCategory : public std::error_category {\npublic:\n    char const* name() const noexcept override final;\n    std::string message(int c) const override final;\n\n    static getaddrinfoErrorCategory const& category();\n};\n\n#endif\n\n\nclass SocketBase {\npublic:\n    virtual ~SocketBase();\n\n    virtual std::unique_ptr<InputStream> makeInputStream() const = 0;\n    virtual std::unique_ptr<OutputStream> makeOutputStream() const = 0;\n};\n\n\nclass SocketFactory {\npublic:\n    virtual ~SocketFactory();\n\n    // TODO: move connection-related options to ConnectionOptions structure.\n\n    virtual std::unique_ptr<SocketBase> connect(const ClientOptions& opts, const Endpoint& endpoint) = 0;\n\n    virtual void sleepFor(const std::chrono::milliseconds& duration);\n};\n\n\nstruct SocketTimeoutParams {\n    std::chrono::milliseconds connect_timeout{ 5000 };\n    std::chrono::milliseconds recv_timeout{ 0 };\n    std::chrono::milliseconds send_timeout{ 0 };\n};\n\nclass Socket : public SocketBase {\npublic:\n    Socket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params);\n    Socket(const NetworkAddress& addr);\n    Socket(Socket&& other) noexcept;\n    Socket& operator=(Socket&& other) noexcept;\n\n    ~Socket() override;\n\n    /// @params idle the time (in seconds) the connection needs to remain\n    ///         idle before TCP starts sending keepalive probes.\n    /// @params intvl the time (in seconds) between individual keepalive probes.\n    /// @params cnt the maximum number of keepalive probes TCP should send\n    ///         before dropping the connection.\n    void SetTcpKeepAlive(int idle, int intvl, int cnt) noexcept;\n\n    /// @params nodelay whether to enable TCP_NODELAY\n    void SetTcpNoDelay(bool nodelay) noexcept;\n\n    std::unique_ptr<InputStream> makeInputStream() const override;\n    std::unique_ptr<OutputStream> makeOutputStream() const override;\n\nprotected:\n    Socket(const Socket&) = delete;\n    Socket& operator = (const Socket&) = delete;\n    void Close();\n\n    SOCKET handle_;\n};\n\n\nclass NonSecureSocketFactory : public SocketFactory {\npublic:\n    ~NonSecureSocketFactory() override;\n\n    std::unique_ptr<SocketBase> connect(const ClientOptions& opts, const Endpoint& endpoint) override;\n\nprotected:\n    virtual std::unique_ptr<Socket> doConnect(const NetworkAddress& address, const ClientOptions& opts);\n\n    void setSocketOptions(Socket& socket, const ClientOptions& opts);\n};\n\n\nclass SocketInput : public InputStream {\npublic:\n    explicit SocketInput(SOCKET s);\n    ~SocketInput();\n\nprotected:\n    bool Skip(size_t bytes) override;\n    size_t DoRead(void* buf, size_t len) override;\n\nprivate:\n    SOCKET s_;\n};\n\nclass SocketOutput : public OutputStream {\npublic:\n    explicit SocketOutput(SOCKET s);\n    ~SocketOutput();\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    SOCKET s_;\n};\n\nstatic struct NetrworkInitializer {\n    NetrworkInitializer();\n} gNetrworkInitializer;\n\n}\n\n// Path: clickhouse/base/wire_format.h\n#pragma once\n\n#include <string>\n#include <cstdint>\n\nnamespace clickhouse {\n\nclass InputStream;\nclass OutputStream;\n\nclass WireFormat {\npublic:\n    template <typename T>\n    static bool ReadFixed(InputStream& input, T* value);\n    static bool ReadString(InputStream& input, std::string* value);\n    static bool SkipString(InputStream& input);\n    static bool ReadBytes(InputStream& input, void* buf, size_t len);\n    static bool ReadUInt64(InputStream& input, uint64_t* value);\n    static bool ReadVarint64(InputStream& output, uint64_t* value);\n\n    template <typename T>\n    static void WriteFixed(OutputStream& output, const T& value);\n    static void WriteBytes(OutputStream& output, const void* buf, size_t len);\n    static void WriteString(OutputStream& output, std::string_view value);\n    static void WriteUInt64(OutputStream& output, const uint64_t value);\n    static void WriteVarint64(OutputStream& output, uint64_t value);\n\nprivate:\n    static bool ReadAll(InputStream& input, void* buf, size_t len);\n    static void WriteAll(OutputStream& output, const void* buf, size_t len);\n};\n\ntemplate <typename T>\ninline bool WireFormat::ReadFixed(InputStream& input, T* value) {\n    return ReadAll(input, value, sizeof(T));\n}\n\ninline bool WireFormat::ReadString(InputStream& input, std::string* value) {\n    uint64_t len = 0;\n    if (ReadVarint64(input, &len)) {\n        if (len > 0x00FFFFFFULL) {\n            return false;\n        }\n        value->resize((size_t)len);\n        return ReadAll(input, value->data(), (size_t)len);\n    }\n\n    return false;\n}\n\ninline bool WireFormat::ReadBytes(InputStream& input, void* buf, size_t len) {\n    return ReadAll(input, buf, len);\n}\n\ninline bool WireFormat::ReadUInt64(InputStream& input, uint64_t* value) {\n    return ReadVarint64(input, value);\n}\n\ntemplate <typename T>\ninline void WireFormat::WriteFixed(OutputStream& output, const T& value) {\n    WriteAll(output, &value, sizeof(T));\n}\n\ninline void WireFormat::WriteBytes(OutputStream& output, const void* buf, size_t len) {\n    WriteAll(output, buf, len);\n}\n\ninline void WireFormat::WriteString(OutputStream& output, std::string_view value) {\n    WriteVarint64(output, value.size());\n    WriteAll(output, value.data(), value.size());\n}\n\ninline void WireFormat::WriteUInt64(OutputStream& output, const uint64_t value) {\n    WriteVarint64(output, value);\n}\n\n}\n\n// Path: clickhouse/columns/factory.h\n#pragma once\n\n#include \"column.h\"\n\nnamespace clickhouse {\n\nstruct CreateColumnByTypeSettings\n{\n    bool low_cardinality_as_wrapped_column = false;\n};\n\nColumnRef CreateColumnByType(const std::string& type_name, CreateColumnByTypeSettings settings = {});\n\n}\n\n// Path: clickhouse/base/sslsocket.h\n#pragma once\n\n#include \"socket.h\"\n\n#include <memory>\n#include <optional>\n#include <vector>\n\ntypedef struct ssl_ctx_st SSL_CTX;\ntypedef struct ssl_st SSL;\n\nnamespace clickhouse {\n\nstruct SSLParams\n{\n    std::vector<std::string> path_to_ca_files;\n    std::string path_to_ca_directory;\n    bool use_default_ca_locations;\n    int context_options;\n    int min_protocol_version;\n    int max_protocol_version;\n    bool use_SNI;\n    bool skip_verification;\n    int host_flags;\n    using ConfigurationType = std::vector<std::pair<std::string, std::optional<std::string>>>;\n    ConfigurationType configuration;\n};\n\nclass SSLContext\n{\npublic:\n    explicit SSLContext(SSL_CTX & context);\n    explicit SSLContext(const SSLParams & context_params);\n    ~SSLContext() = default;\n\n    SSLContext(const SSLContext &) = delete;\n    SSLContext& operator=(const SSLContext &) = delete;\n    SSLContext(SSLContext &&) = delete;\n    SSLContext& operator=(SSLContext &) = delete;\n\nprivate:\n    friend class SSLSocket;\n    SSL_CTX * getContext();\n\nprivate:\n    std::unique_ptr<SSL_CTX, void (*)(SSL_CTX*)> context_;\n};\n\nclass SSLSocket : public Socket {\npublic:\n    explicit SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params,\n                       const SSLParams& ssl_params, SSLContext& context);\n\n    SSLSocket(SSLSocket &&) = default;\n    ~SSLSocket() override = default;\n\n    SSLSocket(const SSLSocket & ) = delete;\n    SSLSocket& operator=(const SSLSocket & ) = delete;\n\n    std::unique_ptr<InputStream> makeInputStream() const override;\n    std::unique_ptr<OutputStream> makeOutputStream() const override;\n\n    static void validateParams(const SSLParams & ssl_params);\nprivate:\n    std::unique_ptr<SSL, void (*)(SSL *s)> ssl_;\n};\n\nclass SSLSocketFactory : public NonSecureSocketFactory {\npublic:\n    explicit SSLSocketFactory(const ClientOptions& opts);\n    ~SSLSocketFactory() override;\n\nprotected:\n    std::unique_ptr<Socket> doConnect(const NetworkAddress& address, const ClientOptions& opts) override;\n\nprivate:\n    const SSLParams ssl_params_;\n    std::unique_ptr<SSLContext> ssl_context_;\n};\n\nclass SSLSocketInput : public InputStream {\npublic:\n    explicit SSLSocketInput(SSL *ssl);\n    ~SSLSocketInput() = default;\n\n    bool Skip(size_t /*bytes*/) override {\n        return false;\n    }\n\nprotected:\n    size_t DoRead(void* buf, size_t len) override;\n\nprivate:\n    // Not owning\n    SSL *ssl_;\n};\n\nclass SSLSocketOutput : public OutputStream {\npublic:\n    explicit SSLSocketOutput(SSL *ssl);\n    ~SSLSocketOutput() = default;\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    // Not owning\n    SSL *ssl_;\n};\n\n}\n\n// Path: clickhouse/client.cpp\n#include \"client.h\"\n#include \"clickhouse/version.h\"\n#include \"protocol.h\"\n\n#include \"base/compressed.h\"\n#include \"base/socket.h\"\n#include \"base/wire_format.h\"\n\n#include \"columns/factory.h\"\n\n#include <assert.h>\n#include <system_error>\n#include <vector>\n#include <sstream>\n\n#if defined(WITH_OPENSSL)\n#include \"base/sslsocket.h\"\n#endif\n\n#define DBMS_NAME                                       \"ClickHouse\"\n\n#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES         50264\n#define DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS   51554\n#define DBMS_MIN_REVISION_WITH_BLOCK_INFO               51903\n#define DBMS_MIN_REVISION_WITH_CLIENT_INFO              54032\n#define DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE          54058\n#define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060\n//#define DBMS_MIN_REVISION_WITH_TABLES_STATUS            54226\n#define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337\n#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME      54372\n#define DBMS_MIN_REVISION_WITH_VERSION_PATCH            54401\n#define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE     54405\n#define DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA 54410\n#define DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO        54420\n#define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429\n#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET       54441\n#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY            54442\n#define DBMS_MIN_REVISION_WITH_DISTRIBUTED_DEPTH        54448\n#define DBMS_MIN_REVISION_WITH_INITIAL_QUERY_START_TIME 54449\n#define DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS 54451\n\n#define DMBS_PROTOCOL_REVISION  DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS\n\nnamespace clickhouse {\n\nstruct ClientInfo {\n    uint8_t iface_type = 1; // TCP\n    uint8_t query_kind;\n    std::string initial_user;\n    std::string initial_query_id;\n    std::string quota_key;\n    std::string os_user;\n    std::string client_hostname;\n    std::string client_name;\n    std::string initial_address = \"[::ffff:127.0.0.1]:0\";\n    uint64_t client_version_major = 0;\n    uint64_t client_version_minor = 0;\n    uint64_t client_version_patch = 0;\n    uint32_t client_revision = 0;\n};\n\nstd::ostream& operator<<(std::ostream& os, const Endpoint& endpoint) {\n    return os << endpoint.host << \":\" << endpoint.port;\n}\n\nstd::ostream& operator<<(std::ostream& os, const ClientOptions& opt) {\n    os << \"Client(\"\n       << \" Endpoints : [\";\n    size_t extra_endpoints = 0;\n\n    if (!opt.host.empty()) {\n        extra_endpoints = 1;\n        os << opt.user << '@' << Endpoint{opt.host, opt.port};\n\n        if (opt.endpoints.size())\n            os << \", \";\n    }\n\n    for (size_t i = 0; i < opt.endpoints.size(); i++) {\n        os << opt.user << '@' << opt.endpoints[i]\n           << ((i == opt.endpoints.size() - 1) ? \"\" : \", \");\n    }\n\n    os << \"] (\" << opt.endpoints.size() + extra_endpoints << \" items )\"\n       << \" ping_before_query:\" << opt.ping_before_query\n       << \" send_retries:\" << opt.send_retries\n       << \" retry_timeout:\" << opt.retry_timeout.count()\n       << \" compression_method:\"\n       << (opt.compression_method == CompressionMethod::LZ4 ? \"LZ4\" : \"None\");\n#if defined(WITH_OPENSSL)\n    if (opt.ssl_options) {\n        const auto & ssl_options = *opt.ssl_options;\n        os << \" SSL (\"\n           << \" ssl_context: \" << (ssl_options.ssl_context ? \"provided by user\" : \"created internally\")\n           << \" use_default_ca_locations: \" << ssl_options.use_default_ca_locations\n           << \" path_to_ca_files: \" << ssl_options.path_to_ca_files.size() << \" items\"\n           << \" path_to_ca_directory: \" << ssl_options.path_to_ca_directory\n           << \" min_protocol_version: \" << ssl_options.min_protocol_version\n           << \" max_protocol_version: \" << ssl_options.max_protocol_version\n           << \" context_options: \" << ssl_options.context_options\n           << \")\";\n    }\n#endif\n    os << \")\";\n    return os;\n}\n\nClientOptions& ClientOptions::SetSSLOptions(ClientOptions::SSLOptions options)\n{\n#ifdef WITH_OPENSSL\n    ssl_options = options;\n    return *this;\n#else\n    (void)options;\n    throw OpenSSLError(\"Library was built with no SSL support\");\n#endif\n}\n\nnamespace {\n\nstd::unique_ptr<SocketFactory> GetSocketFactory(const ClientOptions& opts) {\n    (void)opts;\n#if defined(WITH_OPENSSL)\n    if (opts.ssl_options)\n        return std::make_unique<SSLSocketFactory>(opts);\n    else\n#endif\n        return std::make_unique<NonSecureSocketFactory>();\n}\n\n\nstd::unique_ptr<EndpointsIteratorBase> GetEndpointsIterator(const ClientOptions& opts) {\n    if (opts.endpoints.empty())\n    {\n        throw ValidationError(\"The list of endpoints is empty\");\n    }\n\n    return std::make_unique<RoundRobinEndpointsIterator>(opts.endpoints);\n}\n\n}\n\nclass Client::Impl {\npublic:\n     Impl(const ClientOptions& opts);\n     Impl(const ClientOptions& opts,\n          std::unique_ptr<SocketFactory> socket_factory);\n    ~Impl();\n\n    void ExecuteQuery(Query query);\n\n    void SendCancel();\n\n    void Insert(const std::string& table_name, const std::string& query_id, const Block& block);\n\n    void Ping();\n\n    void ResetConnection();\n\n    void ResetConnectionEndpoint();\n\n    const ServerInfo& GetServerInfo() const;\n\n    const std::optional<Endpoint>& GetCurrentEndpoint() const;\n\nprivate:\n    bool Handshake();\n\n    bool ReceivePacket(uint64_t* server_packet = nullptr);\n\n    void SendQuery(const Query& query);\n\n    void SendData(const Block& block);\n\n    bool SendHello();\n\n    bool ReadBlock(InputStream& input, Block* block);\n\n    bool ReceiveHello();\n\n    /// Reads data packet form input stream.\n    bool ReceiveData();\n\n    /// Reads exception packet form input stream.\n    bool ReceiveException(bool rethrow = false);\n\n    void WriteBlock(const Block& block, OutputStream& output);\n\n    void CreateConnection();\n\n    void InitializeStreams(std::unique_ptr<SocketBase>&& socket);\n\n    inline size_t GetConnectionAttempts() const\n    {\n        return options_.endpoints.size() * options_.send_retries;\n    }\n\nprivate:\n    /// In case of network errors tries to reconnect to server and\n    /// call fuc several times.\n    void RetryGuard(std::function<void()> func);\n\n    void RetryConnectToTheEndpoint(std::function<void()>& func);\n\nprivate:\n    class EnsureNull {\n    public:\n        inline EnsureNull(QueryEvents* ev, QueryEvents** ptr)\n            : ptr_(ptr)\n        {\n            if (ptr_) {\n                *ptr_ = ev;\n            }\n        }\n\n        inline ~EnsureNull() {\n            if (ptr_) {\n                *ptr_ = nullptr;\n            }\n        }\n\n    private:\n        QueryEvents** ptr_;\n\n    };\n\n\n    const ClientOptions options_;\n    QueryEvents* events_;\n    int compression_ = CompressionState::Disable;\n\n    std::unique_ptr<SocketFactory> socket_factory_;\n\n    std::unique_ptr<InputStream> input_;\n    std::unique_ptr<OutputStream> output_;\n    std::unique_ptr<SocketBase> socket_;\n    std::unique_ptr<EndpointsIteratorBase> endpoints_iterator;\n\n    std::optional<Endpoint> current_endpoint_;\n\n    ServerInfo server_info_;\n};\n\nClientOptions modifyClientOptions(ClientOptions opts)\n{\n    if (opts.host.empty())\n        return opts;\n\n    Endpoint default_endpoint({opts.host, opts.port});\n    opts.endpoints.emplace(opts.endpoints.begin(), default_endpoint);\n    return opts;\n}\n\nClient::Impl::Impl(const ClientOptions& opts)\n    : Impl(opts, GetSocketFactory(opts)) {}\n\nClient::Impl::Impl(const ClientOptions& opts,\n                   std::unique_ptr<SocketFactory> socket_factory)\n    : options_(modifyClientOptions(opts))\n    , events_(nullptr)\n    , socket_factory_(std::move(socket_factory))\n    , endpoints_iterator(GetEndpointsIterator(options_))\n{\n    CreateConnection();\n\n    if (options_.compression_method != CompressionMethod::None) {\n        compression_ = CompressionState::Enable;\n    }\n}\n\nClient::Impl::~Impl()\n{ }\n\nvoid Client::Impl::ExecuteQuery(Query query) {\n    EnsureNull en(static_cast<QueryEvents*>(&query), &events_);\n\n    if (options_.ping_before_query) {\n        RetryGuard([this]() { Ping(); });\n    }\n\n    SendQuery(query);\n\n    while (ReceivePacket()) {\n        ;\n    }\n}\n\nstd::string NameToQueryString(const std::string &input)\n{\n    std::string output;\n    output.reserve(input.size() + 2);\n    output += '`';\n\n    for (const auto & c : input) {\n        if (c == '`') {\n            //escape ` with ``\n            output.append(\"``\");\n        } else {\n            output.push_back(c);\n        }\n    }\n\n    output += '`';\n    return output;\n}\n\nvoid Client::Impl::Insert(const std::string& table_name, const std::string& query_id, const Block& block) {\n    if (options_.ping_before_query) {\n        RetryGuard([this]() { Ping(); });\n    }\n\n    std::stringstream fields_section;\n        const auto num_columns = block.GetColumnCount();\n\n    for (unsigned int i = 0; i < num_columns; ++i) {\n        if (i == num_columns - 1) {\n            fields_section << NameToQueryString(block.GetColumnName(i));\n        } else {\n            fields_section << NameToQueryString(block.GetColumnName(i)) << \",\";\n        }\n    }\n\n    Query query(\"INSERT INTO \" + table_name + \" ( \" + fields_section.str() + \" ) VALUES\", query_id);\n    SendQuery(query);\n\n    uint64_t server_packet;\n    // Receive data packet.\n    while (true) {\n        bool ret = ReceivePacket(&server_packet);\n\n        if (!ret) {\n            throw ProtocolError(\"fail to receive data packet\");\n        }\n        if (server_packet == ServerCodes::Data) {\n            break;\n        }\n        if (server_packet == ServerCodes::Progress) {\n            continue;\n        }\n    }\n\n    // Send data.\n    SendData(block);\n    // Send empty block as marker of\n    // end of data.\n    SendData(Block());\n\n    // Wait for EOS.\n    uint64_t eos_packet{0};\n    while (ReceivePacket(&eos_packet)) {\n        ;\n    }\n\n    if (eos_packet != ServerCodes::EndOfStream && eos_packet != ServerCodes::Exception\n        && eos_packet != ServerCodes::Log && options_.rethrow_exceptions) {\n        throw ProtocolError(std::string{\"unexpected packet from server while receiving end of query, expected (expected Exception, EndOfStream or Log, got: \"}\n                            + (eos_packet ? std::to_string(eos_packet) : \"nothing\") + \")\");\n    }\n}\n\nvoid Client::Impl::Ping() {\n    WireFormat::WriteUInt64(*output_, ClientCodes::Ping);\n    output_->Flush();\n\n    uint64_t server_packet;\n    const bool ret = ReceivePacket(&server_packet);\n\n    if (!ret || server_packet != ServerCodes::Pong) {\n        throw ProtocolError(\"fail to ping server\");\n    }\n}\n\nvoid Client::Impl::ResetConnection() {\n    InitializeStreams(socket_factory_->connect(options_, current_endpoint_.value()));\n\n    if (!Handshake()) {\n        throw ProtocolError(\"fail to connect to \" + options_.host);\n    }\n}\n\nvoid Client::Impl::ResetConnectionEndpoint() {\n    current_endpoint_.reset();\n    for (size_t i = 0; i < options_.endpoints.size();)\n    {\n        try\n        {\n            current_endpoint_ = endpoints_iterator->Next();\n            ResetConnection();\n            return;\n        } catch (const std::system_error&) {\n            if (++i == options_.endpoints.size())\n            {\n                current_endpoint_.reset();\n                throw;\n            }\n        }\n    }\n}\n\nvoid Client::Impl::CreateConnection() {\n    // make sure to try to connect to each endpoint at least once even if `options_.send_retries` is 0\n    const size_t max_attempts = (options_.send_retries ? options_.send_retries : 1);\n    for (size_t i = 0; i < max_attempts;)\n    {\n        try\n        {\n            // Try to connect to each endpoint before throwing exception.\n            ResetConnectionEndpoint();\n            return;\n        } catch (const std::system_error&) {\n            if (++i >= max_attempts)\n            {\n                throw;\n            }\n        }\n    }\n}\n\nconst ServerInfo& Client::Impl::GetServerInfo() const {\n    return server_info_;\n}\n\n\nconst std::optional<Endpoint>& Client::Impl::GetCurrentEndpoint() const {\n    return current_endpoint_;\n}\n\nbool Client::Impl::Handshake() {\n    if (!SendHello()) {\n        return false;\n    }\n    if (!ReceiveHello()) {\n        return false;\n    }\n    return true;\n}\n\nbool Client::Impl::ReceivePacket(uint64_t* server_packet) {\n    uint64_t packet_type = 0;\n\n    if (!WireFormat::ReadVarint64(*input_, &packet_type)) {\n        return false;\n    }\n    if (server_packet) {\n        *server_packet = packet_type;\n    }\n\n    switch (packet_type) {\n    case ServerCodes::Data: {\n        if (!ReceiveData()) {\n            throw ProtocolError(\"can't read data packet from input stream\");\n        }\n        return true;\n    }\n\n    case ServerCodes::Exception: {\n        ReceiveException();\n        return false;\n    }\n\n    case ServerCodes::ProfileInfo: {\n        Profile profile;\n\n        if (!WireFormat::ReadUInt64(*input_, &profile.rows)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(*input_, &profile.blocks)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(*input_, &profile.bytes)) {\n            return false;\n        }\n        if (!WireFormat::ReadFixed(*input_, &profile.applied_limit)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(*input_, &profile.rows_before_limit)) {\n            return false;\n        }\n        if (!WireFormat::ReadFixed(*input_, &profile.calculated_rows_before_limit)) {\n            return false;\n        }\n\n        if (events_) {\n            events_->OnProfile(profile);\n        }\n\n        return true;\n    }\n\n    case ServerCodes::Progress: {\n        Progress info;\n\n        if (!WireFormat::ReadUInt64(*input_, &info.rows)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(*input_, &info.bytes)) {\n            return false;\n        }\n        if constexpr(DMBS_PROTOCOL_REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) {\n            if (!WireFormat::ReadUInt64(*input_, &info.total_rows)) {\n                return false;\n            }\n        }\n        if constexpr (DMBS_PROTOCOL_REVISION >= DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO)\n        {\n            if (!WireFormat::ReadUInt64(*input_, &info.written_rows)) {\n                return false;\n            }\n            if (!WireFormat::ReadUInt64(*input_, &info.written_bytes)) {\n                return false;\n            }\n        }\n\n        if (events_) {\n            events_->OnProgress(info);\n        }\n\n        return true;\n    }\n\n    case ServerCodes::Pong: {\n        return true;\n    }\n\n    case ServerCodes::Hello: {\n        return true;\n    }\n\n    case ServerCodes::EndOfStream: {\n        if (events_) {\n            events_->OnFinish();\n        }\n        return false;\n    }\n\n    case ServerCodes::Log: {\n        // log tag\n        if (!WireFormat::SkipString(*input_)) {\n            return false;\n        }\n        Block block;\n\n        // Use uncompressed stream since log blocks usually contain only one row\n        if (!ReadBlock(*input_, &block)) {\n            return false;\n        }\n\n        if (events_) {\n            events_->OnServerLog(block);\n        }\n        return true;\n    }\n\n    case ServerCodes::TableColumns: {\n        // external table name\n        if (!WireFormat::SkipString(*input_)) {\n            return false;\n        }\n\n        //  columns metadata\n        if (!WireFormat::SkipString(*input_)) {\n            return false;\n        }\n        return true;\n    }\n\n    case ServerCodes::ProfileEvents: {\n        if (!WireFormat::SkipString(*input_)) {\n            return false;\n        }\n\n        Block block;\n        if (!ReadBlock(*input_, &block)) {\n            return false;\n        }\n\n        if (events_) {\n            events_->OnProfileEvents(block);\n        }\n        return true;\n    }\n\n    default:\n        throw UnimplementedError(\"unimplemented \" + std::to_string((int)packet_type));\n        break;\n    }\n}\n\nbool Client::Impl::ReadBlock(InputStream& input, Block* block) {\n    // Additional information about block.\n    if constexpr (DMBS_PROTOCOL_REVISION >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) {\n        uint64_t num;\n        BlockInfo info;\n\n        // BlockInfo\n        if (!WireFormat::ReadUInt64(input, &num)) {\n            return false;\n        }\n        if (!WireFormat::ReadFixed(input, &info.is_overflows)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(input, &num)) {\n            return false;\n        }\n        if (!WireFormat::ReadFixed(input, &info.bucket_num)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(input, &num)) {\n            return false;\n        }\n\n        block->SetInfo(std::move(info));\n    }\n\n    uint64_t num_columns = 0;\n    uint64_t num_rows = 0;\n\n    if (!WireFormat::ReadUInt64(input, &num_columns)) {\n        return false;\n    }\n    if (!WireFormat::ReadUInt64(input, &num_rows)) {\n        return false;\n    }\n\n    CreateColumnByTypeSettings create_column_settings;\n    create_column_settings.low_cardinality_as_wrapped_column = options_.backward_compatibility_lowcardinality_as_wrapped_column;\n\n    for (size_t i = 0; i < num_columns; ++i) {\n        std::string name;\n        std::string type;\n        if (!WireFormat::ReadString(input, &name)) {\n            return false;\n        }\n        if (!WireFormat::ReadString(input, &type)) {\n            return false;\n        }\n\n        if (ColumnRef col = CreateColumnByType(type, create_column_settings)) {\n            if (num_rows && !col->Load(&input, num_rows)) {\n                throw ProtocolError(\"can't load column '\" + name + \"' of type \" + type);\n            }\n\n            block->AppendColumn(name, col);\n        } else {\n            throw UnimplementedError(std::string(\"unsupported column type: \") + type);\n        }\n    }\n\n    return true;\n}\n\nbool Client::Impl::ReceiveData() {\n    Block block;\n\n    if constexpr (DMBS_PROTOCOL_REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) {\n        if (!WireFormat::SkipString(*input_)) {\n            return false;\n        }\n    }\n\n    if (compression_ == CompressionState::Enable) {\n        CompressedInput compressed(input_.get());\n        if (!ReadBlock(compressed, &block)) {\n            return false;\n        }\n    } else {\n        if (!ReadBlock(*input_, &block)) {\n            return false;\n        }\n    }\n\n    if (events_) {\n        events_->OnData(block);\n        if (!events_->OnDataCancelable(block)) {\n            SendCancel();\n        }\n    }\n\n    return true;\n}\n\nbool Client::Impl::ReceiveException(bool rethrow) {\n    std::unique_ptr<Exception> e(new Exception);\n    Exception* current = e.get();\n\n    bool exception_received = true;\n    do {\n        bool has_nested = false;\n\n        if (!WireFormat::ReadFixed(*input_, &current->code)) {\n           exception_received = false;\n           break;\n        }\n        if (!WireFormat::ReadString(*input_, &current->name)) {\n            exception_received = false;\n            break;\n        }\n        if (!WireFormat::ReadString(*input_, &current->display_text)) {\n            exception_received = false;\n            break;\n        }\n        if (!WireFormat::ReadString(*input_, &current->stack_trace)) {\n            exception_received = false;\n            break;\n        }\n        if (!WireFormat::ReadFixed(*input_, &has_nested)) {\n            exception_received = false;\n            break;\n        }\n\n        if (has_nested) {\n            current->nested.reset(new Exception);\n            current = current->nested.get();\n        } else {\n            break;\n        }\n    } while (true);\n\n    if (events_) {\n        events_->OnServerException(*e);\n    }\n\n    if (rethrow || options_.rethrow_exceptions) {\n        throw ServerError(std::move(e));\n    }\n\n    return exception_received;\n}\n\nvoid Client::Impl::SendCancel() {\n    WireFormat::WriteUInt64(*output_, ClientCodes::Cancel);\n    output_->Flush();\n}\n\nvoid Client::Impl::SendQuery(const Query& query) {\n    WireFormat::WriteUInt64(*output_, ClientCodes::Query);\n    WireFormat::WriteString(*output_, query.GetQueryID());\n\n    /// Client info.\n    if (server_info_.revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) {\n        ClientInfo info;\n\n        info.query_kind = 1;\n        info.client_name = \"ClickHouse client\";\n        info.client_version_major = CLICKHOUSE_CPP_VERSION_MAJOR;\n        info.client_version_minor = CLICKHOUSE_CPP_VERSION_MINOR;\n        info.client_version_patch = CLICKHOUSE_CPP_VERSION_PATCH;\n        info.client_revision = DMBS_PROTOCOL_REVISION;\n\n\n        WireFormat::WriteFixed(*output_, info.query_kind);\n        WireFormat::WriteString(*output_, info.initial_user);\n        WireFormat::WriteString(*output_, info.initial_query_id);\n        WireFormat::WriteString(*output_, info.initial_address);\n        if (server_info_.revision >= DBMS_MIN_REVISION_WITH_INITIAL_QUERY_START_TIME) {\n            WireFormat::WriteFixed<int64_t>(*output_, 0);\n        }\n        WireFormat::WriteFixed(*output_, info.iface_type);\n\n        WireFormat::WriteString(*output_, info.os_user);\n        WireFormat::WriteString(*output_, info.client_hostname);\n        WireFormat::WriteString(*output_, info.client_name);\n        WireFormat::WriteUInt64(*output_, info.client_version_major);\n        WireFormat::WriteUInt64(*output_, info.client_version_minor);\n        WireFormat::WriteUInt64(*output_, info.client_revision);\n\n        if (server_info_.revision >= DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO)\n            WireFormat::WriteString(*output_, info.quota_key);\n        if (server_info_.revision >= DBMS_MIN_REVISION_WITH_DISTRIBUTED_DEPTH)\n            WireFormat::WriteUInt64(*output_, 0u);\n        if (server_info_.revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) {\n            WireFormat::WriteUInt64(*output_, info.client_version_patch);\n        }\n\n        if (server_info_.revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) {\n            if (const auto& tracing_context = query.GetTracingContext()) {\n                // Have OpenTelemetry header.\n                WireFormat::WriteFixed(*output_, uint8_t(1));\n                // No point writing these numbers with variable length, because they\n                // are random and will probably require the full length anyway.\n                WireFormat::WriteFixed(*output_, tracing_context->trace_id);\n                WireFormat::WriteFixed(*output_, tracing_context->span_id);\n                WireFormat::WriteString(*output_, tracing_context->tracestate);\n                WireFormat::WriteFixed(*output_, tracing_context->trace_flags);\n            } else {\n                // Don't have OpenTelemetry header.\n                WireFormat::WriteFixed(*output_, uint8_t(0));\n            }\n        } else {\n            if (query.GetTracingContext()) {\n                // Current implementation works only for server version >= v20.11.2.1-stable\n                throw UnimplementedError(std::string(\"Can't send open telemetry tracing context to a server, server version is too old\"));\n            }\n        }\n    }\n\n    /// Per query settings\n    if (server_info_.revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) {\n        for(const auto& [name, field] : query.GetQuerySettings()) {\n            WireFormat::WriteString(*output_, name);\n            WireFormat::WriteVarint64(*output_, field.flags);\n            WireFormat::WriteString(*output_, field.value);\n        }\n    }\n    else if (query.GetQuerySettings().size() > 0) {\n        // Current implementation works only for server version >= v20.1.2.4-stable, since we do not implement binary settings serialization.\n        throw UnimplementedError(std::string(\"Can't send query settings to a server, server version is too old\"));\n    }\n    // Empty string signals end of serialized settings\n    WireFormat::WriteString(*output_, std::string());\n\n    if (server_info_.revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET) {\n        WireFormat::WriteString(*output_, \"\");\n    }\n\n    WireFormat::WriteUInt64(*output_, Stages::Complete);\n    WireFormat::WriteUInt64(*output_, compression_);\n    WireFormat::WriteString(*output_, query.GetText());\n    // Send empty block as marker of\n    // end of data\n    SendData(Block());\n\n    output_->Flush();\n}\n\n\nvoid Client::Impl::WriteBlock(const Block& block, OutputStream& output) {\n    // Additional information about block.\n    if (server_info_.revision >= DBMS_MIN_REVISION_WITH_BLOCK_INFO) {\n        WireFormat::WriteUInt64(output, 1);\n        WireFormat::WriteFixed<uint8_t>(output, block.Info().is_overflows);\n        WireFormat::WriteUInt64(output, 2);\n        WireFormat::WriteFixed<int32_t>(output, block.Info().bucket_num);\n        WireFormat::WriteUInt64(output, 0);\n    }\n\n    WireFormat::WriteUInt64(output, block.GetColumnCount());\n    WireFormat::WriteUInt64(output, block.GetRowCount());\n\n    for (Block::Iterator bi(block); bi.IsValid(); bi.Next()) {\n        WireFormat::WriteString(output, bi.Name());\n        WireFormat::WriteString(output, bi.Type()->GetName());\n\n        // Empty columns are not serialized and occupy exactly 0 bytes.\n        // ref https://github.com/ClickHouse/ClickHouse/blob/39b37a3240f74f4871c8c1679910e065af6bea19/src/Formats/NativeWriter.cpp#L163\n        const bool containsData = block.GetRowCount() > 0;\n        if (containsData) {\n            bi.Column()->Save(&output);\n        }\n    }\n    output.Flush();\n}\n\nvoid Client::Impl::SendData(const Block& block) {\n    WireFormat::WriteUInt64(*output_, ClientCodes::Data);\n\n    if (server_info_.revision >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) {\n        WireFormat::WriteString(*output_, std::string());\n    }\n\n    if (compression_ == CompressionState::Enable) {\n        assert(options_.compression_method == CompressionMethod::LZ4);\n\n        std::unique_ptr<OutputStream> compressed_output = std::make_unique<CompressedOutput>(output_.get(), options_.max_compression_chunk_size);\n        BufferedOutput buffered(std::move(compressed_output), options_.max_compression_chunk_size);\n\n        WriteBlock(block, buffered);\n    } else {\n        WriteBlock(block, *output_);\n    }\n\n    output_->Flush();\n}\n\nvoid Client::Impl::InitializeStreams(std::unique_ptr<SocketBase>&& socket) {\n    std::unique_ptr<OutputStream> output = std::make_unique<BufferedOutput>(socket->makeOutputStream());\n    std::unique_ptr<InputStream> input = std::make_unique<BufferedInput>(socket->makeInputStream());\n\n    std::swap(input, input_);\n    std::swap(output, output_);\n    std::swap(socket, socket_);\n}\n\nbool Client::Impl::SendHello() {\n    WireFormat::WriteUInt64(*output_, ClientCodes::Hello);\n    WireFormat::WriteString(*output_, std::string(DBMS_NAME) + \" client\");\n    WireFormat::WriteUInt64(*output_, CLICKHOUSE_CPP_VERSION_MAJOR);\n    WireFormat::WriteUInt64(*output_, CLICKHOUSE_CPP_VERSION_MINOR);\n    WireFormat::WriteUInt64(*output_, DMBS_PROTOCOL_REVISION);\n    WireFormat::WriteString(*output_, options_.default_database);\n    WireFormat::WriteString(*output_, options_.user);\n    WireFormat::WriteString(*output_, options_.password);\n\n    output_->Flush();\n\n    return true;\n}\n\nbool Client::Impl::ReceiveHello() {\n    uint64_t packet_type = 0;\n\n    if (!WireFormat::ReadVarint64(*input_, &packet_type)) {\n        return false;\n    }\n\n    if (packet_type == ServerCodes::Hello) {\n        if (!WireFormat::ReadString(*input_, &server_info_.name)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(*input_, &server_info_.version_major)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(*input_, &server_info_.version_minor)) {\n            return false;\n        }\n        if (!WireFormat::ReadUInt64(*input_, &server_info_.revision)) {\n            return false;\n        }\n\n        if (server_info_.revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) {\n            if (!WireFormat::ReadString(*input_, &server_info_.timezone)) {\n                return false;\n            }\n        }\n\n        if (server_info_.revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) {\n            if (!WireFormat::ReadString(*input_, &server_info_.display_name)) {\n                return false;\n            }\n        }\n\n...\n", "needle_token_start": 8724, "needle_token_end": 8804, "code_context_ntokens": 15981, "output": ["```cpp\nstd::unique_ptr<EndpointsIteratorBase> GetEndpointsIterator(const ClientOptions& opts) {\n    if (opts.endpoints.empty())\n    {\n        throw ValidationError(\"The list of endpoints is empty\");\n    }\n\n    return std::make_unique<RoundRobinEndpointsIterator>(opts.endpoints);\n}\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "swap", "language": "cpp", "path": "clickhouse/columns/column.h", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to exchange the contents of two instances of a class that represents a column in a database or data structure.\n2. **Input**: Two instances of the column class.\n3. **Output**: There is no return value; however, the internal state of the two column instances are exchanged.\n4. **Procedure**: This function directly swaps the internal data of the two column instances, effectively interchanging their states without the need for a temporary variable to hold the data of one during the swap. This is typically implemented to facilitate operations like reordering or algorithmic shuffling where elements need to be swapped efficiently.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/types/types.h\n#pragma once\n\n#include \"absl/numeric/int128.h\"\n\n#include <atomic>\n#include <map>\n#include <memory>\n#include <string>\n#include <vector>\n#include <stdexcept>\n\nnamespace clickhouse {\n\nusing Int128 = absl::int128;\nusing Int64 = int64_t;\n\nusing TypeRef = std::shared_ptr<class Type>;\n\nclass Type {\npublic:\n    enum Code {\n        Void = 0,\n        Int8,\n        Int16,\n        Int32,\n        Int64,\n        UInt8,\n        UInt16,\n        UInt32,\n        UInt64,\n        Float32,\n        Float64,\n        String,\n        FixedString,\n        DateTime,\n        Date,\n        Array,\n        Nullable,\n        Tuple,\n        Enum8,\n        Enum16,\n        UUID,\n        IPv4,\n        IPv6,\n        Int128,\n        Decimal,\n        Decimal32,\n        Decimal64,\n        Decimal128,\n        LowCardinality,\n        DateTime64,\n        Date32,\n        Map,\n        Point,\n        Ring,\n        Polygon,\n        MultiPolygon\n    };\n\n    using EnumItem = std::pair<std::string /* name */, int16_t /* value */>;\n\nprotected:\n    Type(const Code code);\n\npublic:\n    template <typename Derived>\n    auto* As() {\n        return static_cast<Derived*>(this);\n    }\n\n    template <typename Derived>\n    const auto* As() const {\n        return static_cast<const Derived*>(this);\n    }\n\n    /// Type's code.\n    Code GetCode() const { return code_; }\n\n    /// String representation of the type.\n    std::string GetName() const;\n\n    /// Is given type same as current one.\n    bool IsEqual(const Type& other) const {\n        // Types are equal only if both code_ and type_unique_id_ are equal.\n        return this == &other\n                // GetTypeUniqueId() is relatively heavy, so avoid calling it when comparing obviously different types.\n                || (this->GetCode() == other.GetCode() && this->GetTypeUniqueId() == other.GetTypeUniqueId());\n    }\n\n    bool IsEqual(const TypeRef& other) const { return IsEqual(*other); }\n\n    /// Simple name, doesn't depend on parameters and\\or nested types, caller MUST NOT free returned value.\n    static const char* TypeName(Code);\n\npublic:\n    static TypeRef CreateArray(TypeRef item_type);\n\n    static TypeRef CreateDate();\n\n    static TypeRef CreateDate32();\n\n    static TypeRef CreateDateTime(std::string timezone = std::string());\n\n    static TypeRef CreateDateTime64(size_t precision, std::string timezone = std::string());\n\n    static TypeRef CreateDecimal(size_t precision, size_t scale);\n\n    static TypeRef CreateIPv4();\n\n    static TypeRef CreateIPv6();\n\n    static TypeRef CreateNothing();\n\n    static TypeRef CreateNullable(TypeRef nested_type);\n\n    template <typename T>\n    static TypeRef CreateSimple();\n\n    static TypeRef CreateString();\n\n    static TypeRef CreateString(size_t n);\n\n    static TypeRef CreateTuple(const std::vector<TypeRef>& item_types);\n\n    static TypeRef CreateEnum8(const std::vector<EnumItem>& enum_items);\n\n    static TypeRef CreateEnum16(const std::vector<EnumItem>& enum_items);\n\n    static TypeRef CreateUUID();\n\n    static TypeRef CreateLowCardinality(TypeRef item_type);\n\n    static TypeRef CreateMap(TypeRef key_type, TypeRef value_type);\n\n    static TypeRef CreatePoint();\n\n    static TypeRef CreateRing();\n\n    static TypeRef CreatePolygon();\n\n    static TypeRef CreateMultiPolygon();\n\nprivate:\n    uint64_t GetTypeUniqueId() const;\n\n    const Code code_;\n    mutable std::atomic<uint64_t> type_unique_id_;\n};\n\ninline bool operator==(const Type & left, const Type & right) {\n    if (&left == &right)\n        return true;\n    if (typeid(left) == typeid(right))\n        return left.IsEqual(right);\n    return false;\n}\n\ninline bool operator==(const TypeRef & left, const TypeRef & right) {\n    return *left == *right;\n}\n\nclass ArrayType : public Type {\npublic:\n    explicit ArrayType(TypeRef item_type);\n\n    std::string GetName() const { return std::string(\"Array(\") + item_type_->GetName() + \")\"; }\n\n    /// Type of array's elements.\n    inline TypeRef GetItemType() const { return item_type_; }\n\nprivate:\n    TypeRef item_type_;\n};\n\nclass DecimalType : public Type {\npublic:\n    DecimalType(size_t precision, size_t scale);\n\n    std::string GetName() const;\n    friend class EnumType;\n    friend class DateTimeType;\n\n    inline size_t GetScale() const { return scale_; }\n    inline size_t GetPrecision() const { return precision_; }\n\nprivate:\n    const size_t precision_, scale_;\n};\n\nnamespace details\n{\nclass TypeWithTimeZoneMixin\n{\npublic:\n    TypeWithTimeZoneMixin(std::string timezone);\n\n    /// Timezone associated with a data column.\n    const std::string & Timezone() const;\n\nprivate:\n    std::string timezone_;\n};\n}\n\nclass DateTimeType : public Type, public details::TypeWithTimeZoneMixin {\npublic:\n    explicit DateTimeType(std::string timezone);\n\n    std::string GetName() const;\n};\n\nclass DateTime64Type: public Type, public details::TypeWithTimeZoneMixin {\npublic:\n    explicit DateTime64Type(size_t precision, std::string timezone_);\n\n    std::string GetName() const;\n\n    inline size_t GetPrecision() const { return precision_; }\nprivate:\n    size_t precision_;\n};\n\nclass EnumType : public Type {\npublic:\n    EnumType(Type::Code type, const std::vector<EnumItem>& items);\n\n    std::string GetName() const;\n\n    /// Methods to work with enum types.\n    std::string_view GetEnumName(int16_t value) const;\n    int16_t GetEnumValue(const std::string& name) const;\n    bool HasEnumName(const std::string& name) const;\n    bool HasEnumValue(int16_t value) const;\n\nprivate:\n    using ValueToNameType     = std::map<int16_t, std::string_view>;\n    using NameToValueType     = std::map<std::string, int16_t>;\n    using ValueToNameIterator = ValueToNameType::const_iterator;\n\n    ValueToNameType value_to_name_;\n    NameToValueType name_to_value_;\n\npublic:\n    ValueToNameIterator BeginValueToName() const;\n    ValueToNameIterator EndValueToName() const;\n};\n\nclass FixedStringType : public Type {\npublic:\n    explicit FixedStringType(size_t n);\n\n    std::string GetName() const { return std::string(\"FixedString(\") + std::to_string(size_) + \")\"; }\n\n    inline size_t GetSize() const { return size_; }\n\nprivate:\n    size_t size_;\n};\n\nclass NullableType : public Type {\npublic:\n    explicit NullableType(TypeRef nested_type);\n\n    std::string GetName() const { return std::string(\"Nullable(\") + nested_type_->GetName() + \")\"; }\n\n    /// Type of nested nullable element.\n    TypeRef GetNestedType() const { return nested_type_; }\n\nprivate:\n    TypeRef nested_type_;\n};\n\nclass TupleType : public Type {\npublic:\n    explicit TupleType(const std::vector<TypeRef>& item_types);\n\n    std::string GetName() const;\n\n    /// Type of nested Tuple element type.\n    std::vector<TypeRef> GetTupleType() const { return item_types_; }\n\nprivate:\n    std::vector<TypeRef> item_types_;\n};\n\nclass LowCardinalityType : public Type {\npublic:\n    explicit LowCardinalityType(TypeRef nested_type);\n    ~LowCardinalityType();\n\n    std::string GetName() const { return std::string(\"LowCardinality(\") + nested_type_->GetName() + \")\"; }\n\n    /// Type of nested nullable element.\n    TypeRef GetNestedType() const { return nested_type_; }\n\nprivate:\n    TypeRef nested_type_;\n};\n\nclass MapType : public Type {\npublic:\n    explicit MapType(TypeRef key_type, TypeRef value_type);\n\n    std::string GetName() const;\n\n    /// Type of keys.\n    TypeRef GetKeyType() const { return key_type_; }\n\n    /// Type of values.\n    TypeRef GetValueType() const { return value_type_; }\n\nprivate:\n    TypeRef key_type_;\n    TypeRef value_type_;\n};\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int8_t>() {\n    return TypeRef(new Type(Int8));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int16_t>() {\n    return TypeRef(new Type(Int16));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int32_t>() {\n    return TypeRef(new Type(Int32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int64_t>() {\n    return TypeRef(new Type(Int64));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<Int128>() {\n    return TypeRef(new Type(Int128));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint8_t>() {\n    return TypeRef(new Type(UInt8));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint16_t>() {\n    return TypeRef(new Type(UInt16));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint32_t>() {\n    return TypeRef(new Type(UInt32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint64_t>() {\n    return TypeRef(new Type(UInt64));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<float>() {\n    return TypeRef(new Type(Float32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<double>() {\n    return TypeRef(new Type(Float64));\n}\n\n}  // namespace clickhouse\n\n// Path: clickhouse/server_exception.h\n#pragma once\n\n#include <string>\n#include <memory>\n\nnamespace clickhouse {\nstruct Exception {\n    int code = 0;\n    std::string name;\n    std::string display_text;\n    std::string stack_trace;\n    /// Pointer to nested exception.\n    std::unique_ptr<Exception> nested;\n};\n\n}\n\n// Path: clickhouse/exceptions.h\n#pragma once\n\n#include \"server_exception.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nclass Error : public std::runtime_error {\n    using std::runtime_error::runtime_error;\n};\n\n// Caused by any user-related code, like invalid column types or arguments passed to any method.\nclass ValidationError : public Error {\n    using Error::Error;\n};\n\n// Buffers+IO errors, failure to serialize/deserialize, checksum mismatches, etc.\nclass ProtocolError : public Error {\n    using Error::Error;\n};\n\nclass UnimplementedError : public Error {\n    using Error::Error;\n};\n\n// Internal validation error.\nclass AssertionError : public Error {\n    using Error::Error;\n};\n\nclass OpenSSLError : public Error {\n    using Error::Error;\n};\n\nclass LZ4Error : public Error {\n    using Error::Error;\n};\n\n// Exception received from server.\nclass ServerException : public Error {\npublic:\n    ServerException(std::unique_ptr<Exception> e)\n        : Error(std::string())\n        , exception_(std::move(e))\n    {\n    }\n\n    int GetCode() const {\n        return exception_->code;\n    }\n\n    const Exception& GetException() const {\n        return *exception_;\n    }\n\n    const char* what() const noexcept override {\n        return exception_->display_text.c_str();\n    }\n\nprivate:\n    std::unique_ptr<Exception> exception_;\n};\nusing ServerError = ServerException;\n\n}\n\n// Path: clickhouse/columns/itemview.h\n#pragma once\n\n#include \"../types/types.h\"\n#include \"../exceptions.h\"\n\n#include <string_view>\n#include <stdexcept>\n#include <type_traits>\n\nnamespace clickhouse {\n\n/** ItemView is a view on a data stored in Column, safe-ish interface for reading values from Column.\n *\n * Data is not owned (hence the name View) and will be invalidated on column update, load\n * or destruction (basically on calling any non-const method of Column).\n * `type` reflects what is stored in `data` and can be almost any value-type\n * (except Nullable, Array, Tuple, LowCardinality).\n *\n */\nstruct ItemView {\n    using DataType = std::string_view;\n\n    const Type::Code type;\n    const DataType data;\n\nprivate:\n    template <typename T>\n    inline auto ConvertToStorageValue(const T& t) {\n        if constexpr (std::is_same_v<std::string_view, T> || std::is_same_v<std::string, T>) {\n            return std::string_view{t};\n        } else if constexpr (std::is_fundamental_v<T> || std::is_same_v<Int128, std::decay_t<T>>) {\n            return std::string_view{reinterpret_cast<const char*>(&t), sizeof(T)};\n        } else {\n            static_assert(!std::is_same_v<T, T>, \"Unknown type, which can't be stored in ItemView\");\n            return;\n        }\n    }\n\npublic:\n    ItemView(Type::Code type, DataType data)\n        : type(type),\n          data(data)\n    {\n        ValidateData(type, data);\n    }\n\n    ItemView(Type::Code type, ItemView other)\n        : type(type),\n          data(other.data)\n    {\n        ValidateData(type, data);\n    }\n\n    explicit ItemView()\n        : ItemView(Type::Void, std::string_view{})\n    {}\n\n    template <typename T>\n    explicit ItemView(Type::Code type, const T & value)\n        : ItemView(type, ConvertToStorageValue(value))\n    {}\n\n    template <typename T>\n    auto get() const {\n        using ValueType = std::remove_cv_t<std::decay_t<T>>;\n        if constexpr (std::is_same_v<std::string_view, ValueType> || std::is_same_v<std::string, ValueType>) {\n            return data;\n        } else if constexpr (std::is_fundamental_v<ValueType> || std::is_same_v<Int128, ValueType>) {\n            if (sizeof(ValueType) == data.size()) {\n                return *reinterpret_cast<const T*>(data.data());\n            } else {\n                throw AssertionError(\"Incompatitable value type and size. Requested size: \"\n                        + std::to_string(sizeof(ValueType)) + \" stored size: \" + std::to_string(data.size()));\n            }\n        }\n    }\n\n    inline std::string_view AsBinaryData() const {\n        return data;\n    }\n\n    // Validate that value matches type, will throw an exception if validation fails.\n    static void ValidateData(Type::Code type, DataType data);\n};\n\n}\n\n// Path: clickhouse/columns/column.h\n#pragma once\n\n#include \"../types/types.h\"\n#include \"../columns/itemview.h\"\n#include \"../exceptions.h\"\n\n#include <memory>\n#include <stdexcept>\n\nnamespace clickhouse {\n\nclass InputStream;\nclass OutputStream;\n\nusing ColumnRef = std::shared_ptr<class Column>;\n\n/**\n * An abstract base of all columns classes.\n */\nclass Column : public std::enable_shared_from_this<Column> {\npublic:\n    explicit inline Column(TypeRef type) : type_(type) {}\n\n    virtual ~Column() {}\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<T> As() {\n        return std::dynamic_pointer_cast<T>(shared_from_this());\n    }\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<const T> As() const {\n        return std::dynamic_pointer_cast<const T>(shared_from_this());\n    }\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<T> AsStrict() {\n        auto result = std::dynamic_pointer_cast<T>(shared_from_this());\n        if (!result) {\n            throw ValidationError(\"Can't cast from \" + type_->GetName());\n        }\n        return result;\n    }\n\n    /// Get type object of the column.\n    inline TypeRef Type() const { return type_; }\n    inline const class Type& GetType() const { return *type_; }\n\n    /// Appends content of given column to the end of current one.\n    virtual void Append(ColumnRef column) = 0;\n\n    /// Increase the capacity of the column for large block insertion.\n    virtual void Reserve(size_t new_cap) = 0;\n\n    /// Template method to load column data from input stream. It'll call LoadPrefix and LoadBody.\n    /// Should be called only once from the client. Derived classes should not call it.\n    bool Load(InputStream* input, size_t rows);\n\n    /// Loads column prefix from input stream.\n    virtual bool LoadPrefix(InputStream* input, size_t rows);\n\n    /// Loads column data from input stream.\n    virtual bool LoadBody(InputStream* input, size_t rows) = 0;\n\n    /// Saves column prefix to output stream. Column types with prefixes must implement it.\n    virtual void SavePrefix(OutputStream* output);\n\n    /// Saves column body to output stream.\n    virtual void SaveBody(OutputStream* output) = 0;\n\n    /// Template method to save to output stream. It'll call SavePrefix and SaveBody respectively\n    /// Should be called only once from the client. Derived classes should not call it.\n    /// Save is split in Prefix and Body because some data types require prefixes and specific serialization order.\n    /// For instance, Array(LowCardinality(X)) requires LowCardinality.key_version bytes to come before Array.offsets\n    void Save(OutputStream* output);\n\n    /// Clear column data .\n    virtual void Clear() = 0;\n\n    /// Returns count of rows in the column.\n    virtual size_t Size() const = 0;\n\n    /// Makes slice of the current column.\n    virtual ColumnRef Slice(size_t begin, size_t len) const = 0;\n\n    virtual ColumnRef CloneEmpty() const = 0;\n\n    virtual void Swap(Column&) = 0;\n\n    /// Get a view on raw item data if it is supported by column, will throw an exception if index is out of range.\n    /// Please note that view is invalidated once column items are added or deleted, column is loaded from strean or destroyed.\n    virtual ItemView GetItem(size_t) const {\n        throw UnimplementedError(\"GetItem() is not supported for column of \" + type_->GetName());\n    }\n\n    friend \nvoid swap(Column& left, Column& right) {\n        left.Swap(right);\n    }\n\nprotected:\n    TypeRef type_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/block.h\n#pragma once\n\n#include \"columns/column.h\"\n\nnamespace clickhouse {\n\nstruct BlockInfo {\n    uint8_t is_overflows = 0;\n    int32_t bucket_num = -1;\n};\n\nclass Block {\npublic:\n    /// Allow to iterate over block's columns.\n    class Iterator {\n    public:\n        Iterator(const Block& block);\n\n        /// Name of column.\n        const std::string& Name() const;\n\n        /// Type of column.\n        TypeRef Type() const;\n\n        /// Reference to column object.\n        ColumnRef Column() const;\n\n        /// Move to next column, returns false if next call to IsValid() would return false;\n        bool Next();\n\n        /// Is the iterator still valid.\n        bool IsValid() const;\n\n        size_t ColumnIndex() const {\n            return idx_;\n        }\n\n        Iterator& operator*() { return *this; }\n        const Iterator& operator*() const { return *this; }\n\n        bool operator==(const Iterator & other) const {\n            return &block_ == &other.block_ && idx_ == other.idx_;\n        }\n        bool operator!=(const Iterator & other) const {\n            return !(*this == other);\n        }\n\n        Iterator& operator++() {\n            this->Next();\n            return *this;\n        }\n\n    private:\n        friend class Block;\n        struct ConstructAtEndTag {};\n        Iterator(const Block& block, ConstructAtEndTag at_end);\n        Iterator() = delete;\n\n        const Block& block_;\n        size_t idx_;\n    };\n\npublic:\n     Block();\n     Block(size_t cols, size_t rows);\n    ~Block();\n\n    /// Append named column to the block.\n    void AppendColumn(const std::string& name, const ColumnRef& col);\n\n    /// Count of columns in the block.\n    size_t GetColumnCount() const;\n\n    const BlockInfo& Info() const;\n\n    /// Set block info\n    void SetInfo(BlockInfo info);\n\n    /// Count of rows in the block.\n    size_t GetRowCount() const;\n\n    size_t RefreshRowCount();\n\n    const std::string& GetColumnName(size_t idx) const {\n        return columns_.at(idx).name;\n    }\n\n    /// Reference to column by index in the block.\n    ColumnRef operator [] (size_t idx) const;\n\n    Iterator begin() const;\n    Iterator end() const;\n    Iterator cbegin() const { return begin(); }\n    Iterator cend() const { return end(); }\n\nprivate:\n    struct ColumnItem {\n        std::string name;\n        ColumnRef   column;\n    };\n\n    BlockInfo info_;\n    std::vector<ColumnItem> columns_;\n    /// Count of rows in the block.\n    size_t rows_;\n};\n\n}\n\n// Path: clickhouse/block.cpp\n#include \"block.h\"\n\n#include \"exceptions.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nBlock::Iterator::Iterator(const Block& block)\n    : block_(block)\n    , idx_(0)\n{\n}\n\nBlock::Iterator::Iterator(const Block& block, Block::Iterator::ConstructAtEndTag /*at_end*/)\n    : block_(block)\n    , idx_(block.GetColumnCount())\n{}\n\nconst std::string& Block::Iterator::Name() const {\n    return block_.columns_[idx_].name;\n}\n\nTypeRef Block::Iterator::Type() const {\n    return block_.columns_[idx_].column->Type();\n}\n\nColumnRef Block::Iterator::Column() const {\n    return block_.columns_[idx_].column;\n}\n\nbool Block::Iterator::Next() {\n    ++idx_;\n    return IsValid();\n}\n\nbool Block::Iterator::IsValid() const {\n    return idx_ < block_.columns_.size();\n}\n\n\nBlock::Block()\n    : rows_(0)\n{\n}\n\nBlock::Block(size_t cols, size_t rows)\n    : rows_(rows)\n{\n    columns_.reserve(cols);\n}\n\nBlock::~Block() = default;\n\nvoid Block::AppendColumn(const std::string& name, const ColumnRef& col) {\n    if (columns_.empty()) {\n        rows_ = col->Size();\n    } else if (col->Size() != rows_) {\n        throw ValidationError(\"all columns in block must have same count of rows. Name: [\"+name+\"], rows: [\"+std::to_string(rows_)+\"], columns: [\" + std::to_string(col->Size())+\"]\");\n    }\n\n    columns_.push_back(ColumnItem{name, col});\n}\n\n/// Count of columns in the block.\nsize_t Block::GetColumnCount() const {\n    return columns_.size();\n}\n\nconst BlockInfo& Block::Info() const {\n    return info_;\n}\n\n/// Set block info\nvoid Block::SetInfo(BlockInfo info) {\n    info_ = std::move(info);\n}\n\n/// Count of rows in the block.\nsize_t Block::GetRowCount() const {\n    return rows_;\n}\n\nsize_t Block::RefreshRowCount()\n{\n    size_t rows = 0UL;\n\n    for (size_t idx = 0UL; idx < columns_.size(); ++idx)\n    {\n       const std::string& name = columns_[idx].name;\n       const ColumnRef& col = columns_[idx].column;\n\n       if (idx == 0UL)\n           rows = col->Size();\n       else if (rows != col->Size())\n           throw ValidationError(\"all columns in block must have same count of rows. Name: [\"+name+\"], rows: [\"+std::to_string(rows)+\"], columns: [\" + std::to_string(col->Size())+\"]\");\n    }\n\n    rows_ = rows;\n    return rows_;\n}\n\nColumnRef Block::operator [] (size_t idx) const {\n    if (idx < columns_.size()) {\n        return columns_[idx].column;\n    }\n\n    throw std::out_of_range(\"column index is out of range. Index: [\"+std::to_string(idx)+\"], columns: [\" + std::to_string(columns_.size())+\"]\");\n}\n\nBlock::Iterator Block::begin() const {\n    return Iterator(*this);\n}\n\nBlock::Iterator Block::end() const {\n    return Iterator(*this, Iterator::ConstructAtEndTag{});\n}\n\n}\n\n// Path: clickhouse/base/uuid.h\n#pragma once\n\n#include <cstdint>\n#include <utility>\n\nnamespace clickhouse {\n\nusing UInt128 = std::pair<uint64_t, uint64_t>;\n\nusing UUID = UInt128;\n\n}\n\n// Path: clickhouse/base/open_telemetry.h\n#pragma once\n\n#include \"uuid.h\"\n\n#include <string>\n\nnamespace clickhouse::open_telemetry {\n\n/// See https://www.w3.org/TR/trace-context/ for trace_flags definition\nenum TraceFlags : uint8_t {\n    TRACE_FLAG_NONE    = 0,\n    TRACE_FLAG_SAMPLED = 1,\n};\n\n/// The runtime info we need to create new OpenTelemetry spans.\nstruct TracingContext {\n    UUID trace_id{};\n    uint64_t span_id = 0;\n    std::string tracestate;\n    uint8_t trace_flags = TRACE_FLAG_NONE;\n};\n\n}  // namespace clickhouse::open_telemetry\n\n// Path: clickhouse/query.h\n#pragma once\n\n#include \"block.h\"\n#include \"server_exception.h\"\n\n#include \"base/open_telemetry.h\"\n\n#include <cstdint>\n#include <functional>\n#include <memory>\n#include <optional>\n#include <string>\n#include <unordered_map>\n\nnamespace clickhouse {\n\nstruct QuerySettingsField {\n    enum Flags : uint64_t\n    {\n        IMPORTANT = 0x01,\n        CUSTOM = 0x02,\n        OBSOLETE = 0x04,\n    };\n    std::string value;\n    uint64_t flags{0};\n};\n\nusing QuerySettings = std::unordered_map<std::string, QuerySettingsField>;\n\nstruct Profile {\n    uint64_t rows = 0;\n    uint64_t blocks = 0;\n    uint64_t bytes = 0;\n    uint64_t rows_before_limit = 0;\n    bool applied_limit = false;\n    bool calculated_rows_before_limit = false;\n};\n\n\nstruct Progress {\n    uint64_t rows = 0;\n    uint64_t bytes = 0;\n    uint64_t total_rows = 0;\n    uint64_t written_rows = 0;\n    uint64_t written_bytes = 0;\n};\n\n\nclass QueryEvents {\npublic:\n    virtual ~QueryEvents()\n    { }\n\n    /// Some data was received.\n    virtual void OnData(const Block& block) = 0;\n    virtual bool OnDataCancelable(const Block& block) = 0;\n\n    virtual void OnServerException(const Exception& e) = 0;\n\n    virtual void OnProfile(const Profile& profile) = 0;\n\n    virtual void OnProgress(const Progress& progress) = 0;\n\n    /** Handle query execution logs provided by server.\n     *  Amount of logs regulated by `send_logs_level` setting.\n     *  By-default only `fatal` log events are sent to the client side.\n     */\n    virtual void OnServerLog(const Block& block) = 0;\n\n    /// Handle query execution profile events.\n    virtual void OnProfileEvents(const Block& block) = 0;\n\n    virtual void OnFinish() = 0;\n};\n\n\nusing ExceptionCallback        = std::function<void(const Exception& e)>;\nusing ProgressCallback         = std::function<void(const Progress& progress)>;\nusing SelectCallback           = std::function<void(const Block& block)>;\nusing SelectCancelableCallback = std::function<bool(const Block& block)>;\nusing SelectServerLogCallback  = std::function<bool(const Block& block)>;\nusing ProfileEventsCallback    = std::function<bool(const Block& block)>;\nusing ProfileCallbak           = std::function<void(const Profile& profile)>;\n\n\nclass Query : public QueryEvents {\npublic:\n     Query();\n     Query(const char* query, const char* query_id = nullptr);\n     Query(const std::string& query, const std::string& query_id = default_query_id);\n    ~Query() override;\n\n    ///\n    inline const std::string& GetText() const {\n        return query_;\n    }\n\n    inline const std::string& GetQueryID() const {\n        return query_id_;\n    }\n\n    inline const QuerySettings& GetQuerySettings() const {\n        return query_settings_;\n    }\n\n    /// Set per query settings\n    inline Query& SetQuerySettings(QuerySettings query_settings) {\n        query_settings_ = std::move(query_settings);\n        return *this;\n    }\n\n    /// Set per query setting\n    inline Query& SetSetting(const std::string& key, const QuerySettingsField& value) {\n        query_settings_[key] = value;\n        return *this;\n    }\n\n    inline const std::optional<open_telemetry::TracingContext>& GetTracingContext() const {\n        return tracing_context_;\n    }\n\n    /// Set tracing context for open telemetry signals\n    inline Query& SetTracingContext(open_telemetry::TracingContext tracing_context) {\n        tracing_context_ = std::move(tracing_context);\n        return *this;\n    }\n\n    /// Set handler for receiving result data.\n    inline Query& OnData(SelectCallback cb) {\n        select_cb_ = std::move(cb);\n        return *this;\n    }\n\n    inline Query& OnDataCancelable(SelectCancelableCallback cb) {\n        select_cancelable_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving server's exception.\n    inline Query& OnException(ExceptionCallback cb) {\n        exception_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving a progress of query execution.\n    inline Query& OnProgress(ProgressCallback cb) {\n        progress_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving a server log of query exceution.\n    inline Query& OnServerLog(SelectServerLogCallback cb) {\n        select_server_log_cb_ = std::move(cb);\n        return *this;\n    }\n\n    /// Set handler for receiving profile events.\n    inline Query& OnProfileEvents(ProfileEventsCallback cb) {\n        profile_events_callback_cb_ = std::move(cb);\n        return *this;\n    }\n\n    inline Query& OnProfile(ProfileCallbak cb) {\n        profile_callback_cb_ = std::move(cb);\n        return *this;\n    }\n\n    static const std::string default_query_id;\n\nprivate:\n    void OnData(const Block& block) override {\n        if (select_cb_) {\n            select_cb_(block);\n        }\n    }\n\n    bool OnDataCancelable(const Block& block) override {\n        if (select_cancelable_cb_) {\n            return select_cancelable_cb_(block);\n        } else {\n            return true;\n        }\n    }\n\n    void OnServerException(const Exception& e) override {\n        if (exception_cb_) {\n            exception_cb_(e);\n        }\n    }\n\n    void OnProfile(const Profile& profile) override {\n        if (profile_callback_cb_)\n            profile_callback_cb_(profile);\n    }\n\n    void OnProgress(const Progress& progress) override {\n        if (progress_cb_) {\n            progress_cb_(progress);\n        }\n    }\n\n    void OnServerLog(const Block& block) override {\n        if (select_server_log_cb_) {\n            select_server_log_cb_(block);\n        }\n    }\n\n    void OnProfileEvents(const Block& block) override {\n        if (profile_events_callback_cb_) {\n            profile_events_callback_cb_(block);\n        }\n    }\n\n    void OnFinish() override {\n    }\n\nprivate:\n    const std::string query_;\n    const std::string query_id_;\n    std::optional<open_telemetry::TracingContext> tracing_context_;\n    QuerySettings query_settings_;\n    ExceptionCallback exception_cb_;\n    ProgressCallback progress_cb_;\n    SelectCallback select_cb_;\n    SelectCancelableCallback select_cancelable_cb_;\n    SelectServerLogCallback select_server_log_cb_;\n    ProfileEventsCallback profile_events_callback_cb_;\n    ProfileCallbak profile_callback_cb_;\n};\n\n}\n\n// Path: clickhouse/columns/numeric.h\n#pragma once\n\n#include \"column.h\"\n#include \"absl/numeric/int128.h\"\n\nnamespace clickhouse {\n\n/**\n * Represents various numeric columns.\n */\ntemplate <typename T>\nclass ColumnVector : public Column {\npublic:\n    using DataType = T;\n    using ValueType = T;\n\n    ColumnVector();\n\n    explicit ColumnVector(const std::vector<T>& data);\n    explicit ColumnVector(std::vector<T> && data);\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends one element to the end of column.\n    void Append(const T& value);\n\n    /// Returns element at given row number.\n    const T& At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline const T& operator [] (size_t n) const { return At(n); }\n\n    void Erase(size_t pos, size_t count = 1);\n\n    /// Get Raw Vector Contents\n    std::vector<T>& GetWritableData();\n\n    /// Returns the capacity of the column\n    size_t Capacity() const;\n\npublic:\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\n    ItemView GetItem(size_t index) const override;\n\nprivate:\n    std::vector<T> data_;\n};\n\nusing Int128 = absl::int128;\nusing Int64 = int64_t;\n\nusing ColumnUInt8   = ColumnVector<uint8_t>;\nusing ColumnUInt16  = ColumnVector<uint16_t>;\nusing ColumnUInt32  = ColumnVector<uint32_t>;\nusing ColumnUInt64  = ColumnVector<uint64_t>;\n\nusing ColumnInt8    = ColumnVector<int8_t>;\nusing ColumnInt16   = ColumnVector<int16_t>;\nusing ColumnInt32   = ColumnVector<int32_t>;\nusing ColumnInt64   = ColumnVector<int64_t>;\nusing ColumnInt128  = ColumnVector<Int128>;\n\nusing ColumnFloat32 = ColumnVector<float>;\nusing ColumnFloat64 = ColumnVector<double>;\n\n}\n\n// Path: clickhouse/columns/utils.h\n#pragma once\n\n#include <algorithm>\n#include <vector>\n#include <memory>\n\nnamespace clickhouse {\n\ntemplate <typename T>\nstd::vector<T> SliceVector(const std::vector<T>& vec, size_t begin, size_t len) {\n    std::vector<T> result;\n\n    if (begin < vec.size()) {\n        len = std::min(len, vec.size() - begin);\n        result.assign(vec.begin() + begin, vec.begin() + (begin + len));\n    }\n\n    return result;\n}\n\ntemplate <typename T>\nstruct HasWrapMethod {\nprivate:\n    static int detect(...);\n    template <typename U>\n    static decltype(U::Wrap(std::move(std::declval<ColumnRef>()))) detect(const U&);\n\npublic:\n    static constexpr bool value = !std::is_same<int, decltype(detect(std::declval<T>()))>::value;\n};\n\ntemplate <typename T>\ninline std::shared_ptr<T> WrapColumn(ColumnRef&& column) {\n    if constexpr (HasWrapMethod<T>::value) {\n        return T::Wrap(std::move(column));\n    } else {\n        return column->template AsStrict<T>();\n    }\n}\n\n}\n\n// Path: clickhouse/columns/array.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n#include \"utils.h\"\n\n#include <memory>\n\nnamespace clickhouse {\n\ntemplate <typename NestedColumnType>\nclass ColumnArrayT;\n\n/**\n * Represents column of Array(T).\n */\nclass ColumnArray : public Column {\npublic:\n    using ValueType = ColumnRef;\n\n    /** Create an array of given type.\n     *\n     *  `data` is used internally (and modified) by ColumnArray.\n     *  Users are strongly advised against supplying non-empty columns and/or modifying\n     *  contents of `data` afterwards.\n     */\n    explicit ColumnArray(ColumnRef data);\n\n    /** Create an array of given type, with actual values and offsets.\n     *\n     *  Both `data` and `offsets` are used (and modified) internally bye ColumnArray.\n     *  Users are strongly advised against modifying contents of `data` or `offsets` afterwards.\n     */\n    ColumnArray(ColumnRef data, std::shared_ptr<ColumnUInt64> offsets);\n\n    /// Converts input column to array and appends as one row to the current column.\n    void AppendAsColumn(ColumnRef array);\n\n    /// Converts array at pos n to column.\n    /// Type of element of result column same as type of array element.\n    ColumnRef GetAsColumn(size_t n) const;\n\n    /// Shorthand to get a column casted to a proper type.\n    template <typename T>\n    auto GetAsColumnTyped(size_t n) const {\n        return GetAsColumn(n)->AsStrict<T>();\n    }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column prefix from input stream.\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t, size_t) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column&) override;\n\n    void OffsetsIncrease(size_t);\n\n...\n", "needle_token_start": 5049, "needle_token_end": 5073, "code_context_ntokens": 10613, "output": ["```cpp\nvoid Column::Swap(Column& other) override {\n    std::swap(data_, other.data_);\n}\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "CreateColumnByType", "language": "cpp", "path": "clickhouse/columns/factory.cpp", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a column object based on a specified data type description, handling various data structures and configurations.\n2. **Input**: It takes a string representing the type of the column and a settings object that configures specific behaviors during the column creation.\n3. **Output**: Returns a reference to the newly created column object, or a null reference if the creation fails or the type is unsupported.\n4. **Procedure**: \n   - The function begins by parsing the input string to determine the structure and type of the column needed.\n   - Depending on the parsed type, it constructs the appropriate column object. This includes handling complex types like enumerations, maps, and wrapped columns with specific settings.\n   - For each type, it may recursively create nested columns or apply specific settings that affect how the column behaves or is represented.\n   - If the type is unsupported or an error occurs during parsing or creation, it returns a null reference.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/client.cpp\n#include \"client.h\"\n#include \"clickhouse/version.h\"\n#include \"protocol.h\"\n\n#include \"base/compressed.h\"\n#include \"base/socket.h\"\n#include \"base/wire_format.h\"\n\n#include \"columns/factory.h\"\n\n#include <assert.h>\n#include <system_error>\n#include <vector>\n#include <sstream>\n\n#if defined(WITH_OPENSSL)\n#include \"base/sslsocket.h\"\n#endif\n\n#define DBMS_NAME                                       \"ClickHouse\"\n\n#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES         50264\n#define DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS   51554\n#define DBMS_MIN_REVISION_WITH_BLOCK_INFO               51903\n#define DBMS_MIN_REVISION_WITH_CLIENT_INFO              54032\n#define DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE          54058\n#define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060\n//#define DBMS_MIN_REVISION_WITH_TABLES_STATUS            54226\n#define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337\n#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME      54372\n#define DBMS_MIN_REVISION_WITH_VERSION_PATCH            54401\n#define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE     54405\n#define DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA 54410\n#define DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO        54420\n#define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429\n#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET       54441\n#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY            54442\n#define DBMS_MIN_REVISION_WITH_DISTRIBUTED_DEPTH        54448\n#define DBMS_MIN_REVISION_WITH_INITIAL_QUERY_START_TIME 54449\n#define DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS 54451\n\n#define DMBS_PROTOCOL_REVISION  DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS\n\nnamespace clickhouse {\n\nstruct ClientInfo {\n    uint8_t iface_type = 1; // TCP\n    uint8_t query_kind;\n    std::string initial_user;\n    std::string initial_query_id;\n    std::string quota_key;\n    std::string os_user;\n    std::string client_hostname;\n    std::string client_name;\n    std::string initial_address = \"[::ffff:127.0.0.1]:0\";\n    uint64_t client_version_major = 0;\n    uint64_t client_version_minor = 0;\n    uint64_t client_version_patch = 0;\n    uint32_t client_revision = 0;\n};\n\nstd::ostream& operator<<(std::ostream& os, const Endpoint& endpoint) {\n    return os << endpoint.host << \":\" << endpoint.port;\n}\n\nstd::ostream& operator<<(std::ostream& os, const ClientOptions& opt) {\n    os << \"Client(\"\n       << \" Endpoints : [\";\n    size_t extra_endpoints = 0;\n\n    if (!opt.host.empty()) {\n        extra_endpoints = 1;\n        os << opt.user << '@' << Endpoint{opt.host, opt.port};\n\n        if (opt.endpoints.size())\n            os << \", \";\n    }\n\n    for (size_t i = 0; i < opt.endpoints.size(); i++) {\n        os << opt.user << '@' << opt.endpoints[i]\n           << ((i == opt.endpoints.size() - 1) ? \"\" : \", \");\n    }\n\n    os << \"] (\" << opt.endpoints.size() + extra_endpoints << \" items )\"\n       << \" ping_before_query:\" << opt.ping_before_query\n       << \" send_retries:\" << opt.send_retries\n       << \" retry_timeout:\" << opt.retry_timeout.count()\n       << \" compression_method:\"\n       << (opt.compression_method == CompressionMethod::LZ4 ? \"LZ4\" : \"None\");\n#if defined(WITH_OPENSSL)\n    if (opt.ssl_options) {\n        const auto & ssl_options = *opt.ssl_options;\n        os << \" SSL (\"\n           << \" ssl_context: \" << (ssl_options.ssl_context ? \"provided by user\" : \"created internally\")\n           << \" use_default_ca_locations: \" << ssl_options.use_default_ca_locations\n           << \" path_to_ca_files: \" << ssl_options.path_to_ca_files.size() << \" items\"\n           << \" path_to_ca_directory: \" << ssl_options.path_to_ca_directory\n           << \" min_protocol_version: \" << ssl_options.min_protocol_version\n           << \" max_protocol_version: \" << ssl_options.max_protocol_version\n           << \" context_options: \" << ssl_options.context_options\n           << \")\";\n    }\n#endif\n    os << \")\";\n    return os;\n}\n\nClientOptions& ClientOptions::SetSSLOptions(ClientOptions::SSLOptions options)\n{\n#ifdef WITH_OPENSSL\n    ssl_options = options;\n    return *this;\n#else\n    (void)options;\n    throw OpenSSLError(\"Library was built with no SSL support\");\n#endif\n}\n\nnamespace {\n\nstd::unique_ptr<SocketFactory> GetSocketFactory(const ClientOptions& opts) {\n    (void)opts;\n#if defined(WITH_OPENSSL)\n    if (opts.ssl_options)\n        return std::make_unique<SSLSocketFactory>(opts);\n    else\n#endif\n        return std::make_unique<NonSecureSocketFactory>();\n}\n\nstd::unique_ptr<EndpointsIteratorBase> GetEndpointsIterator(const ClientOptions& opts) {\n    if (opts.endpoints.empty())\n    {\n        throw ValidationError(\"The list of endpoints is empty\");\n    }\n\n    return std::make_unique<RoundRobinEndpointsIterator>(opts.endpoints);\n}\n\n}\n\nclass Client::Impl {\npublic:\n     Impl(const ClientOptions& opts);\n     Impl(const ClientOptions& opts,\n          std::unique_ptr<SocketFactory> socket_factory);\n    ~Impl();\n\n    void ExecuteQuery(Query query);\n\n    void SendCancel();\n\n    void Insert(const std::string& table_name, const std::string& query_id, const Block& block);\n\n    void Ping();\n\n    void ResetConnection();\n\n    void ResetConnectionEndpoint();\n\n    const ServerInfo& GetServerInfo() const;\n\n    const std::optional<Endpoint>& GetCurrentEndpoint() const;\n\nprivate:\n    bool Handshake();\n\n    bool ReceivePacket(uint64_t* server_packet = nullptr);\n\n    void SendQuery(const Query& query);\n\n    void SendData(const Block& block);\n\n    bool SendHello();\n\n    bool ReadBlock(InputStream& input, Block* block);\n\n    bool ReceiveHello();\n\n    /// Reads data packet form input stream.\n    bool ReceiveData();\n\n    /// Reads exception packet form input stream.\n...\n// Path: clickhouse/query.cpp\n#include \"query.h\"\n\nnamespace clickhouse {\n\nconst std::string Query::default_query_id = {};\n\nQuery::Query()\n{ }\n\nQuery::Query(const char* query, const char* query_id)\n    : query_(query)\n    , query_id_(query_id ? std::string(query_id): default_query_id)\n{\n}\n\nQuery::Query(const std::string& query, const std::string& query_id)\n    : query_(query)\n    , query_id_(query_id)\n{\n}\n\nQuery::~Query()\n{ }\n\n}\n\n// Path: clickhouse/columns/map.cpp\n#include \"map.h\"\n\n#include <stdexcept>\n\n#include \"../exceptions.h\"\n#include \"utils.h\"\n\nnamespace {\n\nusing namespace clickhouse;\n\nTypeRef GetMapType(const Type& data_type) {\n    auto array = data_type.As<ArrayType>();\n    if (!array) {\n        throw ValidationError(\"Wrong type  \" + data_type.GetName() + \" of data for map\");\n    }\n    auto tuple = array->GetItemType()->As<TupleType>();\n    if (!tuple) {\n        throw ValidationError(\"Wrong type  \" + data_type.GetName() + \" of data for map\");\n    }\n    auto types = tuple->GetTupleType();\n    if (types.size() != 2) {\n        throw ValidationError(\"Wrong type  \" + data_type.GetName() + \" of data for map\");\n    }\n    return Type::CreateMap(types[0], types[1]);\n}\n\n}  // namespace\n\nnamespace clickhouse {\n\nColumnMap::ColumnMap(ColumnRef data)\n    : Column(GetMapType(data->GetType())), data_(data->As<ColumnArray>()) {\n}\n\nvoid ColumnMap::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\nvoid ColumnMap::Clear() {\n    data_->Clear();\n}\n\nvoid ColumnMap::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnMap>()) {\n        data_->Append(col->data_);\n    }\n}\n\nbool ColumnMap::LoadPrefix(InputStream* input, size_t rows) {\n    return data_->LoadPrefix(input, rows);\n}\n\nbool ColumnMap::LoadBody(InputStream* input, size_t rows) {\n    return data_->LoadBody(input, rows);\n}\n\nvoid ColumnMap::SavePrefix(OutputStream* output) {\n    data_->SavePrefix(output);\n}\n\nvoid ColumnMap::SaveBody(OutputStream* output) {\n    data_->SaveBody(output);\n}\n\nsize_t ColumnMap::Size() const {\n    return data_->Size();\n}\n\nColumnRef ColumnMap::Slice(size_t begin, size_t len) const {\n    return std::make_shared<ColumnMap>(data_->Slice(begin, len));\n}\n\nColumnRef ColumnMap::CloneEmpty() const {\n    return std::make_shared<ColumnMap>(data_->CloneEmpty());\n}\n\nvoid ColumnMap::Swap(Column& other) {\n    auto& col = dynamic_cast<ColumnMap&>(other);\n    data_.swap(col.data_);\n}\n\nColumnRef ColumnMap::GetAsColumn(size_t n) const {\n    return data_->GetAsColumn(n);\n}\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/array.cpp\n#include \"array.h\"\n#include \"numeric.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nColumnArray::ColumnArray(ColumnRef data)\n    : ColumnArray(data, std::make_shared<ColumnUInt64>())\n{\n}\n\nColumnArray::ColumnArray(ColumnRef data, std::shared_ptr<ColumnUInt64> offsets)\n    : Column(Type::CreateArray(data->Type()))\n    , data_(data)\n    , offsets_(offsets)\n{\n}\n\nColumnArray::ColumnArray(ColumnArray&& other)\n    : Column(other.Type())\n    , data_(std::move(other.data_))\n    , offsets_(std::move(other.offsets_))\n{\n}\n\nvoid ColumnArray::AppendAsColumn(ColumnRef array) {\n    // appending data may throw (i.e. due to ype check failure), so do it first to avoid partly modified state.\n    data_->Append(array);\n    AddOffset(array->Size());\n}\n\nColumnRef ColumnArray::GetAsColumn(size_t n) const {\n    if (n >= Size())\n        throw ValidationError(\"Index is out ouf bounds: \" + std::to_string(n));\n\n    return data_->Slice(GetOffset(n), GetSize(n));\n}\n\nColumnRef ColumnArray::Slice(size_t begin, size_t size) const {\n    if (size && begin + size > Size())\n        throw ValidationError(\"Slice indexes are out of bounds\");\n\n    auto result = std::make_shared<ColumnArray>(data_->Slice(GetOffset(begin), GetOffset(begin + size) - GetOffset(begin)));\n    for (size_t i = 0; i < size; i++)\n        result->AddOffset(GetSize(begin + i));\n\n    return result;\n}\n\nColumnRef ColumnArray::CloneEmpty() const {\n    return std::make_shared<ColumnArray>(data_->CloneEmpty());\n}\n\nvoid ColumnArray::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n    offsets_->Reserve(new_cap);\n}\n\nvoid ColumnArray::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnArray>()) {\n        for (size_t i = 0; i < col->Size(); ++i) {\n            AppendAsColumn(col->GetAsColumn(i));\n        }\n    }\n}\n\nbool ColumnArray::LoadPrefix(InputStream* input, size_t rows) {\n    if (!rows) {\n        return true;\n    }\n\n    return data_->LoadPrefix(input, rows);\n}\n\nbool ColumnArray::LoadBody(InputStream* input, size_t rows) {\n    if (!rows) {\n        return true;\n    }\n    if (!offsets_->LoadBody(input, rows)) {\n        return false;\n    }\n    const auto nested_rows = (*offsets_)[rows - 1];\n    if (nested_rows == 0) {\n        return true;\n    }\n    if (!data_->LoadBody(input, nested_rows)) {\n        return false;\n    }\n    return true;\n}\n\nvoid ColumnArray::SavePrefix(OutputStream* output) {\n    data_->SavePrefix(output);\n}\n\nvoid ColumnArray::SaveBody(OutputStream* output) {\n    offsets_->SaveBody(output);\n    if (data_->Size() > 0) {\n        data_->SaveBody(output);\n    }\n}\n\nvoid ColumnArray::Clear() {\n    offsets_->Clear();\n    data_->Clear();\n}\n\nsize_t ColumnArray::Size() const {\n    return offsets_->Size();\n}\n\nvoid ColumnArray::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnArray &>(other);\n    data_.swap(col.data_);\n    offsets_.swap(col.offsets_);\n}\n\nvoid ColumnArray::OffsetsIncrease(size_t n) {\n    offsets_->Append(n);\n}\n\nsize_t ColumnArray::GetOffset(size_t n) const {\n\n    return (n == 0) ? 0 : (*offsets_)[n - 1];\n}\n\nvoid ColumnArray::AddOffset(size_t n) {\n    if (offsets_->Size() == 0) {\n        offsets_->Append(n);\n    } else {\n        offsets_->Append((*offsets_)[offsets_->Size() - 1] + n);\n    }\n}\n\nsize_t ColumnArray::GetSize(size_t n) const {\n    return (n == 0) ? (*offsets_)[n] : ((*offsets_)[n] - (*offsets_)[n - 1]);\n}\n\nColumnRef ColumnArray::GetData() {\n    return data_;\n}\n\nvoid ColumnArray::Reset() {\n    data_.reset();\n    offsets_.reset();\n}\n\n}\n\n// Path: clickhouse/columns/geo.cpp\n#include \"geo.h\"\n\n#include \"utils.h\"\n\nnamespace {\nusing namespace ::clickhouse;\n\ntemplate <Type::Code type_code>\nTypeRef CreateGeoType() {\n    if constexpr (type_code == Type::Code::Point) {\n        return Type::CreatePoint();\n    } else if constexpr (type_code == Type::Code::Ring) {\n        return Type::CreateRing();\n    } else if constexpr (type_code == Type::Code::Polygon) {\n        return Type::CreatePolygon();\n    } else if constexpr (type_code == Type::Code::MultiPolygon) {\n        return Type::CreateMultiPolygon();\n    }\n}\n\ntemplate <typename ColumnType>\nstd::shared_ptr<ColumnType> CreateColumn() {\n    if constexpr (std::is_same_v<ColumnType, ColumnTupleT<ColumnFloat64, ColumnFloat64>>) {\n        return std::make_shared<ColumnTupleT<ColumnFloat64, ColumnFloat64>>(\n            std::make_tuple(std::make_shared<ColumnFloat64>(), std::make_shared<ColumnFloat64>()));\n    } else {\n        return std::make_shared<ColumnType>();\n    }\n}\n\n}  // namespace\n\nnamespace clickhouse {\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nColumnGeo<NestedColumnType, type_code>::ColumnGeo()\n    : Column(CreateGeoType<type_code>()),\n      data_(CreateColumn<NestedColumnType>()) {\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nColumnGeo<NestedColumnType, type_code>::ColumnGeo(ColumnRef data)\n    : Column(CreateGeoType<type_code>())\n    , data_(WrapColumn<NestedColumnType>(std::move(data))) {\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nvoid ColumnGeo<NestedColumnType, type_code>::Clear() {\n    data_->Clear();\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nconst typename ColumnGeo<NestedColumnType, type_code>::ValueType ColumnGeo<NestedColumnType, type_code>::At(size_t n) const {\n    return data_->At(n);\n}\n\ntemplate<typename NestedColumnType, Type::Code type_code>\nvoid ColumnGeo<NestedColumnType, type_code>::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nvoid ColumnGeo<NestedColumnType, type_code>::Append(ColumnRef column) {\n    if (auto col = column->template As<ColumnGeo>()) {\n        data_->Append(col->data_->template As<Column>());\n    }\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nbool ColumnGeo<NestedColumnType, type_code>::LoadBody(InputStream* input, size_t rows) {\n    return data_->LoadBody(input, rows);\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nvoid ColumnGeo<NestedColumnType, type_code>::SaveBody(OutputStream* output) {\n    data_->SaveBody(output);\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nsize_t ColumnGeo<NestedColumnType, type_code>::Size() const {\n    return data_->Size();\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nColumnRef ColumnGeo<NestedColumnType, type_code>::Slice(size_t begin, size_t len) const {\n    return std::make_shared<ColumnGeo>(data_->Slice(begin, len));\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nColumnRef ColumnGeo<NestedColumnType, type_code>::CloneEmpty() const {\n    return std::make_shared<ColumnGeo>();\n}\n\ntemplate <typename NestedColumnType, Type::Code type_code>\nvoid ColumnGeo<NestedColumnType, type_code>::Swap(Column& other) {\n    auto& col = dynamic_cast<ColumnGeo&>(other);\n    data_.swap(col.data_);\n}\n\ntemplate class ColumnGeo<ColumnTupleT<ColumnFloat64, ColumnFloat64>, Type::Code::Point>;\n\ntemplate class ColumnGeo<ColumnArrayT<ColumnPoint>, Type::Code::Ring>;\n\ntemplate class ColumnGeo<ColumnArrayT<ColumnRing>, Type::Code::Polygon>;\n\ntemplate class ColumnGeo<ColumnArrayT<ColumnPolygon>, Type::Code::MultiPolygon>;\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/ip6.cpp\n#include \"ip6.h\"\n#include \"../base/socket.h\" // for IPv6 platform-specific stuff\n#include \"../exceptions.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nstatic_assert(sizeof(struct in6_addr) == 16, \"sizeof in6_addr should be 16 bytes\");\n\nColumnIPv6::ColumnIPv6()\n    : Column(Type::CreateIPv6())\n    , data_(std::make_shared<ColumnFixedString>(16))\n{\n}\n\nColumnIPv6::ColumnIPv6(ColumnRef data)\n    : Column(Type::CreateIPv6())\n    , data_(data ? data->As<ColumnFixedString>() : nullptr)\n{\n    if (!data_ || data_->FixedSize() != sizeof(in6_addr))\n        throw ValidationError(\"Expecting ColumnFixedString(16), got \" + (data ? data->GetType().GetName() : \"null\"));\n}\n\nvoid ColumnIPv6::Append(const std::string_view& str) {\n    unsigned char buf[16];\n    if (inet_pton(AF_INET6, str.data(), buf) != 1) {\n        throw ValidationError(\"invalid IPv6 format, ip: \" + std::string(str));\n    }\n    data_->Append(std::string_view((const char*)buf, 16));\n}\n\nvoid ColumnIPv6::Append(const in6_addr* addr) {\n    data_->Append(std::string_view((const char*)addr->s6_addr, 16));\n}\n\nvoid ColumnIPv6::Append(const in6_addr& addr) {\n    Append(&addr);\n}\n\nvoid ColumnIPv6::Clear() {\n    data_->Clear();\n}\n\nstd::string ColumnIPv6::AsString (size_t n) const {\n    const auto& addr = this->At(n);\n\n    char buf[INET6_ADDRSTRLEN];\n    const char* ip_str = inet_ntop(AF_INET6, &addr, buf, INET6_ADDRSTRLEN);\n\n    if (ip_str == nullptr) {\n        throw std::system_error(\n                std::error_code(errno, std::generic_category()),\n                \"Invalid IPv6 data\");\n    }\n\n    return ip_str;\n}\n\nin6_addr ColumnIPv6::At(size_t n) const {\n    return *reinterpret_cast<const in6_addr*>(data_->At(n).data());\n}\n\nin6_addr ColumnIPv6::operator [] (size_t n) const {\n    return *reinterpret_cast<const in6_addr*>(data_->At(n).data());\n}\n\nvoid ColumnIPv6::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\nvoid ColumnIPv6::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnIPv6>()) {\n        data_->Append(col->data_);\n    }\n}\n\nbool ColumnIPv6::LoadBody(InputStream* input, size_t rows) {\n    return data_->LoadBody(input, rows);\n}\n\nvoid ColumnIPv6::SaveBody(OutputStream* output) {\n    data_->SaveBody(output);\n}\n\nsize_t ColumnIPv6::Size() const {\n    return data_->Size();\n}\n\nColumnRef ColumnIPv6::Slice(size_t begin, size_t len) const {\n    return std::make_shared<ColumnIPv6>(data_->Slice(begin, len));\n}\n\nColumnRef ColumnIPv6::CloneEmpty() const {\n    return std::make_shared<ColumnIPv6>(data_->CloneEmpty());\n}\n\nvoid ColumnIPv6::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnIPv6 &>(other);\n    data_.swap(col.data_);\n}\n\nItemView ColumnIPv6::GetItem(size_t index) const {\n    return ItemView{Type::IPv6, data_->GetItem(index)};\n}\n\n}\n\n// Path: clickhouse/columns/lowcardinalityadaptor.h\n#pragma once\n\n#include \"column.h\"\n#include \"lowcardinality.h\"\n\n#include <cassert>\n\nnamespace clickhouse {\n\nclass OutputStream;\nclass CodedInputStream;\n\n/** Adapts any ColumnType to be serialized\\deserialized as LowCardinality,\n *  and to be castable to ColumnType via ColumnPtr->As<ColumnType>().\n *\n * It helps to ease migration of the old codebases, which can't afford to switch\n * to using ColumnLowCardinalityT or ColumnLowCardinality directly,\n * but still want to benefit from smaller on-wire LowCardinality bandwidth footprint.\n *\n * Not intended to be used by users directly.\n *\n * @see ClientOptions, CreateColumnByType\n */\ntemplate <typename AdaptedColumnType>\nclass\n[[deprecated(\"Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) \")]]\nLowCardinalitySerializationAdaptor : public AdaptedColumnType\n{\npublic:\n    using AdaptedColumnType::AdaptedColumnType;\n\n    bool LoadPrefix(InputStream* input, size_t rows) override {\n        auto new_data_column = this->Slice(0, 0)->template As<AdaptedColumnType>();\n        ColumnLowCardinalityT<AdaptedColumnType> low_cardinality_col(new_data_column);\n\n        return low_cardinality_col.LoadPrefix(input, rows);\n    }\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override {\n        auto new_data_column = this->CloneEmpty()->template As<AdaptedColumnType>();\n\n        ColumnLowCardinalityT<AdaptedColumnType> low_cardinality_col(new_data_column);\n        if (!low_cardinality_col.LoadBody(input, rows))\n            return false;\n\n        // It safe to reuse `flat_data_column` later since ColumnLowCardinalityT makes a deep copy, but still check just in case.\n        assert(new_data_column->Size() == 0);\n\n        for (size_t i = 0; i < low_cardinality_col.Size(); ++i)\n            new_data_column->Append(low_cardinality_col[i]);\n\n        this->Swap(*new_data_column);\n        return true;\n    }\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override {\n        ColumnLowCardinalityT<AdaptedColumnType>(this->template As<AdaptedColumnType>()).SaveBody(output);\n    }\n};\n\n}\n\n// Path: clickhouse/columns/nothing.h\n\n#pragma once\n\n#include \"column.h\"\n#include \"../base/input.h\"\n\n#include <stdexcept>\n#include <utility>\n\nnamespace clickhouse {\n\n/**\n * Represents dummy column of NULLs.\n */\nclass ColumnNothing : public Column {\npublic:\n    ColumnNothing()\n        : Column(Type::CreateNothing())\n        , size_(0)\n    {\n    }\n\n    explicit ColumnNothing(size_t n)\n        : Column(Type::CreateNothing())\n        , size_(n)\n    {\n    }\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t) override {};\n\n    /// Appends one element to the column.\n    void Append(std::unique_ptr<void*>) { ++size_; }\n\n    /// Returns element at given row number.\n    std::nullptr_t At(size_t) const { return nullptr; };\n\n    /// Returns element at given row number.\n    inline std::nullptr_t operator [] (size_t) const { return nullptr; };\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t, size_t len) const override {\n        return std::make_shared<ColumnNothing>(len);\n    }\n\n    ColumnRef CloneEmpty() const override {\n        return std::make_shared<ColumnNothing>();\n    }\n\n    ItemView GetItem(size_t /*index*/) const override { return ItemView{}; }\n\npublic:\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override {\n        if (auto col = column->As<ColumnNothing>()) {\n            size_ += col->Size();\n        }\n    }\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override {\n        input->Skip(rows);\n        size_ += rows;\n        return true;\n    }\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream*) override {\n        throw UnimplementedError(\"method SaveBody is not supported for Nothing column\");\n    }\n\n    /// Clear column data .\n    void Clear() override { size_ = 0; }\n\n    /// Returns count of rows in the column.\n    size_t Size() const override { return size_; }\n\n    void Swap(Column& other) override {\n        auto & col = dynamic_cast<ColumnNothing &>(other);\n        std::swap(size_, col.size_);\n    }\n\nprivate:\n    size_t size_;\n};\n\n}\n\n// Path: clickhouse/base/string_view.h\n#pragma once\n\n#include <cassert>\n#include <stdexcept>\n#include <string>\n\n/**\n * A lightweight non-owning read-only view into a subsequence of a string.\n */\ntemplate <\n    typename TChar,\n    typename TTraits = std::char_traits<TChar>\n>\nclass\n[[deprecated(\"Obsolete due to C++17's std::string_view. Will be removed in next major release (3.0) \")]]\nStringViewImpl {\npublic:\n    using size_type = size_t;\n    using traits_type = TTraits;\n    using value_type = typename TTraits::char_type;\n\n    static constexpr size_type npos = size_type(-1);\n\npublic:\n    inline StringViewImpl() noexcept\n        : data_(nullptr)\n        , size_(0)\n    {\n    }\n\n    constexpr inline StringViewImpl(const TChar* data, size_t len) noexcept\n        : data_(data)\n        , size_(len)\n    {\n    }\n\n    template <size_t len>\n    constexpr inline StringViewImpl(const TChar (&str)[len]) noexcept\n        : data_(str)\n        , size_(len - 1)\n    {\n    }\n\n    inline StringViewImpl(const TChar* begin, const TChar* end) noexcept\n        : data_(begin)\n        , size_(end - begin)\n    {\n        assert(begin <= end);\n    }\n\n    inline StringViewImpl(const std::basic_string<TChar>& str) noexcept\n        : data_(str.data())\n        , size_(str.size())\n    {\n    }\n\n    inline TChar at(size_type pos) const {\n        if (pos >= size_)\n            throw std::out_of_range(\"pos must be less than len\");\n        return data_[pos];\n    }\n\n    inline const TChar* data() const noexcept {\n        return data_;\n    }\n\n    inline bool empty() const noexcept {\n        return size_ == 0;\n    }\n\n    inline bool null() const noexcept {\n        assert(size_ == 0);\n        return data_ == nullptr;\n    }\n\n    inline size_type size() const noexcept {\n        return size_;\n    }\n\n    // to mimic std::string and std::string_view\n    inline size_type length() const noexcept {\n        return size();\n    }\n\npublic:\n    // Returns a substring [pos, pos + count).\n    // If the requested substring extends past the end of the string,\n    // or if count == npos, the returned substring is [pos, size()).\n    StringViewImpl substr(size_type pos, size_type count = npos) const {\n        if (pos >= size_)\n            throw std::out_of_range(\"pos must be less than len\");\n        if (pos + count >= size_ || count == npos)\n            return StringViewImpl(data_ + pos, size_ - pos);\n        else\n            return StringViewImpl(data_ + pos, count);\n    }\n\n    inline const std::basic_string<TChar> to_string() const {\n        return std::basic_string<TChar>(data_, size_);\n    }\n\npublic:\n    inline operator bool () const noexcept {\n        return !empty();\n    }\n\n    inline explicit operator const std::basic_string<TChar> () const {\n        return to_string();\n    }\n\n    inline TChar operator [] (size_type pos) const noexcept {\n        return data_[pos];\n    }\n\n    inline bool operator < (const StringViewImpl& other) const noexcept {\n        if (size_ < other.size_)\n            return true;\n        if (size_ > other.size_)\n            return false;\n        return TTraits::compare(data_, other.data_, size_) < 0;\n    }\n\n    inline bool operator == (const StringViewImpl& other) const noexcept {\n        if (size_ == other.size_)\n            return TTraits::compare(data_, other.data_, size_) == 0;\n        return false;\n    }\n\nprivate:\n    const TChar* data_;\n    size_t size_;\n};\n\n\n// It creates StringView from literal constant at compile time.\ntemplate <typename TChar, size_t size>\nconstexpr inline StringViewImpl<TChar> MakeStringView(const TChar (&str)[size]) {\n    return StringViewImpl<TChar>(str, size - 1);\n}\n\n\nusing StringView = StringViewImpl<char>;\n\n// Path: clickhouse/types/type_parser.h\n#pragma once\n\n#include \"../base/string_view.h\"\n#include \"types.h\"\n\n#include <list>\n#include <stack>\n#include <string>\n\nnamespace clickhouse {\n\nstruct TypeAst {\n    enum Meta {\n        Array,\n        Assign,\n        Null,\n        Nullable,\n        Number,\n        String,\n        Terminal,\n        Tuple,\n        Enum,\n        LowCardinality,\n        SimpleAggregateFunction,\n        Map\n    };\n\n    /// Type's category.\n    Meta meta;\n    Type::Code code;\n    /// Type's name.\n    /// Need to cache TypeAst, so can't use StringView for name.\n    std::string name;\n    /// Value associated with the node,\n    /// used for fixed-width types and enum values.\n    int64_t value = 0;\n    std::string value_string;\n    /// Subelements of the type.\n    /// Used to store enum's names and values as well.\n    std::vector<TypeAst> elements;\n\n    bool operator==(const TypeAst & other) const;\n    inline bool operator!=(const TypeAst & other) const {\n        return !(*this == other);\n    }\n};\n\n\nclass TypeParser {\n\n    struct Token {\n        enum Type {\n            Invalid = 0,\n            Assign,\n            Name,\n            Number,\n            String,\n            LPar,\n            RPar,\n            Comma,\n            QuotedString, // string with quotation marks included\n            EOS,\n        };\n\n        Type type;\n        StringView value;\n    };\n\npublic:\n    explicit TypeParser(const StringView& name);\n    ~TypeParser();\n\n    bool Parse(TypeAst* type);\n\nprivate:\n    Token NextToken();\n\nprivate:\n    const char* cur_;\n    const char* end_;\n\n    TypeAst* type_;\n    std::stack<TypeAst*> open_elements_;\n};\n\n\nconst TypeAst* ParseTypeName(const std::string& type_name);\n\n}\n\n// Path: clickhouse/columns/factory.cpp\n#include \"factory.h\"\n\n#include \"array.h\"\n#include \"date.h\"\n#include \"decimal.h\"\n#include \"enum.h\"\n#include \"geo.h\"\n#include \"ip4.h\"\n#include \"ip6.h\"\n#include \"lowcardinality.h\"\n#include \"lowcardinalityadaptor.h\"\n#include \"map.h\"\n#include \"nothing.h\"\n#include \"nullable.h\"\n#include \"numeric.h\"\n#include \"string.h\"\n#include \"tuple.h\"\n#include \"uuid.h\"\n\n\n#include \"../types/type_parser.h\"\n\n#include \"../exceptions.h\"\n\n#include <stdexcept>\n#include <string>\n\nnamespace clickhouse {\nnamespace {\n\n// Like Python's list's []:\n// * 0 - first element\n// * 1 - second element\n// * -1 - last element\n// * -2 - one before last, etc.\nconst auto& GetASTChildElement(const TypeAst & ast, int position) {\n    if (static_cast<size_t>(abs(position)) >= ast.elements.size())\n        throw ValidationError(\"AST child element index out of bounds: \" + std::to_string(position));\n\n    if (position < 0)\n        position = static_cast<int>(ast.elements.size() + position);\n\n    return ast.elements[static_cast<size_t>(position)];\n}\n\nstatic ColumnRef CreateTerminalColumn(const TypeAst& ast) {\n    switch (ast.code) {\n    case Type::Void:\n        return std::make_shared<ColumnNothing>();\n\n    case Type::UInt8:\n        return std::make_shared<ColumnUInt8>();\n    case Type::UInt16:\n        return std::make_shared<ColumnUInt16>();\n    case Type::UInt32:\n        return std::make_shared<ColumnUInt32>();\n    case Type::UInt64:\n        return std::make_shared<ColumnUInt64>();\n\n    case Type::Int8:\n        return std::make_shared<ColumnInt8>();\n    case Type::Int16:\n        return std::make_shared<ColumnInt16>();\n    case Type::Int32:\n        return std::make_shared<ColumnInt32>();\n    case Type::Int64:\n        return std::make_shared<ColumnInt64>();\n    case Type::Int128:\n        return std::make_shared<ColumnInt128>();\n\n    case Type::Float32:\n        return std::make_shared<ColumnFloat32>();\n    case Type::Float64:\n        return std::make_shared<ColumnFloat64>();\n\n    case Type::Decimal:\n        return std::make_shared<ColumnDecimal>(GetASTChildElement(ast, 0).value, GetASTChildElement(ast, -1).value);\n    case Type::Decimal32:\n        return std::make_shared<ColumnDecimal>(9, GetASTChildElement(ast, 0).value);\n    case Type::Decimal64:\n        return std::make_shared<ColumnDecimal>(18, GetASTChildElement(ast, 0).value);\n    case Type::Decimal128:\n        return std::make_shared<ColumnDecimal>(38, GetASTChildElement(ast, 0).value);\n\n    case Type::String:\n        return std::make_shared<ColumnString>();\n    case Type::FixedString:\n        return std::make_shared<ColumnFixedString>(GetASTChildElement(ast, 0).value);\n\n    case Type::DateTime:\n        if (ast.elements.empty()) {\n            return std::make_shared<ColumnDateTime>();\n        } else {\n            return std::make_shared<ColumnDateTime>(GetASTChildElement(ast, 0).value_string);\n        }\n    case Type::DateTime64:\n        if (ast.elements.empty()) {\n            return nullptr;\n        }\n        if (ast.elements.size() == 1) {\n            return std::make_shared<ColumnDateTime64>(ast.elements[0].value);\n        } else {\n            return std::make_shared<ColumnDateTime64>(ast.elements[0].value, ast.elements[1].value_string);\n        }\n    case Type::Date:\n        return std::make_shared<ColumnDate>();\n    case Type::Date32:\n        return std::make_shared<ColumnDate32>();\n\n    case Type::IPv4:\n        return std::make_shared<ColumnIPv4>();\n    case Type::IPv6:\n        return std::make_shared<ColumnIPv6>();\n\n    case Type::UUID:\n        return std::make_shared<ColumnUUID>();\n\n    case Type::Point:\n        return std::make_shared<ColumnPoint>();\n\n    case Type::Ring:\n        return std::make_shared<ColumnRing>();\n\n    case Type::Polygon:\n        return std::make_shared<ColumnPolygon>();\n\n    case Type::MultiPolygon:\n        return std::make_shared<ColumnMultiPolygon>();\n\n    default:\n        return nullptr;\n    }\n}\n\nstatic ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSettings settings) {\n    switch (ast.meta) {\n        case TypeAst::Array: {\n            return std::make_shared<ColumnArray>(\n                CreateColumnFromAst(GetASTChildElement(ast, 0), settings)\n            );\n        }\n\n        case TypeAst::Nullable: {\n            return std::make_shared<ColumnNullable>(\n                CreateColumnFromAst(GetASTChildElement(ast, 0), settings),\n                std::make_shared<ColumnUInt8>()\n            );\n        }\n\n        case TypeAst::Terminal: {\n            return CreateTerminalColumn(ast);\n        }\n\n        case TypeAst::Tuple: {\n            std::vector<ColumnRef> columns;\n\n            columns.reserve(ast.elements.size());\n            for (const auto& elem : ast.elements) {\n                if (auto col = CreateColumnFromAst(elem, settings)) {\n                    columns.push_back(col);\n                } else {\n                    return nullptr;\n                }\n            }\n\n            return std::make_shared<ColumnTuple>(columns);\n        }\n\n        case TypeAst::Enum: {\n            std::vector<Type::EnumItem> enum_items;\n            //ast.elements.size() minimum is 1.\n            if ((ast.elements.size() % 2) != 0) {\n                throw ValidationError(ast.name + \" content is not correct\");\n            }\n\n            enum_items.reserve(ast.elements.size() / 2);\n            for (size_t i = 0; i < ast.elements.size(); i += 2) {\n                enum_items.push_back(Type::EnumItem{\n                    ast.elements[i].value_string,\n                    static_cast<int16_t>(ast.elements[i + 1].value)\n                });\n            }\n\n            if (ast.code == Type::Enum8) {\n                return std::make_shared<ColumnEnum8>(\n                    Type::CreateEnum8(enum_items)\n                );\n            } else if (ast.code == Type::Enum16) {\n                return std::make_shared<ColumnEnum16>(\n                    Type::CreateEnum16(enum_items)\n                );\n            }\n            break;\n        }\n        case TypeAst::LowCardinality: {\n            const auto nested = GetASTChildElement(ast, 0);\n            if (settings.low_cardinality_as_wrapped_column) {\n                switch (nested.code) {\n                    // TODO (nemkov): update this to maximize code reuse.\n                    case Type::String:\n                        return std::make_shared<LowCardinalitySerializationAdaptor<ColumnString>>();\n                    case Type::FixedString:\n                        return std::make_shared<LowCardinalitySerializationAdaptor<ColumnFixedString>>(GetASTChildElement(nested, 0).value);\n                    case Type::Nullable:\n                        throw UnimplementedError(\"LowCardinality(\" + nested.name + \") is not supported with LowCardinalityAsWrappedColumn on\");\n                    default:\n                        throw UnimplementedError(\"LowCardinality(\" + nested.name + \") is not supported\");\n                }\n            }\n            else {\n                switch (nested.code) {\n                    // TODO (nemkov): update this to maximize code reuse.\n                    case Type::String:\n                        return std::make_shared<ColumnLowCardinalityT<ColumnString>>();\n                    case Type::FixedString:\n                        return std::make_shared<ColumnLowCardinalityT<ColumnFixedString>>(GetASTChildElement(nested, 0).value);\n                    case Type::Nullable:\n                        return std::make_shared<ColumnLowCardinality>(\n                            std::make_shared<ColumnNullable>(\n                                CreateColumnFromAst(GetASTChildElement(nested, 0), settings),\n                                std::make_shared<ColumnUInt8>()\n                            )\n                        );\n                    default:\n                        throw UnimplementedError(\"LowCardinality(\" + nested.name + \") is not supported\");\n                }\n            }\n        }\n        case TypeAst::SimpleAggregateFunction: {\n            return CreateTerminalColumn(GetASTChildElement(ast, -1));\n        }\n\n        case TypeAst::Map: {\n            if (ast.elements.size() != 2) {\n                throw ValidationError(ast.name + \" content is not correct\");\n            }\n\n            std::vector<ColumnRef> columns;\n\n            columns.reserve(ast.elements.size());\n            for (const auto& elem : ast.elements) {\n                if (auto col = CreateColumnFromAst(elem, settings)) {\n                    columns.push_back(col);\n                } else {\n                    return nullptr;\n                }\n            }\n\n            return std::make_shared<ColumnMap>(\n                std::make_shared<ColumnArray>(\n                    std::make_shared<ColumnTuple>(columns)));\n        }\n\n        case TypeAst::Assign:\n        case TypeAst::Null:\n        case TypeAst::Number:\n        case TypeAst::String:\n            break;\n    }\n\n    return nullptr;\n}\n\n} // namespace\n\n\n\nColumnRef CreateColumnByType(const std::string& type_name, CreateColumnByTypeSettings settings) {\n    auto ast = ParseTypeName(type_name);\n    if (ast != nullptr) {\n        return CreateColumnFromAst(*ast, settings);\n    }\n\n    return nullptr;\n}\n\n}\n\n// Path: clickhouse/columns/nullable.cpp\n#include \"nullable.h\"\n\n#include <assert.h>\n#include <stdexcept>\n\nnamespace clickhouse {\n\nColumnNullable::ColumnNullable(ColumnRef nested, ColumnRef nulls)\n    : Column(Type::CreateNullable(nested->Type()))\n    , nested_(nested)\n    , nulls_(nulls->As<ColumnUInt8>())\n{\n    if (nested_->Size() != nulls->Size()) {\n        throw ValidationError(\"count of elements in nested and nulls should be the same\");\n    }\n}\n\nvoid ColumnNullable::Append(bool isnull)\n{\n    nulls_->Append(isnull ? 1 : 0);\n}\n\n\nbool ColumnNullable::IsNull(size_t n) const {\n    return nulls_->At(n) != 0;\n}\n\nColumnRef ColumnNullable::Nested() const {\n    return nested_;\n}\n\nColumnRef ColumnNullable::Nulls() const\n{\n       return nulls_;\n}\n\nvoid ColumnNullable::Reserve(size_t new_cap) {\n    nested_->Reserve(new_cap);\n    nulls_->Reserve(new_cap);\n}\n\nvoid ColumnNullable::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnNullable>()) {\n        if (!col->nested_->Type()->IsEqual(nested_->Type())) {\n            return;\n        }\n\n        nested_->Append(col->nested_);\n        nulls_->Append(col->nulls_);\n    }\n}\n\nvoid ColumnNullable::Clear() {\n    nested_->Clear();\n    nulls_->Clear();\n}\n\nbool ColumnNullable::LoadPrefix(InputStream* input, size_t rows) {\n    return nested_->LoadPrefix(input, rows);\n}\n\nbool ColumnNullable::LoadBody(InputStream* input, size_t rows) {\n    if (!nulls_->LoadBody(input, rows)) {\n        return false;\n    }\n    if (!nested_->LoadBody(input, rows)) {\n        return false;\n    }\n    return true;\n}\n\nvoid ColumnNullable::SavePrefix(OutputStream* output) {\n    nested_->SavePrefix(output);\n}\n\nvoid ColumnNullable::SaveBody(OutputStream* output) {\n    nulls_->SaveBody(output);\n    nested_->SaveBody(output);\n}\n\nsize_t ColumnNullable::Size() const {\n    return nulls_->Size();\n}\n\nColumnRef ColumnNullable::Slice(size_t begin, size_t len) const {\n    return std::make_shared<ColumnNullable>(nested_->Slice(begin, len), nulls_->Slice(begin, len));\n}\n\nColumnRef ColumnNullable::CloneEmpty() const {\n    return std::make_shared<ColumnNullable>(nested_->CloneEmpty(), nulls_->CloneEmpty());\n}\n\nvoid ColumnNullable::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnNullable &>(other);\n    if (!nested_->Type()->IsEqual(col.nested_->Type()))\n        throw ValidationError(\"Can't swap() Nullable columns of different types.\");\n\n    nested_.swap(col.nested_);\n    nulls_.swap(col.nulls_);\n}\n\nItemView ColumnNullable::GetItem(size_t index) const  {\n    if (IsNull(index))\n        return ItemView();\n\n    return nested_->GetItem(index);\n}\n\n}\n\n// Path: clickhouse/columns/enum.cpp\n#include \"enum.h\"\n#include \"utils.h\"\n\n#include \"../base/input.h\"\n#include \"../base/output.h\"\n#include \"../base/wire_format.h\"\n\nnamespace clickhouse {\n\ntemplate <typename T>\nColumnEnum<T>::ColumnEnum(TypeRef type)\n    : Column(type)\n{\n}\n\ntemplate <typename T>\nColumnEnum<T>::ColumnEnum(TypeRef type, const std::vector<T>& data)\n    : Column(type)\n    , data_(data)\n{\n}\n\ntemplate <typename T>\nColumnEnum<T>::ColumnEnum(TypeRef type, std::vector<T>&& data)\n    : Column(type)\n    , data_(std::move(data))\n{\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::Append(const T& value, bool checkValue) {\n    if  (checkValue) {\n        // TODO: type_->HasEnumValue(value), \"Enum type doesn't have value \" + std::to_string(value);\n    }\n    data_.push_back(value);\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::Append(const std::string& name) {\n    data_.push_back(static_cast<T>(type_->As<EnumType>()->GetEnumValue(name)));\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::Clear() {\n    data_.clear();\n}\n\ntemplate <typename T>\nconst T& ColumnEnum<T>::At(size_t n) const {\n    return data_.at(n);\n}\n\ntemplate <typename T>\nstd::string_view ColumnEnum<T>::NameAt(size_t n) const {\n    return type_->As<EnumType>()->GetEnumName(data_.at(n));\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::SetAt(size_t n, const T& value, bool checkValue) {\n    if (checkValue) {\n        // TODO: type_->HasEnumValue(value), \"Enum type doesn't have value \" + std::to_string(value);\n    }\n    data_.at(n) = value;\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::SetNameAt(size_t n, const std::string& name) {\n    data_.at(n) = static_cast<T>(type_->As<EnumType>()->GetEnumValue(name));\n}\n\ntemplate<typename T>\nvoid ColumnEnum<T>::Reserve(size_t new_cap) {\n    data_.reserve(new_cap);\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnEnum<T>>()) {\n        data_.insert(data_.end(), col->data_.begin(), col->data_.end());\n    }\n}\n\ntemplate <typename T>\nbool ColumnEnum<T>::LoadBody(InputStream* input, size_t rows) {\n    data_.resize(rows);\n    return WireFormat::ReadBytes(*input, data_.data(), data_.size() * sizeof(T));\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::SaveBody(OutputStream* output) {\n    WireFormat::WriteBytes(*output, data_.data(), data_.size() * sizeof(T));\n}\n\ntemplate <typename T>\nsize_t ColumnEnum<T>::Size() const {\n    return data_.size();\n}\n\ntemplate <typename T>\nColumnRef ColumnEnum<T>::Slice(size_t begin, size_t len) const {\n    return std::make_shared<ColumnEnum<T>>(type_, SliceVector(data_, begin, len));\n}\n\ntemplate <typename T>\nColumnRef ColumnEnum<T>::CloneEmpty() const {\n    return std::make_shared<ColumnEnum<T>>(type_);\n}\n\ntemplate <typename T>\nvoid ColumnEnum<T>::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnEnum<T> &>(other);\n    data_.swap(col.data_);\n}\n\ntemplate <typename T>\nItemView ColumnEnum<T>::GetItem(size_t index) const {\n    return ItemView{type_->GetCode(), data_[index]};\n}\n\ntemplate class ColumnEnum<int8_t>;\ntemplate class ColumnEnum<int16_t>;\n\n}\n\n// Path: clickhouse/columns/date.cpp\n#include \"date.h\"\n#include <cstdint>\n\nnamespace clickhouse {\n\nColumnDate::ColumnDate()\n    : Column(Type::CreateDate())\n    , data_(std::make_shared<ColumnUInt16>())\n{\n}\n\nColumnDate::ColumnDate(std::vector<uint16_t>&& data)\n    : Column(Type::CreateDate())\n    , data_(std::make_shared<ColumnUInt16>(std::move(data)))\n{\n}\n\nvoid ColumnDate::Append(const std::time_t& value) {\n    /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving.\n    data_->Append(static_cast<uint16_t>(value / std::time_t(86400)));\n}\n\nvoid ColumnDate::Clear() {\n    data_->Clear();\n}\n\nstd::time_t ColumnDate::At(size_t n) const {\n    /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving.\n    return static_cast<std::time_t>(data_->At(n)) * 86400;\n}\n\nvoid ColumnDate::AppendRaw(uint16_t value) {\n    data_->Append(value);\n}\n\nuint16_t ColumnDate::RawAt(size_t n) const {\n    return data_->At(n);\n}\n\nvoid ColumnDate::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnDate>()) {\n        data_->Append(col->data_);\n    }\n}\n\nstd::vector<uint16_t>& ColumnDate::GetWritableData() {\n    return data_->GetWritableData();\n}\n\nvoid ColumnDate::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\nsize_t ColumnDate::Capacity() const {\n    return data_->Capacity();\n}\n\nbool ColumnDate::LoadBody(InputStream* input, size_t rows) {\n    return data_->LoadBody(input, rows);\n}\n\nvoid ColumnDate::SaveBody(OutputStream* output) {\n    data_->SaveBody(output);\n}\n\nsize_t ColumnDate::Size() const {\n    return data_->Size();\n}\n\nColumnRef ColumnDate::Slice(size_t begin, size_t len) const {\n    auto col = data_->Slice(begin, len)->As<ColumnUInt16>();\n    auto result = std::make_shared<ColumnDate>();\n\n    result->data_->Append(col);\n\n    return result;\n}\n\nColumnRef ColumnDate::CloneEmpty() const {\n    return std::make_shared<ColumnDate>();\n}\n\nvoid ColumnDate::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnDate &>(other);\n    data_.swap(col.data_);\n}\n\nItemView ColumnDate::GetItem(size_t index) const {\n    return ItemView(Type::Date, data_->GetItem(index));\n}\n\n\nColumnDate32::ColumnDate32()\n    : Column(Type::CreateDate32())\n    , data_(std::make_shared<ColumnInt32>())\n{\n}\n\nColumnDate32::ColumnDate32(std::vector<int32_t>&& data)\n    : Column(Type::CreateDate32())\n    , data_(std::make_shared<ColumnInt32>(std::move(data)))\n{\n}\n\nvoid ColumnDate32::Append(const std::time_t& value) {\n    /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving.\n    data_->Append(static_cast<int32_t>(value / std::time_t(86400)));\n}\n\nvoid ColumnDate32::Clear() {\n    data_->Clear();\n}\n\nstd::time_t ColumnDate32::At(size_t n) const {\n    /// The implementation is fundamentally wrong, ignores timezones, leap years and daylight saving.\n    return static_cast<std::time_t>(data_->At(n)) * 86400;\n}\n\nvoid ColumnDate32::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnDate32>()) {\n        data_->Append(col->data_);\n    }\n}\n\nstd::vector<int32_t>& ColumnDate32::GetWritableData() {\n    return data_->GetWritableData();\n}\n\nvoid ColumnDate32::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\nsize_t ColumnDate32::Capacity() const {\n    return data_->Capacity();\n}\n\nvoid ColumnDate32::AppendRaw(int32_t value) {\n    data_->Append(value);\n}\n\nint32_t ColumnDate32::RawAt(size_t n) const {\n    return data_->At(n);\n}\n\nbool ColumnDate32::LoadBody(InputStream* input, size_t rows) {\n    return data_->LoadBody(input, rows);\n}\n\nvoid ColumnDate32::SaveBody(OutputStream* output) {\n    data_->SaveBody(output);\n}\n\nsize_t ColumnDate32::Size() const {\n    return data_->Size();\n}\n\nColumnRef ColumnDate32::Slice(size_t begin, size_t len) const {\n    auto col = data_->Slice(begin, len)->As<ColumnInt32>();\n    auto result = std::make_shared<ColumnDate32>();\n\n    result->data_->Append(col);\n\n    return result;\n}\n\nColumnRef ColumnDate32::CloneEmpty() const {\n    return std::make_shared<ColumnDate32>();\n}\n\nvoid ColumnDate32::Swap(Column& other) {\n    auto & col = dynamic_cast<ColumnDate32 &>(other);\n    data_.swap(col.data_);\n}\n\nItemView ColumnDate32::GetItem(size_t index) const {\n    return ItemView{Type()->GetCode(), data_->GetItem(index)};\n}\n\nColumnDateTime::ColumnDateTime()\n    : Column(Type::CreateDateTime())\n    , data_(std::make_shared<ColumnUInt32>())\n{\n}\n\nColumnDateTime::ColumnDateTime(std::string timezone)\n    : Column(Type::CreateDateTime(std::move(timezone)))\n    , data_(std::make_shared<ColumnUInt32>())\n{\n}\n\nColumnDateTime::ColumnDateTime(std::vector<uint32_t>&& data)\n    : Column(Type::CreateDateTime())\n    , data_(std::make_shared<ColumnUInt32>(std::move(data))) {\n}\n\nColumnDateTime::ColumnDateTime(std::string timezone, std::vector<uint32_t>&& data)\n    : Column(Type::CreateDateTime(std::move(timezone)))\n    , data_(std::make_shared<ColumnUInt32>(std::move(data))) {\n}\n\nvoid ColumnDateTime::Append(const std::time_t& value) {\n    data_->Append(static_cast<uint32_t>(value));\n}\n\nstd::time_t ColumnDateTime::At(size_t n) const {\n    return data_->At(n);\n}\n\nvoid ColumnDateTime::AppendRaw(uint32_t value) {\n    data_->Append(value);\n}\n\nstd::string ColumnDateTime::Timezone() const {\n    return type_->As<DateTimeType>()->Timezone();\n}\n\nvoid ColumnDateTime::Append(ColumnRef column) {\n    if (auto col = column->As<ColumnDateTime>()) {\n        data_->Append(col->data_);\n    }\n}\n\nstd::vector<uint32_t>& ColumnDateTime::GetWritableData() {\n    return data_->GetWritableData();\n}\n\nvoid ColumnDateTime::Reserve(size_t new_cap) {\n    data_->Reserve(new_cap);\n}\n\n...\n", "needle_token_start": 11960, "needle_token_end": 12032, "code_context_ntokens": 15987, "output": ["```cpp\n#include \"factory.h\"\n\n#include \"array.h\"\n#include \"date.h\"\n#include \"decimal.h\"\n#include \"enum.h\"\n#include \"geo.h\"\n#include \"ip4.h\"\n#include \"ip6.h\"\n#include \"lowcardinality.h\"\n#include \"lowcardinalityadaptor.h\"\n#include \"map.h\"\n#include \"nothing.h\"\n#include \"nullable.h\"\n#include \"numeric.h\"\n#include \"string.h\"\n#include \"tuple.h\"\n#include \"uuid.h\"\n\n#include \"../types/type_parser.h\"\n\n#include \"../exceptions.h\"\n\n#include <stdexcept>\n#include <string>\n\nnamespace clickhouse {\nnamespace {\n\n// Like Python's list's []:\n// * 0 - first element\n// * 1 - second element\n// * -1 - last element\n// * -2 - one before last, etc.\nconst auto& GetASTChildElement(const TypeAst & ast, int position) {\n    if (static_cast<size_t>(abs(position)) >= ast.elements.size())\n        throw ValidationError(\"AST child element index out of bounds: \" + std::to_string(position));\n\n    if (position < 0)\n        position = static_cast<int>(ast.elements.size() + position);\n\n    return ast.elements[static_cast<size_t>(position)];\n}\n\nstatic ColumnRef CreateTerminalColumn(const TypeAst& ast) {\n    switch (ast.code) {\n    case Type::Void:\n        return std::make_shared<ColumnNothing>();\n\n    case Type::UInt8:\n        return std::make_shared<ColumnUInt8>();\n    case Type::UInt16:\n        return std::make_shared<ColumnUInt16>();\n    case Type::UInt32:\n        return std::make_shared<ColumnUInt32>();\n    case Type::UInt64:\n        return std::make_shared<ColumnUInt64>();\n\n    case Type::Int8:\n        return std::make_shared<ColumnInt8>();\n    case Type::Int16:\n        return std::make_shared<ColumnInt16>();\n    case Type::Int32:\n        return std::make_shared<ColumnInt32>();\n    case Type::Int64:\n        return std::make_shared<ColumnInt64>();\n    case Type::Int128:\n        return std::make_shared<ColumnInt128>();\n\n    case Type::Float32:\n        return std::make_shared<ColumnFloat32>();\n    case Type::Float64:\n        return std::make_shared<ColumnFloat64>();\n\n    case Type::Decimal:\n        return std::make_shared<ColumnDecimal>(GetASTChildElement(ast, 0).value, GetASTChildElement(ast, -1).value);\n    case Type::Decimal32:\n        return std::make_shared<ColumnDecimal>(9, GetASTChildElement(ast, 0).value);\n    case Type::Decimal64:\n        return std::make_shared<ColumnDecimal>(18, GetASTChildElement(ast, 0).value);\n    case Type::Decimal128:\n        return std::make_shared<ColumnDecimal>(38, GetASTChildElement(ast, 0).value);\n\n    case Type::String:\n        return std::make_shared<ColumnString>();\n    case Type::FixedString:\n        return std::make_shared<ColumnFixedString>(GetASTChildElement(ast, 0).value);\n\n    case Type::DateTime:\n        if (ast.elements.empty()) {\n            return std::make_shared<ColumnDateTime>();\n        } else {\n            return std::make_shared<ColumnDateTime>(GetASTChildElement(ast, 0).value_string);\n        }\n    case Type::DateTime64:\n        if (ast.elements.empty()) {\n            return nullptr;\n        }\n        if (ast.elements.size() == 1) {\n            return std::make_shared<ColumnDateTime64>(ast.elements[0].value);\n        } else {\n            return std::make_shared<ColumnDateTime64>(ast.elements[0].value, ast.elements[1].value_string);\n        }\n    case Type::Date:\n        return std::make_shared<ColumnDate>();\n    case Type::Date32:\n        return std::make_shared<ColumnDate32>();\n\n    case Type::IPv4:\n        return std::make_shared<ColumnIPv4>();\n    case Type::IPv6:\n        return std::make_shared<ColumnIPv6>();\n\n    case Type::UUID:\n        return std::make_shared<ColumnUUID>();\n\n    case Type::Point:\n        return std::make_shared<ColumnPoint>();\n\n    case Type::Ring:\n        return std::make_shared<ColumnRing>();\n\n    case Type::Polygon:\n        return std::make_shared<ColumnPolygon>();\n\n    case Type::MultiPolygon:\n        return std::make_shared<ColumnMultiPolygon>();\n\n    default:\n        return nullptr;\n    }\n}\n\nstatic ColumnRef CreateColumnFromAst(const TypeAst& ast, CreateColumnByTypeSettings settings) {\n    switch (ast.meta) {\n        case TypeAst::Array: {\n           "]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "ConvertToStorageValue", "language": "cpp", "path": "clickhouse/columns/itemview.h", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert various data types into a uniform format suitable for storage in a specific data structure, ensuring compatibility and consistency in how data is represented.\n2. **Input**: Accepts a generic input of various data types, including strings, fundamental data types, and user-defined types.\n3. **Output**: Returns a uniform string view representation of the input data, facilitating standardized storage.\n4. **Procedure**: The function checks the type of the input data:\n   - If the input is a string or string view, it directly returns a string view of the input.\n   - If the input is a fundamental data type or a specific user-defined type, it converts the data to a string view by interpreting the data's memory representation.\n   - If the input type is not supported, it triggers a compile-time assertion to indicate an error due to an unknown type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/types/types.h\n#pragma once\n\n#include \"absl/numeric/int128.h\"\n\n#include <atomic>\n#include <map>\n#include <memory>\n#include <string>\n#include <vector>\n#include <stdexcept>\n\nnamespace clickhouse {\n\nusing Int128 = absl::int128;\nusing Int64 = int64_t;\n\nusing TypeRef = std::shared_ptr<class Type>;\n\nclass Type {\npublic:\n    enum Code {\n        Void = 0,\n        Int8,\n        Int16,\n        Int32,\n        Int64,\n        UInt8,\n        UInt16,\n        UInt32,\n        UInt64,\n        Float32,\n        Float64,\n        String,\n        FixedString,\n        DateTime,\n        Date,\n        Array,\n        Nullable,\n        Tuple,\n        Enum8,\n        Enum16,\n        UUID,\n        IPv4,\n        IPv6,\n        Int128,\n        Decimal,\n        Decimal32,\n        Decimal64,\n        Decimal128,\n        LowCardinality,\n        DateTime64,\n        Date32,\n        Map,\n        Point,\n        Ring,\n        Polygon,\n        MultiPolygon\n    };\n\n    using EnumItem = std::pair<std::string /* name */, int16_t /* value */>;\n\nprotected:\n    Type(const Code code);\n\npublic:\n    template <typename Derived>\n    auto* As() {\n        return static_cast<Derived*>(this);\n    }\n\n    template <typename Derived>\n    const auto* As() const {\n        return static_cast<const Derived*>(this);\n    }\n\n    /// Type's code.\n    Code GetCode() const { return code_; }\n\n    /// String representation of the type.\n    std::string GetName() const;\n\n    /// Is given type same as current one.\n    bool IsEqual(const Type& other) const {\n        // Types are equal only if both code_ and type_unique_id_ are equal.\n        return this == &other\n                // GetTypeUniqueId() is relatively heavy, so avoid calling it when comparing obviously different types.\n                || (this->GetCode() == other.GetCode() && this->GetTypeUniqueId() == other.GetTypeUniqueId());\n    }\n\n    bool IsEqual(const TypeRef& other) const { return IsEqual(*other); }\n\n    /// Simple name, doesn't depend on parameters and\\or nested types, caller MUST NOT free returned value.\n    static const char* TypeName(Code);\n\npublic:\n    static TypeRef CreateArray(TypeRef item_type);\n\n    static TypeRef CreateDate();\n\n    static TypeRef CreateDate32();\n\n    static TypeRef CreateDateTime(std::string timezone = std::string());\n\n    static TypeRef CreateDateTime64(size_t precision, std::string timezone = std::string());\n\n    static TypeRef CreateDecimal(size_t precision, size_t scale);\n\n    static TypeRef CreateIPv4();\n\n    static TypeRef CreateIPv6();\n\n    static TypeRef CreateNothing();\n\n    static TypeRef CreateNullable(TypeRef nested_type);\n\n    template <typename T>\n    static TypeRef CreateSimple();\n\n    static TypeRef CreateString();\n\n    static TypeRef CreateString(size_t n);\n\n    static TypeRef CreateTuple(const std::vector<TypeRef>& item_types);\n\n    static TypeRef CreateEnum8(const std::vector<EnumItem>& enum_items);\n\n    static TypeRef CreateEnum16(const std::vector<EnumItem>& enum_items);\n\n    static TypeRef CreateUUID();\n\n    static TypeRef CreateLowCardinality(TypeRef item_type);\n\n    static TypeRef CreateMap(TypeRef key_type, TypeRef value_type);\n\n    static TypeRef CreatePoint();\n\n    static TypeRef CreateRing();\n\n    static TypeRef CreatePolygon();\n\n    static TypeRef CreateMultiPolygon();\n\nprivate:\n    uint64_t GetTypeUniqueId() const;\n\n    const Code code_;\n    mutable std::atomic<uint64_t> type_unique_id_;\n};\n\ninline bool operator==(const Type & left, const Type & right) {\n    if (&left == &right)\n        return true;\n    if (typeid(left) == typeid(right))\n        return left.IsEqual(right);\n    return false;\n}\n\ninline bool operator==(const TypeRef & left, const TypeRef & right) {\n    return *left == *right;\n}\n\nclass ArrayType : public Type {\npublic:\n    explicit ArrayType(TypeRef item_type);\n\n    std::string GetName() const { return std::string(\"Array(\") + item_type_->GetName() + \")\"; }\n\n    /// Type of array's elements.\n    inline TypeRef GetItemType() const { return item_type_; }\n\nprivate:\n    TypeRef item_type_;\n};\n\nclass DecimalType : public Type {\npublic:\n    DecimalType(size_t precision, size_t scale);\n\n    std::string GetName() const;\n    friend class EnumType;\n    friend class DateTimeType;\n\n    inline size_t GetScale() const { return scale_; }\n    inline size_t GetPrecision() const { return precision_; }\n\nprivate:\n    const size_t precision_, scale_;\n};\n\nnamespace details\n{\nclass TypeWithTimeZoneMixin\n{\npublic:\n    TypeWithTimeZoneMixin(std::string timezone);\n\n    /// Timezone associated with a data column.\n    const std::string & Timezone() const;\n\nprivate:\n    std::string timezone_;\n};\n}\n\nclass DateTimeType : public Type, public details::TypeWithTimeZoneMixin {\npublic:\n    explicit DateTimeType(std::string timezone);\n\n    std::string GetName() const;\n};\n\nclass DateTime64Type: public Type, public details::TypeWithTimeZoneMixin {\npublic:\n    explicit DateTime64Type(size_t precision, std::string timezone_);\n\n    std::string GetName() const;\n\n    inline size_t GetPrecision() const { return precision_; }\nprivate:\n    size_t precision_;\n};\n\nclass EnumType : public Type {\npublic:\n    EnumType(Type::Code type, const std::vector<EnumItem>& items);\n\n    std::string GetName() const;\n\n    /// Methods to work with enum types.\n    std::string_view GetEnumName(int16_t value) const;\n    int16_t GetEnumValue(const std::string& name) const;\n    bool HasEnumName(const std::string& name) const;\n    bool HasEnumValue(int16_t value) const;\n\nprivate:\n    using ValueToNameType     = std::map<int16_t, std::string_view>;\n    using NameToValueType     = std::map<std::string, int16_t>;\n    using ValueToNameIterator = ValueToNameType::const_iterator;\n\n    ValueToNameType value_to_name_;\n    NameToValueType name_to_value_;\n\npublic:\n    ValueToNameIterator BeginValueToName() const;\n    ValueToNameIterator EndValueToName() const;\n};\n\nclass FixedStringType : public Type {\npublic:\n    explicit FixedStringType(size_t n);\n\n    std::string GetName() const { return std::string(\"FixedString(\") + std::to_string(size_) + \")\"; }\n\n    inline size_t GetSize() const { return size_; }\n\nprivate:\n    size_t size_;\n};\n\nclass NullableType : public Type {\npublic:\n    explicit NullableType(TypeRef nested_type);\n\n    std::string GetName() const { return std::string(\"Nullable(\") + nested_type_->GetName() + \")\"; }\n\n    /// Type of nested nullable element.\n    TypeRef GetNestedType() const { return nested_type_; }\n\nprivate:\n    TypeRef nested_type_;\n};\n\nclass TupleType : public Type {\npublic:\n    explicit TupleType(const std::vector<TypeRef>& item_types);\n\n    std::string GetName() const;\n\n    /// Type of nested Tuple element type.\n    std::vector<TypeRef> GetTupleType() const { return item_types_; }\n\nprivate:\n    std::vector<TypeRef> item_types_;\n};\n\nclass LowCardinalityType : public Type {\npublic:\n    explicit LowCardinalityType(TypeRef nested_type);\n    ~LowCardinalityType();\n\n    std::string GetName() const { return std::string(\"LowCardinality(\") + nested_type_->GetName() + \")\"; }\n\n    /// Type of nested nullable element.\n    TypeRef GetNestedType() const { return nested_type_; }\n\nprivate:\n    TypeRef nested_type_;\n};\n\nclass MapType : public Type {\npublic:\n    explicit MapType(TypeRef key_type, TypeRef value_type);\n\n    std::string GetName() const;\n\n    /// Type of keys.\n    TypeRef GetKeyType() const { return key_type_; }\n\n    /// Type of values.\n    TypeRef GetValueType() const { return value_type_; }\n\nprivate:\n    TypeRef key_type_;\n    TypeRef value_type_;\n};\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int8_t>() {\n    return TypeRef(new Type(Int8));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int16_t>() {\n    return TypeRef(new Type(Int16));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int32_t>() {\n    return TypeRef(new Type(Int32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<int64_t>() {\n    return TypeRef(new Type(Int64));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<Int128>() {\n    return TypeRef(new Type(Int128));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint8_t>() {\n    return TypeRef(new Type(UInt8));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint16_t>() {\n    return TypeRef(new Type(UInt16));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint32_t>() {\n    return TypeRef(new Type(UInt32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<uint64_t>() {\n    return TypeRef(new Type(UInt64));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<float>() {\n    return TypeRef(new Type(Float32));\n}\n\ntemplate <>\ninline TypeRef Type::CreateSimple<double>() {\n    return TypeRef(new Type(Float64));\n}\n\n}  // namespace clickhouse\n\n// Path: clickhouse/server_exception.h\n#pragma once\n\n#include <string>\n#include <memory>\n\nnamespace clickhouse {\nstruct Exception {\n    int code = 0;\n    std::string name;\n    std::string display_text;\n    std::string stack_trace;\n    /// Pointer to nested exception.\n    std::unique_ptr<Exception> nested;\n};\n\n}\n\n// Path: clickhouse/exceptions.h\n#pragma once\n\n#include \"server_exception.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nclass Error : public std::runtime_error {\n    using std::runtime_error::runtime_error;\n};\n\n// Caused by any user-related code, like invalid column types or arguments passed to any method.\nclass ValidationError : public Error {\n    using Error::Error;\n};\n\n// Buffers+IO errors, failure to serialize/deserialize, checksum mismatches, etc.\nclass ProtocolError : public Error {\n    using Error::Error;\n};\n\nclass UnimplementedError : public Error {\n    using Error::Error;\n};\n\n// Internal validation error.\nclass AssertionError : public Error {\n    using Error::Error;\n};\n\nclass OpenSSLError : public Error {\n    using Error::Error;\n};\n\nclass LZ4Error : public Error {\n    using Error::Error;\n};\n\n// Exception received from server.\nclass ServerException : public Error {\npublic:\n    ServerException(std::unique_ptr<Exception> e)\n        : Error(std::string())\n        , exception_(std::move(e))\n    {\n    }\n\n    int GetCode() const {\n        return exception_->code;\n    }\n\n    const Exception& GetException() const {\n        return *exception_;\n    }\n\n    const char* what() const noexcept override {\n        return exception_->display_text.c_str();\n    }\n\nprivate:\n    std::unique_ptr<Exception> exception_;\n};\nusing ServerError = ServerException;\n\n}\n\n// Path: clickhouse/columns/itemview.h\n#pragma once\n\n#include \"../types/types.h\"\n#include \"../exceptions.h\"\n\n#include <string_view>\n#include <stdexcept>\n#include <type_traits>\n\nnamespace clickhouse {\n\n/** ItemView is a view on a data stored in Column, safe-ish interface for reading values from Column.\n *\n * Data is not owned (hence the name View) and will be invalidated on column update, load\n * or destruction (basically on calling any non-const method of Column).\n * `type` reflects what is stored in `data` and can be almost any value-type\n * (except Nullable, Array, Tuple, LowCardinality).\n *\n */\nstruct ItemView {\n    using DataType = std::string_view;\n\n    const Type::Code type;\n    const DataType data;\n\nprivate:\n    template <typename T>\n    \ninline auto ConvertToStorageValue(const T& t) {\n        if constexpr (std::is_same_v<std::string_view, T> || std::is_same_v<std::string, T>) {\n            return std::string_view{t};\n        } else if constexpr (std::is_fundamental_v<T> || std::is_same_v<Int128, std::decay_t<T>>) {\n            return std::string_view{reinterpret_cast<const char*>(&t), sizeof(T)};\n        } else {\n            static_assert(!std::is_same_v<T, T>, \"Unknown type, which can't be stored in ItemView\");\n            return;\n        }\n    }\n\npublic:\n    ItemView(Type::Code type, DataType data)\n        : type(type),\n          data(data)\n    {\n        ValidateData(type, data);\n    }\n\n    ItemView(Type::Code type, ItemView other)\n        : type(type),\n          data(other.data)\n    {\n        ValidateData(type, data);\n    }\n\n    explicit ItemView()\n        : ItemView(Type::Void, std::string_view{})\n    {}\n\n    template <typename T>\n    explicit ItemView(Type::Code type, const T & value)\n        : ItemView(type, ConvertToStorageValue(value))\n    {}\n\n    template <typename T>\n    auto get() const {\n        using ValueType = std::remove_cv_t<std::decay_t<T>>;\n        if constexpr (std::is_same_v<std::string_view, ValueType> || std::is_same_v<std::string, ValueType>) {\n            return data;\n        } else if constexpr (std::is_fundamental_v<ValueType> || std::is_same_v<Int128, ValueType>) {\n            if (sizeof(ValueType) == data.size()) {\n                return *reinterpret_cast<const T*>(data.data());\n            } else {\n                throw AssertionError(\"Incompatitable value type and size. Requested size: \"\n                        + std::to_string(sizeof(ValueType)) + \" stored size: \" + std::to_string(data.size()));\n            }\n        }\n    }\n\n    inline std::string_view AsBinaryData() const {\n        return data;\n    }\n\n    // Validate that value matches type, will throw an exception if validation fails.\n    static void ValidateData(Type::Code type, DataType data);\n};\n\n}\n\n// Path: clickhouse/columns/column.h\n#pragma once\n\n#include \"../types/types.h\"\n#include \"../columns/itemview.h\"\n#include \"../exceptions.h\"\n\n#include <memory>\n#include <stdexcept>\n\nnamespace clickhouse {\n\nclass InputStream;\nclass OutputStream;\n\nusing ColumnRef = std::shared_ptr<class Column>;\n\n/**\n * An abstract base of all columns classes.\n */\nclass Column : public std::enable_shared_from_this<Column> {\npublic:\n    explicit inline Column(TypeRef type) : type_(type) {}\n\n    virtual ~Column() {}\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<T> As() {\n        return std::dynamic_pointer_cast<T>(shared_from_this());\n    }\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<const T> As() const {\n        return std::dynamic_pointer_cast<const T>(shared_from_this());\n    }\n\n    /// Downcast pointer to the specific column's subtype.\n    template <typename T>\n    inline std::shared_ptr<T> AsStrict() {\n        auto result = std::dynamic_pointer_cast<T>(shared_from_this());\n        if (!result) {\n            throw ValidationError(\"Can't cast from \" + type_->GetName());\n        }\n        return result;\n    }\n\n    /// Get type object of the column.\n    inline TypeRef Type() const { return type_; }\n    inline const class Type& GetType() const { return *type_; }\n\n    /// Appends content of given column to the end of current one.\n    virtual void Append(ColumnRef column) = 0;\n\n    /// Increase the capacity of the column for large block insertion.\n    virtual void Reserve(size_t new_cap) = 0;\n\n    /// Template method to load column data from input stream. It'll call LoadPrefix and LoadBody.\n    /// Should be called only once from the client. Derived classes should not call it.\n    bool Load(InputStream* input, size_t rows);\n\n    /// Loads column prefix from input stream.\n    virtual bool LoadPrefix(InputStream* input, size_t rows);\n\n    /// Loads column data from input stream.\n    virtual bool LoadBody(InputStream* input, size_t rows) = 0;\n\n    /// Saves column prefix to output stream. Column types with prefixes must implement it.\n    virtual void SavePrefix(OutputStream* output);\n\n    /// Saves column body to output stream.\n    virtual void SaveBody(OutputStream* output) = 0;\n\n    /// Template method to save to output stream. It'll call SavePrefix and SaveBody respectively\n    /// Should be called only once from the client. Derived classes should not call it.\n    /// Save is split in Prefix and Body because some data types require prefixes and specific serialization order.\n    /// For instance, Array(LowCardinality(X)) requires LowCardinality.key_version bytes to come before Array.offsets\n    void Save(OutputStream* output);\n\n    /// Clear column data .\n    virtual void Clear() = 0;\n\n    /// Returns count of rows in the column.\n    virtual size_t Size() const = 0;\n\n    /// Makes slice of the current column.\n    virtual ColumnRef Slice(size_t begin, size_t len) const = 0;\n\n    virtual ColumnRef CloneEmpty() const = 0;\n\n    virtual void Swap(Column&) = 0;\n\n    /// Get a view on raw item data if it is supported by column, will throw an exception if index is out of range.\n    /// Please note that view is invalidated once column items are added or deleted, column is loaded from strean or destroyed.\n    virtual ItemView GetItem(size_t) const {\n        throw UnimplementedError(\"GetItem() is not supported for column of \" + type_->GetName());\n    }\n\n    friend void swap(Column& left, Column& right) {\n        left.Swap(right);\n    }\n\nprotected:\n    TypeRef type_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/block.h\n#pragma once\n\n#include \"columns/column.h\"\n\nnamespace clickhouse {\n\nstruct BlockInfo {\n    uint8_t is_overflows = 0;\n    int32_t bucket_num = -1;\n};\n\nclass Block {\npublic:\n    /// Allow to iterate over block's columns.\n    class Iterator {\n    public:\n        Iterator(const Block& block);\n\n        /// Name of column.\n        const std::string& Name() const;\n\n        /// Type of column.\n        TypeRef Type() const;\n\n        /// Reference to column object.\n        ColumnRef Column() const;\n\n        /// Move to next column, returns false if next call to IsValid() would return false;\n        bool Next();\n\n        /// Is the iterator still valid.\n        bool IsValid() const;\n\n        size_t ColumnIndex() const {\n            return idx_;\n        }\n\n        Iterator& operator*() { return *this; }\n        const Iterator& operator*() const { return *this; }\n\n        bool operator==(const Iterator & other) const {\n            return &block_ == &other.block_ && idx_ == other.idx_;\n        }\n        bool operator!=(const Iterator & other) const {\n            return !(*this == other);\n        }\n\n        Iterator& operator++() {\n            this->Next();\n            return *this;\n        }\n\n    private:\n        friend class Block;\n        struct ConstructAtEndTag {};\n        Iterator(const Block& block, ConstructAtEndTag at_end);\n        Iterator() = delete;\n\n        const Block& block_;\n        size_t idx_;\n    };\n\npublic:\n     Block();\n     Block(size_t cols, size_t rows);\n    ~Block();\n\n    /// Append named column to the block.\n    void AppendColumn(const std::string& name, const ColumnRef& col);\n\n    /// Count of columns in the block.\n    size_t GetColumnCount() const;\n\n    const BlockInfo& Info() const;\n\n    /// Set block info\n    void SetInfo(BlockInfo info);\n\n    /// Count of rows in the block.\n    size_t GetRowCount() const;\n\n    size_t RefreshRowCount();\n\n    const std::string& GetColumnName(size_t idx) const {\n        return columns_.at(idx).name;\n    }\n\n    /// Reference to column by index in the block.\n    ColumnRef operator [] (size_t idx) const;\n\n    Iterator begin() const;\n    Iterator end() const;\n    Iterator cbegin() const { return begin(); }\n    Iterator cend() const { return end(); }\n\nprivate:\n    struct ColumnItem {\n        std::string name;\n        ColumnRef   column;\n    };\n\n    BlockInfo info_;\n    std::vector<ColumnItem> columns_;\n    /// Count of rows in the block.\n    size_t rows_;\n};\n\n}\n\n// Path: clickhouse/block.cpp\n#include \"block.h\"\n\n#include \"exceptions.h\"\n\n#include <stdexcept>\n\nnamespace clickhouse {\n\nBlock::Iterator::Iterator(const Block& block)\n    : block_(block)\n    , idx_(0)\n{\n}\n\n...\n", "needle_token_start": 3440, "needle_token_end": 3620, "code_context_ntokens": 5891, "output": ["```cpp\ntemplate <>\ninline auto ConvertToStorageValue(const T& t) {\n    if constexpr (std::is_same_v<std::string_view, T> || std::is_same_v<std::string, T>) {\n        return std::string_view{t};\n    } else if constexpr (std::is_fundamental_v<T> || std::is_same_v<Int128, std::decay_t<T>>) {\n        return std::string_view{reinterpret_cast<const char*>(&t), sizeof(T)};\n    } else {\n        static_assert(!std::is_same_v<T, T>, \"Unknown type, which can't be stored in ItemView\");\n        return;\n    }\n}\n```"]}
{"repo": "ClickHouse/clickhouse-cpp", "name": "GetSocketFactory", "language": "cpp", "path": "clickhouse/client.cpp", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create and return an appropriate socket factory based on the security settings specified in the client configuration. It decides whether to create a secure or non-secure socket factory.\n2. **Input**: The function takes a configuration object that includes client options, which may or may not include SSL settings.\n3. **Output**: Returns a unique pointer to a socket factory, which can either be secure or non-secure depending on the input configuration.\n4. **Procedure**: The function checks if SSL options are provided in the input configuration. If SSL options are present and the build supports SSL, it creates and returns a secure socket factory. If no SSL options are provided or the build does not support SSL, it returns a non-secure socket factory.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: clickhouse/columns/lowcardinality.h\n#pragma once\n\n#include \"column.h\"\n#include \"numeric.h\"\n#include \"nullable.h\"\n\n#include <functional>\n#include <string>\n#include <unordered_map>\n#include <utility>\n\nnamespace clickhouse {\n\ntemplate <typename NestedColumnType>\nclass ColumnLowCardinalityT;\n\nnamespace details {\n\n/** LowCardinalityHashKey used as key in unique items hashmap to abstract away key value\n * (type of which depends on dictionary column) and to reduce likelehood of collisions.\n *\n * In order to dramatically reduce collision rate, we use 2 different hashes from 2 different hash functions.\n * First hash is used in hashtable (to calculate item position).\n * Second one is used as part of key value and accessed via `operator==()` upon collision resolution/detection.\n */\nusing LowCardinalityHashKey = std::pair<std::uint64_t, std::uint64_t>;\n\nstruct LowCardinalityHashKeyHash {\n    inline std::size_t operator()(const LowCardinalityHashKey &hash_key) const noexcept {\n        return hash_key.first;\n    }\n};\n\n}\n\n/*\n * LC column contains an \"invisible\" default item at the beginning of the collection. [default, ...]\n * If the nested type is Nullable, it contains a null-item at the beginning and a default item at the second position. [null, default, ...]\n * Null map is not serialized in LC columns. Instead, nulls are tracked by having an index of 0.\n * */\nclass ColumnLowCardinality : public Column {\npublic:\n...\n// Path: clickhouse/base/projected_iterator.h\n#pragma once\n\n#include <iterator>\n#include <type_traits>\n#include <utility>\n\nnamespace clickhouse {\n\ntemplate <typename UnaryFunction, typename Iterator, typename Reference = decltype(std::declval<UnaryFunction>()(std::declval<Iterator>())),\n          typename Value = std::decay_t<Reference>>\nclass ProjectedIterator {\npublic:\n    using value_type        = Value;\n    using reference         = Reference;\n    using pointer           = Reference;\n    using difference_type   = typename std::iterator_traits<Iterator>::difference_type;\n    using iterator_category = typename std::iterator_traits<Iterator>::iterator_category;\n\n    ProjectedIterator() = default;\n\n    inline ProjectedIterator(Iterator const& iterator, UnaryFunction functor)\n        : iterator_(iterator)\n        , functor_(std::move(functor)) {\n    }\n\n    inline UnaryFunction functor() const { return functor; }\n\n    inline Iterator const& base() const { return iterator_; }\n\n    inline reference operator*() const { return functor_(iterator_); }\n\n    inline ProjectedIterator& operator++() {\n        ++iterator_;\n        return *this;\n    }\n\n    inline ProjectedIterator& operator--() {\n        --iterator_;\n        return *this;\n    }\n\n    inline bool operator==(const ProjectedIterator& other) const {\n        return this->iterator_ == other.iterator_;\n    }\n\n    inline bool operator!=(const ProjectedIterator& other) const {\n        return !(*this == other);\n    }\n\nprivate:\n    Iterator iterator_;\n    UnaryFunction functor_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/map.h\n#pragma once\n\n#include \"../base/projected_iterator.h\"\n#include \"array.h\"\n#include \"column.h\"\n#include \"tuple.h\"\n\n#include <functional>\n#include <map>\n\nnamespace clickhouse {\n\ntemplate <typename KeyColumnType, typename ValueColumnType>\nclass ColumnMapT;\n\n/**\n * Represents column of Map(K, V).\n */\nclass ColumnMap : public Column {\npublic:\n    /** Create a map of given type, with actual values and offsets.\n     *\n     *  Both `data` and `offsets` are used (and modified) internally bye ColumnArray.\n     *  Users are strongly advised against modifying contents of `data` or `offsets` afterwards.\n     */\n    explicit ColumnMap(ColumnRef data);\n\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column prefix from input stream.\n    bool LoadPrefix(InputStream* input, size_t rows) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column prefix to output stream.\n    void SavePrefix(OutputStream* output) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t, size_t) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column&) override;\n\n    /// Converts map at pos n to column.\n    /// Type of row is tuple {key, value}.\n    ColumnRef GetAsColumn(size_t n) const;\n\nprotected:\n    template <typename K, typename V>\n    friend class ColumnMapT;\n\n    ColumnMap(ColumnMap&& map);\n\nprivate:\n    std::shared_ptr<ColumnArray> data_;\n};\n\ntemplate <typename K, typename V>\nclass ColumnMapT : public ColumnMap {\npublic:\n    using KeyColumnType   = K;\n    using ValueColumnType = V;\n    using Key             = std::decay_t<decltype(std::declval<KeyColumnType>().At(0))>;\n    using Value           = std::decay_t<decltype(std::declval<ValueColumnType>().At(0))>;\n    using TupleColumnType = ColumnTupleT<KeyColumnType, ValueColumnType>;\n    using ArrayColumnType = ColumnArrayT<TupleColumnType>;\n\n    ColumnMapT(ColumnRef data)\n        : ColumnMap(data), typed_data_(data->AsStrict<ColumnArrayT<TupleColumnType>>()) {}\n\n    ColumnMapT(std::shared_ptr<KeyColumnType> keys, std::shared_ptr<ValueColumnType> values)\n        : ColumnMap(std::make_shared<ArrayColumnType>(std::make_shared<TupleColumnType>(\n              std::make_tuple(std::move(keys), std::move(values))))),\n          typed_data_(data_->template As<ArrayColumnType>()) {}\n\n    ColumnRef Slice(size_t begin, size_t len) const override {\n        return std::make_shared<ColumnMapT<K, V>>(typed_data_->Slice(begin, len));\n    }\n\n    ColumnRef CloneEmpty() const override {\n        return std::make_shared<ColumnMapT<K, V>>(typed_data_->CloneEmpty());\n    }\n\n    void Swap(Column& other) override {\n        auto& col = dynamic_cast<ColumnMapT<K, V>&>(other);\n        col.typed_data_.swap(typed_data_);\n        ColumnMap::Swap(other);\n    }\n\n    /// A single (row) value of the Map-column i.e. read-only map.\n    /// It has a linear time complexity to access items\n    /// Because data base type has same structure\n    /// \"This lookup works now with a linear complexity.\"\n    /// https://clickhouse.com/docs/en/sql-reference/data-types/map\n    /// Convert it to a suitable container required to access more than one element\n\n    class MapValueView {\n        const typename ArrayColumnType::ArrayValueView data_;\n\n    public:\n        using ValueType = std::pair<Key, Value>;\n\n        MapValueView(typename ArrayColumnType::ArrayValueView data) : data_(std::move(data)) {}\n\n        inline auto operator[](const Key& key) const { return (*Find(key)).second; }\n\n        inline auto At(const Key& key) const {\n            auto it = Find(key);\n            if (it == end()) throw ValidationError(\"ColumnMap value key not found\");\n            return (*it).second;\n        }\n\n        class Iterator {\n            typename ArrayColumnType::ArrayValueView::Iterator data_iterator_;\n\n        public:\n            Iterator() = default;\n\n            Iterator(typename ArrayColumnType::ArrayValueView::Iterator data_iterator)\n                : data_iterator_(data_iterator) {}\n\n            using ValueType = std::pair<Key, Value>;\n            using difference_type = size_t;\n            using value_type = ValueType;\n            using pointer = void;\n            using reference = ValueType&;\n            using iterator_category = std::forward_iterator_tag;\n\n            inline auto operator*() const {\n                auto tuple = *data_iterator_;\n                return ValueType{std::get<0>(tuple), std::get<1>(tuple)};\n            }\n\n            inline Iterator& operator++() {\n                ++data_iterator_;\n                return *this;\n            }\n\n            inline bool operator==(const Iterator& other) const {\n                return this->data_iterator_ == other.data_iterator_;\n            }\n\n            inline bool operator!=(const Iterator& other) const { return !(*this == other); }\n        };\n\n        // minimalistic stl-like container interface, hence the lowercase\n        inline Iterator begin() const { return Iterator{data_.begin()}; }\n\n        inline Iterator cbegin() const { return Iterator{data_.cbegin()}; }\n\n        inline Iterator end() const { return Iterator{data_.end()}; }\n\n        inline Iterator cend() const { return Iterator{data_.cend()}; }\n\n        inline size_t size() const { return data_.size(); }\n\n        // It is ugly to have both size() and Size(), but it is for compatitability with both STL\n        // and rest of the clickhouse-cpp.\n        inline size_t Size() const { return data_.Size(); }\n\n        inline size_t Count(const Key& key) const {\n            size_t result = 0;\n            for (auto item : data_) {\n                if (std::get<0>(item) == key) {\n                    ++result;\n                }\n            }\n            return result;\n        }\n\n        inline Iterator Find(const Key& key) const {\n            for (auto it = data_.begin(); it != data_.end(); ++it) {\n                if (std::get<0>(*it) == key) {\n                    return Iterator{it};\n                }\n            }\n            return end();\n        }\n\n        inline bool operator==(const MapValueView& other) const {\n            if (size() != other.size()) {\n                return false;\n            }\n            const auto make_index = [](const auto& data) {\n                std::vector<size_t> result{data.Size()};\n                std::generate(result.begin(), result.end(), [i = 0] () mutable {return i++;});\n                std::sort(result.begin(), result.end(), [&data](size_t l, size_t r) {return data[l] < data[r];});\n                return result;\n            };\n            const auto index = make_index(data_);\n            for (const auto& val : other.data_) {\n                if (!std::binary_search(index.begin(), index.end(), val,\n                        [&data = data_](const auto& l, size_t r) {return l < data[r];})) {\n                    return false;\n                }\n            }\n            return true;\n        }\n\n        inline bool operator!=(const MapValueView& other) const { return !(*this == other); }\n    };\n\n    inline auto At(size_t index) const { return MapValueView{typed_data_->At(index)}; }\n\n    inline auto operator[](size_t index) const { return At(index); }\n\n    using ColumnMap::Append;\n\n    inline void Append(const MapValueView& value) { typed_data_->Append(value.data_); }\n\n    inline void Append(const std::vector<std::tuple<Key, Value>>& tuples) {\n        typed_data_->Append(tuples.begin(), tuples.end());\n    }\n\n    template <typename T>\n    inline void Append(const T& value) {\n        using BaseIter = decltype(value.begin());\n        using KeyOfT = decltype(std::declval<BaseIter>()->first);\n        using ValOfT = decltype(std::declval<BaseIter>()->second);\n        using Functor = std::function<std::tuple<KeyOfT, ValOfT>(const BaseIter&)>;\n        using Iterator = ProjectedIterator<Functor, BaseIter>;\n\n        Functor functor = [](const BaseIter& i) {\n            return std::make_tuple(std::cref(i->first), std::cref(i->second));\n        };\n\n        typed_data_->Append(Iterator{value.begin(), functor}, Iterator{value.end(), functor});\n    }\n\n    static auto Wrap(ColumnMap&& col) {\n        auto data = ArrayColumnType::Wrap(std::move(col.data_));\n        return std::make_shared<ColumnMapT<K, V>>(std::move(data));\n    }\n\n    static auto Wrap(Column&& col) { return Wrap(std::move(dynamic_cast<ColumnMap&&>(col))); }\n\n    // Helper to simplify integration with other APIs\n    static auto Wrap(ColumnRef&& col) { return Wrap(std::move(*col->AsStrict<ColumnMap>())); }\n\nprivate:\n    std::shared_ptr<ArrayColumnType> typed_data_;\n};\n\n}  // namespace clickhouse\n\n// Path: clickhouse/columns/uuid.h\n#pragma once\n\n#include \"../base/uuid.h\"\n#include \"column.h\"\n#include \"numeric.h\"\n\nnamespace clickhouse {\n\n\n/**\n * Represents a UUID column.\n */\nclass ColumnUUID : public Column {\npublic:\n    ColumnUUID();\n\n    explicit ColumnUUID(ColumnRef data);\n\n    /// Appends one element to the end of column.\n    void Append(const UUID& value);\n\n    /// Returns element at given row number.\n    const UUID At(size_t n) const;\n\n    /// Returns element at given row number.\n    inline const UUID operator [] (size_t n) const { return At(n); }\n\npublic:\n    /// Increase the capacity of the column for large block insertion.\n    void Reserve(size_t new_cap) override;\n\n    /// Appends content of given column to the end of current one.\n    void Append(ColumnRef column) override;\n\n    /// Loads column data from input stream.\n    bool LoadBody(InputStream* input, size_t rows) override;\n\n    /// Saves column data to output stream.\n    void SaveBody(OutputStream* output) override;\n\n    /// Clear column data .\n    void Clear() override;\n\n    /// Returns count of rows in the column.\n    size_t Size() const override;\n\n    /// Makes slice of the current column.\n    ColumnRef Slice(size_t begin, size_t len) const override;\n    ColumnRef CloneEmpty() const override;\n    void Swap(Column& other) override;\n\n    ItemView GetItem(size_t) const override;\n\nprivate:\n    std::shared_ptr<ColumnUInt64> data_;\n};\n\n}\n\n// Path: clickhouse/client.h\n#pragma once\n\n#include \"query.h\"\n#include \"exceptions.h\"\n\n#include \"columns/array.h\"\n#include \"columns/date.h\"\n#include \"columns/decimal.h\"\n#include \"columns/enum.h\"\n#include \"columns/geo.h\"\n#include \"columns/ip4.h\"\n#include \"columns/ip6.h\"\n#include \"columns/lowcardinality.h\"\n#include \"columns/nullable.h\"\n#include \"columns/numeric.h\"\n#include \"columns/map.h\"\n#include \"columns/string.h\"\n#include \"columns/tuple.h\"\n#include \"columns/uuid.h\"\n\n#include <chrono>\n#include <cstdint>\n#include <memory>\n#include <ostream>\n#include <string>\n#include <optional>\n\ntypedef struct ssl_ctx_st SSL_CTX;\n\nnamespace clickhouse {\n\nstruct ServerInfo {\n    std::string name;\n    std::string timezone;\n    std::string display_name;\n    uint64_t    version_major;\n    uint64_t    version_minor;\n    uint64_t    version_patch;\n    uint64_t    revision;\n};\n\n/// Methods of block compression.\nenum class CompressionMethod {\n    None    = -1,\n    LZ4     =  1,\n};\n\nstruct Endpoint {\n    std::string host;\n    uint16_t port = 9000;\n    inline bool operator==(const Endpoint& right) const {\n        return host == right.host && port == right.port;\n    }\n};\n\nenum class EndpointsIterationAlgorithm {\n    RoundRobin = 0,\n};\n\nstruct ClientOptions {\n    // Setter goes first, so it is possible to apply 'deprecated' annotation safely.\n#define DECLARE_FIELD(name, type, setter, default_value) \\\n    inline auto & setter(const type& value) { \\\n        name = value; \\\n        return *this; \\\n    } \\\n    type name = default_value\n\n    /// Hostname of the server.\n    DECLARE_FIELD(host, std::string, SetHost, std::string());\n    /// Service port.\n    DECLARE_FIELD(port, uint16_t, SetPort, 9000);\n\n    /** Set endpoints (host+port), only one is used.\n     * Client tries to connect to those endpoints one by one, on the round-robin basis:\n     * first default enpoint (set via SetHost() + SetPort()), then each of endpoints, from begin() to end(),\n     * the first one to establish connection is used for the rest of the session.\n     * If port isn't specified, default(9000) value will be used.\n     */\n    DECLARE_FIELD(endpoints, std::vector<Endpoint>, SetEndpoints, {});\n\n    /// Default database.\n    DECLARE_FIELD(default_database, std::string, SetDefaultDatabase, \"default\");\n    /// User name.\n    DECLARE_FIELD(user, std::string, SetUser, \"default\");\n    /// Access password.\n    DECLARE_FIELD(password, std::string, SetPassword, std::string());\n\n    /// By default all exceptions received during query execution will be\n    /// passed to OnException handler.  Set rethrow_exceptions to true to\n    /// enable throwing exceptions with standard c++ exception mechanism.\n    DECLARE_FIELD(rethrow_exceptions, bool, SetRethrowException, true);\n\n    /// Ping server every time before execute any query.\n    DECLARE_FIELD(ping_before_query, bool, SetPingBeforeQuery, false);\n    /// Count of retry to send request to server.\n    DECLARE_FIELD(send_retries, unsigned int, SetSendRetries, 1);\n    /// Amount of time to wait before next retry.\n    DECLARE_FIELD(retry_timeout, std::chrono::seconds, SetRetryTimeout, std::chrono::seconds(5));\n\n    /// Compression method.\n    DECLARE_FIELD(compression_method, CompressionMethod, SetCompressionMethod, CompressionMethod::None);\n\n    /// TCP Keep alive options\n    DECLARE_FIELD(tcp_keepalive, bool, TcpKeepAlive, false);\n    DECLARE_FIELD(tcp_keepalive_idle, std::chrono::seconds, SetTcpKeepAliveIdle, std::chrono::seconds(60));\n    DECLARE_FIELD(tcp_keepalive_intvl, std::chrono::seconds, SetTcpKeepAliveInterval, std::chrono::seconds(5));\n    DECLARE_FIELD(tcp_keepalive_cnt, unsigned int, SetTcpKeepAliveCount, 3);\n\n    // TCP options\n    DECLARE_FIELD(tcp_nodelay, bool, TcpNoDelay, true);\n\n    /// Connection socket connect timeout. If the timeout is negative then the connect operation will never timeout.\n    DECLARE_FIELD(connection_connect_timeout, std::chrono::milliseconds, SetConnectionConnectTimeout, std::chrono::seconds(5));\n\n    /// Connection socket timeout. If the timeout is set to zero then the operation will never timeout.\n    DECLARE_FIELD(connection_recv_timeout, std::chrono::milliseconds, SetConnectionRecvTimeout, std::chrono::milliseconds(0));\n    DECLARE_FIELD(connection_send_timeout, std::chrono::milliseconds, SetConnectionSendTimeout, std::chrono::milliseconds(0));\n\n    /** It helps to ease migration of the old codebases, which can't afford to switch\n    * to using ColumnLowCardinalityT or ColumnLowCardinality directly,\n    * but still want to benefit from smaller on-wire LowCardinality bandwidth footprint.\n    *\n    * @see LowCardinalitySerializationAdaptor, CreateColumnByType\n    */\n    [[deprecated(\"Makes implementation of LC(X) harder and code uglier. Will be removed in next major release (3.0) \")]]\n    DECLARE_FIELD(backward_compatibility_lowcardinality_as_wrapped_column, bool, SetBakcwardCompatibilityFeatureLowCardinalityAsWrappedColumn, false);\n\n    /** Set max size data to compress if compression enabled.\n     *\n     *  Allows choosing tradeoff between RAM\\CPU:\n     *  - Lower value reduces RAM usage, but slightly increases CPU usage.\n     *  - Higher value increases RAM usage but slightly decreases CPU usage.\n     */\n    DECLARE_FIELD(max_compression_chunk_size, unsigned int, SetMaxCompressionChunkSize, 65535);\n\n    struct SSLOptions {\n        /** There are two ways to configure an SSL connection:\n         *  - provide a pre-configured SSL_CTX, which is not modified and not owned by the Client.\n         *  - provide a set of options and allow the Client to create and configure SSL_CTX by itself.\n         */\n\n        /** Pre-configured SSL-context for SSL-connection.\n         *  If NOT null client DONES NOT take ownership of context and it must be valid for client lifetime.\n         *  If null client initlaizes OpenSSL and creates his own context, initializes it using\n         *  other options, like path_to_ca_files, path_to_ca_directory, use_default_ca_locations, etc.\n         *\n         *  Either way context is used to create an SSL-connection, which is then configured with\n         *  whatever was provided as `configuration`, `host_flags`, `skip_verification` and `use_sni`.\n         */\n        SSL_CTX * ssl_context = nullptr;\n        auto & SetExternalSSLContext(SSL_CTX * new_ssl_context) {\n            ssl_context = new_ssl_context;\n            return *this;\n        }\n\n        /** Means to validate the server-supplied certificate against trusted Certificate Authority (CA).\n         *  If no CAs are configured, the server's identity can't be validated, and the Client would err.\n         *  See https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_default_verify_paths.html\n        */\n        /// Load default CA certificates from default locations.\n        DECLARE_FIELD(use_default_ca_locations, bool, SetUseDefaultCALocations, true);\n        /// Path to the CA files to verify server certificate, may be empty.\n        DECLARE_FIELD(path_to_ca_files, std::vector<std::string>, SetPathToCAFiles, {});\n        /// Path to the directory with CA files used to validate server certificate, may be empty.\n        DECLARE_FIELD(path_to_ca_directory, std::string, SetPathToCADirectory, \"\");\n\n        /** Min and max protocol versions to use, set with SSL_CTX_set_min_proto_version and SSL_CTX_set_max_proto_version\n         *  for details see https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_min_proto_version.html\n         */\n        DECLARE_FIELD(min_protocol_version, int, SetMinProtocolVersion, DEFAULT_VALUE);\n        DECLARE_FIELD(max_protocol_version, int, SetMaxProtocolVersion, DEFAULT_VALUE);\n\n        /** Options to be set with SSL_CTX_set_options,\n         * for details see https://www.openssl.org/docs/man1.1.1/man3/SSL_CTX_set_options.html\n        */\n        DECLARE_FIELD(context_options, int, SetContextOptions, DEFAULT_VALUE);\n\n        /** Use SNI at ClientHello\n         */\n        DECLARE_FIELD(use_sni, bool, SetUseSNI, true);\n\n        /** Skip SSL session verification (server's certificate, etc).\n         *\n         *  WARNING: settig to true will bypass all SSL session checks, which\n         *  is dangerous, but can be used against self-signed certificates, e.g. for testing purposes.\n         */\n        DECLARE_FIELD(skip_verification, bool, SetSkipVerification, false);\n\n        /** Mode of verifying host ssl certificate against name of the host, set with SSL_set_hostflags.\n         *  For details see https://www.openssl.org/docs/man1.1.1/man3/SSL_set_hostflags.html\n         */\n        DECLARE_FIELD(host_flags, int, SetHostVerifyFlags, DEFAULT_VALUE);\n\n        struct CommandAndValue {\n            std::string command;\n            std::optional<std::string> value = std::nullopt;\n        };\n        /** Extra configuration options, set with SSL_CONF_cmd.\n         *  For deatils see https://www.openssl.org/docs/man1.1.1/man3/SSL_CONF_cmd.html\n         *\n         *  Takes multiple pairs of command-value strings, all commands are supported,\n         *  and prefix is empty.\n         *  i.e. pass `sigalgs` or `SignatureAlgorithms` instead of `-sigalgs`.\n         *\n         *  Rewrites any other options/flags if set in other ways.\n         */\n        DECLARE_FIELD(configuration, std::vector<CommandAndValue>, SetConfiguration, {});\n\n        static const int DEFAULT_VALUE = -1;\n    };\n\n    // By default SSL is turned off.\n    std::optional<SSLOptions> ssl_options = std::nullopt;\n\n    // Will throw an exception if client was built without SSL support.\n    ClientOptions& SetSSLOptions(SSLOptions options);\n\n#undef DECLARE_FIELD\n};\n\nstd::ostream& operator<<(std::ostream& os, const ClientOptions& options);\nstd::ostream& operator<<(std::ostream& os, const Endpoint& options);\n\nclass SocketFactory;\n\n/**\n *\n */\nclass Client {\npublic:\n     Client(const ClientOptions& opts);\n     Client(const ClientOptions& opts,\n            std::unique_ptr<SocketFactory> socket_factory);\n    ~Client();\n\n    /// Intends for execute arbitrary queries.\n    void Execute(const Query& query);\n\n    /// Intends for execute select queries.  Data will be returned with\n    /// one or more call of \\p cb.\n    void Select(const std::string& query, SelectCallback cb);\n    void Select(const std::string& query, const std::string& query_id, SelectCallback cb);\n\n    /// Executes a select query which can be canceled by returning false from\n    /// the data handler function \\p cb.\n    void SelectCancelable(const std::string& query, SelectCancelableCallback cb);\n    void SelectCancelable(const std::string& query, const std::string& query_id, SelectCancelableCallback cb);\n\n    /// Alias for Execute.\n    void Select(const Query& query);\n\n    /// Intends for insert block of data into a table \\p table_name.\n    void Insert(const std::string& table_name, const Block& block);\n    void Insert(const std::string& table_name, const std::string& query_id, const Block& block);\n\n    /// Ping server for aliveness.\n    void Ping();\n\n    /// Reset connection with initial params.\n    void ResetConnection();\n\n    const ServerInfo& GetServerInfo() const;\n\n    /// Get current connected endpoint.\n    /// In case when client is not connected to any endpoint, nullopt will returned.\n    const std::optional<Endpoint>& GetCurrentEndpoint() const;\n\n    // Try to connect to different endpoints one by one only one time. If it doesn't work, throw an exception.\n    void ResetConnectionEndpoint();\n\n    struct Version\n    {\n        uint16_t major;\n        uint16_t minor;\n        uint16_t patch;\n        uint16_t build;\n        const char * extra;\n    };\n\n    static Version GetVersion();\n\nprivate:\n    const ClientOptions options_;\n\n    class Impl;\n    std::unique_ptr<Impl> impl_;\n};\n\n}\n\n// Path: clickhouse/protocol.h\n#pragma once\n\nnamespace clickhouse {\n\n    /// Types of packets received from server\n    namespace ServerCodes {\n        enum {\n            Hello                = 0,    /// Name, version, revision.\n            Data                 = 1,    /// `Block` of data, may be compressed.\n            Exception            = 2,    /// Exception that occurred on server side during query execution.\n            Progress             = 3,    /// Query execcution progress: rows and bytes read.\n            Pong                 = 4,    /// response to Ping sent by client.\n            EndOfStream          = 5,    /// All packets were sent.\n            ProfileInfo          = 6,    /// Profiling data\n            Totals               = 7,    /// Block of totals, may be compressed.\n            Extremes             = 8,    /// Block of mins and maxs, may be compressed.\n            TablesStatusResponse = 9,    /// Response to TableStatus.\n            Log                  = 10,   /// Query execution log.\n            TableColumns         = 11,   /// Columns' description for default values calculation\n            PartUUIDs            = 12,   /// List of unique parts ids.\n            ReadTaskRequest      = 13,   /// String (UUID) describes a request for which next task is needed\n                                         /// This is such an inverted logic, where server sends requests\n                                         /// And client returns back response\n            ProfileEvents        = 14,   /// Packet with profile events from server.\n        };\n    }\n\n    /// Types of packets sent by client.\n    namespace ClientCodes {\n        enum {\n            Hello       = 0,    /// Name, version, default database name.\n            Query       = 1,    /** Query id, query settings, query processing stage,\n                                  * compression status, and query text (no INSERT data).\n                                  */\n            Data        = 2,    /// Data `Block` (e.g. INSERT data), may be compressed.\n            Cancel      = 3,    /// Cancel query.\n            Ping        = 4,    /// Check server connection.\n        };\n    }\n\n    /// Should we compress `Block`s of data\n    namespace CompressionState {\n        enum {\n            Disable     = 0,\n            Enable      = 1,\n        };\n    }\n\n    namespace Stages {\n        enum {\n            Complete    = 2,\n        };\n    }\n}\n\n// Path: clickhouse/base/input.h\n#pragma once\n\n#include <cstddef>\n#include <cstdint>\n#include <vector>\n#include <memory>\n\nnamespace clickhouse {\n\nclass InputStream {\npublic:\n    virtual ~InputStream() noexcept (false)\n    { }\n\n    /// Reads one byte from the stream.\n    inline bool ReadByte(uint8_t* byte) {\n        return DoRead(byte, sizeof(uint8_t)) == sizeof(uint8_t);\n    }\n\n    /// Reads some data from the stream.\n    inline size_t Read(void* buf, size_t len) {\n        return DoRead(buf, len);\n    }\n\n    // Skips a number of bytes.  Returns false if an underlying read error occurs.\n    virtual bool Skip(size_t bytes) = 0;\n\nprotected:\n    virtual size_t DoRead(void* buf, size_t len) = 0;\n};\n\n\nclass ZeroCopyInput : public InputStream {\npublic:\n    inline size_t Next(const void** buf, size_t len) {\n        return DoNext(buf, len);\n    }\n\n    bool Skip(size_t bytes) override;\n\nprotected:\n    virtual size_t DoNext(const void** ptr, size_t len) = 0;\n\n    size_t DoRead(void* buf, size_t len) override;\n};\n\n\n/**\n * A ZeroCopyInput stream backed by an in-memory array of bytes.\n */\nclass ArrayInput : public ZeroCopyInput {\npublic:\n     ArrayInput() noexcept;\n     ArrayInput(const void* buf, size_t len) noexcept;\n    ~ArrayInput() override;\n\n    /// Number of bytes available in the stream.\n    inline size_t Avail() const noexcept {\n        return len_;\n    }\n\n    /// Current read position in the memory block used by this stream.\n    inline const uint8_t* Data() const noexcept {\n        return data_;\n    }\n\n    /// Whether there is more data in the stream.\n    inline bool Exhausted() const noexcept {\n        return !Avail();\n    }\n\n    inline void Reset(const void* buf, size_t len) noexcept {\n        data_ = static_cast<const uint8_t*>(buf);\n        len_ = len;\n    }\n\nprivate:\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    const uint8_t* data_;\n    size_t len_;\n};\n\n\nclass BufferedInput : public ZeroCopyInput {\npublic:\n    BufferedInput(std::unique_ptr<InputStream> source, size_t buflen = 8192);\n    ~BufferedInput() override;\n\n    void Reset();\n\nprotected:\n    size_t DoRead(void* buf, size_t len) override;\n    size_t DoNext(const void** ptr, size_t len) override;\n\nprivate:\n    std::unique_ptr<InputStream> const source_;\n    ArrayInput array_input_;\n    std::vector<uint8_t> buffer_;\n};\n\n}\n\n// Path: clickhouse/base/buffer.h\n#pragma once\n\n#include <cstdint>\n#include <vector>\n\nnamespace clickhouse {\n\nusing Buffer = std::vector<uint8_t>;\n\n}\n\n// Path: clickhouse/base/output.h\n#pragma once\n\n#include \"buffer.h\"\n\n#include <cstddef>\n#include <cstdint>\n#include <vector>\n#include <memory.h>\n#include <memory>\n\nnamespace clickhouse {\n\nclass OutputStream {\npublic:\n    virtual ~OutputStream()\n    { }\n\n    inline void Flush() {\n        DoFlush();\n    }\n\n    inline size_t Write(const void* data, size_t len) {\n        return DoWrite(data, len);\n    }\n\nprotected:\n    virtual void DoFlush() { }\n\n    virtual size_t DoWrite(const void* data, size_t len) = 0;\n};\n\n\nclass ZeroCopyOutput : public OutputStream {\npublic:\n    inline size_t Next(void** data, size_t size) {\n        return DoNext(data, size);\n    }\n\nprotected:\n    // Obtains a buffer into which data can be written.  Any data written\n    // into this buffer will eventually (maybe instantly, maybe later on)\n    // be written to the output.\n    virtual size_t DoNext(void** data, size_t len) = 0;\n\n    size_t DoWrite(const void* data, size_t len) override;\n};\n\n\n/**\n * A ZeroCopyOutput stream backed by an in-memory array of bytes.\n */\nclass ArrayOutput : public ZeroCopyOutput {\npublic:\n     ArrayOutput(void* buf, size_t len);\n    ~ArrayOutput() override;\n\n    /// Number of bytes available in the stream.\n    inline size_t Avail() const noexcept {\n        return end_ - buf_;\n    }\n\n    /// Current write position in the memory block used by this stream.\n    inline const uint8_t* Data() const noexcept {\n        return buf_;\n    }\n\n    /// Whether there is more space in the stream.\n    inline bool Exhausted() const noexcept {\n        return !Avail();\n    }\n\n    /// Initializes this stream with a new memory block.\n    inline void Reset(void* buf, size_t len) noexcept {\n        buf_ = static_cast<uint8_t*>(buf);\n        end_ = buf_ + len;\n        buffer_size_ = len;\n    }\n\n    /// Number of bytes written to the buffer.\n    inline size_t Size() const noexcept {\n        return buffer_size_ - Avail();\n    }\n\nprotected:\n    size_t DoNext(void** data, size_t len) override;\n\nprivate:\n    uint8_t* buf_;\n    uint8_t* end_;\n    size_t buffer_size_;\n};\n\n\n/**\n * A ZeroCopyOutput stream backed by a vector.\n *\n * Doesn't Flush() in destructor, client must ensure to do it manually at some point.\n */\nclass BufferOutput : public ZeroCopyOutput {\npublic:\n     BufferOutput(Buffer* buf);\n    ~BufferOutput() override;\n\nprotected:\n    size_t DoNext(void** data, size_t len) override;\n\nprivate:\n    Buffer* buf_;\n    size_t pos_;\n};\n\n/** BufferedOutput writes data to internal buffer first.\n *\n *  Any data goes to underlying stream only if internal buffer is full\n *  or when client invokes Flush() on this.\n *\n * Doesn't Flush() in destructor, client must ensure to do it manually at some point.\n */\nclass BufferedOutput : public ZeroCopyOutput {\npublic:\n    explicit BufferedOutput(std::unique_ptr<OutputStream> destination, size_t buflen = 8192);\n    ~BufferedOutput() override;\n\n    void Reset();\n\nprotected:\n    void DoFlush() override;\n    size_t DoNext(void** data, size_t len) override;\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    std::unique_ptr<OutputStream> const destination_;\n    Buffer buffer_;\n    ArrayOutput array_output_;\n};\n\ntemplate <typename T>\nvoid WriteUnaligned(void* buf, const T& value) {\n    memcpy(buf, &value, sizeof(value));\n}\n\n}\n\n// Path: clickhouse/base/compressed.h\n#pragma once\n\n#include \"input.h\"\n#include \"output.h\"\n#include \"buffer.h\"\n\nnamespace clickhouse {\n\nclass CompressedInput : public ZeroCopyInput {\npublic:\n    explicit CompressedInput(InputStream* input);\n    ~CompressedInput() override;\n\nprotected:\n    size_t DoNext(const void** ptr, size_t len) override;\n\n    bool Decompress();\n\nprivate:\n    InputStream* const input_;\n\n    Buffer data_;\n    ArrayInput mem_;\n};\n\nclass CompressedOutput : public OutputStream {\npublic:\n    explicit CompressedOutput(OutputStream * destination, size_t max_compressed_chunk_size = 0);\n    ~CompressedOutput() override;\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n    void DoFlush() override;\n\nprivate:\n    void Compress(const void * data, size_t len);\n    void PreallocateCompressBuffer(size_t input_size);\n\nprivate:\n    OutputStream * destination_;\n    const size_t max_compressed_chunk_size_;\n    Buffer compressed_buffer_;\n};\n\n}\n\n// Path: clickhouse/base/platform.h\n#pragma once\n\n#if defined(__linux__)\n#   define _linux_\n#elif defined(_WIN64)\n#   define _win64_\n#   define _win32_\n#elif defined(__WIN32__) || defined(_WIN32)\n#   define _win32_\n#elif defined(__APPLE__)\n#   define _darwin_\n#endif\n\n#if defined(_win32_) || defined(_win64_)\n#   define _win_\n#   if !defined(_WIN32_WINNT) || (_WIN32_WINNT < 0x0600)\n#      undef _WIN32_WINNT\n#      define _WIN32_WINNT 0x0600 // The WSAPoll function is defined on Windows Vista and later.\n#   endif\n#   define WIN32_LEAN_AND_MEAN 1  // don't include too much header automatically\n#endif\n\n#if defined(_linux_) || defined (_darwin_)\n#   define _unix_\n#endif\n\n#if defined(_MSC_VER)\n#   undef NOMINMAX\n#   define NOMINMAX\n#   include <basetsd.h>\n#   define ssize_t SSIZE_T\n#   define HAVE_SSIZE_T 1\n#endif\n\n// Path: clickhouse/base/endpoints_iterator.h\n#pragma once\n\n#include \"clickhouse/client.h\"\n#include <vector>\n\nnamespace clickhouse {\n\nstruct ClientOptions;\n\n/**\n * Base class for iterating through endpoints.\n*/\nclass EndpointsIteratorBase\n{\n public:\n   virtual ~EndpointsIteratorBase() = default;\n\n   virtual Endpoint Next() = 0;\n};\n\nclass RoundRobinEndpointsIterator : public EndpointsIteratorBase\n{\n public:\n    explicit RoundRobinEndpointsIterator(const std::vector<Endpoint>& opts);\n    Endpoint Next() override;\n\n    ~RoundRobinEndpointsIterator() override;\n\n private:\n    const std::vector<Endpoint>& endpoints;\n    size_t current_index;\n};\n\n}\n\n// Path: clickhouse/base/socket.h\n#pragma once\n\n#include \"platform.h\"\n#include \"input.h\"\n#include \"output.h\"\n#include \"endpoints_iterator.h\"\n\n#include <cstddef>\n#include <string>\n#include <chrono>\n\n#if defined(_win_)\n#   include <winsock2.h>\n#   include <ws2tcpip.h>\n#else\n#   include <arpa/inet.h>\n#   include <sys/types.h>\n#   include <sys/socket.h>\n#   include <poll.h>\n\n#   if !defined(SOCKET)\n#       define SOCKET int\n#   endif\n#endif\n\n#include <memory>\n#include <system_error>\n\nstruct addrinfo;\n\nnamespace clickhouse {\n\nstruct ClientOptions;\n\n/** Address of a host to establish connection to.\n *\n */\nclass NetworkAddress {\npublic:\n    explicit NetworkAddress(const std::string& host,\n                            const std::string& port = \"0\");\n    ~NetworkAddress();\n\n    const struct addrinfo* Info() const;\n    const std::string & Host() const;\n\nprivate:\n    const std::string host_;\n    struct addrinfo* info_;\n};\n\n#if defined(_win_)\n\nclass windowsErrorCategory : public std::error_category {\npublic:\n    char const* name() const noexcept override final;\n    std::string message(int c) const override final;\n\n    static windowsErrorCategory const& category();\n};\n\n#endif\n\n#if defined(_unix_)\n\nclass getaddrinfoErrorCategory : public std::error_category {\npublic:\n    char const* name() const noexcept override final;\n    std::string message(int c) const override final;\n\n    static getaddrinfoErrorCategory const& category();\n};\n\n#endif\n\n\nclass SocketBase {\npublic:\n    virtual ~SocketBase();\n\n    virtual std::unique_ptr<InputStream> makeInputStream() const = 0;\n    virtual std::unique_ptr<OutputStream> makeOutputStream() const = 0;\n};\n\n\nclass SocketFactory {\npublic:\n    virtual ~SocketFactory();\n\n    // TODO: move connection-related options to ConnectionOptions structure.\n\n    virtual std::unique_ptr<SocketBase> connect(const ClientOptions& opts, const Endpoint& endpoint) = 0;\n\n    virtual void sleepFor(const std::chrono::milliseconds& duration);\n};\n\n\nstruct SocketTimeoutParams {\n    std::chrono::milliseconds connect_timeout{ 5000 };\n    std::chrono::milliseconds recv_timeout{ 0 };\n    std::chrono::milliseconds send_timeout{ 0 };\n};\n\nclass Socket : public SocketBase {\npublic:\n    Socket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params);\n    Socket(const NetworkAddress& addr);\n    Socket(Socket&& other) noexcept;\n    Socket& operator=(Socket&& other) noexcept;\n\n    ~Socket() override;\n\n    /// @params idle the time (in seconds) the connection needs to remain\n    ///         idle before TCP starts sending keepalive probes.\n    /// @params intvl the time (in seconds) between individual keepalive probes.\n    /// @params cnt the maximum number of keepalive probes TCP should send\n    ///         before dropping the connection.\n    void SetTcpKeepAlive(int idle, int intvl, int cnt) noexcept;\n\n    /// @params nodelay whether to enable TCP_NODELAY\n    void SetTcpNoDelay(bool nodelay) noexcept;\n\n    std::unique_ptr<InputStream> makeInputStream() const override;\n    std::unique_ptr<OutputStream> makeOutputStream() const override;\n\nprotected:\n    Socket(const Socket&) = delete;\n    Socket& operator = (const Socket&) = delete;\n    void Close();\n\n    SOCKET handle_;\n};\n\n\nclass NonSecureSocketFactory : public SocketFactory {\npublic:\n    ~NonSecureSocketFactory() override;\n\n    std::unique_ptr<SocketBase> connect(const ClientOptions& opts, const Endpoint& endpoint) override;\n\nprotected:\n    virtual std::unique_ptr<Socket> doConnect(const NetworkAddress& address, const ClientOptions& opts);\n\n    void setSocketOptions(Socket& socket, const ClientOptions& opts);\n};\n\n\nclass SocketInput : public InputStream {\npublic:\n    explicit SocketInput(SOCKET s);\n    ~SocketInput();\n\nprotected:\n    bool Skip(size_t bytes) override;\n    size_t DoRead(void* buf, size_t len) override;\n\nprivate:\n    SOCKET s_;\n};\n\nclass SocketOutput : public OutputStream {\npublic:\n    explicit SocketOutput(SOCKET s);\n    ~SocketOutput();\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    SOCKET s_;\n};\n\nstatic struct NetrworkInitializer {\n    NetrworkInitializer();\n} gNetrworkInitializer;\n\n}\n\n// Path: clickhouse/base/wire_format.h\n#pragma once\n\n#include <string>\n#include <cstdint>\n\nnamespace clickhouse {\n\nclass InputStream;\nclass OutputStream;\n\nclass WireFormat {\npublic:\n    template <typename T>\n    static bool ReadFixed(InputStream& input, T* value);\n    static bool ReadString(InputStream& input, std::string* value);\n    static bool SkipString(InputStream& input);\n    static bool ReadBytes(InputStream& input, void* buf, size_t len);\n    static bool ReadUInt64(InputStream& input, uint64_t* value);\n    static bool ReadVarint64(InputStream& output, uint64_t* value);\n\n    template <typename T>\n    static void WriteFixed(OutputStream& output, const T& value);\n    static void WriteBytes(OutputStream& output, const void* buf, size_t len);\n    static void WriteString(OutputStream& output, std::string_view value);\n    static void WriteUInt64(OutputStream& output, const uint64_t value);\n    static void WriteVarint64(OutputStream& output, uint64_t value);\n\nprivate:\n    static bool ReadAll(InputStream& input, void* buf, size_t len);\n    static void WriteAll(OutputStream& output, const void* buf, size_t len);\n};\n\ntemplate <typename T>\ninline bool WireFormat::ReadFixed(InputStream& input, T* value) {\n    return ReadAll(input, value, sizeof(T));\n}\n\ninline bool WireFormat::ReadString(InputStream& input, std::string* value) {\n    uint64_t len = 0;\n    if (ReadVarint64(input, &len)) {\n        if (len > 0x00FFFFFFULL) {\n            return false;\n        }\n        value->resize((size_t)len);\n        return ReadAll(input, value->data(), (size_t)len);\n    }\n\n    return false;\n}\n\ninline bool WireFormat::ReadBytes(InputStream& input, void* buf, size_t len) {\n    return ReadAll(input, buf, len);\n}\n\ninline bool WireFormat::ReadUInt64(InputStream& input, uint64_t* value) {\n    return ReadVarint64(input, value);\n}\n\ntemplate <typename T>\ninline void WireFormat::WriteFixed(OutputStream& output, const T& value) {\n    WriteAll(output, &value, sizeof(T));\n}\n\ninline void WireFormat::WriteBytes(OutputStream& output, const void* buf, size_t len) {\n    WriteAll(output, buf, len);\n}\n\ninline void WireFormat::WriteString(OutputStream& output, std::string_view value) {\n    WriteVarint64(output, value.size());\n    WriteAll(output, value.data(), value.size());\n}\n\ninline void WireFormat::WriteUInt64(OutputStream& output, const uint64_t value) {\n    WriteVarint64(output, value);\n}\n\n}\n\n// Path: clickhouse/columns/factory.h\n#pragma once\n\n#include \"column.h\"\n\nnamespace clickhouse {\n\nstruct CreateColumnByTypeSettings\n{\n    bool low_cardinality_as_wrapped_column = false;\n};\n\nColumnRef CreateColumnByType(const std::string& type_name, CreateColumnByTypeSettings settings = {});\n\n}\n\n// Path: clickhouse/base/sslsocket.h\n#pragma once\n\n#include \"socket.h\"\n\n#include <memory>\n#include <optional>\n#include <vector>\n\ntypedef struct ssl_ctx_st SSL_CTX;\ntypedef struct ssl_st SSL;\n\nnamespace clickhouse {\n\nstruct SSLParams\n{\n    std::vector<std::string> path_to_ca_files;\n    std::string path_to_ca_directory;\n    bool use_default_ca_locations;\n    int context_options;\n    int min_protocol_version;\n    int max_protocol_version;\n    bool use_SNI;\n    bool skip_verification;\n    int host_flags;\n    using ConfigurationType = std::vector<std::pair<std::string, std::optional<std::string>>>;\n    ConfigurationType configuration;\n};\n\nclass SSLContext\n{\npublic:\n    explicit SSLContext(SSL_CTX & context);\n    explicit SSLContext(const SSLParams & context_params);\n    ~SSLContext() = default;\n\n    SSLContext(const SSLContext &) = delete;\n    SSLContext& operator=(const SSLContext &) = delete;\n    SSLContext(SSLContext &&) = delete;\n    SSLContext& operator=(SSLContext &) = delete;\n\nprivate:\n    friend class SSLSocket;\n    SSL_CTX * getContext();\n\nprivate:\n    std::unique_ptr<SSL_CTX, void (*)(SSL_CTX*)> context_;\n};\n\nclass SSLSocket : public Socket {\npublic:\n    explicit SSLSocket(const NetworkAddress& addr, const SocketTimeoutParams& timeout_params,\n                       const SSLParams& ssl_params, SSLContext& context);\n\n    SSLSocket(SSLSocket &&) = default;\n    ~SSLSocket() override = default;\n\n    SSLSocket(const SSLSocket & ) = delete;\n    SSLSocket& operator=(const SSLSocket & ) = delete;\n\n    std::unique_ptr<InputStream> makeInputStream() const override;\n    std::unique_ptr<OutputStream> makeOutputStream() const override;\n\n    static void validateParams(const SSLParams & ssl_params);\nprivate:\n    std::unique_ptr<SSL, void (*)(SSL *s)> ssl_;\n};\n\nclass SSLSocketFactory : public NonSecureSocketFactory {\npublic:\n    explicit SSLSocketFactory(const ClientOptions& opts);\n    ~SSLSocketFactory() override;\n\nprotected:\n    std::unique_ptr<Socket> doConnect(const NetworkAddress& address, const ClientOptions& opts) override;\n\nprivate:\n    const SSLParams ssl_params_;\n    std::unique_ptr<SSLContext> ssl_context_;\n};\n\nclass SSLSocketInput : public InputStream {\npublic:\n    explicit SSLSocketInput(SSL *ssl);\n    ~SSLSocketInput() = default;\n\n    bool Skip(size_t /*bytes*/) override {\n        return false;\n    }\n\nprotected:\n    size_t DoRead(void* buf, size_t len) override;\n\nprivate:\n    // Not owning\n    SSL *ssl_;\n};\n\nclass SSLSocketOutput : public OutputStream {\npublic:\n    explicit SSLSocketOutput(SSL *ssl);\n    ~SSLSocketOutput() = default;\n\nprotected:\n    size_t DoWrite(const void* data, size_t len) override;\n\nprivate:\n    // Not owning\n    SSL *ssl_;\n};\n\n}\n\n// Path: clickhouse/client.cpp\n#include \"client.h\"\n#include \"clickhouse/version.h\"\n#include \"protocol.h\"\n\n#include \"base/compressed.h\"\n#include \"base/socket.h\"\n#include \"base/wire_format.h\"\n\n#include \"columns/factory.h\"\n\n#include <assert.h>\n#include <system_error>\n#include <vector>\n#include <sstream>\n\n#if defined(WITH_OPENSSL)\n#include \"base/sslsocket.h\"\n#endif\n\n#define DBMS_NAME                                       \"ClickHouse\"\n\n#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES         50264\n#define DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS   51554\n#define DBMS_MIN_REVISION_WITH_BLOCK_INFO               51903\n#define DBMS_MIN_REVISION_WITH_CLIENT_INFO              54032\n#define DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE          54058\n#define DBMS_MIN_REVISION_WITH_QUOTA_KEY_IN_CLIENT_INFO 54060\n//#define DBMS_MIN_REVISION_WITH_TABLES_STATUS            54226\n#define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337\n#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME      54372\n#define DBMS_MIN_REVISION_WITH_VERSION_PATCH            54401\n#define DBMS_MIN_REVISION_WITH_LOW_CARDINALITY_TYPE     54405\n#define DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA 54410\n#define DBMS_MIN_REVISION_WITH_CLIENT_WRITE_INFO        54420\n#define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429\n#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET       54441\n#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY            54442\n#define DBMS_MIN_REVISION_WITH_DISTRIBUTED_DEPTH        54448\n#define DBMS_MIN_REVISION_WITH_INITIAL_QUERY_START_TIME 54449\n#define DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS 54451\n\n#define DMBS_PROTOCOL_REVISION  DBMS_MIN_REVISION_WITH_INCREMENTAL_PROFILE_EVENTS\n\nnamespace clickhouse {\n\nstruct ClientInfo {\n    uint8_t iface_type = 1; // TCP\n    uint8_t query_kind;\n    std::string initial_user;\n    std::string initial_query_id;\n    std::string quota_key;\n    std::string os_user;\n    std::string client_hostname;\n    std::string client_name;\n    std::string initial_address = \"[::ffff:127.0.0.1]:0\";\n    uint64_t client_version_major = 0;\n    uint64_t client_version_minor = 0;\n    uint64_t client_version_patch = 0;\n    uint32_t client_revision = 0;\n};\n\nstd::ostream& operator<<(std::ostream& os, const Endpoint& endpoint) {\n    return os << endpoint.host << \":\" << endpoint.port;\n}\n\nstd::ostream& operator<<(std::ostream& os, const ClientOptions& opt) {\n    os << \"Client(\"\n       << \" Endpoints : [\";\n    size_t extra_endpoints = 0;\n\n    if (!opt.host.empty()) {\n        extra_endpoints = 1;\n        os << opt.user << '@' << Endpoint{opt.host, opt.port};\n\n        if (opt.endpoints.size())\n            os << \", \";\n    }\n\n    for (size_t i = 0; i < opt.endpoints.size(); i++) {\n        os << opt.user << '@' << opt.endpoints[i]\n           << ((i == opt.endpoints.size() - 1) ? \"\" : \", \");\n    }\n\n    os << \"] (\" << opt.endpoints.size() + extra_endpoints << \" items )\"\n       << \" ping_before_query:\" << opt.ping_before_query\n       << \" send_retries:\" << opt.send_retries\n       << \" retry_timeout:\" << opt.retry_timeout.count()\n       << \" compression_method:\"\n       << (opt.compression_method == CompressionMethod::LZ4 ? \"LZ4\" : \"None\");\n#if defined(WITH_OPENSSL)\n    if (opt.ssl_options) {\n        const auto & ssl_options = *opt.ssl_options;\n        os << \" SSL (\"\n           << \" ssl_context: \" << (ssl_options.ssl_context ? \"provided by user\" : \"created internally\")\n           << \" use_default_ca_locations: \" << ssl_options.use_default_ca_locations\n           << \" path_to_ca_files: \" << ssl_options.path_to_ca_files.size() << \" items\"\n           << \" path_to_ca_directory: \" << ssl_options.path_to_ca_directory\n           << \" min_protocol_version: \" << ssl_options.min_protocol_version\n           << \" max_protocol_version: \" << ssl_options.max_protocol_version\n           << \" context_options: \" << ssl_options.context_options\n           << \")\";\n    }\n#endif\n    os << \")\";\n    return os;\n}\n\nClientOptions& ClientOptions::SetSSLOptions(ClientOptions::SSLOptions options)\n{\n#ifdef WITH_OPENSSL\n    ssl_options = options;\n    return *this;\n#else\n    (void)options;\n    throw OpenSSLError(\"Library was built with no SSL support\");\n#endif\n}\n\nnamespace {\n\n\nstd::unique_ptr<SocketFactory> GetSocketFactory(const ClientOptions& opts) {\n    (void)opts;\n#if defined(WITH_OPENSSL)\n    if (opts.ssl_options)\n        return std::make_unique<SSLSocketFactory>(opts);\n    else\n#endif\n        return std::make_unique<NonSecureSocketFactory>();\n}\n\nstd::unique_ptr<EndpointsIteratorBase> GetEndpointsIterator(const ClientOptions& opts) {\n    if (opts.endpoints.empty())\n    {\n        throw ValidationError(\"The list of endpoints is empty\");\n    }\n\n    return std::make_unique<RoundRobinEndpointsIterator>(opts.endpoints);\n}\n\n}\n\nclass Client::Impl {\npublic:\n     Impl(const ClientOptions& opts);\n     Impl(const ClientOptions& opts,\n          std::unique_ptr<SocketFactory> socket_factory);\n    ~Impl();\n\n    void ExecuteQuery(Query query);\n\n    void SendCancel();\n\n    void Insert(const std::string& table_name, const std::string& query_id, const Block& block);\n\n    void Ping();\n\n    void ResetConnection();\n\n    void ResetConnectionEndpoint();\n\n    const ServerInfo& GetServerInfo() const;\n\n    const std::optional<Endpoint>& GetCurrentEndpoint() const;\n\nprivate:\n    bool Handshake();\n\n    bool ReceivePacket(uint64_t* server_packet = nullptr);\n\n    void SendQuery(const Query& query);\n\n    void SendData(const Block& block);\n\n    bool SendHello();\n\n    bool ReadBlock(InputStream& input, Block* block);\n\n    bool ReceiveHello();\n\n    /// Reads data packet form input stream.\n    bool ReceiveData();\n\n    /// Reads exception packet form input stream.\n    bool ReceiveException(bool rethrow = false);\n\n    void WriteBlock(const Block& block, OutputStream& output);\n\n    void CreateConnection();\n\n    void InitializeStreams(std::unique_ptr<SocketBase>&& socket);\n\n    inline size_t GetConnectionAttempts() const\n    {\n        return options_.endpoints.size() * options_.send_retries;\n    }\n\nprivate:\n    /// In case of network errors tries to reconnect to server and\n    /// call fuc several times.\n    void RetryGuard(std::function<void()> func);\n\n    void RetryConnectToTheEndpoint(std::function<void()>& func);\n\nprivate:\n    class EnsureNull {\n    public:\n        inline EnsureNull(QueryEvents* ev, QueryEvents** ptr)\n            : ptr_(ptr)\n        {\n            if (ptr_) {\n                *ptr_ = ev;\n            }\n        }\n\n        inline ~EnsureNull() {\n            if (ptr_) {\n                *ptr_ = nullptr;\n            }\n        }\n\n    private:\n        QueryEvents** ptr_;\n\n    };\n\n\n    const ClientOptions options_;\n    QueryEvents* events_;\n    int compression_ = CompressionState::Disable;\n\n    std::unique_ptr<SocketFactory> socket_factory_;\n\n    std::unique_ptr<InputStream> input_;\n    std::unique_ptr<OutputStream> output_;\n    std::unique_ptr<SocketBase> socket_;\n    std::unique_ptr<EndpointsIteratorBase> endpoints_iterator;\n\n...\n", "needle_token_start": 15113, "needle_token_end": 15201, "code_context_ntokens": 15935, "output": ["```cpp\nstd::unique_ptr<SocketFactory> GetSocketFactory(const ClientOptions& opts) {\n    (void)opts;\n#if defined(WITH_OPENSSL)\n    if (opts.ssl_options)\n        return std::make_unique<SSLSocketFactory>(opts);\n    else\n#endif\n        return std::make_unique<NonSecureSocketFactory>();\n}\n```"]}
{"repo": "polybar/polybar", "name": "make_mask", "language": "cpp", "path": "src/modules/bspwm.cpp", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a bitmask based on the states of a window manager module, allowing for the combination of two different states into a single bitmask.\n2. **Input**: Two enumeration values representing the states of a window manager module, with the second state having a default value indicating no state.\n3. **Output**: An unsigned integer representing the bitmask created from the input states.\n4. **Procedure**: The function checks if each input state is non-zero. For each non-zero state, it shifts a bit left by the state value minus one and combines these bits using a bitwise OR operation to form the final bitmask.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/modules/date.cpp\n#include \"modules/date.hpp\"\n\n#include \"drawtypes/label.hpp\"\n#include \"modules/meta/base.inl\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<date_module>;\n\n  date_module::date_module(const bar_settings& bar, string name_, const config& config)\n      : timer_module<date_module>(bar, move(name_), config) {\n    if (!m_bar.locale.empty()) {\n      datetime_stream.imbue(std::locale(m_bar.locale.c_str()));\n    }\n\n    m_router->register_action(EVENT_TOGGLE, [this]() { action_toggle(); });\n\n    m_dateformat = m_conf.get(name(), \"date\", \"\"s);\n    m_dateformat_alt = m_conf.get(name(), \"date-alt\", \"\"s);\n    m_timeformat = m_conf.get(name(), \"time\", \"\"s);\n    m_timeformat_alt = m_conf.get(name(), \"time-alt\", \"\"s);\n\n    if (m_dateformat.empty() && m_timeformat.empty()) {\n      throw module_error(\"No date or time format specified\");\n    }\n\n    set_interval(1s);\n\n    m_formatter->add(DEFAULT_FORMAT, TAG_LABEL, {TAG_LABEL, TAG_DATE});\n\n    if (m_formatter->has(TAG_DATE)) {\n      m_log.warn(\"%s: The format tag `<date>` is deprecated, use `<label>` instead.\", name());\n\n      m_formatter->get(DEFAULT_FORMAT)->value =\n          string_util::replace_all(m_formatter->get(DEFAULT_FORMAT)->value, TAG_DATE, TAG_LABEL);\n    }\n\n    if (m_formatter->has(TAG_LABEL)) {\n      m_label = load_optional_label(m_conf, name(), \"label\", \"%date%\");\n    }\n  }\n\n  bool date_module::update() {\n    auto time = std::time(nullptr);\n\n    auto date_format = m_toggled ? m_dateformat_alt : m_dateformat;\n    // Clear stream contents\n    datetime_stream.str(\"\");\n    datetime_stream << std::put_time(localtime(&time), date_format.c_str());\n...\n// Path: src/modules/bspwm.cpp\n#include \"modules/bspwm.hpp\"\n\n#include <sys/socket.h>\n\n#include \"drawtypes/iconset.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace {\n  using bspwm_state = modules::bspwm_module::state;\n\n  \nunsigned int make_mask(bspwm_state s1, bspwm_state s2 = bspwm_state::NONE) {\n    unsigned int mask{0U};\n    if (static_cast<unsigned int>(s1)) {\n      mask |= 1U << (static_cast<unsigned int>(s1) - 1U);\n    }\n    if (static_cast<unsigned int>(s2)) {\n      mask |= 1U << (static_cast<unsigned int>(s2) - 1U);\n    }\n    return mask;\n  }\n\n  unsigned int check_mask(unsigned int base, bspwm_state s1, bspwm_state s2 = bspwm_state::NONE) {\n    unsigned int mask{0U};\n    if (static_cast<unsigned int>(s1)) {\n      mask |= 1U << (static_cast<unsigned int>(s1) - 1U);\n    }\n    if (static_cast<unsigned int>(s2)) {\n      mask |= 1U << (static_cast<unsigned int>(s2) - 1U);\n    }\n    return (base & mask) == mask;\n  }\n}  // namespace\n\nnamespace modules {\n  template class module<bspwm_module>;\n\n  bspwm_module::bspwm_module(const bar_settings& bar, string name_, const config& config)\n      : event_module<bspwm_module>(bar, move(name_), config) {\n    m_router->register_action_with_data(EVENT_FOCUS, [this](const std::string& data) { action_focus(data); });\n    m_router->register_action(EVENT_NEXT, [this]() { action_next(); });\n    m_router->register_action(EVENT_PREV, [this]() { action_prev(); });\n\n    auto socket_path = bspwm_util::get_socket_path();\n\n    if (!file_util::exists(socket_path)) {\n      throw module_error(\"Could not find socket: \" + (socket_path.empty() ? \"<empty>\" : socket_path));\n    }\n\n    // Create ipc subscriber\n    m_subscriber = bspwm_util::make_subscriber();\n\n    // Load configuration values\n    m_pinworkspaces = m_conf.get(name(), \"pin-workspaces\", m_pinworkspaces);\n    m_click = m_conf.get(name(), \"enable-click\", m_click);\n    m_scroll = m_conf.get(name(), \"enable-scroll\", m_scroll);\n    m_occscroll = m_conf.get(name(), \"occupied-scroll\", m_occscroll);\n    m_revscroll = m_conf.get(name(), \"reverse-scroll\", m_revscroll);\n    m_inlinemode = m_conf.get(name(), \"inline-mode\", m_inlinemode);\n    m_fuzzy_match = m_conf.get(name(), \"fuzzy-match\", m_fuzzy_match);\n\n    // Add formats and create components\n    m_formatter->add(DEFAULT_FORMAT, TAG_LABEL_STATE, {TAG_LABEL_STATE}, {TAG_LABEL_MONITOR, TAG_LABEL_MODE});\n\n    if (m_formatter->has(TAG_LABEL_MONITOR)) {\n      m_monitorlabel = load_optional_label(m_conf, name(), \"label-monitor\", DEFAULT_MONITOR_LABEL);\n    }\n\n    if (m_formatter->has(TAG_LABEL_STATE)) {\n      // XXX: Warn about deprecated parameters\n      m_conf.warn_deprecated(name(), \"label-dimmed-active\", \"label-dimmed-focused\");\n\n      // clang-format off\n      try {\n        m_statelabels.emplace(make_mask(state::FOCUSED), load_label(m_conf, name(), \"label-active\", DEFAULT_LABEL));\n        m_conf.warn_deprecated(name(), \"label-active\", \"label-focused and label-dimmed-focused\");\n      } catch (const key_error& err) {\n        m_statelabels.emplace(make_mask(state::FOCUSED), load_optional_label(m_conf, name(), \"label-focused\", DEFAULT_LABEL));\n      }\n\n      m_statelabels.emplace(make_mask(state::OCCUPIED),\n          load_optional_label(m_conf, name(), \"label-occupied\", DEFAULT_LABEL));\n      m_statelabels.emplace(make_mask(state::URGENT),\n          load_optional_label(m_conf, name(), \"label-urgent\", DEFAULT_LABEL));\n      m_statelabels.emplace(make_mask(state::EMPTY),\n          load_optional_label(m_conf, name(), \"label-empty\", DEFAULT_LABEL));\n      m_statelabels.emplace(make_mask(state::DIMMED),\n          load_optional_label(m_conf, name(), \"label-dimmed\"));\n\n      vector<pair<state, string>> focused_overrides{\n        {state::OCCUPIED, \"label-focused-occupied\"},\n        {state::URGENT, \"label-focused-urgent\"},\n        {state::EMPTY, \"label-focused-empty\"}};\n\n      for (auto&& os : focused_overrides) {\n        unsigned int mask{make_mask(state::FOCUSED, os.first)};\n        try {\n          m_statelabels.emplace(mask, load_label(m_conf, name(), os.second));\n        } catch (const key_error& err) {\n          m_statelabels.emplace(mask, m_statelabels.at(make_mask(state::FOCUSED))->clone());\n        }\n      }\n\n      vector<pair<state, string>> dimmed_overrides{\n        {state::FOCUSED, \"label-dimmed-focused\"},\n        {state::OCCUPIED, \"label-dimmed-occupied\"},\n        {state::URGENT, \"label-dimmed-urgent\"},\n        {state::EMPTY, \"label-dimmed-empty\"}};\n\n      for (auto&& os : dimmed_overrides) {\n        m_statelabels.emplace(make_mask(state::DIMMED, os.first),\n            load_optional_label(m_conf, name(), os.second, m_statelabels.at(make_mask(os.first))->get()));\n      }\n      // clang-format on\n    }\n\n    if (m_formatter->has(TAG_LABEL_MODE)) {\n      m_modelabels.emplace(mode::LAYOUT_MONOCLE, load_optional_label(m_conf, name(), \"label-monocle\"));\n      m_modelabels.emplace(mode::LAYOUT_TILED, load_optional_label(m_conf, name(), \"label-tiled\"));\n      m_modelabels.emplace(mode::STATE_FULLSCREEN, load_optional_label(m_conf, name(), \"label-fullscreen\"));\n      m_modelabels.emplace(mode::STATE_FLOATING, load_optional_label(m_conf, name(), \"label-floating\"));\n      m_modelabels.emplace(mode::STATE_PSEUDOTILED, load_optional_label(m_conf, name(), \"label-pseudotiled\"));\n      m_modelabels.emplace(mode::NODE_LOCKED, load_optional_label(m_conf, name(), \"label-locked\"));\n      m_modelabels.emplace(mode::NODE_STICKY, load_optional_label(m_conf, name(), \"label-sticky\"));\n      m_modelabels.emplace(mode::NODE_PRIVATE, load_optional_label(m_conf, name(), \"label-private\"));\n      m_modelabels.emplace(mode::NODE_MARKED, load_optional_label(m_conf, name(), \"label-marked\"));\n    }\n\n    m_labelseparator = load_optional_label(m_conf, name(), \"label-separator\", \"\");\n\n    m_icons = std::make_shared<iconset>();\n    m_icons->add(DEFAULT_ICON, std::make_shared<label>(m_conf.get(name(), DEFAULT_ICON, \"\"s)));\n\n    int i = 0;\n    for (const auto& workspace : m_conf.get_list<string>(name(), \"ws-icon\", {})) {\n      auto vec = string_util::tokenize(workspace, ';');\n      if (vec.size() == 2) {\n        m_icons->add(vec[0], std::make_shared<label>(vec[1]));\n      } else {\n        m_log.err(\"%s: Ignoring ws-icon-%d because it has %s semicolons\", name(), i, vec.size() > 2? \"too many\" : \"too few\");\n      }\n\n      i++;\n    }\n  }\n\n  void bspwm_module::stop() {\n    if (m_subscriber) {\n      m_log.info(\"%s: Disconnecting from socket\", name());\n      m_subscriber->disconnect();\n    }\n    event_module::stop();\n  }\n\n  bool bspwm_module::has_event() {\n    if (m_subscriber->poll(POLLHUP, 0)) {\n      m_log.notice(\"%s: Reconnecting to socket...\", name());\n      m_subscriber = bspwm_util::make_subscriber();\n    }\n    return m_subscriber->peek(1);\n  }\n\n  bool bspwm_module::update() {\n    if (!m_subscriber) {\n      return false;\n    }\n\n    string data{m_subscriber->receive(BUFSIZ)};\n    bool result = false;\n\n    for (auto&& status_line : string_util::split(data, '\\n')) {\n      // Need to return true if ANY of the handle_status calls\n      // return true\n      result = this->handle_status(status_line) || result;\n    }\n\n    return result;\n  }\n\n  bool bspwm_module::handle_status(string& data) {\n    if (data.empty()) {\n      return false;\n    }\n\n    size_t prefix_len{strlen(BSPWM_STATUS_PREFIX)};\n    if (data.compare(0, prefix_len, BSPWM_STATUS_PREFIX) != 0) {\n      m_log.err(\"%s: Unknown status '%s'\", name(), data);\n      return false;\n    }\n\n    string_util::hash_type hash;\n    if ((hash = string_util::hash(data)) == m_hash) {\n      return false;\n    }\n\n    m_hash = hash;\n\n    size_t pos;\n\n    // Extract the string for the defined monitor\n    if (m_pinworkspaces) {\n      const auto needle_active = \":M\" + m_bar.monitor->name + \":\";\n      const auto needle_inactive = \":m\" + m_bar.monitor->name + \":\";\n\n      if ((pos = data.find(BSPWM_STATUS_PREFIX)) != string::npos) {\n        data = data.replace(pos, prefix_len, \":\");\n      }\n      if ((pos = data.find(needle_active)) != string::npos) {\n        data.erase(0, pos + 1);\n      }\n      if ((pos = data.find(needle_inactive)) != string::npos) {\n        data.erase(0, pos + 1);\n      }\n      if ((pos = data.find(\":m\", 1)) != string::npos) {\n        data.erase(pos);\n      }\n      if ((pos = data.find(\":M\", 1)) != string::npos) {\n        data.erase(pos);\n      }\n    } else if ((pos = data.find(BSPWM_STATUS_PREFIX)) != string::npos) {\n      data = data.replace(pos, prefix_len, \":\");\n    } else {\n      return false;\n    }\n\n    m_log.info(\"%s: Parsing socket data: %s\", name(), data);\n\n    m_monitors.clear();\n\n    size_t workspace_n{0U};\n\n    for (auto&& tag : string_util::split(data, ':')) {\n      auto value = tag.substr(1);\n      auto mode_flag = mode::NONE;\n      unsigned int workspace_mask{0U};\n\n      if (tag[0] == 'm' || tag[0] == 'M') {\n        m_monitors.emplace_back(std::make_unique<bspwm_monitor>());\n        m_monitors.back()->name = value;\n\n        if (m_monitorlabel) {\n          m_monitors.back()->label = m_monitorlabel->clone();\n          m_monitors.back()->label->replace_token(\"%name%\", value);\n        }\n      }\n\n      switch (tag[0]) {\n        case 'm':\n          m_monitors.back()->focused = false;\n          break;\n        case 'M':\n          m_monitors.back()->focused = true;\n          break;\n        case 'F':\n          workspace_mask = make_mask(state::FOCUSED, state::EMPTY);\n          break;\n        case 'O':\n          workspace_mask = make_mask(state::FOCUSED, state::OCCUPIED);\n          break;\n        case 'U':\n          workspace_mask = make_mask(state::FOCUSED, state::URGENT);\n          break;\n        case 'f':\n          workspace_mask = make_mask(state::EMPTY);\n          break;\n        case 'o':\n          workspace_mask = make_mask(state::OCCUPIED);\n          break;\n        case 'u':\n          workspace_mask = make_mask(state::URGENT);\n          break;\n        case 'L':\n          switch (value[0]) {\n            case 0:\n              break;\n            case 'M':\n              mode_flag = mode::LAYOUT_MONOCLE;\n              break;\n            case 'T':\n              mode_flag = mode::LAYOUT_TILED;\n              break;\n            default:\n              m_log.warn(\"%s: Undefined L => '%s'\", name(), value);\n          }\n          break;\n\n        case 'T':\n          switch (value[0]) {\n            case 0:\n              break;\n            case '@':\n              break;\n            case 'T':\n              break;\n            case '=':\n              mode_flag = mode::STATE_FULLSCREEN;\n              break;\n            case 'F':\n              mode_flag = mode::STATE_FLOATING;\n              break;\n            case 'P':\n              mode_flag = mode::STATE_PSEUDOTILED;\n              break;\n            default:\n              m_log.warn(\"%s: Undefined T => '%s'\", name(), value);\n          }\n          break;\n\n        case 'G':\n          if (!m_monitors.back()->focused) {\n            break;\n          }\n\n          for (size_t i = 0U; i < value.length(); i++) {\n            switch (value[i]) {\n              case 0:\n                break;\n              case 'L':\n                mode_flag = mode::NODE_LOCKED;\n                break;\n              case 'S':\n                mode_flag = mode::NODE_STICKY;\n                break;\n              case 'P':\n                mode_flag = mode::NODE_PRIVATE;\n                break;\n              case 'M':\n                mode_flag = mode::NODE_MARKED;\n                break;\n              default:\n                m_log.warn(\"%s: Undefined G => '%s'\", name(), value.substr(i, 1));\n            }\n\n            if (mode_flag != mode::NONE && !m_modelabels.empty()) {\n              m_monitors.back()->modes.emplace_back(m_modelabels.find(mode_flag)->second->clone());\n            }\n          }\n          continue;\n\n        default:\n          m_log.warn(\"%s: Undefined tag => '%s'\", name(), tag.substr(0, 1));\n          continue;\n      }\n\n      if (!m_monitors.back()) {\n        m_log.warn(\"%s: No monitor created\", name());\n        continue;\n      }\n\n      if (workspace_mask && m_formatter->has(TAG_LABEL_STATE)) {\n        auto icon = m_icons->get(value, DEFAULT_ICON, m_fuzzy_match);\n        auto label = m_statelabels.at(workspace_mask)->clone();\n\n        if (!m_monitors.back()->focused) {\n          if (m_statelabels[make_mask(state::DIMMED)]) {\n            label->replace_defined_values(m_statelabels[make_mask(state::DIMMED)]);\n          }\n          if (workspace_mask & make_mask(state::EMPTY)) {\n            label->replace_defined_values(m_statelabels[make_mask(state::DIMMED, state::EMPTY)]);\n          }\n          if (workspace_mask & make_mask(state::OCCUPIED)) {\n            label->replace_defined_values(m_statelabels[make_mask(state::DIMMED, state::OCCUPIED)]);\n          }\n          if (workspace_mask & make_mask(state::FOCUSED)) {\n            label->replace_defined_values(m_statelabels[make_mask(state::DIMMED, state::FOCUSED)]);\n          }\n          if (workspace_mask & make_mask(state::URGENT)) {\n            label->replace_defined_values(m_statelabels[make_mask(state::DIMMED, state::URGENT)]);\n          }\n        }\n\n        label->reset_tokens();\n        label->replace_token(\"%name%\", value);\n        label->replace_token(\"%icon%\", icon->get());\n        label->replace_token(\"%index%\", to_string(++workspace_n));\n\n        m_monitors.back()->workspaces.emplace_back(workspace_mask, move(label));\n      }\n\n      if (mode_flag != mode::NONE && !m_modelabels.empty()) {\n        m_monitors.back()->modes.emplace_back(m_modelabels.find(mode_flag)->second->clone());\n      }\n    }\n\n    return true;\n  }\n\n  string bspwm_module::get_output() {\n    string output;\n    for (m_index = 0U; m_index < m_monitors.size(); m_index++) {\n      if (m_index > 0) {\n        m_builder->spacing(m_formatter->get(DEFAULT_FORMAT)->spacing);\n      }\n      output += this->event_module::get_output();\n    }\n    return output;\n  }\n\n  bool bspwm_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_LABEL_MONITOR) {\n      builder->node(m_monitors[m_index]->label);\n      return true;\n    } else if (tag == TAG_LABEL_STATE && !m_monitors[m_index]->workspaces.empty()) {\n      size_t workspace_n{0U};\n\n      if (m_scroll) {\n        builder->action(mousebtn::SCROLL_DOWN, *this, m_revscroll ? EVENT_NEXT : EVENT_PREV, \"\");\n        builder->action(mousebtn::SCROLL_UP, *this, m_revscroll ? EVENT_PREV : EVENT_NEXT, \"\");\n      }\n\n      for (auto&& ws : m_monitors[m_index]->workspaces) {\n        if (ws.second.get()) {\n          if (workspace_n != 0 && *m_labelseparator) {\n            builder->node(m_labelseparator);\n          }\n\n          workspace_n++;\n\n          if (m_click) {\n            builder->action(mousebtn::LEFT, *this, EVENT_FOCUS, sstream() << m_index << \"+\" << workspace_n, ws.second);\n          } else {\n            builder->node(ws.second);\n          }\n\n          if (m_inlinemode && m_monitors[m_index]->focused && check_mask(ws.first, bspwm_state::FOCUSED)) {\n            for (auto&& mode : m_monitors[m_index]->modes) {\n              builder->node(mode);\n            }\n          }\n        }\n      }\n\n      if (m_scroll) {\n        builder->action_close();\n        builder->action_close();\n      }\n\n      return workspace_n > 0;\n    } else if (tag == TAG_LABEL_MODE && !m_inlinemode && m_monitors[m_index]->focused &&\n               !m_monitors[m_index]->modes.empty()) {\n      int modes_n = 0;\n\n      for (auto&& mode : m_monitors[m_index]->modes) {\n        if (mode && *mode) {\n          builder->node(mode);\n          modes_n++;\n        }\n      }\n\n      return modes_n > 0;\n    }\n\n    return false;\n  }\n\n  void bspwm_module::action_focus(const string& data) {\n    size_t separator{string_util::find_nth(data, 0, \"+\", 1)};\n    size_t monitor_n{std::strtoul(data.substr(0, separator).c_str(), nullptr, 10)};\n    string workspace_n{data.substr(separator + 1)};\n\n    if (monitor_n < m_monitors.size()) {\n      send_command(\"desktop -f \" + m_monitors[monitor_n]->name + \":^\" + workspace_n,\n          \"Sending desktop focus command to ipc handler\");\n    } else {\n      m_log.err(\"%s: Invalid monitor index in command: %s\", name(), data);\n    }\n  }\n  void bspwm_module::action_next() {\n    focus_direction(true);\n  }\n\n  void bspwm_module::action_prev() {\n    focus_direction(false);\n  }\n\n  void bspwm_module::focus_direction(bool next) {\n    string scrolldir = next ? \"next\" : \"prev\";\n    string modifier;\n\n    if (m_occscroll) {\n      modifier += \".occupied\";\n    }\n\n    if (m_pinworkspaces) {\n      modifier += \".local\";\n      for (const auto& mon : m_monitors) {\n        if (m_bar.monitor->match(mon->name, false) && !mon->focused) {\n          send_command(\"monitor -f \" + mon->name, \"Sending monitor focus command to ipc handler\");\n          break;\n        }\n      }\n    }\n\n    send_command(\"desktop -f \" + scrolldir + modifier, \"Sending desktop \" + scrolldir + \" command to ipc handler\");\n  }\n\n  void bspwm_module::send_command(const string& payload_cmd, const string& log_info) {\n    auto ipc = bspwm_util::make_connection();\n    auto payload = bspwm_util::make_payload(payload_cmd);\n    m_log.info(\"%s: %s\", name(), log_info);\n    ipc->send(payload->data, payload->len, 0);\n    ipc->disconnect();\n  }\n}  // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/xwindow.cpp\n#include \"modules/xwindow.hpp\"\n\n#include \"drawtypes/label.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"x11/atoms.hpp\"\n#include \"x11/connection.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<xwindow_module>;\n\n  /**\n   * Wrapper used to update the event mask of the\n   * currently active to enable title tracking\n   */\n  active_window::active_window(xcb_connection_t* conn, xcb_window_t win) : m_connection(conn), m_window(win) {\n    if (m_window != XCB_NONE) {\n      const unsigned int mask{XCB_EVENT_MASK_PROPERTY_CHANGE};\n      xcb_change_window_attributes(m_connection, m_window, XCB_CW_EVENT_MASK, &mask);\n    }\n  }\n\n  /**\n   * Deconstruct window object\n   */\n  active_window::~active_window() {\n    if (m_window != XCB_NONE) {\n      const unsigned int mask{XCB_EVENT_MASK_NO_EVENT};\n      xcb_change_window_attributes(m_connection, m_window, XCB_CW_EVENT_MASK, &mask);\n    }\n  }\n\n  /**\n   * Check if current window matches passed value\n   */\n  bool active_window::match(const xcb_window_t win) const {\n    return m_window == win;\n  }\n\n  /**\n   * Get the title by returning the first non-empty value of:\n   *  _NET_WM_NAME\n   *  _NET_WM_VISIBLE_NAME\n   *  WM_NAME\n   */\n  string active_window::title() const {\n    string title;\n\n    if (!(title = ewmh_util::get_wm_name(m_window)).empty()) {\n      return title;\n    } else if (!(title = ewmh_util::get_visible_name(m_window)).empty()) {\n      return title;\n    } else if (!(title = icccm_util::get_wm_name(m_connection, m_window)).empty()) {\n      return title;\n    } else {\n      return \"\";\n    }\n  }\n\n  string active_window::instance_name() const {\n    return icccm_util::get_wm_class(m_connection, m_window).first;\n  }\n\n  string active_window::class_name() const {\n    return icccm_util::get_wm_class(m_connection, m_window).second;\n  }\n\n  /**\n   * Construct module\n   */\n  xwindow_module::xwindow_module(const bar_settings& bar, string name_, const config& config)\n      : static_module<xwindow_module>(bar, move(name_), config), m_connection(connection::make()) {\n    // Initialize ewmh atoms\n    ewmh_util::initialize();\n\n    // Check if the WM supports _NET_ACTIVE_WINDOW\n    if (!ewmh_util::supports(_NET_ACTIVE_WINDOW)) {\n      throw module_error(\"The WM does not list _NET_ACTIVE_WINDOW as a supported hint\");\n    }\n\n    // Add formats and elements\n    m_formatter->add(DEFAULT_FORMAT, TAG_LABEL, {TAG_LABEL});\n\n    if (m_formatter->has(TAG_LABEL)) {\n      m_statelabels.emplace(state::ACTIVE, load_optional_label(m_conf, name(), \"label\", \"%title%\"));\n      m_statelabels.emplace(state::EMPTY, load_optional_label(m_conf, name(), \"label-empty\", \"\"));\n    }\n  }\n\n  /**\n   * Handler for XCB_PROPERTY_NOTIFY events\n   */\n  void xwindow_module::handle(const evt::property_notify& evt) {\n    if (evt->atom == _NET_ACTIVE_WINDOW) {\n      reset_active_window();\n      update();\n    } else if (evt->atom == _NET_CURRENT_DESKTOP) {\n      reset_active_window();\n      update();\n    } else if (evt->atom == _NET_WM_NAME || evt->atom == _NET_WM_VISIBLE_NAME || evt->atom == WM_NAME ||\n               evt->atom == WM_CLASS) {\n      update();\n    } else {\n      return;\n    }\n\n    broadcast();\n  }\n\n  void xwindow_module::reset_active_window() {\n    m_active.reset();\n  }\n\n  /**\n   * Update the currently active window and query its title\n   */\n  void xwindow_module::update() {\n    if (!m_active) {\n      xcb_window_t win = ewmh_util::get_active_window();\n      if (win != XCB_NONE) {\n        m_active = make_unique<active_window>(m_connection, win);\n      }\n    }\n\n    if (!m_statelabels.empty()) {\n      if (m_active) {\n        m_label = m_statelabels.at(state::ACTIVE)->clone();\n        m_label->reset_tokens();\n        m_label->replace_token(\"%title%\", m_active->title());\n        m_label->replace_token(\"%instance%\", m_active->instance_name());\n        m_label->replace_token(\"%class%\", m_active->class_name());\n      } else {\n        m_label = m_statelabels.at(state::EMPTY)->clone();\n      }\n    }\n  }\n\n  /**\n   * Output content as defined in the config\n   */\n  bool xwindow_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_LABEL && m_label) {\n      builder->node(m_label);\n      return true;\n    }\n    return false;\n  }\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/fs.cpp\n#include \"modules/fs.hpp\"\n\n#include <sys/statvfs.h>\n\n#include <fstream>\n#include <utility>\n\n#include \"drawtypes/label.hpp\"\n#include \"drawtypes/progressbar.hpp\"\n#include \"drawtypes/ramp.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"utils/math.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\n// Columns in /proc/self/mountinfo\n#define MOUNTINFO_DIR 4\n#define MOUNTINFO_TYPE 7\n#define MOUNTINFO_FSNAME 8\n\nnamespace modules {\n  template class module<fs_module>;\n\n  /**\n   * Bootstrap the module by reading config values and\n   * setting up required components\n   */\n  fs_module::fs_module(const bar_settings& bar, string name_, const config& config)\n      : timer_module<fs_module>(bar, move(name_), config) {\n    m_mountpoints = m_conf.get_list(name(), \"mount\", {});\n    if (m_mountpoints.empty()) {\n      m_log.info(\"%s: No mountpoints specified, using fallback \\\"/\\\"\", name());\n      m_mountpoints.emplace_back(\"/\");\n    }\n    m_remove_unmounted = m_conf.get(name(), \"remove-unmounted\", m_remove_unmounted);\n    m_perc_used_warn = m_conf.get(name(), \"warn-percentage\", 90);\n    m_fixed = m_conf.get(name(), \"fixed-values\", m_fixed);\n    m_spacing = m_conf.get(name(), \"spacing\", m_spacing);\n    set_interval(30s);\n\n    // Add formats and elements\n    m_formatter->add(\n        FORMAT_MOUNTED, TAG_LABEL_MOUNTED, {TAG_LABEL_MOUNTED, TAG_BAR_FREE, TAG_BAR_USED, TAG_RAMP_CAPACITY});\n    m_formatter->add_optional(FORMAT_WARN, {TAG_LABEL_WARN, TAG_BAR_FREE, TAG_BAR_USED, TAG_RAMP_CAPACITY});\n    m_formatter->add(FORMAT_UNMOUNTED, TAG_LABEL_UNMOUNTED, {TAG_LABEL_UNMOUNTED});\n\n    if (m_formatter->has(TAG_LABEL_MOUNTED)) {\n      m_labelmounted = load_optional_label(m_conf, name(), TAG_LABEL_MOUNTED, \"%mountpoint% %percentage_free%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_WARN)) {\n      m_labelwarn = load_optional_label(m_conf, name(), TAG_LABEL_WARN, \"%mountpoint% %percentage_free%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_UNMOUNTED)) {\n      m_labelunmounted = load_optional_label(m_conf, name(), TAG_LABEL_UNMOUNTED, \"%mountpoint% is not mounted\");\n    }\n    if (m_formatter->has(TAG_BAR_FREE)) {\n      m_barfree = load_progressbar(m_bar, m_conf, name(), TAG_BAR_FREE);\n    }\n    if (m_formatter->has(TAG_BAR_USED)) {\n      m_barused = load_progressbar(m_bar, m_conf, name(), TAG_BAR_USED);\n    }\n    if (m_formatter->has(TAG_RAMP_CAPACITY)) {\n      m_rampcapacity = load_ramp(m_conf, name(), TAG_RAMP_CAPACITY);\n    }\n\n    // Warn about \"unreachable\" format tag\n    if (m_formatter->has(TAG_LABEL_UNMOUNTED) && m_remove_unmounted) {\n      m_log.warn(\"%s: Defined format tag \\\"%s\\\" will never be used (reason: `remove-unmounted = true`)\", name(),\n          string{TAG_LABEL_UNMOUNTED});\n    }\n  }\n\n  /**\n   * Update mountpoints\n   */\n  bool fs_module::update() {\n    m_mounts.clear();\n\n    vector<vector<string>> mountinfo;\n    std::ifstream filestream(\"/proc/self/mountinfo\");\n    string line;\n\n    // Get details for mounted filesystems\n    while (std::getline(filestream, line)) {\n      auto cols = string_util::split(line, ' ');\n      if (std::find(m_mountpoints.begin(), m_mountpoints.end(), cols[MOUNTINFO_DIR]) != m_mountpoints.end()) {\n        mountinfo.emplace_back(move(cols));\n      }\n    }\n\n    // Get data for defined mountpoints\n    for (auto&& mountpoint : m_mountpoints) {\n      auto details = std::find_if(mountinfo.begin(), mountinfo.end(),\n          [&](const vector<string>& m) { return m.size() > 4 && m[MOUNTINFO_DIR] == mountpoint; });\n\n      m_mounts.emplace_back(std::make_unique<fs_mount>(mountpoint, details != mountinfo.end()));\n      struct statvfs buffer {};\n\n      if (!m_mounts.back()->mounted) {\n        m_log.warn(\"%s: Mountpoint %s is not mounted\", name(), mountpoint);\n      } else if (statvfs(mountpoint.c_str(), &buffer) == -1) {\n        m_log.err(\"%s: Failed to query filesystem (statvfs() error: %s)\", name(), strerror(errno));\n      } else {\n        auto& mount = m_mounts.back();\n        mount->mountpoint = details->at(MOUNTINFO_DIR);\n        mount->type = details->at(MOUNTINFO_TYPE);\n        mount->fsname = details->at(MOUNTINFO_FSNAME);\n\n        // see: https://en.cppreference.com/w/cpp/filesystem/space\n        mount->bytes_total = static_cast<uint64_t>(buffer.f_frsize) * static_cast<uint64_t>(buffer.f_blocks);\n        mount->bytes_free = static_cast<uint64_t>(buffer.f_frsize) * static_cast<uint64_t>(buffer.f_bfree);\n        mount->bytes_used = mount->bytes_total - mount->bytes_free;\n        mount->bytes_avail = static_cast<uint64_t>(buffer.f_frsize) * static_cast<uint64_t>(buffer.f_bavail);\n\n        mount->percentage_free =\n            math_util::percentage<double>(mount->bytes_avail, mount->bytes_used + mount->bytes_avail);\n        mount->percentage_used =\n            math_util::percentage<double>(mount->bytes_used, mount->bytes_used + mount->bytes_avail);\n      }\n    }\n\n    if (m_remove_unmounted) {\n      auto new_end =\n          std::stable_partition(m_mounts.begin(), m_mounts.end(), [](const auto& mount) { return mount->mounted; });\n\n      for (auto it = new_end; it < m_mounts.end(); ++it) {\n        m_log.info(\"%s: Removing mountpoint \\\"%s\\\" (reason: `remove-unmounted = true`)\", name(), (*it)->mountpoint);\n        m_mountpoints.erase(\n            std::remove(m_mountpoints.begin(), m_mountpoints.end(), (*it)->mountpoint), m_mountpoints.end());\n      }\n\n      m_mounts.erase(new_end, m_mounts.end());\n    }\n\n    return true;\n  }\n\n  /**\n   * Generate the module output\n   */\n  string fs_module::get_output() {\n    string output;\n\n    for (m_index = 0_z; m_index < m_mounts.size(); ++m_index) {\n      string mount_output = timer_module::get_output();\n      /*\n       * Add spacing before the mountpoint, but only if the mountpoint contents\n       * are not empty and there is already other content in the module.\n       */\n      if (!output.empty() && !mount_output.empty()) {\n        m_builder->spacing(m_spacing);\n        output += m_builder->flush();\n      }\n      output += mount_output;\n    }\n\n    return output;\n  }\n\n  /**\n   * Select format based on fs state\n   */\n  string fs_module::get_format() const {\n    if (!m_mounts[m_index]->mounted) {\n      return FORMAT_UNMOUNTED;\n    }\n    if (m_mounts[m_index]->percentage_used >= m_perc_used_warn && m_formatter->has_format(FORMAT_WARN)) {\n      return FORMAT_WARN;\n    }\n    return FORMAT_MOUNTED;\n  }\n\n  /**\n   * Output content using configured format tags\n   */\n  bool fs_module::build(builder* builder, const string& tag) const {\n    auto& mount = m_mounts[m_index];\n\n    auto replace_tokens = [&](const label_t& label) {\n      label->reset_tokens();\n      label->replace_token(\"%mountpoint%\", mount->mountpoint);\n      label->replace_token(\"%type%\", mount->type);\n      label->replace_token(\"%fsname%\", mount->fsname);\n      label->replace_token(\"%percentage_free%\", to_string(mount->percentage_free));\n      label->replace_token(\"%percentage_used%\", to_string(mount->percentage_used));\n      label->replace_token(\n          \"%total%\", string_util::filesize(mount->bytes_total, m_fixed ? 2 : 0, m_fixed, m_bar.locale));\n      label->replace_token(\"%free%\", string_util::filesize(mount->bytes_avail, m_fixed ? 2 : 0, m_fixed, m_bar.locale));\n      label->replace_token(\"%used%\", string_util::filesize(mount->bytes_used, m_fixed ? 2 : 0, m_fixed, m_bar.locale));\n    };\n\n    if (tag == TAG_BAR_FREE) {\n      builder->node(m_barfree->output(mount->percentage_free));\n    } else if (tag == TAG_BAR_USED) {\n      builder->node(m_barused->output(mount->percentage_used));\n    } else if (tag == TAG_RAMP_CAPACITY) {\n      builder->node(m_rampcapacity->get_by_percentage_with_borders(mount->percentage_free, 0, m_perc_used_warn));\n    } else if (tag == TAG_LABEL_MOUNTED) {\n      replace_tokens(m_labelmounted);\n      builder->node(m_labelmounted);\n    } else if (tag == TAG_LABEL_WARN) {\n      replace_tokens(m_labelwarn);\n      builder->node(m_labelwarn);\n    } else if (tag == TAG_LABEL_UNMOUNTED) {\n      m_labelunmounted->reset_tokens();\n      m_labelunmounted->replace_token(\"%mountpoint%\", mount->mountpoint);\n      builder->node(m_labelunmounted);\n    } else {\n      return false;\n    }\n\n    return true;\n  }\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/xkeyboard.cpp\n#include \"modules/xkeyboard.hpp\"\n\n#include \"drawtypes/iconset.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"x11/atoms.hpp\"\n#include \"x11/connection.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<xkeyboard_module>;\n\n  // clang-format off\n  static const keyboard::indicator::type INDICATOR_TYPES[] {\n    keyboard::indicator::type::CAPS_LOCK,\n    keyboard::indicator::type::NUM_LOCK,\n    keyboard::indicator::type::SCROLL_LOCK\n  };\n  // clang-format on\n\n  /**\n   * Construct module\n   */\n  xkeyboard_module::xkeyboard_module(const bar_settings& bar, string name_, const config& config)\n      : static_module<xkeyboard_module>(bar, move(name_), config), m_connection(connection::make()) {\n    m_router->register_action(EVENT_SWITCH, [this]() { action_switch(); });\n\n    // Setup extension\n    // clang-format off\n    m_connection.xkb().select_events_checked(XCB_XKB_ID_USE_CORE_KBD,\n      XCB_XKB_EVENT_TYPE_NEW_KEYBOARD_NOTIFY |\n      XCB_XKB_EVENT_TYPE_STATE_NOTIFY |\n      XCB_XKB_EVENT_TYPE_INDICATOR_STATE_NOTIFY, 0,\n      XCB_XKB_EVENT_TYPE_NEW_KEYBOARD_NOTIFY |\n      XCB_XKB_EVENT_TYPE_STATE_NOTIFY |\n      XCB_XKB_EVENT_TYPE_INDICATOR_STATE_NOTIFY, 0, 0, nullptr);\n    // clang-format on\n\n    // Create keyboard object\n    query_keyboard();\n\n    // Load config values\n    m_blacklist = m_conf.get_list(name(), \"blacklist\", {});\n\n    // load layout icons\n    parse_icons();\n\n    // Add formats and elements\n    m_formatter->add(DEFAULT_FORMAT, FORMAT_DEFAULT, {TAG_LABEL_LAYOUT, TAG_LABEL_INDICATOR});\n\n    if (m_formatter->has(TAG_LABEL_LAYOUT)) {\n      m_layout = load_optional_label(m_conf, name(), TAG_LABEL_LAYOUT, \"%layout%\");\n    }\n\n    if (m_formatter->has(TAG_LABEL_INDICATOR)) {\n      m_conf.warn_deprecated(name(), \"label-indicator\", \"label-indicator-on\");\n      // load an empty label if 'label-indicator-off' is not explicitly specified so\n      // no existing user configs are broken (who expect nothing to be shown when indicator is off)\n      m_indicator_state_off = load_optional_label(m_conf, name(), \"label-indicator-off\"s, \"\"s);\n\n      if (m_conf.has(name(), \"label-indicator-on\"s)) {\n        m_indicator_state_on = load_optional_label(m_conf, name(), \"label-indicator-on\"s, \"%name%\"s);\n      } else {\n        // if 'label-indicator-on' is not explicitly specified, use 'label-indicator'\n        // as to not break existing user configs\n        m_indicator_state_on = load_optional_label(m_conf, name(), TAG_LABEL_INDICATOR, \"%name%\"s);\n      }\n\n      // load indicator icons\n      m_indicator_icons_off = std::make_shared<iconset>();\n      m_indicator_icons_on = std::make_shared<iconset>();\n\n      auto icon_pair = string_util::tokenize(m_conf.get(name(), DEFAULT_INDICATOR_ICON, \"\"s), ';');\n      if (icon_pair.size() == 2) {\n        m_indicator_icons_off->add(DEFAULT_INDICATOR_ICON, std::make_shared<label>(icon_pair[0]));\n        m_indicator_icons_on->add(DEFAULT_INDICATOR_ICON, std::make_shared<label>(icon_pair[1]));\n      } else {\n        m_indicator_icons_off->add(DEFAULT_INDICATOR_ICON, std::make_shared<label>(\"\"s));\n        m_indicator_icons_on->add(DEFAULT_INDICATOR_ICON, std::make_shared<label>(\"\"s));\n      }\n\n      for (const auto& it : m_conf.get_list<string>(name(), \"indicator-icon\", {})) {\n        auto icon_triple = string_util::tokenize(it, ';');\n        if (icon_triple.size() == 3) {\n          auto const indicator_str = string_util::lower(icon_triple[0]);\n          m_indicator_icons_off->add(indicator_str, std::make_shared<label>(icon_triple[1]));\n          m_indicator_icons_on->add(indicator_str, std::make_shared<label>(icon_triple[2]));\n        }\n      }\n\n      for (auto it : INDICATOR_TYPES) {\n        const auto& indicator_str = m_keyboard->indicator_name(it);\n        auto key_name = string_util::replace(string_util::lower(indicator_str), \" \"s, \"\"s);\n        const auto indicator_key_on = \"label-indicator-on-\"s + key_name;\n        const auto indicator_key_off = \"label-indicator-off-\"s + key_name;\n\n        if (m_conf.has(name(), indicator_key_on)) {\n          m_indicator_on_labels.emplace(it, load_label(m_conf, name(), indicator_key_on));\n        }\n        if (m_conf.has(name(), indicator_key_off)) {\n          m_indicator_off_labels.emplace(it, load_label(m_conf, name(), indicator_key_off));\n        }\n      }\n    }\n  }\n\n  /**\n   * Update labels with extension data\n   */\n  void xkeyboard_module::update() {\n    if (m_layout) {\n      m_layout->reset_tokens();\n      m_layout->replace_token(\"%name%\", m_keyboard->group_name(m_keyboard->current()));\n      m_layout->replace_token(\"%variant%\", m_keyboard->variant_name(m_keyboard->current()));\n\n      auto const current_layout = m_keyboard->layout_name(m_keyboard->current());\n      auto const current_variant = m_keyboard->variant_name(m_keyboard->current());\n\n      auto icon = m_layout_icons->get(current_layout, current_variant);\n\n      m_layout->replace_token(\"%icon%\", icon->get());\n      m_layout->replace_token(\"%layout%\", current_layout);\n      m_layout->replace_token(\"%number%\", to_string(m_keyboard->current()));\n    }\n\n    if (m_formatter->has(TAG_LABEL_INDICATOR)) {\n      m_indicators.clear();\n\n      for (auto it : INDICATOR_TYPES) {\n        const auto& indicator_str = m_keyboard->indicator_name(it);\n\n        if (blacklisted(indicator_str)) {\n          continue;\n        }\n\n        auto indicator_on = m_keyboard->on(it);\n        auto& indicator_labels = indicator_on ? m_indicator_on_labels : m_indicator_off_labels;\n        auto& indicator_icons = indicator_on ? m_indicator_icons_on : m_indicator_icons_off;\n        auto& indicator_state = indicator_on ? m_indicator_state_on : m_indicator_state_off;\n\n        label_t indicator;\n        if (indicator_labels.find(it) != indicator_labels.end()) {\n          indicator = indicator_labels[it]->clone();\n        } else {\n          indicator = indicator_state->clone();\n        }\n\n        auto icon = indicator_icons->get(string_util::lower(indicator_str), DEFAULT_INDICATOR_ICON);\n\n        indicator->replace_token(\"%name%\", indicator_str);\n        indicator->replace_token(\"%icon%\", icon->get());\n        m_indicators.emplace(it, move(indicator));\n      }\n    }\n\n    // Trigger redraw\n    broadcast();\n  }\n\n  /**\n   * Build module output and wrap it in a click handler use\n   * to cycle between configured layout groups\n   */\n  string xkeyboard_module::get_output() {\n    string output{module::get_output()};\n\n    if (m_keyboard && m_keyboard->size() > 1) {\n      m_builder->action(mousebtn::LEFT, *this, EVENT_SWITCH, \"\");\n      m_builder->node(output);\n      m_builder->action_close();\n    } else {\n      m_builder->node(output);\n    }\n\n    return m_builder->flush();\n  }\n\n  /**\n   * Map format tags to content\n   */\n  bool xkeyboard_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_LABEL_LAYOUT) {\n      builder->node(m_layout);\n    } else if (tag == TAG_LABEL_INDICATOR && !m_indicators.empty()) {\n      size_t n{0};\n      for (auto&& indicator : m_indicators) {\n        if (*indicator.second) {\n          if (n++) {\n            builder->spacing(m_formatter->get(DEFAULT_FORMAT)->spacing);\n          }\n          builder->node(indicator.second);\n        }\n      }\n      return n > 0;\n    } else {\n      return false;\n    }\n\n    return true;\n  }\n\n  /**\n   * Handle input command\n   */\n  void xkeyboard_module::action_switch() {\n    size_t current_group = m_keyboard->current() + 1;\n\n    if (current_group >= m_keyboard->size()) {\n      current_group = 0;\n    }\n\n    xkb_util::switch_layout(m_connection, XCB_XKB_ID_USE_CORE_KBD, current_group);\n    m_keyboard->current(current_group);\n    m_connection.flush();\n\n    update();\n  }\n\n  /**\n   * Create keyboard object by querying current extension data\n   */\n  bool xkeyboard_module::query_keyboard() {\n    try {\n      auto layouts = xkb_util::get_layouts(m_connection, XCB_XKB_ID_USE_CORE_KBD);\n      auto indicators = xkb_util::get_indicators(m_connection, XCB_XKB_ID_USE_CORE_KBD);\n      auto current_group = xkb_util::get_current_group(m_connection, XCB_XKB_ID_USE_CORE_KBD);\n      m_keyboard = std::make_unique<keyboard>(move(layouts), move(indicators), current_group);\n      return true;\n    } catch (const exception& err) {\n      throw module_error(\"Failed to query keyboard, err: \" + string{err.what()});\n    }\n\n    return false;\n  }\n\n  /**\n   * Check if the indicator has been blacklisted by the user\n   */\n  bool xkeyboard_module::blacklisted(const string& indicator_name) {\n    for (auto&& i : m_blacklist) {\n      if (string_util::compare(i, indicator_name)) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  /**\n   * Handler for XCB_XKB_NEW_KEYBOARD_NOTIFY events\n   */\n  void xkeyboard_module::handle(const evt::xkb_new_keyboard_notify& evt) {\n    if (evt->changed & XCB_XKB_NKN_DETAIL_KEYCODES && m_xkb_newkb_notify.allow(evt->time)) {\n      query_keyboard();\n      update();\n    }\n  }\n\n  /**\n   * Handler for XCB_XKB_STATE_NOTIFY events\n   */\n  void xkeyboard_module::handle(const evt::xkb_state_notify& evt) {\n    if (m_keyboard && evt->changed & XCB_XKB_STATE_PART_GROUP_STATE && m_xkb_state_notify.allow(evt->time)) {\n      m_keyboard->current(evt->group);\n      update();\n    }\n  }\n\n  /**\n   * Handler for XCB_XKB_INDICATOR_STATE_NOTIFY events\n   */\n  void xkeyboard_module::handle(const evt::xkb_indicator_state_notify& evt) {\n    if (m_keyboard && m_xkb_indicator_notify.allow(evt->time)) {\n      m_keyboard->set(m_connection.xkb().get_state(XCB_XKB_ID_USE_CORE_KBD)->lockedMods);\n      update();\n    }\n  }\n\n  void xkeyboard_module::parse_icons() {\n    m_layout_icons = make_shared<layouticonset>(load_optional_label(m_conf, name(), DEFAULT_LAYOUT_ICON, \"\"s));\n\n    for (const auto& it : m_conf.get_list<string>(name(), \"layout-icon\", {})) {\n      auto vec = string_util::tokenize(it, ';');\n\n      size_t size = vec.size();\n      if (size != 2 && size != 3) {\n        m_log.warn(\"%s: Malformed layout-icon '%s'\", name(), it);\n        continue;\n      }\n\n      const string& layout = vec[0];\n\n      if (layout.empty()) {\n        m_log.warn(\"%s: layout-icon '%s' is invalid: there must always be a layout defined\", name(), it);\n        continue;\n      }\n\n      const string& variant = size == 2 ? layouticonset::VARIANT_ANY : vec[1];\n      const string& icon = vec.back();\n\n      if (layout == layouticonset::VARIANT_ANY && variant == layouticonset::VARIANT_ANY) {\n        m_log.warn(\"%s: Using '%s' for layout-icon means declaring a default icon, use 'layout-icon-default' instead\",\n            name(), it);\n        continue;\n      }\n\n      define_layout_icon(it, layout, variant, std::make_shared<label>(icon));\n    }\n  }\n\n  void xkeyboard_module::define_layout_icon(\n      const string& entry, const string& layout, const string& variant, label_t&& icon) {\n    if (m_layout_icons->contains(layout, variant)) {\n      m_log.warn(\n          \"%s: An equivalent matching is already defined for '%s;%s' => ignoring '%s'\", name(), layout, variant, entry);\n    } else if (!m_layout_icons->add(layout, variant, std::forward<label_t>(icon))) {\n      m_log.err(\n          \"%s: '%s' cannot be added to internal structure. This case should never happen and must be reported as a bug\",\n          name(), entry);\n    }\n  }\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/menu.cpp\n#include \"modules/menu.hpp\"\n\n#include \"drawtypes/label.hpp\"\n#include \"events/signal.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"utils/actions.hpp\"\n#include \"utils/scope.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<menu_module>;\n\n  menu_module::menu_module(const bar_settings& bar, string name_, const config& config)\n      : static_module<menu_module>(bar, move(name_), config) {\n    m_expand_right = m_conf.get(name(), \"expand-right\", m_expand_right);\n\n    m_router->register_action_with_data(EVENT_OPEN, [this](const std::string& data) { action_open(data); });\n    m_router->register_action(EVENT_CLOSE, [this]() { action_close(); });\n    m_router->register_action_with_data(EVENT_EXEC, [this](const std::string& data) { action_exec(data); });\n\n    string default_format;\n    if (m_expand_right) {\n      default_format += TAG_LABEL_TOGGLE;\n      default_format += TAG_MENU;\n    } else {\n      default_format += TAG_MENU;\n      default_format += TAG_LABEL_TOGGLE;\n    }\n\n    m_formatter->add(DEFAULT_FORMAT, default_format, {TAG_LABEL_TOGGLE, TAG_MENU});\n\n    if (m_formatter->has(TAG_LABEL_TOGGLE)) {\n      m_labelopen = load_label(m_conf, name(), \"label-open\");\n      m_labelclose = load_optional_label(m_conf, name(), \"label-close\", \"x\");\n    }\n\n    m_labelseparator = load_optional_label(m_conf, name(), \"label-separator\", \"\");\n\n    if (!m_formatter->has(TAG_MENU)) {\n      return;\n    }\n\n    while (true) {\n      string level_param{\"menu-\" + to_string(m_levels.size())};\n\n      if (m_conf.get(name(), level_param + \"-0\", \"\"s).empty()) {\n        break;\n      }\n\n      m_log.trace(\"%s: Creating menu level %i\", name(), m_levels.size());\n      m_levels.emplace_back(std::make_unique<menu_tree>());\n\n      while (true) {\n        string item_param{level_param + \"-\" + to_string(m_levels.back()->items.size())};\n\n        if (m_conf.get(name(), item_param, \"\"s).empty()) {\n          break;\n        }\n\n        m_log.trace(\"%s: Creating menu level item %i\", name(), m_levels.back()->items.size());\n        auto item = std::make_unique<menu_tree_item>();\n        item->label = load_label(m_conf, name(), item_param);\n        item->exec = m_conf.get(name(), item_param + \"-exec\", actions_util::get_action_string(*this, EVENT_CLOSE, \"\"));\n        m_levels.back()->items.emplace_back(move(item));\n      }\n    }\n  }\n\n  bool menu_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_LABEL_TOGGLE && m_level == -1) {\n      builder->action(mousebtn::LEFT, *this, string(EVENT_OPEN), \"0\", m_labelopen);\n    } else if (tag == TAG_LABEL_TOGGLE && m_level > -1) {\n...\n", "needle_token_start": 729, "needle_token_end": 867, "code_context_ntokens": 16153, "output": ["```cpp\nunsigned int make_mask(bspwm_state s1, bspwm_state s2 = bspwm_state::NONE) {\n    unsigned int mask{0U};\n    if (static_cast<unsigned int>(s1)) {\n      mask |= 1U << (static_cast<unsigned int>(s1) - 1U);\n    }\n    if (static_cast<unsigned int>(s2)) {\n      mask |= 1U << (static_cast<unsigned int>(s2) - 1U);\n    }\n    return mask;\n}\n```"]}
{"repo": "polybar/polybar", "name": "get_runtime_path", "language": "cpp", "path": "src/ipc/util.cpp", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To determine and provide a directory path for storing application-specific runtime data.\n2. **Input**: No explicit parameters are passed to the function.\n3. **Output**: Returns a string representing the directory path where runtime data should be stored.\n4. **Procedure**: The function first checks if a specific environment variable is set. If it is, the function constructs the path by appending a specific directory name to the value of this environment variable. If the environment variable is not set, it constructs the path by appending the directory name to a default system temporary directory, also incorporating the user's unique identifier to ensure the path is user-specific.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/ipc/ipc.cpp\n#include \"ipc/ipc.hpp\"\n\n#include <sys/stat.h>\n#include <unistd.h>\n\n#include <cassert>\n\n#include \"components/eventloop.hpp\"\n#include \"components/logger.hpp\"\n#include \"errors.hpp\"\n#include \"events/signal.hpp\"\n#include \"events/signal_emitter.hpp\"\n#include \"ipc/encoder.hpp\"\n#include \"ipc/util.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nusing namespace eventloop;\n\nnamespace ipc {\n\n  /**\n   * Message types\n   */\n  static constexpr const char* ipc_command_prefix{\"cmd:\"};\n  static constexpr const char* ipc_hook_prefix{\"hook:\"};\n  static constexpr const char* ipc_action_prefix{\"action:\"};\n\n  /**\n   * Create instance\n   */\n  ipc::make_type ipc::make(loop& loop) {\n    return std::make_unique<ipc>(signal_emitter::make(), logger::make(), loop);\n  }\n\n  /**\n   * Construct ipc handler\n   */\n  ipc::ipc(signal_emitter& emitter, const logger& logger, loop& loop)\n      : m_sig(emitter), m_log(logger), m_loop(loop), m_socket(loop.handle<PipeHandle>()) {\n    m_pipe_path = string_util::replace(PATH_MESSAGING_FIFO, \"%pid%\", to_string(getpid()));\n\n    if (file_util::exists(m_pipe_path) && unlink(m_pipe_path.c_str()) == -1) {\n      throw system_error(\"Failed to remove ipc channel\");\n    }\n    if (mkfifo(m_pipe_path.c_str(), 0600) == -1) {\n      throw system_error(\"Failed to create ipc channel\");\n    }\n    m_log.info(\"Created legacy ipc fifo at '%s'\", m_pipe_path);\n\n    ipc_pipe = make_unique<fifo>(m_loop, *this, m_pipe_path);\n\n    string sock_path = get_socket_path(getpid());\n\n    m_log.info(\"Opening ipc socket at '%s'\", sock_path);\n    m_log.notice(\"Listening for IPC messages (PID: %d)\", getpid());\n    m_socket->bind(sock_path);\n    m_socket->listen(\n        4, [this]() { on_connection(); },\n        [this](const auto& e) {\n          m_log.err(\"libuv error while listening to IPC socket: %s\", uv_strerror(e.status));\n          m_socket->close();\n        });\n  }\n\n  /**\n   * Deconstruct ipc handler\n   */\n  ipc::~ipc() {\n    m_log.trace(\"ipc: Removing named pipe at: %s\", m_pipe_path);\n    if (unlink(m_pipe_path.c_str()) == -1) {\n      m_log.err(\"Failed to delete ipc named pipe: %s\", strerror(errno));\n    }\n  }\n\n  string ipc::get_socket_path(int pid) {\n    return ensure_runtime_path() + \"/ipc.\" + to_string(pid) + \".sock\";\n  }\n\n...\n// Path: src/ipc/decoder.cpp\n#include \"ipc/decoder.hpp\"\n\n#include <cassert>\n#include <cstring>\n\nPOLYBAR_NS\n\nnamespace ipc {\n  decoder::decoder(const logger& logger, cb callback) : callback(callback), m_log(logger) {}\n\n  void decoder::on_read(const uint8_t* data, size_t size) {\n    if (state == state::CLOSED) {\n      throw error(\"Decoder is closed\");\n    }\n\n    try {\n      process_data(data, size);\n    } catch (const error& e) {\n      close();\n      throw;\n    }\n  }\n\n  void decoder::process_data(const uint8_t* data, size_t size) {\n    m_log.trace(\"ipc: Received %zd bytes\", size);\n\n    size_t buf_pos = 0;\n    size_t remain = size;\n\n    while (remain > 0) {\n      if (state == state::HEADER) {\n        ssize_t num_read = process_header_data(data + buf_pos, remain);\n        assert(num_read > 0);\n        assert(remain >= (size_t)num_read);\n\n        buf_pos += num_read;\n        remain -= num_read;\n\n        /*\n         * If an empty message arrives, we need to explicitly trigger this because there is no further data that would\n         * call process_msg_data.\n         */\n        if (remain == 0 && state == state::PAYLOAD && to_read_buf == 0) {\n          ssize_t num_read_data = process_msg_data(data + buf_pos, remain);\n          assert(num_read_data == 0);\n          (void)num_read_data;\n        }\n\n      } else {\n        assert(to_read_header == 0);\n        ssize_t num_read = process_msg_data(data + buf_pos, remain);\n        assert(num_read > 0);\n        assert(remain >= (size_t)num_read);\n\n        buf_pos += num_read;\n        remain -= num_read;\n      }\n    }\n  }\n\n  void decoder::close() noexcept {\n    state = state::CLOSED;\n  }\n\n  bool decoder::closed() const {\n    return state == state::CLOSED;\n  }\n\n  /**\n   * If we are waiting for header data, read as many bytes as possible from the given buffer.\n   *\n   * @return Number of bytes processed.\n   * @throws decoder::error on message errors\n   */\n  ssize_t decoder::process_header_data(const uint8_t* data, size_t size) {\n    assert(state == state::HEADER);\n    assert(to_read_header > 0);\n\n    size_t num_read = std::min(size, to_read_header);\n\n    std::copy(data, data + num_read, header.d + HEADER_SIZE - to_read_header);\n    to_read_header -= num_read;\n\n    if (to_read_header == 0) {\n      uint8_t version = header.s.version;\n      uint32_t msg_size = header.s.size;\n\n      m_log.trace(\n          \"Received full ipc header (magic=%.*s version=%d size=%zd)\", MAGIC.size(), header.s.magic, version, msg_size);\n\n      if (memcmp(header.s.magic, MAGIC.data(), MAGIC.size()) != 0) {\n        throw error(\"Invalid magic header, expected '\" + MAGIC_STR + \"', got '\" +\n                    string(reinterpret_cast<const char*>(header.s.magic), MAGIC.size()) + \"'\");\n      }\n      if (version != VERSION) {\n        throw error(\"Unsupported message format version \" + to_string(version));\n      }\n\n      assert(buf.empty());\n      state = state::PAYLOAD;\n      to_read_buf = msg_size;\n    }\n\n    return num_read;\n  }\n\n  /**\n   * If we are waiting for message data, read as many bytes as possible from the given buffer.\n   *\n   * @return Number of bytes processed.\n   * @throws decoder::error on message errors\n   */\n  ssize_t decoder::process_msg_data(const uint8_t* data, size_t size) {\n    assert(state == state::PAYLOAD);\n\n    size_t num_read = std::min(size, to_read_buf);\n\n    buf.reserve(buf.size() + num_read);\n    for (size_t i = 0; i < num_read; i++) {\n      buf.push_back(data[i]);\n    }\n    to_read_buf -= num_read;\n\n    if (to_read_buf == 0) {\n      callback(header.s.version, header.s.type, buf);\n\n      state = state::HEADER;\n      to_read_header = HEADER_SIZE;\n      buf.clear();\n    }\n\n    return num_read;\n  }\n}  // namespace ipc\nPOLYBAR_NS_END\n\n// Path: src/ipc/util.cpp\n#include \"ipc/util.hpp\"\n\n#include <sys/stat.h>\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace ipc {\n\n  static constexpr auto SUFFIX = \".sock\";\n  static constexpr auto XDG_RUNTIME_DIR = \"XDG_RUNTIME_DIR\";\n\n  \nstring get_runtime_path() {\n    if (env_util::has(XDG_RUNTIME_DIR)) {\n      return env_util::get(\"XDG_RUNTIME_DIR\") + \"/polybar\";\n    } else {\n      return \"/tmp/polybar-\" + to_string(getuid());\n    }\n    return env_util::get(\"XDG_RUNTIME_DIR\", \"/tmp\") + \"/polybar\";\n  }\n\n  string ensure_runtime_path() {\n    string runtime_path = get_runtime_path();\n    if (!file_util::exists(runtime_path) && mkdir(runtime_path.c_str(), 0700) == -1) {\n      // It's possible the folder was created in the meantime, we have to check again.\n      if (!file_util::exists(runtime_path)) {\n        throw system_error(\"Failed to create ipc socket folders\");\n      }\n    }\n\n    return runtime_path;\n  }\n\n  string get_socket_path(const string& pid_string) {\n    return get_runtime_path() + \"/ipc.\" + pid_string + SUFFIX;\n  }\n\n  string get_socket_path(int pid) {\n    return get_socket_path(to_string(pid));\n  }\n\n  string get_glob_socket_path() {\n    return get_socket_path(\"*\");\n  }\n\n  int get_pid_from_socket(const string& path) {\n    if (!string_util::ends_with(path, SUFFIX)) {\n      return -1;\n    }\n\n    auto stripped = path.substr(0, path.size() - strlen(SUFFIX));\n    auto p = stripped.rfind('.');\n\n    if (p == string::npos) {\n      return -1;\n    }\n\n    try {\n      return std::stoi(stripped.substr(p + 1));\n    } catch (...) {\n      return -1;\n    }\n  }\n} // namespace ipc\n\nPOLYBAR_NS_END\n\n// Path: src/ipc/encoder.cpp\n#include \"ipc/encoder.hpp\"\n\n#include <cassert>\n#include <cstring>\n\nPOLYBAR_NS\n\nnamespace ipc {\n  template <typename V>\n  vector<uint8_t> encode(const type_t type, const V& payload) {\n    size_t total_size = HEADER_SIZE + payload.size();\n    std::vector<uint8_t> data(total_size);\n\n    auto* msg_header = reinterpret_cast<header*>(data.data());\n    std::copy(MAGIC.begin(), MAGIC.end(), msg_header->s.magic);\n    msg_header->s.version = VERSION;\n    msg_header->s.size = payload.size();\n    msg_header->s.type = type;\n\n    std::copy(payload.begin(), payload.end(), data.begin() + HEADER_SIZE);\n    return data;\n  }\n\n  vector<uint8_t> encode(const type_t type, const vector<uint8_t>& payload) {\n    return encode<vector<uint8_t>>(type, payload);\n  }\n\n  vector<uint8_t> encode(const type_t type, const string& payload) {\n    return encode<string>(type, payload);\n  }\n\n}  // namespace ipc\nPOLYBAR_NS_END\n\n// Path: src/modules/alsa.cpp\n#include \"modules/alsa.hpp\"\n\n#include \"adapters/alsa/control.hpp\"\n#include \"adapters/alsa/generic.hpp\"\n#include \"adapters/alsa/mixer.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"drawtypes/progressbar.hpp\"\n#include \"drawtypes/ramp.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"settings.hpp\"\n#include \"utils/math.hpp\"\n\nPOLYBAR_NS\n\nusing namespace alsa;\n\nnamespace modules {\n  template class module<alsa_module>;\n\n  alsa_module::alsa_module(const bar_settings& bar, string name_, const config& config)\n      : event_module<alsa_module>(bar, move(name_), config) {\n    if (m_handle_events) {\n      m_router->register_action(EVENT_DEC, [this]() { action_dec(); });\n      m_router->register_action(EVENT_INC, [this]() { action_inc(); });\n      m_router->register_action(EVENT_TOGGLE, [this]() { action_toggle(); });\n    }\n\n    // Load configuration values\n    m_mapped = m_conf.get(name(), \"mapped\", m_mapped);\n    m_interval = m_conf.get(name(), \"interval\", m_interval);\n    m_unmute_on_scroll = m_conf.get(name(), \"unmute-on-scroll\", m_unmute_on_scroll);\n\n    auto master_mixer_name = m_conf.get(name(), \"master-mixer\", \"Master\"s);\n    auto speaker_mixer_name = m_conf.get(name(), \"speaker-mixer\", \"\"s);\n    auto headphone_mixer_name = m_conf.get(name(), \"headphone-mixer\", \"\"s);\n\n    // m_soundcard_name: Master Soundcard Name\n    // s_soundcard_name: Speaker Soundcard Name\n    // h_soundcard_name: Headphone Soundcard Name\n    auto m_soundcard_name = m_conf.get(name(), \"master-soundcard\", \"default\"s);\n    auto s_soundcard_name = m_conf.get(name(), \"speaker-soundcard\", \"default\"s);\n    auto h_soundcard_name = m_conf.get(name(), \"headphone-soundcard\", \"default\"s);\n\n    if (!headphone_mixer_name.empty()) {\n      m_headphoneid = m_conf.get<decltype(m_headphoneid)>(name(), \"headphone-id\");\n    }\n\n    if (string_util::compare(speaker_mixer_name, \"master\")) {\n      throw module_error(\"Master mixer is already defined\");\n    }\n    if (string_util::compare(headphone_mixer_name, \"master\")) {\n      throw module_error(\"Master mixer is already defined\");\n    }\n\n    // Setup mixers\n    try {\n      if (!master_mixer_name.empty()) {\n        m_mixer[mixer::MASTER].reset(new mixer_t::element_type{move(master_mixer_name), move(m_soundcard_name)});\n      }\n      if (!speaker_mixer_name.empty()) {\n        m_mixer[mixer::SPEAKER].reset(new mixer_t::element_type{move(speaker_mixer_name), move(s_soundcard_name)});\n      }\n      if (!headphone_mixer_name.empty()) {\n        m_mixer[mixer::HEADPHONE].reset(new mixer_t::element_type{move(headphone_mixer_name), move(h_soundcard_name)});\n      }\n      if (m_mixer[mixer::HEADPHONE]) {\n        m_ctrl[control::HEADPHONE].reset(new control_t::element_type{m_headphoneid});\n      }\n      if (m_mixer.empty()) {\n        throw module_error(\"No configured mixers\");\n      }\n    } catch (const mixer_error& err) {\n      throw module_error(err.what());\n    } catch (const control_error& err) {\n      throw module_error(err.what());\n    }\n\n    // Add formats and elements\n    m_formatter->add(FORMAT_VOLUME, TAG_LABEL_VOLUME, {TAG_RAMP_VOLUME, TAG_LABEL_VOLUME, TAG_BAR_VOLUME});\n    m_formatter->add(FORMAT_MUTED, TAG_LABEL_MUTED, {TAG_RAMP_VOLUME, TAG_LABEL_MUTED, TAG_BAR_VOLUME});\n\n    if (m_formatter->has(TAG_BAR_VOLUME)) {\n      m_bar_volume = load_progressbar(m_bar, m_conf, name(), TAG_BAR_VOLUME);\n    }\n    if (m_formatter->has(TAG_LABEL_VOLUME, FORMAT_VOLUME)) {\n      m_label_volume = load_optional_label(m_conf, name(), TAG_LABEL_VOLUME, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_MUTED, FORMAT_MUTED)) {\n      m_label_muted = load_optional_label(m_conf, name(), TAG_LABEL_MUTED, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_RAMP_VOLUME)) {\n      m_ramp_volume = load_ramp(m_conf, name(), TAG_RAMP_VOLUME);\n      m_ramp_headphones = load_ramp(m_conf, name(), TAG_RAMP_HEADPHONES, false);\n    }\n  }\n\n  void alsa_module::teardown() {\n    m_mixer.clear();\n    m_ctrl.clear();\n    snd_config_update_free_global();\n  }\n\n  bool alsa_module::has_event() {\n    // Poll for mixer and control events\n    try {\n      if (m_mixer[mixer::MASTER] && m_mixer[mixer::MASTER]->wait(25)) {\n        return true;\n      }\n      if (m_mixer[mixer::SPEAKER] && m_mixer[mixer::SPEAKER]->wait(25)) {\n        return true;\n      }\n      if (m_mixer[mixer::HEADPHONE] && m_mixer[mixer::HEADPHONE]->wait(25)) {\n        return true;\n      }\n      if (m_ctrl[control::HEADPHONE] && m_ctrl[control::HEADPHONE]->wait(25)) {\n        return true;\n      }\n    } catch (const alsa_exception& e) {\n      m_log.err(\"%s: %s\", name(), e.what());\n    }\n\n    return false;\n  }\n\n  bool alsa_module::update() {\n    // Consume pending events\n    if (m_mixer[mixer::MASTER]) {\n      m_mixer[mixer::MASTER]->process_events();\n    }\n    if (m_mixer[mixer::SPEAKER]) {\n      m_mixer[mixer::SPEAKER]->process_events();\n    }\n    if (m_mixer[mixer::HEADPHONE]) {\n      m_mixer[mixer::HEADPHONE]->process_events();\n    }\n    if (m_ctrl[control::HEADPHONE]) {\n      m_ctrl[control::HEADPHONE]->process_events();\n    }\n\n    // Get volume, mute and headphone state\n    m_volume = 100;\n    m_muted = false;\n    m_headphones = false;\n\n    try {\n      if (m_mixer[mixer::MASTER]) {\n        m_volume = m_volume * (m_mapped ? m_mixer[mixer::MASTER]->get_normalized_volume() / 100.0f\n                                        : m_mixer[mixer::MASTER]->get_volume() / 100.0f);\n        m_muted = m_muted || m_mixer[mixer::MASTER]->is_muted();\n      }\n    } catch (const alsa_exception& err) {\n      m_log.err(\"%s: Failed to query master mixer (%s)\", name(), err.what());\n    }\n\n    try {\n      if (m_ctrl[control::HEADPHONE] && m_ctrl[control::HEADPHONE]->test_device_plugged()) {\n        m_headphones = true;\n        m_volume = m_volume * (m_mapped ? m_mixer[mixer::HEADPHONE]->get_normalized_volume() / 100.0f\n                                        : m_mixer[mixer::HEADPHONE]->get_volume() / 100.0f);\n        m_muted = m_muted || m_mixer[mixer::HEADPHONE]->is_muted();\n      }\n    } catch (const alsa_exception& err) {\n      m_log.err(\"%s: Failed to query headphone mixer (%s)\", name(), err.what());\n    }\n\n    try {\n      if (!m_headphones && m_mixer[mixer::SPEAKER]) {\n        m_volume = m_volume * (m_mapped ? m_mixer[mixer::SPEAKER]->get_normalized_volume() / 100.0f\n                                        : m_mixer[mixer::SPEAKER]->get_volume() / 100.0f);\n        m_muted = m_muted || m_mixer[mixer::SPEAKER]->is_muted();\n      }\n    } catch (const alsa_exception& err) {\n      m_log.err(\"%s: Failed to query speaker mixer (%s)\", name(), err.what());\n    }\n\n    // Replace label tokens\n    if (m_label_volume) {\n      m_label_volume->reset_tokens();\n      m_label_volume->replace_token(\"%percentage%\", to_string(m_volume));\n    }\n\n    if (m_label_muted) {\n      m_label_muted->reset_tokens();\n      m_label_muted->replace_token(\"%percentage%\", to_string(m_volume));\n    }\n\n    return true;\n  }\n\n  string alsa_module::get_format() const {\n    return m_muted ? FORMAT_MUTED : FORMAT_VOLUME;\n  }\n\n  string alsa_module::get_output() {\n    // Get the module output early so that\n    // the format prefix/suffix also gets wrapper\n    // with the cmd handlers\n    string output{module::get_output()};\n\n    if (m_handle_events) {\n      auto click_middle = m_conf.get(name(), \"click-middle\", \"\"s);\n      auto click_right = m_conf.get(name(), \"click-right\", \"\"s);\n\n      if (!click_middle.empty()) {\n        m_builder->action(mousebtn::MIDDLE, click_middle);\n      }\n\n      if (!click_right.empty()) {\n        m_builder->action(mousebtn::RIGHT, click_right);\n      }\n\n      m_builder->action(mousebtn::LEFT, *this, EVENT_TOGGLE, \"\");\n      m_builder->action(mousebtn::SCROLL_UP, *this, EVENT_INC, \"\");\n      m_builder->action(mousebtn::SCROLL_DOWN, *this, EVENT_DEC, \"\");\n    }\n\n    m_builder->node(output);\n\n    return m_builder->flush();\n  }\n\n  bool alsa_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_BAR_VOLUME) {\n      builder->node(m_bar_volume->output(m_volume));\n    } else if (tag == TAG_RAMP_VOLUME && (!m_headphones || !*m_ramp_headphones)) {\n      builder->node(m_ramp_volume->get_by_percentage(m_volume));\n    } else if (tag == TAG_RAMP_VOLUME && m_headphones && *m_ramp_headphones) {\n      builder->node(m_ramp_headphones->get_by_percentage(m_volume));\n    } else if (tag == TAG_LABEL_VOLUME) {\n      builder->node(m_label_volume);\n    } else if (tag == TAG_LABEL_MUTED) {\n      builder->node(m_label_muted);\n    } else {\n      return false;\n    }\n    return true;\n  }\n\n  void alsa_module::action_inc() {\n    change_volume(m_interval);\n  }\n\n  void alsa_module::action_dec() {\n    change_volume(-m_interval);\n  }\n\n  void alsa_module::action_toggle() {\n    if (!m_mixer[mixer::MASTER]) {\n      return;\n    }\n    const auto& mixers = get_mixers();\n    for (auto&& mixer : mixers) {\n      mixer->set_mute(m_muted || mixers[0]->is_muted());\n    }\n\n    action_epilogue(mixers);\n  }\n\n  void alsa_module::change_volume(int interval) {\n    if (!m_mixer[mixer::MASTER]) {\n      return;\n    }\n    const auto& mixers = get_mixers();\n    for (auto&& mixer : mixers) {\n      if (m_unmute_on_scroll) {\n        mixer->set_mute(true);\n      }\n      m_mapped ? mixer->set_normalized_volume(math_util::cap<float>(mixer->get_normalized_volume() + interval, 0, 100))\n               : mixer->set_volume(math_util::cap<float>(mixer->get_volume() + interval, 0, 100));\n    }\n    action_epilogue(mixers);\n  }\n\n  void alsa_module::action_epilogue(const vector<mixer_t>& mixers) {\n    for (auto&& mixer : mixers) {\n      if (mixer->wait(0)) {\n        mixer->process_events();\n      }\n    }\n  }\n\n  vector<mixer_t> alsa_module::get_mixers() {\n    vector<mixer_t> mixers;\n    bool headphones{m_headphones};\n\n    if (m_mixer[mixer::MASTER] && !m_mixer[mixer::MASTER]->get_name().empty()) {\n      mixers.emplace_back(std::make_shared<mixer_t::element_type>(\n          string{m_mixer[mixer::MASTER]->get_name()}, string{m_mixer[mixer::MASTER]->get_sound_card()}));\n    }\n    if (m_mixer[mixer::HEADPHONE] && !m_mixer[mixer::HEADPHONE]->get_name().empty() && headphones) {\n      mixers.emplace_back(std::make_shared<mixer_t::element_type>(\n          string{m_mixer[mixer::HEADPHONE]->get_name()}, string{m_mixer[mixer::HEADPHONE]->get_sound_card()}));\n    }\n    if (m_mixer[mixer::SPEAKER] && !m_mixer[mixer::SPEAKER]->get_name().empty() && !headphones) {\n      mixers.emplace_back(std::make_shared<mixer_t::element_type>(\n          string{m_mixer[mixer::SPEAKER]->get_name()}, string{m_mixer[mixer::SPEAKER]->get_sound_card()}));\n    }\n\n    return mixers;\n  }\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/battery.cpp\n#include \"modules/battery.hpp\"\n\n#include \"drawtypes/animation.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"drawtypes/progressbar.hpp\"\n#include \"drawtypes/ramp.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"utils/file.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<battery_module>;\n\n  template <typename ValueReader>\n  typename ValueReader::return_type read(ValueReader& reader) {\n    std::lock_guard<ValueReader> guard(reader);\n    return reader.read();\n  }\n\n  /**\n   * Bootstrap module by setting up required components\n   */\n  battery_module::battery_module(const bar_settings& bar, string name_, const config& config)\n      : inotify_module<battery_module>(bar, move(name_), config) {\n    // Load configuration values\n    m_fullat = std::min(m_conf.get(name(), \"full-at\", m_fullat), 100);\n    m_lowat = std::max(m_conf.get(name(), \"low-at\", m_lowat), 0);\n    m_interval = m_conf.get<decltype(m_interval)>(name(), \"poll-interval\", 5s);\n    m_lastpoll = chrono::steady_clock::now();\n\n    auto path_adapter = string_util::replace(PATH_ADAPTER, \"%adapter%\", m_conf.get(name(), \"adapter\", \"ADP1\"s)) + \"/\";\n    auto path_battery = string_util::replace(PATH_BATTERY, \"%battery%\", m_conf.get(name(), \"battery\", \"BAT0\"s)) + \"/\";\n\n    // Make state reader\n    if (file_util::exists((m_fstate = path_battery + \"status\"))) {\n      m_state_reader =\n          make_unique<state_reader>([=] { return file_util::contents(m_fstate).compare(0, 8, \"Charging\") == 0; });\n    } else if (file_util::exists((m_fstate = path_adapter + \"online\"))) {\n      m_state_reader = make_unique<state_reader>([=] { return file_util::contents(m_fstate).compare(0, 1, \"1\") == 0; });\n    } else {\n      throw module_error(\"No suitable way to get current charge state\");\n    }\n\n    // Make capacity reader\n    if ((m_fcapnow = file_util::pick({path_battery + \"charge_now\", path_battery + \"energy_now\"})).empty()) {\n      throw module_error(\"No suitable way to get current capacity value\");\n    } else if ((m_fcapfull = file_util::pick({path_battery + \"charge_full\", path_battery + \"energy_full\"})).empty()) {\n      throw module_error(\"No suitable way to get max capacity value\");\n    }\n\n    m_capacity_reader = make_unique<capacity_reader>([=] {\n      auto cap_now = std::strtoul(file_util::contents(m_fcapnow).c_str(), nullptr, 10);\n      auto cap_max = std::strtoul(file_util::contents(m_fcapfull).c_str(), nullptr, 10);\n      return math_util::percentage(cap_now, 0UL, cap_max);\n    });\n\n    // Make rate reader\n    if ((m_fvoltage = file_util::pick({path_battery + \"voltage_now\"})).empty()) {\n      throw module_error(\"No suitable way to get current voltage value\");\n    } else if ((m_frate = file_util::pick({path_battery + \"current_now\", path_battery + \"power_now\"})).empty()) {\n      throw module_error(\"No suitable way to get current charge rate value\");\n    }\n\n    m_rate_reader = make_unique<rate_reader>([this] {\n      unsigned long rate{std::strtoul(file_util::contents(m_frate).c_str(), nullptr, 10)};\n      unsigned long volt{std::strtoul(file_util::contents(m_fvoltage).c_str(), nullptr, 10) / 1000UL};\n      unsigned long now{std::strtoul(file_util::contents(m_fcapnow).c_str(), nullptr, 10)};\n      unsigned long max{std::strtoul(file_util::contents(m_fcapfull).c_str(), nullptr, 10)};\n      unsigned long cap{read(*m_state_reader) ? max - now : now};\n\n      if (rate && volt && cap) {\n        auto remaining = (cap / volt);\n        auto current_rate = (rate / volt);\n\n        if (remaining && current_rate) {\n          return 3600UL * remaining / current_rate;\n        }\n      }\n\n      return 0UL;\n    });\n\n    // Make consumption reader\n    m_consumption_reader = make_unique<consumption_reader>([this] {\n      float consumption;\n\n      // if the rate we found was the current, calculate power (P = I*V)\n      if (string_util::contains(m_frate, \"current_now\")) {\n        unsigned long current{std::strtoul(file_util::contents(m_frate).c_str(), nullptr, 10)};\n        unsigned long voltage{std::strtoul(file_util::contents(m_fvoltage).c_str(), nullptr, 10)};\n\n        consumption = ((voltage / 1000.0) * (current / 1000.0)) / 1e6;\n      } else {\n        // if it was power, just use as is\n        unsigned long power{std::strtoul(file_util::contents(m_frate).c_str(), nullptr, 10)};\n\n        consumption = power / 1e6;\n      }\n\n      // convert to string with 2 decimmal places\n      string rtn(16, '\\0'); // 16 should be plenty big. Cant see it needing more than 6/7..\n      auto written = std::snprintf(&rtn[0], rtn.size(), \"%.2f\", consumption);\n      rtn.resize(written);\n\n      return rtn;\n    });\n\n    // Load state and capacity level\n    m_state = current_state();\n    m_percentage = current_percentage();\n\n    // Add formats and elements\n    m_formatter->add(FORMAT_CHARGING, TAG_LABEL_CHARGING,\n        {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_ANIMATION_CHARGING, TAG_LABEL_CHARGING});\n    m_formatter->add(FORMAT_DISCHARGING, TAG_LABEL_DISCHARGING,\n        {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_ANIMATION_DISCHARGING, TAG_LABEL_DISCHARGING});\n    m_formatter->add_optional(FORMAT_LOW, {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_ANIMATION_LOW, TAG_LABEL_LOW});\n    m_formatter->add(FORMAT_FULL, TAG_LABEL_FULL, {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_LABEL_FULL});\n\n    if (m_formatter->has(TAG_ANIMATION_CHARGING, FORMAT_CHARGING)) {\n      m_animation_charging = load_animation(m_conf, name(), TAG_ANIMATION_CHARGING);\n    }\n    if (m_formatter->has(TAG_ANIMATION_DISCHARGING, FORMAT_DISCHARGING)) {\n      m_animation_discharging = load_animation(m_conf, name(), TAG_ANIMATION_DISCHARGING);\n    }\n    if (m_formatter->has(TAG_ANIMATION_LOW, FORMAT_LOW)) {\n      m_animation_low = load_animation(m_conf, name(), TAG_ANIMATION_LOW);\n    }\n    if (m_formatter->has(TAG_BAR_CAPACITY)) {\n      m_bar_capacity = load_progressbar(m_bar, m_conf, name(), TAG_BAR_CAPACITY);\n    }\n    if (m_formatter->has(TAG_RAMP_CAPACITY)) {\n      m_ramp_capacity = load_ramp(m_conf, name(), TAG_RAMP_CAPACITY);\n    }\n    if (m_formatter->has(TAG_LABEL_CHARGING, FORMAT_CHARGING)) {\n      m_label_charging = load_optional_label(m_conf, name(), TAG_LABEL_CHARGING, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_DISCHARGING, FORMAT_DISCHARGING)) {\n      m_label_discharging = load_optional_label(m_conf, name(), TAG_LABEL_DISCHARGING, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_LOW, FORMAT_LOW)) {\n      m_label_low = load_optional_label(m_conf, name(), TAG_LABEL_LOW, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_FULL, FORMAT_FULL)) {\n      m_label_full = load_optional_label(m_conf, name(), TAG_LABEL_FULL, \"%percentage%%\");\n    }\n\n    // Create inotify watches\n    watch(m_fcapnow, IN_ACCESS);\n    watch(m_fstate, IN_ACCESS);\n\n    // Setup time if token is used\n    if ((m_label_charging && m_label_charging->has_token(\"%time%\")) ||\n        (m_label_discharging && m_label_discharging->has_token(\"%time%\")) ||\n        (m_label_low && m_label_low->has_token(\"%time%\"))) {\n      if (!m_bar.locale.empty()) {\n        setlocale(LC_TIME, m_bar.locale.c_str());\n      }\n      m_timeformat = m_conf.get(name(), \"time-format\", \"%H:%M:%S\"s);\n    }\n  }\n\n  /**\n   * Dispatch the subthread used to update the\n   * charging animation when the module is started\n   */\n  void battery_module::start() {\n    this->inotify_module::start();\n    // We only start animation thread if there is at least one animation.\n    if (m_animation_charging || m_animation_discharging || m_animation_low) {\n      m_subthread = thread(&battery_module::subthread, this);\n    }\n  }\n\n  /**\n   * Release wake lock when stopping the module\n   */\n  void battery_module::teardown() {\n    if (m_subthread.joinable()) {\n      m_subthread.join();\n    }\n  }\n\n  /**\n   * Idle between polling inotify watches for events.\n   *\n   * If the defined interval has been reached, trigger a manual\n   * poll in case the inotify events aren't fired.\n   *\n   * This fallback is needed because some systems won't\n   * report inotify events for files on sysfs.\n   */\n  void battery_module::idle() {\n    if (m_interval.count() > 0) {\n      auto now = chrono::steady_clock::now();\n      if (chrono::duration_cast<decltype(m_interval)>(now - m_lastpoll) > m_interval) {\n        m_lastpoll = now;\n        m_log.info(\"%s: Polling values (inotify fallback)\", name());\n        on_event({});\n      }\n    }\n\n    this->inotify_module::idle();\n  }\n\n  /**\n   * Update values when tracked files have changed\n   */\n  bool battery_module::on_event(const inotify_event& event) {\n    auto state = current_state();\n    auto percentage = current_percentage();\n\n    // Reset timer to avoid unnecessary polling\n    m_lastpoll = chrono::steady_clock::now();\n\n    if (event.is_valid) {\n      m_log.trace(\"%s: Inotify event reported for %s\", name(), event.filename);\n\n      if (state == m_state && percentage == m_percentage && m_unchanged--) {\n        return false;\n      }\n\n      m_unchanged = SKIP_N_UNCHANGED;\n    }\n\n    m_state = state;\n    m_percentage = percentage;\n\n    const auto replace_tokens = [&](label_t& label) {\n      if (!label) {\n        return;\n      }\n\n      label->reset_tokens();\n      label->replace_token(\"%percentage%\", to_string(clamp_percentage(m_percentage, m_state)));\n      label->replace_token(\"%percentage_raw%\", to_string(m_percentage));\n      label->replace_token(\"%consumption%\", current_consumption());\n\n      if (m_state != battery_module::state::FULL && !m_timeformat.empty()) {\n        label->replace_token(\"%time%\", current_time());\n      }\n    };\n\n    replace_tokens(m_label_full);\n    replace_tokens(m_label_discharging);\n    replace_tokens(m_label_low);\n    replace_tokens(m_label_charging);\n\n    return true;\n  }\n\n  /**\n   * Get the output format based on state\n   */\n  string battery_module::get_format() const {\n    switch (m_state) {\n      case battery_module::state::FULL:\n        return FORMAT_FULL;\n      case battery_module::state::LOW:\n        if (m_formatter->has_format(FORMAT_LOW)) {\n          return FORMAT_LOW;\n        }\n        return FORMAT_DISCHARGING;\n      case battery_module::state::DISCHARGING:\n        return FORMAT_DISCHARGING;\n      default:\n        return FORMAT_CHARGING;\n    }\n  }\n\n  /**\n   * Generate module output using defined drawtypes\n   */\n  bool battery_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_ANIMATION_CHARGING) {\n      builder->node(m_animation_charging->get());\n    } else if (tag == TAG_ANIMATION_DISCHARGING) {\n      builder->node(m_animation_discharging->get());\n    } else if (tag == TAG_ANIMATION_LOW) {\n      builder->node(m_animation_low->get());\n    } else if (tag == TAG_BAR_CAPACITY) {\n      builder->node(m_bar_capacity->output(clamp_percentage(m_percentage, m_state)));\n    } else if (tag == TAG_RAMP_CAPACITY) {\n      builder->node(m_ramp_capacity->get_by_percentage_with_borders(m_percentage, m_lowat, m_fullat));\n    } else if (tag == TAG_LABEL_CHARGING) {\n      builder->node(m_label_charging);\n    } else if (tag == TAG_LABEL_DISCHARGING) {\n      builder->node(m_label_discharging);\n    } else if (tag == TAG_LABEL_LOW) {\n      builder->node(m_label_low);\n    } else if (tag == TAG_LABEL_FULL) {\n      builder->node(m_label_full);\n    } else {\n      return false;\n    }\n\n    return true;\n  }\n\n  /**\n   * Get the current battery state\n   */\n  battery_module::state battery_module::current_state() {\n    auto charge = read(*m_capacity_reader);\n    if (charge >= m_fullat) {\n      return battery_module::state::FULL;\n    } else if (!read(*m_state_reader)) {\n      return charge <= m_lowat ? battery_module::state::LOW : battery_module::state::DISCHARGING;\n    } else {\n      return battery_module::state::CHARGING;\n    }\n  }\n\n  /**\n   * Get the current capacity level\n   */\n  int battery_module::current_percentage() {\n    return read(*m_capacity_reader);\n  }\n\n  int battery_module::clamp_percentage(int percentage, state state) const {\n    if (state == battery_module::state::FULL && percentage >= m_fullat) {\n      return 100;\n    }\n    return percentage;\n  }\n\n  /**\n   * Get the current power consumption\n   */\n  string battery_module::current_consumption() {\n    return read(*m_consumption_reader);\n  }\n\n  /**\n   * Get estimate of remaining time until fully dis-/charged\n   */\n  string battery_module::current_time() {\n    struct tm t {\n      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, nullptr\n    };\n\n    chrono::seconds sec{read(*m_rate_reader)};\n    if (sec.count() > 0) {\n      t.tm_hour = chrono::duration_cast<chrono::hours>(sec).count();\n      sec -= chrono::seconds{3600 * t.tm_hour};\n      t.tm_min = chrono::duration_cast<chrono::minutes>(sec).count();\n      sec -= chrono::seconds{60 * t.tm_min};\n      t.tm_sec = chrono::duration_cast<chrono::seconds>(sec).count();\n    }\n\n    char buffer[256]{0};\n    strftime(buffer, sizeof(buffer), m_timeformat.c_str(), &t);\n    return {buffer};\n  }\n\n  /**\n   * Subthread runner that emits update events to refresh <animation-charging>\n   * or <animation-discharging> in case they are used. Note, that it is ok to\n   * use a single thread, because the two animations are never shown at the\n   * same time.\n   */\n  void battery_module::subthread() {\n    m_log.trace(\"%s: Start of subthread\", name());\n\n    while (running()) {\n      auto now = chrono::steady_clock::now();\n      auto framerate = 1000U; // milliseconds\n      if (m_state == battery_module::state::CHARGING && m_animation_charging) {\n        m_animation_charging->increment();\n        broadcast();\n        framerate = m_animation_charging->framerate();\n      } else if (m_state == battery_module::state::DISCHARGING && m_animation_discharging) {\n        m_animation_discharging->increment();\n        broadcast();\n        framerate = m_animation_discharging->framerate();\n      } else if (m_state == battery_module::state::LOW && m_animation_low) {\n        m_animation_low->increment();\n        broadcast();\n        framerate = m_animation_low->framerate();\n      }\n\n      // We don't count the the first part of the loop to be as close as possible to the framerate.\n      now += chrono::milliseconds(framerate);\n      this_thread::sleep_until(now);\n    }\n\n    m_log.trace(\"%s: End of subthread\", name());\n  }\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/pulseaudio.cpp\n#include \"modules/pulseaudio.hpp\"\n\n#include \"adapters/pulseaudio.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"drawtypes/progressbar.hpp\"\n#include \"drawtypes/ramp.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"settings.hpp\"\n#include \"utils/math.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<pulseaudio_module>;\n\n  pulseaudio_module::pulseaudio_module(const bar_settings& bar, string name_, const config& config)\n      : event_module<pulseaudio_module>(bar, move(name_), config) {\n    if (m_handle_events) {\n      m_router->register_action(EVENT_DEC, [this]() { action_dec(); });\n      m_router->register_action(EVENT_INC, [this]() { action_inc(); });\n      m_router->register_action(EVENT_TOGGLE, [this]() { action_toggle(); });\n    }\n\n    // Load configuration values\n    m_interval = m_conf.get(name(), \"interval\", m_interval);\n    m_unmute_on_scroll = m_conf.get(name(), \"unmute-on-scroll\", m_unmute_on_scroll);\n\n    auto sink_name = m_conf.get(name(), \"sink\", \"\"s);\n    bool m_max_volume = m_conf.get(name(), \"use-ui-max\", true);\n    m_reverse_scroll = m_conf.get(name(), \"reverse-scroll\", false);\n\n    try {\n      m_pulseaudio = std::make_unique<pulseaudio>(m_log, move(sink_name), m_max_volume);\n    } catch (const pulseaudio_error& err) {\n      throw module_error(err.what());\n    }\n\n    // Add formats and elements\n    m_formatter->add(FORMAT_VOLUME, TAG_LABEL_VOLUME, {TAG_RAMP_VOLUME, TAG_LABEL_VOLUME, TAG_BAR_VOLUME});\n    m_formatter->add(FORMAT_MUTED, TAG_LABEL_MUTED, {TAG_RAMP_VOLUME, TAG_LABEL_MUTED, TAG_BAR_VOLUME});\n\n    if (m_formatter->has(TAG_BAR_VOLUME)) {\n      m_bar_volume = load_progressbar(m_bar, m_conf, name(), TAG_BAR_VOLUME);\n    }\n    if (m_formatter->has(TAG_LABEL_VOLUME, FORMAT_VOLUME)) {\n      m_label_volume = load_optional_label(m_conf, name(), TAG_LABEL_VOLUME, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_MUTED, FORMAT_MUTED)) {\n      m_label_muted = load_optional_label(m_conf, name(), TAG_LABEL_MUTED, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_RAMP_VOLUME)) {\n      m_ramp_volume = load_ramp(m_conf, name(), TAG_RAMP_VOLUME);\n    }\n  }\n\n  void pulseaudio_module::teardown() {\n    m_pulseaudio.reset();\n  }\n\n  bool pulseaudio_module::has_event() {\n    // Poll for mixer and control events\n    try {\n      if (m_pulseaudio->wait()) {\n        return true;\n      }\n    } catch (const pulseaudio_error& e) {\n      m_log.err(\"%s: %s\", name(), e.what());\n    }\n    return false;\n  }\n\n  bool pulseaudio_module::update() {\n    // Consume pending events\n    m_pulseaudio->process_events();\n\n    // Get volume and mute state\n    m_volume = 100;\n    m_decibels = PA_DECIBEL_MININFTY;\n    m_muted = false;\n\n    try {\n      if (m_pulseaudio) {\n        m_volume = m_volume * m_pulseaudio->get_volume() / 100.0f;\n        m_decibels = m_pulseaudio->get_decibels();\n        m_muted = m_muted || m_pulseaudio->is_muted();\n      }\n    } catch (const pulseaudio_error& err) {\n      m_log.err(\"%s: Failed to query pulseaudio sink (%s)\", name(), err.what());\n    }\n\n    // Replace label tokens\n    if (m_label_volume) {\n      m_label_volume->reset_tokens();\n      m_label_volume->replace_token(\"%percentage%\", to_string(m_volume));\n      m_label_volume->replace_token(\"%decibels%\", string_util::floating_point(m_decibels, 2, true));\n    }\n\n    if (m_label_muted) {\n      m_label_muted->reset_tokens();\n      m_label_muted->replace_token(\"%percentage%\", to_string(m_volume));\n      m_label_muted->replace_token(\"%decibels%\", string_util::floating_point(m_decibels, 2, true));\n    }\n\n    return true;\n  }\n\n  string pulseaudio_module::get_format() const {\n    return m_muted ? FORMAT_MUTED : FORMAT_VOLUME;\n  }\n\n  string pulseaudio_module::get_output() {\n    // Get the module output early so that\n    // the format prefix/suffix also gets wrapper\n    // with the cmd handlers\n    string output{module::get_output()};\n\n    if (m_handle_events) {\n      auto click_middle = m_conf.get(name(), \"click-middle\", \"\"s);\n      auto click_right = m_conf.get(name(), \"click-right\", \"\"s);\n\n      if (!click_middle.empty()) {\n        m_builder->action(mousebtn::MIDDLE, click_middle);\n      }\n\n      if (!click_right.empty()) {\n        m_builder->action(mousebtn::RIGHT, click_right);\n      }\n\n      m_builder->action(mousebtn::LEFT, *this, EVENT_TOGGLE, \"\");\n      if (!m_reverse_scroll) {\n        m_builder->action(mousebtn::SCROLL_UP, *this, EVENT_INC, \"\");\n        m_builder->action(mousebtn::SCROLL_DOWN, *this, EVENT_DEC, \"\");\n      } else {\n        m_builder->action(mousebtn::SCROLL_UP, *this, EVENT_DEC, \"\");\n        m_builder->action(mousebtn::SCROLL_DOWN, *this, EVENT_INC, \"\");\n      }\n    }\n\n    m_builder->node(output);\n\n    return m_builder->flush();\n  }\n\n  bool pulseaudio_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_BAR_VOLUME) {\n      builder->node(m_bar_volume->output(m_volume));\n    } else if (tag == TAG_RAMP_VOLUME) {\n      builder->node(m_ramp_volume->get_by_percentage(m_volume));\n    } else if (tag == TAG_LABEL_VOLUME) {\n      builder->node(m_label_volume);\n    } else if (tag == TAG_LABEL_MUTED) {\n      builder->node(m_label_muted);\n    } else {\n      return false;\n    }\n    return true;\n  }\n\n  void pulseaudio_module::action_inc() {\n    if (m_unmute_on_scroll) {\n      m_pulseaudio->set_mute(false);\n    }\n    m_pulseaudio->inc_volume(m_interval);\n  }\n\n  void pulseaudio_module::action_dec() {\n    if (m_unmute_on_scroll) {\n      m_pulseaudio->set_mute(false);\n    }\n    m_pulseaudio->inc_volume(-m_interval);\n  }\n\n  void pulseaudio_module::action_toggle() {\n    m_pulseaudio->toggle_mute();\n  }\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/ipc.cpp\n#include \"modules/ipc.hpp\"\n\n#include <unistd.h>\n\n#include \"drawtypes/label.hpp\"\n#include \"modules/meta/base.inl\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<ipc_module>;\n\n  /**\n   * Load user-defined ipc hooks and\n   * create formatting tags\n   */\n  ipc_module::ipc_module(const bar_settings& bar, string name_, const config& config)\n      : module<ipc_module>(bar, move(name_), config) {\n    m_router->register_action_with_data(EVENT_SEND, [this](const std::string& data) { action_send(data); });\n    m_router->register_action_with_data(EVENT_HOOK, [this](const std::string& data) { action_hook(data); });\n    m_router->register_action(EVENT_NEXT, [this]() { action_next(); });\n    m_router->register_action(EVENT_PREV, [this]() { action_prev(); });\n    m_router->register_action(EVENT_RESET, [this]() { action_reset(); });\n\n    size_t index = 0;\n\n    for (auto&& command : m_conf.get_list<string>(name(), \"hook\", {})) {\n      m_hooks.emplace_back(std::make_unique<hook>(hook{name() + to_string(++index), command}));\n    }\n\n    m_log.info(\"%s: Loaded %d hooks\", name(), m_hooks.size());\n\n    // Negative initial values should always be -1\n    m_initial = std::max(-1, m_conf.get(name(), \"initial\", 0) - 1);\n    if (has_initial()) {\n      if ((size_t)m_initial >= m_hooks.size()) {\n        throw module_error(\"Initial hook out of bounds: '\" + to_string(m_initial + 1) +\n                           \"'. Defined hooks go from 1 to \" + to_string(m_hooks.size()) + \")\");\n      }\n      m_current_hook = m_initial;\n    } else {\n      m_current_hook = -1;\n    }\n\n    // clang-format off\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::LEFT, m_conf.get(name(), \"click-left\", \"\"s)));\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::MIDDLE, m_conf.get(name(), \"click-middle\", \"\"s)));\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::RIGHT, m_conf.get(name(), \"click-right\", \"\"s)));\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::SCROLL_UP, m_conf.get(name(), \"scroll-up\", \"\"s)));\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::SCROLL_DOWN, m_conf.get(name(), \"scroll-down\", \"\"s)));\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::DOUBLE_LEFT, m_conf.get(name(), \"double-click-left\", \"\"s)));\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::DOUBLE_MIDDLE, m_conf.get(name(), \"double-click-middle\", \"\"s)));\n    m_actions.emplace(make_pair<mousebtn, string>(mousebtn::DOUBLE_RIGHT, m_conf.get(name(), \"double-click-right\", \"\"s)));\n    // clang-format on\n\n    const auto pid_token = [](const string& s) { return string_util::replace_all(s, \"%pid%\", to_string(getpid())); };\n\n    for (auto& action : m_actions) {\n      action.second = pid_token(action.second);\n    }\n    for (auto& hook : m_hooks) {\n      hook->command = pid_token(hook->command);\n    }\n\n    m_formatter->add(DEFAULT_FORMAT, TAG_LABEL, {TAG_LABEL, TAG_OUTPUT});\n\n    m_label = load_optional_label(m_conf, name(), TAG_LABEL, \"%output%\");\n\n    for (size_t i = 0; i < m_hooks.size(); i++) {\n      string format_i = \"format-\" + to_string(i);\n      m_formatter->add_optional(format_i, {TAG_LABEL});\n    }\n  }\n\n  /**\n   * Start module and run first defined hook if configured to\n   */\n  void ipc_module::start() {\n    this->module::start();\n    m_mainthread = thread([&] {\n      m_log.trace(\"%s: Thread id = %i\", this->name(), concurrency_util::thread_id(this_thread::get_id()));\n      update();\n      broadcast();\n    });\n  }\n\n  void ipc_module::update() {\n    if (has_hook()) {\n      exec_hook();\n    }\n  }\n\n  /**\n   * Wrap the output with defined mouse actions\n   */\n  string ipc_module::get_output() {\n    // Get the module output early so that\n    // the format prefix/suffix also gets wrapper\n    // with the cmd handlers\n    string output{module::get_output()};\n    if (output.empty()) {\n      return \"\";\n    }\n\n    for (auto&& action : m_actions) {\n      if (!action.second.empty()) {\n        m_builder->action(action.first, action.second);\n      }\n    }\n\n    m_builder->node(output);\n    return m_builder->flush();\n  }\n\n  string ipc_module::get_format() const {\n    if (m_current_hook != -1 && (size_t)m_current_hook < m_hooks.size()) {\n      string format_i = \"format-\" + to_string(m_current_hook);\n...\n", "needle_token_start": 2341, "needle_token_end": 2450, "code_context_ntokens": 16150, "output": ["```cpp\nstring get_runtime_path() {\n    if (env_util::has(XDG_RUNTIME_DIR)) {\n      return env_util::get(\"XDG_RUNTIME_DIR\") + \"/polybar\";\n    } else {\n      return \"/tmp/polybar-\" + to_string(getuid());\n    }\n    return env_util::get(\"XDG_RUNTIME_DIR\", \"/tmp\") + \"/polybar\";\n  }\n```"]}
{"repo": "polybar/polybar", "name": "m_path", "language": "cpp", "path": "src/utils/inotify.cpp", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To store the file system path that will be monitored for changes.\n2. **Input**: A string representing the file system path.\n3. **Output**: None directly; the path is stored internally for use in monitoring file changes.\n4. **Procedure**: The provided path string is moved into an internal variable to optimize memory usage and avoid copying the string unnecessarily.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/file.cpp\n#include \"utils/file.hpp\"\n\n#include <dirent.h>\n#include <fcntl.h>\n#include <glob.h>\n#include <sys/stat.h>\n#include <unistd.h>\n\n#include <cstdio>\n#include <cstdlib>\n#include <fstream>\n#include <streambuf>\n\n#include \"errors.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\n// implementation of file_descriptor {{{\n\nfile_descriptor::file_descriptor(const string& path, int flags, bool autoclose) : m_autoclose(autoclose) {\n  if ((m_fd = open(path.c_str(), flags)) == -1) {\n    throw system_error(\"Failed to open file descriptor\");\n  }\n}\n\nfile_descriptor::file_descriptor(int fd, bool autoclose) : m_fd(fd), m_autoclose(autoclose) {\n  if (m_fd != -1 && !*this) {\n    throw system_error(\"Given file descriptor (\" + to_string(m_fd) + \") is not valid\");\n  }\n}\n\nfile_descriptor::~file_descriptor() {\n  if (m_autoclose) {\n    close();\n  }\n}\n\nfile_descriptor& file_descriptor::operator=(const int fd) {\n  if (m_autoclose) {\n    close();\n  }\n  m_fd = fd;\n  return *this;\n}\n\nfile_descriptor::operator int() {\n  return static_cast<const file_descriptor&>(*this);\n}\nfile_descriptor::operator int() const {\n  return m_fd;\n}\n\nfile_descriptor::operator bool() {\n  return static_cast<const file_descriptor&>(*this);\n}\nfile_descriptor::operator bool() const {\n  errno = 0;  // reset since fcntl only changes it on error\n  if ((fcntl(m_fd, F_GETFD) == -1) || errno == EBADF) {\n    errno = EBADF;\n    return false;\n  }\n  return true;\n}\n\nvoid file_descriptor::close() {\n  if (m_fd != -1 && ::close(m_fd) == -1) {\n    throw system_error(\"Failed to close file descriptor\");\n  }\n  m_fd = -1;\n}\n\n// }}}\n// implementation of file_streambuf {{{\n\nfd_streambuf::~fd_streambuf() {\n  close();\n}\n\nfd_streambuf::operator int() {\n  return static_cast<const fd_streambuf&>(*this);\n}\nfd_streambuf::operator int() const {\n  return m_fd;\n}\n\nvoid fd_streambuf::open(int fd) {\n  if (m_fd) {\n    close();\n  }\n  m_fd = fd;\n  setg(m_in, m_in + BUFSIZE_IN, m_in + BUFSIZE_IN);\n  setp(m_out, m_out + BUFSIZE_OUT - 1);\n}\n\nvoid fd_streambuf::close() {\n  if (m_fd) {\n    sync();\n    m_fd = -1;\n  }\n}\n\nint fd_streambuf::sync() {\n  if (pbase() != pptr()) {\n    auto size = pptr() - pbase();\n    auto bytes = write(m_fd, m_out, size);\n    if (bytes > 0) {\n      std::copy(pbase() + bytes, pptr(), pbase());\n      setp(pbase(), epptr());\n      pbump(size - bytes);\n    }\n  }\n  return pptr() != epptr() ? 0 : -1;\n}\n\nint fd_streambuf::overflow(int c) {\n  if (!traits_type::eq_int_type(c, traits_type::eof())) {\n    *pptr() = traits_type::to_char_type(c);\n    pbump(1);\n  }\n  return sync() == -1 ? traits_type::eof() : traits_type::not_eof(c);\n}\n\nint fd_streambuf::underflow() {\n  if (gptr() >= egptr()) {\n    int bytes = ::read(m_fd, m_in, BUFSIZE_IN);\n\n    if (bytes <= 0) {\n      setg(eback(), egptr(), egptr());\n      return traits_type::eof();\n    }\n\n    setg(m_in, m_in, m_in + bytes);\n  }\n\n  return traits_type::to_int_type(*gptr());\n}\n\n// }}}\n\nnamespace file_util {\n  /**\n   * Checks if the given file exist\n   *\n   * May also return false if the file status  cannot be read\n   *\n   * Sets errno when returning false\n   */\n  bool exists(const string& filename) {\n    struct stat buffer {};\n    return stat(filename.c_str(), &buffer) == 0;\n  }\n\n  /**\n   * Checks if the given path exists and is a file\n   */\n  bool is_file(const string& filename) {\n    struct stat buffer {};\n\n    if (stat(filename.c_str(), &buffer) != 0) {\n      return false;\n    }\n\n    return S_ISREG(buffer.st_mode);\n  }\n\n  /**\n   * Checks if the given path exists and is a file\n   */\n  bool is_dir(const string& filename) {\n    struct stat buffer {};\n\n    if (stat(filename.c_str(), &buffer) != 0) {\n      return false;\n    }\n\n    return S_ISDIR(buffer.st_mode);\n  }\n\n  /**\n   * Picks the first existing file out of given entries\n   */\n  string pick(const vector<string>& filenames) {\n    for (auto&& f : filenames) {\n      if (exists(f)) {\n        return f;\n      }\n    }\n    return \"\";\n  }\n\n  /**\n   * Gets the contents of the given file\n   */\n  string contents(const string& filename) {\n    try {\n      string contents;\n      string line;\n      std::ifstream in(filename, std::ifstream::in);\n      while (std::getline(in, line)) {\n        contents += line + '\\n';\n      }\n      return contents;\n    } catch (const std::ifstream::failure& e) {\n      return \"\";\n    }\n  }\n\n  /**\n...\n// Path: src/utils/command.cpp\n#include \"utils/command.hpp\"\n\n#include <sys/wait.h>\n#include <unistd.h>\n\n#include <csignal>\n#include <cstdlib>\n#include <utility>\n\n#include \"errors.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/io.hpp\"\n#include \"utils/process.hpp\"\n#include \"utils/string.hpp\"\n\n#ifndef STDOUT_FILENO\n#define STDOUT_FILENO 1\n#endif\n#ifndef STDERR_FILENO\n#define STDERR_FILENO 2\n#endif\n\nPOLYBAR_NS\n\ncommand<output_policy::IGNORED>::command(const logger& logger, string cmd) : m_log(logger), m_cmd(move(cmd)) {}\n\ncommand<output_policy::IGNORED>::~command() {\n  terminate();\n}\n\n/**\n * Execute the command\n */\nint command<output_policy::IGNORED>::exec(bool wait_for_completion) {\n  m_forkpid = process_util::spawn_async([m_cmd = m_cmd] { process_util::exec_sh(m_cmd.c_str()); });\n  if (wait_for_completion) {\n    auto status = wait();\n    m_forkpid = -1;\n    return status;\n  }\n\n  return EXIT_SUCCESS;\n}\n\nvoid command<output_policy::IGNORED>::terminate() {\n  if (is_running()) {\n    m_log.trace(\"command: Sending SIGTERM to running child process (%d)\", m_forkpid);\n    killpg(m_forkpid, SIGTERM);\n    wait();\n  }\n  m_forkpid = -1;\n}\n\n/**\n * Check if command is running\n */\nbool command<output_policy::IGNORED>::is_running() {\n  return m_forkpid > 0 && process_util::wait_for_completion_nohang(m_forkpid, &m_forkstatus) > -1;\n}\n\n/**\n * Wait for the child process to finish\n */\nint command<output_policy::IGNORED>::wait() {\n  do {\n    m_log.trace(\"command: Waiting for pid %d to finish...\", m_forkpid);\n\n    process_util::wait_for_completion(m_forkpid, &m_forkstatus, WCONTINUED | WUNTRACED);\n\n    if (WIFEXITED(m_forkstatus) && m_forkstatus > 0) {\n      m_log.trace(\"command: Exited with failed status %d\", WEXITSTATUS(m_forkstatus));\n    } else if (WIFEXITED(m_forkstatus)) {\n      m_log.trace(\"command: Exited with status %d\", WEXITSTATUS(m_forkstatus));\n    } else if (WIFSIGNALED(m_forkstatus)) {\n      m_log.trace(\"command: killed by signal %d\", WTERMSIG(m_forkstatus));\n    } else if (WIFSTOPPED(m_forkstatus)) {\n      m_log.trace(\"command: Stopped by signal %d\", WSTOPSIG(m_forkstatus));\n    } else if (WIFCONTINUED(m_forkstatus)) {\n      m_log.trace(\"command: Continued\");\n    } else {\n      break;\n    }\n  } while (!WIFEXITED(m_forkstatus) && !WIFSIGNALED(m_forkstatus));\n\n  return WEXITSTATUS(m_forkstatus);\n}\n\n/**\n * Get command pid\n */\npid_t command<output_policy::IGNORED>::get_pid() {\n  return m_forkpid;\n}\n\n/**\n * Get command exit status\n */\nint command<output_policy::IGNORED>::get_exit_status() {\n  return WEXITSTATUS(m_forkstatus);\n}\n\ncommand<output_policy::REDIRECTED>::command(const polybar::logger& logger, std::string cmd)\n    : command<output_policy::IGNORED>(logger, move(cmd)) {\n  if (pipe(m_stdin) != 0) {\n    throw command_error(\"Failed to allocate input stream\");\n  }\n  if (pipe(m_stdout) != 0) {\n    throw command_error(\"Failed to allocate output stream\");\n  }\n}\n\ncommand<output_policy::REDIRECTED>::~command() {\n  if (m_stdin[PIPE_READ] > 0) {\n    close(m_stdin[PIPE_READ]);\n  }\n  if (m_stdin[PIPE_WRITE] > 0) {\n    close(m_stdin[PIPE_WRITE]);\n  }\n  if (m_stdout[PIPE_READ] > 0) {\n    close(m_stdout[PIPE_READ]);\n  }\n  if (m_stdout[PIPE_WRITE] > 0) {\n    close(m_stdout[PIPE_WRITE]);\n  }\n}\n\n/**\n * Execute the command\n */\nint command<output_policy::REDIRECTED>::exec(bool wait_for_completion, const vector<pair<string, string>>& env) {\n  if ((m_forkpid = fork()) == -1) {\n    throw system_error(\"Failed to fork process\");\n  }\n\n  if (process_util::in_forked_process(m_forkpid)) {\n    if (dup2(m_stdin[PIPE_READ], STDIN_FILENO) == -1) {\n      throw command_error(\"Failed to redirect stdin in child process\");\n    }\n    if (dup2(m_stdout[PIPE_WRITE], STDOUT_FILENO) == -1) {\n      throw command_error(\"Failed to redirect stdout in child process\");\n    }\n    if (dup2(m_stdout[PIPE_WRITE], STDERR_FILENO) == -1) {\n      throw command_error(\"Failed to redirect stderr in child process\");\n    }\n\n    // Close file descriptors that won't be used by the child\n    if ((m_stdin[PIPE_READ] = close(m_stdin[PIPE_READ])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdin[PIPE_WRITE] = close(m_stdin[PIPE_WRITE])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdout[PIPE_READ] = close(m_stdout[PIPE_READ])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdout[PIPE_WRITE] = close(m_stdout[PIPE_WRITE])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n\n    setpgid(m_forkpid, 0);\n    process_util::exec_sh(m_cmd.c_str(), env);\n  } else {\n    // Close file descriptors that won't be used by the parent\n    if ((m_stdin[PIPE_READ] = close(m_stdin[PIPE_READ])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdout[PIPE_WRITE] = close(m_stdout[PIPE_WRITE])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n\n    if (wait_for_completion) {\n      auto status = wait();\n      m_forkpid = -1;\n      return status;\n    }\n  }\n\n  return EXIT_SUCCESS;\n}\n\n/**\n * Tail command output\n *\n * @note: This is a blocking call and will not\n * end until the stream is closed\n */\nvoid command<output_policy::REDIRECTED>::tail(std::function<void(string)> cb) {\n  io_util::tail(get_stdout(PIPE_READ), cb);\n}\n\n/**\n * Read a line from the commands output stream\n */\nstring command<output_policy::REDIRECTED>::readline() {\n  if (!m_stdout_reader) {\n    m_stdout_reader = make_unique<fd_stream<std::istream>>(get_stdout(PIPE_READ), false);\n  }\n\n  string s;\n  std::getline(*m_stdout_reader, s);\n  return s;\n}\n\n/**\n * Get command output channel\n */\nint command<output_policy::REDIRECTED>::get_stdout(int c) {\n  return m_stdout[c];\n}\n\n/**\n * Get command input channel\n */\nint command<output_policy::REDIRECTED>::get_stdin(int c) {\n  return m_stdin[c];\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/inotify.cpp\n#include \"utils/inotify.hpp\"\n\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"utils/memory.hpp\"\n\n\nPOLYBAR_NS\n\n/**\n * Construct inotify watch\n */\ninotify_watch::inotify_watch(string path) : m_path(move(path)) {}\n\n/**\n * Deconstruct inotify watch\n */\ninotify_watch::~inotify_watch() {\n  if (m_wd != -1) {\n    inotify_rm_watch(m_fd, m_wd);\n  }\n  if (m_fd != -1) {\n    close(m_fd);\n  }\n}\n\ninotify_watch::inotify_watch(inotify_watch&& other) noexcept {\n  std::swap(m_path, other.m_path);\n  std::swap(m_wd, other.m_wd);\n  std::swap(m_fd, other.m_fd);\n  std::swap(m_mask, other.m_mask);\n}\n\ninotify_watch& inotify_watch::operator=(inotify_watch&& other) noexcept {\n  std::swap(m_path, other.m_path);\n  std::swap(m_wd, other.m_wd);\n  std::swap(m_fd, other.m_fd);\n  std::swap(m_mask, other.m_mask);\n  return *this;\n}\n\n/**\n * Attach inotify watch\n */\nvoid inotify_watch::attach(int mask) {\n  if (m_fd == -1 && (m_fd = inotify_init()) == -1) {\n    throw system_error(\"Failed to allocate inotify fd\");\n  }\n  if ((m_wd = inotify_add_watch(m_fd, m_path.c_str(), mask)) == -1) {\n    throw system_error(\"Failed to attach inotify watch\");\n  }\n  m_mask |= mask;\n}\n\n/**\n * Remove inotify watch\n */\nvoid inotify_watch::remove(bool force) {\n  if (inotify_rm_watch(m_fd, m_wd) == -1 && !force) {\n    throw system_error(\"Failed to remove inotify watch\");\n  }\n  m_wd = -1;\n  m_mask = 0;\n}\n\n/**\n * Poll the inotify fd for events\n *\n * @brief A wait_ms of -1 blocks until an event is fired\n */\nbool inotify_watch::poll(int wait_ms) const {\n  if (m_fd == -1) {\n    return false;\n  }\n\n  struct pollfd fds[1];\n  fds[0].fd = m_fd;\n  fds[0].events = POLLIN;\n\n  ::poll(fds, 1, wait_ms);\n\n  return fds[0].revents & POLLIN;\n}\n\n/**\n * Get the latest inotify event\n */\ninotify_event inotify_watch::get_event() const {\n  inotify_event event;\n\n  if (m_fd == -1 || m_wd == -1) {\n    return event;\n  }\n\n  event.is_valid = true;\n\n  char buffer[1024];\n  auto bytes = read(m_fd, buffer, 1024);\n  auto len = 0;\n\n  while (len < bytes) {\n    auto* e = reinterpret_cast<::inotify_event*>(&buffer[len]);\n\n    event.filename = e->len ? e->name : m_path;\n    event.wd = e->wd;\n    event.cookie = e->cookie;\n    event.is_dir = e->mask & IN_ISDIR;\n    event.mask |= e->mask;\n\n    len += sizeof(*e) + e->len;\n  }\n\n  return event;\n}\n\n/**\n * Get watch file path\n */\nstring inotify_watch::path() const {\n  return m_path;\n}\n\n/**\n * Get the file descriptor associated with the watch\n */\nint inotify_watch::get_file_descriptor() const {\n  return m_fd;\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/restack.cpp\n#include \"utils/restack.hpp\"\n\nPOLYBAR_NS\n\nnamespace restack_util {\n\n/**\n * Restacks the given window relative to a given sibling with some stack order (above, below)\n *\n * Both windows need to be siblings (have the same parent window).\n *\n * @param win The window to restack\n * @param sibling The window relative to which restacking happens\n * @param stack_mode The restacking mode (above, below)\n * @throw Some xpp error if restacking fails\n */\nvoid restack_relative(connection& conn, xcb_window_t win, xcb_window_t sibling, xcb_stack_mode_t stack_mode) {\n  const unsigned int value_mask = XCB_CONFIG_WINDOW_SIBLING | XCB_CONFIG_WINDOW_STACK_MODE;\n  const std::array<uint32_t, 2> value_list = {sibling, stack_mode};\n  conn.configure_window_checked(win, value_mask, value_list.data());\n}\n\nstring stack_mode_to_string(xcb_stack_mode_t mode) {\n  switch (mode) {\n    case XCB_STACK_MODE_ABOVE:\n      return \"ABOVE\";\n    case XCB_STACK_MODE_BELOW:\n      return \"BELOW\";\n    case XCB_STACK_MODE_TOP_IF:\n      return \"TOP_IF\";\n    case XCB_STACK_MODE_BOTTOM_IF:\n      return \"BOTTOM_IF\";\n    case XCB_STACK_MODE_OPPOSITE:\n      return \"OPPOSITE\";\n  }\n\n  return \"UNKNOWN\";\n}\n\n/**\n * @return true iff the two given windows are sibings (are different windows and have same parent).\n */\nbool are_siblings(connection& conn, xcb_window_t win, xcb_window_t sibling) {\n  if (win == XCB_NONE || sibling == XCB_NONE || win == sibling) {\n    return false;\n  }\n\n  auto win_tree = conn.query_tree(win);\n  auto sibling_tree = conn.query_tree(sibling);\n  return win_tree->parent == sibling_tree->parent;\n}\n\n/**\n * \"bottom\" restack strategy.\n *\n * Moves the bar window to the bottom of the window stack\n */\nparams get_bottom_params(connection& conn, xcb_window_t bar_window) {\n  auto children = conn.query_tree(conn.root()).children();\n  if (children.begin() != children.end() && *children.begin() != bar_window) {\n    return {*children.begin(), XCB_STACK_MODE_BELOW};\n  }\n\n  return NONE_PARAMS;\n}\n\n/**\n * EWMH restack strategy.\n *\n * Moves the bar window above the WM meta window (_NET_SUPPORTING_WM_CHECK).\n * This window is generally towards the bottom of the window stack, but still above other windows that could interfere.\n *\n * @see ewmh_util::get_ewmh_meta_window\n */\nparams get_ewmh_params(connection& conn) {\n  if (auto meta_window = ewmh_util::get_ewmh_meta_window(conn.root())) {\n    return {meta_window, XCB_STACK_MODE_ABOVE};\n  }\n\n  return NONE_PARAMS;\n}\n\n/**\n * Generic restack stratgey.\n *\n * Tries to provide the best WM-agnostic restacking.\n *\n * Currently tries to the following stratgies in order:\n * * ewmh\n * * bottom\n */\nparams get_generic_params(connection& conn, xcb_window_t bar_window) {\n  auto ewmh_params = get_ewmh_params(conn);\n\n  if (are_siblings(conn, bar_window, ewmh_params.first)) {\n    return ewmh_params;\n  }\n\n  auto bottom_params = get_bottom_params(conn, bar_window);\n\n  if (are_siblings(conn, bar_window, bottom_params.first)) {\n    return bottom_params;\n  }\n\n  return NONE_PARAMS;\n}\n} // namespace restack_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/color.cpp\n#include \"utils/color.hpp\"\n\n#include <algorithm>\n\nPOLYBAR_NS\n\n/**\n * Takes a hex string as input and brings it into a normalized form\n *\n * The input can either be only an alpha channel #AA\n * or any of these forms: #RGB, #ARGB, #RRGGBB, #AARRGGBB\n *\n * Colors without alpha channel will get an alpha channel of FF\n * The input does not have to start with '#'\n *\n * @returns Empty string for malformed input, either AA for the alpha only\n * input or an 8 character string of the expanded form AARRGGBB\n */\nstatic string normalize_hex(string hex) {\n  if (hex.length() == 0) {\n    return \"\";\n  }\n\n  // We remove the hash because it makes processing easier\n  if (hex[0] == '#') {\n    hex.erase(0, 1);\n  }\n\n  // Check that only valid characters are used\n  if (!std::all_of(hex.cbegin(), hex.cend(), isxdigit)) {\n    return \"\";\n  }\n\n  if (hex.length() == 2) {\n    // We only have an alpha channel\n    return hex;\n  }\n\n  if (hex.length() == 3) {\n    // RGB -> FRGB\n    hex.insert(0, 1, 'f');\n  }\n\n  if (hex.length() == 4) {\n    // ARGB -> AARRGGBB\n    hex = {hex[0], hex[0], hex[1], hex[1], hex[2], hex[2], hex[3], hex[3]};\n  }\n\n  if (hex.length() == 6) {\n    // RRGGBB -> FFRRGGBB\n    hex.insert(0, 2, 'f');\n  }\n\n  if (hex.length() != 8) {\n    return \"\";\n  }\n\n  return hex;\n}\n\nrgba::rgba() : m_value(0), m_type(type::NONE) {}\nrgba::rgba(uint32_t value, enum type type) : m_value(value), m_type(type) {}\nrgba::rgba(string hex) {\n  hex = normalize_hex(hex);\n\n  if (hex.length() == 0) {\n    m_value = 0;\n    m_type = type::NONE;\n  } else if (hex.length() == 2) {\n    m_value = std::strtoul(hex.c_str(), nullptr, 16) << 24;\n    m_type = type::ALPHA_ONLY;\n  } else {\n    m_value = std::strtoul(hex.c_str(), nullptr, 16);\n    m_type = type::ARGB;\n  }\n}\n\nrgba::operator string() const {\n  char s[10];\n  size_t len = snprintf(s, 10, \"#%08x\", m_value);\n  return string(s, len);\n}\n\nbool rgba::operator==(const rgba& other) const {\n  if (m_type != other.m_type) {\n    return false;\n  }\n\n  switch (m_type) {\n    case type::NONE:\n      return true;\n    case type::ARGB:\n      return m_value == other.m_value;\n    case type::ALPHA_ONLY:\n      return alpha_i() == other.alpha_i();\n    default:\n      return false;\n  }\n}\n\nbool rgba::operator!=(const rgba& other) const {\n  return !(*this == other);\n}\n\nrgba::operator uint32_t() const {\n  return m_value;\n}\n\nrgba::operator bool() const {\n  return has_color();\n}\n\nuint32_t rgba::value() const {\n  return this->m_value;\n}\n\nenum rgba::type rgba::get_type() const {\n  return m_type;\n}\n\ndouble rgba::alpha_d() const {\n  return alpha_i() / 255.0;\n}\n\ndouble rgba::red_d() const {\n  return red_i() / 255.0;\n}\n\ndouble rgba::green_d() const {\n  return green_i() / 255.0;\n}\n\ndouble rgba::blue_d() const {\n  return blue_i() / 255.0;\n}\n\nuint8_t rgba::alpha_i() const {\n  return (m_value >> 24) & 0xFF;\n}\n\nuint8_t rgba::red_i() const {\n  return (m_value >> 16) & 0xFF;\n}\n\nuint8_t rgba::green_i() const {\n  return (m_value >> 8) & 0xFF;\n}\n\nuint8_t rgba::blue_i() const {\n  return m_value & 0xFF;\n}\n\nbool rgba::has_color() const {\n  return m_type != type::NONE;\n}\n\nbool rgba::is_transparent() const {\n  return alpha_i() != 0xFF;\n}\n\n/**\n * Applies the alpha channel of this color to the given color.\n */\nrgba rgba::apply_alpha_to(rgba other) const {\n  uint32_t val = (other.value() & 0x00FFFFFF) | (((uint32_t)alpha_i()) << 24);\n  return rgba(val, m_type);\n}\n\n/**\n * If this is an ALPHA_ONLY color, applies this alpha channel to the other\n * color, otherwise just returns this.\n *\n * @returns the new color if this is ALPHA_ONLY or a copy of this otherwise.\n */\nrgba rgba::try_apply_alpha_to(rgba other) const {\n  if (m_type == type::ALPHA_ONLY) {\n    return apply_alpha_to(other);\n  }\n\n  return *this;\n}\n\nstring color_util::simplify_hex(string hex) {\n  // convert #ffrrggbb to #rrggbb\n  if (hex.length() == 9 && std::toupper(hex[1]) == 'F' && std::toupper(hex[2]) == 'F') {\n    hex.erase(1, 2);\n  }\n\n  // convert #rrggbb to #rgb\n  if (hex.length() == 7) {\n    if (hex[1] == hex[2] && hex[3] == hex[4] && hex[5] == hex[6]) {\n      hex = {'#', hex[1], hex[3], hex[5]};\n    }\n  }\n\n  return hex;\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/http.cpp\n#include \"utils/http.hpp\"\n\n#include <curl/curl.h>\n#include <curl/easy.h>\n\n#include <sstream>\n\n#include \"errors.hpp\"\n#include \"settings.hpp\"\n\nPOLYBAR_NS\n\nhttp_downloader::http_downloader(int connection_timeout) {\n  m_curl = curl_easy_init();\n  curl_easy_setopt(m_curl, CURLOPT_ACCEPT_ENCODING, \"deflate\");\n  curl_easy_setopt(m_curl, CURLOPT_CONNECTTIMEOUT, connection_timeout);\n  curl_easy_setopt(m_curl, CURLOPT_FOLLOWLOCATION, true);\n  curl_easy_setopt(m_curl, CURLOPT_NOSIGNAL, true);\n  curl_easy_setopt(m_curl, CURLOPT_USERAGENT, (\"polybar/\" + string{APP_VERSION}).c_str());\n  curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, http_downloader::write);\n  curl_easy_setopt(m_curl, CURLOPT_FORBID_REUSE, true);\n}\n\nhttp_downloader::~http_downloader() {\n  curl_easy_cleanup(m_curl);\n}\n\nstring http_downloader::get(const string& url, const string& user, const string& password) {\n  std::stringstream out{};\n  curl_easy_setopt(m_curl, CURLOPT_URL, url.c_str());\n  curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, &out);\n  if (!user.empty()) {\n    curl_easy_setopt(m_curl, CURLOPT_USERNAME, user.c_str());\n  }\n  if (!password.empty()) {\n    curl_easy_setopt(m_curl, CURLOPT_PASSWORD, password.c_str());\n  }\n\n  auto res = curl_easy_perform(m_curl);\n  if (res != CURLE_OK) {\n    throw application_error(curl_easy_strerror(res), res);\n  }\n\n  return out.str();\n}\n\nlong http_downloader::response_code() {\n  long code{0};\n  curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &code);\n  return code;\n}\n\nsize_t http_downloader::write(void* p, size_t size, size_t bytes, void* stream) {\n  string data{static_cast<const char*>(p), size * bytes};\n  *(static_cast<std::stringstream*>(stream)) << data << '\\n';\n  return size * bytes;\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/string.cpp\n#include \"utils/string.hpp\"\n\n#include <algorithm>\n#include <cassert>\n#include <iomanip>\n#include <sstream>\n#include <utility>\n\nPOLYBAR_NS\n\nnamespace string_util {\n\n/**\n * Prefixes for the leading byte in a UTF8 codepoint\n */\nstatic constexpr uint8_t UTF8_LEADING1_PREFIX = 0b00000000;\nstatic constexpr uint8_t UTF8_LEADING2_PREFIX = 0b11000000;\nstatic constexpr uint8_t UTF8_LEADING3_PREFIX = 0b11100000;\nstatic constexpr uint8_t UTF8_LEADING4_PREFIX = 0b11110000;\n\n/**\n * Masks to extract the prefix from the leading byte in a UTF8 codepoint\n */\nstatic constexpr uint8_t UTF8_LEADING1_MASK = 0b10000000;\nstatic constexpr uint8_t UTF8_LEADING2_MASK = 0b11100000;\nstatic constexpr uint8_t UTF8_LEADING3_MASK = 0b11110000;\nstatic constexpr uint8_t UTF8_LEADING4_MASK = 0b11111000;\n\n/**\n * Prefix for UTF8 continuation bytes\n */\nstatic constexpr uint8_t UTF8_CONTINUATION_PREFIX = 0b10000000;\n/**\n * Mask to extract the UTF8 continuation byte prefix\n */\nstatic constexpr uint8_t UTF8_CONTINUATION_MASK = 0b11000000;\n\n/**\n * Check if haystack contains needle\n */\nbool contains(const string& haystack, const string& needle) {\n  return haystack.find(needle) != string::npos;\n}\n\nbool ends_with(const string& haystack, const string& suffix) {\n  if (haystack.length() < suffix.length()) {\n    return false;\n  }\n\n  return haystack.compare(haystack.length() - suffix.length(), suffix.length(), suffix) == 0;\n}\n\n/**\n * Check if haystack contains needle ignoring case\n */\nbool contains_ignore_case(const string& haystack, const string& needle) {\n  return lower(haystack).find(lower(needle)) != string::npos;\n}\n\n/**\n * Convert string to uppercase\n */\nstring upper(const string& s) {\n  string str(s);\n  for (auto& c : str) {\n    c = toupper(c);\n  }\n  return str;\n}\n\n/**\n * Convert string to lowercase\n */\nstring lower(const string& s) {\n  string str(s);\n  for (auto& c : str) {\n    c = tolower(c);\n  }\n  return str;\n}\n\n/**\n * Test lower case equality\n */\nbool compare(const string& s1, const string& s2) {\n  return lower(s1) == lower(s2);\n}\n\n/**\n * Replace first occurrence of needle in haystack\n */\nstring replace(const string& haystack, const string& needle, const string& replacement, size_t start, size_t end) {\n  string str(haystack);\n  string::size_type pos;\n\n  if (needle != replacement && (pos = str.find(needle, start)) != string::npos) {\n    if (end == string::npos || pos < end) {\n      str = str.replace(pos, needle.length(), replacement);\n    }\n  }\n\n  return str;\n}\n\n/**\n * Replace all occurrences of needle in haystack\n */\nstring replace_all(const string& haystack, const string& needle, const string& replacement, size_t start, size_t end) {\n  string result{haystack};\n  string::size_type pos;\n  while ((pos = result.find(needle, start)) != string::npos && pos < result.length() &&\n         (end == string::npos || pos + needle.length() <= end)) {\n    result.replace(pos, needle.length(), replacement);\n    start = pos + replacement.length();\n  }\n  return result;\n}\n\n/**\n * Replace all consecutive occurrences of needle in haystack\n */\nstring squeeze(const string& haystack, char needle) {\n  string result = haystack;\n  while (result.find({needle, needle}) != string::npos) {\n    result = replace_all(result, {needle, needle}, {needle});\n  }\n  return result;\n}\n\n/**\n * Remove all occurrences of needle in haystack\n */\nstring strip(const string& haystack, char needle) {\n  string str(haystack);\n  string::size_type pos;\n  while ((pos = str.find(needle)) != string::npos) {\n    str.erase(pos, 1);\n  }\n  return str;\n}\n\n/**\n * Remove trailing newline\n */\nstring strip_trailing_newline(const string& haystack) {\n  string str(haystack);\n  if (str[str.length() - 1] == '\\n') {\n    str.erase(str.length() - 1, 1);\n  }\n  return str;\n}\n\n/**\n * Trims all characters that match pred from the left\n */\nstring ltrim(string value, function<bool(char)> pred) {\n  value.erase(value.begin(), find_if(value.begin(), value.end(), std::not_fn(pred)));\n  return value;\n}\n\n/**\n * Trims all characters that match pred from the right\n */\nstring rtrim(string value, function<bool(char)> pred) {\n  value.erase(find_if(value.rbegin(), value.rend(), std::not_fn(pred)).base(), value.end());\n  return value;\n}\n\n/**\n * Trims all characters that match pred from both sides\n */\nstring trim(string value, function<bool(char)> pred) {\n  return ltrim(rtrim(move(value), pred), pred);\n}\n\n/**\n * Remove needle from the start of the string\n */\nstring ltrim(string&& value, const char& needle) {\n  if (value.empty()) {\n    return \"\";\n  }\n  while (*value.begin() == needle) {\n    value.erase(0, 1);\n  }\n  return forward<string>(value);\n}\n\n/**\n * Remove needle from the end of the string\n */\nstring rtrim(string&& value, const char& needle) {\n  if (value.empty()) {\n    return \"\";\n  }\n  while (*(value.end() - 1) == needle) {\n    value.erase(value.length() - 1, 1);\n  }\n  return forward<string>(value);\n}\n\n/**\n * Remove needle from the start and end of the string\n */\nstring trim(string&& value, const char& needle) {\n  if (value.empty()) {\n    return \"\";\n  }\n  return rtrim(ltrim(forward<string>(value), needle), needle);\n}\n\n/**\n * Counts the number of codepoints in a utf8 encoded string.\n */\nsize_t char_len(const string& value) {\n  // utf-8 bytes of the form 10xxxxxx are continuation bytes, so we\n  // simply count the number of bytes not of this form.\n  return std::count_if(\n      value.begin(), value.end(), [](char c) { return (c & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_PREFIX; });\n}\n\n/**\n * Truncates a utf8 string at len number of codepoints. This isn't 100%\n * matching the user-perceived character count, but it should be close\n * enough and avoids having to pull in something like ICU to count actual\n * grapheme clusters.\n */\nstring utf8_truncate(string&& value, size_t len) {\n  if (value.empty()) {\n    return \"\";\n  }\n\n  // utf-8 bytes of the form 10xxxxxx are continuation bytes, so we\n  // simply jump forward to bytes not of that form and truncate starting\n  // at that byte if we've counted too many codepoints\n  auto it = value.begin();\n  auto end = value.end();\n  for (size_t i = 0; i < len; ++i) {\n    if (it == end)\n      break;\n    ++it;\n    it = std::find_if(it, end, [](char c) { return (c & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_PREFIX; });\n  }\n  value.erase(it, end);\n\n  return forward<string>(value);\n}\n\n/**\n * Given a leading byte of a UTF8 codepoint calculates the number of bytes taken up by the codepoint.\n *\n * @returns {len, result} The codepoint is len bytes and result contains the codepoint bits held in the leading byte.\n */\nstatic pair<int, uint32_t> utf8_get_len(uint8_t leading) {\n  if ((leading & UTF8_LEADING1_MASK) == UTF8_LEADING1_PREFIX) {\n    return {1, leading & ~UTF8_LEADING1_MASK};\n  } else if ((leading & UTF8_LEADING2_MASK) == UTF8_LEADING2_PREFIX) {\n    return {2, leading & ~UTF8_LEADING2_MASK};\n  } else if ((leading & UTF8_LEADING3_MASK) == UTF8_LEADING3_PREFIX) {\n    return {3, leading & ~UTF8_LEADING3_MASK};\n  } else if ((leading & UTF8_LEADING4_MASK) == UTF8_LEADING4_PREFIX) {\n    return {4, leading & ~UTF8_LEADING4_MASK};\n  } else {\n    return {-1, 0};\n  }\n}\n\n/**\n * @brief Create a list of UCS-4 codepoint from a utf-8 encoded string\n *\n * If invalid utf8 characters are encountered they are skipped until the next valid codepoint and the function will\n * eventually return false.\n *\n * The result_list is always populated with all valid utf8 codepoints.\n *\n * @return Whether the string is completely valid utf8\n */\nbool utf8_to_ucs4(const string& src, unicode_charlist& result_list) {\n  result_list.reserve(src.size());\n  bool has_errors = false;\n  const auto* begin = reinterpret_cast<const uint8_t*>(src.c_str());\n\n  const auto* current = begin;\n  while (*current) {\n    // Number of bytes taken up by this codepoint and the bits contained in the leading byte.\n    auto [len, result] = utf8_get_len(*current);\n    auto offset = current - begin;\n\n    /*\n     * Invalid lengths, this byte is not a valid leading byte.\n     * Skip it.\n     */\n    if (len <= 0 || len > 4) {\n      has_errors = true;\n      current++;\n      continue;\n    }\n\n    const uint8_t* next = current + 1;\n    for (; ((*next & UTF8_CONTINUATION_MASK) == UTF8_CONTINUATION_PREFIX) && (next - current < len); next++) {\n      result = result << 6;\n      result |= *next & ~UTF8_CONTINUATION_MASK;\n    }\n\n    auto actual_len = next - current;\n    current = next;\n\n    if (actual_len != len) {\n      has_errors = true;\n      continue;\n    }\n\n    result_list.push_back(unicode_character{result, static_cast<int>(offset), static_cast<int>(actual_len)});\n    current = next;\n  }\n\n  return !has_errors;\n}\n\n/**\n * @brief Convert a UCS-4 codepoint to a utf-8 encoded string\n */\nsize_t ucs4_to_utf8(std::array<char, 5>& utf8, uint32_t ucs) {\n  if (ucs <= 0x7f) {\n    utf8[0] = ucs;\n    return 1;\n  } else if (ucs <= 0x07ff) {\n    utf8[0] = ((ucs >> 6) & ~UTF8_LEADING2_MASK) | UTF8_LEADING2_PREFIX;\n    utf8[1] = (ucs & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    return 2;\n  } else if (ucs <= 0xffff) {\n    utf8[0] = ((ucs >> 12) & ~UTF8_LEADING3_MASK) | UTF8_LEADING3_PREFIX;\n    utf8[1] = ((ucs >> 6) & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    utf8[2] = (ucs & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    return 3;\n  } else if (ucs <= 0x10ffff) {\n    utf8[0] = ((ucs >> 18) & ~UTF8_LEADING4_MASK) | UTF8_LEADING4_PREFIX;\n    utf8[1] = ((ucs >> 12) & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    utf8[2] = ((ucs >> 6) & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    utf8[3] = (ucs & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    return 4;\n  } else {\n    return 0;\n  }\n}\n\n/**\n * Join all strings in vector into a single string separated by delim\n */\nstring join(const vector<string>& strs, const string& delim) {\n  string str;\n  for (auto& s : strs) {\n    str += (str.empty() ? \"\" : delim) + s;\n  }\n  return str;\n}\n\n/**\n * Explode string by delim, ignore empty tokens\n */\nvector<string> split(const string& s, char delim) {\n  std::string::size_type pos = 0;\n  std::vector<std::string> result;\n\n  while ((pos = s.find_first_not_of(delim, pos)) != std::string::npos) {\n    auto nextpos = s.find_first_of(delim, pos);\n    result.emplace_back(s.substr(pos, nextpos - pos));\n    pos = nextpos;\n  }\n\n  return result;\n}\n\n/**\n * Explode string by delim, include empty tokens\n */\nstd::vector<std::string> tokenize(const string& str, char delimiters) {\n  std::vector<std::string> tokens;\n  std::string::size_type lastPos = 0;\n  auto pos = str.find_first_of(delimiters, lastPos);\n\n  while (pos != std::string::npos && lastPos != std::string::npos) {\n    tokens.emplace_back(str.substr(lastPos, pos - lastPos));\n\n    lastPos = pos + 1;\n    pos = str.find_first_of(delimiters, lastPos);\n  }\n\n  tokens.emplace_back(str.substr(lastPos, pos - lastPos));\n  return tokens;\n}\n\n/**\n * Find the nth occurrence of needle in haystack starting from pos\n */\nsize_t find_nth(const string& haystack, size_t pos, const string& needle, size_t nth) {\n  size_t found_pos = haystack.find(needle, pos);\n  if (1 == nth || string::npos == found_pos) {\n    return found_pos;\n  }\n  return find_nth(haystack, found_pos + 1, needle, nth - 1);\n}\n\n/**\n * Create a floating point string\n */\nstring floating_point(double value, size_t precision, bool fixed, const string& locale) {\n  std::stringstream ss;\n  ss.imbue(!locale.empty() ? std::locale(locale.c_str()) : std::locale::classic());\n  ss << std::fixed << std::setprecision(precision) << value;\n  return fixed ? ss.str() : replace(ss.str(), \".00\", \"\");\n}\n\n/**\n * Create a MiB filesize string\n */\nstring filesize_mib(unsigned long long kibibytes, size_t precision, const string& locale) {\n  return floating_point(kibibytes / 1024.0, precision, true, locale) + \" MiB\";\n}\n\n/**\n * Create a GiB filesize string\n */\nstring filesize_gib(unsigned long long kibibytes, size_t precision, const string& locale) {\n  return floating_point(kibibytes / 1024.0 / 1024.0, precision, true, locale) + \" GiB\";\n}\n\n/**\n * Create a GiB string, if the value in GiB is >= 1.0. Otherwise, create a MiB string.\n */\nstring filesize_gib_mib(\n    unsigned long long kibibytes, size_t precision_mib, size_t precision_gib, const string& locale) {\n  if (kibibytes < 1024 * 1024) {\n    return filesize_mib(kibibytes, precision_mib, locale);\n  } else {\n    return filesize_gib(kibibytes, precision_gib, locale);\n  }\n}\n\n/**\n * Create a filesize string by converting given bytes to highest unit possible\n */\nstring filesize(unsigned long long bytes, size_t precision, bool fixed, const string& locale) {\n  vector<string> suffixes{\"TB\", \"GB\", \"MB\", \"KB\"};\n  string suffix{\"B\"};\n  double value = bytes;\n  while (!suffixes.empty() && value >= 1024.0) {\n    suffix = suffixes.back();\n    suffixes.pop_back();\n    value /= 1024.0;\n  }\n  return floating_point(value, precision, fixed, locale) + \" \" + suffix;\n}\n\n/**\n * Compute string hash\n */\nhash_type hash(const string& src) {\n  return std::hash<string>()(src);\n}\n} // namespace string_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/concurrency.cpp\n#include \"utils/concurrency.hpp\"\n\n#include <map>\n\nPOLYBAR_NS\n\nnamespace concurrency_util {\n  size_t thread_id(const thread::id id) {\n    static size_t idx{1_z};\n    static mutex_wrapper<std::map<thread::id, size_t>> ids;\n    std::lock_guard<decltype(ids)> lock(ids);\n    if (ids.find(id) == ids.end()) {\n      ids[id] = idx++;\n    }\n    return ids[id];\n  }\n}  // namespace concurrency_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/units.cpp\n#include \"utils/units.hpp\"\n\n#include \"common.hpp\"\n#include \"components/types.hpp\"\n#include \"errors.hpp\"\n#include \"utils/math.hpp\"\n\nPOLYBAR_NS\n\nnamespace units_utils {\n\n  /**\n   * Converts points to pixels under the given DPI (PPI).\n   *\n   * 1 pt = 1/72in, so point / 72 * DPI = #pixels\n   */\n  int point_to_pixel(double point, double dpi) {\n    return dpi * point / 72.0;\n  }\n\n  /**\n   * Converts an extent value to a pixel value according to the given DPI (if needed).\n   */\n  int extent_to_pixel(const extent_val size, double dpi) {\n    if (size.type == extent_type::PIXEL) {\n      return size.value;\n    }\n\n    return point_to_pixel(size.value, dpi);\n  }\n\n  /**\n   * Same as extent_to_pixel but is capped below at 0 pixels.\n   */\n  unsigned extent_to_pixel_nonnegative(const extent_val size, double dpi) {\n    return std::max(0, extent_to_pixel(size, dpi));\n  }\n\n  /**\n   * Converts a percentage with offset into pixels\n   */\n  int percentage_with_offset_to_pixel(percentage_with_offset g_format, double max, double dpi) {\n    int offset_pixel = extent_to_pixel(g_format.offset, dpi);\n\n    return static_cast<int>(math_util::percentage_to_value<double, double>(g_format.percentage, max) + offset_pixel);\n  }\n\n  /**\n   * Same as percentage_with_offset_to_pixel but capped below at 0 pixels\n   */\n  unsigned percentage_with_offset_to_pixel_nonnegative(percentage_with_offset g_format, double max, double dpi) {\n    return std::max<int>(0, percentage_with_offset_to_pixel(g_format, max, dpi));\n  }\n\n  extent_type parse_extent_unit(const string& str) {\n    if (!str.empty()) {\n      if (str == \"px\") {\n        return extent_type::PIXEL;\n      } else if (str == \"pt\") {\n        return extent_type::POINT;\n      } else {\n        throw std::runtime_error(\"Unrecognized unit '\" + str + \"'\");\n      }\n    } else {\n      return extent_type::PIXEL;\n    }\n  }\n\n  extent_val parse_extent(const string& str) {\n    size_t pos;\n    auto size_value = std::stof(str, &pos);\n\n    string unit = string_util::trim(str.substr(pos));\n    extent_type type = parse_extent_unit(unit);\n\n    // Pixel values should be integers\n    if (type == extent_type::PIXEL) {\n      size_value = std::trunc(size_value);\n    }\n\n    return {type, size_value};\n  }\n\n  string extent_to_string(extent_val extent) {\n    std::stringstream ss;\n\n    switch (extent.type) {\n      case extent_type::POINT:\n        ss << extent.value << \"pt\";\n        break;\n      case extent_type::PIXEL:\n        ss << static_cast<int>(extent.value) << \"px\";\n        break;\n    }\n\n    return ss.str();\n  }\n\n  spacing_type parse_spacing_unit(const string& str) {\n    if (!str.empty()) {\n      if (str == \"px\") {\n        return spacing_type::PIXEL;\n      } else if (str == \"pt\") {\n        return spacing_type::POINT;\n      } else {\n        throw std::runtime_error(\"Unrecognized unit '\" + str + \"'\");\n      }\n    } else {\n      return spacing_type::SPACE;\n    }\n  }\n\n  spacing_val parse_spacing(const string& str) {\n    size_t pos;\n    auto size_value = std::stof(str, &pos);\n\n    if (size_value < 0) {\n      throw runtime_error(sstream() << \"value '\" << str << \"' must not be negative\");\n    }\n\n    spacing_type type;\n\n    string unit = string_util::trim(str.substr(pos));\n    if (!unit.empty()) {\n      if (unit == \"px\") {\n        type = spacing_type::PIXEL;\n        size_value = std::trunc(size_value);\n      } else if (unit == \"pt\") {\n        type = spacing_type::POINT;\n      } else {\n        throw runtime_error(\"Unrecognized unit '\" + unit + \"'\");\n      }\n    } else {\n      type = spacing_type::SPACE;\n      size_value = std::trunc(size_value);\n    }\n\n    return {type, size_value};\n  }\n\n  /**\n   * Creates an extent from a spacing value.\n   *\n   * @param spacing Value to convert, must not be SPACE\n   */\n  extent_val spacing_to_extent(spacing_val spacing) {\n    extent_type t;\n\n    switch (spacing.type) {\n      case spacing_type::POINT:\n        t = extent_type::POINT;\n        break;\n      case spacing_type::PIXEL:\n        t = extent_type::PIXEL;\n        break;\n      default:\n        throw std::runtime_error(\"spacing_to_extent: Illegal type: \" + to_string(to_integral(spacing.type)));\n    }\n\n    return {t, spacing.value};\n  }\n\n} // namespace units_utils\n\nPOLYBAR_NS_END\n\n// Path: src/utils/i3.cpp\n#include \"utils/i3.hpp\"\n\n#include <xcb/xcb.h>\n\n#include <i3ipc++/ipc.hpp>\n\n#include \"common.hpp\"\n#include \"settings.hpp\"\n#include \"utils/socket.hpp\"\n#include \"utils/string.hpp\"\n#include \"x11/connection.hpp\"\n#include \"x11/ewmh.hpp\"\n#include \"x11/icccm.hpp\"\n\nPOLYBAR_NS\n\nnamespace i3_util {\n  /**\n   * Get all workspaces for given output\n   */\n  vector<shared_ptr<workspace_t>> workspaces(const connection_t& conn, const string& output, const bool show_urgent) {\n    vector<shared_ptr<workspace_t>> result;\n    for (auto&& ws : conn.get_workspaces()) {\n      if (output.empty() || ws->output == output || (show_urgent && ws->urgent)) {\n        result.emplace_back(forward<decltype(ws)>(ws));\n      }\n    }\n    return result;\n  }\n\n  /**\n   * Get currently focused workspace\n   */\n  shared_ptr<workspace_t> focused_workspace(const connection_t& conn) {\n    for (auto&& ws : conn.get_workspaces()) {\n      if (ws->focused) {\n        return ws;\n      }\n    }\n    return nullptr;\n  }\n\n  /**\n   * Get main root window\n   */\n  xcb_window_t root_window(connection& conn) {\n    auto children = conn.query_tree(conn.root()).children();\n    const auto wm_name = [&](xcb_connection_t* conn, xcb_window_t win) -> string {\n      string title;\n      if (!(title = ewmh_util::get_wm_name(win)).empty()) {\n        return title;\n      } else if (!(title = icccm_util::get_wm_name(conn, win)).empty()) {\n        return title;\n      } else {\n        return \"\";\n      }\n    };\n\n    for (auto it = children.begin(); it != children.end(); it++) {\n      if (wm_name(conn, *it) == \"i3\") {\n        return *it;\n      }\n    }\n\n    return XCB_NONE;\n  }\n\n  /**\n   * Returns window against which to restack.\n   */\n  restack_util::params get_restack_params(connection& conn) {\n    return {root_window(conn), XCB_STACK_MODE_ABOVE};\n  }\n} // namespace i3_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/process.cpp\n#include \"utils/process.hpp\"\n\n#include <fcntl.h>\n#include <sys/stat.h>\n#include <sys/types.h>\n#include <sys/wait.h>\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace process_util {\n  /**\n   * Check if currently in main process\n   */\n  bool in_parent_process(pid_t pid) {\n    return pid != -1 && pid != 0;\n  }\n\n  /**\n   * Check if currently in subprocess\n   */\n  bool in_forked_process(pid_t pid) {\n    return pid == 0;\n  }\n\n  /**\n   * Redirects all io fds (stdin, stdout, stderr) of the current process to /dev/null.\n   */\n  void redirect_stdio_to_dev_null() {\n    auto redirect = [](int fd_to_redirect) {\n      int fd = open(\"/dev/null\", O_WRONLY);\n...\n", "needle_token_start": 3979, "needle_token_end": 4025, "code_context_ntokens": 15940, "output": ["```cpp\ninotify_watch::inotify_watch(string path) : m_path(move(path)) {}\n```"]}
{"repo": "polybar/polybar", "name": "close_walk_cb", "language": "cpp", "path": "src/components/eventloop.cpp", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to ensure that each handle in the event loop is properly closed if it is not already in the process of closing.\n2. **Input**: It takes a handle from the event loop and an optional user-defined data pointer, which is not used in this context.\n3. **Output**: There is no direct output from this function; however, it triggers the closing of handles that are not already being closed.\n4. **Procedure**: The function checks if the given handle is already in the process of closing. If it is not, the function initiates the closing process for that handle.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/components/bar.cpp\n#include \"components/bar.hpp\"\n\n#include <algorithm>\n\n#include \"components/config.hpp\"\n#include \"components/renderer.hpp\"\n#include \"components/screen.hpp\"\n#include \"components/types.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"events/signal.hpp\"\n#include \"events/signal_emitter.hpp\"\n#include \"tags/dispatch.hpp\"\n#include \"utils/bspwm.hpp\"\n#include \"utils/color.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/restack.hpp\"\n#include \"utils/string.hpp\"\n#include \"utils/units.hpp\"\n#include \"x11/atoms.hpp\"\n#include \"x11/connection.hpp\"\n#include \"x11/ewmh.hpp\"\n#include \"x11/extensions/all.hpp\"\n#include \"x11/icccm.hpp\"\n#include \"x11/legacy_tray_manager.hpp\"\n\n#if WITH_XCURSOR\n#include \"x11/cursor.hpp\"\n#endif\n\n#if ENABLE_I3\n#include \"utils/i3.hpp\"\n#endif\n\nPOLYBAR_NS\n\nusing namespace signals::ui;\nusing namespace eventloop;\n\n/**\n * Create instance\n */\nbar::make_type bar::make(loop& loop, const config& config, bool only_initialize_values) {\n  auto action_ctxt = make_unique<tags::action_context>();\n\n  // clang-format off\n  return std::make_unique<bar>(\n      connection::make(),\n      signal_emitter::make(),\n      config,\n      logger::make(),\n      loop,\n      screen::make(config),\n      tags::dispatch::make(*action_ctxt),\n      std::move(action_ctxt),\n      only_initialize_values);\n  // clang-format on\n}\n\n/**\n * Construct bar instance\n */\nbar::bar(connection& conn, signal_emitter& emitter, const config& config, const logger& logger, loop& loop,\n    unique_ptr<screen>&& screen, unique_ptr<tags::dispatch>&& dispatch, unique_ptr<tags::action_context>&& action_ctxt,\n    bool only_initialize_values)\n    : m_connection(conn)\n    , m_sig(emitter)\n    , m_conf(config)\n    , m_log(logger)\n    , m_loop(loop)\n    , m_screen(forward<decltype(screen)>(screen))\n    , m_dispatch(forward<decltype(dispatch)>(dispatch))\n    , m_action_ctxt(forward<decltype(action_ctxt)>(action_ctxt)) {\n  string bs{m_conf.section()};\n\n  m_tray = legacy_tray::tray_manager::make(m_opts);\n\n  // Get available RandR outputs\n  auto monitor_name = m_conf.get(bs, \"monitor\", \"\"s);\n  auto monitor_name_fallback = m_conf.get(bs, \"monitor-fallback\", \"\"s);\n  m_opts.monitor_strict = m_conf.get(bs, \"monitor-strict\", m_opts.monitor_strict);\n  m_opts.monitor_exact = m_conf.get(bs, \"monitor-exact\", m_opts.monitor_exact);\n  auto monitors = randr_util::get_monitors(m_connection, m_opts.monitor_strict, false);\n\n  if (monitors.empty()) {\n    throw application_error(\"No monitors found\");\n  }\n\n  // if monitor_name is not defined, first check for primary monitor\n  if (monitor_name.empty()) {\n    for (auto&& mon : monitors) {\n      if (mon->primary) {\n        monitor_name = mon->name;\n        break;\n      }\n    }\n  }\n\n  // if still not found (and not strict matching), get first connected monitor\n  if (monitor_name.empty() && !m_opts.monitor_strict) {\n    auto connected_monitors = randr_util::get_monitors(m_connection, true, false);\n    if (!connected_monitors.empty()) {\n      monitor_name = connected_monitors[0]->name;\n      m_log.warn(\"No monitor specified, using \\\"%s\\\"\", monitor_name);\n    }\n  }\n\n  // if still not found, get first monitor\n  if (monitor_name.empty()) {\n    monitor_name = monitors[0]->name;\n    m_log.warn(\"No monitor specified, using \\\"%s\\\"\", monitor_name);\n  }\n\n  // get the monitor data based on the name\n  m_opts.monitor = randr_util::match_monitor(monitors, monitor_name, m_opts.monitor_exact);\n  monitor_t fallback{};\n\n  if (!monitor_name_fallback.empty()) {\n    fallback = randr_util::match_monitor(monitors, monitor_name_fallback, m_opts.monitor_exact);\n  }\n\n  if (!m_opts.monitor) {\n    if (fallback) {\n      m_opts.monitor = move(fallback);\n      m_log.warn(\"Monitor \\\"%s\\\" not found, reverting to fallback \\\"%s\\\"\", monitor_name, m_opts.monitor->name);\n    } else {\n      throw application_error(\"Monitor \\\"\" + monitor_name + \"\\\" not found or disconnected\");\n    }\n  }\n\n  m_log.info(\"Loaded monitor %s (%ix%i+%i+%i)\", m_opts.monitor->name, m_opts.monitor->w, m_opts.monitor->h,\n      m_opts.monitor->x, m_opts.monitor->y);\n\n  m_opts.override_redirect = m_conf.deprecated(bs, \"dock\", \"override-redirect\", m_opts.override_redirect);\n\n  m_opts.dimvalue = m_conf.get(bs, \"dim-value\", 1.0);\n  m_opts.dimvalue = math_util::cap(m_opts.dimvalue, 0.0, 1.0);\n\n#if WITH_XCURSOR\n  m_opts.cursor_click = m_conf.get(bs, \"cursor-click\", \"\"s);\n  if (!m_opts.cursor_click.empty() && !cursor_util::valid(m_opts.cursor_click)) {\n    m_log.warn(\"Ignoring unsupported cursor-click option '%s'\", m_opts.cursor_click);\n    m_opts.cursor_click.clear();\n  }\n\n  m_opts.cursor_scroll = m_conf.get(bs, \"cursor-scroll\", \"\"s);\n  if (!m_opts.cursor_scroll.empty() && !cursor_util::valid(m_opts.cursor_scroll)) {\n    m_log.warn(\"Ignoring unsupported cursor-scroll option '%s'\", m_opts.cursor_scroll);\n    m_opts.cursor_scroll.clear();\n  }\n#else\n  if (m_conf.has(bs, \"cursor-click\")) {\n    m_log.warn(\"Polybar was not compiled with xcursor support, ignoring cursor-click option\");\n  }\n\n  if (m_conf.has(bs, \"cursor-scroll\")) {\n    m_log.warn(\"Polybar was not compiled with xcursor support, ignoring cursor-scroll option\");\n  }\n#endif\n\n  // Build WM_NAME\n  m_opts.wmname = m_conf.get(bs, \"wm-name\", \"polybar-\" + bs.substr(4) + \"_\" + m_opts.monitor->name);\n  m_opts.wmname = string_util::replace(m_opts.wmname, \" \", \"-\");\n\n  // Configure DPI\n  {\n    double dpi_x = 96, dpi_y = 96;\n    if (m_conf.has(m_conf.section(), \"dpi\")) {\n      dpi_x = dpi_y = m_conf.get<double>(\"dpi\");\n    } else {\n      if (m_conf.has(m_conf.section(), \"dpi-x\")) {\n        dpi_x = m_conf.get<double>(\"dpi-x\");\n      }\n      if (m_conf.has(m_conf.section(), \"dpi-y\")) {\n        dpi_y = m_conf.get<double>(\"dpi-y\");\n      }\n    }\n\n    // dpi to be computed\n    if (dpi_x <= 0 || dpi_y <= 0) {\n      auto screen = m_connection.screen();\n      if (dpi_x <= 0) {\n        dpi_x = screen->width_in_pixels * 25.4 / screen->width_in_millimeters;\n      }\n      if (dpi_y <= 0) {\n        dpi_y = screen->height_in_pixels * 25.4 / screen->height_in_millimeters;\n      }\n    }\n\n    m_opts.dpi_x = dpi_x;\n    m_opts.dpi_y = dpi_y;\n\n    m_log.info(\"Configured DPI = %gx%g\", dpi_x, dpi_y);\n  }\n\n  // Load configuration values\n\n  m_opts.bottom = m_conf.get(bs, \"bottom\", m_opts.bottom);\n  m_opts.spacing = m_conf.get(bs, \"spacing\", m_opts.spacing);\n  m_opts.separator = drawtypes::load_optional_label(m_conf, bs, \"separator\", \"\");\n  m_opts.locale = m_conf.get(bs, \"locale\", \"\"s);\n\n  auto radius = m_conf.get<double>(bs, \"radius\", 0.0);\n  auto top = m_conf.get(bs, \"radius-top\", radius);\n  m_opts.radius.top_left = m_conf.get(bs, \"radius-top-left\", top);\n  m_opts.radius.top_right = m_conf.get(bs, \"radius-top-right\", top);\n  auto bottom = m_conf.get(bs, \"radius-bottom\", radius);\n  m_opts.radius.bottom_left = m_conf.get(bs, \"radius-bottom-left\", bottom);\n  m_opts.radius.bottom_right = m_conf.get(bs, \"radius-bottom-right\", bottom);\n\n  auto padding = m_conf.get(bs, \"padding\", ZERO_SPACE);\n  m_opts.padding.left = m_conf.get(bs, \"padding-left\", padding);\n  m_opts.padding.right = m_conf.get(bs, \"padding-right\", padding);\n\n  auto margin = m_conf.get(bs, \"module-margin\", ZERO_SPACE);\n  m_opts.module_margin.left = m_conf.get(bs, \"module-margin-left\", margin);\n  m_opts.module_margin.right = m_conf.get(bs, \"module-margin-right\", margin);\n\n  m_opts.double_click_interval = m_conf.get(bs, \"double-click-interval\", m_opts.double_click_interval);\n\n  m_opts.struts = m_conf.get(bs, \"enable-struts\", m_opts.struts);\n\n  if (only_initialize_values) {\n    return;\n  }\n\n  // Load values used to adjust the struts atom\n\n  if (!m_opts.struts) {\n    if (m_conf.has(\"global/wm\", \"margin-bottom\")) {\n      m_log.warn(\"Struts are disabled, ignoring margin-bottom\");\n    }\n    if (m_conf.has(\"global/wm\", \"margin-top\")) {\n      m_log.warn(\"Struts are disabled, ignoring margin-top\");\n    }\n  }\n  auto margin_top = m_conf.get(\"global/wm\", \"margin-top\", percentage_with_offset{});\n  auto margin_bottom = m_conf.get(\"global/wm\", \"margin-bottom\", percentage_with_offset{});\n  m_opts.strut.top = units_utils::percentage_with_offset_to_pixel(margin_top, m_opts.monitor->h, m_opts.dpi_y);\n  m_opts.strut.bottom = units_utils::percentage_with_offset_to_pixel(margin_bottom, m_opts.monitor->h, m_opts.dpi_y);\n\n  // Load commands used for fallback click handlers\n  vector<action> actions;\n  actions.emplace_back(action{mousebtn::LEFT, m_conf.get(bs, \"click-left\", \"\"s)});\n  actions.emplace_back(action{mousebtn::MIDDLE, m_conf.get(bs, \"click-middle\", \"\"s)});\n  actions.emplace_back(action{mousebtn::RIGHT, m_conf.get(bs, \"click-right\", \"\"s)});\n  actions.emplace_back(action{mousebtn::SCROLL_UP, m_conf.get(bs, \"scroll-up\", \"\"s)});\n...\n// Path: src/components/eventloop.cpp\n#include \"components/eventloop.hpp\"\n\n#include <cassert>\n#include <utility>\n\n#include \"errors.hpp\"\n\n#if !(UV_VERSION_MAJOR == 1 && UV_VERSION_MINOR >= 3)\n#error \"Polybar requires libuv 1.x and at least version 1.3\"\n#endif\n\nPOLYBAR_NS\n\nnamespace eventloop {\n\n  /**\n   * Closes the given wrapper.\n   *\n   * We have to have distinct cases for all types because we can't just cast to `Handle` without template\n   * arguments.\n   */\n  void close_handle(uv_handle_t* handle) {\n    switch (handle->type) {\n      case UV_ASYNC:\n        static_cast<AsyncHandle*>(handle->data)->close();\n        break;\n      case UV_FS_EVENT:\n        static_cast<FSEventHandle*>(handle->data)->close();\n        break;\n      case UV_POLL:\n        static_cast<PollHandle*>(handle->data)->close();\n        break;\n      case UV_TIMER:\n        static_cast<TimerHandle*>(handle->data)->close();\n        break;\n      case UV_SIGNAL:\n        static_cast<SignalHandle*>(handle->data)->close();\n        break;\n      case UV_NAMED_PIPE:\n        static_cast<PipeHandle*>(handle->data)->close();\n        break;\n      case UV_PREPARE:\n        static_cast<PrepareHandle*>(handle->data)->close();\n        break;\n      default:\n        assert(false);\n    }\n  }\n\n  // WriteRequest {{{\n  WriteRequest::WriteRequest(cb_write&& user_cb, cb_error&& err_cb)\n      : write_callback(std::move(user_cb)), write_err_cb(std::move(err_cb)) {\n    get()->data = this;\n  }\n\n  WriteRequest& WriteRequest::create(cb_write&& user_cb, cb_error&& err_cb) {\n    auto r = std::make_unique<WriteRequest>(std::move(user_cb), std::move(err_cb));\n    return r->leak(std::move(r));\n  }\n\n  uv_write_t* WriteRequest::get() {\n    return &req;\n  }\n\n  void WriteRequest::trigger(int status) {\n    if (status < 0) {\n      if (write_err_cb) {\n        write_err_cb(ErrorEvent{status});\n      }\n    } else {\n      if (write_callback) {\n        write_callback();\n      }\n    }\n\n    unleak();\n  }\n\n  WriteRequest& WriteRequest::leak(std::unique_ptr<WriteRequest> h) {\n    lifetime_extender = std::move(h);\n    return *lifetime_extender;\n  }\n\n  void WriteRequest::unleak() {\n    reset_callbacks();\n    lifetime_extender.reset();\n  }\n\n  void WriteRequest::reset_callbacks() {\n    write_callback = nullptr;\n    write_err_cb = nullptr;\n  }\n  // }}}\n\n  // SignalHandle {{{\n  void SignalHandle::init() {\n    UV(uv_signal_init, loop(), get());\n  }\n\n  void SignalHandle::start(int signum, cb&& user_cb) {\n    this->callback = std::move(user_cb);\n    UV(uv_signal_start, get(), event_cb<SignalEvent, &SignalHandle::callback>, signum);\n  }\n\n  void SignalHandle::reset_callbacks() {\n    callback = nullptr;\n  }\n  // }}}\n\n  // PollHandle {{{\n  void PollHandle::init(int fd) {\n    UV(uv_poll_init, loop(), get(), fd);\n  }\n\n  void PollHandle::start(int events, cb&& user_cb, cb_error&& err_cb) {\n    this->callback = std::move(user_cb);\n    this->err_cb = std::move(err_cb);\n    UV(uv_poll_start, get(), events, &poll_callback);\n  }\n\n  void PollHandle::poll_callback(uv_poll_t* handle, int status, int events) {\n    auto& self = cast(handle);\n    if (status < 0) {\n      self.err_cb(ErrorEvent{status});\n      return;\n    }\n\n    self.callback(PollEvent{(uv_poll_event)events});\n  }\n\n  void PollHandle::reset_callbacks() {\n    callback = nullptr;\n    err_cb = nullptr;\n  }\n  // }}}\n\n  // FSEventHandle {{{\n  void FSEventHandle::init() {\n    UV(uv_fs_event_init, loop(), get());\n  }\n\n  void FSEventHandle::start(const string& path, int flags, cb&& user_cb, cb_error&& err_cb) {\n    this->callback = std::move(user_cb);\n    this->err_cb = std::move(err_cb);\n    UV(uv_fs_event_start, get(), fs_event_callback, path.c_str(), flags);\n  }\n\n  void FSEventHandle::fs_event_callback(uv_fs_event_t* handle, const char* path, int events, int status) {\n    auto& self = cast(handle);\n    if (status < 0) {\n      self.err_cb(ErrorEvent{status});\n      return;\n    }\n\n    self.callback(FSEvent{path, (uv_fs_event)events});\n  }\n\n  void FSEventHandle::reset_callbacks() {\n    callback = nullptr;\n    err_cb = nullptr;\n  }\n  // }}}\n\n  // PipeHandle {{{\n  void PipeHandle::init(bool ipc) {\n    UV(uv_pipe_init, loop(), get(), ipc);\n  }\n\n  void PipeHandle::open(int fd) {\n    UV(uv_pipe_open, get(), fd);\n  }\n\n  void PipeHandle::bind(const string& path) {\n    UV(uv_pipe_bind, get(), path.c_str());\n  }\n\n  void PipeHandle::connect(const string& name, cb_connect&& user_cb, cb_error&& err_cb) {\n    this->connect_callback = std::move(user_cb);\n    this->connect_err_cb = std::move(err_cb);\n    uv_pipe_connect(new uv_connect_t(), get(), name.c_str(), connect_cb);\n  }\n\n  void PipeHandle::connect_cb(uv_connect_t* req, int status) {\n    auto& self = PipeHandle::cast((uv_pipe_t*)req->handle);\n\n    if (status < 0) {\n      self.connect_err_cb(ErrorEvent{status});\n    } else {\n      self.connect_callback();\n    }\n\n    delete req;\n  }\n\n  void PipeHandle::reset_callbacks() {\n    StreamHandle::reset_callbacks();\n    connect_callback = nullptr;\n    connect_err_cb = nullptr;\n  }\n  // }}}\n\n  // TimerHandle {{{\n  void TimerHandle::init() {\n    UV(uv_timer_init, loop(), get());\n  }\n\n  void TimerHandle::start(uint64_t timeout, uint64_t repeat, cb&& user_cb) {\n    this->callback = std::move(user_cb);\n    UV(uv_timer_start, get(), void_event_cb<&TimerHandle::callback>, timeout, repeat);\n  }\n\n  void TimerHandle::stop() {\n    UV(uv_timer_stop, get());\n  }\n\n  void TimerHandle::reset_callbacks() {\n    callback = nullptr;\n  }\n  // }}}\n\n  // AsyncHandle {{{\n  void AsyncHandle::init(cb&& user_cb) {\n    this->callback = std::move(user_cb);\n    UV(uv_async_init, loop(), get(), void_event_cb<&AsyncHandle::callback>);\n  }\n\n  void AsyncHandle::send() {\n    UV(uv_async_send, get());\n  }\n\n  void AsyncHandle::reset_callbacks() {\n    callback = nullptr;\n  }\n  // }}}\n\n  // PrepareHandle {{{\n  void PrepareHandle::init() {\n    UV(uv_prepare_init, loop(), get());\n  }\n\n  void PrepareHandle::start(cb&& user_cb) {\n    this->callback = std::move(user_cb);\n    UV(uv_prepare_start, get(), void_event_cb<&PrepareHandle::callback>);\n  }\n\n  void PrepareHandle::reset_callbacks() {\n    callback = nullptr;\n  }\n  // }}}\n\n  // eventloop {{{\n  \nstatic void close_walk_cb(uv_handle_t* handle, void*) {\n    if (!uv_is_closing(handle)) {\n      close_handle(handle);\n    }\n  }\n\n  /**\n   * Completely closes everything in the loop.\n   *\n   * After this function returns, uv_loop_close can be called.\n   */\n  static void close_loop(uv_loop_t* loop) {\n    uv_walk(loop, close_walk_cb, nullptr);\n    UV(uv_run, loop, UV_RUN_DEFAULT);\n  }\n\n  loop::loop() {\n    m_loop = std::make_unique<uv_loop_t>();\n    UV(uv_loop_init, m_loop.get());\n    m_loop->data = this;\n  }\n\n  loop::~loop() {\n    if (m_loop) {\n      try {\n        close_loop(m_loop.get());\n        UV(uv_loop_close, m_loop.get());\n      } catch (const std::exception& e) {\n        logger::make().err(\"%s\", e.what());\n      }\n\n      m_loop.reset();\n    }\n  }\n\n  void loop::run() {\n    UV(uv_run, m_loop.get(), UV_RUN_DEFAULT);\n  }\n\n  void loop::stop() {\n    uv_stop(m_loop.get());\n  }\n\n  uint64_t loop::now() const {\n    return uv_now(m_loop.get());\n  }\n\n  uv_loop_t* loop::get() const {\n    return m_loop.get();\n  }\n  // }}}\n\n} // namespace eventloop\n\nPOLYBAR_NS_END\n\n// Path: src/components/command_line.cpp\n#include \"components/command_line.hpp\"\n\n#include <algorithm>\n\nPOLYBAR_NS\n\nnamespace command_line {\n  /**\n   * Create instance\n   */\n  parser::make_type parser::make(string&& scriptname, const options&& opts) {\n    return std::make_unique<parser>(\"Usage: \" + scriptname + \" [OPTION]... [BAR]\", forward<decltype(opts)>(opts));\n  }\n\n  /**\n   * Construct parser\n   */\n  parser::parser(string&& synopsis, const options&& opts)\n      : m_synopsis(forward<decltype(synopsis)>(synopsis)), m_opts(forward<decltype(opts)>(opts)) {}\n\n  /**\n   * Print application usage message\n   */\n  void parser::usage() const {\n    printf(\"%s\\n\\n\", m_synopsis.c_str());\n\n    // get the length of the longest string in the flag column\n    // which is used to align the description fields\n    size_t maxlen{0};\n\n    for (const auto& m_opt : m_opts) {\n      size_t len{m_opt.flag_long.length() + m_opt.flag.length() + 4};\n      maxlen = len > maxlen ? len : maxlen;\n    }\n\n    for (auto& opt : m_opts) {\n      size_t pad = maxlen - opt.flag_long.length() - opt.token.length();\n\n      printf(\"  %s, %s\", opt.flag.c_str(), opt.flag_long.c_str());\n\n      if (!opt.token.empty()) {\n        printf(\"=%s\", opt.token.c_str());\n        pad--;\n      }\n\n      // output the list with accepted values\n      if (!opt.values.empty()) {\n        printf(\"%*s\\n\", static_cast<int>(pad + opt.desc.length()), opt.desc.c_str());\n\n        pad = pad + opt.flag_long.length() + opt.token.length() + 7;\n\n        printf(\"%*c%s is one of: \", static_cast<int>(pad), ' ', opt.token.c_str());\n\n        for (auto& v : opt.values) {\n          printf(\"%s%s\", v.c_str(), v != opt.values.back() ? \", \" : \"\");\n        }\n      } else {\n        printf(\"%*s\", static_cast<int>(pad + opt.desc.length()), opt.desc.c_str());\n      }\n\n      printf(\"\\n\");\n    }\n\n    printf(\"\\n\");\n  }\n\n  /**\n   * Process input values\n   */\n  void parser::process_input(const vector<string>& values) {\n    for (size_t i = 0; i < values.size(); i++) {\n      parse(values[i], values.size() > i + 1 ? values[i + 1] : \"\");\n    }\n  }\n\n  /**\n   * Test if the passed option was provided\n   */\n  bool parser::has(const string& option) const {\n    return m_optvalues.find(option) != m_optvalues.end();\n  }\n\n  /**\n   * Test if a positional argument is defined at given index\n   */\n  bool parser::has(size_t index) const {\n    return m_posargs.size() > index;\n  }\n\n  /**\n   * Get the value defined for given option\n   */\n  string parser::get(string opt) const {\n    if (has(forward<string>(opt))) {\n      return m_optvalues.find(opt)->second;\n    }\n    return \"\";\n  }\n\n  /**\n   * Get the positional argument at given index\n   */\n  string parser::get(size_t index) const {\n    return has(index) ? m_posargs[index] : \"\";\n  }\n\n  /**\n   * Compare option value with given string\n   */\n  bool parser::compare(string opt, const string& val) const {\n    return get(move(opt)) == val;\n  }\n\n  /**\n   * Compare positional argument at given index with given string\n   */\n  bool parser::compare(size_t index, const string& val) const {\n    return get(index) == val;\n  }\n\n  /**\n   * Compare option with its short version\n   */\n  auto parser::is_short(const string& option, const string& opt_short) const {\n    return option.compare(0, opt_short.length(), opt_short) == 0;\n  }\n\n  /**\n   * Compare option with its long version\n   */\n  auto parser::is_long(const string& option, const string& opt_long) const {\n    return option.compare(0, opt_long.length(), opt_long) == 0;\n  }\n\n  /**\n   * Compare option with both versions\n   */\n  auto parser::is(const string& option, string opt_short, string opt_long) const {\n    return is_short(option, move(opt_short)) || is_long(option, move(opt_long));\n  }\n\n  /**\n   * Parse option value\n   */\n  auto parser::parse_value(string input, const string& input_next, choices values) const {\n    string opt = move(input);\n    size_t pos;\n    string value;\n\n    if (input_next.empty() && opt.compare(0, 2, \"--\") != 0) {\n      throw value_error(\"Missing argument for option \" + opt);\n    } else if ((pos = opt.find('=')) == string::npos && opt.compare(0, 2, \"--\") == 0) {\n      throw value_error(\"Missing argument for option \" + opt);\n    } else if (pos == string::npos && !input_next.empty()) {\n      value = input_next;\n    } else {\n      value = opt.substr(pos + 1);\n      opt = opt.substr(0, pos);\n    }\n\n    if (!values.empty() && std::find(values.begin(), values.end(), value) == values.end()) {\n      throw value_error(\"Invalid argument for option \" + opt);\n    }\n\n    return value;\n  }\n\n  /**\n   * Parse and validate passed arguments and flags\n   */\n  void parser::parse(const string& input, const string& input_next) {\n    auto skipped = m_skipnext;\n    if (m_skipnext) {\n      m_skipnext = false;\n      if (!input_next.empty()) {\n        return;\n      }\n    }\n\n    for (auto&& opt : m_opts) {\n      if (is(input, opt.flag, opt.flag_long)) {\n        if (opt.token.empty()) {\n          m_optvalues.insert(make_pair(opt.flag_long.substr(2), \"\"));\n        } else {\n          auto value = parse_value(input, input_next, opt.values);\n          m_skipnext = (value == input_next);\n          m_optvalues.insert(make_pair(opt.flag_long.substr(2), value));\n        }\n        return;\n      }\n    }\n\n    if (skipped) {\n      return;\n    } else if (input[0] != '-') {\n      m_posargs.emplace_back(input);\n    } else {\n      throw argument_error(\"Unrecognized option \" + input);\n    }\n  }\n}  // namespace command_line\n\nPOLYBAR_NS_END\n\n// Path: src/components/logger.cpp\n#include \"components/logger.hpp\"\n\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"settings.hpp\"\n#include \"utils/concurrency.hpp\"\n#include \"utils/factory.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\n/**\n * Convert string\n */\nconst char* logger::convert(string& arg) const {\n  return arg.c_str();\n}\n\nconst char* logger::convert(const string& arg) const {\n  return arg.c_str();\n}\n\n/**\n * Convert thread id\n */\nsize_t logger::convert(const std::thread::id arg) const {\n  return concurrency_util::thread_id(arg);\n}\n\n/**\n * Create instance\n */\nlogger::make_type logger::make(loglevel level) {\n  return *factory_util::singleton<std::remove_reference_t<logger::make_type>>(level);\n}\n\n/**\n * Construct logger\n */\nlogger::logger(loglevel level) : m_level(level) {\n  // clang-format off\n  if (isatty(m_fd)) {\n    m_prefixes[loglevel::TRACE]   = \"\\r\\033[0;32m- \\033[0m\";\n    m_prefixes[loglevel::INFO]    = \"\\r\\033[1;32m* \\033[0m\";\n    m_prefixes[loglevel::NOTICE]  = \"\\r\\033[1;34mnotice: \\033[0m\";\n    m_prefixes[loglevel::WARNING] = \"\\r\\033[1;33mwarn: \\033[0m\";\n    m_prefixes[loglevel::ERROR]   = \"\\r\\033[1;31merror: \\033[0m\";\n    m_suffixes[loglevel::TRACE]   = \"\\033[0m\";\n    m_suffixes[loglevel::INFO]    = \"\\033[0m\";\n    m_suffixes[loglevel::NOTICE]  = \"\\033[0m\";\n    m_suffixes[loglevel::WARNING] = \"\\033[0m\";\n    m_suffixes[loglevel::ERROR]   = \"\\033[0m\";\n  } else {\n    m_prefixes.emplace(make_pair(loglevel::TRACE,   \"polybar|trace: \"));\n    m_prefixes.emplace(make_pair(loglevel::INFO,    \"polybar|info:  \"));\n    m_prefixes.emplace(make_pair(loglevel::NOTICE,  \"polybar|notice:  \"));\n    m_prefixes.emplace(make_pair(loglevel::WARNING, \"polybar|warn:  \"));\n    m_prefixes.emplace(make_pair(loglevel::ERROR,   \"polybar|error: \"));\n    m_suffixes.emplace(make_pair(loglevel::TRACE,   \"\"));\n    m_suffixes.emplace(make_pair(loglevel::INFO,    \"\"));\n    m_suffixes.emplace(make_pair(loglevel::NOTICE,  \"\"));\n    m_suffixes.emplace(make_pair(loglevel::WARNING, \"\"));\n    m_suffixes.emplace(make_pair(loglevel::ERROR,   \"\"));\n  }\n  // clang-format on\n}\n\n/**\n * Set output verbosity\n */\nvoid logger::verbosity(loglevel level) {\n#ifndef DEBUG_LOGGER\n  if (level == loglevel::TRACE) {\n    throw application_error(\"Trace logging is not enabled...\");\n  }\n#endif\n  m_level = level;\n}\n\n/**\n * Convert given loglevel name to its enum type counterpart\n */\nloglevel logger::parse_verbosity(const string& name, loglevel fallback) {\n  if (string_util::compare(name, \"error\")) {\n    return loglevel::ERROR;\n  } else if (string_util::compare(name, \"warning\")) {\n    return loglevel::WARNING;\n  } else if (string_util::compare(name, \"notice\")) {\n    return loglevel::NOTICE;\n  } else if (string_util::compare(name, \"info\")) {\n    return loglevel::INFO;\n  } else if (string_util::compare(name, \"trace\")) {\n    return loglevel::TRACE;\n  } else {\n    return fallback;\n  }\n}\n\nPOLYBAR_NS_END\n\n// Path: src/components/config_parser.cpp\n#include \"components/config_parser.hpp\"\n\n#include <algorithm>\n#include <cerrno>\n#include <cstring>\n#include <fstream>\n\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nconfig_parser::config_parser(const logger& logger, string&& file)\n    : m_log(logger), m_config_file(file_util::expand(file)) {}\n\nconfig config_parser::parse(string barname) {\n  m_log.notice(\"Parsing config file: %s\", m_config_file);\n\n  parse_file(m_config_file, {});\n\n  sectionmap_t sections = create_sectionmap();\n\n  vector<string> bars = get_bars(sections);\n  if (barname.empty()) {\n    if (bars.size() == 1) {\n      barname = bars[0];\n    } else if (bars.empty()) {\n      throw application_error(\"The config file contains no bar.\");\n    } else {\n      throw application_error(\"The config file contains multiple bars, but no bar name was given. Available bars: \" +\n                              string_util::join(bars, \", \"));\n    }\n  } else if (sections.find(\"bar/\" + barname) == sections.end()) {\n    if (bars.empty()) {\n      throw application_error(\"Undefined bar: \" + barname + \". The config file contains no bar.\");\n    } else {\n      throw application_error(\n          \"Undefined bar: \" + barname + \". Available bars: \" + string_util::join(get_bars(sections), \", \"));\n    }\n  }\n\n  /*\n   * The first element in the files vector is always the main config file and\n   * because it has unique filenames, we can use all the elements from the\n   * second element onwards for the included list\n   */\n  file_list included(m_files.begin() + 1, m_files.end());\n  config conf(m_log, move(m_config_file), move(barname));\n\n  conf.set_sections(move(sections));\n  conf.set_included(move(included));\n  if (use_xrm) {\n    conf.use_xrm();\n  }\n\n  return conf;\n}\n\nsectionmap_t config_parser::create_sectionmap() {\n  sectionmap_t sections{};\n\n  string current_section{};\n\n  for (const line_t& line : m_lines) {\n    if (!line.useful) {\n      continue;\n    }\n\n    if (line.is_header) {\n      current_section = line.header;\n    } else {\n      // The first valid line in the config is not a section definition\n      if (current_section.empty()) {\n        throw syntax_error(\"First valid line in config must be section header\", m_files[line.file_index], line.line_no);\n      }\n\n      const string& key = line.key;\n      const string& value = line.value;\n\n      valuemap_t& valuemap = sections[current_section];\n\n      if (valuemap.find(key) == valuemap.end()) {\n        valuemap.emplace(key, value);\n      } else {\n        // Key already exists in this section\n        throw syntax_error(\"Duplicate key name \\\"\" + key + \"\\\" defined in section \\\"\" + current_section + \"\\\"\",\n            m_files[line.file_index], line.line_no);\n      }\n    }\n  }\n\n  return sections;\n}\n\n/**\n * Get the bars declared\n */\nvector<string> config_parser::get_bars(const sectionmap_t& sections) const {\n  vector<string> bars;\n  for (const auto& it : sections) {\n    if (it.first.find(config::BAR_PREFIX) == 0) {\n      bars.push_back(it.first.substr(strlen(config::BAR_PREFIX)));\n    }\n  }\n  return bars;\n}\n\nvoid config_parser::parse_file(const string& file, file_list path) {\n  if (std::find(path.begin(), path.end(), file) != path.end()) {\n    string path_str{};\n\n    for (const auto& p : path) {\n      path_str += \">\\t\" + p + \"\\n\";\n    }\n\n    path_str += \">\\t\" + file;\n\n    // We have already parsed this file in this path, so there are cyclic dependencies\n    throw application_error(\"include-file: Dependency cycle detected:\\n\" + path_str);\n  }\n\n  if (!file_util::exists(file)) {\n    throw application_error(\"Failed to open config file \" + file + \": \" + strerror(errno));\n  }\n\n  if (file_util::is_dir(file)) {\n    throw application_error(\"Config file \" + file + \" is a directory\");\n  }\n\n  m_log.trace(\"config_parser: Parsing %s\", file);\n\n  int file_index;\n\n  auto found = std::find(m_files.begin(), m_files.end(), file);\n\n  if (found == m_files.end()) {\n    file_index = m_files.size();\n    m_files.push_back(file);\n  } else {\n    /*\n     * `file` is already in the `files` vector so we calculate its index.\n     *\n     * This means that the file was already parsed, this can happen without\n     * cyclic dependencies, if the file is included twice\n     */\n    file_index = found - m_files.begin();\n  }\n\n  path.push_back(file);\n\n  int line_no = 0;\n\n  string line_str{};\n\n  std::ifstream in(file);\n\n  if (!in) {\n    throw application_error(\"Failed to open config file \" + file + \": \" + strerror(errno));\n  }\n\n  auto dirname = file_util::dirname(file);\n\n  while (std::getline(in, line_str)) {\n    line_no++;\n    line_t line;\n    line.file_index = file_index;\n    line.line_no = line_no;\n    parse_line(line, line_str);\n\n    // Skip useless lines (comments, empty lines)\n    if (!line.useful) {\n      continue;\n    }\n\n    if (!line.is_header && line.key == \"include-file\") {\n      parse_file(file_util::expand(line.value, dirname), path);\n    } else if (!line.is_header && line.key == \"include-directory\") {\n      const string expanded_path = file_util::expand(line.value, dirname);\n      vector<string> file_list = file_util::list_files(expanded_path);\n      sort(file_list.begin(), file_list.end());\n      for (const auto& filename : file_list) {\n        parse_file(expanded_path + \"/\" + filename, path);\n      }\n    } else {\n      m_lines.push_back(line);\n    }\n  }\n}\n\nvoid config_parser::parse_line(line_t& line, const string& line_str) {\n  if (string_util::contains(line_str, \"\\ufeff\")) {\n    throw syntax_error(\n        \"This config file uses UTF-8 with BOM, which is not supported. Please use plain UTF-8 without BOM.\",\n        m_files[line.file_index], line.line_no);\n  }\n\n  string line_trimmed = string_util::trim(line_str, isspace);\n  line_type type = get_line_type(line_trimmed);\n\n  if (type == line_type::EMPTY || type == line_type::COMMENT) {\n    line.useful = false;\n    return;\n  }\n\n  if (type == line_type::UNKNOWN) {\n    throw syntax_error(\"Unknown line type: \" + line_trimmed, m_files[line.file_index], line.line_no);\n  }\n\n  line.useful = true;\n\n  if (type == line_type::HEADER) {\n    line.is_header = true;\n    line.header = parse_header(line, line_trimmed);\n  } else if (type == line_type::KEY) {\n    line.is_header = false;\n    auto key_value = parse_key(line, line_trimmed);\n    line.key = key_value.first;\n    line.value = key_value.second;\n  }\n}\n\nline_type config_parser::get_line_type(const string& line) {\n  if (line.empty()) {\n    return line_type::EMPTY;\n  }\n\n  switch (line[0]) {\n    case '[':\n      return line_type::HEADER;\n\n    case ';':\n    case '#':\n      return line_type::COMMENT;\n\n    default: {\n      if (string_util::contains(line, \"=\")) {\n        return line_type::KEY;\n      } else {\n        return line_type::UNKNOWN;\n      }\n    }\n  }\n}\n\nstring config_parser::parse_header(const line_t& line, const string& line_str) {\n  if (line_str.back() != ']') {\n    throw syntax_error(\"Missing ']' in header '\" + line_str + \"'\", m_files[line.file_index], line.line_no);\n  }\n\n  // Stripping square brackets\n  string header = line_str.substr(1, line_str.size() - 2);\n\n  if (!is_valid_name(header)) {\n    throw invalid_name_error(\"Section\", header, m_files[line.file_index], line.line_no);\n  }\n\n  if (m_reserved_section_names.find(header) != m_reserved_section_names.end()) {\n    throw syntax_error(\n        \"'\" + header + \"' is reserved and cannot be used as a section name\", m_files[line.file_index], line.line_no);\n  }\n\n  return header;\n}\n\nstd::pair<string, string> config_parser::parse_key(const line_t& line, const string& line_str) {\n  size_t pos = line_str.find_first_of('=');\n\n  string key = string_util::trim(line_str.substr(0, pos), isspace);\n  string value = string_util::trim(line_str.substr(pos + 1), isspace);\n\n  if (!is_valid_name(key)) {\n    throw invalid_name_error(\"Key\", key, m_files[line.file_index], line.line_no);\n  }\n\n  value = parse_escaped_value(line, move(value), key);\n\n  /*\n   * Only if the string is surrounded with double quotes, do we treat them\n   * not as part of the value and remove them.\n   */\n  if (value.size() >= 2 && value.front() == '\"' && value.back() == '\"') {\n    value = value.substr(1, value.size() - 2);\n  }\n\n  // TODO check value for references\n\n#if WITH_XRM\n  // Use xrm, if at least one value is an xrdb reference\n  if (!use_xrm && value.find(\"${xrdb\") == 0) {\n    use_xrm = true;\n  }\n#endif\n\n  return {move(key), move(value)};\n}\n\nbool config_parser::is_valid_name(const string& name) {\n  if (name.empty()) {\n    return false;\n  }\n\n  for (const char c : name) {\n    // Names with forbidden chars or spaces are not valid\n    if (isspace(c) || m_forbidden_chars.find_first_of(c) != string::npos) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nstring config_parser::parse_escaped_value(const line_t& line, string&& value, const string& key) {\n  string cfg_value = value;\n  bool log = false;\n  auto backslash_pos = value.find('\\\\');\n  while (backslash_pos != string::npos) {\n    if (backslash_pos == value.size() - 1 || value[backslash_pos + 1] != '\\\\') {\n      log = true;\n    } else {\n      value = value.replace(backslash_pos, 2, \"\\\\\");\n    }\n    backslash_pos = value.find('\\\\', backslash_pos + 1);\n  }\n  if (log) {\n    m_log.err(\n        \"%s:%d: Value '%s' of key '%s' contains one or more unescaped backslashes, please prepend them with the \"\n        \"backslash \"\n        \"escape character.\",\n        m_files[line.file_index], line.line_no, cfg_value, key);\n  }\n  return move(value);\n}\nPOLYBAR_NS_END\n\n// Path: src/components/controller.cpp\n#include \"components/controller.hpp\"\n\n#include <csignal>\n#include <cassert>\n#include <utility>\n\n#include \"components/bar.hpp\"\n#include \"components/builder.hpp\"\n#include \"components/config.hpp\"\n#include \"components/eventloop.hpp\"\n#include \"components/logger.hpp\"\n#include \"components/types.hpp\"\n#include \"events/signal.hpp\"\n#include \"events/signal_emitter.hpp\"\n#include \"modules/meta/all.hpp\"\n#include \"modules/meta/base.hpp\"\n#include \"modules/meta/event_handler.hpp\"\n#include \"modules/meta/factory.hpp\"\n#include \"utils/actions.hpp\"\n#include \"utils/inotify.hpp\"\n#include \"utils/process.hpp\"\n#include \"utils/string.hpp\"\n#include \"utils/time.hpp\"\n#include \"x11/connection.hpp\"\n#include \"x11/extensions/all.hpp\"\n\nPOLYBAR_NS\n\nusing namespace eventloop;\nusing namespace modules;\n\n/**\n * Build controller instance\n */\ncontroller::make_type controller::make(bool has_ipc, loop& loop, const config& config) {\n  return std::make_unique<controller>(\n      connection::make(), signal_emitter::make(), logger::make(), config, has_ipc, loop);\n}\n\n/**\n * Construct controller\n */\ncontroller::controller(\n    connection& conn, signal_emitter& emitter, const logger& logger, const config& config, bool has_ipc, loop& loop)\n    : m_connection(conn)\n    , m_sig(emitter)\n    , m_log(logger)\n    , m_conf(config)\n    , m_loop(loop)\n    , m_bar(bar::make(m_loop, config))\n    , m_has_ipc(has_ipc) {\n  m_conf.warn_deprecated(\"settings\", \"throttle-input-for\");\n  m_conf.warn_deprecated(\"settings\", \"throttle-output\");\n  m_conf.warn_deprecated(\"settings\", \"throttle-output-for\");\n  m_conf.warn_deprecated(\"settings\", \"eventqueue-swallow\");\n  m_conf.warn_deprecated(\"settings\", \"eventqueue-swallow-time\");\n\n  m_log.trace(\"controller: Setup user-defined modules\");\n  size_t created_modules{0};\n  created_modules += setup_modules(alignment::LEFT);\n  created_modules += setup_modules(alignment::CENTER);\n  created_modules += setup_modules(alignment::RIGHT);\n\n  if (!created_modules) {\n    throw application_error(\"No modules created\");\n  }\n\n  m_log.notice(\"Loaded %zd modules\", created_modules);\n}\n\n/**\n * Deconstruct controller\n */\ncontroller::~controller() {\n  m_log.trace(\"controller: Detach signal receiver\");\n  m_sig.detach(this);\n\n  m_log.trace(\"controller: Stop modules\");\n  for (auto&& module : m_modules) {\n    auto module_name = module->name();\n    auto cleanup_ms = time_util::measure([&module] {\n      module->stop();\n      module->join();\n      module.reset();\n    });\n    m_log.info(\"Deconstruction of %s took %lu ms.\", module_name, cleanup_ms);\n  }\n}\n\n/**\n * Run the main loop\n */\nbool controller::run(bool writeback, string snapshot_dst, bool confwatch) {\n  m_log.info(\"Starting application\");\n  m_log.trace(\"controller: Main thread id = %i\", concurrency_util::thread_id(this_thread::get_id()));\n\n  assert(!m_connection.connection_has_error());\n\n  m_writeback = writeback;\n  m_snapshot_dst = move(snapshot_dst);\n\n  m_sig.attach(this);\n\n  m_connection.flush();\n\n  read_events(confwatch);\n\n  m_log.notice(\"Termination signal received, shutting down...\");\n\n  return !m_reload;\n}\n\n/**\n * Enqueue input data\n */\nvoid controller::trigger_action(string&& input_data) {\n  std::unique_lock<std::mutex> guard(m_notification_mutex);\n  m_log.trace(\"controller: Queueing input event '%s'\", input_data);\n  m_notifications.inputdata.push(std::move(input_data));\n  trigger_notification();\n}\n\nvoid controller::trigger_quit(bool reload) {\n  std::unique_lock<std::mutex> guard(m_notification_mutex);\n  m_notifications.quit = true;\n  m_notifications.reload = m_notifications.reload || reload;\n  trigger_notification();\n}\n\nvoid controller::trigger_update(bool force) {\n  std::unique_lock<std::mutex> guard(m_notification_mutex);\n  m_notifications.update = true;\n  m_notifications.force_update = m_notifications.force_update || force;\n\n  trigger_notification();\n}\n\nvoid controller::trigger_notification() {\n  m_notifier->send();\n}\n\nvoid controller::stop(bool reload) {\n  update_reload(reload);\n  m_loop.stop();\n}\n\nvoid controller::conn_cb() {\n  int xcb_error = m_connection.connection_has_error();\n  if ((xcb_error = m_connection.connection_has_error()) != 0) {\n    m_log.err(\"X connection error, terminating... (what: %s)\", m_connection.error_str(xcb_error));\n    stop(false);\n    return;\n  }\n\n  shared_ptr<xcb_generic_event_t> evt{};\n  while ((evt = m_connection.poll_for_event()) != nullptr) {\n    try {\n      m_connection.dispatch_event(evt);\n    } catch (xpp::connection_error& err) {\n      m_log.err(\"X connection error, terminating... (what: %s)\", m_connection.error_str(err.code()));\n      stop(false);\n    } catch (const exception& err) {\n      // IDs for events are defined in xproto.h\n      m_log.err(\"Error in X event loop while handling event %d: %s\", evt->response_type, err.what());\n    }\n  }\n\n  if ((xcb_error = m_connection.connection_has_error()) != 0) {\n    m_log.err(\"X connection error, terminating... (what: %s)\", m_connection.error_str(xcb_error));\n    stop(false);\n    return;\n  }\n}\n\nvoid controller::signal_handler(int signum) {\n  m_log.notice(\"Received signal(%d): %s\", signum, strsignal(signum));\n  stop(signum == SIGUSR1);\n}\n\nvoid controller::create_config_watcher(const string& filename) {\n  auto fs_event_handle = m_loop.handle<FSEventHandle>();\n  fs_event_handle->start(\n      filename, 0, [this](const auto& e) { confwatch_handler(e.path); },\n      [this, &handle = *fs_event_handle](const auto& e) {\n        m_log.err(\"libuv error while watching included file for changes: %s\", uv_strerror(e.status));\n        handle.close();\n      });\n}\n\nvoid controller::confwatch_handler(const char* filename) {\n  m_log.notice(\"Watched config file changed %s\", filename);\n  stop(true);\n}\n\nvoid controller::notifier_handler() {\n  notifications_t data{};\n\n  {\n    std::unique_lock<std::mutex> guard(m_notification_mutex);\n    std::swap(m_notifications, data);\n  }\n\n  if (data.quit) {\n    stop(data.reload);\n    return;\n  }\n\n  while (!data.inputdata.empty()) {\n    auto inputdata = data.inputdata.front();\n    data.inputdata.pop();\n    m_log.trace(\"controller: Dequeueing inputdata: '%s'\", inputdata);\n    process_inputdata(std::move(inputdata));\n  }\n\n  if (data.update) {\n    process_update(data.force_update);\n  }\n}\n\nvoid controller::screenshot_handler() {\n  m_sig.emit(signals::ui::request_snapshot{move(m_snapshot_dst)});\n  trigger_update(true);\n}\n\nvoid controller::start_modules() {\n  size_t started_modules{0};\n  for (const auto& module : m_modules) {\n    auto evt_handler = dynamic_cast<event_handler_interface*>(&*module);\n\n    if (evt_handler != nullptr) {\n      evt_handler->connect(m_connection);\n    }\n\n    try {\n      m_log.info(\"Starting %s\", module->name());\n      module->start();\n      started_modules++;\n    } catch (const application_error& err) {\n      m_log.err(\"Failed to start '%s' (reason: %s)\", module->name(), err.what());\n    }\n  }\n\n  if (!started_modules) {\n    throw application_error(\"No modules started\");\n  }\n}\n\n/**\n * Read events from configured file descriptors\n */\nvoid controller::read_events(bool confwatch) {\n  m_log.info(\"Entering event loop (thread-id=%lu)\", this_thread::get_id());\n\n  if (!m_writeback) {\n    m_bar->start(m_tray_module_name);\n  }\n\n  start_modules();\n\n  auto x_poll_handle = m_loop.handle<PollHandle>(m_connection.get_file_descriptor());\n  x_poll_handle->start(\n      UV_READABLE, [this](const auto&) { conn_cb(); },\n      [this](const auto& e) {\n        m_log.err(\"libuv error while polling X connection: \"s + uv_strerror(e.status));\n        stop(false);\n      });\n\n  auto x_prepare_handle = m_loop.handle<PrepareHandle>();\n  x_prepare_handle->start([this]() {\n    /*\n     * We have to also handle events in the prepare handle (which runs right\n     * before polling for IO) to process any already queued X events which\n     * wouldn't trigger the uv_poll handle.\n     */\n    conn_cb();\n    m_connection.flush();\n  });\n\n  for (auto s : {SIGINT, SIGQUIT, SIGTERM, SIGUSR1, SIGALRM}) {\n    auto signal_handle = m_loop.handle<SignalHandle>();\n    signal_handle->start(s, [this](const auto& e) { signal_handler(e.signum); });\n  }\n\n  if (confwatch) {\n    create_config_watcher(m_conf.filepath());\n    // also watch the include-files for changes\n    for (auto& module_path : m_conf.get_included_files()) {\n      create_config_watcher(module_path);\n    }\n  }\n\n  if (!m_snapshot_dst.empty()) {\n    // Trigger a single screenshot after 3 seconds\n    auto timer_handle = m_loop.handle<TimerHandle>();\n    timer_handle->start(3000, 0, [this]() { screenshot_handler(); });\n  }\n\n  /*\n   * Immediately trigger and update so that the bar displays something.\n   */\n  trigger_update(true);\n\n  try {\n    m_loop.run();\n  } catch (const exception& err) {\n    m_log.err(\"Fatal Error in eventloop: %s\", err.what());\n    stop(false);\n  }\n\n  m_log.info(\"Eventloop finished\");\n}\n\n/**\n * Tries to match the given command to a legacy action string and sends the\n * appropriate new action (and data) to the right module if possible.\n *\n * @returns true iff the given command matches a legacy action string and was\n *          successfully forwarded to a module\n */\nbool controller::try_forward_legacy_action(const string& cmd) {\n  /*\n   * Maps legacy action names to a module type and the new action name in that module.\n   *\n   * We try to match the old action name as a prefix, and everything after it will also be added to the end of the new\n   * action string (for example \"mpdseek+5\" will be redirected to \"seek.+5\" in the first mpd module).\n   *\n   * The action will be delivered to the first module of that type so that it is consistent with existing behavior.\n   * If the module does not support the action or no matching module is found, the command is forwarded to the shell.\n   *\n   * TODO Remove when deprecated action names are removed\n   */\n// clang-format off\n#define A_MAP(old, module_name, event) {old, {string(module_name::TYPE), string(module_name::event)}}\n\n  static const std::unordered_map<string, std::pair<string, const string>> legacy_actions{\n    A_MAP(\"datetoggle\", date_module, EVENT_TOGGLE),\n#if ENABLE_ALSA\n    A_MAP(\"volup\", alsa_module, EVENT_INC),\n    A_MAP(\"voldown\", alsa_module, EVENT_DEC),\n    A_MAP(\"volmute\", alsa_module, EVENT_TOGGLE),\n#endif\n#if ENABLE_PULSEAUDIO\n    A_MAP(\"pa_volup\", pulseaudio_module, EVENT_INC),\n    A_MAP(\"pa_voldown\", pulseaudio_module, EVENT_DEC),\n    A_MAP(\"pa_volmute\", pulseaudio_module, EVENT_TOGGLE),\n#endif\n    A_MAP(\"xbacklight+\", xbacklight_module, EVENT_INC),\n    A_MAP(\"xbacklight-\", xbacklight_module, EVENT_DEC),\n    A_MAP(\"backlight+\", backlight_module, EVENT_INC),\n    A_MAP(\"backlight-\", backlight_module, EVENT_DEC),\n#if ENABLE_XKEYBOARD\n    A_MAP(\"xkeyboard/switch\", xkeyboard_module, EVENT_SWITCH),\n#endif\n#if ENABLE_MPD\n    A_MAP(\"mpdplay\", mpd_module, EVENT_PLAY),\n    A_MAP(\"mpdpause\", mpd_module, EVENT_PAUSE),\n    A_MAP(\"mpdstop\", mpd_module, EVENT_STOP),\n...\n", "needle_token_start": 5590, "needle_token_end": 5637, "code_context_ntokens": 16040, "output": ["```cpp\nstatic void close_walk_cb(uv_handle_t* handle, void*) {\n    if (!uv_is_closing(handle)) {\n      close_handle(handle);\n    }\n  }\n```"]}
{"repo": "polybar/polybar", "name": "g_mpd_signal_handler", "language": "cpp", "path": "src/adapters/mpd.cpp", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle specific signals received by the process, particularly to update a global state indicating a change in the connection status.\n2. **Input**: It takes an integer representing the signal number.\n3. **Output**: There is no return value; the function directly modifies a global atomic variable.\n4. **Procedure**: The function checks if the received signal is a \"broken pipe\" signal. If it is, it sets a global atomic variable to 1, indicating that the connection has been closed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/adapters/net_iw.cpp\n#include \"adapters/net.hpp\"\n\n#include \"utils/file.hpp\"\n\nPOLYBAR_NS\n\nnamespace net {\n  // class : wireless_network {{{\n\n  /**\n   * Query the wireless device for information\n   * about the current connection\n   */\n  bool wireless_network::query(bool accumulate) {\n    if (!network::query(accumulate)) {\n      return false;\n    }\n\n    auto socket_fd = file_util::make_file_descriptor(iw_sockets_open());\n    if (!*socket_fd) {\n      return false;\n    }\n\n    struct iwreq req {};\n\n    if (iw_get_ext(*socket_fd, m_interface.c_str(), SIOCGIWMODE, &req) == -1) {\n      return false;\n    }\n\n    // Ignore interfaces in ad-hoc mode\n    if (req.u.mode == IW_MODE_ADHOC) {\n      return false;\n    }\n\n    query_essid(*socket_fd);\n...\n// Path: src/adapters/net.cpp\n#include \"adapters/net.hpp\"\n\n#include <arpa/inet.h>\n#include <dirent.h>\n#include <linux/ethtool.h>\n#include <linux/if_link.h>\n#include <linux/sockios.h>\n#include <net/if.h>\n#include <netdb.h>\n#include <netinet/in.h>\n#include <sys/ioctl.h>\n#include <sys/socket.h>\n#include <sys/types.h>\n\n#include <cassert>\n#include <iomanip>\n\n#include \"common.hpp\"\n#include \"settings.hpp\"\n#include \"utils/command.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace net {\n  enum class NetType {\n    WIRELESS,\n    ETHERNET,\n    OTHER,\n  };\n\n  static const string NO_IP = string(\"N/A\");\n  static const string NO_MAC = string(\"N/A\");\n  static const string NET_PATH = \"/sys/class/net/\";\n  static const string VIRTUAL_PATH = \"/sys/devices/virtual/\";\n\n  static bool is_virtual(const std::string& ifname) {\n    char* target = realpath((NET_PATH + ifname).c_str(), nullptr);\n\n    if (!target) {\n      throw system_error(\"realpath\");\n    }\n\n    const std::string real_path{target};\n    free(target);\n    return real_path.rfind(VIRTUAL_PATH, 0) == 0;\n  }\n\n  NetType iface_type(const std::string& ifname) {\n    if (file_util::exists(NET_PATH + ifname + \"/wireless\")) {\n      return NetType::WIRELESS;\n    }\n\n    if (is_virtual(ifname)) {\n      return NetType::OTHER;\n    }\n\n    return NetType::ETHERNET;\n  }\n\n  bool is_interface_valid(const string& ifname) {\n    return if_nametoindex(ifname.c_str()) != 0;\n  }\n\n  /**\n   * Returns the canonical name of the given interface and whether that differs\n   * from the given name.\n   *\n   * The canonical name is the name that has an entry in `/sys/class/net`.\n   *\n   * This resolves any altnames that were defined (see man ip-link).\n   */\n  std::pair<string, bool> get_canonical_interface(const string& ifname) {\n    int idx = if_nametoindex(ifname.c_str());\n\n    if (idx == 0) {\n      throw system_error(\"if_nameindex(\" + ifname + \")\");\n    }\n\n    char canonical[IF_NAMESIZE];\n    if (!if_indextoname(idx, canonical)) {\n      throw system_error(\"if_indextoname(\" + to_string(idx) + \")\");\n    }\n\n    string str{canonical};\n\n    return {str, str != ifname};\n  }\n\n  /**\n   * Test if interface with given name is a wireless device\n   */\n  bool is_wireless_interface(const string& ifname) {\n    return iface_type(ifname) == NetType::WIRELESS;\n  }\n\n  std::string find_interface(NetType type) {\n    struct ifaddrs* ifaddrs;\n    getifaddrs(&ifaddrs);\n\n    struct ifaddrs* candidate_if = nullptr;\n\n    for (struct ifaddrs* i = ifaddrs; i != nullptr; i = i->ifa_next) {\n      const std::string name{i->ifa_name};\n      const NetType iftype = iface_type(name);\n      if (iftype != type) {\n        continue;\n      }\n      if (candidate_if == nullptr) {\n        candidate_if = i;\n      } else if (((candidate_if->ifa_flags & IFF_RUNNING) == 0) && ((i->ifa_flags & IFF_RUNNING) > 0)) {\n        candidate_if = i;\n      }\n    }\n    if (candidate_if) {\n      const std::string name{candidate_if->ifa_name};\n      freeifaddrs(ifaddrs);\n      return name;\n    }\n    freeifaddrs(ifaddrs);\n    return \"\";\n  }\n\n  std::string find_wireless_interface() {\n    return find_interface(NetType::WIRELESS);\n  }\n\n  std::string find_wired_interface() {\n    return find_interface(NetType::ETHERNET);\n  }\n\n  // class : network {{{\n\n  /**\n   * Construct network interface\n   */\n  network::network(string interface) : m_log(logger::make()), m_interface(move(interface)) {\n    assert(is_interface_valid(m_interface));\n\n    m_socketfd = file_util::make_file_descriptor(socket(AF_INET, SOCK_DGRAM, 0));\n    if (!*m_socketfd) {\n      throw network_error(\"Failed to open socket\");\n    }\n\n    check_tuntap_or_bridge();\n  }\n\n  /**\n   * Query device driver for information\n   */\n  bool network::query(bool accumulate) {\n    m_status.previous = m_status.current;\n    m_status.current.transmitted = 0;\n    m_status.current.received = 0;\n    m_status.current.time = std::chrono::steady_clock::now();\n    m_status.ip = NO_IP;\n    m_status.ip6 = NO_IP;\n\n    struct ifaddrs* ifaddr;\n    if (getifaddrs(&ifaddr) == -1 || ifaddr == nullptr) {\n      return false;\n    }\n\n    m_status.mac = string_util::trim(file_util::contents(NET_PATH + m_interface + \"/address\"), isspace);\n    if (m_status.mac == \"\") {\n      m_status.mac = NO_MAC;\n    }\n\n    for (auto ifa = ifaddr; ifa != nullptr; ifa = ifa->ifa_next) {\n      if (ifa->ifa_addr == nullptr) {\n        continue;\n      }\n\n      if (m_interface.compare(0, m_interface.length(), ifa->ifa_name) != 0) {\n        if (!accumulate || (ifa->ifa_data == nullptr && ifa->ifa_addr->sa_family != AF_PACKET)) {\n          continue;\n        }\n      }\n\n      struct sockaddr_in6* sa6;\n\n      switch (ifa->ifa_addr->sa_family) {\n        case AF_INET:\n          char ip_buffer[NI_MAXHOST];\n          getnameinfo(ifa->ifa_addr, sizeof(sockaddr_in), ip_buffer, NI_MAXHOST, nullptr, 0, NI_NUMERICHOST);\n          m_status.ip = string{ip_buffer};\n          break;\n\n        case AF_INET6:\n          char ip6_buffer[INET6_ADDRSTRLEN];\n          sa6 = reinterpret_cast<decltype(sa6)>(ifa->ifa_addr);\n          if (IN6_IS_ADDR_LINKLOCAL(&sa6->sin6_addr)) {\n            continue;\n          }\n          if (IN6_IS_ADDR_SITELOCAL(&sa6->sin6_addr)) {\n            continue;\n          }\n          if ((((unsigned char*)sa6->sin6_addr.s6_addr)[0] & 0xFE) == 0xFC) {\n            /* Skip Unique Local Addresses (fc00::/7) */\n            continue;\n          }\n          if (inet_ntop(AF_INET6, &sa6->sin6_addr, ip6_buffer, INET6_ADDRSTRLEN) == 0) {\n            m_log.warn(\"inet_ntop() \" + string(strerror(errno)));\n            continue;\n          }\n          m_status.ip6 = string{ip6_buffer};\n          break;\n\n        case AF_PACKET:\n          if (ifa->ifa_data == nullptr) {\n            continue;\n          }\n          struct rtnl_link_stats* link_state = reinterpret_cast<decltype(link_state)>(ifa->ifa_data);\n          if (link_state == nullptr) {\n            continue;\n          }\n          m_status.current.transmitted += link_state->tx_bytes;\n          m_status.current.received += link_state->rx_bytes;\n          break;\n      }\n    }\n\n    freeifaddrs(ifaddr);\n\n    return true;\n  }\n\n  /**\n   * Run ping command to test internet connectivity\n   */\n  bool network::ping() const {\n    try {\n      auto exec = \"ping -c 2 -W 2 -I \" + m_interface + \" \" + string(CONNECTION_TEST_IP);\n      command<output_policy::IGNORED> ping(m_log, exec);\n      return ping.exec(true) == EXIT_SUCCESS;\n    } catch (const std::exception& err) {\n      return false;\n    }\n  }\n\n  /**\n   * Get interface ipv4 address\n   */\n  string network::ip() const {\n    return m_status.ip;\n  }\n  /**\n   * Get interface mac address\n   */\n  string network::mac() const {\n    return m_status.mac;\n  }\n\n  /**\n   * Get interface ipv6 address\n   */\n  string network::ip6() const {\n    return m_status.ip6;\n  }\n\n  /**\n   * Get download speed rate\n   */\n  string network::downspeed(int minwidth, const string& unit) const {\n    float bytes_diff = m_status.current.received - m_status.previous.received;\n    return format_speedrate(bytes_diff, minwidth, unit);\n  }\n\n  /**\n   * Get upload speed rate\n   */\n  string network::upspeed(int minwidth, const string& unit) const {\n    float bytes_diff = m_status.current.transmitted - m_status.previous.transmitted;\n    return format_speedrate(bytes_diff, minwidth, unit);\n  }\n\n  /**\n   * Get total net speed rate\n   */\n  string network::netspeed(int minwidth, const string& unit) const {\n    float bytes_diff = m_status.current.received - m_status.previous.received + m_status.current.transmitted -\n                       m_status.previous.transmitted;\n    return format_speedrate(bytes_diff, minwidth, unit);\n  }\n\n  /**\n   * Set if unknown counts as up\n   */\n  void network::set_unknown_up(bool unknown) {\n    m_unknown_up = unknown;\n  }\n\n  /**\n   * Query driver info to check if the\n   * interface is a TUN/TAP device or BRIDGE\n   */\n  void network::check_tuntap_or_bridge() {\n    struct ethtool_drvinfo driver {};\n    struct ifreq request {};\n\n    driver.cmd = ETHTOOL_GDRVINFO;\n\n    memset(&request, 0, sizeof(request));\n\n    /*\n     * Only copy array size minus one bytes over to ensure there is a\n     * terminating NUL byte (which is guaranteed by memset)\n     */\n    strncpy(request.ifr_name, m_interface.c_str(), IFNAMSIZ - 1);\n\n    request.ifr_data = reinterpret_cast<char*>(&driver);\n\n    if (ioctl(*m_socketfd, SIOCETHTOOL, &request) == -1) {\n      return;\n    }\n\n    // Check if it's a TUN/TAP device\n    if (strncmp(driver.bus_info, \"tun\", 3) == 0) {\n      m_tuntap = true;\n    } else if (strncmp(driver.bus_info, \"tap\", 3) == 0) {\n      m_tuntap = true;\n    } else {\n      m_tuntap = false;\n    }\n\n    if (strncmp(driver.driver, \"bridge\", 6) == 0) {\n      m_bridge = true;\n    }\n  }\n\n  /**\n   * Test if the network interface is in a valid state\n   */\n  bool network::test_interface() const {\n    auto operstate = file_util::contents(NET_PATH + m_interface + \"/operstate\");\n    bool up = operstate.compare(0, 2, \"up\") == 0;\n    return m_unknown_up ? (up || operstate.compare(0, 7, \"unknown\") == 0) : up;\n  }\n\n  /**\n   * Format up- and download speed\n   */\n  string network::format_speedrate(float bytes_diff, int minwidth, const string& unit) const {\n    // Get time difference in seconds as a float\n    const std::chrono::duration<float> duration = m_status.current.time - m_status.previous.time;\n    float time_diff = duration.count();\n    float speedrate = bytes_diff / time_diff;\n\n    vector<pair<string, int>> units{make_pair(\"G\", 2), make_pair(\"M\", 1)};\n    string suffix{\"K\"};\n    int precision = 0;\n\n    while ((speedrate /= 1000) > 999) {\n      suffix = units.back().first;\n      precision = units.back().second;\n      units.pop_back();\n    }\n\n    return sstream() << std::setw(minwidth) << std::setfill(' ') << std::setprecision(precision) << std::fixed\n                     << speedrate << \" \" << suffix << unit;\n  }\n\n  // }}}\n  // class : wired_network {{{\n\n  /**\n   * Query device driver for information\n   */\n  bool wired_network::query(bool accumulate) {\n    if (!network::query(accumulate)) {\n      return false;\n    }\n\n    if (m_tuntap) {\n      return true;\n    }\n\n    if (m_bridge) {\n      /* If bridge network then link speed cannot be computed\n       * TODO: Identify the physical network in bridge and compute the link speed\n       */\n      return true;\n    }\n\n    struct ifreq request {};\n    struct ethtool_cmd data {};\n\n    memset(&request, 0, sizeof(request));\n    strncpy(request.ifr_name, m_interface.c_str(), IFNAMSIZ - 1);\n    data.cmd = ETHTOOL_GSET;\n    request.ifr_data = reinterpret_cast<char*>(&data);\n\n    if (ioctl(*m_socketfd, SIOCETHTOOL, &request) == -1) {\n      m_linkspeed = -1;\n    } else {\n      m_linkspeed = data.speed;\n    }\n\n    return true;\n  }\n\n  /**\n   * Check current connection state\n   */\n  bool wired_network::connected() const {\n    if (!m_tuntap && !network::test_interface()) {\n      return false;\n    }\n\n    struct ethtool_value data {};\n    struct ifreq request {};\n\n    memset(&request, 0, sizeof(request));\n    strncpy(request.ifr_name, m_interface.c_str(), IFNAMSIZ - 1);\n    data.cmd = ETHTOOL_GLINK;\n    request.ifr_data = reinterpret_cast<char*>(&data);\n\n    if (ioctl(*m_socketfd, SIOCETHTOOL, &request) == -1) {\n      return false;\n    }\n\n    return data.data != 0;\n  }\n\n  /**\n   *\n   * about the current connection\n   */\n  string wired_network::linkspeed() const {\n    return m_linkspeed == -1 ? \"N/A\"\n                             : (m_linkspeed < 1000 ? (to_string(m_linkspeed) + \" Mbit/s\")\n                                                   : (to_string(m_linkspeed / 1000) + \" Gbit/s\"));\n  }\n\n  // }}}\n\n} // namespace net\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/net_nl.cpp\n#include <linux/nl80211.h>\n#include <netlink/genl/ctrl.h>\n#include <netlink/genl/genl.h>\n\n#include <algorithm>\n\n#include \"adapters/net.hpp\"\n#include \"utils/file.hpp\"\n\nPOLYBAR_NS\n\nnamespace net {\n  // class : wireless_network {{{\n\n  /**\n   * Query the wireless device for information\n   * about the current connection\n   */\n  bool wireless_network::query(bool accumulate) {\n    if (!network::query(accumulate)) {\n      return false;\n    }\n\n    struct nl_sock* sk = nl_socket_alloc();\n    if (sk == nullptr) {\n      return false;\n    }\n\n    if (genl_connect(sk) < 0) {\n      return false;\n    }\n\n    int driver_id = genl_ctrl_resolve(sk, \"nl80211\");\n    if (driver_id < 0) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    if (nl_socket_modify_cb(sk, NL_CB_VALID, NL_CB_CUSTOM, scan_cb, this) != 0) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    struct nl_msg* msg = nlmsg_alloc();\n    if (msg == nullptr) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    if ((genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, driver_id, 0, NLM_F_DUMP, NL80211_CMD_GET_SCAN, 0) == nullptr) ||\n        nla_put_u32(msg, NL80211_ATTR_IFINDEX, m_ifid) < 0) {\n      nlmsg_free(msg);\n      nl_socket_free(sk);\n      return false;\n    }\n\n    // nl_send_sync always frees msg\n    if (nl_send_sync(sk, msg) < 0) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    nl_socket_free(sk);\n\n    return true;\n  }\n\n  /**\n   * Check current connection state\n   */\n  bool wireless_network::connected() const {\n    if (!network::test_interface()) {\n      return false;\n    }\n    return !m_essid.empty();\n  }\n\n  /**\n   * ESSID reported by last query\n   */\n  string wireless_network::essid() const {\n    return m_essid;\n  }\n\n  /**\n   * Signal strength percentage reported by last query\n   */\n  int wireless_network::signal() const {\n    return m_signalstrength.percentage();\n  }\n\n  /**\n   * Link quality percentage reported by last query\n   */\n  int wireless_network::quality() const {\n    return m_linkquality.percentage();\n  }\n\n  /**\n   * Callback to parse scan results\n   */\n  int wireless_network::scan_cb(struct nl_msg* msg, void* instance) {\n    auto wn = static_cast<wireless_network*>(instance);\n    auto gnlh = static_cast<genlmsghdr*>(nlmsg_data(nlmsg_hdr(msg)));\n    struct nlattr* tb[NL80211_ATTR_MAX + 1];\n    struct nlattr* bss[NL80211_BSS_MAX + 1];\n\n    struct nla_policy bss_policy[NL80211_BSS_MAX + 1]{};\n    bss_policy[NL80211_BSS_TSF].type = NLA_U64;\n    bss_policy[NL80211_BSS_FREQUENCY].type = NLA_U32;\n    bss_policy[NL80211_BSS_BSSID].type = NLA_UNSPEC;\n    bss_policy[NL80211_BSS_BEACON_INTERVAL].type = NLA_U16;\n    bss_policy[NL80211_BSS_CAPABILITY].type = NLA_U16;\n    bss_policy[NL80211_BSS_INFORMATION_ELEMENTS].type = NLA_UNSPEC;\n    bss_policy[NL80211_BSS_SIGNAL_MBM].type = NLA_U32;\n    bss_policy[NL80211_BSS_SIGNAL_UNSPEC].type = NLA_U8;\n    bss_policy[NL80211_BSS_STATUS].type = NLA_U32;\n\n    if (nla_parse(tb, NL80211_ATTR_MAX, genlmsg_attrdata(gnlh, 0), genlmsg_attrlen(gnlh, 0), nullptr) < 0) {\n      return NL_SKIP;\n    }\n\n    if (tb[NL80211_ATTR_BSS] == nullptr) {\n      return NL_SKIP;\n    }\n\n    if (nla_parse_nested(bss, NL80211_BSS_MAX, tb[NL80211_ATTR_BSS], bss_policy) != 0) {\n      return NL_SKIP;\n    }\n\n    if (!wn->associated_or_joined(bss)) {\n      return NL_SKIP;\n    }\n\n    wn->parse_essid(bss);\n    wn->parse_frequency(bss);\n    wn->parse_signal(bss);\n    wn->parse_quality(bss);\n\n    return NL_SKIP;\n  }\n\n  /**\n   * Check for a connection to a AP\n   */\n  bool wireless_network::associated_or_joined(struct nlattr** bss) {\n    if (bss[NL80211_BSS_STATUS] == nullptr) {\n      return false;\n    }\n\n    auto status = nla_get_u32(bss[NL80211_BSS_STATUS]);\n\n    switch (status) {\n      case NL80211_BSS_STATUS_ASSOCIATED:\n      case NL80211_BSS_STATUS_IBSS_JOINED:\n      case NL80211_BSS_STATUS_AUTHENTICATED:\n        return true;\n      default:\n        return false;\n    }\n  }\n\n  /**\n   * Set the ESSID\n   */\n  void wireless_network::parse_essid(struct nlattr** bss) {\n    m_essid.clear();\n\n    if (bss[NL80211_BSS_INFORMATION_ELEMENTS] != nullptr) {\n// Information Element ID from ieee80211.h\n#define WLAN_EID_SSID 0\n\n      auto ies = static_cast<char*>(nla_data(bss[NL80211_BSS_INFORMATION_ELEMENTS]));\n      auto ies_len = nla_len(bss[NL80211_BSS_INFORMATION_ELEMENTS]);\n      const auto hdr_len = 2;\n\n      while (ies_len > hdr_len && ies[0] != WLAN_EID_SSID) {\n        ies_len -= ies[1] + hdr_len;\n        ies += ies[1] + hdr_len;\n      }\n\n      if (ies_len > hdr_len && ies_len > ies[1] + hdr_len) {\n        auto essid_begin = ies + hdr_len;\n        auto essid_end = essid_begin + ies[1];\n\n        std::copy(essid_begin, essid_end, std::back_inserter(m_essid));\n      }\n    }\n  }\n\n  /**\n   * Set frequency\n   */\n  void wireless_network::parse_frequency(struct nlattr** bss) {\n    if (bss[NL80211_BSS_FREQUENCY] != nullptr) {\n      // in MHz\n      m_frequency = static_cast<int>(nla_get_u32(bss[NL80211_BSS_FREQUENCY]));\n    }\n  }\n\n  /**\n   * Set device driver quality values\n   */\n  void wireless_network::parse_quality(struct nlattr** bss) {\n    if (bss[NL80211_BSS_SIGNAL_UNSPEC] != nullptr) {\n      // Signal strength in unspecified units, scaled to 0..100 (u8)\n      m_linkquality.val = nla_get_u8(bss[NL80211_BSS_SIGNAL_UNSPEC]);\n      m_linkquality.max = 100;\n    }\n  }\n\n  /**\n   * Set the signalstrength\n   */\n  void wireless_network::parse_signal(struct nlattr** bss) {\n    if (bss[NL80211_BSS_SIGNAL_MBM] != nullptr) {\n      // signalstrength in dBm\n      int signalstrength = static_cast<int>(nla_get_u32(bss[NL80211_BSS_SIGNAL_MBM])) / 100;\n\n      // WiFi-hardware usually operates in the range -90 to -20dBm.\n      const int hardware_max = -20;\n      const int hardware_min = -90;\n      signalstrength = std::max(hardware_min, std::min(signalstrength, hardware_max));\n\n      // Shift for positive values\n      m_signalstrength.val = signalstrength - hardware_min;\n      m_signalstrength.max = hardware_max - hardware_min;\n    }\n  }\n} // namespace net\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/mpd.cpp\n#include <cassert>\n#include <csignal>\n#include <thread>\n#include <utility>\n\n#include \"adapters/mpd.hpp\"\n#include \"components/logger.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\n#define TRACE_BOOL(mode) m_log.trace(\"mpdconnection.%s: %s\", __func__, mode ? \"true\" : \"false\");\n\nnamespace mpd {\n  sig_atomic_t g_connection_closed = 0;\n  \nvoid g_mpd_signal_handler(int signum) {\n    if (signum == SIGPIPE) {\n      g_connection_closed = 1;\n    }\n  }\n\n  void check_connection(mpd_connection* conn) {\n    if (g_connection_closed) {\n      g_connection_closed = 0;\n      throw server_error(\"Connection closed (broken pipe)\");\n    } else if (conn == nullptr) {\n      throw client_error(\"Not connected to server\", MPD_ERROR_STATE);\n    }\n  }\n\n  void check_errors(mpd_connection* conn) {\n    check_connection(conn);\n\n    string err_msg;\n\n    switch (mpd_connection_get_error(conn)) {\n      case MPD_ERROR_SUCCESS:\n        return;\n      case MPD_ERROR_SERVER:\n        {\n          err_msg = mpd_connection_get_error_message(conn);\n          enum mpd_server_error server_err = mpd_connection_get_server_error(conn);\n          mpd_connection_clear_error(conn);\n          throw server_error(err_msg, server_err);\n        }\n      default:\n        {\n          err_msg = mpd_connection_get_error_message(conn);\n          enum mpd_error err = mpd_connection_get_error(conn);\n          mpd_connection_clear_error(conn);\n          throw client_error(err_msg, err);\n        }\n    }\n  }\n\n  // deleters {{{\n\n  namespace details {\n    void mpd_connection_deleter::operator()(mpd_connection* conn) {\n      if (conn != nullptr) {\n        mpd_connection_free(conn);\n      }\n    }\n\n    void mpd_status_deleter::operator()(mpd_status* status) {\n      mpd_status_free(status);\n    }\n\n    void mpd_song_deleter::operator()(mpd_song* song) {\n      mpd_song_free(song);\n    }\n  }\n\n  // }}}\n  // class: mpdsong {{{\n\n  mpdsong::operator bool() {\n    return m_song != nullptr;\n  }\n\n  string mpdsong::get_artist() {\n    assert(m_song);\n    auto tag = mpd_song_get_tag(m_song.get(), MPD_TAG_ARTIST, 0);\n    return string{tag != nullptr ? tag : \"\"};\n  }\n\n  string mpdsong::get_album_artist() {\n    assert(m_song);\n    auto tag = mpd_song_get_tag(m_song.get(), MPD_TAG_ALBUM_ARTIST, 0);\n    return string{tag != nullptr ? tag : \"\"};\n}\n\n  string mpdsong::get_album() {\n    assert(m_song);\n    auto tag = mpd_song_get_tag(m_song.get(), MPD_TAG_ALBUM, 0);\n    return string{tag != nullptr ? tag : \"\"};\n  }\n\n  string mpdsong::get_date() {\n    assert(m_song);\n    auto tag = mpd_song_get_tag(m_song.get(), MPD_TAG_DATE, 0);\n    return string{tag != nullptr ? tag : \"\"};\n  }\n\n  string mpdsong::get_title() {\n    assert(m_song);\n    auto tag = mpd_song_get_tag(m_song.get(), MPD_TAG_TITLE, 0);\n    if (tag == nullptr) {\n      tag = mpd_song_get_tag(m_song.get(), MPD_TAG_NAME, 0);\n      if (tag == nullptr) {\n        auto uri = mpd_song_get_uri(m_song.get());\n        auto name = strrchr(uri, '/');\n        tag = name ? name + 1 : uri;\n      }\n    }\n    return string{tag};\n  }\n\n  unsigned mpdsong::get_duration() {\n    assert(m_song);\n    return mpd_song_get_duration(m_song.get());\n  }\n\n  // }}}\n  // class: mpdconnection {{{\n\n  mpdconnection::mpdconnection(\n      const logger& logger, string host, unsigned int port, string password, unsigned int timeout)\n      : m_log(logger), m_host(move(host)), m_port(port), m_password(move(password)), m_timeout(timeout) {\n    memset(&m_signal_action, 0, sizeof(m_signal_action));\n    m_signal_action.sa_handler = &g_mpd_signal_handler;\n    if (sigaction(SIGPIPE, &m_signal_action, nullptr) == -1) {\n      throw mpd_exception(\"Could not setup signal handler: \"s + std::strerror(errno));\n    }\n  }\n\n  mpdconnection::~mpdconnection() {\n    m_signal_action.sa_handler = SIG_DFL;\n    sigaction(SIGPIPE, &m_signal_action, nullptr);\n  }\n\n  void mpdconnection::connect() {\n    try {\n      m_log.trace(\"mpdconnection.connect: %s, %i, \\\"%s\\\", timeout: %i\", m_host, m_port, m_password, m_timeout);\n      m_connection.reset(mpd_connection_new(m_host.c_str(), m_port, m_timeout * 1000));\n      check_errors(m_connection.get());\n\n      if (!m_password.empty()) {\n        noidle();\n        assert(!m_listactive);\n        mpd_run_password(m_connection.get(), m_password.c_str());\n        check_errors(m_connection.get());\n      }\n\n      m_fd = mpd_connection_get_fd(m_connection.get());\n      check_errors(m_connection.get());\n    } catch (const client_error& e) {\n      disconnect();\n      throw e;\n    }\n  }\n\n  void mpdconnection::disconnect() {\n    m_connection.reset();\n    m_idle = false;\n    m_listactive = false;\n  }\n\n  bool mpdconnection::connected() {\n    return static_cast<bool>(m_connection);\n  }\n\n  bool mpdconnection::retry_connection(int interval) {\n    if (connected()) {\n      return true;\n    }\n\n    while (true) {\n      try {\n        connect();\n        return true;\n      } catch (const mpd_exception& e) {\n        std::this_thread::sleep_for(chrono::duration<double>(interval));\n      }\n    }\n\n    return false;\n  }\n\n  int mpdconnection::get_fd() {\n    return m_fd;\n  }\n\n  void mpdconnection::idle() {\n    check_connection(m_connection.get());\n    if (!m_idle) {\n      mpd_send_idle(m_connection.get());\n      check_errors(m_connection.get());\n      m_idle = true;\n    }\n  }\n\n  int mpdconnection::noidle() {\n    check_connection(m_connection.get());\n    int flags = 0;\n    if (m_idle && mpd_send_noidle(m_connection.get())) {\n      m_idle = false;\n      flags = mpd_recv_idle(m_connection.get(), true);\n      mpd_response_finish(m_connection.get());\n      check_errors(m_connection.get());\n    }\n    return flags;\n  }\n\n  unique_ptr<mpdstatus> mpdconnection::get_status() {\n    check_prerequisites();\n    auto status = make_unique<mpdstatus>(this);\n    check_errors(m_connection.get());\n    return status;\n  }\n\n  unique_ptr<mpdstatus> mpdconnection::get_status_safe() {\n    try {\n      return get_status();\n    } catch (const mpd_exception& e) {\n      return {};\n    }\n  }\n\n  unique_ptr<mpdsong> mpdconnection::get_song() {\n    check_prerequisites_commands_list();\n    mpd_send_current_song(m_connection.get());\n    mpd_song_t song{mpd_recv_song(m_connection.get()), mpd_song_t::deleter_type{}};\n    mpd_response_finish(m_connection.get());\n    check_errors(m_connection.get());\n    if (song != nullptr) {\n      return make_unique<mpdsong>(move(song));\n    }\n    return unique_ptr<mpdsong>{};\n  }\n\n  void mpdconnection::play() {\n    try {\n      check_prerequisites_commands_list();\n      mpd_run_play(m_connection.get());\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.play: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::pause(bool state) {\n    try {\n      TRACE_BOOL(state);\n      check_prerequisites_commands_list();\n      mpd_run_pause(m_connection.get(), state);\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.pause: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::toggle() {\n    try {\n      check_prerequisites_commands_list();\n      mpd_run_toggle_pause(m_connection.get());\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.toggle: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::stop() {\n    try {\n      check_prerequisites_commands_list();\n      mpd_run_stop(m_connection.get());\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.stop: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::prev() {\n    try {\n      check_prerequisites_commands_list();\n      mpd_run_previous(m_connection.get());\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.prev: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::next() {\n    try {\n      check_prerequisites_commands_list();\n      mpd_run_next(m_connection.get());\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.next: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::seek(int songid, int pos) {\n    try {\n      check_prerequisites_commands_list();\n      mpd_run_seek_id(m_connection.get(), songid, pos);\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.seek: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::set_repeat(bool mode) {\n    try {\n      TRACE_BOOL(mode);\n      check_prerequisites_commands_list();\n      mpd_run_repeat(m_connection.get(), mode);\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.set_repeat: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::set_random(bool mode) {\n    try {\n      TRACE_BOOL(mode);\n      check_prerequisites_commands_list();\n      mpd_run_random(m_connection.get(), mode);\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.set_random: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::set_single(bool mode) {\n    try {\n      TRACE_BOOL(mode);\n      check_prerequisites_commands_list();\n      mpd_run_single(m_connection.get(), mode);\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.set_single: %s\", e.what());\n    }\n  }\n\n  void mpdconnection::set_consume(bool mode) {\n    try {\n      TRACE_BOOL(mode);\n      check_prerequisites_commands_list();\n      mpd_run_consume(m_connection.get(), mode);\n      check_errors(m_connection.get());\n    } catch (const mpd_exception& e) {\n      m_log.err(\"mpdconnection.set_consume: %s\", e.what());\n    }\n  }\n\n  mpdconnection::operator mpd_connection_t::element_type*() {\n    return m_connection.get();\n  }\n\n  void mpdconnection::check_prerequisites() {\n    check_connection(m_connection.get());\n    noidle();\n  }\n\n  void mpdconnection::check_prerequisites_commands_list() {\n    noidle();\n    assert(!m_listactive);\n    check_prerequisites();\n  }\n\n  // }}}\n  // class: mpdstatus {{{\n\n  mpdstatus::mpdstatus(mpdconnection* conn, bool autoupdate) {\n    fetch_data(conn);\n    if (autoupdate) {\n      update(-1, conn);\n    }\n  }\n\n  void mpdstatus::fetch_data(mpdconnection* conn) {\n    m_status.reset(mpd_run_status(*conn));\n    m_songid = mpd_status_get_song_id(m_status.get());\n    m_queuelen = mpd_status_get_queue_length(m_status.get());\n    m_random = mpd_status_get_random(m_status.get());\n    m_repeat = mpd_status_get_repeat(m_status.get());\n    m_single = mpd_status_get_single(m_status.get());\n    m_consume = mpd_status_get_consume(m_status.get());\n    m_elapsed_time = mpd_status_get_elapsed_time(m_status.get());\n    m_total_time = mpd_status_get_total_time(m_status.get());\n  }\n\n  void mpdstatus::update(int event, mpdconnection* connection) {\n    /*\n     * Only update if either the player state (play, stop, pause, seek, ...), the options (random, repeat, ...),\n     * or the playlist has been changed\n     */\n    if (connection == nullptr || !static_cast<bool>(event & (MPD_IDLE_PLAYER | MPD_IDLE_OPTIONS | MPD_IDLE_QUEUE))) {\n      return;\n    }\n\n    fetch_data(connection);\n\n    m_elapsed_time_ms = m_elapsed_time * 1000;\n\n    auto state = mpd_status_get_state(m_status.get());\n\n    switch (state) {\n      case MPD_STATE_PAUSE:\n        m_state = mpdstate::PAUSED;\n        break;\n      case MPD_STATE_PLAY:\n        m_state = mpdstate::PLAYING;\n        break;\n      case MPD_STATE_STOP:\n        m_state = mpdstate::STOPPED;\n        break;\n      default:\n        m_state = mpdstate::UNKNOWN;\n    }\n  }\n\n  bool mpdstatus::random() const {\n    return m_random;\n  }\n\n  bool mpdstatus::repeat() const {\n    return m_repeat;\n  }\n\n  bool mpdstatus::single() const {\n    return m_single;\n  }\n\n  bool mpdstatus::consume() const {\n    return m_consume;\n  }\n\n  bool mpdstatus::match_state(mpdstate state) const {\n    return state == m_state;\n  }\n\n  int mpdstatus::get_songid() const {\n    return m_songid;\n  }\n\n  int mpdstatus::get_queuelen() const {\n    return m_queuelen;\n  }\n\n  unsigned mpdstatus::get_total_time() const {\n    return m_total_time;\n  }\n\n  unsigned mpdstatus::get_elapsed_time() const {\n    return m_elapsed_time;\n  }\n\n  unsigned mpdstatus::get_elapsed_percentage() {\n    if (m_total_time == 0) {\n      return 0;\n    }\n    return static_cast<int>(float(m_elapsed_time) / float(m_total_time) * 100.0 + 0.5f);\n  }\n\n  string mpdstatus::get_formatted_elapsed() {\n    char buffer[32];\n    snprintf(buffer, sizeof(buffer), \"%lu:%02lu\", m_elapsed_time / 60, m_elapsed_time % 60);\n    return {buffer};\n  }\n\n  string mpdstatus::get_formatted_total() {\n    char buffer[32];\n    snprintf(buffer, sizeof(buffer), \"%lu:%02lu\", m_total_time / 60, m_total_time % 60);\n    return {buffer};\n  }\n\n  int mpdstatus::get_seek_position(int percentage) {\n    if (m_total_time == 0) {\n      return 0;\n    }\n    math_util::cap<int>(0, 100, percentage);\n    return math_util::percentage_to_value<double>(percentage, m_total_time);\n  }\n\n  // }}}\n}\n\n#undef TRACE_BOOL\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/alsa/control.cpp\n#include \"adapters/alsa/control.hpp\"\n#include \"adapters/alsa/generic.hpp\"\n\nPOLYBAR_NS\n\nnamespace alsa {\n  /**\n   * Construct control object\n   */\n  control::control(int numid) : m_numid(numid) {\n    int err{0};\n\n    if ((err = snd_ctl_open(&m_ctl, ALSA_SOUNDCARD, SND_CTL_NONBLOCK | SND_CTL_READONLY)) == -1) {\n      throw_exception<control_error>(\"Could not open control '\" + string{ALSA_SOUNDCARD} + \"'\", err);\n    }\n\n    snd_config_update_free_global();\n\n    if ((err = snd_hctl_open_ctl(&m_hctl, m_ctl)) == -1) {\n      snd_ctl_close(m_ctl);\n      throw_exception<control_error>(\"Failed to open hctl\", err);\n    }\n\n    snd_config_update_free_global();\n\n    if ((err = snd_hctl_load(m_hctl)) == -1) {\n      throw_exception<control_error>(\"Failed to load hctl\", err);\n    }\n\n    snd_ctl_elem_id_t* m_id{nullptr};\n    snd_ctl_elem_id_alloca(&m_id);\n    snd_ctl_elem_id_set_numid(m_id, m_numid);\n\n    snd_ctl_elem_info_t* m_info{nullptr};\n    snd_ctl_elem_info_alloca(&m_info);\n    snd_ctl_elem_info_set_id(m_info, m_id);\n\n    if ((err = snd_ctl_elem_info(m_ctl, m_info)) == -1) {\n      throw_exception<control_error>(\"Could not get control data\", err);\n    }\n\n    snd_ctl_elem_info_get_id(m_info, m_id);\n\n    if ((m_elem = snd_hctl_find_elem(m_hctl, m_id)) == nullptr) {\n      throw control_error(\"Could not find control with id \" + to_string(snd_ctl_elem_id_get_numid(m_id)));\n    }\n\n    if ((err = snd_ctl_subscribe_events(m_ctl, 1)) == -1) {\n      throw control_error(\"Could not subscribe to events: \" + to_string(snd_ctl_elem_id_get_numid(m_id)));\n    }\n  }\n\n  /**\n   * Deconstruct control object\n   */\n  control::~control() {\n    if (m_hctl != nullptr) {\n      snd_hctl_close(m_hctl);\n    }\n  }\n\n  /**\n   * Get the id number\n   */\n  int control::get_numid() {\n    return m_numid;\n  }\n\n  /**\n   * Wait for events\n   */\n  bool control::wait(int timeout) {\n    assert(m_ctl);\n\n    int err{0};\n\n    if ((err = snd_ctl_wait(m_ctl, timeout)) == -1) {\n      throw_exception<control_error>(\"Failed to wait for events\", err);\n    }\n\n    snd_ctl_event_t* event{nullptr};\n    snd_ctl_event_alloca(&event);\n\n    if ((err = snd_ctl_read(m_ctl, event)) == -1) {\n      return false;\n    }\n\n    if (snd_ctl_event_get_type(event) == SND_CTL_EVENT_ELEM) {\n      return snd_ctl_event_elem_get_mask(event) & SND_CTL_EVENT_MASK_VALUE;\n    }\n\n    return false;\n  }\n\n  /**\n   * Check if the interface is in use\n   */\n  bool control::test_device_plugged() {\n    assert(m_elem);\n\n    snd_ctl_elem_value_t* m_value{nullptr};\n    snd_ctl_elem_value_alloca(&m_value);\n\n    int err{0};\n\n    if ((err = snd_hctl_elem_read(m_elem, m_value)) == -1) {\n      throw_exception<control_error>(\"Could not read control value\", err);\n    }\n\n    return snd_ctl_elem_value_get_boolean(m_value, 0);\n  }\n\n  /**\n   * Process queued events\n   */\n  void control::process_events() {\n    wait(0);\n  }\n}\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/alsa/mixer.cpp\n#include <cmath>\n\n#include \"adapters/alsa/generic.hpp\"\n#include \"adapters/alsa/mixer.hpp\"\n#include \"utils/math.hpp\"\n\n#define MAX_LINEAR_DB_SCALE 24\n\nPOLYBAR_NS\n\nnamespace alsa {\n  /**\n   * Construct mixer object\n   */\n  mixer::mixer(string&& mixer_selem_name, string&& soundcard_name)\n      : m_name(forward<string>(mixer_selem_name)), s_name(soundcard_name) {\n    int err = 0;\n\n    if ((err = snd_mixer_open(&m_mixer, 1)) == -1) {\n      throw_exception<mixer_error>(\"Failed to open hardware mixer\", err);\n    }\n\n    snd_config_update_free_global();\n\n    if ((err = snd_mixer_attach(m_mixer, s_name.c_str())) == -1) {\n      throw_exception<mixer_error>(\"Failed to attach hardware mixer control\", err);\n    }\n    if ((err = snd_mixer_selem_register(m_mixer, nullptr, nullptr)) == -1) {\n      throw_exception<mixer_error>(\"Failed to register simple mixer element\", err);\n    }\n    if ((err = snd_mixer_load(m_mixer)) == -1) {\n      throw_exception<mixer_error>(\"Failed to load mixer\", err);\n    }\n\n    snd_mixer_selem_id_t* sid{nullptr};\n    snd_mixer_selem_id_alloca(&sid);\n    snd_mixer_selem_id_set_index(sid, 0);\n    snd_mixer_selem_id_set_name(sid, m_name.c_str());\n\n    if ((m_elem = snd_mixer_find_selem(m_mixer, sid)) == nullptr) {\n      throw mixer_error(\"Cannot find simple element\");\n    }\n  }\n\n  /**\n   * Deconstruct mixer\n   */\n  mixer::~mixer() {\n    if (m_mixer != nullptr) {\n      snd_mixer_close(m_mixer);\n    }\n  }\n\n  /**\n   * Get mixer name\n   */\n  const string& mixer::get_name() {\n    return m_name;\n  }\n\n  /**\n   * Get the name of the soundcard that is associated with the mixer\n   */\n  const string& mixer::get_sound_card() {\n    return s_name;\n  }\n\n  /**\n   * Wait for events\n   */\n  bool mixer::wait(int timeout) {\n    assert(m_mixer);\n\n    int err = 0;\n\n    if ((err = snd_mixer_wait(m_mixer, timeout)) == -1) {\n      throw_exception<mixer_error>(\"Failed to wait for events\", err);\n    }\n\n    return process_events() > 0;\n  }\n\n  /**\n   * Process queued mixer events\n   */\n  int mixer::process_events() {\n    int num_events{0};\n    if ((num_events = snd_mixer_handle_events(m_mixer)) == -1) {\n      throw_exception<mixer_error>(\"Failed to process pending events\", num_events);\n    }\n\n    return num_events;\n  }\n\n  /**\n   * Get volume in percentage\n   */\n  int mixer::get_volume() {\n    assert(m_elem != nullptr);\n\n    long chan_n = 0, vol_total = 0, vol, vol_min, vol_max;\n\n    snd_mixer_selem_get_playback_volume_range(m_elem, &vol_min, &vol_max);\n\n    for (int i = 0; i <= SND_MIXER_SCHN_LAST; i++) {\n      if (snd_mixer_selem_has_playback_channel(m_elem, static_cast<snd_mixer_selem_channel_id_t>(i))) {\n        snd_mixer_selem_get_playback_volume(m_elem, static_cast<snd_mixer_selem_channel_id_t>(i), &vol);\n        vol_total += vol;\n        chan_n++;\n      }\n    }\n\n    return math_util::unbounded_percentage(vol_total / chan_n, vol_min, vol_max);\n  }\n\n  /**\n   * Get normalized volume in percentage\n   */\n  int mixer::get_normalized_volume() {\n    assert(m_elem != nullptr);\n\n    long chan_n = 0, vol_total = 0, vol, vol_min, vol_max;\n    double normalized, min_norm;\n\n    snd_mixer_selem_get_playback_dB_range(m_elem, &vol_min, &vol_max);\n\n    for (int i = 0; i <= SND_MIXER_SCHN_LAST; i++) {\n      if (snd_mixer_selem_has_playback_channel(m_elem, static_cast<snd_mixer_selem_channel_id_t>(i))) {\n        snd_mixer_selem_get_playback_dB(m_elem, static_cast<snd_mixer_selem_channel_id_t>(i), &vol);\n        vol_total += vol;\n        chan_n++;\n      }\n    }\n\n    if (vol_max - vol_min <= MAX_LINEAR_DB_SCALE * 100) {\n       return math_util::percentage(vol_total / chan_n, vol_min, vol_max);\n    }\n\n    normalized = pow(10, (vol_total / chan_n - vol_max) / 6000.0);\n    if (vol_min != SND_CTL_TLV_DB_GAIN_MUTE) {\n      min_norm = pow(10, (vol_min - vol_max) / 6000.0);\n      normalized = (normalized - min_norm) / (1 - min_norm);\n    }\n\n    return 100.0f * normalized + 0.5f;\n  }\n\n  /**\n   * Set volume to given percentage\n   */\n  void mixer::set_volume(float percentage) {\n    assert(m_elem != nullptr);\n\n    if (is_muted()) {\n      return;\n    }\n\n    long vol_min, vol_max;\n    snd_mixer_selem_get_playback_volume_range(m_elem, &vol_min, &vol_max);\n    snd_mixer_selem_set_playback_volume_all(m_elem, math_util::percentage_to_value<int>(percentage, vol_min, vol_max));\n  }\n\n  /**\n   * Set normalized volume to given percentage\n   */\n  void mixer::set_normalized_volume(float percentage) {\n    assert(m_elem != nullptr);\n\n    if (is_muted()) {\n      return;\n    }\n\n    long vol_min, vol_max;\n    double min_norm;\n    percentage = percentage / 100.0f;\n\n    snd_mixer_selem_get_playback_dB_range(m_elem, &vol_min, &vol_max);\n\n    if (vol_max - vol_min <= MAX_LINEAR_DB_SCALE * 100) {\n      snd_mixer_selem_set_playback_dB_all(m_elem, lrint(percentage * (vol_max - vol_min)) + vol_min, 0);\n      return;\n    }\n\n    if (vol_min != SND_CTL_TLV_DB_GAIN_MUTE) {\n      min_norm = pow(10, (vol_min - vol_max) / 6000.0);\n      percentage = percentage * (1 - min_norm) + min_norm;\n    }\n\n    snd_mixer_selem_set_playback_dB_all(m_elem, lrint(6000.0 * log10(percentage)) + vol_max, 0);\n  }\n\n  /**\n   * Set mute state\n   */\n  void mixer::set_mute(bool mode) {\n    assert(m_elem != nullptr);\n\n    snd_mixer_selem_set_playback_switch_all(m_elem, mode);\n  }\n\n  /**\n   * Toggle mute state\n   */\n  void mixer::toggle_mute() {\n    assert(m_elem != nullptr);\n\n    int state;\n\n    snd_mixer_selem_get_playback_switch(m_elem, SND_MIXER_SCHN_MONO, &state);\n    snd_mixer_selem_set_playback_switch_all(m_elem, !state);\n  }\n\n  /**\n   * Get current mute state\n   */\n  bool mixer::is_muted() {\n    assert(m_elem != nullptr);\n\n    int state = 0;\n\n    for (int i = 0; i <= SND_MIXER_SCHN_LAST; i++) {\n      if (snd_mixer_selem_has_playback_channel(m_elem, static_cast<snd_mixer_selem_channel_id_t>(i))) {\n        int state_ = 0;\n        snd_mixer_selem_get_playback_switch(m_elem, static_cast<snd_mixer_selem_channel_id_t>(i), &state_);\n        state = state || state_;\n      }\n    }\n    return !state;\n  }\n}\n\nPOLYBAR_NS_END\n\n// Path: src/events/signal_receiver.cpp\n#include \"events/signal_receiver.hpp\"\n\n// Path: src/events/signal_emitter.cpp\n#include \"events/signal_emitter.hpp\"\n#include \"utils/factory.hpp\"\n\nPOLYBAR_NS\n\nsignal_receivers_t g_signal_receivers;\n\n/**\n * Create instance\n */\nsignal_emitter::make_type signal_emitter::make() {\n  return static_cast<signal_emitter&>(*factory_util::singleton<signal_emitter>());\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/socket.cpp\n#include <fcntl.h>\n#include <sys/socket.h>\n#include <sys/un.h>\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/mixins.hpp\"\n#include \"utils/socket.hpp\"\n\nPOLYBAR_NS\n\nusing std::snprintf;\nusing std::strlen;\n\nnamespace socket_util {\n  /**\n   * Constructor: establishing socket connection\n   */\n  unix_connection::unix_connection(string&& path) : m_socketpath(path) {\n    struct sockaddr_un socket_addr {};\n    socket_addr.sun_family = AF_UNIX;\n\n    if ((m_fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {\n      throw system_error(\"Failed to open unix connection\");\n    }\n\n    snprintf(socket_addr.sun_path, sizeof(socket_addr.sun_path), \"%s\", m_socketpath.c_str());\n    auto len = sizeof(socket_addr);\n\n...\n", "needle_token_start": 7216, "needle_token_end": 7261, "code_context_ntokens": 16061, "output": ["```cpp\nvoid g_mpd_signal_handler(int signum) {\n    if (signum == SIGPIPE) {\n      g_connection_closed = 1;\n    }\n}\n```"]}
{"repo": "polybar/polybar", "name": "is_virtual", "language": "cpp", "path": "src/adapters/net.cpp", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To determine if a network interface is virtual or physical by checking its system path.\n2. **Input**: A string representing the name of the network interface.\n3. **Output**: A boolean value; `true` if the interface is virtual, otherwise `false`.\n4. **Procedure**: The function constructs the full path to the network interface using a predefined directory path. It then resolves the absolute path of this interface. If the resolved path starts with a specific virtual directory path, the function returns `true`, indicating the interface is virtual; otherwise, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/modules/meta/factory.cpp\n#include \"modules/meta/factory.hpp\"\n\n#include \"modules/meta/all.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n\n  /**\n...\n// Path: src/modules/meta/base.cpp\n#include \"modules/meta/base.hpp\"\n\n#include <utility>\n\n#include \"components/builder.hpp\"\n#include \"drawtypes/label.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  // module_format {{{\n\n  string module_format::decorate(builder* builder, string output) {\n    if (output.empty()) {\n      builder->flush();\n      return \"\";\n    }\n    if (offset) {\n      builder->offset(offset);\n    }\n    if (margin) {\n      builder->spacing(margin);\n    }\n    if (bg.has_color()) {\n      builder->background(bg);\n    }\n    if (fg.has_color()) {\n      builder->foreground(fg);\n    }\n    if (ul.has_color()) {\n      builder->underline(ul);\n    }\n    if (ol.has_color()) {\n      builder->overline(ol);\n    }\n    if (font > 0) {\n      builder->font(font);\n    }\n    if (padding) {\n      builder->spacing(padding);\n    }\n\n    builder->node(prefix);\n\n    if (bg.has_color()) {\n      builder->background(bg);\n    }\n    if (fg.has_color()) {\n      builder->foreground(fg);\n    }\n    if (ul.has_color()) {\n      builder->underline(ul);\n    }\n    if (ol.has_color()) {\n      builder->overline(ol);\n    }\n\n    builder->node(output);\n    builder->node(suffix);\n\n    if (padding) {\n      builder->spacing(padding);\n    }\n    if (font > 0) {\n      builder->font_close();\n    }\n    if (ol.has_color()) {\n      builder->overline_close();\n    }\n    if (ul.has_color()) {\n      builder->underline_close();\n    }\n    if (fg.has_color()) {\n      builder->foreground_close();\n    }\n    if (bg.has_color()) {\n      builder->background_close();\n    }\n    if (margin) {\n      builder->spacing(margin);\n    }\n\n    return builder->flush();\n  }\n\n  // }}}\n  // module_formatter {{{\n\n  void module_formatter::add_value(string&& name, string&& value, vector<string>&& tags, vector<string>&& whitelist) {\n    const auto formatdef = [&](const string& param, const auto& fallback) {\n      return m_conf.get(\"settings\", \"format-\" + param, fallback);\n    };\n\n    auto format = make_unique<module_format>();\n    format->value = std::move(value);\n    format->fg = m_conf.get(m_modname, name + \"-foreground\", formatdef(\"foreground\", format->fg));\n    format->bg = m_conf.get(m_modname, name + \"-background\", formatdef(\"background\", format->bg));\n    format->ul = m_conf.get(m_modname, name + \"-underline\", formatdef(\"underline\", format->ul));\n    format->ol = m_conf.get(m_modname, name + \"-overline\", formatdef(\"overline\", format->ol));\n    format->ulsize = m_conf.get(m_modname, name + \"-underline-size\", formatdef(\"underline-size\", format->ulsize));\n    format->olsize = m_conf.get(m_modname, name + \"-overline-size\", formatdef(\"overline-size\", format->olsize));\n    format->spacing = m_conf.get(m_modname, name + \"-spacing\", formatdef(\"spacing\", format->spacing));\n    format->padding = m_conf.get(m_modname, name + \"-padding\", formatdef(\"padding\", format->padding));\n    format->margin = m_conf.get(m_modname, name + \"-margin\", formatdef(\"margin\", format->margin));\n    format->offset = m_conf.get(m_modname, name + \"-offset\", formatdef(\"offset\", format->offset));\n    format->font = m_conf.get(m_modname, name + \"-font\", formatdef(\"font\", format->font));\n\n    try {\n      format->prefix = load_label(m_conf, m_modname, name + \"-prefix\");\n    } catch (const key_error& err) {\n      // prefix not defined\n    }\n\n    try {\n      format->suffix = load_label(m_conf, m_modname, name + \"-suffix\");\n    } catch (const key_error& err) {\n      // suffix not defined\n    }\n\n    vector<string> tag_collection;\n    tag_collection.reserve(tags.size() + whitelist.size());\n    tag_collection.insert(tag_collection.end(), tags.begin(), tags.end());\n    tag_collection.insert(tag_collection.end(), whitelist.begin(), whitelist.end());\n\n    size_t start = 0;\n    size_t end = 0;\n    while ((start = format->value.find('<', start)) != string::npos && (end = format->value.find('>', start)) != string::npos) {\n      string tag{format->value.substr(start, end - start + 1)};\n      if (find(tag_collection.begin(), tag_collection.end(), tag) == tag_collection.end()) {\n        throw undefined_format_tag(tag + \" is not a valid format tag for \\\"\" + name + \"\\\"\");\n      }\n      start = end + 1;\n    }\n\n    m_formats.insert(make_pair(move(name), move(format)));\n  }\n\n  void module_formatter::add(string name, string fallback, vector<string>&& tags, vector<string>&& whitelist) {\n    string value = m_conf.get(m_modname, name, move(fallback));\n    add_value(move(name), move(value), forward<vector<string>>(tags), forward<vector<string>>(whitelist));\n  }\n\n  void module_formatter::add_optional(string name, vector<string>&& tags, vector<string>&& whitelist) {\n    if (m_conf.has(m_modname, name)) {\n      string value = m_conf.get(m_modname, name);\n      add_value(move(name), move(value), move(tags), move(whitelist));\n    }\n  }\n\n  bool module_formatter::has(const string& tag, const string& format_name) {\n    auto format = m_formats.find(format_name);\n    if (format == m_formats.end()) {\n      return false;\n    }\n    return format->second->value.find(tag) != string::npos;\n  }\n\n  bool module_formatter::has(const string& tag) {\n    for (auto&& format : m_formats) {\n      if (format.second->value.find(tag) != string::npos) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  bool module_formatter::has_format(const string& format_name) {\n    return m_formats.find(format_name) != m_formats.end();\n  }\n\n  shared_ptr<module_format> module_formatter::get(const string& format_name) {\n    auto format = m_formats.find(format_name);\n    if (format == m_formats.end()) {\n      throw undefined_format(\"Format \\\"\" + format_name + \"\\\" has not been added\");\n    }\n    return format->second;\n  }\n\n  // }}}\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/pulseaudio.cpp\n#include \"adapters/pulseaudio.hpp\"\n\n#include \"components/logger.hpp\"\n\nPOLYBAR_NS\n\n/**\n * Construct pulseaudio object\n */\npulseaudio::pulseaudio(const logger& logger, string&& sink_name, bool max_volume)\n    : m_log(logger), spec_s_name(sink_name) {\n  m_mainloop = pa_threaded_mainloop_new();\n  if (!m_mainloop) {\n    throw pulseaudio_error(\"Could not create pulseaudio threaded mainloop.\");\n  }\n  pa_threaded_mainloop_lock(m_mainloop);\n\n  m_context = pa_context_new(pa_threaded_mainloop_get_api(m_mainloop), \"polybar\");\n  if (!m_context) {\n    pa_threaded_mainloop_unlock(m_mainloop);\n    pa_threaded_mainloop_free(m_mainloop);\n    throw pulseaudio_error(\"Could not create pulseaudio context.\");\n  }\n\n  pa_context_set_state_callback(m_context, context_state_callback, this);\n\n  m_state_callback_signal = false;\n\n  if (pa_context_connect(m_context, nullptr, PA_CONTEXT_NOFLAGS, nullptr) < 0) {\n    pa_context_disconnect(m_context);\n    pa_context_unref(m_context);\n    pa_threaded_mainloop_unlock(m_mainloop);\n    pa_threaded_mainloop_free(m_mainloop);\n    throw pulseaudio_error(\"Could not connect pulseaudio context.\");\n  }\n\n  if (pa_threaded_mainloop_start(m_mainloop) < 0) {\n    pa_context_disconnect(m_context);\n    pa_context_unref(m_context);\n    pa_threaded_mainloop_unlock(m_mainloop);\n    pa_threaded_mainloop_free(m_mainloop);\n    throw pulseaudio_error(\"Could not start pulseaudio mainloop.\");\n  }\n\n  m_log.trace(\"pulseaudio: started mainloop\");\n\n  /*\n   * Only wait for signal from the context state callback, if it has not\n   * already signalled the mainloop since pa_context_connect was called,\n   * otherwise, we would wait forever.\n   *\n   * The while loop ensures spurious wakeups are handled.\n   */\n  while (!m_state_callback_signal) {\n    pa_threaded_mainloop_wait(m_mainloop);\n  }\n\n  if (pa_context_get_state(m_context) != PA_CONTEXT_READY) {\n    pa_threaded_mainloop_unlock(m_mainloop);\n    pa_threaded_mainloop_stop(m_mainloop);\n    pa_context_disconnect(m_context);\n    pa_context_unref(m_context);\n    pa_threaded_mainloop_free(m_mainloop);\n    throw pulseaudio_error(\"Could not connect to pulseaudio server.\");\n  }\n\n  pa_operation* op{nullptr};\n  if (!sink_name.empty()) {\n    op = pa_context_get_sink_info_by_name(m_context, sink_name.c_str(), sink_info_callback, this);\n    wait_loop(op, m_mainloop);\n  }\n  if (s_name.empty()) {\n    // get the sink index\n    op = pa_context_get_sink_info_by_name(m_context, DEFAULT_SINK, sink_info_callback, this);\n    wait_loop(op, m_mainloop);\n    m_log.notice(\"pulseaudio: using default sink %s\", s_name);\n  } else {\n    m_log.trace(\"pulseaudio: using sink %s\", s_name);\n  }\n\n  m_max_volume = max_volume ? PA_VOLUME_UI_MAX : PA_VOLUME_NORM;\n\n  auto event_types = static_cast<pa_subscription_mask_t>(PA_SUBSCRIPTION_MASK_SINK | PA_SUBSCRIPTION_MASK_SERVER);\n  op = pa_context_subscribe(m_context, event_types, simple_callback, this);\n  wait_loop(op, m_mainloop);\n  if (!success) {\n    throw pulseaudio_error(\"Failed to subscribe to sink.\");\n  }\n  pa_context_set_subscribe_callback(m_context, subscribe_callback, this);\n\n  update_volume(op);\n\n  pa_threaded_mainloop_unlock(m_mainloop);\n}\n\n/**\n * Deconstruct pulseaudio\n */\npulseaudio::~pulseaudio() {\n  pa_threaded_mainloop_stop(m_mainloop);\n  pa_context_disconnect(m_context);\n  pa_context_unref(m_context);\n  pa_threaded_mainloop_free(m_mainloop);\n}\n\n/**\n * Get sink name\n */\nconst string& pulseaudio::get_name() {\n  return s_name;\n}\n\n/**\n * Wait for events\n */\nbool pulseaudio::wait() {\n  return m_events.size() > 0;\n}\n\n/**\n * Process queued pulseaudio events\n */\nint pulseaudio::process_events() {\n  int ret = m_events.size();\n  pa_threaded_mainloop_lock(m_mainloop);\n  pa_operation* o{nullptr};\n  // clear the queue\n  while (!m_events.empty()) {\n    switch (m_events.front()) {\n      // try to get specified sink\n      case evtype::NEW:\n        // redundant if already using specified sink\n        if (!spec_s_name.empty()) {\n          o = pa_context_get_sink_info_by_name(m_context, spec_s_name.c_str(), sink_info_callback, this);\n          wait_loop(o, m_mainloop);\n          break;\n        }\n        // FALLTHRU\n      case evtype::SERVER:\n        // don't fallthrough only if always using default sink\n        if (!spec_s_name.empty()) {\n          break;\n        }\n        // FALLTHRU\n      // get default sink\n      case evtype::REMOVE:\n        o = pa_context_get_sink_info_by_name(m_context, DEFAULT_SINK, sink_info_callback, this);\n        wait_loop(o, m_mainloop);\n        if (spec_s_name != s_name) {\n          m_log.notice(\"pulseaudio: using default sink %s\", s_name);\n        }\n        break;\n      default:\n        break;\n    }\n    update_volume(o);\n    m_events.pop();\n  }\n  pa_threaded_mainloop_unlock(m_mainloop);\n  return ret;\n}\n\n/**\n * Get volume in percentage\n */\nint pulseaudio::get_volume() {\n  // alternatively, user pa_cvolume_avg_mask() to average selected channels\n  return static_cast<int>(pa_cvolume_max(&cv) * 100.0f / PA_VOLUME_NORM + 0.5f);\n}\n\n/**\n * Get volume in decibels\n */\ndouble pulseaudio::get_decibels() {\n  return pa_sw_volume_to_dB(pa_cvolume_max(&cv));\n}\n\n/**\n * Set volume to given percentage\n */\nvoid pulseaudio::set_volume(float percentage) {\n  pa_threaded_mainloop_lock(m_mainloop);\n  pa_volume_t vol = math_util::percentage_to_value<pa_volume_t>(percentage, PA_VOLUME_MUTED, PA_VOLUME_NORM);\n  pa_cvolume_scale(&cv, vol);\n  pa_operation* op = pa_context_set_sink_volume_by_index(m_context, m_index, &cv, simple_callback, this);\n  wait_loop(op, m_mainloop);\n  if (!success)\n    throw pulseaudio_error(\"Failed to set sink volume.\");\n  pa_threaded_mainloop_unlock(m_mainloop);\n}\n\n/**\n * Increment or decrement volume by given percentage (prevents accumulation of rounding errors from get_volume)\n */\nvoid pulseaudio::inc_volume(int delta_perc) {\n  pa_threaded_mainloop_lock(m_mainloop);\n  pa_volume_t vol = math_util::percentage_to_value<pa_volume_t>(abs(delta_perc), PA_VOLUME_NORM);\n  if (delta_perc > 0) {\n    pa_volume_t current = pa_cvolume_max(&cv);\n    if (current + vol <= m_max_volume) {\n      pa_cvolume_inc(&cv, vol);\n    } else if (current < m_max_volume) {\n      // avoid rounding errors and set to m_max_volume directly\n      pa_cvolume_scale(&cv, m_max_volume);\n    } else {\n      m_log.notice(\"pulseaudio: maximum volume reached\");\n    }\n  } else\n    pa_cvolume_dec(&cv, vol);\n  pa_operation* op = pa_context_set_sink_volume_by_index(m_context, m_index, &cv, simple_callback, this);\n  wait_loop(op, m_mainloop);\n  if (!success)\n    throw pulseaudio_error(\"Failed to set sink volume.\");\n  pa_threaded_mainloop_unlock(m_mainloop);\n}\n\n/**\n * Set mute state\n */\nvoid pulseaudio::set_mute(bool mode) {\n  pa_threaded_mainloop_lock(m_mainloop);\n  pa_operation* op = pa_context_set_sink_mute_by_index(m_context, m_index, mode, simple_callback, this);\n  wait_loop(op, m_mainloop);\n  if (!success)\n    throw pulseaudio_error(\"Failed to mute sink.\");\n  pa_threaded_mainloop_unlock(m_mainloop);\n}\n\n/**\n * Toggle mute state\n */\nvoid pulseaudio::toggle_mute() {\n  set_mute(!is_muted());\n}\n\n/**\n * Get current mute state\n */\nbool pulseaudio::is_muted() {\n  return muted;\n}\n\n/**\n * Update local volume cache\n */\nvoid pulseaudio::update_volume(pa_operation* o) {\n  o = pa_context_get_sink_info_by_index(m_context, m_index, get_sink_volume_callback, this);\n  wait_loop(o, m_mainloop);\n}\n\n/**\n * Callback when getting volume\n */\nvoid pulseaudio::get_sink_volume_callback(pa_context*, const pa_sink_info* info, int, void* userdata) {\n  pulseaudio* This = static_cast<pulseaudio*>(userdata);\n  if (info) {\n    This->cv = info->volume;\n    This->muted = info->mute;\n  }\n  pa_threaded_mainloop_signal(This->m_mainloop, 0);\n}\n\n/**\n * Callback when subscribing to changes\n */\nvoid pulseaudio::subscribe_callback(pa_context*, pa_subscription_event_type_t t, uint32_t idx, void* userdata) {\n  pulseaudio* This = static_cast<pulseaudio*>(userdata);\n  switch (t & PA_SUBSCRIPTION_EVENT_FACILITY_MASK) {\n    case PA_SUBSCRIPTION_EVENT_SERVER:\n      switch (t & PA_SUBSCRIPTION_EVENT_TYPE_MASK) {\n        case PA_SUBSCRIPTION_EVENT_CHANGE:\n          This->m_events.emplace(evtype::SERVER);\n          break;\n      }\n      break;\n    case PA_SUBSCRIPTION_EVENT_SINK:\n      switch (t & PA_SUBSCRIPTION_EVENT_TYPE_MASK) {\n        case PA_SUBSCRIPTION_EVENT_NEW:\n          This->m_events.emplace(evtype::NEW);\n          break;\n        case PA_SUBSCRIPTION_EVENT_CHANGE:\n          if (idx == This->m_index)\n            This->m_events.emplace(evtype::CHANGE);\n          break;\n        case PA_SUBSCRIPTION_EVENT_REMOVE:\n          if (idx == This->m_index)\n            This->m_events.emplace(evtype::REMOVE);\n          break;\n      }\n      break;\n  }\n  pa_threaded_mainloop_signal(This->m_mainloop, 0);\n}\n\n/**\n * Simple callback to check for success\n */\nvoid pulseaudio::simple_callback(pa_context*, int success, void* userdata) {\n  pulseaudio* This = static_cast<pulseaudio*>(userdata);\n  This->success = success;\n  pa_threaded_mainloop_signal(This->m_mainloop, 0);\n}\n\n/**\n * Callback when getting sink info & existence\n */\nvoid pulseaudio::sink_info_callback(pa_context*, const pa_sink_info* info, int eol, void* userdata) {\n  pulseaudio* This = static_cast<pulseaudio*>(userdata);\n  if (!eol && info) {\n    This->m_index = info->index;\n    This->s_name = info->name;\n  }\n  pa_threaded_mainloop_signal(This->m_mainloop, 0);\n}\n\n/**\n * Callback when context state changes\n */\nvoid pulseaudio::context_state_callback(pa_context* context, void* userdata) {\n  pulseaudio* This = static_cast<pulseaudio*>(userdata);\n  switch (pa_context_get_state(context)) {\n    case PA_CONTEXT_READY:\n    case PA_CONTEXT_TERMINATED:\n    case PA_CONTEXT_FAILED:\n      This->m_state_callback_signal = true;\n      pa_threaded_mainloop_signal(This->m_mainloop, 0);\n      break;\n\n    case PA_CONTEXT_UNCONNECTED:\n    case PA_CONTEXT_CONNECTING:\n    case PA_CONTEXT_AUTHORIZING:\n    case PA_CONTEXT_SETTING_NAME:\n      break;\n  }\n}\n\ninline void pulseaudio::wait_loop(pa_operation* op, pa_threaded_mainloop* loop) {\n  while (pa_operation_get_state(op) != PA_OPERATION_DONE) {\n    pa_threaded_mainloop_wait(loop);\n  }\n  pa_operation_unref(op);\n}\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/script_runner.cpp\n#include \"adapters/script_runner.hpp\"\n\n#include <cassert>\n#include <functional>\n\n#include \"modules/meta/base.hpp\"\n#include \"utils/command.hpp\"\n#include \"utils/io.hpp\"\n#include \"utils/scope.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nscript_runner::script_runner(on_update on_update, const string& exec, const string& exec_if, bool tail,\n    interval interval_success, interval interval_fail, const vector<pair<string, string>>& env)\n    : m_log(logger::make())\n    , m_on_update(on_update)\n    , m_exec(exec)\n    , m_exec_if(exec_if)\n    , m_tail(tail)\n    , m_interval_success(interval_success)\n    , m_interval_fail(interval_fail)\n    , m_env(env) {}\n\n/**\n * Check if defined condition is met\n */\nbool script_runner::check_condition() const {\n  if (m_exec_if.empty()) {\n    return true;\n  }\n\n  command<output_policy::IGNORED> exec_if_cmd(m_log, m_exec_if);\n  return exec_if_cmd.exec(true) == 0;\n}\n\n/**\n * Process mutex wrapped script handler\n */\nscript_runner::interval script_runner::process() {\n  if (m_tail) {\n    return run_tail();\n  } else {\n    return run();\n  }\n}\n\nvoid script_runner::clear_output() {\n  auto changed = set_output(\"\");\n  if (changed) {\n    m_on_update(m_data);\n  }\n}\n\nvoid script_runner::stop() {\n  m_stopping = true;\n}\n\nbool script_runner::is_stopping() const {\n  return m_stopping;\n}\n\n/**\n * Updates the current output.\n *\n * Returns true if the output changed.\n */\nbool script_runner::set_output(string&& new_output) {\n  if (m_data.output != new_output) {\n    m_data.output = std::move(new_output);\n    return true;\n  }\n\n  return false;\n}\n\n/**\n * Updates the current exit status\n *\n * Returns true if the exit status changed.\n */\nbool script_runner::set_exit_status(int new_status) {\n  auto changed = (m_data.exit_status != new_status);\n  m_data.exit_status = new_status;\n\n  return changed;\n}\n\nscript_runner::interval script_runner::run() {\n  auto exec = string_util::replace_all(m_exec, \"%counter%\", to_string(++m_data.counter));\n  m_log.info(\"script_runner: Invoking shell command: \\\"%s\\\"\", exec);\n  command<output_policy::REDIRECTED> cmd(m_log, exec);\n\n  try {\n    cmd.exec(false, m_env);\n  } catch (const exception& err) {\n    m_log.err(\"script_runner: %s\", err.what());\n    throw modules::module_error(\"Failed to execute command, stopping module...\");\n  }\n\n  int fd = cmd.get_stdout(PIPE_READ);\n  assert(fd != -1);\n\n  bool changed = false;\n\n  bool got_output = false;\n  while (!m_stopping && cmd.is_running() && !io_util::poll(fd, POLLHUP, 0)) {\n    /**\n     * For non-tailed scripts, we only use the first line. However, to ensure interruptability when the module shuts\n     * down, we still need to continue polling.\n     */\n    if (io_util::poll_read(fd, 250) && !got_output) {\n      changed = set_output(cmd.readline());\n      got_output = true;\n    }\n  }\n\n  if (m_stopping) {\n    cmd.terminate();\n    return 0s;\n  }\n\n  auto exit_status_changed = set_exit_status(cmd.wait());\n\n  if (changed || exit_status_changed) {\n    m_on_update(m_data);\n  }\n\n  if (m_data.exit_status == 0) {\n    return m_interval_success;\n  } else {\n    return std::max(m_interval_fail, interval{1s});\n  }\n}\n\nscript_runner::interval script_runner::run_tail() {\n  auto exec = string_util::replace_all(m_exec, \"%counter%\", to_string(++m_data.counter));\n  m_log.info(\"script_runner: Invoking shell command: \\\"%s\\\"\", exec);\n  command<output_policy::REDIRECTED> cmd(m_log, exec);\n\n  try {\n    cmd.exec(false, m_env);\n  } catch (const exception& err) {\n    throw modules::module_error(\"Failed to execute command: \" + string(err.what()));\n  }\n\n  scope_util::on_exit pid_guard([this]() {\n    m_data.pid = -1;\n    m_on_update(m_data);\n  });\n\n  m_data.pid = cmd.get_pid();\n\n  int fd = cmd.get_stdout(PIPE_READ);\n  assert(fd != -1);\n\n  while (!m_stopping && cmd.is_running() && !io_util::poll(fd, POLLHUP, 0)) {\n    if (io_util::poll_read(fd, 250)) {\n      auto changed = set_output(cmd.readline());\n\n      if (changed) {\n        m_on_update(m_data);\n      }\n    }\n  }\n\n  if (m_stopping) {\n    cmd.terminate();\n    return 0s;\n  }\n\n  auto exit_status = cmd.wait();\n\n  if (exit_status == 0) {\n    return m_interval_success;\n  } else {\n    return std::max(m_interval_fail, interval{1s});\n  }\n}\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/net_iw.cpp\n#include \"adapters/net.hpp\"\n\n#include \"utils/file.hpp\"\n\nPOLYBAR_NS\n\nnamespace net {\n  // class : wireless_network {{{\n\n  /**\n   * Query the wireless device for information\n   * about the current connection\n   */\n  bool wireless_network::query(bool accumulate) {\n    if (!network::query(accumulate)) {\n      return false;\n    }\n\n    auto socket_fd = file_util::make_file_descriptor(iw_sockets_open());\n    if (!*socket_fd) {\n      return false;\n    }\n\n    struct iwreq req {};\n\n    if (iw_get_ext(*socket_fd, m_interface.c_str(), SIOCGIWMODE, &req) == -1) {\n      return false;\n    }\n\n    // Ignore interfaces in ad-hoc mode\n    if (req.u.mode == IW_MODE_ADHOC) {\n      return false;\n    }\n\n    query_essid(*socket_fd);\n    query_quality(*socket_fd);\n\n    return true;\n  }\n\n  /**\n   * Check current connection state\n   */\n  bool wireless_network::connected() const {\n    if (!network::test_interface()) {\n      return false;\n    }\n    return !m_essid.empty();\n  }\n\n  /**\n   * ESSID reported by last query\n   */\n  string wireless_network::essid() const {\n    return m_essid;\n  }\n\n  /**\n   * Signal strength percentage reported by last query\n   */\n  int wireless_network::signal() const {\n    return m_signalstrength.percentage();\n  }\n\n  /**\n   * Link quality percentage reported by last query\n   */\n  int wireless_network::quality() const {\n    return m_linkquality.percentage();\n  }\n\n  /**\n   * Query for ESSID\n   */\n  void wireless_network::query_essid(const int& socket_fd) {\n    char essid[IW_ESSID_MAX_SIZE + 1];\n\n    struct iwreq req {};\n    req.u.essid.pointer = &essid;\n    req.u.essid.length = sizeof(essid);\n    req.u.essid.flags = 0;\n\n    if (iw_get_ext(socket_fd, m_interface.c_str(), SIOCGIWESSID, &req) != -1) {\n      m_essid = string{essid};\n    } else {\n      m_essid.clear();\n    }\n  }\n\n  /**\n   * Query for device driver quality values\n   */\n  void wireless_network::query_quality(const int& socket_fd) {\n    iwrange range{};\n    iwstats stats{};\n\n    // Fill range\n    if (iw_get_range_info(socket_fd, m_interface.c_str(), &range) == -1) {\n      return;\n    }\n    // Fill stats\n    if (iw_get_stats(socket_fd, m_interface.c_str(), &stats, &range, 1) == -1) {\n      return;\n    }\n\n    // Check if the driver supplies the quality value\n    if (stats.qual.updated & IW_QUAL_QUAL_INVALID) {\n      return;\n    }\n    // Check if the driver supplies the quality level value\n    if (stats.qual.updated & IW_QUAL_LEVEL_INVALID) {\n      return;\n    }\n\n    // Check if the link quality has been updated\n    if (stats.qual.updated & IW_QUAL_QUAL_UPDATED) {\n      m_linkquality.val = stats.qual.qual;\n      m_linkquality.max = range.max_qual.qual;\n    }\n\n    // Check if the signal strength has been updated\n    if (stats.qual.updated & IW_QUAL_LEVEL_UPDATED) {\n      m_signalstrength.val = stats.qual.level;\n      m_signalstrength.max = range.max_qual.level;\n\n      // Check if the values are defined in dBm\n      if (stats.qual.level > range.max_qual.level) {\n        m_signalstrength.val -= 0x100;\n        m_signalstrength.max = (stats.qual.level - range.max_qual.level) - 0x100;\n      }\n    }\n  }\n\n  // }}}\n}  // namespace net\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/net.cpp\n#include \"adapters/net.hpp\"\n\n#include <arpa/inet.h>\n#include <dirent.h>\n#include <linux/ethtool.h>\n#include <linux/if_link.h>\n#include <linux/sockios.h>\n#include <net/if.h>\n#include <netdb.h>\n#include <netinet/in.h>\n#include <sys/ioctl.h>\n#include <sys/socket.h>\n#include <sys/types.h>\n\n#include <cassert>\n#include <iomanip>\n\n#include \"common.hpp\"\n#include \"settings.hpp\"\n#include \"utils/command.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace net {\n  enum class NetType {\n    WIRELESS,\n    ETHERNET,\n    OTHER,\n  };\n\n  static const string NO_IP = string(\"N/A\");\n  static const string NO_MAC = string(\"N/A\");\n  static const string NET_PATH = \"/sys/class/net/\";\n  static const string VIRTUAL_PATH = \"/sys/devices/virtual/\";\n\n  \nstatic bool is_virtual(const std::string& ifname) {\n    char* target = realpath((NET_PATH + ifname).c_str(), nullptr);\n\n    if (!target) {\n      throw system_error(\"realpath\");\n    }\n\n    const std::string real_path{target};\n    free(target);\n    return real_path.rfind(VIRTUAL_PATH, 0) == 0;\n  }\n\n  NetType iface_type(const std::string& ifname) {\n    if (file_util::exists(NET_PATH + ifname + \"/wireless\")) {\n      return NetType::WIRELESS;\n    }\n\n    if (is_virtual(ifname)) {\n      return NetType::OTHER;\n    }\n\n    return NetType::ETHERNET;\n  }\n\n  bool is_interface_valid(const string& ifname) {\n    return if_nametoindex(ifname.c_str()) != 0;\n  }\n\n  /**\n   * Returns the canonical name of the given interface and whether that differs\n   * from the given name.\n   *\n   * The canonical name is the name that has an entry in `/sys/class/net`.\n   *\n   * This resolves any altnames that were defined (see man ip-link).\n   */\n  std::pair<string, bool> get_canonical_interface(const string& ifname) {\n    int idx = if_nametoindex(ifname.c_str());\n\n    if (idx == 0) {\n      throw system_error(\"if_nameindex(\" + ifname + \")\");\n    }\n\n    char canonical[IF_NAMESIZE];\n    if (!if_indextoname(idx, canonical)) {\n      throw system_error(\"if_indextoname(\" + to_string(idx) + \")\");\n    }\n\n    string str{canonical};\n\n    return {str, str != ifname};\n  }\n\n  /**\n   * Test if interface with given name is a wireless device\n   */\n  bool is_wireless_interface(const string& ifname) {\n    return iface_type(ifname) == NetType::WIRELESS;\n  }\n\n  std::string find_interface(NetType type) {\n    struct ifaddrs* ifaddrs;\n    getifaddrs(&ifaddrs);\n\n    struct ifaddrs* candidate_if = nullptr;\n\n    for (struct ifaddrs* i = ifaddrs; i != nullptr; i = i->ifa_next) {\n      const std::string name{i->ifa_name};\n      const NetType iftype = iface_type(name);\n      if (iftype != type) {\n        continue;\n      }\n      if (candidate_if == nullptr) {\n        candidate_if = i;\n      } else if (((candidate_if->ifa_flags & IFF_RUNNING) == 0) && ((i->ifa_flags & IFF_RUNNING) > 0)) {\n        candidate_if = i;\n      }\n    }\n    if (candidate_if) {\n      const std::string name{candidate_if->ifa_name};\n      freeifaddrs(ifaddrs);\n      return name;\n    }\n    freeifaddrs(ifaddrs);\n    return \"\";\n  }\n\n  std::string find_wireless_interface() {\n    return find_interface(NetType::WIRELESS);\n  }\n\n  std::string find_wired_interface() {\n    return find_interface(NetType::ETHERNET);\n  }\n\n  // class : network {{{\n\n  /**\n   * Construct network interface\n   */\n  network::network(string interface) : m_log(logger::make()), m_interface(move(interface)) {\n    assert(is_interface_valid(m_interface));\n\n    m_socketfd = file_util::make_file_descriptor(socket(AF_INET, SOCK_DGRAM, 0));\n    if (!*m_socketfd) {\n      throw network_error(\"Failed to open socket\");\n    }\n\n    check_tuntap_or_bridge();\n  }\n\n  /**\n   * Query device driver for information\n   */\n  bool network::query(bool accumulate) {\n    m_status.previous = m_status.current;\n    m_status.current.transmitted = 0;\n    m_status.current.received = 0;\n    m_status.current.time = std::chrono::steady_clock::now();\n    m_status.ip = NO_IP;\n    m_status.ip6 = NO_IP;\n\n    struct ifaddrs* ifaddr;\n    if (getifaddrs(&ifaddr) == -1 || ifaddr == nullptr) {\n      return false;\n    }\n\n    m_status.mac = string_util::trim(file_util::contents(NET_PATH + m_interface + \"/address\"), isspace);\n    if (m_status.mac == \"\") {\n      m_status.mac = NO_MAC;\n    }\n\n    for (auto ifa = ifaddr; ifa != nullptr; ifa = ifa->ifa_next) {\n      if (ifa->ifa_addr == nullptr) {\n        continue;\n      }\n\n      if (m_interface.compare(0, m_interface.length(), ifa->ifa_name) != 0) {\n        if (!accumulate || (ifa->ifa_data == nullptr && ifa->ifa_addr->sa_family != AF_PACKET)) {\n          continue;\n        }\n      }\n\n      struct sockaddr_in6* sa6;\n\n      switch (ifa->ifa_addr->sa_family) {\n        case AF_INET:\n          char ip_buffer[NI_MAXHOST];\n          getnameinfo(ifa->ifa_addr, sizeof(sockaddr_in), ip_buffer, NI_MAXHOST, nullptr, 0, NI_NUMERICHOST);\n          m_status.ip = string{ip_buffer};\n          break;\n\n        case AF_INET6:\n          char ip6_buffer[INET6_ADDRSTRLEN];\n          sa6 = reinterpret_cast<decltype(sa6)>(ifa->ifa_addr);\n          if (IN6_IS_ADDR_LINKLOCAL(&sa6->sin6_addr)) {\n            continue;\n          }\n          if (IN6_IS_ADDR_SITELOCAL(&sa6->sin6_addr)) {\n            continue;\n          }\n          if ((((unsigned char*)sa6->sin6_addr.s6_addr)[0] & 0xFE) == 0xFC) {\n            /* Skip Unique Local Addresses (fc00::/7) */\n            continue;\n          }\n          if (inet_ntop(AF_INET6, &sa6->sin6_addr, ip6_buffer, INET6_ADDRSTRLEN) == 0) {\n            m_log.warn(\"inet_ntop() \" + string(strerror(errno)));\n            continue;\n          }\n          m_status.ip6 = string{ip6_buffer};\n          break;\n\n        case AF_PACKET:\n          if (ifa->ifa_data == nullptr) {\n            continue;\n          }\n          struct rtnl_link_stats* link_state = reinterpret_cast<decltype(link_state)>(ifa->ifa_data);\n          if (link_state == nullptr) {\n            continue;\n          }\n          m_status.current.transmitted += link_state->tx_bytes;\n          m_status.current.received += link_state->rx_bytes;\n          break;\n      }\n    }\n\n    freeifaddrs(ifaddr);\n\n    return true;\n  }\n\n  /**\n   * Run ping command to test internet connectivity\n   */\n  bool network::ping() const {\n    try {\n      auto exec = \"ping -c 2 -W 2 -I \" + m_interface + \" \" + string(CONNECTION_TEST_IP);\n      command<output_policy::IGNORED> ping(m_log, exec);\n      return ping.exec(true) == EXIT_SUCCESS;\n    } catch (const std::exception& err) {\n      return false;\n    }\n  }\n\n  /**\n   * Get interface ipv4 address\n   */\n  string network::ip() const {\n    return m_status.ip;\n  }\n  /**\n   * Get interface mac address\n   */\n  string network::mac() const {\n    return m_status.mac;\n  }\n\n  /**\n   * Get interface ipv6 address\n   */\n  string network::ip6() const {\n    return m_status.ip6;\n  }\n\n  /**\n   * Get download speed rate\n   */\n  string network::downspeed(int minwidth, const string& unit) const {\n    float bytes_diff = m_status.current.received - m_status.previous.received;\n    return format_speedrate(bytes_diff, minwidth, unit);\n  }\n\n  /**\n   * Get upload speed rate\n   */\n  string network::upspeed(int minwidth, const string& unit) const {\n    float bytes_diff = m_status.current.transmitted - m_status.previous.transmitted;\n    return format_speedrate(bytes_diff, minwidth, unit);\n  }\n\n  /**\n   * Get total net speed rate\n   */\n  string network::netspeed(int minwidth, const string& unit) const {\n    float bytes_diff = m_status.current.received - m_status.previous.received + m_status.current.transmitted -\n                       m_status.previous.transmitted;\n    return format_speedrate(bytes_diff, minwidth, unit);\n  }\n\n  /**\n   * Set if unknown counts as up\n   */\n  void network::set_unknown_up(bool unknown) {\n    m_unknown_up = unknown;\n  }\n\n  /**\n   * Query driver info to check if the\n   * interface is a TUN/TAP device or BRIDGE\n   */\n  void network::check_tuntap_or_bridge() {\n    struct ethtool_drvinfo driver {};\n    struct ifreq request {};\n\n    driver.cmd = ETHTOOL_GDRVINFO;\n\n    memset(&request, 0, sizeof(request));\n\n    /*\n     * Only copy array size minus one bytes over to ensure there is a\n     * terminating NUL byte (which is guaranteed by memset)\n     */\n    strncpy(request.ifr_name, m_interface.c_str(), IFNAMSIZ - 1);\n\n    request.ifr_data = reinterpret_cast<char*>(&driver);\n\n    if (ioctl(*m_socketfd, SIOCETHTOOL, &request) == -1) {\n      return;\n    }\n\n    // Check if it's a TUN/TAP device\n    if (strncmp(driver.bus_info, \"tun\", 3) == 0) {\n      m_tuntap = true;\n    } else if (strncmp(driver.bus_info, \"tap\", 3) == 0) {\n      m_tuntap = true;\n    } else {\n      m_tuntap = false;\n    }\n\n    if (strncmp(driver.driver, \"bridge\", 6) == 0) {\n      m_bridge = true;\n    }\n  }\n\n  /**\n   * Test if the network interface is in a valid state\n   */\n  bool network::test_interface() const {\n    auto operstate = file_util::contents(NET_PATH + m_interface + \"/operstate\");\n    bool up = operstate.compare(0, 2, \"up\") == 0;\n    return m_unknown_up ? (up || operstate.compare(0, 7, \"unknown\") == 0) : up;\n  }\n\n  /**\n   * Format up- and download speed\n   */\n  string network::format_speedrate(float bytes_diff, int minwidth, const string& unit) const {\n    // Get time difference in seconds as a float\n    const std::chrono::duration<float> duration = m_status.current.time - m_status.previous.time;\n    float time_diff = duration.count();\n    float speedrate = bytes_diff / time_diff;\n\n    vector<pair<string, int>> units{make_pair(\"G\", 2), make_pair(\"M\", 1)};\n    string suffix{\"K\"};\n    int precision = 0;\n\n    while ((speedrate /= 1000) > 999) {\n      suffix = units.back().first;\n      precision = units.back().second;\n      units.pop_back();\n    }\n\n    return sstream() << std::setw(minwidth) << std::setfill(' ') << std::setprecision(precision) << std::fixed\n                     << speedrate << \" \" << suffix << unit;\n  }\n\n  // }}}\n  // class : wired_network {{{\n\n  /**\n   * Query device driver for information\n   */\n  bool wired_network::query(bool accumulate) {\n    if (!network::query(accumulate)) {\n      return false;\n    }\n\n    if (m_tuntap) {\n      return true;\n    }\n\n    if (m_bridge) {\n      /* If bridge network then link speed cannot be computed\n       * TODO: Identify the physical network in bridge and compute the link speed\n       */\n      return true;\n    }\n\n    struct ifreq request {};\n    struct ethtool_cmd data {};\n\n    memset(&request, 0, sizeof(request));\n    strncpy(request.ifr_name, m_interface.c_str(), IFNAMSIZ - 1);\n    data.cmd = ETHTOOL_GSET;\n    request.ifr_data = reinterpret_cast<char*>(&data);\n\n    if (ioctl(*m_socketfd, SIOCETHTOOL, &request) == -1) {\n      m_linkspeed = -1;\n    } else {\n      m_linkspeed = data.speed;\n    }\n\n    return true;\n  }\n\n  /**\n   * Check current connection state\n   */\n  bool wired_network::connected() const {\n    if (!m_tuntap && !network::test_interface()) {\n      return false;\n    }\n\n    struct ethtool_value data {};\n    struct ifreq request {};\n\n    memset(&request, 0, sizeof(request));\n    strncpy(request.ifr_name, m_interface.c_str(), IFNAMSIZ - 1);\n    data.cmd = ETHTOOL_GLINK;\n    request.ifr_data = reinterpret_cast<char*>(&data);\n\n    if (ioctl(*m_socketfd, SIOCETHTOOL, &request) == -1) {\n      return false;\n    }\n\n    return data.data != 0;\n  }\n\n  /**\n   *\n   * about the current connection\n   */\n  string wired_network::linkspeed() const {\n    return m_linkspeed == -1 ? \"N/A\"\n                             : (m_linkspeed < 1000 ? (to_string(m_linkspeed) + \" Mbit/s\")\n                                                   : (to_string(m_linkspeed / 1000) + \" Gbit/s\"));\n  }\n\n  // }}}\n\n} // namespace net\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/net_nl.cpp\n#include <linux/nl80211.h>\n#include <netlink/genl/ctrl.h>\n#include <netlink/genl/genl.h>\n\n#include <algorithm>\n\n#include \"adapters/net.hpp\"\n#include \"utils/file.hpp\"\n\nPOLYBAR_NS\n\nnamespace net {\n  // class : wireless_network {{{\n\n  /**\n   * Query the wireless device for information\n   * about the current connection\n   */\n  bool wireless_network::query(bool accumulate) {\n    if (!network::query(accumulate)) {\n      return false;\n    }\n\n    struct nl_sock* sk = nl_socket_alloc();\n    if (sk == nullptr) {\n      return false;\n    }\n\n    if (genl_connect(sk) < 0) {\n      return false;\n    }\n\n    int driver_id = genl_ctrl_resolve(sk, \"nl80211\");\n    if (driver_id < 0) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    if (nl_socket_modify_cb(sk, NL_CB_VALID, NL_CB_CUSTOM, scan_cb, this) != 0) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    struct nl_msg* msg = nlmsg_alloc();\n    if (msg == nullptr) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    if ((genlmsg_put(msg, NL_AUTO_PORT, NL_AUTO_SEQ, driver_id, 0, NLM_F_DUMP, NL80211_CMD_GET_SCAN, 0) == nullptr) ||\n        nla_put_u32(msg, NL80211_ATTR_IFINDEX, m_ifid) < 0) {\n      nlmsg_free(msg);\n      nl_socket_free(sk);\n      return false;\n    }\n\n    // nl_send_sync always frees msg\n    if (nl_send_sync(sk, msg) < 0) {\n      nl_socket_free(sk);\n      return false;\n    }\n\n    nl_socket_free(sk);\n\n    return true;\n  }\n\n  /**\n   * Check current connection state\n   */\n  bool wireless_network::connected() const {\n    if (!network::test_interface()) {\n      return false;\n    }\n    return !m_essid.empty();\n  }\n\n  /**\n   * ESSID reported by last query\n   */\n  string wireless_network::essid() const {\n    return m_essid;\n  }\n\n  /**\n   * Signal strength percentage reported by last query\n   */\n  int wireless_network::signal() const {\n    return m_signalstrength.percentage();\n  }\n\n  /**\n   * Link quality percentage reported by last query\n   */\n  int wireless_network::quality() const {\n    return m_linkquality.percentage();\n  }\n\n  /**\n   * Callback to parse scan results\n   */\n  int wireless_network::scan_cb(struct nl_msg* msg, void* instance) {\n    auto wn = static_cast<wireless_network*>(instance);\n    auto gnlh = static_cast<genlmsghdr*>(nlmsg_data(nlmsg_hdr(msg)));\n    struct nlattr* tb[NL80211_ATTR_MAX + 1];\n    struct nlattr* bss[NL80211_BSS_MAX + 1];\n\n    struct nla_policy bss_policy[NL80211_BSS_MAX + 1]{};\n    bss_policy[NL80211_BSS_TSF].type = NLA_U64;\n    bss_policy[NL80211_BSS_FREQUENCY].type = NLA_U32;\n    bss_policy[NL80211_BSS_BSSID].type = NLA_UNSPEC;\n    bss_policy[NL80211_BSS_BEACON_INTERVAL].type = NLA_U16;\n    bss_policy[NL80211_BSS_CAPABILITY].type = NLA_U16;\n    bss_policy[NL80211_BSS_INFORMATION_ELEMENTS].type = NLA_UNSPEC;\n    bss_policy[NL80211_BSS_SIGNAL_MBM].type = NLA_U32;\n    bss_policy[NL80211_BSS_SIGNAL_UNSPEC].type = NLA_U8;\n    bss_policy[NL80211_BSS_STATUS].type = NLA_U32;\n\n    if (nla_parse(tb, NL80211_ATTR_MAX, genlmsg_attrdata(gnlh, 0), genlmsg_attrlen(gnlh, 0), nullptr) < 0) {\n      return NL_SKIP;\n    }\n\n    if (tb[NL80211_ATTR_BSS] == nullptr) {\n      return NL_SKIP;\n    }\n\n    if (nla_parse_nested(bss, NL80211_BSS_MAX, tb[NL80211_ATTR_BSS], bss_policy) != 0) {\n      return NL_SKIP;\n    }\n\n    if (!wn->associated_or_joined(bss)) {\n      return NL_SKIP;\n    }\n\n    wn->parse_essid(bss);\n    wn->parse_frequency(bss);\n    wn->parse_signal(bss);\n    wn->parse_quality(bss);\n\n    return NL_SKIP;\n  }\n\n  /**\n   * Check for a connection to a AP\n   */\n  bool wireless_network::associated_or_joined(struct nlattr** bss) {\n    if (bss[NL80211_BSS_STATUS] == nullptr) {\n      return false;\n    }\n\n    auto status = nla_get_u32(bss[NL80211_BSS_STATUS]);\n\n    switch (status) {\n      case NL80211_BSS_STATUS_ASSOCIATED:\n      case NL80211_BSS_STATUS_IBSS_JOINED:\n      case NL80211_BSS_STATUS_AUTHENTICATED:\n        return true;\n      default:\n        return false;\n    }\n  }\n\n  /**\n   * Set the ESSID\n   */\n  void wireless_network::parse_essid(struct nlattr** bss) {\n    m_essid.clear();\n\n    if (bss[NL80211_BSS_INFORMATION_ELEMENTS] != nullptr) {\n// Information Element ID from ieee80211.h\n#define WLAN_EID_SSID 0\n\n      auto ies = static_cast<char*>(nla_data(bss[NL80211_BSS_INFORMATION_ELEMENTS]));\n      auto ies_len = nla_len(bss[NL80211_BSS_INFORMATION_ELEMENTS]);\n      const auto hdr_len = 2;\n\n      while (ies_len > hdr_len && ies[0] != WLAN_EID_SSID) {\n        ies_len -= ies[1] + hdr_len;\n        ies += ies[1] + hdr_len;\n      }\n\n      if (ies_len > hdr_len && ies_len > ies[1] + hdr_len) {\n        auto essid_begin = ies + hdr_len;\n        auto essid_end = essid_begin + ies[1];\n\n        std::copy(essid_begin, essid_end, std::back_inserter(m_essid));\n      }\n    }\n  }\n\n  /**\n   * Set frequency\n   */\n  void wireless_network::parse_frequency(struct nlattr** bss) {\n    if (bss[NL80211_BSS_FREQUENCY] != nullptr) {\n      // in MHz\n      m_frequency = static_cast<int>(nla_get_u32(bss[NL80211_BSS_FREQUENCY]));\n    }\n  }\n\n  /**\n   * Set device driver quality values\n   */\n  void wireless_network::parse_quality(struct nlattr** bss) {\n    if (bss[NL80211_BSS_SIGNAL_UNSPEC] != nullptr) {\n      // Signal strength in unspecified units, scaled to 0..100 (u8)\n      m_linkquality.val = nla_get_u8(bss[NL80211_BSS_SIGNAL_UNSPEC]);\n      m_linkquality.max = 100;\n    }\n  }\n\n  /**\n   * Set the signalstrength\n   */\n  void wireless_network::parse_signal(struct nlattr** bss) {\n    if (bss[NL80211_BSS_SIGNAL_MBM] != nullptr) {\n      // signalstrength in dBm\n      int signalstrength = static_cast<int>(nla_get_u32(bss[NL80211_BSS_SIGNAL_MBM])) / 100;\n\n      // WiFi-hardware usually operates in the range -90 to -20dBm.\n      const int hardware_max = -20;\n      const int hardware_min = -90;\n      signalstrength = std::max(hardware_min, std::min(signalstrength, hardware_max));\n\n      // Shift for positive values\n      m_signalstrength.val = signalstrength - hardware_min;\n      m_signalstrength.max = hardware_max - hardware_min;\n    }\n  }\n} // namespace net\n\nPOLYBAR_NS_END\n\n// Path: src/adapters/mpd.cpp\n#include <cassert>\n#include <csignal>\n#include <thread>\n#include <utility>\n\n#include \"adapters/mpd.hpp\"\n#include \"components/logger.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\n#define TRACE_BOOL(mode) m_log.trace(\"mpdconnection.%s: %s\", __func__, mode ? \"true\" : \"false\");\n\nnamespace mpd {\n  sig_atomic_t g_connection_closed = 0;\n  void g_mpd_signal_handler(int signum) {\n    if (signum == SIGPIPE) {\n      g_connection_closed = 1;\n    }\n  }\n\n  void check_connection(mpd_connection* conn) {\n    if (g_connection_closed) {\n      g_connection_closed = 0;\n      throw server_error(\"Connection closed (broken pipe)\");\n    } else if (conn == nullptr) {\n      throw client_error(\"Not connected to server\", MPD_ERROR_STATE);\n    }\n  }\n\n  void check_errors(mpd_connection* conn) {\n    check_connection(conn);\n\n    string err_msg;\n\n    switch (mpd_connection_get_error(conn)) {\n      case MPD_ERROR_SUCCESS:\n        return;\n      case MPD_ERROR_SERVER:\n        {\n          err_msg = mpd_connection_get_error_message(conn);\n          enum mpd_server_error server_err = mpd_connection_get_server_error(conn);\n          mpd_connection_clear_error(conn);\n          throw server_error(err_msg, server_err);\n        }\n      default:\n        {\n          err_msg = mpd_connection_get_error_message(conn);\n          enum mpd_error err = mpd_connection_get_error(conn);\n          mpd_connection_clear_error(conn);\n          throw client_error(err_msg, err);\n        }\n    }\n  }\n\n  // deleters {{{\n\n  namespace details {\n    void mpd_connection_deleter::operator()(mpd_connection* conn) {\n      if (conn != nullptr) {\n        mpd_connection_free(conn);\n      }\n    }\n\n    void mpd_status_deleter::operator()(mpd_status* status) {\n      mpd_status_free(status);\n    }\n\n    void mpd_song_deleter::operator()(mpd_song* song) {\n      mpd_song_free(song);\n    }\n  }\n\n  // }}}\n  // class: mpdsong {{{\n\n  mpdsong::operator bool() {\n    return m_song != nullptr;\n  }\n\n  string mpdsong::get_artist() {\n    assert(m_song);\n    auto tag = mpd_song_get_tag(m_song.get(), MPD_TAG_ARTIST, 0);\n    return string{tag != nullptr ? tag : \"\"};\n  }\n\n  string mpdsong::get_album_artist() {\n    assert(m_song);\n...\n", "needle_token_start": 8776, "needle_token_end": 8882, "code_context_ntokens": 16045, "output": ["```cpp\nstatic bool is_virtual(const std::string& ifname) {\n    char* target = realpath((NET_PATH + ifname).c_str(), nullptr);\n\n    if (!target) {\n      throw system_error(\"realpath\");\n    }\n\n    const std::string real_path{target};\n    free(target);\n    return real_path.rfind(VIRTUAL_PATH, 0) == 0;\n}\n```"]}
{"repo": "polybar/polybar", "name": "utf8_get_len", "language": "cpp", "path": "src/utils/string.cpp", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To determine the number of bytes used by a UTF-8 encoded codepoint based on its initial byte and to extract the significant bits from that byte.\n2. **Input**: A single byte (as an unsigned 8-bit integer), which is the leading byte of a UTF-8 encoded codepoint.\n3. **Output**: A pair consisting of two elements: the first is an integer representing the number of bytes that the codepoint occupies, and the second is a 32-bit unsigned integer containing the significant bits of the codepoint extracted from the leading byte.\n4. **Procedure**: The function checks the leading byte against specific bit masks to identify the format of the UTF-8 encoding (ranging from 1 to 4 bytes). Depending on the format, it extracts the significant bits by removing the prefix bits associated with that format. If the byte does not match any known UTF-8 prefix, it returns an error state.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/file.cpp\n#include \"utils/file.hpp\"\n\n#include <dirent.h>\n#include <fcntl.h>\n#include <glob.h>\n#include <sys/stat.h>\n#include <unistd.h>\n\n#include <cstdio>\n#include <cstdlib>\n#include <fstream>\n#include <streambuf>\n\n#include \"errors.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\n// implementation of file_descriptor {{{\n\nfile_descriptor::file_descriptor(const string& path, int flags, bool autoclose) : m_autoclose(autoclose) {\n  if ((m_fd = open(path.c_str(), flags)) == -1) {\n    throw system_error(\"Failed to open file descriptor\");\n  }\n}\n\nfile_descriptor::file_descriptor(int fd, bool autoclose) : m_fd(fd), m_autoclose(autoclose) {\n  if (m_fd != -1 && !*this) {\n    throw system_error(\"Given file descriptor (\" + to_string(m_fd) + \") is not valid\");\n  }\n}\n\nfile_descriptor::~file_descriptor() {\n  if (m_autoclose) {\n    close();\n  }\n}\n\nfile_descriptor& file_descriptor::operator=(const int fd) {\n  if (m_autoclose) {\n    close();\n  }\n  m_fd = fd;\n  return *this;\n}\n\nfile_descriptor::operator int() {\n  return static_cast<const file_descriptor&>(*this);\n}\nfile_descriptor::operator int() const {\n  return m_fd;\n}\n\nfile_descriptor::operator bool() {\n  return static_cast<const file_descriptor&>(*this);\n}\nfile_descriptor::operator bool() const {\n  errno = 0;  // reset since fcntl only changes it on error\n  if ((fcntl(m_fd, F_GETFD) == -1) || errno == EBADF) {\n    errno = EBADF;\n    return false;\n  }\n  return true;\n}\n\nvoid file_descriptor::close() {\n  if (m_fd != -1 && ::close(m_fd) == -1) {\n    throw system_error(\"Failed to close file descriptor\");\n  }\n  m_fd = -1;\n}\n\n// }}}\n// implementation of file_streambuf {{{\n\nfd_streambuf::~fd_streambuf() {\n  close();\n}\n\nfd_streambuf::operator int() {\n  return static_cast<const fd_streambuf&>(*this);\n}\nfd_streambuf::operator int() const {\n  return m_fd;\n}\n\nvoid fd_streambuf::open(int fd) {\n  if (m_fd) {\n    close();\n  }\n  m_fd = fd;\n  setg(m_in, m_in + BUFSIZE_IN, m_in + BUFSIZE_IN);\n  setp(m_out, m_out + BUFSIZE_OUT - 1);\n}\n\nvoid fd_streambuf::close() {\n  if (m_fd) {\n    sync();\n    m_fd = -1;\n  }\n}\n\nint fd_streambuf::sync() {\n  if (pbase() != pptr()) {\n    auto size = pptr() - pbase();\n    auto bytes = write(m_fd, m_out, size);\n    if (bytes > 0) {\n      std::copy(pbase() + bytes, pptr(), pbase());\n      setp(pbase(), epptr());\n      pbump(size - bytes);\n    }\n  }\n  return pptr() != epptr() ? 0 : -1;\n}\n\nint fd_streambuf::overflow(int c) {\n  if (!traits_type::eq_int_type(c, traits_type::eof())) {\n    *pptr() = traits_type::to_char_type(c);\n    pbump(1);\n  }\n  return sync() == -1 ? traits_type::eof() : traits_type::not_eof(c);\n...\n// Path: src/utils/command.cpp\n#include \"utils/command.hpp\"\n\n#include <sys/wait.h>\n#include <unistd.h>\n\n#include <csignal>\n#include <cstdlib>\n#include <utility>\n\n#include \"errors.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/io.hpp\"\n#include \"utils/process.hpp\"\n#include \"utils/string.hpp\"\n\n#ifndef STDOUT_FILENO\n#define STDOUT_FILENO 1\n#endif\n#ifndef STDERR_FILENO\n#define STDERR_FILENO 2\n#endif\n\nPOLYBAR_NS\n\ncommand<output_policy::IGNORED>::command(const logger& logger, string cmd) : m_log(logger), m_cmd(move(cmd)) {}\n\ncommand<output_policy::IGNORED>::~command() {\n  terminate();\n}\n\n/**\n * Execute the command\n */\nint command<output_policy::IGNORED>::exec(bool wait_for_completion) {\n  m_forkpid = process_util::spawn_async([m_cmd = m_cmd] { process_util::exec_sh(m_cmd.c_str()); });\n  if (wait_for_completion) {\n    auto status = wait();\n    m_forkpid = -1;\n    return status;\n  }\n\n  return EXIT_SUCCESS;\n}\n\nvoid command<output_policy::IGNORED>::terminate() {\n  if (is_running()) {\n    m_log.trace(\"command: Sending SIGTERM to running child process (%d)\", m_forkpid);\n    killpg(m_forkpid, SIGTERM);\n    wait();\n  }\n  m_forkpid = -1;\n}\n\n/**\n * Check if command is running\n */\nbool command<output_policy::IGNORED>::is_running() {\n  return m_forkpid > 0 && process_util::wait_for_completion_nohang(m_forkpid, &m_forkstatus) > -1;\n}\n\n/**\n * Wait for the child process to finish\n */\nint command<output_policy::IGNORED>::wait() {\n  do {\n    m_log.trace(\"command: Waiting for pid %d to finish...\", m_forkpid);\n\n    process_util::wait_for_completion(m_forkpid, &m_forkstatus, WCONTINUED | WUNTRACED);\n\n    if (WIFEXITED(m_forkstatus) && m_forkstatus > 0) {\n      m_log.trace(\"command: Exited with failed status %d\", WEXITSTATUS(m_forkstatus));\n    } else if (WIFEXITED(m_forkstatus)) {\n      m_log.trace(\"command: Exited with status %d\", WEXITSTATUS(m_forkstatus));\n    } else if (WIFSIGNALED(m_forkstatus)) {\n      m_log.trace(\"command: killed by signal %d\", WTERMSIG(m_forkstatus));\n    } else if (WIFSTOPPED(m_forkstatus)) {\n      m_log.trace(\"command: Stopped by signal %d\", WSTOPSIG(m_forkstatus));\n    } else if (WIFCONTINUED(m_forkstatus)) {\n      m_log.trace(\"command: Continued\");\n    } else {\n      break;\n    }\n  } while (!WIFEXITED(m_forkstatus) && !WIFSIGNALED(m_forkstatus));\n\n  return WEXITSTATUS(m_forkstatus);\n}\n\n/**\n * Get command pid\n */\npid_t command<output_policy::IGNORED>::get_pid() {\n  return m_forkpid;\n}\n\n/**\n * Get command exit status\n */\nint command<output_policy::IGNORED>::get_exit_status() {\n  return WEXITSTATUS(m_forkstatus);\n}\n\ncommand<output_policy::REDIRECTED>::command(const polybar::logger& logger, std::string cmd)\n    : command<output_policy::IGNORED>(logger, move(cmd)) {\n  if (pipe(m_stdin) != 0) {\n    throw command_error(\"Failed to allocate input stream\");\n  }\n  if (pipe(m_stdout) != 0) {\n    throw command_error(\"Failed to allocate output stream\");\n  }\n}\n\ncommand<output_policy::REDIRECTED>::~command() {\n  if (m_stdin[PIPE_READ] > 0) {\n    close(m_stdin[PIPE_READ]);\n  }\n  if (m_stdin[PIPE_WRITE] > 0) {\n    close(m_stdin[PIPE_WRITE]);\n  }\n  if (m_stdout[PIPE_READ] > 0) {\n    close(m_stdout[PIPE_READ]);\n  }\n  if (m_stdout[PIPE_WRITE] > 0) {\n    close(m_stdout[PIPE_WRITE]);\n  }\n}\n\n/**\n * Execute the command\n */\nint command<output_policy::REDIRECTED>::exec(bool wait_for_completion, const vector<pair<string, string>>& env) {\n  if ((m_forkpid = fork()) == -1) {\n    throw system_error(\"Failed to fork process\");\n  }\n\n  if (process_util::in_forked_process(m_forkpid)) {\n    if (dup2(m_stdin[PIPE_READ], STDIN_FILENO) == -1) {\n      throw command_error(\"Failed to redirect stdin in child process\");\n    }\n    if (dup2(m_stdout[PIPE_WRITE], STDOUT_FILENO) == -1) {\n      throw command_error(\"Failed to redirect stdout in child process\");\n    }\n    if (dup2(m_stdout[PIPE_WRITE], STDERR_FILENO) == -1) {\n      throw command_error(\"Failed to redirect stderr in child process\");\n    }\n\n    // Close file descriptors that won't be used by the child\n    if ((m_stdin[PIPE_READ] = close(m_stdin[PIPE_READ])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdin[PIPE_WRITE] = close(m_stdin[PIPE_WRITE])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdout[PIPE_READ] = close(m_stdout[PIPE_READ])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdout[PIPE_WRITE] = close(m_stdout[PIPE_WRITE])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n\n    setpgid(m_forkpid, 0);\n    process_util::exec_sh(m_cmd.c_str(), env);\n  } else {\n    // Close file descriptors that won't be used by the parent\n    if ((m_stdin[PIPE_READ] = close(m_stdin[PIPE_READ])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n    if ((m_stdout[PIPE_WRITE] = close(m_stdout[PIPE_WRITE])) == -1) {\n      throw command_error(\"Failed to close fd\");\n    }\n\n    if (wait_for_completion) {\n      auto status = wait();\n      m_forkpid = -1;\n      return status;\n    }\n  }\n\n  return EXIT_SUCCESS;\n}\n\n/**\n * Tail command output\n *\n * @note: This is a blocking call and will not\n * end until the stream is closed\n */\nvoid command<output_policy::REDIRECTED>::tail(std::function<void(string)> cb) {\n  io_util::tail(get_stdout(PIPE_READ), cb);\n}\n\n/**\n * Read a line from the commands output stream\n */\nstring command<output_policy::REDIRECTED>::readline() {\n  if (!m_stdout_reader) {\n    m_stdout_reader = make_unique<fd_stream<std::istream>>(get_stdout(PIPE_READ), false);\n  }\n\n  string s;\n  std::getline(*m_stdout_reader, s);\n  return s;\n}\n\n/**\n * Get command output channel\n */\nint command<output_policy::REDIRECTED>::get_stdout(int c) {\n  return m_stdout[c];\n}\n\n/**\n * Get command input channel\n */\nint command<output_policy::REDIRECTED>::get_stdin(int c) {\n  return m_stdin[c];\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/inotify.cpp\n#include \"utils/inotify.hpp\"\n\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"utils/memory.hpp\"\n\nPOLYBAR_NS\n\n/**\n * Construct inotify watch\n */\ninotify_watch::inotify_watch(string path) : m_path(move(path)) {}\n\n/**\n * Deconstruct inotify watch\n */\ninotify_watch::~inotify_watch() {\n  if (m_wd != -1) {\n    inotify_rm_watch(m_fd, m_wd);\n  }\n  if (m_fd != -1) {\n    close(m_fd);\n  }\n}\n\ninotify_watch::inotify_watch(inotify_watch&& other) noexcept {\n  std::swap(m_path, other.m_path);\n  std::swap(m_wd, other.m_wd);\n  std::swap(m_fd, other.m_fd);\n  std::swap(m_mask, other.m_mask);\n}\n\ninotify_watch& inotify_watch::operator=(inotify_watch&& other) noexcept {\n  std::swap(m_path, other.m_path);\n  std::swap(m_wd, other.m_wd);\n  std::swap(m_fd, other.m_fd);\n  std::swap(m_mask, other.m_mask);\n  return *this;\n}\n\n/**\n * Attach inotify watch\n */\nvoid inotify_watch::attach(int mask) {\n  if (m_fd == -1 && (m_fd = inotify_init()) == -1) {\n    throw system_error(\"Failed to allocate inotify fd\");\n  }\n  if ((m_wd = inotify_add_watch(m_fd, m_path.c_str(), mask)) == -1) {\n    throw system_error(\"Failed to attach inotify watch\");\n  }\n  m_mask |= mask;\n}\n\n/**\n * Remove inotify watch\n */\nvoid inotify_watch::remove(bool force) {\n  if (inotify_rm_watch(m_fd, m_wd) == -1 && !force) {\n    throw system_error(\"Failed to remove inotify watch\");\n  }\n  m_wd = -1;\n  m_mask = 0;\n}\n\n/**\n * Poll the inotify fd for events\n *\n * @brief A wait_ms of -1 blocks until an event is fired\n */\nbool inotify_watch::poll(int wait_ms) const {\n  if (m_fd == -1) {\n    return false;\n  }\n\n  struct pollfd fds[1];\n  fds[0].fd = m_fd;\n  fds[0].events = POLLIN;\n\n  ::poll(fds, 1, wait_ms);\n\n  return fds[0].revents & POLLIN;\n}\n\n/**\n * Get the latest inotify event\n */\ninotify_event inotify_watch::get_event() const {\n  inotify_event event;\n\n  if (m_fd == -1 || m_wd == -1) {\n    return event;\n  }\n\n  event.is_valid = true;\n\n  char buffer[1024];\n  auto bytes = read(m_fd, buffer, 1024);\n  auto len = 0;\n\n  while (len < bytes) {\n    auto* e = reinterpret_cast<::inotify_event*>(&buffer[len]);\n\n    event.filename = e->len ? e->name : m_path;\n    event.wd = e->wd;\n    event.cookie = e->cookie;\n    event.is_dir = e->mask & IN_ISDIR;\n    event.mask |= e->mask;\n\n    len += sizeof(*e) + e->len;\n  }\n\n  return event;\n}\n\n/**\n * Get watch file path\n */\nstring inotify_watch::path() const {\n  return m_path;\n}\n\n/**\n * Get the file descriptor associated with the watch\n */\nint inotify_watch::get_file_descriptor() const {\n  return m_fd;\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/restack.cpp\n#include \"utils/restack.hpp\"\n\nPOLYBAR_NS\n\nnamespace restack_util {\n\n/**\n * Restacks the given window relative to a given sibling with some stack order (above, below)\n *\n * Both windows need to be siblings (have the same parent window).\n *\n * @param win The window to restack\n * @param sibling The window relative to which restacking happens\n * @param stack_mode The restacking mode (above, below)\n * @throw Some xpp error if restacking fails\n */\nvoid restack_relative(connection& conn, xcb_window_t win, xcb_window_t sibling, xcb_stack_mode_t stack_mode) {\n  const unsigned int value_mask = XCB_CONFIG_WINDOW_SIBLING | XCB_CONFIG_WINDOW_STACK_MODE;\n  const std::array<uint32_t, 2> value_list = {sibling, stack_mode};\n  conn.configure_window_checked(win, value_mask, value_list.data());\n}\n\nstring stack_mode_to_string(xcb_stack_mode_t mode) {\n  switch (mode) {\n    case XCB_STACK_MODE_ABOVE:\n      return \"ABOVE\";\n    case XCB_STACK_MODE_BELOW:\n      return \"BELOW\";\n    case XCB_STACK_MODE_TOP_IF:\n      return \"TOP_IF\";\n    case XCB_STACK_MODE_BOTTOM_IF:\n      return \"BOTTOM_IF\";\n    case XCB_STACK_MODE_OPPOSITE:\n      return \"OPPOSITE\";\n  }\n\n  return \"UNKNOWN\";\n}\n\n/**\n * @return true iff the two given windows are sibings (are different windows and have same parent).\n */\nbool are_siblings(connection& conn, xcb_window_t win, xcb_window_t sibling) {\n  if (win == XCB_NONE || sibling == XCB_NONE || win == sibling) {\n    return false;\n  }\n\n  auto win_tree = conn.query_tree(win);\n  auto sibling_tree = conn.query_tree(sibling);\n  return win_tree->parent == sibling_tree->parent;\n}\n\n/**\n * \"bottom\" restack strategy.\n *\n * Moves the bar window to the bottom of the window stack\n */\nparams get_bottom_params(connection& conn, xcb_window_t bar_window) {\n  auto children = conn.query_tree(conn.root()).children();\n  if (children.begin() != children.end() && *children.begin() != bar_window) {\n    return {*children.begin(), XCB_STACK_MODE_BELOW};\n  }\n\n  return NONE_PARAMS;\n}\n\n/**\n * EWMH restack strategy.\n *\n * Moves the bar window above the WM meta window (_NET_SUPPORTING_WM_CHECK).\n * This window is generally towards the bottom of the window stack, but still above other windows that could interfere.\n *\n * @see ewmh_util::get_ewmh_meta_window\n */\nparams get_ewmh_params(connection& conn) {\n  if (auto meta_window = ewmh_util::get_ewmh_meta_window(conn.root())) {\n    return {meta_window, XCB_STACK_MODE_ABOVE};\n  }\n\n  return NONE_PARAMS;\n}\n\n/**\n * Generic restack stratgey.\n *\n * Tries to provide the best WM-agnostic restacking.\n *\n * Currently tries to the following stratgies in order:\n * * ewmh\n * * bottom\n */\nparams get_generic_params(connection& conn, xcb_window_t bar_window) {\n  auto ewmh_params = get_ewmh_params(conn);\n\n  if (are_siblings(conn, bar_window, ewmh_params.first)) {\n    return ewmh_params;\n  }\n\n  auto bottom_params = get_bottom_params(conn, bar_window);\n\n  if (are_siblings(conn, bar_window, bottom_params.first)) {\n    return bottom_params;\n  }\n\n  return NONE_PARAMS;\n}\n} // namespace restack_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/color.cpp\n#include \"utils/color.hpp\"\n\n#include <algorithm>\n\nPOLYBAR_NS\n\n/**\n * Takes a hex string as input and brings it into a normalized form\n *\n * The input can either be only an alpha channel #AA\n * or any of these forms: #RGB, #ARGB, #RRGGBB, #AARRGGBB\n *\n * Colors without alpha channel will get an alpha channel of FF\n * The input does not have to start with '#'\n *\n * @returns Empty string for malformed input, either AA for the alpha only\n * input or an 8 character string of the expanded form AARRGGBB\n */\nstatic string normalize_hex(string hex) {\n  if (hex.length() == 0) {\n    return \"\";\n  }\n\n  // We remove the hash because it makes processing easier\n  if (hex[0] == '#') {\n    hex.erase(0, 1);\n  }\n\n  // Check that only valid characters are used\n  if (!std::all_of(hex.cbegin(), hex.cend(), isxdigit)) {\n    return \"\";\n  }\n\n  if (hex.length() == 2) {\n    // We only have an alpha channel\n    return hex;\n  }\n\n  if (hex.length() == 3) {\n    // RGB -> FRGB\n    hex.insert(0, 1, 'f');\n  }\n\n  if (hex.length() == 4) {\n    // ARGB -> AARRGGBB\n    hex = {hex[0], hex[0], hex[1], hex[1], hex[2], hex[2], hex[3], hex[3]};\n  }\n\n  if (hex.length() == 6) {\n    // RRGGBB -> FFRRGGBB\n    hex.insert(0, 2, 'f');\n  }\n\n  if (hex.length() != 8) {\n    return \"\";\n  }\n\n  return hex;\n}\n\nrgba::rgba() : m_value(0), m_type(type::NONE) {}\nrgba::rgba(uint32_t value, enum type type) : m_value(value), m_type(type) {}\nrgba::rgba(string hex) {\n  hex = normalize_hex(hex);\n\n  if (hex.length() == 0) {\n    m_value = 0;\n    m_type = type::NONE;\n  } else if (hex.length() == 2) {\n    m_value = std::strtoul(hex.c_str(), nullptr, 16) << 24;\n    m_type = type::ALPHA_ONLY;\n  } else {\n    m_value = std::strtoul(hex.c_str(), nullptr, 16);\n    m_type = type::ARGB;\n  }\n}\n\nrgba::operator string() const {\n  char s[10];\n  size_t len = snprintf(s, 10, \"#%08x\", m_value);\n  return string(s, len);\n}\n\nbool rgba::operator==(const rgba& other) const {\n  if (m_type != other.m_type) {\n    return false;\n  }\n\n  switch (m_type) {\n    case type::NONE:\n      return true;\n    case type::ARGB:\n      return m_value == other.m_value;\n    case type::ALPHA_ONLY:\n      return alpha_i() == other.alpha_i();\n    default:\n      return false;\n  }\n}\n\nbool rgba::operator!=(const rgba& other) const {\n  return !(*this == other);\n}\n\nrgba::operator uint32_t() const {\n  return m_value;\n}\n\nrgba::operator bool() const {\n  return has_color();\n}\n\nuint32_t rgba::value() const {\n  return this->m_value;\n}\n\nenum rgba::type rgba::get_type() const {\n  return m_type;\n}\n\ndouble rgba::alpha_d() const {\n  return alpha_i() / 255.0;\n}\n\ndouble rgba::red_d() const {\n  return red_i() / 255.0;\n}\n\ndouble rgba::green_d() const {\n  return green_i() / 255.0;\n}\n\ndouble rgba::blue_d() const {\n  return blue_i() / 255.0;\n}\n\nuint8_t rgba::alpha_i() const {\n  return (m_value >> 24) & 0xFF;\n}\n\nuint8_t rgba::red_i() const {\n  return (m_value >> 16) & 0xFF;\n}\n\nuint8_t rgba::green_i() const {\n  return (m_value >> 8) & 0xFF;\n}\n\nuint8_t rgba::blue_i() const {\n  return m_value & 0xFF;\n}\n\nbool rgba::has_color() const {\n  return m_type != type::NONE;\n}\n\nbool rgba::is_transparent() const {\n  return alpha_i() != 0xFF;\n}\n\n/**\n * Applies the alpha channel of this color to the given color.\n */\nrgba rgba::apply_alpha_to(rgba other) const {\n  uint32_t val = (other.value() & 0x00FFFFFF) | (((uint32_t)alpha_i()) << 24);\n  return rgba(val, m_type);\n}\n\n/**\n * If this is an ALPHA_ONLY color, applies this alpha channel to the other\n * color, otherwise just returns this.\n *\n * @returns the new color if this is ALPHA_ONLY or a copy of this otherwise.\n */\nrgba rgba::try_apply_alpha_to(rgba other) const {\n  if (m_type == type::ALPHA_ONLY) {\n    return apply_alpha_to(other);\n  }\n\n  return *this;\n}\n\nstring color_util::simplify_hex(string hex) {\n  // convert #ffrrggbb to #rrggbb\n  if (hex.length() == 9 && std::toupper(hex[1]) == 'F' && std::toupper(hex[2]) == 'F') {\n    hex.erase(1, 2);\n  }\n\n  // convert #rrggbb to #rgb\n  if (hex.length() == 7) {\n    if (hex[1] == hex[2] && hex[3] == hex[4] && hex[5] == hex[6]) {\n      hex = {'#', hex[1], hex[3], hex[5]};\n    }\n  }\n\n  return hex;\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/http.cpp\n#include \"utils/http.hpp\"\n\n#include <curl/curl.h>\n#include <curl/easy.h>\n\n#include <sstream>\n\n#include \"errors.hpp\"\n#include \"settings.hpp\"\n\nPOLYBAR_NS\n\nhttp_downloader::http_downloader(int connection_timeout) {\n  m_curl = curl_easy_init();\n  curl_easy_setopt(m_curl, CURLOPT_ACCEPT_ENCODING, \"deflate\");\n  curl_easy_setopt(m_curl, CURLOPT_CONNECTTIMEOUT, connection_timeout);\n  curl_easy_setopt(m_curl, CURLOPT_FOLLOWLOCATION, true);\n  curl_easy_setopt(m_curl, CURLOPT_NOSIGNAL, true);\n  curl_easy_setopt(m_curl, CURLOPT_USERAGENT, (\"polybar/\" + string{APP_VERSION}).c_str());\n  curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, http_downloader::write);\n  curl_easy_setopt(m_curl, CURLOPT_FORBID_REUSE, true);\n}\n\nhttp_downloader::~http_downloader() {\n  curl_easy_cleanup(m_curl);\n}\n\nstring http_downloader::get(const string& url, const string& user, const string& password) {\n  std::stringstream out{};\n  curl_easy_setopt(m_curl, CURLOPT_URL, url.c_str());\n  curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, &out);\n  if (!user.empty()) {\n    curl_easy_setopt(m_curl, CURLOPT_USERNAME, user.c_str());\n  }\n  if (!password.empty()) {\n    curl_easy_setopt(m_curl, CURLOPT_PASSWORD, password.c_str());\n  }\n\n  auto res = curl_easy_perform(m_curl);\n  if (res != CURLE_OK) {\n    throw application_error(curl_easy_strerror(res), res);\n  }\n\n  return out.str();\n}\n\nlong http_downloader::response_code() {\n  long code{0};\n  curl_easy_getinfo(m_curl, CURLINFO_RESPONSE_CODE, &code);\n  return code;\n}\n\nsize_t http_downloader::write(void* p, size_t size, size_t bytes, void* stream) {\n  string data{static_cast<const char*>(p), size * bytes};\n  *(static_cast<std::stringstream*>(stream)) << data << '\\n';\n  return size * bytes;\n}\n\nPOLYBAR_NS_END\n\n// Path: src/utils/string.cpp\n#include \"utils/string.hpp\"\n\n#include <algorithm>\n#include <cassert>\n#include <iomanip>\n#include <sstream>\n#include <utility>\n\nPOLYBAR_NS\n\nnamespace string_util {\n\n/**\n * Prefixes for the leading byte in a UTF8 codepoint\n */\nstatic constexpr uint8_t UTF8_LEADING1_PREFIX = 0b00000000;\nstatic constexpr uint8_t UTF8_LEADING2_PREFIX = 0b11000000;\nstatic constexpr uint8_t UTF8_LEADING3_PREFIX = 0b11100000;\nstatic constexpr uint8_t UTF8_LEADING4_PREFIX = 0b11110000;\n\n/**\n * Masks to extract the prefix from the leading byte in a UTF8 codepoint\n */\nstatic constexpr uint8_t UTF8_LEADING1_MASK = 0b10000000;\nstatic constexpr uint8_t UTF8_LEADING2_MASK = 0b11100000;\nstatic constexpr uint8_t UTF8_LEADING3_MASK = 0b11110000;\nstatic constexpr uint8_t UTF8_LEADING4_MASK = 0b11111000;\n\n/**\n * Prefix for UTF8 continuation bytes\n */\nstatic constexpr uint8_t UTF8_CONTINUATION_PREFIX = 0b10000000;\n/**\n * Mask to extract the UTF8 continuation byte prefix\n */\nstatic constexpr uint8_t UTF8_CONTINUATION_MASK = 0b11000000;\n\n/**\n * Check if haystack contains needle\n */\nbool contains(const string& haystack, const string& needle) {\n  return haystack.find(needle) != string::npos;\n}\n\nbool ends_with(const string& haystack, const string& suffix) {\n  if (haystack.length() < suffix.length()) {\n    return false;\n  }\n\n  return haystack.compare(haystack.length() - suffix.length(), suffix.length(), suffix) == 0;\n}\n\n/**\n * Check if haystack contains needle ignoring case\n */\nbool contains_ignore_case(const string& haystack, const string& needle) {\n  return lower(haystack).find(lower(needle)) != string::npos;\n}\n\n/**\n * Convert string to uppercase\n */\nstring upper(const string& s) {\n  string str(s);\n  for (auto& c : str) {\n    c = toupper(c);\n  }\n  return str;\n}\n\n/**\n * Convert string to lowercase\n */\nstring lower(const string& s) {\n  string str(s);\n  for (auto& c : str) {\n    c = tolower(c);\n  }\n  return str;\n}\n\n/**\n * Test lower case equality\n */\nbool compare(const string& s1, const string& s2) {\n  return lower(s1) == lower(s2);\n}\n\n/**\n * Replace first occurrence of needle in haystack\n */\nstring replace(const string& haystack, const string& needle, const string& replacement, size_t start, size_t end) {\n  string str(haystack);\n  string::size_type pos;\n\n  if (needle != replacement && (pos = str.find(needle, start)) != string::npos) {\n    if (end == string::npos || pos < end) {\n      str = str.replace(pos, needle.length(), replacement);\n    }\n  }\n\n  return str;\n}\n\n/**\n * Replace all occurrences of needle in haystack\n */\nstring replace_all(const string& haystack, const string& needle, const string& replacement, size_t start, size_t end) {\n  string result{haystack};\n  string::size_type pos;\n  while ((pos = result.find(needle, start)) != string::npos && pos < result.length() &&\n         (end == string::npos || pos + needle.length() <= end)) {\n    result.replace(pos, needle.length(), replacement);\n    start = pos + replacement.length();\n  }\n  return result;\n}\n\n/**\n * Replace all consecutive occurrences of needle in haystack\n */\nstring squeeze(const string& haystack, char needle) {\n  string result = haystack;\n  while (result.find({needle, needle}) != string::npos) {\n    result = replace_all(result, {needle, needle}, {needle});\n  }\n  return result;\n}\n\n/**\n * Remove all occurrences of needle in haystack\n */\nstring strip(const string& haystack, char needle) {\n  string str(haystack);\n  string::size_type pos;\n  while ((pos = str.find(needle)) != string::npos) {\n    str.erase(pos, 1);\n  }\n  return str;\n}\n\n/**\n * Remove trailing newline\n */\nstring strip_trailing_newline(const string& haystack) {\n  string str(haystack);\n  if (str[str.length() - 1] == '\\n') {\n    str.erase(str.length() - 1, 1);\n  }\n  return str;\n}\n\n/**\n * Trims all characters that match pred from the left\n */\nstring ltrim(string value, function<bool(char)> pred) {\n  value.erase(value.begin(), find_if(value.begin(), value.end(), std::not_fn(pred)));\n  return value;\n}\n\n/**\n * Trims all characters that match pred from the right\n */\nstring rtrim(string value, function<bool(char)> pred) {\n  value.erase(find_if(value.rbegin(), value.rend(), std::not_fn(pred)).base(), value.end());\n  return value;\n}\n\n/**\n * Trims all characters that match pred from both sides\n */\nstring trim(string value, function<bool(char)> pred) {\n  return ltrim(rtrim(move(value), pred), pred);\n}\n\n/**\n * Remove needle from the start of the string\n */\nstring ltrim(string&& value, const char& needle) {\n  if (value.empty()) {\n    return \"\";\n  }\n  while (*value.begin() == needle) {\n    value.erase(0, 1);\n  }\n  return forward<string>(value);\n}\n\n/**\n * Remove needle from the end of the string\n */\nstring rtrim(string&& value, const char& needle) {\n  if (value.empty()) {\n    return \"\";\n  }\n  while (*(value.end() - 1) == needle) {\n    value.erase(value.length() - 1, 1);\n  }\n  return forward<string>(value);\n}\n\n/**\n * Remove needle from the start and end of the string\n */\nstring trim(string&& value, const char& needle) {\n  if (value.empty()) {\n    return \"\";\n  }\n  return rtrim(ltrim(forward<string>(value), needle), needle);\n}\n\n/**\n * Counts the number of codepoints in a utf8 encoded string.\n */\nsize_t char_len(const string& value) {\n  // utf-8 bytes of the form 10xxxxxx are continuation bytes, so we\n  // simply count the number of bytes not of this form.\n  return std::count_if(\n      value.begin(), value.end(), [](char c) { return (c & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_PREFIX; });\n}\n\n/**\n * Truncates a utf8 string at len number of codepoints. This isn't 100%\n * matching the user-perceived character count, but it should be close\n * enough and avoids having to pull in something like ICU to count actual\n * grapheme clusters.\n */\nstring utf8_truncate(string&& value, size_t len) {\n  if (value.empty()) {\n    return \"\";\n  }\n\n  // utf-8 bytes of the form 10xxxxxx are continuation bytes, so we\n  // simply jump forward to bytes not of that form and truncate starting\n  // at that byte if we've counted too many codepoints\n  auto it = value.begin();\n  auto end = value.end();\n  for (size_t i = 0; i < len; ++i) {\n    if (it == end)\n      break;\n    ++it;\n    it = std::find_if(it, end, [](char c) { return (c & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_PREFIX; });\n  }\n  value.erase(it, end);\n\n  return forward<string>(value);\n}\n\n/**\n * Given a leading byte of a UTF8 codepoint calculates the number of bytes taken up by the codepoint.\n *\n * @returns {len, result} The codepoint is len bytes and result contains the codepoint bits held in the leading byte.\n */\n\nstatic pair<int, uint32_t> utf8_get_len(uint8_t leading) {\n  if ((leading & UTF8_LEADING1_MASK) == UTF8_LEADING1_PREFIX) {\n    return {1, leading & ~UTF8_LEADING1_MASK};\n  } else if ((leading & UTF8_LEADING2_MASK) == UTF8_LEADING2_PREFIX) {\n    return {2, leading & ~UTF8_LEADING2_MASK};\n  } else if ((leading & UTF8_LEADING3_MASK) == UTF8_LEADING3_PREFIX) {\n    return {3, leading & ~UTF8_LEADING3_MASK};\n  } else if ((leading & UTF8_LEADING4_MASK) == UTF8_LEADING4_PREFIX) {\n    return {4, leading & ~UTF8_LEADING4_MASK};\n  } else {\n    return {-1, 0};\n  }\n}\n\n/**\n * @brief Create a list of UCS-4 codepoint from a utf-8 encoded string\n *\n * If invalid utf8 characters are encountered they are skipped until the next valid codepoint and the function will\n * eventually return false.\n *\n * The result_list is always populated with all valid utf8 codepoints.\n *\n * @return Whether the string is completely valid utf8\n */\nbool utf8_to_ucs4(const string& src, unicode_charlist& result_list) {\n  result_list.reserve(src.size());\n  bool has_errors = false;\n  const auto* begin = reinterpret_cast<const uint8_t*>(src.c_str());\n\n  const auto* current = begin;\n  while (*current) {\n    // Number of bytes taken up by this codepoint and the bits contained in the leading byte.\n    auto [len, result] = utf8_get_len(*current);\n    auto offset = current - begin;\n\n    /*\n     * Invalid lengths, this byte is not a valid leading byte.\n     * Skip it.\n     */\n    if (len <= 0 || len > 4) {\n      has_errors = true;\n      current++;\n      continue;\n    }\n\n    const uint8_t* next = current + 1;\n    for (; ((*next & UTF8_CONTINUATION_MASK) == UTF8_CONTINUATION_PREFIX) && (next - current < len); next++) {\n      result = result << 6;\n      result |= *next & ~UTF8_CONTINUATION_MASK;\n    }\n\n    auto actual_len = next - current;\n    current = next;\n\n    if (actual_len != len) {\n      has_errors = true;\n      continue;\n    }\n\n    result_list.push_back(unicode_character{result, static_cast<int>(offset), static_cast<int>(actual_len)});\n    current = next;\n  }\n\n  return !has_errors;\n}\n\n/**\n * @brief Convert a UCS-4 codepoint to a utf-8 encoded string\n */\nsize_t ucs4_to_utf8(std::array<char, 5>& utf8, uint32_t ucs) {\n  if (ucs <= 0x7f) {\n    utf8[0] = ucs;\n    return 1;\n  } else if (ucs <= 0x07ff) {\n    utf8[0] = ((ucs >> 6) & ~UTF8_LEADING2_MASK) | UTF8_LEADING2_PREFIX;\n    utf8[1] = (ucs & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    return 2;\n  } else if (ucs <= 0xffff) {\n    utf8[0] = ((ucs >> 12) & ~UTF8_LEADING3_MASK) | UTF8_LEADING3_PREFIX;\n    utf8[1] = ((ucs >> 6) & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    utf8[2] = (ucs & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    return 3;\n  } else if (ucs <= 0x10ffff) {\n    utf8[0] = ((ucs >> 18) & ~UTF8_LEADING4_MASK) | UTF8_LEADING4_PREFIX;\n    utf8[1] = ((ucs >> 12) & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    utf8[2] = ((ucs >> 6) & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    utf8[3] = (ucs & ~UTF8_CONTINUATION_MASK) | UTF8_CONTINUATION_PREFIX;\n    return 4;\n  } else {\n    return 0;\n  }\n}\n\n/**\n * Join all strings in vector into a single string separated by delim\n */\nstring join(const vector<string>& strs, const string& delim) {\n  string str;\n  for (auto& s : strs) {\n    str += (str.empty() ? \"\" : delim) + s;\n  }\n  return str;\n}\n\n/**\n * Explode string by delim, ignore empty tokens\n */\nvector<string> split(const string& s, char delim) {\n  std::string::size_type pos = 0;\n  std::vector<std::string> result;\n\n  while ((pos = s.find_first_not_of(delim, pos)) != std::string::npos) {\n    auto nextpos = s.find_first_of(delim, pos);\n    result.emplace_back(s.substr(pos, nextpos - pos));\n    pos = nextpos;\n  }\n\n  return result;\n}\n\n/**\n * Explode string by delim, include empty tokens\n */\nstd::vector<std::string> tokenize(const string& str, char delimiters) {\n  std::vector<std::string> tokens;\n  std::string::size_type lastPos = 0;\n  auto pos = str.find_first_of(delimiters, lastPos);\n\n  while (pos != std::string::npos && lastPos != std::string::npos) {\n    tokens.emplace_back(str.substr(lastPos, pos - lastPos));\n\n    lastPos = pos + 1;\n    pos = str.find_first_of(delimiters, lastPos);\n  }\n\n  tokens.emplace_back(str.substr(lastPos, pos - lastPos));\n  return tokens;\n}\n\n/**\n * Find the nth occurrence of needle in haystack starting from pos\n */\nsize_t find_nth(const string& haystack, size_t pos, const string& needle, size_t nth) {\n  size_t found_pos = haystack.find(needle, pos);\n  if (1 == nth || string::npos == found_pos) {\n    return found_pos;\n  }\n  return find_nth(haystack, found_pos + 1, needle, nth - 1);\n}\n\n/**\n * Create a floating point string\n */\nstring floating_point(double value, size_t precision, bool fixed, const string& locale) {\n  std::stringstream ss;\n  ss.imbue(!locale.empty() ? std::locale(locale.c_str()) : std::locale::classic());\n  ss << std::fixed << std::setprecision(precision) << value;\n  return fixed ? ss.str() : replace(ss.str(), \".00\", \"\");\n}\n\n/**\n * Create a MiB filesize string\n */\nstring filesize_mib(unsigned long long kibibytes, size_t precision, const string& locale) {\n  return floating_point(kibibytes / 1024.0, precision, true, locale) + \" MiB\";\n}\n\n/**\n * Create a GiB filesize string\n */\nstring filesize_gib(unsigned long long kibibytes, size_t precision, const string& locale) {\n  return floating_point(kibibytes / 1024.0 / 1024.0, precision, true, locale) + \" GiB\";\n}\n\n/**\n * Create a GiB string, if the value in GiB is >= 1.0. Otherwise, create a MiB string.\n */\nstring filesize_gib_mib(\n    unsigned long long kibibytes, size_t precision_mib, size_t precision_gib, const string& locale) {\n  if (kibibytes < 1024 * 1024) {\n    return filesize_mib(kibibytes, precision_mib, locale);\n  } else {\n    return filesize_gib(kibibytes, precision_gib, locale);\n  }\n}\n\n/**\n * Create a filesize string by converting given bytes to highest unit possible\n */\nstring filesize(unsigned long long bytes, size_t precision, bool fixed, const string& locale) {\n  vector<string> suffixes{\"TB\", \"GB\", \"MB\", \"KB\"};\n  string suffix{\"B\"};\n  double value = bytes;\n  while (!suffixes.empty() && value >= 1024.0) {\n    suffix = suffixes.back();\n    suffixes.pop_back();\n    value /= 1024.0;\n  }\n  return floating_point(value, precision, fixed, locale) + \" \" + suffix;\n}\n\n/**\n * Compute string hash\n */\nhash_type hash(const string& src) {\n  return std::hash<string>()(src);\n}\n} // namespace string_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/concurrency.cpp\n#include \"utils/concurrency.hpp\"\n\n#include <map>\n\nPOLYBAR_NS\n\nnamespace concurrency_util {\n  size_t thread_id(const thread::id id) {\n    static size_t idx{1_z};\n    static mutex_wrapper<std::map<thread::id, size_t>> ids;\n    std::lock_guard<decltype(ids)> lock(ids);\n    if (ids.find(id) == ids.end()) {\n      ids[id] = idx++;\n    }\n    return ids[id];\n  }\n}  // namespace concurrency_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/units.cpp\n#include \"utils/units.hpp\"\n\n#include \"common.hpp\"\n#include \"components/types.hpp\"\n#include \"errors.hpp\"\n#include \"utils/math.hpp\"\n\nPOLYBAR_NS\n\nnamespace units_utils {\n\n  /**\n   * Converts points to pixels under the given DPI (PPI).\n   *\n   * 1 pt = 1/72in, so point / 72 * DPI = #pixels\n   */\n  int point_to_pixel(double point, double dpi) {\n    return dpi * point / 72.0;\n  }\n\n  /**\n   * Converts an extent value to a pixel value according to the given DPI (if needed).\n   */\n  int extent_to_pixel(const extent_val size, double dpi) {\n    if (size.type == extent_type::PIXEL) {\n      return size.value;\n    }\n\n    return point_to_pixel(size.value, dpi);\n  }\n\n  /**\n   * Same as extent_to_pixel but is capped below at 0 pixels.\n   */\n  unsigned extent_to_pixel_nonnegative(const extent_val size, double dpi) {\n    return std::max(0, extent_to_pixel(size, dpi));\n  }\n\n  /**\n   * Converts a percentage with offset into pixels\n   */\n  int percentage_with_offset_to_pixel(percentage_with_offset g_format, double max, double dpi) {\n    int offset_pixel = extent_to_pixel(g_format.offset, dpi);\n\n    return static_cast<int>(math_util::percentage_to_value<double, double>(g_format.percentage, max) + offset_pixel);\n  }\n\n  /**\n   * Same as percentage_with_offset_to_pixel but capped below at 0 pixels\n   */\n  unsigned percentage_with_offset_to_pixel_nonnegative(percentage_with_offset g_format, double max, double dpi) {\n    return std::max<int>(0, percentage_with_offset_to_pixel(g_format, max, dpi));\n  }\n\n  extent_type parse_extent_unit(const string& str) {\n    if (!str.empty()) {\n      if (str == \"px\") {\n        return extent_type::PIXEL;\n      } else if (str == \"pt\") {\n        return extent_type::POINT;\n      } else {\n        throw std::runtime_error(\"Unrecognized unit '\" + str + \"'\");\n      }\n    } else {\n      return extent_type::PIXEL;\n    }\n  }\n\n  extent_val parse_extent(const string& str) {\n    size_t pos;\n    auto size_value = std::stof(str, &pos);\n\n    string unit = string_util::trim(str.substr(pos));\n    extent_type type = parse_extent_unit(unit);\n\n    // Pixel values should be integers\n    if (type == extent_type::PIXEL) {\n      size_value = std::trunc(size_value);\n    }\n\n    return {type, size_value};\n  }\n\n  string extent_to_string(extent_val extent) {\n    std::stringstream ss;\n\n    switch (extent.type) {\n      case extent_type::POINT:\n        ss << extent.value << \"pt\";\n        break;\n      case extent_type::PIXEL:\n        ss << static_cast<int>(extent.value) << \"px\";\n        break;\n    }\n\n    return ss.str();\n  }\n\n  spacing_type parse_spacing_unit(const string& str) {\n    if (!str.empty()) {\n      if (str == \"px\") {\n        return spacing_type::PIXEL;\n      } else if (str == \"pt\") {\n        return spacing_type::POINT;\n      } else {\n        throw std::runtime_error(\"Unrecognized unit '\" + str + \"'\");\n      }\n    } else {\n      return spacing_type::SPACE;\n    }\n  }\n\n  spacing_val parse_spacing(const string& str) {\n    size_t pos;\n    auto size_value = std::stof(str, &pos);\n\n    if (size_value < 0) {\n      throw runtime_error(sstream() << \"value '\" << str << \"' must not be negative\");\n    }\n\n    spacing_type type;\n\n    string unit = string_util::trim(str.substr(pos));\n    if (!unit.empty()) {\n      if (unit == \"px\") {\n        type = spacing_type::PIXEL;\n        size_value = std::trunc(size_value);\n      } else if (unit == \"pt\") {\n        type = spacing_type::POINT;\n      } else {\n        throw runtime_error(\"Unrecognized unit '\" + unit + \"'\");\n      }\n    } else {\n      type = spacing_type::SPACE;\n      size_value = std::trunc(size_value);\n    }\n\n    return {type, size_value};\n  }\n\n  /**\n   * Creates an extent from a spacing value.\n   *\n   * @param spacing Value to convert, must not be SPACE\n   */\n  extent_val spacing_to_extent(spacing_val spacing) {\n    extent_type t;\n\n    switch (spacing.type) {\n      case spacing_type::POINT:\n        t = extent_type::POINT;\n        break;\n      case spacing_type::PIXEL:\n        t = extent_type::PIXEL;\n        break;\n      default:\n        throw std::runtime_error(\"spacing_to_extent: Illegal type: \" + to_string(to_integral(spacing.type)));\n    }\n\n    return {t, spacing.value};\n  }\n\n} // namespace units_utils\n\nPOLYBAR_NS_END\n\n// Path: src/utils/i3.cpp\n#include \"utils/i3.hpp\"\n\n#include <xcb/xcb.h>\n\n#include <i3ipc++/ipc.hpp>\n\n#include \"common.hpp\"\n#include \"settings.hpp\"\n#include \"utils/socket.hpp\"\n#include \"utils/string.hpp\"\n#include \"x11/connection.hpp\"\n#include \"x11/ewmh.hpp\"\n#include \"x11/icccm.hpp\"\n\nPOLYBAR_NS\n\nnamespace i3_util {\n  /**\n   * Get all workspaces for given output\n   */\n  vector<shared_ptr<workspace_t>> workspaces(const connection_t& conn, const string& output, const bool show_urgent) {\n    vector<shared_ptr<workspace_t>> result;\n    for (auto&& ws : conn.get_workspaces()) {\n      if (output.empty() || ws->output == output || (show_urgent && ws->urgent)) {\n        result.emplace_back(forward<decltype(ws)>(ws));\n      }\n    }\n    return result;\n  }\n\n  /**\n   * Get currently focused workspace\n   */\n  shared_ptr<workspace_t> focused_workspace(const connection_t& conn) {\n    for (auto&& ws : conn.get_workspaces()) {\n      if (ws->focused) {\n        return ws;\n      }\n    }\n    return nullptr;\n  }\n\n  /**\n   * Get main root window\n   */\n  xcb_window_t root_window(connection& conn) {\n    auto children = conn.query_tree(conn.root()).children();\n    const auto wm_name = [&](xcb_connection_t* conn, xcb_window_t win) -> string {\n      string title;\n      if (!(title = ewmh_util::get_wm_name(win)).empty()) {\n        return title;\n      } else if (!(title = icccm_util::get_wm_name(conn, win)).empty()) {\n        return title;\n      } else {\n        return \"\";\n      }\n    };\n\n    for (auto it = children.begin(); it != children.end(); it++) {\n      if (wm_name(conn, *it) == \"i3\") {\n        return *it;\n      }\n    }\n\n    return XCB_NONE;\n  }\n\n  /**\n   * Returns window against which to restack.\n   */\n  restack_util::params get_restack_params(connection& conn) {\n    return {root_window(conn), XCB_STACK_MODE_ABOVE};\n  }\n} // namespace i3_util\n\nPOLYBAR_NS_END\n\n// Path: src/utils/process.cpp\n#include \"utils/process.hpp\"\n\n#include <fcntl.h>\n#include <sys/stat.h>\n#include <sys/types.h>\n#include <sys/wait.h>\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace process_util {\n  /**\n   * Check if currently in main process\n   */\n  bool in_parent_process(pid_t pid) {\n    return pid != -1 && pid != 0;\n  }\n\n  /**\n   * Check if currently in subprocess\n   */\n  bool in_forked_process(pid_t pid) {\n    return pid == 0;\n  }\n\n  /**\n   * Redirects all io fds (stdin, stdout, stderr) of the current process to /dev/null.\n   */\n  void redirect_stdio_to_dev_null() {\n    auto redirect = [](int fd_to_redirect) {\n      int fd = open(\"/dev/null\", O_WRONLY);\n      if (fd < 0 || dup2(fd, fd_to_redirect) < 0) {\n        throw system_error(\"Failed to redirect process output\");\n      }\n      close(fd);\n    };\n\n    redirect(STDIN_FILENO);\n    redirect(STDOUT_FILENO);\n    redirect(STDERR_FILENO);\n  }\n\n  /**\n   * Forks a child process and executes the given lambda function in it.\n   *\n   * Processes spawned this way need to be waited on by the caller.\n   */\n  pid_t spawn_async(std::function<void()> const& lambda) {\n    pid_t pid = fork();\n    switch (pid) {\n      case -1:\n        throw runtime_error(\"fork_detached: Unable to fork: \" + string(strerror(errno)));\n      case 0:\n        // Child\n        setsid();\n        umask(0);\n        redirect_stdio_to_dev_null();\n        lambda();\n        _Exit(0);\n        break;\n      default:\n        return pid;\n    }\n  }\n\n  /**\n   * Forks a child process and completely detaches it.\n   *\n   * In the child process, the given lambda function is executed.\n   * We fork twice so that the first forked process can exit and it's child is\n   * reparented to the init process.\n   *\n   * Ref: https://web.archive.org/web/20120914180018/http://www.steve.org.uk/Reference/Unix/faq_2.html#SEC16\n   *\n   * Use this if you want to run a command and just forget about it.\n   *\n   * @returns The PID of the child process\n   */\n  void fork_detached(std::function<void()> const& lambda) {\n    pid_t pid = fork();\n    switch (pid) {\n      case -1:\n        throw runtime_error(\"fork_detached: Unable to fork: \" + string(strerror(errno)));\n      case 0:\n        // Child\n        setsid();\n\n        pid = fork();\n        switch (pid) {\n          case -1:\n            throw runtime_error(\"fork_detached: Unable to fork: \" + string(strerror(errno)));\n          case 0:\n            // Child\n            umask(0);\n            redirect_stdio_to_dev_null();\n            lambda();\n            _Exit(0);\n        }\n\n        _Exit(0);\n...\n", "needle_token_start": 10213, "needle_token_end": 10463, "code_context_ntokens": 15957, "output": ["```cpp\nstatic pair<int, uint32_t> utf8_get_len(uint8_t leading) {\n  if ((leading & UTF8_LEADING1_MASK) == UTF8_LEADING1_PREFIX) {\n    return {1, leading & ~UTF8_LEADING1_MASK};\n  } else if ((leading & UTF8_LEADING2_MASK) == UTF8_LEADING2_PREFIX) {\n    return {2, leading & ~UTF8_LEADING2_MASK};\n  } else if ((leading & UTF8_LEADING3_MASK) == UTF8_LEADING3_PREFIX) {\n    return {3, leading & ~UTF8_LEADING3_MASK};\n  } else if ((leading & UTF8_LEADING4_MASK) == UTF8_LEADING4_PREFIX) {\n    return {4, leading & ~UTF8_LEADING4_MASK};\n  } else {\n    return {-1, 0};\n  }\n}\n```"]}
{"repo": "polybar/polybar", "name": "get_color", "language": "cpp", "path": "src/tags/context.cpp", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function determines the appropriate color value to use based on whether a reset condition is specified. If a reset is indicated, it returns a default color; otherwise, it returns the specified color.\n2. **Input**: The function takes two parameters: a color structure that indicates whether a reset is needed and the actual color value, and a fallback color value to use in case of a reset.\n3. **Output**: Returns a color value, which is either the specified color or the fallback color depending on the reset condition.\n4. **Procedure**: The function checks if the reset condition is active within the color structure. If active, it returns the fallback color; if not, it returns the specified color from the color structure.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/components/renderer.cpp\n#include \"components/renderer.hpp\"\n\n#include <cassert>\n\n#include \"cairo/context.hpp\"\n#include \"components/config.hpp\"\n#include \"events/signal.hpp\"\n#include \"events/signal_emitter.hpp\"\n#include \"events/signal_receiver.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/units.hpp\"\n#include \"x11/atoms.hpp\"\n#include \"x11/background_manager.hpp\"\n#include \"x11/connection.hpp\"\n#include \"x11/winspec.hpp\"\n\nPOLYBAR_NS\n\nstatic constexpr double BLOCK_GAP{20.0};\n\n/**\n * Create instance\n */\nrenderer::make_type renderer::make(const bar_settings& bar, tags::action_context& action_ctxt, const config& conf) {\n  // clang-format off\n  return std::make_unique<renderer>(\n      connection::make(),\n      signal_emitter::make(),\n      conf,\n      logger::make(),\n      forward<decltype(bar)>(bar),\n      background_manager::make(),\n      action_ctxt);\n  // clang-format on\n}\n\n/**\n * Construct renderer instance\n */\nrenderer::renderer(connection& conn, signal_emitter& sig, const config& conf, const logger& logger,\n    const bar_settings& bar, background_manager& background, tags::action_context& action_ctxt)\n    : renderer_interface(action_ctxt)\n    , m_connection(conn)\n    , m_sig(sig)\n    , m_conf(conf)\n    , m_log(logger)\n    , m_bar(forward<const bar_settings&>(bar))\n    , m_rect(m_bar.inner_area()) {\n  m_sig.attach(this);\n\n  m_log.trace(\"renderer: Get TrueColor visual\");\n  if ((m_visual = m_connection.visual_type(XCB_VISUAL_CLASS_TRUE_COLOR, 32)) != nullptr) {\n    m_depth = 32;\n  } else if ((m_visual = m_connection.visual_type(XCB_VISUAL_CLASS_TRUE_COLOR, 24)) != nullptr) {\n    m_depth = 24;\n  } else {\n    throw application_error(\"Could not find a 24 or 32-bit TrueColor visual\");\n  }\n\n  m_log.info(\"renderer: Using %d-bit TrueColor visual: 0x%x\", m_depth, m_visual->visual_id);\n\n  m_log.trace(\"renderer: Allocate colormap\");\n  m_colormap = m_connection.generate_id();\n  m_connection.create_colormap(XCB_COLORMAP_ALLOC_NONE, m_colormap, m_connection.root(), m_visual->visual_id);\n\n  m_log.trace(\"renderer: Allocate output window\");\n  // clang-format off\n  m_window = winspec(m_connection)\n    << cw_size(m_bar.size)\n    << cw_pos(m_bar.pos)\n    << cw_depth(m_depth)\n    << cw_visual(m_visual->visual_id)\n    << cw_class(XCB_WINDOW_CLASS_INPUT_OUTPUT)\n    << cw_params_back_pixel(0)\n    << cw_params_border_pixel(0)\n    << cw_params_backing_store(XCB_BACKING_STORE_WHEN_MAPPED)\n    << cw_params_colormap(m_colormap)\n    << cw_params_event_mask(XCB_EVENT_MASK_PROPERTY_CHANGE\n                           |XCB_EVENT_MASK_EXPOSURE\n                           |XCB_EVENT_MASK_BUTTON_PRESS)\n    << cw_params_override_redirect(m_bar.override_redirect)\n    << cw_flush(true);\n  // clang-format on\n\n  m_log.trace(\"renderer: Allocate window pixmaps\");\n  {\n    m_pixmap = m_connection.generate_id();\n    m_connection.create_pixmap(m_depth, m_pixmap, m_window, m_bar.size.w, m_bar.size.h);\n\n    uint32_t configure_mask = 0;\n    std::array<uint32_t, 32> configure_values{};\n    xcb_params_cw_t configure_params{};\n\n    XCB_AUX_ADD_PARAM(&configure_mask, &configure_params, back_pixmap, m_pixmap);\n    connection::pack_values(configure_mask, &configure_params, configure_values);\n    m_connection.change_window_attributes_checked(m_window, configure_mask, configure_values.data());\n  }\n\n  m_log.trace(\"renderer: Allocate graphic contexts\");\n  {\n    uint32_t mask{0};\n    std::array<uint32_t, 32> value_list{};\n    xcb_params_gc_t params{};\n    XCB_AUX_ADD_PARAM(&mask, &params, foreground, m_bar.foreground);\n    XCB_AUX_ADD_PARAM(&mask, &params, graphics_exposures, 0);\n    connection::pack_values(mask, &params, value_list);\n    m_gcontext = m_connection.generate_id();\n    m_connection.create_gc(m_gcontext, m_pixmap, mask, value_list.data());\n  }\n\n  m_log.trace(\"renderer: Allocate alignment blocks\");\n  {\n    m_blocks.emplace(alignment::LEFT, alignment_block{nullptr, 0.0, 0.0, 0.});\n    m_blocks.emplace(alignment::CENTER, alignment_block{nullptr, 0.0, 0.0, 0.});\n    m_blocks.emplace(alignment::RIGHT, alignment_block{nullptr, 0.0, 0.0, 0.});\n  }\n\n  m_log.trace(\"renderer: Allocate cairo components\");\n  {\n    m_surface = make_unique<cairo::xcb_surface>(m_connection, m_pixmap, m_visual, m_bar.size.w, m_bar.size.h);\n    m_context = make_unique<cairo::context>(*m_surface, m_log);\n  }\n\n  m_log.trace(\"renderer: Load fonts\");\n  {\n    auto fonts = m_conf.get_list<string>(m_conf.section(), \"font\", {});\n    if (fonts.empty()) {\n      m_log.warn(\"No fonts specified, using fallback font \\\"fixed\\\"\");\n      fonts.emplace_back(\"fixed\");\n    }\n\n    for (const auto& f : fonts) {\n      int offset{0};\n      string pattern{f};\n      size_t pos = pattern.rfind(';');\n      if (pos != string::npos) {\n        offset = std::strtol(pattern.substr(pos + 1).c_str(), nullptr, 10);\n        pattern.erase(pos);\n      }\n      auto font = cairo::make_font(*m_context, string{pattern}, offset, m_bar.dpi_x, m_bar.dpi_y);\n      m_log.notice(\"Loaded font \\\"%s\\\" (name=%s, offset=%i, file=%s)\", pattern, font->name(), offset, font->file());\n      *m_context << move(font);\n    }\n  }\n\n  m_pseudo_transparency = m_conf.get<bool>(\"settings\", \"pseudo-transparency\", m_pseudo_transparency);\n  if (m_pseudo_transparency) {\n    m_log.trace(\"Activate root background manager\");\n    m_background = background.observe(m_bar.outer_area(false), m_window);\n  }\n\n  m_comp_bg = m_conf.get<cairo_operator_t>(\"settings\", \"compositing-background\", m_comp_bg);\n  m_comp_fg = m_conf.get<cairo_operator_t>(\"settings\", \"compositing-foreground\", m_comp_fg);\n  m_comp_ol = m_conf.get<cairo_operator_t>(\"settings\", \"compositing-overline\", m_comp_ol);\n  m_comp_ul = m_conf.get<cairo_operator_t>(\"settings\", \"compositing-underline\", m_comp_ul);\n  m_comp_border = m_conf.get<cairo_operator_t>(\"settings\", \"compositing-border\", m_comp_border);\n\n  m_fixedcenter = m_conf.get(m_conf.section(), \"fixed-center\", true);\n}\n\n/**\n * Deconstruct instance\n */\nrenderer::~renderer() {\n  m_sig.detach(this);\n}\n\n/**\n * Get bar window\n */\nxcb_window_t renderer::window() const {\n  return m_window;\n}\n\n/**\n * Get the bar window visual\n */\nxcb_visualtype_t* renderer::visual() const {\n  return m_visual;\n}\n\n/**\n * Get the bar window depth\n */\nint renderer::depth() const {\n  return m_depth;\n}\n\n/**\n * Begin render routine\n */\nvoid renderer::begin(xcb_rectangle_t rect) {\n  m_log.trace_x(\"renderer: begin (geom=%ix%i+%i+%i)\", rect.width, rect.height, rect.x, rect.y);\n\n  // Reset state\n  m_rect = rect;\n  m_align = alignment::NONE;\n\n  // Clear canvas\n  m_context->save();\n  m_context->clear();\n\n  // when pseudo-transparency is requested, render the bar into a new layer\n  // that will later be composited against the desktop background\n  if (m_pseudo_transparency) {\n    m_context->push();\n  }\n\n  // Create corner mask\n  if (m_bar.radius && m_cornermask == nullptr) {\n    m_context->save();\n    m_context->push();\n    // clang-format off\n    *m_context << cairo::rounded_corners{\n        static_cast<double>(m_rect.x),\n        static_cast<double>(m_rect.y),\n        static_cast<double>(m_rect.width),\n        static_cast<double>(m_rect.height), m_bar.radius};\n    // clang-format on\n    *m_context << rgba{0xffffffff};\n    m_context->fill();\n    m_context->pop(&m_cornermask);\n    m_context->restore();\n  }\n\n  fill_borders();\n\n  // clang-format off\n  m_context->clip(cairo::rect{\n      static_cast<double>(m_rect.x),\n      static_cast<double>(m_rect.y),\n      static_cast<double>(m_rect.width),\n      static_cast<double>(m_rect.height)});\n  // clang-format on\n}\n\n/**\n * End render routine\n */\nvoid renderer::end() {\n  m_log.trace_x(\"renderer: end\");\n\n  if (m_align != alignment::NONE) {\n    m_log.trace_x(\"renderer: pop(%i)\", static_cast<int>(m_align));\n    m_context->pop(&m_blocks[m_align].pattern);\n\n    // Capture the concatenated block contents\n    // so that it can be masked with the corner pattern\n    m_context->push();\n\n    // Draw the background on the new layer to make up for\n    // the areas not covered by the alignment blocks\n    fill_background();\n\n    for (auto&& b : m_blocks) {\n      flush(b.first);\n    }\n\n    cairo_pattern_t* blockcontents{};\n    m_context->pop(&blockcontents);\n\n    if (m_cornermask != nullptr) {\n      *m_context << blockcontents;\n      m_context->mask(m_cornermask);\n    } else {\n      *m_context << blockcontents;\n      m_context->paint();\n    }\n\n    m_context->destroy(&blockcontents);\n  } else {\n    fill_background();\n  }\n\n  // For pseudo-transparency, capture the contents of the rendered bar and\n  // composite it against the desktop wallpaper. This way transparent parts of\n  // the bar will be filled by the wallpaper creating illusion of transparency.\n  if (m_pseudo_transparency) {\n    cairo_pattern_t* barcontents{};\n    m_context->pop(&barcontents); // corresponding push is in renderer::begin\n\n    auto root_bg = m_background->get_surface();\n    if (root_bg != nullptr) {\n      m_log.trace_x(\"renderer: root background\");\n      *m_context << *root_bg;\n      m_context->paint();\n      *m_context << CAIRO_OPERATOR_OVER;\n    }\n    *m_context << barcontents;\n    m_context->paint();\n    m_context->destroy(&barcontents);\n  }\n\n  m_context->restore();\n  m_surface->flush();\n\n  flush();\n\n  m_sig.emit(signals::ui::changed{});\n}\n\n/**\n * Flush contents of given alignment block\n */\nvoid renderer::flush(alignment a) {\n  if (m_blocks[a].pattern == nullptr) {\n    return;\n  }\n\n  m_context->save();\n\n  double x = static_cast<int>(block_x(a) + 0.5);\n  double y = static_cast<int>(block_y(a) + 0.5);\n  double w = static_cast<int>(block_w(a) + 0.5);\n  double h = static_cast<int>(block_h(a) + 0.5);\n  double xw = x + w;\n  bool fits{xw <= m_rect.width};\n\n  m_log.trace(\"renderer: flush(%i geom=%gx%g+%g+%g, falloff=%i)\", static_cast<int>(a), w, h, x, y, !fits);\n\n  // Set block shape\n  *m_context << cairo::abspos{0.0, 0.0};\n  *m_context << cairo::rect{m_rect.x + x, m_rect.y + y, w, h};\n\n  // Restrict drawing to the block rectangle\n  m_context->clip(true);\n\n  // Clear the area covered by the block\n  m_context->clear();\n\n  *m_context << cairo::translate{x, 0.0};\n  *m_context << m_blocks[a].pattern;\n  m_context->paint();\n\n  *m_context << cairo::abspos{0.0, 0.0};\n  m_context->destroy(&m_blocks[a].pattern);\n  m_context->restore();\n\n  if (!fits) {\n    // Paint falloff gradient at the end of the visible block\n    // to indicate that the content expands past the canvas\n\n    /*\n     * How many pixels are hidden\n     */\n    double overflow = xw - m_rect.width;\n    double visible_width = w - overflow;\n\n    /*\n     * Width of the falloff gradient. Depends on how much of the block is hidden\n     */\n    double fsize = std::max(5.0, std::min(std::abs(overflow), 30.0));\n    m_log.trace(\"renderer: Drawing falloff (pos=%g, size=%g, overflow=%g)\", visible_width - fsize, fsize, overflow);\n    m_context->save();\n    *m_context << cairo::translate{(double)m_rect.x, (double)m_rect.y};\n    *m_context << cairo::abspos{0.0, 0.0};\n    *m_context << cairo::rect{x + visible_width - fsize, y, fsize, h};\n    m_context->clip(true);\n    *m_context << cairo::linear_gradient{\n        x + visible_width - fsize, y, x + visible_width, y, {rgba{0x00000000}, rgba{0xFF000000}}};\n    m_context->paint(0.25);\n    m_context->restore();\n  }\n}\n\n/**\n * Flush pixmap contents onto the target window\n */\nvoid renderer::flush() {\n  m_log.trace_x(\"renderer: flush\");\n\n  highlight_clickable_areas();\n\n  m_surface->flush();\n  // Clear entire window so that the new pixmap is shown\n  m_connection.clear_area(0, m_window, 0, 0, m_bar.size.w, m_bar.size.h);\n  m_connection.flush();\n\n  if (!m_snapshot_dst.empty()) {\n    try {\n      m_surface->write_png(m_snapshot_dst);\n      m_log.notice(\"Successfully wrote %s\", m_snapshot_dst);\n    } catch (const exception& err) {\n      m_log.err(\"Failed to write snapshot (err: %s)\", err.what());\n    }\n    m_snapshot_dst.clear();\n  }\n}\n\n/**\n * Get x position of block for given alignment\n *\n * The position is relative to m_rect.x (the left side of the bar w/o borders and tray)\n */\ndouble renderer::block_x(alignment a) const {\n  switch (a) {\n    case alignment::CENTER: {\n      // The leftmost x position this block can start at\n      double min_pos = block_w(alignment::LEFT);\n\n      if (min_pos != 0) {\n        min_pos += BLOCK_GAP;\n      }\n\n      double right_width = block_w(alignment::RIGHT);\n      /*\n       * The rightmost x position this block can end at\n       *\n       * We can't use block_x(alignment::RIGHT) because that would lead to infinite recursion\n       */\n      double max_pos = m_rect.width - right_width;\n\n      if (right_width != 0) {\n        max_pos -= BLOCK_GAP;\n      }\n\n      /*\n       * x position of the center of this block\n       *\n       * With fixed-center this will be the center of the bar unless it is pushed to the left by a large right block\n       * Without fixed-center this will be the middle between the end of the left and the start of the right block.\n       */\n      double base_pos{0.0};\n\n      if (m_fixedcenter) {\n        /*\n         * This is in the middle of the *bar*. Not just the middle of m_rect because this way we need to account for the\n         * tray.\n         *\n         * The resulting position is relative to the very left of the bar (including border and tray), so we need to\n         * compensate for that by subtracting m_rect.x\n         */\n        base_pos = m_bar.size.w / 2.0 - m_rect.x;\n\n        /*\n         * The center block can be moved to the left if the right block is too large\n         */\n        base_pos = std::min(base_pos, max_pos - block_w(a) / 2.0);\n      } else {\n        base_pos = (min_pos + max_pos) / 2.0;\n      }\n\n      /*\n       * The left block always has priority (even with fixed-center = true)\n       */\n      return std::max(base_pos - block_w(a) / 2.0, min_pos);\n    }\n    case alignment::RIGHT: {\n      /*\n       * The block immediately to the left of this block\n       *\n       * Generally the center block unless it is empty.\n       */\n      alignment left_barrier = alignment::CENTER;\n\n      if (block_w(alignment::CENTER) == 0) {\n        left_barrier = alignment::LEFT;\n      }\n\n      // The minimum x position this block can start at\n      double min_pos = block_x(left_barrier) + block_w(left_barrier);\n\n...\n// Path: src/tags/parser.cpp\n#include \"tags/parser.hpp\"\n\n#include <cassert>\n#include <cctype>\n\n#include \"utils/units.hpp\"\n\nPOLYBAR_NS\n\nnamespace tags {\n\n  bool parser::has_next_element() {\n    return buf_pos < buf.size() || has_next();\n  }\n\n  element parser::next_element() {\n    if (!has_next_element()) {\n      throw std::runtime_error(\"tag parser: No next element\");\n    }\n\n    if (buf_pos >= buf.size()) {\n      parse_step();\n    }\n\n    if (buf_pos >= buf.size()) {\n      throw std::runtime_error(\"tag parser: No next element. THIS IS A BUG. (Context: '\" + input + \"')\");\n    }\n\n    element e = buf[buf_pos];\n    buf_pos++;\n\n    if (buf_pos == buf.size()) {\n      buf.clear();\n      buf_pos = 0;\n    }\n\n    return e;\n  }\n\n  format_string parser::parse() {\n    format_string parsed;\n\n    while (has_next_element()) {\n      parsed.push_back(next_element());\n    }\n\n    return parsed;\n  }\n\n  /**\n   * Performs a single parse step.\n   *\n   * This means it will parse text until the next tag is reached or it will\n   * parse an entire %{...} tag.\n   */\n  void parser::parse_step() {\n    char c;\n\n    /*\n     * If we have already parsed text, we can stop if we reach a tag.\n     */\n    bool text_parsed = false;\n\n    size_t start_pos = pos;\n\n    try {\n      while ((c = next())) {\n        // TODO here we could think about how to escape an action tag\n        if (c == '%' && has_next() && peek() == '{') {\n          /*\n           * If we have already parsed text, encountering a tag opening means\n           * we can stop parsing now because we parsed at least one entire\n           * element (the text up to the beginning of the tag).\n           */\n          if (text_parsed) {\n            // Put back the '%'\n            revert();\n            break;\n          }\n\n          consume('{');\n          consume_space();\n          parse_tag();\n          break;\n        } else {\n          push_char(c);\n          text_parsed = true;\n        }\n      }\n    } catch (error& e) {\n      e.set_context(input.substr(start_pos, pos - start_pos));\n      throw;\n    }\n  }\n\n  void parser::set(const string&& input) {\n    this->input = std::move(input);\n    pos = 0;\n    buf.clear();\n    buf_pos = 0;\n  }\n\n  bool parser::has_next() const {\n    return pos < input.size();\n  }\n\n  char parser::next() {\n    char c = peek();\n    pos++;\n    return c;\n  }\n\n  char parser::peek() const {\n    if (!has_next()) {\n      return EOL;\n    }\n\n    return input[pos];\n  }\n\n  /**\n   * Puts back a single character in the input string.\n   */\n  void parser::revert() {\n    assert(pos > 0);\n    pos--;\n  }\n\n  void parser::consume(char c) {\n    char n = next();\n    if (n != c) {\n      throw tags::token_error(n, c);\n    }\n  }\n\n  void parser::consume_space() {\n    while (peek() == ' ') {\n      next();\n    }\n  }\n\n  /**\n   * Parses an entire %{....} tag.\n   *\n   * '%' and '{' were already consumed and we are currently on the first character\n   * inside the tag.\n   * At the end of this method, we should be on the closing '}' character (not\n   * yet consumed).\n   */\n\n  void parser::parse_tag() {\n    if (!has_next()) {\n      throw token_error(EOL, \"Formatting tag content\");\n    }\n\n    while (has_next()) {\n      parse_single_tag_content();\n\n      int p = peek();\n\n      if (p != ' ' && p != '}') {\n        throw tag_end_error(p);\n      } else {\n        /**\n         * Consume whitespace between elements inside the tag\n         */\n        consume_space();\n\n        if (peek() == '}') {\n          consume('}');\n          break;\n        }\n      }\n    }\n  }\n\n  /**\n   * Parses a single element inside a formatting tag.\n   *\n   * For example it would parse the foreground part of the following tag:\n   *\n   * %{F#ff0000 B#ff0000}\n   *   ^       ^\n   *   |       - Pointer at the end\n   *   |\n   *   - Pointer at the start\n   */\n  void parser::parse_single_tag_content() {\n    char c = next();\n\n    /**\n     * %{U...} is a special case because it produces over and underline tags.\n     */\n    if (c == 'U') {\n      element e{};\n      e.is_tag = true;\n      e.tag_data.type = tag_type::FORMAT;\n      e.tag_data.subtype.format = syntaxtag::u;\n      e.tag_data.color = parse_color();\n      buf.emplace_back(e);\n\n      e.tag_data.subtype.format = syntaxtag::o;\n      buf.emplace_back(e);\n      return;\n    }\n\n    tag_type type;\n    tag_subtype sub;\n\n    switch (c) {\n      // clang-format off\n      case 'B': sub.format = syntaxtag::B; break;\n      case 'F': sub.format = syntaxtag::F; break;\n      case 'u': sub.format = syntaxtag::u; break;\n      case 'o': sub.format = syntaxtag::o; break;\n      case 'T': sub.format = syntaxtag::T; break;\n      case 'R': sub.format = syntaxtag::R; break;\n      case 'O': sub.format = syntaxtag::O; break;\n      case 'P': sub.format = syntaxtag::P; break;\n      case 'A': sub.format = syntaxtag::A; break;\n      case 'l': sub.format = syntaxtag::l; break;\n      case 'c': sub.format = syntaxtag::c; break;\n      case 'r': sub.format = syntaxtag::r; break;\n\n      case '+': sub.activation = attr_activation::ON; break;\n      case '-': sub.activation = attr_activation::OFF; break;\n      case '!': sub.activation = attr_activation::TOGGLE; break;\n\n        // clang-format on\n\n      default:\n        throw unrecognized_tag(c);\n    }\n\n    switch (c) {\n      case 'B':\n      case 'F':\n      case 'u':\n      case 'o':\n      case 'T':\n      case 'R':\n      case 'O':\n      case 'P':\n      case 'A':\n      case 'l':\n      case 'c':\n      case 'r':\n        type = tag_type::FORMAT;\n        break;\n\n      case '+':\n      case '-':\n      case '!':\n        type = tag_type::ATTR;\n        break;\n\n      default:\n        throw unrecognized_tag(c);\n    }\n\n    tag tag_data{};\n    tag_data.type = type;\n    tag_data.subtype = sub;\n\n    element e{};\n    e.is_tag = true;\n\n    switch (c) {\n      case 'B':\n      case 'F':\n      case 'u':\n      case 'o':\n        tag_data.color = parse_color();\n        break;\n      case 'T':\n        tag_data.font = parse_fontindex();\n        break;\n      case 'O':\n        tag_data.offset = parse_offset();\n        break;\n      case 'P':\n        tag_data.ctrl = parse_control();\n        break;\n      case 'A':\n        std::tie(tag_data.action, e.data) = parse_action();\n        break;\n\n      case '+':\n      case '-':\n      case '!':\n        tag_data.attr = parse_attribute();\n        break;\n    }\n\n    e.tag_data = tag_data;\n    buf.emplace_back(e);\n  }\n\n  color_value parser::parse_color() {\n    string s = get_tag_value();\n\n    color_value ret;\n\n    if (s.empty() || s == \"-\") {\n      ret.type = color_type::RESET;\n    } else {\n      rgba c{s};\n\n      if (!c.has_color()) {\n        throw color_error(s);\n      }\n\n      ret.type = color_type::COLOR;\n      ret.val = c;\n    }\n\n    return ret;\n  }\n\n  int parser::parse_fontindex() {\n    string s = get_tag_value();\n\n    if (s.empty() || s == \"-\") {\n      return 0;\n    }\n\n    try {\n      size_t ptr;\n      int ret = std::stoi(s, &ptr, 10);\n\n      if (ret < 0) {\n        return 0;\n      }\n\n      if (ptr != s.size()) {\n        throw font_error(s, \"Font index contains non-number characters\");\n      }\n\n      return ret;\n    } catch (const std::exception& err) {\n      throw font_error(s, err.what());\n    }\n  }\n\n  extent_val parser::parse_offset() {\n    string s = get_tag_value();\n\n    if (s.empty()) {\n      return ZERO_PX_EXTENT;\n    }\n\n    try {\n      return units_utils::parse_extent(string{s});\n    } catch (const std::exception& err) {\n      throw offset_error(s, err.what());\n    }\n  }\n\n  controltag parser::parse_control() {\n    string s = get_tag_value();\n\n    if (s.empty()) {\n      throw control_error(s, \"Control tag is empty\");\n    }\n\n    switch (s[0]) {\n      case 'R':\n        if (s.size() != 1) {\n          throw control_error(s, \"Control tag R has extra data\");\n        }\n\n        return controltag::R;\n      case 't':\n        return controltag::t;\n      default:\n        throw control_error(s);\n    }\n  }\n\n  std::pair<action_value, string> parser::parse_action() {\n    mousebtn btn = parse_action_btn();\n\n    action_value ret;\n\n    string cmd;\n\n    if (has_next() && peek() == ':') {\n      ret.btn = btn == mousebtn::NONE ? mousebtn::LEFT : btn;\n      ret.closing = false;\n      cmd = parse_action_cmd();\n    } else {\n      ret.btn = btn;\n      ret.closing = true;\n    }\n\n    return {ret, cmd};\n  }\n\n  /**\n   * Parses the button index after starting an action tag.\n   *\n   * May return mousebtn::NONE if no button was given.\n   */\n  mousebtn parser::parse_action_btn() {\n    if (has_next()) {\n      if (isdigit(peek())) {\n        char c = next();\n        int num = c - '0';\n\n        if (num < static_cast<int>(mousebtn::NONE) || num >= static_cast<int>(mousebtn::BTN_COUNT)) {\n          throw btn_error(string{c});\n        }\n\n        return static_cast<mousebtn>(num);\n      }\n    }\n\n    return mousebtn::NONE;\n  }\n\n  /**\n   * Starts at ':' and parses a complete action string.\n   *\n   * Returns the parsed action string with without escaping backslashes.\n   *\n   * Afterwards the parsers will be at the character immediately after the\n   * closing colon.\n   */\n  string parser::parse_action_cmd() {\n    consume(':');\n\n    string s;\n\n    char prev = EOL;\n\n    while (has_next()) {\n      char c = next();\n\n      if (c == ':') {\n        if (prev == '\\\\') {\n          s.pop_back();\n          s.push_back(c);\n        } else {\n          break;\n        }\n      } else {\n        s.push_back(c);\n      }\n\n      prev = c;\n    }\n\n    return s;\n  }\n\n  attribute parser::parse_attribute() {\n    char c;\n    switch (c = next()) {\n      case 'u':\n        return attribute::UNDERLINE;\n      case 'o':\n        return attribute::OVERLINE;\n      default:\n        throw unrecognized_attr(c);\n    }\n  }\n\n  void parser::push_char(char c) {\n    if (!buf.empty() && buf_pos < buf.size() && !buf.back().is_tag) {\n      buf.back().data += c;\n    } else {\n      buf.emplace_back(string{c});\n    }\n  }\n\n  void parser::push_text(string&& text) {\n    if (text.empty()) {\n      return;\n    }\n\n    if (!buf.empty() && buf_pos < buf.size() && !buf.back().is_tag) {\n      buf.back().data += text;\n    } else {\n      buf.emplace_back(std::move(text));\n    }\n  }\n\n  /**\n   * Will read up until the end of the tag value.\n   *\n   * Afterwards the parser will be at the character directly after the tag\n   * value.\n   *\n   * This function just reads until it encounters a space or a closing curly\n   * bracket, so it is not useful for tag values that can contain these\n   * characters (e.g. action tags).\n   */\n  string parser::get_tag_value() {\n    string s;\n\n    while (has_next() && peek() != ' ' && peek() != '}') {\n      s.push_back(next());\n    }\n\n    return s;\n  }\n} // namespace tags\n\nPOLYBAR_NS_END\n\n// Path: src/tags/action_context.cpp\n#include \"tags/action_context.hpp\"\n\n#include <cassert>\n\nPOLYBAR_NS\n\nnamespace tags {\n\n  void action_context::reset() {\n    m_action_blocks.clear();\n  }\n\n  action_t action_context::action_open(mousebtn btn, const string&& cmd, alignment align, double x) {\n    action_t id = m_action_blocks.size();\n    m_action_blocks.emplace_back(std::move(cmd), btn, align, true);\n    set_start(id, x);\n    return id;\n  }\n\n  std::pair<action_t, mousebtn> action_context::action_close(mousebtn btn, alignment align, double x) {\n    for (auto it = m_action_blocks.rbegin(); it != m_action_blocks.rend(); it++) {\n      if (it->is_open && it->align == align && (btn == mousebtn::NONE || it->button == btn)) {\n        it->is_open = false;\n\n        // Converts a reverse iterator into an index\n        action_t id = std::distance(m_action_blocks.begin(), it.base()) - 1;\n        set_end(id, x);\n        return {id, it->button};\n      }\n    }\n\n    return {NO_ACTION, mousebtn::NONE};\n  }\n\n  void action_context::set_start(action_t id, double x) {\n    m_action_blocks[id].start_x = x;\n  }\n\n  void action_context::set_end(action_t id, double x) {\n    /*\n     * Only ever increase the end position.\n     * A larger end position may have been set before.\n     */\n    m_action_blocks[id].end_x = std::max(m_action_blocks[id].end_x, x);\n  }\n\n  void action_context::compensate_for_negative_move(alignment a, double old_x, double new_x) {\n    assert(new_x < old_x);\n    for (auto& block : m_action_blocks) {\n      if (block.is_open && block.align == a) {\n        // Move back the start position if a smaller position is observed\n        if (block.start_x > new_x) {\n          block.start_x = new_x;\n        }\n\n        // Move forward the end position if a larger position is observed\n        if (old_x > block.end_x) {\n          block.end_x = old_x;\n        }\n      }\n    }\n  }\n\n  void action_context::set_alignment_start(const alignment a, const double x) {\n    m_align_start[a] = x;\n  }\n\n  std::map<mousebtn, tags::action_t> action_context::get_actions(int x) const {\n    std::map<mousebtn, tags::action_t> buttons;\n\n    for (int i = static_cast<int>(mousebtn::NONE); i < static_cast<int>(mousebtn::BTN_COUNT); i++) {\n      buttons[static_cast<mousebtn>(i)] = tags::NO_ACTION;\n    }\n\n    for (action_t id = 0; (unsigned)id < m_action_blocks.size(); id++) {\n      auto action = m_action_blocks[id];\n      mousebtn btn = action.button;\n\n      // Higher IDs are higher in the action stack.\n      if (id > buttons[btn] && action.test(m_align_start.at(action.align), x)) {\n        buttons[action.button] = id;\n      }\n    }\n\n    return buttons;\n  }\n\n  action_t action_context::has_action(mousebtn btn, int x) const {\n    return get_actions(x)[btn];\n  }\n\n  string action_context::get_action(action_t id) const {\n    assert(id >= 0 && (unsigned)id < num_actions());\n\n    return m_action_blocks[id].cmd;\n  }\n\n  bool action_context::has_double_click() const {\n    for (auto&& a : m_action_blocks) {\n      if (a.button == mousebtn::DOUBLE_LEFT || a.button == mousebtn::DOUBLE_MIDDLE ||\n          a.button == mousebtn::DOUBLE_RIGHT) {\n        return true;\n      }\n    }\n\n    return false;\n  }\n\n  size_t action_context::num_actions() const {\n    return m_action_blocks.size();\n  }\n\n  size_t action_context::num_unclosed() const {\n    size_t num = 0;\n\n    for (const auto& a : m_action_blocks) {\n      if (a.is_open) {\n        num++;\n      }\n    }\n\n    return num;\n  }\n\n  const std::vector<action_block>& action_context::get_blocks() const {\n    return m_action_blocks;\n  }\n} // namespace tags\n\nPOLYBAR_NS_END\n\n// Path: src/tags/dispatch.cpp\n#include \"tags/dispatch.hpp\"\n\n#include <algorithm>\n\n#include \"components/renderer.hpp\"\n#include \"events/signal.hpp\"\n#include \"settings.hpp\"\n#include \"tags/parser.hpp\"\n#include \"utils/color.hpp\"\n#include \"components/logger.hpp\"\n\nPOLYBAR_NS\n\nnamespace tags {\n  /**\n   * Create instance\n   */\n  dispatch::make_type dispatch::make(action_context& action_ctxt) {\n    return std::make_unique<dispatch>(logger::make(), action_ctxt);\n  }\n\n  /**\n   * Construct parser instance\n   */\n  dispatch::dispatch(const logger& logger, action_context& action_ctxt) : m_log(logger), m_action_ctxt(action_ctxt) {}\n\n  /**\n   * Process input string\n   */\n  void dispatch::parse(const bar_settings& bar, renderer_interface& renderer, const string&& data) {\n    tags::parser p;\n    p.set(std::move(data));\n\n    m_action_ctxt.reset();\n    m_ctxt = make_unique<context>(bar);\n\n    while (p.has_next_element()) {\n      tags::element el;\n      try {\n        el = p.next_element();\n      } catch (const tags::error& e) {\n        m_log.err(\"Parser error (reason: %s)\", e.what());\n        continue;\n      }\n\n      alignment old_alignment = m_ctxt->get_alignment();\n      double old_x = old_alignment == alignment::NONE ? 0 : renderer.get_x(*m_ctxt);\n\n      if (el.is_tag) {\n        switch (el.tag_data.type) {\n          case tags::tag_type::FORMAT:\n            switch (el.tag_data.subtype.format) {\n              case tags::syntaxtag::A:\n                handle_action(renderer, el.tag_data.action.btn, el.tag_data.action.closing, std::move(el.data));\n                break;\n              case tags::syntaxtag::B:\n                m_ctxt->apply_bg(el.tag_data.color);\n                break;\n              case tags::syntaxtag::F:\n                m_ctxt->apply_fg(el.tag_data.color);\n                break;\n              case tags::syntaxtag::T:\n                m_ctxt->apply_font(el.tag_data.font);\n                break;\n              case tags::syntaxtag::O:\n                handle_offset(renderer, el.tag_data.offset);\n                break;\n              case tags::syntaxtag::R:\n                m_ctxt->apply_reverse();\n                break;\n              case tags::syntaxtag::o:\n                m_ctxt->apply_ol(el.tag_data.color);\n                break;\n              case tags::syntaxtag::u:\n                m_ctxt->apply_ul(el.tag_data.color);\n                break;\n              case tags::syntaxtag::P:\n                handle_control(renderer, el.tag_data.ctrl);\n                break;\n              case tags::syntaxtag::l:\n                handle_alignment(renderer, alignment::LEFT);\n                break;\n              case tags::syntaxtag::r:\n                handle_alignment(renderer, alignment::RIGHT);\n                break;\n              case tags::syntaxtag::c:\n                handle_alignment(renderer, alignment::CENTER);\n                break;\n              default:\n                throw runtime_error(\n                    \"Unrecognized tag format: \" + to_string(static_cast<int>(el.tag_data.subtype.format)));\n            }\n            break;\n          case tags::tag_type::ATTR:\n            m_ctxt->apply_attr(el.tag_data.subtype.activation, el.tag_data.attr);\n            break;\n        }\n      } else {\n        handle_text(renderer, std::move(el.data));\n      }\n\n      if (old_alignment == m_ctxt->get_alignment()) {\n        double new_x = renderer.get_x(*m_ctxt);\n        if (new_x < old_x) {\n          m_action_ctxt.compensate_for_negative_move(old_alignment, old_x, new_x);\n        }\n      }\n    }\n\n    /*\n     * After rendering, we need to tell the action context about the position\n     * of the alignment blocks so that it can do intersection tests.\n     */\n    for (auto a : {alignment::LEFT, alignment::CENTER, alignment::RIGHT}) {\n      m_action_ctxt.set_alignment_start(a, renderer.get_alignment_start(a));\n    }\n    renderer.apply_tray_position(*m_ctxt);\n\n    auto num_unclosed = m_action_ctxt.num_unclosed();\n\n    if (num_unclosed != 0) {\n      throw runtime_error(to_string(num_unclosed) + \" unclosed action block(s)\");\n    }\n  }\n\n  /**\n   * Process text contents\n   */\n  void dispatch::handle_text(renderer_interface& renderer, string&& data) {\n#ifdef DEBUG_WHITESPACE\n    string::size_type p;\n    while ((p = data.find(' ')) != string::npos) {\n      data.replace(p, 1, \"-\"s);\n    }\n#endif\n\n    renderer.render_text(*m_ctxt, std::move(data));\n  }\n\n  void dispatch::handle_action(renderer_interface& renderer, mousebtn btn, bool closing, const string&& cmd) {\n    if (closing) {\n      m_action_ctxt.action_close(btn, m_ctxt->get_alignment(), renderer.get_x(*m_ctxt));\n    } else {\n      m_action_ctxt.action_open(btn, std::move(cmd), m_ctxt->get_alignment(), renderer.get_x(*m_ctxt));\n    }\n  }\n\n  void dispatch::handle_offset(renderer_interface& renderer, extent_val offset) {\n    renderer.render_offset(*m_ctxt, offset);\n  }\n\n  void dispatch::handle_alignment(renderer_interface& renderer, alignment a) {\n    m_ctxt->apply_alignment(a);\n    renderer.change_alignment(*m_ctxt);\n  }\n\n  void dispatch::handle_control(renderer_interface& renderer, controltag ctrl) {\n    switch (ctrl) {\n      case controltag::R:\n        m_ctxt->apply_reset();\n        break;\n      case controltag::t:\n        m_ctxt->store_tray_position(renderer.get_x(*m_ctxt));\n        break;\n      default:\n        throw runtime_error(\"Unrecognized polybar control tag: \" + to_string(static_cast<int>(ctrl)));\n    }\n  }\n\n} // namespace tags\n\nPOLYBAR_NS_END\n\n// Path: src/tags/context.cpp\n#include \"tags/context.hpp\"\n\nPOLYBAR_NS\n\nnamespace tags {\n  \nstatic rgba get_color(color_value c, rgba fallback) {\n    if (c.type == color_type::RESET) {\n      return fallback;\n    } else {\n      return c.val;\n    }\n  }\n\n  context::context(const bar_settings& settings) : m_settings(settings) {\n    reset();\n  }\n\n  void context::reset() {\n    apply_reset();\n    m_align = alignment::NONE;\n  }\n\n  void context::apply_bg(color_value c) {\n    m_bg = get_color(c, m_settings.background);\n  }\n\n  void context::apply_fg(color_value c) {\n    m_fg = get_color(c, m_settings.foreground);\n  }\n\n  void context::apply_ol(color_value c) {\n    m_ol = get_color(c, m_settings.overline.color);\n  }\n\n  void context::apply_ul(color_value c) {\n    m_ul = get_color(c, m_settings.underline.color);\n  }\n\n  void context::apply_font(int font) {\n    m_font = std::max(font, 0);\n  }\n\n  void context::apply_reverse() {\n    std::swap(m_bg, m_fg);\n  }\n\n  void context::apply_alignment(alignment align) {\n    m_align = align;\n  }\n\n  void context::apply_attr(attr_activation act, attribute attr) {\n    if (attr == attribute::NONE) {\n      return;\n    }\n\n    bool& current = attr == attribute::OVERLINE ? m_attr_overline : m_attr_underline;\n\n    switch (act) {\n      case attr_activation::ON:\n        current = true;\n        break;\n      case attr_activation::OFF:\n        current = false;\n        break;\n      case attr_activation::TOGGLE:\n        current = !current;\n        break;\n      default:\n        break;\n    }\n  }\n\n  void context::store_tray_position(int x_pos) {\n    m_relative_tray_position = std::make_pair(get_alignment(), x_pos);\n  }\n\n  void context::apply_reset() {\n    m_bg = m_settings.background;\n    m_fg = m_settings.foreground;\n    m_ul = m_settings.underline.color;\n    m_ol = m_settings.overline.color;\n    m_font = 0;\n    m_attr_overline = false;\n    m_attr_underline = false;\n  }\n\n  rgba context::get_bg() const {\n    return m_bg;\n  }\n\n  rgba context::get_fg() const {\n    return m_fg;\n  }\n\n  rgba context::get_ol() const {\n    return m_ol;\n  }\n\n  rgba context::get_ul() const {\n    return m_ul;\n  }\n\n  int context::get_font() const {\n    return m_font;\n  }\n\n  bool context::has_overline() const {\n    return m_attr_overline;\n  }\n\n  bool context::has_underline() const {\n    return m_attr_underline;\n  }\n\n  alignment context::get_alignment() const {\n    return m_align;\n  }\n\n  std::pair<alignment, int> context::get_relative_tray_position() const {\n    return m_relative_tray_position;\n  }\n}  // namespace tags\n\nPOLYBAR_NS_END\n\n// Path: src/cairo/utils.cpp\n#include \"cairo/utils.hpp\"\n\n#include <map>\n\nPOLYBAR_NS\n\nnamespace cairo {\nnamespace utils {\n\n  // implementation : device_lock {{{\n\n  device_lock::device_lock(cairo_device_t* device) {\n    auto status = cairo_device_acquire(device);\n    if (status == CAIRO_STATUS_SUCCESS) {\n      m_device = device;\n    }\n  }\n  device_lock::~device_lock() {\n    cairo_device_release(m_device);\n  }\n  device_lock::operator bool() const {\n    return m_device != nullptr;\n  }\n  device_lock::operator cairo_device_t*() const {\n    return m_device;\n  }\n\n  // }}}\n  // implementation : ft_face_lock {{{\n\n  ft_face_lock::ft_face_lock(cairo_scaled_font_t* font) : m_font(font) {\n    m_face = cairo_ft_scaled_font_lock_face(m_font);\n  }\n  ft_face_lock::~ft_face_lock() {\n    cairo_ft_scaled_font_unlock_face(m_font);\n  }\n  ft_face_lock::operator FT_Face() const {\n    return m_face;\n  }\n\n  // }}}\n\n  /**\n   * @see <cairo/cairo.h>\n   */\n  cairo_operator_t str2operator(const string& mode, cairo_operator_t fallback) {\n    if (mode.empty()) {\n      return fallback;\n    }\n    static std::map<string, cairo_operator_t> modes;\n    if (modes.empty()) {\n      modes[\"clear\"s] = CAIRO_OPERATOR_CLEAR;\n      modes[\"source\"s] = CAIRO_OPERATOR_SOURCE;\n      modes[\"over\"s] = CAIRO_OPERATOR_OVER;\n      modes[\"in\"s] = CAIRO_OPERATOR_IN;\n      modes[\"out\"s] = CAIRO_OPERATOR_OUT;\n      modes[\"atop\"s] = CAIRO_OPERATOR_ATOP;\n      modes[\"dest\"s] = CAIRO_OPERATOR_DEST;\n      modes[\"dest-over\"s] = CAIRO_OPERATOR_DEST_OVER;\n      modes[\"dest-in\"s] = CAIRO_OPERATOR_DEST_IN;\n      modes[\"dest-out\"s] = CAIRO_OPERATOR_DEST_OUT;\n      modes[\"dest-atop\"s] = CAIRO_OPERATOR_DEST_ATOP;\n      modes[\"xor\"s] = CAIRO_OPERATOR_XOR;\n      modes[\"add\"s] = CAIRO_OPERATOR_ADD;\n      modes[\"saturate\"s] = CAIRO_OPERATOR_SATURATE;\n      modes[\"multiply\"s] = CAIRO_OPERATOR_MULTIPLY;\n      modes[\"screen\"s] = CAIRO_OPERATOR_SCREEN;\n      modes[\"overlay\"s] = CAIRO_OPERATOR_OVERLAY;\n      modes[\"darken\"s] = CAIRO_OPERATOR_DARKEN;\n      modes[\"lighten\"s] = CAIRO_OPERATOR_LIGHTEN;\n      modes[\"color-dodge\"s] = CAIRO_OPERATOR_COLOR_DODGE;\n      modes[\"color-burn\"s] = CAIRO_OPERATOR_COLOR_BURN;\n      modes[\"hard-light\"s] = CAIRO_OPERATOR_HARD_LIGHT;\n      modes[\"soft-light\"s] = CAIRO_OPERATOR_SOFT_LIGHT;\n      modes[\"difference\"s] = CAIRO_OPERATOR_DIFFERENCE;\n      modes[\"exclusion\"s] = CAIRO_OPERATOR_EXCLUSION;\n      modes[\"hsl-hue\"s] = CAIRO_OPERATOR_HSL_HUE;\n      modes[\"hsl-saturation\"s] = CAIRO_OPERATOR_HSL_SATURATION;\n      modes[\"hsl-color\"s] = CAIRO_OPERATOR_HSL_COLOR;\n      modes[\"hsl-luminosity\"s] = CAIRO_OPERATOR_HSL_LUMINOSITY;\n    }\n    auto it = modes.find(mode);\n    return it != modes.end() ? it->second : fallback;\n  }\n} // namespace utils\n} // namespace cairo\n\nPOLYBAR_NS_END\n\n// Path: src/drawtypes/animation.cpp\n#include \"drawtypes/animation.hpp\"\n\n#include \"drawtypes/label.hpp\"\n\nPOLYBAR_NS\n\nnamespace drawtypes {\n  void animation::add(label_t&& frame) {\n    m_frames.emplace_back(forward<decltype(frame)>(frame));\n    m_framecount = m_frames.size();\n    m_frame = m_framecount - 1;\n  }\n\n  label_t animation::get() const {\n    return m_frames[m_frame];\n  }\n\n  unsigned int animation::framerate() const {\n    return m_framerate_ms;\n  }\n\n  animation::operator bool() const {\n    return !m_frames.empty();\n  }\n\n  void animation::increment() {\n    auto tmp = m_frame.load();\n    ++tmp;\n    tmp %= m_framecount;\n\n    m_frame = tmp;\n  }\n\n  /**\n   * Create an animation by loading values\n   * from the configuration\n   */\n  animation_t load_animation(const config& conf, const string& section, string name, bool required) {\n    vector<label_t> vec;\n    vector<string> frames;\n\n    name = string_util::ltrim(string_util::rtrim(move(name), '>'), '<');\n\n    auto anim_defaults = load_optional_label(conf, section, name);\n\n    if (required) {\n      frames = conf.get_list(section, name);\n    } else {\n      frames = conf.get_list(section, name, {});\n    }\n\n    for (size_t i = 0; i < frames.size(); i++) {\n      vec.emplace_back(forward<label_t>(load_optional_label(conf, section, name + \"-\" + to_string(i), frames[i])));\n      vec.back()->copy_undefined(anim_defaults);\n    }\n\n    auto framerate = conf.get(section, name + \"-framerate\", 1000);\n\n    return std::make_shared<animation>(move(vec), framerate);\n  }\n}  // namespace drawtypes\n\nPOLYBAR_NS_END\n\n// Path: src/drawtypes/ramp.cpp\n#include \"drawtypes/ramp.hpp\"\n\n#include \"utils/factory.hpp\"\n#include \"utils/math.hpp\"\n\nPOLYBAR_NS\n\nnamespace drawtypes {\n  void ramp::add(label_t&& icon) {\n    m_icons.emplace_back(forward<decltype(icon)>(icon));\n  }\n\n  void ramp::add(label_t&& icon, unsigned weight) {\n    while (weight--) {\n      m_icons.emplace_back(icon);\n    }\n  }\n\n  label_t ramp::get(size_t index) {\n    return m_icons[index];\n  }\n\n  label_t ramp::get_by_percentage(float percentage) {\n    size_t index = percentage * m_icons.size() / 100.0f;\n    return m_icons[math_util::cap<size_t>(index, 0, m_icons.size() - 1)];\n  }\n\n  label_t ramp::get_by_percentage_with_borders(int value, int min, int max) {\n    return get_by_percentage_with_borders(static_cast<float>(value), static_cast<float>(min), static_cast<float>(max));\n  }\n\n  label_t ramp::get_by_percentage_with_borders(float value, float min, float max) {\n    size_t index;\n    if (value <= min) {\n      index = 0;\n    } else if (value >= max) {\n      index = m_icons.size() - 1;\n    } else {\n      float percentage = math_util::percentage(value, min, max);\n      index = percentage * (m_icons.size() - 2) / 100.0f + 1;\n      index = math_util::cap<size_t>(index, 0, m_icons.size() - 1);\n    }\n    return m_icons[index];\n  }\n\n  ramp::operator bool() {\n    return !m_icons.empty();\n  }\n\n  /**\n   * Create a ramp by loading values\n   * from the configuration\n   */\n  ramp_t load_ramp(const config& conf, const string& section, string name, bool required) {\n    name = string_util::ltrim(string_util::rtrim(move(name), '>'), '<');\n\n    auto ramp_defaults = load_optional_label(conf, section, name);\n\n    vector<label_t> vec;\n    vector<string> icons;\n\n    if (required) {\n      icons = conf.get_list<string>(section, name);\n    } else {\n      icons = conf.get_list<string>(section, name, {});\n    }\n\n    for (size_t i = 0; i < icons.size(); i++) {\n      auto ramp_name = name + \"-\" + to_string(i);\n      auto icon = load_optional_label(conf, section, ramp_name, icons[i]);\n      icon->copy_undefined(ramp_defaults);\n\n      auto weight = conf.get(section, ramp_name + \"-weight\", 1U);\n      while (weight--) {\n        vec.emplace_back(icon);\n      }\n    }\n\n    return std::make_shared<drawtypes::ramp>(move(vec));\n  }\n}  // namespace drawtypes\n\nPOLYBAR_NS_END\n\n// Path: src/drawtypes/label.cpp\n#include \"drawtypes/label.hpp\"\n\n#include <cmath>\n#include <utility>\n\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace drawtypes {\n  /**\n   * Gets the text from the label as it should be rendered\n   *\n   * Here tokens are replaced with values and minlen and maxlen properties are applied\n   */\n  string label::get() const {\n    const size_t len = string_util::char_len(m_tokenized);\n    if (len >= m_minlen) {\n      string text = m_tokenized;\n      if (m_maxlen > 0 && len > m_maxlen) {\n        if (m_ellipsis) {\n          text = string_util::utf8_truncate(std::move(text), m_maxlen - 3) + \"...\";\n        } else {\n          text = string_util::utf8_truncate(std::move(text), m_maxlen);\n        }\n      }\n      return text;\n    }\n\n    const size_t num_fill_chars = m_minlen - len;\n    size_t right_fill_len = 0;\n    size_t left_fill_len = 0;\n    if (m_alignment == alignment::RIGHT) {\n      left_fill_len = num_fill_chars;\n    } else if (m_alignment == alignment::LEFT) {\n      right_fill_len = num_fill_chars;\n    } else {\n      right_fill_len = std::ceil(num_fill_chars / 2.0);\n      left_fill_len = right_fill_len;\n      // The text is positioned one character to the left if we can't perfectly center it\n      if (len + left_fill_len + right_fill_len > m_minlen) {\n        --left_fill_len;\n      }\n    }\n    return string(left_fill_len, ' ') + m_tokenized + string(right_fill_len, ' ');\n  }\n\n  label::operator bool() {\n...\n", "needle_token_start": 12001, "needle_token_end": 12057, "code_context_ntokens": 16046, "output": ["```cpp\nstatic rgba get_color(color_value c, rgba fallback) {\n    if (c.type == color_type::RESET) {\n      return fallback;\n    } else {\n      return c.val;\n    }\n  }\n```"]}
{"repo": "polybar/polybar", "name": "read", "language": "cpp", "path": "src/modules/battery.cpp", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To safely retrieve data from a data source using a reader object, ensuring thread safety during the operation.\n2. **Input**: A reference to a reader object that implements the reading mechanism.\n3. **Output**: The data read from the source, with the type depending on the implementation of the reader.\n4. **Procedure**: The function first locks the reader object to prevent concurrent access, then calls the reader's read method to fetch the data, and finally returns the fetched data.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/drawtypes/label.cpp\n#include \"drawtypes/label.hpp\"\n\n#include <cmath>\n#include <utility>\n\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace drawtypes {\n  /**\n   * Gets the text from the label as it should be rendered\n   *\n   * Here tokens are replaced with values and minlen and maxlen properties are applied\n   */\n  string label::get() const {\n    const size_t len = string_util::char_len(m_tokenized);\n    if (len >= m_minlen) {\n      string text = m_tokenized;\n      if (m_maxlen > 0 && len > m_maxlen) {\n        if (m_ellipsis) {\n          text = string_util::utf8_truncate(std::move(text), m_maxlen - 3) + \"...\";\n        } else {\n          text = string_util::utf8_truncate(std::move(text), m_maxlen);\n        }\n      }\n      return text;\n    }\n\n    const size_t num_fill_chars = m_minlen - len;\n    size_t right_fill_len = 0;\n    size_t left_fill_len = 0;\n    if (m_alignment == alignment::RIGHT) {\n      left_fill_len = num_fill_chars;\n    } else if (m_alignment == alignment::LEFT) {\n      right_fill_len = num_fill_chars;\n    } else {\n      right_fill_len = std::ceil(num_fill_chars / 2.0);\n      left_fill_len = right_fill_len;\n      // The text is positioned one character to the left if we can't perfectly center it\n      if (len + left_fill_len + right_fill_len > m_minlen) {\n        --left_fill_len;\n      }\n    }\n    return string(left_fill_len, ' ') + m_tokenized + string(right_fill_len, ' ');\n  }\n\n  label::operator bool() {\n    return !m_tokenized.empty();\n  }\n\n  label_t label::clone() {\n    vector<token> tokens;\n    if (!m_tokens.empty()) {\n      std::back_insert_iterator<decltype(tokens)> back_it(tokens);\n      std::copy(m_tokens.begin(), m_tokens.end(), back_it);\n    }\n    return std::make_shared<label>(m_text, m_foreground, m_background, m_underline, m_overline, m_font, m_padding,\n        m_margin, m_minlen, m_maxlen, m_alignment, m_ellipsis, move(tokens));\n  }\n\n  void label::clear() {\n    m_tokenized.clear();\n  }\n\n  void label::reset_tokens() {\n    m_tokenized = m_text;\n  }\n\n  void label::reset_tokens(const string& tokenized) {\n    m_tokenized = tokenized;\n  }\n\n  bool label::has_token(const string& token) const {\n    return m_tokenized.find(token) != string::npos;\n  }\n\n  void label::replace_token(const string& token, string replacement) {\n    if (!has_token(token)) {\n      return;\n    }\n\n    for (auto&& tok : m_tokens) {\n      string repl{replacement};\n      if (token == tok.token) {\n        if (tok.max != 0_z && string_util::char_len(repl) > tok.max) {\n          repl = string_util::utf8_truncate(std::move(repl), tok.max) + tok.suffix;\n        } else if (tok.min != 0_z && repl.length() < tok.min) {\n          if (tok.rpadding) {\n            repl.append(tok.min - repl.length(), ' ');\n          } else {\n            repl.insert(0_z, tok.min - repl.length(), tok.zpad ? '0' : ' ');\n          }\n        }\n\n        /*\n         * Only replace first occurence, so that the proper token objects can be used\n         */\n        m_tokenized = string_util::replace(m_tokenized, token, move(repl));\n      }\n    }\n  }\n\n  void label::replace_defined_values(const label_t& label) {\n    if (label->m_foreground.has_color()) {\n      m_foreground = label->m_foreground;\n    }\n    if (label->m_background.has_color()) {\n      m_background = label->m_background;\n    }\n    if (label->m_underline.has_color()) {\n      m_underline = label->m_underline;\n    }\n    if (label->m_overline.has_color()) {\n      m_overline = label->m_overline;\n    }\n    if (label->m_font != 0) {\n      m_font = label->m_font;\n    }\n    if (label->m_padding.left) {\n      m_padding.left = label->m_padding.left;\n    }\n    if (label->m_padding.right) {\n      m_padding.right = label->m_padding.right;\n    }\n    if (label->m_margin.left) {\n      m_margin.left = label->m_margin.left;\n    }\n    if (label->m_margin.right) {\n      m_margin.right = label->m_margin.right;\n    }\n    if (label->m_maxlen != 0_z) {\n      m_maxlen = label->m_maxlen;\n      m_ellipsis = label->m_ellipsis;\n    }\n  }\n\n  void label::copy_undefined(const label_t& label) {\n...\n// Path: src/drawtypes/iconset.cpp\n#include \"drawtypes/iconset.hpp\"\n\n#include <algorithm>\n\nPOLYBAR_NS\n\nnamespace drawtypes {\n  void iconset::add(string id, label_t&& icon) {\n    m_icons.emplace(id, forward<decltype(icon)>(icon));\n  }\n\n  bool iconset::has(const string& id) {\n    return m_icons.find(id) != m_icons.end();\n  }\n\n  label_t iconset::get(const string& id, const string& fallback_id, bool fuzzy_match) {\n    // Try to match exactly first\n    auto icon = m_icons.find(id);\n    if (icon != m_icons.end()) {\n      return icon->second;\n    }\n\n    // If fuzzy matching is turned on, try that first before returning the fallback.\n    if (fuzzy_match) {\n      // works by finding the *longest* matching icon to the given workspace id\n      size_t max_size = -1;\n      label_t max_label;\n\n      for (auto const& icon : m_icons) {\n        if (id.find(icon.first) != std::string::npos) {\n          if (icon.first.length() > max_size || !max_label) {\n            max_size = icon.first.length();\n            max_label = icon.second;\n          }\n        }\n      }\n\n      if (max_label) {\n        return max_label;\n      }\n    }\n\n    return m_icons.find(fallback_id)->second;\n  }\n\n  iconset::operator bool() {\n    return !m_icons.empty();\n  }\n} // namespace drawtypes\n\nPOLYBAR_NS_END\n\n// Path: src/drawtypes/progressbar.cpp\n#include \"drawtypes/progressbar.hpp\"\n\n#include <utility>\n\n#include \"drawtypes/label.hpp\"\n#include \"utils/color.hpp\"\n#include \"utils/math.hpp\"\n\nPOLYBAR_NS\n\nnamespace drawtypes {\n  progressbar::progressbar(const bar_settings& bar, int width, string format)\n      : m_builder(std::make_unique<builder>(bar)), m_format(move(format)), m_width(width) {}\n\n  void progressbar::set_fill(label_t&& fill) {\n    m_fill = forward<decltype(fill)>(fill);\n  }\n\n  void progressbar::set_empty(label_t&& empty) {\n    m_empty = forward<decltype(empty)>(empty);\n  }\n\n  void progressbar::set_indicator(label_t&& indicator) {\n    if (!m_indicator && indicator.get()) {\n      m_width--;\n    }\n    m_indicator = forward<decltype(indicator)>(indicator);\n  }\n\n  void progressbar::set_gradient(bool mode) {\n    m_gradient = mode;\n  }\n\n  void progressbar::set_colors(vector<rgba>&& colors) {\n    m_colors = forward<decltype(colors)>(colors);\n\n    m_colorstep = m_colors.empty() ? 1 : m_width / m_colors.size();\n  }\n\n  string progressbar::output(float percentage) {\n    string output{m_format};\n\n    // Get fill/empty widths based on percentage\n    unsigned int perc = math_util::cap(percentage, 0.0f, 100.0f);\n    unsigned int fill_width = math_util::percentage_to_value(perc, m_width);\n    unsigned int empty_width = m_width - fill_width;\n\n    // Output fill icons\n    fill(perc, fill_width);\n    output = string_util::replace_all(output, \"%fill%\", m_builder->flush());\n\n    // Output indicator icon\n    m_builder->node(m_indicator);\n    output = string_util::replace_all(output, \"%indicator%\", m_builder->flush());\n\n    // Output empty icons\n    m_builder->node_repeat(m_empty, empty_width);\n    output = string_util::replace_all(output, \"%empty%\", m_builder->flush());\n\n    return output;\n  }\n\n  void progressbar::fill(unsigned int perc, unsigned int fill_width) {\n    if (m_colors.empty()) {\n      m_builder->node_repeat(m_fill, fill_width);\n    } else if (m_gradient) {\n      size_t color = 0;\n      for (size_t i = 0; i < fill_width; i++) {\n        if (i % m_colorstep == 0 && color < m_colors.size()) {\n          m_fill->m_foreground = m_colors[color++];\n        }\n        m_builder->node(m_fill);\n      }\n    } else {\n      size_t color = math_util::percentage_to_value<size_t>(perc, m_colors.size() - 1);\n      m_fill->m_foreground = m_colors[color];\n      m_builder->node_repeat(m_fill, fill_width);\n    }\n  }\n\n  /**\n   * Create a progressbar by loading values\n   * from the configuration\n   */\n  progressbar_t load_progressbar(const bar_settings& bar, const config& conf, const string& section, string name) {\n    // Remove the start and end tag from the name in case a format tag is passed\n    name = string_util::ltrim(string_util::rtrim(move(name), '>'), '<');\n\n    string format = \"%fill%%indicator%%empty%\";\n    unsigned int width;\n\n    if ((format = conf.get(section, name + \"-format\", format)).empty()) {\n      throw application_error(\"Invalid format defined at [\" + section + \".\" + name + \"]\");\n    }\n    if ((width = conf.get<decltype(width)>(section, name + \"-width\")) < 1) {\n      throw application_error(\"Invalid width defined at [\" + section + \".\" + name + \"]\");\n    }\n\n    auto pbar = std::make_shared<progressbar>(bar, width, format);\n    pbar->set_gradient(conf.get(section, name + \"-gradient\", true));\n    pbar->set_colors(conf.get_list(section, name + \"-foreground\", vector<rgba>{}));\n\n    label_t icon_empty;\n    label_t icon_fill;\n    label_t icon_indicator;\n\n    if (format.find(\"%empty%\") != string::npos) {\n      icon_empty = load_label(conf, section, name + \"-empty\");\n    }\n    if (format.find(\"%fill%\") != string::npos) {\n      icon_fill = load_label(conf, section, name + \"-fill\");\n    }\n    if (format.find(\"%indicator%\") != string::npos) {\n      icon_indicator = load_label(conf, section, name + \"-indicator\");\n    }\n\n    // If a foreground/background color is defined for the indicator\n    // but not for the empty icon we use the bar's default colors to\n    // avoid color bleed\n    if (icon_empty && icon_indicator) {\n      if (icon_indicator->m_background.has_color() && !icon_empty->m_background.has_color()) {\n        icon_empty->m_background = bar.background;\n      }\n      if (icon_indicator->m_foreground.has_color() && !icon_empty->m_foreground.has_color()) {\n        icon_empty->m_foreground = bar.foreground;\n      }\n    }\n\n    pbar->set_empty(move(icon_empty));\n    pbar->set_fill(move(icon_fill));\n    pbar->set_indicator(move(icon_indicator));\n\n    return pbar;\n  }\n}  // namespace drawtypes\n\nPOLYBAR_NS_END\n\n// Path: src/drawtypes/layouticonset.cpp\n#include \"drawtypes/layouticonset.hpp\"\n\nPOLYBAR_NS\n\nnamespace drawtypes {\n  layouticonset::layouticonset(label_t&& default_icon) : m_default_icon(default_icon) {}\n\n  bool layouticonset::add(const string& layout, const string& variant, label_t&& icon) {\n    if (layout == VARIANT_ANY && variant == VARIANT_ANY) {\n      return false;\n    }\n    m_layout_icons.emplace_back(layout, variant, icon);\n    return true;\n  }\n\n  label_t layouticonset::get(const string& layout, const string& variant) const {\n    // The layout, variant are matched against defined icons in that order:\n    //     1. perfect match on layout and perfect match on variant (ex: us;Colemak;<icon>)\n    //     2. perfect match on layout and case insensitive search on variant (ex: us;coLEmAk;<icon>)\n    //     3. perfect match on layout and the any variant '_' (ex: us;<icon> or us;_;<icon>)\n    //     4. any layout for icon and perfect match on variant (ex: _;Colemak;<icon>)\n    //     5. any layout for icon and case insensitive search on variant (ex: _;coLEmAk;<icon>)\n    //     6. no match at all => default icon if defined\n\n    /*\n     * The minimal case that was matched.\n     * Once a case is matched, this is updated and no case with the same or higher number can be matched again.\n     */\n    int min_case = 6;\n\n    // Case 6: initializing with default\n    label_t icon = m_default_icon;\n\n    for (auto it : m_layout_icons) {\n      const string& icon_layout = std::get<0>(it);\n      const string& icon_variant = std::get<1>(it);\n      label_t icon_label = std::get<2>(it);\n\n      bool is_variant_match = icon_variant == variant;\n\n      bool is_variant_any = icon_variant == VARIANT_ANY;\n\n      bool is_variant_match_fuzzy =\n          !is_variant_any && !icon_variant.empty() && string_util::contains_ignore_case(variant, icon_variant);\n\n      // Which of the 6 match cases is matched here.\n      int current_case = 6;\n\n      if (icon_layout == layout) {\n        if (is_variant_match) {\n          // Case 1\n          current_case = 1;\n        } else if (is_variant_match_fuzzy) {\n          // Case 2\n          current_case = 2;\n        } else if (is_variant_any) {\n          // Case 3\n          current_case = 3;\n        }\n      } else if (icon_layout == VARIANT_ANY) {\n        if (is_variant_match) {\n          // Case 4\n          current_case = 4;\n        } else if (is_variant_match_fuzzy) {\n          // Case 5\n          current_case = 5;\n        }\n      }\n\n      /*\n       * We matched with a higher priority than before -> update icon.\n       */\n      if (current_case < min_case) {\n        icon = icon_label;\n        min_case = current_case;\n      }\n\n      if (current_case == 1) {\n        // Case 1: perfect match, we can break early\n        break;\n      }\n    }\n\n    return icon;\n  }\n\n  bool layouticonset::contains(const string& layout, const string& variant) const {\n    for (auto it : m_layout_icons) {\n      const string& icon_layout = std::get<0>(it);\n      const string& icon_variant = std::get<1>(it);\n      if (icon_layout == layout && icon_variant == variant) {\n        return true;\n      }\n    }\n    return false;\n  }\n}  // namespace drawtypes\n\nPOLYBAR_NS_END\n\n// Path: src/ipc/ipc.cpp\n#include \"ipc/ipc.hpp\"\n\n#include <sys/stat.h>\n#include <unistd.h>\n\n#include <cassert>\n\n#include \"components/eventloop.hpp\"\n#include \"components/logger.hpp\"\n#include \"errors.hpp\"\n#include \"events/signal.hpp\"\n#include \"events/signal_emitter.hpp\"\n#include \"ipc/encoder.hpp\"\n#include \"ipc/util.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nusing namespace eventloop;\n\nnamespace ipc {\n\n  /**\n   * Message types\n   */\n  static constexpr const char* ipc_command_prefix{\"cmd:\"};\n  static constexpr const char* ipc_hook_prefix{\"hook:\"};\n  static constexpr const char* ipc_action_prefix{\"action:\"};\n\n  /**\n   * Create instance\n   */\n  ipc::make_type ipc::make(loop& loop) {\n    return std::make_unique<ipc>(signal_emitter::make(), logger::make(), loop);\n  }\n\n  /**\n   * Construct ipc handler\n   */\n  ipc::ipc(signal_emitter& emitter, const logger& logger, loop& loop)\n      : m_sig(emitter), m_log(logger), m_loop(loop), m_socket(loop.handle<PipeHandle>()) {\n    m_pipe_path = string_util::replace(PATH_MESSAGING_FIFO, \"%pid%\", to_string(getpid()));\n\n    if (file_util::exists(m_pipe_path) && unlink(m_pipe_path.c_str()) == -1) {\n      throw system_error(\"Failed to remove ipc channel\");\n    }\n    if (mkfifo(m_pipe_path.c_str(), 0600) == -1) {\n      throw system_error(\"Failed to create ipc channel\");\n    }\n    m_log.info(\"Created legacy ipc fifo at '%s'\", m_pipe_path);\n\n    ipc_pipe = make_unique<fifo>(m_loop, *this, m_pipe_path);\n\n    string sock_path = get_socket_path(getpid());\n\n    m_log.info(\"Opening ipc socket at '%s'\", sock_path);\n    m_log.notice(\"Listening for IPC messages (PID: %d)\", getpid());\n    m_socket->bind(sock_path);\n    m_socket->listen(\n        4, [this]() { on_connection(); },\n        [this](const auto& e) {\n          m_log.err(\"libuv error while listening to IPC socket: %s\", uv_strerror(e.status));\n          m_socket->close();\n        });\n  }\n\n  /**\n   * Deconstruct ipc handler\n   */\n  ipc::~ipc() {\n    m_log.trace(\"ipc: Removing named pipe at: %s\", m_pipe_path);\n    if (unlink(m_pipe_path.c_str()) == -1) {\n      m_log.err(\"Failed to delete ipc named pipe: %s\", strerror(errno));\n    }\n  }\n\n  string ipc::get_socket_path(int pid) {\n    return ensure_runtime_path() + \"/ipc.\" + to_string(pid) + \".sock\";\n  }\n\n  bool ipc::trigger_ipc(v0::ipc_type type, const string& msg) {\n    switch (type) {\n      case v0::ipc_type::CMD:\n        m_log.info(\"Received ipc command: '%s'\", msg);\n        return m_sig.emit(signals::ipc::command{msg});\n      case v0::ipc_type::ACTION:\n        m_log.info(\"Received ipc action: '%s'\", msg);\n        return m_sig.emit(signals::ipc::action{msg});\n    }\n\n    assert(false);\n    return false;\n  }\n\n  void ipc::trigger_legacy_ipc(const string& msg) {\n    m_log.info(\"Received ipc message: '%s'\", msg);\n    if (msg.find(ipc_command_prefix) == 0) {\n      m_sig.emit(signals::ipc::command{msg.substr(strlen(ipc_command_prefix))});\n    } else if (msg.find(ipc_hook_prefix) == 0) {\n      m_sig.emit(signals::ipc::hook{msg.substr(strlen(ipc_hook_prefix))});\n    } else if (msg.find(ipc_action_prefix) == 0) {\n      m_sig.emit(signals::ipc::action{msg.substr(strlen(ipc_action_prefix))});\n    } else {\n      m_log.warn(\"Received unknown ipc message: (payload=%s)\", msg);\n    }\n  }\n\n  void ipc::on_connection() {\n    auto connection = make_unique<ipc::connection>(\n        m_loop, [this](ipc::connection& c, uint8_t, type_t type, const vector<uint8_t>& msg) {\n          vector<uint8_t> response;\n\n          if (type == to_integral(v0::ipc_type::ACTION) || type == to_integral(v0::ipc_type::CMD)) {\n            auto ipc_type = static_cast<v0::ipc_type>(type);\n            string str;\n            str.insert(str.end(), msg.begin(), msg.end());\n            if (trigger_ipc(ipc_type, str)) {\n              response = encode(TYPE_OK);\n            } else {\n              response = encode(TYPE_ERR, \"Error while executing ipc message, see polybar log for details.\");\n            }\n          } else {\n            response = encode(TYPE_ERR, \"Unrecognized IPC message type \" + to_string(type));\n          }\n          c.client_pipe->write(\n              response, [this, &c]() { remove_client(c); },\n              [this, &c](const auto& e) {\n                m_log.err(\"ipc: libuv error while writing to IPC socket: %s\", uv_strerror(e.status));\n                remove_client(c);\n              });\n        });\n\n    auto& c = *connection;\n    m_socket->accept(*c.client_pipe);\n\n    c.client_pipe->read_start(\n        [this, &c](const auto& e) {\n          try {\n            c.dec.on_read(reinterpret_cast<const uint8_t*>(e.data), e.len);\n          } catch (const decoder::error& e) {\n            m_log.err(\"ipc: Failed to decode IPC message (reason: %s)\", e.what());\n\n            c.client_pipe->write(\n                encode(TYPE_ERR, \"Invalid binary message format: \"s + e.what()), [this, &c]() { remove_client(c); },\n                [this, &c](const auto& e) {\n                  m_log.err(\"ipc: libuv error while writing to IPC socket: %s\", uv_strerror(e.status));\n                  remove_client(c);\n                });\n          }\n        },\n        [this, &c]() { remove_client(c); },\n        [this, &c](const auto& e) {\n          m_log.err(\"ipc: libuv error while listening to IPC socket: %s\", uv_strerror(e.status));\n          remove_client(c);\n        });\n\n    connections.emplace(std::move(connection));\n    m_log.info(\"ipc: New connection (%d clients)\", connections.size());\n  }\n\n  void ipc::remove_client(connection& conn) {\n    connections.erase(connections.find(conn));\n  }\n\n  ipc::connection::connection(loop& loop, cb msg_callback)\n      : client_pipe(loop.handle<PipeHandle>())\n      , dec(logger::make(), [this, msg_callback](uint8_t version, auto type, const auto& msg) {\n        msg_callback(*this, version, type, msg);\n      }) {}\n\n  ipc::connection::~connection() {\n    client_pipe->close();\n  }\n\n  ipc::fifo::fifo(loop& loop, ipc& ipc, const string& path) : pipe_handle(loop.handle<PipeHandle>()) {\n    int fd{};\n    if ((fd = open(path.c_str(), O_RDONLY | O_NONBLOCK)) == -1) {\n      throw system_error(\"Failed to open pipe '\" + path + \"'\");\n    }\n\n    pipe_handle->open(fd);\n    pipe_handle->read_start([&ipc](const auto& e) mutable { ipc.receive_data(string(e.data, e.len)); },\n        [&ipc]() { ipc.receive_eof(); },\n        [&ipc](const auto& e) mutable {\n          ipc.m_log.err(\"libuv error while listening to IPC channel: %s\", uv_strerror(e.status));\n          ipc.ipc_pipe.reset();\n        });\n  }\n\n  ipc::fifo::~fifo() {\n    pipe_handle->close();\n  }\n\n  /**\n   * Receive parts of an IPC message\n   */\n  void ipc::receive_data(const string& buf) {\n    m_pipe_buffer += buf;\n\n    m_log.warn(\"Using the named pipe at '%s' for ipc is deprecated, always use 'polybar-msg'\", m_pipe_path);\n  }\n\n  /**\n   * Called once the end of the message arrives.\n   */\n  void ipc::receive_eof() {\n    ipc_pipe = make_unique<fifo>(m_loop, *this, m_pipe_path);\n\n    if (m_pipe_buffer.empty()) {\n      return;\n    }\n\n    trigger_legacy_ipc(string_util::trim(std::move(m_pipe_buffer), '\\n'));\n    m_pipe_buffer.clear();\n  }\n} // namespace ipc\n\nPOLYBAR_NS_END\n\n// Path: src/ipc/decoder.cpp\n#include \"ipc/decoder.hpp\"\n\n#include <cassert>\n#include <cstring>\n\nPOLYBAR_NS\n\nnamespace ipc {\n  decoder::decoder(const logger& logger, cb callback) : callback(callback), m_log(logger) {}\n\n  void decoder::on_read(const uint8_t* data, size_t size) {\n    if (state == state::CLOSED) {\n      throw error(\"Decoder is closed\");\n    }\n\n    try {\n      process_data(data, size);\n    } catch (const error& e) {\n      close();\n      throw;\n    }\n  }\n\n  void decoder::process_data(const uint8_t* data, size_t size) {\n    m_log.trace(\"ipc: Received %zd bytes\", size);\n\n    size_t buf_pos = 0;\n    size_t remain = size;\n\n    while (remain > 0) {\n      if (state == state::HEADER) {\n        ssize_t num_read = process_header_data(data + buf_pos, remain);\n        assert(num_read > 0);\n        assert(remain >= (size_t)num_read);\n\n        buf_pos += num_read;\n        remain -= num_read;\n\n        /*\n         * If an empty message arrives, we need to explicitly trigger this because there is no further data that would\n         * call process_msg_data.\n         */\n        if (remain == 0 && state == state::PAYLOAD && to_read_buf == 0) {\n          ssize_t num_read_data = process_msg_data(data + buf_pos, remain);\n          assert(num_read_data == 0);\n          (void)num_read_data;\n        }\n\n      } else {\n        assert(to_read_header == 0);\n        ssize_t num_read = process_msg_data(data + buf_pos, remain);\n        assert(num_read > 0);\n        assert(remain >= (size_t)num_read);\n\n        buf_pos += num_read;\n        remain -= num_read;\n      }\n    }\n  }\n\n  void decoder::close() noexcept {\n    state = state::CLOSED;\n  }\n\n  bool decoder::closed() const {\n    return state == state::CLOSED;\n  }\n\n  /**\n   * If we are waiting for header data, read as many bytes as possible from the given buffer.\n   *\n   * @return Number of bytes processed.\n   * @throws decoder::error on message errors\n   */\n  ssize_t decoder::process_header_data(const uint8_t* data, size_t size) {\n    assert(state == state::HEADER);\n    assert(to_read_header > 0);\n\n    size_t num_read = std::min(size, to_read_header);\n\n    std::copy(data, data + num_read, header.d + HEADER_SIZE - to_read_header);\n    to_read_header -= num_read;\n\n    if (to_read_header == 0) {\n      uint8_t version = header.s.version;\n      uint32_t msg_size = header.s.size;\n\n      m_log.trace(\n          \"Received full ipc header (magic=%.*s version=%d size=%zd)\", MAGIC.size(), header.s.magic, version, msg_size);\n\n      if (memcmp(header.s.magic, MAGIC.data(), MAGIC.size()) != 0) {\n        throw error(\"Invalid magic header, expected '\" + MAGIC_STR + \"', got '\" +\n                    string(reinterpret_cast<const char*>(header.s.magic), MAGIC.size()) + \"'\");\n      }\n      if (version != VERSION) {\n        throw error(\"Unsupported message format version \" + to_string(version));\n      }\n\n      assert(buf.empty());\n      state = state::PAYLOAD;\n      to_read_buf = msg_size;\n    }\n\n    return num_read;\n  }\n\n  /**\n   * If we are waiting for message data, read as many bytes as possible from the given buffer.\n   *\n   * @return Number of bytes processed.\n   * @throws decoder::error on message errors\n   */\n  ssize_t decoder::process_msg_data(const uint8_t* data, size_t size) {\n    assert(state == state::PAYLOAD);\n\n    size_t num_read = std::min(size, to_read_buf);\n\n    buf.reserve(buf.size() + num_read);\n    for (size_t i = 0; i < num_read; i++) {\n      buf.push_back(data[i]);\n    }\n    to_read_buf -= num_read;\n\n    if (to_read_buf == 0) {\n      callback(header.s.version, header.s.type, buf);\n\n      state = state::HEADER;\n      to_read_header = HEADER_SIZE;\n      buf.clear();\n    }\n\n    return num_read;\n  }\n}  // namespace ipc\nPOLYBAR_NS_END\n\n// Path: src/ipc/util.cpp\n#include \"ipc/util.hpp\"\n\n#include <sys/stat.h>\n#include <unistd.h>\n\n#include \"errors.hpp\"\n#include \"utils/env.hpp\"\n#include \"utils/file.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace ipc {\n\n  static constexpr auto SUFFIX = \".sock\";\n  static constexpr auto XDG_RUNTIME_DIR = \"XDG_RUNTIME_DIR\";\n\n  string get_runtime_path() {\n    if (env_util::has(XDG_RUNTIME_DIR)) {\n      return env_util::get(\"XDG_RUNTIME_DIR\") + \"/polybar\";\n    } else {\n      return \"/tmp/polybar-\" + to_string(getuid());\n    }\n    return env_util::get(\"XDG_RUNTIME_DIR\", \"/tmp\") + \"/polybar\";\n  }\n\n  string ensure_runtime_path() {\n    string runtime_path = get_runtime_path();\n    if (!file_util::exists(runtime_path) && mkdir(runtime_path.c_str(), 0700) == -1) {\n      // It's possible the folder was created in the meantime, we have to check again.\n      if (!file_util::exists(runtime_path)) {\n        throw system_error(\"Failed to create ipc socket folders\");\n      }\n    }\n\n    return runtime_path;\n  }\n\n  string get_socket_path(const string& pid_string) {\n    return get_runtime_path() + \"/ipc.\" + pid_string + SUFFIX;\n  }\n\n  string get_socket_path(int pid) {\n    return get_socket_path(to_string(pid));\n  }\n\n  string get_glob_socket_path() {\n    return get_socket_path(\"*\");\n  }\n\n  int get_pid_from_socket(const string& path) {\n    if (!string_util::ends_with(path, SUFFIX)) {\n      return -1;\n    }\n\n    auto stripped = path.substr(0, path.size() - strlen(SUFFIX));\n    auto p = stripped.rfind('.');\n\n    if (p == string::npos) {\n      return -1;\n    }\n\n    try {\n      return std::stoi(stripped.substr(p + 1));\n    } catch (...) {\n      return -1;\n    }\n  }\n} // namespace ipc\n\nPOLYBAR_NS_END\n\n// Path: src/ipc/encoder.cpp\n#include \"ipc/encoder.hpp\"\n\n#include <cassert>\n#include <cstring>\n\nPOLYBAR_NS\n\nnamespace ipc {\n  template <typename V>\n  vector<uint8_t> encode(const type_t type, const V& payload) {\n    size_t total_size = HEADER_SIZE + payload.size();\n    std::vector<uint8_t> data(total_size);\n\n    auto* msg_header = reinterpret_cast<header*>(data.data());\n    std::copy(MAGIC.begin(), MAGIC.end(), msg_header->s.magic);\n    msg_header->s.version = VERSION;\n    msg_header->s.size = payload.size();\n    msg_header->s.type = type;\n\n    std::copy(payload.begin(), payload.end(), data.begin() + HEADER_SIZE);\n    return data;\n  }\n\n  vector<uint8_t> encode(const type_t type, const vector<uint8_t>& payload) {\n    return encode<vector<uint8_t>>(type, payload);\n  }\n\n  vector<uint8_t> encode(const type_t type, const string& payload) {\n    return encode<string>(type, payload);\n  }\n\n}  // namespace ipc\nPOLYBAR_NS_END\n\n// Path: src/modules/alsa.cpp\n#include \"modules/alsa.hpp\"\n\n#include \"adapters/alsa/control.hpp\"\n#include \"adapters/alsa/generic.hpp\"\n#include \"adapters/alsa/mixer.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"drawtypes/progressbar.hpp\"\n#include \"drawtypes/ramp.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"settings.hpp\"\n#include \"utils/math.hpp\"\n\nPOLYBAR_NS\n\nusing namespace alsa;\n\nnamespace modules {\n  template class module<alsa_module>;\n\n  alsa_module::alsa_module(const bar_settings& bar, string name_, const config& config)\n      : event_module<alsa_module>(bar, move(name_), config) {\n    if (m_handle_events) {\n      m_router->register_action(EVENT_DEC, [this]() { action_dec(); });\n      m_router->register_action(EVENT_INC, [this]() { action_inc(); });\n      m_router->register_action(EVENT_TOGGLE, [this]() { action_toggle(); });\n    }\n\n    // Load configuration values\n    m_mapped = m_conf.get(name(), \"mapped\", m_mapped);\n    m_interval = m_conf.get(name(), \"interval\", m_interval);\n    m_unmute_on_scroll = m_conf.get(name(), \"unmute-on-scroll\", m_unmute_on_scroll);\n\n    auto master_mixer_name = m_conf.get(name(), \"master-mixer\", \"Master\"s);\n    auto speaker_mixer_name = m_conf.get(name(), \"speaker-mixer\", \"\"s);\n    auto headphone_mixer_name = m_conf.get(name(), \"headphone-mixer\", \"\"s);\n\n    // m_soundcard_name: Master Soundcard Name\n    // s_soundcard_name: Speaker Soundcard Name\n    // h_soundcard_name: Headphone Soundcard Name\n    auto m_soundcard_name = m_conf.get(name(), \"master-soundcard\", \"default\"s);\n    auto s_soundcard_name = m_conf.get(name(), \"speaker-soundcard\", \"default\"s);\n    auto h_soundcard_name = m_conf.get(name(), \"headphone-soundcard\", \"default\"s);\n\n    if (!headphone_mixer_name.empty()) {\n      m_headphoneid = m_conf.get<decltype(m_headphoneid)>(name(), \"headphone-id\");\n    }\n\n    if (string_util::compare(speaker_mixer_name, \"master\")) {\n      throw module_error(\"Master mixer is already defined\");\n    }\n    if (string_util::compare(headphone_mixer_name, \"master\")) {\n      throw module_error(\"Master mixer is already defined\");\n    }\n\n    // Setup mixers\n    try {\n      if (!master_mixer_name.empty()) {\n        m_mixer[mixer::MASTER].reset(new mixer_t::element_type{move(master_mixer_name), move(m_soundcard_name)});\n      }\n      if (!speaker_mixer_name.empty()) {\n        m_mixer[mixer::SPEAKER].reset(new mixer_t::element_type{move(speaker_mixer_name), move(s_soundcard_name)});\n      }\n      if (!headphone_mixer_name.empty()) {\n        m_mixer[mixer::HEADPHONE].reset(new mixer_t::element_type{move(headphone_mixer_name), move(h_soundcard_name)});\n      }\n      if (m_mixer[mixer::HEADPHONE]) {\n        m_ctrl[control::HEADPHONE].reset(new control_t::element_type{m_headphoneid});\n      }\n      if (m_mixer.empty()) {\n        throw module_error(\"No configured mixers\");\n      }\n    } catch (const mixer_error& err) {\n      throw module_error(err.what());\n    } catch (const control_error& err) {\n      throw module_error(err.what());\n    }\n\n    // Add formats and elements\n    m_formatter->add(FORMAT_VOLUME, TAG_LABEL_VOLUME, {TAG_RAMP_VOLUME, TAG_LABEL_VOLUME, TAG_BAR_VOLUME});\n    m_formatter->add(FORMAT_MUTED, TAG_LABEL_MUTED, {TAG_RAMP_VOLUME, TAG_LABEL_MUTED, TAG_BAR_VOLUME});\n\n    if (m_formatter->has(TAG_BAR_VOLUME)) {\n      m_bar_volume = load_progressbar(m_bar, m_conf, name(), TAG_BAR_VOLUME);\n    }\n    if (m_formatter->has(TAG_LABEL_VOLUME, FORMAT_VOLUME)) {\n      m_label_volume = load_optional_label(m_conf, name(), TAG_LABEL_VOLUME, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_MUTED, FORMAT_MUTED)) {\n      m_label_muted = load_optional_label(m_conf, name(), TAG_LABEL_MUTED, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_RAMP_VOLUME)) {\n      m_ramp_volume = load_ramp(m_conf, name(), TAG_RAMP_VOLUME);\n      m_ramp_headphones = load_ramp(m_conf, name(), TAG_RAMP_HEADPHONES, false);\n    }\n  }\n\n  void alsa_module::teardown() {\n    m_mixer.clear();\n    m_ctrl.clear();\n    snd_config_update_free_global();\n  }\n\n  bool alsa_module::has_event() {\n    // Poll for mixer and control events\n    try {\n      if (m_mixer[mixer::MASTER] && m_mixer[mixer::MASTER]->wait(25)) {\n        return true;\n      }\n      if (m_mixer[mixer::SPEAKER] && m_mixer[mixer::SPEAKER]->wait(25)) {\n        return true;\n      }\n      if (m_mixer[mixer::HEADPHONE] && m_mixer[mixer::HEADPHONE]->wait(25)) {\n        return true;\n      }\n      if (m_ctrl[control::HEADPHONE] && m_ctrl[control::HEADPHONE]->wait(25)) {\n        return true;\n      }\n    } catch (const alsa_exception& e) {\n      m_log.err(\"%s: %s\", name(), e.what());\n    }\n\n    return false;\n  }\n\n  bool alsa_module::update() {\n    // Consume pending events\n    if (m_mixer[mixer::MASTER]) {\n      m_mixer[mixer::MASTER]->process_events();\n    }\n    if (m_mixer[mixer::SPEAKER]) {\n      m_mixer[mixer::SPEAKER]->process_events();\n    }\n    if (m_mixer[mixer::HEADPHONE]) {\n      m_mixer[mixer::HEADPHONE]->process_events();\n    }\n    if (m_ctrl[control::HEADPHONE]) {\n      m_ctrl[control::HEADPHONE]->process_events();\n    }\n\n    // Get volume, mute and headphone state\n    m_volume = 100;\n    m_muted = false;\n    m_headphones = false;\n\n    try {\n      if (m_mixer[mixer::MASTER]) {\n        m_volume = m_volume * (m_mapped ? m_mixer[mixer::MASTER]->get_normalized_volume() / 100.0f\n                                        : m_mixer[mixer::MASTER]->get_volume() / 100.0f);\n        m_muted = m_muted || m_mixer[mixer::MASTER]->is_muted();\n      }\n    } catch (const alsa_exception& err) {\n      m_log.err(\"%s: Failed to query master mixer (%s)\", name(), err.what());\n    }\n\n    try {\n      if (m_ctrl[control::HEADPHONE] && m_ctrl[control::HEADPHONE]->test_device_plugged()) {\n        m_headphones = true;\n        m_volume = m_volume * (m_mapped ? m_mixer[mixer::HEADPHONE]->get_normalized_volume() / 100.0f\n                                        : m_mixer[mixer::HEADPHONE]->get_volume() / 100.0f);\n        m_muted = m_muted || m_mixer[mixer::HEADPHONE]->is_muted();\n      }\n    } catch (const alsa_exception& err) {\n      m_log.err(\"%s: Failed to query headphone mixer (%s)\", name(), err.what());\n    }\n\n    try {\n      if (!m_headphones && m_mixer[mixer::SPEAKER]) {\n        m_volume = m_volume * (m_mapped ? m_mixer[mixer::SPEAKER]->get_normalized_volume() / 100.0f\n                                        : m_mixer[mixer::SPEAKER]->get_volume() / 100.0f);\n        m_muted = m_muted || m_mixer[mixer::SPEAKER]->is_muted();\n      }\n    } catch (const alsa_exception& err) {\n      m_log.err(\"%s: Failed to query speaker mixer (%s)\", name(), err.what());\n    }\n\n    // Replace label tokens\n    if (m_label_volume) {\n      m_label_volume->reset_tokens();\n      m_label_volume->replace_token(\"%percentage%\", to_string(m_volume));\n    }\n\n    if (m_label_muted) {\n      m_label_muted->reset_tokens();\n      m_label_muted->replace_token(\"%percentage%\", to_string(m_volume));\n    }\n\n    return true;\n  }\n\n  string alsa_module::get_format() const {\n    return m_muted ? FORMAT_MUTED : FORMAT_VOLUME;\n  }\n\n  string alsa_module::get_output() {\n    // Get the module output early so that\n    // the format prefix/suffix also gets wrapper\n    // with the cmd handlers\n    string output{module::get_output()};\n\n    if (m_handle_events) {\n      auto click_middle = m_conf.get(name(), \"click-middle\", \"\"s);\n      auto click_right = m_conf.get(name(), \"click-right\", \"\"s);\n\n      if (!click_middle.empty()) {\n        m_builder->action(mousebtn::MIDDLE, click_middle);\n      }\n\n      if (!click_right.empty()) {\n        m_builder->action(mousebtn::RIGHT, click_right);\n      }\n\n      m_builder->action(mousebtn::LEFT, *this, EVENT_TOGGLE, \"\");\n      m_builder->action(mousebtn::SCROLL_UP, *this, EVENT_INC, \"\");\n      m_builder->action(mousebtn::SCROLL_DOWN, *this, EVENT_DEC, \"\");\n    }\n\n    m_builder->node(output);\n\n    return m_builder->flush();\n  }\n\n  bool alsa_module::build(builder* builder, const string& tag) const {\n    if (tag == TAG_BAR_VOLUME) {\n      builder->node(m_bar_volume->output(m_volume));\n    } else if (tag == TAG_RAMP_VOLUME && (!m_headphones || !*m_ramp_headphones)) {\n      builder->node(m_ramp_volume->get_by_percentage(m_volume));\n    } else if (tag == TAG_RAMP_VOLUME && m_headphones && *m_ramp_headphones) {\n      builder->node(m_ramp_headphones->get_by_percentage(m_volume));\n    } else if (tag == TAG_LABEL_VOLUME) {\n      builder->node(m_label_volume);\n    } else if (tag == TAG_LABEL_MUTED) {\n      builder->node(m_label_muted);\n    } else {\n      return false;\n    }\n    return true;\n  }\n\n  void alsa_module::action_inc() {\n    change_volume(m_interval);\n  }\n\n  void alsa_module::action_dec() {\n    change_volume(-m_interval);\n  }\n\n  void alsa_module::action_toggle() {\n    if (!m_mixer[mixer::MASTER]) {\n      return;\n    }\n    const auto& mixers = get_mixers();\n    for (auto&& mixer : mixers) {\n      mixer->set_mute(m_muted || mixers[0]->is_muted());\n    }\n\n    action_epilogue(mixers);\n  }\n\n  void alsa_module::change_volume(int interval) {\n    if (!m_mixer[mixer::MASTER]) {\n      return;\n    }\n    const auto& mixers = get_mixers();\n    for (auto&& mixer : mixers) {\n      if (m_unmute_on_scroll) {\n        mixer->set_mute(true);\n      }\n      m_mapped ? mixer->set_normalized_volume(math_util::cap<float>(mixer->get_normalized_volume() + interval, 0, 100))\n               : mixer->set_volume(math_util::cap<float>(mixer->get_volume() + interval, 0, 100));\n    }\n    action_epilogue(mixers);\n  }\n\n  void alsa_module::action_epilogue(const vector<mixer_t>& mixers) {\n    for (auto&& mixer : mixers) {\n      if (mixer->wait(0)) {\n        mixer->process_events();\n      }\n    }\n  }\n\n  vector<mixer_t> alsa_module::get_mixers() {\n    vector<mixer_t> mixers;\n    bool headphones{m_headphones};\n\n    if (m_mixer[mixer::MASTER] && !m_mixer[mixer::MASTER]->get_name().empty()) {\n      mixers.emplace_back(std::make_shared<mixer_t::element_type>(\n          string{m_mixer[mixer::MASTER]->get_name()}, string{m_mixer[mixer::MASTER]->get_sound_card()}));\n    }\n    if (m_mixer[mixer::HEADPHONE] && !m_mixer[mixer::HEADPHONE]->get_name().empty() && headphones) {\n      mixers.emplace_back(std::make_shared<mixer_t::element_type>(\n          string{m_mixer[mixer::HEADPHONE]->get_name()}, string{m_mixer[mixer::HEADPHONE]->get_sound_card()}));\n    }\n    if (m_mixer[mixer::SPEAKER] && !m_mixer[mixer::SPEAKER]->get_name().empty() && !headphones) {\n      mixers.emplace_back(std::make_shared<mixer_t::element_type>(\n          string{m_mixer[mixer::SPEAKER]->get_name()}, string{m_mixer[mixer::SPEAKER]->get_sound_card()}));\n    }\n\n    return mixers;\n  }\n} // namespace modules\n\nPOLYBAR_NS_END\n\n// Path: src/modules/battery.cpp\n#include \"modules/battery.hpp\"\n\n#include \"drawtypes/animation.hpp\"\n#include \"drawtypes/label.hpp\"\n#include \"drawtypes/progressbar.hpp\"\n#include \"drawtypes/ramp.hpp\"\n#include \"modules/meta/base.inl\"\n#include \"utils/file.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/string.hpp\"\n\nPOLYBAR_NS\n\nnamespace modules {\n  template class module<battery_module>;\n\n  template <typename ValueReader>\n  \ntypename ValueReader::return_type read(ValueReader& reader) {\n    std::lock_guard<ValueReader> guard(reader);\n    return reader.read();\n  }\n\n  /**\n   * Bootstrap module by setting up required components\n   */\n  battery_module::battery_module(const bar_settings& bar, string name_, const config& config)\n      : inotify_module<battery_module>(bar, move(name_), config) {\n    // Load configuration values\n    m_fullat = std::min(m_conf.get(name(), \"full-at\", m_fullat), 100);\n    m_lowat = std::max(m_conf.get(name(), \"low-at\", m_lowat), 0);\n    m_interval = m_conf.get<decltype(m_interval)>(name(), \"poll-interval\", 5s);\n    m_lastpoll = chrono::steady_clock::now();\n\n    auto path_adapter = string_util::replace(PATH_ADAPTER, \"%adapter%\", m_conf.get(name(), \"adapter\", \"ADP1\"s)) + \"/\";\n    auto path_battery = string_util::replace(PATH_BATTERY, \"%battery%\", m_conf.get(name(), \"battery\", \"BAT0\"s)) + \"/\";\n\n    // Make state reader\n    if (file_util::exists((m_fstate = path_battery + \"status\"))) {\n      m_state_reader =\n          make_unique<state_reader>([=] { return file_util::contents(m_fstate).compare(0, 8, \"Charging\") == 0; });\n    } else if (file_util::exists((m_fstate = path_adapter + \"online\"))) {\n      m_state_reader = make_unique<state_reader>([=] { return file_util::contents(m_fstate).compare(0, 1, \"1\") == 0; });\n    } else {\n      throw module_error(\"No suitable way to get current charge state\");\n    }\n\n    // Make capacity reader\n    if ((m_fcapnow = file_util::pick({path_battery + \"charge_now\", path_battery + \"energy_now\"})).empty()) {\n      throw module_error(\"No suitable way to get current capacity value\");\n    } else if ((m_fcapfull = file_util::pick({path_battery + \"charge_full\", path_battery + \"energy_full\"})).empty()) {\n      throw module_error(\"No suitable way to get max capacity value\");\n    }\n\n    m_capacity_reader = make_unique<capacity_reader>([=] {\n      auto cap_now = std::strtoul(file_util::contents(m_fcapnow).c_str(), nullptr, 10);\n      auto cap_max = std::strtoul(file_util::contents(m_fcapfull).c_str(), nullptr, 10);\n      return math_util::percentage(cap_now, 0UL, cap_max);\n    });\n\n    // Make rate reader\n    if ((m_fvoltage = file_util::pick({path_battery + \"voltage_now\"})).empty()) {\n      throw module_error(\"No suitable way to get current voltage value\");\n    } else if ((m_frate = file_util::pick({path_battery + \"current_now\", path_battery + \"power_now\"})).empty()) {\n      throw module_error(\"No suitable way to get current charge rate value\");\n    }\n\n    m_rate_reader = make_unique<rate_reader>([this] {\n      unsigned long rate{std::strtoul(file_util::contents(m_frate).c_str(), nullptr, 10)};\n      unsigned long volt{std::strtoul(file_util::contents(m_fvoltage).c_str(), nullptr, 10) / 1000UL};\n      unsigned long now{std::strtoul(file_util::contents(m_fcapnow).c_str(), nullptr, 10)};\n      unsigned long max{std::strtoul(file_util::contents(m_fcapfull).c_str(), nullptr, 10)};\n      unsigned long cap{read(*m_state_reader) ? max - now : now};\n\n      if (rate && volt && cap) {\n        auto remaining = (cap / volt);\n        auto current_rate = (rate / volt);\n\n        if (remaining && current_rate) {\n          return 3600UL * remaining / current_rate;\n        }\n      }\n\n      return 0UL;\n    });\n\n    // Make consumption reader\n    m_consumption_reader = make_unique<consumption_reader>([this] {\n      float consumption;\n\n      // if the rate we found was the current, calculate power (P = I*V)\n      if (string_util::contains(m_frate, \"current_now\")) {\n        unsigned long current{std::strtoul(file_util::contents(m_frate).c_str(), nullptr, 10)};\n        unsigned long voltage{std::strtoul(file_util::contents(m_fvoltage).c_str(), nullptr, 10)};\n\n        consumption = ((voltage / 1000.0) * (current / 1000.0)) / 1e6;\n      } else {\n        // if it was power, just use as is\n        unsigned long power{std::strtoul(file_util::contents(m_frate).c_str(), nullptr, 10)};\n\n        consumption = power / 1e6;\n      }\n\n      // convert to string with 2 decimmal places\n      string rtn(16, '\\0'); // 16 should be plenty big. Cant see it needing more than 6/7..\n      auto written = std::snprintf(&rtn[0], rtn.size(), \"%.2f\", consumption);\n      rtn.resize(written);\n\n      return rtn;\n    });\n\n    // Load state and capacity level\n    m_state = current_state();\n    m_percentage = current_percentage();\n\n    // Add formats and elements\n    m_formatter->add(FORMAT_CHARGING, TAG_LABEL_CHARGING,\n        {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_ANIMATION_CHARGING, TAG_LABEL_CHARGING});\n    m_formatter->add(FORMAT_DISCHARGING, TAG_LABEL_DISCHARGING,\n        {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_ANIMATION_DISCHARGING, TAG_LABEL_DISCHARGING});\n    m_formatter->add_optional(FORMAT_LOW, {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_ANIMATION_LOW, TAG_LABEL_LOW});\n    m_formatter->add(FORMAT_FULL, TAG_LABEL_FULL, {TAG_BAR_CAPACITY, TAG_RAMP_CAPACITY, TAG_LABEL_FULL});\n\n    if (m_formatter->has(TAG_ANIMATION_CHARGING, FORMAT_CHARGING)) {\n      m_animation_charging = load_animation(m_conf, name(), TAG_ANIMATION_CHARGING);\n    }\n    if (m_formatter->has(TAG_ANIMATION_DISCHARGING, FORMAT_DISCHARGING)) {\n      m_animation_discharging = load_animation(m_conf, name(), TAG_ANIMATION_DISCHARGING);\n    }\n    if (m_formatter->has(TAG_ANIMATION_LOW, FORMAT_LOW)) {\n      m_animation_low = load_animation(m_conf, name(), TAG_ANIMATION_LOW);\n    }\n    if (m_formatter->has(TAG_BAR_CAPACITY)) {\n      m_bar_capacity = load_progressbar(m_bar, m_conf, name(), TAG_BAR_CAPACITY);\n    }\n    if (m_formatter->has(TAG_RAMP_CAPACITY)) {\n      m_ramp_capacity = load_ramp(m_conf, name(), TAG_RAMP_CAPACITY);\n    }\n    if (m_formatter->has(TAG_LABEL_CHARGING, FORMAT_CHARGING)) {\n      m_label_charging = load_optional_label(m_conf, name(), TAG_LABEL_CHARGING, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_DISCHARGING, FORMAT_DISCHARGING)) {\n      m_label_discharging = load_optional_label(m_conf, name(), TAG_LABEL_DISCHARGING, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_LOW, FORMAT_LOW)) {\n      m_label_low = load_optional_label(m_conf, name(), TAG_LABEL_LOW, \"%percentage%%\");\n    }\n    if (m_formatter->has(TAG_LABEL_FULL, FORMAT_FULL)) {\n      m_label_full = load_optional_label(m_conf, name(), TAG_LABEL_FULL, \"%percentage%%\");\n    }\n\n    // Create inotify watches\n    watch(m_fcapnow, IN_ACCESS);\n    watch(m_fstate, IN_ACCESS);\n\n    // Setup time if token is used\n    if ((m_label_charging && m_label_charging->has_token(\"%time%\")) ||\n...\n", "needle_token_start": 13684, "needle_token_end": 13725, "code_context_ntokens": 16130, "output": ["```cpp\ntemplate <typename ValueReader>\ntypename ValueReader::return_type read(ValueReader& reader) {\n    std::lock_guard<ValueReader> guard(reader);\n    return reader.read();\n}\n```"]}
{"repo": "polybar/polybar", "name": "query", "language": "cpp", "path": "src/x11/xembed.cpp", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To retrieve and check the XEMBED information of a specific window to determine its embedding properties.\n2. **Input**: Takes a connection to the X server and a window identifier.\n3. **Output**: Returns a boolean indicating whether valid XEMBED information was found and updates the XEMBED data structure with the retrieved information.\n4. **Procedure**: The function requests the XEMBED information property from the specified window. It checks if the property length is zero (indicating no data). If data is present, it extracts the XEMBED version and flags from the property, updates the XEMBED data structure, and returns true. If no data is found, it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/x11/legacy_tray_manager.cpp\n#include \"x11/legacy_tray_manager.hpp\"\n\n#include <xcb/xcb.h>\n#include <xcb/xcb_aux.h>\n#include <xcb/xcb_image.h>\n\n#include <thread>\n\n#include \"cairo/context.hpp\"\n#include \"cairo/surface.hpp\"\n#include \"errors.hpp\"\n#include \"events/signal.hpp\"\n#include \"utils/color.hpp\"\n#include \"utils/factory.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/memory.hpp\"\n#include \"utils/process.hpp\"\n#include \"utils/units.hpp\"\n#include \"x11/background_manager.hpp\"\n#include \"x11/connection.hpp\"\n#include \"x11/ewmh.hpp\"\n#include \"x11/icccm.hpp\"\n#include \"x11/window.hpp\"\n#include \"x11/winspec.hpp\"\n#include \"x11/xembed.hpp\"\n\n/**\n * Legacy tray implementation that does not run as part of a module.\n */\n\nPOLYBAR_NS\n\nnamespace legacy_tray {\n\n/**\n * Create instance\n */\ntray_manager::make_type tray_manager::make(const bar_settings& settings) {\n  return std::make_unique<tray_manager>(\n      connection::make(), signal_emitter::make(), logger::make(), background_manager::make(), settings);\n}\n\ntray_manager::tray_manager(connection& conn, signal_emitter& emitter, const logger& logger, background_manager& back,\n    const bar_settings& settings)\n    : m_connection(conn), m_sig(emitter), m_log(logger), m_background_manager(back), m_bar_opts(settings) {\n  m_connection.attach_sink(this, SINK_PRIORITY_TRAY);\n}\n\ntray_manager::~tray_manager() {\n  if (m_delaythread.joinable()) {\n    m_delaythread.join();\n  }\n  m_connection.detach_sink(this, SINK_PRIORITY_TRAY);\n  deactivate();\n}\n\nvoid tray_manager::setup(const config& conf, const string& tray_module_name) {\n  auto bs = conf.section();\n\n  for (const auto& deprecated : {\"tray-position\", \"tray-detached\", \"tray-maxsize\", \"tray-scale\", \"tray-background\",\n           \"tray-foreground\", \"tray-padding\", \"tray-offset-x\", \"tray-offset-y\"}) {\n    if (conf.has(bs, deprecated)) {\n      m_log.warn(\"tray: %s.%s is deprecated, use the dedicated tray module to display the system tray\", bs, deprecated);\n    }\n  }\n\n  string position = conf.get(bs, \"tray-position\", \"none\"s);\n\n  if (!position.empty() && position != \"none\" && !tray_module_name.empty()) {\n    m_log.err(\n        \"The tray position is manually defined (`tray-position`) and also set by the tray module (%s). `tray-position` \"\n        \"will be ignored\",\n        tray_module_name);\n  }\n\n  if (!tray_module_name.empty()) {\n    m_opts.tray_position = tray_postition::MODULE;\n    return;\n  } else if (position == \"left\") {\n    m_opts.tray_position = tray_postition::LEFT;\n  } else if (position == \"right\") {\n    m_opts.tray_position = tray_postition::RIGHT;\n  } else if (position == \"center\") {\n    m_opts.tray_position = tray_postition::CENTER;\n  } else if (position != \"none\") {\n    return m_log.err(\"Disabling tray manager (reason: Invalid position \\\"\" + position + \"\\\")\");\n  } else {\n    return;\n  }\n\n  m_opts.detached = conf.get(bs, \"tray-detached\", false);\n  m_opts.height = m_bar_opts.size.h;\n  m_opts.height -= m_bar_opts.borders.at(edge::BOTTOM).size;\n  m_opts.height -= m_bar_opts.borders.at(edge::TOP).size;\n  m_opts.height_fill = m_opts.height;\n\n  if (m_opts.height % 2 != 0) {\n    m_opts.height--;\n  }\n\n  auto maxsize = conf.get<unsigned int>(bs, \"tray-maxsize\", 16);\n  if (m_opts.height > maxsize) {\n    m_opts.spacing += (m_opts.height - maxsize) / 2;\n    m_opts.height = maxsize;\n  }\n\n  m_opts.width_max = m_bar_opts.size.w;\n  m_opts.width = m_opts.height;\n  m_opts.orig_y = m_bar_opts.pos.y + m_bar_opts.borders.at(edge::TOP).size;\n\n  // Apply user-defined scaling\n  auto scale = conf.get(bs, \"tray-scale\", 1.0);\n  m_opts.width *= scale;\n  m_opts.height_fill *= scale;\n\n  auto inner_area = m_bar_opts.inner_area(true);\n\n  switch (m_opts.tray_position) {\n    case tray_postition::NONE:\n      break;\n    case tray_postition::LEFT:\n      m_opts.orig_x = inner_area.x;\n      break;\n    case tray_postition::CENTER:\n      m_opts.orig_x = inner_area.x + inner_area.width / 2 - m_opts.width / 2;\n      break;\n    case tray_postition::RIGHT:\n      m_opts.orig_x = inner_area.x + inner_area.width;\n      break;\n    case tray_postition::MODULE:\n      break;\n  }\n\n  if (conf.has(bs, \"tray-transparent\")) {\n    m_log.warn(\"tray-transparent is deprecated, the tray always uses pseudo-transparency. Please remove it.\");\n  }\n\n  // Set user-defined foreground and background colors.\n  m_opts.background = conf.get(bs, \"tray-background\", m_bar_opts.background);\n  m_opts.foreground = conf.get(bs, \"tray-foreground\", m_bar_opts.foreground);\n\n  if (m_opts.background.alpha_i() != 255) {\n    m_log.trace(\"tray: enable transparency\");\n    m_opts.transparent = true;\n  }\n\n  // Add user-defined padding\n  m_opts.spacing += conf.get<unsigned int>(bs, \"tray-padding\", 0);\n\n  // Add user-defiend offset\n  auto offset_x = conf.get(bs, \"tray-offset-x\", percentage_with_offset{});\n  auto offset_y = conf.get(bs, \"tray-offset-y\", percentage_with_offset{});\n\n  int max_x;\n  int max_y;\n\n  if (m_opts.detached) {\n    max_x = m_bar_opts.monitor->w;\n    max_y = m_bar_opts.monitor->h;\n  } else {\n    max_x = inner_area.width;\n    max_y = inner_area.height;\n  }\n\n  m_opts.orig_x += units_utils::percentage_with_offset_to_pixel(offset_x, max_x, m_bar_opts.dpi_x);\n  m_opts.orig_y += units_utils::percentage_with_offset_to_pixel(offset_y, max_y, m_bar_opts.dpi_y);\n  ;\n  m_opts.rel_x = m_opts.orig_x - m_bar_opts.pos.x;\n  m_opts.rel_y = m_opts.orig_y - m_bar_opts.pos.y;\n\n  // Put the tray next to the bar in the window stack\n  m_opts.sibling = m_bar_opts.x_data.window;\n\n  // Activate the tray manager\n  query_atom();\n  activate();\n}\n\n/**\n * Get the settings container\n */\nconst tray_settings tray_manager::settings() const {\n  return m_opts;\n}\n\n/**\n * Activate systray management\n */\nvoid tray_manager::activate() {\n  if (m_activated) {\n    return;\n  }\n\n  m_log.info(\"Activating tray manager\");\n  m_activated = true;\n  m_opts.running = true;\n\n  m_sig.attach(this);\n\n  try {\n    create_window();\n    create_bg();\n    restack_window();\n    set_wm_hints();\n    set_tray_colors();\n  } catch (const exception& err) {\n    m_log.err(err.what());\n    m_log.err(\"Cannot activate tray manager... failed to setup window\");\n    m_activated = false;\n    return;\n  }\n\n  // Attempt to get control of the systray selection then\n  // notify clients waiting for a manager.\n  acquire_selection();\n\n  if (!m_acquired_selection) {\n    deactivate();\n    return;\n  }\n\n  // Send delayed notification\n  if (!m_firstactivation) {\n    notify_clients();\n  } else {\n    notify_clients_delayed();\n  }\n\n  m_firstactivation = false;\n}\n\n/**\n * Deactivate systray management\n */\nvoid tray_manager::deactivate(bool clear_selection) {\n  if (!m_activated) {\n    return;\n  }\n\n  m_log.info(\"Deactivating tray manager\");\n  m_activated = false;\n  m_opts.running = false;\n\n  m_sig.detach(this);\n\n  if (!m_connection.connection_has_error() && clear_selection && m_acquired_selection) {\n    m_log.trace(\"tray: Unset selection owner\");\n    m_connection.set_selection_owner(XCB_NONE, m_atom, XCB_CURRENT_TIME);\n  }\n\n  m_log.trace(\"tray: Unembed clients\");\n  m_clients.clear();\n\n  if (m_tray) {\n    m_log.trace(\"tray: Destroy window\");\n    m_connection.destroy_window(m_tray);\n  }\n  m_context.reset();\n  m_surface.reset();\n  if (m_pixmap) {\n    m_connection.free_pixmap(m_pixmap);\n  }\n  if (m_gc) {\n    m_connection.free_gc(m_pixmap);\n  }\n\n  m_tray = 0;\n  m_pixmap = 0;\n  m_gc = 0;\n  m_prevwidth = 0;\n  m_prevheight = 0;\n  m_opts.configured_x = 0;\n  m_opts.configured_y = 0;\n  m_opts.configured_w = 0;\n  m_opts.configured_h = 0;\n  m_opts.configured_slots = 0;\n  m_acquired_selection = false;\n  m_mapped = false;\n\n  m_connection.flush();\n\n  m_sig.emit(signals::eventqueue::notify_forcechange{});\n}\n\n/**\n * Reconfigure tray\n */\nvoid tray_manager::reconfigure() {\n  if (!m_tray) {\n    return;\n  } else if (m_mtx.try_lock()) {\n    std::unique_lock<mutex> guard(m_mtx, std::adopt_lock);\n\n    try {\n      reconfigure_clients();\n    } catch (const exception& err) {\n      m_log.err(\"Failed to reconfigure tray clients (%s)\", err.what());\n    }\n    try {\n      reconfigure_window();\n    } catch (const exception& err) {\n      m_log.err(\"Failed to reconfigure tray window (%s)\", err.what());\n    }\n    try {\n      reconfigure_bg();\n    } catch (const exception& err) {\n      m_log.err(\"Failed to reconfigure tray background (%s)\", err.what());\n    }\n\n    m_opts.configured_slots = mapped_clients();\n    guard.unlock();\n    refresh_window();\n    m_connection.flush();\n  }\n\n  m_sig.emit(signals::eventqueue::notify_forcechange{});\n}\n\n/**\n * Reconfigure container window\n */\nvoid tray_manager::reconfigure_window() {\n  m_log.trace(\"tray: Reconfigure window (mapped=%i, clients=%i)\", static_cast<bool>(m_mapped), m_clients.size());\n\n  if (!m_tray) {\n    return;\n  }\n\n  auto clients = mapped_clients();\n  if (!clients && m_mapped) {\n    m_log.trace(\"tray: Reconfigure window / unmap\");\n    m_connection.unmap_window_checked(m_tray);\n  } else if (clients && !m_mapped && !m_hidden) {\n    m_log.trace(\"tray: Reconfigure window / map\");\n    m_connection.map_window_checked(m_tray);\n  }\n\n  auto width = calculate_w();\n  auto x = calculate_x(width);\n\n  if (m_opts.transparent) {\n    xcb_rectangle_t rect{0, 0, calculate_w(), calculate_h()};\n    m_bg_slice = m_background_manager.observe(rect, m_tray);\n  }\n\n  if (width > 0) {\n    m_log.trace(\"tray: New window values, width=%d, x=%d\", width, x);\n\n    unsigned int mask = 0;\n    array<uint32_t, 32> values{};\n    xcb_params_configure_window_t params{};\n\n    XCB_AUX_ADD_PARAM(&mask, &params, width, width);\n    XCB_AUX_ADD_PARAM(&mask, &params, x, x);\n    connection::pack_values(mask, &params, values);\n    m_connection.configure_window_checked(m_tray, mask, values.data());\n  }\n\n  m_opts.configured_w = width;\n  m_opts.configured_x = x;\n}\n\n/**\n * Reconfigure clients\n */\nvoid tray_manager::reconfigure_clients() {\n  m_log.trace(\"tray: Reconfigure clients\");\n\n  int x = m_opts.spacing;\n\n  for (auto it = m_clients.rbegin(); it != m_clients.rend(); it++) {\n    auto client = *it;\n\n    try {\n      client->ensure_state();\n      client->reconfigure(x, calculate_client_y());\n\n      x += m_opts.width + m_opts.spacing;\n    } catch (const xpp::x::error::window& err) {\n      remove_client(client, false);\n    }\n  }\n}\n\n/**\n * Reconfigure root pixmap\n */\nvoid tray_manager::reconfigure_bg(bool realloc) {\n  if (!m_opts.transparent || m_clients.empty() || !m_mapped) {\n    return;\n  };\n\n  m_log.trace(\"tray: Reconfigure bg (realloc=%i)\", realloc);\n\n  if (!m_context) {\n    return m_log.err(\"tray: no context for drawing the background\");\n  }\n\n  cairo::surface* surface = m_bg_slice->get_surface();\n  if (!surface) {\n    return;\n  }\n\n  m_context->clear();\n  *m_context << CAIRO_OPERATOR_SOURCE << *m_surface;\n  cairo_set_source_surface(*m_context, *surface, 0, 0);\n  m_context->paint();\n  *m_context << CAIRO_OPERATOR_OVER << m_opts.background;\n  m_context->paint();\n}\n\n/**\n * Refresh the bar window by clearing it along with each client window\n */\nvoid tray_manager::refresh_window() {\n  if (!m_activated || !m_mapped || !m_mtx.try_lock()) {\n    return;\n  }\n\n  std::lock_guard<mutex> lock(m_mtx, std::adopt_lock);\n\n  m_log.trace(\"tray: Refreshing window\");\n\n  auto width = calculate_w();\n  auto height = calculate_h();\n\n  if (m_opts.transparent && !m_context) {\n    xcb_rectangle_t rect{0, 0, static_cast<uint16_t>(width), static_cast<uint16_t>(height)};\n    m_connection.poly_fill_rectangle(m_pixmap, m_gc, 1, &rect);\n  }\n\n  if (m_surface) {\n    m_surface->flush();\n  }\n\n  m_connection.clear_area(0, m_tray, 0, 0, width, height);\n\n  for (auto&& client : m_clients) {\n    try {\n      if (client->mapped()) {\n        client->clear_window();\n      }\n    } catch (const std::exception& e) {\n      m_log.err(\"Failed to clear tray client %s '%s' (%s)\", m_connection.id(client->window()),\n          ewmh_util::get_wm_name(client->window()), e.what());\n    }\n  }\n\n  m_connection.flush();\n\n  if (!mapped_clients()) {\n    m_opts.configured_w = 0;\n  } else {\n    m_opts.configured_w = width;\n  }\n}\n\n/**\n * Redraw window\n */\nvoid tray_manager::redraw_window(bool realloc_bg) {\n  m_log.info(\"Redraw tray container (id=%s)\", m_connection.id(m_tray));\n  reconfigure_bg(realloc_bg);\n  refresh_window();\n}\n\n/**\n * Find the systray selection atom\n */\nvoid tray_manager::query_atom() {\n  m_log.trace(\"tray: Find systray selection atom for the default screen\");\n  string name{\"_NET_SYSTEM_TRAY_S\" + to_string(m_connection.default_screen())};\n  auto reply = m_connection.intern_atom(false, name.length(), name.c_str());\n  m_atom = reply.atom();\n}\n\n/**\n * Create tray window\n */\nvoid tray_manager::create_window() {\n  m_log.trace(\"tray: Create tray window\");\n\n  // clang-format off\n  auto win = winspec(m_connection, m_tray)\n    << cw_size(calculate_w(), calculate_h())\n    << cw_pos(calculate_x(calculate_w()), calculate_y())\n    << cw_class(XCB_WINDOW_CLASS_INPUT_OUTPUT)\n    << cw_params_backing_store(XCB_BACKING_STORE_WHEN_MAPPED)\n    << cw_params_event_mask(XCB_EVENT_MASK_SUBSTRUCTURE_REDIRECT\n        |XCB_EVENT_MASK_STRUCTURE_NOTIFY\n        |XCB_EVENT_MASK_EXPOSURE)\n    << cw_params_override_redirect(true);\n  // clang-format on\n\n  if (!m_opts.transparent) {\n    win << cw_params_back_pixel(m_opts.background);\n    win << cw_params_border_pixel(m_opts.background);\n  }\n\n  m_tray = win << cw_flush(true);\n  m_log.info(\"Tray window: %s\", m_connection.id(m_tray));\n\n  // activate the background manager if we have transparency\n  if (m_opts.transparent) {\n    xcb_rectangle_t rect{0, 0, calculate_w(), calculate_h()};\n    m_bg_slice = m_background_manager.observe(rect, m_tray);\n  }\n\n  const unsigned int shadow{0};\n  m_connection.change_property(XCB_PROP_MODE_REPLACE, m_tray, _COMPTON_SHADOW, XCB_ATOM_CARDINAL, 32, 1, &shadow);\n}\n\n/**\n * Create tray window background components\n */\nvoid tray_manager::create_bg(bool realloc) {\n  if (!m_opts.transparent) {\n    return;\n  }\n  if (!realloc && m_pixmap && m_gc && m_surface && m_context) {\n    return;\n  }\n  if (realloc && m_pixmap) {\n    m_connection.free_pixmap(m_pixmap);\n    m_pixmap = 0;\n  }\n  if (realloc && m_gc) {\n    m_connection.free_gc(m_gc);\n    m_gc = 0;\n  }\n\n  if (realloc && m_surface) {\n    m_surface.reset();\n  }\n  if (realloc && m_context) {\n    m_context.reset();\n  }\n\n  auto w = m_opts.width_max;\n  auto h = calculate_h();\n\n  if (!m_pixmap) {\n    try {\n      m_pixmap = m_connection.generate_id();\n      m_connection.create_pixmap_checked(m_connection.screen()->root_depth, m_pixmap, m_tray, w, h);\n    } catch (const exception& err) {\n      return m_log.err(\"Failed to create pixmap for tray background (err: %s)\", err.what());\n    }\n  }\n\n  if (!m_gc) {\n    try {\n      xcb_params_gc_t params{};\n      unsigned int mask = 0;\n      array<uint32_t, 32> values{};\n      XCB_AUX_ADD_PARAM(&mask, &params, graphics_exposures, 1);\n      connection::pack_values(mask, &params, values);\n      m_gc = m_connection.generate_id();\n      m_connection.create_gc_checked(m_gc, m_pixmap, mask, values.data());\n    } catch (const exception& err) {\n      return m_log.err(\"Failed to create gcontext for tray background (err: %s)\", err.what());\n    }\n  }\n\n  if (!m_surface) {\n    xcb_visualtype_t* visual = m_connection.visual_type_for_id(m_connection.screen()->root_visual);\n    if (!visual) {\n      return m_log.err(\"Failed to get root visual for tray background\");\n    }\n    m_surface = make_unique<cairo::xcb_surface>(m_connection, m_pixmap, visual, w, h);\n  }\n\n  if (!m_context) {\n    m_context = make_unique<cairo::context>(*m_surface, m_log);\n    m_context->clear();\n    *m_context << CAIRO_OPERATOR_SOURCE << m_opts.background;\n    m_context->paint();\n  }\n\n  try {\n    m_connection.change_window_attributes_checked(m_tray, XCB_CW_BACK_PIXMAP, &m_pixmap);\n  } catch (const exception& err) {\n    m_log.err(\"Failed to set tray window back pixmap (%s)\", err.what());\n  }\n}\n\n/**\n * Put tray window above the defined sibling in the window stack\n */\nvoid tray_manager::restack_window() {\n  if (m_opts.sibling == XCB_NONE) {\n    return;\n  }\n\n  try {\n    m_log.trace(\"tray: Restacking tray window\");\n\n    unsigned int mask = 0;\n    array<uint32_t, 32> values{};\n    xcb_params_configure_window_t params{};\n\n    XCB_AUX_ADD_PARAM(&mask, &params, sibling, m_opts.sibling);\n    XCB_AUX_ADD_PARAM(&mask, &params, stack_mode, XCB_STACK_MODE_ABOVE);\n\n    connection::pack_values(mask, &params, values);\n    m_connection.configure_window_checked(m_tray, mask, values.data());\n  } catch (const exception& err) {\n    auto id = m_connection.id(m_opts.sibling);\n    m_log.err(\"tray: Failed to put tray above %s in the stack (%s)\", id, err.what());\n  }\n}\n\n/**\n * Set window WM hints\n */\nvoid tray_manager::set_wm_hints() {\n  const unsigned int visual{m_connection.screen()->root_visual};\n  const unsigned int orientation{_NET_SYSTEM_TRAY_ORIENTATION_HORZ};\n\n  m_log.trace(\"bar: Set window WM_NAME / WM_CLASS\");\n  icccm_util::set_wm_name(m_connection, m_tray, TRAY_WM_NAME, 19_z, TRAY_WM_CLASS, 12_z);\n\n  m_log.trace(\"tray: Set window WM_PROTOCOLS\");\n  icccm_util::set_wm_protocols(m_connection, m_tray, {WM_DELETE_WINDOW, WM_TAKE_FOCUS});\n\n  m_log.trace(\"tray: Set window _NET_WM_WINDOW_TYPE\");\n  ewmh_util::set_wm_window_type(m_tray, {_NET_WM_WINDOW_TYPE_DOCK, _NET_WM_WINDOW_TYPE_NORMAL});\n\n  m_log.trace(\"tray: Set window _NET_WM_STATE\");\n  ewmh_util::set_wm_state(m_tray, {_NET_WM_STATE_SKIP_TASKBAR});\n\n  m_log.trace(\"tray: Set window _NET_WM_PID\");\n  ewmh_util::set_wm_pid(m_tray);\n\n  m_log.trace(\"tray: Set window _NET_SYSTEM_TRAY_VISUAL\");\n  xcb_change_property(\n      m_connection, XCB_PROP_MODE_REPLACE, m_tray, _NET_SYSTEM_TRAY_VISUAL, XCB_ATOM_VISUALID, 32, 1, &visual);\n\n  m_log.trace(\"tray: Set window _NET_SYSTEM_TRAY_ORIENTATION\");\n  xcb_change_property(m_connection, XCB_PROP_MODE_REPLACE, m_tray, _NET_SYSTEM_TRAY_ORIENTATION,\n      _NET_SYSTEM_TRAY_ORIENTATION, 32, 1, &orientation);\n}\n\n/**\n * Set color atom used by clients when determing icon theme\n */\nvoid tray_manager::set_tray_colors() {\n  m_log.trace(\"tray: Set _NET_SYSTEM_TRAY_COLORS to %x\", m_opts.foreground);\n\n  auto r = m_opts.foreground.red_i();\n  auto g = m_opts.foreground.green_i();\n  auto b = m_opts.foreground.blue_i();\n\n  const uint16_t r16 = (r << 8) | r;\n  const uint16_t g16 = (g << 8) | g;\n  const uint16_t b16 = (b << 8) | b;\n\n  const uint32_t colors[12] = {\n      r16, g16, b16, // normal\n      r16, g16, b16, // error\n      r16, g16, b16, // warning\n      r16, g16, b16, // success\n  };\n\n  m_connection.change_property(\n      XCB_PROP_MODE_REPLACE, m_tray, _NET_SYSTEM_TRAY_COLORS, XCB_ATOM_CARDINAL, 32, 12, colors);\n}\n\n/**\n * Acquire the systray selection\n */\n...\n// Path: src/x11/tray_manager.cpp\n#include \"x11/tray_manager.hpp\"\n\n#include <xcb/xcb_image.h>\n\n#include <thread>\n#include <utility>\n\n#include \"cairo/context.hpp\"\n#include \"cairo/surface.hpp\"\n#include \"components/config.hpp\"\n#include \"errors.hpp\"\n#include \"events/signal.hpp\"\n#include \"utils/color.hpp\"\n#include \"utils/factory.hpp\"\n#include \"utils/math.hpp\"\n#include \"utils/memory.hpp\"\n#include \"utils/process.hpp\"\n#include \"utils/units.hpp\"\n#include \"x11/ewmh.hpp\"\n#include \"x11/icccm.hpp\"\n#include \"x11/window.hpp\"\n#include \"x11/winspec.hpp\"\n#include \"x11/xembed.hpp\"\n\n/*\n * Tray implementation according to the System Tray Protocol.\n *\n * Ref: https://specifications.freedesktop.org/systemtray-spec/systemtray-spec-latest.html\n *\n * This class manages embedded tray icons by placing them on the bar in the correct positions; the start position is\n * requested by the renderer.\n *\n * The tray manager needs to trigger bar updates only when the size of the entire tray changes (e.g. when tray icons are\n * added/removed). Everything else can be handled without an update.\n */\n\nPOLYBAR_NS\n\nnamespace tray {\n\nmanager::manager(\n    connection& conn, signal_emitter& emitter, const logger& logger, const bar_settings& bar_opts, on_update on_update)\n    : m_connection(conn), m_sig(emitter), m_log(logger), m_bar_opts(bar_opts), m_on_update(std::move(on_update)) {\n  m_connection.attach_sink(this, SINK_PRIORITY_TRAY);\n}\n\nmanager::~manager() {\n  m_connection.detach_sink(this, SINK_PRIORITY_TRAY);\n  deactivate();\n}\n\nvoid manager::setup(const config& conf, const string& section_name) {\n  unsigned bar_height = m_bar_opts.inner_area().height;\n\n  // Spacing between icons\n  auto spacing = conf.get(section_name, \"tray-spacing\", ZERO_PX_EXTENT);\n  m_opts.spacing = units_utils::extent_to_pixel_nonnegative(spacing, m_bar_opts.dpi_x);\n\n  // Padding before and after each icon\n  auto padding = conf.get(section_name, \"tray-padding\", ZERO_PX_EXTENT);\n  m_opts.padding = units_utils::extent_to_pixel_nonnegative(padding, m_bar_opts.dpi_x);\n\n  auto size = conf.get(section_name, \"tray-size\", percentage_with_offset{66., ZERO_PX_EXTENT});\n  unsigned client_height = std::min(\n      bar_height, units_utils::percentage_with_offset_to_pixel_nonnegative(size, bar_height, m_bar_opts.dpi_y));\n\n  if (client_height == 0) {\n    m_log.warn(\"tray: tray-size has an effective value of 0px, you will not see any tray icons\");\n  }\n\n  m_opts.client_size = {client_height, client_height};\n\n  // Set user-defined foreground and background colors.\n  m_opts.background = conf.get(section_name, \"tray-background\", m_bar_opts.background);\n  m_opts.foreground = conf.get(section_name, \"tray-foreground\", m_bar_opts.foreground);\n\n  m_opts.selection_owner = m_bar_opts.x_data.window;\n\n  m_log.info(\"tray: spacing=%upx padding=%upx size=%upx\", m_opts.spacing, m_opts.padding, client_height);\n\n  if (m_bar_opts.x_data.window == XCB_NONE) {\n    m_log.err(\"tray: No bar window found, disabling tray\");\n    return;\n  }\n\n  // Activate the tray manager\n  query_atom();\n  activate();\n}\n\nunsigned manager::get_width() const {\n  return m_tray_width;\n}\n\nbool manager::is_active() const {\n  return m_state == state::ACTIVE;\n}\n\nbool manager::is_inactive() const {\n  return m_state == state::INACTIVE;\n}\n\nbool manager::is_waiting() const {\n  return m_state == state::WAITING;\n}\n\nbool manager::is_visible() const {\n  return is_active() && !m_hidden;\n}\n\n/**\n * Activate systray management\n */\nvoid manager::activate() {\n  if (is_active()) {\n    return;\n  }\n\n  m_log.info(\"tray: Activating tray manager\");\n\n  try {\n    set_tray_colors();\n    set_tray_orientation();\n  } catch (const exception& err) {\n    m_log.err(err.what());\n    m_log.err(\"Cannot activate tray manager... failed to setup window\");\n    deactivate();\n    return;\n  }\n\n  // Attempt to get control of the systray selection\n  xcb_window_t other_owner = XCB_NONE;\n  if (!acquire_selection(other_owner)) {\n    // Transition to WAITING state\n    wait_for_selection(other_owner);\n    return;\n  }\n\n  m_sig.attach(this);\n\n  m_othermanager = XCB_NONE;\n\n  m_state = state::ACTIVE;\n\n  notify_clients();\n}\n\n/**\n * Transitions tray manager to WAITING state\n *\n * @param other window id for current selection owner\n */\nvoid manager::wait_for_selection(xcb_window_t other) {\n  if (is_waiting() || other == XCB_NONE) {\n    return;\n  }\n\n  m_log.info(\"tray: Waiting for systray selection (current owner: %s)\", m_connection.id(other));\n\n  m_sig.detach(this);\n\n  m_othermanager = other;\n  track_selection_owner(other);\n\n  m_log.trace(\"tray: Unembed clients\");\n  m_clients.clear();\n\n  m_connection.flush();\n\n  m_state = state::WAITING;\n\n  recalculate_width();\n}\n\n/**\n * Deactivate systray management\n */\nvoid manager::deactivate() {\n  if (is_inactive()) {\n    return;\n  }\n\n  m_log.info(\"tray: Deactivating tray manager\");\n\n  m_sig.detach(this);\n\n  // Unset selection owner if we currently own the atom\n  if (!m_connection.connection_has_error() && is_active()) {\n    m_log.trace(\"tray: Unset selection owner\");\n    m_connection.set_selection_owner(XCB_NONE, m_atom, XCB_CURRENT_TIME);\n  }\n\n  m_log.trace(\"tray: Unembed clients\");\n  m_clients.clear();\n\n  m_connection.flush();\n\n  m_othermanager = XCB_NONE;\n\n  m_state = state::INACTIVE;\n\n  recalculate_width();\n}\n\n/**\n * Reconfigure tray\n */\nvoid manager::reconfigure() {\n  if (!m_opts.selection_owner) {\n    return;\n  }\n\n  try {\n    reconfigure_clients();\n  } catch (const exception& err) {\n    m_log.err(\"Failed to reconfigure tray clients (%s)\", err.what());\n  }\n\n  m_connection.flush();\n}\n\n/**\n * Calculates the total width of the tray and potentially runs the update hook.\n *\n * Should be called whenever the number of mapped clients changes\n */\nvoid manager::recalculate_width() {\n  m_log.trace(\"tray: Reconfigure window (hidden=%i, clients=%i)\", m_hidden, m_clients.size());\n  unsigned new_width = calculate_w();\n  if (m_tray_width != new_width) {\n    m_tray_width = new_width;\n    m_log.trace(\"tray: new width (width: %d, clients: %d)\", m_tray_width, m_clients.size());\n    m_on_update();\n  }\n}\n\n/**\n * Reconfigure client positions and mapped state\n */\nvoid manager::reconfigure_clients() {\n  m_log.trace(\"tray: Reconfigure clients\");\n\n  // X-position of the start of the tray area\n  int base_x = calculate_x();\n\n  // X-position of the end of the previous tray icon (including padding)\n  unsigned x = 0;\n\n  bool has_error = false;\n\n  unsigned count = 0;\n\n  for (auto& client : m_clients) {\n    try {\n      client->ensure_state();\n\n      if (client->mapped()) {\n        // Calculate start of tray icon\n        unsigned client_x = x + (count > 0 ? m_opts.spacing : 0) + m_opts.padding;\n        client->set_position(base_x + client_x, calculate_client_y());\n        // Add size and padding to get the end position of the icon\n        x = client_x + m_opts.client_size.w + m_opts.padding;\n        count++;\n      }\n    } catch (const xpp::x::error::window& err) {\n      m_log.err(\"Failed to reconfigure %s, removing ... (%s)\", client->name(), err.what());\n      client.reset();\n      has_error = true;\n    }\n  }\n\n  if (has_error) {\n    clean_clients();\n  }\n\n  // Some clients may have been (un)mapped or even removed\n  recalculate_width();\n  // The final x position should match the width of the entire tray\n  assert(x == m_tray_width);\n}\n\n/**\n * Redraw client windows.\n */\nvoid manager::redraw_clients() {\n  if (!is_visible()) {\n    return;\n  }\n\n  m_log.trace(\"tray: Refreshing clients\");\n\n  for (auto& client : m_clients) {\n    try {\n      if (client->mapped()) {\n        client->update_bg();\n      }\n    } catch (const std::exception& e) {\n      m_log.err(\"tray: Failed to clear %s (%s)\", client->name(), e.what());\n    }\n  }\n\n  m_connection.flush();\n}\n\n/**\n * Find the systray selection atom\n */\nvoid manager::query_atom() {\n  m_log.trace(\"tray: Find systray selection atom for the default screen\");\n  string name{\"_NET_SYSTEM_TRAY_S\" + to_string(m_connection.default_screen())};\n  auto reply = m_connection.intern_atom(false, name.length(), name.c_str());\n  m_atom = reply.atom();\n}\n\n/**\n * Set _NET_SYSTEM_TRAY_COLORS atom used by clients when determing icon theme\n */\nvoid manager::set_tray_colors() {\n  m_log.trace(\"tray: Set _NET_SYSTEM_TRAY_COLORS to 0x%08x\", m_opts.foreground);\n\n  auto r = m_opts.foreground.red_i();\n  auto g = m_opts.foreground.green_i();\n  auto b = m_opts.foreground.blue_i();\n\n  const uint16_t r16 = (r << 8) | r;\n  const uint16_t g16 = (g << 8) | g;\n  const uint16_t b16 = (b << 8) | b;\n\n  const array<uint32_t, 12> colors = {\n      r16, g16, b16, // normal\n      r16, g16, b16, // error\n      r16, g16, b16, // warning\n      r16, g16, b16, // success\n  };\n\n  m_connection.change_property(XCB_PROP_MODE_REPLACE, m_opts.selection_owner, _NET_SYSTEM_TRAY_COLORS,\n      XCB_ATOM_CARDINAL, 32, colors.size(), colors.data());\n}\n\n/**\n * Set the _NET_SYSTEM_TRAY_ORIENTATION atom\n */\nvoid manager::set_tray_orientation() {\n  const uint32_t orientation = _NET_SYSTEM_TRAY_ORIENTATION_HORZ;\n  m_log.trace(\"tray: Set _NET_SYSTEM_TRAY_ORIENTATION to 0x%x\", orientation);\n  m_connection.change_property_checked(XCB_PROP_MODE_REPLACE, m_opts.selection_owner, _NET_SYSTEM_TRAY_ORIENTATION,\n      XCB_ATOM_CARDINAL, 32, 1, &orientation);\n}\n\n/**\n * Acquire the systray selection\n *\n * @param other_owner is set to the current owner if the function fails\n * @returns Whether we acquired the selection\n */\nbool manager::acquire_selection(xcb_window_t& other_owner) {\n  other_owner = XCB_NONE;\n  xcb_window_t owner = m_connection.get_selection_owner(m_atom).owner();\n\n  if (owner == m_opts.selection_owner) {\n    m_log.trace(\"tray: Already managing the systray selection\");\n    return true;\n  } else if (owner == XCB_NONE) {\n    m_log.trace(\"tray: Change selection owner to %s\", m_connection.id(m_opts.selection_owner));\n    m_connection.set_selection_owner_checked(m_opts.selection_owner, m_atom, XCB_CURRENT_TIME);\n    if (m_connection.get_selection_owner_unchecked(m_atom)->owner != m_opts.selection_owner) {\n      throw application_error(\"Failed to get control of the systray selection\");\n    }\n    return true;\n  } else {\n    other_owner = owner;\n    m_log.warn(\"Systray selection already managed (window=%s)\", m_connection.id(owner));\n    return false;\n  }\n}\n\n/**\n * Notify pending clients about the new systray MANAGER\n */\nvoid manager::notify_clients() {\n  if (is_active()) {\n    m_log.info(\"tray: Notifying pending tray clients\");\n    auto message = m_connection.make_client_message(MANAGER, m_connection.root());\n    message.data.data32[0] = XCB_CURRENT_TIME;\n    message.data.data32[1] = m_atom;\n    message.data.data32[2] = m_opts.selection_owner;\n    m_connection.send_client_message(message, m_connection.root());\n  }\n}\n\n/**\n * Track changes to the given selection owner\n * If it gets destroyed or goes away we can reactivate the tray_manager\n */\nvoid manager::track_selection_owner(xcb_window_t owner) {\n  if (owner != XCB_NONE) {\n    const uint32_t mask{XCB_CW_EVENT_MASK};\n    const uint32_t value{XCB_EVENT_MASK_STRUCTURE_NOTIFY};\n    m_connection.change_window_attributes(owner, mask, &value);\n  }\n}\n\n/**\n * Process client docking request\n */\nvoid manager::process_docking_request(xcb_window_t win) {\n  m_log.info(\"tray: Processing docking request from '%s' (%s)\", ewmh_util::get_wm_name(win), m_connection.id(win));\n\n  try {\n    auto cl =\n        make_unique<client>(m_log, m_connection, m_opts.selection_owner, win, m_opts.client_size, m_opts.background);\n\n    try {\n      cl->query_xembed();\n    } catch (const xpp::x::error::window& err) {\n      m_log.err(\"Failed to query _XEMBED_INFO, removing %s ... (%s)\", cl->name(), err.what());\n      return;\n    }\n\n    cl->update_client_attributes();\n\n    cl->reparent();\n\n    cl->add_to_save_set();\n\n    cl->hidden(m_hidden);\n    cl->ensure_state();\n\n    cl->notify_xembed();\n\n    m_clients.emplace_back(std::move(cl));\n  } catch (const std::exception& err) {\n    m_log.err(\"tray: Failed to setup tray client '%s' (%s) removing... (%s)\", ewmh_util::get_wm_name(win),\n        m_connection.id(win), err.what());\n    return;\n  }\n}\n\n/**\n * Final x-position of the tray window relative to the very top-left bar window.\n */\nint manager::calculate_x() const {\n  return m_bar_opts.inner_area(false).x + m_pos.x;\n}\n\n/**\n * Calculates the entire width taken up by the tray area in pixels\n *\n * This many pixels need to be reserved on the bar in order to draw the tray.\n */\nunsigned manager::calculate_w() const {\n  unsigned count =\n      std::count_if(m_clients.begin(), m_clients.end(), [](const auto& client) { return client->mapped(); });\n\n  if (count > 0) {\n    return (count - 1) * m_opts.spacing + count * (2 * m_opts.padding + m_opts.client_size.w);\n  } else {\n    return 0;\n  }\n}\n\n/**\n * Calculate y position of client window to vertically center it in the inner area of the bar.\n */\nint manager::calculate_client_y() {\n  return m_bar_opts.inner_area(false).y + (m_bar_opts.inner_area(false).height - m_opts.client_size.h) / 2;\n}\n\n/**\n * Check if the given window is embedded.\n *\n * The given window ID can be the ID of the wrapper or the embedded window\n */\nbool manager::is_embedded(const xcb_window_t& win) {\n  return find_client(win) != nullptr;\n}\n\n/**\n * Find tray client object from the wrapper or embedded window\n */\nclient* manager::find_client(const xcb_window_t& win) {\n  auto client = std::find_if(m_clients.begin(), m_clients.end(),\n      [win](const auto& client) { return client->match(win) || client->embedder() == win; });\n\n  if (client == m_clients.end()) {\n    return nullptr;\n  } else {\n    return client->get();\n  }\n}\n\n/**\n * Remove tray client\n */\nvoid manager::remove_client(const client& c) {\n  remove_client(c.client_window());\n}\n\n/**\n * Remove tray client by window\n */\nvoid manager::remove_client(xcb_window_t win) {\n  auto old_size = m_clients.size();\n  m_clients.erase(\n      std::remove_if(m_clients.begin(), m_clients.end(), [win](const auto& client) { return client->match(win); }));\n\n  if (old_size != m_clients.size()) {\n    reconfigure();\n  }\n}\n\n/**\n * Remove all null pointers from client list.\n *\n * Removing clients is often done in two steps:\n * 1. When removing a client during iteration, the unique_ptr is reset.\n * 2. Afterwards all null pointers are removed from the list.\n */\nvoid manager::clean_clients() {\n  m_clients.erase(\n      std::remove_if(m_clients.begin(), m_clients.end(), [](const auto& client) { return client.get() == nullptr; }));\n}\n\nbool manager::change_visibility(bool visible) {\n  if (m_hidden == !visible) {\n    return false;\n  }\n\n  m_log.trace(\"tray: visibility_change (new_state)\", visible ? \"visible\" : \"hidden\");\n\n  m_hidden = !visible;\n\n  for (auto& client : m_clients) {\n    client->hidden(m_hidden);\n    client->ensure_state();\n  }\n\n  if (!m_hidden) {\n    redraw_clients();\n  }\n\n  m_connection.flush();\n\n  return true;\n}\n\n/**\n * Event callback : XCB_EXPOSE\n */\nvoid manager::handle(const evt::expose& evt) {\n  if (is_active() && !m_clients.empty() && evt->count == 0) {\n    redraw_clients();\n  }\n}\n\n/**\n * Event callback : XCB_CLIENT_MESSAGE\n */\nvoid manager::handle(const evt::client_message& evt) {\n  if (!is_active()) {\n    return;\n  }\n\n  // Our selection owner window was deleted\n  if (evt->type == WM_PROTOCOLS && evt->data.data32[0] == WM_DELETE_WINDOW && evt->window == m_opts.selection_owner) {\n    m_log.notice(\"Received WM_DELETE for selection owner\");\n    deactivate();\n  } else if (evt->type == _NET_SYSTEM_TRAY_OPCODE && evt->format == 32) {\n    m_log.trace(\"tray: Received client_message\");\n\n    // Docking request\n    if (SYSTEM_TRAY_REQUEST_DOCK == evt->data.data32[1]) {\n      xcb_window_t win = evt->data.data32[2];\n      if (is_embedded(win)) {\n        m_log.warn(\"Tray client %s already embedded, ignoring request...\", m_connection.id(win));\n      } else {\n        process_docking_request(win);\n      }\n    }\n  }\n}\n\n/**\n * Event callback : XCB_CONFIGURE_REQUEST\n *\n * Called when a tray client thinks he's part of the free world and\n * wants to reconfigure its window. This is of course nothing we appreciate\n * so we return an answer that'll put him in place.\n */\nvoid manager::handle(const evt::configure_request& evt) {\n  if (is_active() && is_embedded(evt->window)) {\n    auto client = find_client(evt->window);\n    try {\n      m_log.trace(\"%s: Client configure request\", client->name());\n      client->configure_notify();\n    } catch (const xpp::x::error::window& err) {\n      m_log.err(\"Failed to reconfigure %s, removing... (%s)\", client->name(), err.what());\n      remove_client(evt->window);\n    }\n  }\n}\n\n/**\n * @see manager::handle(const evt::configure_request&);\n */\nvoid manager::handle(const evt::resize_request& evt) {\n  if (is_active() && is_embedded(evt->window)) {\n    auto client = find_client(evt->window);\n    try {\n      m_log.trace(\"%s: Client resize request\", client->name());\n      client->configure_notify();\n    } catch (const xpp::x::error::window& err) {\n      m_log.err(\"Failed to reconfigure %s, removing... (%s)\", client->name(), err.what());\n      remove_client(evt->window);\n    }\n  }\n}\n\n/**\n * Event callback : XCB_SELECTION_CLEAR\n */\nvoid manager::handle(const evt::selection_clear& evt) {\n  if (is_inactive()) {\n    return;\n  } else if (evt->selection != m_atom) {\n    return;\n  } else if (evt->owner != m_opts.selection_owner) {\n    return;\n  }\n\n  m_log.warn(\"Lost systray selection, deactivating...\");\n  wait_for_selection(m_connection.get_selection_owner(m_atom).owner());\n}\n\n/**\n * Event callback : XCB_PROPERTY_NOTIFY\n */\nvoid manager::handle(const evt::property_notify& evt) {\n  if (!is_active()) {\n    return;\n  }\n\n  if (evt->atom != _XEMBED_INFO) {\n    return;\n  }\n\n  auto client = find_client(evt->window);\n\n  if (!client) {\n    return;\n  }\n\n  m_log.trace(\"%s: _XEMBED_INFO\", client->name());\n\n  if (evt->state == XCB_PROPERTY_NEW_VALUE) {\n    m_log.trace(\"tray: _XEMBED_INFO value has changed\");\n  }\n\n  try {\n    client->query_xembed();\n  } catch (const xpp::x::error::window& err) {\n    m_log.err(\"Failed to query _XEMBED_INFO, removing %s ... (%s)\", client->name(), err.what());\n    remove_client(*client);\n    return;\n  }\n\n  client->ensure_state();\n}\n\n/**\n * Event callback : XCB_REPARENT_NOTIFY\n */\nvoid manager::handle(const evt::reparent_notify& evt) {\n  if (!is_active()) {\n    return;\n  }\n\n  auto client = find_client(evt->window);\n\n  if (!client) {\n    return;\n  }\n\n  // Tray client was reparented to another window\n  if (evt->parent != client->embedder()) {\n    m_log.info(\"%s: Received reparent_notify for client, remove...\", client->name());\n    remove_client(*client);\n  }\n}\n\n/**\n * Event callback : XCB_DESTROY_NOTIFY\n */\nvoid manager::handle(const evt::destroy_notify& evt) {\n  if (is_waiting() && evt->window == m_othermanager) {\n    m_log.info(\"Systray selection unmanaged... re-activating\");\n    activate();\n  } else if (is_active() && is_embedded(evt->window)) {\n    m_log.info(\"tray: Received destroy_notify for client, remove...\");\n    remove_client(evt->window);\n    redraw_clients();\n  }\n}\n\n/**\n * Event callback : XCB_MAP_NOTIFY\n */\nvoid manager::handle(const evt::map_notify& evt) {\n  if (is_active() && evt->window == m_opts.selection_owner) {\n    m_log.trace(\"tray: Received map_notify for selection owner\");\n    redraw_clients();\n  } else if (is_embedded(evt->window)) {\n    auto client = find_client(evt->window);\n\n    // If we received a notification on the wrapped window, we don't want to do anything.\n    if (client->embedder() != evt->window) {\n      return;\n    }\n\n    m_log.trace(\"%s: Received map_notify\", client->name());\n\n    if (!client->mapped()) {\n      client->mapped(true);\n      reconfigure();\n    }\n  }\n}\n\n/**\n * Event callback : XCB_UNMAP_NOTIFY\n */\nvoid manager::handle(const evt::unmap_notify& evt) {\n  if (is_active() && is_embedded(evt->window)) {\n    auto client = find_client(evt->window);\n\n    // If we received a notification on the wrapped window, we don't want to do anything.\n    if (client->embedder() != evt->window) {\n      return;\n    }\n\n    m_log.trace(\"%s: Received unmap_notify\", client->name());\n\n    if (client->mapped()) {\n      client->mapped(false);\n      reconfigure();\n    }\n  }\n}\n\nbool manager::on(const signals::ui::update_background&) {\n  redraw_clients();\n\n  return false;\n}\n\nbool manager::on(const signals::ui_tray::tray_pos_change& evt) {\n  int new_x = std::max(0, std::min(evt.cast(), (int)(m_bar_opts.size.w - m_tray_width)));\n\n  if (new_x != m_pos.x) {\n    m_pos.x = new_x;\n    reconfigure();\n  }\n\n  return true;\n}\n\n} // namespace tray\n\nPOLYBAR_NS_END\n\n// Path: src/x11/xembed.cpp\n#include \"x11/xembed.hpp\"\n\n#include <cassert>\n\n#include \"errors.hpp\"\n#include \"utils/string.hpp\"\n#include \"x11/atoms.hpp\"\n#include \"x11/ewmh.hpp\"\n\nPOLYBAR_NS\n\nnamespace xembed {\n\nvoid info::set(const uint32_t* data) {\n  version = data[0];\n  flags = data[1];\n}\n\nuint32_t info::get_version() const {\n  return version;\n}\n\nuint32_t info::get_flags() const {\n  return flags;\n}\n\nbool info::is_mapped() const {\n  return (flags & XEMBED_MAPPED) == XEMBED_MAPPED;\n}\n\nstring info::to_string() const {\n  return sstream() << \"xembed(version=0x\" << std::hex << get_version() << \", flags=0x\" << get_flags()\n                   << \", XEMBED_MAPPED=\" << std::dec << is_mapped() << \")\";\n}\n\n/**\n * Query _XEMBED_INFO for the given window\n *\n * @return Whether valid XEMBED info was found\n */\n\nbool query(connection& conn, xcb_window_t win, info& data) {\n  auto info = conn.get_property(false, win, _XEMBED_INFO, _XEMBED_INFO, 0L, 2);\n\n  if (info->value_len == 0) {\n    return false;\n  }\n\n  std::vector<uint32_t> xembed_data{info.value<uint32_t>().begin(), info.value<uint32_t>().end()};\n  data.set(xembed_data.data());\n\n  return true;\n}\n\n/**\n * Send _XEMBED messages\n */\nvoid send_message(connection& conn, xcb_window_t target, uint32_t message, uint32_t d1, uint32_t d2, uint32_t d3) {\n  auto msg = conn.make_client_message(_XEMBED, target);\n  msg.data.data32[0] = XCB_CURRENT_TIME;\n  msg.data.data32[1] = message;\n  msg.data.data32[2] = d1;\n  msg.data.data32[3] = d2;\n  msg.data.data32[4] = d3;\n  conn.send_client_message(msg, target);\n}\n\n/**\n * Send window focus event\n */\nvoid send_focus_event(connection& conn, xcb_window_t target) {\n  auto msg = conn.make_client_message(WM_PROTOCOLS, target);\n  msg.data.data32[0] = WM_TAKE_FOCUS;\n  msg.data.data32[1] = XCB_CURRENT_TIME;\n  conn.send_client_message(msg, target);\n}\n\n/**\n * Acknowledge window embedding\n */\nvoid notify_embedded(connection& conn, xcb_window_t win, xcb_window_t embedder, uint32_t version) {\n  send_message(conn, win, XEMBED_EMBEDDED_NOTIFY, 0, embedder, std::min(version, XEMBED_MAX_VERSION));\n}\n\n/**\n * Send window activate notification\n */\nvoid notify_activated(connection& conn, xcb_window_t win) {\n  send_message(conn, win, XEMBED_WINDOW_ACTIVATE, 0, 0, 0);\n}\n\n/**\n * Send window deactivate notification\n */\nvoid notify_deactivated(connection& conn, xcb_window_t win) {\n  send_message(conn, win, XEMBED_WINDOW_DEACTIVATE, 0, 0, 0);\n}\n\n/**\n * Send window focused notification\n */\nvoid notify_focused(connection& conn, xcb_window_t win, uint32_t focus_type) {\n  send_message(conn, win, XEMBED_FOCUS_IN, focus_type, 0, 0);\n}\n\n/**\n * Send window unfocused notification\n */\nvoid notify_unfocused(connection& conn, xcb_window_t win) {\n  send_message(conn, win, XEMBED_FOCUS_OUT, 0, 0, 0);\n}\n\n/**\n * Unembed given window\n */\nvoid unembed(connection& conn, xcb_window_t win, xcb_window_t root) {\n  assert(win != XCB_NONE);\n...\n", "needle_token_start": 15084, "needle_token_end": 15231, "code_context_ntokens": 15953, "output": ["```cpp\n/**\n * Query _XEMBED_INFO for the given window\n *\n * @return Whether valid XEMBED info was found\n */\nbool query(connection& conn, xcb_window_t win, info& data) {\n  auto info = conn.get_property(false, win, _XEMBED_INFO, _XEMBED_INFO, 0L, 2);\n\n  if (info->value_len == 0) {\n    return false;\n  }\n\n  std::vector<uint32_t> xembed_data{info.value<uint32_t>().begin(), info.value<uint32_t>().end()};\n  data.set(xembed_data.data());\n\n  return true;\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "doAdvicesChain", "language": "cpp", "path": "lib/src/AOPAdvice.cc", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function sequentially executes a list of advice functions for HTTP requests, allowing each advice to potentially modify the request or response before passing control to the next advice in the chain. It also ensures that the final response is returned through a callback once all advices have been processed or immediately if there are no advices.\n2. **Input**: It takes a list of advice functions, an index to keep track of the current advice being executed, an HTTP request object, and a shared pointer to a callback function that is intended to handle the final HTTP response.\n3. **Output**: There is no direct return value; however, the output is indirectly produced through the callback function which either receives the final HTTP response or `nullptr` if the chain completes without generating a response.\n4. **Procedure**: \n   - Checks if the current index is within the bounds of the advice list.\n   - If within bounds, retrieves and executes the current advice function, passing the HTTP request, a response handler, and a chain continuation callback.\n   - The response handler invokes the callback with the HTTP response.\n   - The chain continuation callback checks if it should run in the current or a different thread (based on the request's associated event loop), and then recursively calls itself with the next index, effectively moving to the next advice in the list.\n   - If the index is out of bounds (i.e., all advices have been processed), the callback is called with `nullptr` to indicate the end of processing.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    doAdvicesChain(preRoutingAdvices_, 0, req, std::move(callbackPtr));\n}\n\nvoid AopAdvice::passPostRoutingObservers(const HttpRequestImplPtr &req) const\n{\n    if (!postRoutingObservers_.empty())\n    {\n        for (auto &observer : postRoutingObservers_)\n        {\n            observer(req);\n        }\n    }\n}\n\nvoid AopAdvice::passPostRoutingAdvices(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    if (postRoutingAdvices_.empty())\n    {\n        callback(nullptr);\n        return;\n    }\n\n    auto callbackPtr =\n        std::make_shared<std::decay_t<decltype(callback)>>(std::move(callback));\n    doAdvicesChain(postRoutingAdvices_, 0, req, std::move(callbackPtr));\n}\n\nvoid AopAdvice::passPreHandlingObservers(const HttpRequestImplPtr &req) const\n{\n    if (!preHandlingObservers_.empty())\n    {\n        for (auto &observer : preHandlingObservers_)\n        {\n            observer(req);\n        }\n    }\n}\n\nvoid AopAdvice::passPreHandlingAdvices(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    if (preHandlingAdvices_.empty())\n    {\n        callback(nullptr);\n        return;\n    }\n\n    auto callbackPtr =\n        std::make_shared<std::decay_t<decltype(callback)>>(std::move(callback));\n    doAdvicesChain(preHandlingAdvices_, 0, req, std::move(callbackPtr));\n}\n\nvoid AopAdvice::passPostHandlingAdvices(const HttpRequestImplPtr &req,\n                                        const HttpResponsePtr &resp) const\n{\n    for (auto &advice : postHandlingAdvices_)\n    {\n        advice(req, resp);\n    }\n}\n\nvoid AopAdvice::passPreSendingAdvices(const HttpRequestImplPtr &req,\n                                      const HttpResponsePtr &resp) const\n{\n    for (auto &advice : preSendingAdvices_)\n    {\n        advice(req, resp);\n    }\n}\n\n\nstatic void doAdvicesChain(\n    const std::vector<std::function<void(const HttpRequestPtr &,\n                                         AdviceCallback &&,\n                                         AdviceChainCallback &&)>> &advices,\n    size_t index,\n    const HttpRequestImplPtr &req,\n    std::shared_ptr<const std::function<void(const HttpResponsePtr &)>>\n        &&callbackPtr)\n{\n    if (index < advices.size())\n    {\n        auto &advice = advices[index];\n        advice(\n            req,\n            [/*copy*/ callbackPtr](const HttpResponsePtr &resp) {\n                (*callbackPtr)(resp);\n            },\n            [index, req, callbackPtr, &advices]() mutable {\n                auto ioLoop = req->getLoop();\n                if (ioLoop && !ioLoop->isInLoopThread())\n                {\n                    ioLoop->queueInLoop([index,\n                                         req,\n                                         callbackPtr = std::move(callbackPtr),\n                                         &advices]() mutable {\n                        doAdvicesChain(advices,\n                                       index + 1,\n                                       req,\n                                       std::move(callbackPtr));\n                    });\n                }\n                else\n                {\n                    doAdvicesChain(advices,\n                                   index + 1,\n                                   req,\n                                   std::move(callbackPtr));\n                }\n            });\n    }\n    else\n    {\n        (*callbackPtr)(nullptr);\n    }\n}\n\n}  // namespace drogon\n\n// Path: lib/src/MultiPart.cc\n/**\n *\n *  @file MultiPart.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpRequestImpl.h\"\n#include \"HttpUtils.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpFileImpl.h\"\n#include <drogon/MultiPart.h>\n#include <drogon/utils/Utilities.h>\n#include <drogon/config.h>\n#include <algorithm>\n#include <fcntl.h>\n#include <fstream>\n#include <iostream>\n#include <sys/stat.h>\n#ifndef _WIN32\n#include <unistd.h>\n#endif\n\nusing namespace drogon;\n\nconst std::vector<HttpFile> &MultiPartParser::getFiles() const\n{\n    return files_;\n}\n\nstd::unordered_map<std::string, HttpFile> MultiPartParser::getFilesMap() const\n{\n    std::unordered_map<std::string, HttpFile> result;\n    for (auto &file : files_)\n    {\n        result.emplace(file.getItemName(), file);\n    }\n    return result;\n}\n\nconst SafeStringMap<std::string> &MultiPartParser::getParameters() const\n{\n    return parameters_;\n}\n\nint MultiPartParser::parse(const HttpRequestPtr &req)\n{\n    if (req->method() != Post && req->method() != Put)\n        return -1;\n    const std::string &contentType =\n        static_cast<HttpRequestImpl *>(req.get())->getHeaderBy(\"content-type\");\n    if (contentType.empty())\n    {\n        return -1;\n    }\n    std::string::size_type pos = contentType.find(';');\n    if (pos == std::string::npos)\n        return -1;\n\n    std::string type = contentType.substr(0, pos);\n    std::transform(type.begin(), type.end(), type.begin(), [](unsigned char c) {\n        return tolower(c);\n    });\n    if (type != \"multipart/form-data\")\n        return -1;\n    pos = contentType.find(\"boundary=\");\n    if (pos == std::string::npos)\n        return -1;\n    auto pos2 = contentType.find(';', pos);\n    if (pos2 == std::string::npos)\n        pos2 = contentType.size();\n    return parse(req, contentType.data() + (pos + 9), pos2 - (pos + 9));\n}\n\nstatic std::pair<std::string_view, std::string_view> parseLine(\n    const char *begin,\n    const char *end)\n{\n    auto p = begin;\n    while (p != end)\n    {\n        if (*p == ':')\n        {\n            if (p + 1 != end && *(p + 1) == ' ')\n            {\n                return std::make_pair(std::string_view(begin, p - begin),\n                                      std::string_view(p + 2, end - p - 2));\n            }\n            else\n            {\n                return std::make_pair(std::string_view(begin, p - begin),\n                                      std::string_view(p + 1, end - p - 1));\n            }\n        }\n        ++p;\n    }\n    return std::make_pair(std::string_view(), std::string_view());\n}\n\nint MultiPartParser::parseEntity(const HttpRequestPtr &req,\n                                 const char *begin,\n                                 const char *end)\n{\n    static const char entityName[] = \"name=\";\n    static const char fileName[] = \"filename=\";\n    static const char CRLF[] = \"\\r\\n\\r\\n\";\n    auto headEnd = std::search(begin, end, CRLF, CRLF + 4);\n    if (headEnd == end)\n    {\n        return -1;\n    }\n    headEnd += 2;\n    auto pos = begin;\n    std::shared_ptr<HttpFileImpl> filePtr = std::make_shared<HttpFileImpl>();\n    while (pos != headEnd)\n    {\n        auto lineEnd = std::search(pos, headEnd, CRLF, CRLF + 2);\n        auto keyAndValue = parseLine(pos, lineEnd);\n        if (keyAndValue.first.empty() || keyAndValue.second.empty())\n        {\n            return -1;\n        }\n        pos = lineEnd + 2;\n        std::string key{keyAndValue.first.data(), keyAndValue.first.size()};\n        std::transform(key.begin(),\n                       key.end(),\n                       key.begin(),\n                       [](unsigned char c) { return tolower(c); });\n        if (key == \"content-disposition\")\n        {\n            auto value = keyAndValue.second;\n            auto valueEnd = value.data() + value.length();\n            auto namePos =\n                std::search(value.data(), valueEnd, entityName, entityName + 5);\n            if (namePos == valueEnd)\n            {\n                return -1;\n            }\n            namePos += 5;\n            const char *nameEnd;\n            if (*namePos == '\"')\n            {\n                ++namePos;\n                nameEnd = std::find(namePos, valueEnd, '\"');\n            }\n            else\n            {\n                nameEnd = std::find(namePos, valueEnd, ';');\n            }\n            std::string name(namePos, nameEnd);\n            auto fileNamePos =\n                std::search(nameEnd, valueEnd, fileName, fileName + 9);\n            if (fileNamePos == valueEnd)\n            {\n                parameters_.emplace(name, std::string(headEnd + 2, end));\n                return 0;\n            }\n            else\n            {\n                fileNamePos += 9;\n                const char *fileNameEnd;\n                if (*fileNamePos == '\"')\n                {\n                    ++fileNamePos;\n                    fileNameEnd = std::find(fileNamePos, valueEnd, '\"');\n                }\n                else\n                {\n                    fileNameEnd = std::find(fileNamePos, valueEnd, ';');\n                }\n                std::string fName{fileNamePos, fileNameEnd};\n                filePtr->setRequest(req);\n                filePtr->setItemName(std::move(name));\n                filePtr->setFileName(std::move(fName));\n                filePtr->setFile(headEnd + 2,\n                                 static_cast<size_t>(end - headEnd - 2));\n            }\n        }\n        else if (key == \"content-type\")\n        {\n            auto value = keyAndValue.second;\n            auto semiColonPos =\n                std::find(value.data(), value.data() + value.length(), ';');\n            std::string_view contentType(value.data(),\n                                         semiColonPos - value.data());\n            filePtr->setContentType(parseContentType(contentType));\n        }\n        else if (key == \"content-transfer-encoding\")\n        {\n            auto value = keyAndValue.second;\n            auto semiColonPos =\n                std::find(value.data(), value.data() + value.length(), ';');\n\n            filePtr->setContentTransferEncoding(\n                std::string{value.data(), semiColonPos});\n        }\n    }\n    if (!filePtr->getFileName().empty())\n    {\n        files_.emplace_back(std::move(filePtr));\n        return 0;\n    }\n    else\n    {\n        return -1;\n    }\n}\n\nint MultiPartParser::parse(const HttpRequestPtr &req,\n                           const char *boundaryData,\n                           size_t boundaryLen)\n{\n    std::string_view boundary{boundaryData, boundaryLen};\n    if (boundary.size() > 2 && boundary[0] == '\\\"')\n        boundary = boundary.substr(1, boundary.size() - 2);\n    std::string_view::size_type pos1, pos2;\n    pos1 = 0;\n    auto content = static_cast<HttpRequestImpl *>(req.get())->bodyView();\n    pos2 = content.find(boundary);\n    while (true)\n    {\n        pos1 = pos2;\n        if (pos1 == std::string_view::npos)\n            break;\n        pos1 += boundary.length();\n        if (content[pos1] == '\\r' && content[pos1 + 1] == '\\n')\n            pos1 += 2;\n        pos2 = content.find(boundary, pos1);\n        if (pos2 == std::string_view::npos)\n            break;\n        bool flag = false;\n        if (content[pos2 - 4] == '\\r' && content[pos2 - 3] == '\\n' &&\n            content[pos2 - 2] == '-' && content[pos2 - 1] == '-')\n        {\n            pos2 -= 4;\n            flag = true;\n        }\n        if (parseEntity(req, content.data() + pos1, content.data() + pos2) != 0)\n            return -1;\n        if (flag)\n            pos2 += 4;\n    }\n    return 0;\n}\n\n// Path: lib/src/SharedLibManager.cc\n/**\n *\n *  @file SharedLibManager.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"SharedLibManager.h\"\n#include <drogon/config.h>\n#include <dirent.h>\n#include <dlfcn.h>\n#include <fstream>\n#include <sys/types.h>\n#include <trantor/utils/Logger.h>\n#include <unistd.h>\n\nstatic void forEachFileIn(\n    const std::string &path,\n    const std::function<void(const std::string &, const struct stat &)> &cb)\n{\n    DIR *dp;\n    struct dirent *dirp;\n    struct stat st;\n\n    /* open dirent directory */\n    if ((dp = opendir(path.c_str())) == NULL)\n    {\n        // perror(\"opendir:\");\n        LOG_ERROR << \"can't open dir,path:\" << path;\n        return;\n    }\n\n    /**\n     * read all files in this dir\n     **/\n    while ((dirp = readdir(dp)) != NULL)\n    {\n        /* ignore hidden files */\n        if (dirp->d_name[0] == '.')\n            continue;\n        /* get dirent status */\n        std::string filename = dirp->d_name;\n        std::string fullname = path;\n        fullname.append(\"/\").append(filename);\n        if (stat(fullname.c_str(), &st) == -1)\n        {\n            perror(\"stat\");\n            closedir(dp);\n            return;\n        }\n\n        /* if dirent is a directory, find files recursively */\n        if (S_ISDIR(st.st_mode))\n        {\n            forEachFileIn(fullname, cb);\n        }\n        else\n        {\n            cb(fullname, st);\n        }\n    }\n    closedir(dp);\n    return;\n}\n\nusing namespace drogon;\n\nSharedLibManager::SharedLibManager(const std::vector<std::string> &libPaths,\n                                   const std::string &outputPath)\n    : libPaths_(libPaths), outputPath_(outputPath)\n{\n    workingThread_.run();\n    timeId_ =\n        workingThread_.getLoop()->runEvery(5.0, [this]() { managerLibs(); });\n}\n\nSharedLibManager::~SharedLibManager()\n{\n    workingThread_.getLoop()->invalidateTimer(timeId_);\n}\n\nvoid SharedLibManager::managerLibs()\n{\n    for (auto const &libPath : libPaths_)\n    {\n        forEachFileIn(\n            libPath,\n            [this, libPath](const std::string &filename,\n                            const struct stat &st) {\n                auto pos = filename.rfind('.');\n                if (pos != std::string::npos)\n                {\n                    auto exName = filename.substr(pos + 1);\n                    if (exName == \"csp\")\n                    {\n                        // compile\n                        auto lockFile = filename + \".lock\";\n                        std::ifstream fin(lockFile);\n                        if (fin)\n                        {\n                            return;\n                        }\n\n                        void *oldHandle = nullptr;\n                        if (dlMap_.find(filename) != dlMap_.end())\n                        {\n#if defined __linux__ || defined __HAIKU__\n                            if (st.st_mtim.tv_sec >\n                                dlMap_[filename].mTime.tv_sec)\n#elif defined _WIN32\n                            if (st.st_mtime > dlMap_[filename].mTime.tv_sec)\n#else\n                            if (st.st_mtimespec.tv_sec >\n                                dlMap_[filename].mTime.tv_sec)\n#endif\n                            {\n                                LOG_TRACE << \"new csp file:\" << filename;\n                                oldHandle = dlMap_[filename].handle;\n                            }\n                            else\n                                return;\n                        }\n\n                        {\n                            std::ofstream fout(lockFile);\n                        }\n\n                        auto srcFile = filename.substr(0, pos);\n                        if (!outputPath_.empty())\n                        {\n                            pos = srcFile.rfind(\"/\");\n                            if (pos != std::string::npos)\n                            {\n                                srcFile = srcFile.substr(pos + 1);\n                            }\n                            srcFile = outputPath_ + \"/\" + srcFile;\n                        }\n                        auto soFile = srcFile + \".so\";\n                        DLStat dlStat;\n                        if (!shouldCompileLib(soFile, st))\n                        {\n                            LOG_TRACE << \"Using already compiled library:\"\n                                      << soFile;\n                            dlStat.handle = loadLib(soFile, oldHandle);\n                        }\n                        else\n                        {\n                            // generate source code and compile it.\n                            std::string cmd = \"drogon_ctl create view \";\n                            if (!outputPath_.empty())\n                            {\n                                cmd.append(filename).append(\" -o \").append(\n                                    outputPath_);\n                            }\n                            else\n                            {\n                                cmd.append(filename).append(\" -o \").append(\n                                    libPath);\n                            }\n                            srcFile.append(\".cc\");\n                            LOG_TRACE << cmd;\n                            auto r = system(cmd.c_str());\n                            // TODO: handle r\n                            (void)(r);\n                            dlStat.handle =\n                                compileAndLoadLib(srcFile, oldHandle);\n                        }\n#if defined __linux__ || defined __HAIKU__\n                        dlStat.mTime = st.st_mtim;\n#elif defined _WIN32\n                        dlStat.mTime.tv_sec = st.st_mtime;\n#else\n                        dlStat.mTime = st.st_mtimespec;\n#endif\n                        if (dlStat.handle)\n                        {\n                            dlMap_[filename] = dlStat;\n                        }\n                        else\n                        {\n                            dlStat.handle = dlMap_[filename].handle;\n                            dlMap_[filename] = dlStat;\n                        }\n                        workingThread_.getLoop()->runAfter(3.5, [lockFile]() {\n                            LOG_TRACE << \"remove file \" << lockFile;\n                            if (unlink(lockFile.c_str()) == -1)\n                                perror(\"\");\n                        });\n                    }\n                }\n            });\n    }\n}\n\nvoid *SharedLibManager::compileAndLoadLib(const std::string &sourceFile,\n                                          void *oldHld)\n{\n    LOG_TRACE << \"src:\" << sourceFile;\n    std::string cmd = COMPILER_COMMAND;\n    cmd.append(\" \")\n        .append(sourceFile)\n        .append(\" \")\n        .append(COMPILATION_FLAGS)\n        .append(\" \")\n        .append(INCLUDING_DIRS);\n    if (std::string(COMPILER_ID).find(\"Clang\") != std::string::npos)\n        cmd.append(\" -shared -fPIC -undefined dynamic_lookup -o \");\n    else\n        cmd.append(\" -shared -fPIC --no-gnu-unique -o \");\n    auto pos = sourceFile.rfind('.');\n    auto soFile = sourceFile.substr(0, pos);\n    soFile.append(\".so\");\n    cmd.append(soFile);\n    LOG_TRACE << cmd;\n\n    if (system(cmd.c_str()) == 0)\n    {\n        LOG_TRACE << \"Compiled successfully:\" << soFile;\n        return loadLib(soFile, oldHld);\n    }\n    else\n    {\n        LOG_DEBUG << \"Could not compile library.\";\n        return nullptr;\n    }\n}\n\nbool SharedLibManager::shouldCompileLib(const std::string &soFile,\n                                        const struct stat &sourceStat)\n{\n#if defined __linux__ || defined __HAIKU__\n    auto sourceModifiedTime = sourceStat.st_mtim.tv_sec;\n#elif defined _WIN32\n    auto sourceModifiedTime = sourceStat.st_mtime;\n#else\n    auto sourceModifiedTime = sourceStat.st_mtimespec.tv_sec;\n#endif\n\n    struct stat soStat;\n    if (stat(soFile.c_str(), &soStat) == -1)\n    {\n        LOG_TRACE << \"Cannot determine modification time for:\" << soFile;\n        return true;\n    }\n\n#if defined __linux__ || defined __HAIKU__\n    auto soModifiedTime = soStat.st_mtim.tv_sec;\n#elif defined _WIN32\n    auto soModifiedTime = soStat.st_mtime;\n#else\n    auto soModifiedTime = soStat.st_mtimespec.tv_sec;\n#endif\n\n    return (sourceModifiedTime > soModifiedTime);\n}\n\nvoid *SharedLibManager::loadLib(const std::string &soFile, void *oldHld)\n{\n    if (oldHld)\n    {\n        if (dlclose(oldHld) == 0)\n        {\n            LOG_TRACE << \"Successfully closed dynamic library:\" << oldHld;\n        }\n        else\n        {\n            LOG_TRACE << dlerror();\n        }\n    }\n    auto Handle = dlopen(soFile.c_str(), RTLD_LAZY);\n    if (!Handle)\n    {\n        LOG_ERROR << \"load \" << soFile << \" error!\";\n        LOG_ERROR << dlerror();\n    }\n    else\n    {\n        LOG_TRACE << \"Successfully loaded library file \" << soFile;\n    }\n\n    return Handle;\n}\n\n// Path: lib/src/RedisClientSkipped.cc\n/**\n *\n *  RedisClientSkipped.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"drogon/nosql/RedisClient.h\"\n\nnamespace drogon\n{\nnamespace nosql\n{\nstd::shared_ptr<RedisClient> RedisClient::newRedisClient(\n    const trantor::InetAddress & /*serverAddress*/,\n    size_t /*numberOfConnections*/,\n    const std::string & /*password*/,\n    const unsigned int /*db*/,\n    const std::string & /*username*/)\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n}  // namespace nosql\n}  // namespace drogon\n\n// Path: lib/src/Cookie.cc\n/**\n *\n *  Cookie.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/Cookie.h>\n#include <drogon/utils/Utilities.h>\n#include <trantor/utils/Logger.h>\nusing namespace drogon;\n\nstd::string Cookie::cookieString() const\n{\n    std::string ret = \"Set-Cookie: \";\n    // reserve space to reduce frequency allocation\n    ret.reserve(ret.size() + key_.size() + value_.size() + 30);\n    ret.append(key_).append(\"=\").append(value_).append(\"; \");\n    if (expiresDate_.microSecondsSinceEpoch() !=\n            (std::numeric_limits<int64_t>::max)() &&\n        expiresDate_.microSecondsSinceEpoch() >= 0)\n    {\n        ret.append(\"Expires=\")\n            .append(utils::getHttpFullDate(expiresDate_))\n            .append(\"; \");\n    }\n    if (maxAge_.has_value())\n    {\n        ret.append(\"Max-Age=\")\n            .append(std::to_string(maxAge_.value()))\n            .append(\"; \");\n    }\n    if (!domain_.empty())\n    {\n        ret.append(\"Domain=\").append(domain_).append(\"; \");\n    }\n    if (!path_.empty())\n    {\n        ret.append(\"Path=\").append(path_).append(\"; \");\n    }\n    if (sameSite_ != SameSite::kNull)\n    {\n        switch (sameSite_)\n        {\n            case SameSite::kLax:\n                ret.append(\"SameSite=Lax; \");\n                break;\n            case SameSite::kStrict:\n                ret.append(\"SameSite=Strict; \");\n                break;\n            case SameSite::kNone:\n                ret.append(\"SameSite=None; \");\n                // Cookies with SameSite=None must now also specify the Secure\n                // attribute (they require a secure context/HTTPS).\n                ret.append(\"Secure; \");\n                break;\n            default:\n                // Lax replaced None as the default value to ensure that users\n                // have reasonably robust defense against some CSRF attacks\n                ret.append(\"SameSite=Lax; \");\n        }\n    }\n    if (secure_ && sameSite_ != SameSite::kNone)\n    {\n        ret.append(\"Secure; \");\n    }\n    if (httpOnly_)\n    {\n        ret.append(\"HttpOnly; \");\n    }\n    ret.resize(ret.length() - 2);  // delete last semicolon\n    ret.append(\"\\r\\n\");\n    return ret;\n}\n\n// Path: lib/src/Utilities.cc\n/**\n *\n *  @file Utilities.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/utils/Utilities.h>\n#include <trantor/utils/Logger.h>\n#include <trantor/utils/Utilities.h>\n#include <drogon/config.h>\n#ifdef USE_BROTLI\n#include <brotli/decode.h>\n#include <brotli/encode.h>\n#endif\n#ifdef _WIN32\n#include <rpc.h>\n#include <direct.h>\n#include <io.h>\n#include <iomanip>\n#else\n#include <uuid.h>\n#include <unistd.h>\n#endif\n#include <zlib.h>\n#include <sstream>\n#include <string>\n#include <mutex>\n#include <algorithm>\n#include <array>\n#include <locale>\n#include <clocale>\n#include <cctype>\n#include <cstdlib>\n#include <filesystem>\n#include <stdio.h>\n#include <string.h>\n#include <sys/stat.h>\n#include <stdarg.h>\n\n#ifdef _WIN32\n\nchar *strptime(const char *s, const char *f, struct tm *tm)\n{\n    // std::get_time is defined such that its\n    // format parameters are the exact same as strptime.\n    std::istringstream input(s);\n    input.imbue(std::locale(setlocale(LC_ALL, nullptr)));\n    input >> std::get_time(tm, f);\n    if (input.fail())\n    {\n        return nullptr;\n    }\n    return (char *)(s + input.tellg());\n}\n\ntime_t timegm(struct tm *tm)\n{\n    struct tm my_tm;\n\n    memcpy(&my_tm, tm, sizeof(struct tm));\n\n    /* _mkgmtime() changes the value of the struct tm* you pass in, so\n     * use a copy\n     */\n    return _mkgmtime(&my_tm);\n}\n#endif\n\n#ifdef __HAIKU__\n// HACK: Haiku has a timegm implementation. But it is not exposed\nextern \"C\" time_t timegm(struct tm *tm);\n#endif\n\nnamespace drogon\n{\nnamespace utils\n{\nstatic const std::string base64Chars =\n    \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n    \"abcdefghijklmnopqrstuvwxyz\"\n    \"0123456789+/\";\n\nstatic const std::string urlBase64Chars =\n    \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n    \"abcdefghijklmnopqrstuvwxyz\"\n    \"0123456789-_\";\n\nclass Base64CharMap\n{\n  public:\n    Base64CharMap()\n    {\n        char index = 0;\n        for (int c = 'A'; c <= 'Z'; ++c)\n        {\n            charMap_[c] = index++;\n        }\n        for (int c = 'a'; c <= 'z'; ++c)\n        {\n            charMap_[c] = index++;\n        }\n        for (int c = '0'; c <= '9'; ++c)\n        {\n            charMap_[c] = index++;\n        }\n        charMap_[static_cast<int>('+')] = charMap_[static_cast<int>('-')] =\n            index++;\n        charMap_[static_cast<int>('/')] = charMap_[static_cast<int>('_')] =\n            index;\n        charMap_[0] = char(0xff);\n    }\n\n    char getIndex(const char c) const noexcept\n    {\n        return charMap_[static_cast<int>(c)];\n    }\n\n  private:\n    char charMap_[256]{0};\n};\n\nstatic const Base64CharMap base64CharMap;\n\nstatic inline bool isBase64(unsigned char c)\n{\n    if (isalnum(c))\n        return true;\n    switch (c)\n    {\n        case '+':\n        case '/':\n        case '-':\n        case '_':\n            return true;\n    }\n    return false;\n}\n\nbool isInteger(std::string_view str)\n{\n    for (auto c : str)\n        if (c < '0' || c > '9')\n            return false;\n    return true;\n}\n\nbool isBase64(std::string_view str)\n{\n    for (auto c : str)\n        if (!isBase64(c))\n            return false;\n    return true;\n}\n\nstd::string genRandomString(int length)\n{\n    static const char char_space[] =\n        \"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\";\n    static std::once_flag once;\n    static const size_t len = strlen(char_space);\n    static const int randMax = RAND_MAX - (RAND_MAX % len);\n    std::call_once(once, []() {\n        std::srand(static_cast<unsigned int>(time(nullptr)));\n    });\n\n    int i;\n    std::string str;\n    str.resize(length);\n\n    for (i = 0; i < length; ++i)\n    {\n        int x = std::rand();\n        while (x >= randMax)\n        {\n            x = std::rand();\n        }\n        x = (x % len);\n        str[i] = char_space[x];\n    }\n\n    return str;\n}\n\nstd::vector<char> hexToBinaryVector(const char *ptr, size_t length)\n{\n    assert(length % 2 == 0);\n    std::vector<char> ret(length / 2, '\\0');\n    for (size_t i = 0; i < ret.size(); ++i)\n    {\n        auto p = i * 2;\n        char c1 = ptr[p];\n        if (c1 >= '0' && c1 <= '9')\n        {\n            c1 -= '0';\n        }\n        else if (c1 >= 'a' && c1 <= 'f')\n        {\n            c1 -= 'a';\n            c1 += 10;\n        }\n        else if (c1 >= 'A' && c1 <= 'F')\n        {\n            c1 -= 'A';\n            c1 += 10;\n        }\n        else\n        {\n            return std::vector<char>();\n        }\n        char c2 = ptr[p + 1];\n        if (c2 >= '0' && c2 <= '9')\n        {\n            c2 -= '0';\n        }\n        else if (c2 >= 'a' && c2 <= 'f')\n        {\n            c2 -= 'a';\n            c2 += 10;\n        }\n        else if (c2 >= 'A' && c2 <= 'F')\n        {\n            c2 -= 'A';\n            c2 += 10;\n        }\n        else\n        {\n            return std::vector<char>();\n        }\n        ret[i] = c1 * 16 + c2;\n    }\n    return ret;\n}\n\nstd::string hexToBinaryString(const char *ptr, size_t length)\n{\n    assert(length % 2 == 0);\n    std::string ret(length / 2, '\\0');\n    for (size_t i = 0; i < ret.length(); ++i)\n    {\n        auto p = i * 2;\n        char c1 = ptr[p];\n        if (c1 >= '0' && c1 <= '9')\n        {\n            c1 -= '0';\n        }\n        else if (c1 >= 'a' && c1 <= 'f')\n        {\n            c1 -= 'a';\n            c1 += 10;\n        }\n        else if (c1 >= 'A' && c1 <= 'F')\n        {\n            c1 -= 'A';\n            c1 += 10;\n        }\n        else\n        {\n            return \"\";\n        }\n        char c2 = ptr[p + 1];\n        if (c2 >= '0' && c2 <= '9')\n        {\n            c2 -= '0';\n        }\n        else if (c2 >= 'a' && c2 <= 'f')\n        {\n            c2 -= 'a';\n            c2 += 10;\n        }\n        else if (c2 >= 'A' && c2 <= 'F')\n        {\n            c2 -= 'A';\n            c2 += 10;\n        }\n        else\n        {\n            return \"\";\n        }\n        ret[i] = c1 * 16 + c2;\n    }\n    return ret;\n}\n\nDROGON_EXPORT void binaryStringToHex(const char *ptr,\n                                     size_t length,\n                                     char *out,\n                                     bool lowerCase)\n{\n    for (size_t i = 0; i < length; ++i)\n    {\n        int value = (ptr[i] & 0xf0) >> 4;\n        if (value < 10)\n        {\n            out[i * 2] = char(value + 48);\n        }\n        else\n        {\n            if (!lowerCase)\n            {\n                out[i * 2] = char(value + 55);\n            }\n            else\n            {\n                out[i * 2] = char(value + 87);\n            }\n        }\n\n        value = (ptr[i] & 0x0f);\n        if (value < 10)\n        {\n            out[i * 2 + 1] = char(value + 48);\n        }\n        else\n        {\n            if (!lowerCase)\n            {\n                out[i * 2 + 1] = char(value + 55);\n            }\n            else\n            {\n                out[i * 2 + 1] = char(value + 87);\n            }\n        }\n    }\n}\n\nstd::string binaryStringToHex(const unsigned char *ptr,\n                              size_t length,\n                              bool lowercase)\n{\n    std::string idString(length * 2, '\\0');\n    binaryStringToHex((const char *)ptr, length, &idString[0], lowercase);\n    return idString;\n}\n\nstd::set<std::string> splitStringToSet(const std::string &str,\n                                       const std::string &separator)\n{\n    std::set<std::string> ret;\n    std::string::size_type pos1, pos2;\n    pos2 = 0;\n    pos1 = str.find(separator);\n    while (pos1 != std::string::npos)\n    {\n        if (pos1 != 0)\n        {\n            std::string item = str.substr(pos2, pos1 - pos2);\n            ret.insert(item);\n        }\n        pos2 = pos1 + separator.length();\n        while (pos2 < str.length() &&\n               str.substr(pos2, separator.length()) == separator)\n            pos2 += separator.length();\n        pos1 = str.find(separator, pos2);\n    }\n    if (pos2 < str.length())\n        ret.insert(str.substr(pos2));\n    return ret;\n}\n\ninline std::string createUuidString(const char *str, size_t len, bool lowercase)\n{\n    assert(len == 16);\n    std::string uuid(36, '\\0');\n    binaryStringToHex(str, 4, &uuid[0], lowercase);\n    uuid[8] = '-';\n    binaryStringToHex(str + 4, 2, &uuid[9], lowercase);\n    uuid[13] = '-';\n    binaryStringToHex(str + 6, 2, &uuid[14], lowercase);\n    uuid[18] = '-';\n    binaryStringToHex(str + 8, 2, &uuid[19], lowercase);\n    uuid[23] = '-';\n    binaryStringToHex(str + 10, 6, &uuid[24], lowercase);\n    return uuid;\n}\n\nstd::string getUuid(bool lowercase)\n{\n#if USE_OSSP_UUID\n    uuid_t *uuid;\n    uuid_create(&uuid);\n    uuid_make(uuid, UUID_MAKE_V4);\n    char *str{nullptr};\n    size_t len{0};\n    uuid_export(uuid, UUID_FMT_BIN, &str, &len);\n    uuid_destroy(uuid);\n    auto ret = createUuidString(str, len, lowercase);\n    free(str);\n    return ret;\n#elif defined __FreeBSD__ || defined __OpenBSD__\n    uuid_t *uuid = new uuid_t;\n    char *binstr = (char *)malloc(16);\n#if defined __FreeBSD__\n    uuidgen(uuid, 1);\n#else\n    uint32_t status;\n    uuid_create(uuid, &status);\n#endif\n#if _BYTE_ORDER == _LITTLE_ENDIAN\n    uuid_enc_le(binstr, uuid);\n#else  /* _BYTE_ORDER != _LITTLE_ENDIAN */\n    uuid_enc_be(binstr, uuid);\n#endif /* _BYTE_ORDER == _LITTLE_ENDIAN */\n    delete uuid;\n    auto ret = createUuidString(binstr, 16, lowercase);\n    free(binstr);\n    return ret;\n#elif defined _WIN32\n    uuid_t uu;\n    UuidCreate(&uu);\n    char tempStr[100];\n    auto len = snprintf(tempStr,\n                        sizeof(tempStr),\n                        \"%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x\",\n                        uu.Data1,\n                        uu.Data2,\n                        uu.Data3,\n                        uu.Data4[0],\n                        uu.Data4[1],\n                        uu.Data4[2],\n                        uu.Data4[3],\n                        uu.Data4[4],\n                        uu.Data4[5],\n                        uu.Data4[6],\n                        uu.Data4[7]);\n    return std::string{tempStr, static_cast<size_t>(len)};\n#else\n    uuid_t uu;\n    uuid_generate(uu);\n    auto uuid = createUuidString((const char *)uu, 16, lowercase);\n    return uuid;\n#endif\n}\n\nstd::string base64Encode(const unsigned char *bytes_to_encode,\n                         size_t in_len,\n                         bool url_safe,\n                         bool padded)\n{\n    std::string ret;\n    ret.reserve(base64EncodedLength(in_len, padded));\n    int i = 0;\n    unsigned char char_array_3[3];\n    unsigned char char_array_4[4];\n\n    const std::string &charSet = url_safe ? urlBase64Chars : base64Chars;\n\n    while (in_len--)\n    {\n        char_array_3[i++] = *(bytes_to_encode++);\n        if (i == 3)\n        {\n            char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;\n            char_array_4[1] = ((char_array_3[0] & 0x03) << 4) +\n                              ((char_array_3[1] & 0xf0) >> 4);\n            char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) +\n                              ((char_array_3[2] & 0xc0) >> 6);\n            char_array_4[3] = char_array_3[2] & 0x3f;\n\n            for (i = 0; (i < 4); ++i)\n                ret += charSet[char_array_4[i]];\n            i = 0;\n        }\n    }\n\n    if (i)\n    {\n        for (int j = i; j < 3; ++j)\n            char_array_3[j] = '\\0';\n\n        char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;\n        char_array_4[1] =\n            ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);\n        char_array_4[2] =\n            ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);\n        char_array_4[3] = char_array_3[2] & 0x3f;\n\n        for (int j = 0; (j <= i); ++j)\n            ret += charSet[char_array_4[j]];\n\n        if (padded)\n            while ((++i < 4))\n                ret += '=';\n    }\n    return ret;\n}\n\nstd::vector<char> base64DecodeToVector(std::string_view encoded_string)\n{\n    auto in_len = encoded_string.size();\n    int i = 0;\n    int in_{0};\n    char char_array_4[4], char_array_3[3];\n    std::vector<char> ret;\n    ret.reserve(base64DecodedLength(in_len));\n\n    while (in_len-- && (encoded_string[in_] != '='))\n    {\n        if (!isBase64(encoded_string[in_]))\n        {\n            ++in_;\n            continue;\n        }\n\n        char_array_4[i++] = encoded_string[in_];\n        ++in_;\n        if (i == 4)\n        {\n            for (i = 0; i < 4; ++i)\n            {\n                char_array_4[i] = base64CharMap.getIndex(char_array_4[i]);\n            }\n\n            char_array_3[0] =\n                (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) +\n                              ((char_array_4[2] & 0x3c) >> 2);\n            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n            for (i = 0; (i < 3); ++i)\n                ret.push_back(char_array_3[i]);\n            i = 0;\n        }\n    }\n\n    if (i)\n    {\n        for (int j = i; j < 4; ++j)\n            char_array_4[j] = 0;\n\n        for (int j = 0; j < 4; ++j)\n        {\n            char_array_4[j] = base64CharMap.getIndex(char_array_4[j]);\n        }\n\n        char_array_3[0] =\n            (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n        char_array_3[1] =\n            ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);\n        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n        --i;\n        for (int j = 0; (j < i); ++j)\n            ret.push_back(char_array_3[j]);\n    }\n\n    return ret;\n}\n\nstd::string base64Decode(std::string_view encoded_string)\n{\n    auto in_len = encoded_string.size();\n    int i = 0;\n    int in_{0};\n    unsigned char char_array_4[4], char_array_3[3];\n    std::string ret;\n    ret.reserve(base64DecodedLength(in_len));\n\n    while (in_len-- && (encoded_string[in_] != '='))\n    {\n        if (!isBase64(encoded_string[in_]))\n        {\n            ++in_;\n            continue;\n        }\n\n        char_array_4[i++] = encoded_string[in_];\n        ++in_;\n        if (i == 4)\n        {\n            for (i = 0; i < 4; ++i)\n            {\n                char_array_4[i] = base64CharMap.getIndex(char_array_4[i]);\n            }\n            char_array_3[0] =\n                (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) +\n                              ((char_array_4[2] & 0x3c) >> 2);\n            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n            for (i = 0; (i < 3); ++i)\n                ret += char_array_3[i];\n            i = 0;\n        }\n    }\n\n    if (i)\n    {\n        for (int j = i; j < 4; ++j)\n            char_array_4[j] = 0;\n\n        for (int j = 0; j < 4; ++j)\n        {\n            char_array_4[j] = base64CharMap.getIndex(char_array_4[j]);\n        }\n\n        char_array_3[0] =\n            (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n        char_array_3[1] =\n            ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);\n        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n        --i;\n        for (int j = 0; (j < i); ++j)\n            ret += char_array_3[j];\n    }\n\n    return ret;\n}\n\nstatic std::string charToHex(char c)\n{\n    std::string result;\n    char first, second;\n\n    first = (c & 0xF0) / 16;\n    first += first > 9 ? 'A' - 10 : '0';\n    second = c & 0x0F;\n    second += second > 9 ? 'A' - 10 : '0';\n\n    result.append(1, first);\n    result.append(1, second);\n\n    return result;\n}\n\nstd::string urlEncodeComponent(const std::string &src)\n{\n    std::string result;\n    std::string::const_iterator iter;\n\n    for (iter = src.begin(); iter != src.end(); ++iter)\n    {\n        switch (*iter)\n        {\n            case ' ':\n                result.append(1, '+');\n                break;\n            // alnum\n            case 'A':\n            case 'B':\n            case 'C':\n            case 'D':\n            case 'E':\n            case 'F':\n            case 'G':\n            case 'H':\n            case 'I':\n            case 'J':\n            case 'K':\n            case 'L':\n            case 'M':\n            case 'N':\n            case 'O':\n            case 'P':\n            case 'Q':\n            case 'R':\n            case 'S':\n            case 'T':\n            case 'U':\n            case 'V':\n            case 'W':\n            case 'X':\n            case 'Y':\n            case 'Z':\n            case 'a':\n            case 'b':\n            case 'c':\n            case 'd':\n            case 'e':\n            case 'f':\n            case 'g':\n            case 'h':\n            case 'i':\n            case 'j':\n            case 'k':\n            case 'l':\n            case 'm':\n            case 'n':\n            case 'o':\n            case 'p':\n            case 'q':\n            case 'r':\n            case 's':\n            case 't':\n            case 'u':\n            case 'v':\n            case 'w':\n            case 'x':\n            case 'y':\n            case 'z':\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            // mark\n            case '-':\n            case '_':\n            case '.':\n            case '!':\n            case '~':\n            case '*':\n            case '(':\n            case ')':\n                result.append(1, *iter);\n                break;\n            // escape\n            default:\n                result.append(1, '%');\n                result.append(charToHex(*iter));\n                break;\n        }\n    }\n\n    return result;\n}\n\nstd::string urlEncode(const std::string &src)\n{\n    std::string result;\n    std::string::const_iterator iter;\n\n    for (iter = src.begin(); iter != src.end(); ++iter)\n    {\n        switch (*iter)\n        {\n            case ' ':\n                result.append(1, '+');\n                break;\n            // alnum\n            case 'A':\n            case 'B':\n            case 'C':\n            case 'D':\n            case 'E':\n            case 'F':\n            case 'G':\n            case 'H':\n            case 'I':\n            case 'J':\n            case 'K':\n            case 'L':\n            case 'M':\n            case 'N':\n            case 'O':\n            case 'P':\n            case 'Q':\n            case 'R':\n            case 'S':\n            case 'T':\n            case 'U':\n            case 'V':\n            case 'W':\n            case 'X':\n            case 'Y':\n            case 'Z':\n            case 'a':\n            case 'b':\n            case 'c':\n            case 'd':\n            case 'e':\n            case 'f':\n            case 'g':\n            case 'h':\n            case 'i':\n            case 'j':\n            case 'k':\n            case 'l':\n            case 'm':\n            case 'n':\n            case 'o':\n            case 'p':\n            case 'q':\n            case 'r':\n            case 's':\n            case 't':\n            case 'u':\n            case 'v':\n            case 'w':\n            case 'x':\n            case 'y':\n            case 'z':\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            // mark\n            case '-':\n            case '_':\n            case '.':\n            case '!':\n            case '~':\n            case '*':\n            case '\\'':\n            case '(':\n            case ')':\n            case '&':\n            case '=':\n            case '/':\n            case '\\\\':\n            case '?':\n                result.append(1, *iter);\n                break;\n            // escape\n            default:\n                result.append(1, '%');\n                result.append(charToHex(*iter));\n                break;\n        }\n    }\n\n    return result;\n}\n\nbool needUrlDecoding(const char *begin, const char *end)\n{\n    return std::find_if(begin, end, [](const char c) {\n               return c == '+' || c == '%';\n           }) != end;\n}\n\nstd::string urlDecode(const char *begin, const char *end)\n{\n    std::string result;\n    size_t len = end - begin;\n    result.reserve(len * 2);\n    int hex = 0;\n    for (size_t i = 0; i < len; ++i)\n    {\n        switch (begin[i])\n        {\n            case '+':\n                result += ' ';\n                break;\n            case '%':\n                if ((i + 2) < len && isxdigit(begin[i + 1]) &&\n                    isxdigit(begin[i + 2]))\n                {\n                    unsigned int x1 = begin[i + 1];\n                    if (x1 >= '0' && x1 <= '9')\n                    {\n                        x1 -= '0';\n                    }\n                    else if (x1 >= 'a' && x1 <= 'f')\n                    {\n                        x1 = x1 - 'a' + 10;\n                    }\n                    else if (x1 >= 'A' && x1 <= 'F')\n                    {\n                        x1 = x1 - 'A' + 10;\n                    }\n                    unsigned int x2 = begin[i + 2];\n                    if (x2 >= '0' && x2 <= '9')\n                    {\n                        x2 -= '0';\n                    }\n                    else if (x2 >= 'a' && x2 <= 'f')\n                    {\n                        x2 = x2 - 'a' + 10;\n                    }\n                    else if (x2 >= 'A' && x2 <= 'F')\n                    {\n                        x2 = x2 - 'A' + 10;\n                    }\n                    hex = x1 * 16 + x2;\n\n                    result += char(hex);\n                    i += 2;\n                }\n                else\n                {\n                    result += '%';\n                }\n                break;\n            default:\n                result += begin[i];\n                break;\n        }\n    }\n    return result;\n}\n\n/* Compress gzip data */\nstd::string gzipCompress(const char *data, const size_t ndata)\n{\n    z_stream strm = {nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     0,\n                     0,\n                     0};\n    if (data && ndata > 0)\n    {\n        if (deflateInit2(&strm,\n                         Z_DEFAULT_COMPRESSION,\n                         Z_DEFLATED,\n                         MAX_WBITS + 16,\n                         8,\n                         Z_DEFAULT_STRATEGY) != Z_OK)\n        {\n            LOG_ERROR << \"deflateInit2 error!\";\n            return std::string{};\n        }\n        std::string outstr;\n        outstr.resize(compressBound(static_cast<uLong>(ndata)));\n        strm.next_in = (Bytef *)data;\n        strm.avail_in = static_cast<uInt>(ndata);\n        int ret;\n        do\n        {\n            if (strm.total_out >= outstr.size())\n            {\n                outstr.resize(strm.total_out * 2);\n            }\n            assert(outstr.size() >= strm.total_out);\n            strm.avail_out = static_cast<uInt>(outstr.size() - strm.total_out);\n            strm.next_out = (Bytef *)outstr.data() + strm.total_out;\n            ret = deflate(&strm, Z_FINISH); /* no bad return value */\n            if (ret == Z_STREAM_ERROR)\n            {\n                (void)deflateEnd(&strm);\n                return std::string{};\n            }\n        } while (strm.avail_out == 0);\n        assert(strm.avail_in == 0);\n        assert(ret == Z_STREAM_END); /* stream will be complete */\n        outstr.resize(strm.total_out);\n        /* clean up and return */\n        (void)deflateEnd(&strm);\n        return outstr;\n    }\n    return std::string{};\n}\n\nstd::string gzipDecompress(const char *data, const size_t ndata)\n{\n    if (ndata == 0)\n        return std::string(data, ndata);\n\n    auto full_length = ndata;\n\n    auto decompressed = std::string(full_length * 2, 0);\n    bool done = false;\n\n    z_stream strm = {nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     0,\n                     0,\n                     0};\n    strm.next_in = (Bytef *)data;\n    strm.avail_in = static_cast<uInt>(ndata);\n    strm.total_out = 0;\n    strm.zalloc = Z_NULL;\n    strm.zfree = Z_NULL;\n    if (inflateInit2(&strm, (15 + 32)) != Z_OK)\n    {\n        LOG_ERROR << \"inflateInit2 error!\";\n        return std::string{};\n    }\n    while (!done)\n    {\n        // Make sure we have enough room and reset the lengths.\n        if (strm.total_out >= decompressed.length())\n        {\n            decompressed.resize(decompressed.length() * 2);\n        }\n        strm.next_out = (Bytef *)decompressed.data() + strm.total_out;\n        strm.avail_out =\n            static_cast<uInt>(decompressed.length() - strm.total_out);\n        // Inflate another chunk.\n        int status = inflate(&strm, Z_SYNC_FLUSH);\n        if (status == Z_STREAM_END)\n        {\n            done = true;\n        }\n        else if (status != Z_OK)\n        {\n            break;\n        }\n    }\n    if (inflateEnd(&strm) != Z_OK)\n        return std::string{};\n    // Set real length.\n    if (done)\n    {\n        decompressed.resize(strm.total_out);\n        return decompressed;\n    }\n    else\n    {\n        return std::string{};\n    }\n}\n\nchar *getHttpFullDate(const trantor::Date &date)\n{\n    static thread_local int64_t lastSecond = 0;\n    static thread_local char lastTimeString[128] = {0};\n    auto nowSecond = date.microSecondsSinceEpoch() / MICRO_SECONDS_PRE_SEC;\n    if (nowSecond == lastSecond)\n    {\n        return lastTimeString;\n    }\n    lastSecond = nowSecond;\n    date.toCustomedFormattedString(\"%a, %d %b %Y %H:%M:%S GMT\",\n                                   lastTimeString,\n                                   sizeof(lastTimeString));\n    return lastTimeString;\n}\n\ntrantor::Date getHttpDate(const std::string &httpFullDateString)\n{\n    static const std::array<const char *, 4> formats = {\n        // RFC822 (default)\n        \"%a, %d %b %Y %H:%M:%S\",\n        // RFC 850 (deprecated)\n        \"%a, %d-%b-%y %H:%M:%S\",\n        // ansi asctime format\n...\n", "needle_token_start": 601, "needle_token_end": 969, "code_context_ntokens": 16173, "output": ["```cpp\nstatic void doAdvicesChain(\n    const std::vector<std::function<void(const HttpRequestPtr &,\n                                         AdviceCallback &&,\n                                         AdviceChainCallback &&)>> &advices,\n    size_t index,\n    const HttpRequestImplPtr &req,\n    std::shared_ptr<const std::function<void(const HttpResponsePtr &)>>\n        &&callbackPtr)\n{\n    if (index < advices.size())\n    {\n        auto &advice = advices[index];\n        advice(\n            req,\n            [/*copy*/ callbackPtr](const HttpResponsePtr &resp) {\n                (*callbackPtr)(resp);\n            },\n            [index, req, callbackPtr, &advices]() mutable {\n                auto ioLoop = req->getLoop();\n                if (ioLoop && !ioLoop->isInLoopThread())\n                {\n                    ioLoop->queueInLoop([index,\n                                         req,\n                                         callbackPtr = std::move(callbackPtr),\n                                         &advices]() mutable {\n                        doAdvicesChain(advices,\n                                       index + 1,\n                                       req,\n                                       std::move(callbackPtr));\n                    });\n                }\n                else\n                {\n                    doAdvicesChain(advices,\n                                   index + 1,\n                                   req,\n                                   std::move(callbackPtr));\n                }\n            });\n    }\n    else\n    {\n        (*callbackPtr)(nullptr);\n    }\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "JsonConfigAdapter", "language": "cpp", "path": "lib/src/JsonConfigAdapter.h", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this component is to interpret and convert configuration data formatted in JSON into a usable structure within the application.\n2. **Input**: It takes a string containing JSON formatted data.\n3. **Output**: It outputs a structured representation of the JSON data, specifically in a format that can be utilized by the application for configuration purposes.\n4. **Procedure**: The component reads the input string, parses it to extract JSON data, and then converts this data into a structured format that the application can use to configure its settings or parameters.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/HttpAppFrameworkImpl.cc\n/**\n *\n *  @file HttpAppFrameworkImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/DrClassMap.h>\n#include <drogon/HttpResponse.h>\n#include <drogon/HttpTypes.h>\n#include <drogon/utils/Utilities.h>\n#include <drogon/version.h>\n#include <json/json.h>\n#include <trantor/utils/AsyncFileLogger.h>\n#include <algorithm>\n#include \"AOPAdvice.h\"\n#include \"ConfigLoader.h\"\n#include \"DbClientManager.h\"\n#include \"HttpClientImpl.h\"\n#include \"HttpConnectionLimit.h\"\n#include \"HttpControllersRouter.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpResponseImpl.h\"\n#include \"HttpServer.h\"\n#include \"HttpUtils.h\"\n#include \"ListenerManager.h\"\n#include \"PluginsManager.h\"\n#include \"RedisClientManager.h\"\n#include \"SessionManager.h\"\n#include \"SharedLibManager.h\"\n#include \"StaticFileRouter.h\"\n\n#include <iostream>\n#include <memory>\n#include <tuple>\n#include <unordered_map>\n#include <utility>\n#include <filesystem>\n\n#include <fcntl.h>\n#include <sys/stat.h>\n#ifndef _WIN32\n#include <sys/wait.h>\n#include <unistd.h>\n#define os_access access\n#elif !defined(_WIN32) || defined(__MINGW32__)\n#include <sys/file.h>\n#include <unistd.h>\n#define os_access access\n#else\n#include <io.h>\n#define os_access _waccess\n#define R_OK 04\n#define W_OK 02\n#endif\n\n#ifdef DROGON_SPDLOG_SUPPORT\n#include <spdlog/spdlog.h>\n#include <spdlog/logger.h>\n#include <spdlog/sinks/stdout_color_sinks.h>\n#include <spdlog/sinks/rotating_file_sink.h>\n#ifdef _WIN32\n#include <spdlog/sinks/msvc_sink.h>\n// Damn antedeluvian M$ macros\n#undef min\n#undef max\n#endif\n#endif  // DROGON_SPDLOG_SUPPORT\n\nusing namespace drogon;\nusing namespace std::placeholders;\n\nHttpAppFrameworkImpl::HttpAppFrameworkImpl()\n    : listenerManagerPtr_(new ListenerManager),\n      pluginsManagerPtr_(new PluginsManager),\n      dbClientManagerPtr_(new orm::DbClientManager),\n      redisClientManagerPtr_(new nosql::RedisClientManager),\n      uploadPath_(rootPath_ + \"uploads\")\n{\n}\n\nstatic std::function<void()> f = [] {\n    LOG_TRACE << \"Initialize the main event loop in the main thread\";\n};\n\n/// Make sure that the main event loop is initialized in the main thread.\ndrogon::InitBeforeMainFunction drogon::HttpAppFrameworkImpl::initFirst_([]() {\n    HttpAppFrameworkImpl::instance().getLoop()->runInLoop(f);\n});\n\nnamespace drogon\n{\nstd::string getVersion()\n{\n    return DROGON_VERSION;\n}\n\nstd::string getGitCommit()\n{\n    return DROGON_VERSION_SHA1;\n}\n\nHttpResponsePtr defaultErrorHandler(HttpStatusCode code, const HttpRequestPtr &)\n{\n    return std::make_shared<HttpResponseImpl>(code, CT_TEXT_HTML);\n}\n\nvoid defaultExceptionHandler(\n    const std::exception &e,\n    const HttpRequestPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    std::string pathWithQuery = req->path();\n    if (req->query().empty() == false)\n        pathWithQuery += \"?\" + req->query();\n    LOG_ERROR << \"Unhandled exception in \" << pathWithQuery\n              << \", what(): \" << e.what();\n    const auto &handler = app().getCustomErrorHandler();\n    callback(handler(k500InternalServerError, req));\n}\n\nstatic void godaemon()\n{\n    printf(\"Initializing daemon mode\\n\");\n#ifndef _WIN32\n    if (getppid() != 1)\n    {\n        pid_t pid;\n        pid = fork();\n        if (pid > 0)\n            exit(0);  // parent\n        if (pid < 0)\n        {\n            perror(\"fork\");\n            exit(1);\n        }\n        setsid();\n    }\n\n    // redirect stdin/stdout/stderr to /dev/null\n    close(0);\n    close(1);\n    close(2);\n\n    int ret = open(\"/dev/null\", O_RDWR);\n    (void)ret;\n    ret = dup(0);\n    (void)ret;\n    ret = dup(0);\n    (void)ret;\n    umask(0);\n#else\n    LOG_ERROR << \"Cannot run as daemon in Windows\";\n    exit(1);\n#endif\n\n    return;\n}\n\nstatic void TERMFunction(int sig)\n{\n    if (sig == SIGTERM)\n    {\n        LOG_WARN << \"SIGTERM signal received.\";\n        HttpAppFrameworkImpl::instance().getTermSignalHandler()();\n    }\n    else if (sig == SIGINT)\n    {\n        LOG_WARN << \"SIGINT signal received.\";\n        HttpAppFrameworkImpl::instance().getIntSignalHandler()();\n    }\n}\n\n}  // namespace drogon\n\nHttpAppFrameworkImpl::~HttpAppFrameworkImpl() noexcept\n{\n// Destroy the following objects before the loop destruction\n#ifndef _WIN32\n    sharedLibManagerPtr_.reset();\n#endif\n    sessionManagerPtr_.reset();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setStaticFilesCacheTime(int cacheTime)\n{\n    StaticFileRouter::instance().setStaticFilesCacheTime(cacheTime);\n    return *this;\n}\n\nint HttpAppFrameworkImpl::staticFilesCacheTime() const\n{\n    return StaticFileRouter::instance().staticFilesCacheTime();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setGzipStatic(bool useGzipStatic)\n{\n    StaticFileRouter::instance().setGzipStatic(useGzipStatic);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setBrStatic(bool useGzipStatic)\n{\n    StaticFileRouter::instance().setBrStatic(useGzipStatic);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setImplicitPageEnable(\n    bool useImplicitPage)\n{\n    StaticFileRouter::instance().setImplicitPageEnable(useImplicitPage);\n    return *this;\n}\n\nbool HttpAppFrameworkImpl::isImplicitPageEnabled() const\n{\n    return StaticFileRouter::instance().isImplicitPageEnabled();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setImplicitPage(\n    const std::string &implicitPageFile)\n{\n    StaticFileRouter::instance().setImplicitPage(implicitPageFile);\n    return *this;\n}\n\nconst std::string &HttpAppFrameworkImpl::getImplicitPage() const\n{\n    return StaticFileRouter::instance().getImplicitPage();\n}\n#ifndef _WIN32\nHttpAppFramework &HttpAppFrameworkImpl::enableDynamicViewsLoading(\n    const std::vector<std::string> &libPaths,\n    const std::string &outputPath)\n{\n    assert(!running_);\n\n    for (auto const &libpath : libPaths)\n    {\n        if (libpath[0] == '/' ||\n            (libpath.length() >= 2 && libpath[0] == '.' && libpath[1] == '/') ||\n            (libpath.length() >= 3 && libpath[0] == '.' && libpath[1] == '.' &&\n             libpath[2] == '/') ||\n            libpath == \".\" || libpath == \"..\")\n        {\n            libFilePaths_.push_back(libpath);\n        }\n        else\n        {\n            if (rootPath_[rootPath_.length() - 1] == '/')\n                libFilePaths_.push_back(rootPath_ + libpath);\n            else\n...\n// Path: lib/src/ConfigAdapter.h\n#pragma once\n#include <json/json.h>\n#include <vector>\n#include <string>\n#include <memory>\n#include <fstream>\n\nnamespace drogon\n{\nclass ConfigAdapter\n{\n  public:\n    virtual ~ConfigAdapter() = default;\n    virtual Json::Value getJson(const std::string &content) const\n        noexcept(false) = 0;\n    virtual std::vector<std::string> getExtensions() const = 0;\n};\n\nusing ConfigAdapterPtr = std::shared_ptr<ConfigAdapter>;\n\n}  // namespace drogon\n\n// Path: lib/src/JsonConfigAdapter.h\n#pragma once\n#include \"ConfigAdapter.h\"\n\nnamespace drogon\n{\nclass JsonConfigAdapter : public ConfigAdapter\n{\n  public:\n    \nJsonConfigAdapter() = default;\n    ~JsonConfigAdapter() override = default;\n    Json::Value getJson(const std::string &content) const\n        noexcept(false) override;\n    std::vector<std::string> getExtensions() const override;\n};\n}  // namespace drogon\n\n// Path: lib/src/JsonConfigAdapter.cc\n#include \"JsonConfigAdapter.h\"\n#include <fstream>\n#include <mutex>\n\nusing namespace drogon;\n\nJson::Value JsonConfigAdapter::getJson(const std::string &content) const\n    noexcept(false)\n{\n    static std::once_flag once;\n    static Json::CharReaderBuilder builder;\n    std::call_once(once, []() { builder[\"collectComments\"] = false; });\n    JSONCPP_STRING errs;\n    std::unique_ptr<Json::CharReader> reader(builder.newCharReader());\n    Json::Value root;\n    if (!reader->parse(\n            content.c_str(), content.c_str() + content.size(), &root, &errs))\n    {\n        throw std::runtime_error(errs);\n    }\n    return root;\n}\n\nstd::vector<std::string> JsonConfigAdapter::getExtensions() const\n{\n    return {\"json\"};\n}\n\n// Path: lib/src/Histogram.cc\n#include <drogon/utils/monitoring/Histogram.h>\nusing namespace drogon;\nusing namespace drogon::monitoring;\n\nvoid Histogram::observe(double value)\n{\n    std::lock_guard<std::mutex> lock(mutex_);\n    if (maxAge_ > std::chrono::seconds(0) &&\n        timerId_ == trantor::InvalidTimerId)\n    {\n        std::weak_ptr<Histogram> weakPtr =\n            std::dynamic_pointer_cast<Histogram>(shared_from_this());\n        timerId_ = loopPtr_->runEvery(maxAge_ / timeBucketCount_, [weakPtr]() {\n            auto thisPtr = weakPtr.lock();\n            if (!thisPtr)\n                return;\n            thisPtr->rotateTimeBuckets();\n        });\n    }\n    auto &currentBucket = timeBuckets_.back();\n    currentBucket.sum += value;\n    currentBucket.count += 1;\n    for (size_t i = 0; i < bucketBoundaries_.size(); i++)\n    {\n        if (value <= bucketBoundaries_[i])\n        {\n            currentBucket.buckets[i] += 1;\n            break;\n        }\n    }\n    if (value > bucketBoundaries_.back())\n    {\n        currentBucket.buckets.back() += 1;\n    }\n}\n\nstd::vector<Sample> Histogram::collect() const\n{\n    std::vector<Sample> samples;\n    std::lock_guard<std::mutex> guard(mutex_);\n    size_t count{0};\n    for (size_t i = 0; i < bucketBoundaries_.size(); i++)\n    {\n        Sample sample;\n        for (auto &bucket : timeBuckets_)\n        {\n            count += bucket.buckets[i];\n        }\n        sample.name = name_ + \"_bucket\";\n        sample.exLabels.emplace_back(\"le\",\n                                     std::to_string(bucketBoundaries_[i]));\n        sample.value = count;\n        samples.emplace_back(std::move(sample));\n    }\n    Sample sample;\n    for (auto &bucket : timeBuckets_)\n    {\n        count += bucket.buckets.back();\n    }\n    sample.name = name_ + \"_bucket\";\n    sample.exLabels.emplace_back(\"le\", \"+Inf\");\n    sample.value = count;\n    samples.emplace_back(std::move(sample));\n    double sum{0};\n    uint64_t totalCount{0};\n    for (auto &bucket : timeBuckets_)\n    {\n        sum += bucket.sum;\n        totalCount += bucket.count;\n    }\n    Sample sumSample;\n    sumSample.name = name_ + \"_sum\";\n    sumSample.value = sum;\n    samples.emplace_back(std::move(sumSample));\n    Sample countSample;\n    countSample.name = name_ + \"_count\";\n    countSample.value = totalCount;\n    samples.emplace_back(std::move(countSample));\n    return samples;\n}\n\n// Path: lib/src/GlobalFilters.cc\n#include <drogon/plugins/GlobalFilters.h>\n#include <drogon/DrClassMap.h>\n#include <drogon/HttpAppFramework.h>\n#include \"FiltersFunction.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpAppFrameworkImpl.h\"\n\nusing namespace drogon::plugin;\n\nvoid GlobalFilters::initAndStart(const Json::Value &config)\n{\n    if (config.isMember(\"filters\") && config[\"filters\"].isArray())\n    {\n        auto &filters = config[\"filters\"];\n\n        for (auto const &filter : filters)\n        {\n            if (filter.isString())\n            {\n                auto filterPtr = std::dynamic_pointer_cast<HttpFilterBase>(\n                    drogon::DrClassMap::getSingleInstance(filter.asString()));\n                if (filterPtr)\n                {\n                    filters_.push_back(filterPtr);\n                }\n                else\n                {\n                    LOG_ERROR << \"Filter \" << filter.asString()\n                              << \" not found!\";\n                }\n            }\n        }\n    }\n    if (config.isMember(\"exempt\"))\n    {\n        auto exempt = config[\"exempt\"];\n        if (exempt.isArray())\n        {\n            std::string regexStr;\n            for (auto const &ex : exempt)\n            {\n                if (ex.isString())\n                {\n                    regexStr.append(\"(\").append(ex.asString()).append(\")|\");\n                }\n                else\n                {\n                    LOG_ERROR << \"exempt must be a string array!\";\n                }\n            }\n            if (!regexStr.empty())\n            {\n                regexStr.pop_back();\n                exemptPegex_ = std::regex(regexStr);\n                regexFlag_ = true;\n            }\n        }\n        else if (exempt.isString())\n        {\n            exemptPegex_ = std::regex(exempt.asString());\n            regexFlag_ = true;\n        }\n        else\n        {\n            LOG_ERROR << \"exempt must be a string or string array!\";\n        }\n    }\n    std::weak_ptr<GlobalFilters> weakPtr = shared_from_this();\n    drogon::app().registerPreRoutingAdvice(\n        [weakPtr](const drogon::HttpRequestPtr &req,\n                  drogon::AdviceCallback &&acb,\n                  drogon::AdviceChainCallback &&accb) {\n            auto thisPtr = weakPtr.lock();\n            if (!thisPtr)\n            {\n                accb();\n                return;\n            }\n            if (thisPtr->regexFlag_)\n            {\n                if (std::regex_match(req->path(), thisPtr->exemptPegex_))\n                {\n                    accb();\n                    return;\n                }\n            }\n\n            drogon::filters_function::doFilters(\n                thisPtr->filters_,\n                std::static_pointer_cast<HttpRequestImpl>(req),\n                [acb = std::move(acb),\n                 accb = std::move(accb)](const HttpResponsePtr &resp) {\n                    if (resp)\n                    {\n                        acb(resp);\n                    }\n                    else\n                    {\n                        accb();\n                    }\n                });\n        });\n}\n\nvoid GlobalFilters::shutdown()\n{\n    filters_.clear();\n}\n\n// Path: lib/src/HttpControllerBinder.cc\n/**\n *\n *  @file HttpControllerBinder.cc\n *  @author Nitromelon\n *\n *  Copyright 2023, Nitromelon.  All rights reserved.\n *  https://github.com/drogonframework/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpControllerBinder.h\"\n#include \"HttpResponseImpl.h\"\n#include <drogon/HttpSimpleController.h>\n#include <drogon/WebSocketController.h>\n\nnamespace drogon\n{\n\nvoid HttpSimpleControllerBinder::handleRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    // Binders without controller should be removed after run()\n    assert(controller_);\n\n    try\n    {\n        auto cb = callback;  // copy\n        controller_->asyncHandleHttpRequest(req, std::move(cb));\n    }\n    catch (const std::exception &e)\n    {\n        app().getExceptionHandler()(e, req, std::move(callback));\n        return;\n    }\n    catch (...)\n    {\n        LOG_ERROR << \"Exception not derived from std::exception\";\n        return;\n    }\n}\n\nvoid HttpControllerBinder::handleRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    auto &paramsVector = req->getRoutingParameters();\n    std::deque<std::string> params(paramsVector.begin(), paramsVector.end());\n    binderPtr_->handleHttpRequest(params, req, std::move(callback));\n}\n\nvoid WebsocketControllerBinder::handleRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    std::string wsKey = req->getHeaderBy(\"sec-websocket-key\");\n    wsKey.append(\"258EAFA5-E914-47DA-95CA-C5AB0DC85B11\");\n    unsigned char accKey[20];\n    auto sha1 = trantor::utils::sha1(wsKey.c_str(), wsKey.length());\n    memcpy(accKey, &sha1, sizeof(sha1));\n    auto base64Key = utils::base64Encode(accKey, sizeof(accKey));\n    auto resp = HttpResponse::newHttpResponse();\n    resp->setStatusCode(k101SwitchingProtocols);\n    resp->addHeader(\"Upgrade\", \"websocket\");\n    resp->addHeader(\"Connection\", \"Upgrade\");\n    resp->addHeader(\"Sec-WebSocket-Accept\", base64Key);\n    callback(resp);\n}\n\nvoid WebsocketControllerBinder::handleNewConnection(\n    const HttpRequestImplPtr &req,\n    const WebSocketConnectionImplPtr &wsConnPtr) const\n{\n    auto ctrlPtr = controller_;\n    assert(ctrlPtr);\n    wsConnPtr->setMessageCallback(\n        [ctrlPtr](std::string &&message,\n                  const WebSocketConnectionImplPtr &connPtr,\n                  const WebSocketMessageType &type) {\n            ctrlPtr->handleNewMessage(connPtr, std::move(message), type);\n        });\n    wsConnPtr->setCloseCallback(\n        [ctrlPtr](const WebSocketConnectionImplPtr &connPtr) {\n            ctrlPtr->handleConnectionClosed(connPtr);\n        });\n    ctrlPtr->handleNewConnection(req, wsConnPtr);\n}\n\n}  // namespace drogon\n\n// Path: lib/src/HttpBinder.cc\n/**\n *\n *  HttpBinder.h\n *  Martin Chang\n *\n *  Copyright 2021, Martin Chang.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/HttpBinder.h>\n#include <drogon/HttpAppFramework.h>\n\nnamespace drogon\n{\nnamespace internal\n{\nvoid handleException(const std::exception &e,\n                     const HttpRequestPtr &req,\n                     std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    app().getExceptionHandler()(e, req, std::move(callback));\n}\n}  // namespace internal\n}  // namespace drogon\n\n// Path: lib/src/YamlConfigAdapter.h\n#pragma once\n#include \"ConfigAdapter.h\"\n\nnamespace drogon\n{\nclass YamlConfigAdapter : public ConfigAdapter\n{\n  public:\n    YamlConfigAdapter() = default;\n    ~YamlConfigAdapter() override = default;\n    Json::Value getJson(const std::string &content) const\n        noexcept(false) override;\n    std::vector<std::string> getExtensions() const override;\n};\n}  // namespace drogon\n\n// Path: lib/src/YamlConfigAdapter.cc\n#include \"YamlConfigAdapter.h\"\n#ifdef HAS_YAML_CPP\n#include <yaml-cpp/yaml.h>\n#endif\nusing namespace drogon;\n#ifdef HAS_YAML_CPP\nnamespace YAML\n{\nstatic bool yaml2json(const Node &node, Json::Value &jsonValue)\n{\n    if (node.IsNull())\n    {\n        return false;\n    }\n    else if (node.IsScalar())\n    {\n        if (node.Tag() != \"!\")\n        {\n            try\n            {\n                jsonValue = node.as<Json::Value::Int64>();\n                return true;\n            }\n            catch (const YAML::BadConversion &e)\n            {\n            }\n            try\n            {\n                jsonValue = node.as<double>();\n                return true;\n            }\n            catch (const YAML::BadConversion &e)\n            {\n            }\n            try\n            {\n                jsonValue = node.as<bool>();\n                return true;\n            }\n            catch (const YAML::BadConversion &e)\n            {\n            }\n        }\n\n        Json::Value v(node.Scalar());\n        jsonValue.swapPayload(v);\n        return true;\n    }\n    else if (node.IsSequence())\n    {\n        for (std::size_t i = 0; i < node.size(); i++)\n        {\n            Json::Value v;\n            if (yaml2json(node[i], v))\n            {\n                jsonValue.append(v);\n            }\n            else\n            {\n                return false;\n            }\n        }\n\n        return true;\n    }\n    else if (node.IsMap())\n    {\n        for (YAML::const_iterator it = node.begin(); it != node.end(); ++it)\n        {\n            Json::Value v;\n            if (yaml2json(it->second, v))\n            {\n                jsonValue[it->first.Scalar()] = v;\n            }\n            else\n            {\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    return false;\n}\n\ntemplate <>\nstruct convert<Json::Value>\n{\n    static bool decode(const Node &node, Json::Value &rhs)\n    {\n        return yaml2json(node, rhs);\n    };\n};\n}  // namespace YAML\n\n#endif\nJson::Value YamlConfigAdapter::getJson(const std::string &content) const\n    noexcept(false)\n{\n#if HAS_YAML_CPP\n    // parse yaml file\n    YAML::Node config = YAML::Load(content);\n    if (!config.IsNull())\n    {\n        return config.as<Json::Value>();\n    }\n    else\n        return Json::Value();\n#else\n    throw std::runtime_error(\"please install yaml-cpp library\");\n#endif\n}\n\nstd::vector<std::string> YamlConfigAdapter::getExtensions() const\n{\n    return {\"yaml\", \"yml\"};\n}\n\n// Path: lib/src/FixedWindowRateLimiter.h\n#pragma once\n\n#include <drogon/RateLimiter.h>\n#include <chrono>\n\nnamespace drogon\n{\nclass FixedWindowRateLimiter : public RateLimiter\n{\n  public:\n    FixedWindowRateLimiter(size_t capacity,\n                           std::chrono::duration<double> timeUnit);\n    bool isAllowed() override;\n    ~FixedWindowRateLimiter() noexcept override = default;\n\n  private:\n    size_t capacity_;\n    size_t currentRequests_{0};\n    std::chrono::steady_clock::time_point lastTime_;\n    std::chrono::duration<double> timeUnit_;\n};\n}  // namespace drogon\n\n// Path: lib/src/SlidingWindowRateLimiter.h\n#pragma once\n#include <drogon/RateLimiter.h>\n#include <chrono>\n\nnamespace drogon\n{\nclass SlidingWindowRateLimiter : public RateLimiter\n{\n  public:\n    SlidingWindowRateLimiter(size_t capacity,\n                             std::chrono::duration<double> timeUnit);\n    bool isAllowed() override;\n    ~SlidingWindowRateLimiter() noexcept override = default;\n\n  private:\n    size_t capacity_;\n    size_t currentRequests_{0};\n    size_t previousRequests_{0};\n    std::chrono::steady_clock::time_point unitStartTime_;\n    std::chrono::steady_clock::time_point lastTime_;\n    std::chrono::duration<double> timeUnit_;\n};\n}  // namespace drogon\n\n// Path: lib/src/TokenBucketRateLimiter.h\n#pragma once\n\n#include <drogon/RateLimiter.h>\n\nnamespace drogon\n{\nclass TokenBucketRateLimiter : public RateLimiter\n{\n  public:\n    TokenBucketRateLimiter(size_t capacity,\n                           std::chrono::duration<double> timeUnit);\n    bool isAllowed() override;\n    ~TokenBucketRateLimiter() noexcept override = default;\n\n  private:\n    size_t capacity_;\n    std::chrono::steady_clock::time_point lastTime_;\n    std::chrono::duration<double> timeUnit_;\n    double tokens_;\n};\n}  // namespace drogon\n\n// Path: lib/src/RateLimiter.cc\n#include <drogon/RateLimiter.h>\n#include \"FixedWindowRateLimiter.h\"\n#include \"SlidingWindowRateLimiter.h\"\n#include \"TokenBucketRateLimiter.h\"\n\nusing namespace drogon;\n\nRateLimiterPtr RateLimiter::newRateLimiter(\n    RateLimiterType type,\n    size_t capacity,\n    std::chrono::duration<double> timeUnit)\n{\n    switch (type)\n    {\n        case RateLimiterType::kFixedWindow:\n            return std::make_shared<FixedWindowRateLimiter>(capacity, timeUnit);\n        case RateLimiterType::kSlidingWindow:\n            return std::make_shared<SlidingWindowRateLimiter>(capacity,\n                                                              timeUnit);\n        case RateLimiterType::kTokenBucket:\n            return std::make_shared<TokenBucketRateLimiter>(capacity, timeUnit);\n    }\n    return std::make_shared<TokenBucketRateLimiter>(capacity, timeUnit);\n}\n\n// Path: lib/src/SlashRemover.cc\n#include <drogon/plugins/SlashRemover.h>\n#include <drogon/plugins/Redirector.h>\n#include <drogon/HttpAppFramework.h>\n#include \"drogon/utils/FunctionTraits.h\"\n#include <cstddef>\n#include <cstdint>\n#include <functional>\n#include <string>\n#include <string_view>\n#include <utility>\n\nusing namespace drogon;\nusing namespace drogon::plugin;\nusing std::string;\nusing std::string_view;\n\nenum removeSlashMode : uint8_t\n{\n    trailing = 1 << 0,\n    duplicate = 1 << 1,\n    both = trailing | duplicate,\n};\n\n/// Returns the index before the trailing slashes,\n/// or 0 if only contains slashes\nstatic inline size_t findTrailingSlashes(string_view url)\n{\n    auto len = url.size();\n    // Must be at least 2 chars and end with a slash\n    if (len < 2 || url.back() != '/')\n        return string::npos;\n\n    size_t a = len - 1;  // We already know the last char is '/',\n                         // we will use pre-decrement to account for this\n    while (--a > 0 && url[a] == '/')\n        ;  // We know the first char is '/', so don't check for 0\n    return a;\n}\n\nstatic inline void removeTrailingSlashes(string &url,\n                                         size_t start,\n                                         string_view originalUrl)\n{\n    url = originalUrl.substr(0, start + 1);\n}\n\n/// Returns the index of the 2nd duplicate slash\nstatic inline size_t findDuplicateSlashes(string_view url)\n{\n    size_t len = url.size();\n    if (len < 2)\n        return string::npos;\n\n    bool startedPair = true;  // Always starts with a slash\n    for (size_t a = 1; a < len; ++a)\n    {\n        if (url[a] != '/')  // Broken pair\n        {\n            startedPair = false;\n            continue;\n        }\n        if (startedPair)  // Matching pair\n            return a;\n        startedPair = true;\n    }\n\n    return string::npos;\n}\n\nstatic inline void removeDuplicateSlashes(string &url, size_t start)\n{\n    // +1 because we don't need to look at the same character again,\n    // which was found by `findDuplicateSlashes`, it saves one iteration\n    for (size_t b = (start--) + 1, len = url.size(); b < len; ++b)\n    {\n        const char c = url[b];\n        if (c != '/' || url[start] != '/')\n        {\n            ++start;\n            url[start] = c;\n        }\n    }\n    url.resize(start + 1);\n}\n\nstatic inline std::pair<size_t, size_t> findExcessiveSlashes(string_view url)\n{\n    size_t len = url.size();\n    if (len < 2)  // Must have at least 2 characters to count as either trailing\n                  // or duplicate slash\n        return {string::npos, string::npos};\n\n    // Trail finder\n    size_t trailIdx = len;  // The pre-decrement will put it on last char\n    while (--trailIdx > 0 && url[trailIdx] == '/')\n        ;  // We know first char is '/', no need to check it\n\n    // Filled with '/'\n    if (trailIdx == 0)\n        return {\n            0,             // Only keep first slash\n            string::npos,  // No duplicate\n        };\n\n    // Look for a duplicate pair\n    size_t dupIdx = 1;\n    for (bool startedPair = true; dupIdx < trailIdx;\n         ++dupIdx)  // Always starts with a slash\n    {\n        if (url[dupIdx] != '/')  // Broken pair\n        {\n            startedPair = false;\n            continue;\n        }\n        if (startedPair)  // Matching pair\n            break;\n        startedPair = true;\n    }\n\n    // Found no duplicate\n    if (dupIdx == trailIdx)\n        return {\n            trailIdx != len - 1\n                ?  // If has gone past last char, then there is a trailing slash\n                trailIdx\n                : string::npos,  // No trail\n            string::npos,        // No duplicate\n        };\n\n    // Duplicate found\n    return {\n        trailIdx != len - 1\n            ?  // If has gone past last char, then there is a trailing slash\n            trailIdx\n            : string::npos,  // No trail\n        dupIdx,\n    };\n}\n\nstatic inline void removeExcessiveSlashes(string &url,\n                                          std::pair<size_t, size_t> start,\n                                          string_view originalUrl)\n{\n    if (start.first != string::npos)\n        removeTrailingSlashes(url, start.first, originalUrl);\n    else\n        url = originalUrl;\n\n    if (start.second != string::npos)\n        removeDuplicateSlashes(url, start.second);\n}\n\nstatic inline bool handleReq(const drogon::HttpRequestPtr &req,\n                             uint8_t removeMode)\n{\n    switch (removeMode)\n    {\n        case trailing:\n        {\n            auto find = findTrailingSlashes(req->path());\n            if (find == string::npos)\n                return false;\n\n            string newPath;\n            removeTrailingSlashes(newPath, find, req->path());\n            req->setPath(std::move(newPath));\n            break;\n        }\n        case duplicate:\n        {\n            auto find = findDuplicateSlashes(req->path());\n            if (find == string::npos)\n                return false;\n\n            string newPath = req->path();\n            removeDuplicateSlashes(newPath, find);\n            req->setPath(std::move(newPath));\n            break;\n        }\n        case both:\n        default:\n        {\n            auto find = findExcessiveSlashes(req->path());\n            if (find.first == string::npos && find.second == string::npos)\n                return false;\n\n            string newPath;\n            removeExcessiveSlashes(newPath, find, req->path());\n            req->setPath(std::move(newPath));\n            break;\n        }\n    }\n    return true;\n}\n\nvoid SlashRemover::initAndStart(const Json::Value &config)\n{\n    trailingSlashes_ = config.get(\"remove_trailing_slashes\", true).asBool();\n    duplicateSlashes_ = config.get(\"remove_duplicate_slashes\", true).asBool();\n    redirect_ = config.get(\"redirect\", true).asBool();\n    const uint8_t removeMode =\n        (trailingSlashes_ * trailing) | (duplicateSlashes_ * duplicate);\n    if (!removeMode)\n        return;\n    auto redirector = app().getPlugin<Redirector>();\n    if (!redirector)\n    {\n        LOG_ERROR << \"Redirector plugin is not found!\";\n        return;\n    }\n    auto func = [removeMode](const HttpRequestPtr &req) -> bool {\n        return handleReq(req, removeMode);\n    };\n    if (redirect_)\n    {\n        redirector->registerPathRewriteHandler(std::move(func));\n    }\n    else\n    {\n        redirector->registerForwardHandler(std::move(func));\n    }\n}\n\nvoid SlashRemover::shutdown()\n{\n    LOG_TRACE << \"SlashRemover plugin is shutdown!\";\n}\n\n// Path: lib/src/NotFound.cc\n// this file is generated by program automatically,don't modify it!\n\n/**\n *\n *  NotFound.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/NotFound.h>\n#include <map>\n#include <set>\n#include <sstream>\n#include <string>\n#include <vector>\n\nusing namespace drogon;\n\nstd::string NotFound::genText(const HttpViewData &NotFound_view_data)\n{\n    std::stringstream NotFound_tmp_stream;\n    NotFound_tmp_stream << \"<html>\\n\";\n    NotFound_tmp_stream << \"<head><title>404 Not Found</title></head>\\n\";\n    NotFound_tmp_stream << \"<body bgcolor=\\\"white\\\" text=\\\"black\\\">\\n\";\n    NotFound_tmp_stream << \"<center><h1>404 Not Found</h1></center>\\n\";\n    NotFound_tmp_stream << \"<hr><center>drogon/\";\n    NotFound_tmp_stream << NotFound_view_data.get<std::string>(\"version\");\n    NotFound_tmp_stream << \"</center>\\n\";\n    NotFound_tmp_stream << \"</body>\\n\";\n    NotFound_tmp_stream << \"</html>\\n\";\n    NotFound_tmp_stream << \"<!-- a padding to disable MSIE and Chrome friendly \"\n                           \"error page -->\\n\";\n    NotFound_tmp_stream << \"<!-- a padding to disable MSIE and Chrome friendly \"\n                           \"error page -->\\n\";\n    NotFound_tmp_stream << \"<!-- a padding to disable MSIE and Chrome friendly \"\n                           \"error page -->\\n\";\n    NotFound_tmp_stream << \"<!-- a padding to disable MSIE and Chrome friendly \"\n                           \"error page -->\\n\";\n    NotFound_tmp_stream << \"<!-- a padding to disable MSIE and Chrome friendly \"\n                           \"error page -->\\n\";\n    NotFound_tmp_stream << \"<!-- a padding to disable MSIE and Chrome friendly \"\n                           \"error page -->\\n\";\n    return NotFound_tmp_stream.str();\n}\n\n// Path: lib/src/LocalHostFilter.cc\n/**\n *\n *  LocalHostFilter.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpResponseImpl.h\"\n#include <drogon/LocalHostFilter.h>\nusing namespace drogon;\n\nvoid LocalHostFilter::doFilter(const HttpRequestPtr &req,\n                               FilterCallback &&fcb,\n                               FilterChainCallback &&fccb)\n{\n    if (req->peerAddr().isLoopbackIp())\n    {\n        fccb();\n        return;\n    }\n    auto res = drogon::HttpResponse::newNotFoundResponse(req);\n    fcb(res);\n}\n\n// Path: lib/src/RedisResultSkipped.cc\n/**\n *\n *  RedisClientSkipped.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"drogon/nosql/RedisResult.h\"\n#include \"trantor/utils/Logger.h\"\n\nnamespace drogon\n{\nnamespace nosql\n{\nstd::string RedisResult::getStringForDisplaying() const noexcept\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n\nstd::string RedisResult::getStringForDisplayingWithIndent(\n    size_t /*indent*/) const noexcept\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n\nstd::string RedisResult::asString() const noexcept(false)\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n\nRedisResultType RedisResult::type() const noexcept\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n\nstd::vector<RedisResult> RedisResult::asArray() const noexcept(false)\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n\nlong long RedisResult::asInteger() const noexcept(false)\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n\nbool RedisResult::isNil() const noexcept\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n}  // namespace nosql\n}  // namespace drogon\n\n// Path: lib/src/TokenBucketRateLimiter.cc\n#include \"TokenBucketRateLimiter.h\"\n\nusing namespace drogon;\n\nTokenBucketRateLimiter::TokenBucketRateLimiter(\n    size_t capacity,\n    std::chrono::duration<double> timeUnit)\n    : capacity_(capacity),\n      lastTime_(std::chrono::steady_clock::now()),\n      timeUnit_(timeUnit),\n      tokens_((double)capacity_)\n{\n}\n\n// implementation of the token bucket algorithm\nbool TokenBucketRateLimiter::isAllowed()\n{\n    auto now = std::chrono::steady_clock::now();\n    auto duration = std::chrono::duration_cast<std::chrono::duration<double>>(\n        now - lastTime_);\n    tokens_ += capacity_ * (duration / timeUnit_);\n    if (tokens_ > capacity_)\n        tokens_ = (double)capacity_;\n    lastTime_ = now;\n    if (tokens_ > 1.0)\n    {\n        tokens_ -= 1.0;\n        return true;\n    }\n    return false;\n}\n\n// Path: lib/src/AccessLogger.cc\n/**\n *\n *  @file AccessLogger.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpUtils.h\"\n#include <drogon/drogon.h>\n#include <drogon/plugins/AccessLogger.h>\n#include <drogon/plugins/RealIpResolver.h>\n#include <regex>\n#include <thread>\n#if !defined _WIN32 && !defined __HAIKU__\n#include <unistd.h>\n#include <sys/syscall.h>\n#elif defined __HAIKU__\n#include <unistd.h>\n#else\n#include <sstream>\n#endif\n#ifdef __FreeBSD__\n#include <pthread_np.h>\n#endif\n\n#ifdef DROGON_SPDLOG_SUPPORT\n#include <spdlog/spdlog.h>\n#include <spdlog/logger.h>\n#include <spdlog/sinks/stdout_color_sinks.h>\n#include <spdlog/sinks/rotating_file_sink.h>\n#ifdef _WIN32\n#include <spdlog/sinks/msvc_sink.h>\n// Damn antedeluvian M$ macros\n#undef min\n#undef max\n#endif\n#ifndef _WIN32\n#include <sys/wait.h>\n#include <unistd.h>\n#define os_access access\n#elif !defined(_WIN32) || defined(__MINGW32__)\n#include <sys/file.h>\n#include <unistd.h>\n#define os_access access\n#else\n#include <io.h>\n#define os_access _waccess\n#define R_OK 04\n#define W_OK 02\n#endif\n#endif\n\nusing namespace drogon;\nusing namespace drogon::plugin;\n\nbool AccessLogger::useRealIp_ = false;\n\nvoid AccessLogger::initAndStart(const Json::Value &config)\n{\n    useLocalTime_ = config.get(\"use_local_time\", true).asBool();\n    showMicroseconds_ = config.get(\"show_microseconds\", true).asBool();\n    timeFormat_ = config.get(\"custom_time_format\", \"\").asString();\n    useCustomTimeFormat_ = !timeFormat_.empty();\n    useRealIp_ = config.get(\"use_real_ip\", false).asBool();\n\n    logFunctionMap_ = {{\"$request_path\", outputReqPath},\n                       {\"$path\", outputReqPath},\n                       {\"$date\",\n                        [this](trantor::LogStream &stream,\n                               const drogon::HttpRequestPtr &req,\n                               const drogon::HttpResponsePtr &resp) {\n                            outputDate(stream, req, resp);\n                        }},\n                       {\"$request_date\",\n                        [this](trantor::LogStream &stream,\n                               const drogon::HttpRequestPtr &req,\n                               const drogon::HttpResponsePtr &resp) {\n                            outputReqDate(stream, req, resp);\n                        }},\n                       {\"$request_query\", outputReqQuery},\n                       {\"$request_url\", outputReqURL},\n                       {\"$query\", outputReqQuery},\n                       {\"$url\", outputReqURL},\n                       {\"$request_version\", outputVersion},\n                       {\"$version\", outputVersion},\n                       {\"$request\", outputReqLine},\n                       {\"$remote_addr\", outputRemoteAddr},\n                       {\"$local_addr\", outputLocalAddr},\n                       {\"$request_len\", outputReqLength},\n                       {\"$body_bytes_received\", outputReqLength},\n                       {\"$method\", outputMethod},\n                       {\"$thread\", outputThreadNumber},\n                       {\"$response_len\", outputRespLength},\n                       {\"$body_bytes_sent\", outputRespLength},\n                       {\"$status\", outputStatusString},\n                       {\"$status_code\", outputStatusCode},\n                       {\"$processing_time\", outputProcessingTime},\n                       {\"$upstream_http_content-type\", outputRespContentType},\n                       {\"$upstream_http_content_type\", outputRespContentType}};\n    auto format = config.get(\"log_format\", \"\").asString();\n    if (format.empty())\n    {\n        format =\n            \"$request_date $method $url [$body_bytes_received] ($remote_addr - \"\n            \"$local_addr) $status $body_bytes_sent $processing_time\";\n    }\n    createLogFunctions(format);\n    auto logPath = config.get(\"log_path\", \"\").asString();\n#ifdef DROGON_SPDLOG_SUPPORT\n    auto logWithSpdlog = trantor::Logger::hasSpdLogSupport() &&\n                         config.get(\"use_spdlog\", false).asBool();\n    if (logWithSpdlog)\n    {\n        logIndex_ = config.get(\"log_index\", 0).asInt();\n        // Do nothing if already initialized...\n        if (!trantor::Logger::getSpdLogger(logIndex_))\n        {\n            trantor::Logger::enableSpdLog(logIndex_);\n            // Get the new logger & replace its sinks with the ones of the\n            // config\n            auto logger = trantor::Logger::getSpdLogger(logIndex_);\n            std::vector<spdlog::sink_ptr> sinks;\n            while (!logPath.empty())\n            {\n                // 1. check existence of folder or try to create it\n                auto fsLogPath =\n                    std::filesystem::path(utils::toNativePath(logPath));\n                std::error_code fsErr;\n                if (!std::filesystem::create_directories(fsLogPath, fsErr) &&\n                    fsErr)\n                {\n                    LOG_ERROR << \"could not create log file path\";\n                    break;\n                }\n                // 2. check if we have rights to create files in the folder\n                if (os_access(fsLogPath.native().c_str(), W_OK) != 0)\n                {\n                    LOG_ERROR << \"cannot create files in log folder\";\n                    break;\n                }\n                std::filesystem::path fileName(\n                    config.get(\"log_file\", \"access.log\").asString());\n                if (fileName.empty())\n                    fileName = \"access.log\";\n                else\n                    fileName.replace_extension(\".log\");\n                auto sizeLimit = config.get(\"log_size_limit\", 0).asUInt64();\n                if (sizeLimit == 0)\n                    sizeLimit = config.get(\"size_limit\", 0).asUInt64();\n                if (sizeLimit == 0)  // 0 is not allowed by this sink\n                    sizeLimit = std::numeric_limits<std::size_t>::max();\n                std::size_t maxFiles = config.get(\"max_files\", 0).asUInt();\n                sinks.push_back(\n                    std::make_shared<spdlog::sinks::rotating_file_sink_mt>(\n                        (fsLogPath / fileName).string(),\n                        sizeLimit,\n                        // spdlog limitation\n                        std::min(maxFiles, std::size_t(20000)),\n                        false));\n                break;\n            }\n            if (sinks.empty())\n                sinks.push_back(\n                    std::make_shared<spdlog::sinks::stderr_color_sink_mt>());\n#if defined(_WIN32) && defined(_DEBUG)\n            // On Windows with debug, it may be interesting to have the logs\n            // directly in the Visual Studio / WinDbg console\n            sinks.push_back(std::make_shared<spdlog::sinks::msvc_sink_mt>());\n#endif\n            logger->sinks() = sinks;\n            // Override the pattern set in\n            // trantor::Logger::getDefaultSpdLogger() and let AccessLogger\n            // format the output\n            logger->set_pattern(\"%v\");\n        }\n    }\n    else\n#endif\n        if (!logPath.empty())\n    {\n        auto fileName = config.get(\"log_file\", \"access.log\").asString();\n        auto extension = std::string(\".log\");\n        auto pos = fileName.rfind('.');\n        if (pos != std::string::npos)\n        {\n            extension = fileName.substr(pos);\n            fileName = fileName.substr(0, pos);\n        }\n        if (fileName.empty())\n        {\n            fileName = \"access\";\n        }\n        asyncFileLogger_.setFileName(fileName, extension, logPath);\n        asyncFileLogger_.startLogging();\n        logIndex_ = config.get(\"log_index\", 0).asInt();\n        trantor::Logger::setOutputFunction(\n            [&](const char *msg, const uint64_t len) {\n                asyncFileLogger_.output(msg, len);\n            },\n            [&]() { asyncFileLogger_.flush(); },\n            logIndex_);\n        auto sizeLimit = config.get(\"log_size_limit\", 0).asUInt64();\n        if (sizeLimit == 0)\n        {\n            // In earlier code, \"size_limit\" is taken instead of\n            // \"log_size_limit\" as it said in the comment in AccessLogger.h.\n            // In order to ensure backward compatibility we still take this\n            // field as a fallback.\n            sizeLimit = config.get(\"size_limit\", 0).asUInt64();\n        }\n        if (sizeLimit > 0)\n        {\n            asyncFileLogger_.setFileSizeLimit(sizeLimit);\n        }\n        auto maxFiles = config.get(\"max_files\", 0).asUInt();\n        if (maxFiles >= 0)\n        {\n            asyncFileLogger_.setMaxFiles(maxFiles);\n        }\n    }\n    drogon::app().registerPreSendingAdvice(\n        [this](const drogon::HttpRequestPtr &req,\n               const drogon::HttpResponsePtr &resp) {\n            logging(LOG_RAW_TO(logIndex_), req, resp);\n        });\n}\n\nvoid AccessLogger::shutdown()\n{\n}\n\nvoid AccessLogger::logging(trantor::LogStream &stream,\n                           const drogon::HttpRequestPtr &req,\n                           const drogon::HttpResponsePtr &resp)\n{\n    for (auto &func : logFunctions_)\n    {\n        func(stream, req, resp);\n    }\n}\n\nvoid AccessLogger::createLogFunctions(std::string format)\n{\n    std::string rawString;\n    while (!format.empty())\n    {\n        LOG_TRACE << format;\n        auto pos = format.find('$');\n        if (pos != std::string::npos)\n        {\n            rawString += format.substr(0, pos);\n\n            format = format.substr(pos);\n            std::regex e{\"^\\\\$[a-zA-Z0-9\\\\-_]+\"};\n            std::smatch m;\n            if (std::regex_search(format, m, e))\n            {\n                if (!rawString.empty())\n                {\n                    logFunctions_.emplace_back(\n                        [rawString](trantor::LogStream &stream,\n                                    const drogon::HttpRequestPtr &,\n                                    const drogon::HttpResponsePtr &) {\n                            stream << rawString;\n                        });\n                    rawString.clear();\n                }\n                auto placeholder = m[0];\n                logFunctions_.emplace_back(newLogFunction(placeholder));\n                format = m.suffix().str();\n            }\n            else\n            {\n                rawString += '$';\n                format = format.substr(1);\n            }\n        }\n        else\n        {\n            rawString += format;\n            break;\n        }\n    }\n    if (!rawString.empty())\n    {\n        logFunctions_.emplace_back(\n            [rawString =\n                 std::move(rawString)](trantor::LogStream &stream,\n                                       const drogon::HttpRequestPtr &,\n                                       const drogon::HttpResponsePtr &) {\n                stream << rawString << \"\\n\";\n            });\n    }\n    else\n    {\n        logFunctions_.emplace_back(\n            [](trantor::LogStream &stream,\n               const drogon::HttpRequestPtr &,\n               const drogon::HttpResponsePtr &) { stream << \"\\n\"; });\n    }\n}\n\nAccessLogger::LogFunction AccessLogger::newLogFunction(\n    const std::string &placeholder)\n{\n    auto iter = logFunctionMap_.find(placeholder);\n    if (iter != logFunctionMap_.end())\n    {\n        return iter->second;\n    }\n    if (placeholder.find(\"$http_\") == 0 && placeholder.size() > 6)\n    {\n        auto headerName = placeholder.substr(6);\n        return [headerName =\n                    std::move(headerName)](trantor::LogStream &stream,\n                                           const drogon::HttpRequestPtr &req,\n                                           const drogon::HttpResponsePtr &) {\n            outputReqHeader(stream, req, headerName);\n        };\n    }\n    if (placeholder.find(\"$cookie_\") == 0 && placeholder.size() > 8)\n    {\n        auto cookieName = placeholder.substr(8);\n        return [cookieName =\n                    std::move(cookieName)](trantor::LogStream &stream,\n                                           const drogon::HttpRequestPtr &req,\n                                           const drogon::HttpResponsePtr &) {\n            outputReqCookie(stream, req, cookieName);\n        };\n    }\n    if (placeholder.find(\"$upstream_http_\") == 0 && placeholder.size() > 15)\n    {\n        auto headerName = placeholder.substr(15);\n        return [headerName = std::move(\n                    headerName)](trantor::LogStream &stream,\n                                 const drogon::HttpRequestPtr &,\n                                 const drogon::HttpResponsePtr &resp) {\n            outputRespHeader(stream, resp, headerName);\n        };\n    }\n    return [placeholder](trantor::LogStream &stream,\n                         const drogon::HttpRequestPtr &,\n                         const drogon::HttpResponsePtr &) {\n        stream << placeholder;\n    };\n}\n\nvoid AccessLogger::outputReqPath(trantor::LogStream &stream,\n                                 const drogon::HttpRequestPtr &req,\n                                 const drogon::HttpResponsePtr &)\n{\n    stream << req->path();\n}\n\nvoid AccessLogger::outputDate(trantor::LogStream &stream,\n                              const drogon::HttpRequestPtr &,\n                              const drogon::HttpResponsePtr &) const\n{\n    if (useCustomTimeFormat_)\n    {\n        if (useLocalTime_)\n        {\n            stream << trantor::Date::now().toCustomedFormattedStringLocal(\n                timeFormat_, showMicroseconds_);\n        }\n        else\n        {\n            stream << trantor::Date::now().toCustomedFormattedString(\n                timeFormat_, showMicroseconds_);\n        }\n    }\n    else\n    {\n        if (useLocalTime_)\n        {\n            stream << trantor::Date::now().toFormattedStringLocal(\n                showMicroseconds_);\n        }\n        else\n        {\n            stream << trantor::Date::now().toFormattedString(showMicroseconds_);\n        }\n    }\n}\n\nvoid AccessLogger::outputReqDate(trantor::LogStream &stream,\n                                 const drogon::HttpRequestPtr &req,\n                                 const drogon::HttpResponsePtr &) const\n{\n    if (useCustomTimeFormat_)\n    {\n        if (useLocalTime_)\n        {\n            stream << req->creationDate().toCustomedFormattedStringLocal(\n                timeFormat_, showMicroseconds_);\n        }\n        else\n        {\n            stream << req->creationDate().toCustomedFormattedString(\n                timeFormat_, showMicroseconds_);\n        }\n    }\n    else\n    {\n        if (useLocalTime_)\n        {\n            stream << req->creationDate().toFormattedStringLocal(\n                showMicroseconds_);\n        }\n        else\n        {\n            stream << req->creationDate().toFormattedString(showMicroseconds_);\n        }\n    }\n}\n\n//$request_query\nvoid AccessLogger::outputReqQuery(trantor::LogStream &stream,\n                                  const drogon::HttpRequestPtr &req,\n                                  const drogon::HttpResponsePtr &)\n{\n    stream << req->query();\n}\n\n//$request_url\nvoid AccessLogger::outputReqURL(trantor::LogStream &stream,\n                                const drogon::HttpRequestPtr &req,\n                                const drogon::HttpResponsePtr &)\n{\n    auto &query = req->query();\n    if (query.empty())\n    {\n        stream << req->path();\n    }\n    else\n    {\n        stream << req->path() << '?' << query;\n    }\n}\n\n//$request_version\nvoid AccessLogger::outputVersion(trantor::LogStream &stream,\n                                 const drogon::HttpRequestPtr &req,\n                                 const drogon::HttpResponsePtr &)\n{\n    stream << req->versionString();\n}\n\n//$request\nvoid AccessLogger::outputReqLine(trantor::LogStream &stream,\n                                 const drogon::HttpRequestPtr &req,\n                                 const drogon::HttpResponsePtr &)\n{\n    auto &query = req->query();\n    if (query.empty())\n    {\n        stream << req->methodString() << \" \" << req->path() << \" \"\n               << req->versionString();\n    }\n    else\n    {\n        stream << req->methodString() << \" \" << req->path() << '?' << query\n               << \" \" << req->versionString();\n    }\n}\n\nvoid AccessLogger::outputRemoteAddr(trantor::LogStream &stream,\n                                    const drogon::HttpRequestPtr &req,\n                                    const drogon::HttpResponsePtr &)\n{\n    if (useRealIp_)\n    {\n        stream << RealIpResolver::GetRealAddr(req).toIpPort();\n    }\n    else\n    {\n        stream << req->peerAddr().toIpPort();\n    }\n}\n\nvoid AccessLogger::outputLocalAddr(trantor::LogStream &stream,\n                                   const drogon::HttpRequestPtr &req,\n                                   const drogon::HttpResponsePtr &)\n{\n    stream << req->localAddr().toIpPort();\n}\n\nvoid AccessLogger::outputReqLength(trantor::LogStream &stream,\n                                   const drogon::HttpRequestPtr &req,\n                                   const drogon::HttpResponsePtr &)\n{\n    stream << req->body().length();\n}\n\nvoid AccessLogger::outputRespLength(trantor::LogStream &stream,\n                                    const drogon::HttpRequestPtr &,\n                                    const drogon::HttpResponsePtr &resp)\n{\n    stream << resp->body().length();\n}\n\nvoid AccessLogger::outputMethod(trantor::LogStream &stream,\n                                const drogon::HttpRequestPtr &req,\n                                const drogon::HttpResponsePtr &)\n{\n    stream << req->methodString();\n}\n\nvoid AccessLogger::outputThreadNumber(trantor::LogStream &stream,\n                                      const drogon::HttpRequestPtr &,\n                                      const drogon::HttpResponsePtr &)\n{\n#ifdef __linux__\n    static thread_local pid_t threadId_{0};\n#else\n    static thread_local uint64_t threadId_{0};\n#endif\n#ifdef __linux__\n    if (threadId_ == 0)\n        threadId_ = static_cast<pid_t>(::syscall(SYS_gettid));\n#elif defined __FreeBSD__\n    if (threadId_ == 0)\n    {\n        threadId_ = pthread_getthreadid_np();\n    }\n#elif defined __OpenBSD__\n    if (threadId_ == 0)\n    {\n        threadId_ = getthrid();\n    }\n#elif defined _WIN32 || defined __HAIKU__\n    if (threadId_ == 0)\n    {\n        std::stringstream ss;\n        ss << std::this_thread::get_id();\n        threadId_ = std::stoull(ss.str());\n    }\n#else\n    if (threadId_ == 0)\n    {\n        pthread_threadid_np(NULL, &threadId_);\n    }\n#endif\n    stream << threadId_;\n}\n\n//$http_[header_name]\nvoid AccessLogger::outputReqHeader(trantor::LogStream &stream,\n...\n", "needle_token_start": 2400, "needle_token_end": 2407, "code_context_ntokens": 16109, "output": ["```cpp\nJson::Value JsonConfigAdapter::getJson(const std::string &content) const\n    noexcept(false)\n{\n    static std::once_flag once;\n    static Json::CharReaderBuilder builder;\n    std::call_once(once, []() { builder[\"collectComments\"] = false; });\n    JSONCPP_STRING errs;\n    std::unique_ptr<Json::CharReader> reader(builder.newCharReader());\n    Json::Value root;\n    if (!reader->parse(\n            content.c_str(), content.c_str() + content.size(), &root, &errs))\n    {\n        throw std::runtime_error(errs);\n    }\n    return root;\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "registerCustomExtensionMime", "language": "cpp", "path": "lib/src/HttpUtils.cc", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to add or update a custom file extension to MIME type mapping in a global map. This allows the system to recognize and handle files with custom extensions appropriately.\n2. **Input**: It takes two strings as parameters: the first represents the file extension (without the dot), and the second is the corresponding MIME type.\n3. **Output**: There is no return value; the function operates by side effects, modifying a global map.\n4. **Procedure**: The function first checks if the input extension is non-empty. If it is, it checks if this extension already has a MIME type associated with it in the map. If an association exists, a warning is logged indicating that the existing MIME type is being overwritten. Finally, the new MIME type is set for the given extension in the map.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        {\"rar\", {FT_ARCHIVE, CT_APPLICATION_VND_RAR}},\n        {\"svg\", {FT_IMAGE, CT_IMAGE_SVG_XML}},\n        {\"tar\", {FT_ARCHIVE, CT_APPLICATION_X_TAR}},\n        {\"targa\", {FT_IMAGE, CT_IMAGE_X_TGA}},\n        {\"tif\", {FT_IMAGE, CT_IMAGE_TIFF}},\n        {\"tiff\", {FT_IMAGE, CT_IMAGE_TIFF}},\n        {\"tga\", {FT_IMAGE, CT_IMAGE_X_TGA}},\n        {\"tgz\", {FT_ARCHIVE, CT_APPLICATION_X_TGZ}},\n        {\"ts\", {FT_MEDIA, CT_VIDEO_MPEG2TS}},\n        {\"tta\", {FT_AUDIO, CT_AUDIO_X_TTA}},\n        {\"ttf\", {FT_DOCUMENT, CT_APPLICATION_X_FONT_TRUETYPE}},\n        {\"txt\", {FT_DOCUMENT, CT_TEXT_PLAIN}},\n        {\"w64\", {FT_AUDIO, CT_AUDIO_WAVE}},\n        {\"wav\", {FT_AUDIO, CT_AUDIO_WAVE}},\n        {\"wave\", {FT_AUDIO, CT_AUDIO_WAVE}},\n        {\"wasm\", {FT_DOCUMENT, CT_APPLICATION_WASM}},\n        {\"weba\", {FT_AUDIO, CT_AUDIO_WEBM}},\n        {\"webm\", {FT_MEDIA, CT_VIDEO_WEBM}},\n        {\"webp\", {FT_IMAGE, CT_IMAGE_WEBP}},\n        {\"wma\", {FT_AUDIO, CT_AUDIO_X_MS_WMA}},\n        {\"woff\", {FT_DOCUMENT, CT_APPLICATION_FONT_WOFF}},\n        {\"woff2\", {FT_DOCUMENT, CT_APPLICATION_FONT_WOFF2}},\n        {\"wv\", {FT_AUDIO, CT_AUDIO_X_WAVPACK}},\n        {\"xht\", {FT_DOCUMENT, CT_APPLICATION_XHTML}},\n        {\"xhtml\", {FT_DOCUMENT, CT_APPLICATION_XHTML}},\n        {\"xml\", {FT_DOCUMENT, CT_APPLICATION_XML}},\n        {\"xsl\", {FT_DOCUMENT, CT_TEXT_XSL}},\n        {\"xz\", {FT_ARCHIVE, CT_APPLICATION_X_XZ}},\n        {\"zip\", {FT_ARCHIVE, CT_APPLICATION_ZIP}},\n        {\"7z\", {FT_ARCHIVE, CT_APPLICATION_X_7Z}},\n    };\n\nconst std::string_view &statusCodeToString(int code)\n{\n    switch (code)\n    {\n        case 100:\n        {\n            static std::string_view sv = \"Continue\";\n            return sv;\n        }\n        case 101:\n        {\n            static std::string_view sv = \"Switching Protocols\";\n            return sv;\n        }\n        case 102:\n        {\n            static std::string_view sv = \"Processing\";\n            return sv;\n        }\n        case 103:\n        {\n            static std::string_view sv = \"Early Hints\";\n            return sv;\n        }\n        case 200:\n        {\n            static std::string_view sv = \"OK\";\n            return sv;\n        }\n        case 201:\n        {\n            static std::string_view sv = \"Created\";\n            return sv;\n        }\n        case 202:\n        {\n            static std::string_view sv = \"Accepted\";\n            return sv;\n        }\n        case 203:\n        {\n            static std::string_view sv = \"Non-Authoritative Information\";\n            return sv;\n        }\n        case 204:\n        {\n            static std::string_view sv = \"No Content\";\n            return sv;\n        }\n        case 205:\n        {\n            static std::string_view sv = \"Reset Content\";\n            return sv;\n        }\n        case 206:\n        {\n            static std::string_view sv = \"Partial Content\";\n            return sv;\n        }\n        case 207:\n        {\n            static std::string_view sv = \"Multi-Status\";\n            return sv;\n        }\n        case 208:\n        {\n            static std::string_view sv = \"Already Reported\";\n            return sv;\n        }\n        case 226:\n        {\n            static std::string_view sv = \"IM Used\";\n            return sv;\n        }\n        case 300:\n        {\n            static std::string_view sv = \"Multiple Choices\";\n            return sv;\n        }\n        case 301:\n        {\n            static std::string_view sv = \"Moved Permanently\";\n            return sv;\n        }\n        case 302:\n        {\n            static std::string_view sv = \"Found\";\n            return sv;\n        }\n        case 303:\n        {\n            static std::string_view sv = \"See Other\";\n            return sv;\n        }\n        case 304:\n        {\n            static std::string_view sv = \"Not Modified\";\n            return sv;\n        }\n        case 305:\n        {\n            static std::string_view sv = \"Use Proxy\";\n            return sv;\n        }\n        case 306:\n        {\n            static std::string_view sv = \"(Unused)\";\n            return sv;\n        }\n        case 307:\n        {\n            static std::string_view sv = \"Temporary Redirect\";\n            return sv;\n        }\n        case 308:\n        {\n            static std::string_view sv = \"Permanent Redirect\";\n            return sv;\n        }\n        case 400:\n        {\n            static std::string_view sv = \"Bad Request\";\n            return sv;\n        }\n        case 401:\n        {\n            static std::string_view sv = \"Unauthorized\";\n            return sv;\n        }\n        case 402:\n        {\n            static std::string_view sv = \"Payment Required\";\n            return sv;\n        }\n        case 403:\n        {\n            static std::string_view sv = \"Forbidden\";\n            return sv;\n        }\n        case 404:\n        {\n            static std::string_view sv = \"Not Found\";\n            return sv;\n        }\n        case 405:\n        {\n            static std::string_view sv = \"Method Not Allowed\";\n            return sv;\n        }\n        case 406:\n        {\n            static std::string_view sv = \"Not Acceptable\";\n            return sv;\n        }\n        case 407:\n        {\n            static std::string_view sv = \"Proxy Authentication Required\";\n            return sv;\n        }\n        case 408:\n        {\n            static std::string_view sv = \"Request Time-out\";\n            return sv;\n        }\n        case 409:\n        {\n            static std::string_view sv = \"Conflict\";\n            return sv;\n        }\n        case 410:\n        {\n            static std::string_view sv = \"Gone\";\n            return sv;\n        }\n        case 411:\n        {\n            static std::string_view sv = \"Length Required\";\n            return sv;\n        }\n        case 412:\n        {\n            static std::string_view sv = \"Precondition Failed\";\n            return sv;\n        }\n        case 413:\n        {\n            static std::string_view sv = \"Request Entity Too Large\";\n            return sv;\n        }\n        case 414:\n        {\n            static std::string_view sv = \"Request-URI Too Large\";\n            return sv;\n        }\n        case 415:\n        {\n            static std::string_view sv = \"Unsupported Media Type\";\n            return sv;\n        }\n        case 416:\n        {\n            static std::string_view sv = \"Requested Range Not Satisfiable\";\n            return sv;\n        }\n        case 417:\n        {\n            static std::string_view sv = \"Expectation Failed\";\n            return sv;\n        }\n        case 418:\n        {\n            static std::string_view sv = \"I'm a Teapot\";\n            return sv;\n        }\n        case 421:\n        {\n            static std::string_view sv = \"Misdirected Request\";\n            return sv;\n        }\n        case 422:\n        {\n            static std::string_view sv = \"Unprocessable Entity\";\n            return sv;\n        }\n        case 423:\n        {\n            static std::string_view sv = \"Locked\";\n            return sv;\n        }\n        case 424:\n        {\n            static std::string_view sv = \"Failed Dependency\";\n            return sv;\n        }\n        case 425:\n        {\n            static std::string_view sv = \"Too Early\";\n            return sv;\n        }\n        case 426:\n        {\n            static std::string_view sv = \"Upgrade Required\";\n            return sv;\n        }\n        case 428:\n        {\n            static std::string_view sv = \"Precondition Required\";\n            return sv;\n        }\n        case 429:\n        {\n            static std::string_view sv = \"Too Many Requests\";\n            return sv;\n        }\n        case 431:\n        {\n            static std::string_view sv = \"Request Header Fields Too Large\";\n            return sv;\n        }\n        case 451:\n        {\n            static std::string_view sv = \"Unavailable For Legal Reasons\";\n            return sv;\n        }\n        case 500:\n        {\n            static std::string_view sv = \"Internal Server Error\";\n            return sv;\n        }\n        case 501:\n        {\n            static std::string_view sv = \"Not Implemented\";\n            return sv;\n        }\n        case 502:\n        {\n            static std::string_view sv = \"Bad Gateway\";\n            return sv;\n        }\n        case 503:\n        {\n            static std::string_view sv = \"Service Unavailable\";\n            return sv;\n        }\n        case 504:\n        {\n            static std::string_view sv = \"Gateway Time-out\";\n            return sv;\n        }\n        case 505:\n        {\n            static std::string_view sv = \"HTTP Version Not Supported\";\n            return sv;\n        }\n        case 506:\n        {\n            static std::string_view sv = \"Variant Also Negotiates\";\n            return sv;\n        }\n        case 507:\n        {\n            static std::string_view sv = \"Insufficient Storage\";\n            return sv;\n        }\n        case 508:\n        {\n            static std::string_view sv = \"Loop Detected\";\n            return sv;\n        }\n        case 510:\n        {\n            static std::string_view sv = \"Not Extended\";\n            return sv;\n        }\n        case 511:\n        {\n            static std::string_view sv = \"Network Authentication Required\";\n            return sv;\n        }\n        default:\n            if (code >= 100 && code < 200)\n            {\n                static std::string_view sv = \"Informational\";\n                return sv;\n            }\n            else if (code >= 200 && code < 300)\n            {\n                static std::string_view sv = \"Successful\";\n                return sv;\n            }\n            else if (code >= 300 && code < 400)\n            {\n                static std::string_view sv = \"Redirection\";\n                return sv;\n            }\n            else if (code >= 400 && code < 500)\n            {\n                static std::string_view sv = \"Bad Request\";\n                return sv;\n            }\n            else if (code >= 500 && code < 600)\n            {\n                static std::string_view sv = \"Server Error\";\n                return sv;\n            }\n            else\n            {\n                static std::string_view sv = \"Undefined Error\";\n                return sv;\n            }\n    }\n}\n\nContentType getContentType(const std::string &fileName)\n{\n    std::string extName;\n    auto pos = fileName.rfind('.');\n    if (pos != std::string::npos)\n    {\n        extName = fileName.substr(pos + 1);\n        transform(extName.begin(),\n                  extName.end(),\n                  extName.begin(),\n                  [](unsigned char c) { return tolower(c); });\n    }\n    auto it = fileTypeDatabase_.find(extName);\n    return (it == fileTypeDatabase_.end()) ? CT_APPLICATION_OCTET_STREAM\n                                           : it->second.second;\n}\n\nContentType parseContentType(const std::string_view &contentType)\n{\n    // Generate map from database for faster query\n    static std::unordered_map<std::string_view, ContentType> contentTypeMap_;\n    // Thread safe initialization\n    static std::once_flag flag;\n    std::call_once(flag, []() {\n        for (const auto &e : mimeTypeDatabase_)\n        {\n            for (const auto &type : e.second.first)\n                contentTypeMap_[type] = e.first;\n        }\n    });\n    auto ext = contentType.find(';');\n    if (ext != std::string_view::npos)\n        return parseContentType(contentType.substr(0, ext));\n    if (contentType == \"application/x-www-form-urlencoded\")\n        return CT_APPLICATION_X_FORM;\n    if (contentType == \"multipart/form-data\")\n        return CT_MULTIPART_FORM_DATA;\n    auto it = contentTypeMap_.find(contentType);\n    return (it == contentTypeMap_.end()) ? CT_CUSTOM : it->second;\n}\n\nFileType parseFileType(const std::string_view &fileExtension)\n{\n    std::string extName(fileExtension);\n    transform(extName.begin(),\n              extName.end(),\n              extName.begin(),\n              [](unsigned char c) { return tolower(c); });\n    auto it = fileTypeDatabase_.find(extName);\n    return (it == fileTypeDatabase_.end()) ? FT_CUSTOM : it->second.first;\n}\n\nFileType getFileType(ContentType contentType)\n{\n    // Generate map from database for faster query\n    static std::unordered_map<ContentType, FileType> fileTypeMap_;\n    // Thread safe initialization\n    static std::once_flag flag;\n    std::call_once(flag, []() {\n        for (const auto &e : fileTypeDatabase_)\n            fileTypeMap_[e.second.second] = e.second.first;\n        fileTypeMap_[CT_NONE] = FT_UNKNOWN;\n        fileTypeMap_[CT_CUSTOM] = FT_CUSTOM;\n    });\n    auto it = fileTypeMap_.find(contentType);\n    return (it == fileTypeMap_.end()) ? FT_UNKNOWN : it->second;\n}\n\nconst std::string_view &contentTypeToMime(ContentType contentType)\n{\n    auto it = mimeTypeDatabase_.find(contentType);\n    return (it == mimeTypeDatabase_.end())\n               ? mimeTypeDatabase_.at(CT_APPLICATION_OCTET_STREAM).first.front()\n               : (it->second.second.empty() ? it->second.first.front()\n                                            : it->second.second);\n}\n\n\nvoid registerCustomExtensionMime(const std::string &ext,\n                                 const std::string &mime)\n{\n    if (ext.empty())\n        return;\n    auto &mimeStr = customMime[ext];\n    if (!mimeStr.empty())\n    {\n        LOG_WARN << ext << \" has already been registered as type \" << mime\n                 << \". Overwriting.\";\n    }\n    mimeStr = mime;\n}\n\nconst std::string_view fileNameToMime(const std::string &fileName)\n{\n    ContentType internalContentType = getContentType(fileName);\n    if (internalContentType != CT_APPLICATION_OCTET_STREAM)\n        return contentTypeToMime(internalContentType);\n\n    std::string extName;\n    auto pos = fileName.rfind('.');\n    if (pos != std::string::npos)\n    {\n        extName = fileName.substr(pos + 1);\n        transform(extName.begin(),\n                  extName.end(),\n                  extName.begin(),\n                  [](unsigned char c) { return tolower(c); });\n    }\n    auto it = customMime.find(extName);\n    if (it == customMime.end())\n        return \"\";\n    return it->second;\n}\n\nstd::pair<ContentType, const std::string_view> fileNameToContentTypeAndMime(\n    const std::string &fileName)\n{\n    ContentType internalContentType = getContentType(fileName);\n    if (internalContentType != CT_APPLICATION_OCTET_STREAM)\n        return {internalContentType, contentTypeToMime(internalContentType)};\n\n    std::string extName;\n    auto pos = fileName.rfind('.');\n    if (pos != std::string::npos)\n    {\n        extName = fileName.substr(pos + 1);\n        transform(extName.begin(),\n                  extName.end(),\n                  extName.begin(),\n                  [](unsigned char c) { return tolower(c); });\n    }\n    auto it = customMime.find(extName);\n    if (it == customMime.end())\n        return {CT_NONE, \"\"};\n    return {CT_CUSTOM, it->second};\n}\n\nconst std::vector<std::string_view> &getFileExtensions(ContentType contentType)\n{\n    // Generate map from database for faster query\n    static std::unordered_map<ContentType, std::vector<std::string_view>>\n        extensionMap_;\n    static std::vector<std::string_view> notFound_;\n    // Thread safe initialization\n    static std::once_flag flag;\n    std::call_once(flag, []() {\n        for (const auto &e : fileTypeDatabase_)\n            if (!e.first.empty())\n                extensionMap_[e.second.second].push_back(e.first);\n        // Add deprecated\n        extensionMap_[CT_APPLICATION_X_JAVASCRIPT] =\n            extensionMap_[CT_TEXT_JAVASCRIPT];\n        extensionMap_[CT_TEXT_XML] = extensionMap_[CT_APPLICATION_XML];\n    });\n    auto it = extensionMap_.find(contentType);\n    if (it == extensionMap_.end())\n        return notFound_;\n    return it->second;\n}\n\n}  // namespace drogon\n\n// Path: lib/src/ConfigAdapterManager.h\n#pragma once\n#include \"ConfigAdapterManager.h\"\n#include \"ConfigAdapter.h\"\n#include <map>\n\nnamespace drogon\n{\nclass ConfigAdapterManager\n{\n  public:\n    static ConfigAdapterManager &instance();\n    Json::Value getJson(const std::string &content, std::string ext) const\n        noexcept(false);\n\n  private:\n    ConfigAdapterManager();\n    std::map<std::string, ConfigAdapterPtr> adapters_;\n};\n}  // namespace drogon\n\n// Path: lib/src/ConfigAdapterManager.cc\n#include \"ConfigAdapterManager.h\"\n#include \"JsonConfigAdapter.h\"\n#include \"YamlConfigAdapter.h\"\n#include <algorithm>\n\nusing namespace drogon;\n#define REGISTER_CONFIG_ADAPTER(adapter)                                    \\\n    {                                                                       \\\n        auto adapterPtr = std::make_shared<adapter>();                      \\\n        auto exts = adapterPtr->getExtensions();                            \\\n        for (auto ext : exts)                                               \\\n        {                                                                   \\\n            std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); \\\n            adapters_[ext] = adapterPtr;                                    \\\n        }                                                                   \\\n    }\n\nConfigAdapterManager &ConfigAdapterManager::instance()\n{\n    static ConfigAdapterManager instance;\n    return instance;\n}\n\nJson::Value ConfigAdapterManager::getJson(const std::string &content,\n                                          std::string ext) const noexcept(false)\n{\n    std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);\n    auto it = adapters_.find(ext);\n    if (it == adapters_.end())\n    {\n        throw std::runtime_error(\"No valid parser for this config file!\");\n    }\n    return it->second->getJson(content);\n}\n\nConfigAdapterManager::ConfigAdapterManager()\n{\n    REGISTER_CONFIG_ADAPTER(JsonConfigAdapter);\n    REGISTER_CONFIG_ADAPTER(YamlConfigAdapter);\n}\n\n// Path: lib/src/DrTemplateBase.cc\n/**\n *\n *  @file DrTemplateBase.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/DrClassMap.h>\n#include <drogon/DrTemplateBase.h>\n#include <trantor/utils/Logger.h>\n#include <memory>\n#include <regex>\n\nusing namespace drogon;\n\nstd::shared_ptr<DrTemplateBase> DrTemplateBase::newTemplate(\n    const std::string &templateName)\n{\n    LOG_TRACE << \"http view name=\" << templateName;\n    auto l = templateName.length();\n    if (l >= 4 && templateName[l - 4] == '.' && templateName[l - 3] == 'c' &&\n        templateName[l - 2] == 's' && templateName[l - 1] == 'p')\n    {\n        std::string::size_type pos = 0;\n        std::string newName;\n        newName.reserve(templateName.size());\n        if (templateName[0] == '/' || templateName[0] == '\\\\')\n        {\n            pos = 1;\n        }\n        else if (templateName[0] == '.' &&\n                 (templateName[1] == '/' || templateName[1] == '\\\\'))\n        {\n            pos = 2;\n        }\n        while (pos < l - 4)\n        {\n            if (templateName[pos] == '/' || templateName[pos] == '\\\\')\n            {\n                newName.append(\"::\");\n            }\n            else\n            {\n                newName.append(1, templateName[pos]);\n            }\n            ++pos;\n        }\n        return std::shared_ptr<DrTemplateBase>(dynamic_cast<DrTemplateBase *>(\n            drogon::DrClassMap::newObject(newName)));\n    }\n    else\n    {\n        return std::shared_ptr<DrTemplateBase>(dynamic_cast<DrTemplateBase *>(\n            drogon::DrClassMap::newObject(templateName)));\n    }\n}\n\n// Path: lib/src/DrClassMap.cc\n/**\n *\n *  DrClassMap.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/DrClassMap.h>\n#include <drogon/DrObject.h>\n#include <trantor/utils/Logger.h>\n\nusing namespace drogon;\n\nnamespace drogon\n{\nnamespace internal\n{\nstatic std::unordered_map<std::string, std::shared_ptr<DrObjectBase>> &\ngetObjsMap()\n{\n    static std::unordered_map<std::string, std::shared_ptr<DrObjectBase>>\n        singleInstanceMap;\n    return singleInstanceMap;\n}\n\nstatic std::mutex &getMapMutex()\n{\n    static std::mutex mtx;\n    return mtx;\n}\n\n}  // namespace internal\n}  // namespace drogon\n\nvoid DrClassMap::registerClass(const std::string &className,\n                               const DrAllocFunc &func,\n                               const DrSharedAllocFunc &sharedFunc)\n{\n    LOG_TRACE << \"Register class:\" << className;\n    getMap().insert(\n        std::make_pair(className, std::make_pair(func, sharedFunc)));\n}\n\nDrObjectBase *DrClassMap::newObject(const std::string &className)\n{\n    auto iter = getMap().find(className);\n    if (iter != getMap().end())\n    {\n        return iter->second.first();\n    }\n    else\n        return nullptr;\n}\n\nstd::shared_ptr<DrObjectBase> DrClassMap::newSharedObject(\n    const std::string &className)\n{\n    auto iter = getMap().find(className);\n    if (iter != getMap().end())\n    {\n        if (iter->second.second)\n            return iter->second.second();\n        else\n            return std::shared_ptr<DrObjectBase>(iter->second.first());\n    }\n    else\n        return nullptr;\n}\n\nconst std::shared_ptr<DrObjectBase> &DrClassMap::getSingleInstance(\n    const std::string &className)\n{\n    auto &mtx = internal::getMapMutex();\n    auto &singleInstanceMap = internal::getObjsMap();\n    {\n        std::lock_guard<std::mutex> lock(mtx);\n        auto iter = singleInstanceMap.find(className);\n        if (iter != singleInstanceMap.end())\n            return iter->second;\n    }\n    auto newObj = newSharedObject(className);\n    {\n        std::lock_guard<std::mutex> lock(mtx);\n        auto ret = singleInstanceMap.insert(\n            std::make_pair(className, std::move(newObj)));\n        return ret.first->second;\n    }\n}\n\nvoid DrClassMap::setSingleInstance(const std::shared_ptr<DrObjectBase> &ins)\n{\n    auto &mtx = internal::getMapMutex();\n    auto &singleInstanceMap = internal::getObjsMap();\n    std::lock_guard<std::mutex> lock(mtx);\n    singleInstanceMap[ins->className()] = ins;\n}\n\nstd::vector<std::string> DrClassMap::getAllClassName()\n{\n    std::vector<std::string> ret;\n    for (auto const &iter : getMap())\n    {\n        ret.push_back(iter.first);\n    }\n    return ret;\n}\n\nstd::unordered_map<std::string, std::pair<DrAllocFunc, DrSharedAllocFunc>> &\nDrClassMap::getMap()\n{\n    static std::unordered_map<std::string,\n                              std::pair<DrAllocFunc, DrSharedAllocFunc>>\n        map;\n    return map;\n}\n\n// Path: lib/src/HttpFileImpl.h\n/**\n *\n *  @file HttpFileImpl.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n#include \"HttpUtils.h\"\n#include <drogon/HttpRequest.h>\n\n#include <map>\n#include <string>\n#include <vector>\n#include <memory>\n#include <filesystem>\n#include <string_view>\n\nnamespace drogon\n{\nclass HttpFileImpl\n{\n  public:\n    /// Return the file name;\n    const std::string &getFileName() const noexcept\n    {\n        return fileName_;\n    }\n\n    /// Set the file name, usually called by the MultiPartParser parser.\n    void setFileName(const std::string &fileName) noexcept\n    {\n        fileName_ = fileName;\n    }\n\n    void setFileName(std::string &&fileName) noexcept\n    {\n        fileName_ = std::move(fileName);\n    }\n\n    /// Return the file extension;\n    std::string_view getFileExtension() const noexcept\n    {\n        return drogon::getFileExtension(fileName_);\n    }\n\n    /// Set the contents of the file, usually called by the MultiPartParser\n    /// parser.\n    void setFile(const char *data, size_t length) noexcept\n    {\n        fileContent_ = std::string_view{data, length};\n    }\n\n    /// Save the file to the file system.\n    /**\n     * The folder saving the file is app().getUploadPath().\n     * The full path is app().getUploadPath()+\"/\"+this->getFileName()\n     */\n    int save() const noexcept;\n\n    /// Save the file to @param path\n    /**\n     * @param path if the parameter is prefixed with \"/\", \"./\" or \"../\", or is\n     * \".\" or \"..\", the full path is path+\"/\"+this->getFileName(),\n     * otherwise the file is saved as\n     * app().getUploadPath()+\"/\"+path+\"/\"+this->getFileName()\n     */\n    int save(const std::string &path) const noexcept;\n\n    /// Save the file to file system with a new name\n    /**\n     * @param fileName if the parameter isn't prefixed with \"/\", \"./\" or \"../\",\n     * the full path is app().getUploadPath()+\"/\"+filename, otherwise the file\n     * is saved as the filename\n     */\n    int saveAs(const std::string &fileName) const noexcept;\n\n    /// Return the file length.\n    size_t fileLength() const noexcept\n    {\n        return fileContent_.length();\n    }\n\n    const char *fileData() const noexcept\n    {\n        return fileContent_.data();\n    }\n\n    const std::string_view &fileContent() const noexcept\n    {\n        return fileContent_;\n    }\n\n    /// Return the name of the item in multiple parts.\n    const std::string &getItemName() const noexcept\n    {\n        return itemName_;\n    }\n\n    void setItemName(const std::string &itemName) noexcept\n    {\n        itemName_ = itemName;\n    }\n\n    void setItemName(std::string &&itemName) noexcept\n    {\n        itemName_ = std::move(itemName);\n    }\n\n    /// Return the type of file.\n    FileType getFileType() const noexcept\n    {\n        auto ft = drogon::getFileType(contentType_);\n        if ((ft != FT_UNKNOWN) && (ft != FT_CUSTOM))\n            return ft;\n        return parseFileType(getFileExtension());\n    }\n\n    /// Return md5 hash of the file\n    std::string getMd5() const noexcept;\n    // Return sha1 hash of the file\n    std::string getSha256() const noexcept;\n    // Return sha512 hash of the file\n    std::string getSha3() const noexcept;\n    //    int saveTo(const std::string &pathAndFileName) const;\n    int saveTo(const std::filesystem::path &pathAndFileName) const noexcept;\n\n    void setRequest(const HttpRequestPtr &req) noexcept\n    {\n        requestPtr_ = req;\n    }\n\n    drogon::ContentType getContentType() const noexcept\n    {\n        return contentType_;\n    }\n\n    void setContentType(drogon::ContentType contentType) noexcept\n    {\n        contentType_ = contentType;\n    }\n\n    void setContentTransferEncoding(\n        const std::string &contentTransferEncoding) noexcept\n    {\n        transferEncoding_ = contentTransferEncoding;\n    }\n\n    void setContentTransferEncoding(\n        std::string &&contentTransferEncoding) noexcept\n    {\n        transferEncoding_ = std::move(contentTransferEncoding);\n    }\n\n    const std::string &getContentTransferEncoding() const noexcept\n    {\n        return transferEncoding_;\n    }\n\n  private:\n    std::string fileName_;\n    std::string itemName_;\n    std::string transferEncoding_;\n    std::string_view fileContent_;\n    HttpRequestPtr requestPtr_;\n    drogon::ContentType contentType_{drogon::CT_NONE};\n};\n}  // namespace drogon\n\n// Path: lib/src/HttpFileImpl.cc\n/**\n *\n *  @file HttpFileImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpFileImpl.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/MultiPart.h>\n#include <drogon/utils/Utilities.h>\n#include <fstream>\n#include <iostream>\n#include <algorithm>\n#include <filesystem>\n\nusing namespace drogon;\n\nint HttpFileImpl::save() const noexcept\n{\n    return save(HttpAppFrameworkImpl::instance().getUploadPath());\n}\n\nint HttpFileImpl::save(const std::string &path) const noexcept\n{\n    assert(!path.empty());\n    if (fileName_.empty())\n        return -1;\n    std::filesystem::path fsUploadDir(utils::toNativePath(path));\n\n    if (fsUploadDir.is_absolute())\n    {  // do nothing\n    }\n    else if ((!fsUploadDir.has_parent_path() ||\n              (fsUploadDir.begin()->string() != \".\" &&\n               fsUploadDir.begin()->string() != \"..\")))\n    {\n        fsUploadDir = utils::toNativePath(\n                          HttpAppFrameworkImpl::instance().getUploadPath()) /\n                      fsUploadDir;\n    }\n    else\n    {\n        fsUploadDir = std::filesystem::current_path() / fsUploadDir;\n    }\n\n    fsUploadDir = std::filesystem::weakly_canonical(fsUploadDir);\n\n    if (!std::filesystem::exists(fsUploadDir))\n    {\n        LOG_TRACE << \"create path:\" << fsUploadDir;\n        std::error_code err;\n        std::filesystem::create_directories(fsUploadDir, err);\n        if (err)\n        {\n            LOG_SYSERR;\n            return -1;\n        }\n    }\n\n    std::filesystem::path fsSaveToPath(std::filesystem::weakly_canonical(\n        fsUploadDir / utils::toNativePath(fileName_)));\n    LOG_TRACE << \"save to path:\" << fsSaveToPath;\n    if (!std::equal(fsUploadDir.begin(),\n                    fsUploadDir.end(),\n                    fsSaveToPath.begin()))\n    {\n        LOG_ERROR\n            << \"Attempt writing outside of upload directory detected. Path: \"\n            << fileName_;\n        return -1;\n    }\n\n    return saveTo(fsSaveToPath);\n}\n\nint HttpFileImpl::saveAs(const std::string &fileName) const noexcept\n{\n    assert(!fileName.empty());\n    std::filesystem::path fsFileName(utils::toNativePath(fileName));\n    if (!fsFileName.is_absolute() && (!fsFileName.has_parent_path() ||\n                                      (fsFileName.begin()->string() != \".\" &&\n                                       fsFileName.begin()->string() != \"..\")))\n    {\n        std::filesystem::path fsUploadPath(utils::toNativePath(\n            HttpAppFrameworkImpl::instance().getUploadPath()));\n        fsFileName = fsUploadPath / fsFileName;\n    }\n    if (fsFileName.has_parent_path() &&\n        !std::filesystem::exists(fsFileName.parent_path()))\n    {\n        LOG_TRACE << \"create path:\" << fsFileName.parent_path();\n        std::error_code err;\n        std::filesystem::create_directories(fsFileName.parent_path(), err);\n        if (err)\n        {\n            LOG_SYSERR;\n            return -1;\n        }\n    }\n    return saveTo(fsFileName);\n}\n\nint HttpFileImpl::saveTo(\n    const std::filesystem::path &pathAndFileName) const noexcept\n{\n    LOG_TRACE << \"save uploaded file:\" << pathAndFileName;\n    auto wPath = utils::toNativePath(pathAndFileName.native());\n    std::ofstream file(wPath, std::ios::binary);\n    if (file.is_open())\n    {\n        file.write(fileContent_.data(), fileContent_.size());\n        file.close();\n        return 0;\n    }\n    else\n    {\n        LOG_ERROR << \"save failed!\";\n        return -1;\n    }\n}\n\nstd::string HttpFileImpl::getMd5() const noexcept\n{\n    return utils::getMd5(fileContent_.data(), fileContent_.size());\n}\n\nstd::string HttpFileImpl::getSha256() const noexcept\n{\n    return utils::getSha256(fileContent_.data(), fileContent_.size());\n}\n\nstd::string HttpFileImpl::getSha3() const noexcept\n{\n    return utils::getSha3(fileContent_.data(), fileContent_.size());\n}\n\nconst std::string &HttpFile::getFileName() const noexcept\n{\n    return implPtr_->getFileName();\n}\n\nvoid HttpFile::setFileName(const std::string &fileName) noexcept\n{\n    implPtr_->setFileName(fileName);\n}\n\nstd::string_view HttpFile::getFileExtension() const noexcept\n{\n    return implPtr_->getFileExtension();\n}\n\nFileType HttpFile::getFileType() const noexcept\n{\n    return implPtr_->getFileType();\n}\n\nvoid HttpFile::setFile(const char *data, size_t length) noexcept\n{\n    implPtr_->setFile(data, length);\n}\n\nint HttpFile::save() const noexcept\n{\n    return implPtr_->save();\n}\n\nint HttpFile::save(const std::string &path) const noexcept\n{\n    return implPtr_->save(path);\n}\n\nint HttpFile::saveAs(const std::string &fileName) const noexcept\n{\n    return implPtr_->saveAs(fileName);\n}\n\nsize_t HttpFile::fileLength() const noexcept\n{\n    return implPtr_->fileLength();\n}\n\ndrogon::ContentType HttpFile::getContentType() const noexcept\n{\n    return implPtr_->getContentType();\n}\n\nconst char *HttpFile::fileData() const noexcept\n{\n    return implPtr_->fileData();\n}\n\nstd::string HttpFile::getMd5() const noexcept\n{\n    return implPtr_->getMd5();\n}\n\nconst std::string &HttpFile::getContentTransferEncoding() const noexcept\n{\n    return implPtr_->getContentTransferEncoding();\n}\n\nHttpFile::HttpFile(std::shared_ptr<HttpFileImpl> &&implPtr) noexcept\n    : implPtr_(std::move(implPtr))\n{\n}\n\nconst std::string &HttpFile::getItemName() const noexcept\n{\n    return implPtr_->getItemName();\n}\n\n// Path: lib/src/ListenerManager.cc\n/**\n *\n *  @file ListenerManager.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"ListenerManager.h\"\n#include <drogon/config.h>\n#include <fcntl.h>\n#include <trantor/utils/Logger.h>\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpServer.h\"\n#ifndef _WIN32\n#include <sys/file.h>\n#include <unistd.h>\n#endif\n\nnamespace drogon\n{\n#ifndef _WIN32\nclass DrogonFileLocker : public trantor::NonCopyable\n{\n  public:\n    DrogonFileLocker()\n    {\n        fd_ = open(\"/tmp/drogon.lock\", O_TRUNC | O_CREAT, 0666);\n        flock(fd_, LOCK_EX);\n    }\n\n    ~DrogonFileLocker()\n    {\n        close(fd_);\n    }\n\n  private:\n    int fd_{0};\n};\n\n#endif\n}  // namespace drogon\n\nusing namespace trantor;\nusing namespace drogon;\n\nvoid ListenerManager::addListener(\n    const std::string &ip,\n    uint16_t port,\n    bool useSSL,\n    const std::string &certFile,\n    const std::string &keyFile,\n    bool useOldTLS,\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n{\n    if (useSSL && !utils::supportsTls())\n        LOG_ERROR << \"Can't use SSL without OpenSSL found in your system\";\n    listeners_.emplace_back(\n        ip, port, useSSL, certFile, keyFile, useOldTLS, sslConfCmds);\n}\n\nstd::vector<trantor::InetAddress> ListenerManager::getListeners() const\n{\n    std::vector<trantor::InetAddress> listeners;\n    for (auto &server : servers_)\n    {\n        listeners.emplace_back(server->address());\n    }\n    return listeners;\n}\n\nvoid ListenerManager::createListeners(\n    const std::string &globalCertFile,\n    const std::string &globalKeyFile,\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds,\n    const std::vector<trantor::EventLoop *> &ioLoops)\n{\n    LOG_TRACE << \"thread num=\" << ioLoops.size();\n#ifdef __linux__\n    for (size_t i = 0; i < ioLoops.size(); ++i)\n    {\n        for (auto const &listener : listeners_)\n        {\n            auto const &ip = listener.ip_;\n            bool isIpv6 = (ip.find(':') != std::string::npos);\n            InetAddress listenAddress(ip, listener.port_, isIpv6);\n            if (listenAddress.isUnspecified())\n            {\n                LOG_FATAL << \"Failed to parse IP address '\" << ip\n                          << \"'. (Note: FQDN/domain names/hostnames are not \"\n                             \"supported. Including 'localhost')\";\n                abort();\n            }\n            if (i == 0 && !app().reusePort())\n            {\n                DrogonFileLocker lock;\n                // Check whether the port is in use.\n                TcpServer server(HttpAppFrameworkImpl::instance().getLoop(),\n                                 listenAddress,\n                                 \"drogonPortTest\",\n                                 true,\n                                 false);\n            }\n            std::shared_ptr<HttpServer> serverPtr =\n                std::make_shared<HttpServer>(ioLoops[i],\n                                             listenAddress,\n                                             \"drogon\");\n\n            if (listener.useSSL_ && utils::supportsTls())\n            {\n                auto cert = listener.certFile_;\n                auto key = listener.keyFile_;\n                if (cert.empty())\n                    cert = globalCertFile;\n                if (key.empty())\n                    key = globalKeyFile;\n                if (cert.empty() || key.empty())\n                {\n                    std::cerr\n                        << \"You can't use https without cert file or key file\"\n                        << std::endl;\n                    exit(1);\n                }\n                auto cmds = sslConfCmds;\n                std::copy(listener.sslConfCmds_.begin(),\n                          listener.sslConfCmds_.end(),\n                          std::back_inserter(cmds));\n                auto policy =\n                    trantor::TLSPolicy::defaultServerPolicy(cert, key);\n                policy->setConfCmds(cmds).setUseOldTLS(listener.useOldTLS_);\n                serverPtr->enableSSL(std::move(policy));\n            }\n            servers_.push_back(serverPtr);\n        }\n    }\n#else\n\n    if (!listeners_.empty())\n    {\n        listeningThread_ =\n            std::make_unique<EventLoopThread>(\"DrogonListeningLoop\");\n        listeningThread_->run();\n        for (auto const &listener : listeners_)\n        {\n            auto ip = listener.ip_;\n            bool isIpv6 = (ip.find(':') != std::string::npos);\n            auto serverPtr = std::make_shared<HttpServer>(\n                listeningThread_->getLoop(),\n                InetAddress(ip, listener.port_, isIpv6),\n                \"drogon\");\n            if (listener.useSSL_ && utils::supportsTls())\n            {\n                auto cert = listener.certFile_;\n                auto key = listener.keyFile_;\n                if (cert.empty())\n                    cert = globalCertFile;\n                if (key.empty())\n                    key = globalKeyFile;\n                if (cert.empty() || key.empty())\n                {\n                    std::cerr\n                        << \"You can't use https without cert file or key file\"\n                        << std::endl;\n                    exit(1);\n                }\n                auto cmds = sslConfCmds;\n                auto policy =\n                    trantor::TLSPolicy::defaultServerPolicy(cert, key);\n                policy->setConfCmds(cmds).setUseOldTLS(listener.useOldTLS_);\n                serverPtr->enableSSL(std::move(policy));\n            }\n            serverPtr->setIoLoops(ioLoops);\n            servers_.push_back(serverPtr);\n        }\n    }\n#endif\n}\n\nvoid ListenerManager::startListening()\n{\n    for (auto &server : servers_)\n    {\n        server->start();\n    }\n}\n\nvoid ListenerManager::stopListening()\n{\n    for (auto &serverPtr : servers_)\n    {\n        serverPtr->stop();\n    }\n    if (listeningThread_)\n    {\n        auto loop = listeningThread_->getLoop();\n        assert(!loop->isInLoopThread());\n        loop->quit();\n        listeningThread_->wait();\n    }\n}\n\n// Path: lib/src/RealIpResolver.cc\n/**\n *\n *  @file RealIpResolver.cc\n *  @author Nitromelon\n *\n *  Copyright 2022, Nitromelon. All rights reserved.\n *  https://github.com/drogonframework/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/drogon.h>\n#include <trantor/utils/Logger.h>\n#include <drogon/plugins/RealIpResolver.h>\n\nusing namespace drogon;\nusing namespace drogon::plugin;\n\nstruct XForwardedForParser : public trantor::NonCopyable\n{\n    explicit XForwardedForParser(std::string value)\n        : value_(std::move(value)), start_(value_.c_str()), len_(value_.size())\n    {\n    }\n\n    std::string getNext()\n    {\n        if (len_ == 0)\n        {\n            return {};\n        }\n        // Skip trailing separators\n        const char *cur;\n        for (cur = start_ + len_ - 1; cur > start_; --cur, --len_)\n        {\n            if (*cur != ' ' && *cur != ',')\n            {\n                break;\n            }\n        }\n        for (; cur > start_; --cur)\n        {\n            if (*cur == ' ' || *cur == ',')\n            {\n                ++cur;\n                break;\n            }\n        }\n        std::string ip{cur, len_ - (cur - start_)};\n        len_ = cur == start_ ? 0 : cur - start_ - 1;\n        return ip;\n    }\n\n  private:\n    std::string value_;\n    const char *start_;\n    size_t len_;\n};\n\nstatic trantor::InetAddress parseAddress(const std::string &addr)\n{\n    auto pos = addr.find(':');\n    uint16_t port = 0;\n    if (pos == std::string::npos)\n    {\n        return trantor::InetAddress(addr, 0);\n    }\n    try\n    {\n        port = std::stoi(addr.substr(pos + 1));\n    }\n    catch (const std::exception &ex)\n    {\n        (void)ex;\n        LOG_ERROR << \"Error in ipv4 address: \" + addr;\n        port = 0;\n    }\n    return trantor::InetAddress(addr.substr(0, pos), port);\n}\n\nvoid RealIpResolver::initAndStart(const Json::Value &config)\n{\n    fromHeader_ = config.get(\"from_header\", \"x-forwarded-for\").asString();\n    attributeKey_ = config.get(\"attribute_key\", \"real-ip\").asString();\n\n    std::transform(fromHeader_.begin(),\n                   fromHeader_.end(),\n                   fromHeader_.begin(),\n                   [](unsigned char c) { return tolower(c); });\n    if (fromHeader_ == \"x-forwarded-for\")\n    {\n        useXForwardedFor_ = true;\n    }\n\n    const Json::Value &trustIps = config[\"trust_ips\"];\n    if (!trustIps.isArray())\n    {\n        throw std::runtime_error(\"Invalid trusted_ips. Should be array.\");\n    }\n    for (const auto &elem : trustIps)\n    {\n        std::string ipOrCidr = elem.asString();\n        trustCIDRs_.emplace_back(ipOrCidr);\n    }\n\n    drogon::app().registerPreRoutingAdvice([this](const HttpRequestPtr &req) {\n        const std::string &ipHeader = req->getHeader(fromHeader_);\n        const trantor::InetAddress &peerAddr = req->getPeerAddr();\n        if (ipHeader.empty() || !matchCidr(peerAddr))\n        {\n            // Target header is empty, or\n            // direct peer is already a non-proxy\n            req->attributes()->insert(attributeKey_, peerAddr);\n            return;\n        }\n        // Use a header field which contains a single ip\n        if (!useXForwardedFor_)\n        {\n            trantor::InetAddress addr = parseAddress(ipHeader);\n            if (addr.isUnspecified())\n            {\n                req->attributes()->insert(attributeKey_, peerAddr);\n            }\n            else\n            {\n                req->attributes()->insert(attributeKey_, addr);\n            }\n            return;\n        }\n        // Use x-forwarded-for header, which may contains multiple ip address,\n        // separated by comma\n        XForwardedForParser parser(ipHeader);\n        std::string ip;\n        while (!(ip = parser.getNext()).empty())\n        {\n            trantor::InetAddress addr = parseAddress(ip);\n            if (addr.isUnspecified() || matchCidr(addr))\n            {\n                continue;\n            }\n            req->attributes()->insert(attributeKey_, addr);\n            return;\n        }\n        // No match, use peerAddr\n        req->attributes()->insert(attributeKey_, peerAddr);\n    });\n}\n\nvoid RealIpResolver::shutdown()\n{\n}\n\nconst trantor::InetAddress &RealIpResolver::GetRealAddr(\n    const HttpRequestPtr &req)\n{\n    auto *plugin = app().getPlugin<drogon::plugin::RealIpResolver>();\n    if (!plugin)\n    {\n        return req->getPeerAddr();\n    }\n    return plugin->getRealAddr(req);\n}\n\nconst trantor::InetAddress &RealIpResolver::getRealAddr(\n    const HttpRequestPtr &req) const\n{\n    const std::shared_ptr<Attributes> &attributesPtr = req->getAttributes();\n    if (!attributesPtr->find(attributeKey_))\n    {\n        return req->getPeerAddr();\n    }\n    return attributesPtr->get<trantor::InetAddress>(attributeKey_);\n}\n\nbool RealIpResolver::matchCidr(const trantor::InetAddress &addr) const\n{\n    for (auto &cidr : trustCIDRs_)\n    {\n        if ((addr.ipNetEndian() & cidr.mask_) == cidr.addr_)\n        {\n            return true;\n        }\n    }\n    return false;\n}\n\nRealIpResolver::CIDR::CIDR(const std::string &ipOrCidr)\n{\n    // Find CIDR slash\n    auto pos = ipOrCidr.find('/');\n    std::string ipv4;\n    if (pos != std::string::npos)\n    {\n        // parameter is a CIDR block\n        std::string prefixLen = ipOrCidr.substr(pos + 1);\n        ipv4 = ipOrCidr.substr(0, pos);\n        uint16_t prefix = std::stoi(prefixLen);\n        if (prefix > 32)\n        {\n            throw std::runtime_error(\"Bad CIDR block: \" + ipOrCidr);\n        }\n        mask_ = htonl(0xffffffffu << (32 - prefix));\n    }\n    else\n    {\n        // parameter is an IP\n        ipv4 = ipOrCidr;\n        mask_ = 0xffffffffu;\n    }\n\n    trantor::InetAddress addr(ipv4, 0);\n    if (addr.isIpV6())\n    {\n        throw std::runtime_error(\"Ipv6 is not supported by RealIpResolver.\");\n    }\n    if (addr.isUnspecified())\n    {\n        throw std::runtime_error(\"Bad ipv4 address: \" + ipv4);\n    }\n    addr_ = addr.ipNetEndian() & mask_;\n}\n\n// Path: lib/src/FiltersFunction.cc\n/**\n *\n *  @file FiltersFunction.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"FiltersFunction.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpResponseImpl.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/HttpFilter.h>\n\n#include <queue>\n\nnamespace drogon\n{\nnamespace filters_function\n{\nstatic void doFilterChains(\n    const std::vector<std::shared_ptr<HttpFilterBase>> &filters,\n    size_t index,\n    const HttpRequestImplPtr &req,\n    std::shared_ptr<const std::function<void(const HttpResponsePtr &)>>\n        &&callbackPtr)\n{\n    if (index < filters.size())\n    {\n        auto &filter = filters[index];\n        filter->doFilter(\n            req,\n            [/*copy*/ callbackPtr](const HttpResponsePtr &resp) {\n                (*callbackPtr)(resp);\n            },\n            [index, req, callbackPtr, &filters]() mutable {\n                auto ioLoop = req->getLoop();\n                if (ioLoop && !ioLoop->isInLoopThread())\n                {\n                    ioLoop->queueInLoop(\n                        [&filters,\n                         index,\n                         req,\n                         callbackPtr = std::move(callbackPtr)]() mutable {\n                            doFilterChains(filters,\n                                           index + 1,\n                                           req,\n                                           std::move(callbackPtr));\n                        });\n                }\n                else\n                {\n                    doFilterChains(filters,\n                                   index + 1,\n                                   req,\n                                   std::move(callbackPtr));\n                }\n            });\n    }\n    else\n    {\n        (*callbackPtr)(nullptr);\n    }\n}\n\nstd::vector<std::shared_ptr<HttpFilterBase>> createFilters(\n    const std::vector<std::string> &filterNames)\n{\n    std::vector<std::shared_ptr<HttpFilterBase>> filters;\n    for (auto const &filter : filterNames)\n    {\n        auto object_ = DrClassMap::getSingleInstance(filter);\n        auto filter_ = std::dynamic_pointer_cast<HttpFilterBase>(object_);\n        if (filter_)\n            filters.push_back(filter_);\n        else\n        {\n            LOG_ERROR << \"filter \" << filter << \" not found\";\n        }\n    }\n    return filters;\n}\n\nvoid doFilters(const std::vector<std::shared_ptr<HttpFilterBase>> &filters,\n               const HttpRequestImplPtr &req,\n               std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    auto callbackPtr =\n        std::make_shared<std::decay_t<decltype(callback)>>(std::move(callback));\n    doFilterChains(filters, 0, req, std::move(callbackPtr));\n}\n\n}  // namespace filters_function\n}  // namespace drogon\n\n// Path: lib/src/RangeParser.cc\n/**\n *\n *  RangeParser.h\n *  He, Wanchen\n *\n *  Copyright 2021, He,Wanchen.  All rights reserved.\n *  https://github.com/drogonframework/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"RangeParser.h\"\n\n#include <limits>\n\nusing namespace drogon;\n\nstatic constexpr size_t MAX_SIZE = std::numeric_limits<size_t>::max();\nstatic constexpr size_t MAX_TEN = MAX_SIZE / 10;\nstatic constexpr size_t MAX_DIGIT = MAX_SIZE % 10;\n\n// clang-format off\n#define DR_SKIP_WHITESPACE(p) while (*p == ' ') { ++(p); }\n#define DR_ISDIGIT(p) ('0' <= *(p) && *(p) <= '9')\n#define DR_WOULD_OVERFLOW(base, digit) \\\n    (static_cast<size_t>(base) > MAX_TEN || \\\n    (static_cast<size_t>(base) >= MAX_TEN && \\\n    static_cast<size_t>(digit) - '0' > MAX_DIGIT))\n\n// clang-format on\n\n/** Following formats are valid range header according to rfc7233`\n * Range: <unit>=<start>-\n * Range: <unit>=<start>-<end>\n * Range: <unit>=<start>-<end>, <start>-<end>\n * Range: <unit>=<start>-<end>, <start>-<end>, <start>-<end>\n * Range: <unit>=-<suffix-length>\n */\n\nFileRangeParseResult drogon::parseRangeHeader(const std::string &rangeStr,\n                                              size_t contentLength,\n                                              std::vector<FileRange> &ranges)\n{\n    if (rangeStr.size() < 7 || rangeStr.compare(0, 6, \"bytes=\") != 0)\n    {\n        return InvalidRange;\n    }\n    const char *iter = rangeStr.c_str() + 6;\n\n    size_t totalSize = 0;\n    while (true)\n    {\n        size_t start = 0;\n        size_t end = 0;\n        // If this is a suffix range: <unit>=-<suffix-length>\n        bool isSuffix = false;\n\n        DR_SKIP_WHITESPACE(iter);\n\n        if (*iter == '-')\n        {\n            isSuffix = true;\n            ++iter;\n        }\n        // Parse start\n        else\n        {\n            if (!DR_ISDIGIT(iter))\n            {\n                return InvalidRange;\n            }\n            while (DR_ISDIGIT(iter))\n            {\n                // integer out of range\n                if (DR_WOULD_OVERFLOW(start, *iter))\n                {\n                    return NotSatisfiable;\n                }\n                start = start * 10 + (*iter++ - '0');\n            }\n            DR_SKIP_WHITESPACE(iter);\n            // should be separator now\n            if (*iter++ != '-')\n            {\n                return InvalidRange;\n            }\n            DR_SKIP_WHITESPACE(iter);\n            // If this is a prefix range <unit>=<range-start>-\n            if (*iter == ',' || *iter == '\\0')\n            {\n                end = contentLength;\n                // Handle found\n                if (start < end)\n                {\n                    if (totalSize > MAX_SIZE - (end - start))\n                    {\n                        return NotSatisfiable;\n                    }\n                    totalSize += end - start;\n                    ranges.push_back({start, end});\n                }\n                if (*iter++ != ',')\n                {\n                    break;\n                }\n                continue;\n            }\n        }\n\n        // Parse end\n        if (!DR_ISDIGIT(iter))\n        {\n            return InvalidRange;\n        }\n        while (DR_ISDIGIT(iter))\n        {\n            if (DR_WOULD_OVERFLOW(end, *iter))\n            {\n                return NotSatisfiable;\n            }\n            end = end * 10 + (*iter++ - '0');\n        }\n        DR_SKIP_WHITESPACE(iter);\n\n        if (*iter != ',' && *iter != '\\0')\n        {\n            return InvalidRange;\n        }\n        if (isSuffix)\n        {\n            start = (end < contentLength) ? contentLength - end : 0;\n            end = contentLength - 1;\n        }\n        // [start, end)\n        if (end >= contentLength)\n        {\n            end = contentLength;\n        }\n        else\n        {\n            ++end;\n        }\n\n        // handle found\n        if (start < end)\n        {\n            ranges.push_back({start, end});\n            if (totalSize > MAX_SIZE - (end - start))\n            {\n                return NotSatisfiable;\n            }\n            totalSize += end - start;\n            // We restrict the number to be under 100, to avoid malicious\n            // requests.\n            // Though rfc does not say anything about max number of ranges,\n            // it does mention that server can ignore range header freely.\n            if (ranges.size() > 100)\n            {\n                return InvalidRange;\n            }\n        }\n        if (*iter++ != ',')\n        {\n            break;\n        }\n    }\n\n    if (ranges.size() == 0 || totalSize > contentLength)\n    {\n        return NotSatisfiable;\n    }\n...\n", "needle_token_start": 4019, "needle_token_end": 4127, "code_context_ntokens": 16101, "output": ["```cpp\nvoid registerCustomExtensionMime(const std::string &ext,\n                                 const std::string &mime)\n{\n    if (ext.empty())\n        return;\n    auto &mimeStr = customMime[ext];\n    if (!mimeStr.empty())\n    {\n        LOG_WARN << ext << \" has already been registered as type \" << mime\n                 << \". Overwriting.\";\n    }\n    mimeStr = mime;\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "genHttpResponse", "language": "cpp", "path": "lib/src/HttpResponseImpl.cc", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To generate an HTTP response based on a specified template and data, or return a not-found response if the template does not exist.\n2. **Input**: A string representing the name of the template, a data structure containing the data to be populated in the template, and a pointer to the HTTP request.\n3. **Output**: A pointer to an HTTP response object, either containing the rendered template or a not-found status.\n4. **Procedure**: \n   - The function attempts to create a new template object using the provided template name.\n   - If the template object is successfully created, it generates the text for the HTTP response body using the provided data.\n   - A new HTTP response object is then created, and the generated text is set as the body of this response.\n   - If the template object cannot be created (i.e., the template does not exist), the function returns a not-found HTTP response, indicating that the requested template could not be found.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/HttpClientImpl.cc\n/**\n *\n *  @file HttpClientImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpClientImpl.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpResponseImpl.h\"\n#include \"HttpResponseParser.h\"\n\n#include <drogon/config.h>\n#include <stdlib.h>\n#include <algorithm>\n\nusing namespace trantor;\nusing namespace drogon;\nusing namespace std::placeholders;\n\nnamespace trantor\n{\nstatic const size_t kDefaultDNSTimeout{600};\n}\n\nvoid HttpClientImpl::createTcpClient()\n{\n    LOG_TRACE << \"New TcpClient,\" << serverAddr_.toIpPort();\n    tcpClientPtr_ =\n        std::make_shared<trantor::TcpClient>(loop_, serverAddr_, \"httpClient\");\n\n    if (useSSL_ && utils::supportsTls())\n    {\n        LOG_TRACE << \"useOldTLS=\" << useOldTLS_;\n        LOG_TRACE << \"domain=\" << domain_;\n        auto policy = trantor::TLSPolicy::defaultClientPolicy();\n        policy->setUseOldTLS(useOldTLS_)\n            .setValidate(validateCert_)\n            .setHostname(domain_)\n            .setConfCmds(sslConfCmds_)\n            .setCertPath(clientCertPath_)\n            .setKeyPath(clientKeyPath_);\n        tcpClientPtr_->enableSSL(std::move(policy));\n    }\n\n    auto thisPtr = shared_from_this();\n    std::weak_ptr<HttpClientImpl> weakPtr = thisPtr;\n    tcpClientPtr_->setSockOptCallback([weakPtr](int fd) {\n        auto thisPtr = weakPtr.lock();\n        if (!thisPtr)\n            return;\n        if (thisPtr->sockOptCallback_)\n            thisPtr->sockOptCallback_(fd);\n    });\n    tcpClientPtr_->setConnectionCallback(\n        [weakPtr](const trantor::TcpConnectionPtr &connPtr) {\n            auto thisPtr = weakPtr.lock();\n            if (!thisPtr)\n                return;\n            if (connPtr->connected())\n            {\n                connPtr->setContext(\n                    std::make_shared<HttpResponseParser>(connPtr));\n                // send request;\n                LOG_TRACE << \"Connection established!\";\n                while (thisPtr->pipeliningCallbacks_.size() <=\n                           thisPtr->pipeliningDepth_ &&\n                       !thisPtr->requestsBuffer_.empty())\n                {\n                    thisPtr->sendReq(connPtr,\n                                     thisPtr->requestsBuffer_.front().first);\n                    thisPtr->pipeliningCallbacks_.push(\n                        std::move(thisPtr->requestsBuffer_.front()));\n                    thisPtr->requestsBuffer_.pop_front();\n                }\n            }\n            else\n            {\n                LOG_TRACE << \"connection disconnect\";\n                auto responseParser = connPtr->getContext<HttpResponseParser>();\n                if (responseParser && responseParser->parseResponseOnClose() &&\n                    responseParser->gotAll())\n                {\n                    auto &firstReq = thisPtr->pipeliningCallbacks_.front();\n                    if (firstReq.first->method() == Head)\n                    {\n                        responseParser->setForHeadMethod();\n                    }\n                    auto resp = responseParser->responseImpl();\n                    responseParser->reset();\n                    // temporary fix of dead tcpClientPtr_\n                    // TODO: fix HttpResponseParser when content-length absence\n                    thisPtr->tcpClientPtr_.reset();\n                    thisPtr->handleResponse(resp, std::move(firstReq), connPtr);\n                    if (!thisPtr->requestsBuffer_.empty())\n                    {\n                        thisPtr->createTcpClient();\n                    }\n                    return;\n                }\n                thisPtr->onError(ReqResult::NetworkFailure);\n            }\n        });\n    tcpClientPtr_->setConnectionErrorCallback([weakPtr]() {\n        auto thisPtr = weakPtr.lock();\n        if (!thisPtr)\n            return;\n        // can't connect to server\n        thisPtr->onError(ReqResult::BadServerAddress);\n    });\n    tcpClientPtr_->setMessageCallback(\n        [weakPtr](const trantor::TcpConnectionPtr &connPtr,\n                  trantor::MsgBuffer *msg) {\n            auto thisPtr = weakPtr.lock();\n            if (thisPtr)\n            {\n                thisPtr->onRecvMessage(connPtr, msg);\n            }\n        });\n    tcpClientPtr_->setSSLErrorCallback([weakPtr](SSLError err) {\n        auto thisPtr = weakPtr.lock();\n        if (!thisPtr)\n            return;\n        if (err == trantor::SSLError::kSSLHandshakeError)\n            thisPtr->onError(ReqResult::HandshakeError);\n        else if (err == trantor::SSLError::kSSLInvalidCertificate)\n            thisPtr->onError(ReqResult::InvalidCertificate);\n        else if (err == trantor::SSLError::kSSLProtocolError)\n            thisPtr->onError(ReqResult::EncryptionFailure);\n        else\n        {\n            LOG_FATAL << \"Invalid value for SSLError\";\n            abort();\n        }\n    });\n    tcpClientPtr_->connect();\n}\n\nHttpClientImpl::HttpClientImpl(trantor::EventLoop *loop,\n                               const trantor::InetAddress &addr,\n                               bool useSSL,\n                               bool useOldTLS,\n                               bool validateCert)\n    : loop_(loop),\n      serverAddr_(addr),\n      useSSL_(useSSL),\n      validateCert_(validateCert),\n      useOldTLS_(useOldTLS)\n{\n}\n\nHttpClientImpl::HttpClientImpl(trantor::EventLoop *loop,\n                               const std::string &hostString,\n                               bool useOldTLS,\n                               bool validateCert)\n    : loop_(loop), validateCert_(validateCert), useOldTLS_(useOldTLS)\n{\n    auto lowerHost = hostString;\n    std::transform(lowerHost.begin(),\n                   lowerHost.end(),\n                   lowerHost.begin(),\n                   [](unsigned char c) { return tolower(c); });\n    if (lowerHost.find(\"https://\") == 0)\n    {\n        useSSL_ = true;\n        lowerHost = lowerHost.substr(8);\n    }\n    else if (lowerHost.find(\"http://\") == 0)\n    {\n        useSSL_ = false;\n        lowerHost = lowerHost.substr(7);\n    }\n    else\n    {\n        return;\n    }\n    auto pos = lowerHost.find(']');\n    if (lowerHost[0] == '[' && pos != std::string::npos)\n    {\n        // ipv6\n        domain_ = lowerHost.substr(1, pos - 1);\n        if (lowerHost[pos + 1] == ':')\n        {\n            auto portStr = lowerHost.substr(pos + 2);\n            pos = portStr.find('/');\n            if (pos != std::string::npos)\n            {\n                portStr = portStr.substr(0, pos);\n            }\n            auto port = atoi(portStr.c_str());\n            if (port > 0 && port < 65536)\n            {\n                serverAddr_ = InetAddress(domain_, port, true);\n            }\n        }\n        else\n        {\n            if (useSSL_)\n            {\n                serverAddr_ = InetAddress(domain_, 443, true);\n            }\n            else\n            {\n                serverAddr_ = InetAddress(domain_, 80, true);\n            }\n        }\n    }\n    else\n    {\n        auto pos = lowerHost.find(':');\n        if (pos != std::string::npos)\n        {\n            domain_ = lowerHost.substr(0, pos);\n            auto portStr = lowerHost.substr(pos + 1);\n            pos = portStr.find('/');\n            if (pos != std::string::npos)\n            {\n                portStr = portStr.substr(0, pos);\n            }\n            auto port = atoi(portStr.c_str());\n            if (port > 0 && port < 65536)\n            {\n                serverAddr_ = InetAddress(domain_, port);\n            }\n        }\n        else\n        {\n            domain_ = lowerHost;\n            pos = domain_.find('/');\n            if (pos != std::string::npos)\n            {\n                domain_ = domain_.substr(0, pos);\n            }\n            if (useSSL_)\n            {\n                serverAddr_ = InetAddress(domain_, 443);\n            }\n            else\n            {\n                serverAddr_ = InetAddress(domain_, 80);\n            }\n        }\n    }\n    if (serverAddr_.isUnspecified())\n    {\n        isDomainName_ = true;\n    }\n    LOG_TRACE << \"userSSL=\" << useSSL_ << \" domain=\" << domain_;\n}\n\nHttpClientImpl::~HttpClientImpl()\n{\n    LOG_TRACE << \"Deconstruction HttpClient\";\n    if (resolverPtr_ && !(loop_->isInLoopThread()))\n    {\n        // Make sure the resolverPtr_ is destroyed in the correct thread.\n        loop_->queueInLoop([resolverPtr = std::move(resolverPtr_)]() {});\n    }\n}\n\nvoid HttpClientImpl::sendRequest(const drogon::HttpRequestPtr &req,\n                                 const drogon::HttpReqCallback &callback,\n                                 double timeout)\n{\n    auto thisPtr = shared_from_this();\n    loop_->runInLoop([thisPtr, req, callback = callback, timeout]() mutable {\n        thisPtr->sendRequestInLoop(req, std::move(callback), timeout);\n    });\n}\n\nvoid HttpClientImpl::sendRequest(const drogon::HttpRequestPtr &req,\n                                 drogon::HttpReqCallback &&callback,\n                                 double timeout)\n{\n    auto thisPtr = shared_from_this();\n    loop_->runInLoop(\n        [thisPtr, req, callback = std::move(callback), timeout]() mutable {\n            thisPtr->sendRequestInLoop(req, std::move(callback), timeout);\n        });\n}\n\nstruct RequestCallbackParams\n{\n    RequestCallbackParams(HttpReqCallback &&cb,\n                          HttpClientImplPtr client,\n                          HttpRequestPtr req)\n        : callback(std::move(cb)),\n          clientPtr(std::move(client)),\n          requestPtr(std::move(req))\n    {\n    }\n\n    const drogon::HttpReqCallback callback;\n    const HttpClientImplPtr clientPtr;\n    const HttpRequestPtr requestPtr;\n    bool timeoutFlag{false};\n};\n\nvoid HttpClientImpl::sendRequestInLoop(const HttpRequestPtr &req,\n                                       HttpReqCallback &&callback,\n                                       double timeout)\n{\n    if (timeout <= 0)\n    {\n        sendRequestInLoop(req, std::move(callback));\n        return;\n    }\n\n    auto callbackParamsPtr =\n        std::make_shared<RequestCallbackParams>(std::move(callback),\n                                                shared_from_this(),\n                                                req);\n\n    loop_->runAfter(\n        timeout,\n        [weakCallbackBackPtr =\n             std::weak_ptr<RequestCallbackParams>(callbackParamsPtr)] {\n            auto callbackParamsPtr = weakCallbackBackPtr.lock();\n            if (callbackParamsPtr != nullptr)\n            {\n                auto &thisPtr = callbackParamsPtr->clientPtr;\n                if (callbackParamsPtr->timeoutFlag)\n                {\n                    return;\n                }\n\n                callbackParamsPtr->timeoutFlag = true;\n\n                for (auto iter = thisPtr->requestsBuffer_.begin();\n                     iter != thisPtr->requestsBuffer_.end();\n                     ++iter)\n                {\n                    if (iter->first == callbackParamsPtr->requestPtr)\n                    {\n                        thisPtr->requestsBuffer_.erase(iter);\n                        break;\n                    }\n                }\n\n                (callbackParamsPtr->callback)(ReqResult::Timeout, nullptr);\n            }\n        });\n    sendRequestInLoop(req,\n                      [callbackParamsPtr](ReqResult r,\n                                          const HttpResponsePtr &resp) {\n                          if (callbackParamsPtr->timeoutFlag)\n                          {\n                              return;\n                          }\n                          callbackParamsPtr->timeoutFlag = true;\n                          (callbackParamsPtr->callback)(r, resp);\n                      });\n}\n\nstatic bool isValidIpAddr(const trantor::InetAddress &addr)\n{\n    if (addr.portNetEndian() == 0)\n    {\n        return false;\n    }\n    if (!addr.isIpV6())\n    {\n        return addr.ipNetEndian() != 0;\n    }\n    // Is ipv6\n    auto ipaddr = addr.ip6NetEndian();\n    for (int i = 0; i < 4; ++i)\n    {\n        if (ipaddr[i] != 0)\n        {\n            return true;\n        }\n    }\n    return false;\n}\n\nvoid HttpClientImpl::sendRequestInLoop(const drogon::HttpRequestPtr &req,\n                                       drogon::HttpReqCallback &&callback)\n{\n    loop_->assertInLoopThread();\n    if (!static_cast<drogon::HttpRequestImpl *>(req.get())->passThrough())\n    {\n        req->addHeader(\"connection\", \"Keep-Alive\");\n        if (!userAgent_.empty())\n            req->addHeader(\"user-agent\", userAgent_);\n    }\n    // Set the host header if not already set\n    if (req->getHeader(\"host\").empty())\n    {\n        if (onDefaultPort())\n        {\n            req->addHeader(\"host\", host());\n        }\n        else\n        {\n...\n// Path: lib/src/AOPAdvice.h\n/**\n *\n *  AOPAdvice.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n#include \"impl_forwards.h\"\n#include <drogon/drogon_callbacks.h>\n#include <trantor/net/InetAddress.h>\n#include <functional>\n#include <vector>\n\nnamespace drogon\n{\nclass AopAdvice\n{\n  public:\n    static AopAdvice &instance()\n    {\n        static AopAdvice inst;\n        return inst;\n    }\n\n    // Getters?\n    bool hasPreRoutingAdvices() const\n    {\n        return !preRoutingAdvices_.empty();\n    }\n\n    bool hasPostRoutingAdvices() const\n    {\n        return !postRoutingAdvices_.empty();\n    }\n\n    bool hasPreHandlingAdvices() const\n    {\n        return !preHandlingAdvices_.empty();\n    }\n\n    // Setters?\n    void registerNewConnectionAdvice(\n        std::function<bool(const trantor::InetAddress &,\n                           const trantor::InetAddress &)> advice)\n    {\n        newConnectionAdvices_.emplace_back(std::move(advice));\n    }\n\n    void registerHttpResponseCreationAdvice(\n        std::function<void(const HttpResponsePtr &)> advice)\n    {\n        responseCreationAdvices_.emplace_back(std::move(advice));\n    }\n\n    void registerSyncAdvice(\n        std::function<HttpResponsePtr(const HttpRequestPtr &)> advice)\n\n    {\n        syncAdvices_.emplace_back(std::move(advice));\n    }\n\n    void registerPreRoutingObserver(\n        std::function<void(const HttpRequestPtr &)> advice)\n    {\n        preRoutingObservers_.emplace_back(std::move(advice));\n    }\n\n    void registerPreRoutingAdvice(\n        std::function<void(const HttpRequestPtr &,\n                           AdviceCallback &&,\n                           AdviceChainCallback &&)> advice)\n    {\n        preRoutingAdvices_.emplace_back(std::move(advice));\n    }\n\n    void registerPostRoutingObserver(\n        std::function<void(const HttpRequestPtr &)> advice)\n    {\n        postRoutingObservers_.emplace_back(std::move(advice));\n    }\n\n    void registerPostRoutingAdvice(\n        std::function<void(const HttpRequestPtr &,\n                           AdviceCallback &&,\n                           AdviceChainCallback &&)> advice)\n    {\n        postRoutingAdvices_.emplace_back(std::move(advice));\n    }\n\n    void registerPreHandlingObserver(\n        std::function<void(const HttpRequestPtr &)> advice)\n    {\n        preHandlingObservers_.emplace_back(std::move(advice));\n    }\n\n    void registerPreHandlingAdvice(\n        std::function<void(const HttpRequestPtr &,\n                           AdviceCallback &&,\n                           AdviceChainCallback &&)> advice)\n    {\n        preHandlingAdvices_.emplace_back(std::move(advice));\n    }\n\n    void registerPostHandlingAdvice(\n        std::function<void(const HttpRequestPtr &, const HttpResponsePtr &)>\n            advice)\n    {\n        postHandlingAdvices_.emplace_back(std::move(advice));\n    }\n\n    void registerPreSendingAdvice(\n        std::function<void(const HttpRequestPtr &, const HttpResponsePtr &)>\n            advice)\n    {\n        preSendingAdvices_.emplace_back(std::move(advice));\n    }\n\n    // Executors\n    bool passNewConnectionAdvices(const trantor::TcpConnectionPtr &conn) const;\n    void passResponseCreationAdvices(const HttpResponsePtr &resp) const;\n\n    HttpResponsePtr passSyncAdvices(const HttpRequestPtr &req) const;\n    void passPreRoutingObservers(const HttpRequestImplPtr &req) const;\n    void passPreRoutingAdvices(\n        const HttpRequestImplPtr &req,\n        std::function<void(const HttpResponsePtr &)> &&callback) const;\n    void passPostRoutingObservers(const HttpRequestImplPtr &req) const;\n    void passPostRoutingAdvices(\n        const HttpRequestImplPtr &req,\n        std::function<void(const HttpResponsePtr &)> &&callback) const;\n    void passPreHandlingObservers(const HttpRequestImplPtr &req) const;\n    void passPreHandlingAdvices(\n        const HttpRequestImplPtr &req,\n        std::function<void(const HttpResponsePtr &)> &&callback) const;\n    void passPostHandlingAdvices(const HttpRequestImplPtr &req,\n                                 const HttpResponsePtr &resp) const;\n    void passPreSendingAdvices(const HttpRequestImplPtr &req,\n                               const HttpResponsePtr &resp) const;\n\n  private:\n    using SyncAdvice = std::function<HttpResponsePtr(const HttpRequestPtr &)>;\n    using SyncReqObserver = std::function<void(const HttpRequestPtr &)>;\n    using SyncObserver =\n        std::function<void(const HttpRequestPtr &, const HttpResponsePtr &)>;\n    using AsyncAdvice = std::function<void(const HttpRequestPtr &,\n                                           AdviceCallback &&,\n                                           AdviceChainCallback &&)>;\n\n    // If we want to add aop functions anytime, we can add a mutex here\n\n    std::vector<std::function<bool(const trantor::InetAddress &,\n                                   const trantor::InetAddress &)>>\n        newConnectionAdvices_;\n    std::vector<std::function<void(const HttpResponsePtr &)>>\n        responseCreationAdvices_;\n\n    std::vector<SyncAdvice> syncAdvices_;\n    std::vector<SyncReqObserver> preRoutingObservers_;\n    std::vector<AsyncAdvice> preRoutingAdvices_;\n    std::vector<SyncReqObserver> postRoutingObservers_;\n    std::vector<AsyncAdvice> postRoutingAdvices_;\n    std::vector<SyncReqObserver> preHandlingObservers_;\n    std::vector<AsyncAdvice> preHandlingAdvices_;\n    std::vector<SyncObserver> postHandlingAdvices_;\n    std::vector<SyncObserver> preSendingAdvices_;\n};\n\n}  // namespace drogon\n\n// Path: lib/src/HttpResponseImpl.cc\n/**\n *\n *  @file HttpResponseImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpResponseImpl.h\"\n#include \"AOPAdvice.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpUtils.h\"\n#include <drogon/HttpViewData.h>\n#include <drogon/IOThreadStorage.h>\n#include <filesystem>\n#include <fstream>\n#include <memory>\n#include <cstdio>\n#include <string>\n#include <sys/stat.h>\n#include <trantor/utils/Logger.h>\n\nusing namespace trantor;\nusing namespace drogon;\nusing namespace std::literals::string_literals;\nusing namespace std::placeholders;\n#ifdef _WIN32\n#undef max\n#endif\n\nnamespace drogon\n{\n// \"Fri, 23 Aug 2019 12:58:03 GMT\" length = 29\nstatic const size_t httpFullDateStringLength = 29;\n\n\nstatic inline HttpResponsePtr genHttpResponse(const std::string &viewName,\n                                              const HttpViewData &data,\n                                              const HttpRequestPtr &req)\n{\n    auto templ = DrTemplateBase::newTemplate(viewName);\n    if (templ)\n    {\n        auto res = HttpResponse::newHttpResponse();\n        res->setBody(templ->genText(data));\n        return res;\n    }\n    return drogon::HttpResponse::newNotFoundResponse(req);\n}\n}  // namespace drogon\n\nHttpResponsePtr HttpResponse::newHttpResponse()\n{\n    auto res = std::make_shared<HttpResponseImpl>(k200OK, CT_TEXT_HTML);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpResponse(HttpStatusCode code,\n                                              ContentType type)\n{\n    auto res = std::make_shared<HttpResponseImpl>(code, type);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpJsonResponse(const Json::Value &data)\n{\n    auto res = std::make_shared<HttpResponseImpl>(k200OK, CT_APPLICATION_JSON);\n    res->setJsonObject(data);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpJsonResponse(Json::Value &&data)\n{\n    auto res = std::make_shared<HttpResponseImpl>(k200OK, CT_APPLICATION_JSON);\n    res->setJsonObject(std::move(data));\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nconst char *HttpResponseImpl::versionString() const\n{\n    const char *result = \"UNKNOWN\";\n    switch (version_)\n    {\n        case Version::kHttp10:\n            result = \"HTTP/1.0\";\n            break;\n\n        case Version::kHttp11:\n            result = \"HTTP/1.1\";\n            break;\n\n        default:\n            break;\n    }\n    return result;\n}\n\nvoid HttpResponseImpl::generateBodyFromJson() const\n{\n    if (!jsonPtr_ || flagForSerializingJson_)\n    {\n        return;\n    }\n    flagForSerializingJson_ = true;\n    static std::once_flag once;\n    static Json::StreamWriterBuilder builder;\n    std::call_once(once, []() {\n        builder[\"commentStyle\"] = \"None\";\n        builder[\"indentation\"] = \"\";\n        if (!app().isUnicodeEscapingUsedInJson())\n        {\n            builder[\"emitUTF8\"] = true;\n        }\n        auto &precision = app().getFloatPrecisionInJson();\n        if (precision.first != 0)\n        {\n            builder[\"precision\"] = precision.first;\n            builder[\"precisionType\"] = precision.second;\n        }\n    });\n    bodyPtr_ = std::make_shared<HttpMessageStringBody>(\n        writeString(builder, *jsonPtr_));\n}\n\nHttpResponsePtr HttpResponse::newNotFoundResponse(const HttpRequestPtr &req)\n{\n    auto loop = trantor::EventLoop::getEventLoopOfCurrentThread();\n    auto &resp = HttpAppFrameworkImpl::instance().getCustom404Page();\n    if (resp)\n    {\n        if (loop && loop->index() < app().getThreadNum())\n        {\n            return resp;\n        }\n        else\n        {\n            return HttpResponsePtr{new HttpResponseImpl(\n                *static_cast<HttpResponseImpl *>(resp.get()))};\n        }\n    }\n    else\n    {\n        if (HttpAppFrameworkImpl::instance().isUsingCustomErrorHandler())\n        {\n            return app().getCustomErrorHandler()(k404NotFound, req);\n        }\n        else if (loop && loop->index() < app().getThreadNum())\n        {\n            // If the current thread is an IO thread\n            static std::once_flag threadOnce;\n            static IOThreadStorage<HttpResponsePtr> thread404Pages;\n            std::call_once(threadOnce, [req = req] {\n                thread404Pages.init([req = req](drogon::HttpResponsePtr &resp,\n                                                size_t /*index*/) {\n                    HttpViewData data;\n                    data.insert(\"version\", drogon::getVersion());\n                    resp = HttpResponse::newHttpViewResponse(\"drogon::NotFound\",\n                                                             data);\n                    resp->setStatusCode(k404NotFound);\n                    resp->setExpiredTime(0);\n                });\n            });\n            LOG_TRACE << \"Use cached 404 response\";\n            return thread404Pages.getThreadData();\n        }\n        else\n        {\n            HttpViewData data;\n            data.insert(\"version\", drogon::getVersion());\n            auto notFoundResp =\n                HttpResponse::newHttpViewResponse(\"drogon::NotFound\", data);\n            notFoundResp->setStatusCode(k404NotFound);\n            return notFoundResp;\n        }\n    }\n}\n\nHttpResponsePtr HttpResponse::newRedirectionResponse(\n    const std::string &location,\n    HttpStatusCode status)\n{\n    auto res = std::make_shared<HttpResponseImpl>();\n    res->setStatusCode(status);\n    res->redirect(location);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpViewResponse(const std::string &viewName,\n                                                  const HttpViewData &data,\n                                                  const HttpRequestPtr &req)\n{\n    return genHttpResponse(viewName, data, req);\n}\n\nHttpResponsePtr HttpResponse::newFileResponse(\n    const unsigned char *pBuffer,\n    size_t bufferLength,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString)\n{\n    // Make Raw HttpResponse\n    auto resp = std::make_shared<HttpResponseImpl>();\n\n    // Set response body and length\n    resp->setBody(\n        std::string(reinterpret_cast<const char *>(pBuffer), bufferLength));\n\n    // Set status of message\n    resp->setStatusCode(k200OK);\n\n    // Check for type and assign proper content type in header\n    if (!typeString.empty())\n    {\n        // auto contentType = type;\n        if (type == CT_NONE)\n            type = parseContentType(typeString);\n        if (type == CT_NONE)\n            type = CT_APPLICATION_OCTET_STREAM;  // XXX: Is this Ok?\n        static_cast<HttpResponse *>(resp.get())\n            ->setContentTypeCodeAndCustomString(type,\n                                                typeString.c_str(),\n                                                typeString.size());\n    }\n    else if (type != CT_NONE)\n    {\n        resp->setContentTypeCode(type);\n    }\n    else if (!attachmentFileName.empty())\n    {\n        resp->setContentTypeCode(drogon::getContentType(attachmentFileName));\n    }\n    else\n    {\n        resp->setContentTypeCode(\n            CT_APPLICATION_OCTET_STREAM);  // default content-type for file;\n    }\n\n    // Add additional header values\n    if (!attachmentFileName.empty())\n    {\n        resp->addHeader(\"Content-Disposition\",\n                        \"attachment; filename=\" + attachmentFileName);\n    }\n\n    // Finalize and return response\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nHttpResponsePtr HttpResponse::newFileResponse(\n    const std::string &fullPath,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString,\n    const HttpRequestPtr &req)\n{\n    return newFileResponse(\n        fullPath, 0, 0, false, attachmentFileName, type, typeString, req);\n}\n\nHttpResponsePtr HttpResponse::newFileResponse(\n    const std::string &fullPath,\n    size_t offset,\n    size_t length,\n    bool setContentRange,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString,\n    const HttpRequestPtr &req)\n{\n    std::ifstream infile(utils::toNativePath(fullPath), std::ifstream::binary);\n    LOG_TRACE << \"send http file:\" << fullPath << \" offset \" << offset\n              << \" length \" << length;\n    if (!infile)\n    {\n        auto resp = HttpResponse::newNotFoundResponse(req);\n        return resp;\n    }\n    auto resp = std::make_shared<HttpResponseImpl>();\n    std::streambuf *pbuf = infile.rdbuf();\n    size_t filesize =\n        static_cast<size_t>(pbuf->pubseekoff(0, std::ifstream::end));\n    if (offset > filesize || length > filesize ||  // in case of overflow\n        offset + length > filesize)\n    {\n        resp->setStatusCode(k416RequestedRangeNotSatisfiable);\n        if (setContentRange)\n        {\n            char buf[64];\n            snprintf(buf, sizeof(buf), \"bytes */%zu\", filesize);\n            resp->addHeader(\"Content-Range\", std::string(buf));\n        }\n        return resp;\n    }\n    if (length == 0)\n    {\n        length = filesize - offset;\n    }\n    pbuf->pubseekoff(offset, std::ifstream::beg);  // rewind\n\n    if (HttpAppFrameworkImpl::instance().useSendfile() && length > 1024 * 200)\n    // TODO : Is 200k an appropriate value? Or set it to be configurable\n    {\n        // The advantages of sendfile() can only be reflected in sending large\n        // files.\n        resp->setSendfile(fullPath);\n        // Must set length with the right value! Content-Length header relies on\n        // this value.\n        resp->setSendfileRange(offset, length);\n    }\n    else\n    {\n        std::string str;\n        str.resize(length);\n        pbuf->sgetn(&str[0], length);\n        resp->setBody(std::move(str));\n        resp->setSendfileRange(offset, length);\n    }\n\n    // Set correct status code\n    if (length < filesize)\n    {\n        resp->setStatusCode(k206PartialContent);\n    }\n    else\n    {\n        resp->setStatusCode(k200OK);\n    }\n\n    // Infer content type\n    if (type == CT_NONE)\n    {\n        if (!typeString.empty())\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            // auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n        else if (!attachmentFileName.empty())\n        {\n            resp->setContentTypeCode(\n                drogon::getContentType(attachmentFileName));\n        }\n        else\n        {\n            resp->setContentTypeCode(drogon::getContentType(fullPath));\n        }\n    }\n    else\n    {\n        if (typeString.empty())\n            resp->setContentTypeCode(type);\n        else\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            // auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n    }\n\n    // Set headers\n    if (!attachmentFileName.empty())\n    {\n        resp->addHeader(\"Content-Disposition\",\n                        \"attachment; filename=\" + attachmentFileName);\n    }\n    if (setContentRange && length > 0)\n    {\n        char buf[128];\n        snprintf(buf,\n                 sizeof(buf),\n                 \"bytes %zu-%zu/%zu\",\n                 offset,\n                 offset + length - 1,\n                 filesize);\n        resp->addHeader(\"Content-Range\", std::string(buf));\n    }\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nHttpResponsePtr HttpResponse::newStreamResponse(\n    const std::function<std::size_t(char *, std::size_t)> &callback,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString,\n    const HttpRequestPtr &req)\n{\n    LOG_TRACE << \"send stream as \"s\n              << (attachmentFileName.empty() ? \"raw data\"s\n                                             : \"file: \"s + attachmentFileName);\n    if (!callback)\n    {\n        auto resp = HttpResponse::newNotFoundResponse();\n        return resp;\n    }\n    auto resp = std::make_shared<HttpResponseImpl>();\n    resp->setStreamCallback(callback);\n    resp->setStatusCode(k200OK);\n\n    // Infer content type\n    if (type == CT_NONE)\n    {\n        if (!typeString.empty())\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n        else if (!attachmentFileName.empty())\n        {\n            resp->setContentTypeCode(\n                drogon::getContentType(attachmentFileName));\n        }\n    }\n    else\n    {\n        if (typeString.empty())\n            resp->setContentTypeCode(type);\n        else\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n    }\n\n    // Set headers\n    if (!attachmentFileName.empty())\n    {\n        resp->addHeader(\"Content-Disposition\",\n                        \"attachment; filename=\" + attachmentFileName);\n    }\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nHttpResponsePtr HttpResponse::newAsyncStreamResponse(\n    const std::function<void(ResponseStreamPtr)> &callback,\n    bool disableKickoffTimeout)\n{\n    if (!callback)\n    {\n        auto resp = HttpResponse::newNotFoundResponse();\n        return resp;\n    }\n    auto resp = std::make_shared<HttpResponseImpl>();\n    resp->setAsyncStreamCallback(callback, disableKickoffTimeout);\n    resp->setStatusCode(k200OK);\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nvoid HttpResponseImpl::makeHeaderString(trantor::MsgBuffer &buffer)\n{\n    buffer.ensureWritableBytes(128);\n    int len{0};\n    if (version_ == Version::kHttp11)\n    {\n        if (customStatusCode_ >= 0)\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.1 %d \",\n                           customStatusCode_);\n        }\n        else\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.1 %d \",\n                           statusCode_);\n        }\n    }\n    else\n    {\n        if (customStatusCode_ >= 0)\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.0 %d \",\n                           customStatusCode_);\n        }\n        else\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.0 %d \",\n                           statusCode_);\n        }\n    }\n    buffer.hasWritten(len);\n\n    if (!statusMessage_.empty())\n        buffer.append(statusMessage_.data(), statusMessage_.length());\n    buffer.append(\"\\r\\n\");\n    generateBodyFromJson();\n    if (!passThrough_)\n    {\n        buffer.ensureWritableBytes(64);\n        if (streamCallback_ || asyncStreamCallback_)\n        {\n            // When the headers are created, it is time to set the transfer\n            // encoding to chunked if the contents size is not specified\n            if (!ifCloseConnection() &&\n                headers_.find(\"content-length\") == headers_.end())\n            {\n                LOG_DEBUG << \"send stream with transfer-encoding chunked\";\n                headers_[\"transfer-encoding\"] = \"chunked\";\n            }\n            len = 0;\n        }\n        else if (sendfileName_.empty())\n        {\n            auto bodyLength = bodyPtr_ ? bodyPtr_->length() : 0;\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           contentLengthFormatString<decltype(bodyLength)>(),\n                           bodyLength);\n        }\n        else\n        {\n            auto bodyLength = sendfileRange_.second;\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           contentLengthFormatString<decltype(bodyLength)>(),\n                           bodyLength);\n        }\n        buffer.hasWritten(len);\n        if (headers_.find(\"connection\") == headers_.end())\n        {\n            if (closeConnection_)\n            {\n                buffer.append(\"connection: close\\r\\n\");\n            }\n            else if (version_ == Version::kHttp10)\n            {\n                buffer.append(\"connection: Keep-Alive\\r\\n\");\n            }\n        }\n\n        if (!contentTypeString_.empty())\n        {\n            buffer.append(\"content-type: \");\n            buffer.append(contentTypeString_);\n            buffer.append(\"\\r\\n\");\n        }\n        if (HttpAppFrameworkImpl::instance().sendServerHeader())\n        {\n            buffer.append(\n                HttpAppFrameworkImpl::instance().getServerHeaderString());\n        }\n    }\n\n    for (auto it = headers_.begin(); it != headers_.end(); ++it)\n    {\n        buffer.append(it->first);\n        buffer.append(\": \");\n        buffer.append(it->second);\n        buffer.append(\"\\r\\n\");\n    }\n}\n\nvoid HttpResponseImpl::renderToBuffer(trantor::MsgBuffer &buffer)\n{\n    if (expriedTime_ >= 0)\n    {\n        auto strPtr = renderToBuffer();\n        buffer.append(strPtr->peek(), strPtr->readableBytes());\n        return;\n    }\n\n    if (!fullHeaderString_)\n    {\n        makeHeaderString(buffer);\n    }\n    else\n    {\n        buffer.append(*fullHeaderString_);\n    }\n\n    // output cookies\n    if (!cookies_.empty())\n    {\n        for (auto it = cookies_.begin(); it != cookies_.end(); ++it)\n        {\n            buffer.append(it->second.cookieString());\n        }\n    }\n\n    // output Date header\n    if (!passThrough_ &&\n        drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n    {\n        buffer.append(\"date: \");\n        buffer.append(utils::getHttpFullDate(trantor::Date::date()),\n                      httpFullDateStringLength);\n        buffer.append(\"\\r\\n\\r\\n\");\n    }\n    else\n    {\n        buffer.append(\"\\r\\n\");\n    }\n    if (bodyPtr_)\n        buffer.append(bodyPtr_->data(), bodyPtr_->length());\n}\n\nstd::shared_ptr<trantor::MsgBuffer> HttpResponseImpl::renderToBuffer()\n{\n    if (expriedTime_ >= 0)\n    {\n        if (!passThrough_ &&\n            drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n        {\n            if (datePos_ != static_cast<size_t>(-1))\n            {\n                auto now = trantor::Date::now();\n                bool isDateChanged =\n                    ((now.microSecondsSinceEpoch() / MICRO_SECONDS_PRE_SEC) !=\n                     httpStringDate_);\n                assert(httpString_);\n                if (isDateChanged)\n                {\n                    httpStringDate_ =\n                        now.microSecondsSinceEpoch() / MICRO_SECONDS_PRE_SEC;\n                    auto newDate = utils::getHttpFullDate(now);\n\n                    httpString_ =\n                        std::make_shared<trantor::MsgBuffer>(*httpString_);\n                    memcpy((void *)&(*httpString_)[datePos_],\n                           newDate,\n                           httpFullDateStringLength);\n                    return httpString_;\n                }\n\n                return httpString_;\n            }\n        }\n        else\n        {\n            if (httpString_)\n                return httpString_;\n        }\n    }\n    auto httpString = std::make_shared<trantor::MsgBuffer>(256);\n    if (!fullHeaderString_)\n    {\n        makeHeaderString(*httpString);\n    }\n    else\n    {\n        httpString->append(*fullHeaderString_);\n    }\n\n    // output cookies\n    if (!cookies_.empty())\n    {\n        for (auto it = cookies_.begin(); it != cookies_.end(); ++it)\n        {\n            httpString->append(it->second.cookieString());\n        }\n    }\n\n    // output Date header\n    if (!passThrough_ &&\n        drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n    {\n        httpString->append(\"date: \");\n        auto datePos = httpString->readableBytes();\n        httpString->append(utils::getHttpFullDate(trantor::Date::date()),\n                           httpFullDateStringLength);\n        httpString->append(\"\\r\\n\\r\\n\");\n        datePos_ = datePos;\n    }\n    else\n    {\n        httpString->append(\"\\r\\n\");\n    }\n\n    LOG_TRACE << \"response(no body):\"\n              << std::string_view{httpString->peek(),\n                                  httpString->readableBytes()};\n    if (bodyPtr_)\n        httpString->append(bodyPtr_->data(), bodyPtr_->length());\n    if (expriedTime_ >= 0)\n    {\n        httpString_ = httpString;\n    }\n    return httpString;\n}\n\nstd::shared_ptr<trantor::MsgBuffer> HttpResponseImpl::\n    renderHeaderForHeadMethod()\n{\n    auto httpString = std::make_shared<trantor::MsgBuffer>(256);\n    if (!fullHeaderString_)\n    {\n        makeHeaderString(*httpString);\n    }\n    else\n    {\n        httpString->append(*fullHeaderString_);\n    }\n\n    // output cookies\n    if (!cookies_.empty())\n    {\n        for (auto it = cookies_.begin(); it != cookies_.end(); ++it)\n        {\n            httpString->append(it->second.cookieString());\n        }\n    }\n\n    // output Date header\n    if (!passThrough_ &&\n        drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n    {\n        httpString->append(\"date: \");\n        httpString->append(utils::getHttpFullDate(trantor::Date::date()),\n                           httpFullDateStringLength);\n        httpString->append(\"\\r\\n\\r\\n\");\n    }\n    else\n    {\n        httpString->append(\"\\r\\n\");\n    }\n\n    return httpString;\n}\n\nvoid HttpResponseImpl::addHeader(const char *start,\n                                 const char *colon,\n                                 const char *end)\n{\n    fullHeaderString_.reset();\n    std::string field(start, colon);\n    transform(field.begin(), field.end(), field.begin(), [](unsigned char c) {\n        return tolower(c);\n    });\n    ++colon;\n    while (colon < end && isspace(static_cast<unsigned char>(*colon)))\n    {\n        ++colon;\n    }\n    std::string value(colon, end);\n    while (!value.empty() &&\n           isspace(static_cast<unsigned char>(value[value.size() - 1])))\n    {\n        value.resize(value.size() - 1);\n    }\n\n    if (field == \"set-cookie\")\n    {\n        // LOG_INFO<<\"cookies!!!:\"<<value;\n        auto values = utils::splitString(value, \";\");\n        Cookie cookie;\n        cookie.setHttpOnly(false);\n        for (size_t i = 0; i < values.size(); ++i)\n        {\n            std::string &coo = values[i];\n            std::string cookie_name;\n            std::string cookie_value;\n            auto epos = coo.find('=');\n            if (epos != std::string::npos)\n            {\n                cookie_name = coo.substr(0, epos);\n                std::string::size_type cpos = 0;\n                while (cpos < cookie_name.length() &&\n                       isspace(static_cast<unsigned char>(cookie_name[cpos])))\n                    ++cpos;\n                cookie_name = cookie_name.substr(cpos);\n                ++epos;\n                while (epos < coo.length() &&\n                       isspace(static_cast<unsigned char>(coo[epos])))\n                    ++epos;\n                cookie_value = coo.substr(epos);\n            }\n            else\n            {\n                std::string::size_type cpos = 0;\n                while (cpos < coo.length() &&\n                       isspace(static_cast<unsigned char>(coo[cpos])))\n                    ++cpos;\n                cookie_name = coo.substr(cpos);\n            }\n            if (i == 0)\n            {\n                cookie.setKey(cookie_name);\n                cookie.setValue(cookie_value);\n            }\n            else\n            {\n                std::transform(cookie_name.begin(),\n                               cookie_name.end(),\n                               cookie_name.begin(),\n                               [](unsigned char c) { return tolower(c); });\n                if (cookie_name == \"path\")\n                {\n                    cookie.setPath(cookie_value);\n                }\n                else if (cookie_name == \"domain\")\n                {\n                    cookie.setDomain(cookie_value);\n                }\n                else if (cookie_name == \"expires\")\n                {\n                    cookie.setExpiresDate(utils::getHttpDate(cookie_value));\n                }\n                else if (cookie_name == \"secure\")\n                {\n                    cookie.setSecure(true);\n                }\n                else if (cookie_name == \"httponly\")\n                {\n                    cookie.setHttpOnly(true);\n                }\n                else if (cookie_name == \"samesite\")\n                {\n                    cookie.setSameSite(\n                        cookie.convertString2SameSite(cookie_value));\n                }\n                else if (cookie_name == \"max-age\")\n                {\n                    cookie.setMaxAge(std::stoi(cookie_value));\n                }\n            }\n        }\n        if (!cookie.key().empty())\n        {\n            cookies_[cookie.key()] = cookie;\n        }\n    }\n    else\n    {\n        headers_[std::move(field)] = std::move(value);\n    }\n}\n\nvoid HttpResponseImpl::swap(HttpResponseImpl &that) noexcept\n{\n    using std::swap;\n    headers_.swap(that.headers_);\n    cookies_.swap(that.cookies_);\n    swap(statusCode_, that.statusCode_);\n    swap(version_, that.version_);\n    swap(statusMessage_, that.statusMessage_);\n    swap(closeConnection_, that.closeConnection_);\n    bodyPtr_.swap(that.bodyPtr_);\n    swap(contentType_, that.contentType_);\n    swap(flagForParsingContentType_, that.flagForParsingContentType_);\n    swap(flagForParsingJson_, that.flagForParsingJson_);\n    swap(sendfileName_, that.sendfileName_);\n    swap(streamCallback_, that.streamCallback_);\n    swap(asyncStreamCallback_, that.asyncStreamCallback_);\n    jsonPtr_.swap(that.jsonPtr_);\n    fullHeaderString_.swap(that.fullHeaderString_);\n    httpString_.swap(that.httpString_);\n    swap(datePos_, that.datePos_);\n    swap(jsonParsingErrorPtr_, that.jsonParsingErrorPtr_);\n}\n\nvoid HttpResponseImpl::clear()\n{\n    statusCode_ = kUnknown;\n    version_ = Version::kHttp11;\n    statusMessage_ = std::string_view{};\n    fullHeaderString_.reset();\n    jsonParsingErrorPtr_.reset();\n    sendfileName_.clear();\n    if (streamCallback_)\n    {\n        LOG_TRACE << \"Cleanup HttpResponse stream callback\";\n        streamCallback_(nullptr, 0);  // callback internal cleanup\n        streamCallback_ = {};\n    }\n    if (asyncStreamCallback_)\n    {\n        // asyncStreamCallback_(nullptr);\n        asyncStreamCallback_ = {};\n    }\n    headers_.clear();\n    cookies_.clear();\n    bodyPtr_.reset();\n    jsonPtr_.reset();\n    expriedTime_ = -1;\n    datePos_ = std::string::npos;\n    flagForParsingContentType_ = false;\n    flagForParsingJson_ = false;\n}\n\nvoid HttpResponseImpl::parseJson() const\n{\n    static std::once_flag once;\n    static Json::CharReaderBuilder builder;\n    std::call_once(once, []() {\n        builder[\"collectComments\"] = false;\n        builder[\"stackLimit\"] =\n            static_cast<Json::UInt>(drogon::app().getJsonParserStackLimit());\n    });\n    JSONCPP_STRING errs;\n    std::unique_ptr<Json::CharReader> reader(builder.newCharReader());\n    if (bodyPtr_)\n    {\n        jsonPtr_ = std::make_shared<Json::Value>();\n        if (!reader->parse(bodyPtr_->data(),\n                           bodyPtr_->data() + bodyPtr_->length(),\n                           jsonPtr_.get(),\n                           &errs))\n        {\n            LOG_ERROR << errs;\n            LOG_ERROR << \"body: \" << bodyPtr_->getString();\n            jsonPtr_.reset();\n            jsonParsingErrorPtr_ =\n                std::make_shared<std::string>(std::move(errs));\n        }\n        else\n        {\n            jsonParsingErrorPtr_.reset();\n        }\n    }\n    else\n    {\n        jsonPtr_.reset();\n        jsonParsingErrorPtr_ =\n            std::make_shared<std::string>(\"empty response body\");\n    }\n}\n\nbool HttpResponseImpl::shouldBeCompressed() const\n{\n    if (streamCallback_ || asyncStreamCallback_ || !sendfileName_.empty() ||\n        contentType() >= CT_APPLICATION_OCTET_STREAM ||\n        getBody().length() < 1024 || !(getHeaderBy(\"content-encoding\").empty()))\n    {\n        return false;\n    }\n    return true;\n}\n\nvoid HttpResponseImpl::setContentTypeString(const char *typeString,\n                                            size_t typeStringLength)\n{\n    std::string sv(typeString, typeStringLength);\n    auto contentType = parseContentType(sv);\n    if (contentType == CT_NONE)\n        contentType = CT_CUSTOM;\n    contentType_ = contentType;\n    contentTypeString_ = std::string(sv);\n    flagForParsingContentType_ = true;\n}\n\n// Path: lib/src/ConfigLoader.h\n/**\n *\n *  ConfigLoader.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <json/json.h>\n#include <string>\n#include <trantor/utils/NonCopyable.h>\n\nnamespace drogon\n{\nclass ConfigLoader : public trantor::NonCopyable\n{\n  public:\n    explicit ConfigLoader(const std::string &configFile) noexcept(false);\n    explicit ConfigLoader(const Json::Value &data);\n    explicit ConfigLoader(Json::Value &&data);\n    ~ConfigLoader();\n\n    const Json::Value &jsonValue() const\n    {\n        return configJsonRoot_;\n    }\n\n    void load() noexcept(false);\n\n  private:\n    std::string configFile_;\n    Json::Value configJsonRoot_;\n};\n}  // namespace drogon\n\n// Path: lib/src/HttpServer.h\n/**\n *\n *  @file HttpServer.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <trantor/net/TcpServer.h>\n#include <trantor/utils/NonCopyable.h>\n#include <functional>\n#include <string>\n#include <vector>\n#include \"impl_forwards.h\"\n\nstruct CallbackParamPack;\n\nnamespace drogon\n{\nclass ControllerBinderBase;\n\nclass HttpServer : trantor::NonCopyable\n{\n  public:\n    HttpServer(trantor::EventLoop *loop,\n               const trantor::InetAddress &listenAddr,\n               std::string name);\n\n    ~HttpServer();\n\n    void setIoLoops(const std::vector<trantor::EventLoop *> &ioLoops)\n    {\n        server_.setIoLoops(ioLoops);\n    }\n\n    void start();\n    void stop();\n\n    void enableSSL(trantor::TLSPolicyPtr policy)\n    {\n        server_.enableSSL(std::move(policy));\n    }\n\n    const trantor::InetAddress &address() const\n    {\n        return server_.address();\n    }\n\n  private:\n    friend class HttpInternalForwardHelper;\n\n    static void onConnection(const trantor::TcpConnectionPtr &conn);\n    static void onMessage(const trantor::TcpConnectionPtr &,\n                          trantor::MsgBuffer *);\n    static void onRequests(const trantor::TcpConnectionPtr &,\n                           const std::vector<HttpRequestImplPtr> &,\n                           const std::shared_ptr<HttpRequestParser> &);\n\n    struct HttpRequestParamPack\n    {\n        std::shared_ptr<ControllerBinderBase> binderPtr;\n        std::function<void(const HttpResponsePtr &)> callback;\n    };\n\n    struct WsRequestParamPack\n    {\n        std::shared_ptr<ControllerBinderBase> binderPtr;\n        std::function<void(const HttpResponsePtr &)> callback;\n        WebSocketConnectionImplPtr wsConnPtr;\n    };\n\n    // Http request handling steps\n    static void onHttpRequest(const HttpRequestImplPtr &,\n                              std::function<void(const HttpResponsePtr &)> &&);\n    static void httpRequestRouting(\n        const HttpRequestImplPtr &req,\n        std::function<void(const HttpResponsePtr &)> &&callback);\n    static void httpRequestHandling(\n        const HttpRequestImplPtr &req,\n        std::shared_ptr<ControllerBinderBase> &&binderPtr,\n        std::function<void(const HttpResponsePtr &)> &&callback);\n\n    // Websocket request handling steps\n    static void onWebsocketRequest(\n        const HttpRequestImplPtr &,\n        std::function<void(const HttpResponsePtr &)> &&,\n        WebSocketConnectionImplPtr &&);\n    static void websocketRequestRouting(\n        const HttpRequestImplPtr &req,\n        std::function<void(const HttpResponsePtr &)> &&callback,\n        WebSocketConnectionImplPtr &&wsConnPtr);\n    static void websocketRequestHandling(\n        const HttpRequestImplPtr &req,\n        std::shared_ptr<ControllerBinderBase> &&binderPtr,\n        std::function<void(const HttpResponsePtr &)> &&callback,\n        WebSocketConnectionImplPtr &&wsConnPtr);\n\n    // Http/Websocket shared handling steps\n    template <typename Pack>\n    static void requestPostRouting(const HttpRequestImplPtr &req, Pack &&pack);\n    template <typename Pack>\n    static void requestPassFilters(const HttpRequestImplPtr &req, Pack &&pack);\n    template <typename Pack>\n    static void requestPreHandling(const HttpRequestImplPtr &req, Pack &&pack);\n\n    // Response buffering and sending\n    static void handleResponse(\n        const HttpResponsePtr &response,\n        const std::shared_ptr<CallbackParamPack> &paramPack,\n        bool *respReadyPtr);\n    static void sendResponse(const trantor::TcpConnectionPtr &,\n                             const HttpResponsePtr &,\n                             bool isHeadMethod);\n    static void sendResponses(\n        const trantor::TcpConnectionPtr &conn,\n        const std::vector<std::pair<HttpResponsePtr, bool>> &responses,\n        trantor::MsgBuffer &buffer);\n\n    trantor::TcpServer server_;\n};\n\nclass HttpInternalForwardHelper\n{\n  public:\n    static void forward(const HttpRequestImplPtr &req,\n                        std::function<void(const HttpResponsePtr &)> &&callback)\n    {\n        return HttpServer::onHttpRequest(req, std::move(callback));\n    }\n};\n\n}  // namespace drogon\n\n// Path: lib/src/ListenerManager.h\n/**\n *\n *  @file ListenerManager.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <trantor/net/EventLoopThreadPool.h>\n#include <trantor/net/callbacks.h>\n#include <trantor/utils/NonCopyable.h>\n#include <memory>\n#include <string>\n#include <utility>\n#include <vector>\n#include \"impl_forwards.h\"\n\nnamespace trantor\n{\nclass InetAddress;\n}\n\nnamespace drogon\n{\nclass ListenerManager : public trantor::NonCopyable\n{\n  public:\n    ~ListenerManager() = default;\n    void addListener(const std::string &ip,\n                     uint16_t port,\n                     bool useSSL = false,\n                     const std::string &certFile = \"\",\n                     const std::string &keyFile = \"\",\n                     bool useOldTLS = false,\n                     const std::vector<std::pair<std::string, std::string>>\n                         &sslConfCmds = {});\n    std::vector<trantor::InetAddress> getListeners() const;\n    void createListeners(\n        const std::string &globalCertFile,\n        const std::string &globalKeyFile,\n        const std::vector<std::pair<std::string, std::string>> &sslConfCmds,\n        const std::vector<trantor::EventLoop *> &ioLoops);\n    void startListening();\n    void stopListening();\n\n  private:\n    struct ListenerInfo\n    {\n        ListenerInfo(\n            std::string ip,\n            uint16_t port,\n            bool useSSL,\n            std::string certFile,\n            std::string keyFile,\n            bool useOldTLS,\n            std::vector<std::pair<std::string, std::string>> sslConfCmds)\n            : ip_(std::move(ip)),\n              port_(port),\n              useSSL_(useSSL),\n              certFile_(std::move(certFile)),\n              keyFile_(std::move(keyFile)),\n              useOldTLS_(useOldTLS),\n              sslConfCmds_(std::move(sslConfCmds))\n        {\n        }\n\n        std::string ip_;\n        uint16_t port_;\n        bool useSSL_;\n        std::string certFile_;\n        std::string keyFile_;\n        bool useOldTLS_;\n        std::vector<std::pair<std::string, std::string>> sslConfCmds_;\n    };\n\n    std::vector<ListenerInfo> listeners_;\n    std::vector<std::shared_ptr<HttpServer>> servers_;\n\n    // should have value when and only when on OS that one port can only be\n    // listened by one thread\n    std::unique_ptr<trantor::EventLoopThread> listeningThread_;\n};\n\n}  // namespace drogon\n\n// Path: lib/src/RedisClientManager.h\n/**\n *\n *  RedisClientManager.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <drogon/nosql/RedisClient.h>\n#include <drogon/HttpAppFramework.h>\n#include <drogon/IOThreadStorage.h>\n#include <trantor/utils/NonCopyable.h>\n...\n", "needle_token_start": 5570, "needle_token_end": 5689, "code_context_ntokens": 16124, "output": ["```cpp\nstatic inline HttpResponsePtr genHttpResponse(const std::string &viewName,\n                                              const HttpViewData &data,\n                                              const HttpRequestPtr &req)\n{\n    auto templ = DrTemplateBase::newTemplate(viewName);\n    if (templ)\n    {\n        auto res = HttpResponse::newHttpResponse();\n        res->setBody(templ->genText(data));\n        return res;\n    }\n    return drogon::HttpResponse::newNotFoundResponse(req);\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "splitStringToSet", "language": "cpp", "path": "lib/src/Utilities.cc", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to divide a given string into distinct substrings based on a specified delimiter, and then store these substrings in a set to ensure all entries are unique and automatically ordered.\n2. **Input**: The function takes two strings: one is the main string to be split, and the other is the delimiter used to split the string.\n3. **Output**: It returns a set of strings, where each element is a unique substring from the original string, split according to the specified delimiter.\n4. **Procedure**: The function starts by initializing two indices to manage the current position within the string and the position of the next delimiter. It iterates through the main string, using the delimiter to find the end of each substring. Each found substring is then added to the set (if it's not an empty string), ensuring no duplicates. This process repeats until the entire string has been processed. If any part of the string remains after the last delimiter, it is also added to the set.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/MultiPart.cc\n/**\n *\n *  @file MultiPart.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpRequestImpl.h\"\n#include \"HttpUtils.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpFileImpl.h\"\n#include <drogon/MultiPart.h>\n#include <drogon/utils/Utilities.h>\n#include <drogon/config.h>\n#include <algorithm>\n#include <fcntl.h>\n#include <fstream>\n#include <iostream>\n#include <sys/stat.h>\n#ifndef _WIN32\n#include <unistd.h>\n#endif\n\nusing namespace drogon;\n\nconst std::vector<HttpFile> &MultiPartParser::getFiles() const\n{\n    return files_;\n}\n\nstd::unordered_map<std::string, HttpFile> MultiPartParser::getFilesMap() const\n{\n    std::unordered_map<std::string, HttpFile> result;\n    for (auto &file : files_)\n    {\n        result.emplace(file.getItemName(), file);\n    }\n    return result;\n}\n\nconst SafeStringMap<std::string> &MultiPartParser::getParameters() const\n{\n    return parameters_;\n}\n\nint MultiPartParser::parse(const HttpRequestPtr &req)\n{\n    if (req->method() != Post && req->method() != Put)\n        return -1;\n    const std::string &contentType =\n        static_cast<HttpRequestImpl *>(req.get())->getHeaderBy(\"content-type\");\n    if (contentType.empty())\n    {\n        return -1;\n    }\n    std::string::size_type pos = contentType.find(';');\n    if (pos == std::string::npos)\n        return -1;\n\n    std::string type = contentType.substr(0, pos);\n    std::transform(type.begin(), type.end(), type.begin(), [](unsigned char c) {\n        return tolower(c);\n    });\n    if (type != \"multipart/form-data\")\n        return -1;\n    pos = contentType.find(\"boundary=\");\n    if (pos == std::string::npos)\n        return -1;\n    auto pos2 = contentType.find(';', pos);\n    if (pos2 == std::string::npos)\n        pos2 = contentType.size();\n    return parse(req, contentType.data() + (pos + 9), pos2 - (pos + 9));\n}\n\nstatic std::pair<std::string_view, std::string_view> parseLine(\n    const char *begin,\n    const char *end)\n{\n    auto p = begin;\n    while (p != end)\n    {\n        if (*p == ':')\n        {\n            if (p + 1 != end && *(p + 1) == ' ')\n            {\n                return std::make_pair(std::string_view(begin, p - begin),\n...\n// Path: lib/src/SharedLibManager.cc\n/**\n *\n *  @file SharedLibManager.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"SharedLibManager.h\"\n#include <drogon/config.h>\n#include <dirent.h>\n#include <dlfcn.h>\n#include <fstream>\n#include <sys/types.h>\n#include <trantor/utils/Logger.h>\n#include <unistd.h>\n\nstatic void forEachFileIn(\n    const std::string &path,\n    const std::function<void(const std::string &, const struct stat &)> &cb)\n{\n    DIR *dp;\n    struct dirent *dirp;\n    struct stat st;\n\n    /* open dirent directory */\n    if ((dp = opendir(path.c_str())) == NULL)\n    {\n        // perror(\"opendir:\");\n        LOG_ERROR << \"can't open dir,path:\" << path;\n        return;\n    }\n\n    /**\n     * read all files in this dir\n     **/\n    while ((dirp = readdir(dp)) != NULL)\n    {\n        /* ignore hidden files */\n        if (dirp->d_name[0] == '.')\n            continue;\n        /* get dirent status */\n        std::string filename = dirp->d_name;\n        std::string fullname = path;\n        fullname.append(\"/\").append(filename);\n        if (stat(fullname.c_str(), &st) == -1)\n        {\n            perror(\"stat\");\n            closedir(dp);\n            return;\n        }\n\n        /* if dirent is a directory, find files recursively */\n        if (S_ISDIR(st.st_mode))\n        {\n            forEachFileIn(fullname, cb);\n        }\n        else\n        {\n            cb(fullname, st);\n        }\n    }\n    closedir(dp);\n    return;\n}\n\nusing namespace drogon;\n\nSharedLibManager::SharedLibManager(const std::vector<std::string> &libPaths,\n                                   const std::string &outputPath)\n    : libPaths_(libPaths), outputPath_(outputPath)\n{\n    workingThread_.run();\n    timeId_ =\n        workingThread_.getLoop()->runEvery(5.0, [this]() { managerLibs(); });\n}\n\nSharedLibManager::~SharedLibManager()\n{\n    workingThread_.getLoop()->invalidateTimer(timeId_);\n}\n\nvoid SharedLibManager::managerLibs()\n{\n    for (auto const &libPath : libPaths_)\n    {\n        forEachFileIn(\n            libPath,\n            [this, libPath](const std::string &filename,\n                            const struct stat &st) {\n                auto pos = filename.rfind('.');\n                if (pos != std::string::npos)\n                {\n                    auto exName = filename.substr(pos + 1);\n                    if (exName == \"csp\")\n                    {\n                        // compile\n                        auto lockFile = filename + \".lock\";\n                        std::ifstream fin(lockFile);\n                        if (fin)\n                        {\n                            return;\n                        }\n\n                        void *oldHandle = nullptr;\n                        if (dlMap_.find(filename) != dlMap_.end())\n                        {\n#if defined __linux__ || defined __HAIKU__\n                            if (st.st_mtim.tv_sec >\n                                dlMap_[filename].mTime.tv_sec)\n#elif defined _WIN32\n                            if (st.st_mtime > dlMap_[filename].mTime.tv_sec)\n#else\n                            if (st.st_mtimespec.tv_sec >\n                                dlMap_[filename].mTime.tv_sec)\n#endif\n                            {\n                                LOG_TRACE << \"new csp file:\" << filename;\n                                oldHandle = dlMap_[filename].handle;\n                            }\n                            else\n                                return;\n                        }\n\n                        {\n                            std::ofstream fout(lockFile);\n                        }\n\n                        auto srcFile = filename.substr(0, pos);\n                        if (!outputPath_.empty())\n                        {\n                            pos = srcFile.rfind(\"/\");\n                            if (pos != std::string::npos)\n                            {\n                                srcFile = srcFile.substr(pos + 1);\n                            }\n                            srcFile = outputPath_ + \"/\" + srcFile;\n                        }\n                        auto soFile = srcFile + \".so\";\n                        DLStat dlStat;\n                        if (!shouldCompileLib(soFile, st))\n                        {\n                            LOG_TRACE << \"Using already compiled library:\"\n                                      << soFile;\n                            dlStat.handle = loadLib(soFile, oldHandle);\n                        }\n                        else\n                        {\n                            // generate source code and compile it.\n                            std::string cmd = \"drogon_ctl create view \";\n                            if (!outputPath_.empty())\n                            {\n                                cmd.append(filename).append(\" -o \").append(\n                                    outputPath_);\n                            }\n                            else\n                            {\n                                cmd.append(filename).append(\" -o \").append(\n                                    libPath);\n                            }\n                            srcFile.append(\".cc\");\n                            LOG_TRACE << cmd;\n                            auto r = system(cmd.c_str());\n                            // TODO: handle r\n                            (void)(r);\n                            dlStat.handle =\n                                compileAndLoadLib(srcFile, oldHandle);\n                        }\n#if defined __linux__ || defined __HAIKU__\n                        dlStat.mTime = st.st_mtim;\n#elif defined _WIN32\n                        dlStat.mTime.tv_sec = st.st_mtime;\n#else\n                        dlStat.mTime = st.st_mtimespec;\n#endif\n                        if (dlStat.handle)\n                        {\n                            dlMap_[filename] = dlStat;\n                        }\n                        else\n                        {\n                            dlStat.handle = dlMap_[filename].handle;\n                            dlMap_[filename] = dlStat;\n                        }\n                        workingThread_.getLoop()->runAfter(3.5, [lockFile]() {\n                            LOG_TRACE << \"remove file \" << lockFile;\n                            if (unlink(lockFile.c_str()) == -1)\n                                perror(\"\");\n                        });\n                    }\n                }\n            });\n    }\n}\n\nvoid *SharedLibManager::compileAndLoadLib(const std::string &sourceFile,\n                                          void *oldHld)\n{\n    LOG_TRACE << \"src:\" << sourceFile;\n    std::string cmd = COMPILER_COMMAND;\n    cmd.append(\" \")\n        .append(sourceFile)\n        .append(\" \")\n        .append(COMPILATION_FLAGS)\n        .append(\" \")\n        .append(INCLUDING_DIRS);\n    if (std::string(COMPILER_ID).find(\"Clang\") != std::string::npos)\n        cmd.append(\" -shared -fPIC -undefined dynamic_lookup -o \");\n    else\n        cmd.append(\" -shared -fPIC --no-gnu-unique -o \");\n    auto pos = sourceFile.rfind('.');\n    auto soFile = sourceFile.substr(0, pos);\n    soFile.append(\".so\");\n    cmd.append(soFile);\n    LOG_TRACE << cmd;\n\n    if (system(cmd.c_str()) == 0)\n    {\n        LOG_TRACE << \"Compiled successfully:\" << soFile;\n        return loadLib(soFile, oldHld);\n    }\n    else\n    {\n        LOG_DEBUG << \"Could not compile library.\";\n        return nullptr;\n    }\n}\n\nbool SharedLibManager::shouldCompileLib(const std::string &soFile,\n                                        const struct stat &sourceStat)\n{\n#if defined __linux__ || defined __HAIKU__\n    auto sourceModifiedTime = sourceStat.st_mtim.tv_sec;\n#elif defined _WIN32\n    auto sourceModifiedTime = sourceStat.st_mtime;\n#else\n    auto sourceModifiedTime = sourceStat.st_mtimespec.tv_sec;\n#endif\n\n    struct stat soStat;\n    if (stat(soFile.c_str(), &soStat) == -1)\n    {\n        LOG_TRACE << \"Cannot determine modification time for:\" << soFile;\n        return true;\n    }\n\n#if defined __linux__ || defined __HAIKU__\n    auto soModifiedTime = soStat.st_mtim.tv_sec;\n#elif defined _WIN32\n    auto soModifiedTime = soStat.st_mtime;\n#else\n    auto soModifiedTime = soStat.st_mtimespec.tv_sec;\n#endif\n\n    return (sourceModifiedTime > soModifiedTime);\n}\n\nvoid *SharedLibManager::loadLib(const std::string &soFile, void *oldHld)\n{\n    if (oldHld)\n    {\n        if (dlclose(oldHld) == 0)\n        {\n            LOG_TRACE << \"Successfully closed dynamic library:\" << oldHld;\n        }\n        else\n        {\n            LOG_TRACE << dlerror();\n        }\n    }\n    auto Handle = dlopen(soFile.c_str(), RTLD_LAZY);\n    if (!Handle)\n    {\n        LOG_ERROR << \"load \" << soFile << \" error!\";\n        LOG_ERROR << dlerror();\n    }\n    else\n    {\n        LOG_TRACE << \"Successfully loaded library file \" << soFile;\n    }\n\n    return Handle;\n}\n\n// Path: lib/src/RedisClientSkipped.cc\n/**\n *\n *  RedisClientSkipped.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"drogon/nosql/RedisClient.h\"\n\nnamespace drogon\n{\nnamespace nosql\n{\nstd::shared_ptr<RedisClient> RedisClient::newRedisClient(\n    const trantor::InetAddress & /*serverAddress*/,\n    size_t /*numberOfConnections*/,\n    const std::string & /*password*/,\n    const unsigned int /*db*/,\n    const std::string & /*username*/)\n{\n    LOG_FATAL << \"Redis is not supported by drogon, please install the \"\n                 \"hiredis library first.\";\n    abort();\n}\n}  // namespace nosql\n}  // namespace drogon\n\n// Path: lib/src/Cookie.cc\n/**\n *\n *  Cookie.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/Cookie.h>\n#include <drogon/utils/Utilities.h>\n#include <trantor/utils/Logger.h>\nusing namespace drogon;\n\nstd::string Cookie::cookieString() const\n{\n    std::string ret = \"Set-Cookie: \";\n    // reserve space to reduce frequency allocation\n    ret.reserve(ret.size() + key_.size() + value_.size() + 30);\n    ret.append(key_).append(\"=\").append(value_).append(\"; \");\n    if (expiresDate_.microSecondsSinceEpoch() !=\n            (std::numeric_limits<int64_t>::max)() &&\n        expiresDate_.microSecondsSinceEpoch() >= 0)\n    {\n        ret.append(\"Expires=\")\n            .append(utils::getHttpFullDate(expiresDate_))\n            .append(\"; \");\n    }\n    if (maxAge_.has_value())\n    {\n        ret.append(\"Max-Age=\")\n            .append(std::to_string(maxAge_.value()))\n            .append(\"; \");\n    }\n    if (!domain_.empty())\n    {\n        ret.append(\"Domain=\").append(domain_).append(\"; \");\n    }\n    if (!path_.empty())\n    {\n        ret.append(\"Path=\").append(path_).append(\"; \");\n    }\n    if (sameSite_ != SameSite::kNull)\n    {\n        switch (sameSite_)\n        {\n            case SameSite::kLax:\n                ret.append(\"SameSite=Lax; \");\n                break;\n            case SameSite::kStrict:\n                ret.append(\"SameSite=Strict; \");\n                break;\n            case SameSite::kNone:\n                ret.append(\"SameSite=None; \");\n                // Cookies with SameSite=None must now also specify the Secure\n                // attribute (they require a secure context/HTTPS).\n                ret.append(\"Secure; \");\n                break;\n            default:\n                // Lax replaced None as the default value to ensure that users\n                // have reasonably robust defense against some CSRF attacks\n                ret.append(\"SameSite=Lax; \");\n        }\n    }\n    if (secure_ && sameSite_ != SameSite::kNone)\n    {\n        ret.append(\"Secure; \");\n    }\n    if (httpOnly_)\n    {\n        ret.append(\"HttpOnly; \");\n    }\n    ret.resize(ret.length() - 2);  // delete last semicolon\n    ret.append(\"\\r\\n\");\n    return ret;\n}\n\n// Path: lib/src/Utilities.cc\n/**\n *\n *  @file Utilities.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/utils/Utilities.h>\n#include <trantor/utils/Logger.h>\n#include <trantor/utils/Utilities.h>\n#include <drogon/config.h>\n#ifdef USE_BROTLI\n#include <brotli/decode.h>\n#include <brotli/encode.h>\n#endif\n#ifdef _WIN32\n#include <rpc.h>\n#include <direct.h>\n#include <io.h>\n#include <iomanip>\n#else\n#include <uuid.h>\n#include <unistd.h>\n#endif\n#include <zlib.h>\n#include <sstream>\n#include <string>\n#include <mutex>\n#include <algorithm>\n#include <array>\n#include <locale>\n#include <clocale>\n#include <cctype>\n#include <cstdlib>\n#include <filesystem>\n#include <stdio.h>\n#include <string.h>\n#include <sys/stat.h>\n#include <stdarg.h>\n\n#ifdef _WIN32\n\nchar *strptime(const char *s, const char *f, struct tm *tm)\n{\n    // std::get_time is defined such that its\n    // format parameters are the exact same as strptime.\n    std::istringstream input(s);\n    input.imbue(std::locale(setlocale(LC_ALL, nullptr)));\n    input >> std::get_time(tm, f);\n    if (input.fail())\n    {\n        return nullptr;\n    }\n    return (char *)(s + input.tellg());\n}\n\ntime_t timegm(struct tm *tm)\n{\n    struct tm my_tm;\n\n    memcpy(&my_tm, tm, sizeof(struct tm));\n\n    /* _mkgmtime() changes the value of the struct tm* you pass in, so\n     * use a copy\n     */\n    return _mkgmtime(&my_tm);\n}\n#endif\n\n#ifdef __HAIKU__\n// HACK: Haiku has a timegm implementation. But it is not exposed\nextern \"C\" time_t timegm(struct tm *tm);\n#endif\n\nnamespace drogon\n{\nnamespace utils\n{\nstatic const std::string base64Chars =\n    \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n    \"abcdefghijklmnopqrstuvwxyz\"\n    \"0123456789+/\";\n\nstatic const std::string urlBase64Chars =\n    \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n    \"abcdefghijklmnopqrstuvwxyz\"\n    \"0123456789-_\";\n\nclass Base64CharMap\n{\n  public:\n    Base64CharMap()\n    {\n        char index = 0;\n        for (int c = 'A'; c <= 'Z'; ++c)\n        {\n            charMap_[c] = index++;\n        }\n        for (int c = 'a'; c <= 'z'; ++c)\n        {\n            charMap_[c] = index++;\n        }\n        for (int c = '0'; c <= '9'; ++c)\n        {\n            charMap_[c] = index++;\n        }\n        charMap_[static_cast<int>('+')] = charMap_[static_cast<int>('-')] =\n            index++;\n        charMap_[static_cast<int>('/')] = charMap_[static_cast<int>('_')] =\n            index;\n        charMap_[0] = char(0xff);\n    }\n\n    char getIndex(const char c) const noexcept\n    {\n        return charMap_[static_cast<int>(c)];\n    }\n\n  private:\n    char charMap_[256]{0};\n};\n\nstatic const Base64CharMap base64CharMap;\n\nstatic inline bool isBase64(unsigned char c)\n{\n    if (isalnum(c))\n        return true;\n    switch (c)\n    {\n        case '+':\n        case '/':\n        case '-':\n        case '_':\n            return true;\n    }\n    return false;\n}\n\nbool isInteger(std::string_view str)\n{\n    for (auto c : str)\n        if (c < '0' || c > '9')\n            return false;\n    return true;\n}\n\nbool isBase64(std::string_view str)\n{\n    for (auto c : str)\n        if (!isBase64(c))\n            return false;\n    return true;\n}\n\nstd::string genRandomString(int length)\n{\n    static const char char_space[] =\n        \"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\";\n    static std::once_flag once;\n    static const size_t len = strlen(char_space);\n    static const int randMax = RAND_MAX - (RAND_MAX % len);\n    std::call_once(once, []() {\n        std::srand(static_cast<unsigned int>(time(nullptr)));\n    });\n\n    int i;\n    std::string str;\n    str.resize(length);\n\n    for (i = 0; i < length; ++i)\n    {\n        int x = std::rand();\n        while (x >= randMax)\n        {\n            x = std::rand();\n        }\n        x = (x % len);\n        str[i] = char_space[x];\n    }\n\n    return str;\n}\n\nstd::vector<char> hexToBinaryVector(const char *ptr, size_t length)\n{\n    assert(length % 2 == 0);\n    std::vector<char> ret(length / 2, '\\0');\n    for (size_t i = 0; i < ret.size(); ++i)\n    {\n        auto p = i * 2;\n        char c1 = ptr[p];\n        if (c1 >= '0' && c1 <= '9')\n        {\n            c1 -= '0';\n        }\n        else if (c1 >= 'a' && c1 <= 'f')\n        {\n            c1 -= 'a';\n            c1 += 10;\n        }\n        else if (c1 >= 'A' && c1 <= 'F')\n        {\n            c1 -= 'A';\n            c1 += 10;\n        }\n        else\n        {\n            return std::vector<char>();\n        }\n        char c2 = ptr[p + 1];\n        if (c2 >= '0' && c2 <= '9')\n        {\n            c2 -= '0';\n        }\n        else if (c2 >= 'a' && c2 <= 'f')\n        {\n            c2 -= 'a';\n            c2 += 10;\n        }\n        else if (c2 >= 'A' && c2 <= 'F')\n        {\n            c2 -= 'A';\n            c2 += 10;\n        }\n        else\n        {\n            return std::vector<char>();\n        }\n        ret[i] = c1 * 16 + c2;\n    }\n    return ret;\n}\n\nstd::string hexToBinaryString(const char *ptr, size_t length)\n{\n    assert(length % 2 == 0);\n    std::string ret(length / 2, '\\0');\n    for (size_t i = 0; i < ret.length(); ++i)\n    {\n        auto p = i * 2;\n        char c1 = ptr[p];\n        if (c1 >= '0' && c1 <= '9')\n        {\n            c1 -= '0';\n        }\n        else if (c1 >= 'a' && c1 <= 'f')\n        {\n            c1 -= 'a';\n            c1 += 10;\n        }\n        else if (c1 >= 'A' && c1 <= 'F')\n        {\n            c1 -= 'A';\n            c1 += 10;\n        }\n        else\n        {\n            return \"\";\n        }\n        char c2 = ptr[p + 1];\n        if (c2 >= '0' && c2 <= '9')\n        {\n            c2 -= '0';\n        }\n        else if (c2 >= 'a' && c2 <= 'f')\n        {\n            c2 -= 'a';\n            c2 += 10;\n        }\n        else if (c2 >= 'A' && c2 <= 'F')\n        {\n            c2 -= 'A';\n            c2 += 10;\n        }\n        else\n        {\n            return \"\";\n        }\n        ret[i] = c1 * 16 + c2;\n    }\n    return ret;\n}\n\nDROGON_EXPORT void binaryStringToHex(const char *ptr,\n                                     size_t length,\n                                     char *out,\n                                     bool lowerCase)\n{\n    for (size_t i = 0; i < length; ++i)\n    {\n        int value = (ptr[i] & 0xf0) >> 4;\n        if (value < 10)\n        {\n            out[i * 2] = char(value + 48);\n        }\n        else\n        {\n            if (!lowerCase)\n            {\n                out[i * 2] = char(value + 55);\n            }\n            else\n            {\n                out[i * 2] = char(value + 87);\n            }\n        }\n\n        value = (ptr[i] & 0x0f);\n        if (value < 10)\n        {\n            out[i * 2 + 1] = char(value + 48);\n        }\n        else\n        {\n            if (!lowerCase)\n            {\n                out[i * 2 + 1] = char(value + 55);\n            }\n            else\n            {\n                out[i * 2 + 1] = char(value + 87);\n            }\n        }\n    }\n}\n\nstd::string binaryStringToHex(const unsigned char *ptr,\n                              size_t length,\n                              bool lowercase)\n{\n    std::string idString(length * 2, '\\0');\n    binaryStringToHex((const char *)ptr, length, &idString[0], lowercase);\n    return idString;\n}\n\n\nstd::set<std::string> splitStringToSet(const std::string &str,\n                                       const std::string &separator)\n{\n    std::set<std::string> ret;\n    std::string::size_type pos1, pos2;\n    pos2 = 0;\n    pos1 = str.find(separator);\n    while (pos1 != std::string::npos)\n    {\n        if (pos1 != 0)\n        {\n            std::string item = str.substr(pos2, pos1 - pos2);\n            ret.insert(item);\n        }\n        pos2 = pos1 + separator.length();\n        while (pos2 < str.length() &&\n               str.substr(pos2, separator.length()) == separator)\n            pos2 += separator.length();\n        pos1 = str.find(separator, pos2);\n    }\n    if (pos2 < str.length())\n        ret.insert(str.substr(pos2));\n    return ret;\n}\n\ninline std::string createUuidString(const char *str, size_t len, bool lowercase)\n{\n    assert(len == 16);\n    std::string uuid(36, '\\0');\n    binaryStringToHex(str, 4, &uuid[0], lowercase);\n    uuid[8] = '-';\n    binaryStringToHex(str + 4, 2, &uuid[9], lowercase);\n    uuid[13] = '-';\n    binaryStringToHex(str + 6, 2, &uuid[14], lowercase);\n    uuid[18] = '-';\n    binaryStringToHex(str + 8, 2, &uuid[19], lowercase);\n    uuid[23] = '-';\n    binaryStringToHex(str + 10, 6, &uuid[24], lowercase);\n    return uuid;\n}\n\nstd::string getUuid(bool lowercase)\n{\n#if USE_OSSP_UUID\n    uuid_t *uuid;\n    uuid_create(&uuid);\n    uuid_make(uuid, UUID_MAKE_V4);\n    char *str{nullptr};\n    size_t len{0};\n    uuid_export(uuid, UUID_FMT_BIN, &str, &len);\n    uuid_destroy(uuid);\n    auto ret = createUuidString(str, len, lowercase);\n    free(str);\n    return ret;\n#elif defined __FreeBSD__ || defined __OpenBSD__\n    uuid_t *uuid = new uuid_t;\n    char *binstr = (char *)malloc(16);\n#if defined __FreeBSD__\n    uuidgen(uuid, 1);\n#else\n    uint32_t status;\n    uuid_create(uuid, &status);\n#endif\n#if _BYTE_ORDER == _LITTLE_ENDIAN\n    uuid_enc_le(binstr, uuid);\n#else  /* _BYTE_ORDER != _LITTLE_ENDIAN */\n    uuid_enc_be(binstr, uuid);\n#endif /* _BYTE_ORDER == _LITTLE_ENDIAN */\n    delete uuid;\n    auto ret = createUuidString(binstr, 16, lowercase);\n    free(binstr);\n    return ret;\n#elif defined _WIN32\n    uuid_t uu;\n    UuidCreate(&uu);\n    char tempStr[100];\n    auto len = snprintf(tempStr,\n                        sizeof(tempStr),\n                        \"%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x\",\n                        uu.Data1,\n                        uu.Data2,\n                        uu.Data3,\n                        uu.Data4[0],\n                        uu.Data4[1],\n                        uu.Data4[2],\n                        uu.Data4[3],\n                        uu.Data4[4],\n                        uu.Data4[5],\n                        uu.Data4[6],\n                        uu.Data4[7]);\n    return std::string{tempStr, static_cast<size_t>(len)};\n#else\n    uuid_t uu;\n    uuid_generate(uu);\n    auto uuid = createUuidString((const char *)uu, 16, lowercase);\n    return uuid;\n#endif\n}\n\nstd::string base64Encode(const unsigned char *bytes_to_encode,\n                         size_t in_len,\n                         bool url_safe,\n                         bool padded)\n{\n    std::string ret;\n    ret.reserve(base64EncodedLength(in_len, padded));\n    int i = 0;\n    unsigned char char_array_3[3];\n    unsigned char char_array_4[4];\n\n    const std::string &charSet = url_safe ? urlBase64Chars : base64Chars;\n\n    while (in_len--)\n    {\n        char_array_3[i++] = *(bytes_to_encode++);\n        if (i == 3)\n        {\n            char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;\n            char_array_4[1] = ((char_array_3[0] & 0x03) << 4) +\n                              ((char_array_3[1] & 0xf0) >> 4);\n            char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) +\n                              ((char_array_3[2] & 0xc0) >> 6);\n            char_array_4[3] = char_array_3[2] & 0x3f;\n\n            for (i = 0; (i < 4); ++i)\n                ret += charSet[char_array_4[i]];\n            i = 0;\n        }\n    }\n\n    if (i)\n    {\n        for (int j = i; j < 3; ++j)\n            char_array_3[j] = '\\0';\n\n        char_array_4[0] = (char_array_3[0] & 0xfc) >> 2;\n        char_array_4[1] =\n            ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4);\n        char_array_4[2] =\n            ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6);\n        char_array_4[3] = char_array_3[2] & 0x3f;\n\n        for (int j = 0; (j <= i); ++j)\n            ret += charSet[char_array_4[j]];\n\n        if (padded)\n            while ((++i < 4))\n                ret += '=';\n    }\n    return ret;\n}\n\nstd::vector<char> base64DecodeToVector(std::string_view encoded_string)\n{\n    auto in_len = encoded_string.size();\n    int i = 0;\n    int in_{0};\n    char char_array_4[4], char_array_3[3];\n    std::vector<char> ret;\n    ret.reserve(base64DecodedLength(in_len));\n\n    while (in_len-- && (encoded_string[in_] != '='))\n    {\n        if (!isBase64(encoded_string[in_]))\n        {\n            ++in_;\n            continue;\n        }\n\n        char_array_4[i++] = encoded_string[in_];\n        ++in_;\n        if (i == 4)\n        {\n            for (i = 0; i < 4; ++i)\n            {\n                char_array_4[i] = base64CharMap.getIndex(char_array_4[i]);\n            }\n\n            char_array_3[0] =\n                (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) +\n                              ((char_array_4[2] & 0x3c) >> 2);\n            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n            for (i = 0; (i < 3); ++i)\n                ret.push_back(char_array_3[i]);\n            i = 0;\n        }\n    }\n\n    if (i)\n    {\n        for (int j = i; j < 4; ++j)\n            char_array_4[j] = 0;\n\n        for (int j = 0; j < 4; ++j)\n        {\n            char_array_4[j] = base64CharMap.getIndex(char_array_4[j]);\n        }\n\n        char_array_3[0] =\n            (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n        char_array_3[1] =\n            ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);\n        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n        --i;\n        for (int j = 0; (j < i); ++j)\n            ret.push_back(char_array_3[j]);\n    }\n\n    return ret;\n}\n\nstd::string base64Decode(std::string_view encoded_string)\n{\n    auto in_len = encoded_string.size();\n    int i = 0;\n    int in_{0};\n    unsigned char char_array_4[4], char_array_3[3];\n    std::string ret;\n    ret.reserve(base64DecodedLength(in_len));\n\n    while (in_len-- && (encoded_string[in_] != '='))\n    {\n        if (!isBase64(encoded_string[in_]))\n        {\n            ++in_;\n            continue;\n        }\n\n        char_array_4[i++] = encoded_string[in_];\n        ++in_;\n        if (i == 4)\n        {\n            for (i = 0; i < 4; ++i)\n            {\n                char_array_4[i] = base64CharMap.getIndex(char_array_4[i]);\n            }\n            char_array_3[0] =\n                (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n            char_array_3[1] = ((char_array_4[1] & 0xf) << 4) +\n                              ((char_array_4[2] & 0x3c) >> 2);\n            char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n            for (i = 0; (i < 3); ++i)\n                ret += char_array_3[i];\n            i = 0;\n        }\n    }\n\n    if (i)\n    {\n        for (int j = i; j < 4; ++j)\n            char_array_4[j] = 0;\n\n        for (int j = 0; j < 4; ++j)\n        {\n            char_array_4[j] = base64CharMap.getIndex(char_array_4[j]);\n        }\n\n        char_array_3[0] =\n            (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4);\n        char_array_3[1] =\n            ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2);\n        char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3];\n\n        --i;\n        for (int j = 0; (j < i); ++j)\n            ret += char_array_3[j];\n    }\n\n    return ret;\n}\n\nstatic std::string charToHex(char c)\n{\n    std::string result;\n    char first, second;\n\n    first = (c & 0xF0) / 16;\n    first += first > 9 ? 'A' - 10 : '0';\n    second = c & 0x0F;\n    second += second > 9 ? 'A' - 10 : '0';\n\n    result.append(1, first);\n    result.append(1, second);\n\n    return result;\n}\n\nstd::string urlEncodeComponent(const std::string &src)\n{\n    std::string result;\n    std::string::const_iterator iter;\n\n    for (iter = src.begin(); iter != src.end(); ++iter)\n    {\n        switch (*iter)\n        {\n            case ' ':\n                result.append(1, '+');\n                break;\n            // alnum\n            case 'A':\n            case 'B':\n            case 'C':\n            case 'D':\n            case 'E':\n            case 'F':\n            case 'G':\n            case 'H':\n            case 'I':\n            case 'J':\n            case 'K':\n            case 'L':\n            case 'M':\n            case 'N':\n            case 'O':\n            case 'P':\n            case 'Q':\n            case 'R':\n            case 'S':\n            case 'T':\n            case 'U':\n            case 'V':\n            case 'W':\n            case 'X':\n            case 'Y':\n            case 'Z':\n            case 'a':\n            case 'b':\n            case 'c':\n            case 'd':\n            case 'e':\n            case 'f':\n            case 'g':\n            case 'h':\n            case 'i':\n            case 'j':\n            case 'k':\n            case 'l':\n            case 'm':\n            case 'n':\n            case 'o':\n            case 'p':\n            case 'q':\n            case 'r':\n            case 's':\n            case 't':\n            case 'u':\n            case 'v':\n            case 'w':\n            case 'x':\n            case 'y':\n            case 'z':\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            // mark\n            case '-':\n            case '_':\n            case '.':\n            case '!':\n            case '~':\n            case '*':\n            case '(':\n            case ')':\n                result.append(1, *iter);\n                break;\n            // escape\n            default:\n                result.append(1, '%');\n                result.append(charToHex(*iter));\n                break;\n        }\n    }\n\n    return result;\n}\n\nstd::string urlEncode(const std::string &src)\n{\n    std::string result;\n    std::string::const_iterator iter;\n\n    for (iter = src.begin(); iter != src.end(); ++iter)\n    {\n        switch (*iter)\n        {\n            case ' ':\n                result.append(1, '+');\n                break;\n            // alnum\n            case 'A':\n            case 'B':\n            case 'C':\n            case 'D':\n            case 'E':\n            case 'F':\n            case 'G':\n            case 'H':\n            case 'I':\n            case 'J':\n            case 'K':\n            case 'L':\n            case 'M':\n            case 'N':\n            case 'O':\n            case 'P':\n            case 'Q':\n            case 'R':\n            case 'S':\n            case 'T':\n            case 'U':\n            case 'V':\n            case 'W':\n            case 'X':\n            case 'Y':\n            case 'Z':\n            case 'a':\n            case 'b':\n            case 'c':\n            case 'd':\n            case 'e':\n            case 'f':\n            case 'g':\n            case 'h':\n            case 'i':\n            case 'j':\n            case 'k':\n            case 'l':\n            case 'm':\n            case 'n':\n            case 'o':\n            case 'p':\n            case 'q':\n            case 'r':\n            case 's':\n            case 't':\n            case 'u':\n            case 'v':\n            case 'w':\n            case 'x':\n            case 'y':\n            case 'z':\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '6':\n            case '7':\n            case '8':\n            case '9':\n            // mark\n            case '-':\n            case '_':\n            case '.':\n            case '!':\n            case '~':\n            case '*':\n            case '\\'':\n            case '(':\n            case ')':\n            case '&':\n            case '=':\n            case '/':\n            case '\\\\':\n            case '?':\n                result.append(1, *iter);\n                break;\n            // escape\n            default:\n                result.append(1, '%');\n                result.append(charToHex(*iter));\n                break;\n        }\n    }\n\n    return result;\n}\n\nbool needUrlDecoding(const char *begin, const char *end)\n{\n    return std::find_if(begin, end, [](const char c) {\n               return c == '+' || c == '%';\n           }) != end;\n}\n\nstd::string urlDecode(const char *begin, const char *end)\n{\n    std::string result;\n    size_t len = end - begin;\n    result.reserve(len * 2);\n    int hex = 0;\n    for (size_t i = 0; i < len; ++i)\n    {\n        switch (begin[i])\n        {\n            case '+':\n                result += ' ';\n                break;\n            case '%':\n                if ((i + 2) < len && isxdigit(begin[i + 1]) &&\n                    isxdigit(begin[i + 2]))\n                {\n                    unsigned int x1 = begin[i + 1];\n                    if (x1 >= '0' && x1 <= '9')\n                    {\n                        x1 -= '0';\n                    }\n                    else if (x1 >= 'a' && x1 <= 'f')\n                    {\n                        x1 = x1 - 'a' + 10;\n                    }\n                    else if (x1 >= 'A' && x1 <= 'F')\n                    {\n                        x1 = x1 - 'A' + 10;\n                    }\n                    unsigned int x2 = begin[i + 2];\n                    if (x2 >= '0' && x2 <= '9')\n                    {\n                        x2 -= '0';\n                    }\n                    else if (x2 >= 'a' && x2 <= 'f')\n                    {\n                        x2 = x2 - 'a' + 10;\n                    }\n                    else if (x2 >= 'A' && x2 <= 'F')\n                    {\n                        x2 = x2 - 'A' + 10;\n                    }\n                    hex = x1 * 16 + x2;\n\n                    result += char(hex);\n                    i += 2;\n                }\n                else\n                {\n                    result += '%';\n                }\n                break;\n            default:\n                result += begin[i];\n                break;\n        }\n    }\n    return result;\n}\n\n/* Compress gzip data */\nstd::string gzipCompress(const char *data, const size_t ndata)\n{\n    z_stream strm = {nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     0,\n                     0,\n                     0};\n    if (data && ndata > 0)\n    {\n        if (deflateInit2(&strm,\n                         Z_DEFAULT_COMPRESSION,\n                         Z_DEFLATED,\n                         MAX_WBITS + 16,\n                         8,\n                         Z_DEFAULT_STRATEGY) != Z_OK)\n        {\n            LOG_ERROR << \"deflateInit2 error!\";\n            return std::string{};\n        }\n        std::string outstr;\n        outstr.resize(compressBound(static_cast<uLong>(ndata)));\n        strm.next_in = (Bytef *)data;\n        strm.avail_in = static_cast<uInt>(ndata);\n        int ret;\n        do\n        {\n            if (strm.total_out >= outstr.size())\n            {\n                outstr.resize(strm.total_out * 2);\n            }\n            assert(outstr.size() >= strm.total_out);\n            strm.avail_out = static_cast<uInt>(outstr.size() - strm.total_out);\n            strm.next_out = (Bytef *)outstr.data() + strm.total_out;\n            ret = deflate(&strm, Z_FINISH); /* no bad return value */\n            if (ret == Z_STREAM_ERROR)\n            {\n                (void)deflateEnd(&strm);\n                return std::string{};\n            }\n        } while (strm.avail_out == 0);\n        assert(strm.avail_in == 0);\n        assert(ret == Z_STREAM_END); /* stream will be complete */\n        outstr.resize(strm.total_out);\n        /* clean up and return */\n        (void)deflateEnd(&strm);\n        return outstr;\n    }\n    return std::string{};\n}\n\nstd::string gzipDecompress(const char *data, const size_t ndata)\n{\n    if (ndata == 0)\n        return std::string(data, ndata);\n\n    auto full_length = ndata;\n\n    auto decompressed = std::string(full_length * 2, 0);\n    bool done = false;\n\n    z_stream strm = {nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     0,\n                     0,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     nullptr,\n                     0,\n                     0,\n                     0};\n    strm.next_in = (Bytef *)data;\n    strm.avail_in = static_cast<uInt>(ndata);\n    strm.total_out = 0;\n    strm.zalloc = Z_NULL;\n    strm.zfree = Z_NULL;\n    if (inflateInit2(&strm, (15 + 32)) != Z_OK)\n    {\n        LOG_ERROR << \"inflateInit2 error!\";\n        return std::string{};\n    }\n    while (!done)\n    {\n        // Make sure we have enough room and reset the lengths.\n        if (strm.total_out >= decompressed.length())\n        {\n            decompressed.resize(decompressed.length() * 2);\n        }\n        strm.next_out = (Bytef *)decompressed.data() + strm.total_out;\n        strm.avail_out =\n            static_cast<uInt>(decompressed.length() - strm.total_out);\n        // Inflate another chunk.\n        int status = inflate(&strm, Z_SYNC_FLUSH);\n        if (status == Z_STREAM_END)\n        {\n            done = true;\n        }\n        else if (status != Z_OK)\n        {\n            break;\n        }\n    }\n    if (inflateEnd(&strm) != Z_OK)\n        return std::string{};\n    // Set real length.\n    if (done)\n    {\n        decompressed.resize(strm.total_out);\n        return decompressed;\n    }\n    else\n    {\n        return std::string{};\n    }\n}\n\nchar *getHttpFullDate(const trantor::Date &date)\n{\n    static thread_local int64_t lastSecond = 0;\n    static thread_local char lastTimeString[128] = {0};\n    auto nowSecond = date.microSecondsSinceEpoch() / MICRO_SECONDS_PRE_SEC;\n    if (nowSecond == lastSecond)\n    {\n        return lastTimeString;\n    }\n    lastSecond = nowSecond;\n    date.toCustomedFormattedString(\"%a, %d %b %Y %H:%M:%S GMT\",\n                                   lastTimeString,\n                                   sizeof(lastTimeString));\n    return lastTimeString;\n}\n\ntrantor::Date getHttpDate(const std::string &httpFullDateString)\n{\n    static const std::array<const char *, 4> formats = {\n        // RFC822 (default)\n        \"%a, %d %b %Y %H:%M:%S\",\n        // RFC 850 (deprecated)\n        \"%a, %d-%b-%y %H:%M:%S\",\n        // ansi asctime format\n        \"%a %b %d %H:%M:%S %Y\",\n        // weird RFC 850-hybrid thing that reddit uses\n        \"%a, %d-%b-%Y %H:%M:%S\",\n    };\n    struct tm tmptm;\n    for (const char *format : formats)\n    {\n        if (strptime(httpFullDateString.c_str(), format, &tmptm) != NULL)\n        {\n            auto epoch = timegm(&tmptm);\n            return trantor::Date(epoch * MICRO_SECONDS_PRE_SEC);\n        }\n    }\n    LOG_WARN << \"invalid datetime format: '\" << httpFullDateString << \"'\";\n    return trantor::Date((std::numeric_limits<int64_t>::max)());\n}\n\nstd::string formattedString(const char *format, ...)\n{\n    std::string strBuffer(128, 0);\n    va_list ap, backup_ap;\n    va_start(ap, format);\n    va_copy(backup_ap, ap);\n    auto result = vsnprintf((char *)strBuffer.data(),\n                            strBuffer.size(),\n                            format,\n                            backup_ap);\n    va_end(backup_ap);\n    if ((result >= 0) && ((std::string::size_type)result < strBuffer.size()))\n    {\n        strBuffer.resize(result);\n    }\n    else\n    {\n        while (true)\n        {\n            if (result < 0)\n            {\n                // Older snprintf() behavior. Just try doubling the buffer size\n                strBuffer.resize(strBuffer.size() * 2);\n            }\n            else\n            {\n                strBuffer.resize(result + 1);\n            }\n\n            va_copy(backup_ap, ap);\n            auto result = vsnprintf((char *)strBuffer.data(),\n                                    strBuffer.size(),\n                                    format,\n                                    backup_ap);\n            va_end(backup_ap);\n\n            if ((result >= 0) &&\n                ((std::string::size_type)result < strBuffer.size()))\n            {\n                strBuffer.resize(result);\n                break;\n            }\n        }\n    }\n    va_end(ap);\n    return strBuffer;\n}\n\nint createPath(const std::string &path)\n{\n    if (path.empty())\n        return 0;\n    auto osPath{toNativePath(path)};\n    if (osPath.back() != std::filesystem::path::preferred_separator)\n        osPath.push_back(std::filesystem::path::preferred_separator);\n    std::filesystem::path fsPath(osPath);\n    std::error_code err;\n    std::filesystem::create_directories(fsPath, err);\n    if (err)\n    {\n        LOG_ERROR << \"Error \" << err.value() << \" creating path \" << osPath\n                  << \": \" << err.message();\n        return -1;\n    }\n    return 0;\n}\n#ifdef USE_BROTLI\nstd::string brotliCompress(const char *data, const size_t ndata)\n{\n    std::string ret;\n    if (ndata == 0)\n        return ret;\n    ret.resize(BrotliEncoderMaxCompressedSize(ndata));\n    size_t encodedSize{ret.size()};\n    auto r = BrotliEncoderCompress(5,\n                                   BROTLI_DEFAULT_WINDOW,\n                                   BROTLI_DEFAULT_MODE,\n                                   ndata,\n                                   (const uint8_t *)(data),\n                                   &encodedSize,\n                                   (uint8_t *)(ret.data()));\n    if (r == BROTLI_FALSE)\n        ret.resize(0);\n    else\n        ret.resize(encodedSize);\n    return ret;\n}\n\nstd::string brotliDecompress(const char *data, const size_t ndata)\n{\n    if (ndata == 0)\n        return std::string(data, ndata);\n\n    size_t availableIn = ndata;\n    auto nextIn = (const uint8_t *)(data);\n    auto decompressed = std::string(availableIn * 3, 0);\n    size_t availableOut = decompressed.size();\n    auto nextOut = (uint8_t *)(decompressed.data());\n    size_t totalOut{0};\n    bool done = false;\n    auto s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);\n    while (!done)\n    {\n        auto result = BrotliDecoderDecompressStream(\n            s, &availableIn, &nextIn, &availableOut, &nextOut, &totalOut);\n        if (result == BROTLI_DECODER_RESULT_SUCCESS)\n        {\n            decompressed.resize(totalOut);\n            done = true;\n        }\n        else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT)\n        {\n            assert(totalOut == decompressed.size());\n            decompressed.resize(totalOut * 2);\n            nextOut = (uint8_t *)(decompressed.data() + totalOut);\n            availableOut = totalOut;\n        }\n        else\n        {\n            decompressed.resize(0);\n            done = true;\n        }\n    }\n    BrotliDecoderDestroyInstance(s);\n    return decompressed;\n}\n#else\nstd::string brotliCompress(const char * /*data*/, const size_t /*ndata*/)\n{\n    LOG_ERROR << \"If you do not have the brotli package installed, you cannot \"\n                 \"use brotliCompress()\";\n    abort();\n}\n\nstd::string brotliDecompress(const char * /*data*/, const size_t /*ndata*/)\n{\n    LOG_ERROR << \"If you do not have the brotli package installed, you cannot \"\n                 \"use brotliDecompress()\";\n    abort();\n}\n#endif\n\nstd::string getMd5(const char *data, const size_t dataLen)\n{\n    return trantor::utils::toHexString(trantor::utils::md5(data, dataLen));\n}\n\nstd::string getSha1(const char *data, const size_t dataLen)\n{\n    return trantor::utils::toHexString(trantor::utils::sha1(data, dataLen));\n}\n\nstd::string getSha256(const char *data, const size_t dataLen)\n{\n    return trantor::utils::toHexString(trantor::utils::sha256(data, dataLen));\n}\n\nstd::string getSha3(const char *data, const size_t dataLen)\n{\n    return trantor::utils::toHexString(trantor::utils::sha3(data, dataLen));\n}\n\nstd::string getBlake2b(const char *data, const size_t dataLen)\n{\n    return trantor::utils::toHexString(trantor::utils::blake2b(data, dataLen));\n}\n\nvoid replaceAll(std::string &s, const std::string &from, const std::string &to)\n{\n    size_t pos = 0;\n    while ((pos = s.find(from, pos)) != std::string::npos)\n    {\n        s.replace(pos, from.size(), to);\n        pos += to.size();\n    }\n}\n\nbool supportsTls() noexcept\n{\n    return trantor::utils::tlsBackend() != \"None\";\n}\n\nbool secureRandomBytes(void *ptr, size_t size)\n{\n    return trantor::utils::secureRandomBytes(ptr, size);\n}\n\nstd::string secureRandomString(size_t size)\n{\n    if (size == 0)\n        return std::string();\n\n    std::string ret(size, 0);\n    const std::string_view chars =\n        \"0123456789\"\n        \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n        \"abcdefghijklmnopqrstuvwxyz\"\n        \"+-\";\n    assert(chars.size() == 64);\n\n    // batch up to 32 bytes of random data for efficiency. Calling\n    // secureRandomBytes can be expensive.\n    auto randByte = []() {\n        thread_local trantor::utils::Hash256 hash;\n        thread_local size_t i = 0;\n        if (i == 0)\n        {\n            bool ok = trantor::utils::secureRandomBytes(&hash, sizeof(hash));\n            if (!ok)\n                throw std::runtime_error(\n                    \"Failed to generate random bytes for secureRandomString\");\n        }\n        unsigned char *hashBytes = reinterpret_cast<unsigned char *>(&hash);\n        auto ret = hashBytes[i];\n        i = (i + 1) % sizeof(hash);\n        return ret;\n    };\n\n    for (size_t i = 0; i < size; ++i)\n        ret[i] = chars[randByte() % 64];\n    return ret;\n}\n\nnamespace internal\n{\nconst size_t fixedRandomNumber = []() {\n    size_t res;\n    utils::secureRandomBytes(&res, sizeof(res));\n    return res;\n}();\n}\n\n}  // namespace utils\n}  // namespace drogon\n\n// Path: lib/src/SpinLock.h\n/**\n *  SpinLock.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n#include <atomic>\n...\n", "needle_token_start": 7115, "needle_token_end": 7355, "code_context_ntokens": 16163, "output": ["```cpp\nstd::set<std::string> splitStringToSet(const std::string &str, const std::string &separator)\n{\n    std::set<std::string> ret;\n    std::string::size_type pos1, pos2;\n    pos2 = 0;\n    pos1 = str.find(separator);\n    while (pos1 != std::string::npos)\n    {\n        if (pos1 != 0)\n        {\n            std::string item = str.substr(pos2, pos1 - pos2);\n            ret.insert(item);\n        }\n        pos2 = pos1 + separator.length();\n        while (pos2 < str.length() &&\n               str.substr(pos2, separator.length()) == separator)\n            pos2 += separator.length();\n        pos1 = str.find(separator, pos2);\n    }\n    if (pos2 < str.length())\n        ret.insert(str.substr(pos2));\n    return ret;\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "HttpMessageStringViewBody", "language": "cpp", "path": "lib/src/HttpMessageBody.h", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To manage a view of a constant segment of data, representing the body of an HTTP message without owning the data itself.\n2. **Input**: A pointer to the beginning of the data segment and the length of the segment.\n3. **Output**: Provides access to the data segment and its properties such as length, but does not modify the original data.\n4. **Procedure**: Initializes with a data pointer and length, setting the internal type to indicate it handles a view of data. Provides methods to access the data and its length, but does not support modifying the data.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/HttpAppFrameworkImpl.h\n/**\n *\n *  @file HttpAppFrameworkImpl.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <drogon/HttpAppFramework.h>\n#include <drogon/config.h>\n#include <json/json.h>\n#include <functional>\n#include <memory>\n#include <string>\n#include <vector>\n#include \"SessionManager.h\"\n#include \"drogon/utils/Utilities.h\"\n#include \"impl_forwards.h\"\n\nnamespace trantor\n{\nclass EventLoopThreadPool;\n}\n\nnamespace drogon\n{\nHttpResponsePtr defaultErrorHandler(HttpStatusCode code,\n                                    const HttpRequestPtr &req);\nvoid defaultExceptionHandler(const std::exception &,\n                             const HttpRequestPtr &,\n                             std::function<void(const HttpResponsePtr &)> &&);\n\nstruct InitBeforeMainFunction\n{\n    explicit InitBeforeMainFunction(const std::function<void()> &func)\n    {\n        func();\n    }\n};\n\nclass HttpAppFrameworkImpl final : public HttpAppFramework\n{\n  public:\n    HttpAppFrameworkImpl();\n\n    const Json::Value &getCustomConfig() const override\n    {\n        return jsonConfig_[\"custom_config\"];\n    }\n\n    PluginBase *getPlugin(const std::string &name) override;\n    std::shared_ptr<PluginBase> getSharedPlugin(\n        const std::string &name) override;\n    void addPlugins(const Json::Value &configs) override;\n    void addPlugin(const std::string &name,\n                   const std::vector<std::string> &dependencies,\n                   const Json::Value &config) override;\n    HttpAppFramework &addListener(\n        const std::string &ip,\n        uint16_t port,\n        bool useSSL,\n        const std::string &certFile,\n        const std::string &keyFile,\n        bool useOldTLS,\n        const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n        override;\n    HttpAppFramework &setThreadNum(size_t threadNum) override;\n\n    size_t getThreadNum() const override\n    {\n        return threadNum_;\n    }\n\n    HttpAppFramework &setSSLConfigCommands(\n        const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n        override;\n    HttpAppFramework &setSSLFiles(const std::string &certPath,\n                                  const std::string &keyPath) override;\n    void run() override;\n    HttpAppFramework &registerWebSocketController(\n        const std::string &pathName,\n        const std::string &ctrlName,\n        const std::vector<internal::HttpConstraint> &filtersAndMethods)\n        override;\n    HttpAppFramework &registerHttpSimpleController(\n        const std::string &pathName,\n        const std::string &ctrlName,\n        const std::vector<internal::HttpConstraint> &filtersAndMethods)\n        override;\n\n    HttpAppFramework &setCustom404Page(const HttpResponsePtr &resp,\n                                       bool set404) override\n    {\n        if (set404)\n        {\n            resp->setStatusCode(k404NotFound);\n        }\n        custom404_ = resp;\n        return *this;\n    }\n\n    HttpAppFramework &setCustomErrorHandler(\n        std::function<HttpResponsePtr(HttpStatusCode,\n                                      const HttpRequestPtr &req)>\n            &&resp_generator) override;\n\n    const HttpResponsePtr &getCustom404Page();\n\n    void forward(const HttpRequestPtr &req,\n                 std::function<void(const HttpResponsePtr &)> &&callback,\n                 const std::string &hostString,\n                 double timeout) override;\n\n    void forward(const HttpRequestImplPtr &req,\n                 std::function<void(const HttpResponsePtr &)> &&callback,\n                 const std::string &hostString,\n                 double timeout = 0);\n\n    HttpAppFramework &registerBeginningAdvice(\n        const std::function<void()> &advice) override\n    {\n        beginningAdvices_.emplace_back(advice);\n        return *this;\n    }\n\n    HttpAppFramework &registerNewConnectionAdvice(\n        const std::function<bool(const trantor::InetAddress &,\n                                 const trantor::InetAddress &)> &advice)\n        override;\n\n    HttpAppFramework &registerHttpResponseCreationAdvice(\n        const std::function<void(const HttpResponsePtr &)> &advice) override;\n\n    HttpAppFramework &registerSyncAdvice(\n        const std::function<HttpResponsePtr(const HttpRequestPtr &)> &advice)\n        override;\n\n    HttpAppFramework &registerPreRoutingAdvice(\n        const std::function<void(const HttpRequestPtr &,\n                                 AdviceCallback &&,\n                                 AdviceChainCallback &&)> &advice) override;\n\n    HttpAppFramework &registerPostRoutingAdvice(\n        const std::function<void(const HttpRequestPtr &,\n                                 AdviceCallback &&,\n                                 AdviceChainCallback &&)> &advice) override;\n\n    HttpAppFramework &registerPreHandlingAdvice(\n        const std::function<void(const HttpRequestPtr &,\n                                 AdviceCallback &&,\n                                 AdviceChainCallback &&)> &advice) override;\n\n    HttpAppFramework &registerPreRoutingAdvice(\n        const std::function<void(const HttpRequestPtr &)> &advice) override;\n\n    HttpAppFramework &registerPostRoutingAdvice(\n        const std::function<void(const HttpRequestPtr &)> &advice) override;\n\n    HttpAppFramework &registerPreHandlingAdvice(\n        const std::function<void(const HttpRequestPtr &)> &advice) override;\n\n    HttpAppFramework &registerPostHandlingAdvice(\n        const std::function<void(const HttpRequestPtr &,\n                                 const HttpResponsePtr &)> &advice) override;\n\n    HttpAppFramework &registerPreSendingAdvice(\n        const std::function<void(const HttpRequestPtr &,\n                                 const HttpResponsePtr &)> &advice) override;\n\n    HttpAppFramework &setDefaultHandler(DefaultHandler handler) override;\n\n    HttpAppFramework &setupFileLogger() override;\n\n    HttpAppFramework &enableSession(\n        const size_t timeout,\n        Cookie::SameSite sameSite = Cookie::SameSite::kNull,\n        const std::string &cookieKey = \"JSESSIONID\",\n        int maxAge = -1,\n        SessionManager::IdGeneratorCallback idGeneratorCallback =\n            nullptr) override\n    {\n        useSession_ = true;\n        sessionTimeout_ = timeout;\n        sessionSameSite_ = sameSite;\n        sessionCookieKey_ = cookieKey;\n        sessionMaxAge_ = maxAge;\n        return setSessionIdGenerator(idGeneratorCallback);\n    }\n\n    HttpAppFramework &setSessionIdGenerator(\n        SessionManager::IdGeneratorCallback idGeneratorCallback = nullptr)\n    {\n        sessionIdGeneratorCallback_ =\n            idGeneratorCallback ? idGeneratorCallback\n                                : []() { return utils::getUuid(true); };\n        return *this;\n    }\n\n    HttpAppFramework &disableSession() override\n    {\n        useSession_ = false;\n        return *this;\n    }\n\n    HttpAppFramework &registerSessionStartAdvice(\n        const AdviceStartSessionCallback &advice) override\n    {\n        sessionStartAdvices_.emplace_back(advice);\n        return *this;\n    }\n\n    HttpAppFramework &registerSessionDestroyAdvice(\n        const AdviceDestroySessionCallback &advice) override\n    {\n        sessionDestroyAdvices_.emplace_back(advice);\n        return *this;\n    }\n\n    const std::string &getDocumentRoot() const override\n    {\n        return rootPath_;\n    }\n\n    HttpAppFramework &setDocumentRoot(const std::string &rootPath) override\n    {\n        rootPath_ = rootPath;\n        return *this;\n    }\n\n    HttpAppFramework &setStaticFileHeaders(\n        const std::vector<std::pair<std::string, std::string>> &headers)\n        override;\n\n    HttpAppFramework &addALocation(\n        const std::string &uriPrefix,\n        const std::string &defaultContentType,\n        const std::string &alias,\n        bool isCaseSensitive,\n        bool allowAll,\n        bool isRecursive,\n        const std::vector<std::string> &filters) override;\n\n    const std::string &getUploadPath() const override\n    {\n        return uploadPath_;\n    }\n\n    const std::shared_ptr<trantor::Resolver> &getResolver() const override\n    {\n        static auto resolver = trantor::Resolver::newResolver(getLoop());\n        return resolver;\n    }\n\n    HttpAppFramework &setUploadPath(const std::string &uploadPath) override;\n    HttpAppFramework &setFileTypes(\n        const std::vector<std::string> &types) override;\n#ifndef _WIN32\n    HttpAppFramework &enableDynamicViewsLoading(\n        const std::vector<std::string> &libPaths,\n        const std::string &outputPath) override;\n#endif\n    HttpAppFramework &setMaxConnectionNum(size_t maxConnections) override;\n    HttpAppFramework &setMaxConnectionNumPerIP(\n        size_t maxConnectionsPerIP) override;\n    HttpAppFramework &loadConfigFile(const std::string &fileName) noexcept(\n        false) override;\n    HttpAppFramework &loadConfigJson(const Json::Value &data) noexcept(\n        false) override;\n    HttpAppFramework &loadConfigJson(Json::Value &&data) noexcept(\n        false) override;\n\n    HttpAppFramework &enableRunAsDaemon() override\n    {\n        runAsDaemon_ = true;\n        return *this;\n...\n// Path: lib/src/HttpUtils.h\n/**\n *\n *  @file HttpUtils.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <trantor/utils/MsgBuffer.h>\n#include <drogon/HttpTypes.h>\n#include <string>\n#include <string_view>\n\nnamespace drogon\n{\nconst std::string_view &contentTypeToMime(ContentType contentType);\nconst std::string_view &statusCodeToString(int code);\nContentType getContentType(const std::string &fileName);\nContentType parseContentType(const std::string_view &contentType);\nFileType parseFileType(const std::string_view &fileExtension);\nFileType getFileType(ContentType contentType);\nvoid registerCustomExtensionMime(const std::string &ext,\n                                 const std::string &mime);\nconst std::string_view fileNameToMime(const std::string &fileName);\nstd::pair<ContentType, const std::string_view> fileNameToContentTypeAndMime(\n    const std::string &filename);\n\ninline std::string_view getFileExtension(const std::string &fileName)\n{\n    auto pos = fileName.rfind('.');\n    if (pos == std::string::npos)\n        return \"\";\n    return std::string_view(&fileName[pos + 1], fileName.length() - pos - 1);\n}\n\nconst std::vector<std::string_view> &getFileExtensions(ContentType contentType);\n\ninline const std::vector<std::string_view> &getFileExtensions(\n    const std::string_view &contentType)\n{\n    return getFileExtensions(parseContentType(contentType));\n}\n\ntemplate <typename T>\ninline constexpr const char *contentLengthFormatString()\n{\n    return \"content-length: %d\\r\\n\";\n}\n\ntemplate <>\ninline constexpr const char *contentLengthFormatString<unsigned int>()\n{\n    return \"content-length: %u\\r\\n\";\n}\n\ntemplate <>\ninline constexpr const char *contentLengthFormatString<long>()\n{\n    return \"content-length: %ld\\r\\n\";\n}\n\ntemplate <>\ninline constexpr const char *contentLengthFormatString<unsigned long>()\n{\n    return \"content-length: %lu\\r\\n\";\n}\n\ntemplate <>\ninline constexpr const char *contentLengthFormatString<long long>()\n{\n    return \"content-length: %lld\\r\\n\";\n}\n\ntemplate <>\ninline constexpr const char *contentLengthFormatString<unsigned long long>()\n{\n    return \"content-length: %llu\\r\\n\";\n}\n}  // namespace drogon\n\n// Path: lib/src/CacheFile.h\n/**\n *\n *  CacheFile.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <trantor/utils/NonCopyable.h>\n#include <string>\n#include <string_view>\n#include <stdio.h>\n\nnamespace drogon\n{\nclass CacheFile : public trantor::NonCopyable\n{\n  public:\n    explicit CacheFile(const std::string &path, bool autoDelete = true);\n    ~CacheFile();\n\n    void append(const std::string &data)\n    {\n        append(data.data(), data.length());\n    }\n\n    void append(const char *data, size_t length);\n\n    std::string_view getStringView()\n    {\n        if (data())\n            return std::string_view(data_, dataLength_);\n        return std::string_view();\n    }\n\n  private:\n    char *data();\n    size_t length();\n    FILE *file_{nullptr};\n    bool autoDelete_{true};\n    const std::string path_;\n    char *data_{nullptr};\n    size_t dataLength_{0};\n};\n}  // namespace drogon\n\n// Path: lib/src/HttpRequestImpl.h\n/**\n *\n *  @file HttpRequestImpl.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include \"HttpUtils.h\"\n#include \"CacheFile.h\"\n#include <drogon/utils/Utilities.h>\n#include <drogon/HttpRequest.h>\n#include <drogon/utils/Utilities.h>\n#include <trantor/net/EventLoop.h>\n#include <trantor/net/InetAddress.h>\n#include <trantor/net/Certificate.h>\n#include <trantor/utils/Logger.h>\n#include <trantor/utils/MsgBuffer.h>\n#include <trantor/utils/NonCopyable.h>\n#include <algorithm>\n#include <string>\n#include <thread>\n#include <unordered_map>\n#include <assert.h>\n#include <stdio.h>\n\nnamespace drogon\n{\nenum class StreamDecompressStatus\n{\n    TooLarge,\n    DecompressError,\n    NotSupported,\n    Ok\n};\n\nclass HttpRequestImpl : public HttpRequest\n{\n  public:\n    friend class HttpRequestParser;\n\n    explicit HttpRequestImpl(trantor::EventLoop *loop)\n        : creationDate_(trantor::Date::now()), loop_(loop)\n    {\n    }\n\n    void reset()\n    {\n        method_ = Invalid;\n        previousMethod_ = Invalid;\n        version_ = Version::kUnknown;\n        flagForParsingJson_ = false;\n        headers_.clear();\n        cookies_.clear();\n        flagForParsingParameters_ = false;\n        path_.clear();\n        originalPath_.clear();\n        pathEncode_ = true;\n        matchedPathPattern_ = \"\";\n        query_.clear();\n        parameters_.clear();\n        jsonPtr_.reset();\n        sessionPtr_.reset();\n        attributesPtr_.reset();\n        cacheFilePtr_.reset();\n        expectPtr_.reset();\n        content_.clear();\n        contentType_ = CT_TEXT_PLAIN;\n        flagForParsingContentType_ = false;\n        contentTypeString_.clear();\n        keepAlive_ = true;\n        jsonParsingErrorPtr_.reset();\n        peerCertificate_.reset();\n        routingParams_.clear();\n    }\n\n    trantor::EventLoop *getLoop()\n    {\n        return loop_;\n    }\n\n    void setVersion(Version v)\n    {\n        version_ = v;\n        if (version_ == Version::kHttp10)\n        {\n            keepAlive_ = false;\n        }\n    }\n\n    Version version() const override\n    {\n        return version_;\n    }\n\n    const char *versionString() const override;\n\n    bool setMethod(const char *start, const char *end);\n\n    void setSecure(bool secure)\n    {\n        isOnSecureConnection_ = secure;\n    }\n\n    void setMethod(const HttpMethod method) override\n    {\n        previousMethod_ = method_;\n        method_ = method;\n        return;\n    }\n\n    HttpMethod method() const override\n    {\n        return method_;\n    }\n\n    bool isHead() const override\n    {\n        return (method_ == HttpMethod::Head) ||\n               ((method_ == HttpMethod::Get) &&\n                (previousMethod_ == HttpMethod::Head));\n    }\n\n    const char *methodString() const override;\n\n    void setPath(const char *start, const char *end)\n    {\n        if (utils::needUrlDecoding(start, end))\n        {\n            originalPath_.append(start, end);\n            path_ = utils::urlDecode(start, end);\n        }\n        else\n        {\n            path_.append(start, end);\n        }\n    }\n\n    const std::vector<std::string> &getRoutingParameters() const override\n    {\n        return routingParams_;\n    }\n\n    void setRoutingParameters(std::vector<std::string> &&params) override\n    {\n        routingParams_ = std::move(params);\n    }\n\n    void setPath(const std::string &path) override\n    {\n        path_ = path;\n    }\n\n    void setPath(std::string &&path) override\n    {\n        path_ = std::move(path);\n    }\n\n    void setPathEncode(bool pathEncode) override\n    {\n        pathEncode_ = pathEncode;\n    }\n\n    const SafeStringMap<std::string> &parameters() const override\n    {\n        parseParametersOnce();\n        return parameters_;\n    }\n\n    const std::string &getParameter(const std::string &key) const override\n    {\n        static const std::string defaultVal;\n        parseParametersOnce();\n        auto iter = parameters_.find(key);\n        if (iter != parameters_.end())\n            return iter->second;\n        return defaultVal;\n    }\n\n    const std::string &path() const override\n    {\n        return path_;\n    }\n\n    const std::string &getOriginalPath() const override\n    {\n        return originalPath_.empty() ? path_ : originalPath_;\n    }\n\n    void setQuery(const char *start, const char *end)\n    {\n        query_.assign(start, end);\n    }\n\n    void setQuery(const std::string &query)\n    {\n        query_ = query;\n    }\n\n    std::string_view bodyView() const\n    {\n        if (cacheFilePtr_)\n        {\n            return cacheFilePtr_->getStringView();\n        }\n        return content_;\n    }\n\n    const char *bodyData() const override\n    {\n        if (cacheFilePtr_)\n        {\n            return cacheFilePtr_->getStringView().data();\n        }\n        return content_.data();\n    }\n\n    size_t bodyLength() const override\n    {\n        if (cacheFilePtr_)\n        {\n            return cacheFilePtr_->getStringView().length();\n        }\n        return content_.length();\n    }\n\n    void appendToBody(const char *data, size_t length);\n\n    void reserveBodySize(size_t length);\n\n    std::string_view queryView() const\n    {\n        return query_;\n    }\n\n    std::string_view contentView() const\n    {\n        if (cacheFilePtr_)\n            return cacheFilePtr_->getStringView();\n        return content_;\n    }\n\n    const std::string &query() const override\n    {\n        return query_;\n    }\n\n    const trantor::InetAddress &peerAddr() const override\n    {\n        return peer_;\n    }\n\n    const trantor::InetAddress &localAddr() const override\n    {\n        return local_;\n    }\n\n    const trantor::Date &creationDate() const override\n    {\n        return creationDate_;\n    }\n\n    const trantor::CertificatePtr &peerCertificate() const override\n    {\n        return peerCertificate_;\n    }\n\n    void setCreationDate(const trantor::Date &date)\n    {\n        creationDate_ = date;\n    }\n\n    void setPeerAddr(const trantor::InetAddress &peer)\n    {\n        peer_ = peer;\n    }\n\n    void setLocalAddr(const trantor::InetAddress &local)\n    {\n        local_ = local;\n    }\n\n    void setPeerCertificate(const trantor::CertificatePtr &cert)\n    {\n        peerCertificate_ = cert;\n    }\n\n    void addHeader(const char *start, const char *colon, const char *end);\n\n    void removeHeader(std::string key) override\n    {\n        transform(key.begin(), key.end(), key.begin(), [](unsigned char c) {\n            return tolower(c);\n        });\n        removeHeaderBy(key);\n    }\n\n    void removeHeaderBy(const std::string &lowerKey)\n    {\n        headers_.erase(lowerKey);\n    }\n\n    const std::string &getHeader(std::string field) const override\n    {\n        std::transform(field.begin(),\n                       field.end(),\n                       field.begin(),\n                       [](unsigned char c) { return tolower(c); });\n        return getHeaderBy(field);\n    }\n\n    const std::string &getHeaderBy(const std::string &lowerField) const\n    {\n        static const std::string defaultVal;\n        auto it = headers_.find(lowerField);\n        if (it != headers_.end())\n        {\n            return it->second;\n        }\n        return defaultVal;\n    }\n\n    const std::string &getCookie(const std::string &field) const override\n    {\n        static const std::string defaultVal;\n        auto it = cookies_.find(field);\n        if (it != cookies_.end())\n        {\n            return it->second;\n        }\n        return defaultVal;\n    }\n\n    const SafeStringMap<std::string> &headers() const override\n    {\n        return headers_;\n    }\n\n    const SafeStringMap<std::string> &cookies() const override\n    {\n        return cookies_;\n    }\n\n    void setParameter(const std::string &key, const std::string &value) override\n    {\n        flagForParsingParameters_ = true;\n        parameters_[key] = value;\n    }\n\n    const std::string &getContent() const\n    {\n        return content_;\n    }\n\n    void swap(HttpRequestImpl &that) noexcept;\n\n    void setContent(const std::string &content)\n    {\n        content_ = content;\n    }\n\n    void setBody(const std::string &body) override\n    {\n        content_ = body;\n    }\n\n    void setBody(std::string &&body) override\n    {\n        content_ = std::move(body);\n    }\n\n    void addHeader(std::string field, const std::string &value) override\n    {\n        transform(field.begin(),\n                  field.end(),\n                  field.begin(),\n                  [](unsigned char c) { return tolower(c); });\n        headers_[std::move(field)] = value;\n    }\n\n    void addHeader(std::string field, std::string &&value) override\n    {\n        transform(field.begin(),\n                  field.end(),\n                  field.begin(),\n                  [](unsigned char c) { return tolower(c); });\n        headers_[std::move(field)] = std::move(value);\n    }\n\n    void addCookie(const std::string &key, const std::string &value) override\n    {\n        cookies_[key] = value;\n    }\n\n    void setPassThrough(bool flag) override\n    {\n        passThrough_ = flag;\n    }\n\n    bool passThrough() const\n    {\n        return passThrough_;\n    }\n\n    void appendToBuffer(trantor::MsgBuffer *output) const;\n\n    const SessionPtr &session() const override\n    {\n        return sessionPtr_;\n    }\n\n    void setSession(const SessionPtr &session)\n    {\n        sessionPtr_ = session;\n    }\n\n    const AttributesPtr &attributes() const override\n    {\n        if (!attributesPtr_)\n        {\n            attributesPtr_ = std::make_shared<Attributes>();\n        }\n        return attributesPtr_;\n    }\n\n    const std::shared_ptr<Json::Value> &jsonObject() const override\n    {\n        // Not multi-thread safe but good, because we basically call this\n        // function in a single thread\n        if (!flagForParsingJson_)\n        {\n            flagForParsingJson_ = true;\n            parseJson();\n        }\n        return jsonPtr_;\n    }\n\n    void setCustomContentTypeString(const std::string &type) override\n    {\n        contentType_ = CT_NONE;\n        flagForParsingContentType_ = true;\n        bool haveHeader = type.find(\"content-type: \") == 0;\n        bool haveCRLF = type.rfind(\"\\r\\n\") == type.size() - 2;\n\n        size_t endOffset = 0;\n        if (haveHeader)\n            endOffset += 14;\n        if (haveCRLF)\n            endOffset += 2;\n        contentTypeString_ = std::string(type.begin() + (haveHeader ? 14 : 0),\n                                         type.end() - endOffset);\n    }\n\n    void setContentTypeCode(const ContentType type) override\n    {\n        contentType_ = type;\n        flagForParsingContentType_ = true;\n        auto &typeStr = contentTypeToMime(type);\n        setContentType(std::string(typeStr.data(), typeStr.length()));\n    }\n\n    void setContentTypeString(const char *typeString,\n                              size_t typeStringLength) override;\n\n    // void setContentTypeCodeAndCharacterSet(ContentType type, const\n    // std::string &charSet = \"utf-8\") override\n    // {\n    //     contentType_ = type;\n    //     setContentType(webContentTypeAndCharsetToString(type, charSet));\n    // }\n\n    ContentType contentType() const override\n    {\n        parseContentTypeAndString();\n        return contentType_;\n    }\n\n    const char *matchedPathPatternData() const override\n    {\n        return matchedPathPattern_.data();\n    }\n\n    size_t matchedPathPatternLength() const override\n    {\n        return matchedPathPattern_.length();\n    }\n\n    void setMatchedPathPattern(const std::string &pathPattern)\n    {\n        matchedPathPattern_ = pathPattern;\n    }\n\n    const std::string &expect() const\n    {\n        static const std::string none{\"\"};\n        if (expectPtr_)\n            return *expectPtr_;\n        return none;\n    }\n\n    bool keepAlive() const\n    {\n        return keepAlive_;\n    }\n\n    bool isOnSecureConnection() const noexcept override\n    {\n        return isOnSecureConnection_;\n    }\n\n    const std::string &getJsonError() const override\n    {\n        static const std::string none{\"\"};\n        if (jsonParsingErrorPtr_)\n            return *jsonParsingErrorPtr_;\n        return none;\n    }\n\n    StreamDecompressStatus decompressBody();\n\n    ~HttpRequestImpl();\n\n  protected:\n    friend class HttpRequest;\n\n    void setContentType(const std::string &contentType)\n    {\n        contentTypeString_ = contentType;\n    }\n\n    void setContentType(std::string &&contentType)\n    {\n        contentTypeString_ = std::move(contentType);\n    }\n\n    void parseContentTypeAndString() const\n    {\n        if (!flagForParsingContentType_)\n        {\n            flagForParsingContentType_ = true;\n            auto &contentTypeString = getHeaderBy(\"content-type\");\n            if (contentTypeString == \"\")\n            {\n                contentType_ = CT_NONE;\n            }\n            else\n            {\n                auto pos = contentTypeString.find(';');\n                if (pos != std::string::npos)\n                {\n                    contentType_ = parseContentType(\n                        std::string_view(contentTypeString.data(), pos));\n                }\n                else\n                {\n                    contentType_ =\n                        parseContentType(std::string_view(contentTypeString));\n                }\n\n                if (contentType_ == CT_NONE)\n                    contentType_ = CT_CUSTOM;\n                contentTypeString_ = contentTypeString;\n            }\n        }\n    }\n\n  private:\n    void parseParameters() const;\n\n    void parseParametersOnce() const\n    {\n        // Not multi-thread safe but good, because we basically call this\n        // function in a single thread\n        if (!flagForParsingParameters_)\n        {\n            flagForParsingParameters_ = true;\n            parseParameters();\n        }\n    }\n\n    void createTmpFile();\n    void parseJson() const;\n#ifdef USE_BROTLI\n    StreamDecompressStatus decompressBodyBrotli() noexcept;\n#endif\n    StreamDecompressStatus decompressBodyGzip() noexcept;\n    mutable bool flagForParsingParameters_{false};\n    mutable bool flagForParsingJson_{false};\n    HttpMethod method_{Invalid};\n    HttpMethod previousMethod_{Invalid};\n    Version version_{Version::kUnknown};\n    std::string path_;\n    std::string originalPath_;\n    bool pathEncode_{true};\n    std::string_view matchedPathPattern_{\"\"};\n    std::string query_;\n    SafeStringMap<std::string> headers_;\n    SafeStringMap<std::string> cookies_;\n    mutable SafeStringMap<std::string> parameters_;\n    mutable std::shared_ptr<Json::Value> jsonPtr_;\n    SessionPtr sessionPtr_;\n    mutable AttributesPtr attributesPtr_;\n    trantor::InetAddress peer_;\n    trantor::InetAddress local_;\n    trantor::Date creationDate_;\n    trantor::CertificatePtr peerCertificate_;\n    std::unique_ptr<CacheFile> cacheFilePtr_;\n    mutable std::unique_ptr<std::string> jsonParsingErrorPtr_;\n    std::unique_ptr<std::string> expectPtr_;\n    bool keepAlive_{true};\n    bool isOnSecureConnection_{false};\n    bool passThrough_{false};\n    std::vector<std::string> routingParams_;\n\n  protected:\n    std::string content_;\n    trantor::EventLoop *loop_;\n    mutable ContentType contentType_{CT_TEXT_PLAIN};\n    mutable bool flagForParsingContentType_{false};\n    mutable std::string contentTypeString_;\n};\n\nusing HttpRequestImplPtr = std::shared_ptr<HttpRequestImpl>;\n\ninline void swap(HttpRequestImpl &one, HttpRequestImpl &two) noexcept\n{\n    one.swap(two);\n}\n\n}  // namespace drogon\n\n// Path: lib/src/HttpMessageBody.h\n/**\n *\n *  HttpMessageBody.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n#include <string_view>\n#include <memory>\n#include <string>\n\nnamespace drogon\n{\nclass HttpMessageBody\n{\n  public:\n    enum class BodyType\n    {\n        kNone = 0,\n        kString,\n        kStringView\n    };\n\n    BodyType bodyType()\n    {\n        return type_;\n    }\n\n    virtual const char *data() const\n    {\n        return nullptr;\n    }\n\n    virtual char *data()\n    {\n        return nullptr;\n    }\n\n    virtual size_t length() const\n    {\n        return 0;\n    }\n\n    virtual std::string_view getString() const = 0;\n\n    virtual void append(const char * /*buf*/, size_t /*len*/)\n    {\n    }\n\n    virtual ~HttpMessageBody()\n    {\n    }\n\n  protected:\n    BodyType type_{BodyType::kNone};\n};\n\nclass HttpMessageStringBody : public HttpMessageBody\n{\n  public:\n    HttpMessageStringBody()\n    {\n        type_ = BodyType::kString;\n    }\n\n    HttpMessageStringBody(const std::string &body) : body_(body)\n    {\n        type_ = BodyType::kString;\n    }\n\n    HttpMessageStringBody(std::string &&body) : body_(std::move(body))\n    {\n        type_ = BodyType::kString;\n    }\n\n    const char *data() const override\n    {\n        return body_.data();\n    }\n\n    char *data() override\n    {\n        return const_cast<char *>(body_.data());\n    }\n\n    size_t length() const override\n    {\n        return body_.length();\n    }\n\n    std::string_view getString() const override\n    {\n        return std::string_view{body_.data(), body_.length()};\n    }\n\n    void append(const char *buf, size_t len) override\n    {\n        body_.append(buf, len);\n    }\n\n  private:\n    std::string body_;\n};\n\nclass HttpMessageStringViewBody : public HttpMessageBody\n{\n  public:\n    \nHttpMessageStringViewBody(const char *buf, size_t len) : body_(buf, len)\n    {\n        type_ = BodyType::kStringView;\n    }\n\n    const char *data() const override\n    {\n        return body_.data();\n    }\n\n    char *data() override\n    {\n        return const_cast<char *>(body_.data());\n    }\n\n    size_t length() const override\n    {\n        return body_.length();\n    }\n\n    std::string_view getString() const override\n    {\n        return body_;\n    }\n\n  private:\n    std::string_view body_;\n};\n\n}  // namespace drogon\n\n// Path: lib/src/HttpResponseImpl.h\n/**\n *\n *  @file HttpResponseImpl.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include \"HttpUtils.h\"\n#include \"HttpMessageBody.h\"\n#include <drogon/exports.h>\n#include <drogon/HttpResponse.h>\n#include <drogon/utils/Utilities.h>\n#include <trantor/net/InetAddress.h>\n#include <trantor/utils/Date.h>\n#include <trantor/utils/MsgBuffer.h>\n#include <memory>\n#include <mutex>\n#include <string>\n#include <atomic>\n#include <unordered_map>\n\nnamespace drogon\n{\nclass DROGON_EXPORT HttpResponseImpl : public HttpResponse\n{\n    friend class HttpResponseParser;\n\n  public:\n    HttpResponseImpl() : creationDate_(trantor::Date::now())\n    {\n    }\n\n    HttpResponseImpl(HttpStatusCode code, ContentType type)\n        : statusCode_(code),\n          statusMessage_(statusCodeToString(code)),\n          creationDate_(trantor::Date::now()),\n          contentType_(type),\n          flagForParsingContentType_(true),\n          contentTypeString_(contentTypeToMime(type))\n    {\n    }\n\n    void setPassThrough(bool flag) override\n    {\n        passThrough_ = flag;\n    }\n\n    HttpStatusCode statusCode() const override\n    {\n        return statusCode_;\n    }\n\n    const trantor::Date &creationDate() const override\n    {\n        return creationDate_;\n    }\n\n    void setStatusCode(HttpStatusCode code) override\n    {\n        statusCode_ = code;\n        setStatusMessage(statusCodeToString(code));\n    }\n\n    void setVersion(const Version v) override\n    {\n        version_ = v;\n        if (version_ == Version::kHttp10)\n        {\n            closeConnection_ = true;\n        }\n    }\n\n    Version version() const override\n    {\n        return version_;\n    }\n\n    const char *versionString() const override;\n\n    void setCloseConnection(bool on) override\n    {\n        closeConnection_ = on;\n    }\n\n    bool ifCloseConnection() const override\n    {\n        return closeConnection_;\n    }\n\n    void setContentTypeCode(ContentType type) override\n    {\n        contentType_ = type;\n        auto ct = contentTypeToMime(type);\n        contentTypeString_ = std::string(ct.data(), ct.size());\n        flagForParsingContentType_ = true;\n    }\n\n    //  void setContentTypeCodeAndCharacterSet(ContentType type, const\n    // std::string &charSet = \"utf-8\") override\n    // {\n    //     contentType_ = type;\n    //     setContentType(webContentTypeAndCharsetToString(type, charSet));\n    // }\n\n    ContentType contentType() const override\n    {\n        parseContentTypeAndString();\n        return contentType_;\n    }\n\n    const std::string &getHeader(std::string key) const override\n    {\n        transform(key.begin(), key.end(), key.begin(), [](unsigned char c) {\n            return tolower(c);\n        });\n        return getHeaderBy(key);\n    }\n\n    void removeHeader(std::string key) override\n    {\n        transform(key.begin(), key.end(), key.begin(), [](unsigned char c) {\n            return tolower(c);\n        });\n        removeHeaderBy(key);\n    }\n\n    const SafeStringMap<std::string> &headers() const override\n    {\n        return headers_;\n    }\n\n    const std::string &getHeaderBy(const std::string &lowerKey) const\n    {\n        static const std::string defaultVal;\n        auto iter = headers_.find(lowerKey);\n        if (iter == headers_.end())\n        {\n            return defaultVal;\n        }\n        return iter->second;\n    }\n\n    void removeHeaderBy(const std::string &lowerKey)\n    {\n        fullHeaderString_.reset();\n        headers_.erase(lowerKey);\n    }\n\n    void addHeader(std::string field, const std::string &value) override\n    {\n        fullHeaderString_.reset();\n        transform(field.begin(),\n                  field.end(),\n                  field.begin(),\n                  [](unsigned char c) { return tolower(c); });\n        headers_[std::move(field)] = value;\n    }\n\n    void addHeader(std::string field, std::string &&value) override\n    {\n        fullHeaderString_.reset();\n        transform(field.begin(),\n                  field.end(),\n                  field.begin(),\n                  [](unsigned char c) { return tolower(c); });\n        headers_[std::move(field)] = std::move(value);\n    }\n\n    void addHeader(const char *start, const char *colon, const char *end);\n\n    void addCookie(const std::string &key, const std::string &value) override\n    {\n        cookies_[key] = Cookie(key, value);\n    }\n\n    void addCookie(const Cookie &cookie) override\n    {\n        cookies_[cookie.key()] = cookie;\n    }\n\n    void addCookie(Cookie &&cookie) override\n    {\n        cookies_[cookie.key()] = std::move(cookie);\n    }\n\n    const Cookie &getCookie(const std::string &key) const override\n    {\n        static const Cookie defaultCookie;\n        auto it = cookies_.find(key);\n        if (it != cookies_.end())\n        {\n            return it->second;\n        }\n        return defaultCookie;\n    }\n\n    const SafeStringMap<Cookie> &cookies() const override\n    {\n        return cookies_;\n    }\n\n    void removeCookie(const std::string &key) override\n    {\n        cookies_.erase(key);\n    }\n\n    void setBody(const std::string &body) override\n    {\n        bodyPtr_ = std::make_shared<HttpMessageStringBody>(body);\n        if (passThrough_)\n        {\n            addHeader(\"content-length\", std::to_string(bodyPtr_->length()));\n        }\n    }\n\n    void setBody(std::string &&body) override\n    {\n        bodyPtr_ = std::make_shared<HttpMessageStringBody>(std::move(body));\n        if (passThrough_)\n        {\n            addHeader(\"content-length\", std::to_string(bodyPtr_->length()));\n        }\n    }\n\n    void redirect(const std::string &url)\n    {\n        headers_[\"location\"] = url;\n    }\n\n    std::shared_ptr<trantor::MsgBuffer> renderToBuffer();\n    void renderToBuffer(trantor::MsgBuffer &buffer);\n    std::shared_ptr<trantor::MsgBuffer> renderHeaderForHeadMethod();\n    void clear() override;\n\n    void setExpiredTime(ssize_t expiredTime) override\n    {\n        expriedTime_ = expiredTime;\n        datePos_ = std::string::npos;\n        if (expriedTime_ < 0 && version_ == Version::kHttp10)\n        {\n            fullHeaderString_.reset();\n        }\n    }\n\n    ssize_t expiredTime() const override\n    {\n        return expriedTime_;\n    }\n\n    const char *getBodyData() const override\n    {\n        if (!flagForSerializingJson_ && jsonPtr_)\n        {\n            generateBodyFromJson();\n        }\n        else if (!bodyPtr_)\n        {\n            return nullptr;\n        }\n        return bodyPtr_->data();\n    }\n\n    size_t getBodyLength() const override\n    {\n        if (bodyPtr_)\n            return bodyPtr_->length();\n        return 0;\n    }\n\n    void swap(HttpResponseImpl &that) noexcept;\n    void parseJson() const;\n\n    const std::shared_ptr<Json::Value> &jsonObject() const override\n    {\n        // Not multi-thread safe but good, because we basically call this\n        // function in a single thread\n        if (!flagForParsingJson_)\n        {\n            flagForParsingJson_ = true;\n            parseJson();\n        }\n        return jsonPtr_;\n    }\n\n    const std::string &getJsonError() const override\n    {\n        static const std::string none;\n        if (jsonParsingErrorPtr_)\n            return *jsonParsingErrorPtr_;\n        return none;\n    }\n\n    void setJsonObject(const Json::Value &pJson)\n    {\n        flagForParsingJson_ = true;\n        flagForSerializingJson_ = false;\n        jsonPtr_ = std::make_shared<Json::Value>(pJson);\n    }\n\n    void setJsonObject(Json::Value &&pJson)\n    {\n        flagForParsingJson_ = true;\n        flagForSerializingJson_ = false;\n        jsonPtr_ = std::make_shared<Json::Value>(std::move(pJson));\n    }\n\n    bool shouldBeCompressed() const;\n    void generateBodyFromJson() const;\n\n    const std::string &sendfileName() const override\n    {\n        return sendfileName_;\n    }\n\n    const SendfileRange &sendfileRange() const override\n    {\n        return sendfileRange_;\n    }\n\n    const trantor::CertificatePtr &peerCertificate() const override\n    {\n        return peerCertificate_;\n    }\n\n    void setPeerCertificate(const trantor::CertificatePtr &cert)\n    {\n        peerCertificate_ = cert;\n    }\n\n    void setSendfile(const std::string &filename)\n    {\n        sendfileName_ = filename;\n    }\n\n    void setSendfileRange(size_t offset, size_t len)\n    {\n        sendfileRange_.first = offset;\n        sendfileRange_.second = len;\n    }\n\n    const std::function<std::size_t(char *, std::size_t)> &streamCallback()\n        const override\n    {\n        return streamCallback_;\n    }\n\n    void setStreamCallback(\n        const std::function<std::size_t(char *, std::size_t)> &callback)\n    {\n        streamCallback_ = callback;\n    }\n\n    const std::function<void(ResponseStreamPtr)> &asyncStreamCallback()\n        const override\n    {\n        return asyncStreamCallback_;\n    }\n\n    void setAsyncStreamCallback(\n        const std::function<void(ResponseStreamPtr)> &callback,\n        bool disableKickoffTimeout)\n    {\n        asyncStreamCallback_ = callback;\n        asyncStreamDisableKickoff_ = disableKickoffTimeout;\n    }\n\n    bool asyncStreamKickoffDisabled() const\n    {\n        return asyncStreamDisableKickoff_;\n    }\n\n    void makeHeaderString()\n    {\n        fullHeaderString_ = std::make_shared<trantor::MsgBuffer>(128);\n        makeHeaderString(*fullHeaderString_);\n    }\n\n    std::string contentTypeString() const override\n    {\n        parseContentTypeAndString();\n        return contentTypeString_;\n    }\n\n    void gunzip()\n    {\n        if (bodyPtr_)\n        {\n            auto gunzipBody =\n                utils::gzipDecompress(bodyPtr_->data(), bodyPtr_->length());\n            removeHeaderBy(\"content-encoding\");\n            bodyPtr_ =\n                std::make_shared<HttpMessageStringBody>(std::move(gunzipBody));\n            addHeader(\"content-length\", std::to_string(bodyPtr_->length()));\n        }\n    }\n#ifdef USE_BROTLI\n    void brDecompress()\n    {\n        if (bodyPtr_)\n        {\n            auto gunzipBody =\n                utils::brotliDecompress(bodyPtr_->data(), bodyPtr_->length());\n            removeHeaderBy(\"content-encoding\");\n            bodyPtr_ =\n                std::make_shared<HttpMessageStringBody>(std::move(gunzipBody));\n            addHeader(\"content-length\", std::to_string(bodyPtr_->length()));\n        }\n    }\n#endif\n    ~HttpResponseImpl() override = default;\n\n  protected:\n    void makeHeaderString(trantor::MsgBuffer &headerString);\n\n    void parseContentTypeAndString() const\n    {\n        if (!flagForParsingContentType_)\n        {\n            flagForParsingContentType_ = true;\n            auto &contentTypeString = getHeaderBy(\"content-type\");\n            if (contentTypeString == \"\")\n            {\n                contentType_ = CT_NONE;\n            }\n            else\n            {\n                auto pos = contentTypeString.find(';');\n                if (pos != std::string::npos)\n                {\n                    contentType_ = parseContentType(\n                        std::string_view(contentTypeString.data(), pos));\n                }\n                else\n                {\n                    contentType_ =\n                        parseContentType(std::string_view(contentTypeString));\n                }\n\n                if (contentType_ == CT_NONE)\n                    contentType_ = CT_CUSTOM;\n                contentTypeString_ = contentTypeString;\n            }\n        }\n    }\n\n  private:\n    void setBody(const char *body, size_t len) override\n    {\n        bodyPtr_ = std::make_shared<HttpMessageStringViewBody>(body, len);\n        if (passThrough_)\n        {\n            addHeader(\"content-length\", std::to_string(bodyPtr_->length()));\n        }\n    }\n\n    void setContentTypeCodeAndCustomString(ContentType type,\n                                           const char *typeString,\n                                           size_t typeStringLength) override\n    {\n        contentType_ = type;\n        flagForParsingContentType_ = true;\n\n        std::string_view sv(typeString, typeStringLength);\n        bool haveHeader = sv.find(\"content-type: \") == 0;\n        bool haveCRLF = sv.rfind(\"\\r\\n\") == sv.size() - 2;\n\n        size_t endOffset = 0;\n        if (haveHeader)\n            endOffset += 14;\n        if (haveCRLF)\n            endOffset += 2;\n        setContentType(std::string_view{typeString + (haveHeader ? 14 : 0),\n                                        typeStringLength - endOffset});\n    }\n\n    void setContentTypeString(const char *typeString,\n                              size_t typeStringLength) override;\n\n    void setCustomStatusCode(int code,\n                             const char *message,\n                             size_t messageLength) override\n    {\n        assert(code >= 0);\n        customStatusCode_ = code;\n        statusMessage_ = std::string_view{message, messageLength};\n    }\n\n    SafeStringMap<std::string> headers_;\n    SafeStringMap<Cookie> cookies_;\n\n    int customStatusCode_{-1};\n    HttpStatusCode statusCode_{kUnknown};\n    std::string_view statusMessage_;\n\n    trantor::Date creationDate_;\n    Version version_{Version::kHttp11};\n    bool closeConnection_{false};\n    mutable std::shared_ptr<HttpMessageBody> bodyPtr_;\n    ssize_t expriedTime_{-1};\n    std::string sendfileName_;\n    SendfileRange sendfileRange_{0, 0};\n    std::function<std::size_t(char *, std::size_t)> streamCallback_;\n    std::function<void(ResponseStreamPtr)> asyncStreamCallback_;\n    bool asyncStreamDisableKickoff_{false};\n\n    mutable std::shared_ptr<Json::Value> jsonPtr_;\n\n    std::shared_ptr<trantor::MsgBuffer> fullHeaderString_;\n    trantor::CertificatePtr peerCertificate_;\n    mutable std::shared_ptr<trantor::MsgBuffer> httpString_;\n    mutable size_t datePos_{static_cast<size_t>(-1)};\n    mutable int64_t httpStringDate_{-1};\n    mutable bool flagForParsingJson_{false};\n    mutable bool flagForSerializingJson_{true};\n    mutable ContentType contentType_{CT_TEXT_PLAIN};\n    mutable bool flagForParsingContentType_{false};\n    mutable std::shared_ptr<std::string> jsonParsingErrorPtr_;\n    mutable std::string contentTypeString_{\"text/html; charset=utf-8\"};\n    bool passThrough_{false};\n\n    void setContentType(const std::string_view &contentType)\n    {\n        contentTypeString_ =\n            std::string(contentType.data(), contentType.size());\n    }\n\n    void setStatusMessage(const std::string_view &message)\n    {\n        statusMessage_ = message;\n    }\n};\n\nusing HttpResponseImplPtr = std::shared_ptr<HttpResponseImpl>;\n\ninline void swap(HttpResponseImpl &one, HttpResponseImpl &two) noexcept\n{\n    one.swap(two);\n}\n\n}  // namespace drogon\n\n// Path: lib/src/RangeParser.h\n/**\n *\n *  RangeParser.h\n *  He, Wanchen\n *\n *  Copyright 2021, He,Wanchen.  All rights reserved.\n *  https://github.com/drogonframework/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n#pragma once\n\n#include <string>\n#include <vector>\n#include <sys/types.h>\n\nnamespace drogon\n{\n// [start, end)\nstruct FileRange\n{\n    size_t start;\n    size_t end;\n};\n\nenum FileRangeParseResult\n{\n    InvalidRange = -1,\n    NotSatisfiable = 0,\n    SinglePart = 1,\n    MultiPart = 2\n};\n\nFileRangeParseResult parseRangeHeader(const std::string &rangeStr,\n                                      size_t contentLength,\n                                      std::vector<FileRange> &ranges);\n\n}  // namespace drogon\n\n// Path: lib/src/StaticFileRouter.cc\n/**\n *\n *  StaticFileRouter.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"StaticFileRouter.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpResponseImpl.h\"\n#include \"RangeParser.h\"\n#include <fstream>\n#include <iostream>\n#include <algorithm>\n#include <memory>\n#include <fcntl.h>\n#ifndef _WIN32\n#include <sys/file.h>\n#elif !defined(__MINGW32__)\n#define stat _wstati64\n#define S_ISREG(m) (((m) & 0170000) == (0100000))\n#define S_ISDIR(m) (((m) & 0170000) == (0040000))\n#endif\n#include <sys/stat.h>\n#include <filesystem>\n\nusing namespace drogon;\n\nvoid StaticFileRouter::init(const std::vector<trantor::EventLoop *> &ioLoops)\n{\n    // Max timeout up to about 70 days;\n    staticFilesCacheMap_ = std::make_unique<\n        IOThreadStorage<std::unique_ptr<CacheMap<std::string, char>>>>();\n    staticFilesCacheMap_->init(\n        [&ioLoops](std::unique_ptr<CacheMap<std::string, char>> &mapPtr,\n                   size_t i) {\n            assert(i == ioLoops[i]->index());\n            mapPtr = std::make_unique<CacheMap<std::string, char>>(ioLoops[i],\n                                                                   1.0,\n                                                                   4,\n                                                                   50);\n        });\n    staticFilesCache_ = std::make_unique<\n        IOThreadStorage<std::unordered_map<std::string, HttpResponsePtr>>>();\n    ioLocationsPtr_ =\n        std::make_shared<IOThreadStorage<std::vector<Location>>>();\n    for (auto *loop : ioLoops)\n    {\n        loop->queueInLoop(\n            [ioLocationsPtr = ioLocationsPtr_, locations = locations_] {\n                **ioLocationsPtr = locations;\n            });\n    }\n}\n\nvoid StaticFileRouter::reset()\n{\n    staticFilesCacheMap_.reset();\n    staticFilesCache_.reset();\n    ioLocationsPtr_.reset();\n    locations_.clear();\n}\n\nvoid StaticFileRouter::route(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    const std::string &path = req->path();\n    if (path.find(\"..\") != std::string::npos)\n    {\n        auto directories = utils::splitString(path, \"/\");\n        int traversalDepth = 0;\n        for (const auto &dir : directories)\n        {\n            if (dir == \"..\")\n            {\n                traversalDepth--;\n            }\n            else if (dir != \".\")\n            {\n                traversalDepth++;\n            }\n\n            if (traversalDepth < 0)\n            {\n                // Downloading files from the parent folder is forbidden.\n                callback(app().getCustomErrorHandler()(k403Forbidden, req));\n                return;\n            }\n        }\n    }\n\n    auto lPath = path;\n    std::transform(lPath.begin(),\n                   lPath.end(),\n                   lPath.begin(),\n                   [](unsigned char c) { return tolower(c); });\n\n    for (auto &location : **ioLocationsPtr_)\n    {\n        auto &URI = location.uriPrefix_;\n        if (location.realLocation_.empty())\n        {\n            if (!location.alias_.empty())\n            {\n                if (location.alias_[0] == '/')\n                {\n                    location.realLocation_ = location.alias_;\n                }\n                else\n                {\n                    location.realLocation_ =\n                        HttpAppFrameworkImpl::instance().getDocumentRoot() +\n                        location.alias_;\n                }\n            }\n            else\n            {\n                location.realLocation_ =\n                    HttpAppFrameworkImpl::instance().getDocumentRoot() +\n                    location.uriPrefix_;\n            }\n            if (location.realLocation_[location.realLocation_.length() - 1] !=\n                '/')\n            {\n                location.realLocation_.append(1, '/');\n            }\n            if (!location.isCaseSensitive_)\n            {\n                std::transform(URI.begin(),\n                               URI.end(),\n                               URI.begin(),\n                               [](unsigned char c) { return tolower(c); });\n            }\n        }\n        auto &tmpPath = location.isCaseSensitive_ ? path : lPath;\n        if (tmpPath.length() >= URI.length() &&\n            std::equal(tmpPath.begin(),\n                       tmpPath.begin() + URI.length(),\n                       URI.begin()))\n        {\n            std::string_view restOfThePath{path.data() + URI.length(),\n                                           path.length() - URI.length()};\n            auto pos = restOfThePath.rfind('/');\n            if (pos != 0 && pos != std::string_view::npos &&\n                !location.isRecursive_)\n            {\n                callback(app().getCustomErrorHandler()(k403Forbidden, req));\n                return;\n            }\n            std::string filePath =\n                location.realLocation_ +\n                std::string{restOfThePath.data(), restOfThePath.length()};\n            std::filesystem::path fsFilePath(utils::toNativePath(filePath));\n            std::error_code err;\n            if (!std::filesystem::exists(fsFilePath, err))\n            {\n                defaultHandler_(req, std::move(callback));\n                return;\n            }\n            if (std::filesystem::is_directory(fsFilePath, err))\n            {\n                // Check if path is eligible for an implicit index.html\n                if (implicitPageEnable_)\n                {\n                    filePath = filePath + \"/\" + implicitPage_;\n                }\n                else\n                {\n                    callback(app().getCustomErrorHandler()(k403Forbidden, req));\n                    return;\n                }\n            }\n            else\n            {\n                if (!location.allowAll_)\n                {\n                    pos = restOfThePath.rfind('.');\n                    if (pos == std::string_view::npos)\n                    {\n                        callback(\n                            app().getCustomErrorHandler()(k403Forbidden, req));\n                        return;\n                    }\n                    std::string extension{restOfThePath.data() + pos + 1,\n                                          restOfThePath.length() - pos - 1};\n                    std::transform(extension.begin(),\n                                   extension.end(),\n                                   extension.begin(),\n                                   [](unsigned char c) { return tolower(c); });\n                    if (fileTypeSet_.find(extension) == fileTypeSet_.end())\n                    {\n                        callback(\n                            app().getCustomErrorHandler()(k403Forbidden, req));\n                        return;\n                    }\n                }\n            }\n\n            if (location.filters_.empty())\n            {\n                sendStaticFileResponse(filePath,\n                                       req,\n                                       std::move(callback),\n                                       std::string_view{\n                                           location.defaultContentType_});\n            }\n            else\n            {\n                filters_function::doFilters(\n                    location.filters_,\n                    req,\n                    [this,\n                     req,\n                     filePath = std::move(filePath),\n                     contentType =\n                         std::string_view{location.defaultContentType_},\n                     callback = std::move(callback)](\n                        const HttpResponsePtr &resp) mutable {\n                        if (resp)\n                        {\n                            callback(resp);\n                        }\n                        else\n                        {\n                            sendStaticFileResponse(filePath,\n                                                   req,\n                                                   std::move(callback),\n                                                   contentType);\n                        }\n                    });\n            }\n            return;\n        }\n    }\n    std::string directoryPath =\n        HttpAppFrameworkImpl::instance().getDocumentRoot() + path;\n    std::filesystem::path fsDirectoryPath(utils::toNativePath(directoryPath));\n    std::error_code err;\n    if (std::filesystem::exists(fsDirectoryPath, err))\n    {\n        if (std::filesystem::is_directory(fsDirectoryPath, err))\n        {\n            // Check if path is eligible for an implicit index.html\n            if (implicitPageEnable_)\n            {\n                std::string filePath = directoryPath + \"/\" + implicitPage_;\n                sendStaticFileResponse(filePath, req, std::move(callback), \"\");\n                return;\n            }\n            else\n            {\n                callback(app().getCustomErrorHandler()(k403Forbidden, req));\n                return;\n            }\n        }\n        else\n        {\n            // This is a normal page\n            auto pos = path.rfind('.');\n            if (pos == std::string::npos)\n            {\n                callback(app().getCustomErrorHandler()(k403Forbidden, req));\n                return;\n            }\n            std::string filetype = lPath.substr(pos + 1);\n            if (fileTypeSet_.find(filetype) != fileTypeSet_.end())\n            {\n                // LOG_INFO << \"file query!\" << path;\n                std::string filePath = directoryPath;\n                sendStaticFileResponse(filePath, req, std::move(callback), \"\");\n                return;\n            }\n        }\n    }\n    defaultHandler_(req, std::move(callback));\n}\n\n...\n", "needle_token_start": 8737, "needle_token_end": 8778, "code_context_ntokens": 15988, "output": ["```cpp\nclass HttpMessageStringViewBody : public HttpMessageBody\n{\n  public:\n    HttpMessageStringViewBody(const char *buf, size_t len) : body_(buf, len)\n    {\n        type_ = BodyType::kStringView;\n    }\n\n    const char *data() const override\n    {\n        return body_.data();\n    }\n\n    char *data() override\n    {\n        return const_cast<char *>(body_.data());\n    }\n\n    size_t length() const override\n    {\n        return body_.length();\n    }\n\n    std::string_view getString() const override\n    {\n        return body_;\n    }\n\n  private:\n    std::string_view body_;\n};\n```"]}
{"repo": "drogonframework/drogon", "name": "ListenerInfo", "language": "cpp", "path": "lib/src/ListenerManager.h", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To encapsulate the configuration details required for setting up a network listener, including SSL/TLS specifics.\n2. **Input**: Receives multiple parameters such as IP address, port number, SSL usage flag, certificate file path, key file path, TLS version flag, and SSL configuration commands.\n3. **Output**: Does not directly produce an output but initializes an object with the provided configuration for use in network operations.\n4. **Procedure**: Takes the provided inputs, moves some of them to optimize memory usage, and stores them within the object for later access when setting up network listeners.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/AOPAdvice.h\n/**\n *\n...\n// Path: lib/src/HttpResponseImpl.cc\n/**\n *\n *  @file HttpResponseImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpResponseImpl.h\"\n#include \"AOPAdvice.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpUtils.h\"\n#include <drogon/HttpViewData.h>\n#include <drogon/IOThreadStorage.h>\n#include <filesystem>\n#include <fstream>\n#include <memory>\n#include <cstdio>\n#include <string>\n#include <sys/stat.h>\n#include <trantor/utils/Logger.h>\n\nusing namespace trantor;\nusing namespace drogon;\nusing namespace std::literals::string_literals;\nusing namespace std::placeholders;\n#ifdef _WIN32\n#undef max\n#endif\n\nnamespace drogon\n{\n// \"Fri, 23 Aug 2019 12:58:03 GMT\" length = 29\nstatic const size_t httpFullDateStringLength = 29;\n\nstatic inline HttpResponsePtr genHttpResponse(const std::string &viewName,\n                                              const HttpViewData &data,\n                                              const HttpRequestPtr &req)\n{\n    auto templ = DrTemplateBase::newTemplate(viewName);\n    if (templ)\n    {\n        auto res = HttpResponse::newHttpResponse();\n        res->setBody(templ->genText(data));\n        return res;\n    }\n    return drogon::HttpResponse::newNotFoundResponse(req);\n}\n}  // namespace drogon\n\nHttpResponsePtr HttpResponse::newHttpResponse()\n{\n    auto res = std::make_shared<HttpResponseImpl>(k200OK, CT_TEXT_HTML);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpResponse(HttpStatusCode code,\n                                              ContentType type)\n{\n    auto res = std::make_shared<HttpResponseImpl>(code, type);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpJsonResponse(const Json::Value &data)\n{\n    auto res = std::make_shared<HttpResponseImpl>(k200OK, CT_APPLICATION_JSON);\n    res->setJsonObject(data);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpJsonResponse(Json::Value &&data)\n{\n    auto res = std::make_shared<HttpResponseImpl>(k200OK, CT_APPLICATION_JSON);\n    res->setJsonObject(std::move(data));\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nconst char *HttpResponseImpl::versionString() const\n{\n    const char *result = \"UNKNOWN\";\n    switch (version_)\n    {\n        case Version::kHttp10:\n            result = \"HTTP/1.0\";\n            break;\n\n        case Version::kHttp11:\n            result = \"HTTP/1.1\";\n            break;\n\n        default:\n            break;\n    }\n    return result;\n}\n\nvoid HttpResponseImpl::generateBodyFromJson() const\n{\n    if (!jsonPtr_ || flagForSerializingJson_)\n    {\n        return;\n    }\n    flagForSerializingJson_ = true;\n    static std::once_flag once;\n    static Json::StreamWriterBuilder builder;\n    std::call_once(once, []() {\n        builder[\"commentStyle\"] = \"None\";\n        builder[\"indentation\"] = \"\";\n        if (!app().isUnicodeEscapingUsedInJson())\n        {\n            builder[\"emitUTF8\"] = true;\n        }\n        auto &precision = app().getFloatPrecisionInJson();\n        if (precision.first != 0)\n        {\n            builder[\"precision\"] = precision.first;\n            builder[\"precisionType\"] = precision.second;\n        }\n    });\n    bodyPtr_ = std::make_shared<HttpMessageStringBody>(\n        writeString(builder, *jsonPtr_));\n}\n\nHttpResponsePtr HttpResponse::newNotFoundResponse(const HttpRequestPtr &req)\n{\n    auto loop = trantor::EventLoop::getEventLoopOfCurrentThread();\n    auto &resp = HttpAppFrameworkImpl::instance().getCustom404Page();\n    if (resp)\n    {\n        if (loop && loop->index() < app().getThreadNum())\n        {\n            return resp;\n        }\n        else\n        {\n            return HttpResponsePtr{new HttpResponseImpl(\n                *static_cast<HttpResponseImpl *>(resp.get()))};\n        }\n    }\n    else\n    {\n        if (HttpAppFrameworkImpl::instance().isUsingCustomErrorHandler())\n        {\n            return app().getCustomErrorHandler()(k404NotFound, req);\n        }\n        else if (loop && loop->index() < app().getThreadNum())\n        {\n            // If the current thread is an IO thread\n            static std::once_flag threadOnce;\n            static IOThreadStorage<HttpResponsePtr> thread404Pages;\n            std::call_once(threadOnce, [req = req] {\n                thread404Pages.init([req = req](drogon::HttpResponsePtr &resp,\n                                                size_t /*index*/) {\n                    HttpViewData data;\n                    data.insert(\"version\", drogon::getVersion());\n                    resp = HttpResponse::newHttpViewResponse(\"drogon::NotFound\",\n                                                             data);\n                    resp->setStatusCode(k404NotFound);\n                    resp->setExpiredTime(0);\n                });\n            });\n            LOG_TRACE << \"Use cached 404 response\";\n            return thread404Pages.getThreadData();\n        }\n        else\n        {\n            HttpViewData data;\n            data.insert(\"version\", drogon::getVersion());\n            auto notFoundResp =\n                HttpResponse::newHttpViewResponse(\"drogon::NotFound\", data);\n            notFoundResp->setStatusCode(k404NotFound);\n            return notFoundResp;\n        }\n    }\n}\n\nHttpResponsePtr HttpResponse::newRedirectionResponse(\n    const std::string &location,\n    HttpStatusCode status)\n{\n    auto res = std::make_shared<HttpResponseImpl>();\n    res->setStatusCode(status);\n    res->redirect(location);\n    AopAdvice::instance().passResponseCreationAdvices(res);\n    return res;\n}\n\nHttpResponsePtr HttpResponse::newHttpViewResponse(const std::string &viewName,\n                                                  const HttpViewData &data,\n                                                  const HttpRequestPtr &req)\n{\n    return genHttpResponse(viewName, data, req);\n}\n\nHttpResponsePtr HttpResponse::newFileResponse(\n    const unsigned char *pBuffer,\n    size_t bufferLength,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString)\n{\n    // Make Raw HttpResponse\n    auto resp = std::make_shared<HttpResponseImpl>();\n\n    // Set response body and length\n    resp->setBody(\n        std::string(reinterpret_cast<const char *>(pBuffer), bufferLength));\n\n    // Set status of message\n    resp->setStatusCode(k200OK);\n\n    // Check for type and assign proper content type in header\n    if (!typeString.empty())\n    {\n        // auto contentType = type;\n        if (type == CT_NONE)\n            type = parseContentType(typeString);\n        if (type == CT_NONE)\n            type = CT_APPLICATION_OCTET_STREAM;  // XXX: Is this Ok?\n        static_cast<HttpResponse *>(resp.get())\n            ->setContentTypeCodeAndCustomString(type,\n                                                typeString.c_str(),\n                                                typeString.size());\n    }\n    else if (type != CT_NONE)\n    {\n        resp->setContentTypeCode(type);\n    }\n    else if (!attachmentFileName.empty())\n    {\n        resp->setContentTypeCode(drogon::getContentType(attachmentFileName));\n    }\n    else\n    {\n        resp->setContentTypeCode(\n            CT_APPLICATION_OCTET_STREAM);  // default content-type for file;\n    }\n\n    // Add additional header values\n    if (!attachmentFileName.empty())\n    {\n        resp->addHeader(\"Content-Disposition\",\n                        \"attachment; filename=\" + attachmentFileName);\n    }\n\n    // Finalize and return response\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nHttpResponsePtr HttpResponse::newFileResponse(\n    const std::string &fullPath,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString,\n    const HttpRequestPtr &req)\n{\n    return newFileResponse(\n        fullPath, 0, 0, false, attachmentFileName, type, typeString, req);\n}\n\nHttpResponsePtr HttpResponse::newFileResponse(\n    const std::string &fullPath,\n    size_t offset,\n    size_t length,\n    bool setContentRange,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString,\n    const HttpRequestPtr &req)\n{\n    std::ifstream infile(utils::toNativePath(fullPath), std::ifstream::binary);\n    LOG_TRACE << \"send http file:\" << fullPath << \" offset \" << offset\n              << \" length \" << length;\n    if (!infile)\n    {\n        auto resp = HttpResponse::newNotFoundResponse(req);\n        return resp;\n    }\n    auto resp = std::make_shared<HttpResponseImpl>();\n    std::streambuf *pbuf = infile.rdbuf();\n    size_t filesize =\n        static_cast<size_t>(pbuf->pubseekoff(0, std::ifstream::end));\n    if (offset > filesize || length > filesize ||  // in case of overflow\n        offset + length > filesize)\n    {\n        resp->setStatusCode(k416RequestedRangeNotSatisfiable);\n        if (setContentRange)\n        {\n            char buf[64];\n            snprintf(buf, sizeof(buf), \"bytes */%zu\", filesize);\n            resp->addHeader(\"Content-Range\", std::string(buf));\n        }\n        return resp;\n    }\n    if (length == 0)\n    {\n        length = filesize - offset;\n    }\n    pbuf->pubseekoff(offset, std::ifstream::beg);  // rewind\n\n    if (HttpAppFrameworkImpl::instance().useSendfile() && length > 1024 * 200)\n    // TODO : Is 200k an appropriate value? Or set it to be configurable\n    {\n        // The advantages of sendfile() can only be reflected in sending large\n        // files.\n        resp->setSendfile(fullPath);\n        // Must set length with the right value! Content-Length header relies on\n        // this value.\n        resp->setSendfileRange(offset, length);\n    }\n    else\n    {\n        std::string str;\n        str.resize(length);\n        pbuf->sgetn(&str[0], length);\n        resp->setBody(std::move(str));\n        resp->setSendfileRange(offset, length);\n    }\n\n    // Set correct status code\n    if (length < filesize)\n    {\n        resp->setStatusCode(k206PartialContent);\n    }\n    else\n    {\n        resp->setStatusCode(k200OK);\n    }\n\n    // Infer content type\n    if (type == CT_NONE)\n    {\n        if (!typeString.empty())\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            // auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n        else if (!attachmentFileName.empty())\n        {\n            resp->setContentTypeCode(\n                drogon::getContentType(attachmentFileName));\n        }\n        else\n        {\n            resp->setContentTypeCode(drogon::getContentType(fullPath));\n        }\n    }\n    else\n    {\n        if (typeString.empty())\n            resp->setContentTypeCode(type);\n        else\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            // auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n    }\n\n    // Set headers\n    if (!attachmentFileName.empty())\n    {\n        resp->addHeader(\"Content-Disposition\",\n                        \"attachment; filename=\" + attachmentFileName);\n    }\n    if (setContentRange && length > 0)\n    {\n        char buf[128];\n        snprintf(buf,\n                 sizeof(buf),\n                 \"bytes %zu-%zu/%zu\",\n                 offset,\n                 offset + length - 1,\n                 filesize);\n        resp->addHeader(\"Content-Range\", std::string(buf));\n    }\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nHttpResponsePtr HttpResponse::newStreamResponse(\n    const std::function<std::size_t(char *, std::size_t)> &callback,\n    const std::string &attachmentFileName,\n    ContentType type,\n    const std::string &typeString,\n    const HttpRequestPtr &req)\n{\n    LOG_TRACE << \"send stream as \"s\n              << (attachmentFileName.empty() ? \"raw data\"s\n                                             : \"file: \"s + attachmentFileName);\n    if (!callback)\n    {\n        auto resp = HttpResponse::newNotFoundResponse();\n        return resp;\n    }\n    auto resp = std::make_shared<HttpResponseImpl>();\n    resp->setStreamCallback(callback);\n    resp->setStatusCode(k200OK);\n\n    // Infer content type\n    if (type == CT_NONE)\n    {\n        if (!typeString.empty())\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n        else if (!attachmentFileName.empty())\n        {\n            resp->setContentTypeCode(\n                drogon::getContentType(attachmentFileName));\n        }\n    }\n    else\n    {\n        if (typeString.empty())\n            resp->setContentTypeCode(type);\n        else\n        {\n            auto r = static_cast<HttpResponse *>(resp.get());\n            auto contentType = type;\n            if (type == CT_NONE)\n                type = parseContentType(typeString);\n            if (type == CT_NONE)\n                type = CT_CUSTOM;  // XXX: Is this Ok?\n            r->setContentTypeCodeAndCustomString(type, typeString);\n        }\n    }\n\n    // Set headers\n    if (!attachmentFileName.empty())\n    {\n        resp->addHeader(\"Content-Disposition\",\n                        \"attachment; filename=\" + attachmentFileName);\n    }\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nHttpResponsePtr HttpResponse::newAsyncStreamResponse(\n    const std::function<void(ResponseStreamPtr)> &callback,\n    bool disableKickoffTimeout)\n{\n    if (!callback)\n    {\n        auto resp = HttpResponse::newNotFoundResponse();\n        return resp;\n    }\n    auto resp = std::make_shared<HttpResponseImpl>();\n    resp->setAsyncStreamCallback(callback, disableKickoffTimeout);\n    resp->setStatusCode(k200OK);\n    AopAdvice::instance().passResponseCreationAdvices(resp);\n    return resp;\n}\n\nvoid HttpResponseImpl::makeHeaderString(trantor::MsgBuffer &buffer)\n{\n    buffer.ensureWritableBytes(128);\n    int len{0};\n    if (version_ == Version::kHttp11)\n    {\n        if (customStatusCode_ >= 0)\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.1 %d \",\n                           customStatusCode_);\n        }\n        else\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.1 %d \",\n                           statusCode_);\n        }\n    }\n    else\n    {\n        if (customStatusCode_ >= 0)\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.0 %d \",\n                           customStatusCode_);\n        }\n        else\n        {\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           \"HTTP/1.0 %d \",\n                           statusCode_);\n        }\n    }\n    buffer.hasWritten(len);\n\n    if (!statusMessage_.empty())\n        buffer.append(statusMessage_.data(), statusMessage_.length());\n    buffer.append(\"\\r\\n\");\n    generateBodyFromJson();\n    if (!passThrough_)\n    {\n        buffer.ensureWritableBytes(64);\n        if (streamCallback_ || asyncStreamCallback_)\n        {\n            // When the headers are created, it is time to set the transfer\n            // encoding to chunked if the contents size is not specified\n            if (!ifCloseConnection() &&\n                headers_.find(\"content-length\") == headers_.end())\n            {\n                LOG_DEBUG << \"send stream with transfer-encoding chunked\";\n                headers_[\"transfer-encoding\"] = \"chunked\";\n            }\n            len = 0;\n        }\n        else if (sendfileName_.empty())\n        {\n            auto bodyLength = bodyPtr_ ? bodyPtr_->length() : 0;\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           contentLengthFormatString<decltype(bodyLength)>(),\n                           bodyLength);\n        }\n        else\n        {\n            auto bodyLength = sendfileRange_.second;\n            len = snprintf(buffer.beginWrite(),\n                           buffer.writableBytes(),\n                           contentLengthFormatString<decltype(bodyLength)>(),\n                           bodyLength);\n        }\n        buffer.hasWritten(len);\n        if (headers_.find(\"connection\") == headers_.end())\n        {\n            if (closeConnection_)\n            {\n                buffer.append(\"connection: close\\r\\n\");\n            }\n            else if (version_ == Version::kHttp10)\n            {\n                buffer.append(\"connection: Keep-Alive\\r\\n\");\n            }\n        }\n\n        if (!contentTypeString_.empty())\n        {\n            buffer.append(\"content-type: \");\n            buffer.append(contentTypeString_);\n            buffer.append(\"\\r\\n\");\n        }\n        if (HttpAppFrameworkImpl::instance().sendServerHeader())\n        {\n            buffer.append(\n                HttpAppFrameworkImpl::instance().getServerHeaderString());\n        }\n    }\n\n    for (auto it = headers_.begin(); it != headers_.end(); ++it)\n    {\n        buffer.append(it->first);\n        buffer.append(\": \");\n        buffer.append(it->second);\n        buffer.append(\"\\r\\n\");\n    }\n}\n\nvoid HttpResponseImpl::renderToBuffer(trantor::MsgBuffer &buffer)\n{\n    if (expriedTime_ >= 0)\n    {\n        auto strPtr = renderToBuffer();\n        buffer.append(strPtr->peek(), strPtr->readableBytes());\n        return;\n    }\n\n    if (!fullHeaderString_)\n    {\n        makeHeaderString(buffer);\n    }\n    else\n    {\n        buffer.append(*fullHeaderString_);\n    }\n\n    // output cookies\n    if (!cookies_.empty())\n    {\n        for (auto it = cookies_.begin(); it != cookies_.end(); ++it)\n        {\n            buffer.append(it->second.cookieString());\n        }\n    }\n\n    // output Date header\n    if (!passThrough_ &&\n        drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n    {\n        buffer.append(\"date: \");\n        buffer.append(utils::getHttpFullDate(trantor::Date::date()),\n                      httpFullDateStringLength);\n        buffer.append(\"\\r\\n\\r\\n\");\n    }\n    else\n    {\n        buffer.append(\"\\r\\n\");\n    }\n    if (bodyPtr_)\n        buffer.append(bodyPtr_->data(), bodyPtr_->length());\n}\n\nstd::shared_ptr<trantor::MsgBuffer> HttpResponseImpl::renderToBuffer()\n{\n    if (expriedTime_ >= 0)\n    {\n        if (!passThrough_ &&\n            drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n        {\n            if (datePos_ != static_cast<size_t>(-1))\n            {\n                auto now = trantor::Date::now();\n                bool isDateChanged =\n                    ((now.microSecondsSinceEpoch() / MICRO_SECONDS_PRE_SEC) !=\n                     httpStringDate_);\n                assert(httpString_);\n                if (isDateChanged)\n                {\n                    httpStringDate_ =\n                        now.microSecondsSinceEpoch() / MICRO_SECONDS_PRE_SEC;\n                    auto newDate = utils::getHttpFullDate(now);\n\n                    httpString_ =\n                        std::make_shared<trantor::MsgBuffer>(*httpString_);\n                    memcpy((void *)&(*httpString_)[datePos_],\n                           newDate,\n                           httpFullDateStringLength);\n                    return httpString_;\n                }\n\n                return httpString_;\n            }\n        }\n        else\n        {\n            if (httpString_)\n                return httpString_;\n        }\n    }\n    auto httpString = std::make_shared<trantor::MsgBuffer>(256);\n    if (!fullHeaderString_)\n    {\n        makeHeaderString(*httpString);\n    }\n    else\n    {\n        httpString->append(*fullHeaderString_);\n    }\n\n    // output cookies\n    if (!cookies_.empty())\n    {\n        for (auto it = cookies_.begin(); it != cookies_.end(); ++it)\n        {\n            httpString->append(it->second.cookieString());\n        }\n    }\n\n    // output Date header\n    if (!passThrough_ &&\n        drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n    {\n        httpString->append(\"date: \");\n        auto datePos = httpString->readableBytes();\n        httpString->append(utils::getHttpFullDate(trantor::Date::date()),\n                           httpFullDateStringLength);\n        httpString->append(\"\\r\\n\\r\\n\");\n        datePos_ = datePos;\n    }\n    else\n    {\n        httpString->append(\"\\r\\n\");\n    }\n\n    LOG_TRACE << \"response(no body):\"\n              << std::string_view{httpString->peek(),\n                                  httpString->readableBytes()};\n    if (bodyPtr_)\n        httpString->append(bodyPtr_->data(), bodyPtr_->length());\n    if (expriedTime_ >= 0)\n    {\n        httpString_ = httpString;\n    }\n    return httpString;\n}\n\nstd::shared_ptr<trantor::MsgBuffer> HttpResponseImpl::\n    renderHeaderForHeadMethod()\n{\n    auto httpString = std::make_shared<trantor::MsgBuffer>(256);\n    if (!fullHeaderString_)\n    {\n        makeHeaderString(*httpString);\n    }\n    else\n    {\n        httpString->append(*fullHeaderString_);\n    }\n\n    // output cookies\n    if (!cookies_.empty())\n    {\n        for (auto it = cookies_.begin(); it != cookies_.end(); ++it)\n        {\n            httpString->append(it->second.cookieString());\n        }\n    }\n\n    // output Date header\n    if (!passThrough_ &&\n        drogon::HttpAppFrameworkImpl::instance().sendDateHeader())\n    {\n        httpString->append(\"date: \");\n        httpString->append(utils::getHttpFullDate(trantor::Date::date()),\n                           httpFullDateStringLength);\n        httpString->append(\"\\r\\n\\r\\n\");\n    }\n    else\n    {\n        httpString->append(\"\\r\\n\");\n    }\n\n    return httpString;\n}\n\nvoid HttpResponseImpl::addHeader(const char *start,\n                                 const char *colon,\n                                 const char *end)\n{\n    fullHeaderString_.reset();\n    std::string field(start, colon);\n    transform(field.begin(), field.end(), field.begin(), [](unsigned char c) {\n        return tolower(c);\n    });\n    ++colon;\n    while (colon < end && isspace(static_cast<unsigned char>(*colon)))\n    {\n        ++colon;\n    }\n    std::string value(colon, end);\n    while (!value.empty() &&\n           isspace(static_cast<unsigned char>(value[value.size() - 1])))\n    {\n        value.resize(value.size() - 1);\n    }\n\n    if (field == \"set-cookie\")\n    {\n        // LOG_INFO<<\"cookies!!!:\"<<value;\n        auto values = utils::splitString(value, \";\");\n        Cookie cookie;\n        cookie.setHttpOnly(false);\n        for (size_t i = 0; i < values.size(); ++i)\n        {\n            std::string &coo = values[i];\n            std::string cookie_name;\n            std::string cookie_value;\n            auto epos = coo.find('=');\n            if (epos != std::string::npos)\n            {\n                cookie_name = coo.substr(0, epos);\n                std::string::size_type cpos = 0;\n                while (cpos < cookie_name.length() &&\n                       isspace(static_cast<unsigned char>(cookie_name[cpos])))\n                    ++cpos;\n                cookie_name = cookie_name.substr(cpos);\n                ++epos;\n                while (epos < coo.length() &&\n                       isspace(static_cast<unsigned char>(coo[epos])))\n                    ++epos;\n                cookie_value = coo.substr(epos);\n            }\n            else\n            {\n                std::string::size_type cpos = 0;\n                while (cpos < coo.length() &&\n                       isspace(static_cast<unsigned char>(coo[cpos])))\n                    ++cpos;\n                cookie_name = coo.substr(cpos);\n            }\n            if (i == 0)\n            {\n                cookie.setKey(cookie_name);\n                cookie.setValue(cookie_value);\n            }\n            else\n            {\n                std::transform(cookie_name.begin(),\n                               cookie_name.end(),\n                               cookie_name.begin(),\n                               [](unsigned char c) { return tolower(c); });\n                if (cookie_name == \"path\")\n                {\n                    cookie.setPath(cookie_value);\n                }\n                else if (cookie_name == \"domain\")\n                {\n                    cookie.setDomain(cookie_value);\n                }\n                else if (cookie_name == \"expires\")\n                {\n                    cookie.setExpiresDate(utils::getHttpDate(cookie_value));\n                }\n                else if (cookie_name == \"secure\")\n                {\n                    cookie.setSecure(true);\n                }\n                else if (cookie_name == \"httponly\")\n                {\n                    cookie.setHttpOnly(true);\n                }\n                else if (cookie_name == \"samesite\")\n                {\n                    cookie.setSameSite(\n                        cookie.convertString2SameSite(cookie_value));\n                }\n                else if (cookie_name == \"max-age\")\n                {\n                    cookie.setMaxAge(std::stoi(cookie_value));\n                }\n            }\n        }\n        if (!cookie.key().empty())\n        {\n            cookies_[cookie.key()] = cookie;\n        }\n    }\n    else\n    {\n        headers_[std::move(field)] = std::move(value);\n    }\n}\n\nvoid HttpResponseImpl::swap(HttpResponseImpl &that) noexcept\n{\n    using std::swap;\n    headers_.swap(that.headers_);\n    cookies_.swap(that.cookies_);\n    swap(statusCode_, that.statusCode_);\n    swap(version_, that.version_);\n    swap(statusMessage_, that.statusMessage_);\n    swap(closeConnection_, that.closeConnection_);\n    bodyPtr_.swap(that.bodyPtr_);\n    swap(contentType_, that.contentType_);\n    swap(flagForParsingContentType_, that.flagForParsingContentType_);\n    swap(flagForParsingJson_, that.flagForParsingJson_);\n    swap(sendfileName_, that.sendfileName_);\n    swap(streamCallback_, that.streamCallback_);\n    swap(asyncStreamCallback_, that.asyncStreamCallback_);\n    jsonPtr_.swap(that.jsonPtr_);\n    fullHeaderString_.swap(that.fullHeaderString_);\n    httpString_.swap(that.httpString_);\n    swap(datePos_, that.datePos_);\n    swap(jsonParsingErrorPtr_, that.jsonParsingErrorPtr_);\n}\n\nvoid HttpResponseImpl::clear()\n{\n    statusCode_ = kUnknown;\n    version_ = Version::kHttp11;\n    statusMessage_ = std::string_view{};\n    fullHeaderString_.reset();\n    jsonParsingErrorPtr_.reset();\n    sendfileName_.clear();\n    if (streamCallback_)\n    {\n        LOG_TRACE << \"Cleanup HttpResponse stream callback\";\n        streamCallback_(nullptr, 0);  // callback internal cleanup\n        streamCallback_ = {};\n    }\n    if (asyncStreamCallback_)\n    {\n        // asyncStreamCallback_(nullptr);\n        asyncStreamCallback_ = {};\n    }\n    headers_.clear();\n    cookies_.clear();\n    bodyPtr_.reset();\n    jsonPtr_.reset();\n    expriedTime_ = -1;\n    datePos_ = std::string::npos;\n    flagForParsingContentType_ = false;\n    flagForParsingJson_ = false;\n}\n\nvoid HttpResponseImpl::parseJson() const\n{\n    static std::once_flag once;\n    static Json::CharReaderBuilder builder;\n    std::call_once(once, []() {\n        builder[\"collectComments\"] = false;\n        builder[\"stackLimit\"] =\n            static_cast<Json::UInt>(drogon::app().getJsonParserStackLimit());\n    });\n    JSONCPP_STRING errs;\n    std::unique_ptr<Json::CharReader> reader(builder.newCharReader());\n    if (bodyPtr_)\n    {\n        jsonPtr_ = std::make_shared<Json::Value>();\n        if (!reader->parse(bodyPtr_->data(),\n                           bodyPtr_->data() + bodyPtr_->length(),\n                           jsonPtr_.get(),\n                           &errs))\n        {\n            LOG_ERROR << errs;\n            LOG_ERROR << \"body: \" << bodyPtr_->getString();\n            jsonPtr_.reset();\n            jsonParsingErrorPtr_ =\n                std::make_shared<std::string>(std::move(errs));\n        }\n        else\n        {\n            jsonParsingErrorPtr_.reset();\n        }\n    }\n    else\n    {\n        jsonPtr_.reset();\n        jsonParsingErrorPtr_ =\n            std::make_shared<std::string>(\"empty response body\");\n    }\n}\n\nbool HttpResponseImpl::shouldBeCompressed() const\n{\n    if (streamCallback_ || asyncStreamCallback_ || !sendfileName_.empty() ||\n        contentType() >= CT_APPLICATION_OCTET_STREAM ||\n        getBody().length() < 1024 || !(getHeaderBy(\"content-encoding\").empty()))\n    {\n        return false;\n    }\n    return true;\n}\n\nvoid HttpResponseImpl::setContentTypeString(const char *typeString,\n                                            size_t typeStringLength)\n{\n    std::string sv(typeString, typeStringLength);\n    auto contentType = parseContentType(sv);\n    if (contentType == CT_NONE)\n        contentType = CT_CUSTOM;\n    contentType_ = contentType;\n    contentTypeString_ = std::string(sv);\n    flagForParsingContentType_ = true;\n}\n\n// Path: lib/src/ConfigLoader.h\n/**\n *\n *  ConfigLoader.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <json/json.h>\n#include <string>\n#include <trantor/utils/NonCopyable.h>\n\nnamespace drogon\n{\nclass ConfigLoader : public trantor::NonCopyable\n{\n  public:\n    explicit ConfigLoader(const std::string &configFile) noexcept(false);\n    explicit ConfigLoader(const Json::Value &data);\n    explicit ConfigLoader(Json::Value &&data);\n    ~ConfigLoader();\n\n    const Json::Value &jsonValue() const\n    {\n        return configJsonRoot_;\n    }\n\n    void load() noexcept(false);\n\n  private:\n    std::string configFile_;\n    Json::Value configJsonRoot_;\n};\n}  // namespace drogon\n\n// Path: lib/src/HttpServer.h\n/**\n *\n *  @file HttpServer.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <trantor/net/TcpServer.h>\n#include <trantor/utils/NonCopyable.h>\n#include <functional>\n#include <string>\n#include <vector>\n#include \"impl_forwards.h\"\n\nstruct CallbackParamPack;\n\nnamespace drogon\n{\nclass ControllerBinderBase;\n\nclass HttpServer : trantor::NonCopyable\n{\n  public:\n    HttpServer(trantor::EventLoop *loop,\n               const trantor::InetAddress &listenAddr,\n               std::string name);\n\n    ~HttpServer();\n\n    void setIoLoops(const std::vector<trantor::EventLoop *> &ioLoops)\n    {\n        server_.setIoLoops(ioLoops);\n    }\n\n    void start();\n    void stop();\n\n    void enableSSL(trantor::TLSPolicyPtr policy)\n    {\n        server_.enableSSL(std::move(policy));\n    }\n\n    const trantor::InetAddress &address() const\n    {\n        return server_.address();\n    }\n\n  private:\n    friend class HttpInternalForwardHelper;\n\n    static void onConnection(const trantor::TcpConnectionPtr &conn);\n    static void onMessage(const trantor::TcpConnectionPtr &,\n                          trantor::MsgBuffer *);\n    static void onRequests(const trantor::TcpConnectionPtr &,\n                           const std::vector<HttpRequestImplPtr> &,\n                           const std::shared_ptr<HttpRequestParser> &);\n\n    struct HttpRequestParamPack\n    {\n        std::shared_ptr<ControllerBinderBase> binderPtr;\n        std::function<void(const HttpResponsePtr &)> callback;\n    };\n\n    struct WsRequestParamPack\n    {\n        std::shared_ptr<ControllerBinderBase> binderPtr;\n        std::function<void(const HttpResponsePtr &)> callback;\n        WebSocketConnectionImplPtr wsConnPtr;\n    };\n\n    // Http request handling steps\n    static void onHttpRequest(const HttpRequestImplPtr &,\n                              std::function<void(const HttpResponsePtr &)> &&);\n    static void httpRequestRouting(\n        const HttpRequestImplPtr &req,\n        std::function<void(const HttpResponsePtr &)> &&callback);\n    static void httpRequestHandling(\n        const HttpRequestImplPtr &req,\n        std::shared_ptr<ControllerBinderBase> &&binderPtr,\n        std::function<void(const HttpResponsePtr &)> &&callback);\n\n    // Websocket request handling steps\n    static void onWebsocketRequest(\n        const HttpRequestImplPtr &,\n        std::function<void(const HttpResponsePtr &)> &&,\n        WebSocketConnectionImplPtr &&);\n    static void websocketRequestRouting(\n        const HttpRequestImplPtr &req,\n        std::function<void(const HttpResponsePtr &)> &&callback,\n        WebSocketConnectionImplPtr &&wsConnPtr);\n    static void websocketRequestHandling(\n        const HttpRequestImplPtr &req,\n        std::shared_ptr<ControllerBinderBase> &&binderPtr,\n        std::function<void(const HttpResponsePtr &)> &&callback,\n        WebSocketConnectionImplPtr &&wsConnPtr);\n\n    // Http/Websocket shared handling steps\n    template <typename Pack>\n    static void requestPostRouting(const HttpRequestImplPtr &req, Pack &&pack);\n    template <typename Pack>\n    static void requestPassFilters(const HttpRequestImplPtr &req, Pack &&pack);\n    template <typename Pack>\n    static void requestPreHandling(const HttpRequestImplPtr &req, Pack &&pack);\n\n    // Response buffering and sending\n    static void handleResponse(\n        const HttpResponsePtr &response,\n        const std::shared_ptr<CallbackParamPack> &paramPack,\n        bool *respReadyPtr);\n    static void sendResponse(const trantor::TcpConnectionPtr &,\n                             const HttpResponsePtr &,\n                             bool isHeadMethod);\n    static void sendResponses(\n        const trantor::TcpConnectionPtr &conn,\n        const std::vector<std::pair<HttpResponsePtr, bool>> &responses,\n        trantor::MsgBuffer &buffer);\n\n    trantor::TcpServer server_;\n};\n\nclass HttpInternalForwardHelper\n{\n  public:\n    static void forward(const HttpRequestImplPtr &req,\n                        std::function<void(const HttpResponsePtr &)> &&callback)\n    {\n        return HttpServer::onHttpRequest(req, std::move(callback));\n    }\n};\n\n}  // namespace drogon\n\n// Path: lib/src/ListenerManager.h\n/**\n *\n *  @file ListenerManager.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <trantor/net/EventLoopThreadPool.h>\n#include <trantor/net/callbacks.h>\n#include <trantor/utils/NonCopyable.h>\n#include <memory>\n#include <string>\n#include <utility>\n#include <vector>\n#include \"impl_forwards.h\"\n\nnamespace trantor\n{\nclass InetAddress;\n}\n\nnamespace drogon\n{\nclass ListenerManager : public trantor::NonCopyable\n{\n  public:\n    ~ListenerManager() = default;\n    void addListener(const std::string &ip,\n                     uint16_t port,\n                     bool useSSL = false,\n                     const std::string &certFile = \"\",\n                     const std::string &keyFile = \"\",\n                     bool useOldTLS = false,\n                     const std::vector<std::pair<std::string, std::string>>\n                         &sslConfCmds = {});\n    std::vector<trantor::InetAddress> getListeners() const;\n    void createListeners(\n        const std::string &globalCertFile,\n        const std::string &globalKeyFile,\n        const std::vector<std::pair<std::string, std::string>> &sslConfCmds,\n        const std::vector<trantor::EventLoop *> &ioLoops);\n    void startListening();\n    void stopListening();\n\n  private:\n    struct ListenerInfo\n    {\n        \nListenerInfo(\n            std::string ip,\n            uint16_t port,\n            bool useSSL,\n            std::string certFile,\n            std::string keyFile,\n            bool useOldTLS,\n            std::vector<std::pair<std::string, std::string>> sslConfCmds)\n            : ip_(std::move(ip)),\n              port_(port),\n              useSSL_(useSSL),\n              certFile_(std::move(certFile)),\n              keyFile_(std::move(keyFile)),\n              useOldTLS_(useOldTLS),\n              sslConfCmds_(std::move(sslConfCmds))\n        {\n        }\n\n        std::string ip_;\n        uint16_t port_;\n        bool useSSL_;\n        std::string certFile_;\n        std::string keyFile_;\n        bool useOldTLS_;\n        std::vector<std::pair<std::string, std::string>> sslConfCmds_;\n    };\n\n    std::vector<ListenerInfo> listeners_;\n    std::vector<std::shared_ptr<HttpServer>> servers_;\n\n    // should have value when and only when on OS that one port can only be\n    // listened by one thread\n    std::unique_ptr<trantor::EventLoopThread> listeningThread_;\n};\n\n}  // namespace drogon\n\n// Path: lib/src/RedisClientManager.h\n/**\n *\n *  RedisClientManager.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <drogon/nosql/RedisClient.h>\n#include <drogon/HttpAppFramework.h>\n#include <drogon/IOThreadStorage.h>\n#include <trantor/utils/NonCopyable.h>\n#include <trantor/net/EventLoop.h>\n#include <string>\n#include <memory>\n\nnamespace drogon\n{\nnamespace nosql\n{\nclass RedisClientManager : public trantor::NonCopyable\n{\n  public:\n    void createRedisClients(const std::vector<trantor::EventLoop *> &ioLoops);\n\n    RedisClientPtr getRedisClient(const std::string &name)\n    {\n        assert(redisClientsMap_.find(name) != redisClientsMap_.end());\n        return redisClientsMap_[name];\n    }\n\n    RedisClientPtr getFastRedisClient(const std::string &name)\n    {\n        auto iter = redisFastClientsMap_.find(name);\n        assert(iter != redisFastClientsMap_.end());\n        return iter->second.getThreadData();\n    }\n\n    void createRedisClient(const std::string &name,\n                           const std::string &host,\n                           unsigned short port,\n                           const std::string &username,\n                           const std::string &password,\n                           size_t connectionNum,\n                           bool isFast,\n                           double timeout,\n                           unsigned int db);\n    // bool areAllRedisClientsAvailable() const noexcept;\n\n    ~RedisClientManager();\n\n  private:\n    std::map<std::string, RedisClientPtr> redisClientsMap_;\n    std::map<std::string, IOThreadStorage<RedisClientPtr>> redisFastClientsMap_;\n\n    struct RedisInfo\n    {\n        std::string name_;\n        std::string addr_;\n        std::string username_;\n        std::string password_;\n        unsigned short port_;\n        bool isFast_;\n        size_t connectionNumber_;\n        double timeout_;\n        unsigned int db_;\n    };\n\n    std::vector<RedisInfo> redisInfos_;\n};\n}  // namespace nosql\n}  // namespace drogon\n\n// Path: lib/src/SharedLibManager.h\n/**\n *\n *  SharedLibManager.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <trantor/net/EventLoopThread.h>\n#include <trantor/utils/NonCopyable.h>\n#include <unordered_map>\n#include <vector>\n#include <sys/stat.h>\n\nnamespace drogon\n{\nclass SharedLibManager : public trantor::NonCopyable\n{\n  public:\n    SharedLibManager(const std::vector<std::string> &libPaths,\n                     const std::string &outputPath);\n    ~SharedLibManager();\n\n  private:\n    void managerLibs();\n    std::vector<std::string> libPaths_;\n    std::string outputPath_;\n\n    struct DLStat\n    {\n        void *handle{nullptr};\n        struct timespec mTime = {0, 0};\n    };\n\n    std::unordered_map<std::string, DLStat> dlMap_;\n    void *compileAndLoadLib(const std::string &sourceFile, void *oldHld);\n    void *loadLib(const std::string &soFile, void *oldHld);\n    bool shouldCompileLib(const std::string &soFile,\n                          const struct stat &sourceStat);\n    trantor::TimerId timeId_;\n    trantor::EventLoopThread workingThread_;\n};\n}  // namespace drogon\n\n// Path: lib/src/HttpAppFrameworkImpl.cc\n/**\n *\n *  @file HttpAppFrameworkImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/DrClassMap.h>\n#include <drogon/HttpResponse.h>\n#include <drogon/HttpTypes.h>\n#include <drogon/utils/Utilities.h>\n#include <drogon/version.h>\n#include <json/json.h>\n#include <trantor/utils/AsyncFileLogger.h>\n#include <algorithm>\n#include \"AOPAdvice.h\"\n#include \"ConfigLoader.h\"\n#include \"DbClientManager.h\"\n#include \"HttpClientImpl.h\"\n#include \"HttpConnectionLimit.h\"\n#include \"HttpControllersRouter.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpResponseImpl.h\"\n#include \"HttpServer.h\"\n#include \"HttpUtils.h\"\n#include \"ListenerManager.h\"\n#include \"PluginsManager.h\"\n#include \"RedisClientManager.h\"\n#include \"SessionManager.h\"\n#include \"SharedLibManager.h\"\n#include \"StaticFileRouter.h\"\n\n#include <iostream>\n#include <memory>\n#include <tuple>\n#include <unordered_map>\n#include <utility>\n#include <filesystem>\n\n#include <fcntl.h>\n#include <sys/stat.h>\n#ifndef _WIN32\n#include <sys/wait.h>\n#include <unistd.h>\n#define os_access access\n#elif !defined(_WIN32) || defined(__MINGW32__)\n#include <sys/file.h>\n#include <unistd.h>\n#define os_access access\n#else\n#include <io.h>\n#define os_access _waccess\n#define R_OK 04\n#define W_OK 02\n#endif\n\n#ifdef DROGON_SPDLOG_SUPPORT\n#include <spdlog/spdlog.h>\n#include <spdlog/logger.h>\n#include <spdlog/sinks/stdout_color_sinks.h>\n#include <spdlog/sinks/rotating_file_sink.h>\n#ifdef _WIN32\n#include <spdlog/sinks/msvc_sink.h>\n// Damn antedeluvian M$ macros\n#undef min\n#undef max\n#endif\n#endif  // DROGON_SPDLOG_SUPPORT\n\nusing namespace drogon;\nusing namespace std::placeholders;\n\nHttpAppFrameworkImpl::HttpAppFrameworkImpl()\n    : listenerManagerPtr_(new ListenerManager),\n      pluginsManagerPtr_(new PluginsManager),\n      dbClientManagerPtr_(new orm::DbClientManager),\n      redisClientManagerPtr_(new nosql::RedisClientManager),\n      uploadPath_(rootPath_ + \"uploads\")\n{\n}\n\nstatic std::function<void()> f = [] {\n    LOG_TRACE << \"Initialize the main event loop in the main thread\";\n};\n\n/// Make sure that the main event loop is initialized in the main thread.\ndrogon::InitBeforeMainFunction drogon::HttpAppFrameworkImpl::initFirst_([]() {\n    HttpAppFrameworkImpl::instance().getLoop()->runInLoop(f);\n});\n\nnamespace drogon\n{\nstd::string getVersion()\n{\n    return DROGON_VERSION;\n}\n\nstd::string getGitCommit()\n{\n    return DROGON_VERSION_SHA1;\n}\n\nHttpResponsePtr defaultErrorHandler(HttpStatusCode code, const HttpRequestPtr &)\n{\n    return std::make_shared<HttpResponseImpl>(code, CT_TEXT_HTML);\n}\n\nvoid defaultExceptionHandler(\n    const std::exception &e,\n    const HttpRequestPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    std::string pathWithQuery = req->path();\n    if (req->query().empty() == false)\n        pathWithQuery += \"?\" + req->query();\n    LOG_ERROR << \"Unhandled exception in \" << pathWithQuery\n              << \", what(): \" << e.what();\n    const auto &handler = app().getCustomErrorHandler();\n    callback(handler(k500InternalServerError, req));\n}\n\nstatic void godaemon()\n{\n    printf(\"Initializing daemon mode\\n\");\n#ifndef _WIN32\n    if (getppid() != 1)\n    {\n        pid_t pid;\n        pid = fork();\n        if (pid > 0)\n            exit(0);  // parent\n        if (pid < 0)\n        {\n            perror(\"fork\");\n            exit(1);\n        }\n        setsid();\n    }\n\n    // redirect stdin/stdout/stderr to /dev/null\n    close(0);\n    close(1);\n    close(2);\n\n    int ret = open(\"/dev/null\", O_RDWR);\n    (void)ret;\n    ret = dup(0);\n    (void)ret;\n    ret = dup(0);\n    (void)ret;\n    umask(0);\n#else\n    LOG_ERROR << \"Cannot run as daemon in Windows\";\n    exit(1);\n#endif\n\n    return;\n}\n\nstatic void TERMFunction(int sig)\n{\n    if (sig == SIGTERM)\n    {\n        LOG_WARN << \"SIGTERM signal received.\";\n        HttpAppFrameworkImpl::instance().getTermSignalHandler()();\n    }\n    else if (sig == SIGINT)\n    {\n        LOG_WARN << \"SIGINT signal received.\";\n        HttpAppFrameworkImpl::instance().getIntSignalHandler()();\n    }\n}\n\n}  // namespace drogon\n\nHttpAppFrameworkImpl::~HttpAppFrameworkImpl() noexcept\n{\n// Destroy the following objects before the loop destruction\n#ifndef _WIN32\n    sharedLibManagerPtr_.reset();\n#endif\n    sessionManagerPtr_.reset();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setStaticFilesCacheTime(int cacheTime)\n{\n    StaticFileRouter::instance().setStaticFilesCacheTime(cacheTime);\n    return *this;\n}\n\nint HttpAppFrameworkImpl::staticFilesCacheTime() const\n{\n    return StaticFileRouter::instance().staticFilesCacheTime();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setGzipStatic(bool useGzipStatic)\n{\n    StaticFileRouter::instance().setGzipStatic(useGzipStatic);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setBrStatic(bool useGzipStatic)\n{\n    StaticFileRouter::instance().setBrStatic(useGzipStatic);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setImplicitPageEnable(\n    bool useImplicitPage)\n{\n    StaticFileRouter::instance().setImplicitPageEnable(useImplicitPage);\n    return *this;\n}\n\nbool HttpAppFrameworkImpl::isImplicitPageEnabled() const\n{\n    return StaticFileRouter::instance().isImplicitPageEnabled();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setImplicitPage(\n    const std::string &implicitPageFile)\n{\n    StaticFileRouter::instance().setImplicitPage(implicitPageFile);\n    return *this;\n}\n\nconst std::string &HttpAppFrameworkImpl::getImplicitPage() const\n{\n    return StaticFileRouter::instance().getImplicitPage();\n}\n#ifndef _WIN32\nHttpAppFramework &HttpAppFrameworkImpl::enableDynamicViewsLoading(\n    const std::vector<std::string> &libPaths,\n    const std::string &outputPath)\n{\n    assert(!running_);\n\n    for (auto const &libpath : libPaths)\n    {\n        if (libpath[0] == '/' ||\n            (libpath.length() >= 2 && libpath[0] == '.' && libpath[1] == '/') ||\n            (libpath.length() >= 3 && libpath[0] == '.' && libpath[1] == '.' &&\n             libpath[2] == '/') ||\n            libpath == \".\" || libpath == \"..\")\n        {\n            libFilePaths_.push_back(libpath);\n        }\n        else\n        {\n            if (rootPath_[rootPath_.length() - 1] == '/')\n                libFilePaths_.push_back(rootPath_ + libpath);\n            else\n                libFilePaths_.push_back(rootPath_ + \"/\" + libpath);\n        }\n    }\n    libFileOutputPath_ = outputPath;\n    if (!libFileOutputPath_.empty())\n    {\n        if (drogon::utils::createPath(libFileOutputPath_) == -1)\n        {\n            LOG_FATAL << \"Can't create \" << libFileOutputPath_\n                      << \" path for dynamic views\";\n            exit(-1);\n        }\n    }\n\n    return *this;\n}\n#endif\nHttpAppFramework &HttpAppFrameworkImpl::setFileTypes(\n    const std::vector<std::string> &types)\n{\n    StaticFileRouter::instance().setFileTypes(types);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::registerWebSocketController(\n    const std::string &pathName,\n    const std::string &ctrlName,\n    const std::vector<internal::HttpConstraint> &filtersAndMethods)\n{\n    assert(!routersInit_);\n    HttpControllersRouter::instance().registerWebSocketController(\n        pathName, ctrlName, filtersAndMethods);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::registerHttpSimpleController(\n    const std::string &pathName,\n    const std::string &ctrlName,\n    const std::vector<internal::HttpConstraint> &filtersAndMethods)\n{\n    assert(!routersInit_);\n    HttpControllersRouter::instance().registerHttpSimpleController(\n        pathName, ctrlName, filtersAndMethods);\n    return *this;\n}\n\nvoid HttpAppFrameworkImpl::registerHttpController(\n    const std::string &pathPattern,\n    const internal::HttpBinderBasePtr &binder,\n    const std::vector<HttpMethod> &validMethods,\n    const std::vector<std::string> &filters,\n    const std::string &handlerName)\n{\n    assert(!pathPattern.empty());\n    assert(binder);\n    assert(!routersInit_);\n    HttpControllersRouter::instance().addHttpPath(\n        pathPattern, binder, validMethods, filters, handlerName);\n}\n\nvoid HttpAppFrameworkImpl::registerHttpControllerViaRegex(\n    const std::string &regExp,\n    const internal::HttpBinderBasePtr &binder,\n    const std::vector<HttpMethod> &validMethods,\n    const std::vector<std::string> &filters,\n    const std::string &handlerName)\n{\n    assert(!regExp.empty());\n    assert(binder);\n    assert(!routersInit_);\n    HttpControllersRouter::instance().addHttpRegex(\n        regExp, binder, validMethods, filters, handlerName);\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setThreadNum(size_t threadNum)\n{\n    if (threadNum == 0)\n    {\n        threadNum_ = std::thread::hardware_concurrency();\n        return *this;\n    }\n    threadNum_ = threadNum;\n    return *this;\n}\n\nPluginBase *HttpAppFrameworkImpl::getPlugin(const std::string &name)\n{\n    return pluginsManagerPtr_->getPlugin(name);\n}\n\nstd::shared_ptr<PluginBase> HttpAppFrameworkImpl::getSharedPlugin(\n    const std::string &name)\n{\n    return pluginsManagerPtr_->getSharedPlugin(name);\n}\n\nvoid HttpAppFrameworkImpl::addPlugin(\n    const std::string &name,\n    const std::vector<std::string> &dependencies,\n    const Json::Value &config)\n{\n    assert(!isRunning());\n    Json::Value pluginConfig;\n    pluginConfig[\"name\"] = name;\n    Json::Value deps(Json::arrayValue);\n    for (const auto dep : dependencies)\n    {\n        deps.append(dep);\n    }\n    pluginConfig[\"dependencies\"] = deps;\n    pluginConfig[\"config\"] = config;\n    auto &plugins = jsonRuntimeConfig_[\"plugins\"];\n    plugins.append(pluginConfig);\n}\n\nvoid HttpAppFrameworkImpl::addPlugins(const Json::Value &configs)\n{\n    assert(!isRunning());\n    assert(configs.isArray());\n    auto &plugins = jsonRuntimeConfig_[\"plugins\"];\n    for (const auto config : configs)\n    {\n        plugins.append(config);\n    }\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::addListener(\n    const std::string &ip,\n    uint16_t port,\n    bool useSSL,\n    const std::string &certFile,\n    const std::string &keyFile,\n    bool useOldTLS,\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n{\n    assert(!running_);\n    listenerManagerPtr_->addListener(\n        ip, port, useSSL, certFile, keyFile, useOldTLS, sslConfCmds);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setMaxConnectionNum(\n    size_t maxConnections)\n{\n    HttpConnectionLimit::instance().setMaxConnectionNum(maxConnections);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setMaxConnectionNumPerIP(\n    size_t maxConnectionsPerIP)\n{\n    HttpConnectionLimit::instance().setMaxConnectionNumPerIP(\n        maxConnectionsPerIP);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::loadConfigFile(\n    const std::string &fileName)\n{\n    ConfigLoader loader(fileName);\n    loader.load();\n    jsonConfig_ = loader.jsonValue();\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::loadConfigJson(const Json::Value &data)\n{\n    ConfigLoader loader(data);\n    loader.load();\n    jsonConfig_ = loader.jsonValue();\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::loadConfigJson(Json::Value &&data)\n{\n    ConfigLoader loader(std::move(data));\n    loader.load();\n    jsonConfig_ = loader.jsonValue();\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setLogPath(\n    const std::string &logPath,\n    const std::string &logfileBaseName,\n    size_t logfileSize,\n    size_t maxFiles,\n    bool useSpdlog)\n{\n#ifdef DROGON_SPDLOG_SUPPORT\n    logWithSpdlog_ = trantor::Logger::hasSpdLogSupport() && useSpdlog;\n#endif\n    if (logPath.empty())\n        return *this;\n    // std::filesystem does not provide a method to check access permissions, so\n    // keep existing code\n    if (os_access(utils::toNativePath(logPath).c_str(), 0) != 0)\n    {\n        std::cerr << \"Log path does not exist!\\n\";\n        exit(1);\n    }\n    if (os_access(utils::toNativePath(logPath).c_str(), R_OK | W_OK) != 0)\n    {\n        std::cerr << \"Unable to access log path!\\n\";\n        exit(1);\n    }\n    logPath_ = logPath;\n    logfileBaseName_ = logfileBaseName;\n    logfileSize_ = logfileSize;\n    logfileMaxNum_ = maxFiles;\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setLogLevel(\n    trantor::Logger::LogLevel level)\n{\n    trantor::Logger::setLogLevel(level);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setLogLocalTime(bool on)\n{\n    trantor::Logger::setDisplayLocalTime(on);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setSSLConfigCommands(\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n{\n    sslConfCmds_ = sslConfCmds;\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setSSLFiles(const std::string &certPath,\n                                                    const std::string &keyPath)\n{\n    sslCertPath_ = certPath;\n    sslKeyPath_ = keyPath;\n    return *this;\n}\n\nvoid HttpAppFrameworkImpl::run()\n{\n    if (!getLoop()->isInLoopThread())\n    {\n        getLoop()->moveToCurrentThread();\n    }\n    LOG_TRACE << \"Start to run...\";\n    // Create dirs for cache files\n...\n", "needle_token_start": 10425, "needle_token_end": 10581, "code_context_ntokens": 16125, "output": ["```cpp\nvoid ListenerManager::addListener(const std::string &ip,\n                                 uint16_t port,\n                                 bool useSSL,\n                                 const std::string &certFile,\n                                 const std::string &keyFile,\n                                 bool useOldTLS,\n                                 const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n{\n    listeners_.emplace_back(ip, port, useSSL, certFile, keyFile, useOldTLS, sslConfCmds);\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "handleException", "language": "cpp", "path": "lib/src/HttpBinder.cc", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To manage exceptions that occur during HTTP request handling by delegating the exception handling to a framework-specific handler.\n2. **Input**: The function takes three parameters: an exception reference, an HTTP request pointer, and a callback function.\n3. **Output**: There is no direct output; however, the exception handling response is managed through the callback function.\n4. **Procedure**: The function invokes a registered exception handler from the application framework, passing it the exception, the HTTP request, and the callback function to handle the response appropriately.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/HttpAppFrameworkImpl.cc\n/**\n *\n *  @file HttpAppFrameworkImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/DrClassMap.h>\n#include <drogon/HttpResponse.h>\n#include <drogon/HttpTypes.h>\n#include <drogon/utils/Utilities.h>\n#include <drogon/version.h>\n#include <json/json.h>\n#include <trantor/utils/AsyncFileLogger.h>\n#include <algorithm>\n#include \"AOPAdvice.h\"\n#include \"ConfigLoader.h\"\n#include \"DbClientManager.h\"\n#include \"HttpClientImpl.h\"\n#include \"HttpConnectionLimit.h\"\n#include \"HttpControllersRouter.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpResponseImpl.h\"\n#include \"HttpServer.h\"\n#include \"HttpUtils.h\"\n#include \"ListenerManager.h\"\n#include \"PluginsManager.h\"\n#include \"RedisClientManager.h\"\n#include \"SessionManager.h\"\n#include \"SharedLibManager.h\"\n#include \"StaticFileRouter.h\"\n\n#include <iostream>\n#include <memory>\n#include <tuple>\n#include <unordered_map>\n#include <utility>\n#include <filesystem>\n\n#include <fcntl.h>\n#include <sys/stat.h>\n#ifndef _WIN32\n#include <sys/wait.h>\n#include <unistd.h>\n#define os_access access\n#elif !defined(_WIN32) || defined(__MINGW32__)\n#include <sys/file.h>\n#include <unistd.h>\n#define os_access access\n#else\n#include <io.h>\n#define os_access _waccess\n#define R_OK 04\n#define W_OK 02\n#endif\n\n#ifdef DROGON_SPDLOG_SUPPORT\n#include <spdlog/spdlog.h>\n#include <spdlog/logger.h>\n#include <spdlog/sinks/stdout_color_sinks.h>\n#include <spdlog/sinks/rotating_file_sink.h>\n#ifdef _WIN32\n#include <spdlog/sinks/msvc_sink.h>\n// Damn antedeluvian M$ macros\n#undef min\n#undef max\n#endif\n#endif  // DROGON_SPDLOG_SUPPORT\n\nusing namespace drogon;\nusing namespace std::placeholders;\n\nHttpAppFrameworkImpl::HttpAppFrameworkImpl()\n    : listenerManagerPtr_(new ListenerManager),\n      pluginsManagerPtr_(new PluginsManager),\n      dbClientManagerPtr_(new orm::DbClientManager),\n      redisClientManagerPtr_(new nosql::RedisClientManager),\n      uploadPath_(rootPath_ + \"uploads\")\n{\n}\n\nstatic std::function<void()> f = [] {\n    LOG_TRACE << \"Initialize the main event loop in the main thread\";\n};\n\n/// Make sure that the main event loop is initialized in the main thread.\ndrogon::InitBeforeMainFunction drogon::HttpAppFrameworkImpl::initFirst_([]() {\n    HttpAppFrameworkImpl::instance().getLoop()->runInLoop(f);\n});\n\nnamespace drogon\n{\nstd::string getVersion()\n{\n    return DROGON_VERSION;\n}\n\nstd::string getGitCommit()\n{\n    return DROGON_VERSION_SHA1;\n}\n\nHttpResponsePtr defaultErrorHandler(HttpStatusCode code, const HttpRequestPtr &)\n{\n    return std::make_shared<HttpResponseImpl>(code, CT_TEXT_HTML);\n}\n\nvoid defaultExceptionHandler(\n    const std::exception &e,\n    const HttpRequestPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    std::string pathWithQuery = req->path();\n    if (req->query().empty() == false)\n        pathWithQuery += \"?\" + req->query();\n    LOG_ERROR << \"Unhandled exception in \" << pathWithQuery\n              << \", what(): \" << e.what();\n    const auto &handler = app().getCustomErrorHandler();\n    callback(handler(k500InternalServerError, req));\n}\n\nstatic void godaemon()\n{\n    printf(\"Initializing daemon mode\\n\");\n#ifndef _WIN32\n    if (getppid() != 1)\n    {\n        pid_t pid;\n        pid = fork();\n        if (pid > 0)\n            exit(0);  // parent\n        if (pid < 0)\n        {\n            perror(\"fork\");\n            exit(1);\n        }\n        setsid();\n    }\n\n    // redirect stdin/stdout/stderr to /dev/null\n    close(0);\n    close(1);\n    close(2);\n\n    int ret = open(\"/dev/null\", O_RDWR);\n    (void)ret;\n    ret = dup(0);\n    (void)ret;\n    ret = dup(0);\n    (void)ret;\n    umask(0);\n#else\n    LOG_ERROR << \"Cannot run as daemon in Windows\";\n    exit(1);\n#endif\n\n    return;\n}\n\nstatic void TERMFunction(int sig)\n{\n    if (sig == SIGTERM)\n    {\n        LOG_WARN << \"SIGTERM signal received.\";\n        HttpAppFrameworkImpl::instance().getTermSignalHandler()();\n    }\n    else if (sig == SIGINT)\n    {\n        LOG_WARN << \"SIGINT signal received.\";\n        HttpAppFrameworkImpl::instance().getIntSignalHandler()();\n    }\n}\n\n}  // namespace drogon\n\nHttpAppFrameworkImpl::~HttpAppFrameworkImpl() noexcept\n{\n// Destroy the following objects before the loop destruction\n#ifndef _WIN32\n    sharedLibManagerPtr_.reset();\n#endif\n    sessionManagerPtr_.reset();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setStaticFilesCacheTime(int cacheTime)\n{\n    StaticFileRouter::instance().setStaticFilesCacheTime(cacheTime);\n    return *this;\n}\n\nint HttpAppFrameworkImpl::staticFilesCacheTime() const\n{\n    return StaticFileRouter::instance().staticFilesCacheTime();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setGzipStatic(bool useGzipStatic)\n{\n    StaticFileRouter::instance().setGzipStatic(useGzipStatic);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setBrStatic(bool useGzipStatic)\n{\n    StaticFileRouter::instance().setBrStatic(useGzipStatic);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setImplicitPageEnable(\n    bool useImplicitPage)\n{\n    StaticFileRouter::instance().setImplicitPageEnable(useImplicitPage);\n    return *this;\n}\n\nbool HttpAppFrameworkImpl::isImplicitPageEnabled() const\n{\n    return StaticFileRouter::instance().isImplicitPageEnabled();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setImplicitPage(\n    const std::string &implicitPageFile)\n{\n    StaticFileRouter::instance().setImplicitPage(implicitPageFile);\n    return *this;\n}\n\nconst std::string &HttpAppFrameworkImpl::getImplicitPage() const\n{\n    return StaticFileRouter::instance().getImplicitPage();\n}\n#ifndef _WIN32\nHttpAppFramework &HttpAppFrameworkImpl::enableDynamicViewsLoading(\n    const std::vector<std::string> &libPaths,\n    const std::string &outputPath)\n{\n    assert(!running_);\n\n    for (auto const &libpath : libPaths)\n    {\n        if (libpath[0] == '/' ||\n            (libpath.length() >= 2 && libpath[0] == '.' && libpath[1] == '/') ||\n            (libpath.length() >= 3 && libpath[0] == '.' && libpath[1] == '.' &&\n             libpath[2] == '/') ||\n            libpath == \".\" || libpath == \"..\")\n        {\n            libFilePaths_.push_back(libpath);\n        }\n        else\n        {\n            if (rootPath_[rootPath_.length() - 1] == '/')\n                libFilePaths_.push_back(rootPath_ + libpath);\n            else\n                libFilePaths_.push_back(rootPath_ + \"/\" + libpath);\n        }\n    }\n    libFileOutputPath_ = outputPath;\n    if (!libFileOutputPath_.empty())\n    {\n        if (drogon::utils::createPath(libFileOutputPath_) == -1)\n        {\n            LOG_FATAL << \"Can't create \" << libFileOutputPath_\n                      << \" path for dynamic views\";\n            exit(-1);\n        }\n    }\n\n    return *this;\n}\n#endif\nHttpAppFramework &HttpAppFrameworkImpl::setFileTypes(\n    const std::vector<std::string> &types)\n{\n    StaticFileRouter::instance().setFileTypes(types);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::registerWebSocketController(\n    const std::string &pathName,\n    const std::string &ctrlName,\n    const std::vector<internal::HttpConstraint> &filtersAndMethods)\n{\n    assert(!routersInit_);\n    HttpControllersRouter::instance().registerWebSocketController(\n        pathName, ctrlName, filtersAndMethods);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::registerHttpSimpleController(\n    const std::string &pathName,\n    const std::string &ctrlName,\n    const std::vector<internal::HttpConstraint> &filtersAndMethods)\n{\n    assert(!routersInit_);\n    HttpControllersRouter::instance().registerHttpSimpleController(\n        pathName, ctrlName, filtersAndMethods);\n    return *this;\n}\n\nvoid HttpAppFrameworkImpl::registerHttpController(\n    const std::string &pathPattern,\n    const internal::HttpBinderBasePtr &binder,\n    const std::vector<HttpMethod> &validMethods,\n    const std::vector<std::string> &filters,\n    const std::string &handlerName)\n{\n    assert(!pathPattern.empty());\n    assert(binder);\n    assert(!routersInit_);\n    HttpControllersRouter::instance().addHttpPath(\n        pathPattern, binder, validMethods, filters, handlerName);\n}\n\nvoid HttpAppFrameworkImpl::registerHttpControllerViaRegex(\n    const std::string &regExp,\n    const internal::HttpBinderBasePtr &binder,\n    const std::vector<HttpMethod> &validMethods,\n    const std::vector<std::string> &filters,\n    const std::string &handlerName)\n{\n    assert(!regExp.empty());\n    assert(binder);\n    assert(!routersInit_);\n    HttpControllersRouter::instance().addHttpRegex(\n        regExp, binder, validMethods, filters, handlerName);\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setThreadNum(size_t threadNum)\n{\n    if (threadNum == 0)\n    {\n        threadNum_ = std::thread::hardware_concurrency();\n        return *this;\n    }\n    threadNum_ = threadNum;\n    return *this;\n}\n\nPluginBase *HttpAppFrameworkImpl::getPlugin(const std::string &name)\n{\n    return pluginsManagerPtr_->getPlugin(name);\n}\n\nstd::shared_ptr<PluginBase> HttpAppFrameworkImpl::getSharedPlugin(\n    const std::string &name)\n{\n    return pluginsManagerPtr_->getSharedPlugin(name);\n}\n\nvoid HttpAppFrameworkImpl::addPlugin(\n    const std::string &name,\n    const std::vector<std::string> &dependencies,\n    const Json::Value &config)\n{\n    assert(!isRunning());\n    Json::Value pluginConfig;\n    pluginConfig[\"name\"] = name;\n    Json::Value deps(Json::arrayValue);\n    for (const auto dep : dependencies)\n    {\n        deps.append(dep);\n    }\n    pluginConfig[\"dependencies\"] = deps;\n    pluginConfig[\"config\"] = config;\n    auto &plugins = jsonRuntimeConfig_[\"plugins\"];\n    plugins.append(pluginConfig);\n}\n\nvoid HttpAppFrameworkImpl::addPlugins(const Json::Value &configs)\n{\n    assert(!isRunning());\n    assert(configs.isArray());\n    auto &plugins = jsonRuntimeConfig_[\"plugins\"];\n    for (const auto config : configs)\n    {\n        plugins.append(config);\n    }\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::addListener(\n    const std::string &ip,\n    uint16_t port,\n    bool useSSL,\n    const std::string &certFile,\n    const std::string &keyFile,\n    bool useOldTLS,\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n{\n    assert(!running_);\n    listenerManagerPtr_->addListener(\n        ip, port, useSSL, certFile, keyFile, useOldTLS, sslConfCmds);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setMaxConnectionNum(\n    size_t maxConnections)\n{\n    HttpConnectionLimit::instance().setMaxConnectionNum(maxConnections);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setMaxConnectionNumPerIP(\n    size_t maxConnectionsPerIP)\n{\n    HttpConnectionLimit::instance().setMaxConnectionNumPerIP(\n        maxConnectionsPerIP);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::loadConfigFile(\n    const std::string &fileName)\n{\n    ConfigLoader loader(fileName);\n    loader.load();\n    jsonConfig_ = loader.jsonValue();\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::loadConfigJson(const Json::Value &data)\n{\n    ConfigLoader loader(data);\n    loader.load();\n    jsonConfig_ = loader.jsonValue();\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::loadConfigJson(Json::Value &&data)\n{\n    ConfigLoader loader(std::move(data));\n    loader.load();\n    jsonConfig_ = loader.jsonValue();\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setLogPath(\n    const std::string &logPath,\n    const std::string &logfileBaseName,\n    size_t logfileSize,\n    size_t maxFiles,\n    bool useSpdlog)\n{\n#ifdef DROGON_SPDLOG_SUPPORT\n    logWithSpdlog_ = trantor::Logger::hasSpdLogSupport() && useSpdlog;\n#endif\n    if (logPath.empty())\n        return *this;\n    // std::filesystem does not provide a method to check access permissions, so\n    // keep existing code\n    if (os_access(utils::toNativePath(logPath).c_str(), 0) != 0)\n    {\n        std::cerr << \"Log path does not exist!\\n\";\n        exit(1);\n    }\n    if (os_access(utils::toNativePath(logPath).c_str(), R_OK | W_OK) != 0)\n    {\n        std::cerr << \"Unable to access log path!\\n\";\n        exit(1);\n    }\n    logPath_ = logPath;\n    logfileBaseName_ = logfileBaseName;\n    logfileSize_ = logfileSize;\n    logfileMaxNum_ = maxFiles;\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setLogLevel(\n    trantor::Logger::LogLevel level)\n{\n    trantor::Logger::setLogLevel(level);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setLogLocalTime(bool on)\n{\n    trantor::Logger::setDisplayLocalTime(on);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setSSLConfigCommands(\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n{\n    sslConfCmds_ = sslConfCmds;\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setSSLFiles(const std::string &certPath,\n                                                    const std::string &keyPath)\n{\n    sslCertPath_ = certPath;\n    sslKeyPath_ = keyPath;\n    return *this;\n}\n\nvoid HttpAppFrameworkImpl::run()\n{\n    if (!getLoop()->isInLoopThread())\n    {\n        getLoop()->moveToCurrentThread();\n    }\n    LOG_TRACE << \"Start to run...\";\n    // Create dirs for cache files\n    for (int i = 0; i < 256; ++i)\n    {\n        char dirName[4];\n        snprintf(dirName, sizeof(dirName), \"%02x\", i);\n        std::transform(dirName, dirName + 2, dirName, [](unsigned char c) {\n            return toupper(c);\n        });\n        utils::createPath(getUploadPath() + \"/tmp/\" + dirName);\n    }\n    if (runAsDaemon_)\n    {\n        // go daemon!\n        godaemon();\n#ifdef __linux__\n        getLoop()->resetTimerQueue();\n#endif\n        getLoop()->resetAfterFork();\n    }\n    // set relaunching\n    if (relaunchOnError_)\n    {\n#ifndef _WIN32\n        while (true)\n        {\n            int child_status = 0;\n            auto child_pid = fork();\n            if (child_pid < 0)\n            {\n                LOG_ERROR << \"fork error\";\n                abort();\n            }\n            else if (child_pid == 0)\n            {\n                // child\n                break;\n            }\n            waitpid(child_pid, &child_status, 0);\n            sleep(1);\n            LOG_INFO << \"start new process\";\n        }\n#ifdef __linux__\n        getLoop()->resetTimerQueue();\n#endif\n        getLoop()->resetAfterFork();\n#endif\n    }\n    if (handleSigterm_)\n    {\n#ifdef WIN32\n        signal(SIGTERM, TERMFunction);\n        signal(SIGINT, TERMFunction);\n#else\n        struct sigaction sa;\n        sa.sa_handler = TERMFunction;\n        sigemptyset(&sa.sa_mask);\n        sa.sa_flags = 0;\n        if (sigaction(SIGINT, &sa, NULL) == -1)\n        {\n            LOG_ERROR << \"sigaction() failed, can't set SIGINT handler\";\n            abort();\n        }\n        if (sigaction(SIGTERM, &sa, NULL) == -1)\n        {\n            LOG_ERROR << \"sigaction() failed, can't set SIGTERM handler\";\n            abort();\n        }\n#endif\n    }\n    setupFileLogger();\n    if (relaunchOnError_)\n    {\n        LOG_INFO << \"Start child process\";\n    }\n\n#ifndef _WIN32\n    if (!libFilePaths_.empty())\n    {\n        sharedLibManagerPtr_ =\n            std::make_unique<SharedLibManager>(libFilePaths_,\n                                               libFileOutputPath_);\n    }\n#endif\n\n    // Create IO threads\n    ioLoopThreadPool_ =\n        std::make_unique<trantor::EventLoopThreadPool>(threadNum_,\n                                                       \"DrogonIoLoop\");\n    std::vector<trantor::EventLoop *> ioLoops = ioLoopThreadPool_->getLoops();\n    for (size_t i = 0; i < threadNum_; ++i)\n    {\n        ioLoops[i]->setIndex(i);\n    }\n    getLoop()->setIndex(threadNum_);\n\n    // Create all listeners.\n    listenerManagerPtr_->createListeners(sslCertPath_,\n                                         sslKeyPath_,\n                                         sslConfCmds_,\n                                         ioLoops);\n\n    // A fast database client instance should be created in the main event\n    // loop, so put the main loop into ioLoops.\n    ioLoops.push_back(getLoop());\n    dbClientManagerPtr_->createDbClients(ioLoops);\n    redisClientManagerPtr_->createRedisClients(ioLoops);\n    if (useSession_)\n    {\n        sessionManagerPtr_ =\n            std::make_unique<SessionManager>(getLoop(),\n                                             sessionTimeout_,\n                                             sessionStartAdvices_,\n                                             sessionDestroyAdvices_,\n                                             sessionIdGeneratorCallback_);\n    }\n    // now start running!!\n    running_ = true;\n    // Initialize plugins\n    auto &pluginConfig = jsonConfig_[\"plugins\"];\n    const auto &runtumePluginConfig = jsonRuntimeConfig_[\"plugins\"];\n    if (!pluginConfig.isNull())\n    {\n        if (!runtumePluginConfig.isNull() && runtumePluginConfig.isArray())\n        {\n            for (const auto &plugin : runtumePluginConfig)\n            {\n                pluginConfig.append(plugin);\n            }\n        }\n    }\n    else\n    {\n        jsonConfig_[\"plugins\"] = runtumePluginConfig;\n    }\n    if (!pluginConfig.isNull())\n    {\n        pluginsManagerPtr_->initializeAllPlugins(pluginConfig,\n                                                 [](PluginBase *plugin) {\n                                                     LOG_TRACE\n                                                         << \"new plugin:\"\n                                                         << plugin->className();\n                                                     // TODO: new plugin\n                                                 });\n    }\n    routersInit_ = true;\n    HttpControllersRouter::instance().init(ioLoops);\n    StaticFileRouter::instance().init(ioLoops);\n    getLoop()->queueInLoop([this]() {\n        for (auto &adv : beginningAdvices_)\n        {\n            adv();\n        }\n        beginningAdvices_.clear();\n        // Let listener event loops run when everything is ready.\n        listenerManagerPtr_->startListening();\n    });\n    // start all loops\n    // TODO: when should IOLoops start?\n    // In before, IOLoops are started in `listenerManagerPtr_->startListening()`\n    // It should be fine for them to start anywhere before `startListening()`.\n    // However, we should consider other components.\n    ioLoopThreadPool_->start();\n    getLoop()->loop();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setUploadPath(\n    const std::string &uploadPath)\n{\n    assert(!uploadPath.empty());\n\n    std::filesystem::path fsUploadPath(utils::toNativePath(uploadPath));\n    if (!fsUploadPath.is_absolute())\n    {\n        std::filesystem::path fsRoot(utils::toNativePath(rootPath_));\n        fsUploadPath = fsRoot / fsUploadPath;\n    }\n    uploadPath_ = utils::fromNativePath(fsUploadPath.native());\n    return *this;\n}\n\nvoid HttpAppFrameworkImpl::findSessionForRequest(const HttpRequestImplPtr &req)\n{\n    if (useSession_)\n    {\n        std::string sessionId = req->getCookie(sessionCookieKey_);\n        bool needSetSessionid = false;\n        if (sessionId.empty())\n        {\n            sessionId = sessionIdGeneratorCallback_();\n            needSetSessionid = true;\n        }\n        req->setSession(\n            sessionManagerPtr_->getSession(sessionId, needSetSessionid));\n    }\n}\n\nstd::vector<HttpHandlerInfo> HttpAppFrameworkImpl::getHandlersInfo() const\n{\n    return HttpControllersRouter::instance().getHandlersInfo();\n}\n\nHttpResponsePtr HttpAppFrameworkImpl::handleSessionForResponse(\n    const HttpRequestImplPtr &req,\n    const HttpResponsePtr &resp)\n{\n    if (useSession_)\n    {\n        auto &sessionPtr = req->getSession();\n        if (!sessionPtr)\n        {\n            return resp;\n        }\n        if (sessionPtr->needToChangeSessionId())\n        {\n            sessionManagerPtr_->changeSessionId(sessionPtr);\n        }\n        if (sessionPtr->needSetToClient())\n        {\n            if (resp->expiredTime() >= 0)\n            {\n                auto newResp = std::make_shared<HttpResponseImpl>(\n                    *static_cast<HttpResponseImpl *>(resp.get()));\n                newResp->setExpiredTime(-1);  // make it temporary\n                auto sessionid =\n                    Cookie(sessionCookieKey_, sessionPtr->sessionId());\n                sessionid.setPath(\"/\");\n                sessionid.setSameSite(sessionSameSite_);\n                if (sessionMaxAge_ >= 0)\n                    sessionid.setMaxAge(sessionMaxAge_);\n                newResp->addCookie(std::move(sessionid));\n                sessionPtr->hasSet();\n\n                return newResp;\n            }\n            else\n            {\n                auto sessionid =\n                    Cookie(sessionCookieKey_, sessionPtr->sessionId());\n                sessionid.setPath(\"/\");\n                sessionid.setSameSite(sessionSameSite_);\n                if (sessionMaxAge_ >= 0)\n                    sessionid.setMaxAge(sessionMaxAge_);\n                resp->addCookie(std::move(sessionid));\n                sessionPtr->hasSet();\n\n                return resp;\n            }\n        }\n        else if (resp->version() != req->version())\n        {\n            auto newResp = std::make_shared<HttpResponseImpl>(\n                *static_cast<HttpResponseImpl *>(resp.get()));\n            newResp->setVersion(req->version());\n            newResp->setExpiredTime(-1);  // make it temporary\n\n            return newResp;\n        }\n        else\n        {\n            return resp;\n        }\n    }\n    else\n    {\n        if (resp->expiredTime() >= 0 && resp->version() != req->version())\n        {\n            auto newResp = std::make_shared<HttpResponseImpl>(\n                *static_cast<HttpResponseImpl *>(resp.get()));\n            newResp->setVersion(req->version());\n            newResp->setExpiredTime(-1);  // make it temporary\n\n            return newResp;\n        }\n        else\n        {\n            return resp;\n        }\n    }\n}\n\ntrantor::EventLoop *HttpAppFrameworkImpl::getLoop() const\n{\n    static trantor::EventLoop loop;\n    return &loop;\n}\n\ntrantor::EventLoop *HttpAppFrameworkImpl::getIOLoop(size_t id) const\n{\n    if (!ioLoopThreadPool_)\n    {\n        LOG_WARN << \"Please call getIOLoop() after drogon::app().run()\";\n        return nullptr;\n    }\n    auto n = ioLoopThreadPool_->size();\n    if (id >= n)\n    {\n        LOG_TRACE << \"Loop id (\" << id << \") out of range [0-\" << n << \").\";\n        id %= n;\n        LOG_TRACE << \"Rounded to : \" << id;\n    }\n    return ioLoopThreadPool_->getLoop(id);\n}\n\nHttpAppFramework &HttpAppFramework::instance()\n{\n    return HttpAppFrameworkImpl::instance();\n}\n\nvoid HttpAppFrameworkImpl::forward(\n    const HttpRequestPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback,\n    const std::string &hostString,\n    double timeout)\n{\n    forward(std::dynamic_pointer_cast<HttpRequestImpl>(req),\n            std::move(callback),\n            hostString,\n            timeout);\n}\n\nvoid HttpAppFrameworkImpl::forward(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback,\n    const std::string &hostString,\n    double timeout)\n{\n    if (hostString.empty())\n    {\n        HttpInternalForwardHelper::forward(req, std::move(callback));\n    }\n    else\n    {\n        /// A tiny implementation of a reverse proxy.\n        static std::unordered_map<std::string, HttpClientImplPtr> clientsMap;\n        HttpClientImplPtr clientPtr;\n        static std::mutex mtx;\n        {\n            std::lock_guard<std::mutex> lock(mtx);\n            auto iter = clientsMap.find(hostString);\n            if (iter != clientsMap.end())\n            {\n                clientPtr = iter->second;\n            }\n            else\n            {\n                clientPtr = std::make_shared<HttpClientImpl>(\n                    trantor::EventLoop::getEventLoopOfCurrentThread()\n                        ? trantor::EventLoop::getEventLoopOfCurrentThread()\n                        : getLoop(),\n                    hostString);\n                clientsMap[hostString] = clientPtr;\n            }\n        }\n        req->setPassThrough(true);\n        clientPtr->sendRequest(\n            req,\n            [callback = std::move(callback), req](ReqResult result,\n                                                  const HttpResponsePtr &resp) {\n                if (result == ReqResult::Ok)\n                {\n                    resp->setPassThrough(true);\n                    callback(resp);\n                }\n                else\n                {\n                    callback(HttpResponse::newNotFoundResponse(req));\n                }\n            },\n            timeout);\n    }\n}\n\norm::DbClientPtr HttpAppFrameworkImpl::getDbClient(const std::string &name)\n{\n    return dbClientManagerPtr_->getDbClient(name);\n}\n\norm::DbClientPtr HttpAppFrameworkImpl::getFastDbClient(const std::string &name)\n{\n    return dbClientManagerPtr_->getFastDbClient(name);\n}\n\nnosql::RedisClientPtr HttpAppFrameworkImpl::getRedisClient(\n    const std::string &name)\n{\n    return redisClientManagerPtr_->getRedisClient(name);\n}\n\nnosql::RedisClientPtr HttpAppFrameworkImpl::getFastRedisClient(\n    const std::string &name)\n{\n    return redisClientManagerPtr_->getFastRedisClient(name);\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::createDbClient(\n    const std::string &dbType,\n    const std::string &host,\n    const unsigned short port,\n    const std::string &databaseName,\n    const std::string &userName,\n    const std::string &password,\n    const size_t connectionNum,\n    const std::string &filename,\n    const std::string &name,\n    const bool isFast,\n    const std::string &characterSet,\n    double timeout,\n    const bool autoBatch)\n{\n    assert(!running_);\n    dbClientManagerPtr_->createDbClient(dbType,\n                                        host,\n                                        port,\n                                        databaseName,\n                                        userName,\n                                        password,\n                                        connectionNum,\n                                        filename,\n                                        name,\n                                        isFast,\n                                        characterSet,\n                                        timeout,\n                                        autoBatch);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::createRedisClient(\n    const std::string &ip,\n    unsigned short port,\n    const std::string &name,\n    const std::string &password,\n    size_t connectionNum,\n    bool isFast,\n    double timeout,\n    unsigned int db,\n    const std::string &username)\n{\n    assert(!running_);\n    redisClientManagerPtr_->createRedisClient(\n        name, ip, port, username, password, connectionNum, isFast, timeout, db);\n    return *this;\n}\n\nvoid HttpAppFrameworkImpl::quit()\n{\n    if (getLoop()->isRunning())\n    {\n        getLoop()->queueInLoop([this]() {\n            // Release members in the reverse order of initialization\n            listenerManagerPtr_->stopListening();\n            listenerManagerPtr_.reset();\n            StaticFileRouter::instance().reset();\n            HttpControllersRouter::instance().reset();\n            pluginsManagerPtr_.reset();\n            redisClientManagerPtr_.reset();\n            dbClientManagerPtr_.reset();\n            running_ = false;\n            getLoop()->quit();\n            for (trantor::EventLoop *loop : ioLoopThreadPool_->getLoops())\n            {\n                loop->quit();\n            }\n            ioLoopThreadPool_->wait();\n        });\n    }\n}\n\nconst HttpResponsePtr &HttpAppFrameworkImpl::getCustom404Page()\n{\n    if (!custom404_)\n    {\n        return custom404_;\n    }\n    auto loop = trantor::EventLoop::getEventLoopOfCurrentThread();\n    if (loop && loop->index() < app().getThreadNum())\n    {\n        // If the current thread is an IO thread\n        static IOThreadStorage<HttpResponsePtr> thread404Pages;\n        static std::once_flag once;\n        std::call_once(once, [this] {\n            thread404Pages.init(\n                [this](HttpResponsePtr &resp, size_t /*index*/) {\n                    resp = std::make_shared<HttpResponseImpl>(\n                        *static_cast<HttpResponseImpl *>(custom404_.get()));\n                });\n        });\n        return thread404Pages.getThreadData();\n    }\n    else\n    {\n        return custom404_;\n    }\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setStaticFileHeaders(\n    const std::vector<std::pair<std::string, std::string>> &headers)\n{\n    StaticFileRouter::instance().setStaticFileHeaders(headers);\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::addALocation(\n    const std::string &uriPrefix,\n    const std::string &defaultContentType,\n    const std::string &alias,\n    bool isCaseSensitive,\n    bool allowAll,\n    bool isRecursive,\n    const std::vector<std::string> &filters)\n{\n    StaticFileRouter::instance().addALocation(uriPrefix,\n                                              defaultContentType,\n                                              alias,\n                                              isCaseSensitive,\n                                              allowAll,\n                                              isRecursive,\n                                              filters);\n    return *this;\n}\n\nbool HttpAppFrameworkImpl::areAllDbClientsAvailable() const noexcept\n{\n    return dbClientManagerPtr_->areAllDbClientsAvailable();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setCustomErrorHandler(\n    std::function<HttpResponsePtr(HttpStatusCode, const HttpRequestPtr &req)>\n        &&resp_generator)\n{\n    customErrorHandler_ = std::move(resp_generator);\n    usingCustomErrorHandler_ = true;\n    return *this;\n}\n\nconst std::function<HttpResponsePtr(HttpStatusCode,\n                                    const HttpRequestPtr &req)> &\nHttpAppFrameworkImpl::getCustomErrorHandler() const\n{\n    return customErrorHandler_;\n}\n\nstd::vector<trantor::InetAddress> HttpAppFrameworkImpl::getListeners() const\n{\n    return listenerManagerPtr_->getListeners();\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setDefaultHandler(\n    DefaultHandler handler)\n{\n    StaticFileRouter::instance().setDefaultHandler(std::move(handler));\n    return *this;\n}\n\nHttpAppFramework &HttpAppFrameworkImpl::setupFileLogger()\n{\n#ifdef DROGON_SPDLOG_SUPPORT\n    if (logWithSpdlog_)\n    {\n...\n// Path: lib/src/ConfigAdapter.h\n#pragma once\n#include <json/json.h>\n#include <vector>\n#include <string>\n#include <memory>\n#include <fstream>\n\nnamespace drogon\n{\nclass ConfigAdapter\n{\n  public:\n    virtual ~ConfigAdapter() = default;\n    virtual Json::Value getJson(const std::string &content) const\n        noexcept(false) = 0;\n    virtual std::vector<std::string> getExtensions() const = 0;\n};\n\nusing ConfigAdapterPtr = std::shared_ptr<ConfigAdapter>;\n\n}  // namespace drogon\n\n// Path: lib/src/JsonConfigAdapter.h\n#pragma once\n#include \"ConfigAdapter.h\"\n\nnamespace drogon\n{\nclass JsonConfigAdapter : public ConfigAdapter\n{\n  public:\n    JsonConfigAdapter() = default;\n    ~JsonConfigAdapter() override = default;\n    Json::Value getJson(const std::string &content) const\n        noexcept(false) override;\n    std::vector<std::string> getExtensions() const override;\n};\n}  // namespace drogon\n\n// Path: lib/src/JsonConfigAdapter.cc\n#include \"JsonConfigAdapter.h\"\n#include <fstream>\n#include <mutex>\n\nusing namespace drogon;\n\nJson::Value JsonConfigAdapter::getJson(const std::string &content) const\n    noexcept(false)\n{\n    static std::once_flag once;\n    static Json::CharReaderBuilder builder;\n    std::call_once(once, []() { builder[\"collectComments\"] = false; });\n    JSONCPP_STRING errs;\n    std::unique_ptr<Json::CharReader> reader(builder.newCharReader());\n    Json::Value root;\n    if (!reader->parse(\n            content.c_str(), content.c_str() + content.size(), &root, &errs))\n    {\n        throw std::runtime_error(errs);\n    }\n    return root;\n}\n\nstd::vector<std::string> JsonConfigAdapter::getExtensions() const\n{\n    return {\"json\"};\n}\n\n// Path: lib/src/Histogram.cc\n#include <drogon/utils/monitoring/Histogram.h>\nusing namespace drogon;\nusing namespace drogon::monitoring;\n\nvoid Histogram::observe(double value)\n{\n    std::lock_guard<std::mutex> lock(mutex_);\n    if (maxAge_ > std::chrono::seconds(0) &&\n        timerId_ == trantor::InvalidTimerId)\n    {\n        std::weak_ptr<Histogram> weakPtr =\n            std::dynamic_pointer_cast<Histogram>(shared_from_this());\n        timerId_ = loopPtr_->runEvery(maxAge_ / timeBucketCount_, [weakPtr]() {\n            auto thisPtr = weakPtr.lock();\n            if (!thisPtr)\n                return;\n            thisPtr->rotateTimeBuckets();\n        });\n    }\n    auto &currentBucket = timeBuckets_.back();\n    currentBucket.sum += value;\n    currentBucket.count += 1;\n    for (size_t i = 0; i < bucketBoundaries_.size(); i++)\n    {\n        if (value <= bucketBoundaries_[i])\n        {\n            currentBucket.buckets[i] += 1;\n            break;\n        }\n    }\n    if (value > bucketBoundaries_.back())\n    {\n        currentBucket.buckets.back() += 1;\n    }\n}\n\nstd::vector<Sample> Histogram::collect() const\n{\n    std::vector<Sample> samples;\n    std::lock_guard<std::mutex> guard(mutex_);\n    size_t count{0};\n    for (size_t i = 0; i < bucketBoundaries_.size(); i++)\n    {\n        Sample sample;\n        for (auto &bucket : timeBuckets_)\n        {\n            count += bucket.buckets[i];\n        }\n        sample.name = name_ + \"_bucket\";\n        sample.exLabels.emplace_back(\"le\",\n                                     std::to_string(bucketBoundaries_[i]));\n        sample.value = count;\n        samples.emplace_back(std::move(sample));\n    }\n    Sample sample;\n    for (auto &bucket : timeBuckets_)\n    {\n        count += bucket.buckets.back();\n    }\n    sample.name = name_ + \"_bucket\";\n    sample.exLabels.emplace_back(\"le\", \"+Inf\");\n    sample.value = count;\n    samples.emplace_back(std::move(sample));\n    double sum{0};\n    uint64_t totalCount{0};\n    for (auto &bucket : timeBuckets_)\n    {\n        sum += bucket.sum;\n        totalCount += bucket.count;\n    }\n    Sample sumSample;\n    sumSample.name = name_ + \"_sum\";\n    sumSample.value = sum;\n    samples.emplace_back(std::move(sumSample));\n    Sample countSample;\n    countSample.name = name_ + \"_count\";\n    countSample.value = totalCount;\n    samples.emplace_back(std::move(countSample));\n    return samples;\n}\n\n// Path: lib/src/GlobalFilters.cc\n#include <drogon/plugins/GlobalFilters.h>\n#include <drogon/DrClassMap.h>\n#include <drogon/HttpAppFramework.h>\n#include \"FiltersFunction.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpAppFrameworkImpl.h\"\n\nusing namespace drogon::plugin;\n\nvoid GlobalFilters::initAndStart(const Json::Value &config)\n{\n    if (config.isMember(\"filters\") && config[\"filters\"].isArray())\n    {\n        auto &filters = config[\"filters\"];\n\n        for (auto const &filter : filters)\n        {\n            if (filter.isString())\n            {\n                auto filterPtr = std::dynamic_pointer_cast<HttpFilterBase>(\n                    drogon::DrClassMap::getSingleInstance(filter.asString()));\n                if (filterPtr)\n                {\n                    filters_.push_back(filterPtr);\n                }\n                else\n                {\n                    LOG_ERROR << \"Filter \" << filter.asString()\n                              << \" not found!\";\n                }\n            }\n        }\n    }\n    if (config.isMember(\"exempt\"))\n    {\n        auto exempt = config[\"exempt\"];\n        if (exempt.isArray())\n        {\n            std::string regexStr;\n            for (auto const &ex : exempt)\n            {\n                if (ex.isString())\n                {\n                    regexStr.append(\"(\").append(ex.asString()).append(\")|\");\n                }\n                else\n                {\n                    LOG_ERROR << \"exempt must be a string array!\";\n                }\n            }\n            if (!regexStr.empty())\n            {\n                regexStr.pop_back();\n                exemptPegex_ = std::regex(regexStr);\n                regexFlag_ = true;\n            }\n        }\n        else if (exempt.isString())\n        {\n            exemptPegex_ = std::regex(exempt.asString());\n            regexFlag_ = true;\n        }\n        else\n        {\n            LOG_ERROR << \"exempt must be a string or string array!\";\n        }\n    }\n    std::weak_ptr<GlobalFilters> weakPtr = shared_from_this();\n    drogon::app().registerPreRoutingAdvice(\n        [weakPtr](const drogon::HttpRequestPtr &req,\n                  drogon::AdviceCallback &&acb,\n                  drogon::AdviceChainCallback &&accb) {\n            auto thisPtr = weakPtr.lock();\n            if (!thisPtr)\n            {\n                accb();\n                return;\n            }\n            if (thisPtr->regexFlag_)\n            {\n                if (std::regex_match(req->path(), thisPtr->exemptPegex_))\n                {\n                    accb();\n                    return;\n                }\n            }\n\n            drogon::filters_function::doFilters(\n                thisPtr->filters_,\n                std::static_pointer_cast<HttpRequestImpl>(req),\n                [acb = std::move(acb),\n                 accb = std::move(accb)](const HttpResponsePtr &resp) {\n                    if (resp)\n                    {\n                        acb(resp);\n                    }\n                    else\n                    {\n                        accb();\n                    }\n                });\n        });\n}\n\nvoid GlobalFilters::shutdown()\n{\n    filters_.clear();\n}\n\n// Path: lib/src/HttpControllerBinder.cc\n/**\n *\n *  @file HttpControllerBinder.cc\n *  @author Nitromelon\n *\n *  Copyright 2023, Nitromelon.  All rights reserved.\n *  https://github.com/drogonframework/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpControllerBinder.h\"\n#include \"HttpResponseImpl.h\"\n#include <drogon/HttpSimpleController.h>\n#include <drogon/WebSocketController.h>\n\nnamespace drogon\n{\n\nvoid HttpSimpleControllerBinder::handleRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    // Binders without controller should be removed after run()\n    assert(controller_);\n\n    try\n    {\n        auto cb = callback;  // copy\n        controller_->asyncHandleHttpRequest(req, std::move(cb));\n    }\n    catch (const std::exception &e)\n    {\n        app().getExceptionHandler()(e, req, std::move(callback));\n        return;\n    }\n    catch (...)\n    {\n        LOG_ERROR << \"Exception not derived from std::exception\";\n        return;\n    }\n}\n\nvoid HttpControllerBinder::handleRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    auto &paramsVector = req->getRoutingParameters();\n    std::deque<std::string> params(paramsVector.begin(), paramsVector.end());\n    binderPtr_->handleHttpRequest(params, req, std::move(callback));\n}\n\nvoid WebsocketControllerBinder::handleRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback) const\n{\n    std::string wsKey = req->getHeaderBy(\"sec-websocket-key\");\n    wsKey.append(\"258EAFA5-E914-47DA-95CA-C5AB0DC85B11\");\n    unsigned char accKey[20];\n    auto sha1 = trantor::utils::sha1(wsKey.c_str(), wsKey.length());\n    memcpy(accKey, &sha1, sizeof(sha1));\n    auto base64Key = utils::base64Encode(accKey, sizeof(accKey));\n    auto resp = HttpResponse::newHttpResponse();\n    resp->setStatusCode(k101SwitchingProtocols);\n    resp->addHeader(\"Upgrade\", \"websocket\");\n    resp->addHeader(\"Connection\", \"Upgrade\");\n    resp->addHeader(\"Sec-WebSocket-Accept\", base64Key);\n    callback(resp);\n}\n\nvoid WebsocketControllerBinder::handleNewConnection(\n    const HttpRequestImplPtr &req,\n    const WebSocketConnectionImplPtr &wsConnPtr) const\n{\n    auto ctrlPtr = controller_;\n    assert(ctrlPtr);\n    wsConnPtr->setMessageCallback(\n        [ctrlPtr](std::string &&message,\n                  const WebSocketConnectionImplPtr &connPtr,\n                  const WebSocketMessageType &type) {\n            ctrlPtr->handleNewMessage(connPtr, std::move(message), type);\n        });\n    wsConnPtr->setCloseCallback(\n        [ctrlPtr](const WebSocketConnectionImplPtr &connPtr) {\n            ctrlPtr->handleConnectionClosed(connPtr);\n        });\n    ctrlPtr->handleNewConnection(req, wsConnPtr);\n}\n\n}  // namespace drogon\n\n// Path: lib/src/HttpBinder.cc\n/**\n *\n *  HttpBinder.h\n *  Martin Chang\n *\n *  Copyright 2021, Martin Chang.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/HttpBinder.h>\n#include <drogon/HttpAppFramework.h>\n\nnamespace drogon\n{\nnamespace internal\n{\n\nvoid handleException(const std::exception &e,\n                     const HttpRequestPtr &req,\n                     std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    app().getExceptionHandler()(e, req, std::move(callback));\n}\n}  // namespace internal\n}  // namespace drogon\n\n// Path: lib/src/YamlConfigAdapter.h\n#pragma once\n#include \"ConfigAdapter.h\"\n\nnamespace drogon\n{\nclass YamlConfigAdapter : public ConfigAdapter\n{\n  public:\n    YamlConfigAdapter() = default;\n    ~YamlConfigAdapter() override = default;\n    Json::Value getJson(const std::string &content) const\n        noexcept(false) override;\n    std::vector<std::string> getExtensions() const override;\n};\n}  // namespace drogon\n\n// Path: lib/src/YamlConfigAdapter.cc\n#include \"YamlConfigAdapter.h\"\n#ifdef HAS_YAML_CPP\n#include <yaml-cpp/yaml.h>\n#endif\nusing namespace drogon;\n#ifdef HAS_YAML_CPP\nnamespace YAML\n{\nstatic bool yaml2json(const Node &node, Json::Value &jsonValue)\n{\n    if (node.IsNull())\n    {\n        return false;\n    }\n    else if (node.IsScalar())\n    {\n        if (node.Tag() != \"!\")\n        {\n            try\n            {\n                jsonValue = node.as<Json::Value::Int64>();\n                return true;\n            }\n            catch (const YAML::BadConversion &e)\n            {\n            }\n            try\n            {\n                jsonValue = node.as<double>();\n                return true;\n            }\n            catch (const YAML::BadConversion &e)\n            {\n            }\n            try\n            {\n                jsonValue = node.as<bool>();\n                return true;\n            }\n            catch (const YAML::BadConversion &e)\n            {\n            }\n        }\n\n        Json::Value v(node.Scalar());\n        jsonValue.swapPayload(v);\n        return true;\n    }\n    else if (node.IsSequence())\n    {\n        for (std::size_t i = 0; i < node.size(); i++)\n        {\n            Json::Value v;\n            if (yaml2json(node[i], v))\n            {\n                jsonValue.append(v);\n            }\n            else\n            {\n                return false;\n            }\n        }\n\n        return true;\n    }\n    else if (node.IsMap())\n    {\n        for (YAML::const_iterator it = node.begin(); it != node.end(); ++it)\n        {\n            Json::Value v;\n            if (yaml2json(it->second, v))\n            {\n                jsonValue[it->first.Scalar()] = v;\n            }\n            else\n            {\n                return false;\n            }\n        }\n\n        return true;\n    }\n\n    return false;\n}\n\ntemplate <>\nstruct convert<Json::Value>\n{\n    static bool decode(const Node &node, Json::Value &rhs)\n    {\n        return yaml2json(node, rhs);\n    };\n};\n}  // namespace YAML\n\n#endif\nJson::Value YamlConfigAdapter::getJson(const std::string &content) const\n    noexcept(false)\n{\n#if HAS_YAML_CPP\n    // parse yaml file\n    YAML::Node config = YAML::Load(content);\n    if (!config.IsNull())\n    {\n        return config.as<Json::Value>();\n    }\n    else\n        return Json::Value();\n#else\n    throw std::runtime_error(\"please install yaml-cpp library\");\n#endif\n}\n\nstd::vector<std::string> YamlConfigAdapter::getExtensions() const\n{\n    return {\"yaml\", \"yml\"};\n}\n\n// Path: lib/src/FixedWindowRateLimiter.h\n#pragma once\n\n#include <drogon/RateLimiter.h>\n#include <chrono>\n\nnamespace drogon\n{\nclass FixedWindowRateLimiter : public RateLimiter\n{\n  public:\n    FixedWindowRateLimiter(size_t capacity,\n                           std::chrono::duration<double> timeUnit);\n    bool isAllowed() override;\n    ~FixedWindowRateLimiter() noexcept override = default;\n\n  private:\n    size_t capacity_;\n    size_t currentRequests_{0};\n    std::chrono::steady_clock::time_point lastTime_;\n    std::chrono::duration<double> timeUnit_;\n};\n}  // namespace drogon\n\n// Path: lib/src/SlidingWindowRateLimiter.h\n#pragma once\n#include <drogon/RateLimiter.h>\n#include <chrono>\n\nnamespace drogon\n{\nclass SlidingWindowRateLimiter : public RateLimiter\n{\n  public:\n    SlidingWindowRateLimiter(size_t capacity,\n                             std::chrono::duration<double> timeUnit);\n    bool isAllowed() override;\n    ~SlidingWindowRateLimiter() noexcept override = default;\n\n  private:\n    size_t capacity_;\n    size_t currentRequests_{0};\n    size_t previousRequests_{0};\n    std::chrono::steady_clock::time_point unitStartTime_;\n    std::chrono::steady_clock::time_point lastTime_;\n    std::chrono::duration<double> timeUnit_;\n};\n}  // namespace drogon\n\n// Path: lib/src/TokenBucketRateLimiter.h\n#pragma once\n\n#include <drogon/RateLimiter.h>\n\nnamespace drogon\n{\nclass TokenBucketRateLimiter : public RateLimiter\n{\n  public:\n    TokenBucketRateLimiter(size_t capacity,\n                           std::chrono::duration<double> timeUnit);\n    bool isAllowed() override;\n    ~TokenBucketRateLimiter() noexcept override = default;\n\n  private:\n    size_t capacity_;\n    std::chrono::steady_clock::time_point lastTime_;\n    std::chrono::duration<double> timeUnit_;\n    double tokens_;\n};\n}  // namespace drogon\n\n// Path: lib/src/RateLimiter.cc\n#include <drogon/RateLimiter.h>\n#include \"FixedWindowRateLimiter.h\"\n#include \"SlidingWindowRateLimiter.h\"\n#include \"TokenBucketRateLimiter.h\"\n\nusing namespace drogon;\n\nRateLimiterPtr RateLimiter::newRateLimiter(\n    RateLimiterType type,\n    size_t capacity,\n    std::chrono::duration<double> timeUnit)\n{\n    switch (type)\n    {\n        case RateLimiterType::kFixedWindow:\n            return std::make_shared<FixedWindowRateLimiter>(capacity, timeUnit);\n        case RateLimiterType::kSlidingWindow:\n            return std::make_shared<SlidingWindowRateLimiter>(capacity,\n                                                              timeUnit);\n        case RateLimiterType::kTokenBucket:\n            return std::make_shared<TokenBucketRateLimiter>(capacity, timeUnit);\n    }\n    return std::make_shared<TokenBucketRateLimiter>(capacity, timeUnit);\n}\n\n// Path: lib/src/SlashRemover.cc\n#include <drogon/plugins/SlashRemover.h>\n#include <drogon/plugins/Redirector.h>\n#include <drogon/HttpAppFramework.h>\n#include \"drogon/utils/FunctionTraits.h\"\n#include <cstddef>\n#include <cstdint>\n#include <functional>\n#include <string>\n#include <string_view>\n#include <utility>\n\nusing namespace drogon;\nusing namespace drogon::plugin;\nusing std::string;\nusing std::string_view;\n\nenum removeSlashMode : uint8_t\n{\n    trailing = 1 << 0,\n    duplicate = 1 << 1,\n    both = trailing | duplicate,\n};\n\n/// Returns the index before the trailing slashes,\n/// or 0 if only contains slashes\nstatic inline size_t findTrailingSlashes(string_view url)\n{\n    auto len = url.size();\n    // Must be at least 2 chars and end with a slash\n    if (len < 2 || url.back() != '/')\n        return string::npos;\n\n    size_t a = len - 1;  // We already know the last char is '/',\n                         // we will use pre-decrement to account for this\n    while (--a > 0 && url[a] == '/')\n        ;  // We know the first char is '/', so don't check for 0\n    return a;\n}\n\nstatic inline void removeTrailingSlashes(string &url,\n                                         size_t start,\n                                         string_view originalUrl)\n{\n    url = originalUrl.substr(0, start + 1);\n}\n\n/// Returns the index of the 2nd duplicate slash\nstatic inline size_t findDuplicateSlashes(string_view url)\n{\n    size_t len = url.size();\n    if (len < 2)\n        return string::npos;\n\n    bool startedPair = true;  // Always starts with a slash\n    for (size_t a = 1; a < len; ++a)\n    {\n        if (url[a] != '/')  // Broken pair\n        {\n            startedPair = false;\n            continue;\n        }\n        if (startedPair)  // Matching pair\n            return a;\n        startedPair = true;\n    }\n\n    return string::npos;\n}\n\nstatic inline void removeDuplicateSlashes(string &url, size_t start)\n{\n    // +1 because we don't need to look at the same character again,\n    // which was found by `findDuplicateSlashes`, it saves one iteration\n    for (size_t b = (start--) + 1, len = url.size(); b < len; ++b)\n    {\n        const char c = url[b];\n        if (c != '/' || url[start] != '/')\n        {\n            ++start;\n            url[start] = c;\n        }\n    }\n    url.resize(start + 1);\n}\n\nstatic inline std::pair<size_t, size_t> findExcessiveSlashes(string_view url)\n{\n    size_t len = url.size();\n    if (len < 2)  // Must have at least 2 characters to count as either trailing\n                  // or duplicate slash\n        return {string::npos, string::npos};\n\n    // Trail finder\n    size_t trailIdx = len;  // The pre-decrement will put it on last char\n    while (--trailIdx > 0 && url[trailIdx] == '/')\n        ;  // We know first char is '/', no need to check it\n\n    // Filled with '/'\n    if (trailIdx == 0)\n        return {\n            0,             // Only keep first slash\n            string::npos,  // No duplicate\n        };\n\n    // Look for a duplicate pair\n    size_t dupIdx = 1;\n    for (bool startedPair = true; dupIdx < trailIdx;\n         ++dupIdx)  // Always starts with a slash\n    {\n        if (url[dupIdx] != '/')  // Broken pair\n        {\n            startedPair = false;\n            continue;\n        }\n        if (startedPair)  // Matching pair\n            break;\n        startedPair = true;\n    }\n\n    // Found no duplicate\n    if (dupIdx == trailIdx)\n        return {\n            trailIdx != len - 1\n                ?  // If has gone past last char, then there is a trailing slash\n                trailIdx\n                : string::npos,  // No trail\n            string::npos,        // No duplicate\n        };\n\n    // Duplicate found\n    return {\n        trailIdx != len - 1\n            ?  // If has gone past last char, then there is a trailing slash\n            trailIdx\n            : string::npos,  // No trail\n        dupIdx,\n    };\n}\n\nstatic inline void removeExcessiveSlashes(string &url,\n                                          std::pair<size_t, size_t> start,\n                                          string_view originalUrl)\n{\n    if (start.first != string::npos)\n        removeTrailingSlashes(url, start.first, originalUrl);\n    else\n        url = originalUrl;\n\n    if (start.second != string::npos)\n        removeDuplicateSlashes(url, start.second);\n}\n\nstatic inline bool handleReq(const drogon::HttpRequestPtr &req,\n                             uint8_t removeMode)\n{\n    switch (removeMode)\n    {\n        case trailing:\n        {\n            auto find = findTrailingSlashes(req->path());\n            if (find == string::npos)\n                return false;\n\n            string newPath;\n            removeTrailingSlashes(newPath, find, req->path());\n            req->setPath(std::move(newPath));\n            break;\n        }\n        case duplicate:\n        {\n            auto find = findDuplicateSlashes(req->path());\n            if (find == string::npos)\n                return false;\n\n            string newPath = req->path();\n            removeDuplicateSlashes(newPath, find);\n            req->setPath(std::move(newPath));\n            break;\n        }\n        case both:\n        default:\n        {\n            auto find = findExcessiveSlashes(req->path());\n            if (find.first == string::npos && find.second == string::npos)\n                return false;\n\n            string newPath;\n            removeExcessiveSlashes(newPath, find, req->path());\n            req->setPath(std::move(newPath));\n            break;\n        }\n    }\n    return true;\n}\n\nvoid SlashRemover::initAndStart(const Json::Value &config)\n{\n    trailingSlashes_ = config.get(\"remove_trailing_slashes\", true).asBool();\n    duplicateSlashes_ = config.get(\"remove_duplicate_slashes\", true).asBool();\n    redirect_ = config.get(\"redirect\", true).asBool();\n    const uint8_t removeMode =\n        (trailingSlashes_ * trailing) | (duplicateSlashes_ * duplicate);\n    if (!removeMode)\n        return;\n    auto redirector = app().getPlugin<Redirector>();\n    if (!redirector)\n    {\n        LOG_ERROR << \"Redirector plugin is not found!\";\n        return;\n    }\n    auto func = [removeMode](const HttpRequestPtr &req) -> bool {\n        return handleReq(req, removeMode);\n    };\n    if (redirect_)\n    {\n        redirector->registerPathRewriteHandler(std::move(func));\n    }\n    else\n    {\n        redirector->registerForwardHandler(std::move(func));\n    }\n}\n\nvoid SlashRemover::shutdown()\n{\n    LOG_TRACE << \"SlashRemover plugin is shutdown!\";\n}\n\n// Path: lib/src/NotFound.cc\n// this file is generated by program automatically,don't modify it!\n\n/**\n *\n *  NotFound.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/NotFound.h>\n#include <map>\n#include <set>\n#include <sstream>\n#include <string>\n#include <vector>\n\nusing namespace drogon;\n\nstd::string NotFound::genText(const HttpViewData &NotFound_view_data)\n{\n    std::stringstream NotFound_tmp_stream;\n    NotFound_tmp_stream << \"<html>\\n\";\n    NotFound_tmp_stream << \"<head><title>404 Not Found</title></head>\\n\";\n...\n", "needle_token_start": 12096, "needle_token_end": 12158, "code_context_ntokens": 16146, "output": ["```cpp\nvoid defaultExceptionHandler(\n    const std::exception &e,\n    const HttpRequestPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    std::string pathWithQuery = req->path();\n    if (req->query().empty() == false)\n        pathWithQuery += \"?\" + req->query();\n    LOG_ERROR << \"Unhandled exception in \" << pathWithQuery\n              << \", what(): \" << e.what();\n    const auto &handler = app().getCustomErrorHandler();\n    callback(handler(k500InternalServerError, req));\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "bytesSize", "language": "cpp", "path": "lib/src/ConfigLoader.cc", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a string representation of a size, potentially suffixed with a unit (e.g., K, M, G, T), into its numerical value in bytes.\n2. **Input**: A string representing the size, which may include a unit suffix to indicate multiplication factors (e.g., \"K\" for kilobytes, \"M\" for megabytes).\n3. **Output**: A boolean indicating success or failure of the conversion, and a size_t variable that holds the computed size in bytes.\n4. **Procedure**: \n   - The function first checks if the input string is empty, setting the size to an undefined large value and returning true.\n   - If not empty, it initializes the size to 1 and checks the last character of the string for a unit suffix to determine the multiplication factor.\n   - The suffix is removed, and the remaining part of the string is converted to a numerical value.\n   - The function then multiplies this value by the determined factor to get the size in bytes.\n   - It checks for overflow during multiplication and returns false if the conversion fails or if the final computed size exceeds the maximum allowable size.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/HttpUtils.cc\n/**\n *\n *  @file HttpUtils.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpUtils.h\"\n#include <drogon/utils/Utilities.h>\n#include <trantor/utils/Logger.h>\n#include <map>\n#include <unordered_map>\n#include <mutex>\n\nnamespace drogon\n{\nstatic std::unordered_map<std::string, std::string> customMime;\n\n// https://en.wikipedia.org/wiki/List_of_file_formats\n// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types\n// https://www.digipres.org/formats/mime-types/\n// https://www.iana.org/assignments/media-types/media-types.xhtml\n// content type -> list of corresponding mime types, the first being the default\n// (more standard) one + the mime type to return in contentTypeToMime() when not\n// empty (mainly to return the charset for text types)\nstatic const std::unordered_map<\n    ContentType,\n    std::pair<std::vector<std::string_view>, std::string_view>>\n    mimeTypeDatabase_{\n        {CT_NONE, {{\"\"}, \"\"}},\n        {CT_APPLICATION_OCTET_STREAM, {{\"application/octet-stream\"}, \"\"}},\n        {CT_APPLICATION_X_FORM, {{\"application/x-www-form-urlencoded\"}, \"\"}},\n        {CT_MULTIPART_FORM_DATA, {{\"multipart/form-data\"}, \"\"}},\n        {CT_APPLICATION_GZIP, {{\"application/gzip\"}, \"\"}},\n        {CT_APPLICATION_JSON,\n         {{\"application/json\"}, \"application/json; charset=utf-8\"}},\n        {CT_APPLICATION_FONT_WOFF, {{\"application/font-woff\"}, \"\"}},\n        {CT_APPLICATION_FONT_WOFF2, {{\"application/font-woff2\"}, \"\"}},\n        {CT_APPLICATION_JAVA_ARCHIVE,\n         {{\"application/java-archive\", \"application/x-java-archive\"}, \"\"}},\n        {CT_APPLICATION_MSWORD, {{\"application/msword\"}, \"\"}},\n        {CT_APPLICATION_MSWORDX,\n         {{\"application/\"\n           \"vnd.openxmlformats-officedocument.wordprocessingml.document\"},\n          \"\"}},\n        {CT_APPLICATION_PDF, {{\"application/pdf\"}, \"\"}},\n        {CT_APPLICATION_VND_MS_FONTOBJ,\n         {{\"application/vnd.ms-fontobject\"}, \"\"}},\n        {CT_APPLICATION_VND_RAR, {{\"application/vnd.rar\"}, \"\"}},\n        {CT_APPLICATION_WASM, {{\"application/wasm\"}, \"\"}},\n        {CT_APPLICATION_X_BZIP, {{\"application/x-bzip\"}, \"\"}},\n        {CT_APPLICATION_X_BZIP2, {{\"application/x-bzip2\"}, \"\"}},\n        {CT_APPLICATION_X_7Z, {{\"application/x-7z-compressed\"}, \"\"}},\n        {CT_APPLICATION_X_HTTPD_PHP, {{\"application/x-httpd-php\"}, \"\"}},\n        {CT_APPLICATION_X_JAVASCRIPT,\n         {{\"application/x-javascript\"},\n          \"application/x-javascript; charset=utf-8\"}},\n        {CT_APPLICATION_X_FONT_OPENTYPE,\n         {{\"application/x-font-opentype\", \"font/otf\"}, \"\"}},\n        {CT_APPLICATION_X_FONT_TRUETYPE,\n         {{\"application/x-font-truetype\", \"font/ttf\"}, \"\"}},\n        {CT_APPLICATION_X_TAR, {{\"application/x-tar\"}, \"\"}},\n        {CT_APPLICATION_X_TGZ, {{\"application/x-tgz\"}, \"\"}},\n        {CT_APPLICATION_X_XZ, {{\"application/x-xz\", \"application/x-lzma\"}, \"\"}},\n        {CT_APPLICATION_XHTML,\n         {{\"application/xhtml+xml\", \"application/xhtml\"},\n          \"application/xhtml+xml; charset=utf-8\"}},\n        {CT_APPLICATION_XML,\n         {{\"application/xml\"}, \"application/xml; charset=utf-8\"}},\n        {CT_APPLICATION_ZIP, {{\"application/zip\"}, \"\"}},\n        {CT_AUDIO_AAC, {{\"audio/aac\", \"audio/aacp\"}, \"\"}},\n        {CT_AUDIO_AC3, {{\"audio/ac3\"}, \"\"}},\n        {CT_AUDIO_AIFF, {{\"audio/aiff\", \"audio/x-aiff\"}, \"\"}},\n        {CT_AUDIO_FLAC, {{\"audio/flac\"}, \"\"}},\n        {CT_AUDIO_MATROSKA, {{\"audio/matroska\", \"audio/x-matroska\"}, \"\"}},\n        {CT_AUDIO_MPEG, {{\"audio/mpeg\"}, \"\"}},\n        {CT_AUDIO_MPEG4, {{\"audio/mp4\", \"audio/x-m4a\"}, \"\"}},\n        {CT_AUDIO_OGG, {{\"audio/ogg\"}, \"\"}},\n        {CT_AUDIO_WAVE, {{\"audio/wav\", \"audio/x-wav\"}, \"\"}},\n        {CT_AUDIO_X_APE, {{\"audio/x-ape\"}, \"\"}},\n        {CT_AUDIO_X_MS_WMA, {{\"audio/x-ms-wma\"}, \"\"}},\n        {CT_AUDIO_X_TTA, {{\"audio/x-tta\"}, \"\"}},\n        {CT_AUDIO_X_WAVPACK, {{\"audio/x-wavpack\"}, \"\"}},\n        {CT_AUDIO_WEBM, {{\"audio/webm\"}, \"\"}},\n        {CT_IMAGE_APNG, {{\"image/apng\"}, \"\"}},\n        {CT_IMAGE_AVIF, {{\"image/avif\"}, \"\"}},\n        {CT_IMAGE_BMP, {{\"image/bmp\"}, \"\"}},\n        {CT_IMAGE_GIF, {{\"image/gif\"}, \"\"}},\n        {CT_IMAGE_ICNS, {{\"image/icns\"}, \"\"}},\n        {CT_IMAGE_JP2, {{\"image/jp2\", \"image/jpx\", \"image/jpm\"}, \"\"}},\n        {CT_IMAGE_JPG, {{\"image/jpeg\"}, \"\"}},\n        {CT_IMAGE_PNG, {{\"image/png\"}, \"\"}},\n        {CT_IMAGE_SVG_XML, {{\"image/svg+xml\"}, \"\"}},\n...\n// Path: lib/src/ConfigAdapterManager.h\n#pragma once\n#include \"ConfigAdapterManager.h\"\n#include \"ConfigAdapter.h\"\n#include <map>\n\nnamespace drogon\n{\nclass ConfigAdapterManager\n{\n  public:\n    static ConfigAdapterManager &instance();\n    Json::Value getJson(const std::string &content, std::string ext) const\n        noexcept(false);\n\n  private:\n    ConfigAdapterManager();\n    std::map<std::string, ConfigAdapterPtr> adapters_;\n};\n}  // namespace drogon\n\n// Path: lib/src/ConfigAdapterManager.cc\n#include \"ConfigAdapterManager.h\"\n#include \"JsonConfigAdapter.h\"\n#include \"YamlConfigAdapter.h\"\n#include <algorithm>\n\nusing namespace drogon;\n#define REGISTER_CONFIG_ADAPTER(adapter)                                    \\\n    {                                                                       \\\n        auto adapterPtr = std::make_shared<adapter>();                      \\\n        auto exts = adapterPtr->getExtensions();                            \\\n        for (auto ext : exts)                                               \\\n        {                                                                   \\\n            std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); \\\n            adapters_[ext] = adapterPtr;                                    \\\n        }                                                                   \\\n    }\n\nConfigAdapterManager &ConfigAdapterManager::instance()\n{\n    static ConfigAdapterManager instance;\n    return instance;\n}\n\nJson::Value ConfigAdapterManager::getJson(const std::string &content,\n                                          std::string ext) const noexcept(false)\n{\n    std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);\n    auto it = adapters_.find(ext);\n    if (it == adapters_.end())\n    {\n        throw std::runtime_error(\"No valid parser for this config file!\");\n    }\n    return it->second->getJson(content);\n}\n\nConfigAdapterManager::ConfigAdapterManager()\n{\n    REGISTER_CONFIG_ADAPTER(JsonConfigAdapter);\n    REGISTER_CONFIG_ADAPTER(YamlConfigAdapter);\n}\n\n// Path: lib/src/DrTemplateBase.cc\n/**\n *\n *  @file DrTemplateBase.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/DrClassMap.h>\n#include <drogon/DrTemplateBase.h>\n#include <trantor/utils/Logger.h>\n#include <memory>\n#include <regex>\n\nusing namespace drogon;\n\nstd::shared_ptr<DrTemplateBase> DrTemplateBase::newTemplate(\n    const std::string &templateName)\n{\n    LOG_TRACE << \"http view name=\" << templateName;\n    auto l = templateName.length();\n    if (l >= 4 && templateName[l - 4] == '.' && templateName[l - 3] == 'c' &&\n        templateName[l - 2] == 's' && templateName[l - 1] == 'p')\n    {\n        std::string::size_type pos = 0;\n        std::string newName;\n        newName.reserve(templateName.size());\n        if (templateName[0] == '/' || templateName[0] == '\\\\')\n        {\n            pos = 1;\n        }\n        else if (templateName[0] == '.' &&\n                 (templateName[1] == '/' || templateName[1] == '\\\\'))\n        {\n            pos = 2;\n        }\n        while (pos < l - 4)\n        {\n            if (templateName[pos] == '/' || templateName[pos] == '\\\\')\n            {\n                newName.append(\"::\");\n            }\n            else\n            {\n                newName.append(1, templateName[pos]);\n            }\n            ++pos;\n        }\n        return std::shared_ptr<DrTemplateBase>(dynamic_cast<DrTemplateBase *>(\n            drogon::DrClassMap::newObject(newName)));\n    }\n    else\n    {\n        return std::shared_ptr<DrTemplateBase>(dynamic_cast<DrTemplateBase *>(\n            drogon::DrClassMap::newObject(templateName)));\n    }\n}\n\n// Path: lib/src/DrClassMap.cc\n/**\n *\n *  DrClassMap.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/DrClassMap.h>\n#include <drogon/DrObject.h>\n#include <trantor/utils/Logger.h>\n\nusing namespace drogon;\n\nnamespace drogon\n{\nnamespace internal\n{\nstatic std::unordered_map<std::string, std::shared_ptr<DrObjectBase>> &\ngetObjsMap()\n{\n    static std::unordered_map<std::string, std::shared_ptr<DrObjectBase>>\n        singleInstanceMap;\n    return singleInstanceMap;\n}\n\nstatic std::mutex &getMapMutex()\n{\n    static std::mutex mtx;\n    return mtx;\n}\n\n}  // namespace internal\n}  // namespace drogon\n\nvoid DrClassMap::registerClass(const std::string &className,\n                               const DrAllocFunc &func,\n                               const DrSharedAllocFunc &sharedFunc)\n{\n    LOG_TRACE << \"Register class:\" << className;\n    getMap().insert(\n        std::make_pair(className, std::make_pair(func, sharedFunc)));\n}\n\nDrObjectBase *DrClassMap::newObject(const std::string &className)\n{\n    auto iter = getMap().find(className);\n    if (iter != getMap().end())\n    {\n        return iter->second.first();\n    }\n    else\n        return nullptr;\n}\n\nstd::shared_ptr<DrObjectBase> DrClassMap::newSharedObject(\n    const std::string &className)\n{\n    auto iter = getMap().find(className);\n    if (iter != getMap().end())\n    {\n        if (iter->second.second)\n            return iter->second.second();\n        else\n            return std::shared_ptr<DrObjectBase>(iter->second.first());\n    }\n    else\n        return nullptr;\n}\n\nconst std::shared_ptr<DrObjectBase> &DrClassMap::getSingleInstance(\n    const std::string &className)\n{\n    auto &mtx = internal::getMapMutex();\n    auto &singleInstanceMap = internal::getObjsMap();\n    {\n        std::lock_guard<std::mutex> lock(mtx);\n        auto iter = singleInstanceMap.find(className);\n        if (iter != singleInstanceMap.end())\n            return iter->second;\n    }\n    auto newObj = newSharedObject(className);\n    {\n        std::lock_guard<std::mutex> lock(mtx);\n        auto ret = singleInstanceMap.insert(\n            std::make_pair(className, std::move(newObj)));\n        return ret.first->second;\n    }\n}\n\nvoid DrClassMap::setSingleInstance(const std::shared_ptr<DrObjectBase> &ins)\n{\n    auto &mtx = internal::getMapMutex();\n    auto &singleInstanceMap = internal::getObjsMap();\n    std::lock_guard<std::mutex> lock(mtx);\n    singleInstanceMap[ins->className()] = ins;\n}\n\nstd::vector<std::string> DrClassMap::getAllClassName()\n{\n    std::vector<std::string> ret;\n    for (auto const &iter : getMap())\n    {\n        ret.push_back(iter.first);\n    }\n    return ret;\n}\n\nstd::unordered_map<std::string, std::pair<DrAllocFunc, DrSharedAllocFunc>> &\nDrClassMap::getMap()\n{\n    static std::unordered_map<std::string,\n                              std::pair<DrAllocFunc, DrSharedAllocFunc>>\n        map;\n    return map;\n}\n\n// Path: lib/src/HttpFileImpl.h\n/**\n *\n *  @file HttpFileImpl.h\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n#include \"HttpUtils.h\"\n#include <drogon/HttpRequest.h>\n\n#include <map>\n#include <string>\n#include <vector>\n#include <memory>\n#include <filesystem>\n#include <string_view>\n\nnamespace drogon\n{\nclass HttpFileImpl\n{\n  public:\n    /// Return the file name;\n    const std::string &getFileName() const noexcept\n    {\n        return fileName_;\n    }\n\n    /// Set the file name, usually called by the MultiPartParser parser.\n    void setFileName(const std::string &fileName) noexcept\n    {\n        fileName_ = fileName;\n    }\n\n    void setFileName(std::string &&fileName) noexcept\n    {\n        fileName_ = std::move(fileName);\n    }\n\n    /// Return the file extension;\n    std::string_view getFileExtension() const noexcept\n    {\n        return drogon::getFileExtension(fileName_);\n    }\n\n    /// Set the contents of the file, usually called by the MultiPartParser\n    /// parser.\n    void setFile(const char *data, size_t length) noexcept\n    {\n        fileContent_ = std::string_view{data, length};\n    }\n\n    /// Save the file to the file system.\n    /**\n     * The folder saving the file is app().getUploadPath().\n     * The full path is app().getUploadPath()+\"/\"+this->getFileName()\n     */\n    int save() const noexcept;\n\n    /// Save the file to @param path\n    /**\n     * @param path if the parameter is prefixed with \"/\", \"./\" or \"../\", or is\n     * \".\" or \"..\", the full path is path+\"/\"+this->getFileName(),\n     * otherwise the file is saved as\n     * app().getUploadPath()+\"/\"+path+\"/\"+this->getFileName()\n     */\n    int save(const std::string &path) const noexcept;\n\n    /// Save the file to file system with a new name\n    /**\n     * @param fileName if the parameter isn't prefixed with \"/\", \"./\" or \"../\",\n     * the full path is app().getUploadPath()+\"/\"+filename, otherwise the file\n     * is saved as the filename\n     */\n    int saveAs(const std::string &fileName) const noexcept;\n\n    /// Return the file length.\n    size_t fileLength() const noexcept\n    {\n        return fileContent_.length();\n    }\n\n    const char *fileData() const noexcept\n    {\n        return fileContent_.data();\n    }\n\n    const std::string_view &fileContent() const noexcept\n    {\n        return fileContent_;\n    }\n\n    /// Return the name of the item in multiple parts.\n    const std::string &getItemName() const noexcept\n    {\n        return itemName_;\n    }\n\n    void setItemName(const std::string &itemName) noexcept\n    {\n        itemName_ = itemName;\n    }\n\n    void setItemName(std::string &&itemName) noexcept\n    {\n        itemName_ = std::move(itemName);\n    }\n\n    /// Return the type of file.\n    FileType getFileType() const noexcept\n    {\n        auto ft = drogon::getFileType(contentType_);\n        if ((ft != FT_UNKNOWN) && (ft != FT_CUSTOM))\n            return ft;\n        return parseFileType(getFileExtension());\n    }\n\n    /// Return md5 hash of the file\n    std::string getMd5() const noexcept;\n    // Return sha1 hash of the file\n    std::string getSha256() const noexcept;\n    // Return sha512 hash of the file\n    std::string getSha3() const noexcept;\n    //    int saveTo(const std::string &pathAndFileName) const;\n    int saveTo(const std::filesystem::path &pathAndFileName) const noexcept;\n\n    void setRequest(const HttpRequestPtr &req) noexcept\n    {\n        requestPtr_ = req;\n    }\n\n    drogon::ContentType getContentType() const noexcept\n    {\n        return contentType_;\n    }\n\n    void setContentType(drogon::ContentType contentType) noexcept\n    {\n        contentType_ = contentType;\n    }\n\n    void setContentTransferEncoding(\n        const std::string &contentTransferEncoding) noexcept\n    {\n        transferEncoding_ = contentTransferEncoding;\n    }\n\n    void setContentTransferEncoding(\n        std::string &&contentTransferEncoding) noexcept\n    {\n        transferEncoding_ = std::move(contentTransferEncoding);\n    }\n\n    const std::string &getContentTransferEncoding() const noexcept\n    {\n        return transferEncoding_;\n    }\n\n  private:\n    std::string fileName_;\n    std::string itemName_;\n    std::string transferEncoding_;\n    std::string_view fileContent_;\n    HttpRequestPtr requestPtr_;\n    drogon::ContentType contentType_{drogon::CT_NONE};\n};\n}  // namespace drogon\n\n// Path: lib/src/HttpFileImpl.cc\n/**\n *\n *  @file HttpFileImpl.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpFileImpl.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/MultiPart.h>\n#include <drogon/utils/Utilities.h>\n#include <fstream>\n#include <iostream>\n#include <algorithm>\n#include <filesystem>\n\nusing namespace drogon;\n\nint HttpFileImpl::save() const noexcept\n{\n    return save(HttpAppFrameworkImpl::instance().getUploadPath());\n}\n\nint HttpFileImpl::save(const std::string &path) const noexcept\n{\n    assert(!path.empty());\n    if (fileName_.empty())\n        return -1;\n    std::filesystem::path fsUploadDir(utils::toNativePath(path));\n\n    if (fsUploadDir.is_absolute())\n    {  // do nothing\n    }\n    else if ((!fsUploadDir.has_parent_path() ||\n              (fsUploadDir.begin()->string() != \".\" &&\n               fsUploadDir.begin()->string() != \"..\")))\n    {\n        fsUploadDir = utils::toNativePath(\n                          HttpAppFrameworkImpl::instance().getUploadPath()) /\n                      fsUploadDir;\n    }\n    else\n    {\n        fsUploadDir = std::filesystem::current_path() / fsUploadDir;\n    }\n\n    fsUploadDir = std::filesystem::weakly_canonical(fsUploadDir);\n\n    if (!std::filesystem::exists(fsUploadDir))\n    {\n        LOG_TRACE << \"create path:\" << fsUploadDir;\n        std::error_code err;\n        std::filesystem::create_directories(fsUploadDir, err);\n        if (err)\n        {\n            LOG_SYSERR;\n            return -1;\n        }\n    }\n\n    std::filesystem::path fsSaveToPath(std::filesystem::weakly_canonical(\n        fsUploadDir / utils::toNativePath(fileName_)));\n    LOG_TRACE << \"save to path:\" << fsSaveToPath;\n    if (!std::equal(fsUploadDir.begin(),\n                    fsUploadDir.end(),\n                    fsSaveToPath.begin()))\n    {\n        LOG_ERROR\n            << \"Attempt writing outside of upload directory detected. Path: \"\n            << fileName_;\n        return -1;\n    }\n\n    return saveTo(fsSaveToPath);\n}\n\nint HttpFileImpl::saveAs(const std::string &fileName) const noexcept\n{\n    assert(!fileName.empty());\n    std::filesystem::path fsFileName(utils::toNativePath(fileName));\n    if (!fsFileName.is_absolute() && (!fsFileName.has_parent_path() ||\n                                      (fsFileName.begin()->string() != \".\" &&\n                                       fsFileName.begin()->string() != \"..\")))\n    {\n        std::filesystem::path fsUploadPath(utils::toNativePath(\n            HttpAppFrameworkImpl::instance().getUploadPath()));\n        fsFileName = fsUploadPath / fsFileName;\n    }\n    if (fsFileName.has_parent_path() &&\n        !std::filesystem::exists(fsFileName.parent_path()))\n    {\n        LOG_TRACE << \"create path:\" << fsFileName.parent_path();\n        std::error_code err;\n        std::filesystem::create_directories(fsFileName.parent_path(), err);\n        if (err)\n        {\n            LOG_SYSERR;\n            return -1;\n        }\n    }\n    return saveTo(fsFileName);\n}\n\nint HttpFileImpl::saveTo(\n    const std::filesystem::path &pathAndFileName) const noexcept\n{\n    LOG_TRACE << \"save uploaded file:\" << pathAndFileName;\n    auto wPath = utils::toNativePath(pathAndFileName.native());\n    std::ofstream file(wPath, std::ios::binary);\n    if (file.is_open())\n    {\n        file.write(fileContent_.data(), fileContent_.size());\n        file.close();\n        return 0;\n    }\n    else\n    {\n        LOG_ERROR << \"save failed!\";\n        return -1;\n    }\n}\n\nstd::string HttpFileImpl::getMd5() const noexcept\n{\n    return utils::getMd5(fileContent_.data(), fileContent_.size());\n}\n\nstd::string HttpFileImpl::getSha256() const noexcept\n{\n    return utils::getSha256(fileContent_.data(), fileContent_.size());\n}\n\nstd::string HttpFileImpl::getSha3() const noexcept\n{\n    return utils::getSha3(fileContent_.data(), fileContent_.size());\n}\n\nconst std::string &HttpFile::getFileName() const noexcept\n{\n    return implPtr_->getFileName();\n}\n\nvoid HttpFile::setFileName(const std::string &fileName) noexcept\n{\n    implPtr_->setFileName(fileName);\n}\n\nstd::string_view HttpFile::getFileExtension() const noexcept\n{\n    return implPtr_->getFileExtension();\n}\n\nFileType HttpFile::getFileType() const noexcept\n{\n    return implPtr_->getFileType();\n}\n\nvoid HttpFile::setFile(const char *data, size_t length) noexcept\n{\n    implPtr_->setFile(data, length);\n}\n\nint HttpFile::save() const noexcept\n{\n    return implPtr_->save();\n}\n\nint HttpFile::save(const std::string &path) const noexcept\n{\n    return implPtr_->save(path);\n}\n\nint HttpFile::saveAs(const std::string &fileName) const noexcept\n{\n    return implPtr_->saveAs(fileName);\n}\n\nsize_t HttpFile::fileLength() const noexcept\n{\n    return implPtr_->fileLength();\n}\n\ndrogon::ContentType HttpFile::getContentType() const noexcept\n{\n    return implPtr_->getContentType();\n}\n\nconst char *HttpFile::fileData() const noexcept\n{\n    return implPtr_->fileData();\n}\n\nstd::string HttpFile::getMd5() const noexcept\n{\n    return implPtr_->getMd5();\n}\n\nconst std::string &HttpFile::getContentTransferEncoding() const noexcept\n{\n    return implPtr_->getContentTransferEncoding();\n}\n\nHttpFile::HttpFile(std::shared_ptr<HttpFileImpl> &&implPtr) noexcept\n    : implPtr_(std::move(implPtr))\n{\n}\n\nconst std::string &HttpFile::getItemName() const noexcept\n{\n    return implPtr_->getItemName();\n}\n\n// Path: lib/src/ListenerManager.cc\n/**\n *\n *  @file ListenerManager.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"ListenerManager.h\"\n#include <drogon/config.h>\n#include <fcntl.h>\n#include <trantor/utils/Logger.h>\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpServer.h\"\n#ifndef _WIN32\n#include <sys/file.h>\n#include <unistd.h>\n#endif\n\nnamespace drogon\n{\n#ifndef _WIN32\nclass DrogonFileLocker : public trantor::NonCopyable\n{\n  public:\n    DrogonFileLocker()\n    {\n        fd_ = open(\"/tmp/drogon.lock\", O_TRUNC | O_CREAT, 0666);\n        flock(fd_, LOCK_EX);\n    }\n\n    ~DrogonFileLocker()\n    {\n        close(fd_);\n    }\n\n  private:\n    int fd_{0};\n};\n\n#endif\n}  // namespace drogon\n\nusing namespace trantor;\nusing namespace drogon;\n\nvoid ListenerManager::addListener(\n    const std::string &ip,\n    uint16_t port,\n    bool useSSL,\n    const std::string &certFile,\n    const std::string &keyFile,\n    bool useOldTLS,\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds)\n{\n    if (useSSL && !utils::supportsTls())\n        LOG_ERROR << \"Can't use SSL without OpenSSL found in your system\";\n    listeners_.emplace_back(\n        ip, port, useSSL, certFile, keyFile, useOldTLS, sslConfCmds);\n}\n\nstd::vector<trantor::InetAddress> ListenerManager::getListeners() const\n{\n    std::vector<trantor::InetAddress> listeners;\n    for (auto &server : servers_)\n    {\n        listeners.emplace_back(server->address());\n    }\n    return listeners;\n}\n\nvoid ListenerManager::createListeners(\n    const std::string &globalCertFile,\n    const std::string &globalKeyFile,\n    const std::vector<std::pair<std::string, std::string>> &sslConfCmds,\n    const std::vector<trantor::EventLoop *> &ioLoops)\n{\n    LOG_TRACE << \"thread num=\" << ioLoops.size();\n#ifdef __linux__\n    for (size_t i = 0; i < ioLoops.size(); ++i)\n    {\n        for (auto const &listener : listeners_)\n        {\n            auto const &ip = listener.ip_;\n            bool isIpv6 = (ip.find(':') != std::string::npos);\n            InetAddress listenAddress(ip, listener.port_, isIpv6);\n            if (listenAddress.isUnspecified())\n            {\n                LOG_FATAL << \"Failed to parse IP address '\" << ip\n                          << \"'. (Note: FQDN/domain names/hostnames are not \"\n                             \"supported. Including 'localhost')\";\n                abort();\n            }\n            if (i == 0 && !app().reusePort())\n            {\n                DrogonFileLocker lock;\n                // Check whether the port is in use.\n                TcpServer server(HttpAppFrameworkImpl::instance().getLoop(),\n                                 listenAddress,\n                                 \"drogonPortTest\",\n                                 true,\n                                 false);\n            }\n            std::shared_ptr<HttpServer> serverPtr =\n                std::make_shared<HttpServer>(ioLoops[i],\n                                             listenAddress,\n                                             \"drogon\");\n\n            if (listener.useSSL_ && utils::supportsTls())\n            {\n                auto cert = listener.certFile_;\n                auto key = listener.keyFile_;\n                if (cert.empty())\n                    cert = globalCertFile;\n                if (key.empty())\n                    key = globalKeyFile;\n                if (cert.empty() || key.empty())\n                {\n                    std::cerr\n                        << \"You can't use https without cert file or key file\"\n                        << std::endl;\n                    exit(1);\n                }\n                auto cmds = sslConfCmds;\n                std::copy(listener.sslConfCmds_.begin(),\n                          listener.sslConfCmds_.end(),\n                          std::back_inserter(cmds));\n                auto policy =\n                    trantor::TLSPolicy::defaultServerPolicy(cert, key);\n                policy->setConfCmds(cmds).setUseOldTLS(listener.useOldTLS_);\n                serverPtr->enableSSL(std::move(policy));\n            }\n            servers_.push_back(serverPtr);\n        }\n    }\n#else\n\n    if (!listeners_.empty())\n    {\n        listeningThread_ =\n            std::make_unique<EventLoopThread>(\"DrogonListeningLoop\");\n        listeningThread_->run();\n        for (auto const &listener : listeners_)\n        {\n            auto ip = listener.ip_;\n            bool isIpv6 = (ip.find(':') != std::string::npos);\n            auto serverPtr = std::make_shared<HttpServer>(\n                listeningThread_->getLoop(),\n                InetAddress(ip, listener.port_, isIpv6),\n                \"drogon\");\n            if (listener.useSSL_ && utils::supportsTls())\n            {\n                auto cert = listener.certFile_;\n                auto key = listener.keyFile_;\n                if (cert.empty())\n                    cert = globalCertFile;\n                if (key.empty())\n                    key = globalKeyFile;\n                if (cert.empty() || key.empty())\n                {\n                    std::cerr\n                        << \"You can't use https without cert file or key file\"\n                        << std::endl;\n                    exit(1);\n                }\n                auto cmds = sslConfCmds;\n                auto policy =\n                    trantor::TLSPolicy::defaultServerPolicy(cert, key);\n                policy->setConfCmds(cmds).setUseOldTLS(listener.useOldTLS_);\n                serverPtr->enableSSL(std::move(policy));\n            }\n            serverPtr->setIoLoops(ioLoops);\n            servers_.push_back(serverPtr);\n        }\n    }\n#endif\n}\n\nvoid ListenerManager::startListening()\n{\n    for (auto &server : servers_)\n    {\n        server->start();\n    }\n}\n\nvoid ListenerManager::stopListening()\n{\n    for (auto &serverPtr : servers_)\n    {\n        serverPtr->stop();\n    }\n    if (listeningThread_)\n    {\n        auto loop = listeningThread_->getLoop();\n        assert(!loop->isInLoopThread());\n        loop->quit();\n        listeningThread_->wait();\n    }\n}\n\n// Path: lib/src/RealIpResolver.cc\n/**\n *\n *  @file RealIpResolver.cc\n *  @author Nitromelon\n *\n *  Copyright 2022, Nitromelon. All rights reserved.\n *  https://github.com/drogonframework/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include <drogon/drogon.h>\n#include <trantor/utils/Logger.h>\n#include <drogon/plugins/RealIpResolver.h>\n\nusing namespace drogon;\nusing namespace drogon::plugin;\n\nstruct XForwardedForParser : public trantor::NonCopyable\n{\n    explicit XForwardedForParser(std::string value)\n        : value_(std::move(value)), start_(value_.c_str()), len_(value_.size())\n    {\n    }\n\n    std::string getNext()\n    {\n        if (len_ == 0)\n        {\n            return {};\n        }\n        // Skip trailing separators\n        const char *cur;\n        for (cur = start_ + len_ - 1; cur > start_; --cur, --len_)\n        {\n            if (*cur != ' ' && *cur != ',')\n            {\n                break;\n            }\n        }\n        for (; cur > start_; --cur)\n        {\n            if (*cur == ' ' || *cur == ',')\n            {\n                ++cur;\n                break;\n            }\n        }\n        std::string ip{cur, len_ - (cur - start_)};\n        len_ = cur == start_ ? 0 : cur - start_ - 1;\n        return ip;\n    }\n\n  private:\n    std::string value_;\n    const char *start_;\n    size_t len_;\n};\n\nstatic trantor::InetAddress parseAddress(const std::string &addr)\n{\n    auto pos = addr.find(':');\n    uint16_t port = 0;\n    if (pos == std::string::npos)\n    {\n        return trantor::InetAddress(addr, 0);\n    }\n    try\n    {\n        port = std::stoi(addr.substr(pos + 1));\n    }\n    catch (const std::exception &ex)\n    {\n        (void)ex;\n        LOG_ERROR << \"Error in ipv4 address: \" + addr;\n        port = 0;\n    }\n    return trantor::InetAddress(addr.substr(0, pos), port);\n}\n\nvoid RealIpResolver::initAndStart(const Json::Value &config)\n{\n    fromHeader_ = config.get(\"from_header\", \"x-forwarded-for\").asString();\n    attributeKey_ = config.get(\"attribute_key\", \"real-ip\").asString();\n\n    std::transform(fromHeader_.begin(),\n                   fromHeader_.end(),\n                   fromHeader_.begin(),\n                   [](unsigned char c) { return tolower(c); });\n    if (fromHeader_ == \"x-forwarded-for\")\n    {\n        useXForwardedFor_ = true;\n    }\n\n    const Json::Value &trustIps = config[\"trust_ips\"];\n    if (!trustIps.isArray())\n    {\n        throw std::runtime_error(\"Invalid trusted_ips. Should be array.\");\n    }\n    for (const auto &elem : trustIps)\n    {\n        std::string ipOrCidr = elem.asString();\n        trustCIDRs_.emplace_back(ipOrCidr);\n    }\n\n    drogon::app().registerPreRoutingAdvice([this](const HttpRequestPtr &req) {\n        const std::string &ipHeader = req->getHeader(fromHeader_);\n        const trantor::InetAddress &peerAddr = req->getPeerAddr();\n        if (ipHeader.empty() || !matchCidr(peerAddr))\n        {\n            // Target header is empty, or\n            // direct peer is already a non-proxy\n            req->attributes()->insert(attributeKey_, peerAddr);\n            return;\n        }\n        // Use a header field which contains a single ip\n        if (!useXForwardedFor_)\n        {\n            trantor::InetAddress addr = parseAddress(ipHeader);\n            if (addr.isUnspecified())\n            {\n                req->attributes()->insert(attributeKey_, peerAddr);\n            }\n            else\n            {\n                req->attributes()->insert(attributeKey_, addr);\n            }\n            return;\n        }\n        // Use x-forwarded-for header, which may contains multiple ip address,\n        // separated by comma\n        XForwardedForParser parser(ipHeader);\n        std::string ip;\n        while (!(ip = parser.getNext()).empty())\n        {\n            trantor::InetAddress addr = parseAddress(ip);\n            if (addr.isUnspecified() || matchCidr(addr))\n            {\n                continue;\n            }\n            req->attributes()->insert(attributeKey_, addr);\n            return;\n        }\n        // No match, use peerAddr\n        req->attributes()->insert(attributeKey_, peerAddr);\n    });\n}\n\nvoid RealIpResolver::shutdown()\n{\n}\n\nconst trantor::InetAddress &RealIpResolver::GetRealAddr(\n    const HttpRequestPtr &req)\n{\n    auto *plugin = app().getPlugin<drogon::plugin::RealIpResolver>();\n    if (!plugin)\n    {\n        return req->getPeerAddr();\n    }\n    return plugin->getRealAddr(req);\n}\n\nconst trantor::InetAddress &RealIpResolver::getRealAddr(\n    const HttpRequestPtr &req) const\n{\n    const std::shared_ptr<Attributes> &attributesPtr = req->getAttributes();\n    if (!attributesPtr->find(attributeKey_))\n    {\n        return req->getPeerAddr();\n    }\n    return attributesPtr->get<trantor::InetAddress>(attributeKey_);\n}\n\nbool RealIpResolver::matchCidr(const trantor::InetAddress &addr) const\n{\n    for (auto &cidr : trustCIDRs_)\n    {\n        if ((addr.ipNetEndian() & cidr.mask_) == cidr.addr_)\n        {\n            return true;\n        }\n    }\n    return false;\n}\n\nRealIpResolver::CIDR::CIDR(const std::string &ipOrCidr)\n{\n    // Find CIDR slash\n    auto pos = ipOrCidr.find('/');\n    std::string ipv4;\n    if (pos != std::string::npos)\n    {\n        // parameter is a CIDR block\n        std::string prefixLen = ipOrCidr.substr(pos + 1);\n        ipv4 = ipOrCidr.substr(0, pos);\n        uint16_t prefix = std::stoi(prefixLen);\n        if (prefix > 32)\n        {\n            throw std::runtime_error(\"Bad CIDR block: \" + ipOrCidr);\n        }\n        mask_ = htonl(0xffffffffu << (32 - prefix));\n    }\n    else\n    {\n        // parameter is an IP\n        ipv4 = ipOrCidr;\n        mask_ = 0xffffffffu;\n    }\n\n    trantor::InetAddress addr(ipv4, 0);\n    if (addr.isIpV6())\n    {\n        throw std::runtime_error(\"Ipv6 is not supported by RealIpResolver.\");\n    }\n    if (addr.isUnspecified())\n    {\n        throw std::runtime_error(\"Bad ipv4 address: \" + ipv4);\n    }\n    addr_ = addr.ipNetEndian() & mask_;\n}\n\n// Path: lib/src/FiltersFunction.cc\n/**\n *\n *  @file FiltersFunction.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"FiltersFunction.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpResponseImpl.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/HttpFilter.h>\n\n#include <queue>\n\nnamespace drogon\n{\nnamespace filters_function\n{\nstatic void doFilterChains(\n    const std::vector<std::shared_ptr<HttpFilterBase>> &filters,\n    size_t index,\n    const HttpRequestImplPtr &req,\n    std::shared_ptr<const std::function<void(const HttpResponsePtr &)>>\n        &&callbackPtr)\n{\n    if (index < filters.size())\n    {\n        auto &filter = filters[index];\n        filter->doFilter(\n            req,\n            [/*copy*/ callbackPtr](const HttpResponsePtr &resp) {\n                (*callbackPtr)(resp);\n            },\n            [index, req, callbackPtr, &filters]() mutable {\n                auto ioLoop = req->getLoop();\n                if (ioLoop && !ioLoop->isInLoopThread())\n                {\n                    ioLoop->queueInLoop(\n                        [&filters,\n                         index,\n                         req,\n                         callbackPtr = std::move(callbackPtr)]() mutable {\n                            doFilterChains(filters,\n                                           index + 1,\n                                           req,\n                                           std::move(callbackPtr));\n                        });\n                }\n                else\n                {\n                    doFilterChains(filters,\n                                   index + 1,\n                                   req,\n                                   std::move(callbackPtr));\n                }\n            });\n    }\n    else\n    {\n        (*callbackPtr)(nullptr);\n    }\n}\n\nstd::vector<std::shared_ptr<HttpFilterBase>> createFilters(\n    const std::vector<std::string> &filterNames)\n{\n    std::vector<std::shared_ptr<HttpFilterBase>> filters;\n    for (auto const &filter : filterNames)\n    {\n        auto object_ = DrClassMap::getSingleInstance(filter);\n        auto filter_ = std::dynamic_pointer_cast<HttpFilterBase>(object_);\n        if (filter_)\n            filters.push_back(filter_);\n        else\n        {\n            LOG_ERROR << \"filter \" << filter << \" not found\";\n        }\n    }\n    return filters;\n}\n\nvoid doFilters(const std::vector<std::shared_ptr<HttpFilterBase>> &filters,\n               const HttpRequestImplPtr &req,\n               std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    auto callbackPtr =\n        std::make_shared<std::decay_t<decltype(callback)>>(std::move(callback));\n    doFilterChains(filters, 0, req, std::move(callbackPtr));\n}\n\n}  // namespace filters_function\n}  // namespace drogon\n\n// Path: lib/src/RangeParser.cc\n/**\n *\n *  RangeParser.h\n *  He, Wanchen\n *\n *  Copyright 2021, He,Wanchen.  All rights reserved.\n *  https://github.com/drogonframework/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"RangeParser.h\"\n\n#include <limits>\n\nusing namespace drogon;\n\nstatic constexpr size_t MAX_SIZE = std::numeric_limits<size_t>::max();\nstatic constexpr size_t MAX_TEN = MAX_SIZE / 10;\nstatic constexpr size_t MAX_DIGIT = MAX_SIZE % 10;\n\n// clang-format off\n#define DR_SKIP_WHITESPACE(p) while (*p == ' ') { ++(p); }\n#define DR_ISDIGIT(p) ('0' <= *(p) && *(p) <= '9')\n#define DR_WOULD_OVERFLOW(base, digit) \\\n    (static_cast<size_t>(base) > MAX_TEN || \\\n    (static_cast<size_t>(base) >= MAX_TEN && \\\n    static_cast<size_t>(digit) - '0' > MAX_DIGIT))\n\n// clang-format on\n\n/** Following formats are valid range header according to rfc7233`\n * Range: <unit>=<start>-\n * Range: <unit>=<start>-<end>\n * Range: <unit>=<start>-<end>, <start>-<end>\n * Range: <unit>=<start>-<end>, <start>-<end>, <start>-<end>\n * Range: <unit>=-<suffix-length>\n */\n\nFileRangeParseResult drogon::parseRangeHeader(const std::string &rangeStr,\n                                              size_t contentLength,\n                                              std::vector<FileRange> &ranges)\n{\n    if (rangeStr.size() < 7 || rangeStr.compare(0, 6, \"bytes=\") != 0)\n    {\n        return InvalidRange;\n    }\n    const char *iter = rangeStr.c_str() + 6;\n\n    size_t totalSize = 0;\n    while (true)\n    {\n        size_t start = 0;\n        size_t end = 0;\n        // If this is a suffix range: <unit>=-<suffix-length>\n        bool isSuffix = false;\n\n        DR_SKIP_WHITESPACE(iter);\n\n        if (*iter == '-')\n        {\n            isSuffix = true;\n            ++iter;\n        }\n        // Parse start\n        else\n        {\n            if (!DR_ISDIGIT(iter))\n            {\n                return InvalidRange;\n            }\n            while (DR_ISDIGIT(iter))\n            {\n                // integer out of range\n                if (DR_WOULD_OVERFLOW(start, *iter))\n                {\n                    return NotSatisfiable;\n                }\n                start = start * 10 + (*iter++ - '0');\n            }\n            DR_SKIP_WHITESPACE(iter);\n            // should be separator now\n            if (*iter++ != '-')\n            {\n                return InvalidRange;\n            }\n            DR_SKIP_WHITESPACE(iter);\n            // If this is a prefix range <unit>=<range-start>-\n            if (*iter == ',' || *iter == '\\0')\n            {\n                end = contentLength;\n                // Handle found\n                if (start < end)\n                {\n                    if (totalSize > MAX_SIZE - (end - start))\n                    {\n                        return NotSatisfiable;\n                    }\n                    totalSize += end - start;\n                    ranges.push_back({start, end});\n                }\n                if (*iter++ != ',')\n                {\n                    break;\n                }\n                continue;\n            }\n        }\n\n        // Parse end\n        if (!DR_ISDIGIT(iter))\n        {\n            return InvalidRange;\n        }\n        while (DR_ISDIGIT(iter))\n        {\n            if (DR_WOULD_OVERFLOW(end, *iter))\n            {\n                return NotSatisfiable;\n            }\n            end = end * 10 + (*iter++ - '0');\n        }\n        DR_SKIP_WHITESPACE(iter);\n\n        if (*iter != ',' && *iter != '\\0')\n        {\n            return InvalidRange;\n        }\n        if (isSuffix)\n        {\n            start = (end < contentLength) ? contentLength - end : 0;\n            end = contentLength - 1;\n        }\n        // [start, end)\n        if (end >= contentLength)\n        {\n            end = contentLength;\n        }\n        else\n        {\n            ++end;\n        }\n\n        // handle found\n        if (start < end)\n        {\n            ranges.push_back({start, end});\n            if (totalSize > MAX_SIZE - (end - start))\n            {\n                return NotSatisfiable;\n            }\n            totalSize += end - start;\n            // We restrict the number to be under 100, to avoid malicious\n            // requests.\n            // Though rfc does not say anything about max number of ranges,\n            // it does mention that server can ignore range header freely.\n            if (ranges.size() > 100)\n            {\n                return InvalidRange;\n            }\n        }\n        if (*iter++ != ',')\n        {\n            break;\n        }\n    }\n\n    if (ranges.size() == 0 || totalSize > contentLength)\n    {\n        return NotSatisfiable;\n    }\n    return ranges.size() == 1 ? SinglePart : MultiPart;\n}\n\n#undef DR_SKIP_WHITESPACE\n#undef DR_ISDIGIT\n#undef DR_WOULD_OVERFLOW\n\n// Path: lib/src/ConfigLoader.cc\n/**\n *\n *  @file ConfigLoader.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"ConfigLoader.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/config.h>\n#include <fstream>\n#include <iostream>\n#include <sstream>\n#include <thread>\n#include <trantor/utils/Logger.h>\n#if !defined(_WIN32)\n#include <unistd.h>\n#define os_access access\n#else\n#include <io.h>\n#ifndef __MINGW32__\n#define os_access _waccess\n#define R_OK 04\n#define W_OK 02\n#else\n#define os_access access\n#endif\n#endif\n\n#include <drogon/utils/Utilities.h>\n#include \"ConfigAdapterManager.h\"\n#include <filesystem>\n\nusing namespace drogon;\n\n\nstatic bool bytesSize(std::string &sizeStr, size_t &size)\n{\n    if (sizeStr.empty())\n    {\n        size = -1;\n        return true;\n    }\n    else\n    {\n        size = 1;\n        switch (sizeStr[sizeStr.length() - 1])\n        {\n            case 'k':\n            case 'K':\n                size = 1024;\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n            case 'M':\n            case 'm':\n                size = (1024 * 1024);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n            case 'g':\n            case 'G':\n                size = (1024 * 1024 * 1024);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n#if ((ULONG_MAX) != (UINT_MAX))\n            // 64bit system\n            case 't':\n            case 'T':\n                size = (1024L * 1024L * 1024L * 1024L);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n#endif\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '7':\n            case '8':\n            case '9':\n                break;\n            default:\n                return false;\n                break;\n        }\n        std::istringstream iss(sizeStr);\n        size_t tmpSize;\n        iss >> tmpSize;\n        if (iss.fail())\n        {\n            return false;\n        }\n        if ((size_t(-1) / tmpSize) >= size)\n            size *= tmpSize;\n        else\n        {\n            size = -1;\n        }\n        return true;\n    }\n}\n\nConfigLoader::ConfigLoader(const std::string &configFile)\n{\n    if (os_access(drogon::utils::toNativePath(configFile).c_str(), 0) != 0)\n    {\n        throw std::runtime_error(\"Config file \" + configFile + \" not found!\");\n    }\n    if (os_access(drogon::utils::toNativePath(configFile).c_str(), R_OK) != 0)\n    {\n        throw std::runtime_error(\"No permission to read config file \" +\n                                 configFile);\n    }\n    configFile_ = configFile;\n    auto pos = configFile.find_last_of('.');\n    if (pos == std::string::npos)\n    {\n        throw std::runtime_error(\"Invalid config file name!\");\n    }\n    auto ext = configFile.substr(pos + 1);\n    std::ifstream infile(drogon::utils::toNativePath(configFile).c_str(),\n                         std::ifstream::in);\n    // get the content of the infile\n    std::string content((std::istreambuf_iterator<char>(infile)),\n                        std::istreambuf_iterator<char>());\n    try\n    {\n        configJsonRoot_ =\n            ConfigAdapterManager::instance().getJson(content, std::move(ext));\n    }\n    catch (std::exception &e)\n    {\n        throw std::runtime_error(\"Error reading config file \" + configFile +\n                                 \": \" + e.what());\n    }\n}\n\nConfigLoader::ConfigLoader(const Json::Value &data) : configJsonRoot_(data)\n{\n}\n\nConfigLoader::ConfigLoader(Json::Value &&data)\n    : configJsonRoot_(std::move(data))\n{\n}\n\nConfigLoader::~ConfigLoader()\n{\n}\n\nstatic void loadLogSetting(const Json::Value &log)\n{\n    if (!log)\n        return;\n    auto useSpdlog = log.get(\"use_spdlog\", false).asBool();\n    auto logPath = log.get(\"log_path\", \"\").asString();\n    auto baseName = log.get(\"logfile_base_name\", \"\").asString();\n    auto logSize = log.get(\"log_size_limit\", 100000000).asUInt64();\n    auto maxFiles = log.get(\"max_files\", 0).asUInt();\n    HttpAppFrameworkImpl::instance().setLogPath(\n        logPath, baseName, logSize, maxFiles, useSpdlog);\n    auto logLevel = log.get(\"log_level\", \"DEBUG\").asString();\n    if (logLevel == \"TRACE\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kTrace);\n    }\n    else if (logLevel == \"DEBUG\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kDebug);\n    }\n    else if (logLevel == \"INFO\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kInfo);\n    }\n    else if (logLevel == \"WARN\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kWarn);\n    }\n    auto localTime = log.get(\"display_local_time\", false).asBool();\n    trantor::Logger::setDisplayLocalTime(localTime);\n}\n\nstatic void loadControllers(const Json::Value &controllers)\n{\n    if (!controllers)\n        return;\n    for (auto const &controller : controllers)\n    {\n        auto path = controller.get(\"path\", \"\").asString();\n        auto ctrlName = controller.get(\"controller\", \"\").asString();\n        if (path == \"\" || ctrlName == \"\")\n            continue;\n        std::vector<internal::HttpConstraint> constraints;\n        if (!controller[\"http_methods\"].isNull())\n        {\n            for (auto const &method : controller[\"http_methods\"])\n            {\n                auto strMethod = method.asString();\n                std::transform(strMethod.begin(),\n                               strMethod.end(),\n                               strMethod.begin(),\n                               [](unsigned char c) { return tolower(c); });\n                if (strMethod == \"get\")\n                {\n                    constraints.push_back(Get);\n                }\n                else if (strMethod == \"post\")\n                {\n                    constraints.push_back(Post);\n                }\n                else if (strMethod == \"head\")  // The branch never work\n                {\n                    constraints.push_back(Head);\n                }\n                else if (strMethod == \"put\")\n                {\n                    constraints.push_back(Put);\n                }\n                else if (strMethod == \"delete\")\n                {\n                    constraints.push_back(Delete);\n                }\n                else if (strMethod == \"patch\")\n                {\n                    constraints.push_back(Patch);\n                }\n            }\n        }\n        if (!controller[\"filters\"].isNull())\n        {\n            for (auto const &filter : controller[\"filters\"])\n            {\n                constraints.push_back(filter.asString());\n            }\n        }\n        drogon::app().registerHttpSimpleController(path, ctrlName, constraints);\n    }\n}\n\nstatic void loadApp(const Json::Value &app)\n{\n    if (!app)\n        return;\n    // threads number\n    auto threadsNum = app.get(\"threads_num\", 1).asUInt64();\n    if (threadsNum == 1)\n    {\n        threadsNum = app.get(\"number_of_threads\", 1).asUInt64();\n    }\n    if (threadsNum == 0)\n    {\n        // set the number to the number of processors.\n        threadsNum = std::thread::hardware_concurrency();\n        LOG_TRACE << \"The number of processors is \" << threadsNum;\n    }\n    if (threadsNum < 1)\n        threadsNum = 1;\n    drogon::app().setThreadNum(threadsNum);\n    // session\n    auto enableSession = app.get(\"enable_session\", false).asBool();\n    if (enableSession)\n    {\n        auto timeout = app.get(\"session_timeout\", 0).asUInt64();\n        auto sameSite = app.get(\"session_same_site\", \"Null\").asString();\n        auto cookieKey = app.get(\"session_cookie_key\", \"JSESSIONID\").asString();\n        auto maxAge = app.get(\"session_max_age\", -1).asInt();\n        drogon::app().enableSession(timeout,\n                                    Cookie::convertString2SameSite(sameSite),\n                                    cookieKey,\n                                    maxAge);\n    }\n    else\n        drogon::app().disableSession();\n    // document root\n    auto documentRoot = app.get(\"document_root\", \"\").asString();\n    if (documentRoot != \"\")\n    {\n        drogon::app().setDocumentRoot(documentRoot);\n    }\n    if (!app[\"static_file_headers\"].empty())\n    {\n        if (app[\"static_file_headers\"].isArray())\n        {\n            std::vector<std::pair<std::string, std::string>> headers;\n            for (auto &header : app[\"static_file_headers\"])\n            {\n                headers.emplace_back(\n                    std::make_pair(header[\"name\"].asString(),\n                                   header[\"value\"].asString()));\n            }\n            drogon::app().setStaticFileHeaders(headers);\n        }\n        else\n        {\n            throw std::runtime_error(\n                \"The static_file_headers option must be an array\");\n        }\n    }\n    // upload path\n    auto uploadPath = app.get(\"upload_path\", \"uploads\").asString();\n    drogon::app().setUploadPath(uploadPath);\n    // file types\n    auto fileTypes = app[\"file_types\"];\n    if (fileTypes.isArray() && !fileTypes.empty())\n    {\n        std::vector<std::string> types;\n        for (auto const &fileType : fileTypes)\n        {\n            types.push_back(fileType.asString());\n            LOG_TRACE << \"file type:\" << types.back();\n        }\n        drogon::app().setFileTypes(types);\n    }\n    // locations\n    if (app.isMember(\"locations\"))\n    {\n        auto &locations = app[\"locations\"];\n        if (!locations.isArray())\n        {\n            throw std::runtime_error(\"The locations option must be an array\");\n        }\n        for (auto &location : locations)\n        {\n            auto uri = location.get(\"uri_prefix\", \"\").asString();\n            if (uri.empty())\n                continue;\n            auto defaultContentType =\n                location.get(\"default_content_type\", \"\").asString();\n            auto alias = location.get(\"alias\", \"\").asString();\n            auto isCaseSensitive =\n                location.get(\"is_case_sensitive\", false).asBool();\n...\n", "needle_token_start": 13407, "needle_token_end": 13892, "code_context_ntokens": 16094, "output": ["```cpp\nstatic bool bytesSize(std::string &sizeStr, size_t &size)\n{\n    if (sizeStr.empty())\n    {\n        size = -1;\n        return true;\n    }\n    else\n    {\n        size = 1;\n        switch (sizeStr[sizeStr.length() - 1])\n        {\n            case 'k':\n            case 'K':\n                size = 1024;\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n            case 'M':\n            case 'm':\n                size = (1024 * 1024);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n            case 'g':\n            case 'G':\n                size = (1024 * 1024 * 1024);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n#if ((ULONG_MAX) != (UINT_MAX))\n            // 64bit system\n            case 't':\n            case 'T':\n                size = (1024L * 1024L * 1024L * 1024L);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n#endif\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '7':\n            case '8':\n            case '9':\n                break;\n            default:\n                return false;\n                break;\n        }\n        std::istringstream iss(sizeStr);\n        size_t tmpSize;\n        iss >> tmpSize;\n        if (iss.fail())\n        {\n            return false;\n        }\n        if ((size_t(-1) / tmpSize) >= size)\n            size *= tmpSize;\n        else\n        {\n            size = -1;\n        }\n        return true;\n    }\n}\n```"]}
{"repo": "drogonframework/drogon", "name": "ChunkingParams", "language": "cpp", "path": "lib/src/HttpServer.cc", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The purpose is to manage the process of dynamically fetching data in chunks for transmission over a network, particularly useful in scenarios where data needs to be sent in segments.\n2. **Input**: The input is a callback function that specifies how to retrieve the data chunks. This function takes a buffer and a size, and returns the amount of data actually filled into the buffer.\n3. **Output**: There is no direct output; however, the callback function indirectly determines the data chunks that are output through its return value, which indicates the size of the data filled.\n4. **Procedure**: The procedure involves initializing with a callback function that is responsible for providing data chunks. When data needs to be sent, this callback is invoked with a buffer and size, and it should fill the buffer with up to 'size' bytes of data, returning the number of bytes actually written.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/src/ConfigLoader.cc\n/**\n *\n *  @file ConfigLoader.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"ConfigLoader.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include <drogon/config.h>\n#include <fstream>\n#include <iostream>\n#include <sstream>\n#include <thread>\n#include <trantor/utils/Logger.h>\n#if !defined(_WIN32)\n#include <unistd.h>\n#define os_access access\n#else\n#include <io.h>\n#ifndef __MINGW32__\n#define os_access _waccess\n#define R_OK 04\n#define W_OK 02\n#else\n#define os_access access\n#endif\n#endif\n\n#include <drogon/utils/Utilities.h>\n#include \"ConfigAdapterManager.h\"\n#include <filesystem>\n\nusing namespace drogon;\n\nstatic bool bytesSize(std::string &sizeStr, size_t &size)\n{\n    if (sizeStr.empty())\n    {\n        size = -1;\n        return true;\n    }\n    else\n    {\n        size = 1;\n        switch (sizeStr[sizeStr.length() - 1])\n        {\n            case 'k':\n            case 'K':\n                size = 1024;\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n            case 'M':\n            case 'm':\n                size = (1024 * 1024);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n            case 'g':\n            case 'G':\n                size = (1024 * 1024 * 1024);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n#if ((ULONG_MAX) != (UINT_MAX))\n            // 64bit system\n            case 't':\n            case 'T':\n                size = (1024L * 1024L * 1024L * 1024L);\n                sizeStr.resize(sizeStr.length() - 1);\n                break;\n#endif\n            case '0':\n            case '1':\n            case '2':\n            case '3':\n            case '4':\n            case '5':\n            case '7':\n            case '8':\n            case '9':\n                break;\n            default:\n                return false;\n                break;\n        }\n        std::istringstream iss(sizeStr);\n        size_t tmpSize;\n        iss >> tmpSize;\n        if (iss.fail())\n        {\n            return false;\n        }\n        if ((size_t(-1) / tmpSize) >= size)\n            size *= tmpSize;\n        else\n        {\n            size = -1;\n        }\n        return true;\n    }\n}\n\nConfigLoader::ConfigLoader(const std::string &configFile)\n{\n    if (os_access(drogon::utils::toNativePath(configFile).c_str(), 0) != 0)\n    {\n        throw std::runtime_error(\"Config file \" + configFile + \" not found!\");\n    }\n    if (os_access(drogon::utils::toNativePath(configFile).c_str(), R_OK) != 0)\n    {\n        throw std::runtime_error(\"No permission to read config file \" +\n                                 configFile);\n    }\n    configFile_ = configFile;\n    auto pos = configFile.find_last_of('.');\n    if (pos == std::string::npos)\n    {\n        throw std::runtime_error(\"Invalid config file name!\");\n    }\n    auto ext = configFile.substr(pos + 1);\n    std::ifstream infile(drogon::utils::toNativePath(configFile).c_str(),\n                         std::ifstream::in);\n    // get the content of the infile\n    std::string content((std::istreambuf_iterator<char>(infile)),\n                        std::istreambuf_iterator<char>());\n    try\n    {\n        configJsonRoot_ =\n            ConfigAdapterManager::instance().getJson(content, std::move(ext));\n    }\n    catch (std::exception &e)\n    {\n        throw std::runtime_error(\"Error reading config file \" + configFile +\n                                 \": \" + e.what());\n    }\n}\n\nConfigLoader::ConfigLoader(const Json::Value &data) : configJsonRoot_(data)\n{\n}\n\nConfigLoader::ConfigLoader(Json::Value &&data)\n    : configJsonRoot_(std::move(data))\n{\n}\n\nConfigLoader::~ConfigLoader()\n{\n}\n\nstatic void loadLogSetting(const Json::Value &log)\n{\n    if (!log)\n        return;\n    auto useSpdlog = log.get(\"use_spdlog\", false).asBool();\n    auto logPath = log.get(\"log_path\", \"\").asString();\n    auto baseName = log.get(\"logfile_base_name\", \"\").asString();\n    auto logSize = log.get(\"log_size_limit\", 100000000).asUInt64();\n    auto maxFiles = log.get(\"max_files\", 0).asUInt();\n    HttpAppFrameworkImpl::instance().setLogPath(\n        logPath, baseName, logSize, maxFiles, useSpdlog);\n    auto logLevel = log.get(\"log_level\", \"DEBUG\").asString();\n    if (logLevel == \"TRACE\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kTrace);\n    }\n    else if (logLevel == \"DEBUG\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kDebug);\n    }\n    else if (logLevel == \"INFO\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kInfo);\n    }\n    else if (logLevel == \"WARN\")\n    {\n        trantor::Logger::setLogLevel(trantor::Logger::kWarn);\n    }\n    auto localTime = log.get(\"display_local_time\", false).asBool();\n    trantor::Logger::setDisplayLocalTime(localTime);\n}\n\nstatic void loadControllers(const Json::Value &controllers)\n{\n    if (!controllers)\n        return;\n    for (auto const &controller : controllers)\n    {\n        auto path = controller.get(\"path\", \"\").asString();\n        auto ctrlName = controller.get(\"controller\", \"\").asString();\n        if (path == \"\" || ctrlName == \"\")\n            continue;\n        std::vector<internal::HttpConstraint> constraints;\n        if (!controller[\"http_methods\"].isNull())\n        {\n            for (auto const &method : controller[\"http_methods\"])\n            {\n                auto strMethod = method.asString();\n                std::transform(strMethod.begin(),\n                               strMethod.end(),\n                               strMethod.begin(),\n                               [](unsigned char c) { return tolower(c); });\n                if (strMethod == \"get\")\n                {\n                    constraints.push_back(Get);\n                }\n                else if (strMethod == \"post\")\n                {\n                    constraints.push_back(Post);\n                }\n                else if (strMethod == \"head\")  // The branch never work\n                {\n                    constraints.push_back(Head);\n                }\n                else if (strMethod == \"put\")\n                {\n                    constraints.push_back(Put);\n                }\n                else if (strMethod == \"delete\")\n                {\n                    constraints.push_back(Delete);\n                }\n                else if (strMethod == \"patch\")\n                {\n                    constraints.push_back(Patch);\n                }\n            }\n        }\n        if (!controller[\"filters\"].isNull())\n        {\n            for (auto const &filter : controller[\"filters\"])\n            {\n                constraints.push_back(filter.asString());\n            }\n        }\n        drogon::app().registerHttpSimpleController(path, ctrlName, constraints);\n    }\n}\n\nstatic void loadApp(const Json::Value &app)\n{\n    if (!app)\n        return;\n    // threads number\n    auto threadsNum = app.get(\"threads_num\", 1).asUInt64();\n    if (threadsNum == 1)\n    {\n        threadsNum = app.get(\"number_of_threads\", 1).asUInt64();\n    }\n    if (threadsNum == 0)\n    {\n        // set the number to the number of processors.\n        threadsNum = std::thread::hardware_concurrency();\n        LOG_TRACE << \"The number of processors is \" << threadsNum;\n    }\n    if (threadsNum < 1)\n        threadsNum = 1;\n    drogon::app().setThreadNum(threadsNum);\n    // session\n    auto enableSession = app.get(\"enable_session\", false).asBool();\n    if (enableSession)\n    {\n        auto timeout = app.get(\"session_timeout\", 0).asUInt64();\n        auto sameSite = app.get(\"session_same_site\", \"Null\").asString();\n        auto cookieKey = app.get(\"session_cookie_key\", \"JSESSIONID\").asString();\n        auto maxAge = app.get(\"session_max_age\", -1).asInt();\n        drogon::app().enableSession(timeout,\n                                    Cookie::convertString2SameSite(sameSite),\n                                    cookieKey,\n                                    maxAge);\n    }\n    else\n        drogon::app().disableSession();\n    // document root\n    auto documentRoot = app.get(\"document_root\", \"\").asString();\n    if (documentRoot != \"\")\n    {\n        drogon::app().setDocumentRoot(documentRoot);\n    }\n    if (!app[\"static_file_headers\"].empty())\n    {\n        if (app[\"static_file_headers\"].isArray())\n        {\n            std::vector<std::pair<std::string, std::string>> headers;\n            for (auto &header : app[\"static_file_headers\"])\n            {\n                headers.emplace_back(\n                    std::make_pair(header[\"name\"].asString(),\n                                   header[\"value\"].asString()));\n            }\n            drogon::app().setStaticFileHeaders(headers);\n        }\n        else\n        {\n            throw std::runtime_error(\n                \"The static_file_headers option must be an array\");\n        }\n    }\n    // upload path\n    auto uploadPath = app.get(\"upload_path\", \"uploads\").asString();\n    drogon::app().setUploadPath(uploadPath);\n    // file types\n    auto fileTypes = app[\"file_types\"];\n    if (fileTypes.isArray() && !fileTypes.empty())\n    {\n        std::vector<std::string> types;\n        for (auto const &fileType : fileTypes)\n        {\n            types.push_back(fileType.asString());\n            LOG_TRACE << \"file type:\" << types.back();\n        }\n        drogon::app().setFileTypes(types);\n    }\n    // locations\n    if (app.isMember(\"locations\"))\n    {\n        auto &locations = app[\"locations\"];\n        if (!locations.isArray())\n        {\n            throw std::runtime_error(\"The locations option must be an array\");\n        }\n        for (auto &location : locations)\n        {\n            auto uri = location.get(\"uri_prefix\", \"\").asString();\n            if (uri.empty())\n                continue;\n            auto defaultContentType =\n                location.get(\"default_content_type\", \"\").asString();\n            auto alias = location.get(\"alias\", \"\").asString();\n            auto isCaseSensitive =\n                location.get(\"is_case_sensitive\", false).asBool();\n            auto allAll = location.get(\"allow_all\", true).asBool();\n            auto isRecursive = location.get(\"is_recursive\", true).asBool();\n            if (!location[\"filters\"].isNull())\n            {\n                if (location[\"filters\"].isArray())\n                {\n                    std::vector<std::string> filters;\n                    for (auto const &filter : location[\"filters\"])\n                    {\n                        filters.push_back(filter.asString());\n                    }\n                    drogon::app().addALocation(uri,\n                                               defaultContentType,\n                                               alias,\n                                               isCaseSensitive,\n                                               allAll,\n                                               isRecursive,\n                                               filters);\n                }\n                else\n                {\n                    throw std::runtime_error(\"the filters of location '\" + uri +\n                                             \"' should be an array\");\n                }\n            }\n            else\n            {\n                drogon::app().addALocation(uri,\n                                           defaultContentType,\n                                           alias,\n                                           isCaseSensitive,\n                                           allAll,\n                                           isRecursive);\n            }\n        }\n    }\n    // max connections\n    auto maxConns = app.get(\"max_connections\", 0).asUInt64();\n    if (maxConns > 0)\n    {\n        drogon::app().setMaxConnectionNum(maxConns);\n    }\n    // max connections per IP\n    auto maxConnsPerIP = app.get(\"max_connections_per_ip\", 0).asUInt64();\n    if (maxConnsPerIP > 0)\n    {\n        drogon::app().setMaxConnectionNumPerIP(maxConnsPerIP);\n    }\n#ifndef _WIN32\n    // dynamic views\n    auto enableDynamicViews = app.get(\"load_dynamic_views\", false).asBool();\n    if (enableDynamicViews)\n    {\n        auto viewsPaths = app[\"dynamic_views_path\"];\n        if (viewsPaths.isArray() && viewsPaths.size() > 0)\n        {\n            std::vector<std::string> paths;\n            for (auto const &viewsPath : viewsPaths)\n            {\n                paths.push_back(viewsPath.asString());\n                LOG_TRACE << \"views path:\" << paths.back();\n            }\n            auto outputPath =\n                app.get(\"dynamic_views_output_path\", \"\").asString();\n            drogon::app().enableDynamicViewsLoading(paths, outputPath);\n        }\n    }\n#endif\n    auto stackLimit = app.get(\"json_parser_stack_limit\", 1000).asUInt64();\n    drogon::app().setJsonParserStackLimit(stackLimit);\n    auto unicodeEscaping =\n        app.get(\"enable_unicode_escaping_in_json\", true).asBool();\n    drogon::app().setUnicodeEscapingInJson(unicodeEscaping);\n    auto &precision = app[\"float_precision_in_json\"];\n    if (!precision.isNull())\n    {\n        auto precisionLength = precision.get(\"precision\", 0).asUInt64();\n        auto precisionType =\n            precision.get(\"precision_type\", \"significant\").asString();\n        drogon::app().setFloatPrecisionInJson((unsigned int)precisionLength,\n                                              precisionType);\n    }\n    // log\n    loadLogSetting(app[\"log\"]);\n    // run as daemon\n    auto runAsDaemon = app.get(\"run_as_daemon\", false).asBool();\n    if (runAsDaemon)\n    {\n        drogon::app().enableRunAsDaemon();\n    }\n    // handle SIGTERM\n    auto handleSigterm = app.get(\"handle_sig_term\", true).asBool();\n    if (!handleSigterm)\n    {\n        drogon::app().disableSigtermHandling();\n    }\n    // relaunch\n    auto relaunch = app.get(\"relaunch_on_error\", false).asBool();\n    if (relaunch)\n    {\n        drogon::app().enableRelaunchOnError();\n    }\n    auto useSendfile = app.get(\"use_sendfile\", true).asBool();\n    drogon::app().enableSendfile(useSendfile);\n    auto useGzip = app.get(\"use_gzip\", true).asBool();\n    drogon::app().enableGzip(useGzip);\n    auto useBr = app.get(\"use_brotli\", false).asBool();\n    drogon::app().enableBrotli(useBr);\n    auto staticFilesCacheTime = app.get(\"static_files_cache_time\", 5).asInt();\n    drogon::app().setStaticFilesCacheTime(staticFilesCacheTime);\n    loadControllers(app[\"simple_controllers_map\"]);\n    // Kick off idle connections\n    auto kickOffTimeout = app.get(\"idle_connection_timeout\", 60).asUInt64();\n    drogon::app().setIdleConnectionTimeout(kickOffTimeout);\n    auto server = app.get(\"server_header_field\", \"\").asString();\n    if (!server.empty())\n        drogon::app().setServerHeaderField(server);\n    auto sendServerHeader = app.get(\"enable_server_header\", true).asBool();\n    drogon::app().enableServerHeader(sendServerHeader);\n    auto sendDateHeader = app.get(\"enable_date_header\", true).asBool();\n    drogon::app().enableDateHeader(sendDateHeader);\n    auto keepaliveReqs = app.get(\"keepalive_requests\", 0).asUInt64();\n    drogon::app().setKeepaliveRequestsNumber(keepaliveReqs);\n    auto pipeliningReqs = app.get(\"pipelining_requests\", 0).asUInt64();\n    drogon::app().setPipeliningRequestsNumber(pipeliningReqs);\n    auto useGzipStatic = app.get(\"gzip_static\", true).asBool();\n    drogon::app().setGzipStatic(useGzipStatic);\n    auto useBrStatic = app.get(\"br_static\", true).asBool();\n    drogon::app().setBrStatic(useBrStatic);\n    auto maxBodySize = app.get(\"client_max_body_size\", \"1M\").asString();\n    size_t size;\n    if (bytesSize(maxBodySize, size))\n    {\n        drogon::app().setClientMaxBodySize(size);\n    }\n    else\n    {\n        throw std::runtime_error(\"Error format of client_max_body_size\");\n    }\n    auto maxMemoryBodySize =\n        app.get(\"client_max_memory_body_size\", \"64K\").asString();\n    if (bytesSize(maxMemoryBodySize, size))\n    {\n        drogon::app().setClientMaxMemoryBodySize(size);\n    }\n    else\n    {\n        throw std::runtime_error(\"Error format of client_max_memory_body_size\");\n    }\n    auto maxWsMsgSize =\n        app.get(\"client_max_websocket_message_size\", \"128K\").asString();\n    if (bytesSize(maxWsMsgSize, size))\n    {\n        drogon::app().setClientMaxWebSocketMessageSize(size);\n    }\n    else\n    {\n        throw std::runtime_error(\n            \"Error format of client_max_websocket_message_size\");\n    }\n    drogon::app().enableReusePort(app.get(\"reuse_port\", false).asBool());\n    drogon::app().setHomePage(app.get(\"home_page\", \"index.html\").asString());\n    drogon::app().setImplicitPageEnable(\n        app.get(\"use_implicit_page\", true).asBool());\n    drogon::app().setImplicitPage(\n        app.get(\"implicit_page\", \"index.html\").asString());\n    auto mimes = app[\"mime\"];\n    if (!mimes.isNull())\n    {\n        auto names = mimes.getMemberNames();\n        for (const auto &mime : names)\n        {\n            auto ext = mimes[mime];\n            std::vector<std::string> exts;\n            if (ext.isString())\n                exts.push_back(ext.asString());\n            else if (ext.isArray())\n            {\n                for (const auto &extension : ext)\n                    exts.push_back(extension.asString());\n            }\n\n            for (const auto &extension : exts)\n                drogon::app().registerCustomExtensionMime(extension, mime);\n        }\n    }\n    bool enableCompressedRequests =\n        app.get(\"enabled_compressed_request\", false).asBool();\n    drogon::app().enableCompressedRequest(enableCompressedRequests);\n}\n\nstatic void loadDbClients(const Json::Value &dbClients)\n{\n    if (!dbClients)\n        return;\n    for (auto const &client : dbClients)\n    {\n        auto type = client.get(\"rdbms\", \"postgresql\").asString();\n        std::transform(type.begin(),\n                       type.end(),\n                       type.begin(),\n                       [](unsigned char c) { return tolower(c); });\n        auto host = client.get(\"host\", \"127.0.0.1\").asString();\n        auto port = client.get(\"port\", 5432).asUInt();\n        auto dbname = client.get(\"dbname\", \"\").asString();\n        if (dbname == \"\" && type != \"sqlite3\")\n        {\n            throw std::runtime_error(\n                \"Please configure dbname in the configuration file\");\n        }\n        auto user = client.get(\"user\", \"postgres\").asString();\n        auto password = client.get(\"passwd\", \"\").asString();\n        if (password.empty())\n        {\n            password = client.get(\"password\", \"\").asString();\n        }\n        auto connNum = client.get(\"connection_number\", 1).asUInt();\n        if (connNum == 1)\n        {\n            connNum = client.get(\"number_of_connections\", 1).asUInt();\n        }\n        auto name = client.get(\"name\", \"default\").asString();\n        auto filename = client.get(\"filename\", \"\").asString();\n        auto isFast = client.get(\"is_fast\", false).asBool();\n        auto characterSet = client.get(\"characterSet\", \"\").asString();\n        if (characterSet.empty())\n        {\n            characterSet = client.get(\"client_encoding\", \"\").asString();\n        }\n        auto timeout = client.get(\"timeout\", -1.0).asDouble();\n        auto autoBatch = client.get(\"auto_batch\", false).asBool();\n        drogon::app().createDbClient(type,\n                                     host,\n                                     (unsigned short)port,\n                                     dbname,\n                                     user,\n                                     password,\n                                     connNum,\n                                     filename,\n                                     name,\n                                     isFast,\n                                     characterSet,\n                                     timeout,\n                                     autoBatch);\n    }\n}\n\nstatic void loadRedisClients(const Json::Value &redisClients)\n{\n    if (!redisClients)\n        return;\n    for (auto const &client : redisClients)\n    {\n        std::promise<std::string> promise;\n        auto future = promise.get_future();\n        auto host = client.get(\"host\", \"127.0.0.1\").asString();\n        trantor::Resolver::newResolver()->resolve(\n            host, [&promise](const trantor::InetAddress &address) {\n                promise.set_value(address.toIp());\n            });\n        auto port = client.get(\"port\", 6379).asUInt();\n        auto username = client.get(\"username\", \"\").asString();\n        auto password = client.get(\"passwd\", \"\").asString();\n        if (password.empty())\n        {\n            password = client.get(\"password\", \"\").asString();\n        }\n        auto connNum = client.get(\"connection_number\", 1).asUInt();\n        if (connNum == 1)\n        {\n            connNum = client.get(\"number_of_connections\", 1).asUInt();\n        }\n        auto name = client.get(\"name\", \"default\").asString();\n        auto isFast = client.get(\"is_fast\", false).asBool();\n        auto timeout = client.get(\"timeout\", -1.0).asDouble();\n        auto db = client.get(\"db\", 0).asUInt();\n        auto hostIp = future.get();\n        drogon::app().createRedisClient(hostIp,\n                                        port,\n                                        name,\n                                        password,\n                                        connNum,\n                                        isFast,\n                                        timeout,\n                                        db,\n                                        username);\n    }\n}\n\nstatic void loadListeners(const Json::Value &listeners)\n{\n    if (!listeners)\n        return;\n    LOG_TRACE << \"Has \" << listeners.size() << \" listeners\";\n    for (auto const &listener : listeners)\n    {\n        auto addr = listener.get(\"address\", \"0.0.0.0\").asString();\n        auto port = (uint16_t)listener.get(\"port\", 0).asUInt();\n        auto useSSL = listener.get(\"https\", false).asBool();\n        auto cert = listener.get(\"cert\", \"\").asString();\n        auto key = listener.get(\"key\", \"\").asString();\n        auto useOldTLS = listener.get(\"use_old_tls\", false).asBool();\n        std::vector<std::pair<std::string, std::string>> sslConfCmds;\n        if (listener.isMember(\"ssl_conf\"))\n        {\n            for (const auto &opt : listener[\"ssl_conf\"])\n            {\n...\n// Path: lib/src/IntranetIpFilter.cc\n/**\n *\n *  IntranetIpFilter.cc\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpResponseImpl.h\"\n#include <drogon/IntranetIpFilter.h>\nusing namespace drogon;\n\nvoid IntranetIpFilter::doFilter(const HttpRequestPtr &req,\n                                FilterCallback &&fcb,\n                                FilterChainCallback &&fccb)\n{\n    if (req->peerAddr().isIntranetIp())\n    {\n        fccb();\n        return;\n    }\n    auto res = drogon::HttpResponse::newNotFoundResponse(req);\n    fcb(res);\n}\n\n// Path: lib/src/FixedWindowRateLimiter.cc\n#include \"FixedWindowRateLimiter.h\"\n\nusing namespace drogon;\n\nFixedWindowRateLimiter::FixedWindowRateLimiter(\n    size_t capacity,\n    std::chrono::duration<double> timeUnit)\n    : capacity_(capacity),\n      lastTime_(std::chrono::steady_clock::now()),\n      timeUnit_(timeUnit)\n{\n}\n\n// implementation of the fixed window algorithm\n\nbool FixedWindowRateLimiter::isAllowed()\n{\n    auto now = std::chrono::steady_clock::now();\n    auto duration = std::chrono::duration_cast<std::chrono::duration<double>>(\n        now - lastTime_);\n    if (duration >= timeUnit_)\n    {\n        currentRequests_ = 0;\n        lastTime_ = now;\n    }\n    if (currentRequests_ < capacity_)\n    {\n        currentRequests_++;\n        return true;\n    }\n    return false;\n}\n\n// Path: lib/src/SlidingWindowRateLimiter.cc\n#include \"SlidingWindowRateLimiter.h\"\n#include <assert.h>\n\nusing namespace drogon;\n\nSlidingWindowRateLimiter::SlidingWindowRateLimiter(\n    size_t capacity,\n    std::chrono::duration<double> timeUnit)\n    : capacity_(capacity),\n      unitStartTime_(std::chrono::steady_clock::now()),\n      lastTime_(unitStartTime_),\n      timeUnit_(timeUnit)\n{\n}\n\n// implementation of the sliding window algorithm\nbool SlidingWindowRateLimiter::isAllowed()\n{\n    auto now = std::chrono::steady_clock::now();\n    unitStartTime_ =\n        unitStartTime_ +\n        std::chrono::duration_cast<decltype(unitStartTime_)::duration>(\n            std::chrono::duration<double>(\n                static_cast<double>(\n                    (uint64_t)(std::chrono::duration_cast<\n                                   std::chrono::duration<double>>(\n                                   now - unitStartTime_)\n                                   .count() /\n                               timeUnit_.count())) *\n                timeUnit_.count()));\n\n    if (unitStartTime_ > lastTime_)\n    {\n        auto duration =\n            std::chrono::duration_cast<std::chrono::duration<double>>(\n                unitStartTime_ - lastTime_);\n        auto startTime = lastTime_;\n        if (duration >= timeUnit_)\n        {\n            previousRequests_ = 0;\n        }\n        else\n        {\n            previousRequests_ = currentRequests_;\n        }\n        currentRequests_ = 0;\n    }\n    auto coef = std::chrono::duration_cast<std::chrono::duration<double>>(\n                    now - unitStartTime_) /\n                timeUnit_;\n    assert(coef <= 1.0);\n    auto count = previousRequests_ * (1.0 - coef) + currentRequests_;\n    if (count < capacity_)\n    {\n        currentRequests_++;\n        lastTime_ = now;\n        return true;\n    }\n    return false;\n}\n\n// Path: lib/src/HttpRequestParser.h\n/**\n *\n *  HttpRequestParser.h\n *  An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#pragma once\n\n#include <drogon/HttpTypes.h>\n#include <trantor/net/TcpConnection.h>\n#include <trantor/utils/MsgBuffer.h>\n#include <trantor/utils/NonCopyable.h>\n#include <deque>\n#include <memory>\n#include <mutex>\n#include \"impl_forwards.h\"\n\nnamespace drogon\n{\nclass HttpRequestParser : public trantor::NonCopyable,\n                          public std::enable_shared_from_this<HttpRequestParser>\n{\n  public:\n    enum class HttpRequestParseStatus\n    {\n        kExpectMethod,\n        kExpectRequestLine,\n        kExpectHeaders,\n        kExpectBody,\n        kExpectChunkLen,\n        kExpectChunkBody,\n        kExpectLastEmptyChunk,\n        kGotAll,\n    };\n\n    explicit HttpRequestParser(const trantor::TcpConnectionPtr &connPtr);\n\n    // return false if any error\n    int parseRequest(trantor::MsgBuffer *buf);\n\n    bool gotAll() const\n    {\n        return status_ == HttpRequestParseStatus::kGotAll;\n    }\n\n    void reset();\n\n    const HttpRequestImplPtr &requestImpl() const\n    {\n        return request_;\n    }\n\n    bool firstReq()\n    {\n        if (firstRequest_)\n        {\n            firstRequest_ = false;\n            return true;\n        }\n        return false;\n    }\n\n    const WebSocketConnectionImplPtr &webSocketConn() const\n    {\n        return websockConnPtr_;\n    }\n\n    void setWebsockConnection(const WebSocketConnectionImplPtr &conn)\n    {\n        websockConnPtr_ = conn;\n    }\n\n    // to support request pipelining(rfc2616-8.1.2.2)\n    void pushRequestToPipelining(const HttpRequestPtr &, bool isHeadMethod);\n    bool pushResponseToPipelining(const HttpRequestPtr &, HttpResponsePtr);\n    void popReadyResponses(std::vector<std::pair<HttpResponsePtr, bool>> &);\n\n    size_t numberOfRequestsInPipelining() const\n    {\n        return requestPipelining_.size();\n    }\n\n    bool emptyPipelining()\n    {\n        return requestPipelining_.empty();\n    }\n\n    bool isStop() const\n    {\n        return stopWorking_;\n    }\n\n    void stop()\n    {\n        stopWorking_ = true;\n    }\n\n    size_t numberOfRequestsParsed() const\n    {\n        return requestsCounter_;\n    }\n\n    trantor::MsgBuffer &getBuffer()\n    {\n        return sendBuffer_;\n    }\n\n    std::vector<std::pair<HttpResponsePtr, bool>> &getResponseBuffer()\n    {\n        assert(loop_->isInLoopThread());\n        if (!responseBuffer_)\n        {\n            responseBuffer_ =\n                std::unique_ptr<std::vector<std::pair<HttpResponsePtr, bool>>>(\n                    new std::vector<std::pair<HttpResponsePtr, bool>>);\n        }\n        return *responseBuffer_;\n    }\n\n    std::vector<HttpRequestImplPtr> &getRequestBuffer()\n    {\n        assert(loop_->isInLoopThread());\n        if (!requestBuffer_)\n        {\n            requestBuffer_ = std::unique_ptr<std::vector<HttpRequestImplPtr>>(\n                new std::vector<HttpRequestImplPtr>);\n        }\n        return *requestBuffer_;\n    }\n\n  private:\n    HttpRequestImplPtr makeRequestForPool(HttpRequestImpl *p);\n    void shutdownConnection(HttpStatusCode code);\n    bool processRequestLine(const char *begin, const char *end);\n    HttpRequestParseStatus status_;\n    trantor::EventLoop *loop_;\n    HttpRequestImplPtr request_;\n    bool firstRequest_{true};\n    WebSocketConnectionImplPtr websockConnPtr_;\n    std::deque<std::pair<HttpRequestPtr, std::pair<HttpResponsePtr, bool>>>\n        requestPipelining_;\n    size_t requestsCounter_{0};\n    std::weak_ptr<trantor::TcpConnection> conn_;\n    bool stopWorking_{false};\n    trantor::MsgBuffer sendBuffer_;\n    std::unique_ptr<std::vector<std::pair<HttpResponsePtr, bool>>>\n        responseBuffer_;\n    std::unique_ptr<std::vector<HttpRequestImplPtr>> requestBuffer_;\n    std::vector<HttpRequestImplPtr> requestsPool_;\n    size_t currentChunkLength_{0};\n    size_t currentContentLength_{0};\n};\n\n}  // namespace drogon\n\n// Path: lib/src/HttpServer.cc\n/**\n *\n *  @file HttpServer.cc\n *  @author An Tao\n *\n *  Copyright 2018, An Tao.  All rights reserved.\n *  https://github.com/an-tao/drogon\n *  Use of this source code is governed by a MIT license\n *  that can be found in the License file.\n *\n *  Drogon\n *\n */\n\n#include \"HttpServer.h\"\n#include <drogon/HttpResponse.h>\n#include <drogon/utils/Utilities.h>\n#include <trantor/utils/Logger.h>\n#include <functional>\n#include <memory>\n#include <utility>\n#include \"AOPAdvice.h\"\n#include \"FiltersFunction.h\"\n#include \"HttpAppFrameworkImpl.h\"\n#include \"HttpConnectionLimit.h\"\n#include \"HttpControllerBinder.h\"\n#include \"HttpRequestImpl.h\"\n#include \"HttpRequestParser.h\"\n#include \"HttpResponseImpl.h\"\n#include \"HttpControllersRouter.h\"\n#include \"StaticFileRouter.h\"\n#include \"WebSocketConnectionImpl.h\"\n\n#if COZ_PROFILING\n#include <coz.h>\n#else\n#define COZ_PROGRESS\n#define COZ_PROGRESS_NAMED(name)\n#define COZ_BEGIN(name)\n#define COZ_END(name)\n#endif\n\nusing namespace std::placeholders;\nusing namespace drogon;\nusing namespace trantor;\n\nstatic inline bool isWebSocket(const HttpRequestImplPtr &req);\nstatic inline HttpResponsePtr tryDecompressRequest(\n    const HttpRequestImplPtr &req);\nstatic inline bool passSyncAdvices(\n    const HttpRequestImplPtr &req,\n    const std::shared_ptr<HttpRequestParser> &requestParser,\n    bool shouldBePipelined,\n    bool isHeadMethod);\nstatic inline HttpResponsePtr getCompressedResponse(\n    const HttpRequestImplPtr &req,\n    const HttpResponsePtr &response,\n    bool isHeadMethod);\n\nstatic void handleInvalidHttpMethod(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback);\n\nstatic void handleHttpOptions(\n    const HttpRequestImplPtr &req,\n    const std::string &allowMethods,\n    std::function<void(const HttpResponsePtr &)> &&callback);\n\nHttpServer::HttpServer(EventLoop *loop,\n                       const InetAddress &listenAddr,\n                       std::string name)\n#ifdef __linux__\n    : server_(loop, listenAddr, std::move(name))\n#else\n    : server_(loop, listenAddr, std::move(name), true, app().reusePort())\n#endif\n{\n    server_.setConnectionCallback(onConnection);\n    server_.setRecvMessageCallback(onMessage);\n    server_.kickoffIdleConnections(\n        HttpAppFrameworkImpl::instance().getIdleConnectionTimeout());\n}\n\nHttpServer::~HttpServer() = default;\n\nvoid HttpServer::start()\n{\n    LOG_TRACE << \"HttpServer[\" << server_.name() << \"] starts listening on \"\n              << server_.ipPort();\n    server_.start();\n}\n\nvoid HttpServer::stop()\n{\n    server_.stop();\n}\n\nvoid HttpServer::onConnection(const TcpConnectionPtr &conn)\n{\n    if (conn->connected())\n    {\n        auto parser = std::make_shared<HttpRequestParser>(conn);\n        parser->reset();\n        conn->setContext(parser);\n        if (!HttpConnectionLimit::instance().tryAddConnection(conn))\n        {\n            LOG_ERROR << \"too much connections!force close!\";\n            conn->forceClose();\n            return;\n        }\n        if (!AopAdvice::instance().passNewConnectionAdvices(conn))\n        {\n            conn->forceClose();\n        }\n    }\n    else if (conn->disconnected())\n    {\n        LOG_TRACE << \"conn disconnected!\";\n        HttpConnectionLimit::instance().releaseConnection(conn);\n        auto requestParser = conn->getContext<HttpRequestParser>();\n        if (requestParser)\n        {\n            if (requestParser->webSocketConn())\n            {\n                requestParser->webSocketConn()->onClose();\n            }\n            conn->clearContext();\n        }\n    }\n}\n\nvoid HttpServer::onMessage(const TcpConnectionPtr &conn, MsgBuffer *buf)\n{\n    if (!conn->hasContext())\n        return;\n    auto requestParser = conn->getContext<HttpRequestParser>();\n    if (!requestParser)\n        return;\n    if (requestParser->webSocketConn())\n    {\n        // Websocket payload\n        requestParser->webSocketConn()->onNewMessage(conn, buf);\n        return;\n    }\n\n    auto &requests = requestParser->getRequestBuffer();\n    // With the pipelining feature or web socket, it is possible to receive\n    // multiple messages at once, so the while loop is necessary\n    while (buf->readableBytes() > 0)\n    {\n        if (requestParser->isStop())\n        {\n            // The number of requests has reached the limit.\n            buf->retrieveAll();\n            return;\n        }\n        int parseRes = requestParser->parseRequest(buf);\n        if (parseRes < 0)\n        {\n            requestParser->reset();\n            conn->forceClose();\n            return;\n        }\n        if (parseRes == 0)\n        {\n            break;\n        }\n        auto &req = requestParser->requestImpl();\n        req->setPeerAddr(conn->peerAddr());\n        req->setLocalAddr(conn->localAddr());\n        req->setCreationDate(trantor::Date::date());\n        req->setSecure(conn->isSSLConnection());\n        req->setPeerCertificate(conn->peerCertificate());\n        requests.push_back(req);\n        requestParser->reset();\n    }\n    onRequests(conn, requests, requestParser);\n    requests.clear();\n}\n\nstruct CallbackParamPack\n{\n    CallbackParamPack(trantor::TcpConnectionPtr conn,\n                      HttpRequestImplPtr req,\n                      std::shared_ptr<bool> loopFlag,\n                      std::shared_ptr<HttpRequestParser> requestParser,\n                      bool isHeadMethod)\n        : conn_(std::move(conn)),\n          req_(std::move(req)),\n          loopFlag_(std::move(loopFlag)),\n          requestParser_(std::move(requestParser)),\n          isHeadMethod_(isHeadMethod)\n    {\n    }\n\n    trantor::TcpConnectionPtr conn_;\n    HttpRequestImplPtr req_;\n    std::shared_ptr<bool> loopFlag_;\n    std::shared_ptr<HttpRequestParser> requestParser_;\n    bool isHeadMethod_;\n    std::atomic<bool> responseSent_{false};\n};\n\nvoid HttpServer::onRequests(\n    const TcpConnectionPtr &conn,\n    const std::vector<HttpRequestImplPtr> &requests,\n    const std::shared_ptr<HttpRequestParser> &requestParser)\n{\n    if (requests.empty())\n        return;\n\n    // will only be checked for the first request\n    if (requestParser->firstReq() && requests.size() == 1 &&\n        isWebSocket(requests[0]))\n    {\n        auto &req = requests[0];\n        if (passSyncAdvices(req,\n                            requestParser,\n                            false /* Not pipelined */,\n                            false /* Not HEAD */))\n        {\n            auto wsConn = std::make_shared<WebSocketConnectionImpl>(conn);\n            wsConn->setPingMessage(\"\", std::chrono::seconds{30});\n            onWebsocketRequest(\n                req,\n                [conn, wsConn, requestParser, req](\n                    const HttpResponsePtr &resp0) mutable {\n                    if (conn->connected())\n                    {\n                        auto resp = HttpAppFrameworkImpl::instance()\n                                        .handleSessionForResponse(req, resp0);\n                        AopAdvice::instance().passPreSendingAdvices(req, resp);\n                        if (resp->statusCode() == k101SwitchingProtocols)\n                        {\n                            requestParser->setWebsockConnection(wsConn);\n                        }\n                        auto httpString =\n                            ((HttpResponseImpl *)resp.get())->renderToBuffer();\n                        conn->send(httpString);\n                        COZ_PROGRESS\n                    }\n                },\n                std::move(wsConn));\n            return;\n        }\n\n        // flush response for not passing sync advices\n        if (conn->connected() && !requestParser->getResponseBuffer().empty())\n        {\n            sendResponses(conn,\n                          requestParser->getResponseBuffer(),\n                          requestParser->getBuffer());\n            requestParser->getResponseBuffer().clear();\n        }\n        return;\n    }\n\n    if (HttpAppFrameworkImpl::instance().keepaliveRequestsNumber() > 0 &&\n        requestParser->numberOfRequestsParsed() >=\n            HttpAppFrameworkImpl::instance().keepaliveRequestsNumber())\n    {\n        requestParser->stop();\n        conn->shutdown();\n        return;\n    }\n    if (HttpAppFrameworkImpl::instance().pipeliningRequestsNumber() > 0 &&\n        requestParser->numberOfRequestsInPipelining() + requests.size() >=\n            HttpAppFrameworkImpl::instance().pipeliningRequestsNumber())\n    {\n        requestParser->stop();\n        conn->shutdown();\n        return;\n    }\n    if (!conn->connected())\n    {\n        return;\n    }\n    auto loopFlagPtr = std::make_shared<bool>(true);\n\n    for (auto &req : requests)\n    {\n        bool isHeadMethod = (req->method() == Head);\n        if (isHeadMethod)\n        {\n            req->setMethod(Get);\n        }\n        bool reqPipelined = false;\n        if (!requestParser->emptyPipelining())\n        {\n            requestParser->pushRequestToPipelining(req, isHeadMethod);\n            reqPipelined = true;\n        }\n        if (!passSyncAdvices(req, requestParser, reqPipelined, isHeadMethod))\n        {\n            continue;\n        }\n\n        // Optimization: Avoids dynamic allocation when copying the callback in\n        // handlers (ex: copying callback into lambda captures in DB calls)\n        bool respReady{false};\n        auto paramPack = std::make_shared<CallbackParamPack>(\n            conn, req, loopFlagPtr, requestParser, isHeadMethod);\n\n        auto errResp = tryDecompressRequest(req);\n        if (errResp)\n        {\n            handleResponse(errResp, paramPack, &respReady);\n        }\n        else\n        {\n            // `handleResponse()` callback may be called synchronously. In this\n            // case, the generated response should not be sent right away, but\n            // be queued in buffer instead. Those ready responses will be sent\n            // together after the end of the for loop.\n            //\n            // By doing this, we could reduce some system calls when sending\n            // through socket. In order to achieve this, we create a\n            // `respReady` variable.\n            onHttpRequest(req,\n                          [respReadyPtr = &respReady,\n                           paramPack = std::move(paramPack)](\n                              const HttpResponsePtr &response) {\n                              handleResponse(response, paramPack, respReadyPtr);\n                          });\n        }\n        if (!reqPipelined && !respReady)\n        {\n            requestParser->pushRequestToPipelining(req, isHeadMethod);\n        }\n    }\n    *loopFlagPtr = false;\n    if (conn->connected() && !requestParser->getResponseBuffer().empty())\n    {\n        sendResponses(conn,\n                      requestParser->getResponseBuffer(),\n                      requestParser->getBuffer());\n        requestParser->getResponseBuffer().clear();\n    }\n}\n\nvoid HttpServer::onHttpRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    LOG_TRACE << \"new request:\" << req->peerAddr().toIpPort() << \"->\"\n              << req->localAddr().toIpPort();\n    LOG_TRACE << \"Headers \" << req->methodString() << \" \" << req->path();\n    LOG_TRACE << \"http path=\" << req->path();\n    if (req->method() == Options && (req->path() == \"*\" || req->path() == \"/*\"))\n    {\n        auto resp = HttpResponse::newHttpResponse();\n        resp->setContentTypeCode(ContentType::CT_TEXT_PLAIN);\n        resp->addHeader(\"Allow\", \"GET,HEAD,POST,PUT,DELETE,OPTIONS,PATCH\");\n        resp->setExpiredTime(0);\n        callback(resp);\n        return;\n    }\n\n    // TODO: move session related codes to its own singleton class\n    HttpAppFrameworkImpl::instance().findSessionForRequest(req);\n    // pre-routing aop\n    auto &aop = AopAdvice::instance();\n    aop.passPreRoutingObservers(req);\n    if (!aop.hasPreRoutingAdvices())\n    {\n        httpRequestRouting(req, std::move(callback));\n        return;\n    }\n    aop.passPreRoutingAdvices(req,\n                              [req, callback = std::move(callback)](\n                                  const HttpResponsePtr &resp) mutable {\n                                  if (resp)\n                                  {\n                                      callback(resp);\n                                  }\n                                  else\n                                  {\n                                      httpRequestRouting(req,\n                                                         std::move(callback));\n                                  }\n                              });\n}\n\nvoid HttpServer::httpRequestRouting(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    // How to access router here?? Make router class singleton?\n    RouteResult result = HttpControllersRouter::instance().route(req);\n    if (result.result == RouteResult::Success)\n    {\n        HttpRequestParamPack pack{std::move(result.binderPtr),\n                                  std::move(callback)};\n        requestPostRouting(req, std::move(pack));\n        return;\n    }\n    if (result.result == RouteResult::MethodNotAllowed)\n    {\n        handleInvalidHttpMethod(req, std::move(callback));\n        return;\n    }\n\n    // Fallback to static file router\n    // TODO: make this router a plugin\n    if (req->path() == \"/\" &&\n        !HttpAppFrameworkImpl::instance().getHomePage().empty())\n    {\n        req->setPath(\"/\" + HttpAppFrameworkImpl::instance().getHomePage());\n    }\n    StaticFileRouter::instance().route(req, std::move(callback));\n}\n\ntemplate <typename Pack>\nvoid HttpServer::requestPostRouting(const HttpRequestImplPtr &req, Pack &&pack)\n{\n    // post-routing aop\n    auto &aop = AopAdvice::instance();\n    aop.passPostRoutingObservers(req);\n    if (!aop.hasPostRoutingAdvices())\n    {\n        requestPassFilters(req, std::forward<Pack>(pack));\n        return;\n    }\n    aop.passPostRoutingAdvices(req,\n                               [req, pack = std::forward<Pack>(pack)](\n                                   const HttpResponsePtr &resp) mutable {\n                                   if (resp)\n                                   {\n                                       pack.callback(resp);\n                                   }\n                                   else\n                                   {\n                                       requestPassFilters(req, std::move(pack));\n                                   }\n                               });\n}\n\ntemplate <typename Pack>\nvoid HttpServer::requestPassFilters(const HttpRequestImplPtr &req, Pack &&pack)\n{\n    // pass filters\n    auto &filters = pack.binderPtr->filters_;\n    if (filters.empty())\n    {\n        requestPreHandling(req, std::forward<Pack>(pack));\n        return;\n    }\n    filters_function::doFilters(filters,\n                                req,\n                                [req, pack = std::forward<Pack>(pack)](\n                                    const HttpResponsePtr &resp) mutable {\n                                    if (resp)\n                                    {\n                                        pack.callback(resp);\n                                    }\n                                    else\n                                    {\n                                        requestPreHandling(req,\n                                                           std::move(pack));\n                                    }\n                                });\n}\n\ntemplate <typename Pack>\nvoid HttpServer::requestPreHandling(const HttpRequestImplPtr &req, Pack &&pack)\n{\n    if (req->method() == Options)\n    {\n        handleHttpOptions(req,\n                          *pack.binderPtr->corsMethods_,\n                          std::move(pack.callback));\n        return;\n    }\n\n    // pre-handling aop\n    auto &aop = AopAdvice::instance();\n    aop.passPreHandlingObservers(req);\n    if (!aop.hasPreHandlingAdvices())\n    {\n        if constexpr (std::is_same_v<std::decay_t<Pack>, HttpRequestParamPack>)\n        {\n            httpRequestHandling(req,\n                                std::move(pack.binderPtr),\n                                std::move(pack.callback));\n        }\n        else\n        {\n            websocketRequestHandling(req,\n                                     std::move(pack.binderPtr),\n                                     std::move(pack.callback),\n                                     std::move(pack.wsConnPtr));\n        }\n        return;\n    }\n    aop.passPreHandlingAdvices(\n        req,\n        [req,\n         pack = std::forward<Pack>(pack)](const HttpResponsePtr &resp) mutable {\n            if (resp)\n            {\n                pack.callback(resp);\n                return;\n            }\n            if constexpr (std::is_same_v<std::decay_t<Pack>,\n                                         HttpRequestParamPack>)\n            {\n                httpRequestHandling(req,\n                                    std::move(pack.binderPtr),\n                                    std::move(pack.callback));\n            }\n            else\n            {\n                websocketRequestHandling(req,\n                                         std::move(pack.binderPtr),\n                                         std::move(pack.callback),\n                                         std::move(pack.wsConnPtr));\n            }\n        });\n}\n\nvoid HttpServer::httpRequestHandling(\n    const HttpRequestImplPtr &req,\n    std::shared_ptr<ControllerBinderBase> &&binderPtr,\n    std::function<void(const HttpResponsePtr &)> &&callback)\n{\n    // Check cached response\n    auto &cachedResp = *(binderPtr->responseCache_);\n    if (cachedResp)\n    {\n        if (cachedResp->expiredTime() == 0 ||\n            (trantor::Date::now() <\n             cachedResp->creationDate().after(\n                 static_cast<double>(cachedResp->expiredTime()))))\n        {\n            // use cached response!\n            LOG_TRACE << \"Use cached response\";\n\n            // post-handling aop\n            AopAdvice::instance().passPostHandlingAdvices(req, cachedResp);\n            callback(cachedResp);\n            return;\n        }\n        else\n        {\n            cachedResp.reset();\n        }\n    }\n\n    auto &binderRef = *binderPtr;\n    binderRef.handleRequest(\n        req,\n        // This is the actual callback being passed to controller\n        [req, binderPtr = std::move(binderPtr), callback = std::move(callback)](\n            const HttpResponsePtr &resp) mutable {\n            // Check if we need to cache the response\n            if (resp->expiredTime() >= 0 && resp->statusCode() != k404NotFound)\n            {\n                static_cast<HttpResponseImpl *>(resp.get())->makeHeaderString();\n                auto loop = req->getLoop();\n                if (loop->isInLoopThread())\n                {\n                    binderPtr->responseCache_.setThreadData(resp);\n                }\n                else\n                {\n                    loop->queueInLoop(\n                        [binderPtr = std::move(binderPtr), resp]() {\n                            binderPtr->responseCache_.setThreadData(resp);\n                        });\n                }\n            }\n            // post-handling aop\n            AopAdvice::instance().passPostHandlingAdvices(req, resp);\n            callback(resp);\n        });\n}\n\nvoid HttpServer::onWebsocketRequest(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback,\n    WebSocketConnectionImplPtr &&wsConnPtr)\n{\n    HttpAppFrameworkImpl::instance().findSessionForRequest(req);\n    // pre-routing aop\n    auto &aop = AopAdvice::instance();\n    aop.passPreRoutingObservers(req);\n    if (!aop.hasPreRoutingAdvices())\n    {\n        websocketRequestRouting(req, std::move(callback), std::move(wsConnPtr));\n        return;\n    }\n    aop.passPreRoutingAdvices(\n        req,\n        [req, wsConnPtr = std::move(wsConnPtr), callback = std::move(callback)](\n            const HttpResponsePtr &resp) mutable {\n            if (resp)\n            {\n                callback(resp);\n            }\n            else\n            {\n                websocketRequestRouting(req,\n                                        std::move(callback),\n                                        std::move(wsConnPtr));\n            }\n        });\n}\n\nvoid HttpServer::websocketRequestRouting(\n    const HttpRequestImplPtr &req,\n    std::function<void(const HttpResponsePtr &)> &&callback,\n    WebSocketConnectionImplPtr &&wsConnPtr)\n{\n    RouteResult result = HttpControllersRouter::instance().routeWs(req);\n\n    if (result.result == RouteResult::Success)\n    {\n        WsRequestParamPack pack{std::move(result.binderPtr),\n                                std::move(callback),\n                                std::move(wsConnPtr)};\n        requestPostRouting(req, std::move(pack));\n        return;\n    }\n    if (result.result == RouteResult::MethodNotAllowed)\n    {\n        handleInvalidHttpMethod(req, std::move(callback));\n        return;\n    }\n\n    // Not found\n    auto resp = drogon::HttpResponse::newNotFoundResponse(req);\n    resp->setCloseConnection(true);\n    callback(resp);\n}\n\nvoid HttpServer::websocketRequestHandling(\n    const HttpRequestImplPtr &req,\n    std::shared_ptr<ControllerBinderBase> &&binderPtr,\n    std::function<void(const HttpResponsePtr &)> &&callback,\n    WebSocketConnectionImplPtr &&wsConnPtr)\n{\n    binderPtr->handleRequest(\n        req,\n        [req, callback = std::move(callback)](const HttpResponsePtr &resp) {\n            AopAdvice::instance().passPostHandlingAdvices(req, resp);\n            callback(resp);\n        });\n\n    // TODO: more elegant?\n    static_cast<WebsocketControllerBinder *>(binderPtr.get())\n        ->handleNewConnection(req, wsConnPtr);\n}\n\nvoid HttpServer::handleResponse(\n    const HttpResponsePtr &response,\n    const std::shared_ptr<CallbackParamPack> &paramPack,\n    bool *respReadyPtr)\n{\n    auto &conn = paramPack->conn_;\n    auto &req = paramPack->req_;\n    auto &requestParser = paramPack->requestParser_;\n    auto &loopFlagPtr = paramPack->loopFlag_;\n    const bool isHeadMethod = paramPack->isHeadMethod_;\n\n    if (!response)\n        return;\n    if (!conn->connected())\n        return;\n\n    if (paramPack->responseSent_.exchange(true, std::memory_order_acq_rel))\n    {\n        LOG_ERROR << \"Sending more than 1 response for request. \"\n                     \"Ignoring later response\";\n        return;\n    }\n\n    auto resp =\n        HttpAppFrameworkImpl::instance().handleSessionForResponse(req,\n                                                                  response);\n    resp->setVersion(req->getVersion());\n    resp->setCloseConnection(!req->keepAlive());\n    AopAdvice::instance().passPreSendingAdvices(req, resp);\n\n    auto newResp = getCompressedResponse(req, resp, isHeadMethod);\n    if (conn->getLoop()->isInLoopThread())\n    {\n        /*\n         * A client that supports persistent connections MAY\n         * \"pipeline\" its requests (i.e., send multiple requests\n         * without waiting for each response). A server MUST send\n         * its responses to those requests in the same order that\n         * the requests were received. rfc2616-8.1.1.2\n         */\n        if (requestParser->emptyPipelining())\n        {\n            // response must have arrived synchronously\n            assert(*loopFlagPtr);\n            // TODO: change to weakPtr to be sure. But may drop performance.\n            *respReadyPtr = true;\n            requestParser->getResponseBuffer().emplace_back(std::move(newResp),\n                                                            isHeadMethod);\n            return;\n        }\n        if (requestParser->pushResponseToPipelining(req, std::move(newResp)))\n        {\n            auto &responseBuffer = requestParser->getResponseBuffer();\n            requestParser->popReadyResponses(responseBuffer);\n            if (!*loopFlagPtr)\n            {\n                // We have passed the point where `onRequests()` sends\n                // responses. So, at here we should send ready responses from\n                // the beginning of pipeline queue.\n                sendResponses(conn, responseBuffer, requestParser->getBuffer());\n                responseBuffer.clear();\n            }\n        }\n    }\n    else\n    {\n        conn->getLoop()->queueInLoop(\n            [conn, req, requestParser, newResp = std::move(newResp)]() mutable {\n                if (!conn->connected())\n                {\n                    return;\n                }\n                if (requestParser->pushResponseToPipelining(req,\n                                                            std::move(newResp)))\n                {\n                    std::vector<std::pair<HttpResponsePtr, bool>> responses;\n                    requestParser->popReadyResponses(responses);\n                    sendResponses(conn, responses, requestParser->getBuffer());\n                }\n            });\n    }\n}\n\nstruct ChunkingParams\n{\n    using DataCallback = std::function<std::size_t(char *, std::size_t)>;\n\n    \nexplicit ChunkingParams(DataCallback cb) : dataCallback(std::move(cb))\n    {\n    }\n\n    DataCallback dataCallback;\n    bool bFinished{false};\n#ifndef NDEBUG  // defined by CMake for release build\n    std::size_t nDataReturned{0};\n#endif\n};\n\nstatic std::size_t chunkingCallback(\n    const std::shared_ptr<ChunkingParams> &cbParams,\n    char *pBuffer,\n    std::size_t nSize)\n{\n    if (!cbParams)\n        return 0;\n    // Cleanup\n    if (pBuffer == nullptr)\n    {\n        LOG_TRACE << \"Chunking callback cleanup\";\n        if (cbParams && cbParams->dataCallback)\n        {\n            cbParams->dataCallback(pBuffer, nSize);\n            cbParams->dataCallback = {};\n        }\n        return 0;\n    }\n    // Terminal chunk already returned\n    if (cbParams->bFinished)\n    {\n        LOG_TRACE << \"Chunking callback has no more data\";\n#ifndef NDEBUG  // defined by CMake for release build\n        LOG_TRACE << \"Chunking callback: total data returned: \"\n                  << cbParams->nDataReturned << \" bytes\";\n#endif\n        return 0;\n    }\n\n    // Reserve size to prepend the chunk size & append cr/lf, and get data\n    struct\n    {\n        std::size_t operator()(std::size_t n)\n        {\n            return n == 0 ? 0 : 1 + (*this)(n >> 4);\n        }\n    } neededDigits;\n\n    auto nHeaderSize = neededDigits(nSize) + 2;\n    auto nDataSize =\n        cbParams->dataCallback(pBuffer + nHeaderSize, nSize - nHeaderSize - 2);\n    if (nDataSize == 0)\n    {\n        // Terminal chunk + cr/lf\n        cbParams->bFinished = true;\n#ifdef _WIN32\n        memcpy_s(pBuffer, nSize, \"0\\r\\n\\r\\n\", 5);\n#else\n        memcpy(pBuffer, \"0\\r\\n\\r\\n\", 5);\n#endif\n        LOG_TRACE << \"Chunking callback: no more data, return last chunk of \"\n                     \"size 0 & end of message\";\n        return 5;\n    }\n    // Non-terminal chunks\n    pBuffer[nHeaderSize + nDataSize] = '\\r';\n    pBuffer[nHeaderSize + nDataSize + 1] = '\\n';\n    // The spec does not say if the chunk size is allowed tohave leading zeroes\n    // Use a fixed size header with leading zeroes\n    // (tested to work with Chrome, Firefox, Safari, Edge, wget, curl and VLC)\n#ifdef _WIN32\n    char pszFormat[]{\"%04llx\\r\"};\n#else\n    char pszFormat[]{\"%04lx\\r\"};\n#endif\n    pszFormat[2] = '0' + char(nHeaderSize - 2);\n    snprintf(pBuffer, nHeaderSize, pszFormat, nDataSize);\n    pBuffer[nHeaderSize - 1] = '\\n';\n    LOG_TRACE << \"Chunking callback: return chunk of size \" << nDataSize;\n...\n", "needle_token_start": 15404, "needle_token_end": 15430, "code_context_ntokens": 16235, "output": ["```cpp\nstatic std::size_t chunkingCallback(\n    const std::shared_ptr<ChunkingParams> &cbParams,\n    char *pBuffer,\n    std::size_t nSize)\n{\n    if (!cbParams)\n        return 0;\n    // Cleanup\n    if (pBuffer == nullptr)\n    {\n        LOG_TRACE << \"Chunking callback cleanup\";\n        if (cbParams && cbParams->dataCallback)\n        {\n            cbParams->dataCallback(pBuffer, nSize);\n            cbParams->dataCallback = {};\n        }\n        return 0;\n    }\n    // Terminal chunk already returned\n    if (cbParams->bFinished)\n    {\n        LOG_TRACE << \"Chunking callback has no more data\";\n#ifndef NDEBUG  // defined by CMake for release build\n        LOG_TRACE << \"Chunking callback: total data returned: \"\n                  << cbParams->nDataReturned << \" bytes\";\n#endif\n        return 0;\n    }\n\n    // Reserve size to prepend the chunk size & append cr/lf, and get data\n    struct\n    {\n        std::size_t operator()(std::size_t n)\n        {\n            return n == 0 ? 0 : 1 + (*this)(n >> 4);\n        }\n    } neededDigits;\n\n    auto nHeaderSize = neededDigits(nSize) + 2;\n    auto nDataSize =\n        cbParams->dataCallback(pBuffer + nHeaderSize, nSize - nHeaderSize - 2);\n    if (nDataSize == 0)\n    {\n        // Terminal chunk + cr/lf\n        cbParams->bFinished = true;\n#ifdef _WIN32\n        memcpy_s(pBuffer, nSize, \"0\\r\\n\\r\\n\", 5);\n#else\n        memcpy(pBuffer, \"0\\r\\n\\r\\n\", 5);\n#endif\n        LOG_TRACE << \"Chunking callback: no more data, return last chunk of \"\n                     \"size 0 & end of message\";\n        return 5;\n    }\n    // Non-terminal chunks\n    pBuffer[nHeaderSize + nDataSize] = '\\r';\n    pBuffer[nHeaderSize + nDataSize + 1] = '\\n';\n    // The spec does not say if the chunk size is allowed tohave leading zeroes\n    // Use a fixed size header with leading zeroes\n    // (tested to work with Chrome, Firefox, Safari, Edge, wget, curl and VLC)\n#ifdef _WIN32\n    char pszFormat[]{\"%04llx\\r\"};\n#else\n    char pszFormat[]{\"%04lx\\r\"};\n#endif\n    pszFormat[2] = '0' + char(nHeaderSize - 2);\n    snprintf(pBuffer, nHeaderSize, pszFormat, nDataSize);\n    pBuffer[nHeaderSize - 1] = '\\n';\n    LOG_TRACE << \"Chunking callback: return chunk of size \" << nDataSize;\n    return nHeaderSize + nDataSize + 2;\n}\n```"]}
{"repo": "sass/node-sass", "name": "compatibility", "language": "cpp", "path": "src/libsass/src/plugins.cpp", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To determine if a plugin's version is compatible with the current library version, focusing on major version compatibility.\n2. **Input**: A string representing the version of the plugin.\n3. **Output**: A boolean value; `true` if the versions are compatible up to the major version, `false` otherwise.\n4. **Procedure**: The function first checks if either the input version or the library's version is marked as \"not available\" (`\"[na]\"`). If either is, it returns `false`. If both versions are available, it locates the position of the second period in the library's version to isolate the major version. If the second period is not found, it directly compares the full version strings. If found, it compares the substrings up to the second period to check major version compatibility.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/libsass/src/environment.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"environment.hpp\"\n\nnamespace Sass {\n\n  template <typename T>\n  Environment<T>::Environment(bool is_shadow)\n  : local_frame_(environment_map<std::string, T>()),\n    parent_(0), is_shadow_(false)\n  { }\n  template <typename T>\n  Environment<T>::Environment(Environment<T>* env, bool is_shadow)\n  : local_frame_(environment_map<std::string, T>()),\n    parent_(env), is_shadow_(is_shadow)\n  { }\n  template <typename T>\n  Environment<T>::Environment(Environment<T>& env, bool is_shadow)\n  : local_frame_(environment_map<std::string, T>()),\n    parent_(&env), is_shadow_(is_shadow)\n  { }\n\n  // link parent to create a stack\n  template <typename T>\n  void Environment<T>::link(Environment& env) { parent_ = &env; }\n  template <typename T>\n  void Environment<T>::link(Environment* env) { parent_ = env; }\n\n  // this is used to find the global frame\n  // which is the second last on the stack\n  template <typename T>\n  bool Environment<T>::is_lexical() const\n  {\n    return !! parent_ && parent_->parent_;\n  }\n\n  // only match the real root scope\n  // there is still a parent around\n  // not sure what it is actually use for\n  // I guess we store functions etc. there\n  template <typename T>\n  bool Environment<T>::is_global() const\n  {\n...\n// Path: src/libsass/src/plugins.cpp\n#include \"sass.hpp\"\n#include <iostream>\n#include \"output.hpp\"\n#include \"plugins.hpp\"\n\n#ifdef _WIN32\n#include <windows.h>\n#else\n#include <sys/types.h>\n#include <dirent.h>\n#include <errno.h>\n#include <dlfcn.h>\n#endif\n\nnamespace Sass {\n\n  Plugins::Plugins(void) { }\n  Plugins::~Plugins(void)\n  {\n    for (auto function : functions) {\n      sass_delete_function(function);\n    }\n    for (auto importer : importers) {\n      sass_delete_importer(importer);\n    }\n    for (auto header : headers) {\n      sass_delete_importer(header);\n    }\n  }\n\n  // check if plugin is compatible with this version\n  // plugins may be linked static against libsass\n  // we try to be compatible between major versions\n  \ninline bool compatibility(const char* their_version)\n  {\n// const char* their_version = \"3.1.2\";\n    // first check if anyone has an unknown version\n    const char* our_version = libsass_version();\n    if (!strcmp(their_version, \"[na]\")) return false;\n    if (!strcmp(our_version, \"[na]\")) return false;\n\n    // find the position of the second dot\n    size_t pos = std::string(our_version).find('.', 0);\n    if (pos != std::string::npos) pos = std::string(our_version).find('.', pos + 1);\n\n    // if we do not have two dots we fallback to compare complete string\n    if (pos == std::string::npos) { return strcmp(their_version, our_version) ? 0 : 1; }\n    // otherwise only compare up to the second dot (major versions)\n    else { return strncmp(their_version, our_version, pos) ? 0 : 1; }\n\n  }\n\n  // load one specific plugin\n  bool Plugins::load_plugin (const std::string& path)\n  {\n\n    typedef const char* (*__plugin_version__)(void);\n    typedef Sass_Function_List (*__plugin_load_fns__)(void);\n    typedef Sass_Importer_List (*__plugin_load_imps__)(void);\n\n    if (LOAD_LIB(plugin, path))\n    {\n      // try to load initial function to query libsass version suppor\n      if (LOAD_LIB_FN(__plugin_version__, plugin_version, \"libsass_get_version\"))\n      {\n        // get the libsass version of the plugin\n        if (!compatibility(plugin_version())) return false;\n        // try to get import address for \"libsass_load_functions\"\n        if (LOAD_LIB_FN(__plugin_load_fns__, plugin_load_functions, \"libsass_load_functions\"))\n        {\n          Sass_Function_List fns = plugin_load_functions(), _p = fns;\n          while (fns && *fns) { functions.push_back(*fns); ++ fns; }\n          sass_free_memory(_p); // only delete the container, items not yet\n        }\n        // try to get import address for \"libsass_load_importers\"\n        if (LOAD_LIB_FN(__plugin_load_imps__, plugin_load_importers, \"libsass_load_importers\"))\n        {\n          Sass_Importer_List imps = plugin_load_importers(), _p = imps;\n          while (imps && *imps) { importers.push_back(*imps); ++ imps; }\n          sass_free_memory(_p); // only delete the container, items not yet\n        }\n        // try to get import address for \"libsass_load_headers\"\n        if (LOAD_LIB_FN(__plugin_load_imps__, plugin_load_headers, \"libsass_load_headers\"))\n        {\n          Sass_Importer_List imps = plugin_load_headers(), _p = imps;\n          while (imps && *imps) { headers.push_back(*imps); ++ imps; }\n          sass_free_memory(_p); // only delete the container, items not yet\n        }\n        // success\n        return true;\n      }\n      else\n      {\n        // print debug message to stderr (should not happen)\n        std::cerr << \"failed loading 'libsass_support' in <\" << path << \">\" << std::endl;\n        if (const char* dlsym_error = dlerror()) std::cerr << dlsym_error << std::endl;\n        CLOSE_LIB(plugin);\n      }\n    }\n    else\n    {\n      // print debug message to stderr (should not happen)\n      std::cerr << \"failed loading plugin <\" << path << \">\" << std::endl;\n      if (const char* dlopen_error = dlerror()) std::cerr << dlopen_error << std::endl;\n    }\n\n    return false;\n\n  }\n\n  size_t Plugins::load_plugins(const std::string& path)\n  {\n\n    // count plugins\n    size_t loaded = 0;\n\n    #ifdef _WIN32\n\n      try\n      {\n\n        // use wchar (utf16)\n        WIN32_FIND_DATAW data;\n        // trailing slash is guaranteed\n        std::string globsrch(path + \"*.dll\");\n        // convert to wide chars (utf16) for system call\n        std::wstring wglobsrch(UTF_8::convert_to_utf16(globsrch));\n        HANDLE hFile = FindFirstFileW(wglobsrch.c_str(), &data);\n        // check if system called returned a result\n        // ToDo: maybe we should print a debug message\n        if (hFile == INVALID_HANDLE_VALUE) return -1;\n\n        // read directory\n        while (true)\n        {\n          try\n          {\n            // the system will report the filenames with wide chars (utf16)\n            std::string entry = UTF_8::convert_from_utf16(data.cFileName);\n            // check if file ending matches exactly\n            if (!ends_with(entry, \".dll\")) continue;\n            // load the plugin and increase counter\n            if (load_plugin(path + entry)) ++ loaded;\n            // check if there should be more entries\n            if (GetLastError() == ERROR_NO_MORE_FILES) break;\n            // load next entry (check for return type)\n            if (!FindNextFileW(hFile, &data)) break;\n          }\n          catch (...)\n          {\n            // report the error to the console (should not happen)\n            // seems like we got strange data from the system call?\n            std::cerr << \"filename in plugin path has invalid utf8?\" << std::endl;\n          }\n        }\n      }\n      catch (utf8::invalid_utf8)\n      {\n        // report the error to the console (should not happen)\n        // implementors should make sure to provide valid utf8\n        std::cerr << \"plugin path contains invalid utf8\" << std::endl;\n      }\n\n    #else\n\n      DIR *dp;\n      struct dirent *dirp;\n      if((dp  = opendir(path.c_str())) == NULL) return -1;\n      while ((dirp = readdir(dp)) != NULL) {\n        #if __APPLE__\n          if (!ends_with(dirp->d_name, \".dylib\")) continue;\n        #else\n          if (!ends_with(dirp->d_name, \".so\")) continue;\n        #endif\n        if (load_plugin(path + dirp->d_name)) ++ loaded;\n      }\n      closedir(dp);\n\n    #endif\n    return loaded;\n\n  }\n\n}\n\n// Path: src/libsass/src/values.hpp\n#ifndef SASS_VALUES_H\n#define SASS_VALUES_H\n\n#include \"ast.hpp\"\n\nnamespace Sass {\n\n  union Sass_Value* ast_node_to_sass_value (const Expression_Ptr val);\n  Value_Ptr sass_value_to_ast_node (const union Sass_Value* val);\n\n}\n#endif\n\n// Path: src/libsass/src/listize.hpp\n#ifndef SASS_LISTIZE_H\n#define SASS_LISTIZE_H\n\n#include <vector>\n#include <iostream>\n\n#include \"ast.hpp\"\n#include \"context.hpp\"\n#include \"operation.hpp\"\n#include \"environment.hpp\"\n\nnamespace Sass {\n\n  struct Backtrace;\n\n  class Listize : public Operation_CRTP<Expression_Ptr, Listize> {\n\n    Expression_Ptr fallback_impl(AST_Node_Ptr n);\n\n  public:\n    Listize();\n    ~Listize() { }\n\n    Expression_Ptr operator()(Selector_List_Ptr);\n    Expression_Ptr operator()(Complex_Selector_Ptr);\n    Expression_Ptr operator()(Compound_Selector_Ptr);\n\n    template <typename U>\n    Expression_Ptr fallback(U x) { return fallback_impl(x); }\n  };\n\n}\n\n#endif\n\n// Path: src/libsass/src/functions.hpp\n#ifndef SASS_FUNCTIONS_H\n#define SASS_FUNCTIONS_H\n\n#include \"listize.hpp\"\n#include \"position.hpp\"\n#include \"environment.hpp\"\n#include \"ast_fwd_decl.hpp\"\n#include \"sass/functions.h\"\n\n#define BUILT_IN(name) Expression_Ptr \\\nname(Env& env, Env& d_env, Context& ctx, Signature sig, ParserState pstate, Backtraces traces, std::vector<Selector_List_Obj> selector_stack)\n\nnamespace Sass {\n  struct Backtrace;\n  typedef const char* Signature;\n  typedef Expression_Ptr (*Native_Function)(Env&, Env&, Context&, Signature, ParserState, Backtraces, std::vector<Selector_List_Obj>);\n\n  Definition_Ptr make_native_function(Signature, Native_Function, Context& ctx);\n  Definition_Ptr make_c_function(Sass_Function_Entry c_func, Context& ctx);\n\n  std::string function_name(Signature);\n\n  namespace Functions {\n\n    extern Signature rgb_sig;\n    extern Signature rgba_4_sig;\n    extern Signature rgba_2_sig;\n    extern Signature red_sig;\n    extern Signature green_sig;\n    extern Signature blue_sig;\n    extern Signature mix_sig;\n    extern Signature hsl_sig;\n    extern Signature hsla_sig;\n    extern Signature hue_sig;\n    extern Signature saturation_sig;\n    extern Signature lightness_sig;\n    extern Signature adjust_hue_sig;\n    extern Signature lighten_sig;\n    extern Signature darken_sig;\n    extern Signature saturate_sig;\n    extern Signature desaturate_sig;\n    extern Signature grayscale_sig;\n    extern Signature complement_sig;\n    extern Signature invert_sig;\n    extern Signature alpha_sig;\n    extern Signature opacity_sig;\n    extern Signature opacify_sig;\n    extern Signature fade_in_sig;\n    extern Signature transparentize_sig;\n    extern Signature fade_out_sig;\n    extern Signature adjust_color_sig;\n    extern Signature scale_color_sig;\n    extern Signature change_color_sig;\n    extern Signature ie_hex_str_sig;\n    extern Signature unquote_sig;\n    extern Signature quote_sig;\n    extern Signature str_length_sig;\n    extern Signature str_insert_sig;\n    extern Signature str_index_sig;\n    extern Signature str_slice_sig;\n    extern Signature to_upper_case_sig;\n    extern Signature to_lower_case_sig;\n    extern Signature percentage_sig;\n    extern Signature round_sig;\n    extern Signature ceil_sig;\n    extern Signature floor_sig;\n    extern Signature abs_sig;\n    extern Signature min_sig;\n    extern Signature max_sig;\n    extern Signature inspect_sig;\n    extern Signature random_sig;\n    extern Signature length_sig;\n    extern Signature nth_sig;\n    extern Signature index_sig;\n    extern Signature join_sig;\n    extern Signature append_sig;\n    extern Signature zip_sig;\n    extern Signature list_separator_sig;\n    extern Signature type_of_sig;\n    extern Signature unit_sig;\n    extern Signature unitless_sig;\n    extern Signature comparable_sig;\n    extern Signature variable_exists_sig;\n    extern Signature global_variable_exists_sig;\n    extern Signature function_exists_sig;\n    extern Signature mixin_exists_sig;\n    extern Signature feature_exists_sig;\n    extern Signature call_sig;\n    extern Signature not_sig;\n    extern Signature if_sig;\n    extern Signature map_get_sig;\n    extern Signature map_merge_sig;\n    extern Signature map_remove_sig;\n    extern Signature map_keys_sig;\n    extern Signature map_values_sig;\n    extern Signature map_has_key_sig;\n    extern Signature keywords_sig;\n    extern Signature set_nth_sig;\n    extern Signature unique_id_sig;\n    extern Signature selector_nest_sig;\n    extern Signature selector_append_sig;\n    extern Signature selector_extend_sig;\n    extern Signature selector_replace_sig;\n    extern Signature selector_unify_sig;\n    extern Signature is_superselector_sig;\n    extern Signature simple_selectors_sig;\n    extern Signature selector_parse_sig;\n    extern Signature is_bracketed_sig;\n    extern Signature content_exists_sig;\n    extern Signature get_function_sig;\n\n    BUILT_IN(rgb);\n    BUILT_IN(rgba_4);\n    BUILT_IN(rgba_2);\n    BUILT_IN(red);\n    BUILT_IN(green);\n    BUILT_IN(blue);\n    BUILT_IN(mix);\n    BUILT_IN(hsl);\n    BUILT_IN(hsla);\n    BUILT_IN(hue);\n    BUILT_IN(saturation);\n    BUILT_IN(lightness);\n    BUILT_IN(adjust_hue);\n    BUILT_IN(lighten);\n    BUILT_IN(darken);\n    BUILT_IN(saturate);\n    BUILT_IN(desaturate);\n    BUILT_IN(grayscale);\n    BUILT_IN(complement);\n    BUILT_IN(invert);\n    BUILT_IN(alpha);\n    BUILT_IN(opacify);\n    BUILT_IN(transparentize);\n    BUILT_IN(adjust_color);\n    BUILT_IN(scale_color);\n    BUILT_IN(change_color);\n    BUILT_IN(ie_hex_str);\n    BUILT_IN(sass_unquote);\n    BUILT_IN(sass_quote);\n    BUILT_IN(str_length);\n    BUILT_IN(str_insert);\n    BUILT_IN(str_index);\n    BUILT_IN(str_slice);\n    BUILT_IN(to_upper_case);\n    BUILT_IN(to_lower_case);\n    BUILT_IN(percentage);\n    BUILT_IN(round);\n    BUILT_IN(ceil);\n    BUILT_IN(floor);\n    BUILT_IN(abs);\n    BUILT_IN(min);\n    BUILT_IN(max);\n    BUILT_IN(inspect);\n    BUILT_IN(random);\n    BUILT_IN(length);\n    BUILT_IN(nth);\n    BUILT_IN(index);\n    BUILT_IN(join);\n    BUILT_IN(append);\n    BUILT_IN(zip);\n    BUILT_IN(list_separator);\n    BUILT_IN(type_of);\n    BUILT_IN(unit);\n    BUILT_IN(unitless);\n    BUILT_IN(comparable);\n    BUILT_IN(variable_exists);\n    BUILT_IN(global_variable_exists);\n    BUILT_IN(function_exists);\n    BUILT_IN(mixin_exists);\n    BUILT_IN(feature_exists);\n    BUILT_IN(call);\n    BUILT_IN(sass_not);\n    BUILT_IN(sass_if);\n    BUILT_IN(map_get);\n    BUILT_IN(map_merge);\n    BUILT_IN(map_remove);\n    BUILT_IN(map_keys);\n    BUILT_IN(map_values);\n    BUILT_IN(map_has_key);\n    BUILT_IN(keywords);\n    BUILT_IN(set_nth);\n    BUILT_IN(unique_id);\n    BUILT_IN(selector_nest);\n    BUILT_IN(selector_append);\n    BUILT_IN(selector_extend);\n    BUILT_IN(selector_replace);\n    BUILT_IN(selector_unify);\n    BUILT_IN(is_superselector);\n    BUILT_IN(simple_selectors);\n    BUILT_IN(selector_parse);\n    BUILT_IN(is_bracketed);\n    BUILT_IN(content_exists);\n    BUILT_IN(get_function);\n  }\n}\n\n#endif\n\n// Path: src/libsass/src/sass_functions.hpp\n#ifndef SASS_SASS_FUNCTIONS_H\n#define SASS_SASS_FUNCTIONS_H\n\n#include \"sass.h\"\n#include \"environment.hpp\"\n#include \"functions.hpp\"\n\n// Struct to hold custom function callback\nstruct Sass_Function {\n  char*            signature;\n  Sass_Function_Fn function;\n  void*            cookie;\n};\n\n// External import entry\nstruct Sass_Import {\n  char* imp_path; // path as found in the import statement\n  char *abs_path; // path after importer has resolved it\n  char* source;\n  char* srcmap;\n  // error handling\n  char* error;\n  size_t line;\n  size_t column;\n};\n\n// External environments\nstruct Sass_Env {\n  // links to parent frames\n  Sass::Env* frame;\n};\n\n// External call entry\nstruct Sass_Callee {\n  const char* name;\n  const char* path;\n  size_t line;\n  size_t column;\n  enum Sass_Callee_Type type;\n  struct Sass_Env env;\n};\n\n// Struct to hold importer callback\nstruct Sass_Importer {\n  Sass_Importer_Fn importer;\n  double           priority;\n  void*            cookie;\n};\n\n#endif\n// Path: src/libsass/src/sass_functions.cpp\n#include \"sass.hpp\"\n#include <cstring>\n#include \"util.hpp\"\n#include \"context.hpp\"\n#include \"values.hpp\"\n#include \"sass/functions.h\"\n#include \"sass_functions.hpp\"\n\nextern \"C\" {\n  using namespace Sass;\n\n  Sass_Function_List ADDCALL sass_make_function_list(size_t length)\n  {\n    return (Sass_Function_List) calloc(length + 1, sizeof(Sass_Function_Entry));\n  }\n\n  Sass_Function_Entry ADDCALL sass_make_function(const char* signature, Sass_Function_Fn function, void* cookie)\n  {\n    Sass_Function_Entry cb = (Sass_Function_Entry) calloc(1, sizeof(Sass_Function));\n    if (cb == 0) return 0;\n    cb->signature = sass_copy_c_string(signature);\n    cb->function = function;\n    cb->cookie = cookie;\n    return cb;\n  }\n\n  void ADDCALL sass_delete_function(Sass_Function_Entry entry)\n  {\n    free(entry->signature);\n    free(entry);\n  }\n\n  // Deallocator for the allocated memory\n  void ADDCALL sass_delete_function_list(Sass_Function_List list)\n  {\n    Sass_Function_List it = list;\n    if (list == 0) return;\n    while(*list) {\n      sass_delete_function(*list);\n      ++list;\n    }\n    free(it);\n  }\n\n  // Setters and getters for callbacks on function lists\n  Sass_Function_Entry ADDCALL sass_function_get_list_entry(Sass_Function_List list, size_t pos) { return list[pos]; }\n  void sass_function_set_list_entry(Sass_Function_List list, size_t pos, Sass_Function_Entry cb) { list[pos] = cb; }\n\n  const char* ADDCALL sass_function_get_signature(Sass_Function_Entry cb) { return cb->signature; }\n  Sass_Function_Fn ADDCALL sass_function_get_function(Sass_Function_Entry cb) { return cb->function; }\n  void* ADDCALL sass_function_get_cookie(Sass_Function_Entry cb) { return cb->cookie; }\n\n  Sass_Importer_Entry ADDCALL sass_make_importer(Sass_Importer_Fn importer, double priority, void* cookie)\n  {\n    Sass_Importer_Entry cb = (Sass_Importer_Entry) calloc(1, sizeof(Sass_Importer));\n    if (cb == 0) return 0;\n    cb->importer = importer;\n    cb->priority = priority;\n    cb->cookie = cookie;\n    return cb;\n  }\n\n  Sass_Importer_Fn ADDCALL sass_importer_get_function(Sass_Importer_Entry cb) { return cb->importer; }\n  double ADDCALL sass_importer_get_priority (Sass_Importer_Entry cb) { return cb->priority; }\n  void* ADDCALL sass_importer_get_cookie(Sass_Importer_Entry cb) { return cb->cookie; }\n\n  // Just in case we have some stray import structs\n  void ADDCALL sass_delete_importer (Sass_Importer_Entry cb)\n  {\n    free(cb);\n  }\n\n  // Creator for sass custom importer function list\n  Sass_Importer_List ADDCALL sass_make_importer_list(size_t length)\n  {\n    return (Sass_Importer_List) calloc(length + 1, sizeof(Sass_Importer_Entry));\n  }\n\n  // Deallocator for the allocated memory\n  void ADDCALL sass_delete_importer_list(Sass_Importer_List list)\n  {\n    Sass_Importer_List it = list;\n    if (list == 0) return;\n    while(*list) {\n      sass_delete_importer(*list);\n      ++list;\n    }\n    free(it);\n  }\n\n  Sass_Importer_Entry ADDCALL sass_importer_get_list_entry(Sass_Importer_List list, size_t idx) { return list[idx]; }\n  void ADDCALL sass_importer_set_list_entry(Sass_Importer_List list, size_t idx, Sass_Importer_Entry cb) { list[idx] = cb; }\n\n  // Creator for sass custom importer return argument list\n  Sass_Import_List ADDCALL sass_make_import_list(size_t length)\n  {\n    return (Sass_Import**) calloc(length + 1, sizeof(Sass_Import*));\n  }\n\n  // Creator for a single import entry returned by the custom importer inside the list\n  // We take ownership of the memory for source and srcmap (freed when context is destroyd)\n  Sass_Import_Entry ADDCALL sass_make_import(const char* imp_path, const char* abs_path, char* source, char* srcmap)\n  {\n    Sass_Import* v = (Sass_Import*) calloc(1, sizeof(Sass_Import));\n    if (v == 0) return 0;\n    v->imp_path = imp_path ? sass_copy_c_string(imp_path) : 0;\n    v->abs_path = abs_path ? sass_copy_c_string(abs_path) : 0;\n    v->source = source;\n    v->srcmap = srcmap;\n    v->error = 0;\n    v->line = -1;\n    v->column = -1;\n    return v;\n  }\n\n  // Older style, but somehow still valid - keep around or deprecate?\n  Sass_Import_Entry ADDCALL sass_make_import_entry(const char* path, char* source, char* srcmap)\n  {\n    return sass_make_import(path, path, source, srcmap);\n  }\n\n  // Upgrade a normal import entry to throw an error (original path can be re-used by error reporting)\n  Sass_Import_Entry ADDCALL sass_import_set_error(Sass_Import_Entry import, const char* error, size_t line, size_t col)\n  {\n    if (import == 0) return 0;\n    if (import->error) free(import->error);\n    import->error = error ? sass_copy_c_string(error) : 0;\n    import->line = line ? line : -1;\n    import->column = col ? col : -1;\n    return import;\n  }\n\n  // Setters and getters for entries on the import list\n  void ADDCALL sass_import_set_list_entry(Sass_Import_List list, size_t idx, Sass_Import_Entry entry) { list[idx] = entry; }\n  Sass_Import_Entry ADDCALL sass_import_get_list_entry(Sass_Import_List list, size_t idx) { return list[idx]; }\n\n  // Deallocator for the allocated memory\n  void ADDCALL sass_delete_import_list(Sass_Import_List list)\n  {\n    Sass_Import_List it = list;\n    if (list == 0) return;\n    while(*list) {\n      sass_delete_import(*list);\n      ++list;\n    }\n    free(it);\n  }\n\n  // Just in case we have some stray import structs\n  void ADDCALL sass_delete_import(Sass_Import_Entry import)\n  {\n    free(import->imp_path);\n    free(import->abs_path);\n    free(import->source);\n    free(import->srcmap);\n    free(import->error);\n    free(import);\n  }\n\n  // Getter for callee entry\n  const char* ADDCALL sass_callee_get_name(Sass_Callee_Entry entry) { return entry->name; }\n  const char* ADDCALL sass_callee_get_path(Sass_Callee_Entry entry) { return entry->path; }\n  size_t ADDCALL sass_callee_get_line(Sass_Callee_Entry entry) { return entry->line; }\n  size_t ADDCALL sass_callee_get_column(Sass_Callee_Entry entry) { return entry->column; }\n  enum Sass_Callee_Type ADDCALL sass_callee_get_type(Sass_Callee_Entry entry) { return entry->type; }\n  Sass_Env_Frame ADDCALL sass_callee_get_env (Sass_Callee_Entry entry) { return &entry->env; }\n\n  // Getters and Setters for environments (lexical, local and global)\n  union Sass_Value* ADDCALL sass_env_get_lexical (Sass_Env_Frame env, const char* name) {\n    Expression_Ptr ex = Cast<Expression>((*env->frame)[name]);\n    return ex != NULL ? ast_node_to_sass_value(ex) : NULL;\n  }\n  void ADDCALL sass_env_set_lexical (Sass_Env_Frame env, const char* name, union Sass_Value* val) {\n    (*env->frame)[name] = sass_value_to_ast_node(val);\n  }\n  union Sass_Value* ADDCALL sass_env_get_local (Sass_Env_Frame env, const char* name) {\n    Expression_Ptr ex = Cast<Expression>(env->frame->get_local(name));\n    return ex != NULL ? ast_node_to_sass_value(ex) : NULL;\n  }\n  void ADDCALL sass_env_set_local (Sass_Env_Frame env, const char* name, union Sass_Value* val) {\n    env->frame->set_local(name, sass_value_to_ast_node(val));\n  }\n  union Sass_Value* ADDCALL sass_env_get_global (Sass_Env_Frame env, const char* name) {\n    Expression_Ptr ex = Cast<Expression>(env->frame->get_global(name));\n    return ex != NULL ? ast_node_to_sass_value(ex) : NULL;\n  }\n  void ADDCALL sass_env_set_global (Sass_Env_Frame env, const char* name, union Sass_Value* val) {\n    env->frame->set_global(name, sass_value_to_ast_node(val));\n  }\n\n  // Getter for import entry\n  const char* ADDCALL sass_import_get_imp_path(Sass_Import_Entry entry) { return entry->imp_path; }\n  const char* ADDCALL sass_import_get_abs_path(Sass_Import_Entry entry) { return entry->abs_path; }\n  const char* ADDCALL sass_import_get_source(Sass_Import_Entry entry) { return entry->source; }\n  const char* ADDCALL sass_import_get_srcmap(Sass_Import_Entry entry) { return entry->srcmap; }\n\n  // Getter for import error entry\n  size_t ADDCALL sass_import_get_error_line(Sass_Import_Entry entry) { return entry->line; }\n  size_t ADDCALL sass_import_get_error_column(Sass_Import_Entry entry) { return entry->column; }\n  const char* ADDCALL sass_import_get_error_message(Sass_Import_Entry entry) { return entry->error; }\n\n  // Explicit functions to take ownership of the memory\n  // Resets our own property since we do not know if it is still alive\n  char* ADDCALL sass_import_take_source(Sass_Import_Entry entry) { char* ptr = entry->source; entry->source = 0; return ptr; }\n  char* ADDCALL sass_import_take_srcmap(Sass_Import_Entry entry) { char* ptr = entry->srcmap; entry->srcmap = 0; return ptr; }\n\n}\n\n// Path: src/libsass/src/subset_map.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"subset_map.hpp\"\n\nnamespace Sass {\n\n  void Subset_Map::put(const Compound_Selector_Obj& sel, const SubSetMapPair& value)\n  {\n    if (sel->empty()) throw std::runtime_error(\"internal error: subset map keys may not be empty\");\n    size_t index = values_.size();\n    values_.push_back(value);\n    for (size_t i = 0, S = sel->length(); i < S; ++i)\n    {\n      hash_[(*sel)[i]].push_back(std::make_pair(sel, index));\n    }\n  }\n\n  std::vector<SubSetMapPair> Subset_Map::get_kv(const Compound_Selector_Obj& sel)\n  {\n    SimpleSelectorDict dict(sel->begin(), sel->end()); // XXX Set\n    std::vector<size_t> indices;\n    for (size_t i = 0, S = sel->length(); i < S; ++i) {\n      if (!hash_.count((*sel)[i])) {\n        continue;\n      }\n      const std::vector<std::pair<Compound_Selector_Obj, size_t> >& subsets = hash_[(*sel)[i]];\n      for (const std::pair<Compound_Selector_Obj, size_t>& item : subsets) {\n        bool include = true;\n        for (const Simple_Selector_Obj& it : item.first->elements()) {\n          auto found = dict.find(it);\n          if (found == dict.end()) {\n            include = false;\n            break;\n          }\n        }\n        if (include) indices.push_back(item.second);\n      }\n    }\n    sort(indices.begin(), indices.end());\n    std::vector<size_t>::iterator indices_end = unique(indices.begin(), indices.end());\n    indices.resize(distance(indices.begin(), indices_end));\n\n    std::vector<SubSetMapPair> results;\n    for (size_t i = 0, S = indices.size(); i < S; ++i) {\n      results.push_back(values_[indices[i]]);\n    }\n    return results;\n  }\n\n  std::vector<SubSetMapPair> Subset_Map::get_v(const Compound_Selector_Obj& sel)\n  {\n    return get_kv(sel);\n  }\n\n}\n// Path: src/libsass/src/to_value.hpp\n#ifndef SASS_TO_VALUE_H\n#define SASS_TO_VALUE_H\n\n#include \"operation.hpp\"\n#include \"sass/values.h\"\n#include \"ast_fwd_decl.hpp\"\n\nnamespace Sass {\n\n  class To_Value : public Operation_CRTP<Value_Ptr, To_Value> {\n\n    Value_Ptr fallback_impl(AST_Node_Ptr n);\n\n  private:\n\n    Context& ctx;\n\n  public:\n\n    To_Value(Context& ctx)\n    : ctx(ctx)\n    { }\n    ~To_Value() { }\n    using Operation<Value_Ptr>::operator();\n\n    Value_Ptr operator()(Argument_Ptr);\n    Value_Ptr operator()(Boolean_Ptr);\n    Value_Ptr operator()(Number_Ptr);\n    Value_Ptr operator()(Color_Ptr);\n    Value_Ptr operator()(String_Constant_Ptr);\n    Value_Ptr operator()(String_Quoted_Ptr);\n    Value_Ptr operator()(Custom_Warning_Ptr);\n    Value_Ptr operator()(Custom_Error_Ptr);\n    Value_Ptr operator()(List_Ptr);\n    Value_Ptr operator()(Map_Ptr);\n    Value_Ptr operator()(Null_Ptr);\n    Value_Ptr operator()(Function_Ptr);\n\n    // convert to string via `To_String`\n    Value_Ptr operator()(Selector_List_Ptr);\n    Value_Ptr operator()(Binary_Expression_Ptr);\n\n    // fallback throws error\n    template <typename U>\n    Value_Ptr fallback(U x) { return fallback_impl(x); }\n  };\n\n}\n\n#endif\n\n// Path: src/libsass/src/to_value.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"to_value.hpp\"\n\nnamespace Sass {\n\n  Value_Ptr To_Value::fallback_impl(AST_Node_Ptr n)\n  {\n    // throw a runtime error if this happens\n    // we want a well defined set of possible nodes\n    throw std::runtime_error(\"invalid node for to_value\");\n  }\n\n  // Custom_Error is a valid value\n  Value_Ptr To_Value::operator()(Custom_Error_Ptr e)\n  {\n    return e;\n  }\n\n  // Custom_Warning is a valid value\n  Value_Ptr To_Value::operator()(Custom_Warning_Ptr w)\n  {\n    return w;\n  }\n\n  // Boolean is a valid value\n  Value_Ptr To_Value::operator()(Boolean_Ptr b)\n  {\n    return b;\n  }\n\n  // Number is a valid value\n  Value_Ptr To_Value::operator()(Number_Ptr n)\n  {\n    return n;\n  }\n\n  // Color is a valid value\n  Value_Ptr To_Value::operator()(Color_Ptr c)\n  {\n    return c;\n  }\n\n  // String_Constant is a valid value\n  Value_Ptr To_Value::operator()(String_Constant_Ptr s)\n  {\n    return s;\n  }\n\n  // String_Quoted is a valid value\n  Value_Ptr To_Value::operator()(String_Quoted_Ptr s)\n  {\n    return s;\n  }\n\n  // List is a valid value\n  Value_Ptr To_Value::operator()(List_Ptr l)\n  {\n    List_Obj ll = SASS_MEMORY_NEW(List,\n                               l->pstate(),\n                               l->length(),\n                               l->separator(),\n                               l->is_arglist(),\n                               l->is_bracketed());\n    for (size_t i = 0, L = l->length(); i < L; ++i) {\n      ll->append((*l)[i]->perform(this));\n    }\n    return ll.detach();\n  }\n\n  // Map is a valid value\n  Value_Ptr To_Value::operator()(Map_Ptr m)\n  {\n    return m;\n  }\n\n  // Null is a valid value\n  Value_Ptr To_Value::operator()(Null_Ptr n)\n  {\n    return n;\n  }\n\n  // Function is a valid value\n  Value_Ptr To_Value::operator()(Function_Ptr n)\n  {\n    return n;\n  }\n\n  // Argument returns its value\n  Value_Ptr To_Value::operator()(Argument_Ptr arg)\n  {\n    if (!arg->name().empty()) return 0;\n    return arg->value()->perform(this);\n  }\n\n  // Selector_List is converted to a string\n  Value_Ptr To_Value::operator()(Selector_List_Ptr s)\n  {\n    return SASS_MEMORY_NEW(String_Quoted,\n                           s->pstate(),\n                           s->to_string(ctx.c_options));\n  }\n\n  // Binary_Expression is converted to a string\n  Value_Ptr To_Value::operator()(Binary_Expression_Ptr s)\n  {\n    return SASS_MEMORY_NEW(String_Quoted,\n                           s->pstate(),\n                           s->to_string(ctx.c_options));\n  }\n\n};\n\n// Path: src/libsass/src/json.hpp\n/*\n  Copyright (C) 2011 Joseph A. Adams (joeyadams3.14159@gmail.com)\n  All rights reserved.\n\n  Permission is hereby granted, free of charge, to any person obtaining a copy\n  of this software and associated documentation files (the \"Software\"), to deal\n  in the Software without restriction, including without limitation the rights\n  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n  copies of the Software, and to permit persons to whom the Software is\n  furnished to do so, subject to the following conditions:\n\n  The above copyright notice and this permission notice shall be included in\n  all copies or substantial portions of the Software.\n\n  THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n  THE SOFTWARE.\n*/\n\n#ifndef CCAN_JSON_H\n#define CCAN_JSON_H\n\n#include <stdbool.h>\n#include <stddef.h>\n\ntypedef enum {\n  JSON_NULL,\n  JSON_BOOL,\n  JSON_STRING,\n  JSON_NUMBER,\n  JSON_ARRAY,\n  JSON_OBJECT,\n} JsonTag;\n\ntypedef struct JsonNode JsonNode;\n\nstruct JsonNode\n{\n  /* only if parent is an object or array (NULL otherwise) */\n  JsonNode *parent;\n  JsonNode *prev, *next;\n\n  /* only if parent is an object (NULL otherwise) */\n  char *key; /* Must be valid UTF-8. */\n\n  JsonTag tag;\n  union {\n    /* JSON_BOOL */\n    bool bool_;\n\n    /* JSON_STRING */\n    char *string_; /* Must be valid UTF-8. */\n\n    /* JSON_NUMBER */\n    double number_;\n\n    /* JSON_ARRAY */\n    /* JSON_OBJECT */\n    struct {\n      JsonNode *head, *tail;\n    } children;\n  };\n};\n\n/*** Encoding, decoding, and validation ***/\n\nJsonNode   *json_decode         (const char *json);\nchar       *json_encode         (const JsonNode *node);\nchar       *json_encode_string  (const char *str);\nchar       *json_stringify      (const JsonNode *node, const char *space);\nvoid        json_delete         (JsonNode *node);\n\nbool        json_validate       (const char *json);\n\n/*** Lookup and traversal ***/\n\nJsonNode   *json_find_element   (JsonNode *array, int index);\nJsonNode   *json_find_member    (JsonNode *object, const char *key);\n\nJsonNode   *json_first_child    (const JsonNode *node);\n\n#define json_foreach(i, object_or_array)            \\\n  for ((i) = json_first_child(object_or_array);   \\\n     (i) != NULL;                               \\\n     (i) = (i)->next)\n\n/*** Construction and manipulation ***/\n\nJsonNode *json_mknull(void);\nJsonNode *json_mkbool(bool b);\nJsonNode *json_mkstring(const char *s);\nJsonNode *json_mknumber(double n);\nJsonNode *json_mkarray(void);\nJsonNode *json_mkobject(void);\n\nvoid json_append_element(JsonNode *array, JsonNode *element);\nvoid json_prepend_element(JsonNode *array, JsonNode *element);\nvoid json_append_member(JsonNode *object, const char *key, JsonNode *value);\nvoid json_prepend_member(JsonNode *object, const char *key, JsonNode *value);\n\nvoid json_remove_from_parent(JsonNode *node);\n\n/*** Debugging ***/\n\n/*\n * Look for structure and encoding problems in a JsonNode or its descendents.\n *\n * If a problem is detected, return false, writing a description of the problem\n * to errmsg (unless errmsg is NULL).\n */\nbool json_check(const JsonNode *node, char errmsg[256]);\n\n#endif\n\n// Path: src/libsass/src/sass_context.cpp\n#include \"sass.hpp\"\n#include <cstring>\n#include <stdexcept>\n#include <sstream>\n#include <string>\n#include <vector>\n\n#include \"sass.h\"\n#include \"ast.hpp\"\n#include \"file.hpp\"\n#include \"json.hpp\"\n#include \"util.hpp\"\n#include \"context.hpp\"\n#include \"sass_context.hpp\"\n#include \"sass_functions.hpp\"\n#include \"ast_fwd_decl.hpp\"\n#include \"error_handling.hpp\"\n\n#define LFEED \"\\n\"\n\n// C++ helper\nnamespace Sass {\n  // see sass_copy_c_string(std::string str)\n  static inline JsonNode* json_mkstream(const std::stringstream& stream)\n  {\n    // hold on to string on stack!\n    std::string str(stream.str());\n    return json_mkstring(str.c_str());\n  }\n\n  static int handle_error(Sass_Context* c_ctx) {\n    try {\n      throw;\n    }\n    catch (Exception::Base& e) {\n      std::stringstream msg_stream;\n      std::string cwd(Sass::File::get_cwd());\n      std::string msg_prefix(e.errtype());\n      bool got_newline = false;\n      msg_stream << msg_prefix << \": \";\n      const char* msg = e.what();\n      while (msg && *msg) {\n        if (*msg == '\\r') {\n          got_newline = true;\n        }\n        else if (*msg == '\\n') {\n          got_newline = true;\n        }\n        else if (got_newline) {\n          msg_stream << std::string(msg_prefix.size() + 2, ' ');\n          got_newline = false;\n        }\n        msg_stream << *msg;\n        ++msg;\n      }\n      if (!got_newline) msg_stream << \"\\n\";\n\n      if (e.traces.empty()) {\n        // we normally should have some traces, still here as a fallback\n        std::string rel_path(Sass::File::abs2rel(e.pstate.path, cwd, cwd));\n        msg_stream << std::string(msg_prefix.size() + 2, ' ');\n        msg_stream << \" on line \" << e.pstate.line + 1 << \" of \" << rel_path << \"\\n\";\n      }\n      else {\n        std::string rel_path(Sass::File::abs2rel(e.pstate.path, cwd, cwd));\n        msg_stream << traces_to_string(e.traces, \"        \");\n      }\n\n      // now create the code trace (ToDo: maybe have util functions?)\n      if (e.pstate.line != std::string::npos && e.pstate.column != std::string::npos) {\n        size_t lines = e.pstate.line;\n        const char* line_beg = e.pstate.src;\n        // scan through src until target line\n        // move line_beg pointer to line start\n        while (line_beg && *line_beg && lines != 0) {\n          if (*line_beg == '\\n') --lines;\n          utf8::unchecked::next(line_beg); \n        }\n        const char* line_end = line_beg;\n        // move line_end before next newline character\n        while (line_end && *line_end && *line_end != '\\n') {\n          if (*line_end == '\\n') break;\n          if (*line_end == '\\r') break;\n          utf8::unchecked::next(line_end); \n        }\n        if (line_end && *line_end != 0) ++ line_end;\n        size_t line_len = line_end - line_beg;\n        size_t move_in = 0; size_t shorten = 0;\n        size_t left_chars = 42; size_t max_chars = 76;\n        // reported excerpt should not exceed `max_chars` chars\n        if (e.pstate.column > line_len) left_chars = e.pstate.column;\n        if (e.pstate.column > left_chars) move_in = e.pstate.column - left_chars;\n        if (line_len > max_chars + move_in) shorten = line_len - move_in - max_chars;\n        utf8::advance(line_beg, move_in, line_end);\n        utf8::retreat(line_end, shorten, line_beg);\n        std::string sanitized; std::string marker(e.pstate.column - move_in, '-');\n        utf8::replace_invalid(line_beg, line_end, std::back_inserter(sanitized));\n        msg_stream << \">> \" << sanitized << \"\\n\";\n        msg_stream << \"   \" << marker << \"^\\n\";\n      }\n\n      JsonNode* json_err = json_mkobject();\n      json_append_member(json_err, \"status\", json_mknumber(1));\n      json_append_member(json_err, \"file\", json_mkstring(e.pstate.path));\n      json_append_member(json_err, \"line\", json_mknumber((double)(e.pstate.line + 1)));\n      json_append_member(json_err, \"column\", json_mknumber((double)(e.pstate.column + 1)));\n      json_append_member(json_err, \"message\", json_mkstring(e.what()));\n      json_append_member(json_err, \"formatted\", json_mkstream(msg_stream));\n      try { c_ctx->error_json = json_stringify(json_err, \"  \"); }\n      catch (...) {}\n      c_ctx->error_message = sass_copy_string(msg_stream.str());\n      c_ctx->error_text = sass_copy_c_string(e.what());\n      c_ctx->error_status = 1;\n      c_ctx->error_file = sass_copy_c_string(e.pstate.path);\n      c_ctx->error_line = e.pstate.line + 1;\n      c_ctx->error_column = e.pstate.column + 1;\n      c_ctx->error_src = e.pstate.src;\n      c_ctx->output_string = 0;\n      c_ctx->source_map_string = 0;\n      json_delete(json_err);\n    }\n    catch (std::bad_alloc& ba) {\n      std::stringstream msg_stream;\n      JsonNode* json_err = json_mkobject();\n      msg_stream << \"Unable to allocate memory: \" << ba.what() << std::endl;\n      json_append_member(json_err, \"status\", json_mknumber(2));\n      json_append_member(json_err, \"message\", json_mkstring(ba.what()));\n      json_append_member(json_err, \"formatted\", json_mkstream(msg_stream));\n      try { c_ctx->error_json = json_stringify(json_err, \"  \"); }\n      catch (...) {}\n      c_ctx->error_message = sass_copy_string(msg_stream.str());\n      c_ctx->error_text = sass_copy_c_string(ba.what());\n      c_ctx->error_status = 2;\n      c_ctx->output_string = 0;\n      c_ctx->source_map_string = 0;\n      json_delete(json_err);\n    }\n    catch (std::exception& e) {\n      std::stringstream msg_stream;\n      JsonNode* json_err = json_mkobject();\n      msg_stream << \"Internal Error: \" << e.what() << std::endl;\n      json_append_member(json_err, \"status\", json_mknumber(3));\n      json_append_member(json_err, \"message\", json_mkstring(e.what()));\n      json_append_member(json_err, \"formatted\", json_mkstream(msg_stream));\n      try { c_ctx->error_json = json_stringify(json_err, \"  \"); }\n      catch (...) {}\n      c_ctx->error_message = sass_copy_string(msg_stream.str());\n      c_ctx->error_text = sass_copy_c_string(e.what());\n      c_ctx->error_status = 3;\n      c_ctx->output_string = 0;\n      c_ctx->source_map_string = 0;\n      json_delete(json_err);\n    }\n    catch (std::string& e) {\n      std::stringstream msg_stream;\n      JsonNode* json_err = json_mkobject();\n      msg_stream << \"Internal Error: \" << e << std::endl;\n      json_append_member(json_err, \"status\", json_mknumber(4));\n      json_append_member(json_err, \"message\", json_mkstring(e.c_str()));\n      json_append_member(json_err, \"formatted\", json_mkstream(msg_stream));\n      try { c_ctx->error_json = json_stringify(json_err, \"  \"); }\n      catch (...) {}\n      c_ctx->error_message = sass_copy_string(msg_stream.str());\n      c_ctx->error_text = sass_copy_c_string(e.c_str());\n      c_ctx->error_status = 4;\n      c_ctx->output_string = 0;\n      c_ctx->source_map_string = 0;\n      json_delete(json_err);\n    }\n    catch (const char* e) {\n      std::stringstream msg_stream;\n      JsonNode* json_err = json_mkobject();\n      msg_stream << \"Internal Error: \" << e << std::endl;\n      json_append_member(json_err, \"status\", json_mknumber(4));\n      json_append_member(json_err, \"message\", json_mkstring(e));\n      json_append_member(json_err, \"formatted\", json_mkstream(msg_stream));\n      try { c_ctx->error_json = json_stringify(json_err, \"  \"); }\n      catch (...) {}\n      c_ctx->error_message = sass_copy_string(msg_stream.str());\n      c_ctx->error_text = sass_copy_c_string(e);\n      c_ctx->error_status = 4;\n      c_ctx->output_string = 0;\n      c_ctx->source_map_string = 0;\n      json_delete(json_err);\n    }\n    catch (...) {\n      std::stringstream msg_stream;\n      JsonNode* json_err = json_mkobject();\n      msg_stream << \"Unknown error occurred\" << std::endl;\n      json_append_member(json_err, \"status\", json_mknumber(5));\n      json_append_member(json_err, \"message\", json_mkstring(\"unknown\"));\n      try { c_ctx->error_json = json_stringify(json_err, \"  \"); }\n      catch (...) {}\n      c_ctx->error_message = sass_copy_string(msg_stream.str());\n      c_ctx->error_text = sass_copy_c_string(\"unknown\");\n      c_ctx->error_status = 5;\n      c_ctx->output_string = 0;\n      c_ctx->source_map_string = 0;\n      json_delete(json_err);\n    }\n    return c_ctx->error_status;\n  }\n\n  // allow one error handler to throw another error\n  // this can happen with invalid utf8 and json lib\n  static int handle_errors(Sass_Context* c_ctx) {\n    try { return handle_error(c_ctx); }\n    catch (...) { return handle_error(c_ctx); }\n  }\n\n  static Block_Obj sass_parse_block(Sass_Compiler* compiler) throw()\n  {\n\n    // assert valid pointer\n    if (compiler == 0) return 0;\n    // The cpp context must be set by now\n    Context* cpp_ctx = compiler->cpp_ctx;\n    Sass_Context* c_ctx = compiler->c_ctx;\n    // We will take care to wire up the rest\n    compiler->cpp_ctx->c_compiler = compiler;\n    compiler->state = SASS_COMPILER_PARSED;\n\n    try {\n\n      // get input/output path from options\n      std::string input_path = safe_str(c_ctx->input_path);\n      std::string output_path = safe_str(c_ctx->output_path);\n\n      // maybe skip some entries of included files\n      // we do not include stdin for data contexts\n      bool skip = c_ctx->type == SASS_CONTEXT_DATA;\n\n      // dispatch parse call\n      Block_Obj root(cpp_ctx->parse());\n      // abort on errors\n      if (!root) return 0;\n\n      // skip all prefixed files? (ToDo: check srcmap)\n      // IMO source-maps should point to headers already\n      // therefore don't skip it for now. re-enable or\n      // remove completely once this is tested\n      size_t headers = cpp_ctx->head_imports;\n\n      // copy the included files on to the context (dont forget to free later)\n      if (copy_strings(cpp_ctx->get_included_files(skip, headers), &c_ctx->included_files) == NULL)\n        throw(std::bad_alloc());\n\n      // return parsed block\n      return root;\n\n    }\n    // pass errors to generic error handler\n    catch (...) { handle_errors(c_ctx); }\n\n    // error\n    return 0;\n\n  }\n\n}\n\nextern \"C\" {\n  using namespace Sass;\n\n  static void sass_clear_options (struct Sass_Options* options);\n  static void sass_reset_options (struct Sass_Options* options);\n  static void copy_options(struct Sass_Options* to, struct Sass_Options* from) {\n    // do not overwrite ourself\n    if (to == from) return;\n    // free assigned memory\n    sass_clear_options(to);\n    // move memory\n    *to = *from;\n    // Reset pointers on source\n    sass_reset_options(from);\n  }\n\n  #define IMPLEMENT_SASS_OPTION_ACCESSOR(type, option) \\\n    type ADDCALL sass_option_get_##option (struct Sass_Options* options) { return options->option; } \\\n    void ADDCALL sass_option_set_##option (struct Sass_Options* options, type option) { options->option = option; }\n  #define IMPLEMENT_SASS_OPTION_STRING_GETTER(type, option, def) \\\n    type ADDCALL sass_option_get_##option (struct Sass_Options* options) { return safe_str(options->option, def); }\n  #define IMPLEMENT_SASS_OPTION_STRING_SETTER(type, option, def) \\\n    void ADDCALL sass_option_set_##option (struct Sass_Options* options, type option) \\\n    { free(options->option); options->option = option || def ? sass_copy_c_string(option ? option : def) : 0; }\n  #define IMPLEMENT_SASS_OPTION_STRING_ACCESSOR(type, option, def) \\\n    IMPLEMENT_SASS_OPTION_STRING_GETTER(type, option, def) \\\n    IMPLEMENT_SASS_OPTION_STRING_SETTER(type, option, def)\n\n  #define IMPLEMENT_SASS_CONTEXT_GETTER(type, option) \\\n    type ADDCALL sass_context_get_##option (struct Sass_Context* ctx) { return ctx->option; }\n  #define IMPLEMENT_SASS_CONTEXT_TAKER(type, option) \\\n    type sass_context_take_##option (struct Sass_Context* ctx) \\\n    { type foo = ctx->option; ctx->option = 0; return foo; }\n\n\n  // generic compilation function (not exported, use file/data compile instead)\n  static Sass_Compiler* sass_prepare_context (Sass_Context* c_ctx, Context* cpp_ctx) throw()\n  {\n    try {\n      // register our custom functions\n      if (c_ctx->c_functions) {\n        auto this_func_data = c_ctx->c_functions;\n        while (this_func_data && *this_func_data) {\n          cpp_ctx->add_c_function(*this_func_data);\n          ++this_func_data;\n        }\n      }\n\n      // register our custom headers\n      if (c_ctx->c_headers) {\n        auto this_head_data = c_ctx->c_headers;\n        while (this_head_data && *this_head_data) {\n          cpp_ctx->add_c_header(*this_head_data);\n          ++this_head_data;\n        }\n      }\n\n      // register our custom importers\n      if (c_ctx->c_importers) {\n        auto this_imp_data = c_ctx->c_importers;\n        while (this_imp_data && *this_imp_data) {\n          cpp_ctx->add_c_importer(*this_imp_data);\n          ++this_imp_data;\n        }\n      }\n\n      // reset error status\n      c_ctx->error_json = 0;\n      c_ctx->error_text = 0;\n      c_ctx->error_message = 0;\n      c_ctx->error_status = 0;\n      // reset error position\n      c_ctx->error_src = 0;\n      c_ctx->error_file = 0;\n      c_ctx->error_line = std::string::npos;\n      c_ctx->error_column = std::string::npos;\n\n      // allocate a new compiler instance\n      void* ctxmem = calloc(1, sizeof(struct Sass_Compiler));\n      if (ctxmem == 0) { std::cerr << \"Error allocating memory for context\" << std::endl; return 0; }\n      Sass_Compiler* compiler = (struct Sass_Compiler*) ctxmem;\n      compiler->state = SASS_COMPILER_CREATED;\n\n      // store in sass compiler\n      compiler->c_ctx = c_ctx;\n      compiler->cpp_ctx = cpp_ctx;\n      cpp_ctx->c_compiler = compiler;\n\n      // use to parse block\n      return compiler;\n\n    }\n    // pass errors to generic error handler\n    catch (...) { handle_errors(c_ctx); }\n\n    // error\n    return 0;\n\n  }\n\n  // generic compilation function (not exported, use file/data compile instead)\n  static int sass_compile_context (Sass_Context* c_ctx, Context* cpp_ctx)\n  {\n\n    // prepare sass compiler with context and options\n...\n", "needle_token_start": 674, "needle_token_end": 931, "code_context_ntokens": 16165, "output": ["```cpp\ninline bool compatibility(const char* their_version)\n{\n  // const char* their_version = \"3.1.2\";\n  // first check if anyone has an unknown version\n  const char* our_version = libsass_version();\n  if (!strcmp(their_version, \"[na]\")) return false;\n  if (!strcmp(our_version, \"[na]\")) return false;\n\n  // find the position of the second dot\n  size_t pos = std::string(our_version).find('.', 0);\n  if (pos != std::string::npos) pos = std::string(our_version).find('.', pos + 1);\n\n  // if we do not have two dots we fallback to compare complete string\n  if (pos == std::string::npos) { return strcmp(their_version, our_version) ? 0 : 1; }\n  // otherwise only compare up to the second dot (major versions)\n  else { return strncmp(their_version, our_version, pos) ? 0 : 1; }\n\n}\n```"]}
{"repo": "sass/node-sass", "name": "Sass_Inspect_Options", "language": "cpp", "path": "src/libsass/src/sass.hpp", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this structure is to configure the output characteristics of CSS code generated from SASS/SCSS files, including the style format, numerical precision, and color compression settings.\n2. **Input**: This structure takes three parameters: a style format for the output CSS (from a predefined set of styles), a numerical precision for fractional values, and a boolean indicating whether to compress color values in selectors.\n3. **Output**: There is no direct output from this structure itself; rather, it influences the output of the CSS generation process by specifying how the CSS should be formatted and how precise numerical values should be.\n4. **Procedure**: Upon initialization, the structure is configured with default values or with values provided by the user. These settings are then used during the CSS generation process to determine the output style, precision of numerical values, and whether colors in selectors should be compressed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/sass_types/string.cpp\n#include <nan.h>\n#include \"string.h\"\n#include \"../create_string.h\"\n\nnamespace SassTypes\n{\n  String::String(Sass_Value* v) : SassValueWrapper(v) {}\n\n  Sass_Value* String::construct(const std::vector<v8::Local<v8::Value>> raw_val, Sass_Value **out) {\n    char const* value = \"\";\n\n    if (raw_val.size() >= 1) {\n      if (!raw_val[0]->IsString()) {\n        return fail(\"Argument should be a string.\", out);\n      }\n\n      value = create_string(raw_val[0]);\n...\n// Path: src/sass_types/null.cpp\n#include <nan.h>\n#include \"null.h\"\n\nnamespace SassTypes\n{\n  Nan::Persistent<v8::Function> Null::constructor;\n  bool Null::constructor_locked = false;\n\n  Null::Null() {\n      value = sass_make_null();\n  }\n\n  Null& Null::get_singleton() {\n    static Null singleton_instance;\n    return singleton_instance;\n  }\n\n  v8::Local<v8::Function> Null::get_constructor() {\n    Nan::EscapableHandleScope scope;\n    v8::Local<v8::Function> conslocal;\n    if (constructor.IsEmpty()) {\n      v8::Local<v8::FunctionTemplate> tpl = Nan::New<v8::FunctionTemplate>(New);\n\n      tpl->SetClassName(Nan::New(\"SassNull\").ToLocalChecked());\n      tpl->InstanceTemplate()->SetInternalFieldCount(1);\n\n      conslocal = Nan::GetFunction(tpl).ToLocalChecked();\n      constructor.Reset(conslocal);\n\n      get_singleton().js_object.Reset(Nan::NewInstance(conslocal).ToLocalChecked());\n      Nan::SetInternalFieldPointer(Nan::New(get_singleton().js_object), 0, &get_singleton());\n      Nan::Set(conslocal, Nan::New(\"NULL\").ToLocalChecked(), Nan::New(get_singleton().js_object));\n\n      constructor_locked = true;\n    } else {\n      conslocal = Nan::New(constructor);\n    }\n\n    return scope.Escape(conslocal);\n  }\n\n  v8::Local<v8::Object> Null::get_js_object() {\n    return Nan::New(this->js_object);\n  }\n\n  NAN_METHOD(Null::New) {\n\n    if (info.IsConstructCall()) {\n      if (constructor_locked) {\n        return Nan::ThrowTypeError(\"Cannot instantiate SassNull\");\n      }\n    }\n    else {\n      info.GetReturnValue().Set(get_singleton().get_js_object());\n    }\n  }\n}\n\n// Path: src/sass_types/boolean.cpp\n#include <nan.h>\n#include \"boolean.h\"\n\nnamespace SassTypes\n{\n  Nan::Persistent<v8::Function> Boolean::constructor;\n  bool Boolean::constructor_locked = false;\n\n  Boolean::Boolean(bool _value) {\n    value = sass_make_boolean(_value);\n  }\n\n  Boolean& Boolean::get_singleton(bool v) {\n    static Boolean instance_false(false), instance_true(true);\n    return v ? instance_true : instance_false;\n  }\n\n  v8::Local<v8::Function> Boolean::get_constructor() {\n    Nan::EscapableHandleScope scope;\n    v8::Local<v8::Function> conslocal;\n    if (constructor.IsEmpty()) {\n      v8::Local<v8::FunctionTemplate> tpl = Nan::New<v8::FunctionTemplate>(New);\n\n      tpl->SetClassName(Nan::New(\"SassBoolean\").ToLocalChecked());\n      tpl->InstanceTemplate()->SetInternalFieldCount(1);\n      Nan::SetPrototypeTemplate(tpl, \"getValue\", Nan::New<v8::FunctionTemplate>(GetValue));\n\n      conslocal = Nan::GetFunction(tpl).ToLocalChecked();\n      constructor.Reset(conslocal);\n\n      get_singleton(false).js_object.Reset(Nan::NewInstance(conslocal).ToLocalChecked());\n      Nan::SetInternalFieldPointer(Nan::New(get_singleton(false).js_object), 0, &get_singleton(false));\n      Nan::Set(conslocal, Nan::New(\"FALSE\").ToLocalChecked(), Nan::New(get_singleton(false).js_object));\n\n      get_singleton(true).js_object.Reset(Nan::NewInstance(conslocal).ToLocalChecked());\n      Nan::SetInternalFieldPointer(Nan::New(get_singleton(true).js_object), 0, &get_singleton(true));\n      Nan::Set(conslocal, Nan::New(\"TRUE\").ToLocalChecked(), Nan::New(get_singleton(true).js_object));\n\n      constructor_locked = true;\n    } else {\n      conslocal = Nan::New(constructor);\n    }\n\n    return scope.Escape(conslocal);\n  }\n\n  v8::Local<v8::Object> Boolean::get_js_object() {\n    return Nan::New(this->js_object);\n  }\n\n  v8::Local<v8::Boolean> Boolean::get_js_boolean() {\n    return sass_boolean_get_value(this->value) ? Nan::True() : Nan::False();\n  }\n\n  NAN_METHOD(Boolean::New) {\n    if (info.IsConstructCall()) {\n      if (constructor_locked) {\n        return Nan::ThrowTypeError(\"Cannot instantiate SassBoolean\");\n      }\n    }\n    else {\n      if (info.Length() != 1 || !info[0]->IsBoolean()) {\n        return Nan::ThrowTypeError(\"Expected one boolean argument\");\n      }\n\n      info.GetReturnValue().Set(get_singleton(Nan::To<bool>(info[0]).FromJust()).get_js_object());\n    }\n  }\n\n  NAN_METHOD(Boolean::GetValue) {\n    info.GetReturnValue().Set(Boolean::Unwrap<Boolean>(info.This())->get_js_boolean());\n  }\n}\n\n// Path: src/libsass/src/sass.hpp\n// must be the first include in all compile units\n#ifndef SASS_SASS_H\n#define SASS_SASS_H\n\n// undefine extensions macro to tell sys includes\n// that we do not want any macros to be exported\n// mainly fixes an issue on SmartOS (SEC macro)\n#undef __EXTENSIONS__\n\n#ifdef _MSC_VER\n#pragma warning(disable : 4005)\n#endif\n\n// aplies to MSVC and MinGW\n#ifdef _WIN32\n// we do not want the ERROR macro\n# define NOGDI\n// we do not want the min/max macro\n# define NOMINMAX\n// we do not want the IN/OUT macro\n# define _NO_W32_PSEUDO_MODIFIERS\n#endif\n\n\n// should we be case insensitive\n// when dealing with files or paths\n#ifndef FS_CASE_SENSITIVE\n# ifdef _WIN32\n#  define FS_CASE_SENSITIVE 0\n# else\n#  define FS_CASE_SENSITIVE 1\n# endif\n#endif\n\n// path separation char\n#ifndef PATH_SEP\n# ifdef _WIN32\n#  define PATH_SEP ';'\n# else\n#  define PATH_SEP ':'\n# endif\n#endif\n\n\n// include C-API header\n#include \"sass/base.h\"\n\n// For C++ helper\n#include <string>\n\n// output behaviours\nnamespace Sass {\n\n  // create some C++ aliases for the most used options\n  const static Sass_Output_Style NESTED = SASS_STYLE_NESTED;\n  const static Sass_Output_Style COMPACT = SASS_STYLE_COMPACT;\n  const static Sass_Output_Style EXPANDED = SASS_STYLE_EXPANDED;\n  const static Sass_Output_Style COMPRESSED = SASS_STYLE_COMPRESSED;\n  // only used internal to trigger ruby inspect behavior\n  const static Sass_Output_Style INSPECT = SASS_STYLE_INSPECT;\n  const static Sass_Output_Style TO_SASS = SASS_STYLE_TO_SASS;\n\n  // helper to aid dreaded MSVC debug mode\n  // see implementation for more details\n  char* sass_copy_string(std::string str);\n\n}\n\n// input behaviours\nenum Sass_Input_Style {\n  SASS_CONTEXT_NULL,\n  SASS_CONTEXT_FILE,\n  SASS_CONTEXT_DATA,\n  SASS_CONTEXT_FOLDER\n};\n\n// simple linked list\nstruct string_list {\n  string_list* next;\n  char* string;\n};\n\n// sass config options structure\nstruct Sass_Inspect_Options {\n\n  // Output style for the generated css code\n  // A value from above SASS_STYLE_* constants\n  enum Sass_Output_Style output_style;\n\n  // Precision for fractional numbers\n  int precision;\n\n  // Do not compress colors in selectors\n  bool in_selector;\n\n  // initialization list (constructor with defaults)\n  \nSass_Inspect_Options(Sass_Output_Style style = Sass::NESTED,\n                       int precision = 5, bool in_selector = false)\n  : output_style(style), precision(precision), in_selector(in_selector)\n  { }\n\n};\n\n// sass config options structure\nstruct Sass_Output_Options : Sass_Inspect_Options {\n\n  // String to be used for indentation\n  const char* indent;\n  // String to be used to for line feeds\n  const char* linefeed;\n\n  // Emit comments in the generated CSS indicating\n  // the corresponding source line.\n  bool source_comments;\n\n  // initialization list (constructor with defaults)\n  Sass_Output_Options(struct Sass_Inspect_Options opt,\n                      const char* indent = \"  \",\n                      const char* linefeed = \"\\n\",\n                      bool source_comments = false)\n  : Sass_Inspect_Options(opt),\n    indent(indent), linefeed(linefeed),\n    source_comments(source_comments)\n  { }\n\n  // initialization list (constructor with defaults)\n  Sass_Output_Options(Sass_Output_Style style = Sass::NESTED,\n                      int precision = 5,\n                      const char* indent = \"  \",\n                      const char* linefeed = \"\\n\",\n                      bool source_comments = false)\n  : Sass_Inspect_Options(style, precision),\n    indent(indent), linefeed(linefeed),\n    source_comments(source_comments)\n  { }\n\n};\n\n#endif\n\n// Path: src/libsass/src/memory/SharedPtr.hpp\n#ifndef SASS_MEMORY_SHARED_PTR_H\n#define SASS_MEMORY_SHARED_PTR_H\n\n#include \"sass/base.h\"\n\n#include <vector>\n\nnamespace Sass {\n\n  class SharedPtr;\n\n  ///////////////////////////////////////////////////////////////////////////////\n  // Use macros for the allocation task, since overloading operator `new`\n  // has been proven to be flaky under certain compilers (see comment below).\n  ///////////////////////////////////////////////////////////////////////////////\n\n  #ifdef DEBUG_SHARED_PTR\n\n    #define SASS_MEMORY_NEW(Class, ...) \\\n      ((Class*)(new Class(__VA_ARGS__))->trace(__FILE__, __LINE__)) \\\n\n    #define SASS_MEMORY_COPY(obj) \\\n      ((obj)->copy(__FILE__, __LINE__)) \\\n\n    #define SASS_MEMORY_CLONE(obj) \\\n      ((obj)->clone(__FILE__, __LINE__)) \\\n\n  #else\n\n    #define SASS_MEMORY_NEW(Class, ...) \\\n      new Class(__VA_ARGS__) \\\n\n    #define SASS_MEMORY_COPY(obj) \\\n      ((obj)->copy()) \\\n\n    #define SASS_MEMORY_CLONE(obj) \\\n      ((obj)->clone()) \\\n\n  #endif\n\n  class SharedObj {\n  protected:\n  friend class SharedPtr;\n  friend class Memory_Manager;\n    #ifdef DEBUG_SHARED_PTR\n      static std::vector<SharedObj*> all;\n      std::string file;\n      size_t line;\n    #endif\n    static bool taint;\n    long refcounter;\n    // long refcount;\n    bool detached;\n    #ifdef DEBUG_SHARED_PTR\n      bool dbg;\n    #endif\n  public:\n    #ifdef DEBUG_SHARED_PTR\n      static void dumpMemLeaks();\n      SharedObj* trace(std::string file, size_t line) {\n        this->file = file;\n        this->line = line;\n        return this;\n      }\n    #endif\n    SharedObj();\n    #ifdef DEBUG_SHARED_PTR\n      std::string getDbgFile() {\n        return file;\n      }\n      size_t getDbgLine() {\n        return line;\n      }\n      void setDbg(bool dbg) {\n        this->dbg = dbg;\n      }\n    #endif\n    static void setTaint(bool val) {\n      taint = val;\n    }\n    virtual ~SharedObj();\n    long getRefCount() {\n      return refcounter;\n    }\n  };\n\n\n  class SharedPtr {\n  protected:\n    SharedObj* node;\n  protected:\n    void decRefCount();\n    void incRefCount();\n  public:\n    // the empty constructor\n    SharedPtr()\n    : node(NULL) {};\n    // the create constructor\n    SharedPtr(SharedObj* ptr);\n    // the copy constructor\n    SharedPtr(const SharedPtr& obj);\n    // the move constructor\n    SharedPtr(SharedPtr&& obj);\n    // copy assignment operator\n    SharedPtr& operator=(const SharedPtr& obj);\n    // move assignment operator\n    SharedPtr& operator=(SharedPtr&& obj);\n    // pure virtual destructor\n    virtual ~SharedPtr() = 0;\n  public:\n    SharedObj* obj () const {\n      return node;\n    };\n    SharedObj* operator-> () const {\n      return node;\n    };\n    bool isNull () {\n      return node == NULL;\n    };\n    bool isNull () const {\n      return node == NULL;\n    };\n    SharedObj* detach() const {\n      if (node) {\n        node->detached = true;\n      }\n      return node;\n    };\n    operator bool() const {\n      return node != NULL;\n    };\n\n  };\n\n  template < class T >\n  class SharedImpl : private SharedPtr {\n  public:\n    SharedImpl()\n    : SharedPtr(NULL) {};\n    SharedImpl(T* node)\n    : SharedPtr(node) {};\n    template < class U >\n    SharedImpl(SharedImpl<U> obj)\n    : SharedPtr(static_cast<T*>(obj.ptr())) {}\n    SharedImpl(T&& node)\n    : SharedPtr(node) {};\n    SharedImpl(const T& node)\n    : SharedPtr(node) {};\n    // the copy constructor\n    SharedImpl(const SharedImpl<T>& impl)\n    : SharedPtr(impl.node) {};\n    // the move constructor\n    SharedImpl(SharedImpl<T>&& impl)\n    : SharedPtr(impl.node) {};\n    // copy assignment operator\n    SharedImpl<T>& operator=(const SharedImpl<T>& rhs) {\n      if (node) decRefCount();\n      node = rhs.node;\n      incRefCount();\n      return *this;\n    }\n    // move assignment operator\n    SharedImpl<T>& operator=(SharedImpl<T>&& rhs) {\n      // don't move our self\n      if (this != &rhs) {\n        if (node) decRefCount();\n        node = std::move(rhs.node);\n        rhs.node = NULL;\n      }\n      return *this;\n    }\n    ~SharedImpl() {};\n  public:\n    operator T*() const {\n      return static_cast<T*>(this->obj());\n    }\n    operator T&() const {\n      return *static_cast<T*>(this->obj());\n    }\n    T& operator* () const {\n      return *static_cast<T*>(this->obj());\n    };\n    T* operator-> () const {\n      return static_cast<T*>(this->obj());\n    };\n    T* ptr () const {\n      return static_cast<T*>(this->obj());\n    };\n    T* detach() const {\n      if (this->obj() == NULL) return NULL;\n      return static_cast<T*>(SharedPtr::detach());\n    }\n    bool isNull() const {\n      return this->obj() == NULL;\n    }\n    bool operator<(const T& rhs) const {\n      return *this->ptr() < rhs;\n    };\n    operator bool() const {\n      return this->obj() != NULL;\n    };\n  };\n\n}\n\n#endif\n// Path: src/libsass/src/ast_fwd_decl.hpp\n#ifndef SASS_AST_FWD_DECL_H\n#define SASS_AST_FWD_DECL_H\n\n#include <map>\n#include <set>\n#include <deque>\n#include <vector>\n#include <typeinfo>\n#include <iostream>\n#include <algorithm>\n#include <unordered_map>\n#include <unordered_set>\n#include \"memory/SharedPtr.hpp\"\n#include \"sass/functions.h\"\n\n/////////////////////////////////////////////\n// Forward declarations for the AST visitors.\n/////////////////////////////////////////////\nnamespace Sass {\n\n  class AST_Node;\n  typedef AST_Node* AST_Node_Ptr;\n  typedef AST_Node const* AST_Node_Ptr_Const;\n\n  class Has_Block;\n  typedef Has_Block* Has_Block_Ptr;\n  typedef Has_Block const* Has_Block_Ptr_Const;\n\n  class Simple_Selector;\n  typedef Simple_Selector* Simple_Selector_Ptr;\n  typedef Simple_Selector const* Simple_Selector_Ptr_Const;\n\n  class PreValue;\n  typedef PreValue* PreValue_Ptr;\n  typedef PreValue const* PreValue_Ptr_Const;\n  class Thunk;\n  typedef Thunk* Thunk_Ptr;\n  typedef Thunk const* Thunk_Ptr_Const;\n  class Block;\n  typedef Block* Block_Ptr;\n  typedef Block const* Block_Ptr_Const;\n  class Expression;\n  typedef Expression* Expression_Ptr;\n  typedef Expression const* Expression_Ptr_Const;\n  class Statement;\n  typedef Statement* Statement_Ptr;\n  typedef Statement const* Statement_Ptr_Const;\n  class Value;\n  typedef Value* Value_Ptr;\n  typedef Value const* Value_Ptr_Const;\n  class Declaration;\n  typedef Declaration* Declaration_Ptr;\n  typedef Declaration const* Declaration_Ptr_Const;\n  class Ruleset;\n  typedef Ruleset* Ruleset_Ptr;\n  typedef Ruleset const* Ruleset_Ptr_Const;\n  class Bubble;\n  typedef Bubble* Bubble_Ptr;\n  typedef Bubble const* Bubble_Ptr_Const;\n  class Trace;\n  typedef Trace* Trace_Ptr;\n  typedef Trace const* Trace_Ptr_Const;\n\n  class Media_Block;\n  typedef Media_Block* Media_Block_Ptr;\n  typedef Media_Block const* Media_Block_Ptr_Const;\n  class Supports_Block;\n  typedef Supports_Block* Supports_Block_Ptr;\n  typedef Supports_Block const* Supports_Block_Ptr_Const;\n  class Directive;\n  typedef Directive* Directive_Ptr;\n  typedef Directive const* Directive_Ptr_Const;\n\n\n  class Keyframe_Rule;\n  typedef Keyframe_Rule* Keyframe_Rule_Ptr;\n  typedef Keyframe_Rule const* Keyframe_Rule_Ptr_Const;\n  class At_Root_Block;\n  typedef At_Root_Block* At_Root_Block_Ptr;\n  typedef At_Root_Block const* At_Root_Block_Ptr_Const;\n  class Assignment;\n  typedef Assignment* Assignment_Ptr;\n  typedef Assignment const* Assignment_Ptr_Const;\n\n  class Import;\n  typedef Import* Import_Ptr;\n  typedef Import const* Import_Ptr_Const;\n  class Import_Stub;\n  typedef Import_Stub* Import_Stub_Ptr;\n  typedef Import_Stub const* Import_Stub_Ptr_Const;\n  class Warning;\n  typedef Warning* Warning_Ptr;\n  typedef Warning const* Warning_Ptr_Const;\n\n  class Error;\n  typedef Error* Error_Ptr;\n  typedef Error const* Error_Ptr_Const;\n  class Debug;\n  typedef Debug* Debug_Ptr;\n  typedef Debug const* Debug_Ptr_Const;\n  class Comment;\n  typedef Comment* Comment_Ptr;\n  typedef Comment const* Comment_Ptr_Const;\n\n  class If;\n  typedef If* If_Ptr;\n  typedef If const* If_Ptr_Const;\n  class For;\n  typedef For* For_Ptr;\n  typedef For const* For_Ptr_Const;\n  class Each;\n  typedef Each* Each_Ptr;\n  typedef Each const* Each_Ptr_Const;\n  class While;\n  typedef While* While_Ptr;\n  typedef While const* While_Ptr_Const;\n  class Return;\n  typedef Return* Return_Ptr;\n  typedef Return const* Return_Ptr_Const;\n  class Content;\n  typedef Content* Content_Ptr;\n  typedef Content const* Content_Ptr_Const;\n  class Extension;\n  typedef Extension* Extension_Ptr;\n  typedef Extension const* Extension_Ptr_Const;\n  class Definition;\n  typedef Definition* Definition_Ptr;\n  typedef Definition const* Definition_Ptr_Const;\n\n  class List;\n  typedef List* List_Ptr;\n  typedef List const* List_Ptr_Const;\n  class Map;\n  typedef Map* Map_Ptr;\n  typedef Map const* Map_Ptr_Const;\n  class Function;\n  typedef Function* Function_Ptr;\n  typedef Function const* Function_Ptr_Const;\n\n  class Mixin_Call;\n  typedef Mixin_Call* Mixin_Call_Ptr;\n  typedef Mixin_Call const* Mixin_Call_Ptr_Const;\n  class Binary_Expression;\n  typedef Binary_Expression* Binary_Expression_Ptr;\n  typedef Binary_Expression const* Binary_Expression_Ptr_Const;\n  class Unary_Expression;\n  typedef Unary_Expression* Unary_Expression_Ptr;\n  typedef Unary_Expression const* Unary_Expression_Ptr_Const;\n  class Function_Call;\n  typedef Function_Call* Function_Call_Ptr;\n  typedef Function_Call const* Function_Call_Ptr_Const;\n  class Function_Call_Schema;\n  typedef Function_Call_Schema* Function_Call_Schema_Ptr;\n  typedef Function_Call_Schema const* Function_Call_Schema_Ptr_Const;\n  class Custom_Warning;\n  typedef Custom_Warning* Custom_Warning_Ptr;\n  typedef Custom_Warning const* Custom_Warning_Ptr_Const;\n  class Custom_Error;\n  typedef Custom_Error* Custom_Error_Ptr;\n  typedef Custom_Error const* Custom_Error_Ptr_Const;\n\n  class Variable;\n  typedef Variable* Variable_Ptr;\n  typedef Variable const* Variable_Ptr_Const;\n  class Number;\n  typedef Number* Number_Ptr;\n  typedef Number const* Number_Ptr_Const;\n  class Color;\n  typedef Color* Color_Ptr;\n  typedef Color const* Color_Ptr_Const;\n  class Boolean;\n  typedef Boolean* Boolean_Ptr;\n  typedef Boolean const* Boolean_Ptr_Const;\n  class String;\n  typedef String* String_Ptr;\n  typedef String const* String_Ptr_Const;\n\n  class String_Schema;\n  typedef String_Schema* String_Schema_Ptr;\n  typedef String_Schema const* String_Schema_Ptr_Const;\n  class String_Constant;\n  typedef String_Constant* String_Constant_Ptr;\n  typedef String_Constant const* String_Constant_Ptr_Const;\n  class String_Quoted;\n  typedef String_Quoted* String_Quoted_Ptr;\n  typedef String_Quoted const* String_Quoted_Ptr_Const;\n\n  class Media_Query;\n  typedef Media_Query* Media_Query_Ptr;\n  typedef Media_Query const* Media_Query_Ptr_Const;\n  class Media_Query_Expression;\n  typedef Media_Query_Expression* Media_Query_Expression_Ptr;\n  typedef Media_Query_Expression const* Media_Query_Expression_Ptr_Const;\n  class Supports_Condition;\n  typedef Supports_Condition* Supports_Condition_Ptr;\n  typedef Supports_Condition const* Supports_Condition_Ptr_Const;\n  class Supports_Operator;\n  typedef Supports_Operator* Supports_Operator_Ptr;\n  typedef Supports_Operator const* Supports_Operator_Ptr_Const;\n  class Supports_Negation;\n  typedef Supports_Negation* Supports_Negation_Ptr;\n  typedef Supports_Negation const* Supports_Negation_Ptr_Const;\n  class Supports_Declaration;\n  typedef Supports_Declaration* Supports_Declaration_Ptr;\n  typedef Supports_Declaration const* Supports_Declaration_Ptr_Const;\n  class Supports_Interpolation;\n  typedef Supports_Interpolation* Supports_Interpolation_Ptr;\n  typedef Supports_Interpolation const* Supports_Interpolation_Ptr_Const;\n\n\n  class Null;\n  typedef Null* Null_Ptr;\n  typedef Null const* Null_Ptr_Const;\n\n  class At_Root_Query;\n  typedef At_Root_Query* At_Root_Query_Ptr;\n  typedef At_Root_Query const* At_Root_Query_Ptr_Const;\n  class Parent_Selector;\n  typedef Parent_Selector* Parent_Selector_Ptr;\n  typedef Parent_Selector const* Parent_Selector_Ptr_Const;\n  class Parameter;\n  typedef Parameter* Parameter_Ptr;\n  typedef Parameter const* Parameter_Ptr_Const;\n  class Parameters;\n  typedef Parameters* Parameters_Ptr;\n  typedef Parameters const* Parameters_Ptr_Const;\n  class Argument;\n  typedef Argument* Argument_Ptr;\n  typedef Argument const* Argument_Ptr_Const;\n  class Arguments;\n  typedef Arguments* Arguments_Ptr;\n  typedef Arguments const* Arguments_Ptr_Const;\n  class Selector;\n  typedef Selector* Selector_Ptr;\n  typedef Selector const* Selector_Ptr_Const;\n\n\n  class Selector_Schema;\n  typedef Selector_Schema* Selector_Schema_Ptr;\n  typedef Selector_Schema const* Selector_Schema_Ptr_Const;\n  class Placeholder_Selector;\n  typedef Placeholder_Selector* Placeholder_Selector_Ptr;\n  typedef Placeholder_Selector const* Placeholder_Selector_Ptr_Const;\n  class Element_Selector;\n  typedef Element_Selector* Element_Selector_Ptr;\n  typedef Element_Selector const* Element_Selector_Ptr_Const;\n  class Class_Selector;\n  typedef Class_Selector* Class_Selector_Ptr;\n  typedef Class_Selector const* Class_Selector_Ptr_Const;\n  class Id_Selector;\n  typedef Id_Selector* Id_Selector_Ptr;\n  typedef Id_Selector const* Id_Selector_Ptr_Const;\n  class Attribute_Selector;\n  typedef Attribute_Selector* Attribute_Selector_Ptr;\n  typedef Attribute_Selector const* Attribute_Selector_Ptr_Const;\n\n  class Pseudo_Selector;\n  typedef Pseudo_Selector* Pseudo_Selector_Ptr;\n  typedef Pseudo_Selector const * Pseudo_Selector_Ptr_Const;\n  class Wrapped_Selector;\n  typedef Wrapped_Selector* Wrapped_Selector_Ptr;\n  typedef Wrapped_Selector const * Wrapped_Selector_Ptr_Const;\n  class Compound_Selector;\n  typedef Compound_Selector* Compound_Selector_Ptr;\n  typedef Compound_Selector const * Compound_Selector_Ptr_Const;\n  class Complex_Selector;\n  typedef Complex_Selector* Complex_Selector_Ptr;\n  typedef Complex_Selector const * Complex_Selector_Ptr_Const;\n  class Selector_List;\n  typedef Selector_List* Selector_List_Ptr;\n  typedef Selector_List const * Selector_List_Ptr_Const;\n\n\n  // common classes\n  class Context;\n  class Expand;\n  class Eval;\n\n  // declare classes that are instances of memory nodes\n  // #define IMPL_MEM_OBJ(type) using type##_Obj = SharedImpl<type>\n  #define IMPL_MEM_OBJ(type) typedef SharedImpl<type> type##_Obj\n\n  IMPL_MEM_OBJ(AST_Node);\n  IMPL_MEM_OBJ(Statement);\n  IMPL_MEM_OBJ(Block);\n  IMPL_MEM_OBJ(Ruleset);\n  IMPL_MEM_OBJ(Bubble);\n  IMPL_MEM_OBJ(Trace);\n  IMPL_MEM_OBJ(Media_Block);\n  IMPL_MEM_OBJ(Supports_Block);\n  IMPL_MEM_OBJ(Directive);\n  IMPL_MEM_OBJ(Keyframe_Rule);\n  IMPL_MEM_OBJ(At_Root_Block);\n  IMPL_MEM_OBJ(Declaration);\n  IMPL_MEM_OBJ(Assignment);\n  IMPL_MEM_OBJ(Import);\n  IMPL_MEM_OBJ(Import_Stub);\n  IMPL_MEM_OBJ(Warning);\n  IMPL_MEM_OBJ(Error);\n  IMPL_MEM_OBJ(Debug);\n  IMPL_MEM_OBJ(Comment);\n  IMPL_MEM_OBJ(PreValue);\n  IMPL_MEM_OBJ(Has_Block);\n  IMPL_MEM_OBJ(Thunk);\n  IMPL_MEM_OBJ(If);\n  IMPL_MEM_OBJ(For);\n  IMPL_MEM_OBJ(Each);\n  IMPL_MEM_OBJ(While);\n  IMPL_MEM_OBJ(Return);\n  IMPL_MEM_OBJ(Content);\n  IMPL_MEM_OBJ(Extension);\n  IMPL_MEM_OBJ(Definition);\n  IMPL_MEM_OBJ(Mixin_Call);\n  IMPL_MEM_OBJ(Value);\n  IMPL_MEM_OBJ(Expression);\n  IMPL_MEM_OBJ(List);\n  IMPL_MEM_OBJ(Map);\n  IMPL_MEM_OBJ(Function);\n  IMPL_MEM_OBJ(Binary_Expression);\n  IMPL_MEM_OBJ(Unary_Expression);\n  IMPL_MEM_OBJ(Function_Call);\n  IMPL_MEM_OBJ(Function_Call_Schema);\n  IMPL_MEM_OBJ(Custom_Warning);\n  IMPL_MEM_OBJ(Custom_Error);\n  IMPL_MEM_OBJ(Variable);\n  IMPL_MEM_OBJ(Number);\n  IMPL_MEM_OBJ(Color);\n  IMPL_MEM_OBJ(Boolean);\n  IMPL_MEM_OBJ(String_Schema);\n  IMPL_MEM_OBJ(String);\n  IMPL_MEM_OBJ(String_Constant);\n  IMPL_MEM_OBJ(String_Quoted);\n  IMPL_MEM_OBJ(Media_Query);\n  IMPL_MEM_OBJ(Media_Query_Expression);\n  IMPL_MEM_OBJ(Supports_Condition);\n  IMPL_MEM_OBJ(Supports_Operator);\n  IMPL_MEM_OBJ(Supports_Negation);\n  IMPL_MEM_OBJ(Supports_Declaration);\n  IMPL_MEM_OBJ(Supports_Interpolation);\n  IMPL_MEM_OBJ(At_Root_Query);\n  IMPL_MEM_OBJ(Null);\n  IMPL_MEM_OBJ(Parent_Selector);\n  IMPL_MEM_OBJ(Parameter);\n  IMPL_MEM_OBJ(Parameters);\n  IMPL_MEM_OBJ(Argument);\n  IMPL_MEM_OBJ(Arguments);\n  IMPL_MEM_OBJ(Selector);\n  IMPL_MEM_OBJ(Selector_Schema);\n  IMPL_MEM_OBJ(Simple_Selector);\n  IMPL_MEM_OBJ(Placeholder_Selector);\n  IMPL_MEM_OBJ(Element_Selector);\n  IMPL_MEM_OBJ(Class_Selector);\n  IMPL_MEM_OBJ(Id_Selector);\n  IMPL_MEM_OBJ(Attribute_Selector);\n  IMPL_MEM_OBJ(Pseudo_Selector);\n  IMPL_MEM_OBJ(Wrapped_Selector);\n  IMPL_MEM_OBJ(Compound_Selector);\n  IMPL_MEM_OBJ(Complex_Selector);\n  IMPL_MEM_OBJ(Selector_List);\n\n  // ###########################################################################\n  // Implement compare, order and hashing operations for AST Nodes\n  // ###########################################################################\n\n  struct HashNodes {\n    template <class T>\n    size_t operator() (const T& ex) const {\n      return ex.isNull() ? 0 : ex->hash();\n    }\n  };\n  struct OrderNodes {\n    template <class T>\n    bool operator() (const T& lhs, const T& rhs) const {\n      return !lhs.isNull() && !rhs.isNull() && *lhs < *rhs;\n    }\n  };\n  struct CompareNodes {\n    template <class T>\n    bool operator() (const T& lhs, const T& rhs) const {\n      // code around sass logic issue. 1px == 1 is true\n      // but both items are still different keys in maps\n      if (dynamic_cast<Number*>(lhs.ptr()))\n        if (dynamic_cast<Number*>(rhs.ptr()))\n          return lhs->hash() == rhs->hash();\n      return !lhs.isNull() && !rhs.isNull() && *lhs == *rhs;\n    }\n  };\n\n  // ###########################################################################\n  // some often used typedefs\n  // ###########################################################################\n\n  typedef std::unordered_map<\n    Expression_Obj, // key\n    Expression_Obj, // value\n    HashNodes, // hasher\n    CompareNodes // compare\n  > ExpressionMap;\n  typedef std::unordered_set<\n    Expression_Obj, // value\n    HashNodes, // hasher\n    CompareNodes // compare\n  > ExpressionSet;\n\n  typedef std::string SubSetMapKey;\n  typedef std::vector<std::string> SubSetMapKeys;\n\n  typedef std::pair<Complex_Selector_Obj, Compound_Selector_Obj> SubSetMapPair;\n  typedef std::pair<Compound_Selector_Obj, Complex_Selector_Obj> SubSetMapLookup;\n  typedef std::vector<SubSetMapPair> SubSetMapPairs;\n  typedef std::vector<SubSetMapLookup> SubSetMapLookups;\n\n  typedef std::pair<Complex_Selector_Obj, SubSetMapPairs> SubSetMapResult;\n  typedef std::vector<SubSetMapResult> SubSetMapResults;\n\n  typedef std::deque<Complex_Selector_Obj> ComplexSelectorDeque;\n  typedef std::set<Simple_Selector_Obj, OrderNodes> SimpleSelectorSet;\n  typedef std::set<Complex_Selector_Obj, OrderNodes> ComplexSelectorSet;\n  typedef std::set<Compound_Selector_Obj, OrderNodes> CompoundSelectorSet;\n  typedef std::unordered_set<Simple_Selector_Obj, HashNodes, CompareNodes> SimpleSelectorDict;\n\n  typedef std::vector<Sass_Import_Entry>* ImporterStack;\n\n  // only to switch implementations for testing\n  #define environment_map std::map\n\n  // ###########################################################################\n  // explicit type conversion functions\n  // ###########################################################################\n\n  template<class T>\n  T* Cast(AST_Node* ptr);\n\n  template<class T>\n  const T* Cast(const AST_Node* ptr);\n\n  // sometimes you know the class you want to cast to is final\n  // in this case a simple typeid check is faster and safe to use\n\n  #define DECLARE_BASE_CAST(T) \\\n  template<> T* Cast(AST_Node* ptr); \\\n  template<> const T* Cast(const AST_Node* ptr); \\\n\n  // ###########################################################################\n  // implement specialization for final classes\n  // ###########################################################################\n\n  DECLARE_BASE_CAST(AST_Node)\n  DECLARE_BASE_CAST(Expression)\n  DECLARE_BASE_CAST(Statement)\n  DECLARE_BASE_CAST(Has_Block)\n  DECLARE_BASE_CAST(PreValue)\n  DECLARE_BASE_CAST(Value)\n  DECLARE_BASE_CAST(List)\n  DECLARE_BASE_CAST(String)\n  DECLARE_BASE_CAST(String_Constant)\n  DECLARE_BASE_CAST(Supports_Condition)\n  DECLARE_BASE_CAST(Selector)\n  DECLARE_BASE_CAST(Simple_Selector)\n\n}\n\n#endif\n\n// Path: src/libsass/src/util.hpp\n#ifndef SASS_UTIL_H\n#define SASS_UTIL_H\n\n#include <vector>\n#include <string>\n#include <assert.h>\n#include \"sass.hpp\"\n#include \"sass/base.h\"\n#include \"ast_fwd_decl.hpp\"\n\n#define SASS_ASSERT(cond, msg) assert(cond && msg)\n\nnamespace Sass {\n\n  double round(double val, size_t precision = 0);\n  double sass_strtod(const char* str);\n  const char* safe_str(const char *, const char* = \"\");\n  void free_string_array(char **);\n  char **copy_strings(const std::vector<std::string>&, char ***, int = 0);\n  std::string read_css_string(const std::string& str, bool css = true);\n  std::string evacuate_escapes(const std::string& str);\n  std::string string_to_output(const std::string& str);\n  std::string comment_to_string(const std::string& text);\n  std::string read_hex_escapes(const std::string& str);\n  std::string escape_string(const std::string& str);\n  void newline_to_space(std::string& str);\n\n  std::string quote(const std::string&, char q = 0);\n  std::string unquote(const std::string&, char* q = 0, bool keep_utf8_sequences = false, bool strict = true);\n  char detect_best_quotemark(const char* s, char qm = '\"');\n\n  bool is_hex_doublet(double n);\n  bool is_color_doublet(double r, double g, double b);\n\n  bool peek_linefeed(const char* start);\n\n  namespace Util {\n\n    std::string rtrim(const std::string& str);\n\n    std::string normalize_underscores(const std::string& str);\n    std::string normalize_decimals(const std::string& str);\n\n    bool isPrintable(Ruleset_Ptr r, Sass_Output_Style style = NESTED);\n    bool isPrintable(Supports_Block_Ptr r, Sass_Output_Style style = NESTED);\n    bool isPrintable(Media_Block_Ptr r, Sass_Output_Style style = NESTED);\n    bool isPrintable(Comment_Ptr b, Sass_Output_Style style = NESTED);\n    bool isPrintable(Block_Obj b, Sass_Output_Style style = NESTED);\n    bool isPrintable(String_Constant_Ptr s, Sass_Output_Style style = NESTED);\n    bool isPrintable(String_Quoted_Ptr s, Sass_Output_Style style = NESTED);\n    bool isPrintable(Declaration_Ptr d, Sass_Output_Style style = NESTED);\n    bool isAscii(const char chr);\n\n  }\n}\n#endif\n\n// Path: src/libsass/src/units.hpp\n#ifndef SASS_UNITS_H\n#define SASS_UNITS_H\n\n#include <cmath>\n#include <string>\n#include <sstream>\n#include <vector>\n\nnamespace Sass {\n\n  const double PI = std::acos(-1);\n\n  enum UnitClass {\n    LENGTH = 0x000,\n    ANGLE = 0x100,\n    TIME = 0x200,\n    FREQUENCY = 0x300,\n    RESOLUTION = 0x400,\n    INCOMMENSURABLE = 0x500\n  };\n\n  enum UnitType {\n\n    // size units\n    IN = UnitClass::LENGTH,\n    CM,\n    PC,\n    MM,\n    PT,\n    PX,\n\n    // angle units\n    DEG = ANGLE,\n    GRAD,\n    RAD,\n    TURN,\n\n    // time units\n    SEC = TIME,\n    MSEC,\n\n    // frequency units\n    HERTZ = FREQUENCY,\n    KHERTZ,\n\n    // resolutions units\n    DPI = RESOLUTION,\n    DPCM,\n    DPPX,\n\n    // for unknown units\n    UNKNOWN = INCOMMENSURABLE\n\n  };\n\n  class Units {\n  public:\n    std::vector<std::string> numerators;\n    std::vector<std::string> denominators;\n  public:\n    // default constructor\n    Units() :\n      numerators(),\n      denominators()\n    { }\n    // copy constructor\n    Units(const Units* ptr) :\n      numerators(ptr->numerators),\n      denominators(ptr->denominators)\n    { }\n    // convert to string\n    std::string unit() const;\n    // get if units are empty\n    bool is_unitless() const;\n    // return if valid for css\n    bool is_valid_css_unit() const;\n    // reduce units for output\n    // returns conversion factor\n    double reduce();\n    // normalize units for compare\n    // returns conversion factor\n    double normalize();\n    // compare operations\n    bool operator< (const Units& rhs) const;\n    bool operator== (const Units& rhs) const;\n    // factor to convert into given units\n    double convert_factor(const Units&) const;\n  };\n\n  extern const double size_conversion_factors[6][6];\n  extern const double angle_conversion_factors[4][4];\n  extern const double time_conversion_factors[2][2];\n  extern const double frequency_conversion_factors[2][2];\n  extern const double resolution_conversion_factors[3][3];\n\n  UnitType get_main_unit(const UnitClass unit);\n  enum Sass::UnitType string_to_unit(const std::string&);\n  const char* unit_to_string(Sass::UnitType unit);\n  enum Sass::UnitClass get_unit_type(Sass::UnitType unit);\n  std::string get_unit_class(Sass::UnitType unit);\n  std::string unit_to_class(const std::string&);\n  // throws incompatibleUnits exceptions\n  double conversion_factor(const std::string&, const std::string&);\n  double conversion_factor(UnitType, UnitType, UnitClass, UnitClass);\n  double convert_units(const std::string&, const std::string&, int&, int&);\n\n}\n\n#endif\n\n// Path: src/libsass/src/b64/cencode.h\n/*\ncencode.h - c header for a base64 encoding algorithm\n\nThis is part of the libb64 project, and has been placed in the public domain.\nFor details, see http://sourceforge.net/projects/libb64\n*/\n\n#ifndef BASE64_CENCODE_H\n#define BASE64_CENCODE_H\n\ntypedef enum\n{\n  step_A, step_B, step_C\n} base64_encodestep;\n\ntypedef struct\n{\n\tbase64_encodestep step;\n\tchar result;\n\tint stepcount;\n} base64_encodestate;\n\nvoid base64_init_encodestate(base64_encodestate* state_in);\n\nchar base64_encode_value(char value_in);\n\nint base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in);\n\nint base64_encode_blockend(char* code_out, base64_encodestate* state_in);\n\n#endif /* BASE64_CENCODE_H */\n\n\n// Path: src/libsass/src/b64/encode.h\n// :mode=c++:\n/*\nencode.h - c++ wrapper for a base64 encoding algorithm\n\nThis is part of the libb64 project, and has been placed in the public domain.\nFor details, see http://sourceforge.net/projects/libb64\n*/\n#ifndef BASE64_ENCODE_H\n#define BASE64_ENCODE_H\n\n#include <iostream>\n\nnamespace base64\n{\n\textern \"C\"\n\t{\n\t\t#include \"cencode.h\"\n\t}\n\n\tstruct encoder\n\t{\n\t\tbase64_encodestate _state;\n\t\tint _buffersize;\n\n\t\tencoder(int buffersize_in = BUFFERSIZE)\n\t\t: _buffersize(buffersize_in)\n\t\t{\n\t\t\tbase64_init_encodestate(&_state);\n\t\t}\n\n\t\tint encode(char value_in)\n\t\t{\n\t\t\treturn base64_encode_value(value_in);\n\t\t}\n\n\t\tint encode(const char* code_in, const int length_in, char* plaintext_out)\n\t\t{\n\t\t\treturn base64_encode_block(code_in, length_in, plaintext_out, &_state);\n\t\t}\n\n\t\tint encode_end(char* plaintext_out)\n\t\t{\n\t\t\treturn base64_encode_blockend(plaintext_out, &_state);\n\t\t}\n\n\t\tvoid encode(std::istream& istream_in, std::ostream& ostream_in)\n\t\t{\n\t\t\tbase64_init_encodestate(&_state);\n\t\t\t//\n\t\t\tconst int N = _buffersize;\n\t\t\tchar* plaintext = new char[N];\n\t\t\tchar* code = new char[2*N];\n\t\t\tint plainlength;\n\t\t\tint codelength;\n\n\t\t\tdo\n\t\t\t{\n\t\t\t\tistream_in.read(plaintext, N);\n\t\t\t\tplainlength = static_cast<int>(istream_in.gcount());\n\t\t\t\t//\n\t\t\t\tcodelength = encode(plaintext, plainlength, code);\n\t\t\t\tostream_in.write(code, codelength);\n\t\t\t}\n\t\t\twhile (istream_in.good() && plainlength > 0);\n\n\t\t\tcodelength = encode_end(code);\n\t\t\tostream_in.write(code, codelength);\n\t\t\t//\n\t\t\tbase64_init_encodestate(&_state);\n\n\t\t\tdelete [] code;\n\t\t\tdelete [] plaintext;\n\t\t}\n\t};\n\n} // namespace base64\n\n#endif // BASE64_ENCODE_H\n\n\n// Path: src/libsass/src/kwd_arg_macros.hpp\n#ifndef SASS_KWD_ARG_MACROS_H\n#define SASS_KWD_ARG_MACROS_H\n\n// Example usage:\n// KWD_ARG_SET(Args) {\n//   KWD_ARG(Args, string, foo);\n//   KWD_ARG(Args, int, bar);\n//   ...\n// };\n//\n// ... and later ...\n//\n// something(Args().foo(\"hey\").bar(3));\n\n#define KWD_ARG_SET(set_name) class set_name\n\n#define KWD_ARG(set_name, type, name) \\\nprivate: \\\n  type name##_; \\\npublic: \\\n  set_name& name(type name##__) { \\\n    name##_ = name##__; \\\n    return *this; \\\n  } \\\n  type name() { return name##_; } \\\nprivate:\n\n#endif\n\n// Path: src/libsass/src/sass_context.hpp\n#ifndef SASS_SASS_CONTEXT_H\n#define SASS_SASS_CONTEXT_H\n\n#include \"sass/base.h\"\n#include \"sass/context.h\"\n#include \"ast_fwd_decl.hpp\"\n\n// sass config options structure\nstruct Sass_Options : Sass_Output_Options {\n\n  // embed sourceMappingUrl as data uri\n  bool source_map_embed;\n\n  // embed include contents in maps\n  bool source_map_contents;\n\n  // create file urls for sources\n  bool source_map_file_urls;\n\n  // Disable sourceMappingUrl in css output\n  bool omit_source_map_url;\n\n  // Treat source_string as sass (as opposed to scss)\n  bool is_indented_syntax_src;\n\n  // The input path is used for source map\n  // generation. It can be used to define\n  // something with string compilation or to\n  // overload the input file path. It is\n  // set to \"stdin\" for data contexts and\n  // to the input file on file contexts.\n  char* input_path;\n\n  // The output path is used for source map\n  // generation. LibSass will not write to\n  // this file, it is just used to create\n  // information in source-maps etc.\n  char* output_path;\n\n  // Colon-separated list of paths\n  // Semicolon-separated on Windows\n  // Maybe use array interface instead?\n  char* include_path;\n  char* plugin_path;\n\n  // Include paths (linked string list)\n  struct string_list* include_paths;\n  // Plugin paths (linked string list)\n  struct string_list* plugin_paths;\n\n  // Path to source map file\n  // Enables source map generation\n  // Used to create sourceMappingUrl\n  char* source_map_file;\n\n  // Directly inserted in source maps\n  char* source_map_root;\n\n  // Custom functions that can be called from sccs code\n  Sass_Function_List c_functions;\n\n  // List of custom importers\n  Sass_Importer_List c_importers;\n\n  // List of custom headers\n  Sass_Importer_List c_headers;\n\n};\n\n\n// base for all contexts\nstruct Sass_Context : Sass_Options\n{\n\n  // store context type info\n  enum Sass_Input_Style type;\n\n  // generated output data\n  char* output_string;\n\n  // generated source map json\n  char* source_map_string;\n\n  // error status\n  int error_status;\n  char* error_json;\n  char* error_text;\n  char* error_message;\n  // error position\n  char* error_file;\n  size_t error_line;\n  size_t error_column;\n  const char* error_src;\n\n  // report imported files\n  char** included_files;\n\n};\n\n// struct for file compilation\nstruct Sass_File_Context : Sass_Context {\n\n  // no additional fields required\n  // input_path is already on options\n\n};\n\n// struct for data compilation\nstruct Sass_Data_Context : Sass_Context {\n\n  // provided source string\n  char* source_string;\n  char* srcmap_string;\n\n};\n\n// link c and cpp context\nstruct Sass_Compiler {\n  // progress status\n  Sass_Compiler_State state;\n  // original c context\n  Sass_Context* c_ctx;\n  // Sass::Context\n  Sass::Context* cpp_ctx;\n  // Sass::Block\n  Sass::Block_Obj root;\n};\n\n#endif\n// Path: src/libsass/src/ast_def_macros.hpp\n#ifndef SASS_AST_DEF_MACROS_H\n#define SASS_AST_DEF_MACROS_H\n\n// Helper class to switch a flag and revert once we go out of scope\ntemplate <class T>\nclass LocalOption {\n  private:\n    T* var; // pointer to original variable\n    T orig; // copy of the original option\n  public:\n    LocalOption(T& var)\n    {\n      this->var = &var;\n      this->orig = var;\n    }\n    LocalOption(T& var, T orig)\n    {\n      this->var = &var;\n      this->orig = var;\n      *(this->var) = orig;\n    }\n    void reset()\n    {\n      *(this->var) = this->orig;\n    }\n    ~LocalOption() {\n      *(this->var) = this->orig;\n    }\n};\n\n#define LOCAL_FLAG(name,opt) LocalOption<bool> flag_##name(name, opt)\n#define LOCAL_COUNT(name,opt) LocalOption<size_t> cnt_##name(name, opt)\n\n#define NESTING_GUARD(name) \\\n  LocalOption<size_t> cnt_##name(name, name + 1); \\\n  if (name > MAX_NESTING) throw Exception::NestingLimitError(pstate, traces); \\\n\n#define ATTACH_OPERATIONS()\\\nvirtual void perform(Operation<void>* op) { (*op)(this); }\\\nvirtual AST_Node_Ptr perform(Operation<AST_Node_Ptr>* op) { return (*op)(this); }\\\nvirtual Statement_Ptr perform(Operation<Statement_Ptr>* op) { return (*op)(this); }\\\nvirtual Expression_Ptr perform(Operation<Expression_Ptr>* op) { return (*op)(this); }\\\nvirtual Selector_Ptr perform(Operation<Selector_Ptr>* op) { return (*op)(this); }\\\nvirtual std::string perform(Operation<std::string>* op) { return (*op)(this); }\\\nvirtual union Sass_Value* perform(Operation<union Sass_Value*>* op) { return (*op)(this); }\\\nvirtual Value_Ptr perform(Operation<Value_Ptr>* op) { return (*op)(this); }\n\n#define ADD_PROPERTY(type, name)\\\nprotected:\\\n  type name##_;\\\npublic:\\\n  type name() const        { return name##_; }\\\n  type name(type name##__) { return name##_ = name##__; }\\\nprivate:\n\n#define HASH_PROPERTY(type, name)\\\nprotected:\\\n  type name##_;\\\npublic:\\\n  type name() const        { return name##_; }\\\n  type name(type name##__) { hash_ = 0; return name##_ = name##__; }\\\nprivate:\n\n#define ADD_CONSTREF(type, name) \\\nprotected: \\\n  type name##_; \\\npublic: \\\n  const type& name() const { return name##_; } \\\n  void name(type name##__) { name##_ = name##__; } \\\nprivate:\n\n#define HASH_CONSTREF(type, name) \\\nprotected: \\\n  type name##_; \\\npublic: \\\n  const type& name() const { return name##_; } \\\n  void name(type name##__) { hash_ = 0; name##_ = name##__; } \\\nprivate:\n\n#endif\n\n// Path: src/libsass/src/environment.hpp\n#ifndef SASS_ENVIRONMENT_H\n#define SASS_ENVIRONMENT_H\n\n#include <string>\n#include \"ast_fwd_decl.hpp\"\n#include \"ast_def_macros.hpp\"\n\nnamespace Sass {\n\n  typedef environment_map<std::string, AST_Node_Obj>::iterator EnvIter;\n\n  class EnvResult {\n    public:\n      EnvIter it;\n      bool found;\n    public:\n      EnvResult(EnvIter it, bool found)\n      : it(it), found(found) {}\n  };\n\n  template <typename T>\n  class Environment {\n    // TODO: test with map\n    environment_map<std::string, T> local_frame_;\n    ADD_PROPERTY(Environment*, parent)\n    ADD_PROPERTY(bool, is_shadow)\n\n  public:\n    Environment(bool is_shadow = false);\n    Environment(Environment* env, bool is_shadow = false);\n    Environment(Environment& env, bool is_shadow = false);\n\n    // link parent to create a stack\n    void link(Environment& env);\n    void link(Environment* env);\n\n    // this is used to find the global frame\n    // which is the second last on the stack\n    bool is_lexical() const;\n\n    // only match the real root scope\n    // there is still a parent around\n    // not sure what it is actually use for\n    // I guess we store functions etc. there\n    bool is_global() const;\n\n    // scope operates on the current frame\n\n    environment_map<std::string, T>& local_frame();\n\n    bool has_local(const std::string& key) const;\n\n    EnvResult find_local(const std::string& key);\n\n    T& get_local(const std::string& key);\n\n    // set variable on the current frame\n    void set_local(const std::string& key, const T& val);\n    void set_local(const std::string& key, T&& val);\n\n    void del_local(const std::string& key);\n\n    // global operates on the global frame\n    // which is the second last on the stack\n    Environment* global_env();\n    // get the env where the variable already exists\n    // if it does not yet exist, we return current env\n    Environment* lexical_env(const std::string& key);\n\n    bool has_global(const std::string& key);\n\n    T& get_global(const std::string& key);\n\n    // set a variable on the global frame\n    void set_global(const std::string& key, const T& val);\n    void set_global(const std::string& key, T&& val);\n\n    void del_global(const std::string& key);\n\n    // see if we have a lexical variable\n    // move down the stack but stop before we\n    // reach the global frame (is not included)\n    bool has_lexical(const std::string& key) const;\n\n    // see if we have a lexical we could update\n    // either update already existing lexical value\n    // or we create a new one on the current frame\n    void set_lexical(const std::string& key, T&& val);\n    void set_lexical(const std::string& key, const T& val);\n\n    // look on the full stack for key\n    // include all scopes available\n    bool has(const std::string& key) const;\n\n    // look on the full stack for key\n    // include all scopes available\n    EnvResult find(const std::string& key);\n\n    // use array access for getter and setter functions\n    T& operator[](const std::string& key);\n\n    #ifdef DEBUG\n    size_t print(std::string prefix = \"\");\n    #endif\n\n  };\n\n  // define typedef for our use case\n  typedef Environment<AST_Node_Obj> Env;\n\n}\n\n#endif\n\n// Path: src/libsass/src/base64vlq.hpp\n#ifndef SASS_BASE64VLQ_H\n#define SASS_BASE64VLQ_H\n\n#include <string>\n\nnamespace Sass {\n\n  class Base64VLQ {\n\n  public:\n\n    std::string encode(const int number) const;\n\n  private:\n\n    char base64_encode(const int number) const;\n\n    int to_vlq_signed(const int number) const;\n\n    static const char* CHARACTERS;\n\n    static const int VLQ_BASE_SHIFT;\n    static const int VLQ_BASE;\n    static const int VLQ_BASE_MASK;\n    static const int VLQ_CONTINUATION_BIT;\n  };\n\n}\n\n#endif\n\n// Path: src/libsass/src/position.hpp\n#ifndef SASS_POSITION_H\n#define SASS_POSITION_H\n\n#include <string>\n#include <cstring>\n// #include <iostream>\n\nnamespace Sass {\n\n\n  class Offset {\n\n    public: // c-tor\n      Offset(const char chr);\n      Offset(const char* string);\n      Offset(const std::string& text);\n      Offset(const size_t line, const size_t column);\n\n      // return new position, incremented by the given string\n      Offset add(const char* begin, const char* end);\n      Offset inc(const char* begin, const char* end) const;\n\n      // init/create instance from const char substring\n      static Offset init(const char* beg, const char* end);\n\n    public: // overload operators for position\n      void operator+= (const Offset &pos);\n      bool operator== (const Offset &pos) const;\n      bool operator!= (const Offset &pos) const;\n      Offset operator+ (const Offset &off) const;\n      Offset operator- (const Offset &off) const;\n\n    public: // overload output stream operator\n      // friend std::ostream& operator<<(std::ostream& strm, const Offset& off);\n\n    public:\n      Offset off() { return *this; }\n\n    public:\n      size_t line;\n      size_t column;\n\n  };\n\n  class Position : public Offset {\n\n    public: // c-tor\n      Position(const size_t file); // line(0), column(0)\n      Position(const size_t file, const Offset& offset);\n      Position(const size_t line, const size_t column); // file(-1)\n      Position(const size_t file, const size_t line, const size_t column);\n\n    public: // overload operators for position\n      void operator+= (const Offset &off);\n      bool operator== (const Position &pos) const;\n      bool operator!= (const Position &pos) const;\n      const Position operator+ (const Offset &off) const;\n      const Offset operator- (const Offset &off) const;\n      // return new position, incremented by the given string\n      Position add(const char* begin, const char* end);\n      Position inc(const char* begin, const char* end) const;\n\n    public: // overload output stream operator\n      // friend std::ostream& operator<<(std::ostream& strm, const Position& pos);\n\n    public:\n      size_t file;\n\n  };\n\n  // Token type for representing lexed chunks of text\n  class Token {\n  public:\n    const char* prefix;\n    const char* begin;\n    const char* end;\n\n    Token()\n...\n", "needle_token_start": 2364, "needle_token_end": 2429, "code_context_ntokens": 16025, "output": ["```cpp\n// Path: src/libsass/src/sass.hpp\n// must be the first include in all compile units\n#ifndef SASS_SASS_H\n#define SASS_SASS_H\n\n// undefine extensions macro to tell sys includes\n// that we do not want any macros to be exported\n// mainly fixes an issue on SmartOS (SEC macro)\n#undef __EXTENSIONS__\n\n#ifdef _MSC_VER\n#pragma warning(disable : 4005)\n#endif\n\n// aplies to MSVC and MinGW\n#ifdef _WIN32\n// we do not want the ERROR macro\n# define NOGDI\n// we do not want the min/max macro\n# define NOMINMAX\n// we do not want the IN/OUT macro\n# define _NO_W32_PSEUDO_MODIFIERS\n#endif\n\n\n// should we be case insensitive\n// when dealing with files or paths\n#ifndef FS_CASE_SENSITIVE\n# ifdef _WIN32\n#  define FS_CASE_SENSITIVE 0\n# else\n#  define FS_CASE_SENSITIVE 1\n# endif\n#endif\n\n// path separation char\n#ifndef PATH_SEP\n# ifdef _WIN32\n#  define PATH_SEP ';'\n# else\n#  define PATH_SEP ':'\n# endif\n#endif\n\n\n// include C-API header\n#include \"sass/base.h\"\n\n// For C++ helper\n#include <string>\n\n// output behaviours\nnamespace Sass {\n\n  // create some C++ aliases for the most used options\n  const static Sass_Output_Style NESTED = SASS_STYLE_NESTED;\n  const static Sass_Output_Style COMPACT = SASS_STYLE_COMPACT;\n  const static Sass_Output_Style EXPANDED = SASS_STYLE_EXPANDED;\n  const static Sass_Output_Style COMPRESSED = SASS_STYLE_COMPRESSED;\n  // only used internal to trigger ruby inspect behavior\n  const static Sass_Output_Style INSPECT = SASS_STYLE_INSPECT;\n  const static Sass_Output_Style TO_SASS = SASS_STYLE_TO_SASS;\n\n  // helper to aid dreaded MSVC debug mode\n  // see implementation for more details\n  char* sass_copy_string(std::string str);\n\n}\n\n// input behaviours\nenum Sass_Input_Style {\n  SASS_CONTEXT_NULL,\n  SASS_CONTEXT_FILE,\n  SASS_CONTEXT_DATA,\n  SASS_CONTEXT_FOLDER\n};\n\n// simple linked list\nstruct string_list {\n  string_list* next;\n  char* string;\n};\n\n// sass config options structure\nstruct Sass_Inspect_Options {\n\n  // Output style for the generated css code\n  // A value from above SASS_STYLE_* constants\n  enum Sass_Output_Style output_style;\n\n  // Precision for fractional numbers\n  int precision;\n\n  // Do not compress colors in selectors\n  bool in_selector;\n\n  // initialization list (constructor with defaults)\n  \nSass_Inspect_Options(Sass_Output_Style style = Sass::NESTED,\n                       int precision = 5, bool in_selector = false)\n  : output_style(style), precision(precision), in_selector(in_selector)\n  { }\n\n};\n\n// sass config options structure\nstruct Sass_Output_Options : Sass_Inspect_Options {\n\n  // String to be used for indentation\n  const char* indent;\n  // String to be used to for line feeds\n  const char* linefeed;\n\n  // Emit comments in the generated CSS indicating\n  // the corresponding source line.\n  bool source_comments;\n\n  // initialization list (constructor with defaults)\n  Sass_Output_Options(struct Sass_Inspect_Options opt,\n                      const char* indent = \"  \",\n                      const char* linefeed = \"\\n\",\n                      bool source_comments = false)\n  : Sass_Inspect_Options(opt),\n    indent(indent), linefeed(linefeed),\n    source_comments(source_comments)\n  { }\n\n  // initialization list (constructor with defaults)\n  Sass_Output_Options(Sass_Output_Style style = Sass::NESTED,\n                      int precision = 5,\n                      const char* indent = \"  \",\n                      const char* linefeed = \"\\n\",\n                      bool source_comments = false)\n  : Sass_Inspect_Options(style, precision),\n    indent(indent), linefeed(linefeed),\n    source_comments(source_comments)\n  { }\n\n};\n\n#endif\n```"]}
{"repo": "sass/node-sass", "name": "Thunk", "language": "cpp", "path": "src/libsass/src/ast.hpp", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this class is to facilitate delayed evaluation of expressions within a specific environment. This is useful in scenarios where the evaluation needs to be deferred until a certain condition or context is met.\n2. **Input**: It takes an expression object and an optional environment pointer. The expression object represents the computation or value to be evaluated later, and the environment pointer refers to the scope or context in which this expression should be evaluated.\n3. **Output**: There is no direct output since this class primarily handles the storage of an expression and its evaluation context. The actual output would be generated when the stored expression is eventually evaluated within its stored environment.\n4. **Procedure**: Upon instantiation, the class stores the provided expression and environment. The evaluation of the expression is deferred until explicitly invoked, at which point the expression is evaluated in the context of the stored environment, ensuring that the correct scope and conditions are applied.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n    std::string type() const { return \"string\"; }\n    static std::string type_name() { return \"string\"; }\n\n    bool is_left_interpolant(void) const;\n    bool is_right_interpolant(void) const;\n    // void has_interpolants(bool tc) { }\n    bool has_interpolants() {\n      for (auto el : elements()) {\n        if (el->is_interpolant()) return true;\n      }\n      return false;\n    }\n    virtual void rtrim();\n\n    virtual size_t hash()\n    {\n      if (hash_ == 0) {\n        for (auto string : elements())\n          hash_combine(hash_, string->hash());\n      }\n      return hash_;\n    }\n\n    virtual void set_delayed(bool delayed) {\n      is_delayed(delayed);\n    }\n\n    virtual bool operator==(const Expression& rhs) const;\n    ATTACH_AST_OPERATIONS(String_Schema)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////////////\n  // Flat strings -- the lowest level of raw textual data.\n  ////////////////////////////////////////////////////////\n  class String_Constant : public String {\n    ADD_PROPERTY(char, quote_mark)\n    ADD_PROPERTY(bool, can_compress_whitespace)\n    HASH_CONSTREF(std::string, value)\n  protected:\n    size_t hash_;\n  public:\n    String_Constant(const String_Constant* ptr)\n    : String(ptr),\n      quote_mark_(ptr->quote_mark_),\n      can_compress_whitespace_(ptr->can_compress_whitespace_),\n      value_(ptr->value_),\n      hash_(ptr->hash_)\n    { }\n    String_Constant(ParserState pstate, std::string val, bool css = true)\n    : String(pstate), quote_mark_(0), can_compress_whitespace_(false), value_(read_css_string(val, css)), hash_(0)\n    { }\n    String_Constant(ParserState pstate, const char* beg, bool css = true)\n    : String(pstate), quote_mark_(0), can_compress_whitespace_(false), value_(read_css_string(std::string(beg), css)), hash_(0)\n    { }\n    String_Constant(ParserState pstate, const char* beg, const char* end, bool css = true)\n    : String(pstate), quote_mark_(0), can_compress_whitespace_(false), value_(read_css_string(std::string(beg, end-beg), css)), hash_(0)\n    { }\n    String_Constant(ParserState pstate, const Token& tok, bool css = true)\n    : String(pstate), quote_mark_(0), can_compress_whitespace_(false), value_(read_css_string(std::string(tok.begin, tok.end), css)), hash_(0)\n    { }\n    std::string type() const { return \"string\"; }\n    static std::string type_name() { return \"string\"; }\n    virtual bool is_invisible() const;\n    virtual void rtrim();\n\n    virtual size_t hash()\n    {\n      if (hash_ == 0) {\n        hash_ = std::hash<std::string>()(value_);\n      }\n      return hash_;\n    }\n\n    virtual bool operator==(const Expression& rhs) const;\n    virtual std::string inspect() const; // quotes are forced on inspection\n\n    // static char auto_quote() { return '*'; }\n    static char double_quote() { return '\"'; }\n    static char single_quote() { return '\\''; }\n\n    ATTACH_AST_OPERATIONS(String_Constant)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////////////\n  // Possibly quoted string (unquote on instantiation)\n  ////////////////////////////////////////////////////////\n  class String_Quoted : public String_Constant {\n  public:\n    String_Quoted(ParserState pstate, std::string val, char q = 0,\n      bool keep_utf8_escapes = false, bool skip_unquoting = false,\n      bool strict_unquoting = true, bool css = true)\n    : String_Constant(pstate, val, css)\n    {\n      if (skip_unquoting == false) {\n        value_ = unquote(value_, &quote_mark_, keep_utf8_escapes, strict_unquoting);\n      }\n      if (q && quote_mark_) quote_mark_ = q;\n    }\n    String_Quoted(const String_Quoted* ptr)\n    : String_Constant(ptr)\n    { }\n    virtual bool operator==(const Expression& rhs) const;\n    virtual std::string inspect() const; // quotes are forced on inspection\n    ATTACH_AST_OPERATIONS(String_Quoted)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////\n  // Media queries.\n  /////////////////\n  class Media_Query : public Expression,\n                      public Vectorized<Media_Query_Expression_Obj> {\n    ADD_PROPERTY(String_Obj, media_type)\n    ADD_PROPERTY(bool, is_negated)\n    ADD_PROPERTY(bool, is_restricted)\n  public:\n    Media_Query(ParserState pstate,\n                String_Obj t = 0, size_t s = 0, bool n = false, bool r = false)\n    : Expression(pstate), Vectorized<Media_Query_Expression_Obj>(s),\n      media_type_(t), is_negated_(n), is_restricted_(r)\n    { }\n    Media_Query(const Media_Query* ptr)\n    : Expression(ptr),\n      Vectorized<Media_Query_Expression_Obj>(*ptr),\n      media_type_(ptr->media_type_),\n      is_negated_(ptr->is_negated_),\n      is_restricted_(ptr->is_restricted_)\n    { }\n    ATTACH_AST_OPERATIONS(Media_Query)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////////\n  // Media expressions (for use inside media queries).\n  ////////////////////////////////////////////////////\n  class Media_Query_Expression : public Expression {\n    ADD_PROPERTY(Expression_Obj, feature)\n    ADD_PROPERTY(Expression_Obj, value)\n    ADD_PROPERTY(bool, is_interpolated)\n  public:\n    Media_Query_Expression(ParserState pstate,\n                           Expression_Obj f, Expression_Obj v, bool i = false)\n    : Expression(pstate), feature_(f), value_(v), is_interpolated_(i)\n    { }\n    Media_Query_Expression(const Media_Query_Expression* ptr)\n    : Expression(ptr),\n      feature_(ptr->feature_),\n      value_(ptr->value_),\n      is_interpolated_(ptr->is_interpolated_)\n    { }\n    ATTACH_AST_OPERATIONS(Media_Query_Expression)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////\n  // `@supports` rule.\n  ////////////////////\n  class Supports_Block : public Has_Block {\n    ADD_PROPERTY(Supports_Condition_Obj, condition)\n  public:\n    Supports_Block(ParserState pstate, Supports_Condition_Obj condition, Block_Obj block = 0)\n    : Has_Block(pstate, block), condition_(condition)\n    { statement_type(SUPPORTS); }\n    Supports_Block(const Supports_Block* ptr)\n    : Has_Block(ptr), condition_(ptr->condition_)\n    { statement_type(SUPPORTS); }\n    bool bubbles() { return true; }\n    ATTACH_AST_OPERATIONS(Supports_Block)\n    ATTACH_OPERATIONS()\n  };\n\n  //////////////////////////////////////////////////////\n  // The abstract superclass of all Supports conditions.\n  //////////////////////////////////////////////////////\n  class Supports_Condition : public Expression {\n  public:\n    Supports_Condition(ParserState pstate)\n    : Expression(pstate)\n    { }\n    Supports_Condition(const Supports_Condition* ptr)\n    : Expression(ptr)\n    { }\n    virtual bool needs_parens(Supports_Condition_Obj cond) const { return false; }\n    ATTACH_AST_OPERATIONS(Supports_Condition)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////////////////\n  // An operator condition (e.g. `CONDITION1 and CONDITION2`).\n  ////////////////////////////////////////////////////////////\n  class Supports_Operator : public Supports_Condition {\n  public:\n    enum Operand { AND, OR };\n  private:\n    ADD_PROPERTY(Supports_Condition_Obj, left);\n    ADD_PROPERTY(Supports_Condition_Obj, right);\n    ADD_PROPERTY(Operand, operand);\n  public:\n    Supports_Operator(ParserState pstate, Supports_Condition_Obj l, Supports_Condition_Obj r, Operand o)\n    : Supports_Condition(pstate), left_(l), right_(r), operand_(o)\n    { }\n    Supports_Operator(const Supports_Operator* ptr)\n    : Supports_Condition(ptr),\n      left_(ptr->left_),\n      right_(ptr->right_),\n      operand_(ptr->operand_)\n    { }\n    virtual bool needs_parens(Supports_Condition_Obj cond) const;\n    ATTACH_AST_OPERATIONS(Supports_Operator)\n    ATTACH_OPERATIONS()\n  };\n\n  //////////////////////////////////////////\n  // A negation condition (`not CONDITION`).\n  //////////////////////////////////////////\n  class Supports_Negation : public Supports_Condition {\n  private:\n    ADD_PROPERTY(Supports_Condition_Obj, condition);\n  public:\n    Supports_Negation(ParserState pstate, Supports_Condition_Obj c)\n    : Supports_Condition(pstate), condition_(c)\n    { }\n    Supports_Negation(const Supports_Negation* ptr)\n    : Supports_Condition(ptr), condition_(ptr->condition_)\n    { }\n    virtual bool needs_parens(Supports_Condition_Obj cond) const;\n    ATTACH_AST_OPERATIONS(Supports_Negation)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////////////////////////\n  // A declaration condition (e.g. `(feature: value)`).\n  /////////////////////////////////////////////////////\n  class Supports_Declaration : public Supports_Condition {\n  private:\n    ADD_PROPERTY(Expression_Obj, feature);\n    ADD_PROPERTY(Expression_Obj, value);\n  public:\n    Supports_Declaration(ParserState pstate, Expression_Obj f, Expression_Obj v)\n    : Supports_Condition(pstate), feature_(f), value_(v)\n    { }\n    Supports_Declaration(const Supports_Declaration* ptr)\n    : Supports_Condition(ptr),\n      feature_(ptr->feature_),\n      value_(ptr->value_)\n    { }\n    virtual bool needs_parens(Supports_Condition_Obj cond) const { return false; }\n    ATTACH_AST_OPERATIONS(Supports_Declaration)\n    ATTACH_OPERATIONS()\n  };\n\n  ///////////////////////////////////////////////\n  // An interpolation condition (e.g. `#{$var}`).\n  ///////////////////////////////////////////////\n  class Supports_Interpolation : public Supports_Condition {\n  private:\n    ADD_PROPERTY(Expression_Obj, value);\n  public:\n    Supports_Interpolation(ParserState pstate, Expression_Obj v)\n    : Supports_Condition(pstate), value_(v)\n    { }\n    Supports_Interpolation(const Supports_Interpolation* ptr)\n    : Supports_Condition(ptr),\n      value_(ptr->value_)\n    { }\n    virtual bool needs_parens(Supports_Condition_Obj cond) const { return false; }\n    ATTACH_AST_OPERATIONS(Supports_Interpolation)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////////////////////\n  // At root expressions (for use inside @at-root).\n  /////////////////////////////////////////////////\n  class At_Root_Query : public Expression {\n  private:\n    ADD_PROPERTY(Expression_Obj, feature)\n    ADD_PROPERTY(Expression_Obj, value)\n  public:\n    At_Root_Query(ParserState pstate, Expression_Obj f = 0, Expression_Obj v = 0, bool i = false)\n    : Expression(pstate), feature_(f), value_(v)\n    { }\n    At_Root_Query(const At_Root_Query* ptr)\n    : Expression(ptr),\n      feature_(ptr->feature_),\n      value_(ptr->value_)\n    { }\n    bool exclude(std::string str);\n    ATTACH_AST_OPERATIONS(At_Root_Query)\n    ATTACH_OPERATIONS()\n  };\n\n  ///////////\n  // At-root.\n  ///////////\n  class At_Root_Block : public Has_Block {\n    ADD_PROPERTY(At_Root_Query_Obj, expression)\n  public:\n    At_Root_Block(ParserState pstate, Block_Obj b = 0, At_Root_Query_Obj e = 0)\n    : Has_Block(pstate, b), expression_(e)\n    { statement_type(ATROOT); }\n    At_Root_Block(const At_Root_Block* ptr)\n    : Has_Block(ptr), expression_(ptr->expression_)\n    { statement_type(ATROOT); }\n    bool bubbles() { return true; }\n    bool exclude_node(Statement_Obj s) {\n      if (expression() == 0)\n      {\n        return s->statement_type() == Statement::RULESET;\n      }\n\n      if (s->statement_type() == Statement::DIRECTIVE)\n      {\n        if (Directive_Obj dir = Cast<Directive>(s))\n        {\n          std::string keyword(dir->keyword());\n          if (keyword.length() > 0) keyword.erase(0, 1);\n          return expression()->exclude(keyword);\n        }\n      }\n      if (s->statement_type() == Statement::MEDIA)\n      {\n        return expression()->exclude(\"media\");\n      }\n      if (s->statement_type() == Statement::RULESET)\n      {\n        return expression()->exclude(\"rule\");\n      }\n      if (s->statement_type() == Statement::SUPPORTS)\n      {\n        return expression()->exclude(\"supports\");\n      }\n      if (Directive_Obj dir = Cast<Directive>(s))\n      {\n        if (dir->is_keyframes()) return expression()->exclude(\"keyframes\");\n      }\n      return false;\n    }\n    ATTACH_AST_OPERATIONS(At_Root_Block)\n    ATTACH_OPERATIONS()\n  };\n\n  //////////////////\n  // The null value.\n  //////////////////\n  class Null : public Value {\n  public:\n    Null(ParserState pstate) : Value(pstate) { concrete_type(NULL_VAL); }\n    Null(const Null* ptr) : Value(ptr) { concrete_type(NULL_VAL); }\n    std::string type() const { return \"null\"; }\n    static std::string type_name() { return \"null\"; }\n    bool is_invisible() const { return true; }\n    operator bool() { return false; }\n    bool is_false() { return true; }\n\n    virtual size_t hash()\n    {\n      return -1;\n    }\n\n    virtual bool operator== (const Expression& rhs) const;\n\n    ATTACH_AST_OPERATIONS(Null)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////\n  // Thunks for delayed evaluation.\n  /////////////////////////////////\n  class Thunk : public Expression {\n    ADD_PROPERTY(Expression_Obj, expression)\n    ADD_PROPERTY(Env*, environment)\n  public:\n    \nThunk(ParserState pstate, Expression_Obj exp, Env* env = 0)\n    : Expression(pstate), expression_(exp), environment_(env)\n    { }\n  };\n\n  /////////////////////////////////////////////////////////\n  // Individual parameter objects for mixins and functions.\n  /////////////////////////////////////////////////////////\n  class Parameter : public AST_Node {\n    ADD_CONSTREF(std::string, name)\n    ADD_PROPERTY(Expression_Obj, default_value)\n    ADD_PROPERTY(bool, is_rest_parameter)\n  public:\n    Parameter(ParserState pstate,\n              std::string n, Expression_Obj def = 0, bool rest = false)\n    : AST_Node(pstate), name_(n), default_value_(def), is_rest_parameter_(rest)\n    {\n      // tried to come up with a spec test for this, but it does no longer\n      // get  past the parser (it error out earlier). A spec test was added!\n      // if (default_value_ && is_rest_parameter_) {\n      //   error(\"variable-length parameter may not have a default value\", pstate_);\n      // }\n    }\n    Parameter(const Parameter* ptr)\n    : AST_Node(ptr),\n      name_(ptr->name_),\n      default_value_(ptr->default_value_),\n      is_rest_parameter_(ptr->is_rest_parameter_)\n    {\n      // tried to come up with a spec test for this, but it does no longer\n      // get  past the parser (it error out earlier). A spec test was added!\n      // if (default_value_ && is_rest_parameter_) {\n      //   error(\"variable-length parameter may not have a default value\", pstate_);\n      // }\n    }\n    ATTACH_AST_OPERATIONS(Parameter)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////////////////////////////////////////////\n  // Parameter lists -- in their own class to facilitate context-sensitive\n  // error checking (e.g., ensuring that all optional parameters follow all\n  // required parameters).\n  /////////////////////////////////////////////////////////////////////////\n  class Parameters : public AST_Node, public Vectorized<Parameter_Obj> {\n    ADD_PROPERTY(bool, has_optional_parameters)\n    ADD_PROPERTY(bool, has_rest_parameter)\n  protected:\n    void adjust_after_pushing(Parameter_Obj p)\n    {\n      if (p->default_value()) {\n        if (has_rest_parameter()) {\n          coreError(\"optional parameters may not be combined with variable-length parameters\", p->pstate());\n        }\n        has_optional_parameters(true);\n      }\n      else if (p->is_rest_parameter()) {\n        if (has_rest_parameter()) {\n          coreError(\"functions and mixins cannot have more than one variable-length parameter\", p->pstate());\n        }\n        has_rest_parameter(true);\n      }\n      else {\n        if (has_rest_parameter()) {\n          coreError(\"required parameters must precede variable-length parameters\", p->pstate());\n        }\n        if (has_optional_parameters()) {\n          coreError(\"required parameters must precede optional parameters\", p->pstate());\n        }\n      }\n    }\n  public:\n    Parameters(ParserState pstate)\n    : AST_Node(pstate),\n      Vectorized<Parameter_Obj>(),\n      has_optional_parameters_(false),\n      has_rest_parameter_(false)\n    { }\n    Parameters(const Parameters* ptr)\n    : AST_Node(ptr),\n      Vectorized<Parameter_Obj>(*ptr),\n      has_optional_parameters_(ptr->has_optional_parameters_),\n      has_rest_parameter_(ptr->has_rest_parameter_)\n    { }\n    ATTACH_AST_OPERATIONS(Parameters)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////////////\n  // Abstract base class for CSS selectors.\n  /////////////////////////////////////////\n  class Selector : public Expression {\n    // ADD_PROPERTY(bool, has_reference)\n    // line break before list separator\n    ADD_PROPERTY(bool, has_line_feed)\n    // line break after list separator\n    ADD_PROPERTY(bool, has_line_break)\n    // maybe we have optional flag\n    ADD_PROPERTY(bool, is_optional)\n    // parent block pointers\n\n    // must not be a reference counted object\n    // otherwise we create circular references\n    ADD_PROPERTY(Media_Block_Ptr, media_block)\n  protected:\n    size_t hash_;\n  public:\n    Selector(ParserState pstate)\n    : Expression(pstate),\n      has_line_feed_(false),\n      has_line_break_(false),\n      is_optional_(false),\n      media_block_(0),\n      hash_(0)\n    { concrete_type(SELECTOR); }\n    Selector(const Selector* ptr)\n    : Expression(ptr),\n      // has_reference_(ptr->has_reference_),\n      has_line_feed_(ptr->has_line_feed_),\n      has_line_break_(ptr->has_line_break_),\n      is_optional_(ptr->is_optional_),\n      media_block_(ptr->media_block_),\n      hash_(ptr->hash_)\n    { concrete_type(SELECTOR); }\n    virtual ~Selector() = 0;\n    virtual size_t hash() = 0;\n    virtual unsigned long specificity() const = 0;\n    virtual void set_media_block(Media_Block_Ptr mb) {\n      media_block(mb);\n    }\n    virtual bool has_parent_ref() const {\n      return false;\n    }\n    virtual bool has_real_parent_ref() const {\n      return false;\n    }\n    // dispatch to correct handlers\n    virtual bool operator<(const Selector& rhs) const = 0;\n    virtual bool operator==(const Selector& rhs) const = 0;\n    ATTACH_VIRTUAL_AST_OPERATIONS(Selector);\n  };\n  inline Selector::~Selector() { }\n\n  /////////////////////////////////////////////////////////////////////////\n  // Interpolated selectors -- the interpolated String will be expanded and\n  // re-parsed into a normal selector class.\n  /////////////////////////////////////////////////////////////////////////\n  class Selector_Schema : public AST_Node {\n    ADD_PROPERTY(String_Obj, contents)\n    ADD_PROPERTY(bool, connect_parent);\n    // must not be a reference counted object\n    // otherwise we create circular references\n    ADD_PROPERTY(Media_Block_Ptr, media_block)\n    // store computed hash\n    size_t hash_;\n  public:\n    Selector_Schema(ParserState pstate, String_Obj c)\n    : AST_Node(pstate),\n      contents_(c),\n      connect_parent_(true),\n      media_block_(NULL),\n      hash_(0)\n    { }\n    Selector_Schema(const Selector_Schema* ptr)\n    : AST_Node(ptr),\n      contents_(ptr->contents_),\n      connect_parent_(ptr->connect_parent_),\n      media_block_(ptr->media_block_),\n      hash_(ptr->hash_)\n    { }\n    virtual bool has_parent_ref() const;\n    virtual bool has_real_parent_ref() const;\n    virtual bool operator<(const Selector& rhs) const;\n    virtual bool operator==(const Selector& rhs) const;\n    // selector schema is not yet a final selector, so we do not\n    // have a specificity for it yet. We need to\n    virtual unsigned long specificity() const { return 0; }\n    virtual size_t hash() {\n      if (hash_ == 0) {\n        hash_combine(hash_, contents_->hash());\n      }\n      return hash_;\n    }\n    ATTACH_AST_OPERATIONS(Selector_Schema)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////\n  // Abstract base class for simple selectors.\n  ////////////////////////////////////////////\n  class Simple_Selector : public Selector {\n    ADD_CONSTREF(std::string, ns)\n    ADD_CONSTREF(std::string, name)\n    ADD_PROPERTY(Simple_Type, simple_type)\n    ADD_PROPERTY(bool, has_ns)\n  public:\n    Simple_Selector(ParserState pstate, std::string n = \"\")\n    : Selector(pstate), ns_(\"\"), name_(n), has_ns_(false)\n    {\n      simple_type(SIMPLE);\n      size_t pos = n.find('|');\n      // found some namespace\n      if (pos != std::string::npos) {\n        has_ns_ = true;\n        ns_ = n.substr(0, pos);\n        name_ = n.substr(pos + 1);\n      }\n    }\n    Simple_Selector(const Simple_Selector* ptr)\n    : Selector(ptr),\n      ns_(ptr->ns_),\n      name_(ptr->name_),\n      has_ns_(ptr->has_ns_)\n    { simple_type(SIMPLE); }\n    virtual std::string ns_name() const\n    {\n      std::string name(\"\");\n      if (has_ns_)\n        name += ns_ + \"|\";\n      return name + name_;\n    }\n    virtual size_t hash()\n    {\n      if (hash_ == 0) {\n        hash_combine(hash_, std::hash<int>()(SELECTOR));\n        hash_combine(hash_, std::hash<std::string>()(ns()));\n        hash_combine(hash_, std::hash<std::string>()(name()));\n      }\n      return hash_;\n    }\n    // namespace compare functions\n    bool is_ns_eq(const Simple_Selector& r) const;\n    // namespace query functions\n    bool is_universal_ns() const\n    {\n      return has_ns_ && ns_ == \"*\";\n    }\n    bool has_universal_ns() const\n    {\n      return !has_ns_ || ns_ == \"*\";\n    }\n    bool is_empty_ns() const\n    {\n      return !has_ns_ || ns_ == \"\";\n    }\n    bool has_empty_ns() const\n    {\n      return has_ns_ && ns_ == \"\";\n    }\n    bool has_qualified_ns() const\n    {\n      return has_ns_ && ns_ != \"\" && ns_ != \"*\";\n    }\n    // name query functions\n    bool is_universal() const\n    {\n      return name_ == \"*\";\n    }\n\n    virtual bool has_placeholder() {\n      return false;\n    }\n\n    virtual ~Simple_Selector() = 0;\n    virtual Compound_Selector_Ptr unify_with(Compound_Selector_Ptr);\n    virtual bool has_parent_ref() const { return false; };\n    virtual bool has_real_parent_ref() const  { return false; };\n    virtual bool is_pseudo_element() const { return false; }\n\n    virtual bool is_superselector_of(Compound_Selector_Obj sub) { return false; }\n\n    virtual bool operator==(const Selector& rhs) const;\n    virtual bool operator==(const Simple_Selector& rhs) const;\n    inline bool operator!=(const Simple_Selector& rhs) const { return !(*this == rhs); }\n\n    bool operator<(const Selector& rhs) const;\n    bool operator<(const Simple_Selector& rhs) const;\n    // default implementation should work for most of the simple selectors (otherwise overload)\n    ATTACH_VIRTUAL_AST_OPERATIONS(Simple_Selector);\n    ATTACH_OPERATIONS();\n  };\n  inline Simple_Selector::~Simple_Selector() { }\n\n\n  //////////////////////////////////\n  // The Parent Selector Expression.\n  //////////////////////////////////\n  // parent selectors can occur in selectors but also\n  // inside strings in declarations (Compound_Selector).\n  // only one simple parent selector means the first case.\n  class Parent_Selector : public Simple_Selector {\n    ADD_PROPERTY(bool, real)\n  public:\n    Parent_Selector(ParserState pstate, bool r = true)\n    : Simple_Selector(pstate, \"&\"), real_(r)\n    { /* has_reference(true); */ }\n    Parent_Selector(const Parent_Selector* ptr)\n    : Simple_Selector(ptr), real_(ptr->real_)\n    { /* has_reference(true); */ }\n    bool is_real_parent_ref() const { return real(); };\n    virtual bool has_parent_ref() const { return true; };\n    virtual bool has_real_parent_ref() const { return is_real_parent_ref(); };\n    virtual unsigned long specificity() const\n    {\n      return 0;\n    }\n    std::string type() const { return \"selector\"; }\n    static std::string type_name() { return \"selector\"; }\n    ATTACH_AST_OPERATIONS(Parent_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////////////////////////////////////////////\n  // Placeholder selectors (e.g., \"%foo\") for use in extend-only selectors.\n  /////////////////////////////////////////////////////////////////////////\n  class Placeholder_Selector : public Simple_Selector {\n  public:\n    Placeholder_Selector(ParserState pstate, std::string n)\n    : Simple_Selector(pstate, n)\n    { }\n    Placeholder_Selector(const Placeholder_Selector* ptr)\n    : Simple_Selector(ptr)\n    { }\n    virtual unsigned long specificity() const\n    {\n      return Constants::Specificity_Base;\n    }\n    virtual bool has_placeholder() {\n      return true;\n    }\n    virtual ~Placeholder_Selector() {};\n    ATTACH_AST_OPERATIONS(Placeholder_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////////////////////////////////////////\n  // Element selectors (and the universal selector) -- e.g., div, span, *.\n  /////////////////////////////////////////////////////////////////////\n  class Element_Selector : public Simple_Selector {\n  public:\n    Element_Selector(ParserState pstate, std::string n)\n    : Simple_Selector(pstate, n)\n    { }\n    Element_Selector(const Element_Selector* ptr)\n    : Simple_Selector(ptr)\n    { }\n    virtual unsigned long specificity() const\n    {\n      if (name() == \"*\") return 0;\n      else               return Constants::Specificity_Element;\n    }\n    virtual Simple_Selector_Ptr unify_with(Simple_Selector_Ptr);\n    virtual Compound_Selector_Ptr unify_with(Compound_Selector_Ptr);\n    virtual bool operator==(const Simple_Selector& rhs) const;\n    virtual bool operator==(const Element_Selector& rhs) const;\n    virtual bool operator<(const Simple_Selector& rhs) const;\n    virtual bool operator<(const Element_Selector& rhs) const;\n    ATTACH_AST_OPERATIONS(Element_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////\n  // Class selectors  -- i.e., .foo.\n  ////////////////////////////////////////////////\n  class Class_Selector : public Simple_Selector {\n  public:\n    Class_Selector(ParserState pstate, std::string n)\n    : Simple_Selector(pstate, n)\n    { }\n    Class_Selector(const Class_Selector* ptr)\n    : Simple_Selector(ptr)\n    { }\n    virtual unsigned long specificity() const\n    {\n      return Constants::Specificity_Class;\n    }\n    virtual Compound_Selector_Ptr unify_with(Compound_Selector_Ptr);\n    ATTACH_AST_OPERATIONS(Class_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////\n  // ID selectors -- i.e., #foo.\n  ////////////////////////////////////////////////\n  class Id_Selector : public Simple_Selector {\n  public:\n    Id_Selector(ParserState pstate, std::string n)\n    : Simple_Selector(pstate, n)\n    { }\n    Id_Selector(const Id_Selector* ptr)\n    : Simple_Selector(ptr)\n    { }\n    virtual unsigned long specificity() const\n    {\n      return Constants::Specificity_ID;\n    }\n    virtual Compound_Selector_Ptr unify_with(Compound_Selector_Ptr);\n    ATTACH_AST_OPERATIONS(Id_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  ///////////////////////////////////////////////////\n  // Attribute selectors -- e.g., [src*=\".jpg\"], etc.\n  ///////////////////////////////////////////////////\n  class Attribute_Selector : public Simple_Selector {\n    ADD_CONSTREF(std::string, matcher)\n    // this cannot be changed to obj atm!!!!!!????!!!!!!!\n    ADD_PROPERTY(String_Obj, value) // might be interpolated\n    ADD_PROPERTY(char, modifier);\n  public:\n    Attribute_Selector(ParserState pstate, std::string n, std::string m, String_Obj v, char o = 0)\n    : Simple_Selector(pstate, n), matcher_(m), value_(v), modifier_(o)\n    { simple_type(ATTR_SEL); }\n    Attribute_Selector(const Attribute_Selector* ptr)\n    : Simple_Selector(ptr),\n      matcher_(ptr->matcher_),\n      value_(ptr->value_),\n      modifier_(ptr->modifier_)\n    { simple_type(ATTR_SEL); }\n    virtual size_t hash()\n    {\n      if (hash_ == 0) {\n        hash_combine(hash_, Simple_Selector::hash());\n        hash_combine(hash_, std::hash<std::string>()(matcher()));\n        if (value_) hash_combine(hash_, value_->hash());\n      }\n      return hash_;\n    }\n    virtual unsigned long specificity() const\n    {\n      return Constants::Specificity_Attr;\n    }\n    virtual bool operator==(const Simple_Selector& rhs) const;\n    virtual bool operator==(const Attribute_Selector& rhs) const;\n    virtual bool operator<(const Simple_Selector& rhs) const;\n    virtual bool operator<(const Attribute_Selector& rhs) const;\n    ATTACH_AST_OPERATIONS(Attribute_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  //////////////////////////////////////////////////////////////////\n  // Pseudo selectors -- e.g., :first-child, :nth-of-type(...), etc.\n  //////////////////////////////////////////////////////////////////\n  /* '::' starts a pseudo-element, ':' a pseudo-class */\n  /* Except :first-line, :first-letter, :before and :after */\n  /* Note that pseudo-elements are restricted to one per selector */\n  /* and occur only in the last simple_selector_sequence. */\n  inline bool is_pseudo_class_element(const std::string& name)\n  {\n    return name == \":before\"       ||\n           name == \":after\"        ||\n           name == \":first-line\"   ||\n           name == \":first-letter\";\n  }\n\n  // Pseudo Selector cannot have any namespace?\n  class Pseudo_Selector : public Simple_Selector {\n    ADD_PROPERTY(String_Obj, expression)\n  public:\n    Pseudo_Selector(ParserState pstate, std::string n, String_Obj expr = 0)\n    : Simple_Selector(pstate, n), expression_(expr)\n    { simple_type(PSEUDO_SEL); }\n    Pseudo_Selector(const Pseudo_Selector* ptr)\n    : Simple_Selector(ptr), expression_(ptr->expression_)\n    { simple_type(PSEUDO_SEL); }\n\n    // A pseudo-element is made of two colons (::) followed by the name.\n    // The `::` notation is introduced by the current document in order to\n    // establish a discrimination between pseudo-classes and pseudo-elements.\n    // For compatibility with existing style sheets, user agents must also\n    // accept the previous one-colon notation for pseudo-elements introduced\n    // in CSS levels 1 and 2 (namely, :first-line, :first-letter, :before and\n    // :after). This compatibility is not allowed for the new pseudo-elements\n    // introduced in this specification.\n    virtual bool is_pseudo_element() const\n    {\n      return (name_[0] == ':' && name_[1] == ':')\n             || is_pseudo_class_element(name_);\n    }\n    virtual size_t hash()\n    {\n      if (hash_ == 0) {\n        hash_combine(hash_, Simple_Selector::hash());\n        if (expression_) hash_combine(hash_, expression_->hash());\n      }\n      return hash_;\n    }\n    virtual unsigned long specificity() const\n    {\n      if (is_pseudo_element())\n        return Constants::Specificity_Element;\n      return Constants::Specificity_Pseudo;\n    }\n    virtual bool operator==(const Simple_Selector& rhs) const;\n    virtual bool operator==(const Pseudo_Selector& rhs) const;\n    virtual bool operator<(const Simple_Selector& rhs) const;\n    virtual bool operator<(const Pseudo_Selector& rhs) const;\n    virtual Compound_Selector_Ptr unify_with(Compound_Selector_Ptr);\n    ATTACH_AST_OPERATIONS(Pseudo_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  /////////////////////////////////////////////////\n  // Wrapped selector -- pseudo selector that takes a list of selectors as argument(s) e.g., :not(:first-of-type), :-moz-any(ol p.blah, ul, menu, dir)\n  /////////////////////////////////////////////////\n  class Wrapped_Selector : public Simple_Selector {\n    ADD_PROPERTY(Selector_List_Obj, selector)\n  public:\n    Wrapped_Selector(ParserState pstate, std::string n, Selector_List_Obj sel)\n    : Simple_Selector(pstate, n), selector_(sel)\n    { simple_type(WRAPPED_SEL); }\n    Wrapped_Selector(const Wrapped_Selector* ptr)\n    : Simple_Selector(ptr), selector_(ptr->selector_)\n    { simple_type(WRAPPED_SEL); }\n    virtual bool is_superselector_of(Wrapped_Selector_Obj sub);\n    // Selectors inside the negation pseudo-class are counted like any\n    // other, but the negation itself does not count as a pseudo-class.\n    virtual size_t hash();\n    virtual bool has_parent_ref() const;\n    virtual bool has_real_parent_ref() const;\n    virtual unsigned long specificity() const;\n    virtual bool find ( bool (*f)(AST_Node_Obj) );\n    virtual bool operator==(const Simple_Selector& rhs) const;\n    virtual bool operator==(const Wrapped_Selector& rhs) const;\n    virtual bool operator<(const Simple_Selector& rhs) const;\n    virtual bool operator<(const Wrapped_Selector& rhs) const;\n    virtual void cloneChildren();\n    ATTACH_AST_OPERATIONS(Wrapped_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////////////////////////////////\n  // Simple selector sequences. Maintains flags indicating whether it contains\n  // any parent references or placeholders, to simplify expansion.\n  ////////////////////////////////////////////////////////////////////////////\n  class Compound_Selector : public Selector, public Vectorized<Simple_Selector_Obj> {\n  private:\n    ComplexSelectorSet sources_;\n    ADD_PROPERTY(bool, extended);\n    ADD_PROPERTY(bool, has_parent_reference);\n  protected:\n    void adjust_after_pushing(Simple_Selector_Obj s)\n    {\n      // if (s->has_reference())   has_reference(true);\n      // if (s->has_placeholder()) has_placeholder(true);\n    }\n  public:\n    Compound_Selector(ParserState pstate, size_t s = 0)\n    : Selector(pstate),\n      Vectorized<Simple_Selector_Obj>(s),\n      extended_(false),\n      has_parent_reference_(false)\n    { }\n    Compound_Selector(const Compound_Selector* ptr)\n    : Selector(ptr),\n      Vectorized<Simple_Selector_Obj>(*ptr),\n      extended_(ptr->extended_),\n      has_parent_reference_(ptr->has_parent_reference_)\n    { }\n    bool contains_placeholder() {\n      for (size_t i = 0, L = length(); i < L; ++i) {\n        if ((*this)[i]->has_placeholder()) return true;\n      }\n      return false;\n    };\n\n    void append(Simple_Selector_Ptr element);\n\n    bool is_universal() const\n    {\n      return length() == 1 && (*this)[0]->is_universal();\n    }\n\n    Complex_Selector_Obj to_complex();\n    Compound_Selector_Ptr unify_with(Compound_Selector_Ptr rhs);\n    // virtual Placeholder_Selector_Ptr find_placeholder();\n    virtual bool has_parent_ref() const;\n    virtual bool has_real_parent_ref() const;\n    Simple_Selector_Ptr base() const {\n      if (length() == 0) return 0;\n      // ToDo: why is this needed?\n      if (Cast<Element_Selector>((*this)[0]))\n        return (*this)[0];\n      return 0;\n    }\n    virtual bool is_superselector_of(Compound_Selector_Obj sub, std::string wrapped = \"\");\n    virtual bool is_superselector_of(Complex_Selector_Obj sub, std::string wrapped = \"\");\n    virtual bool is_superselector_of(Selector_List_Obj sub, std::string wrapped = \"\");\n    virtual size_t hash()\n    {\n      if (Selector::hash_ == 0) {\n        hash_combine(Selector::hash_, std::hash<int>()(SELECTOR));\n        if (length()) hash_combine(Selector::hash_, Vectorized::hash());\n      }\n      return Selector::hash_;\n    }\n    virtual unsigned long specificity() const\n    {\n      int sum = 0;\n      for (size_t i = 0, L = length(); i < L; ++i)\n      { sum += (*this)[i]->specificity(); }\n      return sum;\n    }\n\n    virtual bool has_placeholder()\n    {\n      if (length() == 0) return false;\n      if (Simple_Selector_Obj ss = elements().front()) {\n        if (ss->has_placeholder()) return true;\n      }\n      return false;\n    }\n\n    bool is_empty_reference()\n    {\n      return length() == 1 &&\n             Cast<Parent_Selector>((*this)[0]);\n    }\n\n    virtual bool find ( bool (*f)(AST_Node_Obj) );\n    virtual bool operator<(const Selector& rhs) const;\n    virtual bool operator==(const Selector& rhs) const;\n    virtual bool operator<(const Compound_Selector& rhs) const;\n    virtual bool operator==(const Compound_Selector& rhs) const;\n    inline bool operator!=(const Compound_Selector& rhs) const { return !(*this == rhs); }\n\n    ComplexSelectorSet& sources() { return sources_; }\n    void clearSources() { sources_.clear(); }\n    void mergeSources(ComplexSelectorSet& sources);\n\n    Compound_Selector_Ptr minus(Compound_Selector_Ptr rhs);\n    virtual void cloneChildren();\n    ATTACH_AST_OPERATIONS(Compound_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  ////////////////////////////////////////////////////////////////////////////\n  // General selectors -- i.e., simple sequences combined with one of the four\n  // CSS selector combinators (\">\", \"+\", \"~\", and whitespace). Essentially a\n  // linked list.\n  ////////////////////////////////////////////////////////////////////////////\n  class Complex_Selector : public Selector {\n  public:\n    enum Combinator { ANCESTOR_OF, PARENT_OF, PRECEDES, ADJACENT_TO, REFERENCE };\n  private:\n    HASH_CONSTREF(Combinator, combinator)\n    HASH_PROPERTY(Compound_Selector_Obj, head)\n    HASH_PROPERTY(Complex_Selector_Obj, tail)\n    HASH_PROPERTY(String_Obj, reference);\n  public:\n    bool contains_placeholder() {\n      if (head() && head()->contains_placeholder()) return true;\n      if (tail() && tail()->contains_placeholder()) return true;\n      return false;\n    };\n    Complex_Selector(ParserState pstate,\n                     Combinator c = ANCESTOR_OF,\n                     Compound_Selector_Obj h = 0,\n                     Complex_Selector_Obj t = 0,\n                     String_Obj r = 0)\n    : Selector(pstate),\n      combinator_(c),\n      head_(h), tail_(t),\n      reference_(r)\n    {}\n    Complex_Selector(const Complex_Selector* ptr)\n    : Selector(ptr),\n      combinator_(ptr->combinator_),\n      head_(ptr->head_), tail_(ptr->tail_),\n      reference_(ptr->reference_)\n    {};\n    virtual bool has_parent_ref() const;\n    virtual bool has_real_parent_ref() const;\n\n    Complex_Selector_Obj skip_empty_reference()\n    {\n      if ((!head_ || !head_->length() || head_->is_empty_reference()) &&\n          combinator() == Combinator::ANCESTOR_OF)\n      {\n        if (!tail_) return 0;\n        tail_->has_line_feed_ = this->has_line_feed_;\n        // tail_->has_line_break_ = this->has_line_break_;\n        return tail_->skip_empty_reference();\n      }\n      return this;\n    }\n\n    // can still have a tail\n    bool is_empty_ancestor() const\n    {\n      return (!head() || head()->length() == 0) &&\n             combinator() == Combinator::ANCESTOR_OF;\n    }\n\n    Selector_List_Ptr tails(Selector_List_Ptr tails);\n\n    // front returns the first real tail\n    // skips over parent and empty ones\n    Complex_Selector_Obj first();\n    // last returns the last real tail\n    Complex_Selector_Obj last();\n\n    // some shortcuts that should be removed\n    Complex_Selector_Obj innermost() { return last(); };\n\n    size_t length() const;\n    Selector_List_Ptr resolve_parent_refs(std::vector<Selector_List_Obj>& pstack, Backtraces& traces, bool implicit_parent = true);\n    virtual bool is_superselector_of(Compound_Selector_Obj sub, std::string wrapping = \"\");\n    virtual bool is_superselector_of(Complex_Selector_Obj sub, std::string wrapping = \"\");\n    virtual bool is_superselector_of(Selector_List_Obj sub, std::string wrapping = \"\");\n    Selector_List_Ptr unify_with(Complex_Selector_Ptr rhs);\n    Combinator clear_innermost();\n    void append(Complex_Selector_Obj, Backtraces& traces);\n    void set_innermost(Complex_Selector_Obj, Combinator);\n    virtual size_t hash()\n    {\n      if (hash_ == 0) {\n        hash_combine(hash_, std::hash<int>()(SELECTOR));\n        hash_combine(hash_, std::hash<int>()(combinator_));\n        if (head_) hash_combine(hash_, head_->hash());\n        if (tail_) hash_combine(hash_, tail_->hash());\n      }\n      return hash_;\n    }\n    virtual unsigned long specificity() const\n    {\n      int sum = 0;\n      if (head()) sum += head()->specificity();\n      if (tail()) sum += tail()->specificity();\n      return sum;\n    }\n    virtual void set_media_block(Media_Block_Ptr mb) {\n      media_block(mb);\n      if (tail_) tail_->set_media_block(mb);\n      if (head_) head_->set_media_block(mb);\n    }\n    virtual bool has_placeholder() {\n      if (head_ && head_->has_placeholder()) return true;\n      if (tail_ && tail_->has_placeholder()) return true;\n      return false;\n    }\n    virtual bool find ( bool (*f)(AST_Node_Obj) );\n    virtual bool operator<(const Selector& rhs) const;\n    virtual bool operator==(const Selector& rhs) const;\n    virtual bool operator<(const Complex_Selector& rhs) const;\n    virtual bool operator==(const Complex_Selector& rhs) const;\n    inline bool operator!=(const Complex_Selector& rhs) const { return !(*this == rhs); }\n    const ComplexSelectorSet sources()\n    {\n      //s = Set.new\n      //seq.map {|sseq_or_op| s.merge sseq_or_op.sources if sseq_or_op.is_a?(SimpleSequence)}\n      //s\n\n      ComplexSelectorSet srcs;\n\n      Compound_Selector_Obj pHead = head();\n      Complex_Selector_Obj  pTail = tail();\n\n      if (pHead) {\n        const ComplexSelectorSet& headSources = pHead->sources();\n        srcs.insert(headSources.begin(), headSources.end());\n      }\n\n      if (pTail) {\n        const ComplexSelectorSet& tailSources = pTail->sources();\n        srcs.insert(tailSources.begin(), tailSources.end());\n      }\n\n      return srcs;\n    }\n    void addSources(ComplexSelectorSet& sources) {\n      // members.map! {|m| m.is_a?(SimpleSequence) ? m.with_more_sources(sources) : m}\n      Complex_Selector_Ptr pIter = this;\n      while (pIter) {\n        Compound_Selector_Ptr pHead = pIter->head();\n\n        if (pHead) {\n          pHead->mergeSources(sources);\n        }\n\n        pIter = pIter->tail();\n      }\n    }\n    void clearSources() {\n      Complex_Selector_Ptr pIter = this;\n      while (pIter) {\n        Compound_Selector_Ptr pHead = pIter->head();\n\n        if (pHead) {\n          pHead->clearSources();\n        }\n\n        pIter = pIter->tail();\n      }\n    }\n\n    virtual void cloneChildren();\n    ATTACH_AST_OPERATIONS(Complex_Selector)\n    ATTACH_OPERATIONS()\n  };\n\n  ///////////////////////////////////\n  // Comma-separated selector groups.\n  ///////////////////////////////////\n  class Selector_List : public Selector, public Vectorized<Complex_Selector_Obj> {\n    ADD_PROPERTY(Selector_Schema_Obj, schema)\n    ADD_CONSTREF(std::vector<std::string>, wspace)\n  protected:\n    void adjust_after_pushing(Complex_Selector_Obj c);\n  public:\n    Selector_List(ParserState pstate, size_t s = 0)\n    : Selector(pstate),\n      Vectorized<Complex_Selector_Obj>(s),\n      schema_(NULL),\n      wspace_(0)\n    { }\n    Selector_List(const Selector_List* ptr)\n    : Selector(ptr),\n      Vectorized<Complex_Selector_Obj>(*ptr),\n      schema_(ptr->schema_),\n      wspace_(ptr->wspace_)\n    { }\n    std::string type() const { return \"list\"; }\n    // remove parent selector references\n    // basically unwraps parsed selectors\n    virtual bool has_parent_ref() const;\n    virtual bool has_real_parent_ref() const;\n    void remove_parent_selectors();\n    Selector_List_Ptr resolve_parent_refs(std::vector<Selector_List_Obj>& pstack, Backtraces& traces, bool implicit_parent = true);\n    virtual bool is_superselector_of(Compound_Selector_Obj sub, std::string wrapping = \"\");\n    virtual bool is_superselector_of(Complex_Selector_Obj sub, std::string wrapping = \"\");\n    virtual bool is_superselector_of(Selector_List_Obj sub, std::string wrapping = \"\");\n    Selector_List_Ptr unify_with(Selector_List_Ptr);\n    void populate_extends(Selector_List_Obj, Subset_Map&);\n    Selector_List_Obj eval(Eval& eval);\n    virtual size_t hash()\n    {\n      if (Selector::hash_ == 0) {\n        hash_combine(Selector::hash_, std::hash<int>()(SELECTOR));\n        hash_combine(Selector::hash_, Vectorized::hash());\n      }\n      return Selector::hash_;\n    }\n    virtual unsigned long specificity() const\n    {\n      unsigned long sum = 0;\n      unsigned long specificity;\n      for (size_t i = 0, L = length(); i < L; ++i)\n      {\n        specificity = (*this)[i]->specificity();\n        if (sum < specificity) sum = specificity;\n      }\n      return sum;\n    }\n    virtual void set_media_block(Media_Block_Ptr mb) {\n      media_block(mb);\n      for (Complex_Selector_Obj cs : elements()) {\n        cs->set_media_block(mb);\n      }\n    }\n    virtual bool has_placeholder() {\n      for (Complex_Selector_Obj cs : elements()) {\n        if (cs->has_placeholder()) return true;\n      }\n      return false;\n    }\n    virtual bool find ( bool (*f)(AST_Node_Obj) );\n    virtual bool operator<(const Selector& rhs) const;\n    virtual bool operator==(const Selector& rhs) const;\n    virtual bool operator<(const Selector_List& rhs) const;\n    virtual bool operator==(const Selector_List& rhs) const;\n    // Selector Lists can be compared to comma lists\n    virtual bool operator==(const Expression& rhs) const;\n    virtual void cloneChildren();\n    ATTACH_AST_OPERATIONS(Selector_List)\n    ATTACH_OPERATIONS()\n  };\n\n  // compare function for sorting and probably other other uses\n  struct cmp_complex_selector { inline bool operator() (const Complex_Selector_Obj l, const Complex_Selector_Obj r) { return (*l < *r); } };\n  struct cmp_compound_selector { inline bool operator() (const Compound_Selector_Obj l, const Compound_Selector_Obj r) { return (*l < *r); } };\n  struct cmp_simple_selector { inline bool operator() (const Simple_Selector_Obj l, const Simple_Selector_Obj r) { return (*l < *r); } };\n\n}\n\n#ifdef __clang__\n\n#pragma clang diagnostic pop\n\n#endif\n\n#endif\n\n// Path: src/libsass/src/ast_fwd_decl.cpp\n#include \"ast.hpp\"\n\nnamespace Sass {\n\n  #define IMPLEMENT_BASE_CAST(T) \\\n  template<> \\\n  T* Cast(AST_Node* ptr) { \\\n    return dynamic_cast<T*>(ptr); \\\n  }; \\\n  \\\n  template<> \\\n  const T* Cast(const AST_Node* ptr) { \\\n    return dynamic_cast<const T*>(ptr); \\\n  }; \\\n\n  IMPLEMENT_BASE_CAST(AST_Node)\n  IMPLEMENT_BASE_CAST(Expression)\n  IMPLEMENT_BASE_CAST(Statement)\n  IMPLEMENT_BASE_CAST(Has_Block)\n  IMPLEMENT_BASE_CAST(PreValue)\n  IMPLEMENT_BASE_CAST(Value)\n  IMPLEMENT_BASE_CAST(List)\n  IMPLEMENT_BASE_CAST(String)\n  IMPLEMENT_BASE_CAST(String_Constant)\n  IMPLEMENT_BASE_CAST(Supports_Condition)\n  IMPLEMENT_BASE_CAST(Selector)\n  IMPLEMENT_BASE_CAST(Simple_Selector)\n\n}\n\n// Path: src/libsass/src/backtrace.cpp\n#include \"backtrace.hpp\"\n\nnamespace Sass {\n\n  const std::string traces_to_string(Backtraces traces, std::string indent) {\n\n    std::stringstream ss;\n    std::string cwd(File::get_cwd());\n\n    bool first = true;\n    size_t i_beg = traces.size() - 1;\n    size_t i_end = std::string::npos;\n    for (size_t i = i_beg; i != i_end; i --) {\n\n      const Backtrace& trace = traces[i];\n\n      // make path relative to the current directory\n      std::string rel_path(File::abs2rel(trace.pstate.path, cwd, cwd));\n\n      // skip functions on error cases (unsure why ruby sass does this)\n      // if (trace.caller.substr(0, 6) == \", in f\") continue;\n\n      if (first) {\n        ss << indent;\n        ss << \"on line \";\n        ss << trace.pstate.line + 1;\n        ss << \" of \" << rel_path;\n        // ss << trace.caller;\n        first = false;\n      } else {\n        ss << trace.caller;\n        ss << std::endl;\n        ss << indent;\n        ss << \"from line \";\n        ss << trace.pstate.line + 1;\n        ss << \" of \" << rel_path;\n      }\n\n    }\n\n    ss << std::endl;\n    return ss.str();\n\n  }\n\n};\n\n// Path: src/libsass/src/environment.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"environment.hpp\"\n\nnamespace Sass {\n\n  template <typename T>\n  Environment<T>::Environment(bool is_shadow)\n  : local_frame_(environment_map<std::string, T>()),\n    parent_(0), is_shadow_(false)\n  { }\n  template <typename T>\n  Environment<T>::Environment(Environment<T>* env, bool is_shadow)\n  : local_frame_(environment_map<std::string, T>()),\n    parent_(env), is_shadow_(is_shadow)\n  { }\n  template <typename T>\n  Environment<T>::Environment(Environment<T>& env, bool is_shadow)\n  : local_frame_(environment_map<std::string, T>()),\n    parent_(&env), is_shadow_(is_shadow)\n  { }\n\n  // link parent to create a stack\n  template <typename T>\n  void Environment<T>::link(Environment& env) { parent_ = &env; }\n  template <typename T>\n  void Environment<T>::link(Environment* env) { parent_ = env; }\n\n  // this is used to find the global frame\n  // which is the second last on the stack\n  template <typename T>\n  bool Environment<T>::is_lexical() const\n  {\n    return !! parent_ && parent_->parent_;\n  }\n\n  // only match the real root scope\n  // there is still a parent around\n  // not sure what it is actually use for\n  // I guess we store functions etc. there\n  template <typename T>\n  bool Environment<T>::is_global() const\n  {\n    return parent_ && ! parent_->parent_;\n  }\n\n  template <typename T>\n  environment_map<std::string, T>& Environment<T>::local_frame() {\n    return local_frame_;\n  }\n\n  template <typename T>\n  bool Environment<T>::has_local(const std::string& key) const\n  { return local_frame_.find(key) != local_frame_.end(); }\n\n  template <typename T> EnvResult\n  Environment<T>::find_local(const std::string& key)\n  {\n    auto end = local_frame_.end();\n    auto it = local_frame_.find(key);\n    return EnvResult(it, it != end);\n  }\n\n  template <typename T>\n  T& Environment<T>::get_local(const std::string& key)\n  { return local_frame_[key]; }\n\n  template <typename T>\n  void Environment<T>::set_local(const std::string& key, const T& val)\n  {\n    local_frame_[key] = val;\n  }\n  template <typename T>\n  void Environment<T>::set_local(const std::string& key, T&& val)\n  {\n    local_frame_[key] = val;\n  }\n\n  template <typename T>\n  void Environment<T>::del_local(const std::string& key)\n  { local_frame_.erase(key); }\n\n  template <typename T>\n  Environment<T>* Environment<T>::global_env()\n  {\n    Environment* cur = this;\n    while (cur->is_lexical()) {\n      cur = cur->parent_;\n    }\n    return cur;\n  }\n\n  template <typename T>\n  bool Environment<T>::has_global(const std::string& key)\n  { return global_env()->has(key); }\n\n  template <typename T>\n  T& Environment<T>::get_global(const std::string& key)\n  { return (*global_env())[key]; }\n\n  template <typename T>\n  void Environment<T>::set_global(const std::string& key, const T& val)\n  {\n    global_env()->local_frame_[key] = val;\n  }\n  template <typename T>\n  void Environment<T>::set_global(const std::string& key, T&& val)\n  {\n    global_env()->local_frame_[key] = val;\n  }\n\n  template <typename T>\n  void Environment<T>::del_global(const std::string& key)\n  { global_env()->local_frame_.erase(key); }\n\n  template <typename T>\n  Environment<T>* Environment<T>::lexical_env(const std::string& key)\n  {\n    Environment* cur = this;\n    while (cur) {\n      if (cur->has_local(key)) {\n        return cur;\n      }\n      cur = cur->parent_;\n    }\n    return this;\n  }\n\n  // see if we have a lexical variable\n  // move down the stack but stop before we\n  // reach the global frame (is not included)\n  template <typename T>\n  bool Environment<T>::has_lexical(const std::string& key) const\n  {\n    auto cur = this;\n    while (cur->is_lexical()) {\n      if (cur->has_local(key)) return true;\n      cur = cur->parent_;\n    }\n    return false;\n  }\n\n  // see if we have a lexical we could update\n  // either update already existing lexical value\n  // or if flag is set, we create one if no lexical found\n  template <typename T>\n  void Environment<T>::set_lexical(const std::string& key, const T& val)\n  {\n    Environment<T>* cur = this;\n    bool shadow = false;\n    while ((cur && cur->is_lexical()) || shadow) {\n      EnvResult rv(cur->find_local(key));\n      if (rv.found) {\n        rv.it->second = val;\n        return;\n      }\n      shadow = cur->is_shadow();\n      cur = cur->parent_;\n    }\n    set_local(key, val);\n  }\n  // this one moves the value\n  template <typename T>\n  void Environment<T>::set_lexical(const std::string& key, T&& val)\n  {\n    Environment<T>* cur = this;\n    bool shadow = false;\n    while ((cur && cur->is_lexical()) || shadow) {\n      EnvResult rv(cur->find_local(key));\n      if (rv.found) {\n        rv.it->second = val;\n        return;\n      }\n      shadow = cur->is_shadow();\n      cur = cur->parent_;\n    }\n    set_local(key, val);\n  }\n\n  // look on the full stack for key\n  // include all scopes available\n  template <typename T>\n  bool Environment<T>::has(const std::string& key) const\n  {\n    auto cur = this;\n    while (cur) {\n      if (cur->has_local(key)) {\n        return true;\n      }\n      cur = cur->parent_;\n    }\n    return false;\n  }\n\n  // look on the full stack for key\n  // include all scopes available\n  template <typename T> EnvResult\n  Environment<T>::find(const std::string& key)\n  {\n    auto cur = this;\n    while (true) {\n      EnvResult rv(cur->find_local(key));\n      if (rv.found) return rv;\n      cur = cur->parent_;\n      if (!cur) return rv;\n    }\n  };\n\n  // use array access for getter and setter functions\n  template <typename T>\n  T& Environment<T>::operator[](const std::string& key)\n  {\n    auto cur = this;\n    while (cur) {\n      if (cur->has_local(key)) {\n        return cur->get_local(key);\n      }\n      cur = cur->parent_;\n...\n", "needle_token_start": 4044, "needle_token_end": 4085, "code_context_ntokens": 16259, "output": ["```cpp\nclass Thunk : public Expression {\n    ADD_PROPERTY(Expression_Obj, expression)\n    ADD_PROPERTY(Env*, environment)\npublic:\n    Thunk(ParserState pstate, Expression_Obj exp, Env* env = 0)\n        : Expression(pstate), expression_(exp), environment_(env)\n    { }\n};\n```"]}
{"repo": "sass/node-sass", "name": "vector_to_string", "language": "cpp", "path": "src/libsass/src/paths.hpp", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a list of elements into a human-readable string format, encapsulating the elements within square brackets and separating them with commas.\n2. **Input**: A vector containing elements of a generic type.\n3. **Output**: A string representation of the vector, where elements are enclosed in square brackets and separated by commas. If the vector is empty, it returns \"[]\".\n4. **Procedure**: The function initializes a string stream and begins by adding an opening square bracket. If the vector is not empty, it inserts the first element directly. For vectors with more than one element, it iterates through the remaining elements, appending each with a preceding comma and a space for formatting. Finally, it closes with a square bracket and converts the stream contents into a string for output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/libsass/src/ast.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"context.hpp\"\n#include \"node.hpp\"\n#include \"eval.hpp\"\n#include \"extend.hpp\"\n#include \"emitter.hpp\"\n#include \"color_maps.hpp\"\n#include \"ast_fwd_decl.hpp\"\n#include <set>\n#include <iomanip>\n#include <iostream>\n#include <algorithm>\n#include <functional>\n#include <cctype>\n#include <locale>\n\nnamespace Sass {\n\n  static Null sass_null(ParserState(\"null\"));\n\n  bool Wrapped_Selector::find ( bool (*f)(AST_Node_Obj) )\n  {\n    // check children first\n    if (selector_) {\n      if (selector_->find(f)) return true;\n    }\n    // execute last\n    return f(this);\n  }\n\n  bool Selector_List::find ( bool (*f)(AST_Node_Obj) )\n  {\n    // check children first\n    for (Complex_Selector_Obj sel : elements()) {\n      if (sel->find(f)) return true;\n    }\n    // execute last\n    return f(this);\n  }\n\n  bool Compound_Selector::find ( bool (*f)(AST_Node_Obj) )\n  {\n    // check children first\n    for (Simple_Selector_Obj sel : elements()) {\n      if (sel->find(f)) return true;\n    }\n    // execute last\n    return f(this);\n  }\n\n  bool Complex_Selector::find ( bool (*f)(AST_Node_Obj) )\n  {\n    // check children first\n    if (head_ && head_->find(f)) return true;\n    if (tail_ && tail_->find(f)) return true;\n    // execute last\n    return f(this);\n  }\n\n  bool Supports_Operator::needs_parens(Supports_Condition_Obj cond) const {\n    if (Supports_Operator_Obj op = Cast<Supports_Operator>(cond)) {\n      return op->operand() != operand();\n    }\n    return Cast<Supports_Negation>(cond) != NULL;\n  }\n\n  bool Supports_Negation::needs_parens(Supports_Condition_Obj cond) const {\n    return Cast<Supports_Negation>(cond) ||\n           Cast<Supports_Operator>(cond);\n  }\n\n  void str_rtrim(std::string& str, const std::string& delimiters = \" \\f\\n\\r\\t\\v\")\n  {\n    str.erase( str.find_last_not_of( delimiters ) + 1 );\n  }\n\n  void String_Constant::rtrim()\n  {\n    str_rtrim(value_);\n  }\n\n  void String_Schema::rtrim()\n  {\n    if (!empty()) {\n      if (String_Ptr str = Cast<String>(last())) str->rtrim();\n    }\n  }\n\n  void Argument::set_delayed(bool delayed)\n  {\n    if (value_) value_->set_delayed(delayed);\n    is_delayed(delayed);\n  }\n\n  void Arguments::set_delayed(bool delayed)\n  {\n    for (Argument_Obj arg : elements()) {\n      if (arg) arg->set_delayed(delayed);\n    }\n    is_delayed(delayed);\n  }\n\n\n  bool At_Root_Query::exclude(std::string str)\n  {\n    bool with = feature() && unquote(feature()->to_string()).compare(\"with\") == 0;\n    List_Ptr l = static_cast<List_Ptr>(value().ptr());\n    std::string v;\n\n    if (with)\n    {\n      if (!l || l->length() == 0) return str.compare(\"rule\") != 0;\n      for (size_t i = 0, L = l->length(); i < L; ++i)\n      {\n        v = unquote((*l)[i]->to_string());\n        if (v.compare(\"all\") == 0 || v == str) return false;\n      }\n      return true;\n    }\n    else\n    {\n      if (!l || !l->length()) return str.compare(\"rule\") == 0;\n      for (size_t i = 0, L = l->length(); i < L; ++i)\n      {\n        v = unquote((*l)[i]->to_string());\n        if (v.compare(\"all\") == 0 || v == str) return true;\n      }\n      return false;\n    }\n  }\n\n  void AST_Node::update_pstate(const ParserState& pstate)\n  {\n    pstate_.offset += pstate - pstate_ + pstate.offset;\n  }\n\n  bool Simple_Selector::is_ns_eq(const Simple_Selector& r) const\n  {\n    // https://github.com/sass/sass/issues/2229\n    if ((has_ns_ == r.has_ns_) ||\n        (has_ns_ && ns_.empty()) ||\n        (r.has_ns_ && r.ns_.empty())\n    ) {\n      if (ns_.empty() && r.ns() == \"*\") return false;\n      else if (r.ns().empty() && ns() == \"*\") return false;\n      else return ns() == r.ns();\n    }\n    return false;\n  }\n\n  bool Compound_Selector::operator< (const Compound_Selector& rhs) const\n  {\n    size_t L = std::min(length(), rhs.length());\n    for (size_t i = 0; i < L; ++i)\n    {\n      Simple_Selector_Obj l = (*this)[i];\n      Simple_Selector_Obj r = rhs[i];\n      if (!l && !r) return false;\n      else if (!r) return false;\n      else if (!l) return true;\n      else if (*l != *r)\n      { return *l < *r; }\n    }\n    // just compare the length now\n    return length() < rhs.length();\n  }\n\n  bool Compound_Selector::has_parent_ref() const\n  {\n    for (Simple_Selector_Obj s : *this) {\n      if (s && s->has_parent_ref()) return true;\n    }\n    return false;\n  }\n\n  bool Compound_Selector::has_real_parent_ref() const\n  {\n    for (Simple_Selector_Obj s : *this) {\n      if (s && s->has_real_parent_ref()) return true;\n    }\n    return false;\n  }\n\n  bool Complex_Selector::has_parent_ref() const\n  {\n    return (head() && head()->has_parent_ref()) ||\n           (tail() && tail()->has_parent_ref());\n  }\n\n  bool Complex_Selector::has_real_parent_ref() const\n  {\n    return (head() && head()->has_real_parent_ref()) ||\n           (tail() && tail()->has_real_parent_ref());\n  }\n\n  bool Complex_Selector::operator< (const Complex_Selector& rhs) const\n  {\n    // const iterators for tails\n    Complex_Selector_Ptr_Const l = this;\n    Complex_Selector_Ptr_Const r = &rhs;\n    Compound_Selector_Ptr l_h = NULL;\n    Compound_Selector_Ptr r_h = NULL;\n    if (l) l_h = l->head();\n    if (r) r_h = r->head();\n    // process all tails\n    while (true)\n    {\n      #ifdef DEBUG\n      // skip empty ancestor first\n      if (l && l->is_empty_ancestor())\n      {\n        l_h = NULL;\n        l = l->tail();\n        if(l) l_h = l->head();\n        continue;\n      }\n      // skip empty ancestor first\n      if (r && r->is_empty_ancestor())\n      {\n        r_h = NULL;\n        r = r->tail();\n        if (r) r_h = r->head();\n        continue;\n      }\n      #endif\n      // check for valid selectors\n      if (!l) return !!r;\n      if (!r) return false;\n      // both are null\n      else if (!l_h && !r_h)\n      {\n        // check combinator after heads\n        if (l->combinator() != r->combinator())\n        { return l->combinator() < r->combinator(); }\n        // advance to next tails\n        l = l->tail();\n        r = r->tail();\n        // fetch the next headers\n        l_h = NULL; r_h = NULL;\n        if (l) l_h = l->head();\n        if (r) r_h = r->head();\n      }\n      // one side is null\n      else if (!r_h) return true;\n      else if (!l_h) return false;\n      // heads ok and equal\n      else if (*l_h == *r_h)\n      {\n        // check combinator after heads\n        if (l->combinator() != r->combinator())\n        { return l->combinator() < r->combinator(); }\n        // advance to next tails\n        l = l->tail();\n        r = r->tail();\n        // fetch the next headers\n        l_h = NULL; r_h = NULL;\n        if (l) l_h = l->head();\n        if (r) r_h = r->head();\n      }\n      // heads are not equal\n      else return *l_h < *r_h;\n    }\n  }\n\n  bool Complex_Selector::operator== (const Complex_Selector& rhs) const\n  {\n    // const iterators for tails\n    Complex_Selector_Ptr_Const l = this;\n    Complex_Selector_Ptr_Const r = &rhs;\n    Compound_Selector_Ptr l_h = NULL;\n    Compound_Selector_Ptr r_h = NULL;\n    if (l) l_h = l->head();\n    if (r) r_h = r->head();\n    // process all tails\n    while (true)\n    {\n      #ifdef DEBUG\n      // skip empty ancestor first\n      if (l && l->is_empty_ancestor())\n      {\n        l_h = NULL;\n        l = l->tail();\n        if (l) l_h = l->head();\n        continue;\n      }\n      // skip empty ancestor first\n      if (r && r->is_empty_ancestor())\n      {\n        r_h = NULL;\n        r = r->tail();\n        if (r) r_h = r->head();\n        continue;\n      }\n      #endif\n      // check the pointers\n      if (!r) return !l;\n      if (!l) return !r;\n      // both are null\n      if (!l_h && !r_h)\n      {\n        // check combinator after heads\n        if (l->combinator() != r->combinator())\n        { return l->combinator() < r->combinator(); }\n        // advance to next tails\n        l = l->tail();\n        r = r->tail();\n        // fetch the next heads\n        l_h = NULL; r_h = NULL;\n        if (l) l_h = l->head();\n        if (r) r_h = r->head();\n      }\n      // equals if other head is empty\n      else if ((!l_h && !r_h) ||\n               (!l_h && r_h->empty()) ||\n               (!r_h && l_h->empty()) ||\n               (l_h && r_h && *l_h == *r_h))\n      {\n        // check combinator after heads\n        if (l->combinator() != r->combinator())\n        { return l->combinator() == r->combinator(); }\n        // advance to next tails\n        l = l->tail();\n        r = r->tail();\n        // fetch the next heads\n        l_h = NULL; r_h = NULL;\n        if (l) l_h = l->head();\n        if (r) r_h = r->head();\n      }\n      // abort\n      else break;\n    }\n    // unreachable\n    return false;\n  }\n\n  Compound_Selector_Ptr Compound_Selector::unify_with(Compound_Selector_Ptr rhs)\n  {\n    if (empty()) return rhs;\n    Compound_Selector_Obj unified = SASS_MEMORY_COPY(rhs);\n    for (size_t i = 0, L = length(); i < L; ++i)\n    {\n      if (unified.isNull()) break;\n      unified = at(i)->unify_with(unified);\n    }\n    return unified.detach();\n  }\n\n  bool Complex_Selector::operator== (const Selector& rhs) const\n  {\n    if (const Selector_List* sl = Cast<Selector_List>(&rhs)) return *this == *sl;\n    if (const Simple_Selector* sp = Cast<Simple_Selector>(&rhs)) return *this == *sp;\n    if (const Complex_Selector* cs = Cast<Complex_Selector>(&rhs)) return *this == *cs;\n    if (const Compound_Selector* ch = Cast<Compound_Selector>(&rhs)) return *this == *ch;\n    throw std::runtime_error(\"invalid selector base classes to compare\");\n  }\n\n\n  bool Complex_Selector::operator< (const Selector& rhs) const\n  {\n    if (const Selector_List* sl = Cast<Selector_List>(&rhs)) return *this < *sl;\n    if (const Simple_Selector* sp = Cast<Simple_Selector>(&rhs)) return *this < *sp;\n    if (const Complex_Selector* cs = Cast<Complex_Selector>(&rhs)) return *this < *cs;\n    if (const Compound_Selector* ch = Cast<Compound_Selector>(&rhs)) return *this < *ch;\n    throw std::runtime_error(\"invalid selector base classes to compare\");\n  }\n\n  bool Compound_Selector::operator== (const Selector& rhs) const\n  {\n    if (const Selector_List* sl = Cast<Selector_List>(&rhs)) return *this == *sl;\n    if (const Simple_Selector* sp = Cast<Simple_Selector>(&rhs)) return *this == *sp;\n    if (const Complex_Selector* cs = Cast<Complex_Selector>(&rhs)) return *this == *cs;\n    if (const Compound_Selector* ch = Cast<Compound_Selector>(&rhs)) return *this == *ch;\n    throw std::runtime_error(\"invalid selector base classes to compare\");\n  }\n\n  bool Compound_Selector::operator< (const Selector& rhs) const\n  {\n    if (const Selector_List* sl = Cast<Selector_List>(&rhs)) return *this < *sl;\n    if (const Simple_Selector* sp = Cast<Simple_Selector>(&rhs)) return *this < *sp;\n    if (const Complex_Selector* cs = Cast<Complex_Selector>(&rhs)) return *this < *cs;\n    if (const Compound_Selector* ch = Cast<Compound_Selector>(&rhs)) return *this < *ch;\n    throw std::runtime_error(\"invalid selector base classes to compare\");\n  }\n\n  bool Selector_Schema::operator== (const Selector& rhs) const\n  {\n    if (const Selector_List* sl = Cast<Selector_List>(&rhs)) return *this == *sl;\n    if (const Simple_Selector* sp = Cast<Simple_Selector>(&rhs)) return *this == *sp;\n    if (const Complex_Selector* cs = Cast<Complex_Selector>(&rhs)) return *this == *cs;\n    if (const Compound_Selector* ch = Cast<Compound_Selector>(&rhs)) return *this == *ch;\n    throw std::runtime_error(\"invalid selector base classes to compare\");\n  }\n\n  bool Selector_Schema::operator< (const Selector& rhs) const\n  {\n    if (const Selector_List* sl = Cast<Selector_List>(&rhs)) return *this < *sl;\n    if (const Simple_Selector* sp = Cast<Simple_Selector>(&rhs)) return *this < *sp;\n    if (const Complex_Selector* cs = Cast<Complex_Selector>(&rhs)) return *this < *cs;\n    if (const Compound_Selector* ch = Cast<Compound_Selector>(&rhs)) return *this < *ch;\n    throw std::runtime_error(\"invalid selector base classes to compare\");\n  }\n\n  bool Simple_Selector::operator== (const Selector& rhs) const\n  {\n    if (Simple_Selector_Ptr_Const sp = Cast<Simple_Selector>(&rhs)) return *this == *sp;\n    return false;\n  }\n\n  bool Simple_Selector::operator< (const Selector& rhs) const\n  {\n    if (Simple_Selector_Ptr_Const sp = Cast<Simple_Selector>(&rhs)) return *this < *sp;\n    return false;\n  }\n\n  bool Simple_Selector::operator== (const Simple_Selector& rhs) const\n  {\n    // solve the double dispatch problem by using RTTI information via dynamic cast\n    if (const Pseudo_Selector* lhs = Cast<Pseudo_Selector>(this)) {return *lhs == rhs; }\n    else if (const Wrapped_Selector* lhs = Cast<Wrapped_Selector>(this)) {return *lhs == rhs; }\n    else if (const Element_Selector* lhs = Cast<Element_Selector>(this)) {return *lhs == rhs; }\n    else if (const Attribute_Selector* lhs = Cast<Attribute_Selector>(this)) {return *lhs == rhs; }\n    else if (name_ == rhs.name_)\n    { return is_ns_eq(rhs); }\n    else return false;\n  }\n\n  bool Simple_Selector::operator< (const Simple_Selector& rhs) const\n  {\n    // solve the double dispatch problem by using RTTI information via dynamic cast\n    if (const Pseudo_Selector* lhs = Cast<Pseudo_Selector>(this)) {return *lhs < rhs; }\n    else if (const Wrapped_Selector* lhs = Cast<Wrapped_Selector>(this)) {return *lhs < rhs; }\n    else if (const Element_Selector* lhs = Cast<Element_Selector>(this)) {return *lhs < rhs; }\n    else if (const Attribute_Selector* lhs = Cast<Attribute_Selector>(this)) {return *lhs < rhs; }\n    if (is_ns_eq(rhs))\n    { return name_ < rhs.name_; }\n    return ns_ < rhs.ns_;\n  }\n\n  bool Selector_List::operator== (const Selector& rhs) const\n  {\n    // solve the double dispatch problem by using RTTI information via dynamic cast\n    if (Selector_List_Ptr_Const sl = Cast<Selector_List>(&rhs)) { return *this == *sl; }\n    else if (Complex_Selector_Ptr_Const cpx = Cast<Complex_Selector>(&rhs)) { return *this == *cpx; }\n    else if (Compound_Selector_Ptr_Const cpd = Cast<Compound_Selector>(&rhs)) { return *this == *cpd; }\n    // no compare method\n    return this == &rhs;\n  }\n\n  // Selector lists can be compared to comma lists\n  bool Selector_List::operator== (const Expression& rhs) const\n  {\n    // solve the double dispatch problem by using RTTI information via dynamic cast\n    if (List_Ptr_Const ls = Cast<List>(&rhs)) { return *ls == *this; }\n    if (Selector_Ptr_Const ls = Cast<Selector>(&rhs)) { return *this == *ls; }\n    // compare invalid (maybe we should error?)\n    return false;\n  }\n\n  bool Selector_List::operator== (const Selector_List& rhs) const\n  {\n    // for array access\n    size_t i = 0, n = 0;\n    size_t iL = length();\n    size_t nL = rhs.length();\n    // create temporary vectors and sort them\n    std::vector<Complex_Selector_Obj> l_lst = this->elements();\n...\n// Path: src/libsass/src/base64vlq.cpp\n#include \"sass.hpp\"\n#include \"base64vlq.hpp\"\n\nnamespace Sass {\n\n  std::string Base64VLQ::encode(const int number) const\n  {\n    std::string encoded = \"\";\n\n    int vlq = to_vlq_signed(number);\n\n    do {\n      int digit = vlq & VLQ_BASE_MASK;\n      vlq >>= VLQ_BASE_SHIFT;\n      if (vlq > 0) {\n        digit |= VLQ_CONTINUATION_BIT;\n      }\n      encoded += base64_encode(digit);\n    } while (vlq > 0);\n\n    return encoded;\n  }\n\n  char Base64VLQ::base64_encode(const int number) const\n  {\n    int index = number;\n    if (index < 0) index = 0;\n    if (index > 63) index = 63;\n    return CHARACTERS[index];\n  }\n\n  int Base64VLQ::to_vlq_signed(const int number) const\n  {\n    return (number < 0) ? ((-number) << 1) + 1 : (number << 1) + 0;\n  }\n\n  const char* Base64VLQ::CHARACTERS = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/\";\n\n  const int Base64VLQ::VLQ_BASE_SHIFT = 5;\n  const int Base64VLQ::VLQ_BASE = 1 << VLQ_BASE_SHIFT;\n  const int Base64VLQ::VLQ_BASE_MASK = VLQ_BASE - 1;\n  const int Base64VLQ::VLQ_CONTINUATION_BIT = VLQ_BASE;\n\n}\n\n// Path: src/libsass/src/paths.hpp\n#ifndef SASS_PATHS_H\n#define SASS_PATHS_H\n\n#include <string>\n#include <vector>\n#include <sstream>\n\n\ntemplate<typename T>\n\nstd::string vector_to_string(std::vector<T> v)\n{\n  std::stringstream buffer;\n  buffer << \"[\";\n\n  if (!v.empty())\n  {  buffer << v[0]; }\n  else\n  { buffer << \"]\"; }\n\n  if (v.size() == 1)\n  { buffer << \"]\"; }\n  else\n  {\n    for (size_t i = 1, S = v.size(); i < S; ++i) buffer << \", \" << v[i];\n    buffer << \"]\";\n  }\n\n  return buffer.str();\n}\n\nnamespace Sass {\n\n\n  template<typename T>\n  std::vector<std::vector<T> > paths(std::vector<std::vector<T> > strata, size_t from_end = 0)\n  {\n    if (strata.empty()) {\n      return std::vector<std::vector<T> >();\n    }\n\n    size_t end = strata.size() - from_end;\n    if (end <= 1) {\n      std::vector<std::vector<T> > starting_points;\n      starting_points.reserve(strata[0].size());\n      for (size_t i = 0, S = strata[0].size(); i < S; ++i) {\n        std::vector<T> starting_point;\n        starting_point.push_back(strata[0][i]);\n        starting_points.push_back(starting_point);\n      }\n      return starting_points;\n    }\n\n    std::vector<std::vector<T> > up_to_here = paths(strata, from_end + 1);\n    std::vector<T>          here       = strata[end-1];\n\n    std::vector<std::vector<T> > branches;\n    branches.reserve(up_to_here.size() * here.size());\n    for (size_t i = 0, S1 = up_to_here.size(); i < S1; ++i) {\n      for (size_t j = 0, S2 = here.size(); j < S2; ++j) {\n        std::vector<T> branch = up_to_here[i];\n        branch.push_back(here[j]);\n        branches.push_back(branch);\n      }\n    }\n\n    return branches;\n  }\n\n}\n\n#endif\n\n// Path: src/libsass/src/debug.hpp\n#ifndef SASS_DEBUG_H\n#define SASS_DEBUG_H\n\n#include <stdint.h>\n\n#ifndef UINT32_MAX\n  #define UINT32_MAX 0xffffffffU\n#endif\n\nenum dbg_lvl_t : uint32_t {\n  NONE = 0,\n  TRIM = 1,\n  CHUNKS = 2,\n  SUBWEAVE = 4,\n  WEAVE = 8,\n  EXTEND_COMPOUND = 16,\n  EXTEND_COMPLEX = 32,\n  LCS = 64,\n  EXTEND_OBJECT = 128,\n  ALL = UINT32_MAX\n};\n\n#ifdef DEBUG\n\n#ifndef DEBUG_LVL\nconst uint32_t debug_lvl = UINT32_MAX;\n#else\nconst uint32_t debug_lvl = (DEBUG_LVL);\n#endif // DEBUG_LVL\n\n#define DEBUG_PRINT(lvl, x) if((lvl) & debug_lvl) { std::cerr << x; }\n#define DEBUG_PRINTLN(lvl, x) if((lvl) & debug_lvl) { std::cerr << x << std::endl; }\n#define DEBUG_EXEC(lvl, x) if((lvl) & debug_lvl) { x; }\n\n#else // DEBUG\n\n#define DEBUG_PRINT(lvl, x)\n#define DEBUG_PRINTLN(lvl, x)\n#define DEBUG_EXEC(lvl, x)\n\n#endif // DEBUG\n\n#endif // SASS_DEBUG\n\n// Path: src/libsass/src/sass_util.hpp\n#ifndef SASS_SASS_UTIL_H\n#define SASS_SASS_UTIL_H\n\n#include \"ast.hpp\"\n#include \"node.hpp\"\n#include \"debug.hpp\"\n\nnamespace Sass {\n\n\n\n\n  /*\n   This is for ports of functions in the Sass:Util module.\n   */\n\n\n  /*\n    # Return a Node collection of all possible paths through the given Node collection of Node collections.\n    #\n    # @param arrs [NodeCollection<NodeCollection<Node>>]\n    # @return [NodeCollection<NodeCollection<Node>>]\n    #\n    # @example\n    #   paths([[1, 2], [3, 4], [5]]) #=>\n    #     # [[1, 3, 5],\n    #     #  [2, 3, 5],\n    #     #  [1, 4, 5],\n    #     #  [2, 4, 5]]\n  */\n  Node paths(const Node& arrs);\n\n\n  /*\n  This class is a default implementation of a Node comparator that can be passed to the lcs function below.\n  It uses operator== for equality comparision. It then returns one if the Nodes are equal.\n  */\n  class DefaultLcsComparator {\n  public:\n    bool operator()(const Node& one, const Node& two, Node& out) const {\n      // TODO: Is this the correct C++ interpretation?\n      // block ||= proc {|a, b| a == b && a}\n      if (one == two) {\n        out = one;\n        return true;\n      }\n\n      return false;\n    }\n  };\n\n\n  typedef std::vector<std::vector<int> > LCSTable;\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_backtrace.\n\n  # Computes a single longest common subsequence for arrays x and y.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Reading_out_an_LCS\n  */\n  template<typename ComparatorType>\n  Node lcs_backtrace(const LCSTable& c, const Node& x, const Node& y, int i, int j, const ComparatorType& comparator) {\n    DEBUG_PRINTLN(LCS, \"LCSBACK: X=\" << x << \" Y=\" << y << \" I=\" << i << \" J=\" << j)\n\n    if (i == 0 || j == 0) {\n      DEBUG_PRINTLN(LCS, \"RETURNING EMPTY\")\n      return Node::createCollection();\n    }\n\n    NodeDeque& xChildren = *(x.collection());\n    NodeDeque& yChildren = *(y.collection());\n\n    Node compareOut = Node::createNil();\n    if (comparator(xChildren[i], yChildren[j], compareOut)) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER ELEM COMPARE\")\n      Node result = lcs_backtrace(c, x, y, i - 1, j - 1, comparator);\n      result.collection()->push_back(compareOut);\n      return result;\n    }\n\n    if (c[i][j - 1] > c[i - 1][j]) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER TABLE COMPARE\")\n      return lcs_backtrace(c, x, y, i, j - 1, comparator);\n    }\n\n    DEBUG_PRINTLN(LCS, \"FINAL RETURN\")\n    return lcs_backtrace(c, x, y, i - 1, j, comparator);\n  }\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_table.\n\n  # Calculates the memoization table for the Least Common Subsequence algorithm.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Computing_the_length_of_the_LCS\n  */\n  template<typename ComparatorType>\n  void lcs_table(const Node& x, const Node& y, const ComparatorType& comparator, LCSTable& out) {\n    DEBUG_PRINTLN(LCS, \"LCSTABLE: X=\" << x << \" Y=\" << y)\n\n    NodeDeque& xChildren = *(x.collection());\n    NodeDeque& yChildren = *(y.collection());\n\n    LCSTable c(xChildren.size(), std::vector<int>(yChildren.size()));\n\n    // These shouldn't be necessary since the vector will be initialized to 0 already.\n    // x.size.times {|i| c[i][0] = 0}\n    // y.size.times {|j| c[0][j] = 0}\n\n    for (size_t i = 1; i < xChildren.size(); i++) {\n      for (size_t j = 1; j < yChildren.size(); j++) {\n        Node compareOut = Node::createNil();\n\n        if (comparator(xChildren[i], yChildren[j], compareOut)) {\n          c[i][j] = c[i - 1][j - 1] + 1;\n        } else {\n          c[i][j] = std::max(c[i][j - 1], c[i - 1][j]);\n        }\n      }\n    }\n\n    out = c;\n  }\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs.\n\n  # Computes a single longest common subsequence for `x` and `y`.\n  # If there are more than one longest common subsequences,\n  # the one returned is that which starts first in `x`.\n\n  # @param x [NodeCollection]\n  # @param y [NodeCollection]\n  # @comparator An equality check between elements of `x` and `y`.\n  # @return [NodeCollection] The LCS\n\n  http://en.wikipedia.org/wiki/Longest_common_subsequence_problem\n  */\n  template<typename ComparatorType>\n  Node lcs(Node& x, Node& y, const ComparatorType& comparator) {\n    DEBUG_PRINTLN(LCS, \"LCS: X=\" << x << \" Y=\" << y)\n\n    Node newX = Node::createCollection();\n    newX.collection()->push_back(Node::createNil());\n    newX.plus(x);\n\n    Node newY = Node::createCollection();\n    newY.collection()->push_back(Node::createNil());\n    newY.plus(y);\n\n    LCSTable table;\n    lcs_table(newX, newY, comparator, table);\n\n    return lcs_backtrace(table, newX, newY, static_cast<int>(newX.collection()->size()) - 1, static_cast<int>(newY.collection()->size()) - 1, comparator);\n  }\n\n\n  /*\n  This is the equivalent of ruby sass' Sass::Util.flatten and [].flatten.\n  Sass::Util.flatten requires the number of levels to flatten, while\n  [].flatten doesn't and will flatten the entire array. This function\n  supports both.\n\n  # Flattens the first `n` nested arrays. If n == -1, all arrays will be flattened\n  #\n  # @param arr [NodeCollection] The array to flatten\n  # @param n [int] The number of levels to flatten\n  # @return [NodeCollection] The flattened array\n  */\n  Node flatten(Node& arr, int n = -1);\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.group_by_to_a.\n\n  # Performs the equivalent of `enum.group_by.to_a`, but with a guaranteed\n  # order. Unlike [#hash_to_a], the resulting order isn't sorted key order;\n  # instead, it's the same order as `#group_by` has under Ruby 1.9 (key\n  # appearance order).\n  #\n  # @param enum [Enumerable]\n  # @return [Array<[Object, Array]>] An array of pairs.\n\n  TODO: update @param and @return once I know what those are.\n\n  The following is the modified version of the ruby code that was more portable to C++. You\n  should be able to drop it into ruby 3.2.19 and get the same results from ruby sass.\n\n    def group_by_to_a(enum, &block)\n      order = {}\n\n      arr = []\n\n      grouped = {}\n\n      for e in enum do\n        key = block[e]\n        unless order.include?(key)\n          order[key] = order.size\n        end\n\n        if not grouped.has_key?(key) then\n          grouped[key] = [e]\n        else\n          grouped[key].push(e)\n        end\n      end\n\n      grouped.each do |key, vals|\n        arr[order[key]] = [key, vals]\n      end\n\n      arr\n    end\n\n  */\n  template<typename EnumType, typename KeyType, typename KeyFunctorType>\n  void group_by_to_a(std::vector<EnumType>& enumeration, KeyFunctorType& keyFunc, std::vector<std::pair<KeyType, std::vector<EnumType> > >& arr /*out*/) {\n\n    std::map<unsigned int, KeyType> order;\n\n    std::map<size_t, std::vector<EnumType> > grouped;\n\n    for (typename std::vector<EnumType>::iterator enumIter = enumeration.begin(), enumIterEnd = enumeration.end(); enumIter != enumIterEnd; enumIter++) {\n      EnumType& e = *enumIter;\n\n      KeyType key = keyFunc(e);\n\n      if (grouped.find(key->hash()) == grouped.end()) {\n        order.insert(std::make_pair((unsigned int)order.size(), key));\n\n        std::vector<EnumType> newCollection;\n        newCollection.push_back(e);\n        grouped.insert(std::make_pair(key->hash(), newCollection));\n      } else {\n        std::vector<EnumType>& collection = grouped.at(key->hash());\n        collection.push_back(e);\n      }\n    }\n\n    for (unsigned int index = 0; index < order.size(); index++) {\n      KeyType& key = order.at(index);\n      std::vector<EnumType>& values = grouped.at(key->hash());\n\n      std::pair<KeyType, std::vector<EnumType> > grouping = std::make_pair(key, values);\n\n      arr.push_back(grouping);\n    }\n  }\n\n\n}\n\n#endif\n\n// Path: src/libsass/src/extend.cpp\n#include \"sass.hpp\"\n#include \"extend.hpp\"\n#include \"context.hpp\"\n#include \"backtrace.hpp\"\n#include \"paths.hpp\"\n#include \"parser.hpp\"\n#include \"expand.hpp\"\n#include \"node.hpp\"\n#include \"sass_util.hpp\"\n#include \"remove_placeholders.hpp\"\n#include \"debug.hpp\"\n#include <iostream>\n#include <deque>\n#include <set>\n\n/*\n NOTES:\n\n - The print* functions print to cerr. This allows our testing frameworks (like sass-spec) to ignore the output, which\n   is very helpful when debugging. The format of the output is mainly to wrap things in square brackets to match what\n   ruby already outputs (to make comparisons easier).\n\n - For the direct porting effort, we're trying to port method-for-method until we get all the tests passing.\n   Where applicable, I've tried to include the ruby code above the function for reference until all our tests pass.\n   The ruby code isn't always directly portable, so I've tried to include any modified ruby code that was actually\n   used for the porting.\n\n - DO NOT try to optimize yet. We get a tremendous benefit out of comparing the output of each stage of the extend to the ruby\n   output at the same stage. This makes it much easier to determine where problems are. Try to keep as close to\n   the ruby code as you can until we have all the sass-spec tests passing. Then, we should optimize. However, if you see\n   something that could probably be optimized, let's not forget it. Add a // TODO: or // IMPROVEMENT: comment.\n\n - Coding conventions in this file (these may need to be changed before merging back into master)\n   - Very basic hungarian notation:\n     p prefix for pointers (pSelector)\n     no prefix for value types and references (selector)\n   - Use STL iterators where possible\n   - prefer verbose naming over terse naming\n   - use typedefs for STL container types for make maintenance easier\n\n - You may see a lot of comments that say \"// TODO: is this the correct combinator?\". See the comment referring to combinators\n   in extendCompoundSelector for a more extensive explanation of my confusion. I think our divergence in data model from ruby\n   sass causes this to be necessary.\n\n\n GLOBAL TODOS:\n\n - wrap the contents of the print functions in DEBUG preprocesser conditionals so they will be optimized away in non-debug mode.\n\n - consider making the extend* functions member functions to avoid passing around ctx and subset_map map around. This has the\n   drawback that the implementation details of the operator are then exposed to the outside world, which is not ideal and\n   can cause additional compile time dependencies.\n\n - mark the helper methods in this file static to given them compilation unit linkage.\n\n - implement parent directive matching\n\n - fix compilation warnings for unused Extend members if we really don't need those references anymore.\n */\n\n\nnamespace Sass {\n\n\n\n#ifdef DEBUG\n\n  // TODO: move the ast specific ostream operators into ast.hpp/ast.cpp\n  std::ostream& operator<<(std::ostream& os, const Complex_Selector::Combinator combinator) {\n    switch (combinator) {\n      case Complex_Selector::ANCESTOR_OF: os << \"\\\" \\\"\"; break;\n      case Complex_Selector::PARENT_OF:   os << \"\\\">\\\"\"; break;\n      case Complex_Selector::PRECEDES:    os << \"\\\"~\\\"\"; break;\n      case Complex_Selector::ADJACENT_TO: os << \"\\\"+\\\"\"; break;\n      case Complex_Selector::REFERENCE:   os << \"\\\"/\\\"\"; break;\n    }\n\n    return os;\n  }\n\n\n  std::ostream& operator<<(std::ostream& os, Compound_Selector& compoundSelector) {\n    for (size_t i = 0, L = compoundSelector.length(); i < L; ++i) {\n      if (i > 0) os << \", \";\n      os << compoundSelector[i]->to_string();\n    }\n    return os;\n  }\n\n  std::ostream& operator<<(std::ostream& os, Simple_Selector& simpleSelector) {\n    os << simpleSelector.to_string();\n    return os;\n  }\n\n  // Print a string representation of a Compound_Selector\n  static void printSimpleSelector(Simple_Selector* pSimpleSelector, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    if (pSimpleSelector) {\n      std::cerr << \"[\" << *pSimpleSelector << \"]\";\n    } else {\n      std::cerr << \"NULL\";\n    }\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n  // Print a string representation of a Compound_Selector\n  static void printCompoundSelector(Compound_Selector_Ptr pCompoundSelector, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    if (pCompoundSelector) {\n      std::cerr << \"[\" << *pCompoundSelector << \"]\";\n    } else {\n      std::cerr << \"NULL\";\n    }\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n\n  std::ostream& operator<<(std::ostream& os, Complex_Selector& complexSelector) {\n\n    os << \"[\";\n    Complex_Selector_Ptr pIter = &complexSelector;\n    bool first = true;\n    while (pIter) {\n      if (pIter->combinator() != Complex_Selector::ANCESTOR_OF) {\n        if (!first) {\n          os << \", \";\n        }\n        first = false;\n        os << pIter->combinator();\n      }\n\n      if (!first) {\n        os << \", \";\n      }\n      first = false;\n\n      if (pIter->head()) {\n        os << pIter->head()->to_string();\n      } else {\n        os << \"NULL_HEAD\";\n      }\n\n      pIter = pIter->tail();\n    }\n    os << \"]\";\n\n    return os;\n  }\n\n\n  // Print a string representation of a Complex_Selector\n  static void printComplexSelector(Complex_Selector_Ptr pComplexSelector, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    if (pComplexSelector) {\n      std::cerr << *pComplexSelector;\n    } else {\n      std::cerr << \"NULL\";\n    }\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n  static void printSelsNewSeqPairCollection(SubSetMapLookups& collection, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n    bool first = true;\n    std::cerr << \"[\";\n    for(SubSetMapLookup& pair : collection) {\n      if (first) {\n        first = false;\n      } else {\n        std::cerr << \", \";\n      }\n      std::cerr << \"[\";\n      Compound_Selector_Ptr pSels = pair.first;\n      Complex_Selector_Ptr pNewSelector = pair.second;\n      std::cerr << \"[\" << *pSels << \"], \";\n      printComplexSelector(pNewSelector, NULL, false);\n    }\n    std::cerr << \"]\";\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n  // Print a string representation of a ComplexSelectorSet\n  static void printSourcesSet(ComplexSelectorSet& sources, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    // Convert to a deque of strings so we can sort since order doesn't matter in a set. This should cut down on\n    // the differences we see when debug printing.\n    typedef std::deque<std::string> SourceStrings;\n    SourceStrings sourceStrings;\n    for (ComplexSelectorSet::iterator iterator = sources.begin(), iteratorEnd = sources.end(); iterator != iteratorEnd; ++iterator) {\n      Complex_Selector_Ptr pSource = *iterator;\n      std::stringstream sstream;\n      sstream << complexSelectorToNode(pSource);\n      sourceStrings.push_back(sstream.str());\n    }\n\n    // Sort to get consistent output\n    std::sort(sourceStrings.begin(), sourceStrings.end());\n\n    std::cerr << \"ComplexSelectorSet[\";\n    for (SourceStrings::iterator iterator = sourceStrings.begin(), iteratorEnd = sourceStrings.end(); iterator != iteratorEnd; ++iterator) {\n      std::string source = *iterator;\n      if (iterator != sourceStrings.begin()) {\n        std::cerr << \", \";\n      }\n      std::cerr << source;\n    }\n    std::cerr << \"]\";\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n\n  std::ostream& operator<<(std::ostream& os, SubSetMapPairs& entries) {\n    os << \"SUBSET_MAP_ENTRIES[\";\n\n    for (SubSetMapPairs::iterator iterator = entries.begin(), endIterator = entries.end(); iterator != endIterator; ++iterator) {\n      Complex_Selector_Obj pExtComplexSelector = iterator->first;    // The selector up to where the @extend is (ie, the thing to merge)\n      Compound_Selector_Obj pExtCompoundSelector = iterator->second; // The stuff after the @extend\n\n      if (iterator != entries.begin()) {\n        os << \", \";\n      }\n\n      os << \"(\";\n\n      if (pExtComplexSelector) {\n        std::cerr << *pExtComplexSelector;\n      } else {\n        std::cerr << \"NULL\";\n      }\n\n      os << \" -> \";\n\n      if (pExtCompoundSelector) {\n        std::cerr << *pExtCompoundSelector;\n      } else {\n        std::cerr << \"NULL\";\n      }\n\n      os << \")\";\n\n    }\n\n    os << \"]\";\n\n    return os;\n  }\n#endif\n\n  static bool parentSuperselector(Complex_Selector_Ptr pOne, Complex_Selector_Ptr pTwo) {\n    // TODO: figure out a better way to create a Complex_Selector from scratch\n    // TODO: There's got to be a better way. This got ugly quick...\n    Element_Selector_Obj fakeParent = SASS_MEMORY_NEW(Element_Selector, ParserState(\"[FAKE]\"), \"temp\");\n    Compound_Selector_Obj fakeHead = SASS_MEMORY_NEW(Compound_Selector, ParserState(\"[FAKE]\"), 1 /*size*/);\n    fakeHead->elements().push_back(fakeParent);\n    Complex_Selector_Obj fakeParentContainer = SASS_MEMORY_NEW(Complex_Selector, ParserState(\"[FAKE]\"), Complex_Selector::ANCESTOR_OF, fakeHead /*head*/, NULL /*tail*/);\n\n    pOne->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n    pTwo->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n\n    bool isSuperselector = pOne->is_superselector_of(pTwo);\n\n    pOne->clear_innermost();\n    pTwo->clear_innermost();\n\n    return isSuperselector;\n  }\n\n  void nodeToComplexSelectorDeque(const Node& node, ComplexSelectorDeque& out) {\n    for (NodeDeque::iterator iter = node.collection()->begin(), iterEnd = node.collection()->end(); iter != iterEnd; iter++) {\n      Node& child = *iter;\n      out.push_back(nodeToComplexSelector(child));\n    }\n  }\n\n  Node complexSelectorDequeToNode(const ComplexSelectorDeque& deque) {\n    Node result = Node::createCollection();\n\n    for (ComplexSelectorDeque::const_iterator iter = deque.begin(), iterEnd = deque.end(); iter != iterEnd; iter++) {\n      Complex_Selector_Obj pChild = *iter;\n      result.collection()->push_back(complexSelectorToNode(pChild));\n    }\n\n    return result;\n  }\n\n  class LcsCollectionComparator {\n  public:\n    LcsCollectionComparator() {}\n\n    bool operator()(Complex_Selector_Obj pOne, Complex_Selector_Obj pTwo, Complex_Selector_Obj& pOut) const {\n      /*\n      This code is based on the following block from ruby sass' subweave\n        do |s1, s2|\n          next s1 if s1 == s2\n          next unless s1.first.is_a?(SimpleSequence) && s2.first.is_a?(SimpleSequence)\n          next s2 if parent_superselector?(s1, s2)\n          next s1 if parent_superselector?(s2, s1)\n        end\n      */\n\n      if (*pOne == *pTwo) {\n        pOut = pOne;\n        return true;\n      }\n\n      if (pOne->combinator() != Complex_Selector::ANCESTOR_OF || pTwo->combinator() != Complex_Selector::ANCESTOR_OF) {\n        return false;\n      }\n\n      if (parentSuperselector(pOne, pTwo)) {\n        pOut = pTwo;\n        return true;\n      }\n\n      if (parentSuperselector(pTwo, pOne)) {\n        pOut = pOne;\n        return true;\n      }\n\n      return false;\n    }\n  };\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_backtrace.\n\n  # Computes a single longest common subsequence for arrays x and y.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Reading_out_an_LCS\n  */\n  void lcs_backtrace(const LCSTable& c, ComplexSelectorDeque& x, ComplexSelectorDeque& y, int i, int j, const LcsCollectionComparator& comparator, ComplexSelectorDeque& out) {\n    //DEBUG_PRINTLN(LCS, \"LCSBACK: X=\" << x << \" Y=\" << y << \" I=\" << i << \" J=\" << j)\n    // TODO: make printComplexSelectorDeque and use DEBUG_EXEC AND DEBUG_PRINTLN HERE to get equivalent output\n\n    if (i == 0 || j == 0) {\n      DEBUG_PRINTLN(LCS, \"RETURNING EMPTY\")\n      return;\n    }\n\n\n    Complex_Selector_Obj pCompareOut;\n    if (comparator(x[i], y[j], pCompareOut)) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER ELEM COMPARE\")\n      lcs_backtrace(c, x, y, i - 1, j - 1, comparator, out);\n      out.push_back(pCompareOut);\n      return;\n    }\n\n    if (c[i][j - 1] > c[i - 1][j]) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER TABLE COMPARE\")\n      lcs_backtrace(c, x, y, i, j - 1, comparator, out);\n      return;\n    }\n\n    DEBUG_PRINTLN(LCS, \"FINAL RETURN\")\n    lcs_backtrace(c, x, y, i - 1, j, comparator, out);\n    return;\n  }\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_table.\n\n  # Calculates the memoization table for the Least Common Subsequence algorithm.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Computing_the_length_of_the_LCS\n  */\n  void lcs_table(const ComplexSelectorDeque& x, const ComplexSelectorDeque& y, const LcsCollectionComparator& comparator, LCSTable& out) {\n    //DEBUG_PRINTLN(LCS, \"LCSTABLE: X=\" << x << \" Y=\" << y)\n    // TODO: make printComplexSelectorDeque and use DEBUG_EXEC AND DEBUG_PRINTLN HERE to get equivalent output\n\n    LCSTable c(x.size(), std::vector<int>(y.size()));\n\n    // These shouldn't be necessary since the vector will be initialized to 0 already.\n    // x.size.times {|i| c[i][0] = 0}\n    // y.size.times {|j| c[0][j] = 0}\n\n    for (size_t i = 1; i < x.size(); i++) {\n      for (size_t j = 1; j < y.size(); j++) {\n        Complex_Selector_Obj pCompareOut;\n\n        if (comparator(x[i], y[j], pCompareOut)) {\n          c[i][j] = c[i - 1][j - 1] + 1;\n        } else {\n          c[i][j] = std::max(c[i][j - 1], c[i - 1][j]);\n        }\n      }\n    }\n\n    out = c;\n  }\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs.\n\n  # Computes a single longest common subsequence for `x` and `y`.\n  # If there are more than one longest common subsequences,\n  # the one returned is that which starts first in `x`.\n\n  # @param x [NodeCollection]\n  # @param y [NodeCollection]\n  # @comparator An equality check between elements of `x` and `y`.\n  # @return [NodeCollection] The LCS\n\n  http://en.wikipedia.org/wiki/Longest_common_subsequence_problem\n  */\n  void lcs(ComplexSelectorDeque& x, ComplexSelectorDeque& y, const LcsCollectionComparator& comparator, ComplexSelectorDeque& out) {\n    //DEBUG_PRINTLN(LCS, \"LCS: X=\" << x << \" Y=\" << y)\n    // TODO: make printComplexSelectorDeque and use DEBUG_EXEC AND DEBUG_PRINTLN HERE to get equivalent output\n\n    x.push_front(NULL);\n    y.push_front(NULL);\n\n    LCSTable table;\n    lcs_table(x, y, comparator, table);\n\n    return lcs_backtrace(table, x, y, static_cast<int>(x.size()) - 1, static_cast<int>(y.size()) - 1, comparator, out);\n  }\n\n\n  /*\n   This is the equivalent of ruby's Sequence.trim.\n\n   The following is the modified version of the ruby code that was more portable to C++. You\n   should be able to drop it into ruby 3.2.19 and get the same results from ruby sass.\n\n        # Avoid truly horrific quadratic behavior. TODO: I think there\n        # may be a way to get perfect trimming without going quadratic.\n        return seqses if seqses.size > 100\n\n        # Keep the results in a separate array so we can be sure we aren't\n        # comparing against an already-trimmed selector. This ensures that two\n        # identical selectors don't mutually trim one another.\n        result = seqses.dup\n\n        # This is n^2 on the sequences, but only comparing between\n        # separate sequences should limit the quadratic behavior.\n        seqses.each_with_index do |seqs1, i|\n          tempResult = []\n\n          for seq1 in seqs1 do\n            max_spec = 0\n            for seq in _sources(seq1) do\n              max_spec = [max_spec, seq.specificity].max\n            end\n\n\n            isMoreSpecificOuter = false\n            for seqs2 in result do\n              if seqs1.equal?(seqs2) then\n                next\n              end\n\n              # Second Law of Extend: the specificity of a generated selector\n              # should never be less than the specificity of the extending\n              # selector.\n              #\n              # See https://github.com/nex3/sass/issues/324.\n              isMoreSpecificInner = false\n              for seq2 in seqs2 do\n                isMoreSpecificInner = _specificity(seq2) >= max_spec && _superselector?(seq2, seq1)\n                if isMoreSpecificInner then\n                  break\n                end\n              end\n\n              if isMoreSpecificInner then\n                isMoreSpecificOuter = true\n                break\n              end\n            end\n\n            if !isMoreSpecificOuter then\n              tempResult.push(seq1)\n            end\n          end\n\n          result[i] = tempResult\n\n        end\n\n        result\n   */\n  /*\n   - IMPROVEMENT: We could probably work directly in the output trimmed deque.\n   */\n  Node Extend::trim(Node& seqses, bool isReplace) {\n    // See the comments in the above ruby code before embarking on understanding this function.\n\n    // Avoid poor performance in extreme cases.\n    if (seqses.collection()->size() > 100) {\n      return seqses;\n    }\n\n\n    DEBUG_PRINTLN(TRIM, \"TRIM: \" << seqses)\n\n\n    Node result = Node::createCollection();\n    result.plus(seqses);\n\n    DEBUG_PRINTLN(TRIM, \"RESULT INITIAL: \" << result)\n\n    // Normally we use the standard STL iterators, but in this case, we need to access the result collection by index since we're\n    // iterating the input collection, computing a value, and then setting the result in the output collection. We have to keep track\n    // of the index manually.\n    int toTrimIndex = 0;\n\n    for (NodeDeque::iterator seqsesIter = seqses.collection()->begin(), seqsesIterEnd = seqses.collection()->end(); seqsesIter != seqsesIterEnd; ++seqsesIter) {\n      Node& seqs1 = *seqsesIter;\n\n      DEBUG_PRINTLN(TRIM, \"SEQS1: \" << seqs1 << \" \" << toTrimIndex)\n\n      Node tempResult = Node::createCollection();\n      tempResult.got_line_feed = seqs1.got_line_feed;\n\n      for (NodeDeque::iterator seqs1Iter = seqs1.collection()->begin(), seqs1EndIter = seqs1.collection()->end(); seqs1Iter != seqs1EndIter; ++seqs1Iter) {\n        Node& seq1 = *seqs1Iter;\n\n        Complex_Selector_Obj pSeq1 = nodeToComplexSelector(seq1);\n\n        // Compute the maximum specificity. This requires looking at the \"sources\" of the sequence. See SimpleSequence.sources in the ruby code\n        // for a good description of sources.\n        //\n        // TODO: I'm pretty sure there's a bug in the sources code. It was implemented for sass-spec's 182_test_nested_extend_loop test.\n        // While the test passes, I compared the state of each trim call to verify correctness. The last trim call had incorrect sources. We\n        // had an extra source that the ruby version did not have. Without a failing test case, this is going to be extra hard to find. My\n        // best guess at this point is that we're cloning an object somewhere and maintaining the sources when we shouldn't be. This is purely\n        // a guess though.\n        unsigned long maxSpecificity = isReplace ? pSeq1->specificity() : 0;\n        ComplexSelectorSet sources = pSeq1->sources();\n\n        DEBUG_PRINTLN(TRIM, \"TRIM SEQ1: \" << seq1)\n        DEBUG_EXEC(TRIM, printSourcesSet(sources, \"TRIM SOURCES: \"))\n\n        for (ComplexSelectorSet::iterator sourcesSetIterator = sources.begin(), sourcesSetIteratorEnd = sources.end(); sourcesSetIterator != sourcesSetIteratorEnd; ++sourcesSetIterator) {\n          const Complex_Selector_Obj& pCurrentSelector = *sourcesSetIterator;\n          maxSpecificity = std::max(maxSpecificity, pCurrentSelector->specificity());\n        }\n\n        DEBUG_PRINTLN(TRIM, \"MAX SPECIFICITY: \" << maxSpecificity)\n\n        bool isMoreSpecificOuter = false;\n\n        int resultIndex = 0;\n\n        for (NodeDeque::iterator resultIter = result.collection()->begin(), resultIterEnd = result.collection()->end(); resultIter != resultIterEnd; ++resultIter) {\n          Node& seqs2 = *resultIter;\n\n          DEBUG_PRINTLN(TRIM, \"SEQS1: \" << seqs1)\n          DEBUG_PRINTLN(TRIM, \"SEQS2: \" << seqs2)\n\n          // Do not compare the same sequence to itself. The ruby call we're trying to\n          // emulate is: seqs1.equal?(seqs2). equal? is an object comparison, not an equivalency comparision.\n          // Since we have the same pointers in seqes and results, we can do a pointer comparision. seqs1 is\n          // derived from seqses and seqs2 is derived from result.\n          if (seqs1.collection() == seqs2.collection()) {\n            DEBUG_PRINTLN(TRIM, \"CONTINUE\")\n            continue;\n          }\n\n          bool isMoreSpecificInner = false;\n\n          for (NodeDeque::iterator seqs2Iter = seqs2.collection()->begin(), seqs2IterEnd = seqs2.collection()->end(); seqs2Iter != seqs2IterEnd; ++seqs2Iter) {\n            Node& seq2 = *seqs2Iter;\n\n            Complex_Selector_Obj pSeq2 = nodeToComplexSelector(seq2);\n\n            DEBUG_PRINTLN(TRIM, \"SEQ2 SPEC: \" << pSeq2->specificity())\n            DEBUG_PRINTLN(TRIM, \"IS SPEC: \" << pSeq2->specificity() << \" >= \" << maxSpecificity << \" \" << (pSeq2->specificity() >= maxSpecificity ? \"true\" : \"false\"))\n            DEBUG_PRINTLN(TRIM, \"IS SUPER: \" << (pSeq2->is_superselector_of(pSeq1) ? \"true\" : \"false\"))\n\n            isMoreSpecificInner = pSeq2->specificity() >= maxSpecificity && pSeq2->is_superselector_of(pSeq1);\n\n            if (isMoreSpecificInner) {\n              DEBUG_PRINTLN(TRIM, \"FOUND MORE SPECIFIC\")\n              break;\n            }\n          }\n\n          // If we found something more specific, we're done. Let the outer loop know and stop iterating.\n          if (isMoreSpecificInner) {\n            isMoreSpecificOuter = true;\n            break;\n          }\n\n          resultIndex++;\n        }\n\n        if (!isMoreSpecificOuter) {\n...\n", "needle_token_start": 5601, "needle_token_end": 5748, "code_context_ntokens": 16070, "output": ["```cpp\n#include <string>\n#include <vector>\n#include <sstream>\n\ntemplate<typename T>\nstd::string vector_to_string(std::vector<T> v)\n{\n  std::stringstream buffer;\n  buffer << \"[\";\n\n  if (!v.empty())\n  {  buffer << v[0]; }\n  else\n  { buffer << \"]\"; }\n\n  if (v.size() == 1)\n  { buffer << \"]\"; }\n  else\n  {\n    for (size_t i = 1, S = v.size(); i < S; ++i) buffer << \", \" << v[i];\n    buffer << \"]\";\n  }\n\n  return buffer.str();\n}\n```"]}
{"repo": "sass/node-sass", "name": "debug_subset_map", "language": "cpp", "path": "src/libsass/src/debugger.hpp", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to output a detailed textual representation of a mapping structure used in a style sheet language processor. This helps in debugging by showing how different abstract syntax tree (AST) nodes are related within the map.\n2. **Input**: The function accepts a mapping structure that associates AST nodes with each other, typically representing relationships or transformations in a stylesheet.\n3. **Output**: There is no return value; instead, the function outputs to the standard error stream. It prints a detailed breakdown of each key-value pair in the mapping, showing the structure and content of the associated AST nodes.\n4. **Procedure**: The function begins by checking if the indentation level is at the base level and, if so, prints a start marker. It then iterates over each entry in the map, calling another function to print the AST nodes that form the key and the value of each entry with appropriate indentation. After iterating through all entries, if the indentation level is still at the base, it prints an end marker.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    std::cerr << \" [@media:\" << block->media_block() << \"]\";\n    std::cerr << \" \" << block->tabs() << std::endl;\n  } else if (Cast<Import_Stub>(node)) {\n    Import_Stub_Ptr block = Cast<Import_Stub>(node);\n    std::cerr << ind << \"Import_Stub \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << block->imp_path() << \"] \";\n    std::cerr << \" \" << block->tabs() << std::endl;\n  } else if (Cast<Import>(node)) {\n    Import_Ptr block = Cast<Import>(node);\n    std::cerr << ind << \"Import \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << block->tabs() << std::endl;\n    // std::vector<std::string>         files_;\n    for (auto imp : block->urls()) debug_ast(imp, ind + \"@: \", env);\n    debug_ast(block->import_queries(), ind + \"@@ \");\n  } else if (Cast<Assignment>(node)) {\n    Assignment_Ptr block = Cast<Assignment>(node);\n    std::cerr << ind << \"Assignment \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" <<\" << block->variable() << \">> \" << block->tabs() << std::endl;\n    debug_ast(block->value(), ind + \"=\", env);\n  } else if (Cast<Declaration>(node)) {\n    Declaration_Ptr block = Cast<Declaration>(node);\n    std::cerr << ind << \"Declaration \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [is_custom_property: \" << block->is_custom_property() << \"] \";\n    std::cerr << \" \" << block->tabs() << std::endl;\n    debug_ast(block->property(), ind + \" prop: \", env);\n    debug_ast(block->value(), ind + \" value: \", env);\n    debug_ast(block->block(), ind + \" \", env);\n  } else if (Cast<Keyframe_Rule>(node)) {\n    Keyframe_Rule_Ptr has_block = Cast<Keyframe_Rule>(node);\n    std::cerr << ind << \"Keyframe_Rule \" << has_block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << has_block->tabs() << std::endl;\n    if (has_block->name()) debug_ast(has_block->name(), ind + \"@\");\n    if (has_block->block()) for(const Statement_Obj& i : has_block->block()->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Directive>(node)) {\n    Directive_Ptr block = Cast<Directive>(node);\n    std::cerr << ind << \"Directive \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << block->keyword() << \"] \" << block->tabs() << std::endl;\n    debug_ast(block->selector(), ind + \"~\", env);\n    debug_ast(block->value(), ind + \"+\", env);\n    if (block->block()) for(const Statement_Obj& i : block->block()->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Each>(node)) {\n    Each_Ptr block = Cast<Each>(node);\n    std::cerr << ind << \"Each \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << block->tabs() << std::endl;\n    if (block->block()) for(const Statement_Obj& i : block->block()->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<For>(node)) {\n    For_Ptr block = Cast<For>(node);\n    std::cerr << ind << \"For \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << block->tabs() << std::endl;\n    if (block->block()) for(const Statement_Obj& i : block->block()->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<While>(node)) {\n    While_Ptr block = Cast<While>(node);\n    std::cerr << ind << \"While \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << block->tabs() << std::endl;\n    if (block->block()) for(const Statement_Obj& i : block->block()->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Definition>(node)) {\n    Definition_Ptr block = Cast<Definition>(node);\n    std::cerr << ind << \"Definition \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [name: \" << block->name() << \"] \";\n    std::cerr << \" [type: \" << (block->type() == Sass::Definition::Type::MIXIN ? \"Mixin \" : \"Function \") << \"] \";\n    // this seems to lead to segfaults some times?\n    // std::cerr << \" [signature: \" << block->signature() << \"] \";\n    std::cerr << \" [native: \" << block->native_function() << \"] \";\n    std::cerr << \" \" << block->tabs() << std::endl;\n    debug_ast(block->parameters(), ind + \" params: \", env);\n    if (block->block()) debug_ast(block->block(), ind + \" \", env);\n  } else if (Cast<Mixin_Call>(node)) {\n    Mixin_Call_Ptr block = Cast<Mixin_Call>(node);\n    std::cerr << ind << \"Mixin_Call \" << block << \" \" << block->tabs();\n    std::cerr << \" (\" << pstate_source_position(block) << \")\";\n    std::cerr << \" [\" <<  block->name() << \"]\";\n    std::cerr << \" [has_content: \" << block->has_content() << \"] \" << std::endl;\n    debug_ast(block->arguments(), ind + \" args: \");\n    if (block->block()) debug_ast(block->block(), ind + \" \", env);\n  } else if (Ruleset_Ptr ruleset = Cast<Ruleset>(node)) {\n    std::cerr << ind << \"Ruleset \" << ruleset;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [indent: \" << ruleset->tabs() << \"]\";\n    std::cerr << (ruleset->is_invisible() ? \" [INVISIBLE]\" : \"\");\n    std::cerr << (ruleset->is_root() ? \" [root]\" : \"\");\n    std::cerr << std::endl;\n    debug_ast(ruleset->selector(), ind + \">\");\n    debug_ast(ruleset->block(), ind + \" \");\n  } else if (Cast<Block>(node)) {\n    Block_Ptr block = Cast<Block>(node);\n    std::cerr << ind << \"Block \" << block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << (block->is_invisible() ? \" [INVISIBLE]\" : \"\");\n    std::cerr << \" [indent: \" << block->tabs() << \"]\" << std::endl;\n    for(const Statement_Obj& i : block->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Variable>(node)) {\n    Variable_Ptr expression = Cast<Variable>(node);\n    std::cerr << ind << \"Variable \" << expression;\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << expression->name() << \"]\" << std::endl;\n    std::string name(expression->name());\n    if (env && env->has(name)) debug_ast(Cast<Expression>((*env)[name]), ind + \" -> \", env);\n  } else if (Cast<Function_Call_Schema>(node)) {\n    Function_Call_Schema_Ptr expression = Cast<Function_Call_Schema>(node);\n    std::cerr << ind << \"Function_Call_Schema \" << expression;\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \"\" << std::endl;\n    debug_ast(expression->name(), ind + \"name: \", env);\n    debug_ast(expression->arguments(), ind + \" args: \", env);\n  } else if (Cast<Function_Call>(node)) {\n    Function_Call_Ptr expression = Cast<Function_Call>(node);\n    std::cerr << ind << \"Function_Call \" << expression;\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << expression->name() << \"]\";\n    if (expression->is_delayed()) std::cerr << \" [delayed]\";\n    if (expression->is_interpolant()) std::cerr << \" [interpolant]\";\n    if (expression->is_css()) std::cerr << \" [css]\";\n    std::cerr << std::endl;\n    debug_ast(expression->arguments(), ind + \" args: \", env);\n    debug_ast(expression->func(), ind + \" func: \", env);\n  } else if (Cast<Function>(node)) {\n    Function_Ptr expression = Cast<Function>(node);\n    std::cerr << ind << \"Function \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    if (expression->is_css()) std::cerr << \" [css]\";\n    std::cerr << std::endl;\n    debug_ast(expression->definition(), ind + \" definition: \", env);\n  } else if (Cast<Arguments>(node)) {\n    Arguments_Ptr expression = Cast<Arguments>(node);\n    std::cerr << ind << \"Arguments \" << expression;\n    if (expression->is_delayed()) std::cerr << \" [delayed]\";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    if (expression->has_named_arguments()) std::cerr << \" [has_named_arguments]\";\n    if (expression->has_rest_argument()) std::cerr << \" [has_rest_argument]\";\n    if (expression->has_keyword_argument()) std::cerr << \" [has_keyword_argument]\";\n    std::cerr << std::endl;\n    for(const Argument_Obj& i : expression->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Argument>(node)) {\n    Argument_Ptr expression = Cast<Argument>(node);\n    std::cerr << ind << \"Argument \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << expression->value().ptr() << \"]\";\n    std::cerr << \" [name: \" << expression->name() << \"] \";\n    std::cerr << \" [rest: \" << expression->is_rest_argument() << \"] \";\n    std::cerr << \" [keyword: \" << expression->is_keyword_argument() << \"] \" << std::endl;\n    debug_ast(expression->value(), ind + \" value: \", env);\n  } else if (Cast<Parameters>(node)) {\n    Parameters_Ptr expression = Cast<Parameters>(node);\n    std::cerr << ind << \"Parameters \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [has_optional: \" << expression->has_optional_parameters() << \"] \";\n    std::cerr << \" [has_rest: \" << expression->has_rest_parameter() << \"] \";\n    std::cerr << std::endl;\n    for(const Parameter_Obj& i : expression->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Parameter>(node)) {\n    Parameter_Ptr expression = Cast<Parameter>(node);\n    std::cerr << ind << \"Parameter \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [name: \" << expression->name() << \"] \";\n    std::cerr << \" [default: \" << expression->default_value().ptr() << \"] \";\n    std::cerr << \" [rest: \" << expression->is_rest_parameter() << \"] \" << std::endl;\n  } else if (Cast<Unary_Expression>(node)) {\n    Unary_Expression_Ptr expression = Cast<Unary_Expression>(node);\n    std::cerr << ind << \"Unary_Expression \" << expression;\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" [delayed: \" << expression->is_delayed() << \"] \";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << expression->type() << \"]\" << std::endl;\n    debug_ast(expression->operand(), ind + \" operand: \", env);\n  } else if (Cast<Binary_Expression>(node)) {\n    Binary_Expression_Ptr expression = Cast<Binary_Expression>(node);\n    std::cerr << ind << \"Binary_Expression \" << expression;\n    if (expression->is_interpolant()) std::cerr << \" [is interpolant] \";\n    if (expression->is_left_interpolant()) std::cerr << \" [left interpolant] \";\n    if (expression->is_right_interpolant()) std::cerr << \" [right interpolant] \";\n    std::cerr << \" [delayed: \" << expression->is_delayed() << \"] \";\n    std::cerr << \" [ws_before: \" << expression->op().ws_before << \"] \";\n    std::cerr << \" [ws_after: \" << expression->op().ws_after << \"] \";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << expression->type_name() << \"]\" << std::endl;\n    debug_ast(expression->left(), ind + \" left:  \", env);\n    debug_ast(expression->right(), ind + \" right: \", env);\n  } else if (Cast<Map>(node)) {\n    Map_Ptr expression = Cast<Map>(node);\n    std::cerr << ind << \"Map \" << expression;\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [Hashed]\" << std::endl;\n    for (const auto& i : expression->elements()) {\n      debug_ast(i.first, ind + \" key: \");\n      debug_ast(i.second, ind + \" val: \");\n    }\n  } else if (Cast<List>(node)) {\n    List_Ptr expression = Cast<List>(node);\n    std::cerr << ind << \"List \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" (\" << expression->length() << \") \" <<\n      (expression->separator() == SASS_COMMA ? \"Comma \" : expression->separator() == SASS_HASH ? \"Map \" : \"Space \") <<\n      \" [delayed: \" << expression->is_delayed() << \"] \" <<\n      \" [interpolant: \" << expression->is_interpolant() << \"] \" <<\n      \" [listized: \" << expression->from_selector() << \"] \" <<\n      \" [arglist: \" << expression->is_arglist() << \"] \" <<\n      \" [bracketed: \" << expression->is_bracketed() << \"] \" <<\n      \" [expanded: \" << expression->is_expanded() << \"] \" <<\n      \" [hash: \" << expression->hash() << \"] \" <<\n      std::endl;\n    for(const auto& i : expression->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Content>(node)) {\n    Content_Ptr expression = Cast<Content>(node);\n    std::cerr << ind << \"Content \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [@media:\" << expression->media_block() << \"]\";\n    std::cerr << \" [Statement]\" << std::endl;\n  } else if (Cast<Boolean>(node)) {\n    Boolean_Ptr expression = Cast<Boolean>(node);\n    std::cerr << ind << \"Boolean \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" [\" << expression->value() << \"]\" << std::endl;\n  } else if (Cast<Color>(node)) {\n    Color_Ptr expression = Cast<Color>(node);\n    std::cerr << ind << \"Color \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [delayed: \" << expression->is_delayed() << \"] \";\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" [\" << expression->r() << \":\"  << expression->g() << \":\" << expression->b() << \"@\" << expression->a() << \"]\" << std::endl;\n  } else if (Cast<Number>(node)) {\n    Number_Ptr expression = Cast<Number>(node);\n    std::cerr << ind << \"Number \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [delayed: \" << expression->is_delayed() << \"] \";\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \";\n    std::cerr << \" [\" << expression->value() << expression->unit() << \"]\" <<\n      \" [hash: \" << expression->hash() << \"] \" <<\n      std::endl;\n  } else if (Cast<Null>(node)) {\n    Null_Ptr expression = Cast<Null>(node);\n    std::cerr << ind << \"Null \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [interpolant: \" << expression->is_interpolant() << \"] \"\n      // \" [hash: \" << expression->hash() << \"] \"\n      << std::endl;\n  } else if (Cast<String_Quoted>(node)) {\n    String_Quoted_Ptr expression = Cast<String_Quoted>(node);\n    std::cerr << ind << \"String_Quoted \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << prettyprint(expression->value()) << \"]\";\n    if (expression->is_delayed()) std::cerr << \" [delayed]\";\n    if (expression->is_interpolant()) std::cerr << \" [interpolant]\";\n    if (expression->quote_mark()) std::cerr << \" [quote_mark: \" << expression->quote_mark() << \"]\";\n    std::cerr << \" <\" << prettyprint(expression->pstate().token.ws_before()) << \">\" << std::endl;\n  } else if (Cast<String_Constant>(node)) {\n    String_Constant_Ptr expression = Cast<String_Constant>(node);\n    std::cerr << ind << \"String_Constant \" << expression;\n    if (expression->concrete_type()) {\n      std::cerr << \" \" << expression->concrete_type();\n    }\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" [\" << prettyprint(expression->value()) << \"]\";\n    if (expression->is_delayed()) std::cerr << \" [delayed]\";\n    if (expression->is_interpolant()) std::cerr << \" [interpolant]\";\n    std::cerr << \" <\" << prettyprint(expression->pstate().token.ws_before()) << \">\" << std::endl;\n  } else if (Cast<String_Schema>(node)) {\n    String_Schema_Ptr expression = Cast<String_Schema>(node);\n    std::cerr << ind << \"String_Schema \" << expression;\n    std::cerr << \" (\" << pstate_source_position(expression) << \")\";\n    std::cerr << \" \" << expression->concrete_type();\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    if (expression->css()) std::cerr << \" [css]\";\n    if (expression->is_delayed()) std::cerr << \" [delayed]\";\n    if (expression->is_interpolant()) std::cerr << \" [is interpolant]\";\n    if (expression->has_interpolant()) std::cerr << \" [has interpolant]\";\n    if (expression->is_left_interpolant()) std::cerr << \" [left interpolant] \";\n    if (expression->is_right_interpolant()) std::cerr << \" [right interpolant] \";\n    std::cerr << \" <\" << prettyprint(expression->pstate().token.ws_before()) << \">\" << std::endl;\n    for(const auto& i : expression->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<String>(node)) {\n    String_Ptr expression = Cast<String>(node);\n    std::cerr << ind << \"String \" << expression;\n    std::cerr << \" \" << expression->concrete_type();\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    if (expression->is_interpolant()) std::cerr << \" [interpolant]\";\n    std::cerr << \" <\" << prettyprint(expression->pstate().token.ws_before()) << \">\" << std::endl;\n  } else if (Cast<Expression>(node)) {\n    Expression_Ptr expression = Cast<Expression>(node);\n    std::cerr << ind << \"Expression \" << expression;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    switch (expression->concrete_type()) {\n      case Expression::Concrete_Type::NONE: std::cerr << \" [NONE]\"; break;\n      case Expression::Concrete_Type::BOOLEAN: std::cerr << \" [BOOLEAN]\"; break;\n      case Expression::Concrete_Type::NUMBER: std::cerr << \" [NUMBER]\"; break;\n      case Expression::Concrete_Type::COLOR: std::cerr << \" [COLOR]\"; break;\n      case Expression::Concrete_Type::STRING: std::cerr << \" [STRING]\"; break;\n      case Expression::Concrete_Type::LIST: std::cerr << \" [LIST]\"; break;\n      case Expression::Concrete_Type::MAP: std::cerr << \" [MAP]\"; break;\n      case Expression::Concrete_Type::SELECTOR: std::cerr << \" [SELECTOR]\"; break;\n      case Expression::Concrete_Type::NULL_VAL: std::cerr << \" [NULL_VAL]\"; break;\n      case Expression::Concrete_Type::C_WARNING: std::cerr << \" [C_WARNING]\"; break;\n      case Expression::Concrete_Type::C_ERROR: std::cerr << \" [C_ERROR]\"; break;\n      case Expression::Concrete_Type::FUNCTION: std::cerr << \" [FUNCTION]\"; break;\n      case Expression::Concrete_Type::NUM_TYPES: std::cerr << \" [NUM_TYPES]\"; break;\n      case Expression::Concrete_Type::VARIABLE: std::cerr << \" [VARIABLE]\"; break;\n      case Expression::Concrete_Type::FUNCTION_VAL: std::cerr << \" [FUNCTION_VAL]\"; break;\n    }\n    std::cerr << std::endl;\n  } else if (Cast<Has_Block>(node)) {\n    Has_Block_Ptr has_block = Cast<Has_Block>(node);\n    std::cerr << ind << \"Has_Block \" << has_block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << has_block->tabs() << std::endl;\n    if (has_block->block()) for(const Statement_Obj& i : has_block->block()->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Statement>(node)) {\n    Statement_Ptr statement = Cast<Statement>(node);\n    std::cerr << ind << \"Statement \" << statement;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << statement->tabs() << std::endl;\n  }\n\n  if (ind == \"\") std::cerr << \"####################################################################\\n\";\n}\n\ninline void debug_node(Node* node, std::string ind = \"\")\n{\n  if (ind == \"\") std::cerr << \"#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\\n\";\n  if (node->isCombinator()) {\n    std::cerr << ind;\n    std::cerr << \"Combinator \";\n    std::cerr << node << \" \";\n    if (node->got_line_feed) std::cerr << \"[LF] \";\n    switch (node->combinator()) {\n      case Complex_Selector::ADJACENT_TO: std::cerr << \"{+} \"; break;\n      case Complex_Selector::PARENT_OF:   std::cerr << \"{>} \"; break;\n      case Complex_Selector::PRECEDES:    std::cerr << \"{~} \"; break;\n      case Complex_Selector::REFERENCE:   std::cerr << \"{@} \"; break;\n      case Complex_Selector::ANCESTOR_OF: std::cerr << \"{ } \"; break;\n    }\n    std::cerr << std::endl;\n    // debug_ast(node->combinator(), ind + \"  \");\n  } else if (node->isSelector()) {\n    std::cerr << ind;\n    std::cerr << \"Selector \";\n    std::cerr << node << \" \";\n    if (node->got_line_feed) std::cerr << \"[LF] \";\n    std::cerr << std::endl;\n    debug_ast(node->selector(), ind + \"  \");\n  } else if (node->isCollection()) {\n    std::cerr << ind;\n    std::cerr << \"Collection \";\n    std::cerr << node << \" \";\n    if (node->got_line_feed) std::cerr << \"[LF] \";\n    std::cerr << std::endl;\n    for(auto n : (*node->collection())) {\n      debug_node(&n, ind + \"  \");\n    }\n  } else if (node->isNil()) {\n    std::cerr << ind;\n    std::cerr << \"Nil \";\n    std::cerr << node << \" \";\n    if (node->got_line_feed) std::cerr << \"[LF] \";\n    std::cerr << std::endl;\n  } else {\n    std::cerr << ind;\n    std::cerr << \"OTHER \";\n    std::cerr << node << \" \";\n    if (node->got_line_feed) std::cerr << \"[LF] \";\n    std::cerr << std::endl;\n  }\n  if (ind == \"\") std::cerr << \"#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\\n\";\n}\n\n/*\ninline void debug_ast(const AST_Node_Ptr node, std::string ind = \"\", Env* env = 0)\n{\n  debug_ast(const_cast<AST_Node_Ptr>(node), ind, env);\n}\n*/\ninline void debug_node(const Node* node, std::string ind = \"\")\n{\n  debug_node(const_cast<Node*>(node), ind);\n}\n\n\ninline void debug_subset_map(Sass::Subset_Map& map, std::string ind = \"\")\n{\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n  for(auto const &it : map.values()) {\n    debug_ast(it.first, ind + \"first: \");\n    debug_ast(it.second, ind + \"second: \");\n  }\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n}\n\ninline void debug_subset_entries(SubSetMapPairs* entries, std::string ind = \"\")\n{\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n  for(auto const &pair : *entries) {\n    debug_ast(pair.first, ind + \"first: \");\n    debug_ast(pair.second, ind + \"second: \");\n  }\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n}\n\n#endif // SASS_DEBUGGER\n\n// Path: src/libsass/src/memory/SharedPtr.cpp\n#include \"../sass.hpp\"\n#include <iostream>\n#include <typeinfo>\n\n#include \"SharedPtr.hpp\"\n#include \"../ast_fwd_decl.hpp\"\n\n#ifdef DEBUG_SHARED_PTR\n#include \"../debugger.hpp\"\n#endif\n\nnamespace Sass {\n\n  #ifdef DEBUG_SHARED_PTR\n  void SharedObj::dumpMemLeaks() {\n    if (!all.empty()) {\n      std::cerr << \"###################################\\n\";\n      std::cerr << \"# REPORTING MISSING DEALLOCATIONS #\\n\";\n      std::cerr << \"###################################\\n\";\n      for (SharedObj* var : all) {\n        if (AST_Node_Ptr ast = dynamic_cast<AST_Node*>(var)) {\n          debug_ast(ast);\n        } else {\n          std::cerr << \"LEAKED \" << var << \"\\n\";\n        }\n      }\n    }\n  }\n  std::vector<SharedObj*> SharedObj::all;\n  #endif\n\n  bool SharedObj::taint = false;\n\n  SharedObj::SharedObj()\n  : detached(false)\n    #ifdef DEBUG_SHARED_PTR\n    , dbg(false)\n    #endif\n  {\n    refcounter = 0;\n    #ifdef DEBUG_SHARED_PTR\n      if (taint) all.push_back(this);\n    #endif\n  };\n\n  SharedObj::~SharedObj() {\n    #ifdef DEBUG_SHARED_PTR\n      if (dbg) std::cerr << \"Destruct \" << this << \"\\n\";\n      if(!all.empty()) { // check needed for MSVC (no clue why?)\n        all.erase(std::remove(all.begin(), all.end(), this), all.end());\n      }\n    #endif\n  };\n\n  void SharedPtr::decRefCount() {\n    if (node) {\n      -- node->refcounter;\n      #ifdef DEBUG_SHARED_PTR\n        if (node->dbg)  std::cerr << \"- \" << node << \" X \" << node->refcounter << \" (\" << this << \") \" << \"\\n\";\n      #endif\n      if (node->refcounter == 0) {\n        #ifdef DEBUG_SHARED_PTR\n          // AST_Node_Ptr ast = dynamic_cast<AST_Node*>(node);\n          if (node->dbg) std::cerr << \"DELETE NODE \" << node << \"\\n\";\n        #endif\n        if (!node->detached) {\n          delete(node);\n        }\n      }\n    }\n  }\n\n  void SharedPtr::incRefCount() {\n    if (node) {\n      ++ node->refcounter;\n      node->detached = false;\n      #ifdef DEBUG_SHARED_PTR\n        if (node->dbg) {\n          std::cerr << \"+ \" << node << \" X \" << node->refcounter << \" (\" << this << \") \" << \"\\n\";\n        }\n      #endif\n    }\n  }\n\n  SharedPtr::~SharedPtr() {\n    decRefCount();\n  }\n\n\n  // the create constructor\n  SharedPtr::SharedPtr(SharedObj* ptr)\n  : node(ptr) {\n    incRefCount();\n  }\n  // copy assignment operator\n  SharedPtr& SharedPtr::operator=(const SharedPtr& rhs) {\n    void* cur_ptr = (void*) node;\n    void* rhs_ptr = (void*) rhs.node;\n    if (cur_ptr == rhs_ptr) {\n      return *this;\n    }\n    decRefCount();\n    node = rhs.node;\n    incRefCount();\n    return *this;\n  }\n\n  // the copy constructor\n  SharedPtr::SharedPtr(const SharedPtr& obj)\n  : node(obj.node) {\n    incRefCount();\n  }\n\n}\n// Path: src/libsass/contrib/plugin.cpp\n#include <cstring>\n#include <iostream>\n#include <stdint.h>\n#include <sass.h>\n\n// gcc: g++ -shared plugin.cpp -o plugin.so -fPIC -Llib -lsass\n// mingw: g++ -shared plugin.cpp -o plugin.dll -Llib -lsass\n\nextern \"C\" const char* ADDCALL libsass_get_version() {\n  return libsass_version();\n}\n\nunion Sass_Value* custom_function(const union Sass_Value* s_args, Sass_Function_Entry cb, struct Sass_Compiler* comp)\n{\n  // get context/option struct associated with this compiler\n  struct Sass_Context* ctx = sass_compiler_get_context(comp);\n  struct Sass_Options* opts = sass_compiler_get_options(comp);\n  // get the cookie from function descriptor\n  void* cookie = sass_function_get_cookie(cb);\n  // we actually abuse the void* to store an \"int\"\n  return sass_make_number((intptr_t)cookie, \"px\");\n}\n\nextern \"C\" Sass_Function_List ADDCALL libsass_load_functions()\n{\n  // allocate a custom function caller\n  Sass_Function_Entry c_func =\n    sass_make_function(\"foo()\", custom_function, (void*)42);\n  // create list of all custom functions\n  Sass_Function_List fn_list = sass_make_function_list(1);\n  // put the only function in this plugin to the list\n  sass_function_set_list_entry(fn_list, 0, c_func);\n  // return the list\n  return fn_list;\n}\n\nSass_Import_List custom_importer(const char* cur_path, Sass_Importer_Entry cb, struct Sass_Compiler* comp)\n{\n  // get the cookie from importer descriptor\n  void* cookie = sass_importer_get_cookie(cb);\n  // create a list to hold our import entries\n  Sass_Import_List incs = sass_make_import_list(1);\n  // create our only import entry (route path back)\n  incs[0] = sass_make_import_entry(cur_path, 0, 0);\n  // return imports\n  return incs;\n}\n\nextern \"C\" Sass_Importer_List ADDCALL libsass_load_importers()\n{\n  // allocate a custom function caller\n  Sass_Importer_Entry c_imp =\n    sass_make_importer(custom_importer, - 99, (void*)42);\n  // create list of all custom functions\n  Sass_Importer_List imp_list = sass_make_importer_list(1);\n  // put the only function in this plugin to the list\n  sass_importer_set_list_entry(imp_list, 0, c_imp);\n  // return the list\n  return imp_list;\n}\n\n// Path: src/libsass/test/test_node.cpp\n#include <assert.h>\n#include <sstream>\n\n#include \"node.hpp\"\n#include \"parser.hpp\"\n\n\n#define STATIC_ARRAY_SIZE(array) (sizeof((array))/sizeof((array[0])))\n\n\nnamespace Sass {\n\n  Context ctx = Context::Data();\n\n  const char* const ROUNDTRIP_TESTS[] = {\n    NULL,\n    \"~\",\n    \"CMPD\",\n    \"~ CMPD\",\n    \"CMPD >\",\n    \"> > CMPD\",\n    \"CMPD ~ ~\",\n    \"> + CMPD1.CMPD2 > ~\",\n    \"> + CMPD1.CMPD2 CMPD3.CMPD4 > ~\",\n    \"+ CMPD1 CMPD2 ~ CMPD3 + CMPD4 > CMPD5 > ~\"\n  };\n\n\n\n  static Complex_Selector* createComplexSelector(std::string src) {\n    std::string temp(src);\n    temp += \";\";\n    return (*Parser::from_c_str(temp.c_str(), ctx, \"\", Position()).parse_selector_list())[0];\n  }\n\n\n  void roundtripTest(const char* toTest) {\n\n    // Create the initial selector\n\n    Complex_Selector* pOrigSelector = NULL;\n    if (toTest) {\n      pOrigSelector = createComplexSelector(toTest);\n    }\n\n    std::string expected(pOrigSelector ? pOrigSelector->to_string() : \"NULL\");\n\n\n    // Roundtrip the selector into a node and back\n\n    Node node = complexSelectorToNode(pOrigSelector, ctx);\n\n    std::stringstream nodeStringStream;\n    nodeStringStream << node;\n    std::string nodeString = nodeStringStream.str();\n    cout << \"ASNODE: \" << node << endl;\n\n    Complex_Selector* pNewSelector = nodeToComplexSelector(node, ctx);\n\n    // Show the result\n\n    std::string result(pNewSelector ? pNewSelector->to_string() : \"NULL\");\n\n    cout << \"SELECTOR: \" << expected << endl;\n    cout << \"NEW SELECTOR:   \" << result << endl;\n\n\n    // Test that they are equal using the equality operator\n\n    assert( (!pOrigSelector && !pNewSelector ) || (pOrigSelector && pNewSelector) );\n    if (pOrigSelector) {\n      assert( *pOrigSelector == *pNewSelector );\n    }\n\n\n    // Test that they are equal by comparing the string versions of the selectors\n\n    assert(expected == result);\n\n  }\n\n\n  int main() {\n    for (int index = 0; index < STATIC_ARRAY_SIZE(ROUNDTRIP_TESTS); index++) {\n      const char* const toTest = ROUNDTRIP_TESTS[index];\n      cout << \"\\nINPUT STRING: \" << (toTest ? toTest : \"NULL\") << endl;\n      roundtripTest(toTest);\n    }\n\n    cout << \"\\nTesting Done.\\n\";\n  }\n\n\n}\n\n// Path: src/libsass/test/test_selector_difference.cpp\n#include \"../ast.hpp\"\n#include \"../context.hpp\"\n#include \"../parser.hpp\"\n#include <string>\n#include <iostream>\n\nusing namespace Sass;\n\nContext ctx = Context::Data();\n\nCompound_Selector* selector(std::string src)\n{ return Parser::from_c_str(src.c_str(), ctx, \"\", Position()).parse_compound_selector(); }\n\nvoid diff(std::string s, std::string t)\n{\n  std::cout << s << \" - \" << t << \" = \" << selector(s + \";\")->minus(selector(t + \";\"), ctx)->to_string() << std::endl;\n}\n\nint main()\n{\n  diff(\".a.b.c\", \".c.b\");\n  diff(\".a.b.c\", \".fludge.b\");\n\n  return 0;\n}\n\n// Path: src/libsass/test/test_superselector.cpp\n#include \"../ast.hpp\"\n#include \"../context.hpp\"\n#include \"../parser.hpp\"\n#include <string>\n\nusing namespace Sass;\n\nContext ctx = Context(Context::Data());\n\nCompound_Selector* compound_selector(std::string src)\n{ return Parser::from_c_str(src.c_str(), ctx, \"\", Position()).parse_compound_selector(); }\n\nComplex_Selector* complex_selector(std::string src)\n{ return Parser::from_c_str(src.c_str(), ctx, \"\", Position()).parse_complex_selector(false); }\n\nvoid check_compound(std::string s1, std::string s2)\n{\n  std::cout << \"Is \"\n       << s1\n       << \" a superselector of \"\n       << s2\n       << \"?\\t\"\n       << compound_selector(s1 + \";\")->is_superselector_of(compound_selector(s2 + \";\"))\n       << std::endl;\n}\n\nvoid check_complex(std::string s1, std::string s2)\n{\n  std::cout << \"Is \"\n       << s1\n       << \" a superselector of \"\n       << s2\n       << \"?\\t\"\n       << complex_selector(s1 + \";\")->is_superselector_of(complex_selector(s2 + \";\"))\n       << std::endl;\n}\n\nint main()\n{\n  check_compound(\".foo\", \".foo.bar\");\n  check_compound(\".foo.bar\", \".foo\");\n  check_compound(\".foo.bar\", \"div.foo\");\n  check_compound(\".foo\", \"div.foo\");\n  check_compound(\"div.foo\", \".foo\");\n  check_compound(\"div.foo\", \"div.bar.foo\");\n  check_compound(\"p.foo\", \"div.bar.foo\");\n  check_compound(\".hux\", \".mumble\");\n\n  std::cout << std::endl;\n\n  check_complex(\".foo ~ .bar\", \".foo + .bar\");\n  check_complex(\".foo .bar\", \".foo + .bar\");\n  check_complex(\".foo .bar\", \".foo > .bar\");\n  check_complex(\".foo .bar > .hux\", \".foo.a .bar.b > .hux\");\n  check_complex(\".foo ~ .bar .hux\", \".foo.a + .bar.b > .hux\");\n  check_complex(\".foo\", \".bar .foo\");\n  check_complex(\".foo\", \".foo.a\");\n  check_complex(\".foo.bar\", \".foo\");\n  check_complex(\".foo .bar .hux\", \".bar .hux\");\n  check_complex(\".foo ~ .bar .hux.x\", \".foo.a + .bar.b > .hux.y\");\n  check_complex(\".foo ~ .bar .hux\", \".foo.a + .bar.b > .mumble\");\n  check_complex(\".foo + .bar\", \".foo ~ .bar\");\n  check_complex(\"a c e\", \"a b c d e\");\n  check_complex(\"c a e\", \"a b c d e\");\n\n  return 0;\n}\n\n\n\n// Path: src/libsass/test/test_paths.cpp\n#include <iostream>\n#include \"../paths.hpp\"\n\nusing namespace Sass;\n\ntemplate<typename T>\nstd::vector<T>& operator<<(std::vector<T>& v, const T& e)\n{\n  v.push_back(e);\n  return v;\n}\n\nint main()\n{\n  std::vector<int> v1, v2, v3;\n  v1 << 1 << 2;\n  v2 << 3;\n  v3 << 4 << 5 << 6;\n\n  std::vector<std::vector<int> > ss;\n  ss << v1 << v2 << v3;\n\n  std::vector<std::vector<int> > ps = paths(ss);\n  for (size_t i = 0, S = ps.size(); i < S; ++i) {\n    std::cout << vector_to_string(ps[i]) << std::endl;\n  }\n  return 0;\n}\n\n// Path: src/libsass/test/test_unification.cpp\n#include \"../ast.hpp\"\n#include \"../context.hpp\"\n#include \"../parser.hpp\"\n#include <string>\n\nusing namespace Sass;\n\nContext ctx = Context(Context::Data());\n\nCompound_Selector* selector(std::string src)\n{ return Parser::from_c_str(src.c_str(), ctx, \"\", Position()).parse_compound_selector(); }\n\nvoid unify(std::string lhs, std::string rhs)\n{\n  Compound_Selector* unified = selector(lhs + \";\")->unify_with(selector(rhs + \";\"), ctx);\n  std::cout << lhs << \" UNIFIED WITH \" << rhs << \" =\\t\" << (unified ? unified->to_string() : \"NOTHING\") << std::endl;\n}\n\nint main()\n{\n  unify(\".foo\", \".foo.bar\");\n  unify(\"div:nth-of-type(odd)\", \"div:first-child\");\n  unify(\"div\", \"span:whatever\");\n  unify(\"div\", \"span\");\n  unify(\"foo:bar::after\", \"foo:bar::first-letter\");\n  unify(\".foo#bar.hux\", \".hux.foo#bar\");\n  unify(\".foo#bar.hux\", \".hux.foo#baz\");\n  unify(\"*:blah:fudge\", \"p:fudge:blah\");\n\n  return 0;\n}\n\n// Path: src/libsass/test/test_subset_map.cpp\n#include <string>\n#include <iostream>\n#include <assert.h>\n#include \"../subset_map.hpp\"\n\nSubset_Map<std::string, std::string> ssm;\n\nstring toString(std::vector<std::string> v);\nstring toString(std::vector<std::pair<std::string, std::vector<std::string>>> v);\nvoid assertEqual(string std::sExpected, std::string sResult);\n\nvoid setup() {\n  ssm.clear();\n\n  //@ssm[Set[1, 2]] = \"Foo\"\n  std::vector<std::string> s1;\n  s1.push_back(\"1\");\n  s1.push_back(\"2\");\n  ssm.put(s1, \"Foo\");\n\n  //@ssm[Set[\"fizz\", \"fazz\"]] = \"Bar\"\n  std::vector<std::string> s2;\n  s2.push_back(\"fizz\");\n  s2.push_back(\"fazz\");\n  ssm.put(s2, \"Bar\");\n\n  //@ssm[Set[:foo, :bar]] = \"Baz\"\n  std::vector<std::string> s3;\n  s3.push_back(\":foo\");\n  s3.push_back(\":bar\");\n  ssm.put(s3, \"Baz\");\n\n  //@ssm[Set[:foo, :bar, :baz]] = \"Bang\"\n  std::vector<std::string> s4;\n  s4.push_back(\":foo\");\n  s4.push_back(\":bar\");\n  s4.push_back(\":baz\");\n  ssm.put(s4, \"Bang\");\n\n  //@ssm[Set[:bip, :bop, :blip]] = \"Qux\"\n  std::vector<std::string> s5;\n  s5.push_back(\":bip\");\n  s5.push_back(\":bop\");\n  s5.push_back(\":blip\");\n  ssm.put(s5, \"Qux\");\n\n  //@ssm[Set[:bip, :bop]] = \"Thram\"\n  std::vector<std::string> s6;\n  s6.push_back(\":bip\");\n  s6.push_back(\":bop\");\n  ssm.put(s6, \"Thram\");\n}\n\nvoid testEqualKeys() {\n  std::cout << \"testEqualKeys\" << std::endl;\n\n  //assert_equal [[\"Foo\", Set[1, 2]]], @ssm.get(Set[1, 2])\n  std::vector<std::string> k1;\n  k1.push_back(\"1\");\n  k1.push_back(\"2\");\n  assertEqual(\"[[Foo, Set[1, 2]]]\", toString(ssm.get_kv(k1)));\n\n  //assert_equal [[\"Bar\", Set[\"fizz\", \"fazz\"]]], @ssm.get(Set[\"fizz\", \"fazz\"])\n  std::vector<std::string> k2;\n  k2.push_back(\"fizz\");\n  k2.push_back(\"fazz\");\n  assertEqual(\"[[Bar, Set[fizz, fazz]]]\", toString(ssm.get_kv(k2)));\n\n  std::cout << std::endl;\n}\n\nvoid testSubsetKeys() {\n  std::cout << \"testSubsetKeys\" << std::endl;\n\n  //assert_equal [[\"Foo\", Set[1, 2]]], @ssm.get(Set[1, 2, \"fuzz\"])\n  std::vector<std::string> k1;\n  k1.push_back(\"1\");\n  k1.push_back(\"2\");\n  k1.push_back(\"fuzz\");\n  assertEqual(\"[[Foo, Set[1, 2]]]\", toString(ssm.get_kv(k1)));\n\n  //assert_equal [[\"Bar\", Set[\"fizz\", \"fazz\"]]], @ssm.get(Set[\"fizz\", \"fazz\", 3])\n  std::vector<std::string> k2;\n  k2.push_back(\"fizz\");\n  k2.push_back(\"fazz\");\n  k2.push_back(\"3\");\n  assertEqual(\"[[Bar, Set[fizz, fazz]]]\", toString(ssm.get_kv(k2)));\n\n  std::cout << std::endl;\n}\n\nvoid testSupersetKeys() {\n  std::cout << \"testSupersetKeys\" << std::endl;\n\n  //assert_equal [], @ssm.get(Set[1])\n  std::vector<std::string> k1;\n  k1.push_back(\"1\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k1)));\n\n  //assert_equal [], @ssm.get(Set[2])\n  std::vector<std::string> k2;\n  k2.push_back(\"2\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k2)));\n\n  //assert_equal [], @ssm.get(Set[\"fizz\"])\n  std::vector<std::string> k3;\n  k3.push_back(\"fizz\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k3)));\n\n  //assert_equal [], @ssm.get(Set[\"fazz\"])\n  std::vector<std::string> k4;\n  k4.push_back(\"fazz\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k4)));\n\n  std::cout << std::endl;\n}\n\nvoid testDisjointKeys() {\n  std::cout << \"testDisjointKeys\" << std::endl;\n\n  //assert_equal [], @ssm.get(Set[3, 4])\n  std::vector<std::string> k1;\n  k1.push_back(\"3\");\n  k1.push_back(\"4\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k1)));\n\n  //assert_equal [], @ssm.get(Set[\"fuzz\", \"frizz\"])\n  std::vector<std::string> k2;\n  k2.push_back(\"fuzz\");\n  k2.push_back(\"frizz\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k2)));\n\n  //assert_equal [], @ssm.get(Set[\"gran\", 15])\n  std::vector<std::string> k3;\n  k3.push_back(\"gran\");\n  k3.push_back(\"15\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k3)));\n\n  std::cout << std::endl;\n}\n\nvoid testSemiDisjointKeys() {\n  std::cout << \"testSemiDisjointKeys\" << std::endl;\n\n  //assert_equal [], @ssm.get(Set[2, 3])\n  std::vector<std::string> k1;\n  k1.push_back(\"2\");\n  k1.push_back(\"3\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k1)));\n\n  //assert_equal [], @ssm.get(Set[\"fizz\", \"fuzz\"])\n  std::vector<std::string> k2;\n  k2.push_back(\"fizz\");\n  k2.push_back(\"fuzz\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k2)));\n\n  //assert_equal [], @ssm.get(Set[1, \"fazz\"])\n  std::vector<std::string> k3;\n  k3.push_back(\"1\");\n  k3.push_back(\"fazz\");\n  assertEqual(\"[]\", toString(ssm.get_kv(k3)));\n\n  std::cout << std::endl;\n}\n\nvoid testEmptyKeySet() {\n  std::cout << \"testEmptyKeySet\" << std::endl;\n\n  //assert_raises(ArgumentError) {@ssm[Set[]] = \"Fail\"}\n  std::vector<std::string> s1;\n  try {\n    ssm.put(s1, \"Fail\");\n  }\n  catch (const char* &e) {\n    assertEqual(\"internal error: subset map keys may not be empty\", e);\n  }\n}\n\nvoid testEmptyKeyGet() {\n  std::cout << \"testEmptyKeyGet\" << std::endl;\n\n  //assert_equal [], @ssm.get(Set[])\n  std::vector<std::string> k1;\n  assertEqual(\"[]\", toString(ssm.get_kv(k1)));\n\n  std::cout << std::endl;\n}\nvoid testMultipleSubsets() {\n  std::cout << \"testMultipleSubsets\" << std::endl;\n\n  //assert_equal [[\"Foo\", Set[1, 2]], [\"Bar\", Set[\"fizz\", \"fazz\"]]], @ssm.get(Set[1, 2, \"fizz\", \"fazz\"])\n  std::vector<std::string> k1;\n  k1.push_back(\"1\");\n  k1.push_back(\"2\");\n  k1.push_back(\"fizz\");\n  k1.push_back(\"fazz\");\n  assertEqual(\"[[Foo, Set[1, 2]], [Bar, Set[fizz, fazz]]]\", toString(ssm.get_kv(k1)));\n\n  //assert_equal [[\"Foo\", Set[1, 2]], [\"Bar\", Set[\"fizz\", \"fazz\"]]], @ssm.get(Set[1, 2, 3, \"fizz\", \"fazz\", \"fuzz\"])\n  std::vector<std::string> k2;\n  k2.push_back(\"1\");\n  k2.push_back(\"2\");\n  k2.push_back(\"3\");\n  k2.push_back(\"fizz\");\n  k2.push_back(\"fazz\");\n  k2.push_back(\"fuzz\");\n  assertEqual(\"[[Foo, Set[1, 2]], [Bar, Set[fizz, fazz]]]\", toString(ssm.get_kv(k2)));\n\n  //assert_equal [[\"Baz\", Set[:foo, :bar]]], @ssm.get(Set[:foo, :bar])\n  std::vector<std::string> k3;\n  k3.push_back(\":foo\");\n  k3.push_back(\":bar\");\n  assertEqual(\"[[Baz, Set[:foo, :bar]]]\", toString(ssm.get_kv(k3)));\n\n  //assert_equal [[\"Baz\", Set[:foo, :bar]], [\"Bang\", Set[:foo, :bar, :baz]]], @ssm.get(Set[:foo, :bar, :baz])\n  std::vector<std::string> k4;\n  k4.push_back(\":foo\");\n  k4.push_back(\":bar\");\n  k4.push_back(\":baz\");\n  assertEqual(\"[[Baz, Set[:foo, :bar]], [Bang, Set[:foo, :bar, :baz]]]\", toString(ssm.get_kv(k4)));\n\n  std::cout << std::endl;\n}\nvoid testBracketBracket() {\n  std::cout << \"testBracketBracket\" << std::endl;\n\n  //assert_equal [\"Foo\"], @ssm[Set[1, 2, \"fuzz\"]]\n  std::vector<std::string> k1;\n  k1.push_back(\"1\");\n  k1.push_back(\"2\");\n  k1.push_back(\"fuzz\");\n  assertEqual(\"[Foo]\", toString(ssm.get_v(k1)));\n\n  //assert_equal [\"Baz\", \"Bang\"], @ssm[Set[:foo, :bar, :baz]]\n  std::vector<std::string> k2;\n  k2.push_back(\":foo\");\n  k2.push_back(\":bar\");\n  k2.push_back(\":baz\");\n  assertEqual(\"[Baz, Bang]\", toString(ssm.get_v(k2)));\n\n  std::cout << std::endl;\n}\n\nvoid testKeyOrder() {\n  std::cout << \"testEqualKeys\" << std::endl;\n\n  //assert_equal [[\"Foo\", Set[1, 2]]], @ssm.get(Set[2, 1])\n  std::vector<std::string> k1;\n  k1.push_back(\"2\");\n  k1.push_back(\"1\");\n  assertEqual(\"[[Foo, Set[1, 2]]]\", toString(ssm.get_kv(k1)));\n\n  std::cout << std::endl;\n}\n\nvoid testOrderPreserved() {\n  std::cout << \"testOrderPreserved\" << std::endl;\n  //@ssm[Set[10, 11, 12]] = 1\n  std::vector<std::string> s1;\n  s1.push_back(\"10\");\n  s1.push_back(\"11\");\n  s1.push_back(\"12\");\n  ssm.put(s1, \"1\");\n\n  //@ssm[Set[10, 11]] = 2\n  std::vector<std::string> s2;\n  s2.push_back(\"10\");\n  s2.push_back(\"11\");\n  ssm.put(s2, \"2\");\n\n  //@ssm[Set[11]] = 3\n  std::vector<std::string> s3;\n  s3.push_back(\"11\");\n  ssm.put(s3, \"3\");\n\n  //@ssm[Set[11, 12]] = 4\n  std::vector<std::string> s4;\n  s4.push_back(\"11\");\n  s4.push_back(\"12\");\n  ssm.put(s4, \"4\");\n\n  //@ssm[Set[9, 10, 11, 12, 13]] = 5\n  std::vector<std::string> s5;\n  s5.push_back(\"9\");\n  s5.push_back(\"10\");\n  s5.push_back(\"11\");\n  s5.push_back(\"12\");\n  s5.push_back(\"13\");\n  ssm.put(s5, \"5\");\n\n  //@ssm[Set[10, 13]] = 6\n  std::vector<std::string> s6;\n  s6.push_back(\"10\");\n  s6.push_back(\"13\");\n  ssm.put(s6, \"6\");\n\n  //assert_equal([[1, Set[10, 11, 12]], [2, Set[10, 11]], [3, Set[11]], [4, Set[11, 12]], [5, Set[9, 10, 11, 12, 13]], [6, Set[10, 13]]], @ssm.get(Set[9, 10, 11, 12, 13]))\n  std::vector<std::string> k1;\n  k1.push_back(\"9\");\n  k1.push_back(\"10\");\n  k1.push_back(\"11\");\n  k1.push_back(\"12\");\n  k1.push_back(\"13\");\n  assertEqual(\"[[1, Set[10, 11, 12]], [2, Set[10, 11]], [3, Set[11]], [4, Set[11, 12]], [5, Set[9, 10, 11, 12, 13]], [6, Set[10, 13]]]\", toString(ssm.get_kv(k1)));\n\n  std::cout << std::endl;\n}\nvoid testMultipleEqualValues() {\n  std::cout << \"testMultipleEqualValues\" << std::endl;\n  //@ssm[Set[11, 12]] = 1\n  std::vector<std::string> s1;\n  s1.push_back(\"11\");\n  s1.push_back(\"12\");\n  ssm.put(s1, \"1\");\n\n  //@ssm[Set[12, 13]] = 2\n  std::vector<std::string> s2;\n  s2.push_back(\"12\");\n  s2.push_back(\"13\");\n  ssm.put(s2, \"2\");\n\n  //@ssm[Set[13, 14]] = 1\n  std::vector<std::string> s3;\n  s3.push_back(\"13\");\n  s3.push_back(\"14\");\n  ssm.put(s3, \"1\");\n\n  //@ssm[Set[14, 15]] = 1\n  std::vector<std::string> s4;\n  s4.push_back(\"14\");\n  s4.push_back(\"15\");\n  ssm.put(s4, \"1\");\n\n  //assert_equal([[1, Set[11, 12]], [2, Set[12, 13]], [1, Set[13, 14]], [1, Set[14, 15]]], @ssm.get(Set[11, 12, 13, 14, 15]))\n  std::vector<std::string> k1;\n  k1.push_back(\"11\");\n  k1.push_back(\"12\");\n  k1.push_back(\"13\");\n  k1.push_back(\"14\");\n  k1.push_back(\"15\");\n  assertEqual(\"[[1, Set[11, 12]], [2, Set[12, 13]], [1, Set[13, 14]], [1, Set[14, 15]]]\", toString(ssm.get_kv(k1)));\n\n  std::cout << std::endl;\n}\n\n...\n", "needle_token_start": 7299, "needle_token_end": 7428, "code_context_ntokens": 16222, "output": ["```cpp\ninline void debug_subset_map(Sass::Subset_Map& map, std::string ind = \"\")\n{\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n  for(auto const &it : map.values()) {\n    debug_ast(it.first, ind + \"first: \");\n    debug_ast(it.second, ind + \"second: \");\n  }\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n}\n```"]}
{"repo": "sass/node-sass", "name": "get_unit_type", "language": "cpp", "path": "src/libsass/src/units.cpp", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To determine the category of a given unit based on its type.\n2. **Input**: An integer representing a specific unit type.\n3. **Output**: An enumeration value representing the category of the unit (e.g., length, angle, time).\n4. **Procedure**: The function checks the upper byte of the input integer to match it against predefined categories. It returns the corresponding category if a match is found, otherwise, it returns a default category indicating the unit is not commensurable with known types.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/libsass/src/color_maps.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"color_maps.hpp\"\n\nnamespace Sass {\n\n  namespace ColorNames\n  {\n    const char aliceblue [] = \"aliceblue\";\n    const char antiquewhite [] = \"antiquewhite\";\n    const char cyan [] = \"cyan\";\n    const char aqua [] = \"aqua\";\n    const char aquamarine [] = \"aquamarine\";\n    const char azure [] = \"azure\";\n    const char beige [] = \"beige\";\n    const char bisque [] = \"bisque\";\n    const char black [] = \"black\";\n    const char blanchedalmond [] = \"blanchedalmond\";\n    const char blue [] = \"blue\";\n    const char blueviolet [] = \"blueviolet\";\n    const char brown [] = \"brown\";\n    const char burlywood [] = \"burlywood\";\n    const char cadetblue [] = \"cadetblue\";\n    const char chartreuse [] = \"chartreuse\";\n    const char chocolate [] = \"chocolate\";\n    const char coral [] = \"coral\";\n    const char cornflowerblue [] = \"cornflowerblue\";\n    const char cornsilk [] = \"cornsilk\";\n    const char crimson [] = \"crimson\";\n    const char darkblue [] = \"darkblue\";\n    const char darkcyan [] = \"darkcyan\";\n    const char darkgoldenrod [] = \"darkgoldenrod\";\n    const char darkgray [] = \"darkgray\";\n    const char darkgrey [] = \"darkgrey\";\n    const char darkgreen [] = \"darkgreen\";\n    const char darkkhaki [] = \"darkkhaki\";\n    const char darkmagenta [] = \"darkmagenta\";\n    const char darkolivegreen [] = \"darkolivegreen\";\n    const char darkorange [] = \"darkorange\";\n    const char darkorchid [] = \"darkorchid\";\n    const char darkred [] = \"darkred\";\n    const char darksalmon [] = \"darksalmon\";\n    const char darkseagreen [] = \"darkseagreen\";\n    const char darkslateblue [] = \"darkslateblue\";\n    const char darkslategray [] = \"darkslategray\";\n    const char darkslategrey [] = \"darkslategrey\";\n    const char darkturquoise [] = \"darkturquoise\";\n    const char darkviolet [] = \"darkviolet\";\n    const char deeppink [] = \"deeppink\";\n    const char deepskyblue [] = \"deepskyblue\";\n    const char dimgray [] = \"dimgray\";\n    const char dimgrey [] = \"dimgrey\";\n    const char dodgerblue [] = \"dodgerblue\";\n    const char firebrick [] = \"firebrick\";\n    const char floralwhite [] = \"floralwhite\";\n    const char forestgreen [] = \"forestgreen\";\n    const char magenta [] = \"magenta\";\n    const char fuchsia [] = \"fuchsia\";\n    const char gainsboro [] = \"gainsboro\";\n    const char ghostwhite [] = \"ghostwhite\";\n    const char gold [] = \"gold\";\n    const char goldenrod [] = \"goldenrod\";\n    const char gray [] = \"gray\";\n    const char grey [] = \"grey\";\n    const char green [] = \"green\";\n    const char greenyellow [] = \"greenyellow\";\n    const char honeydew [] = \"honeydew\";\n    const char hotpink [] = \"hotpink\";\n    const char indianred [] = \"indianred\";\n    const char indigo [] = \"indigo\";\n    const char ivory [] = \"ivory\";\n    const char khaki [] = \"khaki\";\n    const char lavender [] = \"lavender\";\n    const char lavenderblush [] = \"lavenderblush\";\n    const char lawngreen [] = \"lawngreen\";\n    const char lemonchiffon [] = \"lemonchiffon\";\n    const char lightblue [] = \"lightblue\";\n    const char lightcoral [] = \"lightcoral\";\n    const char lightcyan [] = \"lightcyan\";\n    const char lightgoldenrodyellow [] = \"lightgoldenrodyellow\";\n    const char lightgray [] = \"lightgray\";\n    const char lightgrey [] = \"lightgrey\";\n    const char lightgreen [] = \"lightgreen\";\n    const char lightpink [] = \"lightpink\";\n    const char lightsalmon [] = \"lightsalmon\";\n    const char lightseagreen [] = \"lightseagreen\";\n    const char lightskyblue [] = \"lightskyblue\";\n    const char lightslategray [] = \"lightslategray\";\n    const char lightslategrey [] = \"lightslategrey\";\n    const char lightsteelblue [] = \"lightsteelblue\";\n    const char lightyellow [] = \"lightyellow\";\n    const char lime [] = \"lime\";\n    const char limegreen [] = \"limegreen\";\n    const char linen [] = \"linen\";\n    const char maroon [] = \"maroon\";\n    const char mediumaquamarine [] = \"mediumaquamarine\";\n    const char mediumblue [] = \"mediumblue\";\n    const char mediumorchid [] = \"mediumorchid\";\n    const char mediumpurple [] = \"mediumpurple\";\n    const char mediumseagreen [] = \"mediumseagreen\";\n    const char mediumslateblue [] = \"mediumslateblue\";\n    const char mediumspringgreen [] = \"mediumspringgreen\";\n    const char mediumturquoise [] = \"mediumturquoise\";\n    const char mediumvioletred [] = \"mediumvioletred\";\n    const char midnightblue [] = \"midnightblue\";\n    const char mintcream [] = \"mintcream\";\n    const char mistyrose [] = \"mistyrose\";\n    const char moccasin [] = \"moccasin\";\n    const char navajowhite [] = \"navajowhite\";\n    const char navy [] = \"navy\";\n    const char oldlace [] = \"oldlace\";\n    const char olive [] = \"olive\";\n    const char olivedrab [] = \"olivedrab\";\n    const char orange [] = \"orange\";\n    const char orangered [] = \"orangered\";\n    const char orchid [] = \"orchid\";\n    const char palegoldenrod [] = \"palegoldenrod\";\n    const char palegreen [] = \"palegreen\";\n    const char paleturquoise [] = \"paleturquoise\";\n    const char palevioletred [] = \"palevioletred\";\n    const char papayawhip [] = \"papayawhip\";\n    const char peachpuff [] = \"peachpuff\";\n    const char peru [] = \"peru\";\n    const char pink [] = \"pink\";\n    const char plum [] = \"plum\";\n    const char powderblue [] = \"powderblue\";\n    const char purple [] = \"purple\";\n    const char red [] = \"red\";\n    const char rosybrown [] = \"rosybrown\";\n    const char royalblue [] = \"royalblue\";\n    const char saddlebrown [] = \"saddlebrown\";\n    const char salmon [] = \"salmon\";\n    const char sandybrown [] = \"sandybrown\";\n    const char seagreen [] = \"seagreen\";\n    const char seashell [] = \"seashell\";\n    const char sienna [] = \"sienna\";\n    const char silver [] = \"silver\";\n    const char skyblue [] = \"skyblue\";\n    const char slateblue [] = \"slateblue\";\n    const char slategray [] = \"slategray\";\n    const char slategrey [] = \"slategrey\";\n    const char snow [] = \"snow\";\n    const char springgreen [] = \"springgreen\";\n    const char steelblue [] = \"steelblue\";\n    const char tan [] = \"tan\";\n    const char teal [] = \"teal\";\n    const char thistle [] = \"thistle\";\n    const char tomato [] = \"tomato\";\n    const char turquoise [] = \"turquoise\";\n    const char violet [] = \"violet\";\n    const char wheat [] = \"wheat\";\n    const char white [] = \"white\";\n    const char whitesmoke [] = \"whitesmoke\";\n    const char yellow [] = \"yellow\";\n    const char yellowgreen [] = \"yellowgreen\";\n    const char rebeccapurple [] = \"rebeccapurple\";\n    const char transparent [] = \"transparent\";\n  }\n\n  namespace Colors {\n    const ParserState color_table(\"[COLOR TABLE]\");\n    const Color aliceblue(color_table, 240, 248, 255, 1);\n    const Color antiquewhite(color_table, 250, 235, 215, 1);\n    const Color cyan(color_table, 0, 255, 255, 1);\n    const Color aqua(color_table, 0, 255, 255, 1);\n    const Color aquamarine(color_table, 127, 255, 212, 1);\n    const Color azure(color_table, 240, 255, 255, 1);\n    const Color beige(color_table, 245, 245, 220, 1);\n    const Color bisque(color_table, 255, 228, 196, 1);\n    const Color black(color_table, 0, 0, 0, 1);\n    const Color blanchedalmond(color_table, 255, 235, 205, 1);\n    const Color blue(color_table, 0, 0, 255, 1);\n    const Color blueviolet(color_table, 138, 43, 226, 1);\n    const Color brown(color_table, 165, 42, 42, 1);\n    const Color burlywood(color_table, 222, 184, 135, 1);\n    const Color cadetblue(color_table, 95, 158, 160, 1);\n    const Color chartreuse(color_table, 127, 255, 0, 1);\n    const Color chocolate(color_table, 210, 105, 30, 1);\n    const Color coral(color_table, 255, 127, 80, 1);\n    const Color cornflowerblue(color_table, 100, 149, 237, 1);\n    const Color cornsilk(color_table, 255, 248, 220, 1);\n    const Color crimson(color_table, 220, 20, 60, 1);\n    const Color darkblue(color_table, 0, 0, 139, 1);\n    const Color darkcyan(color_table, 0, 139, 139, 1);\n    const Color darkgoldenrod(color_table, 184, 134, 11, 1);\n    const Color darkgray(color_table, 169, 169, 169, 1);\n    const Color darkgrey(color_table, 169, 169, 169, 1);\n    const Color darkgreen(color_table, 0, 100, 0, 1);\n    const Color darkkhaki(color_table, 189, 183, 107, 1);\n    const Color darkmagenta(color_table, 139, 0, 139, 1);\n    const Color darkolivegreen(color_table, 85, 107, 47, 1);\n    const Color darkorange(color_table, 255, 140, 0, 1);\n    const Color darkorchid(color_table, 153, 50, 204, 1);\n    const Color darkred(color_table, 139, 0, 0, 1);\n    const Color darksalmon(color_table, 233, 150, 122, 1);\n    const Color darkseagreen(color_table, 143, 188, 143, 1);\n    const Color darkslateblue(color_table, 72, 61, 139, 1);\n    const Color darkslategray(color_table, 47, 79, 79, 1);\n    const Color darkslategrey(color_table, 47, 79, 79, 1);\n    const Color darkturquoise(color_table, 0, 206, 209, 1);\n    const Color darkviolet(color_table, 148, 0, 211, 1);\n    const Color deeppink(color_table, 255, 20, 147, 1);\n    const Color deepskyblue(color_table, 0, 191, 255, 1);\n    const Color dimgray(color_table, 105, 105, 105, 1);\n    const Color dimgrey(color_table, 105, 105, 105, 1);\n...\n// Path: src/libsass/src/values.cpp\n#include \"sass.hpp\"\n#include \"sass.h\"\n#include \"values.hpp\"\n\n#include <stdint.h>\n\nnamespace Sass {\n\n  // convert value from C++ side to C-API\n  union Sass_Value* ast_node_to_sass_value (const Expression_Ptr val)\n  {\n    if (val->concrete_type() == Expression::NUMBER)\n    {\n      Number_Ptr_Const res = Cast<Number>(val);\n      return sass_make_number(res->value(), res->unit().c_str());\n    }\n    else if (val->concrete_type() == Expression::COLOR)\n    {\n      Color_Ptr_Const col = Cast<Color>(val);\n      return sass_make_color(col->r(), col->g(), col->b(), col->a());\n    }\n    else if (val->concrete_type() == Expression::LIST)\n    {\n      List_Ptr_Const l = Cast<List>(val);\n      union Sass_Value* list = sass_make_list(l->size(), l->separator(), l->is_bracketed());\n      for (size_t i = 0, L = l->length(); i < L; ++i) {\n        Expression_Obj obj = l->at(i);\n        auto val = ast_node_to_sass_value(obj);\n        sass_list_set_value(list, i, val);\n      }\n      return list;\n    }\n    else if (val->concrete_type() == Expression::MAP)\n    {\n      Map_Ptr_Const m = Cast<Map>(val);\n      union Sass_Value* map = sass_make_map(m->length());\n      size_t i = 0; for (Expression_Obj key : m->keys()) {\n        sass_map_set_key(map, i, ast_node_to_sass_value(key));\n        sass_map_set_value(map, i, ast_node_to_sass_value(m->at(key)));\n        ++ i;\n      }\n      return map;\n    }\n    else if (val->concrete_type() == Expression::NULL_VAL)\n    {\n      return sass_make_null();\n    }\n    else if (val->concrete_type() == Expression::BOOLEAN)\n    {\n      Boolean_Ptr_Const res = Cast<Boolean>(val);\n      return sass_make_boolean(res->value());\n    }\n    else if (val->concrete_type() == Expression::STRING)\n    {\n      if (String_Quoted_Ptr_Const qstr = Cast<String_Quoted>(val))\n      {\n        return sass_make_qstring(qstr->value().c_str());\n      }\n      else if (String_Constant_Ptr_Const cstr = Cast<String_Constant>(val))\n      {\n        return sass_make_string(cstr->value().c_str());\n      }\n    }\n    return sass_make_error(\"unknown sass value type\");\n  }\n\n  // convert value from C-API to C++ side\n  Value_Ptr sass_value_to_ast_node (const union Sass_Value* val)\n  {\n    switch (sass_value_get_tag(val)) {\n      case SASS_NUMBER:\n        return SASS_MEMORY_NEW(Number,\n                               ParserState(\"[C-VALUE]\"),\n                               sass_number_get_value(val),\n                               sass_number_get_unit(val));\n      case SASS_BOOLEAN:\n        return SASS_MEMORY_NEW(Boolean,\n                               ParserState(\"[C-VALUE]\"),\n                               sass_boolean_get_value(val));\n      case SASS_COLOR:\n        return SASS_MEMORY_NEW(Color,\n                               ParserState(\"[C-VALUE]\"),\n                               sass_color_get_r(val),\n                               sass_color_get_g(val),\n                               sass_color_get_b(val),\n                               sass_color_get_a(val));\n      case SASS_STRING:\n        if (sass_string_is_quoted(val)) {\n          return SASS_MEMORY_NEW(String_Quoted,\n                                 ParserState(\"[C-VALUE]\"),\n                                 sass_string_get_value(val));\n        }\n        return SASS_MEMORY_NEW(String_Constant,\n                                 ParserState(\"[C-VALUE]\"),\n                                 sass_string_get_value(val));\n      case SASS_LIST: {\n        List_Ptr l = SASS_MEMORY_NEW(List,\n                                  ParserState(\"[C-VALUE]\"),\n                                  sass_list_get_length(val),\n                                  sass_list_get_separator(val));\n        for (size_t i = 0, L = sass_list_get_length(val); i < L; ++i) {\n          l->append(sass_value_to_ast_node(sass_list_get_value(val, i)));\n        }\n        l->is_bracketed(sass_list_get_is_bracketed(val));\n        return l;\n      }\n      case SASS_MAP: {\n        Map_Ptr m = SASS_MEMORY_NEW(Map, ParserState(\"[C-VALUE]\"));\n        for (size_t i = 0, L = sass_map_get_length(val); i < L; ++i) {\n          *m << std::make_pair(\n            sass_value_to_ast_node(sass_map_get_key(val, i)),\n            sass_value_to_ast_node(sass_map_get_value(val, i)));\n        }\n        return m;\n      }\n      case SASS_NULL:\n        return SASS_MEMORY_NEW(Null, ParserState(\"[C-VALUE]\"));\n      case SASS_ERROR:\n        return SASS_MEMORY_NEW(Custom_Error,\n                               ParserState(\"[C-VALUE]\"),\n                               sass_error_get_message(val));\n      case SASS_WARNING:\n        return SASS_MEMORY_NEW(Custom_Warning,\n                               ParserState(\"[C-VALUE]\"),\n                               sass_warning_get_message(val));\n      default: break;\n    }\n    return 0;\n  }\n\n}\n\n// Path: src/libsass/src/error_handling.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"prelexer.hpp\"\n#include \"backtrace.hpp\"\n#include \"error_handling.hpp\"\n\n#include <iostream>\n\nnamespace Sass {\n\n  namespace Exception {\n\n    Base::Base(ParserState pstate, std::string msg, Backtraces traces)\n    : std::runtime_error(msg), msg(msg),\n      prefix(\"Error\"), pstate(pstate), traces(traces)\n    { }\n\n    InvalidSass::InvalidSass(ParserState pstate, Backtraces traces, std::string msg)\n    : Base(pstate, msg, traces)\n    { }\n\n\n    InvalidParent::InvalidParent(Selector_Ptr parent, Backtraces traces, Selector_Ptr selector)\n    : Base(selector->pstate(), def_msg, traces), parent(parent), selector(selector)\n    {\n      msg = \"Invalid parent selector for \\\"\";\n      msg += selector->to_string(Sass_Inspect_Options());\n      msg += \"\\\": \\\"\";\n      msg += parent->to_string(Sass_Inspect_Options());\n      msg += \"\\\"\";\n    }\n\n    InvalidVarKwdType::InvalidVarKwdType(ParserState pstate, Backtraces traces, std::string name, const Argument_Ptr arg)\n    : Base(pstate, def_msg, traces), name(name), arg(arg)\n    {\n      msg = \"Variable keyword argument map must have string keys.\\n\";\n      msg += name + \" is not a string in \" + arg->to_string() + \".\";\n    }\n\n    InvalidArgumentType::InvalidArgumentType(ParserState pstate, Backtraces traces, std::string fn, std::string arg, std::string type, const Value_Ptr value)\n    : Base(pstate, def_msg, traces), fn(fn), arg(arg), type(type), value(value)\n    {\n      msg  = arg + \": \\\"\";\n      if (value) msg += value->to_string(Sass_Inspect_Options());\n      msg += \"\\\" is not a \" + type;\n      msg += \" for `\" + fn + \"'\";\n    }\n\n    MissingArgument::MissingArgument(ParserState pstate, Backtraces traces, std::string fn, std::string arg, std::string fntype)\n    : Base(pstate, def_msg, traces), fn(fn), arg(arg), fntype(fntype)\n    {\n      msg  = fntype + \" \" + fn;\n      msg += \" is missing argument \";\n      msg += arg + \".\";\n    }\n\n    InvalidSyntax::InvalidSyntax(ParserState pstate, Backtraces traces, std::string msg)\n    : Base(pstate, msg, traces)\n    { }\n\n    NestingLimitError::NestingLimitError(ParserState pstate, Backtraces traces, std::string msg)\n    : Base(pstate, msg, traces)\n    { }\n\n    DuplicateKeyError::DuplicateKeyError(Backtraces traces, const Map& dup, const Expression& org)\n    : Base(org.pstate(), def_msg, traces), dup(dup), org(org)\n    {\n      msg  = \"Duplicate key \";\n      msg += dup.get_duplicate_key()->inspect();\n      msg += \" in map (\";\n      msg += org.inspect();\n      msg += \").\";\n    }\n\n    TypeMismatch::TypeMismatch(Backtraces traces, const Expression& var, const std::string type)\n    : Base(var.pstate(), def_msg, traces), var(var), type(type)\n    {\n      msg  = var.to_string();\n      msg += \" is not an \";\n      msg += type;\n      msg += \".\";\n    }\n\n    InvalidValue::InvalidValue(Backtraces traces, const Expression& val)\n    : Base(val.pstate(), def_msg, traces), val(val)\n    {\n      msg  = val.to_string();\n      msg += \" isn't a valid CSS value.\";\n    }\n\n    StackError::StackError(Backtraces traces, const AST_Node& node)\n    : Base(node.pstate(), def_msg, traces), node(node)\n    {\n      msg  = \"stack level too deep\";\n    }\n\n    IncompatibleUnits::IncompatibleUnits(const Units& lhs, const Units& rhs)\n    {\n      msg  = \"Incompatible units: '\";\n      msg += rhs.unit();\n      msg += \"' and '\";\n      msg += lhs.unit();\n      msg += \"'.\";\n    }\n\n    IncompatibleUnits::IncompatibleUnits(const UnitType lhs, const UnitType rhs)\n    {\n      msg  = \"Incompatible units: '\";\n      msg += unit_to_string(rhs);\n      msg += \"' and '\";\n      msg += unit_to_string(lhs);\n      msg += \"'.\";\n    }\n\n    AlphaChannelsNotEqual::AlphaChannelsNotEqual(Expression_Ptr_Const lhs, Expression_Ptr_Const rhs, enum Sass_OP op)\n    : OperationError(), lhs(lhs), rhs(rhs), op(op)\n    {\n      msg  = \"Alpha channels must be equal: \";\n      msg += lhs->to_string({ NESTED, 5 });\n      msg += \" \" + sass_op_to_name(op) + \" \";\n      msg += rhs->to_string({ NESTED, 5 });\n      msg += \".\";\n    }\n\n    ZeroDivisionError::ZeroDivisionError(const Expression& lhs, const Expression& rhs)\n    : OperationError(), lhs(lhs), rhs(rhs)\n    {\n      msg  = \"divided by 0\";\n    }\n\n    UndefinedOperation::UndefinedOperation(Expression_Ptr_Const lhs, Expression_Ptr_Const rhs, enum Sass_OP op)\n    : OperationError(), lhs(lhs), rhs(rhs), op(op)\n    {\n      msg  = def_op_msg + \": \\\"\";\n      msg += lhs->to_string({ NESTED, 5 });\n      msg += \" \" + sass_op_to_name(op) + \" \";\n      msg += rhs->to_string({ TO_SASS, 5 });\n      msg += \"\\\".\";\n    }\n\n    InvalidNullOperation::InvalidNullOperation(Expression_Ptr_Const lhs, Expression_Ptr_Const rhs, enum Sass_OP op)\n    : UndefinedOperation(lhs, rhs, op)\n    {\n      msg  = def_op_null_msg + \": \\\"\";\n      msg += lhs->inspect();\n      msg += \" \" + sass_op_to_name(op) + \" \";\n      msg += rhs->inspect();\n      msg += \"\\\".\";\n    }\n\n    SassValueError::SassValueError(Backtraces traces, ParserState pstate, OperationError& err)\n    : Base(pstate, err.what(), traces)\n    {\n      msg = err.what();\n      prefix = err.errtype();\n    }\n\n  }\n\n\n  void warn(std::string msg, ParserState pstate)\n  {\n    std::cerr << \"Warning: \" << msg << std::endl;\n  }\n\n  void warning(std::string msg, ParserState pstate)\n  {\n    std::string cwd(Sass::File::get_cwd());\n    std::string abs_path(Sass::File::rel2abs(pstate.path, cwd, cwd));\n    std::string rel_path(Sass::File::abs2rel(pstate.path, cwd, cwd));\n    std::string output_path(Sass::File::path_for_console(rel_path, abs_path, pstate.path));\n\n    std::cerr << \"WARNING on line \" << pstate.line+1 << \", column \" << pstate.column+1 << \" of \" << output_path << \":\" << std::endl;\n    std::cerr << msg << std::endl << std::endl;\n  }\n\n  void warn(std::string msg, ParserState pstate, Backtrace* bt)\n  {\n    warn(msg, pstate);\n  }\n\n  void deprecated_function(std::string msg, ParserState pstate)\n  {\n    std::string cwd(Sass::File::get_cwd());\n    std::string abs_path(Sass::File::rel2abs(pstate.path, cwd, cwd));\n    std::string rel_path(Sass::File::abs2rel(pstate.path, cwd, cwd));\n    std::string output_path(Sass::File::path_for_console(rel_path, abs_path, pstate.path));\n\n    std::cerr << \"DEPRECATION WARNING: \" << msg << std::endl;\n    std::cerr << \"will be an error in future versions of Sass.\" << std::endl;\n    std::cerr << \"        on line \" << pstate.line+1 << \" of \" << output_path << std::endl;\n  }\n\n  void deprecated(std::string msg, std::string msg2, bool with_column, ParserState pstate)\n  {\n    std::string cwd(Sass::File::get_cwd());\n    std::string abs_path(Sass::File::rel2abs(pstate.path, cwd, cwd));\n    std::string rel_path(Sass::File::abs2rel(pstate.path, cwd, cwd));\n    std::string output_path(Sass::File::path_for_console(rel_path, pstate.path, pstate.path));\n\n    std::cerr << \"DEPRECATION WARNING on line \" << pstate.line + 1;\n    if (with_column) std::cerr << \", column \" << pstate.column + pstate.offset.column + 1;\n    if (output_path.length()) std::cerr << \" of \" << output_path;\n    std::cerr << \":\" << std::endl;\n    std::cerr << msg << std::endl;\n    if (msg2.length()) std::cerr << msg2 << std::endl;\n    std::cerr << std::endl;\n  }\n\n  void deprecated_bind(std::string msg, ParserState pstate)\n  {\n    std::string cwd(Sass::File::get_cwd());\n    std::string abs_path(Sass::File::rel2abs(pstate.path, cwd, cwd));\n    std::string rel_path(Sass::File::abs2rel(pstate.path, cwd, cwd));\n    std::string output_path(Sass::File::path_for_console(rel_path, abs_path, pstate.path));\n\n    std::cerr << \"WARNING: \" << msg << std::endl;\n    std::cerr << \"        on line \" << pstate.line+1 << \" of \" << output_path << std::endl;\n    std::cerr << \"This will be an error in future versions of Sass.\" << std::endl;\n  }\n\n  // should be replaced with error with backtraces\n  void coreError(std::string msg, ParserState pstate)\n  {\n    Backtraces traces;\n    throw Exception::InvalidSyntax(pstate, traces, msg);\n  }\n\n  void error(std::string msg, ParserState pstate, Backtraces& traces)\n  {\n    traces.push_back(Backtrace(pstate));\n    throw Exception::InvalidSyntax(pstate, traces, msg);\n  }\n\n}\n\n// Path: src/libsass/src/units.cpp\n#include \"sass.hpp\"\n#include <stdexcept>\n#include \"units.hpp\"\n#include \"error_handling.hpp\"\n\nnamespace Sass {\n\n  /* the conversion matrix can be readed the following way */\n  /* if you go down, the factor is for the numerator (multiply) */\n  /* if you go right, the factor is for the denominator (divide) */\n  /* and yes, we actually use both, not sure why, but why not!? */\n\n  const double size_conversion_factors[6][6] =\n  {\n             /*  in         cm         pc         mm         pt         px        */\n    /* in   */ { 1,         2.54,      6,         25.4,      72,        96,       },\n    /* cm   */ { 1.0/2.54,  1,         6.0/2.54,  10,        72.0/2.54, 96.0/2.54 },\n    /* pc   */ { 1.0/6.0,   2.54/6.0,  1,         25.4/6.0,  72.0/6.0,  96.0/6.0  },\n    /* mm   */ { 1.0/25.4,  1.0/10.0,  6.0/25.4,  1,         72.0/25.4, 96.0/25.4 },\n    /* pt   */ { 1.0/72.0,  2.54/72.0, 6.0/72.0,  25.4/72.0, 1,         96.0/72.0 },\n    /* px   */ { 1.0/96.0,  2.54/96.0, 6.0/96.0,  25.4/96.0, 72.0/96.0, 1,        }\n  };\n\n  const double angle_conversion_factors[4][4] =\n  {\n             /*  deg        grad       rad        turn      */\n    /* deg  */ { 1,         40.0/36.0, PI/180.0,  1.0/360.0 },\n    /* grad */ { 36.0/40.0, 1,         PI/200.0,  1.0/400.0 },\n    /* rad  */ { 180.0/PI,  200.0/PI,  1,         0.5/PI    },\n    /* turn */ { 360.0,     400.0,     2.0*PI,    1         }\n  };\n\n  const double time_conversion_factors[2][2] =\n  {\n             /*  s          ms        */\n    /* s    */ { 1,         1000.0    },\n    /* ms   */ { 1/1000.0,  1         }\n  };\n  const double frequency_conversion_factors[2][2] =\n  {\n             /*  Hz         kHz       */\n    /* Hz   */ { 1,         1/1000.0  },\n    /* kHz  */ { 1000.0,    1         }\n  };\n  const double resolution_conversion_factors[3][3] =\n  {\n             /*  dpi        dpcm       dppx     */\n    /* dpi  */ { 1,         1/2.54,    1/96.0   },\n    /* dpcm */ { 2.54,      1,         2.54/96  },\n    /* dppx */ { 96,        96/2.54,   1        }\n  };\n\n  \nUnitClass get_unit_type(UnitType unit)\n  {\n    switch (unit & 0xFF00)\n    {\n      case UnitClass::LENGTH:      return UnitClass::LENGTH;\n      case UnitClass::ANGLE:       return UnitClass::ANGLE;\n      case UnitClass::TIME:        return UnitClass::TIME;\n      case UnitClass::FREQUENCY:   return UnitClass::FREQUENCY;\n      case UnitClass::RESOLUTION:  return UnitClass::RESOLUTION;\n      default:                     return UnitClass::INCOMMENSURABLE;\n    }\n  };\n\n  std::string get_unit_class(UnitType unit)\n  {\n    switch (unit & 0xFF00)\n    {\n      case UnitClass::LENGTH:      return \"LENGTH\";\n      case UnitClass::ANGLE:       return \"ANGLE\";\n      case UnitClass::TIME:        return \"TIME\";\n      case UnitClass::FREQUENCY:   return \"FREQUENCY\";\n      case UnitClass::RESOLUTION:  return \"RESOLUTION\";\n      default:                     return \"INCOMMENSURABLE\";\n    }\n  };\n\n  UnitType get_main_unit(const UnitClass unit)\n  {\n    switch (unit)\n    {\n      case UnitClass::LENGTH:      return UnitType::PX;\n      case UnitClass::ANGLE:       return UnitType::DEG;\n      case UnitClass::TIME:        return UnitType::SEC;\n      case UnitClass::FREQUENCY:   return UnitType::HERTZ;\n      case UnitClass::RESOLUTION:  return UnitType::DPI;\n      default:                     return UnitType::UNKNOWN;\n    }\n  };\n\n  UnitType string_to_unit(const std::string& s)\n  {\n    // size units\n    if      (s == \"px\")   return UnitType::PX;\n    else if (s == \"pt\")   return UnitType::PT;\n    else if (s == \"pc\")   return UnitType::PC;\n    else if (s == \"mm\")   return UnitType::MM;\n    else if (s == \"cm\")   return UnitType::CM;\n    else if (s == \"in\")   return UnitType::IN;\n    // angle units\n    else if (s == \"deg\")  return UnitType::DEG;\n    else if (s == \"grad\") return UnitType::GRAD;\n    else if (s == \"rad\")  return UnitType::RAD;\n    else if (s == \"turn\") return UnitType::TURN;\n    // time units\n    else if (s == \"s\")    return UnitType::SEC;\n    else if (s == \"ms\")   return UnitType::MSEC;\n    // frequency units\n    else if (s == \"Hz\")   return UnitType::HERTZ;\n    else if (s == \"kHz\")  return UnitType::KHERTZ;\n    // resolutions units\n    else if (s == \"dpi\")  return UnitType::DPI;\n    else if (s == \"dpcm\") return UnitType::DPCM;\n    else if (s == \"dppx\") return UnitType::DPPX;\n    // for unknown units\n    else return UnitType::UNKNOWN;\n  }\n\n  const char* unit_to_string(UnitType unit)\n  {\n    switch (unit) {\n      // size units\n      case UnitType::PX:      return \"px\";\n      case UnitType::PT:      return \"pt\";\n      case UnitType::PC:      return \"pc\";\n      case UnitType::MM:      return \"mm\";\n      case UnitType::CM:      return \"cm\";\n      case UnitType::IN:      return \"in\";\n      // angle units\n      case UnitType::DEG:     return \"deg\";\n      case UnitType::GRAD:    return \"grad\";\n      case UnitType::RAD:     return \"rad\";\n      case UnitType::TURN:    return \"turn\";\n      // time units\n      case UnitType::SEC:     return \"s\";\n      case UnitType::MSEC:    return \"ms\";\n      // frequency units\n      case UnitType::HERTZ:   return \"Hz\";\n      case UnitType::KHERTZ:  return \"kHz\";\n      // resolutions units\n      case UnitType::DPI:     return \"dpi\";\n      case UnitType::DPCM:    return \"dpcm\";\n      case UnitType::DPPX:    return \"dppx\";\n      // for unknown units\n      default:                return \"\";\n    }\n  }\n\n  std::string unit_to_class(const std::string& s)\n  {\n    if      (s == \"px\")   return \"LENGTH\";\n    else if (s == \"pt\")   return \"LENGTH\";\n    else if (s == \"pc\")   return \"LENGTH\";\n    else if (s == \"mm\")   return \"LENGTH\";\n    else if (s == \"cm\")   return \"LENGTH\";\n    else if (s == \"in\")   return \"LENGTH\";\n    // angle units\n    else if (s == \"deg\")  return \"ANGLE\";\n    else if (s == \"grad\") return \"ANGLE\";\n    else if (s == \"rad\")  return \"ANGLE\";\n    else if (s == \"turn\") return \"ANGLE\";\n    // time units\n    else if (s == \"s\")    return \"TIME\";\n    else if (s == \"ms\")   return \"TIME\";\n    // frequency units\n    else if (s == \"Hz\")   return \"FREQUENCY\";\n    else if (s == \"kHz\")  return \"FREQUENCY\";\n    // resolutions units\n    else if (s == \"dpi\")  return \"RESOLUTION\";\n    else if (s == \"dpcm\") return \"RESOLUTION\";\n    else if (s == \"dppx\") return \"RESOLUTION\";\n    // for unknown units\n    return \"CUSTOM:\" + s;\n  }\n\n  // throws incompatibleUnits exceptions\n  double conversion_factor(const std::string& s1, const std::string& s2)\n  {\n    // assert for same units\n    if (s1 == s2) return 1;\n    // get unit enum from string\n    UnitType u1 = string_to_unit(s1);\n    UnitType u2 = string_to_unit(s2);\n    // query unit group types\n    UnitClass t1 = get_unit_type(u1);\n    UnitClass t2 = get_unit_type(u2);\n    // return the conversion factor\n    return conversion_factor(u1, u2, t1, t2);\n  }\n\n  // throws incompatibleUnits exceptions\n  double conversion_factor(UnitType u1, UnitType u2, UnitClass t1, UnitClass t2)\n  {\n    // can't convert between groups\n    if (t1 != t2) return 0;\n    // get absolute offset\n    // used for array acces\n    size_t i1 = u1 - t1;\n    size_t i2 = u2 - t2;\n    // process known units\n    switch (t1) {\n      case LENGTH:\n        return size_conversion_factors[i1][i2];\n      case ANGLE:\n        return angle_conversion_factors[i1][i2];\n      case TIME:\n        return time_conversion_factors[i1][i2];\n      case FREQUENCY:\n        return frequency_conversion_factors[i1][i2];\n      case RESOLUTION:\n        return resolution_conversion_factors[i1][i2];\n      case INCOMMENSURABLE:\n        return 0;\n    }\n    // fallback\n    return 0;\n  }\n\n  double convert_units(const std::string& lhs, const std::string& rhs, int& lhsexp, int& rhsexp)\n  {\n    double f = 0;\n    // do not convert same ones\n    if (lhs == rhs) return 0;\n    // skip already canceled out unit\n    if (lhsexp == 0) return 0;\n    if (rhsexp == 0) return 0;\n    // check if it can be converted\n    UnitType ulhs = string_to_unit(lhs);\n    UnitType urhs = string_to_unit(rhs);\n    // skip units we cannot convert\n    if (ulhs == UNKNOWN) return 0;\n    if (urhs == UNKNOWN) return 0;\n    // query unit group types\n    UnitClass clhs = get_unit_type(ulhs);\n    UnitClass crhs = get_unit_type(urhs);\n    // skip units we cannot convert\n    if (clhs != crhs) return 0;\n    // if right denominator is bigger than lhs, we want to keep it in rhs unit\n    if (rhsexp < 0 && lhsexp > 0 && - rhsexp > lhsexp) {\n      // get the conversion factor for units\n      f = conversion_factor(urhs, ulhs, clhs, crhs);\n      // left hand side has been consumned\n      f = std::pow(f, lhsexp);\n      rhsexp += lhsexp;\n      lhsexp = 0;\n    }\n    else {\n      // get the conversion factor for units\n      f = conversion_factor(ulhs, urhs, clhs, crhs);\n      // right hand side has been consumned\n      f = std::pow(f, rhsexp);\n      lhsexp += rhsexp;\n      rhsexp = 0;\n    }\n    return f;\n  }\n\n  bool Units::operator< (const Units& rhs) const\n  {\n    return (numerators < rhs.numerators) &&\n           (denominators < rhs.denominators);\n  }\n  bool Units::operator== (const Units& rhs) const\n  {\n    return (numerators == rhs.numerators) &&\n           (denominators == rhs.denominators);\n  }\n\n  double Units::normalize()\n  {\n\n    size_t iL = numerators.size();\n    size_t nL = denominators.size();\n\n    // the final conversion factor\n    double factor = 1;\n\n    for (size_t i = 0; i < iL; i++) {\n      std::string &lhs = numerators[i];\n      UnitType ulhs = string_to_unit(lhs);\n      if (ulhs == UNKNOWN) continue;\n      UnitClass clhs = get_unit_type(ulhs);\n      UnitType umain = get_main_unit(clhs);\n      if (ulhs == umain) continue;\n      double f(conversion_factor(umain, ulhs, clhs, clhs));\n      if (f == 0) throw std::runtime_error(\"INVALID\");\n      numerators[i] = unit_to_string(umain);\n      factor /= f;\n    }\n\n    for (size_t n = 0; n < nL; n++) {\n      std::string &rhs = denominators[n];\n      UnitType urhs = string_to_unit(rhs);\n      if (urhs == UNKNOWN) continue;\n      UnitClass crhs = get_unit_type(urhs);\n      UnitType umain = get_main_unit(crhs);\n      if (urhs == umain) continue;\n      double f(conversion_factor(umain, urhs, crhs, crhs));\n      if (f == 0) throw std::runtime_error(\"INVALID\");\n      denominators[n] = unit_to_string(umain);\n      factor /= f;\n    }\n\n    std::sort (numerators.begin(), numerators.end());\n    std::sort (denominators.begin(), denominators.end());\n\n    // return for conversion\n    return factor;\n  }\n\n  double Units::reduce()\n  {\n\n    size_t iL = numerators.size();\n    size_t nL = denominators.size();\n\n    // have less than two units?\n    if (iL + nL < 2) return 1;\n\n    // first make sure same units cancel each other out\n    // it seems that a map table will fit nicely to do this\n    // we basically construct exponents for each unit\n    // has the advantage that they will be pre-sorted\n    std::map<std::string, int> exponents;\n\n    // initialize by summing up occurences in unit vectors\n    // this will already cancel out equivalent units (e.q. px/px)\n    for (size_t i = 0; i < iL; i ++) exponents[numerators[i]] += 1;\n    for (size_t n = 0; n < nL; n ++) exponents[denominators[n]] -= 1;\n\n    // the final conversion factor\n    double factor = 1;\n\n    // convert between compatible units\n    for (size_t i = 0; i < iL; i++) {\n      for (size_t n = 0; n < nL; n++) {\n        std::string &lhs = numerators[i], &rhs = denominators[n];\n        int &lhsexp = exponents[lhs], &rhsexp = exponents[rhs];\n        double f(convert_units(lhs, rhs, lhsexp, rhsexp));\n        if (f == 0) continue;\n        factor /= f;\n      }\n    }\n\n    // now we can build up the new unit arrays\n    numerators.clear();\n    denominators.clear();\n\n    // recreate sorted units vectors\n    for (auto exp : exponents) {\n      int &exponent = exp.second;\n      while (exponent > 0 && exponent --)\n        numerators.push_back(exp.first);\n      while (exponent < 0 && exponent ++)\n        denominators.push_back(exp.first);\n    }\n\n    // return for conversion\n    return factor;\n\n  }\n\n  std::string Units::unit() const\n  {\n    std::string u;\n    size_t iL = numerators.size();\n    size_t nL = denominators.size();\n    for (size_t i = 0; i < iL; i += 1) {\n      if (i) u += '*';\n      u += numerators[i];\n    }\n    if (nL != 0) u += '/';\n    for (size_t n = 0; n < nL; n += 1) {\n      if (n) u += '*';\n      u += denominators[n];\n    }\n    return u;\n  }\n\n  bool Units::is_unitless() const\n  {\n    return numerators.empty() &&\n           denominators.empty();\n  }\n\n  bool Units::is_valid_css_unit() const\n  {\n    return numerators.size() <= 1 &&\n           denominators.size() == 0;\n  }\n\n  // this does not cover all cases (multiple prefered units)\n  double Units::convert_factor(const Units& r) const\n  {\n\n    std::vector<std::string> miss_nums(0);\n    std::vector<std::string> miss_dens(0);\n    // create copy since we need these for state keeping\n    std::vector<std::string> r_nums(r.numerators);\n    std::vector<std::string> r_dens(r.denominators);\n\n    auto l_num_it = numerators.begin();\n    auto l_num_end = numerators.end();\n\n    bool l_unitless = is_unitless();\n    auto r_unitless = r.is_unitless();\n\n    // overall conversion\n    double factor = 1;\n\n    // process all left numerators\n    while (l_num_it != l_num_end)\n    {\n      // get and increment afterwards\n      const std::string l_num = *(l_num_it ++);\n\n      auto r_num_it = r_nums.begin(), r_num_end = r_nums.end();\n\n      bool found = false;\n      // search for compatible numerator\n      while (r_num_it != r_num_end)\n      {\n        // get and increment afterwards\n        const std::string r_num = *(r_num_it);\n        // get possible conversion factor for units\n        double conversion = conversion_factor(l_num, r_num);\n        // skip incompatible numerator\n        if (conversion == 0) {\n          ++ r_num_it;\n          continue;\n        }\n        // apply to global factor\n        factor *= conversion;\n        // remove item from vector\n        r_nums.erase(r_num_it);\n        // found numerator\n        found = true;\n        break;\n      }\n      // maybe we did not find any\n      // left numerator is leftover\n      if (!found) miss_nums.push_back(l_num);\n    }\n\n    auto l_den_it = denominators.begin();\n    auto l_den_end = denominators.end();\n\n    // process all left denominators\n    while (l_den_it != l_den_end)\n    {\n      // get and increment afterwards\n      const std::string l_den = *(l_den_it ++);\n\n      auto r_den_it = r_dens.begin();\n      auto r_den_end = r_dens.end();\n\n      bool found = false;\n      // search for compatible denominator\n      while (r_den_it != r_den_end)\n      {\n        // get and increment afterwards\n        const std::string r_den = *(r_den_it);\n        // get possible converstion factor for units\n        double conversion = conversion_factor(l_den, r_den);\n        // skip incompatible denominator\n        if (conversion == 0) {\n          ++ r_den_it;\n          continue;\n        }\n        // apply to global factor\n        factor /= conversion;\n        // remove item from vector\n        r_dens.erase(r_den_it);\n        // found denominator\n        found = true;\n        break;\n      }\n      // maybe we did not find any\n      // left denominator is leftover\n      if (!found) miss_dens.push_back(l_den);\n    }\n\n    // check left-overs (ToDo: might cancel out?)\n    if (miss_nums.size() > 0 && !r_unitless) {\n      throw Exception::IncompatibleUnits(r, *this);\n    }\n    else if (miss_dens.size() > 0 && !r_unitless) {\n      throw Exception::IncompatibleUnits(r, *this);\n    }\n    else if (r_nums.size() > 0 && !l_unitless) {\n      throw Exception::IncompatibleUnits(r, *this);\n    }\n    else if (r_dens.size() > 0 && !l_unitless) {\n      throw Exception::IncompatibleUnits(r, *this);\n    }\n\n    return factor;\n  }\n\n}\n\n// Path: src/libsass/src/remove_placeholders.cpp\n#include \"sass.hpp\"\n#include \"remove_placeholders.hpp\"\n#include \"context.hpp\"\n#include \"inspect.hpp\"\n#include <iostream>\n\nnamespace Sass {\n\n    Remove_Placeholders::Remove_Placeholders()\n    { }\n\n    void Remove_Placeholders::operator()(Block_Ptr b) {\n        for (size_t i = 0, L = b->length(); i < L; ++i) {\n            Statement_Ptr st = b->at(i);\n            st->perform(this);\n        }\n    }\n\n    Selector_List_Ptr Remove_Placeholders::remove_placeholders(Selector_List_Ptr sl)\n    {\n      Selector_List_Ptr new_sl = SASS_MEMORY_NEW(Selector_List, sl->pstate());\n\n      for (size_t i = 0, L = sl->length(); i < L; ++i) {\n          if (!sl->at(i)->contains_placeholder()) {\n              new_sl->append(sl->at(i));\n          }\n      }\n\n      return new_sl;\n\n    }\n\n\n    void Remove_Placeholders::operator()(Ruleset_Ptr r) {\n        // Create a new selector group without placeholders\n        Selector_List_Obj sl = Cast<Selector_List>(r->selector());\n\n        if (sl) {\n          // Set the new placeholder selector list\n          r->selector(remove_placeholders(sl));\n          // Remove placeholders in wrapped selectors\n          for (Complex_Selector_Obj cs : sl->elements()) {\n            while (cs) {\n              if (cs->head()) {\n                for (Simple_Selector_Obj& ss : cs->head()->elements()) {\n                  if (Wrapped_Selector_Ptr ws = Cast<Wrapped_Selector>(ss)) {\n                    if (Selector_List_Ptr wsl = Cast<Selector_List>(ws->selector())) {\n                      Selector_List_Ptr clean = remove_placeholders(wsl);\n                      // also clean superflous parent selectors\n                      // probably not really the correct place\n                      clean->remove_parent_selectors();\n                      ws->selector(clean);\n                    }\n                  }\n                }\n              }\n              cs = cs->tail();\n            }\n          }\n        }\n\n        // Iterate into child blocks\n        Block_Obj b = r->block();\n\n        for (size_t i = 0, L = b->length(); i < L; ++i) {\n            if (b->at(i)) {\n                Statement_Obj st = b->at(i);\n                st->perform(this);\n            }\n        }\n    }\n\n    void Remove_Placeholders::operator()(Media_Block_Ptr m) {\n        operator()(m->block());\n    }\n    void Remove_Placeholders::operator()(Supports_Block_Ptr m) {\n        operator()(m->block());\n    }\n\n    void Remove_Placeholders::operator()(Directive_Ptr a) {\n        if (a->block()) a->block()->perform(this);\n    }\n\n}\n\n// Path: src/libsass/src/operators.cpp\n#include \"sass.hpp\"\n#include \"operators.hpp\"\n\nnamespace Sass {\n\n  namespace Operators {\n\n    inline double add(double x, double y) { return x + y; }\n    inline double sub(double x, double y) { return x - y; }\n    inline double mul(double x, double y) { return x * y; }\n    inline double div(double x, double y) { return x / y; } // x/0 checked by caller\n\n    inline double mod(double x, double y) { // x/0 checked by caller\n      if ((x > 0 && y < 0) || (x < 0 && y > 0)) {\n        double ret = std::fmod(x, y);\n        return ret ? ret + y : ret;\n      } else {\n        return std::fmod(x, y);\n      }\n    }\n\n    typedef double (*bop)(double, double);\n    bop ops[Sass_OP::NUM_OPS] = {\n      0, 0, // and, or\n      0, 0, 0, 0, 0, 0, // eq, neq, gt, gte, lt, lte\n      add, sub, mul, div, mod\n    };\n\n    /* static function, has no pstate or traces */\n    bool eq(Expression_Obj lhs, Expression_Obj rhs)\n    {\n      // operation is undefined if one is not a number\n      if (!lhs || !rhs) throw Exception::UndefinedOperation(lhs, rhs, Sass_OP::EQ);\n      // use compare operator from ast node\n      return *lhs == *rhs;\n    }\n\n    /* static function, throws OperationError, has no pstate or traces */\n    bool cmp(Expression_Obj lhs, Expression_Obj rhs, const Sass_OP op)\n    {\n      // can only compare numbers!?\n      Number_Obj l = Cast<Number>(lhs);\n      Number_Obj r = Cast<Number>(rhs);\n      // operation is undefined if one is not a number\n      if (!l || !r) throw Exception::UndefinedOperation(lhs, rhs, op);\n      // use compare operator from ast node\n      return *l < *r;\n    }\n\n    /* static functions, throws OperationError, has no pstate or traces */\n    bool lt(Expression_Obj lhs, Expression_Obj rhs) { return cmp(lhs, rhs, Sass_OP::LT); }\n    bool neq(Expression_Obj lhs, Expression_Obj rhs) { return eq(lhs, rhs) == false; }\n    bool gt(Expression_Obj lhs, Expression_Obj rhs) { return !cmp(lhs, rhs, Sass_OP::GT) && neq(lhs, rhs); }\n    bool lte(Expression_Obj lhs, Expression_Obj rhs) { return cmp(lhs, rhs, Sass_OP::LTE) || eq(lhs, rhs); }\n    bool gte(Expression_Obj lhs, Expression_Obj rhs) { return !cmp(lhs, rhs, Sass_OP::GTE) || eq(lhs, rhs); }\n\n    /* colour math deprecation warning */\n    void op_color_deprecation(enum Sass_OP op, std::string lsh, std::string rhs, const ParserState& pstate)\n    {\n      std::string op_str(\n        op == Sass_OP::ADD ? \"plus\" :\n          op == Sass_OP::DIV ? \"div\" :\n            op == Sass_OP::SUB ? \"minus\" :\n              op == Sass_OP::MUL ? \"times\" : \"\"\n      );\n\n      std::string msg(\"The operation `\" + lsh + \" \" + op_str + \" \" + rhs + \"` is deprecated and will be an error in future versions.\");\n      std::string tail(\"Consider using Sass's color functions instead.\\nhttp://sass-lang.com/documentation/Sass/Script/Functions.html#other_color_functions\");\n\n      deprecated(msg, tail, false, pstate);\n    }\n\n    /* static function, throws OperationError, has no traces but optional pstate for returned value */\n    Value_Ptr op_strings(Sass::Operand operand, Value& lhs, Value& rhs, struct Sass_Inspect_Options opt, const ParserState& pstate, bool delayed)\n    {\n      enum Sass_OP op = operand.operand;\n\n      String_Quoted_Ptr lqstr = Cast<String_Quoted>(&lhs);\n      String_Quoted_Ptr rqstr = Cast<String_Quoted>(&rhs);\n\n      std::string lstr(lqstr ? lqstr->value() : lhs.to_string(opt));\n      std::string rstr(rqstr ? rqstr->value() : rhs.to_string(opt));\n\n      if (Cast<Null>(&lhs)) throw Exception::InvalidNullOperation(&lhs, &rhs, op);\n      if (Cast<Null>(&rhs)) throw Exception::InvalidNullOperation(&lhs, &rhs, op);\n\n      std::string sep;\n      switch (op) {\n        case Sass_OP::ADD: sep = \"\";   break;\n        case Sass_OP::SUB: sep = \"-\";  break;\n        case Sass_OP::DIV: sep = \"/\";  break;\n        case Sass_OP::EQ:  sep = \"==\"; break;\n        case Sass_OP::NEQ: sep = \"!=\"; break;\n        case Sass_OP::LT:  sep = \"<\";  break;\n        case Sass_OP::GT:  sep = \">\";  break;\n        case Sass_OP::LTE: sep = \"<=\"; break;\n        case Sass_OP::GTE: sep = \">=\"; break;\n        default:\n          throw Exception::UndefinedOperation(&lhs, &rhs, op);\n        break;\n      }\n\n      if (op == Sass_OP::ADD) {\n        // create string that might be quoted on output (but do not unquote what we pass)\n        return SASS_MEMORY_NEW(String_Quoted, pstate, lstr + rstr, 0, false, true);\n      }\n\n      // add whitespace around operator\n      // but only if result is not delayed\n      if (sep != \"\" && delayed == false) {\n        if (operand.ws_before) sep = \" \" + sep;\n        if (operand.ws_after) sep = sep + \" \";\n      }\n\n      if (op == Sass_OP::SUB || op == Sass_OP::DIV) {\n        if (lqstr && lqstr->quote_mark()) lstr = quote(lstr);\n        if (rqstr && rqstr->quote_mark()) rstr = quote(rstr);\n      }\n\n      return SASS_MEMORY_NEW(String_Constant, pstate, lstr + sep + rstr);\n    }\n\n    /* static function, throws OperationError, has no traces but optional pstate for returned value */\n    Value_Ptr op_colors(enum Sass_OP op, const Color& lhs, const Color& rhs, struct Sass_Inspect_Options opt, const ParserState& pstate, bool delayed)\n    {\n\n      if (lhs.a() != rhs.a()) {\n        throw Exception::AlphaChannelsNotEqual(&lhs, &rhs, op);\n      }\n      if ((op == Sass_OP::DIV || op == Sass_OP::MOD) && (!rhs.r() || !rhs.g() || !rhs.b())) {\n        throw Exception::ZeroDivisionError(lhs, rhs);\n...\n", "needle_token_start": 8862, "needle_token_end": 9010, "code_context_ntokens": 16225, "output": ["```cpp\nUnitClass get_unit_type(UnitType unit)\n{\n  switch (unit & 0xFF00)\n  {\n    case UnitClass::LENGTH:      return UnitClass::LENGTH;\n    case UnitClass::ANGLE:       return UnitClass::ANGLE;\n    case UnitClass::TIME:        return UnitClass::TIME;\n    case UnitClass::FREQUENCY:   return UnitClass::FREQUENCY;\n    case UnitClass::RESOLUTION:  return UnitClass::RESOLUTION;\n    default:                     return UnitClass::INCOMMENSURABLE;\n  }\n};\n```"]}
{"repo": "sass/node-sass", "name": "ParentSuperselectorChunker", "language": "cpp", "path": "src/libsass/src/extend.cpp", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this class is to manage and process a collection of CSS selectors to determine if one selector is a superselector of another, which means it encompasses all the styles of the other selector but may include additional constraints or styles.\n2. **Input**: The input is a single node representing a collection of CSS selectors.\n3. **Output**: The output is a processed collection where selectors that are less specific or redundant due to more specific superselectors are identified or filtered.\n4. **Procedure**: The procedure involves iterating through a collection of CSS selectors, comparing each selector's specificity and checking if one selector is a superselector of another. This involves creating temporary selectors to facilitate the comparison and updating the collection based on these comparisons to ensure it only contains the most specific and relevant selectors.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/libsass/src/debug.hpp\n#ifndef SASS_DEBUG_H\n#define SASS_DEBUG_H\n\n#include <stdint.h>\n\n#ifndef UINT32_MAX\n  #define UINT32_MAX 0xffffffffU\n#endif\n\nenum dbg_lvl_t : uint32_t {\n  NONE = 0,\n  TRIM = 1,\n  CHUNKS = 2,\n  SUBWEAVE = 4,\n  WEAVE = 8,\n  EXTEND_COMPOUND = 16,\n  EXTEND_COMPLEX = 32,\n  LCS = 64,\n  EXTEND_OBJECT = 128,\n  ALL = UINT32_MAX\n};\n\n#ifdef DEBUG\n\n#ifndef DEBUG_LVL\nconst uint32_t debug_lvl = UINT32_MAX;\n#else\nconst uint32_t debug_lvl = (DEBUG_LVL);\n#endif // DEBUG_LVL\n\n#define DEBUG_PRINT(lvl, x) if((lvl) & debug_lvl) { std::cerr << x; }\n#define DEBUG_PRINTLN(lvl, x) if((lvl) & debug_lvl) { std::cerr << x << std::endl; }\n#define DEBUG_EXEC(lvl, x) if((lvl) & debug_lvl) { x; }\n\n#else // DEBUG\n\n#define DEBUG_PRINT(lvl, x)\n#define DEBUG_PRINTLN(lvl, x)\n#define DEBUG_EXEC(lvl, x)\n\n#endif // DEBUG\n\n...\n// Path: src/libsass/src/sass_util.hpp\n#ifndef SASS_SASS_UTIL_H\n#define SASS_SASS_UTIL_H\n\n#include \"ast.hpp\"\n#include \"node.hpp\"\n#include \"debug.hpp\"\n\nnamespace Sass {\n\n\n\n\n  /*\n   This is for ports of functions in the Sass:Util module.\n   */\n\n\n  /*\n    # Return a Node collection of all possible paths through the given Node collection of Node collections.\n    #\n    # @param arrs [NodeCollection<NodeCollection<Node>>]\n    # @return [NodeCollection<NodeCollection<Node>>]\n    #\n    # @example\n    #   paths([[1, 2], [3, 4], [5]]) #=>\n    #     # [[1, 3, 5],\n    #     #  [2, 3, 5],\n    #     #  [1, 4, 5],\n    #     #  [2, 4, 5]]\n  */\n  Node paths(const Node& arrs);\n\n\n  /*\n  This class is a default implementation of a Node comparator that can be passed to the lcs function below.\n  It uses operator== for equality comparision. It then returns one if the Nodes are equal.\n  */\n  class DefaultLcsComparator {\n  public:\n    bool operator()(const Node& one, const Node& two, Node& out) const {\n      // TODO: Is this the correct C++ interpretation?\n      // block ||= proc {|a, b| a == b && a}\n      if (one == two) {\n        out = one;\n        return true;\n      }\n\n      return false;\n    }\n  };\n\n\n  typedef std::vector<std::vector<int> > LCSTable;\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_backtrace.\n\n  # Computes a single longest common subsequence for arrays x and y.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Reading_out_an_LCS\n  */\n  template<typename ComparatorType>\n  Node lcs_backtrace(const LCSTable& c, const Node& x, const Node& y, int i, int j, const ComparatorType& comparator) {\n    DEBUG_PRINTLN(LCS, \"LCSBACK: X=\" << x << \" Y=\" << y << \" I=\" << i << \" J=\" << j)\n\n    if (i == 0 || j == 0) {\n      DEBUG_PRINTLN(LCS, \"RETURNING EMPTY\")\n      return Node::createCollection();\n    }\n\n    NodeDeque& xChildren = *(x.collection());\n    NodeDeque& yChildren = *(y.collection());\n\n    Node compareOut = Node::createNil();\n    if (comparator(xChildren[i], yChildren[j], compareOut)) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER ELEM COMPARE\")\n      Node result = lcs_backtrace(c, x, y, i - 1, j - 1, comparator);\n      result.collection()->push_back(compareOut);\n      return result;\n    }\n\n    if (c[i][j - 1] > c[i - 1][j]) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER TABLE COMPARE\")\n      return lcs_backtrace(c, x, y, i, j - 1, comparator);\n    }\n\n    DEBUG_PRINTLN(LCS, \"FINAL RETURN\")\n    return lcs_backtrace(c, x, y, i - 1, j, comparator);\n  }\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_table.\n\n  # Calculates the memoization table for the Least Common Subsequence algorithm.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Computing_the_length_of_the_LCS\n  */\n  template<typename ComparatorType>\n  void lcs_table(const Node& x, const Node& y, const ComparatorType& comparator, LCSTable& out) {\n    DEBUG_PRINTLN(LCS, \"LCSTABLE: X=\" << x << \" Y=\" << y)\n\n    NodeDeque& xChildren = *(x.collection());\n    NodeDeque& yChildren = *(y.collection());\n\n    LCSTable c(xChildren.size(), std::vector<int>(yChildren.size()));\n\n    // These shouldn't be necessary since the vector will be initialized to 0 already.\n    // x.size.times {|i| c[i][0] = 0}\n    // y.size.times {|j| c[0][j] = 0}\n\n    for (size_t i = 1; i < xChildren.size(); i++) {\n      for (size_t j = 1; j < yChildren.size(); j++) {\n        Node compareOut = Node::createNil();\n\n        if (comparator(xChildren[i], yChildren[j], compareOut)) {\n          c[i][j] = c[i - 1][j - 1] + 1;\n        } else {\n          c[i][j] = std::max(c[i][j - 1], c[i - 1][j]);\n        }\n      }\n    }\n\n    out = c;\n  }\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs.\n\n  # Computes a single longest common subsequence for `x` and `y`.\n  # If there are more than one longest common subsequences,\n  # the one returned is that which starts first in `x`.\n\n  # @param x [NodeCollection]\n  # @param y [NodeCollection]\n  # @comparator An equality check between elements of `x` and `y`.\n  # @return [NodeCollection] The LCS\n\n  http://en.wikipedia.org/wiki/Longest_common_subsequence_problem\n  */\n  template<typename ComparatorType>\n  Node lcs(Node& x, Node& y, const ComparatorType& comparator) {\n    DEBUG_PRINTLN(LCS, \"LCS: X=\" << x << \" Y=\" << y)\n\n    Node newX = Node::createCollection();\n    newX.collection()->push_back(Node::createNil());\n    newX.plus(x);\n\n    Node newY = Node::createCollection();\n    newY.collection()->push_back(Node::createNil());\n    newY.plus(y);\n\n    LCSTable table;\n    lcs_table(newX, newY, comparator, table);\n\n    return lcs_backtrace(table, newX, newY, static_cast<int>(newX.collection()->size()) - 1, static_cast<int>(newY.collection()->size()) - 1, comparator);\n  }\n\n\n  /*\n  This is the equivalent of ruby sass' Sass::Util.flatten and [].flatten.\n  Sass::Util.flatten requires the number of levels to flatten, while\n  [].flatten doesn't and will flatten the entire array. This function\n  supports both.\n\n  # Flattens the first `n` nested arrays. If n == -1, all arrays will be flattened\n  #\n  # @param arr [NodeCollection] The array to flatten\n  # @param n [int] The number of levels to flatten\n  # @return [NodeCollection] The flattened array\n  */\n  Node flatten(Node& arr, int n = -1);\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.group_by_to_a.\n\n  # Performs the equivalent of `enum.group_by.to_a`, but with a guaranteed\n  # order. Unlike [#hash_to_a], the resulting order isn't sorted key order;\n  # instead, it's the same order as `#group_by` has under Ruby 1.9 (key\n  # appearance order).\n  #\n  # @param enum [Enumerable]\n  # @return [Array<[Object, Array]>] An array of pairs.\n\n  TODO: update @param and @return once I know what those are.\n\n  The following is the modified version of the ruby code that was more portable to C++. You\n  should be able to drop it into ruby 3.2.19 and get the same results from ruby sass.\n\n    def group_by_to_a(enum, &block)\n      order = {}\n\n      arr = []\n\n      grouped = {}\n\n      for e in enum do\n        key = block[e]\n        unless order.include?(key)\n          order[key] = order.size\n        end\n\n        if not grouped.has_key?(key) then\n          grouped[key] = [e]\n        else\n          grouped[key].push(e)\n        end\n      end\n\n      grouped.each do |key, vals|\n        arr[order[key]] = [key, vals]\n      end\n\n      arr\n    end\n\n  */\n  template<typename EnumType, typename KeyType, typename KeyFunctorType>\n  void group_by_to_a(std::vector<EnumType>& enumeration, KeyFunctorType& keyFunc, std::vector<std::pair<KeyType, std::vector<EnumType> > >& arr /*out*/) {\n\n    std::map<unsigned int, KeyType> order;\n\n    std::map<size_t, std::vector<EnumType> > grouped;\n\n    for (typename std::vector<EnumType>::iterator enumIter = enumeration.begin(), enumIterEnd = enumeration.end(); enumIter != enumIterEnd; enumIter++) {\n      EnumType& e = *enumIter;\n\n      KeyType key = keyFunc(e);\n\n      if (grouped.find(key->hash()) == grouped.end()) {\n        order.insert(std::make_pair((unsigned int)order.size(), key));\n\n        std::vector<EnumType> newCollection;\n        newCollection.push_back(e);\n        grouped.insert(std::make_pair(key->hash(), newCollection));\n      } else {\n        std::vector<EnumType>& collection = grouped.at(key->hash());\n        collection.push_back(e);\n      }\n    }\n\n    for (unsigned int index = 0; index < order.size(); index++) {\n      KeyType& key = order.at(index);\n      std::vector<EnumType>& values = grouped.at(key->hash());\n\n      std::pair<KeyType, std::vector<EnumType> > grouping = std::make_pair(key, values);\n\n      arr.push_back(grouping);\n    }\n  }\n\n\n}\n\n#endif\n\n// Path: src/libsass/src/extend.cpp\n#include \"sass.hpp\"\n#include \"extend.hpp\"\n#include \"context.hpp\"\n#include \"backtrace.hpp\"\n#include \"paths.hpp\"\n#include \"parser.hpp\"\n#include \"expand.hpp\"\n#include \"node.hpp\"\n#include \"sass_util.hpp\"\n#include \"remove_placeholders.hpp\"\n#include \"debug.hpp\"\n#include <iostream>\n#include <deque>\n#include <set>\n\n/*\n NOTES:\n\n - The print* functions print to cerr. This allows our testing frameworks (like sass-spec) to ignore the output, which\n   is very helpful when debugging. The format of the output is mainly to wrap things in square brackets to match what\n   ruby already outputs (to make comparisons easier).\n\n - For the direct porting effort, we're trying to port method-for-method until we get all the tests passing.\n   Where applicable, I've tried to include the ruby code above the function for reference until all our tests pass.\n   The ruby code isn't always directly portable, so I've tried to include any modified ruby code that was actually\n   used for the porting.\n\n - DO NOT try to optimize yet. We get a tremendous benefit out of comparing the output of each stage of the extend to the ruby\n   output at the same stage. This makes it much easier to determine where problems are. Try to keep as close to\n   the ruby code as you can until we have all the sass-spec tests passing. Then, we should optimize. However, if you see\n   something that could probably be optimized, let's not forget it. Add a // TODO: or // IMPROVEMENT: comment.\n\n - Coding conventions in this file (these may need to be changed before merging back into master)\n   - Very basic hungarian notation:\n     p prefix for pointers (pSelector)\n     no prefix for value types and references (selector)\n   - Use STL iterators where possible\n   - prefer verbose naming over terse naming\n   - use typedefs for STL container types for make maintenance easier\n\n - You may see a lot of comments that say \"// TODO: is this the correct combinator?\". See the comment referring to combinators\n   in extendCompoundSelector for a more extensive explanation of my confusion. I think our divergence in data model from ruby\n   sass causes this to be necessary.\n\n\n GLOBAL TODOS:\n\n - wrap the contents of the print functions in DEBUG preprocesser conditionals so they will be optimized away in non-debug mode.\n\n - consider making the extend* functions member functions to avoid passing around ctx and subset_map map around. This has the\n   drawback that the implementation details of the operator are then exposed to the outside world, which is not ideal and\n   can cause additional compile time dependencies.\n\n - mark the helper methods in this file static to given them compilation unit linkage.\n\n - implement parent directive matching\n\n - fix compilation warnings for unused Extend members if we really don't need those references anymore.\n */\n\n\nnamespace Sass {\n\n\n\n#ifdef DEBUG\n\n  // TODO: move the ast specific ostream operators into ast.hpp/ast.cpp\n  std::ostream& operator<<(std::ostream& os, const Complex_Selector::Combinator combinator) {\n    switch (combinator) {\n      case Complex_Selector::ANCESTOR_OF: os << \"\\\" \\\"\"; break;\n      case Complex_Selector::PARENT_OF:   os << \"\\\">\\\"\"; break;\n      case Complex_Selector::PRECEDES:    os << \"\\\"~\\\"\"; break;\n      case Complex_Selector::ADJACENT_TO: os << \"\\\"+\\\"\"; break;\n      case Complex_Selector::REFERENCE:   os << \"\\\"/\\\"\"; break;\n    }\n\n    return os;\n  }\n\n\n  std::ostream& operator<<(std::ostream& os, Compound_Selector& compoundSelector) {\n    for (size_t i = 0, L = compoundSelector.length(); i < L; ++i) {\n      if (i > 0) os << \", \";\n      os << compoundSelector[i]->to_string();\n    }\n    return os;\n  }\n\n  std::ostream& operator<<(std::ostream& os, Simple_Selector& simpleSelector) {\n    os << simpleSelector.to_string();\n    return os;\n  }\n\n  // Print a string representation of a Compound_Selector\n  static void printSimpleSelector(Simple_Selector* pSimpleSelector, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    if (pSimpleSelector) {\n      std::cerr << \"[\" << *pSimpleSelector << \"]\";\n    } else {\n      std::cerr << \"NULL\";\n    }\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n  // Print a string representation of a Compound_Selector\n  static void printCompoundSelector(Compound_Selector_Ptr pCompoundSelector, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    if (pCompoundSelector) {\n      std::cerr << \"[\" << *pCompoundSelector << \"]\";\n    } else {\n      std::cerr << \"NULL\";\n    }\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n\n  std::ostream& operator<<(std::ostream& os, Complex_Selector& complexSelector) {\n\n    os << \"[\";\n    Complex_Selector_Ptr pIter = &complexSelector;\n    bool first = true;\n    while (pIter) {\n      if (pIter->combinator() != Complex_Selector::ANCESTOR_OF) {\n        if (!first) {\n          os << \", \";\n        }\n        first = false;\n        os << pIter->combinator();\n      }\n\n      if (!first) {\n        os << \", \";\n      }\n      first = false;\n\n      if (pIter->head()) {\n        os << pIter->head()->to_string();\n      } else {\n        os << \"NULL_HEAD\";\n      }\n\n      pIter = pIter->tail();\n    }\n    os << \"]\";\n\n    return os;\n  }\n\n\n  // Print a string representation of a Complex_Selector\n  static void printComplexSelector(Complex_Selector_Ptr pComplexSelector, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    if (pComplexSelector) {\n      std::cerr << *pComplexSelector;\n    } else {\n      std::cerr << \"NULL\";\n    }\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n  static void printSelsNewSeqPairCollection(SubSetMapLookups& collection, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n    bool first = true;\n    std::cerr << \"[\";\n    for(SubSetMapLookup& pair : collection) {\n      if (first) {\n        first = false;\n      } else {\n        std::cerr << \", \";\n      }\n      std::cerr << \"[\";\n      Compound_Selector_Ptr pSels = pair.first;\n      Complex_Selector_Ptr pNewSelector = pair.second;\n      std::cerr << \"[\" << *pSels << \"], \";\n      printComplexSelector(pNewSelector, NULL, false);\n    }\n    std::cerr << \"]\";\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n  // Print a string representation of a ComplexSelectorSet\n  static void printSourcesSet(ComplexSelectorSet& sources, const char* message=NULL, bool newline=true) {\n\n    if (message) {\n      std::cerr << message;\n    }\n\n    // Convert to a deque of strings so we can sort since order doesn't matter in a set. This should cut down on\n    // the differences we see when debug printing.\n    typedef std::deque<std::string> SourceStrings;\n    SourceStrings sourceStrings;\n    for (ComplexSelectorSet::iterator iterator = sources.begin(), iteratorEnd = sources.end(); iterator != iteratorEnd; ++iterator) {\n      Complex_Selector_Ptr pSource = *iterator;\n      std::stringstream sstream;\n      sstream << complexSelectorToNode(pSource);\n      sourceStrings.push_back(sstream.str());\n    }\n\n    // Sort to get consistent output\n    std::sort(sourceStrings.begin(), sourceStrings.end());\n\n    std::cerr << \"ComplexSelectorSet[\";\n    for (SourceStrings::iterator iterator = sourceStrings.begin(), iteratorEnd = sourceStrings.end(); iterator != iteratorEnd; ++iterator) {\n      std::string source = *iterator;\n      if (iterator != sourceStrings.begin()) {\n        std::cerr << \", \";\n      }\n      std::cerr << source;\n    }\n    std::cerr << \"]\";\n\n    if (newline) {\n      std::cerr << std::endl;\n    }\n  }\n\n\n  std::ostream& operator<<(std::ostream& os, SubSetMapPairs& entries) {\n    os << \"SUBSET_MAP_ENTRIES[\";\n\n    for (SubSetMapPairs::iterator iterator = entries.begin(), endIterator = entries.end(); iterator != endIterator; ++iterator) {\n      Complex_Selector_Obj pExtComplexSelector = iterator->first;    // The selector up to where the @extend is (ie, the thing to merge)\n      Compound_Selector_Obj pExtCompoundSelector = iterator->second; // The stuff after the @extend\n\n      if (iterator != entries.begin()) {\n        os << \", \";\n      }\n\n      os << \"(\";\n\n      if (pExtComplexSelector) {\n        std::cerr << *pExtComplexSelector;\n      } else {\n        std::cerr << \"NULL\";\n      }\n\n      os << \" -> \";\n\n      if (pExtCompoundSelector) {\n        std::cerr << *pExtCompoundSelector;\n      } else {\n        std::cerr << \"NULL\";\n      }\n\n      os << \")\";\n\n    }\n\n    os << \"]\";\n\n    return os;\n  }\n#endif\n\n  static bool parentSuperselector(Complex_Selector_Ptr pOne, Complex_Selector_Ptr pTwo) {\n    // TODO: figure out a better way to create a Complex_Selector from scratch\n    // TODO: There's got to be a better way. This got ugly quick...\n    Element_Selector_Obj fakeParent = SASS_MEMORY_NEW(Element_Selector, ParserState(\"[FAKE]\"), \"temp\");\n    Compound_Selector_Obj fakeHead = SASS_MEMORY_NEW(Compound_Selector, ParserState(\"[FAKE]\"), 1 /*size*/);\n    fakeHead->elements().push_back(fakeParent);\n    Complex_Selector_Obj fakeParentContainer = SASS_MEMORY_NEW(Complex_Selector, ParserState(\"[FAKE]\"), Complex_Selector::ANCESTOR_OF, fakeHead /*head*/, NULL /*tail*/);\n\n    pOne->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n    pTwo->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n\n    bool isSuperselector = pOne->is_superselector_of(pTwo);\n\n    pOne->clear_innermost();\n    pTwo->clear_innermost();\n\n    return isSuperselector;\n  }\n\n  void nodeToComplexSelectorDeque(const Node& node, ComplexSelectorDeque& out) {\n    for (NodeDeque::iterator iter = node.collection()->begin(), iterEnd = node.collection()->end(); iter != iterEnd; iter++) {\n      Node& child = *iter;\n      out.push_back(nodeToComplexSelector(child));\n    }\n  }\n\n  Node complexSelectorDequeToNode(const ComplexSelectorDeque& deque) {\n    Node result = Node::createCollection();\n\n    for (ComplexSelectorDeque::const_iterator iter = deque.begin(), iterEnd = deque.end(); iter != iterEnd; iter++) {\n      Complex_Selector_Obj pChild = *iter;\n      result.collection()->push_back(complexSelectorToNode(pChild));\n    }\n\n    return result;\n  }\n\n  class LcsCollectionComparator {\n  public:\n    LcsCollectionComparator() {}\n\n    bool operator()(Complex_Selector_Obj pOne, Complex_Selector_Obj pTwo, Complex_Selector_Obj& pOut) const {\n      /*\n      This code is based on the following block from ruby sass' subweave\n        do |s1, s2|\n          next s1 if s1 == s2\n          next unless s1.first.is_a?(SimpleSequence) && s2.first.is_a?(SimpleSequence)\n          next s2 if parent_superselector?(s1, s2)\n          next s1 if parent_superselector?(s2, s1)\n        end\n      */\n\n      if (*pOne == *pTwo) {\n        pOut = pOne;\n        return true;\n      }\n\n      if (pOne->combinator() != Complex_Selector::ANCESTOR_OF || pTwo->combinator() != Complex_Selector::ANCESTOR_OF) {\n        return false;\n      }\n\n      if (parentSuperselector(pOne, pTwo)) {\n        pOut = pTwo;\n        return true;\n      }\n\n      if (parentSuperselector(pTwo, pOne)) {\n        pOut = pOne;\n        return true;\n      }\n\n      return false;\n    }\n  };\n\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_backtrace.\n\n  # Computes a single longest common subsequence for arrays x and y.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Reading_out_an_LCS\n  */\n  void lcs_backtrace(const LCSTable& c, ComplexSelectorDeque& x, ComplexSelectorDeque& y, int i, int j, const LcsCollectionComparator& comparator, ComplexSelectorDeque& out) {\n    //DEBUG_PRINTLN(LCS, \"LCSBACK: X=\" << x << \" Y=\" << y << \" I=\" << i << \" J=\" << j)\n    // TODO: make printComplexSelectorDeque and use DEBUG_EXEC AND DEBUG_PRINTLN HERE to get equivalent output\n\n    if (i == 0 || j == 0) {\n      DEBUG_PRINTLN(LCS, \"RETURNING EMPTY\")\n      return;\n    }\n\n\n    Complex_Selector_Obj pCompareOut;\n    if (comparator(x[i], y[j], pCompareOut)) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER ELEM COMPARE\")\n      lcs_backtrace(c, x, y, i - 1, j - 1, comparator, out);\n      out.push_back(pCompareOut);\n      return;\n    }\n\n    if (c[i][j - 1] > c[i - 1][j]) {\n      DEBUG_PRINTLN(LCS, \"RETURNING AFTER TABLE COMPARE\")\n      lcs_backtrace(c, x, y, i, j - 1, comparator, out);\n      return;\n    }\n\n    DEBUG_PRINTLN(LCS, \"FINAL RETURN\")\n    lcs_backtrace(c, x, y, i - 1, j, comparator, out);\n    return;\n  }\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs_table.\n\n  # Calculates the memoization table for the Least Common Subsequence algorithm.\n  # Algorithm from http://en.wikipedia.org/wiki/Longest_common_subsequence_problem#Computing_the_length_of_the_LCS\n  */\n  void lcs_table(const ComplexSelectorDeque& x, const ComplexSelectorDeque& y, const LcsCollectionComparator& comparator, LCSTable& out) {\n    //DEBUG_PRINTLN(LCS, \"LCSTABLE: X=\" << x << \" Y=\" << y)\n    // TODO: make printComplexSelectorDeque and use DEBUG_EXEC AND DEBUG_PRINTLN HERE to get equivalent output\n\n    LCSTable c(x.size(), std::vector<int>(y.size()));\n\n    // These shouldn't be necessary since the vector will be initialized to 0 already.\n    // x.size.times {|i| c[i][0] = 0}\n    // y.size.times {|j| c[0][j] = 0}\n\n    for (size_t i = 1; i < x.size(); i++) {\n      for (size_t j = 1; j < y.size(); j++) {\n        Complex_Selector_Obj pCompareOut;\n\n        if (comparator(x[i], y[j], pCompareOut)) {\n          c[i][j] = c[i - 1][j - 1] + 1;\n        } else {\n          c[i][j] = std::max(c[i][j - 1], c[i - 1][j]);\n        }\n      }\n    }\n\n    out = c;\n  }\n\n  /*\n  This is the equivalent of ruby's Sass::Util.lcs.\n\n  # Computes a single longest common subsequence for `x` and `y`.\n  # If there are more than one longest common subsequences,\n  # the one returned is that which starts first in `x`.\n\n  # @param x [NodeCollection]\n  # @param y [NodeCollection]\n  # @comparator An equality check between elements of `x` and `y`.\n  # @return [NodeCollection] The LCS\n\n  http://en.wikipedia.org/wiki/Longest_common_subsequence_problem\n  */\n  void lcs(ComplexSelectorDeque& x, ComplexSelectorDeque& y, const LcsCollectionComparator& comparator, ComplexSelectorDeque& out) {\n    //DEBUG_PRINTLN(LCS, \"LCS: X=\" << x << \" Y=\" << y)\n    // TODO: make printComplexSelectorDeque and use DEBUG_EXEC AND DEBUG_PRINTLN HERE to get equivalent output\n\n    x.push_front(NULL);\n    y.push_front(NULL);\n\n    LCSTable table;\n    lcs_table(x, y, comparator, table);\n\n    return lcs_backtrace(table, x, y, static_cast<int>(x.size()) - 1, static_cast<int>(y.size()) - 1, comparator, out);\n  }\n\n\n  /*\n   This is the equivalent of ruby's Sequence.trim.\n\n   The following is the modified version of the ruby code that was more portable to C++. You\n   should be able to drop it into ruby 3.2.19 and get the same results from ruby sass.\n\n        # Avoid truly horrific quadratic behavior. TODO: I think there\n        # may be a way to get perfect trimming without going quadratic.\n        return seqses if seqses.size > 100\n\n        # Keep the results in a separate array so we can be sure we aren't\n        # comparing against an already-trimmed selector. This ensures that two\n        # identical selectors don't mutually trim one another.\n        result = seqses.dup\n\n        # This is n^2 on the sequences, but only comparing between\n        # separate sequences should limit the quadratic behavior.\n        seqses.each_with_index do |seqs1, i|\n          tempResult = []\n\n          for seq1 in seqs1 do\n            max_spec = 0\n            for seq in _sources(seq1) do\n              max_spec = [max_spec, seq.specificity].max\n            end\n\n\n            isMoreSpecificOuter = false\n            for seqs2 in result do\n              if seqs1.equal?(seqs2) then\n                next\n              end\n\n              # Second Law of Extend: the specificity of a generated selector\n              # should never be less than the specificity of the extending\n              # selector.\n              #\n              # See https://github.com/nex3/sass/issues/324.\n              isMoreSpecificInner = false\n              for seq2 in seqs2 do\n                isMoreSpecificInner = _specificity(seq2) >= max_spec && _superselector?(seq2, seq1)\n                if isMoreSpecificInner then\n                  break\n                end\n              end\n\n              if isMoreSpecificInner then\n                isMoreSpecificOuter = true\n                break\n              end\n            end\n\n            if !isMoreSpecificOuter then\n              tempResult.push(seq1)\n            end\n          end\n\n          result[i] = tempResult\n\n        end\n\n        result\n   */\n  /*\n   - IMPROVEMENT: We could probably work directly in the output trimmed deque.\n   */\n  Node Extend::trim(Node& seqses, bool isReplace) {\n    // See the comments in the above ruby code before embarking on understanding this function.\n\n    // Avoid poor performance in extreme cases.\n    if (seqses.collection()->size() > 100) {\n      return seqses;\n    }\n\n\n    DEBUG_PRINTLN(TRIM, \"TRIM: \" << seqses)\n\n\n    Node result = Node::createCollection();\n    result.plus(seqses);\n\n    DEBUG_PRINTLN(TRIM, \"RESULT INITIAL: \" << result)\n\n    // Normally we use the standard STL iterators, but in this case, we need to access the result collection by index since we're\n    // iterating the input collection, computing a value, and then setting the result in the output collection. We have to keep track\n    // of the index manually.\n    int toTrimIndex = 0;\n\n    for (NodeDeque::iterator seqsesIter = seqses.collection()->begin(), seqsesIterEnd = seqses.collection()->end(); seqsesIter != seqsesIterEnd; ++seqsesIter) {\n      Node& seqs1 = *seqsesIter;\n\n      DEBUG_PRINTLN(TRIM, \"SEQS1: \" << seqs1 << \" \" << toTrimIndex)\n\n      Node tempResult = Node::createCollection();\n      tempResult.got_line_feed = seqs1.got_line_feed;\n\n      for (NodeDeque::iterator seqs1Iter = seqs1.collection()->begin(), seqs1EndIter = seqs1.collection()->end(); seqs1Iter != seqs1EndIter; ++seqs1Iter) {\n        Node& seq1 = *seqs1Iter;\n\n        Complex_Selector_Obj pSeq1 = nodeToComplexSelector(seq1);\n\n        // Compute the maximum specificity. This requires looking at the \"sources\" of the sequence. See SimpleSequence.sources in the ruby code\n        // for a good description of sources.\n        //\n        // TODO: I'm pretty sure there's a bug in the sources code. It was implemented for sass-spec's 182_test_nested_extend_loop test.\n        // While the test passes, I compared the state of each trim call to verify correctness. The last trim call had incorrect sources. We\n        // had an extra source that the ruby version did not have. Without a failing test case, this is going to be extra hard to find. My\n        // best guess at this point is that we're cloning an object somewhere and maintaining the sources when we shouldn't be. This is purely\n        // a guess though.\n        unsigned long maxSpecificity = isReplace ? pSeq1->specificity() : 0;\n        ComplexSelectorSet sources = pSeq1->sources();\n\n        DEBUG_PRINTLN(TRIM, \"TRIM SEQ1: \" << seq1)\n        DEBUG_EXEC(TRIM, printSourcesSet(sources, \"TRIM SOURCES: \"))\n\n        for (ComplexSelectorSet::iterator sourcesSetIterator = sources.begin(), sourcesSetIteratorEnd = sources.end(); sourcesSetIterator != sourcesSetIteratorEnd; ++sourcesSetIterator) {\n          const Complex_Selector_Obj& pCurrentSelector = *sourcesSetIterator;\n          maxSpecificity = std::max(maxSpecificity, pCurrentSelector->specificity());\n        }\n\n        DEBUG_PRINTLN(TRIM, \"MAX SPECIFICITY: \" << maxSpecificity)\n\n        bool isMoreSpecificOuter = false;\n\n        int resultIndex = 0;\n\n        for (NodeDeque::iterator resultIter = result.collection()->begin(), resultIterEnd = result.collection()->end(); resultIter != resultIterEnd; ++resultIter) {\n          Node& seqs2 = *resultIter;\n\n          DEBUG_PRINTLN(TRIM, \"SEQS1: \" << seqs1)\n          DEBUG_PRINTLN(TRIM, \"SEQS2: \" << seqs2)\n\n          // Do not compare the same sequence to itself. The ruby call we're trying to\n          // emulate is: seqs1.equal?(seqs2). equal? is an object comparison, not an equivalency comparision.\n          // Since we have the same pointers in seqes and results, we can do a pointer comparision. seqs1 is\n          // derived from seqses and seqs2 is derived from result.\n          if (seqs1.collection() == seqs2.collection()) {\n            DEBUG_PRINTLN(TRIM, \"CONTINUE\")\n            continue;\n          }\n\n          bool isMoreSpecificInner = false;\n\n          for (NodeDeque::iterator seqs2Iter = seqs2.collection()->begin(), seqs2IterEnd = seqs2.collection()->end(); seqs2Iter != seqs2IterEnd; ++seqs2Iter) {\n            Node& seq2 = *seqs2Iter;\n\n            Complex_Selector_Obj pSeq2 = nodeToComplexSelector(seq2);\n\n            DEBUG_PRINTLN(TRIM, \"SEQ2 SPEC: \" << pSeq2->specificity())\n            DEBUG_PRINTLN(TRIM, \"IS SPEC: \" << pSeq2->specificity() << \" >= \" << maxSpecificity << \" \" << (pSeq2->specificity() >= maxSpecificity ? \"true\" : \"false\"))\n            DEBUG_PRINTLN(TRIM, \"IS SUPER: \" << (pSeq2->is_superselector_of(pSeq1) ? \"true\" : \"false\"))\n\n            isMoreSpecificInner = pSeq2->specificity() >= maxSpecificity && pSeq2->is_superselector_of(pSeq1);\n\n            if (isMoreSpecificInner) {\n              DEBUG_PRINTLN(TRIM, \"FOUND MORE SPECIFIC\")\n              break;\n            }\n          }\n\n          // If we found something more specific, we're done. Let the outer loop know and stop iterating.\n          if (isMoreSpecificInner) {\n            isMoreSpecificOuter = true;\n            break;\n          }\n\n          resultIndex++;\n        }\n\n        if (!isMoreSpecificOuter) {\n          DEBUG_PRINTLN(TRIM, \"PUSHING: \" << seq1)\n          tempResult.collection()->push_back(seq1);\n        }\n\n      }\n\n      DEBUG_PRINTLN(TRIM, \"RESULT BEFORE ASSIGN: \" << result)\n      DEBUG_PRINTLN(TRIM, \"TEMP RESULT: \" << toTrimIndex << \" \" << tempResult)\n      (*result.collection())[toTrimIndex] = tempResult;\n\n      toTrimIndex++;\n\n      DEBUG_PRINTLN(TRIM, \"RESULT: \" << result)\n    }\n\n    return result;\n  }\n\n\n\n  static bool parentSuperselector(const Node& one, const Node& two) {\n    // TODO: figure out a better way to create a Complex_Selector from scratch\n    // TODO: There's got to be a better way. This got ugly quick...\n    Element_Selector_Obj fakeParent = SASS_MEMORY_NEW(Element_Selector, ParserState(\"[FAKE]\"), \"temp\");\n    Compound_Selector_Obj fakeHead = SASS_MEMORY_NEW(Compound_Selector, ParserState(\"[FAKE]\"), 1 /*size*/);\n    fakeHead->elements().push_back(fakeParent);\n    Complex_Selector_Obj fakeParentContainer = SASS_MEMORY_NEW(Complex_Selector, ParserState(\"[FAKE]\"), Complex_Selector::ANCESTOR_OF, fakeHead /*head*/, NULL /*tail*/);\n\n    Complex_Selector_Obj pOneWithFakeParent = nodeToComplexSelector(one);\n    pOneWithFakeParent->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n    Complex_Selector_Obj pTwoWithFakeParent = nodeToComplexSelector(two);\n    pTwoWithFakeParent->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n\n    return pOneWithFakeParent->is_superselector_of(pTwoWithFakeParent);\n  }\n\n\n  class ParentSuperselectorChunker {\n  public:\n    \nParentSuperselectorChunker(Node& lcs) : mLcs(lcs) {}\n    Node& mLcs;\n\n    bool operator()(const Node& seq) const {\n      // {|s| parent_superselector?(s.first, lcs.first)}\n      if (seq.collection()->size() == 0) return false;\n      return parentSuperselector(seq.collection()->front(), mLcs.collection()->front());\n    }\n  };\n\n  class SubweaveEmptyChunker {\n  public:\n    bool operator()(const Node& seq) const {\n      // {|s| s.empty?}\n\n      return seq.collection()->empty();\n    }\n  };\n\n  /*\n  # Takes initial subsequences of `seq1` and `seq2` and returns all\n  # orderings of those subsequences. The initial subsequences are determined\n  # by a block.\n  #\n  # Destructively removes the initial subsequences of `seq1` and `seq2`.\n  #\n  # For example, given `(A B C | D E)` and `(1 2 | 3 4 5)` (with `|`\n  # denoting the boundary of the initial subsequence), this would return\n  # `[(A B C 1 2), (1 2 A B C)]`. The sequences would then be `(D E)` and\n  # `(3 4 5)`.\n  #\n  # @param seq1 [Array]\n  # @param seq2 [Array]\n  # @yield [a] Used to determine when to cut off the initial subsequences.\n  #   Called repeatedly for each sequence until it returns true.\n  # @yieldparam a [Array] A final subsequence of one input sequence after\n  #   cutting off some initial subsequence.\n  # @yieldreturn [Boolean] Whether or not to cut off the initial subsequence\n  #   here.\n  # @return [Array<Array>] All possible orderings of the initial subsequences.\n  def chunks(seq1, seq2)\n    chunk1 = []\n    chunk1 << seq1.shift until yield seq1\n    chunk2 = []\n    chunk2 << seq2.shift until yield seq2\n    return [] if chunk1.empty? && chunk2.empty?\n    return [chunk2] if chunk1.empty?\n    return [chunk1] if chunk2.empty?\n    [chunk1 + chunk2, chunk2 + chunk1]\n  end\n  */\n  template<typename ChunkerType>\n  static Node chunks(Node& seq1, Node& seq2, const ChunkerType& chunker) {\n    Node chunk1 = Node::createCollection();\n    while (seq1.collection()->size() && !chunker(seq1)) {\n      chunk1.collection()->push_back(seq1.collection()->front());\n      seq1.collection()->pop_front();\n    }\n\n    Node chunk2 = Node::createCollection();\n    while (!seq2.collection()->empty() && !chunker(seq2)) {\n      chunk2.collection()->push_back(seq2.collection()->front());\n      seq2.collection()->pop_front();\n    }\n\n    if (chunk1.collection()->empty() && chunk2.collection()->empty()) {\n      DEBUG_PRINTLN(CHUNKS, \"RETURNING BOTH EMPTY\")\n      return Node::createCollection();\n    }\n\n    if (chunk1.collection()->empty()) {\n      Node chunk2Wrapper = Node::createCollection();\n      chunk2Wrapper.collection()->push_back(chunk2);\n      DEBUG_PRINTLN(CHUNKS, \"RETURNING ONE EMPTY\")\n      return chunk2Wrapper;\n    }\n\n    if (chunk2.collection()->empty()) {\n      Node chunk1Wrapper = Node::createCollection();\n      chunk1Wrapper.collection()->push_back(chunk1);\n      DEBUG_PRINTLN(CHUNKS, \"RETURNING TWO EMPTY\")\n      return chunk1Wrapper;\n    }\n\n    Node perms = Node::createCollection();\n\n    Node firstPermutation = Node::createCollection();\n    firstPermutation.collection()->insert(firstPermutation.collection()->end(), chunk1.collection()->begin(), chunk1.collection()->end());\n    firstPermutation.collection()->insert(firstPermutation.collection()->end(), chunk2.collection()->begin(), chunk2.collection()->end());\n    perms.collection()->push_back(firstPermutation);\n\n    Node secondPermutation = Node::createCollection();\n    secondPermutation.collection()->insert(secondPermutation.collection()->end(), chunk2.collection()->begin(), chunk2.collection()->end());\n    secondPermutation.collection()->insert(secondPermutation.collection()->end(), chunk1.collection()->begin(), chunk1.collection()->end());\n    perms.collection()->push_back(secondPermutation);\n\n    DEBUG_PRINTLN(CHUNKS, \"RETURNING PERM\")\n\n    return perms;\n  }\n\n\n  static Node groupSelectors(Node& seq) {\n    Node newSeq = Node::createCollection();\n\n    Node tail = Node::createCollection();\n    tail.plus(seq);\n\n    while (!tail.collection()->empty()) {\n      Node head = Node::createCollection();\n\n      do {\n        head.collection()->push_back(tail.collection()->front());\n        tail.collection()->pop_front();\n      } while (!tail.collection()->empty() && (head.collection()->back().isCombinator() || tail.collection()->front().isCombinator()));\n\n      newSeq.collection()->push_back(head);\n    }\n\n    return newSeq;\n  }\n\n\n  static void getAndRemoveInitialOps(Node& seq, Node& ops) {\n    NodeDeque& seqCollection = *(seq.collection());\n    NodeDeque& opsCollection = *(ops.collection());\n\n    while (seqCollection.size() > 0 && seqCollection.front().isCombinator()) {\n      opsCollection.push_back(seqCollection.front());\n      seqCollection.pop_front();\n    }\n  }\n\n\n  static void getAndRemoveFinalOps(Node& seq, Node& ops) {\n    NodeDeque& seqCollection = *(seq.collection());\n    NodeDeque& opsCollection = *(ops.collection());\n\n    while (seqCollection.size() > 0 && seqCollection.back().isCombinator()) {\n      opsCollection.push_back(seqCollection.back()); // Purposefully reversed to match ruby code\n      seqCollection.pop_back();\n    }\n  }\n\n\n  /*\n      def merge_initial_ops(seq1, seq2)\n        ops1, ops2 = [], []\n        ops1 << seq1.shift while seq1.first.is_a?(String)\n        ops2 << seq2.shift while seq2.first.is_a?(String)\n\n        newline = false\n        newline ||= !!ops1.shift if ops1.first == \"\\n\"\n        newline ||= !!ops2.shift if ops2.first == \"\\n\"\n\n        # If neither sequence is a subsequence of the other, they cannot be\n        # merged successfully\n        lcs = Sass::Util.lcs(ops1, ops2)\n        return unless lcs == ops1 || lcs == ops2\n        return (newline ? [\"\\n\"] : []) + (ops1.size > ops2.size ? ops1 : ops2)\n      end\n  */\n  static Node mergeInitialOps(Node& seq1, Node& seq2) {\n    Node ops1 = Node::createCollection();\n    Node ops2 = Node::createCollection();\n\n    getAndRemoveInitialOps(seq1, ops1);\n    getAndRemoveInitialOps(seq2, ops2);\n\n    // TODO: Do we have this information available to us?\n    // newline = false\n    // newline ||= !!ops1.shift if ops1.first == \"\\n\"\n    // newline ||= !!ops2.shift if ops2.first == \"\\n\"\n\n    // If neither sequence is a subsequence of the other, they cannot be merged successfully\n    DefaultLcsComparator lcsDefaultComparator;\n    Node opsLcs = lcs(ops1, ops2, lcsDefaultComparator);\n\n    if (!(opsLcs == ops1 || opsLcs == ops2)) {\n      return Node::createNil();\n    }\n\n    // TODO: more newline logic\n    // return (newline ? [\"\\n\"] : []) + (ops1.size > ops2.size ? ops1 : ops2)\n\n    return (ops1.collection()->size() > ops2.collection()->size() ? ops1 : ops2);\n  }\n\n\n  /*\n      def merge_final_ops(seq1, seq2, res = [])\n\n\n        # This code looks complicated, but it's actually just a bunch of special\n        # cases for interactions between different combinators.\n        op1, op2 = ops1.first, ops2.first\n        if op1 && op2\n          sel1 = seq1.pop\n          sel2 = seq2.pop\n          if op1 == '~' && op2 == '~'\n            if sel1.superselector?(sel2)\n              res.unshift sel2, '~'\n            elsif sel2.superselector?(sel1)\n              res.unshift sel1, '~'\n            else\n              merged = sel1.unify(sel2.members, sel2.subject?)\n              res.unshift [\n                [sel1, '~', sel2, '~'],\n                [sel2, '~', sel1, '~'],\n                ([merged, '~'] if merged)\n              ].compact\n            end\n          elsif (op1 == '~' && op2 == '+') || (op1 == '+' && op2 == '~')\n            if op1 == '~'\n              tilde_sel, plus_sel = sel1, sel2\n            else\n              tilde_sel, plus_sel = sel2, sel1\n            end\n\n            if tilde_sel.superselector?(plus_sel)\n              res.unshift plus_sel, '+'\n            else\n              merged = plus_sel.unify(tilde_sel.members, tilde_sel.subject?)\n              res.unshift [\n                [tilde_sel, '~', plus_sel, '+'],\n                ([merged, '+'] if merged)\n              ].compact\n            end\n          elsif op1 == '>' && %w[~ +].include?(op2)\n            res.unshift sel2, op2\n            seq1.push sel1, op1\n          elsif op2 == '>' && %w[~ +].include?(op1)\n            res.unshift sel1, op1\n            seq2.push sel2, op2\n          elsif op1 == op2\n            return unless merged = sel1.unify(sel2.members, sel2.subject?)\n            res.unshift merged, op1\n          else\n            # Unknown selector combinators can't be unified\n            return\n          end\n          return merge_final_ops(seq1, seq2, res)\n        elsif op1\n          seq2.pop if op1 == '>' && seq2.last && seq2.last.superselector?(seq1.last)\n          res.unshift seq1.pop, op1\n          return merge_final_ops(seq1, seq2, res)\n        else # op2\n          seq1.pop if op2 == '>' && seq1.last && seq1.last.superselector?(seq2.last)\n          res.unshift seq2.pop, op2\n          return merge_final_ops(seq1, seq2, res)\n        end\n      end\n  */\n  static Node mergeFinalOps(Node& seq1, Node& seq2, Node& res) {\n\n    Node ops1 = Node::createCollection();\n    Node ops2 = Node::createCollection();\n\n    getAndRemoveFinalOps(seq1, ops1);\n    getAndRemoveFinalOps(seq2, ops2);\n\n    // TODO: do we have newlines to remove?\n    // ops1.reject! {|o| o == \"\\n\"}\n    // ops2.reject! {|o| o == \"\\n\"}\n\n    if (ops1.collection()->empty() && ops2.collection()->empty()) {\n      return res;\n    }\n\n    if (ops1.collection()->size() > 1 || ops2.collection()->size() > 1) {\n      DefaultLcsComparator lcsDefaultComparator;\n      Node opsLcs = lcs(ops1, ops2, lcsDefaultComparator);\n\n      // If there are multiple operators, something hacky's going on. If one is a supersequence of the other, use that, otherwise give up.\n\n      if (!(opsLcs == ops1 || opsLcs == ops2)) {\n        return Node::createNil();\n      }\n\n      if (ops1.collection()->size() > ops2.collection()->size()) {\n        res.collection()->insert(res.collection()->begin(), ops1.collection()->rbegin(), ops1.collection()->rend());\n      } else {\n        res.collection()->insert(res.collection()->begin(), ops2.collection()->rbegin(), ops2.collection()->rend());\n      }\n\n      return res;\n    }\n\n    if (!ops1.collection()->empty() && !ops2.collection()->empty()) {\n\n      Node op1 = ops1.collection()->front();\n      Node op2 = ops2.collection()->front();\n\n      Node sel1 = seq1.collection()->back();\n      seq1.collection()->pop_back();\n\n      Node sel2 = seq2.collection()->back();\n      seq2.collection()->pop_back();\n\n      if (op1.combinator() == Complex_Selector::PRECEDES && op2.combinator() == Complex_Selector::PRECEDES) {\n\n        if (sel1.selector()->is_superselector_of(sel2.selector())) {\n\n          res.collection()->push_front(op1 /*PRECEDES - could have been op2 as well*/);\n          res.collection()->push_front(sel2);\n\n        } else if (sel2.selector()->is_superselector_of(sel1.selector())) {\n\n          res.collection()->push_front(op1 /*PRECEDES - could have been op2 as well*/);\n          res.collection()->push_front(sel1);\n\n        } else {\n\n          DEBUG_PRINTLN(ALL, \"sel1: \" << sel1)\n          DEBUG_PRINTLN(ALL, \"sel2: \" << sel2)\n\n          Complex_Selector_Obj pMergedWrapper = SASS_MEMORY_CLONE(sel1.selector()); // Clone the Complex_Selector to get back to something we can transform to a node once we replace the head with the unification result\n          // TODO: does subject matter? Ruby: return unless merged = sel1.unify(sel2.members, sel2.subject?)\n          Compound_Selector_Ptr pMerged = sel1.selector()->head()->unify_with(sel2.selector()->head());\n          pMergedWrapper->head(pMerged);\n\n          DEBUG_EXEC(ALL, printCompoundSelector(pMerged, \"MERGED: \"))\n\n          Node newRes = Node::createCollection();\n\n          Node firstPerm = Node::createCollection();\n          firstPerm.collection()->push_back(sel1);\n          firstPerm.collection()->push_back(Node::createCombinator(Complex_Selector::PRECEDES));\n          firstPerm.collection()->push_back(sel2);\n          firstPerm.collection()->push_back(Node::createCombinator(Complex_Selector::PRECEDES));\n          newRes.collection()->push_back(firstPerm);\n\n          Node secondPerm = Node::createCollection();\n          secondPerm.collection()->push_back(sel2);\n          secondPerm.collection()->push_back(Node::createCombinator(Complex_Selector::PRECEDES));\n          secondPerm.collection()->push_back(sel1);\n          secondPerm.collection()->push_back(Node::createCombinator(Complex_Selector::PRECEDES));\n          newRes.collection()->push_back(secondPerm);\n\n          if (pMerged) {\n            Node mergedPerm = Node::createCollection();\n            mergedPerm.collection()->push_back(Node::createSelector(pMergedWrapper));\n            mergedPerm.collection()->push_back(Node::createCombinator(Complex_Selector::PRECEDES));\n            newRes.collection()->push_back(mergedPerm);\n          }\n\n          res.collection()->push_front(newRes);\n\n          DEBUG_PRINTLN(ALL, \"RESULT: \" << res)\n\n        }\n\n      } else if (((op1.combinator() == Complex_Selector::PRECEDES && op2.combinator() == Complex_Selector::ADJACENT_TO)) || ((op1.combinator() == Complex_Selector::ADJACENT_TO && op2.combinator() == Complex_Selector::PRECEDES))) {\n\n          Node tildeSel = sel1;\n          Node plusSel = sel2;\n          Node plusOp = op2;\n          if (op1.combinator() != Complex_Selector::PRECEDES) {\n            tildeSel = sel2;\n            plusSel = sel1;\n            plusOp = op1;\n          }\n\n          if (tildeSel.selector()->is_superselector_of(plusSel.selector())) {\n\n            res.collection()->push_front(plusOp);\n            res.collection()->push_front(plusSel);\n\n          } else {\n\n            DEBUG_PRINTLN(ALL, \"PLUS SEL: \" << plusSel)\n            DEBUG_PRINTLN(ALL, \"TILDE SEL: \" << tildeSel)\n\n            Complex_Selector_Obj pMergedWrapper = SASS_MEMORY_CLONE(plusSel.selector()); // Clone the Complex_Selector to get back to something we can transform to a node once we replace the head with the unification result\n            // TODO: does subject matter? Ruby: merged = plus_sel.unify(tilde_sel.members, tilde_sel.subject?)\n            Compound_Selector_Ptr pMerged = plusSel.selector()->head()->unify_with(tildeSel.selector()->head());\n            pMergedWrapper->head(pMerged);\n\n            DEBUG_EXEC(ALL, printCompoundSelector(pMerged, \"MERGED: \"))\n\n            Node newRes = Node::createCollection();\n\n            Node firstPerm = Node::createCollection();\n            firstPerm.collection()->push_back(tildeSel);\n            firstPerm.collection()->push_back(Node::createCombinator(Complex_Selector::PRECEDES));\n            firstPerm.collection()->push_back(plusSel);\n            firstPerm.collection()->push_back(Node::createCombinator(Complex_Selector::ADJACENT_TO));\n            newRes.collection()->push_back(firstPerm);\n\n            if (pMerged) {\n              Node mergedPerm = Node::createCollection();\n              mergedPerm.collection()->push_back(Node::createSelector(pMergedWrapper));\n              mergedPerm.collection()->push_back(Node::createCombinator(Complex_Selector::ADJACENT_TO));\n              newRes.collection()->push_back(mergedPerm);\n            }\n\n            res.collection()->push_front(newRes);\n\n            DEBUG_PRINTLN(ALL, \"RESULT: \" << res)\n\n          }\n      } else if (op1.combinator() == Complex_Selector::PARENT_OF && (op2.combinator() == Complex_Selector::PRECEDES || op2.combinator() == Complex_Selector::ADJACENT_TO)) {\n\n        res.collection()->push_front(op2);\n        res.collection()->push_front(sel2);\n\n        seq1.collection()->push_back(sel1);\n        seq1.collection()->push_back(op1);\n\n      } else if (op2.combinator() == Complex_Selector::PARENT_OF && (op1.combinator() == Complex_Selector::PRECEDES || op1.combinator() == Complex_Selector::ADJACENT_TO)) {\n\n        res.collection()->push_front(op1);\n        res.collection()->push_front(sel1);\n\n        seq2.collection()->push_back(sel2);\n        seq2.collection()->push_back(op2);\n\n      } else if (op1.combinator() == op2.combinator()) {\n\n        DEBUG_PRINTLN(ALL, \"sel1: \" << sel1)\n        DEBUG_PRINTLN(ALL, \"sel2: \" << sel2)\n\n        Complex_Selector_Obj pMergedWrapper = SASS_MEMORY_CLONE(sel1.selector()); // Clone the Complex_Selector to get back to something we can transform to a node once we replace the head with the unification result\n        // TODO: does subject matter? Ruby: return unless merged = sel1.unify(sel2.members, sel2.subject?)\n        Compound_Selector_Ptr pMerged = sel1.selector()->head()->unify_with(sel2.selector()->head());\n        pMergedWrapper->head(pMerged);\n\n        DEBUG_EXEC(ALL, printCompoundSelector(pMerged, \"MERGED: \"))\n\n        if (!pMerged) {\n          return Node::createNil();\n        }\n\n        res.collection()->push_front(op1);\n        res.collection()->push_front(Node::createSelector(pMergedWrapper));\n\n        DEBUG_PRINTLN(ALL, \"RESULT: \" << res)\n\n      } else {\n        return Node::createNil();\n      }\n\n      return mergeFinalOps(seq1, seq2, res);\n\n    } else if (!ops1.collection()->empty()) {\n\n      Node op1 = ops1.collection()->front();\n\n      if (op1.combinator() == Complex_Selector::PARENT_OF && !seq2.collection()->empty() && seq2.collection()->back().selector()->is_superselector_of(seq1.collection()->back().selector())) {\n        seq2.collection()->pop_back();\n      }\n\n      // TODO: consider unshift(NodeCollection, Node)\n      res.collection()->push_front(op1);\n      res.collection()->push_front(seq1.collection()->back());\n      seq1.collection()->pop_back();\n\n      return mergeFinalOps(seq1, seq2, res);\n\n    } else { // !ops2.collection()->empty()\n\n      Node op2 = ops2.collection()->front();\n\n      if (op2.combinator() == Complex_Selector::PARENT_OF && !seq1.collection()->empty() && seq1.collection()->back().selector()->is_superselector_of(seq2.collection()->back().selector())) {\n        seq1.collection()->pop_back();\n      }\n\n      res.collection()->push_front(op2);\n...\n", "needle_token_start": 10389, "needle_token_end": 10409, "code_context_ntokens": 16016, "output": ["```cpp\nstatic bool parentSuperselector(const Node& one, const Node& two) {\n  // TODO: figure out a better way to create a Complex_Selector from scratch\n  // TODO: There's got to be a better way. This got ugly quick...\n  Element_Selector_Obj fakeParent = SASS_MEMORY_NEW(Element_Selector, ParserState(\"[FAKE]\"), \"temp\");\n  Compound_Selector_Obj fakeHead = SASS_MEMORY_NEW(Compound_Selector, ParserState(\"[FAKE]\"), 1 /*size*/);\n  fakeHead->elements().push_back(fakeParent);\n  Complex_Selector_Obj fakeParentContainer = SASS_MEMORY_NEW(Complex_Selector, ParserState(\"[FAKE]\"), Complex_Selector::ANCESTOR_OF, fakeHead /*head*/, NULL /*tail*/);\n\n  Complex_Selector_Obj pOneWithFakeParent = nodeToComplexSelector(one);\n  pOneWithFakeParent->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n  Complex_Selector_Obj pTwoWithFakeParent = nodeToComplexSelector(two);\n  pTwoWithFakeParent->set_innermost(fakeParentContainer, Complex_Selector::ANCESTOR_OF);\n\n  return pOneWithFakeParent->is_superselector_of(pTwoWithFakeParent);\n}\n```"]}
{"repo": "sass/node-sass", "name": "Operand", "language": "cpp", "path": "src/libsass/src/ast.hpp", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this class is to represent an operation in a stylesheet processing context, encapsulating details about the operation and its formatting preferences.\n2. **Input**: It takes an operation type, and two boolean flags indicating whether there should be whitespace before and after the operation.\n3. **Output**: It does not produce a direct output but initializes an object that holds the operation type and whitespace preferences.\n4. **Procedure**: The procedure involves initializing the class with the specified operation type and whitespace flags, storing these values in the object's properties for later use in processing or formatting operations in stylesheets.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/libsass/src/utf8/checked.h\n// Copyright 2006 Nemanja Trifunovic\n\n/*\nPermission is hereby granted, free of charge, to any person or organization\nobtaining a copy of the software and accompanying documentation covered by\nthis license (the \"Software\") to use, reproduce, display, distribute,\nexecute, and transmit the Software, and to prepare derivative works of the\nSoftware, and to permit third-parties to whom the Software is furnished to\ndo so, all subject to the following:\n\n...\n// Path: src/libsass/src/utf8/unchecked.h\n// Copyright 2006 Nemanja Trifunovic\n\n/*\nPermission is hereby granted, free of charge, to any person or organization\nobtaining a copy of the software and accompanying documentation covered by\nthis license (the \"Software\") to use, reproduce, display, distribute,\nexecute, and transmit the Software, and to prepare derivative works of the\nSoftware, and to permit third-parties to whom the Software is furnished to\ndo so, all subject to the following:\n\nThe copyright notices in the Software and this entire statement, including\nthe above license grant, this restriction and the following disclaimer,\nmust be included in all copies of the Software, in whole or in part, and\nall derivative works of the Software, unless such copies or derivative\nworks are solely in the form of machine-executable object code generated by\na source language processor.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT\nSHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE\nFOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,\nARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n*/\n\n\n#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731\n#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731\n\n#include \"core.h\"\n\nnamespace utf8\n{\n    namespace unchecked\n    {\n        template <typename octet_iterator>\n        octet_iterator append(uint32_t cp, octet_iterator result)\n        {\n            if (cp < 0x80)                        // one octet\n                *(result++) = static_cast<uint8_t>(cp);\n            else if (cp < 0x800) {                // two octets\n                *(result++) = static_cast<uint8_t>((cp >> 6)          | 0xc0);\n                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);\n            }\n            else if (cp < 0x10000) {              // three octets\n                *(result++) = static_cast<uint8_t>((cp >> 12)         | 0xe0);\n                *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);\n                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);\n            }\n            else {                                // four octets\n                *(result++) = static_cast<uint8_t>((cp >> 18)         | 0xf0);\n                *(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);\n                *(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);\n                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);\n            }\n            return result;\n        }\n\n        template <typename octet_iterator>\n        uint32_t next(octet_iterator& it)\n        {\n            uint32_t cp = utf8::internal::mask8(*it);\n            typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);\n            switch (length) {\n                case 1:\n                    break;\n                case 2:\n                    it++;\n                    cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);\n                    break;\n                case 3:\n                    ++it;\n                    cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);\n                    ++it;\n                    cp += (*it) & 0x3f;\n                    break;\n                case 4:\n                    ++it;\n                    cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);\n                    ++it;\n                    cp += (utf8::internal::mask8(*it) << 6) & 0xfff;\n                    ++it;\n                    cp += (*it) & 0x3f;\n                    break;\n            }\n            ++it;\n            return cp;\n        }\n\n        template <typename octet_iterator>\n        uint32_t peek_next(octet_iterator it)\n        {\n            return utf8::unchecked::next(it);\n        }\n\n        template <typename octet_iterator>\n        uint32_t prior(octet_iterator& it)\n        {\n            while (utf8::internal::is_trail(*(--it))) ;\n            octet_iterator temp = it;\n            return utf8::unchecked::next(temp);\n        }\n\n        // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous)\n        template <typename octet_iterator>\n        inline uint32_t previous(octet_iterator& it)\n        {\n            return utf8::unchecked::prior(it);\n        }\n\n        template <typename octet_iterator, typename distance_type>\n        void advance (octet_iterator& it, distance_type n)\n        {\n            for (distance_type i = 0; i < n; ++i)\n                utf8::unchecked::next(it);\n        }\n\n        template <typename octet_iterator, typename distance_type>\n        void retreat (octet_iterator& it, distance_type n)\n        {\n            for (distance_type i = 0; i < n; ++i)\n                utf8::unchecked::prior(it);\n        }\n\n        template <typename octet_iterator>\n        typename std::iterator_traits<octet_iterator>::difference_type\n        distance (octet_iterator first, octet_iterator last)\n        {\n            typename std::iterator_traits<octet_iterator>::difference_type dist;\n            for (dist = 0; first < last; ++dist)\n                utf8::unchecked::next(first);\n            return dist;\n        }\n\n        template <typename u16bit_iterator, typename octet_iterator>\n        octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)\n        {\n            while (start != end) {\n                uint32_t cp = utf8::internal::mask16(*start++);\n            // Take care of surrogate pairs first\n                if (utf8::internal::is_lead_surrogate(cp)) {\n                    uint32_t trail_surrogate = utf8::internal::mask16(*start++);\n                    cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;\n                }\n                result = utf8::unchecked::append(cp, result);\n            }\n            return result;\n        }\n\n        template <typename u16bit_iterator, typename octet_iterator>\n        u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)\n        {\n            while (start < end) {\n                uint32_t cp = utf8::unchecked::next(start);\n                if (cp > 0xffff) { //make a surrogate pair\n                    *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);\n                    *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);\n                }\n                else\n                    *result++ = static_cast<uint16_t>(cp);\n            }\n            return result;\n        }\n\n        template <typename octet_iterator, typename u32bit_iterator>\n        octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)\n        {\n            while (start != end)\n                result = utf8::unchecked::append(*(start++), result);\n\n            return result;\n        }\n\n        template <typename octet_iterator, typename u32bit_iterator>\n        u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)\n        {\n            while (start < end)\n                (*result++) = utf8::unchecked::next(start);\n\n            return result;\n        }\n\n        // The iterator class\n        template <typename octet_iterator>\n          class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {\n            octet_iterator it;\n            public:\n            iterator () {}\n            explicit iterator (const octet_iterator& octet_it): it(octet_it) {}\n            // the default \"big three\" are OK\n            octet_iterator base () const { return it; }\n            uint32_t operator * () const\n            {\n                octet_iterator temp = it;\n                return utf8::unchecked::next(temp);\n            }\n            bool operator == (const iterator& rhs) const\n            {\n                return (it == rhs.it);\n            }\n            bool operator != (const iterator& rhs) const\n            {\n                return !(operator == (rhs));\n            }\n            iterator& operator ++ ()\n            {\n                ::std::advance(it, utf8::internal::sequence_length(it));\n                return *this;\n            }\n            iterator operator ++ (int)\n            {\n                iterator temp = *this;\n                ::std::advance(it, utf8::internal::sequence_length(it));\n                return temp;\n            }\n            iterator& operator -- ()\n            {\n                utf8::unchecked::prior(it);\n                return *this;\n            }\n            iterator operator -- (int)\n            {\n                iterator temp = *this;\n                utf8::unchecked::prior(it);\n                return temp;\n            }\n          }; // class iterator\n\n    } // namespace utf8::unchecked\n} // namespace utf8\n\n\n#endif // header guard\n\n\n// Path: src/libsass/src/utf8.h\n// Copyright 2006 Nemanja Trifunovic\n\n/*\nPermission is hereby granted, free of charge, to any person or organization\nobtaining a copy of the software and accompanying documentation covered by\nthis license (the \"Software\") to use, reproduce, display, distribute,\nexecute, and transmit the Software, and to prepare derivative works of the\nSoftware, and to permit third-parties to whom the Software is furnished to\ndo so, all subject to the following:\n\nThe copyright notices in the Software and this entire statement, including\nthe above license grant, this restriction and the following disclaimer,\nmust be included in all copies of the Software, in whole or in part, and\nall derivative works of the Software, unless such copies or derivative\nworks are solely in the form of machine-executable object code generated by\na source language processor.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT\nSHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE\nFOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,\nARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n*/\n\n\n#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731\n#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731\n\n#include \"utf8/checked.h\"\n#include \"utf8/unchecked.h\"\n\n#endif // header guard\n\n// Path: src/libsass/src/utf8_string.hpp\n#ifndef SASS_UTF8_STRING_H\n#define SASS_UTF8_STRING_H\n\n#include <string>\n#include \"utf8.h\"\n\nnamespace Sass {\n  namespace UTF_8 {\n\n    // naming conventions:\n    // offset: raw byte offset (0 based)\n    // position: code point offset (0 based)\n    // index: code point offset (1 based or negative)\n\n    // function that will count the number of code points (utf-8 characters) from the beginning to the given end\n    size_t code_point_count(const std::string& str, size_t start, size_t end);\n    size_t code_point_count(const std::string& str);\n\n    // function that will return the byte offset of a code point in a\n    size_t offset_at_position(const std::string& str, size_t position);\n\n    // function that returns number of bytes in a character in a string\n    size_t code_point_size_at_offset(const std::string& str, size_t offset);\n\n    // function that will return a normalized index, given a crazy one\n    size_t normalize_index(int index, size_t len);\n\n    #ifdef _WIN32\n    // functions to handle unicode paths on windows\n    std::string convert_from_utf16(const std::wstring& wstr);\n    std::wstring convert_to_utf16(const std::string& str);\n    #endif\n\n  }\n}\n\n#endif\n\n// Path: src/libsass/src/plugins.hpp\n#ifndef SASS_PLUGINS_H\n#define SASS_PLUGINS_H\n\n#include <string>\n#include <vector>\n#include \"utf8_string.hpp\"\n#include \"sass/functions.h\"\n\n#ifdef _WIN32\n\n  #define LOAD_LIB(var, path) HMODULE var = LoadLibraryW(UTF_8::convert_to_utf16(path).c_str())\n  #define LOAD_LIB_WCHR(var, path_wide_str) HMODULE var = LoadLibraryW(path_wide_str.c_str())\n  #define LOAD_LIB_FN(type, var, name) type var = (type) GetProcAddress(plugin, name)\n  #define CLOSE_LIB(var) FreeLibrary(var)\n\n  #ifndef dlerror\n  #define dlerror() 0\n  #endif\n\n#else\n\n  #define LOAD_LIB(var, path) void* var = dlopen(path.c_str(), RTLD_LAZY)\n  #define LOAD_LIB_FN(type, var, name) type var = (type) dlsym(plugin, name)\n  #define CLOSE_LIB(var) dlclose(var)\n\n#endif\n\nnamespace Sass {\n\n\n  class Plugins {\n\n    public: // c-tor\n      Plugins(void);\n      ~Plugins(void);\n\n    public: // methods\n      // load one specific plugin\n      bool load_plugin(const std::string& path);\n      // load all plugins from a directory\n      size_t load_plugins(const std::string& path);\n\n    public: // public accessors\n      const std::vector<Sass_Importer_Entry> get_headers(void) { return headers; }\n      const std::vector<Sass_Importer_Entry> get_importers(void) { return importers; }\n      const std::vector<Sass_Function_Entry> get_functions(void) { return functions; }\n\n    private: // private vars\n      std::vector<Sass_Importer_Entry> headers;\n      std::vector<Sass_Importer_Entry> importers;\n      std::vector<Sass_Function_Entry> functions;\n\n  };\n\n}\n\n#endif\n\n// Path: src/libsass/src/context.hpp\n#ifndef SASS_CONTEXT_H\n#define SASS_CONTEXT_H\n\n#include <string>\n#include <vector>\n#include <map>\n\n#define BUFFERSIZE 255\n#include \"b64/encode.h\"\n\n#include \"ast_fwd_decl.hpp\"\n#include \"kwd_arg_macros.hpp\"\n#include \"ast_fwd_decl.hpp\"\n#include \"sass_context.hpp\"\n#include \"environment.hpp\"\n#include \"source_map.hpp\"\n#include \"subset_map.hpp\"\n#include \"backtrace.hpp\"\n#include \"output.hpp\"\n#include \"plugins.hpp\"\n#include \"file.hpp\"\n\n\nstruct Sass_Function;\n\nnamespace Sass {\n\n  class Context {\n  public:\n    void import_url (Import_Ptr imp, std::string load_path, const std::string& ctx_path);\n    bool call_headers(const std::string& load_path, const char* ctx_path, ParserState& pstate, Import_Ptr imp)\n    { return call_loader(load_path, ctx_path, pstate, imp, c_headers, false); };\n    bool call_importers(const std::string& load_path, const char* ctx_path, ParserState& pstate, Import_Ptr imp)\n    { return call_loader(load_path, ctx_path, pstate, imp, c_importers, true); };\n\n  private:\n    bool call_loader(const std::string& load_path, const char* ctx_path, ParserState& pstate, Import_Ptr imp, std::vector<Sass_Importer_Entry> importers, bool only_one = true);\n\n  public:\n    const std::string CWD;\n    struct Sass_Options& c_options;\n    std::string entry_path;\n    size_t head_imports;\n    Plugins plugins;\n    Output emitter;\n\n    // generic ast node garbage container\n    // used to avoid possible circular refs\n    std::vector<AST_Node_Obj> ast_gc;\n    // resources add under our control\n    // these are guaranteed to be freed\n    std::vector<char*> strings;\n    std::vector<Resource> resources;\n    std::map<const std::string, StyleSheet> sheets;\n    Subset_Map subset_map;\n    std::vector<Sass_Import_Entry> import_stack;\n    std::vector<Sass_Callee> callee_stack;\n    std::vector<Backtrace> traces;\n\n    struct Sass_Compiler* c_compiler;\n\n    // absolute paths to includes\n    std::vector<std::string> included_files;\n    // relative includes for sourcemap\n    std::vector<std::string> srcmap_links;\n    // vectors above have same size\n\n    std::vector<std::string> plugin_paths; // relative paths to load plugins\n    std::vector<std::string> include_paths; // lookup paths for includes\n\n\n\n\n\n    void apply_custom_headers(Block_Obj root, const char* path, ParserState pstate);\n\n    std::vector<Sass_Importer_Entry> c_headers;\n    std::vector<Sass_Importer_Entry> c_importers;\n    std::vector<Sass_Function_Entry> c_functions;\n\n    void add_c_header(Sass_Importer_Entry header);\n    void add_c_importer(Sass_Importer_Entry importer);\n    void add_c_function(Sass_Function_Entry function);\n\n    const std::string indent; // String to be used for indentation\n    const std::string linefeed; // String to be used for line feeds\n    const std::string input_path; // for relative paths in src-map\n    const std::string output_path; // for relative paths to the output\n    const std::string source_map_file; // path to source map file (enables feature)\n    const std::string source_map_root; // path for sourceRoot property (pass-through)\n\n    virtual ~Context();\n    Context(struct Sass_Context&);\n    virtual Block_Obj parse() = 0;\n    virtual Block_Obj compile();\n    virtual char* render(Block_Obj root);\n    virtual char* render_srcmap();\n\n    void register_resource(const Include&, const Resource&);\n    void register_resource(const Include&, const Resource&, ParserState&);\n    std::vector<Include> find_includes(const Importer& import);\n    Include load_import(const Importer&, ParserState pstate);\n\n    Sass_Output_Style output_style() { return c_options.output_style; };\n    std::vector<std::string> get_included_files(bool skip = false, size_t headers = 0);\n\n  private:\n    void collect_plugin_paths(const char* paths_str);\n    void collect_plugin_paths(string_list* paths_array);\n    void collect_include_paths(const char* paths_str);\n    void collect_include_paths(string_list* paths_array);\n    std::string format_embedded_source_map();\n    std::string format_source_mapping_url(const std::string& out_path);\n\n\n    // void register_built_in_functions(Env* env);\n    // void register_function(Signature sig, Native_Function f, Env* env);\n    // void register_function(Signature sig, Native_Function f, size_t arity, Env* env);\n    // void register_overload_stub(std::string name, Env* env);\n\n  public:\n    const std::string& cwd() { return CWD; };\n  };\n\n  class File_Context : public Context {\n  public:\n    File_Context(struct Sass_File_Context& ctx)\n    : Context(ctx)\n    { }\n    virtual ~File_Context();\n    virtual Block_Obj parse();\n  };\n\n  class Data_Context : public Context {\n  public:\n    char* source_c_str;\n    char* srcmap_c_str;\n    Data_Context(struct Sass_Data_Context& ctx)\n    : Context(ctx)\n    {\n      source_c_str       = ctx.source_string;\n      srcmap_c_str       = ctx.srcmap_string;\n      ctx.source_string = 0; // passed away\n      ctx.srcmap_string = 0; // passed away\n    }\n    virtual ~Data_Context();\n    virtual Block_Obj parse();\n  };\n\n}\n\n#endif\n\n// Path: src/libsass/src/constants.hpp\n#ifndef SASS_CONSTANTS_H\n#define SASS_CONSTANTS_H\n\nnamespace Sass {\n  namespace Constants {\n\n    // The maximum call stack that can be created\n    extern const unsigned long MaxCallStack;\n\n    // https://developer.mozilla.org/en-US/docs/Web/CSS/Specificity\n    // The following list of selectors is by increasing specificity:\n    extern const unsigned long Specificity_Star;\n    extern const unsigned long Specificity_Universal;\n    extern const unsigned long Specificity_Element;\n    extern const unsigned long Specificity_Base;\n    extern const unsigned long Specificity_Class;\n    extern const unsigned long Specificity_Attr;\n    extern const unsigned long Specificity_Pseudo;\n    extern const unsigned long Specificity_ID;\n\n    // sass keywords\n    extern const char at_root_kwd[];\n    extern const char import_kwd[];\n    extern const char mixin_kwd[];\n    extern const char function_kwd[];\n    extern const char return_kwd[];\n    extern const char include_kwd[];\n    extern const char content_kwd[];\n    extern const char extend_kwd[];\n    extern const char if_kwd[];\n    extern const char else_kwd[];\n    extern const char if_after_else_kwd[];\n    extern const char for_kwd[];\n    extern const char from_kwd[];\n    extern const char to_kwd[];\n    extern const char through_kwd[];\n    extern const char each_kwd[];\n    extern const char in_kwd[];\n    extern const char while_kwd[];\n    extern const char warn_kwd[];\n    extern const char error_kwd[];\n    extern const char debug_kwd[];\n    extern const char default_kwd[];\n    extern const char global_kwd[];\n    extern const char null_kwd[];\n    extern const char optional_kwd[];\n    extern const char with_kwd[];\n    extern const char without_kwd[];\n    extern const char all_kwd[];\n    extern const char rule_kwd[];\n\n    // css standard units\n    extern const char em_kwd[];\n    extern const char ex_kwd[];\n    extern const char px_kwd[];\n    extern const char cm_kwd[];\n    extern const char mm_kwd[];\n    extern const char pt_kwd[];\n    extern const char pc_kwd[];\n    extern const char deg_kwd[];\n    extern const char rad_kwd[];\n    extern const char grad_kwd[];\n    extern const char turn_kwd[];\n    extern const char ms_kwd[];\n    extern const char s_kwd[];\n    extern const char Hz_kwd[];\n    extern const char kHz_kwd[];\n\n    // vendor prefixes\n    extern const char vendor_opera_kwd[];\n    extern const char vendor_webkit_kwd[];\n    extern const char vendor_mozilla_kwd[];\n    extern const char vendor_ms_kwd[];\n    extern const char vendor_khtml_kwd[];\n\n    // css functions and keywords\n    extern const char charset_kwd[];\n    extern const char media_kwd[];\n    extern const char supports_kwd[];\n    extern const char keyframes_kwd[];\n    extern const char only_kwd[];\n    extern const char rgb_fn_kwd[];\n    extern const char url_fn_kwd[];\n    extern const char url_kwd[];\n    // extern const char url_prefix_fn_kwd[];\n    extern const char important_kwd[];\n    extern const char pseudo_not_fn_kwd[];\n    extern const char even_kwd[];\n    extern const char odd_kwd[];\n    extern const char progid_kwd[];\n    extern const char expression_kwd[];\n    extern const char calc_fn_kwd[];\n\n    // char classes for \"regular expressions\"\n    extern const char almost_any_value_class[];\n\n    // css selector keywords\n    extern const char sel_deep_kwd[];\n\n    // css attribute-matching operators\n    extern const char tilde_equal[];\n    extern const char pipe_equal[];\n    extern const char caret_equal[];\n    extern const char dollar_equal[];\n    extern const char star_equal[];\n\n    // relational & logical operators and constants\n    extern const char and_kwd[];\n    extern const char or_kwd[];\n    extern const char not_kwd[];\n    extern const char gt[];\n    extern const char gte[];\n    extern const char lt[];\n    extern const char lte[];\n    extern const char eq[];\n    extern const char neq[];\n    extern const char true_kwd[];\n    extern const char false_kwd[];\n\n    // miscellaneous punctuation and delimiters\n    extern const char percent_str[];\n    extern const char empty_str[];\n    extern const char slash_slash[];\n    extern const char slash_star[];\n    extern const char star_slash[];\n    extern const char hash_lbrace[];\n    extern const char rbrace[];\n    extern const char rparen[];\n    extern const char sign_chars[];\n    extern const char op_chars[];\n    extern const char hyphen[];\n    extern const char ellipsis[];\n    // extern const char url_space_chars[];\n\n    // type names\n    extern const char numeric_name[];\n    extern const char number_name[];\n    extern const char percentage_name[];\n    extern const char dimension_name[];\n    extern const char string_name[];\n    extern const char bool_name[];\n    extern const char color_name[];\n    extern const char list_name[];\n    extern const char map_name[];\n    extern const char arglist_name[];\n\n    // constants for uri parsing (RFC 3986 Appendix A.)\n    extern const char uri_chars[];\n    extern const char real_uri_chars[];\n\n    // some specific constant character classes\n    // they must be static to be useable by lexer\n    extern const char static_ops[];\n    extern const char selector_list_delims[];\n    extern const char complex_selector_delims[];\n    extern const char selector_combinator_ops[];\n    extern const char attribute_compare_modifiers[];\n    extern const char selector_lookahead_ops[];\n\n    // byte order marks\n    // (taken from http://en.wikipedia.org/wiki/Byte_order_mark)\n    extern const unsigned char utf_8_bom[];\n    extern const unsigned char utf_16_bom_be[];\n    extern const unsigned char utf_16_bom_le[];\n    extern const unsigned char utf_32_bom_be[];\n    extern const unsigned char utf_32_bom_le[];\n    extern const unsigned char utf_7_bom_1[];\n    extern const unsigned char utf_7_bom_2[];\n    extern const unsigned char utf_7_bom_3[];\n    extern const unsigned char utf_7_bom_4[];\n    extern const unsigned char utf_7_bom_5[];\n    extern const unsigned char utf_1_bom[];\n    extern const unsigned char utf_ebcdic_bom[];\n    extern const unsigned char scsu_bom[];\n    extern const unsigned char bocu_1_bom[];\n    extern const unsigned char gb_18030_bom[];\n\n  }\n}\n\n#endif\n\n// Path: src/libsass/src/error_handling.hpp\n#ifndef SASS_ERROR_HANDLING_H\n#define SASS_ERROR_HANDLING_H\n\n#include <string>\n#include <sstream>\n#include <stdexcept>\n#include \"position.hpp\"\n#include \"backtrace.hpp\"\n#include \"ast_fwd_decl.hpp\"\n#include \"sass/functions.h\"\n\nnamespace Sass {\n\n  struct Backtrace;\n\n  namespace Exception {\n\n    const std::string def_msg = \"Invalid sass detected\";\n    const std::string def_op_msg = \"Undefined operation\";\n    const std::string def_op_null_msg = \"Invalid null operation\";\n    const std::string def_nesting_limit = \"Code too deeply neested\";\n\n    class Base : public std::runtime_error {\n      protected:\n        std::string msg;\n        std::string prefix;\n      public:\n        ParserState pstate;\n        Backtraces traces;\n      public:\n        Base(ParserState pstate, std::string msg, Backtraces traces);\n        virtual const char* errtype() const { return prefix.c_str(); }\n        virtual const char* what() const throw() { return msg.c_str(); }\n        virtual ~Base() throw() {};\n    };\n\n    class InvalidSass : public Base {\n      public:\n        InvalidSass(ParserState pstate, Backtraces traces, std::string msg);\n        virtual ~InvalidSass() throw() {};\n    };\n\n    class InvalidParent : public Base {\n      protected:\n        Selector_Ptr parent;\n        Selector_Ptr selector;\n      public:\n        InvalidParent(Selector_Ptr parent, Backtraces traces, Selector_Ptr selector);\n        virtual ~InvalidParent() throw() {};\n    };\n\n    class MissingArgument : public Base {\n      protected:\n        std::string fn;\n        std::string arg;\n        std::string fntype;\n      public:\n        MissingArgument(ParserState pstate, Backtraces traces, std::string fn, std::string arg, std::string fntype);\n        virtual ~MissingArgument() throw() {};\n    };\n\n    class InvalidArgumentType : public Base {\n      protected:\n        std::string fn;\n        std::string arg;\n        std::string type;\n        const Value_Ptr value;\n      public:\n        InvalidArgumentType(ParserState pstate, Backtraces traces, std::string fn, std::string arg, std::string type, const Value_Ptr value = 0);\n        virtual ~InvalidArgumentType() throw() {};\n    };\n\n    class InvalidVarKwdType : public Base {\n      protected:\n        std::string name;\n        const Argument_Ptr arg;\n      public:\n        InvalidVarKwdType(ParserState pstate, Backtraces traces, std::string name, const Argument_Ptr arg = 0);\n        virtual ~InvalidVarKwdType() throw() {};\n    };\n\n    class InvalidSyntax : public Base {\n      public:\n        InvalidSyntax(ParserState pstate, Backtraces traces, std::string msg);\n        virtual ~InvalidSyntax() throw() {};\n    };\n\n    class NestingLimitError : public Base {\n      public:\n        NestingLimitError(ParserState pstate, Backtraces traces, std::string msg = def_nesting_limit);\n        virtual ~NestingLimitError() throw() {};\n    };\n\n    class DuplicateKeyError : public Base {\n      protected:\n        const Map& dup;\n        const Expression& org;\n      public:\n        DuplicateKeyError(Backtraces traces, const Map& dup, const Expression& org);\n        virtual const char* errtype() const { return \"Error\"; }\n        virtual ~DuplicateKeyError() throw() {};\n    };\n\n    class TypeMismatch : public Base {\n      protected:\n        const Expression& var;\n        const std::string type;\n      public:\n        TypeMismatch(Backtraces traces, const Expression& var, const std::string type);\n        virtual const char* errtype() const { return \"Error\"; }\n        virtual ~TypeMismatch() throw() {};\n    };\n\n    class InvalidValue : public Base {\n      protected:\n        const Expression& val;\n      public:\n        InvalidValue(Backtraces traces, const Expression& val);\n        virtual const char* errtype() const { return \"Error\"; }\n        virtual ~InvalidValue() throw() {};\n    };\n\n    class StackError : public Base {\n      protected:\n        const AST_Node& node;\n      public:\n        StackError(Backtraces traces, const AST_Node& node);\n        virtual const char* errtype() const { return \"SystemStackError\"; }\n        virtual ~StackError() throw() {};\n    };\n\n    /* common virtual base class (has no pstate or trace) */\n    class OperationError : public std::runtime_error {\n      protected:\n        std::string msg;\n      public:\n        OperationError(std::string msg = def_op_msg)\n        : std::runtime_error(msg), msg(msg)\n        {};\n      public:\n        virtual const char* errtype() const { return \"Error\"; }\n        virtual const char* what() const throw() { return msg.c_str(); }\n        virtual ~OperationError() throw() {};\n    };\n\n    class ZeroDivisionError : public OperationError {\n      protected:\n        const Expression& lhs;\n        const Expression& rhs;\n      public:\n        ZeroDivisionError(const Expression& lhs, const Expression& rhs);\n        virtual const char* errtype() const { return \"ZeroDivisionError\"; }\n        virtual ~ZeroDivisionError() throw() {};\n    };\n\n    class IncompatibleUnits : public OperationError {\n      protected:\n        // const Sass::UnitType lhs;\n        // const Sass::UnitType rhs;\n      public:\n        IncompatibleUnits(const Units& lhs, const Units& rhs);\n        IncompatibleUnits(const UnitType lhs, const UnitType rhs);\n        virtual ~IncompatibleUnits() throw() {};\n    };\n\n    class UndefinedOperation : public OperationError {\n      protected:\n        Expression_Ptr_Const lhs;\n        Expression_Ptr_Const rhs;\n        const Sass_OP op;\n      public:\n        UndefinedOperation(Expression_Ptr_Const lhs, Expression_Ptr_Const rhs, enum Sass_OP op);\n        // virtual const char* errtype() const { return \"Error\"; }\n        virtual ~UndefinedOperation() throw() {};\n    };\n\n    class InvalidNullOperation : public UndefinedOperation {\n      public:\n        InvalidNullOperation(Expression_Ptr_Const lhs, Expression_Ptr_Const rhs, enum Sass_OP op);\n        virtual ~InvalidNullOperation() throw() {};\n    };\n\n    class AlphaChannelsNotEqual : public OperationError {\n      protected:\n        Expression_Ptr_Const lhs;\n        Expression_Ptr_Const rhs;\n        const Sass_OP op;\n      public:\n        AlphaChannelsNotEqual(Expression_Ptr_Const lhs, Expression_Ptr_Const rhs, enum Sass_OP op);\n        // virtual const char* errtype() const { return \"Error\"; }\n        virtual ~AlphaChannelsNotEqual() throw() {};\n    };\n\n    class SassValueError : public Base {\n      public:\n        SassValueError(Backtraces traces, ParserState pstate, OperationError& err);\n        virtual ~SassValueError() throw() {};\n    };\n\n  }\n\n  void warn(std::string msg, ParserState pstate);\n  void warn(std::string msg, ParserState pstate, Backtrace* bt);\n  void warning(std::string msg, ParserState pstate);\n\n  void deprecated_function(std::string msg, ParserState pstate);\n  void deprecated(std::string msg, std::string msg2, bool with_column, ParserState pstate);\n  void deprecated_bind(std::string msg, ParserState pstate);\n  // void deprecated(std::string msg, ParserState pstate, Backtrace* bt);\n\n  void coreError(std::string msg, ParserState pstate);\n  void error(std::string msg, ParserState pstate, Backtraces& traces);\n\n}\n\n#endif\n\n// Path: src/libsass/include/sass/version.h\n#ifndef SASS_VERSION_H\n#define SASS_VERSION_H\n\n#ifndef LIBSASS_VERSION\n#define LIBSASS_VERSION \"[NA]\"\n#endif\n\n#ifndef LIBSASS_LANGUAGE_VERSION\n#define LIBSASS_LANGUAGE_VERSION \"3.5\"\n#endif\n\n#endif\n\n// Path: src/libsass/include/sass2scss.h\n/**\n * sass2scss\n * Licensed under the MIT License\n * Copyright (c) Marcel Greter\n */\n\n#ifndef SASS2SCSS_H\n#define SASS2SCSS_H\n\n#ifdef _WIN32\n\n  /* You should define ADD_EXPORTS *only* when building the DLL. */\n  #ifdef ADD_EXPORTS\n    #define ADDAPI __declspec(dllexport)\n\t#define ADDCALL __cdecl\n  #else\n    #define ADDAPI\n\t#define ADDCALL\n  #endif\n\n#else /* _WIN32 not defined. */\n\n  /* Define with no value on non-Windows OSes. */\n  #define ADDAPI\n  #define ADDCALL\n\n#endif\n\n#ifdef __cplusplus\n\n#include <stack>\n#include <string>\n#include <cstring>\n#include <sstream>\n#include <iostream>\n\n#ifndef SASS2SCSS_VERSION\n// Hardcode once the file is copied from\n// https://github.com/mgreter/sass2scss\n#define SASS2SCSS_VERSION \"1.1.1\"\n#endif\n\n// add namespace for c++\nnamespace Sass\n{\n\n\t// pretty print options\n\tconst int SASS2SCSS_PRETTIFY_0 = 0;\n\tconst int SASS2SCSS_PRETTIFY_1 = 1;\n\tconst int SASS2SCSS_PRETTIFY_2 = 2;\n\tconst int SASS2SCSS_PRETTIFY_3 = 3;\n\n\t// remove one-line comment\n\tconst int SASS2SCSS_KEEP_COMMENT    =  32;\n\t// remove multi-line comments\n\tconst int SASS2SCSS_STRIP_COMMENT   =  64;\n\t// convert one-line to multi-line\n\tconst int SASS2SCSS_CONVERT_COMMENT = 128;\n\n\t// String for finding something interesting\n\tconst std::string SASS2SCSS_FIND_WHITESPACE = \" \\t\\n\\v\\f\\r\";\n\n\t// converter struct\n\t// holding all states\n\tstruct converter\n\t{\n\t\t// bit options\n\t\tint options;\n\t\t// is selector\n\t\tbool selector;\n\t\t// concat lists\n\t\tbool comma;\n\t\t// has property\n\t\tbool property;\n\t\t// has semicolon\n\t\tbool semicolon;\n\t\t// comment context\n\t\tstd::string comment;\n\t\t// flag end of file\n\t\tbool end_of_file;\n\t\t// whitespace buffer\n\t\tstd::string whitespace;\n\t\t// context/block stack\n\t\tstd::stack<std::string> indents;\n\t};\n\n\t// function only available in c++ code\n\tchar* sass2scss (const std::string& sass, const int options);\n\n}\n// EO namespace\n\n// declare for c\nextern \"C\" {\n#endif\n\n\t// prettyfy print options\n\t#define SASS2SCSS_PRETTIFY_0   0\n\t#define SASS2SCSS_PRETTIFY_1   1\n\t#define SASS2SCSS_PRETTIFY_2   2\n\t#define SASS2SCSS_PRETTIFY_3   3\n\n\t// keep one-line comments\n\t#define SASS2SCSS_KEEP_COMMENT     32\n\t// remove multi-line comments\n\t#define SASS2SCSS_STRIP_COMMENT    64\n\t// convert one-line to multi-line\n\t#define SASS2SCSS_CONVERT_COMMENT  128\n\n\t// available to c and c++ code\n\tADDAPI char* ADDCALL sass2scss (const char* sass, const int options);\n\n\t// Get compiled sass2scss version\n\tADDAPI const char* ADDCALL sass2scss_version(void);\n\n#ifdef __cplusplus\n} // __cplusplus defined.\n#endif\n\n#endif\n// Path: src/libsass/include/sass.h\n#ifndef SASS_H\n#define SASS_H\n\n// #define DEBUG 1\n\n// include API headers\n#include <sass/base.h>\n#include <sass/version.h>\n#include <sass/values.h>\n#include <sass/functions.h>\n#include <sass/context.h>\n#include <sass2scss.h>\n\n#endif\n\n\n// Path: src/libsass/src/ast.hpp\n#ifndef SASS_AST_H\n#define SASS_AST_H\n\n#include \"sass.hpp\"\n#include <set>\n#include <deque>\n#include <vector>\n#include <string>\n#include <sstream>\n#include <iostream>\n#include <typeinfo>\n#include <algorithm>\n#include \"sass/base.h\"\n#include \"ast_fwd_decl.hpp\"\n\n#ifdef DEBUG_SHARED_PTR\n\n#define ATTACH_VIRTUAL_AST_OPERATIONS(klass) \\\n  virtual klass##_Ptr copy(std::string, size_t) const = 0; \\\n  virtual klass##_Ptr clone(std::string, size_t) const = 0; \\\n\n#define ATTACH_AST_OPERATIONS(klass) \\\n  virtual klass##_Ptr copy(std::string, size_t) const; \\\n  virtual klass##_Ptr clone(std::string, size_t) const; \\\n\n#else\n\n#define ATTACH_VIRTUAL_AST_OPERATIONS(klass) \\\n  virtual klass##_Ptr copy() const = 0; \\\n  virtual klass##_Ptr clone() const = 0; \\\n\n#define ATTACH_AST_OPERATIONS(klass) \\\n  virtual klass##_Ptr copy() const; \\\n  virtual klass##_Ptr clone() const; \\\n\n#endif\n\n#ifdef __clang__\n\n/*\n * There are some overloads used here that trigger the clang overload\n * hiding warning. Specifically:\n *\n * Type type() which hides string type() from Expression\n *\n */\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Woverloaded-virtual\"\n\n#endif\n\n#include \"util.hpp\"\n#include \"units.hpp\"\n#include \"context.hpp\"\n#include \"position.hpp\"\n#include \"constants.hpp\"\n#include \"operation.hpp\"\n#include \"position.hpp\"\n#include \"inspect.hpp\"\n#include \"source_map.hpp\"\n#include \"environment.hpp\"\n#include \"error_handling.hpp\"\n#include \"ast_def_macros.hpp\"\n#include \"ast_fwd_decl.hpp\"\n#include \"source_map.hpp\"\n\n#include \"sass.h\"\n\nnamespace Sass {\n\n  // easier to search with name\n  const bool DELAYED = true;\n\n  // ToDo: should this really be hardcoded\n  // Note: most methods follow precision option\n  const double NUMBER_EPSILON = 0.00000000000001;\n\n  // macro to test if numbers are equal within a small error margin\n  #define NEAR_EQUAL(lhs, rhs) std::fabs(lhs - rhs) < NUMBER_EPSILON\n\n  // ToDo: where does this fit best?\n  // We don't share this with C-API?\n  class Operand {\n    public:\n      \nOperand(Sass_OP operand, bool ws_before = false, bool ws_after = false)\n      : operand(operand), ws_before(ws_before), ws_after(ws_after)\n      { }\n    public:\n      enum Sass_OP operand;\n      bool ws_before;\n      bool ws_after;\n  };\n\n  //////////////////////////////////////////////////////////\n  // `hash_combine` comes from boost (functional/hash):\n  // http://www.boost.org/doc/libs/1_35_0/doc/html/hash/combine.html\n  // Boost Software License - Version 1.0\n  // http://www.boost.org/users/license.html\n  template <typename T>\n  void hash_combine (std::size_t& seed, const T& val)\n  {\n    seed ^= std::hash<T>()(val) + 0x9e3779b9\n             + (seed<<6) + (seed>>2);\n  }\n  //////////////////////////////////////////////////////////\n\n  //////////////////////////////////////////////////////////\n  // Abstract base class for all abstract syntax tree nodes.\n  //////////////////////////////////////////////////////////\n  class AST_Node : public SharedObj {\n    ADD_PROPERTY(ParserState, pstate)\n  public:\n    AST_Node(ParserState pstate)\n    : pstate_(pstate)\n    { }\n    AST_Node(const AST_Node* ptr)\n    : pstate_(ptr->pstate_)\n    { }\n\n    // AST_Node(AST_Node& ptr) = delete;\n\n    virtual ~AST_Node() = 0;\n    virtual size_t hash() { return 0; }\n    ATTACH_VIRTUAL_AST_OPERATIONS(AST_Node);\n    virtual std::string inspect() const { return to_string({ INSPECT, 5 }); }\n    virtual std::string to_sass() const { return to_string({ TO_SASS, 5 }); }\n    virtual const std::string to_string(Sass_Inspect_Options opt) const;\n    virtual const std::string to_string() const;\n    virtual void cloneChildren() {};\n    // generic find function (not fully implemented yet)\n    // ToDo: add specific implementions to all children\n    virtual bool find ( bool (*f)(AST_Node_Obj) ) { return f(this); };\n  public:\n    void update_pstate(const ParserState& pstate);\n  public:\n    Offset off() { return pstate(); }\n    Position pos() { return pstate(); }\n    ATTACH_OPERATIONS()\n  };\n  inline AST_Node::~AST_Node() { }\n\n  //////////////////////////////////////////////////////////////////////\n  // define cast template now (need complete type)\n  //////////////////////////////////////////////////////////////////////\n\n  template<class T>\n  T* Cast(AST_Node* ptr) {\n    return ptr && typeid(T) == typeid(*ptr) ?\n           static_cast<T*>(ptr) : NULL;\n  };\n\n  template<class T>\n  const T* Cast(const AST_Node* ptr) {\n    return ptr && typeid(T) == typeid(*ptr) ?\n           static_cast<const T*>(ptr) : NULL;\n  };\n\n  //////////////////////////////////////////////////////////////////////\n  // Abstract base class for expressions. This side of the AST hierarchy\n  // represents elements in value contexts, which exist primarily to be\n  // evaluated and returned.\n  //////////////////////////////////////////////////////////////////////\n  class Expression : public AST_Node {\n  public:\n    enum Concrete_Type {\n      NONE,\n      BOOLEAN,\n      NUMBER,\n      COLOR,\n      STRING,\n      LIST,\n      MAP,\n      SELECTOR,\n      NULL_VAL,\n      FUNCTION_VAL,\n      C_WARNING,\n      C_ERROR,\n      FUNCTION,\n      VARIABLE,\n      NUM_TYPES\n    };\n    enum Simple_Type {\n      SIMPLE,\n      ATTR_SEL,\n      PSEUDO_SEL,\n      WRAPPED_SEL,\n    };\n  private:\n    // expressions in some contexts shouldn't be evaluated\n    ADD_PROPERTY(bool, is_delayed)\n    ADD_PROPERTY(bool, is_expanded)\n    ADD_PROPERTY(bool, is_interpolant)\n    ADD_PROPERTY(Concrete_Type, concrete_type)\n  public:\n    Expression(ParserState pstate,\n               bool d = false, bool e = false, bool i = false, Concrete_Type ct = NONE)\n    : AST_Node(pstate),\n      is_delayed_(d),\n      is_expanded_(e),\n      is_interpolant_(i),\n      concrete_type_(ct)\n    { }\n    Expression(const Expression* ptr)\n    : AST_Node(ptr),\n      is_delayed_(ptr->is_delayed_),\n      is_expanded_(ptr->is_expanded_),\n      is_interpolant_(ptr->is_interpolant_),\n      concrete_type_(ptr->concrete_type_)\n    { }\n    virtual operator bool() { return true; }\n    virtual ~Expression() { }\n    virtual std::string type() const { return \"\"; /* TODO: raise an error? */ }\n    virtual bool is_invisible() const { return false; }\n    static std::string type_name() { return \"\"; }\n    virtual bool is_false() { return false; }\n    // virtual bool is_true() { return !is_false(); }\n    virtual bool operator== (const Expression& rhs) const { return false; }\n    virtual bool eq(const Expression& rhs) const { return *this == rhs; };\n    virtual void set_delayed(bool delayed) { is_delayed(delayed); }\n    virtual bool has_interpolant() const { return is_interpolant(); }\n    virtual bool is_left_interpolant() const { return is_interpolant(); }\n    virtual bool is_right_interpolant() const { return is_interpolant(); }\n    virtual std::string inspect() const { return to_string({ INSPECT, 5 }); }\n    virtual std::string to_sass() const { return to_string({ TO_SASS, 5 }); }\n    ATTACH_VIRTUAL_AST_OPERATIONS(Expression);\n    virtual size_t hash() { return 0; }\n  };\n\n  //////////////////////////////////////////////////////////////////////\n  // Still just an expression, but with a to_string method\n  //////////////////////////////////////////////////////////////////////\n  class PreValue : public Expression {\n  public:\n    PreValue(ParserState pstate,\n               bool d = false, bool e = false, bool i = false, Concrete_Type ct = NONE)\n    : Expression(pstate, d, e, i, ct)\n    { }\n    PreValue(const PreValue* ptr)\n    : Expression(ptr)\n    { }\n    ATTACH_VIRTUAL_AST_OPERATIONS(PreValue);\n    virtual ~PreValue() { }\n  };\n\n  //////////////////////////////////////////////////////////////////////\n  // base class for values that support operations\n  //////////////////////////////////////////////////////////////////////\n  class Value : public Expression {\n  public:\n    Value(ParserState pstate,\n          bool d = false, bool e = false, bool i = false, Concrete_Type ct = NONE)\n    : Expression(pstate, d, e, i, ct)\n    { }\n    Value(const Value* ptr)\n    : Expression(ptr)\n    { }\n    ATTACH_VIRTUAL_AST_OPERATIONS(Value);\n    virtual bool operator== (const Expression& rhs) const = 0;\n  };\n}\n\n/////////////////////////////////////////////////////////////////////////////////////\n// Hash method specializations for std::unordered_map to work with Sass::Expression\n/////////////////////////////////////////////////////////////////////////////////////\n\nnamespace std {\n  template<>\n  struct hash<Sass::Expression_Obj>\n  {\n    size_t operator()(Sass::Expression_Obj s) const\n    {\n      return s->hash();\n    }\n  };\n  template<>\n  struct equal_to<Sass::Expression_Obj>\n  {\n    bool operator()( Sass::Expression_Obj lhs,  Sass::Expression_Obj rhs) const\n    {\n      return lhs->hash() == rhs->hash();\n    }\n  };\n}\n\nnamespace Sass {\n\n  /////////////////////////////////////////////////////////////////////////////\n  // Mixin class for AST nodes that should behave like vectors. Uses the\n  // \"Template Method\" design pattern to allow subclasses to adjust their flags\n  // when certain objects are pushed.\n  /////////////////////////////////////////////////////////////////////////////\n  template <typename T>\n  class Vectorized {\n    std::vector<T> elements_;\n  protected:\n    size_t hash_;\n    void reset_hash() { hash_ = 0; }\n    virtual void adjust_after_pushing(T element) { }\n  public:\n    Vectorized(size_t s = 0) : elements_(std::vector<T>()), hash_(0)\n    { elements_.reserve(s); }\n    virtual ~Vectorized() = 0;\n    size_t length() const   { return elements_.size(); }\n    bool empty() const      { return elements_.empty(); }\n    void clear()            { return elements_.clear(); }\n    T last() const          { return elements_.back(); }\n    T first() const         { return elements_.front(); }\n    T& operator[](size_t i) { return elements_[i]; }\n    virtual const T& at(size_t i) const { return elements_.at(i); }\n    virtual T& at(size_t i) { return elements_.at(i); }\n    const T& operator[](size_t i) const { return elements_[i]; }\n    virtual void append(T element)\n    {\n      if (element) {\n        reset_hash();\n        elements_.push_back(element);\n        adjust_after_pushing(element);\n      }\n    }\n    virtual void concat(Vectorized* v)\n    {\n      for (size_t i = 0, L = v->length(); i < L; ++i) this->append((*v)[i]);\n    }\n    Vectorized& unshift(T element)\n    {\n      elements_.insert(elements_.begin(), element);\n      return *this;\n    }\n    std::vector<T>& elements() { return elements_; }\n    const std::vector<T>& elements() const { return elements_; }\n    std::vector<T>& elements(std::vector<T>& e) { elements_ = e; return elements_; }\n\n    virtual size_t hash()\n    {\n      if (hash_ == 0) {\n        for (T& el : elements_) {\n          hash_combine(hash_, el->hash());\n        }\n      }\n      return hash_;\n    }\n\n    typename std::vector<T>::iterator end() { return elements_.end(); }\n    typename std::vector<T>::iterator begin() { return elements_.begin(); }\n    typename std::vector<T>::const_iterator end() const { return elements_.end(); }\n    typename std::vector<T>::const_iterator begin() const { return elements_.begin(); }\n    typename std::vector<T>::iterator erase(typename std::vector<T>::iterator el) { return elements_.erase(el); }\n    typename std::vector<T>::const_iterator erase(typename std::vector<T>::const_iterator el) { return elements_.erase(el); }\n\n  };\n  template <typename T>\n  inline Vectorized<T>::~Vectorized() { }\n\n  /////////////////////////////////////////////////////////////////////////////\n  // Mixin class for AST nodes that should behave like a hash table. Uses an\n  // extra <std::vector> internally to maintain insertion order for interation.\n  /////////////////////////////////////////////////////////////////////////////\n  class Hashed {\n  private:\n    ExpressionMap elements_;\n    std::vector<Expression_Obj> list_;\n  protected:\n    size_t hash_;\n    Expression_Obj duplicate_key_;\n    void reset_hash() { hash_ = 0; }\n    void reset_duplicate_key() { duplicate_key_ = 0; }\n    virtual void adjust_after_pushing(std::pair<Expression_Obj, Expression_Obj> p) { }\n  public:\n    Hashed(size_t s = 0)\n    : elements_(ExpressionMap(s)),\n      list_(std::vector<Expression_Obj>()),\n      hash_(0), duplicate_key_(NULL)\n    { elements_.reserve(s); list_.reserve(s); }\n    virtual ~Hashed();\n    size_t length() const                  { return list_.size(); }\n    bool empty() const                     { return list_.empty(); }\n    bool has(Expression_Obj k) const          { return elements_.count(k) == 1; }\n    Expression_Obj at(Expression_Obj k) const;\n    bool has_duplicate_key() const         { return duplicate_key_ != 0; }\n    Expression_Obj get_duplicate_key() const  { return duplicate_key_; }\n    const ExpressionMap elements() { return elements_; }\n    Hashed& operator<<(std::pair<Expression_Obj, Expression_Obj> p)\n    {\n      reset_hash();\n\n      if (!has(p.first)) list_.push_back(p.first);\n      else if (!duplicate_key_) duplicate_key_ = p.first;\n\n      elements_[p.first] = p.second;\n\n      adjust_after_pushing(p);\n      return *this;\n    }\n    Hashed& operator+=(Hashed* h)\n    {\n      if (length() == 0) {\n        this->elements_ = h->elements_;\n        this->list_ = h->list_;\n        return *this;\n      }\n\n      for (auto key : h->keys()) {\n        *this << std::make_pair(key, h->at(key));\n      }\n\n      reset_duplicate_key();\n      return *this;\n    }\n    const ExpressionMap& pairs() const { return elements_; }\n    const std::vector<Expression_Obj>& keys() const { return list_; }\n\n//    std::unordered_map<Expression_Obj, Expression_Obj>::iterator end() { return elements_.end(); }\n//    std::unordered_map<Expression_Obj, Expression_Obj>::iterator begin() { return elements_.begin(); }\n//    std::unordered_map<Expression_Obj, Expression_Obj>::const_iterator end() const { return elements_.end(); }\n//    std::unordered_map<Expression_Obj, Expression_Obj>::const_iterator begin() const { return elements_.begin(); }\n\n  };\n  inline Hashed::~Hashed() { }\n\n\n  /////////////////////////////////////////////////////////////////////////\n  // Abstract base class for statements. This side of the AST hierarchy\n  // represents elements in expansion contexts, which exist primarily to be\n  // rewritten and macro-expanded.\n  /////////////////////////////////////////////////////////////////////////\n  class Statement : public AST_Node {\n  public:\n    enum Statement_Type {\n      NONE,\n      RULESET,\n      MEDIA,\n      DIRECTIVE,\n      SUPPORTS,\n      ATROOT,\n      BUBBLE,\n      CONTENT,\n      KEYFRAMERULE,\n      DECLARATION,\n      ASSIGNMENT,\n      IMPORT_STUB,\n      IMPORT,\n      COMMENT,\n      WARNING,\n      RETURN,\n      EXTEND,\n      ERROR,\n      DEBUGSTMT,\n      WHILE,\n      EACH,\n      FOR,\n      IF\n    };\n  private:\n    ADD_PROPERTY(Statement_Type, statement_type)\n    ADD_PROPERTY(size_t, tabs)\n    ADD_PROPERTY(bool, group_end)\n  public:\n    Statement(ParserState pstate, Statement_Type st = NONE, size_t t = 0)\n    : AST_Node(pstate), statement_type_(st), tabs_(t), group_end_(false)\n     { }\n    Statement(const Statement* ptr)\n    : AST_Node(ptr),\n      statement_type_(ptr->statement_type_),\n      tabs_(ptr->tabs_),\n      group_end_(ptr->group_end_)\n     { }\n    virtual ~Statement() = 0;\n    // needed for rearranging nested rulesets during CSS emission\n    virtual bool   is_invisible() const { return false; }\n    virtual bool   bubbles() { return false; }\n    virtual bool has_content()\n    {\n      return statement_type_ == CONTENT;\n    }\n  };\n...\n", "needle_token_start": 11993, "needle_token_end": 12046, "code_context_ntokens": 16081, "output": ["```cpp\nclass Operand {\n  public:\n    \nOperand(Sass_OP operand, bool ws_before = false, bool ws_after = false)\n  : operand(operand), ws_before(ws_before), ws_after(ws_after)\n  { }\n  public:\n    enum Sass_OP operand;\n    bool ws_before;\n    bool ws_after;\n};\n```"]}
{"repo": "sass/node-sass", "name": "debug_sources_set", "language": "cpp", "path": "src/libsass/src/debugger.hpp", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to assist in debugging by printing a visual representation of a set of complex selectors to the standard error output, helping developers understand the current state and structure of the data.\n2. **Input**: It takes a set of complex selectors and an optional string indentation parameter for formatting the output.\n3. **Output**: There is no return value; the function outputs directly to the standard error stream.\n4. **Procedure**: The function begins by checking if the indentation parameter is empty and, if so, prints a starting delimiter. It then iterates over each element in the set, using another function to print details of each complex selector with the provided indentation. After iterating through the set, if the indentation was initially empty, it prints an ending delimiter to the standard error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/libsass/src/output.cpp\n#include \"sass.hpp\"\n#include \"ast.hpp\"\n#include \"output.hpp\"\n\nnamespace Sass {\n\n  Output::Output(Sass_Output_Options& opt)\n  : Inspect(Emitter(opt)),\n    charset(\"\"),\n    top_nodes(0)\n  {}\n\n  Output::~Output() { }\n\n  void Output::fallback_impl(AST_Node_Ptr n)\n  {\n    return n->perform(this);\n  }\n\n  void Output::operator()(Number_Ptr n)\n  {\n    // check for a valid unit here\n    // includes result for reporting\n    if (!n->is_valid_css_unit()) {\n      // should be handle in check_expression\n      throw Exception::InvalidValue({}, *n);\n    }\n    // use values to_string facility\n    std::string res = n->to_string(opt);\n    // output the final token\n    append_token(res, n);\n  }\n\n  void Output::operator()(Import_Ptr imp)\n  {\n    top_nodes.push_back(imp);\n  }\n\n  void Output::operator()(Map_Ptr m)\n  {\n    // should be handle in check_expression\n    throw Exception::InvalidValue({}, *m);\n  }\n\n  OutputBuffer Output::get_buffer(void)\n  {\n\n    Emitter emitter(opt);\n    Inspect inspect(emitter);\n\n    size_t size_nodes = top_nodes.size();\n    for (size_t i = 0; i < size_nodes; i++) {\n      top_nodes[i]->perform(&inspect);\n      inspect.append_mandatory_linefeed();\n    }\n\n    // flush scheduled outputs\n    // maybe omit semicolon if possible\n    inspect.finalize(wbuf.buffer.size() == 0);\n    // prepend buffer on top\n    prepend_output(inspect.output());\n    // make sure we end with a linefeed\n    if (!ends_with(wbuf.buffer, opt.linefeed)) {\n      // if the output is not completely empty\n      if (!wbuf.buffer.empty()) append_string(opt.linefeed);\n    }\n\n    // search for unicode char\n    for(const char& chr : wbuf.buffer) {\n      // skip all ascii chars\n      // static cast to unsigned to handle `char` being signed / unsigned\n      if (static_cast<unsigned>(chr) < 128) continue;\n      // declare the charset\n      if (output_style() != COMPRESSED)\n        charset = \"@charset \\\"UTF-8\\\";\"\n                + std::string(opt.linefeed);\n      else charset = \"\\xEF\\xBB\\xBF\";\n      // abort search\n      break;\n    }\n\n    // add charset as first line, before comments and imports\n    if (!charset.empty()) prepend_string(charset);\n\n    return wbuf;\n\n  }\n\n  void Output::operator()(Comment_Ptr c)\n  {\n    std::string txt = c->text()->to_string(opt);\n    // if (indentation && txt == \"/**/\") return;\n    bool important = c->is_important();\n    if (output_style() != COMPRESSED || important) {\n      if (buffer().size() == 0) {\n        top_nodes.push_back(c);\n      } else {\n        in_comment = true;\n        append_indentation();\n        c->text()->perform(this);\n        in_comment = false;\n        if (indentation == 0) {\n          append_mandatory_linefeed();\n        } else {\n          append_optional_linefeed();\n        }\n      }\n    }\n  }\n\n  void Output::operator()(Ruleset_Ptr r)\n  {\n    Selector_Obj s     = r->selector();\n    Block_Obj    b     = r->block();\n\n    // Filter out rulesets that aren't printable (process its children though)\n    if (!Util::isPrintable(r, output_style())) {\n      for (size_t i = 0, L = b->length(); i < L; ++i) {\n        const Statement_Obj& stm = b->at(i);\n        if (Cast<Has_Block>(stm)) {\n          if (!Cast<Declaration>(stm)) {\n            stm->perform(this);\n          }\n        }\n      }\n      return;\n    }\n\n    if (output_style() == NESTED) indentation += r->tabs();\n    if (opt.source_comments) {\n      std::stringstream ss;\n      append_indentation();\n      std::string path(File::abs2rel(r->pstate().path));\n      ss << \"/* line \" << r->pstate().line + 1 << \", \" << path << \" */\";\n      append_string(ss.str());\n      append_optional_linefeed();\n    }\n    scheduled_crutch = s;\n    if (s) s->perform(this);\n    append_scope_opener(b);\n    for (size_t i = 0, L = b->length(); i < L; ++i) {\n      Statement_Obj stm = b->at(i);\n      bool bPrintExpression = true;\n      // Check print conditions\n      if (Declaration_Ptr dec = Cast<Declaration>(stm)) {\n        if (String_Constant_Ptr valConst = Cast<String_Constant>(dec->value())) {\n          std::string val(valConst->value());\n          if (String_Quoted_Ptr qstr = Cast<String_Quoted>(valConst)) {\n            if (!qstr->quote_mark() && val.empty()) {\n              bPrintExpression = false;\n            }\n          }\n        }\n        else if (List_Ptr list = Cast<List>(dec->value())) {\n          bool all_invisible = true;\n          for (size_t list_i = 0, list_L = list->length(); list_i < list_L; ++list_i) {\n            Expression_Ptr item = list->at(list_i);\n            if (!item->is_invisible()) all_invisible = false;\n          }\n          if (all_invisible && !list->is_bracketed()) bPrintExpression = false;\n        }\n      }\n      // Print if OK\n      if (bPrintExpression) {\n        stm->perform(this);\n      }\n    }\n    if (output_style() == NESTED) indentation -= r->tabs();\n    append_scope_closer(b);\n\n  }\n  void Output::operator()(Keyframe_Rule_Ptr r)\n  {\n    Block_Obj b = r->block();\n    Selector_Obj v = r->name();\n\n    if (!v.isNull()) {\n      v->perform(this);\n    }\n\n    if (!b) {\n      append_colon_separator();\n      return;\n    }\n\n    append_scope_opener();\n    for (size_t i = 0, L = b->length(); i < L; ++i) {\n      Statement_Obj stm = b->at(i);\n      stm->perform(this);\n      if (i < L - 1) append_special_linefeed();\n    }\n    append_scope_closer();\n  }\n\n  void Output::operator()(Supports_Block_Ptr f)\n  {\n    if (f->is_invisible()) return;\n\n    Supports_Condition_Obj c = f->condition();\n    Block_Obj b              = f->block();\n\n    // Filter out feature blocks that aren't printable (process its children though)\n    if (!Util::isPrintable(f, output_style())) {\n      for (size_t i = 0, L = b->length(); i < L; ++i) {\n        Statement_Obj stm = b->at(i);\n        if (Cast<Has_Block>(stm)) {\n          stm->perform(this);\n        }\n      }\n      return;\n    }\n\n    if (output_style() == NESTED) indentation += f->tabs();\n    append_indentation();\n    append_token(\"@supports\", f);\n    append_mandatory_space();\n    c->perform(this);\n    append_scope_opener();\n\n    for (size_t i = 0, L = b->length(); i < L; ++i) {\n      Statement_Obj stm = b->at(i);\n      stm->perform(this);\n      if (i < L - 1) append_special_linefeed();\n    }\n\n    if (output_style() == NESTED) indentation -= f->tabs();\n\n    append_scope_closer();\n\n  }\n\n  void Output::operator()(Media_Block_Ptr m)\n  {\n    if (m->is_invisible()) return;\n\n    Block_Obj b     = m->block();\n\n    // Filter out media blocks that aren't printable (process its children though)\n    if (!Util::isPrintable(m, output_style())) {\n      for (size_t i = 0, L = b->length(); i < L; ++i) {\n        Statement_Obj stm = b->at(i);\n        if (Cast<Has_Block>(stm)) {\n          stm->perform(this);\n        }\n      }\n      return;\n    }\n    if (output_style() == NESTED) indentation += m->tabs();\n    append_indentation();\n    append_token(\"@media\", m);\n    append_mandatory_space();\n    in_media_block = true;\n    m->media_queries()->perform(this);\n    in_media_block = false;\n    append_scope_opener();\n\n    for (size_t i = 0, L = b->length(); i < L; ++i) {\n      if (b->at(i)) {\n      Statement_Obj stm = b->at(i);\n        stm->perform(this);\n      }\n      if (i < L - 1) append_special_linefeed();\n    }\n\n    if (output_style() == NESTED) indentation -= m->tabs();\n    append_scope_closer();\n  }\n\n  void Output::operator()(Directive_Ptr a)\n  {\n    std::string      kwd   = a->keyword();\n    Selector_Obj   s     = a->selector();\n    Expression_Obj v     = a->value();\n    Block_Obj      b     = a->block();\n\n    append_indentation();\n    append_token(kwd, a);\n    if (s) {\n      append_mandatory_space();\n      in_wrapped = true;\n      s->perform(this);\n      in_wrapped = false;\n    }\n    if (v) {\n      append_mandatory_space();\n      // ruby sass bug? should use options?\n      append_token(v->to_string(/* opt */), v);\n    }\n    if (!b) {\n      append_delimiter();\n      return;\n    }\n\n    if (b->is_invisible() || b->length() == 0) {\n      append_optional_space();\n      return append_string(\"{}\");\n    }\n\n    append_scope_opener();\n\n    bool format = kwd != \"@font-face\";;\n\n    for (size_t i = 0, L = b->length(); i < L; ++i) {\n      Statement_Obj stm = b->at(i);\n      stm->perform(this);\n      if (i < L - 1 && format) append_special_linefeed();\n    }\n\n    append_scope_closer();\n  }\n\n  void Output::operator()(String_Quoted_Ptr s)\n  {\n    if (s->quote_mark()) {\n      append_token(quote(s->value(), s->quote_mark()), s);\n    } else if (!in_comment) {\n...\n// Path: src/libsass/src/sass.cpp\n#include \"sass.hpp\"\n#include <cstdlib>\n#include <cstring>\n#include <vector>\n#include <sstream>\n\n#include \"sass.h\"\n#include \"file.hpp\"\n#include \"util.hpp\"\n#include \"sass_context.hpp\"\n#include \"sass_functions.hpp\"\n\nnamespace Sass {\n\n  // helper to convert string list to vector\n  std::vector<std::string> list2vec(struct string_list* cur)\n  {\n    std::vector<std::string> list;\n    while (cur) {\n      list.push_back(cur->string);\n      cur = cur->next;\n    }\n    return list;\n  }\n\n}\n\nextern \"C\" {\n  using namespace Sass;\n\n  // Allocate libsass heap memory\n  // Don't forget string termination!\n  void* ADDCALL sass_alloc_memory(size_t size)\n  {\n    void* ptr = malloc(size);\n    if (ptr == NULL) {\n      std::cerr << \"Out of memory.\\n\";\n      exit(EXIT_FAILURE);\n    }\n    return ptr;\n  }\n\n  char* ADDCALL sass_copy_c_string(const char* str)\n  {\n    size_t len = strlen(str) + 1;\n    char* cpy = (char*) sass_alloc_memory(len);\n    std::memcpy(cpy, str, len);\n    return cpy;\n  }\n\n  // Deallocate libsass heap memory\n  void ADDCALL sass_free_memory(void* ptr)\n  {\n    if (ptr) free (ptr);\n  }\n\n  // caller must free the returned memory\n  char* ADDCALL sass_string_quote (const char *str, const char quote_mark)\n  {\n    std::string quoted = quote(str, quote_mark);\n    return sass_copy_c_string(quoted.c_str());\n  }\n\n  // caller must free the returned memory\n  char* ADDCALL sass_string_unquote (const char *str)\n  {\n    std::string unquoted = unquote(str);\n    return sass_copy_c_string(unquoted.c_str());\n  }\n\n  char* ADDCALL sass_compiler_find_include (const char* file, struct Sass_Compiler* compiler)\n  {\n    // get the last import entry to get current base directory\n    Sass_Import_Entry import = sass_compiler_get_last_import(compiler);\n    const std::vector<std::string>& incs = compiler->cpp_ctx->include_paths;\n    // create the vector with paths to lookup\n    std::vector<std::string> paths(1 + incs.size());\n    paths.push_back(File::dir_name(import->abs_path));\n    paths.insert( paths.end(), incs.begin(), incs.end() );\n    // now resolve the file path relative to lookup paths\n    std::string resolved(File::find_include(file, paths));\n    return sass_copy_c_string(resolved.c_str());\n  }\n\n  char* ADDCALL sass_compiler_find_file (const char* file, struct Sass_Compiler* compiler)\n  {\n    // get the last import entry to get current base directory\n    Sass_Import_Entry import = sass_compiler_get_last_import(compiler);\n    const std::vector<std::string>& incs = compiler->cpp_ctx->include_paths;\n    // create the vector with paths to lookup\n    std::vector<std::string> paths(1 + incs.size());\n    paths.push_back(File::dir_name(import->abs_path));\n    paths.insert( paths.end(), incs.begin(), incs.end() );\n    // now resolve the file path relative to lookup paths\n    std::string resolved(File::find_file(file, paths));\n    return sass_copy_c_string(resolved.c_str());\n  }\n\n  // Make sure to free the returned value!\n  // Incs array has to be null terminated!\n  // this has the original resolve logic for sass include\n  char* ADDCALL sass_find_include (const char* file, struct Sass_Options* opt)\n  {\n    std::vector<std::string> vec(list2vec(opt->include_paths));\n    std::string resolved(File::find_include(file, vec));\n    return sass_copy_c_string(resolved.c_str());\n  }\n\n  // Make sure to free the returned value!\n  // Incs array has to be null terminated!\n  char* ADDCALL sass_find_file (const char* file, struct Sass_Options* opt)\n  {\n    std::vector<std::string> vec(list2vec(opt->include_paths));\n    std::string resolved(File::find_file(file, vec));\n    return sass_copy_c_string(resolved.c_str());\n  }\n\n  // Get compiled libsass version\n  const char* ADDCALL libsass_version(void)\n  {\n    return LIBSASS_VERSION;\n  }\n\n  // Get compiled libsass version\n  const char* ADDCALL libsass_language_version(void)\n  {\n    return LIBSASS_LANGUAGE_VERSION;\n  }\n\n}\n\nnamespace Sass {\n\n  // helper to aid dreaded MSVC debug mode\n  char* sass_copy_string(std::string str)\n  {\n    // In MSVC the following can lead to segfault:\n    // sass_copy_c_string(stream.str().c_str());\n    // Reason is that the string returned by str() is disposed before\n    // sass_copy_c_string is invoked. The string is actually a stack\n    // object, so indeed nobody is holding on to it. So it seems\n    // perfectly fair to release it right away. So the const char*\n    // by c_str will point to invalid memory. I'm not sure if this is\n    // the behavior for all compiler, but I'm pretty sure we would\n    // have gotten more issues reported if that would be the case.\n    // Wrapping it in a functions seems the cleanest approach as the\n    // function must hold on to the stack variable until it's done.\n    return sass_copy_c_string(str.c_str());\n  }\n\n}\n// Path: src/libsass/src/emitter.cpp\n#include \"sass.hpp\"\n#include \"util.hpp\"\n#include \"context.hpp\"\n#include \"output.hpp\"\n#include \"emitter.hpp\"\n#include \"utf8_string.hpp\"\n\nnamespace Sass {\n\n  Emitter::Emitter(struct Sass_Output_Options& opt)\n  : wbuf(),\n    opt(opt),\n    indentation(0),\n    scheduled_space(0),\n    scheduled_linefeed(0),\n    scheduled_delimiter(false),\n    scheduled_crutch(0),\n    scheduled_mapping(0),\n    in_custom_property(false),\n    in_comment(false),\n    in_wrapped(false),\n    in_media_block(false),\n    in_declaration(false),\n    in_space_array(false),\n    in_comma_array(false)\n  { }\n\n  // return buffer as string\n  std::string Emitter::get_buffer(void)\n  {\n    return wbuf.buffer;\n  }\n\n  Sass_Output_Style Emitter::output_style(void) const\n  {\n    return opt.output_style;\n  }\n\n  // PROXY METHODS FOR SOURCE MAPS\n\n  void Emitter::add_source_index(size_t idx)\n  { wbuf.smap.source_index.push_back(idx); }\n\n  std::string Emitter::render_srcmap(Context &ctx)\n  { return wbuf.smap.render_srcmap(ctx); }\n\n  void Emitter::set_filename(const std::string& str)\n  { wbuf.smap.file = str; }\n\n  void Emitter::schedule_mapping(const AST_Node_Ptr node)\n  { scheduled_mapping = node; }\n  void Emitter::add_open_mapping(const AST_Node_Ptr node)\n  { wbuf.smap.add_open_mapping(node); }\n  void Emitter::add_close_mapping(const AST_Node_Ptr node)\n  { wbuf.smap.add_close_mapping(node); }\n  ParserState Emitter::remap(const ParserState& pstate)\n  { return wbuf.smap.remap(pstate); }\n\n  // MAIN BUFFER MANIPULATION\n\n  // add outstanding delimiter\n  void Emitter::finalize(bool final)\n  {\n    scheduled_space = 0;\n    if (output_style() == SASS_STYLE_COMPRESSED)\n      if (final) scheduled_delimiter = false;\n    if (scheduled_linefeed)\n      scheduled_linefeed = 1;\n    flush_schedules();\n  }\n\n  // flush scheduled space/linefeed\n  void Emitter::flush_schedules(void)\n  {\n    // check the schedule\n    if (scheduled_linefeed) {\n      std::string linefeeds = \"\";\n\n      for (size_t i = 0; i < scheduled_linefeed; i++)\n        linefeeds += opt.linefeed;\n      scheduled_space = 0;\n      scheduled_linefeed = 0;\n      append_string(linefeeds);\n\n    } else if (scheduled_space) {\n      std::string spaces(scheduled_space, ' ');\n      scheduled_space = 0;\n      append_string(spaces);\n    }\n    if (scheduled_delimiter) {\n      scheduled_delimiter = false;\n      append_string(\";\");\n    }\n  }\n\n  // prepend some text or token to the buffer\n  void Emitter::prepend_output(const OutputBuffer& output)\n  {\n    wbuf.smap.prepend(output);\n    wbuf.buffer = output.buffer + wbuf.buffer;\n  }\n\n  // prepend some text or token to the buffer\n  void Emitter::prepend_string(const std::string& text)\n  {\n    // do not adjust mappings for utf8 bom\n    // seems they are not counted in any UA\n    if (text.compare(\"\\xEF\\xBB\\xBF\") != 0) {\n      wbuf.smap.prepend(Offset(text));\n    }\n    wbuf.buffer = text + wbuf.buffer;\n  }\n\n  char Emitter::last_char()\n  {\n    return wbuf.buffer.back();\n  }\n\n  // append a single char to the buffer\n  void Emitter::append_char(const char chr)\n  {\n    // write space/lf\n    flush_schedules();\n    // add to buffer\n    wbuf.buffer += chr;\n    // account for data in source-maps\n    wbuf.smap.append(Offset(chr));\n  }\n\n  // append some text or token to the buffer\n  void Emitter::append_string(const std::string& text)\n  {\n\n    // write space/lf\n    flush_schedules();\n\n    if (in_comment && output_style() == COMPACT) {\n      // unescape comment nodes\n      std::string out = comment_to_string(text);\n      // add to buffer\n      wbuf.buffer += out;\n      // account for data in source-maps\n      wbuf.smap.append(Offset(out));\n    } else {\n      // add to buffer\n      wbuf.buffer += text;\n      // account for data in source-maps\n      wbuf.smap.append(Offset(text));\n    }\n  }\n\n  // append some white-space only text\n  void Emitter::append_wspace(const std::string& text)\n  {\n    if (text.empty()) return;\n    if (peek_linefeed(text.c_str())) {\n      scheduled_space = 0;\n      append_mandatory_linefeed();\n    }\n  }\n\n  // append some text or token to the buffer\n  // this adds source-mappings for node start and end\n  void Emitter::append_token(const std::string& text, const AST_Node_Ptr node)\n  {\n    flush_schedules();\n    add_open_mapping(node);\n    // hotfix for browser issues\n    // this is pretty ugly indeed\n    if (scheduled_crutch) {\n      add_open_mapping(scheduled_crutch);\n      scheduled_crutch = 0;\n    }\n    append_string(text);\n    add_close_mapping(node);\n  }\n\n  // HELPER METHODS\n\n  void Emitter::append_indentation()\n  {\n    if (output_style() == COMPRESSED) return;\n    if (output_style() == COMPACT) return;\n    if (in_declaration && in_comma_array) return;\n    if (scheduled_linefeed && indentation)\n      scheduled_linefeed = 1;\n    std::string indent = \"\";\n    for (size_t i = 0; i < indentation; i++)\n      indent += opt.indent;\n    append_string(indent);\n  }\n\n  void Emitter::append_delimiter()\n  {\n    scheduled_delimiter = true;\n    if (output_style() == COMPACT) {\n      if (indentation == 0) {\n        append_mandatory_linefeed();\n      } else {\n        append_mandatory_space();\n      }\n    } else if (output_style() != COMPRESSED) {\n      append_optional_linefeed();\n    }\n  }\n\n  void Emitter::append_comma_separator()\n  {\n    // scheduled_space = 0;\n    append_string(\",\");\n    append_optional_space();\n  }\n\n  void Emitter::append_colon_separator()\n  {\n    scheduled_space = 0;\n    append_string(\":\");\n    if (!in_custom_property) append_optional_space();\n  }\n\n  void Emitter::append_mandatory_space()\n  {\n    scheduled_space = 1;\n  }\n\n  void Emitter::append_optional_space()\n  {\n    if ((output_style() != COMPRESSED) && buffer().size()) {\n      unsigned char lst = buffer().at(buffer().length() - 1);\n      if (!isspace(lst) || scheduled_delimiter) {\n        if (last_char() != '(') {\n          append_mandatory_space();\n        }\n      }\n    }\n  }\n\n  void Emitter::append_special_linefeed()\n  {\n    if (output_style() == COMPACT) {\n      append_mandatory_linefeed();\n      for (size_t p = 0; p < indentation; p++)\n        append_string(opt.indent);\n    }\n  }\n\n  void Emitter::append_optional_linefeed()\n  {\n    if (in_declaration && in_comma_array) return;\n    if (output_style() == COMPACT) {\n      append_mandatory_space();\n    } else {\n      append_mandatory_linefeed();\n    }\n  }\n\n  void Emitter::append_mandatory_linefeed()\n  {\n    if (output_style() != COMPRESSED) {\n      scheduled_linefeed = 1;\n      scheduled_space = 0;\n      // flush_schedules();\n    }\n  }\n\n  void Emitter::append_scope_opener(AST_Node_Ptr node)\n  {\n    scheduled_linefeed = 0;\n    append_optional_space();\n    flush_schedules();\n    if (node) add_open_mapping(node);\n    append_string(\"{\");\n    append_optional_linefeed();\n    // append_optional_space();\n    ++ indentation;\n  }\n  void Emitter::append_scope_closer(AST_Node_Ptr node)\n  {\n    -- indentation;\n    scheduled_linefeed = 0;\n    if (output_style() == COMPRESSED)\n      scheduled_delimiter = false;\n    if (output_style() == EXPANDED) {\n      append_optional_linefeed();\n      append_indentation();\n    } else {\n      append_optional_space();\n    }\n    append_string(\"}\");\n    if (node) add_close_mapping(node);\n    append_optional_linefeed();\n    if (indentation != 0) return;\n    if (output_style() != COMPRESSED)\n      scheduled_linefeed = 2;\n  }\n\n}\n\n// Path: src/libsass/src/listize.cpp\n#include \"sass.hpp\"\n#include <iostream>\n#include <typeinfo>\n#include <string>\n\n#include \"listize.hpp\"\n#include \"context.hpp\"\n#include \"backtrace.hpp\"\n#include \"error_handling.hpp\"\n\nnamespace Sass {\n\n  Listize::Listize()\n  {  }\n\n  Expression_Ptr Listize::operator()(Selector_List_Ptr sel)\n  {\n    List_Obj l = SASS_MEMORY_NEW(List, sel->pstate(), sel->length(), SASS_COMMA);\n    l->from_selector(true);\n    for (size_t i = 0, L = sel->length(); i < L; ++i) {\n      if (!sel->at(i)) continue;\n      l->append(sel->at(i)->perform(this));\n    }\n    if (l->length()) return l.detach();\n    return SASS_MEMORY_NEW(Null, l->pstate());\n  }\n\n  Expression_Ptr Listize::operator()(Compound_Selector_Ptr sel)\n  {\n    std::string str;\n    for (size_t i = 0, L = sel->length(); i < L; ++i) {\n      Expression_Ptr e = (*sel)[i]->perform(this);\n      if (e) str += e->to_string();\n    }\n    return SASS_MEMORY_NEW(String_Quoted, sel->pstate(), str);\n  }\n\n  Expression_Ptr Listize::operator()(Complex_Selector_Ptr sel)\n  {\n    List_Obj l = SASS_MEMORY_NEW(List, sel->pstate(), 2);\n    l->from_selector(true);\n    Compound_Selector_Obj head = sel->head();\n    if (head && !head->is_empty_reference())\n    {\n      Expression_Ptr hh = head->perform(this);\n      if (hh) l->append(hh);\n    }\n\n    std::string reference = ! sel->reference() ? \"\"\n      : sel->reference()->to_string();\n    switch(sel->combinator())\n    {\n      case Complex_Selector::PARENT_OF:\n        l->append(SASS_MEMORY_NEW(String_Quoted, sel->pstate(), \">\"));\n      break;\n      case Complex_Selector::ADJACENT_TO:\n        l->append(SASS_MEMORY_NEW(String_Quoted, sel->pstate(), \"+\"));\n      break;\n      case Complex_Selector::REFERENCE:\n        l->append(SASS_MEMORY_NEW(String_Quoted, sel->pstate(), \"/\" + reference + \"/\"));\n      break;\n      case Complex_Selector::PRECEDES:\n        l->append(SASS_MEMORY_NEW(String_Quoted, sel->pstate(), \"~\"));\n      break;\n      case Complex_Selector::ANCESTOR_OF:\n      break;\n      default: break;\n    }\n\n    Complex_Selector_Obj tail = sel->tail();\n    if (tail)\n    {\n      Expression_Obj tt = tail->perform(this);\n      if (List_Ptr ls = Cast<List>(tt))\n      { l->concat(ls); }\n    }\n    if (l->length() == 0) return 0;\n    return l.detach();\n  }\n\n  Expression_Ptr Listize::fallback_impl(AST_Node_Ptr n)\n  {\n    return Cast<Expression>(n);\n  }\n\n}\n\n// Path: src/libsass/src/source_map.cpp\n#include \"sass.hpp\"\n#include <string>\n#include <sstream>\n#include <iostream>\n#include <iomanip>\n\n#include \"ast.hpp\"\n#include \"json.hpp\"\n#include \"context.hpp\"\n#include \"position.hpp\"\n#include \"source_map.hpp\"\n\nnamespace Sass {\n  SourceMap::SourceMap() : current_position(0, 0, 0), file(\"stdin\") { }\n  SourceMap::SourceMap(const std::string& file) : current_position(0, 0, 0), file(file) { }\n\n  std::string SourceMap::render_srcmap(Context &ctx) {\n\n    const bool include_sources = ctx.c_options.source_map_contents;\n    const std::vector<std::string> links = ctx.srcmap_links;\n    const std::vector<Resource>& sources(ctx.resources);\n\n    JsonNode* json_srcmap = json_mkobject();\n\n    json_append_member(json_srcmap, \"version\", json_mknumber(3));\n\n    const char *file_name = file.c_str();\n    JsonNode *json_file_name = json_mkstring(file_name);\n    json_append_member(json_srcmap, \"file\", json_file_name);\n\n    // pass-through sourceRoot option\n    if (!ctx.source_map_root.empty()) {\n      JsonNode* root = json_mkstring(ctx.source_map_root.c_str());\n      json_append_member(json_srcmap, \"sourceRoot\", root);\n    }\n\n    JsonNode *json_sources = json_mkarray();\n    for (size_t i = 0; i < source_index.size(); ++i) {\n      std::string source(links[source_index[i]]);\n      if (ctx.c_options.source_map_file_urls) {\n        source = File::rel2abs(source);\n        // check for windows abs path\n        if (source[0] == '/') {\n          // ends up with three slashes\n          source = \"file://\" + source;\n        } else {\n          // needs an additional slash\n          source = \"file:///\" + source;\n        }\n      }\n      const char* source_name = source.c_str();\n      JsonNode *json_source_name = json_mkstring(source_name);\n      json_append_element(json_sources, json_source_name);\n    }\n    json_append_member(json_srcmap, \"sources\", json_sources);\n\n    if (include_sources && source_index.size()) {\n      JsonNode *json_contents = json_mkarray();\n      for (size_t i = 0; i < source_index.size(); ++i) {\n        const Resource& resource(sources[source_index[i]]);\n        JsonNode *json_content = json_mkstring(resource.contents);\n        json_append_element(json_contents, json_content);\n      }\n      json_append_member(json_srcmap, \"sourcesContent\", json_contents);\n    }\n\n    JsonNode *json_names = json_mkarray();\n    // so far we have no implementation for names\n    // no problem as we do not alter any identifiers\n    json_append_member(json_srcmap, \"names\", json_names);\n\n    std::string mappings = serialize_mappings();\n    JsonNode *json_mappings = json_mkstring(mappings.c_str());\n    json_append_member(json_srcmap, \"mappings\", json_mappings);\n\n    char *str = json_stringify(json_srcmap, \"\\t\");\n    std::string result = std::string(str);\n    free(str);\n    json_delete(json_srcmap);\n    return result;\n  }\n\n  std::string SourceMap::serialize_mappings() {\n    std::string result = \"\";\n\n    size_t previous_generated_line = 0;\n    size_t previous_generated_column = 0;\n    size_t previous_original_line = 0;\n    size_t previous_original_column = 0;\n    size_t previous_original_file = 0;\n    for (size_t i = 0; i < mappings.size(); ++i) {\n      const size_t generated_line = mappings[i].generated_position.line;\n      const size_t generated_column = mappings[i].generated_position.column;\n      const size_t original_line = mappings[i].original_position.line;\n      const size_t original_column = mappings[i].original_position.column;\n      const size_t original_file = mappings[i].original_position.file;\n\n      if (generated_line != previous_generated_line) {\n        previous_generated_column = 0;\n        if (generated_line > previous_generated_line) {\n          result += std::string(generated_line - previous_generated_line, ';');\n          previous_generated_line = generated_line;\n        }\n      }\n      else if (i > 0) {\n        result += \",\";\n      }\n\n      // generated column\n      result += base64vlq.encode(static_cast<int>(generated_column) - static_cast<int>(previous_generated_column));\n      previous_generated_column = generated_column;\n      // file\n      result += base64vlq.encode(static_cast<int>(original_file) - static_cast<int>(previous_original_file));\n      previous_original_file = original_file;\n      // source line\n      result += base64vlq.encode(static_cast<int>(original_line) - static_cast<int>(previous_original_line));\n      previous_original_line = original_line;\n      // source column\n      result += base64vlq.encode(static_cast<int>(original_column) - static_cast<int>(previous_original_column));\n      previous_original_column = original_column;\n    }\n\n    return result;\n  }\n\n  void SourceMap::prepend(const OutputBuffer& out)\n  {\n    Offset size(out.smap.current_position);\n    for (Mapping mapping : out.smap.mappings) {\n      if (mapping.generated_position.line > size.line) {\n        throw(std::runtime_error(\"prepend sourcemap has illegal line\"));\n      }\n      if (mapping.generated_position.line == size.line) {\n        if (mapping.generated_position.column > size.column) {\n          throw(std::runtime_error(\"prepend sourcemap has illegal column\"));\n        }\n      }\n    }\n    // adjust the buffer offset\n    prepend(Offset(out.buffer));\n    // now add the new mappings\n    VECTOR_UNSHIFT(mappings, out.smap.mappings);\n  }\n\n  void SourceMap::append(const OutputBuffer& out)\n  {\n    append(Offset(out.buffer));\n  }\n\n  void SourceMap::prepend(const Offset& offset)\n  {\n    if (offset.line != 0 || offset.column != 0) {\n      for (Mapping& mapping : mappings) {\n        // move stuff on the first old line\n        if (mapping.generated_position.line == 0) {\n          mapping.generated_position.column += offset.column;\n        }\n        // make place for the new lines\n        mapping.generated_position.line += offset.line;\n      }\n    }\n    if (current_position.line == 0) {\n      current_position.column += offset.column;\n    }\n    current_position.line += offset.line;\n  }\n\n  void SourceMap::append(const Offset& offset)\n  {\n    current_position += offset;\n  }\n\n  void SourceMap::add_open_mapping(const AST_Node_Ptr node)\n  {\n    mappings.push_back(Mapping(node->pstate(), current_position));\n  }\n\n  void SourceMap::add_close_mapping(const AST_Node_Ptr node)\n  {\n    mappings.push_back(Mapping(node->pstate() + node->pstate().offset, current_position));\n  }\n\n  ParserState SourceMap::remap(const ParserState& pstate) {\n    for (size_t i = 0; i < mappings.size(); ++i) {\n      if (\n        mappings[i].generated_position.file == pstate.file &&\n        mappings[i].generated_position.line == pstate.line &&\n        mappings[i].generated_position.column == pstate.column\n      ) return ParserState(pstate.path, pstate.src, mappings[i].original_position, pstate.offset);\n    }\n    return ParserState(pstate.path, pstate.src, Position(-1, -1, -1), Offset(0, 0));\n\n  }\n\n}\n\n// Path: src/libsass/src/bind.cpp\n#include \"sass.hpp\"\n#include \"bind.hpp\"\n#include \"ast.hpp\"\n#include \"context.hpp\"\n#include \"expand.hpp\"\n#include \"eval.hpp\"\n#include <map>\n#include <iostream>\n#include <sstream>\n\nnamespace Sass {\n\n  void bind(std::string type, std::string name, Parameters_Obj ps, Arguments_Obj as, Context* ctx, Env* env, Eval* eval)\n  {\n    std::string callee(type + \" \" + name);\n\n    std::map<std::string, Parameter_Obj> param_map;\n    List_Obj varargs = SASS_MEMORY_NEW(List, as->pstate());\n    varargs->is_arglist(true); // enable keyword size handling\n\n    for (size_t i = 0, L = as->length(); i < L; ++i) {\n      if (auto str = Cast<String_Quoted>((*as)[i]->value())) {\n        // force optional quotes (only if needed)\n        if (str->quote_mark()) {\n          str->quote_mark('*');\n        }\n      }\n    }\n\n    // Set up a map to ensure named arguments refer to actual parameters. Also\n    // eval each default value left-to-right, wrt env, populating env as we go.\n    for (size_t i = 0, L = ps->length(); i < L; ++i) {\n      Parameter_Obj  p = ps->at(i);\n      param_map[p->name()] = p;\n      // if (p->default_value()) {\n      //   env->local_frame()[p->name()] = p->default_value()->perform(eval->with(env));\n      // }\n    }\n\n    // plug in all args; if we have leftover params, deal with it later\n    size_t ip = 0, LP = ps->length();\n    size_t ia = 0, LA = as->length();\n    while (ia < LA) {\n      Argument_Obj a = as->at(ia);\n      if (ip >= LP) {\n        // skip empty rest arguments\n        if (a->is_rest_argument()) {\n          if (List_Obj l = Cast<List>(a->value())) {\n            if (l->length() == 0) {\n              ++ ia; continue;\n            }\n          }\n        }\n        std::stringstream msg;\n        msg << \"wrong number of arguments (\" << LA << \" for \" << LP << \")\";\n        msg << \" for `\" << name << \"'\";\n        return error(msg.str(), as->pstate(), eval->exp.traces);\n      }\n      Parameter_Obj p = ps->at(ip);\n\n      // If the current parameter is the rest parameter, process and break the loop\n      if (p->is_rest_parameter()) {\n        // The next argument by coincidence provides a rest argument\n        if (a->is_rest_argument()) {\n\n          // We should always get a list for rest arguments\n          if (List_Obj rest = Cast<List>(a->value())) {\n              // create a new list object for wrapped items\n              List_Ptr arglist = SASS_MEMORY_NEW(List,\n                                              p->pstate(),\n                                              0,\n                                              rest->separator(),\n                                              true);\n              // wrap each item from list as an argument\n              for (Expression_Obj item : rest->elements()) {\n                if (Argument_Obj arg = Cast<Argument>(item)) {\n                  arglist->append(SASS_MEMORY_COPY(arg)); // copy\n                } else {\n                  arglist->append(SASS_MEMORY_NEW(Argument,\n                                                  item->pstate(),\n                                                  item,\n                                                  \"\",\n                                                  false,\n                                                  false));\n                }\n              }\n              // assign new arglist to environment\n              env->local_frame()[p->name()] = arglist;\n            }\n          // invalid state\n          else {\n            throw std::runtime_error(\"invalid state\");\n          }\n        } else if (a->is_keyword_argument()) {\n\n          // expand keyword arguments into their parameters\n          List_Ptr arglist = SASS_MEMORY_NEW(List, p->pstate(), 0, SASS_COMMA, true);\n          env->local_frame()[p->name()] = arglist;\n          Map_Obj argmap = Cast<Map>(a->value());\n          for (auto key : argmap->keys()) {\n            if (String_Constant_Obj str = Cast<String_Constant>(key)) {\n              std::string param = unquote(str->value());\n              arglist->append(SASS_MEMORY_NEW(Argument,\n                                              key->pstate(),\n                                              argmap->at(key),\n                                              \"$\" + param,\n                                              false,\n                                              false));\n            } else {\n              eval->exp.traces.push_back(Backtrace(key->pstate()));\n              throw Exception::InvalidVarKwdType(key->pstate(), eval->exp.traces, key->inspect(), a);\n            }\n          }\n\n        } else {\n\n          // create a new list object for wrapped items\n          List_Obj arglist = SASS_MEMORY_NEW(List,\n                                          p->pstate(),\n                                          0,\n                                          SASS_COMMA,\n                                          true);\n          // consume the next args\n          while (ia < LA) {\n            // get and post inc\n            a = (*as)[ia++];\n            // maybe we have another list as argument\n            List_Obj ls = Cast<List>(a->value());\n            // skip any list completely if empty\n            if (ls && ls->empty() && a->is_rest_argument()) continue;\n\n            Expression_Obj value = a->value();\n            if (Argument_Obj arg = Cast<Argument>(value)) {\n              arglist->append(arg);\n            }\n            // check if we have rest argument\n            else if (a->is_rest_argument()) {\n              // preserve the list separator from rest args\n              if (List_Obj rest = Cast<List>(a->value())) {\n                arglist->separator(rest->separator());\n\n                for (size_t i = 0, L = rest->length(); i < L; ++i) {\n                  Expression_Obj obj = rest->value_at_index(i);\n                  arglist->append(SASS_MEMORY_NEW(Argument,\n                                                obj->pstate(),\n                                                obj,\n                                                \"\",\n                                                false,\n                                                false));\n                }\n              }\n              // no more arguments\n              break;\n            }\n            // wrap all other value types into Argument\n            else {\n              arglist->append(SASS_MEMORY_NEW(Argument,\n                                            a->pstate(),\n                                            a->value(),\n                                            a->name(),\n                                            false,\n                                            false));\n            }\n          }\n          // assign new arglist to environment\n          env->local_frame()[p->name()] = arglist;\n        }\n        // consumed parameter\n        ++ip;\n        // no more paramaters\n        break;\n      }\n\n      // If the current argument is the rest argument, extract a value for processing\n      else if (a->is_rest_argument()) {\n        // normal param and rest arg\n        List_Obj arglist = Cast<List>(a->value());\n        if (!arglist) {\n          if (Expression_Obj arg = Cast<Expression>(a->value())) {\n            arglist = SASS_MEMORY_NEW(List, a->pstate(), 1);\n            arglist->append(arg);\n          }\n        }\n\n        // empty rest arg - treat all args as default values\n        if (!arglist || !arglist->length()) {\n          break;\n        } else {\n          if (arglist->length() > LP - ip && !ps->has_rest_parameter()) {\n            size_t arg_count = (arglist->length() + LA - 1);\n            std::stringstream msg;\n            msg << callee << \" takes \" << LP;\n            msg << (LP == 1 ? \" argument\" : \" arguments\");\n            msg << \" but \" << arg_count;\n            msg << (arg_count == 1 ? \" was passed\" : \" were passed.\");\n            deprecated_bind(msg.str(), as->pstate());\n\n            while (arglist->length() > LP - ip) {\n              arglist->elements().erase(arglist->elements().end() - 1);\n            }\n          }\n        }\n        // otherwise move one of the rest args into the param, converting to argument if necessary\n        Expression_Obj obj = arglist->at(0);\n        if (!(a = Cast<Argument>(obj))) {\n          Expression_Ptr a_to_convert = obj;\n          a = SASS_MEMORY_NEW(Argument,\n                              a_to_convert->pstate(),\n                              a_to_convert,\n                              \"\",\n                              false,\n                              false);\n        }\n        arglist->elements().erase(arglist->elements().begin());\n        if (!arglist->length() || (!arglist->is_arglist() && ip + 1 == LP)) {\n          ++ia;\n        }\n\n      } else if (a->is_keyword_argument()) {\n        Map_Obj argmap = Cast<Map>(a->value());\n\n        for (auto key : argmap->keys()) {\n          String_Constant_Ptr val = Cast<String_Constant>(key);\n          if (val == NULL) {\n            eval->exp.traces.push_back(Backtrace(key->pstate()));\n            throw Exception::InvalidVarKwdType(key->pstate(), eval->exp.traces, key->inspect(), a);\n          }\n          std::string param = \"$\" + unquote(val->value());\n\n          if (!param_map.count(param)) {\n            std::stringstream msg;\n            msg << callee << \" has no parameter named \" << param;\n            error(msg.str(), a->pstate(), eval->exp.traces);\n          }\n          env->local_frame()[param] = argmap->at(key);\n        }\n        ++ia;\n        continue;\n      } else {\n        ++ia;\n      }\n\n      if (a->name().empty()) {\n        if (env->has_local(p->name())) {\n          std::stringstream msg;\n          msg << \"parameter \" << p->name()\n          << \" provided more than once in call to \" << callee;\n          error(msg.str(), a->pstate(), eval->exp.traces);\n        }\n        // ordinal arg -- bind it to the next param\n        env->local_frame()[p->name()] = a->value();\n        ++ip;\n      }\n      else {\n        // named arg -- bind it to the appropriately named param\n        if (!param_map.count(a->name())) {\n          if (ps->has_rest_parameter()) {\n            varargs->append(a);\n          } else {\n            std::stringstream msg;\n            msg << callee << \" has no parameter named \" << a->name();\n            error(msg.str(), a->pstate(), eval->exp.traces);\n          }\n        }\n        if (param_map[a->name()]) {\n          if (param_map[a->name()]->is_rest_parameter()) {\n            std::stringstream msg;\n            msg << \"argument \" << a->name() << \" of \" << callee\n                << \"cannot be used as named argument\";\n            error(msg.str(), a->pstate(), eval->exp.traces);\n          }\n        }\n        if (env->has_local(a->name())) {\n          std::stringstream msg;\n          msg << \"parameter \" << p->name()\n              << \"provided more than once in call to \" << callee;\n          error(msg.str(), a->pstate(), eval->exp.traces);\n        }\n        env->local_frame()[a->name()] = a->value();\n      }\n    }\n    // EO while ia\n\n    // If we make it here, we're out of args but may have leftover params.\n    // That's only okay if they have default values, or were already bound by\n    // named arguments, or if it's a single rest-param.\n    for (size_t i = ip; i < LP; ++i) {\n      Parameter_Obj leftover = ps->at(i);\n      // cerr << \"env for default params:\" << endl;\n      // env->print();\n      // cerr << \"********\" << endl;\n      if (!env->has_local(leftover->name())) {\n        if (leftover->is_rest_parameter()) {\n          env->local_frame()[leftover->name()] = varargs;\n        }\n        else if (leftover->default_value()) {\n          Expression_Ptr dv = leftover->default_value()->perform(eval);\n          env->local_frame()[leftover->name()] = dv;\n        }\n        else {\n          // param is unbound and has no default value -- error\n          throw Exception::MissingArgument(as->pstate(), eval->exp.traces, name, leftover->name(), type);\n        }\n      }\n    }\n\n    return;\n  }\n\n\n}\n\n// Path: src/libsass/src/debugger.hpp\n#ifndef SASS_DEBUGGER_H\n#define SASS_DEBUGGER_H\n\n#include <string>\n#include <sstream>\n#include \"node.hpp\"\n#include \"ast_fwd_decl.hpp\"\n\nusing namespace Sass;\n\ninline void debug_ast(AST_Node_Ptr node, std::string ind = \"\", Env* env = 0);\n\ninline void debug_ast(const AST_Node* node, std::string ind = \"\", Env* env = 0) {\n  debug_ast(const_cast<AST_Node*>(node), ind, env);\n}\n\n\ninline void debug_sources_set(ComplexSelectorSet& set, std::string ind = \"\")\n{\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n  for(auto const &pair : set) {\n    debug_ast(pair, ind + \"\");\n    // debug_ast(set[pair], ind + \"first: \");\n  }\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n}\n\ninline std::string str_replace(std::string str, const std::string& oldStr, const std::string& newStr)\n{\n  size_t pos = 0;\n  while((pos = str.find(oldStr, pos)) != std::string::npos)\n  {\n     str.replace(pos, oldStr.length(), newStr);\n     pos += newStr.length();\n  }\n  return str;\n}\n\ninline std::string prettyprint(const std::string& str) {\n  std::string clean = str_replace(str, \"\\n\", \"\\\\n\");\n  clean = str_replace(clean, \"\t\", \"\\\\t\");\n  clean = str_replace(clean, \"\\r\", \"\\\\r\");\n  return clean;\n}\n\ninline std::string longToHex(long long t) {\n  std::stringstream is;\n  is << std::hex << t;\n  return is.str();\n}\n\ninline std::string pstate_source_position(AST_Node_Ptr node)\n{\n  std::stringstream str;\n  Position start(node->pstate());\n  Position end(start + node->pstate().offset);\n  str << (start.file == std::string::npos ? -1 : start.file)\n    << \"@[\" << start.line << \":\" << start.column << \"]\"\n    << \"-[\" << end.line << \":\" << end.column << \"]\";\n#ifdef DEBUG_SHARED_PTR\n      str << \"x\" << node->getRefCount() << \"\"\n      << \" \" << node->getDbgFile()\n      << \"@\" << node->getDbgLine();\n#endif\n  return str.str();\n}\n\ninline void debug_ast(AST_Node_Ptr node, std::string ind, Env* env)\n{\n  if (node == 0) return;\n  if (ind == \"\") std::cerr << \"####################################################################\\n\";\n  if (Cast<Bubble>(node)) {\n    Bubble_Ptr bubble = Cast<Bubble>(node);\n    std::cerr << ind << \"Bubble \" << bubble;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << bubble->tabs();\n    std::cerr << std::endl;\n    debug_ast(bubble->node(), ind + \" \", env);\n  } else if (Cast<Trace>(node)) {\n    Trace_Ptr trace = Cast<Trace>(node);\n    std::cerr << ind << \"Trace \" << trace;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\"\n    << \" [name:\" << trace->name() << \", type: \" << trace->type() << \"]\"\n    << std::endl;\n    debug_ast(trace->block(), ind + \" \", env);\n  } else if (Cast<At_Root_Block>(node)) {\n    At_Root_Block_Ptr root_block = Cast<At_Root_Block>(node);\n    std::cerr << ind << \"At_Root_Block \" << root_block;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" \" << root_block->tabs();\n    std::cerr << std::endl;\n    debug_ast(root_block->expression(), ind + \":\", env);\n    debug_ast(root_block->block(), ind + \" \", env);\n  } else if (Cast<Selector_List>(node)) {\n    Selector_List_Ptr selector = Cast<Selector_List>(node);\n    std::cerr << ind << \"Selector_List \" << selector;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" <\" << selector->hash() << \">\";\n    std::cerr << \" [@media:\" << selector->media_block() << \"]\";\n    std::cerr << (selector->is_invisible() ? \" [INVISIBLE]\": \" -\");\n    std::cerr << (selector->has_placeholder() ? \" [PLACEHOLDER]\": \" -\");\n    std::cerr << (selector->is_optional() ? \" [is_optional]\": \" -\");\n    std::cerr << (selector->has_parent_ref() ? \" [has-parent]\": \" -\");\n    std::cerr << (selector->has_line_break() ? \" [line-break]\": \" -\");\n    std::cerr << (selector->has_line_feed() ? \" [line-feed]\": \" -\");\n    std::cerr << std::endl;\n    debug_ast(selector->schema(), ind + \"#{} \");\n\n    for(const Complex_Selector_Obj& i : selector->elements()) { debug_ast(i, ind + \" \", env); }\n\n//  } else if (Cast<Expression>(node)) {\n//    Expression_Ptr expression = Cast<Expression>(node);\n//    std::cerr << ind << \"Expression \" << expression << \" \" << expression->concrete_type() << std::endl;\n\n  } else if (Cast<Parent_Selector>(node)) {\n    Parent_Selector_Ptr selector = Cast<Parent_Selector>(node);\n    std::cerr << ind << \"Parent_Selector \" << selector;\n//    if (selector->not_selector()) cerr << \" [in_declaration]\";\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" <\" << selector->hash() << \">\";\n    std::cerr << \" [\" << (selector->is_real_parent_ref() ? \"REAL\" : \"FAKE\") << \"]\";\n    std::cerr << \" <\" << prettyprint(selector->pstate().token.ws_before()) << \">\" << std::endl;\n//    debug_ast(selector->selector(), ind + \"->\", env);\n\n  } else if (Cast<Complex_Selector>(node)) {\n    Complex_Selector_Ptr selector = Cast<Complex_Selector>(node);\n    std::cerr << ind << \"Complex_Selector \" << selector\n      << \" (\" << pstate_source_position(node) << \")\"\n      << \" <\" << selector->hash() << \">\"\n      << \" [length:\" << longToHex(selector->length()) << \"]\"\n      << \" [weight:\" << longToHex(selector->specificity()) << \"]\"\n      << \" [@media:\" << selector->media_block() << \"]\"\n      << (selector->is_invisible() ? \" [INVISIBLE]\": \" -\")\n      << (selector->has_placeholder() ? \" [PLACEHOLDER]\": \" -\")\n      << (selector->is_optional() ? \" [is_optional]\": \" -\")\n      << (selector->has_parent_ref() ? \" [has parent]\": \" -\")\n      << (selector->has_line_feed() ? \" [line-feed]\": \" -\")\n      << (selector->has_line_break() ? \" [line-break]\": \" -\")\n      << \" -- \";\n      std::string del;\n      switch (selector->combinator()) {\n        case Complex_Selector::PARENT_OF:   del = \">\"; break;\n        case Complex_Selector::PRECEDES:    del = \"~\"; break;\n        case Complex_Selector::ADJACENT_TO: del = \"+\"; break;\n        case Complex_Selector::ANCESTOR_OF: del = \" \"; break;\n        case Complex_Selector::REFERENCE:   del = \"//\"; break;\n      }\n      // if (del = \"/\") del += selector->reference()->perform(&to_string) + \"/\";\n    std::cerr << \" <\" << prettyprint(selector->pstate().token.ws_before()) << \">\" << std::endl;\n    debug_ast(selector->head(), ind + \" \" /* + \"[\" + del + \"]\" */, env);\n    if (selector->tail()) {\n      debug_ast(selector->tail(), ind + \"{\" + del + \"}\", env);\n    } else if(del != \" \") {\n      std::cerr << ind << \" |\" << del << \"| {trailing op}\" << std::endl;\n    }\n    ComplexSelectorSet set = selector->sources();\n    // debug_sources_set(set, ind + \"  @--> \");\n  } else if (Cast<Compound_Selector>(node)) {\n    Compound_Selector_Ptr selector = Cast<Compound_Selector>(node);\n    std::cerr << ind << \"Compound_Selector \" << selector;\n    std::cerr << \" (\" << pstate_source_position(node) << \")\";\n    std::cerr << \" <\" << selector->hash() << \">\";\n    std::cerr << \" [weight:\" << longToHex(selector->specificity()) << \"]\";\n    std::cerr << \" [@media:\" << selector->media_block() << \"]\";\n    std::cerr << (selector->extended() ? \" [extended]\": \" -\");\n    std::cerr << (selector->is_optional() ? \" [is_optional]\": \" -\");\n    std::cerr << (selector->has_parent_ref() ? \" [has-parent]\": \" -\");\n    std::cerr << (selector->has_line_break() ? \" [line-break]\": \" -\");\n    std::cerr << (selector->has_line_feed() ? \" [line-feed]\": \" -\");\n    std::cerr << \" <\" << prettyprint(selector->pstate().token.ws_before()) << \">\" << std::endl;\n    for(const Simple_Selector_Obj& i : selector->elements()) { debug_ast(i, ind + \" \", env); }\n  } else if (Cast<Wrapped_Selector>(node)) {\n    Wrapped_Selector_Ptr selector = Cast<Wrapped_Selector>(node);\n...\n", "needle_token_start": 13682, "needle_token_end": 13804, "code_context_ntokens": 16192, "output": ["```cpp\ninline void debug_sources_set(ComplexSelectorSet& set, std::string ind = \"\")\n{\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n  for(auto const &pair : set) {\n    debug_ast(pair, ind + \"\");\n    // debug_ast(set[pair], ind + \"first: \");\n  }\n  if (ind == \"\") std::cerr << \"#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n\";\n}\n```"]}
{"repo": "sass/node-sass", "name": "is_alnum", "language": "cpp", "path": "src/libsass/src/lexer.cpp", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine if a character is either a letter or a digit.\n2. **Input**: A single character.\n3. **Output**: A boolean value; `true` if the character is a letter or a digit, otherwise `false`.\n4. **Procedure**: The function checks if the character is a letter by calling another function designed to identify letters, or if it is a digit by calling a function designed to identify digits. If either condition is met, it returns `true`; otherwise, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/libsass/src/lexer.hpp\n#ifndef SASS_LEXER_H\n#define SASS_LEXER_H\n\n#include <cstring>\n\nnamespace Sass {\n  namespace Prelexer {\n\n    //####################################\n    // BASIC CHARACTER MATCHERS\n    //####################################\n\n    // Match standard control chars\n    const char* kwd_at(const char* src);\n    const char* kwd_dot(const char* src);\n    const char* kwd_comma(const char* src);\n    const char* kwd_colon(const char* src);\n    const char* kwd_star(const char* src);\n    const char* kwd_plus(const char* src);\n    const char* kwd_minus(const char* src);\n    const char* kwd_slash(const char* src);\n\n    //####################################\n    // BASIC CLASS MATCHERS\n    //####################################\n\n    // These are locale independant\n    bool is_space(const char& src);\n    bool is_alpha(const char& src);\n    bool is_punct(const char& src);\n...\n// Path: src/libsass/src/prelexer.hpp\n#ifndef SASS_PRELEXER_H\n#define SASS_PRELEXER_H\n\n#include <cstring>\n#include \"lexer.hpp\"\n\nnamespace Sass {\n  // using namespace Lexer;\n  namespace Prelexer {\n\n    //####################################\n    // KEYWORD \"REGEX\" MATCHERS\n    //####################################\n\n    // Match Sass boolean keywords.\n    const char* kwd_true(const char* src);\n    const char* kwd_false(const char* src);\n    const char* kwd_only(const char* src);\n    const char* kwd_and(const char* src);\n    const char* kwd_or(const char* src);\n    const char* kwd_not(const char* src);\n    const char* kwd_eq(const char* src);\n    const char* kwd_neq(const char* src);\n    const char* kwd_gt(const char* src);\n    const char* kwd_gte(const char* src);\n    const char* kwd_lt(const char* src);\n    const char* kwd_lte(const char* src);\n\n    // Match standard control chars\n    const char* kwd_at(const char* src);\n    const char* kwd_dot(const char* src);\n    const char* kwd_comma(const char* src);\n    const char* kwd_colon(const char* src);\n    const char* kwd_slash(const char* src);\n    const char* kwd_star(const char* src);\n    const char* kwd_plus(const char* src);\n    const char* kwd_minus(const char* src);\n\n    //####################################\n    // SPECIAL \"REGEX\" CONSTRUCTS\n    //####################################\n\n    // Match a sequence of characters delimited by the supplied chars.\n    template <char beg, char end, bool esc>\n    const char* delimited_by(const char* src) {\n      src = exactly<beg>(src);\n      if (!src) return 0;\n      const char* stop;\n      while (true) {\n        if (!*src) return 0;\n        stop = exactly<end>(src);\n        if (stop && (!esc || *(src - 1) != '\\\\')) return stop;\n        src = stop ? stop : src + 1;\n      }\n    }\n\n    // skip to delimiter (mx) inside given range\n    // this will savely skip over all quoted strings\n    // recursive skip stuff delimited by start/stop\n    // first start/opener must be consumed already!\n    template<prelexer start, prelexer stop>\n    const char* skip_over_scopes(const char* src, const char* end) {\n\n      size_t level = 0;\n      bool in_squote = false;\n      bool in_dquote = false;\n      // bool in_braces = false;\n\n      while (*src) {\n\n        // check for abort condition\n        if (end && src >= end) break;\n\n        // has escaped sequence?\n        if (*src == '\\\\') {\n          ++ src; // skip this (and next)\n        }\n        else if (*src == '\"') {\n          in_dquote = ! in_dquote;\n        }\n        else if (*src == '\\'') {\n          in_squote = ! in_squote;\n        }\n        else if (in_dquote || in_squote) {\n          // take everything literally\n        }\n\n        // find another opener inside?\n        else if (const char* pos = start(src)) {\n          ++ level; // increase counter\n          src = pos - 1; // advance position\n        }\n\n        // look for the closer (maybe final, maybe not)\n        else if (const char* final = stop(src)) {\n          // only close one level?\n          if (level > 0) -- level;\n          // return position at end of stop\n          // delimiter may be multiple chars\n          else return final;\n          // advance position\n          src = final - 1;\n        }\n\n        // next\n        ++ src;\n      }\n\n      return 0;\n    }\n\n    // skip to a skip delimited by parentheses\n    // uses smart `skip_over_scopes` internally\n    const char* parenthese_scope(const char* src);\n\n    // skip to delimiter (mx) inside given range\n    // this will savely skip over all quoted strings\n    // recursive skip stuff delimited by start/stop\n    // first start/opener must be consumed already!\n    template<prelexer start, prelexer stop>\n    const char* skip_over_scopes(const char* src) {\n      return skip_over_scopes<start, stop>(src, 0);\n    }\n\n    // Match a sequence of characters delimited by the supplied chars.\n    template <prelexer start, prelexer stop>\n    const char* recursive_scopes(const char* src) {\n      // parse opener\n      src = start(src);\n      // abort if not found\n      if (!src) return 0;\n      // parse the rest until final closer\n      return skip_over_scopes<start, stop>(src);\n    }\n\n    // Match a sequence of characters delimited by the supplied strings.\n    template <const char* beg, const char* end, bool esc>\n    const char* delimited_by(const char* src) {\n      src = exactly<beg>(src);\n      if (!src) return 0;\n      const char* stop;\n      while (true) {\n        if (!*src) return 0;\n        stop = exactly<end>(src);\n        if (stop && (!esc || *(src - 1) != '\\\\')) return stop;\n        src = stop ? stop : src + 1;\n      }\n    }\n\n    // Tries to match a certain number of times (between the supplied interval).\n    template<prelexer mx, size_t lo, size_t hi>\n    const char* between(const char* src) {\n      for (size_t i = 0; i < lo; ++i) {\n        src = mx(src);\n        if (!src) return 0;\n      }\n      for (size_t i = lo; i <= hi; ++i) {\n        const char* new_src = mx(src);\n        if (!new_src) return src;\n        src = new_src;\n      }\n      return src;\n    }\n\n    // equivalent of STRING_REGULAR_EXPRESSIONS\n    const char* re_string_double_open(const char* src);\n    const char* re_string_double_close(const char* src);\n    const char* re_string_single_open(const char* src);\n    const char* re_string_single_close(const char* src);\n    const char* re_string_uri_open(const char* src);\n    const char* re_string_uri_close(const char* src);\n\n    // Match a line comment.\n    const char* line_comment(const char* src);\n\n    // Match a block comment.\n    const char* block_comment(const char* src);\n    // Match either.\n    const char* comment(const char* src);\n    // Match double- and single-quoted strings.\n    const char* double_quoted_string(const char* src);\n    const char* single_quoted_string(const char* src);\n    const char* quoted_string(const char* src);\n    // Match interpolants.\n    const char* interpolant(const char* src);\n    // Match number prefix ([\\+\\-]+)\n    const char* number_prefix(const char* src);\n\n    // Match zero plus white-space or line_comments\n    const char* optional_css_whitespace(const char* src);\n    const char* css_whitespace(const char* src);\n    // Match optional_css_whitepace plus block_comments\n    const char* optional_css_comments(const char* src);\n    const char* css_comments(const char* src);\n\n    // Match one backslash escaped char\n    const char* escape_seq(const char* src);\n\n    // Match CSS css variables.\n    const char* custom_property_name(const char* src);\n    // Match a CSS identifier.\n    const char* identifier(const char* src);\n    const char* identifier_alpha(const char* src);\n    const char* identifier_alnum(const char* src);\n    const char* strict_identifier(const char* src);\n    const char* strict_identifier_alpha(const char* src);\n    const char* strict_identifier_alnum(const char* src);\n    // Match a CSS unit identifier.\n    const char* one_unit(const char* src);\n    const char* multiple_units(const char* src);\n    const char* unit_identifier(const char* src);\n    // const char* strict_identifier_alnums(const char* src);\n    // Match reference selector.\n    const char* re_reference_combinator(const char* src);\n    const char* static_reference_combinator(const char* src);\n    const char* schema_reference_combinator(const char* src);\n\n    // Match interpolant schemas\n    const char* identifier_schema(const char* src);\n    const char* value_schema(const char* src);\n    const char* sass_value(const char* src);\n    // const char* filename(const char* src);\n    // const char* filename_schema(const char* src);\n    // const char* url_schema(const char* src);\n    // const char* url_value(const char* src);\n    const char* vendor_prefix(const char* src);\n\n    const char* re_special_directive(const char* src);\n    const char* re_prefixed_directive(const char* src);\n    const char* re_almost_any_value_token(const char* src);\n\n    // Match CSS '@' keywords.\n    const char* at_keyword(const char* src);\n    const char* kwd_import(const char* src);\n    const char* kwd_at_root(const char* src);\n    const char* kwd_with_directive(const char* src);\n    const char* kwd_without_directive(const char* src);\n    const char* kwd_media(const char* src);\n    const char* kwd_supports_directive(const char* src);\n    // const char* keyframes(const char* src);\n    // const char* keyf(const char* src);\n    const char* kwd_mixin(const char* src);\n    const char* kwd_function(const char* src);\n    const char* kwd_return_directive(const char* src);\n    const char* kwd_include_directive(const char* src);\n    const char* kwd_content_directive(const char* src);\n    const char* kwd_charset_directive(const char* src);\n    const char* kwd_extend(const char* src);\n\n    const char* unicode_seq(const char* src);\n\n    const char* kwd_if_directive(const char* src);\n    const char* kwd_else_directive(const char* src);\n    const char* elseif_directive(const char* src);\n\n    const char* kwd_for_directive(const char* src);\n    const char* kwd_from(const char* src);\n    const char* kwd_to(const char* src);\n    const char* kwd_through(const char* src);\n\n    const char* kwd_each_directive(const char* src);\n    const char* kwd_in(const char* src);\n\n    const char* kwd_while_directive(const char* src);\n\n    const char* re_nothing(const char* src);\n\n    const char* re_special_fun(const char* src);\n\n    const char* kwd_warn(const char* src);\n    const char* kwd_err(const char* src);\n    const char* kwd_dbg(const char* src);\n\n    const char* kwd_null(const char* src);\n\n    const char* re_selector_list(const char* src);\n    const char* re_type_selector(const char* src);\n    const char* re_static_expression(const char* src);\n\n    // identifier that can start with hyphens\n    const char* css_identifier(const char* src);\n    const char* css_ip_identifier(const char* src);\n\n    // Match CSS type selectors\n    const char* namespace_schema(const char* src);\n    const char* namespace_prefix(const char* src);\n    const char* type_selector(const char* src);\n    const char* hyphens_and_identifier(const char* src);\n    const char* hyphens_and_name(const char* src);\n    const char* universal(const char* src);\n    // Match CSS id names.\n    const char* id_name(const char* src);\n    // Match CSS class names.\n    const char* class_name(const char* src);\n    // Attribute name in an attribute selector\n    const char* attribute_name(const char* src);\n    // Match placeholder selectors.\n    const char* placeholder(const char* src);\n    // Match CSS numeric constants.\n    const char* op(const char* src);\n    const char* sign(const char* src);\n    const char* unsigned_number(const char* src);\n    const char* number(const char* src);\n    const char* coefficient(const char* src);\n    const char* binomial(const char* src);\n    const char* percentage(const char* src);\n    const char* ampersand(const char* src);\n    const char* dimension(const char* src);\n    const char* hex(const char* src);\n    const char* hexa(const char* src);\n    const char* hex0(const char* src);\n    // const char* rgb_prefix(const char* src);\n    // Match CSS uri specifiers.\n    const char* uri_prefix(const char* src);\n    // Match CSS \"!important\" keyword.\n    const char* kwd_important(const char* src);\n    // Match CSS \"!optional\" keyword.\n    const char* kwd_optional(const char* src);\n    // Match Sass \"!default\" keyword.\n    const char* default_flag(const char* src);\n    const char* global_flag(const char* src);\n    // Match CSS pseudo-class/element prefixes\n    const char* pseudo_prefix(const char* src);\n    // Match CSS function call openers.\n    const char* re_functional(const char* src);\n    const char* re_pseudo_selector(const char* src);\n    const char* functional_schema(const char* src);\n    const char* pseudo_not(const char* src);\n    // Match CSS 'odd' and 'even' keywords for functional pseudo-classes.\n    const char* even(const char* src);\n    const char* odd(const char* src);\n    // Match CSS attribute-matching operators.\n    const char* exact_match(const char* src);\n    const char* class_match(const char* src);\n    const char* dash_match(const char* src);\n    const char* prefix_match(const char* src);\n    const char* suffix_match(const char* src);\n    const char* substring_match(const char* src);\n    // Match CSS combinators.\n    // const char* adjacent_to(const char* src);\n    // const char* precedes(const char* src);\n    // const char* parent_of(const char* src);\n    // const char* ancestor_of(const char* src);\n\n    // Match SCSS variable names.\n    const char* variable(const char* src);\n    const char* calc_fn_call(const char* src);\n\n    // IE stuff\n    const char* ie_progid(const char* src);\n    const char* ie_expression(const char* src);\n    const char* ie_property(const char* src);\n    const char* ie_keyword_arg(const char* src);\n    const char* ie_keyword_arg_value(const char* src);\n    const char* ie_keyword_arg_property(const char* src);\n\n    // characters that terminate parsing of a list\n    const char* list_terminator(const char* src);\n    const char* space_list_terminator(const char* src);\n\n    // match url()\n    const char* H(const char* src);\n    const char* W(const char* src);\n    // `UNICODE` makes VS sad\n    const char* UUNICODE(const char* src);\n    const char* NONASCII(const char* src);\n    const char* ESCAPE(const char* src);\n    const char* real_uri(const char* src);\n    const char* real_uri_suffix(const char* src);\n    // const char* real_uri_prefix(const char* src);\n    const char* real_uri_value(const char* src);\n\n    // Path matching functions.\n    // const char* folder(const char* src);\n    // const char* folders(const char* src);\n\n\n    const char* static_string(const char* src);\n    const char* static_component(const char* src);\n    const char* static_property(const char* src);\n    const char* static_value(const char* src);\n\n    const char* css_variable_value(const char* src);\n    const char* css_variable_top_level_value(const char* src);\n\n    // Utility functions for finding and counting characters in a string.\n    template<char c>\n    const char* find_first(const char* src) {\n      while (*src && *src != c) ++src;\n      return *src ? src : 0;\n    }\n    template<prelexer mx>\n    const char* find_first(const char* src) {\n      while (*src && !mx(src)) ++src;\n      return *src ? src : 0;\n    }\n    template<prelexer mx>\n    const char* find_first_in_interval(const char* beg, const char* end) {\n      bool esc = false;\n      while ((beg < end) && *beg) {\n        if (esc) esc = false;\n        else if (*beg == '\\\\') esc = true;\n        else if (mx(beg)) return beg;\n        ++beg;\n      }\n      return 0;\n    }\n    template<prelexer mx, prelexer skip>\n    const char* find_first_in_interval(const char* beg, const char* end) {\n      bool esc = false;\n      while ((beg < end) && *beg) {\n        if (esc) esc = false;\n        else if (*beg == '\\\\') esc = true;\n        else if (const char* pos = skip(beg)) beg = pos;\n        else if (mx(beg)) return beg;\n        ++beg;\n      }\n      return 0;\n    }\n    template <prelexer mx>\n    unsigned int count_interval(const char* beg, const char* end) {\n      unsigned int counter = 0;\n      bool esc = false;\n      while (beg < end && *beg) {\n        const char* p;\n        if (esc) {\n          esc = false;\n          ++beg;\n        } else if (*beg == '\\\\') {\n          esc = true;\n          ++beg;\n        } else if ((p = mx(beg))) {\n          ++counter;\n          beg = p;\n        }\n        else {\n          ++beg;\n        }\n      }\n      return counter;\n    }\n\n    template <size_t size, prelexer mx, prelexer pad>\n    const char* padded_token(const char* src)\n    {\n      size_t got = 0;\n      const char* pos = src;\n      while (got < size) {\n        if (!mx(pos)) break;\n        ++ pos; ++ got;\n      }\n      while (got < size) {\n        if (!pad(pos)) break;\n        ++ pos; ++ got;\n      }\n      return got ? pos : 0;\n    }\n\n    template <size_t min, size_t max, prelexer mx>\n    const char* minmax_range(const char* src)\n    {\n      size_t got = 0;\n      const char* pos = src;\n      while (got < max) {\n        if (!mx(pos)) break;\n        ++ pos; ++ got;\n      }\n      if (got < min) return 0;\n      if (got > max) return 0;\n      return pos;\n    }\n\n    template <char min, char max>\n    const char* char_range(const char* src)\n    {\n      if (*src < min) return 0;\n      if (*src > max) return 0;\n      return src + 1;\n    }\n\n  }\n}\n\n#endif\n\n// Path: src/libsass/src/parser.hpp\n#ifndef SASS_PARSER_H\n#define SASS_PARSER_H\n\n#include <string>\n#include <vector>\n\n#include \"ast.hpp\"\n#include \"position.hpp\"\n#include \"context.hpp\"\n#include \"position.hpp\"\n#include \"prelexer.hpp\"\n\n#ifndef MAX_NESTING\n// Note that this limit is not an exact science\n// it depends on various factors, which some are\n// not under our control (compile time or even OS\n// dependent settings on the available stack size)\n// It should fix most common segfault cases though.\n#define MAX_NESTING 512\n#endif\n\nstruct Lookahead {\n  const char* found;\n  const char* error;\n  const char* position;\n  bool parsable;\n  bool has_interpolants;\n  bool is_custom_property;\n};\n\nnamespace Sass {\n\n  class Parser : public ParserState {\n  public:\n\n    enum Scope { Root, Mixin, Function, Media, Control, Properties, Rules, AtRoot };\n\n    Context& ctx;\n    std::vector<Block_Obj> block_stack;\n    std::vector<Scope> stack;\n    Media_Block_Ptr last_media_block;\n    const char* source;\n    const char* position;\n    const char* end;\n    Position before_token;\n    Position after_token;\n    ParserState pstate;\n    Backtraces traces;\n    size_t indentation;\n    size_t nestings;\n\n    Token lexed;\n\n    Parser(Context& ctx, const ParserState& pstate, Backtraces traces)\n    : ParserState(pstate), ctx(ctx), block_stack(), stack(0), last_media_block(),\n      source(0), position(0), end(0), before_token(pstate), after_token(pstate),\n      pstate(pstate), traces(traces), indentation(0), nestings(0)\n    { \n      stack.push_back(Scope::Root);\n    }\n\n    // static Parser from_string(const std::string& src, Context& ctx, ParserState pstate = ParserState(\"[STRING]\"));\n    static Parser from_c_str(const char* src, Context& ctx, Backtraces, ParserState pstate = ParserState(\"[CSTRING]\"), const char* source = 0);\n    static Parser from_c_str(const char* beg, const char* end, Context& ctx, Backtraces, ParserState pstate = ParserState(\"[CSTRING]\"), const char* source = 0);\n    static Parser from_token(Token t, Context& ctx, Backtraces, ParserState pstate = ParserState(\"[TOKEN]\"), const char* source = 0);\n    // special static parsers to convert strings into certain selectors\n    static Selector_List_Obj parse_selector(const char* src, Context& ctx, Backtraces, ParserState pstate = ParserState(\"[SELECTOR]\"), const char* source = 0);\n\n#ifdef __clang__\n\n    // lex and peak uses the template parameter to branch on the action, which\n    // triggers clangs tautological comparison on the single-comparison\n    // branches. This is not a bug, just a merging of behaviour into\n    // one function\n\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wtautological-compare\"\n\n#endif\n\n\n    // skip current token and next whitespace\n    // moves ParserState right before next token\n    void advanceToNextToken();\n\n    bool peek_newline(const char* start = 0);\n\n    // skip over spaces, tabs and line comments\n    template <Prelexer::prelexer mx>\n    const char* sneak(const char* start = 0)\n    {\n      using namespace Prelexer;\n\n      // maybe use optional start position from arguments?\n      const char* it_position = start ? start : position;\n\n      // skip white-space?\n      if (mx == spaces ||\n          mx == no_spaces ||\n          mx == css_comments ||\n          mx == css_whitespace ||\n          mx == optional_spaces ||\n          mx == optional_css_comments ||\n          mx == optional_css_whitespace\n      ) {\n        return it_position;\n      }\n\n      // skip over spaces, tabs and sass line comments\n      const char* pos = optional_css_whitespace(it_position);\n      // always return a valid position\n      return pos ? pos : it_position;\n\n    }\n\n    // match will not skip over space, tabs and line comment\n    // return the position where the lexer match will occur\n    template <Prelexer::prelexer mx>\n    const char* match(const char* start = 0)\n    {\n      // match the given prelexer\n      return mx(position);\n    }\n\n    // peek will only skip over space, tabs and line comment\n    // return the position where the lexer match will occur\n    template <Prelexer::prelexer mx>\n    const char* peek(const char* start = 0)\n    {\n\n      // sneak up to the actual token we want to lex\n      // this should skip over white-space if desired\n      const char* it_before_token = sneak < mx >(start);\n\n      // match the given prelexer\n      const char* match = mx(it_before_token);\n\n      // check if match is in valid range\n      return match <= end ? match : 0;\n\n    }\n\n    // white-space handling is built into the lexer\n    // this way you do not need to parse it yourself\n    // some matchers don't accept certain white-space\n    // we do not support start arg, since we manipulate\n    // sourcemap offset and we modify the position pointer!\n    // lex will only skip over space, tabs and line comment\n    template <Prelexer::prelexer mx>\n    const char* lex(bool lazy = true, bool force = false)\n    {\n\n      if (*position == 0) return 0;\n\n      // position considered before lexed token\n      // we can skip whitespace or comments for\n      // lazy developers (but we need control)\n      const char* it_before_token = position;\n\n      // sneak up to the actual token we want to lex\n      // this should skip over white-space if desired\n      if (lazy) it_before_token = sneak < mx >(position);\n\n      // now call matcher to get position after token\n      const char* it_after_token = mx(it_before_token);\n\n      // check if match is in valid range\n      if (it_after_token > end) return 0;\n\n      // maybe we want to update the parser state anyway?\n      if (force == false) {\n        // assertion that we got a valid match\n        if (it_after_token == 0) return 0;\n        // assertion that we actually lexed something\n        if (it_after_token == it_before_token) return 0;\n      }\n\n      // create new lexed token object (holds the parse results)\n      lexed = Token(position, it_before_token, it_after_token);\n\n      // advance position (add whitespace before current token)\n      before_token = after_token.add(position, it_before_token);\n\n      // update after_token position for current token\n      after_token.add(it_before_token, it_after_token);\n\n      // ToDo: could probably do this incremetal on original object (API wants offset?)\n      pstate = ParserState(path, source, lexed, before_token, after_token - before_token);\n\n      // advance internal char iterator\n      return position = it_after_token;\n\n    }\n\n    // lex_css skips over space, tabs, line and block comment\n    // all block comments will be consumed and thrown away\n    // source-map position will point to token after the comment\n    template <Prelexer::prelexer mx>\n    const char* lex_css()\n    {\n      // copy old token\n      Token prev = lexed;\n      // store previous pointer\n      const char* oldpos = position;\n      Position bt = before_token;\n      Position at = after_token;\n      ParserState op = pstate;\n      // throw away comments\n      // update srcmap position\n      lex < Prelexer::css_comments >();\n      // now lex a new token\n      const char* pos = lex< mx >();\n      // maybe restore prev state\n      if (pos == 0) {\n        pstate = op;\n        lexed = prev;\n        position = oldpos;\n        after_token = at;\n        before_token = bt;\n      }\n      // return match\n      return pos;\n    }\n\n    // all block comments will be skipped and thrown away\n    template <Prelexer::prelexer mx>\n    const char* peek_css(const char* start = 0)\n    {\n      // now peek a token (skip comments first)\n      return peek< mx >(peek < Prelexer::css_comments >(start));\n    }\n\n#ifdef __clang__\n\n#pragma clang diagnostic pop\n\n#endif\n\n    void error(std::string msg);\n    void error(std::string msg, Position pos);\n    // generate message with given and expected sample\n    // text before and in the middle are configurable\n    void css_error(const std::string& msg,\n                   const std::string& prefix = \" after \",\n                   const std::string& middle = \", was: \",\n                   const bool trim = true);\n    void read_bom();\n\n    Block_Obj parse();\n    Import_Obj parse_import();\n    Definition_Obj parse_definition(Definition::Type which_type);\n    Parameters_Obj parse_parameters();\n    Parameter_Obj parse_parameter();\n    Mixin_Call_Obj parse_include_directive();\n    Arguments_Obj parse_arguments();\n    Argument_Obj parse_argument();\n    Assignment_Obj parse_assignment();\n    Ruleset_Obj parse_ruleset(Lookahead lookahead);\n    Selector_List_Obj parse_selector_list(bool chroot);\n    Complex_Selector_Obj parse_complex_selector(bool chroot);\n    Selector_Schema_Obj parse_selector_schema(const char* end_of_selector, bool chroot);\n    Compound_Selector_Obj parse_compound_selector();\n    Simple_Selector_Obj parse_simple_selector();\n    Wrapped_Selector_Obj parse_negated_selector();\n    Simple_Selector_Obj parse_pseudo_selector();\n    Attribute_Selector_Obj parse_attribute_selector();\n    Block_Obj parse_block(bool is_root = false);\n    Block_Obj parse_css_block(bool is_root = false);\n    bool parse_block_nodes(bool is_root = false);\n    bool parse_block_node(bool is_root = false);\n\n    bool parse_number_prefix();\n    Declaration_Obj parse_declaration();\n    Expression_Obj parse_map();\n    Expression_Obj parse_bracket_list();\n    Expression_Obj parse_list(bool delayed = false);\n    Expression_Obj parse_comma_list(bool delayed = false);\n    Expression_Obj parse_space_list();\n    Expression_Obj parse_disjunction();\n    Expression_Obj parse_conjunction();\n    Expression_Obj parse_relation();\n    Expression_Obj parse_expression();\n    Expression_Obj parse_operators();\n    Expression_Obj parse_factor();\n    Expression_Obj parse_value();\n    Function_Call_Obj parse_calc_function();\n    Function_Call_Obj parse_function_call();\n    Function_Call_Schema_Obj parse_function_call_schema();\n    String_Obj parse_url_function_string();\n    String_Obj parse_url_function_argument();\n    String_Obj parse_interpolated_chunk(Token, bool constant = false, bool css = true);\n    String_Obj parse_string();\n    Value_Obj parse_static_value();\n    String_Schema_Obj parse_css_variable_value(bool top_level = true);\n    String_Schema_Obj parse_css_variable_value_token(bool top_level = true);\n    String_Obj parse_ie_property();\n    String_Obj parse_ie_keyword_arg();\n    String_Schema_Obj parse_value_schema(const char* stop);\n    String_Obj parse_identifier_schema();\n    If_Obj parse_if_directive(bool else_if = false);\n    For_Obj parse_for_directive();\n    Each_Obj parse_each_directive();\n    While_Obj parse_while_directive();\n    Return_Obj parse_return_directive();\n    Content_Obj parse_content_directive();\n    void parse_charset_directive();\n    Media_Block_Obj parse_media_block();\n    List_Obj parse_media_queries();\n    Media_Query_Obj parse_media_query();\n    Media_Query_Expression_Obj parse_media_expression();\n    Supports_Block_Obj parse_supports_directive();\n    Supports_Condition_Obj parse_supports_condition();\n    Supports_Condition_Obj parse_supports_negation();\n    Supports_Condition_Obj parse_supports_operator();\n    Supports_Condition_Obj parse_supports_interpolation();\n    Supports_Condition_Obj parse_supports_declaration();\n    Supports_Condition_Obj parse_supports_condition_in_parens();\n    At_Root_Block_Obj parse_at_root_block();\n    At_Root_Query_Obj parse_at_root_query();\n    String_Schema_Obj parse_almost_any_value();\n    Directive_Obj parse_special_directive();\n    Directive_Obj parse_prefixed_directive();\n    Directive_Obj parse_directive();\n    Warning_Obj parse_warning();\n    Error_Obj parse_error();\n    Debug_Obj parse_debug();\n\n    Value_Ptr color_or_string(const std::string& lexed) const;\n\n    // be more like ruby sass\n    Expression_Obj lex_almost_any_value_token();\n    Expression_Obj lex_almost_any_value_chars();\n    Expression_Obj lex_interp_string();\n    Expression_Obj lex_interp_uri();\n    Expression_Obj lex_interpolation();\n\n    // these will throw errors\n    Token lex_variable();\n    Token lex_identifier();\n\n    void parse_block_comments();\n\n    Lookahead lookahead_for_value(const char* start = 0);\n    Lookahead lookahead_for_selector(const char* start = 0);\n    Lookahead lookahead_for_include(const char* start = 0);\n\n    Expression_Obj fold_operands(Expression_Obj base, std::vector<Expression_Obj>& operands, Operand op);\n    Expression_Obj fold_operands(Expression_Obj base, std::vector<Expression_Obj>& operands, std::vector<Operand>& ops, size_t i = 0);\n\n    void throw_syntax_error(std::string message, size_t ln = 0);\n    void throw_read_error(std::string message, size_t ln = 0);\n\n\n    template <Prelexer::prelexer open, Prelexer::prelexer close>\n    Expression_Obj lex_interp()\n    {\n      if (lex < open >(false)) {\n        String_Schema_Obj schema = SASS_MEMORY_NEW(String_Schema, pstate);\n        // std::cerr << \"LEX [[\" << std::string(lexed) << \"]]\\n\";\n        schema->append(SASS_MEMORY_NEW(String_Constant, pstate, lexed));\n        if (position[0] == '#' && position[1] == '{') {\n          Expression_Obj itpl = lex_interpolation();\n          if (!itpl.isNull()) schema->append(itpl);\n          while (lex < close >(false)) {\n            // std::cerr << \"LEX [[\" << std::string(lexed) << \"]]\\n\";\n            schema->append(SASS_MEMORY_NEW(String_Constant, pstate, lexed));\n            if (position[0] == '#' && position[1] == '{') {\n              Expression_Obj itpl = lex_interpolation();\n              if (!itpl.isNull()) schema->append(itpl);\n            } else {\n              return schema;\n            }\n          }\n        } else {\n          return SASS_MEMORY_NEW(String_Constant, pstate, lexed);\n        }\n      }\n      return 0;\n    }\n\n  public:\n    static Number_Ptr lexed_number(const ParserState& pstate, const std::string& parsed);\n    static Number_Ptr lexed_dimension(const ParserState& pstate, const std::string& parsed);\n    static Number_Ptr lexed_percentage(const ParserState& pstate, const std::string& parsed);\n    static Value_Ptr lexed_hex_color(const ParserState& pstate, const std::string& parsed);\n  private:\n    Number_Ptr lexed_number(const std::string& parsed) { return lexed_number(pstate, parsed); };\n    Number_Ptr lexed_dimension(const std::string& parsed) { return lexed_dimension(pstate, parsed); };\n    Number_Ptr lexed_percentage(const std::string& parsed) { return lexed_percentage(pstate, parsed); };\n    Value_Ptr lexed_hex_color(const std::string& parsed) { return lexed_hex_color(pstate, parsed); };\n\n    static const char* re_attr_sensitive_close(const char* src);\n    static const char* re_attr_insensitive_close(const char* src);\n\n  };\n\n  size_t check_bom_chars(const char* src, const char *end, const unsigned char* bom, size_t len);\n}\n\n#endif\n\n// Path: src/libsass/src/node.cpp\n#include \"sass.hpp\"\n#include <vector>\n\n#include \"node.hpp\"\n#include \"context.hpp\"\n#include \"parser.hpp\"\n\nnamespace Sass {\n\n\n  Node Node::createCombinator(const Complex_Selector::Combinator& combinator) {\n    NodeDequePtr null;\n    return Node(COMBINATOR, combinator, NULL /*pSelector*/, null /*pCollection*/);\n  }\n\n\n  Node Node::createSelector(const Complex_Selector& pSelector) {\n    NodeDequePtr null;\n\n    Complex_Selector_Ptr pStripped = SASS_MEMORY_COPY(&pSelector);\n    pStripped->tail(NULL);\n    pStripped->combinator(Complex_Selector::ANCESTOR_OF);\n\n    Node n(SELECTOR, Complex_Selector::ANCESTOR_OF, pStripped, null /*pCollection*/);\n    n.got_line_feed = pSelector.has_line_feed();\n    return n;\n  }\n\n\n  Node Node::createCollection() {\n    NodeDequePtr pEmptyCollection = std::make_shared<NodeDeque>();\n    return Node(COLLECTION, Complex_Selector::ANCESTOR_OF, NULL /*pSelector*/, pEmptyCollection);\n  }\n\n\n  Node Node::createCollection(const NodeDeque& values) {\n    NodeDequePtr pShallowCopiedCollection = std::make_shared<NodeDeque>(values);\n    return Node(COLLECTION, Complex_Selector::ANCESTOR_OF, NULL /*pSelector*/, pShallowCopiedCollection);\n  }\n\n\n  Node Node::createNil() {\n    NodeDequePtr null;\n    return Node(NIL, Complex_Selector::ANCESTOR_OF, NULL /*pSelector*/, null /*pCollection*/);\n  }\n\n\n  Node::Node(const TYPE& type, Complex_Selector::Combinator combinator, Complex_Selector_Ptr pSelector, NodeDequePtr& pCollection)\n  : got_line_feed(false), mType(type), mCombinator(combinator), mpSelector(pSelector), mpCollection(pCollection)\n  { if (pSelector) got_line_feed = pSelector->has_line_feed(); }\n\n\n  Node Node::klone() const {\n    NodeDequePtr pNewCollection = std::make_shared<NodeDeque>();\n    if (mpCollection) {\n      for (NodeDeque::iterator iter = mpCollection->begin(), iterEnd = mpCollection->end(); iter != iterEnd; iter++) {\n        Node& toClone = *iter;\n        pNewCollection->push_back(toClone.klone());\n      }\n    }\n\n    Node n(mType, mCombinator, mpSelector ? SASS_MEMORY_COPY(mpSelector) : NULL, pNewCollection);\n    n.got_line_feed = got_line_feed;\n    return n;\n  }\n\n\n  bool Node::contains(const Node& potentialChild) const {\n    bool found = false;\n\n    for (NodeDeque::iterator iter = mpCollection->begin(), iterEnd = mpCollection->end(); iter != iterEnd; iter++) {\n      Node& toTest = *iter;\n\n      if (toTest == potentialChild) {\n        found = true;\n        break;\n      }\n    }\n\n    return found;\n  }\n\n\n  bool Node::operator==(const Node& rhs) const {\n    if (this->type() != rhs.type()) {\n      return false;\n    }\n\n    if (this->isCombinator()) {\n\n      return this->combinator() == rhs.combinator();\n\n    } else if (this->isNil()) {\n\n      return true; // no state to check\n\n    } else if (this->isSelector()){\n\n      return *this->selector() == *rhs.selector();\n\n    } else if (this->isCollection()) {\n\n      if (this->collection()->size() != rhs.collection()->size()) {\n        return false;\n      }\n\n      for (NodeDeque::iterator lhsIter = this->collection()->begin(), lhsIterEnd = this->collection()->end(),\n           rhsIter = rhs.collection()->begin(); lhsIter != lhsIterEnd; lhsIter++, rhsIter++) {\n\n        if (*lhsIter != *rhsIter) {\n          return false;\n        }\n\n      }\n\n      return true;\n\n    }\n\n    // We shouldn't get here.\n    throw \"Comparing unknown node types. A new type was probably added and this method wasn't implemented for it.\";\n  }\n\n\n  void Node::plus(Node& rhs) {\n    if (!this->isCollection() || !rhs.isCollection()) {\n      throw \"Both the current node and rhs must be collections.\";\n    }\n    this->collection()->insert(this->collection()->end(), rhs.collection()->begin(), rhs.collection()->end());\n  }\n\n#ifdef DEBUG\n  std::ostream& operator<<(std::ostream& os, const Node& node) {\n\n    if (node.isCombinator()) {\n\n      switch (node.combinator()) {\n        case Complex_Selector::ANCESTOR_OF: os << \"\\\" \\\"\"; break;\n        case Complex_Selector::PARENT_OF:   os << \"\\\">\\\"\"; break;\n        case Complex_Selector::PRECEDES:    os << \"\\\"~\\\"\"; break;\n        case Complex_Selector::ADJACENT_TO: os << \"\\\"+\\\"\"; break;\n        case Complex_Selector::REFERENCE: os    << \"\\\"/\\\"\"; break;\n      }\n\n    } else if (node.isNil()) {\n\n      os << \"nil\";\n\n    } else if (node.isSelector()){\n\n      os << node.selector()->head()->to_string();\n\n    } else if (node.isCollection()) {\n\n      os << \"[\";\n\n      for (NodeDeque::iterator iter = node.collection()->begin(), iterBegin = node.collection()->begin(), iterEnd = node.collection()->end(); iter != iterEnd; iter++) {\n        if (iter != iterBegin) {\n          os << \", \";\n        }\n\n        os << (*iter);\n      }\n\n      os << \"]\";\n\n    }\n\n    return os;\n\n  }\n#endif\n\n\n  Node complexSelectorToNode(Complex_Selector_Ptr pToConvert) {\n    if (pToConvert == NULL) {\n      return Node::createNil();\n    }\n    Node node = Node::createCollection();\n    node.got_line_feed = pToConvert->has_line_feed();\n    bool has_lf = pToConvert->has_line_feed();\n\n    // unwrap the selector from parent ref\n    if (pToConvert->head() && pToConvert->head()->has_parent_ref()) {\n      Complex_Selector_Obj tail = pToConvert->tail();\n      if (tail) tail->has_line_feed(pToConvert->has_line_feed());\n      pToConvert = tail;\n    }\n\n    while (pToConvert) {\n\n      bool empty_parent_ref = pToConvert->head() && pToConvert->head()->is_empty_reference();\n\n      // the first Complex_Selector may contain a dummy head pointer, skip it.\n      if (pToConvert->head() && !empty_parent_ref) {\n        node.collection()->push_back(Node::createSelector(*pToConvert));\n        if (has_lf) node.collection()->back().got_line_feed = has_lf;\n        if (pToConvert->head() || empty_parent_ref) {\n          if (pToConvert->tail()) {\n            pToConvert->tail()->has_line_feed(pToConvert->has_line_feed());\n          }\n        }\n        has_lf = false;\n      }\n\n      if (pToConvert->combinator() != Complex_Selector::ANCESTOR_OF) {\n        node.collection()->push_back(Node::createCombinator(pToConvert->combinator()));\n        if (has_lf) node.collection()->back().got_line_feed = has_lf;\n        has_lf = false;\n      }\n\n      if (pToConvert && empty_parent_ref && pToConvert->tail()) {\n        // pToConvert->tail()->has_line_feed(pToConvert->has_line_feed());\n      }\n\n      pToConvert = pToConvert->tail();\n    }\n\n    return node;\n  }\n\n\n  Complex_Selector_Ptr nodeToComplexSelector(const Node& toConvert) {\n    if (toConvert.isNil()) {\n      return NULL;\n    }\n\n\n    if (!toConvert.isCollection()) {\n      throw \"The node to convert to a Complex_Selector_Ptr must be a collection type or nil.\";\n    }\n\n\n    NodeDeque& childNodes = *toConvert.collection();\n\n    std::string noPath(\"\");\n    Complex_Selector_Obj pFirst = SASS_MEMORY_NEW(Complex_Selector, ParserState(\"[NODE]\"), Complex_Selector::ANCESTOR_OF, NULL, NULL);\n\n    Complex_Selector_Obj pCurrent = pFirst;\n\n    if (toConvert.isSelector()) pFirst->has_line_feed(toConvert.got_line_feed);\n    if (toConvert.isCombinator()) pFirst->has_line_feed(toConvert.got_line_feed);\n\n    for (NodeDeque::iterator childIter = childNodes.begin(), childIterEnd = childNodes.end(); childIter != childIterEnd; childIter++) {\n\n      Node& child = *childIter;\n\n      if (child.isSelector()) {\n        // JMA - need to clone the selector, because they can end up getting shared across Node\n        // collections, and can result in an infinite loop during the call to parentSuperselector()\n        pCurrent->tail(SASS_MEMORY_COPY(child.selector()));\n        // if (child.got_line_feed) pCurrent->has_line_feed(child.got_line_feed);\n        pCurrent = pCurrent->tail();\n      } else if (child.isCombinator()) {\n        pCurrent->combinator(child.combinator());\n        if (child.got_line_feed) pCurrent->has_line_feed(child.got_line_feed);\n\n        // if the next node is also a combinator, create another Complex_Selector to hold it so it doesn't replace the current combinator\n        if (childIter+1 != childIterEnd) {\n          Node& nextNode = *(childIter+1);\n          if (nextNode.isCombinator()) {\n            pCurrent->tail(SASS_MEMORY_NEW(Complex_Selector, ParserState(\"[NODE]\"), Complex_Selector::ANCESTOR_OF, NULL, NULL));\n            if (nextNode.got_line_feed) pCurrent->tail()->has_line_feed(nextNode.got_line_feed);\n            pCurrent = pCurrent->tail();\n          }\n        }\n      } else {\n        throw \"The node to convert's children must be only combinators or selectors.\";\n      }\n    }\n\n    // Put the dummy Compound_Selector in the first position, for consistency with the rest of libsass\n    Compound_Selector_Ptr fakeHead = SASS_MEMORY_NEW(Compound_Selector, ParserState(\"[NODE]\"), 1);\n    Parent_Selector_Ptr selectorRef = SASS_MEMORY_NEW(Parent_Selector, ParserState(\"[NODE]\"));\n    fakeHead->elements().push_back(selectorRef);\n    if (toConvert.got_line_feed) pFirst->has_line_feed(toConvert.got_line_feed);\n    // pFirst->has_line_feed(pFirst->has_line_feed() || pFirst->tail()->has_line_feed() || toConvert.got_line_feed);\n    pFirst->head(fakeHead);\n    return SASS_MEMORY_COPY(pFirst);\n  }\n\n  // A very naive trim function, which removes duplicates in a node\n  // This is only used in Complex_Selector::unify_with for now, may need modifications to fit other needs\n  Node Node::naiveTrim(Node& seqses) {\n\n    std::vector<Node*> res;\n    std::vector<Complex_Selector_Obj> known;\n\n    NodeDeque::reverse_iterator seqsesIter = seqses.collection()->rbegin(),\n                                seqsesIterEnd = seqses.collection()->rend();\n\n    for (; seqsesIter != seqsesIterEnd; ++seqsesIter)\n    {\n      Node& seqs1 = *seqsesIter;\n      if( seqs1.isSelector() ) {\n        Complex_Selector_Obj sel = seqs1.selector();\n        std::vector<Complex_Selector_Obj>::iterator it;\n        bool found = false;\n        for (it = known.begin(); it != known.end(); ++it) {\n          if (**it == *sel) { found = true; break; }\n        }\n        if( !found ) {\n          known.push_back(seqs1.selector());\n          res.push_back(&seqs1);\n        }\n      } else {\n        res.push_back(&seqs1);\n      }\n    }\n\n    Node result = Node::createCollection();\n\n    for (size_t i = res.size() - 1; i != std::string::npos; --i) {\n      result.collection()->push_back(*res[i]);\n    }\n\n    return result;\n  }\n}\n\n// Path: src/libsass/src/sass_util.cpp\n#include \"sass.hpp\"\n#include \"node.hpp\"\n\nnamespace Sass {\n\n\n  /*\n    # This is the equivalent of ruby's Sass::Util.paths.\n    #\n    # Return an array of all possible paths through the given arrays.\n    #\n    # @param arrs [NodeCollection<NodeCollection<Node>>]\n    # @return [NodeCollection<NodeCollection<Node>>]\n    #\n    # @example\n    #   paths([[1, 2], [3, 4], [5]]) #=>\n    #     # [[1, 3, 5],\n    #     #  [2, 3, 5],\n    #     #  [1, 4, 5],\n    #     #  [2, 4, 5]]\n\n   The following is the modified version of the ruby code that was more portable to C++. You\n   should be able to drop it into ruby 3.2.19 and get the same results from ruby sass.\n\n    def paths(arrs)\n        // I changed the inject and maps to an iterative approach to make it easier to implement in C++\n      loopStart = [[]]\n\n      for arr in arrs do\n        permutations = []\n        for e in arr do\n          for path in loopStart do\n            permutations.push(path + [e])\n          end\n        end\n        loopStart = permutations\n      end\n    end\n  */\n  Node paths(const Node& arrs) {\n\n    Node loopStart = Node::createCollection();\n    loopStart.collection()->push_back(Node::createCollection());\n\n    for (NodeDeque::iterator arrsIter = arrs.collection()->begin(), arrsEndIter = arrs.collection()->end();\n    \tarrsIter != arrsEndIter; ++arrsIter) {\n\n      Node& arr = *arrsIter;\n\n      Node permutations = Node::createCollection();\n\n      for (NodeDeque::iterator arrIter = arr.collection()->begin(), arrIterEnd = arr.collection()->end();\n      \tarrIter != arrIterEnd; ++arrIter) {\n\n        Node& e = *arrIter;\n\n        for (NodeDeque::iterator loopStartIter = loopStart.collection()->begin(), loopStartIterEnd = loopStart.collection()->end();\n          loopStartIter != loopStartIterEnd; ++loopStartIter) {\n\n          Node& path = *loopStartIter;\n\n          Node newPermutation = Node::createCollection();\n          newPermutation.got_line_feed = arr.got_line_feed;\n          newPermutation.plus(path);\n          newPermutation.collection()->push_back(e);\n\n          permutations.collection()->push_back(newPermutation);\n        }\n      }\n\n      loopStart = permutations;\n    }\n\n    return loopStart;\n  }\n\n\n  /*\n  This is the equivalent of ruby sass' Sass::Util.flatten and [].flatten.\n  Sass::Util.flatten requires the number of levels to flatten, while\n  [].flatten doesn't and will flatten the entire array. This function\n  supports both.\n\n  # Flattens the first `n` nested arrays. If n == -1, all arrays will be flattened\n  #\n  # @param arr [NodeCollection] The array to flatten\n  # @param n [int] The number of levels to flatten\n  # @return [NodeCollection] The flattened array\n\n  The following is the modified version of the ruby code that was more portable to C++. You\n  should be able to drop it into ruby 3.2.19 and get the same results from ruby sass.\n\n  def flatten(arr, n = -1)\n    if n != -1 and n == 0 then\n      return arr\n    end\n\n    flattened = []\n\n    for e in arr do\n      if e.is_a?(Array) then\n        flattened.concat(flatten(e, n - 1))\n      else\n        flattened << e\n      end\n    end\n\n    return flattened\n  end\n  */\n  Node flatten(Node& arr, int n) {\n    if (n != -1 && n == 0) {\n      return arr;\n    }\n\n    Node flattened = Node::createCollection();\n    if (arr.got_line_feed) flattened.got_line_feed = true;\n\n    for (NodeDeque::iterator iter = arr.collection()->begin(), iterEnd = arr.collection()->end();\n    \titer != iterEnd; iter++) {\n    \tNode& e = *iter;\n\n      // e has the lf set\n      if (e.isCollection()) {\n\n      \t// e.collection().got_line_feed = e.got_line_feed;\n      \tNode recurseFlattened = flatten(e, n - 1);\n\n      \tif(e.got_line_feed) {\n      \t\t flattened.got_line_feed = e.got_line_feed;\n      \t  recurseFlattened.got_line_feed = e.got_line_feed;\n      \t}\n\n      \tfor(auto i : (*recurseFlattened.collection())) {\n          if (recurseFlattened.got_line_feed) {\n\n            i.got_line_feed = true;\n          }\n          flattened.collection()->push_back(i);\n      \t}\n\n      } else {\n      \tflattened.collection()->push_back(e);\n      }\n    }\n\n    return flattened;\n  }\n}\n\n// Path: src/libsass/src/lexer.cpp\n#include \"sass.hpp\"\n#include <cctype>\n#include <iostream>\n#include <iomanip>\n#include \"lexer.hpp\"\n#include \"constants.hpp\"\n\n\nnamespace Sass {\n  using namespace Constants;\n\n  namespace Prelexer {\n\n    //####################################\n    // BASIC CHARACTER MATCHERS\n    //####################################\n\n    // Match standard control chars\n    const char* kwd_at(const char* src) { return exactly<'@'>(src); }\n    const char* kwd_dot(const char* src) { return exactly<'.'>(src); }\n    const char* kwd_comma(const char* src) { return exactly<','>(src); };\n    const char* kwd_colon(const char* src) { return exactly<':'>(src); };\n    const char* kwd_star(const char* src) { return exactly<'*'>(src); };\n    const char* kwd_plus(const char* src) { return exactly<'+'>(src); };\n    const char* kwd_minus(const char* src) { return exactly<'-'>(src); };\n    const char* kwd_slash(const char* src) { return exactly<'/'>(src); };\n\n    //####################################\n    // implement some function that do exist in the standard\n    // but those are locale aware which brought some trouble\n    // this even seems to improve performance by quite a bit\n    //####################################\n\n    bool is_alpha(const char& chr)\n    {\n      return unsigned(chr - 'A') <= 'Z' - 'A' ||\n             unsigned(chr - 'a') <= 'z' - 'a';\n    }\n\n    bool is_space(const char& chr)\n    {\n      // adapted the technique from is_alpha\n      return chr == ' ' || unsigned(chr - '\\t') <= '\\r' - '\\t';\n    }\n\n    bool is_digit(const char& chr)\n    {\n      // adapted the technique from is_alpha\n      return unsigned(chr - '0') <= '9' - '0';\n    }\n\n    bool is_number(const char& chr)\n    {\n      // adapted the technique from is_alpha\n      return is_digit(chr) || chr == '-' || chr == '+';\n    }\n\n    bool is_xdigit(const char& chr)\n    {\n      // adapted the technique from is_alpha\n      return unsigned(chr - '0') <= '9' - '0' ||\n             unsigned(chr - 'a') <= 'f' - 'a' ||\n             unsigned(chr - 'A') <= 'F' - 'A';\n    }\n\n    bool is_punct(const char& chr)\n    {\n      // locale independent\n      return chr == '.';\n    }\n\n    \nbool is_alnum(const char& chr)\n    {\n      return is_alpha(chr) || is_digit(chr);\n    }\n\n    // check if char is outside ascii range\n    bool is_unicode(const char& chr)\n    {\n      // check for unicode range\n      return unsigned(chr) > 127;\n    }\n\n    // check if char is outside ascii range\n    // but with specific ranges (copied from Ruby Sass)\n    bool is_nonascii(const char& chr)\n    {\n      unsigned int cmp = unsigned(chr);\n      return (\n        (cmp >= 128 && cmp <= 15572911) ||\n        (cmp >= 15630464 && cmp <= 15712189) ||\n        (cmp >= 4036001920)\n      );\n    }\n\n    // check if char is within a reduced ascii range\n    // valid in a uri (copied from Ruby Sass)\n    bool is_uri_character(const char& chr)\n    {\n      unsigned int cmp = unsigned(chr);\n      return (cmp > 41 && cmp < 127) ||\n             cmp == ':' || cmp == '/';\n    }\n\n    // check if char is within a reduced ascii range\n    // valid for escaping (copied from Ruby Sass)\n    bool is_escapable_character(const char& chr)\n    {\n      unsigned int cmp = unsigned(chr);\n      return cmp > 31 && cmp < 127;\n    }\n\n    // Match word character (look ahead)\n    bool is_character(const char& chr)\n    {\n      // valid alpha, numeric or unicode char (plus hyphen)\n      return is_alnum(chr) || is_unicode(chr) || chr == '-';\n    }\n\n    //####################################\n    // BASIC CLASS MATCHERS\n    //####################################\n\n    // create matchers that advance the position\n    const char* space(const char* src) { return is_space(*src) ? src + 1 : 0; }\n    const char* alpha(const char* src) { return is_alpha(*src) ? src + 1 : 0; }\n    const char* unicode(const char* src) { return is_unicode(*src) ? src + 1 : 0; }\n    const char* nonascii(const char* src) { return is_nonascii(*src) ? src + 1 : 0; }\n    const char* digit(const char* src) { return is_digit(*src) ? src + 1 : 0; }\n    const char* xdigit(const char* src) { return is_xdigit(*src) ? src + 1 : 0; }\n    const char* alnum(const char* src) { return is_alnum(*src) ? src + 1 : 0; }\n    const char* punct(const char* src) { return is_punct(*src) ? src + 1 : 0; }\n    const char* hyphen(const char* src) { return *src && *src == '-' ? src + 1 : 0; }\n    const char* character(const char* src) { return is_character(*src) ? src + 1 : 0; }\n...\n", "needle_token_start": 15278, "needle_token_end": 15311, "code_context_ntokens": 16090, "output": ["```cpp\nbool is_alnum(const char& chr)\n{\n  return is_alpha(chr) || is_digit(chr);\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "vmap_binary_op", "language": "cpp", "path": "mlx/primitives.cpp", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to align two input arrays based on specified axes for vectorized binary operations, ensuring they are compatible for subsequent operations.\n2. **Input**: \n   - A pair of arrays.\n   - A pair of integers representing the axes along which the arrays should be aligned.\n   - A stream object for managing operations.\n3. **Output**: A tuple containing:\n   - The first modified array.\n   - The second modified array, potentially transposed to match the alignment of the first.\n   - The axis along which the alignment was performed.\n4. **Procedure**: \n   - Validates that there are exactly two input arrays and two axes provided.\n   - Determines the new dimensionality needed to accommodate both arrays.\n   - Expands the dimensions of both arrays to match this new dimensionality.\n   - Transposes the second array if necessary to align it with the first array along the specified axes.\n   - Returns the modified arrays and the axis of alignment.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: mlx/random.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <cmath>\n#include <sstream>\n\n#include \"mlx/linalg.h\"\n#include \"mlx/ops.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/random.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core::random {\n\nKeySequence::KeySequence(uint64_t seed) : key_(key(seed)) {}\n\nvoid KeySequence::seed(uint64_t seed) {\n  key_ = key((seed));\n...\n// Path: mlx/device.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include \"mlx/device.h\"\n#include \"mlx/backend/metal/metal.h\"\n\nnamespace mlx::core {\n\nstatic Device default_device_{\n    metal::is_available() ? Device::gpu : Device::cpu};\n\nconst Device& default_device() {\n  return default_device_;\n}\n\nvoid set_default_device(const Device& d) {\n  if (!metal::is_available() && d == Device::gpu) {\n    throw std::invalid_argument(\n        \"[set_default_device] Cannot set gpu device without gpu backend.\");\n  }\n  default_device_ = d;\n}\n\nbool operator==(const Device& lhs, const Device& rhs) {\n  return lhs.type == rhs.type && lhs.index == rhs.index;\n}\n\nbool operator!=(const Device& lhs, const Device& rhs) {\n  return !(lhs == rhs);\n}\n\n} // namespace mlx::core\n\n// Path: mlx/primitives.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n#include <algorithm>\n#include <cassert>\n#include <cmath>\n#include <numeric>\n#include <sstream>\n#include <stdexcept>\n\n#include \"mlx/backend/common/utils.h\"\n#include \"mlx/fft.h\"\n#include \"mlx/ops.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\ns\ntd::tuple<array, array, int> vmap_binary_op(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes,\n    const Stream& stream) {\n  assert(inputs.size() == 2);\n  assert(axes.size() == 2);\n\n  if (axes[0] == -1 && axes[1] == -1) {\n    return {inputs[0], inputs[1], -1};\n  }\n\n  auto a = inputs[0];\n  auto b = inputs[1];\n  int ndim = std::max(a.ndim() + (axes[0] == -1), b.ndim() + (axes[1] == -1));\n\n  auto expand_dims = [stream, ndim](auto in) {\n    auto shape = in.shape();\n    shape.insert(shape.begin(), ndim - shape.size(), 1);\n    return reshape(in, shape, stream);\n  };\n\n  int to_ax = (ndim - a.ndim()) + axes[0];\n  int from_ax = (ndim - b.ndim()) + axes[1];\n  a = expand_dims(a);\n  b = expand_dims(b);\n\n  if (from_ax != to_ax) {\n    std::vector<int> tdims(b.ndim());\n    std::iota(tdims.begin(), tdims.end(), 0);\n    tdims.erase(tdims.begin() + from_ax);\n    tdims.insert(tdims.begin() + to_ax, from_ax);\n    b = transpose(b, tdims, stream);\n  }\n  return {a, b, to_ax};\n}\n\nstd::tuple<array, array, array, int> vmap_ternary_op(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes,\n    const Stream& stream) {\n  assert(inputs.size() == 3);\n  assert(axes.size() == 3);\n\n  if (axes[0] == -1 && axes[1] == -1 && axes[2] == -1) {\n    return {inputs[0], inputs[1], inputs[2], -1};\n  }\n\n  auto a = inputs[0];\n  auto b = inputs[1];\n  auto c = inputs[2];\n  int ndim = std::max(\n      {a.ndim() + (axes[0] == -1),\n       b.ndim() + (axes[1] == -1),\n       c.ndim() + (axes[2] == -1)});\n\n  auto expand_dims = [stream, ndim](auto in) {\n    auto shape = in.shape();\n    shape.insert(shape.begin(), ndim - shape.size(), 1);\n    return reshape(in, shape, stream);\n  };\n\n  int to_ax = (ndim - a.ndim()) + axes[0];\n  int from_ax1 = (ndim - b.ndim()) + axes[1];\n  int from_ax2 = (ndim - c.ndim()) + axes[2];\n  a = expand_dims(a);\n  b = expand_dims(b);\n  c = expand_dims(c);\n\n  auto find_tdims = [](auto x, int to_ax, int from_ax) {\n    std::vector<int> tdims(x.ndim());\n    std::iota(tdims.begin(), tdims.end(), 0);\n    tdims.erase(tdims.begin() + from_ax);\n    tdims.insert(tdims.begin() + to_ax, from_ax);\n    return tdims;\n  };\n\n  if (to_ax != from_ax1) {\n    std::vector<int> tdims = find_tdims(b, to_ax, from_ax1);\n    b = transpose(b, tdims, stream);\n  }\n\n  if (to_ax != from_ax2) {\n    std::vector<int> tdims = find_tdims(c, to_ax, from_ax2);\n    c = transpose(c, tdims, stream);\n  }\n  return {a, b, c, to_ax};\n}\n\n} // namespace\n\nstd::vector<array> Primitive::jvp(\n    const std::vector<array>&,\n    const std::vector<array>&,\n    const std::vector<int>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::jvp] Not implemented for \";\n  print(msg);\n  msg << \".\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::vector<array> Primitive::vjp(\n    const std::vector<array>&,\n    const std::vector<array>&,\n    const std::vector<int>&,\n    const std::vector<array>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::vip] Not implemented for \";\n  print(msg);\n  msg << \".\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::pair<std::vector<array>, std::vector<int>> Primitive::vmap(\n    const std::vector<array>&,\n    const std::vector<int>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::vmap] Not implemented for \";\n  print(msg);\n  msg << \".\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::vector<std::vector<int>> Primitive::output_shapes(\n    const std::vector<array>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::output_shapes] \";\n  this->print(msg);\n  msg << \" cannot infer output shapes.\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::vector<array> Abs::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Abs::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(tangents[0], sign(primals[0], stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Abs::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{abs(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Add::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  return {\n      tangents.size() > 1 ? add(tangents[0], tangents[1], stream())\n                          : tangents[0]};\n}\n\nstd::vector<array> Add::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  if (argnums.size() == 1) {\n    return cotangents;\n  } else {\n    return {cotangents[0], cotangents[0]};\n  }\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Add::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{add(a, b, stream())}, {to_ax}};\n}\n\nstd::vector<array> AddMM::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  auto& cotan = cotangents[0];\n  std::vector<int> reorder(cotan.ndim());\n  std::iota(reorder.begin(), reorder.end(), 0);\n  std::iter_swap(reorder.end() - 1, reorder.end() - 2);\n  for (auto arg : argnums) {\n    if (arg == 0) {\n      // M X N * (K X N).T -> M X K\n      auto cotan_scaled = cotan;\n      if (alpha_ != 1.) {\n        auto alpha_arr = array(alpha_, cotan.dtype());\n        cotan_scaled = (multiply(alpha_arr, cotan_scaled, stream()));\n      }\n      vjps.push_back(matmul(\n          cotan_scaled, transpose(primals[1], reorder, stream()), stream()));\n    } else if (arg == 1) {\n      // (M X K).T * M X N -> K X N\n      auto cotan_scaled = cotan;\n      if (alpha_ != 1.) {\n        auto alpha_arr = array(alpha_, cotan.dtype());\n        cotan_scaled = (multiply(alpha_arr, cotan_scaled, stream()));\n      }\n      vjps.push_back(matmul(\n          transpose(primals[0], reorder, stream()), cotan_scaled, stream()));\n    } else {\n      auto cotan_scaled = cotan;\n      if (beta_ != 1.) {\n        auto beta_arr = array(beta_, cotan.dtype());\n        cotan_scaled = (multiply(beta_arr, cotan_scaled, stream()));\n      }\n      vjps.push_back(cotan_scaled);\n    }\n  }\n  return vjps;\n}\n\nbool AddMM::is_equivalent(const Primitive& other) const {\n  const AddMM& a_other = static_cast<const AddMM&>(other);\n  return (alpha_ == a_other.alpha_ && beta_ == a_other.beta_);\n}\n\nstd::pair<std::vector<array>, std::vector<int>> AddMM::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto maybe_move_ax = [this](auto& arr, auto ax) {\n    return ax > 0 ? moveaxis(arr, ax, 0, stream()) : arr;\n  };\n  auto a = maybe_move_ax(inputs[0], axes[0]);\n  auto b = maybe_move_ax(inputs[1], axes[1]);\n  auto c = maybe_move_ax(inputs[2], axes[2]);\n  return {{addmm(c, a, b, alpha_, beta_, stream())}, {0}};\n}\n\nbool Arange::is_equivalent(const Primitive& other) const {\n  const Arange& a_other = static_cast<const Arange&>(other);\n  return (\n      start_ == a_other.start_ && stop_ == a_other.stop_ &&\n      step_ == a_other.step_);\n}\n\nstd::vector<array> ArcCos::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcCos::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(one, square(primals[0], stream()), stream());\n  array denom = negative(rsqrt(t, stream()), stream());\n  return {multiply(tangents[0], denom, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcCos::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arccos(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcCosh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcCosh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(square(primals[0], stream()), one, stream());\n  return {multiply(tangents[0], rsqrt(t, stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcCosh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arccosh(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcSin::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcSin::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(one, square(primals[0], stream()), stream());\n  return {multiply(tangents[0], rsqrt(t, stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcSin::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arcsin(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcSinh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcSinh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = add(square(primals[0], stream()), one, stream());\n  return {multiply(tangents[0], rsqrt(t, stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcSinh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arcsinh(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcTan::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcTan::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = add(one, square(primals[0], stream()), stream());\n  return {divide(tangents[0], t, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcTan::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arctan(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcTanh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcTanh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(one, square(primals[0], stream()), stream());\n  return {divide(tangents[0], t, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcTanh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arctanh(inputs[0], stream())}, axes};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArgPartition::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n\n  int axis_left = axes[0] >= 0 && axes[0] <= axis_;\n  return {{argpartition(inputs[0], axis_ + axis_left, stream())}, axes};\n}\n\nbool ArgPartition::is_equivalent(const Primitive& other) const {\n  const ArgPartition& r_other = static_cast<const ArgPartition&>(other);\n  return axis_ == r_other.axis_ && kth_ == r_other.kth_;\n}\n\nbool ArgReduce::is_equivalent(const Primitive& other) const {\n  const ArgReduce& r_other = static_cast<const ArgReduce&>(other);\n  return reduce_type_ == r_other.reduce_type_ && axis_ == r_other.axis_;\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArgReduce::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  int reduce_ax = axis_ + (axes[0] >= 0 && axis_ >= axes[0]);\n  auto& in = inputs[0];\n  std::vector<array> out;\n  if (reduce_type_ == ArgReduce::ArgMin) {\n    out.push_back(argmin(in, reduce_ax, true, stream()));\n  } else {\n    out.push_back(argmax(in, reduce_ax, true, stream()));\n  }\n  return {out, axes};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArgSort::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n\n  int axis_left = axes[0] >= 0 && axes[0] <= axis_;\n  return {{argsort(inputs[0], axis_ + axis_left, stream())}, axes};\n}\n\nstd::vector<std::vector<int>> ArgReduce::output_shapes(\n    const std::vector<array>& inputs) {\n  auto out_shape = inputs[0].shape();\n  out_shape[axis_] = 1;\n  return {out_shape};\n}\n\nbool ArgSort::is_equivalent(const Primitive& other) const {\n  const ArgSort& r_other = static_cast<const ArgSort&>(other);\n  return axis_ == r_other.axis_;\n}\n\nstd::vector<array> AsType::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  if (cotangents[0].dtype() != dtype_) {\n    throw std::invalid_argument(\n        \"[astype] Type of cotangentsgent does not much primal output type.\");\n  }\n  return {astype(cotangents[0], primals[0].dtype(), stream())};\n}\n\nstd::vector<array> AsType::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  return {astype(tangents[0], dtype_, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> AsType::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  return {{astype(inputs[0], dtype_, stream())}, axes};\n}\n\nbool AsType::is_equivalent(const Primitive& other) const {\n  const AsType& a_other = static_cast<const AsType&>(other);\n  return dtype_ == a_other.dtype_;\n}\n\nstd::vector<array> AsStrided::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(argnums.size() == 1);\n\n  // Extract the sizes and cast them to ints\n  int grad_size = primals[0].size();\n  int cotangents_size = cotangents[0].size();\n\n  // Make a flat container to hold the gradients\n  auto grad = zeros_like(primals[0], stream());\n  grad = reshape(grad, {grad_size}, stream());\n\n  // Create the indices that map output to input\n  auto idx = arange(grad_size, stream());\n  idx = as_strided(idx, shape_, strides_, offset_, stream());\n  idx = reshape(idx, {cotangents_size}, stream());\n\n  // Reshape the cotangentsgent for use with scatter\n  auto flat_cotangents = reshape(cotangents[0], {cotangents_size, 1}, stream());\n\n  // Finally accumulate the gradients and reshape them to look like the input\n  grad = scatter_add(grad, idx, flat_cotangents, 0, stream());\n  grad = reshape(grad, primals[0].shape(), stream());\n\n  return {grad};\n}\n\nstd::vector<array> AsStrided::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n\n  return {as_strided(tangents[0], shape_, strides_, offset_, stream())};\n}\n\nbool AsStrided::is_equivalent(const Primitive& other) const {\n  const AsStrided& a_other = static_cast<const AsStrided&>(other);\n  return shape_ == a_other.shape_ && strides_ == a_other.strides_ &&\n      offset_ == a_other.offset_;\n}\n\nstd::vector<array> Broadcast::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(argnums.size() == 1);\n\n  // Reduce cotangents to the shape of the primal\n  auto& shape = primals[0].shape();\n  auto& cotan = cotangents[0];\n  int diff = cotan.ndim() - shape.size();\n  std::vector<int> reduce_axes;\n  for (int i = 0; i < cotan.ndim(); ++i) {\n    if (i < diff) {\n      reduce_axes.push_back(i);\n    } else if (shape[i - diff] != cotan.shape(i)) {\n      reduce_axes.push_back(i);\n    }\n  }\n  return {reshape(sum(cotan, reduce_axes, true, stream()), shape, stream())};\n}\n\nstd::vector<array> Broadcast::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(argnums.size() == 1);\n  return {broadcast_to(tangents[0], shape_, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Broadcast::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  auto ax = axes[0];\n  auto in = inputs[0];\n  if (ax >= 0) {\n    auto in_shape = in.shape();\n    int diff = shape_.size() - in.ndim() + 1;\n    assert(diff >= 0);\n    in_shape.insert(in_shape.begin(), diff, 1);\n    ax += diff;\n    shape_.insert(shape_.begin() + ax, in_shape[ax]);\n    in = reshape(in, in_shape, stream());\n  }\n  return {{broadcast_to(in, shape_, stream())}, {ax}};\n}\n\nbool Broadcast::is_equivalent(const Primitive& other) const {\n  const Broadcast& b_other = static_cast<const Broadcast&>(other);\n  return shape_ == b_other.shape_;\n}\n\nstd::vector<array> Ceil::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Ceil::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {zeros_like(primals[0], stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Ceil::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{ceil(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Concatenate::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  auto& cotan = cotangents[0];\n  std::vector<int> start(cotan.ndim(), 0);\n  std::vector<int> stop = cotan.shape();\n\n  std::vector<int> sizes;\n  sizes.push_back(0);\n  for (auto& p : primals) {\n    sizes.push_back(p.shape(axis_));\n  }\n  std::partial_sum(sizes.cbegin(), sizes.cend(), sizes.begin());\n\n  std::vector<array> grads;\n  for (auto i : argnums) {\n    start[axis_] = sizes[i];\n    stop[axis_] = sizes[i + 1];\n    grads.push_back(slice(cotan, start, stop, stream()));\n  }\n  return grads;\n}\n\nstd::vector<array> Concatenate::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  std::vector<int> argidx(argnums.size());\n  std::iota(argidx.begin(), argidx.end(), 0);\n  std::sort(argidx.begin(), argidx.end(), [&argnums](int a, int b) {\n    return argnums[a] < argnums[b];\n  });\n\n  std::vector<array> vals;\n  for (int i = 0, j = 0; i < primals.size(); ++i) {\n    if (j < argnums.size() && argnums[argidx[j]] == i) {\n      vals.push_back(tangents[argidx[j++]]);\n    } else {\n      vals.push_back(zeros_like(primals[i], stream()));\n    }\n  }\n  return {concatenate(vals, axis_, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Concatenate::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  std::vector<array> t_inputs;\n  int out_ax = -1;\n  // Find the first vmapped input\n  int i = 0;\n  for (; i < axes.size(); i++) {\n    t_inputs.push_back(inputs[i]);\n    if (axes[i] >= 0) {\n      out_ax = axes[i];\n      break;\n    }\n  }\n  if (out_ax >= 0) {\n    // Advance to the next input\n    i++;\n\n    // Move vmap axes to the same spot.\n    for (; i < axes.size(); ++i) {\n      if (out_ax != axes[i] && axes[i] >= 0) {\n        t_inputs.push_back(moveaxis(inputs[i], axes[i], out_ax, stream()));\n      } else {\n        t_inputs.push_back(inputs[i]);\n      }\n    }\n  }\n  auto axis = axis_ + (out_ax >= 0 && axis_ >= out_ax);\n  return {{concatenate(t_inputs, axis, stream())}, {out_ax}};\n}\n\nbool Concatenate::is_equivalent(const Primitive& other) const {\n  const Concatenate& c_other = static_cast<const Concatenate&>(other);\n  return axis_ == c_other.axis_;\n}\n\narray conv_weight_backward_patches(\n    const array& in,\n    const array& wt,\n    const array& cotan,\n    const std::vector<int>& kernel_strides,\n    const std::vector<int>& padding,\n    StreamOrDevice s) {\n  // Resolve Padded input shapes and strides\n  std::vector<int> padding_starts(in.ndim(), 0);\n  std::vector<int> padding_ends = in.shape();\n  std::vector<int> in_padded_shape = in.shape();\n\n  // padded shape\n  for (int i = 1; i < in.ndim() - 1; i++) {\n    in_padded_shape[i] += 2 * padding[i - 1];\n    padding_ends[i] += padding[i - 1];\n    padding_starts[i] += padding[i - 1];\n  }\n\n  // padded strides (contiguous)\n  std::vector<size_t> in_padded_strides(in.ndim(), 1);\n  for (int i = in.ndim() - 2; i >= 0; --i) {\n    in_padded_strides[i] = in_padded_strides[i + 1] * in_padded_shape[i + 1];\n  }\n\n  // Pad input\n  std::vector<int> padded_axes(in.ndim() - 2, 0);\n  std::iota(padded_axes.begin(), padded_axes.end(), 1);\n  auto in_padded =\n      pad(in, padded_axes, padding, padding, array(0, in.dtype()), s);\n\n  // Resolve strided patches\n\n  // patches are shaped as\n  // (batch_dim, out_spatial_dims, weight_spatial_dims, in_channels)\n  std::vector<int> patches_shape{\n      cotan.shape().begin(), cotan.shape().end() - 1};\n  patches_shape.insert(\n      patches_shape.end(), wt.shape().begin() + 1, wt.shape().end());\n\n  // Resolve patch strides\n  int n_spatial_dim = in.ndim() - 2;\n  std::vector<size_t> patches_strides(patches_shape.size(), 1);\n  patches_strides[0] = in_padded_strides[0];\n  for (int i = 1; i < n_spatial_dim + 1; i++) {\n    patches_strides[i] = in_padded_strides[i] * kernel_strides[i - 1];\n  }\n  for (int i = 1; i < in.ndim(); i++) {\n    patches_strides[n_spatial_dim + i] = in_padded_strides[i];\n  }\n\n  // Make patches from in\n  auto in_patches = as_strided(in_padded, patches_shape, patches_strides, 0, s);\n\n  // Prepare for matmul\n  int O = wt.shape(0);\n  auto cotan_mat = reshape(cotan, {-1, O}, s);\n  in_patches = reshape(in_patches, {cotan_mat.shape(0), -1}, s);\n\n  auto grad = matmul(transpose(cotan_mat, {1, 0}, s), in_patches, s);\n  grad = reshape(grad, wt.shape(), s);\n  return grad;\n}\n\nstd::vector<array> Convolution::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(primals.size() == 2);\n  std::vector<array> grads;\n\n  // Collect info\n  auto& in = primals[0];\n  auto& wt = primals[1];\n  auto& cotan = cotangents[0];\n\n  for (int a : argnums) {\n    // Grads for input\n    if (a == 0) {\n      std::vector<int> padding_lo = padding_;\n      std::vector<int> padding_hi = padding_;\n\n      for (int i = 0; i < padding_lo.size(); ++i) {\n        int wt_size = 1 + kernel_dilation_[i] * (wt.shape(1 + i) - 1);\n        padding_lo[i] = wt_size - padding_[i] - 1;\n\n        int in_size = 1 + input_dilation_[i] * (in.shape(1 + i) - 1);\n        int out_size = 1 + kernel_strides_[i] * (cotan.shape(1 + i) - 1);\n        padding_hi[i] = in_size - out_size + padding_[i];\n      }\n\n      auto wt_trans = swapaxes(wt, 0, -1, stream());\n\n      auto grad = conv_general(\n          /* const array& input = */ cotan,\n          /* const array& weight = */ wt_trans,\n          /* std::vector<int> stride = */ input_dilation_,\n          /* std::vector<int> padding_lo = */ padding_lo,\n          /* std::vector<int> padding_hi = */ padding_hi,\n          /* std::vector<int> kernel_dilation = */ kernel_dilation_,\n          /* std::vector<int> input_dilation = */ kernel_strides_,\n          /* int groups = */ 1,\n          /* bool flip = */ !flip_,\n          stream());\n\n      grads.push_back(grad);\n    }\n    // Grads for weight\n    else if (a == 1) {\n      bool no_dilation = true;\n\n      for (int i = 0; i < input_dilation_.size(); i++) {\n        no_dilation &= (input_dilation_[i] == 1) && (kernel_dilation_[i] == 1);\n      }\n\n      if (no_dilation) {\n        auto grad = conv_weight_backward_patches(\n            in, wt, cotan, kernel_strides_, padding_, stream());\n        grads.push_back(grad);\n      } else {\n        std::vector<int> padding_lo = padding_;\n        std::vector<int> padding_hi = padding_;\n\n        for (int i = 0; i < padding_hi.size(); ++i) {\n          int in_size = 1 + input_dilation_[i] * (in.shape(1 + i) - 1);\n          int out_size = 1 + kernel_strides_[i] * (cotan.shape(1 + i) - 1);\n          int wt_size = 1 + kernel_dilation_[i] * (wt.shape(1 + i) - 1);\n          padding_hi[i] = out_size - in_size + wt_size - padding_[i] - 1;\n        }\n\n        auto in_trans = swapaxes(in, 0, -1, stream());\n        auto cotan_trans = swapaxes(cotan, 0, -1, stream());\n        auto grad_trans = conv_general(\n            /* const array& input = */ in_trans,\n            /* const array& weight = */ cotan_trans,\n            /* std::vector<int> stride = */ kernel_dilation_,\n            /* std::vector<int> padding_lo = */ padding_lo,\n            /* std::vector<int> padding_hi = */ padding_hi,\n            /* std::vector<int> kernel_dilation = */ kernel_strides_,\n            /* std::vector<int> input_dilation = */ input_dilation_,\n            /* int groups = */ 1,\n            /* bool flip = */ flip_,\n            stream());\n        auto grad = swapaxes(grad_trans, 0, -1, stream());\n        grads.push_back(grad);\n      }\n    }\n  }\n\n  return grads;\n}\n\nbool Convolution::is_equivalent(const Primitive& other) const {\n  const Convolution& c_other = static_cast<const Convolution&>(other);\n  return padding_ == c_other.padding_ &&\n      kernel_strides_ == c_other.kernel_strides_ &&\n      kernel_dilation_ == c_other.kernel_dilation_ &&\n      input_dilation_ == c_other.input_dilation_ &&\n      groups_ == c_other.groups_ && flip_ == c_other.flip_;\n}\n\nstd::vector<array> Copy::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return cotangents;\n}\n\nstd::vector<array> Copy::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return tangents;\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Copy::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{copy(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Cos::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return {jvp(primals, cotangents, argnums)};\n}\n\nstd::vector<array> Cos::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(\n      tangents[0], negative(sin(primals[0], stream()), stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Cos::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{cos(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Cosh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Cosh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(tangents[0], sinh(primals[0], stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Cosh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{cosh(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> CustomVJP::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  std::vector<array> inputs(primals.begin(), primals.end() - outputs.size());\n  auto all_vjps = vjp_fun_(inputs, cotangents, outputs);\n  for (const auto& cot : cotangents) {\n    all_vjps.emplace_back(cot);\n  }\n\n  std::vector<array> vjps;\n  vjps.reserve(argnums.size());\n  for (auto arg : argnums) {\n    vjps.push_back(all_vjps[arg]);\n  }\n\n  return vjps;\n}\n\nstd::vector<array> Depends::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  std::vector<array> vjps;\n\n  for (auto arg : argnums) {\n    if (arg < cotangents.size()) {\n      vjps.push_back(cotangents[arg]);\n    } else {\n      vjps.push_back(zeros_like(primals[arg]));\n    }\n  }\n  return vjps;\n}\n\nstd::vector<array> Divide::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    if (arg == 0) {\n      vjps.push_back(divide(cotangents[0], primals[1], stream()));\n    } else {\n      vjps.push_back(negative(\n          divide(\n              multiply(cotangents[0], primals[0], stream()),\n              square(primals[1], stream()),\n              stream()),\n          stream()));\n    }\n  }\n  return vjps;\n}\n\nstd::vector<array> DivMod::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    vjps.push_back(zeros_like(primals[arg], stream()));\n  }\n  return vjps;\n}\n\nstd::vector<array> DivMod::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  return {zeros_like(primals[0], stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> DivMod::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {divmod(a, b, stream()), {to_ax}};\n}\n\nstd::vector<array> Divide::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  auto jvp_fun = [&](int i) {\n    int arg = argnums[i];\n    if (arg == 0) {\n      return divide(tangents[i], primals[1], stream());\n    } else {\n      return negative(\n          divide(\n              multiply(tangents[i], primals[0], stream()),\n              square(primals[1], stream()),\n              stream()),\n          stream());\n    }\n  };\n  auto out = jvp_fun(0);\n  if (argnums.size() > 1) {\n    out = add(out, jvp_fun(1), stream());\n  }\n  return {out};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Divide::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{divide(a, b, stream())}, {to_ax}};\n}\n\nstd::vector<array> Remainder::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    if (arg == 0) {\n      vjps.push_back(cotangents[0]);\n    } else {\n      auto x_over_y = divide(primals[0], primals[1], stream());\n      x_over_y = floor(x_over_y, stream());\n      vjps.push_back(\n          negative(multiply(x_over_y, cotangents[0], stream()), stream()));\n    }\n  }\n  return vjps;\n}\n\nstd::vector<array> Remainder::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  auto jvp_fun = [&](int i) {\n    int arg = argnums[i];\n    if (arg == 0) {\n      return tangents[i];\n    } else {\n      auto x_over_y = divide(primals[0], primals[1], stream());\n      x_over_y = floor(x_over_y, stream());\n      return negative(multiply(x_over_y, tangents[i], stream()), stream());\n    }\n  };\n  auto out = jvp_fun(0);\n  if (argnums.size() > 1) {\n    out = add(out, jvp_fun(1), stream());\n  }\n  return {out};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Remainder::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{remainder(a, b, stream())}, {to_ax}};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Equal::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{equal(a, b, stream())}, {to_ax}};\n}\n\nstd::vector<array> Equal::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    vjps.push_back(zeros_like(primals[arg], stream()));\n  }\n  return vjps;\n}\n\nstd::vector<array> Equal::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  auto shape = broadcast_shapes(primals[0].shape(), primals[1].shape());\n  return {zeros(shape, bool_, stream())};\n}\n\nstd::vector<array> Erf::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Erf::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  auto dtype = primals[0].dtype();\n  auto scale = multiply(array(M_2_SQRTPI, dtype), tangents[0], stream());\n  return {multiply(\n      scale,\n      exp(negative(square(primals[0], stream()), stream()), stream()),\n      stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Erf::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{erf(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ErfInv::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  auto dtype = primals[0].dtype();\n  auto scale =\n      multiply(array(1.0 / M_2_SQRTPI, dtype), cotangents[0], stream());\n  return {\n      multiply(scale, exp(square(outputs[0], stream()), stream()), stream())};\n}\n\nstd::vector<array> ErfInv::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  auto dtype = primals[0].dtype();\n  auto scale = multiply(array(1.0 / M_2_SQRTPI, dtype), tangents[0], stream());\n  return {multiply(\n      scale,\n      exp(square(erfinv(primals[0], stream()), stream()), stream()),\n      stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ErfInv::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{erfinv(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Exp::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  return {multiply(cotangents[0], outputs[0], stream())};\n}\n\nstd::vector<array> Exp::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(tangents[0], exp(primals[0], stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Exp::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{exp(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Expm1::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  return {multiply(\n      cotangents[0],\n      add(outputs[0], array(1.0f, outputs[0].dtype()), stream()),\n      stream())};\n}\n\nstd::vector<array> Expm1::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(tangents[0], exp(primals[0], stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Expm1::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{expm1(inputs[0], stream())}, axes};\n}\n\nbool FFT::is_equivalent(const Primitive& other) const {\n  const FFT& r_other = static_cast<const FFT&>(other);\n  return axes_ == r_other.axes_ && inverse_ == r_other.inverse_ &&\n      real_ == r_other.real_;\n}\n\nstd::pair<std::vector<array>, std::vector<int>> FFT::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto& in = inputs[0];\n  int ax = axes[0];\n  auto fft_axes = axes_;\n  auto out_shape = in.shape();\n  if (ax >= 0) {\n    for (auto& fft_ax : fft_axes) {\n      if (fft_ax >= ax) {\n        fft_ax++;\n      }\n      if (real_) {\n        auto n = out_shape[fft_ax];\n        out_shape[fft_ax] = inverse_ ? 2 * (n - 1) : n / 2 + 1;\n      }\n    }\n  }\n  return {\n      {array(\n          out_shape,\n          real_ && inverse_ ? float32 : complex64,\n          std::make_shared<FFT>(stream(), fft_axes, inverse_, real_),\n          {in})},\n      {ax}};\n}\n\nstd::vector<array> FFT::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  auto& in = primals[0];\n  std::vector<int> axes(axes_.begin(), axes_.end());\n  if (real_ && inverse_) {\n    auto out = fft::fftn(cotangents[0], axes, stream());\n    auto start = std::vector<int>(out.ndim(), 0);\n    auto stop = in.shape();\n    out = slice(out, start, stop, stream());\n    auto mask_shape = out.shape();\n    mask_shape[axes_.back()] -= 2;\n    auto mask = full(mask_shape, 2.0f, stream());\n    auto pad_shape = out.shape();\n    pad_shape[axes_.back()] = 1;\n    auto pad = full(pad_shape, 1.0f, stream());\n    mask = concatenate({pad, mask, pad}, axes_.back(), stream());\n    return {multiply(mask, out, stream())};\n  } else if (real_) {\n    std::vector<int> n;\n    for (auto ax : axes_) {\n      n.push_back(in.shape()[ax]);\n    }\n    return {astype(\n        fft::fftn(cotangents[0], n, axes, stream()), in.dtype(), stream())};\n  } else if (inverse_) {\n    return {fft::ifftn(cotangents[0], axes, stream())};\n  } else {\n    return {fft::fftn(cotangents[0], axes, stream())};\n  }\n}\n\nstd::vector<array> FFT::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  auto& tan = tangents[0];\n  if (real_ & inverse_) {\n    return {fft::irfftn(tan, stream())};\n  } else if (real_) {\n    return {fft::rfftn(tan, stream())};\n  } else if (inverse_) {\n    return {fft::ifftn(tan, stream())};\n  } else {\n    return {fft::fftn(tan, stream())};\n  }\n}\n\nstd::vector<array> Floor::vjp(\n    const std::vector<array>& primals,\n...\n", "needle_token_start": 585, "needle_token_end": 1018, "code_context_ntokens": 16213, "output": ["```cpp\nstd::tuple<array, array, int> vmap_binary_op(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes,\n    const Stream& stream) {\n  assert(inputs.size() == 2);\n  assert(axes.size() == 2);\n\n  if (axes[0] == -1 && axes[1] == -1) {\n    return {inputs[0], inputs[1], -1};\n  }\n\n  auto a = inputs[0];\n  auto b = inputs[1];\n  int ndim = std::max(a.ndim() + (axes[0] == -1), b.ndim() + (axes[1] == -1));\n\n  auto expand_dims = [stream, ndim](auto in) {\n    auto shape = in.shape();\n    shape.insert(shape.begin(), ndim - shape.size(), 1);\n    return reshape(in, shape, stream);\n  };\n\n  int to_ax = (ndim - a.ndim()) + axes[0];\n  int from_ax = (ndim - b.ndim()) + axes[1];\n  a = expand_dims(a);\n  b = expand_dims(b);\n\n  if (from_ax != to_ax) {\n    std::vector<int> tdims(b.ndim());\n    std::iota(tdims.begin(), tdims.end(), 0);\n    tdims.erase(tdims.begin() + from_ax);\n    tdims.insert(tdims.begin() + to_ax, from_ax);\n    b = transpose(b, tdims, stream);\n  }\n  return {a, b, to_ax};\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "vmap_trace", "language": "cpp", "path": "mlx/transforms.cpp", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to trace a vectorized computation over a batch of inputs, allowing for operations to be applied in parallel across elements of the input arrays specified by axes.\n2. **Input**: This function takes three parameters: a function that operates on vectors of arrays, a vector of arrays representing the inputs to the function, and a vector of integers specifying the axes along which to vectorize each input array.\n3. **Output**: It returns a pair of vectors of arrays. The first vector contains the modified inputs with tracing enabled, and the second vector contains the results of applying the input function to these traced inputs.\n4. **Procedure**: The function first checks that the number of specified axes matches the number of input arrays and validates the specified axes against the dimensions of the input arrays. It then determines the size of the vectorization axis and ensures consistency across all inputs. For each input, if it is to be vectorized, it modifies the shape by removing the vectorization dimension and creates a placeholder input with tracing enabled. These modified inputs are then used to invoke the provided function, capturing the computational graph of the operation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "}\n\nstd::pair<array, array> vjp(\n    const std::function<array(const array&)>& fun,\n    const array& primal,\n    const array& cotan) {\n  auto vec_fun = [fun](const std::vector<array>& inputs) {\n    return std::vector<array>{fun(inputs[0])};\n  };\n  auto [outputs, vjps] = vjp(vec_fun, {primal}, {cotan});\n  return {outputs[0], vjps[0]};\n}\n\nstd::pair<std::vector<array>, std::vector<array>> jvp(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents) {\n  // Set the global tracing flag.\n  detail::InTracing in_tracing;\n\n  if (primals.size() != tangents.size()) {\n    throw std::invalid_argument(\n        \"[jvp] Number of inputs does not match number of tangents.\");\n  }\n  for (int i = 0; i < primals.size(); ++i) {\n    if (primals[i].shape() != tangents[i].shape()) {\n      throw std::invalid_argument(\n          \"[jvp] Input shape does not match shape of tangent.\");\n    }\n  }\n\n  std::vector<array> primals_;\n  for (auto& p : primals) {\n    auto s = p.has_primitive() ? p.primitive().stream()\n                               : default_stream(default_device());\n    primals_.push_back(copy(p, s)); // Does not do a deep copy\n    primals_.back().set_tracer(true);\n  }\n  auto outputs = fun(primals_);\n\n  // Topologically sort the compute graph, record outputs\n  // in the tape if a gradient is needed.\n  std::unordered_set<std::uintptr_t> cache;\n  std::unordered_set<std::uintptr_t> calc_grad;\n  for (auto& primal : primals_) {\n    primal.set_tracer(false);\n    calc_grad.insert(primal.id());\n    cache.insert(primal.id());\n  }\n\n  std::vector<array> tape;\n\n  std::function<void(array&)> recurse;\n  recurse = [&](auto& a) {\n    // Check if visited and add to cache if not\n    if (auto inserted = cache.insert(a.id()); !inserted.second) {\n      return;\n    }\n    a.set_tracer(false);\n    for (auto s : a.siblings()) {\n      s.set_tracer(false);\n      cache.insert(s.id());\n    }\n\n    for (auto input : a.inputs()) {\n      recurse(input);\n    }\n\n    // Stop grad\n    if (a.has_primitive()) {\n      if (auto& p = a.primitive(); typeid(p) == typeid(StopGradient)) {\n        return;\n      }\n    }\n\n    // Calculate gradient if any inputs require gradient\n    for (auto& input : a.inputs()) {\n      if (calc_grad.find(input.id()) != calc_grad.end()) {\n        tape.push_back(a);\n        calc_grad.insert(a.id());\n        for (auto& s : a.siblings()) {\n          calc_grad.insert(s.id());\n        }\n        break;\n      }\n    }\n  };\n\n  for (auto out : outputs) {\n    recurse(out);\n  }\n\n  std::unordered_map<std::uintptr_t, array> tan_map;\n  for (int i = 0; i < primals_.size(); ++i) {\n    tan_map.insert({primals_[i].id(), tangents[i]});\n  }\n\n  for (auto& a : tape) {\n    // Get the arguments used in the jvp\n    std::vector<int> argnums;\n    std::vector<array> tangents;\n    for (int i = 0; i < a.inputs().size(); ++i) {\n      if (auto it = tan_map.find(a.inputs()[i].id()); it != tan_map.end()) {\n        argnums.push_back(i);\n        tangents.push_back(it->second);\n      }\n    }\n\n    auto jvps = a.primitive().jvp(a.inputs(), tangents, argnums);\n    auto outputs = a.outputs();\n    for (int i = 0; i < jvps.size(); ++i) {\n      tan_map.insert({outputs[i].id(), jvps[i]});\n    }\n  }\n\n  std::vector<array> jvps;\n  for (auto& out : outputs) {\n    if (auto it = tan_map.find(out.id()); it != tan_map.end()) {\n      jvps.push_back(it->second);\n    } else {\n      auto s = out.has_primitive() ? out.primitive().stream()\n                                   : default_stream(default_device());\n      jvps.push_back(zeros_like(out, s));\n    }\n  }\n  return {outputs, jvps};\n}\n\nstd::pair<array, array> jvp(\n    const std::function<array(const array&)>& fun,\n    const array& primal,\n    const array& tangent) {\n  auto vec_fun = [fun](const std::vector<array>& inputs) {\n    return std::vector<array>{fun(inputs[0])};\n  };\n  auto [outputs, jvps] = jvp(vec_fun, {primal}, {tangent});\n  return {outputs[0], jvps[0]};\n}\n\nValueAndGradFn value_and_grad(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<int>& argnums) {\n  if (argnums.empty()) {\n    throw std::invalid_argument(\"[grad] Must specify at least one argument.\");\n  }\n  return [fun, argnums](const std::vector<array>& inputs) {\n    std::set<int> args;\n    for (auto& arg : argnums) {\n      args.insert(arg < 0 ? arg + inputs.size() : arg);\n    }\n    if (args.size() != argnums.size()) {\n      throw std::invalid_argument(\n          \"[grad] Repeat argument number not allowed in grad.\");\n    }\n    if (*args.begin() < 0 || *args.rbegin() >= inputs.size()) {\n      std::ostringstream msg;\n      msg << \"[grad] Invalid argument number for function with \"\n          << inputs.size() << \" inputs.\";\n      throw std::invalid_argument(msg.str());\n    }\n\n    auto gfun = [&fun, &inputs, &args](const std::vector<array>& ginputs) {\n      std::vector<array> inputs_(inputs);\n      auto argit = args.begin();\n      for (int i = 0; i < ginputs.size(); ++i) {\n        inputs_[*argit] = ginputs[i];\n        ++argit;\n      }\n      auto outputs = fun(inputs_);\n      for (int i = 1; i < outputs.size(); i++) {\n        auto& out = outputs[i];\n        auto s = out.has_primitive() ? out.primitive().stream()\n                                     : default_stream(default_device());\n        outputs[i] = stop_gradient(out, s);\n      }\n      return outputs;\n    };\n\n    std::vector<array> ginputs;\n    for (auto arg : args) {\n      ginputs.push_back(inputs[arg]);\n    }\n    // Set the incoming gradient to int32, vjp will cast it to the output type\n    auto [outputs, grads] = vjp(gfun, ginputs, {array(1.0f)});\n    return std::make_pair(outputs, grads);\n  };\n}\n\nnamespace detail {\n\ns\ntd::pair<std::vector<array>, std::vector<array>> vmap_trace(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& inputs,\n    const std::vector<int>& in_axes) {\n  // Set the global tracing flag.\n  detail::InTracing in_tracing;\n\n  if (in_axes.size() != inputs.size()) {\n    throw std::invalid_argument(\n        \"[vmap] The number of in axes must match the number of inputs.\");\n  }\n\n  // Some error checking and get the vmap axis size\n  size_t vmap_ax_size;\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      if (inputs[i].ndim() == 0) {\n        throw std::invalid_argument(\n            \"[vmap] Cannot vmap an input with zero dimensions.\");\n      }\n      if (in_axes[i] > inputs[i].ndim()) {\n        std::ostringstream msg;\n        msg << \"[vmap] Axis \" << in_axes[i] << \" invalid for input with \"\n            << inputs[i].ndim() << \" dimensions.\";\n        throw std::invalid_argument(msg.str());\n      }\n      vmap_ax_size = inputs[i].shape(in_axes[i]);\n    }\n  }\n  // Check that all vmapped axes have the same size\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      if (size_t in_ax = inputs[i].shape(in_axes[i]); vmap_ax_size != in_ax) {\n        std::ostringstream msg;\n        msg << \"[vmap] Inconsistent axis sizes: \" << in_ax << \" and \"\n            << vmap_ax_size << \".\";\n        throw std::invalid_argument(msg.str());\n      }\n    }\n  }\n\n  // Run the function on placeholder inputs\n  // to get the original graph\n  std::vector<array> s_inputs;\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      std::vector<int> shape = inputs[i].shape();\n      shape.erase(shape.begin() + in_axes[i]);\n      array in(shape, inputs[i].dtype(), nullptr, {});\n      s_inputs.push_back(in);\n      s_inputs.back().set_tracer(true);\n    } else {\n      s_inputs.push_back(inputs[i]);\n    }\n  }\n  return {s_inputs, fun(s_inputs)};\n}\n\nstd::vector<array> vmap_replace(\n    const std::vector<array>& inputs,\n    const std::vector<array>& s_inputs,\n    const std::vector<array>& s_outputs,\n    const std::vector<int>& in_axes,\n    const std::vector<int>& out_axes) {\n  if (out_axes.size() != s_outputs.size()) {\n    throw std::invalid_argument(\n        \"[vmap] The number of out axes must match the number of outputs.\");\n  }\n\n  std::unordered_map<std::uintptr_t, std::pair<array, int>> tmap;\n  std::unordered_set<std::uintptr_t> needs_vmap;\n  std::unordered_set<std::uintptr_t> cache;\n  for (int i = 0; i < s_inputs.size(); ++i) {\n    auto in = s_inputs[i];\n    if (in_axes[i] != -1) {\n      tmap.insert({in.id(), {inputs[i], in_axes[i]}});\n      needs_vmap.insert(in.id());\n      in.set_tracer(false);\n    }\n    cache.insert(in.id());\n  }\n\n  // Topologically sort the graph\n  std::vector<array> tape;\n\n  std::function<void(const array&)> recurse;\n\n  recurse = [&](const array& a) {\n    auto id = a.id();\n    if (cache.find(id) != cache.end()) {\n      return;\n    }\n    cache.insert(id);\n    for (auto& s : a.siblings()) {\n      cache.insert(s.id());\n    }\n\n    // Recurse on inputs\n    for (auto& input : a.inputs()) {\n      recurse(input);\n    }\n    // If any input needs a vmap, then the outputs also need\n    // a vmap\n    for (auto& input : a.inputs()) {\n      if (needs_vmap.find(input.id()) != needs_vmap.end()) {\n        tape.push_back(a);\n        tape.back().set_tracer(false);\n        needs_vmap.insert(a.id());\n        for (auto s : a.siblings()) {\n          needs_vmap.insert(s.id());\n          s.set_tracer(false);\n        }\n        break;\n      }\n    }\n  };\n\n  for (auto& out : s_outputs) {\n    if (out.has_primitive()) {\n      recurse(out);\n    }\n  }\n\n  // Transform each primitive in the graph with\n  // its vmap implementation\n  for (auto& a : tape) {\n    std::vector<array> v_inputs;\n    std::vector<int> v_axes;\n    for (auto& in : a.inputs()) {\n      auto map_it = tmap.find(in.id());\n      if (map_it != tmap.end()) {\n        v_inputs.push_back(map_it->second.first);\n        v_axes.push_back(map_it->second.second);\n      } else {\n        v_inputs.push_back(in);\n        v_axes.push_back(-1);\n      }\n    }\n\n    auto [v_outputs, v_out_axes] = a.primitive().vmap(v_inputs, v_axes);\n\n    // For each primitive's outputs add its id, the vout id and the vax\n    auto outputs = a.outputs();\n    for (int i = 0; i < v_outputs.size(); ++i) {\n      tmap.insert({outputs[i].id(), {v_outputs[i], v_out_axes[i]}});\n    }\n  }\n\n  // Populate the outputs and make sure all the output axes are\n  // in the right place\n  std::vector<array> outputs;\n  for (int i = 0; i < s_outputs.size(); ++i) {\n    if (auto map_it = tmap.find(s_outputs[i].id()); map_it != tmap.end()) {\n      auto& [out, vdim] = map_it->second;\n      if (vdim != out_axes[i]) {\n        if (out_axes[i] >= out.ndim()) {\n          std::ostringstream msg;\n          msg << \"[vmap] Axis \" << out_axes[i] << \" invalid for output with \"\n              << out.ndim() << \" dimensions.\";\n          throw std::invalid_argument(msg.str());\n        }\n        out = moveaxis(out, vdim, out_axes[i]);\n      }\n      outputs.push_back(out);\n    } else {\n      outputs.push_back(s_outputs[i]);\n    }\n  }\n  return outputs;\n}\n\n} // namespace detail\n\nstd::function<std::vector<array>(const std::vector<array>&)> vmap(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<int>& in_axes /* = {} */,\n    const std::vector<int>& out_axes /* = {} */) {\n  auto infer_axes = [](auto axes) {\n    return !axes.empty() &&\n        std::all_of(axes.begin(), axes.end(), [](int ax) { return ax < 0; });\n  };\n  if (infer_axes(in_axes) != infer_axes(out_axes)) {\n    throw std::invalid_argument(\n        \"[vmap] Input (or output) axes must be \"\n        \"specified if output (or input) axes are.\");\n  }\n  auto vfun = [fun, in_axes = in_axes, out_axes = out_axes](\n                  const std::vector<array>& inputs) mutable {\n    if (in_axes.size() == 0) {\n      in_axes.resize(inputs.size(), 0);\n    }\n\n    auto [trace_inputs, trace_outputs] =\n        detail::vmap_trace(fun, inputs, in_axes);\n\n    if (out_axes.size() == 0) {\n      out_axes.resize(trace_outputs.size(), 0);\n    }\n\n    return detail::vmap_replace(\n        inputs, trace_inputs, trace_outputs, in_axes, out_axes);\n  };\n\n  return vfun;\n}\n\nstd::function<array(const array&, const array&)> vmap(\n    const std::function<array(const array&, const array&)>& fun,\n    int in_axis_a /* = 0 */,\n    int in_axis_b /* = 0 */,\n    int out_axis /* = 0 */) {\n  auto vfun = vmap(\n      [in_axis_a, in_axis_b, out_axis, fun](const std::vector<array>& inputs) {\n        return std::vector<array>{fun(inputs[0], inputs[1])};\n      },\n      {in_axis_a, in_axis_b},\n      {out_axis});\n  return [vfun](const array& a, const array& b) { return vfun({a, b})[0]; };\n}\n\nstd::function<array(const array&)> vmap(\n    const std::function<array(const array&)>& fun,\n    int in_axis /* = 0 */,\n    int out_axis /* = 0 */) {\n  auto vfun = vmap(\n      [in_axis, out_axis, fun](const std::vector<array>& inputs) {\n        return std::vector<array>{fun(inputs[0])};\n      },\n      {in_axis},\n      {out_axis});\n  return [vfun](const array& a) { return vfun({a})[0]; };\n}\n\nstd::function<std::vector<array>(const std::vector<array>&)> custom_vjp(\n    std::function<std::vector<array>(const std::vector<array>&)> fun,\n    std::function<std::vector<array>(\n        const std::vector<array>&,\n        const std::vector<array>&,\n        const std::vector<array>&)> fun_vjp) {\n  return [fun = std::move(fun),\n          fun_vjp = std::move(fun_vjp)](const std::vector<array>& args) {\n    // Compute the outputs\n    auto outputs = fun(args);\n    for (auto& out : outputs) {\n      out = stop_gradient(out);\n    }\n\n    // Prepare the inputs to the primitive\n    // We also add the outputs to the primitive so that it can \"run\" the forward\n    // pass.\n    std::vector<array> inputs = args;\n    inputs.insert(inputs.end(), outputs.begin(), outputs.end());\n\n    // Compute the stream. Maybe do it in a smarter way at some point in the\n    // future.\n    Stream s = (outputs[0].has_primitive()) ? outputs[0].primitive().stream()\n                                            : default_stream(default_device());\n\n    // Make the output info\n    std::vector<std::vector<int>> shapes;\n    std::vector<Dtype> dtypes;\n    for (const auto& out : outputs) {\n      shapes.emplace_back(out.shape());\n      dtypes.emplace_back(out.dtype());\n    }\n\n    return array::make_arrays(\n        std::move(shapes),\n        dtypes,\n        std::make_shared<CustomVJP>(to_stream(s), fun_vjp),\n        inputs);\n  };\n}\n\nstd::function<std::vector<array>(const std::vector<array>&)> checkpoint(\n    std::function<std::vector<array>(const std::vector<array>&)> fun) {\n  auto vjp_fun = [fun](\n                     const std::vector<array>& primals,\n                     const std::vector<array>& cotangents,\n                     const std::vector<array>& outputs) -> std::vector<array> {\n    auto [__, vjps] = vjp(fun, depends(primals, outputs), cotangents);\n    return vjps;\n  };\n\n  return custom_vjp(fun, vjp_fun);\n}\n\n} // namespace mlx::core\n\n// Path: mlx/linalg.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <numeric>\n#include <ostream>\n#include <vector>\n\n#include \"mlx/linalg.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core::linalg {\n\nDtype at_least_float(const Dtype& d) {\n  return issubdtype(d, inexact) ? d : promote_types(d, float32);\n}\n\ninline array l2_norm(\n    const array& a,\n    const std::vector<int>& axis,\n    bool keepdims,\n    StreamOrDevice s) {\n  if (issubdtype(a.dtype(), complexfloating)) {\n    return sqrt(sum(abs(a, s) * abs(a, s), axis, keepdims, s), s);\n  } else {\n    return sqrt(sum(square(a, s), axis, keepdims, s), s);\n  }\n}\n\ninline array vector_norm(\n    const array& a,\n    const double ord,\n    const std::vector<int>& axis,\n    bool keepdims,\n    StreamOrDevice s) {\n  auto dtype = at_least_float(a.dtype());\n  if (ord == 0.0) {\n    return astype(sum(not_equal(a, array(0), s), axis, keepdims, s), dtype, s);\n  } else if (ord == 1.0) {\n    return astype(sum(abs(a, s), axis, keepdims, s), dtype, s);\n  } else if (ord == 2.0) {\n    return l2_norm(a, axis, keepdims, s);\n  } else if (ord == std::numeric_limits<double>::infinity()) {\n    return astype(max(abs(a, s), axis, keepdims, s), dtype, s);\n  } else if (ord == -std::numeric_limits<double>::infinity()) {\n    return astype(min(abs(a, s), axis, keepdims, s), dtype, s);\n  } else {\n    return power(\n        sum(power(abs(a, s), array(ord, dtype), s), axis, keepdims, s),\n        array(1.0 / ord, dtype),\n        s);\n  }\n}\n\ninline array matrix_norm(\n    const array& a,\n    const double ord,\n    const std::vector<int>& axis,\n    bool keepdims,\n    StreamOrDevice s) {\n  auto dtype = at_least_float(a.dtype());\n  auto row_axis = axis[0];\n  auto col_axis = axis[1];\n  if (ord == -1.0) {\n    col_axis -= (!keepdims && col_axis > row_axis && col_axis > 0);\n    return astype(\n        min(sum(abs(a, s), row_axis, keepdims, s), col_axis, keepdims, s),\n        dtype,\n        s);\n  } else if (ord == 1.0) {\n    col_axis -= (!keepdims && col_axis > row_axis && col_axis > 0);\n    return astype(\n        max(sum(abs(a, s), row_axis, keepdims, s), col_axis, keepdims, s),\n        dtype,\n        s);\n  } else if (ord == std::numeric_limits<double>::infinity()) {\n    row_axis -= (!keepdims && row_axis > col_axis && row_axis > 0);\n    return astype(\n        max(sum(abs(a, s), col_axis, keepdims, s), row_axis, keepdims, s),\n        dtype,\n        s);\n  } else if (ord == -std::numeric_limits<double>::infinity()) {\n    row_axis -= (!keepdims && row_axis > col_axis && row_axis > 0);\n    return astype(\n        min(sum(abs(a, s), col_axis, keepdims, s), row_axis, keepdims, s),\n        dtype,\n        s);\n  } else if (ord == 2.0 || ord == -2.0) {\n    throw std::runtime_error(\n        \"[linalg::norm] Singular value norms are not implemented.\");\n  } else {\n    std::ostringstream msg;\n    msg << \"[linalg::norm] Invalid ord \" << ord << \" for matrix norm.\";\n    throw std::invalid_argument(msg.str());\n  }\n}\n\ninline array matrix_norm(\n    const array& a,\n    const std::string& ord,\n    const std::vector<int>& axis,\n    bool keepdims,\n    StreamOrDevice s) {\n  if (ord == \"f\" || ord == \"fro\") {\n    return l2_norm(a, axis, keepdims, s);\n  } else if (ord == \"nuc\") {\n    throw std::runtime_error(\n        \"[linalg::norm] Nuclear norm not yet implemented.\");\n  } else {\n    std::ostringstream msg;\n    msg << \"[linalg::norm] Invalid ord value '\" << ord << \"' for matrix norm.\";\n    throw std::invalid_argument(msg.str());\n  }\n}\n\narray norm(\n    const array& a,\n    const std::optional<std::vector<int>>& axis /* = std::nullopt */,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  if (!axis) {\n    return norm(flatten(a, s), std::vector<int>{0}, keepdims, s);\n  }\n\n  if (axis.value().size() > 2) {\n    throw std::invalid_argument(\n        \"[linalg::norm] Received too many axes for norm.\");\n  }\n  return l2_norm(a, axis.value(), keepdims, s);\n}\n\narray norm(\n    const array& a,\n    const double ord,\n    const std::optional<std::vector<int>>& axis /* = std::nullopt */,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  std::vector<int> ax;\n  if (!axis) {\n    ax.resize(a.ndim());\n    std::iota(ax.begin(), ax.end(), 0);\n  } else {\n    ax = axis.value();\n  }\n  if (ax.size() == 1) {\n    return vector_norm(a, ord, ax, keepdims, s);\n  } else if (ax.size() == 2) {\n    return matrix_norm(a, ord, ax, keepdims, s);\n  } else {\n    throw std::invalid_argument(\n        \"[linalg::norm] Received too many axes for norm.\");\n  }\n}\n\narray norm(\n    const array& a,\n    const std::string& ord,\n    const std::optional<std::vector<int>>& axis /* = std::nullopt */,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  std::vector<int> ax;\n  if (!axis) {\n    ax.resize(a.ndim());\n    std::iota(ax.begin(), ax.end(), 0);\n  } else {\n    ax = axis.value();\n  }\n  if (ax.size() != 2) {\n    std::ostringstream msg;\n    msg << \"[linalg::norm] Norm '\" << ord << \"' only supported for matrices,\"\n        << \" but received \" << ax.size() << \" axis/axes.\";\n    throw std::invalid_argument(msg.str());\n  }\n  return matrix_norm(a, ord, ax, keepdims, s);\n}\n\nstd::pair<array, array> qr(const array& a, StreamOrDevice s /* = {} */) {\n  if (a.dtype() != float32) {\n    std::ostringstream msg;\n    msg << \"[linalg::qr] Arrays must type float32. Received array \"\n        << \"with type \" << a.dtype() << \".\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (a.ndim() < 2) {\n    std::ostringstream msg;\n    msg << \"[linalg::qr] Arrays must have >= 2 dimensions. Received array \"\n           \"with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (a.shape(-1) != a.shape(-2)) {\n    throw std::invalid_argument(\n        \"[linalg::qr] Support for non-square matrices NYI.\");\n  }\n\n  auto out = array::make_arrays(\n      {a.shape(), a.shape()},\n      {a.dtype(), a.dtype()},\n      std::make_shared<QRF>(to_stream(s)),\n      {astype(a, a.dtype(), s)});\n  return std::make_pair(out[0], out[1]);\n}\n\nstd::vector<array> svd(const array& a, StreamOrDevice s /* = {} */) {\n  if (a.dtype() != float32) {\n    std::ostringstream msg;\n    msg << \"[linalg::svd] Input array must have type float32. Received array \"\n        << \"with type \" << a.dtype() << \".\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (a.ndim() < 2) {\n    std::ostringstream msg;\n    msg << \"[linalg::svd] Input array must have >= 2 dimensions. Received array \"\n           \"with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  const auto m = a.shape(-2);\n  const auto n = a.shape(-1);\n  const auto rank = a.ndim();\n\n  std::vector<int> u_shape = a.shape();\n  u_shape[rank - 2] = m;\n  u_shape[rank - 1] = m;\n\n  std::vector<int> s_shape = a.shape();\n  s_shape.pop_back();\n  s_shape[rank - 2] = std::min(m, n);\n\n  std::vector<int> vt_shape = a.shape();\n  vt_shape[rank - 2] = n;\n  vt_shape[rank - 1] = n;\n\n  return array::make_arrays(\n      {u_shape, s_shape, vt_shape},\n      {a.dtype(), a.dtype(), a.dtype()},\n      std::make_shared<SVD>(to_stream(s)),\n      {a});\n}\n\narray inv(const array& a, StreamOrDevice s /* = {} */) {\n  if (a.dtype() != float32) {\n    std::ostringstream msg;\n    msg << \"[linalg::inv] Arrays must type float32. Received array \"\n        << \"with type \" << a.dtype() << \".\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (a.ndim() < 2) {\n    std::ostringstream msg;\n    msg << \"[linalg::inv] Arrays must have >= 2 dimensions. Received array \"\n           \"with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (a.shape(-1) != a.shape(-2)) {\n    throw std::invalid_argument(\n        \"[linalg::inv] Inverses are only defined for square matrices.\");\n  }\n\n  return array(\n      a.shape(), a.dtype(), std::make_shared<Inverse>(to_stream(s)), {a});\n}\n\n} // namespace mlx::core::linalg\n\n// Path: mlx/compile.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <cstdlib>\n#include <map>\n#include <unordered_map>\n#include <unordered_set>\n\n#include \"mlx/allocator.h\"\n#include \"mlx/compile.h\"\n#include \"mlx/compile_impl.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/transforms.h\"\n#include \"mlx/transforms_impl.h\"\n\nnamespace mlx::core {\n\nconstexpr int max_compile_depth = 11;\n\nbool is_unary(const Primitive& p) {\n  return (\n      typeid(p) == typeid(Abs) || typeid(p) == typeid(ArcCos) ||\n      typeid(p) == typeid(ArcCosh) || typeid(p) == typeid(ArcSin) ||\n      typeid(p) == typeid(ArcSinh) || typeid(p) == typeid(ArcTan) ||\n      typeid(p) == typeid(ArcTanh) || typeid(p) == typeid(AsType) ||\n      typeid(p) == typeid(Ceil) || typeid(p) == typeid(Cos) ||\n      typeid(p) == typeid(Cosh) || typeid(p) == typeid(Remainder) ||\n      typeid(p) == typeid(Erf) || typeid(p) == typeid(ErfInv) ||\n      typeid(p) == typeid(Exp) || typeid(p) == typeid(Floor) ||\n      typeid(p) == typeid(Log) || typeid(p) == typeid(Log1p) ||\n      typeid(p) == typeid(LogicalNot) || typeid(p) == typeid(Negative) ||\n      typeid(p) == typeid(Round) || typeid(p) == typeid(Sigmoid) ||\n      typeid(p) == typeid(Sign) || typeid(p) == typeid(Sin) ||\n      typeid(p) == typeid(Sinh) || typeid(p) == typeid(Square) ||\n      typeid(p) == typeid(Sqrt) || typeid(p) == typeid(Tan) ||\n      typeid(p) == typeid(Tanh) || typeid(p) == typeid(Expm1));\n}\n\nbool is_binary(const Primitive& p) {\n  return (\n      typeid(p) == typeid(Add) || typeid(p) == typeid(Divide) ||\n      typeid(p) == typeid(Equal) || typeid(p) == typeid(Greater) ||\n      typeid(p) == typeid(GreaterEqual) || typeid(p) == typeid(Less) ||\n      typeid(p) == typeid(LessEqual) || typeid(p) == typeid(LogicalNot) ||\n      typeid(p) == typeid(LogicalAnd) || typeid(p) == typeid(LogicalOr) ||\n      typeid(p) == typeid(LogAddExp) || typeid(p) == typeid(Maximum) ||\n      typeid(p) == typeid(Minimum) || typeid(p) == typeid(Multiply) ||\n      typeid(p) == typeid(NotEqual) || typeid(p) == typeid(Power) ||\n      typeid(p) == typeid(Subtract));\n}\n\nbool is_ternary(const Primitive& p) {\n  return typeid(p) == typeid(Select);\n}\n\nbool is_broadcast(const Primitive& p) {\n  return typeid(p) == typeid(Broadcast);\n}\n\nbool is_noop(const Primitive& p) {\n  return typeid(p) == typeid(Copy) || typeid(p) == typeid(StopGradient);\n}\n\nbool is_reduction(const Primitive& p) {\n  return typeid(p) == typeid(Reduce) || typeid(p) == typeid(ArgReduce);\n}\n\nbool is_fusable(const Primitive& p) {\n  return is_unary(p) || is_binary(p) || is_ternary(p) || is_broadcast(p) ||\n      is_noop(p);\n}\n\nbool allows_shapeless(const Primitive& p) {\n  return typeid(p) == typeid(Compiled) || is_unary(p) || is_binary(p) ||\n      is_noop(p) || is_reduction(p) || typeid(p) == typeid(Softmax) ||\n      typeid(p) == typeid(Sort) || typeid(p) == typeid(ArgSort) ||\n      typeid(p) == typeid(ArgPartition) || typeid(p) == typeid(Partition) ||\n      typeid(p) == typeid(Select) || typeid(p) == typeid(NumberOfElements);\n}\n\nCompiled::Compiled(\n    Stream stream,\n    std::vector<array> inputs,\n    std::vector<array> outputs,\n    std::vector<array> tape,\n    std::unordered_set<uintptr_t> constant_ids)\n    : Primitive(stream),\n      inputs_(std::move(inputs)),\n      outputs_(std::move(outputs)),\n      tape_(std::move(tape)),\n      constant_ids_(std::move(constant_ids)) {}\n\nstd::vector<array> Compiled::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  throw std::runtime_error(\"[Compiled] Cannot vjp primitive.\");\n}\n\nstd::vector<array> Compiled::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  throw std::runtime_error(\"[Compiled] Cannot jvp primitive.\");\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Compiled::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  throw std::runtime_error(\"[Compiled] Cannot vmap primitive.\");\n}\n\nbool Compiled::is_equivalent(const Primitive& other) const {\n  const Compiled& a_other = static_cast<const Compiled&>(other);\n  return std::equal(\n      tape_.begin(),\n      tape_.end(),\n      a_other.tape_.begin(),\n      a_other.tape_.end(),\n      [](const array& a1, const array& a2) {\n        auto& p1 = a1.primitive();\n        auto& p2 = a2.primitive();\n        return typeid(p1) == typeid(p2) && p1.is_equivalent(p2);\n      });\n}\n\nvoid Compiled::print(std::ostream& os) {\n  os << \"Compiled\";\n  for (auto& a : tape_) {\n    a.primitive().print(os);\n  }\n}\n\nstd::vector<std::vector<int>> Compiled::output_shapes(\n    const std::vector<array>& inputs) {\n  size_t nd = 0;\n  for (auto& in : inputs) {\n    nd = std::max(nd, in.ndim());\n  }\n  std::vector<int> out_shape(nd, 0);\n  for (auto& in : inputs) {\n    auto dd = nd - in.ndim();\n    for (auto i = dd; i < nd; ++i) {\n      out_shape[i] = std::max(out_shape[i], in.shape()[i - dd]);\n    }\n  }\n  // All outputs have the same shape\n  return std::vector<std::vector<int>>(outputs_.size(), out_shape);\n}\n\nnamespace detail {\n\nCompileMode& compile_mode() {\n  auto get_val = []() {\n    if (const char* buff_str = std::getenv(\"MLX_DISABLE_COMPILE\")) {\n      return CompileMode::disabled;\n    } else {\n      return CompileMode::enabled;\n    }\n  };\n  static CompileMode compile_mode_ = get_val();\n  return compile_mode_;\n}\n\nusing ParentsMap =\n    std::unordered_map<std::uintptr_t, std::vector<std::pair<array, int>>>;\n\n// Helper like below but only merges the two provided arrays. If the src has\n// siblings then these won't be merged to the dst.\nvoid merge_one(array& dst, array& src, ParentsMap& parents_map) {\n  auto src_parents = parents_map.find(src.id());\n  if (src_parents == parents_map.end()) {\n    return;\n  }\n  auto& pairs = parents_map[dst.id()];\n  for (auto& parent : src_parents->second) {\n    parent.first.inputs()[parent.second] = dst;\n    pairs.push_back(parent);\n  }\n  // Remove the source from the map to avoid fusing with it again\n  parents_map.erase(src_parents);\n};\n\n// Helper that merges two arrays in the graph by setting the parents of the\n// source to point to the destination. The arrays are assumed to be coming from\n// equivalent primitives so their siblings are merged as well.\nvoid merge(array& dst, array& src, ParentsMap& parents_map) {\n  // Canonicalize the order of the primitives outputs\n  auto sources = src.outputs();\n  auto dests = dst.outputs();\n  // For each src parent, point it to the corresponding dst\n  for (int i = 0; i < sources.size(); ++i) {\n    merge_one(dests[i], sources[i], parents_map);\n  }\n};\n\ntemplate <typename T, typename... U>\nstd::uintptr_t get_function_address(const std::function<T(U...)>& fun) {\n  using FunType = T (*)(U...);\n  const FunType* fun_ptr = fun.template target<FunType>();\n  if (fun_ptr == nullptr) {\n    throw std::invalid_argument(\n        \"[compile] Cannot compile a non-addressable function.\");\n  }\n  return reinterpret_cast<std::uintptr_t>(*fun_ptr);\n}\n\nclass CompilerCache {\n public:\n  struct CacheEntry {\n    std::vector<array> inputs;\n    std::vector<array> outputs;\n    std::vector<array> tape;\n    bool empty{true};\n    std::vector<uint64_t> constants;\n  };\n\n  // Returns a reference to a CacheEntry which can be updated\n  // by the caller to avoid copying large tapes / inputs / outputs\n  CacheEntry& find(\n      std::uintptr_t fun_id,\n      const std::vector<array>& inputs,\n      bool shapeless,\n      const std::vector<uint64_t>& constants) {\n    // Find the cache entries for |fun_id|.\n    std::vector<CacheEntry>& entries = cache_[fun_id];\n    // Compare if 2 arrays have same shape and dtype.\n    auto has_same_shape_and_dtype = [shapeless](\n                                        const std::vector<array>& in1,\n                                        const std::vector<array>& in2) {\n      if (in1.size() != in2.size()) {\n        return false;\n      }\n      for (size_t i = 0; i < in1.size(); ++i) {\n        if (in1[i].ndim() != in2[i].ndim()) {\n          return false;\n        }\n        if (!shapeless && in1[i].shape() != in2[i].shape()) {\n          return false;\n        }\n        if (in1[i].dtype() != in2[i].dtype()) {\n          return false;\n        }\n      }\n      return true;\n    };\n    // Loop over entries and check inputs match i.e. shapes and types must be\n    // equal. Note this could get really slow if one compiles the same\n    // function with many different shapes. May want to store entries in a\n    // more easily searchable structure.\n    for (CacheEntry& entry : entries) {\n      // Check the inputs match and return if so\n      if (has_same_shape_and_dtype(inputs, entry.inputs) &&\n          constants == entry.constants) {\n        return entry;\n      }\n    }\n    // Otherwise append a new cache entry\n    entries.push_back(CacheEntry{});\n    return entries.back();\n  };\n\n  void erase(std::uintptr_t fun_id) {\n    cache_.erase(fun_id);\n  }\n\n private:\n  CompilerCache() {\n    // Make sure the allocator is fully\n    // initialized before the compiler cache\n    allocator::allocator();\n  }\n\n  friend CompilerCache& compiler_cache();\n  std::unordered_map<std::uintptr_t, std::vector<CacheEntry>> cache_;\n};\n\nCompilerCache& compiler_cache() {\n  static CompilerCache compiler_cache_;\n  return compiler_cache_;\n}\n\nstd::pair<std::vector<array>, std::vector<array>> compile_trace(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& inputs) {\n  // Set the global tracing flag.\n  detail::InTracing in_tracing;\n\n  // Run the function on placeholder inputs\n  // to get compute graph\n  std::vector<array> tracer_inputs;\n  for (int i = 0; i < inputs.size(); ++i) {\n    array in(inputs[i].shape(), inputs[i].dtype(), nullptr, {});\n    in.set_tracer(true);\n    tracer_inputs.push_back(std::move(in));\n  }\n  return {tracer_inputs, fun(tracer_inputs)};\n}\n\n// Traverses the graph to build a tape and a map of array ids to their parents\nstd::pair<std::vector<array>, ParentsMap> compile_dfs(\n    const std::vector<array>& inputs,\n    const std::vector<array>& outputs) {\n  std::function<void(const array&)> recurse;\n  std::vector<array> tape;\n  std::unordered_set<std::uintptr_t> input_set;\n  std::unordered_map<std::uintptr_t, std::vector<std::pair<array, int>>>\n      parents_map;\n  for (int i = 0; i < inputs.size(); ++i) {\n    auto in = inputs[i];\n    input_set.insert(in.id());\n  }\n\n  // DFS the graph to build the tape, and log parents and scalars\n  std::unordered_set<std::uintptr_t> cache;\n  recurse = [&](const array& a) {\n    auto id = a.id();\n    if (cache.find(id) != cache.end()) {\n      return;\n    }\n    for (int i = 0; i < a.inputs().size(); i++) {\n      auto& in = a.inputs()[i];\n      parents_map[in.id()].push_back({a, i});\n      for (auto& s : a.siblings()) {\n        parents_map[in.id()].push_back({s, i});\n      }\n      // Don't recurse on inputs (but add them to the tape for the purpose\n      // of future optimizations)\n      if (input_set.find(a.id()) == input_set.end()) {\n        recurse(in);\n      }\n    }\n    cache.insert(id);\n    for (auto& s : a.siblings()) {\n      cache.insert(s.id());\n    }\n    tape.push_back(a);\n  };\n  for (auto& a : outputs) {\n    recurse(a);\n  }\n  return {tape, parents_map};\n}\n\n// Simplify the tape. Note, this function modifies in-place both the tape and\n// the parents map to remove orphaned arrays\nvoid compile_simplify(\n    std::vector<array>& tape,\n    ParentsMap& parents_map,\n    const std::vector<array>& outputs,\n    int passes) {\n  // Helpers to identify identical scalars\n  std::map<std::pair<uint64_t, Dtype::Val>, array> scalars;\n  auto is_scalar = [](const array& a) {\n    return a.is_evaled() && a.ndim() == 0;\n  };\n  auto get_scalar_rep = [](const array& a) {\n    uint64_t v = 0;\n    int dtype;\n    switch (a.dtype().size) {\n      case 1:\n        v = *a.data<uint8_t>();\n        break;\n      case 2:\n        v = *a.data<uint16_t>();\n        break;\n      case 4:\n        v = *a.data<uint32_t>();\n        break;\n      case 8:\n        v = *a.data<uint64_t>();\n        break;\n    }\n    return std::make_pair(v, a.dtype().val);\n  };\n\n  for (auto& a : tape) {\n    if (is_scalar(a)) {\n      scalars.insert({get_scalar_rep(a), a});\n    }\n  }\n\n  // Depth-1 array equivalence check.\n  auto array_equivalent = [](const array& a, const array& b) {\n    if (!a.has_primitive() || !b.has_primitive()) {\n      return false;\n    }\n    if (a.primitive_id() == b.primitive_id()) {\n      return false;\n    }\n    const auto& pa = a.primitive();\n    const auto& pb = b.primitive();\n    if (typeid(pa) != typeid(pb)) {\n      return false;\n    }\n\n    if (a.inputs().size() != b.inputs().size()) {\n      return false;\n    }\n\n    for (int i = 0; i < a.inputs().size(); i++) {\n      if (a.inputs()[i].id() != b.inputs()[i].id()) {\n        return false;\n      }\n    }\n\n    return pa.is_equivalent(pb);\n  };\n\n  // Merge scalars\n  std::vector<array> new_tape;\n  for (auto& arr : tape) {\n    // Check if we can merge scalars\n    if (is_scalar(arr)) {\n      auto scalar = scalars.find(get_scalar_rep(arr));\n      if (scalar->second.id() != arr.id()) {\n        merge(scalar->second, arr, parents_map);\n        // Don't keep orphaned scalars in the tape\n        continue;\n      }\n    }\n    new_tape.push_back(std::move(arr));\n  }\n  tape = std::move(new_tape);\n\n  std::unordered_map<std::uintptr_t, uint32_t> tape_order;\n  for (uint32_t i = 0; i < tape.size(); ++i) {\n    tape_order.insert({tape[i].id(), i});\n  }\n\n  std::unordered_set<uintptr_t> output_set;\n  for (auto& o : outputs) {\n    output_set.insert(o.id());\n  }\n  // Multi-pass merge only keeping non-orphaned arrays in the tape\n  for (int pass = 0; pass < passes; ++pass) {\n    for (auto& arr : tape) {\n      // Helper to check if we can merge the parents of the\n      // given array\n      auto maybe_merge_parents = [&](auto& a) {\n        auto parents = parents_map.find(a.id());\n        if (parents != parents_map.end()) {\n          auto N = parents->second.size();\n          std::vector<bool> mask(N, false);\n          for (int i = 0; i < N; i++) {\n            if (mask[i]) {\n              continue;\n            }\n            for (int j = i + 1; j < N; j++) {\n              if (mask[j]) {\n                continue;\n              }\n              auto src_idx = j;\n              auto dst_idx = i;\n              if (tape_order[parents->second[src_idx].first.id()] <\n                  tape_order[parents->second[dst_idx].first.id()]) {\n                std::swap(src_idx, dst_idx);\n              }\n              auto& src = parents->second[src_idx].first;\n              auto& dst = parents->second[dst_idx].first;\n              if (src.id() != dst.id() && array_equivalent(src, dst) &&\n                  output_set.find(src.id()) == output_set.end()) {\n                merge(dst, src, parents_map);\n                mask[src_idx] = true;\n              }\n            }\n          }\n          // Erase orphaned parents so we don't keep fusing with them\n          for (int i = N - 1; i >= 0; --i) {\n            if (mask[i]) {\n              parents->second.erase(parents->second.begin() + i);\n            }\n          }\n          return false;\n        } else {\n          return output_set.find(a.id()) == output_set.end();\n        }\n      };\n      bool discard = maybe_merge_parents(arr);\n      for (auto& s : arr.siblings()) {\n        discard &= maybe_merge_parents(s);\n      }\n      // If an array and its siblings have no parents, and none of them are\n      // outputs, it is safe to remove it from the tape\n      if (!discard) {\n        new_tape.push_back(std::move(arr));\n      }\n    }\n    tape = std::move(new_tape);\n  }\n}\n\n// Extract sub-graphs of the graph that can be compiled\n// and replace them with a Compiled Primitive.\nvoid compile_fuse(\n    std::vector<array>& tape,\n    ParentsMap& parents_map,\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  // Track outputs to replace with new compiled outputs\n  std::unordered_map<uintptr_t, array> output_map;\n  for (auto& o : outputs) {\n    output_map.insert({o.id(), o});\n  }\n\n  // Set of inputs to distinguish constants\n  std::unordered_set<uintptr_t> input_ids;\n  for (auto& in : inputs) {\n    input_ids.insert(in.id());\n  }\n\n  // Go through the tape in reverse order and check for fusable sub-graphs\n  std::vector<array> new_tape;\n  std::unordered_set<uintptr_t> global_cache;\n  for (int i = tape.size() - 1; i >= 0; --i) {\n    auto& arr = tape[i];\n\n    // Already compiled\n    if (global_cache.find(arr.id()) != global_cache.end()) {\n      continue;\n    }\n\n    // Two pass recursion:\n    // First pass:\n    //  - Collect all the primitives which we can fuse with\n    //  - Keeps a cache of fusable primitives which may be added out of\n    //    DAG order. We have to determine if all of a fused primitive's\n    //    outputs are also in the fused section, and this may not be the\n    //    case the first time we visit it.\n    // Second pass:\n    //  - Collect inputs to the new compiled primitive\n    //  - Add fusable primitives to a tape in the correct order\n\n    std::function<void(\n        const array&, int, const Stream&, const std::vector<int>&)>\n        recurse;\n    std::unordered_set<uintptr_t> cache;\n    recurse = [&](const array& a,\n                  int depth,\n                  const Stream& s,\n                  const std::vector<int>& shape) {\n      if (cache.find(a.id()) != cache.end()) {\n        return;\n      }\n\n      // Stop fusing if:\n      // - Depth limit exceeded\n      // - Constant input\n      // - Stream mismatch\n      // - Non fusable primitive\n      // - Is global output but has a different shape\n      if (depth >= max_compile_depth || !a.has_primitive() ||\n          a.primitive().stream() != s || !is_fusable(a.primitive()) ||\n          (output_map.find(a.id()) != output_map.end() && a.shape() != shape)) {\n        return;\n      }\n\n      bool all_parents_in = true;\n      if (depth > 0) {\n        // Guaranteed to have a parent since nested in the\n        // recursion.\n        auto& parents = parents_map.at(a.id());\n        for (auto& [p, idx] : parents) {\n          auto in_cache = cache.find(p.id()) != cache.end();\n          if (!in_cache) {\n            all_parents_in = false;\n            break;\n          }\n        }\n      }\n\n      // Arrays with a mix of parents outside the compilable section\n      // are not fusable\n      if (!all_parents_in) {\n        return;\n      }\n\n      cache.insert({a.id()});\n\n      for (auto& in : a.inputs()) {\n        recurse(in, depth + 1, s, shape);\n      }\n    };\n\n    if (arr.has_primitive()) {\n      Stream s = arr.primitive().stream();\n      recurse(arr, 0, s, arr.shape());\n    }\n\n    // Not worth fusing a single primitive\n    if (cache.size() <= 1) {\n      new_tape.push_back(arr);\n      continue;\n    }\n\n    // Recurse a second time to build the tape in the right\n    // order and collect the inputs\n    std::unordered_set<uintptr_t> input_set;\n    std::vector<array> inputs;\n    std::vector<array> fused_tape;\n    std::unordered_set<uintptr_t> tape_set;\n    std::function<void(const array&)> recurse_tape;\n    recurse_tape = [&](const array& a) {\n      if (cache.find(a.id()) == cache.end()) {\n        if (input_set.find(a.id()) == input_set.end()) {\n          input_set.insert(a.id());\n          inputs.push_back(a);\n        }\n        return;\n      }\n      if (tape_set.find(a.id()) != tape_set.end()) {\n        return;\n      }\n      tape_set.insert(a.id());\n      for (auto& in : a.inputs()) {\n        recurse_tape(in);\n      }\n      fused_tape.push_back(a);\n    };\n    recurse_tape(arr);\n\n    std::vector<array> old_outputs;\n    // Add to global cache and add any global outputs to outputs\n    // of new primitive\n    for (int j = 0; j < fused_tape.size() - 1; ++j) {\n      auto& f = fused_tape[j];\n      if (output_map.find(f.id()) != output_map.end()) {\n        old_outputs.push_back(f);\n        // Parents are now siblings, update the parent map\n        auto& pairs = parents_map[f.id()];\n        pairs.erase(\n            std::remove_if(\n                pairs.begin(),\n                pairs.end(),\n                [&](auto& p) {\n                  return cache.find(p.first.id()) != cache.end();\n                }),\n            pairs.end());\n      } else {\n        // Remove inner fused arrays parents from the parents map\n        // to keep the parents map in a valid state\n        parents_map.erase(f.id());\n      }\n      global_cache.insert({f.id()});\n    }\n    old_outputs.push_back(arr);\n\n    std::vector<std::vector<int>> shapes;\n    std::vector<Dtype> types;\n    for (auto& o : old_outputs) {\n      if (o.shape() != old_outputs.back().shape()) {\n        throw std::runtime_error(\n            \"[compile] Compilation failed. Tried to fuse operations with different output shapes\");\n      }\n      shapes.push_back(o.shape());\n      types.push_back(o.dtype());\n    }\n    std::unordered_set<uintptr_t> constant_ids;\n    for (auto& in : inputs) {\n      // Scalar constant\n      if (in.size() == 1 && !in.has_primitive() &&\n          input_ids.find(in.id()) == input_ids.end()) {\n        constant_ids.insert(in.id());\n      }\n    }\n    auto compiled_outputs = array::make_arrays(\n        std::move(shapes),\n        types,\n        std::make_shared<Compiled>(\n            old_outputs.back().primitive().stream(),\n            inputs,\n            old_outputs,\n            std::move(fused_tape),\n            std::move(constant_ids)),\n        inputs);\n\n    // One output per primitive\n    new_tape.push_back(compiled_outputs.back());\n\n    // Replace inputs old parents with compiled_outputs\n    for (int i = 0; i < inputs.size(); ++i) {\n      auto& pairs = parents_map[inputs[i].id()];\n      pairs.erase(\n          std::remove_if(\n              pairs.begin(),\n              pairs.end(),\n              [&](auto& p) { return cache.find(p.first.id()) != cache.end(); }),\n          pairs.end());\n      for (auto& o : compiled_outputs) {\n        pairs.push_back({o, i});\n      }\n    }\n\n    // - Update outputs parents to point to compiled outputs\n    // - Update any overall graph outputs to be compiled outputs\n    for (int o = 0; o < old_outputs.size(); ++o) {\n      merge_one(compiled_outputs[o], old_outputs[o], parents_map);\n      if (auto it = output_map.find(old_outputs[o].id());\n          it != output_map.end()) {\n        it->second = compiled_outputs[o];\n      }\n    }\n  }\n\n...\n", "needle_token_start": 2074, "needle_token_end": 2788, "code_context_ntokens": 16226, "output": ["```cpp\nstd::pair<std::vector<array>, std::vector<array>> vmap_trace(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& inputs,\n    const std::vector<int>& in_axes) {\n  // Set the global tracing flag.\n  detail::InTracing in_tracing;\n\n  if (in_axes.size() != inputs.size()) {\n    throw std::invalid_argument(\n        \"[vmap] The number of in axes must match the number of inputs.\");\n  }\n\n  // Some error checking and get the vmap axis size\n  size_t vmap_ax_size;\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      if (inputs[i].ndim() == 0) {\n        throw std::invalid_argument(\n            \"[vmap] Cannot vmap an input with zero dimensions.\");\n      }\n      if (in_axes[i] > inputs[i].ndim()) {\n        std::ostringstream msg;\n        msg << \"[vmap] Axis \" << in_axes[i] << \" invalid for input with \"\n            << inputs[i].ndim() << \" dimensions.\";\n        throw std::invalid_argument(msg.str());\n      }\n      vmap_ax_size = inputs[i].shape(in_axes[i]);\n    }\n  }\n  // Check that all vmapped axes have the same size\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      if (size_t in_ax = inputs[i].shape(in_axes[i]); vmap_ax_size != in_ax) {\n        std::ostringstream msg;\n        msg << \"[vmap] Inconsistent axis sizes: \" << in_ax << \" and \"\n            << vmap_ax_size << \".\";\n        throw std::invalid_argument(msg.str());\n      }\n    }\n  }\n\n  // Run the function on placeholder inputs\n  // to get the original graph\n  std::vector<array> s_inputs;\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      std::vector<int> shape = inputs[i].shape();\n      shape.erase(shape.begin() + in_axes[i]);\n      array in(shape, inputs[i].dtype(), nullptr, {});\n      s_inputs.push_back(in);\n      s_inputs.back().set_tracer(true);\n    } else {\n      s_inputs.push_back(inputs[i]);\n    }\n  }\n  return {s_inputs, fun(s_inputs)};\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "in_tracing", "language": "cpp", "path": "mlx/array.cpp", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To determine if the system is currently in a state of performing transformations on functions specifically for maintaining computational graphs during the evaluation of tracer arrays.\n2. **Input**: None; the function does not take any parameters.\n3. **Output**: Returns a boolean value; true if the system is in the process of function transformation for graph retention, otherwise false.\n4. **Procedure**: The function internally accesses a state-checking utility from a detailed namespace to ascertain whether the current operational mode involves function transformation aimed at graph preservation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: mlx/array.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <functional>\n\n#include \"mlx/array.h\"\n#include \"mlx/ops.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/transforms.h\"\n#include \"mlx/transforms_impl.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\n/** Return true if we are currently performing a function transformation in\n * order to keep the graph when evaluating tracer arrays. */\nb\nool in_tracing() {\n  return detail::InTracing::in_tracing();\n}\n\n} // namespace\n\narray::array(const std::complex<float>& val, Dtype dtype /* = complex64 */)\n    : array_desc_(std::make_shared<ArrayDesc>(std::vector<int>{}, dtype)) {\n  auto cval = static_cast<complex64_t>(val);\n  init(&cval);\n}\n\narray::array(\n    std::vector<int> shape,\n    Dtype dtype,\n    std::shared_ptr<Primitive> primitive,\n    std::vector<array> inputs)\n    : array_desc_(std::make_shared<ArrayDesc>(\n          std::move(shape),\n          dtype,\n          std::move(primitive),\n          std::move(inputs))) {}\n\nstd::vector<array> array::make_arrays(\n    std::vector<std::vector<int>> shapes,\n    const std::vector<Dtype>& dtypes,\n    const std::shared_ptr<Primitive>& primitive,\n    const std::vector<array>& inputs) {\n  std::vector<array> outputs;\n  for (size_t i = 0; i < shapes.size(); ++i) {\n    outputs.emplace_back(std::move(shapes[i]), dtypes[i], primitive, inputs);\n  }\n  // For each node in |outputs|, its siblings are the other nodes.\n  for (size_t i = 0; i < outputs.size(); ++i) {\n    auto siblings = outputs;\n    siblings.erase(siblings.begin() + i);\n    outputs[i].set_siblings(std::move(siblings), i);\n  }\n  return outputs;\n}\n\narray::array(std::initializer_list<float> data)\n    : array_desc_(std::make_shared<ArrayDesc>(\n          std::vector<int>{static_cast<int>(data.size())},\n          float32)) {\n  init(data.begin());\n}\n\narray::array(std::initializer_list<int> data, Dtype dtype)\n    : array_desc_(std::make_shared<ArrayDesc>(\n          std::vector<int>{static_cast<int>(data.size())},\n          dtype)) {\n  init(data.begin());\n}\n\n/* Build an array from a shared buffer */\narray::array(\n    allocator::Buffer data,\n    std::vector<int> shape,\n    Dtype dtype,\n    deleter_t deleter)\n    : array_desc_(std::make_shared<ArrayDesc>(std::move(shape), dtype)) {\n  set_data(data, deleter);\n}\n\nvoid array::detach() {\n  for (auto& s : array_desc_->siblings) {\n    s.array_desc_->inputs.clear();\n    s.array_desc_->siblings.clear();\n    s.array_desc_->position = 0;\n    s.array_desc_->primitive = nullptr;\n  }\n  array_desc_->inputs.clear();\n  array_desc_->siblings.clear();\n  array_desc_->position = 0;\n  array_desc_->primitive = nullptr;\n}\n\nvoid array::eval() {\n  if (!is_evaled()) {\n    mlx::core::eval({*this});\n  }\n}\n\nbool array::is_tracer() const {\n  return array_desc_->is_tracer && in_tracing();\n}\n\nvoid array::set_data(allocator::Buffer buffer, deleter_t d) {\n  array_desc_->data = std::make_shared<Data>(buffer, d);\n  array_desc_->data_ptr = buffer.raw_ptr();\n  array_desc_->data_size = size();\n  array_desc_->flags.contiguous = true;\n  array_desc_->flags.row_contiguous = true;\n  auto max_dim = std::max_element(shape().begin(), shape().end());\n  array_desc_->flags.col_contiguous = size() <= 1 || size() == *max_dim;\n}\n\nvoid array::set_data(\n    allocator::Buffer buffer,\n    size_t data_size,\n    std::vector<size_t> strides,\n    Flags flags,\n    deleter_t d) {\n  array_desc_->data = std::make_shared<Data>(buffer, d);\n  array_desc_->data_ptr = buffer.raw_ptr();\n  array_desc_->data_size = data_size;\n  array_desc_->strides = std::move(strides);\n  array_desc_->flags = flags;\n}\n\nvoid array::copy_shared_buffer(\n    const array& other,\n    const std::vector<size_t>& strides,\n    Flags flags,\n    size_t data_size,\n    size_t offset /* = 0 */) {\n  array_desc_->data = other.array_desc_->data;\n  array_desc_->strides = strides;\n  array_desc_->flags = flags;\n  array_desc_->data_size = data_size;\n  auto char_offset = sizeof(char) * itemsize() * offset;\n  array_desc_->data_ptr = static_cast<void*>(\n      static_cast<char*>(other.array_desc_->data_ptr) + char_offset);\n}\n\nvoid array::copy_shared_buffer(const array& other) {\n  copy_shared_buffer(other, other.strides(), other.flags(), other.data_size());\n}\n\nvoid array::move_shared_buffer(\n    array other,\n    const std::vector<size_t>& strides,\n    Flags flags,\n    size_t data_size,\n    size_t offset /* = 0 */) {\n  array_desc_->data = std::move(other.array_desc_->data);\n  array_desc_->strides = strides;\n  array_desc_->flags = flags;\n  array_desc_->data_size = data_size;\n  auto char_offset = sizeof(char) * itemsize() * offset;\n  array_desc_->data_ptr = static_cast<void*>(\n      static_cast<char*>(other.array_desc_->data_ptr) + char_offset);\n}\n\nvoid array::move_shared_buffer(array other) {\n  move_shared_buffer(other, other.strides(), other.flags(), other.data_size());\n}\n\nvoid array::ArrayDesc::init() {\n  strides.resize(shape.size());\n  size = 1;\n  for (int i = shape.size() - 1; i >= 0; --i) {\n    strides[i] = size;\n    size *= shape[i];\n  }\n  for (auto& in : inputs) {\n    is_tracer |= in.is_tracer();\n  }\n}\n\narray::ArrayDesc::ArrayDesc(std::vector<int> shape, Dtype dtype)\n    : shape(std::move(shape)), dtype(dtype) {\n  init();\n}\n\narray::ArrayDesc::ArrayDesc(\n    std::vector<int> shape,\n    Dtype dtype,\n    std::shared_ptr<Primitive> primitive,\n    std::vector<array> inputs)\n    : shape(std::move(shape)),\n      dtype(dtype),\n      primitive(std::move(primitive)),\n      inputs(std::move(inputs)) {\n  init();\n}\n\narray::ArrayDesc::~ArrayDesc() {\n  // When an array description is destroyed it will delete a bunch of arrays\n  // that may also destory their corresponding descriptions and so on and so\n  // forth.\n  //\n  // This calls recursively the destructor and can result in stack overflow, we\n  // instead put them in a vector and destroy them one at a time resulting in a\n  // max stack depth of 2.\n  std::vector<std::shared_ptr<ArrayDesc>> for_deletion;\n\n  for (array& a : inputs) {\n    if (a.array_desc_.use_count() == 1) {\n      for_deletion.push_back(std::move(a.array_desc_));\n    }\n  }\n\n  while (!for_deletion.empty()) {\n    // top is going to be deleted at the end of the block *after* the arrays\n    // with inputs have been moved into the vector\n    auto top = std::move(for_deletion.back());\n    for_deletion.pop_back();\n\n    for (array& a : top->inputs) {\n      if (a.array_desc_.use_count() == 1) {\n        for_deletion.push_back(std::move(a.array_desc_));\n      }\n    }\n  }\n}\n\narray::ArrayIterator::ArrayIterator(const array& arr, int idx)\n    : arr(arr), idx(idx) {\n  if (arr.ndim() == 0) {\n    throw std::invalid_argument(\"Cannot iterate over 0-d array.\");\n  }\n}\n\narray::ArrayIterator::reference array::ArrayIterator::operator*() const {\n  auto start = std::vector<int>(arr.ndim(), 0);\n  auto end = arr.shape();\n  auto shape = arr.shape();\n  shape.erase(shape.begin());\n  start[0] = idx;\n  end[0] = idx + 1;\n  return reshape(slice(arr, start, end), shape);\n};\n\n} // namespace mlx::core\n\n// Path: mlx/transforms.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n#include <algorithm>\n#include <future>\n#include <numeric>\n#include <set>\n#include <sstream>\n#include <stack>\n#include <unordered_map>\n#include <unordered_set>\n\n#include \"mlx/backend/metal/metal_impl.h\"\n#include \"mlx/ops.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/scheduler.h\"\n#include \"mlx/transforms.h\"\n#include \"mlx/transforms_impl.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core {\n\n/* This class is only meant to be used in eval\n * for synchronizing with the main thread. */\nclass Synchronizer : public Primitive {\n public:\n  explicit Synchronizer(Stream stream) : Primitive(stream){};\n\n  void eval_cpu(const std::vector<array>&, std::vector<array>&) override {};\n  void eval_gpu(const std::vector<array>&, std::vector<array>&) override {};\n\n  DEFINE_PRINT(Synchronize);\n};\n\n// Initialize the static tracing counter from transforms_impl.h .\n//\n// This is used to implement the in_tracing() function the returns true if we\n// are currently under a function transformation.\nint detail::InTracing::tracing_counter{0};\n\nstd::shared_future<void> async_eval(std::vector<array> outputs) {\n  static std::shared_future<void> global_synchronizer;\n  // Catch up with previous async eval if needed\n  if (global_synchronizer.valid()) {\n    global_synchronizer.wait();\n  }\n  std::queue<array> tape;\n  std::unordered_set<std::uintptr_t> cache;\n  std::unordered_map<std::uintptr_t, std::shared_future<void>> deps;\n\n  // Make an effort to choose a good output stream\n  Stream stream = default_stream(default_device());\n  for (auto& o : outputs) {\n    if (!o.is_evaled() && o.has_primitive()) {\n      stream = o.primitive().stream();\n      break;\n    }\n  }\n\n  auto synchronizer = array(\n      {}, bool_, std::make_shared<Synchronizer>(stream), std::move(outputs));\n\n  {\n    std::stack<std::pair<std::reference_wrapper<array>, int>> dfs;\n    dfs.emplace(synchronizer, 0);\n    while (!dfs.empty()) {\n      auto& [a_ref, idx] = dfs.top();\n      auto& a = a_ref.get();\n      if (idx < a.inputs().size()) {\n        // Add an input, and continue\n        auto& in = a.inputs()[idx++];\n        if (!in.is_evaled()) {\n          if (!in.has_primitive()) {\n            throw std::invalid_argument(\n                \"[eval] Attempting to eval an array without a primitive.\");\n          }\n\n          // If the input is being computed on a different stream, we need to\n          // manage the dependency.\n          if (a.primitive().stream() != in.primitive().stream()) {\n            deps.insert({in.output(0).id(), std::shared_future<void>{}});\n          }\n        }\n\n        if (cache.find(in.id()) == cache.end()) {\n          dfs.emplace(in, 0);\n          cache.insert(in.id());\n          for (auto& s : in.siblings()) {\n            cache.insert(s.id());\n          }\n        }\n        continue;\n      }\n\n      // All inputs are done being processed, process this array\n      if (!a.is_evaled() || (!a.is_tracer() && a.has_primitive())) {\n        tape.push(a);\n      }\n      dfs.pop();\n    }\n  }\n  deps.insert({synchronizer.id(), std::shared_future<void>{}});\n\n  std::vector<std::shared_ptr<std::promise<void>>> ps;\n  while (!tape.empty()) {\n    auto arr = std::move(tape.front());\n    tape.pop();\n    if (arr.is_evaled()) {\n      if (!arr.is_tracer() && arr.has_primitive()) {\n        arr.detach();\n      }\n      continue;\n    }\n\n    auto stream = arr.primitive().stream();\n    std::vector<std::shared_future<void>> arr_deps;\n    for (auto& in : arr.inputs()) {\n      if (auto it = deps.find(in.output(0).id()); it != deps.end()) {\n        arr_deps.push_back(it->second);\n      }\n    }\n    std::shared_ptr<std::promise<void>> p;\n    if (auto it = deps.find(arr.output(0).id()); it != deps.end()) {\n      p = std::make_shared<std::promise<void>>();\n      ps.push_back(p);\n      it->second = p->get_future().share();\n    }\n\n    if (arr.primitive().device() == Device::gpu) {\n      if (!metal::is_available()) {\n        throw std::runtime_error(\"Metal GPU is not available.\");\n      }\n      scheduler::enqueue(\n          stream, metal::make_task(arr, std::move(arr_deps), std::move(p)));\n    } else {\n      auto task = [arr,\n                   stream,\n                   deps = std::move(arr_deps),\n                   p = std::move(p)]() mutable {\n        for (auto& d : deps) {\n          d.wait();\n        }\n        scheduler::notify_new_task(stream);\n        auto outputs = arr.outputs();\n        arr.primitive().eval_cpu(arr.inputs(), outputs);\n        if (!arr.is_tracer()) {\n          arr.detach();\n        }\n        if (p) {\n          p->set_value();\n        }\n        scheduler::notify_task_completion(stream);\n      };\n      scheduler::enqueue(stream, std::move(task));\n    }\n  }\n  global_synchronizer = std::move(deps[synchronizer.id()]);\n  return global_synchronizer;\n}\n\nvoid eval(std::vector<array> outputs) {\n  async_eval(std::move(outputs)).wait();\n}\n\nstd::pair<std::vector<array>, std::vector<array>> vjp(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& primals,\n    const std::vector<array>& cotans) {\n  // Set the global tracing flag.\n  detail::InTracing in_tracing;\n\n  // Make tracers from given primals\n  std::vector<array> primals_;\n  for (auto& p : primals) {\n    auto s = p.has_primitive() ? p.primitive().stream()\n                               : default_stream(default_device());\n    primals_.push_back(copy(p, s)); // Does not do a deep copy\n    primals_.back().set_tracer(true);\n  }\n\n  // Pass tracer primals through the function\n  // Any variables that depend on the primals are marked as tracers\n  auto outputs = fun(primals_);\n\n  // Map outputs to passed cotans while ignoring the outputs\n  // that have stop_gradient called on them\n  int cotan_index = 0;\n  std::vector<std::pair<int, int>> output_cotan_pairs;\n  for (int i = 0; i < outputs.size(); ++i) {\n    auto& out = outputs[i];\n    if (out.has_primitive()) {\n      if (auto& p = out.primitive(); typeid(p) == typeid(StopGradient)) {\n        continue;\n      }\n    }\n    if (cotan_index >= cotans.size()) {\n      std::ostringstream msg;\n      msg << \"[vjp] Number of outputs to compute gradients for (\"\n          << outputs.size() << \") does not match number of cotangents (\"\n          << cotans.size() << \").\";\n      throw std::invalid_argument(msg.str());\n    }\n    if (out.shape() != cotans[cotan_index].shape()) {\n      std::ostringstream msg;\n      msg << \"[vjp] Output shape \" << out.shape()\n          << \" does not match cotangent shape \" << cotans[cotan_index].shape()\n          << \".\";\n      if (outputs.size() == 1 && out.size() == 1) {\n        msg << \" If you are using grad your function must return a scalar.\";\n      }\n      throw std::invalid_argument(msg.str());\n    }\n    output_cotan_pairs.emplace_back(i, cotan_index++);\n  }\n\n  // Topologically sort the compute graph, add graph nodes\n  // to the tape which need a gradient.\n  std::unordered_set<std::uintptr_t> cache;\n  std::unordered_set<std::uintptr_t> calc_grad;\n  for (auto& primal : primals_) {\n    primal.set_tracer(false);\n    calc_grad.insert(primal.id());\n    cache.insert(primal.id());\n  }\n\n  std::vector<array> tape;\n\n  std::function<void(array&)> recurse;\n  recurse = [&](auto& a) {\n    // Check if visited and add to cache if not\n    if (auto inserted = cache.insert(a.id()); !inserted.second) {\n      return;\n    }\n    a.set_tracer(false);\n    for (auto s : a.siblings()) {\n      s.set_tracer(false);\n      cache.insert(s.id());\n    }\n\n    for (auto& input : a.inputs()) {\n      recurse(input);\n    }\n\n    // Stop grad\n    if (a.has_primitive()) {\n      if (auto& p = a.primitive(); typeid(p) == typeid(StopGradient)) {\n        return;\n      }\n    }\n\n    // Calculate gradient if any inputs require gradient\n    for (auto& input : a.inputs()) {\n      if (calc_grad.find(input.id()) != calc_grad.end()) {\n        tape.push_back(a);\n        calc_grad.insert(a.id());\n        for (auto& s : a.siblings()) {\n          calc_grad.insert(s.id());\n        }\n        break;\n      }\n    }\n  };\n\n  for (auto out : outputs) {\n    recurse(out);\n  }\n\n  // Run the tape backwards, computing vector-jacobian\n  // products for each primitive\n  std::unordered_map<std::uintptr_t, array> cotan_map;\n  for (auto [out_idx, cotan_idx] : output_cotan_pairs) {\n    auto& o = outputs[out_idx];\n    auto s = o.has_primitive() ? o.primitive().stream()\n                               : default_stream(default_device());\n    cotan_map.insert({o.id(), astype(cotans[cotan_idx], o.dtype(), s)});\n  }\n  for (auto it = tape.rbegin(); it != tape.rend(); ++it) {\n    auto& a = *it;\n\n    // Get the arguments whose gradients are needed\n    std::vector<int> argnums;\n    for (int i = 0; i < a.inputs().size(); ++i) {\n      if (calc_grad.find(a.inputs()[i].id()) != calc_grad.end()) {\n        argnums.push_back(i);\n      }\n    }\n\n    // Check if any of the array or its siblings have cotangents,\n    // if not, we can skip this primitive\n    auto outputs = a.outputs();\n    bool has_cotans =\n        std::any_of(outputs.cbegin(), outputs.cend(), [&cotan_map](auto& s) {\n          return cotan_map.find(s.id()) != cotan_map.end();\n        });\n    if (!has_cotans) {\n      continue;\n    }\n\n    auto s = a.primitive().stream();\n    std::vector<array> cotangents{};\n    for (auto& o : outputs) {\n      if (auto cotan_it = cotan_map.find(o.id()); cotan_it != cotan_map.end()) {\n        cotangents.push_back(cotan_map.extract(cotan_it).mapped());\n      } else {\n        cotangents.push_back(zeros_like(o, s));\n      }\n    }\n\n    auto vjps = a.primitive().vjp(a.inputs(), cotangents, argnums, outputs);\n    // Accumulate the vector-jacobian products for each input\n    for (int i = 0; i < argnums.size(); ++i) {\n      auto in_id = a.inputs()[argnums[i]].id();\n      if (auto cotan_it = cotan_map.find(in_id); cotan_it != cotan_map.end()) {\n        cotan_it->second = add(cotan_it->second, vjps[i], s);\n      } else {\n        cotan_map.insert({in_id, vjps[i]});\n      }\n    }\n  }\n\n  std::vector<array> vjps;\n  for (auto& primal : primals_) {\n    if (auto cotan_it = cotan_map.find(primal.id());\n        cotan_it != cotan_map.end()) {\n      vjps.push_back(cotan_it->second);\n    } else {\n      auto s = primal.has_primitive() ? primal.primitive().stream()\n                                      : default_stream(default_device());\n      vjps.push_back(zeros_like(primal, s));\n    }\n  }\n  return {outputs, vjps};\n}\n\nstd::pair<array, array> vjp(\n    const std::function<array(const array&)>& fun,\n    const array& primal,\n    const array& cotan) {\n  auto vec_fun = [fun](const std::vector<array>& inputs) {\n    return std::vector<array>{fun(inputs[0])};\n  };\n  auto [outputs, vjps] = vjp(vec_fun, {primal}, {cotan});\n  return {outputs[0], vjps[0]};\n}\n\nstd::pair<std::vector<array>, std::vector<array>> jvp(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents) {\n  // Set the global tracing flag.\n  detail::InTracing in_tracing;\n\n  if (primals.size() != tangents.size()) {\n    throw std::invalid_argument(\n        \"[jvp] Number of inputs does not match number of tangents.\");\n  }\n  for (int i = 0; i < primals.size(); ++i) {\n    if (primals[i].shape() != tangents[i].shape()) {\n      throw std::invalid_argument(\n          \"[jvp] Input shape does not match shape of tangent.\");\n    }\n  }\n\n  std::vector<array> primals_;\n  for (auto& p : primals) {\n    auto s = p.has_primitive() ? p.primitive().stream()\n                               : default_stream(default_device());\n    primals_.push_back(copy(p, s)); // Does not do a deep copy\n    primals_.back().set_tracer(true);\n  }\n  auto outputs = fun(primals_);\n\n  // Topologically sort the compute graph, record outputs\n  // in the tape if a gradient is needed.\n  std::unordered_set<std::uintptr_t> cache;\n  std::unordered_set<std::uintptr_t> calc_grad;\n  for (auto& primal : primals_) {\n    primal.set_tracer(false);\n    calc_grad.insert(primal.id());\n    cache.insert(primal.id());\n  }\n\n  std::vector<array> tape;\n\n  std::function<void(array&)> recurse;\n  recurse = [&](auto& a) {\n    // Check if visited and add to cache if not\n    if (auto inserted = cache.insert(a.id()); !inserted.second) {\n      return;\n    }\n    a.set_tracer(false);\n    for (auto s : a.siblings()) {\n      s.set_tracer(false);\n      cache.insert(s.id());\n    }\n\n    for (auto input : a.inputs()) {\n      recurse(input);\n    }\n\n    // Stop grad\n    if (a.has_primitive()) {\n      if (auto& p = a.primitive(); typeid(p) == typeid(StopGradient)) {\n        return;\n      }\n    }\n\n    // Calculate gradient if any inputs require gradient\n    for (auto& input : a.inputs()) {\n      if (calc_grad.find(input.id()) != calc_grad.end()) {\n        tape.push_back(a);\n        calc_grad.insert(a.id());\n        for (auto& s : a.siblings()) {\n          calc_grad.insert(s.id());\n        }\n        break;\n      }\n    }\n  };\n\n  for (auto out : outputs) {\n    recurse(out);\n  }\n\n  std::unordered_map<std::uintptr_t, array> tan_map;\n  for (int i = 0; i < primals_.size(); ++i) {\n    tan_map.insert({primals_[i].id(), tangents[i]});\n  }\n\n  for (auto& a : tape) {\n    // Get the arguments used in the jvp\n    std::vector<int> argnums;\n    std::vector<array> tangents;\n    for (int i = 0; i < a.inputs().size(); ++i) {\n      if (auto it = tan_map.find(a.inputs()[i].id()); it != tan_map.end()) {\n        argnums.push_back(i);\n        tangents.push_back(it->second);\n      }\n    }\n\n    auto jvps = a.primitive().jvp(a.inputs(), tangents, argnums);\n    auto outputs = a.outputs();\n    for (int i = 0; i < jvps.size(); ++i) {\n      tan_map.insert({outputs[i].id(), jvps[i]});\n    }\n  }\n\n  std::vector<array> jvps;\n  for (auto& out : outputs) {\n    if (auto it = tan_map.find(out.id()); it != tan_map.end()) {\n      jvps.push_back(it->second);\n    } else {\n      auto s = out.has_primitive() ? out.primitive().stream()\n                                   : default_stream(default_device());\n      jvps.push_back(zeros_like(out, s));\n    }\n  }\n  return {outputs, jvps};\n}\n\nstd::pair<array, array> jvp(\n    const std::function<array(const array&)>& fun,\n    const array& primal,\n    const array& tangent) {\n  auto vec_fun = [fun](const std::vector<array>& inputs) {\n    return std::vector<array>{fun(inputs[0])};\n  };\n  auto [outputs, jvps] = jvp(vec_fun, {primal}, {tangent});\n  return {outputs[0], jvps[0]};\n}\n\nValueAndGradFn value_and_grad(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<int>& argnums) {\n  if (argnums.empty()) {\n    throw std::invalid_argument(\"[grad] Must specify at least one argument.\");\n  }\n  return [fun, argnums](const std::vector<array>& inputs) {\n    std::set<int> args;\n    for (auto& arg : argnums) {\n      args.insert(arg < 0 ? arg + inputs.size() : arg);\n    }\n    if (args.size() != argnums.size()) {\n      throw std::invalid_argument(\n          \"[grad] Repeat argument number not allowed in grad.\");\n    }\n    if (*args.begin() < 0 || *args.rbegin() >= inputs.size()) {\n      std::ostringstream msg;\n      msg << \"[grad] Invalid argument number for function with \"\n          << inputs.size() << \" inputs.\";\n      throw std::invalid_argument(msg.str());\n    }\n\n    auto gfun = [&fun, &inputs, &args](const std::vector<array>& ginputs) {\n      std::vector<array> inputs_(inputs);\n      auto argit = args.begin();\n      for (int i = 0; i < ginputs.size(); ++i) {\n        inputs_[*argit] = ginputs[i];\n        ++argit;\n      }\n      auto outputs = fun(inputs_);\n      for (int i = 1; i < outputs.size(); i++) {\n        auto& out = outputs[i];\n        auto s = out.has_primitive() ? out.primitive().stream()\n                                     : default_stream(default_device());\n        outputs[i] = stop_gradient(out, s);\n      }\n      return outputs;\n    };\n\n    std::vector<array> ginputs;\n    for (auto arg : args) {\n      ginputs.push_back(inputs[arg]);\n    }\n    // Set the incoming gradient to int32, vjp will cast it to the output type\n    auto [outputs, grads] = vjp(gfun, ginputs, {array(1.0f)});\n    return std::make_pair(outputs, grads);\n  };\n}\n\nnamespace detail {\n\nstd::pair<std::vector<array>, std::vector<array>> vmap_trace(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& inputs,\n    const std::vector<int>& in_axes) {\n  // Set the global tracing flag.\n  detail::InTracing in_tracing;\n\n  if (in_axes.size() != inputs.size()) {\n    throw std::invalid_argument(\n        \"[vmap] The number of in axes must match the number of inputs.\");\n  }\n\n  // Some error checking and get the vmap axis size\n  size_t vmap_ax_size;\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      if (inputs[i].ndim() == 0) {\n        throw std::invalid_argument(\n            \"[vmap] Cannot vmap an input with zero dimensions.\");\n      }\n      if (in_axes[i] > inputs[i].ndim()) {\n        std::ostringstream msg;\n        msg << \"[vmap] Axis \" << in_axes[i] << \" invalid for input with \"\n            << inputs[i].ndim() << \" dimensions.\";\n        throw std::invalid_argument(msg.str());\n      }\n      vmap_ax_size = inputs[i].shape(in_axes[i]);\n    }\n  }\n  // Check that all vmapped axes have the same size\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      if (size_t in_ax = inputs[i].shape(in_axes[i]); vmap_ax_size != in_ax) {\n        std::ostringstream msg;\n        msg << \"[vmap] Inconsistent axis sizes: \" << in_ax << \" and \"\n            << vmap_ax_size << \".\";\n        throw std::invalid_argument(msg.str());\n      }\n    }\n  }\n\n  // Run the function on placeholder inputs\n  // to get the original graph\n  std::vector<array> s_inputs;\n  for (int i = 0; i < inputs.size(); ++i) {\n    if (in_axes[i] != -1) {\n      std::vector<int> shape = inputs[i].shape();\n      shape.erase(shape.begin() + in_axes[i]);\n      array in(shape, inputs[i].dtype(), nullptr, {});\n      s_inputs.push_back(in);\n      s_inputs.back().set_tracer(true);\n    } else {\n      s_inputs.push_back(inputs[i]);\n    }\n  }\n  return {s_inputs, fun(s_inputs)};\n}\n\nstd::vector<array> vmap_replace(\n    const std::vector<array>& inputs,\n    const std::vector<array>& s_inputs,\n    const std::vector<array>& s_outputs,\n    const std::vector<int>& in_axes,\n    const std::vector<int>& out_axes) {\n  if (out_axes.size() != s_outputs.size()) {\n    throw std::invalid_argument(\n        \"[vmap] The number of out axes must match the number of outputs.\");\n  }\n\n  std::unordered_map<std::uintptr_t, std::pair<array, int>> tmap;\n  std::unordered_set<std::uintptr_t> needs_vmap;\n  std::unordered_set<std::uintptr_t> cache;\n  for (int i = 0; i < s_inputs.size(); ++i) {\n    auto in = s_inputs[i];\n    if (in_axes[i] != -1) {\n      tmap.insert({in.id(), {inputs[i], in_axes[i]}});\n      needs_vmap.insert(in.id());\n      in.set_tracer(false);\n    }\n    cache.insert(in.id());\n  }\n\n  // Topologically sort the graph\n  std::vector<array> tape;\n\n  std::function<void(const array&)> recurse;\n\n  recurse = [&](const array& a) {\n    auto id = a.id();\n    if (cache.find(id) != cache.end()) {\n      return;\n    }\n    cache.insert(id);\n    for (auto& s : a.siblings()) {\n      cache.insert(s.id());\n    }\n\n    // Recurse on inputs\n    for (auto& input : a.inputs()) {\n      recurse(input);\n    }\n    // If any input needs a vmap, then the outputs also need\n    // a vmap\n    for (auto& input : a.inputs()) {\n      if (needs_vmap.find(input.id()) != needs_vmap.end()) {\n        tape.push_back(a);\n        tape.back().set_tracer(false);\n        needs_vmap.insert(a.id());\n        for (auto s : a.siblings()) {\n          needs_vmap.insert(s.id());\n          s.set_tracer(false);\n        }\n        break;\n      }\n    }\n  };\n\n  for (auto& out : s_outputs) {\n    if (out.has_primitive()) {\n      recurse(out);\n    }\n  }\n\n  // Transform each primitive in the graph with\n  // its vmap implementation\n  for (auto& a : tape) {\n    std::vector<array> v_inputs;\n    std::vector<int> v_axes;\n    for (auto& in : a.inputs()) {\n      auto map_it = tmap.find(in.id());\n      if (map_it != tmap.end()) {\n        v_inputs.push_back(map_it->second.first);\n        v_axes.push_back(map_it->second.second);\n      } else {\n        v_inputs.push_back(in);\n        v_axes.push_back(-1);\n      }\n    }\n\n    auto [v_outputs, v_out_axes] = a.primitive().vmap(v_inputs, v_axes);\n\n    // For each primitive's outputs add its id, the vout id and the vax\n    auto outputs = a.outputs();\n    for (int i = 0; i < v_outputs.size(); ++i) {\n      tmap.insert({outputs[i].id(), {v_outputs[i], v_out_axes[i]}});\n    }\n  }\n\n  // Populate the outputs and make sure all the output axes are\n  // in the right place\n  std::vector<array> outputs;\n  for (int i = 0; i < s_outputs.size(); ++i) {\n    if (auto map_it = tmap.find(s_outputs[i].id()); map_it != tmap.end()) {\n      auto& [out, vdim] = map_it->second;\n      if (vdim != out_axes[i]) {\n        if (out_axes[i] >= out.ndim()) {\n          std::ostringstream msg;\n          msg << \"[vmap] Axis \" << out_axes[i] << \" invalid for output with \"\n              << out.ndim() << \" dimensions.\";\n          throw std::invalid_argument(msg.str());\n        }\n        out = moveaxis(out, vdim, out_axes[i]);\n      }\n      outputs.push_back(out);\n    } else {\n      outputs.push_back(s_outputs[i]);\n    }\n  }\n  return outputs;\n}\n\n} // namespace detail\n\nstd::function<std::vector<array>(const std::vector<array>&)> vmap(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<int>& in_axes /* = {} */,\n    const std::vector<int>& out_axes /* = {} */) {\n  auto infer_axes = [](auto axes) {\n    return !axes.empty() &&\n        std::all_of(axes.begin(), axes.end(), [](int ax) { return ax < 0; });\n  };\n  if (infer_axes(in_axes) != infer_axes(out_axes)) {\n    throw std::invalid_argument(\n        \"[vmap] Input (or output) axes must be \"\n        \"specified if output (or input) axes are.\");\n  }\n  auto vfun = [fun, in_axes = in_axes, out_axes = out_axes](\n                  const std::vector<array>& inputs) mutable {\n    if (in_axes.size() == 0) {\n      in_axes.resize(inputs.size(), 0);\n    }\n\n    auto [trace_inputs, trace_outputs] =\n        detail::vmap_trace(fun, inputs, in_axes);\n\n    if (out_axes.size() == 0) {\n      out_axes.resize(trace_outputs.size(), 0);\n    }\n\n    return detail::vmap_replace(\n        inputs, trace_inputs, trace_outputs, in_axes, out_axes);\n  };\n\n  return vfun;\n}\n\nstd::function<array(const array&, const array&)> vmap(\n    const std::function<array(const array&, const array&)>& fun,\n    int in_axis_a /* = 0 */,\n    int in_axis_b /* = 0 */,\n    int out_axis /* = 0 */) {\n  auto vfun = vmap(\n      [in_axis_a, in_axis_b, out_axis, fun](const std::vector<array>& inputs) {\n        return std::vector<array>{fun(inputs[0], inputs[1])};\n      },\n      {in_axis_a, in_axis_b},\n      {out_axis});\n  return [vfun](const array& a, const array& b) { return vfun({a, b})[0]; };\n}\n\nstd::function<array(const array&)> vmap(\n    const std::function<array(const array&)>& fun,\n    int in_axis /* = 0 */,\n    int out_axis /* = 0 */) {\n  auto vfun = vmap(\n      [in_axis, out_axis, fun](const std::vector<array>& inputs) {\n        return std::vector<array>{fun(inputs[0])};\n      },\n      {in_axis},\n      {out_axis});\n  return [vfun](const array& a) { return vfun({a})[0]; };\n}\n\nstd::function<std::vector<array>(const std::vector<array>&)> custom_vjp(\n    std::function<std::vector<array>(const std::vector<array>&)> fun,\n    std::function<std::vector<array>(\n        const std::vector<array>&,\n        const std::vector<array>&,\n        const std::vector<array>&)> fun_vjp) {\n  return [fun = std::move(fun),\n          fun_vjp = std::move(fun_vjp)](const std::vector<array>& args) {\n    // Compute the outputs\n    auto outputs = fun(args);\n    for (auto& out : outputs) {\n      out = stop_gradient(out);\n    }\n\n    // Prepare the inputs to the primitive\n    // We also add the outputs to the primitive so that it can \"run\" the forward\n    // pass.\n    std::vector<array> inputs = args;\n    inputs.insert(inputs.end(), outputs.begin(), outputs.end());\n\n    // Compute the stream. Maybe do it in a smarter way at some point in the\n    // future.\n    Stream s = (outputs[0].has_primitive()) ? outputs[0].primitive().stream()\n                                            : default_stream(default_device());\n\n    // Make the output info\n    std::vector<std::vector<int>> shapes;\n    std::vector<Dtype> dtypes;\n    for (const auto& out : outputs) {\n      shapes.emplace_back(out.shape());\n      dtypes.emplace_back(out.dtype());\n    }\n\n    return array::make_arrays(\n        std::move(shapes),\n        dtypes,\n        std::make_shared<CustomVJP>(to_stream(s), fun_vjp),\n        inputs);\n  };\n}\n\nstd::function<std::vector<array>(const std::vector<array>&)> checkpoint(\n    std::function<std::vector<array>(const std::vector<array>&)> fun) {\n  auto vjp_fun = [fun](\n                     const std::vector<array>& primals,\n                     const std::vector<array>& cotangents,\n                     const std::vector<array>& outputs) -> std::vector<array> {\n    auto [__, vjps] = vjp(fun, depends(primals, outputs), cotangents);\n    return vjps;\n  };\n\n  return custom_vjp(fun, vjp_fun);\n}\n\n} // namespace mlx::core\n\n// Path: mlx/linalg.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <numeric>\n#include <ostream>\n#include <vector>\n\n#include \"mlx/linalg.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core::linalg {\n\nDtype at_least_float(const Dtype& d) {\n  return issubdtype(d, inexact) ? d : promote_types(d, float32);\n}\n\ninline array l2_norm(\n    const array& a,\n    const std::vector<int>& axis,\n    bool keepdims,\n    StreamOrDevice s) {\n  if (issubdtype(a.dtype(), complexfloating)) {\n    return sqrt(sum(abs(a, s) * abs(a, s), axis, keepdims, s), s);\n  } else {\n    return sqrt(sum(square(a, s), axis, keepdims, s), s);\n  }\n}\n\ninline array vector_norm(\n    const array& a,\n    const double ord,\n    const std::vector<int>& axis,\n    bool keepdims,\n    StreamOrDevice s) {\n  auto dtype = at_least_float(a.dtype());\n  if (ord == 0.0) {\n    return astype(sum(not_equal(a, array(0), s), axis, keepdims, s), dtype, s);\n  } else if (ord == 1.0) {\n    return astype(sum(abs(a, s), axis, keepdims, s), dtype, s);\n  } else if (ord == 2.0) {\n    return l2_norm(a, axis, keepdims, s);\n  } else if (ord == std::numeric_limits<double>::infinity()) {\n    return astype(max(abs(a, s), axis, keepdims, s), dtype, s);\n  } else if (ord == -std::numeric_limits<double>::infinity()) {\n    return astype(min(abs(a, s), axis, keepdims, s), dtype, s);\n  } else {\n    return power(\n        sum(power(abs(a, s), array(ord, dtype), s), axis, keepdims, s),\n        array(1.0 / ord, dtype),\n        s);\n  }\n}\n\ninline array matrix_norm(\n    const array& a,\n    const double ord,\n    const std::vector<int>& axis,\n    bool keepdims,\n    StreamOrDevice s) {\n  auto dtype = at_least_float(a.dtype());\n  auto row_axis = axis[0];\n  auto col_axis = axis[1];\n  if (ord == -1.0) {\n    col_axis -= (!keepdims && col_axis > row_axis && col_axis > 0);\n    return astype(\n        min(sum(abs(a, s), row_axis, keepdims, s), col_axis, keepdims, s),\n        dtype,\n        s);\n...\n", "needle_token_start": 144, "needle_token_end": 172, "code_context_ntokens": 12323, "output": ["```cpp\nbool in_tracing() {\n  return detail::InTracing::in_tracing();\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "InTracing", "language": "cpp", "path": "mlx/transforms_impl.h", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this function is to manage and indicate the state of tracing operations within a computational graph, ensuring that during these operations, evaluations do not discard the graph.\n2. **Input**: This function does not take any explicit parameters as input.\n3. **Output**: There is no direct output returned by this function; instead, it affects the state of a counter used to track the tracing status.\n4. **Procedure**: Upon instantiation, the function increments a global counter that signifies an ongoing tracing operation. This increment helps other parts of the codebase recognize that a tracing process is active, thereby altering their behavior to preserve computational graphs during evaluations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: mlx/ops.h\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#pragma once\n\n#include <optional>\n\n#include \"mlx/array.h\"\n#include \"mlx/device.h\"\n#include \"mlx/stream.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core {\n\n/** Creation operations */\n\n/**\n * A 1D array of numbers starting at `start` (optional),\n * stopping at stop, stepping by `step` (optional). */\narray arange(\n    double start,\n    double stop,\n    double step,\n    Dtype dtype,\n    StreamOrDevice s = {});\narray arange(double start, double stop, double step, StreamOrDevice s = {});\narray arange(double start, double stop, Dtype dtype, StreamOrDevice s = {});\narray arange(double start, double stop, StreamOrDevice s = {});\narray arange(double stop, Dtype dtype, StreamOrDevice s = {});\narray arange(double stop, StreamOrDevice s = {});\n\narray arange(int start, int stop, int step, StreamOrDevice s = {});\narray arange(int start, int stop, StreamOrDevice s = {});\narray arange(int stop, StreamOrDevice s = {});\n\n/** A 1D array of `num` evenly spaced numbers in the range `[start, stop]` */\narray linspace(\n    double start,\n    double stop,\n    int num = 50,\n    Dtype dtype = float32,\n    StreamOrDevice s = {});\n\n/** Convert an array to the given data type. */\narray astype(array a, Dtype dtype, StreamOrDevice s = {});\n\n/** Create a view of an array with the given shape and strides. */\narray as_strided(\n    array a,\n    std::vector<int> shape,\n    std::vector<size_t> strides,\n    size_t offset,\n    StreamOrDevice s = {});\n\n/** Copy another array. */\narray copy(array a, StreamOrDevice s = {});\n\n/** Fill an array of the given shape with the given value(s). */\narray full(\n    std::vector<int> shape,\n    array vals,\n    Dtype dtype,\n    StreamOrDevice s = {});\narray full(std::vector<int> shape, array vals, StreamOrDevice s = {});\ntemplate <typename T>\narray full(std::vector<int> shape, T val, Dtype dtype, StreamOrDevice s = {}) {\n  return full(std::move(shape), array(val, dtype), to_stream(s));\n}\ntemplate <typename T>\narray full(std::vector<int> shape, T val, StreamOrDevice s = {}) {\n  return full(std::move(shape), array(val), to_stream(s));\n}\n\n/** Fill an array of the given shape with zeros. */\narray zeros(const std::vector<int>& shape, Dtype dtype, StreamOrDevice s = {});\ninline array zeros(const std::vector<int>& shape, StreamOrDevice s = {}) {\n  return zeros(shape, float32, s);\n}\narray zeros_like(const array& a, StreamOrDevice s = {});\n\n/** Fill an array of the given shape with ones. */\narray ones(const std::vector<int>& shape, Dtype dtype, StreamOrDevice s = {});\ninline array ones(const std::vector<int>& shape, StreamOrDevice s = {}) {\n  return ones(shape, float32, s);\n}\narray ones_like(const array& a, StreamOrDevice s = {});\n\n/** Fill an array of the given shape (n,m) with ones in the specified diagonal\n * k, and zeros everywhere else. */\narray eye(int n, int m, int k, Dtype dtype, StreamOrDevice s = {});\ninline array eye(int n, Dtype dtype, StreamOrDevice s = {}) {\n  return eye(n, n, 0, dtype, s);\n}\ninline array eye(int n, int m, StreamOrDevice s = {}) {\n  return eye(n, m, 0, float32, s);\n}\ninline array eye(int n, int m, int k, StreamOrDevice s = {}) {\n  return eye(n, m, k, float32, s);\n}\ninline array eye(int n, StreamOrDevice s = {}) {\n  return eye(n, n, 0, float32, s);\n}\n\n/** Create a square matrix of shape (n,n) of zeros, and ones in the major\n * diagonal. */\narray identity(int n, Dtype dtype, StreamOrDevice s = {});\ninline array identity(int n, StreamOrDevice s = {}) {\n  return identity(n, float32, s);\n}\n\narray tri(int n, int m, int k, Dtype type, StreamOrDevice s = {});\ninline array tri(int n, Dtype type, StreamOrDevice s = {}) {\n  return tri(n, n, 0, type, s);\n}\n\narray tril(array x, int k = 0, StreamOrDevice s = {});\narray triu(array x, int k = 0, StreamOrDevice s = {});\n\n/** array manipulation */\n\n/** Reshape an array to the given shape. */\narray reshape(const array& a, std::vector<int> shape, StreamOrDevice s = {});\n\n/** Flatten the dimensions in the range `[start_axis, end_axis]` . */\narray flatten(\n    const array& a,\n    int start_axis,\n    int end_axis = -1,\n    StreamOrDevice s = {});\n\n/** Flatten the array to 1D. */\narray flatten(const array& a, StreamOrDevice s = {});\n\n/** Remove singleton dimensions at the given axes. */\narray squeeze(\n    const array& a,\n    const std::vector<int>& axes,\n    StreamOrDevice s = {});\n\n/** Remove singleton dimensions at the given axis. */\ninline array squeeze(const array& a, int axis, StreamOrDevice s = {}) {\n  return squeeze(a, std::vector<int>{axis}, s);\n}\n\n/** Remove all singleton dimensions. */\narray squeeze(const array& a, StreamOrDevice s = {});\n\n/** Add a singleton dimension at the given axes. */\narray expand_dims(\n    const array& a,\n    const std::vector<int>& axes,\n    StreamOrDevice s = {});\n\n/** Add a singleton dimension at the given axis. */\narray expand_dims(const array& a, int axis, StreamOrDevice s = {});\n\n/** Slice an array. */\narray slice(\n    const array& a,\n    std::vector<int> start,\n    std::vector<int> stop,\n    std::vector<int> strides,\n    StreamOrDevice s = {});\n\n/** Slice an array with a stride of 1 in each dimension. */\narray slice(\n    const array& a,\n    const std::vector<int>& start,\n    const std::vector<int>& stop,\n    StreamOrDevice s = {});\n\n/** Update a slice from the source array */\narray slice_update(\n    const array& src,\n    const array& update,\n    std::vector<int> start,\n    std::vector<int> stop,\n    std::vector<int> strides,\n    StreamOrDevice s = {});\n\n/** Update a slice from the source array with stride 1 in each dimension */\narray slice_update(\n    const array& src,\n    const array& update,\n    std::vector<int> start,\n    std::vector<int> stop,\n    StreamOrDevice s = {});\n\n/** Split an array into sub-arrays along a given axis. */\nstd::vector<array>\nsplit(const array& a, int num_splits, int axis, StreamOrDevice s = {});\nstd::vector<array> split(const array& a, int num_splits, StreamOrDevice s = {});\nstd::vector<array> split(\n    const array& a,\n    const std::vector<int>& indices,\n    int axis,\n    StreamOrDevice s = {});\nstd::vector<array>\nsplit(const array& a, const std::vector<int>& indices, StreamOrDevice s = {});\n\n/** A vector of coordinate arrays from coordinate vectors. */\nstd::vector<array> meshgrid(\n    const std::vector<array>& arrays,\n    bool sparse = false,\n    std::string indexing = \"xy\",\n    StreamOrDevice s = {});\n\n/**\n * Clip (limit) the values in an array.\n */\narray clip(\n    const array& a,\n    const std::optional<array>& a_min = std::nullopt,\n    const std::optional<array>& a_max = std::nullopt,\n    StreamOrDevice s = {});\n\n/** Concatenate arrays along a given axis. */\narray concatenate(\n    const std::vector<array>& arrays,\n    int axis,\n    StreamOrDevice s = {});\narray concatenate(const std::vector<array>& arrays, StreamOrDevice s = {});\n\n/** Stack arrays along a new axis. */\narray stack(const std::vector<array>& arrays, int axis, StreamOrDevice s = {});\narray stack(const std::vector<array>& arrays, StreamOrDevice s = {});\n\n/** Repeat an array along an axis. */\narray repeat(const array& arr, int repeats, int axis, StreamOrDevice s = {});\narray repeat(const array& arr, int repeats, StreamOrDevice s = {});\n\narray tile(const array& arr, std::vector<int> reps, StreamOrDevice s = {});\n\n/** Permutes the dimensions according to the given axes. */\narray transpose(const array& a, std::vector<int> axes, StreamOrDevice s = {});\ninline array transpose(\n    const array& a,\n    std::initializer_list<int> axes,\n    StreamOrDevice s = {}) {\n  return transpose(a, std::vector<int>(axes), s);\n}\n\n/** Swap two axes of an array. */\narray swapaxes(const array& a, int axis1, int axis2, StreamOrDevice s = {});\n\n/** Move an axis of an array. */\narray moveaxis(\n    const array& a,\n    int source,\n    int destination,\n    StreamOrDevice s = {});\n\n/** Pad an array with a constant value */\narray pad(\n    const array& a,\n    const std::vector<int>& axes,\n    const std::vector<int>& low_pad_size,\n    const std::vector<int>& high_pad_size,\n    const array& pad_value = array(0),\n    StreamOrDevice s = {});\n\n/** Pad an array with a constant value along all axes */\narray pad(\n    const array& a,\n    const std::vector<std::pair<int, int>>& pad_width,\n    const array& pad_value = array(0),\n    StreamOrDevice s = {});\narray pad(\n    const array& a,\n    const std::pair<int, int>& pad_width,\n    const array& pad_value = array(0),\n    StreamOrDevice s = {});\narray pad(\n    const array& a,\n    int pad_width,\n    const array& pad_value = array(0),\n    StreamOrDevice s = {});\n\n/** Permutes the dimensions in reverse order. */\narray transpose(const array& a, StreamOrDevice s = {});\n\n/** Broadcast an array to a given shape. */\narray broadcast_to(\n    const array& a,\n    const std::vector<int>& shape,\n    StreamOrDevice s = {});\n\n/** Broadcast a vector of arrays against one another. */\nstd::vector<array> broadcast_arrays(\n    const std::vector<array>& inputs,\n    StreamOrDevice s = {});\n\n/** Comparison operations */\n\n/** Returns the bool array with (a == b) element-wise. */\narray equal(const array& a, const array& b, StreamOrDevice s = {});\ninline array operator==(const array& a, const array& b) {\n  return equal(a, b);\n}\ntemplate <typename T>\narray operator==(T a, const array& b) {\n  return equal(array(a), b);\n}\ntemplate <typename T>\narray operator==(const array& a, T b) {\n  return equal(a, array(b));\n}\n\n/** Returns the bool array with (a != b) element-wise. */\narray not_equal(const array& a, const array& b, StreamOrDevice s = {});\ninline array operator!=(const array& a, const array& b) {\n  return not_equal(a, b);\n}\ntemplate <typename T>\narray operator!=(T a, const array& b) {\n  return not_equal(array(a), b);\n}\ntemplate <typename T>\narray operator!=(const array& a, T b) {\n  return not_equal(a, array(b));\n}\n\n/** Returns bool array with (a > b) element-wise. */\narray greater(const array& a, const array& b, StreamOrDevice s = {});\ninline array operator>(const array& a, const array& b) {\n...\n// Path: mlx/random.h\n// Copyright \u00a9 2023 Apple Inc.\n\n#pragma once\n\n#include <chrono>\n#include <optional>\n\n#include \"mlx/array.h\"\n#include \"mlx/stream.h\"\n\nnamespace mlx::core::random {\n\nclass KeySequence {\n public:\n  explicit KeySequence(uint64_t seed);\n\n  void seed(uint64_t seed);\n  array next();\n\n  // static default\n  static KeySequence& default_() {\n    static KeySequence ks(get_current_time_seed());\n    return ks;\n  }\n\n private:\n  array key_;\n  static uint64_t get_current_time_seed() {\n    auto now = std::chrono::system_clock::now();\n    return std::chrono::duration_cast<std::chrono::milliseconds>(\n               now.time_since_epoch())\n        .count();\n  }\n};\n\n/** Get a PRNG key from a seed. */\narray key(uint64_t seed);\n\n/** Seed the default PRNG key. */\nvoid seed(uint64_t seed);\n\n/** Generate an array with type uint32 filled with random bits. */\narray bits(\n    const std::vector<int>& shape,\n    int width,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\ninline array bits(\n    const std::vector<int>& shape,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return bits(shape, 4, key, s);\n}\n\n/** Split the rng key into a pair of keys. */\nstd::pair<array, array> split(const array& key, StreamOrDevice s = {});\n\n/** Split the rng key into `num` keys. */\narray split(const array& key, int num, StreamOrDevice s = {});\n\n/** Generate uniform random numbers between low and high. */\narray uniform(\n    const array& low,\n    const array& high,\n    const std::vector<int>& shape,\n    Dtype dtype = float32,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\ntemplate <typename T, typename U>\narray uniform(\n    T low,\n    U high,\n    const std::vector<int>& shape,\n    Dtype dtype = float32,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return uniform(array(low), array(high), shape, dtype, key, to_stream(s));\n}\n\n/** Generate uniform random numbers between 0 and 1. */\narray uniform(\n    const std::vector<int>& shape,\n    Dtype dtype,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\ninline array uniform(\n    const std::vector<int>& shape,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return uniform(shape, float32, key);\n}\n\n/** Generate samples from the standard normal distribution. */\narray normal(\n    const std::vector<int>& shape,\n    Dtype dtype,\n    const float loc,\n    const float scale,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\ninline array normal(\n    const std::vector<int>& shape,\n    const float loc,\n    const float scale,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return normal(shape, float32, loc, scale, key, s);\n}\ninline array normal(\n    const std::vector<int>& shape,\n    const Dtype dtype,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return normal(shape, dtype, 0.0, 1.0, key, s);\n}\ninline array normal(\n    const std::vector<int>& shape,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return normal(shape, float32, 0.0, 1.0, key, s);\n}\n\n/** Generate samples from a multivariate normal distribution. **/\narray multivariate_normal(\n    const array& mean,\n    const array& cov,\n    const std::vector<int>& shape,\n    Dtype dtype,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\n/** Generate integer samples uniformly at random */\narray randint(\n    const array& low,\n    const array& high,\n    const std::vector<int>& shape,\n    Dtype dtype = int32,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\ntemplate <typename T, typename U>\narray randint(\n    T low,\n    U high,\n    const std::vector<int>& shape,\n    Dtype dtype = int32,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return randint(array(low), array(high), shape, dtype, key, to_stream(s));\n};\n\n/** Generate binary variables with probability to be true equal to p */\narray bernoulli(\n    const array& p,\n    const std::vector<int>& shape,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\narray bernoulli(\n    const array& p,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\ntemplate <typename T>\narray bernoulli(\n    T p,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return bernoulli(array(p), key, s);\n};\n\ntemplate <typename T>\narray bernoulli(\n    T p,\n    const std::vector<int>& shape,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {}) {\n  return bernoulli(array(p), shape, key, s);\n};\n\narray bernoulli(\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\narray truncated_normal(\n    const array& lower,\n    const array& upper,\n    const std::vector<int>& shape,\n    Dtype dtype = float32,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\narray truncated_normal(\n    const array& lower,\n    const array& upper,\n    Dtype dtype = float32,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\narray gumbel(\n    const std::vector<int>& shape,\n    Dtype dtype = float32,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\narray categorical(\n    const array& logits,\n    int axis,\n    const std::vector<int>& shape,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\narray categorical(\n    const array& logits_,\n    int axis,\n    int num_samples,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\narray categorical(\n    const array& logits,\n    int axis = -1,\n    const std::optional<array>& key = std::nullopt,\n    StreamOrDevice s = {});\n\n} // namespace mlx::core::random\n\n// Path: mlx/transforms_impl.h\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#pragma once\n\nnamespace mlx::core::detail {\n\nstd::pair<std::vector<array>, std::vector<array>> vmap_trace(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    const std::vector<array>& inputs,\n    const std::vector<int>& in_axes);\n\nstd::vector<array> vmap_replace(\n    const std::vector<array>& inputs,\n    const std::vector<array>& s_inputs,\n    const std::vector<array>& s_outputs,\n    const std::vector<int>& in_axes,\n    const std::vector<int>& out_axes);\n\n// This is not part of the general C++ API as calling with a bad id is a bad\n// idea.\nstd::function<std::vector<array>(const std::vector<array>&)> compile(\n    const std::function<std::vector<array>(const std::vector<array>&)>& fun,\n    std::uintptr_t fun_id,\n    bool shapeless = false,\n    std::vector<uint64_t> constants = {});\n\n// Erase cached compile functions\nvoid compile_erase(std::uintptr_t fun_id);\n\n// Create an InTracing object during tracing operations to signify to the rest\n// of the codebase that we are during tracing so evals should not throw away\n// the graph.\nstruct InTracing {\n  I\nnTracing() {\n    tracing_counter++;\n  }\n  ~InTracing() {\n    tracing_counter--;\n  }\n\n  static bool in_tracing() {\n    return tracing_counter > 0;\n  }\n\n private:\n  static int tracing_counter;\n};\n\n} // namespace mlx::core::detail\n\n// Path: mlx/fast_primitives.h\n// Copyright \u00a9 2024 Apple Inc.\n\n#include \"mlx/primitives.h\"\n\nnamespace mlx::core::fast {\n\n// Custom primitive accepts a fallback function which it uses for\n// transformations. Transformations are virtual so that derived classes may\n// override the default behavior.\nclass Custom : public Primitive {\n public:\n  explicit Custom(\n      Stream stream,\n      std::function<std::vector<array>(std::vector<array>)> fallback)\n      : Primitive(stream), fallback_(fallback){};\n\n  virtual std::pair<std::vector<array>, std::vector<int>> vmap(\n      const std::vector<array>& inputs,\n      const std::vector<int>& axes) override;\n\n  virtual std::vector<array> jvp(\n      const std::vector<array>& primals,\n      const std::vector<array>& tangents,\n      const std::vector<int>& argnums) override;\n\n  virtual std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n private:\n  std::function<std::vector<array>(std::vector<array>)> fallback_;\n};\n\nclass RMSNorm : public Custom {\n public:\n  RMSNorm(\n      Stream stream,\n      std::function<std::vector<array>(std::vector<array>)> fallback,\n      float eps)\n      : Custom(stream, fallback), eps_(eps){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override {\n    throw std::runtime_error(\"NYI\");\n  };\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_PRINT(RMSNorm)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::function<std::vector<array>(std::vector<array>)> fallback_;\n  float eps_;\n};\n\nclass RMSNormVJP : public Custom {\n public:\n  RMSNormVJP(\n      Stream stream,\n      std::function<std::vector<array>(std::vector<array>)> fallback,\n      float eps)\n      : Custom(stream, fallback), eps_(eps){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override {\n    throw std::runtime_error(\"NYI\");\n  };\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  DEFINE_PRINT(RMSNormVJP)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::function<std::vector<array>(std::vector<array>)> fallback_;\n  float eps_;\n};\n\nclass LayerNorm : public Custom {\n public:\n  LayerNorm(\n      Stream stream,\n      std::function<std::vector<array>(std::vector<array>)> fallback,\n      float eps)\n      : Custom(stream, fallback), eps_(eps){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override {\n    throw std::runtime_error(\"NYI\");\n  };\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_PRINT(LayerNorm)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::function<std::vector<array>(std::vector<array>)> fallback_;\n  float eps_;\n};\n\nclass LayerNormVJP : public Custom {\n public:\n  LayerNormVJP(\n      Stream stream,\n      std::function<std::vector<array>(std::vector<array>)> fallback,\n      float eps)\n      : Custom(stream, fallback), eps_(eps){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override {\n    throw std::runtime_error(\"NYI\");\n  };\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  DEFINE_PRINT(LayerNormVJP)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::function<std::vector<array>(std::vector<array>)> fallback_;\n  float eps_;\n};\n\nclass RoPE : public Custom {\n public:\n  RoPE(\n      Stream stream,\n      std::function<std::vector<array>(std::vector<array>)> fallback,\n      int dims,\n      bool traditional,\n      float base,\n      float scale,\n      int offset,\n      bool forward)\n      : Custom(stream, fallback),\n        dims_(dims),\n        traditional_(traditional),\n        base_(base),\n        scale_(scale),\n        offset_(offset),\n        forward_(forward){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override {\n    throw std::runtime_error(\"NYI\");\n  };\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_PRINT(RoPE)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::function<std::vector<array>(std::vector<array>)> fallback_;\n  int dims_;\n  bool traditional_;\n  float base_;\n  float scale_;\n  int offset_;\n  bool forward_;\n};\n\nclass ScaledDotProductAttention : public Custom {\n public:\n  explicit ScaledDotProductAttention(\n      Stream stream,\n      std::function<std::vector<array>(std::vector<array>)> fallback,\n      const float scale,\n      const bool needs_mask)\n      : Custom(stream, fallback), scale_(scale), needs_mask_(needs_mask){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override {\n    throw std::runtime_error(\"NYI\");\n  };\n\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override {\n    eval_gpu(inputs, outputs[0]);\n  };\n\n  void eval_gpu(const std::vector<array>& inputs, array& out);\n  bool is_equivalent(const Primitive& other) const override;\n\n  DEFINE_PRINT(ScaledDotProductAttention)\n\n private:\n  std::function<std::vector<array>(std::vector<array>)> fallback_;\n  float scale_;\n  bool needs_mask_;\n};\n\n} // namespace mlx::core::fast\n\n// Path: mlx/compile_impl.h\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#pragma once\n\n#include \"mlx/device.h\"\n\nnamespace mlx::core::detail {\n\nbool compile_available_for_device(const Device& device);\n\n}\n\n// Path: mlx/allocator.h\n// Copyright \u00a9 2023 Apple Inc.\n\n#pragma once\n\n#include <cstdlib>\n\nnamespace mlx::core::allocator {\n\n// Simple wrapper around buffer pointers\n// WARNING: Only Buffer objects constructed from and those that wrap\n//          raw pointers from mlx::allocator are supported.\nclass Buffer {\n private:\n  void* ptr_;\n\n public:\n  Buffer(void* ptr) : ptr_(ptr){};\n\n  // Get the raw data pointer from the buffer\n  void* raw_ptr();\n\n  // Get the buffer pointer from the buffer\n  const void* ptr() const {\n    return ptr_;\n  };\n  void* ptr() {\n    return ptr_;\n  };\n};\n\nBuffer malloc(size_t size);\n\nvoid free(Buffer buffer);\n\n// Wait for running tasks to finish and free up memory\n// if allocation fails\nBuffer malloc_or_wait(size_t size);\n\nclass Allocator {\n  /** Abstract base class for a memory allocator. */\n public:\n  virtual Buffer malloc(size_t size, bool allow_swap = false) = 0;\n  virtual void free(Buffer buffer) = 0;\n\n  Allocator() = default;\n  Allocator(const Allocator& other) = delete;\n  Allocator(Allocator&& other) = delete;\n  Allocator& operator=(const Allocator& other) = delete;\n  Allocator& operator=(Allocator&& other) = delete;\n  virtual ~Allocator() = default;\n};\n\nAllocator& allocator();\n\nclass CommonAllocator : public Allocator {\n  /** A general CPU allocator. */\n public:\n  virtual Buffer malloc(size_t size, bool allow_swap = false) override;\n  virtual void free(Buffer buffer) override;\n\n private:\n  CommonAllocator() = default;\n  friend Allocator& allocator();\n};\n\n} // namespace mlx::core::allocator\n\n// Path: mlx/io/load.h\n// Copyright \u00a9 2023 Apple Inc.\n\n#pragma once\n\n#include <fstream>\n#include <istream>\n#include <memory>\n\nnamespace mlx::core {\n\nnamespace io {\n\nclass Reader {\n public:\n  virtual bool is_open() const = 0;\n  virtual bool good() const = 0;\n  virtual size_t tell() = 0; // tellp is non-const in iostream\n  virtual void seek(\n      int64_t off,\n      std::ios_base::seekdir way = std::ios_base::beg) = 0;\n  virtual void read(char* data, size_t n) = 0;\n  virtual std::string label() const = 0;\n};\n\nclass Writer {\n public:\n  virtual bool is_open() const = 0;\n  virtual bool good() const = 0;\n  virtual size_t tell() = 0;\n  virtual void seek(\n      int64_t off,\n      std::ios_base::seekdir way = std::ios_base::beg) = 0;\n  virtual void write(const char* data, size_t n) = 0;\n  virtual std::string label() const = 0;\n};\n\nclass FileReader : public Reader {\n public:\n  explicit FileReader(std::ifstream is)\n      : is_(std::move(is)), label_(\"stream\") {}\n  explicit FileReader(std::string file_path)\n      : is_(std::ifstream(file_path, std::ios::binary)),\n        label_(std::move(file_path)) {}\n\n  bool is_open() const override {\n    return is_.is_open();\n  }\n\n  bool good() const override {\n    return is_.good();\n  }\n\n  size_t tell() override {\n    return is_.tellg();\n  }\n\n  void seek(int64_t off, std::ios_base::seekdir way = std::ios_base::beg)\n      override {\n    is_.seekg(off, way);\n  }\n\n  void read(char* data, size_t n) override {\n    is_.read(data, n);\n  }\n\n  std::string label() const override {\n    return \"file \" + label_;\n  }\n\n private:\n  std::ifstream is_;\n  std::string label_;\n};\n\nclass FileWriter : public Writer {\n public:\n  explicit FileWriter(std::ofstream os)\n      : os_(std::move(os)), label_(\"stream\") {}\n  explicit FileWriter(std::string file_path)\n      : os_(std::ofstream(file_path, std::ios::binary)),\n        label_(std::move(file_path)) {}\n\n  bool is_open() const override {\n    return os_.is_open();\n  }\n\n  bool good() const override {\n    return os_.good();\n  }\n\n  size_t tell() override {\n    return os_.tellp();\n  }\n\n  void seek(int64_t off, std::ios_base::seekdir way = std::ios_base::beg)\n      override {\n    os_.seekp(off, way);\n  }\n\n  void write(const char* data, size_t n) override {\n    os_.write(data, n);\n  }\n\n  std::string label() const override {\n    return \"file \" + label_;\n  }\n\n private:\n  std::ofstream os_;\n  std::string label_;\n};\n\n} // namespace io\n} // namespace mlx::core\n\n// Path: mlx/io/gguf.h\n// Copyright \u00a9 2023-2024 Apple Inc.\n#pragma once\n\n#include \"mlx/io.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/transforms.h\"\n#include \"mlx/utils.h\"\n\nextern \"C\" {\n#include <gguflib.h>\n}\n\nnamespace mlx::core {\n\nstd::vector<int> get_shape(const gguf_tensor& tensor);\nvoid gguf_load_quantized(\n    std::unordered_map<std::string, array>& a,\n    const gguf_tensor& tensor);\n\n} // namespace mlx::core\n\n// Path: mlx/types/bf16.h\n// Copyright \u00a9 2023 Apple Inc.\n\n#pragma once\n\n#include <algorithm>\n#include <cmath>\n#include <cstdint>\n#include <vector>\n\n#define __MLX_BFLOAT_NAN__ 0x7FC0\n\nnamespace mlx::core {\n\nnamespace {\nunion float_bits_bf16 {\n  float f;\n  uint32_t u;\n};\n} // namespace\n\nstruct _MLX_BFloat16 {\n  uint16_t bits_;\n\n  // Default constructor\n  _MLX_BFloat16() = default;\n\n  // Default copy constructor\n  _MLX_BFloat16(_MLX_BFloat16 const&) = default;\n\n  // Appease std::vector<bool> for being special\n  _MLX_BFloat16& operator=(std::vector<bool>::reference x) {\n    bits_ = x;\n    return *this;\n  }\n\n  _MLX_BFloat16& operator=(const float& x) {\n    return (*this = _MLX_BFloat16(x));\n  }\n\n  // From float32\n  _MLX_BFloat16(const float& x) {\n    if (std::isnan(x)) {\n      bits_ = __MLX_BFLOAT_NAN__;\n    } else {\n      // Union\n      float_bits_bf16 in;\n\n      // Take bits\n      in.f = x;\n\n      // Round to nearest even\n      in.u += (in.u >> 16 & 1) + uint32_t(0x7FFF);\n\n      // Take upper 16 bits\n      bits_ = in.u >> 16;\n    }\n  }\n\n  // To float32\n  operator float() const {\n    // Union\n    float_bits_bf16 out;\n\n    // Upper 16 bits are the data and lower 16 bits are 0s\n    out.u = ((uint32_t)bits_) << 16;\n\n    return out.f;\n  }\n};\n\n#define bfloat_binop_base(__op__, __operator__, otype, atype, btype, ctype) \\\n  inline otype __operator__(atype lhs, btype rhs) {                         \\\n    return static_cast<ctype>(lhs) __op__ static_cast<ctype>(rhs);          \\\n  }\n\n#define bfloat_binop_helper(__op__, __operator__, otype, itype, ctype) \\\n  inline otype __operator__(_MLX_BFloat16 lhs, itype rhs) {            \\\n    return static_cast<ctype>(lhs) __op__ static_cast<ctype>(rhs);     \\\n  }                                                                    \\\n  inline otype __operator__(itype lhs, _MLX_BFloat16 rhs) {            \\\n    return static_cast<ctype>(lhs) __op__ static_cast<ctype>(rhs);     \\\n  }\n\n// Operators\n#define bfloat_binop(_op_, _operator_)                                       \\\n  bfloat_binop_base(                                                         \\\n      _op_, _operator_, _MLX_BFloat16, _MLX_BFloat16, _MLX_BFloat16, float); \\\n  bfloat_binop_helper(_op_, _operator_, float, float, float);                \\\n  bfloat_binop_helper(_op_, _operator_, double, double, double);             \\\n  bfloat_binop_helper(_op_, _operator_, _MLX_BFloat16, bool, float);         \\\n  bfloat_binop_helper(_op_, _operator_, _MLX_BFloat16, int32_t, float);      \\\n  bfloat_binop_helper(_op_, _operator_, _MLX_BFloat16, uint32_t, float);     \\\n  bfloat_binop_helper(_op_, _operator_, _MLX_BFloat16, int64_t, float);      \\\n  bfloat_binop_helper(_op_, _operator_, _MLX_BFloat16, uint64_t, float);\n\nbfloat_binop(+, operator+);\nbfloat_binop(-, operator-);\nbfloat_binop(*, operator*);\nbfloat_binop(/, operator/);\n\n#undef bfloat_binop\n\n// Comparison ops\n#define bfloat_compop(__op__, __operator__)                             \\\n  bfloat_binop_base(                                                    \\\n      __op__, __operator__, bool, _MLX_BFloat16, _MLX_BFloat16, float); \\\n  bfloat_binop_helper(__op__, __operator__, bool, float, float);        \\\n  bfloat_binop_helper(__op__, __operator__, bool, double, double);      \\\n  bfloat_binop_helper(__op__, __operator__, bool, int32_t, float);      \\\n  bfloat_binop_helper(__op__, __operator__, bool, uint32_t, float);     \\\n  bfloat_binop_helper(__op__, __operator__, bool, int64_t, float);      \\\n  bfloat_binop_helper(__op__, __operator__, bool, uint64_t, float);\n\nbfloat_compop(>, operator>);\nbfloat_compop(<, operator<);\nbfloat_compop(>=, operator>=);\nbfloat_compop(<=, operator<=);\nbfloat_compop(==, operator==);\nbfloat_compop(!=, operator!=);\n\n#undef bfloat_compop\n\n// Negative\ninline _MLX_BFloat16 operator-(_MLX_BFloat16 lhs) {\n  return -static_cast<float>(lhs);\n}\n\n// Inplace ops\n#define bfloat_inplace_op(__op__, __operator__)                              \\\n  inline _MLX_BFloat16& __operator__(_MLX_BFloat16& lhs, const float& rhs) { \\\n    lhs = lhs __op__ rhs;                                                    \\\n    return lhs;                                                              \\\n  }                                                                          \\\n  inline float& __operator__(float& lhs, _MLX_BFloat16 rhs) {                \\\n    lhs = lhs __op__ rhs;                                                    \\\n    return lhs;                                                              \\\n  }\n\nbfloat_inplace_op(+, operator+=);\nbfloat_inplace_op(-, operator-=);\nbfloat_inplace_op(*, operator*=);\nbfloat_inplace_op(/, operator/=);\n\n#undef bfloat_inplace_op\n\n// Bitwise ops\n\n#define bfloat_bitop(__op__, __operator__)                                  \\\n  inline _MLX_BFloat16 __operator__(_MLX_BFloat16 lhs, _MLX_BFloat16 rhs) { \\\n    _MLX_BFloat16 out;                                                      \\\n    out.bits_ = lhs.bits_ __op__ rhs.bits_;                                 \\\n    return out;                                                             \\\n  }                                                                         \\\n  inline _MLX_BFloat16 __operator__(_MLX_BFloat16 lhs, uint16_t rhs) {      \\\n    _MLX_BFloat16 out;                                                      \\\n    out.bits_ = lhs.bits_ __op__ rhs;                                       \\\n    return out;                                                             \\\n  }                                                                         \\\n  inline _MLX_BFloat16 __operator__(uint16_t lhs, _MLX_BFloat16 rhs) {      \\\n    _MLX_BFloat16 out;                                                      \\\n    out.bits_ = lhs __op__ rhs.bits_;                                       \\\n    return out;                                                             \\\n  }\n\nbfloat_bitop(|, operator|);\nbfloat_bitop(&, operator&);\nbfloat_bitop(^, operator^);\n\n#undef bfloat_bitop\n\n#define bfloat_inplace_bitop(__op__, __operator__)                            \\\n  inline _MLX_BFloat16& __operator__(_MLX_BFloat16& lhs, _MLX_BFloat16 rhs) { \\\n    lhs.bits_ = lhs.bits_ __op__ rhs.bits_;                                   \\\n    return lhs;                                                               \\\n  }                                                                           \\\n  inline _MLX_BFloat16& __operator__(_MLX_BFloat16& lhs, uint16_t rhs) {      \\\n    lhs.bits_ = lhs.bits_ __op__ rhs;                                         \\\n    return lhs;                                                               \\\n  }\n\nbfloat_inplace_bitop(|, operator|=);\nbfloat_inplace_bitop(&, operator&=);\nbfloat_inplace_bitop(^, operator^=);\n\n#undef bfloat_inplace_bitop\n\n} // namespace mlx::core\n\n// Path: mlx/types/complex.h\n// Copyright \u00a9 2023 Apple Inc.\n\n#pragma once\n#include <complex>\n#include \"mlx/types/half_types.h\"\n\nnamespace mlx::core {\n\nstruct complex64_t;\nstruct complex128_t;\n\ntemplate <typename T>\ninline constexpr bool can_convert_to_complex128 =\n    !std::is_same_v<T, complex128_t> && std::is_convertible_v<T, double>;\n\nstruct complex128_t : public std::complex<double> {\n  complex128_t(double v, double u) : std::complex<double>(v, u){};\n  complex128_t(std::complex<double> v) : std::complex<double>(v){};\n\n  template <\n      typename T,\n      typename = typename std::enable_if<can_convert_to_complex128<T>>::type>\n  complex128_t(T x) : std::complex<double>(x){};\n\n  operator float() const {\n    return real();\n  };\n};\n\ntemplate <typename T>\ninline constexpr bool can_convert_to_complex64 =\n    !std::is_same_v<T, complex64_t> && std::is_convertible_v<T, float>;\n\nstruct complex64_t : public std::complex<float> {\n  complex64_t(float v, float u) : std::complex<float>(v, u){};\n  complex64_t(std::complex<float> v) : std::complex<float>(v){};\n\n  template <\n      typename T,\n      typename = typename std::enable_if<can_convert_to_complex64<T>>::type>\n  complex64_t(T x) : std::complex<float>(x){};\n\n  operator float() const {\n    return real();\n  };\n};\n\ninline bool operator>=(const complex64_t& a, const complex64_t& b) {\n  return (a.real() > b.real()) ||\n      (a.real() == b.real() && a.imag() >= b.imag());\n}\n\ninline bool operator>(const complex64_t& a, const complex64_t& b) {\n  return (a.real() > b.real()) || (a.real() == b.real() && a.imag() > b.imag());\n}\n\ninline complex64_t operator%(complex64_t a, complex64_t b) {\n  auto real = a.real() - (b.real() * static_cast<int64_t>(a.real() / b.real()));\n  auto imag = a.imag() - (b.imag() * static_cast<int64_t>(a.imag() / b.imag()));\n  if (real != 0 && ((real < 0) != (b.real() < 0)))\n    real += b.real();\n  if (imag != 0 && ((imag < 0) != (b.imag() < 0)))\n    imag += b.imag();\n  return {real, imag};\n}\n\ninline bool operator<=(const complex64_t& a, const complex64_t& b) {\n  return operator>=(b, a);\n}\n\ninline bool operator<(const complex64_t& a, const complex64_t& b) {\n  return operator>(b, a);\n}\n\ninline complex64_t operator-(const complex64_t& v) {\n  return -static_cast<std::complex<float>>(v);\n}\n\n// clang-format off\n#define complex_binop_helper(_op_, _operator_, itype)            \\\n  inline complex64_t _operator_(itype x, const complex64_t& y) { \\\n    return static_cast<complex64_t>(x) _op_ y;           \\\n  }                                                              \\\n  inline complex64_t _operator_(const complex64_t& x, itype y) { \\\n    return x _op_ static_cast<complex64_t>(y);           \\\n  }\n\n#define complex_binop(_op_, _operator_)                                               \\\n  inline complex64_t _operator_(const std::complex<float>& x, const complex64_t& y) { \\\n    return x _op_ static_cast<std::complex<float>>(y);                                \\\n  }                                                                                   \\\n  inline complex64_t _operator_(const complex64_t& x, const std::complex<float>& y) { \\\n    return static_cast<std::complex<float>>(x) _op_ y;                                \\\n  }                                                                                   \\\n  inline complex64_t _operator_(const complex64_t& x, const complex64_t& y) {         \\\n    return static_cast<std::complex<float>>(x)                                        \\\n        _op_ static_cast<std::complex<float>>(y);                                     \\\n  }                                                                                   \\\n  complex_binop_helper(_op_, _operator_, bool)                                        \\\n  complex_binop_helper(_op_, _operator_, uint32_t)                                    \\\n  complex_binop_helper(_op_, _operator_, uint64_t)                                    \\\n  complex_binop_helper(_op_, _operator_, int32_t)                                     \\\n  complex_binop_helper(_op_, _operator_, int64_t)                                     \\\n  complex_binop_helper(_op_, _operator_, float16_t)                                   \\\n  complex_binop_helper(_op_, _operator_, bfloat16_t)                                  \\\n  complex_binop_helper(_op_, _operator_, float)\n// clang-format on\n\ncomplex_binop(+, operator+)\n\n} // namespace mlx::core\n\n// Path: mlx/types/fp16.h\n// Copyright \u00a9 2023 Apple Inc.\n\n#pragma once\n\n#include <algorithm>\n#include <cmath>\n#include <cstdint>\n#include <vector>\n\n#define __MLX_HALF_NAN__ 0x7D00\n\nnamespace mlx::core {\n\nnamespace {\nunion float_bits_fp16 {\n  float f;\n  uint32_t u;\n};\n} // namespace\n\nstruct _MLX_Float16 {\n  uint16_t bits_;\n\n  // Default constructor\n  _MLX_Float16() = default;\n\n  // Default copy constructor\n  _MLX_Float16(_MLX_Float16 const&) = default;\n\n  // Appease std::vector<bool> for being special\n  _MLX_Float16& operator=(std::vector<bool>::reference x) {\n    bits_ = x;\n    return *this;\n  }\n\n  _MLX_Float16& operator=(const float& x) {\n    return (*this = _MLX_Float16(x));\n  }\n\n  // From float32\n  _MLX_Float16(const float& x) : bits_(0) {\n    // Conversion following\n    // https://github.com/Maratyszcza/FP16/blob/master/include/fp16/fp16.h\n\n    // Union\n    float_bits_fp16 in;\n\n    // Take fp32 bits\n    in.f = x;\n\n    // Find and take sign bit\n    uint32_t x_sign_32 = in.u & uint32_t(0x80000000);\n    uint16_t x_sign_16 = (x_sign_32 >> 16);\n\n    if (std::isnan(x)) {\n      bits_ = x_sign_16 | uint16_t(__MLX_HALF_NAN__);\n    } else {\n      // Union\n      float_bits_fp16 inf_scale, zero_scale, magic_bits;\n\n      // Find exponent bits and take the max supported by half\n      uint32_t x_expo_32 = in.u & uint32_t(0x7f800000);\n      uint32_t max_expo_32 = uint32_t(0x38800000);\n      x_expo_32 = x_expo_32 < max_expo_32 ? max_expo_32 : x_expo_32;\n      x_expo_32 += uint32_t(15) << 23;\n\n      // Handle scaling to inf as needed\n      inf_scale.u = uint32_t(0x77800000);\n      zero_scale.u = uint32_t(0x08800000);\n\n      // Combine with magic and let addition do rounding\n      magic_bits.u = x_expo_32;\n      magic_bits.f += (std::abs(x) * inf_scale.f) * zero_scale.f;\n\n      // Take the lower 5 bits of the exponent\n      uint32_t x_expo_16 = ((magic_bits.u >> 13) & uint32_t(0x7c00));\n\n      // Collect the lower 12 bits which have the mantissa\n      uint32_t x_mant_16 = magic_bits.u & uint32_t(0x0fff);\n\n      // Combine sign, exp and mantissa\n      bits_ = (x_sign_16 | uint16_t(x_expo_16 + x_mant_16));\n    }\n  }\n\n  // To float32\n  operator float() const {\n    // Conversion following\n    // https://github.com/Maratyszcza/FP16/blob/master/include/fp16/fp16.h\n\n    // Union\n    float_bits_fp16 out;\n\n    uint32_t x_sign_32 = (bits_ << 16) & uint32_t(0x80000000);\n    uint32_t base = (bits_ << 16);\n    uint32_t two_base = base + base;\n\n    uint32_t denorm_max = 1u << 27;\n    if (two_base < denorm_max) {\n      out.u = uint32_t(126) << 23; // magic mask\n      out.u |= (two_base >> 17); // Bits from fp16\n      out.f -= 0.5f; // magic bias\n    } else {\n      out.u = uint32_t(0xE0) << 23; // exponent offset\n      out.u += (two_base >> 4); // Bits from fp16\n      float out_unscaled = out.f; // Store value\n      out.u = uint32_t(0x7800000); // exponent scale\n      out.f *= out_unscaled;\n    }\n\n    // Add sign\n    out.u |= x_sign_32;\n\n    return out.f;\n  }\n};\n\n#define half_binop_base(__op__, __operator__, otype, atype, btype, ctype) \\\n  inline otype __operator__(atype lhs, btype rhs) {                       \\\n    return static_cast<ctype>(lhs) __op__ static_cast<ctype>(rhs);        \\\n  }\n\n#define half_binop_helper(__op__, __operator__, otype, itype, ctype) \\\n  inline otype __operator__(_MLX_Float16 lhs, itype rhs) {           \\\n    return static_cast<ctype>(lhs) __op__ static_cast<ctype>(rhs);   \\\n  }                                                                  \\\n  inline otype __operator__(itype lhs, _MLX_Float16 rhs) {           \\\n    return static_cast<ctype>(lhs) __op__ static_cast<ctype>(rhs);   \\\n  }\n\n// Operators\n#define half_binop(__op__, __operator__)                                      \\\n  half_binop_base(                                                            \\\n      __op__, __operator__, _MLX_Float16, _MLX_Float16, _MLX_Float16, float); \\\n  half_binop_helper(__op__, __operator__, float, float, float);               \\\n  half_binop_helper(__op__, __operator__, double, double, double);            \\\n  half_binop_helper(__op__, __operator__, _MLX_Float16, bool, float);         \\\n  half_binop_helper(__op__, __operator__, _MLX_Float16, int32_t, float);      \\\n  half_binop_helper(__op__, __operator__, _MLX_Float16, uint32_t, float);     \\\n  half_binop_helper(__op__, __operator__, _MLX_Float16, int64_t, float);      \\\n  half_binop_helper(__op__, __operator__, _MLX_Float16, uint64_t, float);\n\nhalf_binop(+, operator+);\nhalf_binop(-, operator-);\nhalf_binop(*, operator*);\nhalf_binop(/, operator/);\n\n#undef half_binop\n\n// Comparison ops\n#define half_compop(__op__, __operator__)                             \\\n  half_binop_base(                                                    \\\n      __op__, __operator__, bool, _MLX_Float16, _MLX_Float16, float); \\\n  half_binop_helper(__op__, __operator__, bool, float, float);        \\\n  half_binop_helper(__op__, __operator__, bool, double, double);      \\\n  half_binop_helper(__op__, __operator__, bool, int32_t, float);      \\\n  half_binop_helper(__op__, __operator__, bool, uint32_t, float);     \\\n  half_binop_helper(__op__, __operator__, bool, int64_t, float);      \\\n  half_binop_helper(__op__, __operator__, bool, uint64_t, float);\n\nhalf_compop(>, operator>);\nhalf_compop(<, operator<);\nhalf_compop(>=, operator>=);\nhalf_compop(<=, operator<=);\nhalf_compop(==, operator==);\nhalf_compop(!=, operator!=);\n\n#undef half_compop\n\n// Negative\ninline _MLX_Float16 operator-(_MLX_Float16 lhs) {\n  return -static_cast<float>(lhs);\n}\n\n// Inplace ops\n#define half_inplace_op(__op__, __operator__)                              \\\n  inline _MLX_Float16& __operator__(_MLX_Float16& lhs, const float& rhs) { \\\n    lhs = lhs __op__ rhs;                                                  \\\n    return lhs;                                                            \\\n  }                                                                        \\\n  inline float& __operator__(float& lhs, _MLX_Float16 rhs) {               \\\n    lhs = lhs __op__ rhs;                                                  \\\n    return lhs;                                                            \\\n  }\n\nhalf_inplace_op(+, operator+=);\nhalf_inplace_op(-, operator-=);\nhalf_inplace_op(*, operator*=);\nhalf_inplace_op(/, operator/=);\n\n#undef half_inplace_op\n\n// Bitwise ops\n\n#define half_bitop(__op__, __operator__)                                 \\\n  inline _MLX_Float16 __operator__(_MLX_Float16 lhs, _MLX_Float16 rhs) { \\\n    _MLX_Float16 out;                                                    \\\n    out.bits_ = lhs.bits_ __op__ rhs.bits_;                              \\\n    return out;                                                          \\\n  }                                                                      \\\n  inline _MLX_Float16 __operator__(_MLX_Float16 lhs, uint16_t rhs) {     \\\n    _MLX_Float16 out;                                                    \\\n    out.bits_ = lhs.bits_ __op__ rhs;                                    \\\n    return out;                                                          \\\n  }                                                                      \\\n  inline _MLX_Float16 __operator__(uint16_t lhs, _MLX_Float16 rhs) {     \\\n    _MLX_Float16 out;                                                    \\\n    out.bits_ = lhs __op__ rhs.bits_;                                    \\\n    return out;                                                          \\\n  }\n\nhalf_bitop(|, operator|);\nhalf_bitop(&, operator&);\nhalf_bitop(^, operator^);\n\n#undef half_bitop\n\n#define half_inplace_bitop(__op__, __operator__)                           \\\n  inline _MLX_Float16& __operator__(_MLX_Float16& lhs, _MLX_Float16 rhs) { \\\n    lhs.bits_ = lhs.bits_ __op__ rhs.bits_;                                \\\n    return lhs;                                                            \\\n  }                                                                        \\\n  inline _MLX_Float16& __operator__(_MLX_Float16& lhs, uint16_t rhs) {     \\\n    lhs.bits_ = lhs.bits_ __op__ rhs;                                      \\\n    return lhs;                                                            \\\n...\n", "needle_token_start": 5628, "needle_token_end": 5645, "code_context_ntokens": 16065, "output": ["```cpp\nnamespace mlx::core::detail {\n\nstruct InTracing {\n  InTracing() {\n    tracing_counter++;\n  }\n  ~InTracing() {\n    tracing_counter--;\n  }\n\n  static bool in_tracing() {\n    return tracing_counter > 0;\n  }\n\n private:\n  static int tracing_counter;\n};\n\n} // namespace mlx::core::detail\n```"]}
{"repo": "ml-explore/mlx", "name": "copy_inplace_dispatch", "language": "cpp", "path": "mlx/backend/common/copy.cpp", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the copying of data from one array to another, ensuring that the data type of the source array is appropriately managed during the copy process to match the destination array's data type.\n2. **Input**: This function takes a source array, a destination array, a type of copy operation, and additional arguments that may be needed for specific copy operations.\n3. **Output**: There is no direct output returned by the function; however, it modifies the destination array by copying data from the source array into it according to the specified copy type.\n4. **Procedure**: The function first identifies the data type of the source array. Based on this data type, it selects the appropriate templated copy function to execute. The selected function handles the copying process, which may vary depending on the copy type (e.g., scalar, vector, general, or general-general copying). This ensures that the data is copied correctly while respecting the data types and the intended memory layout and transformation specified by the copy type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: mlx/backend/common/quantized.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <cassert>\n\n#include \"mlx/backend/metal/copy.h\"\n#include \"mlx/primitives.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\ntemplate <typename T, int bits, int group_size>\nvoid _qmm(\n    T* result,\n    const T* x,\n    const uint32_t* w,\n    const T* scales,\n    const T* biases,\n    int M,\n    int N,\n    int K) {\n  constexpr int bitmask = (1 << bits) - 1;\n  constexpr int pack_factor = 32 / bits;\n  constexpr int packs_in_group = group_size / pack_factor;\n  const int Ng = N / group_size;\n  const int Nw = N / pack_factor;\n\n  for (int m = 0; m < M; m++) {\n    const uint32_t* w_local = w;\n    const T* scales_local = scales;\n    const T* biases_local = biases;\n\n    std::fill(result, result + N, 0);\n\n    for (int k = 0; k < K; k++) {\n      T* result_local = result;\n      T xi = *x++;\n\n      for (int n = 0; n < N; n += group_size) {\n        T scale = *scales_local++;\n        T bias = *biases_local++;\n        for (int ng = 0; ng < packs_in_group; ng++) {\n          uint32_t wi = *w_local++;\n\n#pragma clang loop unroll(full)\n          for (int p = 0; p < pack_factor; p++) {\n            (*result_local++) +=\n                xi * (scale * static_cast<T>(wi & bitmask) + bias);\n            wi >>= bits;\n          }\n        }\n      }\n    }\n\n    result += N;\n  }\n}\n\ntemplate <typename T, int bits, int group_size>\nvoid _qmm_t(\n    T* result,\n    const T* x,\n    const uint32_t* w,\n    const T* scales,\n    const T* biases,\n    int M,\n    int N,\n    int K) {\n  constexpr int bitmask = (1 << bits) - 1;\n  constexpr int pack_factor = 32 / bits;\n  constexpr int packs_in_group = group_size / pack_factor;\n  const int Kg = K / group_size;\n  const int Kw = K / pack_factor;\n\n  for (int m = 0; m < M; m++) {\n    const uint32_t* w_local = w;\n    const T* scales_local = scales;\n    const T* biases_local = biases;\n\n    for (int n = 0; n < N; n++) {\n      const T* x_local = x;\n      T sum = 0;\n      for (int k = 0; k < K; k += group_size) {\n        T scale = *scales_local++;\n        T bias = *biases_local++;\n\n        for (int kw = 0; kw < packs_in_group; kw++) {\n          uint32_t wi = *w_local++;\n\n#pragma clang loop unroll(full)\n          for (int p = 0; p < pack_factor; p++) {\n            sum += (*x_local++) * (scale * static_cast<T>(wi & bitmask) + bias);\n            wi >>= bits;\n          }\n        }\n      }\n      *result = sum;\n      result++;\n    }\n\n    x += K;\n  }\n}\n\ntemplate <typename T>\nvoid _qmm_dispatch_typed(\n    T* result,\n    const T* x,\n    const uint32_t* w,\n    const T* scales,\n    const T* biases,\n    int M,\n    int N,\n    int K,\n    int group_size,\n    int bits,\n    bool transposed_w) {\n  switch (bits) {\n    case 2: {\n      switch (group_size) {\n        case 32:\n          if (transposed_w) {\n            return _qmm_t<T, 2, 32>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 2, 32>(result, x, w, scales, biases, M, N, K);\n          }\n        case 64:\n          if (transposed_w) {\n            return _qmm_t<T, 2, 64>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 2, 64>(result, x, w, scales, biases, M, N, K);\n          }\n        case 128:\n          if (transposed_w) {\n            return _qmm_t<T, 2, 128>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 2, 128>(result, x, w, scales, biases, M, N, K);\n          }\n      }\n    }\n    case 4: {\n      switch (group_size) {\n        case 32:\n          if (transposed_w) {\n            return _qmm_t<T, 4, 32>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 4, 32>(result, x, w, scales, biases, M, N, K);\n          }\n        case 64:\n          if (transposed_w) {\n            return _qmm_t<T, 4, 64>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 4, 64>(result, x, w, scales, biases, M, N, K);\n          }\n        case 128:\n          if (transposed_w) {\n            return _qmm_t<T, 4, 128>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 4, 128>(result, x, w, scales, biases, M, N, K);\n          }\n      }\n    }\n    case 8: {\n      switch (group_size) {\n        case 32:\n          if (transposed_w) {\n            return _qmm_t<T, 8, 32>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 8, 32>(result, x, w, scales, biases, M, N, K);\n          }\n        case 64:\n          if (transposed_w) {\n            return _qmm_t<T, 8, 64>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 8, 64>(result, x, w, scales, biases, M, N, K);\n          }\n        case 128:\n          if (transposed_w) {\n            return _qmm_t<T, 8, 128>(result, x, w, scales, biases, M, N, K);\n          } else {\n            return _qmm<T, 8, 128>(result, x, w, scales, biases, M, N, K);\n          }\n      }\n    }\n  }\n  std::ostringstream msg;\n  msg << \"Quantization type not supported. Provided bits=\" << bits\n      << \" and group_size=\" << group_size\n      << \". The supported options are bits in \"\n      << \"{2, 4, 8} and group_size in {64, 128}.\";\n  throw std::invalid_argument(msg.str());\n}\n\nvoid _qmm_dispatch(\n    array out,\n    const array& x,\n    const array& w,\n    const array& scales,\n    const array& biases,\n    int bits,\n    int group_size,\n    bool transposed_w) {\n  int K = x.shape(-1);\n  int M = x.size() / K;\n  int N = out.shape(-1);\n\n  switch (x.dtype()) {\n    case float32:\n      _qmm_dispatch_typed<float>(\n          out.data<float>(),\n          x.data<float>(),\n          w.data<uint32_t>(),\n          scales.data<float>(),\n          biases.data<float>(),\n          M,\n          N,\n          K,\n          bits,\n          group_size,\n          transposed_w);\n      break;\n    case float16:\n      _qmm_dispatch_typed<float16_t>(\n          out.data<float16_t>(),\n          x.data<float16_t>(),\n          w.data<uint32_t>(),\n          scales.data<float16_t>(),\n          biases.data<float16_t>(),\n          M,\n          N,\n          K,\n          bits,\n          group_size,\n          transposed_w);\n      break;\n    case bfloat16:\n      _qmm_dispatch_typed<bfloat16_t>(\n          out.data<bfloat16_t>(),\n          x.data<bfloat16_t>(),\n          w.data<uint32_t>(),\n          scales.data<bfloat16_t>(),\n          biases.data<bfloat16_t>(),\n          M,\n          N,\n          K,\n          bits,\n          group_size,\n          transposed_w);\n      break;\n    default:\n      throw std::invalid_argument(\n          \"[quantized_matmul] only floating types are supported\");\n  }\n}\n\n} // namespace\n\nvoid QuantizedMatmul::eval(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 4);\n\n  auto& x_pre = inputs[0];\n  auto& w_pre = inputs[1];\n  auto& scales_pre = inputs[2];\n  auto& biases_pre = inputs[3];\n\n  auto ensure_row_contiguous = [](const array& arr) {\n...\n// Path: mlx/backend/common/copy.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <numeric>\n\n#include \"mlx/allocator.h\"\n#include \"mlx/backend/common/copy.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\ntemplate <typename SrcT, typename DstT>\nvoid copy_single(const array& src, array& dst) {\n  auto val = static_cast<DstT>(src.data<SrcT>()[0]);\n  auto dst_ptr = dst.data<DstT>();\n  for (int i = 0; i < dst.size(); ++i) {\n    dst_ptr[i] = val;\n  }\n}\n\ntemplate <typename SrcT, typename DstT>\nvoid copy_vector(const array& src, array& dst) {\n  auto src_ptr = src.data<SrcT>();\n  auto dst_ptr = dst.data<DstT>();\n  std::copy(src_ptr, src_ptr + src.data_size(), dst_ptr);\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t>\nvoid copy_general_dim1(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    int64_t i_offset) {\n  const SrcT* src_ptr = src.data<SrcT>();\n  DstT* dst_ptr = dst.data<DstT>();\n  stride_t src_idx = i_offset;\n  stride_t dst_idx = 0;\n  for (int i = 0; i < data_shape[0]; ++i) {\n    dst_ptr[dst_idx++] = static_cast<DstT>(src_ptr[src_idx]);\n    src_idx += i_strides[0];\n  }\n}\n\ntemplate <typename SrcT, typename DstT>\ninline void copy_general_dim1(const array& src, array& dst) {\n  return copy_general_dim1<SrcT, DstT, size_t>(\n      src, dst, src.shape(), src.strides(), 0);\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t>\nvoid copy_general_dim2(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    int64_t i_offset) {\n  const SrcT* src_ptr = src.data<SrcT>();\n  DstT* dst_ptr = dst.data<DstT>();\n  stride_t src_idx = i_offset;\n  stride_t dst_idx = 0;\n  for (int i = 0; i < data_shape[0]; ++i) {\n    for (int j = 0; j < data_shape[1]; ++j) {\n      dst_ptr[dst_idx++] = static_cast<DstT>(src_ptr[src_idx]);\n      src_idx += i_strides[1];\n    }\n    src_idx += i_strides[0] - i_strides[1] * data_shape[1];\n  }\n}\n\ntemplate <typename SrcT, typename DstT>\ninline void copy_general_dim2(const array& src, array& dst) {\n  return copy_general_dim2<SrcT, DstT, size_t>(\n      src, dst, src.shape(), src.strides(), 0);\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t>\nvoid copy_general_dim3(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    int64_t i_offset) {\n  const SrcT* src_ptr = src.data<SrcT>();\n  DstT* dst_ptr = dst.data<DstT>();\n  stride_t src_idx = i_offset;\n  stride_t dst_idx = 0;\n  for (int i = 0; i < data_shape[0]; ++i) {\n    for (int j = 0; j < data_shape[1]; ++j) {\n      for (int k = 0; k < data_shape[2]; ++k) {\n        dst_ptr[dst_idx++] = static_cast<DstT>(src_ptr[src_idx]);\n        src_idx += i_strides[2];\n      }\n      src_idx += i_strides[1] - i_strides[2] * data_shape[2];\n    }\n    src_idx += i_strides[0] - i_strides[1] * data_shape[1];\n  }\n}\n\ntemplate <typename SrcT, typename DstT>\ninline void copy_general_dim3(const array& src, array& dst) {\n  return copy_general_dim3<SrcT, DstT, size_t>(\n      src, dst, src.shape(), src.strides(), 0);\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t>\nvoid copy_general_dim4(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    int64_t i_offset) {\n  const SrcT* src_ptr = src.data<SrcT>();\n  DstT* dst_ptr = dst.data<DstT>();\n  stride_t src_idx = i_offset;\n  stride_t dst_idx = 0;\n  for (int i = 0; i < data_shape[0]; ++i) {\n    for (int j = 0; j < data_shape[1]; ++j) {\n      for (int k = 0; k < data_shape[2]; ++k) {\n        for (int ii = 0; ii < data_shape[3]; ++ii) {\n          dst_ptr[dst_idx++] = static_cast<DstT>(src_ptr[src_idx]);\n          src_idx += i_strides[3];\n        }\n        src_idx += i_strides[2] - i_strides[3] * data_shape[3];\n      }\n      src_idx += i_strides[1] - i_strides[2] * data_shape[2];\n    }\n    src_idx += i_strides[0] - i_strides[1] * data_shape[1];\n  }\n}\n\ntemplate <typename SrcT, typename DstT>\ninline void copy_general_dim4(const array& src, array& dst) {\n  return copy_general_dim4<SrcT, DstT, size_t>(\n      src, dst, src.shape(), src.strides(), 0);\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t>\nvoid copy_general(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    int64_t i_offset) {\n  switch (src.ndim()) {\n    case 1:\n      copy_general_dim1<SrcT, DstT, stride_t>(\n          src, dst, data_shape, i_strides, i_offset);\n      return;\n    case 2:\n      copy_general_dim2<SrcT, DstT, stride_t>(\n          src, dst, data_shape, i_strides, i_offset);\n      return;\n    case 3:\n      copy_general_dim3<SrcT, DstT, stride_t>(\n          src, dst, data_shape, i_strides, i_offset);\n      return;\n    case 4:\n      copy_general_dim4<SrcT, DstT, stride_t>(\n          src, dst, data_shape, i_strides, i_offset);\n      return;\n  }\n\n  auto src_ptr = src.data<SrcT>() + i_offset;\n  auto dst_ptr = dst.data<DstT>();\n  for (size_t i = 0; i < dst.size(); ++i) {\n    stride_t src_elem = elem_to_loc(i, data_shape, i_strides);\n    dst_ptr[i] = static_cast<DstT>(src_ptr[src_elem]);\n  }\n}\n\ntemplate <typename SrcT, typename DstT>\ninline void copy_general(const array& src, array& dst) {\n  return copy_general<SrcT, DstT, size_t>(\n      src, dst, src.shape(), src.strides(), 0);\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t>\ninline void copy_general(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    const std::vector<stride_t>& o_strides,\n    int64_t i_offset,\n    int64_t o_offset) {\n  return copy_general<SrcT, DstT, stride_t>(\n      src, dst, data_shape, i_strides, i_offset);\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t, int D>\ninline void copy_general_general_dims(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    const std::vector<stride_t>& o_strides,\n    stride_t i_offset,\n    stride_t o_offset) {\n  if constexpr (D > 1) {\n    int axis = src.ndim() - D;\n    auto stride_src = i_strides[axis];\n    auto stride_dst = o_strides[axis];\n    auto N = data_shape[axis];\n    for (int i = 0; i < N; i++) {\n      copy_general_general_dims<SrcT, DstT, stride_t, D - 1>(\n          src, dst, data_shape, i_strides, o_strides, i_offset, o_offset);\n      i_offset += stride_src;\n      o_offset += stride_dst;\n    }\n  } else {\n    int axis = src.ndim() - 1;\n    auto stride_src = i_strides[axis];\n    auto stride_dst = o_strides[axis];\n    auto N = data_shape[axis];\n    const SrcT* src_ptr = src.data<SrcT>() + i_offset;\n    DstT* dst_ptr = dst.data<DstT>() + o_offset;\n    for (int i = 0; i < N; i++) {\n      *dst_ptr = static_cast<DstT>(*src_ptr);\n      src_ptr += stride_src;\n      dst_ptr += stride_dst;\n    }\n  }\n}\n\ntemplate <typename SrcT, typename DstT, typename stride_t>\nvoid copy_general_general(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    const std::vector<stride_t>& o_strides,\n    stride_t i_offset,\n    stride_t o_offset) {\n  switch (src.ndim()) {\n    case 1:\n      copy_general_general_dims<SrcT, DstT, stride_t, 1>(\n          src, dst, data_shape, i_strides, o_strides, i_offset, o_offset);\n      return;\n    case 2:\n      copy_general_general_dims<SrcT, DstT, stride_t, 2>(\n          src, dst, data_shape, i_strides, o_strides, i_offset, o_offset);\n      return;\n    case 3:\n      copy_general_general_dims<SrcT, DstT, stride_t, 3>(\n          src, dst, data_shape, i_strides, o_strides, i_offset, o_offset);\n      return;\n    case 4:\n      copy_general_general_dims<SrcT, DstT, stride_t, 4>(\n          src, dst, data_shape, i_strides, o_strides, i_offset, o_offset);\n      return;\n    case 5:\n      copy_general_general_dims<SrcT, DstT, stride_t, 5>(\n          src, dst, data_shape, i_strides, o_strides, i_offset, o_offset);\n      return;\n  }\n\n  int size = std::accumulate(\n      data_shape.begin() - 5, data_shape.end(), 1, std::multiplies<int>());\n  for (int i = 0; i < src.size(); i += size) {\n    stride_t src_offset = i_offset + elem_to_loc(i, data_shape, i_strides);\n    stride_t dst_offset = o_offset + elem_to_loc(i, dst.shape(), o_strides);\n    copy_general_general_dims<SrcT, DstT, stride_t, 5>(\n        src, dst, data_shape, i_strides, o_strides, src_offset, dst_offset);\n  }\n}\n\ntemplate <typename SrcT, typename DstT>\ninline void copy_general_general(const array& src, array& dst) {\n  return copy_general_general<SrcT, DstT, size_t>(\n      src, dst, src.shape(), src.strides(), dst.strides(), 0, 0);\n}\n\ntemplate <typename SrcT, typename DstT, typename... Args>\nvoid copy(const array& src, array& dst, CopyType ctype, Args&&... args) {\n  switch (ctype) {\n    case CopyType::Scalar:\n      copy_single<SrcT, DstT>(src, dst);\n      return;\n    case CopyType::Vector:\n      copy_vector<SrcT, DstT>(src, dst);\n      return;\n    case CopyType::General:\n      copy_general<SrcT, DstT>(src, dst, std::forward<Args>(args)...);\n      return;\n    case CopyType::GeneralGeneral:\n      copy_general_general<SrcT, DstT>(src, dst, std::forward<Args>(args)...);\n  }\n}\n\ntemplate <typename SrcT, typename... Args>\nvoid copy(const array& src, array& dst, CopyType ctype, Args&&... args) {\n  switch (dst.dtype()) {\n    case bool_:\n      copy<SrcT, bool>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint8:\n      copy<SrcT, uint8_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint16:\n      copy<SrcT, uint16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint32:\n      copy<SrcT, uint32_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint64:\n      copy<SrcT, uint64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int8:\n      copy<SrcT, int8_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int16:\n      copy<SrcT, int16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int32:\n      copy<SrcT, int32_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int64:\n      copy<SrcT, int64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case float16:\n      copy<SrcT, float16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case float32:\n      copy<SrcT, float>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case bfloat16:\n      copy<SrcT, bfloat16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case complex64:\n      copy<SrcT, complex64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n  }\n}\n\ntemplate <typename... Args>\ni\nnline void copy_inplace_dispatch(\n    const array& src,\n    array& dst,\n    CopyType ctype,\n    Args&&... args) {\n  switch (src.dtype()) {\n    case bool_:\n      copy<bool>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint8:\n      copy<uint8_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint16:\n      copy<uint16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint32:\n      copy<uint32_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint64:\n      copy<uint64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int8:\n      copy<int8_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int16:\n      copy<int16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int32:\n      copy<int32_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int64:\n      copy<int64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case float16:\n      copy<float16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case float32:\n      copy<float>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case bfloat16:\n      copy<bfloat16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case complex64:\n      copy<complex64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n  }\n}\n\n} // namespace\n\nvoid copy_inplace(const array& src, array& dst, CopyType ctype) {\n  return copy_inplace_dispatch(src, dst, ctype);\n}\n\nvoid copy(const array& src, array& dst, CopyType ctype) {\n  // Allocate the output\n  switch (ctype) {\n    case CopyType::Vector:\n      if (src.is_donatable() && src.itemsize() == dst.itemsize()) {\n        dst.copy_shared_buffer(src);\n      } else {\n        auto size = src.data_size();\n        dst.set_data(\n            allocator::malloc_or_wait(size * dst.itemsize()),\n            size,\n            src.strides(),\n            src.flags());\n      }\n      break;\n    case CopyType::Scalar:\n    case CopyType::General:\n    case CopyType::GeneralGeneral:\n      dst.set_data(allocator::malloc_or_wait(dst.nbytes()));\n      break;\n  }\n  if (ctype == CopyType::GeneralGeneral) {\n    ctype = CopyType::General;\n  }\n  copy_inplace(src, dst, ctype);\n}\n\ntemplate <typename stride_t>\nvoid copy_inplace(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<stride_t>& i_strides,\n    const std::vector<stride_t>& o_strides,\n    int64_t i_offset,\n    int64_t o_offset,\n    CopyType ctype) {\n  switch (ctype) {\n    case CopyType::General:\n    case CopyType::GeneralGeneral:\n      return copy_inplace_dispatch(\n          src,\n          dst,\n          ctype,\n          data_shape,\n          i_strides,\n          o_strides,\n          i_offset,\n          o_offset);\n\n    case CopyType::Scalar:\n    case CopyType::Vector:\n      return copy_inplace_dispatch(src, dst, ctype);\n  }\n}\n\ntemplate <>\nvoid copy_inplace<int64_t>(\n    const array& src,\n    array& dst,\n    const std::vector<int>& data_shape,\n    const std::vector<int64_t>& i_strides,\n    const std::vector<int64_t>& o_strides,\n    int64_t i_offset,\n    int64_t o_offset,\n    CopyType ctype) {\n  switch (ctype) {\n    case CopyType::General:\n    case CopyType::GeneralGeneral:\n      return copy_inplace_dispatch(\n          src,\n          dst,\n          ctype,\n          data_shape,\n          i_strides,\n          o_strides,\n          i_offset,\n          o_offset);\n\n    case CopyType::Scalar:\n    case CopyType::Vector:\n      return copy_inplace_dispatch(src, dst, ctype);\n  }\n}\n\n} // namespace mlx::core\n\n// Path: mlx/backend/common/compiled_nocpu.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include \"mlx/backend/common/compiled.h\"\n\nnamespace mlx::core {\n\n// GPU compile is always available if the GPU is available and since we are in\n// this file CPU compile is not available so check if the device is a GPU\n// device.\nnamespace detail {\nbool compile_available_for_device(const Device& device) {\n  return device == Device::gpu;\n}\n} // namespace detail\n\nvoid Compiled::eval_cpu(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  throw std::runtime_error(\n      \"[Compiled::eval_cpu] CPU compialtion not supported on the platform.\");\n}\n\n} // namespace mlx::core\n\n// Path: mlx/backend/common/compiled.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include \"mlx/backend/common/compiled.h\"\n#include \"mlx/graph_utils.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core {\n\nvoid print_constant(std::ostream& os, const array& x) {\n  switch (x.dtype()) {\n    case float32:\n      return print_float_constant<float>(os, x);\n    case float16:\n      return print_float_constant<float16_t>(os, x);\n    case bfloat16:\n      return print_float_constant<bfloat16_t>(os, x);\n    case complex64:\n      return print_complex_constant<complex64_t>(os, x);\n    case int8:\n      return print_int_constant<int8_t>(os, x);\n    case int16:\n      return print_int_constant<int16_t>(os, x);\n    case int32:\n      return print_int_constant<int32_t>(os, x);\n    case int64:\n      return print_int_constant<int64_t>(os, x);\n    case uint8:\n      return print_int_constant<uint8_t>(os, x);\n    case uint16:\n      return print_int_constant<uint16_t>(os, x);\n    case uint32:\n      return print_int_constant<uint32_t>(os, x);\n    case uint64:\n      return print_int_constant<uint64_t>(os, x);\n    case bool_:\n      os << std::boolalpha << x.item<bool>();\n      return;\n    default:\n      throw std::runtime_error(\"Unsupported constant type\");\n  }\n}\n\nstd::string get_type_string(Dtype d) {\n  switch (d) {\n    case float32:\n      return \"float\";\n    case float16:\n      return \"float16_t\";\n    case bfloat16:\n      return \"bfloat16_t\";\n    case complex64:\n      return \"complex64_t\";\n    case bool_:\n      return \"bool\";\n    case int8:\n      return \"int8_t\";\n    case int16:\n      return \"int16_t\";\n    case int32:\n      return \"int32_t\";\n    case int64:\n      return \"int64_t\";\n    case uint8:\n      return \"uint8_t\";\n    case uint16:\n      return \"uint16_t\";\n    case uint32:\n      return \"uint32_t\";\n    case uint64:\n      return \"uint64_t\";\n    default: {\n      std::ostringstream msg;\n      msg << \"Unsupported compilation type \" << d;\n      throw std::runtime_error(msg.str());\n    }\n  }\n}\n\nstd::string build_lib_name(\n    const std::vector<array>& inputs,\n    const std::vector<array>& outputs,\n    const std::vector<array>& tape,\n    const std::unordered_set<uintptr_t>& constant_ids) {\n  NodeNamer namer;\n  std::ostringstream os;\n  std::ostringstream constant_hasher;\n\n  // Fill the input names. This is not really necessary, I just like having A,\n  // B, C, ... as the inputs.\n  for (auto& x : inputs) {\n    namer.get_name(x);\n  }\n\n  // The primitives describing the tape. For unary and binary primitives this\n  // must be enough to describe the full computation.\n  for (auto& a : tape) {\n    // name and type of output\n    os << namer.get_name(a) << kindof(a.dtype()) << a.itemsize();\n    // computation performed\n    a.primitive().print(os);\n    // name of inputs to the function\n    for (auto& inp : a.inputs()) {\n      os << namer.get_name(inp);\n    }\n  }\n  os << \"_\";\n\n  for (auto& x : inputs) {\n    if (constant_ids.find(x.id()) != constant_ids.end()) {\n      os << \"C\";\n      print_constant(constant_hasher, x);\n    } else {\n      os << (is_scalar(x) ? \"S\" : \"V\");\n    }\n  }\n  os << \"_\";\n  for (auto& x : inputs) {\n    if (constant_ids.find(x.id()) != constant_ids.end()) {\n      continue;\n    }\n    os << kindof(x.dtype()) << x.itemsize();\n  }\n  os << \"_\" << std::hash<std::string>{}(constant_hasher.str());\n\n  return os.str();\n}\n\nbool compiled_check_contiguity(\n    const std::vector<array>& inputs,\n    const std::vector<int>& shape) {\n  bool contiguous = true;\n  bool all_contig = true;\n  bool all_row_contig = true;\n  bool all_col_contig = true;\n  int non_scalar_inputs = 0;\n  for (const auto& x : inputs) {\n    if (is_scalar(x)) {\n      continue;\n    }\n    non_scalar_inputs++;\n    bool shape_eq = x.shape() == shape;\n    all_contig &= (x.flags().contiguous && shape_eq);\n    all_row_contig &= (x.flags().row_contiguous && shape_eq);\n    all_col_contig &= (x.flags().col_contiguous && shape_eq);\n  }\n  if (non_scalar_inputs > 1 && !all_row_contig && !all_col_contig) {\n    contiguous = false;\n  } else if (non_scalar_inputs == 1 && !all_contig) {\n    contiguous = false;\n  } else if (non_scalar_inputs == 0 && !shape.empty()) {\n    contiguous = false;\n  }\n  return contiguous;\n}\n\nvoid compiled_allocate_outputs(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs,\n    const std::vector<array>& inputs_,\n    const std::unordered_set<uintptr_t>& constant_ids_,\n    bool contiguous,\n    bool move_buffers /* = false */) {\n  if (contiguous) {\n    int o = 0;\n    std::vector<size_t> strides;\n    size_t data_size;\n    array::Flags flags;\n    for (int i = 0; i < inputs.size() && o < outputs.size(); ++i) {\n      auto& in = inputs[i];\n      // Conditions for donation\n      // - Correct size\n      // - Not a scalar\n      // - Donatable\n      // - Not a constant\n      if (in.itemsize() == outputs[o].itemsize() && !is_scalar(in) &&\n          in.is_donatable() &&\n          constant_ids_.find(inputs_[i].id()) == constant_ids_.end()) {\n        if (move_buffers) {\n          outputs[o++].move_shared_buffer(in);\n        } else {\n          outputs[o++].copy_shared_buffer(in);\n        }\n      }\n      // Get representative input flags to properly set non-donated outputs\n      if (strides.empty() && in.size() == outputs[0].size()) {\n        strides = in.strides();\n        flags = in.flags();\n        data_size = in.data_size();\n      }\n    }\n    for (; o < outputs.size(); ++o) {\n      outputs[o].set_data(\n          allocator::malloc_or_wait(data_size * outputs[o].itemsize()),\n          data_size,\n          strides,\n          flags);\n    }\n  } else {\n    int o = 0;\n    for (int i = 0; i < inputs.size() && o < outputs.size(); ++i) {\n      auto& in = inputs[i];\n      // Conditions for donation\n      // - Row contiguous\n      // - Donatable\n      // - Correct size\n      // - Not a constant\n      if (in.flags().row_contiguous && in.nbytes() == outputs[o].nbytes() &&\n          in.is_donatable() &&\n          constant_ids_.find(inputs_[i].id()) == constant_ids_.end()) {\n        if (move_buffers) {\n          outputs[o].move_shared_buffer(\n              in, outputs[o].strides(), in.flags(), in.data_size());\n        } else {\n          outputs[o].copy_shared_buffer(\n              in, outputs[o].strides(), in.flags(), in.data_size());\n        }\n        o++;\n      }\n    }\n    for (; o < outputs.size(); ++o) {\n      outputs[o].set_data(allocator::malloc_or_wait(outputs[o].nbytes()));\n    }\n  }\n}\n\n} // namespace mlx::core\n\n// Path: mlx/backend/common/select.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <cassert>\n\n#include \"mlx/backend/common/ternary.h\"\n#include \"mlx/primitives.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\ntemplate <typename Op>\nvoid select_op(\n    const array& a,\n    const array& b,\n    const array& c,\n    array& out,\n    Op op) {\n  switch (out.dtype()) {\n    case bool_:\n      ternary_op<bool, bool, bool, bool>(a, b, c, out, op);\n      break;\n    case uint8:\n      ternary_op<bool, uint8_t, uint8_t, uint8_t>(a, b, c, out, op);\n      break;\n    case uint16:\n      ternary_op<bool, uint16_t, uint16_t, uint16_t>(a, b, c, out, op);\n      break;\n    case uint32:\n      ternary_op<bool, uint32_t, uint32_t, uint32_t>(a, b, c, out, op);\n      break;\n    case uint64:\n      ternary_op<bool, uint64_t, uint64_t, uint64_t>(a, b, c, out, op);\n      break;\n    case int8:\n      ternary_op<bool, int8_t, int8_t, int8_t>(a, b, c, out, op);\n      break;\n    case int16:\n      ternary_op<bool, int16_t, int16_t, int16_t>(a, b, c, out, op);\n      break;\n    case int32:\n      ternary_op<bool, int32_t, int32_t, int32_t>(a, b, c, out, op);\n      break;\n    case int64:\n      ternary_op<bool, int64_t, int64_t, int64_t>(a, b, c, out, op);\n      break;\n    case float16:\n      ternary_op<bool, float16_t, float16_t, float16_t>(a, b, c, out, op);\n      break;\n    case float32:\n      ternary_op<bool, float, float, float>(a, b, c, out, op);\n      break;\n    case bfloat16:\n      ternary_op<bool, bfloat16_t, bfloat16_t, bfloat16_t>(a, b, c, out, op);\n      break;\n    case complex64:\n      ternary_op<bool, complex64_t, complex64_t, complex64_t>(a, b, c, out, op);\n      break;\n  }\n}\n\n} // namespace\n\nvoid Select::eval(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 3);\n  const auto& condition = inputs[0];\n  const auto& a = inputs[1];\n  const auto& b = inputs[2];\n  select_op(condition, a, b, out, detail::Select());\n}\n\n} // namespace mlx::core\n\n// Path: mlx/backend/common/threefry.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include \"mlx/backend/common/threefry.h\"\n\nnamespace mlx::core::random {\n\nstd::pair<uint32_t, uint32_t> threefry2x32_hash(\n    const std::pair<uint32_t, uint32_t>& key,\n    std::pair<uint32_t, uint32_t> count) {\n  constexpr static uint32_t rotations[2][4] = {\n      {13, 15, 26, 6}, {17, 29, 16, 24}};\n\n  uint32_t ks[3] = {key.first, key.second, key.first ^ key.second ^ 0x1BD11BDA};\n\n  count.first += ks[0];\n  count.second += ks[1];\n\n  for (int i = 0; i < 5; ++i) {\n    for (auto r : rotations[i % 2]) {\n      count.first += count.second;\n      count.second = (count.second << r) | (count.second >> (32 - r));\n      count.second ^= count.first;\n    }\n    count.first += ks[(i + 1) % 3];\n    count.second += ks[(i + 2) % 3] + i + 1;\n  }\n\n  return count;\n}\n\n} // namespace mlx::core::random\n\n// Path: mlx/backend/common/fft.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <numeric>\n\n#include \"mlx/3rdparty/pocketfft.h\"\n#include \"mlx/allocator.h\"\n#include \"mlx/primitives.h\"\n\nnamespace mlx::core {\n\nvoid FFT::eval(const std::vector<array>& inputs, array& out) {\n  auto& in = inputs[0];\n  std::vector<std::ptrdiff_t> strides_in(\n      in.strides().begin(), in.strides().end());\n  for (auto& s : strides_in) {\n    s *= in.itemsize();\n  }\n  std::vector<std::ptrdiff_t> strides_out(\n      out.strides().begin(), out.strides().end());\n  for (auto& s : strides_out) {\n    s *= out.itemsize();\n  }\n\n  out.set_data(allocator::malloc_or_wait(out.nbytes()));\n\n  std::vector<size_t> shape;\n  if (out.dtype() == float32) {\n    shape.insert(shape.end(), out.shape().begin(), out.shape().end());\n  } else {\n    shape.insert(shape.end(), in.shape().begin(), in.shape().end());\n  }\n\n  float scale = 1.0f;\n  if (inverse_) {\n    size_t nelem = std::accumulate(\n        axes_.begin(), axes_.end(), 1, [&shape](auto x, auto y) {\n          return x * shape[y];\n        });\n    scale /= nelem;\n  }\n  if (in.dtype() == complex64 && out.dtype() == complex64) {\n    auto in_ptr =\n        reinterpret_cast<const std::complex<float>*>(in.data<complex64_t>());\n    auto out_ptr =\n        reinterpret_cast<std::complex<float>*>(out.data<complex64_t>());\n    pocketfft::c2c(\n        shape,\n        strides_in,\n        strides_out,\n        axes_,\n        !inverse_,\n        in_ptr,\n        out_ptr,\n        scale);\n  } else if (in.dtype() == float32 && out.dtype() == complex64) {\n    auto in_ptr = in.data<float>();\n    auto out_ptr =\n        reinterpret_cast<std::complex<float>*>(out.data<complex64_t>());\n    pocketfft::r2c(\n        shape,\n        strides_in,\n        strides_out,\n        axes_,\n        !inverse_,\n        in_ptr,\n        out_ptr,\n        scale);\n  } else if (in.dtype() == complex64 && out.dtype() == float32) {\n    auto in_ptr =\n        reinterpret_cast<const std::complex<float>*>(in.data<complex64_t>());\n    auto out_ptr = out.data<float>();\n    pocketfft::c2r(\n        shape,\n        strides_in,\n        strides_out,\n        axes_,\n        !inverse_,\n        in_ptr,\n        out_ptr,\n        scale);\n  } else {\n    throw std::runtime_error(\n        \"[FFT] Received unexpected input and output type combination.\");\n  }\n}\n\n} // namespace mlx::core\n\n// Path: mlx/backend/common/sort.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <algorithm>\n#include <cassert>\n#include <cmath>\n#include <numeric>\n\n#include \"mlx/backend/common/copy.h\"\n#include \"mlx/backend/common/utils.h\"\n\n#include \"mlx/primitives.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\ntemplate <typename T, typename IdxT = int32_t>\nstruct StridedIterator {\n  using iterator_category = std::random_access_iterator_tag;\n  using difference_type = IdxT;\n  using value_type = T;\n  using reference = value_type&;\n  using pointer = value_type*;\n\n  // Constructors\n  StridedIterator() = default;\n\n  explicit StridedIterator(T* ptr, size_t stride, difference_type offset = 0)\n      : ptr_(ptr + offset * stride), stride_(stride) {}\n\n  explicit StridedIterator(array& arr, int axis, difference_type offset = 0)\n      : StridedIterator(arr.data<T>(), arr.strides()[axis], offset) {}\n\n  // Accessors\n  reference operator*() const {\n    return ptr_[0];\n  }\n\n  reference operator[](difference_type idx) const {\n    return ptr_[idx * stride_];\n  }\n\n  // Comparisons\n  bool operator==(const StridedIterator& other) const {\n    return ptr_ == other.ptr_ && stride_ == other.stride_;\n  }\n\n  bool operator!=(const StridedIterator& other) const {\n    return ptr_ != other.ptr_;\n  }\n\n  bool operator<(const StridedIterator& other) const {\n    return ptr_ < other.ptr_;\n  }\n\n  bool operator>(const StridedIterator& other) const {\n    return ptr_ > other.ptr_;\n  }\n\n  bool operator<=(const StridedIterator& other) const {\n    return ptr_ <= other.ptr_;\n  }\n\n  bool operator>=(const StridedIterator& other) const {\n    return ptr_ >= other.ptr_;\n  }\n\n  difference_type operator-(const StridedIterator& other) const {\n    return (ptr_ - other.ptr_) / stride_;\n  }\n\n  // Moving\n  StridedIterator& operator++() {\n    ptr_ += stride_;\n    return *this;\n  }\n\n  StridedIterator& operator--() {\n    ptr_ -= stride_;\n    return *this;\n  }\n\n  StridedIterator& operator+=(difference_type diff) {\n    ptr_ += diff * stride_;\n    return *this;\n  }\n\n  StridedIterator& operator-=(difference_type diff) {\n    ptr_ -= diff * stride_;\n    return *this;\n  }\n\n  StridedIterator operator+(difference_type diff) {\n    return StridedIterator(ptr_, stride_, diff);\n  }\n\n  StridedIterator operator-(difference_type diff) {\n    return StridedIterator(ptr_, stride_, -diff);\n  }\n\n private:\n  size_t stride_;\n  T* ptr_;\n};\n\ntemplate <typename T, typename IdxT = uint32_t>\nvoid sort(const array& in, array& out, int axis) {\n  // Copy input to output\n  CopyType ctype = in.flags().contiguous ? CopyType::Vector : CopyType::General;\n  copy(in, out, ctype);\n\n  // Get axis, shape and stride info\n  axis = axis < 0 ? axis + in.ndim() : axis;\n  size_t n_rows = in.size() / in.shape(axis);\n\n  auto remaining_shape = in.shape();\n  remaining_shape.erase(remaining_shape.begin() + axis);\n\n  auto remaining_strides = in.strides();\n  remaining_strides.erase(remaining_strides.begin() + axis);\n\n  size_t axis_stride = in.strides()[axis];\n  int axis_size = in.shape(axis);\n\n  // Perform sorting in place\n  for (int i = 0; i < n_rows; i++) {\n    size_t loc = elem_to_loc(i, remaining_shape, remaining_strides);\n    T* data_ptr = out.data<T>() + loc;\n\n    StridedIterator st(data_ptr, axis_stride, 0);\n    StridedIterator ed(data_ptr, axis_stride, axis_size);\n\n    std::stable_sort(st, ed);\n  }\n}\n\ntemplate <typename T, typename IdxT = uint32_t>\nvoid argsort(const array& in, array& out, int axis) {\n  // Allocate output\n  out.set_data(allocator::malloc_or_wait(out.nbytes()));\n\n  // Get axis, shape and stride info\n  axis = axis < 0 ? axis + in.ndim() : axis;\n  size_t n_rows = in.size() / in.shape(axis);\n\n  auto remaining_shape = in.shape();\n  remaining_shape.erase(remaining_shape.begin() + axis);\n\n  auto remaining_strides = in.strides();\n  remaining_strides.erase(remaining_strides.begin() + axis);\n\n  size_t axis_stride = in.strides()[axis];\n  int axis_size = in.shape(axis);\n\n  // Perform sorting\n  for (int i = 0; i < n_rows; i++) {\n    size_t loc = elem_to_loc(i, remaining_shape, remaining_strides);\n    const T* data_ptr = in.data<T>() + loc;\n    IdxT* idx_ptr = out.data<IdxT>() + loc;\n\n    StridedIterator st_(idx_ptr, axis_stride, 0);\n    StridedIterator ed_(idx_ptr, axis_stride, axis_size);\n\n    // Initialize with iota\n    std::iota(st_, ed_, IdxT(0));\n\n    // Sort according to vals\n    StridedIterator st(idx_ptr, axis_stride, 0);\n    StridedIterator ed(idx_ptr, axis_stride, axis_size);\n\n    std::stable_sort(st, ed, [data_ptr, axis_stride](IdxT a, IdxT b) {\n      auto v1 = data_ptr[a * axis_stride];\n      auto v2 = data_ptr[b * axis_stride];\n      return v1 < v2 || (v1 == v2 && a < b);\n    });\n  }\n}\n\ntemplate <typename T, typename IdxT = uint32_t>\nvoid partition(const array& in, array& out, int axis, int kth) {\n  // Copy input to output\n  CopyType ctype = in.flags().contiguous ? CopyType::Vector : CopyType::General;\n  copy(in, out, ctype);\n\n  // Get axis, shape and stride info\n  axis = axis < 0 ? axis + in.ndim() : axis;\n  size_t n_rows = in.size() / in.shape(axis);\n\n  auto remaining_shape = in.shape();\n  remaining_shape.erase(remaining_shape.begin() + axis);\n\n  auto remaining_strides = in.strides();\n  remaining_strides.erase(remaining_strides.begin() + axis);\n\n  size_t axis_stride = in.strides()[axis];\n  int axis_size = in.shape(axis);\n\n  kth = kth < 0 ? kth + axis_size : kth;\n\n  // Perform partition in place\n  for (int i = 0; i < n_rows; i++) {\n    size_t loc = elem_to_loc(i, remaining_shape, remaining_strides);\n    T* data_ptr = out.data<T>() + loc;\n\n    StridedIterator st(data_ptr, axis_stride, 0);\n    StridedIterator md(data_ptr, axis_stride, kth);\n    StridedIterator ed(data_ptr, axis_stride, axis_size);\n\n    std::nth_element(st, md, ed);\n  }\n}\n\ntemplate <typename T, typename IdxT = uint32_t>\nvoid argpartition(const array& in, array& out, int axis, int kth) {\n  // Allocate output\n  out.set_data(allocator::malloc_or_wait(out.nbytes()));\n\n  // Get axis, shape and stride info\n  axis = axis < 0 ? axis + in.ndim() : axis;\n  size_t n_rows = in.size() / in.shape(axis);\n\n  auto remaining_shape = in.shape();\n  remaining_shape.erase(remaining_shape.begin() + axis);\n\n  auto remaining_strides = in.strides();\n  remaining_strides.erase(remaining_strides.begin() + axis);\n\n  size_t axis_stride = in.strides()[axis];\n  int axis_size = in.shape(axis);\n\n  kth = kth < 0 ? kth + axis_size : kth;\n\n  // Perform partition\n  for (int i = 0; i < n_rows; i++) {\n    size_t loc = elem_to_loc(i, remaining_shape, remaining_strides);\n    const T* data_ptr = in.data<T>() + loc;\n    IdxT* idx_ptr = out.data<IdxT>() + loc;\n\n    StridedIterator st_(idx_ptr, axis_stride, 0);\n    StridedIterator ed_(idx_ptr, axis_stride, axis_size);\n\n    // Initialize with iota\n    std::iota(st_, ed_, IdxT(0));\n\n    // Sort according to vals\n    StridedIterator st(idx_ptr, axis_stride, 0);\n    StridedIterator md(idx_ptr, axis_stride, kth);\n    StridedIterator ed(idx_ptr, axis_stride, axis_size);\n\n    std::nth_element(st, md, ed, [data_ptr, axis_stride](IdxT a, IdxT b) {\n      auto v1 = data_ptr[a * axis_stride];\n      auto v2 = data_ptr[b * axis_stride];\n      return v1 < v2 || (v1 == v2 && a < b);\n    });\n  }\n}\n\n} // namespace\n\nvoid ArgSort::eval(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 1);\n  auto& in = inputs[0];\n\n  switch (in.dtype()) {\n    case bool_:\n      return argsort<bool>(in, out, axis_);\n    case uint8:\n      return argsort<uint8_t>(in, out, axis_);\n    case uint16:\n      return argsort<uint16_t>(in, out, axis_);\n    case uint32:\n      return argsort<uint32_t>(in, out, axis_);\n    case uint64:\n      return argsort<uint64_t>(in, out, axis_);\n    case int8:\n      return argsort<int8_t>(in, out, axis_);\n    case int16:\n      return argsort<int16_t>(in, out, axis_);\n    case int32:\n      return argsort<int32_t>(in, out, axis_);\n    case int64:\n      return argsort<int64_t>(in, out, axis_);\n    case float32:\n      return argsort<float>(in, out, axis_);\n    case float16:\n      return argsort<float16_t>(in, out, axis_);\n    case bfloat16:\n      return argsort<bfloat16_t>(in, out, axis_);\n    case complex64:\n      return argsort<complex64_t>(in, out, axis_);\n  }\n}\n\nvoid Sort::eval(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 1);\n  auto& in = inputs[0];\n\n  switch (in.dtype()) {\n    case bool_:\n      return sort<bool>(in, out, axis_);\n    case uint8:\n      return sort<uint8_t>(in, out, axis_);\n    case uint16:\n      return sort<uint16_t>(in, out, axis_);\n    case uint32:\n      return sort<uint32_t>(in, out, axis_);\n    case uint64:\n...\n", "needle_token_start": 7054, "needle_token_end": 7592, "code_context_ntokens": 16205, "output": ["```cpp\ntemplate <typename SrcT, typename DstT, typename... Args>\nvoid copy(const array& src, array& dst, CopyType ctype, Args&&... args) {\n  switch (dst.dtype()) {\n    case bool_:\n      copy<SrcT, bool>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint8:\n      copy<SrcT, uint8_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint16:\n      copy<SrcT, uint16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint32:\n      copy<SrcT, uint32_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case uint64:\n      copy<SrcT, uint64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int8:\n      copy<SrcT, int8_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int16:\n      copy<SrcT, int16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int32:\n      copy<SrcT, int32_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case int64:\n      copy<SrcT, int64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case float16:\n      copy<SrcT, float16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case float32:\n      copy<SrcT, float>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case bfloat16:\n      copy<SrcT, bfloat16_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n    case complex64:\n      copy<SrcT, complex64_t>(src, dst, ctype, std::forward<Args>(args)...);\n      break;\n  }\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "arange_set_scalars", "language": "cpp", "path": "mlx/backend/metal/primitives.cpp", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function configures a command encoder with scalar values necessary for generating a sequence of numbers starting from a specified value and increasing by a constant step.\n2. **Input**: The function accepts two scalar values of a generic type, representing the starting value of the sequence and the next value in the sequence.\n3. **Output**: There is no direct output returned by the function; however, it sets scalar values in a command encoder which are used in subsequent operations.\n4. **Procedure**: The function first sets the starting value of the sequence in the command encoder. It then calculates the step size by subtracting the starting value from the next value. This step size is also set in the command encoder.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: mlx/backend/metal/device.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <dlfcn.h>\n#include <cstdlib>\n#include <filesystem>\n#include <sstream>\n\n#define NS_PRIVATE_IMPLEMENTATION\n#define CA_PRIVATE_IMPLEMENTATION\n#define MTL_PRIVATE_IMPLEMENTATION\n\n#include \"mlx/backend/metal/device.h\"\n#include \"mlx/backend/metal/metal.h\"\n#include \"mlx/backend/metal/metal_impl.h\"\n#include \"mlx/backend/metal/mps/gemm.h\"\n#include \"mlx/backend/metal/utils.h\"\n\nnamespace fs = std::filesystem;\n\nnamespace mlx::core::metal {\n\nnamespace {\n\n// TODO nicer way to set this or possibly expose as an environment variable\nconstexpr int MAX_BUFFERS_PER_QUEUE = 12;\n\nconstexpr const char* default_mtllib_path = METAL_PATH;\n\nauto load_device() {\n  auto devices = MTL::CopyAllDevices();\n  auto device = static_cast<MTL::Device*>(devices->object(0))\n      ?: MTL::CreateSystemDefaultDevice();\n  if (!device) {\n    throw std::runtime_error(\"Failed to load device\");\n  }\n  return device;\n}\n\nstd::pair<MTL::Library*, NS::Error*> load_library_from_path(\n    MTL::Device* device,\n    const char* path) {\n  auto library = NS::String::string(path, NS::UTF8StringEncoding);\n  NS::Error* error;\n  auto lib = device->newLibrary(library, &error);\n\n  return std::make_pair(lib, error);\n}\n\n#ifdef SWIFTPM_BUNDLE\nMTL::Library* try_load_bundle(MTL::Device* device, NS::URL* url) {\n  std::string bundle_path = std::string(url->fileSystemRepresentation()) + \"/\" +\n      SWIFTPM_BUNDLE + \".bundle\";\n  auto bundle = NS::Bundle::alloc()->init(\n      NS::String::string(bundle_path.c_str(), NS::UTF8StringEncoding));\n  if (bundle != nullptr) {\n    std::string resource_path =\n        std::string(bundle->resourceURL()->fileSystemRepresentation()) + \"/\" +\n        \"default.metallib\";\n    auto [lib, error] = load_library_from_path(device, resource_path.c_str());\n    if (lib) {\n      return lib;\n    }\n  }\n  return nullptr;\n}\n#endif\n\nMTL::Library* load_library(\n    MTL::Device* device,\n    const std::string& lib_name = \"mlx\",\n    const char* lib_path = default_mtllib_path) {\n  // Firstly, search for the metallib in the same path as this binary\n  std::string first_path = get_colocated_mtllib_path(lib_name);\n  if (first_path.size() != 0) {\n    auto [lib, error] = load_library_from_path(device, first_path.c_str());\n    if (lib) {\n      return lib;\n    }\n  }\n\n#ifdef SWIFTPM_BUNDLE\n  // try to load from a swiftpm resource bundle -- scan the available bundles to\n  // find one that contains the named bundle\n  {\n    MTL::Library* library =\n        try_load_bundle(device, NS::Bundle::mainBundle()->bundleURL());\n    if (library != nullptr) {\n      return library;\n    }\n    auto bundles = NS::Bundle::allBundles();\n    for (int i = 0, c = (int)bundles->count(); i < c; i++) {\n      auto bundle = reinterpret_cast<NS::Bundle*>(bundles->object(i));\n      library = try_load_bundle(device, bundle->resourceURL());\n      if (library != nullptr) {\n        return library;\n      }\n    }\n  }\n#endif\n\n  // Couldn't find it so let's load it from default_mtllib_path\n  {\n    auto [lib, error] = load_library_from_path(device, lib_path);\n    if (!lib) {\n      std::ostringstream msg;\n      msg << error->localizedDescription()->utf8String() << \"\\n\"\n          << \"Failed to load device library from <\" << lib_path << \">\"\n          << \" or <\" << first_path << \">.\";\n      throw std::runtime_error(msg.str());\n    }\n    return lib;\n  }\n}\n\n} // namespace\n\nDevice::Device() {\n  auto pool = new_scoped_memory_pool();\n  device_ = load_device();\n  library_map_ = {{\"mlx\", load_library(device_)}};\n}\n\nDevice::~Device() {\n  auto pool = new_scoped_memory_pool();\n  for (auto& q : queue_map_) {\n    q.second->release();\n  }\n  for (auto& b : buffer_map_) {\n    b.second.second->release();\n  }\n  for (auto& e : encoder_map_) {\n    e.second->release();\n  }\n  for (auto& k : kernel_map_) {\n    k.second->release();\n  }\n  for (auto& l : library_map_) {\n    l.second->release();\n  }\n  device_->release();\n}\n\nvoid Device::new_queue(int index) {\n  auto thread_pool = metal::new_scoped_memory_pool();\n\n  // Multiple threads can ask the device for queues\n  // We lock this as a critical section for safety\n  const std::lock_guard<std::mutex> lock(mtx_);\n  auto q = device_->newCommandQueue(MAX_BUFFERS_PER_QUEUE);\n  debug_set_stream_queue_label(q, index);\n  if (!q) {\n    throw std::runtime_error(\n        \"[metal::Device] Failed to make new command queue.\");\n  }\n  queue_map_.insert({index, q});\n}\n\nint Device::get_command_buffer_ops(int index) {\n  auto bit = buffer_map_.find(index);\n  return bit->second.first;\n}\n\nvoid Device::increment_command_buffer_ops(int index) {\n  auto bit = buffer_map_.find(index);\n  bit->second.first++;\n}\n\nMTL::CommandBuffer* Device::get_command_buffer(int index) {\n  auto bit = buffer_map_.find(index);\n  return (bit == buffer_map_.end()) ? nullptr : bit->second.second;\n}\n\nMTL::CommandBuffer* Device::new_command_buffer(int index) {\n  auto qit = queue_map_.find(index);\n  if (qit == queue_map_.end()) {\n    throw std::runtime_error(\n        \"[metal::Device] Attempting to get command buffer for invalid queue.\");\n  }\n\n  auto cb = qit->second->commandBufferWithUnretainedReferences();\n\n  if (!cb) {\n    throw std::runtime_error(\n        \"[metal::Device] Unable to create new command buffer\");\n  }\n\n  // Increment ref count so the buffer is not garbage collected\n  cb->retain();\n\n  return buffer_map_.insert({index, {0, cb}}).first->second.second;\n}\n\nvoid Device::commit_command_buffer(int index) {\n  auto bit = buffer_map_.find(index);\n  bit->second.second->commit();\n  bit->second.second->release();\n  buffer_map_.erase(bit);\n}\n\nvoid Device::end_encoding(int index) {\n  auto eit = encoder_map_.find(index);\n  if (eit != encoder_map_.end()) {\n    eit->second->endEncoding();\n    eit->second->release();\n    encoder_map_.erase(eit);\n  }\n}\n\nCommandEncoder& Device::get_command_encoder(int index) {\n  auto eit = encoder_map_.find(index);\n  if (eit == encoder_map_.end()) {\n    auto cb = get_command_buffer(index);\n    auto compute_encoder =\n        cb->computeCommandEncoder(MTL::DispatchTypeConcurrent);\n    // Increment ref count so the buffer is not garbage collected\n    compute_encoder->retain();\n    eit = encoder_map_.emplace(index, CommandEncoder{compute_encoder}).first;\n  }\n  return eit->second;\n}\n\nvoid Device::register_library(\n    const std::string& lib_name,\n    const std::string& lib_path) {\n  if (auto it = library_map_.find(lib_name); it == library_map_.end()) {\n    auto new_lib = load_library(device_, lib_name, lib_path.c_str());\n    library_map_.insert({lib_name, new_lib});\n  }\n}\n\nvoid Device::register_library(\n    const std::string& lib_name,\n    const std::function<std::string(const std::string&)>& lib_path_func) {\n  if (auto it = library_map_.find(lib_name); it == library_map_.end()) {\n    std::string new_lib_path = lib_path_func(lib_name);\n    auto new_lib = load_library(device_, lib_name, new_lib_path.c_str());\n    library_map_.insert({lib_name, new_lib});\n  }\n}\n\nMTL::Library* Device::get_library_cache_(const std::string& lib_name) {\n  // Search for cached metal lib\n  MTL::Library* mtl_lib;\n  if (auto it = library_map_.find(lib_name); it != library_map_.end()) {\n    mtl_lib = it->second;\n  } else { // Look for metallib alongside library\n    register_library(lib_name);\n    mtl_lib = library_map_[lib_name];\n  }\n\n  return mtl_lib;\n}\n\nMTL::Library* Device::get_library_(const std::string& source_string) {\n  auto pool = new_scoped_memory_pool();\n\n  auto ns_code =\n      NS::String::string(source_string.c_str(), NS::ASCIIStringEncoding);\n\n  NS::Error* error = nullptr;\n  auto mtl_lib = device_->newLibrary(ns_code, nullptr, &error);\n\n  // Throw error if unable to compile library\n  if (!mtl_lib) {\n    std::ostringstream msg;\n    msg << \"[metal::Device] Unable to load build metal library from source\"\n        << \"\\n\";\n    if (error) {\n      msg << error->localizedDescription()->utf8String() << \"\\n\";\n    }\n    throw std::runtime_error(msg.str());\n  }\n\n  return mtl_lib;\n}\n\nMTL::Library* Device::get_library_(const MTL::StitchedLibraryDescriptor* desc) {\n  auto pool = new_scoped_memory_pool();\n\n  NS::Error* error = nullptr;\n  auto mtl_lib = device_->newLibrary(desc, &error);\n\n  // Throw error if unable to compile library\n  if (!mtl_lib) {\n    std::ostringstream msg;\n    msg << \"[metal::Device] Unable to load build stitched metal library\"\n        << \"\\n\";\n    if (error) {\n      msg << error->localizedDescription()->utf8String() << \"\\n\";\n    }\n    throw std::runtime_error(msg.str());\n  }\n\n  return mtl_lib;\n}\n\nMTL::Function* Device::get_function_(\n    const std::string& name,\n    MTL::Library* mtl_lib) {\n  // Pull kernel from library\n  auto ns_name = NS::String::string(name.c_str(), NS::ASCIIStringEncoding);\n  auto mtl_function = mtl_lib->newFunction(ns_name);\n\n  return mtl_function;\n}\n\nMTL::Function* Device::get_function_(\n    const std::string& name,\n    const std::string& specialized_name,\n    const MTLFCList& func_consts,\n    MTL::Library* mtl_lib) {\n  if (func_consts.empty() && (specialized_name == name)) {\n    return get_function_(name, mtl_lib);\n  }\n\n  // Prepare function constants\n  auto mtl_func_consts = MTL::FunctionConstantValues::alloc()->init();\n\n  for (auto [value, type, index] : func_consts) {\n    mtl_func_consts->setConstantValue(value, type, index);\n  }\n\n  // Prepare function desc\n  auto desc = MTL::FunctionDescriptor::functionDescriptor();\n  desc->setName(NS::String::string(name.c_str(), NS::ASCIIStringEncoding));\n  desc->setSpecializedName(\n      NS::String::string(specialized_name.c_str(), NS::ASCIIStringEncoding));\n  desc->setConstantValues(mtl_func_consts);\n\n  // Pull kernel from library\n  NS::Error* error = nullptr;\n  auto mtl_function = mtl_lib->newFunction(desc, &error);\n\n  // Throw error if unable to build metal function\n  if (!mtl_function) {\n    std::ostringstream msg;\n    msg << \"[metal::Device] Unable to load function \" << name << \"\\n\";\n    if (error) {\n      msg << error->localizedDescription()->utf8String() << \"\\n\";\n    }\n    throw std::runtime_error(msg.str());\n  }\n\n  mtl_func_consts->release();\n  desc->release();\n\n  return mtl_function;\n}\n\nMTL::ComputePipelineState* Device::get_kernel_(\n    const std::string& name,\n    const MTL::Function* mtl_function) {\n  // Compile kernel to compute pipeline\n  NS::Error* error = nullptr;\n  MTL::ComputePipelineState* kernel;\n\n  if (mtl_function) {\n    kernel = device_->newComputePipelineState(mtl_function, &error);\n  }\n\n  // Throw error if unable to compile metal function\n  if (!mtl_function || !kernel) {\n    std::ostringstream msg;\n    msg << \"[metal::Device] Unable to load kernel \" << name << \"\\n\";\n    if (error) {\n      msg << error->localizedDescription()->utf8String() << \"\\n\";\n    }\n    throw std::runtime_error(msg.str());\n  }\n\n  return kernel;\n}\n\nMTL::ComputePipelineState* Device::get_kernel_(\n    const std::string& name,\n    const MTL::Function* mtl_function,\n    const MTL::LinkedFunctions* linked_functions) {\n  // Check inputs\n  if (!linked_functions) {\n    return get_kernel_(name, mtl_function);\n  }\n\n  if (!mtl_function) {\n    std::ostringstream msg;\n    msg << \"[metal::Device] Unable to load kernel \" << name << \"\\n\";\n    throw std::runtime_error(msg.str());\n  }\n\n  // Prepare compute pipeline state descriptor\n  auto desc = MTL::ComputePipelineDescriptor::alloc()->init();\n  desc->setComputeFunction(mtl_function);\n  desc->setLinkedFunctions(linked_functions);\n\n  // Compile kernel to compute pipeline\n  NS::Error* error = nullptr;\n  auto kernel = device_->newComputePipelineState(\n      desc, MTL::PipelineOptionNone, nullptr, &error);\n\n  // Throw error if unable to compile metal function\n  if (!kernel) {\n    std::ostringstream msg;\n    msg << \"[metal::Device] Unable to load kernel \" << name << \"\\n\";\n    if (error) {\n      msg << error->localizedDescription()->utf8String() << \"\\n\";\n    }\n    throw std::runtime_error(msg.str());\n  }\n\n  return kernel;\n}\n\n...\n// Path: mlx/backend/metal/primitives.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n#include <algorithm>\n#include <cassert>\n#include <numeric>\n#include <sstream>\n\n#include \"mlx/backend/common/binary.h\"\n#include \"mlx/backend/common/ternary.h\"\n#include \"mlx/backend/metal/copy.h\"\n#include \"mlx/backend/metal/device.h\"\n#include \"mlx/backend/metal/kernels/defines.h\"\n#include \"mlx/backend/metal/utils.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\nconstexpr int METAL_MAX_INDEX_ARRAYS = 10;\n\nvoid binary_op(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs,\n    const std::string op) {\n  assert(inputs.size() == 2);\n  auto& a = inputs[0];\n  auto& b = inputs[1];\n  auto bopt = get_binary_op_type(a, b);\n  set_binary_op_output_data(a, b, outputs[0], bopt, true);\n  set_binary_op_output_data(a, b, outputs[1], bopt, true);\n\n  auto& out = outputs[0];\n  if (out.size() == 0) {\n    return;\n  }\n\n  // Try to collapse contiguous dims\n  auto [shape, strides] = collapse_contiguous_dims(a, b, out);\n  auto& strides_a = strides[0];\n  auto& strides_b = strides[1];\n  auto& strides_out = strides[2];\n\n  std::ostringstream kname;\n  switch (bopt) {\n    case BinaryOpType::ScalarScalar:\n      kname << \"ss\";\n      break;\n    case BinaryOpType::ScalarVector:\n      kname << \"sv\";\n      break;\n    case BinaryOpType::VectorScalar:\n      kname << \"vs\";\n      break;\n    case BinaryOpType::VectorVector:\n      kname << \"vv\";\n      break;\n    case BinaryOpType::General:\n      kname << \"g\";\n      break;\n  }\n  kname << op << type_to_name(a);\n  if (bopt == BinaryOpType::General &&\n      shape.size() <= MAX_BINARY_SPECIALIZED_DIMS) {\n    kname << \"_\" << shape.size();\n  }\n\n  auto& s = out.primitive().stream();\n  auto& d = metal::device(s.device);\n  auto kernel = d.get_kernel(kname.str());\n  auto& compute_encoder = d.get_command_encoder(s.index);\n  compute_encoder->setComputePipelineState(kernel);\n  // - If a is donated it goes to the first output\n  // - If b is donated it goes to the first output if a was not donated\n  //   otherwise it goes to the second output\n  bool donate_a = a.data_shared_ptr() == nullptr;\n  bool donate_b = b.data_shared_ptr() == nullptr;\n  compute_encoder.set_input_array(donate_a ? outputs[0] : a, 0);\n  compute_encoder.set_input_array(\n      donate_b ? (donate_a ? outputs[1] : outputs[0]) : b, 1);\n  compute_encoder.set_output_array(outputs[0], 2);\n  compute_encoder.set_output_array(outputs[1], 3);\n\n  if (bopt == BinaryOpType::General) {\n    auto ndim = shape.size();\n    if (ndim > 3) {\n      compute_encoder->setBytes(shape.data(), ndim * sizeof(int), 4);\n      compute_encoder->setBytes(strides_a.data(), ndim * sizeof(size_t), 5);\n      compute_encoder->setBytes(strides_b.data(), ndim * sizeof(size_t), 6);\n    } else {\n      // The shape is implicit in the grid for <= 3D\n      compute_encoder->setBytes(strides_a.data(), ndim * sizeof(size_t), 4);\n      compute_encoder->setBytes(strides_b.data(), ndim * sizeof(size_t), 5);\n    }\n\n    if (ndim > MAX_BINARY_SPECIALIZED_DIMS) {\n      compute_encoder->setBytes(&ndim, sizeof(int), 7);\n    }\n\n    // Launch up to 3D grid of threads\n    size_t dim0 = ndim > 0 ? shape[ndim - 1] : 1;\n    size_t dim1 = ndim > 1 ? shape[ndim - 2] : 1;\n    size_t rest = out.size() / (dim0 * dim1);\n    NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n    if (thread_group_size != 1024) {\n      throw std::runtime_error(\"[Metal::binary] Must use 1024 sized block\");\n    }\n    auto group_dims = get_block_dims(dim0, dim1, rest);\n    MTL::Size grid_dims = MTL::Size(dim0, dim1, rest);\n    compute_encoder->dispatchThreads(grid_dims, group_dims);\n  } else {\n    // Launch a 1D grid of threads\n    size_t nthreads = out.data_size();\n    MTL::Size grid_dims = MTL::Size(nthreads, 1, 1);\n    NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n    if (thread_group_size > nthreads) {\n      thread_group_size = nthreads;\n    }\n    MTL::Size group_dims = MTL::Size(thread_group_size, 1, 1);\n    compute_encoder->dispatchThreads(grid_dims, group_dims);\n  }\n}\n\nvoid binary_op(\n    const std::vector<array>& inputs,\n    array& out,\n    const std::string op) {\n  assert(inputs.size() == 2);\n  auto& a = inputs[0];\n  auto& b = inputs[1];\n  auto bopt = get_binary_op_type(a, b);\n  set_binary_op_output_data(a, b, out, bopt, true);\n  if (out.size() == 0) {\n    return;\n  }\n\n  // Try to collapse contiguous dims\n  auto [shape, strides] = collapse_contiguous_dims(a, b, out);\n  auto& strides_a = strides[0];\n  auto& strides_b = strides[1];\n  auto& strides_out = strides[2];\n\n  std::ostringstream kname;\n  switch (bopt) {\n    case BinaryOpType::ScalarScalar:\n      kname << \"ss\";\n      break;\n    case BinaryOpType::ScalarVector:\n      kname << \"sv\";\n      break;\n    case BinaryOpType::VectorScalar:\n      kname << \"vs\";\n      break;\n    case BinaryOpType::VectorVector:\n      kname << \"vv\";\n      break;\n    case BinaryOpType::General:\n      kname << \"g\";\n      break;\n  }\n  kname << op << type_to_name(a);\n  if (bopt == BinaryOpType::General &&\n      shape.size() <= MAX_BINARY_SPECIALIZED_DIMS) {\n    kname << \"_\" << shape.size();\n  }\n\n  auto& s = out.primitive().stream();\n  auto& d = metal::device(s.device);\n  auto kernel = d.get_kernel(kname.str());\n  auto& compute_encoder = d.get_command_encoder(s.index);\n  compute_encoder->setComputePipelineState(kernel);\n  bool donate_a = a.data_shared_ptr() == nullptr;\n  bool donate_b = b.data_shared_ptr() == nullptr;\n  compute_encoder.set_input_array(donate_a ? out : a, 0);\n  compute_encoder.set_input_array(donate_b ? out : b, 1);\n  compute_encoder.set_output_array(out, 2);\n\n  if (bopt == BinaryOpType::General) {\n    auto ndim = shape.size();\n    if (ndim > 3) {\n      compute_encoder->setBytes(shape.data(), ndim * sizeof(int), 3);\n      compute_encoder->setBytes(strides_a.data(), ndim * sizeof(size_t), 4);\n      compute_encoder->setBytes(strides_b.data(), ndim * sizeof(size_t), 5);\n    } else {\n      // The shape is implicit in the grid for <= 3D\n      compute_encoder->setBytes(strides_a.data(), ndim * sizeof(size_t), 3);\n      compute_encoder->setBytes(strides_b.data(), ndim * sizeof(size_t), 4);\n    }\n\n    if (ndim > MAX_BINARY_SPECIALIZED_DIMS) {\n      compute_encoder->setBytes(&ndim, sizeof(int), 6);\n    }\n\n    // Launch up to 3D grid of threads\n    size_t dim0 = ndim > 0 ? shape[ndim - 1] : 1;\n    size_t dim1 = ndim > 1 ? shape[ndim - 2] : 1;\n    size_t rest = out.size() / (dim0 * dim1);\n    NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n    if (thread_group_size != 1024) {\n      throw std::runtime_error(\"[Metal::binary] Must use 1024 sized block\");\n    }\n    auto group_dims = get_block_dims(dim0, dim1, rest);\n    MTL::Size grid_dims = MTL::Size(dim0, dim1, rest);\n    compute_encoder->dispatchThreads(grid_dims, group_dims);\n  } else {\n    // Launch a 1D grid of threads\n    size_t nthreads =\n        bopt == BinaryOpType::General ? out.size() : out.data_size();\n    MTL::Size grid_dims = MTL::Size(nthreads, 1, 1);\n    NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n    if (thread_group_size > nthreads) {\n      thread_group_size = nthreads;\n    }\n    MTL::Size group_dims = MTL::Size(thread_group_size, 1, 1);\n    compute_encoder->dispatchThreads(grid_dims, group_dims);\n  }\n}\n\nvoid ternary_op(\n    const std::vector<array>& inputs,\n    array& out,\n    const std::string op) {\n  assert(inputs.size() == 3);\n  auto& a = inputs[0];\n  auto& b = inputs[1];\n  auto& c = inputs[2];\n  TernaryOpType topt = get_ternary_op_type(a, b, c);\n  set_ternary_op_output_data(a, b, c, out, topt, true /* donate_with_move */);\n\n  if (out.size() == 0) {\n    return;\n  }\n\n  // Try to collapse contiguous dims\n  auto [shape, strides] = collapse_contiguous_dims(a, b, c, out);\n  auto& strides_a = strides[0];\n  auto& strides_b = strides[1];\n  auto& strides_c = strides[2];\n  auto& strides_out = strides[3];\n\n  std::ostringstream kname;\n  if (topt == TernaryOpType::General) {\n    kname << \"g\";\n    kname << op << type_to_name(b);\n    if (shape.size() <= MAX_BINARY_SPECIALIZED_DIMS) {\n      kname << \"_\" << shape.size();\n    }\n  } else {\n    kname << \"v\";\n    kname << op << type_to_name(b);\n  }\n\n  auto& s = out.primitive().stream();\n  auto& d = metal::device(s.device);\n  auto kernel = d.get_kernel(kname.str());\n  auto& compute_encoder = d.get_command_encoder(s.index);\n  compute_encoder->setComputePipelineState(kernel);\n  compute_encoder.set_input_array(a, 0);\n  compute_encoder.set_input_array(b, 1);\n  compute_encoder.set_input_array(c, 2);\n  compute_encoder.set_output_array(out, 3);\n\n  if (topt == TernaryOpType::General) {\n    auto ndim = shape.size();\n    if (ndim > 3) {\n      compute_encoder->setBytes(shape.data(), ndim * sizeof(int), 4);\n      compute_encoder->setBytes(strides_a.data(), ndim * sizeof(size_t), 5);\n      compute_encoder->setBytes(strides_b.data(), ndim * sizeof(size_t), 6);\n      compute_encoder->setBytes(strides_c.data(), ndim * sizeof(size_t), 7);\n\n      if (ndim > MAX_BINARY_SPECIALIZED_DIMS) {\n        compute_encoder->setBytes(&ndim, sizeof(int), 8);\n      }\n    } else {\n      // The shape is implicit in the grid for <= 3D\n      compute_encoder->setBytes(strides_a.data(), ndim * sizeof(size_t), 4);\n      compute_encoder->setBytes(strides_b.data(), ndim * sizeof(size_t), 5);\n      compute_encoder->setBytes(strides_c.data(), ndim * sizeof(size_t), 6);\n    }\n\n    // Launch up to 3D grid of threads\n    size_t dim0 = ndim > 0 ? shape[ndim - 1] : 1;\n    size_t dim1 = ndim > 1 ? shape[ndim - 2] : 1;\n    size_t rest = out.size() / (dim0 * dim1);\n    NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n    if (thread_group_size != 1024) {\n      throw std::runtime_error(\"[Metal::binary] Must use 1024 sized block\");\n    }\n    MTL::Size group_dims = get_block_dims(dim0, dim1, rest);\n    MTL::Size grid_dims = MTL::Size(dim0, dim1, rest);\n    compute_encoder->dispatchThreads(grid_dims, group_dims);\n  } else {\n    // Launch a 1D grid of threads\n    size_t nthreads = out.data_size();\n    MTL::Size grid_dims = MTL::Size(nthreads, 1, 1);\n    NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n    if (thread_group_size > nthreads) {\n      thread_group_size = nthreads;\n    }\n    MTL::Size group_dims = MTL::Size(thread_group_size, 1, 1);\n    compute_encoder->dispatchThreads(grid_dims, group_dims);\n  }\n}\n\nvoid unary_op(\n    const std::vector<array>& inputs,\n    array& out,\n    const std::string op) {\n  auto& in = inputs[0];\n  bool contig = in.flags().contiguous;\n  if (contig) {\n    if (in.is_donatable() && in.itemsize() == out.itemsize()) {\n      out.move_shared_buffer(in);\n    } else {\n      out.set_data(\n          allocator::malloc_or_wait(in.data_size() * out.itemsize()),\n          in.data_size(),\n          in.strides(),\n          in.flags());\n    }\n  } else {\n    out.set_data(allocator::malloc_or_wait(out.nbytes()));\n  }\n  if (in.size() == 0) {\n    return;\n  }\n\n  auto& s = out.primitive().stream();\n  auto& d = metal::device(s.device);\n  std::string tname = type_to_name(in);\n  std::string opt_name = contig ? \"v\" : \"g\";\n  auto kernel = d.get_kernel(opt_name + op + tname);\n\n  size_t nthreads = contig ? in.data_size() : in.size();\n  MTL::Size grid_dims = MTL::Size(nthreads, 1, 1);\n  NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n  if (thread_group_size > nthreads) {\n    thread_group_size = nthreads;\n  }\n  MTL::Size group_dims = MTL::Size(thread_group_size, 1, 1);\n\n  auto& compute_encoder = d.get_command_encoder(s.index);\n  compute_encoder->setComputePipelineState(kernel);\n  compute_encoder.set_input_array(\n      in.data_shared_ptr() == nullptr ? out : in, 0);\n  compute_encoder.set_output_array(out, 1);\n  if (!contig) {\n    compute_encoder->setBytes(in.shape().data(), in.ndim() * sizeof(int), 2);\n    compute_encoder->setBytes(\n        in.strides().data(), in.ndim() * sizeof(size_t), 3);\n    int ndim = in.ndim();\n    compute_encoder->setBytes(&ndim, sizeof(int), 4);\n  }\n  compute_encoder->dispatchThreads(grid_dims, group_dims);\n}\n\n} // namespace\n\nvoid Abs::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"abs\");\n}\n\nvoid Add::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"add\");\n}\n\ntemplate <typename T>\nv\noid arange_set_scalars(T start, T next, CommandEncoder& enc) {\n  enc->setBytes(&start, sizeof(T), 0);\n  T step = next - start;\n  enc->setBytes(&step, sizeof(T), 1);\n}\n\nvoid Arange::eval_gpu(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 0);\n  out.set_data(allocator::malloc_or_wait(out.nbytes()));\n  if (out.size() == 0) {\n    return;\n  }\n  auto& s = stream();\n  auto& d = metal::device(s.device);\n  auto kernel = d.get_kernel(\"arange\" + type_to_name(out));\n  size_t nthreads = out.size();\n  MTL::Size grid_dims = MTL::Size(nthreads, 1, 1);\n  MTL::Size group_dims = MTL::Size(\n      std::min(nthreads, kernel->maxTotalThreadsPerThreadgroup()), 1, 1);\n  auto& compute_encoder = d.get_command_encoder(s.index);\n  compute_encoder->setComputePipelineState(kernel);\n\n  switch (out.dtype()) {\n    case bool_: // unsupported\n      throw std::runtime_error(\"[Arange::eval_gpu] Does not support bool\");\n    case uint8:\n      arange_set_scalars<uint8_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case uint16:\n      arange_set_scalars<uint16_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case uint32:\n      arange_set_scalars<uint32_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case uint64:\n      arange_set_scalars<uint64_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case int8:\n      arange_set_scalars<int8_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case int16:\n      arange_set_scalars<int16_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case int32:\n      arange_set_scalars<int32_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case int64:\n      arange_set_scalars<int64_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case float16:\n      arange_set_scalars<float16_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case float32:\n      arange_set_scalars<float>(start_, start_ + step_, compute_encoder);\n      break;\n    case bfloat16:\n      arange_set_scalars<bfloat16_t>(start_, start_ + step_, compute_encoder);\n      break;\n    case complex64:\n      throw std::runtime_error(\"[Arange::eval_gpu] Does not support complex64\");\n  }\n\n  compute_encoder.set_output_array(out, 2);\n  compute_encoder->dispatchThreads(grid_dims, group_dims);\n}\n\nvoid ArcCos::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"arccos\");\n}\n\nvoid ArcCosh::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"arccosh\");\n}\n\nvoid ArcSin::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"arcsin\");\n}\n\nvoid ArcSinh::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"arcsinh\");\n}\n\nvoid ArcTan::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"arctan\");\n}\n\nvoid ArcTanh::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"arctanh\");\n}\n\nvoid ArgReduce::eval_gpu(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 1);\n  auto& in = inputs[0];\n  out.set_data(allocator::malloc_or_wait(out.nbytes()));\n  auto& s = stream();\n  auto& d = metal::device(s.device);\n  std::string op_name;\n  switch (reduce_type_) {\n    case ArgReduce::ArgMin:\n      op_name = \"argmin_\";\n      break;\n    case ArgReduce::ArgMax:\n      op_name = \"argmax_\";\n      break;\n  }\n\n  // Prepare the shapes, strides and axis arguments.\n  std::vector<size_t> in_strides = in.strides();\n  std::vector<int> shape = in.shape();\n  std::vector<size_t> out_strides = out.strides();\n  size_t axis_stride = in_strides[axis_];\n  size_t axis_size = shape[axis_];\n  if (out_strides.size() == in_strides.size()) {\n    out_strides.erase(out_strides.begin() + axis_);\n  }\n  in_strides.erase(in_strides.begin() + axis_);\n  shape.erase(shape.begin() + axis_);\n  size_t ndim = shape.size();\n\n  // ArgReduce\n  int simd_size = 32;\n  int n_reads = 4;\n  auto& compute_encoder = d.get_command_encoder(s.index);\n  {\n    auto kernel = d.get_kernel(op_name + type_to_name(in));\n    NS::UInteger thread_group_size = std::min(\n        (axis_size + n_reads - 1) / n_reads,\n        kernel->maxTotalThreadsPerThreadgroup());\n    // round up to the closest number divisible by simd_size\n    thread_group_size =\n        (thread_group_size + simd_size - 1) / simd_size * simd_size;\n    assert(thread_group_size <= kernel->maxTotalThreadsPerThreadgroup());\n\n    size_t n_threads = out.size() * thread_group_size;\n    MTL::Size grid_dims = MTL::Size(n_threads, 1, 1);\n    MTL::Size group_dims = MTL::Size(thread_group_size, 1, 1);\n    compute_encoder->setComputePipelineState(kernel);\n    compute_encoder.set_input_array(in, 0);\n    compute_encoder.set_output_array(out, 1);\n    if (ndim == 0) {\n      // Pass place holders so metal doesn't complain\n      int shape_ = 0;\n      size_t stride_ = 0;\n      compute_encoder->setBytes(&shape_, sizeof(int), 2);\n      compute_encoder->setBytes(&stride_, sizeof(size_t), 3);\n      compute_encoder->setBytes(&stride_, sizeof(size_t), 4);\n    } else {\n      compute_encoder->setBytes(shape.data(), ndim * sizeof(int), 2);\n      compute_encoder->setBytes(in_strides.data(), ndim * sizeof(size_t), 3);\n      compute_encoder->setBytes(out_strides.data(), ndim * sizeof(size_t), 4);\n    }\n    compute_encoder->setBytes(&ndim, sizeof(size_t), 5);\n    compute_encoder->setBytes(&axis_stride, sizeof(size_t), 6);\n    compute_encoder->setBytes(&axis_size, sizeof(size_t), 7);\n    compute_encoder->dispatchThreads(grid_dims, group_dims);\n  }\n}\n\nvoid AsType::eval_gpu(const std::vector<array>& inputs, array& out) {\n  CopyType ctype =\n      inputs[0].flags().contiguous ? CopyType::Vector : CopyType::General;\n  copy_gpu(inputs[0], out, ctype);\n}\n\nvoid AsStrided::eval_gpu(const std::vector<array>& inputs, array& out) {\n  eval(inputs, out);\n}\n\nvoid Broadcast::eval_gpu(const std::vector<array>& inputs, array& out) {\n  eval(inputs, out);\n}\n\nvoid Concatenate::eval_gpu(const std::vector<array>& inputs, array& out) {\n  std::vector<int> sizes;\n  sizes.push_back(0);\n  for (auto& p : inputs) {\n    sizes.push_back(p.shape(axis_));\n  }\n  std::partial_sum(sizes.cbegin(), sizes.cend(), sizes.begin());\n\n  out.set_data(allocator::malloc_or_wait(out.nbytes()));\n\n  auto strides = out.strides();\n  auto flags = out.flags();\n  flags.row_contiguous = false;\n  flags.col_contiguous = false;\n  flags.contiguous = false;\n  auto& d = metal::device(stream().device);\n  auto& compute_encoder = d.get_command_encoder(stream().index);\n  auto concurrent_ctx = compute_encoder.start_concurrent();\n  for (int i = 0; i < inputs.size(); i++) {\n    array out_slice(inputs[i].shape(), out.dtype(), nullptr, {});\n    size_t data_offset = strides[axis_] * sizes[i];\n    out_slice.copy_shared_buffer(\n        out, strides, flags, out_slice.size(), data_offset);\n    copy_gpu_inplace(inputs[i], out_slice, CopyType::GeneralGeneral, stream());\n  }\n}\n\nvoid Copy::eval_gpu(const std::vector<array>& inputs, array& out) {\n  eval(inputs, out);\n}\n\nvoid Cos::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"cos\");\n}\n\nvoid Cosh::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"cosh\");\n}\n\nvoid CustomVJP::eval_gpu(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  eval(inputs, outputs);\n}\n\nvoid Depends::eval_gpu(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  eval(inputs, outputs);\n}\n\nvoid Divide::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"div\");\n}\n\nvoid DivMod::eval_gpu(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  binary_op(inputs, outputs, \"divmod\");\n}\n\nvoid Remainder::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"rem\");\n}\n\nvoid Equal::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, equal_nan_ ? \"naneq\" : \"eq\");\n}\n\nvoid Erf::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"erf\");\n}\n\nvoid ErfInv::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"erfinv\");\n}\n\nvoid Exp::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"exp\");\n}\n\nvoid Expm1::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"expm1\");\n}\n\nvoid Full::eval_gpu(const std::vector<array>& inputs, array& out) {\n  auto in = inputs[0];\n  CopyType ctype;\n  if (in.data_size() == 1) {\n    ctype = CopyType::Scalar;\n  } else if (in.flags().contiguous) {\n    ctype = CopyType::Vector;\n  } else {\n    ctype = CopyType::General;\n  }\n  copy_gpu(in, out, ctype);\n}\n\nvoid Greater::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"ge\");\n}\n\nvoid GreaterEqual::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"geq\");\n}\n\nvoid Less::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"le\");\n}\n\nvoid LessEqual::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"leq\");\n}\n\nvoid Load::eval_gpu(const std::vector<array>& inputs, array& out) {\n  eval(inputs, out);\n}\n\nvoid Log::eval_gpu(const std::vector<array>& inputs, array& out) {\n  switch (base_) {\n    case Base::e:\n      unary_op(inputs, out, \"log\");\n      break;\n    case Base::two:\n      unary_op(inputs, out, \"log2\");\n      break;\n    case Base::ten:\n      unary_op(inputs, out, \"log10\");\n      break;\n  }\n}\n\nvoid Log1p::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"log1p\");\n}\n\nvoid LogicalNot::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"lnot\");\n}\n\nvoid LogicalAnd::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(\n      inputs,\n      out,\n      \"land\"); // Assume \"land\" is the operation identifier for logical AND\n}\n\nvoid LogicalOr::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(\n      inputs,\n      out,\n      \"lor\"); // Assume \"lor\" is the operation identifier for logical OR\n}\n\nvoid LogAddExp::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"lae\");\n}\n\nvoid Maximum::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"max\");\n}\n\nvoid Minimum::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"min\");\n}\n\nvoid NumberOfElements::eval_gpu(const std::vector<array>& inputs, array& out) {\n  eval(inputs, out);\n}\n\nvoid Floor::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"floor\");\n}\n\nvoid Ceil::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"ceil\");\n}\n\nvoid Multiply::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"mul\");\n}\n\nvoid Select::eval_gpu(const std::vector<array>& inputs, array& out) {\n  ternary_op(inputs, out, \"select\");\n}\n\nvoid Negative::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"neg\");\n}\n\nvoid NotEqual::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"neq\");\n}\n\nvoid Pad::eval_gpu(const std::vector<array>& inputs, array& out) {\n  // Inputs must be base input array and scalar val array\n  assert(inputs.size() == 2);\n  auto& in = inputs[0];\n  auto& val = inputs[1];\n\n  // Padding value must be a scalar\n  assert(val.size() == 1);\n\n  // Padding value, input and output must be of the same type\n  assert(val.dtype() == in.dtype() && in.dtype() == out.dtype());\n\n  // Fill output with val\n  copy_gpu(val, out, CopyType::Scalar, stream());\n\n  // Find offset for start of input values\n  size_t data_offset = 0;\n  for (int i = 0; i < axes_.size(); i++) {\n    auto ax = axes_[i] < 0 ? out.ndim() + axes_[i] : axes_[i];\n    data_offset += out.strides()[ax] * low_pad_size_[i];\n  }\n\n  // Extract slice from output where input will be pasted\n  array out_slice(in.shape(), out.dtype(), nullptr, {});\n  out_slice.copy_shared_buffer(\n      out, out.strides(), out.flags(), out_slice.size(), data_offset);\n\n  // Copy input values into the slice\n  copy_gpu_inplace(in, out_slice, CopyType::GeneralGeneral, stream());\n}\n\nvoid Power::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"pow\");\n}\n\nvoid RandomBits::eval_gpu(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 1);\n\n  // keys has shape (N1, ..., NK, 2)\n  // out has shape (N1, ..., NK, M1, M2, ...)\n  auto& keys = inputs[0];\n  size_t num_keys = keys.size() / 2;\n\n  size_t elems_per_key = out.size() / num_keys;\n  size_t bytes_per_key = out.itemsize() * elems_per_key;\n  out.set_data(allocator::malloc_or_wait(out.nbytes()));\n  if (out.size() == 0) {\n    return;\n  }\n\n  size_t out_per_key = (bytes_per_key + 4 - 1) / 4;\n  size_t half_size = out_per_key / 2;\n  bool odd = out_per_key % 2;\n\n  auto& s = stream();\n  auto& d = metal::device(s.device);\n  std::string kname = keys.flags().row_contiguous ? \"rbitsc\" : \"rbits\";\n  auto kernel = d.get_kernel(kname);\n\n  // organize into grid nkeys x elem_per_key\n  MTL::Size grid_dims = MTL::Size(num_keys, half_size + odd, 1);\n  NS::UInteger thread_group_size = kernel->maxTotalThreadsPerThreadgroup();\n  MTL::Size group_dims = MTL::Size(thread_group_size, 1, 1);\n  auto& compute_encoder = d.get_command_encoder(s.index);\n  compute_encoder->setComputePipelineState(kernel);\n  compute_encoder.set_input_array(keys, 0);\n  compute_encoder.set_output_array(out, 1);\n  compute_encoder->setBytes(&odd, sizeof(bool), 2);\n  compute_encoder->setBytes(&bytes_per_key, sizeof(size_t), 3);\n\n  if (!keys.flags().row_contiguous) {\n    int ndim = keys.ndim();\n    compute_encoder->setBytes(&ndim, sizeof(int), 4);\n    compute_encoder->setBytes(\n        keys.shape().data(), keys.ndim() * sizeof(int), 5);\n    compute_encoder->setBytes(\n        keys.strides().data(), keys.ndim() * sizeof(size_t), 6);\n  }\n\n  compute_encoder->dispatchThreads(grid_dims, group_dims);\n}\n\nvoid Reshape::eval_gpu(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 1);\n  const auto& in = inputs[0];\n\n  auto [copy_necessary, out_strides] = prepare_reshape(in, out);\n\n  if (copy_necessary) {\n    copy_gpu(in, out, CopyType::General);\n  } else {\n    shared_buffer_reshape(in, out_strides, out);\n  }\n}\n\nvoid Round::eval_gpu(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 1);\n  const auto& in = inputs[0];\n  if (issubdtype(in.dtype(), inexact)) {\n    unary_op(inputs, out, \"round\");\n  } else {\n    // No-op integer types\n    out.copy_shared_buffer(in);\n  }\n}\n\nvoid Sigmoid::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"sigmoid\");\n}\n\nvoid Sign::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"sign\");\n}\n\nvoid Sin::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"sin\");\n}\n\nvoid Sinh::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"sinh\");\n}\n\nvoid Split::eval_gpu(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  eval(inputs, outputs);\n}\n\nvoid Square::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"square\");\n}\n\nvoid Sqrt::eval_gpu(const std::vector<array>& inputs, array& out) {\n  if (recip_) {\n    unary_op(inputs, out, \"rsqrt\");\n  } else {\n    unary_op(inputs, out, \"sqrt\");\n  }\n}\n\nvoid Slice::eval_gpu(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 1);\n  if (out.size() == 0) {\n    out.set_data(nullptr);\n    return;\n  }\n\n  auto& in = inputs[0];\n\n  // Calculate out strides, initial offset and if copy needs to be made\n  auto [copy_needed, data_offset, inp_strides] = prepare_slice(in);\n\n  // Do copy if needed\n  if (copy_needed) {\n    out.set_data(allocator::malloc_or_wait(out.nbytes()));\n    std::vector<int64_t> ostrides{out.strides().begin(), out.strides().end()};\n    copy_gpu_inplace(\n        /* const array& in = */ in,\n        /* array& out = */ out,\n        /* const std::vector<int>& data_shape = */ out.shape(),\n        /* const std::vector<stride_t>& i_strides = */ inp_strides,\n        /* const std::vector<stride_t>& o_strides = */ ostrides,\n        /* int64_t i_offset = */ data_offset,\n        /* int64_t o_offset = */ 0,\n        /* CopyType ctype = */ CopyType::General,\n        /* const Stream& s = */ stream());\n  } else {\n    std::vector<size_t> ostrides{inp_strides.begin(), inp_strides.end()};\n    shared_buffer_slice(in, ostrides, data_offset, out);\n  }\n}\n\nvoid SliceUpdate::eval_gpu(const std::vector<array>& inputs, array& out) {\n  assert(inputs.size() == 2);\n  if (out.size() == 0) {\n    out.set_data(nullptr);\n    return;\n  }\n\n  auto& in = inputs[0];\n  auto& upd = inputs[1];\n\n  if (upd.size() == 0) {\n    out.copy_shared_buffer(in);\n    return;\n  }\n\n  // Check if materialization is needed\n  auto ctype = in.flags().contiguous && in.size() == in.data_size()\n      ? CopyType::Vector\n      : CopyType::General;\n  copy_gpu(in, out, in.data_size() == 1 ? CopyType::Scalar : ctype, stream());\n\n  // Calculate out strides, initial offset and if copy needs to be made\n  auto [data_offset, out_strides] = prepare_slice(out);\n\n  // Do copy\n  std::vector<int64_t> upd_strides{upd.strides().begin(), upd.strides().end()};\n  copy_gpu_inplace<int64_t>(\n      /* const array& src = */ upd,\n      /* array& dst = */ out,\n      /* const std::vector<int>& data_shape = */ upd.shape(),\n      /* const std::vector<stride_t>& i_strides = */ upd_strides,\n      /* const std::vector<stride_t>& o_strides = */ out_strides,\n      /* int64_t i_offset = */ 0,\n      /* int64_t o_offset = */ data_offset,\n      /* CopyType ctype = */ CopyType::GeneralGeneral,\n      /* const Stream& s = */ stream());\n}\n\nvoid StopGradient::eval_gpu(const std::vector<array>& inputs, array& out) {\n  eval(inputs, out);\n}\n\nvoid Subtract::eval_gpu(const std::vector<array>& inputs, array& out) {\n  binary_op(inputs, out, \"sub\");\n}\n\nvoid Tan::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"tan\");\n}\n\nvoid Tanh::eval_gpu(const std::vector<array>& inputs, array& out) {\n  unary_op(inputs, out, \"tanh\");\n}\n\nvoid Transpose::eval_gpu(const std::vector<array>& inputs, array& out) {\n  eval(inputs, out);\n}\n\nvoid QRF::eval_gpu(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  throw std::runtime_error(\"[QRF::eval_gpu] Metal QR factorization NYI.\");\n}\n\nvoid SVD::eval_gpu(\n    const std::vector<array>& inputs,\n    std::vector<array>& outputs) {\n  throw std::runtime_error(\"[SVD::eval_gpu] Metal SVD NYI.\");\n}\n\nvoid Inverse::eval_gpu(const std::vector<array>& inputs, array& output) {\n  throw std::runtime_error(\"[Inverse::eval_gpu] Metal inversion NYI.\");\n}\n\n} // namespace mlx::core\n\n// Path: mlx/backend/metal/indexing.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n#include <algorithm>\n#include <cassert>\n#include <numeric>\n#include <sstream>\n\n#include \"mlx/backend/common/binary.h\"\n#include \"mlx/backend/metal/copy.h\"\n#include \"mlx/backend/metal/device.h\"\n#include \"mlx/backend/metal/kernels/defines.h\"\n#include \"mlx/backend/metal/utils.h\"\n#include \"mlx/primitives.h\"\n...\n", "needle_token_start": 8875, "needle_token_end": 8942, "code_context_ntokens": 16174, "output": ["```cpp\nvoid arange_set_scalars(T start, T next, CommandEncoder& enc) {\n  enc->setBytes(&start, sizeof(T), 0);\n  T step = next - start;\n  enc->setBytes(&step, sizeof(T), 1);\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "unpack_32_4", "language": "cpp", "path": "mlx/io/gguf_quants.cpp", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to decode and distribute packed bit data into a more accessible byte format, specifically targeting the transformation of compressed data into a format suitable for further processing or analysis.\n2. **Input**: The function takes two parameters: a pointer to an array of 8-bit unsigned integers (the source data array), and a pointer to an array of 8-bit signed integers (the destination array where the unpacked data will be stored).\n3. **Output**: There is no return value; however, the function modifies the destination array by filling it with unpacked and processed values derived from the source data array.\n4. **Procedure**: The function performs the following steps:\n   - Iterates over the first 16 bytes of the source array starting from the third byte, extracting the lower 4 bits of each byte. Depending on the position (odd or even index), these bits are either shifted left by 4 bits or left as is, and then added to the corresponding position in the destination array.\n   - For the next 16 bytes (also starting from the third byte of the source), it extracts the higher 4 bits, applies a similar shifting logic based on the byte's position, and adds these to the next set of positions in the destination array.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: mlx/utils.h\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#pragma once\n\n#include <variant>\n\n#include \"array.h\"\n#include \"device.h\"\n#include \"dtype.h\"\n#include \"stream.h\"\n\nnamespace mlx::core {\n\nusing StreamOrDevice = std::variant<std::monostate, Stream, Device>;\nStream to_stream(StreamOrDevice s);\n\nstruct StreamContext {\n public:\n  StreamContext(StreamOrDevice s) : _stream(default_stream(default_device())) {\n...\n// Path: mlx/utils.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <sstream>\n#include <vector>\n\n#include \"utils.h\"\n\nnamespace mlx::core {\n\nStream to_stream(StreamOrDevice s) {\n  if (std::holds_alternative<std::monostate>(s)) {\n    return default_stream(default_device());\n  } else if (std::holds_alternative<Device>(s)) {\n    return default_stream(std::get<Device>(s));\n  } else {\n    return std::get<Stream>(s);\n  }\n}\n\nvoid PrintFormatter::print(std::ostream& os, bool val) {\n  if (capitalize_bool) {\n    os << (val ? \"True\" : \"False\");\n  } else {\n    os << val;\n  }\n}\ninline void PrintFormatter::print(std::ostream& os, int16_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, uint16_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, int32_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, uint32_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, int64_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, uint64_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, float16_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, bfloat16_t val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, float val) {\n  os << val;\n}\ninline void PrintFormatter::print(std::ostream& os, complex64_t val) {\n  os << val;\n}\n\nPrintFormatter global_formatter;\n\nDtype result_type(const std::vector<array>& arrays) {\n  std::vector<Dtype> dtypes(1, bool_);\n  for (auto& arr : arrays) {\n    dtypes.push_back(promote_types(dtypes.back(), arr.dtype()));\n  }\n  return dtypes.back();\n}\n\nstd::vector<int> broadcast_shapes(\n    const std::vector<int>& s1,\n    const std::vector<int>& s2) {\n  // Use the same broadcasting rules as numpy\n  // https://numpy.org/doc/1.20/user/theory.broadcasting.html\n  // \"The size of the trailing axes for both arrays in an operation must\n  // either be the same size or one of them must be one.\"\n  int ndim1 = s1.size();\n  int ndim2 = s2.size();\n  int ndim = std::max(ndim1, ndim2);\n  int diff = std::abs(ndim1 - ndim2);\n  const auto& big = ndim1 > ndim2 ? s1 : s2;\n  const auto& small = ndim1 > ndim2 ? s2 : s1;\n  std::vector<int> out_shape(ndim);\n  for (int i = ndim - 1; i >= diff; --i) {\n    int a = big[i];\n    int b = small[i - diff];\n    if (b == a) {\n      out_shape[i] = a;\n    } else if (a == 1 || b == 1) {\n      // 0 if a or b is 0 otherwise max(a, b)\n      out_shape[i] = a * b;\n    } else {\n      std::ostringstream msg;\n      msg << \"Shapes \" << s1 << \" and \" << s2 << \" cannot be broadcast.\";\n      throw std::invalid_argument(msg.str());\n    }\n  }\n  for (int i = diff - 1; i >= 0; --i) {\n    out_shape[i] = big[i];\n  }\n  return out_shape;\n}\n\nbool is_same_shape(const std::vector<array>& arrays) {\n  if (arrays.empty()) {\n    return true;\n  }\n  return std::all_of(arrays.begin() + 1, arrays.end(), [&](const array& a) {\n    return (a.shape() == arrays[0].shape());\n  });\n}\n\nint normalize_axis(int axis, int ndim) {\n  if (ndim <= 0) {\n    throw std::invalid_argument(\"Number of dimensions must be positive.\");\n  }\n  if (axis < -ndim || axis >= ndim) {\n    std::ostringstream msg;\n    msg << \"Axis \" << axis << \" is out of bounds for array with \" << ndim\n        << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (axis < 0) {\n    axis += ndim;\n  }\n  return axis;\n}\n\nstd::ostream& operator<<(std::ostream& os, const Device& d) {\n  os << \"Device(\";\n  switch (d.type) {\n    case Device::cpu:\n      os << \"cpu\";\n      break;\n    case Device::gpu:\n      os << \"gpu\";\n      break;\n  }\n  os << \", \" << d.index << \")\";\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, const Stream& s) {\n  os << \"Stream(\";\n  os << s.device;\n  os << \", \" << s.index << \")\";\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, int8_t x) {\n  os << static_cast<int>(x);\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, uint8_t x) {\n  os << static_cast<uint>(x);\n  return os;\n}\n\nnamespace {\n\ninline size_t elem_to_loc(\n    int elem,\n    const std::vector<int>& shape,\n    const std::vector<size_t>& strides) {\n  size_t loc = 0;\n  for (int i = shape.size() - 1; i >= 0; --i) {\n    auto q_and_r = ldiv(elem, shape[i]);\n    loc += q_and_r.rem * strides[i];\n    elem = q_and_r.quot;\n  }\n  return loc;\n}\n\ntemplate <typename T>\nvoid print_subarray(std::ostream& os, const array& a, size_t index, int dim) {\n  int num_print = 3;\n  int n = a.shape(dim);\n  size_t s = a.strides()[dim];\n  bool is_last = dim == a.ndim() - 1;\n  auto prefix = is_last ? \"\" : std::string(7 + dim, ' ');\n  auto postfix = is_last ? \", \" : \",\\n\";\n  os << \"[\";\n  for (int i = 0; i < n; ++i) {\n    os << (i == 0 ? \"\" : prefix);\n    if (i == num_print && n > 2 * num_print) {\n      os << \"...\";\n      i = n - num_print - 1;\n      index += s * (n - 2 * num_print - 1);\n    } else if (is_last) {\n      global_formatter.print(os, a.data<T>()[index]);\n    } else {\n      print_subarray<T>(os, a, index, dim + 1);\n    }\n    os << (i == n - 1 ? \"\" : postfix);\n    index += s;\n  }\n  os << \"]\";\n}\n\ntemplate <typename T>\nvoid print_array(std::ostream& os, const array& a) {\n  std::vector<int> indices(a.ndim(), 0);\n  os << std::boolalpha;\n  os << \"array(\";\n  if (a.ndim() == 0) {\n    auto data = a.data<T>();\n    global_formatter.print(os, data[0]);\n  } else {\n    print_subarray<T>(os, a, 0, 0);\n  }\n  os << \", dtype=\" << a.dtype() << \")\";\n  os << std::noboolalpha;\n}\n\n} // namespace\n\nstd::ostream& operator<<(std::ostream& os, const Dtype& dtype) {\n  switch (dtype) {\n    case bool_:\n      return os << \"bool\";\n    case uint8:\n      return os << \"uint8\";\n    case uint16:\n      return os << \"uint16\";\n    case uint32:\n      return os << \"uint32\";\n    case uint64:\n      return os << \"uint64\";\n    case int8:\n      return os << \"int8\";\n    case int16:\n      return os << \"int16\";\n    case int32:\n      return os << \"int32\";\n    case int64:\n      return os << \"int64\";\n    case float16:\n      return os << \"float16\";\n    case float32:\n      return os << \"float32\";\n    case bfloat16:\n      return os << \"bfloat16\";\n    case complex64:\n      return os << \"complex64\";\n  }\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, const Dtype::Kind& k) {\n  switch (k) {\n    case Dtype::Kind::b:\n      return os << \"b\";\n    case Dtype::Kind::i:\n      return os << \"i\";\n    case Dtype::Kind::u:\n      return os << \"u\";\n    case Dtype::Kind::f:\n      return os << \"f\";\n    case Dtype::Kind::c:\n      return os << \"c\";\n    case Dtype::Kind::V:\n      return os << \"V\";\n  }\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, array a) {\n  if (!a.is_evaled()) {\n    a.eval();\n  }\n  switch (a.dtype()) {\n    case bool_:\n      print_array<bool>(os, a);\n      break;\n    case uint8:\n      print_array<uint8_t>(os, a);\n      break;\n    case uint16:\n      print_array<uint16_t>(os, a);\n      break;\n    case uint32:\n      print_array<uint32_t>(os, a);\n      break;\n    case uint64:\n      print_array<uint64_t>(os, a);\n      break;\n    case int8:\n      print_array<int8_t>(os, a);\n      break;\n    case int16:\n      print_array<int16_t>(os, a);\n      break;\n    case int32:\n      print_array<int32_t>(os, a);\n      break;\n    case int64:\n      print_array<int64_t>(os, a);\n      break;\n    case float16:\n      print_array<float16_t>(os, a);\n      break;\n    case bfloat16:\n      print_array<bfloat16_t>(os, a);\n      break;\n    case float32:\n      print_array<float>(os, a);\n      break;\n    case complex64:\n      print_array<complex64_t>(os, a);\n      break;\n  }\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, const std::vector<int>& v) {\n  os << \"(\";\n  for (int i = 0; i < v.size(); ++i) {\n    os << v[i] << ((i == v.size() - 1) ? \"\" : \",\");\n  }\n  os << \")\";\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, const std::vector<size_t>& v) {\n  os << \"(\";\n  for (int i = 0; i < v.size(); ++i) {\n    os << v[i] << ((i == v.size() - 1) ? \"\" : \",\");\n  }\n  os << \")\";\n  return os;\n}\n\nstd::ostream& operator<<(std::ostream& os, const std::vector<int64_t>& v) {\n  os << \"(\";\n  for (int i = 0; i < v.size(); ++i) {\n    os << v[i] << ((i == v.size() - 1) ? \"\" : \",\");\n  }\n  os << \")\";\n  return os;\n}\n\n} // namespace mlx::core\n\n// Path: mlx/fft.cpp\n// Copyright \u00a9 2023 Apple Inc.\n\n#include <numeric>\n#include <set>\n\n#include \"mlx/fft.h\"\n#include \"mlx/ops.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core::fft {\n\narray fft_impl(\n    const array& a,\n    std::vector<int> n,\n    const std::vector<int>& axes,\n    bool real,\n    bool inverse,\n    StreamOrDevice s) {\n  if (a.ndim() < 1) {\n    throw std::invalid_argument(\n        \"[fftn] Requires array with at least one dimension.\");\n  }\n  if (n.size() != axes.size()) {\n    throw std::invalid_argument(\"[fftn] Shape and axes have different sizes.\");\n  }\n  if (axes.empty()) {\n    return a;\n  }\n\n  std::vector<size_t> valid_axes;\n  for (int ax : axes) {\n    valid_axes.push_back(ax < 0 ? ax + a.ndim() : ax);\n  }\n  std::set<int> unique_axes(valid_axes.begin(), valid_axes.end());\n  if (unique_axes.size() != axes.size()) {\n    std::ostringstream msg;\n    msg << \"[fftn] Duplicated axis received \" << axes;\n    throw std::invalid_argument(msg.str());\n  }\n  if (*unique_axes.begin() < 0 || *unique_axes.rbegin() >= a.ndim()) {\n    std::ostringstream msg;\n    msg << \"[fftn] Invalid axis received for array with \" << a.ndim()\n        << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  // In the following shape manipulations there are three cases to consider:\n  // 1. In a complex to complex transform (fftn / ifftn) the output\n  //    and input shapes are the same.\n  // 2. In a real to complex transform (rfftn) n specifies the input dims\n  //    and the output dims are n[i] / 2 + 1\n  // 3  In a complex to real transform (irfftn) n specifies the output dims\n  //    and the input dims are n[i] / 2 + 1\n\n  if (std::any_of(n.begin(), n.end(), [](auto i) { return i <= 0; })) {\n    std::ostringstream msg;\n    msg << \"[fftn] Invalid FFT output size requested \" << n;\n    throw std::invalid_argument(msg.str());\n  }\n\n  std::vector<int> in_shape = a.shape();\n  for (int i = 0; i < valid_axes.size(); ++i) {\n    in_shape[valid_axes[i]] = n[i];\n  }\n  if (real && inverse) {\n    in_shape[valid_axes.back()] = n.back() / 2 + 1;\n  }\n\n  bool any_greater = false;\n  bool any_less = false;\n  for (int i = 0; i < in_shape.size(); ++i) {\n    any_greater |= in_shape[i] > a.shape()[i];\n    any_less |= in_shape[i] < a.shape()[i];\n  }\n\n  auto in = a;\n  if (any_less) {\n    in = slice(in, std::vector<int>(in.ndim(), 0), in_shape, s);\n  }\n  if (any_greater) {\n    // Pad with zeros\n    auto tmp = zeros(in_shape, a.dtype(), s);\n    in = scatter(tmp, std::vector<array>{}, in, std::vector<int>{}, s);\n  }\n\n  auto out_shape = in_shape;\n  if (real) {\n    auto ax = valid_axes.back();\n    out_shape[ax] = inverse ? n.back() : out_shape[ax] / 2 + 1;\n  }\n\n  auto in_type = real && !inverse ? float32 : complex64;\n  auto out_type = real && inverse ? float32 : complex64;\n  return array(\n      out_shape,\n      out_type,\n      std::make_shared<FFT>(to_stream(s), valid_axes, inverse, real),\n      {astype(in, in_type, s)});\n}\n\narray fft_impl(\n    const array& a,\n    const std::vector<int>& axes,\n    bool real,\n    bool inverse,\n    StreamOrDevice s) {\n  std::vector<int> n;\n  for (auto ax : axes) {\n    n.push_back(a.shape(ax));\n  }\n  if (real && inverse) {\n    n.back() = (n.back() - 1) * 2;\n  }\n  return fft_impl(a, n, axes, real, inverse, s);\n}\n\narray fft_impl(const array& a, bool real, bool inverse, StreamOrDevice s) {\n  std::vector<int> axes(a.ndim());\n  std::iota(axes.begin(), axes.end(), 0);\n  return fft_impl(a, axes, real, inverse, s);\n}\n\narray fftn(\n    const array& a,\n    const std::vector<int>& n,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, n, axes, false, false, s);\n}\narray fftn(\n    const array& a,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, axes, false, false, s);\n}\narray fftn(const array& a, StreamOrDevice s /* = {} */) {\n  return fft_impl(a, false, false, s);\n}\n\narray ifftn(\n    const array& a,\n    const std::vector<int>& n,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, n, axes, false, true, s);\n}\narray ifftn(\n    const array& a,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, axes, false, true, s);\n}\narray ifftn(const array& a, StreamOrDevice s /* = {} */) {\n  return fft_impl(a, false, true, s);\n}\n\narray rfftn(\n    const array& a,\n    const std::vector<int>& n,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, n, axes, true, false, s);\n}\narray rfftn(\n    const array& a,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, axes, true, false, s);\n}\narray rfftn(const array& a, StreamOrDevice s /* = {} */) {\n  return fft_impl(a, true, false, s);\n}\n\narray irfftn(\n    const array& a,\n    const std::vector<int>& n,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, n, axes, true, true, s);\n}\narray irfftn(\n    const array& a,\n    const std::vector<int>& axes,\n    StreamOrDevice s /* = {} */) {\n  return fft_impl(a, axes, true, true, s);\n}\narray irfftn(const array& a, StreamOrDevice s /* = {} */) {\n  return fft_impl(a, true, true, s);\n}\n\n} // namespace mlx::core::fft\n\n// Path: mlx/io/safetensor.cpp\n// Copyright \u00a9 2023 Apple Inc.\n//\n#include <json.hpp>\n#include <stack>\n\n#include \"mlx/io.h\"\n#include \"mlx/io/load.h\"\n#include \"mlx/primitives.h\"\n\nusing json = nlohmann::json;\n\n#define ST_F16 \"F16\"\n#define ST_BF16 \"BF16\"\n#define ST_F32 \"F32\"\n\n#define ST_BOOL \"BOOL\"\n#define ST_I8 \"I8\"\n#define ST_I16 \"I16\"\n#define ST_I32 \"I32\"\n#define ST_I64 \"I64\"\n#define ST_U8 \"U8\"\n#define ST_U16 \"U16\"\n#define ST_U32 \"U32\"\n#define ST_U64 \"U64\"\n\n// Note: Complex numbers aren't in the spec yet so this could change -\n// https://github.com/huggingface/safetensors/issues/389\n#define ST_C64 \"C64\"\n\nnamespace mlx::core {\n\nstd::string dtype_to_safetensor_str(Dtype t) {\n  switch (t) {\n    case float32:\n      return ST_F32;\n    case bfloat16:\n      return ST_BF16;\n    case float16:\n      return ST_F16;\n    case int64:\n      return ST_I64;\n    case int32:\n      return ST_I32;\n    case int16:\n      return ST_I16;\n    case int8:\n      return ST_I8;\n    case uint64:\n      return ST_U64;\n    case uint32:\n      return ST_U32;\n    case uint16:\n      return ST_U16;\n    case uint8:\n      return ST_U8;\n    case bool_:\n      return ST_BOOL;\n    case complex64:\n      return ST_C64;\n  }\n}\n\nDtype dtype_from_safetensor_str(std::string_view str) {\n  if (str == ST_F32) {\n    return float32;\n  } else if (str == ST_F16) {\n    return float16;\n  } else if (str == ST_BF16) {\n    return bfloat16;\n  } else if (str == ST_I64) {\n    return int64;\n  } else if (str == ST_I32) {\n    return int32;\n  } else if (str == ST_I16) {\n    return int16;\n  } else if (str == ST_I8) {\n    return int8;\n  } else if (str == ST_U64) {\n    return uint64;\n  } else if (str == ST_U32) {\n    return uint32;\n  } else if (str == ST_U16) {\n    return uint16;\n  } else if (str == ST_U8) {\n    return uint8;\n  } else if (str == ST_BOOL) {\n    return bool_;\n  } else if (str == ST_C64) {\n    return complex64;\n  } else {\n    throw std::runtime_error(\n        \"[safetensor] unsupported dtype \" + std::string(str));\n  }\n}\n\n/** Load array from reader in safetensor format */\nSafetensorsLoad load_safetensors(\n    std::shared_ptr<io::Reader> in_stream,\n    StreamOrDevice s) {\n  ////////////////////////////////////////////////////////\n  // Open and check file\n  if (!in_stream->good() || !in_stream->is_open()) {\n    throw std::runtime_error(\n        \"[load_safetensors] Failed to open \" + in_stream->label());\n  }\n\n  uint64_t jsonHeaderLength = 0;\n  in_stream->read(reinterpret_cast<char*>(&jsonHeaderLength), 8);\n  if (jsonHeaderLength <= 0) {\n    throw std::runtime_error(\n        \"[load_safetensors] Invalid json header length \" + in_stream->label());\n  }\n  // Load the json metadata\n  char rawJson[jsonHeaderLength];\n  in_stream->read(rawJson, jsonHeaderLength);\n  auto metadata = json::parse(rawJson, rawJson + jsonHeaderLength);\n  // Should always be an object on the top-level\n  if (!metadata.is_object()) {\n    throw std::runtime_error(\n        \"[load_safetensors] Invalid json metadata \" + in_stream->label());\n  }\n  size_t offset = jsonHeaderLength + 8;\n  // Load the arrays using metadata\n  std::unordered_map<std::string, array> res;\n  std::unordered_map<std::string, std::string> metadata_map;\n  for (const auto& item : metadata.items()) {\n    if (item.key() == \"__metadata__\") {\n      for (const auto& meta_item : item.value().items()) {\n        metadata_map.insert({meta_item.key(), meta_item.value()});\n      }\n      continue;\n    }\n    const std::string& dtype = item.value().at(\"dtype\");\n    const std::vector<int>& shape = item.value().at(\"shape\");\n    const std::vector<size_t>& data_offsets = item.value().at(\"data_offsets\");\n    Dtype type = dtype_from_safetensor_str(dtype);\n    auto loaded_array = array(\n        shape,\n        type,\n        std::make_shared<Load>(\n            to_stream(s), in_stream, offset + data_offsets.at(0), false),\n        std::vector<array>{});\n    res.insert({item.key(), loaded_array});\n  }\n  return {res, metadata_map};\n}\n\nSafetensorsLoad load_safetensors(const std::string& file, StreamOrDevice s) {\n  return load_safetensors(std::make_shared<io::FileReader>(file), s);\n}\n\n/** Save array to out stream in .npy format */\nvoid save_safetensors(\n    std::shared_ptr<io::Writer> out_stream,\n    std::unordered_map<std::string, array> a,\n    std::unordered_map<std::string, std::string> metadata /* = {} */) {\n  ////////////////////////////////////////////////////////\n  // Check file\n  if (!out_stream->good() || !out_stream->is_open()) {\n    throw std::runtime_error(\n        \"[save_safetensors] Failed to open \" + out_stream->label());\n  }\n\n  ////////////////////////////////////////////////////////\n  // Check array map\n  json parent;\n  json _metadata;\n  for (auto& [key, value] : metadata) {\n    _metadata[key] = value;\n  }\n  parent[\"__metadata__\"] = _metadata;\n  size_t offset = 0;\n  for (auto& [key, arr] : a) {\n    arr.eval();\n    if (arr.nbytes() == 0) {\n      throw std::invalid_argument(\n          \"[save_safetensors] cannot serialize an empty array key: \" + key);\n    }\n\n    // Try to make it row contiguous\n    if (!arr.flags().row_contiguous) {\n      arr = reshape(flatten(arr), arr.shape());\n      arr.eval();\n    }\n\n    // Has to be row-major now but, check one more time in case\n    // any of the above change in the future\n    if (!arr.flags().row_contiguous) {\n      throw std::invalid_argument(\n          \"[save_safetensors] can only serialize row-major arrays\");\n    }\n\n    json child;\n    child[\"dtype\"] = dtype_to_safetensor_str(arr.dtype());\n    child[\"shape\"] = arr.shape();\n    child[\"data_offsets\"] = std::vector<size_t>{offset, offset + arr.nbytes()};\n    parent[key] = child;\n    offset += arr.nbytes();\n  }\n\n  auto header = parent.dump();\n  uint64_t header_len = header.length();\n  out_stream->write(reinterpret_cast<char*>(&header_len), 8);\n  out_stream->write(header.c_str(), header_len);\n  for (auto& [key, arr] : a) {\n    out_stream->write(arr.data<char>(), arr.nbytes());\n  }\n}\n\nvoid save_safetensors(\n    std::string file,\n    std::unordered_map<std::string, array> a,\n    std::unordered_map<std::string, std::string> metadata /* = {} */) {\n  // Add .safetensors to file name if it is not there\n  if (file.length() < 12 ||\n      file.substr(file.length() - 12, 12) != \".safetensors\")\n    file += \".safetensors\";\n\n  // Serialize array\n  save_safetensors(\n      std::make_shared<io::FileWriter>(std::move(file)), a, metadata);\n}\n\n} // namespace mlx::core\n\n// Path: mlx/io/load.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n#include <algorithm>\n#include <cstring>\n#include <fstream>\n#include <limits>\n#include <sstream>\n\n#include \"mlx/io/load.h\"\n#include \"mlx/ops.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\n// Adapted from\n// https://github.com/angeloskath/supervised-lda/blob/master/include/ldaplusplus/NumpyFormat.hpp\n\nnamespace mlx::core {\n\nnamespace {\n\nconstexpr uint8_t MAGIC[] = {\n    0x93,\n    0x4e,\n    0x55,\n    0x4d,\n    0x50,\n    0x59,\n};\n\n} // namespace\n\n/** Save array to out stream in .npy format */\nvoid save(std::shared_ptr<io::Writer> out_stream, array a) {\n  ////////////////////////////////////////////////////////\n  // Check array\n\n  a.eval();\n\n  if (a.nbytes() == 0) {\n    throw std::invalid_argument(\"[save] cannot serialize an empty array\");\n  }\n\n  if (!(a.flags().row_contiguous || a.flags().col_contiguous)) {\n    a = reshape(flatten(a), a.shape());\n    a.eval();\n  }\n  // Check once more in-case the above ops change\n  if (!(a.flags().row_contiguous || a.flags().col_contiguous)) {\n    throw std::invalid_argument(\n        \"[save] can only serialize row or col contiguous arrays\");\n  }\n\n  ////////////////////////////////////////////////////////\n  // Check file\n  if (!out_stream->good() || !out_stream->is_open()) {\n    throw std::runtime_error(\"[save] Failed to open \" + out_stream->label());\n  }\n\n  ////////////////////////////////////////////////////////\n  // Prepare header\n  std::ostringstream magic_ver_len;\n  magic_ver_len.write(reinterpret_cast<const char*>(MAGIC), 6);\n\n  std::string fortran_order = a.flags().col_contiguous ? \"True\" : \"False\";\n  std::ostringstream header;\n  header << \"{'descr': '\" << dtype_to_array_protocol(a.dtype()) << \"',\"\n         << \" 'fortran_order': \" << fortran_order << \",\" << \" 'shape': (\";\n  for (auto i : a.shape()) {\n    header << i << \", \";\n  }\n  header << \")}\";\n\n  size_t header_len = static_cast<size_t>(header.tellp());\n  bool is_v1 = header_len + 15 < std::numeric_limits<uint16_t>::max();\n\n  // Pad out magic + version + header_len + header + \\n to be divisible by 16\n  size_t padding = (6 + 2 + (2 + 2 * is_v1) + header_len + 1) % 16;\n\n  header << std::string(padding, ' ') << '\\n';\n\n  if (is_v1) {\n    magic_ver_len << (char)0x01 << (char)0x00;\n\n    uint16_t v1_header_len = header.tellp();\n    const char* len_bytes = reinterpret_cast<const char*>(&v1_header_len);\n\n    if (!is_big_endian()) {\n      magic_ver_len.write(len_bytes, 2);\n    } else {\n      magic_ver_len.write(len_bytes + 1, 1);\n      magic_ver_len.write(len_bytes, 1);\n    }\n  } else {\n    magic_ver_len << (char)0x02 << (char)0x00;\n\n    uint32_t v2_header_len = header.tellp();\n    const char* len_bytes = reinterpret_cast<const char*>(&v2_header_len);\n\n    if (!is_big_endian()) {\n      magic_ver_len.write(len_bytes, 4);\n    } else {\n      magic_ver_len.write(len_bytes + 3, 1);\n      magic_ver_len.write(len_bytes + 2, 1);\n      magic_ver_len.write(len_bytes + 1, 1);\n      magic_ver_len.write(len_bytes, 1);\n    }\n  }\n  ////////////////////////////////////////////////////////\n  // Serialize array\n\n  out_stream->write(magic_ver_len.str().c_str(), magic_ver_len.str().length());\n  out_stream->write(header.str().c_str(), header.str().length());\n  out_stream->write(a.data<char>(), a.nbytes());\n}\n\n/** Save array to file in .npy format */\nvoid save(std::string file, array a) {\n  // Add .npy to file name if it is not there\n  if (file.length() < 4 || file.substr(file.length() - 4, 4) != \".npy\")\n    file += \".npy\";\n\n  // Serialize array\n  save(std::make_shared<io::FileWriter>(std::move(file)), a);\n}\n\n/** Load array from reader in .npy format */\narray load(std::shared_ptr<io::Reader> in_stream, StreamOrDevice s) {\n  ////////////////////////////////////////////////////////\n  // Open and check file\n  if (!in_stream->good() || !in_stream->is_open()) {\n    throw std::runtime_error(\"[load] Failed to open \" + in_stream->label());\n  }\n\n  ////////////////////////////////////////////////////////\n  // Read header and prepare array details\n\n  // Read and check magic\n  char read_magic_and_ver[8];\n  in_stream->read(read_magic_and_ver, 8);\n  if (std::memcmp(read_magic_and_ver, MAGIC, 6) != 0) {\n    throw std::runtime_error(\"[load] Invalid header in \" + in_stream->label());\n  }\n\n  // Read and check version\n  if (read_magic_and_ver[6] != 1 && read_magic_and_ver[6] != 2) {\n    throw std::runtime_error(\n        \"[load] Unsupported npy format version in \" + in_stream->label());\n  }\n\n  // Read header len and header\n  int header_len_size = read_magic_and_ver[6] == 1 ? 2 : 4;\n  size_t header_len;\n\n  if (header_len_size == 2) {\n    uint16_t v1_header_len;\n    in_stream->read(reinterpret_cast<char*>(&v1_header_len), header_len_size);\n    header_len = v1_header_len;\n  } else {\n    uint32_t v2_header_len;\n    in_stream->read(reinterpret_cast<char*>(&v2_header_len), header_len_size);\n    header_len = v2_header_len;\n  }\n\n  // Read the header\n  std::vector<char> buffer(header_len + 1);\n  in_stream->read(&buffer[0], header_len);\n  buffer[header_len] = 0;\n  std::string header(&buffer[0]);\n\n  // Read data type from header\n  std::string dtype_str = header.substr(11, 3);\n  bool read_is_big_endian = dtype_str[0] == '>';\n  Dtype dtype = dtype_from_array_protocol(dtype_str);\n\n  // Read contiguity order\n  bool col_contiguous = header[34] == 'T';\n\n  // Read array shape from header\n  std::vector<int> shape;\n\n  size_t st = header.find_last_of('(') + 1;\n  size_t ed = header.find_last_of(')');\n  std::string shape_str = header.substr(st, ed - st);\n\n  while (!shape_str.empty()) {\n    // Read current number and get position of comma\n    size_t pos;\n    int dim = std::stoi(shape_str, &pos);\n    shape.push_back(dim);\n\n    // Skip the comma and space and read the next number\n    if (pos + 2 <= shape_str.length())\n      shape_str = shape_str.substr(pos + 2);\n    else {\n      shape_str = shape_str.substr(pos);\n      if (!shape_str.empty() && shape_str != \" \" && shape_str != \",\") {\n        throw std::runtime_error(\n            \"[load] Unknown error while parsing header in \" +\n            in_stream->label());\n      }\n      shape_str = \"\";\n    }\n  }\n\n  ////////////////////////////////////////////////////////\n  // Build primitive\n\n  size_t offset = 8 + header_len_size + header.length();\n  bool swap_endianness = read_is_big_endian != is_big_endian();\n\n  if (col_contiguous) {\n    std::reverse(shape.begin(), shape.end());\n  }\n  auto loaded_array = array(\n      shape,\n      dtype,\n      std::make_shared<Load>(to_stream(s), in_stream, offset, swap_endianness),\n      std::vector<array>{});\n  if (col_contiguous) {\n    loaded_array = transpose(loaded_array, s);\n  }\n\n  return loaded_array;\n}\n\n/** Load array from file in .npy format */\narray load(std::string file, StreamOrDevice s) {\n  return load(std::make_shared<io::FileReader>(std::move(file)), s);\n}\n\n} // namespace mlx::core\n\n// Path: mlx/io/gguf_quants.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <cstdint>\n#include <cstring>\n\n#include <mlx/io/gguf.h>\n\nnamespace mlx::core {\n\nv\noid unpack_32_4(uint8_t* data, int8_t* dst) {\n  for (int64_t j = 0; j < 16; ++j) {\n    uint8_t x = (data[j + 2] & 0x0F); // j+2 to skip scale bytes.\n    if (j % 2 != 0) {\n      x <<= 4;\n    }\n    dst[j / 2] += x;\n  }\n  // Last 16 weights are in the higher bits\n  for (int64_t j = 0; j < 16; ++j) {\n    uint8_t x = (data[j + 2] >> 4);\n    if (j % 2 != 0) {\n      x <<= 4;\n    }\n    dst[8 + j / 2] += x;\n  }\n}\n\n// Extracts (weight, scales, biases) from Q4_0 tensors.\n// Data layout is: |16 bit scale|32 x 4bit weights|.\nvoid extract_q4_0_data(\n    const gguf_tensor& tensor,\n    array& weights_arr,\n    array& scales_arr,\n    array& biases_arr) {\n  const uint64_t bytes_per_block = 18; // 2 bytes scale, 32x0.5 byte weights\n  auto data = static_cast<uint8_t*>(tensor.weights_data);\n  auto weights = weights_arr.data<int8_t>();\n  auto scales = scales_arr.data<float16_t>();\n  auto biases = biases_arr.data<float16_t>();\n  for (int64_t i = 0; i < scales_arr.size(); i++) {\n    scales[i] = *((float16_t*)data);\n    biases[i] = -8 * scales[i];\n    unpack_32_4(data, weights);\n    weights += 16;\n    data += bytes_per_block;\n  }\n}\n\n// Extracts (weight, scales, biases) from Q4_1 tensors.\n// Data layout is: |16 bit scale|16 bit bias|32 x 4bit weights|.\nvoid extract_q4_1_data(\n    const gguf_tensor& tensor,\n    array& weights_arr,\n    array& scales_arr,\n    array& biases_arr) {\n  const uint64_t bytes_per_block =\n      20; // 2 bytes scale, 2 bytes bias, 32x0.5 byte weights\n  auto data = static_cast<uint8_t*>(tensor.weights_data);\n  auto weights = weights_arr.data<int8_t>();\n  auto scales = scales_arr.data<float16_t>();\n  auto biases = biases_arr.data<float16_t>();\n  for (int64_t i = 0; i < scales_arr.size(); i++) {\n    scales[i] = *((float16_t*)data);\n    biases[i] = *((float16_t*)(data) + 1);\n    unpack_32_4(data, weights);\n    weights += 16;\n    data += bytes_per_block;\n  }\n}\n\n// Extracts (weight, scales, biases) from Q8_0 tensors.\n// Data layout is: |16 bit scale|32 x 8bit weights|.\nvoid extract_q8_0_data(\n    const gguf_tensor& tensor,\n    array& weights_arr,\n    array& scales_arr,\n    array& biases_arr) {\n  const uint64_t weights_per_block = 32;\n  const uint64_t bytes_per_block = 34; // 2 bytes scale, 32x1 byte weights\n  auto data = static_cast<uint8_t*>(tensor.weights_data);\n  auto weights = weights_arr.data<int8_t>();\n  auto scales = scales_arr.data<float16_t>();\n  auto biases = biases_arr.data<float16_t>();\n  for (int64_t i = 0; i < scales_arr.size(); i++) {\n    uint8_t* block_data = data + i * bytes_per_block;\n    scales[i] = *((float16_t*)block_data);\n    biases[i] = -128 * scales[i];\n    for (int64_t j = 0; j < weights_per_block; ++j) {\n      uint8_t x = block_data[j + 2]; // j+2 to skip the scale bytes.\n      // Original data is in int8_t, so we add a bias of -128 and invert the\n      // first bit.\n      x ^= 1 << 7;\n      weights[i * weights_per_block + j] = x;\n    }\n  }\n}\n\nvoid gguf_load_quantized(\n    std::unordered_map<std::string, array>& a,\n    const gguf_tensor& tensor) {\n  uint64_t weights_per_byte;\n  if (tensor.type == GGUF_TYPE_Q4_0 || tensor.type == GGUF_TYPE_Q4_1) {\n    weights_per_byte = 2;\n  } else { // tensor.type == GGUF_TYPE_Q8_0\n    weights_per_byte = 1;\n  }\n\n  std::string name(tensor.name, tensor.namelen);\n\n  std::vector<int> shape = get_shape(tensor);\n  const uint64_t weights_per_block = 32;\n  if (shape[shape.size() - 1] % weights_per_block != 0) {\n    std::ostringstream msg;\n    msg << \"[load_gguf] tensor \" << name\n        << \"has incompatible last dim shape: \" << shape[shape.size() - 1];\n    throw std::runtime_error(msg.str());\n  }\n\n  std::vector<int> weights_shape = shape;\n  weights_shape.back() /= (weights_per_byte * 4);\n\n  array weights(std::move(weights_shape), uint32, nullptr, {});\n  weights.set_data(allocator::malloc(weights.nbytes()));\n\n  // For scales and bias\n  shape[shape.size() - 1] = shape[shape.size() - 1] / weights_per_block;\n  array scales(shape, float16, nullptr, {});\n  array biases(std::move(shape), float16, nullptr, {});\n  scales.set_data(allocator::malloc(scales.nbytes()));\n  biases.set_data(allocator::malloc(biases.nbytes()));\n\n  if (tensor.type == GGUF_TYPE_Q4_0) {\n    extract_q4_0_data(tensor, weights, scales, biases);\n  } else if (tensor.type == GGUF_TYPE_Q4_1) {\n    extract_q4_1_data(tensor, weights, scales, biases);\n  } else if (tensor.type == GGUF_TYPE_Q8_0) {\n    extract_q8_0_data(tensor, weights, scales, biases);\n  }\n\n  a.emplace(name, std::move(weights));\n\n  auto check_insert = [](const auto& inserted) {\n    if (!inserted.second) {\n      std::ostringstream msg;\n      msg << \"[load_gguf] Duplicate parameter name \" << inserted.first->second\n          << \" this can happend when loading quantized tensors.\";\n      throw std::runtime_error(msg.str());\n    }\n  };\n\n  constexpr std::string_view weight_suffix = \".weight\";\n  const std::string name_prefix =\n      name.substr(0, name.length() - weight_suffix.length());\n  check_insert(a.emplace(name_prefix + \".scales\", std::move(scales)));\n  check_insert(a.emplace(name_prefix + \".biases\", std::move(biases)));\n}\n\n} // namespace mlx::core\n\n// Path: mlx/io/gguf.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <cstdint>\n#include <cstring>\n#include <numeric>\n\n#include <mlx/io/gguf.h>\n\nnamespace mlx::core {\n\n// https://github.com/antirez/gguf-tools/blob/af7d88d808a7608a33723fba067036202910acb3/gguflib.h#L102-L108\nconstexpr int gguf_array_header_size = 12;\n\nstd::optional<uint32_t> dtype_to_gguf_tensor_type(const Dtype& dtype) {\n  switch (dtype) {\n    case float32:\n      return GGUF_TYPE_F32;\n    case float16:\n      return GGUF_TYPE_F16;\n    case int8:\n      return GGUF_TYPE_I8;\n    case int16:\n      return GGUF_TYPE_I16;\n    case int32:\n      return GGUF_TYPE_I32;\n    default:\n      return {};\n  }\n}\n\nstd::optional<Dtype> gguf_type_to_dtype(const uint32_t& gguf_type) {\n  switch (gguf_type) {\n    case GGUF_TYPE_F32:\n      return float32;\n    case GGUF_TYPE_F16:\n      return float16;\n    case GGUF_TYPE_I8:\n      return int8;\n    case GGUF_TYPE_I16:\n      return int16;\n    case GGUF_TYPE_I32:\n      return int32;\n    default:\n      return {};\n  }\n}\n\nstd::vector<int> get_shape(const gguf_tensor& tensor) {\n  std::vector<int> shape;\n  // The dimension order in GGML is the reverse of the order used in MLX.\n  for (int i = tensor.ndim - 1; i >= 0; i--) {\n    shape.push_back(tensor.dim[i]);\n  }\n  return shape;\n}\n\nstd::tuple<allocator::Buffer, Dtype> extract_tensor_data(gguf_tensor* tensor) {\n  std::optional<Dtype> equivalent_dtype = gguf_type_to_dtype(tensor->type);\n  // If there's an equivalent type, we can simply copy.\n  if (equivalent_dtype.has_value()) {\n    allocator::Buffer buffer = allocator::malloc(tensor->bsize);\n    memcpy(\n        buffer.raw_ptr(),\n        tensor->weights_data,\n        tensor->num_weights * equivalent_dtype.value().size);\n    return {buffer, equivalent_dtype.value()};\n  }\n  // Otherwise, we convert to float16.\n  // TODO: Add other dequantization options.\n  int16_t* data = gguf_tensor_to_f16(tensor);\n  if (data == NULL) {\n    throw std::runtime_error(\"[load_gguf] gguf_tensor_to_f16 failed\");\n  }\n  const size_t new_size = tensor->num_weights * sizeof(int16_t);\n  allocator::Buffer buffer = allocator::malloc(new_size);\n  memcpy(buffer.raw_ptr(), data, new_size);\n  free(data);\n  return {buffer, float16};\n}\n\nvoid set_mx_value_from_gguf(\n    gguf_ctx* ctx,\n    uint32_t type,\n    gguf_value* val,\n    GGUFMetaData& value) {\n  switch (type) {\n    case GGUF_VALUE_TYPE_UINT8:\n      value = array(val->uint8, uint8);\n      break;\n    case GGUF_VALUE_TYPE_INT8:\n      value = array(val->int8, int8);\n      break;\n    case GGUF_VALUE_TYPE_UINT16:\n      value = array(val->uint16, uint16);\n      break;\n    case GGUF_VALUE_TYPE_INT16:\n      value = array(val->int16, int16);\n      break;\n    case GGUF_VALUE_TYPE_UINT32:\n      value = array(val->uint32, uint32);\n      break;\n    case GGUF_VALUE_TYPE_INT32:\n      value = array(val->int32, int32);\n      break;\n    case GGUF_VALUE_TYPE_UINT64:\n      value = array(val->uint64, uint64);\n      break;\n    case GGUF_VALUE_TYPE_INT64:\n      value = array(val->int64, int64);\n      break;\n    case GGUF_VALUE_TYPE_FLOAT32:\n      value = array(val->float32, float32);\n      break;\n    case GGUF_VALUE_TYPE_BOOL:\n      value = array(val->boolval, bool_);\n      break;\n    case GGUF_VALUE_TYPE_STRING:\n      value =\n          std::string(val->string.string, static_cast<int>(val->string.len));\n      break;\n    case GGUF_VALUE_TYPE_FLOAT64:\n      value = array(val->float64, float32);\n      break;\n    case GGUF_VALUE_TYPE_ARRAY: {\n      ctx->off += gguf_array_header_size; // Skip header\n      char* data = reinterpret_cast<char*>(val) + gguf_array_header_size;\n      auto size = static_cast<int>(val->array.len);\n      if (val->array.type == GGUF_VALUE_TYPE_ARRAY) {\n        throw std::invalid_argument(\n            \"[load_gguf] Only supports loading 1-layer of nested arrays.\");\n      }\n      switch (val->array.type) {\n        case GGUF_VALUE_TYPE_UINT8:\n          value = array(reinterpret_cast<uint8_t*>(data), {size}, uint8);\n          break;\n        case GGUF_VALUE_TYPE_INT8:\n          value = array(reinterpret_cast<int8_t*>(data), {size}, int8);\n          break;\n        case GGUF_VALUE_TYPE_UINT16:\n          value = array(reinterpret_cast<uint16_t*>(data), {size}, uint16);\n          break;\n        case GGUF_VALUE_TYPE_INT16:\n          value = array(reinterpret_cast<int16_t*>(data), {size}, int16);\n          break;\n        case GGUF_VALUE_TYPE_UINT32:\n          value = array(reinterpret_cast<uint32_t*>(data), {size}, uint32);\n          break;\n        case GGUF_VALUE_TYPE_INT32:\n          value = array(reinterpret_cast<int32_t*>(data), {size}, int32);\n          break;\n        case GGUF_VALUE_TYPE_UINT64:\n          value = array(reinterpret_cast<uint64_t*>(data), {size}, uint64);\n          break;\n        case GGUF_VALUE_TYPE_INT64:\n          value = array(reinterpret_cast<uint64_t*>(data), {size}, int64);\n          break;\n        case GGUF_VALUE_TYPE_FLOAT32:\n          value = array(reinterpret_cast<float*>(data), {size}, float32);\n          break;\n        case GGUF_VALUE_TYPE_BOOL:\n          value = array(reinterpret_cast<bool*>(data), {size}, bool_);\n          break;\n        case GGUF_VALUE_TYPE_STRING: {\n          std::vector<std::string> strs(size);\n          for (auto& str : strs) {\n            auto str_val = reinterpret_cast<gguf_string*>(data);\n            data += (str_val->len + sizeof(gguf_string));\n            str = std::string(str_val->string, static_cast<int>(str_val->len));\n            ctx->off += (str_val->len + sizeof(gguf_string));\n          }\n          value = std::move(strs);\n          break;\n        }\n        case GGUF_VALUE_TYPE_FLOAT64:\n          value = array(reinterpret_cast<double*>(data), {size}, float32);\n          break;\n        default:\n          throw std::runtime_error(\n              \"[load_gguf] Multiple levels of nested arrays are not supported.\");\n      }\n      break;\n    }\n    default:\n      throw std::runtime_error(\"[load_gguf] Received unexpected type.\");\n      break;\n  }\n  if (type == GGUF_VALUE_TYPE_STRING) {\n    ctx->off += (sizeof(gguf_string) + std::get<std::string>(value).size());\n  } else if (auto pv = std::get_if<array>(&value); pv) {\n    ctx->off += pv->nbytes();\n  }\n}\n\nstd::unordered_map<std::string, GGUFMetaData> load_metadata(gguf_ctx* ctx) {\n  std::unordered_map<std::string, GGUFMetaData> metadata;\n  gguf_key key;\n  while (gguf_get_key(ctx, &key)) {\n    std::string key_name = std::string(key.name, key.namelen);\n    auto& val = metadata.insert({key_name, GGUFMetaData{}}).first->second;\n    set_mx_value_from_gguf(ctx, key.type, key.val, val);\n  }\n  return metadata;\n}\n\nstd::unordered_map<std::string, array> load_arrays(gguf_ctx* ctx) {\n  std::unordered_map<std::string, array> array_map;\n  gguf_tensor tensor;\n\n  auto check_insert = [](const auto& inserted) {\n    if (!inserted.second) {\n      std::ostringstream msg;\n      msg << \"[load_gguf] Duplicate parameter name \" << inserted.first->second\n          << \" this can happend when loading quantized tensors.\";\n      throw std::runtime_error(msg.str());\n    }\n  };\n\n  while (gguf_get_tensor(ctx, &tensor)) {\n    std::string name(tensor.name, tensor.namelen);\n    if (tensor.type == GGUF_TYPE_Q4_0 || tensor.type == GGUF_TYPE_Q4_1 ||\n        tensor.type == GGUF_TYPE_Q8_0) {\n      gguf_load_quantized(array_map, tensor);\n    } else {\n      std::string name = std::string(tensor.name, tensor.namelen);\n\n      const auto& [data, dtype] = extract_tensor_data(&tensor);\n      array loaded_array = array(data, get_shape(tensor), dtype);\n      check_insert(array_map.insert({name, loaded_array}));\n    }\n  }\n  return array_map;\n}\n\nGGUFLoad load_gguf(const std::string& file, StreamOrDevice s) {\n  gguf_ctx* ctx = gguf_open(file.data());\n  if (!ctx) {\n    throw std::runtime_error(\"[load_gguf] gguf_init failed\");\n  }\n  auto metadata = load_metadata(ctx);\n  auto arrays = load_arrays(ctx);\n  gguf_close(ctx);\n  return {arrays, metadata};\n}\n\nvoid append_kv_array(\n    gguf_ctx* ctx,\n    const std::string& key,\n    array& val,\n    uint32_t gguf_type) {\n  if (val.ndim() == 1) {\n    size_t gguf_size = val.nbytes() + gguf_array_header_size;\n    std::vector<char> val_vec(gguf_size);\n    gguf_value* gguf_val = reinterpret_cast<gguf_value*>(val_vec.data());\n    gguf_val->array.type = gguf_type;\n    gguf_val->array.len = val.size();\n    memcpy(\n        val_vec.data() + gguf_array_header_size,\n        val.data<char>(),\n        val.nbytes());\n    gguf_append_kv(\n        ctx,\n        key.c_str(),\n        key.length(),\n        GGUF_VALUE_TYPE_ARRAY,\n        reinterpret_cast<void*>(val_vec.data()),\n        gguf_size);\n  } else {\n    gguf_append_kv(\n        ctx,\n        key.c_str(),\n        key.length(),\n        gguf_type,\n        reinterpret_cast<void*>(val.data<char>()),\n        val.nbytes());\n  }\n}\n\nvoid save_gguf(\n    std::string file,\n    std::unordered_map<std::string, array> array_map,\n    std::unordered_map<std::string, GGUFMetaData> metadata /* = {} */) {\n  // Add .gguf to file name if it is not there\n  if (file.length() < 5 || file.substr(file.length() - 5, 5) != \".gguf\") {\n    file += \".gguf\";\n  }\n\n  gguf_ctx* ctx = gguf_create(file.c_str(), GGUF_OVERWRITE);\n  if (!ctx) {\n    throw std::runtime_error(\"[save_gguf] gguf_create failed\");\n  }\n\n  auto string_to_gguf = [](char* dst, const std::string& src) {\n    gguf_string* val = reinterpret_cast<gguf_string*>(dst);\n    val->len = src.length();\n    memcpy(val->string, src.c_str(), src.length());\n  };\n\n  // Save any meta data\n  for (auto& [key, value] : metadata) {\n    if (auto pv = std::get_if<std::string>(&value); pv) {\n      const std::string& str = *pv;\n      size_t size = sizeof(gguf_string) + str.length();\n      std::vector<char> val_vec(size);\n      string_to_gguf(val_vec.data(), str);\n      gguf_append_kv(\n          ctx,\n          key.c_str(),\n          key.length(),\n          GGUF_VALUE_TYPE_STRING,\n...\n", "needle_token_start": 10409, "needle_token_end": 10630, "code_context_ntokens": 16217, "output": ["```cpp\nvoid unpack_32_4(uint8_t* data, int8_t* dst) {\n  for (int64_t j = 0; j < 16; ++j) {\n    uint8_t x = (data[j + 2] & 0x0F); // j+2 to skip scale bytes.\n    if (j % 2 != 0) {\n      x <<= 4;\n    }\n    dst[j / 2] += x;\n  }\n  // Last 16 weights are in the higher bits\n  for (int64_t j = 0; j < 16; ++j) {\n    uint8_t x = (data[j + 2] >> 4);\n    if (j % 2 != 0) {\n      x <<= 4;\n    }\n    dst[8 + j / 2] += x;\n  }\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "QuantizedMatmul", "language": "cpp", "path": "mlx/primitives.h", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: This function is designed to perform matrix multiplication on quantized data, which is a method to reduce the computational precision of the operands to lower bit-widths. This is particularly useful in scenarios where memory bandwidth and power consumption are limited, such as in embedded systems or mobile devices.\n2. **Input**: The inputs include two matrices to be multiplied. These matrices are expected to be quantized, meaning their numerical precision is reduced to a specified number of bits.\n3. **Output**: The output is a single matrix that represents the product of the input matrices, also in a quantized format.\n4. **Procedure**: The function first checks if the matrices need to be transposed based on the specified parameter. It then groups the elements of the matrices according to the provided group size, performs the multiplication in the reduced bit-width format, and finally aggregates the results to form the output matrix.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "      const std::vector<array>& inputs,\n      const std::vector<int>& axes);\n\n  /** Print the primitive. */\n  virtual void print(std::ostream& os) = 0;\n\n  /** Equivalence check defaults to false unless overridden by the primitive */\n  virtual bool is_equivalent(const Primitive& other) const {\n    return false;\n  }\n\n  /** Get the output shapes of the primitive. This is not required to be\n   * implemented by derived classes, in which case it will throw. */\n  virtual std::vector<std::vector<int>> output_shapes(\n      const std::vector<array>& inputs);\n\n  virtual ~Primitive() = default;\n  Primitive(const Primitive& other) = delete;\n  Primitive(Primitive&& other) = delete;\n  Primitive& operator=(const Primitive& other) = delete;\n  Primitive& operator=(Primitive&& other) = delete;\n\n private:\n  // Every primitive stores the stream it should run in\n  Stream stream_;\n};\n\nclass UnaryPrimitive : public Primitive {\n  /**\n   * An abstract base class for a primitive with a single output.\n   */\n public:\n  explicit UnaryPrimitive(Stream stream) : Primitive(stream) {}\n\n  virtual void eval_cpu(const std::vector<array>& inputs, array& output) = 0;\n  virtual void eval_gpu(const std::vector<array>& inputs, array& output) = 0;\n\n  inline void eval_cpu(\n      const std::vector<array>& inputs,\n      std::vector<array>& outputs) override {\n    eval_cpu(inputs, outputs[0]);\n  }\n  inline void eval_gpu(\n      const std::vector<array>& inputs,\n      std::vector<array>& outputs) override {\n    eval_gpu(inputs, outputs[0]);\n  }\n\n  virtual ~UnaryPrimitive() = default;\n  UnaryPrimitive(const UnaryPrimitive& other) = delete;\n  UnaryPrimitive(UnaryPrimitive&& other) = delete;\n  UnaryPrimitive& operator=(const UnaryPrimitive& other) = delete;\n  UnaryPrimitive& operator=(UnaryPrimitive&& other) = delete;\n};\n\nclass Abs : public UnaryPrimitive {\n public:\n  explicit Abs(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Abs)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Add : public UnaryPrimitive {\n public:\n  explicit Add(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Add)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass AddMM : public UnaryPrimitive {\n public:\n  explicit AddMM(Stream stream, float alpha, float beta)\n      : UnaryPrimitive(stream), alpha_(alpha), beta_(beta){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_VMAP()\n  DEFINE_PRINT(AddMM)\n\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  const float alpha_;\n  const float beta_;\n};\n\nclass Arange : public UnaryPrimitive {\n public:\n  explicit Arange(Stream stream, double start, double stop, double step)\n      : UnaryPrimitive(stream), start_(start), stop_(stop), step_(step){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_PRINT(Arange)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  double start_;\n  double stop_;\n  double step_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArcCos : public UnaryPrimitive {\n public:\n  explicit ArcCos(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(ArcCos)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArcCosh : public UnaryPrimitive {\n public:\n  explicit ArcCosh(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(ArcCosh)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArcSin : public UnaryPrimitive {\n public:\n  explicit ArcSin(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(ArcSin)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArcSinh : public UnaryPrimitive {\n public:\n  explicit ArcSinh(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(ArcSinh)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArcTan : public UnaryPrimitive {\n public:\n  explicit ArcTan(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(ArcTan)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArcTanh : public UnaryPrimitive {\n public:\n  explicit ArcTanh(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(ArcTanh)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArgPartition : public UnaryPrimitive {\n public:\n  explicit ArgPartition(Stream stream, int kth, int axis)\n      : UnaryPrimitive(stream), kth_(kth), axis_(axis){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_PRINT(ArgPartition)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  int kth_;\n  int axis_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArgReduce : public UnaryPrimitive {\n public:\n  enum ReduceType {\n    ArgMin,\n    ArgMax,\n  };\n\n  explicit ArgReduce(Stream stream, ReduceType reduce_type, int axis)\n      : UnaryPrimitive(stream), reduce_type_(reduce_type), axis_(axis){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_PRINT(ArgReduce)\n  bool is_equivalent(const Primitive& other) const override;\n  std::vector<std::vector<int>> output_shapes(\n      const std::vector<array>& inputs) override;\n\n private:\n  ReduceType reduce_type_;\n  int axis_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ArgSort : public UnaryPrimitive {\n public:\n  explicit ArgSort(Stream stream, int axis)\n      : UnaryPrimitive(stream), axis_(axis){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_PRINT(ArgSort)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  int axis_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass AsType : public UnaryPrimitive {\n public:\n  explicit AsType(Stream stream, Dtype dtype)\n      : UnaryPrimitive(stream), dtype_(dtype){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(AsType)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  Dtype dtype_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass AsStrided : public UnaryPrimitive {\n public:\n  explicit AsStrided(\n      Stream stream,\n      std::vector<int> shape,\n      std::vector<size_t> strides,\n      size_t offset)\n      : UnaryPrimitive(stream),\n        shape_(std::move(shape)),\n        strides_(std::move(strides)),\n        offset_(offset){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_GRADS()\n  DEFINE_PRINT(AsStrided)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> shape_;\n  std::vector<size_t> strides_;\n  size_t offset_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Broadcast : public UnaryPrimitive {\n public:\n  explicit Broadcast(Stream stream, const std::vector<int>& shape)\n      : UnaryPrimitive(stream), shape_(shape){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Broadcast)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> shape_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Ceil : public UnaryPrimitive {\n public:\n  explicit Ceil(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Ceil)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Compiled : public Primitive {\n public:\n  /*\n   * The inputs, outputs and tape are either tracers or constants.\n   * - The tape should not contain the inputs, but it should contain the\n   *   outputs.\n   * - The tape should also have only one array per primitive for multi-output\n   *   primitives.\n   * - The constant_ids contains ids of arrays in the input list that are safe\n   *   to treat as scalar constants.\n   */\n  explicit Compiled(\n      Stream stream,\n      std::vector<array> inputs,\n      std::vector<array> outputs,\n      std::vector<array> tape,\n      std::unordered_set<uintptr_t> constant_ids);\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  std::vector<std::vector<int>> output_shapes(\n      const std::vector<array>& inputs) override;\n  void print(std::ostream& os) override;\n  bool is_equivalent(const Primitive& other) const override;\n\n  std::string lib_name() const {\n    return kernel_lib_;\n  }\n\n private:\n  const std::vector<array> inputs_;\n  const std::vector<array> outputs_;\n  const std::vector<array> tape_;\n  const std::unordered_set<uintptr_t> constant_ids_;\n\n  std::string kernel_lib_;\n};\n\nclass Concatenate : public UnaryPrimitive {\n public:\n  explicit Concatenate(Stream stream, int axis)\n      : UnaryPrimitive(stream), axis_(axis){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Concatenate)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  int axis_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Convolution : public UnaryPrimitive {\n public:\n  explicit Convolution(\n      Stream stream,\n      const std::vector<int>& kernel_strides,\n      const std::vector<int>& padding,\n      const std::vector<int>& kernel_dilation,\n      const std::vector<int>& input_dilation,\n      const int groups = 1,\n      const bool flip = false)\n      : UnaryPrimitive(stream),\n        padding_(padding),\n        kernel_strides_(kernel_strides),\n        kernel_dilation_(kernel_dilation),\n        input_dilation_(input_dilation),\n        groups_(groups),\n        flip_(flip){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_PRINT(Convolution)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> padding_;\n  std::vector<int> kernel_strides_;\n  std::vector<int> kernel_dilation_;\n  std::vector<int> input_dilation_;\n  int groups_;\n  bool flip_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Copy : public UnaryPrimitive {\n public:\n  explicit Copy(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Copy)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Cos : public UnaryPrimitive {\n public:\n  explicit Cos(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Cos)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Cosh : public UnaryPrimitive {\n public:\n  explicit Cosh(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Cosh)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass CustomVJP : public Primitive {\n public:\n  explicit CustomVJP(\n      Stream stream,\n      std::function<std::vector<array>(\n          const std::vector<array>&,\n          const std::vector<array>&,\n          const std::vector<array>&)> fun)\n      : Primitive(stream), vjp_fun_(std::move(fun)) {}\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotan,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_PRINT(CustomVJP);\n\n private:\n  void eval(const std::vector<array>& inputs, std::vector<array>& outputs);\n\n  std::function<std::vector<array>(\n      const std::vector<array>&,\n      const std::vector<array>&,\n      const std::vector<array>&)>\n      vjp_fun_;\n};\n\nclass Depends : public Primitive {\n public:\n  explicit Depends(Stream stream) : Primitive(stream) {}\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotan,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_PRINT(Depends);\n\n private:\n  void eval(const std::vector<array>& inputs, std::vector<array>& outputs);\n};\n\nclass Divide : public UnaryPrimitive {\n public:\n  explicit Divide(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Divide)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass DivMod : public Primitive {\n public:\n  explicit DivMod(Stream stream) : Primitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(DivMod)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  std::vector<std::vector<int>> output_shapes(\n      const std::vector<array>& inputs) override {\n    return std::vector{inputs[0].shape(), inputs[0].shape()};\n  };\n\n private:\n  void eval(const std::vector<array>& inputs, std::vector<array>& outputs);\n};\n\nclass Select : public UnaryPrimitive {\n public:\n  explicit Select(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Select)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Remainder : public UnaryPrimitive {\n public:\n  explicit Remainder(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Remainder)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Equal : public UnaryPrimitive {\n public:\n  explicit Equal(Stream stream, bool equal_nan = false)\n      : UnaryPrimitive(stream), equal_nan_(equal_nan){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n  void print(std::ostream& os) override {\n    if (equal_nan_) {\n      os << \"NanEqual\";\n    } else {\n      os << \"Equal\";\n    }\n  }\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n  bool equal_nan_;\n};\n\nclass Erf : public UnaryPrimitive {\n public:\n  explicit Erf(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Erf)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass ErfInv : public UnaryPrimitive {\n public:\n  explicit ErfInv(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(ErfInv)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Exp : public UnaryPrimitive {\n public:\n  explicit Exp(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Exp)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Expm1 : public UnaryPrimitive {\n public:\n  explicit Expm1(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Expm1)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass FFT : public UnaryPrimitive {\n public:\n  explicit FFT(\n      Stream stream,\n      const std::vector<size_t>& axes,\n      bool inverse,\n      bool real)\n      : UnaryPrimitive(stream), axes_(axes), inverse_(inverse), real_(real){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(FFT)\n\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<size_t> axes_;\n  bool inverse_;\n  bool real_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Floor : public UnaryPrimitive {\n public:\n  explicit Floor(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Floor)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Full : public UnaryPrimitive {\n public:\n  explicit Full(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Full)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Gather : public UnaryPrimitive {\n public:\n  explicit Gather(\n      Stream stream,\n      const std::vector<int>& axes,\n      const std::vector<int>& slice_sizes)\n      : UnaryPrimitive(stream), axes_(axes), slice_sizes_(slice_sizes){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Gather)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n  std::vector<int> axes_;\n  std::vector<int> slice_sizes_;\n};\n\nclass Greater : public UnaryPrimitive {\n public:\n  explicit Greater(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Greater)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass GreaterEqual : public UnaryPrimitive {\n public:\n  explicit GreaterEqual(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(GreaterEqual)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Less : public UnaryPrimitive {\n public:\n  explicit Less(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Less)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass LessEqual : public UnaryPrimitive {\n public:\n  explicit LessEqual(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(LessEqual)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Load : public UnaryPrimitive {\n public:\n  explicit Load(\n      Stream stream,\n      std::shared_ptr<io::Reader> reader,\n      size_t offset,\n      bool swap_endianness = false)\n      : UnaryPrimitive(stream),\n        reader_(reader),\n        offset_(offset),\n        swap_endianness_(swap_endianness){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_PRINT(Load)\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n  std::shared_ptr<io::Reader> reader_;\n  size_t offset_;\n  bool swap_endianness_;\n};\n\nclass Log : public UnaryPrimitive {\n public:\n  enum Base { two, ten, e };\n\n  explicit Log(Stream stream, Base base)\n      : UnaryPrimitive(stream), base_(base){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n  void print(std::ostream& os) override {\n    switch (base_) {\n      case e:\n        os << \"Log\";\n        break;\n      case two:\n        os << \"Log2\";\n        break;\n      case ten:\n        os << \"Log10\";\n        break;\n    }\n  }\n\n private:\n  Base base_;\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Log1p : public UnaryPrimitive {\n public:\n  explicit Log1p(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Log1p)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass LogicalNot : public UnaryPrimitive {\n public:\n  explicit LogicalNot(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(LogicalNot)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass LogicalAnd : public UnaryPrimitive {\n public:\n  explicit LogicalAnd(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(LogicalAnd)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass LogicalOr : public UnaryPrimitive {\n public:\n  explicit LogicalOr(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(LogicalOr)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass LogAddExp : public UnaryPrimitive {\n public:\n  explicit LogAddExp(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(LogAddExp)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Matmul : public UnaryPrimitive {\n public:\n  explicit Matmul(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  DEFINE_VMAP()\n  DEFINE_PRINT(Matmul)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n};\n\nclass Maximum : public UnaryPrimitive {\n public:\n  explicit Maximum(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Maximum)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Minimum : public UnaryPrimitive {\n public:\n  explicit Minimum(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Minimum)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Multiply : public UnaryPrimitive {\n public:\n  explicit Multiply(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Multiply)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Negative : public UnaryPrimitive {\n public:\n  explicit Negative(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Negative)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass NotEqual : public UnaryPrimitive {\n public:\n  explicit NotEqual(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(NotEqual)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass NumberOfElements : public UnaryPrimitive {\n public:\n  explicit NumberOfElements(\n      Stream stream,\n      std::vector<int> axes,\n      bool inverted,\n      Dtype dtype)\n      : UnaryPrimitive(stream),\n        axes_(std::move(axes)),\n        inverted_(inverted),\n        dtype_(dtype) {}\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_PRINT(NumberOfElements)\n  bool is_equivalent(const Primitive& other) const override;\n  std::vector<std::vector<int>> output_shapes(\n      const std::vector<array>& inputs) override {\n    return {{}};\n  }\n\n private:\n  std::vector<int> axes_;\n  bool inverted_;\n  Dtype dtype_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Pad : public UnaryPrimitive {\n public:\n  explicit Pad(\n      Stream stream,\n      const std::vector<int>& axes,\n      const std::vector<int>& low_pad_size,\n      const std::vector<int>& high_pad_size)\n      : UnaryPrimitive(stream),\n        axes_(axes),\n        low_pad_size_(low_pad_size),\n        high_pad_size_(high_pad_size){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Pad)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> axes_;\n  std::vector<int> low_pad_size_;\n  std::vector<int> high_pad_size_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Partition : public UnaryPrimitive {\n public:\n  explicit Partition(Stream stream, int kth, int axis)\n      : UnaryPrimitive(stream), kth_(kth), axis_(axis){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Partition)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  int kth_;\n  int axis_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Power : public UnaryPrimitive {\n public:\n  explicit Power(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Power)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass QuantizedMatmul : public UnaryPrimitive {\n public:\n  e\nxplicit QuantizedMatmul(\n      Stream stream,\n      int group_size,\n      int bits,\n      bool transpose)\n      : UnaryPrimitive(stream),\n        group_size_(group_size),\n        bits_(bits),\n        transpose_(transpose){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(QuantizedMatmul)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  int group_size_;\n  int bits_;\n  bool transpose_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass RandomBits : public UnaryPrimitive {\n public:\n  explicit RandomBits(Stream stream, const std::vector<int>& shape, int width)\n      : UnaryPrimitive(stream), shape_(shape), width_(width){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_PRINT(RandomBits)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> shape_;\n  int width_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Reshape : public UnaryPrimitive {\n public:\n  explicit Reshape(Stream stream, const std::vector<int>& shape)\n      : UnaryPrimitive(stream), shape_(shape){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Reshape)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> shape_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n\n  std::pair<bool, std::vector<size_t>> prepare_reshape(\n      const array& in,\n      const array& out);\n  void shared_buffer_reshape(\n      const array& in,\n      const std::vector<size_t>& out_strides,\n      array& out);\n};\n\nclass Reduce : public UnaryPrimitive {\n public:\n  enum ReduceType { And, Or, Sum, Prod, Min, Max };\n\n  explicit Reduce(\n      Stream stream,\n      ReduceType reduce_type,\n      const std::vector<int>& axes)\n      : UnaryPrimitive(stream), reduce_type_(reduce_type), axes_(axes){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n\n  std::vector<array> vjp(\n      const std::vector<array>& primals,\n      const std::vector<array>& cotangents,\n      const std::vector<int>& argnums,\n      const std::vector<array>& outputs) override;\n\n  std::vector<std::vector<int>> output_shapes(\n      const std::vector<array>& inputs) override;\n\n  void print(std::ostream& os) override {\n    switch (reduce_type_) {\n      case And:\n        os << \"And\";\n      case Or:\n        os << \"And\";\n        break;\n      case Sum:\n        os << \"Sum\";\n        break;\n      case Prod:\n        os << \"Prod\";\n        break;\n      case Min:\n        os << \"Min\";\n        break;\n      case Max:\n        os << \"Max\";\n        break;\n    }\n    os << \" Reduce\";\n  }\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  ReduceType reduce_type_;\n  std::vector<int> axes_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Round : public UnaryPrimitive {\n public:\n  explicit Round(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Round)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Scan : public UnaryPrimitive {\n public:\n  enum ReduceType { Max, Min, Sum, Prod };\n\n  explicit Scan(\n      Stream stream,\n      ReduceType reduce_type,\n      int axis,\n      bool reverse,\n      bool inclusive)\n      : UnaryPrimitive(stream),\n        reduce_type_(reduce_type),\n        axis_(axis),\n        reverse_(reverse),\n        inclusive_(inclusive){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS();\n\n  void print(std::ostream& os) override {\n    os << \"Cum\";\n    switch (reduce_type_) {\n      case Sum:\n        os << \"Sum\";\n        break;\n      case Prod:\n        os << \"Prod\";\n        break;\n      case Min:\n        os << \"Min\";\n        break;\n      case Max:\n        os << \"Max\";\n        break;\n    }\n    os << \" Reduce\";\n  }\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  ReduceType reduce_type_;\n  int axis_;\n  bool reverse_;\n  bool inclusive_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Scatter : public UnaryPrimitive {\n public:\n  enum ReduceType { Max, Min, Sum, Prod, None };\n\n  explicit Scatter(\n      Stream stream,\n      ReduceType reduce_type,\n      const std::vector<int>& axes)\n      : UnaryPrimitive(stream), reduce_type_(reduce_type), axes_(axes){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_GRADS();\n  void print(std::ostream& os) override {\n    os << \"Scatter\";\n    switch (reduce_type_) {\n      case Sum:\n        os << \" Sum\";\n        break;\n      case Prod:\n        os << \" Prod\";\n        break;\n      case Min:\n        os << \" Min\";\n        break;\n      case Max:\n        os << \" Max\";\n        break;\n      case None:\n        break;\n    }\n  }\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n  ReduceType reduce_type_;\n  std::vector<int> axes_;\n};\n\nclass Sigmoid : public UnaryPrimitive {\n public:\n  explicit Sigmoid(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Sigmoid)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Sign : public UnaryPrimitive {\n public:\n  explicit Sign(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Sign)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Sin : public UnaryPrimitive {\n public:\n  explicit Sin(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Sin)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Sinh : public UnaryPrimitive {\n public:\n  explicit Sinh(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Sinh)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Slice : public UnaryPrimitive {\n public:\n  explicit Slice(\n      Stream stream,\n      const std::vector<int>& start_indices,\n      const std::vector<int>& end_indices,\n      const std::vector<int>& strides)\n      : UnaryPrimitive(stream),\n        start_indices_(start_indices),\n        end_indices_(end_indices),\n        strides_(strides){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Slice)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> start_indices_;\n  std::vector<int> end_indices_;\n  std::vector<int> strides_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n\n  std::tuple<bool, int64_t, std::vector<int64_t>> prepare_slice(\n      const array& in);\n  void shared_buffer_slice(\n      const array& in,\n      const std::vector<size_t>& out_strides,\n      size_t data_offset,\n      array& out);\n};\n\nclass SliceUpdate : public UnaryPrimitive {\n public:\n  explicit SliceUpdate(\n      Stream stream,\n      const std::vector<int>& start_indices,\n      const std::vector<int>& end_indices,\n      const std::vector<int>& strides)\n      : UnaryPrimitive(stream),\n        start_indices_(start_indices),\n        end_indices_(end_indices),\n        strides_(strides){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(SliceUpdate)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  std::vector<int> start_indices_;\n  std::vector<int> end_indices_;\n  std::vector<int> strides_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n\n  std::tuple<int64_t, std::vector<int64_t>> prepare_slice(const array& in);\n};\n\nclass Softmax : public UnaryPrimitive {\n public:\n  explicit Softmax(Stream stream, bool precise)\n      : UnaryPrimitive(stream), precise_(precise){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Softmax)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n  bool precise_;\n};\n\nclass Sort : public UnaryPrimitive {\n public:\n  explicit Sort(Stream stream, int axis)\n      : UnaryPrimitive(stream), axis_(axis){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Sort)\n  DEFINE_INPUT_OUTPUT_SHAPE()\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  int axis_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Split : public Primitive {\n public:\n  explicit Split(Stream stream, const std::vector<int>& indices, int axis)\n      : Primitive(stream), indices_(indices), axis_(axis){};\n\n  void eval_cpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n  void eval_gpu(const std::vector<array>& inputs, std::vector<array>& outputs)\n      override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Split)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  void eval(const std::vector<array>& inputs, std::vector<array>& outputs);\n\n  std::vector<int> indices_;\n  int axis_;\n};\n\nclass Square : public UnaryPrimitive {\n public:\n  explicit Square(Stream stream) : UnaryPrimitive(stream){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(Square)\n  DEFINE_DEFAULT_IS_EQUIVALENT()\n  DEFINE_INPUT_OUTPUT_SHAPE()\n\n private:\n  void eval(const std::vector<array>& inputs, array& out);\n};\n\nclass Sqrt : public UnaryPrimitive {\n public:\n  explicit Sqrt(Stream stream, bool recip = false)\n...\n", "needle_token_start": 11860, "needle_token_end": 11926, "code_context_ntokens": 15856, "output": ["```cpp\nclass QuantizedMatmul : public UnaryPrimitive {\n public:\n  explicit QuantizedMatmul(\n      Stream stream,\n      int group_size,\n      int bits,\n      bool transpose)\n      : UnaryPrimitive(stream),\n        group_size_(group_size),\n        bits_(bits),\n        transpose_(transpose){};\n\n  void eval_cpu(const std::vector<array>& inputs, array& out) override;\n  void eval_gpu(const std::vector<array>& inputs, array& out) override;\n\n  DEFINE_VMAP()\n  DEFINE_GRADS()\n  DEFINE_PRINT(QuantizedMatmul)\n  bool is_equivalent(const Primitive& other) const override;\n\n private:\n  int group_size_;\n  int bits_;\n  bool transpose_;\n\n  void eval(const std::vector<array>& inputs, array& out);\n};\n```"]}
{"repo": "ml-explore/mlx", "name": "sqrt", "language": "cpp", "path": "mlx/ops.cpp", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: Computes the square root of each element in a numerical array.\n2. **Input**: A numerical array and an optional specification of the computational context (like CPU or GPU).\n3. **Output**: A new array of the same shape as the input, containing the square roots of the original elements, ensuring the data type is at least a floating-point type to accommodate non-integer results.\n4. **Procedure**: The function first ensures the data type of the input array can handle floating-point results. It then applies a square root operation to each element of the input array, producing an output array with the results in the same shape as the input.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    const std::vector<int>& axes,\n    bool keepdims /* = false */,\n    int ddof /* = 0*/,\n    StreamOrDevice s /* = {}*/) {\n  auto dtype = at_least_float(a.dtype());\n  auto mu2 = square(mean(a, axes, keepdims, s), s);\n  auto a2 = mean(square(a, s), axes, keepdims, s);\n  auto v = subtract(a2, mu2, s);\n\n  if (ddof != 0) {\n    auto nelements = number_of_elements(a, axes, false, dtype, s);\n    auto factor = divide(\n        nelements,\n        maximum(subtract(nelements, array(ddof, dtype), s), array(0, dtype), s),\n        s);\n    v = multiply(v, factor, s);\n  }\n\n  return v;\n}\n\narray var(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    int ddof /* = 0*/,\n    StreamOrDevice s /* = {} */) {\n  return var(a, std::vector<int>{axis}, keepdims, ddof, to_stream(s));\n}\n\narray std(\n    const array& a,\n    bool keepdims,\n    int ddof /* = 0*/,\n    StreamOrDevice s /* = {}*/) {\n  std::vector<int> axes(a.ndim());\n  std::iota(axes.begin(), axes.end(), 0);\n  return std(a, axes, keepdims, ddof, to_stream(s));\n}\n\narray std(\n    const array& a,\n    const std::vector<int>& axes,\n    bool keepdims /* = false */,\n    int ddof /* = 0*/,\n    StreamOrDevice s /* = {}*/) {\n  return sqrt(var(a, axes, keepdims, ddof, s), s);\n}\n\narray std(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    int ddof /* = 0*/,\n    StreamOrDevice s /* = {} */) {\n  return std(a, std::vector<int>{axis}, keepdims, ddof, to_stream(s));\n}\n\narray prod(const array& a, bool keepdims, StreamOrDevice s /* = {}*/) {\n  std::vector<int> axes(a.ndim());\n  std::iota(axes.begin(), axes.end(), 0);\n  return prod(a, axes, keepdims, s);\n}\n\narray prod(\n    const array& a,\n    const std::vector<int>& axes,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {}*/) {\n  if (axes.empty()) {\n    return a;\n  }\n  auto [out_shape, sorted_axes] = compute_reduce_shape(axes, a.shape());\n  auto out = array(\n      out_shape,\n      a.dtype(),\n      std::make_shared<Reduce>(to_stream(s), Reduce::Prod, sorted_axes),\n      {a});\n  if (!keepdims) {\n    out = squeeze(out, sorted_axes, s);\n  }\n  return out;\n}\n\narray prod(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  return prod(a, std::vector<int>{axis}, keepdims, s);\n}\n\narray max(const array& a, bool keepdims, StreamOrDevice s /* = {}*/) {\n  std::vector<int> axes(a.ndim());\n  std::iota(axes.begin(), axes.end(), 0);\n  return max(a, axes, keepdims, s);\n}\n\narray max(\n    const array& a,\n    const std::vector<int>& axes,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {}*/) {\n  if (a.size() == 0) {\n    throw std::invalid_argument(\"[max] Cannot max reduce zero size array.\");\n  }\n  if (axes.empty()) {\n    return a;\n  }\n  auto [out_shape, sorted_axes] = compute_reduce_shape(axes, a.shape());\n  auto out = array(\n      out_shape,\n      a.dtype(),\n      std::make_shared<Reduce>(to_stream(s), Reduce::Max, sorted_axes),\n      {a});\n  if (!keepdims) {\n    out = squeeze(out, sorted_axes, s);\n  }\n  return out;\n}\n\narray max(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  return max(a, std::vector<int>{axis}, keepdims, s);\n}\n\narray min(const array& a, bool keepdims, StreamOrDevice s /* = {}*/) {\n  std::vector<int> axes(a.ndim());\n  std::iota(axes.begin(), axes.end(), 0);\n  return min(a, axes, keepdims, s);\n}\n\narray min(\n    const array& a,\n    const std::vector<int>& axes,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {}*/) {\n  if (a.size() == 0) {\n    throw std::invalid_argument(\"[min] Cannot min reduce zero size array.\");\n  }\n  if (axes.empty()) {\n    return a;\n  }\n  auto [out_shape, sorted_axes] = compute_reduce_shape(axes, a.shape());\n  auto out = array(\n      out_shape,\n      a.dtype(),\n      std::make_shared<Reduce>(to_stream(s), Reduce::Min, sorted_axes),\n      {a});\n  if (!keepdims) {\n    out = squeeze(out, sorted_axes, s);\n  }\n  return out;\n}\n\narray min(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  return min(a, std::vector<int>{axis}, keepdims, s);\n}\n\narray argmin(const array& a, bool keepdims, StreamOrDevice s /* = {} */) {\n  int size = a.size();\n  auto result = argmin(reshape(a, {size}, s), 0, true, s);\n  if (keepdims) {\n    result = reshape(result, std::vector<int>(a.shape().size(), 1), s);\n  } else {\n    result = squeeze(result, s);\n  }\n  return result;\n}\n\narray argmin(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  if (a.size() == 0) {\n    throw std::invalid_argument(\n        \"[argmin] Cannot argmin reduce zero size array.\");\n  }\n  auto [out_shape, sorted_axes] = compute_reduce_shape({axis}, a.shape());\n  auto out = array(\n      out_shape,\n      uint32,\n      std::make_shared<ArgReduce>(\n          to_stream(s), ArgReduce::ArgMin, sorted_axes[0]),\n      {a});\n  if (!keepdims) {\n    out = squeeze(out, sorted_axes, s);\n  }\n  return out;\n}\n\narray argmax(const array& a, bool keepdims, StreamOrDevice s /* = {} */) {\n  int size = a.size();\n  auto result = argmax(reshape(a, {size}, s), 0, true, s);\n  if (keepdims) {\n    result = reshape(result, std::vector<int>(a.shape().size(), 1), s);\n  } else {\n    result = squeeze(result, s);\n  }\n  return result;\n}\n\narray argmax(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  if (a.size() == 0) {\n    throw std::invalid_argument(\n        \"[argmax] Cannot argmax reduce zero size array.\");\n  }\n  auto [out_shape, sorted_axes] = compute_reduce_shape({axis}, a.shape());\n  auto out = array(\n      out_shape,\n      uint32,\n      std::make_shared<ArgReduce>(\n          to_stream(s), ArgReduce::ArgMax, sorted_axes[0]),\n      {a});\n  if (!keepdims) {\n    out = squeeze(out, sorted_axes, s);\n  }\n  return out;\n}\n\n/** Returns a sorted copy of the flattened array. */\narray sort(const array& a, StreamOrDevice s /* = {} */) {\n  int size = a.size();\n  return sort(reshape(a, {size}, s), 0, s);\n}\n\n/** Returns a sorted copy of the array along a given axis. */\narray sort(const array& a, int axis, StreamOrDevice s /* = {} */) {\n  // Check for valid axis\n  if (axis + static_cast<int>(a.ndim()) < 0 ||\n      axis >= static_cast<int>(a.ndim())) {\n    std::ostringstream msg;\n    msg << \"[sort] Received invalid axis \" << axis << \" for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  // TODO: Fix GPU kernel\n  if (a.shape(axis) >= (1u << 21) && to_stream(s).device.type == Device::gpu) {\n    std::ostringstream msg;\n    msg << \"[sort] GPU sort cannot handle sort axis of >= 2M elements,\"\n        << \" got array with sort axis size \" << a.shape(axis) << \".\"\n        << \" Please place this operation on the CPU instead.\";\n    throw std::runtime_error(msg.str());\n  }\n\n  return array(\n      a.shape(), a.dtype(), std::make_shared<Sort>(to_stream(s), axis), {a});\n}\n\n/** Returns indices that sort the flattened array. */\narray argsort(const array& a, StreamOrDevice s /* = {} */) {\n  int size = a.size();\n  return argsort(reshape(a, {size}, s), 0, s);\n}\n\n/** Returns indices that sort the array along a given axis. */\narray argsort(const array& a, int axis, StreamOrDevice s /* = {} */) {\n  // Check for valid axis\n  if (axis + static_cast<int>(a.ndim()) < 0 ||\n      axis >= static_cast<int>(a.ndim())) {\n    std::ostringstream msg;\n    msg << \"[argsort] Received invalid axis \" << axis << \" for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  // TODO: Fix GPU kernel\n  if (a.shape(axis) >= (1u << 21) && to_stream(s).device.type == Device::gpu) {\n    std::ostringstream msg;\n    msg << \"[argsort] GPU sort cannot handle sort axis of >= 2M elements,\"\n        << \" got array with sort axis size \" << a.shape(axis) << \".\"\n        << \" Please place this operation on the CPU instead.\";\n    throw std::runtime_error(msg.str());\n  }\n\n  return array(\n      a.shape(), uint32, std::make_shared<ArgSort>(to_stream(s), axis), {a});\n}\n\n/**\n * Returns a partitioned copy of the flattened array\n * such that the smaller kth elements are first.\n **/\narray partition(const array& a, int kth, StreamOrDevice s /* = {} */) {\n  int size = a.size();\n  return partition(reshape(a, {size}, s), kth, 0, s);\n}\n\n/**\n * Returns a partitioned copy of the array along a given axis\n * such that the smaller kth elements are first.\n **/\narray partition(\n    const array& a,\n    int kth,\n    int axis,\n    StreamOrDevice s /* = {} */) {\n  // Check for valid axis\n  if (axis + static_cast<int>(a.ndim()) < 0 ||\n      axis >= static_cast<int>(a.ndim())) {\n    std::ostringstream msg;\n    msg << \"[partition] Received invalid axis \" << axis << \" for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  int axis_ = axis < 0 ? axis + a.ndim() : axis;\n  int kth_ = kth < 0 ? kth + a.shape(axis) : kth;\n  if (kth_ < 0 || kth_ >= a.shape(axis_)) {\n    std::ostringstream msg;\n    msg << \"[partition] Received invalid kth \" << kth << \"along axis \" << axis\n        << \" for array with shape: \" << a.shape();\n    throw std::invalid_argument(msg.str());\n  }\n  return array(\n      a.shape(),\n      a.dtype(),\n      std::make_shared<Partition>(to_stream(s), kth_, axis_),\n      {a});\n}\n\n/**\n * Returns indices that partition the flattened array\n * such that the smaller kth elements are first.\n **/\narray argpartition(const array& a, int kth, StreamOrDevice s /* = {} */) {\n  int size = a.size();\n  return argpartition(reshape(a, {size}, s), kth, 0, s);\n}\n\n/**\n * Returns indices that partition the array along a given axis\n * such that the smaller kth elements are first.\n **/\narray argpartition(\n    const array& a,\n    int kth,\n    int axis,\n    StreamOrDevice s /* = {} */) {\n  // Check for valid axis\n  if (axis + static_cast<int>(a.ndim()) < 0 ||\n      axis >= static_cast<int>(a.ndim())) {\n    std::ostringstream msg;\n    msg << \"[argpartition] Received invalid axis \" << axis << \" for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  int axis_ = axis < 0 ? axis + a.ndim() : axis;\n  int kth_ = kth < 0 ? kth + a.shape(axis) : kth;\n  if (kth_ < 0 || kth_ >= a.shape(axis_)) {\n    std::ostringstream msg;\n    msg << \"[argpartition] Received invalid kth \" << kth << \" along axis \"\n        << axis << \" for array with shape: \" << a.shape();\n    throw std::invalid_argument(msg.str());\n  }\n  return array(\n      a.shape(),\n      uint32,\n      std::make_shared<ArgPartition>(to_stream(s), kth_, axis_),\n      {a});\n}\n\n/** Returns topk elements of the flattened array. */\narray topk(const array& a, int k, StreamOrDevice s /* = {}*/) {\n  int size = a.size();\n  return topk(reshape(a, {size}, s), k, 0, s);\n}\n\n/** Returns topk elements of the array along a given axis. */\narray topk(const array& a, int k, int axis, StreamOrDevice s /* = {}*/) {\n  // Check for valid axis\n  int axis_ = axis < 0 ? axis + a.ndim() : axis;\n  if (axis_ < 0 || axis_ >= static_cast<int>(a.ndim())) {\n    std::ostringstream msg;\n    msg << \"[topk] Received invalid axis \" << axis << \" for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (k < 0 || k > a.shape(axis_)) {\n    std::ostringstream msg;\n    msg << \"[topk] Received invalid k=\" << k << \" along axis \" << axis\n        << \" for array with shape: \" << a.shape();\n    throw std::invalid_argument(msg.str());\n  }\n\n  // Return early if the whole input was requested.\n  if (k == a.shape(axis_)) {\n    return a;\n  }\n\n  array a_partitioned = partition(a, -k, axis_, s);\n  std::vector<int> slice_starts(a.ndim(), 0);\n  std::vector<int> slice_ends = a.shape();\n  slice_starts[axis_] = a.shape(axis_) - k;\n  return slice(a_partitioned, slice_starts, slice_ends, s);\n}\n\narray logsumexp(const array& a, bool keepdims, StreamOrDevice s /* = {}*/) {\n  std::vector<int> axes(a.ndim());\n  std::iota(axes.begin(), axes.end(), 0);\n  return logsumexp(a, axes, keepdims, s);\n}\n\narray logsumexp(\n    const array& a,\n    const std::vector<int>& axes,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {}*/) {\n  auto maxval = stop_gradient(max(a, axes, true, s), s);\n  auto out = log(sum(exp(subtract(a, maxval, s), s), axes, keepdims, s), s);\n  out = add(out, reshape(maxval, out.shape(), s), s);\n  if (!keepdims) {\n    maxval = squeeze(maxval, axes, s);\n  }\n  return where(isinf(maxval, s), maxval, out, s);\n}\n\narray logsumexp(\n    const array& a,\n    int axis,\n    bool keepdims /* = false */,\n    StreamOrDevice s /* = {} */) {\n  return logsumexp(a, std::vector<int>{axis}, keepdims, s);\n}\n\narray abs(const array& a, StreamOrDevice s /* = {} */) {\n  auto out =\n      array(a.shape(), a.dtype(), std::make_shared<Abs>(to_stream(s)), {a});\n  if (a.dtype() == complex64) {\n    out = astype(out, float32, s);\n  }\n  return out;\n}\n\narray negative(const array& a, StreamOrDevice s /* = {} */) {\n  if (a.dtype() == bool_) {\n    auto msg = \"[negative] Not supported for bool, use logical_not instead.\";\n    throw std::invalid_argument(msg);\n  }\n  return array(\n      a.shape(), a.dtype(), std::make_shared<Negative>(to_stream(s)), {a});\n}\narray operator-(const array& a) {\n  return negative(a);\n}\n\narray sign(const array& a, StreamOrDevice s /* = {} */) {\n  return array(a.shape(), a.dtype(), std::make_shared<Sign>(to_stream(s)), {a});\n}\n\narray logical_not(const array& a, StreamOrDevice s /* = {} */) {\n  return array(\n      a.shape(),\n      bool_,\n      std::make_shared<LogicalNot>(to_stream(s)),\n      {astype(a, bool_, s)});\n}\n\narray logical_and(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  // Broadcast arrays to a common shape\n  auto inputs = broadcast_arrays(astype(a, bool_, s), astype(b, bool_, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      bool_,\n      std::make_shared<LogicalAnd>(to_stream(s)),\n      std::move(inputs));\n}\narray operator&&(const array& a, const array& b) {\n  return logical_and(a, b);\n}\n\narray logical_or(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  // Broadcast arrays to a common shape\n  auto inputs = broadcast_arrays(astype(a, bool_, s), astype(b, bool_, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      bool_,\n      std::make_shared<LogicalOr>(to_stream(s)),\n      std::move(inputs));\n}\narray operator||(const array& a, const array& b) {\n  return logical_or(a, b);\n}\n\narray reciprocal(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  return divide(array(1.0f, dtype), a, to_stream(s));\n}\n\narray add(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto out_type = promote_types(a.dtype(), b.dtype());\n  auto inputs =\n      broadcast_arrays(astype(a, out_type, s), astype(b, out_type, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape, out_type, std::make_shared<Add>(to_stream(s)), std::move(inputs));\n}\n\narray operator+(const array& a, const array& b) {\n  return add(a, b);\n}\n\narray subtract(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto out_type = promote_types(a.dtype(), b.dtype());\n  auto inputs =\n      broadcast_arrays(astype(a, out_type, s), astype(b, out_type, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      out_type,\n      std::make_shared<Subtract>(to_stream(s)),\n      std::move(inputs));\n}\n\narray operator-(const array& a, const array& b) {\n  return subtract(a, b);\n}\n\narray multiply(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto out_type = promote_types(a.dtype(), b.dtype());\n  auto inputs =\n      broadcast_arrays(astype(a, out_type, s), astype(b, out_type, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      out_type,\n      std::make_shared<Multiply>(to_stream(s)),\n      std::move(inputs));\n}\n\narray operator*(const array& a, const array& b) {\n  return multiply(a, b);\n}\n\narray divide(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(promote_types(a.dtype(), b.dtype()));\n  auto inputs =\n      broadcast_arrays(astype(a, dtype, s), astype(b, dtype, to_stream(s)), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape, dtype, std::make_shared<Divide>(to_stream(s)), std::move(inputs));\n}\narray operator/(const array& a, const array& b) {\n  return divide(a, b);\n}\narray operator/(double a, const array& b) {\n  return divide(array(a), b);\n}\narray operator/(const array& a, double b) {\n  return divide(a, array(b));\n}\n\narray floor_divide(\n    const array& a,\n    const array& b,\n    StreamOrDevice s /* = {} */) {\n  auto dtype = promote_types(a.dtype(), b.dtype());\n  if (issubdtype(dtype, inexact)) {\n    return floor(divide(a, b, s), s);\n  }\n\n  auto inputs = broadcast_arrays(astype(a, dtype, s), astype(b, dtype, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape, dtype, std::make_shared<Divide>(to_stream(s)), std::move(inputs));\n}\n\narray remainder(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto dtype = promote_types(a.dtype(), b.dtype());\n  auto inputs =\n      broadcast_arrays(astype(a, dtype, s), astype(b, dtype, to_stream(s)), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      dtype,\n      std::make_shared<Remainder>(to_stream(s)),\n      std::move(inputs));\n}\narray operator%(const array& a, const array& b) {\n  return remainder(a, b);\n}\n\nstd::vector<array>\ndivmod(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto dtype = promote_types(a.dtype(), b.dtype());\n  if (issubdtype(dtype, complexfloating)) {\n    throw std::invalid_argument(\"[divmod] Complex type not supported.\");\n  }\n  auto inputs =\n      broadcast_arrays(astype(a, dtype, s), astype(b, dtype, to_stream(s)), s);\n  return array::make_arrays(\n      {inputs[0].shape(), inputs[0].shape()},\n      {inputs[0].dtype(), inputs[0].dtype()},\n      std::make_shared<DivMod>(to_stream(s)),\n      inputs);\n}\n\narray maximum(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto out_type = promote_types(a.dtype(), b.dtype());\n  auto inputs =\n      broadcast_arrays(astype(a, out_type, s), astype(b, out_type, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      out_type,\n      std::make_shared<Maximum>(to_stream(s)),\n      std::move(inputs));\n}\n\narray minimum(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto out_type = promote_types(a.dtype(), b.dtype());\n  auto inputs =\n      broadcast_arrays(astype(a, out_type, s), astype(b, out_type, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      out_type,\n      std::make_shared<Minimum>(to_stream(s)),\n      std::move(inputs));\n}\n\narray floor(const array& a, StreamOrDevice s /* = {} */) {\n  if (a.dtype() == complex64) {\n    throw std::invalid_argument(\"[floor] Not supported for complex64.\");\n  }\n  return array(\n      a.shape(), a.dtype(), std::make_shared<Floor>(to_stream(s)), {a});\n}\n\narray ceil(const array& a, StreamOrDevice s /* = {} */) {\n  if (a.dtype() == complex64) {\n    throw std::invalid_argument(\"[floor] Not supported for complex64.\");\n  }\n  return array(a.shape(), a.dtype(), std::make_shared<Ceil>(to_stream(s)), {a});\n}\n\narray square(const array& a, StreamOrDevice s /* = {} */) {\n  return array(\n      a.shape(), a.dtype(), std::make_shared<Square>(to_stream(s)), {a});\n}\n\narray exp(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(a.shape(), dtype, std::make_shared<Exp>(to_stream(s)), {input});\n}\n\narray expm1(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<Expm1>(to_stream(s)), {input});\n}\n\narray sin(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(a.shape(), dtype, std::make_shared<Sin>(to_stream(s)), {input});\n}\n\narray cos(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(a.shape(), dtype, std::make_shared<Cos>(to_stream(s)), {input});\n}\n\narray tan(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(a.shape(), dtype, std::make_shared<Tan>(to_stream(s)), {input});\n}\n\narray arcsin(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<ArcSin>(to_stream(s)), {input});\n}\n\narray arccos(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<ArcCos>(to_stream(s)), {input});\n}\n\narray arctan(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<ArcTan>(to_stream(s)), {input});\n}\n\narray sinh(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(a.shape(), dtype, std::make_shared<Sinh>(to_stream(s)), {input});\n}\n\narray cosh(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(a.shape(), dtype, std::make_shared<Cosh>(to_stream(s)), {input});\n}\n\narray tanh(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(a.shape(), dtype, std::make_shared<Tanh>(to_stream(s)), {input});\n}\n\narray arcsinh(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<ArcSinh>(to_stream(s)), {input});\n}\n\narray arccosh(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<ArcCosh>(to_stream(s)), {input});\n}\n\narray arctanh(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<ArcTanh>(to_stream(s)), {input});\n}\n\narray log(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<Log>(to_stream(s), Log::Base::e),\n      {input});\n}\n\narray log2(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<Log>(to_stream(s), Log::Base::two),\n      {input});\n}\n\narray log10(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<Log>(to_stream(s), Log::Base::ten),\n      {input});\n}\n\narray log1p(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<Log1p>(to_stream(s)), {input});\n}\n\narray logaddexp(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  // Make sure out type is floating point\n  auto out_type = at_least_float(promote_types(a.dtype(), b.dtype()));\n  auto inputs =\n      broadcast_arrays(astype(a, out_type, s), astype(b, out_type, s), s);\n  auto& shape = inputs[0].shape();\n  return array(\n      shape,\n      out_type,\n      std::make_shared<LogAddExp>(to_stream(s)),\n      std::move(inputs));\n}\n\narray sigmoid(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  auto input = astype(a, dtype, s);\n  return array(\n      a.shape(), dtype, std::make_shared<Sigmoid>(to_stream(s)), {input});\n}\n\narray erf(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<Erf>(to_stream(s)),\n      {astype(a, dtype, s)});\n}\n\narray erfinv(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<ErfInv>(to_stream(s)),\n      {astype(a, dtype, s)});\n}\n\narray stop_gradient(const array& a, StreamOrDevice s /* = {} */) {\n  return array(\n      a.shape(), a.dtype(), std::make_shared<StopGradient>(to_stream(s)), {a});\n}\n\narray round(const array& a, int decimals, StreamOrDevice s /* = {} */) {\n  if (decimals == 0) {\n    return array(\n        a.shape(), a.dtype(), std::make_shared<Round>(to_stream(s)), {a});\n  }\n\n  auto dtype = at_least_float(a.dtype());\n  float scale = std::pow(10, decimals);\n  auto result = multiply(a, array(scale, dtype), s);\n  result = round(result, 0, s);\n  result = multiply(result, array(1 / scale, dtype), s);\n\n  return astype(result, a.dtype(), s);\n}\n\narray matmul(\n    const array& in_a,\n    const array& in_b,\n    StreamOrDevice s /* = {} */) {\n  auto a = in_a;\n  auto b = in_b;\n  if (a.ndim() == 0 || b.ndim() == 0) {\n    throw std::invalid_argument(\n        \"[matmul] Got 0 dimension input. Inputs must \"\n        \"have at least one dimension.\");\n  }\n  if (a.ndim() == 1) {\n    // Insert a singleton dim in the beginning\n    a = reshape(a, {1, -1}, s);\n  }\n  if (b.ndim() == 1) {\n    // Insert a singleton dim at the end\n    b = reshape(b, {-1, 1}, s);\n  }\n  if (a.shape(-1) != b.shape(-2)) {\n    std::ostringstream msg;\n    msg << \"[matmul] Last dimension of first input with shape \" << a.shape()\n        << \" must match second to last dimension of\"\n        << \" second input with shape \" << b.shape() << \".\";\n    throw std::invalid_argument(msg.str());\n  }\n  // Type promotion\n  auto out_type = promote_types(a.dtype(), b.dtype());\n  if (!issubdtype(out_type, floating)) {\n    std::ostringstream msg;\n    msg << \"[matmul] Only real floating point types are supported but \"\n        << a.dtype() << \" and \" << b.dtype() << \" were provided which results\"\n        << \" in \" << out_type << \", which is not a real floating point type.\";\n    throw std::invalid_argument(msg.str());\n  }\n  if (a.dtype() != out_type) {\n    a = astype(a, out_type, s);\n  }\n  if (b.dtype() != out_type) {\n    b = astype(b, out_type, s);\n  }\n\n  // We can batch the multiplication by reshaping a\n  if (a.ndim() > 2 && b.ndim() == 2) {\n    std::vector<int> out_shape = a.shape();\n    a = reshape(a, {-1, out_shape.back()}, s);\n    out_shape.back() = b.shape(-1);\n    if (in_b.ndim() == 1) {\n      out_shape.pop_back();\n    }\n    auto out = array(\n        {a.shape(0), b.shape(1)},\n        out_type,\n        std::make_shared<Matmul>(to_stream(s)),\n        {a, b});\n    return reshape(out, out_shape, s);\n  }\n\n  if (a.ndim() > 2 || b.ndim() > 2) {\n    std::vector<int> bsx_a(a.shape().begin(), a.shape().end() - 2);\n    std::vector<int> bsx_b(b.shape().begin(), b.shape().end() - 2);\n    auto inner_shape = broadcast_shapes(bsx_a, bsx_b);\n\n    // Broadcast a\n    inner_shape.push_back(a.shape(-2));\n    inner_shape.push_back(a.shape(-1));\n    a = broadcast_to(a, inner_shape, s);\n\n    // Broadcast b\n    *(inner_shape.end() - 2) = b.shape(-2);\n    *(inner_shape.end() - 1) = b.shape(-1);\n    b = broadcast_to(b, inner_shape, s);\n  }\n\n  auto out_shape = a.shape();\n  out_shape.back() = b.shape(-1);\n\n  auto p = std::make_shared<Matmul>(to_stream(s));\n\n  // Remove the possibly inserted singleton dimensions\n  if (in_a.ndim() == 1 || in_b.ndim() == 1) {\n    auto out = array(out_shape, out_type, std::move(p), {a, b});\n    out_shape.erase(\n        out_shape.end() - ((in_a.ndim() == 1) ? 2 : 1),\n        out_shape.end() - ((in_b.ndim() == 1) ? 0 : 1));\n    return reshape(out, std::move(out_shape), s);\n  }\n  return array(std::move(out_shape), out_type, std::move(p), {a, b});\n}\n\narray gather(\n    const array& a,\n    const std::vector<array>& indices,\n    const std::vector<int>& axes,\n    const std::vector<int>& slice_sizes,\n    StreamOrDevice s /* = {} */) {\n  // Checks that indices, dimensions, and slice_sizes are all valid\n  if (indices.size() > a.ndim()) {\n    std::ostringstream msg;\n    msg << \"[gather] Too many index arrays. Got \" << indices.size()\n        << \" index arrays for input with \" << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  std::set dims(axes.begin(), axes.end());\n  if (dims.size() != axes.size()) {\n    throw std::invalid_argument(\"[gather] Repeat axes not allowed in gather.\");\n  }\n  if (!dims.empty() && (*dims.begin() < 0 || *dims.rbegin() >= a.ndim())) {\n    throw std::invalid_argument(\"[gather] Axes don't match array dimensions.\");\n  }\n  if (indices.size() != axes.size()) {\n    throw std::invalid_argument(\n        \"[gather] Number of index arrays does not match number of axes.\");\n  }\n  for (auto& x : indices) {\n    if (x.dtype() == bool_) {\n      throw(\"[Gather] Boolean indices not supported.\");\n    }\n  }\n\n  if (slice_sizes.size() != a.ndim()) {\n    std::ostringstream msg;\n    msg << \"[gather] Got slice_sizes with size \" << slice_sizes.size()\n        << \" for array with \" << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  for (int i = 0; i < a.ndim(); ++i) {\n    if (slice_sizes[i] < 0 || slice_sizes[i] > a.shape(i)) {\n      std::ostringstream msg;\n      msg << \"[gather] Slice sizes must be in [0, a.shape(i)]. Got \"\n          << slice_sizes << \" for array with shape \" << a.shape() << \".\";\n      throw std::invalid_argument(msg.str());\n    }\n  }\n\n  // Promote indices to the same type\n  auto dtype = result_type(indices);\n  if (issubdtype(dtype, inexact)) {\n    throw std::invalid_argument(\n        \"[gather] Got indices with invalid dtype. Indices must be integral.\");\n  }\n\n  // Broadcast and cast indices if necessary\n  auto inputs = broadcast_arrays(indices);\n  for (auto& idx : inputs) {\n    idx = astype(idx, dtype, s);\n  }\n\n  std::vector<int> out_shape;\n  if (!inputs.empty()) {\n    out_shape = inputs[0].shape();\n  }\n  out_shape.insert(out_shape.end(), slice_sizes.begin(), slice_sizes.end());\n\n  inputs.insert(inputs.begin(), a);\n  return array(\n      out_shape,\n      a.dtype(),\n      std::make_shared<Gather>(to_stream(s), axes, slice_sizes),\n      inputs);\n}\n\narray take(\n    const array& a,\n    const array& indices,\n    int axis,\n    StreamOrDevice s /* = {} */) {\n  // Check for valid axis\n  if (axis + static_cast<int>(a.ndim()) < 0 ||\n      axis >= static_cast<int>(a.ndim())) {\n    std::ostringstream msg;\n    msg << \"[take] Received invalid axis \" << axis << \" for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  // Check for valid take\n  if (a.size() == 0 && indices.size() != 0) {\n    throw std::invalid_argument(\n        \"[take] Cannot do a non-empty take from an array with zero elements.\");\n  }\n\n  // Handle negative axis\n  axis = axis < 0 ? a.ndim() + axis : axis;\n\n  // Make slice sizes to pass to gather\n  std::vector<int> slice_sizes = a.shape();\n  slice_sizes[axis] = indices.size() > 0 ? 1 : 0;\n\n  auto out = gather(a, indices, axis, slice_sizes, s);\n\n  // Transpose indices dimensions to axis dimension\n  if (axis != 0) {\n    std::vector<int> t_axes(out.ndim());\n    std::iota(t_axes.begin(), t_axes.begin() + axis, indices.ndim());\n    std::iota(t_axes.begin() + axis, t_axes.begin() + axis + indices.ndim(), 0);\n    std::iota(\n        t_axes.begin() + axis + indices.ndim(),\n        t_axes.end(),\n        indices.ndim() + axis);\n    out = transpose(out, t_axes, s);\n  }\n\n  // Squeeze the axis we take over\n  std::vector<int> out_shape = out.shape();\n  out_shape.erase(out_shape.begin() + indices.ndim() + axis);\n  return reshape(out, out_shape, s);\n}\n\narray take(const array& a, const array& indices, StreamOrDevice s /* = {} */) {\n  return take(reshape(a, {-1}, s), indices, 0, s);\n}\n\narray take_along_axis(\n    const array& a,\n    const array& indices,\n    int axis,\n    StreamOrDevice s /* = {} */) {\n  if (axis + a.ndim() < 0 || axis >= static_cast<int>(a.ndim())) {\n    std::ostringstream msg;\n    msg << \"[take_along_axis] Received invalid axis \" << \" for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  if (indices.ndim() != a.ndim()) {\n    std::ostringstream msg;\n    msg << \"[take_along_axis] Indices of dimension \" << indices.ndim()\n        << \" does not match array of dimension \" << a.ndim() << \".\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  // Allow negative axis\n  axis = axis < 0 ? a.ndim() + axis : axis;\n\n  std::vector<array> nd_indices;\n  std::vector<int> index_shape(a.ndim(), 1);\n  for (int i = 0; i < a.ndim(); ++i) {\n    if (i == axis) {\n      nd_indices.push_back(indices);\n    } else {\n      // Reshape so they can be broadcast\n      index_shape[i] = a.shape(i);\n      nd_indices.push_back(reshape(arange(a.shape(i), s), index_shape, s));\n      index_shape[i] = 1;\n    }\n  }\n  std::vector<int> dims(a.ndim());\n  std::iota(dims.begin(), dims.end(), 0);\n  std::vector<int> slice_sizes(a.ndim(), a.size() > 0);\n  auto out = gather(a, nd_indices, dims, slice_sizes, s);\n\n  // Squeeze out the slice shape\n  std::vector<int> out_shape(\n      out.shape().begin(), out.shape().begin() + a.ndim());\n  return reshape(out, out_shape, s);\n}\n\n/** Scatter updates to given indices */\narray scatter(\n    const array& a,\n    const std::vector<array>& indices,\n    const array& updates,\n    const std::vector<int>& axes,\n    Scatter::ReduceType mode /*= Scatter::ReduceType::None*/,\n    StreamOrDevice s /*= {}*/) {\n  // Checks that indices, dimensions, and slice_sizes are all valid\n  if (indices.size() > a.ndim()) {\n    std::ostringstream msg;\n    msg << \"[scatter] Too many index arrays. Got \" << indices.size()\n        << \" index arrays for input with \" << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  for (auto& x : indices) {\n    if (x.dtype() == bool_) {\n      throw(\"[scatter] Boolean indices not supported.\");\n    }\n  }\n\n  std::set dims(axes.begin(), axes.end());\n  if (dims.size() != axes.size()) {\n    throw std::invalid_argument(\n        \"[scatter] Repeat axes not allowed in scatter.\");\n  }\n  if (!dims.empty() && (*dims.begin() < 0 || *dims.rbegin() >= a.ndim())) {\n    throw std::invalid_argument(\"[scatter] Axes don't match array dimensions.\");\n  }\n  if (indices.size() != axes.size()) {\n    throw std::invalid_argument(\n        \"[scatter] Number of index arrays does not match number of axes.\");\n  }\n\n  // Broadcast and cast indices if necessary\n  auto inputs = broadcast_arrays(indices);\n\n  std::vector<int> idx_shape;\n  if (!inputs.empty()) {\n    idx_shape = inputs[0].shape();\n  }\n\n  if (updates.ndim() != (a.ndim() + idx_shape.size())) {\n    std::ostringstream msg;\n    msg << \"[scatter] Updates with \" << updates.ndim()\n        << \" dimensions does not match the sum of the array and indices \"\n           \"dimensions \"\n        << a.ndim() + idx_shape.size() << \".\";\n    throw std::invalid_argument(msg.str());\n  }\n  for (int i = 0; i < idx_shape.size(); ++i) {\n    if (updates.shape(i) != idx_shape[i]) {\n      std::ostringstream msg;\n      msg << \"[scatter] Update shape \" << updates.shape()\n          << \" is not valid for broadcasted index shape \" << idx_shape << \".\";\n      throw std::invalid_argument(msg.str());\n    }\n  }\n  for (int i = 0; i < a.ndim(); ++i) {\n    auto up_shape = updates.shape(i + idx_shape.size());\n    if (up_shape > a.shape(i)) {\n      std::ostringstream msg;\n      msg << \"[scatter] Updates with shape \" << updates.shape()\n          << \" are too large for array with shape \" << a.shape() << \".\";\n      throw std::invalid_argument(msg.str());\n    }\n  }\n\n  // Promote indices to the same type\n  auto dtype = result_type(indices);\n  if (issubdtype(dtype, inexact)) {\n    throw std::invalid_argument(\n        \"[scatter] Got indices with invalid dtype. Indices must be integral.\");\n  }\n  for (auto& idx : inputs) {\n    idx = astype(idx, dtype, s);\n  }\n\n  inputs.insert(inputs.begin(), a);\n  // TODO promote or cast?\n  inputs.push_back(astype(updates, a.dtype(), s));\n  return array(\n      a.shape(),\n      a.dtype(),\n      std::make_shared<Scatter>(to_stream(s), mode, axes),\n      inputs);\n}\n\narray scatter(\n    const array& a,\n    const std::vector<array>& indices,\n    const array& updates,\n    const std::vector<int>& axes,\n    StreamOrDevice s /*= {}*/) {\n  return scatter(a, indices, updates, axes, Scatter::None, s);\n}\n\narray scatter_add(\n    const array& a,\n    const std::vector<array>& indices,\n    const array& updates,\n    const std::vector<int>& axes,\n    StreamOrDevice s /*= {}*/) {\n  return scatter(a, indices, updates, axes, Scatter::Sum, s);\n}\n\narray scatter_prod(\n    const array& a,\n    const std::vector<array>& indices,\n    const array& updates,\n    const std::vector<int>& axes,\n    StreamOrDevice s /*= {}*/) {\n  return scatter(a, indices, updates, axes, Scatter::Prod, s);\n}\n\narray scatter_max(\n    const array& a,\n    const std::vector<array>& indices,\n    const array& updates,\n    const std::vector<int>& axes,\n    StreamOrDevice s /*= {}*/) {\n  return scatter(a, indices, updates, axes, Scatter::Max, s);\n}\n\narray scatter_min(\n    const array& a,\n    const std::vector<array>& indices,\n    const array& updates,\n    const std::vector<int>& axes,\n    StreamOrDevice s /*= {}*/) {\n  return scatter(a, indices, updates, axes, Scatter::Min, s);\n}\n\na\nrray sqrt(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<Sqrt>(to_stream(s)),\n      {astype(a, dtype, s)});\n}\n\narray rsqrt(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<Sqrt>(to_stream(s), true),\n      {astype(a, dtype, s)});\n}\n\narray softmax(\n    const array& a,\n    const std::vector<int>& axes,\n    bool precise /* = false */,\n    StreamOrDevice s /* = {}*/) {\n  if (axes.size() == 1 && (a.ndim() == axes[0] + 1 || axes[0] == -1)) {\n    auto dtype = at_least_float(a.dtype());\n    return array(\n        a.shape(),\n        dtype,\n        std::make_shared<Softmax>(to_stream(s), precise),\n        {astype(a, dtype, s)});\n  } else {\n    auto in = a;\n    if (precise) {\n      in = astype(a, float32, s);\n    }\n    auto a_max = stop_gradient(max(in, axes, /*keepdims = */ true, s), s);\n    auto ex = exp(subtract(in, a_max, s), s);\n    return astype(\n        divide(ex, sum(ex, axes, /*keepdims = */ true, s), s), a.dtype(), s);\n  }\n}\n\narray softmax(\n    const array& a,\n    bool precise /* = false */,\n    StreamOrDevice s /* = {}*/) {\n  std::vector<int> axes(a.ndim());\n  std::iota(axes.begin(), axes.end(), 0);\n  return softmax(a, axes, precise, s);\n}\n\narray power(const array& a, const array& b, StreamOrDevice s /* = {} */) {\n  auto dtype = promote_types(a.dtype(), b.dtype());\n  std::vector<array> inputs = {astype(a, dtype, s), astype(b, dtype, s)};\n  if (a.shape() != b.shape()) {\n    inputs = broadcast_arrays(inputs, s);\n  }\n  return array(\n      inputs[0].shape(), dtype, std::make_shared<Power>(to_stream(s)), inputs);\n}\n\narray cumsum(\n    const array& a,\n    int axis,\n    bool reverse /* = false*/,\n    bool inclusive /* = true*/,\n    StreamOrDevice s /* = {}*/) {\n  int ndim = a.ndim();\n  if (axis >= ndim || axis < -ndim) {\n    std::ostringstream msg;\n    msg << \"[cumsum] Axis \" << axis << \" is out of bounds for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  axis = (axis + a.ndim()) % a.ndim();\n  auto out_type = a.dtype() == bool_ ? int32 : a.dtype();\n  return array(\n      a.shape(),\n      out_type,\n      std::make_shared<Scan>(\n          to_stream(s), Scan::ReduceType::Sum, axis, reverse, inclusive),\n      {a});\n}\n\narray cumprod(\n    const array& a,\n    int axis,\n    bool reverse /* = false*/,\n    bool inclusive /* = true*/,\n    StreamOrDevice s /* = {}*/) {\n  int ndim = a.ndim();\n  if (axis >= ndim || axis < -ndim) {\n    std::ostringstream msg;\n    msg << \"[cumprod] Axis \" << axis << \" is out of bounds for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  axis = (axis + a.ndim()) % a.ndim();\n  return array(\n      a.shape(),\n      a.dtype(),\n      std::make_shared<Scan>(\n          to_stream(s), Scan::ReduceType::Prod, axis, reverse, inclusive),\n      {a});\n}\n\narray cummax(\n    const array& a,\n    int axis,\n    bool reverse /* = false*/,\n    bool inclusive /* = true*/,\n    StreamOrDevice s /* = {}*/) {\n  int ndim = a.ndim();\n  if (axis >= ndim || axis < -ndim) {\n    std::ostringstream msg;\n    msg << \"[cummax] Axis \" << axis << \" is out of bounds for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  axis = (axis + a.ndim()) % a.ndim();\n  return array(\n      a.shape(),\n      a.dtype(),\n      std::make_shared<Scan>(\n          to_stream(s), Scan::ReduceType::Max, axis, reverse, inclusive),\n      {a});\n}\n\narray cummin(\n    const array& a,\n    int axis,\n    bool reverse /* = false*/,\n    bool inclusive /* = true*/,\n    StreamOrDevice s /* = {}*/) {\n  int ndim = a.ndim();\n  if (axis >= ndim || axis < -ndim) {\n    std::ostringstream msg;\n    msg << \"[cummin] Axis \" << axis << \" is out of bounds for array with \"\n        << a.ndim() << \" dimensions.\";\n    throw std::invalid_argument(msg.str());\n  }\n  axis = (axis + a.ndim()) % a.ndim();\n  return array(\n      a.shape(),\n      a.dtype(),\n      std::make_shared<Scan>(\n          to_stream(s), Scan::ReduceType::Min, axis, reverse, inclusive),\n      {a});\n}\n\n/** Convolution operations */\n\nnamespace {\n\n// Conv helpers\ninline int conv_out_axis_size(int in_dim, int wt_dim, int stride, int padding) {\n  return ((in_dim + padding - wt_dim) / stride) + 1;\n}\n\n// Conv helpers\ninline int dilate_size(int dim, int dil) {\n  return 1 + dil * (dim - 1);\n}\n\ninline std::vector<int> conv_out_shape(\n    const std::vector<int>& in_shape,\n    const std::vector<int>& wt_shape,\n    const std::vector<int>& strides,\n    const std::vector<int>& pads_lo,\n    const std::vector<int>& pads_hi,\n    const std::vector<int>& kernel_dilation,\n    const std::vector<int>& input_dilation) {\n  int N = in_shape[0];\n  int O = wt_shape[0];\n  std::vector<int> out_shape(in_shape.size());\n  int i = 0;\n  out_shape[i++] = N;\n\n  int spatial_dims = in_shape.size() - 2;\n\n  if (strides.size() != spatial_dims) {\n    std::ostringstream msg;\n    msg << \"[conv] Invalid strides \" << strides << \"for \" << spatial_dims\n        << \"D convolution.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  if (pads_lo.size() != spatial_dims || pads_hi.size() != spatial_dims) {\n    std::ostringstream msg;\n    msg << \"[conv] Invalid pading \" << pads_lo << \" | \" << pads_hi << \"for \"\n        << spatial_dims << \"D convolution.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  if (kernel_dilation.size() != spatial_dims) {\n    std::ostringstream msg;\n    msg << \"[conv] Invalid kernel dilation \" << kernel_dilation << \"for \"\n        << spatial_dims << \"D convolution.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  if (input_dilation.size() != spatial_dims) {\n    std::ostringstream msg;\n    msg << \"[conv] Invalid input dilation \" << input_dilation << \"for \"\n        << spatial_dims << \"D convolution.\";\n    throw std::invalid_argument(msg.str());\n  }\n\n  for (; i < in_shape.size() - 1; i++) {\n    if (kernel_dilation[i - 1] <= 0) {\n      std::ostringstream msg;\n      msg << \"[conv] Kernel dilation sizes must be positive.\"\n          << \" Got kernel dilation \" << kernel_dilation << \".\";\n      throw std::invalid_argument(msg.str());\n    }\n\n    if (input_dilation[i - 1] <= 0) {\n      std::ostringstream msg;\n      msg << \"[conv] Input dilation sizes must be positive.\"\n          << \" Got input dilation \" << input_dilation << \".\";\n      throw std::invalid_argument(msg.str());\n    }\n\n    if (pads_lo[i - 1] < 0 || pads_hi[i - 1] < 0) {\n      std::ostringstream msg;\n      msg << \"[conv] Padding sizes must be non-negative.\" << \" Got padding \"\n          << pads_lo << \" | \" << pads_hi << \".\";\n      throw std::invalid_argument(msg.str());\n    }\n\n...\n", "needle_token_start": 13741, "needle_token_end": 13827, "code_context_ntokens": 16222, "output": ["```cpp\narray sqrt(const array& a, StreamOrDevice s /* = {} */) {\n  auto dtype = at_least_float(a.dtype());\n  return array(\n      a.shape(),\n      dtype,\n      std::make_shared<Sqrt>(to_stream(s)),\n      {astype(a, dtype, s)});\n}\n```"]}
{"repo": "ml-explore/mlx", "name": "promote_types", "language": "cpp", "path": "mlx/dtype.cpp", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine the most appropriate data type for operations involving two different data types, ensuring that no precision is lost.\n2. **Input**: Two data type identifiers representing the types of two operands.\n3. **Output**: A single data type identifier that represents the promoted data type suitable for handling operations between the two input types without loss of information.\n4. **Procedure**: The function uses a predefined matrix that maps pairs of input data types to a resulting data type. It retrieves the appropriate data type from this matrix using the indices derived from the input data types.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: mlx/primitives.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n#include <algorithm>\n#include <cassert>\n#include <cmath>\n#include <numeric>\n#include <sstream>\n#include <stdexcept>\n\n#include \"mlx/backend/common/utils.h\"\n#include \"mlx/fft.h\"\n#include \"mlx/ops.h\"\n#include \"mlx/primitives.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\nstd::tuple<array, array, int> vmap_binary_op(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes,\n    const Stream& stream) {\n  assert(inputs.size() == 2);\n  assert(axes.size() == 2);\n\n  if (axes[0] == -1 && axes[1] == -1) {\n    return {inputs[0], inputs[1], -1};\n  }\n\n  auto a = inputs[0];\n  auto b = inputs[1];\n  int ndim = std::max(a.ndim() + (axes[0] == -1), b.ndim() + (axes[1] == -1));\n\n  auto expand_dims = [stream, ndim](auto in) {\n    auto shape = in.shape();\n    shape.insert(shape.begin(), ndim - shape.size(), 1);\n    return reshape(in, shape, stream);\n  };\n\n  int to_ax = (ndim - a.ndim()) + axes[0];\n  int from_ax = (ndim - b.ndim()) + axes[1];\n  a = expand_dims(a);\n  b = expand_dims(b);\n\n  if (from_ax != to_ax) {\n    std::vector<int> tdims(b.ndim());\n    std::iota(tdims.begin(), tdims.end(), 0);\n    tdims.erase(tdims.begin() + from_ax);\n    tdims.insert(tdims.begin() + to_ax, from_ax);\n    b = transpose(b, tdims, stream);\n  }\n  return {a, b, to_ax};\n}\n\nstd::tuple<array, array, array, int> vmap_ternary_op(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes,\n    const Stream& stream) {\n  assert(inputs.size() == 3);\n  assert(axes.size() == 3);\n\n  if (axes[0] == -1 && axes[1] == -1 && axes[2] == -1) {\n    return {inputs[0], inputs[1], inputs[2], -1};\n  }\n\n  auto a = inputs[0];\n  auto b = inputs[1];\n  auto c = inputs[2];\n  int ndim = std::max(\n      {a.ndim() + (axes[0] == -1),\n       b.ndim() + (axes[1] == -1),\n       c.ndim() + (axes[2] == -1)});\n\n  auto expand_dims = [stream, ndim](auto in) {\n    auto shape = in.shape();\n    shape.insert(shape.begin(), ndim - shape.size(), 1);\n    return reshape(in, shape, stream);\n  };\n\n  int to_ax = (ndim - a.ndim()) + axes[0];\n  int from_ax1 = (ndim - b.ndim()) + axes[1];\n  int from_ax2 = (ndim - c.ndim()) + axes[2];\n  a = expand_dims(a);\n  b = expand_dims(b);\n  c = expand_dims(c);\n\n  auto find_tdims = [](auto x, int to_ax, int from_ax) {\n    std::vector<int> tdims(x.ndim());\n    std::iota(tdims.begin(), tdims.end(), 0);\n    tdims.erase(tdims.begin() + from_ax);\n    tdims.insert(tdims.begin() + to_ax, from_ax);\n    return tdims;\n  };\n\n  if (to_ax != from_ax1) {\n    std::vector<int> tdims = find_tdims(b, to_ax, from_ax1);\n    b = transpose(b, tdims, stream);\n  }\n\n  if (to_ax != from_ax2) {\n    std::vector<int> tdims = find_tdims(c, to_ax, from_ax2);\n    c = transpose(c, tdims, stream);\n  }\n  return {a, b, c, to_ax};\n}\n\n} // namespace\n\nstd::vector<array> Primitive::jvp(\n    const std::vector<array>&,\n    const std::vector<array>&,\n    const std::vector<int>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::jvp] Not implemented for \";\n  print(msg);\n  msg << \".\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::vector<array> Primitive::vjp(\n    const std::vector<array>&,\n    const std::vector<array>&,\n    const std::vector<int>&,\n    const std::vector<array>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::vip] Not implemented for \";\n  print(msg);\n  msg << \".\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::pair<std::vector<array>, std::vector<int>> Primitive::vmap(\n    const std::vector<array>&,\n    const std::vector<int>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::vmap] Not implemented for \";\n  print(msg);\n  msg << \".\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::vector<std::vector<int>> Primitive::output_shapes(\n    const std::vector<array>&) {\n  std::ostringstream msg;\n  msg << \"[Primitive::output_shapes] \";\n  this->print(msg);\n  msg << \" cannot infer output shapes.\";\n  throw std::invalid_argument(msg.str());\n};\n\nstd::vector<array> Abs::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Abs::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(tangents[0], sign(primals[0], stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Abs::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{abs(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Add::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  return {\n      tangents.size() > 1 ? add(tangents[0], tangents[1], stream())\n                          : tangents[0]};\n}\n\nstd::vector<array> Add::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  if (argnums.size() == 1) {\n    return cotangents;\n  } else {\n    return {cotangents[0], cotangents[0]};\n  }\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Add::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{add(a, b, stream())}, {to_ax}};\n}\n\nstd::vector<array> AddMM::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  auto& cotan = cotangents[0];\n  std::vector<int> reorder(cotan.ndim());\n  std::iota(reorder.begin(), reorder.end(), 0);\n  std::iter_swap(reorder.end() - 1, reorder.end() - 2);\n  for (auto arg : argnums) {\n    if (arg == 0) {\n      // M X N * (K X N).T -> M X K\n      auto cotan_scaled = cotan;\n      if (alpha_ != 1.) {\n        auto alpha_arr = array(alpha_, cotan.dtype());\n        cotan_scaled = (multiply(alpha_arr, cotan_scaled, stream()));\n      }\n      vjps.push_back(matmul(\n          cotan_scaled, transpose(primals[1], reorder, stream()), stream()));\n    } else if (arg == 1) {\n      // (M X K).T * M X N -> K X N\n      auto cotan_scaled = cotan;\n      if (alpha_ != 1.) {\n        auto alpha_arr = array(alpha_, cotan.dtype());\n        cotan_scaled = (multiply(alpha_arr, cotan_scaled, stream()));\n      }\n      vjps.push_back(matmul(\n          transpose(primals[0], reorder, stream()), cotan_scaled, stream()));\n    } else {\n      auto cotan_scaled = cotan;\n      if (beta_ != 1.) {\n        auto beta_arr = array(beta_, cotan.dtype());\n        cotan_scaled = (multiply(beta_arr, cotan_scaled, stream()));\n      }\n      vjps.push_back(cotan_scaled);\n    }\n  }\n  return vjps;\n}\n\nbool AddMM::is_equivalent(const Primitive& other) const {\n  const AddMM& a_other = static_cast<const AddMM&>(other);\n  return (alpha_ == a_other.alpha_ && beta_ == a_other.beta_);\n}\n\nstd::pair<std::vector<array>, std::vector<int>> AddMM::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto maybe_move_ax = [this](auto& arr, auto ax) {\n    return ax > 0 ? moveaxis(arr, ax, 0, stream()) : arr;\n  };\n  auto a = maybe_move_ax(inputs[0], axes[0]);\n  auto b = maybe_move_ax(inputs[1], axes[1]);\n  auto c = maybe_move_ax(inputs[2], axes[2]);\n  return {{addmm(c, a, b, alpha_, beta_, stream())}, {0}};\n}\n\nbool Arange::is_equivalent(const Primitive& other) const {\n  const Arange& a_other = static_cast<const Arange&>(other);\n  return (\n      start_ == a_other.start_ && stop_ == a_other.stop_ &&\n      step_ == a_other.step_);\n}\n\nstd::vector<array> ArcCos::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcCos::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(one, square(primals[0], stream()), stream());\n  array denom = negative(rsqrt(t, stream()), stream());\n  return {multiply(tangents[0], denom, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcCos::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arccos(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcCosh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcCosh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(square(primals[0], stream()), one, stream());\n  return {multiply(tangents[0], rsqrt(t, stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcCosh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arccosh(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcSin::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcSin::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(one, square(primals[0], stream()), stream());\n  return {multiply(tangents[0], rsqrt(t, stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcSin::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arcsin(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcSinh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcSinh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = add(square(primals[0], stream()), one, stream());\n  return {multiply(tangents[0], rsqrt(t, stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcSinh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arcsinh(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcTan::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcTan::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = add(one, square(primals[0], stream()), stream());\n  return {divide(tangents[0], t, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcTan::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arctan(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> ArcTanh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> ArcTanh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  array one = array(1., primals[0].dtype());\n  array t = subtract(one, square(primals[0], stream()), stream());\n  return {divide(tangents[0], t, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArcTanh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{arctanh(inputs[0], stream())}, axes};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArgPartition::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n\n  int axis_left = axes[0] >= 0 && axes[0] <= axis_;\n  return {{argpartition(inputs[0], axis_ + axis_left, stream())}, axes};\n}\n\nbool ArgPartition::is_equivalent(const Primitive& other) const {\n  const ArgPartition& r_other = static_cast<const ArgPartition&>(other);\n  return axis_ == r_other.axis_ && kth_ == r_other.kth_;\n}\n\nbool ArgReduce::is_equivalent(const Primitive& other) const {\n  const ArgReduce& r_other = static_cast<const ArgReduce&>(other);\n  return reduce_type_ == r_other.reduce_type_ && axis_ == r_other.axis_;\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArgReduce::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  int reduce_ax = axis_ + (axes[0] >= 0 && axis_ >= axes[0]);\n  auto& in = inputs[0];\n  std::vector<array> out;\n  if (reduce_type_ == ArgReduce::ArgMin) {\n    out.push_back(argmin(in, reduce_ax, true, stream()));\n  } else {\n    out.push_back(argmax(in, reduce_ax, true, stream()));\n  }\n  return {out, axes};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> ArgSort::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n\n  int axis_left = axes[0] >= 0 && axes[0] <= axis_;\n  return {{argsort(inputs[0], axis_ + axis_left, stream())}, axes};\n}\n\nstd::vector<std::vector<int>> ArgReduce::output_shapes(\n    const std::vector<array>& inputs) {\n  auto out_shape = inputs[0].shape();\n  out_shape[axis_] = 1;\n  return {out_shape};\n}\n\nbool ArgSort::is_equivalent(const Primitive& other) const {\n  const ArgSort& r_other = static_cast<const ArgSort&>(other);\n  return axis_ == r_other.axis_;\n}\n\nstd::vector<array> AsType::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  if (cotangents[0].dtype() != dtype_) {\n    throw std::invalid_argument(\n        \"[astype] Type of cotangentsgent does not much primal output type.\");\n  }\n  return {astype(cotangents[0], primals[0].dtype(), stream())};\n}\n\nstd::vector<array> AsType::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  return {astype(tangents[0], dtype_, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> AsType::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  return {{astype(inputs[0], dtype_, stream())}, axes};\n}\n\nbool AsType::is_equivalent(const Primitive& other) const {\n  const AsType& a_other = static_cast<const AsType&>(other);\n  return dtype_ == a_other.dtype_;\n}\n\nstd::vector<array> AsStrided::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(argnums.size() == 1);\n\n  // Extract the sizes and cast them to ints\n  int grad_size = primals[0].size();\n  int cotangents_size = cotangents[0].size();\n\n  // Make a flat container to hold the gradients\n  auto grad = zeros_like(primals[0], stream());\n  grad = reshape(grad, {grad_size}, stream());\n\n  // Create the indices that map output to input\n  auto idx = arange(grad_size, stream());\n  idx = as_strided(idx, shape_, strides_, offset_, stream());\n  idx = reshape(idx, {cotangents_size}, stream());\n\n  // Reshape the cotangentsgent for use with scatter\n  auto flat_cotangents = reshape(cotangents[0], {cotangents_size, 1}, stream());\n\n  // Finally accumulate the gradients and reshape them to look like the input\n  grad = scatter_add(grad, idx, flat_cotangents, 0, stream());\n  grad = reshape(grad, primals[0].shape(), stream());\n\n  return {grad};\n}\n\nstd::vector<array> AsStrided::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n\n  return {as_strided(tangents[0], shape_, strides_, offset_, stream())};\n}\n\nbool AsStrided::is_equivalent(const Primitive& other) const {\n  const AsStrided& a_other = static_cast<const AsStrided&>(other);\n  return shape_ == a_other.shape_ && strides_ == a_other.strides_ &&\n      offset_ == a_other.offset_;\n}\n\nstd::vector<array> Broadcast::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(argnums.size() == 1);\n\n  // Reduce cotangents to the shape of the primal\n  auto& shape = primals[0].shape();\n  auto& cotan = cotangents[0];\n  int diff = cotan.ndim() - shape.size();\n  std::vector<int> reduce_axes;\n  for (int i = 0; i < cotan.ndim(); ++i) {\n    if (i < diff) {\n      reduce_axes.push_back(i);\n    } else if (shape[i - diff] != cotan.shape(i)) {\n      reduce_axes.push_back(i);\n    }\n  }\n  return {reshape(sum(cotan, reduce_axes, true, stream()), shape, stream())};\n}\n\nstd::vector<array> Broadcast::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(argnums.size() == 1);\n  return {broadcast_to(tangents[0], shape_, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Broadcast::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  auto ax = axes[0];\n  auto in = inputs[0];\n  if (ax >= 0) {\n    auto in_shape = in.shape();\n    int diff = shape_.size() - in.ndim() + 1;\n    assert(diff >= 0);\n    in_shape.insert(in_shape.begin(), diff, 1);\n    ax += diff;\n    shape_.insert(shape_.begin() + ax, in_shape[ax]);\n    in = reshape(in, in_shape, stream());\n  }\n  return {{broadcast_to(in, shape_, stream())}, {ax}};\n}\n\nbool Broadcast::is_equivalent(const Primitive& other) const {\n  const Broadcast& b_other = static_cast<const Broadcast&>(other);\n  return shape_ == b_other.shape_;\n}\n\nstd::vector<array> Ceil::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Ceil::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {zeros_like(primals[0], stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Ceil::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{ceil(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Concatenate::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  auto& cotan = cotangents[0];\n  std::vector<int> start(cotan.ndim(), 0);\n  std::vector<int> stop = cotan.shape();\n\n  std::vector<int> sizes;\n  sizes.push_back(0);\n  for (auto& p : primals) {\n    sizes.push_back(p.shape(axis_));\n  }\n  std::partial_sum(sizes.cbegin(), sizes.cend(), sizes.begin());\n\n  std::vector<array> grads;\n  for (auto i : argnums) {\n    start[axis_] = sizes[i];\n    stop[axis_] = sizes[i + 1];\n    grads.push_back(slice(cotan, start, stop, stream()));\n  }\n  return grads;\n}\n\nstd::vector<array> Concatenate::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  std::vector<int> argidx(argnums.size());\n  std::iota(argidx.begin(), argidx.end(), 0);\n  std::sort(argidx.begin(), argidx.end(), [&argnums](int a, int b) {\n    return argnums[a] < argnums[b];\n  });\n\n  std::vector<array> vals;\n  for (int i = 0, j = 0; i < primals.size(); ++i) {\n    if (j < argnums.size() && argnums[argidx[j]] == i) {\n      vals.push_back(tangents[argidx[j++]]);\n    } else {\n      vals.push_back(zeros_like(primals[i], stream()));\n    }\n  }\n  return {concatenate(vals, axis_, stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Concatenate::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  std::vector<array> t_inputs;\n  int out_ax = -1;\n  // Find the first vmapped input\n  int i = 0;\n  for (; i < axes.size(); i++) {\n    t_inputs.push_back(inputs[i]);\n    if (axes[i] >= 0) {\n      out_ax = axes[i];\n      break;\n    }\n  }\n  if (out_ax >= 0) {\n    // Advance to the next input\n    i++;\n\n    // Move vmap axes to the same spot.\n    for (; i < axes.size(); ++i) {\n      if (out_ax != axes[i] && axes[i] >= 0) {\n        t_inputs.push_back(moveaxis(inputs[i], axes[i], out_ax, stream()));\n      } else {\n        t_inputs.push_back(inputs[i]);\n      }\n    }\n  }\n  auto axis = axis_ + (out_ax >= 0 && axis_ >= out_ax);\n  return {{concatenate(t_inputs, axis, stream())}, {out_ax}};\n}\n\nbool Concatenate::is_equivalent(const Primitive& other) const {\n  const Concatenate& c_other = static_cast<const Concatenate&>(other);\n  return axis_ == c_other.axis_;\n}\n\narray conv_weight_backward_patches(\n    const array& in,\n    const array& wt,\n    const array& cotan,\n    const std::vector<int>& kernel_strides,\n    const std::vector<int>& padding,\n    StreamOrDevice s) {\n  // Resolve Padded input shapes and strides\n  std::vector<int> padding_starts(in.ndim(), 0);\n  std::vector<int> padding_ends = in.shape();\n  std::vector<int> in_padded_shape = in.shape();\n\n  // padded shape\n  for (int i = 1; i < in.ndim() - 1; i++) {\n    in_padded_shape[i] += 2 * padding[i - 1];\n    padding_ends[i] += padding[i - 1];\n    padding_starts[i] += padding[i - 1];\n  }\n\n  // padded strides (contiguous)\n  std::vector<size_t> in_padded_strides(in.ndim(), 1);\n  for (int i = in.ndim() - 2; i >= 0; --i) {\n    in_padded_strides[i] = in_padded_strides[i + 1] * in_padded_shape[i + 1];\n  }\n\n  // Pad input\n  std::vector<int> padded_axes(in.ndim() - 2, 0);\n  std::iota(padded_axes.begin(), padded_axes.end(), 1);\n  auto in_padded =\n      pad(in, padded_axes, padding, padding, array(0, in.dtype()), s);\n\n  // Resolve strided patches\n\n  // patches are shaped as\n  // (batch_dim, out_spatial_dims, weight_spatial_dims, in_channels)\n  std::vector<int> patches_shape{\n      cotan.shape().begin(), cotan.shape().end() - 1};\n  patches_shape.insert(\n      patches_shape.end(), wt.shape().begin() + 1, wt.shape().end());\n\n  // Resolve patch strides\n  int n_spatial_dim = in.ndim() - 2;\n  std::vector<size_t> patches_strides(patches_shape.size(), 1);\n  patches_strides[0] = in_padded_strides[0];\n  for (int i = 1; i < n_spatial_dim + 1; i++) {\n    patches_strides[i] = in_padded_strides[i] * kernel_strides[i - 1];\n  }\n  for (int i = 1; i < in.ndim(); i++) {\n    patches_strides[n_spatial_dim + i] = in_padded_strides[i];\n  }\n\n  // Make patches from in\n  auto in_patches = as_strided(in_padded, patches_shape, patches_strides, 0, s);\n\n  // Prepare for matmul\n  int O = wt.shape(0);\n  auto cotan_mat = reshape(cotan, {-1, O}, s);\n  in_patches = reshape(in_patches, {cotan_mat.shape(0), -1}, s);\n\n  auto grad = matmul(transpose(cotan_mat, {1, 0}, s), in_patches, s);\n  grad = reshape(grad, wt.shape(), s);\n  return grad;\n}\n\nstd::vector<array> Convolution::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(primals.size() == 2);\n  std::vector<array> grads;\n\n  // Collect info\n  auto& in = primals[0];\n  auto& wt = primals[1];\n  auto& cotan = cotangents[0];\n\n  for (int a : argnums) {\n    // Grads for input\n    if (a == 0) {\n      std::vector<int> padding_lo = padding_;\n      std::vector<int> padding_hi = padding_;\n\n      for (int i = 0; i < padding_lo.size(); ++i) {\n        int wt_size = 1 + kernel_dilation_[i] * (wt.shape(1 + i) - 1);\n        padding_lo[i] = wt_size - padding_[i] - 1;\n\n        int in_size = 1 + input_dilation_[i] * (in.shape(1 + i) - 1);\n        int out_size = 1 + kernel_strides_[i] * (cotan.shape(1 + i) - 1);\n        padding_hi[i] = in_size - out_size + padding_[i];\n      }\n\n      auto wt_trans = swapaxes(wt, 0, -1, stream());\n\n      auto grad = conv_general(\n          /* const array& input = */ cotan,\n          /* const array& weight = */ wt_trans,\n          /* std::vector<int> stride = */ input_dilation_,\n          /* std::vector<int> padding_lo = */ padding_lo,\n          /* std::vector<int> padding_hi = */ padding_hi,\n          /* std::vector<int> kernel_dilation = */ kernel_dilation_,\n          /* std::vector<int> input_dilation = */ kernel_strides_,\n          /* int groups = */ 1,\n          /* bool flip = */ !flip_,\n          stream());\n\n      grads.push_back(grad);\n    }\n    // Grads for weight\n    else if (a == 1) {\n      bool no_dilation = true;\n\n      for (int i = 0; i < input_dilation_.size(); i++) {\n        no_dilation &= (input_dilation_[i] == 1) && (kernel_dilation_[i] == 1);\n      }\n\n      if (no_dilation) {\n        auto grad = conv_weight_backward_patches(\n            in, wt, cotan, kernel_strides_, padding_, stream());\n        grads.push_back(grad);\n      } else {\n        std::vector<int> padding_lo = padding_;\n        std::vector<int> padding_hi = padding_;\n\n        for (int i = 0; i < padding_hi.size(); ++i) {\n          int in_size = 1 + input_dilation_[i] * (in.shape(1 + i) - 1);\n          int out_size = 1 + kernel_strides_[i] * (cotan.shape(1 + i) - 1);\n          int wt_size = 1 + kernel_dilation_[i] * (wt.shape(1 + i) - 1);\n          padding_hi[i] = out_size - in_size + wt_size - padding_[i] - 1;\n        }\n\n        auto in_trans = swapaxes(in, 0, -1, stream());\n        auto cotan_trans = swapaxes(cotan, 0, -1, stream());\n        auto grad_trans = conv_general(\n            /* const array& input = */ in_trans,\n            /* const array& weight = */ cotan_trans,\n            /* std::vector<int> stride = */ kernel_dilation_,\n            /* std::vector<int> padding_lo = */ padding_lo,\n            /* std::vector<int> padding_hi = */ padding_hi,\n            /* std::vector<int> kernel_dilation = */ kernel_strides_,\n            /* std::vector<int> input_dilation = */ input_dilation_,\n            /* int groups = */ 1,\n            /* bool flip = */ flip_,\n            stream());\n        auto grad = swapaxes(grad_trans, 0, -1, stream());\n        grads.push_back(grad);\n      }\n    }\n  }\n\n  return grads;\n}\n\nbool Convolution::is_equivalent(const Primitive& other) const {\n  const Convolution& c_other = static_cast<const Convolution&>(other);\n  return padding_ == c_other.padding_ &&\n      kernel_strides_ == c_other.kernel_strides_ &&\n      kernel_dilation_ == c_other.kernel_dilation_ &&\n      input_dilation_ == c_other.input_dilation_ &&\n      groups_ == c_other.groups_ && flip_ == c_other.flip_;\n}\n\nstd::vector<array> Copy::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return cotangents;\n}\n\nstd::vector<array> Copy::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return tangents;\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Copy::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{copy(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Cos::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return {jvp(primals, cotangents, argnums)};\n}\n\nstd::vector<array> Cos::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(\n      tangents[0], negative(sin(primals[0], stream()), stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Cos::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{cos(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> Cosh::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Cosh::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  assert(primals.size() == 1);\n  assert(argnums.size() == 1);\n  return {multiply(tangents[0], sinh(primals[0], stream()), stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Cosh::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  assert(inputs.size() == 1);\n  assert(axes.size() == 1);\n  return {{cosh(inputs[0], stream())}, axes};\n}\n\nstd::vector<array> CustomVJP::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  std::vector<array> inputs(primals.begin(), primals.end() - outputs.size());\n  auto all_vjps = vjp_fun_(inputs, cotangents, outputs);\n  for (const auto& cot : cotangents) {\n    all_vjps.emplace_back(cot);\n  }\n\n  std::vector<array> vjps;\n  vjps.reserve(argnums.size());\n  for (auto arg : argnums) {\n    vjps.push_back(all_vjps[arg]);\n  }\n\n  return vjps;\n}\n\nstd::vector<array> Depends::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>& outputs) {\n  std::vector<array> vjps;\n\n  for (auto arg : argnums) {\n    if (arg < cotangents.size()) {\n      vjps.push_back(cotangents[arg]);\n    } else {\n      vjps.push_back(zeros_like(primals[arg]));\n    }\n  }\n  return vjps;\n}\n\nstd::vector<array> Divide::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    if (arg == 0) {\n      vjps.push_back(divide(cotangents[0], primals[1], stream()));\n    } else {\n      vjps.push_back(negative(\n          divide(\n              multiply(cotangents[0], primals[0], stream()),\n              square(primals[1], stream()),\n              stream()),\n          stream()));\n    }\n  }\n  return vjps;\n}\n\nstd::vector<array> DivMod::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    vjps.push_back(zeros_like(primals[arg], stream()));\n  }\n  return vjps;\n}\n\nstd::vector<array> DivMod::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  return {zeros_like(primals[0], stream())};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> DivMod::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {divmod(a, b, stream()), {to_ax}};\n}\n\nstd::vector<array> Divide::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  auto jvp_fun = [&](int i) {\n    int arg = argnums[i];\n    if (arg == 0) {\n      return divide(tangents[i], primals[1], stream());\n    } else {\n      return negative(\n          divide(\n              multiply(tangents[i], primals[0], stream()),\n              square(primals[1], stream()),\n              stream()),\n          stream());\n    }\n  };\n  auto out = jvp_fun(0);\n  if (argnums.size() > 1) {\n    out = add(out, jvp_fun(1), stream());\n  }\n  return {out};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Divide::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{divide(a, b, stream())}, {to_ax}};\n}\n\nstd::vector<array> Remainder::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    if (arg == 0) {\n      vjps.push_back(cotangents[0]);\n    } else {\n      auto x_over_y = divide(primals[0], primals[1], stream());\n      x_over_y = floor(x_over_y, stream());\n      vjps.push_back(\n          negative(multiply(x_over_y, cotangents[0], stream()), stream()));\n    }\n  }\n  return vjps;\n}\n\nstd::vector<array> Remainder::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  auto jvp_fun = [&](int i) {\n    int arg = argnums[i];\n    if (arg == 0) {\n      return tangents[i];\n    } else {\n      auto x_over_y = divide(primals[0], primals[1], stream());\n      x_over_y = floor(x_over_y, stream());\n      return negative(multiply(x_over_y, tangents[i], stream()), stream());\n    }\n  };\n  auto out = jvp_fun(0);\n  if (argnums.size() > 1) {\n    out = add(out, jvp_fun(1), stream());\n  }\n  return {out};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Remainder::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{remainder(a, b, stream())}, {to_ax}};\n}\n\nstd::pair<std::vector<array>, std::vector<int>> Equal::vmap(\n    const std::vector<array>& inputs,\n    const std::vector<int>& axes) {\n  auto [a, b, to_ax] = vmap_binary_op(inputs, axes, stream());\n  return {{equal(a, b, stream())}, {to_ax}};\n}\n\nstd::vector<array> Equal::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  std::vector<array> vjps;\n  for (auto arg : argnums) {\n    vjps.push_back(zeros_like(primals[arg], stream()));\n  }\n  return vjps;\n}\n\nstd::vector<array> Equal::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n    const std::vector<int>& argnums) {\n  auto shape = broadcast_shapes(primals[0].shape(), primals[1].shape());\n  return {zeros(shape, bool_, stream())};\n}\n\nstd::vector<array> Erf::vjp(\n    const std::vector<array>& primals,\n    const std::vector<array>& cotangents,\n    const std::vector<int>& argnums,\n    const std::vector<array>&) {\n  return jvp(primals, cotangents, argnums);\n}\n\nstd::vector<array> Erf::jvp(\n    const std::vector<array>& primals,\n    const std::vector<array>& tangents,\n...\n// Path: mlx/dtype.cpp\n// Copyright \u00a9 2023-2024 Apple Inc.\n\n#include <cstdint>\n#include <sstream>\n#include <vector>\n\n#include \"mlx/dtype.h\"\n#include \"mlx/utils.h\"\n\nnamespace mlx::core {\n\nnamespace {\n\nconstexpr int num_types = 13;\nconstexpr int num_cats = 8;\n\nconstexpr Dtype::Kind type_kinds[num_types] = {\n    Dtype::Kind::b, // bool_,\n    Dtype::Kind::u, // uint8,\n    Dtype::Kind::u, // uint16,\n    Dtype::Kind::u, // uint32,\n    Dtype::Kind::u, // uint64,\n    Dtype::Kind::i, // int8,\n    Dtype::Kind::i, // int16,\n    Dtype::Kind::i, // int32,\n    Dtype::Kind::i, // int64,\n    Dtype::Kind::f, // float16,\n    Dtype::Kind::f, // float32,\n    Dtype::Kind::V, // bfloat16,\n    Dtype::Kind::c // complex64,\n};\n\n// Following Jax type promotion rules:\n// https://jax.readthedocs.io/en/latest/type_promotion.html\n// clang-format off\nconstexpr Dtype type_rules[num_types][num_types] = {\n// bool       uint8      uint16     uint32     uint64     int8       int16      int32      int64      float16    float32    bfloat16   complex64\n  {bool_,     uint8,     uint16,    uint32,    uint64,    int8,      int16,     int32,     int64,     float16,   float32,   bfloat16,  complex64}, // bool\n  {uint8,     uint8,     uint16,    uint32,    uint64,    int16,     int16,     int32,     int64,     float16,   float32,   bfloat16,  complex64}, // uint8\n  {uint16,    uint16,    uint16,    uint32,    uint64,    int32,     int32,     int32,     int64,     float16,   float32,   bfloat16,  complex64}, // uint16\n  {uint32,    uint32,    uint32,    uint32,    uint64,    int64,     int64,     int64,     int64,     float16,   float32,   bfloat16,  complex64}, // uint32\n  {uint64,    uint64,    uint64,    uint64,    uint64,    float32,   float32,   float32,   float32,   float16,   float32,   bfloat16,  complex64}, // uint64\n  {int8,      int16,     int32,     int64,     float32,   int8,      int16,     int32,     int64,     float16,   float32,   bfloat16,  complex64}, // int8\n  {int16,     int16,     int32,     int64,     float32,   int16,     int16,     int32,     int64,     float16,   float32,   bfloat16,  complex64}, // int16\n  {int32,     int32,     int32,     int64,     float32,   int32,     int32,     int32,     int64,     float16,   float32,   bfloat16,  complex64}, // int32\n  {int64,     int64,     int64,     int64,     float32,   int64,     int64,     int64,     int64,     float16,   float32,   bfloat16,  complex64}, // int64\n  {float16,   float16,   float16,   float16,   float16,   float16,   float16,   float16,   float16,   float16,   float32,   float32,   complex64}, // float16\n  {float32,   float32,   float32,   float32,   float32,   float32,   float32,   float32,   float32,   float32,   float32,   float32,   complex64}, // float32\n  {bfloat16,  bfloat16,  bfloat16,  bfloat16,  bfloat16,  bfloat16,  bfloat16,  bfloat16,  bfloat16,  float32,   float32,   bfloat16,  complex64}, // bfloat16\n  {complex64, complex64, complex64, complex64, complex64, complex64, complex64, complex64, complex64, complex64, complex64, complex64, complex64}, // complex64\n};\n\n\nconstexpr bool subcategory_to_category[num_cats][num_cats] = {\n// complexfloating floating inexact signedinteger unsignedinteger integer number generic\n  {true,           false,   true,   false,        false,          false,  true,  true}, // complexfloating\n  {false,          true,    true,   false,        false,          false,  true,  true}, // floating\n  {false,          false,   true,   false,        false,          false,  true,  true}, // inexact\n  {false,          false,   false,  true,         false,          true,   true,  true}, // signedinteger\n  {false,          false,   false,  false,        true,           true,   true,  true}, // unsignedinteger\n  {false,          false,   false,  false,        false,          true,   true,  true}, // integer\n  {false,          false,   false,  false,        false,          false,  true,  true}, // number\n  {false,          false,   false,  false,        false,          false,  false, true}, // generic\n};\n\nconstexpr Dtype::Category type_to_category[num_types] = {\n    Dtype::Category::generic, // bool_,\n    Dtype::Category::unsignedinteger, // uint8,\n    Dtype::Category::unsignedinteger, // uint16,\n    Dtype::Category::unsignedinteger, // uint32,\n    Dtype::Category::unsignedinteger, // uint64,\n    Dtype::Category::signedinteger, // int8,\n    Dtype::Category::signedinteger, // int16,\n    Dtype::Category::signedinteger, // int32,\n    Dtype::Category::signedinteger, // int64,\n    Dtype::Category::floating, // float16,\n    Dtype::Category::floating, // float32,\n    Dtype::Category::floating, // bfloat16,\n    Dtype::Category::complexfloating, // complex64,\n};\n\n// clang-format on\n\n} // namespace\n\nD\ntype promote_types(const Dtype& t1, const Dtype& t2) {\n  return Dtype(type_rules[static_cast<int>(t1.val)][static_cast<int>(t2.val)]);\n}\n\nDtype::Kind kindof(const Dtype& t) {\n  return type_kinds[static_cast<int>(t.val)];\n}\n\ntemplate <>\nTypeToDtype<bool>::operator Dtype() {\n  return bool_;\n}\n\ntemplate <>\nTypeToDtype<uint8_t>::operator Dtype() {\n  return uint8;\n}\n\ntemplate <>\nTypeToDtype<uint16_t>::operator Dtype() {\n  return uint16;\n}\n\ntemplate <>\nTypeToDtype<uint32_t>::operator Dtype() {\n  return uint32;\n}\n\ntemplate <>\nTypeToDtype<uint64_t>::operator Dtype() {\n  return uint64;\n}\n\ntemplate <>\nTypeToDtype<int8_t>::operator Dtype() {\n  return int8;\n}\n\ntemplate <>\nTypeToDtype<int16_t>::operator Dtype() {\n  return int16;\n}\n\ntemplate <>\nTypeToDtype<int32_t>::operator Dtype() {\n  return int32;\n}\n\ntemplate <>\nTypeToDtype<int64_t>::operator Dtype() {\n  return int64;\n}\n\ntemplate <>\nTypeToDtype<float16_t>::operator Dtype() {\n  return float16;\n}\n\ntemplate <>\nTypeToDtype<float>::operator Dtype() {\n  return float32;\n}\n\ntemplate <>\nTypeToDtype<double>::operator Dtype() {\n  return float32;\n}\n\ntemplate <>\nTypeToDtype<bfloat16_t>::operator Dtype() {\n  return bfloat16;\n}\n\ntemplate <>\nTypeToDtype<complex64_t>::operator Dtype() {\n  return complex64;\n}\n\nbool issubdtype(const Dtype& a, const Dtype& b) {\n  return a == b;\n}\n\nbool issubdtype(const Dtype::Category& cat, const Dtype& type) {\n  return false;\n}\n\nbool issubdtype(const Dtype& type, const Dtype::Category& cat) {\n  return issubdtype(type_to_category[static_cast<uint32_t>(type.val)], cat);\n}\n\nbool issubdtype(const Dtype::Category& a, const Dtype::Category& b) {\n  return subcategory_to_category[static_cast<uint32_t>(a)]\n                                [static_cast<uint32_t>(b)];\n}\n\n// Array protocol typestring for Dtype\nstd::string dtype_to_array_protocol(const Dtype& t) {\n  std::ostringstream r;\n  if (size_of(t) > 1)\n    r << (is_big_endian() ? \">\" : \"<\");\n  else\n    r << \"|\";\n  r << kindof(t) << (int)size_of(t);\n  return r.str();\n}\n\n// Dtype from array protocol type string\nDtype dtype_from_array_protocol(std::string_view t) {\n...\n", "needle_token_start": 15381, "needle_token_end": 15438, "code_context_ntokens": 16200, "output": ["```cpp\nDtype promote_types(const Dtype& t1, const Dtype& t2) {\n  return Dtype(type_rules[static_cast<int>(t1.val)][static_cast<int>(t2.val)]);\n}\n```"]}
{"repo": "scylladb/seastar", "name": "send", "language": "cpp", "path": "src/net/dhcp.cc", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transmit a DHCP packet over a UDP socket to a predefined server port, with the packet containing specific DHCP fields set for communication.\n2. **Input**: The function accepts a DHCP packet which has been prepared for transmission.\n3. **Output**: The function returns a future object that resolves when the packet sending operation is complete, although the actual sending operation's result is not tracked due to the future being discarded.\n4. **Procedure**: \n   - The function first sets the transaction ID and client hardware address in the DHCP packet.\n   - It then converts the packet from host byte order to network byte order.\n   - The packet is sent to the broadcast address on the server port using a UDP socket.\n   - A future object is returned immediately, indicating the operation is complete, but without tracking the success of the send operation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            _state = state::DONE;\n            _result.set_value(info);\n            break;\n        default:\n            break;\n        }\n        return make_ready_future<>();\n    }\n\n    future<> handle(packet& p, ip_hdr* iph, ethernet_address from, bool & handled) override {\n        if (_state == state::NONE || p.len() < sizeof(dhcp_packet_base)) {\n            return make_ready_future<>();\n        }\n\n        auto ipl = iph->ihl * 4;\n        auto udp = p.get_header<udp_hdr>(ipl);\n        auto dhp = p.get_header<dhcp_payload>(ipl + sizeof(*udp));\n\n        const auto opt_off = ipl + sizeof(*udp) + sizeof(dhcp_payload);\n\n        if (udp == nullptr || dhp == nullptr\n                || iph->ip_proto != uint8_t(ip_protocol_num::udp)\n                || (::ntohs)(udp->dst_port) != client_port\n                || iph->len < (opt_off + sizeof(option_mark))\n                || dhp->magic != options_magic) {\n            return make_ready_future<>();\n        }\n        handled = true;\n        auto src_cpu = this_shard_id();\n        if (src_cpu == 0) {\n            return process_packet(std::move(p), dhp, opt_off);\n        }\n        // FIXME: future is discarded\n        (void)smp::submit_to(0, [this, p = std::move(p), src_cpu, dhp, opt_off]() mutable {\n            return process_packet(p.free_on_cpu(src_cpu), dhp, opt_off);\n        });\n        return make_ready_future<>();\n    }\n\n    future<std::optional<lease>> run(const lease & l,\n            const steady_clock_type::duration & timeout) {\n\n        _state = state::NONE;\n        _timer.set_callback([this]() {\n            _state = state::FAIL;\n            log() << \"timeout\" << std::endl;\n            _retry_timer.cancel();\n            _result.set_value(std::nullopt);\n        });\n\n        log() << \"sending discover\" << std::endl;\n        (void)send_discover(l.ip); // FIXME: ignoring return\n        if (timeout.count()) {\n            _timer.arm(timeout);\n        }\n        _retry_timer.set_callback([this, l] {\n            // FIXME: ignoring return\n            (void)send_discover(l.ip);\n        });\n        _retry_timer.arm_periodic(1s);\n        return _result.get_future();\n    }\n\n    template<typename T>\n    \nfuture<> send(T && pkt) {\n        pkt.dhp.bootp.xid = _xid;\n        auto ipf = _stack.netif();\n        auto mac = ipf->hw_address().mac;\n        std::copy(mac.begin(), mac.end(), std::begin(pkt.dhp.bootp.chaddr));\n\n        pkt = hton(pkt);\n\n        // FIXME: future is discarded\n        (void)_sock.send({0xffffffff, server_port}, packet(reinterpret_cast<char *>(&pkt), sizeof(pkt)));\n\n        return make_ready_future<>();\n    }\n\n    future<> send_discover(const ipv4_address & ip = ipv4_address()) {\n        struct discover : public dhcp_packet_base {\n            type_option type = type_option(msg_type::DISCOVER);\n            ip_option requested_ip;\n            requested_option req;\n            option_mark end;\n        } __attribute__((packed));\n\n        discover d;\n\n        d.requested_ip = ip_option(opt_type::REQUESTED_ADDRESS, ip);\n\n        std::random_device rd;\n        std::default_random_engine e1(rd());\n        std::uniform_int_distribution<uint32_t> xid_dist{};\n\n        _xid = xid_dist(e1);\n        _state = state::DISCOVER;\n        return send(d);\n    }\n\n    future<> send_request(const lease & info) {\n        struct request : public dhcp_packet_base {\n            type_option type = type_option(msg_type::REQUEST);\n            ip_option dhcp_server;\n            ip_option requested_ip;\n            requested_option req;\n            option_mark end;\n        } __attribute__((packed));\n\n        request d;\n\n        d.dhcp_server = ip_option(opt_type::DHCP_SERVER, info.dhcp_server);\n        d.requested_ip = ip_option(opt_type::REQUESTED_ADDRESS, info.ip);\n\n        log() << \"sending request for \" << info.ip << std::endl;\n        _state = state::REQUEST;\n        return send(d);\n    }\n\nprivate:\n    promise<std::optional<lease>> _result;\n    state _state = state::NONE;\n    timer<> _timer;\n    timer<> _retry_timer;\n    ipv4 & _stack;\n    udp_channel _sock;\n    uint32_t _xid = 0;\n};\n\nconst net::dhcp::impl::req_opt_type net::dhcp::impl::requested_options = { {\n        opt_type::SUBNET_MASK, opt_type::ROUTER, opt_type::DOMAIN_NAME_SERVERS,\n        opt_type::INTERFACE_MTU, opt_type::BROADCAST_ADDRESS } };\n\nconst net::dhcp::impl::magic_tag net::dhcp::impl::options_magic = { { 0x63, 0x82, 0x53,\n        0x63 } };\n\nconst uint16_t net::dhcp::impl::client_port;\nconst uint16_t net::dhcp::impl::server_port;\n\nconst steady_clock_type::duration net::dhcp::default_timeout = std::chrono::duration_cast<steady_clock_type::duration>(std::chrono::seconds(30));\n\nnet::dhcp::dhcp(ipv4 & ip)\n: _impl(std::make_unique<impl>(ip))\n{}\n\nnet::dhcp::dhcp(dhcp && v) noexcept = default;\n\nnet::dhcp::~dhcp()\n{}\n\nnet::dhcp::result_type net::dhcp::discover(const steady_clock_type::duration & timeout) {\n    return _impl->run(lease(), timeout);\n}\n\nnet::dhcp::result_type net::dhcp::renew(const lease & l, const steady_clock_type::duration & timeout) {\n    return _impl->run(l, timeout);\n}\n\nnet::ip_packet_filter* net::dhcp::get_ipv4_filter() {\n    return _impl.get();\n}\n\n}\n\n// Path: src/net/ip.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n *\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <chrono>\n#include <string>\n#include <boost/asio/ip/address_v4.hpp>\n#include <fmt/core.h>\nmodule seastar;\n#else\n#include <seastar/net/ip.hh>\n#include <seastar/core/print.hh>\n#include <seastar/core/shared_ptr.hh>\n#include <seastar/net/toeplitz.hh>\n#include <seastar/core/metrics.hh>\n#endif\n\nnamespace seastar {\n\nnamespace net {\n\nipv4_address::ipv4_address(const std::string& addr) {\n    boost::system::error_code ec;\n    auto ipv4 = boost::asio::ip::address_v4::from_string(addr, ec);\n    if (ec) {\n        throw std::runtime_error(\n            fmt::format(\"Wrong format for IPv4 address {}. Please ensure it's in dotted-decimal format\", addr));\n    }\n    ip = static_cast<uint32_t>(std::move(ipv4).to_ulong());\n}\n\nipv4::ipv4(interface* netif)\n    : _netif(netif)\n    , _global_arp(netif)\n    , _arp(_global_arp)\n    , _host_address(0)\n    , _gw_address(0)\n    , _netmask(0)\n    , _l3(netif, eth_protocol_num::ipv4, [this] { return get_packet(); })\n    , _tcp(*this)\n    , _icmp(*this)\n    , _udp(*this)\n    , _l4({ { uint8_t(ip_protocol_num::tcp), &_tcp }, { uint8_t(ip_protocol_num::icmp), &_icmp }, { uint8_t(ip_protocol_num::udp), &_udp }})\n{\n    namespace sm = seastar::metrics;\n    // FIXME: ignored future\n    (void)_l3.receive(\n        [this](packet p, ethernet_address ea) {\n            return handle_received_packet(std::move(p), ea);\n        },\n        [this](forward_hash& out_hash_data, packet& p, size_t off) {\n            return forward(out_hash_data, p, off);\n        });\n\n    _metrics.add_group(\"ipv4\", {\n        //\n        // Linearized events: DERIVE:0:u\n        //\n        sm::make_counter(\"linearizations\", [] { return ipv4_packet_merger::linearizations(); },\n                        sm::description(\"Counts a number of times a buffer linearization was invoked during buffers merge process. \"\n                                        \"Divide it by a total IPv4 receive packet rate to get an average number of lineraizations per packet.\"))\n    });\n    _frag_timer.set_callback([this] { frag_timeout(); });\n}\n\nbool ipv4::forward(forward_hash& out_hash_data, packet& p, size_t off)\n{\n    auto iph = p.get_header<ip_hdr>(off);\n\n    out_hash_data.push_back(iph->src_ip.ip);\n    out_hash_data.push_back(iph->dst_ip.ip);\n\n    auto h = ntoh(*iph);\n    auto l4 = _l4[h.ip_proto];\n    if (l4) {\n        if (h.mf() == false && h.offset() == 0) {\n            // This IP datagram is atomic, forward according to tcp or udp connection hash\n            l4->forward(out_hash_data, p, off + sizeof(ip_hdr));\n        }\n        // else forward according to ip fields only\n    }\n    return true;\n}\n\nbool ipv4::in_my_netmask(ipv4_address a) const {\n    return !((a.ip ^ _host_address.ip) & _netmask.ip);\n}\n\nbool ipv4::needs_frag(packet& p, ip_protocol_num prot_num, net::hw_features hw_features) {\n    if (p.len() + ipv4_hdr_len_min <= hw_features.mtu) {\n        return false;\n    }\n\n    if ((prot_num == ip_protocol_num::tcp && hw_features.tx_tso) ||\n        (prot_num == ip_protocol_num::udp && hw_features.tx_ufo)) {\n        return false;\n    }\n\n    return true;\n}\n\nfuture<>\nipv4::handle_received_packet(packet p, ethernet_address from) {\n    auto iph = p.get_header<ip_hdr>(0);\n    if (!iph) {\n        return make_ready_future<>();\n    }\n\n    // Skip checking csum of reassembled IP datagram\n    if (!hw_features().rx_csum_offload && !p.offload_info_ref().reassembled) {\n        checksummer csum;\n        csum.sum(reinterpret_cast<char*>(iph), sizeof(*iph));\n        if (csum.get() != 0) {\n            return make_ready_future<>();\n        }\n    }\n\n    auto h = ntoh(*iph);\n    unsigned ip_len = h.len;\n    unsigned ip_hdr_len = h.ihl * 4;\n    unsigned pkt_len = p.len();\n    auto offset = h.offset();\n    if (pkt_len > ip_len) {\n        // Trim extra data in the packet beyond IP total length\n        p.trim_back(pkt_len - ip_len);\n    } else if (pkt_len < ip_len) {\n        // Drop if it contains less than IP total length\n        return make_ready_future<>();\n    }\n    // Drop if the reassembled datagram will be larger than maximum IP size\n    if (offset + p.len() > net::ip_packet_len_max) {\n        return make_ready_future<>();\n    }\n\n    // FIXME: process options\n    if (in_my_netmask(h.src_ip) && h.src_ip != _host_address) {\n        _arp.learn(from, h.src_ip);\n    }\n\n    if (_packet_filter) {\n        bool handled = false;\n        auto r = _packet_filter->handle(p, &h, from, handled);\n        if (handled) {\n            return r;\n        }\n    }\n\n    if (h.dst_ip != _host_address) {\n        // FIXME: forward\n        return make_ready_future<>();\n    }\n\n    // Does this IP datagram need reassembly\n    auto mf = h.mf();\n    if (mf == true || offset != 0) {\n        frag_limit_mem();\n        auto frag_id = ipv4_frag_id{h.src_ip, h.dst_ip, h.id, h.ip_proto};\n        auto& frag = _frags[frag_id];\n        if (mf == false) {\n            frag.last_frag_received = true;\n        }\n        // This is a newly created frag_id\n        if (frag.mem_size == 0) {\n            _frags_age.push_back(frag_id);\n            frag.rx_time = clock_type::now();\n        }\n        auto added_size = frag.merge(h, offset, std::move(p));\n        _frag_mem += added_size;\n        if (frag.is_complete()) {\n            // All the fragments are received\n            auto dropped_size = frag.mem_size;\n            auto& ip_data = frag.data.map.begin()->second;\n            // Choose a cpu to forward this packet\n            auto cpu_id = this_shard_id();\n            auto l4 = _l4[h.ip_proto];\n            if (l4) {\n                if (smp::count == 1) {\n                    l4->received(std::move(ip_data), h.src_ip, h.dst_ip);\n                } else {\n                    size_t l4_offset = 0;\n                    forward_hash hash_data;\n                    hash_data.push_back(hton(h.src_ip.ip));\n                    hash_data.push_back(hton(h.dst_ip.ip));\n                    auto forwarded = l4->forward(hash_data, ip_data, l4_offset);\n                    if (forwarded) {\n                        cpu_id = _netif->hash2cpu(toeplitz_hash(_netif->rss_key(), hash_data));\n                        // No need to forward if the dst cpu is the current cpu\n                        if (cpu_id == this_shard_id()) {\n                            l4->received(std::move(ip_data), h.src_ip, h.dst_ip);\n                        } else {\n                            auto to = _netif->hw_address();\n                            auto pkt = frag.get_assembled_packet(from, to);\n                            _netif->forward(cpu_id, std::move(pkt));\n                        }\n                    }\n                }\n            }\n\n            // Delete this frag from _frags and _frags_age\n            frag_drop(frag_id, dropped_size);\n            _frags_age.remove(frag_id);\n        } else {\n            // Some of the fragments are missing\n            if (!_frag_timer.armed()) {\n                frag_arm();\n            }\n        }\n        return make_ready_future<>();\n    }\n\n    auto l4 = _l4[h.ip_proto];\n    if (l4) {\n        // Trim IP header and pass to upper layer\n        p.trim_front(ip_hdr_len);\n        l4->received(std::move(p), h.src_ip, h.dst_ip);\n    }\n    return make_ready_future<>();\n}\n\nfuture<ethernet_address> ipv4::get_l2_dst_address(ipv4_address to) {\n    // Figure out where to send the packet to. If it is a directly connected\n    // host, send to it directly, otherwise send to the default gateway.\n    ipv4_address dst;\n    if (in_my_netmask(to)) {\n        dst = to;\n    } else {\n        dst = _gw_address;\n    }\n\n    return _arp.lookup(dst);\n}\n\nvoid ipv4::send(ipv4_address to, ip_protocol_num proto_num, packet p, ethernet_address e_dst) {\n    auto needs_frag = this->needs_frag(p, proto_num, hw_features());\n\n    auto send_pkt = [this, to, proto_num, needs_frag, e_dst] (packet& pkt, uint16_t remaining, uint16_t offset) mutable  {\n        auto iph = pkt.prepend_header<ip_hdr>();\n        iph->ihl = sizeof(*iph) / 4;\n        iph->ver = 4;\n        iph->dscp = 0;\n        iph->ecn = 0;\n        iph->len = pkt.len();\n        // FIXME: a proper id\n        iph->id = 0;\n        if (needs_frag) {\n            uint16_t mf = remaining > 0;\n            // The fragment offset is measured in units of 8 octets (64 bits)\n            auto off = offset / 8;\n            iph->frag = (mf << uint8_t(ip_hdr::frag_bits::mf)) | off;\n        } else {\n            iph->frag = 0;\n        }\n        iph->ttl = 64;\n        iph->ip_proto = (uint8_t)proto_num;\n        iph->csum = 0;\n        iph->src_ip = _host_address;\n        iph->dst_ip = to;\n        *iph = hton(*iph);\n\n        if (hw_features().tx_csum_ip_offload) {\n            iph->csum = 0;\n            pkt.offload_info_ref().needs_ip_csum = true;\n        } else {\n            checksummer csum;\n            csum.sum(reinterpret_cast<char*>(iph), sizeof(*iph));\n            iph->csum = csum.get();\n        }\n\n        _packetq.push_back(l3_protocol::l3packet{eth_protocol_num::ipv4, e_dst, std::move(pkt)});\n    };\n\n    if (needs_frag) {\n        uint16_t offset = 0;\n        uint16_t remaining = p.len();\n        auto mtu = hw_features().mtu;\n\n        while (remaining) {\n            auto can_send = std::min(uint16_t(mtu - net::ipv4_hdr_len_min), remaining);\n            remaining -= can_send;\n            auto pkt = p.share(offset, can_send);\n            send_pkt(pkt, remaining, offset);\n            offset += can_send;\n        }\n    } else {\n        // The whole packet can be send in one shot\n        send_pkt(p, 0, 0);\n    }\n}\n\nstd::optional<l3_protocol::l3packet> ipv4::get_packet() {\n    // _packetq will be mostly empty here unless it hold remnants of previously\n    // fragmented packet\n    if (_packetq.empty()) {\n        for (size_t i = 0; i < _pkt_providers.size(); i++) {\n            auto l4p = _pkt_providers[_pkt_provider_idx++]();\n            if (_pkt_provider_idx == _pkt_providers.size()) {\n                _pkt_provider_idx = 0;\n            }\n            if (l4p) {\n                auto l4pv = std::move(l4p.value());\n                send(l4pv.to, l4pv.proto_num, std::move(l4pv.p), l4pv.e_dst);\n                break;\n            }\n        }\n    }\n\n    std::optional<l3_protocol::l3packet> p;\n    if (!_packetq.empty()) {\n        p = std::move(_packetq.front());\n        _packetq.pop_front();\n    }\n    return p;\n}\n\nvoid ipv4::set_host_address(ipv4_address ip) {\n    _host_address = ip;\n    _arp.set_self_addr(ip);\n}\n\nipv4_address ipv4::host_address() const {\n    return _host_address;\n}\n\nvoid ipv4::set_gw_address(ipv4_address ip) {\n    _gw_address = ip;\n}\n\nipv4_address ipv4::gw_address() const {\n    return _gw_address;\n}\n\nvoid ipv4::set_netmask_address(ipv4_address ip) {\n    _netmask = ip;\n}\n\nipv4_address ipv4::netmask_address() const {\n    return _netmask;\n}\n\nvoid ipv4::set_packet_filter(ip_packet_filter * f) {\n    _packet_filter = f;\n}\n\nip_packet_filter * ipv4::packet_filter() const {\n    return _packet_filter;\n}\n\nvoid ipv4::frag_limit_mem() {\n    if (_frag_mem <= _frag_high_thresh) {\n        return;\n    }\n    auto drop = _frag_mem - _frag_low_thresh;\n    while (drop) {\n        if (_frags_age.empty()) {\n            return;\n        }\n        // Drop the oldest frag (first element) from _frags_age\n        auto frag_id = _frags_age.front();\n        _frags_age.pop_front();\n\n        // Drop from _frags as well\n        auto& frag = _frags[frag_id];\n        auto dropped_size = frag.mem_size;\n        frag_drop(frag_id, dropped_size);\n\n        drop -= std::min(drop, dropped_size);\n    }\n}\n\nvoid ipv4::frag_timeout() {\n    if (_frags.empty()) {\n        return;\n    }\n    auto now = clock_type::now();\n    for (auto it = _frags_age.begin(); it != _frags_age.end();) {\n        auto frag_id = *it;\n        auto& frag = _frags[frag_id];\n        if (now > frag.rx_time + _frag_timeout) {\n            auto dropped_size = frag.mem_size;\n            // Drop from _frags\n            frag_drop(frag_id, dropped_size);\n            // Drop from _frags_age\n            it = _frags_age.erase(it);\n        } else {\n            // The further items can only be younger\n            break;\n        }\n    }\n    if (_frags.size() != 0) {\n        frag_arm(now);\n    } else {\n        _frag_mem = 0;\n    }\n}\n\nvoid ipv4::frag_drop(ipv4_frag_id frag_id, uint32_t dropped_size) {\n    _frags.erase(frag_id);\n    _frag_mem -= dropped_size;\n}\n\nint32_t ipv4::frag::merge(ip_hdr &h, uint16_t offset, packet p) {\n    uint32_t old = mem_size;\n    unsigned ip_hdr_len = h.ihl * 4;\n    // Store IP header\n    if (offset == 0) {\n        header = p.share(0, ip_hdr_len);\n    }\n    // Sotre IP payload\n    p.trim_front(ip_hdr_len);\n    data.merge(offset, std::move(p));\n    // Update mem size\n    mem_size = header.memory();\n    for (const auto& x : data.map) {\n        mem_size += x.second.memory();\n    }\n    auto added_size = mem_size - old;\n    return added_size;\n}\n\nbool ipv4::frag::is_complete() {\n    // If all the fragments are received, ipv4::frag::merge() should merge all\n    // the fragments into a single packet\n    auto offset = data.map.begin()->first;\n    auto nr_packet = data.map.size();\n    return last_frag_received && nr_packet == 1 && offset == 0;\n}\n\npacket ipv4::frag::get_assembled_packet(ethernet_address from, ethernet_address to) {\n    auto& ip_header = header;\n    auto& ip_data = data.map.begin()->second;\n    // Append a ethernet header, needed for forwarding\n    auto eh = ip_header.prepend_header<eth_hdr>();\n    eh->src_mac = from;\n    eh->dst_mac = to;\n    eh->eth_proto = uint16_t(eth_protocol_num::ipv4);\n    *eh = hton(*eh);\n    // Prepare a packet contains both ethernet header, ip header and ip data\n    ip_header.append(std::move(ip_data));\n    auto pkt = std::move(ip_header);\n    auto iph = pkt.get_header<ip_hdr>(sizeof(eth_hdr));\n    // len is the sum of each fragment\n    iph->len = hton(uint16_t(pkt.len() - sizeof(eth_hdr)));\n    // No fragmentation for the assembled datagram\n    iph->frag = 0;\n    // Since each fragment's csum is checked, no need to csum\n    // again for the assembled datagram\n    offload_info oi;\n    oi.reassembled = true;\n    pkt.set_offload_info(oi);\n    return pkt;\n}\n\nvoid icmp::received(packet p, ipaddr from, ipaddr to) {\n    auto hdr = p.get_header<icmp_hdr>(0);\n    if (!hdr || hdr->type != icmp_hdr::msg_type::echo_request) {\n        return;\n    }\n    hdr->type = icmp_hdr::msg_type::echo_reply;\n    hdr->code = 0;\n    hdr->csum = 0;\n    checksummer csum;\n    csum.sum(reinterpret_cast<char*>(hdr), p.len());\n    hdr->csum = csum.get();\n\n    if (_queue_space.try_wait(p.len())) { // drop packets that do not fit the queue\n        // FIXME: future is discarded\n        (void)_inet.get_l2_dst_address(from).then([this, from, p = std::move(p)] (ethernet_address e_dst) mutable {\n            _packetq.emplace_back(ipv4_traits::l4packet{from, std::move(p), e_dst, ip_protocol_num::icmp});\n        });\n    }\n}\n\n}\n\n}\n\n// Path: src/net/tcp.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <compare>\n#include <atomic>\n#include <cassert>\n#include <cstdint>\n#include <memory>\n#include <utility>\nmodule seastar;\n#else\n#include <seastar/net/tcp.hh>\n#include <seastar/net/tcp-stack.hh>\n#include <seastar/net/ip.hh>\n#include <seastar/core/align.hh>\n#include <seastar/core/future.hh>\n#include \"net/native-stack-impl.hh\"\n#endif\n\nnamespace seastar {\n\nnamespace net {\n\nvoid tcp_option::parse(uint8_t* beg1, uint8_t* end1) {\n    const char* beg = reinterpret_cast<const char*>(beg1);\n    const char* end = reinterpret_cast<const char*>(end1);\n    while (beg < end) {\n        auto kind = option_kind(*beg);\n        if (kind != option_kind::nop && kind != option_kind::eol) {\n            // Make sure there is enough room for this option\n            auto len = uint8_t(beg[1]);\n            if (beg + len > end) {\n                return;\n            }\n        }\n        switch (kind) {\n        case option_kind::mss:\n            _mss_received = true;\n            _remote_mss = mss::read(beg).mss;\n            beg += option_len::mss;\n            break;\n        case option_kind::win_scale:\n            _win_scale_received = true;\n            _remote_win_scale = win_scale::read(beg).shift;\n            // We can turn on win_scale option, 7 is Linux's default win scale size\n            _local_win_scale = 7;\n            beg += option_len::win_scale;\n            break;\n        case option_kind::sack:\n            _sack_received = true;\n            beg += option_len::sack;\n            break;\n        case option_kind::nop:\n            beg += option_len::nop;\n            break;\n        case option_kind::eol:\n            return;\n        default:\n            // Ignore options we do not understand\n            uint8_t len = *(beg + 1);\n            beg += len;\n            // Prevent infinite loop\n            if (len == 0) {\n                return;\n            }\n            break;\n        }\n    }\n}\n\nuint8_t tcp_option::fill(void* h, const tcp_hdr* th, uint8_t options_size) {\n    auto hdr = reinterpret_cast<char*>(h);\n    auto off = hdr + tcp_hdr::len;\n    uint8_t size = 0;\n    bool syn_on = th->f_syn;\n    bool ack_on = th->f_ack;\n\n    if (syn_on) {\n        if (_mss_received || !ack_on) {\n            auto mss = tcp_option::mss();\n            mss.mss = _local_mss;\n            mss.write(off);\n            off += mss.len;\n            size += mss.len;\n        }\n        if (_win_scale_received || !ack_on) {\n            auto win_scale = tcp_option::win_scale();\n            win_scale.shift = _local_win_scale;\n            win_scale.write(off);\n            off += win_scale.len;\n            size += win_scale.len;\n        }\n    }\n    if (size > 0) {\n        // Insert NOP option\n        auto size_max = align_up(uint8_t(size + 1), tcp_option::align);\n        while (size < size_max - uint8_t(option_len::eol)) {\n            auto nop = tcp_option::nop();\n            nop.write(off);\n            off += option_len::nop;\n            size += option_len::nop;\n        }\n        auto eol = tcp_option::eol();\n        eol.write(off);\n        size += option_len::eol;\n    }\n    assert(size == options_size);\n\n    return size;\n}\n\nuint8_t tcp_option::get_size(bool syn_on, bool ack_on) {\n    uint8_t size = 0;\n    if (syn_on) {\n        if (_mss_received || !ack_on) {\n            size += option_len::mss;\n        }\n        if (_win_scale_received || !ack_on) {\n            size += option_len::win_scale;\n        }\n    }\n    if (size > 0) {\n        size += option_len::eol;\n        // Insert NOP option to align on 32-bit\n        size = align_up(size, tcp_option::align);\n    }\n    return size;\n}\n\nipv4_tcp::ipv4_tcp(ipv4& inet)\n\t: _inet_l4(inet), _tcp(std::make_unique<tcp<ipv4_traits>>(_inet_l4)) {\n}\n\nipv4_tcp::~ipv4_tcp() {\n}\n\nvoid ipv4_tcp::received(packet p, ipv4_address from, ipv4_address to) {\n    _tcp->received(std::move(p), from, to);\n}\n\nbool ipv4_tcp::forward(forward_hash& out_hash_data, packet& p, size_t off) {\n\n    return _tcp->forward(out_hash_data, p, off);\n}\n\nserver_socket\ntcpv4_listen(tcp<ipv4_traits>& tcpv4, uint16_t port, listen_options opts) {\n\treturn server_socket(std::make_unique<native_server_socket_impl<tcp<ipv4_traits>>>(\n\t\t\ttcpv4, port, opts));\n}\n\n::seastar::socket\ntcpv4_socket(tcp<ipv4_traits>& tcpv4) {\n    return ::seastar::socket(std::make_unique<native_socket_impl<tcp<ipv4_traits>>>(\n            tcpv4));\n}\n\n}\n\n}\n\n// Path: src/net/ip_checksum.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <arpa/inet.h>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/net/ip_checksum.hh>\n#include <seastar/net/net.hh>\n#endif\n\nnamespace seastar {\n\nnamespace net {\n\nvoid checksummer::sum(const char* data, size_t len) {\n    auto orig_len = len;\n    if (odd) {\n        csum += uint8_t(*data++);\n        --len;\n    }\n    auto p64 = reinterpret_cast<const packed<uint64_t>*>(data);\n    while (len >= 8) {\n        csum += ntohq(*p64++);\n        len -= 8;\n    }\n    auto p16 = reinterpret_cast<const packed<uint16_t>*>(p64);\n    while (len >= 2) {\n        csum += ntohs(*p16++);\n        len -= 2;\n    }\n    auto p8 = reinterpret_cast<const uint8_t*>(p16);\n    if (len) {\n        csum += *p8++ << 8;\n        len -= 1;\n    }\n    odd ^= orig_len & 1;\n}\n\nuint16_t checksummer::get() const {\n    __int128 csum1 = (csum & 0xffff'ffff'ffff'ffff) + (csum >> 64);\n    uint64_t csum = (csum1 & 0xffff'ffff'ffff'ffff) + (csum1 >> 64);\n    csum = (csum & 0xffff) + ((csum >> 16) & 0xffff) + ((csum >> 32) & 0xffff) + (csum >> 48);\n    csum = (csum & 0xffff) + (csum >> 16);\n    csum = (csum & 0xffff) + (csum >> 16);\n    return htons(~csum);\n}\n\nvoid checksummer::sum(const packet& p) {\n    for (auto&& f : p.fragments()) {\n        sum(f.base, f.size);\n    }\n}\n\nuint16_t ip_checksum(const void* data, size_t len) {\n    checksummer cksum;\n    cksum.sum(reinterpret_cast<const char*>(data), len);\n    return cksum.get();\n}\n\n\n}\n\n}\n\n// Path: src/net/udp.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cstdint>\n#include <utility>\n#include <cstring>\n#include <exception>\n#include <system_error>\n#include <optional>\n#include <memory>\nmodule seastar;\n#else\n#include <seastar/net/ip.hh>\n#include <seastar/net/stack.hh>\n#include <seastar/net/inet_address.hh>\n#endif\n\nnamespace seastar {\n\nusing namespace net;\n\nnamespace net {\nnamespace ipv4_udp_impl {\n\nstatic inline\nipv4_addr\nto_ipv4_addr(ipv4_address a, uint16_t port) {\n    return {a.ip, port};\n}\n\nclass native_datagram : public datagram_impl {\nprivate:\n    ipv4_addr _src;\n    ipv4_addr _dst;\n    packet _p;\npublic:\n    native_datagram(ipv4_address src, ipv4_address dst, packet p)\n            : _p(std::move(p)) {\n        udp_hdr* hdr = _p.get_header<udp_hdr>();\n        auto h = ntoh(*hdr);\n        _p.trim_front(sizeof(*hdr));\n        _src = to_ipv4_addr(src, h.src_port);\n        _dst = to_ipv4_addr(dst, h.dst_port);\n    }\n\n    virtual socket_address get_src() override {\n        return _src;\n    };\n\n    virtual socket_address get_dst() override {\n        return _dst;\n    };\n\n    virtual uint16_t get_dst_port() override {\n        return _dst.port;\n    }\n\n    virtual packet& get_data() override {\n        return _p;\n    }\n};\n\nclass native_channel : public datagram_channel_impl {\nprivate:\n    ipv4_udp& _proto;\n    ipv4_udp::registration _reg;\n    bool _closed;\n    lw_shared_ptr<udp_channel_state> _state;\n\npublic:\n    native_channel(ipv4_udp &proto, ipv4_udp::registration reg, lw_shared_ptr<udp_channel_state> state)\n            : _proto(proto)\n            , _reg(reg)\n            , _closed(false)\n            , _state(state)\n    {\n    }\n\n    ~native_channel()\n    {\n        if (!_closed)\n            close();\n    }\n\n    socket_address local_address() const override {\n        return socket_address(_proto.inet().host_address(), _reg.port());\n    }\n\n    virtual future<datagram> receive() override {\n        return _state->_queue.pop_eventually();\n    }\n\n    virtual future<> send(const socket_address& dst, const char* msg) override {\n        return send(dst, packet::from_static_data(msg, strlen(msg)));\n    }\n\n    virtual future<> send(const socket_address& dst, packet p) override {\n        auto len = p.len();\n        return _state->wait_for_send_buffer(len).then([this, dst, p = std::move(p), len] () mutable {\n            p = packet(std::move(p), make_deleter([s = _state, len] { s->complete_send(len); }));\n            _proto.send(_reg.port(), dst, std::move(p));\n        });\n    }\n\n    virtual bool is_closed() const override {\n        return _closed;\n    }\n\n    virtual void shutdown_input() override {\n        _state->_queue.abort(std::make_exception_ptr(std::system_error(EBADF, std::system_category())));\n    }\n\n    virtual void shutdown_output() override {\n        _state->_queue.abort(std::make_exception_ptr(std::system_error(EPIPE, std::system_category())));\n    }\n\n    virtual void close() override {\n        _reg.unregister();\n        _closed = true;\n    }\n};\n\n} /* namespace ipv4_udp_impl */\n\nusing namespace net::ipv4_udp_impl;\n\nconst int ipv4_udp::default_queue_size = 1024;\n\nipv4_udp::ipv4_udp(ipv4& inet)\n    : _inet(inet)\n{\n    _inet.register_packet_provider([this] {\n        std::optional<ipv4_traits::l4packet> l4p;\n        if (!_packetq.empty()) {\n            l4p = std::move(_packetq.front());\n            _packetq.pop_front();\n        }\n        return l4p;\n    });\n}\n\nbool ipv4_udp::forward(forward_hash& out_hash_data, packet& p, size_t off)\n{\n    auto uh = p.get_header<udp_hdr>(off);\n\n    if (uh) {\n        out_hash_data.push_back(uh->src_port);\n        out_hash_data.push_back(uh->dst_port);\n    }\n    return true;\n}\n\nvoid ipv4_udp::received(packet p, ipv4_address from, ipv4_address to)\n{\n    datagram dgram(std::make_unique<native_datagram>(from, to, std::move(p)));\n\n    auto chan_it = _channels.find(dgram.get_dst_port());\n    if (chan_it != _channels.end()) {\n        auto chan = chan_it->second;\n        chan->_queue.push(std::move(dgram));\n    }\n}\n\nvoid ipv4_udp::send(uint16_t src_port, ipv4_addr dst, packet &&p)\n{\n    auto src = _inet.host_address();\n    auto hdr = p.prepend_header<udp_hdr>();\n    hdr->src_port = src_port;\n    hdr->dst_port = dst.port;\n    hdr->len = p.len();\n    *hdr = hton(*hdr);\n\n    offload_info oi;\n    checksummer csum;\n    ipv4_traits::udp_pseudo_header_checksum(csum, src, dst, p.len());\n    bool needs_frag = ipv4::needs_frag(p, ip_protocol_num::udp, _inet.hw_features());\n    if (_inet.hw_features().tx_csum_l4_offload && !needs_frag) {\n        hdr->cksum = ~csum.get();\n        oi.needs_csum = true;\n    } else {\n        csum.sum(p);\n        hdr->cksum = csum.get();\n        oi.needs_csum = false;\n    }\n    oi.protocol = ip_protocol_num::udp;\n    p.set_offload_info(oi);\n\n    // FIXME: future is discarded\n    (void)_inet.get_l2_dst_address(dst).then([this, dst, p = std::move(p)] (ethernet_address e_dst) mutable {\n        _packetq.emplace_back(ipv4_traits::l4packet{dst, std::move(p), e_dst, ip_protocol_num::udp});\n    });\n}\n\nuint16_t ipv4_udp::next_port(uint16_t port) {\n    return (port + 1) == 0 ? min_anonymous_port : port + 1;\n}\n\nudp_channel\nipv4_udp::make_channel(ipv4_addr addr) {\n    if (!is_ip_unspecified(addr)) {\n        throw std::runtime_error(\"Binding to specific IP not supported yet\");\n    }\n\n    uint16_t bind_port;\n\n    if (!is_port_unspecified(addr)) {\n        if (_channels.count(addr.port)) {\n            throw std::runtime_error(\"Address already in use\");\n        }\n        bind_port = addr.port;\n    } else {\n        auto starting_port = _next_anonymous_port;\n        while (_channels.count(_next_anonymous_port)) {\n            _next_anonymous_port = next_port(_next_anonymous_port);\n            if (starting_port == _next_anonymous_port) {\n                throw std::runtime_error(\"No free port\");\n            }\n        }\n\n        bind_port = _next_anonymous_port;\n        _next_anonymous_port = next_port(_next_anonymous_port);\n    }\n\n    auto chan_state = make_lw_shared<udp_channel_state>(_queue_size);\n    _channels[bind_port] = chan_state;\n    return udp_channel(std::make_unique<native_channel>(*this, registration(*this, bind_port), chan_state));\n}\n\n} /* namespace net */\n\n}\n\n\n// Path: src/net/stack.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2015 Cloudius Systems\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <memory>\n#include <type_traits>\n#include <utility>\n#include <vector>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/net/stack.hh>\n#include <seastar/net/inet_address.hh>\n#endif\n\nnamespace seastar {\n\nstatic_assert(std::is_nothrow_default_constructible_v<connected_socket>);\nstatic_assert(std::is_nothrow_move_constructible_v<connected_socket>);\n\nstatic_assert(std::is_nothrow_default_constructible_v<socket>);\nstatic_assert(std::is_nothrow_move_constructible_v<socket>);\n\nstatic_assert(std::is_nothrow_default_constructible_v<server_socket>);\nstatic_assert(std::is_nothrow_move_constructible_v<server_socket>);\n\nnet::datagram_channel::datagram_channel() noexcept\n{}\n\nnet::datagram_channel::datagram_channel(std::unique_ptr<datagram_channel_impl> impl) noexcept : _impl(std::move(impl))\n{}\n\nnet::datagram_channel::~datagram_channel()\n{}\n\nnet::datagram_channel::datagram_channel(datagram_channel&&) noexcept = default;\nnet::datagram_channel& net::datagram_channel::operator=(datagram_channel&&) noexcept = default;\n\nsocket_address net::datagram_channel::local_address() const {\n    if (_impl) {\n        return _impl->local_address();\n    } else {\n        return {};\n    }\n}\n\nfuture<net::datagram> net::datagram_channel::receive() {\n    return _impl->receive();\n}\n\nfuture<> net::datagram_channel::send(const socket_address& dst, const char* msg) {\n    return _impl->send(dst, msg);\n}\n\nfuture<> net::datagram_channel::send(const socket_address& dst, packet p) {\n    return _impl->send(dst, std::move(p));\n}\n\nbool net::datagram_channel::is_closed() const {\n    return _impl->is_closed();\n}\n\nvoid net::datagram_channel::shutdown_input() {\n    _impl->shutdown_input();\n}\n\nvoid net::datagram_channel::shutdown_output() {\n    _impl->shutdown_output();\n}\n\n\nvoid net::datagram_channel::close() {\n    return _impl->close();\n}\n\nconnected_socket::connected_socket() noexcept\n{}\n\nconnected_socket::connected_socket(\n        std::unique_ptr<net::connected_socket_impl> csi) noexcept\n        : _csi(std::move(csi)) {\n}\n\nconnected_socket::connected_socket(connected_socket&& cs) noexcept = default;\nconnected_socket& connected_socket::operator=(connected_socket&& cs) noexcept = default;\n\nconnected_socket::~connected_socket()\n{}\n\ninput_stream<char> connected_socket::input(connected_socket_input_stream_config csisc) {\n    return input_stream<char>(_csi->source(csisc));\n}\n\noutput_stream<char> connected_socket::output(size_t buffer_size) {\n    output_stream_options opts;\n    opts.batch_flushes = true;\n    // TODO: allow user to determine buffer size etc\n    return output_stream<char>(_csi->sink(), buffer_size, opts);\n}\n\nvoid connected_socket::set_nodelay(bool nodelay) {\n    _csi->set_nodelay(nodelay);\n}\n\nbool connected_socket::get_nodelay() const {\n    return _csi->get_nodelay();\n}\nvoid connected_socket::set_keepalive(bool keepalive) {\n    _csi->set_keepalive(keepalive);\n}\nbool connected_socket::get_keepalive() const {\n    return _csi->get_keepalive();\n}\nvoid connected_socket::set_keepalive_parameters(const net::keepalive_params& p) {\n    _csi->set_keepalive_parameters(p);\n}\nnet::keepalive_params connected_socket::get_keepalive_parameters() const {\n    return _csi->get_keepalive_parameters();\n}\nvoid connected_socket::set_sockopt(int level, int optname, const void* data, size_t len) {\n    _csi->set_sockopt(level, optname, data, len);\n}\nint connected_socket::get_sockopt(int level, int optname, void* data, size_t len) const {\n    return _csi->get_sockopt(level, optname, data, len);\n}\n\nsocket_address connected_socket::local_address() const noexcept {\n    return _csi->local_address();\n}\n\nsocket_address connected_socket::remote_address() const noexcept {\n    return _csi->remote_address();\n}\n\nvoid connected_socket::shutdown_output() {\n    _csi->shutdown_output();\n}\n\nvoid connected_socket::shutdown_input() {\n    _csi->shutdown_input();\n}\n\nfuture<> connected_socket::wait_input_shutdown() {\n    return _csi->wait_input_shutdown();\n}\n\ndata_source\nnet::connected_socket_impl::source(connected_socket_input_stream_config csisc) {\n    // Default implementation falls back to non-parameterized data_source\n    return source();\n}\n\nsocket::~socket()\n{}\n\nsocket::socket(\n        std::unique_ptr<net::socket_impl> si) noexcept\n        : _si(std::move(si)) {\n}\n\nsocket::socket(socket&&) noexcept = default;\nsocket& socket::operator=(socket&&) noexcept = default;\n\nfuture<connected_socket> socket::connect(socket_address sa, socket_address local, transport proto) {\n    return _si->connect(sa, local, proto);\n}\n\nvoid socket::set_reuseaddr(bool reuseaddr) {\n    _si->set_reuseaddr(reuseaddr);\n}\n\nbool socket::get_reuseaddr() const {\n    return _si->get_reuseaddr();\n}\n\nvoid socket::shutdown() {\n    _si->shutdown();\n}\n\nserver_socket::server_socket() noexcept {\n}\n\nserver_socket::server_socket(std::unique_ptr<net::server_socket_impl> ssi) noexcept\n        : _ssi(std::move(ssi)) {\n}\nserver_socket::server_socket(server_socket&& ss) noexcept = default;\nserver_socket& server_socket::operator=(server_socket&& cs) noexcept = default;\n\nserver_socket::~server_socket() {\n}\n\nfuture<accept_result> server_socket::accept() {\n    if (_aborted) {\n        return make_exception_future<accept_result>(std::system_error(ECONNABORTED, std::system_category()));\n    }\n    return _ssi->accept();\n}\n\nvoid server_socket::abort_accept() {\n    _ssi->abort_accept();\n    _aborted = true;\n}\n\nsocket_address server_socket::local_address() const noexcept {\n    return _ssi->local_address();\n}\n\nnetwork_interface::network_interface(shared_ptr<net::network_interface_impl> impl) noexcept\n    : _impl(std::move(impl))\n{}\n\nnetwork_interface::network_interface(network_interface&&) noexcept = default;\nnetwork_interface& network_interface::operator=(network_interface&&) noexcept = default;\n    \nuint32_t network_interface::index() const {\n    return _impl->index();\n}\n\nuint32_t network_interface::mtu() const {\n    return _impl->mtu();\n}\n\nconst sstring& network_interface::name() const {\n    return _impl->name();\n}\n\nconst sstring& network_interface::display_name() const {\n    return _impl->display_name();\n}\n\nconst std::vector<net::inet_address>& network_interface::addresses() const {\n    return _impl->addresses();\n}\n\nconst std::vector<uint8_t> network_interface::hardware_address() const {\n    return _impl->hardware_address();\n}\n\nbool network_interface::is_loopback() const {\n    return _impl->is_loopback();\n}\n\nbool network_interface::is_virtual() const {\n    return _impl->is_virtual();\n}\n\nbool network_interface::is_up() const {\n    return _impl->is_up();\n}\n\nbool network_interface::supports_ipv6() const {\n    return _impl->supports_ipv6();\n}\n\n\nfuture<connected_socket>\nnetwork_stack::connect(socket_address sa, socket_address local, transport proto) {\n    return do_with(socket(), [sa, local, proto](::seastar::socket& s) {\n        return s.connect(sa, local, proto);\n    });\n}\n\nstd::vector<network_interface> network_stack::network_interfaces() {\n    return {};\n}\n\n}\n\n// Path: src/net/config.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2017 Marek Waszkiewicz ( marek.waszkiewicz77@gmail.com )\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <boost/algorithm/cxx11/all_of.hpp>\n#include <boost/algorithm/cxx11/none_of.hpp>\n#include <boost/next_prior.hpp>\n#include <yaml-cpp/yaml.h>\n#include <algorithm>\n#include <istream>\n#include <unordered_map>\n#include <string>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/net/config.hh>\n#include <seastar/core/print.hh>\n#endif\n\nusing namespace boost::algorithm;\n\nnamespace seastar {\nnamespace net {\n\n    // list of supported config keys\n    std::string config_keys[]{ \"pci-address\", \"port-index\", \"ip\", \"gateway\", \"netmask\", \"dhcp\", \"lro\", \"tso\", \"ufo\", \"hw-fc\", \"event-index\", \"csum-offload\",\"ring-size\" };\n\n    std::unordered_map<std::string, device_config>\n    parse_config(std::istream& input) {\n        std::unordered_map<std::string, device_config> device_configs;\n\n        YAML::Node doc = YAML::Load(input);\n        for (auto&& item : doc) {\n            device_configs[item.first.as<std::string>()] = item.second.as<device_config>();\n        }\n\n        bool port_index_used = false;\n        bool pci_address_used = false;\n\n        for (auto&& item : device_configs) {\n\n            if (item.second.hw_cfg.port_index) {\n                port_index_used = true;\n            }\n\n            if (!item.second.hw_cfg.pci_address.empty()) {\n                pci_address_used = true;\n            }\n\n            if (port_index_used && pci_address_used) {\n                throw config_exception(\"port_index and pci_address cannot be used together\");\n            }\n        }\n\n...\n", "needle_token_start": 717, "needle_token_end": 877, "code_context_ntokens": 16030, "output": ["```cpp\nfuture<> send(T && pkt) {\n    pkt.dhp.bootp.xid = _xid;\n    auto ipf = _stack.netif();\n    auto mac = ipf->hw_address().mac;\n    std::copy(mac.begin(), mac.end(), std::begin(pkt.dhp.bootp.chaddr));\n\n    pkt = hton(pkt);\n\n    // FIXME: future is discarded\n    (void)_sock.send({0xffffffff, server_port}, packet(reinterpret_cast<char *>(&pkt), sizeof(pkt)));\n\n    return make_ready_future<>();\n}\n```"]}
{"repo": "scylladb/seastar", "name": "sharded_parallel_for_each", "language": "cpp", "path": "src/core/sharded.cc", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To execute a given function concurrently across multiple shards, facilitating parallel processing in a distributed system.\n2. **Input**: The number of shards to operate on and a function that defines the operation to be performed on each shard.\n3. **Output**: A future object that completes when all shard operations have finished.\n4. **Procedure**: The function iterates over a range from 0 to the specified number of shards, applying the provided function concurrently to each shard using a parallel execution mechanism.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/core/fair_queue.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2019 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <chrono>\n#include <functional>\n#include <queue>\n#include <unordered_set>\n#include <utility>\n#include <boost/container/small_vector.hpp>\n#include <boost/intrusive/parent_from_member.hpp>\n\n#include \"fmt/format.h\"\n#include \"fmt/ostream.h\"\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/fair_queue.hh>\n#include <seastar/core/future.hh>\n#include <seastar/core/shared_ptr.hh>\n#include <seastar/core/circular_buffer.hh>\n#include <seastar/util/noncopyable_function.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/metrics.hh>\n#endif\n\nnamespace seastar {\n\nstatic_assert(sizeof(fair_queue_ticket) == sizeof(uint64_t), \"unexpected fair_queue_ticket size\");\nstatic_assert(sizeof(fair_queue_entry) <= 3 * sizeof(void*), \"unexpected fair_queue_entry::_hook size\");\nstatic_assert(sizeof(fair_queue_entry::container_list_t) == 2 * sizeof(void*), \"unexpected priority_class::_queue size\");\n\nfair_queue_ticket::fair_queue_ticket(uint32_t weight, uint32_t size) noexcept\n    : _weight(weight)\n    , _size(size)\n{}\n\nfloat fair_queue_ticket::normalize(fair_queue_ticket denominator) const noexcept {\n    return float(_weight) / denominator._weight + float(_size) / denominator._size;\n}\n\nfair_queue_ticket fair_queue_ticket::operator+(fair_queue_ticket desc) const noexcept {\n    return fair_queue_ticket(_weight + desc._weight, _size + desc._size);\n}\n\n...\n// Path: src/core/dpdk_rte.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#ifdef SEASTAR_HAVE_DPDK\n\n#include <cinttypes>\n#include <seastar/net/dpdk.hh>\n#include <seastar/core/dpdk_rte.hh>\n#include <seastar/util/conversions.hh>\n#include <seastar/util/std-compat.hh>\n#include <rte_pci.h>\n\nnamespace seastar {\n\nnamespace dpdk {\n\nbool eal::initialized = false;\n\nvoid eal::init(cpuset cpus, const std::string& argv0, const std::optional<std::string>& hugepages_path, bool dpdk_pmd)\n{\n    if (initialized) {\n        return;\n    }\n\n    size_t cpu_count = cpus.count();\n    std::stringstream mask;\n    cpuset nibble = 0xF;\n    while (cpus.any()) {\n        mask << std::hex << (cpus & nibble).to_ulong();\n        cpus >>= 4;\n    }\n\n    std::string mask_str = mask.str();\n    std::reverse(mask_str.begin(), mask_str.end());\n\n    std::vector<std::vector<char>> args {\n        string2vector(argv0),\n        string2vector(\"-c\"), string2vector(mask_str),\n        string2vector(\"-n\"), string2vector(\"1\")\n    };\n\n    // If \"hugepages\" is not provided and DPDK PMD drivers mode is requested -\n    // use the default DPDK huge tables configuration.\n    if (hugepages_path) {\n        args.push_back(string2vector(\"--huge-dir\"));\n        args.push_back(string2vector(hugepages_path.value()));\n\n        //\n        // We don't know what is going to be our networking configuration so we\n        // assume there is going to be a queue per-CPU. Plus we'll give a DPDK\n        // 64MB for \"other stuff\".\n        //\n        size_t size_MB = mem_size(cpu_count) >> 20;\n        std::stringstream size_MB_str;\n        size_MB_str << size_MB;\n\n        args.push_back(string2vector(\"-m\"));\n        args.push_back(string2vector(size_MB_str.str()));\n    } else if (!dpdk_pmd) {\n        args.push_back(string2vector(\"--no-huge\"));\n    }\n#ifdef HAVE_OSV\n    args.push_back(string2vector(\"--no-shconf\"));\n#endif\n\n    std::vector<char*> cargs;\n\n    for (auto&& a: args) {\n        cargs.push_back(a.data());\n    }\n    /* initialise the EAL for all */\n    int ret = rte_eal_init(cargs.size(), cargs.data());\n    if (ret < 0) {\n        rte_exit(EXIT_FAILURE, \"Cannot init EAL\\n\");\n    }\n\n    initialized = true;\n}\n\nuint32_t __attribute__((weak)) qp_mempool_obj_size(bool hugetlbfs_membackend)\n{\n    return 0;\n}\n\nsize_t eal::mem_size(int num_cpus, bool hugetlbfs_membackend)\n{\n    size_t memsize = 0;\n    //\n    // PMD mempool memory:\n    //\n    // We don't know what is going to be our networking configuration so we\n    // assume there is going to be a queue per-CPU.\n    //\n    memsize += num_cpus * qp_mempool_obj_size(hugetlbfs_membackend);\n\n    // Plus we'll give a DPDK 64MB for \"other stuff\".\n    memsize += (64UL << 20);\n\n    return memsize;\n}\n\n} // namespace dpdk\n\n}\n\n#endif // SEASTAR_HAVE_DPDK\n\n// Path: src/core/sharded.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2018 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <boost/range/irange.hpp>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/sharded.hh>\n#include <seastar/core/loop.hh>\n#endif\n\nnamespace seastar {\n\nnamespace internal {\n\n\n\nfuture<>\nsharded_parallel_for_each(unsigned nr_shards, on_each_shard_func on_each_shard) noexcept(std::is_nothrow_move_constructible_v<on_each_shard_func>) {\n    return parallel_for_each(boost::irange<unsigned>(0, nr_shards), std::move(on_each_shard));\n}\n\n}\n\n}\n\n// Path: src/core/uname.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * Copyright (C) 2019 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <memory>\n#include <optional>\n#include <regex>\n#include <boost/algorithm/cxx11/any_of.hpp>\n#include <sys/utsname.h>\n#include <iostream>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/internal/uname.hh>\n#endif\n\nnamespace seastar {\n\nnamespace internal {\n\nint uname_t::component_count() const {\n    if (distro_patch) {\n        return 5;\n    }\n    if (subsublevel) {\n        return 4;\n    }\n    if (sublevel) {\n        return 3;\n    }\n    return 2;\n}\n\nbool uname_t::has_distro_extra(std::string extra) const {\n    return distro_extra.find(extra) != std::string::npos;\n}\n\n// Can't use optional compares, C++17 only\nstatic int cmp(const std::optional<int>& u1, const std::optional<int>& u2) {\n    return int(u1.value_or(0) - u2.value_or(0));\n}\n\nbool uname_t::same_as_or_descendant_of(const uname_t& x) const {\n    if (version < x.version) {\n        return false; // 4.2 vs. 5.1\n    }\n    if (version == x.version && patchlevel < x.patchlevel) {\n        return false; // 4.0 vs 4.1\n    }\n    if (!has_distro_extra(x.distro_extra)) {\n        return false;\n    }\n    switch (x.component_count()) {\n    case 5:\n        return version == x.version\n                && patchlevel == x.patchlevel\n                && cmp(sublevel, x.sublevel) == 0\n                && cmp(subsublevel, x.subsublevel) == 0\n                && cmp(distro_patch, x.distro_patch) >= 0;\n    case 4:\n        return version == x.version\n                && patchlevel == x.patchlevel\n                && cmp(sublevel, x.sublevel) == 0\n                && cmp(subsublevel, x.subsublevel) >= 0;\n    case 3:\n        return version == x.version\n                && patchlevel == x.patchlevel\n                && cmp(sublevel, x.sublevel) >= 0;\n    case 2:\n        return true;\n    default:\n        return false;\n    }\n}\n\nuname_t parse_uname(const char* u) {\n    static std::regex re(R\"XX((\\d+)\\.(\\d+)(?:\\.(\\d+)(?:\\.(\\d+))?)?(?:-(\\d*)(.+))?)XX\");\n    std::cmatch m;\n    if (std::regex_match(u, m, re)) {\n        auto num = [] (std::csub_match sm) -> std::optional<int> {\n            if (sm.length() > 0) {\n                return std::atoi(sm.str().c_str());\n            } else {\n                return std::nullopt;\n            }\n        };\n        return uname_t{*num(m[1]), *num(m[2]), num(m[3]), num(m[4]), num(m[5]), m[6].str()};\n    } else {\n        return uname_t{0, 0, std::nullopt, std::nullopt, std::nullopt, \"\"};\n    }\n}\n\n\nbool uname_t::whitelisted(std::initializer_list<const char*> wl) const {\n    return boost::algorithm::any_of(wl, [this] (const char* v) {\n        return same_as_or_descendant_of(parse_uname(v));\n    });\n}\n\nstd::ostream& operator<<(std::ostream& os, const uname_t& u) {\n    os << u.version << \".\" << u.patchlevel;\n    if (u.sublevel) {\n        os << \".\" << *u.sublevel;\n    }\n    if (u.subsublevel) {\n        os << \".\" << *u.subsublevel;\n    }\n    if (u.distro_patch || !u.distro_extra.empty()) {\n        os << \"-\";\n    }\n    if (u.distro_patch) {\n        os << *u.distro_patch;\n    }\n    os << u.distro_extra;\n    return os;\n}\n\n\nuname_t kernel_uname() {\n    struct ::utsname buf;\n    ::uname(&buf);\n    return parse_uname(buf.release);\n}\n\n}\n}\n\n// Path: src/core/app-template.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <fstream>\n#include <cstdlib>\n#include <chrono>\n#include <iostream>\n#include <fmt/format.h>\n#include <boost/program_options.hpp>\n#include <boost/make_shared.hpp>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/app-template.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/alien.hh>\n#include <seastar/core/scollectd.hh>\n#include <seastar/core/metrics_api.hh>\n#include <seastar/core/print.hh>\n#include <seastar/util/log.hh>\n#include <seastar/util/log-cli.hh>\n#include <seastar/net/native-stack.hh>\n#include \"program_options.hh\"\n#endif\n\nnamespace seastar {\n\nnamespace bpo = boost::program_options;\n\nusing namespace std::chrono_literals;\n\nstatic\napp_template::seastar_options\nseastar_options_from_config(app_template::config cfg) {\n    app_template::seastar_options opts;\n    opts.name = std::move(cfg.name);\n    opts.description = std::move(cfg.description);\n    opts.auto_handle_sigint_sigterm = std::move(cfg.auto_handle_sigint_sigterm);\n    opts.reactor_opts.task_quota_ms.set_default_value(cfg.default_task_quota / 1ms);\n    opts.reactor_opts.max_networking_io_control_blocks.set_default_value(cfg.max_networking_aio_io_control_blocks);\n    opts.smp_opts.reserve_additional_memory_per_shard = cfg.reserve_additional_memory_per_shard;\n    return opts;\n}\n\napp_template::seastar_options::seastar_options()\n    : program_options::option_group(nullptr, \"seastar\")\n    , reactor_opts(this)\n    , metrics_opts(this)\n    , smp_opts(this)\n    , scollectd_opts(this)\n    , log_opts(this)\n{\n}\n\napp_template::app_template(app_template::seastar_options opts)\n    : _alien(std::make_unique<alien::instance>())\n    , _smp(std::make_shared<smp>(*_alien))\n    , _opts(std::move(opts))\n    , _app_opts(_opts.name + \" options\")\n    , _conf_reader(get_default_configuration_reader()) {\n\n        if (!alien::internal::default_instance) {\n            alien::internal::default_instance = _alien.get();\n        }\n        _app_opts.add_options()\n                (\"help,h\", \"show help message\")\n                ;\n        _app_opts.add_options()\n                (\"help-seastar\", \"show help message about seastar options\")\n                ;\n        _app_opts.add_options()\n                (\"help-loggers\", \"print a list of logger names and exit\")\n                ;\n\n        {\n            program_options::options_description_building_visitor visitor;\n            _opts.describe(visitor);\n            _opts_conf_file.add(std::move(visitor).get_options_description());\n        }\n\n        _seastar_opts.add(_opts_conf_file);\n}\n\napp_template::app_template(app_template::config cfg)\n    : app_template(seastar_options_from_config(std::move(cfg)))\n{\n}\n\napp_template::~app_template() = default;\n\nconst app_template::seastar_options& app_template::options() const {\n    return _opts;\n}\n\napp_template::configuration_reader app_template::get_default_configuration_reader() {\n    return [this] (bpo::variables_map& configuration) {\n        auto home = std::getenv(\"HOME\");\n        if (home) {\n            std::ifstream ifs(std::string(home) + \"/.config/seastar/seastar.conf\");\n            if (ifs) {\n                bpo::store(bpo::parse_config_file(ifs, _opts_conf_file), configuration);\n            }\n            std::ifstream ifs_io(std::string(home) + \"/.config/seastar/io.conf\");\n            if (ifs_io) {\n                bpo::store(bpo::parse_config_file(ifs_io, _opts_conf_file), configuration);\n            }\n        }\n    };\n}\n\nvoid app_template::set_configuration_reader(configuration_reader conf_reader) {\n    _conf_reader = conf_reader;\n}\n\nboost::program_options::options_description& app_template::get_options_description() {\n    return _app_opts;\n}\n\nboost::program_options::options_description& app_template::get_conf_file_options_description() {\n    return _opts_conf_file;\n}\n\nboost::program_options::options_description_easy_init\napp_template::add_options() {\n    return _app_opts.add_options();\n}\n\nvoid\napp_template::add_positional_options(std::initializer_list<positional_option> options) {\n    for (auto&& o : options) {\n        _app_opts.add(boost::make_shared<bpo::option_description>(o.name, o.value_semantic, o.help));\n        _pos_opts.add(o.name, o.max_count);\n    }\n}\n\n\nbpo::variables_map&\napp_template::configuration() {\n    return *_configuration;\n}\n\nint\napp_template::run(int ac, char ** av, std::function<future<int> ()>&& func) noexcept {\n    return run_deprecated(ac, av, [func = std::move(func)] () mutable {\n        auto func_done = make_lw_shared<promise<>>();\n        engine().at_exit([func_done] { return func_done->get_future(); });\n        // No need to wait for this future.\n        // func's returned exit_code is communicated via engine().exit()\n        (void)futurize_invoke(func).finally([func_done] {\n            func_done->set_value();\n        }).then([] (int exit_code) {\n            return engine().exit(exit_code);\n        }).or_terminate();\n    });\n}\n\nint\napp_template::run(int ac, char ** av, std::function<future<> ()>&& func) noexcept {\n    return run(ac, av, [func = std::move(func)] {\n        return func().then([] () {\n            return 0;\n        });\n    });\n}\n\nint\napp_template::run_deprecated(int ac, char ** av, std::function<void ()>&& func) noexcept {\n#ifdef SEASTAR_DEBUG\n    fmt::print(std::cerr, \"WARNING: debug mode. Not for benchmarking or production\\n\");\n#endif\n    boost::program_options::options_description all_opts;\n    all_opts.add(_app_opts);\n    all_opts.add(_seastar_opts);\n\n    bpo::variables_map configuration;\n    try {\n        bpo::store(bpo::command_line_parser(ac, av)\n                    .options(all_opts)\n                    .positional(_pos_opts)\n                    .run()\n            , configuration);\n        _conf_reader(configuration);\n    } catch (bpo::error& e) {\n        fmt::print(\"error: {}\\n\\nTry --help.\\n\", e.what());\n        return 2;\n    }\n    if (configuration.count(\"help\")) {\n        if (!_opts.description.empty()) {\n            std::cout << _opts.description << \"\\n\";\n        }\n        std::cout << _app_opts << \"\\n\";\n        return 1;\n    }\n    if (configuration.count(\"help-seastar\")) {\n        std::cout << _seastar_opts << \"\\n\";\n        return 1;\n    }\n    if (configuration.count(\"help-loggers\")) {\n        log_cli::print_available_loggers(std::cout);\n        return 1;\n    }\n\n    try {\n        bpo::notify(configuration);\n    } catch (const bpo::error& ex) {\n        std::cout << ex.what() << std::endl;\n        return 1;\n    }\n\n    {\n        program_options::variables_map_extracting_visitor visitor(configuration);\n        _opts.mutate(visitor);\n    }\n    _opts.reactor_opts._argv0 = std::string(av[0]);\n    _opts.reactor_opts._auto_handle_sigint_sigterm = _opts.auto_handle_sigint_sigterm;\n    if (auto* native_stack = dynamic_cast<net::native_stack_options*>(_opts.reactor_opts.network_stack.get_selected_candidate_opts())) {\n        native_stack->_hugepages = _opts.smp_opts.hugepages;\n    }\n\n    // Needs to be before `smp::configure()`.\n    try {\n        apply_logging_settings(log_cli::extract_settings(_opts.log_opts));\n    } catch (const std::runtime_error& exn) {\n        std::cout << \"logging configuration error: \" << exn.what() << '\\n';\n        return 1;\n    }\n\n    try {\n        _smp->configure(_opts.smp_opts, _opts.reactor_opts);\n    } catch (...) {\n        std::cerr << \"Could not initialize seastar: \" << std::current_exception() << std::endl;\n        return 1;\n    }\n    _configuration = {std::move(configuration)};\n    // No need to wait for this future.\n    // func is waited on via engine().run()\n    (void)engine().when_started().then([this] {\n        return seastar::metrics::configure(_opts.metrics_opts).then([this] {\n            // set scollectd use the metrics configuration, so the later\n            // need to be set first\n            scollectd::configure( _opts.scollectd_opts);\n        });\n    }).then(\n        std::move(func)\n    ).then_wrapped([] (auto&& f) {\n        try {\n            f.get();\n        } catch (std::exception& ex) {\n            std::cout << \"program failed with uncaught exception: \" << ex.what() << \"\\n\";\n            engine().exit(1);\n        }\n    });\n    auto exit_code = engine().run();\n    _smp->cleanup();\n    return exit_code;\n}\n\n}\n\n// Path: src/core/program_options.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2021 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cstdlib>\n#include <string>\n#include <stdexcept>\n#include <boost/program_options.hpp>\n#include <boost/type.hpp>\n#include <fmt/format.h>\nmodule seastar;\n#else\n#include \"core/program_options.hh\"\n\n#include <seastar/util/log-cli.hh>\n#include <seastar/util/memory_diagnostics.hh>\n#include <seastar/core/reactor_config.hh>\n#include <seastar/core/resource.hh>\n#include <seastar/core/smp.hh>\n#endif\n\nnamespace seastar::program_options {\n\nnamespace {\n\nconst char* to_string(memory::alloc_failure_kind val) {\n    switch (val) {\n        case memory::alloc_failure_kind::none: return \"none\";\n        case memory::alloc_failure_kind::critical: return \"critical\";\n        case memory::alloc_failure_kind::all: return \"all\";\n    }\n    std::abort();\n}\n\nconst char* to_string(log_level val) {\n    switch (val) {\n        case log_level::error: return \"error\";\n        case log_level::warn: return \"warn\";\n        case log_level::info: return \"info\";\n        case log_level::debug: return \"debug\";\n        case log_level::trace: return \"trace\";\n    }\n    std::abort();\n}\n\nconst char* to_string(logger_timestamp_style val) {\n    switch (val) {\n        case logger_timestamp_style::none: return \"none\";\n        case logger_timestamp_style::boot: return \"boot\";\n        case logger_timestamp_style::real: return \"real\";\n    }\n    std::abort();\n}\n\nconst char* to_string(logger_ostream_type val) {\n    switch (val) {\n        case logger_ostream_type::none: return \"none\";\n        case logger_ostream_type::cout: return \"stdout\";\n        case logger_ostream_type::cerr: return \"stderr\";\n    }\n    std::abort();\n}\n\nmemory::alloc_failure_kind from_string(const std::string& val, boost::type<memory::alloc_failure_kind>) {\n    if (val == \"none\") {\n        return memory::alloc_failure_kind::none;\n    } else if (val == \"critical\") {\n        return memory::alloc_failure_kind::critical;\n    } else if (val == \"all\") {\n        return memory::alloc_failure_kind::all;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum memory::alloc_failure_kind: {}\", val));\n}\n\nlog_level from_string(const std::string& val, boost::type<log_level>) {\n    if (val == \"error\") {\n        return log_level::error;\n    } else if (val == \"warn\") {\n        return log_level::warn;\n    } else if (val == \"info\") {\n        return log_level::info;\n    } else if (val == \"debug\") {\n        return log_level::debug;\n    } else if (val == \"trace\") {\n        return log_level::trace;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum log_level: {}\", val));\n}\n\nlogger_timestamp_style from_string(const std::string& val, boost::type<logger_timestamp_style>) {\n    if (val == \"none\") {\n        return logger_timestamp_style::none;\n    } else if (val == \"boot\") {\n        return logger_timestamp_style::boot;\n    } else if (val == \"real\") {\n        return logger_timestamp_style::real;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum logger_timestamp_style: {}\", val));\n}\n\nlogger_ostream_type from_string(const std::string& val, boost::type<logger_ostream_type>) {\n    if (val == \"none\") {\n        return logger_ostream_type::none;\n    } else if (val == \"stdout\") {\n        return logger_ostream_type::cout;\n    } else if (val == \"stderr\") {\n        return logger_ostream_type::cerr;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum logger_ostream_type: {}\", val));\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const std::string& name, const std::string& description, const Type& default_value) {\n    opts.add_options()(name.c_str(), boost::program_options::value<Type>()->default_value(default_value), description.c_str());\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d, const Type& default_value) {\n    describe_value(opts, d.name, d.description, default_value);\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const std::string& name, const std::string& description) {\n    opts.add_options()(name.c_str(), boost::program_options::value<Type>(), description.c_str());\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d) {\n    describe_value<Type>(opts, d.name, d.description);\n}\n\ntemplate <typename Type>\nvoid describe_value_maybe_default(bpo::options_description& opts, const std::string& name, const std::string& description, const Type* default_value) {\n    if (default_value) {\n        describe_value(opts, name, description, *default_value);\n    } else {\n        describe_value<Type>(opts, name, description);\n    }\n}\n\ntemplate <typename Type>\nvoid describe_value_maybe_default(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d, const Type* default_value) {\n    describe_value_maybe_default(opts, d.name, d.description, default_value);\n}\n\ntemplate <typename Enum>\nvoid describe_enum_value(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d, const Enum* default_value) {\n    if (default_value) {\n        opts.add_options()(d.name.c_str(), boost::program_options::value<std::string>()->default_value(to_string(*default_value)), d.description.c_str());\n    } else {\n        opts.add_options()(d.name.c_str(), boost::program_options::value<std::string>(), d.description.c_str());\n    }\n}\n\ntemplate <typename T>\nbool extract_value(const bpo::variables_map& values, const std::string& current_name, T& val) {\n    auto it = values.find(current_name);\n    if (it == values.end() || it->second.defaulted()) {\n        return false;\n    }\n    val = it->second.as<T>();\n    return true;\n}\n\ntemplate <typename T>\nbool extract_enum_value(const bpo::variables_map& values, const std::string& current_name, T& val) {\n    auto it = values.find(current_name);\n    if (it == values.end() || it->second.defaulted()) {\n        return false;\n    }\n    val = from_string(it->second.as<std::string>(), boost::type<T>{});\n    return true;\n}\n\n} // anonymous namespace\n\nbool options_description_building_visitor::visit_group_start(const std::string& name, bool used) {\n    _groups.push({name, bpo::options_description(name.c_str()), used});\n    return used;\n}\nvoid options_description_building_visitor::visit_group_end() {\n    if (_groups.size() == 1) {\n        return;\n    }\n    auto grp = std::move(_groups.top());\n    _groups.pop();\n    if (grp.used && grp.values) {\n        _groups.top().description.add(std::move(grp.description));\n    }\n}\n\nbool options_description_building_visitor::visit_value_metadata(const std::string& name, const std::string& description, bool used) {\n    if (!used) {\n        return false;\n    }\n    ++_groups.top().values;\n    _current_metadata.emplace(value_metadata{name, description});\n    return true;\n}\n\nvoid options_description_building_visitor::visit_value() {\n    _groups.top().description.add_options()(_current_metadata->name.c_str(), _current_metadata->description.c_str());\n}\n\nvoid options_description_building_visitor::visit_value(const bool* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const int* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const unsigned* default_value) {\n    auto name = _current_metadata->name;\n    if (_current_metadata->name == \"smp\") {\n        name = \"smp,c\";\n    }\n    describe_value_maybe_default(_groups.top().description, name, _current_metadata->description, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const float* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const double* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const std::string* default_value) {\n    auto name = _current_metadata->name;\n    if (_current_metadata->name == \"memory\") {\n        name = \"memory,m\";\n    }\n    describe_value_maybe_default(_groups.top().description, name, _current_metadata->description, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const std::set<unsigned>*) {\n    describe_value<std::string>(_groups.top().description, *_current_metadata);\n}\n\nvoid options_description_building_visitor::visit_value(const memory::alloc_failure_kind* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const log_level* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const logger_timestamp_style* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const logger_ostream_type* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const std::unordered_map<sstring, log_level>*) {\n    describe_value<std::vector<std::string>>(_groups.top().description, *_current_metadata);\n}\n\nvoid options_description_building_visitor::visit_selection_value(const std::vector<std::string>& candidates, const std::size_t* selected_candidate) {\n    if (selected_candidate) {\n        describe_value<std::string>(_groups.top().description, *_current_metadata, candidates.at(*selected_candidate));\n    } else {\n        describe_value<std::string>(_groups.top().description, *_current_metadata);\n    }\n}\n\nvariables_map_extracting_visitor::variables_map_extracting_visitor(const bpo::variables_map& values) : _values(values) {\n}\n\nbool variables_map_extracting_visitor::visit_group_start(const std::string& name, bool used) {\n    return used;\n}\n\nvoid variables_map_extracting_visitor::visit_group_end() {\n}\n\nbool variables_map_extracting_visitor::visit_value_metadata(const std::string& name, bool used) {\n    if (used) {\n        _current_name = &name;\n        return true;\n    } else {\n        _current_name = nullptr;\n        return false;\n    }\n}\n\nbool variables_map_extracting_visitor::visit_value() {\n    return _values.count(*_current_name);\n}\n\nbool variables_map_extracting_visitor::visit_value(bool& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(int& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(unsigned& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(float& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(double& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(std::string& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(std::set<unsigned>& val) {\n    std::string raw_val;\n    if (!extract_value(_values, *_current_name, raw_val)) {\n        return false;\n    }\n    if (auto parsed_cpu_set = resource::parse_cpuset(raw_val)) {\n        val = std::move(*parsed_cpu_set);\n        return true;\n    }\n    throw std::invalid_argument(fmt::format(\"invalid value for option {}: failed to parse cpuset: {}\", *_current_name, raw_val));\n}\n\nbool variables_map_extracting_visitor::visit_value(log_level& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(logger_timestamp_style& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(logger_ostream_type& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(memory::alloc_failure_kind& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(std::unordered_map<sstring, log_level>& val) {\n    std::vector<std::string> raw_val;\n    if (!extract_value(_values, *_current_name, raw_val)) {\n        return false;\n    }\n    for (const auto& e : raw_val) {\n        log_cli::parse_map_associations(e, [&val] (std::string k, std::string v) { val[std::move(k)] = log_cli::parse_log_level(v); });\n    }\n    return !val.empty();\n}\n\nbool variables_map_extracting_visitor::visit_selection_value(const std::vector<std::string>& candidates, std::size_t& selected_candidate) {\n    std::string candidate_name;\n    if (!extract_value(_values, *_current_name, candidate_name)) {\n        return false;\n    }\n    auto it = std::find(candidates.begin(), candidates.end(), candidate_name);\n    if (it == candidates.end()) {\n        throw std::invalid_argument(fmt::format(\"invalid value for option {}: selected candidate doesn't exist: {}\", *_current_name, candidate_name));\n    }\n    selected_candidate = it - candidates.begin();\n    return true;\n}\n\n} // namespace seastar::program_options\n\n// Path: src/core/future-util.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2017 ScyllaDB\n */\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cstddef>\n#include <exception>\n#include <iosfwd>\n#include <memory>\n#include <optional>\n#include <utility>\n#include <vector>\nmodule seastar;\n#else\n#include <seastar/core/future-util.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/sleep.hh>\n#include <seastar/core/print.hh>\n#include <seastar/core/semaphore.hh>\n#endif\n\nnamespace seastar {\n\nparallel_for_each_state::parallel_for_each_state(size_t n) {\n    _incomplete.reserve(n);\n}\n\nfuture<> parallel_for_each_state::get_future() {\n    auto ret = _result.get_future();\n    wait_for_one();\n    return ret;\n}\n\nvoid parallel_for_each_state::add_future(future<>&& f) {\n    _incomplete.push_back(std::move(f));\n}\n\nvoid parallel_for_each_state::wait_for_one() noexcept {\n    // Process from back to front, on the assumption that the front\n    // futures are likely to complete earlier than the back futures.\n    // If that's indeed the case, then the front futures will be\n    // available and we won't have to wait for them.\n\n    // Skip over futures that happen to be complete already.\n    while (!_incomplete.empty() && _incomplete.back().available()) {\n        if (_incomplete.back().failed()) {\n            _ex = _incomplete.back().get_exception();\n        }\n        _incomplete.pop_back();\n    }\n\n    // If there's an incompelete future, wait for it.\n    if (!_incomplete.empty()) {\n        internal::set_callback(std::move(_incomplete.back()), static_cast<continuation_base<>*>(this));\n        // This future's state will be collected in run_and_dispose(), so we can drop it.\n        _incomplete.pop_back();\n        return;\n    }\n\n    // Everything completed, report a result.\n    if (__builtin_expect(bool(_ex), false)) {\n        _result.set_exception(std::move(_ex));\n    } else {\n        _result.set_value();\n    }\n    delete this;\n}\n\nvoid parallel_for_each_state::run_and_dispose() noexcept {\n    if (_state.failed()) {\n        _ex = std::move(_state).get_exception();\n    }\n    _state = {};\n    wait_for_one();\n}\n\ntemplate <typename Clock>\nfuture<> sleep_abortable(typename Clock::duration dur) {\n    return engine().wait_for_stop(dur).then([] {\n        throw sleep_aborted();\n    }).handle_exception([] (std::exception_ptr ep) {\n        try {\n            std::rethrow_exception(ep);\n        } catch(condition_variable_timed_out&) {};\n    });\n}\n\ntemplate future<> sleep_abortable<steady_clock_type>(typename steady_clock_type::duration);\ntemplate future<> sleep_abortable<lowres_clock>(typename lowres_clock::duration);\n\ntemplate <typename Clock>\nfuture<> sleep_abortable(typename Clock::duration dur, abort_source& as) {\n    struct sleeper {\n        promise<> done;\n        timer<Clock> tmr;\n        abort_source::subscription sc;\n\n        sleeper(typename Clock::duration dur, abort_source& as)\n                : tmr([this] { done.set_value(); }) {\n            auto sc_opt = as.subscribe([this] (const std::optional<std::exception_ptr>& opt_ex) noexcept {\n                if (tmr.cancel()) {\n                    done.set_exception(opt_ex.value_or(std::make_exception_ptr(sleep_aborted())));\n                }\n            });\n            if (sc_opt) {\n                sc = std::move(*sc_opt);\n                tmr.arm(dur);\n            } else {\n                done.set_exception(sleep_aborted());\n            }\n        }\n    };\n    //FIXME: Use do_with() after #373\n    auto s = std::make_unique<sleeper>(dur, as);\n    auto fut = s->done.get_future();\n    return fut.finally([s = std::move(s)] { });\n}\n\ntemplate future<> sleep_abortable<steady_clock_type>(typename steady_clock_type::duration, abort_source&);\ntemplate future<> sleep_abortable<lowres_clock>(typename lowres_clock::duration, abort_source&);\ntemplate future<> sleep_abortable<manual_clock>(typename manual_clock::duration, abort_source&);\n\n}\n\n// Path: src/core/future.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2020 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cassert>\n#include <exception>\n#include <tuple>\n#include <type_traits>\n#include <utility>\nmodule seastar;\n#else\n#include <seastar/core/future.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/thread.hh>\n#include <seastar/core/report_exception.hh>\n#include <seastar/util/backtrace.hh>\n#endif\n\nnamespace seastar {\n\n// We can't test future_state_base directly because its private\n// destructor is protected.\nstatic_assert(std::is_nothrow_move_constructible_v<future_state<std::tuple<int>>>,\n              \"future_state's move constructor must not throw\");\n\nstatic_assert(sizeof(future_state<std::tuple<>>) <= 8, \"future_state<std::tuple<>> is too large\");\nstatic_assert(sizeof(future_state<std::tuple<long>>) <= 16, \"future_state<std::tuple<long>> is too large\");\nstatic_assert(future_state<std::tuple<>>::has_trivial_move_and_destroy, \"future_state<std::tuple<>> not trivial\");\nstatic_assert(future_state<long>::has_trivial_move_and_destroy, \"future_state<long> not trivial\");\n\n// We need to be able to move and copy std::exception_ptr in and out\n// of future/promise/continuations without that producing a new\n// exception.\nstatic_assert(std::is_nothrow_copy_constructible_v<std::exception_ptr>,\n    \"std::exception_ptr's copy constructor must not throw\");\nstatic_assert(std::is_nothrow_move_constructible_v<std::exception_ptr>,\n    \"std::exception_ptr's move constructor must not throw\");\n\nnamespace internal {\n\nstatic_assert(std::is_empty_v<uninitialized_wrapper<std::tuple<>>>, \"This should still be empty\");\n\nvoid promise_base::move_it(promise_base&& x) noexcept {\n    // Don't use std::exchange to make sure x's values are nulled even\n    // if &x == this.\n    _task = x._task;\n    x._task = nullptr;\n#ifdef SEASTAR_DEBUG_PROMISE\n    _task_shard = x._task_shard;\n#endif\n    _state = x._state;\n    x._state = nullptr;\n    _future = x._future;\n    if (auto* fut = _future) {\n        fut->detach_promise();\n        fut->_promise = this;\n    }\n}\n\nstatic void set_to_broken_promise(future_state_base& state) noexcept {\n    try {\n        // Constructing broken_promise may throw (std::logic_error ctor is not noexcept).\n        state.set_exception(std::make_exception_ptr(broken_promise{}));\n    } catch (...) {\n        state.set_exception(std::current_exception());\n    }\n}\n\npromise_base::promise_base(promise_base&& x) noexcept {\n    move_it(std::move(x));\n}\n\nvoid promise_base::clear() noexcept {\n    if (__builtin_expect(bool(_task), false)) {\n        assert(_state && !_state->available());\n        set_to_broken_promise(*_state);\n        ::seastar::schedule(std::exchange(_task, nullptr));\n    }\n    if (_future) {\n        assert(_state);\n        if (!_state->available()) {\n            set_to_broken_promise(*_state);\n        }\n        _future->detach_promise();\n    }\n}\n\npromise_base& promise_base::operator=(promise_base&& x) noexcept {\n    clear();\n    move_it(std::move(x));\n    return *this;\n}\n\nvoid promise_base::set_to_current_exception() noexcept {\n    set_exception(std::current_exception());\n}\n\n#ifdef SEASTAR_DEBUG_PROMISE\n\nvoid promise_base::assert_task_shard() const noexcept {\n    if (_task_shard >= 0 && static_cast<shard_id>(_task_shard) != this_shard_id()) {\n        on_fatal_internal_error(seastar_logger, format(\"Promise task was set on shard {} but made ready on shard {}\", _task_shard, this_shard_id()));\n    }\n}\n\n#endif\n\ntemplate <promise_base::urgent Urgent>\nvoid promise_base::make_ready() noexcept {\n    if (_task) {\n        assert_task_shard();\n        if (Urgent == urgent::yes) {\n            ::seastar::schedule_urgent(std::exchange(_task, nullptr));\n        } else {\n            ::seastar::schedule(std::exchange(_task, nullptr));\n        }\n    }\n}\n\ntemplate void promise_base::make_ready<promise_base::urgent::no>() noexcept;\ntemplate void promise_base::make_ready<promise_base::urgent::yes>() noexcept;\n}\n\ntemplate\nfuture<void> current_exception_as_future() noexcept;\n\n/**\n * engine_exit() exits the reactor. It should be given a pointer to the\n * exception which prompted this exit - or a null pointer if the exit\n * request was not caused by any exception.\n */\nvoid engine_exit(std::exception_ptr eptr) {\n    if (!eptr) {\n        engine().exit(0);\n        return;\n    }\n    report_exception(\"Exiting on unhandled exception\", eptr);\n    engine().exit(1);\n}\n\nbroken_promise::broken_promise() : logic_error(\"broken promise\") { }\n\nfuture_state_base::future_state_base(current_exception_future_marker) noexcept\n    : future_state_base(std::current_exception()) { }\n\nvoid future_state_base::ignore() noexcept {\n    switch (_u.st) {\n    case state::invalid:\n    case state::future:\n    case state::result_unavailable:\n        assert(0 && \"invalid state for ignore\");\n    case state::result:\n        _u.st = state::result_unavailable;\n        break;\n    default:\n        // Ignore the exception\n        _u.take_exception();\n    }\n}\n\nnested_exception::nested_exception(std::exception_ptr inner, std::exception_ptr outer) noexcept\n    : inner(std::move(inner)), outer(std::move(outer)) {}\n\nnested_exception::nested_exception(nested_exception&&) noexcept = default;\n\nnested_exception::nested_exception(const nested_exception&) noexcept = default;\n\nconst char* nested_exception::what() const noexcept {\n    return \"seastar::nested_exception\";\n}\n\n[[noreturn]] void nested_exception::rethrow_nested() const {\n    std::rethrow_exception(outer);\n}\n\nstatic std::exception_ptr make_nested(std::exception_ptr&& inner, future_state_base&& old) noexcept {\n    std::exception_ptr outer = std::move(old).get_exception();\n    nested_exception nested{std::move(inner), std::move(outer)};\n    return std::make_exception_ptr<nested_exception>(std::move(nested));\n}\n\nfuture_state_base::future_state_base(nested_exception_marker, future_state_base&& n, future_state_base&& old) noexcept {\n    std::exception_ptr inner = std::move(n).get_exception();\n    if (!old.failed()) {\n        new (this) future_state_base(std::move(inner));\n    } else {\n        new (this) future_state_base(make_nested(std::move(inner), std::move(old)));\n    }\n}\n\nfuture_state_base::future_state_base(nested_exception_marker, future_state_base&& old) noexcept {\n    if (!old.failed()) {\n        new (this) future_state_base(current_exception_future_marker());\n        return;\n    } else {\n        new (this) future_state_base(make_nested(std::current_exception(), std::move(old)));\n    }\n}\n\nvoid future_state_base::rethrow_exception() && {\n    // Move ex out so future::~future() knows we've handled it\n    std::rethrow_exception(std::move(*this).get_exception());\n}\n\nvoid future_state_base::rethrow_exception() const& {\n    std::rethrow_exception(_u.ex);\n}\n\nvoid report_failed_future(const std::exception_ptr& eptr) noexcept {\n    ++engine()._abandoned_failed_futures;\n    seastar_logger.warn(\"Exceptional future ignored: {}, backtrace: {}\", eptr, current_backtrace());\n}\n\nvoid report_failed_future(const future_state_base& state) noexcept {\n    report_failed_future(state._u.ex);\n}\n\nvoid report_failed_future(future_state_base::any&& state) noexcept {\n    report_failed_future(std::move(state).take_exception());\n}\n\nvoid reactor::test::with_allow_abandoned_failed_futures(unsigned count, noncopyable_function<void ()> func) {\n    auto before = engine()._abandoned_failed_futures;\n    auto old_level = seastar_logger.level();\n    seastar_logger.set_level(log_level::error);\n    func();\n    auto after = engine()._abandoned_failed_futures;\n    assert(after - before == count);\n    engine()._abandoned_failed_futures = before;\n    seastar_logger.set_level(old_level);\n}\n\nnamespace {\nclass thread_wake_task final : public task {\n    thread_context* _thread;\npublic:\n    thread_wake_task(thread_context* thread) noexcept : _thread(thread) {}\n    virtual void run_and_dispose() noexcept override {\n        thread_impl::switch_in(_thread);\n        // no need to delete, since this is always allocated on\n        // _thread's stack.\n    }\n    /// Returns the task which is waiting for this thread to be done, or nullptr.\n    virtual task* waiting_task() noexcept override {\n        return _thread->waiting_task();\n    }\n};\n}\n\nvoid internal::future_base::do_wait() noexcept {\n    auto thread = thread_impl::get();\n    assert(thread);\n    thread_wake_task wake_task{thread};\n    wake_task.make_backtrace();\n    _promise->set_task(&wake_task);\n    thread_impl::switch_out(thread);\n}\n\n#ifdef SEASTAR_COROUTINES_ENABLED\nvoid internal::future_base::set_coroutine(task& coroutine) noexcept {\n    assert(_promise);\n    _promise->set_task(&coroutine);\n}\n#endif\n\n}\n\n// Path: src/core/resource.cc\n\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n...\n", "needle_token_start": 2313, "needle_token_end": 2410, "code_context_ntokens": 16017, "output": ["```cpp\nfuture<>\nsharded_parallel_for_each(unsigned nr_shards, on_each_shard_func on_each_shard) noexcept(std::is_nothrow_move_constructible_v<on_each_shard_func>) {\n    return parallel_for_each(boost::irange<unsigned>(0, nr_shards), std::move(on_each_shard));\n}\n```"]}
{"repo": "scylladb/seastar", "name": "enable_abort_on_allocation_failure", "language": "cpp", "path": "src/core/memory.cc", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To configure the system to terminate the process when memory allocation fails.\n2. **Input**: None.\n3. **Output**: None.\n4. **Procedure**: This function calls another function that sets a flag to true, indicating that the system should abort when it cannot allocate memory.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/core/file.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2019 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <algorithm>\n#include <atomic>\n#ifdef SEASTAR_COROUTINES_ENABLED\n#include <coroutine>\n#endif\n#include <deque>\n#include <filesystem>\n#include <functional>\n#include <memory>\n#include <optional>\n#include <vector>\n\n#define __user /* empty */  // for xfs includes, below\n\n#include <sys/syscall.h>\n#include <dirent.h>\n#include <linux/types.h> // for xfs, below\n#include <linux/fs.h> // BLKBSZGET\n#include <linux/major.h>\n#include <sys/ioctl.h>\n#include <unistd.h>\n#include <fcntl.h>\n#include <xfs/linux.h>\n/*\n * With package xfsprogs-devel >= 5.14.1, `fallthrough` has defined to\n * fix compilation warning in header <xfs/linux.h>,\n * (see: https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git/commit/?id=df9c7d8d8f3ed0785ed83e7fd0c7ddc92cbfbe15)\n * There is a confliction with c++ keyword `fallthrough`, so undefine fallthrough here.\n */\n#undef fallthrough   \n#define min min    /* prevent xfs.h from defining min() as a macro */\n#include <xfs/xfs.h>\n#undef min\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/internal/read_state.hh>\n#include <seastar/core/internal/uname.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/file.hh>\n#include <seastar/core/report_exception.hh>\n#include <seastar/core/linux-aio.hh>\n#include <seastar/util/later.hh>\n#include <seastar/util/internal/magic.hh>\n#include <seastar/util/internal/iovec_utils.hh>\n#include <seastar/core/io_queue.hh>\n#include <seastar/core/queue.hh>\n#include \"core/file-impl.hh\"\n#include \"core/syscall_result.hh\"\n#include \"core/thread_pool.hh\"\n#endif\n\nnamespace seastar {\n\n#if SEASTAR_API_LEVEL < 7\nstatic_assert(std::is_nothrow_copy_constructible_v<io_priority_class>);\nstatic_assert(std::is_nothrow_move_constructible_v<io_priority_class>);\n#endif\n\nnamespace internal {\n\nstruct fs_info {\n    uint32_t block_size;\n    bool append_challenged;\n    unsigned append_concurrency;\n    bool fsync_is_exclusive;\n    bool nowait_works;\n    std::optional<dioattr> dioinfo;\n};\n\n};\n\nusing namespace internal::linux_abi;\n\nfile_handle::file_handle(const file_handle& x)\n        : _impl(x._impl ? x._impl->clone() : std::unique_ptr<file_handle_impl>()) {\n}\n\nfile_handle::file_handle(file_handle&& x) noexcept = default;\n\nfile_handle&\nfile_handle::operator=(const file_handle& x) {\n    return operator=(file_handle(x));\n}\n\nfile_handle&\nfile_handle::operator=(file_handle&&) noexcept = default;\n\nfile\nfile_handle::to_file() const & {\n    return file_handle(*this).to_file();\n}\n\nfile\nfile_handle::to_file() && {\n    return file(std::move(*_impl).to_file());\n}\n\nposix_file_impl::posix_file_impl(int fd, open_flags f, file_open_options options, dev_t device_id, bool nowait_works)\n        : _device_id(device_id)\n        , _nowait_works(nowait_works)\n        , _io_queue(engine().get_io_queue(_device_id))\n        , _open_flags(f)\n        , _fd(fd)\n{\n    configure_io_lengths();\n}\n\nposix_file_impl::posix_file_impl(int fd, open_flags f, file_open_options options, dev_t device_id, const internal::fs_info& fsi)\n        : posix_file_impl(fd, f, options, device_id, fsi.nowait_works)\n{\n    configure_dma_alignment(fsi);\n}\n\nposix_file_impl::~posix_file_impl() {\n    if (_refcount && _refcount->fetch_add(-1, std::memory_order_relaxed) != 1) {\n        return;\n    }\n    delete _refcount;\n    if (_fd != -1) {\n        // Note: close() can be a blocking operation on NFS\n        ::close(_fd);\n    }\n}\n\nvoid\nposix_file_impl::configure_dma_alignment(const internal::fs_info& fsi) {\n    if (fsi.dioinfo) {\n        const dioattr& da = *fsi.dioinfo;\n        _memory_dma_alignment = da.d_mem;\n        _disk_read_dma_alignment = da.d_miniosz;\n        // xfs wants at least the block size for writes\n        // FIXME: really read the block size\n        _disk_write_dma_alignment = std::max<unsigned>(da.d_miniosz, fsi.block_size);\n        static bool xfs_with_relaxed_overwrite_alignment = internal::kernel_uname().whitelisted({\"5.12\"});\n        _disk_overwrite_dma_alignment = xfs_with_relaxed_overwrite_alignment ? da.d_miniosz : _disk_write_dma_alignment;\n    }\n}\n\nvoid posix_file_impl::configure_io_lengths() noexcept {\n    auto limits = _io_queue.get_request_limits();\n    _read_max_length = std::min<size_t>(_read_max_length, limits.max_read);\n    _write_max_length = std::min<size_t>(_write_max_length, limits.max_write);\n}\n\nstd::unique_ptr<seastar::file_handle_impl>\nposix_file_impl::dup() {\n    if (!_refcount) {\n        _refcount = new std::atomic<unsigned>(1u);\n    }\n    auto ret = std::make_unique<posix_file_handle_impl>(_fd, _open_flags, _refcount, _device_id,\n            _memory_dma_alignment, _disk_read_dma_alignment, _disk_write_dma_alignment, _disk_overwrite_dma_alignment,\n            _nowait_works);\n    _refcount->fetch_add(1, std::memory_order_relaxed);\n    return ret;\n}\n\nposix_file_impl::posix_file_impl(int fd, open_flags f, std::atomic<unsigned>* refcount, dev_t device_id,\n        uint32_t memory_dma_alignment,\n        uint32_t disk_read_dma_alignment,\n        uint32_t disk_write_dma_alignment,\n        uint32_t disk_overwrite_dma_alignment,\n        bool nowait_works)\n        : _refcount(refcount)\n        , _device_id(device_id)\n        , _nowait_works(nowait_works)\n        , _io_queue(engine().get_io_queue(_device_id))\n        , _open_flags(f)\n        , _fd(fd) {\n...\n// Path: src/core/memory.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n\n/// \\cond internal\n\n//\n// Seastar memory allocator\n//\n// This is a share-nothing allocator (memory allocated on one cpu must\n// be freed on the same cpu).\n//\n// Inspired by gperftools' tcmalloc.\n//\n// Memory map:\n//\n// 0x0000'sccc'vvvv'vvvv\n//\n// 0000 - required by architecture (only 48 bits of address space)\n// s    - chosen to satisfy system allocator (1-7)\n// ccc  - cpu number (0-12 bits allocated vary according to system)\n// v    - virtual address within cpu (32-44 bits, according to how much ccc\n//        leaves us\n//\n// Each page has a page structure that describes it.  Within a cpu's\n// memory pool, the page array starts at offset 0, describing all pages\n// within that pool.  Page 0 does not describe a valid page.\n//\n// Each pool can contain at most 2^32 pages (or 44 address bits), so we can\n// use a 32-bit integer to identify a page.\n//\n// Runs of pages are organized into spans.  Free spans are organized into lists,\n// by size.  When spans are broken up or coalesced, they may move into new lists.\n// Spans have a size that is a power-of-two and are naturally aligned (aka buddy\n// allocator)\n//\n// If compiled with SEASTAR_HEAPPROF seastar features a sampling memory\n// profiler. Allocations are sampled at random (see `sampler` for the sampling\n// logic) and tracked. The sampled live set can be retrieved with\n// `sampled_memory_profile()`. Sampled allocations carry an extra\n// allocation_site pointer with them which is used on free to remove them from\n// the sampled live set.\n//\n// Large allocations are tracked via a pointer to the allocation_site which is\n// stored on the page structure. To check whether an allocation was sampled or\n// not this pointer is being looked at on free.\n//\n// Small allocations store an extra 8 bytes at the end of their allocation.\n// Sampled allocations are allocated in a separate set of small pools. Hence, to\n// check whether an allocation was sampled or not one only has to look at the\n// tag in pool.\n//\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <cassert>\n#include <concepts>\n#include <unordered_set>\n#include <iostream>\n#include <optional>\n#include <thread>\n#include <memory_resource>\n\n#include <fmt/format.h>\n#include <fmt/ostream.h>\n\n#include <boost/container/static_vector.hpp>\n\n#include <dlfcn.h>\n\n#ifndef SEASTAR_DEFAULT_ALLOCATOR\n#include <new>\n#include <cstdint>\n#include <algorithm>\n#include <limits>\n#include <cassert>\n#include <atomic>\n#include <mutex>\n#include <functional>\n#include <cstring>\n#include <utility>\n#include <boost/intrusive/list.hpp>\n#include <sys/mman.h>\n\n#ifdef SEASTAR_HAVE_NUMA\n#include <numaif.h>\n#endif\n#endif // !defined(SEASTAR_DEFAULT_ALLOCATOR)\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/cacheline.hh>\n#include <seastar/core/memory.hh>\n#include <seastar/core/print.hh>\n#include <seastar/util/alloc_failure_injector.hh>\n#include <seastar/util/memory_diagnostics.hh>\n#include <seastar/util/std-compat.hh>\n#include <seastar/util/sampler.hh>\n#include <seastar/util/log.hh>\n#include <seastar/core/aligned_buffer.hh>\n#ifndef SEASTAR_DEFAULT_ALLOCATOR\n#include <seastar/core/bitops.hh>\n#include <seastar/core/align.hh>\n#include <seastar/core/posix.hh>\n#include <seastar/core/shared_ptr.hh>\n#include <seastar/util/backtrace.hh>\n#endif\n#endif\n\n#ifdef SEASTAR_DEBUG\n#define dassert(expr) assert(expr)\n#else\n#define dassert(expr) do {} while(false)\n#endif\n\nnamespace seastar {\n\nextern seastar::logger seastar_logger;\n\nvoid* internal::allocate_aligned_buffer_impl(size_t size, size_t align) {\n    void *ret;\n    auto r = posix_memalign(&ret, align, size);\n    if (r == ENOMEM) {\n        throw std::bad_alloc();\n    } else if (r == EINVAL) {\n        throw std::runtime_error(format(\"Invalid alignment of {:d}; allocating {:d} bytes\", align, size));\n    } else {\n        assert(r == 0);\n        return ret;\n    }\n}\n\nnamespace memory {\n\n// We always create the logger object for memory disagnostics, even in\n// in SEASTAR_DEFAULT_ALLOCATOR builds, though it only logs when the\n// seastar allocator is enabled.\nseastar::logger seastar_memory_logger(\"seastar_memory\");\n\nstatic thread_local int abort_on_alloc_failure_suppressed = 0;\n\ndisable_abort_on_alloc_failure_temporarily::disable_abort_on_alloc_failure_temporarily() {\n    ++abort_on_alloc_failure_suppressed;\n}\n\ndisable_abort_on_alloc_failure_temporarily::~disable_abort_on_alloc_failure_temporarily() noexcept {\n    --abort_on_alloc_failure_suppressed;\n}\n\n\nvoid enable_abort_on_allocation_failure() {\n    set_abort_on_allocation_failure(true);\n}\n\nstatic std::pmr::polymorphic_allocator<char> static_malloc_allocator{std::pmr::get_default_resource()};;\nstd::pmr::polymorphic_allocator<char>* malloc_allocator{&static_malloc_allocator};\n\nnamespace internal {\n\n#ifdef __cpp_constinit\n#define SEASTAR_CONSTINIT constinit\n#else\n#define SEASTAR_CONSTINIT\n#endif\n\n#ifdef SEASTAR_ENABLE_ALLOC_FAILURE_INJECTION\n\n#ifdef __cpp_constinit\nthread_local constinit volatile int critical_alloc_section = 0;\n#else\n__thread volatile int critical_alloc_section = 0;\n#endif\n\n#endif  // SEASTAR_ENABLE_ALLOC_FAILURE_INJECTION\n\nnuma_layout\nmerge(numa_layout one, numa_layout two) {\n    // There's no chance to merge, so just concatenate\n    one.ranges.insert(one.ranges.end(), two.ranges.begin(), two.ranges.end());\n    return one;\n}\n\n} // namespace internal\n\n}\n\n}\n\n#ifndef SEASTAR_DEFAULT_ALLOCATOR\n\n#if FMT_VERSION >= 90000\nnamespace seastar::memory {\nstruct human_readable_value;\n}\ntemplate <> struct fmt::formatter<struct seastar::memory::human_readable_value> : fmt::ostream_formatter {};\n#endif\n\nnamespace seastar {\n\nusing allocation_site_ptr = const memory::allocation_site*;\n\nnamespace memory {\n\n[[gnu::unused]]\nstatic allocation_site_ptr get_allocation_site();\n\n[[gnu::noinline]]\nstatic void on_allocation_failure(size_t size);\n\nstatic constexpr unsigned cpu_id_shift = 36; // FIXME: make dynamic\nstatic constexpr unsigned max_cpus = 256;\nstatic constexpr uintptr_t cpu_id_and_mem_base_mask = ~((uintptr_t(1) << cpu_id_shift) - 1);\n\nusing pageidx = uint32_t;\n\nstruct page;\nclass page_list;\n\nstatic std::atomic<bool> live_cpus[max_cpus];\n\nusing std::optional;\n\n// is_reactor_thread gets set to true when memory::configure() gets called\n// it is used to identify seastar threads and hence use system memory allocator\n// for those threads\nstatic thread_local bool is_reactor_thread = false;\n\n// We default transparent hugepages to true since we prefer to transiently\n// use a transparent hugepage and then break it, to having the kernel\n// work to rearrange a broken transparent hugepage.\nstd::atomic<bool> use_transparent_hugepages = true;\n\nnamespace alloc_stats {\n\nenum class types { allocs, frees, cross_cpu_frees, reclaims, large_allocs, failed_allocs,\n    foreign_mallocs, foreign_frees, foreign_cross_frees, enum_size };\n\nusing stats_array = std::array<uint64_t, static_cast<std::size_t>(types::enum_size)>;\nusing stats_atomic_array = std::array<std::atomic_uint64_t, static_cast<std::size_t>(types::enum_size)>;\n\nstatic thread_local SEASTAR_CONSTINIT stats_array stats{};\nstd::array<stats_atomic_array, max_cpus> alien_stats{};\n\nstatic void increment_local(types stat_type, uint64_t size = 1) {\n    stats[static_cast<std::size_t>(stat_type)] += size;\n}\n\nstatic void increment(types stat_type, uint64_t size=1)\n{\n    // fast path, reactor threads takes thread local statistics\n    if (is_reactor_thread) {\n        increment_local(stat_type, size);\n    } else {\n        auto hash = std::hash<std::thread::id>()(std::this_thread::get_id());\n        auto i = static_cast<std::size_t>(stat_type);\n        alien_stats[hash % alien_stats.size()][i].fetch_add(size, std::memory_order_relaxed);\n    }\n}\n\nstatic uint64_t get(types stat_type)\n{\n    auto i = static_cast<std::size_t>(stat_type);\n    // fast path, reactor threads takes thread local statistics\n    if (is_reactor_thread) {\n        return stats[i];\n    } else {\n        auto hash = std::hash<std::thread::id>()(std::this_thread::get_id());\n        return alien_stats[hash % alien_stats.size()][i].load();\n    }\n}\n\n}\n\n// original memory allocator support\n// note: allocations before calling the constructor would use seastar allocator\nusing malloc_func_type = void * (*)(size_t);\nusing free_func_type = void * (*)(void *);\nusing realloc_func_type = void * (*)(void *, size_t);\nusing aligned_alloc_type = void * (*)(size_t alignment, size_t size);\nusing malloc_trim_type = int (*)(size_t);\nusing malloc_usable_size_type = size_t (*)(void *);\n\nmalloc_func_type original_malloc_func = reinterpret_cast<malloc_func_type>(dlsym(RTLD_NEXT, \"malloc\"));\nfree_func_type original_free_func = reinterpret_cast<free_func_type>(dlsym(RTLD_NEXT, \"free\"));\nrealloc_func_type original_realloc_func = reinterpret_cast<realloc_func_type>(dlsym(RTLD_NEXT, \"realloc\"));\naligned_alloc_type original_aligned_alloc_func = reinterpret_cast<aligned_alloc_type>(dlsym(RTLD_NEXT, \"aligned_alloc\"));\nmalloc_trim_type original_malloc_trim_func = reinterpret_cast<malloc_trim_type>(dlsym(RTLD_NEXT, \"malloc_trim\"));\nmalloc_usable_size_type original_malloc_usable_size_func = reinterpret_cast<malloc_usable_size_type>(dlsym(RTLD_NEXT, \"malloc_usable_size\"));\n\nusing allocate_system_memory_fn\n        = std::function<mmap_area (void* where, size_t how_much)>;\n\nnamespace bi = boost::intrusive;\n\nstatic thread_local uintptr_t local_expected_cpu_id = std::numeric_limits<uintptr_t>::max();\n\ninline\nunsigned object_cpu_id(const void* ptr) {\n    return (reinterpret_cast<uintptr_t>(ptr) >> cpu_id_shift) & 0xff;\n}\n\nclass page_list_link {\n    uint32_t _prev;\n    uint32_t _next;\n    friend class page_list;\n    friend seastar::internal::log_buf::inserter_iterator do_dump_memory_diagnostics(seastar::internal::log_buf::inserter_iterator);\n};\n\nconstexpr size_t mem_base_alloc = size_t(1) << 44;\n\nstatic char* mem_base() {\n    static char* known;\n    static std::once_flag flag;\n    std::call_once(flag, [] {\n        auto r = ::mmap(NULL, 2 * mem_base_alloc,\n                    PROT_NONE,\n                    MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE,\n                    -1, 0);\n        if (r == MAP_FAILED) {\n            abort();\n        }\n        ::madvise(r, 2 * mem_base_alloc, MADV_DONTDUMP);\n        auto cr = reinterpret_cast<char*>(r);\n        known = align_up(cr, mem_base_alloc);\n        ::munmap(cr, known - cr);\n        ::munmap(known + mem_base_alloc, cr + 2 * mem_base_alloc - (known + mem_base_alloc));\n        // extremely unlikely for mmap to return a mapping at 0, but our detection of free(null)\n        // depends on it not doing that so check it\n        assert(known != nullptr);\n        assert(reinterpret_cast<uintptr_t>(known) != 0);\n    });\n    return known;\n}\n\nbool is_seastar_memory(void * ptr)\n{\n    auto begin = mem_base();\n    auto end = begin + mem_base_alloc;\n    return ptr >= begin && ptr < end;\n}\n\nconstexpr bool is_page_aligned(size_t size) {\n    return (size & (page_size - 1)) == 0;\n}\n\nconstexpr size_t next_page_aligned(size_t size) {\n    return (size + (page_size - 1)) & ~(page_size - 1);\n}\n\nclass small_pool;\n\nstruct free_object {\n    free_object* next;\n};\n\nstruct page {\n    bool free;\n    uint8_t offset_in_span;\n    uint16_t nr_small_alloc;\n    uint32_t span_size; // in pages, if we're the head or the tail\n    page_list_link link;\n    small_pool* pool;  // if used in a small_pool\n    free_object* freelist;\n#ifdef SEASTAR_HEAPPROF\n    allocation_site_ptr alloc_site; // for objects whose size is multiple of page size, valid for head only\n#endif\n};\n\nclass page_list {\n    uint32_t _front = 0;\n    uint32_t _back = 0;\npublic:\n    page& front(page* ary) { return ary[_front]; }\n    page& back(page* ary) { return ary[_back]; }\n    bool empty() const { return !_front; }\n    void erase(page* ary, page& span) {\n        if (span.link._next) {\n            ary[span.link._next].link._prev = span.link._prev;\n        } else {\n            _back = span.link._prev;\n        }\n        if (span.link._prev) {\n            ary[span.link._prev].link._next = span.link._next;\n        } else {\n            _front = span.link._next;\n        }\n    }\n    void push_front(page* ary, page& span) {\n        auto idx = &span - ary;\n        if (_front) {\n            ary[_front].link._prev = idx;\n        } else {\n            _back = idx;\n        }\n        span.link._next = _front;\n        span.link._prev = 0;\n        _front = idx;\n    }\n    void pop_front(page* ary) {\n        if (ary[_front].link._next) {\n            ary[ary[_front].link._next].link._prev = 0;\n        } else {\n            _back = 0;\n        }\n        _front = ary[_front].link._next;\n    }\n    friend seastar::internal::log_buf::inserter_iterator do_dump_memory_diagnostics(seastar::internal::log_buf::inserter_iterator);\n};\n\nclass small_pool {\n    struct span_sizes {\n        uint8_t preferred;\n        uint8_t fallback;\n    };\n    free_object* _free = nullptr;\n    unsigned _object_size;\n    span_sizes _span_sizes;\n    unsigned _free_count = 0;\n    unsigned _min_free;\n    unsigned _max_free;\n    unsigned _pages_in_use = 0;\n    // Flag to indicate whether this pool stores sampled allocations.\n    // When freeing small allocations this flag is checked to see whether an\n    // allocation site pointer is part of the object and the allocation needs\n    // removal from the allocation_site tracking\n#ifdef SEASTAR_HEAPPROF\n    bool _sampled_pool = false;\n#endif\n    page_list _span_list;\n    static constexpr unsigned idx_frac_bits = 2;\npublic:\n    explicit small_pool(unsigned object_size, bool is_sampled) noexcept;\n    ~small_pool();\n    inline void* allocate();\n    void deallocate(void* object);\n    unsigned object_size() const { return _object_size; }\n    /// See _sampled_pool\n    bool is_sampled_pool() const {\n#ifdef SEASTAR_HEAPPROF\n        return _sampled_pool;\n#else\n        return false;\n#endif\n    }\n    bool objects_page_aligned() const { return is_page_aligned(_object_size); }\n    static constexpr unsigned size_to_idx(unsigned size);\n    static constexpr unsigned idx_to_size(unsigned idx);\n    allocation_site_ptr& alloc_site_holder(void* ptr);\nprivate:\n    inline void* pop_free();\n    [[gnu::noinline]] void* add_more_objects();\n    void trim_free_list();\n    friend seastar::internal::log_buf::inserter_iterator do_dump_memory_diagnostics(seastar::internal::log_buf::inserter_iterator);\n};\n\n// index 0b0001'1100 -> size (1 << 4) + 0b11 << (4 - 2)\n\nconstexpr unsigned\nsmall_pool::idx_to_size(unsigned idx) {\n    size_t s = (((1 << idx_frac_bits) | (idx & ((1 << idx_frac_bits) - 1)))\n              << (idx >> idx_frac_bits))\n                  >> idx_frac_bits;\n    // If size is larger than max_align_t, force it to be a multiple of\n    // max_align_t. Clang relies in this property to use aligned mov\n    // instructions (e.g. movaps)\n    //\n    // Note this function is used at initialization time only, so it doesn't\n    // need to be especially fast.\n    if (s > alignof(std::max_align_t)) {\n\ts = align_up(s, alignof(std::max_align_t));\n    }\n    return s;\n}\n\nconstexpr unsigned\nsmall_pool::size_to_idx(unsigned size) {\n    return ((log2floor(size) << idx_frac_bits) - ((1 << idx_frac_bits) - 1))\n            + ((size - 1) >> (log2floor(size) - idx_frac_bits));\n}\n\ntemplate<bool sampled> // tag the pools in this array as sampled, see small_pool._sampled_pool\nclass small_pool_array {\npublic:\n    static constexpr unsigned nr_small_pools = small_pool::size_to_idx(4 * page_size) + 1;\nprivate:\n    union u {\n        small_pool a[nr_small_pools];\n        u() {\n            for (unsigned i = 0; i < nr_small_pools; ++i) {\n                new (&a[i]) small_pool(small_pool::idx_to_size(i), sampled);\n            }\n        }\n        ~u() {\n            // cannot really call destructor, since other\n            // objects may be freed after we are gone.\n        }\n    } _u;\npublic:\n    small_pool& operator[](unsigned idx) { return _u.a[idx]; }\n};\n\nstatic constexpr size_t max_small_allocation\n    = small_pool::idx_to_size(small_pool_array<false>::nr_small_pools - 1);\n\n#ifdef SEASTAR_HEAPPROF\nconstexpr size_t object_size_with_alloc_site(size_t size) {\n    // For page-aligned sizes, allocation_site* lives in page::alloc_site, not with the object.\n    static_assert(is_page_aligned(max_small_allocation), \"assuming that max_small_allocation is page aligned so that we\"\n            \" don't need to add allocation_site_ptr to objects of size close to it\");\n    size_t next_page_aligned_size = next_page_aligned(size);\n    if (next_page_aligned_size - size > sizeof(allocation_site_ptr)) {\n        size += sizeof(allocation_site_ptr);\n    } else {\n        return next_page_aligned_size;\n    }\n    return size;\n}\n\n// Ensure that object_size_with_alloc_site() does not exceed max_small_allocation\nstatic_assert(object_size_with_alloc_site(max_small_allocation) == max_small_allocation, \"\");\nstatic_assert(object_size_with_alloc_site(max_small_allocation - 1) == max_small_allocation, \"\");\nstatic_assert(object_size_with_alloc_site(max_small_allocation - sizeof(allocation_site_ptr) + 1) == max_small_allocation, \"\");\nstatic_assert(object_size_with_alloc_site(max_small_allocation - sizeof(allocation_site_ptr)) == max_small_allocation, \"\");\nstatic_assert(object_size_with_alloc_site(max_small_allocation - sizeof(allocation_site_ptr) - 1) == max_small_allocation - 1, \"\");\nstatic_assert(object_size_with_alloc_site(max_small_allocation - sizeof(allocation_site_ptr) - 2) == max_small_allocation - 2, \"\");\n#endif\n\nstruct cross_cpu_free_item {\n    cross_cpu_free_item* next;\n};\n\nstruct cpu_pages {\n    small_pool_array<false> small_pools;\n    uint32_t min_free_pages = 20000000 / page_size;\n    char* memory;\n    page* pages;\n    uint32_t nr_pages;\n    uint32_t nr_free_pages;\n    uint32_t current_min_free_pages = 0;\n    size_t large_allocation_warning_threshold = std::numeric_limits<size_t>::max();\n    unsigned cpu_id = -1U;\n    std::function<void (std::function<void ()>)> reclaim_hook;\n    std::vector<reclaimer*> reclaimers;\n    static constexpr unsigned nr_span_lists = 32;\n    page_list free_spans[nr_span_lists];  // contains aligned spans with span_size == 2^idx\n    alignas(seastar::cache_line_size) std::atomic<cross_cpu_free_item*> xcpu_freelist;\n    static std::atomic<unsigned> cpu_id_gen;\n    static cpu_pages* all_cpus[max_cpus];\n    union asu {\n        using alloc_sites_type = std::unordered_set<allocation_site>;\n        asu() : alloc_sites{} {\n        }\n        ~asu() {} // alloc_sites live forever\n        alloc_sites_type alloc_sites;\n    } asu;\n    allocation_site_ptr alloc_site_list_head = nullptr; // For easy traversal of asu.alloc_sites from scylla-gdb.py\n    sampler heap_prof_sampler;\n    small_pool_array<true> sampled_small_pools;\n\n    char* mem() { return memory; }\n\n    void link(page_list& list, page* span);\n    void unlink(page_list& list, page* span);\n    struct trim {\n        unsigned offset;\n        unsigned nr_pages;\n    };\n    void maybe_reclaim();\n    void* allocate_large_and_trim(unsigned nr_pages, bool should_sample);\n    void* allocate_large(unsigned nr_pages, bool should_sample);\n    void* allocate_large_aligned(unsigned align_pages, unsigned nr_pages, bool should_sample);\n    page* find_and_unlink_span(unsigned nr_pages);\n    page* find_and_unlink_span_reclaiming(unsigned n_pages);\n    void free_large(void* ptr);\n    bool grow_span(pageidx& start, uint32_t& nr_pages, unsigned idx);\n    void free_span(pageidx start, uint32_t nr_pages);\n    void free_span_no_merge(pageidx start, uint32_t nr_pages);\n    void free_span_unaligned(pageidx start, uint32_t nr_pages);\n    void free(void* ptr);\n    void free(void* ptr, size_t size);\n    static bool try_free_fastpath(void* ptr);\n    static bool is_local_pointer(void* ptr);\n    static void do_foreign_free(void* ptr);\n    void shrink(void* ptr, size_t new_size);\n    static void free_cross_cpu(unsigned cpu_id, void* ptr);\n    bool drain_cross_cpu_freelist();\n    size_t object_size(void* ptr);\n\n    page* to_page(void* p) {\n        size_t page_idx = ((uintptr_t)p) << (64 - cpu_id_shift) >> (64 - cpu_id_shift + page_bits);\n        return &pages[page_idx];\n    }\n\n    bool is_initialized() const;\n    bool initialize();\n    reclaiming_result run_reclaimers(reclaimer_scope, size_t pages_to_reclaim);\n    void schedule_reclaim();\n    void set_reclaim_hook(std::function<void (std::function<void ()>)> hook);\n    void set_min_free_pages(size_t pages);\n    void resize(size_t new_size, allocate_system_memory_fn alloc_sys_mem);\n    void do_resize(size_t new_size, allocate_system_memory_fn alloc_sys_mem);\n    void replace_memory_backing(allocate_system_memory_fn alloc_sys_mem);\n    void check_large_allocation(size_t size);\n    void warn_large_allocation(size_t size);\n    allocation_site_ptr add_alloc_site(size_t allocated_size);\n    void remove_alloc_site(allocation_site_ptr alloc_site, size_t deallocated_size);\n    bool maybe_sample(size_t size);\n    bool definitely_sample(size_t size);\n    memory::memory_layout memory_layout();\n    ~cpu_pages();\n};\n\nstatic thread_local cpu_pages cpu_mem;\nstd::atomic<unsigned> cpu_pages::cpu_id_gen;\ncpu_pages* cpu_pages::all_cpus[max_cpus];\n\nstatic cpu_pages& get_cpu_mem();\n\n#ifdef SEASTAR_HEAPPROF\n\nvoid set_heap_profiling_sampling_rate(size_t sample_rate) {\n    bool current_sample_rate = get_cpu_mem().heap_prof_sampler.sampling_interval();\n    if (sample_rate) {\n        if (!current_sample_rate) {\n            seastar_logger.info(\"Enabling heap profiler - using {} bytes sampling rate\", sample_rate);\n        } else {\n            seastar_logger.warn(\"Ignoring change to heap profiler sample rate as heap profiling is already turned on\");\n            return;\n        }\n    } else {\n        if (current_sample_rate) {\n            seastar_logger.info(\"Disabling heap profiler\");\n        }\n    }\n    get_cpu_mem().heap_prof_sampler.set_sampling_interval(sample_rate);\n}\n\nsize_t get_heap_profiling_sample_rate() {\n    return get_cpu_mem().heap_prof_sampler.sampling_interval();\n}\n\nstatic thread_local int64_t scoped_heap_profiling_embed_count = 0;\n\nscoped_heap_profiling::scoped_heap_profiling(size_t sample_rate) noexcept {\n    ++scoped_heap_profiling_embed_count;\n    set_heap_profiling_sampling_rate(sample_rate);\n}\n\nscoped_heap_profiling::~scoped_heap_profiling() {\n    if (!--scoped_heap_profiling_embed_count) {\n        set_heap_profiling_sampling_rate(0);\n    }\n}\n\n#else\n\nvoid set_heap_profiling_sampling_rate(size_t enable) {\n    seastar_logger.warn(\"Seastar compiled without heap profiling support, heap profiler not supported;\"\n            \" compile with the Seastar_HEAP_PROFILING=ON CMake option to add heap profiling support\");\n}\n\nsize_t get_heap_profiling_sample_rate() {\n    // don't log here, called on all paths\n    return 0;\n}\n\nscoped_heap_profiling::scoped_heap_profiling(size_t sample_rate) noexcept {\n    set_heap_profiling_sampling_rate(sample_rate); // let it print the warning\n}\n\nscoped_heap_profiling::~scoped_heap_profiling() {\n}\n\n#endif\n\n// Smallest index i such that all spans stored in the index are >= pages.\nstatic inline\nunsigned index_of(unsigned pages) {\n    if (pages == 1) {\n        return 0;\n    }\n    return std::numeric_limits<unsigned>::digits - count_leading_zeros(pages - 1);\n}\n\nvoid\ncpu_pages::unlink(page_list& list, page* span) {\n    list.erase(pages, *span);\n}\n\nvoid\ncpu_pages::link(page_list& list, page* span) {\n    list.push_front(pages, *span);\n}\n\nvoid cpu_pages::free_span_no_merge(uint32_t span_start, uint32_t nr_pages) {\n    assert(nr_pages);\n    nr_free_pages += nr_pages;\n    auto span = &pages[span_start];\n    auto span_end = &pages[span_start + nr_pages - 1];\n    span->free = span_end->free = true;\n    span->span_size = span_end->span_size = nr_pages;\n    auto idx = index_of(nr_pages);\n    link(free_spans[idx], span);\n}\n\nbool cpu_pages::grow_span(uint32_t& span_start, uint32_t& nr_pages, unsigned idx) {\n    auto which = (span_start >> idx) & 1; // 0=lower, 1=upper\n    // locate first page of upper buddy or last page of lower buddy\n    // examples: span_start = 0x10 nr_pages = 0x08 -> buddy = 0x18  (which = 0)\n    //           span_start = 0x18 nr_pages = 0x08 -> buddy = 0x17  (which = 1)\n    // delta = which ? -1u : nr_pages\n    auto delta = ((which ^ 1) << idx) | -which;\n    auto buddy = span_start + delta;\n    if (pages[buddy].free && pages[buddy].span_size == nr_pages) {\n        unlink(free_spans[idx], &pages[span_start ^ nr_pages]);\n        nr_free_pages -= nr_pages; // free_span_no_merge() will restore\n        span_start &= ~nr_pages;\n        nr_pages *= 2;\n        return true;\n    }\n    return false;\n}\n\nvoid cpu_pages::free_span(uint32_t span_start, uint32_t nr_pages) {\n    auto idx = index_of(nr_pages);\n    while (grow_span(span_start, nr_pages, idx)) {\n        ++idx;\n    }\n    free_span_no_merge(span_start, nr_pages);\n}\n\n// Internal, used during startup. Span is not aligned so needs to be broken up\nvoid cpu_pages::free_span_unaligned(uint32_t span_start, uint32_t nr_pages) {\n    while (nr_pages) {\n        auto start_nr_bits = span_start ? count_trailing_zeros(span_start) : 32;\n        auto size_nr_bits = count_trailing_zeros(nr_pages);\n        auto now = 1u << std::min(start_nr_bits, size_nr_bits);\n        free_span(span_start, now);\n        span_start += now;\n        nr_pages -= now;\n    }\n}\n\npage*\ncpu_pages::find_and_unlink_span(unsigned n_pages) {\n    auto idx = index_of(n_pages);\n    if (n_pages >= (2u << idx)) {\n        return nullptr;\n    }\n    while (idx < nr_span_lists && free_spans[idx].empty()) {\n        ++idx;\n    }\n    if (idx == nr_span_lists) {\n        if (initialize()) {\n            return find_and_unlink_span(n_pages);\n        }\n        return nullptr;\n    }\n    auto& list = free_spans[idx];\n    page* span = &list.front(pages);\n    unlink(list, span);\n    return span;\n}\n\npage*\ncpu_pages::find_and_unlink_span_reclaiming(unsigned n_pages) {\n    while (true) {\n        auto span = find_and_unlink_span(n_pages);\n        if (span) {\n            return span;\n        }\n        if (run_reclaimers(reclaimer_scope::sync, n_pages) == reclaiming_result::reclaimed_nothing) {\n            return nullptr;\n        }\n    }\n}\n\nvoid cpu_pages::maybe_reclaim() {\n    if (nr_free_pages < current_min_free_pages) {\n        drain_cross_cpu_freelist();\n        if (nr_free_pages < current_min_free_pages) {\n            run_reclaimers(reclaimer_scope::sync, current_min_free_pages - nr_free_pages);\n        }\n        if (nr_free_pages < current_min_free_pages) {\n            schedule_reclaim();\n        }\n    }\n}\n\nvoid*\ncpu_pages::allocate_large_and_trim(unsigned n_pages, bool should_sample) {\n    // Avoid exercising the reclaimers for requests we'll not be able to satisfy\n    // nr_pages might be zero during startup, so check for that too\n    if (nr_pages && n_pages >= nr_pages) {\n        return nullptr;\n    }\n    page* span = find_and_unlink_span_reclaiming(n_pages);\n    if (!span) {\n        return nullptr;\n    }\n    auto span_size = span->span_size;\n    auto span_idx = span - pages;\n    nr_free_pages -= span->span_size;\n    while (span_size >= n_pages * 2) {\n        span_size /= 2;\n        auto other_span_idx = span_idx + span_size;\n        free_span_no_merge(other_span_idx, span_size);\n    }\n    auto span_end = &pages[span_idx + span_size - 1];\n    span->free = span_end->free = false;\n    span->span_size = span_end->span_size = span_size;\n    span->pool = nullptr;\n#ifdef SEASTAR_HEAPPROF\n    if (should_sample) {\n        auto alloc_site = add_alloc_site(span->span_size * page_size);\n        span->alloc_site = alloc_site;\n    }\n    else {\n        span->alloc_site = nullptr;\n    }\n#endif\n    maybe_reclaim();\n    return mem() + span_idx * page_size;\n}\n\nvoid\ncpu_pages::warn_large_allocation(size_t size) {\n    alloc_stats::increment_local(alloc_stats::types::large_allocs);\n    seastar_memory_logger.warn(\"oversized allocation: {} bytes. This is non-fatal, but could lead to latency and/or fragmentation issues. Please report: at {}\", size, current_backtrace());\n    large_allocation_warning_threshold *= 1.618; // prevent spam\n}\n\nallocation_site_ptr\ncpu_pages::add_alloc_site(size_t allocated_size) {\n    allocation_site_ptr alloc_site = get_allocation_site();\n    if (alloc_site) {\n        ++alloc_site->count;\n        alloc_site->size += heap_prof_sampler.sample_size(allocated_size);\n    }\n\n    return alloc_site;\n}\n\nvoid\ncpu_pages::remove_alloc_site(allocation_site_ptr alloc_site, size_t deallocated_size) {\n    if (alloc_site) {\n        --alloc_site->count;\n        auto sample_size = heap_prof_sampler.sample_size(deallocated_size);\n        // prevent underflow in case sample rate changed\n        alloc_site->size -= alloc_site->size < sample_size ? alloc_site->size : sample_size;\n        if (alloc_site->count == 0) {\n            if (alloc_site->prev) {\n                alloc_site->prev->next = alloc_site->next;\n            }\n            if (alloc_site->next) {\n                alloc_site->next->prev = alloc_site->prev;\n            }\n            if (alloc_site_list_head == alloc_site) {\n                alloc_site_list_head = alloc_site->next;\n            }\n\n            asu.alloc_sites.erase(*alloc_site);\n        }\n    }\n}\n\n[[gnu::always_inline]]\ninline bool\ncpu_pages::maybe_sample(size_t size) {\n#ifdef SEASTAR_HEAPPROF\n    return heap_prof_sampler.maybe_sample(size);\n#else\n    return false;\n#endif\n}\n\n[[gnu::always_inline]]\ninline bool\ncpu_pages::definitely_sample(size_t size) {\n#ifdef SEASTAR_HEAPPROF\n    return heap_prof_sampler.definitely_sample(size);\n#else\n    return false;\n#endif\n}\n\n[[gnu::always_inline]]\nvoid\ninline\ncpu_pages::check_large_allocation(size_t size) {\n    if (size >= large_allocation_warning_threshold) {\n        warn_large_allocation(size);\n    }\n}\n\n[[gnu::always_inline]]\ninline void*\ncpu_pages::allocate_large(unsigned n_pages, bool should_sample) {\n    check_large_allocation(n_pages * page_size);\n    return allocate_large_and_trim(n_pages, should_sample);\n}\n\nvoid*\ncpu_pages::allocate_large_aligned(unsigned align_pages, unsigned n_pages, bool should_sample) {\n    check_large_allocation(n_pages * page_size);\n    // buddy allocation is always aligned\n    return allocate_large_and_trim(n_pages, should_sample);\n}\n\ndisable_backtrace_temporarily::disable_backtrace_temporarily()\n    : _disable_sampling(cpu_mem.heap_prof_sampler.pause_sampling()) {\n}\n\nstatic\nsimple_backtrace get_backtrace() noexcept {\n    disable_backtrace_temporarily dbt;\n    return current_backtrace_tasklocal();\n}\n\nstatic\nallocation_site_ptr get_allocation_site() {\n    if (!cpu_mem.is_initialized() || !cpu_mem.heap_prof_sampler.sampling_interval()) {\n        return nullptr;\n    }\n    disable_backtrace_temporarily dbt;\n    allocation_site new_alloc_site;\n    new_alloc_site.backtrace = get_backtrace();\n    if (cpu_mem.asu.alloc_sites.size() >= 1000\n        && cpu_mem.asu.alloc_sites.find(new_alloc_site) == cpu_mem.asu.alloc_sites.end()) {\n        // Drop sample for now. Could do something smarter like dropping a\n        // current one at random but needs more work in remove_alloc_site as we\n        // might then have allocations for which the allocsite is no longer\n        // alive\n        return nullptr;\n    }\n    auto insert_result = cpu_mem.asu.alloc_sites.insert(std::move(new_alloc_site));\n    allocation_site_ptr alloc_site = &*insert_result.first;\n    if (insert_result.second) {\n        alloc_site->next = cpu_mem.alloc_site_list_head;\n        if (cpu_mem.alloc_site_list_head) {\n            cpu_mem.alloc_site_list_head->prev = alloc_site;\n        }\n        cpu_mem.alloc_site_list_head = alloc_site;\n    }\n    return alloc_site;\n}\n\n#ifdef SEASTAR_HEAPPROF\n\nallocation_site_ptr&\nsmall_pool::alloc_site_holder(void* ptr) {\n    if (objects_page_aligned()) {\n        return get_cpu_mem().to_page(ptr)->alloc_site;\n    } else {\n        return *reinterpret_cast<allocation_site_ptr*>(reinterpret_cast<char*>(ptr) + _object_size - sizeof(allocation_site_ptr));\n    }\n}\n\n#endif\n\nstatic\nvoid\nmaybe_enable_transparent_hugepages(void* addr, size_t len) {\n    if (use_transparent_hugepages.load(std::memory_order_relaxed)) {\n        ::madvise(addr, len, MADV_HUGEPAGE);\n    }\n}\n\nstatic\nvoid\nmaybe_disable_transparent_hugepages(void* addr, size_t len) {\n    if (!use_transparent_hugepages.load(std::memory_order_relaxed)) {\n        ::madvise(addr, len, MADV_NOHUGEPAGE);\n    }\n}\n\nvoid cpu_pages::free_large(void* ptr) {\n    pageidx idx = (reinterpret_cast<char*>(ptr) - mem()) / page_size;\n    page* span = &pages[idx];\n#ifdef SEASTAR_HEAPPROF\n    if (span->alloc_site) {\n        auto alloc_site = span->alloc_site;\n        remove_alloc_site(alloc_site, span->span_size * page_size);\n    }\n#endif\n    free_span(idx, span->span_size);\n}\n\nsize_t cpu_pages::object_size(void* ptr) {\n    page* span = to_page(ptr);\n    if (span->pool) {\n        auto s = span->pool->object_size();\n#ifdef SEASTAR_HEAPPROF\n        if (span->pool->is_sampled_pool()) {\n            // We must not allow the object to be extended onto the allocation_site_ptr field.\n            if (!span->pool->objects_page_aligned()) {\n                s -= sizeof(allocation_site_ptr);\n            }\n        }\n#endif\n        return s;\n    } else {\n        return size_t(span->span_size) * page_size;\n    }\n}\n\nvoid cpu_pages::free_cross_cpu(unsigned cpu_id, void* ptr) {\n    if (!live_cpus[cpu_id].load(std::memory_order_relaxed)) {\n        // Thread was destroyed; leak object\n        // should only happen for boost unit-tests.\n        return;\n    }\n    auto p = reinterpret_cast<cross_cpu_free_item*>(ptr);\n    auto& list = all_cpus[cpu_id]->xcpu_freelist;\n    auto old = list.load(std::memory_order_relaxed);\n    do {\n        p->next = old;\n    } while (!list.compare_exchange_weak(old, p, std::memory_order_release, std::memory_order_relaxed));\n    alloc_stats::increment(alloc_stats::types::cross_cpu_frees);\n}\n\nbool cpu_pages::drain_cross_cpu_freelist() {\n    if (!xcpu_freelist.load(std::memory_order_relaxed)) {\n        return false;\n    }\n    auto p = xcpu_freelist.exchange(nullptr, std::memory_order_acquire);\n    while (p) {\n        auto n = p->next;\n        alloc_stats::increment_local(alloc_stats::types::frees);\n        free(p);\n        p = n;\n    }\n    return true;\n}\n\n[[gnu::always_inline]]\ninline void cpu_pages::free(void* ptr) {\n    page* span = to_page(ptr);\n    if (span->pool) {\n        small_pool& pool = *span->pool;\n#ifdef SEASTAR_HEAPPROF\n        if (pool.is_sampled_pool()) {\n            allocation_site_ptr alloc_site = pool.alloc_site_holder(ptr);\n            remove_alloc_site(alloc_site, pool.object_size());\n        }\n#endif\n        pool.deallocate(ptr);\n    } else {\n        free_large(ptr);\n    }\n}\n\nvoid cpu_pages::free(void* ptr, size_t size) {\n#ifdef SEASTAR_HEAPPROF\n    // sized free can avoid accessing the `page` structure as an optimization.\n    // With memory sampling we always need to check the pool though to see\n    // whether this allocation was sampled. Hence, just defer to the non-sized\n    // implementation\n    (void) size;\n    free(ptr);\n#else\n    // match action on allocate() so hit the right pool\n    if (size <= sizeof(free_object)) {\n        size = sizeof(free_object);\n    }\n    if (size <= max_small_allocation) {\n        auto pool = &small_pools[small_pool::size_to_idx(size)];\n        pool->deallocate(ptr);\n    } else {\n        free_large(ptr);\n    }\n#endif\n}\n\n// Is the passed pointer a local pointer, i.e., allocated on the current shard from the \n// per-shard allocator.\n[[gnu::always_inline]]\ninline bool\ncpu_pages::is_local_pointer(void* ptr) {\n    return (reinterpret_cast<uintptr_t>(ptr) & cpu_id_and_mem_base_mask) == local_expected_cpu_id;\n}\n\n// Try to execute free on the fast path, which succeeds if:\n//\n// 1) The pointer is local to this shard\n// 2) The pointer is from a small pool\n// 3) The small pool is not sampled\n//\n// In this case, complete the de-allocation and return true.\n// Otherwise, modify nothing and return false.\n[[gnu::always_inline]]\ninline bool\ncpu_pages::try_free_fastpath(void* ptr) {\n    if (__builtin_expect(is_local_pointer(ptr), true)) {\n        auto pool = get_cpu_mem().to_page(ptr)->pool;\n        if (__builtin_expect(pool && !pool->is_sampled_pool(), true)) {\n            alloc_stats::increment_local(alloc_stats::types::frees);\n            pool->deallocate(ptr);\n            return true;\n        }\n    }\n    return false;\n}\n\n/// Helper to allow a single implementation for sized and non-sized functions.\n/// Indicator to allow a single implementation for sized and non-sized functions.\n/// The size parameter will be either no_size tag type or size_t, and most\n/// of the implementation can be shared, using constexpr if or other dispatch\n/// in the places where there should be a difference of behavior.\nstruct no_size {};\n\ntemplate <typename S>\nrequires std::same_as<S, size_t> || std::same_as<S, no_size>\n[[gnu::noinline]]\nstatic void free_slowpath(void* obj, S size) {\n    if (cpu_pages::is_local_pointer(obj)) {\n        alloc_stats::increment_local(alloc_stats::types::frees);\n        if constexpr (std::is_same_v<decltype(size), no_size>) {\n            get_cpu_mem().free(obj);\n        } else {\n            get_cpu_mem().free(obj, size);\n        }\n    } else {\n        cpu_pages::do_foreign_free(obj);\n    }\n}\n\n[[gnu::noinline]]\nvoid\ncpu_pages::do_foreign_free(void* ptr) {\n    // handles:\n    // 1) non-seastar pointers\n    // 2) cross-shard frees\n    // 3) null pointer\n\n    if (!ptr) {\n        return;\n    }\n\n    if (!is_seastar_memory(ptr)) {\n        if (is_reactor_thread) {\n            alloc_stats::increment_local(alloc_stats::types::foreign_cross_frees);\n        } else {\n            alloc_stats::increment(alloc_stats::types::foreign_frees);\n        }\n        original_free_func(ptr);\n        return;\n    }\n    free_cross_cpu(object_cpu_id(ptr), ptr);\n}\n\nvoid cpu_pages::shrink(void* ptr, size_t new_size) {\n    assert(object_cpu_id(ptr) == cpu_id);\n    page* span = to_page(ptr);\n    if (span->pool) {\n        return;\n    }\n    auto old_size_pages = span->span_size;\n    size_t new_size_pages = old_size_pages;\n    while (new_size_pages / 2 * page_size >= new_size) {\n        new_size_pages /= 2;\n    }\n    if (new_size_pages == old_size_pages) {\n        return;\n    }\n#ifdef SEASTAR_HEAPPROF\n    auto alloc_site = span->alloc_site;\n    if (alloc_site) {\n        alloc_site->size -= span->span_size * page_size;\n        alloc_site->size += new_size_pages * page_size;\n    }\n#endif\n    span->span_size = new_size_pages;\n    span[new_size_pages - 1].free = false;\n    span[new_size_pages - 1].span_size = new_size_pages;\n    pageidx idx = span - pages;\n    free_span_unaligned(idx + new_size_pages, old_size_pages - new_size_pages);\n}\n\ncpu_pages::~cpu_pages() {\n    if (is_initialized()) {\n        live_cpus[cpu_id].store(false, std::memory_order_relaxed);\n    }\n}\n\nbool cpu_pages::is_initialized() const {\n    return bool(nr_pages);\n}\n\nbool cpu_pages::initialize() {\n    if (is_initialized()) {\n        return false;\n    }\n    cpu_id = cpu_id_gen.fetch_add(1, std::memory_order_relaxed);\n    local_expected_cpu_id = (static_cast<uint64_t>(cpu_id) << cpu_id_shift)\n\t        | reinterpret_cast<uintptr_t>(mem_base());\n    assert(cpu_id < max_cpus);\n    all_cpus[cpu_id] = this;\n    auto base = mem_base() + (size_t(cpu_id) << cpu_id_shift);\n    auto size = 32 << 20;  // Small size for bootstrap\n    auto r = ::mmap(base, size,\n            PROT_READ | PROT_WRITE,\n            MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,\n            -1, 0);\n    if (r == MAP_FAILED) {\n        abort();\n    }\n    maybe_enable_transparent_hugepages(base, size);\n    pages = reinterpret_cast<page*>(base);\n    memory = base;\n    nr_pages = size / page_size;\n    // we reserve the end page so we don't have to special case\n...\n", "needle_token_start": 3993, "needle_token_end": 4027, "code_context_ntokens": 16177, "output": ["```cpp\nvoid enable_abort_on_allocation_failure() {\n    set_abort_on_allocation_failure(true);\n}\n```"]}
{"repo": "scylladb/seastar", "name": "make_shard_local_buffer_copy", "language": "cpp", "path": "src/rpc/rpc.cc", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a local copy of a buffer that may originally reside on a different processing shard, ensuring that the buffer's data is accessible locally without the need for inter-shard communication when accessed.\n2. **Input**: It takes a pointer to a unique buffer object that may belong to a different shard.\n3. **Output**: Returns a new buffer object that contains either a direct move of the original buffer if it's already local, or a new buffer with shared ownership of the original data to avoid unnecessary data copying.\n4. **Procedure**: \n   - First, the function checks if the original buffer is owned by the current shard. If so, it directly moves the buffer to the output.\n   - If the buffer is from a different shard, it initializes a new buffer of the same size.\n   - It then checks the type of the original buffer (single or multiple chunks).\n   - For a single chunk, it creates a new buffer that points to the original data and sets up a deleter that will handle the original buffer's deletion once no longer needed.\n   - For multiple chunks, it creates a new vector of buffers, each pointing to a chunk of the original data, with a shared deleter across all new chunks to manage the original data's lifecycle.\n   - The newly created buffer or vector of buffers is then returned, ensuring local access with minimal overhead.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/http/json_path.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2015 Cloudius Systems\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <vector>\nmodule seastar;\n#else\n#include <seastar/http/json_path.hh>\n#endif\n\nnamespace seastar {\n\nnamespace httpd {\n\nusing namespace std;\n\nvoid path_description::set(routes& _routes, handler_base* handler) const {\n    for (auto& i : mandatory_queryparams) {\n        handler->mandatory(i);\n    }\n\n    if (params.size() == 0)\n        _routes.put(operations.method, path, handler);\n    else {\n        match_rule* rule = new match_rule(handler);\n        rule->add_str(path);\n        for (auto&& i : params) {\n            if (i.type == url_component_type::FIXED_STRING) {\n                rule->add_str(i.name);\n            } else {\n                rule->add_param(i.name, i.type == url_component_type::PARAM_UNTIL_END_OF_PATH);\n            }\n        }\n        _cookie = _routes.add_cookie(rule, operations.method);\n    }\n}\n\nvoid path_description::set(routes& _routes,\n        const json_request_function& f) const {\n    set(_routes, new function_handler(f));\n}\n\nvoid path_description::set(routes& _routes, const future_json_function& f) const {\n    set(_routes, new function_handler(f));\n}\n\nvoid path_description::unset(routes& _routes) const {\n    if (params.size() == 0) {\n...\n// Path: src/testing/test_runner.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2015 Cloudius Systems, Ltd.\n */\n\n#include <iostream>\n\n#include <seastar/core/app-template.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/posix.hh>\n#include <seastar/testing/test_runner.hh>\n\nnamespace seastar {\n\nnamespace testing {\n\nstatic test_runner instance;\n\nstruct stop_execution : public std::exception {};\n\ntest_runner::~test_runner() {\n    finalize();\n}\n\nbool\ntest_runner::start(int ac, char** av) {\n    bool expected = false;\n    if (!_started.compare_exchange_strong(expected, true, std::memory_order_acquire)) {\n        return true;\n    }\n\n    // Don't interfere with seastar signal handling\n    sigset_t mask;\n    sigfillset(&mask);            \n    for (auto sig : { SIGSEGV }) {\n        sigdelset(&mask, sig);\n    }\n    auto r = ::pthread_sigmask(SIG_BLOCK, &mask, NULL);\n    if (r) {\n        std::cerr << \"Error blocking signals. Aborting.\" << std::endl;\n        abort();\n    }\n\n    _st_args = std::make_unique<start_thread_args>(ac, av);\n    return true;\n}\n\nint test_runner::start_thread(int ac, char** av) {\n    auto init_outcome = std::make_shared<exchanger<int>>();\n\n    namespace bpo = boost::program_options;\n    _thread = std::make_unique<posix_thread>([this, ac, av, init_outcome]() mutable {\n        app_template app;\n        app.add_options()\n            (\"random-seed\", bpo::value<unsigned>(), \"Random number generator seed\")\n            (\"fail-on-abandoned-failed-futures\", bpo::value<bool>()->default_value(true), \"Fail the test if there are any abandoned failed futures\");\n        // We guarantee that only one thread is running.\n        // We only read this after that one thread is joined, so this is safe.\n        _exit_code = app.run(ac, av, [this, &app, init_outcome = init_outcome.get()] {\n            init_outcome->give(0);\n            auto init = [&app] {\n                auto conf_seed = app.configuration()[\"random-seed\"];\n                auto seed = conf_seed.empty() ? std::random_device()():  conf_seed.as<unsigned>();\n                std::cout << \"random-seed=\" << seed << std::endl;\n                return smp::invoke_on_all([seed] {\n                    auto local_seed = seed + this_shard_id();\n                    local_random_engine.seed(local_seed);\n                });\n            };\n\n            return init().then([this] {\n              return do_until([this] { return _done; }, [this] {\n                // this will block the reactor briefly, but we don't care\n                try {\n                    auto func = _task.take();\n                    return func();\n                } catch (const stop_execution&) {\n                    _done = true;\n                    return make_ready_future<>();\n                }\n              }).or_terminate();\n            }).then([&app] {\n                if (engine().abandoned_failed_futures()) {\n                    std::cerr << \"*** \" << engine().abandoned_failed_futures() << \" abandoned failed future(s) detected\" << std::endl;\n                    if (app.configuration()[\"fail-on-abandoned-failed-futures\"].as<bool>()) {\n                        std::cerr << \"Failing the test because fail was requested by --fail-on-abandoned-failed-futures\" << std::endl;\n                        return 3;\n                    }\n                }\n                return 0;\n            });\n        });\n        init_outcome->give(_exit_code);\n    });\n\n    return init_outcome->take();\n}\n\nvoid\ntest_runner::run_sync(std::function<future<>()> task) {\n    if (_st_args) {\n        start_thread_args sa = *_st_args;\n        _st_args.reset();\n        if (int start_status = start_thread(sa.ac, sa.av); start_status != 0) {\n            // something bad happened when starting the reactor or app, and\n            // the _thread has exited before taking any task. but we need to\n            // move on. let's report this bad news with exit code\n            _exit_code = start_status;\n        }\n    }\n    if (_exit_code != 0) {\n        // we failed to start the worker reactor, so we cannot send the task to\n        // it.\n        return;\n    }\n\n    exchanger<std::exception_ptr> e;\n    _task.give([task = std::move(task), &e] {\n        assert(engine_is_ready());\n        try {\n            return task().then_wrapped([&e](auto&& f) {\n                try {\n                    f.get();\n                    e.give({});\n                } catch (...) {\n                    e.give(std::current_exception());\n                }\n            });\n        } catch (...) {\n            e.give(std::current_exception());\n            return make_ready_future<>();\n        }\n    });\n    auto maybe_exception = e.take();\n    if (maybe_exception) {\n        std::rethrow_exception(maybe_exception);\n    }\n}\n\nint test_runner::finalize() {\n    if (_thread) {\n        _task.interrupt(stop_execution());\n        _thread->join();\n        _thread = nullptr;\n    }\n    return _exit_code;\n}\n\ntest_runner& global_test_runner() {\n    return instance;\n}\n\n}\n\n}\n\n// Path: src/testing/entry_point.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * Copyright (C) 2018 ScyllaDB Ltd.\n */\n\n#include <seastar/testing/entry_point.hh>\n#include <seastar/testing/seastar_test.hh>\n#include <seastar/testing/test_runner.hh>\n\nnamespace seastar {\n\nnamespace testing {\n\nstatic bool init_unit_test_suite() {\n    auto&& ts = boost::unit_test::framework::master_test_suite();\n    return global_test_runner().start(ts.argc, ts.argv);\n}\n\nstatic void dummy_handler(int) {\n    // This handler should have been replaced.\n    _exit(1);\n}\n\nstatic void install_dummy_handler(int sig) {\n    struct sigaction sa {};\n    sa.sa_handler = dummy_handler;\n    sigaction(sig, &sa, nullptr);\n}\n\nint entry_point(int argc, char** argv) {\n#ifndef SEASTAR_ASAN_ENABLED\n    // Before we call into boost, install some dummy signal\n    // handlers. This seems to be the only way to stop boost from\n    // installing its own handlers, which disables our backtrace\n    // printer. The real handler will be installed when the reactor is\n    // constructed.\n    // If we are using ASAN, it has already installed a signal handler\n    // that does its own stack printing.\n    for (int sig : {SIGSEGV, SIGABRT}) {\n        install_dummy_handler(sig);\n    }\n#else\n    (void)install_dummy_handler;\n#endif\n\n    const int boost_exit_code = ::boost::unit_test::unit_test_main(&init_unit_test_suite, argc, argv);\n    const int seastar_exit_code = seastar::testing::global_test_runner().finalize();\n    if (boost_exit_code) {\n        return boost_exit_code;\n    }\n    return seastar_exit_code;\n}\n\n}\n\n}\n\n// Path: src/testing/seastar_test.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * Copyright (C) 2015 Cloudius Systems, Ltd.\n */\n\n#include <thread>\n#include <iostream>\n\n#include <seastar/testing/entry_point.hh>\n#include <seastar/testing/seastar_test.hh>\n#include <seastar/testing/test_runner.hh>\n#include <seastar/core/future.hh>\n#include <seastar/core/on_internal_error.hh>\n#include <seastar/core/app-template.hh>\n#include <seastar/testing/on_internal_error.hh>\n\nnamespace seastar {\n\nnamespace testing {\n\nexchanger_base::exchanger_base() { }\nexchanger_base::~exchanger_base() { }\n\nvoid seastar_test::run() {\n    // HACK: please see https://github.com/cloudius-systems/seastar/issues/10\n    BOOST_REQUIRE(true);\n\n    // HACK: please see https://github.com/cloudius-systems/seastar/issues/10\n    boost::program_options::variables_map()[\"dummy\"];\n\n    set_abort_on_internal_error(true);\n\n    global_test_runner().run_sync([this] {\n        return run_test_case();\n    });\n}\n\nseastar_test::seastar_test(const char* test_name, const char* test_file, int test_line)\n    : seastar_test(test_name, test_file, test_line, boost::unit_test::decorator::collector_t::instance()) {}\n\nseastar_test::seastar_test(const char* test_name, const char* test_file, int test_line,\n                           boost::unit_test::decorator::collector_t& decorators)\n    : _test_file{test_file} {\n    auto test = boost::unit_test::make_test_case([this] { run(); }, test_name, test_file, test_line);\n    decorators.store_in(*test);\n    decorators.reset();\n    boost::unit_test::framework::current_auto_test_suite().add(test);\n}\n\nconst std::string& seastar_test::get_name() {\n    const auto& current_test = boost::unit_test::framework::current_test_unit();\n    return current_test.p_name.get();\n}\n\nnamespace exception_predicate {\n\nstd::function<bool(const std::exception&)> message_equals(std::string_view expected_message) {\n    return [expected_message] (const std::exception& e) {\n        std::string error = e.what();\n        if (error == expected_message) {\n            return true;\n        } else {\n            std::cerr << \"Expected \\\"\" << expected_message << \"\\\" but got \\\"\" << error << '\"' << std::endl;\n            return false;\n        }\n    };\n}\n\nstd::function<bool(const std::exception&)> message_contains(std::string_view expected_message) {\n    return [expected_message] (const std::exception& e) {\n        std::string error = e.what();\n        if (error.find(expected_message.data()) != std::string::npos) {\n            return true;\n        } else {\n            std::cerr << \"Expected \\\"\" << expected_message << \"\\\" but got \\\"\" << error << '\"' << std::endl;\n            return false;\n        }\n    };\n}\n\n} // exception_predicate\n\nscoped_no_abort_on_internal_error::scoped_no_abort_on_internal_error() noexcept\n    : _prev(set_abort_on_internal_error(false))\n{\n}\n\nscoped_no_abort_on_internal_error::~scoped_no_abort_on_internal_error() {\n    set_abort_on_internal_error(_prev);\n}\n\n}\n\n}\n\n// Path: src/testing/random.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2020 Cloudius Systems, Ltd.\n */\n\n#include <seastar/testing/test_runner.hh>\n\nnamespace seastar {\n\nnamespace testing {\n\nthread_local std::default_random_engine local_random_engine;\n\n} // namespace testing\n\n} // namespace seastar\n\n// Path: src/rpc/rpc.cc\n#include <seastar/rpc/rpc.hh>\n#include <seastar/core/align.hh>\n#include <seastar/core/seastar.hh>\n#include <seastar/core/print.hh>\n#include <seastar/core/future-util.hh>\n#include <seastar/core/metrics.hh>\n#include <boost/range/adaptor/map.hpp>\n\n#if FMT_VERSION >= 90000\ntemplate <> struct fmt::formatter<seastar::rpc::streaming_domain_type> : fmt::ostream_formatter {};\n#endif\n\nnamespace seastar {\n\nnamespace rpc {\n\n    void logger::operator()(const client_info& info, id_type msg_id, const sstring& str) const {\n        log(format(\"client {} msg_id {}:  {}\", info.addr, msg_id, str));\n    }\n\n    void logger::operator()(const client_info& info, id_type msg_id, log_level level, std::string_view str) const {\n        log(level, \"client {} msg_id {}:  {}\", info.addr, msg_id, str);\n    }\n\n    void logger::operator()(const client_info& info, const sstring& str) const {\n        (*this)(info.addr, str);\n    }\n\n    void logger::operator()(const client_info& info, log_level level, std::string_view str) const {\n        (*this)(info.addr, level, str);\n    }\n\n    void logger::operator()(const socket_address& addr, const sstring& str) const {\n        log(format(\"client {}: {}\", addr, str));\n    }\n\n    void logger::operator()(const socket_address& addr, log_level level, std::string_view str) const {\n        log(level, \"client {}: {}\", addr, str);\n    }\n\n  no_wait_type no_wait;\n\n  snd_buf::snd_buf(size_t size_) : size(size_) {\n      if (size <= chunk_size) {\n          bufs = temporary_buffer<char>(size);\n      } else {\n          std::vector<temporary_buffer<char>> v;\n          v.reserve(align_up(size_t(size), chunk_size) / chunk_size);\n          while (size_) {\n              v.push_back(temporary_buffer<char>(std::min(chunk_size, size_)));\n              size_ -= v.back().size();\n          }\n          bufs = std::move(v);\n      }\n  }\n\n  snd_buf::snd_buf(snd_buf&&) noexcept = default;\n  snd_buf& snd_buf::operator=(snd_buf&&) noexcept = default;\n\n  temporary_buffer<char>& snd_buf::front() {\n      auto* one = std::get_if<temporary_buffer<char>>(&bufs);\n      if (one) {\n          return *one;\n      } else {\n          return std::get<std::vector<temporary_buffer<char>>>(bufs).front();\n      }\n  }\n\n  // Make a copy of a remote buffer. No data is actually copied, only pointers and\n  // a deleter of a new buffer takes care of deleting the original buffer\n  template<typename T> // T is either snd_buf or rcv_buf\n  \nT make_shard_local_buffer_copy(foreign_ptr<std::unique_ptr<T>> org) {\n      if (org.get_owner_shard() == this_shard_id()) {\n          return std::move(*org);\n      }\n      T buf(org->size);\n      auto* one = std::get_if<temporary_buffer<char>>(&org->bufs);\n\n      if (one) {\n          buf.bufs = temporary_buffer<char>(one->get_write(), one->size(), make_object_deleter(std::move(org)));\n      } else {\n          auto& orgbufs = std::get<std::vector<temporary_buffer<char>>>(org->bufs);\n          std::vector<temporary_buffer<char>> newbufs;\n          newbufs.reserve(orgbufs.size());\n          deleter d = make_object_deleter(std::move(org));\n          for (auto&& b : orgbufs) {\n              newbufs.push_back(temporary_buffer<char>(b.get_write(), b.size(), d.share()));\n          }\n          buf.bufs = std::move(newbufs);\n      }\n\n      return buf;\n  }\n\n  template snd_buf make_shard_local_buffer_copy(foreign_ptr<std::unique_ptr<snd_buf>>);\n  template rcv_buf make_shard_local_buffer_copy(foreign_ptr<std::unique_ptr<rcv_buf>>);\n\n  static void log_exception(connection& c, log_level level, const char* log, std::exception_ptr eptr) {\n      const char* s;\n      try {\n          std::rethrow_exception(eptr);\n      } catch (std::exception& ex) {\n          s = ex.what();\n      } catch (...) {\n          s = \"unknown exception\";\n      }\n      auto formatted = format(\"{}: {}\", log, s);\n      c.get_logger()(c.peer_address(), level, std::string_view(formatted.data(), formatted.size()));\n  }\n\n  snd_buf connection::compress(snd_buf buf) {\n      if (_compressor) {\n          buf = _compressor->compress(4, std::move(buf));\n          static_assert(snd_buf::chunk_size >= 4, \"send buffer chunk size is too small\");\n          write_le<uint32_t>(buf.front().get_write(), buf.size - 4);\n          return buf;\n      }\n      return buf;\n  }\n\n  future<> connection::send_buffer(snd_buf buf) {\n      auto* b = std::get_if<temporary_buffer<char>>(&buf.bufs);\n      if (b) {\n          return _write_buf.write(std::move(*b));\n      } else {\n          return do_with(std::move(std::get<std::vector<temporary_buffer<char>>>(buf.bufs)),\n                  [this] (std::vector<temporary_buffer<char>>& ar) {\n              return do_for_each(ar.begin(), ar.end(), [this] (auto& b) {\n                  return _write_buf.write(std::move(b));\n              });\n          });\n      }\n  }\n\n  future<> connection::send_entry(outgoing_entry& d) noexcept {\n    return futurize_invoke([this, &d] {\n      if (d.buf.size && _propagate_timeout) {\n          static_assert(snd_buf::chunk_size >= sizeof(uint64_t), \"send buffer chunk size is too small\");\n          if (_timeout_negotiated) {\n              auto expire = d.t.get_timeout();\n              uint64_t left = 0;\n              if (expire != typename timer<rpc_clock_type>::time_point()) {\n                  left = std::chrono::duration_cast<std::chrono::milliseconds>(expire - timer<rpc_clock_type>::clock::now()).count();\n              }\n              write_le<uint64_t>(d.buf.front().get_write(), left);\n          } else {\n              d.buf.front().trim_front(sizeof(uint64_t));\n              d.buf.size -= sizeof(uint64_t);\n          }\n      }\n      auto buf = compress(std::move(d.buf));\n      return send_buffer(std::move(buf)).then([this] {\n          _stats.sent_messages++;\n          return _write_buf.flush();\n      });\n    });\n  }\n\n  void connection::set_negotiated() noexcept {\n      _negotiated->set_value();\n      _negotiated = std::nullopt;\n  }\n\n  future<> connection::stop_send_loop(std::exception_ptr ex) {\n      _error = true;\n      if (_connected) {\n          _fd.shutdown_output();\n      }\n      if (ex == nullptr) {\n          ex = std::make_exception_ptr(closed_error());\n      }\n      while (!_outgoing_queue.empty()) {\n          auto it = std::prev(_outgoing_queue.end());\n          // Cancel all but front entry normally. The front entry is sitting in the\n          // send_entry() and cannot be withdrawn, except when _negotiated is still\n          // engaged. In the latter case when it will be aborted below the entry's\n          // continuation will not be called and its done promise will not resolve\n          // the _outgoing_queue_ready, so do it here\n          if (it != _outgoing_queue.begin()) {\n              withdraw(it, ex);\n          } else {\n              if (_negotiated) {\n                  it->done.set_exception(ex);\n              }\n              break;\n          }\n      }\n      if (_negotiated) {\n          _negotiated->set_exception(ex);\n      }\n      return when_all(std::move(_outgoing_queue_ready), std::move(_sink_closed_future)).then([this] (std::tuple<future<>, future<bool>> res){\n          // _outgoing_queue_ready might be exceptional if queue drain or\n          // _negotiated abortion set it such\n          std::get<0>(res).ignore_ready_future();\n          // _sink_closed_future is never exceptional\n          bool sink_closed = std::get<1>(res).get();\n          return _connected && !sink_closed ? _write_buf.close() : make_ready_future();\n      });\n  }\n\n  void connection::set_socket(connected_socket&& fd) {\n      if (_connected) {\n          throw std::runtime_error(\"already connected\");\n      }\n      _fd = std::move(fd);\n      _read_buf =_fd.input();\n      _write_buf = _fd.output();\n      _connected = true;\n  }\n\n  future<> connection::send_negotiation_frame(feature_map features) {\n      auto negotiation_frame_feature_record_size = [] (const feature_map::value_type& e) {\n          return 8 + e.second.size();\n      };\n      auto extra_len = boost::accumulate(\n              features | boost::adaptors::transformed(negotiation_frame_feature_record_size),\n              uint32_t(0));\n      temporary_buffer<char> reply(sizeof(negotiation_frame) + extra_len);\n      auto p = reply.get_write();\n      p = std::copy_n(rpc_magic, 8, p);\n      write_le<uint32_t>(p, extra_len);\n      p += 4;\n      for (auto&& e : features) {\n          write_le<uint32_t>(p, static_cast<uint32_t>(e.first));\n          p += 4;\n          write_le<uint32_t>(p, e.second.size());\n          p += 4;\n          p = std::copy_n(e.second.begin(), e.second.size(), p);\n      }\n      return _write_buf.write(std::move(reply)).then([this] {\n          _stats.sent_messages++;\n          return _write_buf.flush();\n      });\n  }\n\n  void connection::withdraw(outgoing_entry::container_t::iterator it, std::exception_ptr ex) {\n      assert(it != _outgoing_queue.end());\n\n      auto pit = std::prev(it);\n      // Previous entry's (pit's) done future will schedule current entry (it)\n      // continuation. Similarly, it.done will schedule next entry continuation\n      // or will resolve _outgoing_queue_ready future.\n      //\n      // To withdraw \"it\" we need to do two things:\n      // - make pit.done resolve it->next (some time later)\n      // - resolve \"it\"'s continuation right now\n      //\n      // The latter is achieved by resolving pit.done immediatelly, the former\n      // by moving it.done into pit.done. For simplicity (verging on obscurity?)\n      // both done's are just swapped and \"it\" resolves its new promise\n\n      std::swap(it->done, pit->done);\n      it->uncancellable();\n      it->unlink();\n      if (ex == nullptr) {\n          it->done.set_value();\n      } else {\n          it->done.set_exception(ex);\n      }\n  }\n\n  future<> connection::send(snd_buf buf, std::optional<rpc_clock_type::time_point> timeout, cancellable* cancel) {\n      if (!_error) {\n          if (timeout && *timeout <= rpc_clock_type::now()) {\n              return make_ready_future<>();\n          }\n\n          auto p = std::make_unique<outgoing_entry>(std::move(buf));\n          auto& d = *p;\n          _outgoing_queue.push_back(d);\n          _outgoing_queue_size++;\n          auto deleter = [this, it = _outgoing_queue.iterator_to(d)] {\n              // Front entry is most likely (unless _negotiated is unresolved, check enqueue_zero_frame()) sitting\n              // inside send_entry() continuations and thus it cannot be cancelled.\n              if (it != _outgoing_queue.begin()) {\n                  withdraw(it);\n              }\n          };\n\n          if (timeout) {\n              auto& t = d.t;\n              t.set_callback(deleter);\n              t.arm(timeout.value());\n          }\n          if (cancel) {\n              cancel->cancel_send = std::move(deleter);\n              cancel->send_back_pointer = &d.pcancel;\n              d.pcancel = cancel;\n          }\n\n          // New entry should continue (do its .then() lambda) after _outgoing_queue_ready\n          // resolves. Next entry will need to do the same after this entry's done resolves.\n          // Thus -- replace _outgoing_queue_ready with d's future and chain its continuation\n          // on ..._ready's old value.\n          return std::exchange(_outgoing_queue_ready, d.done.get_future()).then([this, p = std::move(p)] () mutable {\n              _outgoing_queue_size--;\n              if (__builtin_expect(!p->is_linked(), false)) {\n                  // If withdrawn the entry is unlinked and this lambda is fired right at once\n                  return make_ready_future<>();\n              }\n\n              p->uncancellable();\n              return send_entry(*p).then_wrapped([this, p = std::move(p)] (auto f) mutable {\n                  if (f.failed()) {\n                      f.ignore_ready_future();\n                      abort();\n                  }\n                  p->done.set_value();\n              });\n          });\n      } else {\n          return make_exception_future<>(closed_error());\n      }\n  }\n\n  void connection::abort() {\n      if (!_error) {\n          _error = true;\n          _fd.shutdown_input();\n      }\n  }\n\n  future<> connection::stop() noexcept {\n      try {\n          abort();\n      } catch (...) {\n          log_exception(*this, log_level::error, \"fail to shutdown connection while stopping\", std::current_exception());\n      }\n      return _stopped.get_future();\n  }\n\n  template<typename Connection>\n  static bool verify_frame(Connection& c, temporary_buffer<char>& buf, size_t expected, const char* log) {\n      if (buf.size() != expected) {\n          if (buf.size() != 0) {\n              c.get_logger()(c.peer_address(), log);\n          }\n          return false;\n      }\n      return true;\n  }\n\n  template<typename Connection>\n  static\n  future<feature_map>\n  receive_negotiation_frame(Connection& c, input_stream<char>& in) {\n      return in.read_exactly(sizeof(negotiation_frame)).then([&c, &in] (temporary_buffer<char> neg) {\n          if (!verify_frame(c, neg, sizeof(negotiation_frame), \"unexpected eof during negotiation frame\")) {\n              return make_exception_future<feature_map>(closed_error());\n          }\n          negotiation_frame frame;\n          std::copy_n(neg.get_write(), sizeof(frame.magic), frame.magic);\n          frame.len = read_le<uint32_t>(neg.get_write() + 8);\n          if (std::memcmp(frame.magic, rpc_magic, sizeof(frame.magic)) != 0) {\n              c.get_logger()(c.peer_address(), format(\"wrong protocol magic: {:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}\",\n                    frame.magic[0], frame.magic[1], frame.magic[2], frame.magic[3], frame.magic[4], frame.magic[5], frame.magic[6], frame.magic[7]));\n              return make_exception_future<feature_map>(closed_error());\n          }\n          auto len = frame.len;\n          return in.read_exactly(len).then([&c, len] (temporary_buffer<char> extra) {\n              if (extra.size() != len) {\n                  c.get_logger()(c.peer_address(), \"unexpected eof during negotiation frame\");\n                  return make_exception_future<feature_map>(closed_error());\n              }\n              feature_map map;\n              auto p = extra.get();\n              auto end = p + extra.size();\n              while (p != end) {\n                  if (end - p < 8) {\n                      c.get_logger()(c.peer_address(), \"bad feature data format in negotiation frame\");\n                      return make_exception_future<feature_map>(closed_error());\n                  }\n                  auto feature = static_cast<protocol_features>(read_le<uint32_t>(p));\n                  auto f_len = read_le<uint32_t>(p + 4);\n                  p += 8;\n                  if (f_len > end - p) {\n                      c.get_logger()(c.peer_address(), \"buffer underflow in feature data in negotiation frame\");\n                      return make_exception_future<feature_map>(closed_error());\n                  }\n                  auto data = sstring(p, f_len);\n                  p += f_len;\n                  map.emplace(feature, std::move(data));\n              }\n              return make_ready_future<feature_map>(std::move(map));\n          });\n      });\n  }\n\n  inline future<rcv_buf>\n  read_rcv_buf(input_stream<char>& in, uint32_t size) {\n      return in.read_up_to(size).then([&, size] (temporary_buffer<char> data) mutable {\n          rcv_buf rb(size);\n          if (data.size() == 0) {\n              return make_ready_future<rcv_buf>(rcv_buf());\n          } else if (data.size() == size) {\n              rb.bufs = std::move(data);\n              return make_ready_future<rcv_buf>(std::move(rb));\n          } else {\n              size -= data.size();\n              std::vector<temporary_buffer<char>> v;\n              v.push_back(std::move(data));\n              rb.bufs = std::move(v);\n              return do_with(std::move(rb), std::move(size), [&in] (rcv_buf& rb, uint32_t& left) {\n                  return repeat([&] () {\n                      return in.read_up_to(left).then([&] (temporary_buffer<char> data) {\n                          if (!data.size()) {\n                              rb.size -= left;\n                              return stop_iteration::yes;\n                          } else {\n                              left -= data.size();\n                              std::get<std::vector<temporary_buffer<char>>>(rb.bufs).push_back(std::move(data));\n                              return left ? stop_iteration::no : stop_iteration::yes;\n                          }\n                      });\n                  }).then([&rb] {\n                      return std::move(rb);\n                  });\n              });\n          }\n      });\n  }\n\n  template<typename FrameType>\n  future<typename FrameType::return_type>\n  connection::read_frame(socket_address info, input_stream<char>& in) {\n      auto header_size = FrameType::header_size();\n      return in.read_exactly(header_size).then([this, header_size, info, &in] (temporary_buffer<char> header) {\n          if (header.size() != header_size) {\n              if (header.size() != 0) {\n                  _logger(info, format(\"unexpected eof on a {} while reading header: expected {:d} got {:d}\", FrameType::role(), header_size, header.size()));\n              }\n              return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n          }\n          auto [size, h] = FrameType::decode_header(header.get());\n          if (!size) {\n              return make_ready_future<typename FrameType::return_type>(FrameType::make_value(h, rcv_buf()));\n          } else {\n              return read_rcv_buf(in, size).then([this, info, h = std::move(h), size] (rcv_buf rb) {\n                  if (rb.size != size) {\n                      _logger(info, format(\"unexpected eof on a {} while reading data: expected {:d} got {:d}\", FrameType::role(), size, rb.size));\n                      return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n                  } else {\n                      return make_ready_future<typename FrameType::return_type>(FrameType::make_value(h, std::move(rb)));\n                  }\n              });\n          }\n      });\n  }\n\n  template<typename FrameType>\n  future<typename FrameType::return_type>\n  connection::read_frame_compressed(socket_address info, std::unique_ptr<compressor>& compressor, input_stream<char>& in) {\n      if (compressor) {\n          return in.read_exactly(4).then([this, info, &in, &compressor] (temporary_buffer<char> compress_header) {\n              if (compress_header.size() != 4) {\n                  if (compress_header.size() != 0) {\n                      _logger(info, format(\"unexpected eof on a {} while reading compression header: expected 4 got {:d}\", FrameType::role(), compress_header.size()));\n                  }\n                  return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n              }\n              auto ptr = compress_header.get();\n              auto size = read_le<uint32_t>(ptr);\n              return read_rcv_buf(in, size).then([this, size, &compressor, info, &in] (rcv_buf compressed_data) {\n                  if (compressed_data.size != size) {\n                      _logger(info, format(\"unexpected eof on a {} while reading compressed data: expected {:d} got {:d}\", FrameType::role(), size, compressed_data.size));\n                      return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n                  }\n                  auto eb = compressor->decompress(std::move(compressed_data));\n                  if (eb.size == 0) {\n                      // Empty frames might be sent as means of communication between the compressors, and should be skipped by the RPC layer.\n                      // We skip the empty frame here. We recursively restart the function, as if the empty frame didn't happen.\n                      // The yield() is here to limit the stack depth of the recursion to 1.\n                      return yield().then([this, info, &in, &compressor] { return read_frame_compressed<FrameType>(info, compressor, in); });\n                  }\n                  net::packet p;\n                  auto* one = std::get_if<temporary_buffer<char>>(&eb.bufs);\n                  if (one) {\n                      p = net::packet(std::move(p), std::move(*one));\n                  } else {\n                      auto&& bufs = std::get<std::vector<temporary_buffer<char>>>(eb.bufs);\n                      p.reserve(bufs.size());\n                      for (auto&& b : bufs) {\n                          p = net::packet(std::move(p), std::move(b));\n                      }\n                  }\n                  return do_with(as_input_stream(std::move(p)), [this, info] (input_stream<char>& in) {\n                      return read_frame<FrameType>(info, in);\n                  });\n              });\n          });\n      } else {\n          return read_frame<FrameType>(info, in);\n      }\n  }\n\n  struct stream_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = opt_buf_type;\n      struct header_type {\n          bool eos;\n      };\n      static size_t header_size() {\n          return 4;\n      }\n      static const char* role() {\n          return \"stream\";\n      }\n      static auto empty_value() {\n          return std::nullopt;\n      }\n      static std::pair<uint32_t, header_type> decode_header(const char* ptr) {\n          auto size = read_le<uint32_t>(ptr);\n          return size != -1U ? std::make_pair(size, header_type{false}) : std::make_pair(0U, header_type{true});\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          if (t.eos) {\n              data.size = -1U;\n          }\n          return data;\n      }\n  };\n\n  future<std::optional<rcv_buf>>\n  connection::read_stream_frame_compressed(input_stream<char>& in) {\n      return read_frame_compressed<stream_frame>(peer_address(), _compressor, in);\n  }\n\n  future<> connection::stream_close() {\n      auto f = make_ready_future<>();\n      if (!error()) {\n          promise<bool> p;\n          _sink_closed_future = p.get_future();\n          // stop_send_loop(), which also calls _write_buf.close(), and this code can run in parallel.\n          // Use _sink_closed_future to serialize them and skip second call to close()\n          f = _write_buf.close().finally([p = std::move(p)] () mutable { p.set_value(true);});\n      }\n      return f.finally([this] () mutable { return stop(); });\n  }\n\n  future<> connection::stream_process_incoming(rcv_buf&& buf) {\n      // we do not want to dead lock on huge packets, so let them in\n      // but only one at a time\n      auto size = std::min(size_t(buf.size), max_stream_buffers_memory);\n      return get_units(_stream_sem, size).then([this, buf = std::move(buf)] (semaphore_units<>&& su) mutable {\n          buf.su = std::move(su);\n          return _stream_queue.push_eventually(std::move(buf));\n      });\n  }\n\n  future<> connection::handle_stream_frame() {\n      return read_stream_frame_compressed(_read_buf).then([this] (std::optional<rcv_buf> data) {\n          if (!data) {\n              _error = true;\n              return make_ready_future<>();\n          }\n          return stream_process_incoming(std::move(*data));\n      });\n  }\n\n  future<> connection::stream_receive(circular_buffer<foreign_ptr<std::unique_ptr<rcv_buf>>>& bufs) {\n      return _stream_queue.not_empty().then([this, &bufs] {\n          bool eof = !_stream_queue.consume([&bufs] (rcv_buf&& b) {\n              if (b.size == -1U) { // max fragment length marks an end of a stream\n                  return false;\n              } else {\n                  bufs.push_back(make_foreign(std::make_unique<rcv_buf>(std::move(b))));\n                  return true;\n              }\n          });\n          if (eof && !bufs.empty()) {\n              assert(_stream_queue.empty());\n              _stream_queue.push(rcv_buf(-1U)); // push eof marker back for next read to notice it\n          }\n      });\n  }\n\n  void connection::register_stream(connection_id id, xshard_connection_ptr c) {\n      _streams.emplace(id, std::move(c));\n  }\n\n  xshard_connection_ptr connection::get_stream(connection_id id) const {\n      auto it = _streams.find(id);\n      if (it == _streams.end()) {\n          throw std::logic_error(format(\"rpc stream id {} not found\", id).c_str());\n      }\n      return it->second;\n  }\n\n  // The request frame is\n  //   le64 optional timeout (see request_frame_with_timeout below)\n  //   le64 message type a.k.a. verb ID\n  //   le64 message ID\n  //   le32 payload length\n  //   ...  payload\n  struct request_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = std::tuple<std::optional<uint64_t>, uint64_t, int64_t, opt_buf_type>;\n      using header_type = std::tuple<std::optional<uint64_t>, uint64_t, int64_t>;\n      static constexpr size_t raw_header_size = sizeof(uint64_t) + sizeof(int64_t) + sizeof(uint32_t);\n      static size_t header_size() {\n          static_assert(request_frame_headroom >= raw_header_size);\n          return raw_header_size;\n      }\n      static const char* role() {\n          return \"server\";\n      }\n      static auto empty_value() {\n          return std::make_tuple(std::nullopt, uint64_t(0), 0, std::nullopt);\n      }\n      static std::pair<size_t, header_type> decode_header(const char* ptr) {\n          auto type = read_le<uint64_t>(ptr);\n          auto msgid = read_le<int64_t>(ptr + 8);\n          auto size = read_le<uint32_t>(ptr + 16);\n          return std::make_pair(size, std::make_tuple(std::nullopt, type, msgid));\n      }\n      static void encode_header(uint64_t type, int64_t msg_id, snd_buf& buf, size_t off) {\n          auto p = buf.front().get_write() + off;\n          write_le<uint64_t>(p, type);\n          write_le<int64_t>(p + 8, msg_id);\n          write_le<uint32_t>(p + 16, buf.size - raw_header_size - off);\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          return std::make_tuple(std::get<0>(t), std::get<1>(t), std::get<2>(t), std::move(data));\n      }\n  };\n\n  // This frame is used if protocol_features.TIMEOUT was negotiated\n  struct request_frame_with_timeout : request_frame {\n      using super = request_frame;\n      static constexpr size_t raw_header_size = sizeof(uint64_t) + request_frame::raw_header_size;\n      static size_t header_size() {\n          static_assert(request_frame_headroom >= raw_header_size);\n          return raw_header_size;\n      }\n      static std::pair<uint32_t, typename super::header_type> decode_header(const char* ptr) {\n          auto h = super::decode_header(ptr + 8);\n          std::get<0>(h.second) = read_le<uint64_t>(ptr);\n          return h;\n      }\n      static void encode_header(uint64_t type, int64_t msg_id, snd_buf& buf) {\n          static_assert(snd_buf::chunk_size >= raw_header_size, \"send buffer chunk size is too small\");\n          // expiration timer is encoded later\n          request_frame::encode_header(type, msg_id, buf, 8);\n      }\n  };\n\n  future<> client::request(uint64_t type, int64_t msg_id, snd_buf buf, std::optional<rpc_clock_type::time_point> timeout, cancellable* cancel) {\n      request_frame_with_timeout::encode_header(type, msg_id, buf);\n      return send(std::move(buf), timeout, cancel);\n  }\n\n  void\n  client::negotiate(feature_map provided) {\n      // record features returned here\n      for (auto&& e : provided) {\n          auto id = e.first;\n          switch (id) {\n          // supported features go here\n          case protocol_features::COMPRESS:\n              if (_options.compressor_factory) {\n                  _compressor = _options.compressor_factory->negotiate(e.second, false, [this] { return send({}); });\n              }\n              if (!_compressor) {\n                  throw std::runtime_error(format(\"RPC server responded with compression {} - unsupported\", e.second));\n              }\n              break;\n          case protocol_features::TIMEOUT:\n              _timeout_negotiated = true;\n              break;\n          case protocol_features::CONNECTION_ID: {\n              _id = deserialize_connection_id(e.second);\n              break;\n          }\n          default:\n              // nothing to do\n              ;\n          }\n      }\n  }\n\n  future<> client::negotiate_protocol(feature_map features) {\n      return send_negotiation_frame(std::move(features)).then([this] {\n          return receive_negotiation_frame(*this, _read_buf).then([this] (feature_map features) {\n              return negotiate(std::move(features));\n          });\n      });\n  }\n\n  // The response frame is\n  //   le64 message ID\n  //   le32 payload size\n  //   ...  payload\n  struct response_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = std::tuple<int64_t, opt_buf_type>;\n      using header_type = std::tuple<int64_t>;\n      static constexpr size_t raw_header_size = sizeof(int64_t) + sizeof(uint32_t);\n      static size_t header_size() {\n          static_assert(response_frame_headroom >= raw_header_size);\n          return raw_header_size;\n      }\n      static const char* role() {\n          return \"client\";\n      }\n      static auto empty_value() {\n          return std::make_tuple(0, std::nullopt);\n      }\n      static std::pair<uint32_t, header_type> decode_header(const char* ptr) {\n          auto msgid = read_le<int64_t>(ptr);\n          auto size = read_le<uint32_t>(ptr + 8);\n          return std::make_pair(size, std::make_tuple(msgid));\n      }\n      static void encode_header(int64_t msg_id, snd_buf& data) {\n          static_assert(snd_buf::chunk_size >= raw_header_size, \"send buffer chunk size is too small\");\n          auto p = data.front().get_write();\n          write_le<int64_t>(p, msg_id);\n          write_le<uint32_t>(p + 8, data.size - raw_header_size);\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          return std::make_tuple(std::get<0>(t), std::move(data));\n      }\n  };\n\n\n  future<response_frame::return_type>\n  client::read_response_frame_compressed(input_stream<char>& in) {\n      return read_frame_compressed<response_frame>(_server_addr, _compressor, in);\n  }\n\n  stats client::get_stats() const {\n      stats res = _stats;\n      res.wait_reply = incoming_queue_length();\n      res.pending = outgoing_queue_length();\n      return res;\n  }\n\n  void client::wait_for_reply(id_type id, std::unique_ptr<reply_handler_base>&& h, std::optional<rpc_clock_type::time_point> timeout, cancellable* cancel) {\n      if (timeout) {\n          h->t.set_callback(std::bind(std::mem_fn(&client::wait_timed_out), this, id));\n          h->t.arm(timeout.value());\n      }\n      if (cancel) {\n          cancel->cancel_wait = [this, id] {\n              _outstanding[id]->cancel();\n              _outstanding.erase(id);\n          };\n          h->pcancel = cancel;\n          cancel->wait_back_pointer = &h->pcancel;\n      }\n      _outstanding.emplace(id, std::move(h));\n  }\n  void client::wait_timed_out(id_type id) {\n      _stats.timeout++;\n      _outstanding[id]->timeout();\n      _outstanding.erase(id);\n  }\n\n  future<> client::stop() noexcept {\n      _error = true;\n      try {\n          _socket.shutdown();\n      } catch(...) {\n          log_exception(*this, log_level::error, \"fail to shutdown connection while stopping\", std::current_exception());\n      }\n      return _stopped.get_future();\n  }\n\n  void client::abort_all_streams() {\n      while (!_streams.empty()) {\n          auto&& s = _streams.begin();\n          assert(s->second->get_owner_shard() == this_shard_id()); // abort can be called only locally\n          s->second->get()->abort();\n          _streams.erase(s);\n      }\n  }\n\n  void client::deregister_this_stream() {\n      if (_parent) {\n          _parent->_streams.erase(_id);\n      }\n  }\n\n  // This is the enlightened copy of the connection::send() method. Its intention is to\n  // keep a dummy entry in front of the queue while connect+negotiate is happenning so\n  // that all subsequent entries could abort on timeout or explicit cancellation.\n  void client::enqueue_zero_frame() {\n      if (_error) {\n          return;\n      }\n\n      auto p = std::make_unique<outgoing_entry>(snd_buf(0));\n      auto& d = *p;\n      _outgoing_queue.push_back(d);\n\n      // Make it in the background. Even if the client is stopped it will pick\n      // up all the entries hanging around\n      (void)std::exchange(_outgoing_queue_ready, d.done.get_future()).then_wrapped([p = std::move(p)] (auto f) mutable {\n          if (f.failed()) {\n              f.ignore_ready_future();\n          } else {\n              p->done.set_value();\n          }\n      });\n  }\n\n  struct client::metrics::domain {\n      metrics::domain_list_t list;\n      stats dead;\n      seastar::metrics::metric_groups metric_groups;\n\n      static thread_local std::unordered_map<sstring, domain> all;\n      static domain& find_or_create(sstring name);\n\n      stats::counter_type count_all(stats::counter_type stats::* field) noexcept {\n          stats::counter_type res = dead.*field;\n          for (const auto& m : list) {\n              res += m._c._stats.*field;\n          }\n          return res;\n      }\n\n      size_t count_all_fn(size_t (client::*fn)(void) const) noexcept {\n          size_t res = 0;\n          for (const auto& m : list) {\n              res += (m._c.*fn)();\n          }\n          return res;\n      }\n\n      domain(sstring name)\n      {\n          namespace sm = seastar::metrics;\n          auto domain_l = sm::label(\"domain\")(name);\n\n          metric_groups.add_group(\"rpc_client\", {\n                sm::make_gauge(\"count\", [this] { return list.size(); },\n                        sm::description(\"Total number of clients\"), { domain_l }),\n                sm::make_counter(\"sent_messages\", std::bind(&domain::count_all, this, &stats::sent_messages),\n                        sm::description(\"Total number of messages sent\"), { domain_l }),\n                sm::make_counter(\"replied\", std::bind(&domain::count_all, this, &stats::replied),\n                        sm::description(\"Total number of responses received\"), { domain_l }),\n                sm::make_counter(\"exception_received\", std::bind(&domain::count_all, this, &stats::exception_received),\n                        sm::description(\"Total number of exceptional responses received\"), { domain_l }).set_skip_when_empty(),\n                sm::make_counter(\"timeout\", std::bind(&domain::count_all, this, &stats::timeout),\n                        sm::description(\"Total number of timeout responses\"), { domain_l }).set_skip_when_empty(),\n                sm::make_gauge(\"pending\", std::bind(&domain::count_all_fn, this, &client::outgoing_queue_length),\n                    sm::description(\"Number of queued outbound messages\"), { domain_l }),\n                sm::make_gauge(\"wait_reply\", std::bind(&domain::count_all_fn, this, &client::incoming_queue_length),\n                    sm::description(\"Number of replies waiting for\"), { domain_l }),\n          });\n      }\n  };\n\n  thread_local std::unordered_map<sstring, client::metrics::domain> client::metrics::domain::all;\n\n  client::metrics::domain& client::metrics::domain::find_or_create(sstring name) {\n      auto i = all.try_emplace(name, name);\n      return i.first->second;\n  }\n\n  client::metrics::metrics(const client& c)\n        : _c(c)\n        , _domain(domain::find_or_create(_c._options.metrics_domain))\n  {\n      _domain.list.push_back(*this);\n  }\n\n  client::metrics::~metrics() {\n      _domain.dead.replied += _c._stats.replied;\n      _domain.dead.exception_received += _c._stats.exception_received;\n      _domain.dead.sent_messages += _c._stats.sent_messages;\n      _domain.dead.timeout += _c._stats.timeout;\n  }\n\n  client::client(const logger& l, void* s, client_options ops, socket socket, const socket_address& addr, const socket_address& local)\n  : rpc::connection(l, s), _socket(std::move(socket)), _server_addr(addr), _local_addr(local), _options(ops), _metrics(*this)\n  {\n       _socket.set_reuseaddr(ops.reuseaddr);\n      // Run client in the background.\n      // Communicate result via _stopped.\n      // The caller has to call client::stop() to synchronize.\n      (void)_socket.connect(addr, local).then([this, ops = std::move(ops)] (connected_socket fd) {\n          fd.set_nodelay(ops.tcp_nodelay);\n          if (ops.keepalive) {\n              fd.set_keepalive(true);\n              fd.set_keepalive_parameters(ops.keepalive.value());\n          }\n          set_socket(std::move(fd));\n\n          feature_map features;\n          if (_options.compressor_factory) {\n              features[protocol_features::COMPRESS] = _options.compressor_factory->supported();\n          }\n          if (_options.send_timeout_data) {\n              features[protocol_features::TIMEOUT] = \"\";\n          }\n          if (_options.stream_parent) {\n              features[protocol_features::STREAM_PARENT] = serialize_connection_id(_options.stream_parent);\n          }\n          if (!_options.isolation_cookie.empty()) {\n              features[protocol_features::ISOLATION] = _options.isolation_cookie;\n          }\n\n          return negotiate_protocol(std::move(features)).then([this] {\n...\n", "needle_token_start": 5395, "needle_token_end": 5702, "code_context_ntokens": 16135, "output": ["```cpp\ntemplate<typename T> // T is either snd_buf or rcv_buf\nT make_shard_local_buffer_copy(foreign_ptr<std::unique_ptr<T>> org) {\n    if (org.get_owner_shard() == this_shard_id()) {\n        return std::move(*org);\n    }\n    T buf(org->size);\n    auto* one = std::get_if<temporary_buffer<char>>(&org->bufs);\n\n    if (one) {\n        buf.bufs = temporary_buffer<char>(one->get_write(), one->size(), make_object_deleter(std::move(org)));\n    } else {\n        auto& orgbufs = std::get<std::vector<temporary_buffer<char>>>(org->bufs);\n        std::vector<temporary_buffer<char>> newbufs;\n        newbufs.reserve(orgbufs.size());\n        deleter d = make_object_deleter(std::move(org));\n        for (auto&& b : orgbufs) {\n            newbufs.push_back(temporary_buffer<char>(b.get_write(), b.size(), d.share()));\n        }\n        buf.bufs = std::move(newbufs);\n    }\n\n    return buf;\n}\n```"]}
{"repo": "scylladb/seastar", "name": "describe_enum_value", "language": "cpp", "path": "src/core/program_options.cc", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To add an enumeration option to a program's options description, optionally setting a default value if provided.\n2. **Input**: Takes a structured description of the option (including its name and description), the enumeration type, and optionally a pointer to a default value of the enumeration type.\n3. **Output**: There is no direct output; however, the function modifies the options description to include the new enumeration option.\n4. **Procedure**: The function checks if a default value is provided. If so, it converts this enumeration value to its corresponding string representation and adds it as an option with a default value to the options description. If no default value is provided, it simply adds the option without a default value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/core/dpdk_rte.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n#ifdef SEASTAR_HAVE_DPDK\n\n#include <cinttypes>\n#include <seastar/net/dpdk.hh>\n#include <seastar/core/dpdk_rte.hh>\n#include <seastar/util/conversions.hh>\n#include <seastar/util/std-compat.hh>\n#include <rte_pci.h>\n\nnamespace seastar {\n\nnamespace dpdk {\n\nbool eal::initialized = false;\n\nvoid eal::init(cpuset cpus, const std::string& argv0, const std::optional<std::string>& hugepages_path, bool dpdk_pmd)\n{\n    if (initialized) {\n        return;\n    }\n\n    size_t cpu_count = cpus.count();\n    std::stringstream mask;\n    cpuset nibble = 0xF;\n    while (cpus.any()) {\n...\n// Path: src/core/sharded.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2018 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <boost/range/irange.hpp>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/sharded.hh>\n#include <seastar/core/loop.hh>\n#endif\n\nnamespace seastar {\n\nnamespace internal {\n\n\nfuture<>\nsharded_parallel_for_each(unsigned nr_shards, on_each_shard_func on_each_shard) noexcept(std::is_nothrow_move_constructible_v<on_each_shard_func>) {\n    return parallel_for_each(boost::irange<unsigned>(0, nr_shards), std::move(on_each_shard));\n}\n\n}\n\n}\n\n// Path: src/core/uname.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n\n/*\n * Copyright (C) 2019 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <memory>\n#include <optional>\n#include <regex>\n#include <boost/algorithm/cxx11/any_of.hpp>\n#include <sys/utsname.h>\n#include <iostream>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/internal/uname.hh>\n#endif\n\nnamespace seastar {\n\nnamespace internal {\n\nint uname_t::component_count() const {\n    if (distro_patch) {\n        return 5;\n    }\n    if (subsublevel) {\n        return 4;\n    }\n    if (sublevel) {\n        return 3;\n    }\n    return 2;\n}\n\nbool uname_t::has_distro_extra(std::string extra) const {\n    return distro_extra.find(extra) != std::string::npos;\n}\n\n// Can't use optional compares, C++17 only\nstatic int cmp(const std::optional<int>& u1, const std::optional<int>& u2) {\n    return int(u1.value_or(0) - u2.value_or(0));\n}\n\nbool uname_t::same_as_or_descendant_of(const uname_t& x) const {\n    if (version < x.version) {\n        return false; // 4.2 vs. 5.1\n    }\n    if (version == x.version && patchlevel < x.patchlevel) {\n        return false; // 4.0 vs 4.1\n    }\n    if (!has_distro_extra(x.distro_extra)) {\n        return false;\n    }\n    switch (x.component_count()) {\n    case 5:\n        return version == x.version\n                && patchlevel == x.patchlevel\n                && cmp(sublevel, x.sublevel) == 0\n                && cmp(subsublevel, x.subsublevel) == 0\n                && cmp(distro_patch, x.distro_patch) >= 0;\n    case 4:\n        return version == x.version\n                && patchlevel == x.patchlevel\n                && cmp(sublevel, x.sublevel) == 0\n                && cmp(subsublevel, x.subsublevel) >= 0;\n    case 3:\n        return version == x.version\n                && patchlevel == x.patchlevel\n                && cmp(sublevel, x.sublevel) >= 0;\n    case 2:\n        return true;\n    default:\n        return false;\n    }\n}\n\nuname_t parse_uname(const char* u) {\n    static std::regex re(R\"XX((\\d+)\\.(\\d+)(?:\\.(\\d+)(?:\\.(\\d+))?)?(?:-(\\d*)(.+))?)XX\");\n    std::cmatch m;\n    if (std::regex_match(u, m, re)) {\n        auto num = [] (std::csub_match sm) -> std::optional<int> {\n            if (sm.length() > 0) {\n                return std::atoi(sm.str().c_str());\n            } else {\n                return std::nullopt;\n            }\n        };\n        return uname_t{*num(m[1]), *num(m[2]), num(m[3]), num(m[4]), num(m[5]), m[6].str()};\n    } else {\n        return uname_t{0, 0, std::nullopt, std::nullopt, std::nullopt, \"\"};\n    }\n}\n\n\nbool uname_t::whitelisted(std::initializer_list<const char*> wl) const {\n    return boost::algorithm::any_of(wl, [this] (const char* v) {\n        return same_as_or_descendant_of(parse_uname(v));\n    });\n}\n\nstd::ostream& operator<<(std::ostream& os, const uname_t& u) {\n    os << u.version << \".\" << u.patchlevel;\n    if (u.sublevel) {\n        os << \".\" << *u.sublevel;\n    }\n    if (u.subsublevel) {\n        os << \".\" << *u.subsublevel;\n    }\n    if (u.distro_patch || !u.distro_extra.empty()) {\n        os << \"-\";\n    }\n    if (u.distro_patch) {\n        os << *u.distro_patch;\n    }\n    os << u.distro_extra;\n    return os;\n}\n\n\nuname_t kernel_uname() {\n    struct ::utsname buf;\n    ::uname(&buf);\n    return parse_uname(buf.release);\n}\n\n}\n}\n\n// Path: src/core/app-template.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <fstream>\n#include <cstdlib>\n#include <chrono>\n#include <iostream>\n#include <fmt/format.h>\n#include <boost/program_options.hpp>\n#include <boost/make_shared.hpp>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/app-template.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/alien.hh>\n#include <seastar/core/scollectd.hh>\n#include <seastar/core/metrics_api.hh>\n#include <seastar/core/print.hh>\n#include <seastar/util/log.hh>\n#include <seastar/util/log-cli.hh>\n#include <seastar/net/native-stack.hh>\n#include \"program_options.hh\"\n#endif\n\nnamespace seastar {\n\nnamespace bpo = boost::program_options;\n\nusing namespace std::chrono_literals;\n\nstatic\napp_template::seastar_options\nseastar_options_from_config(app_template::config cfg) {\n    app_template::seastar_options opts;\n    opts.name = std::move(cfg.name);\n    opts.description = std::move(cfg.description);\n    opts.auto_handle_sigint_sigterm = std::move(cfg.auto_handle_sigint_sigterm);\n    opts.reactor_opts.task_quota_ms.set_default_value(cfg.default_task_quota / 1ms);\n    opts.reactor_opts.max_networking_io_control_blocks.set_default_value(cfg.max_networking_aio_io_control_blocks);\n    opts.smp_opts.reserve_additional_memory_per_shard = cfg.reserve_additional_memory_per_shard;\n    return opts;\n}\n\napp_template::seastar_options::seastar_options()\n    : program_options::option_group(nullptr, \"seastar\")\n    , reactor_opts(this)\n    , metrics_opts(this)\n    , smp_opts(this)\n    , scollectd_opts(this)\n    , log_opts(this)\n{\n}\n\napp_template::app_template(app_template::seastar_options opts)\n    : _alien(std::make_unique<alien::instance>())\n    , _smp(std::make_shared<smp>(*_alien))\n    , _opts(std::move(opts))\n    , _app_opts(_opts.name + \" options\")\n    , _conf_reader(get_default_configuration_reader()) {\n\n        if (!alien::internal::default_instance) {\n            alien::internal::default_instance = _alien.get();\n        }\n        _app_opts.add_options()\n                (\"help,h\", \"show help message\")\n                ;\n        _app_opts.add_options()\n                (\"help-seastar\", \"show help message about seastar options\")\n                ;\n        _app_opts.add_options()\n                (\"help-loggers\", \"print a list of logger names and exit\")\n                ;\n\n        {\n            program_options::options_description_building_visitor visitor;\n            _opts.describe(visitor);\n            _opts_conf_file.add(std::move(visitor).get_options_description());\n        }\n\n        _seastar_opts.add(_opts_conf_file);\n}\n\napp_template::app_template(app_template::config cfg)\n    : app_template(seastar_options_from_config(std::move(cfg)))\n{\n}\n\napp_template::~app_template() = default;\n\nconst app_template::seastar_options& app_template::options() const {\n    return _opts;\n}\n\napp_template::configuration_reader app_template::get_default_configuration_reader() {\n    return [this] (bpo::variables_map& configuration) {\n        auto home = std::getenv(\"HOME\");\n        if (home) {\n            std::ifstream ifs(std::string(home) + \"/.config/seastar/seastar.conf\");\n            if (ifs) {\n                bpo::store(bpo::parse_config_file(ifs, _opts_conf_file), configuration);\n            }\n            std::ifstream ifs_io(std::string(home) + \"/.config/seastar/io.conf\");\n            if (ifs_io) {\n                bpo::store(bpo::parse_config_file(ifs_io, _opts_conf_file), configuration);\n            }\n        }\n    };\n}\n\nvoid app_template::set_configuration_reader(configuration_reader conf_reader) {\n    _conf_reader = conf_reader;\n}\n\nboost::program_options::options_description& app_template::get_options_description() {\n    return _app_opts;\n}\n\nboost::program_options::options_description& app_template::get_conf_file_options_description() {\n    return _opts_conf_file;\n}\n\nboost::program_options::options_description_easy_init\napp_template::add_options() {\n    return _app_opts.add_options();\n}\n\nvoid\napp_template::add_positional_options(std::initializer_list<positional_option> options) {\n    for (auto&& o : options) {\n        _app_opts.add(boost::make_shared<bpo::option_description>(o.name, o.value_semantic, o.help));\n        _pos_opts.add(o.name, o.max_count);\n    }\n}\n\n\nbpo::variables_map&\napp_template::configuration() {\n    return *_configuration;\n}\n\nint\napp_template::run(int ac, char ** av, std::function<future<int> ()>&& func) noexcept {\n    return run_deprecated(ac, av, [func = std::move(func)] () mutable {\n        auto func_done = make_lw_shared<promise<>>();\n        engine().at_exit([func_done] { return func_done->get_future(); });\n        // No need to wait for this future.\n        // func's returned exit_code is communicated via engine().exit()\n        (void)futurize_invoke(func).finally([func_done] {\n            func_done->set_value();\n        }).then([] (int exit_code) {\n            return engine().exit(exit_code);\n        }).or_terminate();\n    });\n}\n\nint\napp_template::run(int ac, char ** av, std::function<future<> ()>&& func) noexcept {\n    return run(ac, av, [func = std::move(func)] {\n        return func().then([] () {\n            return 0;\n        });\n    });\n}\n\nint\napp_template::run_deprecated(int ac, char ** av, std::function<void ()>&& func) noexcept {\n#ifdef SEASTAR_DEBUG\n    fmt::print(std::cerr, \"WARNING: debug mode. Not for benchmarking or production\\n\");\n#endif\n    boost::program_options::options_description all_opts;\n    all_opts.add(_app_opts);\n    all_opts.add(_seastar_opts);\n\n    bpo::variables_map configuration;\n    try {\n        bpo::store(bpo::command_line_parser(ac, av)\n                    .options(all_opts)\n                    .positional(_pos_opts)\n                    .run()\n            , configuration);\n        _conf_reader(configuration);\n    } catch (bpo::error& e) {\n        fmt::print(\"error: {}\\n\\nTry --help.\\n\", e.what());\n        return 2;\n    }\n    if (configuration.count(\"help\")) {\n        if (!_opts.description.empty()) {\n            std::cout << _opts.description << \"\\n\";\n        }\n        std::cout << _app_opts << \"\\n\";\n        return 1;\n    }\n    if (configuration.count(\"help-seastar\")) {\n        std::cout << _seastar_opts << \"\\n\";\n        return 1;\n    }\n    if (configuration.count(\"help-loggers\")) {\n        log_cli::print_available_loggers(std::cout);\n        return 1;\n    }\n\n    try {\n        bpo::notify(configuration);\n    } catch (const bpo::error& ex) {\n        std::cout << ex.what() << std::endl;\n        return 1;\n    }\n\n    {\n        program_options::variables_map_extracting_visitor visitor(configuration);\n        _opts.mutate(visitor);\n    }\n    _opts.reactor_opts._argv0 = std::string(av[0]);\n    _opts.reactor_opts._auto_handle_sigint_sigterm = _opts.auto_handle_sigint_sigterm;\n    if (auto* native_stack = dynamic_cast<net::native_stack_options*>(_opts.reactor_opts.network_stack.get_selected_candidate_opts())) {\n        native_stack->_hugepages = _opts.smp_opts.hugepages;\n    }\n\n    // Needs to be before `smp::configure()`.\n    try {\n        apply_logging_settings(log_cli::extract_settings(_opts.log_opts));\n    } catch (const std::runtime_error& exn) {\n        std::cout << \"logging configuration error: \" << exn.what() << '\\n';\n        return 1;\n    }\n\n    try {\n        _smp->configure(_opts.smp_opts, _opts.reactor_opts);\n    } catch (...) {\n        std::cerr << \"Could not initialize seastar: \" << std::current_exception() << std::endl;\n        return 1;\n    }\n    _configuration = {std::move(configuration)};\n    // No need to wait for this future.\n    // func is waited on via engine().run()\n    (void)engine().when_started().then([this] {\n        return seastar::metrics::configure(_opts.metrics_opts).then([this] {\n            // set scollectd use the metrics configuration, so the later\n            // need to be set first\n            scollectd::configure( _opts.scollectd_opts);\n        });\n    }).then(\n        std::move(func)\n    ).then_wrapped([] (auto&& f) {\n        try {\n            f.get();\n        } catch (std::exception& ex) {\n            std::cout << \"program failed with uncaught exception: \" << ex.what() << \"\\n\";\n            engine().exit(1);\n        }\n    });\n    auto exit_code = engine().run();\n    _smp->cleanup();\n    return exit_code;\n}\n\n}\n\n// Path: src/core/program_options.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2021 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cstdlib>\n#include <string>\n#include <stdexcept>\n#include <boost/program_options.hpp>\n#include <boost/type.hpp>\n#include <fmt/format.h>\nmodule seastar;\n#else\n#include \"core/program_options.hh\"\n\n#include <seastar/util/log-cli.hh>\n#include <seastar/util/memory_diagnostics.hh>\n#include <seastar/core/reactor_config.hh>\n#include <seastar/core/resource.hh>\n#include <seastar/core/smp.hh>\n#endif\n\nnamespace seastar::program_options {\n\nnamespace {\n\nconst char* to_string(memory::alloc_failure_kind val) {\n    switch (val) {\n        case memory::alloc_failure_kind::none: return \"none\";\n        case memory::alloc_failure_kind::critical: return \"critical\";\n        case memory::alloc_failure_kind::all: return \"all\";\n    }\n    std::abort();\n}\n\nconst char* to_string(log_level val) {\n    switch (val) {\n        case log_level::error: return \"error\";\n        case log_level::warn: return \"warn\";\n        case log_level::info: return \"info\";\n        case log_level::debug: return \"debug\";\n        case log_level::trace: return \"trace\";\n    }\n    std::abort();\n}\n\nconst char* to_string(logger_timestamp_style val) {\n    switch (val) {\n        case logger_timestamp_style::none: return \"none\";\n        case logger_timestamp_style::boot: return \"boot\";\n        case logger_timestamp_style::real: return \"real\";\n    }\n    std::abort();\n}\n\nconst char* to_string(logger_ostream_type val) {\n    switch (val) {\n        case logger_ostream_type::none: return \"none\";\n        case logger_ostream_type::cout: return \"stdout\";\n        case logger_ostream_type::cerr: return \"stderr\";\n    }\n    std::abort();\n}\n\nmemory::alloc_failure_kind from_string(const std::string& val, boost::type<memory::alloc_failure_kind>) {\n    if (val == \"none\") {\n        return memory::alloc_failure_kind::none;\n    } else if (val == \"critical\") {\n        return memory::alloc_failure_kind::critical;\n    } else if (val == \"all\") {\n        return memory::alloc_failure_kind::all;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum memory::alloc_failure_kind: {}\", val));\n}\n\nlog_level from_string(const std::string& val, boost::type<log_level>) {\n    if (val == \"error\") {\n        return log_level::error;\n    } else if (val == \"warn\") {\n        return log_level::warn;\n    } else if (val == \"info\") {\n        return log_level::info;\n    } else if (val == \"debug\") {\n        return log_level::debug;\n    } else if (val == \"trace\") {\n        return log_level::trace;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum log_level: {}\", val));\n}\n\nlogger_timestamp_style from_string(const std::string& val, boost::type<logger_timestamp_style>) {\n    if (val == \"none\") {\n        return logger_timestamp_style::none;\n    } else if (val == \"boot\") {\n        return logger_timestamp_style::boot;\n    } else if (val == \"real\") {\n        return logger_timestamp_style::real;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum logger_timestamp_style: {}\", val));\n}\n\nlogger_ostream_type from_string(const std::string& val, boost::type<logger_ostream_type>) {\n    if (val == \"none\") {\n        return logger_ostream_type::none;\n    } else if (val == \"stdout\") {\n        return logger_ostream_type::cout;\n    } else if (val == \"stderr\") {\n        return logger_ostream_type::cerr;\n    }\n    throw std::runtime_error(fmt::format(\"Invalid value for enum logger_ostream_type: {}\", val));\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const std::string& name, const std::string& description, const Type& default_value) {\n    opts.add_options()(name.c_str(), boost::program_options::value<Type>()->default_value(default_value), description.c_str());\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d, const Type& default_value) {\n    describe_value(opts, d.name, d.description, default_value);\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const std::string& name, const std::string& description) {\n    opts.add_options()(name.c_str(), boost::program_options::value<Type>(), description.c_str());\n}\n\ntemplate <typename Type>\nvoid describe_value(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d) {\n    describe_value<Type>(opts, d.name, d.description);\n}\n\ntemplate <typename Type>\nvoid describe_value_maybe_default(bpo::options_description& opts, const std::string& name, const std::string& description, const Type* default_value) {\n    if (default_value) {\n        describe_value(opts, name, description, *default_value);\n    } else {\n        describe_value<Type>(opts, name, description);\n    }\n}\n\ntemplate <typename Type>\nvoid describe_value_maybe_default(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d, const Type* default_value) {\n    describe_value_maybe_default(opts, d.name, d.description, default_value);\n}\n\ntemplate <typename Enum>\n\nvoid describe_enum_value(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d, const Enum* default_value) {\n    if (default_value) {\n        opts.add_options()(d.name.c_str(), boost::program_options::value<std::string>()->default_value(to_string(*default_value)), d.description.c_str());\n    } else {\n        opts.add_options()(d.name.c_str(), boost::program_options::value<std::string>(), d.description.c_str());\n    }\n}\n\ntemplate <typename T>\nbool extract_value(const bpo::variables_map& values, const std::string& current_name, T& val) {\n    auto it = values.find(current_name);\n    if (it == values.end() || it->second.defaulted()) {\n        return false;\n    }\n    val = it->second.as<T>();\n    return true;\n}\n\ntemplate <typename T>\nbool extract_enum_value(const bpo::variables_map& values, const std::string& current_name, T& val) {\n    auto it = values.find(current_name);\n    if (it == values.end() || it->second.defaulted()) {\n        return false;\n    }\n    val = from_string(it->second.as<std::string>(), boost::type<T>{});\n    return true;\n}\n\n} // anonymous namespace\n\nbool options_description_building_visitor::visit_group_start(const std::string& name, bool used) {\n    _groups.push({name, bpo::options_description(name.c_str()), used});\n    return used;\n}\nvoid options_description_building_visitor::visit_group_end() {\n    if (_groups.size() == 1) {\n        return;\n    }\n    auto grp = std::move(_groups.top());\n    _groups.pop();\n    if (grp.used && grp.values) {\n        _groups.top().description.add(std::move(grp.description));\n    }\n}\n\nbool options_description_building_visitor::visit_value_metadata(const std::string& name, const std::string& description, bool used) {\n    if (!used) {\n        return false;\n    }\n    ++_groups.top().values;\n    _current_metadata.emplace(value_metadata{name, description});\n    return true;\n}\n\nvoid options_description_building_visitor::visit_value() {\n    _groups.top().description.add_options()(_current_metadata->name.c_str(), _current_metadata->description.c_str());\n}\n\nvoid options_description_building_visitor::visit_value(const bool* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const int* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const unsigned* default_value) {\n    auto name = _current_metadata->name;\n    if (_current_metadata->name == \"smp\") {\n        name = \"smp,c\";\n    }\n    describe_value_maybe_default(_groups.top().description, name, _current_metadata->description, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const float* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const double* default_value) {\n    describe_value_maybe_default(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const std::string* default_value) {\n    auto name = _current_metadata->name;\n    if (_current_metadata->name == \"memory\") {\n        name = \"memory,m\";\n    }\n    describe_value_maybe_default(_groups.top().description, name, _current_metadata->description, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const std::set<unsigned>*) {\n    describe_value<std::string>(_groups.top().description, *_current_metadata);\n}\n\nvoid options_description_building_visitor::visit_value(const memory::alloc_failure_kind* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const log_level* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const logger_timestamp_style* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const logger_ostream_type* default_value) {\n    describe_enum_value(_groups.top().description, *_current_metadata, default_value);\n}\n\nvoid options_description_building_visitor::visit_value(const std::unordered_map<sstring, log_level>*) {\n    describe_value<std::vector<std::string>>(_groups.top().description, *_current_metadata);\n}\n\nvoid options_description_building_visitor::visit_selection_value(const std::vector<std::string>& candidates, const std::size_t* selected_candidate) {\n    if (selected_candidate) {\n        describe_value<std::string>(_groups.top().description, *_current_metadata, candidates.at(*selected_candidate));\n    } else {\n        describe_value<std::string>(_groups.top().description, *_current_metadata);\n    }\n}\n\nvariables_map_extracting_visitor::variables_map_extracting_visitor(const bpo::variables_map& values) : _values(values) {\n}\n\nbool variables_map_extracting_visitor::visit_group_start(const std::string& name, bool used) {\n    return used;\n}\n\nvoid variables_map_extracting_visitor::visit_group_end() {\n}\n\nbool variables_map_extracting_visitor::visit_value_metadata(const std::string& name, bool used) {\n    if (used) {\n        _current_name = &name;\n        return true;\n    } else {\n        _current_name = nullptr;\n        return false;\n    }\n}\n\nbool variables_map_extracting_visitor::visit_value() {\n    return _values.count(*_current_name);\n}\n\nbool variables_map_extracting_visitor::visit_value(bool& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(int& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(unsigned& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(float& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(double& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(std::string& val) {\n    return extract_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(std::set<unsigned>& val) {\n    std::string raw_val;\n    if (!extract_value(_values, *_current_name, raw_val)) {\n        return false;\n    }\n    if (auto parsed_cpu_set = resource::parse_cpuset(raw_val)) {\n        val = std::move(*parsed_cpu_set);\n        return true;\n    }\n    throw std::invalid_argument(fmt::format(\"invalid value for option {}: failed to parse cpuset: {}\", *_current_name, raw_val));\n}\n\nbool variables_map_extracting_visitor::visit_value(log_level& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(logger_timestamp_style& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(logger_ostream_type& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(memory::alloc_failure_kind& val) {\n    return extract_enum_value(_values, *_current_name, val);\n}\n\nbool variables_map_extracting_visitor::visit_value(std::unordered_map<sstring, log_level>& val) {\n    std::vector<std::string> raw_val;\n    if (!extract_value(_values, *_current_name, raw_val)) {\n        return false;\n    }\n    for (const auto& e : raw_val) {\n        log_cli::parse_map_associations(e, [&val] (std::string k, std::string v) { val[std::move(k)] = log_cli::parse_log_level(v); });\n    }\n    return !val.empty();\n}\n\nbool variables_map_extracting_visitor::visit_selection_value(const std::vector<std::string>& candidates, std::size_t& selected_candidate) {\n    std::string candidate_name;\n    if (!extract_value(_values, *_current_name, candidate_name)) {\n        return false;\n    }\n    auto it = std::find(candidates.begin(), candidates.end(), candidate_name);\n    if (it == candidates.end()) {\n        throw std::invalid_argument(fmt::format(\"invalid value for option {}: selected candidate doesn't exist: {}\", *_current_name, candidate_name));\n    }\n    selected_candidate = it - candidates.begin();\n    return true;\n}\n\n} // namespace seastar::program_options\n\n// Path: src/core/future-util.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2017 ScyllaDB\n */\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cstddef>\n#include <exception>\n#include <iosfwd>\n#include <memory>\n#include <optional>\n#include <utility>\n#include <vector>\nmodule seastar;\n#else\n#include <seastar/core/future-util.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/sleep.hh>\n#include <seastar/core/print.hh>\n#include <seastar/core/semaphore.hh>\n#endif\n\nnamespace seastar {\n\nparallel_for_each_state::parallel_for_each_state(size_t n) {\n    _incomplete.reserve(n);\n}\n\nfuture<> parallel_for_each_state::get_future() {\n    auto ret = _result.get_future();\n    wait_for_one();\n    return ret;\n}\n\nvoid parallel_for_each_state::add_future(future<>&& f) {\n    _incomplete.push_back(std::move(f));\n}\n\nvoid parallel_for_each_state::wait_for_one() noexcept {\n    // Process from back to front, on the assumption that the front\n    // futures are likely to complete earlier than the back futures.\n    // If that's indeed the case, then the front futures will be\n    // available and we won't have to wait for them.\n\n    // Skip over futures that happen to be complete already.\n    while (!_incomplete.empty() && _incomplete.back().available()) {\n        if (_incomplete.back().failed()) {\n            _ex = _incomplete.back().get_exception();\n        }\n        _incomplete.pop_back();\n    }\n\n    // If there's an incompelete future, wait for it.\n    if (!_incomplete.empty()) {\n        internal::set_callback(std::move(_incomplete.back()), static_cast<continuation_base<>*>(this));\n        // This future's state will be collected in run_and_dispose(), so we can drop it.\n        _incomplete.pop_back();\n        return;\n    }\n\n    // Everything completed, report a result.\n    if (__builtin_expect(bool(_ex), false)) {\n        _result.set_exception(std::move(_ex));\n    } else {\n        _result.set_value();\n    }\n    delete this;\n}\n\nvoid parallel_for_each_state::run_and_dispose() noexcept {\n    if (_state.failed()) {\n        _ex = std::move(_state).get_exception();\n    }\n    _state = {};\n    wait_for_one();\n}\n\ntemplate <typename Clock>\nfuture<> sleep_abortable(typename Clock::duration dur) {\n    return engine().wait_for_stop(dur).then([] {\n        throw sleep_aborted();\n    }).handle_exception([] (std::exception_ptr ep) {\n        try {\n            std::rethrow_exception(ep);\n        } catch(condition_variable_timed_out&) {};\n    });\n}\n\ntemplate future<> sleep_abortable<steady_clock_type>(typename steady_clock_type::duration);\ntemplate future<> sleep_abortable<lowres_clock>(typename lowres_clock::duration);\n\ntemplate <typename Clock>\nfuture<> sleep_abortable(typename Clock::duration dur, abort_source& as) {\n    struct sleeper {\n        promise<> done;\n        timer<Clock> tmr;\n        abort_source::subscription sc;\n\n        sleeper(typename Clock::duration dur, abort_source& as)\n                : tmr([this] { done.set_value(); }) {\n            auto sc_opt = as.subscribe([this] (const std::optional<std::exception_ptr>& opt_ex) noexcept {\n                if (tmr.cancel()) {\n                    done.set_exception(opt_ex.value_or(std::make_exception_ptr(sleep_aborted())));\n                }\n            });\n            if (sc_opt) {\n                sc = std::move(*sc_opt);\n                tmr.arm(dur);\n            } else {\n                done.set_exception(sleep_aborted());\n            }\n        }\n    };\n    //FIXME: Use do_with() after #373\n    auto s = std::make_unique<sleeper>(dur, as);\n    auto fut = s->done.get_future();\n    return fut.finally([s = std::move(s)] { });\n}\n\ntemplate future<> sleep_abortable<steady_clock_type>(typename steady_clock_type::duration, abort_source&);\ntemplate future<> sleep_abortable<lowres_clock>(typename lowres_clock::duration, abort_source&);\ntemplate future<> sleep_abortable<manual_clock>(typename manual_clock::duration, abort_source&);\n\n}\n\n// Path: src/core/future.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2020 ScyllaDB\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cassert>\n#include <exception>\n#include <tuple>\n#include <type_traits>\n#include <utility>\nmodule seastar;\n#else\n#include <seastar/core/future.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/thread.hh>\n#include <seastar/core/report_exception.hh>\n#include <seastar/util/backtrace.hh>\n#endif\n\nnamespace seastar {\n\n// We can't test future_state_base directly because its private\n// destructor is protected.\nstatic_assert(std::is_nothrow_move_constructible_v<future_state<std::tuple<int>>>,\n              \"future_state's move constructor must not throw\");\n\nstatic_assert(sizeof(future_state<std::tuple<>>) <= 8, \"future_state<std::tuple<>> is too large\");\nstatic_assert(sizeof(future_state<std::tuple<long>>) <= 16, \"future_state<std::tuple<long>> is too large\");\nstatic_assert(future_state<std::tuple<>>::has_trivial_move_and_destroy, \"future_state<std::tuple<>> not trivial\");\nstatic_assert(future_state<long>::has_trivial_move_and_destroy, \"future_state<long> not trivial\");\n\n// We need to be able to move and copy std::exception_ptr in and out\n// of future/promise/continuations without that producing a new\n// exception.\nstatic_assert(std::is_nothrow_copy_constructible_v<std::exception_ptr>,\n    \"std::exception_ptr's copy constructor must not throw\");\nstatic_assert(std::is_nothrow_move_constructible_v<std::exception_ptr>,\n    \"std::exception_ptr's move constructor must not throw\");\n\nnamespace internal {\n\nstatic_assert(std::is_empty_v<uninitialized_wrapper<std::tuple<>>>, \"This should still be empty\");\n\nvoid promise_base::move_it(promise_base&& x) noexcept {\n    // Don't use std::exchange to make sure x's values are nulled even\n    // if &x == this.\n    _task = x._task;\n    x._task = nullptr;\n#ifdef SEASTAR_DEBUG_PROMISE\n    _task_shard = x._task_shard;\n#endif\n    _state = x._state;\n    x._state = nullptr;\n    _future = x._future;\n    if (auto* fut = _future) {\n        fut->detach_promise();\n        fut->_promise = this;\n    }\n}\n\nstatic void set_to_broken_promise(future_state_base& state) noexcept {\n    try {\n        // Constructing broken_promise may throw (std::logic_error ctor is not noexcept).\n        state.set_exception(std::make_exception_ptr(broken_promise{}));\n    } catch (...) {\n        state.set_exception(std::current_exception());\n    }\n}\n\npromise_base::promise_base(promise_base&& x) noexcept {\n    move_it(std::move(x));\n}\n\nvoid promise_base::clear() noexcept {\n    if (__builtin_expect(bool(_task), false)) {\n        assert(_state && !_state->available());\n        set_to_broken_promise(*_state);\n        ::seastar::schedule(std::exchange(_task, nullptr));\n    }\n    if (_future) {\n        assert(_state);\n        if (!_state->available()) {\n            set_to_broken_promise(*_state);\n        }\n        _future->detach_promise();\n    }\n}\n\npromise_base& promise_base::operator=(promise_base&& x) noexcept {\n    clear();\n    move_it(std::move(x));\n    return *this;\n}\n\nvoid promise_base::set_to_current_exception() noexcept {\n    set_exception(std::current_exception());\n}\n\n#ifdef SEASTAR_DEBUG_PROMISE\n\nvoid promise_base::assert_task_shard() const noexcept {\n    if (_task_shard >= 0 && static_cast<shard_id>(_task_shard) != this_shard_id()) {\n        on_fatal_internal_error(seastar_logger, format(\"Promise task was set on shard {} but made ready on shard {}\", _task_shard, this_shard_id()));\n    }\n}\n\n#endif\n\ntemplate <promise_base::urgent Urgent>\nvoid promise_base::make_ready() noexcept {\n    if (_task) {\n        assert_task_shard();\n        if (Urgent == urgent::yes) {\n            ::seastar::schedule_urgent(std::exchange(_task, nullptr));\n        } else {\n            ::seastar::schedule(std::exchange(_task, nullptr));\n        }\n    }\n}\n\ntemplate void promise_base::make_ready<promise_base::urgent::no>() noexcept;\ntemplate void promise_base::make_ready<promise_base::urgent::yes>() noexcept;\n}\n\ntemplate\nfuture<void> current_exception_as_future() noexcept;\n\n/**\n * engine_exit() exits the reactor. It should be given a pointer to the\n * exception which prompted this exit - or a null pointer if the exit\n * request was not caused by any exception.\n */\nvoid engine_exit(std::exception_ptr eptr) {\n    if (!eptr) {\n        engine().exit(0);\n        return;\n    }\n    report_exception(\"Exiting on unhandled exception\", eptr);\n    engine().exit(1);\n}\n\nbroken_promise::broken_promise() : logic_error(\"broken promise\") { }\n\nfuture_state_base::future_state_base(current_exception_future_marker) noexcept\n    : future_state_base(std::current_exception()) { }\n\nvoid future_state_base::ignore() noexcept {\n    switch (_u.st) {\n    case state::invalid:\n    case state::future:\n    case state::result_unavailable:\n        assert(0 && \"invalid state for ignore\");\n    case state::result:\n        _u.st = state::result_unavailable;\n        break;\n    default:\n        // Ignore the exception\n        _u.take_exception();\n    }\n}\n\nnested_exception::nested_exception(std::exception_ptr inner, std::exception_ptr outer) noexcept\n    : inner(std::move(inner)), outer(std::move(outer)) {}\n\nnested_exception::nested_exception(nested_exception&&) noexcept = default;\n\nnested_exception::nested_exception(const nested_exception&) noexcept = default;\n\nconst char* nested_exception::what() const noexcept {\n    return \"seastar::nested_exception\";\n}\n\n[[noreturn]] void nested_exception::rethrow_nested() const {\n    std::rethrow_exception(outer);\n}\n\nstatic std::exception_ptr make_nested(std::exception_ptr&& inner, future_state_base&& old) noexcept {\n    std::exception_ptr outer = std::move(old).get_exception();\n    nested_exception nested{std::move(inner), std::move(outer)};\n    return std::make_exception_ptr<nested_exception>(std::move(nested));\n}\n\nfuture_state_base::future_state_base(nested_exception_marker, future_state_base&& n, future_state_base&& old) noexcept {\n    std::exception_ptr inner = std::move(n).get_exception();\n    if (!old.failed()) {\n        new (this) future_state_base(std::move(inner));\n    } else {\n        new (this) future_state_base(make_nested(std::move(inner), std::move(old)));\n    }\n}\n\nfuture_state_base::future_state_base(nested_exception_marker, future_state_base&& old) noexcept {\n    if (!old.failed()) {\n        new (this) future_state_base(current_exception_future_marker());\n        return;\n    } else {\n        new (this) future_state_base(make_nested(std::current_exception(), std::move(old)));\n    }\n}\n\nvoid future_state_base::rethrow_exception() && {\n    // Move ex out so future::~future() knows we've handled it\n    std::rethrow_exception(std::move(*this).get_exception());\n}\n\nvoid future_state_base::rethrow_exception() const& {\n    std::rethrow_exception(_u.ex);\n}\n\nvoid report_failed_future(const std::exception_ptr& eptr) noexcept {\n    ++engine()._abandoned_failed_futures;\n    seastar_logger.warn(\"Exceptional future ignored: {}, backtrace: {}\", eptr, current_backtrace());\n}\n\nvoid report_failed_future(const future_state_base& state) noexcept {\n    report_failed_future(state._u.ex);\n}\n\nvoid report_failed_future(future_state_base::any&& state) noexcept {\n    report_failed_future(std::move(state).take_exception());\n}\n\nvoid reactor::test::with_allow_abandoned_failed_futures(unsigned count, noncopyable_function<void ()> func) {\n    auto before = engine()._abandoned_failed_futures;\n    auto old_level = seastar_logger.level();\n    seastar_logger.set_level(log_level::error);\n    func();\n    auto after = engine()._abandoned_failed_futures;\n    assert(after - before == count);\n    engine()._abandoned_failed_futures = before;\n    seastar_logger.set_level(old_level);\n}\n\nnamespace {\nclass thread_wake_task final : public task {\n    thread_context* _thread;\npublic:\n    thread_wake_task(thread_context* thread) noexcept : _thread(thread) {}\n    virtual void run_and_dispose() noexcept override {\n        thread_impl::switch_in(_thread);\n        // no need to delete, since this is always allocated on\n        // _thread's stack.\n    }\n    /// Returns the task which is waiting for this thread to be done, or nullptr.\n    virtual task* waiting_task() noexcept override {\n        return _thread->waiting_task();\n    }\n};\n}\n\nvoid internal::future_base::do_wait() noexcept {\n    auto thread = thread_impl::get();\n    assert(thread);\n    thread_wake_task wake_task{thread};\n    wake_task.make_backtrace();\n    _promise->set_task(&wake_task);\n    thread_impl::switch_out(thread);\n}\n\n#ifdef SEASTAR_COROUTINES_ENABLED\nvoid internal::future_base::set_coroutine(task& coroutine) noexcept {\n    assert(_promise);\n    _promise->set_task(&coroutine);\n}\n#endif\n\n}\n\n// Path: src/core/resource.cc\n\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <boost/program_options.hpp>\n#include <boost/algorithm/string.hpp>\n#include <boost/range/adaptor/map.hpp>\n#include <boost/range/algorithm/copy.hpp>\n#include <boost/range/irange.hpp>\n#include <regex>\n#include <stdlib.h>\n#include <unistd.h>\n#include <limits>\n#include <filesystem>\n#include <unordered_map>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/resource.hh>\n#include <seastar/core/align.hh>\n#include <seastar/core/print.hh>\n#include <seastar/util/defer.hh>\n#include <seastar/util/read_first_line.hh>\n#include <seastar/util/log.hh>\n#include <seastar/core/io_queue.hh>\n#include <seastar/core/print.hh>\n#include \"cgroup.hh\"\n\n#if SEASTAR_HAVE_HWLOC\n#include <hwloc/glibc-sched.h>\n#endif\n\n#endif\n\nnamespace seastar {\n\nextern logger seastar_logger;\n\nnamespace resource {\n\n// This function was made optional because of validate. It needs to\n// throw an error when a non parseable input is given.\nstd::optional<cpuset> parse_cpuset(std::string value) {\n    static std::regex r(\"(\\\\d+-)?(\\\\d+)(,(\\\\d+-)?(\\\\d+))*\");\n\n    std::smatch match;\n    if (std::regex_match(value, match, r)) {\n        std::vector<std::string> ranges;\n        boost::split(ranges, value, boost::is_any_of(\",\"));\n        resource::cpuset ret;\n        for (auto&& range: ranges) {\n            std::string beg = range;\n            std::string end = range;\n            auto dash = range.find('-');\n            if (dash != range.npos) {\n                beg = range.substr(0, dash);\n                end = range.substr(dash + 1);\n            }\n            auto b = boost::lexical_cast<unsigned>(beg);\n            auto e = boost::lexical_cast<unsigned>(end);\n\n            if (b > e) {\n                return std::nullopt;\n            }\n\n            for (auto i = b; i <= e; ++i) {\n                ret.insert(i);\n            }\n        }\n        return ret;\n    }\n    return std::nullopt;\n}\n\n}\n\nnamespace cgroup {\n\nnamespace fs = std::filesystem;\n\noptional<cpuset> cpu_set() {\n    auto cpuset = read_setting_V1V2_as<std::string>(\n                              \"cpuset/cpuset.cpus\",\n                              \"cpuset.cpus.effective\");\n    if (cpuset) {\n        return seastar::resource::parse_cpuset(*cpuset);\n    }\n\n    seastar_logger.warn(\"Unable to parse cgroup's cpuset. Ignoring.\");\n    return std::nullopt;\n}\n\nsize_t memory_limit() {\n    return read_setting_V1V2_as<size_t>(\n                             \"memory/memory.limit_in_bytes\",\n                             \"memory.max\")\n        .value_or(std::numeric_limits<size_t>::max());\n}\n\ntemplate <typename T>\noptional<T> read_setting_as(std::string path) {\n    try {\n        auto line = read_first_line(path);\n        return boost::lexical_cast<T>(line);\n    } catch (...) {\n        seastar_logger.warn(\"Couldn't read cgroup file {}.\", path);\n    }\n\n    return std::nullopt;\n}\n\n/*\n * what cgroup do we belong to?\n *\n * For cgroups V2, /proc/self/cgroup should read \"0::<cgroup-dir-path>\"\n * Note: true only for V2-only systems, but there is no reason to support\n * a hybrid configuration.\n */\nstatic optional<fs::path> cgroup2_path_my_pid() {\n    seastar::sstring cline;\n    try {\n        cline = read_first_line(fs::path{\"/proc/self/cgroup\"});\n    } catch (...) {\n        // '/proc/self/cgroup' must be there. If not - there is an issue\n        // with the system configuration.\n        throw std::runtime_error(\"no cgroup data for our process\");\n    }\n\n    // for a V2-only system, we expect exactly one line:\n    // 0::<abs-path-to-cgroup>\n    if (cline.at(0) != '0') {\n        // This is either a v1 system, or system configured with a hybrid of v1 & v2.\n        // We do not support such combinations of v1 and v2 at this point.\n        seastar_logger.debug(\"Not a cgroups-v2-only system\");\n        return std::nullopt;\n    }\n\n    // the path is guaranteed to start with '0::/'\n    return fs::path{\"/sys/fs/cgroup/\" + cline.substr(4)};\n}\n\n/*\n * traverse the cgroups V2 hierarchy bottom-up, starting from our process'\n * specific cgroup up to /sys/fs/cgroup, looking for the named file.\n */\nstatic optional<fs::path> locate_lowest_cgroup2(fs::path lowest_subdir, std::string filename) {\n    // locate the lowest subgroup containing the named file (i.e.\n    // handles the requested control by itself)\n...\n", "needle_token_start": 7115, "needle_token_end": 7263, "code_context_ntokens": 16022, "output": ["```cpp\ntemplate <typename Enum>\nvoid describe_enum_value(bpo::options_description& opts, const options_description_building_visitor::value_metadata& d, const Enum* default_value) {\n    if (default_value) {\n        opts.add_options()(d.name.c_str(), boost::program_options::value<std::string>()->default_value(to_string(*default_value)), d.description.c_str());\n    } else {\n        opts.add_options()(d.name.c_str(), boost::program_options::value<std::string>(), d.description.c_str());\n    }\n}\n```"]}
{"repo": "scylladb/seastar", "name": "posix_datagram", "language": "cpp", "path": "src/net/posix-stack.cc", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The purpose is to encapsulate a network message, including its source and destination addresses along with the data packet, for transmission over a network.\n2. **Input**: The input includes the source address, destination address, and the data packet to be transmitted.\n3. **Output**: There is no direct output as this is a constructor for setting up the network message object. The object itself represents the encapsulated data ready for network operations.\n4. **Procedure**: The procedure involves initializing the object with the provided source and destination addresses and the data packet. The data packet is moved into the object to manage ownership and ensure efficient data handling.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    std::pmr::polymorphic_allocator<char>* _allocator;\nprivate:\n    explicit posix_connected_socket_impl(sa_family_t family, int protocol, pollable_fd fd, std::pmr::polymorphic_allocator<char>* allocator=memory::malloc_allocator) :\n        _fd(std::move(fd)), _ops(get_posix_connected_socket_ops(family, protocol)), _allocator(allocator) {}\n    explicit posix_connected_socket_impl(sa_family_t family, int protocol, pollable_fd fd, conntrack::handle&& handle,\n        std::pmr::polymorphic_allocator<char>* allocator=memory::malloc_allocator) : _fd(std::move(fd))\n                , _ops(get_posix_connected_socket_ops(family, protocol)), _handle(std::move(handle)), _allocator(allocator) {}\npublic:\n    virtual data_source source() override {\n        return source(connected_socket_input_stream_config());\n    }\n    virtual data_source source(connected_socket_input_stream_config csisc) override {\n        return data_source(std::make_unique<posix_data_source_impl>(_fd, csisc, _allocator));\n    }\n    virtual data_sink sink() override {\n        return data_sink(std::make_unique< posix_data_sink_impl>(_fd));\n    }\n    virtual void shutdown_input() override {\n        shutdown_socket_fd(_fd, SHUT_RD);\n    }\n    virtual void shutdown_output() override {\n        shutdown_socket_fd(_fd, SHUT_WR);\n    }\n    virtual void set_nodelay(bool nodelay) override {\n        return _ops->set_nodelay(_fd.get_file_desc(), nodelay);\n    }\n    virtual bool get_nodelay() const override {\n        return _ops->get_nodelay(_fd.get_file_desc());\n    }\n    void set_keepalive(bool keepalive) override {\n        return _ops->set_keepalive(_fd.get_file_desc(), keepalive);\n    }\n    bool get_keepalive() const override {\n        return _ops->get_keepalive(_fd.get_file_desc());\n    }\n    void set_keepalive_parameters(const keepalive_params& p) override {\n        return _ops->set_keepalive_parameters(_fd.get_file_desc(), p);\n    }\n    keepalive_params get_keepalive_parameters() const override {\n        return _ops->get_keepalive_parameters(_fd.get_file_desc());\n    }\n    void set_sockopt(int level, int optname, const void* data, size_t len) override {\n        return _ops->set_sockopt(_fd.get_file_desc(), level, optname, data, len);\n    }\n    int get_sockopt(int level, int optname, void* data, size_t len) const override {\n        return _ops->get_sockopt(_fd.get_file_desc(), level, optname, data, len);\n    }\n    socket_address local_address() const noexcept override {\n        return _ops->local_address(_fd.get_file_desc());\n    }\n    socket_address remote_address() const noexcept override {\n        return _ops->remote_address(_fd.get_file_desc());\n    }\n    future<> wait_input_shutdown() override {\n        return _fd.poll_rdhup();\n    }\n\n    friend class posix_server_socket_impl;\n    friend class posix_ap_server_socket_impl;\n    friend class posix_reuseport_server_socket_impl;\n    friend class posix_network_stack;\n    friend class posix_ap_network_stack;\n    friend class posix_socket_impl;\n};\n\nstatic void resolve_outgoing_address(socket_address& a) {\n    if (a.family() != AF_INET6\n        || a.as_posix_sockaddr_in6().sin6_scope_id != inet_address::invalid_scope\n        || !IN6_IS_ADDR_LINKLOCAL(&a.as_posix_sockaddr_in6().sin6_addr)\n    ) {\n        return;\n    }\n\n    FILE *f;\n\n    if (!(f = fopen(\"/proc/net/ipv6_route\", \"r\"))) {\n        throw std::system_error(errno, std::system_category(), \"resolve_address\");\n    }\n\n    auto holder = std::unique_ptr<FILE, int(*)(FILE *)>(f, &::fclose);\n\n    /**\n      Here all configured IPv6 routes are shown in a special format. The example displays for loopback interface only. The meaning is shown below (see net/ipv6/route.c for more).\n\n    # cat /proc/net/ipv6_route\n    00000000000000000000000000000000 00 00000000000000000000000000000000 00 00000000000000000000000000000000 ffffffff 00000001 00000001 00200200 lo\n    +------------------------------+ ++ +------------------------------+ ++ +------------------------------+ +------+ +------+ +------+ +------+ ++\n    |                                |  |                                |  |                                |        |        |        |        |\n    1                                2  3                                4  5                                6        7        8        9        10\n\n    1: IPv6 destination network displayed in 32 hexadecimal chars without colons as separator\n\n    2: IPv6 destination prefix length in hexadecimal\n\n    3: IPv6 source network displayed in 32 hexadecimal chars without colons as separator\n\n    4: IPv6 source prefix length in hexadecimal\n\n    5: IPv6 next hop displayed in 32 hexadecimal chars without colons as separator\n\n    6: Metric in hexadecimal\n\n    7: Reference counter\n\n    8: Use counter\n\n    9: Flags\n\n    10: Device name\n\n    */\n\n    uint32_t prefix_len, src_prefix_len;\n    unsigned long flags;\n    char device[16];\n    char dest_str[40];\n\n    for (;;) {\n        auto n = fscanf(f, \"%4s%4s%4s%4s%4s%4s%4s%4s %02x \"\n                            \"%*4s%*4s%*4s%*4s%*4s%*4s%*4s%*4s %02x \"\n                            \"%*4s%*4s%*4s%*4s%*4s%*4s%*4s%*4s \"\n                            \"%*08x %*08x %*08x %08lx %8s\",\n                            &dest_str[0], &dest_str[5], &dest_str[10], &dest_str[15],\n                            &dest_str[20], &dest_str[25], &dest_str[30], &dest_str[35],\n                            &prefix_len,\n                            &src_prefix_len,\n                            &flags, device);\n        if (n != 12) {\n            break;\n        }\n\n        if ((prefix_len > 128)  || (src_prefix_len != 0)\n            || (flags & (RTF_POLICY | RTF_FLOW))\n            || ((flags & RTF_REJECT) && prefix_len == 0) /* reject all */) {\n            continue;\n        }\n\n        dest_str[4] = dest_str[9] = dest_str[14] = dest_str[19] = dest_str[24] = dest_str[29] = dest_str[34] = ':';\n        dest_str[39] = '\\0';\n\n        struct in6_addr addr;\n        if (inet_pton(AF_INET6, dest_str, &addr) < 0) {\n            /* not an Ipv6 address */\n            continue;\n        }\n\n        auto bytes = prefix_len / 8;\n        auto bits = prefix_len % 8;\n\n        auto& src = a.as_posix_sockaddr_in6().sin6_addr;\n\n        if (bytes > 0 && memcmp(&src, &addr, bytes)) {\n            continue;\n        }\n        if (bits > 0) {\n            auto c1 = src.s6_addr[bytes];\n            auto c2 = addr.s6_addr[bytes];\n            auto mask = 0xffu << (8 - bits);\n            if ((c1 & mask) != (c2 & mask)) {\n                continue;\n            }\n        }\n\n        // found the route.\n        for (auto& nif : engine().net().network_interfaces()) {\n            if (nif.name() == device || nif.display_name() == device) {\n                a.as_posix_sockaddr_in6().sin6_scope_id = nif.index();\n                return;\n            }\n        }\n    }\n}\n\nclass posix_socket_impl final : public socket_impl {\n    pollable_fd _fd;\n    std::pmr::polymorphic_allocator<char>* _allocator;\n    bool _reuseaddr = false;\n\n    future<> find_port_and_connect(socket_address sa, socket_address local, transport proto = transport::TCP) {\n        static thread_local std::default_random_engine random_engine{std::random_device{}()};\n        static thread_local std::uniform_int_distribution<uint16_t> u(49152/smp::count + 1, 65535/smp::count - 1);\n        // If no explicit local address, set to dest address family wildcard. \n        if (local.is_unspecified()) {\n            local = net::inet_address(sa.addr().in_family());\n        }\n        resolve_outgoing_address(sa);\n        return repeat([this, sa, local, proto, attempts = 0, requested_port = ntoh(local.as_posix_sockaddr_in().sin_port)] () mutable {\n            _fd = engine().make_pollable_fd(sa, int(proto));\n            _fd.get_file_desc().setsockopt(SOL_SOCKET, SO_REUSEADDR, int(_reuseaddr));\n            uint16_t port = attempts++ < 5 && requested_port == 0 && proto == transport::TCP ? u(random_engine) * smp::count + this_shard_id() : requested_port;\n            local.as_posix_sockaddr_in().sin_port = hton(port);\n            return futurize_invoke([this, sa, local] { return engine().posix_connect(_fd, sa, local); }).then_wrapped([port, requested_port] (future<> f) {\n                try {\n                    f.get();\n                    return stop_iteration::yes;\n                } catch (std::system_error& err) {\n                    if (port != requested_port && (err.code().value() == EADDRINUSE || err.code().value() == EADDRNOTAVAIL)) {\n                        return stop_iteration::no;\n                    }\n                    throw;\n                }\n            });\n        });\n    }\n\n    /// an aux function to handle unix-domain-specific requests\n    future<connected_socket> connect_unix_domain(socket_address sa, socket_address local) {\n        // note that if the 'local' address was not set by the client, it was created as an undefined address\n        if (local.is_unspecified()) {\n            local = socket_address{unix_domain_addr{std::string{}}};\n        }\n\n        _fd = engine().make_pollable_fd(sa, 0);\n        return engine().posix_connect(_fd, sa, local).then(\n            [fd = _fd, allocator = _allocator](){\n                // a problem with 'private' interaction with 'unique_ptr'\n                std::unique_ptr<connected_socket_impl> csi;\n                csi.reset(new posix_connected_socket_impl{AF_UNIX, 0, std::move(fd), allocator});\n                return make_ready_future<connected_socket>(connected_socket(std::move(csi)));\n            }\n        );\n    }\n\npublic:\n    explicit posix_socket_impl(std::pmr::polymorphic_allocator<char>* allocator=memory::malloc_allocator) : _allocator(allocator) {}\n\n    virtual future<connected_socket> connect(socket_address sa, socket_address local, transport proto = transport::TCP) override {\n        if (sa.is_af_unix()) {\n            return connect_unix_domain(sa, local);\n        }\n        return find_port_and_connect(sa, local, proto).then([this, sa, proto, allocator = _allocator] () mutable {\n            std::unique_ptr<connected_socket_impl> csi;\n            csi.reset(new posix_connected_socket_impl(sa.family(), static_cast<int>(proto), _fd, allocator));\n            return make_ready_future<connected_socket>(connected_socket(std::move(csi)));\n        });\n    }\n\n    void set_reuseaddr(bool reuseaddr) override {\n        _reuseaddr = reuseaddr;\n        if (_fd) {\n            _fd.get_file_desc().setsockopt(SOL_SOCKET, SO_REUSEADDR, int(reuseaddr));\n        }\n    }\n\n    bool get_reuseaddr() const override {\n        if(_fd) {\n            return _fd.get_file_desc().getsockopt<int>(SOL_SOCKET, SO_REUSEADDR);\n        } else {\n            return _reuseaddr;\n        }\n    }\n\n    virtual void shutdown() override {\n        if (_fd) {\n            try {\n                _fd.shutdown(SHUT_RDWR);\n            } catch (std::system_error& e) {\n                if (e.code().value() != ENOTCONN) {\n                    throw;\n                }\n            }\n        }\n    }\n};\n\nfuture<accept_result>\nposix_server_socket_impl::accept() {\n    return _lfd.accept().then([this] (std::tuple<pollable_fd, socket_address> fd_sa) {\n        auto& fd = std::get<0>(fd_sa);\n        auto& sa = std::get<1>(fd_sa);\n        auto cth = [this, &sa] {\n            switch(_lba) {\n            case server_socket::load_balancing_algorithm::connection_distribution:\n                return _conntrack.get_handle();\n            case server_socket::load_balancing_algorithm::port:\n                return _conntrack.get_handle(ntoh(sa.as_posix_sockaddr_in().sin_port) % smp::count);\n            case server_socket::load_balancing_algorithm::fixed:\n                return _conntrack.get_handle(_fixed_cpu);\n            default: abort();\n            }\n        } ();\n        auto cpu = cth.cpu();\n        if (cpu == this_shard_id()) {\n            std::unique_ptr<connected_socket_impl> csi(\n                    new posix_connected_socket_impl(sa.family(), _protocol, std::move(fd), std::move(cth), _allocator));\n            return make_ready_future<accept_result>(\n                    accept_result{connected_socket(std::move(csi)), sa});\n        } else {\n            // FIXME: future is discarded\n            (void)smp::submit_to(cpu, [protocol = _protocol, ssa = _sa, fd = std::move(fd.get_file_desc()), sa, cth = std::move(cth), allocator = _allocator] () mutable {\n                posix_ap_server_socket_impl::move_connected_socket(protocol, ssa, pollable_fd(std::move(fd)), sa, std::move(cth), allocator);\n            });\n            return accept();\n        }\n    });\n}\n\nvoid\nposix_server_socket_impl::abort_accept() {\n    _lfd.shutdown(SHUT_RD, pollable_fd::shutdown_kernel_only::no);\n}\n\nsocket_address posix_server_socket_impl::local_address() const {\n    return _lfd.get_file_desc().get_address();\n}\n\nposix_ap_server_socket_impl::posix_ap_server_socket_impl(int protocol, socket_address sa, std::pmr::polymorphic_allocator<char>* allocator)\n        : _protocol(protocol), _sa(sa), _allocator(allocator)\n{\n    auto it = ports.emplace(std::make_tuple(_protocol, _sa));\n    if (!it.second) {\n        throw std::system_error(EADDRINUSE, std::system_category());\n    }\n}\n\nposix_ap_server_socket_impl::~posix_ap_server_socket_impl() {\n    ports.erase(std::make_tuple(_protocol, _sa));\n}\n\nfuture<accept_result> posix_ap_server_socket_impl::accept() {\n    auto t_sa = std::make_tuple(_protocol, _sa);\n    auto conni = conn_q.find(t_sa);\n    if (conni != conn_q.end()) {\n        connection c = std::move(conni->second);\n        conn_q.erase(conni);\n        try {\n            std::unique_ptr<connected_socket_impl> csi(\n                    new posix_connected_socket_impl(_sa.family(), _protocol, std::move(c.fd), std::move(c.connection_tracking_handle), _allocator));\n            return make_ready_future<accept_result>(accept_result{connected_socket(std::move(csi)), std::move(c.addr)});\n        } catch (...) {\n            return make_exception_future<accept_result>(std::current_exception());\n        }\n    } else {\n        try {\n            auto i = sockets.emplace(std::piecewise_construct, std::make_tuple(t_sa), std::make_tuple());\n            assert(i.second);\n            return i.first->second.get_future();\n        } catch (...) {\n            return make_exception_future<accept_result>(std::current_exception());\n        }\n    }\n}\n\nvoid\nposix_ap_server_socket_impl::abort_accept() {\n    auto t_sa = std::make_tuple(_protocol, _sa);\n    conn_q.erase(t_sa);\n    auto i = sockets.find(t_sa);\n    if (i != sockets.end()) {\n        i->second.set_exception(std::system_error(ECONNABORTED, std::system_category()));\n        sockets.erase(i);\n    }\n}\n\nfuture<accept_result>\nposix_reuseport_server_socket_impl::accept() {\n    return _lfd.accept().then([allocator = _allocator, protocol = _protocol] (std::tuple<pollable_fd, socket_address> fd_sa) {\n        auto& fd = std::get<0>(fd_sa);\n        auto& sa = std::get<1>(fd_sa);\n        std::unique_ptr<connected_socket_impl> csi(\n                new posix_connected_socket_impl(sa.family(), protocol, std::move(fd), allocator));\n        return make_ready_future<accept_result>(\n            accept_result{connected_socket(std::move(csi)), sa});\n    });\n}\n\nvoid\nposix_reuseport_server_socket_impl::abort_accept() {\n    _lfd.shutdown(SHUT_RD, pollable_fd::shutdown_kernel_only::no);\n}\n\nsocket_address posix_reuseport_server_socket_impl::local_address() const {\n    return _lfd.get_file_desc().get_address();\n}\n\nvoid\nposix_ap_server_socket_impl::move_connected_socket(int protocol, socket_address sa, pollable_fd fd, socket_address addr, conntrack::handle cth, std::pmr::polymorphic_allocator<char>* allocator) {\n    auto t_sa = std::make_tuple(protocol, sa);\n    auto i = sockets.find(t_sa);\n    if (i != sockets.end()) {\n        try {\n            std::unique_ptr<connected_socket_impl> csi(new posix_connected_socket_impl(sa.family(), protocol, std::move(fd), std::move(cth), allocator));\n            i->second.set_value(accept_result{connected_socket(std::move(csi)), std::move(addr)});\n        } catch (...) {\n            i->second.set_exception(std::current_exception());\n        }\n        sockets.erase(i);\n    } else {\n        conn_q.emplace(std::piecewise_construct, std::make_tuple(t_sa), std::make_tuple(std::move(fd), std::move(addr), std::move(cth)));\n    }\n}\n\nfuture<temporary_buffer<char>>\nposix_data_source_impl::get() {\n    return _fd.recv_some(static_cast<internal::buffer_allocator*>(this)).then([this] (temporary_buffer<char> b) {\n        if (b.size() >= _config.buffer_size) {\n            _config.buffer_size *= 2;\n            _config.buffer_size = std::min(_config.buffer_size, _config.max_buffer_size);\n        } else if (b.size() <= _config.buffer_size / 4) {\n            _config.buffer_size /= 2;\n            _config.buffer_size = std::max(_config.buffer_size, _config.min_buffer_size);\n        }\n        return b;\n    });\n}\n\ntemporary_buffer<char>\nposix_data_source_impl::allocate_buffer() {\n    return make_temporary_buffer<char>(_buffer_allocator, _config.buffer_size);\n}\n\nfuture<> posix_data_source_impl::close() {\n    _fd.shutdown(SHUT_RD);\n    return make_ready_future<>();\n}\n\nstd::vector<struct iovec> to_iovec(const packet& p) {\n    std::vector<struct iovec> v;\n    v.reserve(p.nr_frags());\n    for (auto&& f : p.fragments()) {\n        v.push_back({.iov_base = f.base, .iov_len = f.size});\n    }\n    return v;\n}\n\nstd::vector<iovec> to_iovec(std::vector<temporary_buffer<char>>& buf_vec) {\n    std::vector<iovec> v;\n    v.reserve(buf_vec.size());\n    for (auto& buf : buf_vec) {\n        v.push_back({.iov_base = buf.get_write(), .iov_len = buf.size()});\n    }\n    return v;\n}\n\nfuture<>\nposix_data_sink_impl::put(temporary_buffer<char> buf) {\n    return _fd.write_all(buf.get(), buf.size()).then([d = buf.release()] {});\n}\n\nfuture<>\nposix_data_sink_impl::put(packet p) {\n    _p = std::move(p);\n    return _fd.write_all(_p).then([this] { _p.reset(); });\n}\n\nfuture<>\nposix_data_sink_impl::close() {\n    _fd.shutdown(SHUT_WR);\n    return make_ready_future<>();\n}\n\nvoid posix_data_sink_impl::on_batch_flush_error() noexcept {\n    shutdown_socket_fd(_fd, SHUT_RD);\n}\n\nposix_network_stack::posix_network_stack(const program_options::option_group& opts, std::pmr::polymorphic_allocator<char>* allocator)\n        : _reuseport(engine().posix_reuseport_available()), _allocator(allocator) {\n}\n\nserver_socket\nposix_network_stack::listen(socket_address sa, listen_options opt) {\n    using server_socket = seastar::server_socket;\n    // allow unspecified bind address -> default to ipv4 wildcard\n    if (sa.is_unspecified()) {\n        sa = inet_address(inet_address::family::INET);\n    }\n    if (sa.is_af_unix()) {\n        return server_socket(std::make_unique<posix_server_socket_impl>(0, sa, engine().posix_listen(sa, opt), opt.lba, opt.fixed_cpu, _allocator));\n    }\n    auto protocol = static_cast<int>(opt.proto);\n    return _reuseport ?\n        server_socket(std::make_unique<posix_reuseport_server_socket_impl>(protocol, sa, engine().posix_listen(sa, opt), _allocator))\n        :\n        server_socket(std::make_unique<posix_server_socket_impl>(protocol, sa, engine().posix_listen(sa, opt), opt.lba, opt.fixed_cpu, _allocator));\n}\n\n::seastar::socket posix_network_stack::socket() {\n    return ::seastar::socket(std::make_unique<posix_socket_impl>(_allocator));\n}\n\nposix_ap_network_stack::posix_ap_network_stack(const program_options::option_group& opts, std::pmr::polymorphic_allocator<char>* allocator)\n        : posix_network_stack(opts, allocator), _reuseport(engine().posix_reuseport_available()) {\n}\n\nserver_socket\nposix_ap_network_stack::listen(socket_address sa, listen_options opt) {\n    using server_socket = seastar::server_socket;\n    // allow unspecified bind address -> default to ipv4 wildcard\n    if (sa.is_unspecified()) {\n        sa = inet_address(inet_address::family::INET);\n    }\n    if (sa.is_af_unix()) {\n        return server_socket(std::make_unique<posix_ap_server_socket_impl>(0, sa, _allocator));\n    }\n    auto protocol = static_cast<int>(opt.proto);\n    return _reuseport ?\n        server_socket(std::make_unique<posix_reuseport_server_socket_impl>(protocol, sa, engine().posix_listen(sa, opt), _allocator))\n        :\n        server_socket(std::make_unique<posix_ap_server_socket_impl>(protocol, sa, _allocator));\n}\n\nstruct cmsg_with_pktinfo {\n    struct cmsghdrcmh;\n    union {\n        struct in_pktinfo pktinfo;\n        struct in6_pktinfo pkt6info;\n    };\n};\n\nclass posix_datagram_channel : public datagram_channel_impl {\nprivate:\n    static constexpr int MAX_DATAGRAM_SIZE = 65507;\n    struct recv_ctx {\n        struct msghdr _hdr;\n        struct iovec _iov;\n        socket_address _src_addr;\n        char* _buffer;\n        cmsg_with_pktinfo _cmsg;\n\n        recv_ctx(bool use_pktinfo) {\n            memset(&_hdr, 0, sizeof(_hdr));\n            _hdr.msg_iov = &_iov;\n            _hdr.msg_iovlen = 1;\n            _hdr.msg_name = &_src_addr.u.sa;\n            _hdr.msg_namelen = sizeof(_src_addr.u.sas);\n\n            if (use_pktinfo) {\n                memset(&_cmsg, 0, sizeof(_cmsg));\n                _hdr.msg_control = &_cmsg;\n                _hdr.msg_controllen = sizeof(_cmsg);\n            } else {\n                _hdr.msg_control = nullptr;\n                _hdr.msg_controllen = 0;\n            }\n        }\n\n        recv_ctx(const recv_ctx&) = delete;\n        recv_ctx(recv_ctx&&) = delete;\n\n        void prepare() {\n            _buffer = new char[MAX_DATAGRAM_SIZE];\n            _iov.iov_base = _buffer;\n            _iov.iov_len = MAX_DATAGRAM_SIZE;\n        }\n    };\n    struct send_ctx {\n        struct msghdr _hdr;\n        std::vector<struct iovec> _iovecs;\n        socket_address _dst;\n        packet _p;\n\n        send_ctx() {\n            memset(&_hdr, 0, sizeof(_hdr));\n            _hdr.msg_name = &_dst.u.sa;\n            _hdr.msg_namelen = _dst.addr_length;\n        }\n\n        send_ctx(const send_ctx&) = delete;\n        send_ctx(send_ctx&&) = delete;\n\n        void prepare(const socket_address& dst, packet p) {\n            _dst = dst;\n            _hdr.msg_namelen = _dst.addr_length;\n            _p = std::move(p);\n            _iovecs = to_iovec(_p);\n            _hdr.msg_iov = _iovecs.data();\n            _hdr.msg_iovlen = _iovecs.size();\n            resolve_outgoing_address(_dst);\n        }\n    };\n\n    static bool is_inet(sa_family_t family) {\n        return family == AF_INET || family == AF_INET6;\n    }\n\n    static file_desc create_socket(sa_family_t family) {\n        file_desc fd = file_desc::socket(family, SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);\n\n        if (is_inet(family)) {\n            fd.setsockopt(SOL_IP, IP_PKTINFO, true);\n            if (engine().posix_reuseport_available()) {\n                fd.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1);\n            }\n        }\n\n        return fd;\n    }\n\n    pollable_fd _fd;\n    socket_address _address;\n    recv_ctx _recv;\n    send_ctx _send;\n    bool _closed;\npublic:\n    /// Creates a channel that is not bound to any socket address. The channel\n    /// can be used to communicate with adressess that belong to the \\param\n    /// family.\n    posix_datagram_channel(sa_family_t family)\n        : _recv(is_inet(family)), _closed(false) {\n        auto fd = create_socket(family);\n\n        _address = fd.get_address();\n        _fd = std::move(fd);\n    }\n\n    /// Creates a channel that is bound to the specified local address. It can be used to\n    /// communicate with addresses that belong to the family of \\param local.\n    posix_datagram_channel(socket_address local)\n        : _recv(is_inet(local.family())), _closed(false) {\n        auto fd = create_socket(local.family());\n        fd.bind(local.u.sa, local.addr_length);\n\n        _address = fd.get_address();\n        _fd = std::move(fd);\n    }\n\n    virtual ~posix_datagram_channel() { if (!_closed) close(); };\n    virtual future<datagram> receive() override;\n    virtual future<> send(const socket_address& dst, const char *msg) override;\n    virtual future<> send(const socket_address& dst, packet p) override;\n    virtual void shutdown_input() override {\n        _fd.shutdown(SHUT_RD, pollable_fd::shutdown_kernel_only::no);\n    }\n    virtual void shutdown_output() override {\n        _fd.shutdown(SHUT_WR, pollable_fd::shutdown_kernel_only::no);\n    }\n    virtual void close() override {\n        _closed = true;\n        _fd = {};\n    }\n    virtual bool is_closed() const override { return _closed; }\n    socket_address local_address() const override {\n        assert(_address.u.sas.ss_family != AF_INET6 || (_address.addr_length > 20));\n        return _address;\n    }\n};\n\nfuture<> posix_datagram_channel::send(const socket_address& dst, const char *message) {\n    auto len = strlen(message);\n    auto a = dst;\n    resolve_outgoing_address(a);\n    return _fd.sendto(a, message, len)\n            .then([len] (size_t size) { assert(size == len); });\n}\n\nfuture<> posix_datagram_channel::send(const socket_address& dst, packet p) {\n    auto len = p.len();\n    _send.prepare(dst, std::move(p));\n    return _fd.sendmsg(&_send._hdr)\n            .then([len] (size_t size) { assert(size == len); });\n}\n\nudp_channel\nposix_network_stack::make_udp_channel(const socket_address& addr) {\n    if (!addr.is_unspecified()) {\n        return make_bound_datagram_channel(addr);\n    } else {\n        // Preserve the default behavior of make_udp_channel({}) which is to\n        // create an unbound channel that can be used to send UDP datagrams.\n        return make_unbound_datagram_channel(AF_INET);\n    }\n}\n\ndatagram_channel\nposix_network_stack::make_unbound_datagram_channel(sa_family_t family) {\n    return datagram_channel(std::make_unique<posix_datagram_channel>(family));\n}\n\ndatagram_channel\nposix_network_stack::make_bound_datagram_channel(const socket_address& local) {\n    return datagram_channel(std::make_unique<posix_datagram_channel>(local));\n}\n\nbool\nposix_network_stack::supports_ipv6() const {\n    static bool has_ipv6 = [] {\n        try {\n            posix_datagram_channel c(ipv6_addr{\"::1\"});\n            c.close();\n            return true;\n        } catch (...) {}\n        return false;\n    }();\n\n    return has_ipv6;\n}\n\nclass posix_datagram : public datagram_impl {\nprivate:\n    socket_address _src;\n    socket_address _dst;\n    packet _p;\npublic:\n    \nposix_datagram(const socket_address& src, const socket_address& dst, packet p) : _src(src), _dst(dst), _p(std::move(p)) {}\n    virtual socket_address get_src() override { return _src; }\n    virtual socket_address get_dst() override { return _dst; }\n    virtual uint16_t get_dst_port() override {\n        if (_dst.family() != AF_INET && _dst.family() != AF_INET6) {\n            throw std::runtime_error(format(\"get_dst_port() called on non-IP address: {}\", _dst));\n        }\n        return _dst.port();\n    }\n    virtual packet& get_data() override { return _p; }\n};\n\nfuture<datagram>\nposix_datagram_channel::receive() {\n    _recv.prepare();\n    return _fd.recvmsg(&_recv._hdr).then([this] (size_t size) {\n        std::optional<socket_address> dst;\n        for (auto* cmsg = CMSG_FIRSTHDR(&_recv._hdr); cmsg != nullptr; cmsg = CMSG_NXTHDR(&_recv._hdr, cmsg)) {\n            if (cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_PKTINFO) {\n                dst = ipv4_addr(copy_reinterpret_cast<in_pktinfo>(CMSG_DATA(cmsg)).ipi_addr, _address.port());\n                break;\n            } else if (cmsg->cmsg_level == IPPROTO_IPV6 && cmsg->cmsg_type == IPV6_PKTINFO) {\n                dst = ipv6_addr(copy_reinterpret_cast<in6_pktinfo>(CMSG_DATA(cmsg)).ipi6_addr, _address.port());\n                break;\n            }\n        }\n        return make_ready_future<datagram>(datagram(std::make_unique<posix_datagram>(\n            _recv._src_addr, dst ? *dst : _address, packet(fragment{_recv._buffer, size}, make_deleter([buf = _recv._buffer] { delete[] buf; })))));\n    }).handle_exception([p = _recv._buffer](auto ep) {\n        delete[] p;\n        return make_exception_future<datagram>(std::move(ep));\n    });\n}\n\nnetwork_stack_entry register_posix_stack() {\n    return network_stack_entry{\n        \"posix\", std::make_unique<program_options::option_group>(nullptr, \"Posix\"),\n        [](const program_options::option_group& ops) {\n            return smp::main_thread() ? posix_network_stack::create(ops)\n                                      : posix_ap_network_stack::create(ops);\n        },\n        true};\n}\n\n// nw interface stuff\n\nstd::vector<network_interface> posix_network_stack::network_interfaces() {\n    class posix_network_interface_impl final : public network_interface_impl {\n    public:\n        uint32_t _index = 0, _mtu = 0;\n        sstring _name, _display_name;\n        std::vector<net::inet_address> _addresses;\n        std::vector<uint8_t> _hardware_address;\n        bool _loopback = false, _virtual = false, _up = false;\n\n        uint32_t index() const override {\n            return _index;\n        }\n        uint32_t mtu() const override {\n            return _mtu;\n        }\n        const sstring& name() const override {\n            return _name;   \n        }\n        const sstring& display_name() const override {\n            return _display_name.empty() ? name() : _display_name;\n        }\n        const std::vector<net::inet_address>& addresses() const override {\n            return _addresses;            \n        }\n        const std::vector<uint8_t> hardware_address() const override {\n            return _hardware_address;\n        }\n        bool is_loopback() const override {\n            return _loopback;   \n        }\n        bool is_virtual() const override {\n            return _virtual;\n        }\n        bool is_up() const override {\n            // TODO: should be checked on query?\n            return _up;\n        }\n        bool supports_ipv6() const override {\n            // TODO: this is not 100% correct.\n            return std::any_of(_addresses.begin(), _addresses.end(), std::mem_fn(&inet_address::is_ipv6));\n        }\n    };\n\n    // For now, keep an immutable set of interfaces created on start, shared across \n    // shards\n    static const std::vector<posix_network_interface_impl> global_interfaces = [] {\n        auto fd = ::socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);\n        throw_system_error_on(fd < 0, \"could not open netlink socket\");\n\n        std::unique_ptr<int, void(*)(int*)> fd_guard(&fd, [](int* p) { ::close(*p); });\n\n        auto pid = ::getpid();\n\n        sockaddr_nl local = {\n          .nl_family = AF_NETLINK,\n          .nl_pid = static_cast<unsigned int>(pid),\n          .nl_groups = RTMGRP_IPV6_IFADDR|RTMGRP_IPV4_IFADDR,\n        };\n\n        throw_system_error_on(bind(fd, (struct sockaddr *) &local, sizeof(local)) < 0, \"could not bind netlink socket\");\n\n        /* RTNL socket is ready for use, prepare and send requests */\n\n        std::vector<posix_network_interface_impl> res;\n\n        for (auto msg : { RTM_GETLINK, RTM_GETADDR}) {\n            struct nl_req {\n                nlmsghdr hdr;\n                union {\n                    rtgenmsg gen;\n                    ifaddrmsg addr; \n                }; \n            } req = {};\n\n            req.hdr.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtgenmsg));\n            req.hdr.nlmsg_type = msg;\n            req.hdr.nlmsg_flags = NLM_F_REQUEST | NLM_F_ROOT; \n            req.hdr.nlmsg_seq = 1;\n            req.hdr.nlmsg_pid = pid;\n\n            if (msg == RTM_GETLINK) {\n                req.gen.rtgen_family = AF_PACKET; /*  no preferred AF, we will get *all* interfaces */\n            } else {\n                req.addr.ifa_family = AF_UNSPEC;\n            }\n\n            sockaddr_nl kernel = {\n              .nl_family = AF_NETLINK, /* fill-in kernel address (destination of our message) */\n            };\n            iovec io = {\n              .iov_base = &req,\n              .iov_len = req.hdr.nlmsg_len,\n            };\n            msghdr rtnl_msg = {\n              .msg_name = &kernel,\n              .msg_namelen = sizeof(kernel),\n              .msg_iov = &io,\n              .msg_iovlen = 1,\n            };\n\n            throw_system_error_on(::sendmsg(fd, (struct msghdr *) &rtnl_msg, 0) < 0, \"could not send netlink request\");\n            /* parse reply */\n\n            constexpr size_t reply_buffer_size = 8192;\n            char reply[reply_buffer_size]; \n\n            bool done = false;\n\n            while (!done) {\n                iovec io_reply = {\n                  .iov_base = reply,\n                  .iov_len = reply_buffer_size,\n                };\n                msghdr rtnl_reply = {\n                  .msg_name = &kernel,\n                  .msg_namelen = sizeof(kernel),\n                  .msg_iov = &io_reply,\n                  .msg_iovlen = 1,\n                };\n\n                auto len = ::recvmsg(fd, &rtnl_reply, 0); /* read as much data as fits in the receive buffer */\n                if (len <= 0) {\n                    return res;\n                }\n\n                for (auto* msg_ptr = (struct nlmsghdr *) reply; NLMSG_OK(msg_ptr, len); msg_ptr = NLMSG_NEXT(msg_ptr, len)) {\n                    switch(msg_ptr->nlmsg_type) {\n                    case NLMSG_DONE: // that is all\n                        done = true;\n                        break;                    \n                    case RTM_NEWLINK: \n                    {\n                        auto* iface = reinterpret_cast<const ifinfomsg*>(NLMSG_DATA(msg_ptr));\n                        auto ilen = msg_ptr->nlmsg_len - NLMSG_LENGTH(sizeof(ifinfomsg));\n\n                        // todo: filter any non-network interfaces (family)\n\n                        posix_network_interface_impl nwif;\n                        \n                        nwif._index = iface->ifi_index;\n                        nwif._loopback = (iface->ifi_flags & IFF_LOOPBACK) != 0;\n                        nwif._up = (iface->ifi_flags & IFF_UP) != 0;\n    #if defined(IFF_802_1Q_VLAN) && defined(IFF_EBRIDGE) && defined(IFF_SLAVE_INACTIVE)\n                        nwif._virtual = (iface->ifi_flags & (IFF_802_1Q_VLAN|IFF_EBRIDGE|IFF_SLAVE_INACTIVE)) != 0;\n    #endif                                        \n                        for (auto* attribute = IFLA_RTA(iface); RTA_OK(attribute, ilen); attribute = RTA_NEXT(attribute, ilen)) {\n                            switch(attribute->rta_type) {\n                            case IFLA_IFNAME:\n                                nwif._name = reinterpret_cast<const char *>(RTA_DATA(attribute));\n                                break;\n                            case IFLA_MTU:\n                                nwif._mtu = *reinterpret_cast<const uint32_t *>(RTA_DATA(attribute));                            \n                                break;\n                            case IFLA_ADDRESS:\n                                nwif._hardware_address.assign(reinterpret_cast<const uint8_t *>(RTA_DATA(attribute)), reinterpret_cast<const uint8_t *>(RTA_DATA(attribute)) + RTA_PAYLOAD(attribute));\n                                break;\n                            default:\n                                break;\n                            }\n                        }\n\n                        res.emplace_back(std::move(nwif));\n\n                        break;\n                    }\n                    case RTM_NEWADDR:\n                    {\n                        auto* addr = reinterpret_cast<const ifaddrmsg*>(NLMSG_DATA(msg_ptr));\n                        auto ilen = msg_ptr->nlmsg_len - NLMSG_LENGTH(sizeof(ifaddrmsg));\n                        \n                        for (auto& nwif : res) {\n                            if (nwif._index == addr->ifa_index) {\n                                for (auto* attribute = IFA_RTA(addr); RTA_OK(attribute, ilen); attribute = RTA_NEXT(attribute, ilen)) {\n                                    std::optional<inet_address> ia;\n                                    \n                                    switch(attribute->rta_type) {\n                                    case IFA_LOCAL:\n                                    case IFA_ADDRESS: // ipv6 addresses are reported only as \"ADDRESS\"\n\n                                        if (RTA_PAYLOAD(attribute) == sizeof(::in_addr)) {\n                                            ia.emplace(*reinterpret_cast<const ::in_addr *>(RTA_DATA(attribute)));\n                                        } else if (RTA_PAYLOAD(attribute) == sizeof(::in6_addr)) {\n                                            ia.emplace(*reinterpret_cast<const ::in6_addr *>(RTA_DATA(attribute)), nwif.index());\n                                        }\n                                        \n                                        if (ia && std::find(nwif._addresses.begin(), nwif._addresses.end(), *ia) == nwif._addresses.end()) {\n                                            nwif._addresses.emplace_back(*ia);\n                                        }\n\n                                        break;\n                                    default:\n                                        break;\n                                    }\n                                }\n\n                                break;\n                            }\n                        }\n\n                        break;\n                    }\n                    default:\n                        break;\n                    }\n                }      \n            }\n        }\n\n        return res;\n    }();\n\n    // And a similarly immutable set of shared_ptr to network_interface_impl per shard, ready \n    // to be handed out to callers with minimal overhead\n    static const thread_local std::vector<shared_ptr<posix_network_interface_impl>> thread_local_interfaces = [] {\n        std::vector<shared_ptr<posix_network_interface_impl>> res;\n        res.reserve(global_interfaces.size());\n        std::transform(global_interfaces.begin(), global_interfaces.end(), std::back_inserter(res), [](const posix_network_interface_impl& impl) {\n            return make_shared<posix_network_interface_impl>(impl);\n        });\n        return res;\n    }();\n\n    return std::vector<network_interface>(thread_local_interfaces.begin(), thread_local_interfaces.end());\n}\n\n}\n\n}\n\n// Path: src/net/arp.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cstdint>\n#include <optional>\n#include <utility>\nmodule seastar;\n#else\n#include <seastar/net/arp.hh>\n#endif\n\nnamespace seastar {\n\nnamespace net {\n\narp_for_protocol::arp_for_protocol(arp& a, uint16_t proto_num)\n    : _arp(a), _proto_num(proto_num) {\n    _arp.add(proto_num, this);\n}\n\narp_for_protocol::~arp_for_protocol() {\n    _arp.del(_proto_num);\n}\n\narp::arp(interface* netif) : _netif(netif), _proto(netif, eth_protocol_num::arp, [this] { return get_packet(); })\n{\n    // FIXME: ignored future\n    (void)_proto.receive(\n        [this](packet p, ethernet_address ea) {\n            return process_packet(std::move(p), ea);\n        },\n        [this](forward_hash& out_hash_data, packet& p, size_t off) {\n            return forward(out_hash_data, p, off);\n        });\n}\n\nstd::optional<l3_protocol::l3packet> arp::get_packet() {\n    std::optional<l3_protocol::l3packet> p;\n    if (!_packetq.empty()) {\n        p = std::move(_packetq.front());\n        _packetq.pop_front();\n    }\n    return p;\n}\n\nbool arp::forward(forward_hash& out_hash_data, packet& p, size_t off) {\n    auto ah = p.get_header<arp_hdr>(off);\n    auto i = _arp_for_protocol.find(ntoh(ah->ptype));\n    if (i != _arp_for_protocol.end()) {\n        return i->second->forward(out_hash_data, p, off);\n    }\n    return false;\n}\n\nvoid arp::add(uint16_t proto_num, arp_for_protocol* afp) {\n    _arp_for_protocol[proto_num] = afp;\n}\n\nvoid arp::del(uint16_t proto_num) {\n    _arp_for_protocol.erase(proto_num);\n}\n\nfuture<>\narp::process_packet(packet p, ethernet_address from) {\n    auto h = p.get_header(0, arp_hdr::size());\n    if (!h) {\n        return make_ready_future<>();\n    }\n    auto ah = arp_hdr::read(h);\n    auto i = _arp_for_protocol.find(ah.ptype);\n    if (i != _arp_for_protocol.end()) {\n        return i->second->received(std::move(p));\n    }\n    return make_ready_future<>();\n}\n\n}\n\n}\n\n// Path: src/core/condition-variable.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2020 ScyllaDB, Ltd.\n */\n#ifdef SEASTAR_MODULE\nmodule;\n#include <cassert>\n#include <exception>\n#include <utility>\nmodule seastar;\n#else\n#include <seastar/core/condition-variable.hh>\n#endif\n\nnamespace seastar {\n\nconst char* broken_condition_variable::what() const noexcept {\n    return \"Condition variable is broken\";\n}\n\nconst char* condition_variable_timed_out::what() const noexcept {\n    return \"Condition variable timed out\";\n}\n\ncondition_variable::~condition_variable() {\n    broken();\n}\n\nvoid condition_variable::add_waiter(waiter& w) noexcept {\n    assert(!_signalled); // should not have snuck between\n    if (_ex) {\n        w.set_exception(_ex);\n        return;\n    }\n    _waiters.push_back(w);\n}\n\nvoid condition_variable::waiter::timeout() noexcept {\n    this->unlink();\n    this->set_exception(std::make_exception_ptr(condition_variable_timed_out()));\n}\n\nbool condition_variable::wakeup_first() noexcept {\n    if (_waiters.empty()) {\n        return false;\n    }\n    auto& w = _waiters.front();\n    _waiters.pop_front();\n    if (_ex) {\n        w.set_exception(_ex);\n    } else {\n        w.signal();\n    }\n    return true;\n}\n\nbool condition_variable::check_and_consume_signal() noexcept {\n    return std::exchange(_signalled, false);\n}\n\nvoid condition_variable::signal() noexcept {\n    if (!wakeup_first()) {\n        _signalled = true;\n    }\n}\n\n/// Notify variable and wake up all waiter\nvoid condition_variable::broadcast() noexcept {\n    auto tmp(std::move(_waiters));\n    while (!tmp.empty()) {\n        auto& w = tmp.front();\n        tmp.pop_front();\n        if (_ex) {\n            w.set_exception(_ex);\n        } else {\n            w.signal();\n        }\n    }\n}\n\n/// Signal to waiters that an error occurred.  \\ref wait() will see\n/// an exceptional future<> containing the provided exception parameter.\n/// The future is made available immediately.\nvoid condition_variable::broken() noexcept {\n    broken(std::make_exception_ptr(broken_condition_variable()));\n}\n\nvoid condition_variable::broken(std::exception_ptr ep) noexcept {\n    _ex = ep;\n    broadcast();\n}\n\n} // namespace seastar\n\n// Path: src/core/smp.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2019 ScyllaDB\n */\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <boost/range/algorithm/find_if.hpp>\n#include <memory>\n#include <vector>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/smp.hh>\n#include <seastar/core/resource.hh>\n#include <seastar/core/loop.hh>\n#include <seastar/core/semaphore.hh>\n#include <seastar/core/print.hh>\n#include <seastar/core/on_internal_error.hh>\n#include \"prefault.hh\"\n#endif\n\nnamespace seastar {\n\nextern logger seastar_logger;\n\n#ifdef SEASTAR_BUILD_SHARED_LIBS\nshard_id* internal::this_shard_id_ptr() noexcept {\n    static thread_local shard_id g_this_shard_id;\n    return &g_this_shard_id;\n}\n#endif\n\nvoid smp_message_queue::work_item::process() {\n    schedule(this);\n}\n\nstruct smp_service_group_impl {\n    std::vector<smp_service_group_semaphore> clients;   // one client per server shard\n#ifdef SEASTAR_DEBUG\n    unsigned version = 0;\n#endif\n};\n\nstatic thread_local smp_service_group_semaphore smp_service_group_management_sem{1, named_semaphore_exception_factory{\"smp_service_group_management_sem\"}};\nstatic thread_local std::vector<smp_service_group_impl> smp_service_groups;\n\nstatic named_semaphore_exception_factory make_service_group_semaphore_exception_factory(unsigned id, shard_id client_cpu, shard_id this_cpu, std::optional<sstring> smp_group_name) {\n    if (smp_group_name) {\n        return named_semaphore_exception_factory{format(\"smp_service_group:'{}' (#{}) {}->{} semaphore\", *smp_group_name, id, client_cpu, this_cpu)};\n    } else {\n        return named_semaphore_exception_factory{format(\"smp_service_group#{} {}->{} semaphore\", id, client_cpu, this_cpu)};\n    }\n\n}\n\nstatic_assert(std::is_nothrow_copy_constructible_v<smp_service_group>);\nstatic_assert(std::is_nothrow_move_constructible_v<smp_service_group>);\n\nstatic_assert(std::is_nothrow_default_constructible_v<smp_submit_to_options>);\nstatic_assert(std::is_nothrow_copy_constructible_v<smp_submit_to_options>);\nstatic_assert(std::is_nothrow_move_constructible_v<smp_submit_to_options>);\n\nfuture<smp_service_group> create_smp_service_group(smp_service_group_config ssgc) noexcept {\n    ssgc.max_nonlocal_requests = std::max(ssgc.max_nonlocal_requests, smp::count - 1);\n    return smp::submit_to(0, [ssgc] {\n        return with_semaphore(smp_service_group_management_sem, 1, [ssgc] {\n            auto it = boost::range::find_if(smp_service_groups, [&] (smp_service_group_impl& ssgi) { return ssgi.clients.empty(); });\n            size_t id = it - smp_service_groups.begin();\n            return parallel_for_each(smp::all_cpus(), [ssgc, id] (unsigned cpu) {\n              return smp::submit_to(cpu, [ssgc, id, cpu] {\n                if (id >= smp_service_groups.size()) {\n                    smp_service_groups.resize(id + 1); // may throw\n                }\n                smp_service_groups[id].clients.reserve(smp::count); // may throw\n                auto per_client = smp::count > 1 ? ssgc.max_nonlocal_requests / (smp::count - 1) : 0u;\n                for (unsigned i = 0; i != smp::count; ++i) {\n                    smp_service_groups[id].clients.emplace_back(per_client, make_service_group_semaphore_exception_factory(id, i, cpu, ssgc.group_name));\n                }\n              });\n            }).handle_exception([id] (std::exception_ptr e) {\n                // rollback\n                return smp::invoke_on_all([id] {\n                    if (smp_service_groups.size() > id) {\n                        smp_service_groups[id].clients.clear();\n                    }\n                }).then([e = std::move(e)] () mutable {\n                    std::rethrow_exception(std::move(e));\n                });\n            }).then([id] {\n                auto ret = smp_service_group(id);\n#ifdef SEASTAR_DEBUG\n                ret._version = smp_service_groups[id].version;\n#endif\n                return ret;\n            });\n        });\n    });\n}\n\nfuture<> destroy_smp_service_group(smp_service_group ssg) noexcept {\n    return smp::submit_to(0, [ssg] {\n        return with_semaphore(smp_service_group_management_sem, 1, [ssg] {\n            auto id = internal::smp_service_group_id(ssg);\n            if (id >= smp_service_groups.size()) {\n                on_fatal_internal_error(seastar_logger, format(\"destroy_smp_service_group id={}: out of range\", id));\n            }\n#ifdef SEASTAR_DEBUG\n            if (ssg._version != smp_service_groups[id].version) {\n                on_fatal_internal_error(seastar_logger, format(\"destroy_smp_service_group id={}: stale version={}: current_version={}\", id, ssg._version, smp_service_groups[id].version));\n            }\n#endif\n            return smp::invoke_on_all([id] {\n                smp_service_groups[id].clients.clear();\n#ifdef SEASTAR_DEBUG\n                ++smp_service_groups[id].version;\n#endif\n            });\n        });\n    });\n}\n\nvoid init_default_smp_service_group(shard_id cpu) {\n...\n", "needle_token_start": 8908, "needle_token_end": 8952, "code_context_ntokens": 16161, "output": ["```cpp\nclass posix_datagram : public datagram_impl {\nprivate:\n    socket_address _src;\n    socket_address _dst;\n    packet _p;\npublic:\n    posix_datagram(const socket_address& src, const socket_address& dst, packet p) : _src(src), _dst(dst), _p(std::move(p)) {}\n    virtual socket_address get_src() override { return _src; }\n    virtual socket_address get_dst() override { return _dst; }\n    virtual uint16_t get_dst_port() override {\n        if (_dst.family() != AF_INET && _dst.family() != AF_INET6) {\n            throw std::runtime_error(format(\"get_dst_port() called on non-IP address: {}\", _dst));\n        }\n        return _dst.port();\n    }\n    virtual packet& get_data() override { return _p; }\n};\n```"]}
{"repo": "scylladb/seastar", "name": "arp_learn", "language": "cpp", "path": "src/net/native-stack.cc", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to update the Address Resolution Protocol (ARP) table across all processing units in a distributed system, associating a network layer address with a data link layer address.\n2. **Input**: It accepts two parameters: a data link layer address (such as a MAC address) and a network layer address (such as an IP address).\n3. **Output**: There is no direct output returned to the caller as the function operates asynchronously in the background.\n4. **Procedure**: The function broadcasts the task of updating the ARP table with the provided addresses to all processing units in the system. Each unit executes this update independently.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/net/virtio.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <atomic>\n#include <algorithm>\n#include <cassert>\n#include <cstring>\n#include <memory>\n#include <queue>\n#include <string>\n#include <vector>\n#include <fcntl.h>\n#include <seastar/net/virtio-interface.hh>\n#include <linux/vhost.h>\n#include <linux/if_tun.h>\n#include <net/if.h>\n#ifdef HAVE_OSV\n#include <osv/virtio-assign.hh>\n#endif\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/net/virtio.hh>\n#include <seastar/core/posix.hh>\n#include <seastar/core/internal/pollable_fd.hh>\n#include <seastar/core/internal/poll.hh>\n#include \"core/vla.hh\"\n#include <seastar/core/reactor.hh>\n#include <seastar/core/stream.hh>\n#include <seastar/core/circular_buffer.hh>\n#include <seastar/core/align.hh>\n#include <seastar/core/metrics.hh>\n#include <seastar/util/function_input_iterator.hh>\n#include <seastar/util/transform_iterator.hh>\n#include <seastar/net/ip.hh>\n#include <seastar/net/const.hh>\n#include <seastar/net/native-stack.hh>\n#endif\n\nnamespace seastar {\n\nusing namespace net;\n\nnamespace virtio {\n\nusing phys = uint64_t;\n\n#ifndef HAVE_OSV\n\nphys virt_to_phys(void* p) {\n    return reinterpret_cast<uintptr_t>(p);\n}\n\n#else\n\nphys virt_to_phys(void* p) {\n    return osv::assigned_virtio::virt_to_phys(p);\n}\n\n#endif\n\nclass device : public net::device {\nprivate:\n    net::hw_features _hw_features;\n    uint64_t _features;\n\nprivate:\n    uint64_t setup_features(const net::virtio_options& opts, const program_options::value<std::string>& lro) {\n        int64_t seastar_supported_features = VIRTIO_RING_F_INDIRECT_DESC | VIRTIO_NET_F_MRG_RXBUF;\n\n        if (!(opts.event_index && opts.event_index.get_value() == \"off\")) {\n            seastar_supported_features |= VIRTIO_RING_F_EVENT_IDX;\n        }\n        if (!(opts.csum_offload && opts.csum_offload.get_value() == \"off\")) {\n            seastar_supported_features |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;\n            _hw_features.tx_csum_l4_offload = true;\n            _hw_features.rx_csum_offload = true;\n        } else {\n            _hw_features.tx_csum_l4_offload = false;\n            _hw_features.rx_csum_offload = false;\n        }\n        if (!(opts.tso && opts.tso.get_value() == \"off\")) {\n            seastar_supported_features |= VIRTIO_NET_F_HOST_TSO4;\n            _hw_features.tx_tso = true;\n        } else {\n            _hw_features.tx_tso = false;\n        }\n\n        if (!(lro && lro.get_value() == \"off\")) {\n            seastar_supported_features |= VIRTIO_NET_F_GUEST_TSO4;\n            _hw_features.rx_lro = true;\n        } else {\n            _hw_features.rx_lro = false;\n        }\n\n        if (!(opts.ufo && opts.ufo.get_value() == \"off\")) {\n            seastar_supported_features |= VIRTIO_NET_F_HOST_UFO;\n            seastar_supported_features |= VIRTIO_NET_F_GUEST_UFO;\n            _hw_features.tx_ufo = true;\n        } else {\n            _hw_features.tx_ufo = false;\n        }\n\n        seastar_supported_features |= VIRTIO_NET_F_MAC;\n        return seastar_supported_features;\n    }\n\npublic:\n    device(const virtio_options& opts, const program_options::value<std::string>& lro)\n       : _features(setup_features(opts, lro))\n       {}\n    ethernet_address hw_address() override {\n        return { 0x12, 0x23, 0x34, 0x56, 0x67, 0x78 };\n    }\n\n    net::hw_features hw_features() override {\n        return _hw_features;\n    }\n\n    uint64_t features() {\n        return _features;\n    }\n\n    virtual std::unique_ptr<net::qp> init_local_queue(const program_options::option_group& opts, uint16_t qid) override;\n};\n\n/* The virtio_notifier class determines how to do host-to-guest and guest-to-\n * host notifications. We have two different implementations - one for vhost\n * (where both notifications occur through eventfds) and one for an assigned\n * virtio device from OSv.\n */\nclass notifier {\npublic:\n    // Notify the host\n    virtual void notify() = 0;\n    // Do whatever it takes to wake wait(). A notifier does not need to\n    // implement this function if wait() waits for an external even which is\n    // generated by an external process (e.g., virtio_notifier_host doesn't\n    // need to implement this).\n    virtual void wake_wait() {\n        abort();\n    }\n    virtual ~notifier() {\n    }\n};\n\nclass notifier_vhost : public notifier {\nprivate:\n    writeable_eventfd _kick;\npublic:\n    virtual void notify() override {\n        _kick.signal(1);\n    }\n    notifier_vhost(writeable_eventfd &&kick)\n        : _kick(std::move(kick)) {}\n};\n\n#ifdef HAVE_OSV\nclass notifier_osv : public notifier {\nprivate:\n    uint16_t _q_index;\n...\n// Path: src/net/inet_address.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2016 ScyllaDB.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <algorithm>\n#include <ostream>\n#include <arpa/inet.h>\n#include <boost/functional/hash.hpp>\n#include <fmt/ostream.h>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/net/inet_address.hh>\n#include <seastar/net/socket_defs.hh>\n#include <seastar/net/dns.hh>\n#include <seastar/net/ip.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/print.hh>\n#endif\n\nstatic_assert(std::is_nothrow_default_constructible_v<seastar::net::ipv4_address>);\nstatic_assert(std::is_nothrow_copy_constructible_v<seastar::net::ipv4_address>);\nstatic_assert(std::is_nothrow_move_constructible_v<seastar::net::ipv4_address>);\n\nstatic_assert(std::is_nothrow_default_constructible_v<seastar::net::ipv6_address>);\nstatic_assert(std::is_nothrow_copy_constructible_v<seastar::net::ipv6_address>);\nstatic_assert(std::is_nothrow_move_constructible_v<seastar::net::ipv6_address>);\n\nstatic_assert(std::is_nothrow_default_constructible_v<seastar::net::inet_address>);\nstatic_assert(std::is_nothrow_copy_constructible_v<seastar::net::inet_address>);\nstatic_assert(std::is_nothrow_move_constructible_v<seastar::net::inet_address>);\n\nseastar::net::inet_address::inet_address() noexcept\n                : inet_address(::in6_addr{})\n{}\n\nseastar::net::inet_address::inet_address(family f) noexcept\n                : _in_family(f)\n{\n    memset(&_in6, 0, sizeof(_in6));\n}\n\nseastar::net::inet_address::inet_address(::in_addr i) noexcept\n                : _in_family(family::INET), _in(i) {\n}\n\nseastar::net::inet_address::inet_address(::in6_addr i, uint32_t scope) noexcept\n                : _in_family(family::INET6), _in6(i), _scope(scope) {\n}\n\nstd::optional<seastar::net::inet_address> \nseastar::net::inet_address::parse_numerical(const sstring& addr) {\n    inet_address in;\n    if (::inet_pton(AF_INET, addr.c_str(), &in._in)) {\n        in._in_family = family::INET;\n        return in;\n    }\n    auto i = addr.find_last_of('%');\n    if (i != sstring::npos) {\n        auto ext = addr.substr(i + 1);\n        auto src = addr.substr(0, i);\n        auto res = parse_numerical(src);\n\n        if (res) {\n            uint32_t index = std::numeric_limits<uint32_t>::max();\n            try {\n                index = std::stoul(ext);\n            } catch (...) {\n            }\n            for (auto& nwif : engine().net().network_interfaces()) {\n                if (nwif.index() == index || nwif.name() == ext || nwif.display_name() == ext) {\n                    res->_scope = nwif.index();\n                    break;\n                }\n            }\n            return *res;\n        }\n    }\n    if (::inet_pton(AF_INET6, addr.c_str(), &in._in6)) {\n        in._in_family = family::INET6;\n        return in;\n    }\n    return {};\n}\n\nseastar::net::inet_address::inet_address(const sstring& addr)\n                : inet_address([&addr] {\n    auto res = parse_numerical(addr);                        \n    if (res) {\n        return std::move(*res);\n    }\n    throw std::invalid_argument(addr);\n}())\n{}\n\nseastar::net::inet_address::inet_address(const ipv4_address& in) noexcept\n    : inet_address(::in_addr{hton(in.ip)})\n{}\n\nseastar::net::inet_address::inet_address(const ipv6_address& in, uint32_t scope) noexcept\n    : inet_address([&] {\n        ::in6_addr tmp;\n        std::copy(in.bytes().begin(), in.bytes().end(), tmp.s6_addr);\n        return tmp;\n    }(), scope)\n{}\n\nseastar::net::ipv4_address seastar::net::inet_address::as_ipv4_address() const {\n    in_addr in = *this;\n    return ipv4_address(ntoh(in.s_addr));\n}\n\nseastar::net::ipv6_address seastar::net::inet_address::as_ipv6_address() const noexcept {\n    in6_addr in6 = *this;\n    return ipv6_address{in6};\n}\n\nbool seastar::net::inet_address::operator==(const inet_address& o) const noexcept {\n    if (o._in_family != _in_family) {\n        return false;\n    }\n    switch (_in_family) {\n    case family::INET:\n        return _in.s_addr == o._in.s_addr;\n    case family::INET6:\n        return std::equal(std::begin(_in6.s6_addr), std::end(_in6.s6_addr), std::begin(o._in6.s6_addr));\n    default:\n        return false;\n    }\n}\n\nseastar::net::inet_address::operator ::in_addr() const {\n    if (_in_family != family::INET) {\n        if (IN6_IS_ADDR_V4MAPPED(&_in6)) {\n            ::in_addr in;\n            in.s_addr = _in6.s6_addr32[3];\n            return in;\n        }\n        throw std::invalid_argument(\"Not an IPv4 address\");\n    }\n    return _in;\n}\n\nseastar::net::inet_address::operator ::in6_addr() const noexcept {\n    if (_in_family == family::INET) {\n        in6_addr in6 = IN6ADDR_ANY_INIT;\n        in6.s6_addr32[2] = htonl(0xffff);\n        in6.s6_addr32[3] = _in.s_addr;\n        return in6;\n    }\n    return _in6;\n}\n\nseastar::net::inet_address::operator seastar::net::ipv6_address() const noexcept {\n    return as_ipv6_address();\n}\n\nsize_t seastar::net::inet_address::size() const noexcept {\n    switch (_in_family) {\n    case family::INET:\n        return sizeof(::in_addr);\n    case family::INET6:\n        return sizeof(::in6_addr);\n    default:\n        return 0;\n    }\n}\n\nconst void * seastar::net::inet_address::data() const noexcept {\n    return &_in;\n}\n\nbool seastar::net::inet_address::is_loopback() const noexcept {\n    switch (_in_family) {\n    case family::INET:\n        return (net::ntoh(_in.s_addr) & 0xff000000) == 0x7f000000;\n    case family::INET6:\n        return std::equal(std::begin(_in6.s6_addr), std::end(_in6.s6_addr), std::begin(::in6addr_loopback.s6_addr));\n    default:\n        return false;\n    }\n}\n\nbool seastar::net::inet_address::is_addr_any() const noexcept {\n    switch (_in_family) {\n    case family::INET:\n        return _in.s_addr == INADDR_ANY;\n    case family::INET6:\n        return std::equal(std::begin(_in6.s6_addr), std::end(_in6.s6_addr), std::begin(::in6addr_any.s6_addr));\n    default:\n        return false;\n    }\n}\n\nseastar::net::ipv6_address::ipv6_address(const ::in6_addr& in) noexcept {\n    std::copy(std::begin(in.s6_addr), std::end(in.s6_addr), ip.begin());\n}\n\nseastar::net::ipv6_address::ipv6_address(const ipv6_bytes& in) noexcept\n    : ip(in)\n{}\n\nseastar::net::ipv6_address::ipv6_address(const ipv6_addr& addr) noexcept\n    : ipv6_address(addr.ip)\n{}\n\nseastar::net::ipv6_address::ipv6_address() noexcept\n    : ipv6_address(::in6addr_any)\n{}\n\nseastar::net::ipv6_address::ipv6_address(const std::string& addr) {\n    if (!::inet_pton(AF_INET6, addr.c_str(), ip.data())) {\n        throw std::runtime_error(fmt::format(\"Wrong format for IPv6 address {}. Please ensure it's in colon-hex format\",\n                                        addr));\n    }\n}\n\nseastar::net::ipv6_address seastar::net::ipv6_address::read(const char* s) noexcept {\n    auto* b = reinterpret_cast<const uint8_t *>(s);\n    ipv6_address in;\n    std::copy(b, b + ipv6_address::size(), in.ip.begin());\n    return in;\n}\n\nseastar::net::ipv6_address seastar::net::ipv6_address::consume(const char*& p) noexcept {\n    auto res = read(p);\n    p += size();\n    return res;\n}\n\nvoid seastar::net::ipv6_address::write(char* p) const noexcept {\n    std::copy(ip.begin(), ip.end(), p);\n}\n\nvoid seastar::net::ipv6_address::produce(char*& p) const noexcept {\n    write(p);\n    p += size();\n}\n\nbool seastar::net::ipv6_address::is_unspecified() const noexcept {\n    return std::all_of(ip.begin(), ip.end(), [](uint8_t b) { return b == 0; });\n}\n\nstd::ostream& seastar::net::operator<<(std::ostream& os, const ipv4_address& a) {\n    auto ip = a.ip;\n    fmt::print(os, \"{:d}.{:d}.{:d}.{:d}\",\n            (ip >> 24) & 0xff,\n            (ip >> 16) & 0xff,\n            (ip >> 8) & 0xff,\n            (ip >> 0) & 0xff);\n    return os;\n}\n\nstd::ostream& seastar::net::operator<<(std::ostream& os, const ipv6_address& a) {\n    char buffer[64];\n    return os << ::inet_ntop(AF_INET6, a.ip.data(), buffer, sizeof(buffer));\n}\n\nseastar::ipv6_addr::ipv6_addr(const ipv6_bytes& b, uint16_t p) noexcept\n    : ip(b), port(p)\n{}\n\nseastar::ipv6_addr::ipv6_addr(uint16_t p) noexcept\n    : ipv6_addr(net::inet_address(), p)\n{}\n\nseastar::ipv6_addr::ipv6_addr(const ::in6_addr& in6, uint16_t p) noexcept\n    : ipv6_addr(net::ipv6_address(in6).bytes(), p)\n{}\n\nseastar::ipv6_addr::ipv6_addr(const std::string& s)\n    : ipv6_addr([&] {\n        auto lc = s.find_last_of(']');\n        auto cp = s.find_first_of(':', lc);\n        auto port = cp != std::string::npos ? std::stoul(s.substr(cp + 1)) : 0;\n        auto ss = lc != std::string::npos ? s.substr(1, lc - 1) : s;\n        return ipv6_addr(net::ipv6_address(ss).bytes(), uint16_t(port));\n    }())\n{}\n\nseastar::ipv6_addr::ipv6_addr(const std::string& s, uint16_t p)\n    : ipv6_addr(net::ipv6_address(s).bytes(), p)\n{}\n\nseastar::ipv6_addr::ipv6_addr(const net::inet_address& i, uint16_t p) noexcept\n    : ipv6_addr(i.as_ipv6_address().bytes(), p)\n{}\n\nseastar::ipv6_addr::ipv6_addr(const ::sockaddr_in6& s) noexcept\n    : ipv6_addr(s.sin6_addr, net::ntoh(s.sin6_port))\n{}\n\nseastar::ipv6_addr::ipv6_addr(const socket_address& s) noexcept\n    : ipv6_addr(s.as_posix_sockaddr_in6())\n{}\n\nbool seastar::ipv6_addr::is_ip_unspecified() const noexcept {\n    return std::all_of(ip.begin(), ip.end(), [](uint8_t b) { return b == 0; });\n}\n\n\nseastar::net::inet_address seastar::socket_address::addr() const noexcept {\n    switch (family()) {\n    case AF_INET:\n        return net::inet_address(as_posix_sockaddr_in().sin_addr);\n    case AF_INET6:\n        return net::inet_address(as_posix_sockaddr_in6().sin6_addr, as_posix_sockaddr_in6().sin6_scope_id);\n    default:\n        return net::inet_address();\n    }\n}\n\n::in_port_t seastar::socket_address::port() const noexcept {\n    return net::ntoh(u.in.sin_port);\n}\n\nbool seastar::socket_address::is_wildcard() const noexcept {\n    switch (family()) {\n    case AF_INET: {\n            ipv4_addr addr(*this);\n            return addr.is_ip_unspecified() && addr.is_port_unspecified();\n        }\n    default:\n    case AF_INET6: {\n            ipv6_addr addr(*this);\n            return addr.is_ip_unspecified() && addr.is_port_unspecified();\n        }\n    case AF_UNIX:\n        return length() <= sizeof(::sa_family_t);\n    }\n}\n\nstd::ostream& seastar::net::operator<<(std::ostream& os, const inet_address& addr) {\n    char buffer[64];\n    os << inet_ntop(int(addr.in_family()), addr.data(), buffer, sizeof(buffer));\n    if (addr.scope() != inet_address::invalid_scope) {\n        os << \"%\" << addr.scope();\n    }\n    return os;\n}\n\nstd::ostream& seastar::net::operator<<(std::ostream& os, const inet_address::family& f) {\n    switch (f) {\n    case inet_address::family::INET:\n        os << \"INET\";\n        break;\n    case inet_address::family::INET6:\n        os << \"INET6\";\n        break;\n    default:\n        break;\n    }\n    return os;\n}\n\nstd::ostream& seastar::operator<<(std::ostream& os, const ipv4_addr& a) {\n    return os << seastar::socket_address(a);\n}\n\nstd::ostream& seastar::operator<<(std::ostream& os, const ipv6_addr& a) {\n    return os << seastar::socket_address(a);\n}\n\nsize_t std::hash<seastar::net::inet_address>::operator()(const seastar::net::inet_address& a) const {\n    switch (a.in_family()) {\n    case seastar::net::inet_address::family::INET:\n        return std::hash<seastar::net::ipv4_address>()(a.as_ipv4_address());\n    case seastar::net::inet_address::family::INET6:\n        return std::hash<seastar::net::ipv6_address>()(a.as_ipv6_address());\n    default:\n        return 0;\n    }\n}\n\nsize_t std::hash<seastar::net::ipv6_address>::operator()(const seastar::net::ipv6_address& a) const {\n    return boost::hash_range(a.ip.begin(), a.ip.end());\n}\n\nsize_t std::hash<seastar::ipv4_addr>::operator()(const seastar::ipv4_addr& x) const {\n    size_t h = x.ip;\n    boost::hash_combine(h, x.port);\n    return h;\n}\n\n// Path: src/net/native-stack.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2014 Cloudius Systems, Ltd.\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <cassert>\n#include <chrono>\n#include <fstream>\n#include <functional>\n#include <map>\n#include <memory>\n#include <optional>\n#include <queue>\n\n#include <sys/types.h>\n#include <sys/stat.h>\n#include <arpa/inet.h>\n#include <unistd.h>\n\n#ifdef HAVE_OSV\n#include <osv/firmware.hh>\n#include <gnu/libc-version.h>\n#endif\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/net/native-stack.hh>\n#include \"net/native-stack-impl.hh\"\n#include <seastar/net/net.hh>\n#include <seastar/net/ip.hh>\n#include <seastar/net/tcp-stack.hh>\n#include <seastar/net/tcp.hh>\n#include <seastar/net/udp.hh>\n#include <seastar/net/virtio.hh>\n#include <seastar/net/dpdk.hh>\n#include <seastar/net/proxy.hh>\n#include <seastar/net/dhcp.hh>\n#include <seastar/net/config.hh>\n#include <seastar/core/reactor.hh>\n#endif\n\nnamespace seastar {\n\nnamespace net {\n\nusing namespace seastar;\n\nvoid create_native_net_device(const native_stack_options& opts) {\n\n    bool deprecated_config_used = true;\n\n    std::stringstream net_config;\n\n    if ( opts.net_config) {\n        deprecated_config_used = false;\n        net_config << opts.net_config.get_value();\n    }\n    if ( opts.net_config_file) {\n        deprecated_config_used = false;\n        std::fstream fs(opts.net_config_file.get_value());\n        net_config << fs.rdbuf();\n    }\n\n    std::unique_ptr<device> dev;\n\n    if ( deprecated_config_used) {\n#ifdef SEASTAR_HAVE_DPDK\n        if ( opts.dpdk_pmd) {\n             dev = create_dpdk_net_device(opts.dpdk_opts.dpdk_port_index.get_value(), smp::count,\n                !(opts.lro && opts.lro.get_value() == \"off\"),\n                !(opts.dpdk_opts.hw_fc && opts.dpdk_opts.hw_fc.get_value() == \"off\"));\n       } else \n#endif  \n        dev = create_virtio_net_device(opts.virtio_opts, opts.lro);\n    }\n    else {\n        auto device_configs = parse_config(net_config);\n\n        if ( device_configs.size() > 1) {\n            std::runtime_error(\"only one network interface is supported\");\n        }\n\n        for ( auto&& device_config : device_configs) {\n            auto& hw_config = device_config.second.hw_cfg;   \n#ifdef SEASTAR_HAVE_DPDK\n            if ( hw_config.port_index || !hw_config.pci_address.empty() ) {\n\t            dev = create_dpdk_net_device(hw_config);\n\t        } else \n#endif  \n            {\n                (void)hw_config;        \n                std::runtime_error(\"only DPDK supports new configuration format\"); \n            }\n        }\n    }\n\n    auto sem = std::make_shared<semaphore>(0);\n    std::shared_ptr<device> sdev(dev.release());\n    // set_local_queue on all shard in the background,\n    // signal when done.\n    // FIXME: handle exceptions\n    for (unsigned i = 0; i < smp::count; i++) {\n        (void)smp::submit_to(i, [&opts, sdev] {\n            uint16_t qid = this_shard_id();\n            if (qid < sdev->hw_queues_count()) {\n                auto qp = sdev->init_local_queue(opts, qid);\n                std::map<unsigned, float> cpu_weights;\n                for (unsigned i = sdev->hw_queues_count() + qid % sdev->hw_queues_count(); i < smp::count; i+= sdev->hw_queues_count()) {\n                    cpu_weights[i] = 1;\n                }\n                cpu_weights[qid] = opts.hw_queue_weight.get_value();\n                qp->configure_proxies(cpu_weights);\n                sdev->set_local_queue(std::move(qp));\n            } else {\n                auto master = qid % sdev->hw_queues_count();\n                sdev->set_local_queue(create_proxy_net_device(master, sdev.get()));\n            }\n        }).then([sem] {\n            sem->signal();\n        });\n    }\n    // wait for all shards to set their local queue,\n    // then when link is ready, communicate the native_stack to the caller\n    // via `create_native_stack` (that sets the ready_promise value)\n    (void)sem->wait(smp::count).then([&opts, sdev] {\n        // FIXME: future is discarded\n        (void)sdev->link_ready().then([&opts, sdev] {\n            for (unsigned i = 0; i < smp::count; i++) {\n                // FIXME: future is discarded\n                (void)smp::submit_to(i, [&opts, sdev] {\n                    create_native_stack(opts, sdev);\n                });\n            }\n        });\n    });\n}\n\n// native_network_stack\nclass native_network_stack : public network_stack {\npublic:\n    static thread_local promise<std::unique_ptr<network_stack>> ready_promise;\nprivate:\n    interface _netif;\n    ipv4 _inet;\n    bool _dhcp = false;\n    promise<> _config;\n    timer<> _timer;\n\n    future<> run_dhcp(bool is_renew = false, const dhcp::lease & res = dhcp::lease());\n    void on_dhcp(std::optional<dhcp::lease> lease, bool is_renew);\n    void set_ipv4_packet_filter(ip_packet_filter* filter) {\n        _inet.set_packet_filter(filter);\n    }\n    using tcp4 = tcp<ipv4_traits>;\npublic:\n    explicit native_network_stack(const native_stack_options& opts, std::shared_ptr<device> dev);\n    virtual server_socket listen(socket_address sa, listen_options opt) override;\n    virtual ::seastar::socket socket() override;\n    virtual udp_channel make_udp_channel(const socket_address& addr) override;\n    virtual net::datagram_channel make_unbound_datagram_channel(sa_family_t) override;\n    virtual net::datagram_channel make_bound_datagram_channel(const socket_address& local) override;\n    virtual future<> initialize() override;\n    static future<std::unique_ptr<network_stack>> create(const program_options::option_group& opts) {\n        auto ns_opts = dynamic_cast<const native_stack_options*>(&opts);\n        assert(ns_opts);\n        if (this_shard_id() == 0) {\n            create_native_net_device(*ns_opts);\n        }\n        return ready_promise.get_future();\n    }\n    virtual bool has_per_core_namespace() override { return true; };\n    void arp_learn(ethernet_address l2, ipv4_address l3) {\n        _inet.learn(l2, l3);\n    }\n    friend class native_server_socket_impl<tcp4>;\n\n    class native_network_interface;\n    friend class native_network_interface;\n\n    std::vector<network_interface> network_interfaces() override;\n};\n\nthread_local promise<std::unique_ptr<network_stack>> native_network_stack::ready_promise;\n\nudp_channel\nnative_network_stack::make_udp_channel(const socket_address& addr) {\n    return _inet.get_udp().make_channel(addr);\n}\n\nnet::datagram_channel native_network_stack::make_unbound_datagram_channel(sa_family_t family) {\n    if (family != AF_INET) {\n        throw std::runtime_error(\"Unsupported address family\");\n    }\n\n    return _inet.get_udp().make_channel({});\n}\n\nnet::datagram_channel native_network_stack::make_bound_datagram_channel(const socket_address& local) {\n    return _inet.get_udp().make_channel(local);\n}\n\nnative_network_stack::native_network_stack(const native_stack_options& opts, std::shared_ptr<device> dev)\n    : _netif(std::move(dev))\n    , _inet(&_netif) {\n    _inet.get_udp().set_queue_size(opts.udpv4_queue_size.get_value());\n    _dhcp = opts.host_ipv4_addr.defaulted()\n            && opts.gw_ipv4_addr.defaulted()\n            && opts.netmask_ipv4_addr.defaulted() && opts.dhcp.get_value();\n    if (!_dhcp) {\n        _inet.set_host_address(ipv4_address(opts.host_ipv4_addr.get_value()));\n        _inet.set_gw_address(ipv4_address(opts.gw_ipv4_addr.get_value()));\n        _inet.set_netmask_address(ipv4_address(opts.netmask_ipv4_addr.get_value()));\n    }\n}\n\nserver_socket\nnative_network_stack::listen(socket_address sa, listen_options opts) {\n    assert(sa.family() == AF_INET || sa.is_unspecified());\n    return tcpv4_listen(_inet.get_tcp(), ntohs(sa.as_posix_sockaddr_in().sin_port), opts);\n}\n\nseastar::socket native_network_stack::socket() {\n    return tcpv4_socket(_inet.get_tcp());\n}\n\nusing namespace std::chrono_literals;\n\nfuture<> native_network_stack::run_dhcp(bool is_renew, const dhcp::lease& res) {\n    dhcp d(_inet);\n    // Hijack the ip-stack.\n    auto f = d.get_ipv4_filter();\n    return smp::invoke_on_all([f] {\n        auto & ns = static_cast<native_network_stack&>(engine().net());\n        ns.set_ipv4_packet_filter(f);\n    }).then([this, d = std::move(d), is_renew, res = res]() mutable {\n        net::dhcp::result_type fut = is_renew ? d.renew(res) : d.discover();\n        return fut.then([this, is_renew](std::optional<dhcp::lease> lease) {\n            return smp::invoke_on_all([] {\n                auto & ns = static_cast<native_network_stack&>(engine().net());\n                ns.set_ipv4_packet_filter(nullptr);\n            }).then(std::bind(&net::native_network_stack::on_dhcp, this, lease, is_renew));\n        }).finally([d = std::move(d)] {});\n    });\n}\n\nvoid native_network_stack::on_dhcp(std::optional<dhcp::lease> lease, bool is_renew) {\n    if (lease) {\n        auto& res = *lease;\n        _inet.set_host_address(res.ip);\n        _inet.set_gw_address(res.gateway);\n        _inet.set_netmask_address(res.netmask);\n    }\n    // Signal waiters.\n    if (!is_renew) {\n        _config.set_value();\n    }\n\n    if (this_shard_id() == 0) {\n        // And the other cpus, which, in the case of initial discovery,\n        // will be waiting for us.\n        for (unsigned i = 1; i < smp::count; i++) {\n            (void)smp::submit_to(i, [lease, is_renew]() {\n                auto & ns = static_cast<native_network_stack&>(engine().net());\n                ns.on_dhcp(lease, is_renew);\n            });\n        }\n        if (lease) {\n            // And set up to renew the lease later on.\n            auto& res = *lease;\n            _timer.set_callback(\n                    [this, res]() {\n                        _config = promise<>();\n                        // callback ignores future result\n                        (void)run_dhcp(true, res);\n                    });\n            _timer.arm(\n                    std::chrono::duration_cast<steady_clock_type::duration>(\n                            res.lease_time));\n        }\n    }\n}\n\nfuture<> native_network_stack::initialize() {\n    return network_stack::initialize().then([this]() {\n        if (!_dhcp) {\n            return make_ready_future();\n        }\n\n        // Only run actual discover on main cpu.\n        // All other cpus must simply for main thread to complete and signal them.\n        if (this_shard_id() == 0) {\n            // FIXME: future is discarded\n            (void)run_dhcp();\n        }\n        return _config.get_future();\n    });\n}\n\n\nvoid arp_learn(ethernet_address l2, ipv4_address l3)\n{\n    // Run arp_learn on all shard in the background\n    (void)smp::invoke_on_all([l2, l3] {\n        auto & ns = static_cast<native_network_stack&>(engine().net());\n        ns.arp_learn(l2, l3);\n    });\n}\n\nvoid create_native_stack(const native_stack_options& opts, std::shared_ptr<device> dev) {\n    native_network_stack::ready_promise.set_value(std::unique_ptr<network_stack>(std::make_unique<native_network_stack>(opts, std::move(dev))));\n}\n\nnative_stack_options::native_stack_options()\n    : program_options::option_group(nullptr, \"Native networking stack options\")\n    // these two are ghost options\n    , net_config(*this, \"net-config\", program_options::unused{})\n    , net_config_file(*this, \"net-config-file\", program_options::unused{})\n    , tap_device(*this, \"tap-device\",\n                \"tap0\",\n                \"tap device to connect to\")\n    , host_ipv4_addr(*this, \"host-ipv4-addr\",\n                \"192.168.122.2\",\n                \"static IPv4 address to use\")\n    , gw_ipv4_addr(*this, \"gw-ipv4-addr\",\n                \"192.168.122.1\",\n                \"static IPv4 gateway to use\")\n    , netmask_ipv4_addr(*this, \"netmask-ipv4-addr\",\n                \"255.255.255.0\",\n                \"static IPv4 netmask to use\")\n    , udpv4_queue_size(*this, \"udpv4-queue-size\",\n                ipv4_udp::default_queue_size,\n                \"Default size of the UDPv4 per-channel packet queue\")\n    , dhcp(*this, \"dhcp\",\n                true,\n                        \"Use DHCP discovery\")\n    , hw_queue_weight(*this, \"hw-queue-weight\",\n                1.0f,\n                \"Weighing of a hardware network queue relative to a software queue (0=no work, 1=equal share)\")\n#ifdef SEASTAR_HAVE_DPDK\n    , dpdk_pmd(*this, \"dpdk-pmd\", \"Use DPDK PMD drivers\")\n#else\n    , dpdk_pmd(*this, \"dpdk-pmd\", program_options::unused{})\n#endif\n    , lro(*this, \"lro\",\n                \"on\",\n                \"Enable LRO\")\n    , virtio_opts(this)\n    , dpdk_opts(this)\n{\n}\n\nnetwork_stack_entry register_native_stack() {\n    return network_stack_entry{\"native\", std::make_unique<native_stack_options>(), native_network_stack::create, false};\n}\n\nclass native_network_stack::native_network_interface : public net::network_interface_impl {\n    const native_network_stack& _stack;\n    std::vector<net::inet_address> _addresses;\n    std::vector<uint8_t> _hardware_address;\npublic:\n    native_network_interface(const native_network_stack& stack)\n        : _stack(stack)\n        , _addresses(1, _stack._inet.host_address())\n    {\n        const auto mac = _stack._inet.netif()->hw_address().mac;\n        _hardware_address = std::vector<uint8_t>{mac.cbegin(), mac.cend()};\n    }\n    native_network_interface(const native_network_interface&) = default;\n\n    uint32_t index() const override {\n        return 0;\n    }\n    uint32_t mtu() const override {\n        return _stack._inet.netif()->hw_features().mtu;\n    }\n    const sstring& name() const override {\n        static const sstring name = \"if0\";\n        return name;\n    }\n    const sstring& display_name() const override {\n        return name();\n    }\n    const std::vector<net::inet_address>& addresses() const override {\n        return _addresses;            \n    }\n    const std::vector<uint8_t> hardware_address() const override {\n        return _hardware_address;\n    }\n    bool is_loopback() const override {\n        return false;   \n    }\n    bool is_virtual() const override {\n        return false;\n    }\n    bool is_up() const override {\n        return true;\n    }\n    bool supports_ipv6() const override {\n        return false;\n    }\n};\n\nstd::vector<network_interface> native_network_stack::network_interfaces() {\n    if (!_inet.netif()) {\n        return {};\n    }\n\n    static const native_network_interface nwif(*this);\n\n    std::vector<network_interface> res;\n    res.emplace_back(make_shared<native_network_interface>(nwif));\n    return res;\n}\n\n}\n\n}\n\n// Path: src/net/tls.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2015 Cloudius Systems\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <filesystem>\n#include <stdexcept>\n#include <system_error>\n#include <memory>\n#include <chrono>\n#include <unordered_set>\n\n#include <netinet/in.h>\n#include <sys/stat.h>\n#include <gnutls/gnutls.h>\n#include <gnutls/x509.h>\n\n#include <boost/any.hpp>\n#include <boost/range/iterator_range.hpp>\n#include <boost/range/adaptor/map.hpp>\n\n#include <fmt/core.h>\n#include <fmt/ostream.h>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/core/loop.hh>\n#include <seastar/core/reactor.hh>\n#include <seastar/core/seastar.hh>\n#include <seastar/core/file.hh>\n#include <seastar/core/thread.hh>\n#include <seastar/core/sstring.hh>\n#include <seastar/core/semaphore.hh>\n#include <seastar/core/timer.hh>\n#include <seastar/core/print.hh>\n#include <seastar/core/with_timeout.hh>\n#include <seastar/net/tls.hh>\n#include <seastar/net/stack.hh>\n#include <seastar/util/std-compat.hh>\n#include <seastar/util/variant_utils.hh>\n#include <seastar/core/fsnotify.hh>\n#endif\n\nnamespace seastar {\n\nclass net::get_impl {\npublic:\n    static std::unique_ptr<connected_socket_impl> get(connected_socket s) {\n        return std::move(s._csi);\n    }\n\n    static connected_socket_impl* maybe_get_ptr(connected_socket& s) {\n        if (s._csi) {\n            return s._csi.get();\n        }\n        return nullptr;\n    }\n};\n\nclass blob_wrapper: public gnutls_datum_t {\npublic:\n    blob_wrapper(const tls::blob& in)\n            : gnutls_datum_t {\n                    reinterpret_cast<uint8_t *>(const_cast<char *>(in.data())),\n                    unsigned(in.size()) } {\n    }\n};\n\nclass gnutlsinit {\npublic:\n    gnutlsinit() {\n        gnutls_global_init();\n    }\n    ~gnutlsinit() {\n        gnutls_global_deinit();\n    }\n};\n\n// Helper to ensure gnutls legacy init\n// is handled properly with regards to\n// object life spans. Could be better,\n// this version will not destroy the\n// gnutls stack until process exit.\nclass gnutlsobj {\npublic:\n    gnutlsobj() {\n        static gnutlsinit init;\n    }\n};\n\n// Helper\nstruct file_info {\n    sstring filename;\n    std::chrono::system_clock::time_point modified;\n};\n\nstruct file_result {\n    temporary_buffer<char> buf;\n    file_info file;\n    operator temporary_buffer<char>&&() && {\n        return std::move(buf);\n    }\n};\n\nstatic future<file_result> read_fully(const sstring& name, const sstring& what) {\n    return open_file_dma(name, open_flags::ro).then([name = name](file f) mutable {\n        return do_with(std::move(f), [name = std::move(name)](file& f) mutable {\n            return f.stat().then([&f, name = std::move(name)](struct stat s) mutable {\n                return f.dma_read_bulk<char>(0, s.st_size).then([s, name = std::move(name)](temporary_buffer<char> buf) mutable {\n                    return file_result{ std::move(buf), file_info{ \n                        std::move(name), std::chrono::system_clock::from_time_t(s.st_mtim.tv_sec) +\n                            std::chrono::duration_cast<std::chrono::system_clock::duration>(std::chrono::nanoseconds(s.st_mtim.tv_nsec))\n                    } };\n                });\n            }).finally([&f]() {\n                return f.close();\n            });\n        });\n    }).handle_exception([name = name, what = what](std::exception_ptr ep) -> future<file_result> {\n       try {\n           std::rethrow_exception(std::move(ep));\n       } catch (...) {\n           std::throw_with_nested(std::runtime_error(sstring(\"Could not read \") + what + \" \" + name));\n       }\n    });\n}\n\n// Note: we are not using gnutls++ interfaces, mainly because we\n// want to keep _our_ interface reasonably non-gnutls (well...)\n// and once we get to this level, their abstractions don't help\n// that much anyway. And they are sooo c++98...\nclass gnutls_error_category : public std::error_category {\npublic:\n    constexpr gnutls_error_category() noexcept : std::error_category{} {}\n    const char * name() const noexcept override {\n        return \"GnuTLS\";\n    }\n    std::string message(int error) const override {\n        return gnutls_strerror(error);\n    }\n};\n\nconst std::error_category& tls::error_category() {\n    static const gnutls_error_category ec;\n    return ec;\n}\n\n// Checks a gnutls return value.\n// < 0 -> error.\nstatic void gtls_chk(int res) {\n    if (res < 0) {\n        throw std::system_error(res, tls::error_category());\n    }\n}\n\nnamespace {\n\n// helper for gnutls-functions for receiving a string\n// arguments\n//  func - the gnutls function that is returning a string (e.g. gnutls_x509_crt_get_issuer_dn)\n//  args - the arguments to func that come before the char array's ptr and size args\n// returns\n//  pair<int, string> - [gnutls error code, extracted string],\n//                      in case of no errors, the error code is zero\nstatic auto get_gtls_string = [](auto func, auto... args) noexcept {\n    size_t size = 0;\n    int ret = func(args..., nullptr, &size);\n\n    // by construction, we expect the SHORT_MEMORY_BUFFER error code here\n    if (ret != GNUTLS_E_SHORT_MEMORY_BUFFER) {\n        return std::make_pair(ret, sstring{});\n    }\n    assert(size != 0);\n    sstring res(sstring::initialized_later{}, size - 1);\n    ret = func(args..., res.data(), &size);\n    return std::make_pair(ret, res);\n};\n\n}\n\nclass tls::dh_params::impl : gnutlsobj {\n    static gnutls_sec_param_t to_gnutls_level(level l) {\n        switch (l) {\n            case level::LEGACY: return GNUTLS_SEC_PARAM_LEGACY;\n#if GNUTLS_VERSION_NUMBER >= 0x030300\n            case level::MEDIUM: return GNUTLS_SEC_PARAM_MEDIUM;\n#else\n            case level::MEDIUM: return GNUTLS_SEC_PARAM_NORMAL;\n#endif\n            case level::HIGH: return GNUTLS_SEC_PARAM_HIGH;\n            case level::ULTRA: return GNUTLS_SEC_PARAM_ULTRA;\n            default:\n                throw std::runtime_error(format(\"Unknown value of dh_params::level: {:d}\", static_cast<std::underlying_type_t<level>>(l)));\n        }\n    }\n    using dh_ptr = std::unique_ptr<std::remove_pointer_t<gnutls_dh_params_t>, void(*)(gnutls_dh_params_t)>;\n\n    static dh_ptr new_dh_params() {\n        gnutls_dh_params_t params;\n        gtls_chk(gnutls_dh_params_init(&params));\n        return dh_ptr(params, &gnutls_dh_params_deinit);\n    }\npublic:\n    impl(dh_ptr p) \n        : _params(std::move(p)) \n    {}\n    impl(level lvl)\n#if GNUTLS_VERSION_NUMBER >= 0x030506\n        : _params(nullptr, &gnutls_dh_params_deinit)\n        , _sec_param(to_gnutls_level(lvl))\n#else        \n        : impl([&] {\n            auto bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH, to_gnutls_level(lvl));\n            auto ptr = new_dh_params();\n            gtls_chk(gnutls_dh_params_generate2(ptr.get(), bits));\n            return ptr;\n        }())\n#endif\n    {}\n    impl(const blob& pkcs3, x509_crt_format fmt)\n        : impl([&] {\n            auto ptr = new_dh_params();\n            blob_wrapper w(pkcs3);\n            gtls_chk(gnutls_dh_params_import_pkcs3(ptr.get(), &w, gnutls_x509_crt_fmt_t(fmt)));\n            return ptr;\n        }()) \n    {}\n    impl(const impl& v)\n        : impl([&v] {\n            auto ptr = new_dh_params();\n            gtls_chk(gnutls_dh_params_cpy(ptr.get(), v));\n            return ptr;\n        }()) \n    {}\n    ~impl() = default;\n\n    operator gnutls_dh_params_t() const {\n        return _params.get();\n    }\n#if GNUTLS_VERSION_NUMBER >= 0x030506\n    std::optional<gnutls_sec_param_t> sec_param() const {\n        return _sec_param;\n    }\n#endif\nprivate:\n    dh_ptr _params;\n#if GNUTLS_VERSION_NUMBER >= 0x030506\n    std::optional<gnutls_sec_param_t> _sec_param;\n#endif\n};\n\ntls::dh_params::dh_params(level lvl) : _impl(std::make_unique<impl>(lvl))\n{}\n\ntls::dh_params::dh_params(const blob& b, x509_crt_format fmt)\n        : _impl(std::make_unique<impl>(b, fmt)) {\n}\n\ntls::dh_params::~dh_params() {\n}\n\ntls::dh_params::dh_params(dh_params&&) noexcept = default;\ntls::dh_params& tls::dh_params::operator=(dh_params&&) noexcept = default;\n\nfuture<tls::dh_params> tls::dh_params::from_file(\n        const sstring& filename, x509_crt_format fmt) {\n    return read_fully(filename, \"dh parameters\").then([fmt](temporary_buffer<char> buf) {\n        return make_ready_future<dh_params>(dh_params(blob(buf.get()), fmt));\n    });\n}\n\nclass tls::x509_cert::impl : gnutlsobj {\npublic:\n    impl()\n            : _cert([] {\n                gnutls_x509_crt_t cert;\n                gtls_chk(gnutls_x509_crt_init(&cert));\n                return cert;\n            }()) {\n    }\n    impl(const blob& b, x509_crt_format fmt)\n        : impl()\n    {\n        blob_wrapper w(b);\n        gtls_chk(gnutls_x509_crt_import(*this, &w, gnutls_x509_crt_fmt_t(fmt)));\n    }\n    ~impl() {\n        if (_cert != nullptr) {\n            gnutls_x509_crt_deinit(_cert);\n        }\n    }\n    operator gnutls_x509_crt_t() const {\n        return _cert;\n    }\n\nprivate:\n    gnutls_x509_crt_t _cert;\n};\n\ntls::x509_cert::x509_cert(shared_ptr<impl> impl)\n        : _impl(std::move(impl)) {\n}\n\ntls::x509_cert::x509_cert(const blob& b, x509_crt_format fmt)\n        : x509_cert(::seastar::make_shared<impl>(b, fmt)) {\n}\n\nfuture<tls::x509_cert> tls::x509_cert::from_file(\n        const sstring& filename, x509_crt_format fmt) {\n    return read_fully(filename, \"x509 certificate\").then([fmt](temporary_buffer<char> buf) {\n        return make_ready_future<x509_cert>(x509_cert(blob(buf.get()), fmt));\n    });\n}\n\nclass tls::certificate_credentials::impl: public gnutlsobj {\npublic:\n    impl()\n            : _creds([] {\n                gnutls_certificate_credentials_t xcred;\n                gnutls_certificate_allocate_credentials(&xcred);\n                if (xcred == nullptr) {\n                    throw std::bad_alloc();\n                }\n                return xcred;\n            }()), _priority(nullptr, &gnutls_priority_deinit)\n    {}\n    ~impl() {\n        if (_creds != nullptr) {\n            gnutls_certificate_free_credentials (_creds);\n        }\n    }\n\n    operator gnutls_certificate_credentials_t() const {\n        return _creds;\n    }\n\n    void set_x509_trust(const blob& b, x509_crt_format fmt) {\n        blob_wrapper w(b);\n        gtls_chk(\n                gnutls_certificate_set_x509_trust_mem(_creds, &w,\n                        gnutls_x509_crt_fmt_t(fmt)));\n    }\n    void set_x509_crl(const blob& b, x509_crt_format fmt) {\n        blob_wrapper w(b);\n        gtls_chk(\n                gnutls_certificate_set_x509_crl_mem(_creds, &w,\n                        gnutls_x509_crt_fmt_t(fmt)));\n    }\n    void set_x509_key(const blob& cert, const blob& key, x509_crt_format fmt) {\n        blob_wrapper w1(cert);\n        blob_wrapper w2(key);\n        gtls_chk(\n                gnutls_certificate_set_x509_key_mem(_creds, &w1, &w2,\n                        gnutls_x509_crt_fmt_t(fmt)));\n    }\n    void set_simple_pkcs12(const blob& b, x509_crt_format fmt,\n            const sstring& password) {\n        blob_wrapper w(b);\n        gtls_chk(\n                gnutls_certificate_set_x509_simple_pkcs12_mem(_creds, &w,\n                        gnutls_x509_crt_fmt_t(fmt), password.c_str()));\n    }\n    void dh_params(const tls::dh_params& dh) {\n#if GNUTLS_VERSION_NUMBER >= 0x030506\n        auto sec_param = dh._impl->sec_param();\n        if (sec_param) {\n            gnutls_certificate_set_known_dh_params(*this, *sec_param);\n            return;\n        }\n#endif\n...\n", "needle_token_start": 10413, "needle_token_end": 10513, "code_context_ntokens": 16118, "output": ["```cpp\nvoid arp_learn(ethernet_address l2, ipv4_address l3)\n{\n    // Run arp_learn on all shard in the background\n    (void)smp::invoke_on_all([l2, l3] {\n        auto & ns = static_cast<native_network_stack&>(engine().net());\n        ns.arp_learn(l2, l3);\n    });\n}\n```"]}
{"repo": "scylladb/seastar", "name": "RTE_LCORE_FOREACH_WORKER", "language": "cpp", "path": "src/core/reactor.cc", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function iterates over each worker logical core (excluding the main core) available in a multi-core system, typically used in environments managed by the Data Plane Development Kit (DPDK).\n2. **Input**: There are no direct inputs as it functions as a macro iterating over available worker cores.\n3. **Output**: There is no direct output; however, it controls the flow to execute specific operations on each worker core.\n4. **Procedure**: This macro retrieves the list of all worker cores, excluding the main core, and iterates over them. During each iteration, it allows the execution of specified operations on each worker core, such as launching threads or processes tailored to run on these specific cores.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    _metrics.add_group(\"smp\", {\n            // queue_length     value:GAUGE:0:U\n            // Absolute value of num packets in last tx batch.\n            sm::make_queue_length(\"send_batch_queue_length\", _last_snt_batch, sm::description(\"Current send batch queue length\"), {sm::shard_label(instance)})(sm::metric_disabled),\n            sm::make_queue_length(\"receive_batch_queue_length\", _last_rcv_batch, sm::description(\"Current receive batch queue length\"), {sm::shard_label(instance)})(sm::metric_disabled),\n            sm::make_queue_length(\"complete_batch_queue_length\", _last_cmpl_batch, sm::description(\"Current complete batch queue length\"), {sm::shard_label(instance)})(sm::metric_disabled),\n            sm::make_queue_length(\"send_queue_length\", _current_queue_length, sm::description(\"Current send queue length\"), {sm::shard_label(instance)})(sm::metric_disabled),\n            // total_operations value:DERIVE:0:U\n            sm::make_counter(\"total_received_messages\", _received, sm::description(\"Total number of received messages\"), {sm::shard_label(instance)})(sm::metric_disabled),\n            // total_operations value:DERIVE:0:U\n            sm::make_counter(\"total_sent_messages\", _sent, sm::description(\"Total number of sent messages\"), {sm::shard_label(instance)})(sm::metric_disabled),\n            // total_operations value:DERIVE:0:U\n            sm::make_counter(\"total_completed_messages\", _compl, sm::description(\"Total number of messages completed\"), {sm::shard_label(instance)})(sm::metric_disabled)\n    });\n}\n\nreadable_eventfd writeable_eventfd::read_side() {\n    return readable_eventfd(_fd.dup());\n}\n\nfile_desc writeable_eventfd::try_create_eventfd(size_t initial) {\n    assert(size_t(int(initial)) == initial);\n    return file_desc::eventfd(initial, EFD_CLOEXEC);\n}\n\nvoid writeable_eventfd::signal(size_t count) {\n    uint64_t c = count;\n    auto r = _fd.write(&c, sizeof(c));\n    assert(r == sizeof(c));\n}\n\nwriteable_eventfd readable_eventfd::write_side() {\n    return writeable_eventfd(_fd.get_file_desc().dup());\n}\n\nfile_desc readable_eventfd::try_create_eventfd(size_t initial) {\n    assert(size_t(int(initial)) == initial);\n    return file_desc::eventfd(initial, EFD_CLOEXEC | EFD_NONBLOCK);\n}\n\nfuture<size_t> readable_eventfd::wait() {\n    return engine().readable(*_fd._s).then([this] {\n        uint64_t count;\n        int r = ::read(_fd.get_fd(), &count, sizeof(count));\n        assert(r == sizeof(count));\n        return make_ready_future<size_t>(count);\n    });\n}\n\nvoid schedule(task* t) noexcept {\n    engine().add_task(t);\n}\n\nvoid schedule_checked(task* t) noexcept {\n    if (t->group().is_at_exit()) {\n        // trying to schedule a task in at_destroy. Not allowed\n        on_internal_error(seastar_logger, \"Cannot schedule tasks in at_destroy queue. Use reactor::at_destroy.\");\n    }\n    engine().add_task(t);\n}\n\nvoid schedule_urgent(task* t) noexcept {\n    engine().add_urgent_task(t);\n}\n\n}\n\nbool operator==(const ::sockaddr_in a, const ::sockaddr_in b) {\n    return (a.sin_addr.s_addr == b.sin_addr.s_addr) && (a.sin_port == b.sin_port);\n}\n\nnamespace seastar {\n\nstatic bool kernel_supports_aio_fsync() {\n    return internal::kernel_uname().whitelisted({\"4.18\"});\n}\n\nstatic program_options::selection_value<network_stack_factory> create_network_stacks_option(reactor_options& zis) {\n    using value_type = program_options::selection_value<network_stack_factory>;\n    value_type::candidates candidates;\n    std::vector<std::string> net_stack_names;\n\n    auto deleter = [] (network_stack_factory* p) { delete p; };\n\n    std::string default_stack;\n    for (auto reg_func : {register_native_stack, register_posix_stack}) {\n        auto s = reg_func();\n        if (s.is_default) {\n            default_stack = s.name;\n        }\n        candidates.push_back({s.name, {new network_stack_factory(std::move(s.factory)), deleter}, std::move(s.opts)});\n        net_stack_names.emplace_back(s.name);\n    }\n\n    return program_options::selection_value<network_stack_factory>(zis, \"network-stack\", std::move(candidates), default_stack,\n            fmt::format(\"select network stack (valid values: {})\", fmt::join(net_stack_names, \", \")));\n}\n\nstatic program_options::selection_value<reactor_backend_selector>::candidates backend_selector_candidates() {\n    using value_type = program_options::selection_value<reactor_backend_selector>;\n    value_type::candidates candidates;\n\n    auto deleter = [] (reactor_backend_selector* p) { delete p; };\n\n    for (auto&& be : reactor_backend_selector::available()) {\n        auto name = be.name();\n        candidates.push_back({std::move(name), {new reactor_backend_selector(std::move(be)), deleter}, {}});\n    }\n    return candidates;\n}\n\nreactor_options::reactor_options(program_options::option_group* parent_group)\n    : program_options::option_group(parent_group, \"Core options\")\n    , network_stack(create_network_stacks_option(*this))\n    , poll_mode(*this, \"poll-mode\", \"poll continuously (100% cpu use)\")\n    , idle_poll_time_us(*this, \"idle-poll-time-us\", reactor::calculate_poll_time() / 1us,\n                \"idle polling time in microseconds (reduce for overprovisioned environments or laptops)\")\n    , poll_aio(*this, \"poll-aio\", true,\n                \"busy-poll for disk I/O (reduces latency and increases throughput)\")\n    , task_quota_ms(*this, \"task-quota-ms\", 0.5, \"Max time (ms) between polls\")\n    , io_latency_goal_ms(*this, \"io-latency-goal-ms\", {}, \"Max time (ms) io operations must take (1.5 * task-quota-ms if not set)\")\n    , io_flow_ratio_threshold(*this, \"io-flow-rate-threshold\", 1.1, \"Dispatch rate to completion rate threshold\")\n    , max_task_backlog(*this, \"max-task-backlog\", 1000, \"Maximum number of task backlog to allow; above this we ignore I/O\")\n    , blocked_reactor_notify_ms(*this, \"blocked-reactor-notify-ms\", 25, \"threshold in miliseconds over which the reactor is considered blocked if no progress is made\")\n    , blocked_reactor_reports_per_minute(*this, \"blocked-reactor-reports-per-minute\", 5, \"Maximum number of backtraces reported by stall detector per minute\")\n    , blocked_reactor_report_format_oneline(*this, \"blocked-reactor-report-format-oneline\", true, \"Print a simplified backtrace on a single line\")\n    , relaxed_dma(*this, \"relaxed-dma\", \"allow using buffered I/O if DMA is not available (reduces performance)\")\n    , linux_aio_nowait(*this, \"linux-aio-nowait\", aio_nowait_supported,\n                \"use the Linux NOWAIT AIO feature, which reduces reactor stalls due to aio (autodetected)\")\n    , unsafe_bypass_fsync(*this, \"unsafe-bypass-fsync\", false, \"Bypass fsync(), may result in data loss. Use for testing on consumer drives\")\n    , kernel_page_cache(*this, \"kernel-page-cache\", false,\n                \"Use the kernel page cache. This disables DMA (O_DIRECT).\"\n                \" Useful for short-lived functional tests with a small data set.\")\n    , overprovisioned(*this, \"overprovisioned\", \"run in an overprovisioned environment (such as docker or a laptop); equivalent to --idle-poll-time-us 0 --thread-affinity 0 --poll-aio 0\")\n    , abort_on_seastar_bad_alloc(*this, \"abort-on-seastar-bad-alloc\", \"abort when seastar allocator cannot allocate memory\")\n    , force_aio_syscalls(*this, \"force-aio-syscalls\", false,\n                \"Force io_getevents(2) to issue a system call, instead of bypassing the kernel when possible.\"\n                \" This makes strace output more useful, but slows down the application\")\n    , dump_memory_diagnostics_on_alloc_failure_kind(*this, \"dump-memory-diagnostics-on-alloc-failure-kind\", memory::alloc_failure_kind::critical,\n                \"Dump diagnostics of the seastar allocator state on allocation failure.\"\n                 \" Accepted values: none, critical (default), all. When set to critical, only allocations marked as critical will trigger diagnostics dump.\"\n                 \" The diagnostics will be written to the seastar_memory logger, with error level.\"\n                 \" Note that if the seastar_memory logger is set to debug or trace level, the diagnostics will be logged irrespective of this setting.\")\n    , reactor_backend(*this, \"reactor-backend\", backend_selector_candidates(), reactor_backend_selector::default_backend().name(),\n                fmt::format(\"Internal reactor implementation ({})\", reactor_backend_selector::available()))\n    , aio_fsync(*this, \"aio-fsync\", kernel_supports_aio_fsync(),\n                \"Use Linux aio for fsync() calls. This reduces latency; requires Linux 4.18 or later.\")\n    , max_networking_io_control_blocks(*this, \"max-networking-io-control-blocks\", 10000,\n                \"Maximum number of I/O control blocks (IOCBs) to allocate per shard. This translates to the number of sockets supported per shard.\"\n                \" Requires tuning /proc/sys/fs/aio-max-nr. Only valid for the linux-aio reactor backend (see --reactor-backend).\")\n#ifdef SEASTAR_HEAPPROF\n    , heapprof(*this, \"heapprof\", 0, \"Enable seastar heap profiling. Sample every ARG bytes. 0 means off\")\n#else\n    , heapprof(*this, \"heapprof\", program_options::unused{})\n#endif\n    , no_handle_interrupt(*this, \"no-handle-interrupt\", \"ignore SIGINT (for gdb)\")\n{\n}\n\nsmp_options::smp_options(program_options::option_group* parent_group)\n    : program_options::option_group(parent_group, \"SMP options\")\n    , smp(*this, \"smp\", {}, \"number of threads (default: one per CPU)\")\n    , cpuset(*this, \"cpuset\", {}, \"CPUs to use (in cpuset(7) list format (ex: 0,1-3,7); default: all))\")\n    , memory(*this, \"memory\", std::nullopt, \"memory to use, in bytes (ex: 4G) (default: all)\")\n    , reserve_memory(*this, \"reserve-memory\", {}, \"memory reserved to OS (if --memory not specified)\")\n    , hugepages(*this, \"hugepages\", {}, \"path to accessible hugetlbfs mount (typically /dev/hugepages/something)\")\n    , lock_memory(*this, \"lock-memory\", {}, \"lock all memory (prevents swapping)\")\n    , thread_affinity(*this, \"thread-affinity\", true, \"pin threads to their cpus (disable for overprovisioning)\")\n#ifdef SEASTAR_HAVE_HWLOC\n    , num_io_groups(*this, \"num-io-groups\", {}, \"Number of IO groups. Each IO group will be responsible for a fraction of the IO requests. Defaults to the number of NUMA nodes\")\n#else\n    , num_io_groups(*this, \"num-io-groups\", program_options::unused{})\n#endif\n    , io_properties_file(*this, \"io-properties-file\", {}, \"path to a YAML file describing the characteristics of the I/O Subsystem\")\n    , io_properties(*this, \"io-properties\", {}, \"a YAML string describing the characteristics of the I/O Subsystem\")\n    , mbind(*this, \"mbind\", true, \"enable mbind\")\n#ifndef SEASTAR_NO_EXCEPTION_HACK\n    , enable_glibc_exception_scaling_workaround(*this, \"enable-glibc-exception-scaling-workaround\", true, \"enable workaround for glibc/gcc c++ exception scalablity problem\")\n#else\n    , enable_glibc_exception_scaling_workaround(*this, \"enable-glibc-exception-scaling-workaround\", program_options::unused{})\n#endif\n#ifdef SEASTAR_HAVE_HWLOC\n    , allow_cpus_in_remote_numa_nodes(*this, \"allow-cpus-in-remote-numa-nodes\", true, \"if some CPUs are found not to have any local NUMA nodes, allow assigning them to remote ones\")\n#else\n    , allow_cpus_in_remote_numa_nodes(*this, \"allow-cpus-in-remote-numa-nodes\", program_options::unused{})\n#endif\n{\n}\n\nstruct reactor_deleter {\n    void operator()(reactor* p) {\n        p->~reactor();\n        free(p);\n    }\n};\n\nthread_local std::unique_ptr<reactor, reactor_deleter> reactor_holder;\n\nthread_local smp_message_queue** smp::_qs;\nthread_local std::thread::id smp::_tmain;\nunsigned smp::count = 0;\n\nvoid smp::start_all_queues()\n{\n    for (unsigned c = 0; c < count; c++) {\n        if (c != this_shard_id()) {\n            _qs[c][this_shard_id()].start(c);\n        }\n    }\n    _alien._qs[this_shard_id()].start();\n}\n\n#ifdef SEASTAR_HAVE_DPDK\n\nint dpdk_thread_adaptor(void* f)\n{\n    (*static_cast<std::function<void ()>*>(f))();\n    return 0;\n}\n\n#endif\n\nvoid smp::join_all()\n{\n#ifdef SEASTAR_HAVE_DPDK\n    if (_using_dpdk) {\n        rte_eal_mp_wait_lcore();\n        return;\n    }\n#endif\n    for (auto&& t: smp::_threads) {\n        t.join();\n    }\n}\n\nvoid smp::pin(unsigned cpu_id) {\n    if (_using_dpdk) {\n        // dpdk does its own pinning\n        return;\n    }\n    pin_this_thread(cpu_id);\n}\n\nvoid smp::arrive_at_event_loop_end() {\n    if (_all_event_loops_done) {\n        _all_event_loops_done->wait();\n    }\n}\n\nvoid smp::allocate_reactor(unsigned id, reactor_backend_selector rbs, reactor_config cfg) {\n    assert(!reactor_holder);\n\n    // we cannot just write \"local_engin = new reactor\" since reactor's constructor\n    // uses local_engine\n    void *buf;\n    int r = posix_memalign(&buf, cache_line_size, sizeof(reactor));\n    assert(r == 0);\n    *internal::this_shard_id_ptr() = id;\n    local_engine = new (buf) reactor(this->shared_from_this(), _alien, id, std::move(rbs), cfg);\n    reactor_holder.reset(local_engine);\n}\n\nvoid smp::cleanup() noexcept {\n    smp::_threads = std::vector<posix_thread>();\n    _thread_loops.clear();\n    reactor_holder.reset();\n    local_engine = nullptr;\n}\n\nvoid smp::cleanup_cpu() {\n    size_t cpuid = this_shard_id();\n\n    if (_qs) {\n        for(unsigned i = 0; i < smp::count; i++) {\n            _qs[i][cpuid].stop();\n        }\n    }\n    if (_alien._qs) {\n        _alien._qs[cpuid].stop();\n    }\n}\n\nvoid smp::create_thread(std::function<void ()> thread_loop) {\n    if (_using_dpdk) {\n        _thread_loops.push_back(std::move(thread_loop));\n    } else {\n        _threads.emplace_back(std::move(thread_loop));\n    }\n}\n\n// Installs handler for Signal which ensures that Func is invoked only once\n// in the whole program and that after it is invoked the default handler is restored.\ntemplate<int Signal, void(*Func)()>\nvoid install_oneshot_signal_handler() {\n    static bool handled = false;\n    static util::spinlock lock;\n\n    struct sigaction sa;\n    sa.sa_sigaction = [](int sig, siginfo_t *info, void *p) {\n        std::lock_guard<util::spinlock> g(lock);\n        if (!handled) {\n            handled = true;\n            Func();\n            signal(sig, SIG_DFL);\n        }\n    };\n    sigfillset(&sa.sa_mask);\n    sa.sa_flags = SA_SIGINFO | SA_RESTART;\n    if (Signal == SIGSEGV) {\n        sa.sa_flags |= SA_ONSTACK;\n    }\n    auto r = ::sigaction(Signal, &sa, nullptr);\n    throw_system_error_on(r == -1);\n}\n\nstatic void reraise_signal(int signo) {\n    signal(signo, SIG_DFL);\n    pthread_kill(pthread_self(), signo);\n}\n\nstatic void sigsegv_action() noexcept {\n    print_with_backtrace(\"Segmentation fault\");\n    reraise_signal(SIGSEGV);\n}\n\nstatic void sigabrt_action() noexcept {\n    print_with_backtrace(\"Aborting\");\n    reraise_signal(SIGABRT);\n}\n\n// We don't need to handle SIGSEGV when asan is enabled.\n#ifdef SEASTAR_ASAN_ENABLED\ntemplate<>\nvoid install_oneshot_signal_handler<SIGSEGV, sigsegv_action>() {\n    (void)sigsegv_action;\n}\n#endif\n\nvoid smp::qs_deleter::operator()(smp_message_queue** qs) const {\n    for (unsigned i = 0; i < smp::count; i++) {\n        for (unsigned j = 0; j < smp::count; j++) {\n            qs[i][j].~smp_message_queue();\n        }\n        ::operator delete[](qs[i], std::align_val_t(alignof(smp_message_queue))\n        );\n    }\n    delete[](qs);\n}\n\nclass disk_config_params {\nprivate:\n    const unsigned _max_queues;\n    unsigned _num_io_groups = 0;\n    std::unordered_map<dev_t, mountpoint_params> _mountpoints;\n    std::chrono::duration<double> _latency_goal;\n    double _flow_ratio_backpressure_threshold;\n\npublic:\n    explicit disk_config_params(unsigned max_queues) noexcept\n            : _max_queues(max_queues)\n    {}\n\n    uint64_t per_io_group(uint64_t qty, unsigned nr_groups) const noexcept {\n        return std::max(qty / nr_groups, 1ul);\n    }\n\n    unsigned num_io_groups() const noexcept { return _num_io_groups; }\n\n    std::chrono::duration<double> latency_goal() const {\n        return _latency_goal;\n    }\n\n    double latency_goal_opt(const reactor_options& opts) const {\n        return opts.io_latency_goal_ms ?\n                opts.io_latency_goal_ms.get_value() :\n                opts.task_quota_ms.get_value() * 1.5;\n    }\n\n    void parse_config(const smp_options& smp_opts, const reactor_options& reactor_opts) {\n        seastar_logger.debug(\"smp::count: {}\", smp::count);\n        _latency_goal = std::chrono::duration_cast<std::chrono::duration<double>>(latency_goal_opt(reactor_opts) * 1ms);\n        seastar_logger.debug(\"latency_goal: {}\", latency_goal().count());\n        _flow_ratio_backpressure_threshold = reactor_opts.io_flow_ratio_threshold.get_value();\n        seastar_logger.debug(\"flow-ratio threshold: {}\", _flow_ratio_backpressure_threshold);\n\n        if (smp_opts.num_io_groups) {\n            _num_io_groups = smp_opts.num_io_groups.get_value();\n            if (!_num_io_groups) {\n                throw std::runtime_error(\"num-io-groups must be greater than zero\");\n            }\n        }\n        if (smp_opts.io_properties_file && smp_opts.io_properties) {\n            throw std::runtime_error(\"Both io-properties and io-properties-file specified. Don't know which to trust!\");\n        }\n\n        std::optional<YAML::Node> doc;\n        if (smp_opts.io_properties_file) {\n            doc = YAML::LoadFile(smp_opts.io_properties_file.get_value());\n        } else if (smp_opts.io_properties) {\n            doc = YAML::Load(smp_opts.io_properties.get_value());\n        }\n\n        if (doc) {\n            if (!doc->IsMap()) {\n                throw std::runtime_error(\"Bogus io-properties (did you mix up --io-properties and --io-properties-file?)\");\n            }\n            for (auto&& section : *doc) {\n                auto sec_name = section.first.as<std::string>();\n                if (sec_name != \"disks\") {\n                    throw std::runtime_error(fmt::format(\"While parsing I/O options: section {} currently unsupported.\", sec_name));\n                }\n                auto disks = section.second.as<std::vector<mountpoint_params>>();\n                for (auto& d : disks) {\n                    struct ::stat buf;\n                    auto ret = stat(d.mountpoint.c_str(), &buf);\n                    if (ret < 0) {\n                        throw std::runtime_error(fmt::format(\"Couldn't stat {}\", d.mountpoint));\n                    }\n\n                    auto st_dev = S_ISBLK(buf.st_mode) ? buf.st_rdev : buf.st_dev;\n                    if (_mountpoints.count(st_dev)) {\n                        throw std::runtime_error(fmt::format(\"Mountpoint {} already configured\", d.mountpoint));\n                    }\n                    if (_mountpoints.size() >= _max_queues) {\n                        throw std::runtime_error(fmt::format(\"Configured number of queues {} is larger than the maximum {}\",\n                                                 _mountpoints.size(), _max_queues));\n                    }\n\n                    d.read_bytes_rate *= d.rate_factor;\n                    d.write_bytes_rate *= d.rate_factor;\n                    d.read_req_rate *= d.rate_factor;\n                    d.write_req_rate *= d.rate_factor;\n\n                    if (d.read_bytes_rate == 0 || d.write_bytes_rate == 0 ||\n                            d.read_req_rate == 0 || d.write_req_rate == 0) {\n                        throw std::runtime_error(fmt::format(\"R/W bytes and req rates must not be zero\"));\n                    }\n\n                    seastar_logger.debug(\"dev_id: {} mountpoint: {}\", st_dev, d.mountpoint);\n                    _mountpoints.emplace(st_dev, d);\n                }\n            }\n        }\n\n        // Placeholder for unconfigured disks.\n        mountpoint_params d = {};\n        _mountpoints.emplace(0, d);\n    }\n\n    struct io_queue::config generate_config(dev_t devid, unsigned nr_groups) const {\n        seastar_logger.debug(\"generate_config dev_id: {}\", devid);\n        const mountpoint_params& p = _mountpoints.at(devid);\n        struct io_queue::config cfg;\n\n        cfg.devid = devid;\n\n        if (p.read_bytes_rate != std::numeric_limits<uint64_t>::max()) {\n            cfg.blocks_count_rate = (io_queue::read_request_base_count * (unsigned long)per_io_group(p.read_bytes_rate, nr_groups)) >> io_queue::block_size_shift;\n            cfg.disk_blocks_write_to_read_multiplier = (io_queue::read_request_base_count * p.read_bytes_rate) / p.write_bytes_rate;\n        }\n        if (p.read_req_rate != std::numeric_limits<uint64_t>::max()) {\n            cfg.req_count_rate = io_queue::read_request_base_count * (unsigned long)per_io_group(p.read_req_rate, nr_groups);\n            cfg.disk_req_write_to_read_multiplier = (io_queue::read_request_base_count * p.read_req_rate) / p.write_req_rate;\n        }\n        if (p.read_saturation_length != std::numeric_limits<uint64_t>::max()) {\n            cfg.disk_read_saturation_length = p.read_saturation_length;\n        }\n        if (p.write_saturation_length != std::numeric_limits<uint64_t>::max()) {\n            cfg.disk_write_saturation_length = p.write_saturation_length;\n        }\n        cfg.mountpoint = p.mountpoint;\n        cfg.duplex = p.duplex;\n        cfg.rate_limit_duration = latency_goal();\n        cfg.flow_ratio_backpressure_threshold = _flow_ratio_backpressure_threshold;\n        // Block count limit should not be less than the minimal IO size on the device\n        // On the other hand, even this is not good enough -- in the worst case the\n        // scheduler will self-tune to allow for the single 64k request, while it would\n        // be better to sacrifice some IO latency, but allow for larger concurrency\n        cfg.block_count_limit_min = (64 << 10) >> io_queue::block_size_shift;\n\n        return cfg;\n    }\n\n    auto device_ids() {\n        return boost::adaptors::keys(_mountpoints);\n    }\n};\n\nunsigned smp::adjust_max_networking_aio_io_control_blocks(unsigned network_iocbs)\n{\n    static unsigned constexpr storage_iocbs = reactor::max_aio;\n    static unsigned constexpr preempt_iocbs = 2;\n\n    auto aio_max_nr = read_first_line_as<unsigned>(\"/proc/sys/fs/aio-max-nr\");\n    auto aio_nr = read_first_line_as<unsigned>(\"/proc/sys/fs/aio-nr\");\n    auto available_aio = aio_max_nr - aio_nr;\n    auto requested_aio_network = network_iocbs * smp::count;\n    auto requested_aio_other = (storage_iocbs + preempt_iocbs) * smp::count;\n    auto requested_aio = requested_aio_network + requested_aio_other;\n    auto network_iocbs_old = network_iocbs;\n\n    if (available_aio < requested_aio) {\n        seastar_logger.warn(\"Requested AIO slots too large, please increase request capacity in /proc/sys/fs/aio-max-nr. configured:{} available:{} requested:{}\", aio_max_nr, available_aio, requested_aio);\n        if (available_aio >= requested_aio_other + smp::count) { // at least one queue for each shard\n            network_iocbs = (available_aio - requested_aio_other) / smp::count;\n            seastar_logger.warn(\"max-networking-io-control-blocks adjusted from {} to {}, since AIO slots are unavailable\", network_iocbs_old, network_iocbs);\n        } else {\n            throw std::runtime_error(\"Could not setup Async I/O: Not enough request capacity in /proc/sys/fs/aio-max-nr. Try increasing that number or reducing the amount of logical CPUs available for your application\");\n        }\n    }\n\n    return network_iocbs;\n}\n\nvoid smp::configure(const smp_options& smp_opts, const reactor_options& reactor_opts)\n{\n    bool use_transparent_hugepages = !reactor_opts.overprovisioned;\n\n#ifndef SEASTAR_NO_EXCEPTION_HACK\n    if (smp_opts.enable_glibc_exception_scaling_workaround.get_value()) {\n        init_phdr_cache();\n    }\n#endif\n\n    // Mask most, to prevent threads (esp. dpdk helper threads)\n    // from servicing a signal.  Individual reactors will unmask signals\n    // as they become prepared to handle them.\n    //\n    // We leave some signals unmasked since we don't handle them ourself.\n    sigset_t sigs;\n    sigfillset(&sigs);\n    for (auto sig : {SIGHUP, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV,\n            SIGALRM, SIGCONT, SIGSTOP, SIGTSTP, SIGTTIN, SIGTTOU}) {\n        sigdelset(&sigs, sig);\n    }\n    if (!reactor_opts._auto_handle_sigint_sigterm) {\n        sigdelset(&sigs, SIGINT);\n        sigdelset(&sigs, SIGTERM);\n    }\n    pthread_sigmask(SIG_BLOCK, &sigs, nullptr);\n\n    install_oneshot_signal_handler<SIGSEGV, sigsegv_action>();\n    install_oneshot_signal_handler<SIGABRT, sigabrt_action>();\n\n#ifdef SEASTAR_HAVE_DPDK\n    const auto* native_stack = dynamic_cast<const net::native_stack_options*>(reactor_opts.network_stack.get_selected_candidate_opts());\n    _using_dpdk = native_stack && native_stack->dpdk_pmd;\n#endif\n    auto thread_affinity = smp_opts.thread_affinity.get_value();\n    if (reactor_opts.overprovisioned\n           && smp_opts.thread_affinity.defaulted()) {\n        thread_affinity = false;\n    }\n    if (!thread_affinity && _using_dpdk) {\n        fmt::print(\"warning: --thread-affinity 0 ignored in dpdk mode\\n\");\n    }\n    auto mbind = smp_opts.mbind.get_value();\n    if (!thread_affinity) {\n        mbind = false;\n    }\n\n    resource::configuration rc;\n\n    smp::_tmain = std::this_thread::get_id();\n    resource::cpuset cpu_set = get_current_cpuset();\n\n    if (smp_opts.cpuset) {\n        auto opts_cpuset = smp_opts.cpuset.get_value();\n        // CPUs that are not available are those pinned by\n        // --cpuset but not present in current task set\n        std::set<unsigned int> not_available_cpus;\n        std::set_difference(opts_cpuset.begin(), opts_cpuset.end(),\n                            cpu_set.begin(), cpu_set.end(),\n                            std::inserter(not_available_cpus, not_available_cpus.end()));\n\n        if (!not_available_cpus.empty()) {\n            std::ostringstream not_available_cpus_list;\n            for (auto cpu_id : not_available_cpus) {\n                not_available_cpus_list << \" \" << cpu_id;\n            }\n            seastar_logger.error(\"Bad value for --cpuset:{} not allowed. Shutting down.\", not_available_cpus_list.str());\n            exit(1);\n        }\n        cpu_set = opts_cpuset;\n    }\n\n    if (smp_opts.smp) {\n        smp::count = smp_opts.smp.get_value();\n    } else {\n        smp::count = cpu_set.size();\n    }\n    std::vector<reactor*> reactors(smp::count);\n    if (smp_opts.memory) {\n#ifdef SEASTAR_DEFAULT_ALLOCATOR\n        seastar_logger.warn(\"Seastar compiled with default allocator, --memory option won't take effect\");\n#endif\n        rc.total_memory = parse_memory_size(smp_opts.memory.get_value());\n#ifdef SEASTAR_HAVE_DPDK\n        if (smp_opts.hugepages &&\n            !reactor_opts.network_stack.get_selected_candidate_name().compare(\"native\") &&\n            _using_dpdk) {\n            size_t dpdk_memory = dpdk::eal::mem_size(smp::count);\n\n            if (dpdk_memory >= rc.total_memory) {\n                seastar_logger.error(\"Can't run with the given amount of memory: {}. \"\n                                     \"Consider giving more.\",\n                                     smp_opts.memory.get_value());\n                exit(1);\n            }\n\n            //\n            // Subtract the memory we are about to give to DPDK from the total\n            // amount of memory we are allowed to use.\n            //\n            rc.total_memory.value() -= dpdk_memory;\n        }\n#endif\n    }\n    if (smp_opts.reserve_memory) {\n        rc.reserve_memory = parse_memory_size(smp_opts.reserve_memory.get_value());\n    }\n    rc.reserve_additional_memory_per_shard = smp_opts.reserve_additional_memory_per_shard;\n    std::optional<std::string> hugepages_path;\n    if (smp_opts.hugepages) {\n        hugepages_path = smp_opts.hugepages.get_value();\n    }\n    auto mlock = false;\n    if (smp_opts.lock_memory) {\n        mlock = smp_opts.lock_memory.get_value();\n    }\n    if (mlock) {\n        auto extra_flags = 0;\n#ifdef MCL_ONFAULT\n        // Linux will serialize faulting in anonymous memory, and also\n        // serialize marking them as locked. This can take many minutes on\n        // terabyte class machines, so fault them in the future to spread\n        // out the cost. This isn't good since we'll see contention if\n        // multiple shards fault in memory at once, but if that work can be\n        // in parallel to regular reactor work on other shards.\n        extra_flags |= MCL_ONFAULT; // Linux 4.4+\n#endif\n        auto r = mlockall(MCL_CURRENT | MCL_FUTURE | extra_flags);\n        if (r) {\n            // Don't hard fail for now, it's hard to get the configuration right\n            fmt::print(\"warning: failed to mlockall: {}\\n\", strerror(errno));\n        }\n    }\n\n    rc.cpus = smp::count;\n    rc.cpu_set = std::move(cpu_set);\n\n    disk_config_params disk_config(reactor::max_queues);\n    disk_config.parse_config(smp_opts, reactor_opts);\n    for (auto& id : disk_config.device_ids()) {\n        rc.devices.push_back(id);\n    }\n    rc.num_io_groups = disk_config.num_io_groups();\n\n#ifdef SEASTAR_HAVE_HWLOC\n    if (smp_opts.allow_cpus_in_remote_numa_nodes.get_value()) {\n        rc.assign_orphan_cpus = true;\n    }\n#endif\n\n    auto resources = resource::allocate(rc);\n    logger::set_shard_field_width(std::ceil(std::log10(smp::count)));\n    std::vector<resource::cpu> allocations = std::move(resources.cpus);\n    if (thread_affinity) {\n        smp::pin(allocations[0].cpu_id);\n    }\n    std::optional<memory::internal::numa_layout> layout;\n    if (smp_opts.memory_allocator == memory_allocator::seastar) {\n        layout = memory::configure(allocations[0].mem, mbind, use_transparent_hugepages, hugepages_path);\n    } else {\n        // #2148 - if running seastar allocator but options that contradict this, we still need to\n        // init memory at least minimally, otherwise a bunch of stuff breaks.\n        // Previously, we got away wth this by accident due to #2137.\n        memory::configure_minimal();\n    }\n\n    if (reactor_opts.abort_on_seastar_bad_alloc) {\n        memory::set_abort_on_allocation_failure(true);\n    }\n\n    if (reactor_opts.dump_memory_diagnostics_on_alloc_failure_kind) {\n        memory::set_dump_memory_diagnostics_on_alloc_failure_kind(reactor_opts.dump_memory_diagnostics_on_alloc_failure_kind.get_value());\n    }\n\n    reactor_config reactor_cfg;\n    reactor_cfg.auto_handle_sigint_sigterm = reactor_opts._auto_handle_sigint_sigterm;\n    reactor_cfg.max_networking_aio_io_control_blocks = adjust_max_networking_aio_io_control_blocks(reactor_opts.max_networking_io_control_blocks.get_value());\n\n    std::mutex mtx;\n\n#ifdef SEASTAR_HEAPPROF\n    size_t heapprof_sampling_rate = reactor_opts.heapprof.get_value();\n    if (heapprof_sampling_rate) {\n        memory::set_heap_profiling_sampling_rate(heapprof_sampling_rate);\n    }\n#else\n    size_t heapprof_sampling_rate = 0;\n#endif\n\n#ifdef SEASTAR_HAVE_DPDK\n    if (_using_dpdk) {\n        dpdk::eal::cpuset cpus;\n        for (auto&& a : allocations) {\n            cpus[a.cpu_id] = true;\n        }\n        dpdk::eal::init(cpus, reactor_opts._argv0, hugepages_path, native_stack ? bool(native_stack->dpdk_pmd) : false);\n    }\n#endif\n\n    // Better to put it into the smp class, but at smp construction time\n    // correct smp::count is not known.\n    boost::barrier reactors_registered(smp::count);\n    boost::barrier smp_queues_constructed(smp::count);\n    // We use shared_ptr since this thread can exit while other threads are still unlocking\n    auto inited = std::make_shared<boost::barrier>(smp::count);\n\n    auto ioq_topology = std::move(resources.ioq_topology);\n\n    // ATTN: The ioq_topology value is referenced by below lambdas which are\n    // then copied to other shard's threads, so each shard has a copy of the\n    // ioq_topology on stack, but (!) still references and uses the value\n    // from shard-0. This access is race-free because\n    //  1. The .shard_to_group is not modified\n    //  2. The .queues is pre-resize()-d in advance, so the vector itself\n    //     doesn't change; existing slots are accessed by owning shards only\n    //     without interference\n    //  3. The .groups manipulations are guarded by the .lock lock (but it's\n    //     also pre-resize()-d in advance)\n\n    auto alloc_io_queues = [&ioq_topology, &disk_config] (shard_id shard) {\n        for (auto& [dev, io_info] : ioq_topology) {\n            auto group_idx = io_info.shard_to_group[shard];\n            std::shared_ptr<io_group> group;\n\n            {\n                std::lock_guard _(io_info.lock);\n                auto& iog = io_info.groups[group_idx];\n                if (!iog) {\n                    struct io_queue::config qcfg = disk_config.generate_config(dev, io_info.groups.size());\n                    iog = std::make_shared<io_group>(std::move(qcfg), io_info.shards_in_group[group_idx]);\n                    seastar_logger.debug(\"allocate {} IO group with {} queues, dev {}\", group_idx, io_info.shards_in_group[group_idx], dev);\n                }\n                group = iog;\n            }\n\n            io_info.queues[shard] = std::make_unique<io_queue>(std::move(group), engine()._io_sink);\n            seastar_logger.debug(\"attached {} queue to {} IO group, dev {}\", shard, group_idx, dev);\n        }\n    };\n\n    auto assign_io_queues = [&ioq_topology] (shard_id shard) {\n        for (auto& [dev, io_info] : ioq_topology) {\n            auto queue = std::move(io_info.queues[shard]);\n            assert(queue);\n            engine()._io_queues.emplace(dev, std::move(queue));\n\n            auto num_io_groups = io_info.groups.size();\n            if (engine()._num_io_groups == 0) {\n                engine()._num_io_groups = num_io_groups;\n            } else if (engine()._num_io_groups != num_io_groups) {\n                throw std::logic_error(format(\"Number of IO-groups mismatch, {} != {}\", engine()._num_io_groups, num_io_groups));\n            }\n        }\n    };\n\n    _all_event_loops_done.emplace(smp::count);\n\n    auto backend_selector = reactor_opts.reactor_backend.get_selected_candidate();\n    seastar_logger.info(\"Reactor backend: {}\", backend_selector);\n\n    unsigned i;\n    auto smp_tmain = smp::_tmain;\n    for (i = 1; i < smp::count; i++) {\n        auto allocation = allocations[i];\n        create_thread([this, smp_tmain, inited, &reactors_registered, &smp_queues_constructed, &smp_opts, &reactor_opts, &reactors, hugepages_path, i, allocation, assign_io_queues, alloc_io_queues, thread_affinity, heapprof_sampling_rate, mbind, backend_selector, reactor_cfg, &mtx, &layout, use_transparent_hugepages] {\n          try {\n            // initialize thread_locals that are equal across all reacto threads of this smp instance\n            smp::_tmain = smp_tmain;\n            auto thread_name = fmt::format(\"reactor-{}\", i);\n            pthread_setname_np(pthread_self(), thread_name.c_str());\n            if (thread_affinity) {\n                smp::pin(allocation.cpu_id);\n            }\n            if (smp_opts.memory_allocator == memory_allocator::seastar) {\n                auto another_layout = memory::configure(allocation.mem, mbind, use_transparent_hugepages, hugepages_path);\n                auto guard = std::lock_guard(mtx);\n                *layout = memory::internal::merge(std::move(*layout), std::move(another_layout));\n            } else {\n                // See comment above (shard 0)\n                memory::configure_minimal();\n            }\n            if (heapprof_sampling_rate) {\n                memory::set_heap_profiling_sampling_rate(heapprof_sampling_rate);\n            }\n            sigset_t mask;\n            sigfillset(&mask);\n            for (auto sig : { SIGSEGV }) {\n                sigdelset(&mask, sig);\n            }\n            auto r = ::pthread_sigmask(SIG_BLOCK, &mask, NULL);\n            throw_pthread_error(r);\n            init_default_smp_service_group(i);\n            lowres_clock::update();\n            allocate_reactor(i, backend_selector, reactor_cfg);\n            reactors[i] = &engine();\n            alloc_io_queues(i);\n            reactors_registered.wait();\n            smp_queues_constructed.wait();\n            // _qs_owner is only initialized here\n            _qs = _qs_owner.get();\n            start_all_queues();\n            assign_io_queues(i);\n            inited->wait();\n            engine().configure(reactor_opts);\n            engine().do_run();\n          } catch (const std::exception& e) {\n              seastar_logger.error(\"{}\", e.what());\n              _exit(1);\n          }\n        });\n    }\n\n    init_default_smp_service_group(0);\n    lowres_clock::update();\n    try {\n        allocate_reactor(0, backend_selector, reactor_cfg);\n    } catch (const std::exception& e) {\n        seastar_logger.error(\"{}\", e.what());\n        _exit(1);\n    }\n\n    reactors[0] = &engine();\n    alloc_io_queues(0);\n\n#ifdef SEASTAR_HAVE_DPDK\n    if (_using_dpdk) {\n        auto it = _thread_loops.begin();\n        \nRTE_LCORE_FOREACH_WORKER(i) {\n            rte_eal_remote_launch(dpdk_thread_adaptor, static_cast<void*>(&*(it++)), i);\n        }\n    }\n#endif\n\n    reactors_registered.wait();\n    _qs_owner = decltype(smp::_qs_owner){new smp_message_queue* [smp::count], qs_deleter{}};\n    _qs = _qs_owner.get();\n    for(unsigned i = 0; i < smp::count; i++) {\n        smp::_qs_owner[i] = reinterpret_cast<smp_message_queue*>(operator new[] (sizeof(smp_message_queue) * smp::count\n        // smp_message_queue has members with hefty alignment requirements.\n        // if we are reactor thread, or not running with dpdk, doing this\n        // new default aligned seemingly works, as does reordering\n        // dlinit dependencies (ugh). But we should enforce calling out to\n        // aligned_alloc, instead of pure malloc, if possible.\n            , std::align_val_t(alignof(smp_message_queue))\n        ));\n        for (unsigned j = 0; j < smp::count; ++j) {\n            new (&smp::_qs_owner[i][j]) smp_message_queue(reactors[j], reactors[i]);\n        }\n    }\n    _alien._qs = alien::instance::create_qs(reactors);\n    smp_queues_constructed.wait();\n    start_all_queues();\n    assign_io_queues(0);\n    inited->wait();\n\n    engine().configure(reactor_opts);\n\n    if (smp_opts.lock_memory && smp_opts.lock_memory.get_value() && layout && !layout->ranges.empty()) {\n        smp::setup_prefaulter(resources, std::move(*layout));\n    }\n}\n\nbool smp::poll_queues() {\n    size_t got = 0;\n    for (unsigned i = 0; i < count; i++) {\n        if (this_shard_id() != i) {\n            auto& rxq = _qs[this_shard_id()][i];\n            rxq.flush_response_batch();\n            got += rxq.has_unflushed_responses();\n            got += rxq.process_incoming();\n            auto& txq = _qs[i][this_shard_id()];\n            txq.flush_request_batch();\n            got += txq.process_completions(i);\n        }\n    }\n    return got != 0;\n}\n\nbool smp::pure_poll_queues() {\n    for (unsigned i = 0; i < count; i++) {\n        if (this_shard_id() != i) {\n            auto& rxq = _qs[this_shard_id()][i];\n            rxq.flush_response_batch();\n            auto& txq = _qs[i][this_shard_id()];\n            txq.flush_request_batch();\n            if (rxq.pure_poll_rx() || txq.pure_poll_tx() || rxq.has_unflushed_responses()) {\n                return true;\n            }\n        }\n    }\n    return false;\n}\n\n__thread reactor* local_engine;\n\nvoid report_exception(std::string_view message, std::exception_ptr eptr) noexcept {\n    seastar_logger.error(\"{}: {}\", message, eptr);\n}\n\nfuture<> check_direct_io_support(std::string_view path) noexcept {\n    struct w {\n        sstring path;\n        open_flags flags;\n        std::function<future<>()> cleanup;\n\n        static w parse(sstring path, std::optional<directory_entry_type> type) {\n            if (!type) {\n                throw std::invalid_argument(format(\"Could not open file at {}. Make sure it exists\", path));\n            }\n\n            if (type == directory_entry_type::directory) {\n                auto fpath = path + \"/.o_direct_test\";\n                return w{fpath, open_flags::wo | open_flags::create | open_flags::truncate, [fpath] { return remove_file(fpath); }};\n            } else if ((type == directory_entry_type::regular) || (type == directory_entry_type::link)) {\n                return w{path, open_flags::ro, [] { return make_ready_future<>(); }};\n            } else {\n                throw std::invalid_argument(format(\"{} neither a directory nor file. Can't be opened with O_DIRECT\", path));\n            }\n        };\n    };\n\n    // Allocating memory for a sstring can throw, hence the futurize_invoke\n    return futurize_invoke([path] {\n        return engine().file_type(path).then([path = sstring(path)] (auto type) {\n            auto w = w::parse(path, type);\n            return open_file_dma(w.path, w.flags).then_wrapped([path = w.path, cleanup = std::move(w.cleanup)] (future<file> f) {\n                try {\n                    auto fd = f.get();\n                    return cleanup().finally([fd = std::move(fd)] () mutable {\n                        return fd.close();\n                    });\n                } catch (std::system_error& e) {\n                    if (e.code() == std::error_code(EINVAL, std::system_category())) {\n                        report_exception(format(\"Could not open file at {}. Does your filesystem support O_DIRECT?\", path), std::current_exception());\n                    }\n                    throw;\n                }\n            });\n        });\n    });\n}\n\nserver_socket listen(socket_address sa) {\n    return engine().listen(sa);\n}\n\nserver_socket listen(socket_address sa, listen_options opts) {\n    return engine().listen(sa, opts);\n}\n\nfuture<connected_socket> connect(socket_address sa) {\n    return engine().connect(sa);\n}\n\nfuture<connected_socket> connect(socket_address sa, socket_address local, transport proto = transport::TCP) {\n    return engine().connect(sa, local, proto);\n}\n\nsocket make_socket() {\n    return engine().net().socket();\n}\n\nnet::udp_channel make_udp_channel() {\n    return make_unbound_datagram_channel(AF_INET);\n}\n\nnet::udp_channel make_udp_channel(const socket_address& local) {\n    return make_bound_datagram_channel(local);\n}\n\nnet::datagram_channel make_unbound_datagram_channel(sa_family_t family) {\n    return engine().net().make_unbound_datagram_channel(family);\n}\n\nnet::datagram_channel make_bound_datagram_channel(const socket_address& local) {\n    return engine().net().make_bound_datagram_channel(local);\n}\n\nvoid reactor::add_high_priority_task(task* t) noexcept {\n    add_urgent_task(t);\n    // break .then() chains\n    request_preemption();\n}\n\n\nvoid set_idle_cpu_handler(idle_cpu_handler&& handler) {\n    engine().set_idle_cpu_handler(std::move(handler));\n}\n\nnamespace experimental {\nfuture<std::tuple<file_desc, file_desc>> make_pipe() {\n    return engine().make_pipe();\n}\n\nfuture<process> spawn_process(const std::filesystem::path& pathname,\n                              spawn_parameters params) {\n    return process::spawn(pathname, std::move(params));\n}\n\nfuture<process> spawn_process(const std::filesystem::path& pathname) {\n    return process::spawn(pathname);\n}\n}\n\nstatic\nbool\nvirtualized() {\n    return fs::exists(\"/sys/hypervisor/type\");\n}\n\nstd::chrono::nanoseconds\nreactor::calculate_poll_time() {\n    // In a non-virtualized environment, select a poll time\n    // that is competitive with halt/unhalt.\n    //\n    // In a virutalized environment, IPIs are slow and dominate\n    // sleep/wake (mprotect/tgkill), so increase poll time to reduce\n    // so we don't sleep in a request/reply workload\n    return virtualized() ? 2000us : 200us;\n}\n\nfuture<>\nyield() noexcept {\n    memory::scoped_critical_alloc_section _;\n    auto tsk = make_task([] {});\n    schedule(tsk);\n    return tsk->get_future();\n}\n\nfuture<> check_for_io_immediately() noexcept {\n    memory::scoped_critical_alloc_section _;\n    engine().force_poll();\n    auto tsk = make_task(default_scheduling_group(), [] {});\n    schedule(tsk);\n    return tsk->get_future();\n}\n\nfuture<> later() noexcept {\n    return check_for_io_immediately();\n}\n\nsteady_clock_type::duration reactor::total_idle_time() {\n    return _total_idle;\n}\n\nsteady_clock_type::duration reactor::total_busy_time() {\n    return now() - _start_time - _total_idle;\n}\n\nstd::chrono::nanoseconds reactor::total_steal_time() {\n    // Steal time: this mimics the concept some Hypervisors have about Steal time.\n    // That is the time in which a VM has something to run, but is not running because some other\n    // process (another VM or the hypervisor itself) is in control.\n    //\n    // For us, we notice that during the time in which we were not sleeping (either running or busy\n    // polling while idle), we should be accumulating thread runtime. If we are not, that's because\n    // someone stole it from us.\n    //\n    // Because this is totally in userspace we can miss some events. For instance, if the seastar\n    // process is ready to run but the kernel hasn't scheduled us yet, that would be technically\n    // steal time but we have no ways to account it.\n    //\n    // But what we have here should be good enough and at least has a well defined meaning.\n    return std::chrono::duration_cast<std::chrono::nanoseconds>(now() - _start_time - _total_sleep) -\n           std::chrono::duration_cast<std::chrono::nanoseconds>(thread_cputime_clock::now().time_since_epoch());\n}\n\nstatic std::atomic<unsigned long> s_used_scheduling_group_ids_bitmap{3}; // 0=main, 1=atexit\nstatic std::atomic<unsigned long> s_next_scheduling_group_specific_key{0};\n\nstatic\nint\nallocate_scheduling_group_id() noexcept {\n    static_assert(max_scheduling_groups() <= std::numeric_limits<unsigned long>::digits, \"more scheduling groups than available bits\");\n    auto b = s_used_scheduling_group_ids_bitmap.load(std::memory_order_relaxed);\n    auto nb = b;\n    unsigned i = 0;\n    do {\n        if (__builtin_popcountl(b) == max_scheduling_groups()) {\n            return -1;\n        }\n        i = count_trailing_zeros(~b);\n        nb = b | (1ul << i);\n    } while (!s_used_scheduling_group_ids_bitmap.compare_exchange_weak(b, nb, std::memory_order_relaxed));\n    return i;\n}\n\nstatic\nunsigned long\nallocate_scheduling_group_specific_key() noexcept {\n    return  s_next_scheduling_group_specific_key.fetch_add(1, std::memory_order_relaxed);\n}\n\nstatic\nvoid\ndeallocate_scheduling_group_id(unsigned id) noexcept {\n    s_used_scheduling_group_ids_bitmap.fetch_and(~(1ul << id), std::memory_order_relaxed);\n}\n\nvoid\nreactor::allocate_scheduling_group_specific_data(scheduling_group sg, scheduling_group_key key) {\n    auto& sg_data = _scheduling_group_specific_data;\n    auto& this_sg = sg_data.per_scheduling_group_data[sg._id];\n    this_sg.specific_vals.resize(std::max<size_t>(this_sg.specific_vals.size(), key.id()+1));\n    this_sg.specific_vals[key.id()] =\n        aligned_alloc(sg_data.scheduling_group_key_configs[key.id()].alignment,\n                sg_data.scheduling_group_key_configs[key.id()].allocation_size);\n    if (!this_sg.specific_vals[key.id()]) {\n        std::abort();\n    }\n    if (sg_data.scheduling_group_key_configs[key.id()].constructor) {\n        sg_data.scheduling_group_key_configs[key.id()].constructor(this_sg.specific_vals[key.id()]);\n    }\n}\n\nfuture<>\nreactor::rename_scheduling_group_specific_data(scheduling_group sg) {\n    return with_scheduling_group(sg, [this, sg] {\n        auto& sg_data = _scheduling_group_specific_data;\n        auto& this_sg = sg_data.per_scheduling_group_data[sg._id];\n        for (size_t i = 0; i < sg_data.scheduling_group_key_configs.size(); ++i) {\n            auto &c = sg_data.scheduling_group_key_configs[i];\n            if (c.rename) {\n                (c.rename)(this_sg.specific_vals[i]);\n            }\n        }\n    });\n}\n\nfuture<>\nreactor::init_scheduling_group(seastar::scheduling_group sg, sstring name, sstring shortname, float shares) {\n    auto& sg_data = _scheduling_group_specific_data;\n    auto& this_sg = sg_data.per_scheduling_group_data[sg._id];\n    this_sg.queue_is_initialized = true;\n    _task_queues.resize(std::max<size_t>(_task_queues.size(), sg._id + 1));\n    _task_queues[sg._id] = std::make_unique<task_queue>(sg._id, name, shortname, shares);\n    unsigned long num_keys = s_next_scheduling_group_specific_key.load(std::memory_order_relaxed);\n\n    return with_scheduling_group(sg, [this, num_keys, sg] () {\n        for (unsigned long key_id = 0; key_id < num_keys; key_id++) {\n            allocate_scheduling_group_specific_data(sg, scheduling_group_key(key_id));\n        }\n    });\n}\n\nfuture<>\nreactor::init_new_scheduling_group_key(scheduling_group_key key, scheduling_group_key_config cfg) {\n    auto& sg_data = _scheduling_group_specific_data;\n    sg_data.scheduling_group_key_configs.resize(std::max<size_t>(sg_data.scheduling_group_key_configs.size(), key.id() + 1));\n    sg_data.scheduling_group_key_configs[key.id()] = cfg;\n    return parallel_for_each(_task_queues, [this, cfg, key] (std::unique_ptr<task_queue>& tq) {\n...\n", "needle_token_start": 12125, "needle_token_end": 12182, "code_context_ntokens": 16228, "output": ["```cpp\n#ifdef SEASTAR_HAVE_DPDK\n\nint dpdk_thread_adaptor(void* f)\n{\n    (*static_cast<std::function<void ()>*>(f))();\n    return 0;\n}\n\n#endif\n```"]}
{"repo": "scylladb/seastar", "name": "default_isolate_connection", "language": "cpp", "path": "src/rpc/rpc.cc", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide a default configuration for isolating a connection based on a specific identifier.\n2. **Input**: A string representing an identifier for the isolation configuration.\n3. **Output**: An object representing the configuration settings for isolating a connection.\n4. **Procedure**: The function receives an identifier as input and returns a new configuration object with default settings, without modifying any properties based on the input.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "      //\n      // The latter is achieved by resolving pit.done immediatelly, the former\n      // by moving it.done into pit.done. For simplicity (verging on obscurity?)\n      // both done's are just swapped and \"it\" resolves its new promise\n\n      std::swap(it->done, pit->done);\n      it->uncancellable();\n      it->unlink();\n      if (ex == nullptr) {\n          it->done.set_value();\n      } else {\n          it->done.set_exception(ex);\n      }\n  }\n\n  future<> connection::send(snd_buf buf, std::optional<rpc_clock_type::time_point> timeout, cancellable* cancel) {\n      if (!_error) {\n          if (timeout && *timeout <= rpc_clock_type::now()) {\n              return make_ready_future<>();\n          }\n\n          auto p = std::make_unique<outgoing_entry>(std::move(buf));\n          auto& d = *p;\n          _outgoing_queue.push_back(d);\n          _outgoing_queue_size++;\n          auto deleter = [this, it = _outgoing_queue.iterator_to(d)] {\n              // Front entry is most likely (unless _negotiated is unresolved, check enqueue_zero_frame()) sitting\n              // inside send_entry() continuations and thus it cannot be cancelled.\n              if (it != _outgoing_queue.begin()) {\n                  withdraw(it);\n              }\n          };\n\n          if (timeout) {\n              auto& t = d.t;\n              t.set_callback(deleter);\n              t.arm(timeout.value());\n          }\n          if (cancel) {\n              cancel->cancel_send = std::move(deleter);\n              cancel->send_back_pointer = &d.pcancel;\n              d.pcancel = cancel;\n          }\n\n          // New entry should continue (do its .then() lambda) after _outgoing_queue_ready\n          // resolves. Next entry will need to do the same after this entry's done resolves.\n          // Thus -- replace _outgoing_queue_ready with d's future and chain its continuation\n          // on ..._ready's old value.\n          return std::exchange(_outgoing_queue_ready, d.done.get_future()).then([this, p = std::move(p)] () mutable {\n              _outgoing_queue_size--;\n              if (__builtin_expect(!p->is_linked(), false)) {\n                  // If withdrawn the entry is unlinked and this lambda is fired right at once\n                  return make_ready_future<>();\n              }\n\n              p->uncancellable();\n              return send_entry(*p).then_wrapped([this, p = std::move(p)] (auto f) mutable {\n                  if (f.failed()) {\n                      f.ignore_ready_future();\n                      abort();\n                  }\n                  p->done.set_value();\n              });\n          });\n      } else {\n          return make_exception_future<>(closed_error());\n      }\n  }\n\n  void connection::abort() {\n      if (!_error) {\n          _error = true;\n          _fd.shutdown_input();\n      }\n  }\n\n  future<> connection::stop() noexcept {\n      try {\n          abort();\n      } catch (...) {\n          log_exception(*this, log_level::error, \"fail to shutdown connection while stopping\", std::current_exception());\n      }\n      return _stopped.get_future();\n  }\n\n  template<typename Connection>\n  static bool verify_frame(Connection& c, temporary_buffer<char>& buf, size_t expected, const char* log) {\n      if (buf.size() != expected) {\n          if (buf.size() != 0) {\n              c.get_logger()(c.peer_address(), log);\n          }\n          return false;\n      }\n      return true;\n  }\n\n  template<typename Connection>\n  static\n  future<feature_map>\n  receive_negotiation_frame(Connection& c, input_stream<char>& in) {\n      return in.read_exactly(sizeof(negotiation_frame)).then([&c, &in] (temporary_buffer<char> neg) {\n          if (!verify_frame(c, neg, sizeof(negotiation_frame), \"unexpected eof during negotiation frame\")) {\n              return make_exception_future<feature_map>(closed_error());\n          }\n          negotiation_frame frame;\n          std::copy_n(neg.get_write(), sizeof(frame.magic), frame.magic);\n          frame.len = read_le<uint32_t>(neg.get_write() + 8);\n          if (std::memcmp(frame.magic, rpc_magic, sizeof(frame.magic)) != 0) {\n              c.get_logger()(c.peer_address(), format(\"wrong protocol magic: {:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}\",\n                    frame.magic[0], frame.magic[1], frame.magic[2], frame.magic[3], frame.magic[4], frame.magic[5], frame.magic[6], frame.magic[7]));\n              return make_exception_future<feature_map>(closed_error());\n          }\n          auto len = frame.len;\n          return in.read_exactly(len).then([&c, len] (temporary_buffer<char> extra) {\n              if (extra.size() != len) {\n                  c.get_logger()(c.peer_address(), \"unexpected eof during negotiation frame\");\n                  return make_exception_future<feature_map>(closed_error());\n              }\n              feature_map map;\n              auto p = extra.get();\n              auto end = p + extra.size();\n              while (p != end) {\n                  if (end - p < 8) {\n                      c.get_logger()(c.peer_address(), \"bad feature data format in negotiation frame\");\n                      return make_exception_future<feature_map>(closed_error());\n                  }\n                  auto feature = static_cast<protocol_features>(read_le<uint32_t>(p));\n                  auto f_len = read_le<uint32_t>(p + 4);\n                  p += 8;\n                  if (f_len > end - p) {\n                      c.get_logger()(c.peer_address(), \"buffer underflow in feature data in negotiation frame\");\n                      return make_exception_future<feature_map>(closed_error());\n                  }\n                  auto data = sstring(p, f_len);\n                  p += f_len;\n                  map.emplace(feature, std::move(data));\n              }\n              return make_ready_future<feature_map>(std::move(map));\n          });\n      });\n  }\n\n  inline future<rcv_buf>\n  read_rcv_buf(input_stream<char>& in, uint32_t size) {\n      return in.read_up_to(size).then([&, size] (temporary_buffer<char> data) mutable {\n          rcv_buf rb(size);\n          if (data.size() == 0) {\n              return make_ready_future<rcv_buf>(rcv_buf());\n          } else if (data.size() == size) {\n              rb.bufs = std::move(data);\n              return make_ready_future<rcv_buf>(std::move(rb));\n          } else {\n              size -= data.size();\n              std::vector<temporary_buffer<char>> v;\n              v.push_back(std::move(data));\n              rb.bufs = std::move(v);\n              return do_with(std::move(rb), std::move(size), [&in] (rcv_buf& rb, uint32_t& left) {\n                  return repeat([&] () {\n                      return in.read_up_to(left).then([&] (temporary_buffer<char> data) {\n                          if (!data.size()) {\n                              rb.size -= left;\n                              return stop_iteration::yes;\n                          } else {\n                              left -= data.size();\n                              std::get<std::vector<temporary_buffer<char>>>(rb.bufs).push_back(std::move(data));\n                              return left ? stop_iteration::no : stop_iteration::yes;\n                          }\n                      });\n                  }).then([&rb] {\n                      return std::move(rb);\n                  });\n              });\n          }\n      });\n  }\n\n  template<typename FrameType>\n  future<typename FrameType::return_type>\n  connection::read_frame(socket_address info, input_stream<char>& in) {\n      auto header_size = FrameType::header_size();\n      return in.read_exactly(header_size).then([this, header_size, info, &in] (temporary_buffer<char> header) {\n          if (header.size() != header_size) {\n              if (header.size() != 0) {\n                  _logger(info, format(\"unexpected eof on a {} while reading header: expected {:d} got {:d}\", FrameType::role(), header_size, header.size()));\n              }\n              return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n          }\n          auto [size, h] = FrameType::decode_header(header.get());\n          if (!size) {\n              return make_ready_future<typename FrameType::return_type>(FrameType::make_value(h, rcv_buf()));\n          } else {\n              return read_rcv_buf(in, size).then([this, info, h = std::move(h), size] (rcv_buf rb) {\n                  if (rb.size != size) {\n                      _logger(info, format(\"unexpected eof on a {} while reading data: expected {:d} got {:d}\", FrameType::role(), size, rb.size));\n                      return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n                  } else {\n                      return make_ready_future<typename FrameType::return_type>(FrameType::make_value(h, std::move(rb)));\n                  }\n              });\n          }\n      });\n  }\n\n  template<typename FrameType>\n  future<typename FrameType::return_type>\n  connection::read_frame_compressed(socket_address info, std::unique_ptr<compressor>& compressor, input_stream<char>& in) {\n      if (compressor) {\n          return in.read_exactly(4).then([this, info, &in, &compressor] (temporary_buffer<char> compress_header) {\n              if (compress_header.size() != 4) {\n                  if (compress_header.size() != 0) {\n                      _logger(info, format(\"unexpected eof on a {} while reading compression header: expected 4 got {:d}\", FrameType::role(), compress_header.size()));\n                  }\n                  return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n              }\n              auto ptr = compress_header.get();\n              auto size = read_le<uint32_t>(ptr);\n              return read_rcv_buf(in, size).then([this, size, &compressor, info, &in] (rcv_buf compressed_data) {\n                  if (compressed_data.size != size) {\n                      _logger(info, format(\"unexpected eof on a {} while reading compressed data: expected {:d} got {:d}\", FrameType::role(), size, compressed_data.size));\n                      return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n                  }\n                  auto eb = compressor->decompress(std::move(compressed_data));\n                  if (eb.size == 0) {\n                      // Empty frames might be sent as means of communication between the compressors, and should be skipped by the RPC layer.\n                      // We skip the empty frame here. We recursively restart the function, as if the empty frame didn't happen.\n                      // The yield() is here to limit the stack depth of the recursion to 1.\n                      return yield().then([this, info, &in, &compressor] { return read_frame_compressed<FrameType>(info, compressor, in); });\n                  }\n                  net::packet p;\n                  auto* one = std::get_if<temporary_buffer<char>>(&eb.bufs);\n                  if (one) {\n                      p = net::packet(std::move(p), std::move(*one));\n                  } else {\n                      auto&& bufs = std::get<std::vector<temporary_buffer<char>>>(eb.bufs);\n                      p.reserve(bufs.size());\n                      for (auto&& b : bufs) {\n                          p = net::packet(std::move(p), std::move(b));\n                      }\n                  }\n                  return do_with(as_input_stream(std::move(p)), [this, info] (input_stream<char>& in) {\n                      return read_frame<FrameType>(info, in);\n                  });\n              });\n          });\n      } else {\n          return read_frame<FrameType>(info, in);\n      }\n  }\n\n  struct stream_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = opt_buf_type;\n      struct header_type {\n          bool eos;\n      };\n      static size_t header_size() {\n          return 4;\n      }\n      static const char* role() {\n          return \"stream\";\n      }\n      static auto empty_value() {\n          return std::nullopt;\n      }\n      static std::pair<uint32_t, header_type> decode_header(const char* ptr) {\n          auto size = read_le<uint32_t>(ptr);\n          return size != -1U ? std::make_pair(size, header_type{false}) : std::make_pair(0U, header_type{true});\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          if (t.eos) {\n              data.size = -1U;\n          }\n          return data;\n      }\n  };\n\n  future<std::optional<rcv_buf>>\n  connection::read_stream_frame_compressed(input_stream<char>& in) {\n      return read_frame_compressed<stream_frame>(peer_address(), _compressor, in);\n  }\n\n  future<> connection::stream_close() {\n      auto f = make_ready_future<>();\n      if (!error()) {\n          promise<bool> p;\n          _sink_closed_future = p.get_future();\n          // stop_send_loop(), which also calls _write_buf.close(), and this code can run in parallel.\n          // Use _sink_closed_future to serialize them and skip second call to close()\n          f = _write_buf.close().finally([p = std::move(p)] () mutable { p.set_value(true);});\n      }\n      return f.finally([this] () mutable { return stop(); });\n  }\n\n  future<> connection::stream_process_incoming(rcv_buf&& buf) {\n      // we do not want to dead lock on huge packets, so let them in\n      // but only one at a time\n      auto size = std::min(size_t(buf.size), max_stream_buffers_memory);\n      return get_units(_stream_sem, size).then([this, buf = std::move(buf)] (semaphore_units<>&& su) mutable {\n          buf.su = std::move(su);\n          return _stream_queue.push_eventually(std::move(buf));\n      });\n  }\n\n  future<> connection::handle_stream_frame() {\n      return read_stream_frame_compressed(_read_buf).then([this] (std::optional<rcv_buf> data) {\n          if (!data) {\n              _error = true;\n              return make_ready_future<>();\n          }\n          return stream_process_incoming(std::move(*data));\n      });\n  }\n\n  future<> connection::stream_receive(circular_buffer<foreign_ptr<std::unique_ptr<rcv_buf>>>& bufs) {\n      return _stream_queue.not_empty().then([this, &bufs] {\n          bool eof = !_stream_queue.consume([&bufs] (rcv_buf&& b) {\n              if (b.size == -1U) { // max fragment length marks an end of a stream\n                  return false;\n              } else {\n                  bufs.push_back(make_foreign(std::make_unique<rcv_buf>(std::move(b))));\n                  return true;\n              }\n          });\n          if (eof && !bufs.empty()) {\n              assert(_stream_queue.empty());\n              _stream_queue.push(rcv_buf(-1U)); // push eof marker back for next read to notice it\n          }\n      });\n  }\n\n  void connection::register_stream(connection_id id, xshard_connection_ptr c) {\n      _streams.emplace(id, std::move(c));\n  }\n\n  xshard_connection_ptr connection::get_stream(connection_id id) const {\n      auto it = _streams.find(id);\n      if (it == _streams.end()) {\n          throw std::logic_error(format(\"rpc stream id {} not found\", id).c_str());\n      }\n      return it->second;\n  }\n\n  // The request frame is\n  //   le64 optional timeout (see request_frame_with_timeout below)\n  //   le64 message type a.k.a. verb ID\n  //   le64 message ID\n  //   le32 payload length\n  //   ...  payload\n  struct request_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = std::tuple<std::optional<uint64_t>, uint64_t, int64_t, opt_buf_type>;\n      using header_type = std::tuple<std::optional<uint64_t>, uint64_t, int64_t>;\n      static constexpr size_t raw_header_size = sizeof(uint64_t) + sizeof(int64_t) + sizeof(uint32_t);\n      static size_t header_size() {\n          static_assert(request_frame_headroom >= raw_header_size);\n          return raw_header_size;\n      }\n      static const char* role() {\n          return \"server\";\n      }\n      static auto empty_value() {\n          return std::make_tuple(std::nullopt, uint64_t(0), 0, std::nullopt);\n      }\n      static std::pair<size_t, header_type> decode_header(const char* ptr) {\n          auto type = read_le<uint64_t>(ptr);\n          auto msgid = read_le<int64_t>(ptr + 8);\n          auto size = read_le<uint32_t>(ptr + 16);\n          return std::make_pair(size, std::make_tuple(std::nullopt, type, msgid));\n      }\n      static void encode_header(uint64_t type, int64_t msg_id, snd_buf& buf, size_t off) {\n          auto p = buf.front().get_write() + off;\n          write_le<uint64_t>(p, type);\n          write_le<int64_t>(p + 8, msg_id);\n          write_le<uint32_t>(p + 16, buf.size - raw_header_size - off);\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          return std::make_tuple(std::get<0>(t), std::get<1>(t), std::get<2>(t), std::move(data));\n      }\n  };\n\n  // This frame is used if protocol_features.TIMEOUT was negotiated\n  struct request_frame_with_timeout : request_frame {\n      using super = request_frame;\n      static constexpr size_t raw_header_size = sizeof(uint64_t) + request_frame::raw_header_size;\n      static size_t header_size() {\n          static_assert(request_frame_headroom >= raw_header_size);\n          return raw_header_size;\n      }\n      static std::pair<uint32_t, typename super::header_type> decode_header(const char* ptr) {\n          auto h = super::decode_header(ptr + 8);\n          std::get<0>(h.second) = read_le<uint64_t>(ptr);\n          return h;\n      }\n      static void encode_header(uint64_t type, int64_t msg_id, snd_buf& buf) {\n          static_assert(snd_buf::chunk_size >= raw_header_size, \"send buffer chunk size is too small\");\n          // expiration timer is encoded later\n          request_frame::encode_header(type, msg_id, buf, 8);\n      }\n  };\n\n  future<> client::request(uint64_t type, int64_t msg_id, snd_buf buf, std::optional<rpc_clock_type::time_point> timeout, cancellable* cancel) {\n      request_frame_with_timeout::encode_header(type, msg_id, buf);\n      return send(std::move(buf), timeout, cancel);\n  }\n\n  void\n  client::negotiate(feature_map provided) {\n      // record features returned here\n      for (auto&& e : provided) {\n          auto id = e.first;\n          switch (id) {\n          // supported features go here\n          case protocol_features::COMPRESS:\n              if (_options.compressor_factory) {\n                  _compressor = _options.compressor_factory->negotiate(e.second, false, [this] { return send({}); });\n              }\n              if (!_compressor) {\n                  throw std::runtime_error(format(\"RPC server responded with compression {} - unsupported\", e.second));\n              }\n              break;\n          case protocol_features::TIMEOUT:\n              _timeout_negotiated = true;\n              break;\n          case protocol_features::CONNECTION_ID: {\n              _id = deserialize_connection_id(e.second);\n              break;\n          }\n          default:\n              // nothing to do\n              ;\n          }\n      }\n  }\n\n  future<> client::negotiate_protocol(feature_map features) {\n      return send_negotiation_frame(std::move(features)).then([this] {\n          return receive_negotiation_frame(*this, _read_buf).then([this] (feature_map features) {\n              return negotiate(std::move(features));\n          });\n      });\n  }\n\n  // The response frame is\n  //   le64 message ID\n  //   le32 payload size\n  //   ...  payload\n  struct response_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = std::tuple<int64_t, opt_buf_type>;\n      using header_type = std::tuple<int64_t>;\n      static constexpr size_t raw_header_size = sizeof(int64_t) + sizeof(uint32_t);\n      static size_t header_size() {\n          static_assert(response_frame_headroom >= raw_header_size);\n          return raw_header_size;\n      }\n      static const char* role() {\n          return \"client\";\n      }\n      static auto empty_value() {\n          return std::make_tuple(0, std::nullopt);\n      }\n      static std::pair<uint32_t, header_type> decode_header(const char* ptr) {\n          auto msgid = read_le<int64_t>(ptr);\n          auto size = read_le<uint32_t>(ptr + 8);\n          return std::make_pair(size, std::make_tuple(msgid));\n      }\n      static void encode_header(int64_t msg_id, snd_buf& data) {\n          static_assert(snd_buf::chunk_size >= raw_header_size, \"send buffer chunk size is too small\");\n          auto p = data.front().get_write();\n          write_le<int64_t>(p, msg_id);\n          write_le<uint32_t>(p + 8, data.size - raw_header_size);\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          return std::make_tuple(std::get<0>(t), std::move(data));\n      }\n  };\n\n\n  future<response_frame::return_type>\n  client::read_response_frame_compressed(input_stream<char>& in) {\n      return read_frame_compressed<response_frame>(_server_addr, _compressor, in);\n  }\n\n  stats client::get_stats() const {\n      stats res = _stats;\n      res.wait_reply = incoming_queue_length();\n      res.pending = outgoing_queue_length();\n      return res;\n  }\n\n  void client::wait_for_reply(id_type id, std::unique_ptr<reply_handler_base>&& h, std::optional<rpc_clock_type::time_point> timeout, cancellable* cancel) {\n      if (timeout) {\n          h->t.set_callback(std::bind(std::mem_fn(&client::wait_timed_out), this, id));\n          h->t.arm(timeout.value());\n      }\n      if (cancel) {\n          cancel->cancel_wait = [this, id] {\n              _outstanding[id]->cancel();\n              _outstanding.erase(id);\n          };\n          h->pcancel = cancel;\n          cancel->wait_back_pointer = &h->pcancel;\n      }\n      _outstanding.emplace(id, std::move(h));\n  }\n  void client::wait_timed_out(id_type id) {\n      _stats.timeout++;\n      _outstanding[id]->timeout();\n      _outstanding.erase(id);\n  }\n\n  future<> client::stop() noexcept {\n      _error = true;\n      try {\n          _socket.shutdown();\n      } catch(...) {\n          log_exception(*this, log_level::error, \"fail to shutdown connection while stopping\", std::current_exception());\n      }\n      return _stopped.get_future();\n  }\n\n  void client::abort_all_streams() {\n      while (!_streams.empty()) {\n          auto&& s = _streams.begin();\n          assert(s->second->get_owner_shard() == this_shard_id()); // abort can be called only locally\n          s->second->get()->abort();\n          _streams.erase(s);\n      }\n  }\n\n  void client::deregister_this_stream() {\n      if (_parent) {\n          _parent->_streams.erase(_id);\n      }\n  }\n\n  // This is the enlightened copy of the connection::send() method. Its intention is to\n  // keep a dummy entry in front of the queue while connect+negotiate is happenning so\n  // that all subsequent entries could abort on timeout or explicit cancellation.\n  void client::enqueue_zero_frame() {\n      if (_error) {\n          return;\n      }\n\n      auto p = std::make_unique<outgoing_entry>(snd_buf(0));\n      auto& d = *p;\n      _outgoing_queue.push_back(d);\n\n      // Make it in the background. Even if the client is stopped it will pick\n      // up all the entries hanging around\n      (void)std::exchange(_outgoing_queue_ready, d.done.get_future()).then_wrapped([p = std::move(p)] (auto f) mutable {\n          if (f.failed()) {\n              f.ignore_ready_future();\n          } else {\n              p->done.set_value();\n          }\n      });\n  }\n\n  struct client::metrics::domain {\n      metrics::domain_list_t list;\n      stats dead;\n      seastar::metrics::metric_groups metric_groups;\n\n      static thread_local std::unordered_map<sstring, domain> all;\n      static domain& find_or_create(sstring name);\n\n      stats::counter_type count_all(stats::counter_type stats::* field) noexcept {\n          stats::counter_type res = dead.*field;\n          for (const auto& m : list) {\n              res += m._c._stats.*field;\n          }\n          return res;\n      }\n\n      size_t count_all_fn(size_t (client::*fn)(void) const) noexcept {\n          size_t res = 0;\n          for (const auto& m : list) {\n              res += (m._c.*fn)();\n          }\n          return res;\n      }\n\n      domain(sstring name)\n      {\n          namespace sm = seastar::metrics;\n          auto domain_l = sm::label(\"domain\")(name);\n\n          metric_groups.add_group(\"rpc_client\", {\n                sm::make_gauge(\"count\", [this] { return list.size(); },\n                        sm::description(\"Total number of clients\"), { domain_l }),\n                sm::make_counter(\"sent_messages\", std::bind(&domain::count_all, this, &stats::sent_messages),\n                        sm::description(\"Total number of messages sent\"), { domain_l }),\n                sm::make_counter(\"replied\", std::bind(&domain::count_all, this, &stats::replied),\n                        sm::description(\"Total number of responses received\"), { domain_l }),\n                sm::make_counter(\"exception_received\", std::bind(&domain::count_all, this, &stats::exception_received),\n                        sm::description(\"Total number of exceptional responses received\"), { domain_l }).set_skip_when_empty(),\n                sm::make_counter(\"timeout\", std::bind(&domain::count_all, this, &stats::timeout),\n                        sm::description(\"Total number of timeout responses\"), { domain_l }).set_skip_when_empty(),\n                sm::make_gauge(\"pending\", std::bind(&domain::count_all_fn, this, &client::outgoing_queue_length),\n                    sm::description(\"Number of queued outbound messages\"), { domain_l }),\n                sm::make_gauge(\"wait_reply\", std::bind(&domain::count_all_fn, this, &client::incoming_queue_length),\n                    sm::description(\"Number of replies waiting for\"), { domain_l }),\n          });\n      }\n  };\n\n  thread_local std::unordered_map<sstring, client::metrics::domain> client::metrics::domain::all;\n\n  client::metrics::domain& client::metrics::domain::find_or_create(sstring name) {\n      auto i = all.try_emplace(name, name);\n      return i.first->second;\n  }\n\n  client::metrics::metrics(const client& c)\n        : _c(c)\n        , _domain(domain::find_or_create(_c._options.metrics_domain))\n  {\n      _domain.list.push_back(*this);\n  }\n\n  client::metrics::~metrics() {\n      _domain.dead.replied += _c._stats.replied;\n      _domain.dead.exception_received += _c._stats.exception_received;\n      _domain.dead.sent_messages += _c._stats.sent_messages;\n      _domain.dead.timeout += _c._stats.timeout;\n  }\n\n  client::client(const logger& l, void* s, client_options ops, socket socket, const socket_address& addr, const socket_address& local)\n  : rpc::connection(l, s), _socket(std::move(socket)), _server_addr(addr), _local_addr(local), _options(ops), _metrics(*this)\n  {\n       _socket.set_reuseaddr(ops.reuseaddr);\n      // Run client in the background.\n      // Communicate result via _stopped.\n      // The caller has to call client::stop() to synchronize.\n      (void)_socket.connect(addr, local).then([this, ops = std::move(ops)] (connected_socket fd) {\n          fd.set_nodelay(ops.tcp_nodelay);\n          if (ops.keepalive) {\n              fd.set_keepalive(true);\n              fd.set_keepalive_parameters(ops.keepalive.value());\n          }\n          set_socket(std::move(fd));\n\n          feature_map features;\n          if (_options.compressor_factory) {\n              features[protocol_features::COMPRESS] = _options.compressor_factory->supported();\n          }\n          if (_options.send_timeout_data) {\n              features[protocol_features::TIMEOUT] = \"\";\n          }\n          if (_options.stream_parent) {\n              features[protocol_features::STREAM_PARENT] = serialize_connection_id(_options.stream_parent);\n          }\n          if (!_options.isolation_cookie.empty()) {\n              features[protocol_features::ISOLATION] = _options.isolation_cookie;\n          }\n\n          return negotiate_protocol(std::move(features)).then([this] {\n              _propagate_timeout = !is_stream();\n              set_negotiated();\n              return do_until([this] { return _read_buf.eof() || _error; }, [this] () mutable {\n                  if (is_stream()) {\n                      return handle_stream_frame();\n                  }\n                  return read_response_frame_compressed(_read_buf).then([this] (response_frame::return_type msg_id_and_data) {\n                      auto& msg_id = std::get<0>(msg_id_and_data);\n                      auto& data = std::get<1>(msg_id_and_data);\n                      auto it = _outstanding.find(std::abs(msg_id));\n                      if (!data) {\n                          _error = true;\n                      } else if (it != _outstanding.end()) {\n                          auto handler = std::move(it->second);\n                          _outstanding.erase(it);\n                          (*handler)(*this, msg_id, std::move(data.value()));\n                      } else if (msg_id < 0) {\n                          try {\n                              std::rethrow_exception(unmarshal_exception(data.value()));\n                          } catch(const unknown_verb_error& ex) {\n                              // if this is unknown verb exception with unknown id ignore it\n                              // can happen if unknown verb was used by no_wait client\n                              get_logger()(peer_address(), format(\"unknown verb exception {:d} ignored\", ex.type));\n                          } catch(...) {\n                              // We've got error response but handler is no longer waiting, could be timed out.\n                              log_exception(*this, log_level::info, \"ignoring error response\", std::current_exception());\n                          }\n                      } else {\n                          // we get a reply for a message id not in _outstanding\n                          // this can happened if the message id is timed out already\n                          get_logger()(peer_address(), log_level::debug, \"got a reply for an expired message id\");\n                      }\n                  });\n              });\n          });\n      }).then_wrapped([this] (future<> f) {\n          std::exception_ptr ep;\n          if (f.failed()) {\n              ep = f.get_exception();\n              if (_connected) {\n                  if (is_stream()) {\n                      log_exception(*this, log_level::error, \"client stream connection dropped\", ep);\n                  } else {\n                      log_exception(*this, log_level::error, \"client connection dropped\", ep);\n                  }\n              } else {\n                  if (is_stream()) {\n                      log_exception(*this, log_level::debug, \"stream fail to connect\", ep);\n                  } else {\n                      log_exception(*this, log_level::debug, \"fail to connect\", ep);\n                  }\n              }\n          }\n          _error = true;\n          _stream_queue.abort(std::make_exception_ptr(stream_closed()));\n          return stop_send_loop(ep).then_wrapped([this] (future<> f) {\n              f.ignore_ready_future();\n              _outstanding.clear();\n              if (is_stream()) {\n                  deregister_this_stream();\n              } else {\n                  abort_all_streams();\n              }\n          }).finally([this] {\n              return _compressor ? _compressor->close() : make_ready_future();\n          }).finally([this]{\n              _stopped.set_value();\n          });\n      });\n      enqueue_zero_frame();\n  }\n\n  client::client(const logger& l, void* s, const socket_address& addr, const socket_address& local)\n  : client(l, s, client_options{}, make_socket(), addr, local)\n  {}\n\n  client::client(const logger& l, void* s, client_options options, const socket_address& addr, const socket_address& local)\n  : client(l, s, options, make_socket(), addr, local)\n  {}\n\n  client::client(const logger& l, void* s, socket socket, const socket_address& addr, const socket_address& local)\n  : client(l, s, client_options{}, std::move(socket), addr, local)\n  {}\n\n\n  future<feature_map>\n  server::connection::negotiate(feature_map requested) {\n      feature_map ret;\n      future<> f = make_ready_future<>();\n      for (auto&& e : requested) {\n          auto id = e.first;\n          switch (id) {\n          // supported features go here\n          case protocol_features::COMPRESS: {\n              if (get_server()._options.compressor_factory) {\n                  _compressor = get_server()._options.compressor_factory->negotiate(e.second, true, [this] { return send({}); });\n                  if (_compressor) {\n                       ret[protocol_features::COMPRESS] = _compressor->name();\n                  }\n              }\n          }\n          break;\n          case protocol_features::TIMEOUT:\n              _timeout_negotiated = true;\n              ret[protocol_features::TIMEOUT] = \"\";\n              break;\n          case protocol_features::STREAM_PARENT: {\n              if (!get_server()._options.streaming_domain) {\n                  f = f.then([] {\n                      return make_exception_future<>(std::runtime_error(\"streaming is not configured for the server\"));\n                  });\n              } else {\n                  _parent_id = deserialize_connection_id(e.second);\n                  _is_stream = true;\n                  // remove stream connection from rpc connection list\n                  get_server()._conns.erase(get_connection_id());\n                  f = f.then([this, c = shared_from_this()] () mutable {\n                      return smp::submit_to(_parent_id.shard(), [this, c = make_foreign(static_pointer_cast<rpc::connection>(c))] () mutable {\n                          auto sit = _servers.find(*get_server()._options.streaming_domain);\n                          if (sit == _servers.end()) {\n                              throw std::logic_error(format(\"Shard {:d} does not have server with streaming domain {}\", this_shard_id(), *get_server()._options.streaming_domain).c_str());\n                          }\n                          auto s = sit->second;\n                          auto it = s->_conns.find(_parent_id);\n                          if (it == s->_conns.end()) {\n                              throw std::logic_error(format(\"Unknown parent connection {} on shard {:d}\", _parent_id, this_shard_id()).c_str());\n                          }\n                          if (it->second->_error) {\n                              throw std::runtime_error(format(\"Parent connection {} is aborting on shard {:d}\", _parent_id, this_shard_id()).c_str());\n                          }\n                          auto id = c->get_connection_id();\n                          it->second->register_stream(id, make_lw_shared(std::move(c)));\n                      });\n                  });\n              }\n              break;\n          }\n          case protocol_features::ISOLATION: {\n              auto&& isolation_cookie = e.second;\n              struct isolation_function_visitor {\n                  isolation_function_visitor(const sstring& isolation_cookie) :\n                        _isolation_cookie(isolation_cookie) { }\n                  future<isolation_config> operator() (resource_limits::syncronous_isolation_function f) const {\n                      return futurize_invoke(f, _isolation_cookie);\n                  }\n                  future<isolation_config> operator() (resource_limits::asyncronous_isolation_function f) const {\n                      return f(_isolation_cookie);\n                  }\n              private:\n                  sstring _isolation_cookie;\n              };\n\n              auto visitor = isolation_function_visitor(isolation_cookie);\n              f = f.then([visitor = std::move(visitor), this] () mutable {\n                  return std::visit(visitor, get_server()._limits.isolate_connection).then([this] (isolation_config conf) {\n                      _isolation_config = conf;\n                  });\n              });\n              ret.emplace(e);\n              break;\n          }\n          default:\n              // nothing to do\n              ;\n          }\n      }\n      if (get_server()._options.streaming_domain) {\n          ret[protocol_features::CONNECTION_ID] = serialize_connection_id(_id);\n      }\n      return f.then([ret = std::move(ret)] {\n          return ret;\n      });\n  }\n\n  future<>\n  server::connection::negotiate_protocol() {\n      return receive_negotiation_frame(*this, _read_buf).then([this] (feature_map requested_features) {\n          return negotiate(std::move(requested_features)).then([this] (feature_map returned_features) {\n              return send_negotiation_frame(std::move(returned_features));\n          });\n      });\n  }\n\n  future<request_frame::return_type>\n  server::connection::read_request_frame_compressed(input_stream<char>& in) {\n      if (_timeout_negotiated) {\n          return read_frame_compressed<request_frame_with_timeout>(_info.addr, _compressor, in);\n      } else {\n          return read_frame_compressed<request_frame>(_info.addr, _compressor, in);\n      }\n  }\n\n  future<>\n  server::connection::respond(int64_t msg_id, snd_buf&& data, std::optional<rpc_clock_type::time_point> timeout) {\n      response_frame::encode_header(msg_id, data);\n      return send(std::move(data), timeout);\n  }\n\nfuture<> server::connection::send_unknown_verb_reply(std::optional<rpc_clock_type::time_point> timeout, int64_t msg_id, uint64_t type) {\n    return wait_for_resources(28, timeout).then([this, timeout, msg_id, type] (auto permit) {\n        // send unknown_verb exception back\n        snd_buf data(28);\n        static_assert(snd_buf::chunk_size >= 28, \"send buffer chunk size is too small\");\n        auto p = data.front().get_write() + 12;\n        write_le<uint32_t>(p, uint32_t(exception_type::UNKNOWN_VERB));\n        write_le<uint32_t>(p + 4, uint32_t(8));\n        write_le<uint64_t>(p + 8, type);\n        try {\n            // Send asynchronously.\n            // This is safe since connection::stop() will wait for background work.\n            (void)with_gate(get_server()._reply_gate, [this, timeout, msg_id, data = std::move(data), permit = std::move(permit)] () mutable {\n                // workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83268\n                auto c = shared_from_this();\n                return respond(-msg_id, std::move(data), timeout).then([c = std::move(c), permit = std::move(permit)] {});\n            });\n        } catch(gate_closed_exception&) {/* ignore */}\n    });\n}\n\n  future<> server::connection::process() {\n      return negotiate_protocol().then([this] () mutable {\n        auto sg = _isolation_config ? _isolation_config->sched_group : current_scheduling_group();\n        return with_scheduling_group(sg, [this] {\n          set_negotiated();\n          return do_until([this] { return _read_buf.eof() || _error; }, [this] () mutable {\n              if (is_stream()) {\n                  return handle_stream_frame();\n              }\n              return read_request_frame_compressed(_read_buf).then([this] (request_frame::return_type header_and_buffer) {\n                  auto& expire = std::get<0>(header_and_buffer);\n                  auto& type = std::get<1>(header_and_buffer);\n                  auto& msg_id = std::get<2>(header_and_buffer);\n                  auto& data = std::get<3>(header_and_buffer);\n                  if (!data) {\n                      _error = true;\n                      return make_ready_future<>();\n                  } else {\n                      std::optional<rpc_clock_type::time_point> timeout;\n                      if (expire && *expire) {\n                          timeout = relative_timeout_to_absolute(std::chrono::milliseconds(*expire));\n                      }\n                      auto h = get_server()._proto.get_handler(type);\n                      if (!h) {\n                          return send_unknown_verb_reply(timeout, msg_id, type);\n                      }\n\n                      // If the new method of per-connection scheduling group was used, honor it.\n                      // Otherwise, use the old per-handler scheduling group.\n                      auto sg = _isolation_config ? _isolation_config->sched_group : h->handler.sg;\n                      return with_scheduling_group(sg, [this, timeout, msg_id, &h = h->handler, data = std::move(data.value()), guard = std::move(h->holder)] () mutable {\n                          return h.func(shared_from_this(), timeout, msg_id, std::move(data), std::move(guard));\n                      });\n                  }\n              });\n          });\n        });\n      }).then_wrapped([this] (future<> f) {\n          std::exception_ptr ep;\n          if (f.failed()) {\n              ep = f.get_exception();\n              log_exception(*this, log_level::error,\n                      format(\"server{} connection dropped\", is_stream() ? \" stream\" : \"\").c_str(), ep);\n          }\n          _fd.shutdown_input();\n          _error = true;\n          _stream_queue.abort(std::make_exception_ptr(stream_closed()));\n          return stop_send_loop(ep).then_wrapped([this] (future<> f) {\n              f.ignore_ready_future();\n              get_server()._conns.erase(get_connection_id());\n              if (is_stream()) {\n                  return deregister_this_stream();\n              } else {\n                  return abort_all_streams();\n              }\n          }).finally([this] {\n              return _compressor ? _compressor->close() : make_ready_future();\n          }).finally([this] {\n              _stopped.set_value();\n          });\n      }).finally([conn_ptr = shared_from_this()] {\n          // hold onto connection pointer until do_until() exists\n      });\n  }\n\n  server::connection::connection(server& s, connected_socket&& fd, socket_address&& addr, const logger& l, void* serializer, connection_id id)\n          : rpc::connection(std::move(fd), l, serializer, id)\n          , _info{.addr{std::move(addr)}, .server{s}, .conn_id{id}} {\n  }\n\n  future<> server::connection::deregister_this_stream() {\n      if (!get_server()._options.streaming_domain) {\n          return make_ready_future<>();\n      }\n      return smp::submit_to(_parent_id.shard(), [this] () mutable {\n          auto sit = server::_servers.find(*get_server()._options.streaming_domain);\n          if (sit != server::_servers.end()) {\n              auto s = sit->second;\n              auto it = s->_conns.find(_parent_id);\n              if (it != s->_conns.end()) {\n                  it->second->_streams.erase(get_connection_id());\n              }\n          }\n      });\n  }\n\n  future<> server::connection::abort_all_streams() {\n      return parallel_for_each(_streams | boost::adaptors::map_values, [] (xshard_connection_ptr s) {\n          return smp::submit_to(s->get_owner_shard(), [s] {\n              s->get()->abort();\n          });\n      }).then([this] {\n          _streams.clear();\n      });\n  }\n\n  thread_local std::unordered_map<streaming_domain_type, server*> server::_servers;\n\n  server::server(protocol_base* proto, const socket_address& addr, resource_limits limits)\n      : server(proto, seastar::listen(addr, listen_options{true}), limits, server_options{})\n  {}\n\n  server::server(protocol_base* proto, server_options opts, const socket_address& addr, resource_limits limits)\n      : server(proto, seastar::listen(addr, listen_options{true, opts.load_balancing_algorithm}), limits, opts)\n  {}\n\n  server::server(protocol_base* proto, server_socket ss, resource_limits limits, server_options opts)\n          : _proto(*proto), _ss(std::move(ss)), _limits(limits), _resources_available(limits.max_memory), _options(opts)\n  {\n      if (_options.streaming_domain) {\n          if (_servers.find(*_options.streaming_domain) != _servers.end()) {\n              throw std::runtime_error(format(\"An RPC server with the streaming domain {} is already exist\", *_options.streaming_domain));\n          }\n          _servers[*_options.streaming_domain] = this;\n      }\n      accept();\n  }\n\n  server::server(protocol_base* proto, server_options opts, server_socket ss, resource_limits limits)\n          : server(proto, std::move(ss), limits, opts)\n  {}\n\n  void server::accept() {\n      // Run asynchronously in background.\n      // Communicate result via __ss_stopped.\n      // The caller has to call server::stop() to synchronize.\n      (void)keep_doing([this] () mutable {\n          return _ss.accept().then([this] (accept_result ar) mutable {\n              if (_options.filter_connection && !_options.filter_connection(ar.remote_address)) {\n                  return;\n              }\n              auto fd = std::move(ar.connection);\n              auto addr = std::move(ar.remote_address);\n              fd.set_nodelay(_options.tcp_nodelay);\n              connection_id id = _options.streaming_domain ?\n                      connection_id::make_id(_next_client_id++, uint16_t(this_shard_id())) :\n                      connection_id::make_invalid_id(_next_client_id++);\n              auto conn = _proto.make_server_connection(*this, std::move(fd), std::move(addr), id);\n              auto r = _conns.emplace(id, conn);\n              assert(r.second);\n              // Process asynchronously in background.\n              (void)conn->process();\n          });\n      }).then_wrapped([this] (future<>&& f){\n          try {\n              f.get();\n              assert(false);\n          } catch (...) {\n              _ss_stopped.set_value();\n          }\n      });\n  }\n\n  future<> server::shutdown() {\n      if (_shutdown) {\n          return make_ready_future<>();\n      }\n\n      _ss.abort_accept();\n      _resources_available.broken();\n      if (_options.streaming_domain) {\n          _servers.erase(*_options.streaming_domain);\n      }\n      return _ss_stopped.get_future().then([this] {\n          return parallel_for_each(_conns | boost::adaptors::map_values, [] (shared_ptr<connection> conn) {\n              return conn->stop();\n          });\n      }).finally([this] {\n          _shutdown = true;\n      });\n  }\n\n  future<> server::stop() {\n      return when_all(\n          shutdown(),\n          _reply_gate.close()\n      ).discard_result();\n  }\n\n  void server::abort_connection(connection_id id) {\n      auto it = _conns.find(id);\n      if (it == _conns.end()) {\n          return;\n      }\n      try {\n          it->second->abort();\n      } catch (...) {\n          log_exception(*it->second, log_level::error,\n                        \"fail to shutdown connection on user request\", std::current_exception());\n      }\n  }\n\n  std::ostream& operator<<(std::ostream& os, const connection_id& id) {\n      fmt::print(os, \"{:x}\", id.id());\n      return os;\n  }\n\n  std::ostream& operator<<(std::ostream& os, const streaming_domain_type& domain) {\n      fmt::print(os, \"{:d}\", domain._id);\n      return os;\n  }\n\n  \nisolation_config default_isolate_connection(sstring isolation_cookie) {\n      return isolation_config{};\n  }\n\n}\n\n}\n\n// Path: src/rpc/lz4_compressor.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2016 Scylladb, Ltd.\n */\n\n#include <seastar/rpc/lz4_compressor.hh>\n#include <seastar/core/byteorder.hh>\n#include <lz4.h>\n\nnamespace seastar {\n\nnamespace rpc {\n\nsstring lz4_compressor::name() const {\n    return factory{}.supported();\n}\n\nconst sstring& lz4_compressor::factory::supported() const {\n    const static sstring name = \"LZ4\";\n    return name;\n}\n\nstd::unique_ptr<rpc::compressor> lz4_compressor::factory::negotiate(sstring feature, bool is_server) const {\n    return feature == supported() ? std::make_unique<lz4_compressor>() : nullptr;\n}\n\n// Reusable contiguous buffers needed for LZ4 compression and decompression functions.\nclass reusable_buffer {\n    static constexpr size_t chunk_size = 128 * 1024;\n    static_assert(snd_buf::chunk_size == chunk_size, \"snd_buf::chunk_size == chunk_size\");\n\n    std::unique_ptr<char[]> _data;\n    size_t _size;\nprivate:\n    void reserve(size_t n) {\n        if (_size < n) {\n            _data.reset();\n            // Not using std::make_unique to avoid value-initialisation.\n            _data = std::unique_ptr<char[]>(new char[n]);\n            _size = n;\n        }\n    }\npublic:\n    // Returns a pointer to a contiguous buffer containing all data stored in input.\n    // The pointer remains valid until next call to this.\n    const char* prepare(const std::variant<std::vector<temporary_buffer<char>>, temporary_buffer<char>>& input, size_t size) {\n        if (const auto single = std::get_if<temporary_buffer<char>>(&input)) {\n            return single->get();\n        }\n        reserve(size);\n        auto dst = _data.get();\n        for (const auto& fragment : std::get<std::vector<temporary_buffer<char>>>(input)) {\n            dst = std::copy_n(fragment.begin(), fragment.size(), dst);\n        }\n        return _data.get();\n    }\n\n    // Calls function fn passing to it a pointer to a temporary contigiuous max_size\n    // buffer.\n    // fn is supposed to return the actual size of the data.\n    // with_reserved() returns an Output object (snd_buf or rcv_buf or compatible),\n    // containing data that was written to the temporary buffer.\n    // Output should be either snd_buf or rcv_buf.\n    template<typename Output, typename Function>\n    requires requires (Function fn, char* ptr) {\n        { fn(ptr) } -> std::convertible_to<size_t>;\n    } && (std::is_same_v<Output, snd_buf> || std::is_same_v<Output, rcv_buf>)\n    Output with_reserved(size_t max_size, Function&& fn) {\n        if (max_size <= chunk_size) {\n            auto dst = temporary_buffer<char>(max_size);\n            size_t dst_size = fn(dst.get_write());\n            dst.trim(dst_size);\n            return Output(std::move(dst));\n        }\n\n        reserve(max_size);\n        size_t dst_size = fn(_data.get());\n        if (dst_size <= chunk_size) {\n            return Output(temporary_buffer<char>(_data.get(), dst_size));\n        }\n\n        auto left = dst_size;\n        auto pos = _data.get();\n        std::vector<temporary_buffer<char>> buffers;\n        while (left) {\n            auto this_size = std::min(left, chunk_size);\n            buffers.emplace_back(this_size);\n            std::copy_n(pos, this_size, buffers.back().get_write());\n            pos += this_size;\n            left -= this_size;\n        }\n        return Output(std::move(buffers), dst_size);\n    }\n\n    void clear() noexcept {\n        _data.reset();\n        _size = 0;\n    }\n};\n\nstatic thread_local reusable_buffer reusable_buffer_compressed_data;\nstatic thread_local reusable_buffer reusable_buffer_decompressed_data;\nstatic thread_local size_t buffer_use_count = 0;\nstatic constexpr size_t drop_buffers_trigger = 100'000;\n\nstatic void after_buffer_use() noexcept {\n    if (buffer_use_count++ == drop_buffers_trigger) {\n        reusable_buffer_compressed_data.clear();\n        reusable_buffer_decompressed_data.clear();\n        buffer_use_count = 0;\n    }\n}\n\nsnd_buf lz4_compressor::compress(size_t head_space, snd_buf data) {\n    head_space += 4;\n    auto dst_size = head_space + LZ4_compressBound(data.size);\n    auto dst = reusable_buffer_compressed_data.with_reserved<snd_buf>(dst_size, [&] (char* dst) {\n        auto src_size = data.size;\n        auto src = reusable_buffer_decompressed_data.prepare(data.bufs, data.size);\n\n        auto size = LZ4_compress_default(src, dst + head_space, src_size, LZ4_compressBound(src_size));\n        if (size == 0) {\n            throw std::runtime_error(\"RPC frame LZ4 compression failure\");\n        }\n        write_le<uint32_t>(dst + (head_space - 4), src_size);\n        return size + head_space;\n    });\n    after_buffer_use();\n    return dst;\n}\n\nrcv_buf lz4_compressor::decompress(rcv_buf data) {\n    if (data.size < 4) {\n        return rcv_buf();\n    } else {\n        auto src_size = data.size;\n        auto src = reusable_buffer_decompressed_data.prepare(data.bufs, data.size);\n\n        auto dst_size = read_le<uint32_t>(src);\n        if (!dst_size) {\n            throw std::runtime_error(\"RPC frame LZ4 decompression failure: decompressed size cannot be zero\");\n        }\n        src += sizeof(uint32_t);\n        src_size -= sizeof(uint32_t);\n\n        auto dst = reusable_buffer_compressed_data.with_reserved<rcv_buf>(dst_size, [&] (char* dst) {\n            if (LZ4_decompress_safe(src, dst, src_size, dst_size) < 0) {\n                throw std::runtime_error(\"RPC frame LZ4 decompression failure\");\n            }\n            return dst_size;\n        });\n        after_buffer_use();\n        return dst;\n    }\n}\n\n}\n\n}\n\n// Path: src/rpc/lz4_fragmented_compressor.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2019 Scylladb, Ltd.\n */\n\n#include <seastar/rpc/lz4_fragmented_compressor.hh>\n#include <seastar/core/byteorder.hh>\n\n#include <lz4.h>\n// LZ4_DECODER_RING_BUFFER_SIZE macro is introduced since v1.8.2\n// To work with previous lz4 release, copied the definition in lz4 here\n...\n", "needle_token_start": 13829, "needle_token_end": 13860, "code_context_ntokens": 16232, "output": ["```cpp\nisolation_config default_isolate_connection(sstring isolation_cookie) {\n    return isolation_config{};\n}\n```"]}
{"repo": "scylladb/seastar", "name": "is_control_char", "language": "cpp", "path": "src/json/formatter.cc", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function checks if a given character is a control character based on its ASCII value.\n2. **Input**: A single character.\n3. **Output**: A boolean value indicating whether the character is a control character.\n4. **Procedure**: The function evaluates if the ASCII value of the character is between 0 and 31 (inclusive), which are recognized as control characters in the ASCII table.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/rpc/rpc.cc\n#include <seastar/rpc/rpc.hh>\n#include <seastar/core/align.hh>\n#include <seastar/core/seastar.hh>\n#include <seastar/core/print.hh>\n#include <seastar/core/future-util.hh>\n#include <seastar/core/metrics.hh>\n#include <boost/range/adaptor/map.hpp>\n\n#if FMT_VERSION >= 90000\ntemplate <> struct fmt::formatter<seastar::rpc::streaming_domain_type> : fmt::ostream_formatter {};\n#endif\n\nnamespace seastar {\n\nnamespace rpc {\n\n    void logger::operator()(const client_info& info, id_type msg_id, const sstring& str) const {\n        log(format(\"client {} msg_id {}:  {}\", info.addr, msg_id, str));\n    }\n\n    void logger::operator()(const client_info& info, id_type msg_id, log_level level, std::string_view str) const {\n        log(level, \"client {} msg_id {}:  {}\", info.addr, msg_id, str);\n    }\n\n    void logger::operator()(const client_info& info, const sstring& str) const {\n        (*this)(info.addr, str);\n    }\n\n    void logger::operator()(const client_info& info, log_level level, std::string_view str) const {\n        (*this)(info.addr, level, str);\n    }\n\n    void logger::operator()(const socket_address& addr, const sstring& str) const {\n        log(format(\"client {}: {}\", addr, str));\n    }\n\n    void logger::operator()(const socket_address& addr, log_level level, std::string_view str) const {\n        log(level, \"client {}: {}\", addr, str);\n    }\n\n  no_wait_type no_wait;\n\n  snd_buf::snd_buf(size_t size_) : size(size_) {\n      if (size <= chunk_size) {\n          bufs = temporary_buffer<char>(size);\n      } else {\n          std::vector<temporary_buffer<char>> v;\n          v.reserve(align_up(size_t(size), chunk_size) / chunk_size);\n          while (size_) {\n              v.push_back(temporary_buffer<char>(std::min(chunk_size, size_)));\n              size_ -= v.back().size();\n          }\n          bufs = std::move(v);\n      }\n  }\n\n  snd_buf::snd_buf(snd_buf&&) noexcept = default;\n  snd_buf& snd_buf::operator=(snd_buf&&) noexcept = default;\n\n  temporary_buffer<char>& snd_buf::front() {\n      auto* one = std::get_if<temporary_buffer<char>>(&bufs);\n      if (one) {\n          return *one;\n      } else {\n          return std::get<std::vector<temporary_buffer<char>>>(bufs).front();\n      }\n  }\n\n  // Make a copy of a remote buffer. No data is actually copied, only pointers and\n  // a deleter of a new buffer takes care of deleting the original buffer\n  template<typename T> // T is either snd_buf or rcv_buf\n  T make_shard_local_buffer_copy(foreign_ptr<std::unique_ptr<T>> org) {\n      if (org.get_owner_shard() == this_shard_id()) {\n          return std::move(*org);\n      }\n      T buf(org->size);\n      auto* one = std::get_if<temporary_buffer<char>>(&org->bufs);\n\n      if (one) {\n          buf.bufs = temporary_buffer<char>(one->get_write(), one->size(), make_object_deleter(std::move(org)));\n      } else {\n          auto& orgbufs = std::get<std::vector<temporary_buffer<char>>>(org->bufs);\n          std::vector<temporary_buffer<char>> newbufs;\n          newbufs.reserve(orgbufs.size());\n          deleter d = make_object_deleter(std::move(org));\n          for (auto&& b : orgbufs) {\n              newbufs.push_back(temporary_buffer<char>(b.get_write(), b.size(), d.share()));\n          }\n          buf.bufs = std::move(newbufs);\n      }\n\n      return buf;\n  }\n\n  template snd_buf make_shard_local_buffer_copy(foreign_ptr<std::unique_ptr<snd_buf>>);\n  template rcv_buf make_shard_local_buffer_copy(foreign_ptr<std::unique_ptr<rcv_buf>>);\n\n  static void log_exception(connection& c, log_level level, const char* log, std::exception_ptr eptr) {\n      const char* s;\n      try {\n          std::rethrow_exception(eptr);\n      } catch (std::exception& ex) {\n          s = ex.what();\n      } catch (...) {\n          s = \"unknown exception\";\n      }\n      auto formatted = format(\"{}: {}\", log, s);\n      c.get_logger()(c.peer_address(), level, std::string_view(formatted.data(), formatted.size()));\n  }\n\n  snd_buf connection::compress(snd_buf buf) {\n      if (_compressor) {\n          buf = _compressor->compress(4, std::move(buf));\n          static_assert(snd_buf::chunk_size >= 4, \"send buffer chunk size is too small\");\n          write_le<uint32_t>(buf.front().get_write(), buf.size - 4);\n          return buf;\n      }\n      return buf;\n  }\n\n  future<> connection::send_buffer(snd_buf buf) {\n      auto* b = std::get_if<temporary_buffer<char>>(&buf.bufs);\n      if (b) {\n          return _write_buf.write(std::move(*b));\n      } else {\n          return do_with(std::move(std::get<std::vector<temporary_buffer<char>>>(buf.bufs)),\n                  [this] (std::vector<temporary_buffer<char>>& ar) {\n              return do_for_each(ar.begin(), ar.end(), [this] (auto& b) {\n                  return _write_buf.write(std::move(b));\n              });\n          });\n      }\n  }\n\n  future<> connection::send_entry(outgoing_entry& d) noexcept {\n    return futurize_invoke([this, &d] {\n      if (d.buf.size && _propagate_timeout) {\n          static_assert(snd_buf::chunk_size >= sizeof(uint64_t), \"send buffer chunk size is too small\");\n          if (_timeout_negotiated) {\n              auto expire = d.t.get_timeout();\n              uint64_t left = 0;\n              if (expire != typename timer<rpc_clock_type>::time_point()) {\n                  left = std::chrono::duration_cast<std::chrono::milliseconds>(expire - timer<rpc_clock_type>::clock::now()).count();\n              }\n              write_le<uint64_t>(d.buf.front().get_write(), left);\n          } else {\n              d.buf.front().trim_front(sizeof(uint64_t));\n              d.buf.size -= sizeof(uint64_t);\n          }\n      }\n      auto buf = compress(std::move(d.buf));\n      return send_buffer(std::move(buf)).then([this] {\n          _stats.sent_messages++;\n          return _write_buf.flush();\n      });\n    });\n  }\n\n  void connection::set_negotiated() noexcept {\n      _negotiated->set_value();\n      _negotiated = std::nullopt;\n  }\n\n  future<> connection::stop_send_loop(std::exception_ptr ex) {\n      _error = true;\n      if (_connected) {\n          _fd.shutdown_output();\n      }\n      if (ex == nullptr) {\n          ex = std::make_exception_ptr(closed_error());\n      }\n      while (!_outgoing_queue.empty()) {\n          auto it = std::prev(_outgoing_queue.end());\n          // Cancel all but front entry normally. The front entry is sitting in the\n          // send_entry() and cannot be withdrawn, except when _negotiated is still\n          // engaged. In the latter case when it will be aborted below the entry's\n          // continuation will not be called and its done promise will not resolve\n          // the _outgoing_queue_ready, so do it here\n          if (it != _outgoing_queue.begin()) {\n              withdraw(it, ex);\n          } else {\n              if (_negotiated) {\n                  it->done.set_exception(ex);\n              }\n              break;\n          }\n      }\n      if (_negotiated) {\n          _negotiated->set_exception(ex);\n      }\n      return when_all(std::move(_outgoing_queue_ready), std::move(_sink_closed_future)).then([this] (std::tuple<future<>, future<bool>> res){\n          // _outgoing_queue_ready might be exceptional if queue drain or\n          // _negotiated abortion set it such\n          std::get<0>(res).ignore_ready_future();\n          // _sink_closed_future is never exceptional\n          bool sink_closed = std::get<1>(res).get();\n          return _connected && !sink_closed ? _write_buf.close() : make_ready_future();\n      });\n  }\n\n  void connection::set_socket(connected_socket&& fd) {\n      if (_connected) {\n          throw std::runtime_error(\"already connected\");\n      }\n      _fd = std::move(fd);\n      _read_buf =_fd.input();\n      _write_buf = _fd.output();\n      _connected = true;\n  }\n\n  future<> connection::send_negotiation_frame(feature_map features) {\n      auto negotiation_frame_feature_record_size = [] (const feature_map::value_type& e) {\n          return 8 + e.second.size();\n      };\n      auto extra_len = boost::accumulate(\n              features | boost::adaptors::transformed(negotiation_frame_feature_record_size),\n              uint32_t(0));\n      temporary_buffer<char> reply(sizeof(negotiation_frame) + extra_len);\n      auto p = reply.get_write();\n      p = std::copy_n(rpc_magic, 8, p);\n      write_le<uint32_t>(p, extra_len);\n      p += 4;\n      for (auto&& e : features) {\n          write_le<uint32_t>(p, static_cast<uint32_t>(e.first));\n          p += 4;\n          write_le<uint32_t>(p, e.second.size());\n          p += 4;\n          p = std::copy_n(e.second.begin(), e.second.size(), p);\n      }\n      return _write_buf.write(std::move(reply)).then([this] {\n          _stats.sent_messages++;\n          return _write_buf.flush();\n      });\n  }\n\n  void connection::withdraw(outgoing_entry::container_t::iterator it, std::exception_ptr ex) {\n      assert(it != _outgoing_queue.end());\n\n      auto pit = std::prev(it);\n      // Previous entry's (pit's) done future will schedule current entry (it)\n      // continuation. Similarly, it.done will schedule next entry continuation\n      // or will resolve _outgoing_queue_ready future.\n      //\n      // To withdraw \"it\" we need to do two things:\n      // - make pit.done resolve it->next (some time later)\n      // - resolve \"it\"'s continuation right now\n      //\n      // The latter is achieved by resolving pit.done immediatelly, the former\n      // by moving it.done into pit.done. For simplicity (verging on obscurity?)\n      // both done's are just swapped and \"it\" resolves its new promise\n\n      std::swap(it->done, pit->done);\n      it->uncancellable();\n      it->unlink();\n      if (ex == nullptr) {\n          it->done.set_value();\n      } else {\n          it->done.set_exception(ex);\n      }\n  }\n\n  future<> connection::send(snd_buf buf, std::optional<rpc_clock_type::time_point> timeout, cancellable* cancel) {\n      if (!_error) {\n          if (timeout && *timeout <= rpc_clock_type::now()) {\n              return make_ready_future<>();\n          }\n\n          auto p = std::make_unique<outgoing_entry>(std::move(buf));\n          auto& d = *p;\n          _outgoing_queue.push_back(d);\n          _outgoing_queue_size++;\n          auto deleter = [this, it = _outgoing_queue.iterator_to(d)] {\n              // Front entry is most likely (unless _negotiated is unresolved, check enqueue_zero_frame()) sitting\n              // inside send_entry() continuations and thus it cannot be cancelled.\n              if (it != _outgoing_queue.begin()) {\n                  withdraw(it);\n              }\n          };\n\n          if (timeout) {\n              auto& t = d.t;\n              t.set_callback(deleter);\n              t.arm(timeout.value());\n          }\n          if (cancel) {\n              cancel->cancel_send = std::move(deleter);\n              cancel->send_back_pointer = &d.pcancel;\n              d.pcancel = cancel;\n          }\n\n          // New entry should continue (do its .then() lambda) after _outgoing_queue_ready\n          // resolves. Next entry will need to do the same after this entry's done resolves.\n          // Thus -- replace _outgoing_queue_ready with d's future and chain its continuation\n          // on ..._ready's old value.\n          return std::exchange(_outgoing_queue_ready, d.done.get_future()).then([this, p = std::move(p)] () mutable {\n              _outgoing_queue_size--;\n              if (__builtin_expect(!p->is_linked(), false)) {\n                  // If withdrawn the entry is unlinked and this lambda is fired right at once\n                  return make_ready_future<>();\n              }\n\n              p->uncancellable();\n              return send_entry(*p).then_wrapped([this, p = std::move(p)] (auto f) mutable {\n                  if (f.failed()) {\n                      f.ignore_ready_future();\n                      abort();\n                  }\n                  p->done.set_value();\n              });\n          });\n      } else {\n          return make_exception_future<>(closed_error());\n      }\n  }\n\n  void connection::abort() {\n      if (!_error) {\n          _error = true;\n          _fd.shutdown_input();\n      }\n  }\n\n  future<> connection::stop() noexcept {\n      try {\n          abort();\n      } catch (...) {\n          log_exception(*this, log_level::error, \"fail to shutdown connection while stopping\", std::current_exception());\n      }\n      return _stopped.get_future();\n  }\n\n  template<typename Connection>\n  static bool verify_frame(Connection& c, temporary_buffer<char>& buf, size_t expected, const char* log) {\n      if (buf.size() != expected) {\n          if (buf.size() != 0) {\n              c.get_logger()(c.peer_address(), log);\n          }\n          return false;\n      }\n      return true;\n  }\n\n  template<typename Connection>\n  static\n  future<feature_map>\n  receive_negotiation_frame(Connection& c, input_stream<char>& in) {\n      return in.read_exactly(sizeof(negotiation_frame)).then([&c, &in] (temporary_buffer<char> neg) {\n          if (!verify_frame(c, neg, sizeof(negotiation_frame), \"unexpected eof during negotiation frame\")) {\n              return make_exception_future<feature_map>(closed_error());\n          }\n          negotiation_frame frame;\n          std::copy_n(neg.get_write(), sizeof(frame.magic), frame.magic);\n          frame.len = read_le<uint32_t>(neg.get_write() + 8);\n          if (std::memcmp(frame.magic, rpc_magic, sizeof(frame.magic)) != 0) {\n              c.get_logger()(c.peer_address(), format(\"wrong protocol magic: {:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}\",\n                    frame.magic[0], frame.magic[1], frame.magic[2], frame.magic[3], frame.magic[4], frame.magic[5], frame.magic[6], frame.magic[7]));\n              return make_exception_future<feature_map>(closed_error());\n          }\n          auto len = frame.len;\n          return in.read_exactly(len).then([&c, len] (temporary_buffer<char> extra) {\n              if (extra.size() != len) {\n                  c.get_logger()(c.peer_address(), \"unexpected eof during negotiation frame\");\n                  return make_exception_future<feature_map>(closed_error());\n              }\n              feature_map map;\n              auto p = extra.get();\n              auto end = p + extra.size();\n              while (p != end) {\n                  if (end - p < 8) {\n                      c.get_logger()(c.peer_address(), \"bad feature data format in negotiation frame\");\n                      return make_exception_future<feature_map>(closed_error());\n                  }\n                  auto feature = static_cast<protocol_features>(read_le<uint32_t>(p));\n                  auto f_len = read_le<uint32_t>(p + 4);\n                  p += 8;\n                  if (f_len > end - p) {\n                      c.get_logger()(c.peer_address(), \"buffer underflow in feature data in negotiation frame\");\n                      return make_exception_future<feature_map>(closed_error());\n                  }\n                  auto data = sstring(p, f_len);\n                  p += f_len;\n                  map.emplace(feature, std::move(data));\n              }\n              return make_ready_future<feature_map>(std::move(map));\n          });\n      });\n  }\n\n  inline future<rcv_buf>\n  read_rcv_buf(input_stream<char>& in, uint32_t size) {\n      return in.read_up_to(size).then([&, size] (temporary_buffer<char> data) mutable {\n          rcv_buf rb(size);\n          if (data.size() == 0) {\n              return make_ready_future<rcv_buf>(rcv_buf());\n          } else if (data.size() == size) {\n              rb.bufs = std::move(data);\n              return make_ready_future<rcv_buf>(std::move(rb));\n          } else {\n              size -= data.size();\n              std::vector<temporary_buffer<char>> v;\n              v.push_back(std::move(data));\n              rb.bufs = std::move(v);\n              return do_with(std::move(rb), std::move(size), [&in] (rcv_buf& rb, uint32_t& left) {\n                  return repeat([&] () {\n                      return in.read_up_to(left).then([&] (temporary_buffer<char> data) {\n                          if (!data.size()) {\n                              rb.size -= left;\n                              return stop_iteration::yes;\n                          } else {\n                              left -= data.size();\n                              std::get<std::vector<temporary_buffer<char>>>(rb.bufs).push_back(std::move(data));\n                              return left ? stop_iteration::no : stop_iteration::yes;\n                          }\n                      });\n                  }).then([&rb] {\n                      return std::move(rb);\n                  });\n              });\n          }\n      });\n  }\n\n  template<typename FrameType>\n  future<typename FrameType::return_type>\n  connection::read_frame(socket_address info, input_stream<char>& in) {\n      auto header_size = FrameType::header_size();\n      return in.read_exactly(header_size).then([this, header_size, info, &in] (temporary_buffer<char> header) {\n          if (header.size() != header_size) {\n              if (header.size() != 0) {\n                  _logger(info, format(\"unexpected eof on a {} while reading header: expected {:d} got {:d}\", FrameType::role(), header_size, header.size()));\n              }\n              return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n          }\n          auto [size, h] = FrameType::decode_header(header.get());\n          if (!size) {\n              return make_ready_future<typename FrameType::return_type>(FrameType::make_value(h, rcv_buf()));\n          } else {\n              return read_rcv_buf(in, size).then([this, info, h = std::move(h), size] (rcv_buf rb) {\n                  if (rb.size != size) {\n                      _logger(info, format(\"unexpected eof on a {} while reading data: expected {:d} got {:d}\", FrameType::role(), size, rb.size));\n                      return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n                  } else {\n                      return make_ready_future<typename FrameType::return_type>(FrameType::make_value(h, std::move(rb)));\n                  }\n              });\n          }\n      });\n  }\n\n  template<typename FrameType>\n  future<typename FrameType::return_type>\n  connection::read_frame_compressed(socket_address info, std::unique_ptr<compressor>& compressor, input_stream<char>& in) {\n      if (compressor) {\n          return in.read_exactly(4).then([this, info, &in, &compressor] (temporary_buffer<char> compress_header) {\n              if (compress_header.size() != 4) {\n                  if (compress_header.size() != 0) {\n                      _logger(info, format(\"unexpected eof on a {} while reading compression header: expected 4 got {:d}\", FrameType::role(), compress_header.size()));\n                  }\n                  return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n              }\n              auto ptr = compress_header.get();\n              auto size = read_le<uint32_t>(ptr);\n              return read_rcv_buf(in, size).then([this, size, &compressor, info, &in] (rcv_buf compressed_data) {\n                  if (compressed_data.size != size) {\n                      _logger(info, format(\"unexpected eof on a {} while reading compressed data: expected {:d} got {:d}\", FrameType::role(), size, compressed_data.size));\n                      return make_ready_future<typename FrameType::return_type>(FrameType::empty_value());\n                  }\n                  auto eb = compressor->decompress(std::move(compressed_data));\n                  if (eb.size == 0) {\n                      // Empty frames might be sent as means of communication between the compressors, and should be skipped by the RPC layer.\n                      // We skip the empty frame here. We recursively restart the function, as if the empty frame didn't happen.\n                      // The yield() is here to limit the stack depth of the recursion to 1.\n                      return yield().then([this, info, &in, &compressor] { return read_frame_compressed<FrameType>(info, compressor, in); });\n                  }\n                  net::packet p;\n                  auto* one = std::get_if<temporary_buffer<char>>(&eb.bufs);\n                  if (one) {\n                      p = net::packet(std::move(p), std::move(*one));\n                  } else {\n                      auto&& bufs = std::get<std::vector<temporary_buffer<char>>>(eb.bufs);\n                      p.reserve(bufs.size());\n                      for (auto&& b : bufs) {\n                          p = net::packet(std::move(p), std::move(b));\n                      }\n                  }\n                  return do_with(as_input_stream(std::move(p)), [this, info] (input_stream<char>& in) {\n                      return read_frame<FrameType>(info, in);\n                  });\n              });\n          });\n      } else {\n          return read_frame<FrameType>(info, in);\n      }\n  }\n\n  struct stream_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = opt_buf_type;\n      struct header_type {\n          bool eos;\n      };\n      static size_t header_size() {\n          return 4;\n      }\n      static const char* role() {\n          return \"stream\";\n      }\n      static auto empty_value() {\n          return std::nullopt;\n      }\n      static std::pair<uint32_t, header_type> decode_header(const char* ptr) {\n          auto size = read_le<uint32_t>(ptr);\n          return size != -1U ? std::make_pair(size, header_type{false}) : std::make_pair(0U, header_type{true});\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          if (t.eos) {\n              data.size = -1U;\n          }\n          return data;\n      }\n  };\n\n  future<std::optional<rcv_buf>>\n  connection::read_stream_frame_compressed(input_stream<char>& in) {\n      return read_frame_compressed<stream_frame>(peer_address(), _compressor, in);\n  }\n\n  future<> connection::stream_close() {\n      auto f = make_ready_future<>();\n      if (!error()) {\n          promise<bool> p;\n          _sink_closed_future = p.get_future();\n          // stop_send_loop(), which also calls _write_buf.close(), and this code can run in parallel.\n          // Use _sink_closed_future to serialize them and skip second call to close()\n          f = _write_buf.close().finally([p = std::move(p)] () mutable { p.set_value(true);});\n      }\n      return f.finally([this] () mutable { return stop(); });\n  }\n\n  future<> connection::stream_process_incoming(rcv_buf&& buf) {\n      // we do not want to dead lock on huge packets, so let them in\n      // but only one at a time\n      auto size = std::min(size_t(buf.size), max_stream_buffers_memory);\n      return get_units(_stream_sem, size).then([this, buf = std::move(buf)] (semaphore_units<>&& su) mutable {\n          buf.su = std::move(su);\n          return _stream_queue.push_eventually(std::move(buf));\n      });\n  }\n\n  future<> connection::handle_stream_frame() {\n      return read_stream_frame_compressed(_read_buf).then([this] (std::optional<rcv_buf> data) {\n          if (!data) {\n              _error = true;\n              return make_ready_future<>();\n          }\n          return stream_process_incoming(std::move(*data));\n      });\n  }\n\n  future<> connection::stream_receive(circular_buffer<foreign_ptr<std::unique_ptr<rcv_buf>>>& bufs) {\n      return _stream_queue.not_empty().then([this, &bufs] {\n          bool eof = !_stream_queue.consume([&bufs] (rcv_buf&& b) {\n              if (b.size == -1U) { // max fragment length marks an end of a stream\n                  return false;\n              } else {\n                  bufs.push_back(make_foreign(std::make_unique<rcv_buf>(std::move(b))));\n                  return true;\n              }\n          });\n          if (eof && !bufs.empty()) {\n              assert(_stream_queue.empty());\n              _stream_queue.push(rcv_buf(-1U)); // push eof marker back for next read to notice it\n          }\n      });\n  }\n\n  void connection::register_stream(connection_id id, xshard_connection_ptr c) {\n      _streams.emplace(id, std::move(c));\n  }\n\n  xshard_connection_ptr connection::get_stream(connection_id id) const {\n      auto it = _streams.find(id);\n      if (it == _streams.end()) {\n          throw std::logic_error(format(\"rpc stream id {} not found\", id).c_str());\n      }\n      return it->second;\n  }\n\n  // The request frame is\n  //   le64 optional timeout (see request_frame_with_timeout below)\n  //   le64 message type a.k.a. verb ID\n  //   le64 message ID\n  //   le32 payload length\n  //   ...  payload\n  struct request_frame {\n      using opt_buf_type = std::optional<rcv_buf>;\n      using return_type = std::tuple<std::optional<uint64_t>, uint64_t, int64_t, opt_buf_type>;\n      using header_type = std::tuple<std::optional<uint64_t>, uint64_t, int64_t>;\n      static constexpr size_t raw_header_size = sizeof(uint64_t) + sizeof(int64_t) + sizeof(uint32_t);\n      static size_t header_size() {\n          static_assert(request_frame_headroom >= raw_header_size);\n          return raw_header_size;\n      }\n      static const char* role() {\n          return \"server\";\n      }\n      static auto empty_value() {\n          return std::make_tuple(std::nullopt, uint64_t(0), 0, std::nullopt);\n      }\n      static std::pair<size_t, header_type> decode_header(const char* ptr) {\n          auto type = read_le<uint64_t>(ptr);\n          auto msgid = read_le<int64_t>(ptr + 8);\n          auto size = read_le<uint32_t>(ptr + 16);\n          return std::make_pair(size, std::make_tuple(std::nullopt, type, msgid));\n      }\n      static void encode_header(uint64_t type, int64_t msg_id, snd_buf& buf, size_t off) {\n          auto p = buf.front().get_write() + off;\n          write_le<uint64_t>(p, type);\n          write_le<int64_t>(p + 8, msg_id);\n          write_le<uint32_t>(p + 16, buf.size - raw_header_size - off);\n      }\n      static auto make_value(const header_type& t, rcv_buf data) {\n          return std::make_tuple(std::get<0>(t), std::get<1>(t), std::get<2>(t), std::move(data));\n      }\n  };\n\n  // This frame is used if protocol_features.TIMEOUT was negotiated\n...\n// Path: src/rpc/lz4_compressor.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2016 Scylladb, Ltd.\n */\n\n#include <seastar/rpc/lz4_compressor.hh>\n#include <seastar/core/byteorder.hh>\n#include <lz4.h>\n\nnamespace seastar {\n\nnamespace rpc {\n\nsstring lz4_compressor::name() const {\n    return factory{}.supported();\n}\n\nconst sstring& lz4_compressor::factory::supported() const {\n    const static sstring name = \"LZ4\";\n    return name;\n}\n\nstd::unique_ptr<rpc::compressor> lz4_compressor::factory::negotiate(sstring feature, bool is_server) const {\n    return feature == supported() ? std::make_unique<lz4_compressor>() : nullptr;\n}\n\n// Reusable contiguous buffers needed for LZ4 compression and decompression functions.\nclass reusable_buffer {\n    static constexpr size_t chunk_size = 128 * 1024;\n    static_assert(snd_buf::chunk_size == chunk_size, \"snd_buf::chunk_size == chunk_size\");\n\n    std::unique_ptr<char[]> _data;\n    size_t _size;\nprivate:\n    void reserve(size_t n) {\n        if (_size < n) {\n            _data.reset();\n            // Not using std::make_unique to avoid value-initialisation.\n            _data = std::unique_ptr<char[]>(new char[n]);\n            _size = n;\n        }\n    }\npublic:\n    // Returns a pointer to a contiguous buffer containing all data stored in input.\n    // The pointer remains valid until next call to this.\n    const char* prepare(const std::variant<std::vector<temporary_buffer<char>>, temporary_buffer<char>>& input, size_t size) {\n        if (const auto single = std::get_if<temporary_buffer<char>>(&input)) {\n            return single->get();\n        }\n        reserve(size);\n        auto dst = _data.get();\n        for (const auto& fragment : std::get<std::vector<temporary_buffer<char>>>(input)) {\n            dst = std::copy_n(fragment.begin(), fragment.size(), dst);\n        }\n        return _data.get();\n    }\n\n    // Calls function fn passing to it a pointer to a temporary contigiuous max_size\n    // buffer.\n    // fn is supposed to return the actual size of the data.\n    // with_reserved() returns an Output object (snd_buf or rcv_buf or compatible),\n    // containing data that was written to the temporary buffer.\n    // Output should be either snd_buf or rcv_buf.\n    template<typename Output, typename Function>\n    requires requires (Function fn, char* ptr) {\n        { fn(ptr) } -> std::convertible_to<size_t>;\n    } && (std::is_same_v<Output, snd_buf> || std::is_same_v<Output, rcv_buf>)\n    Output with_reserved(size_t max_size, Function&& fn) {\n        if (max_size <= chunk_size) {\n            auto dst = temporary_buffer<char>(max_size);\n            size_t dst_size = fn(dst.get_write());\n            dst.trim(dst_size);\n            return Output(std::move(dst));\n        }\n\n        reserve(max_size);\n        size_t dst_size = fn(_data.get());\n        if (dst_size <= chunk_size) {\n            return Output(temporary_buffer<char>(_data.get(), dst_size));\n        }\n\n        auto left = dst_size;\n        auto pos = _data.get();\n        std::vector<temporary_buffer<char>> buffers;\n        while (left) {\n            auto this_size = std::min(left, chunk_size);\n            buffers.emplace_back(this_size);\n            std::copy_n(pos, this_size, buffers.back().get_write());\n            pos += this_size;\n            left -= this_size;\n        }\n        return Output(std::move(buffers), dst_size);\n    }\n\n    void clear() noexcept {\n        _data.reset();\n        _size = 0;\n    }\n};\n\nstatic thread_local reusable_buffer reusable_buffer_compressed_data;\nstatic thread_local reusable_buffer reusable_buffer_decompressed_data;\nstatic thread_local size_t buffer_use_count = 0;\nstatic constexpr size_t drop_buffers_trigger = 100'000;\n\nstatic void after_buffer_use() noexcept {\n    if (buffer_use_count++ == drop_buffers_trigger) {\n        reusable_buffer_compressed_data.clear();\n        reusable_buffer_decompressed_data.clear();\n        buffer_use_count = 0;\n    }\n}\n\nsnd_buf lz4_compressor::compress(size_t head_space, snd_buf data) {\n    head_space += 4;\n    auto dst_size = head_space + LZ4_compressBound(data.size);\n    auto dst = reusable_buffer_compressed_data.with_reserved<snd_buf>(dst_size, [&] (char* dst) {\n        auto src_size = data.size;\n        auto src = reusable_buffer_decompressed_data.prepare(data.bufs, data.size);\n\n        auto size = LZ4_compress_default(src, dst + head_space, src_size, LZ4_compressBound(src_size));\n        if (size == 0) {\n            throw std::runtime_error(\"RPC frame LZ4 compression failure\");\n        }\n        write_le<uint32_t>(dst + (head_space - 4), src_size);\n        return size + head_space;\n    });\n    after_buffer_use();\n    return dst;\n}\n\nrcv_buf lz4_compressor::decompress(rcv_buf data) {\n    if (data.size < 4) {\n        return rcv_buf();\n    } else {\n        auto src_size = data.size;\n        auto src = reusable_buffer_decompressed_data.prepare(data.bufs, data.size);\n\n        auto dst_size = read_le<uint32_t>(src);\n        if (!dst_size) {\n            throw std::runtime_error(\"RPC frame LZ4 decompression failure: decompressed size cannot be zero\");\n        }\n        src += sizeof(uint32_t);\n        src_size -= sizeof(uint32_t);\n\n        auto dst = reusable_buffer_compressed_data.with_reserved<rcv_buf>(dst_size, [&] (char* dst) {\n            if (LZ4_decompress_safe(src, dst, src_size, dst_size) < 0) {\n                throw std::runtime_error(\"RPC frame LZ4 decompression failure\");\n            }\n            return dst_size;\n        });\n        after_buffer_use();\n        return dst;\n    }\n}\n\n}\n\n}\n\n// Path: src/rpc/lz4_fragmented_compressor.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright (C) 2019 Scylladb, Ltd.\n */\n\n#include <seastar/rpc/lz4_fragmented_compressor.hh>\n#include <seastar/core/byteorder.hh>\n\n#include <lz4.h>\n// LZ4_DECODER_RING_BUFFER_SIZE macro is introduced since v1.8.2\n// To work with previous lz4 release, copied the definition in lz4 here\n#ifndef LZ4_DECODER_RING_BUFFER_SIZE\n#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize))\n#endif\n\nnamespace seastar {\nnamespace rpc {\n\nsstring lz4_fragmented_compressor::name() const {\n    return factory{}.supported();\n}\n\nconst sstring& lz4_fragmented_compressor::factory::supported() const {\n    const static sstring name = \"LZ4_FRAGMENTED\";\n    return name;\n}\n\nstd::unique_ptr<rpc::compressor> lz4_fragmented_compressor::factory::negotiate(sstring feature, bool is_server) const {\n    return feature == supported() ? std::make_unique<lz4_fragmented_compressor>() : nullptr;\n}\n\n// Compressed message format:\n// The message consists of one or more data chunks each preceeded by a 4 byte header.\n// The value of the header detrmines the size of the chunk:\n// - most significant bit is cleared: intermediate chunk, 31 least significant bits\n//   contain the compressed size of the chunk (i.e. how it appears on wire), the\n//   decompressed size is 32 kB.\n// - most significant bit is set: last chunk, 31 least significant bits contain the\n//   decompressed size of the chunk, the compressed size is the remaining part of\n//   the message.\n// Compression and decompression is done using LZ4 streaming interface. Each chunk\n// depends on the one that precedes it.\n// All metadata is little-endian.\n\nstatic constexpr uint32_t last_chunk_flag = uint32_t(1) << 31;\nstatic constexpr size_t chunk_header_size = sizeof(uint32_t);\nstatic constexpr size_t chunk_size = 32 * 1024;\n\nnamespace {\n\nstruct compression_stream_deleter {\n    void operator()(LZ4_stream_t* stream) const noexcept {\n        LZ4_freeStream(stream);\n    }\n};\n\nstruct decompression_stream_deleter {\n    void operator()(LZ4_streamDecode_t* stream) const noexcept {\n        LZ4_freeStreamDecode(stream);\n    }\n};\n\n}\n\nsnd_buf lz4_fragmented_compressor::compress(size_t head_space, snd_buf data) {\n    static thread_local auto stream = std::unique_ptr<LZ4_stream_t, compression_stream_deleter>(LZ4_createStream());\n    static_assert(chunk_size <= snd_buf::chunk_size, \"chunk_size <= snd_buf::chunk_size\");\n\n    LZ4_resetStream(stream.get());\n\n    auto size_left = data.size;\n    auto src = std::get_if<temporary_buffer<char>>(&data.bufs);\n    if (!src) {\n        src = std::get<std::vector<temporary_buffer<char>>>(data.bufs).data();\n    }\n\n    auto single_chunk_size = LZ4_COMPRESSBOUND(size_left) + head_space + chunk_header_size;\n    if (single_chunk_size <= chunk_size && size_left <= chunk_size && src->size() == size_left) {\n        // faster path for small messages\n        auto dst = temporary_buffer<char>(single_chunk_size);\n        auto header = dst.get_write() + head_space;\n        auto compressed_data = header + chunk_header_size;\n        auto compressed_size = LZ4_compress_fast_continue(stream.get(), src->get(), compressed_data, size_left, LZ4_COMPRESSBOUND(size_left), 0);\n        write_le(header, last_chunk_flag | size_left);\n        dst.trim(head_space + chunk_header_size + compressed_size);\n        return snd_buf(std::move(dst));\n    }\n\n    static constexpr size_t chunk_compress_bound = LZ4_COMPRESSBOUND(chunk_size);\n    static constexpr size_t chunk_maximum_compressed_size = chunk_compress_bound + chunk_header_size;\n    static_assert(chunk_maximum_compressed_size < snd_buf::chunk_size, \"chunk_maximum_compressed_size is too large\");\n\n    std::vector<temporary_buffer<char>> dst_buffers;\n    size_t dst_offset = head_space;\n\n    dst_buffers.emplace_back(std::max<size_t>(head_space, snd_buf::chunk_size));\n\n    // Intermediate chunks\n    size_t total_compressed_size = 0;\n    auto src_left = data.size;\n    size_t src_current_offset = 0;\n\n    // Advance offset in the current source fragment, move to the next fragment if needed.\n    auto advance_src = [&] (size_t n) {\n        src_current_offset += n;\n        if (src_current_offset >= src->size()) {\n            ++src;\n            src_current_offset = 0;\n        }\n        src_left -= n;\n    };\n\n    // Input chunks do not have to be multiplies of chunk_size.\n    // We handle such cases by reassembling a chunk in this temporary buffer.\n    // Note that this is similar to the ring buffer compression case in docs, \n    // we need to ensure that a suitable amount of (maybe) previous data is \n    // stable in this or input buffer, thus make the temp buffer \n    // LZ4_DECODER_RING_BUFFER_SIZE(chunk_size) large, and treat it as a ring.\n    static constexpr auto lin_buf_size = LZ4_DECODER_RING_BUFFER_SIZE(chunk_size);\n    static thread_local char temporary_chunk_data[lin_buf_size];\n    size_t lin_off = 0;\n\n    auto maybe_linearize = [&](size_t size) {\n        auto src_ptr = src->get() + src_current_offset;\n        if (src->size() - src_current_offset < size) {\n            auto left = size;\n            assert(lin_buf_size > size);\n            if (lin_buf_size - lin_off < size) {\n                lin_off = 0;\n            }\n            auto tmp = temporary_chunk_data + std::exchange(lin_off, lin_off + size);\n            src_ptr = tmp;\n            while (left) {\n                auto this_size = std::min(src->size() - src_current_offset, left);\n                tmp = std::copy_n(src->get() + src_current_offset, this_size, tmp);\n                left -= this_size;\n                advance_src(this_size);\n            }\n        } else {\n            advance_src(chunk_size);\n            lin_off = 0;\n        }\n        return src_ptr;\n    };\n\n    while (src_left > chunk_size) {\n        // Check if we can fit another chunk in the current destination fragment.\n        // If not allocate a new one.\n        if (dst_offset + chunk_maximum_compressed_size > dst_buffers.back().size()) {\n            dst_buffers.back().trim(dst_offset);\n            dst_buffers.emplace_back(snd_buf::chunk_size);\n            dst_offset = 0;\n        }\n\n        // Check if there is at least a contiguous chunk_size of data in the current\n        // source fragment. If not, linearise that into temporary_chunk_data.\n        auto src_ptr = maybe_linearize(chunk_size);\n        auto header = dst_buffers.back().get_write() + dst_offset;\n        auto dst = header + chunk_header_size;\n\n        auto compressed_size = LZ4_compress_fast_continue(stream.get(), src_ptr, dst, chunk_size, chunk_compress_bound, 0);\n        total_compressed_size += compressed_size + chunk_header_size;\n\n        dst_offset += compressed_size + chunk_header_size;\n        write_le<uint32_t>(header, compressed_size);\n    }\n\n    // Last chunk\n    auto last_chunk_compress_bound = LZ4_COMPRESSBOUND(src_left);\n    auto last_chunk_maximum_compressed_size = last_chunk_compress_bound + chunk_header_size;\n\n    // Check if we can fit the last chunk in the current destination fragment. Allocate a new one if not.\n    if (dst_offset + last_chunk_maximum_compressed_size > dst_buffers.back().size()) {\n        dst_buffers.back().trim(dst_offset);\n        dst_buffers.emplace_back(snd_buf::chunk_size);\n        dst_offset = 0;\n    }\n    auto header = dst_buffers.back().get_write() + dst_offset;\n    auto dst = header + chunk_header_size;\n\n    // Check if all remaining source data is contiguous. If not linearise it into temporary_chunk_data.\n    auto rem = src_left;\n    auto src_ptr = maybe_linearize(src_left);\n\n    auto compressed_size = LZ4_compress_fast_continue(stream.get(), src_ptr, dst, rem, last_chunk_compress_bound, 0);\n    dst_offset += compressed_size + chunk_header_size;\n    write_le<uint32_t>(header, last_chunk_flag | rem);\n    total_compressed_size += compressed_size + chunk_header_size + head_space;\n\n    auto& last = dst_buffers.back();\n    last.trim(dst_offset);\n\n    if (dst_buffers.size() == 1) {\n        return snd_buf(std::move(dst_buffers.front()));\n    }\n    return snd_buf(std::move(dst_buffers), total_compressed_size);\n}\n\nrcv_buf lz4_fragmented_compressor::decompress(rcv_buf data) {\n    if (data.size < 4) {\n        return rcv_buf();\n    }\n\n    static thread_local auto stream = std::unique_ptr<LZ4_streamDecode_t, decompression_stream_deleter>(LZ4_createStreamDecode());\n\n    if (!LZ4_setStreamDecode(stream.get(), nullptr, 0)) {\n        throw std::runtime_error(\"RPC frame LZ4_FRAGMENTED decompression failed to reset state\");\n    }\n\n    auto src = std::get_if<temporary_buffer<char>>(&data.bufs);\n    size_t src_left = data.size;\n    size_t src_offset = 0;\n\n    // Prepare source data. Returns pointer to n contiguous bytes of source data.\n    // Avoids copy if possible, otherwise uses dst as a temporary storage.\n    auto copy_src = [&] (char* dst, size_t n) -> const char* {\n        // Fast path, no need to copy anything.\n        if (src->size() - src_offset >= n) {\n            auto ptr = src->get() + src_offset;\n            src_left -= n;\n            src_offset += n;\n            return ptr;\n        }\n\n        // Need to linearise source chunk into dst.\n        auto ptr = dst;\n        src_left -= n;\n        while (n) {\n            if (src_offset == src->size()) {\n                ++src;\n                src_offset = 0;\n            }\n            auto this_size = std::min(n, src->size() - src_offset);\n            std::copy_n(src->get() + src_offset, this_size, dst);\n            n -= this_size;\n            dst += this_size;\n            src_offset += this_size;\n        }\n        return ptr;\n    };\n\n    // Read, possibly fragmented, header.\n    auto read_header = [&] {\n        uint32_t header_value;\n        auto ptr = copy_src(reinterpret_cast<char*>(&header_value), chunk_header_size);\n        if (ptr != reinterpret_cast<char*>(&header_value)) {\n            std::copy_n(ptr, sizeof(uint32_t), reinterpret_cast<char*>(&header_value));\n        }\n        return le_to_cpu(header_value);\n    };\n\n    if (src) {\n        auto header = read_le<uint32_t>(src->get());\n        if (header & last_chunk_flag) {\n            // faster path for small messages: single chunk in a single buffer\n            header &= ~last_chunk_flag;\n            src_offset += chunk_header_size;\n            src_left -= chunk_header_size;\n            auto dst = temporary_buffer<char>(header);\n            if (LZ4_decompress_safe_continue(stream.get(), src->get() + src_offset, dst.get_write(), src_left, header) < 0) {\n                throw std::runtime_error(\"RPC frame LZ4_FRAGMENTED decompression failure (short)\");\n            }\n            return rcv_buf(std::move(dst));\n        }\n        // not eligible for fast path: multiple chunks in a single buffer\n    } else {\n        // not eligible for fast path: multiple buffers\n        src = std::get<std::vector<temporary_buffer<char>>>(data.bufs).data();\n    }\n\n    // Let's be a bit paranoid and not assume that the remote has the same\n    // LZ4_COMPRESSBOUND as we do and allow any compressed chunk size.\n    static thread_local auto chunk_buffer = temporary_buffer<char>(LZ4_COMPRESSBOUND(chunk_size));\n\n    std::vector<temporary_buffer<char>> dst_buffers;\n    size_t total_size = 0;\n\n    // Decompressing requires either dest to be fully split or\n    // \"preserved\" in 64KB or larger, depending on how it was\n    // compressed. If not, decompression will fail, typically\n    // on text-like constructs. Making our dest buffers 64K\n    // ensures we retain a suitable dictionary region for all\n    // passes. \n    constexpr auto buf_size = 64 * 1024;\n    size_t dst_offset = 0;\n\n    auto get_dest = [&](size_t size) {\n        if (dst_buffers.empty()) {\n            dst_buffers.emplace_back(buf_size);\n        }\n        if (dst_buffers.back().size() - dst_offset < size) {\n            dst_buffers.back().trim(dst_offset);\n            dst_buffers.emplace_back(buf_size);\n            dst_offset = 0;\n        }\n        return dst_buffers.back().get_write() + std::exchange(dst_offset, dst_offset + size);\n    };\n\n    // Intermediate chunks\n    uint32_t header_value = read_header();\n    while (!(header_value & last_chunk_flag)) {\n        total_size += chunk_size;\n        if (chunk_buffer.size() < header_value) {\n            chunk_buffer = temporary_buffer<char>(header_value);\n        }\n        auto src_ptr = copy_src(chunk_buffer.get_write(), header_value);\n        auto dst = get_dest(chunk_size);\n        if (LZ4_decompress_safe_continue(stream.get(), src_ptr, /*dst_buffers.back().get_write()*/dst, header_value, chunk_size) < 0) {\n            throw std::runtime_error(format(\"RPC frame LZ4_FRAGMENTED decompression failure (long, at {} bytes)\", total_size - chunk_size));\n        }\n        header_value = read_header();\n    }\n\n    // Last chunk\n    header_value &= ~last_chunk_flag;\n    total_size += header_value;\n    auto dst = get_dest(header_value);\n    if (chunk_buffer.size() < src_left) {\n        chunk_buffer = temporary_buffer<char>(src_left);\n    }\n    auto last_chunk_compressed_size = src_left;\n    auto src_ptr = copy_src(chunk_buffer.get_write(), src_left);\n    if (LZ4_decompress_safe_continue(stream.get(), src_ptr, /*dst_buffers.back().get_write()*/dst, last_chunk_compressed_size, header_value) < 0) {\n        throw std::runtime_error(format(\"RPC frame LZ4_FRAGMENTED decompression failure (long, last frame, at {} bytes)\", total_size - header_value));\n    }\n\n    dst_buffers.back().trim(dst_offset);\n\n    if (dst_buffers.size() == 1) {\n        return rcv_buf(std::move(dst_buffers.front()));\n    }\n    return rcv_buf(std::move(dst_buffers), total_size);\n}\n\n}\n}\n\n// Path: src/json/formatter.cc\n/*\n * This file is open source software, licensed to you under the terms\n * of the Apache License, Version 2.0 (the \"License\").  See the NOTICE file\n * distributed with this work for additional information regarding copyright\n * ownership.  You may not use this file except in compliance with the License.\n *\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied.  See the License for the\n * specific language governing permissions and limitations\n * under the License.\n */\n/*\n * Copyright 2015 Cloudius Systems\n */\n\n#ifdef SEASTAR_MODULE\nmodule;\n#endif\n\n#include <cmath>\n#include <algorithm>\n#include <iomanip>\n#include <ios>\n#include <sstream>\n#include <string_view>\n\n#ifdef SEASTAR_MODULE\nmodule seastar;\n#else\n#include <seastar/json/formatter.hh>\n#include <seastar/json/json_elements.hh>\n#endif\n\nnamespace seastar {\n\nusing namespace std;\n\nnamespace json {\n\nsstring formatter::begin(state s) {\n    switch (s) {\n    case state::array: return \"[\";\n    case state::map: return \"{\";\n    default: return {};\n    }\n}\n\nsstring formatter::end(state s) {\n    switch (s) {\n    case state::array: return \"]\";\n    case state::map: return \"}\";\n    default: return {};\n    }\n}\n\n\nstatic inline bool is_control_char(char c) {\n    return c >= 0 && c <= 0x1F;\n}\n\nstatic bool needs_escaping(const string_view& str) {\n    return std::any_of(str.begin(), str.end(), [] (char c) {\n        return is_control_char(c) || c == '\"' || c == '\\\\';\n    });\n}\n\nstatic sstring string_view_to_json(const string_view& str) {\n    if (!needs_escaping(str)) {\n        return format(\"\\\"{}\\\"\", str);\n    }\n\n    ostringstream oss;\n    oss << std::hex << std::uppercase << std::setfill('0');\n    oss.put('\"');\n    for (char c : str) {\n        switch (c) {\n        case '\"':\n            oss.put('\\\\').put('\"');\n            break;\n        case '\\\\':\n            oss.put('\\\\').put('\\\\');\n            break;\n        case '\\b':\n            oss.put('\\\\').put('b');\n            break;\n        case '\\f':\n            oss.put('\\\\').put('f');\n            break;\n        case '\\n':\n            oss.put('\\\\').put('n');\n            break;\n        case '\\r':\n            oss.put('\\\\').put('r');\n            break;\n        case '\\t':\n            oss.put('\\\\').put('t');\n            break;\n        default:\n            if (is_control_char(c)) {\n                oss.put('\\\\').put('u') << std::setw(4) << static_cast<int>(c);\n            } else {\n                oss.put(c);\n            }\n            break;\n        }\n    }\n    oss.put('\"');\n    return oss.str();\n}\n\nsstring formatter::to_json(const sstring& str) {\n    return string_view_to_json(str);\n}\n\nsstring formatter::to_json(const char* str) {\n    return string_view_to_json(str);\n}\n\nsstring formatter::to_json(const char* str, size_t len) {\n    return string_view_to_json(string_view{str, len});\n}\n\nsstring formatter::to_json(int n) {\n    return to_string(n);\n}\n\nsstring formatter::to_json(unsigned n) {\n    return to_string(n);\n}\n\nsstring formatter::to_json(long n) {\n    return to_string(n);\n}\n\nsstring formatter::to_json(float f) {\n    if (std::isinf(f)) {\n        throw out_of_range(\"Infinite float value is not supported\");\n    } else if (std::isnan(f)) {\n        throw invalid_argument(\"Invalid float value\");\n    }\n    return to_sstring(f);\n}\n\nsstring formatter::to_json(double d) {\n    if (std::isinf(d)) {\n        throw out_of_range(\"Infinite double value is not supported\");\n    } else if (std::isnan(d)) {\n        throw invalid_argument(\"Invalid double value\");\n    }\n    return to_sstring(d);\n}\n\n...\n", "needle_token_start": 15350, "needle_token_end": 15381, "code_context_ntokens": 16167, "output": ["```cpp\nstatic inline bool is_control_char(char c) {\n    return c >= 0 && c <= 0x1F;\n}\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "packVal", "language": "cpp", "path": "lib/executor/engine/refInstr.cpp", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to conditionally modify a value based on its type, specifically for types that require packing into smaller bit widths.\n2. **Input**: It takes two parameters: a type descriptor indicating the data type and the value to be potentially modified.\n3. **Output**: It returns a modified value if the type requires packing, otherwise it returns the original value unchanged.\n4. **Procedure**: The function first checks if the type descriptor indicates a packing requirement. If so, it checks the specific type code and applies a bitwise operation to limit the value to the appropriate size (8 or 16 bits). If no packing is required, the original value is returned as is.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/executor/engine/engine.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <array>\n#include <cstdint>\n#include <cstring>\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void> Executor::runExpression(Runtime::StackManager &StackMgr,\n                                     AST::InstrView Instrs) {\n  return execute(StackMgr, Instrs.begin(), Instrs.end());\n}\n\nExpect<void>\nExecutor::runFunction(Runtime::StackManager &StackMgr,\n                      const Runtime::Instance::FunctionInstance &Func,\n                      Span<const ValVariant> Params) {\n  // Set start time.\n  if (Stat && Conf.getStatisticsConfigure().isTimeMeasuring()) {\n    Stat->startRecordWasm();\n  }\n\n  // Reset and push a dummy frame into stack.\n  StackMgr.pushFrame(nullptr, AST::InstrView::iterator(), 0, 0);\n\n  // Push arguments.\n  const auto &PTypes = Func.getFuncType().getParamTypes();\n  for (uint32_t I = 0; I < Params.size(); I++) {\n    // For the references, transform to non-null reference type if the value not\n    // null.\n    if (PTypes[I].isRefType() && Params[I].get<RefVariant>().getPtr<void>() &&\n        Params[I].get<RefVariant>().getType().isNullableRefType()) {\n      auto Val = Params[I];\n      Val.get<RefVariant>().getType().toNonNullableRef();\n      StackMgr.push(Val);\n    } else {\n      StackMgr.push(Params[I]);\n    }\n  }\n\n  // Enter and execute function.\n  AST::InstrView::iterator StartIt = {};\n  Expect<void> Res = {};\n  if (auto GetIt = enterFunction(StackMgr, Func, Func.getInstrs().end())) {\n    StartIt = *GetIt;\n  } else {\n    if (GetIt.error() == ErrCode::Value::Terminated) {\n      // Handle the terminated case in entering AOT or host functions.\n      // For the terminated case, not return now to print the statistics.\n      Res = Unexpect(GetIt.error());\n    } else {\n      return Unexpect(GetIt);\n    }\n  }\n...\n// Path: lib/executor/engine/refInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\nnamespace WasmEdge {\nnamespace Executor {\n\nnamespace {\n\nValVariant packVal(const ValType &Type, const ValVariant &Val) {\n  if (Type.isPackType()) {\n    switch (Type.getCode()) {\n    case TypeCode::I8:\n      return ValVariant(Val.get<uint32_t>() & 0xFFU);\n    case TypeCode::I16:\n      return ValVariant(Val.get<uint32_t>() & 0xFFFFU);\n    default:\n      assumingUnreachable();\n    }\n  }\n  return Val;\n}\n\nValVariant unpackVal(const ValType &Type, const ValVariant &Val,\n                     bool IsSigned = false) {\n  if (Type.isPackType()) {\n    uint32_t Num = Val.get<uint32_t>();\n    switch (Type.getCode()) {\n    case TypeCode::I8:\n      if (IsSigned) {\n        return static_cast<uint32_t>(static_cast<int8_t>(Num));\n      } else {\n        return static_cast<uint32_t>(static_cast<uint8_t>(Num));\n      }\n    case TypeCode::I16:\n      if (IsSigned) {\n        return static_cast<uint32_t>(static_cast<int16_t>(Num));\n      } else {\n        return static_cast<uint32_t>(static_cast<uint16_t>(Num));\n      }\n    default:\n      assumingUnreachable();\n    }\n  }\n  return Val;\n}\n\nstd::vector<ValVariant> packVals(const ValType &Type,\n                                 std::vector<ValVariant> &&Vals) {\n  for (uint32_t I = 0; I < Vals.size(); I++) {\n    Vals[I] = packVal(Type, Vals[I]);\n  }\n  return std::move(Vals);\n}\n} // namespace\n\nExpect<void> Executor::runRefNullOp(Runtime::StackManager &StackMgr,\n                                    const ValType &Type) const noexcept {\n  // A null reference is typed with the least type in its respective hierarchy.\n  StackMgr.push(RefVariant(toBottomType(StackMgr, Type)));\n  return {};\n}\n\nExpect<void> Executor::runRefIsNullOp(ValVariant &Val) const noexcept {\n  Val.emplace<uint32_t>(Val.get<RefVariant>().isNull() ? 1U : 0U);\n  return {};\n}\n\nExpect<void> Executor::runRefFuncOp(Runtime::StackManager &StackMgr,\n                                    uint32_t Idx) const noexcept {\n  const auto *FuncInst = getFuncInstByIdx(StackMgr, Idx);\n  StackMgr.push(RefVariant(FuncInst->getDefType(), FuncInst));\n  return {};\n}\n\nExpect<void> Executor::runRefEqOp(ValVariant &Val1,\n                                  const ValVariant &Val2) const noexcept {\n  Val1.emplace<uint32_t>(Val1.get<RefVariant>().getPtr<void>() ==\n                                 Val2.get<RefVariant>().getPtr<void>()\n                             ? 1U\n                             : 0U);\n  return {};\n}\n\nExpect<void>\nExecutor::runRefAsNonNullOp(RefVariant &Ref,\n                            const AST::Instruction &Instr) const noexcept {\n  if (Ref.isNull()) {\n    spdlog::error(ErrCode::Value::CastNullToNonNull);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::CastNullToNonNull);\n  }\n  Ref.getType().toNonNullableRef();\n  return {};\n}\n\nExpect<void> Executor::runStructNewOp(Runtime::StackManager &StackMgr,\n                                      const uint32_t DefIndex,\n                                      bool IsDefault) const noexcept {\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, DefIndex)->getCompositeType();\n  uint32_t N = static_cast<uint32_t>(CompType.getFieldTypes().size());\n  std::vector<WasmEdge::ValVariant> Vals;\n  if (IsDefault) {\n    Vals.resize(N);\n    for (uint32_t I = 0; I < N; I++) {\n      const auto &VType = CompType.getFieldTypes()[I].getStorageType();\n      Vals[I] = VType.isRefType()\n                    ? ValVariant(RefVariant(toBottomType(StackMgr, VType)))\n                    : ValVariant(static_cast<uint128_t>(0));\n    }\n  } else {\n    Vals = StackMgr.pop(N);\n    for (uint32_t I = 0; I < N; I++) {\n      Vals[I] = packVal(CompType.getFieldTypes()[I].getStorageType(), Vals[I]);\n    }\n  }\n  auto *Inst =\n      const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n          ->newStruct(DefIndex, std::move(Vals));\n  StackMgr.push(RefVariant(Inst->getDefType(), Inst));\n\n  return {};\n}\n\nExpect<void> Executor::runStructGetOp(ValVariant &Val, const uint32_t Idx,\n                                      const AST::CompositeType &CompType,\n                                      const AST::Instruction &Instr,\n                                      bool IsSigned) const noexcept {\n  const auto *Inst =\n      Val.get<RefVariant>().getPtr<Runtime::Instance::StructInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullStruct);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullStruct);\n  }\n  const auto &SType = CompType.getFieldTypes()[Idx].getStorageType();\n  Val = unpackVal(SType, Inst->getField(Idx), IsSigned);\n  return {};\n}\n\nExpect<void>\nExecutor::runStructSetOp(const ValVariant &Val, const RefVariant &InstRef,\n                         const AST::CompositeType &CompType, uint32_t Idx,\n                         const AST::Instruction &Instr) const noexcept {\n  auto *Inst = InstRef.getPtr<Runtime::Instance::StructInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullStruct);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullStruct);\n  }\n  const auto &SType = CompType.getFieldTypes()[Idx].getStorageType();\n  Inst->getField(Idx) = packVal(SType, Val);\n  return {};\n}\n\nExpect<void> Executor::runArrayNewOp(Runtime::StackManager &StackMgr,\n                                     const uint32_t DefIndex, uint32_t InitCnt,\n                                     uint32_t ValCnt) const noexcept {\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  assuming(InitCnt == 0 || InitCnt == 1 || InitCnt == ValCnt);\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, DefIndex)->getCompositeType();\n  const auto &VType = CompType.getFieldTypes()[0].getStorageType();\n  if (InitCnt == 0) {\n    auto InitVal = VType.isRefType()\n                       ? ValVariant(RefVariant(toBottomType(StackMgr, VType)))\n                       : ValVariant(static_cast<uint128_t>(0));\n    auto *Inst =\n        const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n            ->newArray(DefIndex, ValCnt, InitVal);\n    StackMgr.push(RefVariant(Inst->getDefType(), Inst));\n  } else if (InitCnt == 1) {\n    auto *Inst =\n        const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n            ->newArray(DefIndex, ValCnt, packVal(VType, StackMgr.getTop()));\n    StackMgr.getTop().emplace<RefVariant>(Inst->getDefType(), Inst);\n  } else {\n    auto *Inst =\n        const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n            ->newArray(DefIndex, packVals(VType, StackMgr.pop(ValCnt)));\n    StackMgr.push(RefVariant(Inst->getDefType(), Inst));\n  }\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayNewDataOp(Runtime::StackManager &StackMgr,\n                            const Runtime::Instance::DataInstance &DataInst,\n                            const AST::Instruction &Instr) const noexcept {\n  const uint32_t N = StackMgr.pop().get<uint32_t>();\n  const uint32_t S = StackMgr.getTop().get<uint32_t>();\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, Instr.getTargetIndex())->getCompositeType();\n  const uint32_t BSize =\n      CompType.getFieldTypes()[0].getStorageType().getBitWidth() / 8;\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) * BSize >\n      DataInst.getData().size()) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N * BSize,\n        DataInst.getData().size() > 0\n            ? static_cast<uint32_t>(DataInst.getData().size() - 1)\n            : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  auto *Inst =\n      const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n          ->newArray(Instr.getTargetIndex(), N, 0U);\n  for (uint32_t Idx = 0; Idx < N; Idx++) {\n    // The value has been packed.\n    Inst->getData(Idx) = DataInst.loadValue(S + Idx * BSize, BSize);\n  }\n  StackMgr.getTop().emplace<RefVariant>(Inst->getDefType(), Inst);\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayNewElemOp(Runtime::StackManager &StackMgr,\n                            const Runtime::Instance::ElementInstance &ElemInst,\n                            const AST::Instruction &Instr) const noexcept {\n  const uint32_t N = StackMgr.pop().get<uint32_t>();\n  const uint32_t S = StackMgr.getTop().get<uint32_t>();\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, Instr.getTargetIndex())->getCompositeType();\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  auto ElemSrc = ElemInst.getRefs();\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) > ElemSrc.size()) {\n    spdlog::error(ErrCode::Value::TableOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N,\n        ElemSrc.size() > 0 ? static_cast<uint32_t>(ElemSrc.size() - 1) : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::TableOutOfBounds);\n  }\n  std::vector<ValVariant> Refs(ElemSrc.begin() + S, ElemSrc.begin() + S + N);\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  auto *Inst =\n      const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n          ->newArray(Instr.getTargetIndex(), packVals(SType, std::move(Refs)));\n  StackMgr.getTop().emplace<RefVariant>(Inst->getDefType(), Inst);\n  return {};\n}\n\nExpect<void>\nExecutor::runArraySetOp(const ValVariant &Val, const uint32_t Idx,\n                        const RefVariant &InstRef,\n                        const AST::CompositeType &CompType,\n                        const AST::Instruction &Instr) const noexcept {\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (Idx >= Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(Idx, 1, Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  Inst->getData(Idx) = packVal(SType, Val);\n  return {};\n}\n\nExpect<void> Executor::runArrayGetOp(ValVariant &Val, const uint32_t Idx,\n                                     const AST::CompositeType &CompType,\n                                     const AST::Instruction &Instr,\n                                     bool IsSigned) const noexcept {\n  const auto *Inst =\n      Val.get<RefVariant>().getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (Idx >= Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(Idx, 1, Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  Val = unpackVal(SType, Inst->getData(Idx), IsSigned);\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayLenOp(ValVariant &Val,\n                        const AST::Instruction &Instr) const noexcept {\n  const auto *Inst =\n      Val.get<RefVariant>().getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  Val.emplace<uint32_t>(Inst->getLength());\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayFillOp(uint32_t N, const ValVariant &Val, uint32_t D,\n                         const RefVariant &InstRef,\n                         const AST::CompositeType &CompType,\n                         const AST::Instruction &Instr) const noexcept {\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) > Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  auto Arr = Inst->getArray();\n  std::fill(Arr.begin() + D, Arr.begin() + D + N, packVal(SType, Val));\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayCopyOp(uint32_t N, uint32_t S, const RefVariant &SrcInstRef,\n                         uint32_t D, const RefVariant &DstInstRef,\n                         const AST::CompositeType &SrcCompType,\n                         const AST::CompositeType &DstCompType,\n                         const AST::Instruction &Instr) const noexcept {\n  auto *SrcInst = SrcInstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  auto *DstInst = DstInstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (SrcInst == nullptr || DstInst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) >\n      SrcInst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(S), N,\n                                        SrcInst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) >\n      DstInst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        DstInst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SrcSType = SrcCompType.getFieldTypes()[0].getStorageType();\n  const auto &DstSType = DstCompType.getFieldTypes()[0].getStorageType();\n  auto SrcArr = SrcInst->getArray();\n  auto DstArr = DstInst->getArray();\n  if (D <= S) {\n    std::transform(SrcArr.begin() + S, SrcArr.begin() + S + N,\n                   DstArr.begin() + D, [&](const ValVariant &V) {\n                     return packVal(DstSType, unpackVal(SrcSType, V));\n                   });\n  } else {\n    std::transform(std::make_reverse_iterator(SrcArr.begin() + S + N),\n                   std::make_reverse_iterator(SrcArr.begin() + S),\n                   std::make_reverse_iterator(DstArr.begin() + D + N),\n                   [&](const ValVariant &V) {\n                     return packVal(DstSType, unpackVal(SrcSType, V));\n                   });\n  }\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayInitDataOp(uint32_t N, uint32_t S, uint32_t D,\n                             const RefVariant &InstRef,\n                             const AST::CompositeType &CompType,\n                             const Runtime::Instance::DataInstance &DataInst,\n                             const AST::Instruction &Instr) const noexcept {\n  const uint32_t BSize =\n      CompType.getFieldTypes()[0].getStorageType().getBitWidth() / 8;\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) > Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) * BSize >\n      DataInst.getData().size()) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N * BSize,\n        DataInst.getData().size() > 0\n            ? static_cast<uint32_t>(DataInst.getData().size() - 1)\n            : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n  for (uint32_t Off = 0; Off < N; Off++) {\n    // The value has been packed.\n    Inst->getData(D + Off) = DataInst.loadValue(S + Off * BSize, BSize);\n  }\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayInitElemOp(uint32_t N, uint32_t S, uint32_t D,\n                             const RefVariant &InstRef,\n                             const AST::CompositeType &CompType,\n                             const Runtime::Instance::ElementInstance &ElemInst,\n                             const AST::Instruction &Instr) const noexcept {\n  auto ElemSrc = ElemInst.getRefs();\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) > Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) > ElemSrc.size()) {\n    spdlog::error(ErrCode::Value::TableOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N,\n        ElemSrc.size() > 0 ? static_cast<uint32_t>(ElemSrc.size() - 1) : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::TableOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n\n  auto Arr = Inst->getArray();\n  // The value has been packed.\n  std::transform(ElemSrc.begin() + S, ElemSrc.begin() + S + N, Arr.begin() + D,\n                 [&](const RefVariant &V) { return packVal(SType, V); });\n  return {};\n}\n\nExpect<void>\nExecutor::runRefTestOp(const Runtime::Instance::ModuleInstance *ModInst,\n                       ValVariant &Val, const AST::Instruction &Instr,\n                       bool IsCast) const noexcept {\n  // Copy the value type here due to handling the externalized case.\n  auto &VT = Val.get<RefVariant>().getType();\n  if (VT.isExternalized()) {\n    VT = ValType(TypeCode::Ref, TypeCode::ExternRef);\n  }\n  Span<const AST::SubType *const> GotTypeList = ModInst->getTypeList();\n  if (!VT.isAbsHeapType()) {\n    auto *Inst =\n        Val.get<RefVariant>().getPtr<Runtime::Instance::CompositeBase>();\n    // Reference must not be nullptr here because the null references are typed\n    // with the least abstract heap type.\n    if (Inst->getModule()) {\n      GotTypeList = Inst->getModule()->getTypeList();\n    }\n  }\n\n  if (AST::TypeMatcher::matchType(ModInst->getTypeList(), Instr.getValType(),\n                                  GotTypeList, VT)) {\n    if (!IsCast) {\n      Val.emplace<uint32_t>(1U);\n    }\n  } else {\n    if (IsCast) {\n      spdlog::error(ErrCode::Value::CastFailed);\n      spdlog::error(ErrInfo::InfoMismatch(Instr.getValType(), VT));\n      spdlog::error(\n          ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n      return Unexpect(ErrCode::Value::CastFailed);\n    } else {\n      Val.emplace<uint32_t>(0U);\n    }\n  }\n  return {};\n}\n\nExpect<void> Executor::runRefConvOp(RefVariant &Ref,\n                                    TypeCode TCode) const noexcept {\n\n  if (TCode == TypeCode::AnyRef) {\n    // Internalize.\n    if (Ref.isNull()) {\n      Ref = RefVariant(ValType(TypeCode::RefNull, TypeCode::NullRef));\n    } else {\n      Ref.getType().setInternalized();\n      if (Ref.getType().isExternRefType()) {\n        Ref.getType() = ValType(TypeCode::Ref, TypeCode::AnyRef);\n      }\n    }\n  } else {\n    // Externalize.\n    if (Ref.isNull()) {\n      Ref = RefVariant(ValType(TypeCode::RefNull, TypeCode::NullExternRef));\n    } else {\n      // Use the externalize flag because the value type information should be\n      // reserved when a reference being externalized and internalized.\n      Ref.getType().setExternalized();\n    }\n  }\n  return {};\n}\n\nExpect<void> Executor::runRefI31Op(ValVariant &Val) const noexcept {\n  uint32_t RefNum = (Val.get<uint32_t>() & 0x7FFFFFFFU) | 0x80000000U;\n  Val = RefVariant(ValType(TypeCode::Ref, TypeCode::I31Ref),\n                   reinterpret_cast<void *>(static_cast<uint64_t>(RefNum)));\n  return {};\n}\n\nExpect<void> Executor::runI31GetOp(ValVariant &Val,\n                                   const AST::Instruction &Instr,\n                                   bool IsSigned) const noexcept {\n  uint32_t RefNum = static_cast<uint32_t>(\n      reinterpret_cast<uintptr_t>(Val.get<RefVariant>().getPtr<void>()));\n  if ((RefNum & 0x80000000U) == 0) {\n    spdlog::error(ErrCode::Value::AccessNullI31);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullI31);\n  }\n  RefNum &= 0x7FFFFFFFU;\n  if (IsSigned) {\n    RefNum |= ((RefNum & 0x40000000U) << 1);\n  }\n  Val.emplace<uint32_t>(RefNum);\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/memoryInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void>\nExecutor::runMemorySizeOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInst) {\n  // Push SZ = page size to stack.\n  StackMgr.push(MemInst.getPageSize());\n  return {};\n}\n\nExpect<void>\nExecutor::runMemoryGrowOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInst) {\n  // Pop N for growing page size.\n  uint32_t &N = StackMgr.getTop().get<uint32_t>();\n\n  // Grow page and push result.\n  const uint32_t CurrPageSize = static_cast<uint32_t>(MemInst.getPageSize());\n  if (MemInst.growPage(N)) {\n    N = CurrPageSize;\n  } else {\n    N = static_cast<uint32_t>(-1);\n  }\n  return {};\n}\n\nExpect<void> Executor::runMemoryInitOp(\n    Runtime::StackManager &StackMgr, Runtime::Instance::MemoryInstance &MemInst,\n    Runtime::Instance::DataInstance &DataInst, const AST::Instruction &Instr) {\n  // Pop the length, source, and destination from stack.\n  uint32_t Len = StackMgr.pop().get<uint32_t>();\n  uint32_t Src = StackMgr.pop().get<uint32_t>();\n  uint32_t Dst = StackMgr.pop().get<uint32_t>();\n\n  // Replace mem[Dst : Dst + Len] with data[Src : Src + Len].\n  if (auto Res = MemInst.setBytes(DataInst.getData(), Dst, Src, Len)) {\n    return {};\n  } else {\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Res);\n  }\n}\n\nExpect<void>\nExecutor::runDataDropOp(Runtime::Instance::DataInstance &DataInst) {\n  // Clear data instance.\n  DataInst.clear();\n  return {};\n}\n\nExpect<void>\nExecutor::runMemoryCopyOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInstDst,\n                          Runtime::Instance::MemoryInstance &MemInstSrc,\n                          const AST::Instruction &Instr) {\n  // Pop the length, source, and destination from stack.\n  uint32_t Len = StackMgr.pop().get<uint32_t>();\n  uint32_t Src = StackMgr.pop().get<uint32_t>();\n  uint32_t Dst = StackMgr.pop().get<uint32_t>();\n\n  // Replace mem[Dst : Dst + Len] with mem[Src : Src + Len].\n  if (auto Data = MemInstSrc.getBytes(Src, Len)) {\n    if (auto Res = MemInstDst.setBytes(*Data, Dst, 0, Len)) {\n      return {};\n    } else {\n      spdlog::error(\n          ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n      return Unexpect(Res);\n    }\n  } else {\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Data);\n  }\n}\n\nExpect<void>\nExecutor::runMemoryFillOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInst,\n                          const AST::Instruction &Instr) {\n  // Pop the length, value, and offset from stack.\n  uint32_t Len = StackMgr.pop().get<uint32_t>();\n  uint8_t Val = static_cast<uint8_t>(StackMgr.pop().get<uint32_t>());\n  uint32_t Off = StackMgr.pop().get<uint32_t>();\n\n  // Fill data with Val.\n  if (auto Res = MemInst.fillBytes(Val, Off, Len)) {\n    return {};\n  } else {\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Res);\n  }\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/threadInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void>\nExecutor::runAtomicNotifyOp(Runtime::StackManager &StackMgr,\n                            Runtime::Instance::MemoryInstance &MemInst,\n                            const AST::Instruction &Instr) {\n  ValVariant RawCount = StackMgr.pop();\n  ValVariant &RawAddress = StackMgr.getTop();\n\n  uint32_t Address = RawAddress.get<uint32_t>();\n\n  if (Address >\n      std::numeric_limits<uint32_t>::max() - Instr.getMemoryOffset()) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        Address + static_cast<uint64_t>(Instr.getMemoryOffset()),\n        sizeof(uint32_t), MemInst.getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n  Address += Instr.getMemoryOffset();\n\n  if (Address % sizeof(uint32_t) != 0) {\n    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);\n  }\n\n  uint32_t Count = RawCount.get<uint32_t>();\n  if (auto Res = atomicNotify(MemInst, Address, Count); unlikely(!Res)) {\n    spdlog::error(Res.error());\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Res);\n  } else {\n    RawAddress.emplace<uint32_t>(*Res);\n  }\n  return {};\n}\n\nExpect<void> Executor::runMemoryFenceOp() {\n  std::atomic_thread_fence(std::memory_order_release);\n  return {};\n}\n\nExpect<uint32_t>\nExecutor::atomicNotify(Runtime::Instance::MemoryInstance &MemInst,\n                       uint32_t Address, uint32_t Count) noexcept {\n  // The error message should be handled by the caller, or the AOT mode will\n  // produce the duplicated messages.\n  if (auto *AtomicObj = MemInst.getPointer<std::atomic<uint32_t> *>(Address);\n      !AtomicObj) {\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n\n  std::unique_lock<decltype(WaiterMapMutex)> Locker(WaiterMapMutex);\n  uint32_t Total = 0;\n  auto Range = WaiterMap.equal_range(Address);\n  for (auto Iterator = Range.first; Total < Count && Iterator != Range.second;\n       ++Iterator) {\n    if (likely(&MemInst == Iterator->second.MemInst)) {\n      Iterator->second.Cond.notify_all();\n      ++Total;\n    }\n  }\n  return Total;\n}\n\nvoid Executor::atomicNotifyAll() noexcept {\n  std::unique_lock<decltype(WaiterMapMutex)> Locker(WaiterMapMutex);\n  for (auto Iterator = WaiterMap.begin(); Iterator != WaiterMap.end();\n       ++Iterator) {\n    Iterator->second.Cond.notify_all();\n  }\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/loader/loader.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/loader.h\"\n\n#include \"aot/version.h\"\n\n#include <algorithm>\n#include <cstddef>\n#include <fstream>\n#include <limits>\n#include <memory>\n#include <system_error>\n#include <utility>\n#include <variant>\n\nnamespace WasmEdge {\nnamespace Loader {\n\n// Load data from file path. See \"include/loader/loader.h\".\nExpect<std::vector<Byte>>\nLoader::loadFile(const std::filesystem::path &FilePath) {\n  std::error_code EC;\n  size_t FileSize = std::filesystem::file_size(FilePath, EC);\n  if (EC) {\n    spdlog::error(ErrCode::Value::IllegalPath);\n    spdlog::error(ErrInfo::InfoFile(FilePath));\n    return Unexpect(ErrCode::Value::IllegalPath);\n  }\n\n  std::ifstream Fin(FilePath, std::ios::in | std::ios::binary);\n  if (!Fin) {\n    spdlog::error(ErrCode::Value::IllegalPath);\n    spdlog::error(ErrInfo::InfoFile(FilePath));\n    return Unexpect(ErrCode::Value::IllegalPath);\n  }\n\n  std::vector<Byte> Buf(FileSize);\n  size_t Index = 0;\n  while (FileSize > 0) {\n    const uint32_t BlockSize = static_cast<uint32_t>(\n        std::min<size_t>(FileSize, std::numeric_limits<uint32_t>::max()));\n    Fin.read(reinterpret_cast<char *>(Buf.data()) + Index, BlockSize);\n    const uint32_t ReadCount = static_cast<uint32_t>(Fin.gcount());\n    if (ReadCount != BlockSize) {\n      if (Fin.eof()) {\n        spdlog::error(ErrCode::Value::UnexpectedEnd);\n        spdlog::error(ErrInfo::InfoLoading(ReadCount));\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(ErrCode::Value::UnexpectedEnd);\n      } else {\n        spdlog::error(ErrCode::Value::ReadError);\n        spdlog::error(ErrInfo::InfoLoading(ReadCount));\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(ErrCode::Value::ReadError);\n      }\n    }\n    Index += static_cast<size_t>(BlockSize);\n    FileSize -= static_cast<size_t>(BlockSize);\n  }\n  return Buf;\n}\n\nExpect<std::variant<std::unique_ptr<AST::Component::Component>,\n                    std::unique_ptr<AST::Module>>>\nLoader::parseWasmUnit(const std::filesystem::path &FilePath) {\n  std::lock_guard Lock(Mutex);\n  // Set path and check the header.\n  if (auto Res = FMgr.setPath(FilePath); !Res) {\n    spdlog::error(Res.error());\n    spdlog::error(ErrInfo::InfoFile(FilePath));\n    return Unexpect(Res);\n  }\n\n  switch (FMgr.getHeaderType()) {\n  // Filter out the Windows .dll, MacOS .dylib, or Linux .so AOT compiled\n  // shared-library-WASM.\n  case FileMgr::FileHeader::ELF:\n  case FileMgr::FileHeader::DLL:\n  case FileMgr::FileHeader::MachO_32:\n  case FileMgr::FileHeader::MachO_64: {\n    // AOT compiled shared-library-WASM cases. Use ldmgr to load the module.\n    WASMType = InputType::SharedLibrary;\n    FMgr.reset();\n    std::shared_ptr<SharedLibrary> Library = std::make_shared<SharedLibrary>();\n    if (auto Res = Library->load(FilePath); !Res) {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Res);\n    }\n    if (auto Res = Library->getVersion()) {\n      if (*Res != AOT::kBinaryVersion) {\n        spdlog::error(ErrInfo::InfoMismatch(AOT::kBinaryVersion, *Res));\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(ErrCode::Value::MalformedVersion);\n      }\n    } else {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Res);\n    }\n\n    std::unique_ptr<AST::Module> Mod;\n    if (auto Code = Library->getWasm()) {\n      // Set the binary and load module.\n      // Not to use parseModule() here to keep the `WASMType` value.\n      if (auto Res = FMgr.setCode(*Code); !Res) {\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(Res);\n      }\n      if (auto Res = loadUnit()) {\n        if (std::holds_alternative<std::unique_ptr<AST::Module>>(*Res)) {\n          Mod = std::move(std::get<std::unique_ptr<AST::Module>>(*Res));\n        }\n      } else {\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(Res);\n      }\n    } else {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Code);\n    }\n    if (!Conf.getRuntimeConfigure().isForceInterpreter()) {\n      // If the configure is set to force interpreter mode, not to load the AOT\n      // related data.\n      if (auto Res = loadExecutable(*Mod, Library); unlikely(!Res)) {\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(Res);\n      }\n    }\n    return Mod;\n  }\n  default:\n    // Universal WASM, WASM, or other cases. Load and parse the module directly.\n    WASMType = InputType::WASM;\n    auto Unit = loadUnit();\n    if (!Unit) {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Unit);\n    }\n    switch (Unit->index()) {\n    case 0: // component\n      return Unit;\n    case 1: // module\n    default: {\n      auto Mod = std::move(std::get<std::unique_ptr<AST::Module>>(*Unit));\n      if (!Conf.getRuntimeConfigure().isForceInterpreter()) {\n        // If the configure is set to force interpreter mode, not to set the\n        // symbol.\n        if (auto &Symbol = Mod->getSymbol()) {\n          *Symbol = IntrinsicsTable;\n        }\n      }\n      return Mod;\n    }\n    }\n  }\n}\n\nExpect<std::variant<std::unique_ptr<AST::Component::Component>,\n                    std::unique_ptr<AST::Module>>>\nLoader::parseWasmUnit(Span<const uint8_t> Code) {\n  std::lock_guard Lock(Mutex);\n  if (auto Res = FMgr.setCode(Code); !Res) {\n    return Unexpect(Res);\n  }\n  switch (FMgr.getHeaderType()) {\n  // Filter out the Windows .dll, MacOS .dylib, or Linux .so AOT compiled\n  // shared-library-WASM.\n  case FileMgr::FileHeader::ELF:\n  case FileMgr::FileHeader::DLL:\n  case FileMgr::FileHeader::MachO_32:\n  case FileMgr::FileHeader::MachO_64:\n    spdlog::error(\"Might an invalid wasm file\");\n    spdlog::error(ErrCode::Value::MalformedMagic);\n    spdlog::error(\n        \"    The AOT compiled WASM shared library is not supported for loading \"\n        \"from memory. Please use the universal WASM binary or pure WASM, or \"\n        \"load the AOT compiled WASM shared library from file.\");\n    return Unexpect(ErrCode::Value::MalformedMagic);\n  default:\n    break;\n  }\n  // For malformed header checking, handle in the module loading.\n  WASMType = InputType::WASM;\n  return loadUnit();\n}\n\n// Parse module from file path. See \"include/loader/loader.h\".\nExpect<std::unique_ptr<AST::Module>>\nLoader::parseModule(const std::filesystem::path &FilePath) {\n  if (auto R = parseWasmUnit(FilePath)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*R)) {\n      return std::move(std::get<std::unique_ptr<AST::Module>>(*R));\n    }\n    return Unexpect(ErrCode::Value::MalformedVersion);\n  } else {\n    return Unexpect(R);\n  }\n}\n\n// Parse module from byte code. See \"include/loader/loader.h\".\nExpect<std::unique_ptr<AST::Module>>\nLoader::parseModule(Span<const uint8_t> Code) {\n  if (auto R = parseWasmUnit(Code)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*R)) {\n      return std::move(std::get<std::unique_ptr<AST::Module>>(*R));\n    }\n    return Unexpect(ErrCode::Value::MalformedVersion);\n  } else {\n    return Unexpect(R);\n  }\n}\n\n// Serialize module into byte code. See \"include/loader/loader.h\".\nExpect<std::vector<Byte>> Loader::serializeModule(const AST::Module &Mod) {\n  return Ser.serializeModule(Mod);\n}\n\n} // namespace Loader\n} // namespace WasmEdge\n\n// Path: lib/loader/aot_section.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/aot_section.h\"\n#include \"common/log.h\"\n#include \"system/allocator.h\"\n\n#if WASMEDGE_OS_LINUX || WASMEDGE_OS_MACOS\nextern \"C\" {\nextern void __register_frame(void *);\nextern void __deregister_frame(void *);\n}\n#endif\n\nnamespace {\ninline constexpr uint64_t roundDownPageBoundary(const uint64_t Value) {\n// ARM64 Mac has a special page size\n#if WASMEDGE_OS_MACOS && defined(__aarch64__)\n  return Value & ~UINT64_C(16383);\n#else\n  return Value & ~UINT64_C(4095);\n#endif\n}\ninline constexpr uint64_t roundUpPageBoundary(const uint64_t Value) {\n// ARM64 Mac has a special page size\n#if WASMEDGE_OS_MACOS && defined(__aarch64__)\n  return roundDownPageBoundary(Value + UINT64_C(16383));\n#else\n  return roundDownPageBoundary(Value + UINT64_C(4095));\n#endif\n}\n} // namespace\n\nnamespace WasmEdge::Loader {\n\nExpect<void> AOTSection::load(const AST::AOTSection &AOTSec) noexcept {\n  BinarySize = 0;\n  for (const auto &Section : AOTSec.getSections()) {\n    const auto Offset = std::get<1>(Section);\n    const auto Size = std::get<2>(Section);\n    BinarySize = std::max(BinarySize, Offset + Size);\n  }\n  BinarySize = roundUpPageBoundary(BinarySize);\n\n  Binary = Allocator::allocate_chunk(BinarySize);\n  if (unlikely(!Binary)) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n\n  std::vector<std::pair<uint8_t *, uint64_t>> ExecutableRanges;\n  for (const auto &Section : AOTSec.getSections()) {\n    const auto Offset = std::get<1>(Section);\n    const auto Size = std::get<2>(Section);\n    const auto &Content = std::get<3>(Section);\n    if (Size > BinarySize || Offset > BinarySize ||\n        Offset + Size > BinarySize || Content.size() > Size) {\n      return Unexpect(ErrCode::Value::IntegerTooLarge);\n    }\n    std::copy(Content.begin(), Content.end(), Binary + Offset);\n    switch (std::get<0>(Section)) {\n    case 1: { // Text\n      const auto O = roundDownPageBoundary(Offset);\n      const auto S = roundUpPageBoundary(Size + (Offset - O));\n      ExecutableRanges.emplace_back(Binary + O, S);\n      break;\n    }\n    case 2: // Data\n      break;\n    case 3: // BSS\n      break;\n#if WASMEDGE_OS_LINUX\n    case 4: // EHFrame\n      EHFrameAddress = reinterpret_cast<void *>(Binary + Offset);\n      break;\n#elif WASMEDGE_OS_MACOS\n    case 4: // EHFrame\n      EHFrameAddress = reinterpret_cast<uint8_t *>(Binary + Offset);\n      EHFrameSize = Size;\n      break;\n#elif WASMEDGE_OS_WINDOWS\n    case 4: // PData\n      PDataAddress = reinterpret_cast<void *>(Binary + Offset);\n      PDataSize =\n          static_cast<uint32_t>(Size / sizeof(winapi::RUNTIME_FUNCTION_));\n      break;\n#endif\n    default:\n      return Unexpect(ErrCode::Value::IntegerTooLarge);\n    }\n  }\n\n  for (const auto &[Pointer, Size] : ExecutableRanges) {\n    if (!Allocator::set_chunk_executable(Pointer, Size)) {\n      spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n      spdlog::error(\"    set_chunk_executable failed:{}\", std::strerror(errno));\n      return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n    }\n  }\n\n  IntrinsicsAddress = AOTSec.getIntrinsicsAddress();\n  TypesAddress = AOTSec.getTypesAddress();\n  CodesAddress = AOTSec.getCodesAddress();\n\n#if WASMEDGE_OS_LINUX\n  if (EHFrameAddress) {\n    __register_frame(EHFrameAddress);\n  }\n#elif WASMEDGE_OS_MACOS\n  if (EHFrameAddress) {\n    auto Iter = EHFrameAddress;\n    const auto End = EHFrameAddress + EHFrameSize - 4;\n\n    while (Iter < End) {\n      if (Iter != EHFrameAddress) {\n        __register_frame(Iter);\n      }\n      const uint32_t Length = *reinterpret_cast<const uint32_t *>(Iter);\n      Iter += Length + 4;\n    }\n  }\n#elif WASMEDGE_OS_WINDOWS\n  if (PDataSize != 0) {\n    winapi::RtlAddFunctionTable(\n        static_cast<winapi::PRUNTIME_FUNCTION_>(PDataAddress), PDataSize,\n        reinterpret_cast<winapi::ULONG_PTR_>(Binary));\n  }\n#endif\n\n  return {};\n}\n\nvoid AOTSection::unload() noexcept {\n  if (Binary) {\n#if WASMEDGE_OS_LINUX\n    if (EHFrameAddress) {\n      __deregister_frame(EHFrameAddress);\n    }\n#elif WASMEDGE_OS_MACOS\n    if (EHFrameAddress) {\n      auto Iter = EHFrameAddress;\n      const auto End = EHFrameAddress + EHFrameSize - 4;\n\n      while (Iter < End) {\n        if (Iter != EHFrameAddress) {\n          __deregister_frame(Iter);\n        }\n        const uint32_t Length = *reinterpret_cast<const uint32_t *>(Iter);\n        Iter += Length + 4;\n      }\n    }\n#elif WASMEDGE_OS_WINDOWS\n    if (PDataSize != 0) {\n      winapi::RtlDeleteFunctionTable(\n          static_cast<winapi::PRUNTIME_FUNCTION_>(PDataAddress));\n    }\n#endif\n    Allocator::set_chunk_readable_writable(Binary, BinarySize);\n    Allocator::release_chunk(Binary, BinarySize);\n    Binary = nullptr;\n  }\n}\n\n} // namespace WasmEdge::Loader\n\n// Path: lib/loader/filemgr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/filemgr.h\"\n\n#include <algorithm>\n#include <iterator>\n\n// Error logging of file manager need to be handled in caller.\n\nnamespace WasmEdge {\n\n// Set path to file manager. See \"include/loader/filemgr.h\".\nExpect<void> FileMgr::setPath(const std::filesystem::path &FilePath) {\n  reset();\n  std::error_code ErrCode;\n  Size = std::filesystem::file_size(FilePath, ErrCode);\n  if (likely(!ErrCode)) {\n    if (!MMap::supported()) {\n      Size = 0;\n      Status = ErrCode::Value::IllegalPath;\n      return Unexpect(Status);\n    }\n    FileMap.emplace(FilePath);\n    if (auto *Pointer = FileMap->address(); likely(Pointer)) {\n      Data = reinterpret_cast<const Byte *>(Pointer);\n      Status = ErrCode::Value::Success;\n    } else {\n      // File size is 0, mmap failed.\n      // Will get 'UnexpectedEnd' error while the first reading.\n      FileMap.reset();\n    }\n    return {};\n  }\n  Size = 0;\n  Status = ErrCode::Value::IllegalPath;\n  return Unexpect(Status);\n}\n\n// Set code data. See \"include/loader/filemgr.h\".\nExpect<void> FileMgr::setCode(Span<const Byte> CodeData) {\n  reset();\n  Data = CodeData.data();\n  Size = CodeData.size();\n  Status = ErrCode::Value::Success;\n  return {};\n}\n\n// Set code data. See \"include/loader/filemgr.h\".\nExpect<void> FileMgr::setCode(std::vector<Byte> CodeData) {\n  reset();\n  DataHolder.emplace(std::move(CodeData));\n  Data = DataHolder->data();\n  Size = DataHolder->size();\n  Status = ErrCode::Value::Success;\n  return {};\n}\n\n// Read one byte. See \"include/loader/filemgr.h\".\nExpect<Byte> FileMgr::readByte() {\n  if (unlikely(Status != ErrCode::Value::Success)) {\n    return Unexpect(Status);\n  }\n  // Set the flag to the start offset.\n  LastPos = Pos;\n  // Check if exceed the data boundary and section boundary.\n  if (auto Res = testRead(1); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n  return Data[Pos++];\n}\n\n// Read number of bytes. See \"include/loader/filemgr.h\".\nExpect<std::vector<Byte>> FileMgr::readBytes(size_t SizeToRead) {\n  // Set the flag to the start offset.\n  LastPos = Pos;\n  // Read bytes into vector.\n  std::vector<Byte> Buf(SizeToRead);\n  if (auto Res = readBytes(Buf); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n  return Buf;\n}\n\n// Decode and read an unsigned int. See \"include/loader/filemgr.h\".\nExpect<uint32_t> FileMgr::readU32() {\n  if (unlikely(Status != ErrCode::Value::Success)) {\n    return Unexpect(Status);\n  }\n  // Set the flag to the start offset.\n  LastPos = Pos;\n\n  // Read and decode U32.\n  uint32_t Result = 0;\n  uint32_t Offset = 0;\n  Byte Byte = 0x80;\n  while (Byte & 0x80) {\n    if (unlikely(Offset >= 32)) {\n      Status = ErrCode::Value::IntegerTooLong;\n      return Unexpect(Status);\n    }\n    if (auto Res = testRead(1); unlikely(!Res)) {\n      return Unexpect(Res);\n    }\n    Byte = Data[Pos++];\n    Result |= (Byte & UINT32_C(0x7F)) << Offset;\n    if (Offset == 28 && unlikely((Byte & UINT32_C(0x70)) != 0)) {\n      Status = ErrCode::Value::IntegerTooLarge;\n      return Unexpect(Status);\n    }\n    Offset += 7;\n  }\n  return Result;\n}\n\n// Decode and read an unsigned long long int. See \"include/loader/filemgr.h\".\nExpect<uint64_t> FileMgr::readU64() {\n...\n", "needle_token_start": 737, "needle_token_end": 870, "code_context_ntokens": 16238, "output": ["```cpp\nValVariant packVal(const ValType &Type, const ValVariant &Val) {\n  if (Type.isPackType()) {\n    switch (Type.getCode()) {\n    case TypeCode::I8:\n      return ValVariant(Val.get<uint32_t>() & 0xFFU);\n    case TypeCode::I16:\n      return ValVariant(Val.get<uint32_t>() & 0xFFFFU);\n    default:\n      assumingUnreachable();\n    }\n  }\n  return Val;\n}\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "HostVersion", "language": "cpp", "path": "lib/loader/ast/section.cpp", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the version number of the binary format supported by the host environment.\n2. **Input**: None, as it does not require any parameters.\n3. **Output**: Returns a constant integer representing the version number.\n4. **Procedure**: Simply accesses a predefined constant that holds the version number and returns this value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "          return {};\n        });\n  });\n}\n\n// Load vector of table section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::TableSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::TableSegment &TabSeg) { return loadSegment(TabSeg); });\n  });\n}\n\n// Load vector of memory section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::MemorySection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::MemoryType &MemType) { return loadType(MemType); });\n  });\n}\n\n// Load vector of global section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::GlobalSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(Sec, [this](AST::GlobalSegment &GlobSeg) {\n      return loadSegment(GlobSeg);\n    });\n  });\n}\n\n// Load vector of export section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::ExportSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::ExportDesc &ExpDesc) { return loadDesc(ExpDesc); });\n  });\n}\n\n// Load start function index. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::StartSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() -> Expect<void> {\n    // Read u32 of start function index.\n    if (auto Res = FMgr.readU32()) {\n      Sec.setContent(*Res);\n    } else {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Sec_Start);\n    }\n    return {};\n  });\n}\n\n// Load vector of element section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::ElementSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(Sec, [this](AST::ElementSegment &ElemSeg) {\n      return loadSegment(ElemSeg);\n    });\n  });\n}\n\n// Load vector of code section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::CodeSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(Sec, [this](AST::CodeSegment &CodeSeg) {\n      return loadSegment(CodeSeg);\n    });\n  });\n}\n\n// Load vector of data section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::DataSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(Sec, [this](AST::DataSegment &DataSeg) {\n      return loadSegment(DataSeg);\n    });\n  });\n}\n\n// Load content of data count section. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::DataCountSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() -> Expect<void> {\n    // Read u32 of data count.\n    if (auto Res = FMgr.readU32()) {\n      Sec.setContent(*Res);\n    } else {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Sec_DataCount);\n    }\n    return {};\n  });\n}\n\nExpect<void> Loader::loadSection(AST::Component::ComponentSection &Sec) {\n  auto ResPreamble = Loader::loadPreamble();\n  if (!ResPreamble) {\n    return Unexpect(ResPreamble);\n  }\n  auto WasmMagic = ResPreamble->first;\n  auto Ver = ResPreamble->second;\n  if (unlikely(Ver != ComponentVersion)) {\n    return logLoadError(ErrCode::Value::MalformedVersion, FMgr.getLastOffset(),\n                        ASTNodeAttr::Component);\n  }\n  auto NestedComp = std::make_shared<AST::Component::Component>();\n  NestedComp->getMagic() = WasmMagic;\n  NestedComp->getVersion() = {Ver[0], Ver[1]};\n  NestedComp->getLayer() = {Ver[2], Ver[3]};\n  if (auto Res = loadComponent(*NestedComp); !Res) {\n    spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n    return Unexpect(Res);\n  }\n  Sec.getContent() = NestedComp;\n  return {};\n}\n\nExpect<void> Loader::loadSection(AST::CoreModuleSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() -> Expect<void> {\n    auto ExpectedSize = Sec.getContentSize();\n    auto StartOffset = FMgr.getOffset();\n    auto ResPreamble = Loader::loadPreamble();\n    if (!ResPreamble) {\n      spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Module));\n      return Unexpect(ResPreamble);\n    }\n    auto WasmMagic = ResPreamble->first;\n    auto Ver = ResPreamble->second;\n    if (unlikely(Ver != ModuleVersion)) {\n      return logLoadError(ErrCode::Value::MalformedVersion,\n                          FMgr.getLastOffset(), ASTNodeAttr::Module);\n    }\n    AST::Module CoreMod;\n    CoreMod.getMagic() = WasmMagic;\n    CoreMod.getVersion() = Ver;\n\n    auto Offset = FMgr.getOffset();\n    ExpectedSize -= (Offset - StartOffset);\n\n    if (auto Res = loadModuleInBound(CoreMod, ExpectedSize); !Res) {\n      spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Module));\n      return Unexpect(Res);\n    }\n    Sec.getContent() = CoreMod;\n    return {};\n  });\n}\n\n// Load vector of component alias section.\n// See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::Component::AliasSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::Alias &Alias) { return loadAlias(Alias); });\n  });\n}\n\n// Load vector of component core:instance section.\n// See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::Component::CoreInstanceSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::CoreInstanceExpr &InstanceExpr) {\n          return loadCoreInstance(InstanceExpr);\n        });\n  });\n}\n\n// Load vector of core type section.\n// See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::Component::CoreTypeSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::CoreDefType &Ty) { return loadType(Ty); });\n  });\n}\n\n// Load vector of component type section.\n// See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::Component::TypeSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::DefType &Ty) { return loadType(Ty); });\n  });\n}\n\nExpect<void> Loader::loadSection(AST::Component::StartSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() -> Expect<void> {\n    return loadStart(Sec.getContent());\n  });\n}\n\nExpect<void> Loader::loadSection(AST::Component::CanonSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::Canon &C) { return loadCanonical(C); });\n  });\n}\n\nExpect<void> Loader::loadSection(AST::Component::ImportSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::Import &C) { return loadImport(C); });\n  });\n}\nExpect<void> Loader::loadSection(AST::Component::ExportSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::Export &C) { return loadExport(C); });\n  });\n}\n\n// Load vector of component instance section.\n// See \"include/loader/loader.h\".\nExpect<void> Loader::loadSection(AST::Component::InstanceSection &Sec) {\n  return loadSectionContent(Sec, [this, &Sec]() {\n    return loadSectionContentVec(\n        Sec, [this](AST::Component::InstanceExpr &InstanceExpr) {\n          return loadInstance(InstanceExpr);\n        });\n  });\n}\n\nnamespace {\n\n\ninline constexpr uint32_t HostVersion() noexcept {\n  return WasmEdge::AOT::kBinaryVersion;\n}\n\ninline constexpr uint8_t HostOSType() noexcept {\n#if WASMEDGE_OS_LINUX\n  return UINT8_C(1);\n#elif WASMEDGE_OS_MACOS\n  return UINT8_C(2);\n#elif WASMEDGE_OS_WINDOWS\n  return UINT8_C(3);\n#else\n  // Means WasmEdge is not yet supported on this OS.\n  return UINT8_C(-1);\n#endif\n}\n\ninline constexpr uint8_t HostArchType() noexcept {\n#if defined(__x86_64__) || defined(_M_X64)\n  return UINT8_C(1);\n#elif defined(__aarch64__)\n  return UINT8_C(2);\n#elif defined(__riscv) && __riscv_xlen == 64\n  return UINT8_C(3);\n#elif defined(__arm__) && __ARM_ARCH == 7\n  return UINT8_C(4);\n#else\n  // Means universal wasm binary is not yet supported on this arch.\n  return UINT8_C(-1);\n#endif\n}\n\n} // namespace\n\n// If there is any loader error occurs in the loadSection, then fallback\n// to the interpreter mode with info level log.\nExpect<void> Loader::loadSection(FileMgr &VecMgr, AST::AOTSection &Sec) {\n  if (auto Res = VecMgr.readU32(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT binary version read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    Sec.setVersion(*Res);\n  }\n  if (unlikely(Sec.getVersion() != HostVersion())) {\n    spdlog::info(ErrCode::Value::MalformedSection);\n    spdlog::info(\"    AOT binary version unmatched.\");\n    return Unexpect(ErrCode::Value::MalformedSection);\n  }\n\n  if (auto Res = VecMgr.readByte(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT os type read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    Sec.setOSType(*Res);\n  }\n  if (unlikely(Sec.getOSType() != HostOSType())) {\n    spdlog::info(ErrCode::Value::MalformedSection);\n    spdlog::info(\"    AOT OS type unmatched.\");\n    return Unexpect(ErrCode::Value::MalformedSection);\n  }\n\n  if (auto Res = VecMgr.readByte(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT arch type read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    Sec.setArchType(*Res);\n  }\n  if (unlikely(Sec.getArchType() != HostArchType())) {\n    spdlog::info(ErrCode::Value::MalformedSection);\n    spdlog::info(\"    AOT arch type unmatched.\");\n    return Unexpect(ErrCode::Value::MalformedSection);\n  }\n\n  if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT version address read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    Sec.setVersionAddress(*Res);\n  }\n  if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT intrinsics address read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    Sec.setIntrinsicsAddress(*Res);\n  }\n  if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT types size read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    const uint64_t Size = *Res;\n    if (Size > VecMgr.getRemainSize()) {\n      spdlog::info(ErrCode::Value::IntegerTooLong);\n      spdlog::info(\"    AOT types size too large\");\n      return Unexpect(ErrCode::Value::IntegerTooLong);\n    }\n    Sec.getTypesAddress().resize(Size);\n  }\n  for (size_t I = 0; I < Sec.getTypesAddress().size(); ++I) {\n    if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n      spdlog::info(Res.error());\n      spdlog::info(\"    AOT type address read error:{}\", Res.error());\n      return Unexpect(Res);\n    } else {\n      Sec.getTypesAddress()[I] = *Res;\n    }\n  }\n  if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT code size read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    const uint64_t Size = *Res;\n    if (Size > VecMgr.getRemainSize()) {\n      spdlog::info(ErrCode::Value::IntegerTooLong);\n      spdlog::info(\"    AOT code size too large\");\n      return Unexpect(ErrCode::Value::IntegerTooLong);\n    }\n    Sec.getCodesAddress().resize(Size);\n  }\n  for (size_t I = 0; I < Sec.getCodesAddress().size(); ++I) {\n    if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n      spdlog::info(Res.error());\n      spdlog::info(\"    AOT code address read error:{}\", Res.error());\n      return Unexpect(Res);\n    } else {\n      const uint64_t Address = *Res;\n      Sec.getCodesAddress()[I] = Address;\n    }\n  }\n\n  if (auto Res = VecMgr.readU32(); unlikely(!Res)) {\n    spdlog::info(Res.error());\n    spdlog::info(\"    AOT section count read error:{}\", Res.error());\n    return Unexpect(Res);\n  } else {\n    const uint32_t Size = *Res;\n    if (Size > VecMgr.getRemainSize()) {\n      spdlog::info(ErrCode::Value::IntegerTooLong);\n      spdlog::info(\"    AOT section count too large\");\n      return Unexpect(ErrCode::Value::IntegerTooLong);\n    }\n    Sec.getSections().resize(Size);\n  }\n\n  for (auto &Section : Sec.getSections()) {\n    if (auto Res = VecMgr.readByte(); unlikely(!Res)) {\n      spdlog::info(Res.error());\n      spdlog::info(\"    AOT section type read error:{}\", Res.error());\n      return Unexpect(Res);\n    } else {\n      std::get<0>(Section) = *Res;\n    }\n    if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n      spdlog::info(Res.error());\n      spdlog::info(\"    AOT section offset read error:{}\", Res.error());\n      return Unexpect(Res);\n    } else {\n      std::get<1>(Section) = *Res;\n    }\n    if (auto Res = VecMgr.readU64(); unlikely(!Res)) {\n      spdlog::info(Res.error());\n      spdlog::info(\"    AOT section size read error:{}\", Res.error());\n      return Unexpect(Res);\n    } else {\n      std::get<2>(Section) = *Res;\n    }\n    uint32_t ContentSize;\n    if (auto Res = VecMgr.readU32(); unlikely(!Res)) {\n      spdlog::info(Res.error());\n      spdlog::info(\"    AOT section data size read error:{}\", Res.error());\n      return Unexpect(Res);\n    } else {\n      ContentSize = *Res;\n      if (ContentSize > VecMgr.getRemainSize()) {\n        spdlog::info(ErrCode::Value::IntegerTooLong);\n        spdlog::info(\"    AOT section data size is too large\");\n        return Unexpect(ErrCode::Value::IntegerTooLong);\n      }\n      if (std::get<2>(Section) < ContentSize) {\n        spdlog::info(ErrCode::Value::IntegerTooLong);\n        spdlog::info(\"    AOT section data size is larger then section size\");\n        return Unexpect(ErrCode::Value::IntegerTooLong);\n      }\n    }\n    if (auto Res = VecMgr.readBytes(ContentSize); unlikely(!Res)) {\n      spdlog::info(Res.error());\n      spdlog::info(\"    AOT section data read error:{}\", Res.error());\n      return Unexpect(Res);\n    } else {\n      std::get<3>(Section) = std::move(*Res);\n    }\n  }\n  return {};\n}\n\n} // namespace Loader\n} // namespace WasmEdge\n\n// Path: lib/loader/ast/component.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/loader.h\"\n#include \"spdlog/common.h\"\n#include \"spdlog/spdlog.h\"\n\n#include <cstdint>\n#include <memory>\n#include <utility>\n#include <vector>\n\nnamespace WasmEdge {\nnamespace Loader {\n\nExpect<std::pair<std::vector<Byte>, std::vector<Byte>>> Loader::loadPreamble() {\n  // component ::= <preamble> s*:<section>* => (component flatten(s*))\n  // preamble  ::= <magic> <version> <layer>\n  // magic     ::= 0x00 0x61 0x73 0x6D\n  // version   ::= 0x0a 0x00\n  // layer     ::= 0x01 0x00\n\n  // The combination of version and layer is corresponding to the version of\n  // core wasm.\n  // The core module has same magic but the different version:\n  // 0x01 0x00 0x00 0x00\n  auto Magic = FMgr.readBytes(4);\n  if (!Magic) {\n    return logLoadError(Magic.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Component);\n  }\n  std::vector<Byte> WasmMagic = {0x00, 0x61, 0x73, 0x6D};\n  if (*Magic != WasmMagic) {\n    spdlog::error(\"Might an invalid wasm file\");\n    return logLoadError(ErrCode::Value::MalformedMagic, FMgr.getLastOffset(),\n                        ASTNodeAttr::Component);\n  }\n  auto Ver = FMgr.readBytes(4);\n  if (!Ver) {\n    return logLoadError(Ver.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Component);\n  }\n  return std::make_pair(*Magic, *Ver);\n}\n\nExpect<std::variant<std::unique_ptr<AST::Component::Component>,\n                    std::unique_ptr<AST::Module>>>\nLoader::loadUnit() {\n  auto ResPreamble = Loader::loadPreamble();\n  if (!ResPreamble) {\n    return Unexpect(ResPreamble);\n  }\n  auto WasmMagic = ResPreamble->first;\n  auto Ver = ResPreamble->second;\n  if (Ver == ModuleVersion) {\n    auto Mod = std::make_unique<AST::Module>();\n    Mod->getMagic() = WasmMagic;\n    Mod->getVersion() = Ver;\n    if (!Conf.getRuntimeConfigure().isForceInterpreter()) {\n      if (auto Res = loadModuleAOT(Mod->getAOTSection()); !Res) {\n        return Unexpect(Res);\n      }\n    }\n    // Seek to the position after the binary header.\n    FMgr.seek(8);\n    if (auto Res = loadModule(*Mod); !Res) {\n      return Unexpect(Res);\n    }\n\n    // Load library from AOT Section for the universal WASM case.\n    // For the force interpreter mode, skip this.\n    if (!Conf.getRuntimeConfigure().isForceInterpreter() &&\n        WASMType == InputType::UniversalWASM) {\n      if (auto Res = loadUniversalWASM(*Mod); !Res) {\n        return Unexpect(Res);\n      }\n    }\n    return Mod;\n  } else if (Ver == ComponentVersion) {\n    if (!Conf.hasProposal(Proposal::Component)) {\n      return logNeedProposal(ErrCode::Value::IllegalOpCode, Proposal::Component,\n                             FMgr.getLastOffset(), ASTNodeAttr::Component);\n    }\n    spdlog::warn(\"component model is an experimental proposal\");\n    auto Comp = std::make_unique<AST::Component::Component>();\n    Comp->getMagic() = WasmMagic;\n    Comp->getVersion() = {Ver[0], Ver[1]};\n    Comp->getLayer() = {Ver[2], Ver[3]};\n    if (auto Res = loadComponent(*Comp); !Res) {\n      return Unexpect(Res);\n    }\n    return Comp;\n  } else {\n    return logLoadError(ErrCode::Value::MalformedVersion, FMgr.getLastOffset(),\n                        ASTNodeAttr::Component);\n  }\n}\n\nExpect<void> Loader::loadComponent(AST::Component::Component &Comp) {\n  using namespace AST::Component;\n\n  while (auto ResSecId = FMgr.readByte()) {\n    if (!ResSecId) {\n      return logLoadError(ResSecId.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Component);\n    }\n    // keep going only if we have new section ID\n    uint8_t NewSectionId = *ResSecId;\n\n    switch (NewSectionId) {\n    case 0x00:\n      Comp.getSections().emplace_back();\n      if (auto Res = loadSection(\n              Comp.getSections().back().emplace<AST::CustomSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    case 0x01:\n      Comp.getSections().emplace_back();\n      if (auto Res = loadSection(\n              Comp.getSections().back().emplace<AST::CoreModuleSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    case 0x02: {\n      Comp.getSections().emplace_back();\n      if (auto Res = loadSection(\n              Comp.getSections().back().emplace<CoreInstanceSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x03: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<CoreTypeSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x04:\n      Comp.getSections().emplace_back();\n      if (auto Res = loadSection(\n              Comp.getSections().back().emplace<ComponentSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    case 0x05: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<InstanceSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x06: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<AliasSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x07: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<TypeSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x08: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<CanonSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x09: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<StartSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x0A: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<ImportSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    case 0x0B: {\n      Comp.getSections().emplace_back();\n      if (auto Res =\n              loadSection(Comp.getSections().back().emplace<ExportSection>());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Component));\n        return Unexpect(Res);\n      }\n      break;\n    }\n    default:\n      return logLoadError(ErrCode::Value::MalformedSection,\n                          FMgr.getLastOffset(), ASTNodeAttr::Component);\n    }\n  }\n\n  return {};\n}\n\n} // namespace Loader\n} // namespace WasmEdge\n\n// Path: lib/loader/ast/expression.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/loader.h\"\n\n#include <utility>\n\nnamespace WasmEdge {\nnamespace Loader {\n\n// Load to construct Expression node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadExpression(AST::Expression &Expr,\n                                    std::optional<uint64_t> SizeBound) {\n  if (auto Res = loadInstrSeq(SizeBound)) {\n    // For the section size mismatch case, check in caller.\n    Expr.getInstrs() = std::move(*Res);\n  } else {\n    spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Expression));\n    return Unexpect(Res);\n  }\n  return {};\n}\n\n} // namespace Loader\n} // namespace WasmEdge\n\n// Path: lib/loader/ast/description.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/loader.h\"\n\nnamespace WasmEdge {\nnamespace Loader {\n\n// Load binary of Import description. See \"include/loader/loader.h\".\nExpect<void> Loader::loadDesc(AST::ImportDesc &ImpDesc) {\n  // Read the module name.\n  if (auto Res = FMgr.readName()) {\n    ImpDesc.setModuleName(*Res);\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Desc_Import);\n  }\n\n  // Read the external name.\n  if (auto Res = FMgr.readName()) {\n    ImpDesc.setExternalName(*Res);\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Desc_Import);\n  }\n\n  // Read the external type.\n  if (auto Res = FMgr.readByte()) {\n    ImpDesc.setExternalType(static_cast<ExternalType>(*Res));\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Desc_Import);\n  }\n\n  // Make content node according to external type.\n  switch (ImpDesc.getExternalType()) {\n  case ExternalType::Function: {\n    // Read the function type index.\n    if (auto Res = FMgr.readU32()) {\n      ImpDesc.setExternalFuncTypeIdx(*Res);\n    } else {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Desc_Import);\n    }\n    break;\n  }\n  case ExternalType::Table: {\n    // Read the table type node.\n    return loadType(ImpDesc.getExternalTableType());\n  }\n  case ExternalType::Memory: {\n    // Read the memory type node.\n    return loadType(ImpDesc.getExternalMemoryType());\n  }\n  case ExternalType::Global: {\n    // Read the global type node.\n    if (auto Res = loadType(ImpDesc.getExternalGlobalType()); !Res) {\n      return Unexpect(Res.error());\n    }\n    // Import the mutable globals are for ImportExportMutGlobals proposal.\n    if (ImpDesc.getExternalGlobalType().getValMut() == ValMut::Var &&\n        unlikely(!Conf.hasProposal(Proposal::ImportExportMutGlobals))) {\n      return logNeedProposal(ErrCode::Value::InvalidMut,\n                             Proposal::ImportExportMutGlobals,\n                             FMgr.getLastOffset(), ASTNodeAttr::Desc_Import);\n    }\n    return {};\n  }\n  default:\n    return logLoadError(ErrCode::Value::MalformedImportKind,\n                        FMgr.getLastOffset(), ASTNodeAttr::Desc_Import);\n  }\n  return {};\n}\n\n// Load binary of Export description. See \"include/loader/loader.h\".\nExpect<void> Loader::loadDesc(AST::ExportDesc &ExpDesc) {\n  // Read external name to export.\n  if (auto Res = FMgr.readName()) {\n    ExpDesc.setExternalName(*Res);\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Desc_Export);\n  }\n\n  // Read external type.\n  if (auto Res = FMgr.readByte()) {\n    ExpDesc.setExternalType(static_cast<ExternalType>(*Res));\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Desc_Export);\n  }\n  switch (ExpDesc.getExternalType()) {\n  case ExternalType::Function:\n  case ExternalType::Table:\n  case ExternalType::Memory:\n  case ExternalType::Global:\n    break;\n  default:\n    return logLoadError(ErrCode::Value::MalformedExportKind,\n                        FMgr.getLastOffset(), ASTNodeAttr::Desc_Export);\n  }\n\n  // Read external index to export.\n  if (auto Res = FMgr.readU32()) {\n    ExpDesc.setExternalIndex(*Res);\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Desc_Export);\n  }\n  return {};\n}\n\n} // namespace Loader\n} // namespace WasmEdge\n\n// Path: lib/loader/ast/type.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/loader.h\"\n\n#include <cstdint>\n\nnamespace WasmEdge {\nnamespace Loader {\n\n// Load binary and decode HeapType. See \"include/loader/loader.h\".\nExpect<ValType> Loader::loadHeapType(TypeCode TC, ASTNodeAttr From) {\n  if (auto Res = FMgr.readS33()) {\n    if (*Res < 0) {\n      // FuncRef or ExternRef case.\n      TypeCode HTCode =\n          static_cast<TypeCode>(static_cast<uint8_t>((*Res) & INT64_C(0x7F)));\n      switch (HTCode) {\n      case TypeCode::ExternRef:\n        // For the ref.func instruction, the immediate changed to store the heap\n        // type directly instead of the reference type after applying the\n        // typed function reference proposal. Therefore the reference-types\n        // proposal should be checked here.\n        if (!Conf.hasProposal(Proposal::ReferenceTypes)) {\n          return logNeedProposal(ErrCode::Value::MalformedElemType,\n                                 Proposal::ReferenceTypes, FMgr.getLastOffset(),\n                                 From);\n        }\n        [[fallthrough]];\n      case TypeCode::FuncRef:\n        return ValType(TC, HTCode);\n      case TypeCode::NullFuncRef:\n      case TypeCode::NullExternRef:\n      case TypeCode::NullRef:\n      case TypeCode::AnyRef:\n      case TypeCode::EqRef:\n      case TypeCode::I31Ref:\n      case TypeCode::StructRef:\n      case TypeCode::ArrayRef:\n        if (!Conf.hasProposal(Proposal::GC)) {\n          return logNeedProposal(ErrCode::Value::MalformedRefType, Proposal::GC,\n                                 FMgr.getLastOffset(), From);\n        }\n        return ValType(TC, HTCode);\n      default:\n        return logLoadError(ErrCode::Value::MalformedRefType,\n                            FMgr.getLastOffset(), From);\n      }\n    } else {\n      // Type index case. Legal if the function reference proposal is enabled.\n      if (!Conf.hasProposal(Proposal::FunctionReferences)) {\n        return logNeedProposal(ErrCode::Value::MalformedRefType,\n                               Proposal::FunctionReferences,\n                               FMgr.getLastOffset(), From);\n      }\n      return ValType(TC, static_cast<uint32_t>(*Res));\n    }\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(), From);\n  }\n}\n\n// Load binary and decode RefType. See \"include/loader/loader.h\".\nExpect<ValType> Loader::loadRefType(ASTNodeAttr From) {\n  if (auto Res = FMgr.readByte()) {\n    // The error code is different when the reference-types proposal turned off.\n    ErrCode::Value FailCode = Conf.hasProposal(Proposal::ReferenceTypes)\n                                  ? ErrCode::Value::MalformedRefType\n                                  : ErrCode::Value::MalformedElemType;\n    TypeCode Code = static_cast<TypeCode>(*Res);\n    switch (Code) {\n    case TypeCode::ExternRef:\n      if (!Conf.hasProposal(Proposal::ReferenceTypes)) {\n        return logNeedProposal(FailCode, Proposal::ReferenceTypes,\n                               FMgr.getLastOffset(), From);\n      }\n      [[fallthrough]];\n    case TypeCode::FuncRef:\n      // The FuncRef (0x70) is always allowed in the RefType even if the\n      // reference-types proposal not enabled.\n      return ValType(Code);\n    case TypeCode::NullFuncRef:\n    case TypeCode::NullExternRef:\n    case TypeCode::NullRef:\n    case TypeCode::AnyRef:\n    case TypeCode::EqRef:\n    case TypeCode::I31Ref:\n    case TypeCode::StructRef:\n    case TypeCode::ArrayRef:\n      if (!Conf.hasProposal(Proposal::GC)) {\n        return logNeedProposal(FailCode, Proposal::GC, FMgr.getLastOffset(),\n                               From);\n      }\n      return ValType(Code);\n    case TypeCode::Ref:\n    case TypeCode::RefNull: {\n      if (!Conf.hasProposal(Proposal::FunctionReferences)) {\n        return logNeedProposal(FailCode, Proposal::FunctionReferences,\n                               FMgr.getLastOffset(), From);\n      }\n      return loadHeapType(Code, From);\n    }\n    default:\n      return logLoadError(FailCode, FMgr.getLastOffset(), From);\n    }\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(), From);\n  }\n}\n\n// Load binary and decode ValType. See \"include/loader/loader.h\".\nExpect<ValType> Loader::loadValType(ASTNodeAttr From, bool IsStorageType) {\n  if (auto Res = FMgr.readByte()) {\n    TypeCode Code = static_cast<TypeCode>(*Res);\n    switch (Code) {\n    case TypeCode::V128:\n      if (!Conf.hasProposal(Proposal::SIMD)) {\n        return logNeedProposal(ErrCode::Value::MalformedValType, Proposal::SIMD,\n                               FMgr.getLastOffset(), From);\n      }\n      [[fallthrough]];\n    case TypeCode::I32:\n    case TypeCode::I64:\n    case TypeCode::F32:\n    case TypeCode::F64:\n      return ValType(Code);\n    case TypeCode::I8:\n    case TypeCode::I16:\n      if (!IsStorageType) {\n        break;\n      }\n      if (!Conf.hasProposal(Proposal::GC)) {\n        return logNeedProposal(ErrCode::Value::MalformedValType, Proposal::GC,\n                               FMgr.getLastOffset(), From);\n      }\n      return ValType(Code);\n    case TypeCode::FuncRef:\n      if (!Conf.hasProposal(Proposal::ReferenceTypes) &&\n          !Conf.hasProposal(Proposal::BulkMemoryOperations)) {\n        return logNeedProposal(ErrCode::Value::MalformedElemType,\n                               Proposal::ReferenceTypes, FMgr.getLastOffset(),\n                               From);\n      }\n      return ValType(Code);\n    case TypeCode::ExternRef:\n      if (!Conf.hasProposal(Proposal::ReferenceTypes)) {\n        return logNeedProposal(ErrCode::Value::MalformedElemType,\n                               Proposal::ReferenceTypes, FMgr.getLastOffset(),\n                               From);\n      }\n      return ValType(Code);\n    case TypeCode::NullFuncRef:\n    case TypeCode::NullExternRef:\n    case TypeCode::NullRef:\n    case TypeCode::AnyRef:\n    case TypeCode::EqRef:\n    case TypeCode::I31Ref:\n    case TypeCode::StructRef:\n    case TypeCode::ArrayRef:\n      if (!Conf.hasProposal(Proposal::GC)) {\n        return logNeedProposal(ErrCode::Value::MalformedValType, Proposal::GC,\n                               FMgr.getLastOffset(), From);\n      }\n      return ValType(Code);\n    case TypeCode::Ref:\n    case TypeCode::RefNull:\n      if (!Conf.hasProposal(Proposal::FunctionReferences)) {\n        return logNeedProposal(ErrCode::Value::MalformedValType,\n                               Proposal::FunctionReferences,\n                               FMgr.getLastOffset(), From);\n      }\n      return loadHeapType(Code, From);\n    default:\n      break;\n    }\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(), From);\n  }\n  return logLoadError(ErrCode::Value::MalformedValType, FMgr.getLastOffset(),\n                      From);\n}\n\nExpect<ValMut> Loader::loadMutability(ASTNodeAttr From) {\n  if (auto Res = FMgr.readByte()) {\n    switch (static_cast<ValMut>(*Res)) {\n    case ValMut::Const:\n    case ValMut::Var:\n      return static_cast<ValMut>(*Res);\n    default:\n      return logLoadError(ErrCode::Value::InvalidMut, FMgr.getLastOffset(),\n                          From);\n    }\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(), From);\n  }\n}\n\nExpect<void> Loader::loadFieldType(AST::FieldType &FType) {\n  if (auto Res = loadValType(ASTNodeAttr::Type_Rec, true)) {\n    FType.setStorageType(*Res);\n  } else {\n    // The error code logging is handled.\n    return Unexpect(Res);\n  }\n  if (auto Res = loadMutability(ASTNodeAttr::Type_Rec)) {\n    FType.setValMut(*Res);\n  } else {\n    // The error code logging is handled.\n    return Unexpect(Res);\n  }\n  return {};\n}\n\nExpect<void> Loader::loadCompositeType(AST::CompositeType &CType) {\n  if (auto CodeByte = FMgr.readByte()) {\n    switch (static_cast<TypeCode>(*CodeByte)) {\n    case TypeCode::Array: {\n      AST::FieldType FType;\n      if (auto Res = loadFieldType(FType); unlikely(!Res)) {\n        return Unexpect(Res);\n      }\n      CType.setArrayType(std::move(FType));\n      return {};\n    }\n    case TypeCode::Struct: {\n      std::vector<AST::FieldType> FList;\n      if (auto Res = loadVec<AST::SubType>(\n              FList,\n              [this](AST::FieldType &FType) -> Expect<void> {\n                // The error code logging is handled.\n                return loadFieldType(FType);\n              });\n          !Res) {\n        return Unexpect(Res);\n      }\n      CType.setStructType(std::move(FList));\n      return {};\n    }\n    case TypeCode::Func: {\n      AST::FunctionType FuncType;\n      if (auto Res = loadType(FuncType); unlikely(!Res)) {\n        return Unexpect(Res);\n      }\n      CType.setFunctionType(std::move(FuncType));\n      return {};\n    }\n    default:\n      return logLoadError(ErrCode::Value::IntegerTooLong, FMgr.getLastOffset(),\n                          ASTNodeAttr::Type_Rec);\n    }\n  } else {\n    return logLoadError(CodeByte.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Type_Rec);\n  }\n}\n\n// Load binary to construct Limit node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadLimit(AST::Limit &Lim) {\n  // Read limit.\n  if (auto Res = FMgr.readByte()) {\n    switch (static_cast<AST::Limit::LimitType>(*Res)) {\n    case AST::Limit::LimitType::HasMin:\n      Lim.setType(AST::Limit::LimitType::HasMin);\n      break;\n    case AST::Limit::LimitType::HasMinMax:\n      Lim.setType(AST::Limit::LimitType::HasMinMax);\n      break;\n    case AST::Limit::LimitType::SharedNoMax:\n      if (Conf.hasProposal(Proposal::Threads)) {\n        return logLoadError(ErrCode::Value::SharedMemoryNoMax,\n                            FMgr.getLastOffset(), ASTNodeAttr::Type_Limit);\n      } else {\n        return logLoadError(ErrCode::Value::IntegerTooLarge,\n                            FMgr.getLastOffset(), ASTNodeAttr::Type_Limit);\n      }\n    case AST::Limit::LimitType::Shared:\n      Lim.setType(AST::Limit::LimitType::Shared);\n      break;\n    default:\n      if (*Res == 0x80 || *Res == 0x81) {\n        // LEB128 cases will fail.\n        return logLoadError(ErrCode::Value::IntegerTooLong,\n                            FMgr.getLastOffset(), ASTNodeAttr::Type_Limit);\n      } else {\n        return logLoadError(ErrCode::Value::IntegerTooLarge,\n                            FMgr.getLastOffset(), ASTNodeAttr::Type_Limit);\n      }\n    }\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Type_Limit);\n  }\n\n  // Read min and max number.\n  if (auto Res = FMgr.readU32()) {\n    Lim.setMin(*Res);\n    Lim.setMax(*Res);\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Type_Limit);\n  }\n  if (Lim.hasMax()) {\n    if (auto Res = FMgr.readU32()) {\n      Lim.setMax(*Res);\n    } else {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Type_Limit);\n    }\n  }\n  return {};\n}\n\n// Load binary to construct SubType node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadType(AST::SubType &SType) {\n  if (auto CodeByte = FMgr.peekByte()) {\n    switch (static_cast<TypeCode>(*CodeByte)) {\n    default:\n      // Case: comptype.\n      SType.setFinal(true);\n      return loadCompositeType(SType.getCompositeType());\n    case TypeCode::Sub:\n      // Case: 0x50 vec(typeidx) comptype.\n      SType.setFinal(false);\n      break;\n    case TypeCode::SubFinal:\n      // Case: 0x4F vec(typeidx) comptype.\n      SType.setFinal(true);\n      break;\n    }\n    FMgr.readByte();\n    if (auto Res = loadVec<AST::SubType>(\n            SType.getSuperTypeIndices(),\n            [this](uint32_t &Idx) -> Expect<void> {\n              if (auto Num = FMgr.readU32()) {\n                Idx = *Num;\n              } else {\n                return logLoadError(Num.error(), FMgr.getLastOffset(),\n                                    ASTNodeAttr::Type_Sub);\n              }\n              return {};\n            });\n        !Res) {\n      return Unexpect(Res);\n    }\n    return loadCompositeType(SType.getCompositeType());\n  } else {\n    return logLoadError(CodeByte.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Type_Rec);\n  }\n}\n\n// Load binary to construct FunctionType node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadType(AST::FunctionType &FuncType) {\n  // Read type of Func (0x60). Moved into the composite type.\n  auto LoadValType = [this](ValType &VT) -> Expect<void> {\n    if (auto Res = loadValType(ASTNodeAttr::Type_Function)) {\n      VT = *Res;\n    } else {\n      // The error code logging is handled.\n      return Unexpect(Res);\n    }\n    return {};\n  };\n  // Read vector of parameter types.\n  if (auto Res =\n          loadVec<AST::FunctionType>(FuncType.getParamTypes(), LoadValType);\n      !Res) {\n    return Unexpect(Res);\n  }\n\n  // Read vector of result types.\n  if (auto Res =\n          loadVec<AST::FunctionType>(FuncType.getReturnTypes(), LoadValType);\n      !Res) {\n    return Unexpect(Res);\n  }\n\n  if (unlikely(!Conf.hasProposal(Proposal::MultiValue)) &&\n      FuncType.getReturnTypes().size() > 1) {\n    return logNeedProposal(ErrCode::Value::MalformedValType,\n                           Proposal::MultiValue, FMgr.getLastOffset(),\n                           ASTNodeAttr::Type_Function);\n  }\n  return {};\n}\n\n// Load binary to construct MemoryType node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadType(AST::MemoryType &MemType) {\n  // Read limit.\n  if (auto Res = loadLimit(MemType.getLimit()); !Res) {\n    spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Type_Memory));\n    return Unexpect(Res);\n  }\n  return {};\n}\n\n// Load binary to construct TableType node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadType(AST::TableType &TabType) {\n  // Read reference type.\n  if (auto Res = loadRefType(ASTNodeAttr::Type_Table)) {\n    TabType.setRefType(*Res);\n  } else {\n    // The AST node information is handled.\n    return Unexpect(Res);\n  }\n\n  // Read limit.\n  if (auto Res = loadLimit(TabType.getLimit()); !Res) {\n    spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Type_Table));\n    return Unexpect(Res);\n  }\n  return {};\n}\n\n// Load binary to construct GlobalType node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadType(AST::GlobalType &GlobType) {\n  // Read value type.\n  if (auto Res = loadValType(ASTNodeAttr::Type_Global)) {\n    GlobType.setValType(*Res);\n  } else {\n    // The AST node information is handled.\n    return Unexpect(Res);\n  }\n\n  // Read mutability.\n  if (auto Res = loadMutability(ASTNodeAttr::Type_Global)) {\n    GlobType.setValMut(*Res);\n  } else {\n    // The AST node information is handled.\n    return Unexpect(Res);\n  }\n  return {};\n}\n\n} // namespace Loader\n} // namespace WasmEdge\n\n// Path: lib/loader/ast/segment.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/loader.h\"\n\n#include <cstdint>\n#include <utility>\n\nnamespace WasmEdge {\nnamespace Loader {\n\n// Load binary of TableSegment node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSegment(AST::TableSegment &TabSeg) {\n  // Check the first byte is the reftype in table type or not.\n  if (auto CheckByte = FMgr.peekByte()) {\n    if (*CheckByte == 0x40U) {\n      // Table segment case is for FunctionReferences proposal.\n      if (!Conf.hasProposal(Proposal::FunctionReferences)) {\n        return logNeedProposal(ErrCode::Value::MalformedTable,\n                               Proposal::FunctionReferences,\n                               FMgr.getLastOffset(), ASTNodeAttr::Seg_Table);\n      }\n      FMgr.readByte();\n\n      // Check the second byte.\n      if (auto Res = FMgr.readByte()) {\n        if (*Res != 0x00U) {\n          return logLoadError(ErrCode::Value::MalformedTable,\n                              FMgr.getLastOffset(), ASTNodeAttr::Seg_Table);\n        }\n      } else {\n        return logLoadError(Res.error(), FMgr.getLastOffset(),\n                            ASTNodeAttr::Seg_Table);\n      }\n\n      // Read the table type.\n      if (auto Res = loadType(TabSeg.getTableType()); unlikely(!Res)) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Table));\n        return Unexpect(Res);\n      }\n\n      // Read the expression.\n      if (auto Res = loadExpression(TabSeg.getExpr()); unlikely(!Res)) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Global));\n        return Unexpect(Res);\n      }\n    } else {\n      // The table type case.\n      if (auto Res = loadType(TabSeg.getTableType()); unlikely(!Res)) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Table));\n        return Unexpect(Res);\n      }\n    }\n  } else {\n    return logLoadError(CheckByte.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Seg_Table);\n  }\n\n  return {};\n}\n\n// Load binary of GlobalSegment node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSegment(AST::GlobalSegment &GlobSeg) {\n  // Read global type node.\n  if (auto Res = loadType(GlobSeg.getGlobalType()); unlikely(!Res)) {\n    spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Global));\n    return Unexpect(Res);\n  }\n\n  // Read the expression.\n  if (auto Res = loadExpression(GlobSeg.getExpr()); unlikely(!Res)) {\n    spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Global));\n    return Unexpect(Res);\n  }\n\n  return {};\n}\n\n// Load binary of ElementSegment node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSegment(AST::ElementSegment &ElemSeg) {\n  // Element segment binary format:\n  // ---------------------------------------------------------------------------\n  //  Mode | TableIdx | OffExpr | ElemKind | RefType | vec(FuncIdx) | vec(expr)\n  // ------|----------|---------|----------|---------|--------------|-----------\n  //    0  |          |    v    |          |         |       v      |\n  //    1  |          |         |    v     |         |       v      |\n  //    2  |    v     |    v    |    v     |         |       v      |\n  //    3  |          |         |    v     |         |       v      |\n  //    4  |          |    v    |          |         |              |     v\n  //    5  |          |         |          |    v    |              |     v\n  //    6  |    v     |    v    |          |    v    |              |     v\n  //    7  |          |         |          |    v    |              |     v\n  // ---------------------------------------------------------------------------\n  // Mode: element initial integer, u32\n  // TableIdx: target table index, u32\n  // OffExpr: init offset expression, expr\n  // ElemKind: byte 0x00, RefType::FuncRef\n  // RefType: reference type, RefType\n  // vec(FuncIdx): function index vector, vec(u32)\n  // vec(expr): reference init list, vec(expr)\n\n  // Read the checking byte.\n  uint32_t Check;\n  if (auto Res = FMgr.readU32()) {\n    Check = *Res;\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Seg_Element);\n  }\n  // Check > 0 cases are for BulkMemoryOperations or ReferenceTypes proposal.\n  if (Check > 0 && !Conf.hasProposal(Proposal::BulkMemoryOperations) &&\n      !Conf.hasProposal(Proposal::ReferenceTypes)) {\n    return logNeedProposal(ErrCode::Value::ExpectedZeroByte,\n                           Proposal::BulkMemoryOperations, FMgr.getLastOffset(),\n                           ASTNodeAttr::Seg_Element);\n  }\n\n  // Check the prefix byte.\n  switch (Check) {\n  case 0x00:\n  case 0x02:\n  case 0x04:\n  case 0x06:\n    ElemSeg.setMode(AST::ElementSegment::ElemMode::Active);\n    break;\n\n  case 0x01:\n  case 0x05:\n    ElemSeg.setMode(AST::ElementSegment::ElemMode::Passive);\n    break;\n\n  case 0x03:\n  case 0x07:\n    ElemSeg.setMode(AST::ElementSegment::ElemMode::Declarative);\n    break;\n\n  default:\n    // TODO: Correctness the error code once there's spec test.\n    return logLoadError(ErrCode::Value::IllegalGrammar, FMgr.getLastOffset(),\n                        ASTNodeAttr::Seg_Element);\n  }\n\n  // Read the table index.\n  ElemSeg.setIdx(0);\n  switch (Check) {\n  case 0x02:\n  case 0x06:\n    if (auto Res = FMgr.readU32()) {\n      ElemSeg.setIdx(*Res);\n    } else {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Seg_Element);\n    }\n    break;\n\n  default:\n    break;\n  }\n\n  // Read the expression.\n  switch (Check) {\n  case 0x00:\n  case 0x02:\n  case 0x04:\n  case 0x06:\n    if (auto Res = loadExpression(ElemSeg.getExpr()); unlikely(!Res)) {\n      spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Element));\n      return Unexpect(Res);\n    }\n    break;\n\n  default:\n    break;\n  }\n\n  // Read element kind and init function indices.\n  ElemSeg.setRefType(TypeCode::FuncRef);\n  switch (Check) {\n  case 0x01:\n  case 0x02:\n  case 0x03:\n    if (auto Res = FMgr.readByte()) {\n      if (*Res != 0x00U) {\n        return logLoadError(ErrCode::Value::ExpectedZeroByte,\n                            FMgr.getLastOffset(), ASTNodeAttr::Seg_Element);\n      }\n    } else {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Seg_Element);\n    }\n    [[fallthrough]];\n\n  case 0x00: {\n    uint32_t VecCnt = 0;\n    if (auto Res = FMgr.readU32()) {\n      VecCnt = *Res;\n    } else {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Seg_Element);\n    }\n    for (uint32_t I = 0; I < VecCnt; ++I) {\n      // For each element in vec(funcidx), make expr(ref.func idx end).\n      ElemSeg.getInitExprs().emplace_back();\n      AST::Instruction RefFunc(OpCode::Ref__func);\n      AST::Instruction End(OpCode::End);\n      if (auto Res = loadInstruction(RefFunc); unlikely(!Res)) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Element));\n        return Unexpect(Res);\n      }\n      ElemSeg.getInitExprs().back().getInstrs().emplace_back(\n          std::move(RefFunc));\n      ElemSeg.getInitExprs().back().getInstrs().emplace_back(std::move(End));\n    }\n    break;\n  }\n  default:\n    break;\n  }\n\n  // Read the reference type and init expressions.\n  switch (Check) {\n  case 0x05:\n  case 0x06:\n  case 0x07:\n    if (auto Res = loadRefType(ASTNodeAttr::Seg_Element)) {\n      ElemSeg.setRefType(*Res);\n    } else {\n      // The AST node information is handled.\n      return Unexpect(Res);\n    }\n    [[fallthrough]];\n  case 0x04: {\n    return loadVec<AST::ElementSegment>(\n        ElemSeg.getInitExprs(), [this](AST::Expression &Expr) -> Expect<void> {\n          return loadExpression(Expr);\n        });\n  }\n\n  default:\n    break;\n  }\n\n  return {};\n}\n\n// Load binary of CodeSegment node. See \"include/loader/loader.h\".\nExpect<void> Loader::loadSegment(AST::CodeSegment &CodeSeg) {\n  // Read the code segment size.\n  if (auto Res = FMgr.readU32()) {\n    CodeSeg.setSegSize(*Res);\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Seg_Code);\n  }\n  auto ExprSizeBound = FMgr.getOffset() + CodeSeg.getSegSize();\n\n  // Read the vector of local variable counts and types.\n  uint32_t VecCnt = 0;\n  if (auto Res = loadVecCnt()) {\n    VecCnt = *Res;\n  } else {\n    return logLoadError(Res.error(), FMgr.getLastOffset(),\n                        ASTNodeAttr::Seg_Element);\n  }\n  CodeSeg.getLocals().clear();\n  CodeSeg.getLocals().reserve(VecCnt);\n  uint32_t TotalLocalCnt = 0;\n  for (uint32_t I = 0; I < VecCnt; ++I) {\n    uint32_t LocalCnt = 0;\n    ValType LocalType;\n    if (auto Res = FMgr.readU32(); unlikely(!Res)) {\n      return logLoadError(Res.error(), FMgr.getLastOffset(),\n                          ASTNodeAttr::Seg_Code);\n...\n", "needle_token_start": 2416, "needle_token_end": 2446, "code_context_ntokens": 16264, "output": ["```cpp\ninline constexpr uint32_t HostVersion() noexcept {\n  return WasmEdge::AOT::kBinaryVersion;\n}\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "toLLVMConstantZero", "language": "cpp", "path": "lib/llvm/compiler.cpp", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function generates a zero-initialized constant value corresponding to a specified data type in the LLVM context.\n2. **Input**: It takes a context for LLVM operations and a value type that specifies the data type for which a zero value is needed.\n3. **Output**: It returns an LLVM value that represents a zero-initialized constant of the specified data type.\n4. **Procedure**: The function checks the code of the input value type and, based on this code, it creates and returns a zero-initialized constant using the appropriate LLVM data type. For integer types, vector types, and floating-point types, it uses specific LLVM functions to create a null (zero) value of the corresponding type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "  LLVM::Type Int64PtrTy;\n  LLVM::Type Int128PtrTy;\n  LLVM::Type ExecCtxTy;\n  LLVM::Type ExecCtxPtrTy;\n  LLVM::Type IntrinsicsTableTy;\n  LLVM::Type IntrinsicsTablePtrTy;\n  LLVM::Message SubtargetFeatures;\n\n#if defined(__x86_64__)\n#if defined(__XOP__)\n  bool SupportXOP = true;\n#else\n  bool SupportXOP = false;\n#endif\n\n#if defined(__SSE4_1__)\n  bool SupportSSE4_1 = true;\n#else\n  bool SupportSSE4_1 = false;\n#endif\n\n#if defined(__SSSE3__)\n  bool SupportSSSE3 = true;\n#else\n  bool SupportSSSE3 = false;\n#endif\n\n#if defined(__SSE2__)\n  bool SupportSSE2 = true;\n#else\n  bool SupportSSE2 = false;\n#endif\n#endif\n\n#if defined(__aarch64__)\n#if defined(__ARM_NEON__) || defined(__ARM_NEON) || defined(__ARM_NEON_FP)\n  bool SupportNEON = true;\n#else\n  bool SupportNEON = false;\n#endif\n#endif\n\n  std::vector<const AST::FunctionType *> FunctionTypes;\n  std::vector<LLVM::Value> FunctionWrappers;\n  std::vector<std::tuple<uint32_t, LLVM::FunctionCallee,\n                         const WasmEdge::AST::CodeSegment *>>\n      Functions;\n  std::vector<LLVM::Type> Globals;\n  LLVM::Value IntrinsicsTable;\n  LLVM::FunctionCallee Trap;\n  CompileContext(LLVM::Context C, LLVM::Module &M,\n                 bool IsGenericBinary) noexcept\n      : LLContext(C), LLModule(M),\n        Cold(LLVM::Attribute::createEnum(C, LLVM::Core::Cold, 0)),\n        NoAlias(LLVM::Attribute::createEnum(C, LLVM::Core::NoAlias, 0)),\n        NoInline(LLVM::Attribute::createEnum(C, LLVM::Core::NoInline, 0)),\n        NoReturn(LLVM::Attribute::createEnum(C, LLVM::Core::NoReturn, 0)),\n        ReadOnly(LLVM::Attribute::createEnum(C, LLVM::Core::ReadOnly, 0)),\n        StrictFP(LLVM::Attribute::createEnum(C, LLVM::Core::StrictFP, 0)),\n        UWTable(LLVM::Attribute::createEnum(C, LLVM::Core::UWTable,\n                                            LLVM::Core::UWTableDefault)),\n        NoStackArgProbe(\n            LLVM::Attribute::createString(C, \"no-stack-arg-probe\"sv, {})),\n        VoidTy(LLContext.getVoidTy()), Int8Ty(LLContext.getInt8Ty()),\n        Int16Ty(LLContext.getInt16Ty()), Int32Ty(LLContext.getInt32Ty()),\n        Int64Ty(LLContext.getInt64Ty()), Int128Ty(LLContext.getInt128Ty()),\n        FloatTy(LLContext.getFloatTy()), DoubleTy(LLContext.getDoubleTy()),\n        Int8x16Ty(LLVM::Type::getVectorType(Int8Ty, 16)),\n        Int16x8Ty(LLVM::Type::getVectorType(Int16Ty, 8)),\n        Int32x4Ty(LLVM::Type::getVectorType(Int32Ty, 4)),\n        Floatx4Ty(LLVM::Type::getVectorType(FloatTy, 4)),\n        Int64x2Ty(LLVM::Type::getVectorType(Int64Ty, 2)),\n        Doublex2Ty(LLVM::Type::getVectorType(DoubleTy, 2)),\n        Int128x1Ty(LLVM::Type::getVectorType(Int128Ty, 1)),\n        Int8PtrTy(Int8Ty.getPointerTo()), Int32PtrTy(Int32Ty.getPointerTo()),\n        Int64PtrTy(Int64Ty.getPointerTo()),\n        Int128PtrTy(Int128Ty.getPointerTo()),\n        ExecCtxTy(LLVM::Type::getStructType(\n            \"ExecCtx\",\n            std::initializer_list<LLVM::Type>{\n                // Memory\n                Int8PtrTy.getPointerTo(),\n                // Globals\n                Int128PtrTy.getPointerTo(),\n                // InstrCount\n                Int64PtrTy,\n                // CostTable\n                LLVM::Type::getArrayType(Int64Ty, UINT16_MAX + 1)\n                    .getPointerTo(),\n                // Gas\n                Int64PtrTy,\n                // GasLimit\n                Int64Ty,\n                // StopToken\n                Int32PtrTy,\n            })),\n        ExecCtxPtrTy(ExecCtxTy.getPointerTo()),\n        IntrinsicsTableTy(LLVM::Type::getArrayType(\n            Int8PtrTy,\n            static_cast<uint32_t>(Executable::Intrinsics::kIntrinsicMax))),\n        IntrinsicsTablePtrTy(IntrinsicsTableTy.getPointerTo()),\n        IntrinsicsTable(LLModule.addGlobal(IntrinsicsTablePtrTy, true,\n                                           LLVMExternalLinkage, LLVM::Value(),\n                                           \"intrinsics\")) {\n    Trap.Ty = LLVM::Type::getFunctionType(VoidTy, {Int32Ty});\n    Trap.Fn = LLModule.addFunction(Trap.Ty, LLVMPrivateLinkage, \"trap\");\n    Trap.Fn.setDSOLocal(true);\n    Trap.Fn.addFnAttr(NoStackArgProbe);\n    Trap.Fn.addFnAttr(StrictFP);\n    Trap.Fn.addFnAttr(UWTable);\n    Trap.Fn.addFnAttr(NoReturn);\n    Trap.Fn.addFnAttr(Cold);\n    Trap.Fn.addFnAttr(NoInline);\n\n    LLModule.addGlobal(Int32Ty, true, LLVMExternalLinkage,\n                       LLVM::Value::getConstInt(Int32Ty, AOT::kBinaryVersion),\n                       \"version\");\n\n    if (!IsGenericBinary) {\n      SubtargetFeatures = LLVM::getHostCPUFeatures();\n      auto Features = SubtargetFeatures.string_view();\n      while (!Features.empty()) {\n        std::string_view Feature;\n        if (auto Pos = Features.find(','); Pos != std::string_view::npos) {\n          Feature = Features.substr(0, Pos);\n          Features = Features.substr(Pos + 1);\n        } else {\n          Feature = std::exchange(Features, std::string_view());\n        }\n        if (Feature[0] != '+') {\n          continue;\n        }\n        Feature = Feature.substr(1);\n\n#if defined(__x86_64__)\n        if (!SupportXOP && Feature == \"xop\"sv) {\n          SupportXOP = true;\n        }\n        if (!SupportSSE4_1 && Feature == \"sse4.1\"sv) {\n          SupportSSE4_1 = true;\n        }\n        if (!SupportSSSE3 && Feature == \"ssse3\"sv) {\n          SupportSSSE3 = true;\n        }\n        if (!SupportSSE2 && Feature == \"sse2\"sv) {\n          SupportSSE2 = true;\n        }\n#elif defined(__aarch64__)\n        if (!SupportNEON && Feature == \"neon\"sv) {\n          SupportNEON = true;\n        }\n#endif\n      }\n    }\n\n    {\n      // create trap\n      LLVM::Builder Builder(LLContext);\n      Builder.positionAtEnd(\n          LLVM::BasicBlock::create(LLContext, Trap.Fn, \"entry\"));\n      auto FnTy = LLVM::Type::getFunctionType(VoidTy, {Int32Ty});\n      auto CallTrap = Builder.createCall(\n          getIntrinsic(Builder, Executable::Intrinsics::kTrap, FnTy),\n          {Trap.Fn.getFirstParam()});\n      CallTrap.addCallSiteAttribute(NoReturn);\n      Builder.createUnreachable();\n    }\n  }\n  LLVM::Value getMemory(LLVM::Builder &Builder, LLVM::Value ExecCtx,\n                        uint32_t Index) noexcept {\n    auto Array = Builder.createExtractValue(ExecCtx, 0);\n    auto VPtr = Builder.createLoad(\n        Int8PtrTy, Builder.createInBoundsGEP1(Int8PtrTy, Array,\n                                              LLContext.getInt64(Index)));\n    VPtr.setMetadata(LLContext, LLVM::Core::InvariantGroup,\n                     LLVM::Metadata(LLContext, {}));\n    return Builder.createBitCast(VPtr, Int8PtrTy);\n  }\n  std::pair<LLVM::Type, LLVM::Value> getGlobal(LLVM::Builder &Builder,\n                                               LLVM::Value ExecCtx,\n                                               uint32_t Index) noexcept {\n    auto Ty = Globals[Index];\n    auto Array = Builder.createExtractValue(ExecCtx, 1);\n    auto VPtr = Builder.createLoad(\n        Int128PtrTy, Builder.createInBoundsGEP1(Int8PtrTy, Array,\n                                                LLContext.getInt64(Index)));\n    VPtr.setMetadata(LLContext, LLVM::Core::InvariantGroup,\n                     LLVM::Metadata(LLContext, {}));\n    auto Ptr = Builder.createBitCast(VPtr, Ty.getPointerTo());\n    return {Ty, Ptr};\n  }\n  LLVM::Value getInstrCount(LLVM::Builder &Builder,\n                            LLVM::Value ExecCtx) noexcept {\n    return Builder.createExtractValue(ExecCtx, 2);\n  }\n  LLVM::Value getCostTable(LLVM::Builder &Builder,\n                           LLVM::Value ExecCtx) noexcept {\n    return Builder.createExtractValue(ExecCtx, 3);\n  }\n  LLVM::Value getGas(LLVM::Builder &Builder, LLVM::Value ExecCtx) noexcept {\n    return Builder.createExtractValue(ExecCtx, 4);\n  }\n  LLVM::Value getGasLimit(LLVM::Builder &Builder,\n                          LLVM::Value ExecCtx) noexcept {\n    return Builder.createExtractValue(ExecCtx, 5);\n  }\n  LLVM::Value getStopToken(LLVM::Builder &Builder,\n                           LLVM::Value ExecCtx) noexcept {\n    return Builder.createExtractValue(ExecCtx, 6);\n  }\n  LLVM::FunctionCallee getIntrinsic(LLVM::Builder &Builder,\n                                    Executable::Intrinsics Index,\n                                    LLVM::Type Ty) noexcept {\n    const auto Value = static_cast<uint32_t>(Index);\n    auto PtrTy = Ty.getPointerTo();\n    auto PtrPtrTy = PtrTy.getPointerTo();\n    auto IT = Builder.createLoad(IntrinsicsTablePtrTy, IntrinsicsTable);\n    IT.setMetadata(LLContext, LLVM::Core::InvariantGroup,\n                   LLVM::Metadata(LLContext, {}));\n    auto VPtr =\n        Builder.createInBoundsGEP2(IntrinsicsTableTy, IT, LLContext.getInt64(0),\n                                   LLContext.getInt64(Value));\n    auto Ptr = Builder.createBitCast(VPtr, PtrPtrTy);\n    return {Ty, Builder.createLoad(PtrTy, Ptr)};\n  }\n  std::pair<std::vector<ValType>, std::vector<ValType>>\n  resolveBlockType(const BlockType &BType) const noexcept {\n    using VecT = std::vector<ValType>;\n    using RetT = std::pair<VecT, VecT>;\n    if (BType.isEmpty()) {\n      return RetT{};\n    }\n    if (BType.isValType()) {\n      return RetT{{}, {BType.getValType()}};\n    } else {\n      // Type index case. t2* = type[index].returns\n      const uint32_t TypeIdx = BType.getTypeIndex();\n      const auto &FType = *FunctionTypes[TypeIdx];\n      return RetT{\n          VecT(FType.getParamTypes().begin(), FType.getParamTypes().end()),\n          VecT(FType.getReturnTypes().begin(), FType.getReturnTypes().end())};\n    }\n  }\n};\n\nnamespace {\n\nusing namespace WasmEdge;\n\nstatic bool isVoidReturn(Span<const ValType> ValTypes) noexcept {\n  return ValTypes.empty();\n}\n\nstatic LLVM::Type toLLVMType(LLVM::Context LLContext,\n                             const ValType &ValType) noexcept {\n  switch (ValType.getCode()) {\n  case TypeCode::I32:\n    return LLContext.getInt32Ty();\n  case TypeCode::I64:\n    return LLContext.getInt64Ty();\n  case TypeCode::Ref:\n  case TypeCode::RefNull:\n  case TypeCode::V128:\n    return LLVM::Type::getVectorType(LLContext.getInt64Ty(), 2);\n  case TypeCode::F32:\n    return LLContext.getFloatTy();\n  case TypeCode::F64:\n    return LLContext.getDoubleTy();\n  default:\n    assumingUnreachable();\n  }\n}\n\nstatic std::vector<LLVM::Type>\ntoLLVMTypeVector(LLVM::Context LLContext,\n                 Span<const ValType> ValTypes) noexcept {\n  std::vector<LLVM::Type> Result;\n  Result.reserve(ValTypes.size());\n  for (const auto &Type : ValTypes) {\n    Result.push_back(toLLVMType(LLContext, Type));\n  }\n  return Result;\n}\n\nstatic std::vector<LLVM::Type>\ntoLLVMArgsType(LLVM::Context LLContext, LLVM::Type ExecCtxPtrTy,\n               Span<const ValType> ValTypes) noexcept {\n  auto Result = toLLVMTypeVector(LLContext, ValTypes);\n  Result.insert(Result.begin(), ExecCtxPtrTy);\n  return Result;\n}\n\nstatic LLVM::Type toLLVMRetsType(LLVM::Context LLContext,\n                                 Span<const ValType> ValTypes) noexcept {\n  if (isVoidReturn(ValTypes)) {\n    return LLContext.getVoidTy();\n  }\n  if (ValTypes.size() == 1) {\n    return toLLVMType(LLContext, ValTypes.front());\n  }\n  std::vector<LLVM::Type> Result;\n  Result.reserve(ValTypes.size());\n  for (const auto &Type : ValTypes) {\n    Result.push_back(toLLVMType(LLContext, Type));\n  }\n  return LLVM::Type::getStructType(Result);\n}\n\nstatic LLVM::Type toLLVMType(LLVM::Context LLContext, LLVM::Type ExecCtxPtrTy,\n                             const AST::FunctionType &FuncType) noexcept {\n  auto ArgsTy =\n      toLLVMArgsType(LLContext, ExecCtxPtrTy, FuncType.getParamTypes());\n  auto RetTy = toLLVMRetsType(LLContext, FuncType.getReturnTypes());\n  return LLVM::Type::getFunctionType(RetTy, ArgsTy);\n}\n\n\nstatic LLVM::Value toLLVMConstantZero(LLVM::Context LLContext,\n                                      const ValType &ValType) noexcept {\n  switch (ValType.getCode()) {\n  case TypeCode::I32:\n    return LLVM::Value::getConstNull(LLContext.getInt32Ty());\n  case TypeCode::I64:\n    return LLVM::Value::getConstNull(LLContext.getInt64Ty());\n  case TypeCode::Ref:\n  case TypeCode::RefNull:\n  case TypeCode::V128:\n    return LLVM::Value::getConstNull(\n        LLVM::Type::getVectorType(LLContext.getInt64Ty(), 2));\n  case TypeCode::F32:\n    return LLVM::Value::getConstNull(LLContext.getFloatTy());\n  case TypeCode::F64:\n    return LLVM::Value::getConstNull(LLContext.getDoubleTy());\n  default:\n    assumingUnreachable();\n  }\n}\n\nclass FunctionCompiler {\n  struct Control;\n\npublic:\n  FunctionCompiler(LLVM::Compiler::CompileContext &Context,\n                   LLVM::FunctionCallee F, Span<const ValType> Locals,\n                   bool Interruptible, bool InstructionCounting,\n                   bool GasMeasuring) noexcept\n      : Context(Context), LLContext(Context.LLContext),\n        Interruptible(Interruptible), F(F), Builder(LLContext) {\n    if (F.Fn) {\n      Builder.positionAtEnd(LLVM::BasicBlock::create(LLContext, F.Fn, \"entry\"));\n      ExecCtx = Builder.createLoad(Context.ExecCtxTy, F.Fn.getFirstParam());\n\n      if (InstructionCounting) {\n        LocalInstrCount = Builder.createAlloca(Context.Int64Ty);\n        Builder.createStore(LLContext.getInt64(0), LocalInstrCount);\n      }\n\n      if (GasMeasuring) {\n        LocalGas = Builder.createAlloca(Context.Int64Ty);\n        Builder.createStore(LLContext.getInt64(0), LocalGas);\n      }\n\n      for (LLVM::Value Arg = F.Fn.getFirstParam().getNextParam(); Arg;\n           Arg = Arg.getNextParam()) {\n        LLVM::Type Ty = Arg.getType();\n        LLVM::Value ArgPtr = Builder.createAlloca(Ty);\n        Builder.createStore(Arg, ArgPtr);\n        Local.emplace_back(Ty, ArgPtr);\n      }\n\n      for (const auto &Type : Locals) {\n        LLVM::Type Ty = toLLVMType(LLContext, Type);\n        LLVM::Value ArgPtr = Builder.createAlloca(Ty);\n        Builder.createStore(toLLVMConstantZero(LLContext, Type), ArgPtr);\n        Local.emplace_back(Ty, ArgPtr);\n      }\n    }\n  }\n\n  LLVM::BasicBlock getTrapBB(ErrCode::Value Error) noexcept {\n    if (auto Iter = TrapBB.find(Error); Iter != TrapBB.end()) {\n      return Iter->second;\n    }\n    auto BB = LLVM::BasicBlock::create(LLContext, F.Fn, \"trap\");\n    TrapBB.emplace(Error, BB);\n    return BB;\n  }\n\n  void\n  compile(const AST::CodeSegment &Code,\n          std::pair<std::vector<ValType>, std::vector<ValType>> Type) noexcept {\n    auto RetBB = LLVM::BasicBlock::create(LLContext, F.Fn, \"ret\");\n    Type.first.clear();\n    enterBlock(RetBB, {}, {}, {}, std::move(Type));\n    compile(Code.getExpr().getInstrs());\n    assuming(ControlStack.empty());\n    compileReturn();\n\n    for (auto &[Error, BB] : TrapBB) {\n      Builder.positionAtEnd(BB);\n      updateInstrCount();\n      updateGasAtTrap();\n      auto CallTrap = Builder.createCall(\n          Context.Trap, {LLContext.getInt32(static_cast<uint32_t>(Error))});\n      CallTrap.addCallSiteAttribute(Context.NoReturn);\n      Builder.createUnreachable();\n    }\n  }\n\n  void compile(AST::InstrView Instrs) noexcept {\n    auto Dispatch = [this](const AST::Instruction &Instr) -> void {\n      switch (Instr.getOpCode()) {\n      case OpCode::Block: {\n        auto Block = LLVM::BasicBlock::create(LLContext, F.Fn, \"block\");\n        auto EndBlock = LLVM::BasicBlock::create(LLContext, F.Fn, \"block.end\");\n        Builder.createBr(Block);\n\n        Builder.positionAtEnd(Block);\n        auto Type = Context.resolveBlockType(Instr.getBlockType());\n        const auto Arity = Type.first.size();\n        std::vector<LLVM::Value> Args(Arity);\n        if (isUnreachable()) {\n          for (size_t I = 0; I < Arity; ++I) {\n            auto Ty = toLLVMType(LLContext, Type.first[I]);\n            Args[I] = LLVM::Value::getUndef(Ty);\n          }\n        } else {\n          for (size_t I = 0; I < Arity; ++I) {\n            const size_t J = Arity - 1 - I;\n            Args[J] = stackPop();\n          }\n        }\n        enterBlock(EndBlock, {}, {}, std::move(Args), std::move(Type));\n        checkStop();\n        updateGas();\n        return;\n      }\n      case OpCode::Loop: {\n        auto Curr = Builder.getInsertBlock();\n        auto Loop = LLVM::BasicBlock::create(LLContext, F.Fn, \"loop\");\n        auto EndLoop = LLVM::BasicBlock::create(LLContext, F.Fn, \"loop.end\");\n        Builder.createBr(Loop);\n\n        Builder.positionAtEnd(Loop);\n        auto Type = Context.resolveBlockType(Instr.getBlockType());\n        const auto Arity = Type.first.size();\n        std::vector<LLVM::Value> Args(Arity);\n        if (isUnreachable()) {\n          for (size_t I = 0; I < Arity; ++I) {\n            auto Ty = toLLVMType(LLContext, Type.first[I]);\n            auto Value = LLVM::Value::getUndef(Ty);\n            auto PHINode = Builder.createPHI(Ty);\n            PHINode.addIncoming(Value, Curr);\n            Args[I] = PHINode;\n          }\n        } else {\n          for (size_t I = 0; I < Arity; ++I) {\n            const size_t J = Arity - 1 - I;\n            auto Value = stackPop();\n            auto PHINode = Builder.createPHI(Value.getType());\n            PHINode.addIncoming(Value, Curr);\n            Args[J] = PHINode;\n          }\n        }\n        enterBlock(Loop, EndLoop, {}, std::move(Args), std::move(Type));\n        checkStop();\n        updateGas();\n        return;\n      }\n      case OpCode::If: {\n        auto Then = LLVM::BasicBlock::create(LLContext, F.Fn, \"then\");\n        auto Else = LLVM::BasicBlock::create(LLContext, F.Fn, \"else\");\n        auto EndIf = LLVM::BasicBlock::create(LLContext, F.Fn, \"if.end\");\n        LLVM::Value Cond;\n        if (isUnreachable()) {\n          Cond = LLVM::Value::getUndef(LLContext.getInt1Ty());\n        } else {\n          Cond = Builder.createICmpNE(stackPop(), LLContext.getInt32(0));\n        }\n        Builder.createCondBr(Cond, Then, Else);\n\n        Builder.positionAtEnd(Then);\n        auto Type = Context.resolveBlockType(Instr.getBlockType());\n        const auto Arity = Type.first.size();\n        std::vector<LLVM::Value> Args(Arity);\n        if (isUnreachable()) {\n          for (size_t I = 0; I < Arity; ++I) {\n            auto Ty = toLLVMType(LLContext, Type.first[I]);\n            Args[I] = LLVM::Value::getUndef(Ty);\n          }\n        } else {\n          for (size_t I = 0; I < Arity; ++I) {\n            const size_t J = Arity - 1 - I;\n            Args[J] = stackPop();\n          }\n        }\n        enterBlock(EndIf, {}, Else, std::move(Args), std::move(Type));\n        return;\n      }\n      case OpCode::End: {\n        auto Entry = leaveBlock();\n        if (Entry.ElseBlock) {\n          auto Block = Builder.getInsertBlock();\n          Builder.positionAtEnd(Entry.ElseBlock);\n          enterBlock(Block, {}, {}, std::move(Entry.Args),\n                     std::move(Entry.Type), std::move(Entry.ReturnPHI));\n          Entry = leaveBlock();\n        }\n        buildPHI(Entry.Type.second, Entry.ReturnPHI);\n        return;\n      }\n      case OpCode::Else: {\n        auto Entry = leaveBlock();\n        Builder.positionAtEnd(Entry.ElseBlock);\n        enterBlock(Entry.JumpBlock, {}, {}, std::move(Entry.Args),\n                   std::move(Entry.Type), std::move(Entry.ReturnPHI));\n        return;\n      }\n      default:\n        break;\n      }\n\n      if (isUnreachable()) {\n        return;\n      }\n\n      switch (Instr.getOpCode()) {\n      case OpCode::Unreachable:\n        Builder.createBr(getTrapBB(ErrCode::Value::Unreachable));\n        setUnreachable();\n        Builder.positionAtEnd(\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"unreachable.end\"));\n        break;\n      case OpCode::Nop:\n        break;\n      case OpCode::Return:\n        compileReturn();\n        setUnreachable();\n        Builder.positionAtEnd(\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"ret.end\"));\n        break;\n      case OpCode::Br: {\n        const auto Label = Instr.getJump().TargetIndex;\n        setLableJumpPHI(Label);\n        Builder.createBr(getLabel(Label));\n        setUnreachable();\n        Builder.positionAtEnd(\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"br.end\"));\n        break;\n      }\n      case OpCode::Br_if: {\n        const auto Label = Instr.getJump().TargetIndex;\n        auto Cond = Builder.createICmpNE(stackPop(), LLContext.getInt32(0));\n        setLableJumpPHI(Label);\n        auto Next = LLVM::BasicBlock::create(LLContext, F.Fn, \"br_if.end\");\n        Builder.createCondBr(Cond, getLabel(Label), Next);\n        Builder.positionAtEnd(Next);\n        break;\n      }\n      case OpCode::Br_table: {\n        auto LabelTable = Instr.getLabelList();\n        assuming(LabelTable.size() <= std::numeric_limits<uint32_t>::max());\n        const auto LabelTableSize =\n            static_cast<uint32_t>(LabelTable.size() - 1);\n        auto Value = stackPop();\n        setLableJumpPHI(LabelTable[LabelTableSize].TargetIndex);\n        auto Switch = Builder.createSwitch(\n            Value, getLabel(LabelTable[LabelTableSize].TargetIndex),\n            LabelTableSize);\n        for (uint32_t I = 0; I < LabelTableSize; ++I) {\n          setLableJumpPHI(LabelTable[I].TargetIndex);\n          Switch.addCase(LLContext.getInt32(I),\n                         getLabel(LabelTable[I].TargetIndex));\n        }\n        setUnreachable();\n        Builder.positionAtEnd(\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"br_table.end\"));\n        break;\n      }\n      case OpCode::Br_on_null: {\n        const auto Label = Instr.getJump().TargetIndex;\n        auto Value = Builder.createBitCast(stackPop(), Context.Int64x2Ty);\n        auto Cond = Builder.createICmpEQ(\n            Builder.createExtractElement(Value, LLContext.getInt64(1)),\n            LLContext.getInt64(0));\n        setLableJumpPHI(Label);\n        auto Next = LLVM::BasicBlock::create(LLContext, F.Fn, \"br_on_null.end\");\n        Builder.createCondBr(Cond, getLabel(Label), Next);\n        Builder.positionAtEnd(Next);\n        stackPush(Value);\n        break;\n      }\n      case OpCode::Br_on_non_null: {\n        const auto Label = Instr.getJump().TargetIndex;\n        auto Cond = Builder.createICmpNE(\n            Builder.createExtractElement(\n                Builder.createBitCast(Stack.back(), Context.Int64x2Ty),\n                LLContext.getInt64(1)),\n            LLContext.getInt64(0));\n        setLableJumpPHI(Label);\n        auto Next =\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"br_on_non_null.end\");\n        Builder.createCondBr(Cond, getLabel(Label), Next);\n        Builder.positionAtEnd(Next);\n        stackPop();\n        break;\n      }\n      case OpCode::Call:\n        updateInstrCount();\n        updateGas();\n        compileCallOp(Instr.getTargetIndex());\n        break;\n      case OpCode::Call_indirect:\n        updateInstrCount();\n        updateGas();\n        compileIndirectCallOp(Instr.getSourceIndex(), Instr.getTargetIndex());\n        break;\n      case OpCode::Return_call:\n        updateInstrCount();\n        updateGas();\n        compileReturnCallOp(Instr.getTargetIndex());\n        setUnreachable();\n        Builder.positionAtEnd(\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"ret_call.end\"));\n        break;\n      case OpCode::Return_call_indirect:\n        updateInstrCount();\n        updateGas();\n        compileReturnIndirectCallOp(Instr.getSourceIndex(),\n                                    Instr.getTargetIndex());\n        setUnreachable();\n        Builder.positionAtEnd(\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"ret_call_indir.end\"));\n        break;\n      case OpCode::Call_ref:\n        updateInstrCount();\n        updateGas();\n        compileCallRefOp(Instr.getTargetIndex());\n        break;\n      case OpCode::Return_call_ref:\n        updateInstrCount();\n        updateGas();\n        compileReturnCallRefOp(Instr.getTargetIndex());\n        setUnreachable();\n        Builder.positionAtEnd(\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"ret_call_ref.end\"));\n        break;\n      case OpCode::Ref__null: {\n        std::array<uint8_t, 16> Val = {0};\n        // For null references, the dynamic type down scaling is needed.\n        ValType VType;\n        if (Instr.getValType().isAbsHeapType()) {\n          switch (Instr.getValType().getHeapTypeCode()) {\n          case TypeCode::NullFuncRef:\n          case TypeCode::FuncRef:\n            VType = TypeCode::NullFuncRef;\n            break;\n          case TypeCode::NullExternRef:\n          case TypeCode::ExternRef:\n            VType = TypeCode::NullExternRef;\n            break;\n          case TypeCode::NullRef:\n          case TypeCode::AnyRef:\n          case TypeCode::EqRef:\n          case TypeCode::I31Ref:\n          case TypeCode::StructRef:\n          case TypeCode::ArrayRef:\n            VType = TypeCode::NullRef;\n            break;\n          default:\n            assumingUnreachable();\n          }\n        } else {\n          // TODO: GC - AOT: support other composite here.\n          VType = TypeCode::NullFuncRef;\n        }\n        std::copy_n(VType.getRawData().cbegin(), 8, Val.begin());\n        auto Vector = LLVM::Value::getConstVector8(LLContext, Val);\n        stackPush(Builder.createBitCast(Vector, Context.Int64x2Ty));\n        break;\n      }\n      case OpCode::Ref__is_null:\n        stackPush(Builder.createZExt(\n            Builder.createICmpEQ(\n                Builder.createExtractElement(\n                    Builder.createBitCast(stackPop(), Context.Int64x2Ty),\n                    LLContext.getInt64(1)),\n                LLContext.getInt64(0)),\n            Context.Int32Ty));\n        break;\n      case OpCode::Ref__func:\n        stackPush(Builder.createCall(\n            Context.getIntrinsic(Builder, Executable::Intrinsics::kRefFunc,\n                                 LLVM::Type::getFunctionType(Context.Int64x2Ty,\n                                                             {Context.Int32Ty},\n                                                             false)),\n            {LLContext.getInt32(Instr.getTargetIndex())}));\n        break;\n      case OpCode::Ref__as_non_null: {\n        auto Next =\n            LLVM::BasicBlock::create(LLContext, F.Fn, \"ref_as_non_null.ok\");\n        Stack.back() = Builder.createBitCast(Stack.back(), Context.Int64x2Ty);\n        auto IsNotNull = Builder.createLikely(Builder.createICmpNE(\n            Builder.createExtractElement(Stack.back(), LLContext.getInt64(1)),\n            LLContext.getInt64(0)));\n        Builder.createCondBr(IsNotNull, Next,\n                             getTrapBB(ErrCode::Value::CastNullToNonNull));\n        Builder.positionAtEnd(Next);\n        break;\n      }\n      case OpCode::Drop:\n        stackPop();\n        break;\n      case OpCode::Select:\n      case OpCode::Select_t: {\n        auto Cond = Builder.createICmpNE(stackPop(), LLContext.getInt32(0));\n        auto False = stackPop();\n        auto True = stackPop();\n        stackPush(Builder.createSelect(Cond, True, False));\n        break;\n      }\n      case OpCode::Local__get: {\n        const auto &L = Local[Instr.getTargetIndex()];\n        stackPush(Builder.createLoad(L.first, L.second));\n        break;\n      }\n      case OpCode::Local__set:\n        Builder.createStore(stackPop(), Local[Instr.getTargetIndex()].second);\n        break;\n      case OpCode::Local__tee:\n        Builder.createStore(Stack.back(), Local[Instr.getTargetIndex()].second);\n        break;\n      case OpCode::Global__get: {\n        const auto G =\n            Context.getGlobal(Builder, ExecCtx, Instr.getTargetIndex());\n        stackPush(Builder.createLoad(G.first, G.second));\n        break;\n      }\n      case OpCode::Global__set:\n        Builder.createStore(\n            stackPop(),\n            Context.getGlobal(Builder, ExecCtx, Instr.getTargetIndex()).second);\n        break;\n      case OpCode::Table__get: {\n        auto Idx = stackPop();\n        stackPush(Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kTableGet,\n                LLVM::Type::getFunctionType(Context.Int64x2Ty,\n                                            {Context.Int32Ty, Context.Int32Ty},\n                                            false)),\n            {LLContext.getInt32(Instr.getTargetIndex()), Idx}));\n        break;\n      }\n      case OpCode::Table__set: {\n        auto Ref = stackPop();\n        auto Idx = stackPop();\n        Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kTableSet,\n                LLVM::Type::getFunctionType(\n                    Context.Int64Ty,\n                    {Context.Int32Ty, Context.Int32Ty, Context.Int64x2Ty},\n                    false)),\n            {LLContext.getInt32(Instr.getTargetIndex()), Idx, Ref});\n        break;\n      }\n      case OpCode::Table__init: {\n        auto Len = stackPop();\n        auto Src = stackPop();\n        auto Dst = stackPop();\n        Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kTableInit,\n                LLVM::Type::getFunctionType(Context.VoidTy,\n                                            {Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty},\n                                            false)),\n            {LLContext.getInt32(Instr.getTargetIndex()),\n             LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});\n        break;\n      }\n      case OpCode::Elem__drop: {\n        Builder.createCall(\n            Context.getIntrinsic(Builder, Executable::Intrinsics::kElemDrop,\n                                 LLVM::Type::getFunctionType(\n                                     Context.VoidTy, {Context.Int32Ty}, false)),\n            {LLContext.getInt32(Instr.getTargetIndex())});\n        break;\n      }\n      case OpCode::Table__copy: {\n        auto Len = stackPop();\n        auto Src = stackPop();\n        auto Dst = stackPop();\n        Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kTableCopy,\n                LLVM::Type::getFunctionType(Context.VoidTy,\n                                            {Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty},\n                                            false)),\n            {LLContext.getInt32(Instr.getTargetIndex()),\n             LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});\n        break;\n      }\n      case OpCode::Table__grow: {\n        auto NewSize = stackPop();\n        auto Val = stackPop();\n        stackPush(Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kTableGrow,\n                LLVM::Type::getFunctionType(\n                    Context.Int32Ty,\n                    {Context.Int32Ty, Context.Int64x2Ty, Context.Int32Ty},\n                    false)),\n            {LLContext.getInt32(Instr.getTargetIndex()), Val, NewSize}));\n        break;\n      }\n      case OpCode::Table__size: {\n        stackPush(Builder.createCall(\n            Context.getIntrinsic(Builder, Executable::Intrinsics::kTableSize,\n                                 LLVM::Type::getFunctionType(Context.Int32Ty,\n                                                             {Context.Int32Ty},\n                                                             false)),\n            {LLContext.getInt32(Instr.getTargetIndex())}));\n        break;\n      }\n      case OpCode::Table__fill: {\n        auto Len = stackPop();\n        auto Val = stackPop();\n        auto Off = stackPop();\n        Builder.createCall(\n            Context.getIntrinsic(Builder, Executable::Intrinsics::kTableFill,\n                                 LLVM::Type::getFunctionType(\n                                     Context.Int32Ty,\n                                     {Context.Int32Ty, Context.Int32Ty,\n                                      Context.Int64x2Ty, Context.Int32Ty},\n                                     false)),\n            {LLContext.getInt32(Instr.getTargetIndex()), Off, Val, Len});\n        break;\n      }\n      case OpCode::I32__load:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int32Ty);\n        break;\n      case OpCode::I64__load:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int64Ty);\n        break;\n      case OpCode::F32__load:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.FloatTy);\n        break;\n      case OpCode::F64__load:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.DoubleTy);\n        break;\n      case OpCode::I32__load8_s:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,\n                      true);\n        break;\n      case OpCode::I32__load8_u:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,\n                      false);\n        break;\n      case OpCode::I32__load16_s:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,\n                      true);\n        break;\n      case OpCode::I32__load16_u:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,\n                      false);\n        break;\n      case OpCode::I64__load8_s:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,\n                      true);\n        break;\n      case OpCode::I64__load8_u:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,\n                      false);\n        break;\n      case OpCode::I64__load16_s:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,\n                      true);\n        break;\n      case OpCode::I64__load16_u:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,\n                      false);\n        break;\n      case OpCode::I64__load32_s:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,\n                      true);\n        break;\n      case OpCode::I64__load32_u:\n        compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                      Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,\n                      false);\n        break;\n\n      case OpCode::I32__store:\n        compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                       Instr.getMemoryAlign(), Context.Int32Ty);\n        break;\n      case OpCode::I64__store:\n        compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                       Instr.getMemoryAlign(), Context.Int64Ty);\n        break;\n      case OpCode::F32__store:\n        compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                       Instr.getMemoryAlign(), Context.FloatTy);\n        break;\n      case OpCode::F64__store:\n        compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                       Instr.getMemoryAlign(), Context.DoubleTy);\n        break;\n      case OpCode::I32__store8:\n      case OpCode::I64__store8:\n        compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                       Instr.getMemoryAlign(), Context.Int8Ty, true);\n        break;\n      case OpCode::I32__store16:\n      case OpCode::I64__store16:\n        compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                       Instr.getMemoryAlign(), Context.Int16Ty, true);\n        break;\n      case OpCode::I64__store32:\n        compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),\n                       Instr.getMemoryAlign(), Context.Int32Ty, true);\n        break;\n      case OpCode::Memory__size:\n        stackPush(Builder.createCall(\n            Context.getIntrinsic(Builder, Executable::Intrinsics::kMemSize,\n                                 LLVM::Type::getFunctionType(Context.Int32Ty,\n                                                             {Context.Int32Ty},\n                                                             false)),\n            {LLContext.getInt32(Instr.getTargetIndex())}));\n        break;\n      case OpCode::Memory__grow: {\n        auto Diff = stackPop();\n        stackPush(Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kMemGrow,\n                LLVM::Type::getFunctionType(Context.Int32Ty,\n                                            {Context.Int32Ty, Context.Int32Ty},\n                                            false)),\n            {LLContext.getInt32(Instr.getTargetIndex()), Diff}));\n        break;\n      }\n      case OpCode::Memory__init: {\n        auto Len = stackPop();\n        auto Src = stackPop();\n        auto Dst = stackPop();\n        Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kMemInit,\n                LLVM::Type::getFunctionType(Context.VoidTy,\n                                            {Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty},\n                                            false)),\n            {LLContext.getInt32(Instr.getTargetIndex()),\n             LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});\n        break;\n      }\n      case OpCode::Data__drop: {\n        Builder.createCall(\n            Context.getIntrinsic(Builder, Executable::Intrinsics::kDataDrop,\n                                 LLVM::Type::getFunctionType(\n                                     Context.VoidTy, {Context.Int32Ty}, false)),\n            {LLContext.getInt32(Instr.getTargetIndex())});\n        break;\n      }\n      case OpCode::Memory__copy: {\n        auto Len = stackPop();\n        auto Src = stackPop();\n        auto Dst = stackPop();\n        Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kMemCopy,\n                LLVM::Type::getFunctionType(Context.VoidTy,\n                                            {Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int32Ty},\n                                            false)),\n            {LLContext.getInt32(Instr.getTargetIndex()),\n             LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});\n        break;\n      }\n      case OpCode::Memory__fill: {\n        auto Len = stackPop();\n        auto Val = Builder.createTrunc(stackPop(), Context.Int8Ty);\n        auto Off = stackPop();\n        Builder.createCall(\n            Context.getIntrinsic(\n                Builder, Executable::Intrinsics::kMemFill,\n                LLVM::Type::getFunctionType(Context.VoidTy,\n                                            {Context.Int32Ty, Context.Int32Ty,\n                                             Context.Int8Ty, Context.Int32Ty},\n                                            false)),\n            {LLContext.getInt32(Instr.getTargetIndex()), Off, Val, Len});\n        break;\n      }\n      case OpCode::I32__const:\n        stackPush(LLContext.getInt32(Instr.getNum().get<uint32_t>()));\n        break;\n      case OpCode::I64__const:\n        stackPush(LLContext.getInt64(Instr.getNum().get<uint64_t>()));\n        break;\n      case OpCode::F32__const:\n        stackPush(LLContext.getFloat(Instr.getNum().get<float>()));\n        break;\n      case OpCode::F64__const:\n        stackPush(LLContext.getDouble(Instr.getNum().get<double>()));\n        break;\n      case OpCode::I32__eqz:\n        stackPush(Builder.createZExt(\n            Builder.createICmpEQ(stackPop(), LLContext.getInt32(0)),\n            Context.Int32Ty));\n        break;\n      case OpCode::I64__eqz:\n        stackPush(Builder.createZExt(\n            Builder.createICmpEQ(stackPop(), LLContext.getInt64(0)),\n            Context.Int32Ty));\n        break;\n      case OpCode::I32__clz:\n        assuming(LLVM::Core::Ctlz != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createIntrinsic(LLVM::Core::Ctlz, {Context.Int32Ty},\n                                          {stackPop(), LLContext.getFalse()}));\n        break;\n      case OpCode::I64__clz:\n        assuming(LLVM::Core::Ctlz != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createIntrinsic(LLVM::Core::Ctlz, {Context.Int64Ty},\n                                          {stackPop(), LLContext.getFalse()}));\n        break;\n      case OpCode::I32__ctz:\n        assuming(LLVM::Core::Cttz != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createIntrinsic(LLVM::Core::Cttz, {Context.Int32Ty},\n                                          {stackPop(), LLContext.getFalse()}));\n        break;\n      case OpCode::I64__ctz:\n        assuming(LLVM::Core::Cttz != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createIntrinsic(LLVM::Core::Cttz, {Context.Int64Ty},\n                                          {stackPop(), LLContext.getFalse()}));\n        break;\n      case OpCode::I32__popcnt:\n      case OpCode::I64__popcnt:\n        assuming(LLVM::Core::Ctpop != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Ctpop, stackPop()));\n        break;\n      case OpCode::F32__abs:\n      case OpCode::F64__abs:\n        assuming(LLVM::Core::Fabs != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Fabs, stackPop()));\n        break;\n      case OpCode::F32__neg:\n      case OpCode::F64__neg:\n        stackPush(Builder.createFNeg(stackPop()));\n        break;\n      case OpCode::F32__ceil:\n      case OpCode::F64__ceil:\n        assuming(LLVM::Core::Ceil != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Ceil, stackPop()));\n        break;\n      case OpCode::F32__floor:\n      case OpCode::F64__floor:\n        assuming(LLVM::Core::Floor != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Floor, stackPop()));\n        break;\n      case OpCode::F32__trunc:\n      case OpCode::F64__trunc:\n        assuming(LLVM::Core::Trunc != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Trunc, stackPop()));\n        break;\n      case OpCode::F32__nearest:\n      case OpCode::F64__nearest: {\n        const bool IsFloat = Instr.getOpCode() == OpCode::F32__nearest;\n        LLVM::Value Value = stackPop();\n\n#if LLVM_VERSION_MAJOR >= 12\n        assuming(LLVM::Core::Roundeven != LLVM::Core::NotIntrinsic);\n        if (LLVM::Core::Roundeven != LLVM::Core::NotIntrinsic) {\n          stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Roundeven, Value));\n          break;\n        }\n#endif\n\n        // The VectorSize is only used when SSE4_1 or NEON is supported.\n        [[maybe_unused]] const uint32_t VectorSize = IsFloat ? 4 : 2;\n#if defined(__x86_64__)\n        if (Context.SupportSSE4_1) {\n          auto Zero = LLContext.getInt64(0);\n          auto VectorTy =\n              LLVM::Type::getVectorType(Value.getType(), VectorSize);\n          LLVM::Value Ret = LLVM::Value::getUndef(VectorTy);\n          Ret = Builder.createInsertElement(Ret, Value, Zero);\n          auto ID = IsFloat ? LLVM::Core::X86SSE41RoundSs\n                            : LLVM::Core::X86SSE41RoundSd;\n          assuming(ID != LLVM::Core::NotIntrinsic);\n          Ret = Builder.createIntrinsic(ID, {},\n                                        {Ret, Ret, LLContext.getInt32(8)});\n          Ret = Builder.createExtractElement(Ret, Zero);\n          stackPush(Ret);\n          break;\n        }\n#endif\n\n#if defined(__aarch64__)\n        if (Context.SupportNEON &&\n            LLVM::Core::AArch64NeonFRIntN != LLVM::Core::NotIntrinsic) {\n          auto Zero = LLContext.getInt64(0);\n          auto VectorTy =\n              LLVM::Type::getVectorType(Value.getType(), VectorSize);\n          LLVM::Value Ret = LLVM::Value::getUndef(VectorTy);\n          Ret = Builder.createInsertElement(Ret, Value, Zero);\n          Ret =\n              Builder.createUnaryIntrinsic(LLVM::Core::AArch64NeonFRIntN, Ret);\n          Ret = Builder.createExtractElement(Ret, Zero);\n          stackPush(Ret);\n          break;\n        }\n#endif\n\n        // Fallback case.\n        // If the SSE4.1 is not supported on the x86_64 platform or\n        // the NEON is not supported on the aarch64 platform,\n        // then fallback to this.\n        assuming(LLVM::Core::Nearbyint != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Nearbyint, Value));\n        break;\n      }\n      case OpCode::F32__sqrt:\n      case OpCode::F64__sqrt:\n        assuming(LLVM::Core::Sqrt != LLVM::Core::NotIntrinsic);\n        stackPush(Builder.createUnaryIntrinsic(LLVM::Core::Sqrt, stackPop()));\n        break;\n      case OpCode::I32__wrap_i64:\n        stackPush(Builder.createTrunc(stackPop(), Context.Int32Ty));\n        break;\n      case OpCode::I32__trunc_f32_s:\n        compileSignedTrunc(Context.Int32Ty);\n        break;\n      case OpCode::I32__trunc_f64_s:\n        compileSignedTrunc(Context.Int32Ty);\n        break;\n      case OpCode::I32__trunc_f32_u:\n        compileUnsignedTrunc(Context.Int32Ty);\n        break;\n      case OpCode::I32__trunc_f64_u:\n        compileUnsignedTrunc(Context.Int32Ty);\n        break;\n      case OpCode::I64__extend_i32_s:\n        stackPush(Builder.createSExt(stackPop(), Context.Int64Ty));\n        break;\n      case OpCode::I64__extend_i32_u:\n        stackPush(Builder.createZExt(stackPop(), Context.Int64Ty));\n        break;\n      case OpCode::I64__trunc_f32_s:\n        compileSignedTrunc(Context.Int64Ty);\n        break;\n      case OpCode::I64__trunc_f64_s:\n        compileSignedTrunc(Context.Int64Ty);\n        break;\n      case OpCode::I64__trunc_f32_u:\n        compileUnsignedTrunc(Context.Int64Ty);\n        break;\n      case OpCode::I64__trunc_f64_u:\n        compileUnsignedTrunc(Context.Int64Ty);\n        break;\n      case OpCode::F32__convert_i32_s:\n      case OpCode::F32__convert_i64_s:\n        stackPush(Builder.createSIToFP(stackPop(), Context.FloatTy));\n        break;\n      case OpCode::F32__convert_i32_u:\n      case OpCode::F32__convert_i64_u:\n        stackPush(Builder.createUIToFP(stackPop(), Context.FloatTy));\n        break;\n      case OpCode::F64__convert_i32_s:\n      case OpCode::F64__convert_i64_s:\n        stackPush(Builder.createSIToFP(stackPop(), Context.DoubleTy));\n        break;\n      case OpCode::F64__convert_i32_u:\n      case OpCode::F64__convert_i64_u:\n        stackPush(Builder.createUIToFP(stackPop(), Context.DoubleTy));\n        break;\n      case OpCode::F32__demote_f64:\n        stackPush(Builder.createFPTrunc(stackPop(), Context.FloatTy));\n        break;\n      case OpCode::F64__promote_f32:\n        stackPush(Builder.createFPExt(stackPop(), Context.DoubleTy));\n        break;\n      case OpCode::I32__reinterpret_f32:\n        stackPush(Builder.createBitCast(stackPop(), Context.Int32Ty));\n        break;\n      case OpCode::I64__reinterpret_f64:\n        stackPush(Builder.createBitCast(stackPop(), Context.Int64Ty));\n        break;\n      case OpCode::F32__reinterpret_i32:\n        stackPush(Builder.createBitCast(stackPop(), Context.FloatTy));\n        break;\n      case OpCode::F64__reinterpret_i64:\n        stackPush(Builder.createBitCast(stackPop(), Context.DoubleTy));\n        break;\n      case OpCode::I32__extend8_s:\n        stackPush(Builder.createSExt(\n            Builder.createTrunc(stackPop(), Context.Int8Ty), Context.Int32Ty));\n        break;\n      case OpCode::I32__extend16_s:\n        stackPush(Builder.createSExt(\n            Builder.createTrunc(stackPop(), Context.Int16Ty), Context.Int32Ty));\n        break;\n      case OpCode::I64__extend8_s:\n        stackPush(Builder.createSExt(\n            Builder.createTrunc(stackPop(), Context.Int8Ty), Context.Int64Ty));\n        break;\n      case OpCode::I64__extend16_s:\n        stackPush(Builder.createSExt(\n            Builder.createTrunc(stackPop(), Context.Int16Ty), Context.Int64Ty));\n        break;\n      case OpCode::I64__extend32_s:\n        stackPush(Builder.createSExt(\n            Builder.createTrunc(stackPop(), Context.Int32Ty), Context.Int64Ty));\n        break;\n      case OpCode::I32__trunc_sat_f32_s:\n        compileSignedTruncSat(Context.Int32Ty);\n        break;\n      case OpCode::I32__trunc_sat_f32_u:\n        compileUnsignedTruncSat(Context.Int32Ty);\n        break;\n      case OpCode::I32__trunc_sat_f64_s:\n        compileSignedTruncSat(Context.Int32Ty);\n        break;\n      case OpCode::I32__trunc_sat_f64_u:\n        compileUnsignedTruncSat(Context.Int32Ty);\n        break;\n      case OpCode::I64__trunc_sat_f32_s:\n        compileSignedTruncSat(Context.Int64Ty);\n        break;\n      case OpCode::I64__trunc_sat_f32_u:\n        compileUnsignedTruncSat(Context.Int64Ty);\n        break;\n      case OpCode::I64__trunc_sat_f64_s:\n        compileSignedTruncSat(Context.Int64Ty);\n        break;\n      case OpCode::I64__trunc_sat_f64_u:\n        compileUnsignedTruncSat(Context.Int64Ty);\n        break;\n      case OpCode::I32__eq:\n      case OpCode::I64__eq: {\n        LLVM::Value RHS = stackPop();\n        LLVM::Value LHS = stackPop();\n        stackPush(Builder.createZExt(Builder.createICmpEQ(LHS, RHS),\n                                     Context.Int32Ty));\n        break;\n      }\n      case OpCode::I32__ne:\n      case OpCode::I64__ne: {\n        LLVM::Value RHS = stackPop();\n        LLVM::Value LHS = stackPop();\n        stackPush(Builder.createZExt(Builder.createICmpNE(LHS, RHS),\n                                     Context.Int32Ty));\n        break;\n      }\n      case OpCode::I32__lt_s:\n      case OpCode::I64__lt_s: {\n        LLVM::Value RHS = stackPop();\n        LLVM::Value LHS = stackPop();\n        stackPush(Builder.createZExt(Builder.createICmpSLT(LHS, RHS),\n                                     Context.Int32Ty));\n        break;\n      }\n      case OpCode::I32__lt_u:\n      case OpCode::I64__lt_u: {\n        LLVM::Value RHS = stackPop();\n        LLVM::Value LHS = stackPop();\n        stackPush(Builder.createZExt(Builder.createICmpULT(LHS, RHS),\n                                     Context.Int32Ty));\n        break;\n      }\n      case OpCode::I32__gt_s:\n      case OpCode::I64__gt_s: {\n        LLVM::Value RHS = stackPop();\n        LLVM::Value LHS = stackPop();\n        stackPush(Builder.createZExt(Builder.createICmpSGT(LHS, RHS),\n                                     Context.Int32Ty));\n        break;\n      }\n      case OpCode::I32__gt_u:\n      case OpCode::I64__gt_u: {\n        LLVM::Value RHS = stackPop();\n        LLVM::Value LHS = stackPop();\n        stackPush(Builder.createZExt(Builder.createICmpUGT(LHS, RHS),\n                                     Context.Int32Ty));\n        break;\n      }\n      case OpCode::I32__le_s:\n      case OpCode::I64__le_s: {\n...\n", "needle_token_start": 3944, "needle_token_end": 4188, "code_context_ntokens": 16326, "output": ["```cpp\nstatic LLVM::Value toLLVMConstantZero(LLVM::Context LLContext,\n                                      const ValType &ValType) noexcept {\n  switch (ValType.getCode()) {\n  case TypeCode::I32:\n    return LLVM::Value::getConstNull(LLContext.getInt32Ty());\n  case TypeCode::I64:\n    return LLVM::Value::getConstNull(LLContext.getInt64Ty());\n  case TypeCode::Ref:\n  case TypeCode::RefNull:\n  case TypeCode::V128:\n    return LLVM::Value::getConstNull(\n        LLVM::Type::getVectorType(LLContext.getInt64Ty(), 2));\n  case TypeCode::F32:\n    return LLVM::Value::getConstNull(LLContext.getFloatTy());\n  case TypeCode::F64:\n    return LLVM::Value::getConstNull(LLContext.getDoubleTy());\n  default:\n    assumingUnreachable();\n  }\n}\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator", "language": "cpp", "path": "lib/llvm/llvm.h", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function configures a Just-In-Time (JIT) builder to use a custom object linking layer, which is responsible for managing memory and handling symbols during the JIT compilation process.\n2. **Input**: \n   - A reference to the JIT builder.\n   - A function pointer that defines how to create the object linking layer.\n   - A context pointer that can be used to pass additional information to the layer creation function.\n3. **Output**: There is no direct output; the function modifies the state of the JIT builder by setting the custom object linking layer creator.\n4. **Procedure**: \n   - The function receives the JIT builder reference and the function pointer for creating the object linking layer.\n   - It sets this function as the method to create the object linking layer within the JIT compilation process.\n   - This setup allows the JIT compiler to utilize a customized linking layer that can include specific behaviors, such as custom memory management or symbol resolution strategies, tailored to the needs of the application.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                   OrcThreadSafeContext &RHS) noexcept {\n    using std::swap;\n    swap(LHS.Ref, RHS.Ref);\n  }\n\n  Context getContext() noexcept {\n    return LLVMOrcThreadSafeContextGetContext(Ref);\n  }\n\nprivate:\n  LLVMOrcThreadSafeContextRef Ref = nullptr;\n};\n\nclass OrcThreadSafeModule {\npublic:\n  constexpr OrcThreadSafeModule() noexcept = default;\n  constexpr OrcThreadSafeModule(LLVMOrcThreadSafeModuleRef R) noexcept\n      : Ref(R) {}\n  OrcThreadSafeModule(const OrcThreadSafeModule &) = delete;\n  OrcThreadSafeModule &operator=(const OrcThreadSafeModule &) = delete;\n  OrcThreadSafeModule(OrcThreadSafeModule &&B) noexcept\n      : OrcThreadSafeModule() {\n    swap(*this, B);\n  }\n  OrcThreadSafeModule &operator=(OrcThreadSafeModule &&B) noexcept {\n    swap(*this, B);\n    return *this;\n  }\n\n  OrcThreadSafeModule(Module &&M, OrcThreadSafeContext &C) noexcept\n      : Ref(LLVMOrcCreateNewThreadSafeModule(M.release(), C.unwrap())) {}\n  ~OrcThreadSafeModule() noexcept { LLVMOrcDisposeThreadSafeModule(Ref); }\n\n  constexpr operator bool() const noexcept { return Ref != nullptr; }\n  constexpr auto &unwrap() const noexcept { return Ref; }\n  constexpr auto &unwrap() noexcept { return Ref; }\n  LLVMOrcThreadSafeModuleRef release() noexcept {\n    return std::exchange(Ref, nullptr);\n  }\n  friend void swap(OrcThreadSafeModule &LHS,\n                   OrcThreadSafeModule &RHS) noexcept {\n    using std::swap;\n    swap(LHS.Ref, RHS.Ref);\n  }\n  Error withModuleDo(LLVMOrcGenericIRModuleOperationFunction F,\n                     void *Ctx) noexcept {\n    return LLVMOrcThreadSafeModuleWithModuleDo(Ref, F, Ctx);\n  }\n\nprivate:\n  LLVMOrcThreadSafeModuleRef Ref = nullptr;\n};\n\nclass OrcJITDylib {\npublic:\n  constexpr OrcJITDylib() noexcept = default;\n  constexpr OrcJITDylib(LLVMOrcJITDylibRef R) noexcept : Ref(R) {}\n  OrcJITDylib(const OrcJITDylib &) = delete;\n  OrcJITDylib &operator=(const OrcJITDylib &) = delete;\n  OrcJITDylib(OrcJITDylib &&B) noexcept : OrcJITDylib() { swap(*this, B); }\n  OrcJITDylib &operator=(OrcJITDylib &&B) noexcept {\n    swap(*this, B);\n    return *this;\n  }\n\n  constexpr operator bool() const noexcept { return Ref != nullptr; }\n  constexpr auto &unwrap() const noexcept { return Ref; }\n  constexpr auto &unwrap() noexcept { return Ref; }\n  friend void swap(OrcJITDylib &LHS, OrcJITDylib &RHS) noexcept {\n    using std::swap;\n    swap(LHS.Ref, RHS.Ref);\n  }\n\nprivate:\n  LLVMOrcJITDylibRef Ref = nullptr;\n};\n\nclass OrcIRTransformLayer {\npublic:\n  constexpr OrcIRTransformLayer() noexcept = default;\n  constexpr OrcIRTransformLayer(LLVMOrcIRTransformLayerRef R) noexcept\n      : Ref(R) {}\n  OrcIRTransformLayer(const OrcIRTransformLayer &) = delete;\n  OrcIRTransformLayer &operator=(const OrcIRTransformLayer &) = delete;\n  OrcIRTransformLayer(OrcIRTransformLayer &&B) noexcept\n      : OrcIRTransformLayer() {\n    swap(*this, B);\n  }\n  OrcIRTransformLayer &operator=(OrcIRTransformLayer &&B) noexcept {\n    swap(*this, B);\n    return *this;\n  }\n\n  constexpr operator bool() const noexcept { return Ref != nullptr; }\n  constexpr auto &unwrap() const noexcept { return Ref; }\n  constexpr auto &unwrap() noexcept { return Ref; }\n  friend void swap(OrcIRTransformLayer &LHS,\n                   OrcIRTransformLayer &RHS) noexcept {\n    using std::swap;\n    swap(LHS.Ref, RHS.Ref);\n  }\n\n  void setTransform(LLVMOrcIRTransformLayerTransformFunction TransformFunction,\n                    void *Ctx) noexcept {\n    LLVMOrcIRTransformLayerSetTransform(Ref, TransformFunction, Ctx);\n  }\n\nprivate:\n  LLVMOrcIRTransformLayerRef Ref = nullptr;\n};\n\nclass OrcLLJIT {\npublic:\n  constexpr OrcLLJIT() noexcept = default;\n  constexpr OrcLLJIT(LLVMOrcLLJITRef R) noexcept : Ref(R) {}\n  OrcLLJIT(const OrcLLJIT &) = delete;\n  OrcLLJIT &operator=(const OrcLLJIT &) = delete;\n  OrcLLJIT(OrcLLJIT &&B) noexcept : OrcLLJIT() { swap(*this, B); }\n  OrcLLJIT &operator=(OrcLLJIT &&B) noexcept {\n    swap(*this, B);\n    return *this;\n  }\n\n  ~OrcLLJIT() noexcept { LLVMOrcDisposeLLJIT(Ref); }\n\n  constexpr operator bool() const noexcept { return Ref != nullptr; }\n  constexpr auto &unwrap() const noexcept { return Ref; }\n  constexpr auto &unwrap() noexcept { return Ref; }\n  friend void swap(OrcLLJIT &LHS, OrcLLJIT &RHS) noexcept {\n    using std::swap;\n    swap(LHS.Ref, RHS.Ref);\n  }\n\n  static cxx20::expected<OrcLLJIT, Error> create() noexcept {\n    OrcLLJIT Result;\n    if (auto Err = LLVMOrcCreateLLJIT(&Result.Ref, getBuilder())) {\n      return cxx20::unexpected(Err);\n    } else {\n      return Result;\n    }\n  }\n\n  OrcJITDylib getMainJITDylib() noexcept {\n    return LLVMOrcLLJITGetMainJITDylib(Ref);\n  }\n\n  Error addLLVMIRModule(const OrcJITDylib &L, OrcThreadSafeModule M) noexcept {\n    return LLVMOrcLLJITAddLLVMIRModule(Ref, L.unwrap(), M.release());\n  }\n\n  template <typename T>\n  cxx20::expected<T *, Error> lookup(const char *Name) noexcept {\n    LLVMOrcJITTargetAddress Addr;\n    if (auto Err = LLVMOrcLLJITLookup(Ref, &Addr, Name)) {\n      return cxx20::unexpected(Err);\n    }\n    return reinterpret_cast<T *>(Addr);\n  }\n\n  OrcIRTransformLayer getIRTransformLayer() noexcept {\n    return LLVMOrcLLJITGetIRTransformLayer(Ref);\n  }\n\nprivate:\n  LLVMOrcLLJITRef Ref = nullptr;\n\n  static inline LLVMOrcLLJITBuilderRef getBuilder() noexcept;\n};\n\n} // namespace WasmEdge::LLVM\n\n#include <llvm/IR/GlobalValue.h>\n#include <llvm/Object/ObjectFile.h>\n#include <llvm/Transforms/Utils/BasicBlockUtils.h>\n#if LLVM_VERSION_MAJOR < 12 || WASMEDGE_OS_WINDOWS\n#include <llvm/ExecutionEngine/Orc/Core.h>\n#endif\n#if LLVM_VERSION_MAJOR < 13\n#include <llvm/ExecutionEngine/Orc/LLJIT.h>\n#include <llvm/Support/CBindingWrapping.h>\n#include <llvm/Support/Error.h>\n#endif\n\n#if WASMEDGE_OS_WINDOWS\n#include <llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h>\n#include <llvm/ExecutionEngine/SectionMemoryManager.h>\n#include <llvm/Support/Process.h>\n#include <system/winapi.h>\n#endif\n\nnamespace llvm {\n#if WASMEDGE_OS_WINDOWS\nDEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ExecutionSession,\n                                   LLVMOrcExecutionSessionRef)\nDEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ObjectLayer, LLVMOrcObjectLayerRef)\n#endif\n#if LLVM_VERSION_MAJOR < 12\nDEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::LLJITBuilder, LLVMOrcLLJITBuilderRef)\n#endif\n#if LLVM_VERSION_MAJOR < 13\nDEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeModule,\n                                   LLVMOrcThreadSafeModuleRef)\nDEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::IRTransformLayer,\n                                   LLVMOrcIRTransformLayerRef)\nDEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::MaterializationResponsibility,\n                                   LLVMOrcMaterializationResponsibilityRef)\nDEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::LLJIT, LLVMOrcLLJITRef)\n#endif\n} // namespace llvm\n\nnamespace WasmEdge::LLVM {\n\nvoid Value::setDSOLocal(bool Local) noexcept {\n  llvm::cast<llvm::GlobalValue>(reinterpret_cast<llvm::Value *>(Ref))\n      ->setDSOLocal(Local);\n}\n\nvoid Value::eliminateUnreachableBlocks() noexcept {\n  llvm::EliminateUnreachableBlocks(\n      *llvm::cast<llvm::Function>(reinterpret_cast<llvm::Value *>(Ref)));\n}\n\nbool SectionIterator::isText() const noexcept {\n  auto *S = reinterpret_cast<const llvm::object::section_iterator *>(Ref);\n  return (*S)->isText();\n}\n\nbool SectionIterator::isData() const noexcept {\n  auto *S = reinterpret_cast<const llvm::object::section_iterator *>(Ref);\n  return (*S)->isData();\n}\n\nbool SectionIterator::isBSS() const noexcept {\n  auto *S = reinterpret_cast<const llvm::object::section_iterator *>(Ref);\n  return (*S)->isBSS();\n}\n\nbool SectionIterator::isPData() const noexcept {\n#if WASMEDGE_OS_WINDOWS\n  using namespace std::literals;\n  return \".pdata\"sv == getName();\n#else\n  return false;\n#endif\n}\n\nbool SectionIterator::isEHFrame() const noexcept {\n#if WASMEDGE_OS_LINUX\n  using namespace std::literals;\n  return \".eh_frame\"sv == getName();\n#elif WASMEDGE_OS_MACOS\n  using namespace std::literals;\n  return \"__eh_frame\"sv == getName();\n#else\n  return false;\n#endif\n}\n\n#if WASMEDGE_OS_WINDOWS\nclass DefaultMMapper final : public llvm::SectionMemoryManager::MemoryMapper {\npublic:\n  llvm::sys::MemoryBlock allocateMappedMemory(\n      llvm::SectionMemoryManager::AllocationPurpose /*Purpose*/,\n      size_t NumBytes, const llvm::sys::MemoryBlock *const NearBlock,\n      unsigned Flags, std::error_code &EC) override {\n    return llvm::sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags,\n                                                   EC);\n  }\n  std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &Block,\n                                      unsigned Flags) override {\n    return llvm::sys::Memory::protectMappedMemory(Block, Flags);\n  }\n\n  std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &M) override {\n    return llvm::sys::Memory::releaseMappedMemory(M);\n  }\n};\n\nclass ContiguousSectionMemoryManager : public llvm::RTDyldMemoryManager {\npublic:\n  explicit ContiguousSectionMemoryManager(\n      llvm::SectionMemoryManager::MemoryMapper *UnownedMM = nullptr)\n      : MMapper(UnownedMM), OwnedMMapper(nullptr) {\n    if (!MMapper) {\n      OwnedMMapper = std::make_unique<DefaultMMapper>();\n      MMapper = OwnedMMapper.get();\n    }\n  }\n\n  ~ContiguousSectionMemoryManager() noexcept override {\n    using namespace std::literals;\n    if (Preallocated.allocatedSize() != 0) {\n      auto EC = MMapper->releaseMappedMemory(Preallocated);\n      if (EC) {\n        spdlog::error(\"releaseMappedMemory failed with error: {}\"sv,\n                      EC.message());\n      }\n    }\n  }\n\n  bool needsToReserveAllocationSpace() override { return true; }\n\n  void reserveAllocationSpace(uintptr_t CodeSize, llvm::Align CodeAlign,\n                              uintptr_t RODataSize, llvm::Align RODataAlign,\n                              uintptr_t RWDataSize,\n                              llvm::Align RWDataAlign) override {\n    using namespace std::literals;\n    assuming(Preallocated.allocatedSize() == 0);\n\n    static const size_t PageSize = llvm::sys::Process::getPageSizeEstimate();\n    assuming(CodeAlign.value() <= PageSize);\n    assuming(RODataAlign.value() <= PageSize);\n    assuming(RWDataAlign.value() <= PageSize);\n    CodeSize = roundUpTo(CodeSize + CodeAlign.value(), PageSize);\n    RODataSize = roundUpTo(RODataSize + RODataAlign.value(), PageSize);\n    RWDataSize = roundUpTo(RWDataSize + RWDataAlign.value(), PageSize);\n    const uintptr_t TotalSize =\n        CodeSize + RODataSize + RWDataSize + PageSize * 3;\n\n    std::error_code EC;\n    Preallocated = MMapper->allocateMappedMemory(\n        llvm::SectionMemoryManager::AllocationPurpose::Code, TotalSize, nullptr,\n        llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE, EC);\n    if (EC) {\n      spdlog::error(\"allocateMappedMemory failed with error: {}\"sv,\n                    EC.message());\n      return;\n    }\n\n    auto base = reinterpret_cast<std::uintptr_t>(Preallocated.base());\n    CodeMem = CodeFree =\n        llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), CodeSize);\n    base += CodeSize;\n    RODataMem = RODataFree =\n        llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), RODataSize);\n    base += RODataSize;\n    RWDataMem = RWDataFree =\n        llvm::sys::MemoryBlock(reinterpret_cast<void *>(base), RWDataSize);\n  }\n\n  uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,\n                               unsigned /*SectionID*/,\n                               llvm::StringRef /*SectionName*/,\n                               bool IsReadOnly) override {\n    if (IsReadOnly) {\n      return Allocate(RODataFree, Size, Alignment);\n    } else {\n      return Allocate(RWDataFree, Size, Alignment);\n    }\n  }\n\n  uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,\n                               unsigned /*SectionID*/,\n                               llvm::StringRef /*SectionName*/) override {\n    return Allocate(CodeFree, Size, Alignment);\n  }\n\n  bool finalizeMemory(std::string *ErrMsg) override {\n    std::error_code EC;\n\n    EC = MMapper->protectMappedMemory(CodeMem, llvm::sys::Memory::MF_READ |\n                                                   llvm::sys::Memory::MF_EXEC);\n    if (EC) {\n      if (ErrMsg) {\n        *ErrMsg = EC.message();\n      }\n      return true;\n    }\n    EC = MMapper->protectMappedMemory(RODataMem, llvm::sys::Memory::MF_READ);\n    if (EC) {\n      if (ErrMsg) {\n        *ErrMsg = EC.message();\n      }\n      return true;\n    }\n\n    llvm::sys::Memory::InvalidateInstructionCache(CodeMem.base(),\n                                                  CodeMem.allocatedSize());\n    return false;\n  }\n\nprivate:\n  llvm::sys::MemoryBlock Preallocated;\n\n  // Sections must be in the order code < rodata < rwdata.\n  llvm::sys::MemoryBlock CodeMem;\n  llvm::sys::MemoryBlock RODataMem;\n  llvm::sys::MemoryBlock RWDataMem;\n\n  llvm::sys::MemoryBlock CodeFree;\n  llvm::sys::MemoryBlock RODataFree;\n  llvm::sys::MemoryBlock RWDataFree;\n\n  llvm::SectionMemoryManager::MemoryMapper *MMapper;\n  std::unique_ptr<llvm::SectionMemoryManager::MemoryMapper> OwnedMMapper;\n\n  uint8_t *Allocate(llvm::sys::MemoryBlock &FreeBlock, std::uintptr_t Size,\n                    unsigned alignment) {\n    using namespace std::literals;\n    const auto Base = reinterpret_cast<uintptr_t>(FreeBlock.base());\n    const auto Start = roundUpTo(Base, alignment);\n    const uintptr_t PaddedSize = (Start - Base) + Size;\n    if (PaddedSize > FreeBlock.allocatedSize()) {\n      spdlog::error(\"Failed to satisfy suballocation request for {}\"sv, Size);\n      return nullptr;\n    }\n    FreeBlock =\n        llvm::sys::MemoryBlock(reinterpret_cast<void *>(Base + PaddedSize),\n                               FreeBlock.allocatedSize() - PaddedSize);\n    return reinterpret_cast<uint8_t *>(Start);\n  }\n\n  static uintptr_t roundUpTo(uintptr_t Value, uintptr_t Divisor) noexcept {\n    return ((Value + (Divisor - 1)) / Divisor) * Divisor;\n  }\n};\n\n// Register stack unwind info for JIT functions\nclass Win64EHManager : public ContiguousSectionMemoryManager {\n  using Base = ContiguousSectionMemoryManager;\n  uint64_t CodeAddress = 0;\n\npublic:\n  ~Win64EHManager() noexcept override {}\n\n  uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,\n                               unsigned SectionID,\n                               llvm::StringRef SectionName) override {\n    using namespace std::literals;\n    const auto Allocated =\n        Base::allocateCodeSection(Size, Alignment, SectionID, SectionName);\n    if (SectionName == llvm::StringRef(\".text\"sv)) {\n      CodeAddress = reinterpret_cast<uint64_t>(Allocated);\n    }\n    return Allocated;\n  }\n\n  void registerEHFrames(uint8_t *Addr, uint64_t /*LoadAddr*/,\n                        size_t Size) noexcept override {\n    using namespace std::literals;\n    winapi::RUNTIME_FUNCTION_ *const FunctionTable =\n        reinterpret_cast<winapi::RUNTIME_FUNCTION_ *>(Addr);\n    const uint32_t EntryCount =\n        static_cast<uint32_t>(Size / sizeof(winapi::RUNTIME_FUNCTION_));\n    if (EntryCount == 0)\n      return;\n    // Calculate object image base address by assuming that address of the first\n    // function is equal to the address of the code section\n    const auto ImageBase = CodeAddress - FunctionTable[0].BeginAddress;\n    winapi::RtlAddFunctionTable(FunctionTable, EntryCount, ImageBase);\n    EHFrames.push_back({Addr, Size});\n  }\n  void deregisterEHFrames() noexcept override {\n    using namespace std::literals;\n    for (auto &Frame : EHFrames) {\n      winapi::RtlDeleteFunctionTable(\n          reinterpret_cast<winapi::RUNTIME_FUNCTION_ *>(Frame.Addr));\n    }\n    EHFrames.clear();\n  }\n};\n\nLLVMOrcLLJITBuilderRef OrcLLJIT::getBuilder() noexcept {\n  using llvm::unwrap;\n  using llvm::wrap;\n  const LLVMOrcLLJITBuilderRef Builder = LLVMOrcCreateLLJITBuilder();\n  LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(\n      Builder,\n      [](void *, LLVMOrcExecutionSessionRef ES, const char *) noexcept {\n        auto Layer = std::make_unique<llvm::orc::RTDyldObjectLinkingLayer>(\n            *unwrap(ES), []() { return std::make_unique<Win64EHManager>(); });\n        Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);\n        Layer->setAutoClaimResponsibilityForObjectSymbols(true);\n        return wrap(static_cast<llvm::orc::ObjectLayer *>(Layer.release()));\n      },\n      nullptr);\n  return Builder;\n}\n#else\nLLVMOrcLLJITBuilderRef OrcLLJIT::getBuilder() noexcept { return nullptr; }\n#endif\n\n} // namespace WasmEdge::LLVM\n\n#if LLVM_VERSION_MAJOR < 12 && WASMEDGE_OS_WINDOWS\n\nvoid LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(\n    LLVMOrcLLJITBuilderRef Builder,\n    LLVMOrcLLJITBuilderObjectLinkingLayerCreatorFunction F,\n    void *Ctx) noexcept {\n  using llvm::unwrap;\n  using llvm::wrap;\n  unwrap(Builder)->setObjectLinkingLayerCreator(\n      [=](llvm::orc::ExecutionSession &ES, const llvm::Triple &TT) {\n        auto TTStr = TT.str();\n        return std::unique_ptr<llvm::orc::ObjectLayer>(\n            unwrap(F(Ctx, wrap(&ES), TTStr.c_str())));\n      });\n}\n#endif\n#if LLVM_VERSION_MAJOR < 13\nLLVMOrcIRTransformLayerRef\nLLVMOrcLLJITGetIRTransformLayer(LLVMOrcLLJITRef J) noexcept {\n  using llvm::unwrap;\n  using llvm::wrap;\n  return wrap(&(unwrap(J)->getIRTransformLayer()));\n}\nvoid LLVMOrcIRTransformLayerSetTransform(\n    LLVMOrcIRTransformLayerRef IRTransformLayer,\n    LLVMOrcIRTransformLayerTransformFunction TransformFunction,\n    void *Ctx) noexcept {\n  using llvm::unwrap;\n  using llvm::wrap;\n  unwrap(IRTransformLayer)\n      ->setTransform([=](llvm::orc::ThreadSafeModule TSM,\n                         llvm::orc::MaterializationResponsibility &R)\n                         -> llvm::Expected<llvm::orc::ThreadSafeModule> {\n        LLVMOrcThreadSafeModuleRef TSMRef =\n            wrap(new llvm::orc::ThreadSafeModule(std::move(TSM)));\n        if (LLVMErrorRef Err = TransformFunction(Ctx, &TSMRef, wrap(&R))) {\n          return unwrap(Err);\n        }\n        return std::move(*unwrap(TSMRef));\n      });\n}\n\nLLVMErrorRef\nLLVMOrcThreadSafeModuleWithModuleDo(LLVMOrcThreadSafeModuleRef TSM,\n                                    LLVMOrcGenericIRModuleOperationFunction F,\n                                    void *Ctx) noexcept {\n  using llvm::unwrap;\n  using llvm::wrap;\n  return wrap(unwrap(TSM)->withModuleDo(\n      [&](llvm::Module &M) { return unwrap(F(Ctx, wrap(&M))); }));\n}\n#endif\n\n// Path: lib/llvm/data.h\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n#pragma once\n\n#include \"llvm.h\"\n#include \"llvm/data.h\"\n\nstruct WasmEdge::LLVM::Data::DataContext {\n  LLVM::OrcThreadSafeContext TSContext;\n  LLVM::Module LLModule;\n  LLVM::TargetMachine TM;\n  DataContext() noexcept : TSContext(), LLModule(LLContext(), \"wasm\") {}\n  LLVM::Context LLContext() noexcept { return TSContext.getContext(); }\n};\n\n// Path: lib/llvm/data.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"llvm/data.h\"\n#include \"data.h\"\n#include \"llvm.h\"\n\nnamespace LLVM = WasmEdge::LLVM;\n\nLLVM::Data::Data() noexcept : Context(std::make_unique<DataContext>()) {}\n\nLLVM::Data::~Data() noexcept {}\n\nLLVM::Data::Data(LLVM::Data &&RHS) noexcept : Context(std::move(RHS.Context)) {}\nLLVM::Data &LLVM::Data::operator=(LLVM::Data &&RHS) noexcept {\n  using std::swap;\n  swap(Context, RHS.Context);\n  return *this;\n}\n\n// Path: lib/llvm/jit.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"llvm/jit.h\"\n#include \"common/log.h\"\n\n#include \"data.h\"\n#include \"llvm.h\"\n\nnamespace LLVM = WasmEdge::LLVM;\nusing namespace std::literals;\n\nnamespace WasmEdge::LLVM {\n\nJITLibrary::JITLibrary(OrcLLJIT JIT) noexcept\n    : J(std::make_unique<OrcLLJIT>(std::move(JIT)).release()) {}\n\nJITLibrary::~JITLibrary() noexcept {\n  std::unique_ptr<OrcLLJIT> JIT(std::exchange(J, nullptr));\n}\n\nSymbol<const Executable::IntrinsicsTable *>\nJITLibrary::getIntrinsics() noexcept {\n  if (auto Symbol = J->lookup<const IntrinsicsTable *>(\"intrinsics\")) {\n    return createSymbol<const IntrinsicsTable *>(*Symbol);\n  } else {\n    spdlog::error(\"{}\"sv, Symbol.error().message().string_view());\n    return {};\n  }\n}\n\nstd::vector<Symbol<Executable::Wrapper>>\nJITLibrary::getTypes(size_t Size) noexcept {\n  std::vector<Symbol<Wrapper>> Result;\n  Result.reserve(Size);\n  for (size_t I = 0; I < Size; ++I) {\n    const std::string Name = fmt::format(\"t{}\"sv, I);\n    if (auto Symbol = J->lookup<Wrapper>(Name.c_str())) {\n      Result.push_back(createSymbol<Wrapper>(*Symbol));\n    } else {\n      spdlog::error(\"{}\"sv, Symbol.error().message().string_view());\n      Result.emplace_back();\n    }\n  }\n\n  return Result;\n}\n\nstd::vector<Symbol<void>> JITLibrary::getCodes(size_t Offset,\n                                               size_t Size) noexcept {\n  std::vector<Symbol<void>> Result;\n  Result.reserve(Size);\n  for (size_t I = 0; I < Size; ++I) {\n    const std::string Name = fmt::format(\"f{}\"sv, I + Offset);\n    if (auto Symbol = J->lookup<void>(Name.c_str())) {\n      Result.push_back(createSymbol<void>(*Symbol));\n    } else {\n      spdlog::error(\"{}\"sv, Symbol.error().message().string_view());\n      Result.emplace_back();\n    }\n  }\n\n  return Result;\n}\n\nExpect<std::shared_ptr<Executable>> JIT::load(Data D) noexcept {\n  OrcLLJIT J;\n  if (auto Res = OrcLLJIT::create(); !Res) {\n    spdlog::error(\"{}\"sv, Res.error().message().string_view());\n    return Unexpect(ErrCode::Value::HostFuncError);\n  } else {\n    J = std::move(*Res);\n  }\n\n  auto &LLModule = D.extract().LLModule;\n\n  if (Conf.getCompilerConfigure().isDumpIR()) {\n    if (auto ErrorMessage = LLModule.printModuleToFile(\"wasm-jit.ll\")) {\n      spdlog::error(\"printModuleToFile failed\");\n    }\n  }\n\n  auto MainJD = J.getMainJITDylib();\n  if (auto Err = J.addLLVMIRModule(\n          MainJD,\n          OrcThreadSafeModule(LLModule.release(), D.extract().TSContext))) {\n    spdlog::error(\"{}\"sv, Err.message().string_view());\n    return Unexpect(ErrCode::Value::HostFuncError);\n  }\n\n  return std::make_shared<JITLibrary>(std::move(J));\n}\n} // namespace WasmEdge::LLVM\n\n// Path: lib/llvm/codegen.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"llvm/codegen.h\"\n\n#include \"aot/version.h\"\n#include \"common/defines.h\"\n#include \"data.h\"\n#include \"llvm.h\"\n\n#include <charconv>\n#include <fstream>\n#include <lld/Common/Driver.h>\n#include <random>\n#include <sstream>\n\n#if LLVM_VERSION_MAJOR >= 14\n#include <lld/Common/CommonLinkerContext.h>\n#endif\n#if LLVM_VERSION_MAJOR >= 17\n#if WASMEDGE_OS_MACOS\nLLD_HAS_DRIVER(macho)\n#elif WASMEDGE_OS_LINUX\nLLD_HAS_DRIVER(elf)\n#elif WASMEDGE_OS_WINDOWS\nLLD_HAS_DRIVER(coff)\n#endif\n#endif\n\n#if WASMEDGE_OS_MACOS\n#include <sys/utsname.h>\n#include <unistd.h>\n#endif\n#if WASMEDGE_OS_WINDOWS\n#include <llvm/Object/COFF.h>\n#endif\n\n#if WASMEDGE_OS_LINUX\n#define SYMBOL(X) X\n#elif WASMEDGE_OS_MACOS\n#define SYMBOL(X) \"_\" X\n#elif WASMEDGE_OS_WINDOWS\n#define SYMBOL(X) X\n#endif\n\nnamespace LLVM = WasmEdge::LLVM;\nusing namespace std::literals;\n\nnamespace {\n\nusing namespace WasmEdge;\n\n#if WASMEDGE_OS_MACOS\n// Get current OS version\nstd::string getOSVersion() noexcept {\n  struct utsname Info;\n  if (::uname(&Info)) {\n    // default os version\n    return \"13.0.0\"s;\n  }\n  std::string_view Release = Info.release;\n  auto GetNum = [](std::string_view &String) noexcept {\n    uint64_t Result = 0;\n    while (!String.empty() && std::isdigit(String[0])) {\n      Result = Result * 10 + (String[0] - '0');\n      String = String.substr(1);\n    }\n    return Result;\n  };\n  auto SkipDot = [](std::string_view &String) noexcept {\n    if (!String.empty() && String[0] == '.')\n      String = String.substr(1);\n  };\n  uint64_t Major = GetNum(Release);\n  SkipDot(Release);\n  uint64_t Minor = GetNum(Release);\n  SkipDot(Release);\n  uint64_t Micro = GetNum(Release);\n\n  if (Major == 0) {\n    Major = 8;\n  }\n  if (Major <= 19) {\n    Micro = 0;\n    Minor = Major - 4;\n    Major = 10;\n  } else {\n    Micro = 0;\n    Minor = 0;\n    Major = 11 + Major - 20;\n  }\n\n  return fmt::format(\"{}.{}.{}\"sv, Major, Minor, Micro);\n}\n// Get current SDK version\nstd::string getSDKVersion() noexcept {\n  // TODO: parse SDKSettings.json to get real version\n  return \"12.1\"s;\n}\n// Get current SDK version in pair\nstd::pair<uint32_t, uint32_t> getSDKVersionPair() noexcept {\n  // TODO: parse SDKSettings.json to get real version\n  return {UINT32_C(12), UINT32_C(1)};\n}\n#endif\n\nExpect<void> WriteByte(std::ostream &OS, uint8_t Data) noexcept {\n  OS.put(static_cast<char>(Data));\n  return {};\n}\n\nExpect<void> WriteU32(std::ostream &OS, uint32_t Data) noexcept {\n  do {\n    uint8_t Byte = static_cast<uint8_t>(Data & UINT32_C(0x7f));\n    Data >>= 7;\n    if (Data > UINT32_C(0)) {\n      Byte |= UINT8_C(0x80);\n    }\n    WriteByte(OS, Byte);\n  } while (Data > UINT32_C(0));\n  return {};\n}\n\nExpect<void> WriteU64(std::ostream &OS, uint64_t Data) noexcept {\n  do {\n    uint8_t Byte = static_cast<uint8_t>(Data & UINT64_C(0x7f));\n    Data >>= 7;\n    if (Data > UINT64_C(0)) {\n      Byte |= UINT8_C(0x80);\n    }\n    WriteByte(OS, Byte);\n  } while (Data > UINT64_C(0));\n  return {};\n}\n\nExpect<void> WriteName(std::ostream &OS, std::string_view Data) noexcept {\n  WriteU32(OS, static_cast<uint32_t>(Data.size()));\n  for (const auto C : Data) {\n    WriteByte(OS, static_cast<uint8_t>(C));\n  }\n  return {};\n}\n\ninline constexpr bool startsWith(std::string_view Value,\n                                 std::string_view Prefix) noexcept {\n  return Value.size() >= Prefix.size() &&\n         Value.substr(0, Prefix.size()) == Prefix;\n}\n\nstd::filesystem::path uniquePath(const std::filesystem::path Model) noexcept {\n  using size_type = std::filesystem::path::string_type::size_type;\n  using value_type = std::filesystem::path::value_type;\n  static const auto Hex = \"0123456789abcdef\"sv;\n  std::random_device Device;\n  std::default_random_engine Engine(Device());\n  std::uniform_int_distribution<size_type> Distribution(0, Hex.size() - 1);\n  auto String = Model.native();\n  for (size_type N = String.size(), I = 0; I < N; ++I) {\n    if (String[I] == static_cast<value_type>('%')) {\n      String[I] = static_cast<value_type>(Hex[Distribution(Engine)]);\n    }\n  }\n  return String;\n}\n\nstd::filesystem::path createTemp(const std::filesystem::path Model) noexcept {\n  while (true) {\n    auto Result = uniquePath(Model);\n    std::error_code Error;\n    if (!std::filesystem::exists(Result, Error)) {\n      if (Error) {\n        return {};\n      }\n      return Result;\n    }\n  }\n}\n\n// Write output object and link\nExpect<void> outputNativeLibrary(const std::filesystem::path &OutputPath,\n                                 const LLVM::MemoryBuffer &OSVec) noexcept {\n  spdlog::info(\"output start\");\n  std::filesystem::path ObjectName;\n  {\n    // tempfile\n    std::filesystem::path OPath(OutputPath);\n#if WASMEDGE_OS_WINDOWS\n    OPath.replace_extension(\"%%%%%%%%%%.obj\"sv);\n#else\n    OPath.replace_extension(\"%%%%%%%%%%.o\"sv);\n#endif\n    ObjectName = createTemp(OPath);\n    if (ObjectName.empty()) {\n      // TODO:return error\n      spdlog::error(\"so file creation failed:{}\", OPath.u8string());\n      return Unexpect(ErrCode::Value::IllegalPath);\n    }\n    std::ofstream OS(ObjectName, std::ios_base::binary);\n    OS.write(OSVec.data(), static_cast<std::streamsize>(OSVec.size()));\n    OS.close();\n  }\n\n  // link\n  bool LinkResult = false;\n#if WASMEDGE_OS_MACOS\n  const auto OSVersion = getOSVersion();\n  const auto SDKVersion = getSDKVersion();\n#if LLVM_VERSION_MAJOR >= 14\n  // LLVM 14 replaces the older mach_o lld implementation with the new one.\n  // So we need to change the namespace after LLVM 14.x released.\n  // Reference: https://reviews.llvm.org/D114842\n  LinkResult = lld::macho::link(\n#else\n  LinkResult = lld::mach_o::link(\n#endif\n      std::initializer_list<const char *> {\n        \"lld\", \"-arch\",\n#if defined(__x86_64__)\n            \"x86_64\",\n#elif defined(__aarch64__)\n            \"arm64\",\n#else\n#error Unsupported architecture on the MacOS!\n#endif\n#if LLVM_VERSION_MAJOR >= 14\n            // LLVM 14 replaces the older mach_o lld implementation with the new\n            // one. And it require -arch and -platform_version to always be\n            // specified. Reference: https://reviews.llvm.org/D97799\n            \"-platform_version\", \"macos\", OSVersion.c_str(), SDKVersion.c_str(),\n#else\n            \"-sdk_version\", SDKVersion.c_str(),\n#endif\n            \"-dylib\", \"-demangle\", \"-macosx_version_min\", OSVersion.c_str(),\n            \"-syslibroot\",\n            \"/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk\",\n            ObjectName.u8string().c_str(), \"-o\", OutputPath.u8string().c_str()\n      },\n#elif WASMEDGE_OS_LINUX\n  LinkResult = lld::elf::link(\n      std::initializer_list<const char *>{\"ld.lld\", \"--eh-frame-hdr\",\n                                          \"--shared\", \"--gc-sections\",\n                                          \"--discard-all\", ObjectName.c_str(),\n                                          \"-o\", OutputPath.u8string().c_str()},\n#elif WASMEDGE_OS_WINDOWS\n  LinkResult = lld::coff::link(\n      std::initializer_list<const char *>{\n          \"lld-link\", \"-dll\", \"-base:0\", \"-nologo\",\n          ObjectName.u8string().c_str(),\n          (\"-out:\" + OutputPath.u8string()).c_str()},\n#endif\n\n#if LLVM_VERSION_MAJOR >= 14\n      llvm::outs(), llvm::errs(), false, false\n#elif LLVM_VERSION_MAJOR >= 10\n      false, llvm::outs(), llvm::errs()\n#else\n      false, llvm::errs()\n#endif\n  );\n\n#if LLVM_VERSION_MAJOR >= 14\n  lld::CommonLinkerContext::destroy();\n#endif\n\n  if (LinkResult) {\n    std::error_code Error;\n    std::filesystem::remove(ObjectName, Error);\n#if WASMEDGE_OS_WINDOWS\n    std::filesystem::path LibPath(OutputPath);\n    LibPath.replace_extension(\".lib\"sv);\n    std::filesystem::remove(LibPath, Error);\n#endif\n\n    spdlog::info(\"codegen done\");\n  } else {\n    spdlog::error(\"link error\");\n  }\n\n#if WASMEDGE_OS_MACOS\n  // codesign\n  if (LinkResult) {\n    pid_t PID = ::fork();\n    if (PID == -1) {\n      spdlog::error(\"codesign error on fork:{}\", std::strerror(errno));\n    } else if (PID == 0) {\n      execlp(\"/usr/bin/codesign\", \"codesign\", \"-s\", \"-\",\n             OutputPath.u8string().c_str(), nullptr);\n      std::exit(256);\n    } else {\n      int ChildStat;\n      waitpid(PID, &ChildStat, 0);\n      if (const int Status = WEXITSTATUS(ChildStat); Status != 0) {\n        spdlog::error(\"codesign exited with status {}\", Status);\n      }\n    }\n  }\n#endif\n\n  return {};\n}\n\nExpect<void> outputWasmLibrary(LLVM::Context LLContext,\n                               const std::filesystem::path &OutputPath,\n                               Span<const Byte> Data,\n                               const LLVM::MemoryBuffer &OSVec) noexcept {\n  std::filesystem::path SharedObjectName;\n  {\n    // tempfile\n    std::filesystem::path SOPath(OutputPath);\n    SOPath.replace_extension(\"%%%%%%%%%%\" WASMEDGE_LIB_EXTENSION);\n    SharedObjectName = createTemp(SOPath);\n    if (SharedObjectName.empty()) {\n      // TODO:return error\n      spdlog::error(\"so file creation failed:{}\", SOPath.u8string());\n      return Unexpect(ErrCode::Value::IllegalPath);\n    }\n    std::ofstream OS(SharedObjectName, std::ios_base::binary);\n    OS.write(OSVec.data(), static_cast<std::streamsize>(OSVec.size()));\n    OS.close();\n  }\n\n  if (auto Res = outputNativeLibrary(SharedObjectName, OSVec); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  LLVM::MemoryBuffer SOFile;\n  if (auto [Res, ErrorMessage] =\n          LLVM::MemoryBuffer::getFile(SharedObjectName.u8string().c_str());\n      unlikely(ErrorMessage)) {\n    spdlog::error(\"object file open error:{}\", ErrorMessage.string_view());\n    return Unexpect(ErrCode::Value::IllegalPath);\n  } else {\n    SOFile = std::move(Res);\n  }\n\n  LLVM::Binary ObjFile;\n  if (auto [Res, ErrorMessage] = LLVM::Binary::create(SOFile, LLContext);\n      unlikely(ErrorMessage)) {\n    spdlog::error(\"object file parse error:{}\", ErrorMessage.string_view());\n    return Unexpect(ErrCode::Value::IllegalPath);\n  } else {\n    ObjFile = std::move(Res);\n  }\n\n  std::string OSCustomSecVec;\n  {\n    std::ostringstream OS;\n    WriteName(OS, \"wasmedge\"sv);\n    WriteU32(OS, AOT::kBinaryVersion);\n\n#if WASMEDGE_OS_LINUX\n    WriteByte(OS, UINT8_C(1));\n#elif WASMEDGE_OS_MACOS\n    WriteByte(OS, UINT8_C(2));\n#elif WASMEDGE_OS_WINDOWS\n    WriteByte(OS, UINT8_C(3));\n#else\n#error Unsupported operating system!\n#endif\n\n#if defined(__x86_64__)\n    WriteByte(OS, UINT8_C(1));\n#elif defined(__aarch64__)\n    WriteByte(OS, UINT8_C(2));\n#elif defined(__riscv) && __riscv_xlen == 64\n    WriteByte(OS, UINT8_C(3));\n#elif defined(__arm__) && __ARM_ARCH == 7\n    WriteByte(OS, UINT8_C(4));\n#else\n#error Unsupported hardware architecture!\n#endif\n\n    std::vector<std::pair<std::string, uint64_t>> SymbolTable;\n#if !WASMEDGE_OS_WINDOWS\n    for (auto Symbol = ObjFile.symbols();\n         Symbol && !ObjFile.isSymbolEnd(Symbol); Symbol.next()) {\n      SymbolTable.emplace_back(Symbol.getName(), Symbol.getAddress());\n    }\n#else\n    for (auto &Symbol :\n         llvm::object::unwrap<llvm::object::COFFObjectFile>(ObjFile.unwrap())\n             ->export_directories()) {\n      llvm::StringRef Name;\n      if (auto Error = Symbol.getSymbolName(Name); unlikely(!!Error)) {\n        continue;\n      } else if (Name.empty()) {\n        continue;\n      }\n      uint32_t Offset = 0;\n      if (auto Error = Symbol.getExportRVA(Offset); unlikely(!!Error)) {\n        continue;\n      }\n      SymbolTable.emplace_back(Name.str(), Offset);\n    }\n#endif\n    uint64_t VersionAddress = 0, IntrinsicsAddress = 0;\n    std::vector<uint64_t> Types;\n    std::vector<uint64_t> Codes;\n    uint64_t CodesMin = std::numeric_limits<uint64_t>::max();\n    for (const auto &[Name, Address] : SymbolTable) {\n      if (Name == SYMBOL(\"version\"sv)) {\n        VersionAddress = Address;\n      } else if (Name == SYMBOL(\"intrinsics\"sv)) {\n        IntrinsicsAddress = Address;\n      } else if (startsWith(Name, SYMBOL(\"t\"sv))) {\n        uint64_t Index = 0;\n        std::from_chars(Name.data() + SYMBOL(\"t\"sv).size(),\n                        Name.data() + Name.size(), Index);\n        if (Types.size() < Index + 1) {\n          Types.resize(Index + 1);\n        }\n        Types[Index] = Address;\n      } else if (startsWith(Name, SYMBOL(\"f\"sv))) {\n        uint64_t Index = 0;\n        std::from_chars(Name.data() + SYMBOL(\"f\"sv).size(),\n                        Name.data() + Name.size(), Index);\n        if (Codes.size() < Index + 1) {\n          Codes.resize(Index + 1);\n        }\n        CodesMin = std::min(CodesMin, Index);\n        Codes[Index] = Address;\n      }\n    }\n    if (CodesMin != std::numeric_limits<uint64_t>::max()) {\n      Codes.erase(Codes.begin(),\n                  Codes.begin() + static_cast<int64_t>(CodesMin));\n    }\n    WriteU64(OS, VersionAddress);\n    WriteU64(OS, IntrinsicsAddress);\n    WriteU64(OS, Types.size());\n    for (const uint64_t TypeAddress : Types) {\n      WriteU64(OS, TypeAddress);\n    }\n    WriteU64(OS, Codes.size());\n    for (const uint64_t CodeAddress : Codes) {\n      WriteU64(OS, CodeAddress);\n    }\n\n    uint32_t SectionCount = 0;\n    for (auto Section = ObjFile.sections(); !ObjFile.isSectionEnd(Section);\n         Section.next()) {\n      if (Section.getSize() == 0) {\n        continue;\n      }\n      if (!Section.isEHFrame() && !Section.isPData() && !Section.isText() &&\n          !Section.isData() && !Section.isBSS()) {\n        continue;\n      }\n      ++SectionCount;\n    }\n    WriteU32(OS, SectionCount);\n\n    for (auto Section = ObjFile.sections(); !ObjFile.isSectionEnd(Section);\n         Section.next()) {\n      if (Section.getSize() == 0) {\n        continue;\n      }\n      std::vector<char> Content;\n      if (auto Res = Section.getContents(); unlikely(Res.empty())) {\n        assumingUnreachable();\n      } else {\n        Content.assign(Res.begin(), Res.end());\n      }\n      if (Section.isEHFrame() || Section.isPData()) {\n        WriteByte(OS, UINT8_C(4));\n      } else if (Section.isText()) {\n        WriteByte(OS, UINT8_C(1));\n      } else if (Section.isData()) {\n        WriteByte(OS, UINT8_C(2));\n      } else if (Section.isBSS()) {\n        WriteByte(OS, UINT8_C(3));\n      } else {\n        continue;\n      }\n\n      WriteU64(OS, Section.getAddress());\n      WriteU64(OS, Content.size());\n      WriteName(OS, std::string_view(Content.data(), Content.size()));\n    }\n    OSCustomSecVec = OS.str();\n  }\n\n  spdlog::info(\"output start\");\n\n  std::ofstream OS(OutputPath, std::ios_base::binary);\n  if (!OS) {\n    spdlog::error(\"output failed.\");\n    return Unexpect(ErrCode::Value::IllegalPath);\n  }\n  OS.write(reinterpret_cast<const char *>(Data.data()),\n           static_cast<std::streamsize>(Data.size()));\n  // Custom section id\n  WriteByte(OS, UINT8_C(0x00));\n  WriteName(OS, std::string_view(OSCustomSecVec.data(), OSCustomSecVec.size()));\n\n  std::error_code Error;\n  std::filesystem::remove(SharedObjectName, Error);\n\n  spdlog::info(\"output done\");\n  return {};\n}\n\n} // namespace\n\nnamespace WasmEdge::LLVM {\n\nExpect<void> CodeGen::codegen(Span<const Byte> WasmData, Data D,\n                              std::filesystem::path OutputPath) noexcept {\n  auto LLContext = D.extract().LLContext();\n  auto &LLModule = D.extract().LLModule;\n  auto &TM = D.extract().TM;\n  std::filesystem::path LLPath(OutputPath);\n  LLPath.replace_extension(\"ll\"sv);\n\n#if WASMEDGE_OS_WINDOWS\n  {\n    // create dummy dllmain function\n    auto FTy = LLVM::Type::getFunctionType(LLContext.getInt32Ty(), {});\n    auto F =\n        LLModule.addFunction(FTy, LLVMExternalLinkage, \"_DllMainCRTStartup\");\n    F.setVisibility(LLVMProtectedVisibility);\n    F.setDSOLocal(true);\n    F.addFnAttr(\n        LLVM::Attribute::createString(LLContext, \"no-stack-arg-probe\"sv, {}));\n    F.addFnAttr(\n        LLVM::Attribute::createEnum(LLContext, LLVM::Core::StrictFP, 0));\n    F.addFnAttr(LLVM::Attribute::createEnum(LLContext, LLVM::Core::UWTable,\n                                            LLVM::Core::UWTableDefault));\n    F.addFnAttr(\n        LLVM::Attribute::createEnum(LLContext, LLVM::Core::NoReturn, 0));\n    LLVM::Builder Builder(LLContext);\n    Builder.positionAtEnd(LLVM::BasicBlock::create(LLContext, F, \"entry\"));\n    Builder.createRet(LLContext.getInt32(1u));\n\n    auto A = LLModule.addAlias(F.getType(), F, \"_fltused\");\n    A.setLinkage(LLVMExternalLinkage);\n    A.setVisibility(LLVMProtectedVisibility);\n    A.setDSOLocal(true);\n  }\n#endif\n#if WASMEDGE_OS_MACOS\n  {\n    const auto [Major, Minor] = getSDKVersionPair();\n    LLModule.addFlag(LLVMModuleFlagBehaviorError, \"SDK Version\"sv,\n                     LLVM::Value::getConstVector32(LLContext, {Major, Minor}));\n  }\n#endif\n\n  if (Conf.getCompilerConfigure().getOutputFormat() !=\n      CompilerConfigure::OutputFormat::Wasm) {\n    // create wasm.code and wasm.size\n    auto Int32Ty = LLContext.getInt32Ty();\n    auto Content = LLVM::Value::getConstString(\n        LLContext,\n        {reinterpret_cast<const char *>(WasmData.data()), WasmData.size()},\n        true);\n    LLModule.addGlobal(Content.getType(), true, LLVMExternalLinkage, Content,\n                       \"wasm.code\");\n    LLModule.addGlobal(Int32Ty, true, LLVMExternalLinkage,\n                       LLVM::Value::getConstInt(Int32Ty, WasmData.size()),\n                       \"wasm.size\");\n    for (auto Fn = LLModule.getFirstFunction(); Fn; Fn = Fn.getNextFunction()) {\n      if (Fn.getLinkage() == LLVMInternalLinkage) {\n        Fn.setLinkage(LLVMExternalLinkage);\n        Fn.setVisibility(LLVMProtectedVisibility);\n        Fn.setDSOLocal(true);\n        Fn.setDLLStorageClass(LLVMDLLExportStorageClass);\n      }\n    }\n  } else {\n    for (auto Fn = LLModule.getFirstFunction(); Fn; Fn = Fn.getNextFunction()) {\n      if (Fn.getLinkage() == LLVMInternalLinkage) {\n        Fn.setLinkage(LLVMPrivateLinkage);\n        Fn.setDSOLocal(true);\n        Fn.setDLLStorageClass(LLVMDefaultStorageClass);\n      }\n    }\n  }\n\n  // set dllexport\n  for (auto GV = LLModule.getFirstGlobal(); GV; GV = GV.getNextGlobal()) {\n    if (GV.getLinkage() == LLVMExternalLinkage) {\n      GV.setVisibility(LLVMProtectedVisibility);\n      GV.setDSOLocal(true);\n      GV.setDLLStorageClass(LLVMDLLExportStorageClass);\n    }\n  }\n\n  if (Conf.getCompilerConfigure().isDumpIR()) {\n    if (auto ErrorMessage = LLModule.printModuleToFile(\"wasm.ll\");\n        unlikely(ErrorMessage)) {\n      spdlog::error(\"wasm.ll open error:{}\", ErrorMessage.string_view());\n      return WasmEdge::Unexpect(WasmEdge::ErrCode::Value::IllegalPath);\n    }\n  }\n\n  spdlog::info(\"codegen start\");\n  // codegen\n  {\n    if (Conf.getCompilerConfigure().isDumpIR()) {\n      if (auto ErrorMessage = LLModule.printModuleToFile(\"wasm-opt.ll\")) {\n        // TODO:return error\n        spdlog::error(\"printModuleToFile failed\");\n        return Unexpect(ErrCode::Value::IllegalPath);\n      }\n    }\n\n    auto [OSVec, ErrorMessage] =\n        TM.emitToMemoryBuffer(LLModule, LLVMObjectFile);\n    if (ErrorMessage) {\n      // TODO:return error\n      spdlog::error(\"addPassesToEmitFile failed\");\n      return Unexpect(ErrCode::Value::IllegalPath);\n    }\n\n    if (Conf.getCompilerConfigure().getOutputFormat() ==\n        CompilerConfigure::OutputFormat::Wasm) {\n      if (auto Res = outputWasmLibrary(LLContext, OutputPath, WasmData, OSVec);\n          unlikely(!Res)) {\n        return Unexpect(Res);\n      }\n    } else {\n      if (auto Res = outputNativeLibrary(OutputPath, OSVec); unlikely(!Res)) {\n        return Unexpect(Res);\n      }\n    }\n  }\n\n  return {};\n}\n\n} // namespace WasmEdge::LLVM\n\n// Path: lib/llvm/compiler.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"llvm/compiler.h\"\n\n#include \"aot/version.h\"\n#include \"common/defines.h\"\n#include \"common/filesystem.h\"\n#include \"common/log.h\"\n#include \"data.h\"\n#include \"llvm.h\"\n\n#include <algorithm>\n#include <array>\n#include <cinttypes>\n#include <cstdint>\n#include <cstdlib>\n#include <limits>\n#include <memory>\n#include <numeric>\n#include <string>\n#include <string_view>\n#include <system_error>\n\nnamespace LLVM = WasmEdge::LLVM;\nusing namespace std::literals;\n\nnamespace {\n\nstatic bool\nisVoidReturn(WasmEdge::Span<const WasmEdge::ValType> ValTypes) noexcept;\nstatic LLVM::Type toLLVMType(LLVM::Context LLContext,\n                             const WasmEdge::ValType &ValType) noexcept;\nstatic std::vector<LLVM::Type>\ntoLLVMArgsType(LLVM::Context LLContext, LLVM::Type ExecCtxPtrTy,\n               WasmEdge::Span<const WasmEdge::ValType> ValTypes) noexcept;\nstatic LLVM::Type\ntoLLVMRetsType(LLVM::Context LLContext,\n               WasmEdge::Span<const WasmEdge::ValType> ValTypes) noexcept;\nstatic LLVM::Type\ntoLLVMType(LLVM::Context LLContext, LLVM::Type ExecCtxPtrTy,\n           const WasmEdge::AST::FunctionType &FuncType) noexcept;\nstatic LLVM::Value\ntoLLVMConstantZero(LLVM::Context LLContext,\n                   const WasmEdge::ValType &ValType) noexcept;\nstatic std::vector<LLVM::Value> unpackStruct(LLVM::Builder &Builder,\n                                             LLVM::Value Struct) noexcept;\nclass FunctionCompiler;\n\n// XXX: Misalignment handler not implemented yet, forcing unalignment\n// force unalignment load/store\nstatic inline constexpr const bool kForceUnalignment = true;\n\n// force checking div/rem on zero\nstatic inline constexpr const bool kForceDivCheck = true;\n\n// Size of a ValVariant\nstatic inline constexpr const uint32_t kValSize = sizeof(WasmEdge::ValVariant);\n\n// Translate Compiler::OptimizationLevel to llvm::PassBuilder version\n#if LLVM_VERSION_MAJOR >= 13\nstatic inline const char *\ntoLLVMLevel(WasmEdge::CompilerConfigure::OptimizationLevel Level) noexcept {\n  using OL = WasmEdge::CompilerConfigure::OptimizationLevel;\n  switch (Level) {\n  case OL::O0:\n    return \"default<O0>,function(tailcallelim)\";\n  case OL::O1:\n    return \"default<O1>,function(tailcallelim)\";\n  case OL::O2:\n    return \"default<O2>\";\n  case OL::O3:\n    return \"default<O3>\";\n  case OL::Os:\n    return \"default<Os>\";\n  case OL::Oz:\n    return \"default<Oz>\";\n  default:\n    assumingUnreachable();\n  }\n}\n#else\nstatic inline std::pair<unsigned int, unsigned int>\ntoLLVMLevel(WasmEdge::CompilerConfigure::OptimizationLevel Level) noexcept {\n  using OL = WasmEdge::CompilerConfigure::OptimizationLevel;\n  switch (Level) {\n  case OL::O0:\n    return {0, 0};\n  case OL::O1:\n    return {1, 0};\n  case OL::O2:\n    return {2, 0};\n  case OL::O3:\n    return {3, 0};\n  case OL::Os:\n    return {2, 1};\n  case OL::Oz:\n    return {2, 2};\n  default:\n    assumingUnreachable();\n  }\n}\n#endif\n\nstatic inline LLVMCodeGenOptLevel toLLVMCodeGenLevel(\n    WasmEdge::CompilerConfigure::OptimizationLevel Level) noexcept {\n  using OL = WasmEdge::CompilerConfigure::OptimizationLevel;\n  switch (Level) {\n  case OL::O0:\n    return LLVMCodeGenLevelNone;\n  case OL::O1:\n    return LLVMCodeGenLevelLess;\n  case OL::O2:\n    return LLVMCodeGenLevelDefault;\n  case OL::O3:\n    return LLVMCodeGenLevelAggressive;\n  case OL::Os:\n    return LLVMCodeGenLevelDefault;\n  case OL::Oz:\n    return LLVMCodeGenLevelDefault;\n  default:\n    assumingUnreachable();\n  }\n}\n} // namespace\n\nstruct LLVM::Compiler::CompileContext {\n  LLVM::Context LLContext;\n  LLVM::Module &LLModule;\n  LLVM::Attribute Cold;\n  LLVM::Attribute NoAlias;\n  LLVM::Attribute NoInline;\n  LLVM::Attribute NoReturn;\n  LLVM::Attribute ReadOnly;\n  LLVM::Attribute StrictFP;\n  LLVM::Attribute UWTable;\n  LLVM::Attribute NoStackArgProbe;\n  LLVM::Type VoidTy;\n  LLVM::Type Int8Ty;\n  LLVM::Type Int16Ty;\n  LLVM::Type Int32Ty;\n  LLVM::Type Int64Ty;\n  LLVM::Type Int128Ty;\n  LLVM::Type FloatTy;\n  LLVM::Type DoubleTy;\n  LLVM::Type Int8x16Ty;\n  LLVM::Type Int16x8Ty;\n  LLVM::Type Int32x4Ty;\n  LLVM::Type Floatx4Ty;\n  LLVM::Type Int64x2Ty;\n  LLVM::Type Doublex2Ty;\n  LLVM::Type Int128x1Ty;\n  LLVM::Type Int8PtrTy;\n  LLVM::Type Int32PtrTy;\n  LLVM::Type Int64PtrTy;\n  LLVM::Type Int128PtrTy;\n  LLVM::Type ExecCtxTy;\n  LLVM::Type ExecCtxPtrTy;\n  LLVM::Type IntrinsicsTableTy;\n  LLVM::Type IntrinsicsTablePtrTy;\n  LLVM::Message SubtargetFeatures;\n\n#if defined(__x86_64__)\n#if defined(__XOP__)\n  bool SupportXOP = true;\n#else\n  bool SupportXOP = false;\n#endif\n\n#if defined(__SSE4_1__)\n...\n", "needle_token_start": 5571, "needle_token_end": 5746, "code_context_ntokens": 16207, "output": ["```cpp\nLLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(\n    LLVMOrcLLJITBuilderRef Builder,\n    LLVMOrcLLJITBuilderObjectLinkingLayerCreatorFunction F,\n    void *Ctx) noexcept {\n  using llvm::unwrap;\n  using llvm::wrap;\n  unwrap(Builder)->setObjectLinkingLayerCreator(\n      [=](llvm::orc::ExecutionSession &ES, const llvm::Triple &TT) {\n        auto TTStr = TT.str();\n        return std::unique_ptr<llvm::orc::ObjectLayer>(\n            unwrap(F(Ctx, wrap(&ES), TTStr.c_str())));\n      });\n}\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "toAdvice", "language": "cpp", "path": "lib/host/wasi/linux.h", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function converts a WebAssembly System Interface (WASI) file advice constant to a POSIX file advice constant used to optimize file I/O operations.\n2. **Input**: A WASI file advice constant.\n3. **Output**: A POSIX file advice constant corresponding to the input WASI constant.\n4. **Procedure**: The function takes the WASI advice constant, checks it against known WASI constants using a switch statement, and returns the corresponding POSIX constant. If the input constant is not recognized, the function reaches an unreachable state, indicating an error or unexpected input.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/host/wasi/macos.h\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/defines.h\"\n#if !WASMEDGE_OS_MACOS\n#error\n#endif\n\n#define _DARWIN_C_SOURCE\n#include \"common/errcode.h\"\n#include \"wasi/api.hpp\"\n#include <Availability.h>\n#include <AvailabilityMacros.h>\n#include <TargetConditionals.h>\n#include <cerrno>\n#include <chrono>\n#include <csignal>\n#include <dirent.h>\n#include <fcntl.h>\n#include <netdb.h>\n#include <sys/event.h>\n#include <sys/ioctl.h>\n#include <sys/socket.h>\n#include <sys/stat.h>\n#include <sys/time.h>\n#include <sys/types.h>\n#include <sys/uio.h>\n#include <sys/un.h>\n#include <unistd.h>\n\nnamespace WasmEdge {\nnamespace Host {\nnamespace WASI {\ninline namespace detail {\n\ninline constexpr __wasi_errno_t fromErrNo(int ErrNo) noexcept {\n  switch (ErrNo) {\n  case 0:\n    return __WASI_ERRNO_SUCCESS;\n  case E2BIG:\n    return __WASI_ERRNO_2BIG;\n  case EACCES:\n    return __WASI_ERRNO_ACCES;\n  case EADDRINUSE:\n    return __WASI_ERRNO_ADDRINUSE;\n  case EADDRNOTAVAIL:\n    return __WASI_ERRNO_ADDRNOTAVAIL;\n  case EAFNOSUPPORT:\n    return __WASI_ERRNO_AFNOSUPPORT;\n  case EAGAIN:\n    return __WASI_ERRNO_AGAIN;\n  case EALREADY:\n    return __WASI_ERRNO_ALREADY;\n  case EBADF:\n    return __WASI_ERRNO_BADF;\n  case EBADMSG:\n    return __WASI_ERRNO_BADMSG;\n  case EBUSY:\n    return __WASI_ERRNO_BUSY;\n  case ECANCELED:\n    return __WASI_ERRNO_CANCELED;\n  case ECHILD:\n    return __WASI_ERRNO_CHILD;\n  case ECONNABORTED:\n    return __WASI_ERRNO_CONNABORTED;\n  case ECONNREFUSED:\n    return __WASI_ERRNO_CONNREFUSED;\n  case ECONNRESET:\n    return __WASI_ERRNO_CONNRESET;\n  case EDEADLK:\n    return __WASI_ERRNO_DEADLK;\n  case EDESTADDRREQ:\n    return __WASI_ERRNO_DESTADDRREQ;\n  case EDOM:\n    return __WASI_ERRNO_DOM;\n  case EDQUOT:\n    return __WASI_ERRNO_DQUOT;\n  case EEXIST:\n    return __WASI_ERRNO_EXIST;\n  case EFAULT:\n    return __WASI_ERRNO_FAULT;\n  case EFBIG:\n    return __WASI_ERRNO_FBIG;\n  case EHOSTUNREACH:\n    return __WASI_ERRNO_HOSTUNREACH;\n  case EIDRM:\n    return __WASI_ERRNO_IDRM;\n  case EILSEQ:\n    return __WASI_ERRNO_ILSEQ;\n  case EINPROGRESS:\n    return __WASI_ERRNO_INPROGRESS;\n  case EINTR:\n    return __WASI_ERRNO_INTR;\n  case EINVAL:\n    return __WASI_ERRNO_INVAL;\n  case EIO:\n    return __WASI_ERRNO_IO;\n  case EISCONN:\n    return __WASI_ERRNO_ISCONN;\n  case EISDIR:\n    return __WASI_ERRNO_ISDIR;\n  case ELOOP:\n    return __WASI_ERRNO_LOOP;\n  case EMFILE:\n    return __WASI_ERRNO_MFILE;\n  case EMLINK:\n    return __WASI_ERRNO_MLINK;\n  case EMSGSIZE:\n    return __WASI_ERRNO_MSGSIZE;\n  case EMULTIHOP:\n    return __WASI_ERRNO_MULTIHOP;\n  case ENAMETOOLONG:\n    return __WASI_ERRNO_NAMETOOLONG;\n  case ENETDOWN:\n    return __WASI_ERRNO_NETDOWN;\n  case ENETRESET:\n    return __WASI_ERRNO_NETRESET;\n  case ENETUNREACH:\n    return __WASI_ERRNO_NETUNREACH;\n  case ENFILE:\n    return __WASI_ERRNO_NFILE;\n  case ENOBUFS:\n    return __WASI_ERRNO_NOBUFS;\n  case ENODEV:\n    return __WASI_ERRNO_NODEV;\n  case ENOENT:\n    return __WASI_ERRNO_NOENT;\n  case ENOEXEC:\n    return __WASI_ERRNO_NOEXEC;\n  case ENOLCK:\n    return __WASI_ERRNO_NOLCK;\n  case ENOLINK:\n    return __WASI_ERRNO_NOLINK;\n  case ENOMEM:\n    return __WASI_ERRNO_NOMEM;\n  case ENOMSG:\n    return __WASI_ERRNO_NOMSG;\n  case ENOPROTOOPT:\n    return __WASI_ERRNO_NOPROTOOPT;\n  case ENOSPC:\n    return __WASI_ERRNO_NOSPC;\n  case ENOSYS:\n    return __WASI_ERRNO_NOSYS;\n  case ENOTCONN:\n    return __WASI_ERRNO_NOTCONN;\n  case ENOTDIR:\n    return __WASI_ERRNO_NOTDIR;\n  case ENOTEMPTY:\n    return __WASI_ERRNO_NOTEMPTY;\n  case ENOTRECOVERABLE:\n    return __WASI_ERRNO_NOTRECOVERABLE;\n  case ENOTSOCK:\n    return __WASI_ERRNO_NOTSOCK;\n  case ENOTSUP:\n    return __WASI_ERRNO_NOTSUP;\n  case ENOTTY:\n    return __WASI_ERRNO_NOTTY;\n  case ENXIO:\n    return __WASI_ERRNO_NXIO;\n  case EOVERFLOW:\n    return __WASI_ERRNO_OVERFLOW;\n  case EOWNERDEAD:\n    return __WASI_ERRNO_OWNERDEAD;\n  case EPERM:\n    return __WASI_ERRNO_PERM;\n  case EPIPE:\n    return __WASI_ERRNO_PIPE;\n  case EPROTO:\n    return __WASI_ERRNO_PROTO;\n  case EPROTONOSUPPORT:\n    return __WASI_ERRNO_PROTONOSUPPORT;\n  case EPROTOTYPE:\n    return __WASI_ERRNO_PROTOTYPE;\n  case ERANGE:\n    return __WASI_ERRNO_RANGE;\n  case EROFS:\n    return __WASI_ERRNO_ROFS;\n  case ESPIPE:\n    return __WASI_ERRNO_SPIPE;\n  case ESRCH:\n    return __WASI_ERRNO_SRCH;\n  case ESTALE:\n    return __WASI_ERRNO_STALE;\n  case ETIMEDOUT:\n    return __WASI_ERRNO_TIMEDOUT;\n  case ETXTBSY:\n    return __WASI_ERRNO_TXTBSY;\n  case EXDEV:\n    return __WASI_ERRNO_XDEV;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr __wasi_errno_t fromEAIErrNo(int ErrNo) noexcept {\n  switch (ErrNo) {\n  case EAI_ADDRFAMILY:\n    return __WASI_ERRNO_AIADDRFAMILY;\n  case EAI_AGAIN:\n    return __WASI_ERRNO_AIAGAIN;\n  case EAI_BADFLAGS:\n    return __WASI_ERRNO_AIBADFLAG;\n  case EAI_FAIL:\n    return __WASI_ERRNO_AIFAIL;\n  case EAI_FAMILY:\n    return __WASI_ERRNO_AIFAMILY;\n  case EAI_MEMORY:\n    return __WASI_ERRNO_AIMEMORY;\n  case EAI_NODATA:\n    return __WASI_ERRNO_AINODATA;\n  case EAI_NONAME:\n    return __WASI_ERRNO_AINONAME;\n  case EAI_SERVICE:\n    return __WASI_ERRNO_AISERVICE;\n  case EAI_SOCKTYPE:\n    return __WASI_ERRNO_AISOCKTYPE;\n  case EAI_SYSTEM:\n    return __WASI_ERRNO_AISYSTEM;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr clockid_t toClockId(__wasi_clockid_t Clock) noexcept {\n  switch (Clock) {\n  case __WASI_CLOCKID_REALTIME:\n    return CLOCK_REALTIME;\n  case __WASI_CLOCKID_MONOTONIC:\n    return CLOCK_MONOTONIC;\n  case __WASI_CLOCKID_PROCESS_CPUTIME_ID:\n    return CLOCK_PROCESS_CPUTIME_ID;\n  case __WASI_CLOCKID_THREAD_CPUTIME_ID:\n    return CLOCK_THREAD_CPUTIME_ID;\n  default:\n    assumingUnreachable();\n  }\n}\n\n...\n// Path: lib/host/wasi/environ-macos.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/defines.h\"\n#if WASMEDGE_OS_MACOS\n\n#include \"common/errcode.h\"\n#include \"host/wasi/environ.h\"\n#include \"macos.h\"\n\nnamespace WasmEdge {\nnamespace Host {\nnamespace WASI {\n\nWasiExpect<void> Environ::procRaise(__wasi_signal_t Signal) const noexcept {\n  int SysSignal;\n  switch (Signal) {\n  case __WASI_SIGNAL_NONE:\n    SysSignal = 0;\n    break;\n  case __WASI_SIGNAL_HUP:\n    SysSignal = SIGHUP;\n    break;\n  case __WASI_SIGNAL_INT:\n    SysSignal = SIGINT;\n    break;\n  case __WASI_SIGNAL_QUIT:\n    SysSignal = SIGQUIT;\n    break;\n  case __WASI_SIGNAL_ILL:\n    SysSignal = SIGILL;\n    break;\n  case __WASI_SIGNAL_TRAP:\n    SysSignal = SIGTRAP;\n    break;\n  case __WASI_SIGNAL_ABRT:\n    SysSignal = SIGABRT;\n    break;\n  case __WASI_SIGNAL_BUS:\n    SysSignal = SIGBUS;\n    break;\n  case __WASI_SIGNAL_FPE:\n    SysSignal = SIGFPE;\n    break;\n  case __WASI_SIGNAL_KILL:\n    SysSignal = SIGKILL;\n    break;\n  case __WASI_SIGNAL_USR1:\n    SysSignal = SIGUSR1;\n    break;\n  case __WASI_SIGNAL_SEGV:\n    SysSignal = SIGSEGV;\n    break;\n  case __WASI_SIGNAL_USR2:\n    SysSignal = SIGUSR2;\n    break;\n  case __WASI_SIGNAL_PIPE:\n    SysSignal = SIGPIPE;\n    break;\n  case __WASI_SIGNAL_ALRM:\n    SysSignal = SIGALRM;\n    break;\n  case __WASI_SIGNAL_TERM:\n    SysSignal = SIGTERM;\n    break;\n  case __WASI_SIGNAL_CHLD:\n    SysSignal = SIGCHLD;\n    break;\n  case __WASI_SIGNAL_CONT:\n    SysSignal = SIGCONT;\n    break;\n  case __WASI_SIGNAL_STOP:\n    SysSignal = SIGSTOP;\n    break;\n  case __WASI_SIGNAL_TSTP:\n    SysSignal = SIGTSTP;\n    break;\n  case __WASI_SIGNAL_TTIN:\n    SysSignal = SIGTTIN;\n    break;\n  case __WASI_SIGNAL_TTOU:\n    SysSignal = SIGTTOU;\n    break;\n  case __WASI_SIGNAL_URG:\n    SysSignal = SIGURG;\n    break;\n  case __WASI_SIGNAL_XCPU:\n    SysSignal = SIGXCPU;\n    break;\n  case __WASI_SIGNAL_XFSZ:\n    SysSignal = SIGXFSZ;\n    break;\n  case __WASI_SIGNAL_VTALRM:\n    SysSignal = SIGVTALRM;\n    break;\n  case __WASI_SIGNAL_PROF:\n    SysSignal = SIGPROF;\n    break;\n  case __WASI_SIGNAL_WINCH:\n    SysSignal = SIGWINCH;\n    break;\n  case __WASI_SIGNAL_SYS:\n    SysSignal = SIGSYS;\n    break;\n  case __WASI_SIGNAL_POLL:\n  case __WASI_SIGNAL_PWR:\n  default:\n    return WasiUnexpect(__WASI_ERRNO_NOTSUP);\n  }\n  if (auto Res = std::raise(SysSignal); Res != 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  return {};\n}\n\nWasiExpect<void> Environ::schedYield() const noexcept {\n  ::sched_yield();\n  return {};\n}\n\n} // namespace WASI\n} // namespace Host\n} // namespace WasmEdge\n\n#endif\n\n// Path: lib/host/wasi/linux.h\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/defines.h\"\n#if !WASMEDGE_OS_LINUX\n#error\n#endif\n\n// Uncomment these flag to test CentOS 6\n// #undef __GLIBC_MINOR__\n// #define __GLIBC_MINOR__ 5\n\n#include \"common/errcode.h\"\n#include \"wasi/api.hpp\"\n#include <cerrno>\n#include <chrono>\n#include <csignal>\n#include <cstdio>\n#include <ctime>\n#include <dirent.h>\n#include <fcntl.h>\n#include <sched.h>\n#include <sys/epoll.h>\n#include <sys/ioctl.h>\n#include <sys/time.h>\n#include <sys/types.h>\n#include <sys/uio.h>\n#include <sys/un.h>\n#include <sys/wait.h>\n#include <unistd.h>\n\n// socket include\n#include <arpa/inet.h>\n#include <netdb.h>\n#include <netinet/in.h>\n#include <sys/socket.h>\n\n#include <sys/stat.h>\n\n#if defined(__GLIBC_PREREQ)\n#if defined(_LIBCPP_GLIBC_PREREQ)\n#undef _LIBCPP_GLIBC_PREREQ\n#endif\n#define _LIBCPP_GLIBC_PREREQ(a, b) 0\n#else\n#if defined(_LIBCPP_GLIBC_PREREQ)\n#define __GLIBC_PREREQ(a, b) _LIBCPP_GLIBC_PREREQ(a, b)\n#else\n#define __GLIBC_PREREQ(a, b) 1\n#endif\n#endif\n\n#if __GLIBC_PREREQ(2, 8)\n#include <sys/timerfd.h>\n#endif\n\nnamespace WasmEdge {\nnamespace Host {\nnamespace WASI {\ninline namespace detail {\n\ninline constexpr __wasi_errno_t fromErrNo(int ErrNo) noexcept {\n  switch (ErrNo) {\n  case 0:\n    return __WASI_ERRNO_SUCCESS;\n  case E2BIG:\n    return __WASI_ERRNO_2BIG;\n  case EACCES:\n    return __WASI_ERRNO_ACCES;\n  case EADDRINUSE:\n    return __WASI_ERRNO_ADDRINUSE;\n  case EADDRNOTAVAIL:\n    return __WASI_ERRNO_ADDRNOTAVAIL;\n  case EAFNOSUPPORT:\n    return __WASI_ERRNO_AFNOSUPPORT;\n  case EAGAIN:\n    return __WASI_ERRNO_AGAIN;\n  case EALREADY:\n    return __WASI_ERRNO_ALREADY;\n  case EBADF:\n    return __WASI_ERRNO_BADF;\n  case EBADMSG:\n    return __WASI_ERRNO_BADMSG;\n  case EBUSY:\n    return __WASI_ERRNO_BUSY;\n  case ECANCELED:\n    return __WASI_ERRNO_CANCELED;\n  case ECHILD:\n    return __WASI_ERRNO_CHILD;\n  case ECONNABORTED:\n    return __WASI_ERRNO_CONNABORTED;\n  case ECONNREFUSED:\n    return __WASI_ERRNO_CONNREFUSED;\n  case ECONNRESET:\n    return __WASI_ERRNO_CONNRESET;\n  case EDEADLK:\n    return __WASI_ERRNO_DEADLK;\n  case EDESTADDRREQ:\n    return __WASI_ERRNO_DESTADDRREQ;\n  case EDOM:\n    return __WASI_ERRNO_DOM;\n  case EDQUOT:\n    return __WASI_ERRNO_DQUOT;\n  case EEXIST:\n    return __WASI_ERRNO_EXIST;\n  case EFAULT:\n    return __WASI_ERRNO_FAULT;\n  case EFBIG:\n    return __WASI_ERRNO_FBIG;\n  case EHOSTUNREACH:\n    return __WASI_ERRNO_HOSTUNREACH;\n  case EIDRM:\n    return __WASI_ERRNO_IDRM;\n  case EILSEQ:\n    return __WASI_ERRNO_ILSEQ;\n  case EINPROGRESS:\n    return __WASI_ERRNO_INPROGRESS;\n  case EINTR:\n    return __WASI_ERRNO_INTR;\n  case EINVAL:\n    return __WASI_ERRNO_INVAL;\n  case EIO:\n    return __WASI_ERRNO_IO;\n  case EISCONN:\n    return __WASI_ERRNO_ISCONN;\n  case EISDIR:\n    return __WASI_ERRNO_ISDIR;\n  case ELOOP:\n    return __WASI_ERRNO_LOOP;\n  case EMFILE:\n    return __WASI_ERRNO_MFILE;\n  case EMLINK:\n    return __WASI_ERRNO_MLINK;\n  case EMSGSIZE:\n    return __WASI_ERRNO_MSGSIZE;\n  case EMULTIHOP:\n    return __WASI_ERRNO_MULTIHOP;\n  case ENAMETOOLONG:\n    return __WASI_ERRNO_NAMETOOLONG;\n  case ENETDOWN:\n    return __WASI_ERRNO_NETDOWN;\n  case ENETRESET:\n    return __WASI_ERRNO_NETRESET;\n  case ENETUNREACH:\n    return __WASI_ERRNO_NETUNREACH;\n  case ENFILE:\n    return __WASI_ERRNO_NFILE;\n  case ENOBUFS:\n    return __WASI_ERRNO_NOBUFS;\n  case ENODEV:\n    return __WASI_ERRNO_NODEV;\n  case ENOENT:\n    return __WASI_ERRNO_NOENT;\n  case ENOEXEC:\n    return __WASI_ERRNO_NOEXEC;\n  case ENOLCK:\n    return __WASI_ERRNO_NOLCK;\n  case ENOLINK:\n    return __WASI_ERRNO_NOLINK;\n  case ENOMEM:\n    return __WASI_ERRNO_NOMEM;\n  case ENOMSG:\n    return __WASI_ERRNO_NOMSG;\n  case ENOPROTOOPT:\n    return __WASI_ERRNO_NOPROTOOPT;\n  case ENOSPC:\n    return __WASI_ERRNO_NOSPC;\n  case ENOSYS:\n    return __WASI_ERRNO_NOSYS;\n  case ENOTCONN:\n    return __WASI_ERRNO_NOTCONN;\n  case ENOTDIR:\n    return __WASI_ERRNO_NOTDIR;\n  case ENOTEMPTY:\n    return __WASI_ERRNO_NOTEMPTY;\n  case ENOTRECOVERABLE:\n    return __WASI_ERRNO_NOTRECOVERABLE;\n  case ENOTSOCK:\n    return __WASI_ERRNO_NOTSOCK;\n  case ENOTSUP:\n    return __WASI_ERRNO_NOTSUP;\n  case ENOTTY:\n    return __WASI_ERRNO_NOTTY;\n  case ENXIO:\n    return __WASI_ERRNO_NXIO;\n  case EOVERFLOW:\n    return __WASI_ERRNO_OVERFLOW;\n  case EOWNERDEAD:\n    return __WASI_ERRNO_OWNERDEAD;\n  case EPERM:\n    return __WASI_ERRNO_PERM;\n  case EPIPE:\n    return __WASI_ERRNO_PIPE;\n  case EPROTO:\n    return __WASI_ERRNO_PROTO;\n  case EPROTONOSUPPORT:\n    return __WASI_ERRNO_PROTONOSUPPORT;\n  case EPROTOTYPE:\n    return __WASI_ERRNO_PROTOTYPE;\n  case ERANGE:\n    return __WASI_ERRNO_RANGE;\n  case EROFS:\n    return __WASI_ERRNO_ROFS;\n  case ESPIPE:\n    return __WASI_ERRNO_SPIPE;\n  case ESRCH:\n    return __WASI_ERRNO_SRCH;\n  case ESTALE:\n    return __WASI_ERRNO_STALE;\n  case ETIMEDOUT:\n    return __WASI_ERRNO_TIMEDOUT;\n  case ETXTBSY:\n    return __WASI_ERRNO_TXTBSY;\n  case EXDEV:\n    return __WASI_ERRNO_XDEV;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr __wasi_errno_t fromEAIErrNo(int ErrNo) noexcept {\n  switch (ErrNo) {\n  case EAI_ADDRFAMILY:\n    return __WASI_ERRNO_AIADDRFAMILY;\n  case EAI_AGAIN:\n    return __WASI_ERRNO_AIAGAIN;\n  case EAI_BADFLAGS:\n    return __WASI_ERRNO_AIBADFLAG;\n  case EAI_FAIL:\n    return __WASI_ERRNO_AIFAIL;\n  case EAI_FAMILY:\n    return __WASI_ERRNO_AIFAMILY;\n  case EAI_MEMORY:\n    return __WASI_ERRNO_AIMEMORY;\n  case EAI_NODATA:\n    return __WASI_ERRNO_AINODATA;\n  case EAI_NONAME:\n    return __WASI_ERRNO_AINONAME;\n  case EAI_SERVICE:\n    return __WASI_ERRNO_AISERVICE;\n  case EAI_SOCKTYPE:\n    return __WASI_ERRNO_AISOCKTYPE;\n  case EAI_SYSTEM:\n    return __WASI_ERRNO_AISYSTEM;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr clockid_t toClockId(__wasi_clockid_t Clock) noexcept {\n  switch (Clock) {\n  case __WASI_CLOCKID_REALTIME:\n    return CLOCK_REALTIME;\n  case __WASI_CLOCKID_MONOTONIC:\n    return CLOCK_MONOTONIC;\n  case __WASI_CLOCKID_PROCESS_CPUTIME_ID:\n    return CLOCK_PROCESS_CPUTIME_ID;\n  case __WASI_CLOCKID_THREAD_CPUTIME_ID:\n    return CLOCK_THREAD_CPUTIME_ID;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr timespec toTimespec(__wasi_timestamp_t Timestamp) noexcept {\n  using namespace std::chrono;\n  const auto Total = duration<uint64_t, std::nano>(Timestamp);\n  const auto Second = duration_cast<seconds>(Total);\n  const auto Nano = Total - Second;\n  timespec Result{};\n  Result.tv_sec = Second.count();\n  Result.tv_nsec = Nano.count();\n  return Result;\n}\n\ninline constexpr __wasi_timestamp_t\nfromTimespec(const timespec &Time) noexcept {\n  using namespace std::chrono;\n  const auto Result = seconds(Time.tv_sec) + nanoseconds(Time.tv_nsec);\n  return Result.count();\n}\n\n#if !__GLIBC_PREREQ(2, 6)\ninline constexpr timeval toTimeval(__wasi_timestamp_t Timestamp) noexcept {\n  using namespace std::chrono;\n  const auto Total = duration_cast<microseconds>(nanoseconds(Timestamp));\n  const auto Second = duration_cast<seconds>(Total);\n  const auto Micro = Total - Second;\n  timeval Result{};\n  Result.tv_sec = Second.count();\n  Result.tv_usec = Micro.count();\n  return Result;\n}\ninline constexpr timeval toTimeval(timespec Timespec) noexcept {\n  using namespace std::chrono;\n  timeval Result{};\n  Result.tv_sec = Timespec.tv_sec;\n  Result.tv_usec =\n      duration_cast<microseconds>(nanoseconds(Timespec.tv_nsec)).count();\n  return Result;\n}\n#endif\n\n\ninline constexpr int toAdvice(__wasi_advice_t Advice) noexcept {\n  switch (Advice) {\n  case __WASI_ADVICE_NORMAL:\n    return POSIX_FADV_NORMAL;\n  case __WASI_ADVICE_SEQUENTIAL:\n    return POSIX_FADV_SEQUENTIAL;\n  case __WASI_ADVICE_RANDOM:\n    return POSIX_FADV_RANDOM;\n  case __WASI_ADVICE_WILLNEED:\n    return POSIX_FADV_WILLNEED;\n  case __WASI_ADVICE_DONTNEED:\n    return POSIX_FADV_DONTNEED;\n  case __WASI_ADVICE_NOREUSE:\n    return POSIX_FADV_NOREUSE;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr __wasi_filetype_t fromFileType(mode_t Mode) noexcept {\n  switch (Mode & S_IFMT) {\n  case S_IFBLK:\n    return __WASI_FILETYPE_BLOCK_DEVICE;\n  case S_IFCHR:\n    return __WASI_FILETYPE_CHARACTER_DEVICE;\n  case S_IFDIR:\n    return __WASI_FILETYPE_DIRECTORY;\n  case S_IFREG:\n    return __WASI_FILETYPE_REGULAR_FILE;\n  case S_IFSOCK:\n    return __WASI_FILETYPE_SOCKET_STREAM;\n  case S_IFLNK:\n    return __WASI_FILETYPE_SYMBOLIC_LINK;\n  case S_IFIFO:\n  default:\n    return __WASI_FILETYPE_UNKNOWN;\n  }\n}\n\ninline constexpr __wasi_filetype_t fromFileType(uint8_t Type) noexcept {\n  switch (Type) {\n  case DT_BLK:\n    return __WASI_FILETYPE_BLOCK_DEVICE;\n  case DT_CHR:\n    return __WASI_FILETYPE_CHARACTER_DEVICE;\n  case DT_DIR:\n    return __WASI_FILETYPE_DIRECTORY;\n  case DT_LNK:\n    return __WASI_FILETYPE_SYMBOLIC_LINK;\n  case DT_REG:\n    return __WASI_FILETYPE_REGULAR_FILE;\n  case DT_SOCK:\n    return __WASI_FILETYPE_SOCKET_STREAM;\n  case DT_FIFO:\n  case DT_UNKNOWN:\n  default:\n    return __WASI_FILETYPE_UNKNOWN;\n  }\n}\n\ninline constexpr int toWhence(__wasi_whence_t Whence) noexcept {\n  switch (Whence) {\n  case __WASI_WHENCE_CUR:\n    return SEEK_CUR;\n  case __WASI_WHENCE_END:\n    return SEEK_END;\n  case __WASI_WHENCE_SET:\n    return SEEK_SET;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr int toSockOptLevel(__wasi_sock_opt_level_t Level) noexcept {\n  switch (Level) {\n  case __WASI_SOCK_OPT_LEVEL_SOL_SOCKET:\n    return SOL_SOCKET;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr int toSockOptSoName(__wasi_sock_opt_so_t SoName) noexcept {\n  switch (SoName) {\n  case __WASI_SOCK_OPT_SO_REUSEADDR:\n    return SO_REUSEADDR;\n  case __WASI_SOCK_OPT_SO_TYPE:\n    return SO_TYPE;\n  case __WASI_SOCK_OPT_SO_ERROR:\n    return SO_ERROR;\n  case __WASI_SOCK_OPT_SO_DONTROUTE:\n    return SO_DONTROUTE;\n  case __WASI_SOCK_OPT_SO_BROADCAST:\n    return SO_BROADCAST;\n  case __WASI_SOCK_OPT_SO_SNDBUF:\n    return SO_SNDBUF;\n  case __WASI_SOCK_OPT_SO_RCVBUF:\n    return SO_RCVBUF;\n  case __WASI_SOCK_OPT_SO_KEEPALIVE:\n    return SO_KEEPALIVE;\n  case __WASI_SOCK_OPT_SO_OOBINLINE:\n    return SO_OOBINLINE;\n  case __WASI_SOCK_OPT_SO_LINGER:\n    return SO_LINGER;\n  case __WASI_SOCK_OPT_SO_RCVLOWAT:\n    return SO_RCVLOWAT;\n  case __WASI_SOCK_OPT_SO_RCVTIMEO:\n    return SO_RCVTIMEO;\n  case __WASI_SOCK_OPT_SO_SNDTIMEO:\n    return SO_SNDTIMEO;\n  case __WASI_SOCK_OPT_SO_ACCEPTCONN:\n    return SO_ACCEPTCONN;\n  case __WASI_SOCK_OPT_SO_BINDTODEVICE:\n    return SO_BINDTODEVICE;\n\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr __wasi_aiflags_t fromAIFlags(int AIFlags) noexcept {\n  __wasi_aiflags_t Result = static_cast<__wasi_aiflags_t>(0);\n\n  if (AIFlags & AI_PASSIVE) {\n    Result |= __WASI_AIFLAGS_AI_PASSIVE;\n  }\n  if (AIFlags & AI_CANONNAME) {\n    Result |= __WASI_AIFLAGS_AI_CANONNAME;\n  }\n  if (AIFlags & AI_NUMERICHOST) {\n    Result |= __WASI_AIFLAGS_AI_NUMERICHOST;\n  }\n  if (AIFlags & AI_NUMERICSERV) {\n    Result |= __WASI_AIFLAGS_AI_NUMERICSERV;\n  }\n  if (AIFlags & AI_V4MAPPED) {\n    Result |= __WASI_AIFLAGS_AI_V4MAPPED;\n  }\n  if (AIFlags & AI_ALL) {\n    Result |= __WASI_AIFLAGS_AI_ALL;\n  }\n  if (AIFlags & AI_ADDRCONFIG) {\n    Result |= __WASI_AIFLAGS_AI_ADDRCONFIG;\n  }\n\n  return Result;\n}\n\ninline constexpr int toAIFlags(__wasi_aiflags_t AIFlags) noexcept {\n  int Result = 0;\n\n  if (AIFlags & __WASI_AIFLAGS_AI_PASSIVE) {\n    Result |= AI_PASSIVE;\n  }\n  if (AIFlags & __WASI_AIFLAGS_AI_CANONNAME) {\n    Result |= AI_CANONNAME;\n  }\n  if (AIFlags & __WASI_AIFLAGS_AI_NUMERICHOST) {\n    Result |= AI_NUMERICHOST;\n  }\n  if (AIFlags & __WASI_AIFLAGS_AI_NUMERICSERV) {\n    Result |= AI_NUMERICSERV;\n  }\n  if (AIFlags & __WASI_AIFLAGS_AI_V4MAPPED) {\n    Result |= AI_V4MAPPED;\n  }\n  if (AIFlags & __WASI_AIFLAGS_AI_ALL) {\n    Result |= AI_ALL;\n  }\n  if (AIFlags & __WASI_AIFLAGS_AI_ADDRCONFIG) {\n    Result |= AI_ADDRCONFIG;\n  }\n\n  return Result;\n}\n\ninline constexpr __wasi_sock_type_t fromSockType(int SockType) noexcept {\n  switch (SockType) {\n  case 0:\n    return __WASI_SOCK_TYPE_SOCK_ANY;\n  case SOCK_DGRAM:\n    return __WASI_SOCK_TYPE_SOCK_DGRAM;\n  case SOCK_STREAM:\n    return __WASI_SOCK_TYPE_SOCK_STREAM;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr int toSockType(__wasi_sock_type_t SockType) noexcept {\n  switch (SockType) {\n  case __WASI_SOCK_TYPE_SOCK_ANY:\n    return 0;\n  case __WASI_SOCK_TYPE_SOCK_DGRAM:\n    return SOCK_DGRAM;\n  case __WASI_SOCK_TYPE_SOCK_STREAM:\n    return SOCK_STREAM;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr __wasi_protocol_t fromProtocol(int Protocol) noexcept {\n  switch (Protocol) {\n  case IPPROTO_IP:\n    return __WASI_PROTOCOL_IPPROTO_IP;\n  case IPPROTO_TCP:\n    return __WASI_PROTOCOL_IPPROTO_TCP;\n  case IPPROTO_UDP:\n    return __WASI_PROTOCOL_IPPROTO_UDP;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr int toProtocol(__wasi_protocol_t Protocol) noexcept {\n  switch (Protocol) {\n  case __WASI_PROTOCOL_IPPROTO_IP:\n    return IPPROTO_IP;\n  case __WASI_PROTOCOL_IPPROTO_TCP:\n    return IPPROTO_TCP;\n  case __WASI_PROTOCOL_IPPROTO_UDP:\n    return IPPROTO_UDP;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr __wasi_address_family_t\nfromAddressFamily(int AddressFamily) noexcept {\n  switch (AddressFamily) {\n  case PF_UNSPEC:\n    return __WASI_ADDRESS_FAMILY_UNSPEC;\n  case PF_INET:\n    return __WASI_ADDRESS_FAMILY_INET4;\n  case PF_INET6:\n    return __WASI_ADDRESS_FAMILY_INET6;\n  case PF_UNIX:\n    return __WASI_ADDRESS_FAMILY_AF_UNIX;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr int\ntoAddressFamily(__wasi_address_family_t AddressFamily) noexcept {\n  switch (AddressFamily) {\n  case __WASI_ADDRESS_FAMILY_UNSPEC:\n    return PF_UNSPEC;\n  case __WASI_ADDRESS_FAMILY_INET4:\n    return PF_INET;\n  case __WASI_ADDRESS_FAMILY_INET6:\n    return PF_INET6;\n  case __WASI_ADDRESS_FAMILY_AF_UNIX:\n    return PF_UNIX;\n  default:\n    assumingUnreachable();\n  }\n}\n\n} // namespace detail\n} // namespace WASI\n} // namespace Host\n} // namespace WasmEdge\n\n// Path: lib/host/wasi/inode-linux.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/defines.h\"\n#if WASMEDGE_OS_LINUX\n\n#include \"common/errcode.h\"\n#include \"common/variant.h\"\n#include \"host/wasi/environ.h\"\n#include \"host/wasi/inode.h\"\n#include \"host/wasi/vfs.h\"\n#include \"linux.h\"\n#include <algorithm>\n#include <cstddef>\n#include <cstdint>\n#include <cstring>\n#include <new>\n#include <string>\n#include <string_view>\n#include <vector>\n\nnamespace WasmEdge {\nnamespace Host {\nnamespace WASI {\n\nnamespace {\n\ninline constexpr bool isSpecialFd(int Fd) noexcept {\n  switch (Fd) {\n  case STDIN_FILENO:\n  case STDOUT_FILENO:\n  case STDERR_FILENO:\n    return true;\n  default:\n    return false;\n  }\n}\n\ninline constexpr __wasi_size_t\ncalculateAddrinfoLinkedListSize(struct addrinfo *const Addrinfo) {\n  __wasi_size_t Length = 0;\n  for (struct addrinfo *TmpPointer = Addrinfo; TmpPointer != nullptr;\n       TmpPointer = TmpPointer->ai_next) {\n    Length++;\n  }\n  return Length;\n};\n\nconstexpr int openFlags(__wasi_oflags_t OpenFlags, __wasi_fdflags_t FdFlags,\n                        VFS::Flags VFSFlags) noexcept {\n  int Flags = O_NOFOLLOW;\n#ifdef O_CLOEXEC\n  Flags |= O_CLOEXEC;\n#endif\n\n  if (VFSFlags & VFS::Read) {\n    if (VFSFlags & VFS::Write) {\n      Flags |= O_RDWR;\n    } else {\n      Flags |= O_RDONLY;\n    }\n  } else if (VFSFlags & VFS::Write) {\n    Flags |= O_WRONLY;\n  } else {\n#ifdef O_PATH\n    if (OpenFlags == __WASI_OFLAGS_DIRECTORY) {\n      Flags |= O_PATH;\n    } else {\n      Flags |= O_RDONLY;\n    }\n#else\n    Flags |= O_RDONLY;\n#endif\n  }\n\n  if (OpenFlags & __WASI_OFLAGS_CREAT) {\n    Flags |= O_CREAT;\n  }\n  if (OpenFlags & __WASI_OFLAGS_DIRECTORY) {\n    Flags |= O_DIRECTORY;\n  }\n  if (OpenFlags & __WASI_OFLAGS_EXCL) {\n    Flags |= O_EXCL;\n  }\n  if (OpenFlags & __WASI_OFLAGS_TRUNC) {\n    Flags |= O_TRUNC;\n  }\n\n  // Convert file descriptor flags.\n  if ((FdFlags & __WASI_FDFLAGS_APPEND) != 0) {\n    Flags |= O_APPEND;\n  }\n  if ((FdFlags & __WASI_FDFLAGS_DSYNC) != 0) {\n#ifdef O_DSYNC\n    Flags |= O_DSYNC;\n#else\n    Flags |= O_SYNC;\n#endif\n  }\n  if ((FdFlags & __WASI_FDFLAGS_NONBLOCK) != 0) {\n    Flags |= O_NONBLOCK;\n  }\n  if ((FdFlags & __WASI_FDFLAGS_RSYNC) != 0) {\n#ifdef O_RSYNC\n    Flags |= O_RSYNC;\n#else\n    Flags |= O_SYNC;\n#endif\n  }\n  if ((FdFlags & __WASI_FDFLAGS_SYNC) != 0) {\n    Flags |= O_SYNC;\n  }\n\n  return Flags;\n}\n\nstd::pair<const char *, std::unique_ptr<char[]>>\ncreateNullTerminatedString(std::string_view View) noexcept {\n  const char *CStr = nullptr;\n  std::unique_ptr<char[]> Buffer;\n  if (!View.empty()) {\n    if (const auto Pos = View.find_first_of('\\0');\n        Pos != std::string_view::npos) {\n      CStr = View.data();\n    } else {\n      Buffer = std::make_unique<char[]>(View.size() + 1);\n      std::copy(View.begin(), View.end(), Buffer.get());\n      CStr = Buffer.get();\n    }\n  }\n  return {CStr, std::move(Buffer)};\n}\n\n} // namespace\n\nvoid FdHolder::reset() noexcept {\n  if (likely(ok())) {\n    if (likely(!isSpecialFd(Fd))) {\n      ::close(Fd);\n    }\n    Fd = -1;\n  }\n}\n\nvoid TimerHolder::reset() noexcept {\n  if (likely(Id.has_value())) {\n    timer_delete(*Id);\n    Id.reset();\n  }\n}\n\nvoid DirHolder::reset() noexcept {\n  if (likely(Dir != nullptr)) {\n    closedir(Dir);\n    Dir = nullptr;\n    Cookie = 0;\n  }\n}\n\nINode INode::stdIn() noexcept { return INode(STDIN_FILENO); }\n\nINode INode::stdOut() noexcept { return INode(STDOUT_FILENO); }\n\nINode INode::stdErr() noexcept { return INode(STDERR_FILENO); }\n\nWasiExpect<INode> INode::open(std::string Path, __wasi_oflags_t OpenFlags,\n                              __wasi_fdflags_t FdFlags,\n                              VFS::Flags VFSFlags) noexcept {\n  const int Flags = openFlags(OpenFlags, FdFlags, VFSFlags);\n\n  if (auto NewFd = ::open(Path.c_str(), Flags, 0644); unlikely(NewFd < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    INode New(NewFd);\n#ifndef O_CLOEXEC\n    if (auto Res = ::fcntl(New.Fd, F_SETFD, FD_CLOEXEC); unlikely(Res != 0)) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n#endif\n    return New;\n  }\n}\n\nWasiExpect<void> INode::fdAdvise(__wasi_filesize_t Offset,\n                                 __wasi_filesize_t Len,\n                                 __wasi_advice_t Advice) const noexcept {\n  if (auto Res = ::posix_fadvise(Fd, Offset, Len, toAdvice(Advice));\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdAllocate(__wasi_filesize_t Offset,\n                                   __wasi_filesize_t Len) const noexcept {\n  if (auto Res = ::posix_fallocate(Fd, Offset, Len); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdDatasync() const noexcept {\n  if (auto Res = ::fdatasync(Fd); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdFdstatGet(__wasi_fdstat_t &FdStat) const noexcept {\n  if (auto Res = updateStat(); unlikely(!Res)) {\n    return WasiUnexpect(Res);\n  }\n\n  if (int FdFlags = ::fcntl(Fd, F_GETFL); unlikely(FdFlags < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    FdStat.fs_filetype = unsafeFiletype();\n\n    FdStat.fs_flags = static_cast<__wasi_fdflags_t>(0);\n    if (FdFlags & O_APPEND) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_APPEND;\n    }\n    if (FdFlags & O_DSYNC) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_DSYNC;\n    }\n    if (FdFlags & O_NONBLOCK) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_NONBLOCK;\n    }\n    if (FdFlags & O_SYNC) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_RSYNC | __WASI_FDFLAGS_SYNC;\n    }\n  }\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFdstatSetFlags(__wasi_fdflags_t FdFlags) const noexcept {\n  int SysFlag = 0;\n  if (FdFlags & __WASI_FDFLAGS_NONBLOCK) {\n    SysFlag |= O_NONBLOCK;\n  }\n  if (FdFlags & __WASI_FDFLAGS_APPEND) {\n    SysFlag |= O_APPEND;\n  }\n  if (FdFlags & __WASI_FDFLAGS_DSYNC) {\n    SysFlag |= O_DSYNC;\n  }\n  if (FdFlags & __WASI_FDFLAGS_RSYNC) {\n    SysFlag |= O_RSYNC;\n  }\n  if (FdFlags & __WASI_FDFLAGS_SYNC) {\n    SysFlag |= O_SYNC;\n  }\n\n  if (auto Res = ::fcntl(Fd, F_SETFL, SysFlag); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFilestatGet(__wasi_filestat_t &Filestat) const noexcept {\n  if (auto Res = updateStat(); unlikely(!Res)) {\n    return WasiUnexpect(Res);\n  }\n\n  // Zeroing out these values to prevent leaking information about the host\n  // environment from special fd such as stdin, stdout and stderr.\n  Filestat.dev = isSpecialFd(Fd) ? 0 : Stat->st_dev;\n  Filestat.ino = isSpecialFd(Fd) ? 0 : Stat->st_ino;\n  Filestat.filetype = unsafeFiletype();\n  Filestat.nlink = isSpecialFd(Fd) ? 0 : Stat->st_nlink;\n  Filestat.size = isSpecialFd(Fd) ? 0 : Stat->st_size;\n  Filestat.atim = isSpecialFd(Fd) ? 0 : fromTimespec(Stat->st_atim);\n  Filestat.mtim = isSpecialFd(Fd) ? 0 : fromTimespec(Stat->st_mtim);\n  Filestat.ctim = isSpecialFd(Fd) ? 0 : fromTimespec(Stat->st_ctim);\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFilestatSetSize(__wasi_filesize_t Size) const noexcept {\n  if (auto Res = ::ftruncate(Fd, Size); unlikely(Res == -1)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFilestatSetTimes(__wasi_timestamp_t ATim, __wasi_timestamp_t MTim,\n                          __wasi_fstflags_t FstFlags) const noexcept {\n#if __GLIBC_PREREQ(2, 6) || __BIONIC__\n  timespec SysTimespec[2];\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    SysTimespec[0] = toTimespec(ATim);\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    SysTimespec[0].tv_nsec = UTIME_NOW;\n  } else {\n    SysTimespec[0].tv_nsec = UTIME_OMIT;\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    SysTimespec[1] = toTimespec(MTim);\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    SysTimespec[1].tv_nsec = UTIME_NOW;\n  } else {\n    SysTimespec[1].tv_nsec = UTIME_OMIT;\n  }\n\n  if (auto Res = ::futimens(Fd, SysTimespec); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n#else\n  bool NeedNow = false;\n  bool NeedFile = false;\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    // Nothing to do.\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    NeedNow = true;\n  } else {\n    NeedFile = true;\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    // Nothing to do.\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    NeedNow = true;\n  } else {\n    NeedFile = true;\n  }\n\n  if (NeedFile) {\n    if (auto Res = updateStat(); unlikely(!Res)) {\n      return WasiUnexpect(Res);\n    }\n  }\n\n  timespec Now;\n  if (NeedNow) {\n    if (auto Res = ::clock_gettime(CLOCK_REALTIME, &Now); unlikely(Res != 0)) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n  }\n\n  timeval SysTimeval[2];\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    SysTimeval[0] = toTimeval(ATim);\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    SysTimeval[0] = toTimeval(Now);\n  } else {\n    SysTimeval[0] = toTimeval(Stat->st_atim);\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    SysTimeval[1] = toTimeval(MTim);\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    SysTimeval[1] = toTimeval(Now);\n  } else {\n    SysTimeval[1] = toTimeval(Stat->st_mtim);\n  }\n\n  if (auto Res = ::futimes(Fd, SysTimeval); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n#endif\n\n  return {};\n}\n\nWasiExpect<void> INode::fdPread(Span<Span<uint8_t>> IOVs,\n                                __wasi_filesize_t Offset,\n                                __wasi_size_t &NRead) const noexcept {\n  iovec SysIOVs[kIOVMax];\n  size_t SysIOVsSize = 0;\n  for (auto &IOV : IOVs) {\n    SysIOVs[SysIOVsSize].iov_base = IOV.data();\n    SysIOVs[SysIOVsSize].iov_len = IOV.size();\n    ++SysIOVsSize;\n  }\n\n#if __GLIBC_PREREQ(2, 10)\n  // Store read bytes length.\n  if (auto Res = ::preadv(Fd, SysIOVs, SysIOVsSize, Offset);\n      unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NRead = Res;\n  }\n#else\n  const auto OldOffset = ::lseek(Fd, 0, SEEK_CUR);\n  if (OldOffset < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (::lseek(Fd, Offset, SEEK_SET) < 0 ||\n      ::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (auto Res = ::readv(Fd, SysIOVs, SysIOVsSize); unlikely(Res < 0)) {\n    ::lseek(Fd, OldOffset, SEEK_SET);\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    if (::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n    NRead = Res;\n  }\n#endif\n\n  return {};\n}\n\nWasiExpect<void> INode::fdPwrite(Span<Span<const uint8_t>> IOVs,\n                                 __wasi_filesize_t Offset,\n                                 __wasi_size_t &NWritten) const noexcept {\n  iovec SysIOVs[kIOVMax];\n  size_t SysIOVsSize = 0;\n  for (auto &IOV : IOVs) {\n    SysIOVs[SysIOVsSize].iov_base = const_cast<uint8_t *>(IOV.data());\n    SysIOVs[SysIOVsSize].iov_len = IOV.size();\n    ++SysIOVsSize;\n  }\n\n#if __GLIBC_PREREQ(2, 10)\n  if (auto Res = ::pwritev(Fd, SysIOVs, SysIOVsSize, Offset);\n      unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NWritten = Res;\n  }\n#else\n  const auto OldOffset = ::lseek(Fd, 0, SEEK_CUR);\n  if (OldOffset < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (::lseek(Fd, Offset, SEEK_SET) < 0 ||\n      ::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (auto Res = ::writev(Fd, SysIOVs, SysIOVsSize); unlikely(Res < 0)) {\n    ::lseek(Fd, OldOffset, SEEK_SET);\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    if (::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n    NWritten = Res;\n  }\n#endif\n\n  return {};\n}\n\nWasiExpect<void> INode::fdRead(Span<Span<uint8_t>> IOVs,\n                               __wasi_size_t &NRead) const noexcept {\n  iovec SysIOVs[kIOVMax];\n  size_t SysIOVsSize = 0;\n  for (auto &IOV : IOVs) {\n    SysIOVs[SysIOVsSize].iov_base = IOV.data();\n    SysIOVs[SysIOVsSize].iov_len = IOV.size();\n    ++SysIOVsSize;\n  }\n\n  if (auto Res = ::readv(Fd, SysIOVs, SysIOVsSize); unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NRead = Res;\n  }\n\n  return {};\n}\n\n// Due to the unfortunate design of wasi::fd_readdir, It's nearly impossible to\n// provide a correct implementation. The below implementation is just a\n// workaround for most usages and may not be correct in some edge cases. The\n// readdir entry API is going to be updated to use a stream type, so we don't\n// have to deal with it right now.\nWasiExpect<void> INode::fdReaddir(Span<uint8_t> Buffer,\n                                  __wasi_dircookie_t Cookie,\n                                  __wasi_size_t &Size) noexcept {\n  if (unlikely(!Dir.ok())) {\n    if (FdHolder NewFd(::dup(Fd)); unlikely(!NewFd.ok())) {\n      return WasiUnexpect(fromErrNo(errno));\n    } else if (DIR *D = ::fdopendir(NewFd.Fd); unlikely(!D)) {\n      return WasiUnexpect(fromErrNo(errno));\n    } else {\n      NewFd.release();\n      Dir.emplace(D);\n    }\n  }\n\n  if (Cookie == 0) {\n    ::rewinddir(Dir.Dir);\n  } else if (unlikely(Cookie != Dir.Cookie)) {\n    ::seekdir(Dir.Dir, Cookie);\n  }\n\n  Size = 0;\n  do {\n    if (!Dir.Buffer.empty()) {\n      const auto NewDataSize =\n          std::min<uint32_t>(Buffer.size(), Dir.Buffer.size());\n      std::copy(Dir.Buffer.begin(), Dir.Buffer.begin() + NewDataSize,\n                Buffer.begin());\n      Buffer = Buffer.subspan(NewDataSize);\n      Size += NewDataSize;\n      Dir.Buffer.clear();\n      if (unlikely(Buffer.empty())) {\n        break;\n      }\n    }\n    errno = 0;\n    dirent *SysDirent = ::readdir(Dir.Dir);\n    if (SysDirent == nullptr) {\n      if (errno != 0) {\n        return WasiUnexpect(fromErrNo(errno));\n      }\n      // End of entries\n      break;\n    }\n    Dir.Cookie = SysDirent->d_off;\n    std::string_view Name = SysDirent->d_name;\n\n    Dir.Buffer.resize(sizeof(__wasi_dirent_t) + Name.size());\n\n    __wasi_dirent_t *const Dirent =\n        reinterpret_cast<__wasi_dirent_t *>(Dir.Buffer.data());\n    Dirent->d_next = Dir.Cookie;\n    Dirent->d_ino = SysDirent->d_ino;\n    Dirent->d_type = fromFileType(SysDirent->d_type);\n...\n", "needle_token_start": 7214, "needle_token_end": 7450, "code_context_ntokens": 16260, "output": ["```cpp\ninline constexpr int toAdvice(__wasi_advice_t Advice) noexcept {\n  switch (Advice) {\n  case __WASI_ADVICE_NORMAL:\n    return POSIX_FADV_NORMAL;\n  case __WASI_ADVICE_SEQUENTIAL:\n    return POSIX_FADV_SEQUENTIAL;\n  case __WASI_ADVICE_RANDOM:\n    return POSIX_FADV_RANDOM;\n  case __WASI_ADVICE_WILLNEED:\n    return POSIX_FADV_WILLNEED;\n  case __WASI_ADVICE_DONTNEED:\n    return POSIX_FADV_DONTNEED;\n  case __WASI_ADVICE_NOREUSE:\n    return POSIX_FADV_NOREUSE;\n  default:\n    assumingUnreachable();\n  }\n}\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "WasmEdge_CompilerContext", "language": "cpp", "path": "lib/api/wasmedge.cpp", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this structure is to initialize and manage the compilation process for WebAssembly modules, integrating various components necessary for compilation, code generation, loading, and validation.\n2. **Input**: It takes a configuration object which contains settings that influence the behavior of the compilation process.\n3. **Output**: It does not directly produce an output but sets up an environment for WebAssembly module compilation by initializing several internal components based on the provided configuration.\n4. **Procedure**: The procedure involves initializing internal components responsible for compiling, generating machine code, loading, and validating WebAssembly modules. These components are configured based on the settings provided in the input configuration object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/validator/formchecker.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"validator/formchecker.h\"\n\n#include \"common/errinfo.h\"\n#include \"common/log.h\"\n\n#include <algorithm>\n#include <array>\n#include <cstdint>\n#include <tuple>\n\nnamespace WasmEdge {\nnamespace Validator {\n\nnamespace {\n\n// Helper function for printing error log of index out of range.\nauto logOutOfRange(ErrCode Code, ErrInfo::IndexCategory Cate, uint32_t Idx,\n                   uint32_t Bound) {\n  spdlog::error(Code);\n  spdlog::error(ErrInfo::InfoForbidIndex(Cate, Idx, Bound));\n  return Unexpect(Code);\n}\n\n} // namespace\n\nvoid FormChecker::reset(bool CleanGlobal) {\n  ValStack.clear();\n  CtrlStack.clear();\n  Locals.clear();\n  Returns.clear();\n\n  if (CleanGlobal) {\n    Types.clear();\n    Funcs.clear();\n    Tables.clear();\n    Mems = 0;\n    Globals.clear();\n    Datas.clear();\n    Elems.clear();\n    Refs.clear();\n    NumImportFuncs = 0;\n    NumImportGlobals = 0;\n  }\n}\n\nExpect<void> FormChecker::validate(AST::InstrView Instrs,\n                                   Span<const ValType> RetVals) {\n  for (const ValType &Val : RetVals) {\n    Returns.push_back(Val);\n  }\n  return checkExpr(Instrs);\n}\n\nExpect<void> FormChecker::validate(const ValType &VT) const noexcept {\n  // The value type should be validated for the type index case.\n  if (VT.isRefType() && VT.getHeapTypeCode() == TypeCode::TypeIndex) {\n    if (VT.getTypeIndex() >= Types.size()) {\n      spdlog::error(ErrCode::Value::InvalidFuncTypeIdx);\n      spdlog::error(ErrInfo::InfoForbidIndex(\n          ErrInfo::IndexCategory::FunctionType, VT.getTypeIndex(),\n          static_cast<uint32_t>(Types.size())));\n      return Unexpect(ErrCode::Value::InvalidFuncTypeIdx);\n    }\n  }\n  return {};\n}\n\nvoid FormChecker::addType(const AST::SubType &Type) { Types.push_back(&Type); }\n\nvoid FormChecker::addFunc(const uint32_t TypeIdx, const bool IsImport) {\n  if (Types.size() > TypeIdx) {\n    Funcs.emplace_back(TypeIdx);\n  }\n  if (IsImport) {\n    NumImportFuncs++;\n  }\n}\n\nvoid FormChecker::addTable(const AST::TableType &Tab) {\n  Tables.push_back(Tab.getRefType());\n}\n\nvoid FormChecker::addMemory(const AST::MemoryType &) { Mems++; }\n\nvoid FormChecker::addGlobal(const AST::GlobalType &Glob, const bool IsImport) {\n  // Type in global is confirmed in loading phase.\n  Globals.emplace_back(Glob.getValType(), Glob.getValMut());\n  if (IsImport) {\n    NumImportGlobals++;\n  }\n}\n\nvoid FormChecker::addData(const AST::DataSegment &) {\n  Datas.emplace_back(static_cast<uint32_t>(Datas.size()));\n}\n\n...\n// Path: lib/aot/blake3.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"aot/blake3.h\"\n\n#include \"common/config.h\"\n#include \"common/defines.h\"\n\nnamespace WasmEdge {\nnamespace AOT {\n\nnamespace {}\n\nBlake3::Blake3() noexcept { blake3_hasher_init(&Hasher); }\n\nvoid Blake3::update(Span<const Byte> Data) noexcept {\n  blake3_hasher_update(&Hasher, Data.data(), Data.size());\n}\n\nvoid Blake3::finalize(Span<Byte> Output) noexcept {\n  blake3_hasher_finalize(&Hasher, Output.data(), Output.size());\n}\n\n} // namespace AOT\n} // namespace WasmEdge\n\n// Path: lib/aot/cache.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"aot/cache.h\"\n\n#include \"aot/blake3.h\"\n#include \"common/config.h\"\n#include \"common/defines.h\"\n#include \"common/hexstr.h\"\n#include \"system/path.h\"\n\n#include <array>\n#include <string>\n#include <system_error>\n\nnamespace WasmEdge {\nnamespace AOT {\n\nnamespace {\nstd::filesystem::path getRoot(Cache::StorageScope Scope) {\n  switch (Scope) {\n  case Cache::StorageScope::Global:\n    return std::filesystem::u8path(kCacheRoot);\n  case Cache::StorageScope::Local: {\n    if (const auto Home = Path::home(); !Home.empty()) {\n      return Home / \"cache\"sv;\n    }\n    return {};\n  }\n  default:\n    assumingUnreachable();\n  }\n}\n} // namespace\n\nExpect<std::filesystem::path> Cache::getPath(Span<const Byte> Data,\n                                             Cache::StorageScope Scope,\n                                             std::string_view Key) {\n  auto Root = getRoot(Scope);\n  if (!Key.empty()) {\n    Root /= std::filesystem::u8path(Key);\n  }\n\n  Blake3 Hasher;\n  Hasher.update(Data);\n  std::array<Byte, 32> Hash;\n  Hasher.finalize(Hash);\n  std::string HexStr;\n  convertBytesToHexStr(Hash, HexStr);\n\n  return Root / HexStr;\n}\n\nvoid Cache::clear(Cache::StorageScope Scope, std::string_view Key) {\n  auto Root = getRoot(Scope);\n  if (!Key.empty()) {\n    Root /= std::filesystem::u8path(Key);\n  }\n  std::error_code ErrCode;\n  std::filesystem::remove_all(Root, ErrCode);\n}\n\n} // namespace AOT\n} // namespace WasmEdge\n\n// Path: lib/vm/vm.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"vm/vm.h\"\n\n#include \"host/wasi/wasimodule.h\"\n#include \"plugin/plugin.h\"\n#include \"llvm/compiler.h\"\n#include \"llvm/jit.h\"\n\n#include \"host/mock/wasi_crypto_module.h\"\n#include \"host/mock/wasi_logging_module.h\"\n#include \"host/mock/wasi_nn_module.h\"\n#include \"host/mock/wasmedge_image_module.h\"\n#include \"host/mock/wasmedge_process_module.h\"\n#include \"host/mock/wasmedge_tensorflow_module.h\"\n#include \"host/mock/wasmedge_tensorflowlite_module.h\"\n#include <variant>\n\nnamespace WasmEdge {\nnamespace VM {\n\nnamespace {\ntemplate <typename T>\nstd::unique_ptr<Runtime::Instance::ModuleInstance>\ncreatePluginModule(std::string_view PName, std::string_view MName) {\n  using namespace std::literals::string_view_literals;\n  if (const auto *Plugin = Plugin::Plugin::find(PName)) {\n    if (const auto *Module = Plugin->findModule(MName)) {\n      return Module->create();\n    }\n  }\n  spdlog::debug(\"Plugin: {} , module name: {} not found. Mock instead.\"sv,\n                PName, MName);\n  return std::make_unique<T>();\n}\n} // namespace\n\nVM::VM(const Configure &Conf)\n    : Conf(Conf), Stage(VMStage::Inited),\n      LoaderEngine(Conf, &Executor::Executor::Intrinsics),\n      ValidatorEngine(Conf), ExecutorEngine(Conf, &Stat),\n      Store(std::make_unique<Runtime::StoreManager>()), StoreRef(*Store.get()) {\n  unsafeInitVM();\n}\n\nVM::VM(const Configure &Conf, Runtime::StoreManager &S)\n    : Conf(Conf), Stage(VMStage::Inited),\n      LoaderEngine(Conf, &Executor::Executor::Intrinsics),\n      ValidatorEngine(Conf), ExecutorEngine(Conf, &Stat), StoreRef(S) {\n  unsafeInitVM();\n}\n\nvoid VM::unsafeInitVM() {\n  // Load the built-in modules and the plug-ins.\n  unsafeLoadBuiltInHosts();\n  unsafeLoadPlugInHosts();\n\n  // Register all module instances.\n  unsafeRegisterBuiltInHosts();\n  unsafeRegisterPlugInHosts();\n}\n\nvoid VM::unsafeLoadBuiltInHosts() {\n  // Load the built-in host modules from configuration.\n  // TODO: This will be extended for the versionlized WASI in the future.\n  BuiltInModInsts.clear();\n  if (Conf.hasHostRegistration(HostRegistration::Wasi)) {\n    std::unique_ptr<Runtime::Instance::ModuleInstance> WasiMod =\n        std::make_unique<Host::WasiModule>();\n    BuiltInModInsts.insert({HostRegistration::Wasi, std::move(WasiMod)});\n  }\n}\n\nvoid VM::unsafeLoadPlugInHosts() {\n  // Load the plugins and mock them if not found.\n  using namespace std::literals::string_view_literals;\n  PlugInModInsts.clear();\n\n  PlugInModInsts.push_back(\n      createPluginModule<Host::WasiNNModuleMock>(\"wasi_nn\"sv, \"wasi_nn\"sv));\n  PlugInModInsts.push_back(createPluginModule<Host::WasiCryptoCommonModuleMock>(\n      \"wasi_crypto\"sv, \"wasi_crypto_common\"sv));\n  PlugInModInsts.push_back(\n      createPluginModule<Host::WasiCryptoAsymmetricCommonModuleMock>(\n          \"wasi_crypto\"sv, \"wasi_crypto_asymmetric_common\"sv));\n  PlugInModInsts.push_back(createPluginModule<Host::WasiCryptoKxModuleMock>(\n      \"wasi_crypto\"sv, \"wasi_crypto_kx\"sv));\n  PlugInModInsts.push_back(\n      createPluginModule<Host::WasiCryptoSignaturesModuleMock>(\n          \"wasi_crypto\"sv, \"wasi_crypto_signatures\"sv));\n  PlugInModInsts.push_back(\n      createPluginModule<Host::WasiCryptoSymmetricModuleMock>(\n          \"wasi_crypto\"sv, \"wasi_crypto_symmetric\"sv));\n  PlugInModInsts.push_back(createPluginModule<Host::WasmEdgeProcessModuleMock>(\n      \"wasmedge_process\"sv, \"wasmedge_process\"sv));\n  PlugInModInsts.push_back(createPluginModule<Host::WasiLoggingModuleMock>(\n      \"wasi_logging\"sv, \"wasi:logging/logging\"sv));\n  PlugInModInsts.push_back(\n      createPluginModule<Host::WasmEdgeTensorflowModuleMock>(\n          \"wasmedge_tensorflow\"sv, \"wasmedge_tensorflow\"sv));\n  PlugInModInsts.push_back(\n      createPluginModule<Host::WasmEdgeTensorflowLiteModuleMock>(\n          \"wasmedge_tensorflowlite\"sv, \"wasmedge_tensorflowlite\"sv));\n  PlugInModInsts.push_back(createPluginModule<Host::WasmEdgeImageModuleMock>(\n      \"wasmedge_image\"sv, \"wasmedge_image\"sv));\n\n  // Load the other non-official plugins.\n  for (const auto &Plugin : Plugin::Plugin::plugins()) {\n    if (Conf.isForbiddenPlugins(Plugin.name())) {\n      continue;\n    }\n    // Skip wasi_crypto, wasi_nn, wasi_logging, WasmEdge_Process,\n    // WasmEdge_Tensorflow, WasmEdge_TensorflowLite, and WasmEdge_Image.\n    if (Plugin.name() == \"wasi_crypto\"sv || Plugin.name() == \"wasi_nn\"sv ||\n        Plugin.name() == \"wasi_logging\"sv ||\n        Plugin.name() == \"wasmedge_process\"sv ||\n        Plugin.name() == \"wasmedge_tensorflow\"sv ||\n        Plugin.name() == \"wasmedge_tensorflowlite\"sv ||\n        Plugin.name() == \"wasmedge_image\"sv) {\n      continue;\n    }\n    for (const auto &Module : Plugin.modules()) {\n      PlugInModInsts.push_back(Module.create());\n    }\n  }\n}\n\nvoid VM::unsafeRegisterBuiltInHosts() {\n  // Register all created WASI host modules.\n  for (auto &It : BuiltInModInsts) {\n    ExecutorEngine.registerModule(StoreRef, *(It.second.get()));\n  }\n}\n\nvoid VM::unsafeRegisterPlugInHosts() {\n  // Register all created module instances from plugins.\n  for (auto &It : PlugInModInsts) {\n    ExecutorEngine.registerModule(StoreRef, *(It.get()));\n  }\n}\n\nExpect<void> VM::unsafeRegisterModule(std::string_view Name,\n                                      const std::filesystem::path &Path) {\n  if (Stage == VMStage::Instantiated) {\n    // When registering module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  // Load module.\n  if (auto Res = LoaderEngine.parseModule(Path)) {\n    return unsafeRegisterModule(Name, *(*Res).get());\n  } else {\n    return Unexpect(Res);\n  }\n}\n\nExpect<void> VM::unsafeRegisterModule(std::string_view Name,\n                                      Span<const Byte> Code) {\n  if (Stage == VMStage::Instantiated) {\n    // When registering module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  // Load module.\n  if (auto Res = LoaderEngine.parseModule(Code)) {\n    return unsafeRegisterModule(Name, *(*Res).get());\n  } else {\n    return Unexpect(Res);\n  }\n}\n\nExpect<void> VM::unsafeRegisterModule(std::string_view Name,\n                                      const AST::Module &Module) {\n  if (Stage == VMStage::Instantiated) {\n    // When registering module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  // Validate module.\n  if (auto Res = ValidatorEngine.validate(Module); !Res) {\n    return Unexpect(Res);\n  }\n  // Instantiate and register module.\n  if (auto Res = ExecutorEngine.registerModule(StoreRef, Module, Name)) {\n    RegModInsts.push_back(std::move(*Res));\n    return {};\n  } else {\n    return Unexpect(Res);\n  }\n}\n\nExpect<void>\nVM::unsafeRegisterModule(const Runtime::Instance::ModuleInstance &ModInst) {\n  if (Stage == VMStage::Instantiated) {\n    // When registering module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  return ExecutorEngine.registerModule(StoreRef, ModInst);\n}\n\nExpect<std::vector<std::pair<ValVariant, ValType>>>\nVM::unsafeRunWasmFile(const std::filesystem::path &Path, std::string_view Func,\n                      Span<const ValVariant> Params,\n                      Span<const ValType> ParamTypes) {\n  if (Stage == VMStage::Instantiated) {\n    // When running another module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  // Load wasm unit.\n  if (auto Res = LoaderEngine.parseWasmUnit(Path)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*Res)) {\n      auto M = std::move(std::get<std::unique_ptr<AST::Module>>(*Res));\n      return unsafeRunWasmFile(*M, Func, Params, ParamTypes);\n    } else {\n      return unsafeRunWasmFile(\n          (*std::get<std::unique_ptr<AST::Component::Component>>(*Res)), Func,\n          Params, ParamTypes);\n    }\n  } else {\n    return Unexpect(Res);\n  }\n}\n\nExpect<std::vector<std::pair<ValVariant, ValType>>>\nVM::unsafeRunWasmFile(Span<const Byte> Code, std::string_view Func,\n                      Span<const ValVariant> Params,\n                      Span<const ValType> ParamTypes) {\n  if (Stage == VMStage::Instantiated) {\n    // When running another module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  // Load wasm unit.\n  if (auto Res = LoaderEngine.parseWasmUnit(Code)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*Res)) {\n      std::unique_ptr<AST::Module> M =\n          std::move(std::get<std::unique_ptr<AST::Module>>(*Res));\n      return unsafeRunWasmFile(*M, Func, Params, ParamTypes);\n    } else {\n      std::unique_ptr<AST::Component::Component> C =\n          std::move(std::get<std::unique_ptr<AST::Component::Component>>(*Res));\n      return unsafeRunWasmFile(*C, Func, Params, ParamTypes);\n    }\n  } else {\n    return Unexpect(Res);\n  }\n}\n\nExpect<std::vector<std::pair<ValVariant, ValType>>>\nVM::unsafeRunWasmFile(const AST::Component::Component &Component,\n                      std::string_view, Span<const ValVariant>,\n                      Span<const ValType>) {\n  if (Stage == VMStage::Instantiated) {\n    // When running another module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  if (auto Res = ValidatorEngine.validate(Component); !Res) {\n    return Unexpect(Res);\n  }\n  spdlog::error(\"component execution is not done yet.\");\n  return Unexpect(ErrCode::Value::RuntimeError);\n}\n\nExpect<std::vector<std::pair<ValVariant, ValType>>>\nVM::unsafeRunWasmFile(const AST::Module &Module, std::string_view Func,\n                      Span<const ValVariant> Params,\n                      Span<const ValType> ParamTypes) {\n  if (Stage == VMStage::Instantiated) {\n    // When running another module, instantiated module in store will be reset.\n    // Therefore the instantiation should restart.\n    Stage = VMStage::Validated;\n  }\n  if (auto Res = ValidatorEngine.validate(Module); !Res) {\n    return Unexpect(Res);\n  }\n  if (auto Res = ExecutorEngine.instantiateModule(StoreRef, Module)) {\n    ActiveModInst = std::move(*Res);\n  } else {\n    return Unexpect(Res);\n  }\n  // Get module instance.\n  if (ActiveModInst) {\n    // Execute function and return values with the module instance.\n    return unsafeExecute(ActiveModInst.get(), Func, Params, ParamTypes);\n  } else {\n    spdlog::error(ErrCode::Value::WrongInstanceAddress);\n    spdlog::error(ErrInfo::InfoExecuting(\"\", Func));\n    return Unexpect(ErrCode::Value::WrongInstanceAddress);\n  }\n}\n\nAsync<Expect<std::vector<std::pair<ValVariant, ValType>>>>\nVM::asyncRunWasmFile(const std::filesystem::path &Path, std::string_view Func,\n                     Span<const ValVariant> Params,\n                     Span<const ValType> ParamTypes) {\n  Expect<std::vector<std::pair<ValVariant, ValType>>> (VM::*FPtr)(\n      const std::filesystem::path &, std::string_view, Span<const ValVariant>,\n      Span<const ValType>) = &VM::runWasmFile;\n  return {FPtr,\n          *this,\n          std::filesystem::path(Path),\n          std::string(Func),\n          std::vector(Params.begin(), Params.end()),\n          std::vector(ParamTypes.begin(), ParamTypes.end())};\n}\n\nAsync<Expect<std::vector<std::pair<ValVariant, ValType>>>>\nVM::asyncRunWasmFile(Span<const Byte> Code, std::string_view Func,\n                     Span<const ValVariant> Params,\n                     Span<const ValType> ParamTypes) {\n  Expect<std::vector<std::pair<ValVariant, ValType>>> (VM::*FPtr)(\n      Span<const Byte>, std::string_view, Span<const ValVariant>,\n      Span<const ValType>) = &VM::runWasmFile;\n  return {FPtr,\n          *this,\n          Code,\n          std::string(Func),\n          std::vector(Params.begin(), Params.end()),\n          std::vector(ParamTypes.begin(), ParamTypes.end())};\n}\n\nAsync<Expect<std::vector<std::pair<ValVariant, ValType>>>>\nVM::asyncRunWasmFile(const AST::Module &Module, std::string_view Func,\n                     Span<const ValVariant> Params,\n                     Span<const ValType> ParamTypes) {\n  Expect<std::vector<std::pair<ValVariant, ValType>>> (VM::*FPtr)(\n      const AST::Module &, std::string_view, Span<const ValVariant>,\n      Span<const ValType>) = &VM::runWasmFile;\n  return {FPtr,\n          *this,\n          Module,\n          std::string(Func),\n          std::vector(Params.begin(), Params.end()),\n          std::vector(ParamTypes.begin(), ParamTypes.end())};\n}\n\nExpect<void> VM::unsafeLoadWasm(const std::filesystem::path &Path) {\n  // If not load successfully, the previous status will be reserved.\n  if (auto Res = LoaderEngine.parseWasmUnit(Path)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*Res)) {\n      Mod = std::move(std::get<std::unique_ptr<AST::Module>>(*Res));\n    } else if (std::holds_alternative<\n                   std::unique_ptr<AST::Component::Component>>(*Res)) {\n      spdlog::error(\"component execution is not done yet.\");\n    } else {\n      return Unexpect(Res);\n    }\n    Stage = VMStage::Loaded;\n  } else {\n    return Unexpect(Res);\n  }\n  return {};\n}\n\nExpect<void> VM::unsafeLoadWasm(Span<const Byte> Code) {\n  // If not load successfully, the previous status will be reserved.\n  if (auto Res = LoaderEngine.parseWasmUnit(Code)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*Res)) {\n      Mod = std::move(std::get<std::unique_ptr<AST::Module>>(*Res));\n    } else if (std::holds_alternative<\n                   std::unique_ptr<AST::Component::Component>>(*Res)) {\n      spdlog::error(\"component execution is not done yet.\");\n    } else {\n      return Unexpect(Res);\n    }\n    Stage = VMStage::Loaded;\n  } else {\n    return Unexpect(Res);\n  }\n  return {};\n}\n\nExpect<void> VM::unsafeLoadWasm(const AST::Module &Module) {\n  Mod = std::make_unique<AST::Module>(Module);\n  Stage = VMStage::Loaded;\n  return {};\n}\n\nExpect<void> VM::unsafeValidate() {\n  if (Stage < VMStage::Loaded) {\n    // When module is not loaded, not validate.\n    spdlog::error(ErrCode::Value::WrongVMWorkflow);\n    return Unexpect(ErrCode::Value::WrongVMWorkflow);\n  }\n  if (auto Res = ValidatorEngine.validate(*Mod.get())) {\n    Stage = VMStage::Validated;\n    return {};\n  } else {\n    return Unexpect(Res);\n  }\n}\n\nExpect<void> VM::unsafeInstantiate() {\n  if (Stage < VMStage::Validated) {\n    // When module is not validated, not instantiate.\n    spdlog::error(ErrCode::Value::WrongVMWorkflow);\n    return Unexpect(ErrCode::Value::WrongVMWorkflow);\n  }\n\n  if (Mod) {\n    if (Conf.getRuntimeConfigure().isEnableJIT() && !Mod->getSymbol()) {\n#ifdef WASMEDGE_USE_LLVM\n      LLVM::Compiler Compiler(Conf);\n      LLVM::JIT JIT(Conf);\n      if (auto Res = Compiler.compile(*Mod); !Res) {\n        const auto Err = static_cast<uint32_t>(Res.error());\n        spdlog::error(\n            \"Compilation failed. Error code: {}, use interpreter mode instead.\"sv,\n            Err);\n      } else if (auto Res2 = JIT.load(std::move(*Res)); !Res2) {\n        const auto Err = static_cast<uint32_t>(Res2.error());\n        spdlog::warn(\n            \"JIT failed. Error code: {}, use interpreter mode instead.\"sv, Err);\n      } else {\n        LoaderEngine.loadExecutable(*Mod, std::move(*Res2));\n      }\n#else\n      spdlog::error(\"LLVM disabled, JIT is unsupported!\");\n#endif\n    }\n  }\n\n  if (auto Res = ExecutorEngine.instantiateModule(StoreRef, *Mod.get())) {\n    Stage = VMStage::Instantiated;\n    ActiveModInst = std::move(*Res);\n    return {};\n  } else {\n    return Unexpect(Res);\n  }\n}\n\nExpect<std::vector<std::pair<ValVariant, ValType>>>\nVM::unsafeExecute(std::string_view Func, Span<const ValVariant> Params,\n                  Span<const ValType> ParamTypes) {\n  if (ActiveModInst) {\n    // Execute function and return values with the module instance.\n    return unsafeExecute(ActiveModInst.get(), Func, Params, ParamTypes);\n  } else {\n    spdlog::error(ErrCode::Value::WrongInstanceAddress);\n    spdlog::error(ErrInfo::InfoExecuting(\"\", Func));\n    return Unexpect(ErrCode::Value::WrongInstanceAddress);\n  }\n}\n\nExpect<std::vector<std::pair<ValVariant, ValType>>>\nVM::unsafeExecute(std::string_view ModName, std::string_view Func,\n                  Span<const ValVariant> Params,\n                  Span<const ValType> ParamTypes) {\n  // Find module instance by name.\n  const auto *FindModInst = StoreRef.findModule(ModName);\n  if (FindModInst != nullptr) {\n    // Execute function and return values with the module instance.\n    return unsafeExecute(FindModInst, Func, Params, ParamTypes);\n  } else {\n    spdlog::error(ErrCode::Value::WrongInstanceAddress);\n    spdlog::error(ErrInfo::InfoExecuting(ModName, Func));\n    return Unexpect(ErrCode::Value::WrongInstanceAddress);\n  }\n}\n\nExpect<std::vector<std::pair<ValVariant, ValType>>>\nVM::unsafeExecute(const Runtime::Instance::ModuleInstance *ModInst,\n                  std::string_view Func, Span<const ValVariant> Params,\n                  Span<const ValType> ParamTypes) {\n  // Find exported function by name.\n  Runtime::Instance::FunctionInstance *FuncInst =\n      ModInst->findFuncExports(Func);\n\n  // Execute function.\n  if (auto Res = ExecutorEngine.invoke(FuncInst, Params, ParamTypes);\n      unlikely(!Res)) {\n    if (Res.error() != ErrCode::Value::Terminated) {\n      spdlog::error(ErrInfo::InfoExecuting(ModInst->getModuleName(), Func));\n    }\n    return Unexpect(Res);\n  } else {\n    return Res;\n  }\n}\n\nAsync<Expect<std::vector<std::pair<ValVariant, ValType>>>>\nVM::asyncExecute(std::string_view Func, Span<const ValVariant> Params,\n                 Span<const ValType> ParamTypes) {\n  Expect<std::vector<std::pair<ValVariant, ValType>>> (VM::*FPtr)(\n      std::string_view, Span<const ValVariant>, Span<const ValType>) =\n      &VM::execute;\n  return {FPtr, *this, std::string(Func),\n          std::vector(Params.begin(), Params.end()),\n          std::vector(ParamTypes.begin(), ParamTypes.end())};\n}\n\nAsync<Expect<std::vector<std::pair<ValVariant, ValType>>>>\nVM::asyncExecute(std::string_view ModName, std::string_view Func,\n                 Span<const ValVariant> Params,\n                 Span<const ValType> ParamTypes) {\n  Expect<std::vector<std::pair<ValVariant, ValType>>> (VM::*FPtr)(\n      std::string_view, std::string_view, Span<const ValVariant>,\n      Span<const ValType>) = &VM::execute;\n  return {FPtr,\n          *this,\n          std::string(ModName),\n          std::string(Func),\n          std::vector(Params.begin(), Params.end()),\n          std::vector(ParamTypes.begin(), ParamTypes.end())};\n}\n\nvoid VM::unsafeCleanup() {\n  Mod.reset();\n  ActiveModInst.reset();\n  StoreRef.reset();\n  RegModInsts.clear();\n  Stat.clear();\n  unsafeLoadBuiltInHosts();\n  unsafeLoadPlugInHosts();\n  unsafeRegisterBuiltInHosts();\n  unsafeRegisterPlugInHosts();\n  LoaderEngine.reset();\n  Stage = VMStage::Inited;\n}\n\nstd::vector<std::pair<std::string, const AST::FunctionType &>>\nVM::unsafeGetFunctionList() const {\n  std::vector<std::pair<std::string, const AST::FunctionType &>> Map;\n  if (ActiveModInst) {\n    ActiveModInst->getFuncExports([&](const auto &FuncExports) {\n      Map.reserve(FuncExports.size());\n      for (auto &&Func : FuncExports) {\n        const auto &FuncType = (Func.second)->getFuncType();\n        Map.emplace_back(Func.first, FuncType);\n      }\n    });\n  }\n  return Map;\n}\n\nRuntime::Instance::ModuleInstance *\nVM::unsafeGetImportModule(const HostRegistration Type) const {\n  if (auto Iter = BuiltInModInsts.find(Type); Iter != BuiltInModInsts.cend()) {\n    return Iter->second.get();\n  }\n  return nullptr;\n}\n\nconst Runtime::Instance::ModuleInstance *VM::unsafeGetActiveModule() const {\n  if (ActiveModInst) {\n    return ActiveModInst.get();\n  }\n  return nullptr;\n};\n\n} // namespace VM\n} // namespace WasmEdge\n\n// Path: lib/api/wasmedge.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"wasmedge/wasmedge.h\"\n\n#include \"common/defines.h\"\n#include \"driver/compiler.h\"\n#include \"driver/tool.h\"\n#include \"driver/unitool.h\"\n#include \"host/wasi/wasimodule.h\"\n#include \"plugin/plugin.h\"\n#include \"system/winapi.h\"\n#include \"vm/vm.h\"\n#include \"llvm/codegen.h\"\n#include \"llvm/compiler.h\"\n\n#ifdef WASMEDGE_BUILD_FUZZING\n#include \"driver/fuzzPO.h\"\n#include \"driver/fuzzTool.h\"\n#endif\n\n#ifdef WASMEDGE_BUILD_WASI_NN_RPC\n#include \"driver/wasiNNRPCServerTool.h\"\n#endif\n\n#include <algorithm>\n#include <chrono>\n#include <cstdint>\n#include <cstdlib>\n#include <cstring>\n#include <functional>\n#include <map>\n#include <memory>\n#include <string>\n#include <string_view>\n#include <utility>\n#include <vector>\n\n// WasmEdge_ConfigureContext implementation.\nstruct WasmEdge_ConfigureContext {\n  WasmEdge::Configure Conf;\n};\n\n// WasmEdge_StatisticsContext implementation.\nstruct WasmEdge_StatisticsContext {};\n\n// WasmEdge_ASTModuleContext implementation.\nstruct WasmEdge_ASTModuleContext {};\n\n// WasmEdge_FunctionTypeContext implementation.\nstruct WasmEdge_FunctionTypeContext {};\n\n// WasmEdge_TableTypeContext implementation.\nstruct WasmEdge_TableTypeContext {};\n\n// WasmEdge_MemoryTypeContext implementation.\nstruct WasmEdge_MemoryTypeContext {};\n\n// WasmEdge_GlobalTypeContext implementation.\nstruct WasmEdge_GlobalTypeContext {};\n\n// WasmEdge_ImportTypeContext implementation.\nstruct WasmEdge_ImportTypeContext {};\n\n// WasmEdge_ExportTypeContext implementation.\nstruct WasmEdge_ExportTypeContext {};\n\n// WasmEdge_CompilerContext implementation.\nstruct WasmEdge_CompilerContext {\n#ifdef WASMEDGE_USE_LLVM\n  \nWasmEdge_CompilerContext(const WasmEdge::Configure &Conf) noexcept\n      : Compiler(Conf), CodeGen(Conf), Load(Conf), Valid(Conf) {}\n  WasmEdge::LLVM::Compiler Compiler;\n  WasmEdge::LLVM::CodeGen CodeGen;\n  WasmEdge::Loader::Loader Load;\n  WasmEdge::Validator::Validator Valid;\n#endif\n};\n\n// WasmEdge_LoaderContext implementation.\nstruct WasmEdge_LoaderContext {};\n\n// WasmEdge_ValidatorContext implementation.\nstruct WasmEdge_ValidatorContext {};\n\n// WasmEdge_ExecutorContext implementation.\nstruct WasmEdge_ExecutorContext {};\n\n// WasmEdge_StoreContext implementation.\nstruct WasmEdge_StoreContext {};\n\n// WasmEdge_ModuleInstanceContext implementation.\nstruct WasmEdge_ModuleInstanceContext {};\n\n// WasmEdge_FunctionInstanceContext implementation.\nstruct WasmEdge_FunctionInstanceContext {};\n\n// WasmEdge_TableInstanceContext implementation.\nstruct WasmEdge_TableInstanceContext {};\n\n// WasmEdge_MemoryInstanceContext implementation.\nstruct WasmEdge_MemoryInstanceContext {};\n\n// WasmEdge_GlobalInstanceContext implementation.\nstruct WasmEdge_GlobalInstanceContext {};\n\n// WasmEdge_CallingFrameContext implementation.\nstruct WasmEdge_CallingFrameContext {};\n\n// WasmEdge_Async implementation.\nstruct WasmEdge_Async {\n  template <typename... Args>\n  WasmEdge_Async(Args &&...Vals) noexcept\n      : Async(std::forward<Args>(Vals)...) {}\n  WasmEdge::Async<WasmEdge::Expect<\n      std::vector<std::pair<WasmEdge::ValVariant, WasmEdge::ValType>>>>\n      Async;\n};\n\n// WasmEdge_VMContext implementation.\nstruct WasmEdge_VMContext {\n  template <typename... Args>\n  WasmEdge_VMContext(Args &&...Vals) noexcept\n      : VM(std::forward<Args>(Vals)...) {}\n  WasmEdge::VM::VM VM;\n};\n\n// WasmEdge_PluginContext implementation.\nstruct WasmEdge_PluginContext {};\n\nnamespace {\n\nusing namespace WasmEdge;\n\n// Helper function for returning a WasmEdge_Result by error code.\ninline constexpr WasmEdge_Result\ngenWasmEdge_Result(const ErrCode::Value &Code) noexcept {\n  return WasmEdge_Result{/* Code */ static_cast<uint32_t>(Code) & 0x00FFFFFFU};\n}\ninline constexpr WasmEdge_Result\ngenWasmEdge_Result(const ErrCode &Code) noexcept {\n  return WasmEdge_Result{/* Code */ Code.operator uint32_t()};\n}\n\n// Helper function for returning a struct uint128_t / int128_t\n// from class WasmEdge::uint128_t / WasmEdge::int128_t.\ntemplate <typename C>\ninline constexpr ::uint128_t to_uint128_t(C Val) noexcept {\n#if defined(__x86_64__) || defined(__aarch64__) ||                             \\\n    (defined(__riscv) && __riscv_xlen == 64)\n  return Val;\n#else\n  return {/* Low */ Val.low(), /* High */ static_cast<uint64_t>(Val.high())};\n#endif\n}\ntemplate <typename C> inline constexpr ::int128_t to_int128_t(C Val) noexcept {\n#if defined(__x86_64__) || defined(__aarch64__) ||                             \\\n    (defined(__riscv) && __riscv_xlen == 64)\n  return Val;\n#else\n  return {/* Low */ Val.low(), /* High */ Val.high()};\n#endif\n}\n\n// Helper function for returning a class WasmEdge::uint128_t /\n// WasmEdge::int128_t from struct uint128_t / int128_t.\ntemplate <typename C, typename T>\ninline constexpr C to_WasmEdge_128_t(T Val) noexcept {\n#if defined(__x86_64__) || defined(__aarch64__) ||                             \\\n    (defined(__riscv) && __riscv_xlen == 64)\n  return Val;\n#else\n  return C(Val.High, Val.Low);\n#endif\n}\n\n// Helper functions for returning a WasmEdge::ValType by WasmEdge_ValType.\ninline ValType genValType(const WasmEdge_ValType &T) noexcept {\n  std::array<uint8_t, 8> R;\n  std::copy_n(T.Data, 8, R.begin());\n  return ValType(R);\n}\n\n// Helper functions for returning a WasmEdge_ValType by WasmEdge::ValType.\ninline WasmEdge_ValType genWasmEdge_ValType(const ValType &T) noexcept {\n  WasmEdge_ValType VT;\n  std::copy_n(T.getRawData().cbegin(), 8, VT.Data);\n  return VT;\n}\n\n// Helper functions for returning a WasmEdge_Value by various values.\ntemplate <typename T>\ninline WasmEdge_Value genWasmEdge_Value(const T &Val) noexcept {\n  return WasmEdge_Value{\n      /* Value */ to_uint128_t(ValVariant(Val).unwrap()),\n      /* Type */ genWasmEdge_ValType(WasmEdge::ValTypeFromType<T>())};\n}\ninline WasmEdge_Value genWasmEdge_Value(const ValVariant &Val,\n                                        const ValType &T) noexcept {\n  return WasmEdge_Value{/* Value */ to_uint128_t(Val.unwrap()),\n                        /* Type */ genWasmEdge_ValType(T)};\n}\n\n// Helper function for converting a WasmEdge_Value array to a ValVariant\n// vector.\ninline std::pair<std::vector<ValVariant>, std::vector<ValType>>\ngenParamPair(const WasmEdge_Value *Val, const uint32_t Len) noexcept {\n  // The nullable value in reference types checking is handled in executor.\n  std::vector<ValVariant> VVec;\n  std::vector<ValType> TVec;\n  if (Val == nullptr) {\n    return {VVec, TVec};\n  }\n  VVec.resize(Len);\n  TVec.resize(Len);\n  for (uint32_t I = 0; I < Len; I++) {\n    TVec[I] = genValType(Val[I].Type);\n    switch (TVec[I].getCode()) {\n    case TypeCode::I32:\n      VVec[I] = ValVariant::wrap<uint32_t>(\n          to_WasmEdge_128_t<WasmEdge::uint128_t>(Val[I].Value));\n      break;\n    case TypeCode::I64:\n      VVec[I] = ValVariant::wrap<uint64_t>(\n          to_WasmEdge_128_t<WasmEdge::uint128_t>(Val[I].Value));\n      break;\n    case TypeCode::F32:\n      VVec[I] = ValVariant::wrap<float>(\n          to_WasmEdge_128_t<WasmEdge::uint128_t>(Val[I].Value));\n      break;\n    case TypeCode::F64:\n      VVec[I] = ValVariant::wrap<double>(\n          to_WasmEdge_128_t<WasmEdge::uint128_t>(Val[I].Value));\n      break;\n    case TypeCode::V128:\n      VVec[I] = ValVariant::wrap<WasmEdge::uint128_t>(\n          to_WasmEdge_128_t<WasmEdge::uint128_t>(Val[I].Value));\n      break;\n    case TypeCode::Ref:\n    case TypeCode::RefNull: {\n      VVec[I] = ValVariant::wrap<RefVariant>(\n          to_WasmEdge_128_t<WasmEdge::uint128_t>(Val[I].Value));\n      break;\n    }\n    default:\n      assumingUnreachable();\n    }\n  }\n  return {VVec, TVec};\n}\n\n// Helper function for making a Span to a uint8_t array.\ntemplate <typename T>\ninline constexpr Span<const T> genSpan(const T *Buf,\n                                       const uint32_t Len) noexcept {\n  if (Buf && Len > 0) {\n    return Span<const T>(Buf, Len);\n  }\n  return Span<const T>();\n}\n\n// Helper functions for converting WasmEdge_String to std::String.\ninline std::string_view genStrView(const WasmEdge_String S) noexcept {\n  return std::string_view(S.Buf, S.Length);\n}\n\n// Helper functions for converting a ValVariant vector to a WasmEdge_Value\n// array.\ninline constexpr void\nfillWasmEdge_ValueArr(Span<const std::pair<ValVariant, ValType>> Vec,\n                      WasmEdge_Value *Val, const uint32_t Len) noexcept {\n  if (Val == nullptr) {\n    return;\n  }\n  for (uint32_t I = 0; I < Len && I < Vec.size(); I++) {\n    Val[I] = genWasmEdge_Value(Vec[I].first, Vec[I].second);\n  }\n}\n\n// Helper template to run and return result.\nauto EmptyThen = [](auto &&) noexcept {};\ntemplate <typename T> inline bool isContext(T *Cxt) noexcept {\n  return (Cxt != nullptr);\n}\ntemplate <typename T, typename... Args>\ninline bool isContext(T *Cxt, Args *...Cxts) noexcept {\n  return isContext(Cxt) && isContext(Cxts...);\n}\ntemplate <typename T, typename U, typename... CxtT>\ninline WasmEdge_Result wrap(T &&Proc, U &&Then, CxtT *...Cxts) noexcept {\n  if (isContext(Cxts...)) {\n    if (auto Res = Proc()) {\n      Then(Res);\n      return genWasmEdge_Result(ErrCode::Value::Success);\n    } else {\n      return genWasmEdge_Result(Res.error());\n    }\n  } else {\n    return genWasmEdge_Result(ErrCode::Value::WrongVMWorkflow);\n  }\n}\n\n// Helper function of retrieving exported maps.\ntemplate <typename T>\ninline uint32_t fillMap(const std::map<std::string, T *, std::less<>> &Map,\n                        WasmEdge_String *Names, const uint32_t Len) noexcept {\n  uint32_t I = 0;\n  for (auto &&Pair : Map) {\n    if (I >= Len) {\n      break;\n    }\n    if (Names) {\n      Names[I] = WasmEdge_String{\n          /* Length */ static_cast<uint32_t>(Pair.first.length()),\n          /* Buf */ Pair.first.data()};\n    }\n    I++;\n  }\n  return static_cast<uint32_t>(Map.size());\n}\n\n// Helper functions of context conversions.\n#define CONVTO(SIMP, INST, NAME, QUANT)                                        \\\n  inline QUANT auto *to##SIMP##Cxt(QUANT INST *Cxt) noexcept {                 \\\n    return reinterpret_cast<QUANT WasmEdge_##NAME##Context *>(Cxt);            \\\n  }\nCONVTO(Stat, Statistics::Statistics, Statistics, )\nCONVTO(ASTMod, AST::Module, ASTModule, )\nCONVTO(FuncType, AST::FunctionType, FunctionType, )\nCONVTO(FuncType, AST::FunctionType, FunctionType, const)\nCONVTO(TabType, AST::TableType, TableType, )\nCONVTO(TabType, AST::TableType, TableType, const)\nCONVTO(MemType, AST::MemoryType, MemoryType, )\nCONVTO(MemType, AST::MemoryType, MemoryType, const)\nCONVTO(GlobType, AST::GlobalType, GlobalType, )\nCONVTO(GlobType, AST::GlobalType, GlobalType, const)\nCONVTO(ImpType, AST::ImportDesc, ImportType, const)\nCONVTO(ExpType, AST::ExportDesc, ExportType, const)\nCONVTO(Store, Runtime::StoreManager, Store, )\nCONVTO(Loader, Loader::Loader, Loader, )\nCONVTO(Validator, Validator::Validator, Validator, )\nCONVTO(Executor, Executor::Executor, Executor, )\nCONVTO(Mod, Runtime::Instance::ModuleInstance, ModuleInstance, )\nCONVTO(Mod, Runtime::Instance::ModuleInstance, ModuleInstance, const)\nCONVTO(Func, Runtime::Instance::FunctionInstance, FunctionInstance, )\nCONVTO(Func, Runtime::Instance::FunctionInstance, FunctionInstance, const)\nCONVTO(Tab, Runtime::Instance::TableInstance, TableInstance, )\nCONVTO(Mem, Runtime::Instance::MemoryInstance, MemoryInstance, )\nCONVTO(Glob, Runtime::Instance::GlobalInstance, GlobalInstance, )\nCONVTO(CallFrame, Runtime::CallingFrame, CallingFrame, const)\nCONVTO(Plugin, Plugin::Plugin, Plugin, const)\n#undef CONVTO\n\n#define CONVFROM(SIMP, INST, NAME, QUANT)                                      \\\n  inline QUANT auto *from##SIMP##Cxt(                                          \\\n      QUANT WasmEdge_##NAME##Context *Cxt) noexcept {                          \\\n    return reinterpret_cast<QUANT INST *>(Cxt);                                \\\n  }\nCONVFROM(Stat, Statistics::Statistics, Statistics, )\nCONVFROM(Stat, Statistics::Statistics, Statistics, const)\nCONVFROM(ASTMod, AST::Module, ASTModule, )\nCONVFROM(ASTMod, AST::Module, ASTModule, const)\nCONVFROM(FuncType, AST::FunctionType, FunctionType, )\nCONVFROM(FuncType, AST::FunctionType, FunctionType, const)\nCONVFROM(TabType, AST::TableType, TableType, )\nCONVFROM(TabType, AST::TableType, TableType, const)\nCONVFROM(MemType, AST::MemoryType, MemoryType, )\nCONVFROM(MemType, AST::MemoryType, MemoryType, const)\nCONVFROM(GlobType, AST::GlobalType, GlobalType, )\nCONVFROM(GlobType, AST::GlobalType, GlobalType, const)\nCONVFROM(ImpType, AST::ImportDesc, ImportType, const)\nCONVFROM(ExpType, AST::ExportDesc, ExportType, const)\nCONVFROM(Store, Runtime::StoreManager, Store, )\nCONVFROM(Store, Runtime::StoreManager, Store, const)\nCONVFROM(Loader, Loader::Loader, Loader, )\nCONVFROM(Validator, Validator::Validator, Validator, )\nCONVFROM(Executor, Executor::Executor, Executor, )\nCONVFROM(Mod, Runtime::Instance::ModuleInstance, ModuleInstance, )\nCONVFROM(Mod, Runtime::Instance::ModuleInstance, ModuleInstance, const)\nCONVFROM(Func, Runtime::Instance::FunctionInstance, FunctionInstance, )\nCONVFROM(Func, Runtime::Instance::FunctionInstance, FunctionInstance, const)\nCONVFROM(Tab, Runtime::Instance::TableInstance, TableInstance, )\nCONVFROM(Tab, Runtime::Instance::TableInstance, TableInstance, const)\nCONVFROM(Mem, Runtime::Instance::MemoryInstance, MemoryInstance, )\nCONVFROM(Mem, Runtime::Instance::MemoryInstance, MemoryInstance, const)\nCONVFROM(Glob, Runtime::Instance::GlobalInstance, GlobalInstance, )\nCONVFROM(Glob, Runtime::Instance::GlobalInstance, GlobalInstance, const)\nCONVFROM(CallFrame, Runtime::CallingFrame, CallingFrame, const)\nCONVFROM(Plugin, Plugin::Plugin, Plugin, const)\n#undef CONVFROM\n\n// C API Host function class\nclass CAPIHostFunc : public Runtime::HostFunctionBase {\npublic:\n  CAPIHostFunc(const AST::FunctionType *Type, WasmEdge_HostFunc_t FuncPtr,\n               void *ExtData, const uint64_t FuncCost = 0) noexcept\n      : Runtime::HostFunctionBase(FuncCost), Func(FuncPtr), Wrap(nullptr),\n        Binding(nullptr), Data(ExtData) {\n    DefType.getCompositeType().getFuncType() = *Type;\n  }\n  CAPIHostFunc(const AST::FunctionType *Type, WasmEdge_WrapFunc_t WrapPtr,\n               void *BindingPtr, void *ExtData,\n               const uint64_t FuncCost = 0) noexcept\n      : Runtime::HostFunctionBase(FuncCost), Func(nullptr), Wrap(WrapPtr),\n        Binding(BindingPtr), Data(ExtData) {\n    DefType.getCompositeType().getFuncType() = *Type;\n  }\n  ~CAPIHostFunc() noexcept override = default;\n\n  Expect<void> run(const Runtime::CallingFrame &CallFrame,\n                   Span<const ValVariant> Args,\n                   Span<ValVariant> Rets) override {\n    auto &FuncType = DefType.getCompositeType().getFuncType();\n    std::vector<WasmEdge_Value> Params(FuncType.getParamTypes().size()),\n        Returns(FuncType.getReturnTypes().size());\n    for (uint32_t I = 0; I < Args.size(); I++) {\n      Params[I] = genWasmEdge_Value(Args[I], FuncType.getParamTypes()[I]);\n    }\n    WasmEdge_Value *PPtr = Params.size() ? (&Params[0]) : nullptr;\n    WasmEdge_Value *RPtr = Returns.size() ? (&Returns[0]) : nullptr;\n    auto *CallFrameCxt = toCallFrameCxt(&CallFrame);\n    WasmEdge_Result Stat;\n    if (Func) {\n      Stat = Func(Data, CallFrameCxt, PPtr, RPtr);\n    } else {\n      Stat = Wrap(Binding, Data, CallFrameCxt, PPtr,\n                  static_cast<uint32_t>(Params.size()), RPtr,\n                  static_cast<uint32_t>(Returns.size()));\n    }\n    for (uint32_t I = 0; I < Rets.size(); I++) {\n      Rets[I] = to_WasmEdge_128_t<WasmEdge::uint128_t>(Returns[I].Value);\n    }\n    if (WasmEdge_ResultOK(Stat)) {\n      if (WasmEdge_ResultGetCode(Stat) == 0x01U) {\n        return Unexpect(ErrCode::Value::Terminated);\n      }\n    } else {\n      return Unexpect(\n          static_cast<ErrCategory>(WasmEdge_ResultGetCategory(Stat)),\n          WasmEdge_ResultGetCode(Stat));\n    }\n    return {};\n  }\n  void *getData() const noexcept { return Data; }\n\nprivate:\n  WasmEdge_HostFunc_t Func;\n  WasmEdge_WrapFunc_t Wrap;\n  void *Binding;\n  void *Data;\n};\n\n} // namespace\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n// >>>>>>>> WasmEdge version functions >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\nWASMEDGE_CAPI_EXPORT const char *WasmEdge_VersionGet(void) {\n  return WASMEDGE_VERSION;\n}\n\nWASMEDGE_CAPI_EXPORT uint32_t WasmEdge_VersionGetMajor(void) {\n  return WASMEDGE_VERSION_MAJOR;\n}\n\nWASMEDGE_CAPI_EXPORT uint32_t WasmEdge_VersionGetMinor(void) {\n  return WASMEDGE_VERSION_MINOR;\n}\n\nWASMEDGE_CAPI_EXPORT uint32_t WasmEdge_VersionGetPatch(void) {\n  return WASMEDGE_VERSION_PATCH;\n}\n\n// <<<<<<<< WasmEdge version functions <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n\n// >>>>>>>> WasmEdge logging functions >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\nWASMEDGE_CAPI_EXPORT void WasmEdge_LogSetErrorLevel(void) {\n  WasmEdge::Log::setErrorLoggingLevel();\n}\n\nWASMEDGE_CAPI_EXPORT void WasmEdge_LogSetDebugLevel(void) {\n  WasmEdge::Log::setDebugLoggingLevel();\n}\n\nWASMEDGE_CAPI_EXPORT void WasmEdge_LogOff(void) { WasmEdge::Log::setLogOff(); }\n\n// <<<<<<<< WasmEdge logging functions <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n\n// >>>>>>>> WasmEdge valtype functions >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\nWASMEDGE_CAPI_EXPORT WasmEdge_ValType WasmEdge_ValTypeGenI32(void) {\n  return genWasmEdge_ValType(ValType(TypeCode::I32));\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_ValType WasmEdge_ValTypeGenI64(void) {\n  return genWasmEdge_ValType(ValType(TypeCode::I64));\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_ValType WasmEdge_ValTypeGenF32(void) {\n  return genWasmEdge_ValType(ValType(TypeCode::F32));\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_ValType WasmEdge_ValTypeGenF64(void) {\n  return genWasmEdge_ValType(ValType(TypeCode::F64));\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_ValType WasmEdge_ValTypeGenV128(void) {\n  return genWasmEdge_ValType(ValType(TypeCode::V128));\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_ValType WasmEdge_ValTypeGenFuncRef(void) {\n  return genWasmEdge_ValType(ValType(TypeCode::FuncRef));\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_ValType WasmEdge_ValTypeGenExternRef(void) {\n  return genWasmEdge_ValType(ValType(TypeCode::ExternRef));\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsEqual(const WasmEdge_ValType ValType1,\n                        const WasmEdge_ValType ValType2) {\n  return genValType(ValType1) == genValType(ValType2);\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsI32(const WasmEdge_ValType ValType) {\n  return genValType(ValType).getCode() == WasmEdge::TypeCode::I32;\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsI64(const WasmEdge_ValType ValType) {\n  return genValType(ValType).getCode() == WasmEdge::TypeCode::I64;\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsF32(const WasmEdge_ValType ValType) {\n  return genValType(ValType).getCode() == WasmEdge::TypeCode::F32;\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsF64(const WasmEdge_ValType ValType) {\n  return genValType(ValType).getCode() == WasmEdge::TypeCode::F64;\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsV128(const WasmEdge_ValType ValType) {\n  return genValType(ValType).getCode() == WasmEdge::TypeCode::V128;\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsFuncRef(const WasmEdge_ValType ValType) {\n  return genValType(ValType).isFuncRefType();\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsExternRef(const WasmEdge_ValType ValType) {\n  return genValType(ValType).isExternRefType();\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsRef(const WasmEdge_ValType ValType) {\n  return genValType(ValType).isRefType();\n}\n\nWASMEDGE_CAPI_EXPORT bool\nWasmEdge_ValTypeIsRefNull(const WasmEdge_ValType ValType) {\n  return genValType(ValType).isNullableRefType();\n}\n\n// <<<<<<<< WasmEdge valtype functions <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n\n// >>>>>>>> WasmEdge value functions >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n\nWASMEDGE_CAPI_EXPORT WasmEdge_Value WasmEdge_ValueGenI32(const int32_t Val) {\n  return genWasmEdge_Value(Val);\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_Value WasmEdge_ValueGenI64(const int64_t Val) {\n  return genWasmEdge_Value(Val);\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_Value WasmEdge_ValueGenF32(const float Val) {\n  return genWasmEdge_Value(Val);\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_Value WasmEdge_ValueGenF64(const double Val) {\n  return genWasmEdge_Value(Val);\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_Value\nWasmEdge_ValueGenV128(const ::int128_t Val) {\n  return genWasmEdge_Value(to_WasmEdge_128_t<WasmEdge::int128_t>(Val));\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_Value\nWasmEdge_ValueGenFuncRef(const WasmEdge_FunctionInstanceContext *Cxt) {\n  return genWasmEdge_Value(WasmEdge::RefVariant(fromFuncCxt(Cxt)),\n                           TypeCode::FuncRef);\n}\n\nWASMEDGE_CAPI_EXPORT WasmEdge_Value WasmEdge_ValueGenExternRef(void *Ref) {\n  return genWasmEdge_Value(WasmEdge::RefVariant(Ref), TypeCode::ExternRef);\n}\n\nWASMEDGE_CAPI_EXPORT int32_t WasmEdge_ValueGetI32(const WasmEdge_Value Val) {\n  return WasmEdge::ValVariant::wrap<int32_t>(\n             to_WasmEdge_128_t<WasmEdge::uint128_t>(Val.Value))\n      .get<int32_t>();\n}\n\nWASMEDGE_CAPI_EXPORT int64_t WasmEdge_ValueGetI64(const WasmEdge_Value Val) {\n  return WasmEdge::ValVariant::wrap<int64_t>(\n             to_WasmEdge_128_t<WasmEdge::uint128_t>(Val.Value))\n      .get<int64_t>();\n}\n\nWASMEDGE_CAPI_EXPORT float WasmEdge_ValueGetF32(const WasmEdge_Value Val) {\n  return WasmEdge::ValVariant::wrap<float>(\n             to_WasmEdge_128_t<WasmEdge::uint128_t>(Val.Value))\n      .get<float>();\n}\n\nWASMEDGE_CAPI_EXPORT double WasmEdge_ValueGetF64(const WasmEdge_Value Val) {\n  return WasmEdge::ValVariant::wrap<double>(\n             to_WasmEdge_128_t<WasmEdge::uint128_t>(Val.Value))\n      .get<double>();\n}\n\nWASMEDGE_CAPI_EXPORT ::int128_t\nWasmEdge_ValueGetV128(const WasmEdge_Value Val) {\n  return to_int128_t(WasmEdge::ValVariant::wrap<WasmEdge::int128_t>(\n...\n", "needle_token_start": 8876, "needle_token_end": 8917, "code_context_ntokens": 16178, "output": ["```cpp\nWasmEdge_CompilerContext(const WasmEdge::Configure &Conf) noexcept\n      : Compiler(Conf), CodeGen(Conf), Load(Conf), Valid(Conf) {}\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "proxy", "language": "cpp", "path": "lib/executor/engine/proxy.cpp", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this function is to serve as an intermediary that invokes another function, handling the result and potentially converting the output format, while also managing errors through a fault emission mechanism.\n2. **Input**: The inputs are various arguments required by the target function it intends to call. These arguments are passed through dynamically.\n3. **Output**: The output varies based on the return type of the function being proxied. It could be a direct value, a reference, or a data structure, potentially transformed for compatibility with specific compiler requirements.\n4. **Procedure**: The procedure involves:\n   - Invoking a member function of a class using the current execution context and stack.\n   - Checking if the function call was successful.\n   - Emitting a fault if the function call resulted in an error.\n   - Transforming the output data to a specific format if required, before returning it.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/executor/instantiate/import.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include \"common/errinfo.h\"\n#include \"common/log.h\"\n\n#include <cstdint>\n#include <string_view>\n#include <utility>\n\nnamespace WasmEdge {\nnamespace Executor {\n\nnamespace {\ntemplate <typename... Args>\nauto logMatchError(std::string_view ModName, std::string_view ExtName,\n                   ExternalType ExtType, Args &&...Values) {\n  spdlog::error(ErrCode::Value::IncompatibleImportType);\n  spdlog::error(ErrInfo::InfoMismatch(std::forward<Args>(Values)...));\n  spdlog::error(ErrInfo::InfoLinking(ModName, ExtName, ExtType));\n  spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Desc_Import));\n  return Unexpect(ErrCode::Value::IncompatibleImportType);\n}\n\nauto logUnknownError(std::string_view ModName, std::string_view ExtName,\n                     ExternalType ExtType) {\n  spdlog::error(ErrCode::Value::UnknownImport);\n  spdlog::error(ErrInfo::InfoLinking(ModName, ExtName, ExtType));\n  spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Desc_Import));\n  return Unexpect(ErrCode::Value::UnknownImport);\n}\n\nbool matchLimit(const AST::Limit &Exp, const AST::Limit &Got) {\n  if (Exp.isShared() != Got.isShared()) {\n    return false;\n  }\n  if ((Got.getMin() < Exp.getMin()) || (Exp.hasMax() && !Got.hasMax())) {\n    return false;\n  }\n  if (Exp.hasMax() && Got.hasMax() && Got.getMax() > Exp.getMax()) {\n    return false;\n  }\n  return true;\n}\n\nExpect<void>\ncheckImportMatched(std::string_view ModName, std::string_view ExtName,\n                   const ExternalType ExtType,\n                   const Runtime::Instance::ModuleInstance &ModInst) {\n  switch (ExtType) {\n  case ExternalType::Function:\n    if (auto Res = ModInst.findFuncExports(ExtName); likely(Res != nullptr)) {\n      return {};\n    }\n    break;\n  case ExternalType::Table:\n    if (auto Res = ModInst.findTableExports(ExtName); likely(Res != nullptr)) {\n      return {};\n    }\n    break;\n  case ExternalType::Memory:\n    if (auto Res = ModInst.findMemoryExports(ExtName); likely(Res != nullptr)) {\n      return {};\n    }\n    break;\n  case ExternalType::Global:\n    if (auto Res = ModInst.findGlobalExports(ExtName); likely(Res != nullptr)) {\n      return {};\n    }\n    break;\n  default:\n    return logUnknownError(ModName, ExtName, ExtType);\n  }\n\n  // Check is error external type or unknown imports.\n  if (ModInst.findFuncExports(ExtName)) {\n    return logMatchError(ModName, ExtName, ExtType, ExtType,\n                         ExternalType::Function);\n  }\n  if (ModInst.findTableExports(ExtName)) {\n    return logMatchError(ModName, ExtName, ExtType, ExtType,\n                         ExternalType::Table);\n  }\n  if (ModInst.findMemoryExports(ExtName)) {\n    return logMatchError(ModName, ExtName, ExtType, ExtType,\n                         ExternalType::Memory);\n  }\n  if (ModInst.findGlobalExports(ExtName)) {\n    return logMatchError(ModName, ExtName, ExtType, ExtType,\n                         ExternalType::Global);\n  }\n\n  return logUnknownError(ModName, ExtName, ExtType);\n}\n} // namespace\n\n// Instantiate imports. See \"include/executor/executor.h\".\nExpect<void> Executor::instantiate(Runtime::StoreManager &StoreMgr,\n                                   Runtime::Instance::ModuleInstance &ModInst,\n                                   const AST::ImportSection &ImportSec) {\n  // Iterate and instantiate import descriptions.\n  for (const auto &ImpDesc : ImportSec.getContent()) {\n    // Get data from import description and find import module.\n    auto ExtType = ImpDesc.getExternalType();\n    auto ModName = ImpDesc.getModuleName();\n    auto ExtName = ImpDesc.getExternalName();\n    const auto *ImpModInst = StoreMgr.findModule(ModName);\n    if (unlikely(ImpModInst == nullptr)) {\n      auto Res = logUnknownError(ModName, ExtName, ExtType);\n      if (ModName == \"wasi_snapshot_preview1\") {\n        spdlog::error(\"    This is a WASI related import. Please ensure that \"\n                      \"you've turned on the WASI configuration.\");\n      } else if (ModName == \"wasi_nn\") {\n        spdlog::error(\"    This is a WASI-NN related import. Please ensure \"\n                      \"that you've turned on the WASI-NN configuration and \"\n                      \"installed the WASI-NN plug-in.\");\n      } else if (ModName == \"wasi_crypto_common\" ||\n                 ModName == \"wasi_crypto_asymmetric_common\" ||\n                 ModName == \"wasi_crypto_kx\" ||\n                 ModName == \"wasi_crypto_signatures\" ||\n                 ModName == \"wasi_crypto_symmetric\") {\n        spdlog::error(\"    This is a WASI-Crypto related import. Please ensure \"\n                      \"that you've turned on the WASI-Crypto configuration and \"\n                      \"installed the WASI-Crypto plug-in.\");\n      } else if (ModName == \"env\") {\n        spdlog::error(\n            \"    This may be the import of host environment like JavaScript or \"\n            \"Golang. Please check that you've registered the necessary host \"\n            \"modules from the host programming language.\");\n      }\n      return Res;\n    }\n    if (auto Res = checkImportMatched(ModName, ExtName, ExtType, *ImpModInst);\n        unlikely(!Res)) {\n      return Unexpect(Res);\n    }\n\n    // Add the imports into module instance.\n    switch (ExtType) {\n    case ExternalType::Function: {\n      // Get function type index. External type checked in validation.\n      uint32_t TypeIdx = ImpDesc.getExternalFuncTypeIdx();\n      // Import matching.\n      auto *ImpInst = ImpModInst->findFuncExports(ExtName);\n      const auto &ExpDefType = **ModInst.getType(TypeIdx);\n      // External function type should match the import function type in\n      // description.\n\n      if (!AST::TypeMatcher::matchType(\n              ModInst.getTypeList(), *ExpDefType.getTypeIndex(),\n              ImpModInst->getTypeList(), ImpInst->getTypeIndex())) {\n        bool IsMatchV2 = false;\n        const auto &ExpFuncType = ExpDefType.getCompositeType().getFuncType();\n        const auto &ImpFuncType = ImpInst->getFuncType();\n        if (ModName == \"wasi_snapshot_preview1\") {\n          /*\n           * The following functions should provide V1 and V2.\n             \"sock_open_v2\",\n             \"sock_bind_v2\",\n             \"sock_connect_v2\",\n             \"sock_listen_v2\",\n             \"sock_accept_v2\",\n             \"sock_recv_v2\",\n             \"sock_recv_from_v2\",\n             \"sock_send_v2\",\n             \"sock_send_to_v2\",\n             \"sock_getlocaladdr_v2\",\n             \"sock_getpeeraddr_v2\"\n             */\n          std::vector<std::string> CompatibleWASISocketAPI = {\n              \"sock_open\",         \"sock_bind\",       \"sock_connect\",\n              \"sock_listen\",       \"sock_accept\",     \"sock_recv\",\n              \"sock_recv_from\",    \"sock_send\",       \"sock_send_to\",\n              \"sock_getlocaladdr\", \"sock_getpeeraddr\"};\n          for (auto Iter = CompatibleWASISocketAPI.begin();\n               Iter != CompatibleWASISocketAPI.end(); Iter++) {\n            if (ExtName == *Iter) {\n              auto *ImpInstV2 = ImpModInst->findFuncExports(*Iter + \"_v2\");\n              if (!AST::TypeMatcher::matchType(\n                      ModInst.getTypeList(), *ExpDefType.getTypeIndex(),\n                      ImpModInst->getTypeList(), ImpInst->getTypeIndex())) {\n                // Try to match the new version\n                ImpInst = ImpInstV2;\n                IsMatchV2 = true;\n                break;\n              }\n            }\n          }\n        }\n        if (!IsMatchV2) {\n          return logMatchError(\n              ModName, ExtName, ExtType, ExpFuncType.getParamTypes(),\n              ExpFuncType.getReturnTypes(), ImpFuncType.getParamTypes(),\n              ImpFuncType.getReturnTypes());\n        }\n      }\n      // Set the matched function address to module instance.\n      ModInst.importFunction(ImpInst);\n      break;\n    }\n    case ExternalType::Table: {\n      // Get table type. External type checked in validation.\n      const auto &TabType = ImpDesc.getExternalTableType();\n      const auto &TabLim = TabType.getLimit();\n      // Import matching. External table type should match the one in import\n      // description.\n      auto *ImpInst = ImpModInst->findTableExports(ExtName);\n      const auto &ImpType = ImpInst->getTableType();\n      const auto &ImpLim = ImpType.getLimit();\n      // External table reference type should match the import table reference\n      // type in description, and vice versa.\n      if (!AST::TypeMatcher::matchType(\n              ModInst.getTypeList(), TabType.getRefType(),\n              ImpModInst->getTypeList(), ImpType.getRefType()) ||\n          !AST::TypeMatcher::matchType(\n              ImpModInst->getTypeList(), ImpType.getRefType(),\n              ModInst.getTypeList(), TabType.getRefType()) ||\n          !matchLimit(TabLim, ImpLim)) {\n        return logMatchError(ModName, ExtName, ExtType, TabType.getRefType(),\n                             TabLim.hasMax(), TabLim.getMin(), TabLim.getMax(),\n                             ImpType.getRefType(), ImpLim.hasMax(),\n                             ImpLim.getMin(), ImpLim.getMax());\n      }\n      // Set the matched table address to module instance.\n      ModInst.importTable(ImpInst);\n      break;\n    }\n    case ExternalType::Memory: {\n      // Get memory type. External type checked in validation.\n      const auto &MemType = ImpDesc.getExternalMemoryType();\n      const auto &MemLim = MemType.getLimit();\n      // Import matching. External memory type should match the one in import\n      // description.\n      auto *ImpInst = ImpModInst->findMemoryExports(ExtName);\n      const auto &ImpLim = ImpInst->getMemoryType().getLimit();\n      if (!matchLimit(MemLim, ImpLim)) {\n        return logMatchError(ModName, ExtName, ExtType, MemLim.hasMax(),\n                             MemLim.getMin(), MemLim.getMax(), ImpLim.hasMax(),\n                             ImpLim.getMin(), ImpLim.getMax());\n      }\n      // Set the matched memory address to module instance.\n      ModInst.importMemory(ImpInst);\n      break;\n    }\n    case ExternalType::Global: {\n      // Get global type. External type checked in validation.\n      const auto &GlobType = ImpDesc.getExternalGlobalType();\n      // Import matching. External global type should match the one in\n      // import description.\n      auto *ImpInst = ImpModInst->findGlobalExports(ExtName);\n      const auto &ImpType = ImpInst->getGlobalType();\n      bool IsMatch = false;\n      if (ImpType.getValMut() == GlobType.getValMut()) {\n        // For both const or both var: external global value type should match\n        // the import global value type in description.\n        IsMatch = AST::TypeMatcher::matchType(\n            ModInst.getTypeList(), GlobType.getValType(),\n            ImpModInst->getTypeList(), ImpType.getValType());\n        if (ImpType.getValMut() == ValMut::Var) {\n          // If both var: import global value type in description should also\n          // match the external global value type.\n          IsMatch &= AST::TypeMatcher::matchType(\n              ImpModInst->getTypeList(), ImpType.getValType(),\n              ModInst.getTypeList(), GlobType.getValType());\n        }\n      }\n      if (!IsMatch) {\n...\n// Path: lib/executor/instantiate/function.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <cstdint>\n#include <utility>\n\nnamespace WasmEdge {\nnamespace Executor {\n\n// Instantiate function instance. See \"include/executor/executor.h\".\nExpect<void> Executor::instantiate(Runtime::Instance::ModuleInstance &ModInst,\n                                   const AST::FunctionSection &FuncSec,\n                                   const AST::CodeSection &CodeSec) {\n\n  // Get the function type indices.\n  auto TypeIdxs = FuncSec.getContent();\n  auto CodeSegs = CodeSec.getContent();\n\n  if (CodeSegs.size() == 0) {\n    return {};\n  }\n  // The module will always choose the `for` loop in `else` case under\n  // interpreter mode. Instead, if we do branch in the `for` loop which might\n  // cause meaningless branch misses. Therefore we should check the first item\n  // and dispatch it into different cases to reduce branch misses.\n  if (CodeSegs[0].getSymbol() != false) {\n    for (uint32_t I = 0; I < CodeSegs.size(); ++I) {\n      auto Symbol = CodeSegs[I].getSymbol();\n      ModInst.addFunc(\n          TypeIdxs[I],\n          (*ModInst.getType(TypeIdxs[I]))->getCompositeType().getFuncType(),\n          std::move(Symbol));\n    }\n  } else {\n    // Iterate through the code segments to instantiate function instances.\n    for (uint32_t I = 0; I < CodeSegs.size(); ++I) {\n      // Create and add the function instance into the module instance.\n      ModInst.addFunc(\n          TypeIdxs[I],\n          (*ModInst.getType(TypeIdxs[I]))->getCompositeType().getFuncType(),\n          CodeSegs[I].getLocals(), CodeSegs[I].getExpr().getInstrs());\n    }\n  }\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/instantiate/table.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <cstdint>\n\nnamespace WasmEdge {\nnamespace Executor {\n\n// Instantiate table instance. See \"include/executor/executor.h\".\nExpect<void> Executor::instantiate(Runtime::StackManager &StackMgr,\n                                   Runtime::Instance::ModuleInstance &ModInst,\n                                   const AST::TableSection &TabSec) {\n  // A frame with temp. module is pushed into the stack in caller.\n\n  // Iterate through the table segments to instantiate and initialize table\n  // instances.\n  for (const auto &TabSeg : TabSec.getContent()) {\n    if (TabSeg.getExpr().getInstrs().size() > 0) {\n      // Run initialize expression.\n      if (auto Res = runExpression(StackMgr, TabSeg.getExpr().getInstrs());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Expression));\n        return Unexpect(Res);\n      }\n      // Pop result from stack.\n      RefVariant InitTabValue = StackMgr.pop().get<RefVariant>();\n      // Create and add the table instance into the module instance.\n      ModInst.addTable(TabSeg.getTableType(), InitTabValue);\n    } else {\n      // No init expression case. Use the null reference to initialize.\n      ModInst.addTable(TabSeg.getTableType());\n    }\n  }\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/instantiate/memory.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <cstdint>\n\nnamespace WasmEdge {\nnamespace Executor {\n\n// Instantiate memory instance. See \"include/executor/executor.h\".\nExpect<void> Executor::instantiate(Runtime::Instance::ModuleInstance &ModInst,\n                                   const AST::MemorySection &MemSec) {\n  // Prepare pointers vector for compiled functions.\n  ModInst.MemoryPtrs.resize(ModInst.getMemoryNum() +\n                            MemSec.getContent().size());\n\n  // Iterate through the memory types to instantiate memory instances.\n  for (const auto &MemType : MemSec.getContent()) {\n    // Create and add the memory instance into the module instance.\n    ModInst.addMemory(MemType, Conf.getRuntimeConfigure().getMaxMemoryPage());\n  }\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/instantiate/global.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <cstdint>\n\nnamespace WasmEdge {\nnamespace Executor {\n\n// Instantiate global instance. See \"include/executor/executor.h\".\nExpect<void> Executor::instantiate(Runtime::StackManager &StackMgr,\n                                   Runtime::Instance::ModuleInstance &ModInst,\n                                   const AST::GlobalSection &GlobSec) {\n  // A frame with temp. module is pushed into the stack in caller.\n\n  // Prepare pointers for compiled functions.\n  ModInst.GlobalPtrs.resize(ModInst.getGlobalNum() +\n                            GlobSec.getContent().size());\n\n  // Set the global pointers of imported globals.\n  for (uint32_t I = 0; I < ModInst.getGlobalNum(); ++I) {\n    ModInst.GlobalPtrs[I] = &((*ModInst.getGlobal(I))->getValue());\n  }\n\n  // Iterate through the global segments to instantiate and initialize global\n  // instances.\n  for (const auto &GlobSeg : GlobSec.getContent()) {\n    // Run initialize expression.\n    if (auto Res = runExpression(StackMgr, GlobSeg.getExpr().getInstrs());\n        !Res) {\n      spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Expression));\n      return Unexpect(Res);\n    }\n\n    // Pop result from stack.\n    ValVariant InitValue = StackMgr.pop();\n\n    // Create and add the global instance into the module instance.\n    ModInst.addGlobal(GlobSeg.getGlobalType(), InitValue);\n    const auto Index = ModInst.getGlobalNum() - 1;\n    Runtime::Instance::GlobalInstance *GlobInst = *ModInst.getGlobal(Index);\n\n    // Set the global pointers of instantiated globals.\n    ModInst.GlobalPtrs[Index] = &(GlobInst->getValue());\n  }\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/instantiate/export.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <cstdint>\n#include <string_view>\n\nnamespace WasmEdge {\nnamespace Executor {\n\n// Instantiate exports. See \"include/executor/executor.h\".\nExpect<void> Executor::instantiate(Runtime::Instance::ModuleInstance &ModInst,\n                                   const AST::ExportSection &ExportSec) {\n  // Iterate through the export descriptions and instantiate the exports.\n  for (const auto &ExpDesc : ExportSec.getContent()) {\n    // Get data from the export description.\n    const auto ExtType = ExpDesc.getExternalType();\n    std::string_view ExtName = ExpDesc.getExternalName();\n    const uint32_t ExtIdx = ExpDesc.getExternalIndex();\n\n    // Export the instance with the name.\n    switch (ExtType) {\n    case ExternalType::Function:\n      ModInst.exportFunction(ExtName, ExtIdx);\n      break;\n    case ExternalType::Global:\n      ModInst.exportGlobal(ExtName, ExtIdx);\n      break;\n    case ExternalType::Memory:\n      ModInst.exportMemory(ExtName, ExtIdx);\n      break;\n    case ExternalType::Table:\n      ModInst.exportTable(ExtName, ExtIdx);\n      break;\n    default:\n      break;\n    }\n  }\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/instantiate/elem.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include \"common/errinfo.h\"\n#include \"common/log.h\"\n#include <cstdint>\n#include <vector>\n\nnamespace WasmEdge {\nnamespace Executor {\n\n// Instantiate element instance. See \"include/executor/executor.h\".\nExpect<void> Executor::instantiate(Runtime::StackManager &StackMgr,\n                                   Runtime::Instance::ModuleInstance &ModInst,\n                                   const AST::ElementSection &ElemSec) {\n  // A frame with the current module has been pushed into the stack outside.\n\n  // Iterate through the element segments to instantiate element instances.\n  for (const auto &ElemSeg : ElemSec.getContent()) {\n    std::vector<RefVariant> InitVals;\n    for (const auto &Expr : ElemSeg.getInitExprs()) {\n      // Run init expr of every elements and get the result reference.\n      if (auto Res = runExpression(StackMgr, Expr.getInstrs()); !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Expression));\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Element));\n        return Unexpect(Res);\n      }\n      // Pop result from stack.\n      InitVals.push_back(StackMgr.pop().get<RefVariant>());\n    }\n\n    uint32_t Offset = 0;\n    if (ElemSeg.getMode() == AST::ElementSegment::ElemMode::Active) {\n      // Run initialize expression.\n      if (auto Res = runExpression(StackMgr, ElemSeg.getExpr().getInstrs());\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Expression));\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Element));\n        return Unexpect(Res);\n      }\n      Offset = StackMgr.pop().get<uint32_t>();\n\n      // Check boundary unless ReferenceTypes or BulkMemoryOperations proposal\n      // enabled.\n      if (!Conf.hasProposal(Proposal::ReferenceTypes) &&\n          !Conf.hasProposal(Proposal::BulkMemoryOperations)) {\n        // Table index should be 0. Checked in validation phase.\n        auto *TabInst = getTabInstByIdx(StackMgr, ElemSeg.getIdx());\n        // Check elements fits.\n        assuming(TabInst);\n        if (!TabInst->checkAccessBound(\n                Offset, static_cast<uint32_t>(InitVals.size()))) {\n          spdlog::error(ErrCode::Value::ElemSegDoesNotFit);\n          spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Element));\n          return Unexpect(ErrCode::Value::ElemSegDoesNotFit);\n        }\n      }\n    }\n\n    // Create and add the element instance into the module instance.\n    ModInst.addElem(Offset, ElemSeg.getRefType(), InitVals);\n  }\n  return {};\n}\n\n// Initialize table with Element section. See \"include/executor/executor.h\".\nExpect<void> Executor::initTable(Runtime::StackManager &StackMgr,\n                                 const AST::ElementSection &ElemSec) {\n  // Initialize tables.\n  uint32_t Idx = 0;\n  for (const auto &ElemSeg : ElemSec.getContent()) {\n    auto *ElemInst = getElemInstByIdx(StackMgr, Idx);\n    assuming(ElemInst);\n    if (ElemSeg.getMode() == AST::ElementSegment::ElemMode::Active) {\n      // Table index is checked in validation phase.\n      auto *TabInst = getTabInstByIdx(StackMgr, ElemSeg.getIdx());\n      assuming(TabInst);\n      const uint32_t Off = ElemInst->getOffset();\n\n      // Replace table[Off : Off + n] with elem[0 : n].\n      if (auto Res = TabInst->setRefs(\n              ElemInst->getRefs(), Off, 0,\n              static_cast<uint32_t>(ElemInst->getRefs().size()));\n          !Res) {\n        spdlog::error(ErrInfo::InfoAST(ASTNodeAttr::Seg_Element));\n        return Unexpect(Res);\n      }\n\n      // Drop the element instance.\n      ElemInst->clear();\n\n      // Operation above is equal to the following instruction sequence:\n      //   expr(init) -> i32.const off\n      //   i32.const 0\n      //   i32.const n\n      //   table.init idx\n      //   elem.drop idx\n    } else if (ElemSeg.getMode() ==\n               AST::ElementSegment::ElemMode::Declarative) {\n      // Drop the element instance.\n      ElemInst->clear();\n    }\n    Idx++;\n  }\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/variableInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <cstdint>\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void> Executor::runLocalGetOp(Runtime::StackManager &StackMgr,\n                                     uint32_t StackOffset) const noexcept {\n  StackMgr.push(StackMgr.getTopN(StackOffset));\n  return {};\n}\n\nExpect<void> Executor::runLocalSetOp(Runtime::StackManager &StackMgr,\n                                     uint32_t StackOffset) const noexcept {\n  StackMgr.getTopN(StackOffset - 1) = StackMgr.pop();\n  return {};\n}\n\nExpect<void> Executor::runLocalTeeOp(Runtime::StackManager &StackMgr,\n                                     uint32_t StackOffset) const noexcept {\n  const ValVariant &Val = StackMgr.getTop();\n  StackMgr.getTopN(StackOffset) = Val;\n  return {};\n}\n\nExpect<void> Executor::runGlobalGetOp(Runtime::StackManager &StackMgr,\n                                      uint32_t Idx) const noexcept {\n  auto *GlobInst = getGlobInstByIdx(StackMgr, Idx);\n  assuming(GlobInst);\n  StackMgr.push(GlobInst->getValue());\n  return {};\n}\n\nExpect<void> Executor::runGlobalSetOp(Runtime::StackManager &StackMgr,\n                                      uint32_t Idx) const noexcept {\n  auto *GlobInst = getGlobInstByIdx(StackMgr, Idx);\n  assuming(GlobInst);\n  GlobInst->setValue(StackMgr.pop());\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/controlInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <cstdint>\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void> Executor::runIfElseOp(Runtime::StackManager &StackMgr,\n                                   const AST::Instruction &Instr,\n                                   AST::InstrView::iterator &PC) noexcept {\n  // Get condition.\n  uint32_t Cond = StackMgr.pop().get<uint32_t>();\n\n  // If non-zero, run if-statement; else, run else-statement.\n  if (Cond == 0) {\n    if (Instr.getJumpElse() == Instr.getJumpEnd()) {\n      // No else-statement case. Jump to right before End instruction.\n      PC += (Instr.getJumpEnd() - 1);\n    } else {\n      if (Stat) {\n        Stat->incInstrCount();\n        if (unlikely(!Stat->addInstrCost(OpCode::Else))) {\n          return Unexpect(ErrCode::Value::CostLimitExceeded);\n        }\n      }\n      // Have else-statement case. Jump to Else instruction to continue.\n      PC += Instr.getJumpElse();\n    }\n  }\n  return {};\n}\n\nExpect<void> Executor::runBrOp(Runtime::StackManager &StackMgr,\n                               const AST::Instruction &Instr,\n                               AST::InstrView::iterator &PC) noexcept {\n  return branchToLabel(StackMgr, Instr.getJump().StackEraseBegin,\n                       Instr.getJump().StackEraseEnd, Instr.getJump().PCOffset,\n                       PC);\n}\n\nExpect<void> Executor::runBrIfOp(Runtime::StackManager &StackMgr,\n                                 const AST::Instruction &Instr,\n                                 AST::InstrView::iterator &PC) noexcept {\n  if (StackMgr.pop().get<uint32_t>() != 0) {\n    return runBrOp(StackMgr, Instr, PC);\n  }\n  return {};\n}\n\nExpect<void> Executor::runBrOnNullOp(Runtime::StackManager &StackMgr,\n                                     const AST::Instruction &Instr,\n                                     AST::InstrView::iterator &PC) noexcept {\n  if (StackMgr.getTop().get<RefVariant>().isNull()) {\n    StackMgr.pop();\n    return runBrOp(StackMgr, Instr, PC);\n  }\n  return {};\n}\n\nExpect<void> Executor::runBrOnNonNullOp(Runtime::StackManager &StackMgr,\n                                        const AST::Instruction &Instr,\n                                        AST::InstrView::iterator &PC) noexcept {\n  if (!StackMgr.getTop().get<RefVariant>().isNull()) {\n    return runBrOp(StackMgr, Instr, PC);\n  }\n  StackMgr.pop();\n  return {};\n}\n\nExpect<void> Executor::runBrTableOp(Runtime::StackManager &StackMgr,\n                                    const AST::Instruction &Instr,\n                                    AST::InstrView::iterator &PC) noexcept {\n  // Get value on top of stack.\n  uint32_t Value = StackMgr.pop().get<uint32_t>();\n\n  // Do branch.\n  auto LabelTable = Instr.getLabelList();\n  const auto LabelTableSize = static_cast<uint32_t>(LabelTable.size() - 1);\n  if (Value < LabelTableSize) {\n    return branchToLabel(StackMgr, LabelTable[Value].StackEraseBegin,\n                         LabelTable[Value].StackEraseEnd,\n                         LabelTable[Value].PCOffset, PC);\n  }\n  return branchToLabel(StackMgr, LabelTable[LabelTableSize].StackEraseBegin,\n                       LabelTable[LabelTableSize].StackEraseEnd,\n                       LabelTable[LabelTableSize].PCOffset, PC);\n}\n\nExpect<void> Executor::runBrOnCastOp(Runtime::StackManager &StackMgr,\n                                     const AST::Instruction &Instr,\n                                     AST::InstrView::iterator &PC,\n                                     bool IsReverse) noexcept {\n  // Get value on top of stack.\n  const auto *ModInst = StackMgr.getModule();\n  const auto &Val = StackMgr.getTop().get<RefVariant>();\n  const auto &VT = Val.getType();\n  Span<const AST::SubType *const> GotTypeList = ModInst->getTypeList();\n  if (!VT.isAbsHeapType()) {\n    auto *Inst = Val.getPtr<Runtime::Instance::CompositeBase>();\n    // Reference must not be nullptr here because the null references are typed\n    // with the least abstract heap type.\n    if (Inst->getModule()) {\n      GotTypeList = Inst->getModule()->getTypeList();\n    }\n  }\n\n  if (AST::TypeMatcher::matchType(ModInst->getTypeList(),\n                                  Instr.getBrCast().RType2, GotTypeList,\n                                  VT) != IsReverse) {\n    return branchToLabel(StackMgr, Instr.getBrCast().Jump.StackEraseBegin,\n                         Instr.getBrCast().Jump.StackEraseEnd,\n                         Instr.getBrCast().Jump.PCOffset, PC);\n  }\n  return {};\n}\n\nExpect<void> Executor::runReturnOp(Runtime::StackManager &StackMgr,\n                                   AST::InstrView::iterator &PC) noexcept {\n  // Check stop token\n  if (unlikely(StopToken.exchange(0, std::memory_order_relaxed))) {\n    spdlog::error(ErrCode::Value::Interrupted);\n    return Unexpect(ErrCode::Value::Interrupted);\n  }\n  PC = StackMgr.popFrame();\n  return {};\n}\n\nExpect<void> Executor::runCallOp(Runtime::StackManager &StackMgr,\n                                 const AST::Instruction &Instr,\n                                 AST::InstrView::iterator &PC,\n                                 bool IsTailCall) noexcept {\n  // Get Function address.\n  const auto *FuncInst = getFuncInstByIdx(StackMgr, Instr.getTargetIndex());\n  if (auto Res = enterFunction(StackMgr, *FuncInst, PC + 1, IsTailCall); !Res) {\n    return Unexpect(Res);\n  } else {\n    PC = (*Res) - 1;\n  }\n  return {};\n}\n\nExpect<void> Executor::runCallRefOp(Runtime::StackManager &StackMgr,\n                                    const AST::Instruction &Instr,\n                                    AST::InstrView::iterator &PC,\n                                    bool IsTailCall) noexcept {\n\n  const auto Ref = StackMgr.pop().get<RefVariant>();\n  if (Ref.isNull()) {\n    spdlog::error(ErrCode::Value::AccessNullFunc);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullFunc);\n  }\n\n  // Get Function address.\n  const auto *FuncInst = retrieveFuncRef(Ref);\n  if (auto Res = enterFunction(StackMgr, *FuncInst, PC + 1, IsTailCall); !Res) {\n    return Unexpect(Res);\n  } else {\n    PC = (*Res) - 1;\n  }\n  return {};\n}\n\nExpect<void> Executor::runCallIndirectOp(Runtime::StackManager &StackMgr,\n                                         const AST::Instruction &Instr,\n                                         AST::InstrView::iterator &PC,\n                                         bool IsTailCall) noexcept {\n  // Get Table Instance\n  const auto *TabInst = getTabInstByIdx(StackMgr, Instr.getSourceIndex());\n\n  // Get function type at index x.\n  const auto *ModInst = StackMgr.getModule();\n  const auto &ExpDefType = **ModInst->getType(Instr.getTargetIndex());\n\n  // Pop the value i32.const i from the Stack.\n  uint32_t Idx = StackMgr.pop().get<uint32_t>();\n\n  // If idx not small than tab.elem, trap.\n  if (Idx >= TabInst->getSize()) {\n    spdlog::error(ErrCode::Value::UndefinedElement);\n    spdlog::error(ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset(),\n                                           {Idx},\n                                           {ValTypeFromType<uint32_t>()}));\n    return Unexpect(ErrCode::Value::UndefinedElement);\n  }\n\n  // Get function address. The bound is guaranteed.\n  RefVariant Ref = *TabInst->getRefAddr(Idx);\n  if (Ref.isNull()) {\n    spdlog::error(ErrCode::Value::UninitializedElement);\n    spdlog::error(ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset(),\n                                           {Idx},\n                                           {ValTypeFromType<uint32_t>()}));\n    return Unexpect(ErrCode::Value::UninitializedElement);\n  }\n\n  // Check function type.\n  const auto *FuncInst = retrieveFuncRef(Ref);\n  bool IsMatch = false;\n  if (FuncInst->getModule()) {\n    IsMatch = AST::TypeMatcher::matchType(\n        ModInst->getTypeList(), *ExpDefType.getTypeIndex(),\n        FuncInst->getModule()->getTypeList(), FuncInst->getTypeIndex());\n  } else {\n    // Independent host module instance case. Matching the composite type\n    // directly.\n    IsMatch = AST::TypeMatcher::matchType(\n        ModInst->getTypeList(), ExpDefType.getCompositeType(),\n        FuncInst->getHostFunc().getDefinedType().getCompositeType());\n  }\n  if (!IsMatch) {\n    auto &ExpFuncType = ExpDefType.getCompositeType().getFuncType();\n    auto &GotFuncType = FuncInst->getFuncType();\n    spdlog::error(ErrCode::Value::IndirectCallTypeMismatch);\n    spdlog::error(ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset(),\n                                           {Idx},\n                                           {ValTypeFromType<uint32_t>()}));\n    spdlog::error(ErrInfo::InfoMismatch(\n        ExpFuncType.getParamTypes(), ExpFuncType.getReturnTypes(),\n        GotFuncType.getParamTypes(), GotFuncType.getReturnTypes()));\n    return Unexpect(ErrCode::Value::IndirectCallTypeMismatch);\n  }\n\n  // Enter the function.\n  if (auto Res = enterFunction(StackMgr, *FuncInst, PC + 1, IsTailCall); !Res) {\n    return Unexpect(Res);\n  } else {\n    PC = (*Res) - 1;\n  }\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/proxy.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n#include \"system/fault.h\"\n\n#include <cstdint>\n\nnamespace WasmEdge {\nnamespace Executor {\n\nthread_local Executor *Executor::This = nullptr;\nthread_local Runtime::StackManager *Executor::CurrentStack = nullptr;\nthread_local Executor::ExecutionContextStruct Executor::ExecutionContext;\n\ntemplate <typename RetT, typename... ArgsT>\nstruct Executor::ProxyHelper<Expect<RetT> (Executor::*)(Runtime::StackManager &,\n                                                        ArgsT...) noexcept> {\n  template <Expect<RetT> (Executor::*Func)(Runtime::StackManager &,\n                                           ArgsT...) noexcept>\n  \nstatic auto proxy(ArgsT... Args) {\n    Expect<RetT> Res = (This->*Func)(*CurrentStack, Args...);\n    if (unlikely(!Res)) {\n      Fault::emitFault(Res.error());\n    }\n    if constexpr (std::is_same_v<RetT, RefVariant>) {\n#if defined(_MSC_VER) && !defined(__clang__) // MSVC\n      return *reinterpret_cast<__m128 *>((*Res).getRawData().data());\n#else\n      return (*Res).getRawData();\n#endif // MSVC\n    } else if constexpr (!std::is_void_v<RetT>) {\n      return *Res;\n    }\n  }\n};\n\n#if defined(__clang_major__) && __clang_major__ >= 10\n#pragma clang diagnostic push\n#pragma clang diagnostic ignored \"-Wc99-designator\"\n#endif\n\n// Intrinsics table\nconst Executable::IntrinsicsTable Executor::Intrinsics = {\n#if defined(_MSC_VER) && !defined(__clang__)\n#define ENTRY(NAME, FUNC)                                                      \\\n  reinterpret_cast<void *>(&Executor::ProxyHelper<                             \\\n                           decltype(&Executor::FUNC)>::proxy<&Executor::FUNC>)\n#else\n#define ENTRY(NAME, FUNC)                                                      \\\n  [uint8_t(Executable::Intrinsics::NAME)] = reinterpret_cast<void *>(          \\\n      &Executor::ProxyHelper<decltype(&Executor::FUNC)>::proxy<                \\\n          &Executor::FUNC>)\n#endif\n    ENTRY(kTrap, trap),\n    ENTRY(kCall, call),\n    ENTRY(kCallIndirect, callIndirect),\n    ENTRY(kMemCopy, memCopy),\n    ENTRY(kMemFill, memFill),\n    ENTRY(kMemGrow, memGrow),\n    ENTRY(kMemSize, memSize),\n    ENTRY(kMemInit, memInit),\n    ENTRY(kDataDrop, dataDrop),\n    ENTRY(kTableGet, tableGet),\n    ENTRY(kTableSet, tableSet),\n    ENTRY(kTableCopy, tableCopy),\n    ENTRY(kTableFill, tableFill),\n    ENTRY(kTableGrow, tableGrow),\n    ENTRY(kTableSize, tableSize),\n    ENTRY(kTableInit, tableInit),\n    ENTRY(kElemDrop, elemDrop),\n    ENTRY(kRefFunc, refFunc),\n    ENTRY(kTableGetFuncSymbol, tableGetFuncSymbol),\n    ENTRY(kMemoryAtomicNotify, memoryAtomicNotify),\n    ENTRY(kMemoryAtomicWait, memoryAtomicWait),\n    ENTRY(kCallRef, callRef),\n    ENTRY(kRefGetFuncSymbol, refGetFuncSymbol),\n#undef ENTRY\n};\n\n#if defined(__clang_major__) && __clang_major__ >= 10\n#pragma clang diagnostic pop\n#endif\n\nExpect<void> Executor::trap(Runtime::StackManager &,\n                            const uint32_t Code) noexcept {\n  return Unexpect(static_cast<ErrCategory>(Code >> 24), Code);\n}\n\nExpect<void> Executor::call(Runtime::StackManager &StackMgr,\n                            const uint32_t FuncIdx, const ValVariant *Args,\n                            ValVariant *Rets) noexcept {\n  const auto *FuncInst = getFuncInstByIdx(StackMgr, FuncIdx);\n  const auto &FuncType = FuncInst->getFuncType();\n  const uint32_t ParamsSize =\n      static_cast<uint32_t>(FuncType.getParamTypes().size());\n  const uint32_t ReturnsSize =\n      static_cast<uint32_t>(FuncType.getReturnTypes().size());\n\n  for (uint32_t I = 0; I < ParamsSize; ++I) {\n    StackMgr.push(Args[I]);\n  }\n\n  auto Instrs = FuncInst->getInstrs();\n  AST::InstrView::iterator StartIt;\n  if (auto Res = enterFunction(StackMgr, *FuncInst, Instrs.end())) {\n    StartIt = *Res;\n  } else {\n    return Unexpect(Res);\n  }\n  if (auto Res = execute(StackMgr, StartIt, Instrs.end()); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  for (uint32_t I = 0; I < ReturnsSize; ++I) {\n    Rets[ReturnsSize - 1 - I] = StackMgr.pop();\n  }\n\n  return {};\n}\n\nExpect<void *> Executor::tableGetFuncSymbol(Runtime::StackManager &StackMgr,\n                                            const uint32_t TableIdx,\n                                            const uint32_t FuncTypeIdx,\n                                            const uint32_t FuncIdx) noexcept {\n  const auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n\n  if (unlikely(FuncIdx >= TabInst->getSize())) {\n    return Unexpect(ErrCode::Value::UndefinedElement);\n  }\n\n  auto Ref = TabInst->getRefAddr(FuncIdx);\n  assuming(Ref);\n  if (unlikely(Ref->isNull())) {\n    return Unexpect(ErrCode::Value::UninitializedElement);\n  }\n\n  const auto *ModInst = StackMgr.getModule();\n  assuming(ModInst);\n  const auto &ExpDefType = **ModInst->getType(FuncTypeIdx);\n  const auto *FuncInst = retrieveFuncRef(*Ref);\n  assuming(FuncInst);\n  bool IsMatch = false;\n  if (FuncInst->getModule()) {\n    IsMatch = AST::TypeMatcher::matchType(\n        ModInst->getTypeList(), *ExpDefType.getTypeIndex(),\n        FuncInst->getModule()->getTypeList(), FuncInst->getTypeIndex());\n  } else {\n    // Independent host module instance case. Matching the composite type\n    // directly.\n    IsMatch = AST::TypeMatcher::matchType(\n        ModInst->getTypeList(), ExpDefType.getCompositeType(),\n        FuncInst->getHostFunc().getDefinedType().getCompositeType());\n  }\n  if (!IsMatch) {\n    return Unexpect(ErrCode::Value::IndirectCallTypeMismatch);\n  }\n\n  if (unlikely(!FuncInst->isCompiledFunction())) {\n    return nullptr;\n  }\n\n  return FuncInst->getSymbol().get();\n}\n\nExpect<void>\nExecutor::callIndirect(Runtime::StackManager &StackMgr, const uint32_t TableIdx,\n                       const uint32_t FuncTypeIdx, const uint32_t FuncIdx,\n                       const ValVariant *Args, ValVariant *Rets) noexcept {\n  const auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n\n  if (unlikely(FuncIdx >= TabInst->getSize())) {\n    return Unexpect(ErrCode::Value::UndefinedElement);\n  }\n\n  auto Ref = TabInst->getRefAddr(FuncIdx);\n  assuming(Ref);\n  if (unlikely(Ref->isNull())) {\n    return Unexpect(ErrCode::Value::UninitializedElement);\n  }\n\n  const auto *ModInst = StackMgr.getModule();\n  assuming(ModInst);\n  const auto &ExpDefType = **ModInst->getType(FuncTypeIdx);\n  const auto *FuncInst = retrieveFuncRef(*Ref);\n  assuming(FuncInst);\n  bool IsMatch = false;\n  if (FuncInst->getModule()) {\n    IsMatch = AST::TypeMatcher::matchType(\n        ModInst->getTypeList(), *ExpDefType.getTypeIndex(),\n        FuncInst->getModule()->getTypeList(), FuncInst->getTypeIndex());\n  } else {\n    // Independent host module instance case. Matching the composite type\n    // directly.\n    IsMatch = AST::TypeMatcher::matchType(\n        ModInst->getTypeList(), ExpDefType.getCompositeType(),\n        FuncInst->getHostFunc().getDefinedType().getCompositeType());\n  }\n  if (!IsMatch) {\n    return Unexpect(ErrCode::Value::IndirectCallTypeMismatch);\n  }\n\n  const auto &FuncType = FuncInst->getFuncType();\n  const uint32_t ParamsSize =\n      static_cast<uint32_t>(FuncType.getParamTypes().size());\n  const uint32_t ReturnsSize =\n      static_cast<uint32_t>(FuncType.getReturnTypes().size());\n\n  for (uint32_t I = 0; I < ParamsSize; ++I) {\n    StackMgr.push(Args[I]);\n  }\n\n  auto Instrs = FuncInst->getInstrs();\n  AST::InstrView::iterator StartIt;\n  if (auto Res = enterFunction(StackMgr, *FuncInst, Instrs.end())) {\n    StartIt = *Res;\n  } else {\n    return Unexpect(Res);\n  }\n  if (auto Res = execute(StackMgr, StartIt, Instrs.end()); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  for (uint32_t I = 0; I < ReturnsSize; ++I) {\n    Rets[ReturnsSize - 1 - I] = StackMgr.pop();\n  }\n\n  return {};\n}\n\nExpect<uint32_t> Executor::memGrow(Runtime::StackManager &StackMgr,\n                                   const uint32_t MemIdx,\n                                   const uint32_t NewSize) noexcept {\n  auto *MemInst = getMemInstByIdx(StackMgr, MemIdx);\n  assuming(MemInst);\n  const uint32_t CurrPageSize = MemInst->getPageSize();\n  if (MemInst->growPage(NewSize)) {\n    return CurrPageSize;\n  } else {\n    return static_cast<uint32_t>(-1);\n  }\n}\n\nExpect<uint32_t> Executor::memSize(Runtime::StackManager &StackMgr,\n                                   const uint32_t MemIdx) noexcept {\n  auto *MemInst = getMemInstByIdx(StackMgr, MemIdx);\n  assuming(MemInst);\n  return MemInst->getPageSize();\n}\n\nExpect<void> Executor::memCopy(Runtime::StackManager &StackMgr,\n                               const uint32_t DstMemIdx,\n                               const uint32_t SrcMemIdx, const uint32_t DstOff,\n                               const uint32_t SrcOff,\n                               const uint32_t Len) noexcept {\n  auto *MemInstDst = getMemInstByIdx(StackMgr, DstMemIdx);\n  assuming(MemInstDst);\n  auto *MemInstSrc = getMemInstByIdx(StackMgr, SrcMemIdx);\n  assuming(MemInstSrc);\n\n  if (auto Data = MemInstSrc->getBytes(SrcOff, Len); unlikely(!Data)) {\n    return Unexpect(Data);\n  } else {\n    if (auto Res = MemInstDst->setBytes(*Data, DstOff, 0, Len);\n        unlikely(!Res)) {\n      return Unexpect(Res);\n    }\n  }\n\n  return {};\n}\n\nExpect<void> Executor::memFill(Runtime::StackManager &StackMgr,\n                               const uint32_t MemIdx, const uint32_t Off,\n                               const uint8_t Val, const uint32_t Len) noexcept {\n  auto *MemInst = getMemInstByIdx(StackMgr, MemIdx);\n  assuming(MemInst);\n  if (auto Res = MemInst->fillBytes(Val, Off, Len); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  return {};\n}\n\nExpect<void> Executor::memInit(Runtime::StackManager &StackMgr,\n                               const uint32_t MemIdx, const uint32_t DataIdx,\n                               const uint32_t DstOff, const uint32_t SrcOff,\n                               const uint32_t Len) noexcept {\n  auto *MemInst = getMemInstByIdx(StackMgr, MemIdx);\n  assuming(MemInst);\n  auto *DataInst = getDataInstByIdx(StackMgr, DataIdx);\n  assuming(DataInst);\n\n  if (auto Res = MemInst->setBytes(DataInst->getData(), DstOff, SrcOff, Len);\n      unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  return {};\n}\n\nExpect<void> Executor::dataDrop(Runtime::StackManager &StackMgr,\n                                const uint32_t DataIdx) noexcept {\n  auto *DataInst = getDataInstByIdx(StackMgr, DataIdx);\n  assuming(DataInst);\n  DataInst->clear();\n\n  return {};\n}\n\nExpect<RefVariant> Executor::tableGet(Runtime::StackManager &StackMgr,\n                                      const uint32_t TableIdx,\n                                      const uint32_t Off) noexcept {\n  auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n  if (auto Res = TabInst->getRefAddr(Off); unlikely(!Res)) {\n    return Unexpect(Res);\n  } else {\n    return *Res;\n  }\n}\n\nExpect<void> Executor::tableSet(Runtime::StackManager &StackMgr,\n                                const uint32_t TableIdx, const uint32_t Off,\n                                const RefVariant Ref) noexcept {\n  auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n  if (auto Res = TabInst->setRefAddr(Off, Ref); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  return {};\n}\n\nExpect<void> Executor::tableCopy(Runtime::StackManager &StackMgr,\n                                 const uint32_t TableIdxDst,\n                                 const uint32_t TableIdxSrc,\n                                 const uint32_t DstOff, const uint32_t SrcOff,\n                                 const uint32_t Len) noexcept {\n  auto *TabInstDst = getTabInstByIdx(StackMgr, TableIdxDst);\n  assuming(TabInstDst);\n  auto *TabInstSrc = getTabInstByIdx(StackMgr, TableIdxSrc);\n  assuming(TabInstSrc);\n\n  if (auto Refs = TabInstSrc->getRefs(0, SrcOff + Len); unlikely(!Refs)) {\n    return Unexpect(Refs);\n  } else {\n    if (auto Res = TabInstDst->setRefs(*Refs, DstOff, SrcOff, Len);\n        unlikely(!Res)) {\n      return Unexpect(Res);\n    }\n  }\n\n  return {};\n}\n\nExpect<uint32_t> Executor::tableGrow(Runtime::StackManager &StackMgr,\n                                     const uint32_t TableIdx,\n                                     const RefVariant Val,\n                                     const uint32_t NewSize) noexcept {\n  auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n  const uint32_t CurrTableSize = TabInst->getSize();\n  if (likely(TabInst->growTable(NewSize, Val))) {\n    return CurrTableSize;\n  } else {\n    return static_cast<uint32_t>(-1);\n  }\n}\n\nExpect<uint32_t> Executor::tableSize(Runtime::StackManager &StackMgr,\n                                     const uint32_t TableIdx) noexcept {\n  auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n  return TabInst->getSize();\n}\n\nExpect<void> Executor::tableFill(Runtime::StackManager &StackMgr,\n                                 const uint32_t TableIdx, const uint32_t Off,\n                                 const RefVariant Ref,\n                                 const uint32_t Len) noexcept {\n  auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n  if (auto Res = TabInst->fillRefs(Ref, Off, Len); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  return {};\n}\n\nExpect<void> Executor::tableInit(Runtime::StackManager &StackMgr,\n                                 const uint32_t TableIdx,\n                                 const uint32_t ElemIdx, const uint32_t DstOff,\n                                 const uint32_t SrcOff,\n                                 const uint32_t Len) noexcept {\n  auto *TabInst = getTabInstByIdx(StackMgr, TableIdx);\n  assuming(TabInst);\n  auto *ElemInst = getElemInstByIdx(StackMgr, ElemIdx);\n  assuming(ElemInst);\n  if (auto Res = TabInst->setRefs(ElemInst->getRefs(), DstOff, SrcOff, Len);\n      unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  return {};\n}\n\nExpect<void> Executor::elemDrop(Runtime::StackManager &StackMgr,\n                                const uint32_t ElemIdx) noexcept {\n  auto *ElemInst = getElemInstByIdx(StackMgr, ElemIdx);\n  assuming(ElemInst);\n  ElemInst->clear();\n\n  return {};\n}\n\nExpect<RefVariant> Executor::refFunc(Runtime::StackManager &StackMgr,\n                                     const uint32_t FuncIdx) noexcept {\n  auto *FuncInst = getFuncInstByIdx(StackMgr, FuncIdx);\n  assuming(FuncInst);\n  return RefVariant(FuncInst);\n}\n\nExpect<uint32_t> Executor::memoryAtomicNotify(Runtime::StackManager &StackMgr,\n                                              const uint32_t MemIdx,\n                                              const uint32_t Offset,\n                                              const uint32_t Count) noexcept {\n  auto *MemInst = getMemInstByIdx(StackMgr, MemIdx);\n  assuming(MemInst);\n\n  return atomicNotify(*MemInst, Offset, Count);\n}\n\nExpect<uint32_t> Executor::memoryAtomicWait(Runtime::StackManager &StackMgr,\n                                            const uint32_t MemIdx,\n                                            const uint32_t Offset,\n                                            const uint64_t Expected,\n                                            const int64_t Timeout,\n                                            const uint32_t BitWidth) noexcept {\n  auto *MemInst = getMemInstByIdx(StackMgr, MemIdx);\n  assuming(MemInst);\n\n  if (BitWidth == 64) {\n    return atomicWait<uint64_t>(*MemInst, Offset, Expected, Timeout);\n  } else if (BitWidth == 32) {\n    return atomicWait<uint32_t>(*MemInst, Offset,\n                                static_cast<uint32_t>(Expected), Timeout);\n  }\n\n  assumingUnreachable();\n}\n\nExpect<void> Executor::callRef(Runtime::StackManager &StackMgr,\n                               const RefVariant Ref, const ValVariant *Args,\n                               ValVariant *Rets) noexcept {\n  const auto *FuncInst = retrieveFuncRef(Ref);\n  const auto &FuncType = FuncInst->getFuncType();\n  const uint32_t ParamsSize =\n      static_cast<uint32_t>(FuncType.getParamTypes().size());\n  const uint32_t ReturnsSize =\n      static_cast<uint32_t>(FuncType.getReturnTypes().size());\n\n  for (uint32_t I = 0; I < ParamsSize; ++I) {\n    StackMgr.push(Args[I]);\n  }\n\n  auto Instrs = FuncInst->getInstrs();\n  AST::InstrView::iterator StartIt;\n  if (auto Res = enterFunction(StackMgr, *FuncInst, Instrs.end())) {\n    StartIt = *Res;\n  } else {\n    return Unexpect(Res);\n  }\n  if (auto Res = execute(StackMgr, StartIt, Instrs.end()); unlikely(!Res)) {\n    return Unexpect(Res);\n  }\n\n  for (uint32_t I = 0; I < ReturnsSize; ++I) {\n    Rets[ReturnsSize - 1 - I] = StackMgr.pop();\n  }\n\n  return {};\n}\n\nExpect<void *> Executor::refGetFuncSymbol(Runtime::StackManager &,\n                                          const RefVariant Ref) noexcept {\n  const auto *FuncInst = retrieveFuncRef(Ref);\n  assuming(FuncInst);\n  if (unlikely(!FuncInst->isCompiledFunction())) {\n    return nullptr;\n  }\n  return FuncInst->getSymbol().get();\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/tableInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void> Executor::runTableGetOp(Runtime::StackManager &StackMgr,\n                                     Runtime::Instance::TableInstance &TabInst,\n                                     const AST::Instruction &Instr) {\n  // Pop Idx from Stack.\n  uint32_t Idx = StackMgr.pop().get<uint32_t>();\n\n  // Get table[Idx] and push to Stack.\n  if (auto Res = TabInst.getRefAddr(Idx)) {\n    StackMgr.push(*Res);\n  } else {\n    spdlog::error(ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset(),\n                                           {Idx},\n                                           {ValTypeFromType<uint32_t>()}));\n    return Unexpect(Res);\n  }\n  return {};\n}\n\n...\n", "needle_token_start": 10437, "needle_token_end": 10621, "code_context_ntokens": 16183, "output": ["```cpp\ntemplate <typename RetT, typename... ArgsT>\nstruct Executor::ProxyHelper<Expect<RetT> (Executor::*)(Runtime::StackManager &,\n                                                        ArgsT...) noexcept> {\n  template <Expect<RetT> (Executor::*Func)(Runtime::StackManager &,\n                                           ArgsT...) noexcept>\n  static auto proxy(ArgsT... Args) {\n    Expect<RetT> Res = (This->*Func)(*CurrentStack, Args...);\n    if (unlikely(!Res)) {\n      Fault::emitFault(Res.error());\n    }\n    if constexpr (std::is_same_v<RetT, RefVariant>) {\n#if defined(_MSC_VER) && !defined(__clang__) // MSVC\n      return *reinterpret_cast<__m128 *>((*Res).getRawData().data());\n#else\n      return (*Res).getRawData();\n#endif // MSVC\n    } else if constexpr (!std::is_void_v<RetT>) {\n      return *Res;\n    }\n  }\n};\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "loadPlugins", "language": "cpp", "path": "lib/driver/wasiNNRPCServerTool.cpp", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to initialize and register plugins from predefined paths, ensuring that the system is equipped with necessary extensions or functionalities before performing further operations.\n2. **Input**: There are no direct inputs from the user or function parameters; it operates based on internally defined plugin paths.\n3. **Output**: This function does not return any values but outputs log messages indicating the status of plugin loading, such as successful loads or failures.\n4. **Procedure**: \n   - Iterates over a list of default plugin paths.\n   - Attempts to load plugins from each path.\n   - Logs the outcome of each attempt, whether successful or not.\n   - After attempting to load plugins from all paths, it logs the names of all successfully loaded plugins.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/driver/uniTool.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"driver/unitool.h\"\n#include \"common/log.h\"\n#include \"driver/compiler.h\"\n#include \"driver/tool.h\"\n#include \"po/argument_parser.h\"\n\n#include <iostream>\n#include <string_view>\n\nnamespace WasmEdge {\nnamespace Driver {\n\nint UniTool(int Argc, const char *Argv[], const ToolType ToolSelect) noexcept {\n  using namespace std::literals;\n\n  std::ios::sync_with_stdio(false);\n  Log::setInfoLoggingLevel();\n\n  auto Parser = PO::ArgumentParser();\n\n  PO::SubCommand ToolSubCommand(\n      PO::Description(\"Wasmedge runtime tool subcommand\"sv));\n  PO::SubCommand CompilerSubCommand(\n      PO::Description(\"Wasmedge compiler subcommand\"sv));\n  struct DriverToolOptions ToolOptions;\n  struct DriverCompilerOptions CompilerOptions;\n\n  // Construct Parser Subcommands and Options\n  if (ToolSelect == ToolType::All) {\n    ToolOptions.add_option(Parser);\n\n    Parser.begin_subcommand(CompilerSubCommand, \"compile\"sv);\n    CompilerOptions.add_option(Parser);\n    Parser.end_subcommand();\n\n    Parser.begin_subcommand(ToolSubCommand, \"run\"sv);\n    ToolOptions.add_option(Parser);\n    Parser.end_subcommand();\n  } else if (ToolSelect == ToolType::Tool) {\n    ToolOptions.add_option(Parser);\n  } else if (ToolSelect == ToolType::Compiler) {\n    CompilerOptions.add_option(Parser);\n  } else {\n    return EXIT_FAILURE;\n  }\n\n  // Parse\n  if (!Parser.parse(stdout, Argc, Argv)) {\n    return EXIT_FAILURE;\n  }\n  if (Parser.isVersion()) {\n    std::cout << Argv[0] << \" version \"sv << kVersionString << '\\n';\n    for (const auto &Plugin : Plugin::Plugin::plugins()) {\n      auto PluginVersion = Plugin.version();\n      std::cout << Plugin.path().string() << \" (plugin \\\"\"sv << Plugin.name()\n                << \"\\\") version \"sv << PluginVersion.Major << '.'\n                << PluginVersion.Minor << '.' << PluginVersion.Patch << '.'\n                << PluginVersion.Build << '\\n';\n    }\n    return EXIT_SUCCESS;\n  }\n  if (Parser.isHelp()) {\n    return EXIT_SUCCESS;\n  }\n\n  // Forward Results\n  if (ToolSubCommand.is_selected() || ToolSelect == ToolType::Tool) {\n    return Tool(ToolOptions);\n  } else if (CompilerSubCommand.is_selected() ||\n             ToolSelect == ToolType::Compiler) {\n    return Compiler(CompilerOptions);\n  } else {\n    return Tool(ToolOptions);\n  }\n}\n} // namespace Driver\n} // namespace WasmEdge\n\n// Path: lib/driver/compilerTool.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/configure.h\"\n#include \"common/defines.h\"\n#include \"common/filesystem.h\"\n#include \"common/version.h\"\n#include \"driver/compiler.h\"\n#include \"loader/loader.h\"\n#include \"validator/validator.h\"\n#include \"llvm/codegen.h\"\n#include \"llvm/compiler.h\"\n#include <cstdint>\n#include <cstdlib>\n#include <iostream>\n#include <memory>\n#include <string>\n#include <utility>\n#include <vector>\n\nnamespace WasmEdge {\nnamespace Driver {\n\nint Compiler([[maybe_unused]] struct DriverCompilerOptions &Opt) noexcept {\n  using namespace std::literals;\n\n  std::ios::sync_with_stdio(false);\n  Log::setInfoLoggingLevel();\n\n#ifdef WASMEDGE_USE_LLVM\n\n  Configure Conf;\n  if (Opt.PropMutGlobals.value()) {\n    Conf.removeProposal(Proposal::ImportExportMutGlobals);\n  }\n  if (Opt.PropNonTrapF2IConvs.value()) {\n    Conf.removeProposal(Proposal::NonTrapFloatToIntConversions);\n  }\n  if (Opt.PropSignExtendOps.value()) {\n    Conf.removeProposal(Proposal::SignExtensionOperators);\n  }\n  if (Opt.PropMultiValue.value()) {\n    Conf.removeProposal(Proposal::MultiValue);\n  }\n  if (Opt.PropBulkMemOps.value()) {\n    Conf.removeProposal(Proposal::BulkMemoryOperations);\n  }\n  if (Opt.PropRefTypes.value()) {\n    Conf.removeProposal(Proposal::ReferenceTypes);\n  }\n  if (Opt.PropSIMD.value()) {\n    Conf.removeProposal(Proposal::SIMD);\n  }\n  if (Opt.PropMultiMem.value()) {\n    Conf.addProposal(Proposal::MultiMemories);\n  }\n  if (Opt.PropTailCall.value()) {\n    Conf.addProposal(Proposal::TailCall);\n  }\n  if (Opt.PropExtendConst.value()) {\n    Conf.addProposal(Proposal::ExtendedConst);\n  }\n  if (Opt.PropThreads.value()) {\n    Conf.addProposal(Proposal::Threads);\n  }\n  if (Opt.PropAll.value()) {\n    Conf.addProposal(Proposal::MultiMemories);\n    Conf.addProposal(Proposal::TailCall);\n    Conf.addProposal(Proposal::ExtendedConst);\n    Conf.addProposal(Proposal::Threads);\n  }\n\n  if (Opt.PropOptimizationLevel.value() == \"0\") {\n    Conf.getCompilerConfigure().setOptimizationLevel(\n        WasmEdge::CompilerConfigure::OptimizationLevel::O0);\n  } else if (Opt.PropOptimizationLevel.value() == \"1\") {\n    Conf.getCompilerConfigure().setOptimizationLevel(\n        WasmEdge::CompilerConfigure::OptimizationLevel::O1);\n  } else if (Opt.PropOptimizationLevel.value() == \"3\") {\n    Conf.getCompilerConfigure().setOptimizationLevel(\n        WasmEdge::CompilerConfigure::OptimizationLevel::O3);\n  } else if (Opt.PropOptimizationLevel.value() == \"s\") {\n    Conf.getCompilerConfigure().setOptimizationLevel(\n        WasmEdge::CompilerConfigure::OptimizationLevel::Os);\n  } else if (Opt.PropOptimizationLevel.value() == \"z\") {\n    Conf.getCompilerConfigure().setOptimizationLevel(\n        WasmEdge::CompilerConfigure::OptimizationLevel::Oz);\n  } else {\n    Conf.getCompilerConfigure().setOptimizationLevel(\n        WasmEdge::CompilerConfigure::OptimizationLevel::O2);\n  }\n\n  // Set force interpreter here to load instructions of function body forcibly.\n  Conf.getRuntimeConfigure().setForceInterpreter(true);\n\n  std::filesystem::path InputPath =\n      std::filesystem::absolute(std::filesystem::u8path(Opt.WasmName.value()));\n  std::filesystem::path OutputPath =\n      std::filesystem::absolute(std::filesystem::u8path(Opt.SoName.value()));\n  Loader::Loader Loader(Conf);\n\n  std::vector<Byte> Data;\n  if (auto Res = Loader.loadFile(InputPath)) {\n    Data = std::move(*Res);\n  } else {\n    const auto Err = static_cast<uint32_t>(Res.error());\n    spdlog::error(\"Load failed. Error code: {}\", Err);\n    return EXIT_FAILURE;\n  }\n\n  std::unique_ptr<AST::Module> Module;\n  if (auto Res = Loader.parseModule(Data)) {\n    Module = std::move(*Res);\n  } else {\n    const auto Err = static_cast<uint32_t>(Res.error());\n    spdlog::error(\"Parse Module failed. Error code: {}\", Err);\n    return EXIT_FAILURE;\n  }\n\n  {\n    Validator::Validator ValidatorEngine(Conf);\n    if (auto Res = ValidatorEngine.validate(*Module); !Res) {\n      const auto Err = static_cast<uint32_t>(Res.error());\n      spdlog::error(\"Validate Module failed. Error code: {}\", Err);\n      return EXIT_FAILURE;\n    }\n  }\n\n  {\n    if (Opt.ConfDumpIR.value()) {\n      Conf.getCompilerConfigure().setDumpIR(true);\n    }\n    if (Opt.ConfInterruptible.value()) {\n      Conf.getCompilerConfigure().setInterruptible(true);\n    }\n    if (Opt.ConfEnableAllStatistics.value()) {\n      Conf.getStatisticsConfigure().setInstructionCounting(true);\n      Conf.getStatisticsConfigure().setCostMeasuring(true);\n      Conf.getStatisticsConfigure().setTimeMeasuring(true);\n    } else {\n      if (Opt.ConfEnableInstructionCounting.value()) {\n        Conf.getStatisticsConfigure().setInstructionCounting(true);\n      }\n      if (Opt.ConfEnableGasMeasuring.value()) {\n        Conf.getStatisticsConfigure().setCostMeasuring(true);\n      }\n      if (Opt.ConfEnableTimeMeasuring.value()) {\n        Conf.getStatisticsConfigure().setTimeMeasuring(true);\n      }\n    }\n    if (Opt.ConfGenericBinary.value()) {\n      Conf.getCompilerConfigure().setGenericBinary(true);\n    }\n    if (OutputPath.extension().u8string() == WASMEDGE_LIB_EXTENSION) {\n      Conf.getCompilerConfigure().setOutputFormat(\n          CompilerConfigure::OutputFormat::Native);\n    }\n    LLVM::Compiler Compiler(Conf);\n    LLVM::CodeGen CodeGen(Conf);\n    if (auto Res = Compiler.compile(*Module); !Res) {\n      const auto Err = static_cast<uint32_t>(Res.error());\n      spdlog::error(\"Compilation failed. Error code: {}\", Err);\n      return EXIT_FAILURE;\n    } else if (auto Res2 = CodeGen.codegen(Data, std::move(*Res), OutputPath);\n               !Res2) {\n      const auto Err = static_cast<uint32_t>(Res2.error());\n      spdlog::error(\"Code Generation failed. Error code: {}\", Err);\n      return EXIT_FAILURE;\n    }\n  }\n\n  return EXIT_SUCCESS;\n#else\n  spdlog::error(\"Compilation is not supported!\");\n\n  return EXIT_FAILURE;\n#endif\n}\n\n} // namespace Driver\n} // namespace WasmEdge\n\n// Path: lib/driver/runtimeTool.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/configure.h\"\n#include \"common/filesystem.h\"\n#include \"common/log.h\"\n#include \"common/types.h\"\n#include \"common/version.h\"\n#include \"driver/tool.h\"\n#include \"host/wasi/wasimodule.h\"\n#include \"vm/vm.h\"\n\n#include <chrono>\n#include <cstdint>\n#include <cstdlib>\n#include <iostream>\n#include <optional>\n#include <string>\n#include <string_view>\n#include <vector>\n\nnamespace WasmEdge {\nnamespace Driver {\n\nint Tool(struct DriverToolOptions &Opt) noexcept {\n  using namespace std::literals;\n\n  std::ios::sync_with_stdio(false);\n  Log::setInfoLoggingLevel();\n\n  Configure Conf;\n  if (Opt.PropAFUNIX.value()) {\n    Conf.getRuntimeConfigure().setAllowAFUNIX(true);\n  }\n  if (Opt.PropMutGlobals.value()) {\n    Conf.removeProposal(Proposal::ImportExportMutGlobals);\n  }\n  if (Opt.PropNonTrapF2IConvs.value()) {\n    Conf.removeProposal(Proposal::NonTrapFloatToIntConversions);\n  }\n  if (Opt.PropSignExtendOps.value()) {\n    Conf.removeProposal(Proposal::SignExtensionOperators);\n  }\n  if (Opt.PropMultiValue.value()) {\n    Conf.removeProposal(Proposal::MultiValue);\n  }\n  if (Opt.PropBulkMemOps.value()) {\n    Conf.removeProposal(Proposal::BulkMemoryOperations);\n  }\n  if (Opt.PropRefTypes.value()) {\n    Conf.removeProposal(Proposal::ReferenceTypes);\n  }\n  if (Opt.PropSIMD.value()) {\n    Conf.removeProposal(Proposal::SIMD);\n  }\n  if (Opt.PropMultiMem.value()) {\n    Conf.addProposal(Proposal::MultiMemories);\n  }\n  if (Opt.PropTailCall.value()) {\n    Conf.addProposal(Proposal::TailCall);\n  }\n  if (Opt.PropExtendConst.value()) {\n    Conf.addProposal(Proposal::ExtendedConst);\n  }\n  if (Opt.PropThreads.value()) {\n    Conf.addProposal(Proposal::Threads);\n  }\n  if (Opt.PropFunctionReference.value()) {\n    Conf.addProposal(Proposal::FunctionReferences);\n  }\n  if (Opt.PropGC.value()) {\n    Conf.addProposal(Proposal::GC);\n    spdlog::warn(\"GC proposal is enabled, this is experimental.\");\n  }\n  if (Opt.PropComponent.value()) {\n    Conf.addProposal(Proposal::Component);\n    spdlog::warn(\"component model is enabled, this is experimental.\");\n  }\n  if (Opt.PropAll.value()) {\n    Conf.addProposal(Proposal::MultiMemories);\n    Conf.addProposal(Proposal::TailCall);\n    Conf.addProposal(Proposal::ExtendedConst);\n    Conf.addProposal(Proposal::Threads);\n    Conf.addProposal(Proposal::GC);\n    Conf.addProposal(Proposal::Component);\n    spdlog::warn(\"GC proposal is enabled, this is experimental.\");\n    spdlog::warn(\"component model is enabled, this is experimental.\");\n  }\n\n  std::optional<std::chrono::system_clock::time_point> Timeout;\n  if (Opt.TimeLim.value() > 0) {\n    Timeout = std::chrono::system_clock::now() +\n              std::chrono::milliseconds(Opt.TimeLim.value());\n  }\n  if (Opt.GasLim.value().size() > 0) {\n    Conf.getStatisticsConfigure().setCostMeasuring(true);\n    Conf.getStatisticsConfigure().setCostLimit(\n        static_cast<uint32_t>(Opt.GasLim.value().back()));\n  }\n  if (Opt.MemLim.value().size() > 0) {\n    Conf.getRuntimeConfigure().setMaxMemoryPage(\n        static_cast<uint32_t>(Opt.MemLim.value().back()));\n  }\n  if (Opt.ConfEnableAllStatistics.value()) {\n    Conf.getStatisticsConfigure().setInstructionCounting(true);\n    Conf.getStatisticsConfigure().setCostMeasuring(true);\n    Conf.getStatisticsConfigure().setTimeMeasuring(true);\n  } else {\n    if (Opt.ConfEnableInstructionCounting.value()) {\n      Conf.getStatisticsConfigure().setInstructionCounting(true);\n    }\n    if (Opt.ConfEnableGasMeasuring.value()) {\n      Conf.getStatisticsConfigure().setCostMeasuring(true);\n    }\n    if (Opt.ConfEnableTimeMeasuring.value()) {\n      Conf.getStatisticsConfigure().setTimeMeasuring(true);\n    }\n  }\n  if (Opt.ConfEnableJIT.value()) {\n    Conf.getRuntimeConfigure().setEnableJIT(true);\n    Conf.getCompilerConfigure().setOptimizationLevel(\n        WasmEdge::CompilerConfigure::OptimizationLevel::O1);\n  }\n  if (Opt.ConfForceInterpreter.value()) {\n    Conf.getRuntimeConfigure().setForceInterpreter(true);\n  }\n\n  for (const auto &Name : Opt.ForbiddenPlugins.value()) {\n    Conf.addForbiddenPlugins(Name);\n  }\n\n  Conf.addHostRegistration(HostRegistration::Wasi);\n  const auto InputPath =\n      std::filesystem::absolute(std::filesystem::u8path(Opt.SoName.value()));\n  VM::VM VM(Conf);\n\n  Host::WasiModule *WasiMod = dynamic_cast<Host::WasiModule *>(\n      VM.getImportModule(HostRegistration::Wasi));\n\n  if (auto Result = VM.loadWasm(InputPath.u8string()); !Result) {\n    return EXIT_FAILURE;\n  }\n  if (auto Result = VM.validate(); !Result) {\n    return EXIT_FAILURE;\n  }\n  if (auto Result = VM.instantiate(); !Result) {\n    return EXIT_FAILURE;\n  }\n\n  auto HasValidCommandModStartFunc = [&]() {\n    bool HasStart = false;\n    bool Valid = false;\n\n    auto Functions = VM.getFunctionList();\n    for (auto &[FuncName, Type] : Functions) {\n      if (FuncName == \"_start\") {\n        HasStart = true;\n        if (Type.getReturnTypes().size() == 0 &&\n            Type.getParamTypes().size() == 0) {\n          Valid = true;\n          break;\n        }\n      }\n    }\n\n    // if HasStart but not Valid, insert _start to enter reactor mode\n    if (HasStart && !Valid) {\n      Opt.Args.value().insert(Opt.Args.value().begin(), \"_start\");\n    }\n\n    return HasStart && Valid;\n  };\n\n  bool EnterCommandMode = !Opt.Reactor.value() && HasValidCommandModStartFunc();\n\n  WasiMod->getEnv().init(\n      Opt.Dir.value(),\n      InputPath.filename()\n          .replace_extension(std::filesystem::u8path(\"wasm\"sv))\n          .u8string(),\n      Opt.Args.value(), Opt.Env.value());\n\n  if (EnterCommandMode) {\n    // command mode\n    auto AsyncResult = VM.asyncExecute(\"_start\"sv);\n    if (Timeout.has_value()) {\n      if (!AsyncResult.waitUntil(*Timeout)) {\n        AsyncResult.cancel();\n      }\n    }\n    if (auto Result = AsyncResult.get();\n        Result || Result.error() == ErrCode::Value::Terminated) {\n      return static_cast<int>(WasiMod->getEnv().getExitCode());\n    } else {\n      // It indicates that the execution of wasm has been aborted\n      return 128 + SIGABRT;\n    }\n  } else {\n    // reactor mode\n    if (Opt.Args.value().empty()) {\n      std::cerr\n          << \"A function name is required when reactor mode is enabled.\\n\";\n      return EXIT_FAILURE;\n    }\n    const auto &FuncName = Opt.Args.value().front();\n\n    using namespace std::literals::string_literals;\n    const auto InitFunc = \"_initialize\"s;\n\n    bool HasInit = false;\n    AST::FunctionType FuncType;\n\n    for (const auto &Func : VM.getFunctionList()) {\n      if (Func.first == InitFunc) {\n        HasInit = true;\n      } else if (Func.first == FuncName) {\n        FuncType = Func.second;\n      }\n    }\n\n    if (HasInit) {\n      auto AsyncResult = VM.asyncExecute(InitFunc);\n      if (Timeout.has_value()) {\n        if (!AsyncResult.waitUntil(*Timeout)) {\n          AsyncResult.cancel();\n        }\n      }\n      if (auto Result = AsyncResult.get(); unlikely(!Result)) {\n        // It indicates that the execution of wasm has been aborted\n        return 128 + SIGABRT;\n      }\n    }\n\n    std::vector<ValVariant> FuncArgs;\n    std::vector<ValType> FuncArgTypes;\n    for (size_t I = 0;\n         I < FuncType.getParamTypes().size() && I + 1 < Opt.Args.value().size();\n         ++I) {\n      switch (FuncType.getParamTypes()[I].getCode()) {\n      case TypeCode::I32: {\n        const uint32_t Value =\n            static_cast<uint32_t>(std::stol(Opt.Args.value()[I + 1]));\n        FuncArgs.emplace_back(Value);\n        FuncArgTypes.emplace_back(TypeCode::I32);\n        break;\n      }\n      case TypeCode::I64: {\n        const uint64_t Value =\n            static_cast<uint64_t>(std::stoll(Opt.Args.value()[I + 1]));\n        FuncArgs.emplace_back(Value);\n        FuncArgTypes.emplace_back(TypeCode::I64);\n        break;\n      }\n      case TypeCode::F32: {\n        const float Value = std::stof(Opt.Args.value()[I + 1]);\n        FuncArgs.emplace_back(Value);\n        FuncArgTypes.emplace_back(TypeCode::F32);\n        break;\n      }\n      case TypeCode::F64: {\n        const double Value = std::stod(Opt.Args.value()[I + 1]);\n        FuncArgs.emplace_back(Value);\n        FuncArgTypes.emplace_back(TypeCode::F64);\n        break;\n      }\n      /// TODO: FuncRef and ExternRef\n      default:\n        break;\n      }\n    }\n    if (FuncType.getParamTypes().size() + 1 < Opt.Args.value().size()) {\n      for (size_t I = FuncType.getParamTypes().size() + 1;\n           I < Opt.Args.value().size(); ++I) {\n        const uint64_t Value =\n            static_cast<uint64_t>(std::stoll(Opt.Args.value()[I]));\n        FuncArgs.emplace_back(Value);\n        FuncArgTypes.emplace_back(TypeCode::I64);\n      }\n    }\n\n    auto AsyncResult = VM.asyncExecute(FuncName, FuncArgs, FuncArgTypes);\n    if (Timeout.has_value()) {\n      if (!AsyncResult.waitUntil(*Timeout)) {\n        AsyncResult.cancel();\n      }\n    }\n    if (auto Result = AsyncResult.get()) {\n      /// Print results.\n      for (size_t I = 0; I < Result->size(); ++I) {\n        switch ((*Result)[I].second.getCode()) {\n        case TypeCode::I32:\n          std::cout << (*Result)[I].first.get<uint32_t>() << '\\n';\n          break;\n        case TypeCode::I64:\n          std::cout << (*Result)[I].first.get<uint64_t>() << '\\n';\n          break;\n        case TypeCode::F32:\n          std::cout << (*Result)[I].first.get<float>() << '\\n';\n          break;\n        case TypeCode::F64:\n          std::cout << (*Result)[I].first.get<double>() << '\\n';\n          break;\n        case TypeCode::V128:\n          std::cout << (*Result)[I].first.get<uint128_t>() << '\\n';\n          break;\n        /// TODO: FuncRef and ExternRef\n        default:\n          break;\n        }\n      }\n      return EXIT_SUCCESS;\n    } else {\n      // It indicates that the execution of wasm has been aborted\n      return 128 + SIGABRT;\n    }\n  }\n}\n\n} // namespace Driver\n} // namespace WasmEdge\n\n// Path: lib/driver/wasiNNRPCServerTool.cpp\n#include \"common/log.h\"\n#include \"driver/wasi_nn_rpc/wasi_nn_rpcserver/wasi_nn_rpcserver.h\"\n#include \"plugin/plugin.h\"\n#include \"po/argument_parser.h\"\n\n#include <grpcpp/server_builder.h>\n#include <string_view>\n\nusing namespace std::literals;\nusing namespace WasmEdge;\n\nnamespace WasmEdge {\nnamespace Driver {\n\n\nvoid loadPlugins(void) {\n  for (const auto &Path : Plugin::Plugin::getDefaultPluginPaths()) {\n    spdlog::info(\"Loading plugin path {}\"sv, Path);\n    if (Plugin::Plugin::load(Path)) {\n      spdlog::info(\"Loaded plugin path {}\"sv, Path);\n    } else {\n      spdlog::info(\"Nothing was loaded from plugin path {}\"sv, Path);\n    }\n  }\n  for (const auto &Plugin : Plugin::Plugin::plugins()) {\n    spdlog::info(\"Plugin: {}\", Plugin.name());\n  }\n}\n\nRuntime::Instance::ModuleInstance *createWasiNNModule() {\n  if (const auto *Plugin = Plugin::Plugin::find(\"wasi_nn\"sv)) {\n    if (const auto *Module = Plugin->findModule(\"wasi_nn\"sv)) {\n      return Module->create().release();\n    }\n  }\n  return nullptr;\n}\n\nint WasiNNRPCServer(int Argc, const char *Argv[]) noexcept {\n  std::ios::sync_with_stdio(false);\n  Log::setInfoLoggingLevel();\n  setenv(\"_WASI_NN_RPCSERVER\", \"1\", 1); // wasi_nn plugin checks this env var\n\n  // Parse the args\n  PO::Option<std::string> NNRPCURI(\n      PO::Description(\"Specify NN RPC URI to serve (\\\"unix://...\\\")\"sv),\n      PO::MetaVar(\"URI\"sv), PO::DefaultValue(std::string(\"\")));\n  auto Parser = PO::ArgumentParser();\n  Parser.add_option(\"nn-rpc-uri\"sv, NNRPCURI);\n  loadPlugins();\n  Plugin::Plugin::addPluginOptions(Parser); // Register \"nn-preload\", etc.\n  if (!Parser.parse(stdout, Argc, Argv)) {\n    return EXIT_FAILURE;\n  }\n  if (Parser.isHelp()) {\n    return EXIT_SUCCESS;\n  }\n  auto URI = NNRPCURI.value();\n  if (URI.empty()) {\n    spdlog::error(\"--nn-rpc-uri has to be specified\"sv);\n    return EXIT_FAILURE;\n  }\n\n  // Create the wasi_nn module\n  auto *NNMod = createWasiNNModule();\n  if (NNMod == nullptr) {\n    spdlog::error(\n        \"Failed to get the wasi_nn module (Hint: set $WASMEDGE_PLUGIN_PATH to \"\n        \"the directory where libwasmedgePluginWasiNN.* exists\"sv);\n    return EXIT_FAILURE;\n  }\n\n  // Create the services\n  WasiNNRPC::Server::ServiceSet ServiceSet(*NNMod);\n\n  // Create the gRPC server\n  grpc::ServerBuilder Builder;\n  spdlog::info(\"Listening on \\\"{}\\\"\"sv, URI);\n  std::string_view UnixPrefix = \"unix://\";\n  if (URI.substr(0, UnixPrefix.length()) != UnixPrefix) {\n    spdlog::warn(\"Expected \\\"unix://...\\\", got \\\"{}\\\"\"sv, URI);\n  }\n  auto Cred = grpc::InsecureServerCredentials(); // safe for unix://...\n  Builder.AddListeningPort(URI, Cred);\n  for (auto *Service : ServiceSet.services()) {\n    Builder.RegisterService(Service);\n  }\n\n  // Start the gRPC server\n  auto Server = Builder.BuildAndStart();\n  if (Server == nullptr) {\n    return EXIT_FAILURE;\n  }\n  Server->Wait();\n  return EXIT_SUCCESS;\n}\n} // namespace Driver\n} // namespace WasmEdge\n\n// Path: lib/driver/fuzzPO.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#ifdef WASMEDGE_BUILD_FUZZING\n#include \"driver/fuzzPO.h\"\n#include \"common/log.h\"\n#include \"common/version.h\"\n#include \"po/argument_parser.h\"\n\n#include <algorithm>\n#include <array>\n#include <cstdio>\n#include <iostream>\n#include <type_traits>\n#include <utility>\n#include <vector>\nnamespace {\ntemplate <class Key, class Value, class Hash, class BinaryPredicate>\nclass SkipTable {\nprivate:\n  using UnsignedKey = std::make_unsigned_t<Key>;\n  std::array<Value,\n             static_cast<std::size_t>(std::numeric_limits<UnsignedKey>::max()) +\n                 1u>\n      Table;\n\npublic:\n  SkipTable(std::size_t, Value Default, Hash, BinaryPredicate) {\n    std::fill_n(Table.begin(), Table.size(), Default);\n  }\n\n  void insert(const Key &K, Value V) { Table[static_cast<UnsignedKey>(K)] = V; }\n\n  const Value &at(const Key &K) const {\n    return Table[static_cast<UnsignedKey>(K)];\n  }\n};\n\ntemplate <class RandomIt1,\n          class Hash =\n              std::hash<typename std::iterator_traits<RandomIt1>::value_type>,\n          class BinaryPredicate = std::equal_to<>>\nclass BoyerMooreHorspoolSearcher {\nprivate:\n  using Key = typename std::iterator_traits<RandomIt1>::value_type;\n  using Value = typename std::iterator_traits<RandomIt1>::difference_type;\n  static_assert(std::is_integral_v<Key> && sizeof(Key) == 1 &&\n                std::is_same_v<Hash, std::hash<Key>> &&\n                std::is_same_v<BinaryPredicate, std::equal_to<>>);\n  using SkipTableType = SkipTable<Key, Value, Hash, BinaryPredicate>;\n\npublic:\n  BoyerMooreHorspoolSearcher(RandomIt1 First, RandomIt1 Last, Hash HF = Hash(),\n                             BinaryPredicate Pred = BinaryPredicate())\n      : Pattern(First), PatternLength(std::distance(First, Last)), Pred(Pred),\n        Table(PatternLength, PatternLength, HF, Pred) {\n    if (First != Last) {\n      --Last;\n      for (Value I = 0; First != Last; ++First, ++I) {\n        Table.insert(*First, PatternLength - 1 - I);\n      }\n    }\n  }\n\n  template <class RandomIt2>\n  std::pair<RandomIt2, RandomIt2> operator()(RandomIt2 First,\n                                             RandomIt2 Last) const {\n    static_assert(\n        std::is_same_v<\n            std::remove_cv_t<std::remove_reference_t<\n                typename std::iterator_traits<RandomIt1>::value_type>>,\n            std::remove_cv_t<std::remove_reference_t<\n                typename std::iterator_traits<RandomIt2>::value_type>>>,\n        \"Corpus and Pattern iterators must point to the same type\");\n    if (First == Last) {\n      // empty corpus\n      return {Last, Last};\n    }\n    if (PatternLength == 0) {\n      // empty pattern\n      return {First, First};\n    }\n    // the pattern is larger than the corpus\n    if (PatternLength > std::distance(First, Last)) {\n      return {Last, Last};\n    }\n\n    RandomIt2 Curr = First;\n    const RandomIt2 End = Last - PatternLength;\n    while (Curr <= End) {\n      Value J = PatternLength;\n      while (Pred(Pattern[J - 1], Curr[J - 1])) {\n        --J;\n        if (J == 0) {\n          // found\n          return {Curr, Curr + PatternLength};\n        }\n      }\n      const auto K = Curr[PatternLength - 1];\n      const auto D = Table.at(K);\n      Curr += D;\n    }\n    return {Last, Last};\n  }\n\nprivate:\n  RandomIt1 Pattern;\n  Value PatternLength;\n  BinaryPredicate Pred;\n  SkipTableType Table;\n};\n} // namespace\n\nnamespace WasmEdge {\nnamespace Driver {\n\nint FuzzPO(const uint8_t *Data, size_t Size) noexcept {\n  using namespace std::literals;\n\n  std::ios::sync_with_stdio(false);\n  spdlog::set_level(spdlog::level::info);\n\n  PO::Option<std::string> SoName(PO::Description(\"Wasm or so file\"sv),\n                                 PO::MetaVar(\"WASM_OR_SO\"sv));\n  PO::List<std::string> Args(PO::Description(\"Execution arguments\"sv),\n                             PO::MetaVar(\"ARG\"sv));\n\n  PO::Option<PO::Toggle> Reactor(PO::Description(\n      \"Enable reactor mode. Reactor mode calls `_initialize` if exported.\"));\n\n  PO::List<std::string> Dir(\n      PO::Description(\n          \"Binding directories into WASI virtual filesystem. Each directories \"\n          \"can specified as --dir `guest_path:host_path`, where `guest_path` \"\n          \"specifies the path that will correspond to `host_path` for calls \"\n          \"like `fopen` in the guest.\"sv),\n      PO::MetaVar(\"PREOPEN_DIRS\"sv));\n\n  PO::List<std::string> Env(\n      PO::Description(\n          \"Environ variables. Each variable can be specified as --env `NAME=VALUE`.\"sv),\n      PO::MetaVar(\"ENVS\"sv));\n\n  PO::Option<PO::Toggle> PropMutGlobals(\n      PO::Description(\"Disable Import/Export of mutable globals proposal\"sv));\n  PO::Option<PO::Toggle> PropNonTrapF2IConvs(PO::Description(\n      \"Disable Non-trapping float-to-int conversions proposal\"sv));\n  PO::Option<PO::Toggle> PropSignExtendOps(\n      PO::Description(\"Disable Sign-extension operators proposal\"sv));\n  PO::Option<PO::Toggle> PropMultiValue(\n      PO::Description(\"Disable Multi-value proposal\"sv));\n  PO::Option<PO::Toggle> PropBulkMemOps(\n      PO::Description(\"Disable Bulk memory operations proposal\"sv));\n  PO::Option<PO::Toggle> PropRefTypes(\n      PO::Description(\"Disable Reference types proposal\"sv));\n  PO::Option<PO::Toggle> PropSIMD(PO::Description(\"Disable SIMD proposal\"sv));\n  PO::Option<PO::Toggle> PropMultiMem(\n      PO::Description(\"Enable Multiple memories proposal\"sv));\n  PO::Option<PO::Toggle> PropTailCall(\n      PO::Description(\"Enable Tail-call proposal\"sv));\n  PO::Option<PO::Toggle> PropExtendConst(\n      PO::Description(\"Enable Extended-const proposal\"sv));\n  PO::Option<PO::Toggle> PropThreads(\n      PO::Description(\"Enable Threads proposal\"sv));\n  PO::Option<PO::Toggle> PropAll(PO::Description(\"Enable all features\"sv));\n\n  PO::Option<PO::Toggle> ConfEnableInstructionCounting(PO::Description(\n      \"Enable generating code for counting Wasm instructions executed.\"sv));\n  PO::Option<PO::Toggle> ConfEnableGasMeasuring(PO::Description(\n      \"Enable generating code for counting gas burned during execution.\"sv));\n  PO::Option<PO::Toggle> ConfEnableTimeMeasuring(PO::Description(\n      \"Enable generating code for counting time during execution.\"sv));\n  PO::Option<PO::Toggle> ConfEnableAllStatistics(PO::Description(\n      \"Enable generating code for all statistics options include instruction counting, gas measuring, and execution time\"sv));\n\n  PO::Option<uint64_t> TimeLim(\n      PO::Description(\n          \"Limitation of maximum time(in milliseconds) for execution, default value is 0 for no limitations\"sv),\n      PO::MetaVar(\"TIMEOUT\"sv), PO::DefaultValue<uint64_t>(0));\n\n  PO::List<int> GasLim(\n      PO::Description(\n          \"Limitation of execution gas. Upper bound can be specified as --gas-limit `GAS_LIMIT`.\"sv),\n      PO::MetaVar(\"GAS_LIMIT\"sv));\n\n  PO::List<int> MemLim(\n      PO::Description(\n          \"Limitation of pages(as size of 64 KiB) in every memory instance. Upper bound can be specified as --memory-page-limit `PAGE_COUNT`.\"sv),\n      PO::MetaVar(\"PAGE_COUNT\"sv));\n\n  PO::List<std::string> ForbiddenPlugins(\n      PO::Description(\"List of plugins to ignore.\"sv), PO::MetaVar(\"NAMES\"sv));\n\n  auto Parser = PO::ArgumentParser();\n  Parser.add_option(SoName)\n      .add_option(Args)\n      .add_option(\"reactor\"sv, Reactor)\n      .add_option(\"dir\"sv, Dir)\n      .add_option(\"env\"sv, Env)\n      .add_option(\"enable-instruction-count\"sv, ConfEnableInstructionCounting)\n      .add_option(\"enable-gas-measuring\"sv, ConfEnableGasMeasuring)\n      .add_option(\"enable-time-measuring\"sv, ConfEnableTimeMeasuring)\n      .add_option(\"enable-all-statistics\"sv, ConfEnableAllStatistics)\n      .add_option(\"disable-import-export-mut-globals\"sv, PropMutGlobals)\n      .add_option(\"disable-non-trap-float-to-int\"sv, PropNonTrapF2IConvs)\n      .add_option(\"disable-sign-extension-operators\"sv, PropSignExtendOps)\n      .add_option(\"disable-multi-value\"sv, PropMultiValue)\n      .add_option(\"disable-bulk-memory\"sv, PropBulkMemOps)\n      .add_option(\"disable-reference-types\"sv, PropRefTypes)\n      .add_option(\"disable-simd\"sv, PropSIMD)\n      .add_option(\"enable-multi-memory\"sv, PropMultiMem)\n      .add_option(\"enable-tail-call\"sv, PropTailCall)\n      .add_option(\"enable-extended-const\"sv, PropExtendConst)\n      .add_option(\"enable-threads\"sv, PropThreads)\n      .add_option(\"enable-all\"sv, PropAll)\n      .add_option(\"time-limit\"sv, TimeLim)\n      .add_option(\"gas-limit\"sv, GasLim)\n      .add_option(\"memory-page-limit\"sv, MemLim)\n      .add_option(\"forbidden-plugin\"sv, ForbiddenPlugins);\n\n  static constexpr const std::array<char, 4> Separator = {'\\xde', '\\xad',\n                                                          '\\xbe', '\\xef'};\n  static const BoyerMooreHorspoolSearcher Searcher(Separator.begin(),\n                                                   Separator.end());\n  Span<const char> RawArgs(reinterpret_cast<const char *>(Data), Size);\n  std::vector<std::string> ArgvStr;\n  std::vector<const char *> Argv;\n  while (!RawArgs.empty()) {\n    const auto It = std::search(RawArgs.begin(), RawArgs.end(), Searcher);\n    ArgvStr.emplace_back(RawArgs.begin(), It);\n    RawArgs = RawArgs.subspan(std::min<size_t>(\n        std::distance(RawArgs.begin(), It) + 4, RawArgs.size()));\n  }\n  for (const auto &Arg : ArgvStr) {\n    Argv.push_back(Arg.c_str());\n  }\n\n  std::unique_ptr<std::FILE, decltype(&std::fclose)> Out{\n      std::fopen(\"/dev/null\", \"w\"), std::fclose};\n  if (!Parser.parse(Out.get(), Argv.size(), Argv.data())) {\n    return EXIT_FAILURE;\n  }\n  if (Parser.isVersion()) {\n    fmt::print(Out.get(), \"{} version {}\\n\"sv, Argv.empty() ? \"\" : Argv[0],\n               kVersionString);\n    return EXIT_SUCCESS;\n  }\n\n  return EXIT_SUCCESS;\n}\n\n} // namespace Driver\n} // namespace WasmEdge\n#endif\n\n// Path: lib/driver/fuzzTool.cpp\n...\n", "needle_token_start": 6402, "needle_token_end": 6548, "code_context_ntokens": 10528, "output": ["```cpp\n#include \"common/log.h\"\n#include \"plugin/plugin.h\"\n#include \"po/argument_parser.h\"\n\n#include <iostream>\n#include <string_view>\n\nusing namespace std::literals;\nusing namespace WasmEdge;\n\nnamespace WasmEdge {\nnamespace Driver {\n\nvoid loadPlugins(void) {\n  for (const auto &Path : Plugin::Plugin::getDefaultPluginPaths()) {\n    spdlog::info(\"Loading plugin path {}\"sv, Path);\n    if (Plugin::Plugin::load(Path)) {\n      spdlog::info(\"Loaded plugin path {}\"sv, Path);\n    } else {\n      spdlog::info(\"Nothing was loaded from plugin path {}\"sv, Path);\n    }\n  }\n  for (const auto &Plugin : Plugin::Plugin::plugins()) {\n    spdlog::info(\"Plugin: {}\", Plugin.name());\n  }\n}\n\n} // namespace Driver\n} // namespace WasmEdge\n```"]}
{"repo": "WasmEdge/WasmEdge", "name": "fromLastError", "language": "cpp", "path": "lib/host/wasi/win.h", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function translates system-specific error codes into standardized error codes defined by a WebAssembly system interface.\n2. **Input**: A system-specific error code (numerical value).\n3. **Output**: A standardized error code as defined by the WebAssembly system interface.\n4. **Procedure**: The function checks the input error code against a series of predefined system-specific error codes. Depending on the input error code, it returns the corresponding standardized error code. If the input error code does not match any predefined codes, it defaults to returning a standardized error code indicating that the operation is not implemented.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/host/wasi/inode-linux.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/defines.h\"\n#if WASMEDGE_OS_LINUX\n\n#include \"common/errcode.h\"\n#include \"common/variant.h\"\n#include \"host/wasi/environ.h\"\n#include \"host/wasi/inode.h\"\n#include \"host/wasi/vfs.h\"\n#include \"linux.h\"\n#include <algorithm>\n#include <cstddef>\n#include <cstdint>\n#include <cstring>\n#include <new>\n#include <string>\n#include <string_view>\n#include <vector>\n\nnamespace WasmEdge {\nnamespace Host {\nnamespace WASI {\n\nnamespace {\n\ninline constexpr bool isSpecialFd(int Fd) noexcept {\n  switch (Fd) {\n  case STDIN_FILENO:\n  case STDOUT_FILENO:\n  case STDERR_FILENO:\n    return true;\n  default:\n    return false;\n  }\n}\n\ninline constexpr __wasi_size_t\ncalculateAddrinfoLinkedListSize(struct addrinfo *const Addrinfo) {\n  __wasi_size_t Length = 0;\n  for (struct addrinfo *TmpPointer = Addrinfo; TmpPointer != nullptr;\n       TmpPointer = TmpPointer->ai_next) {\n    Length++;\n  }\n  return Length;\n};\n\nconstexpr int openFlags(__wasi_oflags_t OpenFlags, __wasi_fdflags_t FdFlags,\n                        VFS::Flags VFSFlags) noexcept {\n  int Flags = O_NOFOLLOW;\n#ifdef O_CLOEXEC\n  Flags |= O_CLOEXEC;\n#endif\n\n  if (VFSFlags & VFS::Read) {\n    if (VFSFlags & VFS::Write) {\n      Flags |= O_RDWR;\n    } else {\n      Flags |= O_RDONLY;\n    }\n  } else if (VFSFlags & VFS::Write) {\n    Flags |= O_WRONLY;\n  } else {\n#ifdef O_PATH\n    if (OpenFlags == __WASI_OFLAGS_DIRECTORY) {\n      Flags |= O_PATH;\n    } else {\n      Flags |= O_RDONLY;\n    }\n#else\n    Flags |= O_RDONLY;\n#endif\n  }\n\n  if (OpenFlags & __WASI_OFLAGS_CREAT) {\n    Flags |= O_CREAT;\n  }\n  if (OpenFlags & __WASI_OFLAGS_DIRECTORY) {\n    Flags |= O_DIRECTORY;\n  }\n  if (OpenFlags & __WASI_OFLAGS_EXCL) {\n    Flags |= O_EXCL;\n  }\n  if (OpenFlags & __WASI_OFLAGS_TRUNC) {\n    Flags |= O_TRUNC;\n  }\n\n  // Convert file descriptor flags.\n  if ((FdFlags & __WASI_FDFLAGS_APPEND) != 0) {\n    Flags |= O_APPEND;\n  }\n  if ((FdFlags & __WASI_FDFLAGS_DSYNC) != 0) {\n#ifdef O_DSYNC\n    Flags |= O_DSYNC;\n#else\n    Flags |= O_SYNC;\n#endif\n  }\n  if ((FdFlags & __WASI_FDFLAGS_NONBLOCK) != 0) {\n    Flags |= O_NONBLOCK;\n  }\n  if ((FdFlags & __WASI_FDFLAGS_RSYNC) != 0) {\n#ifdef O_RSYNC\n    Flags |= O_RSYNC;\n#else\n    Flags |= O_SYNC;\n#endif\n  }\n  if ((FdFlags & __WASI_FDFLAGS_SYNC) != 0) {\n    Flags |= O_SYNC;\n  }\n\n  return Flags;\n}\n\nstd::pair<const char *, std::unique_ptr<char[]>>\ncreateNullTerminatedString(std::string_view View) noexcept {\n  const char *CStr = nullptr;\n  std::unique_ptr<char[]> Buffer;\n  if (!View.empty()) {\n    if (const auto Pos = View.find_first_of('\\0');\n        Pos != std::string_view::npos) {\n      CStr = View.data();\n    } else {\n      Buffer = std::make_unique<char[]>(View.size() + 1);\n      std::copy(View.begin(), View.end(), Buffer.get());\n      CStr = Buffer.get();\n    }\n  }\n  return {CStr, std::move(Buffer)};\n}\n\n} // namespace\n\nvoid FdHolder::reset() noexcept {\n  if (likely(ok())) {\n    if (likely(!isSpecialFd(Fd))) {\n      ::close(Fd);\n    }\n    Fd = -1;\n  }\n}\n\nvoid TimerHolder::reset() noexcept {\n  if (likely(Id.has_value())) {\n    timer_delete(*Id);\n    Id.reset();\n  }\n}\n\nvoid DirHolder::reset() noexcept {\n  if (likely(Dir != nullptr)) {\n    closedir(Dir);\n    Dir = nullptr;\n    Cookie = 0;\n  }\n}\n\nINode INode::stdIn() noexcept { return INode(STDIN_FILENO); }\n\nINode INode::stdOut() noexcept { return INode(STDOUT_FILENO); }\n\nINode INode::stdErr() noexcept { return INode(STDERR_FILENO); }\n\nWasiExpect<INode> INode::open(std::string Path, __wasi_oflags_t OpenFlags,\n                              __wasi_fdflags_t FdFlags,\n                              VFS::Flags VFSFlags) noexcept {\n  const int Flags = openFlags(OpenFlags, FdFlags, VFSFlags);\n\n  if (auto NewFd = ::open(Path.c_str(), Flags, 0644); unlikely(NewFd < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    INode New(NewFd);\n#ifndef O_CLOEXEC\n    if (auto Res = ::fcntl(New.Fd, F_SETFD, FD_CLOEXEC); unlikely(Res != 0)) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n#endif\n    return New;\n  }\n}\n\nWasiExpect<void> INode::fdAdvise(__wasi_filesize_t Offset,\n                                 __wasi_filesize_t Len,\n                                 __wasi_advice_t Advice) const noexcept {\n  if (auto Res = ::posix_fadvise(Fd, Offset, Len, toAdvice(Advice));\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdAllocate(__wasi_filesize_t Offset,\n                                   __wasi_filesize_t Len) const noexcept {\n  if (auto Res = ::posix_fallocate(Fd, Offset, Len); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdDatasync() const noexcept {\n  if (auto Res = ::fdatasync(Fd); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdFdstatGet(__wasi_fdstat_t &FdStat) const noexcept {\n  if (auto Res = updateStat(); unlikely(!Res)) {\n    return WasiUnexpect(Res);\n  }\n\n  if (int FdFlags = ::fcntl(Fd, F_GETFL); unlikely(FdFlags < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    FdStat.fs_filetype = unsafeFiletype();\n\n    FdStat.fs_flags = static_cast<__wasi_fdflags_t>(0);\n    if (FdFlags & O_APPEND) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_APPEND;\n    }\n    if (FdFlags & O_DSYNC) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_DSYNC;\n    }\n    if (FdFlags & O_NONBLOCK) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_NONBLOCK;\n    }\n    if (FdFlags & O_SYNC) {\n      FdStat.fs_flags |= __WASI_FDFLAGS_RSYNC | __WASI_FDFLAGS_SYNC;\n    }\n  }\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFdstatSetFlags(__wasi_fdflags_t FdFlags) const noexcept {\n  int SysFlag = 0;\n  if (FdFlags & __WASI_FDFLAGS_NONBLOCK) {\n    SysFlag |= O_NONBLOCK;\n  }\n  if (FdFlags & __WASI_FDFLAGS_APPEND) {\n    SysFlag |= O_APPEND;\n  }\n  if (FdFlags & __WASI_FDFLAGS_DSYNC) {\n    SysFlag |= O_DSYNC;\n  }\n  if (FdFlags & __WASI_FDFLAGS_RSYNC) {\n    SysFlag |= O_RSYNC;\n  }\n  if (FdFlags & __WASI_FDFLAGS_SYNC) {\n    SysFlag |= O_SYNC;\n  }\n\n  if (auto Res = ::fcntl(Fd, F_SETFL, SysFlag); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFilestatGet(__wasi_filestat_t &Filestat) const noexcept {\n  if (auto Res = updateStat(); unlikely(!Res)) {\n    return WasiUnexpect(Res);\n  }\n\n  // Zeroing out these values to prevent leaking information about the host\n  // environment from special fd such as stdin, stdout and stderr.\n  Filestat.dev = isSpecialFd(Fd) ? 0 : Stat->st_dev;\n  Filestat.ino = isSpecialFd(Fd) ? 0 : Stat->st_ino;\n  Filestat.filetype = unsafeFiletype();\n  Filestat.nlink = isSpecialFd(Fd) ? 0 : Stat->st_nlink;\n  Filestat.size = isSpecialFd(Fd) ? 0 : Stat->st_size;\n  Filestat.atim = isSpecialFd(Fd) ? 0 : fromTimespec(Stat->st_atim);\n  Filestat.mtim = isSpecialFd(Fd) ? 0 : fromTimespec(Stat->st_mtim);\n  Filestat.ctim = isSpecialFd(Fd) ? 0 : fromTimespec(Stat->st_ctim);\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFilestatSetSize(__wasi_filesize_t Size) const noexcept {\n  if (auto Res = ::ftruncate(Fd, Size); unlikely(Res == -1)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void>\nINode::fdFilestatSetTimes(__wasi_timestamp_t ATim, __wasi_timestamp_t MTim,\n                          __wasi_fstflags_t FstFlags) const noexcept {\n#if __GLIBC_PREREQ(2, 6) || __BIONIC__\n  timespec SysTimespec[2];\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    SysTimespec[0] = toTimespec(ATim);\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    SysTimespec[0].tv_nsec = UTIME_NOW;\n  } else {\n    SysTimespec[0].tv_nsec = UTIME_OMIT;\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    SysTimespec[1] = toTimespec(MTim);\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    SysTimespec[1].tv_nsec = UTIME_NOW;\n  } else {\n    SysTimespec[1].tv_nsec = UTIME_OMIT;\n  }\n\n  if (auto Res = ::futimens(Fd, SysTimespec); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n#else\n  bool NeedNow = false;\n  bool NeedFile = false;\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    // Nothing to do.\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    NeedNow = true;\n  } else {\n    NeedFile = true;\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    // Nothing to do.\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    NeedNow = true;\n  } else {\n    NeedFile = true;\n  }\n\n  if (NeedFile) {\n    if (auto Res = updateStat(); unlikely(!Res)) {\n      return WasiUnexpect(Res);\n    }\n  }\n\n  timespec Now;\n  if (NeedNow) {\n    if (auto Res = ::clock_gettime(CLOCK_REALTIME, &Now); unlikely(Res != 0)) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n  }\n\n  timeval SysTimeval[2];\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    SysTimeval[0] = toTimeval(ATim);\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    SysTimeval[0] = toTimeval(Now);\n  } else {\n    SysTimeval[0] = toTimeval(Stat->st_atim);\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    SysTimeval[1] = toTimeval(MTim);\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    SysTimeval[1] = toTimeval(Now);\n  } else {\n    SysTimeval[1] = toTimeval(Stat->st_mtim);\n  }\n\n  if (auto Res = ::futimes(Fd, SysTimeval); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n#endif\n\n  return {};\n}\n\nWasiExpect<void> INode::fdPread(Span<Span<uint8_t>> IOVs,\n                                __wasi_filesize_t Offset,\n                                __wasi_size_t &NRead) const noexcept {\n  iovec SysIOVs[kIOVMax];\n  size_t SysIOVsSize = 0;\n  for (auto &IOV : IOVs) {\n    SysIOVs[SysIOVsSize].iov_base = IOV.data();\n    SysIOVs[SysIOVsSize].iov_len = IOV.size();\n    ++SysIOVsSize;\n  }\n\n#if __GLIBC_PREREQ(2, 10)\n  // Store read bytes length.\n  if (auto Res = ::preadv(Fd, SysIOVs, SysIOVsSize, Offset);\n      unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NRead = Res;\n  }\n#else\n  const auto OldOffset = ::lseek(Fd, 0, SEEK_CUR);\n  if (OldOffset < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (::lseek(Fd, Offset, SEEK_SET) < 0 ||\n      ::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (auto Res = ::readv(Fd, SysIOVs, SysIOVsSize); unlikely(Res < 0)) {\n    ::lseek(Fd, OldOffset, SEEK_SET);\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    if (::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n    NRead = Res;\n  }\n#endif\n\n  return {};\n}\n\nWasiExpect<void> INode::fdPwrite(Span<Span<const uint8_t>> IOVs,\n                                 __wasi_filesize_t Offset,\n                                 __wasi_size_t &NWritten) const noexcept {\n  iovec SysIOVs[kIOVMax];\n  size_t SysIOVsSize = 0;\n  for (auto &IOV : IOVs) {\n    SysIOVs[SysIOVsSize].iov_base = const_cast<uint8_t *>(IOV.data());\n    SysIOVs[SysIOVsSize].iov_len = IOV.size();\n    ++SysIOVsSize;\n  }\n\n#if __GLIBC_PREREQ(2, 10)\n  if (auto Res = ::pwritev(Fd, SysIOVs, SysIOVsSize, Offset);\n      unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NWritten = Res;\n  }\n#else\n  const auto OldOffset = ::lseek(Fd, 0, SEEK_CUR);\n  if (OldOffset < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (::lseek(Fd, Offset, SEEK_SET) < 0 ||\n      ::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n  if (auto Res = ::writev(Fd, SysIOVs, SysIOVsSize); unlikely(Res < 0)) {\n    ::lseek(Fd, OldOffset, SEEK_SET);\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    if (::lseek(Fd, OldOffset, SEEK_SET) < 0) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n    NWritten = Res;\n  }\n#endif\n\n  return {};\n}\n\nWasiExpect<void> INode::fdRead(Span<Span<uint8_t>> IOVs,\n                               __wasi_size_t &NRead) const noexcept {\n  iovec SysIOVs[kIOVMax];\n  size_t SysIOVsSize = 0;\n  for (auto &IOV : IOVs) {\n    SysIOVs[SysIOVsSize].iov_base = IOV.data();\n    SysIOVs[SysIOVsSize].iov_len = IOV.size();\n    ++SysIOVsSize;\n  }\n\n  if (auto Res = ::readv(Fd, SysIOVs, SysIOVsSize); unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NRead = Res;\n  }\n\n  return {};\n}\n\n// Due to the unfortunate design of wasi::fd_readdir, It's nearly impossible to\n// provide a correct implementation. The below implementation is just a\n// workaround for most usages and may not be correct in some edge cases. The\n// readdir entry API is going to be updated to use a stream type, so we don't\n// have to deal with it right now.\nWasiExpect<void> INode::fdReaddir(Span<uint8_t> Buffer,\n                                  __wasi_dircookie_t Cookie,\n                                  __wasi_size_t &Size) noexcept {\n  if (unlikely(!Dir.ok())) {\n    if (FdHolder NewFd(::dup(Fd)); unlikely(!NewFd.ok())) {\n      return WasiUnexpect(fromErrNo(errno));\n    } else if (DIR *D = ::fdopendir(NewFd.Fd); unlikely(!D)) {\n      return WasiUnexpect(fromErrNo(errno));\n    } else {\n      NewFd.release();\n      Dir.emplace(D);\n    }\n  }\n\n  if (Cookie == 0) {\n    ::rewinddir(Dir.Dir);\n  } else if (unlikely(Cookie != Dir.Cookie)) {\n    ::seekdir(Dir.Dir, Cookie);\n  }\n\n  Size = 0;\n  do {\n    if (!Dir.Buffer.empty()) {\n      const auto NewDataSize =\n          std::min<uint32_t>(Buffer.size(), Dir.Buffer.size());\n      std::copy(Dir.Buffer.begin(), Dir.Buffer.begin() + NewDataSize,\n                Buffer.begin());\n      Buffer = Buffer.subspan(NewDataSize);\n      Size += NewDataSize;\n      Dir.Buffer.clear();\n      if (unlikely(Buffer.empty())) {\n        break;\n      }\n    }\n    errno = 0;\n    dirent *SysDirent = ::readdir(Dir.Dir);\n    if (SysDirent == nullptr) {\n      if (errno != 0) {\n        return WasiUnexpect(fromErrNo(errno));\n      }\n      // End of entries\n      break;\n    }\n    Dir.Cookie = SysDirent->d_off;\n    std::string_view Name = SysDirent->d_name;\n\n    Dir.Buffer.resize(sizeof(__wasi_dirent_t) + Name.size());\n\n    __wasi_dirent_t *const Dirent =\n        reinterpret_cast<__wasi_dirent_t *>(Dir.Buffer.data());\n    Dirent->d_next = Dir.Cookie;\n    Dirent->d_ino = SysDirent->d_ino;\n    Dirent->d_type = fromFileType(SysDirent->d_type);\n    Dirent->d_namlen = Name.size();\n    std::copy(Name.cbegin(), Name.cend(),\n              Dir.Buffer.begin() + sizeof(__wasi_dirent_t));\n  } while (!Buffer.empty());\n\n  return {};\n}\n\nWasiExpect<void> INode::fdSeek(__wasi_filedelta_t Offset,\n                               __wasi_whence_t Whence,\n                               __wasi_filesize_t &Size) const noexcept {\n  if (auto Res = ::lseek(Fd, Offset, toWhence(Whence)); unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    Size = Res;\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdSync() const noexcept {\n  if (auto Res = ::fsync(Fd); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdTell(__wasi_filesize_t &Size) const noexcept {\n  if (auto Res = ::lseek(Fd, 0, SEEK_CUR); unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    Size = Res;\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::fdWrite(Span<Span<const uint8_t>> IOVs,\n                                __wasi_size_t &NWritten) const noexcept {\n  iovec SysIOVs[kIOVMax];\n  size_t SysIOVsSize = 0;\n  for (auto &IOV : IOVs) {\n    SysIOVs[SysIOVsSize].iov_base = const_cast<uint8_t *>(IOV.data());\n    SysIOVs[SysIOVsSize].iov_len = IOV.size();\n    ++SysIOVsSize;\n  }\n\n  if (auto Res = ::writev(Fd, SysIOVs, SysIOVsSize); unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NWritten = Res;\n  }\n\n  return {};\n}\n\nWasiExpect<uint64_t> INode::getNativeHandler() const noexcept {\n  return static_cast<uint64_t>(Fd);\n}\n\nWasiExpect<void> INode::pathCreateDirectory(std::string Path) const noexcept {\n  if (auto Res = ::mkdirat(Fd, Path.c_str(), 0755); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void>\nINode::pathFilestatGet(std::string Path,\n                       __wasi_filestat_t &Filestat) const noexcept {\n  struct stat SysFStat;\n  if (int Res = ::fstatat(Fd, Path.c_str(), &SysFStat, AT_SYMLINK_NOFOLLOW);\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  Filestat.dev = SysFStat.st_dev;\n  Filestat.ino = SysFStat.st_ino;\n  Filestat.filetype = fromFileType(static_cast<mode_t>(SysFStat.st_mode));\n  Filestat.nlink = SysFStat.st_nlink;\n  Filestat.size = SysFStat.st_size;\n  Filestat.atim = fromTimespec(SysFStat.st_atim);\n  Filestat.mtim = fromTimespec(SysFStat.st_mtim);\n  Filestat.ctim = fromTimespec(SysFStat.st_ctim);\n\n  return {};\n}\n\nWasiExpect<void>\nINode::pathFilestatSetTimes(std::string Path, __wasi_timestamp_t ATim,\n                            __wasi_timestamp_t MTim,\n                            __wasi_fstflags_t FstFlags) const noexcept {\n#if __GLIBC_PREREQ(2, 6) || __BIONIC__\n  timespec SysTimespec[2];\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    SysTimespec[0] = toTimespec(ATim);\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    SysTimespec[0].tv_nsec = UTIME_NOW;\n  } else {\n    SysTimespec[0].tv_nsec = UTIME_OMIT;\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    SysTimespec[1] = toTimespec(MTim);\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    SysTimespec[1].tv_nsec = UTIME_NOW;\n  } else {\n    SysTimespec[1].tv_nsec = UTIME_OMIT;\n  }\n\n  if (auto Res =\n          ::utimensat(Fd, Path.c_str(), SysTimespec, AT_SYMLINK_NOFOLLOW);\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n#else\n  bool NeedNow = false;\n  bool NeedFile = false;\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    // Nothing to do.\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    NeedNow = true;\n  } else {\n    NeedFile = true;\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    // Nothing to do.\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    NeedNow = true;\n  } else {\n    NeedFile = true;\n  }\n\n#ifdef O_PATH\n  const int OFlags = O_PATH | O_SYMLINK;\n#else\n  const int OFlags = O_RDONLY | O_SYMLINK;\n#endif\n\n  FdHolder Target(::openat(Fd, Path.c_str(), OFlags));\n  if (unlikely(!Target.ok())) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  struct stat SysStat;\n  if (NeedFile) {\n    if (auto Res = ::fstat(Target.Fd, &SysStat); unlikely(Res != 0)) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n  }\n\n  timespec Now;\n  if (NeedNow) {\n    if (auto Res = ::clock_gettime(CLOCK_REALTIME, &Now); unlikely(Res != 0)) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n  }\n\n  timeval SysTimeval[2];\n  if (FstFlags & __WASI_FSTFLAGS_ATIM) {\n    SysTimeval[0] = toTimeval(ATim);\n  } else if (FstFlags & __WASI_FSTFLAGS_ATIM_NOW) {\n    SysTimeval[0] = toTimeval(Now);\n  } else {\n    SysTimeval[0] = toTimeval(SysStat.st_atim);\n  }\n  if (FstFlags & __WASI_FSTFLAGS_MTIM) {\n    SysTimeval[1] = toTimeval(MTim);\n  } else if (FstFlags & __WASI_FSTFLAGS_MTIM_NOW) {\n    SysTimeval[1] = toTimeval(Now);\n  } else {\n    SysTimeval[1] = toTimeval(SysStat.st_mtim);\n  }\n\n  if (auto Res = ::futimes(Target.Fd, SysTimeval); unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n#endif\n\n  return {};\n}\n\nWasiExpect<void> INode::pathLink(const INode &Old, std::string OldPath,\n                                 const INode &New,\n                                 std::string NewPath) noexcept {\n  if (auto Res = ::linkat(Old.Fd, OldPath.c_str(), New.Fd, NewPath.c_str(), 0);\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<INode> INode::pathOpen(std::string Path, __wasi_oflags_t OpenFlags,\n                                  __wasi_fdflags_t FdFlags,\n                                  VFS::Flags VFSFlags) const noexcept {\n  const int Flags = openFlags(OpenFlags, FdFlags, VFSFlags);\n\n  if (auto NewFd = ::openat(Fd, Path.c_str(), Flags, 0644);\n      unlikely(NewFd < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    INode New(NewFd);\n#ifndef O_CLOEXEC\n    if (auto Res = ::fcntl(New.Fd, F_SETFD, FD_CLOEXEC); unlikely(Res != 0)) {\n      return WasiUnexpect(fromErrNo(errno));\n    }\n#endif\n    return New;\n  }\n}\n\nWasiExpect<void> INode::pathReadlink(std::string Path, Span<char> Buffer,\n                                     __wasi_size_t &NRead) const noexcept {\n  if (auto Res = ::readlinkat(Fd, Path.c_str(), Buffer.data(), Buffer.size());\n      unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    NRead = Res;\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::pathRemoveDirectory(std::string Path) const noexcept {\n  if (auto Res = ::unlinkat(Fd, Path.c_str(), AT_REMOVEDIR);\n      unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::pathRename(const INode &Old, std::string OldPath,\n                                   const INode &New,\n                                   std::string NewPath) noexcept {\n  if (auto Res = ::renameat(Old.Fd, OldPath.c_str(), New.Fd, NewPath.c_str());\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::pathSymlink(std::string OldPath,\n                                    std::string NewPath) const noexcept {\n  if (auto Res = ::symlinkat(OldPath.c_str(), Fd, NewPath.c_str());\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::pathUnlinkFile(std::string Path) const noexcept {\n  if (auto Res = ::unlinkat(Fd, Path.c_str(), 0); unlikely(Res < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  return {};\n}\n\nWasiExpect<void> INode::getAddrinfo(std::string_view Node,\n                                    std::string_view Service,\n                                    const __wasi_addrinfo_t &Hint,\n                                    uint32_t MaxResLength,\n                                    Span<__wasi_addrinfo_t *> WasiAddrinfoArray,\n                                    Span<__wasi_sockaddr_t *> WasiSockaddrArray,\n                                    Span<char *> AiAddrSaDataArray,\n                                    Span<char *> AiCanonnameArray,\n                                    /*Out*/ __wasi_size_t &ResLength) noexcept {\n  const auto [NodeCStr, NodeBuf] = createNullTerminatedString(Node);\n  const auto [ServiceCStr, ServiceBuf] = createNullTerminatedString(Service);\n\n  struct addrinfo SysHint;\n  SysHint.ai_flags = toAIFlags(Hint.ai_flags);\n  SysHint.ai_family = toAddressFamily(Hint.ai_family);\n  SysHint.ai_socktype = toSockType(Hint.ai_socktype);\n  SysHint.ai_protocol = toProtocol(Hint.ai_protocol);\n  SysHint.ai_addrlen = Hint.ai_addrlen;\n  SysHint.ai_addr = nullptr;\n  SysHint.ai_canonname = nullptr;\n  SysHint.ai_next = nullptr;\n\n  struct addrinfo *SysResPtr = nullptr;\n  if (auto Res = ::getaddrinfo(NodeCStr, ServiceCStr, &SysHint, &SysResPtr);\n      unlikely(Res < 0)) {\n    return WasiUnexpect(fromEAIErrNo(Res));\n  }\n  // calculate ResLength\n  if (ResLength = calculateAddrinfoLinkedListSize(SysResPtr);\n      ResLength > MaxResLength) {\n    ResLength = MaxResLength;\n  }\n\n  struct addrinfo *SysResItem = SysResPtr;\n  for (uint32_t Idx = 0; Idx < ResLength; Idx++) {\n    auto &CurAddrinfo = WasiAddrinfoArray[Idx];\n    CurAddrinfo->ai_flags = fromAIFlags(SysResItem->ai_flags);\n    CurAddrinfo->ai_socktype = fromSockType(SysResItem->ai_socktype);\n    CurAddrinfo->ai_protocol = fromProtocol(SysResItem->ai_protocol);\n    CurAddrinfo->ai_family = fromAddressFamily(SysResItem->ai_family);\n    CurAddrinfo->ai_addrlen = SysResItem->ai_addrlen;\n\n    // process ai_canonname in addrinfo\n    if (SysResItem->ai_canonname != nullptr) {\n      CurAddrinfo->ai_canonname_len = std::strlen(SysResItem->ai_canonname);\n      auto &CurAiCanonname = AiCanonnameArray[Idx];\n      std::memcpy(CurAiCanonname, SysResItem->ai_canonname,\n                  CurAddrinfo->ai_canonname_len + 1);\n    } else {\n      CurAddrinfo->ai_canonname_len = 0;\n    }\n\n    // process socket address\n    if (SysResItem->ai_addrlen > 0) {\n      auto &CurSockaddr = WasiSockaddrArray[Idx];\n      CurSockaddr->sa_family =\n          fromAddressFamily(SysResItem->ai_addr->sa_family);\n\n      // process sa_data in socket address\n      size_t SaSize = 0;\n      switch (CurSockaddr->sa_family) {\n      case __WASI_ADDRESS_FAMILY_INET4:\n        SaSize = sizeof(sockaddr_in) - offsetof(sockaddr_in, sin_port);\n        break;\n      case __WASI_ADDRESS_FAMILY_INET6:\n        SaSize = sizeof(sockaddr_in6) - offsetof(sockaddr_in6, sin6_port);\n        break;\n      default:\n        assumingUnreachable();\n      }\n      std::memcpy(AiAddrSaDataArray[Idx], SysResItem->ai_addr->sa_data, SaSize);\n      CurSockaddr->sa_data_len = __wasi_size_t(SaSize);\n    }\n    // process ai_next in addrinfo\n    SysResItem = SysResItem->ai_next;\n  }\n  ::freeaddrinfo(SysResPtr);\n\n  return {};\n}\n\nWasiExpect<INode> INode::sockOpen(__wasi_address_family_t AddressFamily,\n                                  __wasi_sock_type_t SockType) noexcept {\n  int SysProtocol = IPPROTO_IP;\n  int SysDomain = 0;\n  int SysType = 0;\n\n  switch (AddressFamily) {\n  case __WASI_ADDRESS_FAMILY_INET4:\n    SysDomain = AF_INET;\n    break;\n  case __WASI_ADDRESS_FAMILY_INET6:\n    SysDomain = AF_INET6;\n    break;\n  case __WASI_ADDRESS_FAMILY_AF_UNIX:\n    SysDomain = AF_UNIX;\n    break;\n  default:\n    return WasiUnexpect(__WASI_ERRNO_INVAL);\n  }\n\n  switch (SockType) {\n  case __WASI_SOCK_TYPE_SOCK_DGRAM:\n    SysType = SOCK_DGRAM;\n    break;\n  case __WASI_SOCK_TYPE_SOCK_STREAM:\n    SysType = SOCK_STREAM;\n    break;\n  default:\n    return WasiUnexpect(__WASI_ERRNO_INVAL);\n  }\n\n  if (auto NewFd = ::socket(SysDomain, SysType, SysProtocol);\n      unlikely(NewFd < 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  } else {\n    INode New(NewFd);\n    return New;\n  }\n}\n\nstruct SockEmptyAddr {};\nusing VarAddrT = std::variant<SockEmptyAddr, sockaddr_storage, sockaddr,\n                              sockaddr_in, sockaddr_in6, sockaddr_un>;\n\nstruct VarAddrBuf {\n  template <typename T> sockaddr *operator()(T &V) {\n    return reinterpret_cast<struct sockaddr *>(&V);\n  }\n  sockaddr *operator()(SockEmptyAddr &) { return nullptr; }\n};\n\nstruct VarAddrSize {\n  template <typename T> int operator()(const T &) { return sizeof(T); }\n  int operator()(const SockEmptyAddr &) { return 0; }\n};\n\nstatic VarAddrT sockAddressAssignHelper(__wasi_address_family_t AddrFamily,\n                                        const Span<const uint8_t> &Address,\n                                        uint16_t Port) {\n  VarAddrT Addr;\n  if (Address.size() == 0) {\n    Addr.emplace<SockEmptyAddr>();\n  } else if (AddrFamily == __WASI_ADDRESS_FAMILY_INET4) {\n    auto &ServerAddr4 = Addr.emplace<sockaddr_in>();\n\n    ServerAddr4.sin_family = AF_INET;\n    ServerAddr4.sin_port = htons(Port);\n    assuming(Address.size() >= sizeof(in_addr));\n    std::memcpy(&ServerAddr4.sin_addr, Address.data(), sizeof(in_addr));\n  } else if (AddrFamily == __WASI_ADDRESS_FAMILY_INET6) {\n    auto &ServerAddr6 = Addr.emplace<sockaddr_in6>();\n\n    ServerAddr6.sin6_family = AF_INET6;\n    ServerAddr6.sin6_port = htons(Port);\n    ServerAddr6.sin6_flowinfo = 0;\n    assuming(Address.size() >= sizeof(in6_addr));\n    std::memcpy(&ServerAddr6.sin6_addr, Address.data(), sizeof(in6_addr));\n  } else if (AddrFamily == __WASI_ADDRESS_FAMILY_AF_UNIX) {\n...\n// Path: lib/host/wasi/clock-macos.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/defines.h\"\n#if WASMEDGE_OS_MACOS\n\n#include \"host/wasi/clock.h\"\n#include \"macos.h\"\n\nnamespace WasmEdge {\nnamespace Host {\nnamespace WASI {\n\nWasiExpect<void> Clock::clockResGet(__wasi_clockid_t Id,\n                                    __wasi_timestamp_t &Resolution) noexcept {\n  timespec SysTimespec;\n  if (auto Res = ::clock_getres(toClockId(Id), &SysTimespec);\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  Resolution = fromTimespec(SysTimespec);\n  return {};\n}\n\nWasiExpect<void> Clock::clockTimeGet(__wasi_clockid_t Id, __wasi_timestamp_t,\n                                     __wasi_timestamp_t &Time) noexcept {\n  timespec SysTimespec;\n  if (auto Res = ::clock_gettime(toClockId(Id), &SysTimespec);\n      unlikely(Res != 0)) {\n    return WasiUnexpect(fromErrNo(errno));\n  }\n\n  Time = fromTimespec(SysTimespec);\n  return {};\n}\n\n} // namespace WASI\n} // namespace Host\n} // namespace WasmEdge\n\n#endif\n\n// Path: lib/host/wasi/win.h\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"common/defines.h\"\n#if !WASMEDGE_OS_WINDOWS\n#error\n#endif\n\n#include \"common/errcode.h\"\n#include \"system/winapi.h\"\n#include \"wasi/api.hpp\"\n#include <cerrno>\n#include <chrono>\n#include <fcntl.h>\n#include <io.h>\n#include <sys/stat.h>\n#include <sys/types.h>\n\nnamespace WasmEdge {\nnamespace Host {\nnamespace WASI {\ninline namespace detail {\nusing namespace winapi;\n\ninline constexpr __wasi_errno_t fromErrNo(int ErrNo) noexcept {\n  switch (ErrNo) {\n  case 0:\n    return __WASI_ERRNO_SUCCESS;\n  case E2BIG:\n    return __WASI_ERRNO_2BIG;\n  case EACCES:\n    return __WASI_ERRNO_ACCES;\n  case EADDRINUSE:\n    return __WASI_ERRNO_ADDRINUSE;\n  case EADDRNOTAVAIL:\n    return __WASI_ERRNO_ADDRNOTAVAIL;\n  case EAFNOSUPPORT:\n    return __WASI_ERRNO_AFNOSUPPORT;\n  case EAGAIN:\n    return __WASI_ERRNO_AGAIN;\n  case EALREADY:\n    return __WASI_ERRNO_ALREADY;\n  case EBADF:\n    return __WASI_ERRNO_BADF;\n  case EBADMSG:\n    return __WASI_ERRNO_BADMSG;\n  case EBUSY:\n    return __WASI_ERRNO_BUSY;\n  case ECANCELED:\n    return __WASI_ERRNO_CANCELED;\n  case ECHILD:\n    return __WASI_ERRNO_CHILD;\n  case ECONNABORTED:\n    return __WASI_ERRNO_CONNABORTED;\n  case ECONNREFUSED:\n    return __WASI_ERRNO_CONNREFUSED;\n  case ECONNRESET:\n    return __WASI_ERRNO_CONNRESET;\n  case EDEADLK:\n    return __WASI_ERRNO_DEADLK;\n  case EDESTADDRREQ:\n    return __WASI_ERRNO_DESTADDRREQ;\n  case EDOM:\n    return __WASI_ERRNO_DOM;\n  case EEXIST:\n    return __WASI_ERRNO_EXIST;\n  case EFAULT:\n    return __WASI_ERRNO_FAULT;\n  case EFBIG:\n    return __WASI_ERRNO_FBIG;\n  case EHOSTUNREACH:\n    return __WASI_ERRNO_HOSTUNREACH;\n  case EIDRM:\n    return __WASI_ERRNO_IDRM;\n  case EILSEQ:\n    return __WASI_ERRNO_ILSEQ;\n  case EINPROGRESS:\n    return __WASI_ERRNO_INPROGRESS;\n  case EINTR:\n    return __WASI_ERRNO_INTR;\n  case EINVAL:\n    return __WASI_ERRNO_INVAL;\n  case EIO:\n    return __WASI_ERRNO_IO;\n  case EISCONN:\n    return __WASI_ERRNO_ISCONN;\n  case EISDIR:\n    return __WASI_ERRNO_ISDIR;\n  case ELOOP:\n    return __WASI_ERRNO_LOOP;\n  case EMFILE:\n    return __WASI_ERRNO_MFILE;\n  case EMLINK:\n    return __WASI_ERRNO_MLINK;\n  case EMSGSIZE:\n    return __WASI_ERRNO_MSGSIZE;\n  case ENAMETOOLONG:\n    return __WASI_ERRNO_NAMETOOLONG;\n  case ENETDOWN:\n    return __WASI_ERRNO_NETDOWN;\n  case ENETRESET:\n    return __WASI_ERRNO_NETRESET;\n  case ENETUNREACH:\n    return __WASI_ERRNO_NETUNREACH;\n  case ENFILE:\n    return __WASI_ERRNO_NFILE;\n  case ENOBUFS:\n    return __WASI_ERRNO_NOBUFS;\n  case ENODEV:\n    return __WASI_ERRNO_NODEV;\n  case ENOENT:\n    return __WASI_ERRNO_NOENT;\n  case ENOEXEC:\n    return __WASI_ERRNO_NOEXEC;\n  case ENOLCK:\n    return __WASI_ERRNO_NOLCK;\n  case ENOLINK:\n    return __WASI_ERRNO_NOLINK;\n  case ENOMEM:\n    return __WASI_ERRNO_NOMEM;\n  case ENOMSG:\n    return __WASI_ERRNO_NOMSG;\n  case ENOPROTOOPT:\n    return __WASI_ERRNO_NOPROTOOPT;\n  case ENOSPC:\n    return __WASI_ERRNO_NOSPC;\n  case ENOSYS:\n    return __WASI_ERRNO_NOSYS;\n  case ENOTCONN:\n    return __WASI_ERRNO_NOTCONN;\n  case ENOTDIR:\n    return __WASI_ERRNO_NOTDIR;\n  case ENOTEMPTY:\n    return __WASI_ERRNO_NOTEMPTY;\n  case ENOTRECOVERABLE:\n    return __WASI_ERRNO_NOTRECOVERABLE;\n  case ENOTSOCK:\n    return __WASI_ERRNO_NOTSOCK;\n  case ENOTSUP:\n    return __WASI_ERRNO_NOTSUP;\n  case ENOTTY:\n    return __WASI_ERRNO_NOTTY;\n  case ENXIO:\n    return __WASI_ERRNO_NXIO;\n  case EOVERFLOW:\n    return __WASI_ERRNO_OVERFLOW;\n  case EOWNERDEAD:\n    return __WASI_ERRNO_OWNERDEAD;\n  case EPERM:\n    return __WASI_ERRNO_PERM;\n  case EPIPE:\n    return __WASI_ERRNO_PIPE;\n  case EPROTO:\n    return __WASI_ERRNO_PROTO;\n  case EPROTONOSUPPORT:\n    return __WASI_ERRNO_PROTONOSUPPORT;\n  case EPROTOTYPE:\n    return __WASI_ERRNO_PROTOTYPE;\n  case ERANGE:\n    return __WASI_ERRNO_RANGE;\n  case EROFS:\n    return __WASI_ERRNO_ROFS;\n  case ESPIPE:\n    return __WASI_ERRNO_SPIPE;\n  case ESRCH:\n    return __WASI_ERRNO_SRCH;\n  case ETIMEDOUT:\n    return __WASI_ERRNO_TIMEDOUT;\n  case ETXTBSY:\n    return __WASI_ERRNO_TXTBSY;\n  case EXDEV:\n    return __WASI_ERRNO_XDEV;\n  default:\n    assumingUnreachable();\n  }\n}\n\n\ninline __wasi_errno_t fromLastError(DWORD_ Code) noexcept {\n  switch (Code) {\n  case ERROR_INVALID_PARAMETER_: // MultiByteToWideChar\n  case ERROR_INVALID_HANDLE_:    // GetFinalPathNameByHandleW\n    return __WASI_ERRNO_INVAL;\n  case ERROR_SHARING_VIOLATION_: // CreateFile2\n  case ERROR_PIPE_BUSY_:         // CreateFile2\n    return __WASI_ERRNO_BUSY;\n  case ERROR_ACCESS_DENIED_: // CreateFile2\n    return __WASI_ERRNO_ACCES;\n  case ERROR_ALREADY_EXISTS_: // CreateFile2\n  case ERROR_FILE_EXISTS_:    // CreateFile2\n    return __WASI_ERRNO_EXIST;\n  case ERROR_FILE_NOT_FOUND_: // CreateFile2\n    return __WASI_ERRNO_NOENT;\n  case ERROR_PRIVILEGE_NOT_HELD_: // CreateSymbolicLinkW\n    return __WASI_ERRNO_PERM;\n\n  case ERROR_IO_PENDING_:             // ReadFileEx\n  case ERROR_HANDLE_EOF_:             // ReadFileEx\n  case ERROR_INSUFFICIENT_BUFFER_:    // MultiByteToWideChar\n  case ERROR_INVALID_FLAGS_:          // MultiByteToWideChar\n  case ERROR_NO_UNICODE_TRANSLATION_: // MultiByteToWideChar\n  default:\n    return __WASI_ERRNO_NOSYS;\n  }\n}\n\nusing FiletimeDuration = std::chrono::duration<\n    uint64_t,\n    std::ratio_multiply<std::ratio<100, 1>, std::chrono::nanoseconds::period>>;\n/// from 1601-01-01 to 1970-01-01, 134774 days\nstatic inline constexpr const FiletimeDuration NTToUnixEpoch =\n    std::chrono::seconds{134774u * 86400u};\n\nstatic constexpr __wasi_timestamp_t fromFiletime(FILETIME_ FileTime) noexcept {\n  using std::chrono::duration_cast;\n  using std::chrono::nanoseconds;\n  ULARGE_INTEGER_ Temp = {/* LowPart */ FileTime.dwLowDateTime,\n                          /* HighPart */ FileTime.dwHighDateTime};\n  auto Duration = duration_cast<nanoseconds>(FiletimeDuration{Temp.QuadPart} -\n                                             NTToUnixEpoch);\n  return static_cast<__wasi_timestamp_t>(Duration.count());\n}\n\nstatic constexpr FILETIME_ toFiletime(__wasi_timestamp_t TimeStamp) noexcept {\n  using std::chrono::duration_cast;\n  using std::chrono::nanoseconds;\n  auto Duration =\n      duration_cast<FiletimeDuration>(nanoseconds{TimeStamp}) + NTToUnixEpoch;\n  ULARGE_INTEGER_ Temp = ULARGE_INTEGER_(Duration.count());\n  return FILETIME_{/* dwLowDateTime */ Temp.LowPart,\n                   /* dwHighDateTime */ Temp.HighPart};\n}\n\ninline __wasi_errno_t fromWSALastError() noexcept {\n  switch (WSAGetLastError()) {\n  case WSASYSNOTREADY_: // WSAStartup\n  case WSAEWOULDBLOCK_: // closesocket\n    return __WASI_ERRNO_AGAIN;\n  case WSAVERNOTSUPPORTED_: // WSAStartup\n    return __WASI_ERRNO_NOTSUP;\n  case WSAEINPROGRESS_: // WSAStartup, socket, closesocket\n    return __WASI_ERRNO_INPROGRESS;\n  case WSAEPROCLIM_: // WSAStartup\n    return __WASI_ERRNO_BUSY;\n  case WSAEFAULT_: // WSAStartup\n    return __WASI_ERRNO_FAULT;\n  case WSAENETDOWN_: // socket, closesocket\n    return __WASI_ERRNO_NETDOWN;\n  case WSAENOTSOCK_: // closesocket\n    return __WASI_ERRNO_NOTSOCK;\n  case WSAEINTR_: // closesocket\n    return __WASI_ERRNO_INTR;\n  case WSAEAFNOSUPPORT_: // socket\n    return __WASI_ERRNO_AIFAMILY;\n  case WSAEMFILE_: // socket\n    return __WASI_ERRNO_NFILE;\n  case WSAEINVAL_: // socket\n    return __WASI_ERRNO_INVAL;\n  case WSAENOBUFS_: // socket\n    return __WASI_ERRNO_NOBUFS;\n  case WSAEPROTONOSUPPORT_: // socket\n    return __WASI_ERRNO_PROTONOSUPPORT;\n  case WSAEPROTOTYPE_: // socket\n    return __WASI_ERRNO_PROTOTYPE;\n  case WSAESOCKTNOSUPPORT_: // socket\n    return __WASI_ERRNO_AISOCKTYPE;\n  case WSAEINVALIDPROCTABLE_:   // socket\n  case WSAEINVALIDPROVIDER_:    // socket\n  case WSAEPROVIDERFAILEDINIT_: // socket\n  case WSANOTINITIALISED_:      // socket, closesocket\n  default:\n    return __WASI_ERRNO_NOSYS;\n  }\n}\n\ninline constexpr __wasi_errno_t fromWSAError(int WSAError) noexcept {\n  switch (WSAError) {\n  case WSATRY_AGAIN_:\n    return __WASI_ERRNO_AIAGAIN;\n  case WSAEINVAL_:\n    return __WASI_ERRNO_AIBADFLAG;\n  case WSANO_RECOVERY_:\n    return __WASI_ERRNO_AIFAIL;\n  case WSAEAFNOSUPPORT_:\n    return __WASI_ERRNO_AIFAMILY;\n  case ERROR_NOT_ENOUGH_MEMORY_:\n    return __WASI_ERRNO_AIMEMORY;\n  case WSAHOST_NOT_FOUND_:\n    return __WASI_ERRNO_AINONAME;\n  case WSATYPE_NOT_FOUND_:\n    return __WASI_ERRNO_AISERVICE;\n  case WSAESOCKTNOSUPPORT_:\n    return __WASI_ERRNO_AISOCKTYPE;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline WasiExpect<void> ensureWSAStartup() noexcept {\n  static std::once_flag InitFlag;\n  try {\n    std::call_once(InitFlag, []() {\n      WSADATA_ WSAData;\n      if (unlikely(WSAStartup(0x0202, &WSAData) != 0)) {\n        throw detail::fromWSALastError();\n      }\n      if (unlikely(WSAData.wVersion != 0x0202)) {\n        throw __WASI_ERRNO_NOSYS;\n      }\n    });\n    return {};\n  } catch (__wasi_errno_t &Error) {\n    return WasiUnexpect(Error);\n  }\n}\n\ninline constexpr DWORD_ toWhence(__wasi_whence_t Whence) noexcept {\n  switch (Whence) {\n  case __WASI_WHENCE_SET:\n    return FILE_BEGIN_;\n  case __WASI_WHENCE_END:\n    return FILE_END_;\n  case __WASI_WHENCE_CUR:\n    return FILE_CURRENT_;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr int toSockOptLevel(__wasi_sock_opt_level_t Level) noexcept {\n  switch (Level) {\n  case __WASI_SOCK_OPT_LEVEL_SOL_SOCKET:\n    return SOL_SOCKET;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr int toSockOptSoName(__wasi_sock_opt_so_t SoName) noexcept {\n  switch (SoName) {\n  case __WASI_SOCK_OPT_SO_REUSEADDR:\n    return SO_REUSEADDR;\n  case __WASI_SOCK_OPT_SO_TYPE:\n    return SO_TYPE;\n  case __WASI_SOCK_OPT_SO_ERROR:\n    return SO_ERROR;\n  case __WASI_SOCK_OPT_SO_DONTROUTE:\n    return SO_DONTROUTE;\n  case __WASI_SOCK_OPT_SO_BROADCAST:\n    return SO_BROADCAST;\n  case __WASI_SOCK_OPT_SO_SNDBUF:\n    return SO_SNDBUF;\n  case __WASI_SOCK_OPT_SO_RCVBUF:\n    return SO_RCVBUF;\n  case __WASI_SOCK_OPT_SO_KEEPALIVE:\n    return SO_KEEPALIVE;\n  case __WASI_SOCK_OPT_SO_OOBINLINE:\n    return SO_OOBINLINE;\n  case __WASI_SOCK_OPT_SO_LINGER:\n    return SO_LINGER;\n  case __WASI_SOCK_OPT_SO_RCVLOWAT:\n    return SO_RCVLOWAT;\n  case __WASI_SOCK_OPT_SO_RCVTIMEO:\n    return SO_RCVTIMEO;\n  case __WASI_SOCK_OPT_SO_SNDTIMEO:\n    return SO_SNDTIMEO;\n  case __WASI_SOCK_OPT_SO_ACCEPTCONN:\n    return SO_ACCEPTCONN;\n  default:\n    assumingUnreachable();\n  }\n}\n\ninline constexpr __wasi_aiflags_t fromAIFlags(int AIFlags) noexcept {\n  __wasi_aiflags_t Result = static_cast<__wasi_aiflags_t>(0);\n\n...\n", "needle_token_start": 13569, "needle_token_end": 13978, "code_context_ntokens": 16213, "output": ["```cpp\ninline constexpr __wasi_errno_t fromErrNo(int ErrNo) noexcept {\n  switch (ErrNo) {\n  case 0:\n    return __WASI_ERRNO_SUCCESS;\n  case E2BIG:\n    return __WASI_ERRNO_2BIG;\n  case EACCES:\n    return __WASI_ERRNO_ACCES;\n  case EADDRINUSE:\n    return __WASI_ERRNO_ADDRINUSE;\n  case EADDRNOTAVAIL:\n    return __WASI_ERRNO_ADDRNOTAVAIL;\n  case EAFNOSUPPORT:\n    return __WASI_ERRNO_AFNOSUPPORT;\n  case EAGAIN:\n    return __WASI_ERRNO_AGAIN;\n  case EALREADY:\n    return __WASI_ERRNO_ALREADY;\n  case EBADF:\n    return __WASI_ERRNO_BADF;\n  case EBADMSG:\n    return __WASI_ERRNO_BADMSG;\n  case EBUSY:\n    return __WASI_ERRNO_BUSY;\n  case ECANCELED:\n    return __WASI_ERRNO_CANCELED;\n  case ECHILD:\n    return __WASI_ERRNO_CHILD;\n  case ECONNABORTED:\n    return __WASI_ERRNO_CONNABORTED;\n  case ECONNREFUSED:\n    return __WASI_ERRNO_CONNREFUSED;\n  case ECONNRESET:\n    return __WASI_ERRNO_CONNRESET;\n  case EDEADLK:\n    return __WASI_ERRNO_DEADLK;\n  case EDESTADDRREQ:\n    return __WASI_ERRNO_DESTADDRREQ;\n  case EDOM:\n    return __WASI_ERRNO_DOM;\n  case EEXIST:\n    return __WASI_ERRNO_EXIST;\n  case EFAULT:\n    return __WASI_ERRNO_FAULT;\n  case EFBIG:\n    return __WASI_ERRNO_FBIG;\n  case EHOSTUNREACH:\n    return __WASI_ERRNO_HOSTUNREACH;\n  case EIDRM:\n    return __WASI_ERRNO_IDRM;\n  case EILSEQ:\n    return __WASI_ERRNO_ILSEQ;\n  case EINPROGRESS:\n    return __WASI_ERRNO_INPROGRESS;\n  case EINTR:\n    return __WASI_ERRNO_INTR;\n  case EINVAL:\n    return __WASI_ERRNO_INVAL;\n  case EIO:\n    return __WASI_ERRNO_IO;\n  case EISCONN:\n    return __WASI_ERRNO_ISCONN;\n  case EISDIR:\n    return __WASI_ERRNO_ISDIR;\n  case ELOOP:\n    return __WASI_ERRNO_LOOP;\n  case EMFILE:\n    return __WASI_ERRNO_MFILE;\n  case EMLINK:\n    return __WASI_ERRNO_MLINK;\n  case EMSGSIZE:\n    return __WASI_ERRNO_MSGSIZE;\n  case ENAMETOOLONG:\n    return __WASI_ERRNO_NAMETOOLONG;\n  case ENETDOWN:\n    return __WASI_ERRNO_NETDOWN;\n  case ENETRESET:\n    return __WASI_ERRNO_NETRESET;\n  case ENETUNREACH:\n    return __WASI_ERRNO_NETUNREACH;\n  case ENFILE:\n    return __WASI_ERRNO_NFILE;\n  case ENOBUFS:\n    return __WASI_ERRNO_NOBUFS;\n  case ENODEV:\n    return __WASI_ERRNO_NODEV;\n  case ENOENT:\n    return __WASI_ERRNO_NOENT;\n  case ENOEXEC:\n    return __WASI_ERRNO_NOEXEC;\n  case ENOLCK:\n    return __WASI_ERRNO_NOLCK;\n  case ENOLINK:\n    return __WASI_ERRNO_NOLINK;\n  case ENOMEM:\n    return __WASI_ERRNO_NOMEM;\n  case ENOMSG:\n    return __WASI_ERRNO_NOMSG;\n  case ENOPROTOOPT:\n    return __WASI_ERRNO_NOPROTOOPT;\n  case ENOSPC:\n    return __WASI_ERRNO_NOSPC;\n  case ENOSYS:\n    return __WASI_ERRNO_NOSYS;\n  case ENOTCONN:\n    return __WASI_ERRNO_NOTCONN;\n  case ENOTDIR:\n    return __WASI_ERRNO_NOTDIR;\n  case ENOTEMPTY:\n    return __WASI_ERRNO_NOTEMPTY;\n  case ENOTRECOVERABLE:\n    return __WASI_ERRNO_NOTRECOVERABLE;\n  case ENOTSOCK:\n    return __WASI_ERRNO_NOTSOCK;\n  case ENOTSUP:\n    return __WASI_ERRNO_NOTSUP;\n  case ENOTTY:\n    return __WASI_ERRNO_NOTTY;\n  case ENXIO:\n    return __WASI_ERRNO_NXIO;\n  case EOVERFLOW:\n    return __WASI_ERRNO_OVERFLOW;\n  case EOWNERDEAD:\n    return __WASI_ERRNO_OWNER"]}
{"repo": "WasmEdge/WasmEdge", "name": "roundDownPageBoundary", "language": "cpp", "path": "lib/loader/aot_section.cpp", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To adjust a given numerical value downwards to the nearest boundary of a memory page size, which varies depending on the operating system and architecture.\n2. **Input**: A 64-bit unsigned integer representing an address or size in memory.\n3. **Output**: A 64-bit unsigned integer that has been adjusted down to the nearest page boundary.\n4. **Procedure**: The function checks the operating system and architecture to determine the appropriate page size (standard or specific to ARM64 on macOS). It then uses a bitwise AND operation with the complement of the page size minus one to clear the lower bits of the input value, effectively rounding it down to the nearest multiple of the page size.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/executor/engine/engine.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\n#include <array>\n#include <cstdint>\n#include <cstring>\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void> Executor::runExpression(Runtime::StackManager &StackMgr,\n                                     AST::InstrView Instrs) {\n  return execute(StackMgr, Instrs.begin(), Instrs.end());\n}\n\nExpect<void>\nExecutor::runFunction(Runtime::StackManager &StackMgr,\n                      const Runtime::Instance::FunctionInstance &Func,\n                      Span<const ValVariant> Params) {\n  // Set start time.\n  if (Stat && Conf.getStatisticsConfigure().isTimeMeasuring()) {\n    Stat->startRecordWasm();\n  }\n\n  // Reset and push a dummy frame into stack.\n  StackMgr.pushFrame(nullptr, AST::InstrView::iterator(), 0, 0);\n\n  // Push arguments.\n  const auto &PTypes = Func.getFuncType().getParamTypes();\n  for (uint32_t I = 0; I < Params.size(); I++) {\n    // For the references, transform to non-null reference type if the value not\n    // null.\n    if (PTypes[I].isRefType() && Params[I].get<RefVariant>().getPtr<void>() &&\n        Params[I].get<RefVariant>().getType().isNullableRefType()) {\n      auto Val = Params[I];\n      Val.get<RefVariant>().getType().toNonNullableRef();\n      StackMgr.push(Val);\n    } else {\n      StackMgr.push(Params[I]);\n    }\n  }\n\n  // Enter and execute function.\n  AST::InstrView::iterator StartIt = {};\n  Expect<void> Res = {};\n  if (auto GetIt = enterFunction(StackMgr, Func, Func.getInstrs().end())) {\n    StartIt = *GetIt;\n  } else {\n    if (GetIt.error() == ErrCode::Value::Terminated) {\n      // Handle the terminated case in entering AOT or host functions.\n      // For the terminated case, not return now to print the statistics.\n      Res = Unexpect(GetIt.error());\n    } else {\n      return Unexpect(GetIt);\n    }\n  }\n  if (Res) {\n    // If not terminated, execute the instructions in interpreter mode.\n    // For the entering AOT or host functions, the `StartIt` is equal to the end\n    // of instruction list, therefore the execution will return immediately.\n    Res = execute(StackMgr, StartIt, Func.getInstrs().end());\n  }\n\n  if (Res) {\n    spdlog::debug(\" Execution succeeded.\");\n  } else if (Res.error() == ErrCode::Value::Terminated) {\n    spdlog::debug(\" Terminated.\");\n  }\n\n  if (Stat && Conf.getStatisticsConfigure().isTimeMeasuring()) {\n    Stat->stopRecordWasm();\n  }\n\n  // If Statistics is enabled, then dump it here.\n  if (Stat) {\n    Stat->dumpToLog(Conf);\n  }\n\n  if (Res) {\n    return {};\n  }\n  if (Res.error() == ErrCode::Value::Terminated) {\n    StackMgr.reset();\n  }\n  return Unexpect(Res);\n}\n\nExpect<void> Executor::execute(Runtime::StackManager &StackMgr,\n                               const AST::InstrView::iterator Start,\n                               const AST::InstrView::iterator End) {\n  AST::InstrView::iterator PC = Start;\n  AST::InstrView::iterator PCEnd = End;\n\n  auto Dispatch = [this, &PC, &StackMgr]() -> Expect<void> {\n    const AST::Instruction &Instr = *PC;\n\n    auto GetDstCompType = [&StackMgr, &Instr, this]() {\n      return getDefTypeByIdx(StackMgr, Instr.getTargetIndex())\n          ->getCompositeType();\n    };\n    auto GetSrcCompType = [&StackMgr, &Instr, this]() {\n      return getDefTypeByIdx(StackMgr, Instr.getSourceIndex())\n          ->getCompositeType();\n    };\n\n    switch (Instr.getOpCode()) {\n    // Control instructions.\n    case OpCode::Unreachable:\n      spdlog::error(ErrCode::Value::Unreachable);\n      spdlog::error(\n          ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n      return Unexpect(ErrCode::Value::Unreachable);\n    case OpCode::Nop:\n      return {};\n    case OpCode::Block:\n      return {};\n    case OpCode::Loop:\n      return {};\n    case OpCode::If:\n      return runIfElseOp(StackMgr, Instr, PC);\n    case OpCode::Else:\n      if (Stat && Conf.getStatisticsConfigure().isCostMeasuring()) {\n        // Reach here means end of if-statement.\n        if (unlikely(!Stat->subInstrCost(Instr.getOpCode()))) {\n          spdlog::error(ErrCode::Value::CostLimitExceeded);\n          spdlog::error(\n              ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n          return Unexpect(ErrCode::Value::CostLimitExceeded);\n        }\n        if (unlikely(!Stat->addInstrCost(OpCode::End))) {\n          spdlog::error(ErrCode::Value::CostLimitExceeded);\n          spdlog::error(\n              ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n          return Unexpect(ErrCode::Value::CostLimitExceeded);\n        }\n      }\n      PC += PC->getJumpEnd();\n      [[fallthrough]];\n    case OpCode::End:\n      PC = StackMgr.maybePopFrame(PC);\n      return {};\n    case OpCode::Br:\n      return runBrOp(StackMgr, Instr, PC);\n    case OpCode::Br_if:\n      return runBrIfOp(StackMgr, Instr, PC);\n    case OpCode::Br_table:\n      return runBrTableOp(StackMgr, Instr, PC);\n    case OpCode::Br_on_null:\n      return runBrOnNullOp(StackMgr, Instr, PC);\n    case OpCode::Br_on_non_null:\n      return runBrOnNonNullOp(StackMgr, Instr, PC);\n    case OpCode::Br_on_cast:\n      return runBrOnCastOp(StackMgr, Instr, PC);\n    case OpCode::Br_on_cast_fail:\n      return runBrOnCastOp(StackMgr, Instr, PC, true);\n    case OpCode::Return:\n      return runReturnOp(StackMgr, PC);\n    case OpCode::Call:\n      return runCallOp(StackMgr, Instr, PC);\n    case OpCode::Call_indirect:\n      return runCallIndirectOp(StackMgr, Instr, PC);\n    case OpCode::Return_call:\n      return runCallOp(StackMgr, Instr, PC, true);\n    case OpCode::Return_call_indirect:\n      return runCallIndirectOp(StackMgr, Instr, PC, true);\n    case OpCode::Call_ref:\n      return runCallRefOp(StackMgr, Instr, PC);\n    case OpCode::Return_call_ref:\n      return runCallRefOp(StackMgr, Instr, PC, true);\n\n    // Reference Instructions\n    case OpCode::Ref__null:\n      return runRefNullOp(StackMgr, Instr.getValType());\n    case OpCode::Ref__is_null:\n      return runRefIsNullOp(StackMgr.getTop());\n    case OpCode::Ref__func:\n      return runRefFuncOp(StackMgr, Instr.getTargetIndex());\n    case OpCode::Ref__eq: {\n      ValVariant Rhs = StackMgr.pop();\n      return runRefEqOp(StackMgr.getTop(), Rhs);\n    }\n    case OpCode::Ref__as_non_null:\n      return runRefAsNonNullOp(StackMgr.getTop().get<RefVariant>(), Instr);\n\n      // GC Instructions\n    case OpCode::Struct__new:\n      return runStructNewOp(StackMgr, Instr.getTargetIndex());\n    case OpCode::Struct__new_default:\n      return runStructNewOp(StackMgr, Instr.getTargetIndex(), true);\n    case OpCode::Struct__get:\n    case OpCode::Struct__get_u:\n      return runStructGetOp(StackMgr.getTop(), Instr.getSourceIndex(),\n                            GetDstCompType(), Instr);\n    case OpCode::Struct__get_s:\n      return runStructGetOp(StackMgr.getTop(), Instr.getSourceIndex(),\n                            GetDstCompType(), Instr, true);\n    case OpCode::Struct__set: {\n      const ValVariant Val = StackMgr.pop();\n      RefVariant StructRef = StackMgr.pop().get<RefVariant>();\n      return runStructSetOp(Val, StructRef, GetDstCompType(),\n                            Instr.getSourceIndex(), Instr);\n    }\n    case OpCode::Array__new:\n      return runArrayNewOp(StackMgr, Instr.getTargetIndex(), 1,\n                           StackMgr.pop().get<uint32_t>());\n    case OpCode::Array__new_default:\n      return runArrayNewOp(StackMgr, Instr.getTargetIndex(), 0,\n                           StackMgr.pop().get<uint32_t>());\n    case OpCode::Array__new_fixed:\n      return runArrayNewOp(StackMgr, Instr.getTargetIndex(),\n                           Instr.getSourceIndex(), Instr.getSourceIndex());\n    case OpCode::Array__new_data:\n      return runArrayNewDataOp(\n          StackMgr, *getDataInstByIdx(StackMgr, Instr.getSourceIndex()), Instr);\n    case OpCode::Array__new_elem:\n      return runArrayNewElemOp(\n...\n// Path: lib/executor/engine/refInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\nnamespace WasmEdge {\nnamespace Executor {\n\nnamespace {\nValVariant packVal(const ValType &Type, const ValVariant &Val) {\n  if (Type.isPackType()) {\n    switch (Type.getCode()) {\n    case TypeCode::I8:\n      return ValVariant(Val.get<uint32_t>() & 0xFFU);\n    case TypeCode::I16:\n      return ValVariant(Val.get<uint32_t>() & 0xFFFFU);\n    default:\n      assumingUnreachable();\n    }\n  }\n  return Val;\n}\n\nValVariant unpackVal(const ValType &Type, const ValVariant &Val,\n                     bool IsSigned = false) {\n  if (Type.isPackType()) {\n    uint32_t Num = Val.get<uint32_t>();\n    switch (Type.getCode()) {\n    case TypeCode::I8:\n      if (IsSigned) {\n        return static_cast<uint32_t>(static_cast<int8_t>(Num));\n      } else {\n        return static_cast<uint32_t>(static_cast<uint8_t>(Num));\n      }\n    case TypeCode::I16:\n      if (IsSigned) {\n        return static_cast<uint32_t>(static_cast<int16_t>(Num));\n      } else {\n        return static_cast<uint32_t>(static_cast<uint16_t>(Num));\n      }\n    default:\n      assumingUnreachable();\n    }\n  }\n  return Val;\n}\n\nstd::vector<ValVariant> packVals(const ValType &Type,\n                                 std::vector<ValVariant> &&Vals) {\n  for (uint32_t I = 0; I < Vals.size(); I++) {\n    Vals[I] = packVal(Type, Vals[I]);\n  }\n  return std::move(Vals);\n}\n} // namespace\n\nExpect<void> Executor::runRefNullOp(Runtime::StackManager &StackMgr,\n                                    const ValType &Type) const noexcept {\n  // A null reference is typed with the least type in its respective hierarchy.\n  StackMgr.push(RefVariant(toBottomType(StackMgr, Type)));\n  return {};\n}\n\nExpect<void> Executor::runRefIsNullOp(ValVariant &Val) const noexcept {\n  Val.emplace<uint32_t>(Val.get<RefVariant>().isNull() ? 1U : 0U);\n  return {};\n}\n\nExpect<void> Executor::runRefFuncOp(Runtime::StackManager &StackMgr,\n                                    uint32_t Idx) const noexcept {\n  const auto *FuncInst = getFuncInstByIdx(StackMgr, Idx);\n  StackMgr.push(RefVariant(FuncInst->getDefType(), FuncInst));\n  return {};\n}\n\nExpect<void> Executor::runRefEqOp(ValVariant &Val1,\n                                  const ValVariant &Val2) const noexcept {\n  Val1.emplace<uint32_t>(Val1.get<RefVariant>().getPtr<void>() ==\n                                 Val2.get<RefVariant>().getPtr<void>()\n                             ? 1U\n                             : 0U);\n  return {};\n}\n\nExpect<void>\nExecutor::runRefAsNonNullOp(RefVariant &Ref,\n                            const AST::Instruction &Instr) const noexcept {\n  if (Ref.isNull()) {\n    spdlog::error(ErrCode::Value::CastNullToNonNull);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::CastNullToNonNull);\n  }\n  Ref.getType().toNonNullableRef();\n  return {};\n}\n\nExpect<void> Executor::runStructNewOp(Runtime::StackManager &StackMgr,\n                                      const uint32_t DefIndex,\n                                      bool IsDefault) const noexcept {\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, DefIndex)->getCompositeType();\n  uint32_t N = static_cast<uint32_t>(CompType.getFieldTypes().size());\n  std::vector<WasmEdge::ValVariant> Vals;\n  if (IsDefault) {\n    Vals.resize(N);\n    for (uint32_t I = 0; I < N; I++) {\n      const auto &VType = CompType.getFieldTypes()[I].getStorageType();\n      Vals[I] = VType.isRefType()\n                    ? ValVariant(RefVariant(toBottomType(StackMgr, VType)))\n                    : ValVariant(static_cast<uint128_t>(0));\n    }\n  } else {\n    Vals = StackMgr.pop(N);\n    for (uint32_t I = 0; I < N; I++) {\n      Vals[I] = packVal(CompType.getFieldTypes()[I].getStorageType(), Vals[I]);\n    }\n  }\n  auto *Inst =\n      const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n          ->newStruct(DefIndex, std::move(Vals));\n  StackMgr.push(RefVariant(Inst->getDefType(), Inst));\n\n  return {};\n}\n\nExpect<void> Executor::runStructGetOp(ValVariant &Val, const uint32_t Idx,\n                                      const AST::CompositeType &CompType,\n                                      const AST::Instruction &Instr,\n                                      bool IsSigned) const noexcept {\n  const auto *Inst =\n      Val.get<RefVariant>().getPtr<Runtime::Instance::StructInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullStruct);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullStruct);\n  }\n  const auto &SType = CompType.getFieldTypes()[Idx].getStorageType();\n  Val = unpackVal(SType, Inst->getField(Idx), IsSigned);\n  return {};\n}\n\nExpect<void>\nExecutor::runStructSetOp(const ValVariant &Val, const RefVariant &InstRef,\n                         const AST::CompositeType &CompType, uint32_t Idx,\n                         const AST::Instruction &Instr) const noexcept {\n  auto *Inst = InstRef.getPtr<Runtime::Instance::StructInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullStruct);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullStruct);\n  }\n  const auto &SType = CompType.getFieldTypes()[Idx].getStorageType();\n  Inst->getField(Idx) = packVal(SType, Val);\n  return {};\n}\n\nExpect<void> Executor::runArrayNewOp(Runtime::StackManager &StackMgr,\n                                     const uint32_t DefIndex, uint32_t InitCnt,\n                                     uint32_t ValCnt) const noexcept {\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  assuming(InitCnt == 0 || InitCnt == 1 || InitCnt == ValCnt);\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, DefIndex)->getCompositeType();\n  const auto &VType = CompType.getFieldTypes()[0].getStorageType();\n  if (InitCnt == 0) {\n    auto InitVal = VType.isRefType()\n                       ? ValVariant(RefVariant(toBottomType(StackMgr, VType)))\n                       : ValVariant(static_cast<uint128_t>(0));\n    auto *Inst =\n        const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n            ->newArray(DefIndex, ValCnt, InitVal);\n    StackMgr.push(RefVariant(Inst->getDefType(), Inst));\n  } else if (InitCnt == 1) {\n    auto *Inst =\n        const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n            ->newArray(DefIndex, ValCnt, packVal(VType, StackMgr.getTop()));\n    StackMgr.getTop().emplace<RefVariant>(Inst->getDefType(), Inst);\n  } else {\n    auto *Inst =\n        const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n            ->newArray(DefIndex, packVals(VType, StackMgr.pop(ValCnt)));\n    StackMgr.push(RefVariant(Inst->getDefType(), Inst));\n  }\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayNewDataOp(Runtime::StackManager &StackMgr,\n                            const Runtime::Instance::DataInstance &DataInst,\n                            const AST::Instruction &Instr) const noexcept {\n  const uint32_t N = StackMgr.pop().get<uint32_t>();\n  const uint32_t S = StackMgr.getTop().get<uint32_t>();\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, Instr.getTargetIndex())->getCompositeType();\n  const uint32_t BSize =\n      CompType.getFieldTypes()[0].getStorageType().getBitWidth() / 8;\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) * BSize >\n      DataInst.getData().size()) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N * BSize,\n        DataInst.getData().size() > 0\n            ? static_cast<uint32_t>(DataInst.getData().size() - 1)\n            : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  auto *Inst =\n      const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n          ->newArray(Instr.getTargetIndex(), N, 0U);\n  for (uint32_t Idx = 0; Idx < N; Idx++) {\n    // The value has been packed.\n    Inst->getData(Idx) = DataInst.loadValue(S + Idx * BSize, BSize);\n  }\n  StackMgr.getTop().emplace<RefVariant>(Inst->getDefType(), Inst);\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayNewElemOp(Runtime::StackManager &StackMgr,\n                            const Runtime::Instance::ElementInstance &ElemInst,\n                            const AST::Instruction &Instr) const noexcept {\n  const uint32_t N = StackMgr.pop().get<uint32_t>();\n  const uint32_t S = StackMgr.getTop().get<uint32_t>();\n  const auto &CompType =\n      getDefTypeByIdx(StackMgr, Instr.getTargetIndex())->getCompositeType();\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  auto ElemSrc = ElemInst.getRefs();\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) > ElemSrc.size()) {\n    spdlog::error(ErrCode::Value::TableOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N,\n        ElemSrc.size() > 0 ? static_cast<uint32_t>(ElemSrc.size() - 1) : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::TableOutOfBounds);\n  }\n  std::vector<ValVariant> Refs(ElemSrc.begin() + S, ElemSrc.begin() + S + N);\n  /// TODO: The array and struct instances are owned by the module instance\n  /// currently because of referring the defined types of the module instances.\n  /// This may be changed after applying the garbage collection mechanism.\n  auto *Inst =\n      const_cast<Runtime::Instance::ModuleInstance *>(StackMgr.getModule())\n          ->newArray(Instr.getTargetIndex(), packVals(SType, std::move(Refs)));\n  StackMgr.getTop().emplace<RefVariant>(Inst->getDefType(), Inst);\n  return {};\n}\n\nExpect<void>\nExecutor::runArraySetOp(const ValVariant &Val, const uint32_t Idx,\n                        const RefVariant &InstRef,\n                        const AST::CompositeType &CompType,\n                        const AST::Instruction &Instr) const noexcept {\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (Idx >= Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(Idx, 1, Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  Inst->getData(Idx) = packVal(SType, Val);\n  return {};\n}\n\nExpect<void> Executor::runArrayGetOp(ValVariant &Val, const uint32_t Idx,\n                                     const AST::CompositeType &CompType,\n                                     const AST::Instruction &Instr,\n                                     bool IsSigned) const noexcept {\n  const auto *Inst =\n      Val.get<RefVariant>().getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (Idx >= Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(Idx, 1, Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  Val = unpackVal(SType, Inst->getData(Idx), IsSigned);\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayLenOp(ValVariant &Val,\n                        const AST::Instruction &Instr) const noexcept {\n  const auto *Inst =\n      Val.get<RefVariant>().getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  Val.emplace<uint32_t>(Inst->getLength());\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayFillOp(uint32_t N, const ValVariant &Val, uint32_t D,\n                         const RefVariant &InstRef,\n                         const AST::CompositeType &CompType,\n                         const AST::Instruction &Instr) const noexcept {\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) > Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n  auto Arr = Inst->getArray();\n  std::fill(Arr.begin() + D, Arr.begin() + D + N, packVal(SType, Val));\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayCopyOp(uint32_t N, uint32_t S, const RefVariant &SrcInstRef,\n                         uint32_t D, const RefVariant &DstInstRef,\n                         const AST::CompositeType &SrcCompType,\n                         const AST::CompositeType &DstCompType,\n                         const AST::Instruction &Instr) const noexcept {\n  auto *SrcInst = SrcInstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  auto *DstInst = DstInstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (SrcInst == nullptr || DstInst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) >\n      SrcInst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(S), N,\n                                        SrcInst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) >\n      DstInst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        DstInst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  const auto &SrcSType = SrcCompType.getFieldTypes()[0].getStorageType();\n  const auto &DstSType = DstCompType.getFieldTypes()[0].getStorageType();\n  auto SrcArr = SrcInst->getArray();\n  auto DstArr = DstInst->getArray();\n  if (D <= S) {\n    std::transform(SrcArr.begin() + S, SrcArr.begin() + S + N,\n                   DstArr.begin() + D, [&](const ValVariant &V) {\n                     return packVal(DstSType, unpackVal(SrcSType, V));\n                   });\n  } else {\n    std::transform(std::make_reverse_iterator(SrcArr.begin() + S + N),\n                   std::make_reverse_iterator(SrcArr.begin() + S),\n                   std::make_reverse_iterator(DstArr.begin() + D + N),\n                   [&](const ValVariant &V) {\n                     return packVal(DstSType, unpackVal(SrcSType, V));\n                   });\n  }\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayInitDataOp(uint32_t N, uint32_t S, uint32_t D,\n                             const RefVariant &InstRef,\n                             const AST::CompositeType &CompType,\n                             const Runtime::Instance::DataInstance &DataInst,\n                             const AST::Instruction &Instr) const noexcept {\n  const uint32_t BSize =\n      CompType.getFieldTypes()[0].getStorageType().getBitWidth() / 8;\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) > Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) * BSize >\n      DataInst.getData().size()) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N * BSize,\n        DataInst.getData().size() > 0\n            ? static_cast<uint32_t>(DataInst.getData().size() - 1)\n            : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n  for (uint32_t Off = 0; Off < N; Off++) {\n    // The value has been packed.\n    Inst->getData(D + Off) = DataInst.loadValue(S + Off * BSize, BSize);\n  }\n  return {};\n}\n\nExpect<void>\nExecutor::runArrayInitElemOp(uint32_t N, uint32_t S, uint32_t D,\n                             const RefVariant &InstRef,\n                             const AST::CompositeType &CompType,\n                             const Runtime::Instance::ElementInstance &ElemInst,\n                             const AST::Instruction &Instr) const noexcept {\n  auto ElemSrc = ElemInst.getRefs();\n  auto *Inst = InstRef.getPtr<Runtime::Instance::ArrayInstance>();\n  if (Inst == nullptr) {\n    spdlog::error(ErrCode::Value::AccessNullArray);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullArray);\n  }\n  if (static_cast<uint64_t>(D) + static_cast<uint64_t>(N) > Inst->getLength()) {\n    spdlog::error(ErrCode::Value::ArrayOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(static_cast<uint64_t>(D), N,\n                                        Inst->getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::ArrayOutOfBounds);\n  }\n  if (static_cast<uint64_t>(S) + static_cast<uint64_t>(N) > ElemSrc.size()) {\n    spdlog::error(ErrCode::Value::TableOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        static_cast<uint64_t>(S), N,\n        ElemSrc.size() > 0 ? static_cast<uint32_t>(ElemSrc.size() - 1) : 0U));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::TableOutOfBounds);\n  }\n  const auto &SType = CompType.getFieldTypes()[0].getStorageType();\n\n  auto Arr = Inst->getArray();\n  // The value has been packed.\n  std::transform(ElemSrc.begin() + S, ElemSrc.begin() + S + N, Arr.begin() + D,\n                 [&](const RefVariant &V) { return packVal(SType, V); });\n  return {};\n}\n\nExpect<void>\nExecutor::runRefTestOp(const Runtime::Instance::ModuleInstance *ModInst,\n                       ValVariant &Val, const AST::Instruction &Instr,\n                       bool IsCast) const noexcept {\n  // Copy the value type here due to handling the externalized case.\n  auto &VT = Val.get<RefVariant>().getType();\n  if (VT.isExternalized()) {\n    VT = ValType(TypeCode::Ref, TypeCode::ExternRef);\n  }\n  Span<const AST::SubType *const> GotTypeList = ModInst->getTypeList();\n  if (!VT.isAbsHeapType()) {\n    auto *Inst =\n        Val.get<RefVariant>().getPtr<Runtime::Instance::CompositeBase>();\n    // Reference must not be nullptr here because the null references are typed\n    // with the least abstract heap type.\n    if (Inst->getModule()) {\n      GotTypeList = Inst->getModule()->getTypeList();\n    }\n  }\n\n  if (AST::TypeMatcher::matchType(ModInst->getTypeList(), Instr.getValType(),\n                                  GotTypeList, VT)) {\n    if (!IsCast) {\n      Val.emplace<uint32_t>(1U);\n    }\n  } else {\n    if (IsCast) {\n      spdlog::error(ErrCode::Value::CastFailed);\n      spdlog::error(ErrInfo::InfoMismatch(Instr.getValType(), VT));\n      spdlog::error(\n          ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n      return Unexpect(ErrCode::Value::CastFailed);\n    } else {\n      Val.emplace<uint32_t>(0U);\n    }\n  }\n  return {};\n}\n\nExpect<void> Executor::runRefConvOp(RefVariant &Ref,\n                                    TypeCode TCode) const noexcept {\n\n  if (TCode == TypeCode::AnyRef) {\n    // Internalize.\n    if (Ref.isNull()) {\n      Ref = RefVariant(ValType(TypeCode::RefNull, TypeCode::NullRef));\n    } else {\n      Ref.getType().setInternalized();\n      if (Ref.getType().isExternRefType()) {\n        Ref.getType() = ValType(TypeCode::Ref, TypeCode::AnyRef);\n      }\n    }\n  } else {\n    // Externalize.\n    if (Ref.isNull()) {\n      Ref = RefVariant(ValType(TypeCode::RefNull, TypeCode::NullExternRef));\n    } else {\n      // Use the externalize flag because the value type information should be\n      // reserved when a reference being externalized and internalized.\n      Ref.getType().setExternalized();\n    }\n  }\n  return {};\n}\n\nExpect<void> Executor::runRefI31Op(ValVariant &Val) const noexcept {\n  uint32_t RefNum = (Val.get<uint32_t>() & 0x7FFFFFFFU) | 0x80000000U;\n  Val = RefVariant(ValType(TypeCode::Ref, TypeCode::I31Ref),\n                   reinterpret_cast<void *>(static_cast<uint64_t>(RefNum)));\n  return {};\n}\n\nExpect<void> Executor::runI31GetOp(ValVariant &Val,\n                                   const AST::Instruction &Instr,\n                                   bool IsSigned) const noexcept {\n  uint32_t RefNum = static_cast<uint32_t>(\n      reinterpret_cast<uintptr_t>(Val.get<RefVariant>().getPtr<void>()));\n  if ((RefNum & 0x80000000U) == 0) {\n    spdlog::error(ErrCode::Value::AccessNullI31);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::AccessNullI31);\n  }\n  RefNum &= 0x7FFFFFFFU;\n  if (IsSigned) {\n    RefNum |= ((RefNum & 0x40000000U) << 1);\n  }\n  Val.emplace<uint32_t>(RefNum);\n  return {};\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/memoryInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void>\nExecutor::runMemorySizeOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInst) {\n  // Push SZ = page size to stack.\n  StackMgr.push(MemInst.getPageSize());\n  return {};\n}\n\nExpect<void>\nExecutor::runMemoryGrowOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInst) {\n  // Pop N for growing page size.\n  uint32_t &N = StackMgr.getTop().get<uint32_t>();\n\n  // Grow page and push result.\n  const uint32_t CurrPageSize = static_cast<uint32_t>(MemInst.getPageSize());\n  if (MemInst.growPage(N)) {\n    N = CurrPageSize;\n  } else {\n    N = static_cast<uint32_t>(-1);\n  }\n  return {};\n}\n\nExpect<void> Executor::runMemoryInitOp(\n    Runtime::StackManager &StackMgr, Runtime::Instance::MemoryInstance &MemInst,\n    Runtime::Instance::DataInstance &DataInst, const AST::Instruction &Instr) {\n  // Pop the length, source, and destination from stack.\n  uint32_t Len = StackMgr.pop().get<uint32_t>();\n  uint32_t Src = StackMgr.pop().get<uint32_t>();\n  uint32_t Dst = StackMgr.pop().get<uint32_t>();\n\n  // Replace mem[Dst : Dst + Len] with data[Src : Src + Len].\n  if (auto Res = MemInst.setBytes(DataInst.getData(), Dst, Src, Len)) {\n    return {};\n  } else {\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Res);\n  }\n}\n\nExpect<void>\nExecutor::runDataDropOp(Runtime::Instance::DataInstance &DataInst) {\n  // Clear data instance.\n  DataInst.clear();\n  return {};\n}\n\nExpect<void>\nExecutor::runMemoryCopyOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInstDst,\n                          Runtime::Instance::MemoryInstance &MemInstSrc,\n                          const AST::Instruction &Instr) {\n  // Pop the length, source, and destination from stack.\n  uint32_t Len = StackMgr.pop().get<uint32_t>();\n  uint32_t Src = StackMgr.pop().get<uint32_t>();\n  uint32_t Dst = StackMgr.pop().get<uint32_t>();\n\n  // Replace mem[Dst : Dst + Len] with mem[Src : Src + Len].\n  if (auto Data = MemInstSrc.getBytes(Src, Len)) {\n    if (auto Res = MemInstDst.setBytes(*Data, Dst, 0, Len)) {\n      return {};\n    } else {\n      spdlog::error(\n          ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n      return Unexpect(Res);\n    }\n  } else {\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Data);\n  }\n}\n\nExpect<void>\nExecutor::runMemoryFillOp(Runtime::StackManager &StackMgr,\n                          Runtime::Instance::MemoryInstance &MemInst,\n                          const AST::Instruction &Instr) {\n  // Pop the length, value, and offset from stack.\n  uint32_t Len = StackMgr.pop().get<uint32_t>();\n  uint8_t Val = static_cast<uint8_t>(StackMgr.pop().get<uint32_t>());\n  uint32_t Off = StackMgr.pop().get<uint32_t>();\n\n  // Fill data with Val.\n  if (auto Res = MemInst.fillBytes(Val, Off, Len)) {\n    return {};\n  } else {\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Res);\n  }\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/executor/engine/threadInstr.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"executor/executor.h\"\n\nnamespace WasmEdge {\nnamespace Executor {\n\nExpect<void>\nExecutor::runAtomicNotifyOp(Runtime::StackManager &StackMgr,\n                            Runtime::Instance::MemoryInstance &MemInst,\n                            const AST::Instruction &Instr) {\n  ValVariant RawCount = StackMgr.pop();\n  ValVariant &RawAddress = StackMgr.getTop();\n\n  uint32_t Address = RawAddress.get<uint32_t>();\n\n  if (Address >\n      std::numeric_limits<uint32_t>::max() - Instr.getMemoryOffset()) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    spdlog::error(ErrInfo::InfoBoundary(\n        Address + static_cast<uint64_t>(Instr.getMemoryOffset()),\n        sizeof(uint32_t), MemInst.getBoundIdx()));\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n  Address += Instr.getMemoryOffset();\n\n  if (Address % sizeof(uint32_t) != 0) {\n    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);\n  }\n\n  uint32_t Count = RawCount.get<uint32_t>();\n  if (auto Res = atomicNotify(MemInst, Address, Count); unlikely(!Res)) {\n    spdlog::error(Res.error());\n    spdlog::error(\n        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));\n    return Unexpect(Res);\n  } else {\n    RawAddress.emplace<uint32_t>(*Res);\n  }\n  return {};\n}\n\nExpect<void> Executor::runMemoryFenceOp() {\n  std::atomic_thread_fence(std::memory_order_release);\n  return {};\n}\n\nExpect<uint32_t>\nExecutor::atomicNotify(Runtime::Instance::MemoryInstance &MemInst,\n                       uint32_t Address, uint32_t Count) noexcept {\n  // The error message should be handled by the caller, or the AOT mode will\n  // produce the duplicated messages.\n  if (auto *AtomicObj = MemInst.getPointer<std::atomic<uint32_t> *>(Address);\n      !AtomicObj) {\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n\n  std::unique_lock<decltype(WaiterMapMutex)> Locker(WaiterMapMutex);\n  uint32_t Total = 0;\n  auto Range = WaiterMap.equal_range(Address);\n  for (auto Iterator = Range.first; Total < Count && Iterator != Range.second;\n       ++Iterator) {\n    if (likely(&MemInst == Iterator->second.MemInst)) {\n      Iterator->second.Cond.notify_all();\n      ++Total;\n    }\n  }\n  return Total;\n}\n\nvoid Executor::atomicNotifyAll() noexcept {\n  std::unique_lock<decltype(WaiterMapMutex)> Locker(WaiterMapMutex);\n  for (auto Iterator = WaiterMap.begin(); Iterator != WaiterMap.end();\n       ++Iterator) {\n    Iterator->second.Cond.notify_all();\n  }\n}\n\n} // namespace Executor\n} // namespace WasmEdge\n\n// Path: lib/loader/loader.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/loader.h\"\n\n#include \"aot/version.h\"\n\n#include <algorithm>\n#include <cstddef>\n#include <fstream>\n#include <limits>\n#include <memory>\n#include <system_error>\n#include <utility>\n#include <variant>\n\nnamespace WasmEdge {\nnamespace Loader {\n\n// Load data from file path. See \"include/loader/loader.h\".\nExpect<std::vector<Byte>>\nLoader::loadFile(const std::filesystem::path &FilePath) {\n  std::error_code EC;\n  size_t FileSize = std::filesystem::file_size(FilePath, EC);\n  if (EC) {\n    spdlog::error(ErrCode::Value::IllegalPath);\n    spdlog::error(ErrInfo::InfoFile(FilePath));\n    return Unexpect(ErrCode::Value::IllegalPath);\n  }\n\n  std::ifstream Fin(FilePath, std::ios::in | std::ios::binary);\n  if (!Fin) {\n    spdlog::error(ErrCode::Value::IllegalPath);\n    spdlog::error(ErrInfo::InfoFile(FilePath));\n    return Unexpect(ErrCode::Value::IllegalPath);\n  }\n\n  std::vector<Byte> Buf(FileSize);\n  size_t Index = 0;\n  while (FileSize > 0) {\n    const uint32_t BlockSize = static_cast<uint32_t>(\n        std::min<size_t>(FileSize, std::numeric_limits<uint32_t>::max()));\n    Fin.read(reinterpret_cast<char *>(Buf.data()) + Index, BlockSize);\n    const uint32_t ReadCount = static_cast<uint32_t>(Fin.gcount());\n    if (ReadCount != BlockSize) {\n      if (Fin.eof()) {\n        spdlog::error(ErrCode::Value::UnexpectedEnd);\n        spdlog::error(ErrInfo::InfoLoading(ReadCount));\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(ErrCode::Value::UnexpectedEnd);\n      } else {\n        spdlog::error(ErrCode::Value::ReadError);\n        spdlog::error(ErrInfo::InfoLoading(ReadCount));\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(ErrCode::Value::ReadError);\n      }\n    }\n    Index += static_cast<size_t>(BlockSize);\n    FileSize -= static_cast<size_t>(BlockSize);\n  }\n  return Buf;\n}\n\nExpect<std::variant<std::unique_ptr<AST::Component::Component>,\n                    std::unique_ptr<AST::Module>>>\nLoader::parseWasmUnit(const std::filesystem::path &FilePath) {\n  std::lock_guard Lock(Mutex);\n  // Set path and check the header.\n  if (auto Res = FMgr.setPath(FilePath); !Res) {\n    spdlog::error(Res.error());\n    spdlog::error(ErrInfo::InfoFile(FilePath));\n    return Unexpect(Res);\n  }\n\n  switch (FMgr.getHeaderType()) {\n  // Filter out the Windows .dll, MacOS .dylib, or Linux .so AOT compiled\n  // shared-library-WASM.\n  case FileMgr::FileHeader::ELF:\n  case FileMgr::FileHeader::DLL:\n  case FileMgr::FileHeader::MachO_32:\n  case FileMgr::FileHeader::MachO_64: {\n    // AOT compiled shared-library-WASM cases. Use ldmgr to load the module.\n    WASMType = InputType::SharedLibrary;\n    FMgr.reset();\n    std::shared_ptr<SharedLibrary> Library = std::make_shared<SharedLibrary>();\n    if (auto Res = Library->load(FilePath); !Res) {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Res);\n    }\n    if (auto Res = Library->getVersion()) {\n      if (*Res != AOT::kBinaryVersion) {\n        spdlog::error(ErrInfo::InfoMismatch(AOT::kBinaryVersion, *Res));\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(ErrCode::Value::MalformedVersion);\n      }\n    } else {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Res);\n    }\n\n    std::unique_ptr<AST::Module> Mod;\n    if (auto Code = Library->getWasm()) {\n      // Set the binary and load module.\n      // Not to use parseModule() here to keep the `WASMType` value.\n      if (auto Res = FMgr.setCode(*Code); !Res) {\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(Res);\n      }\n      if (auto Res = loadUnit()) {\n        if (std::holds_alternative<std::unique_ptr<AST::Module>>(*Res)) {\n          Mod = std::move(std::get<std::unique_ptr<AST::Module>>(*Res));\n        }\n      } else {\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(Res);\n      }\n    } else {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Code);\n    }\n    if (!Conf.getRuntimeConfigure().isForceInterpreter()) {\n      // If the configure is set to force interpreter mode, not to load the AOT\n      // related data.\n      if (auto Res = loadExecutable(*Mod, Library); unlikely(!Res)) {\n        spdlog::error(ErrInfo::InfoFile(FilePath));\n        return Unexpect(Res);\n      }\n    }\n    return Mod;\n  }\n  default:\n    // Universal WASM, WASM, or other cases. Load and parse the module directly.\n    WASMType = InputType::WASM;\n    auto Unit = loadUnit();\n    if (!Unit) {\n      spdlog::error(ErrInfo::InfoFile(FilePath));\n      return Unexpect(Unit);\n    }\n    switch (Unit->index()) {\n    case 0: // component\n      return Unit;\n    case 1: // module\n    default: {\n      auto Mod = std::move(std::get<std::unique_ptr<AST::Module>>(*Unit));\n      if (!Conf.getRuntimeConfigure().isForceInterpreter()) {\n        // If the configure is set to force interpreter mode, not to set the\n        // symbol.\n        if (auto &Symbol = Mod->getSymbol()) {\n          *Symbol = IntrinsicsTable;\n        }\n      }\n      return Mod;\n    }\n    }\n  }\n}\n\nExpect<std::variant<std::unique_ptr<AST::Component::Component>,\n                    std::unique_ptr<AST::Module>>>\nLoader::parseWasmUnit(Span<const uint8_t> Code) {\n  std::lock_guard Lock(Mutex);\n  if (auto Res = FMgr.setCode(Code); !Res) {\n    return Unexpect(Res);\n  }\n  switch (FMgr.getHeaderType()) {\n  // Filter out the Windows .dll, MacOS .dylib, or Linux .so AOT compiled\n  // shared-library-WASM.\n  case FileMgr::FileHeader::ELF:\n  case FileMgr::FileHeader::DLL:\n  case FileMgr::FileHeader::MachO_32:\n  case FileMgr::FileHeader::MachO_64:\n    spdlog::error(\"Might an invalid wasm file\");\n    spdlog::error(ErrCode::Value::MalformedMagic);\n    spdlog::error(\n        \"    The AOT compiled WASM shared library is not supported for loading \"\n        \"from memory. Please use the universal WASM binary or pure WASM, or \"\n        \"load the AOT compiled WASM shared library from file.\");\n    return Unexpect(ErrCode::Value::MalformedMagic);\n  default:\n    break;\n  }\n  // For malformed header checking, handle in the module loading.\n  WASMType = InputType::WASM;\n  return loadUnit();\n}\n\n// Parse module from file path. See \"include/loader/loader.h\".\nExpect<std::unique_ptr<AST::Module>>\nLoader::parseModule(const std::filesystem::path &FilePath) {\n  if (auto R = parseWasmUnit(FilePath)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*R)) {\n      return std::move(std::get<std::unique_ptr<AST::Module>>(*R));\n    }\n    return Unexpect(ErrCode::Value::MalformedVersion);\n  } else {\n    return Unexpect(R);\n  }\n}\n\n// Parse module from byte code. See \"include/loader/loader.h\".\nExpect<std::unique_ptr<AST::Module>>\nLoader::parseModule(Span<const uint8_t> Code) {\n  if (auto R = parseWasmUnit(Code)) {\n    if (std::holds_alternative<std::unique_ptr<AST::Module>>(*R)) {\n      return std::move(std::get<std::unique_ptr<AST::Module>>(*R));\n    }\n    return Unexpect(ErrCode::Value::MalformedVersion);\n  } else {\n    return Unexpect(R);\n  }\n}\n\n// Serialize module into byte code. See \"include/loader/loader.h\".\nExpect<std::vector<Byte>> Loader::serializeModule(const AST::Module &Mod) {\n  return Ser.serializeModule(Mod);\n}\n\n} // namespace Loader\n} // namespace WasmEdge\n\n// Path: lib/loader/aot_section.cpp\n// SPDX-License-Identifier: Apache-2.0\n// SPDX-FileCopyrightText: 2019-2022 Second State INC\n\n#include \"loader/aot_section.h\"\n#include \"common/log.h\"\n#include \"system/allocator.h\"\n\n#if WASMEDGE_OS_LINUX || WASMEDGE_OS_MACOS\nextern \"C\" {\nextern void __register_frame(void *);\nextern void __deregister_frame(void *);\n}\n#endif\n\nnamespace {\n\ninline constexpr uint64_t roundDownPageBoundary(const uint64_t Value) {\n// ARM64 Mac has a special page size\n#if WASMEDGE_OS_MACOS && defined(__aarch64__)\n  return Value & ~UINT64_C(16383);\n#else\n  return Value & ~UINT64_C(4095);\n#endif\n}\ninline constexpr uint64_t roundUpPageBoundary(const uint64_t Value) {\n// ARM64 Mac has a special page size\n#if WASMEDGE_OS_MACOS && defined(__aarch64__)\n  return roundDownPageBoundary(Value + UINT64_C(16383));\n#else\n  return roundDownPageBoundary(Value + UINT64_C(4095));\n#endif\n}\n} // namespace\n\nnamespace WasmEdge::Loader {\n\nExpect<void> AOTSection::load(const AST::AOTSection &AOTSec) noexcept {\n  BinarySize = 0;\n  for (const auto &Section : AOTSec.getSections()) {\n    const auto Offset = std::get<1>(Section);\n    const auto Size = std::get<2>(Section);\n    BinarySize = std::max(BinarySize, Offset + Size);\n  }\n  BinarySize = roundUpPageBoundary(BinarySize);\n\n  Binary = Allocator::allocate_chunk(BinarySize);\n  if (unlikely(!Binary)) {\n    spdlog::error(ErrCode::Value::MemoryOutOfBounds);\n    return Unexpect(ErrCode::Value::MemoryOutOfBounds);\n  }\n\n  std::vector<std::pair<uint8_t *, uint64_t>> ExecutableRanges;\n  for (const auto &Section : AOTSec.getSections()) {\n    const auto Offset = std::get<1>(Section);\n    const auto Size = std::get<2>(Section);\n    const auto &Content = std::get<3>(Section);\n    if (Size > BinarySize || Offset > BinarySize ||\n        Offset + Size > BinarySize || Content.size() > Size) {\n      return Unexpect(ErrCode::Value::IntegerTooLarge);\n    }\n    std::copy(Content.begin(), Content.end(), Binary + Offset);\n    switch (std::get<0>(Section)) {\n    case 1: { // Text\n      const auto O = roundDownPageBoundary(Offset);\n      const auto S = roundUpPageBoundary(Size + (Offset - O));\n      ExecutableRanges.emplace_back(Binary + O, S);\n      break;\n    }\n    case 2: // Data\n      break;\n    case 3: // BSS\n      break;\n#if WASMEDGE_OS_LINUX\n    case 4: // EHFrame\n      EHFrameAddress = reinterpret_cast<void *>(Binary + Offset);\n      break;\n#elif WASMEDGE_OS_MACOS\n    case 4: // EHFrame\n      EHFrameAddress = reinterpret_cast<uint8_t *>(Binary + Offset);\n      EHFrameSize = Size;\n      break;\n#elif WASMEDGE_OS_WINDOWS\n    case 4: // PData\n      PDataAddress = reinterpret_cast<void *>(Binary + Offset);\n      PDataSize =\n...\n", "needle_token_start": 15392, "needle_token_end": 15495, "code_context_ntokens": 16235, "output": ["```cpp\ninline constexpr uint64_t roundDownPageBoundary(const uint64_t Value) {\n  // ARM64 Mac has a special page size\n#if WASMEDGE_OS_MACOS && defined(__aarch64__)\n  return Value & ~UINT64_C(16383);\n#else\n  return Value & ~UINT64_C(4095);\n#endif\n}\n```"]}
{"repo": "oatpp/oatpp", "name": "Pool", "language": "cpp", "path": "src/oatpp/core/provider/Pool.hpp", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this class is to manage a pool of resources, ensuring efficient resource allocation and deallocation, maintaining a limit on the maximum number of resources, and handling resource lifecycle based on a time-to-live (TTL) value.\n2. **Input**: The inputs include a provider for the resources, a maximum number of resources allowed in the pool, a maximum TTL for each resource, and an optional timeout for resource acquisition.\n3. **Output**: This class does not directly produce a conventional output but manages the lifecycle and availability of resources. It provides mechanisms to acquire and release resources, and to query the current count of active resources.\n4. **Procedure**: The procedure involves initializing the resource pool with specified parameters, managing resource allocation requests through an acquisition proxy, and handling errors during resource acquisition. It also includes a cleanup task to remove expired resources and a stop mechanism to clean up all resources when the pool is no longer needed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "      }\n\n      async::Action handleError(oatpp::async::Error* error) override {\n        {\n          /* Careful!!! Using non-async lock */\n          std::lock_guard<std::mutex> guard(m_pool->m_lock);\n          -- m_pool->m_counter;\n        }\n        return error;\n      }\n\n    };\n\n    return GetCoroutine::startForResult(_this);\n\n  }\n\npublic:\n\n  static std::shared_ptr<PoolTemplate> createShared(const std::shared_ptr<Provider<TResource>>& provider,\n                                                    v_int64 maxResources,\n                                                    const std::chrono::duration<v_int64, std::micro>& maxResourceTTL,\n                                                    const std::chrono::duration<v_int64, std::micro>& timeout)\n  {\n    /* \"new\" is called directly to keep constructor private */\n    auto ptr = std::shared_ptr<PoolTemplate>(new PoolTemplate(provider, maxResources, maxResourceTTL.count(), timeout));\n    startCleanupTask(ptr);\n    return ptr;\n  }\n\n  virtual ~PoolTemplate() override {\n    stop();\n  }\n\n  void stop() {\n\n    {\n      std::lock_guard<std::mutex> guard(m_lock);\n\n      if (!m_running) {\n        return;\n      }\n\n      m_running = false;\n      m_counter -= static_cast<v_int64>(m_bench.size());\n      m_bench.clear();\n    }\n\n    m_condition.notify_all();\n    m_waitList.notifyAll();\n\n    {\n      std::unique_lock<std::mutex> guard(m_lock);\n      while (!m_finished) {\n        m_condition.wait(guard);\n      }\n\n    }\n\n    m_provider->stop();\n\n  }\n\n  v_int64 getCounter() {\n    std::lock_guard<std::mutex> guard(m_lock);\n    return m_counter;\n  }\n\n};\n\n/**\n * Pool template class.\n * @tparam TProvider - base class for pool to inherit, ex.: ServerConnectionProvider.\n * @tparam TResource - abstract resource interface type, Ex.: `IOStream`. Must be the same as a return-type of Provider.\n * @tparam AcquisitionProxyImpl - implementation of &l:AcquisitionProxy;.\n */\ntemplate<class TProvider, class TResource, class AcquisitionProxyImpl>\nclass Pool :\n  public TProvider,\n  public std::enable_shared_from_this<Pool<TProvider, TResource, AcquisitionProxyImpl>>,\n  public PoolTemplate<TResource, AcquisitionProxyImpl> {\nprivate:\n  typedef PoolTemplate<TResource, AcquisitionProxyImpl> TPool;\nprotected:\n\n  /*\n   * Protected Constructor.\n   * @param provider\n   * @param maxResources\n   * @param maxResourceTTL\n   * @param timeout\n   */\n  \nPool(const std::shared_ptr<TProvider>& provider,\n       v_int64 maxResources,\n       v_int64 maxResourceTTL,\n       const std::chrono::duration<v_int64, std::micro>& timeout = std::chrono::microseconds::zero()\n  )\n    : PoolTemplate<TResource, AcquisitionProxyImpl>(provider, maxResources, maxResourceTTL, timeout)\n  {\n    TProvider::m_properties = provider->getProperties();\n  }\n\npublic:\n\n  /**\n   * Create shared Pool.\n   * @param provider - resource provider.\n   * @param maxResources - max resource count in the pool.\n   * @param maxResourceTTL - max time-to-live for unused resource in the pool.\n   * @param timeout - optional timeout on &l:Pool::get (); and &l:Pool::getAsync (); operations.\n   * @return - `std::shared_ptr` of `Pool`.\n   */\n  static std::shared_ptr<Pool> createShared(const std::shared_ptr<TProvider>& provider,\n                                            v_int64 maxResources,\n                                            const std::chrono::duration<v_int64, std::micro>& maxResourceTTL,\n                                            const std::chrono::duration<v_int64, std::micro>& timeout = std::chrono::microseconds::zero())\n  {\n    /* \"new\" is called directly to keep constructor private */\n    auto ptr = std::shared_ptr<Pool>(new Pool(provider, maxResources, maxResourceTTL.count(), timeout));\n    ptr->startCleanupTask(ptr);\n    return ptr;\n  }\n\n  /**\n   * Get resource.\n   * @return\n   */\n  provider::ResourceHandle<TResource> get() override {\n    return TPool::get(this->shared_from_this());\n  }\n\n  /**\n   * Get resource asynchronously.\n   * @return\n   */\n  async::CoroutineStarterForResult<const provider::ResourceHandle<TResource>&> getAsync() override {\n    return TPool::getAsync(this->shared_from_this());\n  }\n\n  /**\n   * Stop pool. <br>\n   * *Note: call to stop() may block.*\n   */\n  void stop() override {\n    TPool::stop();\n  }\n\n  /**\n   * Get pool resource count. Both acquired and available.\n   * @return\n   */\n  v_int64 getCounter() {\n    return TPool::getCounter();\n  }\n\n};\n\n}}\n\n#endif // oatpp_provider_Pool_hpp\n\n// Path: src/oatpp/core/base/ObjectHandle.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_base_ObjectHandle_hpp\n#define oatpp_base_ObjectHandle_hpp\n\n#include \"./Environment.hpp\"\n\nnamespace oatpp { namespace base {\n\ntemplate<class T>\nclass ObjectHandle {\nprivate:\n  T* m_object;\n  std::shared_ptr<T> m_ptr;\npublic:\n\n  ObjectHandle(T* object)\n    : m_object(object)\n  {}\n\n  template<class Q>\n  ObjectHandle(const std::shared_ptr<Q>& sharedObject)\n    : m_object(sharedObject.get())\n    , m_ptr(sharedObject)\n  {}\n\n  std::shared_ptr<T> getPtr() const {\n    return m_ptr;\n  }\n\n  T* get() const {\n    return m_object;\n  }\n\n  T* operator->() const {\n    return m_object;\n  }\n\n  explicit operator bool() const {\n    return m_object != nullptr;\n  }\n\n};\n\n}}\n\n#endif //oatpp_base_ObjectHandle_hpp\n\n// Path: src/oatpp/core/async/utils/FastQueue.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_async_utils_FastQueue_hpp\n#define oatpp_async_utils_FastQueue_hpp\n\n#include \"oatpp/core/concurrency/SpinLock.hpp\"\n#include \"oatpp/core/base/Environment.hpp\"\n\nnamespace oatpp { namespace async { namespace utils {\n  \ntemplate<typename T>\nclass FastQueue {\npublic:\n  \n  FastQueue()\n    : first(nullptr)\n    , last(nullptr)\n    , count(0)\n  {}\n  \n  ~FastQueue(){\n    clear();\n  }\n\n  FastQueue(const FastQueue &) = delete;\n\n  FastQueue(FastQueue &&other) noexcept\n    : FastQueue() {\n      using std::swap;\n      swap(first, other.first);\n      swap(last, other.last);\n      swap(count, other.count);\n  }\n\n  FastQueue &operator=(const FastQueue &) = delete;\n\n  FastQueue &operator=(FastQueue &&other) noexcept {\n      if (this != std::addressof(other)) {\n          using std::swap;\n          swap(first, other.first);\n          swap(last, other.last);\n          swap(count, other.count);\n      }\n      return *this;\n  }\n\n  \n  T* first;\n  T* last;\n  v_int32 count{};\n\n  v_int32 Count() {\n      return count;\n  }\n\n  bool empty() {\n      return count == 0;\n  }\n  \n  void pushFront(T* entry) {\n    entry->_ref = first;\n    first = entry;\n    if(last == nullptr) {\n      last = first;\n    }\n    ++ count;\n  }\n  \n  void pushBack(T* entry) {\n    entry->_ref = nullptr;\n    if(last == nullptr) {\n      first = entry;\n      last = entry;\n    } else {\n      last->_ref = entry;\n      last = entry;\n    }\n    ++ count;\n  }\n  \n  void round(){\n    if(count > 1) {\n      last->_ref = first;\n      last = first;\n      first = first->_ref;\n      last->_ref = nullptr;\n    }\n  }\n  \n  T* popFront() {\n    T* result = first;\n    first = first->_ref;\n    if(first == nullptr) {\n      last = nullptr;\n    }\n    -- count;\n    return result;\n  }\n  \n  void popFrontNoData() {\n    T* result = first;\n    first = first->_ref;\n    if(first == nullptr) {\n      last = nullptr;\n    }\n    delete result;\n    -- count;\n  }\n\n  static void moveAll(FastQueue& fromQueue, FastQueue& toQueue) {\n\n    if(fromQueue.count > 0) {\n\n      if (toQueue.last == nullptr) {\n        toQueue.first = fromQueue.first;\n        toQueue.last = fromQueue.last;\n      } else {\n        toQueue.last->_ref = fromQueue.first;\n        toQueue.last = fromQueue.last;\n      }\n\n      toQueue.count += fromQueue.count;\n      fromQueue.count = 0;\n\n      fromQueue.first = nullptr;\n      fromQueue.last = nullptr;\n\n    }\n\n  }\n\n  void cutEntry(T* entry, T* prevEntry){\n\n    if(prevEntry == nullptr) {\n      popFront();\n    } else {\n      prevEntry->_ref = entry->_ref;\n      -- count;\n      if(prevEntry->_ref == nullptr) {\n        last = prevEntry;\n      }\n    }\n\n  }\n  \n  void clear() {\n    T* curr = first;\n    while (curr != nullptr) {\n      T* next = curr->_ref;\n      delete curr;\n      curr = next;\n    }\n    first = nullptr;\n    last = nullptr;\n    count = 0;\n  }\n  \n};\n  \n}}}\n\n#endif /* oatpp_async_utils_FastQueue_hpp */\n\n", "needle_token_start": 734, "needle_token_end": 853, "code_context_ntokens": 3291, "output": ["```cpp\ntemplate<class TProvider, class TResource, class AcquisitionProxyImpl>\nclass Pool :\n  public TProvider,\n  public std::enable_shared_from_this<Pool<TProvider, TResource, AcquisitionProxyImpl>>,\n  public PoolTemplate<TResource, AcquisitionProxyImpl> {\nprivate:\n  typedef PoolTemplate<TResource, AcquisitionProxyImpl> TPool;\nprotected:\n\n  /*\n   * Protected Constructor.\n   * @param provider\n   * @param maxResources\n   * @param maxResourceTTL\n   * @param timeout\n   */\n  \nPool(const std::shared_ptr<TProvider>& provider,\n       v_int64 maxResources,\n       v_int64 maxResourceTTL,\n       const std::chrono::duration<v_int64, std::micro>& timeout = std::chrono::microseconds::zero()\n  )\n    : PoolTemplate<TResource, AcquisitionProxyImpl>(provider, maxResources, maxResourceTTL, timeout)\n  {\n    TProvider::m_properties = provider->getProperties();\n  }\n\npublic:\n\n  /**\n   * Create shared Pool.\n   * @param provider - resource provider.\n   * @param maxResources - max resource count in the pool.\n   * @param maxResourceTTL - max time-to-live for unused resource in the pool.\n   * @param timeout - optional timeout on &l:Pool::get (); and &l:Pool::getAsync (); operations.\n   * @return - `std::shared_ptr` of `Pool`.\n   */\n  static std::shared_ptr<Pool> createShared(const std::shared_ptr<TProvider>& provider,\n                                            v_int64 maxResources,\n                                            const std::chrono::duration<v_int64, std::micro>& maxResourceTTL,\n                                            const std::chrono::duration<v_int64, std::micro>& timeout = std::chrono::microseconds::zero())\n  {\n    /* \"new\" is called directly to keep constructor private */\n    auto ptr = std::shared_ptr<Pool>(new Pool(provider, maxResources, maxResourceTTL.count(), timeout));\n    ptr->startCleanupTask(ptr);\n    return ptr;\n  }\n\n  /**\n   * Get resource.\n   * @return\n   */\n  provider::ResourceHandle<TResource> get() override {\n    return TPool::get(this->shared_from_this());\n  }\n\n  /**\n   * Get resource asynchronously.\n   * @return\n   */\n  async::CoroutineStarterForResult<const provider::ResourceHandle<TResource>&> getAsync() override {\n    return TPool::getAsync(this->shared_from_this());\n  }\n\n  /**\n   * Stop pool. <br>\n   * *Note: call to stop() may block.*\n   */\n  void stop() override {\n    TPool::stop();\n  }\n\n  /**\n   * Get pool resource count. Both acquired and available.\n   * @return\n   */\n  v_int64 getCounter() {\n    return TPool::getCounter();\n  }\n\n};\n```"]}
{"repo": "oatpp/oatpp", "name": "HandlerCoroutine", "language": "cpp", "path": "src/oatpp/web/server/api/ApiController.hpp", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: This template class is designed to handle HTTP requests in an asynchronous manner within a web server framework, enabling non-blocking operations that can scale to handle high loads efficiently.\n2. **Input**: It takes two parameters: a pointer to a controller object that manages the logic specific to the application, and a shared pointer to the incoming HTTP request.\n3. **Output**: It returns a shared pointer to an outgoing HTTP response after processing the request.\n4. **Procedure**: Upon instantiation, the class initializes with the provided controller and request. It then processes the incoming request asynchronously, leveraging the controller's logic to generate the appropriate HTTP response.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/web/server/api/Endpoint.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Endpoint.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace api {\n\nEndpoint::Info::Param::Param()\n  : name(nullptr)\n  , type(nullptr)\n{}\n\nEndpoint::Info::Param::Param(const oatpp::String& pName,\n...\n// Path: src/oatpp/web/server/api/ApiController.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_api_Controller_hpp\n#define oatpp_web_server_api_Controller_hpp\n\n#include \"./Endpoint.hpp\"\n\n#include \"oatpp/web/server/handler/AuthorizationHandler.hpp\"\n#include \"oatpp/web/server/handler/ErrorHandler.hpp\"\n#include \"oatpp/web/server/handler/AuthorizationHandler.hpp\"\n#include \"oatpp/web/protocol/http/incoming/Response.hpp\"\n#include \"oatpp/web/protocol/http/outgoing/Request.hpp\"\n#include \"oatpp/web/protocol/http/outgoing/ResponseFactory.hpp\"\n\n#include \"oatpp/core/utils/ConversionUtils.hpp\"\n\n#include <list>\n#include <unordered_map>\n\nnamespace oatpp { namespace web { namespace server { namespace api {\n\n/**\n * Class responsible for implementation and management of endpoints.<br>\n * For details see [ApiController](https://oatpp.io/docs/components/api-controller/).\n */\nclass ApiController : public oatpp::base::Countable {\nprotected:\n  typedef ApiController __ControllerType;\npublic:\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::ResponseFactory;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::ResponseFactory ResponseFactory;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Request;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Request IncomingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Request;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::Request OutgoingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Response;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Response IncomingResponse;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::Response OutgoingResponse;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Status;.\n   */\n  typedef oatpp::web::protocol::http::Status Status;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Header;.\n   */\n  typedef oatpp::web::protocol::http::Header Header;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::QueryParams;.\n   */\n  typedef oatpp::web::protocol::http::QueryParams QueryParams;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::server::HttpRequestHandler;.\n   */\n  typedef oatpp::web::server::HttpRequestHandler RequestHandler;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::server::handler::AuthorizationHandler;.\n   */\n  typedef oatpp::web::server::handler::AuthorizationHandler AuthorizationHandler;\n  \npublic:\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::ObjectMapper;.\n   */\n  typedef oatpp::data::mapping::ObjectMapper ObjectMapper;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::String;.\n   */\n  typedef oatpp::String String;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int8;.\n   */\n  typedef oatpp::Int8 Int8;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt8;.\n   */\n  typedef oatpp::UInt8 UInt8;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int16;.\n   */\n  typedef oatpp::Int16 Int16;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt16;.\n   */\n  typedef oatpp::UInt16 UInt16;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int32;.\n   */\n  typedef oatpp::Int32 Int32;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt32;.\n   */\n  typedef oatpp::UInt32 UInt32;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int64;.\n   */\n  typedef oatpp::Int64 Int64;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt64;.\n   */\n  typedef oatpp::UInt64 UInt64;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Float32;.\n   */\n  typedef oatpp::Float32 Float32;\n\n  /**\n   * Convenience typedef for &id:atpp::data::mapping::type::Float64;.\n   */\n  typedef oatpp::Float64 Float64;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Boolean;.\n   */\n  typedef oatpp::Boolean Boolean;\n\n  /*\n   * Convenience typedef for std::function<std::shared_ptr<Endpoint::Info>()>.\n   */\n  typedef std::function<std::shared_ptr<Endpoint::Info>()> EndpointInfoBuilder;\n\n  template <class T>\n  using Object = oatpp::Object<T>;\n\n  template <class T>\n  using List = oatpp::List<T>;\n\n  template <class Value>\n  using Fields = oatpp::Fields<Value>;\n\n  template <class T>\n  using Enum = oatpp::data::mapping::type::Enum<T>;\n\nprotected:\n  \n  /*\n   * Endpoint Coroutine base class\n   */\n  template<class CoroutineT, class ControllerT>\n  class HandlerCoroutine : public oatpp::async::CoroutineWithResult<CoroutineT, const std::shared_ptr<OutgoingResponse>&> {\n  public:\n    \n    \nHandlerCoroutine(ControllerT* pController, const std::shared_ptr<IncomingRequest>& pRequest)\n      : controller(pController)\n      , request(pRequest)\n    {}\n    \n    ControllerT* const controller;\n    std::shared_ptr<IncomingRequest> request;\n    \n  };\n  \n  /*\n   * Handler which subscribes to specific URL in Router and delegates calls endpoints \n   */\n  template<class T>\n  class Handler : public RequestHandler {\n  public:\n    typedef std::shared_ptr<OutgoingResponse> (T::*Method)(const std::shared_ptr<IncomingRequest>&);\n    typedef oatpp::async::CoroutineStarterForResult<const std::shared_ptr<OutgoingResponse>&>\n            (T::*MethodAsync)(const std::shared_ptr<IncomingRequest>&);\n\n  private:\n\n    class ErrorHandlingCoroutine : public oatpp::async::CoroutineWithResult<ErrorHandlingCoroutine, const std::shared_ptr<OutgoingResponse>&> {\n    private:\n      Handler* m_handler;\n      std::shared_ptr<IncomingRequest> m_request;\n    public:\n\n      ErrorHandlingCoroutine(Handler* handler, const std::shared_ptr<IncomingRequest>& request)\n        : m_handler(handler)\n        , m_request(request)\n      {}\n\n      async::Action act() override {\n        return (m_handler->m_controller->*m_handler->m_methodAsync)(m_request)\n          .callbackTo(&ErrorHandlingCoroutine::onResponse);\n      }\n\n      async::Action onResponse(const std::shared_ptr<OutgoingResponse>& response) {\n        return this->_return(response);\n      }\n\n      async::Action handleError(async::Error* error) override {\n        auto eptr = std::make_exception_ptr(*error);\n        auto response = m_handler->m_controller->m_errorHandler->handleError(eptr);\n        return this->_return(response);\n      }\n\n    };\n\n  private:\n    T* m_controller;\n    Method m_method;\n    MethodAsync m_methodAsync;\n  public:\n    Handler(T* controller, Method method, MethodAsync methodAsync)\n      : m_controller(controller)\n      , m_method(method)\n      , m_methodAsync(methodAsync)\n    {}\n  public:\n    \n    static std::shared_ptr<Handler> createShared(T* controller, Method method, MethodAsync methodAsync){\n      return std::make_shared<Handler>(controller, method, methodAsync);\n    }\n    \n    std::shared_ptr<OutgoingResponse> handle(const std::shared_ptr<IncomingRequest>& request) override {\n\n      if(m_method == nullptr) {\n        if(m_methodAsync == nullptr) {\n          throw protocol::http::HttpError(Status::CODE_500, \"[ApiController]: Error. Handler method is nullptr.\");\n        }\n        throw protocol::http::HttpError(Status::CODE_500, \"[ApiController]: Error. Non-async call to async endpoint.\");\n      }\n\n      try {\n        return (m_controller->*m_method)(request);\n      } catch (...) {\n        auto response = m_controller->handleError(std::current_exception());\n        if(response != nullptr) {\n          return response;\n        }\n\n        throw;\n      }\n      \n    }\n    \n    oatpp::async::CoroutineStarterForResult<const std::shared_ptr<OutgoingResponse>&>\n    handleAsync(const std::shared_ptr<protocol::http::incoming::Request>& request) override {\n\n      if(m_methodAsync == nullptr) {\n        if(m_method == nullptr) {\n          throw oatpp::web::protocol::http::HttpError(Status::CODE_500, \"[ApiController]: Error. Handler method is nullptr.\");\n        }\n        throw oatpp::web::protocol::http::HttpError(Status::CODE_500, \"[ApiController]: Error. Async call to non-async endpoint.\");\n      }\n\n      if(m_controller->m_errorHandler) {\n        return ErrorHandlingCoroutine::startForResult(this, request);\n      }\n\n      return (m_controller->*m_methodAsync)(request);\n\n    }\n\n    Method setMethod(Method method) {\n      auto prev = m_method;\n      m_method = method;\n      return prev;\n    }\n\n    Method getMethod() {\n      return m_method;\n    }\n\n    MethodAsync setMethodAsync(MethodAsync methodAsync) {\n      auto prev = m_methodAsync;\n      m_methodAsync = methodAsync;\n      return prev;\n    }\n\n    MethodAsync getMethodAsync() {\n      return m_methodAsync;\n    }\n    \n  };\n\nprotected:\n\n  /*\n   * Set endpoint info by endpoint name. (Endpoint name is the 'NAME' parameter of the ENDPOINT macro)\n   * Info should be set before call to addEndpointsToRouter();\n   */\n  void setEndpointInfo(const std::string& endpointName, const std::shared_ptr<Endpoint::Info>& info);\n\n  /*\n   * Get endpoint info by endpoint name. (Endpoint name is the 'NAME' parameter of the ENDPOINT macro)\n   */\n  std::shared_ptr<Endpoint::Info> getEndpointInfo(const std::string& endpointName);\n\n  /*\n   * Set endpoint Request handler.\n   * @param endpointName\n   * @param handler\n   */\n  void setEndpointHandler(const std::string& endpointName, const std::shared_ptr<RequestHandler>& handler);\n\n  /*\n   * Get endpoint Request handler.\n   * @param endpointName\n   * @return\n   */\n  std::shared_ptr<RequestHandler> getEndpointHandler(const std::string& endpointName);\n  \nprotected:\n  Endpoints m_endpoints;\n  std::shared_ptr<handler::ErrorHandler> m_errorHandler;\n  std::shared_ptr<handler::AuthorizationHandler> m_defaultAuthorizationHandler;\n  std::shared_ptr<oatpp::data::mapping::ObjectMapper> m_defaultObjectMapper;\n  std::unordered_map<std::string, std::shared_ptr<Endpoint::Info>> m_endpointInfo;\n  std::unordered_map<std::string, std::shared_ptr<RequestHandler>> m_endpointHandlers;\n  const oatpp::String m_routerPrefix;\npublic:\n  ApiController(const std::shared_ptr<oatpp::data::mapping::ObjectMapper>& defaultObjectMapper, const oatpp::String &routerPrefix = nullptr)\n    : m_defaultObjectMapper(defaultObjectMapper)\n    , m_routerPrefix(routerPrefix)\n  {\n\n  }\npublic:\n  \n  template<class T>\n  static std::shared_ptr<Endpoint> createEndpoint(Endpoints& endpoints,\n                                                  const std::shared_ptr<Handler<T>>& handler,\n                                                  const EndpointInfoBuilder& infoBuilder)\n  {\n    auto endpoint = Endpoint::createShared(handler, infoBuilder);\n    endpoints.append(endpoint);\n    return endpoint;\n  }\n  \n  /**\n   * Get list of Endpoints created via ENDPOINT macro\n   */\n  const Endpoints& getEndpoints();\n\n  /**\n   * Set error handler to handle errors that occur during the endpoint's execution\n   */\n  void setErrorHandler(const std::shared_ptr<handler::ErrorHandler>& errorHandler);\n\n  /**\n   * Handle the exception using the registered ErrorHandler or if no handler has been set, uses the DefaultErrorHandler::handleError\n   * @note Does not rethrow an exception anymore, OutgoingResponse has to be returned by the caller!\n   * @note If this handler fails to handle the exception, it will be handled by the connection handlers ErrorHandler.\n   */\n  std::shared_ptr<OutgoingResponse> handleError(const std::exception_ptr& exceptionPtr) const;\n\n  /**\n   * [under discussion]\n   * Set authorization handler to handle calls to handleAuthorization.\n   * Must be called before controller is added to a router or swagger-doc if an endpoint uses the AUTHORIZATION macro\n   */\n  void setDefaultAuthorizationHandler(const std::shared_ptr<handler::AuthorizationHandler>& authorizationHandler);\n\n  /**\n   * Get authorization handler.\n   * @return\n   */\n  std::shared_ptr<handler::AuthorizationHandler> getDefaultAuthorizationHandler();\n\n  /**\n   * [under discussion]\n   * Do not use it directly. This method is under discussion.\n   * Currently returns AuthorizationObject created by AuthorizationHandler or return DefaultAuthorizationObject by DefaultAuthorizationHandler if AuthorizationHandler is null\n   */\n  std::shared_ptr<handler::AuthorizationObject> handleDefaultAuthorization(const String &authHeader) const;\n  \n  const std::shared_ptr<oatpp::data::mapping::ObjectMapper>& getDefaultObjectMapper() const;\n  \n  // Helper methods\n  \n  std::shared_ptr<OutgoingResponse> createResponse(const Status& status,\n                                                   const oatpp::String& str) const;\n  \n  std::shared_ptr<OutgoingResponse> createResponse(const Status& status) const;\n\n  std::shared_ptr<OutgoingResponse> createDtoResponse(const Status& status,\n                                                      const oatpp::Void& dto,\n                                                      const std::shared_ptr<oatpp::data::mapping::ObjectMapper>& objectMapper) const;\n  \n  std::shared_ptr<OutgoingResponse> createDtoResponse(const Status& status,\n                                                      const oatpp::Void& dto) const;\n\npublic:\n\n  template<typename T>\n  struct TypeInterpretation {\n\n    static T fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n      (void) text;\n      success = false;\n      OATPP_LOGE(\"[oatpp::web::server::api::ApiController::TypeInterpretation::fromString()]\",\n                 \"Error. No conversion from '%s' to '%s' is defined.\", \"oatpp::String\", typeName->c_str())\n      throw std::runtime_error(\"[oatpp::web::server::api::ApiController::TypeInterpretation::fromString()]: Error. \"\n                               \"No conversion from 'oatpp::String' to '\" + *typeName + \"' is defined. \"\n                               \"Please define type conversion.\");\n    }\n\n  };\n\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::String> {\n  static oatpp::String fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    success = true;\n    return text;\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::Int8> {\n  static oatpp::Int8 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    //TODO: check the range and perhaps throw an exception if the variable doesn't fit\n    return static_cast<Int8::UnderlyingType>(utils::conversion::strToInt32(text, success));\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::UInt8> {\n  static oatpp::UInt8 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    //TODO: check the range and perhaps throw an exception if the variable doesn't fit\n    return static_cast<UInt8::UnderlyingType>(utils::conversion::strToUInt32(text, success));\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::Int16> {\n  static oatpp::Int16 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    //TODO: check the range and perhaps throw an exception if the variable doesn't fit\n    return static_cast<Int16::UnderlyingType>(utils::conversion::strToInt32(text, success));\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::UInt16> {\n  static oatpp::UInt16 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    //TODO: check the range and perhaps throw an exception if the variable doesn't fit\n    return static_cast<UInt16::UnderlyingType>(utils::conversion::strToUInt32(text, success));\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::Int32> {\n  static oatpp::Int32 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    return utils::conversion::strToInt32(text, success);\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::UInt32> {\n  static oatpp::UInt32 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    return utils::conversion::strToUInt32(text, success);\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::Int64> {\n  static oatpp::Int64 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    return utils::conversion::strToInt64(text, success);\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::UInt64> {\n  static oatpp::UInt64 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    return utils::conversion::strToUInt64(text, success);\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::Float32> {\n  static oatpp::Float32 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    return utils::conversion::strToFloat32(text, success);\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::Float64> {\n  static oatpp::Float64 fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    return utils::conversion::strToFloat64(text, success);\n  }\n};\n\ntemplate<>\nstruct ApiController::TypeInterpretation <oatpp::Boolean> {\n  static oatpp::Boolean fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    (void) typeName;\n    return utils::conversion::strToBool(text, success);\n  }\n};\n\ntemplate<class T, class I>\nstruct ApiController::TypeInterpretation <data::mapping::type::EnumObjectWrapper<T, I>> {\n\n  typedef data::mapping::type::EnumObjectWrapper<T, I> EnumOW;\n  typedef typename I::UnderlyingTypeObjectWrapper UTOW;\n\n  static EnumOW fromString(const oatpp::String& typeName, const oatpp::String& text, bool& success) {\n    const auto& parsedValue = ApiController::TypeInterpretation<UTOW>::fromString(typeName, text, success);\n    if(success) {\n      data::mapping::type::EnumInterpreterError error = data::mapping::type::EnumInterpreterError::OK;\n      const auto& result = I::fromInterpretation(parsedValue, error);\n      if(error == data::mapping::type::EnumInterpreterError::OK) {\n        return result.template cast<EnumOW>();\n      }\n      success = false;\n    }\n    return nullptr;\n  }\n\n};\n\n}}}}\n\n#endif /* oatpp_web_server_api_Controller_hpp */\n\n// Path: src/oatpp/web/server/api/ApiController.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"ApiController.hpp\"\n\n#include \"oatpp/web/server/handler/ErrorHandler.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace api {\n\nconst Endpoints& ApiController::getEndpoints() {\n  return m_endpoints;\n}\n\nvoid ApiController::setEndpointInfo(const std::string& endpointName, const std::shared_ptr<Endpoint::Info>& info){\n  m_endpointInfo[endpointName] = info;\n}\n\nstd::shared_ptr<Endpoint::Info> ApiController::getEndpointInfo(const std::string& endpointName) {\n  return m_endpointInfo[endpointName];\n}\n\nvoid ApiController::setEndpointHandler(const std::string& endpointName, const std::shared_ptr<RequestHandler>& handler) {\n  m_endpointHandlers[endpointName] = handler;\n}\n\nstd::shared_ptr<ApiController::RequestHandler> ApiController::getEndpointHandler(const std::string& endpointName) {\n  return m_endpointHandlers[endpointName];\n}\n\nvoid ApiController::setErrorHandler(const std::shared_ptr<handler::ErrorHandler>& errorHandler){\n  m_errorHandler = errorHandler;\n  if(!m_errorHandler) {\n    m_errorHandler = handler::DefaultErrorHandler::createShared();\n  }\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::handleError(const std::exception_ptr& exceptionPtr) const {\n\n  if(m_errorHandler) {\n    return m_errorHandler->handleError(exceptionPtr);\n  }\n\n  if(exceptionPtr) {\n    std::rethrow_exception(exceptionPtr);\n  }\n\n  throw std::runtime_error(\"[oatpp::web::server::api::ApiController::handleError()]: Error. 'exceptionPtr' is not set.\");\n\n}\n\nvoid ApiController::setDefaultAuthorizationHandler(const std::shared_ptr<handler::AuthorizationHandler>& authorizationHandler){\n  m_defaultAuthorizationHandler = authorizationHandler;\n}\n\nstd::shared_ptr<handler::AuthorizationHandler> ApiController::getDefaultAuthorizationHandler() {\n  return m_defaultAuthorizationHandler;\n}\n\nstd::shared_ptr<handler::AuthorizationObject> ApiController::handleDefaultAuthorization(const String &authHeader) const {\n  if(m_defaultAuthorizationHandler) {\n    return m_defaultAuthorizationHandler->handleAuthorization(authHeader);\n  }\n  // If Authorization is not setup on the server then it's 500\n  throw oatpp::web::protocol::http::HttpError(Status::CODE_500, \"Authorization is not setup.\");\n}\n\nconst std::shared_ptr<oatpp::data::mapping::ObjectMapper>& ApiController::getDefaultObjectMapper() const {\n  return m_defaultObjectMapper;\n}\n\n// Helper methods\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createResponse(const Status& status,\n                                                                               const oatpp::String& str) const {\n  return ResponseFactory::createResponse(status, str);\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createResponse(const ApiController::Status &status) const {\n  return ResponseFactory::createResponse(status);\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createDtoResponse(const Status& status,\n                                                                                  const oatpp::Void& dto,\n                                                                                  const std::shared_ptr<oatpp::data::mapping::ObjectMapper>& objectMapper) const {\n  return ResponseFactory::createResponse(status, dto, objectMapper);\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createDtoResponse(const Status& status,\n                                                                                  const oatpp::Void& dto) const {\n  return ResponseFactory::createResponse(status, dto, m_defaultObjectMapper);\n}\n\n}}}}\n\n// Path: src/oatpp/web/protocol/CommunicationError.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_protocol_CommunicationError_hpp\n#define oatpp_web_protocol_CommunicationError_hpp\n\n#include \"oatpp/core/IODefinitions.hpp\"\n\nnamespace oatpp { namespace web { namespace protocol {\n\n/**\n * Communication Error\n */\nclass CommunicationError : public std::runtime_error {\nprivate:\n  oatpp::v_io_size m_ioStatus;\n  oatpp::String m_message;\npublic:\n\n  /**\n   * Constructor.\n   * @param ioStatus - I/O error. See &id:oatpp::v_io_size;.\n   * @param message - error message.\n   */\n  CommunicationError(oatpp::v_io_size ioStatus, const oatpp::String& message);\n\n  /**\n   * Get I/O error. See &id:oatpp::v_io_size;.\n   * @return &id:oatpp::v_io_size;.\n   */\n  oatpp::v_io_size getIOStatus();\n\n  /**\n   * Get error message.\n   * @return - error message.\n   */\n  oatpp::String& getMessage();\n  \n};\n\n/**\n * Protocol Error Info.\n */\ntemplate<class Status>\nstruct ProtocolErrorInfo {\n\n  /**\n   * Default Constructor.\n   */\n  ProtocolErrorInfo()\n    : ioStatus(0)\n  {}\n\n  /**\n   * Constructor.\n   * @param pIOStatus - I/O level error. See &id:oatpp::v_io_size;.\n   * @param pStatus - configurable arbitrary data type.\n   */\n  ProtocolErrorInfo(oatpp::v_io_size pIOStatus, const Status& pStatus)\n    : ioStatus(pIOStatus)\n    , status(pStatus)\n  {}\n\n  /**\n   * Get I/O level error.\n   */\n  oatpp::v_io_size ioStatus;\n\n  /**\n   * Configurable arbitrary data type.\n   */\n  Status status;\n\n};\n\n/**\n * Protocol Error template.\n * @tparam Status - arbitrary data type.\n */\ntemplate<class Status>\nclass ProtocolError : public CommunicationError {\npublic:\n  /**\n   * Cenvenience typedef for ProtocolErrorInfo\n   */\n  typedef ProtocolErrorInfo<Status> Info;\nprivate:\n  Info m_info;\npublic:\n\n  /**\n   * Constructor.\n   * @param info - &l:ProtocolError::Info ;.\n   * @param message - error message.\n   */\n  ProtocolError(const Info& info, const oatpp::String& message)\n    : CommunicationError(info.ioStatus, message)\n    , m_info(info)\n  {}\n\n  /**\n   * Get error info.\n   * @return - error info.\n   */\n  Info getInfo() {\n    return m_info;\n  }\n  \n};\n\n\n/**\n * Protocol Error template.\n * @tparam Status - arbitrary data type.\n */\ntemplate<class Status>\nclass AsyncProtocolError : public oatpp::AsyncIOError {\npublic:\n  /**\n   * Cenvenience typedef for ProtocolErrorInfo\n   */\n  typedef ProtocolErrorInfo<Status> Info;\nprivate:\n  Info m_info;\n  oatpp::String m_message;\npublic:\n\n  /**\n   * Constructor.\n   * @param info - &l:ProtocolError::Info ;.\n   * @param message - error message.\n   */\n  AsyncProtocolError(const Info& info, const oatpp::String& message)\n    : oatpp::AsyncIOError(\"AsyncProtocolError\", info.ioStatus)\n    , m_info(info)\n    , m_message(message)\n  {}\n\n  /**\n   * Error message.\n   * @return - error message.\n   */\n  oatpp::String getMessage() {\n    return m_message;\n  }\n\n  /**\n   * Get error info.\n   * @return - error info.\n   */\n  Info getInfo() {\n    return m_info;\n  }\n\n};\n  \n}}}\n\n#endif /* oatpp_web_protocol_CommunicationError_hpp */\n\n// Path: src/oatpp/web/protocol/CommunicationError.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"CommunicationError.hpp\"\n\nnamespace oatpp { namespace web { namespace protocol {\n  \nCommunicationError::CommunicationError(oatpp::v_io_size ioStatus, const oatpp::String& message)\n  :std::runtime_error(*message)\n  , m_ioStatus(ioStatus)\n  , m_message(message)\n{}\n  \noatpp::v_io_size CommunicationError::getIOStatus() {\n  return m_ioStatus;\n}\n\noatpp::String& CommunicationError::getMessage(){\n  return m_message;\n}\n  \n}}}\n\n// Path: src/oatpp/web/protocol/http/Http.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_protocol_http_Http_hpp\n#define oatpp_web_protocol_http_Http_hpp\n\n#include \"oatpp/network/tcp/Connection.hpp\"\n\n#include \"oatpp/web/protocol/CommunicationError.hpp\"\n\n#include \"oatpp/core/parser/Caret.hpp\"\n#include \"oatpp/core/data/share/LazyStringMap.hpp\"\n#include \"oatpp/core/Types.hpp\"\n\n#include <unordered_map>\n#include <unordered_set>\n\nnamespace oatpp { namespace web { namespace protocol { namespace http {\n\n/**\n * Typedef for headers map. Headers map key is case-insensitive.\n * For more info see &id:oatpp::data::share::LazyStringMultimap;.\n */\ntypedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\n\n/**\n * Typedef for query parameters map.\n * For more info see &id:oatpp::data::share::LazyStringMultimap;.\n */\ntypedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabel> QueryParams;\n\n/**\n * Http status.\n */\nclass Status{\npublic:\n\n  /**\n   * Continue.\n   */\n  static const Status CODE_100;// Continue\n\n  /**\n   * Switching Protocols.\n   */\n  static const Status CODE_101;// Switching\n\n  /**\n   * Processing.\n   */\n  static const Status CODE_102;// Processing\n\n  /**\n   * OK.\n   */\n  static const Status CODE_200;// OK\n\n  /**\n   * Created.\n   */\n  static const Status CODE_201;// Created\n\n  /**\n   * Accepted.\n   */\n  static const Status CODE_202;// Accepted\n\n  /**\n   * Non-Authoritative Information.\n   */\n  static const Status CODE_203;// Non-Authoritative Information\n\n  /**\n   * No Content.\n   */\n  static const Status CODE_204;// No Content\n\n  /**\n   * Reset Content.\n   */\n  static const Status CODE_205;// Reset Content\n\n  /**\n   * Partial Content.\n   */\n  static const Status CODE_206;// Partial Content\n\n  /**\n   * Multi-Status.\n   */\n  static const Status CODE_207;// Multi-Status\n\n  /**\n   * IM Used.\n   */\n  static const Status CODE_226;// IM Used\n\n  /**\n   * Multiple Choices.\n   */\n  static const Status CODE_300;// Multiple Choices\n\n  /**\n   * Moved Permanently.\n   */\n  static const Status CODE_301;// Moved Permanently\n\n  /**\n   * Moved Temporarily.\n   */\n  static const Status CODE_302;// Moved Temporarily\n\n  /**\n   * See Other.\n   */\n  static const Status CODE_303;// See Other\n\n  /**\n   * Not Modified.\n   */\n  static const Status CODE_304;// Not Modified\n\n  /**\n   * Use Proxy.\n   */\n  static const Status CODE_305;// Use Proxy\n\n  /**\n   * Reserved.\n   */\n  static const Status CODE_306;// Reserved\n\n  /**\n   * Temporary Redirect.\n   */\n  static const Status CODE_307;// Temporary Redirect\n\n  /**\n   * Bad Request.\n   */\n  static const Status CODE_400;// Bad Request\n\n  /**\n   * Unauthorized.\n   */\n  static const Status CODE_401;// Unauthorized\n\n  /**\n   * Payment Required.\n   */\n  static const Status CODE_402;// Payment Required\n\n  /**\n   * Forbidden.\n   */\n  static const Status CODE_403;// Forbidden\n\n  /**\n   * Not Found.\n   */\n  static const Status CODE_404;// Not Found\n\n  /**\n   * Method Not Allowed.\n   */\n  static const Status CODE_405;// Method Not Allowed\n\n  /**\n   * Not Acceptable.\n   */\n  static const Status CODE_406;// Not Acceptable\n\n  /**\n   * Proxy Authentication Required.\n   */\n  static const Status CODE_407;// Proxy Authentication Required\n\n  /**\n   * Request Timeout.\n   */\n  static const Status CODE_408;// Request Timeout\n\n  /**\n   * Conflict.\n   */\n  static const Status CODE_409;// Conflict\n\n  /**\n   * Gone\n   */\n  static const Status CODE_410;// Gone\n\n  /**\n   * Length Required.\n   */\n  static const Status CODE_411;// Length Required\n\n  /**\n   * Precondition Failed.\n   */\n  static const Status CODE_412;// Precondition Failed\n\n  /**\n   * Request Entity Too Large.\n   */\n  static const Status CODE_413;// Request Entity Too Large\n\n  /**\n   * Request-URI Too Large.\n   */\n  static const Status CODE_414;// Request-URI Too Large\n\n  /**\n   * Unsupported Media Type.\n   */\n  static const Status CODE_415;// Unsupported Media Type\n\n  /**\n   * Requested Range Not Satisfiable.\n   */\n  static const Status CODE_416;// Requested Range Not Satisfiable\n\n  /**\n   * Expectation Failed.\n   */\n  static const Status CODE_417;// Expectation Failed\n\n  /**\n   * I'm a Teapot (rfc7168 2.3.3)\n   */\n  static const Status CODE_418;// I'm a teapot\n\n  /**\n   * Unprocessable Entity.\n   */\n  static const Status CODE_422;// Unprocessable Entity\n\n  /**\n   * Locked.\n   */\n  static const Status CODE_423;// Locked\n\n  /**\n   * Failed Dependency.\n   */\n  static const Status CODE_424;// Failed Dependency\n\n  /**\n   * Unordered Collection.\n   */\n  static const Status CODE_425;// Unordered Collection\n\n  /**\n   * Upgrade Required.\n   */\n  static const Status CODE_426;// Upgrade Required\n\n  /**\n   * Precondition Required.\n   */\n  static const Status CODE_428;// Precondition Required\n\n  /**\n   * Too Many Requests.\n   */\n  static const Status CODE_429;// Too Many Requests\n\n  /**\n   * Request Header Fields Too Large.\n   */\n  static const Status CODE_431;// Request Header Fields Too Large\n\n  /**\n   * Requested host unavailable.\n   */\n  static const Status CODE_434;// Requested host unavailable\n\n  /**\n   * Close connection withot sending headers.\n   */\n  static const Status CODE_444;// Close connection withot sending headers\n\n  /**\n   * Retry With.\n   */\n  static const Status CODE_449;// Retry With\n\n  /**\n   * Unavailable For Legal Reasons.\n   */\n  static const Status CODE_451;// Unavailable For Legal Reasons\n\n  /**\n   * Internal Server Error.\n   */\n  static const Status CODE_500;// Internal Server Error\n\n  /**\n   * Not Implemented.\n   */\n  static const Status CODE_501;// Not Implemented\n\n  /**\n   * Bad Gateway.\n   */\n  static const Status CODE_502;// Bad Gateway\n\n  /**\n   * Service Unavailable.\n   */\n  static const Status CODE_503;// Service Unavailable\n\n  /**\n   * Gateway Timeout.\n   */\n  static const Status CODE_504;// Gateway Timeout\n\n  /**\n   * HTTP Version Not Supported.\n   */\n  static const Status CODE_505;// HTTP Version Not Supported\n\n  /**\n   * Variant Also Negotiates.\n   */\n  static const Status CODE_506;// Variant Also Negotiates\n\n  /**\n   * Insufficient Storage.\n   */\n  static const Status CODE_507;// Insufficient Storage\n\n  /**\n   * Loop Detected.\n   */\n  static const Status CODE_508;// Loop Detected\n\n  /**\n   * Bandwidth Limit Exceeded.\n   */\n  static const Status CODE_509;// Bandwidth Limit Exceeded\n\n  /**\n   * Not Extended.\n   */\n  static const Status CODE_510;// Not Extended\n\n  /**\n   * Network Authentication Required.\n   */\n  static const Status CODE_511;// Network Authentication Required\n\n  /**\n   * Constructor.\n   */\n  Status()\n    : code(0)\n    , description(nullptr)\n  {}\n\n  /**\n   * Constructor.\n   * @param pCode - status code.\n   * @param pDesc - description.\n   */\n  Status(v_int32 pCode, const char* pDesc)\n    : code(pCode)\n    , description(pDesc)\n  {}\n\n  /**\n   * Status code.\n   */\n  v_int32 code;\n\n  /**\n   * Description.\n   */\n  const char* description;\n  \n  bool operator==(const Status& other) const {\n    return this->code == other.code;\n  }\n  \n  bool operator!=(const Status& other) const {\n    return this->code != other.code;\n  }\n  \n};\n\n/**\n * HttpError extends &id:oatpp::web::protocol::ProtocolError;<&l:Status;>.\n */\nclass HttpError : public protocol::ProtocolError<Status> {\nprivate:\n  Headers m_headers;\npublic:\n\n  /**\n   * Constructor.\n   * @param info\n   * @param message\n   */\n  HttpError(const Info& info, const oatpp::String& message)\n    : protocol::ProtocolError<Status>(info, message)\n  {}\n\n  /**\n   * Constructor.\n   * @param status\n   * @param message\n   */\n  HttpError(const Status& status, const oatpp::String& message)\n    : protocol::ProtocolError<Status>(Info(0, status), message)\n  {}\n\n  /**\n   * Constructor.\n   * @param status\n   * @param message\n   * @param headers\n   */\n  HttpError(const Status& status, const oatpp::String& message, const Headers& headers)\n    : protocol::ProtocolError<Status>(Info(0, status), message)\n    , m_headers(headers)\n  {}\n\n  /**\n   * Get headers\n   * @return\n   */\n  const Headers& getHeaders() const {\n    return m_headers;\n  }\n  \n};\n\n/**\n * Throw &l:HttpError; if assertion failed.\n * @param COND - boolean statement. If evaluates to false - throw error.\n * @param STATUS - &l:Status;.\n * @param MESSAGE - String message.\n */\n#define OATPP_ASSERT_HTTP(COND, STATUS, MESSAGE) \\\nif(!(COND)) { throw oatpp::web::protocol::http::HttpError(STATUS, MESSAGE); }\n\n/**\n * Collection of HTTP Header constants.\n */\nclass Header {\npublic:\n\n  /**\n   * Possible values for headers.\n   */\n  class Value {\n  public:\n    static const char* const CONNECTION_CLOSE;\n    static const char* const CONNECTION_KEEP_ALIVE;\n    static const char* const CONNECTION_UPGRADE;\n    \n    static const char* const SERVER;\n    static const char* const USER_AGENT;\n    \n    static const char* const TRANSFER_ENCODING_CHUNKED;\n    static const char* const CONTENT_TYPE_APPLICATION_JSON;\n\n    static const char* const EXPECT_100_CONTINUE;\n  };\npublic:\n  static const char* const ACCEPT;              // \"Accept\"\n  static const char* const AUTHORIZATION;       // \"Authorization\"\n  static const char* const WWW_AUTHENTICATE;    // \"WWW-Authenticate\"\n  static const char* const CONNECTION;          // \"Connection\"\n  static const char* const TRANSFER_ENCODING;   // \"Transfer-Encoding\"\n  static const char* const CONTENT_ENCODING;    // \"Content-Encoding\"\n  static const char* const CONTENT_LENGTH;      // \"Content-Length\"\n  static const char* const CONTENT_TYPE;        // \"Content-Type\"\n  static const char* const CONTENT_RANGE;       // \"Content-Range\"\n  static const char* const RANGE;               // \"Range\"\n  static const char* const HOST;                // \"Host\"\n  static const char* const USER_AGENT;          // \"User-Agent\"\n  static const char* const SERVER;              // \"Server\"\n  static const char* const UPGRADE;             // \"Upgrade\"\n  static const char* const CORS_ORIGIN;         // Access-Control-Allow-Origin\n  static const char* const CORS_METHODS;        // Access-Control-Allow-Methods\n  static const char* const CORS_HEADERS;        // Access-Control-Allow-Headers\n  static const char* const CORS_MAX_AGE;        // Access-Control-Max-Age\n  static const char* const ACCEPT_ENCODING;     // Accept-Encoding\n  static const char* const EXPECT;              // Expect\n};\n  \nclass Range {\npublic:\n  static const char* const UNIT_BYTES;\nprivate:\n  Range()\n    : units(nullptr)\n  {}\npublic:\n  \n  Range(const oatpp::String& pUnits,\n        v_int64 pStart,\n        v_int64 pEnd)\n    : units(pUnits)\n    , start(pStart)\n    , end(pEnd)\n  {}\n  \n  oatpp::String units;\n  v_int64 start;\n  v_int64 end;\n  \n  oatpp::String toString() const;\n  \n  bool isValid() const {\n    return units.get() != nullptr;\n  }\n  \n  static Range parse(oatpp::parser::Caret& caret);\n  static Range parse(const oatpp::String& str);\n  \n};\n  \nclass ContentRange {\npublic:\n  static const char* const UNIT_BYTES;\nprivate:\n  ContentRange()\n    : units(nullptr)\n  {}\npublic:\n  \n  ContentRange(const oatpp::String& pUnits,\n               v_int64 pStart,\n               v_int64 pEnd,\n               v_int64 pSize,\n               bool pIsSizeKnown)\n    : units(pUnits)\n    , start(pStart)\n    , end(pEnd)\n    , size(pSize)\n    , isSizeKnown(pIsSizeKnown)\n  {}\n  \n  oatpp::String units;\n  v_int64 start;\n  v_int64 end;\n  v_int64 size;\n  bool isSizeKnown;\n  \n  oatpp::String toString() const;\n  \n  bool isValid() const {\n    return units.get() != nullptr;\n  }\n  \n  static ContentRange parse(oatpp::parser::Caret& caret);\n  static ContentRange parse(const oatpp::String& str);\n  \n};\n\n/**\n * Struct representing HTTP request starting line.\n * Example request starting line: `GET /path/to/resource/ HTTP/1.1`.\n */\nstruct RequestStartingLine {\n  /**\n   * Method as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel method; // GET, POST ...\n\n  /**\n   * Path as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel path;\n\n  /**\n   * Protocol as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel protocol;\n};\n\n/**\n * Struct representing HTTP response starting line.\n * Example response starting line: `HTTP/1.1 200 OK`.\n */\nstruct ResponseStartingLine {\n  /**\n   * Protocol as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel protocol;\n\n  /**\n   * Status code as v_int32.\n   */\n  v_int32 statusCode;\n\n  /**\n   * Description as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel description;\n};\n\n/**\n * Data contained in the value of one header.\n */\nstruct HeaderValueData {\n\n  /**\n   * value tokens.\n   */\n  std::unordered_set<data::share::StringKeyLabelCI> tokens;\n\n  /**\n   * Title params.\n   */\n  std::unordered_map<data::share::StringKeyLabelCI, data::share::StringKeyLabel> titleParams;\n\n  /**\n   * Get title parm value by key.\n   * @param key\n   * @return\n   */\n  oatpp::String getTitleParamValue(const data::share::StringKeyLabelCI& key) const;\n\n};\n\n/**\n * Oatpp Http parser.\n */\nclass Parser {\nprivate:\n  static oatpp::data::share::StringKeyLabelCI parseHeaderNameLabel(const std::shared_ptr<std::string>& headersText,\n                                                                   oatpp::parser::Caret& caret);\npublic:\n\n  /**\n   * Parse &l:RequestStartingLine;.\n   * @param line - &l:RequestStartingLine;. Values will be set to line's fields.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:RequestStartingLine; fields. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseRequestStartingLine(RequestStartingLine& line,\n                                       const std::shared_ptr<std::string>& headersText,\n                                       oatpp::parser::Caret& caret,\n                                       Status& error);\n\n  /**\n   * Parse &l:ResponseStartingLine;.\n   * @param line - &l:ResponseStartingLine;. Values will be set to line's fields.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:ResponseStartingLine; fields. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseResponseStartingLine(ResponseStartingLine& line,\n                                        const std::shared_ptr<std::string>& headersText,\n                                        oatpp::parser::Caret& caret,\n                                        Status& error);\n\n  /**\n   * Parse one header line. Example of the header line:\n   * `\"Content-Type: application/json\\r\\n\"`.\n   * @param headers - &l:Headers; map to put parsed header to.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:Headers; values. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseOneHeader(Headers& headers,\n                             const std::shared_ptr<std::string>& headersText,\n                             oatpp::parser::Caret& caret,\n                             Status& error);\n\n  /**\n   * Parse HTTP headers to &l:Headers; map.\n   * @param headers - &l:Headers; map to put parsed headers to.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:Headers; values. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseHeaders(Headers& headers,\n                           const std::shared_ptr<std::string>& headersText,\n                           oatpp::parser::Caret& caret,\n                           Status& error);\n\n  /**\n   * Parse data that is contained in a one header.\n   * @param data - out. parsed data.\n   * @param headerValue - header value string.\n   * @param separator - subvalues separator.\n   */\n  static void parseHeaderValueData(HeaderValueData& data, const oatpp::data::share::StringKeyLabel& headerValue, char separator);\n\n};\n\n/**\n * Http utils.\n */\nclass Utils {\npublic:\n\n  /**\n   * Write headers map to stream.\n   * @param headers\n   * @param stream\n   */\n  static void writeHeaders(const Headers& headers, data::stream::ConsistentOutputStream* stream);\n\n};\n  \n}}}}\n\nnamespace std {\n  \n  template<>\n  struct hash<oatpp::web::protocol::http::Status> {\n    \n    typedef oatpp::web::protocol::http::Status argument_type;\n    typedef v_uint64 result_type;\n    \n    result_type operator()(oatpp::web::protocol::http::Status const& s) const noexcept {\n      return static_cast<result_type>(s.code);\n    }\n    \n  };\n}\n\n#endif /* oatpp_web_protocol_http_Http_hpp */\n\n// Path: src/oatpp/web/protocol/http/Http.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"./Http.hpp\"\n\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n#include \"oatpp/core/utils/ConversionUtils.hpp\"\n\nnamespace oatpp { namespace web { namespace protocol { namespace http {\n  \nconst Status Status::CODE_100(100, \"Continue\");\nconst Status Status::CODE_101(101, \"Switching\");\nconst Status Status::CODE_102(102, \"Processing\");\n\n...\n", "needle_token_start": 2342, "needle_token_end": 2387, "code_context_ntokens": 15914, "output": ["```cpp\ntemplate <class CoroutineT, class ControllerT>\nclass HandlerCoroutine : public oatpp::async::CoroutineWithResult<CoroutineT, const std::shared_ptr<OutgoingResponse>&> {\npublic:\n  HandlerCoroutine(ControllerT* pController, const std::shared_ptr<IncomingRequest>& pRequest)\n    : controller(pController)\n    , request(pRequest)\n  {}\n  \n  ControllerT* const controller;\n  std::shared_ptr<IncomingRequest> request;\n};\n```"]}
{"repo": "oatpp/oatpp", "name": "ProtocolError", "language": "cpp", "path": "src/oatpp/web/protocol/CommunicationError.hpp", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The purpose is to handle and report errors related to communication protocols by encapsulating both I/O status and a configurable status type, providing detailed context about the error.\n2. **Input**: The input includes an I/O status code indicating the nature of the I/O error and a configurable status type that can hold additional error information specific to the protocol.\n3. **Output**: There is no direct output; however, the class provides methods to retrieve the I/O status and the additional status information encapsulated within the error.\n4. **Procedure**: Upon instantiation, the class is initialized with the I/O status and the additional status. It inherits from a base error class, which takes the I/O status and a message as parameters. The class stores these inputs and makes them accessible through methods for further error handling or logging.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/web/server/api/ApiController.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_api_Controller_hpp\n#define oatpp_web_server_api_Controller_hpp\n\n#include \"./Endpoint.hpp\"\n\n#include \"oatpp/web/server/handler/AuthorizationHandler.hpp\"\n#include \"oatpp/web/server/handler/ErrorHandler.hpp\"\n#include \"oatpp/web/server/handler/AuthorizationHandler.hpp\"\n#include \"oatpp/web/protocol/http/incoming/Response.hpp\"\n#include \"oatpp/web/protocol/http/outgoing/Request.hpp\"\n#include \"oatpp/web/protocol/http/outgoing/ResponseFactory.hpp\"\n\n#include \"oatpp/core/utils/ConversionUtils.hpp\"\n\n#include <list>\n#include <unordered_map>\n\nnamespace oatpp { namespace web { namespace server { namespace api {\n\n/**\n * Class responsible for implementation and management of endpoints.<br>\n * For details see [ApiController](https://oatpp.io/docs/components/api-controller/).\n */\nclass ApiController : public oatpp::base::Countable {\nprotected:\n  typedef ApiController __ControllerType;\npublic:\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::ResponseFactory;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::ResponseFactory ResponseFactory;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Request;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Request IncomingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Request;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::Request OutgoingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Response;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Response IncomingResponse;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::Response OutgoingResponse;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Status;.\n   */\n  typedef oatpp::web::protocol::http::Status Status;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Header;.\n   */\n  typedef oatpp::web::protocol::http::Header Header;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::QueryParams;.\n   */\n  typedef oatpp::web::protocol::http::QueryParams QueryParams;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::server::HttpRequestHandler;.\n   */\n  typedef oatpp::web::server::HttpRequestHandler RequestHandler;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::server::handler::AuthorizationHandler;.\n   */\n  typedef oatpp::web::server::handler::AuthorizationHandler AuthorizationHandler;\n  \npublic:\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::ObjectMapper;.\n   */\n  typedef oatpp::data::mapping::ObjectMapper ObjectMapper;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::String;.\n   */\n  typedef oatpp::String String;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int8;.\n   */\n  typedef oatpp::Int8 Int8;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt8;.\n   */\n  typedef oatpp::UInt8 UInt8;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int16;.\n   */\n  typedef oatpp::Int16 Int16;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt16;.\n   */\n  typedef oatpp::UInt16 UInt16;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int32;.\n   */\n  typedef oatpp::Int32 Int32;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt32;.\n   */\n  typedef oatpp::UInt32 UInt32;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Int64;.\n   */\n  typedef oatpp::Int64 Int64;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::UInt64;.\n   */\n  typedef oatpp::UInt64 UInt64;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Float32;.\n   */\n  typedef oatpp::Float32 Float32;\n\n  /**\n   * Convenience typedef for &id:atpp::data::mapping::type::Float64;.\n   */\n  typedef oatpp::Float64 Float64;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::mapping::type::Boolean;.\n...\n// Path: src/oatpp/web/server/api/ApiController.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"ApiController.hpp\"\n\n#include \"oatpp/web/server/handler/ErrorHandler.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace api {\n\nconst Endpoints& ApiController::getEndpoints() {\n  return m_endpoints;\n}\n\nvoid ApiController::setEndpointInfo(const std::string& endpointName, const std::shared_ptr<Endpoint::Info>& info){\n  m_endpointInfo[endpointName] = info;\n}\n\nstd::shared_ptr<Endpoint::Info> ApiController::getEndpointInfo(const std::string& endpointName) {\n  return m_endpointInfo[endpointName];\n}\n\nvoid ApiController::setEndpointHandler(const std::string& endpointName, const std::shared_ptr<RequestHandler>& handler) {\n  m_endpointHandlers[endpointName] = handler;\n}\n\nstd::shared_ptr<ApiController::RequestHandler> ApiController::getEndpointHandler(const std::string& endpointName) {\n  return m_endpointHandlers[endpointName];\n}\n\nvoid ApiController::setErrorHandler(const std::shared_ptr<handler::ErrorHandler>& errorHandler){\n  m_errorHandler = errorHandler;\n  if(!m_errorHandler) {\n    m_errorHandler = handler::DefaultErrorHandler::createShared();\n  }\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::handleError(const std::exception_ptr& exceptionPtr) const {\n\n  if(m_errorHandler) {\n    return m_errorHandler->handleError(exceptionPtr);\n  }\n\n  if(exceptionPtr) {\n    std::rethrow_exception(exceptionPtr);\n  }\n\n  throw std::runtime_error(\"[oatpp::web::server::api::ApiController::handleError()]: Error. 'exceptionPtr' is not set.\");\n\n}\n\nvoid ApiController::setDefaultAuthorizationHandler(const std::shared_ptr<handler::AuthorizationHandler>& authorizationHandler){\n  m_defaultAuthorizationHandler = authorizationHandler;\n}\n\nstd::shared_ptr<handler::AuthorizationHandler> ApiController::getDefaultAuthorizationHandler() {\n  return m_defaultAuthorizationHandler;\n}\n\nstd::shared_ptr<handler::AuthorizationObject> ApiController::handleDefaultAuthorization(const String &authHeader) const {\n  if(m_defaultAuthorizationHandler) {\n    return m_defaultAuthorizationHandler->handleAuthorization(authHeader);\n  }\n  // If Authorization is not setup on the server then it's 500\n  throw oatpp::web::protocol::http::HttpError(Status::CODE_500, \"Authorization is not setup.\");\n}\n\nconst std::shared_ptr<oatpp::data::mapping::ObjectMapper>& ApiController::getDefaultObjectMapper() const {\n  return m_defaultObjectMapper;\n}\n\n// Helper methods\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createResponse(const Status& status,\n                                                                               const oatpp::String& str) const {\n  return ResponseFactory::createResponse(status, str);\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createResponse(const ApiController::Status &status) const {\n  return ResponseFactory::createResponse(status);\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createDtoResponse(const Status& status,\n                                                                                  const oatpp::Void& dto,\n                                                                                  const std::shared_ptr<oatpp::data::mapping::ObjectMapper>& objectMapper) const {\n  return ResponseFactory::createResponse(status, dto, objectMapper);\n}\n\nstd::shared_ptr<ApiController::OutgoingResponse> ApiController::createDtoResponse(const Status& status,\n                                                                                  const oatpp::Void& dto) const {\n  return ResponseFactory::createResponse(status, dto, m_defaultObjectMapper);\n}\n\n}}}}\n\n// Path: src/oatpp/web/protocol/CommunicationError.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_protocol_CommunicationError_hpp\n#define oatpp_web_protocol_CommunicationError_hpp\n\n#include \"oatpp/core/IODefinitions.hpp\"\n\nnamespace oatpp { namespace web { namespace protocol {\n\n/**\n * Communication Error\n */\nclass CommunicationError : public std::runtime_error {\nprivate:\n  oatpp::v_io_size m_ioStatus;\n  oatpp::String m_message;\npublic:\n\n  /**\n   * Constructor.\n   * @param ioStatus - I/O error. See &id:oatpp::v_io_size;.\n   * @param message - error message.\n   */\n  CommunicationError(oatpp::v_io_size ioStatus, const oatpp::String& message);\n\n  /**\n   * Get I/O error. See &id:oatpp::v_io_size;.\n   * @return &id:oatpp::v_io_size;.\n   */\n  oatpp::v_io_size getIOStatus();\n\n  /**\n   * Get error message.\n   * @return - error message.\n   */\n  oatpp::String& getMessage();\n  \n};\n\n/**\n * Protocol Error Info.\n */\ntemplate<class Status>\nstruct ProtocolErrorInfo {\n\n  /**\n   * Default Constructor.\n   */\n  ProtocolErrorInfo()\n    : ioStatus(0)\n  {}\n\n  /**\n   * Constructor.\n   * @param pIOStatus - I/O level error. See &id:oatpp::v_io_size;.\n   * @param pStatus - configurable arbitrary data type.\n   */\n  ProtocolErrorInfo(oatpp::v_io_size pIOStatus, const Status& pStatus)\n    : ioStatus(pIOStatus)\n    , status(pStatus)\n  {}\n\n  /**\n   * Get I/O level error.\n   */\n  oatpp::v_io_size ioStatus;\n\n  /**\n   * Configurable arbitrary data type.\n   */\n  Status status;\n\n};\n\n/**\n * Protocol Error template.\n * @tparam Status - arbitrary data type.\n */\ntemplate<class Status>\nclass ProtocolError : public CommunicationError {\npublic:\n  /**\n   * Cenvenience typedef for ProtocolErrorInfo\n   */\n  typedef ProtocolErrorInfo<Status> Info;\nprivate:\n  Info m_info;\npublic:\n\n  /**\n   * Constructor.\n   * @param info - &l:ProtocolError::Info ;.\n   * @param message - error message.\n   */\n  \nProtocolError(const Info& info, const oatpp::String& message)\n    : CommunicationError(info.ioStatus, message)\n    , m_info(info)\n  {}\n\n  /**\n   * Get error info.\n   * @return - error info.\n   */\n  Info getInfo() {\n    return m_info;\n  }\n  \n};\n\n\n/**\n * Protocol Error template.\n * @tparam Status - arbitrary data type.\n */\ntemplate<class Status>\nclass AsyncProtocolError : public oatpp::AsyncIOError {\npublic:\n  /**\n   * Cenvenience typedef for ProtocolErrorInfo\n   */\n  typedef ProtocolErrorInfo<Status> Info;\nprivate:\n  Info m_info;\n  oatpp::String m_message;\npublic:\n\n  /**\n   * Constructor.\n   * @param info - &l:ProtocolError::Info ;.\n   * @param message - error message.\n   */\n  AsyncProtocolError(const Info& info, const oatpp::String& message)\n    : oatpp::AsyncIOError(\"AsyncProtocolError\", info.ioStatus)\n    , m_info(info)\n    , m_message(message)\n  {}\n\n  /**\n   * Error message.\n   * @return - error message.\n   */\n  oatpp::String getMessage() {\n    return m_message;\n  }\n\n  /**\n   * Get error info.\n   * @return - error info.\n   */\n  Info getInfo() {\n    return m_info;\n  }\n\n};\n  \n}}}\n\n#endif /* oatpp_web_protocol_CommunicationError_hpp */\n\n// Path: src/oatpp/web/protocol/CommunicationError.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"CommunicationError.hpp\"\n\nnamespace oatpp { namespace web { namespace protocol {\n  \nCommunicationError::CommunicationError(oatpp::v_io_size ioStatus, const oatpp::String& message)\n  :std::runtime_error(*message)\n  , m_ioStatus(ioStatus)\n  , m_message(message)\n{}\n  \noatpp::v_io_size CommunicationError::getIOStatus() {\n  return m_ioStatus;\n}\n\noatpp::String& CommunicationError::getMessage(){\n  return m_message;\n}\n  \n}}}\n\n// Path: src/oatpp/web/protocol/http/Http.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_protocol_http_Http_hpp\n#define oatpp_web_protocol_http_Http_hpp\n\n#include \"oatpp/network/tcp/Connection.hpp\"\n\n#include \"oatpp/web/protocol/CommunicationError.hpp\"\n\n#include \"oatpp/core/parser/Caret.hpp\"\n#include \"oatpp/core/data/share/LazyStringMap.hpp\"\n#include \"oatpp/core/Types.hpp\"\n\n#include <unordered_map>\n#include <unordered_set>\n\nnamespace oatpp { namespace web { namespace protocol { namespace http {\n\n/**\n * Typedef for headers map. Headers map key is case-insensitive.\n * For more info see &id:oatpp::data::share::LazyStringMultimap;.\n */\ntypedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\n\n/**\n * Typedef for query parameters map.\n * For more info see &id:oatpp::data::share::LazyStringMultimap;.\n */\ntypedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabel> QueryParams;\n\n/**\n * Http status.\n */\nclass Status{\npublic:\n\n  /**\n   * Continue.\n   */\n  static const Status CODE_100;// Continue\n\n  /**\n   * Switching Protocols.\n   */\n  static const Status CODE_101;// Switching\n\n  /**\n   * Processing.\n   */\n  static const Status CODE_102;// Processing\n\n  /**\n   * OK.\n   */\n  static const Status CODE_200;// OK\n\n  /**\n   * Created.\n   */\n  static const Status CODE_201;// Created\n\n  /**\n   * Accepted.\n   */\n  static const Status CODE_202;// Accepted\n\n  /**\n   * Non-Authoritative Information.\n   */\n  static const Status CODE_203;// Non-Authoritative Information\n\n  /**\n   * No Content.\n   */\n  static const Status CODE_204;// No Content\n\n  /**\n   * Reset Content.\n   */\n  static const Status CODE_205;// Reset Content\n\n  /**\n   * Partial Content.\n   */\n  static const Status CODE_206;// Partial Content\n\n  /**\n   * Multi-Status.\n   */\n  static const Status CODE_207;// Multi-Status\n\n  /**\n   * IM Used.\n   */\n  static const Status CODE_226;// IM Used\n\n  /**\n   * Multiple Choices.\n   */\n  static const Status CODE_300;// Multiple Choices\n\n  /**\n   * Moved Permanently.\n   */\n  static const Status CODE_301;// Moved Permanently\n\n  /**\n   * Moved Temporarily.\n   */\n  static const Status CODE_302;// Moved Temporarily\n\n  /**\n   * See Other.\n   */\n  static const Status CODE_303;// See Other\n\n  /**\n   * Not Modified.\n   */\n  static const Status CODE_304;// Not Modified\n\n  /**\n   * Use Proxy.\n   */\n  static const Status CODE_305;// Use Proxy\n\n  /**\n   * Reserved.\n   */\n  static const Status CODE_306;// Reserved\n\n  /**\n   * Temporary Redirect.\n   */\n  static const Status CODE_307;// Temporary Redirect\n\n  /**\n   * Bad Request.\n   */\n  static const Status CODE_400;// Bad Request\n\n  /**\n   * Unauthorized.\n   */\n  static const Status CODE_401;// Unauthorized\n\n  /**\n   * Payment Required.\n   */\n  static const Status CODE_402;// Payment Required\n\n  /**\n   * Forbidden.\n   */\n  static const Status CODE_403;// Forbidden\n\n  /**\n   * Not Found.\n   */\n  static const Status CODE_404;// Not Found\n\n  /**\n   * Method Not Allowed.\n   */\n  static const Status CODE_405;// Method Not Allowed\n\n  /**\n   * Not Acceptable.\n   */\n  static const Status CODE_406;// Not Acceptable\n\n  /**\n   * Proxy Authentication Required.\n   */\n  static const Status CODE_407;// Proxy Authentication Required\n\n  /**\n   * Request Timeout.\n   */\n  static const Status CODE_408;// Request Timeout\n\n  /**\n   * Conflict.\n   */\n  static const Status CODE_409;// Conflict\n\n  /**\n   * Gone\n   */\n  static const Status CODE_410;// Gone\n\n  /**\n   * Length Required.\n   */\n  static const Status CODE_411;// Length Required\n\n  /**\n   * Precondition Failed.\n   */\n  static const Status CODE_412;// Precondition Failed\n\n  /**\n   * Request Entity Too Large.\n   */\n  static const Status CODE_413;// Request Entity Too Large\n\n  /**\n   * Request-URI Too Large.\n   */\n  static const Status CODE_414;// Request-URI Too Large\n\n  /**\n   * Unsupported Media Type.\n   */\n  static const Status CODE_415;// Unsupported Media Type\n\n  /**\n   * Requested Range Not Satisfiable.\n   */\n  static const Status CODE_416;// Requested Range Not Satisfiable\n\n  /**\n   * Expectation Failed.\n   */\n  static const Status CODE_417;// Expectation Failed\n\n  /**\n   * I'm a Teapot (rfc7168 2.3.3)\n   */\n  static const Status CODE_418;// I'm a teapot\n\n  /**\n   * Unprocessable Entity.\n   */\n  static const Status CODE_422;// Unprocessable Entity\n\n  /**\n   * Locked.\n   */\n  static const Status CODE_423;// Locked\n\n  /**\n   * Failed Dependency.\n   */\n  static const Status CODE_424;// Failed Dependency\n\n  /**\n   * Unordered Collection.\n   */\n  static const Status CODE_425;// Unordered Collection\n\n  /**\n   * Upgrade Required.\n   */\n  static const Status CODE_426;// Upgrade Required\n\n  /**\n   * Precondition Required.\n   */\n  static const Status CODE_428;// Precondition Required\n\n  /**\n   * Too Many Requests.\n   */\n  static const Status CODE_429;// Too Many Requests\n\n  /**\n   * Request Header Fields Too Large.\n   */\n  static const Status CODE_431;// Request Header Fields Too Large\n\n  /**\n   * Requested host unavailable.\n   */\n  static const Status CODE_434;// Requested host unavailable\n\n  /**\n   * Close connection withot sending headers.\n   */\n  static const Status CODE_444;// Close connection withot sending headers\n\n  /**\n   * Retry With.\n   */\n  static const Status CODE_449;// Retry With\n\n  /**\n   * Unavailable For Legal Reasons.\n   */\n  static const Status CODE_451;// Unavailable For Legal Reasons\n\n  /**\n   * Internal Server Error.\n   */\n  static const Status CODE_500;// Internal Server Error\n\n  /**\n   * Not Implemented.\n   */\n  static const Status CODE_501;// Not Implemented\n\n  /**\n   * Bad Gateway.\n   */\n  static const Status CODE_502;// Bad Gateway\n\n  /**\n   * Service Unavailable.\n   */\n  static const Status CODE_503;// Service Unavailable\n\n  /**\n   * Gateway Timeout.\n   */\n  static const Status CODE_504;// Gateway Timeout\n\n  /**\n   * HTTP Version Not Supported.\n   */\n  static const Status CODE_505;// HTTP Version Not Supported\n\n  /**\n   * Variant Also Negotiates.\n   */\n  static const Status CODE_506;// Variant Also Negotiates\n\n  /**\n   * Insufficient Storage.\n   */\n  static const Status CODE_507;// Insufficient Storage\n\n  /**\n   * Loop Detected.\n   */\n  static const Status CODE_508;// Loop Detected\n\n  /**\n   * Bandwidth Limit Exceeded.\n   */\n  static const Status CODE_509;// Bandwidth Limit Exceeded\n\n  /**\n   * Not Extended.\n   */\n  static const Status CODE_510;// Not Extended\n\n  /**\n   * Network Authentication Required.\n   */\n  static const Status CODE_511;// Network Authentication Required\n\n  /**\n   * Constructor.\n   */\n  Status()\n    : code(0)\n    , description(nullptr)\n  {}\n\n  /**\n   * Constructor.\n   * @param pCode - status code.\n   * @param pDesc - description.\n   */\n  Status(v_int32 pCode, const char* pDesc)\n    : code(pCode)\n    , description(pDesc)\n  {}\n\n  /**\n   * Status code.\n   */\n  v_int32 code;\n\n  /**\n   * Description.\n   */\n  const char* description;\n  \n  bool operator==(const Status& other) const {\n    return this->code == other.code;\n  }\n  \n  bool operator!=(const Status& other) const {\n    return this->code != other.code;\n  }\n  \n};\n\n/**\n * HttpError extends &id:oatpp::web::protocol::ProtocolError;<&l:Status;>.\n */\nclass HttpError : public protocol::ProtocolError<Status> {\nprivate:\n  Headers m_headers;\npublic:\n\n  /**\n   * Constructor.\n   * @param info\n   * @param message\n   */\n  HttpError(const Info& info, const oatpp::String& message)\n    : protocol::ProtocolError<Status>(info, message)\n  {}\n\n  /**\n   * Constructor.\n   * @param status\n   * @param message\n   */\n  HttpError(const Status& status, const oatpp::String& message)\n    : protocol::ProtocolError<Status>(Info(0, status), message)\n  {}\n\n  /**\n   * Constructor.\n   * @param status\n   * @param message\n   * @param headers\n   */\n  HttpError(const Status& status, const oatpp::String& message, const Headers& headers)\n    : protocol::ProtocolError<Status>(Info(0, status), message)\n    , m_headers(headers)\n  {}\n\n  /**\n   * Get headers\n   * @return\n   */\n  const Headers& getHeaders() const {\n    return m_headers;\n  }\n  \n};\n\n/**\n * Throw &l:HttpError; if assertion failed.\n * @param COND - boolean statement. If evaluates to false - throw error.\n * @param STATUS - &l:Status;.\n * @param MESSAGE - String message.\n */\n#define OATPP_ASSERT_HTTP(COND, STATUS, MESSAGE) \\\nif(!(COND)) { throw oatpp::web::protocol::http::HttpError(STATUS, MESSAGE); }\n\n/**\n * Collection of HTTP Header constants.\n */\nclass Header {\npublic:\n\n  /**\n   * Possible values for headers.\n   */\n  class Value {\n  public:\n    static const char* const CONNECTION_CLOSE;\n    static const char* const CONNECTION_KEEP_ALIVE;\n    static const char* const CONNECTION_UPGRADE;\n    \n    static const char* const SERVER;\n    static const char* const USER_AGENT;\n    \n    static const char* const TRANSFER_ENCODING_CHUNKED;\n    static const char* const CONTENT_TYPE_APPLICATION_JSON;\n\n    static const char* const EXPECT_100_CONTINUE;\n  };\npublic:\n  static const char* const ACCEPT;              // \"Accept\"\n  static const char* const AUTHORIZATION;       // \"Authorization\"\n  static const char* const WWW_AUTHENTICATE;    // \"WWW-Authenticate\"\n  static const char* const CONNECTION;          // \"Connection\"\n  static const char* const TRANSFER_ENCODING;   // \"Transfer-Encoding\"\n  static const char* const CONTENT_ENCODING;    // \"Content-Encoding\"\n  static const char* const CONTENT_LENGTH;      // \"Content-Length\"\n  static const char* const CONTENT_TYPE;        // \"Content-Type\"\n  static const char* const CONTENT_RANGE;       // \"Content-Range\"\n  static const char* const RANGE;               // \"Range\"\n  static const char* const HOST;                // \"Host\"\n  static const char* const USER_AGENT;          // \"User-Agent\"\n  static const char* const SERVER;              // \"Server\"\n  static const char* const UPGRADE;             // \"Upgrade\"\n  static const char* const CORS_ORIGIN;         // Access-Control-Allow-Origin\n  static const char* const CORS_METHODS;        // Access-Control-Allow-Methods\n  static const char* const CORS_HEADERS;        // Access-Control-Allow-Headers\n  static const char* const CORS_MAX_AGE;        // Access-Control-Max-Age\n  static const char* const ACCEPT_ENCODING;     // Accept-Encoding\n  static const char* const EXPECT;              // Expect\n};\n  \nclass Range {\npublic:\n  static const char* const UNIT_BYTES;\nprivate:\n  Range()\n    : units(nullptr)\n  {}\npublic:\n  \n  Range(const oatpp::String& pUnits,\n        v_int64 pStart,\n        v_int64 pEnd)\n    : units(pUnits)\n    , start(pStart)\n    , end(pEnd)\n  {}\n  \n  oatpp::String units;\n  v_int64 start;\n  v_int64 end;\n  \n  oatpp::String toString() const;\n  \n  bool isValid() const {\n    return units.get() != nullptr;\n  }\n  \n  static Range parse(oatpp::parser::Caret& caret);\n  static Range parse(const oatpp::String& str);\n  \n};\n  \nclass ContentRange {\npublic:\n  static const char* const UNIT_BYTES;\nprivate:\n  ContentRange()\n    : units(nullptr)\n  {}\npublic:\n  \n  ContentRange(const oatpp::String& pUnits,\n               v_int64 pStart,\n               v_int64 pEnd,\n               v_int64 pSize,\n               bool pIsSizeKnown)\n    : units(pUnits)\n    , start(pStart)\n    , end(pEnd)\n    , size(pSize)\n    , isSizeKnown(pIsSizeKnown)\n  {}\n  \n  oatpp::String units;\n  v_int64 start;\n  v_int64 end;\n  v_int64 size;\n  bool isSizeKnown;\n  \n  oatpp::String toString() const;\n  \n  bool isValid() const {\n    return units.get() != nullptr;\n  }\n  \n  static ContentRange parse(oatpp::parser::Caret& caret);\n  static ContentRange parse(const oatpp::String& str);\n  \n};\n\n/**\n * Struct representing HTTP request starting line.\n * Example request starting line: `GET /path/to/resource/ HTTP/1.1`.\n */\nstruct RequestStartingLine {\n  /**\n   * Method as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel method; // GET, POST ...\n\n  /**\n   * Path as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel path;\n\n  /**\n   * Protocol as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel protocol;\n};\n\n/**\n * Struct representing HTTP response starting line.\n * Example response starting line: `HTTP/1.1 200 OK`.\n */\nstruct ResponseStartingLine {\n  /**\n   * Protocol as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel protocol;\n\n  /**\n   * Status code as v_int32.\n   */\n  v_int32 statusCode;\n\n  /**\n   * Description as &id:oatpp::data::share::StringKeyLabel;.\n   */\n  oatpp::data::share::StringKeyLabel description;\n};\n\n/**\n * Data contained in the value of one header.\n */\nstruct HeaderValueData {\n\n  /**\n   * value tokens.\n   */\n  std::unordered_set<data::share::StringKeyLabelCI> tokens;\n\n  /**\n   * Title params.\n   */\n  std::unordered_map<data::share::StringKeyLabelCI, data::share::StringKeyLabel> titleParams;\n\n  /**\n   * Get title parm value by key.\n   * @param key\n   * @return\n   */\n  oatpp::String getTitleParamValue(const data::share::StringKeyLabelCI& key) const;\n\n};\n\n/**\n * Oatpp Http parser.\n */\nclass Parser {\nprivate:\n  static oatpp::data::share::StringKeyLabelCI parseHeaderNameLabel(const std::shared_ptr<std::string>& headersText,\n                                                                   oatpp::parser::Caret& caret);\npublic:\n\n  /**\n   * Parse &l:RequestStartingLine;.\n   * @param line - &l:RequestStartingLine;. Values will be set to line's fields.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:RequestStartingLine; fields. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseRequestStartingLine(RequestStartingLine& line,\n                                       const std::shared_ptr<std::string>& headersText,\n                                       oatpp::parser::Caret& caret,\n                                       Status& error);\n\n  /**\n   * Parse &l:ResponseStartingLine;.\n   * @param line - &l:ResponseStartingLine;. Values will be set to line's fields.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:ResponseStartingLine; fields. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseResponseStartingLine(ResponseStartingLine& line,\n                                        const std::shared_ptr<std::string>& headersText,\n                                        oatpp::parser::Caret& caret,\n                                        Status& error);\n\n  /**\n   * Parse one header line. Example of the header line:\n   * `\"Content-Type: application/json\\r\\n\"`.\n   * @param headers - &l:Headers; map to put parsed header to.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:Headers; values. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseOneHeader(Headers& headers,\n                             const std::shared_ptr<std::string>& headersText,\n                             oatpp::parser::Caret& caret,\n                             Status& error);\n\n  /**\n   * Parse HTTP headers to &l:Headers; map.\n   * @param headers - &l:Headers; map to put parsed headers to.\n   * @param headersText - `std::shared_ptr` to `std::string` needed as a \"memory handle\" for\n   * &l:Headers; values. See &id:oatpp::data::share::MemoryLabel;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param error - out parameter &l:Status;.\n   */\n  static void parseHeaders(Headers& headers,\n                           const std::shared_ptr<std::string>& headersText,\n                           oatpp::parser::Caret& caret,\n                           Status& error);\n\n  /**\n   * Parse data that is contained in a one header.\n   * @param data - out. parsed data.\n   * @param headerValue - header value string.\n   * @param separator - subvalues separator.\n   */\n  static void parseHeaderValueData(HeaderValueData& data, const oatpp::data::share::StringKeyLabel& headerValue, char separator);\n\n};\n\n/**\n * Http utils.\n */\nclass Utils {\npublic:\n\n  /**\n   * Write headers map to stream.\n   * @param headers\n   * @param stream\n   */\n  static void writeHeaders(const Headers& headers, data::stream::ConsistentOutputStream* stream);\n\n};\n  \n}}}}\n\nnamespace std {\n  \n  template<>\n  struct hash<oatpp::web::protocol::http::Status> {\n    \n    typedef oatpp::web::protocol::http::Status argument_type;\n    typedef v_uint64 result_type;\n    \n    result_type operator()(oatpp::web::protocol::http::Status const& s) const noexcept {\n      return static_cast<result_type>(s.code);\n    }\n    \n  };\n}\n\n#endif /* oatpp_web_protocol_http_Http_hpp */\n\n// Path: src/oatpp/web/protocol/http/Http.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"./Http.hpp\"\n\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n#include \"oatpp/core/utils/ConversionUtils.hpp\"\n\nnamespace oatpp { namespace web { namespace protocol { namespace http {\n  \nconst Status Status::CODE_100(100, \"Continue\");\nconst Status Status::CODE_101(101, \"Switching\");\nconst Status Status::CODE_102(102, \"Processing\");\n\nconst Status Status::CODE_200(200, \"OK\");\nconst Status Status::CODE_201(201, \"Created\");\nconst Status Status::CODE_202(202, \"Accepted\");\nconst Status Status::CODE_203(203, \"Non-Authoritative Information\");\nconst Status Status::CODE_204(204, \"No Content\");\nconst Status Status::CODE_205(205, \"Reset Content\");\nconst Status Status::CODE_206(206, \"Partial Content\");\nconst Status Status::CODE_207(207, \"Multi-Status\");\nconst Status Status::CODE_226(226, \"IM Used\");\n\nconst Status Status::CODE_300(300, \"Multiple Choices\");\nconst Status Status::CODE_301(301, \"Moved Permanently\");\nconst Status Status::CODE_302(302, \"Moved Temporarily\");\nconst Status Status::CODE_303(303, \"See Other\");\nconst Status Status::CODE_304(304, \"Not Modified\");\nconst Status Status::CODE_305(305, \"Use Proxy\");\nconst Status Status::CODE_306(306, \"Reserved\");\nconst Status Status::CODE_307(307, \"Temporary Redirect\");\n\nconst Status Status::CODE_400(400, \"Bad Request\");\nconst Status Status::CODE_401(401, \"Unauthorized\");\nconst Status Status::CODE_402(402, \"Payment Required\");\nconst Status Status::CODE_403(403, \"Forbidden\");\nconst Status Status::CODE_404(404, \"Not Found\");\nconst Status Status::CODE_405(405, \"Method Not Allowed\");\nconst Status Status::CODE_406(406, \"Not Acceptable\");\nconst Status Status::CODE_407(407, \"Proxy Authentication Required\");\nconst Status Status::CODE_408(408, \"Request Timeout\");\nconst Status Status::CODE_409(409, \"Conflict\");\nconst Status Status::CODE_410(410, \"Gone\");\nconst Status Status::CODE_411(411, \"Length Required\");\nconst Status Status::CODE_412(412, \"Precondition Failed\");\nconst Status Status::CODE_413(413, \"Request Entity Too Large\");\nconst Status Status::CODE_414(414, \"Request-URI Too Large\");\nconst Status Status::CODE_415(415, \"Unsupported Media Type\");\nconst Status Status::CODE_416(416, \"Requested Range Not Satisfiable\");\nconst Status Status::CODE_417(417, \"Expectation Failed\");\nconst Status Status::CODE_418(418, \"I'm a Teapot\");\nconst Status Status::CODE_422(422, \"Unprocessable Entity\");\nconst Status Status::CODE_423(423, \"Locked\");\nconst Status Status::CODE_424(424, \"Failed Dependency\");\nconst Status Status::CODE_425(425, \"Unordered Collection\");\nconst Status Status::CODE_426(426, \"Upgrade Required\");\nconst Status Status::CODE_428(428, \"Precondition Required\");\nconst Status Status::CODE_429(429, \"Too Many Requests\");\nconst Status Status::CODE_431(431, \"Request Header Fields Too Large\");\nconst Status Status::CODE_434(434, \"Requested host unavailable\");\nconst Status Status::CODE_444(444, \"Close connection withot sending headers\");\nconst Status Status::CODE_449(449, \"Retry With\");\nconst Status Status::CODE_451(451, \"Unavailable For Legal Reasons\");\n\nconst Status Status::CODE_500(500, \"Internal Server Error\");\nconst Status Status::CODE_501(501, \"Not Implemented\");\nconst Status Status::CODE_502(502, \"Bad Gateway\");\nconst Status Status::CODE_503(503, \"Service Unavailable\");\nconst Status Status::CODE_504(504, \"Gateway Timeout\");\nconst Status Status::CODE_505(505, \"HTTP Version Not Supported\");\nconst Status Status::CODE_506(506, \"Variant Also Negotiates\");\nconst Status Status::CODE_507(507, \"Insufficient Storage\");\nconst Status Status::CODE_508(508, \"Loop Detected\");\nconst Status Status::CODE_509(509, \"Bandwidth Limit Exceeded\");\nconst Status Status::CODE_510(510, \"Not Extended\");\nconst Status Status::CODE_511(511, \"Network Authentication Required\");\n\nconst char* const Header::Value::CONNECTION_CLOSE = \"close\";\nconst char* const Header::Value::CONNECTION_KEEP_ALIVE = \"keep-alive\";\nconst char* const Header::Value::CONNECTION_UPGRADE = \"Upgrade\";\n  \nconst char* const Header::Value::SERVER = \"oatpp/\" OATPP_VERSION;\nconst char* const Header::Value::USER_AGENT = \"oatpp/\" OATPP_VERSION;\n  \nconst char* const Header::Value::TRANSFER_ENCODING_CHUNKED = \"chunked\";\n  \nconst char* const Header::Value::CONTENT_TYPE_APPLICATION_JSON = \"application/json\";\n\nconst char* const Header::Value::EXPECT_100_CONTINUE = \"100-continue\";\n  \nconst char* const Header::ACCEPT = \"Accept\";\nconst char* const Header::AUTHORIZATION = \"Authorization\";\nconst char* const Header::WWW_AUTHENTICATE = \"WWW-Authenticate\";\nconst char* const Header::CONNECTION = \"Connection\";\nconst char* const Header::TRANSFER_ENCODING = \"Transfer-Encoding\";\nconst char* const Header::CONTENT_ENCODING = \"Content-Encoding\";\nconst char* const Header::CONTENT_LENGTH = \"Content-Length\";\nconst char* const Header::CONTENT_TYPE = \"Content-Type\";\nconst char* const Header::CONTENT_RANGE = \"Content-Range\";\nconst char* const Header::RANGE = \"Range\";\nconst char* const Header::HOST = \"Host\";\nconst char* const Header::USER_AGENT = \"User-Agent\";\nconst char* const Header::SERVER = \"Server\";\nconst char* const Header::UPGRADE = \"Upgrade\";\n\n\nconst char* const Header::CORS_ORIGIN = \"Access-Control-Allow-Origin\";\nconst char* const Header::CORS_METHODS = \"Access-Control-Allow-Methods\";\nconst char* const Header::CORS_HEADERS = \"Access-Control-Allow-Headers\";\nconst char* const Header::CORS_MAX_AGE = \"Access-Control-Max-Age\";\n\nconst char* const Header::ACCEPT_ENCODING = \"Accept-Encoding\";\n\nconst char* const Header::EXPECT = \"Expect\";\n\nconst char* const Range::UNIT_BYTES = \"bytes\";\nconst char* const ContentRange::UNIT_BYTES = \"bytes\";\n  \noatpp::String Range::toString() const {\n  data::stream::BufferOutputStream stream(256);\n  stream.writeSimple(units->data(), static_cast<v_buff_size>(units->size()));\n  stream.writeSimple(\"=\", 1);\n  stream.writeAsString(start);\n  stream.writeSimple(\"-\", 1);\n  stream.writeAsString(end);\n  return stream.toString();\n}\n\nRange Range::parse(oatpp::parser::Caret& caret) {\n\n  auto unitsLabel = caret.putLabel();\n  if(caret.findChar('=')) {\n    unitsLabel.end();\n    caret.inc();\n  } else {\n    caret.setError(\"'=' - expected\");\n    return Range();\n  }\n\n  auto startLabel = caret.putLabel();\n  if(caret.findChar('-')) {\n    startLabel.end();\n    caret.inc();\n  } else {\n    caret.setError(\"'-' - expected\");\n    return Range();\n  }\n\n  auto endLabel = caret.putLabel();\n  caret.findRN();\n  endLabel.end();\n\n  auto start = oatpp::utils::conversion::strToInt64(startLabel.getData());\n  auto end = oatpp::utils::conversion::strToInt64(endLabel.getData());\n  return Range(unitsLabel.toString(), start, end);\n  \n}\n\nRange Range::parse(const oatpp::String& str) {\n  oatpp::parser::Caret caret(str);\n  return parse(caret);\n}\n\noatpp::String ContentRange::toString() const {\n  data::stream::BufferOutputStream stream(256);\n  stream.writeSimple(units->data(), static_cast<v_buff_size>(units->size()));\n  stream.writeSimple(\" \", 1);\n  stream.writeAsString(start);\n  stream.writeSimple(\"-\", 1);\n  stream.writeAsString(end);\n  stream.writeSimple(\"/\", 1);\n  if(isSizeKnown) {\n    stream.writeAsString(size);\n  } else {\n    stream.writeSimple(\"*\", 1);\n  }\n  return stream.toString();\n}\n\nContentRange ContentRange::parse(oatpp::parser::Caret& caret) {\n\n  auto unitsLabel = caret.putLabel();\n  if(caret.findChar(' ')) {\n    unitsLabel.end();\n    caret.inc();\n  } else {\n    caret.setError(\"' ' - expected\");\n    return ContentRange();\n  }\n\n  auto startLabel = caret.putLabel();\n  if(caret.findChar('-')) {\n    startLabel.end();\n    caret.inc();\n  } else {\n    caret.setError(\"'-' - expected\");\n    return ContentRange();\n  }\n\n  auto endLabel = caret.putLabel();\n  if(caret.findChar('/')) {\n    endLabel.end();\n    caret.inc();\n  } else {\n    caret.setError(\"'/' - expected\");\n    return ContentRange();\n  }\n\n  auto sizeLabel = caret.putLabel();\n  caret.findRN();\n  sizeLabel.end();\n  \n  v_int64 start = oatpp::utils::conversion::strToInt64(startLabel.getData());\n  v_int64 end = oatpp::utils::conversion::strToInt64(endLabel.getData());\n  v_int64 size = 0;\n  bool isSizeKnown = false;\n  if(sizeLabel.getData()[0] != '*') {\n    isSizeKnown = true;\n    size = oatpp::utils::conversion::strToInt64(sizeLabel.getData());\n  }\n  \n  return ContentRange(unitsLabel.toString(), start, end, size, isSizeKnown);\n  \n}\n\nContentRange ContentRange::parse(const oatpp::String& str) {\n  oatpp::parser::Caret caret(str);\n  return parse(caret);\n}\n\n\noatpp::String HeaderValueData::getTitleParamValue(const data::share::StringKeyLabelCI& key) const {\n  auto it = titleParams.find(key);\n  if(it != titleParams.end()) {\n    return it->second.toString();\n  }\n  return nullptr;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// Parser\n  \noatpp::data::share::StringKeyLabelCI Parser::parseHeaderNameLabel(const std::shared_ptr<std::string>& headersText,\n                                                                  oatpp::parser::Caret& caret) {\n  const char* data = caret.getData();\n  for(v_buff_size i = caret.getPosition(); i < caret.getDataSize(); i++) {\n    v_char8 a = static_cast<v_char8>(data[i]);\n    if(a == ':' || a == ' '){\n      oatpp::data::share::StringKeyLabelCI label(headersText, &data[caret.getPosition()], i - caret.getPosition());\n      caret.setPosition(i);\n      return label;\n      \n    }\n  }\n  return oatpp::data::share::StringKeyLabelCI(nullptr, nullptr, 0);\n}\n  \nvoid Parser::parseRequestStartingLine(RequestStartingLine& line,\n                                      const std::shared_ptr<std::string>& headersText,\n                                      oatpp::parser::Caret& caret,\n                                      Status& error) {\n\n  auto methodLabel = caret.putLabel();\n  if(caret.findChar(' ')){\n    line.method = oatpp::data::share::StringKeyLabel(headersText, methodLabel.getData(), methodLabel.getSize());\n    caret.inc();\n  } else {\n    error = Status::CODE_400;\n    return;\n  }\n\n  auto pathLabel = caret.putLabel();\n  if(caret.findChar(' ')){\n    line.path = oatpp::data::share::StringKeyLabel(headersText, pathLabel.getData(), pathLabel.getSize());\n    caret.inc();\n  } else {\n    error = Status::CODE_400;\n    return;\n  }\n\n  auto protocolLabel = caret.putLabel();\n  if(caret.findRN()){\n    line.protocol = oatpp::data::share::StringKeyLabel(headersText, protocolLabel.getData(), protocolLabel.getSize());\n    caret.skipRN();\n  } else {\n    error = Status::CODE_400;\n    return;\n  }\n  \n}\n  \nvoid Parser::parseResponseStartingLine(ResponseStartingLine& line,\n                                       const std::shared_ptr<std::string>& headersText,\n                                       oatpp::parser::Caret& caret,\n                                       Status& error) {\n\n  auto protocolLabel = caret.putLabel();\n  if(caret.findChar(' ')){\n    line.protocol = oatpp::data::share::StringKeyLabel(headersText, protocolLabel.getData(), protocolLabel.getSize());\n    caret.inc();\n  } else {\n    error = Status::CODE_400;\n    return;\n  }\n\n  line.statusCode = static_cast<v_int32>(caret.parseInt());\n\n  auto descriptionLabel = caret.putLabel();\n  if(caret.findRN()){\n    line.description = oatpp::data::share::StringKeyLabel(headersText, descriptionLabel.getData(), descriptionLabel.getSize());\n    caret.skipRN();\n  } else {\n    error = Status::CODE_400;\n    return;\n  }\n  \n}\n  \nvoid Parser::parseOneHeader(Headers& headers,\n                            const std::shared_ptr<std::string>& headersText,\n                            oatpp::parser::Caret& caret,\n                            Status& error)\n{\n  caret.skipChar(' ');\n  auto name = parseHeaderNameLabel(headersText, caret);\n  if(name.getData() != nullptr) {\n    caret.skipChar(' ');\n    if(!caret.canContinueAtChar(':', 1)) {\n      error = Status::CODE_400;\n      return;\n    }\n    caret.skipChar(' ');\n    v_buff_size valuePos0 = caret.getPosition();\n    caret.findRN();\n    headers.put_LockFree(name, oatpp::data::share::StringKeyLabel(headersText, &caret.getData()[valuePos0], caret.getPosition() - valuePos0));\n    caret.skipRN();\n  } else {\n    error = Status::CODE_431;\n    return;\n  }\n}\n\nvoid Parser::parseHeaders(Headers& headers,\n                          const std::shared_ptr<std::string>& headersText,\n                          oatpp::parser::Caret& caret,\n                          Status& error)\n{\n  \n  while (!caret.isAtRN()) {\n    parseOneHeader(headers, headersText, caret, error);\n    if(error.code != 0) {\n      return;\n    }\n  }\n  \n  caret.skipRN();\n  \n}\n\nvoid Parser::parseHeaderValueData(HeaderValueData& data, const oatpp::data::share::StringKeyLabel& headerValue, char separator) {\n\n  oatpp::parser::Caret caret(reinterpret_cast<const char*>(headerValue.getData()), headerValue.getSize());\n\n  const char charSet[5] = {' ', '=', separator, '\\r', '\\n'};\n  const char charSet2[4] = {' ', separator, '\\r', '\\n'};\n\n  while (caret.canContinue()) {\n\n    caret.skipChar(' ');\n\n    auto label = caret.putLabel();\n    auto res = caret.findCharFromSet(charSet, 5);\n\n    if (res == '=') {\n\n      data::share::StringKeyLabelCI key(headerValue.getMemoryHandle(), label.getData(), label.getSize());\n      caret.inc();\n\n      if (caret.isAtChar('\"')) {\n        label = caret.parseStringEnclosed('\"', '\"', '\\\\');\n      } else if (caret.isAtChar('\\'')) {\n        label = caret.parseStringEnclosed('\\'', '\\'', '\\\\');\n      } else {\n        label = caret.putLabel();\n        caret.findCharFromSet(charSet2, 4);\n      }\n\n      data.titleParams[key] = data::share::StringKeyLabel(headerValue.getMemoryHandle(), label.getData(),\n                                                          label.getSize());\n\n    } else {\n      data.tokens.insert(\n        data::share::StringKeyLabelCI(headerValue.getMemoryHandle(), label.getData(), label.getSize()));\n    }\n\n    if (caret.isAtCharFromSet(\"\\r\\n\", 2)) {\n      break;\n    } else if (caret.isAtChar(separator)) {\n      caret.inc();\n    }\n\n  }\n\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// Utils\n\nvoid Utils::writeHeaders(const Headers& headers, data::stream::ConsistentOutputStream* stream) {\n\n...\n", "needle_token_start": 3928, "needle_token_end": 3972, "code_context_ntokens": 15952, "output": ["```cpp\nclass CommunicationError : public std::runtime_error {\nprivate:\n  oatpp::v_io_size m_ioStatus;\n  oatpp::String m_message;\npublic:\n\n  CommunicationError(oatpp::v_io_size ioStatus, const oatpp::String& message);\n\n  oatpp::v_io_size getIOStatus();\n\n  oatpp::String& getMessage();\n  \n};\n```"]}
{"repo": "oatpp/oatpp", "name": "serializePrimitive", "language": "cpp", "path": "src/oatpp/parser/json/mapping/Serializer.hpp", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a basic data type into a JSON-compatible string format and write it to an output stream. If the data is null, it writes \"null\" to the stream.\n2. **Input**: It accepts three parameters: a pointer to the serializer instance, a pointer to a consistent output stream where the data will be written, and a polymorphic wrapper around the data to be serialized.\n3. **Output**: There is no return value; the function writes the serialized data directly to the provided output stream.\n4. **Procedure**: The function first checks if the provided data is not null. If it is not null, it casts the data to its original type and writes it as a string to the output stream. If the data is null, it writes the string \"null\" to the output stream.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/parser/json/Utils.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Utils.hpp\"\n\n#include \"oatpp/encoding/Unicode.hpp\"\n#include \"oatpp/encoding/Hex.hpp\"\n\nnamespace oatpp { namespace parser { namespace json{\n\nv_buff_size Utils::calcEscapedStringSize(const char* data, v_buff_size size, v_buff_size& safeSize, v_uint32 flags) {\n  v_buff_size result = 0;\n  v_buff_size i = 0;\n  safeSize = size;\n  while (i < size) {\n    v_char8 a = static_cast<v_char8>(data[i]);\n    if(a < 32) {\n      i ++;\n\n      switch (a) {\n\n        case '\\b':\n        case '\\f':\n        case '\\n':\n        case '\\r':\n        case '\\t': result += 2; break; // '\\n'\n\n        default:\n          result += 6; // '\\uFFFF' - 6 chars\n          break;\n\n      }\n\n    } else if(a < 128){\n      i ++;\n\n      switch (a) {\n        case '\\\"':\n        case '\\\\': result += 2; break; // '\\/'\n\n        case '/':\n          result ++;\n          if((flags & FLAG_ESCAPE_SOLIDUS) > 0) result ++;\n          break;\n\n        default:\n          result ++;\n          break;\n\n      }\n\n    } else {\n      v_buff_size charSize = oatpp::encoding::Unicode::getUtf8CharSequenceLength(a);\n      if(charSize != 0) {\n        if(i + charSize > size) {\n          safeSize = i;\n        }\n        i += charSize;\n        if (!(flags & FLAG_ESCAPE_UTF8CHAR)) {\n          result += charSize; // output as-is\n        } else if(charSize < 4) {\n          result += 6; // '\\uFFFF' - 6 chars\n        } else if(charSize == 4) {\n          result += 12; // '\\uFFFF\\uFFFF' - 12 chars surrogate pair\n        } else {\n          result += 11; // '\\u+FFFFFFFF' - 11 chars NOT JSON standard case\n        }\n      } else {\n        // invalid char\n        i ++;\n        result ++;\n      }\n    }\n  }\n  return result;\n}\n\nv_buff_size Utils::calcUnescapedStringSize(const char* data, v_buff_size size, v_int64& errorCode, v_buff_size& errorPosition) {\n  errorCode = 0;\n  v_buff_size result = 0;\n  v_buff_size i = 0;\n  \n  while (i < size) {\n    v_char8 a = static_cast<v_char8>(data[i]);\n    if(a == '\\\\'){\n      \n      if(i + 1 == size){\n        errorCode = ERROR_CODE_INVALID_ESCAPED_CHAR;\n        errorPosition = i;\n        return 0;\n      }\n      \n      v_char8 b = static_cast<v_char8>(data[i + 1]);\n      \n      if(b == '\"' || b == '\\\\' || b == '/' || b == 'b' || b == 'f' || b == 'n' || b == 'r' || b == 't'){\n        result += 1;\n        i += 2;\n      } else if(b == 'u'){\n        \n        if(i + 6 > size){\n          errorCode = ERROR_CODE_INVALID_ESCAPED_CHAR;\n          errorPosition = i;\n          return 0;\n        }\n        \n        if(data[i + 2] == '+') { // not JSON standard case\n          if(i + 11 > size){\n            errorCode = ERROR_CODE_INVALID_ESCAPED_CHAR;\n            errorPosition = i;\n            return 0;\n          }\n          v_uint32 code;\n          errorCode = encoding::Hex::readUInt32(&data[i + 3], code);\n          if(errorCode != 0){\n            errorPosition = i + 3;\n            return 0;\n          }\n          i += 11;\n          result += encoding::Unicode::getUtf8CharSequenceLengthForCode(code);\n        } else {\n          v_uint16 code;\n          errorCode = encoding::Hex::readUInt16(&data[i + 2], code);\n          if(errorCode != 0){\n            errorPosition = i + 2;\n            return 0;\n          }\n          \n          if(code >= 0xD800 && code <= 0xDBFF){\n            if(i + 12 > size){\n              errorCode = ERROR_CODE_INVALID_SURROGATE_PAIR;\n              errorPosition = i;\n              return 0;\n            }\n            v_uint16 low;\n            errorCode = encoding::Hex::readUInt16(&data[i + 8], low);\n            if(errorCode != 0){\n              errorPosition = i + 8;\n              return 0;\n            }\n            \n            if(low >= 0xDC00 && low <= 0xDFFF){\n              v_uint32 bigCode = static_cast<v_uint32>(encoding::Unicode::utf16SurrogatePairToCode(static_cast<v_int16>(code), static_cast<v_int16>(low)));\n              i += 12;\n              result += encoding::Unicode::getUtf8CharSequenceLengthForCode(bigCode);\n            } else {\n              errorCode = ERROR_CODE_INVALID_SURROGATE_PAIR;\n              errorPosition = i;\n              return 0;\n            }\n            \n          } else {\n            i += 6;\n            result += encoding::Unicode::getUtf8CharSequenceLengthForCode(code);\n          }\n        }\n        \n      } else {\n        errorCode = ERROR_CODE_INVALID_ESCAPED_CHAR;\n        errorPosition = i;\n        return 0;\n      }\n      \n    } else {\n      i ++;\n      result ++;\n    }\n    \n  }\n  \n  return result;\n}\n  \nv_buff_size Utils::escapeUtf8Char(const char* sequence, p_char8 buffer){\n  v_buff_size length;\n  v_int32 code = oatpp::encoding::Unicode::encodeUtf8Char(sequence, length);\n  if(code < 0x00010000) {\n    buffer[0] = '\\\\';\n    buffer[1] = 'u';\n    oatpp::encoding::Hex::writeUInt16(v_uint16(code), &buffer[2]);\n    return 6;\n  } else if(code < 0x00200000) {\n    v_int16 high;\n    v_int16 low;\n    oatpp::encoding::Unicode::codeToUtf16SurrogatePair(code, high, low);\n    buffer[0] = '\\\\';\n    buffer[1] = 'u';\n    oatpp::encoding::Hex::writeUInt16(static_cast<v_uint16>(high), &buffer[2]);\n    buffer[6] = '\\\\';\n    buffer[7] = 'u';\n    oatpp::encoding::Hex::writeUInt16(static_cast<v_uint16>(low), &buffer[8]);\n    return 12;\n  } else {\n    buffer[0] = '\\\\';\n    buffer[1] = 'u';\n    buffer[2] = '+';\n    oatpp::encoding::Hex::writeUInt32(static_cast<v_uint32>(code), &buffer[2]);\n    return 11;\n  }\n}\n\noatpp::String Utils::escapeString(const char* data, v_buff_size size, v_uint32 flags) {\n  v_buff_size safeSize;\n  v_buff_size escapedSize = calcEscapedStringSize(data, size, safeSize, flags);\n  if(escapedSize == size) {\n    return String(data, size);\n  }\n  auto result = String(escapedSize);\n  p_char8 resultData = reinterpret_cast<p_char8>(const_cast<char*>(result->data()));\n  v_buff_size pos = 0;\n\n  {\n    v_buff_size i = 0;\n    while (i < safeSize) {\n      v_char8 a = static_cast<v_char8>(data[i]);\n      if (a < 32) {\n\n        switch (a) {\n\n          case '\\b': resultData[pos] = '\\\\'; resultData[pos + 1] = 'b'; pos += 2; break;\n          case '\\f': resultData[pos] = '\\\\'; resultData[pos + 1] = 'f'; pos += 2; break;\n          case '\\n': resultData[pos] = '\\\\'; resultData[pos + 1] = 'n'; pos += 2; break;\n          case '\\r': resultData[pos] = '\\\\'; resultData[pos + 1] = 'r'; pos += 2; break;\n          case '\\t': resultData[pos] = '\\\\'; resultData[pos + 1] = 't'; pos += 2; break;\n\n          default:\n            resultData[pos] = '\\\\';\n            resultData[pos + 1] = 'u';\n            oatpp::encoding::Hex::writeUInt16(a, &resultData[pos + 2]);\n            pos += 6;\n            break;\n\n        }\n\n        i++;\n\n      }\n      else if (a < 128) {\n\n        switch (a) {\n          case '\\\"': resultData[pos] = '\\\\'; resultData[pos + 1] = '\"'; pos += 2; break;\n          case '\\\\': resultData[pos] = '\\\\'; resultData[pos + 1] = '\\\\'; pos += 2; break;\n\n          case '/':\n            if((flags & FLAG_ESCAPE_SOLIDUS) > 0) {\n              resultData[pos] = '\\\\';\n              resultData[pos + 1] = '/';\n              pos += 2;\n            } else {\n              resultData[pos] = static_cast<v_char8>(data[i]);\n              pos++;\n            }\n            break;\n\n          default:\n            resultData[pos] = static_cast<v_char8>(data[i]);\n            pos++;\n            break;\n\n        }\n\n        i++;\n      }\n      else {\n        v_buff_size charSize = oatpp::encoding::Unicode::getUtf8CharSequenceLength(a);\n        if (charSize != 0) {\n          if (!(flags & FLAG_ESCAPE_UTF8CHAR)) {\n            std::memcpy(reinterpret_cast<void*>(&resultData[pos]), reinterpret_cast<void*>(const_cast<char*>(&data[i])), static_cast<size_t>(charSize));\n            pos += charSize;\n          }\n          else {\n            pos += escapeUtf8Char(&data[i], &resultData[pos]);\n          }\n          i += charSize;\n        }\n        else {\n          // invalid char\n          resultData[pos] = static_cast<v_char8>(data[i]);\n          i++;\n          pos++;\n        }\n      }\n    }\n  }\n  \n  if(size > safeSize){\n    for(v_buff_size i = pos; static_cast<size_t>(i) < result->size(); i ++){\n      resultData[i] = '?';\n    }\n  }\n  \n  return result;\n}\n\nvoid Utils::unescapeStringToBuffer(const char* data, v_buff_size size, p_char8 resultData){\n  \n  v_buff_size i = 0;\n  v_buff_size pos = 0;\n  \n  while (i < size) {\n    v_char8 a = static_cast<v_char8>(data[i]);\n    \n    if(a == '\\\\'){\n      v_char8 b = static_cast<v_char8>(data[i + 1]);\n      if(b != 'u'){\n        switch (b) {\n          case '\"': resultData[pos] = '\"'; pos ++; break;\n          case '\\\\': resultData[pos] = '\\\\'; pos ++; break;\n          case '/': resultData[pos] = '/'; pos ++; break;\n          case 'b': resultData[pos] = '\\b'; pos ++; break;\n          case 'f': resultData[pos] = '\\f'; pos ++; break;\n          case 'n': resultData[pos] = '\\n'; pos ++; break;\n          case 'r': resultData[pos] = '\\r'; pos ++; break;\n          case 't': resultData[pos] = '\\t'; pos ++; break;\n          default: break;\n        }\n        i += 2;\n      } else {\n        if(data[i + 2] == '+'){ // Not JSON standard case\n          v_uint32 code;\n          encoding::Hex::readUInt32(&data[i + 3], code);\n          i += 11;\n          pos += encoding::Unicode::decodeUtf8Char(static_cast<v_int32>(code), &resultData[pos]);\n        } else {\n          \n          v_uint16 code;\n          encoding::Hex::readUInt16(&data[i + 2], code);\n          \n          if(code >= 0xD800 && code <= 0xDBFF){\n            v_uint16 low;\n            encoding::Hex::readUInt16(&data[i + 8], low);\n            v_uint32 bigCode = static_cast<v_uint32>(encoding::Unicode::utf16SurrogatePairToCode(static_cast<v_int16>(code), static_cast<v_int16>(low)));\n            pos += encoding::Unicode::decodeUtf8Char(static_cast<v_int32>(bigCode), &resultData[pos]);\n            i += 12;\n          } else {\n            pos += encoding::Unicode::decodeUtf8Char(code, &resultData[pos]);\n            i += 6;\n          }\n          \n        }\n      }\n    } else {\n      resultData[pos] = a;\n      pos ++;\n      i++;\n    }\n    \n  }\n  \n}\n  \noatpp::String Utils::unescapeString(const char* data, v_buff_size size, v_int64& errorCode, v_buff_size& errorPosition) {\n  \n  v_buff_size unescapedSize = calcUnescapedStringSize(data, size, errorCode, errorPosition);\n  if(errorCode != 0){\n    return nullptr;\n  }\n  auto result = String(unescapedSize);\n  if(unescapedSize == size) {\n    std::memcpy(reinterpret_cast<void*>(const_cast<char*>(result->data())), data, static_cast<size_t>(size));\n  } else {\n    unescapeStringToBuffer(data, size, reinterpret_cast<p_char8>(const_cast<char*>(result->data())));\n  }\n  return result;\n  \n}\n  \nstd::string Utils::unescapeStringToStdString(const char* data, v_buff_size size, v_int64& errorCode, v_buff_size& errorPosition){\n  \n  v_buff_size unescapedSize = calcUnescapedStringSize(data, size, errorCode, errorPosition);\n  if(errorCode != 0){\n    return \"\";\n  }\n  std::string result;\n  result.resize(static_cast<size_t>(unescapedSize));\n  if(unescapedSize == size) {\n    std::memcpy(reinterpret_cast<void*>(const_cast<char*>(result.data())), data, static_cast<size_t>(size));\n  } else {\n    unescapeStringToBuffer(data, size, reinterpret_cast<p_char8>(const_cast<char*>(result.data())));\n  }\n  return result;\n  \n}\n  \nconst char* Utils::preparseString(ParsingCaret& caret, v_buff_size& size){\n  \n  if(caret.canContinueAtChar('\"', 1)){\n    \n    const char* data = caret.getData();\n    v_buff_size pos = caret.getPosition();\n...\n// Path: src/oatpp/parser/json/mapping/Serializer.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_parser_json_mapping_Serializer_hpp\n#define oatpp_parser_json_mapping_Serializer_hpp\n\n#include \"oatpp/parser/json/Utils.hpp\"\n#include \"oatpp/parser/json/Beautifier.hpp\"\n#include \"oatpp/core/Types.hpp\"\n#include <vector>\n\nnamespace oatpp { namespace parser { namespace json { namespace mapping {\n\n/**\n * Json Serializer.\n * Serializes oatpp DTO object to json. See [Data Transfer Object(DTO) component](https://oatpp.io/docs/components/dto/).\n */\nclass Serializer {\npublic:\n  typedef oatpp::data::mapping::type::Type Type;\n  typedef oatpp::data::mapping::type::BaseObject::Property Property;\n  typedef oatpp::data::mapping::type::BaseObject::Properties Properties;\n\n  typedef oatpp::String String;\npublic:\n  /**\n   * Serializer config.\n   */\n  class Config : public oatpp::base::Countable {\n  public:\n    /**\n     * Constructor.\n     */\n    Config()\n    {}\n  public:\n\n    /**\n     * Create shared config.\n     * @return - `std::shared_ptr` to Config.\n     */\n    static std::shared_ptr<Config> createShared(){\n      return std::make_shared<Config>();\n    }\n\n    /**\n     * Include fields with value == nullptr into serialized json.\n     * Field will still be included when field-info `required` is set to true and &id:alwaysIncludeRequired is set to true.\n     */\n    bool includeNullFields = true;\n\n    /**\n     * Always include required fields (set in in DTO_FIELD_INFO) even if they are `value == nullptr`\n     */\n    bool alwaysIncludeRequired = false;\n\n    /**\n     * Always include array or map elements, even if their value is `nullptr`.\n     */\n    bool alwaysIncludeNullCollectionElements = false;\n\n    /**\n     * If `true` - insert string `\"<unknown-type>\"` in json field value in case unknown field found.\n     * Fail if `false`.\n     * Known types for this serializer are:<br>\n     * (String, Int8, Int16, Int32, Int64, Float32, Float64, Boolean, DTOs, List, Fields).\n     */\n    bool throwOnUnknownTypes = true;\n\n    /**\n     * Use JSON Beautifier.\n     */\n    bool useBeautifier = false;\n\n    /**\n     * Beautifier Indent.\n     */\n    oatpp::String beautifierIndent = \"  \";\n\n    /**\n     * Beautifier new line.\n     */\n    oatpp::String beautifierNewLine = \"\\n\";\n\n    /**\n     * Enable type interpretations.\n     */\n    std::vector<std::string> enabledInterpretations = {};\n\n    /**\n     * Escape flags.\n     */\n    v_uint32 escapeFlags = json::Utils::FLAG_ESCAPE_ALL;\n\n  };\npublic:\n  typedef void (*SerializerMethod)(Serializer*,\n                                   data::stream::ConsistentOutputStream*,\n                                   const oatpp::Void&);\nprivate:\n\n  template<class T>\n  \nstatic void serializePrimitive(Serializer* serializer,\n                                 data::stream::ConsistentOutputStream* stream,\n                                 const oatpp::Void& polymorph){\n    (void) serializer;\n\n    if(polymorph){\n      stream->writeAsString(* static_cast<typename T::ObjectType*>(polymorph.get()));\n    } else {\n      stream->writeSimple(\"null\", 4);\n    }\n  }\n  \n  static void serializeString(oatpp::data::stream::ConsistentOutputStream* stream,\n                              const char* data,\n                              v_buff_size size,\n                              v_uint32 escapeFlags);\n\n  static void serializeString(Serializer* serializer,\n                              data::stream::ConsistentOutputStream* stream,\n                              const oatpp::Void& polymorph);\n\n  static void serializeAny(Serializer* serializer,\n                           data::stream::ConsistentOutputStream* stream,\n                           const oatpp::Void& polymorph);\n\n  static void serializeEnum(Serializer* serializer,\n                            data::stream::ConsistentOutputStream* stream,\n                            const oatpp::Void& polymorph);\n\n  static void serializeCollection(Serializer* serializer,\n                                  data::stream::ConsistentOutputStream* stream,\n                                  const oatpp::Void& polymorph);\n\n  static void serializeMap(Serializer* serializer,\n                           data::stream::ConsistentOutputStream* stream,\n                           const oatpp::Void& polymorph);\n\n  static void serializeObject(Serializer* serializer,\n                              data::stream::ConsistentOutputStream* stream,\n                              const oatpp::Void& polymorph);\n\n  void serialize(data::stream::ConsistentOutputStream* stream, const oatpp::Void& polymorph);\n\nprivate:\n  std::shared_ptr<Config> m_config;\n  std::vector<SerializerMethod> m_methods;\npublic:\n\n  /**\n   * Constructor.\n   * @param config - serializer config.\n   */\n  Serializer(const std::shared_ptr<Config>& config = std::make_shared<Config>());\n\n  /**\n   * Set serializer method for type.\n   * @param classId - &id:oatpp::data::mapping::type::ClassId;.\n   * @param method - `typedef void (*SerializerMethod)(Serializer*, data::stream::ConsistentOutputStream*, const oatpp::Void&)`.\n   */\n  void setSerializerMethod(const data::mapping::type::ClassId& classId, SerializerMethod method);\n\n  /**\n   * Serialize object to stream.\n   * @param stream - &id:oatpp::data::stream::ConsistentOutputStream;.\n   * @param polymorph - DTO as &id:oatpp::Void;.\n   */\n  void serializeToStream(data::stream::ConsistentOutputStream* stream, const oatpp::Void& polymorph);\n\n  /**\n   * Get serializer config.\n   * @return\n   */\n  const std::shared_ptr<Config>& getConfig();\n\n};\n\n}}}}\n\n#endif /* oatpp_parser_json_mapping_Serializer_hpp */\n\n// Path: src/oatpp/parser/json/mapping/Deserializer.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_parser_json_mapping_Deserializer_hpp\n#define oatpp_parser_json_mapping_Deserializer_hpp\n\n#include \"oatpp/parser/json/Utils.hpp\"\n#include \"oatpp/core/parser/Caret.hpp\"\n#include \"oatpp/core/Types.hpp\"\n\n#include <vector>\n\nnamespace oatpp { namespace parser { namespace json { namespace mapping {\n\n/**\n * Json Deserializer.\n * Deserialize oatpp DTO object from json. See [Data Transfer Object(DTO) component](https://oatpp.io/docs/components/dto/).\n */\nclass Deserializer {\npublic:\n  typedef oatpp::data::mapping::type::Type Type;\n  typedef oatpp::data::mapping::type::BaseObject::Property Property;\n  typedef oatpp::data::mapping::type::BaseObject::Properties Properties;\n\n  typedef oatpp::String String;\n\npublic:\n\n  /**\n * \"'{' - expected\"\n */\n  static constexpr v_int32 ERROR_CODE_OBJECT_SCOPE_OPEN = 1;\n\n  /**\n   * \"'}' - expected\"\n   */\n  static constexpr v_int32 ERROR_CODE_OBJECT_SCOPE_CLOSE = 2;\n\n  /**\n   * \"Unknown field\"\n   */\n  static constexpr v_int32 ERROR_CODE_OBJECT_SCOPE_UNKNOWN_FIELD = 3;\n\n  /**\n   * \"':' - expected\"\n   */\n  static constexpr v_int32 ERROR_CODE_OBJECT_SCOPE_COLON_MISSING = 4;\n\n  /**\n   * \"'[' - expected\"\n   */\n  static constexpr v_int32 ERROR_CODE_ARRAY_SCOPE_OPEN = 5;\n\n  /**\n   * \"']' - expected\"\n   */\n  static constexpr v_int32 ERROR_CODE_ARRAY_SCOPE_CLOSE = 6;\n\n  /**\n   * \"'true' or 'false' - expected\"\n   */\n  static constexpr v_int32 ERROR_CODE_VALUE_BOOLEAN = 7;\n\npublic:\n\n  /**\n   * Deserializer config.\n   */\n  class Config : public oatpp::base::Countable {\n  public:\n    /**\n     * Constructor.\n     */\n    Config()\n    {}\n  public:\n\n    /**\n     * Create shared Config.\n     * @return - `std::shared_ptr` to Config.\n     */\n    static std::shared_ptr<Config> createShared(){\n      return std::make_shared<Config>();\n    }\n\n    /**\n     * Do not fail if unknown field is found in json.\n     * \"unknown field\" is the one which is not present in DTO object class.\n     */\n    bool allowUnknownFields = true;\n\n    /**\n     * Enable type interpretations.\n     */\n    std::vector<std::string> enabledInterpretations = {};\n\n  };\n\npublic:\n  typedef oatpp::Void (*DeserializerMethod)(Deserializer*, parser::Caret&, const Type* const);\nprivate:\n  static void skipScope(oatpp::parser::Caret& caret, v_char8 charOpen, v_char8 charClose);\n  static void skipString(oatpp::parser::Caret& caret);\n  static void skipToken(oatpp::parser::Caret& caret);\n  static void skipValue(oatpp::parser::Caret& caret);\nprivate:\n  static const Type* guessNumberType(oatpp::parser::Caret& caret);\n  static const Type* guessType(oatpp::parser::Caret& caret);\nprivate:\n\n  template<class T>\n  static oatpp::Void deserializeInt(Deserializer* deserializer, parser::Caret& caret, const Type* const type){\n\n    (void) deserializer;\n    (void) type;\n\n    if(caret.isAtText(\"null\", true)){\n      return oatpp::Void(T::Class::getType());\n    } else {\n      //TODO: shall we handle overflow cases like\n      // oatpp::String json = \"128\";\n      // auto value = jsonObjectMapper->readFromString<oatpp::Int8>(json); // UInt8 will overflow to -128\n      return T(static_cast<typename T::UnderlyingType>(caret.parseInt()));\n    }\n\n  }\n\n  template<class T>\n  static oatpp::Void deserializeUInt(Deserializer* deserializer, parser::Caret& caret, const Type* const type){\n\n    (void) deserializer;\n    (void) type;\n\n    if(caret.isAtText(\"null\", true)){\n      return oatpp::Void(T::Class::getType());\n    } else {\n      //TODO: shall we handle overflow cases like\n      // oatpp::String json = \"256\";\n      // auto value = jsonObjectMapper->readFromString<oatpp::UInt8>(json); // UInt8 will overflow to 0\n      return T(static_cast<typename T::UnderlyingType>(caret.parseUnsignedInt()));\n    }\n\n  }\n\n  static oatpp::Void deserializeFloat32(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n  static oatpp::Void deserializeFloat64(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n  static oatpp::Void deserializeBoolean(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n  static oatpp::Void deserializeString(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n  static oatpp::Void deserializeAny(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n  static oatpp::Void deserializeEnum(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n\n  static oatpp::Void deserializeCollection(Deserializer* deserializer, parser::Caret& caret, const Type* type);\n  static oatpp::Void deserializeMap(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n\n  static oatpp::Void deserializeObject(Deserializer* deserializer, parser::Caret& caret, const Type* const type);\n\nprivate:\n  std::shared_ptr<Config> m_config;\n  std::vector<DeserializerMethod> m_methods;\npublic:\n\n  /**\n   * Constructor.\n   * @param config\n   */\n  Deserializer(const std::shared_ptr<Config>& config = std::make_shared<Config>());\n\n  /**\n   * Set deserializer method for type.\n   * @param classId - &id:oatpp::data::mapping::type::ClassId;.\n   * @param method - `typedef oatpp::Void (*DeserializerMethod)(Deserializer*, parser::Caret&, const Type* const)`.\n   */\n  void setDeserializerMethod(const data::mapping::type::ClassId& classId, DeserializerMethod method);\n\n  /**\n   * Deserialize text.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param type - &id:oatpp::data::mapping::type::Type;\n   * @return - `oatpp::Void` over deserialized object.\n   */\n  oatpp::Void deserialize(parser::Caret& caret, const Type* const type);\n\n  /**\n   * Get deserializer config.\n   * @return\n   */\n  const std::shared_ptr<Config>& getConfig();\n\n};\n\n}}}}\n\n#endif /* oatpp_parser_json_mapping_Deserializer_hpp */\n\n// Path: src/oatpp/parser/json/mapping/ObjectMapper.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_parser_json_mapping_ObjectMapper_hpp\n#define oatpp_parser_json_mapping_ObjectMapper_hpp\n\n#include \"./Serializer.hpp\"\n#include \"./Deserializer.hpp\"\n\n#include \"oatpp/core/data/mapping/ObjectMapper.hpp\"\n\nnamespace oatpp { namespace parser { namespace json { namespace mapping {\n\n/**\n * Json ObjectMapper. Serialized/Deserializes oatpp DTO objects to/from JSON.\n * See [Data Transfer Object(DTO) component](https://oatpp.io/docs/components/dto/). <br>\n * Extends &id:oatpp::base::Countable;, &id:oatpp::data::mapping::ObjectMapper;.\n */\nclass ObjectMapper : public oatpp::base::Countable, public oatpp::data::mapping::ObjectMapper {\nprivate:\n  static Info& getMapperInfo() {\n    static Info info(\"application/json\");\n    return info;\n  }\nprivate:\n  std::shared_ptr<Serializer> m_serializer;\n  std::shared_ptr<Deserializer> m_deserializer;\npublic:\n  /**\n   * Constructor.\n   * @param serializerConfig - &id:oatpp::parser::json::mapping::Serializer::Config;.\n   * @param deserializerConfig - &id:oatpp::parser::json::mapping::Deserializer::Config;.\n   */\n  ObjectMapper(const std::shared_ptr<Serializer::Config>& serializerConfig,\n               const std::shared_ptr<Deserializer::Config>& deserializerConfig);\n\n  /**\n   * Constructor.\n   * @param serializer\n   * @param deserializer\n   */\n  ObjectMapper(const std::shared_ptr<Serializer>& serializer = std::make_shared<Serializer>(),\n               const std::shared_ptr<Deserializer>& deserializer = std::make_shared<Deserializer>());\npublic:\n\n  /**\n   * Create shared ObjectMapper.\n   * @param serializerConfig - &id:oatpp::parser::json::mapping::Serializer::Config;.\n   * @param deserializerConfig - &id:oatpp::parser::json::mapping::Deserializer::Config;.\n   * @return - `std::shared_ptr` to ObjectMapper.\n   */\n  static std::shared_ptr<ObjectMapper>\n  createShared(const std::shared_ptr<Serializer::Config>& serializerConfig,\n               const std::shared_ptr<Deserializer::Config>& deserializerConfig);\n\n  /**\n   * Create shared ObjectMapper.\n   * @param serializer\n   * @param deserializer\n   * @return\n   */\n  static std::shared_ptr<ObjectMapper>\n  createShared(const std::shared_ptr<Serializer>& serializer = std::make_shared<Serializer>(),\n               const std::shared_ptr<Deserializer>& deserializer = std::make_shared<Deserializer>());\n\n  /**\n   * Implementation of &id:oatpp::data::mapping::ObjectMapper::write;.\n   * @param stream - stream to write serializerd data to &id:oatpp::data::stream::ConsistentOutputStream;.\n   * @param variant - object to serialize &id:oatpp::Void;.\n   */\n  void write(data::stream::ConsistentOutputStream* stream, const oatpp::Void& variant) const override;\n\n  /**\n   * Implementation of &id:oatpp::data::mapping::ObjectMapper::read;.\n   * @param caret - &id:oatpp::parser::Caret;.\n   * @param type - type of resultant object &id:oatpp::data::mapping::type::Type;.\n   * @return - &id:oatpp::Void; holding resultant object.\n   */\n  oatpp::Void read(oatpp::parser::Caret& caret, const oatpp::data::mapping::type::Type* const type) const override;\n\n\n  /**\n   * Get serializer.\n   * @return\n   */\n  std::shared_ptr<Serializer> getSerializer();\n\n  /**\n   * Get deserializer.\n   * @return\n   */\n  std::shared_ptr<Deserializer> getDeserializer();\n  \n};\n  \n}}}}\n\n#endif /* oatpp_parser_json_mapping_ObjectMapper_hpp */\n\n// Path: src/oatpp/parser/json/mapping/ObjectMapper.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"ObjectMapper.hpp\"\n\nnamespace oatpp { namespace parser { namespace json { namespace mapping {\n\nObjectMapper::ObjectMapper(const std::shared_ptr<Serializer::Config>& serializerConfig,\n                           const std::shared_ptr<Deserializer::Config>& deserializerConfig)\n  : data::mapping::ObjectMapper(getMapperInfo())\n  , m_serializer(std::make_shared<Serializer>(serializerConfig))\n  , m_deserializer(std::make_shared<Deserializer>(deserializerConfig))\n{}\n\nObjectMapper::ObjectMapper(const std::shared_ptr<Serializer>& serializer,\n                           const std::shared_ptr<Deserializer>& deserializer)\n  : data::mapping::ObjectMapper(getMapperInfo())\n  , m_serializer(serializer)\n  , m_deserializer(deserializer)\n{}\n\nstd::shared_ptr<ObjectMapper> ObjectMapper::createShared(const std::shared_ptr<Serializer::Config>& serializerConfig,\n                                                         const std::shared_ptr<Deserializer::Config>& deserializerConfig){\n  return std::make_shared<ObjectMapper>(serializerConfig, deserializerConfig);\n}\n\nstd::shared_ptr<ObjectMapper> ObjectMapper::createShared(const std::shared_ptr<Serializer>& serializer,\n                                                         const std::shared_ptr<Deserializer>& deserializer){\n  return std::make_shared<ObjectMapper>(serializer, deserializer);\n}\n\nvoid ObjectMapper::write(data::stream::ConsistentOutputStream* stream,\n                         const oatpp::Void& variant) const {\n  m_serializer->serializeToStream(stream, variant);\n}\n\noatpp::Void ObjectMapper::read(oatpp::parser::Caret& caret,\n                                                                     const oatpp::data::mapping::type::Type* const type) const {\n  return m_deserializer->deserialize(caret, type);\n}\n\nstd::shared_ptr<Serializer> ObjectMapper::getSerializer() {\n  return m_serializer;\n}\n\nstd::shared_ptr<Deserializer> ObjectMapper::getDeserializer() {\n  return m_deserializer;\n}\n\n}}}}\n\n// Path: src/oatpp/parser/json/mapping/Deserializer.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Deserializer.hpp\"\n\n#include \"oatpp/core/utils/ConversionUtils.hpp\"\n\nnamespace oatpp { namespace parser { namespace json { namespace mapping {\n\nDeserializer::Deserializer(const std::shared_ptr<Config>& config)\n  : m_config(config)\n{\n\n  m_methods.resize(static_cast<size_t>(data::mapping::type::ClassId::getClassCount()), nullptr);\n\n  setDeserializerMethod(data::mapping::type::__class::String::CLASS_ID, &Deserializer::deserializeString);\n  setDeserializerMethod(data::mapping::type::__class::Any::CLASS_ID, &Deserializer::deserializeAny);\n\n  setDeserializerMethod(data::mapping::type::__class::Int8::CLASS_ID, &Deserializer::deserializeInt<oatpp::Int8>);\n  setDeserializerMethod(data::mapping::type::__class::UInt8::CLASS_ID, &Deserializer::deserializeUInt<oatpp::UInt8>);\n\n  setDeserializerMethod(data::mapping::type::__class::Int16::CLASS_ID, &Deserializer::deserializeInt<oatpp::Int16>);\n  setDeserializerMethod(data::mapping::type::__class::UInt16::CLASS_ID, &Deserializer::deserializeUInt<oatpp::UInt16>);\n\n  setDeserializerMethod(data::mapping::type::__class::Int32::CLASS_ID, &Deserializer::deserializeInt<oatpp::Int32>);\n  setDeserializerMethod(data::mapping::type::__class::UInt32::CLASS_ID, &Deserializer::deserializeUInt<oatpp::UInt32>);\n\n  setDeserializerMethod(data::mapping::type::__class::Int64::CLASS_ID, &Deserializer::deserializeInt<oatpp::Int64>);\n  setDeserializerMethod(data::mapping::type::__class::UInt64::CLASS_ID, &Deserializer::deserializeUInt<oatpp::UInt64>);\n\n  setDeserializerMethod(data::mapping::type::__class::Float32::CLASS_ID, &Deserializer::deserializeFloat32);\n  setDeserializerMethod(data::mapping::type::__class::Float64::CLASS_ID, &Deserializer::deserializeFloat64);\n  setDeserializerMethod(data::mapping::type::__class::Boolean::CLASS_ID, &Deserializer::deserializeBoolean);\n\n  setDeserializerMethod(data::mapping::type::__class::AbstractObject::CLASS_ID, &Deserializer::deserializeObject);\n  setDeserializerMethod(data::mapping::type::__class::AbstractEnum::CLASS_ID, &Deserializer::deserializeEnum);\n\n  setDeserializerMethod(data::mapping::type::__class::AbstractVector::CLASS_ID, &Deserializer::deserializeCollection);\n  setDeserializerMethod(data::mapping::type::__class::AbstractList::CLASS_ID, &Deserializer::deserializeCollection);\n  setDeserializerMethod(data::mapping::type::__class::AbstractUnorderedSet::CLASS_ID, &Deserializer::deserializeCollection);\n\n  setDeserializerMethod(data::mapping::type::__class::AbstractPairList::CLASS_ID, &Deserializer::deserializeMap);\n  setDeserializerMethod(data::mapping::type::__class::AbstractUnorderedMap::CLASS_ID, &Deserializer::deserializeMap);\n\n}\n\nvoid Deserializer::setDeserializerMethod(const data::mapping::type::ClassId& classId, DeserializerMethod method) {\n  const v_uint32 id = static_cast<v_uint32>(classId.id);\n  if(id >= m_methods.size()) {\n    m_methods.resize(id + 1, nullptr);\n  }\n  m_methods[id] = method;\n}\n\nvoid Deserializer::skipScope(oatpp::parser::Caret& caret, v_char8 charOpen, v_char8 charClose){\n\n  const char* data = caret.getData();\n  v_buff_size size = caret.getDataSize();\n  v_buff_size pos = caret.getPosition();\n  v_int32 scopeCounter = 0;\n\n  bool isInString = false;\n\n  while(pos < size){\n    v_char8 a = static_cast<v_char8>(data[pos]);\n    if(a == charOpen){\n      if(!isInString){\n        scopeCounter ++;\n      }\n    } else if(a == charClose){\n      if(!isInString){\n        scopeCounter --;\n        if(scopeCounter == 0){\n          caret.setPosition(pos + 1);\n          return;\n        }\n      }\n    } else if(a == '\"') {\n      isInString = !isInString;\n    } else if(a == '\\\\'){\n      pos ++;\n    }\n\n    pos ++;\n\n  }\n}\n\nvoid Deserializer::skipString(oatpp::parser::Caret& caret){\n  const char* data = caret.getData();\n  v_buff_size size = caret.getDataSize();\n  v_buff_size pos = caret.getPosition();\n  v_int32 scopeCounter = 0;\n  while(pos < size){\n    v_char8 a = static_cast<v_char8>(data[pos]);\n    if(a == '\"'){\n      scopeCounter ++;\n      if(scopeCounter == 2) {\n        caret.setPosition(pos + 1);\n        return;\n      }\n    } else if(a == '\\\\'){\n      pos ++;\n    }\n    pos ++;\n  }\n}\n\nvoid Deserializer::skipToken(oatpp::parser::Caret& caret){\n  const char* data = caret.getData();\n  v_buff_size size = caret.getDataSize();\n  v_buff_size pos = caret.getPosition();\n  while(pos < size){\n    v_char8 a = static_cast<v_char8>(data[pos]);\n    if(a == ' ' || a == '\\t' || a == '\\n' || a == '\\r' || a == '\\b' || a == '\\f' ||\n       a == '}' || a == ',' || a == ']') {\n      caret.setPosition(pos);\n      return;\n    }\n    pos ++;\n  }\n}\n\nvoid Deserializer::skipValue(oatpp::parser::Caret& caret){\n  if(caret.isAtChar('{')){\n    skipScope(caret, '{', '}');\n  } else if(caret.isAtChar('[')){\n    skipScope(caret, '[', ']');\n  } else if(caret.isAtChar('\"')){\n    skipString(caret);\n  } else {\n    skipToken(caret);\n  }\n}\n\noatpp::Void Deserializer::deserializeFloat32(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n\n  (void) deserializer;\n  (void) type;\n\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(Float32::Class::getType());\n  } else {\n    return Float32(caret.parseFloat32());\n  }\n}\n\noatpp::Void Deserializer::deserializeFloat64(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n\n  (void) deserializer;\n  (void) type;\n\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(Float64::Class::getType());\n  } else {\n    return Float64(caret.parseFloat64());\n  }\n\n}\n\noatpp::Void Deserializer::deserializeBoolean(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n\n  (void) deserializer;\n  (void) type;\n\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(Boolean::Class::getType());\n  } else {\n    if(caret.isAtText(\"true\", true)) {\n      return Boolean(true);\n    } else if(caret.isAtText(\"false\", true)) {\n      return Boolean(false);\n    } else {\n      caret.setError(\"[oatpp::parser::json::mapping::Deserializer::readBooleanValue()]: Error. 'true' or 'false' - expected.\", ERROR_CODE_VALUE_BOOLEAN);\n      return oatpp::Void(Boolean::Class::getType());\n    }\n  }\n\n}\n\noatpp::Void Deserializer::deserializeString(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n\n  (void) deserializer;\n  (void) type;\n\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(String::Class::getType());\n  } else {\n    return oatpp::Void(oatpp::parser::json::Utils::parseString(caret).getPtr(), String::Class::getType());\n  }\n}\n\nconst data::mapping::type::Type* Deserializer::guessNumberType(oatpp::parser::Caret& caret) {\n  if (!Utils::findDecimalSeparatorInCurrentNumber(caret)) {\n    if (*caret.getCurrData() == '-') {\n      return Int64::Class::getType();\n    } else {\n      return UInt64::Class::getType();\n    }\n  }\n  return Float64::Class::getType();\n}\n\nconst data::mapping::type::Type* Deserializer::guessType(oatpp::parser::Caret& caret) {\n  {\n    parser::Caret::StateSaveGuard stateGuard(caret);\n    v_char8 c = static_cast<v_char8>(*caret.getCurrData());\n    switch (c) {\n      case '\"':\n        return String::Class::getType();\n      case '{':\n        return oatpp::Fields<Any>::Class::getType();\n      case '[':\n        return oatpp::List<Any>::Class::getType();\n      case 't':\n        if(caret.isAtText(\"true\")) return Boolean::Class::getType();\n        break;\n      case 'f':\n        if(caret.isAtText(\"false\")) return Boolean::Class::getType();\n        break;\n      default:\n        if (c == '-' || caret.isAtDigitChar()) {\n          return guessNumberType(caret);\n        }\n    }\n  }\n  caret.setError(\"[oatpp::parser::json::mapping::Deserializer::guessType()]: Error. Can't guess type for oatpp::Any.\");\n  return nullptr;\n}\n\noatpp::Void Deserializer::deserializeAny(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n  (void) type;\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(Any::Class::getType());\n  } else {\n    const Type* const fieldType = guessType(caret);\n    if(fieldType != nullptr) {\n      auto fieldValue = deserializer->deserialize(caret, fieldType);\n      auto anyHandle = std::make_shared<data::mapping::type::AnyHandle>(fieldValue.getPtr(), fieldValue.getValueType());\n      return oatpp::Void(anyHandle, Any::Class::getType());\n    }\n  }\n  return oatpp::Void(Any::Class::getType());\n}\n\noatpp::Void Deserializer::deserializeEnum(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n\n  auto polymorphicDispatcher = static_cast<const data::mapping::type::__class::AbstractEnum::PolymorphicDispatcher*>(\n    type->polymorphicDispatcher\n  );\n\n  data::mapping::type::EnumInterpreterError e = data::mapping::type::EnumInterpreterError::OK;\n  const auto& value = deserializer->deserialize(caret, polymorphicDispatcher->getInterpretationType());\n  if(caret.hasError()) {\n    return nullptr;\n  }\n  const auto& result = polymorphicDispatcher->fromInterpretation(value, e);\n\n  if(e == data::mapping::type::EnumInterpreterError::OK) {\n    return result;\n  }\n\n  switch(e) {\n    case data::mapping::type::EnumInterpreterError::CONSTRAINT_NOT_NULL:\n      caret.setError(\"[oatpp::parser::json::mapping::Deserializer::deserializeEnum()]: Error. Enum constraint violated - 'NotNull'.\");\n      break;\n    case data::mapping::type::EnumInterpreterError::OK:\n    case data::mapping::type::EnumInterpreterError::TYPE_MISMATCH_ENUM:\n    case data::mapping::type::EnumInterpreterError::TYPE_MISMATCH_ENUM_VALUE:\n    case data::mapping::type::EnumInterpreterError::ENTRY_NOT_FOUND:\n    default:\n      caret.setError(\"[oatpp::parser::json::mapping::Deserializer::deserializeEnum()]: Error. Can't deserialize Enum.\");\n  }\n\n  return nullptr;\n\n}\n\noatpp::Void Deserializer::deserializeCollection(Deserializer* deserializer, parser::Caret& caret, const Type* type) {\n\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(type);\n  }\n\n  if(caret.canContinueAtChar('[', 1)) {\n\n    auto dispatcher = static_cast<const data::mapping::type::__class::Collection::PolymorphicDispatcher*>(type->polymorphicDispatcher);\n    auto collection = dispatcher->createObject();\n\n    auto itemType = dispatcher->getItemType();\n\n    caret.skipBlankChars();\n\n    while(!caret.isAtChar(']') && caret.canContinue()){\n\n      caret.skipBlankChars();\n      auto item = deserializer->deserialize(caret, itemType);\n      if(caret.hasError()){\n        return nullptr;\n      }\n\n      dispatcher->addItem(collection, item);\n      caret.skipBlankChars();\n\n      caret.canContinueAtChar(',', 1);\n\n    }\n\n    if(!caret.canContinueAtChar(']', 1)){\n      if(!caret.hasError()){\n        caret.setError(\"[oatpp::parser::json::mapping::Deserializer::deserializeCollection()]: Error. ']' - expected\", ERROR_CODE_ARRAY_SCOPE_CLOSE);\n      }\n      return nullptr;\n    }\n\n    return collection;\n\n  } else {\n    caret.setError(\"[oatpp::parser::json::mapping::Deserializer::deserializeCollection()]: Error. '[' - expected\", ERROR_CODE_ARRAY_SCOPE_OPEN);\n    return nullptr;\n  }\n\n}\n\noatpp::Void Deserializer::deserializeMap(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(type);\n  }\n\n  if(caret.canContinueAtChar('{', 1)) {\n\n    auto dispatcher = static_cast<const data::mapping::type::__class::Map::PolymorphicDispatcher*>(type->polymorphicDispatcher);\n    auto map = dispatcher->createObject();\n\n    auto keyType = dispatcher->getKeyType();\n    if(keyType->classId != oatpp::String::Class::CLASS_ID){\n      throw std::runtime_error(\"[oatpp::parser::json::mapping::Deserializer::deserializeMap()]: Invalid json map key. Key should be String\");\n    }\n    auto valueType = dispatcher->getValueType();\n\n    caret.skipBlankChars();\n\n    while (!caret.isAtChar('}') && caret.canContinue()) {\n\n      caret.skipBlankChars();\n      auto key = Utils::parseString(caret);\n      if(caret.hasError()){\n        return nullptr;\n      }\n\n      caret.skipBlankChars();\n      if(!caret.canContinueAtChar(':', 1)){\n        caret.setError(\"[oatpp::parser::json::mapping::Deserializer::deserializeMap()]: Error. ':' - expected\", ERROR_CODE_OBJECT_SCOPE_COLON_MISSING);\n        return nullptr;\n      }\n\n      caret.skipBlankChars();\n\n      auto item = deserializer->deserialize(caret, valueType);\n      if(caret.hasError()){\n        return nullptr;\n      }\n      dispatcher->addItem(map, key, item);\n\n      caret.skipBlankChars();\n      caret.canContinueAtChar(',', 1);\n\n    }\n\n    if(!caret.canContinueAtChar('}', 1)){\n      if(!caret.hasError()){\n        caret.setError(\"[oatpp::parser::json::mapping::Deserializer::deserializeMap()]: Error. '}' - expected\", ERROR_CODE_OBJECT_SCOPE_CLOSE);\n      }\n      return nullptr;\n    }\n\n    return map;\n\n  } else {\n    caret.setError(\"[oatpp::parser::json::mapping::Deserializer::deserializeMap()]: Error. '{' - expected\", ERROR_CODE_OBJECT_SCOPE_OPEN);\n  }\n\n  return nullptr;\n\n}\n\noatpp::Void Deserializer::deserializeObject(Deserializer* deserializer, parser::Caret& caret, const Type* const type) {\n\n  if(caret.isAtText(\"null\", true)){\n    return oatpp::Void(type);\n  }\n\n  if(caret.canContinueAtChar('{', 1)) {\n\n    auto dispatcher = static_cast<const oatpp::data::mapping::type::__class::AbstractObject::PolymorphicDispatcher*>(type->polymorphicDispatcher);\n...\n", "needle_token_start": 5590, "needle_token_end": 5693, "code_context_ntokens": 16020, "output": ["```cpp\nstatic void serializePrimitive(Serializer* serializer,\n                                 data::stream::ConsistentOutputStream* stream,\n                                 const oatpp::Void& polymorph){\n    (void) serializer;\n\n    if(polymorph){\n      stream->writeAsString(* static_cast<typename T::ObjectType*>(polymorph.get()));\n    } else {\n      stream->writeSimple(\"null\", 4);\n    }\n  }\n```"]}
{"repo": "oatpp/oatpp", "name": "DataHandle", "language": "cpp", "path": "src/oatpp/core/data/resource/InMemoryData.hpp", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To encapsulate and manage a string of data within a resource context.\n2. **Input**: A string representing the data to be managed.\n3. **Output**: An instance that holds the provided string data.\n4. **Procedure**: Upon instantiation, the input string is stored within the instance, making it accessible for further operations or retrieval within the resource management context.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/core/data/stream/FileStream.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"FileStream.hpp\"\n\nnamespace oatpp { namespace data{ namespace stream {\n\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// FileInputStream\n\noatpp::data::stream::DefaultInitializedContext FileInputStream::DEFAULT_CONTEXT(data::stream::StreamType::STREAM_FINITE);\n\nFileInputStream::FileInputStream(FileInputStream&& other)\n  : m_file(other.m_file)\n  , m_ownsFile(other.m_ownsFile)\n  , m_ioMode(other.m_ioMode)\n{\n  other.m_file = nullptr;\n  other.m_ownsFile = false;\n}\n\nFileInputStream::FileInputStream(std::FILE* file, bool ownsFile, const std::shared_ptr<void>& captureData)\n  : m_file(file)\n  , m_ownsFile(ownsFile)\n  , m_ioMode(IOMode::ASYNCHRONOUS)\n  , m_capturedData(captureData)\n{}\n\nFileInputStream::FileInputStream(const char* filename, const std::shared_ptr<void>& captureData)\n  : FileInputStream(std::fopen(filename, \"rb\"), true, captureData)\n{\n  if(!m_file) {\n    OATPP_LOGE(\"[oatpp::data::stream::FileInputStream::FileInputStream(filename)]\", \"Error. Can't open file '%s'.\", filename)\n    throw std::runtime_error(\"[oatpp::data::stream::FileInputStream::FileInputStream(filename)]: Error. Can't open file.\");\n  }\n}\n\nFileInputStream::~FileInputStream() {\n  this->close();\n}\n\nstd::FILE* FileInputStream::getFile() {\n  return m_file;\n}\n\n...\n// Path: src/oatpp/core/data/stream/BufferStream.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_stream_BufferStream_hpp\n#define oatpp_data_stream_BufferStream_hpp\n\n#include \"Stream.hpp\"\n\nnamespace oatpp { namespace data{ namespace stream {\n\n/**\n * BufferOutputStream\n */\nclass BufferOutputStream : public ConsistentOutputStream {\npublic:\n  static data::stream::DefaultInitializedContext DEFAULT_CONTEXT;\nprivate:\n  p_char8 m_data;\n  v_buff_size m_capacity;\n  v_buff_size m_position;\n  v_buff_size m_maxCapacity;\n  IOMode m_ioMode;\nprivate:\n  std::shared_ptr<void> m_capturedData;\npublic:\n\n  /**\n   * Constructor.\n   * @param growBytes\n   * @param captureData - capture auxiliary data to not get deleted until it's done with the stream.\n   */\n  BufferOutputStream(v_buff_size initialCapacity = 2048, const std::shared_ptr<void>& captureData = nullptr);\n\n  /**\n   * Virtual destructor.\n   */\n  ~BufferOutputStream() override;\n\n  /**\n   * Write `count` of bytes to stream.\n   * @param data - data to write.\n   * @param count - number of bytes to write.\n   * @param action - async specific action. If action is NOT &id:oatpp::async::Action::TYPE_NONE;, then\n   * caller MUST return this action on coroutine iteration.\n   * @return - actual number of bytes written. &id:oatpp::v_io_size;.\n   */\n  v_io_size write(const void *data, v_buff_size count, async::Action& action) override;\n\n  /**\n   * Set stream I/O mode.\n   * @throws\n   */\n  void setOutputStreamIOMode(IOMode ioMode) override;\n\n  /**\n   * Get stream I/O mode.\n   * @return\n   */\n  IOMode getOutputStreamIOMode() override;\n\n  /**\n   * Get stream context.\n   * @return\n   */\n  Context& getOutputStreamContext() override;\n\n  /**\n   * Reserve bytes for future writes.\n   */\n  void reserveBytesUpfront(v_buff_size count);\n\n  /**\n   * Get pointer to data.\n   * @return - pointer to data.\n   */\n  p_char8 getData();\n\n  /**\n   * Get current capacity.\n   * Capacity may change.\n   * @return\n   */\n  v_buff_size getCapacity();\n\n  /**\n   * Get current data write position.\n   * @return - current data write position.\n   */\n  v_buff_size getCurrentPosition();\n\n  /**\n   * Set current data write position.\n   * @param position - data write position.\n   */\n  void setCurrentPosition(v_buff_size position);\n\n  /**\n   * Reset stream buffer and its capacity. Also reset write position.\n   * @param initialCapacity\n   */\n  void reset(v_buff_size initialCapacity = 2048);\n\n  /**\n   * Copy data to &id:oatpp::String;.\n   * @return\n   */\n  oatpp::String toString();\n\n  /**\n   * Create &id:oatpp::String; from part of buffer.\n   * @param pos - starting position in buffer.\n   * @param count - size of bytes to write to substring.\n   * @return - &id:oatpp::String;\n   */\n  oatpp::String getSubstring(v_buff_size pos, v_buff_size count);\n\n  /**\n   * Write all bytes from buffer to stream.\n   * @param stream - stream to flush all data to.\n   * @return - actual amount of bytes flushed.\n   */\n  oatpp::v_io_size flushToStream(OutputStream* stream);\n\n  /**\n   * Write all bytes from buffer to stream in async manner.\n   * @param _this - pointer to `this` buffer.\n   * @param stream - stream to flush all data to.\n   * @return - &id:oatpp::async::CoroutineStarter;.\n   */\n  static oatpp::async::CoroutineStarter flushToStreamAsync(const std::shared_ptr<BufferOutputStream>& _this, const std::shared_ptr<OutputStream>& stream);\n\n};\n\n/**\n * BufferInputStream\n */\nclass BufferInputStream : public BufferedInputStream {\npublic:\n  static data::stream::DefaultInitializedContext DEFAULT_CONTEXT;\nprivate:\n  std::shared_ptr<std::string> m_memoryHandle;\n  p_char8 m_data;\n  v_buff_size m_size;\n  v_buff_size m_position;\n  IOMode m_ioMode;\nprivate:\n  std::shared_ptr<void> m_capturedData;\npublic:\n\n  /**\n   * Constructor.\n   * @param memoryHandle - buffer memory handle. May be nullptr.\n   * @param data - pointer to buffer data.\n   * @param size - size of the buffer.\n   * @param captureData - capture auxiliary data to not get deleted until it's done with the stream.\n   */\n  BufferInputStream(const std::shared_ptr<std::string>& memoryHandle,\n                    const void* data,\n                    v_buff_size size,\n                    const std::shared_ptr<void>& captureData = nullptr);\n\n  /**\n   * Constructor.\n   * @param data - buffer.\n   * @param captureData - capture auxiliary data to not get deleted until it's done with the stream.\n   */\n  BufferInputStream(const oatpp::String& data, const std::shared_ptr<void>& captureData = nullptr);\n\n  /**\n   * Reset stream data and set position to `0`.\n   * @param memoryHandle - buffer memory handle. May be nullptr.\n   * @param data - pointer to buffer data.\n   * @param size - size of the buffer.\n   * @param captureData - capture auxiliary data to not get deleted until it's done with the stream.\n   */\n  void reset(const std::shared_ptr<std::string>& memoryHandle,\n             p_char8 data,\n             v_buff_size size,\n             const std::shared_ptr<void>& captureData = nullptr);\n\n\n  /**\n   * Same as `reset(nullptr, nullptr, 0);.`\n   */\n  void reset();\n\n  /**\n   * Read data from stream. <br>\n   * It is a legal case if return result < count. Caller should handle this!\n   * *Calls to this method are always NON-BLOCKING*\n   * @param data - buffer to read data to.\n   * @param count - size of the buffer.\n   * @param action - async specific action. If action is NOT &id:oatpp::async::Action::TYPE_NONE;, then\n   * caller MUST return this action on coroutine iteration.\n   * @return - actual number of bytes read. 0 - designates end of the buffer.\n   */\n  v_io_size read(void *data, v_buff_size count, async::Action& action) override;\n\n  /**\n   * Set stream I/O mode.\n   * @throws\n   */\n  void setInputStreamIOMode(IOMode ioMode) override;\n\n  /**\n   * Get stream I/O mode.\n   * @return\n   */\n  IOMode getInputStreamIOMode() override;\n\n  /**\n   * Get stream context.\n   * @return\n   */\n  Context& getInputStreamContext() override;\n\n  /**\n   * Get data memory handle.\n   * @return - data memory handle.\n   */\n  std::shared_ptr<std::string> getDataMemoryHandle();\n\n  /**\n   * Get pointer to data.\n   * @return - pointer to data.\n   */\n  p_char8 getData();\n\n  /**\n   * Get data size.\n   * @return - data size.\n   */\n  v_buff_size getDataSize();\n\n  /**\n   * Get current data read position.\n   * @return - current data read position.\n   */\n  v_buff_size getCurrentPosition();\n\n  /**\n   * Set current data read position.\n   * @param position - data read position.\n   */\n  void setCurrentPosition(v_buff_size position);\n\n  /**\n   * Peek up to count of bytes int he buffer\n   * @param data\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size peek(void *data, v_buff_size count, async::Action& action) override;\n\n  /**\n   * Amount of bytes currently available to read from buffer.\n   * @return &id:oatpp::v_io_size;.\n   */\n  v_io_size availableToRead() const override;\n\n  /**\n   * Commit read offset\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size commitReadOffset(v_buff_size count) override;\n\n};\n\n}}}\n\n#endif // oatpp_data_stream_BufferStream_hpp\n\n// Path: src/oatpp/core/data/stream/BufferStream.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"BufferStream.hpp\"\n\n#include \"oatpp/core/utils/Binary.hpp\"\n\nnamespace oatpp { namespace data{ namespace stream {\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// BufferOutputStream\n\ndata::stream::DefaultInitializedContext BufferOutputStream::DEFAULT_CONTEXT(data::stream::StreamType::STREAM_INFINITE);\n\nBufferOutputStream::BufferOutputStream(v_buff_size initialCapacity, const std::shared_ptr<void>& captureData)\n  : m_data(new v_char8[initialCapacity])\n  , m_capacity(initialCapacity)\n  , m_position(0)\n  , m_maxCapacity(-1)\n  , m_ioMode(IOMode::ASYNCHRONOUS)\n  , m_capturedData(captureData)\n{}\n\nBufferOutputStream::~BufferOutputStream() {\n  m_capturedData.reset(); // reset capture data before deleting data.\n  delete [] m_data;\n}\n\nv_io_size BufferOutputStream::write(const void *data, v_buff_size count, async::Action& action) {\n\n  (void) action;\n\n  reserveBytesUpfront(count);\n\n  std::memcpy(m_data + m_position, data, static_cast<size_t>(count));\n  m_position += count;\n\n  return count;\n\n}\n\nvoid BufferOutputStream::setOutputStreamIOMode(IOMode ioMode) {\n  m_ioMode = ioMode;\n}\n\nIOMode BufferOutputStream::getOutputStreamIOMode() {\n  return m_ioMode;\n}\n\nContext& BufferOutputStream::getOutputStreamContext() {\n  return DEFAULT_CONTEXT;\n}\n\nvoid BufferOutputStream::reserveBytesUpfront(v_buff_size count) {\n\n  v_buff_size capacityNeeded = m_position + count;\n\n  if(capacityNeeded > m_capacity) {\n\n    v_buff_size newCapacity = utils::Binary::nextP2(capacityNeeded);\n\n    if(newCapacity < 0 || (m_maxCapacity > 0 && newCapacity > m_maxCapacity)) {\n      newCapacity = m_maxCapacity;\n    }\n\n    if(newCapacity < capacityNeeded) {\n      throw std::runtime_error(\"[oatpp::data::stream::BufferOutputStream::reserveBytesUpfront()]: Error. Unable to allocate requested memory.\");\n    }\n\n    p_char8 newData = new v_char8[newCapacity];\n\n    std::memcpy(newData, m_data, static_cast<size_t>(m_position));\n    delete [] m_data;\n    m_data = newData;\n    m_capacity = newCapacity;\n\n  }\n\n}\n\np_char8 BufferOutputStream::getData() {\n  return m_data;\n}\n\n\nv_buff_size BufferOutputStream::getCapacity() {\n  return m_capacity;\n}\n\n\nv_buff_size BufferOutputStream::getCurrentPosition() {\n  return m_position;\n}\n\n\nvoid BufferOutputStream::setCurrentPosition(v_buff_size position) {\n  m_position = position;\n}\n\nvoid BufferOutputStream::reset(v_buff_size initialCapacity) {\n  delete [] m_data;\n  m_data = new v_char8[initialCapacity];\n  m_capacity = initialCapacity;\n  m_position = 0;\n}\n\noatpp::String BufferOutputStream::toString() {\n  return oatpp::String(reinterpret_cast<const char*>(m_data), m_position);\n}\n\noatpp::String BufferOutputStream::getSubstring(v_buff_size pos, v_buff_size count) {\n  if(pos + count <= m_position) {\n    return oatpp::String(reinterpret_cast<const char*>(m_data + pos), count);\n  } else {\n    return oatpp::String(reinterpret_cast<const char*>(m_data + pos), m_position - pos);\n  }\n}\n\noatpp::v_io_size BufferOutputStream::flushToStream(OutputStream* stream) {\n  return stream->writeExactSizeDataSimple(m_data, m_position);\n}\n\noatpp::async::CoroutineStarter BufferOutputStream::flushToStreamAsync(const std::shared_ptr<BufferOutputStream>& _this, const std::shared_ptr<OutputStream>& stream) {\n\n  class WriteDataCoroutine : public oatpp::async::Coroutine<WriteDataCoroutine> {\n  private:\n    std::shared_ptr<BufferOutputStream> m_this;\n    std::shared_ptr<oatpp::data::stream::OutputStream> m_stream;\n    data::buffer::InlineWriteData m_inlineData;\n  public:\n\n    WriteDataCoroutine(const std::shared_ptr<BufferOutputStream>& _this,\n                       const std::shared_ptr<oatpp::data::stream::OutputStream>& stream)\n      : m_this(_this)\n      , m_stream(stream)\n    {}\n\n    Action act() override {\n      if(m_inlineData.currBufferPtr == nullptr) {\n        m_inlineData.currBufferPtr = m_this->m_data;\n        m_inlineData.bytesLeft = m_this->m_position;\n      }\n      return m_stream.get()->writeExactSizeDataAsyncInline(m_inlineData, finish());\n    }\n\n  };\n\n  return WriteDataCoroutine::start(_this, stream);\n\n}\n\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// BufferInputStream\n\ndata::stream::DefaultInitializedContext BufferInputStream::DEFAULT_CONTEXT(data::stream::StreamType::STREAM_FINITE);\n\nBufferInputStream::BufferInputStream(const std::shared_ptr<std::string>& memoryHandle,\n                                     const void* data,\n                                     v_buff_size size,\n                                     const std::shared_ptr<void>& captureData)\n  : m_memoryHandle(memoryHandle)\n  , m_data(reinterpret_cast<p_char8>(const_cast<void*>(data)))\n  , m_size(size)\n  , m_position(0)\n  , m_ioMode(IOMode::ASYNCHRONOUS)\n  , m_capturedData(captureData)\n{}\n\nBufferInputStream::BufferInputStream(const oatpp::String& data, const std::shared_ptr<void>& captureData)\n  : BufferInputStream(data.getPtr(), reinterpret_cast<p_char8>(const_cast<char*>(data->data())), static_cast<v_buff_size>(data->size()), captureData)\n{}\n\nvoid BufferInputStream::reset(const std::shared_ptr<std::string>& memoryHandle,\n                              p_char8 data,\n                              v_buff_size size,\n                              const std::shared_ptr<void>& captureData)\n{\n  m_memoryHandle = memoryHandle;\n  m_data = data;\n  m_size = size;\n  m_position = 0;\n  m_capturedData = captureData;\n}\n\nvoid BufferInputStream::reset() {\n  m_memoryHandle = nullptr;\n  m_data = nullptr;\n  m_size = 0;\n  m_position = 0;\n  m_capturedData.reset();\n}\n\nv_io_size BufferInputStream::read(void *data, v_buff_size count, async::Action& action) {\n\n  (void) action;\n\n  v_buff_size desiredAmount = count;\n  if(desiredAmount > m_size - m_position) {\n    desiredAmount = m_size - m_position;\n  }\n  std::memcpy(data, &m_data[m_position], static_cast<size_t>(desiredAmount));\n  m_position += desiredAmount;\n  return desiredAmount;\n}\n\nvoid BufferInputStream::setInputStreamIOMode(IOMode ioMode) {\n  m_ioMode = ioMode;\n}\n\nIOMode BufferInputStream::getInputStreamIOMode() {\n  return m_ioMode;\n}\n\nContext& BufferInputStream::getInputStreamContext() {\n  return DEFAULT_CONTEXT;\n}\n\nstd::shared_ptr<std::string> BufferInputStream::getDataMemoryHandle() {\n  return m_memoryHandle;\n}\n\np_char8 BufferInputStream::getData() {\n  return m_data;\n}\n\nv_buff_size BufferInputStream::getDataSize() {\n  return m_size;\n}\n\nv_buff_size BufferInputStream::getCurrentPosition() {\n  return m_position;\n}\n\nvoid BufferInputStream::setCurrentPosition(v_buff_size position) {\n  m_position = position;\n}\n\nv_io_size BufferInputStream::peek(void *data, v_buff_size count, async::Action &action) {\n  (void) action;\n\n  v_buff_size desiredAmount = count;\n  if(desiredAmount > m_size - m_position) {\n    desiredAmount = m_size - m_position;\n  }\n  std::memcpy(data, &m_data[m_position], static_cast<size_t>(desiredAmount));\n  return desiredAmount;\n}\n\nv_io_size BufferInputStream::availableToRead() const {\n  return m_size - m_position;\n}\n\nv_io_size BufferInputStream::commitReadOffset(v_buff_size count) {\n  if(count > m_size - m_position) {\n    count = m_size - m_position;\n  }\n  m_position += count;\n  return count;\n}\n\n}}}\n\n// Path: src/oatpp/core/data/resource/Resource.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_resource_Resource_hpp\n#define oatpp_data_resource_Resource_hpp\n\n#include \"oatpp/core/data/stream/Stream.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\n/**\n * Abstract data resource\n */\nclass Resource : public oatpp::base::Countable {\npublic:\n\n  /**\n   * virtual destructor.\n   */\n  virtual ~Resource() override = default;\n\n  /**\n   * Open output stream.\n   * @return\n   */\n  virtual std::shared_ptr<data::stream::OutputStream> openOutputStream() = 0;\n\n  /**\n   * Open input stream.\n   * @return\n   */\n  virtual std::shared_ptr<data::stream::InputStream> openInputStream() = 0;\n\n  /**\n   * Get in-memory data if applicable.\n   * @return - `&id:oatpp::String;` or `nullptr` if not applicable.\n   */\n  virtual oatpp::String getInMemoryData() = 0;\n\n  /**\n   * Get known data size if applicable.\n   * @return - known size of the data. `-1` - if size is unknown.\n   */\n  virtual v_int64 getKnownSize() = 0;\n\n  /**\n   * Get resource location if applicable. <br>\n   * location can be for example a file name.\n   * @return - `&id:oatpp::String;` or `nullptr` if not applicable.\n   */\n  virtual oatpp::String getLocation() = 0;\n\n};\n\n}}}\n\n#endif //oatpp_data_resource_Resource_hpp\n\n// Path: src/oatpp/core/data/resource/InMemoryData.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_resource_InMemoryData_hpp\n#define oatpp_data_resource_InMemoryData_hpp\n\n#include \"./Resource.hpp\"\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\nclass InMemoryData : public Resource {\nprivate:\n\n  struct DataHandle {\n\n    oatpp::String data;\n\n    \nDataHandle(const oatpp::String& pData)\n      : data(pData)\n    {}\n\n  };\n\n  struct OutputDataHandle {\n\n    std::shared_ptr<DataHandle> dataHandle;\n    oatpp::String data;\n    data::stream::BufferOutputStream* stream;\n\n    ~OutputDataHandle();\n\n  };\n\nprivate:\n  std::shared_ptr<DataHandle> m_handle;\npublic:\n\n  /**\n   * Default constructor.\n   */\n  InMemoryData() = default;\n\n  /**\n   * Constructor.\n   * @param fullInMemoryDataname\n   */\n  InMemoryData(const oatpp::String& data);\n\n  /**\n   * Open output stream to an InMemoryData. <br>\n   * NOT thread-safe. <br>\n   * *Note: data is committed once stream is closed.* <br>\n   * *Note: stream also captures data-handle. The data won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` to &id:oatpp::data::stream::OutputStream;.\n   */\n  std::shared_ptr<data::stream::OutputStream> openOutputStream() override;\n\n  /**\n   * Open input stream to an InMemoryData. <br>\n   * NOT thread-safe. <br>\n   * *Note: once the stream is open no subsequent writes through the output stream won't affect currently opened input streams.* <br>\n   * *Note: stream also captures file-handle. The data won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` &id:oatpp::data::stream::InputStream;.\n   */\n  std::shared_ptr<data::stream::InputStream> openInputStream() override;\n\n  /**\n   * Get in-memory-data.\n   * @return - always returns `nullptr`.\n   */\n  oatpp::String getInMemoryData() override;\n\n  /**\n   * Get size of an in-memory-data.\n   * @return - size of the data in bytes.\n   */\n  v_int64 getKnownSize() override;\n\n  /**\n   * Not applicable.\n   * @return - always returns `nullptr`.\n   */\n  oatpp::String getLocation() override;\n\n};\n\n}}}\n\n#endif //oatpp_data_resource_InMemoryData_hpp\n\n// Path: src/oatpp/core/data/resource/InMemoryData.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"InMemoryData.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\nInMemoryData::OutputDataHandle::~OutputDataHandle() {\n  dataHandle->data = data + stream->toString();\n}\n\nInMemoryData::InMemoryData(const oatpp::String& data)\n  : m_handle(std::make_shared<DataHandle>(data))\n{}\n\nstd::shared_ptr<data::stream::OutputStream> InMemoryData::openOutputStream() {\n  auto outputDataHandle = std::make_shared<OutputDataHandle>();\n  if(!m_handle) {\n    m_handle = std::make_shared<DataHandle>(\"\");\n  }\n  if(!m_handle->data){\n    m_handle->data = \"\";\n  }\n  outputDataHandle->dataHandle = m_handle;\n  outputDataHandle->data = m_handle->data;\n\n  auto stream = std::make_shared<data::stream::BufferOutputStream>(1024, outputDataHandle);\n  outputDataHandle->stream = stream.get();\n  return stream;\n}\n\nstd::shared_ptr<data::stream::InputStream> InMemoryData::openInputStream() {\n  if(!m_handle) {\n    m_handle = std::make_shared<DataHandle>(\"\");\n  }\n  if(!m_handle->data){\n    m_handle->data = \"\";\n  }\n  return std::make_shared<data::stream::BufferInputStream>(m_handle->data, m_handle);\n}\n\noatpp::String InMemoryData::getInMemoryData() {\n  if(m_handle && m_handle->data) {\n    return m_handle->data;\n  }\n  return nullptr;\n}\n\nv_int64 InMemoryData::getKnownSize() {\n  if(m_handle && m_handle->data) {\n    return static_cast<v_int64>(m_handle->data->size());\n  }\n  return 0;\n}\n\noatpp::String InMemoryData::getLocation() {\n  return nullptr;\n}\n\n}}}\n\n// Path: src/oatpp/core/data/resource/File.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_resource_File_hpp\n#define oatpp_data_resource_File_hpp\n\n#include \"./Resource.hpp\"\n#include \"oatpp/core/data/stream/Stream.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\n/**\n * File.\n * @extends - &id:oatpp::data::Resource;.\n */\nclass File : public Resource {\nprivate:\n  \n  struct FileHandle {\n\n    oatpp::String fileName;\n\n    FileHandle(const oatpp::String& fullFileName)\n      : fileName(fullFileName)\n    {}\n\n  };\n\npublic:\n  static oatpp::String concatDirAndName(const oatpp::String& dir, const oatpp::String& filename);\nprivate:\n  std::shared_ptr<FileHandle> m_handle;\npublic:\n\n  /**\n   * Default constructor.\n   */\n  File() = default;\n\n  /**\n   * Constructor.\n   * @param fullFilename\n   */\n  File(const oatpp::String& fullFilename);\n\n  /**\n   * Constructor.\n   * @param directory\n   * @param filename\n   */\n  File(const oatpp::String& directory, const oatpp::String& filename);\n\n  /**\n   * Open output stream to a file. <br>\n   * *Note: stream also captures file-handle. The file object won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` to &id:oatpp::data::stream::OutputStream;.\n   */\n  std::shared_ptr<data::stream::OutputStream> openOutputStream() override;\n\n  /**\n   * Open input stream to a temporary file. <br>\n   * *Note: stream also captures file-handle. The file won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` &id:oatpp::data::stream::InputStream;.\n   */\n  std::shared_ptr<data::stream::InputStream> openInputStream() override;\n\n  /**\n   * Not applicable.\n   * @return - always returns `nullptr`.\n   */\n  oatpp::String getInMemoryData() override;\n\n  /**\n   * Not applicable.\n   * @return - always returns `-1`.\n   */\n  v_int64 getKnownSize() override;\n\n  /**\n   * Get location where temporary data is stored.\n   * @return - `&id:oatpp::String;`.\n   */\n  oatpp::String getLocation() override;\n\n};\n\n}}}\n\n#endif //oatpp_data_resource_File_hpp\n\n// Path: src/oatpp/core/data/resource/File.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"File.hpp\"\n\n#include \"oatpp/core/data/stream/FileStream.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\noatpp::String File::concatDirAndName(const oatpp::String& dir, const oatpp::String& filename) {\n  if(dir && dir->size() > 0) {\n    auto lastChar = dir->data()[dir->size() - 1];\n    if(lastChar != '/' && lastChar != '\\\\') {\n      return dir + \"/\" + filename;\n    }\n    return dir + filename;\n  }\n  return filename;\n}\n\nFile::File(const oatpp::String& fullFileName)\n  : m_handle(std::make_shared<FileHandle>(fullFileName))\n{}\n\nFile::File(const oatpp::String& tmpDirectory, const oatpp::String& tmpFileName)\n  : m_handle(std::make_shared<FileHandle>(concatDirAndName(tmpDirectory, tmpFileName)))\n{}\n\nstd::shared_ptr<data::stream::OutputStream> File::openOutputStream() {\n  if(m_handle) {\n    return std::make_shared<data::stream::FileOutputStream>(m_handle->fileName->c_str(), \"wb\", m_handle);\n  }\n  throw std::runtime_error(\"[oatpp::data::resource::File::openOutputStream()]: Error. FileHandle is NOT initialized.\");\n}\n\nstd::shared_ptr<data::stream::InputStream> File::openInputStream() {\n  if(m_handle) {\n    return std::make_shared<data::stream::FileInputStream>(m_handle->fileName->c_str(), m_handle);\n  }\n  throw std::runtime_error(\"[oatpp::data::resource::File::openInputStream()]: Error. FileHandle is NOT initialized.\");\n}\n\noatpp::String File::getInMemoryData() {\n  return nullptr;\n}\n\nv_int64 File::getKnownSize() {\n  return -1;\n}\n\noatpp::String File::getLocation() {\n  if(m_handle) {\n    return m_handle->fileName;\n  }\n  return nullptr;\n}\n\n}}}\n\n// Path: src/oatpp/core/data/resource/TemporaryFile.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_resource_TemporaryFile_hpp\n#define oatpp_data_resource_TemporaryFile_hpp\n\n#include \"./Resource.hpp\"\n#include \"oatpp/core/Types.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\n/**\n * Temporary file - the file which gets deleted when the destructor is called\n * (more precisely when all copies of the same `TemporaryFile` object deleted). <br>\n * The `TemporaryFile` object internally stores a `shared_ptr` to a file handle.\n * When file handle deleted it also deletes the underlying file. <br>\n * Thus it's safe to copy `TemporaryFile` object and you may treat `TemporaryFile` object\n * as a shared_ptr to a temporary file. <br>\n * @extends - &id:oatpp::data::Resource;.\n */\nclass TemporaryFile : public Resource {\nprivate:\n\n  /*\n   * Shared handle.\n   * File is deleted on handle destroy.\n   */\n  struct FileHandle {\n\n    oatpp::String fileName;\n\n    FileHandle(const oatpp::String& fullFileName)\n      : fileName(fullFileName)\n    {}\n\n    ~FileHandle();\n  };\n\nprivate:\n  static oatpp::String constructRandomFilename(const oatpp::String &dir, v_int32 randomWordSizeBytes, const oatpp::String &extension);\nprivate:\n  std::shared_ptr<FileHandle> m_handle;\npublic:\n\n  /**\n   * Default constructor.\n   */\n  TemporaryFile() = default;\n\n  /**\n   * Constructor. <br>\n   * Create temporary file with a random name in the `tmpDirectory`. <br>\n   * The actual file will be created only after first write to that file. <br>\n   * Example of the generated random file name: `f7c6ecd44024ff31.tmp`.\n   * @param tmpDirectory - directory where to create a temporary file.\n   * @param randomWordSizeBytes - number of random bytes to generate file name.\n   */\n  TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes = 8);\n\n  /**\n   * Constructor.<br>\n   * Create temporary file with the `tmpFileName` name in the `tmpDirectory`. <br>\n   * @param tmpDirectory - directory where to create a temporary file.\n   * @param tmpFileName - predefined name for the temporary file.\n   */\n  TemporaryFile(const oatpp::String& tmpDirectory, const oatpp::String& tmpFileName);\n\n  /**\n   * Constructor. <br>\n   * Create temporary file with a random name and specified extension in the `tmpDirectory`. <br>\n   * The actual file will be created only after first write to that file. <br>\n   * Example of the generated random file name: `f7c6ecd44024ff31.txt`.\n   * @param tmpDirectory - directory where to create a temporary file.\n   * @param randomWordSizeBytes - number of random bytes to generate file name.\n   * @param extension - extension of the temporary file, e.g. txt or .txt\n   */\n  TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes, const oatpp::String& extension);\n\n  /**\n   * Open output stream to a temporary file. <br>\n   * *Note: stream also captures file-handle. The temporary file won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` to &id:oatpp::data::stream::OutputStream;.\n   */\n  std::shared_ptr<data::stream::OutputStream> openOutputStream() override;\n\n  /**\n   * Open input stream to a temporary file. <br>\n   * *Note: stream also captures file-handle. The temporary file won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` &id:oatpp::data::stream::InputStream;.\n   */\n  std::shared_ptr<data::stream::InputStream> openInputStream() override;\n\n  /**\n   * Not applicable.\n   * @return - always returns `nullptr`.\n   */\n  oatpp::String getInMemoryData() override;\n\n  /**\n   * Not applicable.\n   * @return - always returns `-1`.\n   */\n  v_int64 getKnownSize() override;\n\n  /**\n   * Get location where temporary data is stored.\n   * @return - `&id:oatpp::String;`.\n   */\n  oatpp::String getLocation() override;\n\n  /**\n   * Move payload to a different file. <br>\n   * @param fullFileName - full-file-name. Note: all the parent folders must exist.\n   * @return - `true` - file was successfully moved, `false` - otherwise.\n   */\n  bool moveFile(const oatpp::String& fullFileName);\n\n};\n\n}}}\n\n#endif //oatpp_data_resource_TemporaryFile_hpp\n\n// Path: src/oatpp/core/data/resource/TemporaryFile.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"TemporaryFile.hpp\"\n\n#include \"./File.hpp\"\n\n#include \"oatpp/core/data/stream/FileStream.hpp\"\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n#include \"oatpp/encoding/Hex.hpp\"\n#include \"oatpp/core/utils/Random.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\nTemporaryFile::FileHandle::~FileHandle() {\n  if(fileName) {\n    std::remove(fileName->c_str());\n  }\n}\n\noatpp::String TemporaryFile::constructRandomFilename(const oatpp::String &dir, v_int32 randomWordSizeBytes, const oatpp::String &extension) {\n\n  std::unique_ptr<v_char8[]> buff(new v_char8[randomWordSizeBytes]);\n  utils::random::Random::randomBytes(buff.get(), randomWordSizeBytes);\n  data::stream::BufferOutputStream s(randomWordSizeBytes * 2 + 4);\n  encoding::Hex::encode(&s, buff.get(), randomWordSizeBytes, encoding::Hex::ALPHABET_LOWER);\n  if (extension->at(0) != '.') {\n    s << \".\";\n  }\n  s << extension;\n\n  return File::concatDirAndName(dir, s.toString());\n\n}\n\nTemporaryFile::TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes)\n  : m_handle(std::make_shared<FileHandle>(constructRandomFilename(tmpDirectory, randomWordSizeBytes, \"tmp\")))\n{}\n\nTemporaryFile::TemporaryFile(const oatpp::String& tmpDirectory, const oatpp::String& tmpFileName)\n  : m_handle(std::make_shared<FileHandle>(File::concatDirAndName(tmpDirectory, tmpFileName)))\n{}\n\nTemporaryFile::TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes, const oatpp::String& extension)\n  : m_handle(std::make_shared<FileHandle>(constructRandomFilename(tmpDirectory, randomWordSizeBytes, extension)))\n{}\n\nstd::shared_ptr<data::stream::OutputStream> TemporaryFile::openOutputStream() {\n  if(m_handle) {\n    return std::make_shared<data::stream::FileOutputStream>(m_handle->fileName->c_str(), \"wb\", m_handle);\n  }\n  throw std::runtime_error(\"[oatpp::data::resource::TemporaryFile::openOutputStream()]: Error. FileHandle is NOT initialized.\");\n}\n\nstd::shared_ptr<data::stream::InputStream> TemporaryFile::openInputStream() {\n  if(m_handle) {\n    return std::make_shared<data::stream::FileInputStream>(m_handle->fileName->c_str(), m_handle);\n  }\n  throw std::runtime_error(\"[oatpp::data::resource::TemporaryFile::openInputStream()]: Error. FileHandle is NOT initialized.\");\n}\n\noatpp::String TemporaryFile::getInMemoryData() {\n  return nullptr;\n}\n\nv_int64 TemporaryFile::getKnownSize() {\n  return -1;\n}\n\noatpp::String TemporaryFile::getLocation() {\n  if(m_handle) {\n    return m_handle->fileName;\n  }\n  return nullptr;\n}\n\nbool TemporaryFile::moveFile(const oatpp::String& fullFileName) {\n  if(m_handle) {\n    return std::rename(m_handle->fileName->c_str(), fullFileName->c_str()) == 0;\n  }\n  return false;\n}\n\n}}}\n\n// Path: src/oatpp/core/data/buffer/Processor.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_buffer_Processor_hpp\n#define oatpp_data_buffer_Processor_hpp\n\n#include \"oatpp/core/IODefinitions.hpp\"\n#include \"oatpp/core/base/ObjectHandle.hpp\"\n#include <vector>\n\nnamespace oatpp { namespace data { namespace buffer {\n\n/**\n * Convenience structure for stream Async-Inline read operations.\n */\nstruct InlineReadData {\n\n  /**\n   * Pointer to current position in the buffer.\n   */\n  void* currBufferPtr;\n\n  /**\n   * Bytes left to read to the buffer.\n   */\n  v_buff_size bytesLeft;\n\n  /**\n   * Default constructor.\n   */\n  InlineReadData();\n\n  /**\n   * Constructor.\n   * @param data\n   * @param size\n   */\n  InlineReadData(void* data, v_buff_size size);\n\n  /**\n   * Set `currBufferPtr` and `bytesLeft` values. <br>\n   * @param data - pointer to buffer to store read data.\n   * @param size - size in bytes of the buffer.\n   */\n  void set(void* data, v_buff_size size);\n\n  /**\n   * Increase position in the read buffer by `amount` bytes. <br>\n   * This will increase `currBufferPtr` and descrease `bytesLeft` values.\n   * @param amount\n   */\n  void inc(v_buff_size amount);\n\n  /**\n   * Same as `inc(bytesLeft).`\n   */\n  void setEof();\n\n};\n\n/**\n * Convenience structure for stream Async-Inline write operations.\n */\nstruct InlineWriteData {\n\n  /**\n   * Pointer to current position in the buffer.\n   */\n  const void* currBufferPtr;\n\n  /**\n   * Bytes left to write from the buffer.\n   */\n  v_buff_size bytesLeft;\n\n  /**\n   * Default constructor.\n   */\n  InlineWriteData();\n\n  /**\n   * Constructor.\n   * @param data\n   * @param size\n   */\n  InlineWriteData(const void* data, v_buff_size size);\n\n  /**\n   * Set `currBufferPtr` and `bytesLeft` values. <br>\n   * @param data - pointer to buffer containing data to be written.\n   * @param size - size in bytes of the buffer.\n   */\n  void set(const void* data, v_buff_size size);\n\n  /**\n   * Increase position in the write buffer by `amount` bytes. <br>\n   * This will increase `currBufferPtr` and descrease `bytesLeft` values.\n   * @param amount\n   */\n  void inc(v_buff_size amount);\n\n  /**\n   * Same as `inc(bytesLeft).`\n   */\n  void setEof();\n\n};\n\n/**\n * Buffer processor.\n * Note: all processors are considered to be stateful.\n */\nclass Processor {\npublic:\n\n  /**\n   * Enum of processing errors.\n   */\n  enum Error : v_int32 {\n\n    /**\n     * No error.\n     */\n    OK = 0,\n\n    /**\n     * Caller must set fields of `dataIn` parameter.\n     */\n    PROVIDE_DATA_IN = 1,\n\n    /**\n     * Caller must read all the data from the `dataOut`.\n     */\n    FLUSH_DATA_OUT = 2,\n\n    /**\n     * Processing is finished.\n     */\n    FINISHED = 3\n\n    //*********************************************//\n    // Other values are processor-specific errors. //\n    //*********************************************//\n  };\n\npublic:\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~Processor() = default;\n\n  /**\n   * If the client is using the input stream to read data and add it to the processor,\n   * the client MAY ask the processor for a suggested read size.\n   * @return - suggested read size.\n   */\n  virtual v_io_size suggestInputStreamReadSize() = 0;\n\n  /**\n   * Process data.\n   * @param dataIn - data provided by client to processor. Input data. &id:data::buffer::InlineReadData;.\n   * Set `dataIn` buffer pointer to `nullptr` to designate the end of input.\n   * @param dataOut - data provided to client by processor. Output data. &id:data::buffer::InlineReadData;.\n   * @return - &l:Processor::Error;.\n   */\n  virtual v_int32 iterate(data::buffer::InlineReadData& dataIn,\n                          data::buffer::InlineReadData& dataOut) = 0;\n\n};\n\n/**\n * Pipeline of buffer processors.\n */\nclass ProcessingPipeline : public Processor {\nprivate:\n  std::vector<base::ObjectHandle<Processor>> m_processors;\n  std::vector<data::buffer::InlineReadData> m_intermediateData;\npublic:\n\n  /**\n   * Constructor.\n   * @param m_processors - the array of processors defining the pipeline.\n   */\n  ProcessingPipeline(const std::vector<base::ObjectHandle<Processor>>& m_processors);\n\n  /**\n   * If the client is using the input stream to read data and add it to the processor,\n   * the client MAY ask the processor for a suggested read size.\n   * @return - suggested read size.\n   */\n  v_io_size suggestInputStreamReadSize() override;\n\n  /**\n   * Process data.\n   * @param dataIn - data provided by client to processor. Input data. &id:data::buffer::InlineReadData;.\n   * Set `dataIn` buffer pointer to `nullptr` to designate the end of input.\n   * @param dataOut - data provided to client by processor. Output data. &id:data::buffer::InlineReadData;.\n   * @return - &l:Processor::Error;.\n   */\n  v_int32 iterate(data::buffer::InlineReadData& dataIn,\n                  data::buffer::InlineReadData& dataOut) override;\n\n};\n\n}}}\n\n#endif // oatpp_data_buffer_Processor_hpp\n\n// Path: src/oatpp/core/data/buffer/Processor.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Processor.hpp\"\n\nnamespace oatpp { namespace data{ namespace buffer {\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// InlineReadData\n\nInlineReadData::InlineReadData()\n  : currBufferPtr(nullptr)\n  , bytesLeft(0)\n{}\n\nInlineReadData::InlineReadData(void* data, v_buff_size size)\n  : currBufferPtr(data)\n  , bytesLeft(size)\n{}\n\nvoid InlineReadData::set(void* data, v_buff_size size) {\n  currBufferPtr = data;\n  bytesLeft = size;\n}\n\nvoid InlineReadData::inc(v_buff_size amount) {\n  currBufferPtr = &(reinterpret_cast<p_char8>(currBufferPtr))[amount];\n  bytesLeft -= amount;\n}\n\nvoid InlineReadData::setEof() {\n  currBufferPtr = &(reinterpret_cast<p_char8>(currBufferPtr))[bytesLeft];\n  bytesLeft = 0;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// InlineWriteData\n\nInlineWriteData::InlineWriteData()\n  : currBufferPtr(nullptr)\n  , bytesLeft(0)\n{}\n\nInlineWriteData::InlineWriteData(const void* data, v_buff_size size)\n  : currBufferPtr(data)\n  , bytesLeft(size)\n{}\n\nvoid InlineWriteData::set(const void* data, v_buff_size size) {\n  currBufferPtr = data;\n  bytesLeft = size;\n}\n\nvoid InlineWriteData::inc(v_buff_size amount) {\n  currBufferPtr = &(reinterpret_cast<p_char8>(const_cast<void*>(currBufferPtr)))[amount];\n  bytesLeft -= amount;\n}\n\nvoid InlineWriteData::setEof() {\n  currBufferPtr = &(reinterpret_cast<p_char8>(const_cast<void*>(currBufferPtr)))[bytesLeft];\n  bytesLeft = 0;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n...\n", "needle_token_start": 7087, "needle_token_end": 7111, "code_context_ntokens": 15764, "output": ["```cpp\nclass InMemoryData : public Resource {\nprivate:\n  struct DataHandle {\n    oatpp::String data;\n    DataHandle(const oatpp::String& pData) : data(pData) {}\n  };\n  std::shared_ptr<DataHandle> m_handle;\npublic:\n  InMemoryData() = default;\n  InMemoryData(const oatpp::String& data);\n  std::shared_ptr<data::stream::OutputStream> openOutputStream() override;\n  std::shared_ptr<data::stream::InputStream> openInputStream() override;\n  oatpp::String getInMemoryData() override;\n  v_int64 getKnownSize() override;\n  oatpp::String getLocation() override;\n};\n```"]}
{"repo": "oatpp/oatpp", "name": "ListenerCall", "language": "cpp", "path": "src/oatpp/web/mime/multipart/StatefulParser.hpp", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this class is to manage the invocation of specific callback functions based on the type of event occurring during the parsing of multipart data streams. It determines whether headers or data chunks are being processed and triggers the appropriate event handlers.\n2. **Input**: This class takes no direct inputs when instantiated but uses internal state variables to manage the type of callback to invoke (either for headers or data chunks) and the associated data.\n3. **Output**: There is no direct output produced by instances of this class. Instead, it indirectly affects the system by triggering specific callback functions which handle the parsed data.\n4. **Procedure**: Upon instantiation, the class initializes with no scheduled callback. When the parsing process identifies a new part or data chunk, the class sets the appropriate callback type and data pointers. Depending on the internal state, it then triggers either the header processing or data processing callbacks at the appropriate times in the parsing lifecycle.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/encoding/Unicode.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Unicode.hpp\"\n\n#if defined(WIN32) || defined(_WIN32)\n  #include <winsock2.h>\n#else\n  #include <arpa/inet.h>\n#endif\n\nnamespace oatpp { namespace encoding {\n  \nv_buff_size Unicode::getUtf8CharSequenceLength(v_char8 firstByte) {\n  \n  if(firstByte < 128){\n    return 1;\n  }\n  \n  if((firstByte | 192) != firstByte){\n    return 0;\n  }\n  \n  if((firstByte | 32) != firstByte){\n    return 2;\n  } else if((firstByte | 16) != firstByte){\n    return 3;\n  } else if((firstByte | 8) != firstByte){\n    return 4;\n  } else if((firstByte | 4) != firstByte){\n    return 5;\n  } else if((firstByte | 2) != firstByte){\n    return 6;\n  } else {\n    return 0;\n  }\n  \n}\n  \nv_buff_size Unicode::getUtf8CharSequenceLengthForCode(v_uint32 code){\n  if(code < 128) {\n    return 1;\n  } else if(code < 0x00000800){\n    return 2;\n  } else if(code < 0x00010000){\n    return 3;\n  } else if(code < 0x00200000){\n    return 4;\n  } else if(code < 0x04000000){\n    return 5;\n  } else {\n    return 6;\n  }\n}\n  \nv_int32 Unicode::encodeUtf8Char(const char* sequence, v_buff_size& length){\n  v_char8 byte = static_cast<v_char8>(sequence[0]);\n  if(byte > 127){\n    v_int32 code;\n    if((byte | 32) != byte){\n      length = 2;\n      code =  ((31 & byte) << 6) | (sequence[1] & 63);\n      return code;\n    } else if((byte | 16) != byte){\n      code = (15 & byte) << 12;\n      length = 3;\n    } else if((byte | 8) != byte){\n      length = 4;\n      v_int32 value = *(reinterpret_cast<p_int32>(const_cast<char*>(sequence)));\n      code =  ((7 & byte) << 18)                  |\n              (((value >> 24) & 0xFF) & 63)       |\n              (((value >> 16) & 0xFF) & 63) << 6  |\n              (((value >>  8) & 0xFF) & 63) << 12;\n      return code;\n    } else if((byte | 4) != byte){\n      code = (3 & byte) << 24;\n      length = 5;\n    } else if((byte | 2) != byte){\n      code = (1 & byte) << 30;\n      length = 6;\n    } else {\n      return -1;\n    }\n    \n    v_char8 bitIndex = 0;\n    for(v_buff_size i = length; i > 1; i--){\n      code |= (sequence[i - 1] & 63) << bitIndex;\n      bitIndex = static_cast<v_char8>(bitIndex + 6);\n    }\n    return code;\n  } else {\n    length = 1;\n    return byte;\n  }\n}\n  \nv_buff_size Unicode::decodeUtf8Char(v_int32 signed_code, p_char8 buffer) {\n  v_uint32 code = static_cast<v_uint32>(signed_code);\n  if(code >= 0x00000080 && code < 0x00000800){\n    *(reinterpret_cast<p_int16>(buffer)) = static_cast<v_int16>(htons(((((code >> 6) & 31) | 192) << 8) | ((code & 63) | 128)));\n    return 2;\n  } else if(code >= 0x00000800 && code < 0x00010000){\n...\n// Path: src/oatpp/encoding/Url.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_encoding_Url_hpp\n#define oatpp_encoding_Url_hpp\n\n#include \"Hex.hpp\"\n#include \"oatpp/core/data/stream/Stream.hpp\"\n\nnamespace oatpp { namespace encoding {\n\nclass Url {\npublic:\n\n  struct Config {\n\n    bool spaceToPlus = false;\n    const char* hexAlphabet = Hex::ALPHABET_UPPER;\n    bool allowedChars[256];\n\n    Config();\n\n    void allowChar(v_char8 c);\n    void allowCharRange(v_char8 from, v_char8 to);\n\n    void disallowChar(v_char8 c);\n    void disallowCharRange(v_char8 from, v_char8 to);\n\n  };\n\npublic:\n\n  static void encode(data::stream::ConsistentOutputStream* stream, const void* data, v_buff_size size, const Config& config);\n  static void decode(data::stream::ConsistentOutputStream* stream, const void* data, v_buff_size size);\n\n  static oatpp::String encode(const oatpp::String& data, const Config& config);\n  static oatpp::String decode(const oatpp::String& data);\n\n};\n\n}}\n\n#endif //oatpp_encoding_Url_hpp\n\n// Path: src/oatpp/encoding/Url.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Url.hpp\"\n\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n\nnamespace oatpp { namespace encoding {\n\nUrl::Config::Config() {\n\n  disallowCharRange(0, 255);\n\n  allowCharRange('0', '9');\n  allowCharRange('a', 'z');\n  allowCharRange('A', 'Z');\n\n  allowChar('-');\n  allowChar('.');\n  allowChar('_');\n  allowChar('~');\n\n}\n\nvoid Url::Config::allowChar(v_char8 c) {\n  allowedChars[c] = true;\n}\n\nvoid Url::Config::allowCharRange(v_char8 from, v_char8 to) {\n  for(v_int32 i = from; i <= to; i++) {\n    allowedChars[i] = true;\n  }\n}\n\nvoid Url::Config::disallowChar(v_char8 c) {\n  allowedChars[c] = false;\n}\n\nvoid Url::Config::disallowCharRange(v_char8 from, v_char8 to) {\n  for(v_int32 i = from; i <= to; i++) {\n    allowedChars[i] = false;\n  }\n}\n\nvoid Url::encode(data::stream::ConsistentOutputStream *stream, const void *data, v_buff_size size, const Config& config) {\n\n  auto pdata = reinterpret_cast<const char*>(data);\n\n  for(v_buff_size i = 0; i < size; i++) {\n    v_char8 c = static_cast<v_char8>(pdata[i]);\n    if(config.allowedChars[c]) {\n      stream->writeCharSimple(c);\n    } else if(c == ' ' && config.spaceToPlus) {\n      stream->writeCharSimple('+');\n    } else {\n        stream->writeCharSimple('%');\n        Hex::encode(stream, pdata + i, 1, config.hexAlphabet);\n    }\n  }\n\n}\n\nvoid Url::decode(data::stream::ConsistentOutputStream* stream, const void* data, v_buff_size size) {\n\n  auto pdata = reinterpret_cast<const char*>(data);\n  v_buff_size i = 0;\n\n  while (i < size) {\n\n    v_char8 c = static_cast<v_char8>(pdata[i]);\n    if(c == '%') {\n      if(size - i > 1) {\n        Hex::decode(stream, pdata + i + 1, 2);\n        i += 3;\n      } else {\n        break;\n      }\n    } else if (c == '+') {\n      stream->writeCharSimple(' ');\n      i ++;\n    } else {\n        stream->writeCharSimple(c);\n        i ++;\n    }\n\n  }\n\n}\n\noatpp::String Url::encode(const oatpp::String& data, const Config& config) {\n  if(!data) return nullptr;\n  data::stream::BufferOutputStream stream(static_cast<v_buff_size>(data->size() * 3));\n  encode(&stream, data->data(), static_cast<v_buff_size>(data->size()), config);\n  return stream.toString();\n}\n\noatpp::String Url::decode(const oatpp::String& data) {\n  if(!data) return nullptr;\n  data::stream::BufferOutputStream stream(static_cast<v_buff_size>(data->size()));\n  decode(&stream, data->data(), static_cast<v_buff_size>(data->size()));\n  return stream.toString();\n}\n\n}}\n\n// Path: src/oatpp/algorithm/CRC.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_algorithm_CRC_hpp\n#define oatpp_algorithm_CRC_hpp\n\n#include \"oatpp/core/base/Environment.hpp\"\n\n#include \"oatpp/encoding/Hex.hpp\"\n\nnamespace oatpp { namespace algorithm {\n\n/**\n * Implementation of CRC-32. Cyclic redundancy check algorithm.\n */\nclass CRC32 {\npublic:\n\n  /**\n   * Precalculated table\n   */\n  static const p_uint32 TABLE_04C11DB7;\npublic:\n\n  static v_uint32 bitReverse(v_uint32 poly);\n  \n  /**\n   *  Generates v_uint32 table[256] for polynomial\n   */\n  static p_uint32 generateTable(v_uint32 poly);\n\n  /**\n   * Calculate CRC32 value for buffer of defined size\n   * @param buffer\n   * @param size\n   * @param crc\n   * @param initValue\n   * @param xorOut\n   * @param table\n   * @return - CRC32 value (v_uint32)\n   */\n  static v_uint32 calc(const void *buffer, v_buff_size size, v_uint32 crc = 0, v_uint32 initValue = 0xFFFFFFFF, v_uint32 xorOut = 0xFFFFFFFF, p_uint32 table = TABLE_04C11DB7);\n  \n};\n    \n}}\n\n#endif /* oatpp_algorithm_CRC_hpp */\n\n// Path: src/oatpp/algorithm/CRC.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"CRC.hpp\"\n\nnamespace oatpp { namespace algorithm {\n  \nconst p_uint32 CRC32::TABLE_04C11DB7 = generateTable(0x04C11DB7);\n  \nv_uint32 CRC32::bitReverse(v_uint32 poly) {\n  v_uint32 result = 0;\n  for(v_int32 i = 0; i < 32; i ++) {\n    if((poly & (1U << i)) > 0) {\n      result |= (1U << (31 - i));\n    }\n  }\n  return result;\n}\n  \np_uint32 CRC32::generateTable(v_uint32 poly) {\n  \n  p_uint32 result = new v_uint32[256];\n  v_uint32 polyReverse = bitReverse(poly);\n  v_uint32 value;\n  \n  for(v_uint32 i = 0; i < 256; i++) {\n    value = i;\n    for (v_int32 bit = 0; bit < 8; bit++) {\n      if (value & 1) {\n        value = (value >> 1) ^ polyReverse;\n      } else {\n        value = (value >> 1);\n      }\n    }\n    \n    result[i] = value;\n    \n  }\n  \n  return result;\n  \n}\n  \nv_uint32 CRC32::calc(const void *buffer, v_buff_size size, v_uint32 crc, v_uint32 initValue, v_uint32 xorOut, p_uint32 table) {\n  \n  auto data = reinterpret_cast<const unsigned char*>(buffer);\n  crc = crc ^ initValue;\n  \n  for(v_buff_size i = 0; i < size; i++) {\n    crc = table[(crc & 0xFF) ^ data[i]] ^ (crc >> 8);\n  }\n  \n  return crc ^ xorOut;\n}\n  \n}}\n\n// Path: src/oatpp/web/mime/multipart/Part.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_mime_multipart_Part_hpp\n#define oatpp_web_mime_multipart_Part_hpp\n\n#include \"oatpp/core/data/share/LazyStringMap.hpp\"\n#include \"oatpp/core/data/resource/Resource.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\n/**\n * One part of the multipart.\n */\nclass Part {\npublic:\n  /**\n   * Typedef for headers map. Headers map key is case-insensitive.\n   * For more info see &id:oatpp::data::share::LazyStringMap;.\n   */\n  typedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\nprivate:\n  oatpp::String m_name;\n  oatpp::String m_filename;\n  Headers m_headers;\n  std::shared_ptr<data::resource::Resource> m_payload;\nprivate:\n  const char* m_tagName;\n  std::shared_ptr<oatpp::base::Countable> m_tagObject;\npublic:\n\n  /**\n   * Default constructor.\n   */\n  Part() = default;\n\n  /**\n   * Constructor.\n   * @param headers - headers of the part.\n   * @param payload - part payload.\n   */\n  Part(const Headers& headers, const std::shared_ptr<data::resource::Resource>& payload = nullptr);\n\n  /**\n   * Set payload.\n   * @param payload\n   */\n  void setPayload(const std::shared_ptr<data::resource::Resource>& payload);\n\n  /**\n   * Get payload.\n   * @return\n   */\n  std::shared_ptr<data::resource::Resource> getPayload();\n\n  /**\n   * Get name of the part.\n   * @return - name of the part.\n   */\n  oatpp::String getName() const;\n\n  /**\n   * Get filename of the part (if applicable).\n   * @return - filename.\n   */\n  oatpp::String getFilename() const;\n\n  /**\n   * Get request's headers map.\n   * @return Headers map\n   */\n  const Headers& getHeaders() const;\n\n  /**\n   * Get header value\n   * @param headerName\n   * @return header value\n   */\n  oatpp::String getHeader(const oatpp::data::share::StringKeyLabelCI& headerName) const;\n\n  /**\n   * Add http header.\n   * @param key - &id:oatpp::data::share::StringKeyLabelCI;.\n   * @param value - &id:oatpp::data::share::StringKeyLabel;.\n   */\n  void putHeader(const oatpp::data::share::StringKeyLabelCI& key, const oatpp::data::share::StringKeyLabel& value);\n\n  /**\n   * Add http header if not already exists.\n   * @param key - &id:oatpp::data::share::StringKeyLabelCI;.\n   * @param value - &id:oatpp::data::share::StringKeyLabel;.\n   * @return - `true` if header was added.\n   */\n  bool putHeaderIfNotExists(const oatpp::data::share::StringKeyLabelCI& key, const oatpp::data::share::StringKeyLabel& value);\n\n  /**\n   * Tag-object - object used to associate some data with the Part. <br>\n   * Ex.: used by &id:oatpp::web::mime::multipart::InMemoryPartReader;. to\n   * associate intermediate buffer with the part.\n   * @param tagName\n   * @param tagObject\n   */\n  void setTag(const char* tagName, const std::shared_ptr<oatpp::base::Countable>& tagObject);\n\n  /**\n   * Get tag name.\n   * @return\n   */\n  const char* getTagName();\n\n  /**\n   * Get tag object.\n   * @return\n   */\n  std::shared_ptr<oatpp::base::Countable> getTagObject();\n\n  /**\n   * Clear the tag.\n   */\n  void clearTag();\n\n};\n\n}}}}\n\n#endif // oatpp_web_mime_multipart_Part_hpp\n\n// Path: src/oatpp/web/mime/multipart/Part.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Part.hpp\"\n\n#include \"oatpp/web/protocol/http/Http.hpp\"\n#include \"oatpp/core/parser/Caret.hpp\"\n\n#include <cstring>\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\nPart::Part(const Headers &headers, const std::shared_ptr<data::resource::Resource>& payload)\n  : m_headers(headers)\n  , m_payload(payload)\n{\n\n  auto contentDisposition = m_headers.getAsMemoryLabel<oatpp::data::share::StringKeyLabel>(\"Content-Disposition\");\n  if(contentDisposition) {\n\n    oatpp::web::protocol::http::HeaderValueData valueData;\n    oatpp::web::protocol::http::Parser::parseHeaderValueData(valueData, contentDisposition, ';');\n\n    m_name = valueData.getTitleParamValue(\"name\");\n    m_filename = valueData.getTitleParamValue(\"filename\");\n\n  }\n\n}\n\nvoid Part::setPayload(const std::shared_ptr<data::resource::Resource>& payload) {\n  m_payload = payload;\n}\n\nstd::shared_ptr<data::resource::Resource> Part::getPayload() {\n  return m_payload;\n}\n\noatpp::String Part::getName() const {\n  return m_name;\n}\n\n\noatpp::String Part::getFilename() const {\n  return m_filename;\n}\n\n\nconst Part::Headers& Part::getHeaders() const {\n  return m_headers;\n}\n\noatpp::String Part::getHeader(const oatpp::data::share::StringKeyLabelCI& headerName) const {\n  return m_headers.get(headerName);\n}\n\nvoid Part::putHeader(const oatpp::data::share::StringKeyLabelCI& key, const oatpp::data::share::StringKeyLabel& value) {\n  m_headers.put(key, value);\n}\n\nbool Part::putHeaderIfNotExists(const oatpp::data::share::StringKeyLabelCI& key, const oatpp::data::share::StringKeyLabel& value) {\n  return m_headers.putIfNotExists(key, value);\n}\n\nvoid Part::setTag(const char* tagName, const std::shared_ptr<oatpp::base::Countable>& tagObject) {\n  m_tagName = tagName;\n  m_tagObject = tagObject;\n}\n\nconst char* Part::getTagName() {\n  return m_tagName;\n}\n\nstd::shared_ptr<oatpp::base::Countable> Part::getTagObject() {\n  return m_tagObject;\n}\n\nvoid Part::clearTag() {\n  m_tagName = nullptr;\n  m_tagObject.reset();\n}\n\n}}}}\n\n// Path: src/oatpp/web/mime/multipart/StatefulParser.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_mime_multipart_StatefulParser_hpp\n#define oatpp_web_mime_multipart_StatefulParser_hpp\n\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n#include \"oatpp/core/data/share/LazyStringMap.hpp\"\n#include \"oatpp/core/Types.hpp\"\n\n#include <unordered_map>\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\n/**\n * Stateful parser of multipart-data stream.\n * Parser designed to work with stream-like data in order to store minimum data in the memory.\n */\nclass StatefulParser {\nprivate:\n  static constexpr v_int32 STATE_BOUNDARY = 0;\n  static constexpr v_int32 STATE_AFTER_BOUNDARY = 1;\n  static constexpr v_int32 STATE_HEADERS = 2;\n  static constexpr v_int32 STATE_DATA = 3;\n  static constexpr v_int32 STATE_DONE = 4;\nprivate:\n  static constexpr v_uint32 HEADERS_SECTION_END = ('\\r' << 24) | ('\\n' << 16) | ('\\r' << 8) | ('\\n');\nprivate:\n  /**\n   * Typedef for headers map. Headers map key is case-insensitive.\n   * For more info see &id:oatpp::data::share::LazyStringMap;.\n   */\n  typedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\npublic:\n\n  /**\n   * Listener for parsed items.\n   */\n  class Listener {\n  public:\n    /**\n     * Typedef for headers map. Headers map key is case-insensitive.\n     * For more info see &id:oatpp::data::share::LazyStringMap;.\n     */\n    typedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\n  public:\n\n    /**\n     * Default virtual Destructor.\n     */\n    virtual ~Listener() = default;\n\n    /**\n     * Called on new part found in the stream.\n     * Always called before `onPartData` events.\n     * @param partHeaders - complete set of part headers.\n     */\n    virtual void onPartHeaders(const Headers& partHeaders) = 0;\n\n    /**\n     * Called on each new chunk of bytes parsed from the part body.\n     * When all data of message is read, readMessage is called again with size == 0 to\n     * indicate end of the part.\n     * @param data - pointer to data.\n     * @param size - size of the data in bytes.\n     */\n    virtual void onPartData(const char* data, v_buff_size size) = 0;\n\n  };\n\npublic:\n\n  /**\n   * Async Listener for parsed items.\n   */\n  class AsyncListener {\n  public:\n    /**\n     * Typedef for headers map. Headers map key is case-insensitive.\n     * For more info see &id:oatpp::data::share::LazyStringMap;.\n     */\n    typedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\n  public:\n\n    /**\n     * Default virtual Destructor.\n     */\n    virtual ~AsyncListener() = default;\n\n    /**\n     * Called on new part found in the stream.\n     * Always called before `onPartData` events.\n     * @param partHeaders - complete set of part headers.\n     */\n    virtual async::CoroutineStarter onPartHeadersAsync(const Headers& partHeaders) = 0;\n\n    /**\n     * Called on each new chunk of bytes parsed from the part body.\n     * When all data of message is read, readMessage is called again with size == 0 to\n     * indicate end of the part.\n     * @param data - pointer to data.\n     * @param size - size of the data in bytes.\n     */\n    virtual async::CoroutineStarter onPartDataAsync(const char* data, v_buff_size size) = 0;\n\n  };\n\nprivate:\n\n  class ListenerCall {\n  public:\n\n    static constexpr v_int32 CALL_NONE = 0;\n    static constexpr v_int32 CALL_ON_HEADERS = 1;\n    static constexpr v_int32 CALL_ON_DATA = 2;\n\n  public:\n\n    \nListenerCall()\n      : callType(CALL_NONE)\n      , data(nullptr)\n      , size(0)\n    {}\n\n    v_int32 callType;\n    const char* data;\n    v_io_size size;\n\n    void setOnHeadersCall();\n    void setOnDataCall(const char* pData, v_buff_size pSize);\n\n    void call(StatefulParser* parser);\n    async::CoroutineStarter callAsync(StatefulParser* parser);\n\n    explicit operator bool() const;\n\n  };\n\nprivate:\n\n  v_int32 m_state;\n  v_int64 m_currPartIndex;\n  v_buff_size m_currBoundaryCharIndex;\n  bool m_checkForBoundary;\n  bool m_finishingBoundary;\n  bool m_readingBody;\n\n  v_uint32 m_headerSectionEndAccumulator;\n\n  oatpp::String m_firstBoundarySample;\n  oatpp::String m_nextBoundarySample;\n\n  /*\n   * Headers of the part are stored in the buffer and are parsed as one chunk.\n   */\n  data::stream::BufferOutputStream m_headersBuffer;\n\n  /*\n   * Max length of all headers per one part.\n   * Default value = 4096 bytes.\n   */\n  v_buff_size m_maxPartHeadersSize;\n\n  std::shared_ptr<Listener> m_listener;\n  std::shared_ptr<AsyncListener> m_asyncListener;\n\nprivate:\n\n  void parseHeaders(Headers& headers);\n\nprivate:\n\n  ListenerCall parseNext_Boundary(data::buffer::InlineWriteData& inlineData);\n  void         parseNext_AfterBoundary(data::buffer::InlineWriteData& inlineData);\n  ListenerCall parseNext_Headers(data::buffer::InlineWriteData& inlineData);\n  ListenerCall parseNext_Data(data::buffer::InlineWriteData& inlineData);\n\npublic:\n\n  /**\n   * Constructor.\n   * @param boundary - value of multipart boundary.\n   * @param listener - &l:StatefulParser::Listener;.\n   * @param asyncListener - &l:StatefulParser::AsyncListener;.\n   */\n  StatefulParser(const oatpp::String& boundary,\n                 const std::shared_ptr<Listener>& listener,\n                 const std::shared_ptr<AsyncListener>& asyncListener);\n\n  /**\n   * Parse next chunk of bytes.\n   * @param inlineData - inline data.\n   * @param action - Async Action in case Async Listener was provided in constructor.\n   */\n  void parseNext(data::buffer::InlineWriteData& inlineData, async::Action& action);\n\n  /**\n   * Check if parser done parsing data.\n   * @return - `true` or `false`.\n   */\n  bool finished();\n\n};\n\n}}}}\n\n#endif // oatpp_web_mime_multipart_StatefulParser_hpp\n\n// Path: src/oatpp/web/mime/multipart/StatefulParser.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"StatefulParser.hpp\"\n\n#include \"oatpp/web/protocol/http/Http.hpp\"\n\n#include \"oatpp/core/parser/Caret.hpp\"\n#include \"oatpp/core/parser/ParsingError.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// StatefulParser::ListenerCall\n\nvoid StatefulParser::ListenerCall::setOnHeadersCall() {\n  callType = CALL_ON_HEADERS;\n  data = nullptr;\n  size = 0;\n}\n\nvoid StatefulParser::ListenerCall::setOnDataCall(const char* pData, v_buff_size pSize) {\n  callType = CALL_ON_DATA;\n  data = pData;\n  size = pSize;\n}\n\nvoid StatefulParser::ListenerCall::call(StatefulParser* parser) {\n\n  if(parser->m_listener) {\n\n    switch(callType) {\n\n      case CALL_ON_HEADERS:\n        {\n          Headers headers;\n          parser->parseHeaders(headers);\n          parser->m_listener->onPartHeaders(headers);\n        }\n        break;\n\n      case CALL_ON_DATA:\n        parser->m_listener->onPartData(data, size);\n        break;\n\n      default:\n        break;\n\n    }\n\n  }\n\n}\n\nasync::CoroutineStarter StatefulParser::ListenerCall::callAsync(StatefulParser* parser) {\n\n  if(parser->m_asyncListener) {\n\n    switch(callType) {\n\n      case CALL_ON_HEADERS: {\n        Headers headers;\n        parser->parseHeaders(headers);\n        return parser->m_asyncListener->onPartHeadersAsync(headers);\n      }\n\n      case CALL_ON_DATA:\n        return parser->m_asyncListener->onPartDataAsync(data, size);\n\n      default:\n        break;\n\n    }\n\n  }\n\n  return nullptr;\n\n}\n\nStatefulParser::ListenerCall::operator bool() const {\n  return callType != CALL_NONE;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// StatefulParser\n\nStatefulParser::StatefulParser(const oatpp::String& boundary,\n                               const std::shared_ptr<Listener>& listener,\n                               const std::shared_ptr<AsyncListener>& asyncListener)\n  : m_state(STATE_BOUNDARY)\n  , m_currPartIndex(0)\n  , m_currBoundaryCharIndex(0)\n  , m_checkForBoundary(true)\n  , m_finishingBoundary(false)\n  , m_readingBody(false)\n  , m_headerSectionEndAccumulator(0)\n  , m_firstBoundarySample(\"--\" + boundary)\n  , m_nextBoundarySample(\"\\r\\n--\" + boundary)\n  , m_maxPartHeadersSize(4092)\n  , m_listener(listener)\n  , m_asyncListener(asyncListener)\n{}\n\nvoid StatefulParser::parseHeaders(Headers& headers) {\n\n  m_currPartIndex ++;\n\n  auto headersText = m_headersBuffer.toString();\n  m_headersBuffer.setCurrentPosition(0);\n\n  protocol::http::Status status;\n  parser::Caret caret(headersText);\n\n  protocol::http::Parser::parseHeaders(headers, headersText.getPtr(), caret, status);\n\n}\n\nStatefulParser::ListenerCall StatefulParser::parseNext_Boundary(data::buffer::InlineWriteData& inlineData) {\n\n  ListenerCall result;\n  auto data = inlineData.currBufferPtr;\n  auto size = inlineData.bytesLeft;\n\n  auto sampleData = m_nextBoundarySample->data();\n  v_io_size sampleSize = static_cast<v_io_size>(m_nextBoundarySample->size());\n\n  if (m_currPartIndex == 0) {\n    sampleData = m_firstBoundarySample->data();\n    sampleSize = static_cast<v_io_size>(m_firstBoundarySample->size());\n  } else {\n    sampleData = m_nextBoundarySample->data();\n    sampleSize = static_cast<v_io_size>(m_nextBoundarySample->size());\n  }\n\n  v_io_size checkSize = sampleSize - m_currBoundaryCharIndex;\n  if(checkSize > size) {\n    checkSize = size;\n  }\n\n  parser::Caret caret(reinterpret_cast<const char*>(data), size);\n\n  if(caret.isAtText(&sampleData[m_currBoundaryCharIndex], checkSize, true)) {\n\n    m_currBoundaryCharIndex += caret.getPosition();\n\n    if(m_currBoundaryCharIndex == sampleSize) {\n      m_state = STATE_AFTER_BOUNDARY;\n      m_currBoundaryCharIndex = 0;\n      m_readingBody = false;\n      if(m_currPartIndex > 0) {\n        result.setOnDataCall(nullptr, 0);\n      }\n    }\n\n    inlineData.inc(caret.getPosition());\n    return result;\n\n  } else if(m_readingBody) {\n\n    if(m_currBoundaryCharIndex > 0) {\n      result.setOnDataCall(sampleData, m_currBoundaryCharIndex);\n    } else {\n      m_checkForBoundary = false;\n    }\n\n    m_state = STATE_DATA;\n    m_currBoundaryCharIndex = 0;\n\n    return result;\n\n  }\n\n  throw std::runtime_error(\"[oatpp::web::mime::multipart::StatefulParser::parseNext_Boundary()]: Error. Invalid state.\");\n\n}\n\nvoid StatefulParser::parseNext_AfterBoundary(data::buffer::InlineWriteData& inlineData) {\n\n  p_char8 data = reinterpret_cast<p_char8>(const_cast<void*>(inlineData.currBufferPtr));\n  auto size = inlineData.bytesLeft;\n\n  if(m_currBoundaryCharIndex == 0) {\n\n    if(data[0] == '-') {\n      m_finishingBoundary = true;\n    } else if(data[0] != '\\r') {\n      throw std::runtime_error(\"[oatpp::web::mime::multipart::StatefulParser::parseNext_AfterBoundary()]: Error. Invalid char.\");\n    }\n\n  }\n\n  if(size > 1 || m_currBoundaryCharIndex == 1) {\n\n    if (m_finishingBoundary && data[1 - m_currBoundaryCharIndex] == '-') {\n      auto result = 2 - m_currBoundaryCharIndex;\n      m_state = STATE_DONE;\n      m_currBoundaryCharIndex = 0;\n      inlineData.inc(result);\n      return;\n    } else if (!m_finishingBoundary && data[1 - m_currBoundaryCharIndex] == '\\n') {\n      auto result = 2 - m_currBoundaryCharIndex;\n      m_state = STATE_HEADERS;\n      m_currBoundaryCharIndex = 0;\n      m_headerSectionEndAccumulator = 0;\n      inlineData.inc(result);\n      return;\n    } else {\n      throw std::runtime_error(\"[oatpp::web::mime::multipart::StatefulParser::parseNext_AfterBoundary()]: Error. Invalid trailing char.\");\n    }\n\n  }\n\n  m_currBoundaryCharIndex = 1;\n  inlineData.inc(1);\n  return;\n\n}\n\nStatefulParser::ListenerCall StatefulParser::parseNext_Headers(data::buffer::InlineWriteData& inlineData) {\n\n  ListenerCall result;\n\n  p_char8 data = reinterpret_cast<p_char8>(const_cast<void*>(inlineData.currBufferPtr));\n  auto size = inlineData.bytesLeft;\n\n  for(v_buff_size i = 0; i < size; i ++) {\n\n    m_headerSectionEndAccumulator <<= 8;\n    m_headerSectionEndAccumulator |= data[i];\n\n    if(m_headerSectionEndAccumulator == HEADERS_SECTION_END) {\n\n      if(m_headersBuffer.getCurrentPosition() + i > m_maxPartHeadersSize) {\n        throw std::runtime_error(\"[oatpp::web::mime::multipart::StatefulParser::parseNext_Headers()]: Error. Too large heades.\");\n      }\n\n      m_headersBuffer.writeSimple(data, i);\n\n      result.setOnHeadersCall();\n\n      m_state = STATE_DATA;\n      m_checkForBoundary = true;\n\n      inlineData.inc(i + 1);\n      return result;\n\n    }\n\n  }\n\n  if(m_headersBuffer.getCurrentPosition() + size > m_maxPartHeadersSize) {\n    throw std::runtime_error(\"[oatpp::web::mime::multipart::StatefulParser::parseNext_Headers()]: Error. Headers section is too large.\");\n  }\n\n  m_headersBuffer.writeSimple(data, size);\n\n  inlineData.inc(size);\n\n  return result;\n\n}\n\nStatefulParser::ListenerCall StatefulParser::parseNext_Data(data::buffer::InlineWriteData& inlineData) {\n\n  ListenerCall result;\n\n  const char* data = reinterpret_cast<const char*>(inlineData.currBufferPtr);\n  auto size = inlineData.bytesLeft;\n\n  parser::Caret caret(data, size);\n\n  bool rFound = caret.findChar('\\r');\n  if(rFound && !m_checkForBoundary) {\n    caret.inc();\n    rFound = caret.findChar('\\r');\n  }\n\n  m_checkForBoundary = true;\n\n  if(rFound) {\n    if(caret.getPosition() > 0) {\n      result.setOnDataCall(data, caret.getPosition());\n    }\n    m_state = STATE_BOUNDARY;\n    m_readingBody = true;\n    inlineData.inc(caret.getPosition());\n  } else {\n    result.setOnDataCall(data, size);\n    inlineData.inc(size);\n  }\n\n  return result;\n\n}\n\nvoid StatefulParser::parseNext(data::buffer::InlineWriteData& inlineData, async::Action& action) {\n\n  while(inlineData.bytesLeft > 0) {\n\n    ListenerCall listenerCall;\n\n    switch (m_state) {\n      case STATE_BOUNDARY:\n        listenerCall = parseNext_Boundary(inlineData);\n        break;\n      case STATE_AFTER_BOUNDARY:\n        parseNext_AfterBoundary(inlineData);\n        break;\n      case STATE_HEADERS:\n        listenerCall = parseNext_Headers(inlineData);\n        break;\n      case STATE_DATA:\n        listenerCall = parseNext_Data(inlineData);\n        break;\n      case STATE_DONE:\n        return;\n      default:\n        throw std::runtime_error(\"[oatpp::web::mime::multipart::StatefulParser::parseNext()]: Error. Invalid state.\");\n    }\n\n    if(listenerCall) {\n      if(m_asyncListener) {\n        action = listenerCall.callAsync(this).next(async::Action::createActionByType(async::Action::TYPE_REPEAT));\n        break;\n      }else {\n        listenerCall.call(this);\n      }\n    }\n\n  }\n\n}\n\nbool StatefulParser::finished() {\n  return m_state == STATE_DONE;\n}\n\n}}}}\n\n// Path: src/oatpp/web/mime/multipart/Multipart.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_mime_multipart_Multipart_hpp\n#define oatpp_web_mime_multipart_Multipart_hpp\n\n#include \"Part.hpp\"\n#include <list>\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\n/**\n * Typedef for headers map. Headers map key is case-insensitive.\n * For more info see &id:oatpp::data::share::LazyStringMap;.\n */\ntypedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\n\n/**\n * Abstract Multipart.\n */\nclass Multipart {\nprivate:\n  oatpp::String m_boundary;\npublic:\n\n  /**\n   * Constructor.\n   * @param boundary - multipart boundary value.\n   */\n  Multipart(const oatpp::String& boundary);\n\n  /**\n   * Default virtual Destructor.\n   */\n  virtual ~Multipart() = default;\n\n  /**\n   * Get multipart boundary value.\n   * @return - multipart boundary value.\n   */\n  oatpp::String getBoundary();\n\n  /**\n   * Read part-by-part from Multipart.\n   * @return\n   */\n  virtual std::shared_ptr<Part> readNextPart(async::Action& action) = 0;\n\n  /**\n   * Write part-by-part to Multipart.\n   * @param part\n   */\n  virtual void writeNextPart(const std::shared_ptr<Part>& part, async::Action& action) = 0;\n\n  /**\n   * Read part-by-part from Multipart. <br>\n   * Call writeNextPart(...) and throw if `action.isNone() == false`.\n   * @return\n   */\n  std::shared_ptr<Part> readNextPartSimple();\n\n  /**\n   * Write part-by-part to Multipart.\n   * Call writeNextPartSimple(...) and throw if `action.isNone() == false`.\n   * @param part\n   */\n  void writeNextPartSimple(const std::shared_ptr<Part>& part);\n\npublic:\n\n  /**\n   * Generate random boundary for Multipart object. Base64 encoded.\n   * @param boundarySize - size in bytes of random vector.\n   * @return - &id:oatpp::String;.\n   */\n  static oatpp::String generateRandomBoundary(v_int32 boundarySize = 15);\n\n  /**\n   * Parse boundary value from headers\n   * @param headers\n   * @return\n   */\n  static oatpp::String parseBoundaryFromHeaders(const Headers& requestHeaders);\n\n};\n\n\n\n}}}}\n\n\n#endif // oatpp_web_mime_multipart_Multipart_hpp\n\n// Path: src/oatpp/web/mime/multipart/PartReader.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_mime_multipart_PartReader_hpp\n#define oatpp_web_mime_multipart_PartReader_hpp\n\n#include \"./Multipart.hpp\"\n#include \"oatpp/core/data/stream/Stream.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\n/**\n * Abstract read handler of multipart parts.\n */\nclass PartReader {\npublic:\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~PartReader() = default;\n\n  /**\n   * Called when new part headers are parsed and part object is created.\n   * @param part\n   */\n  virtual void onNewPart(const std::shared_ptr<Part>& part) = 0;\n\n  /**\n   * Called on each new chunk of data is parsed for the multipart-part. <br>\n   * When all data is read, called again with `data == nullptr && size == 0` to indicate end of the part.\n   * @param part\n   * @param data - pointer to buffer containing chunk data.\n   * @param size - size of the buffer.\n   */\n  virtual void onPartData(const std::shared_ptr<Part>& part, const char* data, oatpp::v_io_size size) = 0;\n\n};\n\n/**\n * Abstract Async read handler of multipart parts.\n */\nclass AsyncPartReader {\npublic:\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~AsyncPartReader() = default;\n\n  /**\n   * Called when new part headers are parsed and part object is created.\n   * @param part\n   * @return - &id:oatpp::async::CoroutineStarter;.\n   */\n  virtual async::CoroutineStarter onNewPartAsync(const std::shared_ptr<Part>& part) = 0;\n\n  /**\n   * Called on each new chunk of data is parsed for the multipart-part. <br>\n   * When all data is read, called again with `data == nullptr && size == 0` to indicate end of the part.\n   * @param part\n   * @param data - pointer to buffer containing chunk data.\n   * @param size - size of the buffer.\n   * @return - &id:oatpp::async::CoroutineStarter;.\n   */\n  virtual async::CoroutineStarter onPartDataAsync(const std::shared_ptr<Part>& part, const char* data, oatpp::v_io_size size) = 0;\n\n};\n\n/**\n * Resource provider for `StreamPartReader`.\n */\nclass PartReaderResourceProvider {\npublic:\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~PartReaderResourceProvider() = default;\n\n  /**\n   * Get data resource to write (save) part data in.\n   * @param part\n   * @return\n   */\n  virtual std::shared_ptr<data::resource::Resource> getResource(const std::shared_ptr<Part>& part) = 0;\n\n  /**\n   * Get data resource to write (save) part data in.\n   * @param part\n   * @param resource - put here pointer to obtained resource.\n   * @return\n   */\n  virtual async::CoroutineStarter getResourceAsync(const std::shared_ptr<Part>& part,\n                                                   std::shared_ptr<data::resource::Resource>& resource) = 0;\n\n};\n\n/**\n * Part reader used in order to stream part data.\n */\nclass StreamPartReader : public PartReader {\nprivate:\n  static const char* const TAG_NAME;\nprivate:\n\n  class TagObject : public oatpp::base::Countable {\n  public:\n    v_io_size size = 0;\n    std::shared_ptr<data::resource::Resource> resource;\n    std::shared_ptr<data::stream::OutputStream> outputStream;\n  };\n\nprivate:\n  std::shared_ptr<PartReaderResourceProvider> m_resourceProvider;\n  v_io_size m_maxDataSize;\npublic:\n\n  /**\n   * Constructor.\n   * @param resourceProvider\n   * @param maxDataSize - use `-1` for no limit.\n   */\n  StreamPartReader(const std::shared_ptr<PartReaderResourceProvider>& resourceProvider, v_io_size maxDataSize = -1);\n\n  /**\n   * Called when new part headers are parsed and part object is created.\n   * @param part\n   */\n  void onNewPart(const std::shared_ptr<Part>& part) override;\n\n  /**\n   * Called on each new chunk of data is parsed for the multipart-part. <br>\n   * When all data is read, called again with `data == nullptr && size == 0` to indicate end of the part.\n   * @param part\n   * @param data - pointer to buffer containing chunk data.\n   * @param size - size of the buffer.\n   */\n  void onPartData(const std::shared_ptr<Part>& part, const char* data, oatpp::v_io_size size) override;\n\n};\n\n/**\n * Async part reader used in order to stream part data in Asynchronous manner.\n */\nclass AsyncStreamPartReader : public AsyncPartReader {\nprivate:\n  static const char* const TAG_NAME;\nprivate:\n\n  class TagObject : public oatpp::base::Countable {\n  public:\n    v_io_size size = 0;\n    std::shared_ptr<data::resource::Resource> resource;\n    std::shared_ptr<data::stream::OutputStream> outputStream;\n  };\n\nprivate:\n  std::shared_ptr<PartReaderResourceProvider> m_resourceProvider;\n  v_io_size m_maxDataSize;\npublic:\n\n  /**\n   * Constructor.\n   * @param resourceProvider\n   * @param maxDataSize - use `-1` for no limit.\n   */\n  AsyncStreamPartReader(const std::shared_ptr<PartReaderResourceProvider>& resourceProvider, v_io_size maxDataSize = -1);\n\n  /**\n   * Called when new part headers are parsed and part object is created.\n   * @param part\n   * @return - &id:oatpp::async::CoroutineStarter;.\n   */\n  async::CoroutineStarter onNewPartAsync(const std::shared_ptr<Part>& part) override;\n\n  /**\n   * Called on each new chunk of data is parsed for the multipart-part. <br>\n   * When all data is read, called again with `data == nullptr && size == 0` to indicate end of the part.\n   * @param part\n   * @param data - pointer to buffer containing chunk data.\n   * @param size - size of the buffer.\n   * @return - &id:oatpp::async::CoroutineStarter;.\n   */\n  async::CoroutineStarter onPartDataAsync(const std::shared_ptr<Part>& part, const char* data, oatpp::v_io_size size) override;\n\n};\n\n}}}}\n\n#endif //oatpp_web_mime_multipart_PartReader_hpp\n\n// Path: src/oatpp/web/mime/multipart/PartReader.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n...\n", "needle_token_start": 8672, "needle_token_end": 8706, "code_context_ntokens": 15778, "output": ["```cpp\nclass StatefulParser {\nprivate:\n  static constexpr v_int32 STATE_BOUNDARY = 0;\n  static constexpr v_int32 STATE_AFTER_BOUNDARY = 1;\n  static constexpr v_int32 STATE_HEADERS = 2;\n  static constexpr v_int32 STATE_DATA = 3;\n  static constexpr v_int32 STATE_DONE = 4;\nprivate:\n  static constexpr v_uint32 HEADERS_SECTION_END = ('\\r' << 24) | ('\\n' << 16) | ('\\r' << 8) | ('\\n');\nprivate:\n  /**\n   * Typedef for headers map. Headers map key is case-insensitive.\n   * For more info see &id:oatpp::data::share::LazyStringMap;.\n   */\n  typedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\npublic:\n\n  /**\n   * Listener for parsed items.\n   */\n  class Listener {\n  public:\n    /**\n     * Typedef for headers map. Headers map key is case-insensitive.\n     * For more info see &id:oatpp::data::share::LazyStringMap;.\n     */\n    typedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\n  public:\n\n    /**\n     * Default virtual Destructor.\n     */\n    virtual ~Listener() = default;\n\n    /**\n     * Called on new part found in the stream.\n     * Always called before `onPartData` events.\n     * @param partHeaders - complete set of part headers.\n     */\n    virtual void onPartHeaders(const Headers& partHeaders) = 0;\n\n    /**\n     * Called on each new chunk of bytes parsed from the part body.\n     * When all data of message is read, readMessage is called again with size == 0 to\n     * indicate end of the part.\n     * @param data - pointer to data.\n     * @param size - size of the data in bytes.\n     */\n    virtual void onPartData(const char* data, v_buff_size size) = 0;\n\n  };\n\npublic:\n\n  /**\n   * Async Listener for parsed items.\n   */\n  class AsyncListener {\n  public:\n    /**\n     * Typedef for headers map. Headers map key is case-insensitive.\n     * For more info see &id:oatpp::data::share::LazyStringMap;.\n     */\n    typedef oatpp::data::share::LazyStringMultimap<oatpp::data::share::StringKeyLabelCI> Headers;\n  public:\n\n    /**\n     * Default virtual Destructor.\n     */\n    virtual ~AsyncListener() = default;\n\n    /**\n     * Called on new part found in the stream.\n     * Always called before `onPartData` events.\n     * @param partHeaders - complete set of part headers.\n     */\n    virtual async::CoroutineStarter onPartHeadersAsync(const Headers& partHeaders) = 0;\n\n    /**\n     * Called on each new chunk of bytes parsed from the part body.\n     * When all data of message is read, readMessage is called again with size == 0 to\n     * indicate end of the part.\n     * @param data - pointer to data.\n     * @param size - size of the data in bytes.\n     */\n    virtual async::CoroutineStarter onPartDataAsync(const char* data, v_buff_size size) = 0;\n\n  };\n\nprivate:\n\n  class ListenerCall {\n  public:\n\n    static constexpr v_int32 CALL_NONE = 0;\n    static constexpr v_int32 CALL_ON_HEADERS = 1;\n    static constexpr v_int32 CALL_ON_DATA = 2;\n\n  public:\n\n    \nListenerCall()\n      : callType(CALL_NONE)\n      , data(nullptr)\n      , size(0)\n    {}\n\n    v_int32 callType;\n    const char* data;\n    v_io_size size;\n\n    void setOnHeadersCall();\n    void setOnDataCall(const char* pData, v_buff_size pSize);\n\n    void call(StatefulParser* parser);\n    async::CoroutineStarter callAsync(StatefulParser* parser);\n\n    explicit operator bool() const;\n\n  };\n\nprivate:\n\n  v_int32 m_state;\n  v_int64 m_currPartIndex;\n  v_buff_size m_currBoundaryCharIndex;\n  bool m_checkForBoundary;\n  bool m_finishingBoundary;\n  bool m_readingBody;\n\n  v_uint32 m_headerSectionEndAccumulator;\n\n  oatpp::String m_firstBoundarySample;\n  oatpp::String m_nextBoundarySample;\n\n  /*\n   * Headers of the part are stored in the buffer and are parsed as one chunk.\n   */\n  data::stream::BufferOutputStream m_headersBuffer;\n\n  /*\n   * Max length of all headers per one part.\n   * Default value = 4096 bytes.\n"]}
{"repo": "oatpp/oatpp", "name": "PairListObjectWrapper", "language": "cpp", "path": "src/oatpp/core/data/mapping/type/PairList.hpp", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To provide a wrapper around a list of key-value pairs, enabling object-oriented operations and type safety in a generic programming context.\n2. **Input**: Receives an initializer list of key-value pairs where keys and values are of templated types.\n3. **Output**: Creates an instance that encapsulates a list of key-value pairs, allowing for further manipulation and access in an object-oriented manner.\n4. **Procedure**: Initializes the encapsulated list using the provided initializer list of key-value pairs, leveraging the capabilities of smart pointers for memory management.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/core/data/mapping/type/Primitive.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_type_Primitive_hpp\n#define oatpp_data_type_Primitive_hpp\n\n#include \"./Type.hpp\"\n\n...\n// Path: src/oatpp/core/data/mapping/type/Enum.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_Enum_hpp\n#define oatpp_data_mapping_type_Enum_hpp\n\n#include \"./Any.hpp\"\n#include \"./Primitive.hpp\"\n#include \"oatpp/core/data/share/MemoryLabel.hpp\"\n\n#include <type_traits>\n#include <unordered_map>\n#include <vector>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\n/**\n * Errors of enum interpretation.\n */\nenum class EnumInterpreterError : v_int32 {\n\n  /**\n   * The interpretation was successful.\n   */\n  OK = 0,\n\n  /**\n   * Wrong `Interpreter` is used to interpret the variable. <br>\n   * This may also occur if for example: <br>\n   * `oatpp::Enum<T>` is passed to interpreter of `oatpp::Enum<T>::NotNull`.\n   */\n  TYPE_MISMATCH_ENUM = 1,\n\n  /**\n   * Wrong &id:oatpp::data::mapping::type::Primitive; is passed to interpreter.\n   */\n  TYPE_MISMATCH_ENUM_VALUE = 2,\n\n  /**\n   * Interpreter constraint is violated. <br>\n   * The constraint was set to `NotNull` but interpretation to/from `nullptr` is requested.\n   */\n  CONSTRAINT_NOT_NULL = 3,\n\n  /**\n   * Enum entry not found.\n   */\n  ENTRY_NOT_FOUND = 4,\n\n};\n\nnamespace __class {\n\n  /**\n   * Abstract Enum class.\n   */\n  class AbstractEnum {\n  public:\n    static const ClassId CLASS_ID;\n  public:\n\n    class PolymorphicDispatcher {\n    public:\n\n      PolymorphicDispatcher(bool pNotNull)\n        : notNull(pNotNull)\n      {}\n\n      virtual ~PolymorphicDispatcher() = default;\n\n      const bool notNull;\n\n      virtual type::Void createObject() const = 0;\n\n      virtual type::Void toInterpretation(const type::Void& enumValue, EnumInterpreterError& error) const = 0;\n      virtual type::Void fromInterpretation(const type::Void& interValue, EnumInterpreterError& error) const = 0;\n      virtual type::Type* getInterpretationType() const = 0;\n      virtual std::vector<type::Any> getInterpretedEnum() const = 0;\n\n    };\n\n  };\n\n  template<class T, class Interpreter>\n  class Enum;\n\n}\n\n/**\n * Enum value info.\n * @tparam T - underlying enum type.\n */\ntemplate<typename T>\nstruct EnumValueInfo {\n  /**\n   * Entry value. T - enum type.\n   */\n  const T value;\n\n  /**\n   * Index of the entry.\n   */\n  const v_int32 index;\n\n  /**\n   * Name of the enum entry or name-qualifier, if qualifier was specified. <br>\n   * &id:oatpp::data::share::StringKeyLabel;.\n   */\n  const data::share::StringKeyLabel name;\n  \n  /**\n   * Description of the enum etry. <br>\n   * &id:oatpp::data::share::StringKeyLabel;.\n   */\n  const data::share::StringKeyLabel description;\n};\n\ntemplate<typename T>\nstruct EnumInfo {\npublic:\n  const char* nameQualifier = nullptr;\n  std::unordered_map<data::share::StringKeyLabel, EnumValueInfo<T>> byName;\n  std::unordered_map<v_uint64, EnumValueInfo<T>> byValue;\n  std::vector<EnumValueInfo<T>> byIndex;\n};\n\ntemplate<class T, class Interpreter>\nclass EnumObjectWrapper; // FWD\n\ntemplate<class T>\nclass EnumMeta {\n\n  template<class Type, class Interpreter>\n  friend class __class::Enum;\n\n  template<class Type, class Interpreter>\n  friend class EnumObjectWrapper;\n\npublic:\n  typedef T EnumType;\nprotected:\n  static EnumInfo<T>* getInfo() {\n    static EnumInfo<T> info;\n    return &info;\n  }\n};\n\n/**\n * Enum interpreter `AsString`\n * @tparam T \n * @tparam notnull \n */\ntemplate<class T, bool notnull>\nclass EnumInterpreterAsString {\npublic:\n  typedef String UnderlyingTypeObjectWrapper;\npublic:\n  template <bool N>\n  using InterpreterType = EnumInterpreterAsString<T, N>;\npublic:\n  constexpr static bool notNull = notnull;\npublic:\n  static Void toInterpretation(const Void& enumValue, EnumInterpreterError& error);\n  static Void fromInterpretation(const Void& interValue, EnumInterpreterError& error);\n  static Type* getInterpretationType();\n};\n\n/**\n * Enum interpreter `AsNumber`\n * @tparam T \n * @tparam notnull \n */\ntemplate<class T, bool notnull>\nclass EnumInterpreterAsNumber {\nprivate:\n  typedef typename std::underlying_type<T>::type EnumUnderlyingType;\npublic:\n  typedef typename ObjectWrapperByUnderlyingType<EnumUnderlyingType>::ObjectWrapper UnderlyingTypeObjectWrapper;\npublic:\n  template <bool N>\n  using InterpreterType = EnumInterpreterAsNumber<T, N>;\npublic:\n  constexpr static bool notNull = notnull;\npublic:\n  static Void toInterpretation(const Void& enumValue, EnumInterpreterError& error);\n  static Void fromInterpretation(const Void& interValue, EnumInterpreterError& error);\n  static Type* getInterpretationType();\n};\n\n/**\n * Template class for `oatpp::Enum<T>`.\n * @tparam T - enum type.\n * @tparam EnumInterpreter - enum interpreter.\n */\ntemplate<class T, class EnumInterpreter>\nclass EnumObjectWrapper : public ObjectWrapper<T, __class::Enum<T, EnumInterpreter>>{\n  template<class Type, class Interpreter>\n  friend class EnumObjectWrapper;\npublic:\n  typedef typename std::underlying_type<T>::type UnderlyingEnumType;\n  typedef T Z__EnumType;\n  typedef __class::Enum<T, EnumInterpreter> EnumObjectClass;\n\n  /**\n   * Template parameter - `Interpreter`.\n   */\n  typedef EnumInterpreter Interpreter;\npublic:\n  /**\n   * Enum interpreted `AsString`.\n   */\n  typedef EnumObjectWrapper<T, EnumInterpreterAsString<T, EnumInterpreter::notNull>> AsString;\n\n  /**\n   * Enum interpreted `AsNumber`.\n   */\n  typedef EnumObjectWrapper<T, EnumInterpreterAsNumber<T, EnumInterpreter::notNull>> AsNumber;\n\n  /**\n   * Enum with `NotNull` interpretation constraint.\n   */\n  typedef EnumObjectWrapper<T, typename EnumInterpreter::template InterpreterType<true>> NotNull;\npublic:\n\n  EnumObjectWrapper(const std::shared_ptr<T>& ptr, const type::Type* const valueType)\n    : type::ObjectWrapper<T, EnumObjectClass>(ptr, valueType)\n  {}\n\n  /**\n   * Default constructor.\n   */\n  EnumObjectWrapper() {}\n\n  /**\n   * Nullptr constructor.\n   */\n  EnumObjectWrapper(std::nullptr_t) {}\n\n  /**\n   * Constructor.\n   * @param ptr\n   */\n  EnumObjectWrapper(const std::shared_ptr<T>& ptr)\n    : type::ObjectWrapper<T, EnumObjectClass>(ptr)\n  {}\n\n  /**\n   * Constructor.\n   * @param ptr\n   */\n  EnumObjectWrapper(std::shared_ptr<T>&& ptr)\n    : type::ObjectWrapper<T, EnumObjectClass>(std::forward<std::shared_ptr<T>>(ptr))\n  {}\n\n  /**\n   * Copy-constructor.\n   * @tparam OtherInter\n   * @param other\n   */\n  template<class OtherInter>\n  EnumObjectWrapper(const EnumObjectWrapper<T, OtherInter>& other)\n    : type::ObjectWrapper<T, EnumObjectClass>(other.m_ptr)\n  {}\n\n  /**\n   * Move-constructor.\n   * @tparam OtherInter\n   * @param other\n   */\n  template<class OtherInter>\n  EnumObjectWrapper(EnumObjectWrapper<T, OtherInter>&& other)\n    : type::ObjectWrapper<T, EnumObjectClass>(std::move(other.m_ptr))\n  {}\n\n  inline EnumObjectWrapper& operator = (std::nullptr_t) {\n    this->m_ptr.reset();\n    return *this;\n  }\n\n  template<class OtherInter>\n  inline EnumObjectWrapper& operator = (const EnumObjectWrapper<T, OtherInter>& other) {\n    this->m_ptr = other.m_ptr;\n    return *this;\n  }\n\n  template<class OtherInter>\n  inline EnumObjectWrapper& operator = (EnumObjectWrapper<T, OtherInter>&& other) {\n    this->m_ptr = std::move(other.m_ptr);\n    return *this;\n  }\n\npublic:\n\n  /**\n   * Constructor by value.\n   * @param value\n   */\n  EnumObjectWrapper(T value)\n    : type::ObjectWrapper<T, EnumObjectClass>(std::make_shared<T>(value))\n  {}\n\n  EnumObjectWrapper& operator = (T value) {\n    this->m_ptr = std::make_shared<T>(value);\n    return *this;\n  }\n\n  T operator*() const {\n    return this->m_ptr.operator*();\n  }\n\n  template<typename TP,\n    typename enabled = typename std::enable_if<std::is_same<TP, std::nullptr_t>::value, void>::type\n  >\n  inline bool operator == (TP){\n    return this->m_ptr.get() == nullptr;\n  }\n\n  template<typename TP,\n    typename enabled = typename std::enable_if<std::is_same<TP, std::nullptr_t>::value, void>::type\n  >\n  inline bool operator != (TP){\n    return this->m_ptr.get() != nullptr;\n  }\n\n  template<typename TP,\n    typename enabled = typename std::enable_if<std::is_same<TP, T>::value, void>::type\n  >\n  inline bool operator == (TP value) const {\n    if(!this->m_ptr) return false;\n    return *this->m_ptr == value;\n  }\n\n  template<typename TP,\n    typename enabled = typename std::enable_if<std::is_same<TP, T>::value, void>::type\n  >\n  inline bool operator != (TP value) const {\n    if(!this->m_ptr) return true;\n    return *this->m_ptr != value;\n  }\n\n  template<typename TP,\n    typename enabled = typename std::enable_if<std::is_same<typename TP::Z__EnumType, Z__EnumType>::value, void>::type\n  >\n  inline bool operator == (const TP &other) const {\n    if(this->m_ptr.get() == other.m_ptr.get()) return true;\n    if(!this->m_ptr || !other.m_ptr) return false;\n    return *this->m_ptr == *other.m_ptr;\n  }\n\n  template<typename TP,\n    typename enabled = typename std::enable_if<std::is_same<typename TP::Z__EnumType, Z__EnumType>::value, void>::type\n  >\n  inline bool operator != (const TP &other) const {\n    return !operator == (other);\n  }\n\n  template<typename TP,\n    typename enabled = typename std::enable_if<std::is_same<TP, T>::value, void>::type\n  >\n  inline operator TP() const {\n    return *this->m_ptr;\n  }\n\npublic:\n\n  /**\n   * Get &l:EnumValueInfo <T>; by name.\n   * @param name - name or name-qualifier of the enum entry.\n   * @return - &l:EnumValueInfo <T>;.\n   * @throws - `std::runtime_error` if not found.\n   */\n  static const EnumValueInfo<T>& getEntryByName(const String& name) {\n    auto it = EnumMeta<T>::getInfo()->byName.find(name);\n    if(it != EnumMeta<T>::getInfo()->byName.end()) {\n      return it->second;\n    }\n    throw std::runtime_error(\"[oatpp::data::mapping::type::Enum::getEntryByName()]: Error. Entry not found.\");\n  }\n\n  /**\n   * Get &l:EnumValueInfo <T>; by enum value.\n   * @param value - enum value.\n   * @return - &l:EnumValueInfo <T>;.\n   * @throws - `std::runtime_error` if not found.\n   */\n  static const EnumValueInfo<T>& getEntryByValue(T value) {\n    auto it = EnumMeta<T>::getInfo()->byValue.find(static_cast<v_uint64>(value));\n    if(it != EnumMeta<T>::getInfo()->byValue.end()) {\n      return it->second;\n    }\n    throw std::runtime_error(\"[oatpp::data::mapping::type::Enum::getEntryByValue()]: Error. Entry not found.\");\n  }\n\n  /**\n   * Get &l:EnumValueInfo <T>; by integer value.\n   * @param value - integer value.\n   * @return - &l:EnumValueInfo <T>;.\n   * @throws - `std::runtime_error` if not found.\n   */\n  static const EnumValueInfo<T>& getEntryByUnderlyingValue(UnderlyingEnumType value) {\n    auto it = EnumMeta<T>::getInfo()->byValue.find(static_cast<v_uint64>(value));\n    if(it != EnumMeta<T>::getInfo()->byValue.end()) {\n      return it->second;\n    }\n    throw std::runtime_error(\"[oatpp::data::mapping::type::Enum::getEntryByUnderlyingValue()]: Error. Entry not found.\");\n  }\n\n  /**\n   * Get &l:EnumValueInfo <T>; by index.\n   * @param index - index of the entry in the enum.\n   * @return - &l:EnumValueInfo <T>;.\n   * @throws - `std::runtime_error` if not found.\n   */\n  static const EnumValueInfo<T>& getEntryByIndex(v_int32 index) {\n    if(index >= 0 && index < EnumMeta<T>::getInfo()->byIndex.size()) {\n      return EnumMeta<T>::getInfo()->byIndex[index];\n    }\n    throw std::runtime_error(\"[oatpp::data::mapping::type::Enum::getEntryByIndex()]: Error. Entry not found.\");\n  }\n\n  /**\n   * Get `std::vector` of &l:EnumValueInfo <T>;.\n   * @return - `std::vector` of &l:EnumValueInfo <T>;.\n   */\n  static const std::vector<EnumValueInfo<T>>& getEntries() {\n    return EnumMeta<T>::getInfo()->byIndex;\n  }\n\n};\n\n/**\n * Mapping-enabled Enum. See &l:EnumObjectWrapper;.\n */\ntemplate <class T>\nusing Enum = EnumObjectWrapper<T, EnumInterpreterAsString<T, false>>;\n\ntemplate<class T, bool notnull>\nVoid EnumInterpreterAsString<T, notnull>::toInterpretation(const Void& enumValue, EnumInterpreterError& error) {\n  typedef EnumObjectWrapper<T, EnumInterpreterAsString<T, notnull>> EnumOW;\n\n  if(enumValue.getValueType() != EnumOW::Class::getType()) {\n    error = EnumInterpreterError::TYPE_MISMATCH_ENUM;\n    return Void(nullptr, String::Class::getType());\n  }\n\n  if(!enumValue) {\n    if(notnull) {\n      error = EnumInterpreterError::CONSTRAINT_NOT_NULL;\n      return Void(nullptr, String::Class::getType());\n    }\n    return Void(nullptr, String::Class::getType());\n  }\n\n  const auto& ow = enumValue.template cast<EnumOW>();\n  const auto& entry = EnumOW::getEntryByValue(*ow);\n  return entry.name.toString();\n}\n\ntemplate<class T, bool notnull>\nVoid EnumInterpreterAsString<T, notnull>::fromInterpretation(const Void& interValue, EnumInterpreterError& error) {\n  typedef EnumObjectWrapper<T, EnumInterpreterAsString<T, notnull>> EnumOW;\n\n  if(interValue.getValueType() != String::Class::getType()) {\n    error = EnumInterpreterError::TYPE_MISMATCH_ENUM_VALUE;\n    return Void(nullptr, EnumOW::Class::getType());\n  }\n\n  if(!interValue) {\n    if(notnull) {\n      error = EnumInterpreterError::CONSTRAINT_NOT_NULL;\n      return Void(nullptr, EnumOW::Class::getType());\n    }\n    return Void(nullptr, EnumOW::Class::getType());\n  }\n\n  try {\n    return EnumOW(EnumOW::getEntryByName(interValue.template cast<String>()).value);\n  } catch (const std::runtime_error&) { // TODO - add a specific error for this.\n    error = EnumInterpreterError::ENTRY_NOT_FOUND;\n  }\n  return Void(nullptr, EnumOW::Class::getType());\n}\n\ntemplate<class T, bool notnull>\nType* EnumInterpreterAsString<T, notnull>::getInterpretationType() {\n  return String::Class::getType();\n}\n\ntemplate<class T, bool notnull>\nVoid EnumInterpreterAsNumber<T, notnull>::toInterpretation(const Void& enumValue, EnumInterpreterError& error) {\n\n  typedef EnumObjectWrapper<T, EnumInterpreterAsNumber<T, notnull>> EnumOW;\n  typedef typename std::underlying_type<T>::type EnumUT;\n  typedef typename ObjectWrapperByUnderlyingType<EnumUT>::ObjectWrapper UTOW;\n\n  if(enumValue.getValueType() != EnumOW::Class::getType()) {\n    error = EnumInterpreterError::TYPE_MISMATCH_ENUM;\n    return Void(nullptr, UTOW::Class::getType());\n  }\n\n  if(!enumValue) {\n    if(notnull) {\n      error = EnumInterpreterError::CONSTRAINT_NOT_NULL;\n      return Void(nullptr, UTOW::Class::getType());\n    }\n    return Void(nullptr, UTOW::Class::getType());\n  }\n\n  const auto& ow = enumValue.template cast<EnumOW>();\n  return UTOW(static_cast<EnumUT>(*ow));\n\n}\n\ntemplate<class T, bool notnull>\nVoid EnumInterpreterAsNumber<T, notnull>::fromInterpretation(const Void& interValue, EnumInterpreterError& error) {\n  typedef EnumObjectWrapper<T, EnumInterpreterAsNumber<T, notnull>> EnumOW;\n\n  typedef typename std::underlying_type<T>::type EnumUT;\n  typedef typename ObjectWrapperByUnderlyingType<EnumUT>::ObjectWrapper OW;\n\n  if(interValue.getValueType() != OW::Class::getType()) {\n    error = EnumInterpreterError::TYPE_MISMATCH_ENUM_VALUE;\n    return Void(nullptr, EnumOW::Class::getType());\n  }\n\n  if(!interValue) {\n    if(notnull) {\n      error = EnumInterpreterError::CONSTRAINT_NOT_NULL;\n      return Void(nullptr, EnumOW::Class::getType());\n    }\n    return Void(nullptr, EnumOW::Class::getType());\n  }\n\n  try{\n    const auto& entry = EnumOW::getEntryByUnderlyingValue(\n      interValue.template cast<OW>()\n    );\n    return EnumOW(entry.value);\n  } catch (const std::runtime_error&) { // TODO - add a specific error for this.\n    error = EnumInterpreterError::ENTRY_NOT_FOUND;\n  }\n  return Void(nullptr, EnumOW::Class::getType());\n}\n\ntemplate<class T, bool notnull>\nType* EnumInterpreterAsNumber<T, notnull>::getInterpretationType() {\n  typedef typename std::underlying_type<T>::type EnumUT;\n  return ObjectWrapperByUnderlyingType<EnumUT>::ObjectWrapper::Class::getType();\n}\n\nnamespace __class {\n\n  template<class T, class Interpreter>\n  class Enum : public AbstractEnum {\n  private:\n\n    class PolymorphicDispatcher : public AbstractEnum::PolymorphicDispatcher {\n    public:\n\n      PolymorphicDispatcher()\n        : AbstractEnum::PolymorphicDispatcher(Interpreter::notNull)\n      {}\n\n      type::Void createObject() const override {\n        return type::Void(std::make_shared<T>(), getType());\n      }\n\n      type::Void toInterpretation(const type::Void& enumValue, EnumInterpreterError& error) const override {\n        return Interpreter::toInterpretation(enumValue, error);\n      }\n\n      type::Void fromInterpretation(const type::Void& interValue, EnumInterpreterError& error) const override {\n        return Interpreter::fromInterpretation(interValue, error);\n      }\n\n      type::Type* getInterpretationType() const override {\n        return Interpreter::getInterpretationType();\n      }\n\n      std::vector<type::Any> getInterpretedEnum() const override {\n\n        typedef type::EnumObjectWrapper<T, Interpreter> EnumOW;\n\n        std::vector<type::Any> result({});\n\n        for(const auto& e : EnumOW::getEntries()) {\n          type::EnumInterpreterError error = type::EnumInterpreterError::OK;\n          result.push_back(type::Any(toInterpretation(EnumOW(e.value), error)));\n          if(error != type::EnumInterpreterError::OK) {\n            throw std::runtime_error(\"[oatpp::data::mapping::type::__class::Enum<T, Interpreter>::getInterpretedEnum()]: Unknown error.\");\n          }\n        }\n\n        return result;\n\n      }\n\n    };\n\n  private:\n\n    static Type createType() {\n      Type::Info info;\n      info.nameQualifier = type::EnumMeta<T>::getInfo()->nameQualifier;\n      info.polymorphicDispatcher = new PolymorphicDispatcher();\n      return Type(__class::AbstractEnum::CLASS_ID, info);\n    }\n\n  public:\n\n    static Type* getType() {\n      static Type type = createType();\n      return &type;\n    }\n\n  };\n\n}\n\n}}}}\n\nnamespace std {\n\n  template<class T, class I>\n  struct hash <oatpp::data::mapping::type::EnumObjectWrapper<T, I>> {\n\n    typedef oatpp::data::mapping::type::EnumObjectWrapper<T, I> argument_type;\n    typedef v_uint64 result_type;\n\n    result_type operator()(argument_type const &e) const noexcept {\n      if (e.get() == nullptr) return 0;\n      return static_cast<v_uint64>(*e);\n    }\n\n  };\n\n}\n\n#endif // oatpp_data_mapping_type_Enum_hpp\n\n// Path: src/oatpp/core/data/mapping/type/Map.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_Map_hpp\n#define oatpp_data_mapping_type_Map_hpp\n\n#include \"./Type.hpp\"\n#include <list>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nnamespace __class {\n\n/**\n * Abstract Map. <br>\n * Ex.: UnorderedMap, Fields.\n */\nclass Map {\npublic:\n\n  /**\n   * Iterator.\n   */\n  struct Iterator {\n\n    /**\n     * Default virtual destructor.\n     */\n    virtual ~Iterator() = default;\n\n    /**\n     * Get current item key.\n     * @return\n     */\n    virtual type::Void getKey() = 0;\n\n    /**\n     * Get current item value.\n     * @return\n     */\n    virtual type::Void getValue() = 0;\n\n    /**\n     * Iterate to next item.\n     */\n    virtual void next() = 0;\n\n    /**\n     * Check if iterator finished.\n     * @return\n     */\n    virtual bool finished() = 0;\n\n  };\n\npublic:\n\n  /**\n   * Polymorphic Dispatcher\n   */\n  class PolymorphicDispatcher {\n  public:\n\n    /**\n     * Virtual destructor.\n     */\n    virtual ~PolymorphicDispatcher() = default;\n\n    /**\n     * Create Map.\n     * @return\n     */\n    virtual type::Void createObject() const = 0;\n\n    /**\n     * Get type of map keys.\n     * @return\n     */\n    virtual const type::Type* getKeyType() const = 0;\n\n    /**\n     * Get type of map values.\n     * @return\n     */\n    virtual const type::Type* getValueType() const = 0;\n\n    /**\n     * Get map size.\n     * @param object - map object.\n     * @return - size of the map.\n     */\n    virtual v_int64 getMapSize(const type::Void& object) const = 0;\n\n    /**\n     * Add item.\n     * @param object - Map.\n     * @param key\n     * @param value\n     */\n    virtual void addItem(const type::Void& object, const type::Void& key, const type::Void& value) const = 0;\n\n    /**\n     * Begin map iteration.\n     * @param object - Map.\n     * @return\n     */\n    virtual std::unique_ptr<Iterator> beginIteration(const type::Void& object) const = 0;\n\n  };\n\n  template<class ContainerType, class KeyType, class ValueType>\n  struct Inserter {\n\n    static void insert(ContainerType* c, const KeyType& k, const ValueType& v) {\n      (*c)[k] = v;\n    }\n\n  };\n\n};\n\ntemplate<class ContainerType, class KeyType, class ValueType, class Clazz>\nclass StandardMap {\npublic:\n\n  struct Iterator : public Map::Iterator {\n\n    typename ContainerType::iterator iterator;\n    typename ContainerType::iterator end;\n\n    type::Void getKey() override {\n      return iterator->first;\n    }\n\n    type::Void getValue() override {\n      return iterator->second;\n    }\n\n    void next() override {\n      std::advance(iterator, 1);\n    }\n\n    bool finished() override {\n      return iterator == end;\n    }\n\n  };\n\npublic:\n\n  class PolymorphicDispatcher : public Map::PolymorphicDispatcher {\n  public:\n\n    type::Void createObject() const override {\n      return type::Void(std::make_shared<ContainerType>(), Clazz::getType());\n    }\n\n    const type::Type* getKeyType() const override {\n      const type::Type* mapType = Clazz::getType();\n      return mapType->params[0];\n    }\n\n    const type::Type* getValueType() const override {\n      const type::Type* mapType = Clazz::getType();\n      return mapType->params[1];\n    }\n\n    v_int64 getMapSize(const type::Void& object) const override {\n      ContainerType* map = static_cast<ContainerType*>(object.get());\n      return static_cast<v_int64>(map->size());\n    }\n\n    void addItem(const type::Void& object, const type::Void& key, const type::Void& value) const override {\n      ContainerType* map = static_cast<ContainerType*>(object.get());\n      const auto& mapKey = key.template cast<KeyType>();\n      const auto& mapValue = value.template cast<ValueType>();\n      Map::Inserter<ContainerType, KeyType, ValueType>::insert(map, mapKey, mapValue);\n    }\n\n    std::unique_ptr<Map::Iterator> beginIteration(const type::Void& object) const override {\n      ContainerType* map = static_cast<ContainerType*>(object.get());\n      auto iterator = new Iterator();\n      iterator->iterator = map->begin();\n      iterator->end = map->end();\n      return std::unique_ptr<Map::Iterator>(iterator);\n    }\n\n  };\n\n};\n\ntemplate<class KeyType, class ValueType>\nstruct Map::Inserter<std::list<std::pair<KeyType, ValueType>>, KeyType, ValueType> {\n  static void insert(std::list<std::pair<KeyType, ValueType>>* c, const KeyType& k, const ValueType& v) {\n    c->push_back({k, v});\n  }\n};\n\n}\n\n}}}}\n\n#endif //oatpp_data_mapping_type_Map_hpp\n\n// Path: src/oatpp/core/data/mapping/type/UnorderedMap.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_UnorderedMap_hpp\n#define oatpp_data_mapping_type_UnorderedMap_hpp\n\n#include \"./Map.hpp\"\n#include \"./Type.hpp\"\n\n#include <unordered_map>\n#include <initializer_list>\n#include <utility>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nnamespace __class {\n\n  /**\n   * Abstract Unordered Map class.\n   */\n  class AbstractUnorderedMap {\n  public:\n    /**\n     * Class Id.\n     */\n    static const ClassId CLASS_ID;\n  };\n\n  template<class Key, class Value>\n  class UnorderedMap;\n\n}\n\n/**\n * `ObjectWrapper` for `std::unordered_map<Key, Value>`\n * @tparam Key - Key `ObjectWrapper` type.\n * @tparam Value - Value `ObjectWrapper` type.\n * @tparam C - Class.\n */\ntemplate<class Key, class Value, class C>\nclass UnorderedMapObjectWrapper : public type::ObjectWrapper<std::unordered_map<Key, Value>, C> {\npublic:\n  typedef std::unordered_map<Key, Value> TemplateObjectType;\n  typedef C TemplateObjectClass;\npublic:\n\n  OATPP_DEFINE_OBJECT_WRAPPER_DEFAULTS(UnorderedMapObjectWrapper, TemplateObjectType, TemplateObjectClass)\n\n  UnorderedMapObjectWrapper(std::initializer_list<std::pair<const Key, Value>> ilist)\n    : type::ObjectWrapper<TemplateObjectType, TemplateObjectClass>(std::make_shared<TemplateObjectType>(ilist))\n  {}\n\n  static UnorderedMapObjectWrapper createShared() {\n    return std::make_shared<TemplateObjectType>();\n  }\n\n  UnorderedMapObjectWrapper& operator = (std::initializer_list<std::pair<const Key, Value>> ilist) {\n    this->m_ptr = std::make_shared<TemplateObjectType>(ilist);\n    return *this;\n  }\n\n  Value& operator[] (const Key& key) const {\n    return this->m_ptr->operator [] (key);\n  }\n\n  TemplateObjectType& operator*() const {\n    return this->m_ptr.operator*();\n  }\n\n};\n\n/**\n * Mapping-Enables UnorderedMap<Key, Value>. See &l:UnorderedMapObjectWrapper;.\n */\ntemplate<class Key, class Value>\nusing UnorderedMap = UnorderedMapObjectWrapper<Key, Value, __class::UnorderedMap<Key, Value>>;\n\nnamespace __class {\n\n  template<class Key, class Value>\n  class UnorderedMap : public AbstractUnorderedMap {\n  private:\n\n    static Type createType() {\n      Type::Info info;\n      info.params.push_back(Key::Class::getType());\n      info.params.push_back(Value::Class::getType());\n      info.polymorphicDispatcher =\n        new typename __class::StandardMap<std::unordered_map<Key, Value>, Key, Value, UnorderedMap>::PolymorphicDispatcher();\n      info.isMap = true;\n      return Type(__class::AbstractUnorderedMap::CLASS_ID, info);\n    }\n\n  public:\n\n    static Type* getType() {\n      static Type type = createType();\n      return &type;\n    }\n\n  };\n\n}\n\n}}}}\n\n#endif // oatpp_data_mapping_type_UnorderedMap_hpp\n\n// Path: src/oatpp/core/data/mapping/type/PairList.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_PairList_hpp\n#define oatpp_data_mapping_type_PairList_hpp\n\n#include \"./Map.hpp\"\n#include \"./Type.hpp\"\n\n#include <list>\n#include <initializer_list>\n#include <utility>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nnamespace __class {\n\n  /**\n   * Abstract PairList class.\n   */\n  class AbstractPairList {\n  public:\n    /**\n     * Class id.\n     */\n    static const ClassId CLASS_ID;\n  };\n\n  template<class Key, class Value>\n  class PairList;\n\n}\n\n/**\n * `ObjectWrapper` over `std::list<std::pair<Key, Value>>`\n * @tparam Key - Key `ObjectWrapper` type.\n * @tparam Value - Value `ObjectWrapper` type.\n * @tparam C - Class.\n */\ntemplate<class Key, class Value, class C>\nclass PairListObjectWrapper : public type::ObjectWrapper<std::list<std::pair<Key, Value>>, C> {\npublic:\n  typedef std::list<std::pair<Key, Value>> TemplateObjectType;\n  typedef C TemplateObjectClass;\npublic:\n\n  OATPP_DEFINE_OBJECT_WRAPPER_DEFAULTS(PairListObjectWrapper, TemplateObjectType, TemplateObjectClass)\n\n  \nPairListObjectWrapper(std::initializer_list<std::pair<Key, Value>> ilist)\n    : type::ObjectWrapper<TemplateObjectType, TemplateObjectClass>(std::make_shared<TemplateObjectType>(ilist))\n  {}\n\n  static PairListObjectWrapper createShared() {\n    return std::make_shared<TemplateObjectType>();\n  }\n\n  PairListObjectWrapper& operator = (std::initializer_list<std::pair<Key, Value>> ilist) {\n    this->m_ptr = std::make_shared<TemplateObjectType>(ilist);\n    return *this;\n  }\n\n  std::pair<Key, Value>& operator[] (v_buff_usize index) const {\n    auto it = this->m_ptr->begin();\n    std::advance(it, index);\n    return *it;\n  }\n  \n  Value& operator[] (const Key& key) const {\n    auto& list = *(this->m_ptr.get());\n    auto it = list.begin();\n    while(it != list.end()) {\n      if(it->first == key) {\n        return it->second;\n      }\n      it ++;\n    }\n    list.push_back({key, nullptr});\n    return list.back().second;\n  }\n\n  Value getValueByKey(const Key& key, const Value& defValue = nullptr) const {\n    auto& list = *(this->m_ptr.get());\n    auto it = list.begin();\n    while(it != list.end()) {\n      if(it->first == key) {\n        return it->second;\n      }\n      it ++;\n    }\n    return defValue;\n  }\n\n  TemplateObjectType& operator*() const {\n    return this->m_ptr.operator*();\n  }\n\n};\n\n/**\n * Mapping-Enables PairList<Key, Value>. See &l:PairListObjectWrapper;.\n */\ntemplate<class Key, class Value>\nusing PairList = PairListObjectWrapper<Key, Value, __class::PairList<Key, Value>>;\n\nnamespace __class {\n\ntemplate<class Key, class Value>\nclass PairList : public AbstractPairList {\nprivate:\n\n  static Type createType() {\n    Type::Info info;\n    info.params.push_back(Key::Class::getType());\n    info.params.push_back(Value::Class::getType());\n    info.polymorphicDispatcher =\n      new typename __class::StandardMap<std::list<std::pair<Key, Value>>, Key, Value, PairList>::PolymorphicDispatcher();\n    info.isMap = true;\n    return Type(__class::AbstractPairList::CLASS_ID, info);\n  }\n\npublic:\n\n  static Type* getType() {\n    static Type type = createType();\n    return &type;\n  }\n\n};\n\n}\n\n}}}}\n\n#endif // oatpp_data_mapping_type_PairList_hpp\n\n// Path: src/oatpp/core/data/mapping/type/Collection.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_Collection_hpp\n#define oatpp_data_mapping_type_Collection_hpp\n\n#include \"./Type.hpp\"\n#include <unordered_set>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nnamespace __class {\n\n/**\n * Abstract Collection. <br>\n * Ex.: Vector, List, Set.\n */\nclass Collection {\npublic:\n\n  /**\n   * Iterator.\n   */\n  struct Iterator {\n\n    /**\n     * Default virtual destructor.\n     */\n    virtual ~Iterator() = default;\n\n    /**\n     * Get current item.\n     * @return\n     */\n    virtual type::Void get() = 0;\n\n    /**\n     * Iterate to next item.\n     */\n    virtual void next() = 0;\n\n    /**\n     * Check if iterator finished.\n     * @return\n     */\n    virtual bool finished() = 0;\n\n  };\n\npublic:\n\n  /**\n   * Polymorphic Dispatcher\n   */\n  class PolymorphicDispatcher {\n  public:\n\n    /**\n     * Virtual destructor.\n     */\n    virtual ~PolymorphicDispatcher() = default;\n\n    /**\n     * Create Collection.\n     * @return\n     */\n    virtual type::Void createObject() const = 0;\n\n    /**\n     * Get type of collection items.\n     * @return\n     */\n    virtual const type::Type* getItemType() const = 0;\n\n    /**\n     * Get collection size.\n     * @param object - collection.\n     * @return - size of the collection (elements count).\n     */\n    virtual v_int64 getCollectionSize(const type::Void& object) const = 0;\n\n    /**\n     * Add item.\n     * @param object - Collection.\n     * @param item - Item to add.\n     */\n    virtual void addItem(const type::Void& object, const type::Void& item) const = 0;\n\n    /**\n     * Begin collection iteration.\n     * @param object - Collection.\n     * @return\n     */\n    virtual std::unique_ptr<Iterator> beginIteration(const type::Void& object) const = 0;\n\n  };\n\n  template<class ContainerType, class ItemType>\n  struct Inserter {\n\n    static void insert(ContainerType* c, const ItemType& i) {\n      c->emplace_back(i);\n    }\n\n  };\n\n};\n\ntemplate<class ContainerType, class ItemType, class Clazz>\nclass StandardCollection {\npublic:\n\n  struct Iterator : public Collection::Iterator {\n\n    typename ContainerType::iterator iterator;\n    typename ContainerType::iterator end;\n\n    type::Void get() override {\n      return *iterator;\n    }\n\n    void next() override {\n      std::advance(iterator, 1);\n    }\n\n    bool finished() override {\n      return iterator == end;\n    }\n\n  };\n\npublic:\n\n  class PolymorphicDispatcher : public Collection::PolymorphicDispatcher {\n  public:\n\n    type::Void createObject() const override {\n      return type::Void(std::make_shared<ContainerType>(), Clazz::getType());\n    }\n\n    const type::Type* getItemType() const override {\n      const type::Type* collectionType = Clazz::getType();\n      return collectionType->params[0];\n    }\n\n    v_int64 getCollectionSize(const type::Void& object) const override {\n      ContainerType* collection = static_cast<ContainerType*>(object.get());\n      return static_cast<v_int64>(collection->size());\n    }\n\n    void addItem(const type::Void& object, const type::Void& item) const override {\n      ContainerType* collection = static_cast<ContainerType*>(object.get());\n      const auto& collectionItem = item.template cast<ItemType>();\n      Collection::Inserter<ContainerType, ItemType>::insert(collection, collectionItem);\n    }\n\n    std::unique_ptr<Collection::Iterator> beginIteration(const type::Void& object) const override {\n      ContainerType* collection = static_cast<ContainerType*>(object.get());\n      auto iterator = new Iterator();\n      iterator->iterator = collection->begin();\n      iterator->end = collection->end();\n      return std::unique_ptr<Collection::Iterator>(iterator);\n    }\n\n  };\n\n};\n\ntemplate<class ItemType>\nstruct Collection::Inserter<std::unordered_set<ItemType>, ItemType> {\n  static void insert(std::unordered_set<ItemType>* c, const ItemType& i) {\n    c->emplace(i);\n  }\n};\n\n}\n\n}}}}\n\n#endif //oatpp_data_mapping_type_Collection_hpp\n\n// Path: src/oatpp/core/data/mapping/type/List.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_List_hpp\n#define oatpp_data_mapping_type_List_hpp\n\n#include \"./Collection.hpp\"\n#include \"./Type.hpp\"\n\n#include <list>\n#include <initializer_list>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nnamespace __class {\n\n  /**\n   * Abstract list class.\n   */\n  class AbstractList {\n  public:\n\n    /**\n     * Class Id.\n     */\n    static const ClassId CLASS_ID;\n\n  };\n\n  template<class T>\n  class List;\n\n}\n\n/**\n * `ObjectWrapper` over `std::list<T>`\n * @tparam T - Item `ObjectWrapper` type.\n * @tparam C - Class.\n */\ntemplate<class T, class C>\nclass ListObjectWrapper : public type::ObjectWrapper<std::list<T>, C> {\npublic:\n  typedef std::list<T> TemplateObjectType;\n  typedef C TemplateObjectClass;\npublic:\n\n  OATPP_DEFINE_OBJECT_WRAPPER_DEFAULTS(ListObjectWrapper, TemplateObjectType, TemplateObjectClass)\n\n  ListObjectWrapper(std::initializer_list<T> ilist)\n    : type::ObjectWrapper<TemplateObjectType, TemplateObjectClass>(std::make_shared<TemplateObjectType>(ilist))\n  {}\n\n  static ListObjectWrapper createShared() {\n    return std::make_shared<TemplateObjectType>();\n  }\n\n  ListObjectWrapper& operator = (std::initializer_list<T> ilist) {\n    this->m_ptr = std::make_shared<TemplateObjectType>(ilist);\n    return *this;\n  }\n\n  T& operator[] (v_buff_usize index) const {\n    auto it = this->m_ptr->begin();\n    std::advance(it, index);\n    return *it;\n  }\n\n  TemplateObjectType& operator*() const {\n    return this->m_ptr.operator*();\n  }\n\n};\n\n/**\n * Mapping-Enabled List. See - &l:ListObjectWrapper;.\n */\ntemplate<class T>\nusing List = ListObjectWrapper<T, __class::List<T>>;\n\ntypedef List<Void> AbstractList;\n\nnamespace __class {\n\n  template<class T>\n  class List : public AbstractList {\n  private:\n\n    static Type createType() {\n      Type::Info info;\n      info.params.push_back(T::Class::getType());\n      info.polymorphicDispatcher = new typename StandardCollection<std::list<T>, T, List>::PolymorphicDispatcher();\n      info.isCollection = true;\n      return Type(__class::AbstractList::CLASS_ID, info);\n    }\n\n  public:\n\n    static Type* getType() {\n      static Type type = createType();\n      return &type;\n    }\n\n  };\n\n}\n\n}}}}\n\n#endif // oatpp_data_mapping_type_List_hpp\n\n// Path: src/oatpp/core/data/mapping/type/Vector.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_Vector_hpp\n#define oatpp_data_mapping_type_Vector_hpp\n\n#include \"./Collection.hpp\"\n#include \"./Type.hpp\"\n\n#include <vector>\n#include <initializer_list>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nnamespace __class {\n\n  /**\n   * Abstract Vector Class.\n   */\n  class AbstractVector {\n  public:\n    /**\n     * Class Id.\n     */\n    static const ClassId CLASS_ID;\n  };\n\n  template<class T>\n  class Vector;\n\n}\n\n/**\n * `ObjectWrapper` over `std::vector<T>`.\n * @tparam T - Item `ObjectWrapper` type.\n * @tparam C - Class.\n */\ntemplate<class T, class C>\nclass VectorObjectWrapper : public type::ObjectWrapper<std::vector<T>, C> {\npublic:\n  typedef std::vector<T> TemplateObjectType;\n  typedef C TemplateObjectClass;\npublic:\n\n  OATPP_DEFINE_OBJECT_WRAPPER_DEFAULTS(VectorObjectWrapper, TemplateObjectType, TemplateObjectClass)\n\n  VectorObjectWrapper(std::initializer_list<T> ilist)\n    : type::ObjectWrapper<TemplateObjectType, TemplateObjectClass>(std::make_shared<TemplateObjectType>(ilist))\n  {}\n\n  static VectorObjectWrapper createShared() {\n    return std::make_shared<TemplateObjectType>();\n  }\n\n  VectorObjectWrapper& operator = (std::initializer_list<T> ilist) {\n    this->m_ptr = std::make_shared<TemplateObjectType>(ilist);\n    return *this;\n  }\n\n  T& operator[] (v_buff_usize index) const {\n    return this->m_ptr->operator [] (index);\n  }\n\n  TemplateObjectType& operator*() const {\n    return this->m_ptr.operator*();\n  }\n\n};\n\n/**\n * Mapping-enabled Vector. See &l:VectorObjectWrapper;.\n */\ntemplate<class T>\nusing Vector = VectorObjectWrapper<T, __class::Vector<T>>;\n\ntypedef Vector<Void> AbstractVector;\n\nnamespace __class {\n\n  template<class T>\n  class Vector : public AbstractVector {\n  private:\n\n    static Type createType() {\n      Type::Info info;\n      info.params.push_back(T::Class::getType());\n      info.polymorphicDispatcher = new typename StandardCollection<std::vector<T>, T, Vector>::PolymorphicDispatcher();\n      info.isCollection = true;\n      return Type(__class::AbstractVector::CLASS_ID, info);\n    }\n\n  public:\n\n    static Type* getType() {\n      static Type type = createType();\n      return &type;\n    }\n\n  };\n\n}\n\n}}}}\n\n#endif // oatpp_data_mapping_type_Vector_hpp\n\n// Path: src/oatpp/core/data/mapping/type/UnorderedSet.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_mapping_type_UnorderedSet_hpp\n#define oatpp_data_mapping_type_UnorderedSet_hpp\n\n#include \"./Collection.hpp\"\n#include \"./Type.hpp\"\n\n#include <unordered_set>\n#include <initializer_list>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nnamespace __class {\n\n  /**\n   * Abstract Unordered Set class.\n   */\n  class AbstractUnorderedSet {\n  public:\n    /**\n     * Class Id.\n     */\n    static const ClassId CLASS_ID;\n  };\n\n  template<class T>\n  class UnorderedSet;\n\n}\n\n/**\n * `ObjectWrapper` over `std::unordered_set<T>`\n * @tparam T - Key `ObjectWrapper` type.\n * @tparam C - Class.\n */\ntemplate<class T, class C>\nclass UnorderedSetObjectWrapper : public type::ObjectWrapper<std::unordered_set<T>, C> {\npublic:\n  typedef std::unordered_set<T> TemplateObjectType;\n  typedef C TemplateObjectClass;\npublic:\n\n  OATPP_DEFINE_OBJECT_WRAPPER_DEFAULTS(UnorderedSetObjectWrapper, TemplateObjectType, TemplateObjectClass)\n\n  UnorderedSetObjectWrapper(std::initializer_list<T> ilist)\n    : type::ObjectWrapper<TemplateObjectType, TemplateObjectClass>(std::make_shared<TemplateObjectType>(ilist))\n  {}\n\n  static UnorderedSetObjectWrapper createShared() {\n    return std::make_shared<TemplateObjectType>();\n  }\n\n  UnorderedSetObjectWrapper& operator = (std::initializer_list<T> ilist) {\n    this->m_ptr = std::make_shared<TemplateObjectType>(ilist);\n    return *this;\n  }\n\n  bool operator[] (const T& key) const {\n    if(this->m_ptr) {\n      auto it = this->m_ptr->find(key);\n      return it != this->m_ptr->end();\n    }\n    return false;\n  }\n\n  TemplateObjectType& operator*() const {\n    return this->m_ptr.operator*();\n  }\n\n};\n\n/**\n * Mapping-Enabled UnorderedSet. See &l:UnorderedSetObjectWrapper;.\n */\ntemplate<class T>\nusing UnorderedSet = UnorderedSetObjectWrapper<T, __class::UnorderedSet<T>>;\n\ntypedef UnorderedSet<Void> AbstractUnorderedSet;\n\nnamespace __class {\n\ntemplate<class T>\nclass UnorderedSet : public AbstractUnorderedSet {\nprivate:\n\n  static Type createType() {\n    Type::Info info;\n    info.params.push_back(T::Class::getType());\n    info.polymorphicDispatcher = new typename StandardCollection<std::unordered_set<T>, T, UnorderedSet>::PolymorphicDispatcher();\n    info.isCollection = true;\n    return Type(__class::AbstractUnorderedSet::CLASS_ID, info);\n  }\n\npublic:\n\n  static Type* getType() {\n    static Type type = createType();\n    return &type;\n  }\n\n};\n\n}\n\n}}}}\n\n#endif // oatpp_data_mapping_type_UnorderedSet_hpp\n\n// Path: src/oatpp/core/data/mapping/type/Object.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n...\n", "needle_token_start": 10275, "needle_token_end": 10331, "code_context_ntokens": 15782, "output": ["```cpp\ntemplate<class Key, class Value, class C>\nclass PairListObjectWrapper : public type::ObjectWrapper<std::list<std::pair<Key, Value>>, C> {\npublic:\n  typedef std::list<std::pair<Key, Value>> TemplateObjectType;\n  typedef C TemplateObjectClass;\npublic:\n\n  OATPP_DEFINE_OBJECT_WRAPPER_DEFAULTS(PairListObjectWrapper, TemplateObjectType, TemplateObjectClass)\n\n  PairListObjectWrapper(std::initializer_list<std::pair<Key, Value>> ilist)\n    : type::ObjectWrapper<TemplateObjectType, TemplateObjectClass>(std::make_shared<TemplateObjectType>(ilist))\n  {}\n\n  static PairListObjectWrapper createShared() {\n    return std::make_shared<TemplateObjectType>();\n  }\n\n  PairListObjectWrapper& operator = (std::initializer_list<std::pair<Key, Value>> ilist) {\n    this->m_ptr = std::make_shared<TemplateObjectType>(ilist);\n    return *this;\n  }\n\n  std::pair<Key, Value>& operator[] (v_buff_usize index) const {\n    auto it = this->m_ptr->begin();\n    std::advance(it, index);\n    return *it;\n  }\n  \n  Value& operator[] (const Key& key) const {\n    auto& list = *(this->m_ptr.get());\n    auto it = list.begin();\n    while(it != list.end()) {\n      if(it->first == key) {\n        return it->second;\n      }\n      it ++;\n    }\n    list.push_back({key, nullptr});\n    return list.back().second;\n  }\n\n  Value getValueByKey(const Key& key, const Value& defValue = nullptr) const {\n    auto& list = *(this->m_ptr.get());\n    auto it = list.begin();\n    while(it != list.end()) {\n      if(it->first == key) {\n        return it->second;\n      }\n      it ++;\n    }\n    return defValue;\n  }\n\n  TemplateObjectType& operator*() const {\n    return this->m_ptr.operator*();\n  }\n\n};\n```"]}
{"repo": "oatpp/oatpp", "name": "FlushCoroutine", "language": "cpp", "path": "src/oatpp/core/data/buffer/FIFOBuffer.cpp", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this coroutine is to asynchronously transfer all available data from a buffer to an output stream until the buffer is empty.\n2. **Input**: The coroutine takes two inputs: a reference to a buffer from which data is read, and a shared pointer to an output stream where data is written.\n3. **Output**: The coroutine does not return a value directly but performs asynchronous I/O operations, resulting in data being written to the output stream.\n4. **Procedure**: The coroutine first checks if there is data available in the buffer. If data is available, it determines the segments of the buffer to write based on the current read and write positions. It then performs asynchronous writes to the output stream in one or two parts, depending on whether the data wraps around the end of the buffer. After writing, it resets the buffer's positions, effectively emptying it.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "...\n// Path: src/oatpp/core/data/resource/TemporaryFile.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_resource_TemporaryFile_hpp\n#define oatpp_data_resource_TemporaryFile_hpp\n\n#include \"./Resource.hpp\"\n#include \"oatpp/core/Types.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\n/**\n * Temporary file - the file which gets deleted when the destructor is called\n * (more precisely when all copies of the same `TemporaryFile` object deleted). <br>\n * The `TemporaryFile` object internally stores a `shared_ptr` to a file handle.\n * When file handle deleted it also deletes the underlying file. <br>\n * Thus it's safe to copy `TemporaryFile` object and you may treat `TemporaryFile` object\n * as a shared_ptr to a temporary file. <br>\n * @extends - &id:oatpp::data::Resource;.\n */\nclass TemporaryFile : public Resource {\nprivate:\n\n  /*\n   * Shared handle.\n   * File is deleted on handle destroy.\n   */\n  struct FileHandle {\n\n    oatpp::String fileName;\n\n    FileHandle(const oatpp::String& fullFileName)\n      : fileName(fullFileName)\n    {}\n\n    ~FileHandle();\n  };\n\nprivate:\n  static oatpp::String constructRandomFilename(const oatpp::String &dir, v_int32 randomWordSizeBytes, const oatpp::String &extension);\nprivate:\n  std::shared_ptr<FileHandle> m_handle;\npublic:\n\n  /**\n   * Default constructor.\n   */\n  TemporaryFile() = default;\n\n  /**\n   * Constructor. <br>\n   * Create temporary file with a random name in the `tmpDirectory`. <br>\n   * The actual file will be created only after first write to that file. <br>\n   * Example of the generated random file name: `f7c6ecd44024ff31.tmp`.\n   * @param tmpDirectory - directory where to create a temporary file.\n   * @param randomWordSizeBytes - number of random bytes to generate file name.\n   */\n  TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes = 8);\n\n  /**\n   * Constructor.<br>\n   * Create temporary file with the `tmpFileName` name in the `tmpDirectory`. <br>\n   * @param tmpDirectory - directory where to create a temporary file.\n   * @param tmpFileName - predefined name for the temporary file.\n   */\n  TemporaryFile(const oatpp::String& tmpDirectory, const oatpp::String& tmpFileName);\n\n  /**\n   * Constructor. <br>\n   * Create temporary file with a random name and specified extension in the `tmpDirectory`. <br>\n   * The actual file will be created only after first write to that file. <br>\n   * Example of the generated random file name: `f7c6ecd44024ff31.txt`.\n   * @param tmpDirectory - directory where to create a temporary file.\n   * @param randomWordSizeBytes - number of random bytes to generate file name.\n   * @param extension - extension of the temporary file, e.g. txt or .txt\n   */\n  TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes, const oatpp::String& extension);\n\n  /**\n   * Open output stream to a temporary file. <br>\n   * *Note: stream also captures file-handle. The temporary file won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` to &id:oatpp::data::stream::OutputStream;.\n   */\n  std::shared_ptr<data::stream::OutputStream> openOutputStream() override;\n\n  /**\n   * Open input stream to a temporary file. <br>\n   * *Note: stream also captures file-handle. The temporary file won't be deleted until the stream is deleted.*\n   * @return - `std::shared_ptr` &id:oatpp::data::stream::InputStream;.\n   */\n  std::shared_ptr<data::stream::InputStream> openInputStream() override;\n\n  /**\n   * Not applicable.\n   * @return - always returns `nullptr`.\n   */\n  oatpp::String getInMemoryData() override;\n\n  /**\n   * Not applicable.\n   * @return - always returns `-1`.\n   */\n  v_int64 getKnownSize() override;\n\n  /**\n   * Get location where temporary data is stored.\n   * @return - `&id:oatpp::String;`.\n   */\n  oatpp::String getLocation() override;\n\n  /**\n   * Move payload to a different file. <br>\n   * @param fullFileName - full-file-name. Note: all the parent folders must exist.\n   * @return - `true` - file was successfully moved, `false` - otherwise.\n   */\n  bool moveFile(const oatpp::String& fullFileName);\n\n};\n\n}}}\n\n#endif //oatpp_data_resource_TemporaryFile_hpp\n\n// Path: src/oatpp/core/data/resource/TemporaryFile.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"TemporaryFile.hpp\"\n\n#include \"./File.hpp\"\n\n#include \"oatpp/core/data/stream/FileStream.hpp\"\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n#include \"oatpp/encoding/Hex.hpp\"\n#include \"oatpp/core/utils/Random.hpp\"\n\nnamespace oatpp { namespace data { namespace resource {\n\nTemporaryFile::FileHandle::~FileHandle() {\n  if(fileName) {\n    std::remove(fileName->c_str());\n  }\n}\n\noatpp::String TemporaryFile::constructRandomFilename(const oatpp::String &dir, v_int32 randomWordSizeBytes, const oatpp::String &extension) {\n\n  std::unique_ptr<v_char8[]> buff(new v_char8[randomWordSizeBytes]);\n  utils::random::Random::randomBytes(buff.get(), randomWordSizeBytes);\n  data::stream::BufferOutputStream s(randomWordSizeBytes * 2 + 4);\n  encoding::Hex::encode(&s, buff.get(), randomWordSizeBytes, encoding::Hex::ALPHABET_LOWER);\n  if (extension->at(0) != '.') {\n    s << \".\";\n  }\n  s << extension;\n\n  return File::concatDirAndName(dir, s.toString());\n\n}\n\nTemporaryFile::TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes)\n  : m_handle(std::make_shared<FileHandle>(constructRandomFilename(tmpDirectory, randomWordSizeBytes, \"tmp\")))\n{}\n\nTemporaryFile::TemporaryFile(const oatpp::String& tmpDirectory, const oatpp::String& tmpFileName)\n  : m_handle(std::make_shared<FileHandle>(File::concatDirAndName(tmpDirectory, tmpFileName)))\n{}\n\nTemporaryFile::TemporaryFile(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes, const oatpp::String& extension)\n  : m_handle(std::make_shared<FileHandle>(constructRandomFilename(tmpDirectory, randomWordSizeBytes, extension)))\n{}\n\nstd::shared_ptr<data::stream::OutputStream> TemporaryFile::openOutputStream() {\n  if(m_handle) {\n    return std::make_shared<data::stream::FileOutputStream>(m_handle->fileName->c_str(), \"wb\", m_handle);\n  }\n  throw std::runtime_error(\"[oatpp::data::resource::TemporaryFile::openOutputStream()]: Error. FileHandle is NOT initialized.\");\n}\n\nstd::shared_ptr<data::stream::InputStream> TemporaryFile::openInputStream() {\n  if(m_handle) {\n    return std::make_shared<data::stream::FileInputStream>(m_handle->fileName->c_str(), m_handle);\n  }\n  throw std::runtime_error(\"[oatpp::data::resource::TemporaryFile::openInputStream()]: Error. FileHandle is NOT initialized.\");\n}\n\noatpp::String TemporaryFile::getInMemoryData() {\n  return nullptr;\n}\n\nv_int64 TemporaryFile::getKnownSize() {\n  return -1;\n}\n\noatpp::String TemporaryFile::getLocation() {\n  if(m_handle) {\n    return m_handle->fileName;\n  }\n  return nullptr;\n}\n\nbool TemporaryFile::moveFile(const oatpp::String& fullFileName) {\n  if(m_handle) {\n    return std::rename(m_handle->fileName->c_str(), fullFileName->c_str()) == 0;\n  }\n  return false;\n}\n\n}}}\n\n// Path: src/oatpp/core/data/buffer/Processor.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_buffer_Processor_hpp\n#define oatpp_data_buffer_Processor_hpp\n\n#include \"oatpp/core/IODefinitions.hpp\"\n#include \"oatpp/core/base/ObjectHandle.hpp\"\n#include <vector>\n\nnamespace oatpp { namespace data { namespace buffer {\n\n/**\n * Convenience structure for stream Async-Inline read operations.\n */\nstruct InlineReadData {\n\n  /**\n   * Pointer to current position in the buffer.\n   */\n  void* currBufferPtr;\n\n  /**\n   * Bytes left to read to the buffer.\n   */\n  v_buff_size bytesLeft;\n\n  /**\n   * Default constructor.\n   */\n  InlineReadData();\n\n  /**\n   * Constructor.\n   * @param data\n   * @param size\n   */\n  InlineReadData(void* data, v_buff_size size);\n\n  /**\n   * Set `currBufferPtr` and `bytesLeft` values. <br>\n   * @param data - pointer to buffer to store read data.\n   * @param size - size in bytes of the buffer.\n   */\n  void set(void* data, v_buff_size size);\n\n  /**\n   * Increase position in the read buffer by `amount` bytes. <br>\n   * This will increase `currBufferPtr` and descrease `bytesLeft` values.\n   * @param amount\n   */\n  void inc(v_buff_size amount);\n\n  /**\n   * Same as `inc(bytesLeft).`\n   */\n  void setEof();\n\n};\n\n/**\n * Convenience structure for stream Async-Inline write operations.\n */\nstruct InlineWriteData {\n\n  /**\n   * Pointer to current position in the buffer.\n   */\n  const void* currBufferPtr;\n\n  /**\n   * Bytes left to write from the buffer.\n   */\n  v_buff_size bytesLeft;\n\n  /**\n   * Default constructor.\n   */\n  InlineWriteData();\n\n  /**\n   * Constructor.\n   * @param data\n   * @param size\n   */\n  InlineWriteData(const void* data, v_buff_size size);\n\n  /**\n   * Set `currBufferPtr` and `bytesLeft` values. <br>\n   * @param data - pointer to buffer containing data to be written.\n   * @param size - size in bytes of the buffer.\n   */\n  void set(const void* data, v_buff_size size);\n\n  /**\n   * Increase position in the write buffer by `amount` bytes. <br>\n   * This will increase `currBufferPtr` and descrease `bytesLeft` values.\n   * @param amount\n   */\n  void inc(v_buff_size amount);\n\n  /**\n   * Same as `inc(bytesLeft).`\n   */\n  void setEof();\n\n};\n\n/**\n * Buffer processor.\n * Note: all processors are considered to be stateful.\n */\nclass Processor {\npublic:\n\n  /**\n   * Enum of processing errors.\n   */\n  enum Error : v_int32 {\n\n    /**\n     * No error.\n     */\n    OK = 0,\n\n    /**\n     * Caller must set fields of `dataIn` parameter.\n     */\n    PROVIDE_DATA_IN = 1,\n\n    /**\n     * Caller must read all the data from the `dataOut`.\n     */\n    FLUSH_DATA_OUT = 2,\n\n    /**\n     * Processing is finished.\n     */\n    FINISHED = 3\n\n    //*********************************************//\n    // Other values are processor-specific errors. //\n    //*********************************************//\n  };\n\npublic:\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~Processor() = default;\n\n  /**\n   * If the client is using the input stream to read data and add it to the processor,\n   * the client MAY ask the processor for a suggested read size.\n   * @return - suggested read size.\n   */\n  virtual v_io_size suggestInputStreamReadSize() = 0;\n\n  /**\n   * Process data.\n   * @param dataIn - data provided by client to processor. Input data. &id:data::buffer::InlineReadData;.\n   * Set `dataIn` buffer pointer to `nullptr` to designate the end of input.\n   * @param dataOut - data provided to client by processor. Output data. &id:data::buffer::InlineReadData;.\n   * @return - &l:Processor::Error;.\n   */\n  virtual v_int32 iterate(data::buffer::InlineReadData& dataIn,\n                          data::buffer::InlineReadData& dataOut) = 0;\n\n};\n\n/**\n * Pipeline of buffer processors.\n */\nclass ProcessingPipeline : public Processor {\nprivate:\n  std::vector<base::ObjectHandle<Processor>> m_processors;\n  std::vector<data::buffer::InlineReadData> m_intermediateData;\npublic:\n\n  /**\n   * Constructor.\n   * @param m_processors - the array of processors defining the pipeline.\n   */\n  ProcessingPipeline(const std::vector<base::ObjectHandle<Processor>>& m_processors);\n\n  /**\n   * If the client is using the input stream to read data and add it to the processor,\n   * the client MAY ask the processor for a suggested read size.\n   * @return - suggested read size.\n   */\n  v_io_size suggestInputStreamReadSize() override;\n\n  /**\n   * Process data.\n   * @param dataIn - data provided by client to processor. Input data. &id:data::buffer::InlineReadData;.\n   * Set `dataIn` buffer pointer to `nullptr` to designate the end of input.\n   * @param dataOut - data provided to client by processor. Output data. &id:data::buffer::InlineReadData;.\n   * @return - &l:Processor::Error;.\n   */\n  v_int32 iterate(data::buffer::InlineReadData& dataIn,\n                  data::buffer::InlineReadData& dataOut) override;\n\n};\n\n}}}\n\n#endif // oatpp_data_buffer_Processor_hpp\n\n// Path: src/oatpp/core/data/buffer/Processor.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Processor.hpp\"\n\nnamespace oatpp { namespace data{ namespace buffer {\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// InlineReadData\n\nInlineReadData::InlineReadData()\n  : currBufferPtr(nullptr)\n  , bytesLeft(0)\n{}\n\nInlineReadData::InlineReadData(void* data, v_buff_size size)\n  : currBufferPtr(data)\n  , bytesLeft(size)\n{}\n\nvoid InlineReadData::set(void* data, v_buff_size size) {\n  currBufferPtr = data;\n  bytesLeft = size;\n}\n\nvoid InlineReadData::inc(v_buff_size amount) {\n  currBufferPtr = &(reinterpret_cast<p_char8>(currBufferPtr))[amount];\n  bytesLeft -= amount;\n}\n\nvoid InlineReadData::setEof() {\n  currBufferPtr = &(reinterpret_cast<p_char8>(currBufferPtr))[bytesLeft];\n  bytesLeft = 0;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// InlineWriteData\n\nInlineWriteData::InlineWriteData()\n  : currBufferPtr(nullptr)\n  , bytesLeft(0)\n{}\n\nInlineWriteData::InlineWriteData(const void* data, v_buff_size size)\n  : currBufferPtr(data)\n  , bytesLeft(size)\n{}\n\nvoid InlineWriteData::set(const void* data, v_buff_size size) {\n  currBufferPtr = data;\n  bytesLeft = size;\n}\n\nvoid InlineWriteData::inc(v_buff_size amount) {\n  currBufferPtr = &(reinterpret_cast<p_char8>(const_cast<void*>(currBufferPtr)))[amount];\n  bytesLeft -= amount;\n}\n\nvoid InlineWriteData::setEof() {\n  currBufferPtr = &(reinterpret_cast<p_char8>(const_cast<void*>(currBufferPtr)))[bytesLeft];\n  bytesLeft = 0;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// ProcessingPipeline\n\nProcessingPipeline::ProcessingPipeline(const std::vector<base::ObjectHandle<Processor>>& processors)\n  : m_processors(processors), m_intermediateData(processors.size() - 1)\n{\n}\n\nv_io_size ProcessingPipeline::suggestInputStreamReadSize() {\n  return m_processors[0]->suggestInputStreamReadSize();\n}\n\nv_int32 ProcessingPipeline::iterate(data::buffer::InlineReadData& dataIn,\n                                    data::buffer::InlineReadData& dataOut)\n{\n\n  if(dataOut.bytesLeft > 0) {\n    return Error::FLUSH_DATA_OUT;\n  }\n\n  size_t i = 0;\n  v_int32 res = Error::OK;\n\n  while(res == Error::OK) {\n\n    auto& p = m_processors[i];\n\n    data::buffer::InlineReadData* currDataIn = &dataIn;\n    if(i > 0) {\n      currDataIn = &m_intermediateData[i - 1];\n    }\n\n    data::buffer::InlineReadData* currDataOut = &dataOut;\n    if(i < m_intermediateData.size()) {\n      currDataOut = &m_intermediateData[i];\n    }\n\n    while(res == Error::OK) {\n      res = p->iterate(*currDataIn, *currDataOut);\n    }\n\n    const size_t numOfProcessors = m_processors.size();\n\n    switch (res) {\n      case Error::PROVIDE_DATA_IN:\n        if (i > 0) {\n          i --;\n          res = Error::OK;\n        }\n        break;\n\n\n      case Error::FLUSH_DATA_OUT:\n        if (i < numOfProcessors - 1) {\n          i ++;\n          res = Error::OK;\n        }\n        break;\n\n\n      case Error::FINISHED:\n        if (i < numOfProcessors - 1) {\n          i ++;\n          res = Error::OK;\n        }\n        break;\n\n      default:\n        break;\n    }\n\n  }\n\n  return res;\n\n}\n\n}}}\n\n// Path: src/oatpp/core/data/buffer/FIFOBuffer.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_buffer_FIFOBuffer_hpp\n#define oatpp_data_buffer_FIFOBuffer_hpp\n\n#include \"oatpp/core/data/stream/Stream.hpp\"\n#include \"oatpp/core/IODefinitions.hpp\"\n#include \"oatpp/core/async/Coroutine.hpp\"\n#include \"oatpp/core/concurrency/SpinLock.hpp\"\n\nnamespace oatpp { namespace data { namespace buffer {\n\n/**\n * FIFO operations over the buffer\n * !FIFOBuffer is NOT an IOStream despite having similar APIs!\n */\nclass FIFOBuffer {\nprivate:\n  p_char8 m_buffer;\n  v_buff_size m_bufferSize;\n  v_buff_size m_readPosition;\n  v_buff_size m_writePosition;\n  bool m_canRead;\npublic:\n\n  /**\n   * Constructor.\n   * @param buffer - pointer to buffer used for reads/writes.\n   * @param bufferSize - buffer size.\n   * @param readPosition - initial read position in buffer.\n   * @param writePosition - initial write position in buffer.\n   * @param canRead - flag to resolve ambiguity when readPosition == writePosition. If(readPosition == writePosition && canRead) then\n   * &l:FIFOBuffer::availableToRead (); returns buffer size, and &l:FIFOBuffer::availableToWrite (); returns 0.\n   */\n  FIFOBuffer(void* buffer, v_buff_size bufferSize,\n             v_buff_size readPosition = 0, v_buff_size writePosition = 0,\n             bool canRead = false);\n\n  /**\n   * Set read and write positions in buffer.\n   * @param readPosition - read position in buffer.\n   * @param writePosition - write position in buffer.\n   * @param canRead - flag to resolve ambiguity when readPosition == writePosition. If(readPosition == writePosition && canRead) then\n   * &l:FIFOBuffer::availableToRead (); returns buffer size, and &l:FIFOBuffer::availableToWrite (); returns 0.\n   */\n  void setBufferPosition(v_buff_size readPosition, v_buff_size writePosition, bool canRead);\n\n  /**\n   * Amount of bytes currently available to read from buffer.\n   * @return &id:oatpp::v_io_size;.\n   */\n  v_io_size availableToRead() const;\n\n  /**\n   * Amount of buffer space currently available for data writes.\n   * @return &id:oatpp::v_io_size;.\n   */\n  v_io_size availableToWrite() const;\n\n  /**\n   * Get FIFOBuffer size.\n   * @return - FIFOBuffer size.\n   */\n  v_buff_size getBufferSize() const;\n\n  /**\n   * read up to count bytes from the buffer to data\n   * @param data\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size read(void *data, v_buff_size count);\n\n  /**\n   * Peek up to count of bytes int he buffer\n   * @param data\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size peek(void *data, v_buff_size count);\n\n  /**\n   * Commit read offset\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size commitReadOffset(v_buff_size count);\n\n  /**\n   * write up to count bytes from data to buffer\n   * @param data\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size write(const void *data, v_buff_size count);\n\n  /**\n   * call read and then write bytes read to output stream\n   * @param stream\n   * @param count\n   * @param action\n   * @return [1..count], IOErrors.\n   */\n  v_io_size readAndWriteToStream(data::stream::WriteCallback* stream, v_buff_size count, async::Action& action);\n\n  /**\n   * call stream.read() and then write bytes read to buffer\n   * @param stream\n   * @param count\n   * @param action\n   * @return\n   */\n  v_io_size readFromStreamAndWrite(data::stream::ReadCallback* stream, v_buff_size count, async::Action& action);\n\n  /**\n   * flush all availableToRead bytes to stream\n   * @param stream\n   * @return\n   */\n  v_io_size flushToStream(data::stream::OutputStream* stream);\n\n  /**\n   * flush all availableToRead bytes to stream in asynchronous manner\n   * @param stream - &id:data::stream::OutputStream;.\n   * @return - &id:async::CoroutineStarter;.\n   */\n  async::CoroutineStarter flushToStreamAsync(const std::shared_ptr<data::stream::OutputStream>& stream);\n\n  \n};\n\n/**\n * Same as FIFOBuffer + synchronization with SpinLock\n */\nclass SynchronizedFIFOBuffer {\nprivate:\n  FIFOBuffer m_fifo;\n  oatpp::concurrency::SpinLock m_lock;\npublic:\n\n  /**\n   * Constructor.\n   * @param buffer - pointer to buffer used for reads/writes.\n   * @param bufferSize - buffer size.\n   * @param readPosition - initial read position in buffer.\n   * @param writePosition - initial write position in buffer.\n   * @param canRead - flag to resolve ambiguity when readPosition == writePosition. If(readPosition == writePosition && canRead) then\n   * &l:SynchronizedFIFOBuffer::availableToRead (); returns buffer size, and &l:SynchronizedFIFOBuffer::availableToWrite (); returns 0.\n   */\n  SynchronizedFIFOBuffer(void* buffer, v_buff_size bufferSize,\n                         v_buff_size readPosition = 0, v_buff_size writePosition = 0,\n                         bool canRead = false);\n\n  /**\n   * Set read and write positions in buffer.\n   * @param readPosition - read position in buffer.\n   * @param writePosition - write position in buffer.\n   * @param canRead - flag to resolve ambiguity when readPosition == writePosition. If(readPosition == writePosition && canRead) then\n   * &l:SynchronizedFIFOBuffer::availableToRead (); returns buffer size, and &l:SynchronizedFIFOBuffer::availableToWrite (); returns 0.\n   */\n  void setBufferPosition(v_buff_size readPosition, v_buff_size writePosition, bool canRead);\n\n  /**\n   * Amount of bytes currently available to read from buffer.\n   * @return &id:oatpp::v_io_size;.\n   */\n  v_io_size availableToRead();\n\n  /**\n   * Amount of buffer space currently available for data writes.\n   * @return &id:oatpp::v_io_size;.\n   */\n  v_io_size availableToWrite();\n\n  /**\n   * read up to count bytes from the buffer to data\n   * @param data\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size read(void *data, v_buff_size count);\n\n  /**\n   * write up to count bytes from data to buffer\n   * @param data\n   * @param count\n   * @return [1..count], IOErrors.\n   */\n  v_io_size write(const void *data, v_buff_size count);\n\n  /* No implementation of other methods */\n  /* User should implement his own synchronization for other methods */\n\n};\n  \n}}}\n\n#endif /* FIFOBuffer_hpp */\n\n// Path: src/oatpp/core/data/buffer/FIFOBuffer.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"FIFOBuffer.hpp\"\n#include <mutex>\n\nnamespace oatpp { namespace data{ namespace buffer {\n\nFIFOBuffer::FIFOBuffer(void* buffer, v_buff_size bufferSize,\n                       v_buff_size readPosition, v_buff_size writePosition,\n                       bool canRead)\n  : m_buffer(reinterpret_cast<p_char8>(buffer))\n  , m_bufferSize(bufferSize)\n  , m_readPosition(readPosition)\n  , m_writePosition(writePosition)\n  , m_canRead(canRead)\n{}\n\nvoid FIFOBuffer::setBufferPosition(v_buff_size readPosition, v_buff_size writePosition, bool canRead) {\n  m_readPosition = readPosition;\n  m_writePosition = writePosition;\n  m_canRead = canRead;\n}\n\nv_io_size FIFOBuffer::availableToRead() const {\n  if(!m_canRead) {\n    return 0;\n  }\n  if(m_readPosition < m_writePosition) {\n    return m_writePosition - m_readPosition;\n  }\n  return (m_bufferSize - m_readPosition + m_writePosition);\n}\n\nv_io_size FIFOBuffer::availableToWrite() const {\n  if(m_canRead && m_writePosition == m_readPosition) {\n    return 0;\n  }\n  if(m_writePosition < m_readPosition) {\n    return m_readPosition - m_writePosition;\n  }\n  return (m_bufferSize - m_writePosition + m_readPosition);\n}\n\nv_buff_size FIFOBuffer::getBufferSize() const {\n  return m_bufferSize;\n}\n\nv_io_size FIFOBuffer::read(void *data, v_buff_size count) {\n  \n  if(!m_canRead) {\n    return IOError::RETRY_READ;\n  }\n  \n  if(count == 0) {\n    return 0;\n  } else if(count < 0) {\n    throw std::runtime_error(\"[oatpp::data::buffer::FIFOBuffer::read(...)]: count < 0\");\n  }\n  \n  if(m_readPosition < m_writePosition) {\n    auto size = m_writePosition - m_readPosition;\n    if(size > count) {\n      size = count;\n    }\n    std::memcpy(data, &m_buffer[m_readPosition], static_cast<size_t>(size));\n    m_readPosition += size;\n    if(m_readPosition == m_writePosition) {\n      m_canRead = false;\n    }\n    return size;\n  }\n  \n  auto size = m_bufferSize - m_readPosition;\n  \n  if(size > count){\n    std::memcpy(data, &m_buffer[m_readPosition], static_cast<size_t>(count));\n    m_readPosition += count;\n    return count;\n  }\n  \n  std::memcpy(data, &m_buffer[m_readPosition], static_cast<size_t>(size));\n  auto size2 = m_writePosition;\n  if(size2 > count - size) {\n    size2 = count - size;\n  }\n  \n  std::memcpy(&(reinterpret_cast<p_char8>(data))[size], m_buffer, static_cast<size_t>(size2));\n  m_readPosition = size2;\n  if(m_readPosition == m_writePosition) {\n    m_canRead = false;\n  }\n  \n  return (size + size2);\n  \n}\n\nv_io_size FIFOBuffer::peek(void *data, v_buff_size count) {\n\n  if(!m_canRead) {\n    return IOError::RETRY_READ;\n  }\n\n  if(count == 0) {\n    return 0;\n  } else if(count < 0) {\n    throw std::runtime_error(\"[oatpp::data::buffer::FIFOBuffer::peek(...)]: count < 0\");\n  }\n\n  if(m_readPosition < m_writePosition) {\n    auto size = m_writePosition - m_readPosition;\n    if(size > count) {\n      size = count;\n    }\n    std::memcpy(data, &m_buffer[m_readPosition], static_cast<size_t>(size));\n    return size;\n  }\n\n  auto size = m_bufferSize - m_readPosition;\n\n  if(size > count){\n    std::memcpy(data, &m_buffer[m_readPosition], static_cast<size_t>(count));\n    return count;\n  }\n\n  std::memcpy(data, &m_buffer[m_readPosition], static_cast<size_t>(size));\n  auto size2 = m_writePosition;\n  if(size2 > count - size) {\n    size2 = count - size;\n  }\n\n  std::memcpy(&(reinterpret_cast<p_char8>(data))[size], m_buffer, static_cast<size_t>(size2));\n\n  return (size + size2);\n\n}\n\nv_io_size FIFOBuffer::commitReadOffset(v_buff_size count) {\n\n  if(!m_canRead) {\n    return IOError::RETRY_READ;\n  }\n\n  if(count == 0) {\n    return 0;\n  } else if(count < 0) {\n    throw std::runtime_error(\"[oatpp::data::buffer::FIFOBuffer::commitReadOffset(...)]: count < 0\");\n  }\n\n  if(m_readPosition < m_writePosition) {\n    auto size = m_writePosition - m_readPosition;\n    if(size > count) {\n      size = count;\n    }\n    m_readPosition += size;\n    if(m_readPosition == m_writePosition) {\n      m_canRead = false;\n    }\n    return size;\n  }\n\n  auto size = m_bufferSize - m_readPosition;\n\n  if(size > count){\n    m_readPosition += count;\n    return count;\n  }\n\n  auto size2 = m_writePosition;\n  if(size2 > count - size) {\n    size2 = count - size;\n  }\n\n  m_readPosition = size2;\n  if(m_readPosition == m_writePosition) {\n    m_canRead = false;\n  }\n\n  return (size + size2);\n\n}\n\nv_io_size FIFOBuffer::write(const void *data, v_buff_size count) {\n  \n  if(m_canRead && m_writePosition == m_readPosition) {\n    return IOError::RETRY_WRITE;\n  }\n  \n  if(count == 0) {\n    return 0;\n  } else if(count < 0) {\n    throw std::runtime_error(\"[oatpp::data::buffer::FIFOBuffer::write(...)]: count < 0\");\n  } else {\n    m_canRead = true;\n  }\n  \n  if(m_writePosition < m_readPosition) {\n    auto size = m_readPosition - m_writePosition;\n    if(size > count) {\n      size = count;\n    }\n    std::memcpy(&m_buffer[m_writePosition], data, static_cast<size_t>(size));\n    m_writePosition += size;\n    return size;\n  }\n  \n  auto size = m_bufferSize - m_writePosition;\n  \n  if(size > count){\n    std::memcpy(&m_buffer[m_writePosition], data, static_cast<size_t>(count));\n    m_writePosition += count;\n    return count;\n  }\n  \n  std::memcpy(&m_buffer[m_writePosition], data, static_cast<size_t>(size));\n  auto size2 = m_readPosition;\n  if(size2 > count - size) {\n    size2 = count - size;\n  }\n  \n  std::memcpy(m_buffer, &(reinterpret_cast<const char*>(data))[size], static_cast<size_t>(size2));\n  m_writePosition = size2;\n  \n  return (size + size2);\n  \n}\n\nv_io_size FIFOBuffer::readAndWriteToStream(data::stream::WriteCallback* stream, v_buff_size count, async::Action& action) {\n\n  if(!m_canRead) {\n    return IOError::RETRY_READ;\n  }\n\n  if(count == 0) {\n    return 0;\n  } else if(count < 0) {\n    throw std::runtime_error(\"[oatpp::data::buffer::FIFOBuffer::readAndWriteToStream(...)]: count < 0\");\n  }\n\n  if(m_readPosition < m_writePosition) {\n    auto size = m_writePosition - m_readPosition;\n    if(size > count) {\n      size = count;\n    }\n    auto bytesWritten = stream->write(&m_buffer[m_readPosition], size, action);\n    if(bytesWritten > 0) {\n      m_readPosition += bytesWritten;\n      if (m_readPosition == m_writePosition) {\n        m_canRead = false;\n      }\n    }\n    return bytesWritten;\n  }\n\n  auto size = m_bufferSize - m_readPosition;\n\n  /* DO NOT call stream.write() twice if size > count !!! */\n  if(size > count){\n    size = count;\n  } else if(size == 0) {\n\n    auto bytesWritten = stream->write(m_buffer, m_writePosition, action);\n    if(bytesWritten > 0) {\n      m_readPosition = bytesWritten;\n      if (m_readPosition == m_writePosition) {\n        m_canRead = false;\n      }\n    }\n    return bytesWritten;\n\n  }\n\n  auto bytesWritten = stream->write(&m_buffer[m_readPosition], size, action);\n  if(bytesWritten > 0) {\n    m_readPosition += bytesWritten;\n  }\n  return bytesWritten;\n\n}\n\nv_io_size FIFOBuffer::readFromStreamAndWrite(data::stream::ReadCallback* stream, v_buff_size count, async::Action& action) {\n\n  if(m_canRead && m_writePosition == m_readPosition) {\n    return IOError::RETRY_WRITE;\n  }\n\n  if(count == 0) {\n    return 0;\n  } else if(count < 0) {\n    throw std::runtime_error(\"[oatpp::data::buffer::FIFOBuffer::readFromStreamAndWrite(...)]: count < 0\");\n  }\n\n  if(m_writePosition < m_readPosition) {\n    auto size = m_readPosition - m_writePosition;\n    if(size > count) {\n      size = count;\n    }\n    auto bytesRead = stream->read(&m_buffer[m_writePosition], size, action);\n    if(bytesRead > 0) {\n      m_writePosition += bytesRead;\n      m_canRead = true;\n    }\n    return bytesRead;\n  }\n\n  auto size = m_bufferSize - m_writePosition;\n\n  /* DO NOT call stream.read() twice if size > count !!! */\n  if(size > count){\n    size = count;\n  } else if(size == 0) {\n\n    auto bytesRead = stream->read(m_buffer, m_readPosition, action);\n    if(bytesRead > 0) {\n      m_writePosition = bytesRead;\n      m_canRead = true;\n    }\n\n    return bytesRead;\n\n  }\n\n  auto bytesRead = stream->read(&m_buffer[m_writePosition], size, action);\n  if(bytesRead > 0) {\n    m_writePosition += bytesRead;\n    m_canRead = true;\n  }\n\n  return bytesRead;\n\n}\n\nv_io_size FIFOBuffer::flushToStream(data::stream::OutputStream* stream) {\n\n  if(!m_canRead) {\n    return 0;\n  }\n\n  v_io_size result = 0;\n\n  if(m_readPosition < m_writePosition) {\n    result = stream->writeExactSizeDataSimple(&m_buffer[m_readPosition], m_writePosition - m_readPosition);\n  } else {\n    result = stream->writeExactSizeDataSimple(&m_buffer[m_readPosition], m_bufferSize - m_readPosition);\n    result += stream->writeExactSizeDataSimple(m_buffer, m_writePosition);\n  }\n\n  setBufferPosition(0, 0, false);\n\n  return result;\n\n}\n\nasync::CoroutineStarter FIFOBuffer::flushToStreamAsync(const std::shared_ptr<data::stream::OutputStream>& stream)\n{\n\n  class FlushCoroutine : public oatpp::async::Coroutine<FlushCoroutine> {\n  private:\n    FIFOBuffer* m_fifo;\n    std::shared_ptr<data::stream::OutputStream> m_stream;\n  private:\n    data::buffer::InlineWriteData m_data1;\n    data::buffer::InlineWriteData m_data2;\n  public:\n\n    \nFlushCoroutine(FIFOBuffer* fifo, const std::shared_ptr<data::stream::OutputStream>& stream)\n      : m_fifo(fifo)\n      , m_stream(stream)\n    {}\n\n    Action act() override {\n\n      if(!m_fifo->m_canRead) {\n        return finish();\n      }\n\n      if(m_fifo->m_readPosition < m_fifo->m_writePosition) {\n\n        m_data1.set(&m_fifo->m_buffer[m_fifo->m_readPosition], m_fifo->m_writePosition - m_fifo->m_readPosition);\n        return yieldTo(&FlushCoroutine::fullFlush);\n\n      } else {\n\n        m_data1.set(&m_fifo->m_buffer[m_fifo->m_readPosition], m_fifo->m_bufferSize - m_fifo->m_readPosition);\n        m_data2.set(m_fifo->m_buffer, m_fifo->m_writePosition);\n        return yieldTo(&FlushCoroutine::partialFlush1);\n\n      }\n    }\n\n    Action fullFlush() {\n      return m_stream->writeExactSizeDataAsyncInline(m_data1, yieldTo(&FlushCoroutine::beforeFinish));\n    }\n\n    Action partialFlush1() {\n      return m_stream->writeExactSizeDataAsyncInline(m_data1, yieldTo(&FlushCoroutine::partialFlush2));\n    }\n\n    Action partialFlush2() {\n      return m_stream->writeExactSizeDataAsyncInline(m_data2, yieldTo(&FlushCoroutine::beforeFinish));\n    }\n\n    Action beforeFinish() {\n      m_fifo->setBufferPosition(0, 0, false);\n      return finish();\n    }\n\n  };\n\n  return FlushCoroutine::start(this, stream);\n\n}\n\n//////////////////////////////////////////////////////////////////////////////////////////\n// SynchronizedFIFOBuffer\n\nSynchronizedFIFOBuffer::SynchronizedFIFOBuffer(void* buffer, v_buff_size bufferSize,\n                                               v_buff_size readPosition, v_buff_size writePosition,\n                                               bool canRead)\n  : m_fifo(buffer, bufferSize, readPosition, writePosition, canRead)\n{}\n\nvoid SynchronizedFIFOBuffer::setBufferPosition(v_buff_size readPosition, v_buff_size writePosition, bool canRead) {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_lock);\n  m_fifo.setBufferPosition(readPosition, writePosition, canRead);\n}\n\nv_io_size SynchronizedFIFOBuffer::availableToRead() {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_lock);\n  return m_fifo.availableToRead();\n}\n\nv_io_size SynchronizedFIFOBuffer::availableToWrite() {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_lock);\n  return m_fifo.availableToWrite();\n}\n\nv_io_size SynchronizedFIFOBuffer::read(void *data, v_buff_size count) {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_lock);\n  return m_fifo.read(data, count);\n}\n\nv_io_size SynchronizedFIFOBuffer::write(const void *data, v_buff_size count) {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_lock);\n  return m_fifo.write(data, count);\n}\n\n}}}\n\n// Path: src/oatpp/core/data/buffer/IOBuffer.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_buffer_IOBuffer_hpp\n#define oatpp_data_buffer_IOBuffer_hpp\n\n#include \"oatpp/core/base/Countable.hpp\"\n\nnamespace oatpp { namespace data{ namespace buffer {\n\n/**\n * Predefined buffer implementation for I/O operations.\n * Allocates buffer bytes using &id:oatpp::base::memory::ThreadDistributedMemoryPool;.\n */\nclass IOBuffer : public oatpp::base::Countable {\npublic:\n  /**\n   * Buffer size constant.\n   */\n  static const v_buff_size BUFFER_SIZE;\nprivate:\n  p_char8 m_entry;\npublic:\n  /**\n   * Constructor.\n   */\n  IOBuffer();\npublic:\n\n  /**\n   * Create shared IOBuffer.\n   * @return\n   */\n  static std::shared_ptr<IOBuffer> createShared();\n\n  /**\n   * Virtual destructor.\n   */\n  ~IOBuffer() override;\n\n  /**\n   * Get pointer to buffer data.\n   * @return\n   */\n  void* getData();\n\n  /**\n   * Get buffer size.\n   * @return - should always return &l:IOBuffer::BUFFER_SIZE;.\n   */\n  v_buff_size getSize();\n  \n};\n  \n}}}\n\n#endif /* oatpp_data_buffer_IOBuffer_hpp */\n\n// Path: src/oatpp/core/data/buffer/IOBuffer.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"IOBuffer.hpp\"\n\nnamespace oatpp { namespace data { namespace buffer {\n\nconst v_buff_size IOBuffer::BUFFER_SIZE = 4096;\n\nIOBuffer::IOBuffer()\n  : m_entry(new v_char8[BUFFER_SIZE])\n{}\n\nstd::shared_ptr<IOBuffer> IOBuffer::createShared(){\n  return std::make_shared<IOBuffer>();\n}\n\nIOBuffer::~IOBuffer() {\n  delete [] m_entry;\n}\n\nvoid* IOBuffer::getData(){\n  return m_entry;\n}\n\nv_buff_size IOBuffer::getSize(){\n  return BUFFER_SIZE;\n}\n  \n}}}\n\n// Path: src/oatpp/core/data/mapping/type/Type.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_data_type_Type_hpp\n#define oatpp_data_type_Type_hpp\n\n#include \"oatpp/core/base/Countable.hpp\"\n#include \"oatpp/core/base/Environment.hpp\"\n\n#include <list>\n#include <unordered_map>\n#include <vector>\n#include <string>\n\nnamespace oatpp { namespace data { namespace mapping { namespace type {\n\nclass Type; // FWD\n\n/**\n * Structure representing `ID` of the type class.\n */\nclass ClassId {\nprivate:\n  static std::mutex& getClassMutex();\n  static std::vector<const char*>& getClassNames();\n  static v_int32 registerClassName(const char* name);\npublic:\n  /**\n   * Get count of all type classes created.\n   * @return\n   */\n  static int getClassCount();\n\n  /**\n   * Get registered class names.\n   * @return\n   */\n  static std::vector<const char*> getRegisteredClassNames();\npublic:\n\n  /**\n   * Constructor.\n   * @param pName\n   */\n  ClassId(const char* pName);\n\n  /**\n   * Name of the type class.\n   */\n  const char* const name;\n\n  /**\n   * Integer ID of the type class. <br>\n   * *Note: class type IDs are integer values incremented continuously from [0 to `getClassCount()`]*\n   */\n  const v_int32 id;\n\npublic:\n\n  inline bool operator == (const ClassId& other) const {\n    return id == other.id;\n  }\n\n  inline bool operator != (const ClassId& other) const {\n    return id != other.id;\n  }\n\n};\n\n\nnamespace __class {\n  /**\n   * Void Object Class.\n   */\n  class Void {\n  public:\n    /**\n     * Class id.\n     */\n    static const ClassId CLASS_ID;\n\n    /**\n     * Get class type information.\n     * @return - &l:Type;\n     */\n    static Type* getType();\n  };\n\n}\n\nclass Void; // FWD\n\n/**\n * ObjectWrapper holds std::shared_ptr to object, object static type, plus object dynamic type information.\n * @tparam T - Object Type.\n * @tparam Clazz - Static type info.\n */\ntemplate <class T, class Clazz = __class::Void>\nclass ObjectWrapper {\n  friend Void;\n  template <class Q, class W>\n  friend class ObjectWrapper;\nprotected:\n  static void checkType(const Type* _this, const Type* other);\nprotected:\n  std::shared_ptr<T> m_ptr;\n  const Type* m_valueType;\npublic:\n\n  /**\n   * Static object type\n   */\n  typedef T ObjectType;\n\n  /**\n   * Static object class information.\n   */\n  typedef Clazz Class;\npublic:\n\n  ObjectWrapper(const std::shared_ptr<T>& ptr)\n    : m_ptr(ptr)\n    , m_valueType(Class::getType())\n  {}\n\n  ObjectWrapper(const std::shared_ptr<T>& ptr, const Type* const type)\n    : m_ptr(ptr)\n    , m_valueType(type)\n  {}\n\n  ObjectWrapper(std::shared_ptr<T>&& ptr, const Type* const type)\n    : m_ptr(std::move(ptr))\n    , m_valueType(type)\n  {}\n  \npublic:\n\n  ObjectWrapper()\n    : m_valueType(Class::getType())\n  {}\n\n  ObjectWrapper(std::nullptr_t)\n    : m_valueType(Class::getType())\n  {}\n\n  ObjectWrapper(const Type* const type)\n    : m_valueType(type)\n  {}\n\n  ObjectWrapper(const ObjectWrapper& other)\n    : m_ptr(other.m_ptr)\n    , m_valueType(other.m_valueType)\n  {}\n\n  ObjectWrapper(ObjectWrapper&& other)\n    : m_ptr(std::move(other.m_ptr))\n    , m_valueType(other.m_valueType)\n  {}\n\n  template <class Q, class W>\n  ObjectWrapper(const ObjectWrapper<Q, W>& other)\n    : m_ptr(other.m_ptr)\n    , m_valueType(other.m_valueType)\n  {}\n\n  template <class Q, class W>\n  ObjectWrapper(ObjectWrapper<Q, W>&& other)\n    : m_ptr(std::move(other.m_ptr))\n    , m_valueType(other.m_valueType)\n  {}\n\n  inline ObjectWrapper& operator=(const ObjectWrapper& other){\n    checkType(m_valueType, other.m_valueType);\n    m_ptr = other.m_ptr;\n    return *this;\n  }\n\n  inline ObjectWrapper& operator=(ObjectWrapper&& other){\n    checkType(m_valueType, other.m_valueType);\n    m_ptr = std::move(other.m_ptr);\n    return *this;\n  }\n\n  template <class Q, class W>\n  inline ObjectWrapper& operator=(const ObjectWrapper<Q, W>& other){\n    checkType(m_valueType, other.m_valueType);\n    m_ptr = other.m_ptr;\n    return *this;\n  }\n\n  template <class Q, class W>\n  inline ObjectWrapper& operator=(ObjectWrapper<Q, W>&& other){\n    checkType(m_valueType, other.m_valueType);\n    m_ptr = std::move(other.m_ptr);\n    return *this;\n  }\n\n  template<class Wrapper>\n  Wrapper cast() const;\n\n  inline T* operator->() const {\n    return m_ptr.operator->();\n  }\n  \n  T* get() const {\n    return m_ptr.get();\n  }\n  \n  void resetPtr(const std::shared_ptr<T>& ptr = nullptr) {\n    m_ptr = ptr;\n  }\n  \n  std::shared_ptr<T> getPtr() const {\n    return m_ptr;\n  }\n\n  inline bool operator == (std::nullptr_t) const {\n    return m_ptr.get() == nullptr;\n  }\n\n  inline bool operator != (std::nullptr_t) const {\n    return m_ptr.get() != nullptr;\n  }\n...\n", "needle_token_start": 11876, "needle_token_end": 11927, "code_context_ntokens": 15839, "output": ["```cpp\nasync::CoroutineStarter FIFOBuffer::flushToStreamAsync(const std::shared_ptr<data::stream::OutputStream>& stream)\n{\n\n  class FlushCoroutine : public oatpp::async::Coroutine<FlushCoroutine> {\n  private:\n    FIFOBuffer* m_fifo;\n    std::shared_ptr<data::stream::OutputStream> m_stream;\n  private:\n    data::buffer::InlineWriteData m_data1;\n    data::buffer::InlineWriteData m_data2;\n  public:\n\n    \nFlushCoroutine(FIFOBuffer* fifo, const std::shared_ptr<data::stream::OutputStream>& stream)\n      : m_fifo(fifo)\n      , m_stream(stream)\n    {}\n\n    Action act() override {\n\n      if(!m_fifo->m_canRead) {\n        return finish();\n      }\n\n      if(m_fifo->m_readPosition < m_fifo->m_writePosition) {\n\n        m_data1.set(&m_fifo->m_buffer[m_fifo->m_readPosition], m_fifo->m_writePosition - m_fifo->m_readPosition);\n        return yieldTo(&FlushCoroutine::fullFlush);\n\n      } else {\n\n        m_data1.set(&m_fifo->m_buffer[m_fifo->m_readPosition], m_fifo->m_bufferSize - m_fifo->m_readPosition);\n        m_data2.set(m_fifo->m_buffer, m_fifo->m_writePosition);\n        return yieldTo(&FlushCoroutine::partialFlush1);\n\n      }\n    }\n\n    Action fullFlush() {\n      return m_stream->writeExactSizeDataAsyncInline(m_data1, yieldTo(&FlushCoroutine::beforeFinish));\n    }\n\n    Action partialFlush1() {\n      return m_stream->writeExactSizeDataAsyncInline(m_data1, yieldTo(&FlushCoroutine::partialFlush2));\n    }\n\n    Action partialFlush2() {\n      return m_stream->writeExactSizeDataAsyncInline(m_data2, yieldTo(&FlushCoroutine::beforeFinish));\n    }\n\n    Action beforeFinish() {\n      m_fifo->setBufferPosition(0, 0, false);\n      return finish();\n    }\n\n  };\n\n  return FlushCoroutine::start(this, stream);\n\n}\n```"]}
{"repo": "oatpp/oatpp", "name": "HttpRouterTemplate", "language": "cpp", "path": "src/oatpp/web/server/HttpRouter.hpp", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The primary function is to route HTTP requests to the appropriate handlers based on the HTTP method and the path pattern specified in the request.\n2. **Input**: The input includes an HTTP method and a path pattern.\n3. **Output**: The output is a handler that is associated with the given HTTP method and path pattern.\n4. **Procedure**: The procedure involves checking if a routing branch exists for the specified HTTP method. If it does not exist, a new branch is created. Then, the request is forwarded to the corresponding handler based on the path pattern within the specified method's branch.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/web/mime/multipart/Reader.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Reader.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// PartsParser\n\nPartsParser::PartsParser(Multipart* multipart)\n  : m_multipart(multipart)\n{}\n\nPartsParser::PartsParser(Multipart* multipart, const PartReadersMap& readersMap)\n  : m_readers(readersMap)\n  , m_multipart(multipart)\n{}\n\nvoid PartsParser::onPartHeaders(const Headers& partHeaders) {\n\n  m_currPart = std::make_shared<Part>(partHeaders);\n\n  if(m_currPart->getName()) {\n    auto it = m_readers.find(m_currPart->getName());\n    if(it != m_readers.end()) {\n      m_currReader = it->second;\n    } else {\n      m_currReader = m_defaultReader;\n    }\n  }\n\n  if(m_currReader) {\n    m_currReader->onNewPart(m_currPart);\n  }\n\n}\n\nvoid PartsParser::onPartData(const char* data, v_buff_size size) {\n  if(size > 0) {\n    if(m_currReader) {\n      m_currReader->onPartData(m_currPart, data, size);\n    }\n  } else {\n    if(m_currReader) {\n      m_currReader->onPartData(m_currPart, nullptr, 0);\n    }\n    m_multipart->writeNextPartSimple(m_currPart);\n  }\n}\n\nvoid PartsParser::setPartReader(const oatpp::String& partName, const std::shared_ptr<PartReader>& reader) {\n  m_readers[partName] = reader;\n}\n\nvoid PartsParser::setDefaultPartReader(const std::shared_ptr<PartReader>& reader) {\n  m_defaultReader = reader;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// AsyncPartsParser\n\nAsyncPartsParser::AsyncPartsParser(Multipart* multipart)\n  : m_multipart(multipart)\n{}\n\nAsyncPartsParser::AsyncPartsParser(Multipart* multipart, const AsyncPartReadersMap& readersMap)\n  : m_readers(readersMap)\n  , m_multipart(multipart)\n{}\n\nasync::CoroutineStarter AsyncPartsParser::onPartHeadersAsync(const Headers& partHeaders) {\n\n  m_currPart = std::make_shared<Part>(partHeaders);\n\n  if(m_currPart->getName()) {\n    auto it = m_readers.find(m_currPart->getName());\n    if(it != m_readers.end()) {\n      m_currReader = it->second;\n    } else {\n      m_currReader = m_defaultReader;\n    }\n  }\n\n  if(m_currReader) {\n    return m_currReader->onNewPartAsync(m_currPart);\n  }\n\n  return nullptr;\n\n}\n\nasync::CoroutineStarter AsyncPartsParser::onPartDone(const std::shared_ptr<Part>& part) {\n\n  class PutPartCoroutine : public async::Coroutine<PutPartCoroutine> {\n  private:\n    Multipart* m_multipart;\n    std::shared_ptr<AsyncPartReader> m_currReader;\n    std::shared_ptr<Part> m_part;\n  public:\n\n    PutPartCoroutine(Multipart* multipart,\n                     const std::shared_ptr<AsyncPartReader>& currReader,\n                     const std::shared_ptr<Part>& part)\n      : m_multipart(multipart)\n      , m_currReader(currReader)\n      , m_part(part)\n    {}\n\n    Action act() override {\n      return m_currReader->onPartDataAsync(m_part, nullptr, 0).next(yieldTo(&PutPartCoroutine::putPart));\n    }\n\n    Action putPart() {\n      async::Action action;\n      m_multipart->writeNextPart(m_part, action);\n      if(!action.isNone()) {\n        return action;\n      }\n      return finish();\n    }\n\n  };\n\n  return PutPartCoroutine::start(m_multipart, m_currReader, part);\n\n}\n\nasync::CoroutineStarter AsyncPartsParser::onPartDataAsync(const char* data, v_buff_size size) {\n  if(size > 0) {\n    if(m_currReader) {\n      return m_currReader->onPartDataAsync(m_currPart, data, size);\n...\n// Path: src/oatpp/web/mime/multipart/FileProvider.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_mime_multipart_FileProvider_hpp\n#define oatpp_web_mime_multipart_FileProvider_hpp\n\n#include \"PartReader.hpp\"\n#include \"Reader.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\nclass FileProvider : public PartReaderResourceProvider {\nprivate:\n  oatpp::String m_filename;\npublic:\n\n  FileProvider(const oatpp::String& filename);\n\n  std::shared_ptr<data::resource::Resource> getResource(const std::shared_ptr<Part>& part) override;\n\n  async::CoroutineStarter getResourceAsync(const std::shared_ptr<Part>& part,\n                                           std::shared_ptr<data::resource::Resource>& resource) override;\n\n};\n\n/**\n * Create file part reader. <br>\n * Reader will save part to a specified file.\n * @param filename - name of the file.\n * @param maxDataSize - max size of the received data. put `-1` for no-limit.\n * @return - `std::shared_ptr` to &id:oatpp::web::mime::multipart::PartReader;.\n */\nstd::shared_ptr<PartReader> createFilePartReader(const oatpp::String& filename, v_io_size maxDataSize = -1);\n\n/**\n * Create async file part reader. <br>\n * Reader will save part to a specified file.\n * @param filename - name of the file.\n * @param maxDataSize - max size of the received data. put `-1` for no-limit.\n * @return - `std::shared_ptr` to &id:oatpp::web::mime::multipart::AsyncPartReader;.\n */\nstd::shared_ptr<AsyncPartReader> createAsyncFilePartReader(const oatpp::String& filename, v_io_size maxDataSize = -1);\n\n}}}}\n\n#endif //oatpp_web_mime_multipart_FileProvider_hpp\n\n// Path: src/oatpp/web/mime/multipart/FileProvider.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"FileProvider.hpp\"\n\n#include \"oatpp/core/data/resource/File.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\nFileProvider::FileProvider(const oatpp::String& filename)\n  : m_filename(filename)\n{}\n\nstd::shared_ptr<data::resource::Resource> FileProvider::getResource(const std::shared_ptr<Part>& part) {\n  (void)part;\n  return std::make_shared<data::resource::File>(m_filename->c_str());\n}\n\nasync::CoroutineStarter FileProvider::getResourceAsync(const std::shared_ptr<Part>& part,\n                                                       std::shared_ptr<data::resource::Resource>& stream)\n{\n  (void)part;\n  stream = std::make_shared<data::resource::File>(m_filename->c_str());\n  return nullptr;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// Other functions\n\nstd::shared_ptr<PartReader> createFilePartReader(const oatpp::String& filename, v_io_size maxDataSize) {\n  auto provider = std::make_shared<FileProvider>(filename);\n  auto reader = std::make_shared<StreamPartReader>(provider, maxDataSize);\n  return reader;\n}\n\nstd::shared_ptr<AsyncPartReader> createAsyncFilePartReader(const oatpp::String& filename, v_io_size maxDataSize) {\n  auto provider = std::make_shared<FileProvider>(filename);\n  auto reader = std::make_shared<AsyncStreamPartReader>(provider, maxDataSize);\n  return reader;\n}\n\n}}}}\n\n// Path: src/oatpp/web/mime/multipart/TemporaryFileProvider.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_mime_multipart_TemporaryFileProvider_hpp\n#define oatpp_web_mime_multipart_TemporaryFileProvider_hpp\n\n#include \"PartReader.hpp\"\n#include \"Reader.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\nclass TemporaryFileProvider : public PartReaderResourceProvider {\nprivate:\n  oatpp::String m_tmpDirectory;\n  v_int32 m_randomWordSizeBytes;\npublic:\n\n  TemporaryFileProvider(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes = 8);\n\n  std::shared_ptr<data::resource::Resource> getResource(const std::shared_ptr<Part>& part) override;\n\n  async::CoroutineStarter getResourceAsync(const std::shared_ptr<Part>& part,\n                                           std::shared_ptr<data::resource::Resource>& resource) override;\n\n};\n\n/**\n * Create part reader to a temporary file.\n * @param tmpDirectory - directory for temporary files.\n * @param randomWordSizeBytes - number of random bytes to generate file name.\n * @param maxDataSize - max size of the received data. put `-1` for no-limit.\n * @return - `std::shared_ptr` to &id:oatpp::web::mime::multipart::PartReader;.\n */\nstd::shared_ptr<PartReader> createTemporaryFilePartReader(const oatpp::String& tmpDirectory,\n                                                          v_int32 randomWordSizeBytes = 8,\n                                                          v_io_size maxDataSize = -1);\n\n/**\n * Create async part reader to a temporary file.\n * @param tmpDirectory - directory for temporary files.\n * @param randomWordSizeBytes - number of random bytes to generate file name.\n * @param maxDataSize - max size of the received data. put `-1` for no-limit.\n * @return - `std::shared_ptr` to &id:oatpp::web::mime::multipart::AsyncPartReader;.\n */\nstd::shared_ptr<AsyncPartReader> createAsyncTemporaryFilePartReader(const oatpp::String& tmpDirectory,\n                                                                    v_int32 randomWordSizeBytes = 8,\n                                                                    v_io_size maxDataSize = -1);\n\n}}}}\n\n#endif //oatpp_web_mime_multipart_TemporaryFileProvider_hpp\n\n// Path: src/oatpp/web/mime/multipart/TemporaryFileProvider.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"TemporaryFileProvider.hpp\"\n\n#include \"oatpp/core/data/resource/TemporaryFile.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\nTemporaryFileProvider::TemporaryFileProvider(const oatpp::String& tmpDirectory, v_int32 randomWordSizeBytes)\n  : m_tmpDirectory(tmpDirectory)\n  , m_randomWordSizeBytes(randomWordSizeBytes)\n{}\n\nstd::shared_ptr<data::resource::Resource> TemporaryFileProvider::getResource(const std::shared_ptr<Part>& part) {\n  (void)part;\n  return std::make_shared<data::resource::TemporaryFile>(m_tmpDirectory, m_randomWordSizeBytes);\n}\n\nasync::CoroutineStarter TemporaryFileProvider::getResourceAsync(const std::shared_ptr<Part>& part,\n                                                                std::shared_ptr<data::resource::Resource>& stream)\n{\n  (void)part;\n  stream = std::make_shared<data::resource::TemporaryFile>(m_tmpDirectory, m_randomWordSizeBytes);\n  return nullptr;\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// Other functions\n\nstd::shared_ptr<PartReader> createTemporaryFilePartReader(const oatpp::String& tmpDirectory,\n                                                          v_int32 randomWordSizeBytes,\n                                                          v_io_size maxDataSize)\n{\n  auto provider = std::make_shared<TemporaryFileProvider>(tmpDirectory, randomWordSizeBytes);\n  auto reader = std::make_shared<StreamPartReader>(provider, maxDataSize);\n  return reader;\n}\n\nstd::shared_ptr<AsyncPartReader> createAsyncTemporaryFilePartReader(const oatpp::String& tmpDirectory,\n                                                                    v_int32 randomWordSizeBytes,\n                                                                    v_io_size maxDataSize)\n{\n  auto provider = std::make_shared<TemporaryFileProvider>(tmpDirectory, randomWordSizeBytes);\n  auto reader = std::make_shared<AsyncStreamPartReader>(provider, maxDataSize);\n  return reader;\n}\n\n}}}}\n\n// Path: src/oatpp/web/mime/multipart/Multipart.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"Multipart.hpp\"\n\n#include \"oatpp/web/protocol/http/Http.hpp\"\n#include \"oatpp/encoding/Base64.hpp\"\n#include \"oatpp/core/utils/Random.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\nMultipart::Multipart(const oatpp::String& boundary)\n  : m_boundary(boundary)\n{}\n\noatpp::String Multipart::getBoundary() {\n  return m_boundary;\n}\n\nstd::shared_ptr<Part> Multipart::readNextPartSimple() {\n  async::Action action;\n  auto result = readNextPart(action);\n  if(!action.isNone()) {\n    throw std::runtime_error(\"[oatpp::web::mime::multipart::Multipart::readNextPartSimple()]. Error.\"\n                             \"Async method is called for non-async API.\");\n  }\n  return result;\n}\n\nvoid Multipart::writeNextPartSimple(const std::shared_ptr<Part>& part) {\n  async::Action action;\n  writeNextPart(part, action);\n  if(!action.isNone()) {\n    throw std::runtime_error(\"[oatpp::web::mime::multipart::Multipart::writeNextPartSimple()]. Error.\"\n                             \"Async method is called for non-async API.\");\n  }\n}\n\noatpp::String Multipart::generateRandomBoundary(v_int32 boundarySize) {\n  std::unique_ptr<v_char8[]> buffer(new v_char8[boundarySize]);\n  utils::random::Random::randomBytes(buffer.get(), boundarySize);\n  return encoding::Base64::encode(buffer.get(), boundarySize, encoding::Base64::ALPHABET_BASE64_URL_SAFE);\n}\n\noatpp::String Multipart::parseBoundaryFromHeaders(const Headers& requestHeaders) {\n\n  oatpp::String boundary;\n  auto contentType = requestHeaders.getAsMemoryLabel<oatpp::data::share::StringKeyLabel>(\"Content-Type\");\n\n  if(contentType) {\n    oatpp::web::protocol::http::HeaderValueData valueData;\n    oatpp::web::protocol::http::Parser::parseHeaderValueData(valueData, contentType, ';');\n    boundary = valueData.getTitleParamValue(\"boundary\");\n  }\n\n  return boundary;\n\n}\n\n}}}}\n\n// Path: src/oatpp/web/mime/multipart/PartList.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>,\n * Matthias Haselmaier <mhaselmaier@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_mime_multipart_PartList_hpp\n#define oatpp_web_mime_multipart_PartList_hpp\n\n#include \"Multipart.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\n/**\n * Structure that holds Multipart parts in the `std::list`.\n */\nclass PartList : public Multipart {\nprivate:\n  std::unordered_map<oatpp::String, std::list<std::shared_ptr<Part>>> m_namedParts;\n  bool m_readIteratorInitialized;\n  std::list<std::shared_ptr<Part>> m_parts;\n  std::list<std::shared_ptr<Part>>::const_iterator m_iterator;\npublic:\n\n  /**\n   * Constructor.\n   * @param boundary - multipart boundary value.\n   */\n  PartList(const oatpp::String& boundary);\n\n  /**\n   * Constructor.\n   * @param requestHeaders - request headers. Headers must contain \"Content-Type\" header.\n   */\n  PartList(const Headers& requestHeaders);\n\n  /**\n   * Create Multipart object with random boundary. <br>\n   * It will generate random vector of size `boundarySize` in bytes encoded in base64.\n   * @param boundarySize - size of the random vecrot in bytes.\n   * @return - `std::shared_ptr` to Multipart.\n   */\n  static std::shared_ptr<PartList> createSharedWithRandomBoundary(v_int32 boundarySize = 15);\n\n  /**\n   * Read part-by-part from Multipart.\n   * @return\n   */\n  std::shared_ptr<Part> readNextPart(async::Action& action) override;\n\n  /**\n   * Write part-by-part to Multipart.\n   * @param part\n   */\n  void writeNextPart(const std::shared_ptr<Part>& part, async::Action& action) override;\n\n  /**\n   * Get part by name <br>\n   * Returns the first part if multiple parts with same name exist.\n   * Applicable to named parts only.\n   * @param name - &id:oatpp::String;.\n   * @return - `std::shared_ptr` to &id:oatpp::web::mime::multipart::Part;.\n   */\n  std::shared_ptr<Part> getNamedPart(const oatpp::String& name);\n\n  /**\n   * Get all parts by name <br>\n   * Applicable to named parts only.\n   * @param name - &id:oatpp::String;.\n   * @return - `std::list` of `std::shared_ptr` to &id:oatpp::web::mime::multipart::Part;.\n   */\n  std::list<std::shared_ptr<Part>> getNamedParts(const oatpp::String& name);\n\n  /**\n   * Get list of all parts.\n   * @return - `std::list` of `std::shared_ptr` to &id:oatpp::web::mime::multipart::Part;.\n   */\n  const std::list<std::shared_ptr<Part>>& getAllParts();\n\n  /**\n   * Get parts count.\n   * @return - parts count.\n   */\n  v_int64 count();\n\n};\n\n}}}}\n\n#endif //oatpp_web_mime_multipart_PartList_hpp\n\n// Path: src/oatpp/web/mime/multipart/PartList.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>,\n * Matthias Haselmaier <mhaselmaier@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"PartList.hpp\"\n\nnamespace oatpp { namespace web { namespace mime { namespace multipart {\n\nPartList::PartList(const oatpp::String& boundary)\n  : Multipart(boundary)\n  , m_readIteratorInitialized(false)\n{}\n\nPartList::PartList(const Headers& requestHeaders)\n  : Multipart(parseBoundaryFromHeaders(requestHeaders))\n  , m_readIteratorInitialized(false)\n{\n  if(!getBoundary()) {\n    throw std::runtime_error(\"[oatpp::web::mime::multipart::PartList::PartList]: Error. No 'boundary' value found in headers.\");\n  }\n}\n\nstd::shared_ptr<PartList> PartList::createSharedWithRandomBoundary(v_int32 boundarySize) {\n  auto boundary = generateRandomBoundary(boundarySize);\n  return std::make_shared<PartList>(boundary);\n}\n\nstd::shared_ptr<Part> PartList::readNextPart(async::Action& action) {\n  if(!m_readIteratorInitialized) {\n    m_readIteratorInitialized = true;\n    m_iterator = m_parts.begin();\n  }\n  if(m_iterator == m_parts.end()) {\n    return nullptr;\n  }\n  return *m_iterator ++;\n}\n\nvoid PartList::writeNextPart(const std::shared_ptr<Part>& part, async::Action& action) {\n\n  if(part->getName()) {\n    m_namedParts[part->getName()].push_back(part);\n  }\n\n  m_parts.push_back(part);\n\n}\n\nstd::shared_ptr<Part> PartList::getNamedPart(const oatpp::String& name) {\n\n  auto it = m_namedParts.find(name);\n  if(it != m_namedParts.end()) {\n    return it->second.front();\n  }\n\n  return nullptr;\n\n}\n\nstd::list<std::shared_ptr<Part>> PartList::getNamedParts(const oatpp::String& name) {\n\n  auto it = m_namedParts.find(name);\n  if(it != m_namedParts.end()) {\n    return it->second;\n  }\n\n  return std::list<std::shared_ptr<Part>>{};\n\n}\n\nconst std::list<std::shared_ptr<Part>>& PartList::getAllParts() {\n  return m_parts;\n}\n\nv_int64 PartList::count() {\n  return static_cast<v_int64>(m_parts.size());\n}\n\n}}}}\n\n// Path: src/oatpp/web/server/AsyncHttpConnectionHandler.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_AsyncHttpConnectionHandler_hpp\n#define oatpp_web_server_AsyncHttpConnectionHandler_hpp\n\n#include \"oatpp/web/server/HttpProcessor.hpp\"\n#include \"oatpp/network/ConnectionHandler.hpp\"\n#include \"oatpp/core/async/Executor.hpp\"\n#include \"oatpp/core/concurrency/SpinLock.hpp\"\n\n#include <unordered_map>\n\nnamespace oatpp { namespace web { namespace server {\n\n/**\n * Asynchronous &id:oatpp::network::ConnectionHandler; for handling http communication.\n */\nclass AsyncHttpConnectionHandler : public base::Countable, public network::ConnectionHandler, public HttpProcessor::TaskProcessingListener {\nprotected:\n\n  void onTaskStart(const provider::ResourceHandle<data::stream::IOStream>& connection) override;\n  void onTaskEnd(const provider::ResourceHandle<data::stream::IOStream>& connection) override;\n\n  void invalidateAllConnections();\n\nprivate:\n  std::shared_ptr<oatpp::async::Executor> m_executor;\n  std::shared_ptr<HttpProcessor::Components> m_components;\n  std::atomic_bool m_continue;\n  std::unordered_map<v_uint64, provider::ResourceHandle<data::stream::IOStream>> m_connections;\n  oatpp::concurrency::SpinLock m_connectionsLock;\npublic:\n\n  /**\n   * Constructor.\n   * @param components - &id:oatpp::web::server::HttpProcessor::Components;.\n   * @param threadCount - number of threads.\n   */\n  AsyncHttpConnectionHandler(const std::shared_ptr<HttpProcessor::Components>& components, v_int32 threadCount = oatpp::async::Executor::VALUE_SUGGESTED);\n\n  /**\n   * Constructor.\n   * @param components - &id:oatpp::web::server::HttpProcessor::Components;.\n   * @param executor - &id:oatpp::async::Executor;.\n   */\n  AsyncHttpConnectionHandler(const std::shared_ptr<HttpProcessor::Components>& components, const std::shared_ptr<oatpp::async::Executor>& executor);\n\n  /**\n   * Constructor.\n   * @param router - &id:oatpp::web::server::HttpRouter; to route incoming requests.\n   * @param threadCount - number of threads.\n   */\n  AsyncHttpConnectionHandler(const std::shared_ptr<HttpRouter>& router, v_int32 threadCount = oatpp::async::Executor::VALUE_SUGGESTED)\n    : AsyncHttpConnectionHandler(std::make_shared<HttpProcessor::Components>(router), threadCount)\n  {}\n\n  /**\n   * Constructor.\n   * @param router - &id:oatpp::web::server::HttpRouter; to route incoming requests.\n   * @param executor - &id:oatpp::async::Executor;.\n   */\n  AsyncHttpConnectionHandler(const std::shared_ptr<HttpRouter>& router, const std::shared_ptr<oatpp::async::Executor>& executor)\n    : AsyncHttpConnectionHandler(std::make_shared<HttpProcessor::Components>(router), executor)\n  {}\n\n  /**\n   * Constructor.\n   * @param router - &id:oatpp::web::server::HttpRouter; to route incoming requests.\n   * @param config - &id:oatpp::web::server::HttpProcessor::Config;.\n   * @param threadCount - number of threads.\n   */\n  AsyncHttpConnectionHandler(const std::shared_ptr<HttpRouter>& router,\n                             const std::shared_ptr<HttpProcessor::Config>& config,\n                             v_int32 threadCount = oatpp::async::Executor::VALUE_SUGGESTED)\n    : AsyncHttpConnectionHandler(std::make_shared<HttpProcessor::Components>(router, config), threadCount)\n  {}\n\n  /**\n   * Constructor.\n   * @param router - &id:oatpp::web::server::HttpRouter; to route incoming requests.\n   * @param config - &id:oatpp::web::server::HttpProcessor::Config;.\n   * @param executor - &id:oatpp::async::Executor;.\n   */\n  AsyncHttpConnectionHandler(const std::shared_ptr<HttpRouter>& router,\n                             const std::shared_ptr<HttpProcessor::Config>& config,\n                             const std::shared_ptr<oatpp::async::Executor>& executor)\n    : AsyncHttpConnectionHandler(std::make_shared<HttpProcessor::Components>(router, config), executor)\n  {}\n\npublic:\n  \n  static std::shared_ptr<AsyncHttpConnectionHandler> createShared(const std::shared_ptr<HttpRouter>& router,\n                                                                  v_int32 threadCount = oatpp::async::Executor::VALUE_SUGGESTED);\n  \n  static std::shared_ptr<AsyncHttpConnectionHandler> createShared(const std::shared_ptr<HttpRouter>& router,\n                                                                  const std::shared_ptr<oatpp::async::Executor>& executor);\n  \n  static std::shared_ptr<AsyncHttpConnectionHandler> createShared(const std::shared_ptr<HttpProcessor::Components>& components,\n                                                                  const std::shared_ptr<oatpp::async::Executor>& executor);\n\n  static std::shared_ptr<AsyncHttpConnectionHandler> createShared(const std::shared_ptr<HttpProcessor::Components>& components,\n                                                                  v_int32 threadCount = oatpp::async::Executor::VALUE_SUGGESTED);\n\n  void setErrorHandler(const std::shared_ptr<handler::ErrorHandler>& errorHandler);\n\n  /**\n   * Add request interceptor. Request interceptors are called before routing happens.\n   * If multiple interceptors set then the order of interception is the same as the order of calls to `addRequestInterceptor`.\n   * @param interceptor - &id:oatpp::web::server::interceptor::RequestInterceptor;.\n   */\n  void addRequestInterceptor(const std::shared_ptr<interceptor::RequestInterceptor>& interceptor);\n\n  /**\n   * Add response interceptor.\n   * If multiple interceptors set then the order of interception is the same as the order of calls to `addResponseInterceptor`.\n   * @param interceptor - &id:oatpp::web::server::interceptor::RequestInterceptor;.\n   */\n  void addResponseInterceptor(const std::shared_ptr<interceptor::ResponseInterceptor>& interceptor);\n\n  \n  void handleConnection(const provider::ResourceHandle<IOStream>& connection,\n                        const std::shared_ptr<const ParameterMap>& params) override;\n\n  /**\n   * Will call m_executor.stop()\n   */\n  void stop() override;\n\n  /**\n   * Get connections count.\n   * @return\n   */\n  v_uint64 getConnectionsCount();\n  \n};\n  \n}}}\n\n#endif /* oatpp_web_server_AsyncHttpConnectionHandler_hpp */\n\n\n// Path: src/oatpp/web/server/AsyncHttpConnectionHandler.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"./AsyncHttpConnectionHandler.hpp\"\n\nnamespace oatpp { namespace web { namespace server {\n\nvoid AsyncHttpConnectionHandler::onTaskStart(const provider::ResourceHandle<data::stream::IOStream>& connection) {\n\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  m_connections.insert({reinterpret_cast<v_uint64>(connection.object.get()), connection});\n\n  if(!m_continue.load()) {\n    connection.invalidator->invalidate(connection.object);\n  }\n\n}\n\nvoid AsyncHttpConnectionHandler::onTaskEnd(const provider::ResourceHandle<data::stream::IOStream>& connection) {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  m_connections.erase(reinterpret_cast<v_uint64>(connection.object.get()));\n}\n\nvoid AsyncHttpConnectionHandler::invalidateAllConnections() {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  for(auto& c : m_connections) {\n    const auto& handle = c.second;\n    handle.invalidator->invalidate(handle.object);\n  }\n}\n\nv_uint64 AsyncHttpConnectionHandler::getConnectionsCount() {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  return m_connections.size();\n}\n\nAsyncHttpConnectionHandler::AsyncHttpConnectionHandler(const std::shared_ptr<HttpProcessor::Components>& components,\n                                                       v_int32 threadCount)\n  : m_executor(std::make_shared<oatpp::async::Executor>(threadCount))\n  , m_components(components)\n  , m_continue(true)\n{\n  m_executor->detach();\n}\n\nAsyncHttpConnectionHandler::AsyncHttpConnectionHandler(const std::shared_ptr<HttpProcessor::Components>& components,\n                                                       const std::shared_ptr<oatpp::async::Executor>& executor)\n  : m_executor(executor)\n  , m_components(components)\n  , m_continue(true)\n{}\n\nstd::shared_ptr<AsyncHttpConnectionHandler> AsyncHttpConnectionHandler::createShared(const std::shared_ptr<HttpRouter>& router, v_int32 threadCount){\n  return std::make_shared<AsyncHttpConnectionHandler>(router, threadCount);\n}\n\nstd::shared_ptr<AsyncHttpConnectionHandler> AsyncHttpConnectionHandler::createShared(const std::shared_ptr<HttpRouter>& router, const std::shared_ptr<oatpp::async::Executor>& executor){\n  return std::make_shared<AsyncHttpConnectionHandler>(router, executor);\n}\n\nstd::shared_ptr<AsyncHttpConnectionHandler> AsyncHttpConnectionHandler::createShared(const std::shared_ptr<HttpProcessor::Components>& components, const std::shared_ptr<oatpp::async::Executor>& executor){\n  return std::make_shared<AsyncHttpConnectionHandler>(components, executor);\n}\n\nstd::shared_ptr<AsyncHttpConnectionHandler> AsyncHttpConnectionHandler::createShared(const std::shared_ptr<HttpProcessor::Components>& components, v_int32 threadCount){\n  return std::make_shared<AsyncHttpConnectionHandler>(components, threadCount);\n}\n\nvoid AsyncHttpConnectionHandler::setErrorHandler(const std::shared_ptr<handler::ErrorHandler>& errorHandler){\n  m_components->errorHandler = errorHandler;\n  if(!m_components->errorHandler) {\n    m_components->errorHandler = handler::DefaultErrorHandler::createShared();\n  }\n}\n\nvoid AsyncHttpConnectionHandler::addRequestInterceptor(const std::shared_ptr<interceptor::RequestInterceptor>& interceptor) {\n  m_components->requestInterceptors.push_back(interceptor);\n}\n\nvoid AsyncHttpConnectionHandler::addResponseInterceptor(const std::shared_ptr<interceptor::ResponseInterceptor>& interceptor) {\n  m_components->responseInterceptors.push_back(interceptor);\n}\n\nvoid AsyncHttpConnectionHandler::handleConnection(const provider::ResourceHandle<IOStream>& connection,\n                                                  const std::shared_ptr<const ParameterMap>& params)\n{\n\n  (void)params;\n\n  if (m_continue.load()) {\n\n    connection.object->setOutputStreamIOMode(oatpp::data::stream::IOMode::ASYNCHRONOUS);\n    connection.object->setInputStreamIOMode(oatpp::data::stream::IOMode::ASYNCHRONOUS);\n\n    m_executor->execute<HttpProcessor::Coroutine>(m_components, connection, this);\n\n  }\n  \n}\n\nvoid AsyncHttpConnectionHandler::stop() {\n  m_continue.store(false);\n\n  /* invalidate all connections */\n  invalidateAllConnections();\n\n  /* Wait until all connection-threads are done */\n  while(getConnectionsCount() > 0) {\n    std::this_thread::sleep_for(std::chrono::milliseconds(100));\n  }\n}\n  \n}}}\n\n// Path: src/oatpp/web/server/HttpRequestHandler.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_HttpRequestHandler_hpp\n#define oatpp_web_server_HttpRequestHandler_hpp\n\n#include \"oatpp/web/protocol/http/outgoing/ResponseFactory.hpp\"\n#include \"oatpp/web/protocol/http/outgoing/Response.hpp\"\n#include \"oatpp/web/protocol/http/incoming/Request.hpp\"\n\nnamespace oatpp { namespace web { namespace server {\n\n/**\n * HTTP request handler.\n */\nclass HttpRequestHandler {\npublic:\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Status;.\n   */\n  typedef oatpp::web::protocol::http::Status Status;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Header;.\n   */\n  typedef oatpp::web::protocol::http::Header Header;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Headers;.\n   */\n  typedef oatpp::web::protocol::http::Headers Headers;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::QueryParams;.\n   */\n  typedef oatpp::web::protocol::http::QueryParams QueryParams;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Request;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Request IncomingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::Response OutgoingResponse;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::ResponseFactory;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::ResponseFactory ResponseFactory;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::HttpError;.\n   */\n  typedef oatpp::web::protocol::http::HttpError HttpError;\n\npublic:\n\n  /**\n   * Handle incoming http request. <br>\n   * *Implement this method.*\n   * @param request - incoming http request. &id:oatpp::web::protocol::http::incoming::Request;.\n   * @return - outgoing http response. &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  virtual std::shared_ptr<OutgoingResponse> handle(const std::shared_ptr<IncomingRequest>& request) {\n    (void)request;\n    throw HttpError(Status::CODE_501, \"Endpoint not implemented.\");\n  }\n\n  /**\n   * Handle incoming http request in Asynchronous manner. <br>\n   * *Implement this method.*\n   * @param request - &id:oatpp::web::protocol::http::incoming::Request;.\n   * @return - &id:oatpp::async::CoroutineStarterForResult; of &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  virtual oatpp::async::CoroutineStarterForResult<const std::shared_ptr<OutgoingResponse>&>\n  handleAsync(const std::shared_ptr<IncomingRequest>& request) {\n    (void)request;\n    throw HttpError(Status::CODE_501, \"Asynchronous endpoint not implemented.\");\n  }\n\n  /**\n   * You have to provide a definition for destructors, otherwise its undefined behaviour.\n   */\n  virtual ~HttpRequestHandler() = default;\n};\n\n}}}\n\n#endif // oatpp_web_server_HttpRequestHandler_hpp\n\n// Path: src/oatpp/web/server/HttpRouter.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_HttpRouter_hpp\n#define oatpp_web_server_HttpRouter_hpp\n\n#include \"./HttpRequestHandler.hpp\"\n\n#include \"oatpp/web/server/api/ApiController.hpp\"\n#include \"oatpp/web/server/api/Endpoint.hpp\"\n#include \"oatpp/web/url/mapping/Router.hpp\"\n\nnamespace oatpp { namespace web { namespace server {\n\n/**\n * HttpRouter is responsible for routing http requests by method and path-pattern.\n */\ntemplate<typename RouterEndpoint>\nclass HttpRouterTemplate : public oatpp::base::Countable {\nprivate:\n  /**\n   * Convenience typedef for &id:oatpp::data::share::StringKeyLabel;.\n   */\n  typedef data::share::StringKeyLabel StringKeyLabel;\npublic:\n\n  /**\n   * &id:oatpp::web::url::mapping::Router;\n   */\n  typedef web::url::mapping::Router<RouterEndpoint> BranchRouter;\n\n  /**\n   * Http method to &l:HttpRouter::BranchRouter; map.\n   * Meaning that for each http method like [\"GET\", \"POST\", ...] there is a separate &l:HttpRouter::BranchRouter;.\n   */\n  typedef std::unordered_map<StringKeyLabel, std::shared_ptr<BranchRouter>> BranchMap;\nprotected:\n  BranchMap m_branchMap;\nprotected:\n\n  const std::shared_ptr<BranchRouter>& getBranch(const StringKeyLabel& name){\n    auto it = m_branchMap.find(name);\n    if(it == m_branchMap.end()){\n      m_branchMap[name] = BranchRouter::createShared();\n    }\n    return m_branchMap[name];\n  }\n\npublic:\n\n  /**\n   * Default Constructor.\n   */\n  \nHttpRouterTemplate() = default;\n\n  /**\n   * Create shared HttpRouter.\n   * @return - `std::shared_ptr` to HttpRouter.\n   */\n  static std::shared_ptr<HttpRouterTemplate> createShared() {\n    return std::make_shared<HttpRouterTemplate>();\n  }\n\n  /**\n   * Route URL to Endpoint by method, and pathPattern.\n   * @param method - http method like [\"GET\", \"POST\", etc.].\n   * @param pathPattern - url path pattern. ex.: `\"/path/to/resource/with/{param1}/{param2}\"`.\n   * @param endpoint - router endpoint.\n   */\n  void route(const oatpp::String& method, const oatpp::String& pathPattern, const RouterEndpoint& endpoint) {\n    getBranch(method)->route(pathPattern, endpoint);\n  }\n\n  /**\n   * Resolve http method and path to &id:oatpp::web::url::mapping::Router::Route;\n   * @param method - http method like [\"GET\", \"POST\", etc.].\n   * @param url - url path. \"Path\" part of url only.\n   * @return - &id:oatpp::web::url::mapping::Router::Route;.\n   */\n  typename BranchRouter::Route getRoute(const StringKeyLabel& method, const StringKeyLabel& path){\n    auto it = m_branchMap.find(method);\n    if(it != m_branchMap.end()) {\n      return m_branchMap[method]->getRoute(path);\n    }\n    return typename BranchRouter::Route();\n  }\n\n  /**\n   * Print out all router mapping.\n   */\n  void logRouterMappings() {\n    for(auto it : m_branchMap) {\n      it.second->logRouterMappings(it.first);\n    }\n  }\n  \n};\n\n/**\n * Default HttpRouter.\n */\nclass HttpRouter : public HttpRouterTemplate<std::shared_ptr<HttpRequestHandler>> {\nprivate:\n  std::list<std::shared_ptr<server::api::ApiController>> m_controllers;\npublic:\n\n  /**\n   * Create shared HttpRouter.\n   * @return\n   */\n  static std::shared_ptr<HttpRouter> createShared();\n\n  using HttpRouterTemplate::route;\n  void route(const std::shared_ptr<server::api::Endpoint>& endpoint);\n  void route(const server::api::Endpoints& endpoints);\n\n  /**\n   * Add controller and route its' endpoints.\n   * @param controller\n   * @return - `std::shared_ptr` to the controller added.\n   */\n  std::shared_ptr<server::api::ApiController> addController(const std::shared_ptr<server::api::ApiController>& controller);\n\n};\n  \n}}}\n\n#endif /* oatpp_web_server_HttpRouter_hpp */\n\n// Path: src/oatpp/web/server/HttpRouter.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"HttpRouter.hpp\"\n\nnamespace oatpp { namespace web { namespace server {\n\nstd::shared_ptr<HttpRouter> HttpRouter::createShared() {\n  return std::make_shared<HttpRouter>();\n}\n\nvoid HttpRouter::route(const std::shared_ptr<server::api::Endpoint>& endpoint) {\n  route(endpoint->info()->method, endpoint->info()->path, endpoint->handler);\n}\n\nvoid HttpRouter::route(const server::api::Endpoints& endpoints) {\n  for(auto& e : endpoints.list) {\n    route(e);\n  }\n}\n\nstd::shared_ptr<server::api::ApiController> HttpRouter::addController(const std::shared_ptr<server::api::ApiController>& controller) {\n  m_controllers.push_back(controller);\n  route(controller->getEndpoints());\n  return controller;\n}\n\n}}}\n\n// Path: src/oatpp/web/server/interceptor/RequestInterceptor.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_interceptor_RequestInterceptor_hpp\n#define oatpp_web_server_interceptor_RequestInterceptor_hpp\n\n#include \"oatpp/web/protocol/http/outgoing/Response.hpp\"\n#include \"oatpp/web/protocol/http/incoming/Request.hpp\"\n#include \"oatpp/web/protocol/http/Http.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace interceptor {\n\n/**\n * RequestInterceptor.\n */\nclass RequestInterceptor {\npublic:\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Request;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Request IncomingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::Response OutgoingResponse;\npublic:\n  \n  /**\n   * Default virtual destructor.\n   */\n  virtual ~RequestInterceptor() = default;\n  \n  /**\n   *\n   *  This method should not do any \"heavy\" nor I/O operations\n   *  as it is used for both \"Simple\" and \"Async\" API\n   *  NOT FOR I/O operations!!!\n   *\n   *  - return nullptr to continue.\n   *  - return OutgoingResponse to send response immediately\n   *\n   *  possible usage ex: return 301 - redirect if needed\n   *\n   */\n  virtual std::shared_ptr<OutgoingResponse> intercept(const std::shared_ptr<IncomingRequest>& request) = 0;\n  \n};\n  \n}}}}\n\n#endif /* oatpp_web_server_interceptor_RequestInterceptor_hpp */\n\n// Path: src/oatpp/web/server/interceptor/ResponseInterceptor.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_interceptor_ResponseInterceptor_hpp\n#define oatpp_web_server_interceptor_ResponseInterceptor_hpp\n\n#include \"oatpp/web/protocol/http/incoming/Request.hpp\"\n...\n", "needle_token_start": 13398, "needle_token_end": 13405, "code_context_ntokens": 15751, "output": ["```cpp\nvoid HttpRouterTemplate::route(const oatpp::String& method, const oatpp::String& pathPattern, const RouterEndpoint& endpoint) {\n  getBranch(method)->route(pathPattern, endpoint);\n}\n```"]}
{"repo": "oatpp/oatpp", "name": "AuthorizationObject", "language": "cpp", "path": "src/oatpp/web/server/handler/AuthorizationHandler.hpp", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To serve as a base class for creating objects that manage and validate user access permissions within a web server environment.\n2. **Input**: Typically receives HTTP request data indirectly through methods implemented in subclasses.\n3. **Output**: Does not directly produce output but provides a framework for subclasses to generate authorization decisions, which may affect the HTTP response.\n4. **Procedure**: Acts as a foundational structure that other specific authorization classes extend. It initializes essential properties and functionalities needed for handling authorization but does not implement specific authorization logic itself.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/oatpp/web/server/interceptor/RequestInterceptor.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_interceptor_RequestInterceptor_hpp\n#define oatpp_web_server_interceptor_RequestInterceptor_hpp\n\n#include \"oatpp/web/protocol/http/outgoing/Response.hpp\"\n#include \"oatpp/web/protocol/http/incoming/Request.hpp\"\n#include \"oatpp/web/protocol/http/Http.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace interceptor {\n\n/**\n * RequestInterceptor.\n */\nclass RequestInterceptor {\npublic:\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Request;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Request IncomingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n...\n// Path: src/oatpp/web/server/interceptor/ResponseInterceptor.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_interceptor_ResponseInterceptor_hpp\n#define oatpp_web_server_interceptor_ResponseInterceptor_hpp\n\n#include \"oatpp/web/protocol/http/incoming/Request.hpp\"\n#include \"oatpp/web/protocol/http/outgoing/Response.hpp\"\n#include \"oatpp/web/protocol/http/Http.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace interceptor {\n\n/**\n * ResponseInterceptor.\n */\nclass ResponseInterceptor {\npublic:\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::incoming::Request;.\n   */\n  typedef oatpp::web::protocol::http::incoming::Request IncomingRequest;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  typedef oatpp::web::protocol::http::outgoing::Response OutgoingResponse;\npublic:\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~ResponseInterceptor() = default;\n\n  /**\n   *\n   * This method should not do any \"heavy\" nor I/O operations <br>\n   * as it is used for both \"Simple\" and \"Async\" API <br>\n   * NOT FOR I/O operations!!! <br>\n   * <br>\n   * - return the same response, or the new one. <br>\n   * - do **NOT** return `nullptr`.\n   * <br><br>\n   * possible usage ex: add extra headers to the response.\n   *\n   * @param request - the corresponding request.\n   * @param response - response to the request\n   * @return - &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  virtual std::shared_ptr<OutgoingResponse> intercept(const std::shared_ptr<IncomingRequest>& request,\n                                                      const std::shared_ptr<OutgoingResponse>& response) = 0;\n\n};\n\n}}}}\n\n#endif /* oatpp_web_server_interceptor_ResponseInterceptor_hpp */\n\n// Path: src/oatpp/web/server/handler/ErrorHandler.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_handler_ErrorHandler_hpp\n#define oatpp_web_server_handler_ErrorHandler_hpp\n\n#include \"oatpp/web/protocol/http/outgoing/Response.hpp\"\n#include \"oatpp/web/protocol/http/Http.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace handler {\n\n/**\n * Error Handler.\n */\nclass ErrorHandler {\npublic:\n  /**\n   * Convenience typedef for Headers. <br>\n   * See &id:oatpp::web::protocol::http::Headers;\n   */\n  typedef web::protocol::http::Headers Headers;\npublic:\n  /**\n   * Virtual destructor since the class is meant to be derived from.\n   * */\n  virtual ~ErrorHandler() = default;\n\n  /**\n   * Implement this method!\n   * @param error - &std::exception;.\n   * @return - std::shared_ptr to &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  virtual std::shared_ptr<protocol::http::outgoing::Response> handleError(const std::exception_ptr& exceptionPtr);\n\n  /**\n   * Implement this method!\n   * @param status - &id:oatpp::web::protocol::http::Status;.\n   * @param message - &id:oatpp::String;.\n   * @param Headers - &id:oatpp::web::protocol::http::Headers;\n   * @return - std::shared_ptr to &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  [[deprecated]]\n  virtual\n  std::shared_ptr<protocol::http::outgoing::Response>\n  handleError(const protocol::http::Status& status, const oatpp::String& message, const Headers& headers) = 0;\n\n  /**\n   * Convenience method to call `handleError` method with no headers.\n   * @param status - &id:oatpp::web::protocol::http::Status;\n   * @param message - &id:oatpp::String;.\n   * @return - std::shared_ptr to &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  [[deprecated]]\n  std::shared_ptr<protocol::http::outgoing::Response> handleError(const protocol::http::Status& status, const oatpp::String& message);\n  \n};\n\n/**\n * Default Error Handler.\n */\nclass DefaultErrorHandler : public oatpp::base::Countable, public ErrorHandler {\npublic:\n  /**\n   * Constructor.\n   */\n  DefaultErrorHandler() = default;\npublic:\n\n  /**\n   * Create shared DefaultErrorHandler.\n   * @return - `std::shared_ptr` to DefaultErrorHandler.\n   */\n  static std::shared_ptr<DefaultErrorHandler> createShared() {\n    return std::make_shared<DefaultErrorHandler>();\n  }\n\n  std::shared_ptr<protocol::http::outgoing::Response> handleError(const std::exception_ptr& error) override;\n\n  /**\n   * Implementation of &l:ErrorHandler::handleError ();\n   * @param status - &id:oatpp::web::protocol::http::Status;.\n   * @param message - &id:oatpp::String;.\n   * @return - &id:oatpp::web::protocol::http::outgoing::Response;.\n   */\n  std::shared_ptr<protocol::http::outgoing::Response>\n  handleError(const protocol::http::Status& status, const oatpp::String& message, const Headers& headers) override;\n\n};\n  \n}}}}\n\n#endif /* oatpp_web_server_handler_ErrorHandler_hpp */\n\n// Path: src/oatpp/web/server/HttpProcessor.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_HttpProcessor_hpp\n#define oatpp_web_server_HttpProcessor_hpp\n\n#include \"./HttpRouter.hpp\"\n\n#include \"./interceptor/RequestInterceptor.hpp\"\n#include \"./interceptor/ResponseInterceptor.hpp\"\n#include \"./handler/ErrorHandler.hpp\"\n\n#include \"oatpp/web/protocol/http/encoding/ProviderCollection.hpp\"\n\n#include \"oatpp/web/protocol/http/incoming/RequestHeadersReader.hpp\"\n#include \"oatpp/web/protocol/http/incoming/Request.hpp\"\n\n#include \"oatpp/web/protocol/http/outgoing/Response.hpp\"\n#include \"oatpp/web/protocol/http/utils/CommunicationUtils.hpp\"\n\n#include \"oatpp/core/data/stream/StreamBufferedProxy.hpp\"\n#include \"oatpp/core/async/Processor.hpp\"\n\nnamespace oatpp { namespace web { namespace server {\n\n/**\n * HttpProcessor. Helper class to handle HTTP processing.\n */\nclass HttpProcessor {\npublic:\n  typedef std::list<std::shared_ptr<web::server::interceptor::RequestInterceptor>> RequestInterceptors;\n  typedef std::list<std::shared_ptr<web::server::interceptor::ResponseInterceptor>> ResponseInterceptors;\n  typedef web::protocol::http::incoming::RequestHeadersReader RequestHeadersReader;\n  typedef protocol::http::utils::CommunicationUtils::ConnectionState ConnectionState;\n\n  HttpProcessor() = default;\n\npublic:\n\n  /**\n   * Resource config per connection.\n   */\n  struct Config {\n\n    /**\n     * Buffer used to read headers in request. Initial size of the buffer.\n     */\n    v_buff_size headersInBufferInitial = 2048;\n\n    /**\n     * Buffer used to write headers in response. Initial size of the buffer.\n     */\n    v_buff_size headersOutBufferInitial = 2048;\n\n    /**\n     * Size of the chunk used for iterative-read of headers.\n     */\n    v_buff_size headersReaderChunkSize = 2048;\n\n    /**\n     * Maximum allowed size of requests headers. The overall size of all headers in the request.\n     */\n    v_buff_size headersReaderMaxSize = 4096;\n\n  };\n\npublic:\n\n  /**\n   * Collection of components needed to serve http-connection.\n   */\n  struct Components {\n\n    /**\n     * Constructor.\n     * @param pRouter\n     * @param pContentEncodingProviders\n     * @param pBodyDecoder\n     * @param pErrorHandler\n     * @param pRequestInterceptors\n     * @param pConfig\n     */\n    Components(const std::shared_ptr<HttpRouter>& pRouter,\n               const std::shared_ptr<protocol::http::encoding::ProviderCollection>& pContentEncodingProviders,\n               const std::shared_ptr<const oatpp::web::protocol::http::incoming::BodyDecoder>& pBodyDecoder,\n               const std::shared_ptr<handler::ErrorHandler>& pErrorHandler,\n               const RequestInterceptors& pRequestInterceptors,\n               const ResponseInterceptors& pResponseInterceptors,\n               const std::shared_ptr<Config>& pConfig);\n\n    /**\n     * Constructor.\n     * @param pRouter\n     */\n    Components(const std::shared_ptr<HttpRouter>& pRouter);\n\n    /**\n     * Constructor.\n     * @param pRouter\n     * @param pConfig\n     */\n    Components(const std::shared_ptr<HttpRouter>& pRouter, const std::shared_ptr<Config>& pConfig);\n\n    /**\n     * Router to route incoming requests. &id:oatpp::web::server::HttpRouter;.\n     */\n    std::shared_ptr<HttpRouter> router;\n\n    /**\n     * Content-encoding providers. &id:oatpp::web::protocol::encoding::ProviderCollection;.\n     */\n    std::shared_ptr<protocol::http::encoding::ProviderCollection> contentEncodingProviders;\n\n    /**\n     * Body decoder. &id:oatpp::web::protocol::http::incoming::BodyDecoder;.\n     */\n    std::shared_ptr<const oatpp::web::protocol::http::incoming::BodyDecoder> bodyDecoder;\n\n    /**\n     * Error handler. &id:oatpp::web::server::handler::ErrorHandler;.\n     */\n    std::shared_ptr<handler::ErrorHandler> errorHandler;\n\n    /**\n     * Collection of request interceptors. &id:oatpp::web::server::interceptor::RequestInterceptor;.\n     */\n    RequestInterceptors requestInterceptors;\n\n    /**\n     * Collection of request interceptors. &id:oatpp::web::server::interceptor::ResponseInterceptor;.\n     */\n    ResponseInterceptors responseInterceptors;\n\n    /**\n     * Resource allocation config. &l:HttpProcessor::Config;.\n     */\n    std::shared_ptr<Config> config;\n\n  };\n\nprivate:\n\n  struct ProcessingResources {\n\n    ProcessingResources(const std::shared_ptr<Components>& pComponents,\n                        const provider::ResourceHandle<oatpp::data::stream::IOStream>& pConnection);\n\n    std::shared_ptr<Components> components;\n    provider::ResourceHandle<oatpp::data::stream::IOStream> connection;\n    oatpp::data::stream::BufferOutputStream headersInBuffer;\n    oatpp::data::stream::BufferOutputStream headersOutBuffer;\n    RequestHeadersReader headersReader;\n    std::shared_ptr<oatpp::data::stream::InputStreamBufferedProxy> inStream;\n\n  };\n\n  static\n  std::shared_ptr<protocol::http::outgoing::Response>\n  processNextRequest(ProcessingResources& resources,\n                     const std::shared_ptr<protocol::http::incoming::Request>& request,\n                     ConnectionState& connectionState);\n  static ConnectionState processNextRequest(ProcessingResources& resources);\n\npublic:\n\n  /**\n   * Listener of the connection processing task.\n   */\n  class TaskProcessingListener {\n  public:\n    virtual void onTaskStart(const provider::ResourceHandle<data::stream::IOStream>& connection) = 0;\n    virtual void onTaskEnd(const provider::ResourceHandle<data::stream::IOStream>& connection) = 0;\n  };\n\npublic:\n\n  /**\n   * Connection serving task. <br>\n   * Usege example: <br>\n   * `std::thread thread(&HttpProcessor::Task::run, HttpProcessor::Task(components, connection));`\n   */\n  class Task : public base::Countable {\n  private:\n    std::shared_ptr<Components> m_components;\n    provider::ResourceHandle<oatpp::data::stream::IOStream> m_connection;\n    TaskProcessingListener* m_taskListener;\n  public:\n\n    /**\n     * Constructor.\n     * @param components - &l:HttpProcessor::Components;.\n     * @param connection - &id:oatpp::data::stream::IOStream;.\n     */\n    Task(const std::shared_ptr<Components>& components,\n         const provider::ResourceHandle<oatpp::data::stream::IOStream>& connection,\n         TaskProcessingListener* taskListener);\n\n    Task(const Task&) = delete;\n    Task &operator=(const Task&) = delete;\n\n    /**\n     * Move-Constructor to correclty count tasks;\n     */\n     Task(Task &&other);\n\n     /**\n      * Move-Assignment to correctly count tasks.\n      * @param t\n      * @return\n      */\n    Task &operator=(Task &&other);\n\n    /**\n     * Destructor, needed for counting.\n     */\n    ~Task() override;\n\n  public:\n\n    /**\n     * Run loop.\n     */\n    void run();\n\n  };\n  \npublic:\n\n  /**\n   * Connection serving coroutiner - &id:oatpp::async::Coroutine;.\n   */\n  class Coroutine : public oatpp::async::Coroutine<HttpProcessor::Coroutine> {\n  private:\n    std::shared_ptr<Components> m_components;\n    provider::ResourceHandle<oatpp::data::stream::IOStream> m_connection;\n    oatpp::data::stream::BufferOutputStream m_headersInBuffer;\n    RequestHeadersReader m_headersReader;\n    std::shared_ptr<oatpp::data::stream::BufferOutputStream> m_headersOutBuffer;\n    std::shared_ptr<oatpp::data::stream::InputStreamBufferedProxy> m_inStream;\n    ConnectionState m_connectionState;\n  private:\n    oatpp::web::server::HttpRouter::BranchRouter::Route m_currentRoute;\n    std::shared_ptr<protocol::http::incoming::Request> m_currentRequest;\n    std::shared_ptr<protocol::http::outgoing::Response> m_currentResponse;\n    TaskProcessingListener* m_taskListener;\n  public:\n\n    /**\n     * Constructor.\n     * @param components - &l:HttpProcessor::Components;.\n     * @param connection - &id:oatpp::data::stream::IOStream;.\n     */\n    Coroutine(const std::shared_ptr<Components>& components,\n              const provider::ResourceHandle<oatpp::data::stream::IOStream>& connection,\n              TaskProcessingListener* taskListener);\n\n    ~Coroutine() override;\n\n    Action act() override;\n\n    Action parseHeaders();\n    \n    Action onHeadersParsed(const RequestHeadersReader::Result& headersReadResult);\n    \n    Action onRequestFormed();\n    Action onResponse(const std::shared_ptr<protocol::http::outgoing::Response>& response);\n    Action onResponseFormed();\n    Action onRequestDone();\n    \n    Action handleError(Error* error) override;\n    \n  };\n  \n};\n  \n}}}\n\n#endif /* oatpp_web_server_HttpProcessor_hpp */\n\n// Path: src/oatpp/web/server/HttpProcessor.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"HttpProcessor.hpp\"\n\n#include \"oatpp/web/protocol/http/incoming/SimpleBodyDecoder.hpp\"\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n\nnamespace oatpp { namespace web { namespace server {\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// Components\n\nHttpProcessor::Components::Components(const std::shared_ptr<HttpRouter>& pRouter,\n                                      const std::shared_ptr<protocol::http::encoding::ProviderCollection>& pContentEncodingProviders,\n                                      const std::shared_ptr<const oatpp::web::protocol::http::incoming::BodyDecoder>& pBodyDecoder,\n                                      const std::shared_ptr<handler::ErrorHandler>& pErrorHandler,\n                                      const RequestInterceptors& pRequestInterceptors,\n                                      const ResponseInterceptors& pResponseInterceptors,\n                                      const std::shared_ptr<Config>& pConfig)\n  : router(pRouter)\n  , contentEncodingProviders(pContentEncodingProviders)\n  , bodyDecoder(pBodyDecoder)\n  , errorHandler(pErrorHandler)\n  , requestInterceptors(pRequestInterceptors)\n  , responseInterceptors(pResponseInterceptors)\n  , config(pConfig)\n{}\n\nHttpProcessor::Components::Components(const std::shared_ptr<HttpRouter>& pRouter)\n  : Components(pRouter,\n               nullptr,\n               std::make_shared<oatpp::web::protocol::http::incoming::SimpleBodyDecoder>(),\n               handler::DefaultErrorHandler::createShared(),\n               {},\n               {},\n               std::make_shared<Config>())\n{}\n\nHttpProcessor::Components::Components(const std::shared_ptr<HttpRouter>& pRouter, const std::shared_ptr<Config>& pConfig)\n  : Components(pRouter,\n               nullptr,\n               std::make_shared<oatpp::web::protocol::http::incoming::SimpleBodyDecoder>(),\n               handler::DefaultErrorHandler::createShared(),\n               {},\n               {},\n               pConfig)\n{}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// Other\n\nHttpProcessor::ProcessingResources::ProcessingResources(const std::shared_ptr<Components>& pComponents,\n                                                        const provider::ResourceHandle<oatpp::data::stream::IOStream>& pConnection)\n  : components(pComponents)\n  , connection(pConnection)\n  , headersInBuffer(components->config->headersInBufferInitial)\n  , headersOutBuffer(components->config->headersOutBufferInitial)\n  , headersReader(&headersInBuffer, components->config->headersReaderChunkSize, components->config->headersReaderMaxSize)\n  , inStream(data::stream::InputStreamBufferedProxy::createShared(connection.object, std::make_shared<std::string>(data::buffer::IOBuffer::BUFFER_SIZE, 0)))\n{}\n\nstd::shared_ptr<protocol::http::outgoing::Response>\nHttpProcessor::processNextRequest(ProcessingResources& resources,\n                                  const std::shared_ptr<protocol::http::incoming::Request>& request,\n                                  ConnectionState& connectionState)\n{\n\n  std::shared_ptr<protocol::http::outgoing::Response> response;\n\n  try{\n\n    for(auto& interceptor : resources.components->requestInterceptors) {\n      response = interceptor->intercept(request);\n      if(response) {\n        return response;\n      }\n    }\n\n    auto route = resources.components->router->getRoute(request->getStartingLine().method, request->getStartingLine().path);\n\n    if(!route) {\n\n      data::stream::BufferOutputStream ss;\n      ss << \"No mapping for HTTP-method: '\" << request->getStartingLine().method.toString()\n      << \"', URL: '\" << request->getStartingLine().path.toString() << \"'\";\n\n      connectionState = ConnectionState::CLOSING;\n      oatpp::web::protocol::http::HttpError error(protocol::http::Status::CODE_404, ss.toString());\n      auto ptr = std::make_exception_ptr(error);\n      return resources.components->errorHandler->handleError(ptr);\n\n    }\n\n    request->setPathVariables(route.getMatchMap());\n    return route.getEndpoint()->handle(request);\n\n  } catch (...) {\n    response = resources.components->errorHandler->handleError(std::current_exception());\n    connectionState = ConnectionState::CLOSING;\n  }\n\n  return response;\n\n}\n\nHttpProcessor::ConnectionState HttpProcessor::processNextRequest(ProcessingResources& resources) {\n\n  oatpp::web::protocol::http::HttpError::Info error;\n  auto headersReadResult = resources.headersReader.readHeaders(resources.inStream.get(), error);\n\n  if(error.ioStatus <= 0) {\n    return ConnectionState::DEAD;\n  }\n\n  ConnectionState connectionState = ConnectionState::ALIVE;\n  std::shared_ptr<protocol::http::incoming::Request> request;\n  std::shared_ptr<protocol::http::outgoing::Response> response;\n\n  if(error.status.code != 0) {\n    oatpp::web::protocol::http::HttpError httpError(error.status, \"Invalid Request Headers\");\n    auto eptr = std::make_exception_ptr(httpError);\n    response = resources.components->errorHandler->handleError(eptr);\n    connectionState = ConnectionState::CLOSING;\n  } else {\n\n    request = protocol::http::incoming::Request::createShared(resources.connection.object,\n                                                              headersReadResult.startingLine,\n                                                              headersReadResult.headers,\n                                                              resources.inStream,\n                                                              resources.components->bodyDecoder);\n\n    response = processNextRequest(resources, request, connectionState);\n\n    try {\n\n      for (auto& interceptor : resources.components->responseInterceptors) {\n        response = interceptor->intercept(request, response);\n        if (!response) {\n          oatpp::web::protocol::http::HttpError httpError(protocol::http::Status::CODE_500, \"Response Interceptor returned an Invalid Response - 'null'\");\n          auto eptr = std::make_exception_ptr(httpError);\n          response = resources.components->errorHandler->handleError(eptr);\n          connectionState = ConnectionState::CLOSING;\n        }\n      }\n\n    } catch (...) {\n      response = resources.components->errorHandler->handleError(std::current_exception());\n      connectionState = ConnectionState::CLOSING;\n    }\n\n    response->putHeaderIfNotExists(protocol::http::Header::SERVER, protocol::http::Header::Value::SERVER);\n    protocol::http::utils::CommunicationUtils::considerConnectionState(request, response, connectionState);\n\n    switch(connectionState) {\n\n      case ConnectionState::ALIVE :\n        response->putHeaderIfNotExists(protocol::http::Header::CONNECTION, protocol::http::Header::Value::CONNECTION_KEEP_ALIVE);\n        break;\n\n      case ConnectionState::CLOSING:\n      case ConnectionState::DEAD:\n        response->putHeaderIfNotExists(protocol::http::Header::CONNECTION, protocol::http::Header::Value::CONNECTION_CLOSE);\n        break;\n\n      case ConnectionState::DELEGATED:\n      default:\n        break;\n\n    }\n\n  }\n\n  auto contentEncoderProvider =\n    protocol::http::utils::CommunicationUtils::selectEncoder(request, resources.components->contentEncodingProviders);\n\n  response->send(resources.connection.object.get(), &resources.headersOutBuffer, contentEncoderProvider.get());\n\n  /* Delegate connection handling to another handler only after the response is sent to the client */\n  if(connectionState == ConnectionState::DELEGATED) {\n    auto handler = response->getConnectionUpgradeHandler();\n    if(handler) {\n      handler->handleConnection(resources.connection, response->getConnectionUpgradeParameters());\n      connectionState = ConnectionState::DELEGATED;\n    } else {\n      OATPP_LOGW(\"[oatpp::web::server::HttpProcessor::processNextRequest()]\", \"Warning. ConnectionUpgradeHandler not set!\")\n      connectionState = ConnectionState::CLOSING;\n    }\n  }\n\n  return connectionState;\n\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// Task\n\nHttpProcessor::Task::Task(const std::shared_ptr<Components>& components,\n                          const provider::ResourceHandle<oatpp::data::stream::IOStream>& connection,\n                          TaskProcessingListener* taskListener)\n  : m_components(components)\n  , m_connection(connection)\n  , m_taskListener(taskListener)\n{\n  m_taskListener->onTaskStart(m_connection);\n}\n\nHttpProcessor::Task::Task(HttpProcessor::Task &&other)\n  : m_components(std::move(other.m_components))\n  , m_connection(std::move(other.m_connection))\n  , m_taskListener(other.m_taskListener)\n{\n  other.m_taskListener = nullptr;\n}\n\nHttpProcessor::Task::~Task() {\n  if (m_taskListener != nullptr) {\n    m_taskListener->onTaskEnd(m_connection);\n  }\n}\n\nHttpProcessor::Task &HttpProcessor::Task::operator=(HttpProcessor::Task &&other) {\n  m_components = std::move(other.m_components);\n  m_connection = std::move(other.m_connection);\n  m_taskListener = other.m_taskListener;\n  other.m_taskListener = nullptr;\n  return *this;\n}\n\nvoid HttpProcessor::Task::run(){\n\n  m_connection.object->initContexts();\n\n  ProcessingResources resources(m_components, m_connection);\n\n  ConnectionState connectionState;\n\n  try {\n\n    do {\n\n      connectionState = HttpProcessor::processNextRequest(resources);\n\n    } while (connectionState == ConnectionState::ALIVE);\n\n  } catch (...) {\n    // DO NOTHING\n  }\n\n}\n\n////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n// HttpProcessor::Coroutine\n\nHttpProcessor::Coroutine::Coroutine(const std::shared_ptr<Components>& components,\n                                    const provider::ResourceHandle<oatpp::data::stream::IOStream>& connection,\n                                    TaskProcessingListener* taskListener)\n  : m_components(components)\n  , m_connection(connection)\n  , m_headersInBuffer(components->config->headersInBufferInitial)\n  , m_headersReader(&m_headersInBuffer, components->config->headersReaderChunkSize, components->config->headersReaderMaxSize)\n  , m_headersOutBuffer(std::make_shared<oatpp::data::stream::BufferOutputStream>(components->config->headersOutBufferInitial))\n  , m_inStream(data::stream::InputStreamBufferedProxy::createShared(m_connection.object, std::make_shared<std::string>(data::buffer::IOBuffer::BUFFER_SIZE, 0)))\n  , m_connectionState(ConnectionState::ALIVE)\n  , m_taskListener(taskListener)\n{\n  m_taskListener->onTaskStart(m_connection);\n}\n\nHttpProcessor::Coroutine::~Coroutine() {\n  m_taskListener->onTaskEnd(m_connection);\n}\n\nHttpProcessor::Coroutine::Action HttpProcessor::Coroutine::act() {\n  return m_connection.object->initContextsAsync().next(yieldTo(&HttpProcessor::Coroutine::parseHeaders));\n}\n\nHttpProcessor::Coroutine::Action HttpProcessor::Coroutine::parseHeaders() {\n  return m_headersReader.readHeadersAsync(m_inStream).callbackTo(&HttpProcessor::Coroutine::onHeadersParsed);\n}\n\noatpp::async::Action HttpProcessor::Coroutine::onHeadersParsed(const RequestHeadersReader::Result& headersReadResult) {\n\n  m_currentRequest = protocol::http::incoming::Request::createShared(m_connection.object,\n                                                                     headersReadResult.startingLine,\n                                                                     headersReadResult.headers,\n                                                                     m_inStream,\n                                                                     m_components->bodyDecoder);\n\n  for(auto& interceptor : m_components->requestInterceptors) {\n    m_currentResponse = interceptor->intercept(m_currentRequest);\n    if(m_currentResponse) {\n      return yieldTo(&HttpProcessor::Coroutine::onResponseFormed);\n    }\n  }\n\n  m_currentRoute = m_components->router->getRoute(headersReadResult.startingLine.method.toString(), headersReadResult.startingLine.path.toString());\n\n  if(!m_currentRoute) {\n\n    data::stream::BufferOutputStream ss;\n    ss << \"No mapping for HTTP-method: '\" << headersReadResult.startingLine.method.toString()\n       << \"', URL: '\" << headersReadResult.startingLine.path.toString() << \"'\";\n    oatpp::web::protocol::http::HttpError error(protocol::http::Status::CODE_404, ss.toString());\n    auto eptr = std::make_exception_ptr(error);\n    m_currentResponse = m_components->errorHandler->handleError(eptr);\n    m_connectionState = ConnectionState::CLOSING;\n    return yieldTo(&HttpProcessor::Coroutine::onResponseFormed);\n  }\n\n  m_currentRequest->setPathVariables(m_currentRoute.getMatchMap());\n\n  return yieldTo(&HttpProcessor::Coroutine::onRequestFormed);\n\n}\n\nHttpProcessor::Coroutine::Action HttpProcessor::Coroutine::onRequestFormed() {\n  return m_currentRoute.getEndpoint()->handleAsync(m_currentRequest).callbackTo(&HttpProcessor::Coroutine::onResponse);\n}\n\nHttpProcessor::Coroutine::Action HttpProcessor::Coroutine::onResponse(const std::shared_ptr<protocol::http::outgoing::Response>& response) {\n  m_currentResponse = response;\n  return yieldTo(&HttpProcessor::Coroutine::onResponseFormed);\n}\n  \nHttpProcessor::Coroutine::Action HttpProcessor::Coroutine::onResponseFormed() {\n\n  for(auto& interceptor : m_components->responseInterceptors) {\n    m_currentResponse = interceptor->intercept(m_currentRequest, m_currentResponse);\n    if(!m_currentResponse) {\n      oatpp::web::protocol::http::HttpError error(protocol::http::Status::CODE_500, \"Response Interceptor returned an Invalid Response - 'null'\");\n      auto eptr = std::make_exception_ptr(error);\n      m_currentResponse = m_components->errorHandler->handleError(eptr);\n    }\n  }\n\n  m_currentResponse->putHeaderIfNotExists(protocol::http::Header::SERVER, protocol::http::Header::Value::SERVER);\n  oatpp::web::protocol::http::utils::CommunicationUtils::considerConnectionState(m_currentRequest, m_currentResponse, m_connectionState);\n\n  switch(m_connectionState) {\n\n    case ConnectionState::ALIVE :\n      m_currentResponse->putHeaderIfNotExists(protocol::http::Header::CONNECTION, protocol::http::Header::Value::CONNECTION_KEEP_ALIVE);\n      break;\n\n    case ConnectionState::CLOSING:\n    case ConnectionState::DEAD:\n      m_currentResponse->putHeaderIfNotExists(protocol::http::Header::CONNECTION, protocol::http::Header::Value::CONNECTION_CLOSE);\n      break;\n\n    case ConnectionState::DELEGATED:\n    default:\n      break;\n\n  }\n\n  auto contentEncoderProvider =\n    protocol::http::utils::CommunicationUtils::selectEncoder(m_currentRequest, m_components->contentEncodingProviders);\n\n  return protocol::http::outgoing::Response::sendAsync(m_currentResponse, m_connection.object, m_headersOutBuffer, contentEncoderProvider)\n         .next(yieldTo(&HttpProcessor::Coroutine::onRequestDone));\n\n}\n  \nHttpProcessor::Coroutine::Action HttpProcessor::Coroutine::onRequestDone() {\n\n  switch (m_connectionState) {\n    case ConnectionState::ALIVE:\n      return yieldTo(&HttpProcessor::Coroutine::parseHeaders);\n\n    /* Delegate connection handling to another handler only after the response is sent to the client */\n    case ConnectionState::DELEGATED: {\n      auto handler = m_currentResponse->getConnectionUpgradeHandler();\n      if(handler) {\n        handler->handleConnection(m_connection, m_currentResponse->getConnectionUpgradeParameters());\n        m_connectionState = ConnectionState::DELEGATED;\n      } else {\n        OATPP_LOGW(\"[oatpp::web::server::HttpProcessor::Coroutine::onResponseFormed()]\", \"Warning. ConnectionUpgradeHandler not set!\")\n        m_connectionState = ConnectionState::CLOSING;\n      }\n      break;\n    }\n\n    case ConnectionState::CLOSING:\n    case ConnectionState::DEAD:\n    default:\n      break;\n\n  }\n  \n  return finish();\n\n}\n  \nHttpProcessor::Coroutine::Action HttpProcessor::Coroutine::handleError(Error* error) {\n\n  if(error) {\n\n    if(error->is<oatpp::AsyncIOError>()) {\n      auto aioe = static_cast<oatpp::AsyncIOError*>(error);\n      if(aioe->getCode() == oatpp::IOError::BROKEN_PIPE) {\n        return aioe; // do not report BROKEN_PIPE error\n      }\n    }\n\n    if(m_currentResponse) {\n      //OATPP_LOGE(\"[oatpp::web::server::HttpProcessor::Coroutine::handleError()]\", \"Unhandled error. '%s'. Dropping connection\", error->what())\n      return error;\n    }\n\n    oatpp::web::protocol::http::HttpError httpError(protocol::http::Status::CODE_500, error->what());\n    auto eptr = std::make_exception_ptr(httpError);\n    m_currentResponse = m_components->errorHandler->handleError(eptr);\n    return yieldTo(&HttpProcessor::Coroutine::onResponseFormed);\n\n  }\n\n  return error;\n\n}\n  \n}}}\n\n// Path: src/oatpp/web/server/HttpConnectionHandler.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_HttpConnectionHandler_hpp\n#define oatpp_web_server_HttpConnectionHandler_hpp\n\n#include \"oatpp/web/server/HttpProcessor.hpp\"\n#include \"oatpp/network/ConnectionHandler.hpp\"\n#include \"oatpp/core/concurrency/SpinLock.hpp\"\n\n#include <unordered_map>\n\nnamespace oatpp { namespace web { namespace server {\n\n/**\n * Simple ConnectionHandler (&id:oatpp::network::ConnectionHandler;) for handling HTTP communication. <br>\n * Will create one thread per each connection to handle communication.\n */\nclass HttpConnectionHandler : public base::Countable, public network::ConnectionHandler, public HttpProcessor::TaskProcessingListener {\nprotected:\n\n  void onTaskStart(const provider::ResourceHandle<data::stream::IOStream>& connection) override;\n  void onTaskEnd(const provider::ResourceHandle<data::stream::IOStream>& connection) override;\n\n  void invalidateAllConnections();\n\nprivate:\n  std::shared_ptr<HttpProcessor::Components> m_components;\n  std::atomic_bool m_continue;\n  std::unordered_map<v_uint64, provider::ResourceHandle<data::stream::IOStream>> m_connections;\n  oatpp::concurrency::SpinLock m_connectionsLock;\npublic:\n\n  /**\n   * Constructor.\n   * @param components - &id:oatpp::web::server::HttpProcessor::Components;.\n   */\n  HttpConnectionHandler(const std::shared_ptr<HttpProcessor::Components>& components);\n\n  /**\n   * Constructor.\n   * @param router - &id:oatpp::web::server::HttpRouter; to route incoming requests.\n   */\n  HttpConnectionHandler(const std::shared_ptr<HttpRouter>& router)\n    : HttpConnectionHandler(std::make_shared<HttpProcessor::Components>(router))\n  {}\n\n  /**\n   * Constructor.\n   * @param router - &id:oatpp::web::server::HttpRouter; to route incoming requests.\n   * @param config - &id:oatpp::web::server::HttpProcessor::Config;.\n   */\n  HttpConnectionHandler(const std::shared_ptr<HttpRouter>& router,\n                        const std::shared_ptr<HttpProcessor::Config>& config)\n    : HttpConnectionHandler(std::make_shared<HttpProcessor::Components>(router, config))\n  {}\n\npublic:\n\n  /**\n   * Create shared HttpConnectionHandler.\n   * @param router - &id:oatpp::web::server::HttpRouter; to route incoming requests.\n   * @return - `std::shared_ptr` to HttpConnectionHandler.\n   */\n  static std::shared_ptr<HttpConnectionHandler> createShared(const std::shared_ptr<HttpRouter>& router);\n\n  /**\n   * Set root error handler for all requests coming through this Connection Handler.\n   * All unhandled errors will be handled by this error handler.\n   * @param errorHandler - &id:oatpp::web::server::handler::ErrorHandler;.\n   */\n  void setErrorHandler(const std::shared_ptr<handler::ErrorHandler>& errorHandler);\n\n  /**\n   * Add request interceptor. Request interceptors are called before routing happens.\n   * If multiple interceptors set then the order of interception is the same as the order of calls to `addRequestInterceptor`.\n   * @param interceptor - &id:oatpp::web::server::interceptor::RequestInterceptor;.\n   */\n  void addRequestInterceptor(const std::shared_ptr<interceptor::RequestInterceptor>& interceptor);\n\n  /**\n   * Add response interceptor.\n   * If multiple interceptors set then the order of interception is the same as the order of calls to `addResponseInterceptor`.\n   * @param interceptor - &id:oatpp::web::server::interceptor::RequestInterceptor;.\n   */\n  void addResponseInterceptor(const std::shared_ptr<interceptor::ResponseInterceptor>& interceptor);\n\n  /**\n   * Implementation of &id:oatpp::network::ConnectionHandler::handleConnection;.\n   * @param connection - &id:oatpp::data::stream::IOStream; representing connection.\n   */\n  void handleConnection(const provider::ResourceHandle<IOStream>& connection,\n                        const std::shared_ptr<const ParameterMap>& params) override;\n\n  /**\n   * Tell all worker threads to exit when done.\n   */\n  void stop() override;\n\n  /**\n   * Get connections count.\n   * @return\n   */\n  v_uint64 getConnectionsCount();\n  \n};\n  \n}}}\n\n#endif /* oatpp_web_server_HttpConnectionHandler_hpp */\n\n// Path: src/oatpp/web/server/HttpConnectionHandler.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"./HttpConnectionHandler.hpp\"\n\n#include \"oatpp/web/protocol/http/incoming/Request.hpp\"\n#include \"oatpp/web/protocol/http/Http.hpp\"\n\n#include \"oatpp/core/concurrency/Thread.hpp\"\n\n#include \"oatpp/core/data/buffer/IOBuffer.hpp\"\n\n#include \"oatpp/core/data/stream/BufferStream.hpp\"\n#include \"oatpp/core/data/stream/StreamBufferedProxy.hpp\"\n\n\nnamespace oatpp { namespace web { namespace server {\n\nvoid HttpConnectionHandler::onTaskStart(const provider::ResourceHandle<data::stream::IOStream>& connection) {\n\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  m_connections.insert({reinterpret_cast<v_uint64>(connection.object.get()), connection});\n\n  if(!m_continue.load()) {\n    connection.invalidator->invalidate(connection.object);\n  }\n\n}\n\nvoid HttpConnectionHandler::onTaskEnd(const provider::ResourceHandle<data::stream::IOStream>& connection) {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  m_connections.erase(reinterpret_cast<v_uint64>(connection.object.get()));\n}\n\nvoid HttpConnectionHandler::invalidateAllConnections() {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  for(auto& c : m_connections) {\n    const auto& handle = c.second;\n    handle.invalidator->invalidate(handle.object);\n  }\n}\n\nv_uint64 HttpConnectionHandler::getConnectionsCount() {\n  std::lock_guard<oatpp::concurrency::SpinLock> lock(m_connectionsLock);\n  return m_connections.size();\n}\n\nHttpConnectionHandler::HttpConnectionHandler(const std::shared_ptr<HttpProcessor::Components>& components)\n  : m_components(components)\n  , m_continue(true)\n{}\n\nstd::shared_ptr<HttpConnectionHandler> HttpConnectionHandler::createShared(const std::shared_ptr<HttpRouter>& router){\n  return std::make_shared<HttpConnectionHandler>(router);\n}\n\nvoid HttpConnectionHandler::setErrorHandler(const std::shared_ptr<handler::ErrorHandler>& errorHandler){\n  m_components->errorHandler = errorHandler;\n  if(!m_components->errorHandler) {\n    m_components->errorHandler = handler::DefaultErrorHandler::createShared();\n  }\n}\n\nvoid HttpConnectionHandler::addRequestInterceptor(const std::shared_ptr<interceptor::RequestInterceptor>& interceptor) {\n  m_components->requestInterceptors.push_back(interceptor);\n}\n\nvoid HttpConnectionHandler::addResponseInterceptor(const std::shared_ptr<interceptor::ResponseInterceptor>& interceptor) {\n  m_components->responseInterceptors.push_back(interceptor);\n}\n  \nvoid HttpConnectionHandler::handleConnection(const provider::ResourceHandle<data::stream::IOStream>& connection,\n                                             const std::shared_ptr<const ParameterMap>& params)\n{\n\n  (void)params;\n\n  if (m_continue.load()) {\n\n    connection.object->setOutputStreamIOMode(oatpp::data::stream::IOMode::BLOCKING);\n    connection.object->setInputStreamIOMode(oatpp::data::stream::IOMode::BLOCKING);\n\n    /* Create working thread */\n    std::thread thread(&HttpProcessor::Task::run, std::move(HttpProcessor::Task(m_components, connection, this)));\n\n    /* Get hardware concurrency -1 in order to have 1cpu free of workers. */\n    v_int32 concurrency = oatpp::concurrency::getHardwareConcurrency();\n    if (concurrency > 1) {\n      concurrency -= 1;\n    }\n\n    /* Set thread affinity group CPUs [0..cpu_count - 1]. Leave one cpu free of workers */\n    oatpp::concurrency::setThreadAffinityToCpuRange(thread.native_handle(),\n                                                    0,\n                                                    concurrency - 1 /* -1 because 0-based index */);\n\n    thread.detach();\n  }\n\n}\n\nvoid HttpConnectionHandler::stop() {\n  m_continue.store(false);\n\n  /* invalidate all connections */\n  invalidateAllConnections();\n\n  /* Wait until all connection-threads are done */\n  while(getConnectionsCount() > 0) {\n    std::this_thread::sleep_for(std::chrono::milliseconds(100));\n  }\n}\n\n}}}\n\n// Path: src/oatpp/web/server/interceptor/AllowCorsGlobal.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_interceptor_AllowCorsGlobal_hpp\n#define oatpp_web_server_interceptor_AllowCorsGlobal_hpp\n\n#include \"oatpp/web/server/interceptor/ResponseInterceptor.hpp\"\n#include \"oatpp/web/server/interceptor/RequestInterceptor.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace interceptor {\n\nclass AllowOptionsGlobal : public RequestInterceptor {\npublic:\n  std::shared_ptr<OutgoingResponse> intercept(const std::shared_ptr<IncomingRequest>& request) override;\n};\n\nclass AllowCorsGlobal : public ResponseInterceptor {\nprivate:\n  oatpp::String m_origin;\n  oatpp::String m_methods;\n  oatpp::String m_headers;\n  oatpp::String m_maxAge;\npublic:\n\n  AllowCorsGlobal(const oatpp::String &origin = \"*\",\n                  const oatpp::String &methods = \"GET, POST, OPTIONS\",\n                  const oatpp::String &headers = \"DNT, User-Agent, X-Requested-With, If-Modified-Since, Cache-Control, Content-Type, Range, Authorization\",\n                  const oatpp::String &maxAge = \"1728000\");\n\n  std::shared_ptr<OutgoingResponse> intercept(const std::shared_ptr<IncomingRequest>& request,\n                                              const std::shared_ptr<OutgoingResponse>& response) override;\n\n};\n\n}}}}\n\n#endif // oatpp_web_server_interceptor_AllowCorsGlobal_hpp\n\n// Path: src/oatpp/web/server/interceptor/AllowCorsGlobal.cpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#include \"AllowCorsGlobal.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace interceptor {\n\nstd::shared_ptr<protocol::http::outgoing::Response> AllowOptionsGlobal::intercept(const std::shared_ptr<IncomingRequest> &request) {\n\n  const auto &line = request->getStartingLine();\n\n  if (line.method == \"OPTIONS\") {\n    return OutgoingResponse::createShared(protocol::http::Status::CODE_204, nullptr);\n  }\n\n  return nullptr;\n\n}\n\nAllowCorsGlobal::AllowCorsGlobal(const oatpp::String &origin,\n                                 const oatpp::String &methods,\n                                 const oatpp::String &headers,\n                                 const oatpp::String &maxAge)\n  : m_origin(origin)\n  , m_methods(methods)\n  , m_headers(headers)\n  , m_maxAge(maxAge)\n{}\n\nstd::shared_ptr<protocol::http::outgoing::Response> AllowCorsGlobal::intercept(const std::shared_ptr<IncomingRequest>& request,\n                                                                               const std::shared_ptr<OutgoingResponse>& response)\n{\n  response->putHeaderIfNotExists(protocol::http::Header::CORS_ORIGIN, m_origin);\n  response->putHeaderIfNotExists(protocol::http::Header::CORS_METHODS, m_methods);\n  response->putHeaderIfNotExists(protocol::http::Header::CORS_HEADERS, m_headers);\n  response->putHeaderIfNotExists(protocol::http::Header::CORS_MAX_AGE, m_maxAge);\n  return response;\n}\n\n}}}}\n\n// Path: src/oatpp/web/server/handler/AuthorizationHandler.hpp\n/***************************************************************************\n *\n * Project         _____    __   ____   _      _\n *                (  _  )  /__\\ (_  _)_| |_  _| |_\n *                 )(_)(  /(__)\\  )( (_   _)(_   _)\n *                (_____)(__)(__)(__)  |_|    |_|\n *\n *\n * Copyright 2018-present, Leonid Stryzhevskyi <lganzzzo@gmail.com>\n *                         Benedikt-Alexander Mokro\u00df <bam@icognize.de>\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n ***************************************************************************/\n\n#ifndef oatpp_web_server_handler_AuthorizationHandler_hpp\n#define oatpp_web_server_handler_AuthorizationHandler_hpp\n\n#include <oatpp/web/protocol/http/incoming/Request.hpp>\n#include \"oatpp/web/protocol/http/Http.hpp\"\n#include \"oatpp/core/macro/codegen.hpp\"\n#include \"oatpp/core/data/mapping/type/Type.hpp\"\n\nnamespace oatpp { namespace web { namespace server { namespace handler {\n\n/**\n * The AuthorizationObject superclass, all AuthorizationObjects have to extend this class.\n */\nclass AuthorizationObject : public oatpp::base::Countable {\nprotected:\n  A\nuthorizationObject() = default;\n};\n\n/**\n * Abstract Authorization Handler.\n */\nclass AuthorizationHandler {\npublic:\n  /**\n   * Convenience typedef for &l:AuthorizationObject;.\n   */\n  typedef oatpp::web::server::handler::AuthorizationObject AuthorizationObject;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::stream::BufferOutputStream;.\n   */\n  typedef oatpp::data::stream::BufferOutputStream BufferOutputStream;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Headers;.\n   */\n  typedef oatpp::web::protocol::http::Headers Headers;\nprivate:\n  oatpp::String m_scheme;\n  oatpp::String m_realm;\npublic:\n\n  /**\n   * Constructor.\n   * @param scheme - authorization type scheme. &id:oatpp::String;.\n   * @param realm - realm. &id:oatpp::String;.\n   */\n  AuthorizationHandler(const oatpp::String& scheme, const oatpp::String& realm);\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~AuthorizationHandler() = default;\n\n  /**\n   * Implement this method! Return nullptr if authorization should be denied.\n   * @param header - `Authorization` header. &id:oatpp::String;.\n   * @return - `std::shared_ptr` to &id:oatpp::web::server::handler::AuthorizationObject;.\n   */\n  virtual std::shared_ptr<AuthorizationObject> handleAuthorization(const oatpp::String& authorizationHeader) = 0;\n\n  /**\n   * Render WWW-Authenicate header value. <br>\n   * Custom Authorization handlers may override this method in order to provide additional information.\n   * @param stream - &id:oatpp::data::stream::BufferOutputStream;.\n   */\n  virtual void renderAuthenticateHeaderValue(BufferOutputStream& stream);\n\n  /**\n   * Add authorization error headers to the headers map. <br>\n   * @param headers - &id:oatpp::web::protocol::http::Headers;.\n   */\n  virtual void addErrorResponseHeaders(Headers& headers);\n\n  /**\n   * Get authorization scheme.\n   * @return\n   */\n  oatpp::String getScheme();\n\n  /**\n   * Get authorization realm.\n   * @return\n   */\n  oatpp::String getRealm();\n\n};\n\n/**\n * Default Basic AuthorizationObject - Convenience object to enable Basic-Authorization without the need to implement anything.\n */\nclass DefaultBasicAuthorizationObject : public AuthorizationObject {\npublic:\n\n  /**\n   * User-Id. &id:oatpp::String;.\n   */\n  oatpp::String userId;\n\n  /**\n   * Password. &id:oatpp::String;.\n   */\n  oatpp::String password;\n\n};\n\n/**\n * AuthorizationHandler for Authorization Type `Basic`. <br>\n * See [RFC 7617](https://tools.ietf.org/html/rfc7617). <br>\n * Extend this class to implement Custom Basic Authorization.\n */\nclass BasicAuthorizationHandler : public AuthorizationHandler {\n...\n", "needle_token_start": 15039, "needle_token_end": 15049, "code_context_ntokens": 15841, "output": ["```cpp\nclass AuthorizationHandler {\npublic:\n  /**\n   * Convenience typedef for &l:AuthorizationObject;.\n   */\n  typedef oatpp::web::server::handler::AuthorizationObject AuthorizationObject;\n\n  /**\n   * Convenience typedef for &id:oatpp::data::stream::BufferOutputStream;.\n   */\n  typedef oatpp::data::stream::BufferOutputStream BufferOutputStream;\n\n  /**\n   * Convenience typedef for &id:oatpp::web::protocol::http::Headers;.\n   */\n  typedef oatpp::web::protocol::http::Headers Headers;\nprivate:\n  oatpp::String m_scheme;\n  oatpp::String m_realm;\npublic:\n\n  /**\n   * Constructor.\n   * @param scheme - authorization type scheme. &id:oatpp::String;.\n   * @param realm - realm. &id:oatpp::String;.\n   */\n  AuthorizationHandler(const oatpp::String& scheme, const oatpp::String& realm);\n\n  /**\n   * Default virtual destructor.\n   */\n  virtual ~AuthorizationHandler() = default;\n\n  /**\n   * Implement this method! Return nullptr if authorization should be denied.\n   * @param header - `Authorization` header. &id:oatpp::String;.\n   * @return - `std::shared_ptr` to &id:oatpp::web::server::handler::AuthorizationObject;.\n   */\n  virtual std::shared_ptr<AuthorizationObject> handleAuthorization(const oatpp::String& authorizationHeader) = 0;\n\n  /**\n   * Render WWW-Authenicate header value. <br>\n   * Custom Authorization handlers may override this method in order to provide additional information.\n   * @param stream - &id:oatpp::data::stream::BufferOutputStream;.\n   */\n  virtual void renderAuthenticateHeaderValue(BufferOutputStream& stream);\n\n  /**\n   * Add authorization error headers to the headers map. <br>\n   * @param headers - &id:oatpp::web::protocol::http::Headers;.\n   */\n  virtual void addErrorResponseHeaders(Headers& headers);\n\n  /**\n   * Get authorization scheme.\n   * @return\n   */\n  oatpp::String getScheme();\n\n  /**\n   * Get authorization realm.\n   * @return\n   */\n  oatpp::String getRealm();\n\n};\n```"]}
{"repo": "google/gson", "name": "addTypeAdaptersForDate", "language": "java", "path": "gson/src/main/java/com/google/gson/GsonBuilder.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To configure serialization and deserialization adapters for date-related types based on specified patterns or styles.\n2. **Input**: A string pattern for dates, two integers representing date and time styles, and a list to which the adapters will be added.\n3. **Output**: There is no direct output as it modifies the list of factories by adding new type adapters.\n4. **Procedure**: The function checks if a non-empty date pattern is provided and creates date adapters accordingly. If no pattern is provided but styles are specified, it creates adapters based on these styles. It also checks for SQL types support and adds corresponding SQL date adapters if supported. These adapters are then added to the provided list of factories.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "   * which no {@link TypeAdapter} has been registered, and for which no built-in Gson {@code\n   * TypeAdapter} exists.\n   *\n   * <p>The created Gson instance might only use an access filter once for a class or its members\n   * and cache the result. It is not guaranteed that the filter will be used again every time a\n   * class or its members are accessed during serialization or deserialization.\n   *\n   * @param filter filter to add\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 2.9.1\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder addReflectionAccessFilter(ReflectionAccessFilter filter) {\n    Objects.requireNonNull(filter);\n    reflectionFilters.addFirst(filter);\n    return this;\n  }\n\n  /**\n   * Creates a {@link Gson} instance based on the current configuration. This method is free of\n   * side-effects to this {@code GsonBuilder} instance and hence can be called multiple times.\n   *\n   * @return an instance of Gson configured with the options currently set in this builder\n   */\n  public Gson create() {\n    List<TypeAdapterFactory> factories =\n        new ArrayList<>(this.factories.size() + this.hierarchyFactories.size() + 3);\n    factories.addAll(this.factories);\n    Collections.reverse(factories);\n\n    List<TypeAdapterFactory> hierarchyFactories = new ArrayList<>(this.hierarchyFactories);\n    Collections.reverse(hierarchyFactories);\n    factories.addAll(hierarchyFactories);\n\n    addTypeAdaptersForDate(datePattern, dateStyle, timeStyle, factories);\n\n    return new Gson(\n        excluder,\n        fieldNamingPolicy,\n        new HashMap<>(instanceCreators),\n        serializeNulls,\n        complexMapKeySerialization,\n        generateNonExecutableJson,\n        escapeHtmlChars,\n        formattingStyle,\n        strictness,\n        serializeSpecialFloatingPointValues,\n        useJdkUnsafe,\n        longSerializationPolicy,\n        datePattern,\n        dateStyle,\n        timeStyle,\n        new ArrayList<>(this.factories),\n        new ArrayList<>(this.hierarchyFactories),\n        factories,\n        objectToNumberStrategy,\n        numberToNumberStrategy,\n        new ArrayList<>(reflectionFilters));\n  }\n\n  \nprivate static void addTypeAdaptersForDate(\n      String datePattern, int dateStyle, int timeStyle, List<TypeAdapterFactory> factories) {\n    TypeAdapterFactory dateAdapterFactory;\n    boolean sqlTypesSupported = SqlTypesSupport.SUPPORTS_SQL_TYPES;\n    TypeAdapterFactory sqlTimestampAdapterFactory = null;\n    TypeAdapterFactory sqlDateAdapterFactory = null;\n\n    if (datePattern != null && !datePattern.trim().isEmpty()) {\n      dateAdapterFactory = DefaultDateTypeAdapter.DateType.DATE.createAdapterFactory(datePattern);\n\n      if (sqlTypesSupported) {\n        sqlTimestampAdapterFactory =\n            SqlTypesSupport.TIMESTAMP_DATE_TYPE.createAdapterFactory(datePattern);\n        sqlDateAdapterFactory = SqlTypesSupport.DATE_DATE_TYPE.createAdapterFactory(datePattern);\n      }\n    } else if (dateStyle != DateFormat.DEFAULT || timeStyle != DateFormat.DEFAULT) {\n      dateAdapterFactory =\n          DefaultDateTypeAdapter.DateType.DATE.createAdapterFactory(dateStyle, timeStyle);\n\n      if (sqlTypesSupported) {\n        sqlTimestampAdapterFactory =\n            SqlTypesSupport.TIMESTAMP_DATE_TYPE.createAdapterFactory(dateStyle, timeStyle);\n        sqlDateAdapterFactory =\n            SqlTypesSupport.DATE_DATE_TYPE.createAdapterFactory(dateStyle, timeStyle);\n      }\n    } else {\n      return;\n    }\n\n    factories.add(dateAdapterFactory);\n    if (sqlTypesSupported) {\n      factories.add(sqlTimestampAdapterFactory);\n      factories.add(sqlDateAdapterFactory);\n    }\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/stream/JsonScope.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.stream;\n\n/**\n * Lexical scoping elements within a JSON reader or writer.\n *\n * @author Jesse Wilson\n * @since 1.6\n */\nfinal class JsonScope {\n  private JsonScope() {}\n\n  /** An array with no elements requires no separator before the next element. */\n  static final int EMPTY_ARRAY = 1;\n\n  /** An array with at least one value requires a separator before the next element. */\n  static final int NONEMPTY_ARRAY = 2;\n\n  /** An object with no name/value pairs requires no separator before the next element. */\n  static final int EMPTY_OBJECT = 3;\n\n  /** An object whose most recent element is a key. The next element must be a value. */\n  static final int DANGLING_NAME = 4;\n\n  /** An object with at least one name/value pair requires a separator before the next element. */\n  static final int NONEMPTY_OBJECT = 5;\n\n  /** No top-level value has been started yet. */\n  static final int EMPTY_DOCUMENT = 6;\n\n  /** A top-level value has already been started. */\n  static final int NONEMPTY_DOCUMENT = 7;\n\n  /** A document that's been closed and cannot be accessed. */\n  static final int CLOSED = 8;\n}\n\n// Path: gson/src/main/java/com/google/gson/stream/JsonWriter.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.stream;\n\nimport static com.google.gson.stream.JsonScope.DANGLING_NAME;\nimport static com.google.gson.stream.JsonScope.EMPTY_ARRAY;\nimport static com.google.gson.stream.JsonScope.EMPTY_DOCUMENT;\nimport static com.google.gson.stream.JsonScope.EMPTY_OBJECT;\nimport static com.google.gson.stream.JsonScope.NONEMPTY_ARRAY;\nimport static com.google.gson.stream.JsonScope.NONEMPTY_DOCUMENT;\nimport static com.google.gson.stream.JsonScope.NONEMPTY_OBJECT;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.FormattingStyle;\nimport com.google.gson.Gson;\nimport com.google.gson.GsonBuilder;\nimport com.google.gson.Strictness;\nimport java.io.Closeable;\nimport java.io.Flushable;\nimport java.io.IOException;\nimport java.io.Writer;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.util.Arrays;\nimport java.util.Objects;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.regex.Pattern;\n\n/**\n * Writes a JSON (<a href=\"https://www.ietf.org/rfc/rfc8259.txt\">RFC 8259</a>) encoded value to a\n * stream, one token at a time. The stream includes both literal values (strings, numbers, booleans\n * and nulls) as well as the begin and end delimiters of objects and arrays.\n *\n * <h2>Encoding JSON</h2>\n *\n * To encode your data as JSON, create a new {@code JsonWriter}. Call methods on the writer as you\n * walk the structure's contents, nesting arrays and objects as necessary:\n *\n * <ul>\n *   <li>To write <strong>arrays</strong>, first call {@link #beginArray()}. Write each of the\n *       array's elements with the appropriate {@link #value} methods or by nesting other arrays and\n *       objects. Finally close the array using {@link #endArray()}.\n *   <li>To write <strong>objects</strong>, first call {@link #beginObject()}. Write each of the\n *       object's properties by alternating calls to {@link #name} with the property's value. Write\n *       property values with the appropriate {@link #value} method or by nesting other objects or\n *       arrays. Finally close the object using {@link #endObject()}.\n * </ul>\n *\n * <h2>Configuration</h2>\n *\n * The behavior of this writer can be customized with the following methods:\n *\n * <ul>\n *   <li>{@link #setFormattingStyle(FormattingStyle)}, the default is {@link\n *       FormattingStyle#COMPACT}\n *   <li>{@link #setHtmlSafe(boolean)}, by default HTML characters are not escaped in the JSON\n *       output\n *   <li>{@link #setStrictness(Strictness)}, the default is {@link Strictness#LEGACY_STRICT}\n *   <li>{@link #setSerializeNulls(boolean)}, by default {@code null} is serialized\n * </ul>\n *\n * The default configuration of {@code JsonWriter} instances used internally by the {@link Gson}\n * class differs, and can be adjusted with the various {@link GsonBuilder} methods.\n *\n * <h2>Example</h2>\n *\n * Suppose we'd like to encode a stream of messages such as the following:\n *\n * <pre>{@code\n * [\n *   {\n *     \"id\": 912345678901,\n *     \"text\": \"How do I stream JSON in Java?\",\n *     \"geo\": null,\n *     \"user\": {\n *       \"name\": \"json_newb\",\n *       \"followers_count\": 41\n *      }\n *   },\n *   {\n *     \"id\": 912345678902,\n *     \"text\": \"@json_newb just use JsonWriter!\",\n *     \"geo\": [50.454722, -104.606667],\n *     \"user\": {\n *       \"name\": \"jesse\",\n *       \"followers_count\": 2\n *     }\n *   }\n * ]\n * }</pre>\n *\n * This code encodes the above structure:\n *\n * <pre>{@code\n * public void writeJsonStream(OutputStream out, List<Message> messages) throws IOException {\n *   JsonWriter writer = new JsonWriter(new OutputStreamWriter(out, \"UTF-8\"));\n *   writer.setIndent(\"    \");\n *   writeMessagesArray(writer, messages);\n *   writer.close();\n * }\n *\n * public void writeMessagesArray(JsonWriter writer, List<Message> messages) throws IOException {\n *   writer.beginArray();\n *   for (Message message : messages) {\n *     writeMessage(writer, message);\n *   }\n *   writer.endArray();\n * }\n *\n * public void writeMessage(JsonWriter writer, Message message) throws IOException {\n *   writer.beginObject();\n *   writer.name(\"id\").value(message.getId());\n *   writer.name(\"text\").value(message.getText());\n *   if (message.getGeo() != null) {\n *     writer.name(\"geo\");\n *     writeDoublesArray(writer, message.getGeo());\n *   } else {\n *     writer.name(\"geo\").nullValue();\n *   }\n *   writer.name(\"user\");\n *   writeUser(writer, message.getUser());\n *   writer.endObject();\n * }\n *\n * public void writeUser(JsonWriter writer, User user) throws IOException {\n *   writer.beginObject();\n *   writer.name(\"name\").value(user.getName());\n *   writer.name(\"followers_count\").value(user.getFollowersCount());\n *   writer.endObject();\n * }\n *\n * public void writeDoublesArray(JsonWriter writer, List<Double> doubles) throws IOException {\n *   writer.beginArray();\n *   for (Double value : doubles) {\n *     writer.value(value);\n *   }\n *   writer.endArray();\n * }\n * }</pre>\n *\n * <p>Each {@code JsonWriter} may be used to write a single JSON stream. Instances of this class are\n * not thread safe. Calls that would result in a malformed JSON string will fail with an {@link\n * IllegalStateException}.\n *\n * @author Jesse Wilson\n * @since 1.6\n */\npublic class JsonWriter implements Closeable, Flushable {\n\n  // Syntax as defined by https://datatracker.ietf.org/doc/html/rfc8259#section-6\n  private static final Pattern VALID_JSON_NUMBER_PATTERN =\n      Pattern.compile(\"-?(?:0|[1-9][0-9]*)(?:\\\\.[0-9]+)?(?:[eE][-+]?[0-9]+)?\");\n\n  /*\n   * From RFC 8259, \"All Unicode characters may be placed within the\n   * quotation marks except for the characters that must be escaped:\n   * quotation mark, reverse solidus, and the control characters\n   * (U+0000 through U+001F).\"\n   *\n   * We also escape '\\u2028' and '\\u2029', which JavaScript interprets as\n   * newline characters. This prevents eval() from failing with a syntax\n   * error. http://code.google.com/p/google-gson/issues/detail?id=341\n   */\n  private static final String[] REPLACEMENT_CHARS;\n  private static final String[] HTML_SAFE_REPLACEMENT_CHARS;\n\n  static {\n    REPLACEMENT_CHARS = new String[128];\n    for (int i = 0; i <= 0x1f; i++) {\n      REPLACEMENT_CHARS[i] = String.format(\"\\\\u%04x\", i);\n    }\n    REPLACEMENT_CHARS['\"'] = \"\\\\\\\"\";\n    REPLACEMENT_CHARS['\\\\'] = \"\\\\\\\\\";\n    REPLACEMENT_CHARS['\\t'] = \"\\\\t\";\n    REPLACEMENT_CHARS['\\b'] = \"\\\\b\";\n    REPLACEMENT_CHARS['\\n'] = \"\\\\n\";\n    REPLACEMENT_CHARS['\\r'] = \"\\\\r\";\n    REPLACEMENT_CHARS['\\f'] = \"\\\\f\";\n    HTML_SAFE_REPLACEMENT_CHARS = REPLACEMENT_CHARS.clone();\n    HTML_SAFE_REPLACEMENT_CHARS['<'] = \"\\\\u003c\";\n    HTML_SAFE_REPLACEMENT_CHARS['>'] = \"\\\\u003e\";\n    HTML_SAFE_REPLACEMENT_CHARS['&'] = \"\\\\u0026\";\n    HTML_SAFE_REPLACEMENT_CHARS['='] = \"\\\\u003d\";\n    HTML_SAFE_REPLACEMENT_CHARS['\\''] = \"\\\\u0027\";\n  }\n\n  /** The JSON output destination */\n  private final Writer out;\n\n  private int[] stack = new int[32];\n  private int stackSize = 0;\n\n  {\n    push(EMPTY_DOCUMENT);\n  }\n\n  private FormattingStyle formattingStyle;\n  // These fields cache data derived from the formatting style, to avoid having to\n  // re-evaluate it every time something is written\n  private String formattedColon;\n  private String formattedComma;\n  private boolean usesEmptyNewlineAndIndent;\n\n  private Strictness strictness = Strictness.LEGACY_STRICT;\n\n  private boolean htmlSafe;\n\n  private String deferredName;\n\n  private boolean serializeNulls = true;\n\n  /**\n   * Creates a new instance that writes a JSON-encoded stream to {@code out}. For best performance,\n   * ensure {@link Writer} is buffered; wrapping in {@link java.io.BufferedWriter BufferedWriter} if\n   * necessary.\n   */\n  public JsonWriter(Writer out) {\n    this.out = Objects.requireNonNull(out, \"out == null\");\n    setFormattingStyle(FormattingStyle.COMPACT);\n  }\n\n  /**\n   * Sets the indentation string to be repeated for each level of indentation in the encoded\n   * document. If {@code indent.isEmpty()} the encoded document will be compact. Otherwise the\n   * encoded document will be more human-readable.\n   *\n   * <p>This is a convenience method which overwrites any previously {@linkplain\n   * #setFormattingStyle(FormattingStyle) set formatting style} with either {@link\n   * FormattingStyle#COMPACT} if the given indent string is empty, or {@link FormattingStyle#PRETTY}\n   * with the given indent if not empty.\n   *\n   * @param indent a string containing only whitespace.\n   */\n  public final void setIndent(String indent) {\n    if (indent.isEmpty()) {\n      setFormattingStyle(FormattingStyle.COMPACT);\n    } else {\n      setFormattingStyle(FormattingStyle.PRETTY.withIndent(indent));\n    }\n  }\n\n  /**\n   * Sets the formatting style to be used in the encoded document.\n   *\n   * <p>The formatting style specifies for example the indentation string to be repeated for each\n   * level of indentation, or the newline style, to accommodate various OS styles.\n   *\n   * @param formattingStyle the formatting style to use, must not be {@code null}.\n   * @since $next-version$\n   */\n  public final void setFormattingStyle(FormattingStyle formattingStyle) {\n    this.formattingStyle = Objects.requireNonNull(formattingStyle);\n\n    this.formattedComma = \",\";\n    if (this.formattingStyle.usesSpaceAfterSeparators()) {\n      this.formattedColon = \": \";\n\n      // Only add space if no newline is written\n      if (this.formattingStyle.getNewline().isEmpty()) {\n        this.formattedComma = \", \";\n      }\n    } else {\n      this.formattedColon = \":\";\n    }\n\n    this.usesEmptyNewlineAndIndent =\n        this.formattingStyle.getNewline().isEmpty() && this.formattingStyle.getIndent().isEmpty();\n  }\n\n  /**\n   * Returns the pretty printing style used by this writer.\n   *\n   * @return the {@code FormattingStyle} that will be used.\n   * @since $next-version$\n   */\n  public final FormattingStyle getFormattingStyle() {\n    return formattingStyle;\n  }\n\n  /**\n   * Sets the strictness of this writer.\n   *\n   * @deprecated Please use {@link #setStrictness(Strictness)} instead. {@code\n   *     JsonWriter.setLenient(true)} should be replaced by {@code\n   *     JsonWriter.setStrictness(Strictness.LENIENT)} and {@code JsonWriter.setLenient(false)}\n   *     should be replaced by {@code JsonWriter.setStrictness(Strictness.LEGACY_STRICT)}.<br>\n   *     However, if you used {@code setLenient(false)} before, you might prefer {@link\n   *     Strictness#STRICT} now instead.\n   * @param lenient whether this writer should be lenient. If true, the strictness is set to {@link\n   *     Strictness#LENIENT}. If false, the strictness is set to {@link Strictness#LEGACY_STRICT}.\n   * @see #setStrictness(Strictness)\n   */\n  @Deprecated\n  // Don't specify @InlineMe, so caller with `setLenient(false)` becomes aware of new\n  // Strictness.STRICT\n  @SuppressWarnings(\"InlineMeSuggester\")\n  public final void setLenient(boolean lenient) {\n    setStrictness(lenient ? Strictness.LENIENT : Strictness.LEGACY_STRICT);\n  }\n\n  /**\n   * Returns true if the {@link Strictness} of this writer is equal to {@link Strictness#LENIENT}.\n   *\n   * @see JsonWriter#setStrictness(Strictness)\n   */\n  public boolean isLenient() {\n    return strictness == Strictness.LENIENT;\n  }\n\n  /**\n   * Configures how strict this writer is with regard to the syntax rules specified in <a\n   * href=\"https://www.ietf.org/rfc/rfc8259.txt\">RFC 8259</a>. By default, {@link\n   * Strictness#LEGACY_STRICT} is used.\n   *\n   * <dl>\n   *   <dt>{@link Strictness#STRICT} &amp; {@link Strictness#LEGACY_STRICT}\n   *   <dd>The behavior of these is currently identical. In these strictness modes, the writer only\n   *       writes JSON in accordance with RFC 8259.\n   *   <dt>{@link Strictness#LENIENT}\n   *   <dd>This mode relaxes the behavior of the writer to allow the writing of {@link\n   *       Double#isNaN() NaNs} and {@link Double#isInfinite() infinities}. It also allows writing\n   *       multiple top level values.\n   * </dl>\n   *\n   * @param strictness the new strictness of this writer. May not be {@code null}.\n   * @since $next-version$\n   */\n  public final void setStrictness(Strictness strictness) {\n    this.strictness = Objects.requireNonNull(strictness);\n  }\n\n  /**\n   * Returns the {@linkplain Strictness strictness} of this writer.\n   *\n   * @see #setStrictness(Strictness)\n   * @since $next-version$\n   */\n  public final Strictness getStrictness() {\n    return strictness;\n  }\n\n  /**\n   * Configures this writer to emit JSON that's safe for direct inclusion in HTML and XML documents.\n   * This escapes the HTML characters {@code <}, {@code >}, {@code &}, {@code =} and {@code '}\n   * before writing them to the stream. Without this setting, your XML/HTML encoder should replace\n   * these characters with the corresponding escape sequences.\n   */\n  public final void setHtmlSafe(boolean htmlSafe) {\n    this.htmlSafe = htmlSafe;\n  }\n\n  /**\n   * Returns true if this writer writes JSON that's safe for inclusion in HTML and XML documents.\n   */\n  public final boolean isHtmlSafe() {\n    return htmlSafe;\n  }\n\n  /**\n   * Sets whether object members are serialized when their value is null. This has no impact on\n   * array elements. The default is true.\n   */\n  public final void setSerializeNulls(boolean serializeNulls) {\n    this.serializeNulls = serializeNulls;\n  }\n\n  /**\n   * Returns true if object members are serialized when their value is null. This has no impact on\n   * array elements. The default is true.\n   */\n  public final boolean getSerializeNulls() {\n    return serializeNulls;\n  }\n\n  /**\n   * Begins encoding a new array. Each call to this method must be paired with a call to {@link\n   * #endArray}.\n   *\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter beginArray() throws IOException {\n    writeDeferredName();\n    return openScope(EMPTY_ARRAY, '[');\n  }\n\n  /**\n   * Ends encoding the current array.\n   *\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter endArray() throws IOException {\n    return closeScope(EMPTY_ARRAY, NONEMPTY_ARRAY, ']');\n  }\n\n  /**\n   * Begins encoding a new object. Each call to this method must be paired with a call to {@link\n   * #endObject}.\n   *\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter beginObject() throws IOException {\n    writeDeferredName();\n    return openScope(EMPTY_OBJECT, '{');\n  }\n\n  /**\n   * Ends encoding the current object.\n   *\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter endObject() throws IOException {\n    return closeScope(EMPTY_OBJECT, NONEMPTY_OBJECT, '}');\n  }\n\n  /** Enters a new scope by appending any necessary whitespace and the given bracket. */\n  @CanIgnoreReturnValue\n  private JsonWriter openScope(int empty, char openBracket) throws IOException {\n    beforeValue();\n    push(empty);\n    out.write(openBracket);\n    return this;\n  }\n\n  /** Closes the current scope by appending any necessary whitespace and the given bracket. */\n  @CanIgnoreReturnValue\n  private JsonWriter closeScope(int empty, int nonempty, char closeBracket) throws IOException {\n    int context = peek();\n    if (context != nonempty && context != empty) {\n      throw new IllegalStateException(\"Nesting problem.\");\n    }\n    if (deferredName != null) {\n      throw new IllegalStateException(\"Dangling name: \" + deferredName);\n    }\n\n    stackSize--;\n    if (context == nonempty) {\n      newline();\n    }\n    out.write(closeBracket);\n    return this;\n  }\n\n  private void push(int newTop) {\n    if (stackSize == stack.length) {\n      stack = Arrays.copyOf(stack, stackSize * 2);\n    }\n    stack[stackSize++] = newTop;\n  }\n\n  /** Returns the value on the top of the stack. */\n  private int peek() {\n    if (stackSize == 0) {\n      throw new IllegalStateException(\"JsonWriter is closed.\");\n    }\n    return stack[stackSize - 1];\n  }\n\n  /** Replace the value on the top of the stack with the given value. */\n  private void replaceTop(int topOfStack) {\n    stack[stackSize - 1] = topOfStack;\n  }\n\n  /**\n   * Encodes the property name.\n   *\n   * @param name the name of the forthcoming value. May not be {@code null}.\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter name(String name) throws IOException {\n    Objects.requireNonNull(name, \"name == null\");\n    if (deferredName != null) {\n      throw new IllegalStateException(\"Already wrote a name, expecting a value.\");\n    }\n    int context = peek();\n    if (context != EMPTY_OBJECT && context != NONEMPTY_OBJECT) {\n      throw new IllegalStateException(\"Please begin an object before writing a name.\");\n    }\n    deferredName = name;\n    return this;\n  }\n\n  private void writeDeferredName() throws IOException {\n    if (deferredName != null) {\n      beforeName();\n      string(deferredName);\n      deferredName = null;\n    }\n  }\n\n  /**\n   * Encodes {@code value}.\n   *\n   * @param value the literal string value, or null to encode a null literal.\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter value(String value) throws IOException {\n    if (value == null) {\n      return nullValue();\n    }\n    writeDeferredName();\n    beforeValue();\n    string(value);\n    return this;\n  }\n\n  /**\n   * Encodes {@code value}.\n   *\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter value(boolean value) throws IOException {\n    writeDeferredName();\n    beforeValue();\n    out.write(value ? \"true\" : \"false\");\n    return this;\n  }\n\n  /**\n   * Encodes {@code value}.\n   *\n   * @return this writer.\n   * @since 2.7\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter value(Boolean value) throws IOException {\n    if (value == null) {\n      return nullValue();\n    }\n    writeDeferredName();\n    beforeValue();\n    out.write(value ? \"true\" : \"false\");\n    return this;\n  }\n\n  /**\n   * Encodes {@code value}.\n   *\n   * @param value a finite value, or if {@link #setStrictness(Strictness) lenient}, also {@link\n   *     Float#isNaN() NaN} or {@link Float#isInfinite() infinity}.\n   * @return this writer.\n   * @throws IllegalArgumentException if the value is NaN or Infinity and this writer is not {@link\n   *     #setStrictness(Strictness) lenient}.\n   * @since 2.9.1\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter value(float value) throws IOException {\n    writeDeferredName();\n    if (strictness != Strictness.LENIENT && (Float.isNaN(value) || Float.isInfinite(value))) {\n      throw new IllegalArgumentException(\"Numeric values must be finite, but was \" + value);\n    }\n    beforeValue();\n    out.append(Float.toString(value));\n    return this;\n  }\n\n  /**\n   * Encodes {@code value}.\n   *\n   * @param value a finite value, or if {@link #setStrictness(Strictness) lenient}, also {@link\n   *     Double#isNaN() NaN} or {@link Double#isInfinite() infinity}.\n   * @return this writer.\n   * @throws IllegalArgumentException if the value is NaN or Infinity and this writer is not {@link\n   *     #setStrictness(Strictness) lenient}.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter value(double value) throws IOException {\n    writeDeferredName();\n    if (strictness != Strictness.LENIENT && (Double.isNaN(value) || Double.isInfinite(value))) {\n      throw new IllegalArgumentException(\"Numeric values must be finite, but was \" + value);\n    }\n    beforeValue();\n    out.append(Double.toString(value));\n    return this;\n  }\n\n  /**\n   * Encodes {@code value}.\n   *\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter value(long value) throws IOException {\n    writeDeferredName();\n    beforeValue();\n    out.write(Long.toString(value));\n    return this;\n  }\n\n  /**\n   * Encodes {@code value}. The value is written by directly writing the {@link Number#toString()}\n   * result to JSON. Implementations must make sure that the result represents a valid JSON number.\n   *\n   * @param value a finite value, or if {@link #setStrictness(Strictness) lenient}, also {@link\n   *     Double#isNaN() NaN} or {@link Double#isInfinite() infinity}.\n   * @return this writer.\n   * @throws IllegalArgumentException if the value is NaN or Infinity and this writer is not {@link\n   *     #setStrictness(Strictness) lenient}; or if the {@code toString()} result is not a valid\n   *     JSON number.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter value(Number value) throws IOException {\n    if (value == null) {\n      return nullValue();\n    }\n\n    writeDeferredName();\n    String string = value.toString();\n    if (string.equals(\"-Infinity\") || string.equals(\"Infinity\") || string.equals(\"NaN\")) {\n      if (strictness != Strictness.LENIENT) {\n        throw new IllegalArgumentException(\"Numeric values must be finite, but was \" + string);\n      }\n    } else {\n      Class<? extends Number> numberClass = value.getClass();\n      // Validate that string is valid before writing it directly to JSON output\n      if (!isTrustedNumberType(numberClass)\n          && !VALID_JSON_NUMBER_PATTERN.matcher(string).matches()) {\n        throw new IllegalArgumentException(\n            \"String created by \" + numberClass + \" is not a valid JSON number: \" + string);\n      }\n    }\n\n    beforeValue();\n    out.append(string);\n    return this;\n  }\n\n  /**\n   * Encodes {@code null}.\n   *\n   * @return this writer.\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter nullValue() throws IOException {\n    if (deferredName != null) {\n      if (serializeNulls) {\n        writeDeferredName();\n      } else {\n        deferredName = null;\n        return this; // skip the name and the value\n      }\n    }\n    beforeValue();\n    out.write(\"null\");\n    return this;\n  }\n\n  /**\n   * Writes {@code value} directly to the writer without quoting or escaping. This might not be\n   * supported by all implementations, if not supported an {@code UnsupportedOperationException} is\n   * thrown.\n   *\n   * @param value the literal string value, or null to encode a null literal.\n   * @return this writer.\n   * @throws UnsupportedOperationException if this writer does not support writing raw JSON values.\n   * @since 2.4\n   */\n  @CanIgnoreReturnValue\n  public JsonWriter jsonValue(String value) throws IOException {\n    if (value == null) {\n      return nullValue();\n    }\n    writeDeferredName();\n    beforeValue();\n    out.append(value);\n    return this;\n  }\n\n  /**\n   * Ensures all buffered data is written to the underlying {@link Writer} and flushes that writer.\n   */\n  @Override\n  public void flush() throws IOException {\n    if (stackSize == 0) {\n      throw new IllegalStateException(\"JsonWriter is closed.\");\n    }\n    out.flush();\n  }\n\n  /**\n   * Flushes and closes this writer and the underlying {@link Writer}.\n   *\n   * @throws IOException if the JSON document is incomplete.\n   */\n  @Override\n  public void close() throws IOException {\n    out.close();\n\n    int size = stackSize;\n    if (size > 1 || (size == 1 && stack[size - 1] != NONEMPTY_DOCUMENT)) {\n      throw new IOException(\"Incomplete document\");\n    }\n    stackSize = 0;\n  }\n\n  /**\n   * Returns whether the {@code toString()} of {@code c} can be trusted to return a valid JSON\n   * number.\n   */\n  private static boolean isTrustedNumberType(Class<? extends Number> c) {\n    // Note: Don't consider LazilyParsedNumber trusted because it could contain\n    // an arbitrary malformed string\n    return c == Integer.class\n        || c == Long.class\n        || c == Double.class\n        || c == Float.class\n        || c == Byte.class\n        || c == Short.class\n        || c == BigDecimal.class\n        || c == BigInteger.class\n        || c == AtomicInteger.class\n        || c == AtomicLong.class;\n  }\n\n  private void string(String value) throws IOException {\n    String[] replacements = htmlSafe ? HTML_SAFE_REPLACEMENT_CHARS : REPLACEMENT_CHARS;\n    out.write('\\\"');\n    int last = 0;\n    int length = value.length();\n    for (int i = 0; i < length; i++) {\n      char c = value.charAt(i);\n      String replacement;\n      if (c < 128) {\n        replacement = replacements[c];\n        if (replacement == null) {\n          continue;\n        }\n      } else if (c == '\\u2028') {\n        replacement = \"\\\\u2028\";\n      } else if (c == '\\u2029') {\n        replacement = \"\\\\u2029\";\n      } else {\n        continue;\n      }\n      if (last < i) {\n        out.write(value, last, i - last);\n      }\n      out.write(replacement);\n      last = i + 1;\n    }\n    if (last < length) {\n      out.write(value, last, length - last);\n    }\n    out.write('\\\"');\n  }\n\n  private void newline() throws IOException {\n    if (usesEmptyNewlineAndIndent) {\n      return;\n    }\n\n    out.write(formattingStyle.getNewline());\n    for (int i = 1, size = stackSize; i < size; i++) {\n      out.write(formattingStyle.getIndent());\n    }\n  }\n\n  /**\n   * Inserts any necessary separators and whitespace before a name. Also adjusts the stack to expect\n   * the name's value.\n   */\n  private void beforeName() throws IOException {\n    int context = peek();\n    if (context == NONEMPTY_OBJECT) { // first in object\n      out.write(formattedComma);\n    } else if (context != EMPTY_OBJECT) { // not in an object!\n      throw new IllegalStateException(\"Nesting problem.\");\n    }\n    newline();\n    replaceTop(DANGLING_NAME);\n  }\n\n  /**\n   * Inserts any necessary separators and whitespace before a literal value, inline array, or inline\n   * object. Also adjusts the stack to expect either a closing bracket or another element.\n   */\n  @SuppressWarnings(\"fallthrough\")\n  private void beforeValue() throws IOException {\n    switch (peek()) {\n      case NONEMPTY_DOCUMENT:\n        if (strictness != Strictness.LENIENT) {\n          throw new IllegalStateException(\"JSON must have only one top-level value.\");\n        }\n        // fall-through\n      case EMPTY_DOCUMENT: // first in document\n        replaceTop(NONEMPTY_DOCUMENT);\n        break;\n\n      case EMPTY_ARRAY: // first in array\n        replaceTop(NONEMPTY_ARRAY);\n        newline();\n        break;\n\n      case NONEMPTY_ARRAY: // another in array\n        out.append(formattedComma);\n        newline();\n        break;\n\n      case DANGLING_NAME: // value for name\n        out.append(formattedColon);\n        replaceTop(NONEMPTY_OBJECT);\n        break;\n\n      default:\n        throw new IllegalStateException(\"Nesting problem.\");\n    }\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/FormattingStyle.java\n/*\n * Copyright (C) 2022 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.gson.stream.JsonWriter;\nimport java.util.Objects;\n\n/**\n * A class used to control what the serialization output looks like.\n *\n * <p>It currently has the following configuration methods, but more methods might be added in the\n * future:\n *\n * <ul>\n *   <li>{@link #withNewline(String)}\n *   <li>{@link #withIndent(String)}\n *   <li>{@link #withSpaceAfterSeparators(boolean)}\n * </ul>\n *\n * @see GsonBuilder#setFormattingStyle(FormattingStyle)\n * @see JsonWriter#setFormattingStyle(FormattingStyle)\n * @see <a href=\"https://en.wikipedia.org/wiki/Newline\">Wikipedia Newline article</a>\n * @since $next-version$\n */\npublic class FormattingStyle {\n  private final String newline;\n  private final String indent;\n  private final boolean spaceAfterSeparators;\n\n  /**\n   * The default compact formatting style:\n   *\n   * <ul>\n   *   <li>no newline\n   *   <li>no indent\n   *   <li>no space after {@code ','} and {@code ':'}\n   * </ul>\n   */\n  public static final FormattingStyle COMPACT = new FormattingStyle(\"\", \"\", false);\n\n  /**\n   * The default pretty printing formatting style:\n   *\n   * <ul>\n   *   <li>{@code \"\\n\"} as newline\n   *   <li>two spaces as indent\n   *   <li>a space between {@code ':'} and the subsequent value\n   * </ul>\n   */\n  public static final FormattingStyle PRETTY = new FormattingStyle(\"\\n\", \"  \", true);\n\n  private FormattingStyle(String newline, String indent, boolean spaceAfterSeparators) {\n    Objects.requireNonNull(newline, \"newline == null\");\n    Objects.requireNonNull(indent, \"indent == null\");\n    if (!newline.matches(\"[\\r\\n]*\")) {\n      throw new IllegalArgumentException(\n          \"Only combinations of \\\\n and \\\\r are allowed in newline.\");\n    }\n    if (!indent.matches(\"[ \\t]*\")) {\n      throw new IllegalArgumentException(\n          \"Only combinations of spaces and tabs are allowed in indent.\");\n    }\n    this.newline = newline;\n    this.indent = indent;\n    this.spaceAfterSeparators = spaceAfterSeparators;\n  }\n\n  /**\n   * Creates a {@link FormattingStyle} with the specified newline setting.\n   *\n   * <p>It can be used to accommodate certain OS convention, for example hardcode {@code \"\\n\"} for\n   * Linux and macOS, {@code \"\\r\\n\"} for Windows, or call {@link java.lang.System#lineSeparator()}\n   * to match the current OS.\n   *\n   * <p>Only combinations of {@code \\n} and {@code \\r} are allowed.\n   *\n   * @param newline the string value that will be used as newline.\n   * @return a newly created {@link FormattingStyle}\n   */\n  public FormattingStyle withNewline(String newline) {\n    return new FormattingStyle(newline, this.indent, this.spaceAfterSeparators);\n  }\n\n  /**\n   * Creates a {@link FormattingStyle} with the specified indent string.\n   *\n   * <p>Only combinations of spaces and tabs allowed in indent.\n   *\n   * @param indent the string value that will be used as indent.\n   * @return a newly created {@link FormattingStyle}\n   */\n  public FormattingStyle withIndent(String indent) {\n    return new FormattingStyle(this.newline, indent, this.spaceAfterSeparators);\n  }\n\n  /**\n   * Creates a {@link FormattingStyle} which either uses a space after the separators {@code ','}\n   * and {@code ':'} in the JSON output, or not.\n   *\n   * <p>This setting has no effect on the {@linkplain #withNewline(String) configured newline}. If a\n   * non-empty newline is configured, it will always be added after {@code ','} and no space is\n   * added after the {@code ','} in that case.\n   *\n   * @param spaceAfterSeparators whether to output a space after {@code ','} and {@code ':'}.\n   * @return a newly created {@link FormattingStyle}\n   */\n  public FormattingStyle withSpaceAfterSeparators(boolean spaceAfterSeparators) {\n    return new FormattingStyle(this.newline, this.indent, spaceAfterSeparators);\n  }\n\n  /**\n   * Returns the string value that will be used as a newline.\n   *\n   * @return the newline value.\n   */\n  public String getNewline() {\n    return this.newline;\n  }\n\n  /**\n   * Returns the string value that will be used as indent.\n   *\n   * @return the indent value.\n   */\n  public String getIndent() {\n    return this.indent;\n  }\n\n  /** Returns whether a space will be used after {@code ','} and {@code ':'}. */\n  public boolean usesSpaceAfterSeparators() {\n    return this.spaceAfterSeparators;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/annotations/JsonAdapter.java\n/*\n * Copyright (C) 2014 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.annotations;\n\nimport com.google.gson.Gson;\nimport com.google.gson.GsonBuilder;\nimport com.google.gson.InstanceCreator;\nimport com.google.gson.JsonDeserializer;\nimport com.google.gson.JsonSerializer;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.TypeAdapterFactory;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * An annotation that indicates the Gson {@link TypeAdapter} to use with a class or field.\n *\n * <p>Here is an example of how this annotation is used:\n *\n * <pre>\n * &#64;JsonAdapter(UserJsonAdapter.class)\n * public class User {\n *   public final String firstName, lastName;\n *\n *   private User(String firstName, String lastName) {\n *     this.firstName = firstName;\n *     this.lastName = lastName;\n *   }\n * }\n *\n * public class UserJsonAdapter extends TypeAdapter&lt;User&gt; {\n *   &#64;Override public void write(JsonWriter out, User user) throws IOException {\n *     // implement write: combine firstName and lastName into name\n *     out.beginObject();\n *     out.name(\"name\");\n *     out.value(user.firstName + \" \" + user.lastName);\n *     out.endObject();\n *   }\n *\n *   &#64;Override public User read(JsonReader in) throws IOException {\n *     // implement read: split name into firstName and lastName\n *     in.beginObject();\n *     in.nextName();\n *     String[] nameParts = in.nextString().split(\" \");\n *     in.endObject();\n *     return new User(nameParts[0], nameParts[1]);\n *   }\n * }\n * </pre>\n *\n * Since {@code User} class specified {@code UserJsonAdapter.class} in {@code @JsonAdapter}\n * annotation, it will automatically be invoked to serialize/deserialize {@code User} instances.\n *\n * <p>Here is an example of how to apply this annotation to a field.\n *\n * <pre>\n * private static final class Gadget {\n *   &#64;JsonAdapter(UserJsonAdapter.class)\n *   final User user;\n *\n *   Gadget(User user) {\n *     this.user = user;\n *   }\n * }\n * </pre>\n *\n * It's possible to specify different type adapters on a field, that field's type, and in the {@link\n * GsonBuilder}. Field annotations take precedence over {@code GsonBuilder}-registered type\n * adapters, which in turn take precedence over annotated types.\n *\n * <p>The class referenced by this annotation must be either a {@link TypeAdapter} or a {@link\n * TypeAdapterFactory}, or must implement one or both of {@link JsonDeserializer} or {@link\n * JsonSerializer}. Using {@link TypeAdapterFactory} makes it possible to delegate to the enclosing\n * {@link Gson} instance. By default the specified adapter will not be called for {@code null}\n * values; set {@link #nullSafe()} to {@code false} to let the adapter handle {@code null} values\n * itself.\n *\n * <p>The type adapter is created in the same way Gson creates instances of custom classes during\n * deserialization, that means:\n *\n * <ol>\n *   <li>If a custom {@link InstanceCreator} has been registered for the adapter class, it will be\n *       used to create the instance\n *   <li>Otherwise, if the adapter class has a no-args constructor (regardless of which visibility),\n *       it will be invoked to create the instance\n *   <li>Otherwise, JDK {@code Unsafe} will be used to create the instance; see {@link\n *       GsonBuilder#disableJdkUnsafe()} for the unexpected side-effects this might have\n * </ol>\n *\n * <p>{@code Gson} instances might cache the adapter they create for a {@code @JsonAdapter}\n * annotation. It is not guaranteed that a new adapter is created every time the annotated class or\n * field is serialized or deserialized.\n *\n * @since 2.3\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @author Jesse Wilson\n */\n// Note that the above example is taken from AdaptAnnotationTest.\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.TYPE, ElementType.FIELD})\npublic @interface JsonAdapter {\n\n  /**\n   * Either a {@link TypeAdapter} or {@link TypeAdapterFactory}, or one or both of {@link\n   * JsonDeserializer} or {@link JsonSerializer}.\n   */\n  Class<?> value();\n\n  /**\n   * Whether the adapter referenced by {@link #value()} should be made {@linkplain\n   * TypeAdapter#nullSafe() null-safe}.\n   *\n   * <p>If {@code true} (the default), it will be made null-safe and Gson will handle {@code null}\n   * Java objects on serialization and JSON {@code null} on deserialization without calling the\n   * adapter. If {@code false}, the adapter will have to handle the {@code null} values.\n   */\n  boolean nullSafe() default true;\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/ReflectionAccessFilterHelper.java\n/*\n * Copyright (C) 2022 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport com.google.gson.ReflectionAccessFilter;\nimport com.google.gson.ReflectionAccessFilter.FilterResult;\nimport java.lang.reflect.AccessibleObject;\nimport java.lang.reflect.Method;\nimport java.util.List;\n\n/** Internal helper class for {@link ReflectionAccessFilter}. */\npublic class ReflectionAccessFilterHelper {\n  private ReflectionAccessFilterHelper() {}\n\n  // Platform type detection is based on Moshi's Util.isPlatformType(Class)\n  // See\n  // https://github.com/square/moshi/blob/3c108919ee1cce88a433ffda04eeeddc0341eae7/moshi/src/main/java/com/squareup/moshi/internal/Util.java#L141\n\n  public static boolean isJavaType(Class<?> c) {\n    return isJavaType(c.getName());\n  }\n\n  private static boolean isJavaType(String className) {\n    return className.startsWith(\"java.\") || className.startsWith(\"javax.\");\n  }\n\n  public static boolean isAndroidType(Class<?> c) {\n    return isAndroidType(c.getName());\n  }\n\n  private static boolean isAndroidType(String className) {\n    return className.startsWith(\"android.\")\n        || className.startsWith(\"androidx.\")\n        || isJavaType(className);\n  }\n\n  public static boolean isAnyPlatformType(Class<?> c) {\n    String className = c.getName();\n    return isAndroidType(className) // Covers Android and Java\n        || className.startsWith(\"kotlin.\")\n        || className.startsWith(\"kotlinx.\")\n        || className.startsWith(\"scala.\");\n  }\n\n  /**\n   * Gets the result of applying all filters until the first one returns a result other than {@link\n   * FilterResult#INDECISIVE}, or {@link FilterResult#ALLOW} if the list of filters is empty or all\n   * returned {@code INDECISIVE}.\n   */\n  public static FilterResult getFilterResult(\n      List<ReflectionAccessFilter> reflectionFilters, Class<?> c) {\n    for (ReflectionAccessFilter filter : reflectionFilters) {\n      FilterResult result = filter.check(c);\n      if (result != FilterResult.INDECISIVE) {\n        return result;\n      }\n    }\n    return FilterResult.ALLOW;\n  }\n\n  /** See {@link AccessibleObject#canAccess(Object)} (Java >= 9) */\n  public static boolean canAccess(AccessibleObject accessibleObject, Object object) {\n    return AccessChecker.INSTANCE.canAccess(accessibleObject, object);\n  }\n\n  private abstract static class AccessChecker {\n    public static final AccessChecker INSTANCE;\n\n    static {\n      AccessChecker accessChecker = null;\n      // TODO: Ideally should use Multi-Release JAR for this version specific code\n      if (JavaVersion.isJava9OrLater()) {\n        try {\n          final Method canAccessMethod =\n              AccessibleObject.class.getDeclaredMethod(\"canAccess\", Object.class);\n          accessChecker =\n              new AccessChecker() {\n                @Override\n                public boolean canAccess(AccessibleObject accessibleObject, Object object) {\n                  try {\n                    return (Boolean) canAccessMethod.invoke(accessibleObject, object);\n                  } catch (Exception e) {\n                    throw new RuntimeException(\"Failed invoking canAccess\", e);\n                  }\n                }\n              };\n        } catch (NoSuchMethodException ignored) {\n          // OK: will assume everything is accessible\n        }\n      }\n\n      if (accessChecker == null) {\n        accessChecker =\n            new AccessChecker() {\n              @Override\n              public boolean canAccess(AccessibleObject accessibleObject, Object object) {\n                // Cannot determine whether object can be accessed, so assume it can be accessed\n                return true;\n              }\n            };\n      }\n      INSTANCE = accessChecker;\n    }\n\n    public abstract boolean canAccess(AccessibleObject accessibleObject, Object object);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/ReflectionAccessFilter.java\n/*\n * Copyright (C) 2022 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.gson.internal.ReflectionAccessFilterHelper;\nimport java.lang.reflect.AccessibleObject;\n\n/**\n * Filter for determining whether reflection based serialization and deserialization is allowed for\n * a class.\n *\n * <p>A filter can be useful in multiple scenarios, for example when upgrading to newer Java\n * versions which use the Java Platform Module System (JPMS). A filter then allows to {@linkplain\n * FilterResult#BLOCK_INACCESSIBLE prevent making inaccessible members accessible}, even if the used\n * Java version might still allow illegal access (but logs a warning), or if {@code java} command\n * line arguments are used to open the inaccessible packages to other parts of the application. This\n * interface defines some convenience filters for this task, such as {@link\n * #BLOCK_INACCESSIBLE_JAVA}.\n *\n * <p>A filter can also be useful to prevent mixing model classes of a project with other non-model\n * classes; the filter could {@linkplain FilterResult#BLOCK_ALL block all reflective access} to\n * non-model classes.\n *\n * <p>A reflection access filter is similar to an {@link ExclusionStrategy} with the major\n * difference that a filter will cause an exception to be thrown when access is disallowed while an\n * exclusion strategy just skips fields and classes.\n *\n * @see GsonBuilder#addReflectionAccessFilter(ReflectionAccessFilter)\n * @since 2.9.1\n */\npublic interface ReflectionAccessFilter {\n  /**\n   * Result of a filter check.\n   *\n   * @since 2.9.1\n   */\n  enum FilterResult {\n    /**\n     * Reflection access for the class is allowed.\n     *\n     * <p>Note that this does not affect the Java access checks in any way, it only permits Gson to\n     * try using reflection for a class. The Java runtime might still deny such access.\n     */\n    ALLOW,\n    /**\n     * The filter is indecisive whether reflection access should be allowed. The next registered\n     * filter will be consulted to get the result. If there is no next filter, this result acts like\n     * {@link #ALLOW}.\n     */\n    INDECISIVE,\n    /**\n     * Blocks reflection access if a member of the class is not accessible by default and would have\n     * to be made accessible. This is unaffected by any {@code java} command line arguments being\n     * used to make packages accessible, or by module declaration directives which <i>open</i> the\n     * complete module or certain packages for reflection and will consider such packages\n     * inaccessible.\n     *\n     * <p>Note that this <b>only works for Java 9 and higher</b>, for older Java versions its\n     * functionality will be limited and it might behave like {@link #ALLOW}. Access checks are only\n     * performed as defined by the Java Language Specification (<a\n     * href=\"https://docs.oracle.com/javase/specs/jls/se11/html/jls-6.html#jls-6.6\">JLS 11\n     * &sect;6.6</a>), restrictions imposed by a {@link SecurityManager} are not considered.\n     *\n     * <p>This result type is mainly intended to help enforce the access checks of the Java Platform\n     * Module System. It allows detecting illegal access, even if the used Java version would only\n     * log a warning, or is configured to open packages for reflection using command line arguments.\n     *\n     * @see AccessibleObject#canAccess(Object)\n     */\n    BLOCK_INACCESSIBLE,\n    /**\n     * Blocks all reflection access for the class. Other means for serializing and deserializing the\n     * class, such as a {@link TypeAdapter}, have to be used.\n     */\n    BLOCK_ALL\n  }\n\n  /**\n   * Blocks all reflection access to members of standard Java classes which are not accessible by\n   * default. However, reflection access is still allowed for classes for which all fields are\n   * accessible and which have an accessible no-args constructor (or for which an {@link\n   * InstanceCreator} has been registered).\n   *\n   * <p>If this filter encounters a class other than a standard Java class it returns {@link\n   * FilterResult#INDECISIVE}.\n   *\n   * <p>This filter is mainly intended to help enforcing the access checks of Java Platform Module\n   * System. It allows detecting illegal access, even if the used Java version would only log a\n   * warning, or is configured to open packages for reflection. However, this filter <b>only works\n   * for Java 9 and higher</b>, when using an older Java version its functionality will be limited.\n   *\n   * <p>Note that this filter might not cover all standard Java classes. Currently only classes in a\n   * {@code java.*} or {@code javax.*} package are considered. The set of detected classes might be\n   * expanded in the future without prior notice.\n   *\n   * @see FilterResult#BLOCK_INACCESSIBLE\n   */\n  ReflectionAccessFilter BLOCK_INACCESSIBLE_JAVA =\n      new ReflectionAccessFilter() {\n        @Override\n        public FilterResult check(Class<?> rawClass) {\n          return ReflectionAccessFilterHelper.isJavaType(rawClass)\n              ? FilterResult.BLOCK_INACCESSIBLE\n              : FilterResult.INDECISIVE;\n        }\n\n        @Override\n        public String toString() {\n          return \"ReflectionAccessFilter#BLOCK_INACCESSIBLE_JAVA\";\n        }\n      };\n\n  /**\n   * Blocks all reflection access to members of standard Java classes.\n   *\n   * <p>If this filter encounters a class other than a standard Java class it returns {@link\n   * FilterResult#INDECISIVE}.\n   *\n   * <p>This filter is mainly intended to prevent depending on implementation details of the Java\n   * platform and to help applications prepare for upgrading to the Java Platform Module System.\n   *\n   * <p>Note that this filter might not cover all standard Java classes. Currently only classes in a\n   * {@code java.*} or {@code javax.*} package are considered. The set of detected classes might be\n   * expanded in the future without prior notice.\n   *\n   * @see #BLOCK_INACCESSIBLE_JAVA\n   * @see FilterResult#BLOCK_ALL\n   */\n  ReflectionAccessFilter BLOCK_ALL_JAVA =\n      new ReflectionAccessFilter() {\n        @Override\n...\n", "needle_token_start": 598, "needle_token_end": 986, "code_context_ntokens": 15904, "output": ["```java\nprivate static void addTypeAdaptersForDate(\n      String datePattern, int dateStyle, int timeStyle, List<TypeAdapterFactory> factories) {\n    TypeAdapterFactory dateAdapterFactory;\n    boolean sqlTypesSupported = SqlTypesSupport.SUPPORTS_SQL_TYPES;\n    TypeAdapterFactory sqlTimestampAdapterFactory = null;\n    TypeAdapterFactory sqlDateAdapterFactory = null;\n\n    if (datePattern != null && !datePattern.trim().isEmpty()) {\n      dateAdapterFactory = DefaultDateTypeAdapter.DateType.DATE.createAdapterFactory(datePattern);\n\n      if (sqlTypesSupported) {\n        sqlTimestampAdapterFactory =\n            SqlTypesSupport.TIMESTAMP_DATE_TYPE.createAdapterFactory(datePattern);\n        sqlDateAdapterFactory = SqlTypesSupport.DATE_DATE_TYPE.createAdapterFactory(datePattern);\n      }\n    } else if (dateStyle != DateFormat.DEFAULT || timeStyle != DateFormat.DEFAULT) {\n      dateAdapterFactory =\n          DefaultDateTypeAdapter.DateType.DATE.createAdapterFactory(dateStyle, timeStyle);\n\n      if (sqlTypesSupported) {\n        sqlTimestampAdapterFactory =\n            SqlTypesSupport.TIMESTAMP_DATE_TYPE.createAdapterFactory(dateStyle, timeStyle);\n        sqlDateAdapterFactory =\n            SqlTypesSupport.DATE_DATE_TYPE.createAdapterFactory(dateStyle, timeStyle);\n      }\n    } else {\n      return;\n    }\n\n    factories.add(dateAdapterFactory);\n    if (sqlTypesSupported) {\n      factories.add(sqlTimestampAdapterFactory);\n      factories.add(sqlDateAdapterFactory);\n    }\n  }\n```"]}
{"repo": "google/gson", "name": "parseString", "language": "java", "path": "gson/src/main/java/com/google/gson/JsonParser.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To convert a JSON formatted string into a structured parse tree of elements that can be easily manipulated within a program.\n2. **Input**: A string formatted in JSON.\n3. **Output**: A structured parse tree of elements that represent the JSON data.\n4. **Procedure**: The function reads the input string using a reader tailored for JSON, processes it in a lenient mode allowing some flexibility in JSON syntax, and constructs a parse tree that represents all the elements and structures in the JSON string. If the input string is not valid JSON, it throws an exception indicating the syntax error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/com/google/gson/ToNumberPolicy.java\n/*\n * Copyright (C) 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.gson.internal.LazilyParsedNumber;\nimport com.google.gson.internal.NumberLimits;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.MalformedJsonException;\n...\n// Path: gson/src/main/java/com/google/gson/JsonStreamParser.java\n/*\n * Copyright (C) 2009 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson;\n\nimport com.google.gson.internal.Streams;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.MalformedJsonException;\nimport java.io.IOException;\nimport java.io.Reader;\nimport java.io.StringReader;\nimport java.util.Iterator;\nimport java.util.NoSuchElementException;\n\n/**\n * A streaming parser that allows reading of multiple {@link JsonElement}s from the specified reader\n * asynchronously. The JSON data is parsed in lenient mode, see also {@link\n * JsonReader#setStrictness(Strictness)}.\n *\n * <p>This class is conditionally thread-safe (see Item 70, Effective Java second edition). To\n * properly use this class across multiple threads, you will need to add some external\n * synchronization. For example:\n *\n * <pre>\n * JsonStreamParser parser = new JsonStreamParser(\"['first'] {'second':10} 'third'\");\n * JsonElement element;\n * synchronized (parser) {  // synchronize on an object shared by threads\n *   if (parser.hasNext()) {\n *     element = parser.next();\n *   }\n * }\n * </pre>\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @since 1.4\n */\npublic final class JsonStreamParser implements Iterator<JsonElement> {\n  private final JsonReader parser;\n  private final Object lock;\n\n  /**\n   * @param json The string containing JSON elements concatenated to each other.\n   * @since 1.4\n   */\n  public JsonStreamParser(String json) {\n    this(new StringReader(json));\n  }\n\n  /**\n   * @param reader The data stream containing JSON elements concatenated to each other.\n   * @since 1.4\n   */\n  public JsonStreamParser(Reader reader) {\n    parser = new JsonReader(reader);\n    parser.setStrictness(Strictness.LENIENT);\n    lock = new Object();\n  }\n\n  /**\n   * Returns the next available {@link JsonElement} on the reader. Throws a {@link\n   * NoSuchElementException} if no element is available.\n   *\n   * @return the next available {@code JsonElement} on the reader.\n   * @throws JsonParseException if the incoming stream is malformed JSON.\n   * @throws NoSuchElementException if no {@code JsonElement} is available.\n   * @since 1.4\n   */\n  @Override\n  public JsonElement next() throws JsonParseException {\n    if (!hasNext()) {\n      throw new NoSuchElementException();\n    }\n\n    try {\n      return Streams.parse(parser);\n    } catch (StackOverflowError e) {\n      throw new JsonParseException(\"Failed parsing JSON source to Json\", e);\n    } catch (OutOfMemoryError e) {\n      throw new JsonParseException(\"Failed parsing JSON source to Json\", e);\n    }\n  }\n\n  /**\n   * Returns true if a {@link JsonElement} is available on the input for consumption\n   *\n   * @return true if a {@link JsonElement} is available on the input, false otherwise\n   * @throws JsonParseException if the incoming stream is malformed JSON.\n   * @since 1.4\n   */\n  @Override\n  public boolean hasNext() {\n    synchronized (lock) {\n      try {\n        return parser.peek() != JsonToken.END_DOCUMENT;\n      } catch (MalformedJsonException e) {\n        throw new JsonSyntaxException(e);\n      } catch (IOException e) {\n        throw new JsonIOException(e);\n      }\n    }\n  }\n\n  /**\n   * This optional {@link Iterator} method is not relevant for stream parsing and hence is not\n   * implemented.\n   *\n   * @since 1.4\n   */\n  @Override\n  public void remove() {\n    throw new UnsupportedOperationException();\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/package-info.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * This package provides the {@link com.google.gson.Gson} class to convert Json to Java and\n * vice-versa.\n *\n * <p>The primary class to use is {@link com.google.gson.Gson} which can be constructed with {@code\n * new Gson()} (using default settings) or by using {@link com.google.gson.GsonBuilder} (to\n * configure various options such as using versioning and so on).\n *\n * @author Inderjeet Singh, Joel Leitch\n */\npackage com.google.gson;\n\n// Path: gson/src/main/java/com/google/gson/JsonParser.java\n/*\n * Copyright (C) 2009 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson;\n\nimport com.google.errorprone.annotations.InlineMe;\nimport com.google.gson.internal.Streams;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.MalformedJsonException;\nimport java.io.IOException;\nimport java.io.Reader;\nimport java.io.StringReader;\n\n/**\n * A parser to parse JSON into a parse tree of {@link JsonElement}s.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @since 1.3\n */\npublic final class JsonParser {\n  /**\n   * @deprecated No need to instantiate this class, use the static methods instead.\n   */\n  @Deprecated\n  public JsonParser() {}\n\n  /**\n   * Parses the specified JSON string into a parse tree. An exception is thrown if the JSON string\n   * has multiple top-level JSON elements, or if there is trailing data.\n   *\n   * <p>The JSON string is parsed in {@linkplain JsonReader#setStrictness(Strictness) lenient mode}.\n   *\n   * @param json JSON text\n   * @return a parse tree of {@link JsonElement}s corresponding to the specified JSON\n   * @throws JsonParseException if the specified text is not valid JSON\n   * @since 2.8.6\n   */\n  \npublic static JsonElement parseString(String json) throws JsonSyntaxException {\n    return parseReader(new StringReader(json));\n  }\n\n  /**\n   * Parses the complete JSON string provided by the reader into a parse tree. An exception is\n   * thrown if the JSON string has multiple top-level JSON elements, or if there is trailing data.\n   *\n   * <p>The JSON data is parsed in {@linkplain JsonReader#setStrictness(Strictness) lenient mode}.\n   *\n   * @param reader JSON text\n   * @return a parse tree of {@link JsonElement}s corresponding to the specified JSON\n   * @throws JsonParseException if there is an IOException or if the specified text is not valid\n   *     JSON\n   * @since 2.8.6\n   */\n  public static JsonElement parseReader(Reader reader) throws JsonIOException, JsonSyntaxException {\n    try {\n      JsonReader jsonReader = new JsonReader(reader);\n      JsonElement element = parseReader(jsonReader);\n      if (!element.isJsonNull() && jsonReader.peek() != JsonToken.END_DOCUMENT) {\n        throw new JsonSyntaxException(\"Did not consume the entire document.\");\n      }\n      return element;\n    } catch (MalformedJsonException e) {\n      throw new JsonSyntaxException(e);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    } catch (NumberFormatException e) {\n      throw new JsonSyntaxException(e);\n    }\n  }\n\n  /**\n   * Returns the next value from the JSON stream as a parse tree. Unlike the other {@code parse}\n   * methods, no exception is thrown if the JSON data has multiple top-level JSON elements, or if\n   * there is trailing data.\n   *\n   * <p>The JSON data is parsed in {@linkplain JsonReader#setStrictness(Strictness) lenient mode},\n   * regardless of the strictness setting of the provided reader. The strictness setting of the\n   * reader is restored once this method returns.\n   *\n   * @throws JsonParseException if there is an IOException or if the specified text is not valid\n   *     JSON\n   * @since 2.8.6\n   */\n  public static JsonElement parseReader(JsonReader reader)\n      throws JsonIOException, JsonSyntaxException {\n    Strictness strictness = reader.getStrictness();\n    reader.setStrictness(Strictness.LENIENT);\n    try {\n      return Streams.parse(reader);\n    } catch (StackOverflowError e) {\n      throw new JsonParseException(\"Failed parsing JSON source: \" + reader + \" to Json\", e);\n    } catch (OutOfMemoryError e) {\n      throw new JsonParseException(\"Failed parsing JSON source: \" + reader + \" to Json\", e);\n    } finally {\n      reader.setStrictness(strictness);\n    }\n  }\n\n  /**\n   * @deprecated Use {@link JsonParser#parseString}\n   */\n  @Deprecated\n  @InlineMe(replacement = \"JsonParser.parseString(json)\", imports = \"com.google.gson.JsonParser\")\n  public JsonElement parse(String json) throws JsonSyntaxException {\n    return parseString(json);\n  }\n\n  /**\n   * @deprecated Use {@link JsonParser#parseReader(Reader)}\n   */\n  @Deprecated\n  @InlineMe(replacement = \"JsonParser.parseReader(json)\", imports = \"com.google.gson.JsonParser\")\n  public JsonElement parse(Reader json) throws JsonIOException, JsonSyntaxException {\n    return parseReader(json);\n  }\n\n  /**\n   * @deprecated Use {@link JsonParser#parseReader(JsonReader)}\n   */\n  @Deprecated\n  @InlineMe(replacement = \"JsonParser.parseReader(json)\", imports = \"com.google.gson.JsonParser\")\n  public JsonElement parse(JsonReader json) throws JsonIOException, JsonSyntaxException {\n    return parseReader(json);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/stream/package-info.java\n/*\n * Copyright (C) 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/** This package provides classes for processing JSON in an efficient streaming way. */\npackage com.google.gson.stream;\n\n// Path: gson/src/main/java/com/google/gson/reflect/package-info.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * This package provides utility classes for finding type information for generic types.\n *\n * @author Inderjeet Singh, Joel Leitch\n */\npackage com.google.gson.reflect;\n\n// Path: gson/src/main/java/com/google/gson/internal/package-info.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Do NOT use any class in this package as they are meant for internal use in Gson. These classes\n * will very likely change incompatibly in future versions. You have been warned.\n *\n * @author Inderjeet Singh, Joel Leitch, Jesse Wilson\n */\npackage com.google.gson.internal;\n\n// Path: gson/src/main/java/com/google/gson/internal/bind/TypeAdapterRuntimeTypeWrapper.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson.internal.bind;\n\nimport com.google.gson.Gson;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\n\nfinal class TypeAdapterRuntimeTypeWrapper<T> extends TypeAdapter<T> {\n  private final Gson context;\n  private final TypeAdapter<T> delegate;\n  private final Type type;\n\n  TypeAdapterRuntimeTypeWrapper(Gson context, TypeAdapter<T> delegate, Type type) {\n    this.context = context;\n    this.delegate = delegate;\n    this.type = type;\n  }\n\n  @Override\n  public T read(JsonReader in) throws IOException {\n    return delegate.read(in);\n  }\n\n  @Override\n  public void write(JsonWriter out, T value) throws IOException {\n    // Order of preference for choosing type adapters\n    // First preference: a type adapter registered for the runtime type\n    // Second preference: a type adapter registered for the declared type\n    // Third preference: reflective type adapter for the runtime type\n    //                   (if it is a subclass of the declared type)\n    // Fourth preference: reflective type adapter for the declared type\n\n    TypeAdapter<T> chosen = delegate;\n    Type runtimeType = getRuntimeTypeIfMoreSpecific(type, value);\n    if (runtimeType != type) {\n      @SuppressWarnings(\"unchecked\")\n      TypeAdapter<T> runtimeTypeAdapter =\n          (TypeAdapter<T>) context.getAdapter(TypeToken.get(runtimeType));\n      // For backward compatibility only check ReflectiveTypeAdapterFactory.Adapter here but not any\n      // other wrapping adapters, see\n      // https://github.com/google/gson/pull/1787#issuecomment-1222175189\n      if (!(runtimeTypeAdapter instanceof ReflectiveTypeAdapterFactory.Adapter)) {\n        // The user registered a type adapter for the runtime type, so we will use that\n        chosen = runtimeTypeAdapter;\n      } else if (!isReflective(delegate)) {\n        // The user registered a type adapter for Base class, so we prefer it over the\n        // reflective type adapter for the runtime type\n        chosen = delegate;\n      } else {\n        // Use the type adapter for runtime type\n        chosen = runtimeTypeAdapter;\n      }\n    }\n    chosen.write(out, value);\n  }\n\n  /**\n   * Returns whether the type adapter uses reflection.\n   *\n   * @param typeAdapter the type adapter to check.\n   */\n  private static boolean isReflective(TypeAdapter<?> typeAdapter) {\n    // Run this in loop in case multiple delegating adapters are nested\n    while (typeAdapter instanceof SerializationDelegatingTypeAdapter) {\n      TypeAdapter<?> delegate =\n          ((SerializationDelegatingTypeAdapter<?>) typeAdapter).getSerializationDelegate();\n      // Break if adapter does not delegate serialization\n      if (delegate == typeAdapter) {\n        break;\n      }\n      typeAdapter = delegate;\n    }\n\n    return typeAdapter instanceof ReflectiveTypeAdapterFactory.Adapter;\n  }\n\n  /** Finds a compatible runtime type if it is more specific */\n  private static Type getRuntimeTypeIfMoreSpecific(Type type, Object value) {\n    if (value != null && (type instanceof Class<?> || type instanceof TypeVariable<?>)) {\n      type = value.getClass();\n    }\n    return type;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/annotations/package-info.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * This package provides annotations that can be used with {@link com.google.gson.Gson}.\n *\n * @author Inderjeet Singh, Joel Leitch\n */\npackage com.google.gson.annotations;\n\n", "needle_token_start": 2332, "needle_token_end": 2362, "code_context_ntokens": 5389, "output": ["```java\n/**\n * Parses the specified JSON string into a parse tree. An exception is thrown if the JSON string\n * has multiple top-level JSON elements, or if there is trailing data.\n *\n * <p>The JSON string is parsed in {@linkplain JsonReader#setStrictness(Strictness) lenient mode}.\n *\n * @param json JSON text\n * @return a parse tree of {@link JsonElement}s corresponding to the specified JSON\n * @throws JsonParseException if the specified text is not valid JSON\n * @since 2.8.6\n */\npublic static JsonElement parseString(String json) throws JsonSyntaxException {\n    return parseReader(new StringReader(json));\n}\n```"]}
{"repo": "google/gson", "name": "newBuilder", "language": "java", "path": "gson/src/main/java/com/google/gson/Gson.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To create a new builder instance that retains all custom settings and factory configurations from the current instance.\n2. **Input**: None.\n3. **Output**: A new instance of a builder with all the configurations and customizations copied from the existing instance.\n4. **Procedure**: The method initializes a new builder instance by passing the current instance to its constructor, ensuring that all customizations and settings are preserved in the new instance.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " *   <li>When using an existing {@code JsonWriter}, manually apply the writer settings of this\n *       {@code Gson} instance listed by {@link #newJsonWriter(Writer)}.<br>\n *       Otherwise, when not using an existing {@code JsonWriter}, use {@link\n *       #newJsonWriter(Writer)} to construct one.\n *   <li>Call {@link TypeAdapter#write(JsonWriter, Object)}\n * </ol>\n *\n * <h3>Deserialization</h3>\n *\n * <ol>\n *   <li>Use {@link #getAdapter(Class)} to obtain the adapter for the type to be deserialized\n *   <li>When using an existing {@code JsonReader}, manually apply the reader settings of this\n *       {@code Gson} instance listed by {@link #newJsonReader(Reader)}.<br>\n *       Otherwise, when not using an existing {@code JsonReader}, use {@link\n *       #newJsonReader(Reader)} to construct one.\n *   <li>Call {@link TypeAdapter#read(JsonReader)}\n *   <li>Call {@link JsonReader#peek()} and verify that the result is {@link JsonToken#END_DOCUMENT}\n *       to make sure there is no trailing data\n * </ol>\n *\n * Note that the {@code JsonReader} created this way is only 'legacy strict', it mostly adheres to\n * the JSON specification but allows small deviations. See {@link\n * JsonReader#setStrictness(Strictness)} for details.\n *\n * @see TypeToken\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @author Jesse Wilson\n */\npublic final class Gson {\n\n  static final boolean DEFAULT_JSON_NON_EXECUTABLE = false;\n  // Strictness of `null` is the legacy mode where some Gson APIs are always lenient\n  static final Strictness DEFAULT_STRICTNESS = null;\n  static final FormattingStyle DEFAULT_FORMATTING_STYLE = FormattingStyle.COMPACT;\n  static final boolean DEFAULT_ESCAPE_HTML = true;\n  static final boolean DEFAULT_SERIALIZE_NULLS = false;\n  static final boolean DEFAULT_COMPLEX_MAP_KEYS = false;\n  static final boolean DEFAULT_SPECIALIZE_FLOAT_VALUES = false;\n  static final boolean DEFAULT_USE_JDK_UNSAFE = true;\n  static final String DEFAULT_DATE_PATTERN = null;\n  static final FieldNamingStrategy DEFAULT_FIELD_NAMING_STRATEGY = FieldNamingPolicy.IDENTITY;\n  static final ToNumberStrategy DEFAULT_OBJECT_TO_NUMBER_STRATEGY = ToNumberPolicy.DOUBLE;\n  static final ToNumberStrategy DEFAULT_NUMBER_TO_NUMBER_STRATEGY =\n      ToNumberPolicy.LAZILY_PARSED_NUMBER;\n\n  private static final String JSON_NON_EXECUTABLE_PREFIX = \")]}'\\n\";\n\n  /**\n   * This thread local guards against reentrant calls to {@link #getAdapter(TypeToken)}. In certain\n   * object graphs, creating an adapter for a type may recursively require an adapter for the same\n   * type! Without intervention, the recursive lookup would stack overflow. We cheat by returning a\n   * proxy type adapter, {@link FutureTypeAdapter}, which is wired up once the initial adapter has\n   * been created.\n   *\n   * <p>The map stores the type adapters for ongoing {@code getAdapter} calls, with the type token\n   * provided to {@code getAdapter} as key and either {@code FutureTypeAdapter} or a regular {@code\n   * TypeAdapter} as value.\n   */\n  @SuppressWarnings(\"ThreadLocalUsage\")\n  private final ThreadLocal<Map<TypeToken<?>, TypeAdapter<?>>> threadLocalAdapterResults =\n      new ThreadLocal<>();\n\n  private final ConcurrentMap<TypeToken<?>, TypeAdapter<?>> typeTokenCache =\n      new ConcurrentHashMap<>();\n\n  private final ConstructorConstructor constructorConstructor;\n  private final JsonAdapterAnnotationTypeAdapterFactory jsonAdapterFactory;\n\n  final List<TypeAdapterFactory> factories;\n\n  final Excluder excluder;\n  final FieldNamingStrategy fieldNamingStrategy;\n  final Map<Type, InstanceCreator<?>> instanceCreators;\n  final boolean serializeNulls;\n  final boolean complexMapKeySerialization;\n  final boolean generateNonExecutableJson;\n  final boolean htmlSafe;\n  final FormattingStyle formattingStyle;\n  final Strictness strictness;\n  final boolean serializeSpecialFloatingPointValues;\n  final boolean useJdkUnsafe;\n  final String datePattern;\n  final int dateStyle;\n  final int timeStyle;\n  final LongSerializationPolicy longSerializationPolicy;\n  final List<TypeAdapterFactory> builderFactories;\n  final List<TypeAdapterFactory> builderHierarchyFactories;\n  final ToNumberStrategy objectToNumberStrategy;\n  final ToNumberStrategy numberToNumberStrategy;\n  final List<ReflectionAccessFilter> reflectionFilters;\n\n  /**\n   * Constructs a Gson object with default configuration. The default configuration has the\n   * following settings:\n   *\n   * <ul>\n   *   <li>The JSON generated by {@code toJson} methods is in compact representation. This means\n   *       that all the unneeded white-space is removed. You can change this behavior with {@link\n   *       GsonBuilder#setPrettyPrinting()}.\n   *   <li>When the JSON generated contains more than one line, the kind of newline and indent to\n   *       use can be configured with {@link GsonBuilder#setFormattingStyle(FormattingStyle)}.\n   *   <li>The generated JSON omits all the fields that are null. Note that nulls in arrays are kept\n   *       as is since an array is an ordered list. Moreover, if a field is not null, but its\n   *       generated JSON is empty, the field is kept. You can configure Gson to serialize null\n   *       values by setting {@link GsonBuilder#serializeNulls()}.\n   *   <li>Gson provides default serialization and deserialization for Enums, {@link Map}, {@link\n   *       java.net.URL}, {@link java.net.URI}, {@link java.util.Locale}, {@link java.util.Date},\n   *       {@link java.math.BigDecimal}, and {@link java.math.BigInteger} classes. If you would\n   *       prefer to change the default representation, you can do so by registering a type adapter\n   *       through {@link GsonBuilder#registerTypeAdapter(Type, Object)}.\n   *   <li>The default Date format is same as {@link java.text.DateFormat#DEFAULT}. This format\n   *       ignores the millisecond portion of the date during serialization. You can change this by\n   *       invoking {@link GsonBuilder#setDateFormat(int, int)} or {@link\n   *       GsonBuilder#setDateFormat(String)}.\n   *   <li>By default, Gson ignores the {@link com.google.gson.annotations.Expose} annotation. You\n   *       can enable Gson to serialize/deserialize only those fields marked with this annotation\n   *       through {@link GsonBuilder#excludeFieldsWithoutExposeAnnotation()}.\n   *   <li>By default, Gson ignores the {@link com.google.gson.annotations.Since} annotation. You\n   *       can enable Gson to use this annotation through {@link GsonBuilder#setVersion(double)}.\n   *   <li>The default field naming policy for the output JSON is same as in Java. So, a Java class\n   *       field {@code versionNumber} will be output as {@code \"versionNumber\"} in JSON. The same\n   *       rules are applied for mapping incoming JSON to the Java classes. You can change this\n   *       policy through {@link GsonBuilder#setFieldNamingPolicy(FieldNamingPolicy)}.\n   *   <li>By default, Gson excludes {@code transient} or {@code static} fields from consideration\n   *       for serialization and deserialization. You can change this behavior through {@link\n   *       GsonBuilder#excludeFieldsWithModifiers(int...)}.\n   *   <li>No explicit strictness is set. You can change this by calling {@link\n   *       GsonBuilder#setStrictness(Strictness)}.\n   * </ul>\n   */\n  public Gson() {\n    this(\n        Excluder.DEFAULT,\n        DEFAULT_FIELD_NAMING_STRATEGY,\n        Collections.<Type, InstanceCreator<?>>emptyMap(),\n        DEFAULT_SERIALIZE_NULLS,\n        DEFAULT_COMPLEX_MAP_KEYS,\n        DEFAULT_JSON_NON_EXECUTABLE,\n        DEFAULT_ESCAPE_HTML,\n        DEFAULT_FORMATTING_STYLE,\n        DEFAULT_STRICTNESS,\n        DEFAULT_SPECIALIZE_FLOAT_VALUES,\n        DEFAULT_USE_JDK_UNSAFE,\n        LongSerializationPolicy.DEFAULT,\n        DEFAULT_DATE_PATTERN,\n        DateFormat.DEFAULT,\n        DateFormat.DEFAULT,\n        Collections.<TypeAdapterFactory>emptyList(),\n        Collections.<TypeAdapterFactory>emptyList(),\n        Collections.<TypeAdapterFactory>emptyList(),\n        DEFAULT_OBJECT_TO_NUMBER_STRATEGY,\n        DEFAULT_NUMBER_TO_NUMBER_STRATEGY,\n        Collections.<ReflectionAccessFilter>emptyList());\n  }\n\n  Gson(\n      Excluder excluder,\n      FieldNamingStrategy fieldNamingStrategy,\n      Map<Type, InstanceCreator<?>> instanceCreators,\n      boolean serializeNulls,\n      boolean complexMapKeySerialization,\n      boolean generateNonExecutableGson,\n      boolean htmlSafe,\n      FormattingStyle formattingStyle,\n      Strictness strictness,\n      boolean serializeSpecialFloatingPointValues,\n      boolean useJdkUnsafe,\n      LongSerializationPolicy longSerializationPolicy,\n      String datePattern,\n      int dateStyle,\n      int timeStyle,\n      List<TypeAdapterFactory> builderFactories,\n      List<TypeAdapterFactory> builderHierarchyFactories,\n      List<TypeAdapterFactory> factoriesToBeAdded,\n      ToNumberStrategy objectToNumberStrategy,\n      ToNumberStrategy numberToNumberStrategy,\n      List<ReflectionAccessFilter> reflectionFilters) {\n    this.excluder = excluder;\n    this.fieldNamingStrategy = fieldNamingStrategy;\n    this.instanceCreators = instanceCreators;\n    this.constructorConstructor =\n        new ConstructorConstructor(instanceCreators, useJdkUnsafe, reflectionFilters);\n    this.serializeNulls = serializeNulls;\n    this.complexMapKeySerialization = complexMapKeySerialization;\n    this.generateNonExecutableJson = generateNonExecutableGson;\n    this.htmlSafe = htmlSafe;\n    this.formattingStyle = formattingStyle;\n    this.strictness = strictness;\n    this.serializeSpecialFloatingPointValues = serializeSpecialFloatingPointValues;\n    this.useJdkUnsafe = useJdkUnsafe;\n    this.longSerializationPolicy = longSerializationPolicy;\n    this.datePattern = datePattern;\n    this.dateStyle = dateStyle;\n    this.timeStyle = timeStyle;\n    this.builderFactories = builderFactories;\n    this.builderHierarchyFactories = builderHierarchyFactories;\n    this.objectToNumberStrategy = objectToNumberStrategy;\n    this.numberToNumberStrategy = numberToNumberStrategy;\n    this.reflectionFilters = reflectionFilters;\n\n    List<TypeAdapterFactory> factories = new ArrayList<>();\n\n    // built-in type adapters that cannot be overridden\n    factories.add(TypeAdapters.JSON_ELEMENT_FACTORY);\n    factories.add(ObjectTypeAdapter.getFactory(objectToNumberStrategy));\n\n    // the excluder must precede all adapters that handle user-defined types\n    factories.add(excluder);\n\n    // users' type adapters\n    factories.addAll(factoriesToBeAdded);\n\n    // type adapters for basic platform types\n    factories.add(TypeAdapters.STRING_FACTORY);\n    factories.add(TypeAdapters.INTEGER_FACTORY);\n    factories.add(TypeAdapters.BOOLEAN_FACTORY);\n    factories.add(TypeAdapters.BYTE_FACTORY);\n    factories.add(TypeAdapters.SHORT_FACTORY);\n    TypeAdapter<Number> longAdapter = longAdapter(longSerializationPolicy);\n    factories.add(TypeAdapters.newFactory(long.class, Long.class, longAdapter));\n    factories.add(\n        TypeAdapters.newFactory(\n            double.class, Double.class, doubleAdapter(serializeSpecialFloatingPointValues)));\n    factories.add(\n        TypeAdapters.newFactory(\n            float.class, Float.class, floatAdapter(serializeSpecialFloatingPointValues)));\n    factories.add(NumberTypeAdapter.getFactory(numberToNumberStrategy));\n    factories.add(TypeAdapters.ATOMIC_INTEGER_FACTORY);\n    factories.add(TypeAdapters.ATOMIC_BOOLEAN_FACTORY);\n    factories.add(TypeAdapters.newFactory(AtomicLong.class, atomicLongAdapter(longAdapter)));\n    factories.add(\n        TypeAdapters.newFactory(AtomicLongArray.class, atomicLongArrayAdapter(longAdapter)));\n    factories.add(TypeAdapters.ATOMIC_INTEGER_ARRAY_FACTORY);\n    factories.add(TypeAdapters.CHARACTER_FACTORY);\n    factories.add(TypeAdapters.STRING_BUILDER_FACTORY);\n    factories.add(TypeAdapters.STRING_BUFFER_FACTORY);\n    factories.add(TypeAdapters.newFactory(BigDecimal.class, TypeAdapters.BIG_DECIMAL));\n    factories.add(TypeAdapters.newFactory(BigInteger.class, TypeAdapters.BIG_INTEGER));\n    // Add adapter for LazilyParsedNumber because user can obtain it from Gson and then try to\n    // serialize it again\n    factories.add(\n        TypeAdapters.newFactory(LazilyParsedNumber.class, TypeAdapters.LAZILY_PARSED_NUMBER));\n    factories.add(TypeAdapters.URL_FACTORY);\n    factories.add(TypeAdapters.URI_FACTORY);\n    factories.add(TypeAdapters.UUID_FACTORY);\n    factories.add(TypeAdapters.CURRENCY_FACTORY);\n    factories.add(TypeAdapters.LOCALE_FACTORY);\n    factories.add(TypeAdapters.INET_ADDRESS_FACTORY);\n    factories.add(TypeAdapters.BIT_SET_FACTORY);\n    factories.add(DefaultDateTypeAdapter.DEFAULT_STYLE_FACTORY);\n    factories.add(TypeAdapters.CALENDAR_FACTORY);\n\n    if (SqlTypesSupport.SUPPORTS_SQL_TYPES) {\n      factories.add(SqlTypesSupport.TIME_FACTORY);\n      factories.add(SqlTypesSupport.DATE_FACTORY);\n      factories.add(SqlTypesSupport.TIMESTAMP_FACTORY);\n    }\n\n    factories.add(ArrayTypeAdapter.FACTORY);\n    factories.add(TypeAdapters.CLASS_FACTORY);\n\n    // type adapters for composite and user-defined types\n    factories.add(new CollectionTypeAdapterFactory(constructorConstructor));\n    factories.add(new MapTypeAdapterFactory(constructorConstructor, complexMapKeySerialization));\n    this.jsonAdapterFactory = new JsonAdapterAnnotationTypeAdapterFactory(constructorConstructor);\n    factories.add(jsonAdapterFactory);\n    factories.add(TypeAdapters.ENUM_FACTORY);\n    factories.add(\n        new ReflectiveTypeAdapterFactory(\n            constructorConstructor,\n            fieldNamingStrategy,\n            excluder,\n            jsonAdapterFactory,\n            reflectionFilters));\n\n    this.factories = Collections.unmodifiableList(factories);\n  }\n\n  /**\n   * Returns a new GsonBuilder containing all custom factories and configuration used by the current\n   * instance.\n   *\n   * @return a GsonBuilder instance.\n   * @since 2.8.3\n   */\n  \npublic GsonBuilder newBuilder() {\n    return new GsonBuilder(this);\n  }\n\n  /**\n   * @deprecated This method by accident exposes an internal Gson class; it might be removed in a\n   *     future version.\n   */\n  @Deprecated\n  public Excluder excluder() {\n    return excluder;\n  }\n\n  /**\n   * Returns the field naming strategy used by this Gson instance.\n   *\n   * @see GsonBuilder#setFieldNamingStrategy(FieldNamingStrategy)\n   */\n  public FieldNamingStrategy fieldNamingStrategy() {\n    return fieldNamingStrategy;\n  }\n\n  /**\n   * Returns whether this Gson instance is serializing JSON object properties with {@code null}\n   * values, or just omits them.\n   *\n   * @see GsonBuilder#serializeNulls()\n   */\n  public boolean serializeNulls() {\n    return serializeNulls;\n  }\n\n  /**\n   * Returns whether this Gson instance produces JSON output which is HTML-safe, that means all HTML\n   * characters are escaped.\n   *\n   * @see GsonBuilder#disableHtmlEscaping()\n   */\n  public boolean htmlSafe() {\n    return htmlSafe;\n  }\n\n  private TypeAdapter<Number> doubleAdapter(boolean serializeSpecialFloatingPointValues) {\n    if (serializeSpecialFloatingPointValues) {\n      return TypeAdapters.DOUBLE;\n    }\n    return new TypeAdapter<Number>() {\n      @Override\n      public Double read(JsonReader in) throws IOException {\n        if (in.peek() == JsonToken.NULL) {\n          in.nextNull();\n          return null;\n        }\n        return in.nextDouble();\n      }\n\n      @Override\n      public void write(JsonWriter out, Number value) throws IOException {\n        if (value == null) {\n          out.nullValue();\n          return;\n        }\n        double doubleValue = value.doubleValue();\n        checkValidFloatingPoint(doubleValue);\n        out.value(doubleValue);\n      }\n    };\n  }\n\n  private TypeAdapter<Number> floatAdapter(boolean serializeSpecialFloatingPointValues) {\n    if (serializeSpecialFloatingPointValues) {\n      return TypeAdapters.FLOAT;\n    }\n    return new TypeAdapter<Number>() {\n      @Override\n      public Float read(JsonReader in) throws IOException {\n        if (in.peek() == JsonToken.NULL) {\n          in.nextNull();\n          return null;\n        }\n        return (float) in.nextDouble();\n      }\n\n      @Override\n      public void write(JsonWriter out, Number value) throws IOException {\n        if (value == null) {\n          out.nullValue();\n          return;\n        }\n        float floatValue = value.floatValue();\n        checkValidFloatingPoint(floatValue);\n        // For backward compatibility don't call `JsonWriter.value(float)` because that method has\n        // been newly added and not all custom JsonWriter implementations might override it yet\n        Number floatNumber = value instanceof Float ? value : floatValue;\n        out.value(floatNumber);\n      }\n    };\n  }\n\n  static void checkValidFloatingPoint(double value) {\n    if (Double.isNaN(value) || Double.isInfinite(value)) {\n      throw new IllegalArgumentException(\n          value\n              + \" is not a valid double value as per JSON specification. To override this\"\n              + \" behavior, use GsonBuilder.serializeSpecialFloatingPointValues() method.\");\n    }\n  }\n\n  private static TypeAdapter<Number> longAdapter(LongSerializationPolicy longSerializationPolicy) {\n    if (longSerializationPolicy == LongSerializationPolicy.DEFAULT) {\n      return TypeAdapters.LONG;\n    }\n    return new TypeAdapter<Number>() {\n      @Override\n      public Number read(JsonReader in) throws IOException {\n        if (in.peek() == JsonToken.NULL) {\n          in.nextNull();\n          return null;\n        }\n        return in.nextLong();\n      }\n\n      @Override\n      public void write(JsonWriter out, Number value) throws IOException {\n        if (value == null) {\n          out.nullValue();\n          return;\n        }\n        out.value(value.toString());\n      }\n    };\n  }\n\n  private static TypeAdapter<AtomicLong> atomicLongAdapter(final TypeAdapter<Number> longAdapter) {\n    return new TypeAdapter<AtomicLong>() {\n      @Override\n      public void write(JsonWriter out, AtomicLong value) throws IOException {\n        longAdapter.write(out, value.get());\n      }\n\n      @Override\n      public AtomicLong read(JsonReader in) throws IOException {\n        Number value = longAdapter.read(in);\n        return new AtomicLong(value.longValue());\n      }\n    }.nullSafe();\n  }\n\n  private static TypeAdapter<AtomicLongArray> atomicLongArrayAdapter(\n      final TypeAdapter<Number> longAdapter) {\n    return new TypeAdapter<AtomicLongArray>() {\n      @Override\n      public void write(JsonWriter out, AtomicLongArray value) throws IOException {\n        out.beginArray();\n        for (int i = 0, length = value.length(); i < length; i++) {\n          longAdapter.write(out, value.get(i));\n        }\n        out.endArray();\n      }\n\n      @Override\n      public AtomicLongArray read(JsonReader in) throws IOException {\n        List<Long> list = new ArrayList<>();\n        in.beginArray();\n        while (in.hasNext()) {\n          long value = longAdapter.read(in).longValue();\n          list.add(value);\n        }\n        in.endArray();\n        int length = list.size();\n        AtomicLongArray array = new AtomicLongArray(length);\n        for (int i = 0; i < length; ++i) {\n          array.set(i, list.get(i));\n        }\n        return array;\n      }\n    }.nullSafe();\n  }\n\n  /**\n   * Returns the type adapter for {@code type}.\n   *\n   * <p>When calling this method concurrently from multiple threads and requesting an adapter for\n   * the same type this method may return different {@code TypeAdapter} instances. However, that\n   * should normally not be an issue because {@code TypeAdapter} implementations are supposed to be\n   * stateless.\n   *\n   * @throws IllegalArgumentException if this Gson instance cannot serialize and deserialize {@code\n   *     type}.\n   */\n  public <T> TypeAdapter<T> getAdapter(TypeToken<T> type) {\n    Objects.requireNonNull(type, \"type must not be null\");\n    TypeAdapter<?> cached = typeTokenCache.get(type);\n    if (cached != null) {\n      @SuppressWarnings(\"unchecked\")\n      TypeAdapter<T> adapter = (TypeAdapter<T>) cached;\n      return adapter;\n    }\n\n    Map<TypeToken<?>, TypeAdapter<?>> threadCalls = threadLocalAdapterResults.get();\n    boolean isInitialAdapterRequest = false;\n    if (threadCalls == null) {\n      threadCalls = new HashMap<>();\n      threadLocalAdapterResults.set(threadCalls);\n      isInitialAdapterRequest = true;\n    } else {\n      // the key and value type parameters always agree\n      @SuppressWarnings(\"unchecked\")\n      TypeAdapter<T> ongoingCall = (TypeAdapter<T>) threadCalls.get(type);\n      if (ongoingCall != null) {\n        return ongoingCall;\n      }\n    }\n\n    TypeAdapter<T> candidate = null;\n    try {\n      FutureTypeAdapter<T> call = new FutureTypeAdapter<>();\n      threadCalls.put(type, call);\n\n      for (TypeAdapterFactory factory : factories) {\n        candidate = factory.create(this, type);\n        if (candidate != null) {\n          call.setDelegate(candidate);\n          // Replace future adapter with actual adapter\n          threadCalls.put(type, candidate);\n          break;\n        }\n      }\n    } finally {\n      if (isInitialAdapterRequest) {\n        threadLocalAdapterResults.remove();\n      }\n    }\n\n    if (candidate == null) {\n      throw new IllegalArgumentException(\n          \"GSON (\" + GsonBuildConfig.VERSION + \") cannot handle \" + type);\n    }\n\n    if (isInitialAdapterRequest) {\n      /*\n       * Publish resolved adapters to all threads\n       * Can only do this for the initial request because cyclic dependency TypeA -> TypeB -> TypeA\n       * would otherwise publish adapter for TypeB which uses not yet resolved adapter for TypeA\n       * See https://github.com/google/gson/issues/625\n       */\n      typeTokenCache.putAll(threadCalls);\n    }\n    return candidate;\n  }\n\n  /**\n   * Returns the type adapter for {@code type}.\n   *\n   * @throws IllegalArgumentException if this Gson instance cannot serialize and deserialize {@code\n   *     type}.\n   */\n  public <T> TypeAdapter<T> getAdapter(Class<T> type) {\n    return getAdapter(TypeToken.get(type));\n  }\n\n  /**\n   * This method is used to get an alternate type adapter for the specified type. This is used to\n   * access a type adapter that is overridden by a {@link TypeAdapterFactory} that you may have\n   * registered. This feature is typically used when you want to register a type adapter that does a\n   * little bit of work but then delegates further processing to the Gson default type adapter. Here\n   * is an example:\n   *\n   * <p>Let's say we want to write a type adapter that counts the number of objects being read from\n   * or written to JSON. We can achieve this by writing a type adapter factory that uses the {@code\n   * getDelegateAdapter} method:\n   *\n   * <pre>{@code\n   * class StatsTypeAdapterFactory implements TypeAdapterFactory {\n   *   public int numReads = 0;\n   *   public int numWrites = 0;\n   *   public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {\n   *     final TypeAdapter<T> delegate = gson.getDelegateAdapter(this, type);\n   *     return new TypeAdapter<T>() {\n   *       public void write(JsonWriter out, T value) throws IOException {\n   *         ++numWrites;\n   *         delegate.write(out, value);\n   *       }\n   *       public T read(JsonReader in) throws IOException {\n   *         ++numReads;\n   *         return delegate.read(in);\n   *       }\n   *     };\n   *   }\n   * }\n   * }</pre>\n   *\n   * This factory can now be used like this:\n   *\n   * <pre>{@code\n   * StatsTypeAdapterFactory stats = new StatsTypeAdapterFactory();\n   * Gson gson = new GsonBuilder().registerTypeAdapterFactory(stats).create();\n   * // Call gson.toJson() and fromJson methods on objects\n   * System.out.println(\"Num JSON reads: \" + stats.numReads);\n   * System.out.println(\"Num JSON writes: \" + stats.numWrites);\n   * }</pre>\n   *\n   * Note that this call will skip all factories registered before {@code skipPast}. In case of\n   * multiple TypeAdapterFactories registered it is up to the caller of this function to ensure that\n   * the order of registration does not prevent this method from reaching a factory they would\n   * expect to reply from this call. Note that since you can not override the type adapter factories\n   * for some types, see {@link GsonBuilder#registerTypeAdapter(Type, Object)}, our stats factory\n   * will not count the number of instances of those types that will be read or written.\n   *\n   * <p>If {@code skipPast} is a factory which has neither been registered on the {@link\n   * GsonBuilder} nor specified with the {@link JsonAdapter @JsonAdapter} annotation on a class,\n   * then this method behaves as if {@link #getAdapter(TypeToken)} had been called. This also means\n   * that for fields with {@code @JsonAdapter} annotation this method behaves normally like {@code\n   * getAdapter} (except for corner cases where a custom {@link InstanceCreator} is used to create\n   * an instance of the factory).\n   *\n   * @param skipPast The type adapter factory that needs to be skipped while searching for a\n   *     matching type adapter. In most cases, you should just pass <i>this</i> (the type adapter\n   *     factory from where {@code getDelegateAdapter} method is being invoked).\n   * @param type Type for which the delegate adapter is being searched for.\n   * @since 2.2\n   */\n  public <T> TypeAdapter<T> getDelegateAdapter(TypeAdapterFactory skipPast, TypeToken<T> type) {\n    Objects.requireNonNull(skipPast, \"skipPast must not be null\");\n    Objects.requireNonNull(type, \"type must not be null\");\n\n    if (jsonAdapterFactory.isClassJsonAdapterFactory(type, skipPast)) {\n      skipPast = jsonAdapterFactory;\n    }\n\n    boolean skipPastFound = false;\n    for (TypeAdapterFactory factory : factories) {\n      if (!skipPastFound) {\n        if (factory == skipPast) {\n          skipPastFound = true;\n        }\n        continue;\n      }\n\n      TypeAdapter<T> candidate = factory.create(this, type);\n      if (candidate != null) {\n        return candidate;\n      }\n    }\n\n    if (skipPastFound) {\n      throw new IllegalArgumentException(\"GSON cannot serialize or deserialize \" + type);\n    } else {\n      // Probably a factory from @JsonAdapter on a field\n      return getAdapter(type);\n    }\n  }\n\n  /**\n   * This method serializes the specified object into its equivalent representation as a tree of\n   * {@link JsonElement}s. This method should be used when the specified object is not a generic\n   * type. This method uses {@link Class#getClass()} to get the type for the specified object, but\n   * the {@code getClass()} loses the generic type information because of the Type Erasure feature\n   * of Java. Note that this method works fine if any of the object fields are of generic type, just\n   * the object itself should not be of a generic type. If the object is of generic type, use {@link\n   * #toJsonTree(Object, Type)} instead.\n   *\n   * @param src the object for which JSON representation is to be created\n   * @return JSON representation of {@code src}.\n   * @since 1.4\n   * @see #toJsonTree(Object, Type)\n   */\n  public JsonElement toJsonTree(Object src) {\n    if (src == null) {\n      return JsonNull.INSTANCE;\n    }\n    return toJsonTree(src, src.getClass());\n  }\n\n  /**\n   * This method serializes the specified object, including those of generic types, into its\n   * equivalent representation as a tree of {@link JsonElement}s. This method must be used if the\n   * specified object is a generic type. For non-generic objects, use {@link #toJsonTree(Object)}\n   * instead.\n   *\n   * @param src the object for which JSON representation is to be created\n   * @param typeOfSrc The specific genericized type of src. You can obtain this type by using the\n   *     {@link com.google.gson.reflect.TypeToken} class. For example, to get the type for {@code\n   *     Collection<Foo>}, you should use:\n   *     <pre>\n   * Type typeOfSrc = new TypeToken&lt;Collection&lt;Foo&gt;&gt;(){}.getType();\n   * </pre>\n   *\n   * @return JSON representation of {@code src}.\n   * @since 1.4\n   * @see #toJsonTree(Object)\n   */\n  public JsonElement toJsonTree(Object src, Type typeOfSrc) {\n    JsonTreeWriter writer = new JsonTreeWriter();\n    toJson(src, typeOfSrc, writer);\n    return writer.get();\n  }\n\n  /**\n   * This method serializes the specified object into its equivalent JSON representation. This\n   * method should be used when the specified object is not a generic type. This method uses {@link\n   * Class#getClass()} to get the type for the specified object, but the {@code getClass()} loses\n   * the generic type information because of the Type Erasure feature of Java. Note that this method\n   * works fine if any of the object fields are of generic type, just the object itself should not\n   * be of a generic type. If the object is of generic type, use {@link #toJson(Object, Type)}\n   * instead. If you want to write out the object to a {@link Writer}, use {@link #toJson(Object,\n   * Appendable)} instead.\n   *\n   * @param src the object for which JSON representation is to be created\n   * @return JSON representation of {@code src}.\n   * @see #toJson(Object, Appendable)\n   * @see #toJson(Object, Type)\n   */\n  public String toJson(Object src) {\n    if (src == null) {\n      return toJson(JsonNull.INSTANCE);\n    }\n    return toJson(src, src.getClass());\n  }\n\n  /**\n   * This method serializes the specified object, including those of generic types, into its\n   * equivalent JSON representation. This method must be used if the specified object is a generic\n   * type. For non-generic objects, use {@link #toJson(Object)} instead. If you want to write out\n   * the object to a {@link Appendable}, use {@link #toJson(Object, Type, Appendable)} instead.\n   *\n   * @param src the object for which JSON representation is to be created\n   * @param typeOfSrc The specific genericized type of src. You can obtain this type by using the\n   *     {@link com.google.gson.reflect.TypeToken} class. For example, to get the type for {@code\n   *     Collection<Foo>}, you should use:\n   *     <pre>\n   * Type typeOfSrc = new TypeToken&lt;Collection&lt;Foo&gt;&gt;(){}.getType();\n   * </pre>\n   *\n   * @return JSON representation of {@code src}.\n   * @see #toJson(Object, Type, Appendable)\n   * @see #toJson(Object)\n   */\n  public String toJson(Object src, Type typeOfSrc) {\n    StringWriter writer = new StringWriter();\n    toJson(src, typeOfSrc, writer);\n    return writer.toString();\n  }\n\n  /**\n   * This method serializes the specified object into its equivalent JSON representation and writes\n   * it to the writer. This method should be used when the specified object is not a generic type.\n   * This method uses {@link Class#getClass()} to get the type for the specified object, but the\n   * {@code getClass()} loses the generic type information because of the Type Erasure feature of\n   * Java. Note that this method works fine if any of the object fields are of generic type, just\n   * the object itself should not be of a generic type. If the object is of generic type, use {@link\n   * #toJson(Object, Type, Appendable)} instead.\n   *\n   * @param src the object for which JSON representation is to be created\n   * @param writer Writer to which the JSON representation needs to be written\n   * @throws JsonIOException if there was a problem writing to the writer\n   * @since 1.2\n   * @see #toJson(Object)\n   * @see #toJson(Object, Type, Appendable)\n   */\n  public void toJson(Object src, Appendable writer) throws JsonIOException {\n    if (src != null) {\n      toJson(src, src.getClass(), writer);\n    } else {\n      toJson(JsonNull.INSTANCE, writer);\n    }\n  }\n\n  /**\n   * This method serializes the specified object, including those of generic types, into its\n   * equivalent JSON representation and writes it to the writer. This method must be used if the\n   * specified object is a generic type. For non-generic objects, use {@link #toJson(Object,\n   * Appendable)} instead.\n   *\n   * @param src the object for which JSON representation is to be created\n   * @param typeOfSrc The specific genericized type of src. You can obtain this type by using the\n   *     {@link com.google.gson.reflect.TypeToken} class. For example, to get the type for {@code\n   *     Collection<Foo>}, you should use:\n   *     <pre>\n   * Type typeOfSrc = new TypeToken&lt;Collection&lt;Foo&gt;&gt;(){}.getType();\n   * </pre>\n   *\n   * @param writer Writer to which the JSON representation of src needs to be written\n   * @throws JsonIOException if there was a problem writing to the writer\n   * @since 1.2\n   * @see #toJson(Object, Type)\n   * @see #toJson(Object, Appendable)\n   */\n  public void toJson(Object src, Type typeOfSrc, Appendable writer) throws JsonIOException {\n    try {\n      JsonWriter jsonWriter = newJsonWriter(Streams.writerForAppendable(writer));\n      toJson(src, typeOfSrc, jsonWriter);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    }\n  }\n\n  /**\n   * Writes the JSON representation of {@code src} of type {@code typeOfSrc} to {@code writer}.\n   *\n   * <p>If the {@code Gson} instance has an {@linkplain GsonBuilder#setStrictness(Strictness)\n   * explicit strictness setting}, this setting will be used for writing the JSON regardless of the\n   * {@linkplain JsonWriter#getStrictness() strictness} of the provided {@link JsonWriter}. For\n   * legacy reasons, if the {@code Gson} instance has no explicit strictness setting and the writer\n   * does not have the strictness {@link Strictness#STRICT}, the JSON will be written in {@link\n   * Strictness#LENIENT} mode.<br>\n   * Note that in all cases the old strictness setting of the writer will be restored when this\n   * method returns.\n   *\n   * <p>The 'HTML-safe' and 'serialize {@code null}' settings of this {@code Gson} instance\n   * (configured by the {@link GsonBuilder}) are applied, and the original settings of the writer\n   * are restored once this method returns.\n   *\n   * @param src the object for which JSON representation is to be created\n   * @param typeOfSrc the type of the object to be written\n   * @param writer Writer to which the JSON representation of src needs to be written\n   * @throws JsonIOException if there was a problem writing to the writer\n   */\n  public void toJson(Object src, Type typeOfSrc, JsonWriter writer) throws JsonIOException {\n    @SuppressWarnings(\"unchecked\")\n    TypeAdapter<Object> adapter = (TypeAdapter<Object>) getAdapter(TypeToken.get(typeOfSrc));\n\n    Strictness oldStrictness = writer.getStrictness();\n    if (this.strictness != null) {\n      writer.setStrictness(this.strictness);\n    } else if (writer.getStrictness() != Strictness.STRICT) {\n      writer.setStrictness(Strictness.LENIENT);\n    }\n\n    boolean oldHtmlSafe = writer.isHtmlSafe();\n    boolean oldSerializeNulls = writer.getSerializeNulls();\n\n    writer.setHtmlSafe(htmlSafe);\n    writer.setSerializeNulls(serializeNulls);\n    try {\n      adapter.write(writer, src);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    } catch (AssertionError e) {\n      throw new AssertionError(\n          \"AssertionError (GSON \" + GsonBuildConfig.VERSION + \"): \" + e.getMessage(), e);\n    } finally {\n      writer.setStrictness(oldStrictness);\n      writer.setHtmlSafe(oldHtmlSafe);\n      writer.setSerializeNulls(oldSerializeNulls);\n    }\n  }\n\n  /**\n   * Converts a tree of {@link JsonElement}s into its equivalent JSON representation.\n   *\n   * @param jsonElement root of a tree of {@link JsonElement}s\n   * @return JSON String representation of the tree.\n   * @since 1.4\n   */\n  public String toJson(JsonElement jsonElement) {\n    StringWriter writer = new StringWriter();\n    toJson(jsonElement, writer);\n    return writer.toString();\n  }\n\n  /**\n   * Writes out the equivalent JSON for a tree of {@link JsonElement}s.\n   *\n   * @param jsonElement root of a tree of {@link JsonElement}s\n   * @param writer Writer to which the JSON representation needs to be written\n   * @throws JsonIOException if there was a problem writing to the writer\n   * @since 1.4\n   */\n  public void toJson(JsonElement jsonElement, Appendable writer) throws JsonIOException {\n    try {\n      JsonWriter jsonWriter = newJsonWriter(Streams.writerForAppendable(writer));\n      toJson(jsonElement, jsonWriter);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    }\n  }\n\n  /**\n   * Writes the JSON for {@code jsonElement} to {@code writer}.\n   *\n   * <p>If the {@code Gson} instance has an {@linkplain GsonBuilder#setStrictness(Strictness)\n   * explicit strictness setting}, this setting will be used for writing the JSON regardless of the\n   * {@linkplain JsonWriter#getStrictness() strictness} of the provided {@link JsonWriter}. For\n   * legacy reasons, if the {@code Gson} instance has no explicit strictness setting and the writer\n   * does not have the strictness {@link Strictness#STRICT}, the JSON will be written in {@link\n   * Strictness#LENIENT} mode.<br>\n   * Note that in all cases the old strictness setting of the writer will be restored when this\n   * method returns.\n   *\n   * <p>The 'HTML-safe' and 'serialize {@code null}' settings of this {@code Gson} instance\n   * (configured by the {@link GsonBuilder}) are applied, and the original settings of the writer\n   * are restored once this method returns.\n   *\n   * @param jsonElement the JSON element to be written\n   * @param writer the JSON writer to which the provided element will be written\n   * @throws JsonIOException if there was a problem writing to the writer\n   */\n  public void toJson(JsonElement jsonElement, JsonWriter writer) throws JsonIOException {\n    Strictness oldStrictness = writer.getStrictness();\n    boolean oldHtmlSafe = writer.isHtmlSafe();\n    boolean oldSerializeNulls = writer.getSerializeNulls();\n\n    writer.setHtmlSafe(htmlSafe);\n    writer.setSerializeNulls(serializeNulls);\n\n    if (this.strictness != null) {\n      writer.setStrictness(this.strictness);\n    } else if (writer.getStrictness() != Strictness.STRICT) {\n      writer.setStrictness(Strictness.LENIENT);\n    }\n\n    try {\n      Streams.write(jsonElement, writer);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    } catch (AssertionError e) {\n      throw new AssertionError(\n          \"AssertionError (GSON \" + GsonBuildConfig.VERSION + \"): \" + e.getMessage(), e);\n    } finally {\n      writer.setStrictness(oldStrictness);\n      writer.setHtmlSafe(oldHtmlSafe);\n      writer.setSerializeNulls(oldSerializeNulls);\n    }\n  }\n\n  /**\n   * Returns a new JSON writer configured for the settings on this Gson instance.\n   *\n   * <p>The following settings are considered:\n   *\n   * <ul>\n   *   <li>{@link GsonBuilder#disableHtmlEscaping()}\n   *   <li>{@link GsonBuilder#generateNonExecutableJson()}\n   *   <li>{@link GsonBuilder#serializeNulls()}\n   *   <li>{@link GsonBuilder#setStrictness(Strictness)}. If no {@linkplain\n   *       GsonBuilder#setStrictness(Strictness) explicit strictness has been set} the created\n   *       writer will have a strictness of {@link Strictness#LEGACY_STRICT}. Otherwise, the\n   *       strictness of the {@code Gson} instance will be used for the created writer.\n   *   <li>{@link GsonBuilder#setPrettyPrinting()}\n   *   <li>{@link GsonBuilder#setFormattingStyle(FormattingStyle)}\n   * </ul>\n   */\n  public JsonWriter newJsonWriter(Writer writer) throws IOException {\n    if (generateNonExecutableJson) {\n      writer.write(JSON_NON_EXECUTABLE_PREFIX);\n    }\n    JsonWriter jsonWriter = new JsonWriter(writer);\n    jsonWriter.setFormattingStyle(formattingStyle);\n    jsonWriter.setHtmlSafe(htmlSafe);\n    jsonWriter.setStrictness(strictness == null ? Strictness.LEGACY_STRICT : strictness);\n    jsonWriter.setSerializeNulls(serializeNulls);\n    return jsonWriter;\n  }\n\n  /**\n   * Returns a new JSON reader configured for the settings on this Gson instance.\n   *\n   * <p>The following settings are considered:\n   *\n   * <ul>\n   *   <li>{@link GsonBuilder#setStrictness(Strictness)}. If no {@linkplain\n   *       GsonBuilder#setStrictness(Strictness) explicit strictness has been set} the created\n   *       reader will have a strictness of {@link Strictness#LEGACY_STRICT}. Otherwise, the\n   *       strictness of the {@code Gson} instance will be used for the created reader.\n   * </ul>\n   */\n  public JsonReader newJsonReader(Reader reader) {\n    JsonReader jsonReader = new JsonReader(reader);\n    jsonReader.setStrictness(strictness == null ? Strictness.LEGACY_STRICT : strictness);\n    return jsonReader;\n  }\n\n  /**\n   * This method deserializes the specified JSON into an object of the specified class. It is not\n   * suitable to use if the specified class is a generic type since it will not have the generic\n   * type information because of the Type Erasure feature of Java. Therefore, this method should not\n   * be used if the desired type is a generic type. Note that this method works fine if any of the\n   * fields of the specified object are generics, just the object itself should not be a generic\n   * type. For the cases when the object is of generic type, invoke {@link #fromJson(String,\n   * TypeToken)}. If you have the JSON in a {@link Reader} instead of a String, use {@link\n   * #fromJson(Reader, Class)} instead.\n   *\n   * <p>An exception is thrown if the JSON string has multiple top-level JSON elements, or if there\n   * is trailing data. Use {@link #fromJson(JsonReader, Type)} if this behavior is not desired.\n   *\n   * @param <T> the type of the desired object\n   * @param json the string from which the object is to be deserialized\n   * @param classOfT the class of T\n   * @return an object of type T from the string. Returns {@code null} if {@code json} is {@code\n   *     null} or if {@code json} is empty.\n   * @throws JsonSyntaxException if json is not a valid representation for an object of type\n   *     classOfT\n   * @see #fromJson(Reader, Class)\n   * @see #fromJson(String, TypeToken)\n   */\n  public <T> T fromJson(String json, Class<T> classOfT) throws JsonSyntaxException {\n    T object = fromJson(json, TypeToken.get(classOfT));\n    return Primitives.wrap(classOfT).cast(object);\n  }\n\n  /**\n   * This method deserializes the specified JSON into an object of the specified type. This method\n   * is useful if the specified object is a generic type. For non-generic objects, use {@link\n   * #fromJson(String, Class)} instead. If you have the JSON in a {@link Reader} instead of a\n   * String, use {@link #fromJson(Reader, Type)} instead.\n   *\n   * <p>Since {@code Type} is not parameterized by T, this method is not type-safe and should be\n   * used carefully. If you are creating the {@code Type} from a {@link TypeToken}, prefer using\n   * {@link #fromJson(String, TypeToken)} instead since its return type is based on the {@code\n   * TypeToken} and is therefore more type-safe.\n   *\n   * <p>An exception is thrown if the JSON string has multiple top-level JSON elements, or if there\n   * is trailing data. Use {@link #fromJson(JsonReader, Type)} if this behavior is not desired.\n   *\n   * @param <T> the type of the desired object\n   * @param json the string from which the object is to be deserialized\n   * @param typeOfT The specific genericized type of src\n   * @return an object of type T from the string. Returns {@code null} if {@code json} is {@code\n   *     null} or if {@code json} is empty.\n   * @throws JsonSyntaxException if json is not a valid representation for an object of type typeOfT\n   * @see #fromJson(Reader, Type)\n   * @see #fromJson(String, Class)\n   * @see #fromJson(String, TypeToken)\n   */\n  @SuppressWarnings({\"unchecked\", \"TypeParameterUnusedInFormals\"})\n  public <T> T fromJson(String json, Type typeOfT) throws JsonSyntaxException {\n    return (T) fromJson(json, TypeToken.get(typeOfT));\n  }\n\n  /**\n   * This method deserializes the specified JSON into an object of the specified type. This method\n   * is useful if the specified object is a generic type. For non-generic objects, use {@link\n   * #fromJson(String, Class)} instead. If you have the JSON in a {@link Reader} instead of a\n   * String, use {@link #fromJson(Reader, TypeToken)} instead.\n   *\n   * <p>An exception is thrown if the JSON string has multiple top-level JSON elements, or if there\n   * is trailing data. Use {@link #fromJson(JsonReader, TypeToken)} if this behavior is not desired.\n   *\n   * @param <T> the type of the desired object\n   * @param json the string from which the object is to be deserialized\n   * @param typeOfT The specific genericized type of src. You should create an anonymous subclass of\n   *     {@code TypeToken} with the specific generic type arguments. For example, to get the type\n   *     for {@code Collection<Foo>}, you should use:\n   *     <pre>\n   * new TypeToken&lt;Collection&lt;Foo&gt;&gt;(){}\n   * </pre>\n   *\n   * @return an object of type T from the string. Returns {@code null} if {@code json} is {@code\n   *     null} or if {@code json} is empty.\n   * @throws JsonSyntaxException if json is not a valid representation for an object of the type\n   *     typeOfT\n   * @see #fromJson(Reader, TypeToken)\n   * @see #fromJson(String, Class)\n   * @since 2.10\n   */\n  public <T> T fromJson(String json, TypeToken<T> typeOfT) throws JsonSyntaxException {\n    if (json == null) {\n      return null;\n    }\n    StringReader reader = new StringReader(json);\n    return fromJson(reader, typeOfT);\n  }\n\n  /**\n   * This method deserializes the JSON read from the specified reader into an object of the\n   * specified class. It is not suitable to use if the specified class is a generic type since it\n   * will not have the generic type information because of the Type Erasure feature of Java.\n   * Therefore, this method should not be used if the desired type is a generic type. Note that this\n   * method works fine if any of the fields of the specified object are generics, just the object\n   * itself should not be a generic type. For the cases when the object is of generic type, invoke\n   * {@link #fromJson(Reader, TypeToken)}. If you have the JSON in a String form instead of a {@link\n   * Reader}, use {@link #fromJson(String, Class)} instead.\n   *\n   * <p>An exception is thrown if the JSON data has multiple top-level JSON elements, or if there is\n   * trailing data. Use {@link #fromJson(JsonReader, Type)} if this behavior is not desired.\n   *\n   * @param <T> the type of the desired object\n   * @param json the reader producing the JSON from which the object is to be deserialized.\n   * @param classOfT the class of T\n   * @return an object of type T from the Reader. Returns {@code null} if {@code json} is at EOF.\n   * @throws JsonIOException if there was a problem reading from the Reader\n   * @throws JsonSyntaxException if json is not a valid representation for an object of type typeOfT\n   * @since 1.2\n   * @see #fromJson(String, Class)\n   * @see #fromJson(Reader, TypeToken)\n   */\n  public <T> T fromJson(Reader json, Class<T> classOfT)\n      throws JsonSyntaxException, JsonIOException {\n    T object = fromJson(json, TypeToken.get(classOfT));\n    return Primitives.wrap(classOfT).cast(object);\n  }\n\n  /**\n   * This method deserializes the JSON read from the specified reader into an object of the\n   * specified type. This method is useful if the specified object is a generic type. For\n   * non-generic objects, use {@link #fromJson(Reader, Class)} instead. If you have the JSON in a\n   * String form instead of a {@link Reader}, use {@link #fromJson(String, Type)} instead.\n   *\n   * <p>Since {@code Type} is not parameterized by T, this method is not type-safe and should be\n   * used carefully. If you are creating the {@code Type} from a {@link TypeToken}, prefer using\n   * {@link #fromJson(Reader, TypeToken)} instead since its return type is based on the {@code\n   * TypeToken} and is therefore more type-safe.\n   *\n   * <p>An exception is thrown if the JSON data has multiple top-level JSON elements, or if there is\n   * trailing data. Use {@link #fromJson(JsonReader, Type)} if this behavior is not desired.\n   *\n   * @param <T> the type of the desired object\n   * @param json the reader producing JSON from which the object is to be deserialized\n   * @param typeOfT The specific genericized type of src\n   * @return an object of type T from the Reader. Returns {@code null} if {@code json} is at EOF.\n   * @throws JsonIOException if there was a problem reading from the Reader\n   * @throws JsonSyntaxException if json is not a valid representation for an object of type typeOfT\n   * @since 1.2\n   * @see #fromJson(String, Type)\n   * @see #fromJson(Reader, Class)\n   * @see #fromJson(Reader, TypeToken)\n   */\n  @SuppressWarnings({\"unchecked\", \"TypeParameterUnusedInFormals\"})\n  public <T> T fromJson(Reader json, Type typeOfT) throws JsonIOException, JsonSyntaxException {\n    return (T) fromJson(json, TypeToken.get(typeOfT));\n  }\n\n  /**\n   * This method deserializes the JSON read from the specified reader into an object of the\n   * specified type. This method is useful if the specified object is a generic type. For\n   * non-generic objects, use {@link #fromJson(Reader, Class)} instead. If you have the JSON in a\n   * String form instead of a {@link Reader}, use {@link #fromJson(String, TypeToken)} instead.\n   *\n   * <p>An exception is thrown if the JSON data has multiple top-level JSON elements, or if there is\n   * trailing data. Use {@link #fromJson(JsonReader, TypeToken)} if this behavior is not desired.\n   *\n   * @param <T> the type of the desired object\n   * @param json the reader producing JSON from which the object is to be deserialized\n   * @param typeOfT The specific genericized type of src. You should create an anonymous subclass of\n   *     {@code TypeToken} with the specific generic type arguments. For example, to get the type\n   *     for {@code Collection<Foo>}, you should use:\n   *     <pre>\n   * new TypeToken&lt;Collection&lt;Foo&gt;&gt;(){}\n   * </pre>\n   *\n   * @return an object of type T from the Reader. Returns {@code null} if {@code json} is at EOF.\n   * @throws JsonIOException if there was a problem reading from the Reader\n   * @throws JsonSyntaxException if json is not a valid representation for an object of type of\n   *     typeOfT\n   * @see #fromJson(String, TypeToken)\n   * @see #fromJson(Reader, Class)\n   * @since 2.10\n   */\n  public <T> T fromJson(Reader json, TypeToken<T> typeOfT)\n      throws JsonIOException, JsonSyntaxException {\n    JsonReader jsonReader = newJsonReader(json);\n    T object = fromJson(jsonReader, typeOfT);\n    assertFullConsumption(object, jsonReader);\n    return object;\n  }\n\n  // fromJson(JsonReader, Class) is unfortunately missing and cannot be added now without breaking\n  // source compatibility in certain cases, see\n  // https://github.com/google/gson/pull/1700#discussion_r973764414\n\n  /**\n   * Reads the next JSON value from {@code reader} and converts it to an object of type {@code\n   * typeOfT}. Returns {@code null}, if the {@code reader} is at EOF.\n   *\n   * <p>Since {@code Type} is not parameterized by T, this method is not type-safe and should be\n   * used carefully. If you are creating the {@code Type} from a {@link TypeToken}, prefer using\n   * {@link #fromJson(JsonReader, TypeToken)} instead since its return type is based on the {@code\n   * TypeToken} and is therefore more type-safe. If the provided type is a {@code Class} the {@code\n   * TypeToken} can be created with {@link TypeToken#get(Class)}.\n   *\n   * <p>Unlike the other {@code fromJson} methods, no exception is thrown if the JSON data has\n   * multiple top-level JSON elements, or if there is trailing data.\n   *\n   * <p>If the {@code Gson} instance has an {@linkplain GsonBuilder#setStrictness(Strictness)\n   * explicit strictness setting}, this setting will be used for reading the JSON regardless of the\n   * {@linkplain JsonReader#getStrictness() strictness} of the provided {@link JsonReader}. For\n   * legacy reasons, if the {@code Gson} instance has no explicit strictness setting and the reader\n   * does not have the strictness {@link Strictness#STRICT}, the JSON will be written in {@link\n   * Strictness#LENIENT} mode.<br>\n   * Note that in all cases the old strictness setting of the reader will be restored when this\n   * method returns.\n   *\n   * @param <T> the type of the desired object\n   * @param reader the reader whose next JSON value should be deserialized\n   * @param typeOfT The specific genericized type of src\n   * @return an object of type T from the JsonReader. Returns {@code null} if {@code reader} is at\n   *     EOF.\n   * @throws JsonIOException if there was a problem reading from the JsonReader\n   * @throws JsonSyntaxException if json is not a valid representation for an object of type typeOfT\n   * @see #fromJson(Reader, Type)\n   * @see #fromJson(JsonReader, TypeToken)\n   */\n  @SuppressWarnings({\"unchecked\", \"TypeParameterUnusedInFormals\"})\n  public <T> T fromJson(JsonReader reader, Type typeOfT)\n      throws JsonIOException, JsonSyntaxException {\n    return (T) fromJson(reader, TypeToken.get(typeOfT));\n  }\n\n  /**\n   * Reads the next JSON value from {@code reader} and converts it to an object of type {@code\n   * typeOfT}. Returns {@code null}, if the {@code reader} is at EOF. This method is useful if the\n   * specified object is a generic type. For non-generic objects, {@link #fromJson(JsonReader,\n   * Type)} can be called, or {@link TypeToken#get(Class)} can be used to create the type token.\n   *\n   * <p>Unlike the other {@code fromJson} methods, no exception is thrown if the JSON data has\n   * multiple top-level JSON elements, or if there is trailing data.\n   *\n   * <p>If the {@code Gson} instance has an {@linkplain GsonBuilder#setStrictness(Strictness)\n   * explicit strictness setting}, this setting will be used for reading the JSON regardless of the\n   * {@linkplain JsonReader#getStrictness() strictness} of the provided {@link JsonReader}. For\n   * legacy reasons, if the {@code Gson} instance has no explicit strictness setting and the reader\n   * does not have the strictness {@link Strictness#STRICT}, the JSON will be written in {@link\n   * Strictness#LENIENT} mode.<br>\n   * Note that in all cases the old strictness setting of the reader will be restored when this\n   * method returns.\n   *\n   * @param <T> the type of the desired object\n   * @param reader the reader whose next JSON value should be deserialized\n   * @param typeOfT The specific genericized type of src. You should create an anonymous subclass of\n   *     {@code TypeToken} with the specific generic type arguments. For example, to get the type\n   *     for {@code Collection<Foo>}, you should use:\n   *     <pre>\n   * new TypeToken&lt;Collection&lt;Foo&gt;&gt;(){}\n   * </pre>\n   *\n   * @return an object of type T from the JsonReader. Returns {@code null} if {@code reader} is at\n   *     EOF.\n   * @throws JsonIOException if there was a problem reading from the JsonReader\n   * @throws JsonSyntaxException if json is not a valid representation for an object of the type\n   *     typeOfT\n   * @see #fromJson(Reader, TypeToken)\n   * @see #fromJson(JsonReader, Type)\n   * @since 2.10\n   */\n  public <T> T fromJson(JsonReader reader, TypeToken<T> typeOfT)\n      throws JsonIOException, JsonSyntaxException {\n    boolean isEmpty = true;\n    Strictness oldStrictness = reader.getStrictness();\n\n    if (this.strictness != null) {\n      reader.setStrictness(this.strictness);\n    } else if (reader.getStrictness() != Strictness.STRICT) {\n      reader.setStrictness(Strictness.LENIENT);\n    }\n\n    try {\n      JsonToken unused = reader.peek();\n      isEmpty = false;\n      TypeAdapter<T> typeAdapter = getAdapter(typeOfT);\n      return typeAdapter.read(reader);\n    } catch (EOFException e) {\n      /*\n       * For compatibility with JSON 1.5 and earlier, we return null for empty\n       * documents instead of throwing.\n       */\n      if (isEmpty) {\n        return null;\n      }\n      throw new JsonSyntaxException(e);\n    } catch (IllegalStateException e) {\n      throw new JsonSyntaxException(e);\n    } catch (IOException e) {\n      // TODO(inder): Figure out whether it is indeed right to rethrow this as JsonSyntaxException\n      throw new JsonSyntaxException(e);\n    } catch (AssertionError e) {\n      throw new AssertionError(\n          \"AssertionError (GSON \" + GsonBuildConfig.VERSION + \"): \" + e.getMessage(), e);\n    } finally {\n      reader.setStrictness(oldStrictness);\n    }\n  }\n\n  /**\n   * This method deserializes the JSON read from the specified parse tree into an object of the\n   * specified type. It is not suitable to use if the specified class is a generic type since it\n   * will not have the generic type information because of the Type Erasure feature of Java.\n   * Therefore, this method should not be used if the desired type is a generic type. Note that this\n   * method works fine if any of the fields of the specified object are generics, just the object\n   * itself should not be a generic type. For the cases when the object is of generic type, invoke\n   * {@link #fromJson(JsonElement, TypeToken)}.\n   *\n   * @param <T> the type of the desired object\n   * @param json the root of the parse tree of {@link JsonElement}s from which the object is to be\n   *     deserialized\n   * @param classOfT The class of T\n   * @return an object of type T from the JSON. Returns {@code null} if {@code json} is {@code null}\n   *     or if {@code json} is empty.\n...\n", "needle_token_start": 4029, "needle_token_end": 4050, "code_context_ntokens": 16261, "output": ["```java\n/**\n * Returns a new GsonBuilder containing all custom factories and configuration used by the current\n * instance.\n *\n * @return a GsonBuilder instance.\n * @since 2.8.3\n */\npublic GsonBuilder newBuilder() {\n    return new GsonBuilder(this);\n}\n```"]}
{"repo": "google/gson", "name": "getAsJsonNull", "language": "java", "path": "gson/src/main/java/com/google/gson/JsonElement.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the current element as a specific null type representation if it is indeed that type, ensuring type safety before casting.\n2. **Input**: No parameters are required as it operates on the current instance of the element.\n3. **Output**: Returns the element as a null type representation if it is valid; otherwise, it throws an exception.\n4. **Procedure**: The method first checks if the current element is of the null type using a type-checking method. If the check passes, the element is cast to the null type and returned. If the check fails, an exception is thrown indicating that the element is not of the expected null type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/com/google/gson/internal/bind/TypeAdapters.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.bind;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonArray;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonIOException;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonPrimitive;\nimport com.google.gson.JsonSyntaxException;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.TypeAdapterFactory;\nimport com.google.gson.annotations.SerializedName;\nimport com.google.gson.internal.LazilyParsedNumber;\nimport com.google.gson.internal.NumberLimits;\nimport com.google.gson.internal.TroubleshootingGuide;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.lang.reflect.AccessibleObject;\nimport java.lang.reflect.Field;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.net.InetAddress;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.net.URL;\nimport java.security.AccessController;\nimport java.security.PrivilegedAction;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.BitSet;\nimport java.util.Calendar;\nimport java.util.Currency;\nimport java.util.Deque;\nimport java.util.GregorianCalendar;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.StringTokenizer;\nimport java.util.UUID;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicIntegerArray;\n\n/** Type adapters for basic types. */\npublic final class TypeAdapters {\n  private TypeAdapters() {\n    throw new UnsupportedOperationException();\n  }\n\n  @SuppressWarnings(\"rawtypes\")\n  public static final TypeAdapter<Class> CLASS =\n      new TypeAdapter<Class>() {\n        @Override\n        public void write(JsonWriter out, Class value) throws IOException {\n          throw new UnsupportedOperationException(\n              \"Attempted to serialize java.lang.Class: \"\n                  + value.getName()\n                  + \". Forgot to register a type adapter?\"\n                  + \"\\nSee \"\n                  + TroubleshootingGuide.createUrl(\"java-lang-class-unsupported\"));\n        }\n\n        @Override\n        public Class read(JsonReader in) throws IOException {\n          throw new UnsupportedOperationException(\n              \"Attempted to deserialize a java.lang.Class. Forgot to register a type adapter?\"\n                  + \"\\nSee \"\n                  + TroubleshootingGuide.createUrl(\"java-lang-class-unsupported\"));\n        }\n      }.nullSafe();\n\n  public static final TypeAdapterFactory CLASS_FACTORY = newFactory(Class.class, CLASS);\n\n  public static final TypeAdapter<BitSet> BIT_SET =\n      new TypeAdapter<BitSet>() {\n        @Override\n        public BitSet read(JsonReader in) throws IOException {\n          BitSet bitset = new BitSet();\n          in.beginArray();\n          int i = 0;\n          JsonToken tokenType = in.peek();\n          while (tokenType != JsonToken.END_ARRAY) {\n            boolean set;\n            switch (tokenType) {\n              case NUMBER:\n              case STRING:\n                int intValue = in.nextInt();\n                if (intValue == 0) {\n                  set = false;\n                } else if (intValue == 1) {\n                  set = true;\n                } else {\n                  throw new JsonSyntaxException(\n                      \"Invalid bitset value \"\n                          + intValue\n                          + \", expected 0 or 1; at path \"\n                          + in.getPreviousPath());\n                }\n                break;\n              case BOOLEAN:\n                set = in.nextBoolean();\n                break;\n              default:\n                throw new JsonSyntaxException(\n                    \"Invalid bitset value type: \" + tokenType + \"; at path \" + in.getPath());\n            }\n            if (set) {\n              bitset.set(i);\n            }\n            ++i;\n            tokenType = in.peek();\n          }\n          in.endArray();\n          return bitset;\n        }\n\n        @Override\n        public void write(JsonWriter out, BitSet src) throws IOException {\n          out.beginArray();\n          for (int i = 0, length = src.length(); i < length; i++) {\n            int value = src.get(i) ? 1 : 0;\n            out.value(value);\n          }\n          out.endArray();\n        }\n      }.nullSafe();\n\n  public static final TypeAdapterFactory BIT_SET_FACTORY = newFactory(BitSet.class, BIT_SET);\n\n  public static final TypeAdapter<Boolean> BOOLEAN =\n      new TypeAdapter<Boolean>() {\n        @Override\n        public Boolean read(JsonReader in) throws IOException {\n          JsonToken peek = in.peek();\n          if (peek == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          } else if (peek == JsonToken.STRING) {\n            // support strings for compatibility with GSON 1.7\n            return Boolean.parseBoolean(in.nextString());\n          }\n          return in.nextBoolean();\n        }\n\n        @Override\n        public void write(JsonWriter out, Boolean value) throws IOException {\n          out.value(value);\n        }\n      };\n\n  /**\n   * Writes a boolean as a string. Useful for map keys, where booleans aren't otherwise permitted.\n   */\n  public static final TypeAdapter<Boolean> BOOLEAN_AS_STRING =\n      new TypeAdapter<Boolean>() {\n        @Override\n        public Boolean read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          return Boolean.valueOf(in.nextString());\n        }\n\n        @Override\n        public void write(JsonWriter out, Boolean value) throws IOException {\n          out.value(value == null ? \"null\" : value.toString());\n        }\n      };\n\n  public static final TypeAdapterFactory BOOLEAN_FACTORY =\n      newFactory(boolean.class, Boolean.class, BOOLEAN);\n\n  public static final TypeAdapter<Number> BYTE =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n\n          int intValue;\n          try {\n            intValue = in.nextInt();\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(e);\n          }\n          // Allow up to 255 to support unsigned values\n          if (intValue > 255 || intValue < Byte.MIN_VALUE) {\n            throw new JsonSyntaxException(\n                \"Lossy conversion from \" + intValue + \" to byte; at path \" + in.getPreviousPath());\n          }\n          return (byte) intValue;\n        }\n\n        @Override\n        public void write(JsonWriter out, Number value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n          } else {\n            out.value(value.byteValue());\n          }\n        }\n      };\n\n  public static final TypeAdapterFactory BYTE_FACTORY = newFactory(byte.class, Byte.class, BYTE);\n\n  public static final TypeAdapter<Number> SHORT =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n\n          int intValue;\n          try {\n            intValue = in.nextInt();\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(e);\n          }\n          // Allow up to 65535 to support unsigned values\n...\n// Path: gson/src/main/java/com/google/gson/stream/MalformedJsonException.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.stream;\n\nimport com.google.gson.Strictness;\nimport java.io.IOException;\n\n/**\n * Thrown when a reader encounters malformed JSON. Some syntax errors can be ignored by using {@link\n * Strictness#LENIENT} for {@link JsonReader#setStrictness(Strictness)}.\n */\npublic final class MalformedJsonException extends IOException {\n  private static final long serialVersionUID = 1L;\n\n  public MalformedJsonException(String msg) {\n    super(msg);\n  }\n\n  public MalformedJsonException(String msg, Throwable throwable) {\n    super(msg, throwable);\n  }\n\n  public MalformedJsonException(Throwable throwable) {\n    super(throwable);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/Streams.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonIOException;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonParseException;\nimport com.google.gson.JsonSyntaxException;\nimport com.google.gson.internal.bind.TypeAdapters;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.JsonWriter;\nimport com.google.gson.stream.MalformedJsonException;\nimport java.io.EOFException;\nimport java.io.IOException;\nimport java.io.Writer;\nimport java.util.Objects;\n\n/** Reads and writes GSON parse trees over streams. */\npublic final class Streams {\n  private Streams() {\n    throw new UnsupportedOperationException();\n  }\n\n  /** Takes a reader in any state and returns the next value as a JsonElement. */\n  public static JsonElement parse(JsonReader reader) throws JsonParseException {\n    boolean isEmpty = true;\n    try {\n      JsonToken unused = reader.peek();\n      isEmpty = false;\n      return TypeAdapters.JSON_ELEMENT.read(reader);\n    } catch (EOFException e) {\n      /*\n       * For compatibility with JSON 1.5 and earlier, we return a JsonNull for\n       * empty documents instead of throwing.\n       */\n      if (isEmpty) {\n        return JsonNull.INSTANCE;\n      }\n      // The stream ended prematurely so it is likely a syntax error.\n      throw new JsonSyntaxException(e);\n    } catch (MalformedJsonException e) {\n      throw new JsonSyntaxException(e);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    } catch (NumberFormatException e) {\n      throw new JsonSyntaxException(e);\n    }\n  }\n\n  /** Writes the JSON element to the writer, recursively. */\n  public static void write(JsonElement element, JsonWriter writer) throws IOException {\n    TypeAdapters.JSON_ELEMENT.write(writer, element);\n  }\n\n  public static Writer writerForAppendable(Appendable appendable) {\n    return appendable instanceof Writer ? (Writer) appendable : new AppendableWriter(appendable);\n  }\n\n  /** Adapts an {@link Appendable} so it can be passed anywhere a {@link Writer} is used. */\n  private static final class AppendableWriter extends Writer {\n    private final Appendable appendable;\n    private final CurrentWrite currentWrite = new CurrentWrite();\n\n    AppendableWriter(Appendable appendable) {\n      this.appendable = appendable;\n    }\n\n    @SuppressWarnings(\"UngroupedOverloads\") // this is intentionally ungrouped, see comment below\n    @Override\n    public void write(char[] chars, int offset, int length) throws IOException {\n      currentWrite.setChars(chars);\n      appendable.append(currentWrite, offset, offset + length);\n    }\n\n    @Override\n    public void flush() {}\n\n    @Override\n    public void close() {}\n\n    // Override these methods for better performance\n    // They would otherwise unnecessarily create Strings or char arrays\n\n    @Override\n    public void write(int i) throws IOException {\n      appendable.append((char) i);\n    }\n\n    @Override\n    public void write(String str, int off, int len) throws IOException {\n      // Appendable.append turns null -> \"null\", which is not desired here\n      Objects.requireNonNull(str);\n      appendable.append(str, off, off + len);\n    }\n\n    @Override\n    public Writer append(CharSequence csq) throws IOException {\n      appendable.append(csq);\n      return this;\n    }\n\n    @Override\n    public Writer append(CharSequence csq, int start, int end) throws IOException {\n      appendable.append(csq, start, end);\n      return this;\n    }\n\n    /** A mutable char sequence pointing at a single char[]. */\n    private static class CurrentWrite implements CharSequence {\n      private char[] chars;\n      private String cachedString;\n\n      void setChars(char[] chars) {\n        this.chars = chars;\n        this.cachedString = null;\n      }\n\n      @Override\n      public int length() {\n        return chars.length;\n      }\n\n      @Override\n      public char charAt(int i) {\n        return chars[i];\n      }\n\n      @Override\n      public CharSequence subSequence(int start, int end) {\n        return new String(chars, start, end - start);\n      }\n\n      // Must return string representation to satisfy toString() contract\n      @Override\n      public String toString() {\n        if (cachedString == null) {\n          cachedString = new String(chars);\n        }\n        return cachedString;\n      }\n    }\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonElement.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.internal.Streams;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.io.StringWriter;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\n\n/**\n * A class representing an element of JSON. It could either be a {@link JsonObject}, a {@link\n * JsonArray}, a {@link JsonPrimitive} or a {@link JsonNull}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic abstract class JsonElement {\n  /**\n   * @deprecated Creating custom {@code JsonElement} subclasses is highly discouraged and can lead\n   *     to undefined behavior.<br>\n   *     This constructor is only kept for backward compatibility.\n   */\n  @Deprecated\n  public JsonElement() {}\n\n  /**\n   * Returns a deep copy of this element. Immutable elements like primitives and nulls are not\n   * copied.\n   *\n   * @since 2.8.2\n   */\n  public abstract JsonElement deepCopy();\n\n  /**\n   * Provides a check for verifying if this element is a JSON array or not.\n   *\n   * @return true if this element is of type {@link JsonArray}, false otherwise.\n   */\n  public boolean isJsonArray() {\n    return this instanceof JsonArray;\n  }\n\n  /**\n   * Provides a check for verifying if this element is a JSON object or not.\n   *\n   * @return true if this element is of type {@link JsonObject}, false otherwise.\n   */\n  public boolean isJsonObject() {\n    return this instanceof JsonObject;\n  }\n\n  /**\n   * Provides a check for verifying if this element is a primitive or not.\n   *\n   * @return true if this element is of type {@link JsonPrimitive}, false otherwise.\n   */\n  public boolean isJsonPrimitive() {\n    return this instanceof JsonPrimitive;\n  }\n\n  /**\n   * Provides a check for verifying if this element represents a null value or not.\n   *\n   * @return true if this element is of type {@link JsonNull}, false otherwise.\n   * @since 1.2\n   */\n  public boolean isJsonNull() {\n    return this instanceof JsonNull;\n  }\n\n  /**\n   * Convenience method to get this element as a {@link JsonObject}. If this element is of some\n   * other type, an {@link IllegalStateException} will result. Hence it is best to use this method\n   * after ensuring that this element is of the desired type by calling {@link #isJsonObject()}\n   * first.\n   *\n   * @return this element as a {@link JsonObject}.\n   * @throws IllegalStateException if this element is of another type.\n   */\n  public JsonObject getAsJsonObject() {\n    if (isJsonObject()) {\n      return (JsonObject) this;\n    }\n    throw new IllegalStateException(\"Not a JSON Object: \" + this);\n  }\n\n  /**\n   * Convenience method to get this element as a {@link JsonArray}. If this element is of some other\n   * type, an {@link IllegalStateException} will result. Hence it is best to use this method after\n   * ensuring that this element is of the desired type by calling {@link #isJsonArray()} first.\n   *\n   * @return this element as a {@link JsonArray}.\n   * @throws IllegalStateException if this element is of another type.\n   */\n  public JsonArray getAsJsonArray() {\n    if (isJsonArray()) {\n      return (JsonArray) this;\n    }\n    throw new IllegalStateException(\"Not a JSON Array: \" + this);\n  }\n\n  /**\n   * Convenience method to get this element as a {@link JsonPrimitive}. If this element is of some\n   * other type, an {@link IllegalStateException} will result. Hence it is best to use this method\n   * after ensuring that this element is of the desired type by calling {@link #isJsonPrimitive()}\n   * first.\n   *\n   * @return this element as a {@link JsonPrimitive}.\n   * @throws IllegalStateException if this element is of another type.\n   */\n  public JsonPrimitive getAsJsonPrimitive() {\n    if (isJsonPrimitive()) {\n      return (JsonPrimitive) this;\n    }\n    throw new IllegalStateException(\"Not a JSON Primitive: \" + this);\n  }\n\n  /**\n   * Convenience method to get this element as a {@link JsonNull}. If this element is of some other\n   * type, an {@link IllegalStateException} will result. Hence it is best to use this method after\n   * ensuring that this element is of the desired type by calling {@link #isJsonNull()} first.\n   *\n   * @return this element as a {@link JsonNull}.\n   * @throws IllegalStateException if this element is of another type.\n   * @since 1.2\n   */\n  \n@CanIgnoreReturnValue // When this method is used only to verify that the value is JsonNull\n  public JsonNull getAsJsonNull() {\n    if (isJsonNull()) {\n      return (JsonNull) this;\n    }\n    throw new IllegalStateException(\"Not a JSON Null: \" + this);\n  }\n\n  /**\n   * Convenience method to get this element as a boolean value.\n   *\n   * @return this element as a primitive boolean value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public boolean getAsBoolean() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a {@link Number}.\n   *\n   * @return this element as a {@link Number}.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}, or cannot be converted to a number.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public Number getAsNumber() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a string value.\n   *\n   * @return this element as a string value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public String getAsString() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive double value.\n   *\n   * @return this element as a primitive double value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if the value contained is not a valid double.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public double getAsDouble() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive float value.\n   *\n   * @return this element as a primitive float value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if the value contained is not a valid float.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public float getAsFloat() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive long value.\n   *\n   * @return this element as a primitive long value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if the value contained is not a valid long.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public long getAsLong() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive integer value.\n   *\n   * @return this element as a primitive integer value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if the value contained is not a valid integer.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public int getAsInt() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive byte value.\n   *\n   * @return this element as a primitive byte value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if the value contained is not a valid byte.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   * @since 1.3\n   */\n  public byte getAsByte() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get the first character of the string value of this element.\n   *\n   * @return the first character of the string value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}, or if its string value is empty.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   * @since 1.3\n   * @deprecated This method is misleading, as it does not get this element as a char but rather as\n   *     a string's first character.\n   */\n  @Deprecated\n  public char getAsCharacter() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a {@link BigDecimal}.\n   *\n   * @return this element as a {@link BigDecimal}.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if this element is not a valid {@link BigDecimal}.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   * @since 1.2\n   */\n  public BigDecimal getAsBigDecimal() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a {@link BigInteger}.\n   *\n   * @return this element as a {@link BigInteger}.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if this element is not a valid {@link BigInteger}.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   * @since 1.2\n   */\n  public BigInteger getAsBigInteger() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive short value.\n   *\n   * @return this element as a primitive short value.\n   * @throws UnsupportedOperationException if this element is not a {@link JsonPrimitive} or {@link\n   *     JsonArray}.\n   * @throws NumberFormatException if the value contained is not a valid short.\n   * @throws IllegalStateException if this element is of the type {@link JsonArray} but contains\n   *     more than a single element.\n   */\n  public short getAsShort() {\n    throw new UnsupportedOperationException(getClass().getSimpleName());\n  }\n\n  /** Returns a String representation of this element. */\n  @Override\n  public String toString() {\n    try {\n      StringWriter stringWriter = new StringWriter();\n      JsonWriter jsonWriter = new JsonWriter(stringWriter);\n      // Make writer lenient because toString() must not fail, even if for example JsonPrimitive\n      // contains NaN\n      jsonWriter.setStrictness(Strictness.LENIENT);\n      Streams.write(this, jsonWriter);\n      return stringWriter.toString();\n    } catch (IOException e) {\n      throw new AssertionError(e);\n    }\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/bind/JsonTreeReader.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.bind;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.JsonArray;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonPrimitive;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.MalformedJsonException;\nimport java.io.IOException;\nimport java.io.Reader;\nimport java.util.Arrays;\nimport java.util.Iterator;\nimport java.util.Map;\n\n/**\n * This reader walks the elements of a JsonElement as if it was coming from a character stream.\n *\n * @author Jesse Wilson\n */\npublic final class JsonTreeReader extends JsonReader {\n  private static final Reader UNREADABLE_READER =\n      new Reader() {\n        @Override\n        public int read(char[] buffer, int offset, int count) {\n          throw new AssertionError();\n        }\n\n        @Override\n        public void close() {\n          throw new AssertionError();\n        }\n      };\n  private static final Object SENTINEL_CLOSED = new Object();\n\n  /*\n   * The nesting stack. Using a manual array rather than an ArrayList saves 20%.\n   */\n  private Object[] stack = new Object[32];\n  private int stackSize = 0;\n\n  /*\n   * The path members. It corresponds directly to stack: At indices where the\n   * stack contains an object (EMPTY_OBJECT, DANGLING_NAME or NONEMPTY_OBJECT),\n   * pathNames contains the name at this scope. Where it contains an array\n   * (EMPTY_ARRAY, NONEMPTY_ARRAY) pathIndices contains the current index in\n   * that array. Otherwise the value is undefined, and we take advantage of that\n   * by incrementing pathIndices when doing so isn't useful.\n   */\n  private String[] pathNames = new String[32];\n  private int[] pathIndices = new int[32];\n\n  public JsonTreeReader(JsonElement element) {\n    super(UNREADABLE_READER);\n    push(element);\n  }\n\n  @Override\n  public void beginArray() throws IOException {\n    expect(JsonToken.BEGIN_ARRAY);\n    JsonArray array = (JsonArray) peekStack();\n    push(array.iterator());\n    pathIndices[stackSize - 1] = 0;\n  }\n\n  @Override\n  public void endArray() throws IOException {\n    expect(JsonToken.END_ARRAY);\n    popStack(); // empty iterator\n    popStack(); // array\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n  }\n\n  @Override\n  public void beginObject() throws IOException {\n    expect(JsonToken.BEGIN_OBJECT);\n    JsonObject object = (JsonObject) peekStack();\n    push(object.entrySet().iterator());\n  }\n\n  @Override\n  public void endObject() throws IOException {\n    expect(JsonToken.END_OBJECT);\n    pathNames[stackSize - 1] = null; // Free the last path name so that it can be garbage collected\n    popStack(); // empty iterator\n    popStack(); // object\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n  }\n\n  @Override\n  public boolean hasNext() throws IOException {\n    JsonToken token = peek();\n    return token != JsonToken.END_OBJECT\n        && token != JsonToken.END_ARRAY\n        && token != JsonToken.END_DOCUMENT;\n  }\n\n  @Override\n  public JsonToken peek() throws IOException {\n    if (stackSize == 0) {\n      return JsonToken.END_DOCUMENT;\n    }\n\n    Object o = peekStack();\n    if (o instanceof Iterator) {\n      boolean isObject = stack[stackSize - 2] instanceof JsonObject;\n      Iterator<?> iterator = (Iterator<?>) o;\n      if (iterator.hasNext()) {\n        if (isObject) {\n          return JsonToken.NAME;\n        } else {\n          push(iterator.next());\n          return peek();\n        }\n      } else {\n        return isObject ? JsonToken.END_OBJECT : JsonToken.END_ARRAY;\n      }\n    } else if (o instanceof JsonObject) {\n      return JsonToken.BEGIN_OBJECT;\n    } else if (o instanceof JsonArray) {\n      return JsonToken.BEGIN_ARRAY;\n    } else if (o instanceof JsonPrimitive) {\n      JsonPrimitive primitive = (JsonPrimitive) o;\n      if (primitive.isString()) {\n        return JsonToken.STRING;\n      } else if (primitive.isBoolean()) {\n        return JsonToken.BOOLEAN;\n      } else if (primitive.isNumber()) {\n        return JsonToken.NUMBER;\n      } else {\n        throw new AssertionError();\n      }\n    } else if (o instanceof JsonNull) {\n      return JsonToken.NULL;\n    } else if (o == SENTINEL_CLOSED) {\n      throw new IllegalStateException(\"JsonReader is closed\");\n    } else {\n      throw new MalformedJsonException(\n          \"Custom JsonElement subclass \" + o.getClass().getName() + \" is not supported\");\n    }\n  }\n\n  private Object peekStack() {\n    return stack[stackSize - 1];\n  }\n\n  @CanIgnoreReturnValue\n  private Object popStack() {\n    Object result = stack[--stackSize];\n    stack[stackSize] = null;\n    return result;\n  }\n\n  private void expect(JsonToken expected) throws IOException {\n    if (peek() != expected) {\n      throw new IllegalStateException(\n          \"Expected \" + expected + \" but was \" + peek() + locationString());\n    }\n  }\n\n  private String nextName(boolean skipName) throws IOException {\n    expect(JsonToken.NAME);\n    Iterator<?> i = (Iterator<?>) peekStack();\n    Map.Entry<?, ?> entry = (Map.Entry<?, ?>) i.next();\n    String result = (String) entry.getKey();\n    pathNames[stackSize - 1] = skipName ? \"<skipped>\" : result;\n    push(entry.getValue());\n    return result;\n  }\n\n  @Override\n  public String nextName() throws IOException {\n    return nextName(false);\n  }\n\n  @Override\n  public String nextString() throws IOException {\n    JsonToken token = peek();\n    if (token != JsonToken.STRING && token != JsonToken.NUMBER) {\n      throw new IllegalStateException(\n          \"Expected \" + JsonToken.STRING + \" but was \" + token + locationString());\n    }\n    String result = ((JsonPrimitive) popStack()).getAsString();\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n    return result;\n  }\n\n  @Override\n  public boolean nextBoolean() throws IOException {\n    expect(JsonToken.BOOLEAN);\n    boolean result = ((JsonPrimitive) popStack()).getAsBoolean();\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n    return result;\n  }\n\n  @Override\n  public void nextNull() throws IOException {\n    expect(JsonToken.NULL);\n    popStack();\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n  }\n\n  @Override\n  public double nextDouble() throws IOException {\n    JsonToken token = peek();\n    if (token != JsonToken.NUMBER && token != JsonToken.STRING) {\n      throw new IllegalStateException(\n          \"Expected \" + JsonToken.NUMBER + \" but was \" + token + locationString());\n    }\n    double result = ((JsonPrimitive) peekStack()).getAsDouble();\n    if (!isLenient() && (Double.isNaN(result) || Double.isInfinite(result))) {\n      throw new MalformedJsonException(\"JSON forbids NaN and infinities: \" + result);\n    }\n    popStack();\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n    return result;\n  }\n\n  @Override\n  public long nextLong() throws IOException {\n    JsonToken token = peek();\n    if (token != JsonToken.NUMBER && token != JsonToken.STRING) {\n      throw new IllegalStateException(\n          \"Expected \" + JsonToken.NUMBER + \" but was \" + token + locationString());\n    }\n    long result = ((JsonPrimitive) peekStack()).getAsLong();\n    popStack();\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n    return result;\n  }\n\n  @Override\n  public int nextInt() throws IOException {\n    JsonToken token = peek();\n    if (token != JsonToken.NUMBER && token != JsonToken.STRING) {\n      throw new IllegalStateException(\n          \"Expected \" + JsonToken.NUMBER + \" but was \" + token + locationString());\n    }\n    int result = ((JsonPrimitive) peekStack()).getAsInt();\n    popStack();\n    if (stackSize > 0) {\n      pathIndices[stackSize - 1]++;\n    }\n    return result;\n  }\n\n  JsonElement nextJsonElement() throws IOException {\n    final JsonToken peeked = peek();\n    if (peeked == JsonToken.NAME\n        || peeked == JsonToken.END_ARRAY\n        || peeked == JsonToken.END_OBJECT\n        || peeked == JsonToken.END_DOCUMENT) {\n      throw new IllegalStateException(\"Unexpected \" + peeked + \" when reading a JsonElement.\");\n    }\n    final JsonElement element = (JsonElement) peekStack();\n    skipValue();\n    return element;\n  }\n\n  @Override\n  public void close() throws IOException {\n    stack = new Object[] {SENTINEL_CLOSED};\n    stackSize = 1;\n  }\n\n  @Override\n  public void skipValue() throws IOException {\n    JsonToken peeked = peek();\n    switch (peeked) {\n      case NAME:\n        @SuppressWarnings(\"unused\")\n        String unused = nextName(true);\n        break;\n      case END_ARRAY:\n        endArray();\n        break;\n      case END_OBJECT:\n        endObject();\n        break;\n      case END_DOCUMENT:\n        // Do nothing\n        break;\n      default:\n        popStack();\n        if (stackSize > 0) {\n          pathIndices[stackSize - 1]++;\n        }\n        break;\n    }\n  }\n\n  @Override\n  public String toString() {\n    return getClass().getSimpleName() + locationString();\n  }\n\n  public void promoteNameToValue() throws IOException {\n    expect(JsonToken.NAME);\n    Iterator<?> i = (Iterator<?>) peekStack();\n    Map.Entry<?, ?> entry = (Map.Entry<?, ?>) i.next();\n    push(entry.getValue());\n    push(new JsonPrimitive((String) entry.getKey()));\n  }\n\n  private void push(Object newTop) {\n    if (stackSize == stack.length) {\n      int newLength = stackSize * 2;\n      stack = Arrays.copyOf(stack, newLength);\n      pathIndices = Arrays.copyOf(pathIndices, newLength);\n      pathNames = Arrays.copyOf(pathNames, newLength);\n    }\n    stack[stackSize++] = newTop;\n  }\n\n  private String getPath(boolean usePreviousPath) {\n    StringBuilder result = new StringBuilder().append('$');\n    for (int i = 0; i < stackSize; i++) {\n      if (stack[i] instanceof JsonArray) {\n        if (++i < stackSize && stack[i] instanceof Iterator) {\n          int pathIndex = pathIndices[i];\n          // If index is last path element it points to next array element; have to decrement\n          // `- 1` covers case where iterator for next element is on stack\n          // `- 2` covers case where peek() already pushed next element onto stack\n          if (usePreviousPath && pathIndex > 0 && (i == stackSize - 1 || i == stackSize - 2)) {\n            pathIndex--;\n          }\n          result.append('[').append(pathIndex).append(']');\n        }\n      } else if (stack[i] instanceof JsonObject) {\n        if (++i < stackSize && stack[i] instanceof Iterator) {\n          result.append('.');\n          if (pathNames[i] != null) {\n            result.append(pathNames[i]);\n          }\n        }\n      }\n    }\n    return result.toString();\n  }\n\n  @Override\n  public String getPath() {\n    return getPath(false);\n  }\n\n  @Override\n  public String getPreviousPath() {\n    return getPath(true);\n  }\n\n  private String locationString() {\n    return \" at path \" + getPath();\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/bind/JsonTreeWriter.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.bind;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.JsonArray;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonPrimitive;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.io.Writer;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Objects;\n\n/** This writer creates a JsonElement. */\npublic final class JsonTreeWriter extends JsonWriter {\n  private static final Writer UNWRITABLE_WRITER =\n      new Writer() {\n        @Override\n        public void write(char[] buffer, int offset, int counter) {\n          throw new AssertionError();\n        }\n\n        @Override\n        public void flush() {\n          throw new AssertionError();\n        }\n\n        @Override\n        public void close() {\n          throw new AssertionError();\n        }\n      };\n\n  /** Added to the top of the stack when this writer is closed to cause following ops to fail. */\n  private static final JsonPrimitive SENTINEL_CLOSED = new JsonPrimitive(\"closed\");\n\n  /** The JsonElements and JsonArrays under modification, outermost to innermost. */\n  private final List<JsonElement> stack = new ArrayList<>();\n\n  /** The name for the next JSON object value. If non-null, the top of the stack is a JsonObject. */\n  private String pendingName;\n\n  /** the JSON element constructed by this writer. */\n  private JsonElement product = JsonNull.INSTANCE; // TODO: is this really what we want?;\n\n  public JsonTreeWriter() {\n    super(UNWRITABLE_WRITER);\n  }\n\n  /** Returns the top level object produced by this writer. */\n  public JsonElement get() {\n    if (!stack.isEmpty()) {\n      throw new IllegalStateException(\"Expected one JSON element but was \" + stack);\n    }\n    return product;\n  }\n\n  private JsonElement peek() {\n    return stack.get(stack.size() - 1);\n  }\n\n  private void put(JsonElement value) {\n    if (pendingName != null) {\n      if (!value.isJsonNull() || getSerializeNulls()) {\n        JsonObject object = (JsonObject) peek();\n        object.add(pendingName, value);\n      }\n      pendingName = null;\n    } else if (stack.isEmpty()) {\n      product = value;\n    } else {\n      JsonElement element = peek();\n      if (element instanceof JsonArray) {\n        ((JsonArray) element).add(value);\n      } else {\n        throw new IllegalStateException();\n      }\n    }\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter beginArray() throws IOException {\n    JsonArray array = new JsonArray();\n    put(array);\n    stack.add(array);\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter endArray() throws IOException {\n    if (stack.isEmpty() || pendingName != null) {\n      throw new IllegalStateException();\n    }\n    JsonElement element = peek();\n    if (element instanceof JsonArray) {\n      stack.remove(stack.size() - 1);\n      return this;\n    }\n    throw new IllegalStateException();\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter beginObject() throws IOException {\n    JsonObject object = new JsonObject();\n    put(object);\n    stack.add(object);\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter endObject() throws IOException {\n    if (stack.isEmpty() || pendingName != null) {\n      throw new IllegalStateException();\n    }\n    JsonElement element = peek();\n    if (element instanceof JsonObject) {\n      stack.remove(stack.size() - 1);\n      return this;\n    }\n    throw new IllegalStateException();\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter name(String name) throws IOException {\n    Objects.requireNonNull(name, \"name == null\");\n    if (stack.isEmpty() || pendingName != null) {\n      throw new IllegalStateException(\"Did not expect a name\");\n    }\n    JsonElement element = peek();\n    if (element instanceof JsonObject) {\n      pendingName = name;\n      return this;\n    }\n    throw new IllegalStateException(\"Please begin an object before writing a name.\");\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter value(String value) throws IOException {\n    if (value == null) {\n      return nullValue();\n    }\n    put(new JsonPrimitive(value));\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter value(boolean value) throws IOException {\n    put(new JsonPrimitive(value));\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter value(Boolean value) throws IOException {\n    if (value == null) {\n      return nullValue();\n    }\n    put(new JsonPrimitive(value));\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter value(float value) throws IOException {\n    if (!isLenient() && (Float.isNaN(value) || Float.isInfinite(value))) {\n      throw new IllegalArgumentException(\"JSON forbids NaN and infinities: \" + value);\n    }\n    put(new JsonPrimitive(value));\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter value(double value) throws IOException {\n    if (!isLenient() && (Double.isNaN(value) || Double.isInfinite(value))) {\n      throw new IllegalArgumentException(\"JSON forbids NaN and infinities: \" + value);\n    }\n    put(new JsonPrimitive(value));\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter value(long value) throws IOException {\n    put(new JsonPrimitive(value));\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter value(Number value) throws IOException {\n    if (value == null) {\n      return nullValue();\n    }\n\n    if (!isLenient()) {\n      double d = value.doubleValue();\n      if (Double.isNaN(d) || Double.isInfinite(d)) {\n        throw new IllegalArgumentException(\"JSON forbids NaN and infinities: \" + value);\n      }\n    }\n\n    put(new JsonPrimitive(value));\n    return this;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public JsonWriter nullValue() throws IOException {\n    put(JsonNull.INSTANCE);\n    return this;\n  }\n\n  @Override\n  public JsonWriter jsonValue(String value) throws IOException {\n    throw new UnsupportedOperationException();\n  }\n\n  @Override\n  public void flush() throws IOException {}\n\n  @Override\n  public void close() throws IOException {\n    if (!stack.isEmpty()) {\n      throw new IOException(\"Incomplete document\");\n    }\n    stack.add(SENTINEL_CLOSED);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/TypeAdapter.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.gson.internal.bind.JsonTreeReader;\nimport com.google.gson.internal.bind.JsonTreeWriter;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.io.Reader;\nimport java.io.StringReader;\nimport java.io.StringWriter;\nimport java.io.Writer;\n\n/**\n * Converts Java objects to and from JSON.\n *\n * <h2>Defining a type's JSON form</h2>\n *\n * By default Gson converts application classes to JSON using its built-in type adapters. If Gson's\n * default JSON conversion isn't appropriate for a type, extend this class to customize the\n * conversion. Here's an example of a type adapter for an (X,Y) coordinate point:\n *\n * <pre>{@code\n * public class PointAdapter extends TypeAdapter<Point> {\n *   public Point read(JsonReader reader) throws IOException {\n *     if (reader.peek() == JsonToken.NULL) {\n *       reader.nextNull();\n *       return null;\n *     }\n *     String xy = reader.nextString();\n *     String[] parts = xy.split(\",\");\n *     int x = Integer.parseInt(parts[0]);\n *     int y = Integer.parseInt(parts[1]);\n *     return new Point(x, y);\n *   }\n *   public void write(JsonWriter writer, Point value) throws IOException {\n *     if (value == null) {\n *       writer.nullValue();\n *       return;\n *     }\n *     String xy = value.getX() + \",\" + value.getY();\n *     writer.value(xy);\n *   }\n * }\n * }</pre>\n *\n * With this type adapter installed, Gson will convert {@code Points} to JSON as strings like {@code\n * \"5,8\"} rather than objects like {@code {\"x\":5,\"y\":8}}. In this case the type adapter binds a rich\n * Java class to a compact JSON value.\n *\n * <p>The {@link #read(JsonReader) read()} method must read exactly one value and {@link\n * #write(JsonWriter,Object) write()} must write exactly one value. For primitive types this is\n * means readers should make exactly one call to {@code nextBoolean()}, {@code nextDouble()}, {@code\n * nextInt()}, {@code nextLong()}, {@code nextString()} or {@code nextNull()}. Writers should make\n * exactly one call to one of {@code value()} or {@code nullValue()}. For arrays, type adapters\n * should start with a call to {@code beginArray()}, convert all elements, and finish with a call to\n * {@code endArray()}. For objects, they should start with {@code beginObject()}, convert the\n * object, and finish with {@code endObject()}. Failing to convert a value or converting too many\n * values may cause the application to crash.\n *\n * <p>Type adapters should be prepared to read null from the stream and write it to the stream.\n * Alternatively, they should use {@link #nullSafe()} method while registering the type adapter with\n * Gson. If your {@code Gson} instance has been configured to {@link GsonBuilder#serializeNulls()},\n * these nulls will be written to the final document. Otherwise the value (and the corresponding\n * name when writing to a JSON object) will be omitted automatically. In either case your type\n * adapter must handle null.\n *\n * <p>Type adapters should be stateless and thread-safe, otherwise the thread-safety guarantees of\n * {@link Gson} might not apply.\n *\n * <p>To use a custom type adapter with Gson, you must <i>register</i> it with a {@link\n * GsonBuilder}:\n *\n * <pre>{@code\n * GsonBuilder builder = new GsonBuilder();\n * builder.registerTypeAdapter(Point.class, new PointAdapter());\n * // if PointAdapter didn't check for nulls in its read/write methods, you should instead use\n * // builder.registerTypeAdapter(Point.class, new PointAdapter().nullSafe());\n * ...\n * Gson gson = builder.create();\n * }</pre>\n *\n * @since 2.1\n */\n// non-Javadoc:\n//\n// <h2>JSON Conversion</h2>\n// <p>A type adapter registered with Gson is automatically invoked while serializing\n// or deserializing JSON. However, you can also use type adapters directly to serialize\n// and deserialize JSON. Here is an example for deserialization: <pre>{@code\n//   String json = \"{'origin':'0,0','points':['1,2','3,4']}\";\n//   TypeAdapter<Graph> graphAdapter = gson.getAdapter(Graph.class);\n//   Graph graph = graphAdapter.fromJson(json);\n// }</pre>\n// And an example for serialization: <pre>{@code\n//   Graph graph = new Graph(...);\n//   TypeAdapter<Graph> graphAdapter = gson.getAdapter(Graph.class);\n//   String json = graphAdapter.toJson(graph);\n// }</pre>\n//\n// <p>Type adapters are <strong>type-specific</strong>. For example, a {@code\n// TypeAdapter<Date>} can convert {@code Date} instances to JSON and JSON to\n// instances of {@code Date}, but cannot convert any other types.\n//\npublic abstract class TypeAdapter<T> {\n\n  public TypeAdapter() {}\n\n  /**\n   * Writes one JSON value (an array, object, string, number, boolean or null) for {@code value}.\n   *\n   * @param value the Java object to write. May be null.\n   */\n  public abstract void write(JsonWriter out, T value) throws IOException;\n\n  /**\n   * Converts {@code value} to a JSON document and writes it to {@code out}.\n   *\n   * <p>A {@link JsonWriter} with default configuration is used for writing the JSON data. To\n   * customize this behavior, create a {@link JsonWriter}, configure it and then use {@link\n   * #write(JsonWriter, Object)} instead.\n   *\n   * @param value the Java object to convert. May be {@code null}.\n   * @since 2.2\n   */\n  public final void toJson(Writer out, T value) throws IOException {\n    JsonWriter writer = new JsonWriter(out);\n    write(writer, value);\n  }\n\n  /**\n   * Converts {@code value} to a JSON document.\n   *\n   * <p>A {@link JsonWriter} with default configuration is used for writing the JSON data. To\n   * customize this behavior, create a {@link JsonWriter}, configure it and then use {@link\n   * #write(JsonWriter, Object)} instead.\n   *\n   * @throws JsonIOException wrapping {@code IOException}s thrown by {@link #write(JsonWriter,\n   *     Object)}\n   * @param value the Java object to convert. May be {@code null}.\n   * @since 2.2\n   */\n  public final String toJson(T value) {\n    StringWriter stringWriter = new StringWriter();\n    try {\n      toJson(stringWriter, value);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    }\n    return stringWriter.toString();\n  }\n\n  /**\n   * Converts {@code value} to a JSON tree.\n   *\n   * @param value the Java object to convert. May be {@code null}.\n   * @return the converted JSON tree. May be {@link JsonNull}.\n   * @throws JsonIOException wrapping {@code IOException}s thrown by {@link #write(JsonWriter,\n   *     Object)}\n   * @since 2.2\n   */\n  public final JsonElement toJsonTree(T value) {\n    try {\n      JsonTreeWriter jsonWriter = new JsonTreeWriter();\n      write(jsonWriter, value);\n      return jsonWriter.get();\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    }\n  }\n\n  /**\n   * Reads one JSON value (an array, object, string, number, boolean or null) and converts it to a\n   * Java object. Returns the converted object.\n   *\n   * @return the converted Java object. May be {@code null}.\n   */\n  public abstract T read(JsonReader in) throws IOException;\n\n  /**\n   * Converts the JSON document in {@code in} to a Java object.\n   *\n   * <p>A {@link JsonReader} with default configuration (that is with {@link\n   * Strictness#LEGACY_STRICT} as strictness) is used for reading the JSON data. To customize this\n   * behavior, create a {@link JsonReader}, configure it and then use {@link #read(JsonReader)}\n   * instead.\n   *\n   * <p>No exception is thrown if the JSON data has multiple top-level JSON elements, or if there is\n   * trailing data.\n   *\n   * @return the converted Java object. May be {@code null}.\n   * @since 2.2\n   */\n  public final T fromJson(Reader in) throws IOException {\n    JsonReader reader = new JsonReader(in);\n    return read(reader);\n  }\n\n  /**\n   * Converts the JSON document in {@code json} to a Java object.\n   *\n   * <p>A {@link JsonReader} with default configuration (that is with {@link\n   * Strictness#LEGACY_STRICT} as strictness) is used for reading the JSON data. To customize this\n   * behavior, create a {@link JsonReader}, configure it and then use {@link #read(JsonReader)}\n   * instead.\n   *\n   * <p>No exception is thrown if the JSON data has multiple top-level JSON elements, or if there is\n   * trailing data.\n   *\n   * @return the converted Java object. May be {@code null}.\n   * @since 2.2\n   */\n  public final T fromJson(String json) throws IOException {\n    return fromJson(new StringReader(json));\n  }\n\n  /**\n   * Converts {@code jsonTree} to a Java object.\n   *\n   * @param jsonTree the JSON element to convert. May be {@link JsonNull}.\n   * @return the converted Java object. May be {@code null}.\n   * @throws JsonIOException wrapping {@code IOException}s thrown by {@link #read(JsonReader)}\n   * @since 2.2\n   */\n  public final T fromJsonTree(JsonElement jsonTree) {\n    try {\n      JsonReader jsonReader = new JsonTreeReader(jsonTree);\n      return read(jsonReader);\n    } catch (IOException e) {\n      throw new JsonIOException(e);\n    }\n  }\n\n  /**\n   * This wrapper method is used to make a type adapter null tolerant. In general, a type adapter is\n   * required to handle nulls in write and read methods. Here is how this is typically done:<br>\n   *\n...\n", "needle_token_start": 5553, "needle_token_end": 5626, "code_context_ntokens": 16030, "output": ["```java\n/**\n * Returns a deep copy of this element. Immutable elements like primitives and nulls are not\n * copied.\n *\n * @since 2.8.2\n */\npublic abstract JsonElement deepCopy();\n```"]}
{"repo": "google/gson", "name": "nonNull", "language": "java", "path": "gson/src/main/java/com/google/gson/internal/NonNullElementWrapperList.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function ensures that the provided element is not null, throwing an exception if it is.\n2. **Input**: A single element of a generic type.\n3. **Output**: The same element if it is not null; otherwise, it throws a `NullPointerException`.\n4. **Procedure**: The function checks if the input element is null. If it is, it throws a `NullPointerException` with a specific error message. If the element is not null, it simply returns the element.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/com/google/gson/FieldNamingPolicy.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport java.lang.reflect.Field;\nimport java.util.Locale;\n\n/**\n * An enumeration that defines a few standard naming conventions for JSON field names. This\n * enumeration should be used in conjunction with {@link com.google.gson.GsonBuilder} to configure a\n * {@link com.google.gson.Gson} instance to properly translate Java field names into the desired\n * JSON field names.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic enum FieldNamingPolicy implements FieldNamingStrategy {\n\n  /** Using this naming policy with Gson will ensure that the field name is unchanged. */\n  IDENTITY() {\n    @Override\n    public String translateName(Field f) {\n      return f.getName();\n    }\n  },\n\n  /**\n   * Using this naming policy with Gson will ensure that the first \"letter\" of the Java field name\n   * is capitalized when serialized to its JSON form.\n   *\n   * <p>Here are a few examples of the form \"Java Field Name\" ---&gt; \"JSON Field Name\":\n   *\n   * <ul>\n   *   <li>someFieldName ---&gt; SomeFieldName\n   *   <li>_someFieldName ---&gt; _SomeFieldName\n   * </ul>\n   */\n  UPPER_CAMEL_CASE() {\n    @Override\n    public String translateName(Field f) {\n      return upperCaseFirstLetter(f.getName());\n    }\n  },\n\n  /**\n   * Using this naming policy with Gson will ensure that the first \"letter\" of the Java field name\n   * is capitalized when serialized to its JSON form and the words will be separated by a space.\n   *\n   * <p>Here are a few examples of the form \"Java Field Name\" ---&gt; \"JSON Field Name\":\n   *\n   * <ul>\n   *   <li>someFieldName ---&gt; Some Field Name\n   *   <li>_someFieldName ---&gt; _Some Field Name\n   * </ul>\n   *\n   * @since 1.4\n   */\n  UPPER_CAMEL_CASE_WITH_SPACES() {\n    @Override\n    public String translateName(Field f) {\n      return upperCaseFirstLetter(separateCamelCase(f.getName(), ' '));\n    }\n  },\n\n  /**\n   * Using this naming policy with Gson will modify the Java Field name from its camel cased form to\n   * an upper case field name where each word is separated by an underscore (_).\n   *\n   * <p>Here are a few examples of the form \"Java Field Name\" ---&gt; \"JSON Field Name\":\n   *\n   * <ul>\n   *   <li>someFieldName ---&gt; SOME_FIELD_NAME\n   *   <li>_someFieldName ---&gt; _SOME_FIELD_NAME\n   *   <li>aStringField ---&gt; A_STRING_FIELD\n   *   <li>aURL ---&gt; A_U_R_L\n   * </ul>\n   *\n   * @since 2.9.0\n   */\n  UPPER_CASE_WITH_UNDERSCORES() {\n    @Override\n    public String translateName(Field f) {\n...\n// Path: gson/src/main/java/com/google/gson/LongSerializationPolicy.java\n/*\n * Copyright (C) 2009 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\n/**\n * Defines the expected format for a {@code long} or {@code Long} type when it is serialized.\n *\n * @since 1.3\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic enum LongSerializationPolicy {\n  /**\n   * This is the \"default\" serialization policy that will output a {@code Long} object as a JSON\n   * number. For example, assume an object has a long field named \"f\" then the serialized output\n   * would be: {@code {\"f\":123}}\n   *\n   * <p>A {@code null} value is serialized as {@link JsonNull}.\n   */\n  DEFAULT() {\n    @Override\n    public JsonElement serialize(Long value) {\n      if (value == null) {\n        return JsonNull.INSTANCE;\n      }\n      return new JsonPrimitive(value);\n    }\n  },\n\n  /**\n   * Serializes a long value as a quoted string. For example, assume an object has a long field\n   * named \"f\" then the serialized output would be: {@code {\"f\":\"123\"}}\n   *\n   * <p>A {@code null} value is serialized as {@link JsonNull}.\n   */\n  STRING() {\n    @Override\n    public JsonElement serialize(Long value) {\n      if (value == null) {\n        return JsonNull.INSTANCE;\n      }\n      return new JsonPrimitive(value.toString());\n    }\n  };\n\n  /**\n   * Serialize this {@code value} using this serialization policy.\n   *\n   * @param value the long value to be serialized into a {@link JsonElement}\n   * @return the serialized version of {@code value}\n   */\n  public abstract JsonElement serialize(Long value);\n}\n\n// Path: gson/src/main/java/com/google/gson/Strictness.java\npackage com.google.gson;\n\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonWriter;\n\n/**\n * Modes that indicate how strictly a JSON {@linkplain JsonReader reader} or {@linkplain JsonWriter\n * writer} follows the syntax laid out in the <a href=\"https://www.ietf.org/rfc/rfc8259.txt\">RFC\n * 8259 JSON specification</a>.\n *\n * <p>You can look at {@link JsonReader#setStrictness(Strictness)} to see how the strictness affects\n * the {@link JsonReader} and you can look at {@link JsonWriter#setStrictness(Strictness)} to see\n * how the strictness affects the {@link JsonWriter}.\n *\n * @see JsonReader#setStrictness(Strictness)\n * @see JsonWriter#setStrictness(Strictness)\n * @since $next-version$\n */\npublic enum Strictness {\n  /** Allow large deviations from the JSON specification. */\n  LENIENT,\n\n  /** Allow certain small deviations from the JSON specification for legacy reasons. */\n  LEGACY_STRICT,\n\n  /** Strict compliance with the JSON specification. */\n  STRICT\n}\n\n// Path: gson/src/main/java/com/google/gson/annotations/Since.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.annotations;\n\nimport com.google.gson.GsonBuilder;\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * An annotation that indicates the version number since a member or a type has been present. This\n * annotation is useful to manage versioning of your JSON classes for a web-service.\n *\n * <p>This annotation has no effect unless you build {@link com.google.gson.Gson} with a {@code\n * GsonBuilder} and invoke the {@link GsonBuilder#setVersion(double)} method.\n *\n * <p>Here is an example of how this annotation is meant to be used:\n *\n * <pre>\n * public class User {\n *   private String firstName;\n *   private String lastName;\n *   &#64;Since(1.0) private String emailAddress;\n *   &#64;Since(1.0) private String password;\n *   &#64;Since(1.1) private Address address;\n * }\n * </pre>\n *\n * <p>If you created Gson with {@code new Gson()}, the {@code toJson()} and {@code fromJson()}\n * methods will use all the fields for serialization and deserialization. However, if you created\n * Gson with {@code Gson gson = new GsonBuilder().setVersion(1.0).create()} then the {@code\n * toJson()} and {@code fromJson()} methods of Gson will exclude the {@code address} field since\n * it's version number is set to {@code 1.1}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @see GsonBuilder#setVersion(double)\n * @see Until\n */\n@Documented\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.FIELD, ElementType.TYPE})\npublic @interface Since {\n  /**\n   * The value indicating a version number since this member or type has been present. The number is\n   * inclusive; annotated elements will be included if {@code gsonVersion >= value}.\n   */\n  double value();\n}\n\n// Path: gson/src/main/java/com/google/gson/annotations/Until.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.annotations;\n\nimport com.google.gson.GsonBuilder;\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * An annotation that indicates the version number until a member or a type should be present.\n * Basically, if Gson is created with a version number that is equal to or exceeds the value stored\n * in the {@code Until} annotation then the field will be ignored from the JSON output. This\n * annotation is useful to manage versioning of your JSON classes for a web-service.\n *\n * <p>This annotation has no effect unless you build {@link com.google.gson.Gson} with a {@code\n * GsonBuilder} and invoke the {@link GsonBuilder#setVersion(double)} method.\n *\n * <p>Here is an example of how this annotation is meant to be used:\n *\n * <pre>\n * public class User {\n *   private String firstName;\n *   private String lastName;\n *   &#64;Until(1.1) private String emailAddress;\n *   &#64;Until(1.1) private String password;\n * }\n * </pre>\n *\n * <p>If you created Gson with {@code new Gson()}, the {@code toJson()} and {@code fromJson()}\n * methods will use all the fields for serialization and deserialization. However, if you created\n * Gson with {@code Gson gson = new GsonBuilder().setVersion(1.2).create()} then the {@code\n * toJson()} and {@code fromJson()} methods of Gson will exclude the {@code emailAddress} and {@code\n * password} fields from the example above, because the version number passed to the GsonBuilder,\n * {@code 1.2}, exceeds the version number set on the {@code Until} annotation, {@code 1.1}, for\n * those fields.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @see GsonBuilder#setVersion(double)\n * @see Since\n * @since 1.3\n */\n@Documented\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.FIELD, ElementType.TYPE})\npublic @interface Until {\n\n  /**\n   * The value indicating a version number until this member or type should be included. The number\n   * is exclusive; annotated elements will be included if {@code gsonVersion < value}.\n   */\n  double value();\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/$Gson$Preconditions.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport java.util.Objects;\n\n/**\n * A simple utility class used to check method Preconditions.\n *\n * <pre>\n * public long divideBy(long value) {\n *   Preconditions.checkArgument(value != 0);\n *   return this.value / value;\n * }\n * </pre>\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class $Gson$Preconditions {\n  private $Gson$Preconditions() {\n    throw new UnsupportedOperationException();\n  }\n\n  /**\n   * @deprecated This is an internal Gson method. Use {@link Objects#requireNonNull(Object)}\n   *     instead.\n   */\n  // Only deprecated for now because external projects might be using this by accident\n  @Deprecated\n  public static <T> T checkNotNull(T obj) {\n    if (obj == null) {\n      throw new NullPointerException();\n    }\n    return obj;\n  }\n\n  public static void checkArgument(boolean condition) {\n    if (!condition) {\n      throw new IllegalArgumentException();\n    }\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/ExclusionStrategy.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\n/**\n * A strategy (or policy) definition that is used to decide whether or not a field or class should\n * be serialized or deserialized as part of the JSON output/input.\n *\n * <p>The following are a few examples that shows how you can use this exclusion mechanism.\n *\n * <p><strong>Exclude fields and objects based on a particular class type:</strong>\n *\n * <pre class=\"code\">\n * private static class SpecificClassExclusionStrategy implements ExclusionStrategy {\n *   private final Class&lt;?&gt; excludedThisClass;\n *\n *   public SpecificClassExclusionStrategy(Class&lt;?&gt; excludedThisClass) {\n *     this.excludedThisClass = excludedThisClass;\n *   }\n *\n *   public boolean shouldSkipClass(Class&lt;?&gt; clazz) {\n *     return excludedThisClass.equals(clazz);\n *   }\n *\n *   public boolean shouldSkipField(FieldAttributes f) {\n *     return excludedThisClass.equals(f.getDeclaredClass());\n *   }\n * }\n * </pre>\n *\n * <p><strong>Excludes fields and objects based on a particular annotation:</strong>\n *\n * <pre class=\"code\">\n * public &#64;interface FooAnnotation {\n *   // some implementation here\n * }\n *\n * // Excludes any field (or class) that is tagged with an \"&#64;FooAnnotation\"\n * private static class FooAnnotationExclusionStrategy implements ExclusionStrategy {\n *   public boolean shouldSkipClass(Class&lt;?&gt; clazz) {\n *     return clazz.getAnnotation(FooAnnotation.class) != null;\n *   }\n *\n *   public boolean shouldSkipField(FieldAttributes f) {\n *     return f.getAnnotation(FooAnnotation.class) != null;\n *   }\n * }\n * </pre>\n *\n * <p>Now if you want to configure {@code Gson} to use a user defined exclusion strategy, then the\n * {@code GsonBuilder} is required. The following is an example of how you can use the {@code\n * GsonBuilder} to configure Gson to use one of the above samples:\n *\n * <pre class=\"code\">\n * ExclusionStrategy excludeStrings = new UserDefinedExclusionStrategy(String.class);\n * Gson gson = new GsonBuilder()\n *     .setExclusionStrategies(excludeStrings)\n *     .create();\n * </pre>\n *\n * <p>For certain model classes, you may only want to serialize a field, but exclude it for\n * deserialization. To do that, you can write an {@code ExclusionStrategy} as per normal; however,\n * you would register it with the {@link\n * GsonBuilder#addDeserializationExclusionStrategy(ExclusionStrategy)} method. For example:\n *\n * <pre class=\"code\">\n * ExclusionStrategy excludeStrings = new UserDefinedExclusionStrategy(String.class);\n * Gson gson = new GsonBuilder()\n *     .addDeserializationExclusionStrategy(excludeStrings)\n *     .create();\n * </pre>\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @see GsonBuilder#setExclusionStrategies(ExclusionStrategy...)\n * @see GsonBuilder#addDeserializationExclusionStrategy(ExclusionStrategy)\n * @see GsonBuilder#addSerializationExclusionStrategy(ExclusionStrategy)\n * @since 1.4\n */\npublic interface ExclusionStrategy {\n\n  /**\n   * Decides if a field should be skipped during serialization or deserialization.\n   *\n   * @param f the field object that is under test\n   * @return true if the field should be ignored; otherwise false\n   */\n  public boolean shouldSkipField(FieldAttributes f);\n\n  /**\n   * Decides if a class should be serialized or deserialized\n   *\n   * @param clazz the class object that is under test\n   * @return true if the class should be ignored; otherwise false\n   */\n  public boolean shouldSkipClass(Class<?> clazz);\n}\n\n// Path: gson/src/main/java/com/google/gson/FieldAttributes.java\n/*\n * Copyright (C) 2009 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Field;\nimport java.lang.reflect.Type;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.Objects;\n\n/**\n * A data object that stores attributes of a field.\n *\n * <p>This class is immutable; therefore, it can be safely shared across threads.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @since 1.4\n */\npublic final class FieldAttributes {\n  private final Field field;\n\n  /**\n   * Constructs a Field Attributes object from the {@code f}.\n   *\n   * @param f the field to pull attributes from\n   */\n  public FieldAttributes(Field f) {\n    this.field = Objects.requireNonNull(f);\n  }\n\n  /**\n   * Gets the declaring Class that contains this field\n   *\n   * @return the declaring class that contains this field\n   */\n  public Class<?> getDeclaringClass() {\n    return field.getDeclaringClass();\n  }\n\n  /**\n   * Gets the name of the field\n   *\n   * @return the name of the field\n   */\n  public String getName() {\n    return field.getName();\n  }\n\n  /**\n   * Returns the declared generic type of the field.\n   *\n   * <p>For example, assume the following class definition:\n   *\n   * <pre class=\"code\">\n   * public class Foo {\n   *   private String bar;\n   *   private List&lt;String&gt; red;\n   * }\n   *\n   * Type listParameterizedType = new TypeToken&lt;List&lt;String&gt;&gt;() {}.getType();\n   * </pre>\n   *\n   * <p>This method would return {@code String.class} for the {@code bar} field and {@code\n   * listParameterizedType} for the {@code red} field.\n   *\n   * @return the specific type declared for this field\n   */\n  public Type getDeclaredType() {\n    return field.getGenericType();\n  }\n\n  /**\n   * Returns the {@code Class} object that was declared for this field.\n   *\n   * <p>For example, assume the following class definition:\n   *\n   * <pre class=\"code\">\n   * public class Foo {\n   *   private String bar;\n   *   private List&lt;String&gt; red;\n   * }\n   * </pre>\n   *\n   * <p>This method would return {@code String.class} for the {@code bar} field and {@code\n   * List.class} for the {@code red} field.\n   *\n   * @return the specific class object that was declared for the field\n   */\n  public Class<?> getDeclaredClass() {\n    return field.getType();\n  }\n\n  /**\n   * Returns the {@code T} annotation object from this field if it exists; otherwise returns {@code\n   * null}.\n   *\n   * @param annotation the class of the annotation that will be retrieved\n   * @return the annotation instance if it is bound to the field; otherwise {@code null}\n   */\n  public <T extends Annotation> T getAnnotation(Class<T> annotation) {\n    return field.getAnnotation(annotation);\n  }\n\n  /**\n   * Returns the annotations that are present on this field.\n   *\n   * @return an array of all the annotations set on the field\n   * @since 1.4\n   */\n  public Collection<Annotation> getAnnotations() {\n    return Arrays.asList(field.getAnnotations());\n  }\n\n  /**\n   * Returns {@code true} if the field is defined with the {@code modifier}.\n   *\n   * <p>This method is meant to be called as:\n   *\n   * <pre class=\"code\">\n   * boolean hasPublicModifier = fieldAttribute.hasModifier(java.lang.reflect.Modifier.PUBLIC);\n   * </pre>\n   *\n   * @see java.lang.reflect.Modifier\n   */\n  public boolean hasModifier(int modifier) {\n    return (field.getModifiers() & modifier) != 0;\n  }\n\n  @Override\n  public String toString() {\n    return field.toString();\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/NonNullElementWrapperList.java\n/*\n * Copyright (C) 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport java.util.AbstractList;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.RandomAccess;\n\n/**\n * {@link List} which wraps another {@code List} but prevents insertion of {@code null} elements.\n * Methods which only perform checks with the element argument (e.g. {@link #contains(Object)}) do\n * not throw exceptions for {@code null} arguments.\n */\npublic class NonNullElementWrapperList<E> extends AbstractList<E> implements RandomAccess {\n  // Explicitly specify ArrayList as type to guarantee that delegate implements RandomAccess\n  private final ArrayList<E> delegate;\n\n  @SuppressWarnings(\"NonApiType\")\n  public NonNullElementWrapperList(ArrayList<E> delegate) {\n    this.delegate = Objects.requireNonNull(delegate);\n  }\n\n  @Override\n  public E get(int index) {\n    return delegate.get(index);\n  }\n\n  @Override\n  public int size() {\n    return delegate.size();\n  }\n\n  \nprivate E nonNull(E element) {\n    if (element == null) {\n      throw new NullPointerException(\"Element must be non-null\");\n    }\n    return element;\n  }\n\n  @Override\n  public E set(int index, E element) {\n    return delegate.set(index, nonNull(element));\n  }\n\n  @Override\n  public void add(int index, E element) {\n    delegate.add(index, nonNull(element));\n  }\n\n  @Override\n  public E remove(int index) {\n    return delegate.remove(index);\n  }\n\n  /* The following methods are overridden because their default implementation is inefficient */\n\n  @Override\n  public void clear() {\n    delegate.clear();\n  }\n\n  @SuppressWarnings(\"UngroupedOverloads\") // this is intentionally ungrouped, see comment above\n  @Override\n  public boolean remove(Object o) {\n    return delegate.remove(o);\n  }\n\n  @Override\n  public boolean removeAll(Collection<?> c) {\n    return delegate.removeAll(c);\n  }\n\n  @Override\n  public boolean retainAll(Collection<?> c) {\n    return delegate.retainAll(c);\n  }\n\n  @Override\n  public boolean contains(Object o) {\n    return delegate.contains(o);\n  }\n\n  @Override\n  public int indexOf(Object o) {\n    return delegate.indexOf(o);\n  }\n\n  @Override\n  public int lastIndexOf(Object o) {\n    return delegate.lastIndexOf(o);\n  }\n\n  @Override\n  public Object[] toArray() {\n    return delegate.toArray();\n  }\n\n  @Override\n  public <T> T[] toArray(T[] a) {\n    return delegate.toArray(a);\n  }\n\n  @Override\n  public boolean equals(Object o) {\n    return delegate.equals(o);\n  }\n\n  @Override\n  public int hashCode() {\n    return delegate.hashCode();\n  }\n\n  // TODO: Once Gson targets Java 8 also override List.sort\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonArray.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.internal.NonNullElementWrapperList;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\n\n/**\n * A class representing an array type in JSON. An array is a list of {@link JsonElement}s each of\n * which can be of a different type. This is an ordered list, meaning that the order in which\n * elements are added is preserved. This class does not support {@code null} values. If {@code null}\n * is provided as element argument to any of the methods, it is converted to a {@link JsonNull}.\n *\n * <p>{@code JsonArray} only implements the {@link Iterable} interface but not the {@link List}\n * interface. A {@code List} view of it can be obtained with {@link #asList()}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonArray extends JsonElement implements Iterable<JsonElement> {\n  private final ArrayList<JsonElement> elements;\n\n  /** Creates an empty JsonArray. */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonArray() {\n    elements = new ArrayList<>();\n  }\n\n  /**\n   * Creates an empty JsonArray with the desired initial capacity.\n   *\n   * @param capacity initial capacity.\n   * @throws IllegalArgumentException if the {@code capacity} is negative\n   * @since 2.8.1\n   */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonArray(int capacity) {\n    elements = new ArrayList<>(capacity);\n  }\n\n  /**\n   * Creates a deep copy of this element and all its children.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonArray deepCopy() {\n    if (!elements.isEmpty()) {\n      JsonArray result = new JsonArray(elements.size());\n      for (JsonElement element : elements) {\n        result.add(element.deepCopy());\n      }\n      return result;\n    }\n    return new JsonArray();\n  }\n\n  /**\n   * Adds the specified boolean to self.\n   *\n   * @param bool the boolean that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(Boolean bool) {\n    elements.add(bool == null ? JsonNull.INSTANCE : new JsonPrimitive(bool));\n  }\n\n  /**\n   * Adds the specified character to self.\n   *\n   * @param character the character that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(Character character) {\n    elements.add(character == null ? JsonNull.INSTANCE : new JsonPrimitive(character));\n  }\n\n  /**\n   * Adds the specified number to self.\n   *\n   * @param number the number that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(Number number) {\n    elements.add(number == null ? JsonNull.INSTANCE : new JsonPrimitive(number));\n  }\n\n  /**\n   * Adds the specified string to self.\n   *\n   * @param string the string that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(String string) {\n    elements.add(string == null ? JsonNull.INSTANCE : new JsonPrimitive(string));\n  }\n\n  /**\n   * Adds the specified element to self.\n   *\n   * @param element the element that needs to be added to the array.\n   */\n  public void add(JsonElement element) {\n    if (element == null) {\n      element = JsonNull.INSTANCE;\n    }\n    elements.add(element);\n  }\n\n  /**\n   * Adds all the elements of the specified array to self.\n   *\n   * @param array the array whose elements need to be added to the array.\n   */\n  public void addAll(JsonArray array) {\n    elements.addAll(array.elements);\n  }\n\n  /**\n   * Replaces the element at the specified position in this array with the specified element.\n   *\n   * @param index index of the element to replace\n   * @param element element to be stored at the specified position\n   * @return the element previously at the specified position\n   * @throws IndexOutOfBoundsException if the specified index is outside the array bounds\n   */\n  @CanIgnoreReturnValue\n  public JsonElement set(int index, JsonElement element) {\n    return elements.set(index, element == null ? JsonNull.INSTANCE : element);\n  }\n\n  /**\n   * Removes the first occurrence of the specified element from this array, if it is present. If the\n   * array does not contain the element, it is unchanged.\n   *\n   * @param element element to be removed from this array, if present\n   * @return true if this array contained the specified element, false otherwise\n   * @since 2.3\n   */\n  @CanIgnoreReturnValue\n  public boolean remove(JsonElement element) {\n    return elements.remove(element);\n  }\n\n  /**\n   * Removes the element at the specified position in this array. Shifts any subsequent elements to\n   * the left (subtracts one from their indices). Returns the element that was removed from the\n   * array.\n   *\n   * @param index index the index of the element to be removed\n   * @return the element previously at the specified position\n   * @throws IndexOutOfBoundsException if the specified index is outside the array bounds\n   * @since 2.3\n   */\n  @CanIgnoreReturnValue\n  public JsonElement remove(int index) {\n    return elements.remove(index);\n  }\n\n  /**\n   * Returns true if this array contains the specified element.\n   *\n   * @return true if this array contains the specified element.\n   * @param element whose presence in this array is to be tested\n   * @since 2.3\n   */\n  public boolean contains(JsonElement element) {\n    return elements.contains(element);\n  }\n\n  /**\n   * Returns the number of elements in the array.\n   *\n   * @return the number of elements in the array.\n   */\n  public int size() {\n    return elements.size();\n  }\n\n  /**\n   * Returns true if the array is empty.\n   *\n   * @return true if the array is empty.\n   * @since 2.8.7\n   */\n  public boolean isEmpty() {\n    return elements.isEmpty();\n  }\n\n  /**\n   * Returns an iterator to navigate the elements of the array. Since the array is an ordered list,\n   * the iterator navigates the elements in the order they were inserted.\n   *\n   * @return an iterator to navigate the elements of the array.\n   */\n  @Override\n  public Iterator<JsonElement> iterator() {\n    return elements.iterator();\n  }\n\n  /**\n   * Returns the i-th element of the array.\n   *\n   * @param i the index of the element that is being sought.\n   * @return the element present at the i-th index.\n   * @throws IndexOutOfBoundsException if {@code i} is negative or greater than or equal to the\n   *     {@link #size()} of the array.\n   */\n  public JsonElement get(int i) {\n    return elements.get(i);\n  }\n\n  private JsonElement getAsSingleElement() {\n    int size = elements.size();\n    if (size == 1) {\n      return elements.get(0);\n    }\n    throw new IllegalStateException(\"Array must have size 1, but has size \" + size);\n  }\n\n  /**\n   * Convenience method to get this array as a {@link Number} if it contains a single element. This\n   * method calls {@link JsonElement#getAsNumber()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a number if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public Number getAsNumber() {\n    return getAsSingleElement().getAsNumber();\n  }\n\n  /**\n   * Convenience method to get this array as a {@link String} if it contains a single element. This\n   * method calls {@link JsonElement#getAsString()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a String if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public String getAsString() {\n    return getAsSingleElement().getAsString();\n  }\n\n  /**\n   * Convenience method to get this array as a double if it contains a single element. This method\n   * calls {@link JsonElement#getAsDouble()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a double if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public double getAsDouble() {\n    return getAsSingleElement().getAsDouble();\n  }\n\n  /**\n   * Convenience method to get this array as a {@link BigDecimal} if it contains a single element.\n   * This method calls {@link JsonElement#getAsBigDecimal()} on the element, therefore any of the\n   * exceptions declared by that method can occur.\n   *\n   * @return this element as a {@link BigDecimal} if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   * @since 1.2\n   */\n  @Override\n  public BigDecimal getAsBigDecimal() {\n    return getAsSingleElement().getAsBigDecimal();\n  }\n\n  /**\n   * Convenience method to get this array as a {@link BigInteger} if it contains a single element.\n   * This method calls {@link JsonElement#getAsBigInteger()} on the element, therefore any of the\n   * exceptions declared by that method can occur.\n   *\n   * @return this element as a {@link BigInteger} if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   * @since 1.2\n   */\n  @Override\n  public BigInteger getAsBigInteger() {\n    return getAsSingleElement().getAsBigInteger();\n  }\n\n  /**\n   * Convenience method to get this array as a float if it contains a single element. This method\n   * calls {@link JsonElement#getAsFloat()} on the element, therefore any of the exceptions declared\n   * by that method can occur.\n   *\n   * @return this element as a float if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public float getAsFloat() {\n    return getAsSingleElement().getAsFloat();\n  }\n\n  /**\n   * Convenience method to get this array as a long if it contains a single element. This method\n   * calls {@link JsonElement#getAsLong()} on the element, therefore any of the exceptions declared\n   * by that method can occur.\n   *\n   * @return this element as a long if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public long getAsLong() {\n    return getAsSingleElement().getAsLong();\n  }\n\n  /**\n   * Convenience method to get this array as an integer if it contains a single element. This method\n   * calls {@link JsonElement#getAsInt()} on the element, therefore any of the exceptions declared\n   * by that method can occur.\n   *\n   * @return this element as an integer if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public int getAsInt() {\n    return getAsSingleElement().getAsInt();\n  }\n\n  /**\n   * Convenience method to get this array as a primitive byte if it contains a single element. This\n   * method calls {@link JsonElement#getAsByte()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a primitive byte if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public byte getAsByte() {\n    return getAsSingleElement().getAsByte();\n  }\n\n  /**\n   * Convenience method to get this array as a character if it contains a single element. This\n   * method calls {@link JsonElement#getAsCharacter()} on the element, therefore any of the\n   * exceptions declared by that method can occur.\n   *\n   * @return this element as a primitive short if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   * @deprecated This method is misleading, as it does not get this element as a char but rather as\n   *     a string's first character.\n   */\n  @Deprecated\n  @Override\n  public char getAsCharacter() {\n    return getAsSingleElement().getAsCharacter();\n  }\n\n  /**\n   * Convenience method to get this array as a primitive short if it contains a single element. This\n   * method calls {@link JsonElement#getAsShort()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a primitive short if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public short getAsShort() {\n    return getAsSingleElement().getAsShort();\n  }\n\n  /**\n   * Convenience method to get this array as a boolean if it contains a single element. This method\n   * calls {@link JsonElement#getAsBoolean()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a boolean if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public boolean getAsBoolean() {\n    return getAsSingleElement().getAsBoolean();\n  }\n\n  /**\n   * Returns a mutable {@link List} view of this {@code JsonArray}. Changes to the {@code List} are\n   * visible in this {@code JsonArray} and the other way around.\n   *\n   * <p>The {@code List} does not permit {@code null} elements. Unlike {@code JsonArray}'s {@code\n   * null} handling, a {@link NullPointerException} is thrown when trying to add {@code null}. Use\n   * {@link JsonNull} for JSON null values.\n   *\n   * @return mutable {@code List} view\n   * @since 2.10\n   */\n  public List<JsonElement> asList() {\n    return new NonNullElementWrapperList<>(elements);\n  }\n\n  /**\n   * Returns whether the other object is equal to this. This method only considers the other object\n   * to be equal if it is an instance of {@code JsonArray} and has equal elements in the same order.\n   */\n  @Override\n  public boolean equals(Object o) {\n    return (o == this) || (o instanceof JsonArray && ((JsonArray) o).elements.equals(elements));\n  }\n\n  /**\n   * Returns the hash code of this array. This method calculates the hash code based on the elements\n   * of this array.\n   */\n  @Override\n  public int hashCode() {\n    return elements.hashCode();\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonIOException.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson;\n\n/**\n * This exception is raised when Gson was unable to read an input stream or write to one.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonIOException extends JsonParseException {\n  private static final long serialVersionUID = 1L;\n\n  public JsonIOException(String msg) {\n    super(msg);\n  }\n\n  public JsonIOException(String msg, Throwable cause) {\n    super(msg, cause);\n  }\n\n  /**\n   * Creates exception with the specified cause. Consider using {@link #JsonIOException(String,\n   * Throwable)} instead if you can describe what happened.\n   *\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonIOException(Throwable cause) {\n    super(cause);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonSyntaxException.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson;\n\n/**\n * This exception is raised when Gson attempts to read (or write) a malformed JSON element.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonSyntaxException extends JsonParseException {\n\n  private static final long serialVersionUID = 1L;\n\n  public JsonSyntaxException(String msg) {\n    super(msg);\n  }\n\n  public JsonSyntaxException(String msg, Throwable cause) {\n    super(msg, cause);\n  }\n\n  /**\n   * Creates exception with the specified cause. Consider using {@link #JsonSyntaxException(String,\n   * Throwable)} instead if you can describe what actually happened.\n   *\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonSyntaxException(Throwable cause) {\n    super(cause);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonPrimitive.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.gson.internal.LazilyParsedNumber;\nimport com.google.gson.internal.NumberLimits;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.util.Objects;\n\n/**\n * A class representing a JSON primitive value. A primitive value is either a String, a Java\n * primitive, or a Java primitive wrapper type.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonPrimitive extends JsonElement {\n\n  private final Object value;\n\n  /**\n   * Create a primitive containing a boolean value.\n   *\n   * @param bool the value to create the primitive with.\n   */\n  // \"deprecation\" suppression for superclass constructor\n  // \"UnnecessaryBoxedVariable\" Error Prone warning is correct since method does not accept\n  // null, but cannot be changed anymore since this is public API\n  @SuppressWarnings({\"deprecation\", \"UnnecessaryBoxedVariable\"})\n  public JsonPrimitive(Boolean bool) {\n    value = Objects.requireNonNull(bool);\n  }\n\n  /**\n   * Create a primitive containing a {@link Number}.\n   *\n   * @param number the value to create the primitive with.\n   */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonPrimitive(Number number) {\n    value = Objects.requireNonNull(number);\n  }\n\n  /**\n   * Create a primitive containing a String value.\n   *\n   * @param string the value to create the primitive with.\n   */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonPrimitive(String string) {\n    value = Objects.requireNonNull(string);\n  }\n\n  /**\n   * Create a primitive containing a character. The character is turned into a one character String\n   * since JSON only supports String.\n   *\n   * @param c the value to create the primitive with.\n   */\n  // \"deprecation\" suppression for superclass constructor\n  // \"UnnecessaryBoxedVariable\" Error Prone warning is correct since method does not accept\n  // null, but cannot be changed anymore since this is public API\n  @SuppressWarnings({\"deprecation\", \"UnnecessaryBoxedVariable\"})\n  public JsonPrimitive(Character c) {\n    // convert characters to strings since in JSON, characters are represented as a single\n    // character string\n    value = Objects.requireNonNull(c).toString();\n  }\n\n  /**\n   * Returns the same value as primitives are immutable.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonPrimitive deepCopy() {\n    return this;\n  }\n\n  /**\n   * Check whether this primitive contains a boolean value.\n   *\n   * @return true if this primitive contains a boolean value, false otherwise.\n   */\n  public boolean isBoolean() {\n    return value instanceof Boolean;\n  }\n\n  /**\n   * Convenience method to get this element as a boolean value. If this primitive {@linkplain\n   * #isBoolean() is not a boolean}, the string value is parsed using {@link\n   * Boolean#parseBoolean(String)}. This means {@code \"true\"} (ignoring case) is considered {@code\n   * true} and any other value is considered {@code false}.\n   */\n  @Override\n  public boolean getAsBoolean() {\n    if (isBoolean()) {\n      return (Boolean) value;\n    }\n    // Check to see if the value as a String is \"true\" in any case.\n    return Boolean.parseBoolean(getAsString());\n  }\n\n  /**\n   * Check whether this primitive contains a Number.\n   *\n   * @return true if this primitive contains a Number, false otherwise.\n   */\n  public boolean isNumber() {\n    return value instanceof Number;\n  }\n\n  /**\n   * Convenience method to get this element as a {@link Number}. If this primitive {@linkplain\n   * #isString() is a string}, a lazily parsed {@code Number} is constructed which parses the string\n   * when any of its methods are called (which can lead to a {@link NumberFormatException}).\n   *\n   * @throws UnsupportedOperationException if this primitive is neither a number nor a string.\n   */\n  @Override\n  public Number getAsNumber() {\n    if (value instanceof Number) {\n      return (Number) value;\n    } else if (value instanceof String) {\n      return new LazilyParsedNumber((String) value);\n    }\n    throw new UnsupportedOperationException(\"Primitive is neither a number nor a string\");\n  }\n\n  /**\n   * Check whether this primitive contains a String value.\n   *\n   * @return true if this primitive contains a String value, false otherwise.\n   */\n  public boolean isString() {\n    return value instanceof String;\n  }\n\n  // Don't add Javadoc, inherit it from super implementation; no exceptions are thrown here\n  @Override\n  public String getAsString() {\n    if (value instanceof String) {\n      return (String) value;\n    } else if (isNumber()) {\n      return getAsNumber().toString();\n    } else if (isBoolean()) {\n      return ((Boolean) value).toString();\n    }\n    throw new AssertionError(\"Unexpected value type: \" + value.getClass());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public double getAsDouble() {\n    return isNumber() ? getAsNumber().doubleValue() : Double.parseDouble(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public BigDecimal getAsBigDecimal() {\n    return value instanceof BigDecimal\n        ? (BigDecimal) value\n        : NumberLimits.parseBigDecimal(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public BigInteger getAsBigInteger() {\n    return value instanceof BigInteger\n        ? (BigInteger) value\n        : isIntegral(this)\n            ? BigInteger.valueOf(this.getAsNumber().longValue())\n            : NumberLimits.parseBigInteger(this.getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public float getAsFloat() {\n    return isNumber() ? getAsNumber().floatValue() : Float.parseFloat(getAsString());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive long.\n   *\n   * @return this element as a primitive long.\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public long getAsLong() {\n    return isNumber() ? getAsNumber().longValue() : Long.parseLong(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public short getAsShort() {\n    return isNumber() ? getAsNumber().shortValue() : Short.parseShort(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public int getAsInt() {\n    return isNumber() ? getAsNumber().intValue() : Integer.parseInt(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public byte getAsByte() {\n    return isNumber() ? getAsNumber().byteValue() : Byte.parseByte(getAsString());\n  }\n\n  /**\n   * @throws UnsupportedOperationException if the string value of this primitive is empty.\n   * @deprecated This method is misleading, as it does not get this element as a char but rather as\n   *     a string's first character.\n   */\n  @Deprecated\n  @Override\n  public char getAsCharacter() {\n    String s = getAsString();\n    if (s.isEmpty()) {\n      throw new UnsupportedOperationException(\"String value is empty\");\n    } else {\n      return s.charAt(0);\n    }\n  }\n\n  /** Returns the hash code of this object. */\n  @Override\n  public int hashCode() {\n    if (value == null) {\n      return 31;\n    }\n    // Using recommended hashing algorithm from Effective Java for longs and doubles\n    if (isIntegral(this)) {\n      long value = getAsNumber().longValue();\n      return (int) (value ^ (value >>> 32));\n    }\n    if (value instanceof Number) {\n      long value = Double.doubleToLongBits(getAsNumber().doubleValue());\n      return (int) (value ^ (value >>> 32));\n    }\n    return value.hashCode();\n  }\n\n  /**\n   * Returns whether the other object is equal to this. This method only considers the other object\n   * to be equal if it is an instance of {@code JsonPrimitive} and has an equal value.\n   */\n  @Override\n  public boolean equals(Object obj) {\n    if (this == obj) {\n      return true;\n    }\n    if (obj == null || getClass() != obj.getClass()) {\n      return false;\n    }\n    JsonPrimitive other = (JsonPrimitive) obj;\n    if (value == null) {\n      return other.value == null;\n    }\n    if (isIntegral(this) && isIntegral(other)) {\n      return (this.value instanceof BigInteger || other.value instanceof BigInteger)\n          ? this.getAsBigInteger().equals(other.getAsBigInteger())\n          : this.getAsNumber().longValue() == other.getAsNumber().longValue();\n    }\n    if (value instanceof Number && other.value instanceof Number) {\n      if (value instanceof BigDecimal && other.value instanceof BigDecimal) {\n        // Uses compareTo to ignore scale of values, e.g. `0` and `0.00` should be considered equal\n        return this.getAsBigDecimal().compareTo(other.getAsBigDecimal()) == 0;\n      }\n\n      double thisAsDouble = this.getAsDouble();\n      double otherAsDouble = other.getAsDouble();\n      // Don't use Double.compare(double, double) because that considers -0.0 and +0.0 not equal\n      return (thisAsDouble == otherAsDouble)\n          || (Double.isNaN(thisAsDouble) && Double.isNaN(otherAsDouble));\n    }\n    return value.equals(other.value);\n  }\n\n  /**\n   * Returns true if the specified number is an integral type (Long, Integer, Short, Byte,\n   * BigInteger)\n   */\n  private static boolean isIntegral(JsonPrimitive primitive) {\n    if (primitive.value instanceof Number) {\n      Number number = (Number) primitive.value;\n      return number instanceof BigInteger\n          || number instanceof Long\n          || number instanceof Integer\n          || number instanceof Short\n          || number instanceof Byte;\n    }\n    return false;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/$Gson$Types.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport static com.google.gson.internal.$Gson$Preconditions.checkArgument;\nimport static java.util.Objects.requireNonNull;\n\nimport java.io.Serializable;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Modifier;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport java.util.Properties;\n\n/**\n * Static methods for working with types.\n *\n * @author Bob Lee\n * @author Jesse Wilson\n */\npublic final class $Gson$Types {\n...\n", "needle_token_start": 6981, "needle_token_end": 7025, "code_context_ntokens": 15800, "output": ["```java\nprivate E nonNull(E element) {\n  if (element == null) {\n    throw new NullPointerException(\"Element must be non-null\");\n  }\n  return element;\n}\n```"]}
{"repo": "google/gson", "name": "rotateLeft", "language": "java", "path": "gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To adjust the structure of a binary tree by promoting the right child of a given node to be the new root of the subtree, which helps maintain or restore the balance of the tree after modifications such as insertions or deletions.\n2. **Input**: A node in a binary tree that is currently unbalanced due to a heavier right subtree.\n3. **Output**: There is no direct output returned by the function; however, it modifies the structure of the tree to balance the heights of the subtrees.\n4. **Procedure**: \n   - The function identifies the right child of the given node and its children.\n   - It then reassigns the left child of the right child to be the right child of the original node.\n   - The original node is set as the left child of its former right child, effectively making the right child the new root of the subtree.\n   - The parent-child relationships are updated to reflect these changes.\n   - Finally, the heights of the affected nodes are recalculated to ensure they accurately represent the heights of their respective subtrees post-rotation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/module-info.java\n/*\n * Copyright (C) 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Defines the Gson serialization/deserialization API.\n *\n * @since 2.8.6\n */\nmodule com.google.gson {\n  exports com.google.gson;\n  exports com.google.gson.annotations;\n  exports com.google.gson.reflect;\n  exports com.google.gson.stream;\n\n  // Dependency on Error Prone Annotations\n  requires static com.google.errorprone.annotations;\n\n  // Optional dependency on java.sql\n  requires static java.sql;\n\n  // Optional dependency on jdk.unsupported for JDK's sun.misc.Unsafe\n  requires static jdk.unsupported;\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonParseException.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\n/**\n * This exception is raised if there is a serious issue that occurs during parsing of a Json string.\n * One of the main usages for this class is for the Gson infrastructure. If the incoming Json is\n * bad/malicious, an instance of this exception is raised.\n *\n * <p>This exception is a {@link RuntimeException} because it is exposed to the client. Using a\n * {@link RuntimeException} avoids bad coding practices on the client side where they catch the\n * exception and do nothing. It is often the case that you want to blow up if there is a parsing\n * error (i.e. often clients do not know how to recover from a {@link JsonParseException}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic class JsonParseException extends RuntimeException {\n  static final long serialVersionUID = -4086729973971783390L;\n\n  /**\n   * Creates exception with the specified message. If you are wrapping another exception, consider\n   * using {@link #JsonParseException(String, Throwable)} instead.\n   *\n   * @param msg error message describing a possible cause of this exception.\n   */\n  public JsonParseException(String msg) {\n    super(msg);\n  }\n\n  /**\n   * Creates exception with the specified message and cause.\n   *\n   * @param msg error message describing what happened.\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonParseException(String msg, Throwable cause) {\n    super(msg, cause);\n  }\n\n  /**\n   * Creates exception with the specified cause. Consider using {@link #JsonParseException(String,\n   * Throwable)} instead if you can describe what happened.\n   *\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonParseException(Throwable cause) {\n    super(cause);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/InstanceCreator.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport java.lang.reflect.Type;\n\n/**\n * This interface is implemented to create instances of a class that does not define a no-args\n * constructor. If you can modify the class, you should instead add a private, or public no-args\n * constructor. However, that is not possible for library classes, such as JDK classes, or a\n * third-party library that you do not have source-code of. In such cases, you should define an\n * instance creator for the class. Implementations of this interface should be registered with\n * {@link GsonBuilder#registerTypeAdapter(Type, Object)} method before Gson will be able to use\n * them.\n *\n * <p>Let us look at an example where defining an InstanceCreator might be useful. The {@code Id}\n * class defined below does not have a default no-args constructor.\n *\n * <pre>\n * public class Id&lt;T&gt; {\n *   private final Class&lt;T&gt; clazz;\n *   private final long value;\n *   public Id(Class&lt;T&gt; clazz, long value) {\n *     this.clazz = clazz;\n *     this.value = value;\n *   }\n * }\n * </pre>\n *\n * <p>If Gson encounters an object of type {@code Id} during deserialization, it will throw an\n * exception. The easiest way to solve this problem will be to add a (public or private) no-args\n * constructor as follows:\n *\n * <pre>\n * private Id() {\n *   this(Object.class, 0L);\n * }\n * </pre>\n *\n * <p>However, let us assume that the developer does not have access to the source-code of the\n * {@code Id} class, or does not want to define a no-args constructor for it. The developer can\n * solve this problem by defining an {@code InstanceCreator} for {@code Id}:\n *\n * <pre>\n * class IdInstanceCreator implements InstanceCreator&lt;Id&gt; {\n *   public Id createInstance(Type type) {\n *     return new Id(Object.class, 0L);\n *   }\n * }\n * </pre>\n *\n * <p>Note that it does not matter what the fields of the created instance contain since Gson will\n * overwrite them with the deserialized values specified in JSON. You should also ensure that a\n * <i>new</i> object is returned, not a common object since its fields will be overwritten. The\n * developer will need to register {@code IdInstanceCreator} with Gson as follows:\n *\n * <pre>\n * Gson gson = new GsonBuilder().registerTypeAdapter(Id.class, new IdInstanceCreator()).create();\n * </pre>\n *\n * @param <T> the type of object that will be created by this implementation.\n * @see GsonBuilder#registerTypeAdapter(Type, Object)\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic interface InstanceCreator<T> {\n\n  /**\n   * Gson invokes this call-back method during deserialization to create an instance of the\n   * specified type. The fields of the returned instance are overwritten with the data present in\n   * the JSON. Since the prior contents of the object are destroyed and overwritten, do not return\n   * an instance that is useful elsewhere. In particular, do not return a common instance, always\n   * use {@code new} to create a new instance.\n   *\n   * @param type the parameterized T represented as a {@link Type}.\n   * @return a default object instance of type T.\n   */\n  public T createInstance(Type type);\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonNull.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\n/**\n * A class representing a JSON {@code null} value.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @since 1.2\n */\npublic final class JsonNull extends JsonElement {\n  /**\n   * Singleton for {@code JsonNull}.\n   *\n   * @since 1.8\n   */\n  public static final JsonNull INSTANCE = new JsonNull();\n\n  /**\n   * Creates a new {@code JsonNull} object.\n   *\n   * @deprecated Deprecated since Gson version 1.8, use {@link #INSTANCE} instead.\n   */\n  @Deprecated\n  public JsonNull() {\n    // Do nothing\n  }\n\n  /**\n   * Returns the same instance since it is an immutable value.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonNull deepCopy() {\n    return INSTANCE;\n  }\n\n  /** All instances of {@code JsonNull} have the same hash code since they are indistinguishable. */\n  @Override\n  public int hashCode() {\n    return JsonNull.class.hashCode();\n  }\n\n  /** All instances of {@code JsonNull} are considered equal. */\n  @Override\n  public boolean equals(Object other) {\n    return other instanceof JsonNull;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java\n/*\n * Copyright (C) 2010 The Android Open Source Project\n * Copyright (C) 2012 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport java.io.IOException;\nimport java.io.InvalidObjectException;\nimport java.io.ObjectInputStream;\nimport java.io.ObjectStreamException;\nimport java.io.Serializable;\nimport java.util.AbstractMap;\nimport java.util.AbstractSet;\nimport java.util.Comparator;\nimport java.util.ConcurrentModificationException;\nimport java.util.Iterator;\nimport java.util.LinkedHashMap;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport java.util.Set;\n\n/**\n * A map of comparable keys to values. Unlike {@code TreeMap}, this class uses insertion order for\n * iteration order. Comparison order is only used as an optimization for efficient insertion and\n * removal.\n *\n * <p>This implementation was derived from Android 4.1's TreeMap class.\n */\n@SuppressWarnings(\"serial\") // ignore warning about missing serialVersionUID\npublic final class LinkedTreeMap<K, V> extends AbstractMap<K, V> implements Serializable {\n  @SuppressWarnings({\"unchecked\", \"rawtypes\"}) // to avoid Comparable<Comparable<Comparable<...>>>\n  private static final Comparator<Comparable> NATURAL_ORDER =\n      new Comparator<Comparable>() {\n        @Override\n        public int compare(Comparable a, Comparable b) {\n          return a.compareTo(b);\n        }\n      };\n\n  private final Comparator<? super K> comparator;\n  private final boolean allowNullValues;\n  Node<K, V> root;\n  int size = 0;\n  int modCount = 0;\n\n  // Used to preserve iteration order\n  final Node<K, V> header;\n\n  /**\n   * Create a natural order, empty tree map whose keys must be mutually comparable and non-null, and\n   * whose values can be {@code null}.\n   */\n  @SuppressWarnings(\"unchecked\") // unsafe! this assumes K is comparable\n  public LinkedTreeMap() {\n    this((Comparator<? super K>) NATURAL_ORDER, true);\n  }\n\n  /**\n   * Create a natural order, empty tree map whose keys must be mutually comparable and non-null.\n   *\n   * @param allowNullValues whether {@code null} is allowed as entry value\n   */\n  @SuppressWarnings(\"unchecked\") // unsafe! this assumes K is comparable\n  public LinkedTreeMap(boolean allowNullValues) {\n    this((Comparator<? super K>) NATURAL_ORDER, allowNullValues);\n  }\n\n  /**\n   * Create a tree map ordered by {@code comparator}. This map's keys may only be null if {@code\n   * comparator} permits.\n   *\n   * @param comparator the comparator to order elements with, or {@code null} to use the natural\n   *     ordering.\n   * @param allowNullValues whether {@code null} is allowed as entry value\n   */\n  // unsafe! if comparator is null, this assumes K is comparable\n  @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n  public LinkedTreeMap(Comparator<? super K> comparator, boolean allowNullValues) {\n    this.comparator = comparator != null ? comparator : (Comparator) NATURAL_ORDER;\n    this.allowNullValues = allowNullValues;\n    this.header = new Node<>(allowNullValues);\n  }\n\n  @Override\n  public int size() {\n    return size;\n  }\n\n  @Override\n  public V get(Object key) {\n    Node<K, V> node = findByObject(key);\n    return node != null ? node.value : null;\n  }\n\n  @Override\n  public boolean containsKey(Object key) {\n    return findByObject(key) != null;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public V put(K key, V value) {\n    if (key == null) {\n      throw new NullPointerException(\"key == null\");\n    }\n    if (value == null && !allowNullValues) {\n      throw new NullPointerException(\"value == null\");\n    }\n    Node<K, V> created = find(key, true);\n    V result = created.value;\n    created.value = value;\n    return result;\n  }\n\n  @Override\n  public void clear() {\n    root = null;\n    size = 0;\n    modCount++;\n\n    // Clear iteration order\n    Node<K, V> header = this.header;\n    header.next = header.prev = header;\n  }\n\n  @Override\n  public V remove(Object key) {\n    Node<K, V> node = removeInternalByKey(key);\n    return node != null ? node.value : null;\n  }\n\n  /**\n   * Returns the node at or adjacent to the given key, creating it if requested.\n   *\n   * @throws ClassCastException if {@code key} and the tree's keys aren't mutually comparable.\n   */\n  Node<K, V> find(K key, boolean create) {\n    Comparator<? super K> comparator = this.comparator;\n    Node<K, V> nearest = root;\n    int comparison = 0;\n\n    if (nearest != null) {\n      // Micro-optimization: avoid polymorphic calls to Comparator.compare().\n      @SuppressWarnings(\"unchecked\") // Throws a ClassCastException below if there's trouble.\n      Comparable<Object> comparableKey =\n          (comparator == NATURAL_ORDER) ? (Comparable<Object>) key : null;\n\n      while (true) {\n        comparison =\n            (comparableKey != null)\n                ? comparableKey.compareTo(nearest.key)\n                : comparator.compare(key, nearest.key);\n\n        // We found the requested key.\n        if (comparison == 0) {\n          return nearest;\n        }\n\n        // If it exists, the key is in a subtree. Go deeper.\n        Node<K, V> child = (comparison < 0) ? nearest.left : nearest.right;\n        if (child == null) {\n          break;\n        }\n\n        nearest = child;\n      }\n    }\n\n    // The key doesn't exist in this tree.\n    if (!create) {\n      return null;\n    }\n\n    // Create the node and add it to the tree or the table.\n    Node<K, V> header = this.header;\n    Node<K, V> created;\n    if (nearest == null) {\n      // Check that the value is comparable if we didn't do any comparisons.\n      if (comparator == NATURAL_ORDER && !(key instanceof Comparable)) {\n        throw new ClassCastException(key.getClass().getName() + \" is not Comparable\");\n      }\n      created = new Node<>(allowNullValues, nearest, key, header, header.prev);\n      root = created;\n    } else {\n      created = new Node<>(allowNullValues, nearest, key, header, header.prev);\n      if (comparison < 0) { // nearest.key is higher\n        nearest.left = created;\n      } else { // comparison > 0, nearest.key is lower\n        nearest.right = created;\n      }\n      rebalance(nearest, true);\n    }\n    size++;\n    modCount++;\n\n    return created;\n  }\n\n  @SuppressWarnings(\"unchecked\")\n  Node<K, V> findByObject(Object key) {\n    try {\n      return key != null ? find((K) key, false) : null;\n    } catch (ClassCastException e) {\n      return null;\n    }\n  }\n\n  /**\n   * Returns this map's entry that has the same key and value as {@code entry}, or null if this map\n   * has no such entry.\n   *\n   * <p>This method uses the comparator for key equality rather than {@code equals}. If this map's\n   * comparator isn't consistent with equals (such as {@code String.CASE_INSENSITIVE_ORDER}), then\n   * {@code remove()} and {@code contains()} will violate the collections API.\n   */\n  Node<K, V> findByEntry(Entry<?, ?> entry) {\n    Node<K, V> mine = findByObject(entry.getKey());\n    boolean valuesEqual = mine != null && equal(mine.value, entry.getValue());\n    return valuesEqual ? mine : null;\n  }\n\n  private static boolean equal(Object a, Object b) {\n    return Objects.equals(a, b);\n  }\n\n  /**\n   * Removes {@code node} from this tree, rearranging the tree's structure as necessary.\n   *\n   * @param unlink true to also unlink this node from the iteration linked list.\n   */\n  void removeInternal(Node<K, V> node, boolean unlink) {\n    if (unlink) {\n      node.prev.next = node.next;\n      node.next.prev = node.prev;\n    }\n\n    Node<K, V> left = node.left;\n    Node<K, V> right = node.right;\n    Node<K, V> originalParent = node.parent;\n    if (left != null && right != null) {\n\n      /*\n       * To remove a node with both left and right subtrees, move an\n       * adjacent node from one of those subtrees into this node's place.\n       *\n       * Removing the adjacent node may change this node's subtrees. This\n       * node may no longer have two subtrees once the adjacent node is\n       * gone!\n       */\n\n      Node<K, V> adjacent = (left.height > right.height) ? left.last() : right.first();\n      removeInternal(adjacent, false); // takes care of rebalance and size--\n\n      int leftHeight = 0;\n      left = node.left;\n      if (left != null) {\n        leftHeight = left.height;\n        adjacent.left = left;\n        left.parent = adjacent;\n        node.left = null;\n      }\n\n      int rightHeight = 0;\n      right = node.right;\n      if (right != null) {\n        rightHeight = right.height;\n        adjacent.right = right;\n        right.parent = adjacent;\n        node.right = null;\n      }\n\n      adjacent.height = Math.max(leftHeight, rightHeight) + 1;\n      replaceInParent(node, adjacent);\n      return;\n    } else if (left != null) {\n      replaceInParent(node, left);\n      node.left = null;\n    } else if (right != null) {\n      replaceInParent(node, right);\n      node.right = null;\n    } else {\n      replaceInParent(node, null);\n    }\n\n    rebalance(originalParent, false);\n    size--;\n    modCount++;\n  }\n\n  Node<K, V> removeInternalByKey(Object key) {\n    Node<K, V> node = findByObject(key);\n    if (node != null) {\n      removeInternal(node, true);\n    }\n    return node;\n  }\n\n  @SuppressWarnings(\"ReferenceEquality\")\n  private void replaceInParent(Node<K, V> node, Node<K, V> replacement) {\n    Node<K, V> parent = node.parent;\n    node.parent = null;\n    if (replacement != null) {\n      replacement.parent = parent;\n    }\n\n    if (parent != null) {\n      if (parent.left == node) {\n        parent.left = replacement;\n      } else {\n        assert parent.right == node;\n        parent.right = replacement;\n      }\n    } else {\n      root = replacement;\n    }\n  }\n\n  /**\n   * Rebalances the tree by making any AVL rotations necessary between the newly-unbalanced node and\n   * the tree's root.\n   *\n   * @param insert true if the node was unbalanced by an insert; false if it was by a removal.\n   */\n  private void rebalance(Node<K, V> unbalanced, boolean insert) {\n    for (Node<K, V> node = unbalanced; node != null; node = node.parent) {\n      Node<K, V> left = node.left;\n      Node<K, V> right = node.right;\n      int leftHeight = left != null ? left.height : 0;\n      int rightHeight = right != null ? right.height : 0;\n\n      int delta = leftHeight - rightHeight;\n      if (delta == -2) {\n        Node<K, V> rightLeft = right.left;\n        Node<K, V> rightRight = right.right;\n        int rightRightHeight = rightRight != null ? rightRight.height : 0;\n        int rightLeftHeight = rightLeft != null ? rightLeft.height : 0;\n\n        int rightDelta = rightLeftHeight - rightRightHeight;\n        if (rightDelta == -1 || (rightDelta == 0 && !insert)) {\n          rotateLeft(node); // AVL right right\n        } else {\n          assert (rightDelta == 1);\n          rotateRight(right); // AVL right left\n          rotateLeft(node);\n        }\n        if (insert) {\n          break; // no further rotations will be necessary\n        }\n\n      } else if (delta == 2) {\n        Node<K, V> leftLeft = left.left;\n        Node<K, V> leftRight = left.right;\n        int leftRightHeight = leftRight != null ? leftRight.height : 0;\n        int leftLeftHeight = leftLeft != null ? leftLeft.height : 0;\n\n        int leftDelta = leftLeftHeight - leftRightHeight;\n        if (leftDelta == 1 || (leftDelta == 0 && !insert)) {\n          rotateRight(node); // AVL left left\n        } else {\n          assert (leftDelta == -1);\n          rotateLeft(left); // AVL left right\n          rotateRight(node);\n        }\n        if (insert) {\n          break; // no further rotations will be necessary\n        }\n\n      } else if (delta == 0) {\n        node.height = leftHeight + 1; // leftHeight == rightHeight\n        if (insert) {\n          break; // the insert caused balance, so rebalancing is done!\n        }\n\n      } else {\n        assert (delta == -1 || delta == 1);\n        node.height = Math.max(leftHeight, rightHeight) + 1;\n        if (!insert) {\n          break; // the height hasn't changed, so rebalancing is done!\n        }\n      }\n    }\n  }\n\n  /** Rotates the subtree so that its root's right child is the new root. */\n  \nprivate void rotateLeft(Node<K, V> root) {\n    Node<K, V> left = root.left;\n    Node<K, V> pivot = root.right;\n    Node<K, V> pivotLeft = pivot.left;\n    Node<K, V> pivotRight = pivot.right;\n\n    // move the pivot's left child to the root's right\n    root.right = pivotLeft;\n    if (pivotLeft != null) {\n      pivotLeft.parent = root;\n    }\n\n    replaceInParent(root, pivot);\n\n    // move the root to the pivot's left\n    pivot.left = root;\n    root.parent = pivot;\n\n    // fix heights\n    root.height =\n        Math.max(left != null ? left.height : 0, pivotLeft != null ? pivotLeft.height : 0) + 1;\n    pivot.height = Math.max(root.height, pivotRight != null ? pivotRight.height : 0) + 1;\n  }\n\n  /** Rotates the subtree so that its root's left child is the new root. */\n  private void rotateRight(Node<K, V> root) {\n    Node<K, V> pivot = root.left;\n    Node<K, V> right = root.right;\n    Node<K, V> pivotLeft = pivot.left;\n    Node<K, V> pivotRight = pivot.right;\n\n    // move the pivot's right child to the root's left\n    root.left = pivotRight;\n    if (pivotRight != null) {\n      pivotRight.parent = root;\n    }\n\n    replaceInParent(root, pivot);\n\n    // move the root to the pivot's right\n    pivot.right = root;\n    root.parent = pivot;\n\n    // fixup heights\n    root.height =\n        Math.max(right != null ? right.height : 0, pivotRight != null ? pivotRight.height : 0) + 1;\n    pivot.height = Math.max(root.height, pivotLeft != null ? pivotLeft.height : 0) + 1;\n  }\n\n  private EntrySet entrySet;\n  private KeySet keySet;\n\n  @Override\n  public Set<Entry<K, V>> entrySet() {\n    EntrySet result = entrySet;\n    return result != null ? result : (entrySet = new EntrySet());\n  }\n\n  @Override\n  public Set<K> keySet() {\n    KeySet result = keySet;\n    return result != null ? result : (keySet = new KeySet());\n  }\n\n  static final class Node<K, V> implements Entry<K, V> {\n    Node<K, V> parent;\n    Node<K, V> left;\n    Node<K, V> right;\n    Node<K, V> next;\n    Node<K, V> prev;\n    final K key;\n    final boolean allowNullValue;\n    V value;\n    int height;\n\n    /** Create the header entry */\n    Node(boolean allowNullValue) {\n      key = null;\n      this.allowNullValue = allowNullValue;\n      next = prev = this;\n    }\n\n    /** Create a regular entry */\n    Node(boolean allowNullValue, Node<K, V> parent, K key, Node<K, V> next, Node<K, V> prev) {\n      this.parent = parent;\n      this.key = key;\n      this.allowNullValue = allowNullValue;\n      this.height = 1;\n      this.next = next;\n      this.prev = prev;\n      prev.next = this;\n      next.prev = this;\n    }\n\n    @Override\n    public K getKey() {\n      return key;\n    }\n\n    @Override\n    public V getValue() {\n      return value;\n    }\n\n    @Override\n    public V setValue(V value) {\n      if (value == null && !allowNullValue) {\n        throw new NullPointerException(\"value == null\");\n      }\n      V oldValue = this.value;\n      this.value = value;\n      return oldValue;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      if (o instanceof Entry) {\n        Entry<?, ?> other = (Entry<?, ?>) o;\n        return (key == null ? other.getKey() == null : key.equals(other.getKey()))\n            && (value == null ? other.getValue() == null : value.equals(other.getValue()));\n      }\n      return false;\n    }\n\n    @Override\n    public int hashCode() {\n      return (key == null ? 0 : key.hashCode()) ^ (value == null ? 0 : value.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      return key + \"=\" + value;\n    }\n\n    /** Returns the first node in this subtree. */\n    public Node<K, V> first() {\n      Node<K, V> node = this;\n      Node<K, V> child = node.left;\n      while (child != null) {\n        node = child;\n        child = node.left;\n      }\n      return node;\n    }\n\n    /** Returns the last node in this subtree. */\n    public Node<K, V> last() {\n      Node<K, V> node = this;\n      Node<K, V> child = node.right;\n      while (child != null) {\n        node = child;\n        child = node.right;\n      }\n      return node;\n    }\n  }\n\n  private abstract class LinkedTreeMapIterator<T> implements Iterator<T> {\n    Node<K, V> next = header.next;\n    Node<K, V> lastReturned = null;\n    int expectedModCount = modCount;\n\n    LinkedTreeMapIterator() {}\n\n    @Override\n    @SuppressWarnings(\"ReferenceEquality\")\n    public final boolean hasNext() {\n      return next != header;\n    }\n\n    @SuppressWarnings(\"ReferenceEquality\")\n    final Node<K, V> nextNode() {\n      Node<K, V> e = next;\n      if (e == header) {\n        throw new NoSuchElementException();\n      }\n      if (modCount != expectedModCount) {\n        throw new ConcurrentModificationException();\n      }\n      next = e.next;\n      return lastReturned = e;\n    }\n\n    @Override\n    public final void remove() {\n      if (lastReturned == null) {\n        throw new IllegalStateException();\n      }\n      removeInternal(lastReturned, true);\n      lastReturned = null;\n      expectedModCount = modCount;\n    }\n  }\n\n  class EntrySet extends AbstractSet<Entry<K, V>> {\n    @Override\n    public int size() {\n      return size;\n    }\n\n    @Override\n    public Iterator<Entry<K, V>> iterator() {\n      return new LinkedTreeMapIterator<Entry<K, V>>() {\n        @Override\n        public Entry<K, V> next() {\n          return nextNode();\n        }\n      };\n    }\n\n    @Override\n    public boolean contains(Object o) {\n      return o instanceof Entry && findByEntry((Entry<?, ?>) o) != null;\n    }\n\n    @Override\n    public boolean remove(Object o) {\n      if (!(o instanceof Entry)) {\n        return false;\n      }\n\n      Node<K, V> node = findByEntry((Entry<?, ?>) o);\n      if (node == null) {\n        return false;\n      }\n      removeInternal(node, true);\n      return true;\n    }\n\n    @Override\n    public void clear() {\n      LinkedTreeMap.this.clear();\n    }\n  }\n\n  final class KeySet extends AbstractSet<K> {\n    @Override\n    public int size() {\n      return size;\n    }\n\n    @Override\n    public Iterator<K> iterator() {\n      return new LinkedTreeMapIterator<K>() {\n        @Override\n        public K next() {\n          return nextNode().key;\n        }\n      };\n    }\n\n    @Override\n    public boolean contains(Object o) {\n      return containsKey(o);\n    }\n\n    @Override\n    public boolean remove(Object key) {\n      return removeInternalByKey(key) != null;\n    }\n\n    @Override\n    public void clear() {\n      LinkedTreeMap.this.clear();\n    }\n  }\n\n  /**\n   * If somebody is unlucky enough to have to serialize one of these, serialize it as a\n   * LinkedHashMap so that they won't need Gson on the other side to deserialize it. Using\n   * serialization defeats our DoS defence, so most apps shouldn't use it.\n   */\n  private Object writeReplace() throws ObjectStreamException {\n    return new LinkedHashMap<>(this);\n  }\n\n  private void readObject(ObjectInputStream in) throws IOException {\n    // Don't permit directly deserializing this class; writeReplace() should have written a\n    // replacement\n    throw new InvalidObjectException(\"Deserialization is unsupported\");\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonObject.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.internal.LinkedTreeMap;\nimport java.util.Map;\nimport java.util.Set;\n\n/**\n * A class representing an object type in Json. An object consists of name-value pairs where names\n * are strings, and values are any other type of {@link JsonElement}. This allows for a creating a\n * tree of JsonElements. The member elements of this object are maintained in order they were added.\n * This class does not support {@code null} values. If {@code null} is provided as value argument to\n * any of the methods, it is converted to a {@link JsonNull}.\n *\n * <p>{@code JsonObject} does not implement the {@link Map} interface, but a {@code Map} view of it\n * can be obtained with {@link #asMap()}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonObject extends JsonElement {\n  private final LinkedTreeMap<String, JsonElement> members = new LinkedTreeMap<>(false);\n\n  /** Creates an empty JsonObject. */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonObject() {}\n\n  /**\n   * Creates a deep copy of this element and all its children.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonObject deepCopy() {\n    JsonObject result = new JsonObject();\n    for (Map.Entry<String, JsonElement> entry : members.entrySet()) {\n      result.add(entry.getKey(), entry.getValue().deepCopy());\n    }\n    return result;\n  }\n\n  /**\n   * Adds a member, which is a name-value pair, to self. The name must be a String, but the value\n   * can be an arbitrary {@link JsonElement}, thereby allowing you to build a full tree of\n   * JsonElements rooted at this node.\n   *\n   * @param property name of the member.\n   * @param value the member object.\n   */\n  public void add(String property, JsonElement value) {\n    members.put(property, value == null ? JsonNull.INSTANCE : value);\n  }\n\n  /**\n   * Removes the {@code property} from this object.\n   *\n   * @param property name of the member that should be removed.\n   * @return the {@link JsonElement} object that is being removed, or {@code null} if no member with\n   *     this name exists.\n   * @since 1.3\n   */\n  @CanIgnoreReturnValue\n  public JsonElement remove(String property) {\n    return members.remove(property);\n  }\n\n  /**\n   * Convenience method to add a string member. The specified value is converted to a {@link\n   * JsonPrimitive} of String.\n   *\n   * @param property name of the member.\n   * @param value the string value associated with the member.\n   */\n  public void addProperty(String property, String value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Convenience method to add a number member. The specified value is converted to a {@link\n   * JsonPrimitive} of Number.\n   *\n   * @param property name of the member.\n   * @param value the number value associated with the member.\n   */\n  public void addProperty(String property, Number value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Convenience method to add a boolean member. The specified value is converted to a {@link\n   * JsonPrimitive} of Boolean.\n   *\n   * @param property name of the member.\n   * @param value the boolean value associated with the member.\n   */\n  public void addProperty(String property, Boolean value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Convenience method to add a char member. The specified value is converted to a {@link\n   * JsonPrimitive} of Character.\n   *\n   * @param property name of the member.\n   * @param value the char value associated with the member.\n   */\n  public void addProperty(String property, Character value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Returns a set of members of this object. The set is ordered, and the order is in which the\n   * elements were added.\n   *\n   * @return a set of members of this object.\n   */\n  public Set<Map.Entry<String, JsonElement>> entrySet() {\n    return members.entrySet();\n  }\n\n  /**\n   * Returns a set of members key values.\n   *\n   * @return a set of member keys as Strings\n   * @since 2.8.1\n   */\n  public Set<String> keySet() {\n    return members.keySet();\n  }\n\n  /**\n   * Returns the number of key/value pairs in the object.\n   *\n   * @return the number of key/value pairs in the object.\n   * @since 2.7\n   */\n  public int size() {\n    return members.size();\n  }\n\n  /**\n   * Returns true if the number of key/value pairs in the object is zero.\n   *\n   * @return true if the number of key/value pairs in the object is zero.\n   * @since 2.10.1\n   */\n  public boolean isEmpty() {\n    return members.size() == 0;\n  }\n\n  /**\n   * Convenience method to check if a member with the specified name is present in this object.\n   *\n   * @param memberName name of the member that is being checked for presence.\n   * @return true if there is a member with the specified name, false otherwise.\n   */\n  public boolean has(String memberName) {\n    return members.containsKey(memberName);\n  }\n\n  /**\n   * Returns the member with the specified name.\n   *\n   * @param memberName name of the member that is being requested.\n   * @return the member matching the name, or {@code null} if no such member exists.\n   */\n  public JsonElement get(String memberName) {\n    return members.get(memberName);\n  }\n\n  /**\n   * Convenience method to get the specified member as a {@link JsonPrimitive}.\n   *\n   * @param memberName name of the member being requested.\n   * @return the {@code JsonPrimitive} corresponding to the specified member, or {@code null} if no\n   *     member with this name exists.\n   * @throws ClassCastException if the member is not of type {@code JsonPrimitive}.\n   */\n  public JsonPrimitive getAsJsonPrimitive(String memberName) {\n    return (JsonPrimitive) members.get(memberName);\n  }\n\n  /**\n   * Convenience method to get the specified member as a {@link JsonArray}.\n   *\n   * @param memberName name of the member being requested.\n   * @return the {@code JsonArray} corresponding to the specified member, or {@code null} if no\n   *     member with this name exists.\n   * @throws ClassCastException if the member is not of type {@code JsonArray}.\n   */\n  public JsonArray getAsJsonArray(String memberName) {\n    return (JsonArray) members.get(memberName);\n  }\n\n  /**\n   * Convenience method to get the specified member as a {@link JsonObject}.\n   *\n   * @param memberName name of the member being requested.\n   * @return the {@code JsonObject} corresponding to the specified member, or {@code null} if no\n   *     member with this name exists.\n   * @throws ClassCastException if the member is not of type {@code JsonObject}.\n   */\n  public JsonObject getAsJsonObject(String memberName) {\n    return (JsonObject) members.get(memberName);\n  }\n\n  /**\n   * Returns a mutable {@link Map} view of this {@code JsonObject}. Changes to the {@code Map} are\n   * visible in this {@code JsonObject} and the other way around.\n   *\n   * <p>The {@code Map} does not permit {@code null} keys or values. Unlike {@code JsonObject}'s\n   * {@code null} handling, a {@link NullPointerException} is thrown when trying to add {@code\n   * null}. Use {@link JsonNull} for JSON null values.\n   *\n   * @return mutable {@code Map} view\n   * @since 2.10\n   */\n  public Map<String, JsonElement> asMap() {\n    // It is safe to expose the underlying map because it disallows null keys and values\n    return members;\n  }\n\n  /**\n   * Returns whether the other object is equal to this. This method only considers the other object\n   * to be equal if it is an instance of {@code JsonObject} and has equal members, ignoring order.\n   */\n  @Override\n  public boolean equals(Object o) {\n    return (o == this) || (o instanceof JsonObject && ((JsonObject) o).members.equals(members));\n  }\n\n  /**\n   * Returns the hash code of this object. This method calculates the hash code based on the members\n   * of this object, ignoring order.\n   */\n  @Override\n  public int hashCode() {\n    return members.hashCode();\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonSerializationContext.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport java.lang.reflect.Type;\n\n/**\n * Context for serialization that is passed to a custom serializer during invocation of its {@link\n * JsonSerializer#serialize(Object, Type, JsonSerializationContext)} method.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic interface JsonSerializationContext {\n\n  /**\n   * Invokes default serialization on the specified object.\n   *\n   * @param src the object that needs to be serialized.\n   * @return a tree of {@link JsonElement}s corresponding to the serialized form of {@code src}.\n   */\n  public JsonElement serialize(Object src);\n\n  /**\n   * Invokes default serialization on the specified object passing the specific type information. It\n   * should never be invoked on the element received as a parameter of the {@link\n   * JsonSerializer#serialize(Object, Type, JsonSerializationContext)} method. Doing so will result\n   * in an infinite loop since Gson will in-turn call the custom serializer again.\n   *\n   * @param src the object that needs to be serialized.\n   * @param typeOfSrc the actual genericized type of src object.\n   * @return a tree of {@link JsonElement}s corresponding to the serialized form of {@code src}.\n   */\n  public JsonElement serialize(Object src, Type typeOfSrc);\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonSerializer.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport java.lang.reflect.Type;\n\n/**\n * Interface representing a custom serializer for JSON. You should write a custom serializer, if you\n * are not happy with the default serialization done by Gson. You will also need to register this\n * serializer through {@link com.google.gson.GsonBuilder#registerTypeAdapter(Type, Object)}.\n *\n * <p>Let us look at example where defining a serializer will be useful. The {@code Id} class\n * defined below has two fields: {@code clazz} and {@code value}.\n *\n * <pre>\n * public class Id&lt;T&gt; {\n *   private final Class&lt;T&gt; clazz;\n *   private final long value;\n *\n *   public Id(Class&lt;T&gt; clazz, long value) {\n *     this.clazz = clazz;\n *     this.value = value;\n *   }\n *\n *   public long getValue() {\n *     return value;\n *   }\n * }\n * </pre>\n *\n * <p>The default serialization of {@code Id(com.foo.MyObject.class, 20L)} will be <code>\n * {\"clazz\":\"com.foo.MyObject\",\"value\":20}</code>. Suppose, you just want the output to be the value\n * instead, which is {@code 20} in this case. You can achieve that by writing a custom serializer:\n *\n * <pre>\n * class IdSerializer implements JsonSerializer&lt;Id&gt; {\n *   public JsonElement serialize(Id id, Type typeOfId, JsonSerializationContext context) {\n *     return new JsonPrimitive(id.getValue());\n *   }\n * }\n * </pre>\n *\n * <p>You will also need to register {@code IdSerializer} with Gson as follows:\n *\n * <pre>\n * Gson gson = new GsonBuilder().registerTypeAdapter(Id.class, new IdSerializer()).create();\n * </pre>\n *\n * <p>Serializers should be stateless and thread-safe, otherwise the thread-safety guarantees of\n * {@link Gson} might not apply.\n *\n * <p>New applications should prefer {@link TypeAdapter}, whose streaming API is more efficient than\n * this interface's tree API.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @param <T> type for which the serializer is being registered. It is possible that a serializer\n *     may be asked to serialize a specific generic type of the T.\n */\npublic interface JsonSerializer<T> {\n\n  /**\n   * Gson invokes this call-back method during serialization when it encounters a field of the\n   * specified type.\n   *\n   * <p>In the implementation of this call-back method, you should consider invoking {@link\n   * JsonSerializationContext#serialize(Object, Type)} method to create JsonElements for any\n   * non-trivial field of the {@code src} object. However, you should never invoke it on the {@code\n   * src} object itself since that will cause an infinite loop (Gson will call your call-back method\n   * again).\n   *\n   * @param src the object that needs to be converted to Json.\n   * @param typeOfSrc the actual type (fully genericized version) of the source object.\n   * @return a JsonElement corresponding to the specified object.\n   */\n  public JsonElement serialize(T src, Type typeOfSrc, JsonSerializationContext context);\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/NumberLimits.java\npackage com.google.gson.internal;\n\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\n\n/**\n * This class enforces limits on numbers parsed from JSON to avoid potential performance problems\n * when extremely large numbers are used.\n */\npublic class NumberLimits {\n  private NumberLimits() {}\n\n  private static final int MAX_NUMBER_STRING_LENGTH = 10_000;\n\n  private static void checkNumberStringLength(String s) {\n    if (s.length() > MAX_NUMBER_STRING_LENGTH) {\n      throw new NumberFormatException(\"Number string too large: \" + s.substring(0, 30) + \"...\");\n    }\n  }\n\n  public static BigDecimal parseBigDecimal(String s) throws NumberFormatException {\n    checkNumberStringLength(s);\n    BigDecimal decimal = new BigDecimal(s);\n\n    // Cast to long to avoid issues with abs when value is Integer.MIN_VALUE\n    if (Math.abs((long) decimal.scale()) >= 10_000) {\n      throw new NumberFormatException(\"Number has unsupported scale: \" + s);\n    }\n    return decimal;\n  }\n\n  public static BigInteger parseBigInteger(String s) throws NumberFormatException {\n    checkNumberStringLength(s);\n    return new BigInteger(s);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/LazilyParsedNumber.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson.internal;\n\nimport java.io.IOException;\nimport java.io.InvalidObjectException;\nimport java.io.ObjectInputStream;\nimport java.io.ObjectStreamException;\nimport java.math.BigDecimal;\n\n/**\n * This class holds a number value that is lazily converted to a specific number type\n *\n * @author Inderjeet Singh\n */\n@SuppressWarnings(\"serial\") // ignore warning about missing serialVersionUID\npublic final class LazilyParsedNumber extends Number {\n  private final String value;\n\n  /**\n   * @param value must not be null\n   */\n  public LazilyParsedNumber(String value) {\n    this.value = value;\n  }\n\n  private BigDecimal asBigDecimal() {\n    return NumberLimits.parseBigDecimal(value);\n  }\n\n  @Override\n  public int intValue() {\n    try {\n      return Integer.parseInt(value);\n    } catch (NumberFormatException e) {\n      try {\n        return (int) Long.parseLong(value);\n      } catch (NumberFormatException nfe) {\n        return asBigDecimal().intValue();\n      }\n    }\n  }\n\n  @Override\n  public long longValue() {\n    try {\n      return Long.parseLong(value);\n    } catch (NumberFormatException e) {\n      return asBigDecimal().longValue();\n    }\n  }\n\n  @Override\n  public float floatValue() {\n    return Float.parseFloat(value);\n  }\n\n  @Override\n  public double doubleValue() {\n    return Double.parseDouble(value);\n  }\n\n  @Override\n  public String toString() {\n...\n", "needle_token_start": 6506, "needle_token_end": 6748, "code_context_ntokens": 13775, "output": ["```java\nprivate void rotateLeft(Node<K, V> root) {\n    Node<K, V> left = root.left;\n    Node<K, V> pivot = root.right;\n    Node<K, V> pivotLeft = pivot.left;\n    Node<K, V> pivotRight = pivot.right;\n\n    // move the pivot's left child to the root's right\n    root.right = pivotLeft;\n    if (pivotLeft != null) {\n      pivotLeft.parent = root;\n    }\n\n    replaceInParent(root, pivot);\n\n    // move the root to the pivot's left\n    pivot.left = root;\n    root.parent = pivot;\n\n    // fix heights\n    root.height =\n        Math.max(left != null ? left.height : 0, pivotLeft != null ? pivotLeft.height : 0) + 1;\n    pivot.height = Math.max(root.height, pivotRight != null ? pivotRight.height : 0) + 1;\n  }\n```"]}
{"repo": "google/gson", "name": "getArrayComponentType", "language": "java", "path": "gson/src/main/java/com/google/gson/internal/$Gson$Types.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the type of elements contained within an array type, whether it is a generic or non-generic array.\n2. **Input**: A `Type` object representing an array.\n3. **Output**: A `Type` object representing the component type of the input array.\n4. **Procedure**: The function first checks if the input type is a generic array. If it is, it extracts the generic component type. If the input type is a standard array, it retrieves the component type directly. If the input is not an array type, the function throws a `ClassCastException`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/com/google/gson/JsonArray.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.internal.NonNullElementWrapperList;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\n\n/**\n * A class representing an array type in JSON. An array is a list of {@link JsonElement}s each of\n * which can be of a different type. This is an ordered list, meaning that the order in which\n * elements are added is preserved. This class does not support {@code null} values. If {@code null}\n * is provided as element argument to any of the methods, it is converted to a {@link JsonNull}.\n *\n * <p>{@code JsonArray} only implements the {@link Iterable} interface but not the {@link List}\n * interface. A {@code List} view of it can be obtained with {@link #asList()}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonArray extends JsonElement implements Iterable<JsonElement> {\n  private final ArrayList<JsonElement> elements;\n\n  /** Creates an empty JsonArray. */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonArray() {\n    elements = new ArrayList<>();\n  }\n\n  /**\n   * Creates an empty JsonArray with the desired initial capacity.\n   *\n   * @param capacity initial capacity.\n   * @throws IllegalArgumentException if the {@code capacity} is negative\n   * @since 2.8.1\n   */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonArray(int capacity) {\n    elements = new ArrayList<>(capacity);\n  }\n\n  /**\n   * Creates a deep copy of this element and all its children.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonArray deepCopy() {\n    if (!elements.isEmpty()) {\n      JsonArray result = new JsonArray(elements.size());\n      for (JsonElement element : elements) {\n        result.add(element.deepCopy());\n      }\n      return result;\n    }\n    return new JsonArray();\n  }\n\n  /**\n   * Adds the specified boolean to self.\n   *\n   * @param bool the boolean that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(Boolean bool) {\n    elements.add(bool == null ? JsonNull.INSTANCE : new JsonPrimitive(bool));\n  }\n\n  /**\n   * Adds the specified character to self.\n   *\n   * @param character the character that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(Character character) {\n    elements.add(character == null ? JsonNull.INSTANCE : new JsonPrimitive(character));\n  }\n\n  /**\n   * Adds the specified number to self.\n   *\n   * @param number the number that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(Number number) {\n    elements.add(number == null ? JsonNull.INSTANCE : new JsonPrimitive(number));\n  }\n\n  /**\n   * Adds the specified string to self.\n   *\n   * @param string the string that needs to be added to the array.\n   * @since 2.4\n   */\n  public void add(String string) {\n    elements.add(string == null ? JsonNull.INSTANCE : new JsonPrimitive(string));\n  }\n\n  /**\n   * Adds the specified element to self.\n   *\n   * @param element the element that needs to be added to the array.\n   */\n  public void add(JsonElement element) {\n    if (element == null) {\n      element = JsonNull.INSTANCE;\n    }\n    elements.add(element);\n  }\n\n  /**\n   * Adds all the elements of the specified array to self.\n   *\n   * @param array the array whose elements need to be added to the array.\n   */\n  public void addAll(JsonArray array) {\n    elements.addAll(array.elements);\n  }\n\n  /**\n   * Replaces the element at the specified position in this array with the specified element.\n   *\n   * @param index index of the element to replace\n   * @param element element to be stored at the specified position\n   * @return the element previously at the specified position\n   * @throws IndexOutOfBoundsException if the specified index is outside the array bounds\n   */\n  @CanIgnoreReturnValue\n  public JsonElement set(int index, JsonElement element) {\n    return elements.set(index, element == null ? JsonNull.INSTANCE : element);\n  }\n\n  /**\n   * Removes the first occurrence of the specified element from this array, if it is present. If the\n   * array does not contain the element, it is unchanged.\n   *\n   * @param element element to be removed from this array, if present\n   * @return true if this array contained the specified element, false otherwise\n   * @since 2.3\n   */\n  @CanIgnoreReturnValue\n  public boolean remove(JsonElement element) {\n    return elements.remove(element);\n  }\n\n  /**\n   * Removes the element at the specified position in this array. Shifts any subsequent elements to\n   * the left (subtracts one from their indices). Returns the element that was removed from the\n   * array.\n   *\n   * @param index index the index of the element to be removed\n   * @return the element previously at the specified position\n   * @throws IndexOutOfBoundsException if the specified index is outside the array bounds\n   * @since 2.3\n   */\n  @CanIgnoreReturnValue\n  public JsonElement remove(int index) {\n    return elements.remove(index);\n  }\n\n  /**\n   * Returns true if this array contains the specified element.\n   *\n   * @return true if this array contains the specified element.\n   * @param element whose presence in this array is to be tested\n   * @since 2.3\n   */\n  public boolean contains(JsonElement element) {\n    return elements.contains(element);\n  }\n\n  /**\n   * Returns the number of elements in the array.\n   *\n   * @return the number of elements in the array.\n   */\n  public int size() {\n    return elements.size();\n  }\n\n  /**\n   * Returns true if the array is empty.\n   *\n   * @return true if the array is empty.\n   * @since 2.8.7\n   */\n  public boolean isEmpty() {\n    return elements.isEmpty();\n  }\n\n  /**\n   * Returns an iterator to navigate the elements of the array. Since the array is an ordered list,\n   * the iterator navigates the elements in the order they were inserted.\n   *\n   * @return an iterator to navigate the elements of the array.\n   */\n  @Override\n  public Iterator<JsonElement> iterator() {\n    return elements.iterator();\n  }\n\n  /**\n   * Returns the i-th element of the array.\n   *\n   * @param i the index of the element that is being sought.\n   * @return the element present at the i-th index.\n   * @throws IndexOutOfBoundsException if {@code i} is negative or greater than or equal to the\n   *     {@link #size()} of the array.\n   */\n  public JsonElement get(int i) {\n    return elements.get(i);\n  }\n\n  private JsonElement getAsSingleElement() {\n    int size = elements.size();\n    if (size == 1) {\n      return elements.get(0);\n    }\n    throw new IllegalStateException(\"Array must have size 1, but has size \" + size);\n  }\n\n  /**\n   * Convenience method to get this array as a {@link Number} if it contains a single element. This\n   * method calls {@link JsonElement#getAsNumber()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a number if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public Number getAsNumber() {\n    return getAsSingleElement().getAsNumber();\n  }\n\n  /**\n   * Convenience method to get this array as a {@link String} if it contains a single element. This\n   * method calls {@link JsonElement#getAsString()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a String if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public String getAsString() {\n    return getAsSingleElement().getAsString();\n  }\n\n  /**\n   * Convenience method to get this array as a double if it contains a single element. This method\n   * calls {@link JsonElement#getAsDouble()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a double if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public double getAsDouble() {\n    return getAsSingleElement().getAsDouble();\n  }\n\n  /**\n   * Convenience method to get this array as a {@link BigDecimal} if it contains a single element.\n   * This method calls {@link JsonElement#getAsBigDecimal()} on the element, therefore any of the\n   * exceptions declared by that method can occur.\n   *\n   * @return this element as a {@link BigDecimal} if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   * @since 1.2\n   */\n  @Override\n  public BigDecimal getAsBigDecimal() {\n    return getAsSingleElement().getAsBigDecimal();\n  }\n\n  /**\n   * Convenience method to get this array as a {@link BigInteger} if it contains a single element.\n   * This method calls {@link JsonElement#getAsBigInteger()} on the element, therefore any of the\n   * exceptions declared by that method can occur.\n   *\n   * @return this element as a {@link BigInteger} if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   * @since 1.2\n   */\n  @Override\n  public BigInteger getAsBigInteger() {\n    return getAsSingleElement().getAsBigInteger();\n  }\n\n  /**\n   * Convenience method to get this array as a float if it contains a single element. This method\n   * calls {@link JsonElement#getAsFloat()} on the element, therefore any of the exceptions declared\n   * by that method can occur.\n   *\n   * @return this element as a float if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public float getAsFloat() {\n    return getAsSingleElement().getAsFloat();\n  }\n\n  /**\n   * Convenience method to get this array as a long if it contains a single element. This method\n   * calls {@link JsonElement#getAsLong()} on the element, therefore any of the exceptions declared\n   * by that method can occur.\n   *\n   * @return this element as a long if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public long getAsLong() {\n    return getAsSingleElement().getAsLong();\n  }\n\n  /**\n   * Convenience method to get this array as an integer if it contains a single element. This method\n   * calls {@link JsonElement#getAsInt()} on the element, therefore any of the exceptions declared\n   * by that method can occur.\n   *\n   * @return this element as an integer if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public int getAsInt() {\n    return getAsSingleElement().getAsInt();\n  }\n\n  /**\n   * Convenience method to get this array as a primitive byte if it contains a single element. This\n   * method calls {@link JsonElement#getAsByte()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n   *\n   * @return this element as a primitive byte if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   */\n  @Override\n  public byte getAsByte() {\n    return getAsSingleElement().getAsByte();\n  }\n\n  /**\n   * Convenience method to get this array as a character if it contains a single element. This\n   * method calls {@link JsonElement#getAsCharacter()} on the element, therefore any of the\n   * exceptions declared by that method can occur.\n   *\n   * @return this element as a primitive short if it is single element array.\n   * @throws IllegalStateException if the array is empty or has more than one element.\n   * @deprecated This method is misleading, as it does not get this element as a char but rather as\n   *     a string's first character.\n   */\n  @Deprecated\n  @Override\n  public char getAsCharacter() {\n    return getAsSingleElement().getAsCharacter();\n  }\n\n  /**\n   * Convenience method to get this array as a primitive short if it contains a single element. This\n   * method calls {@link JsonElement#getAsShort()} on the element, therefore any of the exceptions\n   * declared by that method can occur.\n...\n// Path: gson/src/main/java/com/google/gson/JsonIOException.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson;\n\n/**\n * This exception is raised when Gson was unable to read an input stream or write to one.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonIOException extends JsonParseException {\n  private static final long serialVersionUID = 1L;\n\n  public JsonIOException(String msg) {\n    super(msg);\n  }\n\n  public JsonIOException(String msg, Throwable cause) {\n    super(msg, cause);\n  }\n\n  /**\n   * Creates exception with the specified cause. Consider using {@link #JsonIOException(String,\n   * Throwable)} instead if you can describe what happened.\n   *\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonIOException(Throwable cause) {\n    super(cause);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonSyntaxException.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.google.gson;\n\n/**\n * This exception is raised when Gson attempts to read (or write) a malformed JSON element.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonSyntaxException extends JsonParseException {\n\n  private static final long serialVersionUID = 1L;\n\n  public JsonSyntaxException(String msg) {\n    super(msg);\n  }\n\n  public JsonSyntaxException(String msg, Throwable cause) {\n    super(msg, cause);\n  }\n\n  /**\n   * Creates exception with the specified cause. Consider using {@link #JsonSyntaxException(String,\n   * Throwable)} instead if you can describe what actually happened.\n   *\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonSyntaxException(Throwable cause) {\n    super(cause);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonPrimitive.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.gson.internal.LazilyParsedNumber;\nimport com.google.gson.internal.NumberLimits;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.util.Objects;\n\n/**\n * A class representing a JSON primitive value. A primitive value is either a String, a Java\n * primitive, or a Java primitive wrapper type.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonPrimitive extends JsonElement {\n\n  private final Object value;\n\n  /**\n   * Create a primitive containing a boolean value.\n   *\n   * @param bool the value to create the primitive with.\n   */\n  // \"deprecation\" suppression for superclass constructor\n  // \"UnnecessaryBoxedVariable\" Error Prone warning is correct since method does not accept\n  // null, but cannot be changed anymore since this is public API\n  @SuppressWarnings({\"deprecation\", \"UnnecessaryBoxedVariable\"})\n  public JsonPrimitive(Boolean bool) {\n    value = Objects.requireNonNull(bool);\n  }\n\n  /**\n   * Create a primitive containing a {@link Number}.\n   *\n   * @param number the value to create the primitive with.\n   */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonPrimitive(Number number) {\n    value = Objects.requireNonNull(number);\n  }\n\n  /**\n   * Create a primitive containing a String value.\n   *\n   * @param string the value to create the primitive with.\n   */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonPrimitive(String string) {\n    value = Objects.requireNonNull(string);\n  }\n\n  /**\n   * Create a primitive containing a character. The character is turned into a one character String\n   * since JSON only supports String.\n   *\n   * @param c the value to create the primitive with.\n   */\n  // \"deprecation\" suppression for superclass constructor\n  // \"UnnecessaryBoxedVariable\" Error Prone warning is correct since method does not accept\n  // null, but cannot be changed anymore since this is public API\n  @SuppressWarnings({\"deprecation\", \"UnnecessaryBoxedVariable\"})\n  public JsonPrimitive(Character c) {\n    // convert characters to strings since in JSON, characters are represented as a single\n    // character string\n    value = Objects.requireNonNull(c).toString();\n  }\n\n  /**\n   * Returns the same value as primitives are immutable.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonPrimitive deepCopy() {\n    return this;\n  }\n\n  /**\n   * Check whether this primitive contains a boolean value.\n   *\n   * @return true if this primitive contains a boolean value, false otherwise.\n   */\n  public boolean isBoolean() {\n    return value instanceof Boolean;\n  }\n\n  /**\n   * Convenience method to get this element as a boolean value. If this primitive {@linkplain\n   * #isBoolean() is not a boolean}, the string value is parsed using {@link\n   * Boolean#parseBoolean(String)}. This means {@code \"true\"} (ignoring case) is considered {@code\n   * true} and any other value is considered {@code false}.\n   */\n  @Override\n  public boolean getAsBoolean() {\n    if (isBoolean()) {\n      return (Boolean) value;\n    }\n    // Check to see if the value as a String is \"true\" in any case.\n    return Boolean.parseBoolean(getAsString());\n  }\n\n  /**\n   * Check whether this primitive contains a Number.\n   *\n   * @return true if this primitive contains a Number, false otherwise.\n   */\n  public boolean isNumber() {\n    return value instanceof Number;\n  }\n\n  /**\n   * Convenience method to get this element as a {@link Number}. If this primitive {@linkplain\n   * #isString() is a string}, a lazily parsed {@code Number} is constructed which parses the string\n   * when any of its methods are called (which can lead to a {@link NumberFormatException}).\n   *\n   * @throws UnsupportedOperationException if this primitive is neither a number nor a string.\n   */\n  @Override\n  public Number getAsNumber() {\n    if (value instanceof Number) {\n      return (Number) value;\n    } else if (value instanceof String) {\n      return new LazilyParsedNumber((String) value);\n    }\n    throw new UnsupportedOperationException(\"Primitive is neither a number nor a string\");\n  }\n\n  /**\n   * Check whether this primitive contains a String value.\n   *\n   * @return true if this primitive contains a String value, false otherwise.\n   */\n  public boolean isString() {\n    return value instanceof String;\n  }\n\n  // Don't add Javadoc, inherit it from super implementation; no exceptions are thrown here\n  @Override\n  public String getAsString() {\n    if (value instanceof String) {\n      return (String) value;\n    } else if (isNumber()) {\n      return getAsNumber().toString();\n    } else if (isBoolean()) {\n      return ((Boolean) value).toString();\n    }\n    throw new AssertionError(\"Unexpected value type: \" + value.getClass());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public double getAsDouble() {\n    return isNumber() ? getAsNumber().doubleValue() : Double.parseDouble(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public BigDecimal getAsBigDecimal() {\n    return value instanceof BigDecimal\n        ? (BigDecimal) value\n        : NumberLimits.parseBigDecimal(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public BigInteger getAsBigInteger() {\n    return value instanceof BigInteger\n        ? (BigInteger) value\n        : isIntegral(this)\n            ? BigInteger.valueOf(this.getAsNumber().longValue())\n            : NumberLimits.parseBigInteger(this.getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public float getAsFloat() {\n    return isNumber() ? getAsNumber().floatValue() : Float.parseFloat(getAsString());\n  }\n\n  /**\n   * Convenience method to get this element as a primitive long.\n   *\n   * @return this element as a primitive long.\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public long getAsLong() {\n    return isNumber() ? getAsNumber().longValue() : Long.parseLong(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public short getAsShort() {\n    return isNumber() ? getAsNumber().shortValue() : Short.parseShort(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public int getAsInt() {\n    return isNumber() ? getAsNumber().intValue() : Integer.parseInt(getAsString());\n  }\n\n  /**\n   * @throws NumberFormatException {@inheritDoc}\n   */\n  @Override\n  public byte getAsByte() {\n    return isNumber() ? getAsNumber().byteValue() : Byte.parseByte(getAsString());\n  }\n\n  /**\n   * @throws UnsupportedOperationException if the string value of this primitive is empty.\n   * @deprecated This method is misleading, as it does not get this element as a char but rather as\n   *     a string's first character.\n   */\n  @Deprecated\n  @Override\n  public char getAsCharacter() {\n    String s = getAsString();\n    if (s.isEmpty()) {\n      throw new UnsupportedOperationException(\"String value is empty\");\n    } else {\n      return s.charAt(0);\n    }\n  }\n\n  /** Returns the hash code of this object. */\n  @Override\n  public int hashCode() {\n    if (value == null) {\n      return 31;\n    }\n    // Using recommended hashing algorithm from Effective Java for longs and doubles\n    if (isIntegral(this)) {\n      long value = getAsNumber().longValue();\n      return (int) (value ^ (value >>> 32));\n    }\n    if (value instanceof Number) {\n      long value = Double.doubleToLongBits(getAsNumber().doubleValue());\n      return (int) (value ^ (value >>> 32));\n    }\n    return value.hashCode();\n  }\n\n  /**\n   * Returns whether the other object is equal to this. This method only considers the other object\n   * to be equal if it is an instance of {@code JsonPrimitive} and has an equal value.\n   */\n  @Override\n  public boolean equals(Object obj) {\n    if (this == obj) {\n      return true;\n    }\n    if (obj == null || getClass() != obj.getClass()) {\n      return false;\n    }\n    JsonPrimitive other = (JsonPrimitive) obj;\n    if (value == null) {\n      return other.value == null;\n    }\n    if (isIntegral(this) && isIntegral(other)) {\n      return (this.value instanceof BigInteger || other.value instanceof BigInteger)\n          ? this.getAsBigInteger().equals(other.getAsBigInteger())\n          : this.getAsNumber().longValue() == other.getAsNumber().longValue();\n    }\n    if (value instanceof Number && other.value instanceof Number) {\n      if (value instanceof BigDecimal && other.value instanceof BigDecimal) {\n        // Uses compareTo to ignore scale of values, e.g. `0` and `0.00` should be considered equal\n        return this.getAsBigDecimal().compareTo(other.getAsBigDecimal()) == 0;\n      }\n\n      double thisAsDouble = this.getAsDouble();\n      double otherAsDouble = other.getAsDouble();\n      // Don't use Double.compare(double, double) because that considers -0.0 and +0.0 not equal\n      return (thisAsDouble == otherAsDouble)\n          || (Double.isNaN(thisAsDouble) && Double.isNaN(otherAsDouble));\n    }\n    return value.equals(other.value);\n  }\n\n  /**\n   * Returns true if the specified number is an integral type (Long, Integer, Short, Byte,\n   * BigInteger)\n   */\n  private static boolean isIntegral(JsonPrimitive primitive) {\n    if (primitive.value instanceof Number) {\n      Number number = (Number) primitive.value;\n      return number instanceof BigInteger\n          || number instanceof Long\n          || number instanceof Integer\n          || number instanceof Short\n          || number instanceof Byte;\n    }\n    return false;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/$Gson$Types.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport static com.google.gson.internal.$Gson$Preconditions.checkArgument;\nimport static java.util.Objects.requireNonNull;\n\nimport java.io.Serializable;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Modifier;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport java.util.Properties;\n\n/**\n * Static methods for working with types.\n *\n * @author Bob Lee\n * @author Jesse Wilson\n */\npublic final class $Gson$Types {\n  static final Type[] EMPTY_TYPE_ARRAY = new Type[] {};\n\n  private $Gson$Types() {\n    throw new UnsupportedOperationException();\n  }\n\n  /**\n   * Returns a new parameterized type, applying {@code typeArguments} to {@code rawType} and\n   * enclosed by {@code ownerType}.\n   *\n   * @return a {@link java.io.Serializable serializable} parameterized type.\n   */\n  public static ParameterizedType newParameterizedTypeWithOwner(\n      Type ownerType, Type rawType, Type... typeArguments) {\n    return new ParameterizedTypeImpl(ownerType, rawType, typeArguments);\n  }\n\n  /**\n   * Returns an array type whose elements are all instances of {@code componentType}.\n   *\n   * @return a {@link java.io.Serializable serializable} generic array type.\n   */\n  public static GenericArrayType arrayOf(Type componentType) {\n    return new GenericArrayTypeImpl(componentType);\n  }\n\n  /**\n   * Returns a type that represents an unknown type that extends {@code bound}. For example, if\n   * {@code bound} is {@code CharSequence.class}, this returns {@code ? extends CharSequence}. If\n   * {@code bound} is {@code Object.class}, this returns {@code ?}, which is shorthand for {@code ?\n   * extends Object}.\n   */\n  public static WildcardType subtypeOf(Type bound) {\n    Type[] upperBounds;\n    if (bound instanceof WildcardType) {\n      upperBounds = ((WildcardType) bound).getUpperBounds();\n    } else {\n      upperBounds = new Type[] {bound};\n    }\n    return new WildcardTypeImpl(upperBounds, EMPTY_TYPE_ARRAY);\n  }\n\n  /**\n   * Returns a type that represents an unknown supertype of {@code bound}. For example, if {@code\n   * bound} is {@code String.class}, this returns {@code ? super String}.\n   */\n  public static WildcardType supertypeOf(Type bound) {\n    Type[] lowerBounds;\n    if (bound instanceof WildcardType) {\n      lowerBounds = ((WildcardType) bound).getLowerBounds();\n    } else {\n      lowerBounds = new Type[] {bound};\n    }\n    return new WildcardTypeImpl(new Type[] {Object.class}, lowerBounds);\n  }\n\n  /**\n   * Returns a type that is functionally equal but not necessarily equal according to {@link\n   * Object#equals(Object) Object.equals()}. The returned type is {@link java.io.Serializable}.\n   */\n  public static Type canonicalize(Type type) {\n    if (type instanceof Class) {\n      Class<?> c = (Class<?>) type;\n      return c.isArray() ? new GenericArrayTypeImpl(canonicalize(c.getComponentType())) : c;\n\n    } else if (type instanceof ParameterizedType) {\n      ParameterizedType p = (ParameterizedType) type;\n      return new ParameterizedTypeImpl(\n          p.getOwnerType(), p.getRawType(), p.getActualTypeArguments());\n\n    } else if (type instanceof GenericArrayType) {\n      GenericArrayType g = (GenericArrayType) type;\n      return new GenericArrayTypeImpl(g.getGenericComponentType());\n\n    } else if (type instanceof WildcardType) {\n      WildcardType w = (WildcardType) type;\n      return new WildcardTypeImpl(w.getUpperBounds(), w.getLowerBounds());\n\n    } else {\n      // type is either serializable as-is or unsupported\n      return type;\n    }\n  }\n\n  public static Class<?> getRawType(Type type) {\n    if (type instanceof Class<?>) {\n      // type is a normal class.\n      return (Class<?>) type;\n\n    } else if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n\n      // getRawType() returns Type instead of Class; that seems to be an API mistake,\n      // see https://bugs.openjdk.org/browse/JDK-8250659\n      Type rawType = parameterizedType.getRawType();\n      checkArgument(rawType instanceof Class);\n      return (Class<?>) rawType;\n\n    } else if (type instanceof GenericArrayType) {\n      Type componentType = ((GenericArrayType) type).getGenericComponentType();\n      return Array.newInstance(getRawType(componentType), 0).getClass();\n\n    } else if (type instanceof TypeVariable) {\n      // we could use the variable's bounds, but that won't work if there are multiple.\n      // having a raw type that's more general than necessary is okay\n      return Object.class;\n\n    } else if (type instanceof WildcardType) {\n      Type[] bounds = ((WildcardType) type).getUpperBounds();\n      // Currently the JLS only permits one bound for wildcards so using first bound is safe\n      assert bounds.length == 1;\n      return getRawType(bounds[0]);\n\n    } else {\n      String className = type == null ? \"null\" : type.getClass().getName();\n      throw new IllegalArgumentException(\n          \"Expected a Class, ParameterizedType, or GenericArrayType, but <\"\n              + type\n              + \"> is of type \"\n              + className);\n    }\n  }\n\n  private static boolean equal(Object a, Object b) {\n    return Objects.equals(a, b);\n  }\n\n  /** Returns true if {@code a} and {@code b} are equal. */\n  public static boolean equals(Type a, Type b) {\n    if (a == b) {\n      // also handles (a == null && b == null)\n      return true;\n\n    } else if (a instanceof Class) {\n      // Class already specifies equals().\n      return a.equals(b);\n\n    } else if (a instanceof ParameterizedType) {\n      if (!(b instanceof ParameterizedType)) {\n        return false;\n      }\n\n      // TODO: save a .clone() call\n      ParameterizedType pa = (ParameterizedType) a;\n      ParameterizedType pb = (ParameterizedType) b;\n      return equal(pa.getOwnerType(), pb.getOwnerType())\n          && pa.getRawType().equals(pb.getRawType())\n          && Arrays.equals(pa.getActualTypeArguments(), pb.getActualTypeArguments());\n\n    } else if (a instanceof GenericArrayType) {\n      if (!(b instanceof GenericArrayType)) {\n        return false;\n      }\n\n      GenericArrayType ga = (GenericArrayType) a;\n      GenericArrayType gb = (GenericArrayType) b;\n      return equals(ga.getGenericComponentType(), gb.getGenericComponentType());\n\n    } else if (a instanceof WildcardType) {\n      if (!(b instanceof WildcardType)) {\n        return false;\n      }\n\n      WildcardType wa = (WildcardType) a;\n      WildcardType wb = (WildcardType) b;\n      return Arrays.equals(wa.getUpperBounds(), wb.getUpperBounds())\n          && Arrays.equals(wa.getLowerBounds(), wb.getLowerBounds());\n\n    } else if (a instanceof TypeVariable) {\n      if (!(b instanceof TypeVariable)) {\n        return false;\n      }\n      TypeVariable<?> va = (TypeVariable<?>) a;\n      TypeVariable<?> vb = (TypeVariable<?>) b;\n      return Objects.equals(va.getGenericDeclaration(), vb.getGenericDeclaration())\n          && va.getName().equals(vb.getName());\n\n    } else {\n      // This isn't a type we support. Could be a generic array type, wildcard type, etc.\n      return false;\n    }\n  }\n\n  public static String typeToString(Type type) {\n    return type instanceof Class ? ((Class<?>) type).getName() : type.toString();\n  }\n\n  /**\n   * Returns the generic supertype for {@code supertype}. For example, given a class {@code\n   * IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>} and the\n   * result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.\n   */\n  private static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> supertype) {\n    if (supertype == rawType) {\n      return context;\n    }\n\n    // we skip searching through interfaces if unknown is an interface\n    if (supertype.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == supertype) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (supertype.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], supertype);\n        }\n      }\n    }\n\n    // check our supertypes\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == supertype) {\n          return rawType.getGenericSuperclass();\n        } else if (supertype.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, supertype);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // we can't resolve this further\n    return supertype;\n  }\n\n  /**\n   * Returns the generic form of {@code supertype}. For example, if this is {@code\n   * ArrayList<String>}, this returns {@code Iterable<String>} given the input {@code\n   * Iterable.class}.\n   *\n   * @param supertype a superclass of, or interface implemented by, this.\n   */\n  private static Type getSupertype(Type context, Class<?> contextRawType, Class<?> supertype) {\n    if (context instanceof WildcardType) {\n      // Wildcards are useless for resolving supertypes. As the upper bound has the same raw type,\n      // use it instead\n      Type[] bounds = ((WildcardType) context).getUpperBounds();\n      // Currently the JLS only permits one bound for wildcards so using first bound is safe\n      assert bounds.length == 1;\n      context = bounds[0];\n    }\n    checkArgument(supertype.isAssignableFrom(contextRawType));\n    return resolve(\n        context,\n        contextRawType,\n        $Gson$Types.getGenericSupertype(context, contextRawType, supertype));\n  }\n\n  /**\n   * Returns the component type of this array type.\n   *\n   * @throws ClassCastException if this type is not an array.\n   */\n  \npublic static Type getArrayComponentType(Type array) {\n    return array instanceof GenericArrayType\n        ? ((GenericArrayType) array).getGenericComponentType()\n        : ((Class<?>) array).getComponentType();\n  }\n\n  /**\n   * Returns the element type of this collection type.\n   *\n   * @throws IllegalArgumentException if this type is not a collection.\n   */\n  public static Type getCollectionElementType(Type context, Class<?> contextRawType) {\n    Type collectionType = getSupertype(context, contextRawType, Collection.class);\n\n    if (collectionType instanceof ParameterizedType) {\n      return ((ParameterizedType) collectionType).getActualTypeArguments()[0];\n    }\n    return Object.class;\n  }\n\n  /**\n   * Returns a two element array containing this map's key and value types in positions 0 and 1\n   * respectively.\n   */\n  public static Type[] getMapKeyAndValueTypes(Type context, Class<?> contextRawType) {\n    /*\n     * Work around a problem with the declaration of java.util.Properties. That\n     * class should extend Hashtable<String, String>, but it's declared to\n     * extend Hashtable<Object, Object>.\n     */\n    if (context == Properties.class) {\n      return new Type[] {String.class, String.class}; // TODO: test subclasses of Properties!\n    }\n\n    Type mapType = getSupertype(context, contextRawType, Map.class);\n    // TODO: strip wildcards?\n    if (mapType instanceof ParameterizedType) {\n      ParameterizedType mapParameterizedType = (ParameterizedType) mapType;\n      return mapParameterizedType.getActualTypeArguments();\n    }\n    return new Type[] {Object.class, Object.class};\n  }\n\n  public static Type resolve(Type context, Class<?> contextRawType, Type toResolve) {\n\n    return resolve(context, contextRawType, toResolve, new HashMap<TypeVariable<?>, Type>());\n  }\n\n  private static Type resolve(\n      Type context,\n      Class<?> contextRawType,\n      Type toResolve,\n      Map<TypeVariable<?>, Type> visitedTypeVariables) {\n    // this implementation is made a little more complicated in an attempt to avoid object-creation\n    TypeVariable<?> resolving = null;\n    while (true) {\n      if (toResolve instanceof TypeVariable) {\n        TypeVariable<?> typeVariable = (TypeVariable<?>) toResolve;\n        Type previouslyResolved = visitedTypeVariables.get(typeVariable);\n        if (previouslyResolved != null) {\n          // cannot reduce due to infinite recursion\n          return (previouslyResolved == Void.TYPE) ? toResolve : previouslyResolved;\n        }\n\n        // Insert a placeholder to mark the fact that we are in the process of resolving this type\n        visitedTypeVariables.put(typeVariable, Void.TYPE);\n        if (resolving == null) {\n          resolving = typeVariable;\n        }\n\n        toResolve = resolveTypeVariable(context, contextRawType, typeVariable);\n        if (toResolve == typeVariable) {\n          break;\n        }\n\n      } else if (toResolve instanceof Class && ((Class<?>) toResolve).isArray()) {\n        Class<?> original = (Class<?>) toResolve;\n        Type componentType = original.getComponentType();\n        Type newComponentType =\n            resolve(context, contextRawType, componentType, visitedTypeVariables);\n        toResolve = equal(componentType, newComponentType) ? original : arrayOf(newComponentType);\n        break;\n\n      } else if (toResolve instanceof GenericArrayType) {\n        GenericArrayType original = (GenericArrayType) toResolve;\n        Type componentType = original.getGenericComponentType();\n        Type newComponentType =\n            resolve(context, contextRawType, componentType, visitedTypeVariables);\n        toResolve = equal(componentType, newComponentType) ? original : arrayOf(newComponentType);\n        break;\n\n      } else if (toResolve instanceof ParameterizedType) {\n        ParameterizedType original = (ParameterizedType) toResolve;\n        Type ownerType = original.getOwnerType();\n        Type newOwnerType = resolve(context, contextRawType, ownerType, visitedTypeVariables);\n        boolean changed = !equal(newOwnerType, ownerType);\n\n        Type[] args = original.getActualTypeArguments();\n        for (int t = 0, length = args.length; t < length; t++) {\n          Type resolvedTypeArgument =\n              resolve(context, contextRawType, args[t], visitedTypeVariables);\n          if (!equal(resolvedTypeArgument, args[t])) {\n            if (!changed) {\n              args = args.clone();\n              changed = true;\n            }\n            args[t] = resolvedTypeArgument;\n          }\n        }\n\n        toResolve =\n            changed\n                ? newParameterizedTypeWithOwner(newOwnerType, original.getRawType(), args)\n                : original;\n        break;\n\n      } else if (toResolve instanceof WildcardType) {\n        WildcardType original = (WildcardType) toResolve;\n        Type[] originalLowerBound = original.getLowerBounds();\n        Type[] originalUpperBound = original.getUpperBounds();\n\n        if (originalLowerBound.length == 1) {\n          Type lowerBound =\n              resolve(context, contextRawType, originalLowerBound[0], visitedTypeVariables);\n          if (lowerBound != originalLowerBound[0]) {\n            toResolve = supertypeOf(lowerBound);\n            break;\n          }\n        } else if (originalUpperBound.length == 1) {\n          Type upperBound =\n              resolve(context, contextRawType, originalUpperBound[0], visitedTypeVariables);\n          if (upperBound != originalUpperBound[0]) {\n            toResolve = subtypeOf(upperBound);\n            break;\n          }\n        }\n        toResolve = original;\n        break;\n\n      } else {\n        break;\n      }\n    }\n    // ensure that any in-process resolution gets updated with the final result\n    if (resolving != null) {\n      visitedTypeVariables.put(resolving, toResolve);\n    }\n    return toResolve;\n  }\n\n  private static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // we can't reduce this further\n    if (declaredByRaw == null) {\n      return unknown;\n    }\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  private static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0, length = array.length; i < length; i++) {\n      if (toFind.equals(array[i])) {\n        return i;\n      }\n    }\n    throw new NoSuchElementException();\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n\n  static void checkNotPrimitive(Type type) {\n    checkArgument(!(type instanceof Class<?>) || !((Class<?>) type).isPrimitive());\n  }\n\n  /**\n   * Whether an {@linkplain ParameterizedType#getOwnerType() owner type} must be specified when\n   * constructing a {@link ParameterizedType} for {@code rawType}.\n   *\n   * <p>Note that this method might not require an owner type for all cases where Java reflection\n   * would create parameterized types with owner type.\n   */\n  public static boolean requiresOwnerType(Type rawType) {\n    if (rawType instanceof Class<?>) {\n      Class<?> rawTypeAsClass = (Class<?>) rawType;\n      return !Modifier.isStatic(rawTypeAsClass.getModifiers())\n          && rawTypeAsClass.getDeclaringClass() != null;\n    }\n    return false;\n  }\n\n  // Here and below we put @SuppressWarnings(\"serial\") on fields of type `Type`. Recent Java\n  // compilers complain that the declared type is not Serializable. But in this context we go out of\n  // our way to ensure that the Type in question is either Class (which is serializable) or one of\n  // the nested Type implementations here (which are also serializable).\n  private static final class ParameterizedTypeImpl implements ParameterizedType, Serializable {\n    @SuppressWarnings(\"serial\")\n    private final Type ownerType;\n\n    @SuppressWarnings(\"serial\")\n    private final Type rawType;\n\n    @SuppressWarnings(\"serial\")\n    private final Type[] typeArguments;\n\n    public ParameterizedTypeImpl(Type ownerType, Type rawType, Type... typeArguments) {\n      // TODO: Should this enforce that rawType is a Class? See JDK implementation of\n      // the ParameterizedType interface and https://bugs.openjdk.org/browse/JDK-8250659\n      requireNonNull(rawType);\n      if (ownerType == null && requiresOwnerType(rawType)) {\n        throw new IllegalArgumentException(\"Must specify owner type for \" + rawType);\n      }\n\n      this.ownerType = ownerType == null ? null : canonicalize(ownerType);\n      this.rawType = canonicalize(rawType);\n      this.typeArguments = typeArguments.clone();\n      for (int t = 0, length = this.typeArguments.length; t < length; t++) {\n        requireNonNull(this.typeArguments[t]);\n        checkNotPrimitive(this.typeArguments[t]);\n        this.typeArguments[t] = canonicalize(this.typeArguments[t]);\n      }\n    }\n\n    @Override\n    public Type[] getActualTypeArguments() {\n      return typeArguments.clone();\n    }\n\n    @Override\n    public Type getRawType() {\n      return rawType;\n    }\n\n    @Override\n    public Type getOwnerType() {\n      return ownerType;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof ParameterizedType\n          && $Gson$Types.equals(this, (ParameterizedType) other);\n    }\n\n    private static int hashCodeOrZero(Object o) {\n      return o != null ? o.hashCode() : 0;\n    }\n\n    @Override\n    public int hashCode() {\n      return Arrays.hashCode(typeArguments) ^ rawType.hashCode() ^ hashCodeOrZero(ownerType);\n    }\n\n    @Override\n    public String toString() {\n      int length = typeArguments.length;\n      if (length == 0) {\n        return typeToString(rawType);\n      }\n\n      StringBuilder stringBuilder = new StringBuilder(30 * (length + 1));\n      stringBuilder\n          .append(typeToString(rawType))\n          .append(\"<\")\n          .append(typeToString(typeArguments[0]));\n      for (int i = 1; i < length; i++) {\n        stringBuilder.append(\", \").append(typeToString(typeArguments[i]));\n      }\n      return stringBuilder.append(\">\").toString();\n    }\n\n    private static final long serialVersionUID = 0;\n  }\n\n  private static final class GenericArrayTypeImpl implements GenericArrayType, Serializable {\n    @SuppressWarnings(\"serial\")\n    private final Type componentType;\n\n    public GenericArrayTypeImpl(Type componentType) {\n      requireNonNull(componentType);\n      this.componentType = canonicalize(componentType);\n    }\n\n    @Override\n    public Type getGenericComponentType() {\n      return componentType;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      return o instanceof GenericArrayType && $Gson$Types.equals(this, (GenericArrayType) o);\n    }\n\n    @Override\n    public int hashCode() {\n      return componentType.hashCode();\n    }\n\n    @Override\n    public String toString() {\n      return typeToString(componentType) + \"[]\";\n    }\n\n    private static final long serialVersionUID = 0;\n  }\n\n  /**\n   * The WildcardType interface supports multiple upper bounds and multiple lower bounds. We only\n   * support what the target Java version supports - at most one bound, see also\n   * https://bugs.openjdk.java.net/browse/JDK-8250660. If a lower bound is set, the upper bound must\n   * be Object.class.\n   */\n  private static final class WildcardTypeImpl implements WildcardType, Serializable {\n    @SuppressWarnings(\"serial\")\n    private final Type upperBound;\n\n    @SuppressWarnings(\"serial\")\n    private final Type lowerBound;\n\n    public WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) {\n      checkArgument(lowerBounds.length <= 1);\n      checkArgument(upperBounds.length == 1);\n\n      if (lowerBounds.length == 1) {\n        requireNonNull(lowerBounds[0]);\n        checkNotPrimitive(lowerBounds[0]);\n        checkArgument(upperBounds[0] == Object.class);\n        this.lowerBound = canonicalize(lowerBounds[0]);\n        this.upperBound = Object.class;\n\n      } else {\n        requireNonNull(upperBounds[0]);\n        checkNotPrimitive(upperBounds[0]);\n        this.lowerBound = null;\n        this.upperBound = canonicalize(upperBounds[0]);\n      }\n    }\n\n    @Override\n    public Type[] getUpperBounds() {\n      return new Type[] {upperBound};\n    }\n\n    @Override\n    public Type[] getLowerBounds() {\n      return lowerBound != null ? new Type[] {lowerBound} : EMPTY_TYPE_ARRAY;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof WildcardType && $Gson$Types.equals(this, (WildcardType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      // this equals Arrays.hashCode(getLowerBounds()) ^ Arrays.hashCode(getUpperBounds());\n      return (lowerBound != null ? 31 + lowerBound.hashCode() : 1) ^ (31 + upperBound.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      if (lowerBound != null) {\n        return \"? super \" + typeToString(lowerBound);\n      } else if (upperBound == Object.class) {\n        return \"?\";\n      } else {\n        return \"? extends \" + typeToString(upperBound);\n      }\n    }\n\n    private static final long serialVersionUID = 0;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/TroubleshootingGuide.java\npackage com.google.gson.internal;\n\npublic class TroubleshootingGuide {\n  private TroubleshootingGuide() {}\n\n  /** Creates a URL referring to the specified troubleshooting section. */\n  public static String createUrl(String id) {\n    return \"https://github.com/google/gson/blob/main/Troubleshooting.md#\" + id;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/reflect/TypeToken.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.reflect;\n\nimport com.google.gson.internal.$Gson$Types;\nimport com.google.gson.internal.TroubleshootingGuide;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Objects;\n\n/**\n * Represents a generic type {@code T}. Java doesn't yet provide a way to represent generic types,\n * so this class does. Forces clients to create a subclass of this class which enables retrieval the\n * type information even at runtime.\n *\n * <p>For example, to create a type literal for {@code List<String>}, you can create an empty\n * anonymous class:\n *\n * <p>{@code TypeToken<List<String>> list = new TypeToken<List<String>>() {};}\n *\n * <p>Capturing a type variable as type argument of an anonymous {@code TypeToken} subclass is not\n * allowed, for example {@code TypeToken<List<T>>}. Due to type erasure the runtime type of a type\n * variable is not available to Gson and therefore it cannot provide the functionality one might\n * expect. This would give a false sense of type-safety at compile time and could lead to an\n * unexpected {@code ClassCastException} at runtime.\n *\n * <p>If the type arguments of the parameterized type are only available at runtime, for example\n * when you want to create a {@code List<E>} based on a {@code Class<E>} representing the element\n * type, the method {@link #getParameterized(Type, Type...)} can be used.\n *\n * @author Bob Lee\n * @author Sven Mawson\n * @author Jesse Wilson\n */\npublic class TypeToken<T> {\n  private final Class<? super T> rawType;\n  private final Type type;\n  private final int hashCode;\n\n  /**\n   * Constructs a new type literal. Derives represented class from type parameter.\n   *\n   * <p>Clients create an empty anonymous subclass. Doing so embeds the type parameter in the\n   * anonymous class's type hierarchy so we can reconstitute it at runtime despite erasure, for\n   * example:\n   *\n   * <p>{@code new TypeToken<List<String>>() {}}\n   *\n   * @throws IllegalArgumentException If the anonymous {@code TypeToken} subclass captures a type\n   *     variable, for example {@code TypeToken<List<T>>}. See the {@code TypeToken} class\n   *     documentation for more details.\n   */\n  @SuppressWarnings(\"unchecked\")\n  protected TypeToken() {\n    this.type = getTypeTokenTypeArgument();\n    this.rawType = (Class<? super T>) $Gson$Types.getRawType(type);\n    this.hashCode = type.hashCode();\n  }\n\n  /** Unsafe. Constructs a type literal manually. */\n  @SuppressWarnings(\"unchecked\")\n  private TypeToken(Type type) {\n    this.type = $Gson$Types.canonicalize(Objects.requireNonNull(type));\n    this.rawType = (Class<? super T>) $Gson$Types.getRawType(this.type);\n    this.hashCode = this.type.hashCode();\n  }\n\n  private static boolean isCapturingTypeVariablesForbidden() {\n    return !Objects.equals(System.getProperty(\"gson.allowCapturingTypeVariables\"), \"true\");\n  }\n\n  /**\n   * Verifies that {@code this} is an instance of a direct subclass of TypeToken and returns the\n   * type argument for {@code T} in {@link $Gson$Types#canonicalize canonical form}.\n   */\n  private Type getTypeTokenTypeArgument() {\n    Type superclass = getClass().getGenericSuperclass();\n    if (superclass instanceof ParameterizedType) {\n      ParameterizedType parameterized = (ParameterizedType) superclass;\n      if (parameterized.getRawType() == TypeToken.class) {\n        Type typeArgument = $Gson$Types.canonicalize(parameterized.getActualTypeArguments()[0]);\n\n        if (isCapturingTypeVariablesForbidden()) {\n          verifyNoTypeVariable(typeArgument);\n        }\n        return typeArgument;\n      }\n    }\n    // Check for raw TypeToken as superclass\n    else if (superclass == TypeToken.class) {\n      throw new IllegalStateException(\n          \"TypeToken must be created with a type argument: new TypeToken<...>() {}; When using code\"\n              + \" shrinkers (ProGuard, R8, ...) make sure that generic signatures are preserved.\"\n              + \"\\nSee \"\n              + TroubleshootingGuide.createUrl(\"type-token-raw\"));\n    }\n\n    // User created subclass of subclass of TypeToken\n    throw new IllegalStateException(\"Must only create direct subclasses of TypeToken\");\n  }\n\n  private static void verifyNoTypeVariable(Type type) {\n    if (type instanceof TypeVariable) {\n      TypeVariable<?> typeVariable = (TypeVariable<?>) type;\n      throw new IllegalArgumentException(\n          \"TypeToken type argument must not contain a type variable; captured type variable \"\n              + typeVariable.getName()\n              + \" declared by \"\n              + typeVariable.getGenericDeclaration()\n              + \"\\nSee \"\n              + TroubleshootingGuide.createUrl(\"typetoken-type-variable\"));\n    } else if (type instanceof GenericArrayType) {\n      verifyNoTypeVariable(((GenericArrayType) type).getGenericComponentType());\n    } else if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n      Type ownerType = parameterizedType.getOwnerType();\n      if (ownerType != null) {\n        verifyNoTypeVariable(ownerType);\n      }\n\n      for (Type typeArgument : parameterizedType.getActualTypeArguments()) {\n        verifyNoTypeVariable(typeArgument);\n      }\n    } else if (type instanceof WildcardType) {\n      WildcardType wildcardType = (WildcardType) type;\n      for (Type bound : wildcardType.getLowerBounds()) {\n        verifyNoTypeVariable(bound);\n      }\n      for (Type bound : wildcardType.getUpperBounds()) {\n        verifyNoTypeVariable(bound);\n      }\n    } else if (type == null) {\n      // Occurs in Eclipse IDE and certain Java versions (e.g. Java 11.0.18) when capturing type\n      // variable declared by method of local class, see\n...\n", "needle_token_start": 10412, "needle_token_end": 10464, "code_context_ntokens": 16061, "output": ["```java\n/**\n * Returns the component type of this array type.\n *\n * @throws ClassCastException if this type is not an array.\n */\npublic static Type getArrayComponentType(Type array) {\n  return array instanceof GenericArrayType\n      ? ((GenericArrayType) array).getGenericComponentType()\n      : ((Class<?>) array).getComponentType();\n}\n```"]}
{"repo": "google/gson", "name": "setExclusionStrategies", "language": "java", "path": "gson/src/main/java/com/google/gson/GsonBuilder.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To configure a JSON serialization and deserialization library to selectively ignore certain fields or classes based on specified rules, enhancing control over the output and input process.\n2. **Input**: An array of strategy objects that define conditions under which fields or classes should be skipped during serialization and deserialization.\n3. **Output**: A reference to the builder object itself, allowing for method chaining in accordance with the builder pattern.\n4. **Procedure**: The method iterates over each provided strategy object, applying these to the internal configuration. Each strategy is checked to determine if a field or class should be excluded during the serialization or deserialization process. If any strategy suggests exclusion, the respective field or class is skipped.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/com/google/gson/JsonDeserializer.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport java.lang.reflect.Type;\n\n/**\n * Interface representing a custom deserializer for JSON. You should write a custom deserializer, if\n * you are not happy with the default deserialization done by Gson. You will also need to register\n * this deserializer through {@link GsonBuilder#registerTypeAdapter(Type, Object)}.\n *\n * <p>Let us look at example where defining a deserializer will be useful. The {@code Id} class\n * defined below has two fields: {@code clazz} and {@code value}.\n *\n * <pre>\n * public class Id&lt;T&gt; {\n *   private final Class&lt;T&gt; clazz;\n *   private final long value;\n *   public Id(Class&lt;T&gt; clazz, long value) {\n *     this.clazz = clazz;\n *     this.value = value;\n *   }\n *   public long getValue() {\n *     return value;\n *   }\n * }\n * </pre>\n *\n * <p>The default deserialization of {@code Id(com.foo.MyObject.class, 20L)} will require the JSON\n * string to be <code>{\"clazz\":\"com.foo.MyObject\",\"value\":20}</code>. Suppose, you already know the\n * type of the field that the {@code Id} will be deserialized into, and hence just want to\n * deserialize it from a JSON string {@code 20}. You can achieve that by writing a custom\n * deserializer:\n *\n * <pre>\n * class IdDeserializer implements JsonDeserializer&lt;Id&gt; {\n *   public Id deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)\n *       throws JsonParseException {\n *     long idValue = json.getAsJsonPrimitive().getAsLong();\n *     return new Id((Class) typeOfT, idValue);\n *   }\n * }\n * </pre>\n *\n...\n// Path: gson/src/main/java/com/google/gson/internal/bind/TreeTypeAdapter.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.bind;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonDeserializationContext;\nimport com.google.gson.JsonDeserializer;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonParseException;\nimport com.google.gson.JsonSerializationContext;\nimport com.google.gson.JsonSerializer;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.TypeAdapterFactory;\nimport com.google.gson.internal.$Gson$Preconditions;\nimport com.google.gson.internal.Streams;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.lang.reflect.Type;\n\n/**\n * Adapts a Gson 1.x tree-style adapter as a streaming TypeAdapter. Since the tree adapter may be\n * serialization-only or deserialization-only, this class has a facility to look up a delegate type\n * adapter on demand.\n */\npublic final class TreeTypeAdapter<T> extends SerializationDelegatingTypeAdapter<T> {\n  private final JsonSerializer<T> serializer;\n  private final JsonDeserializer<T> deserializer;\n  final Gson gson;\n  private final TypeToken<T> typeToken;\n\n  /**\n   * Only intended as {@code skipPast} for {@link Gson#getDelegateAdapter(TypeAdapterFactory,\n   * TypeToken)}, must not be used in any other way.\n   */\n  private final TypeAdapterFactory skipPastForGetDelegateAdapter;\n\n  private final GsonContextImpl context = new GsonContextImpl();\n  private final boolean nullSafe;\n\n  /**\n   * The delegate is lazily created because it may not be needed, and creating it may fail. Field\n   * has to be {@code volatile} because {@link Gson} guarantees to be thread-safe.\n   */\n  private volatile TypeAdapter<T> delegate;\n\n  public TreeTypeAdapter(\n      JsonSerializer<T> serializer,\n      JsonDeserializer<T> deserializer,\n      Gson gson,\n      TypeToken<T> typeToken,\n      TypeAdapterFactory skipPast,\n      boolean nullSafe) {\n    this.serializer = serializer;\n    this.deserializer = deserializer;\n    this.gson = gson;\n    this.typeToken = typeToken;\n    this.skipPastForGetDelegateAdapter = skipPast;\n    this.nullSafe = nullSafe;\n  }\n\n  public TreeTypeAdapter(\n      JsonSerializer<T> serializer,\n      JsonDeserializer<T> deserializer,\n      Gson gson,\n      TypeToken<T> typeToken,\n      TypeAdapterFactory skipPast) {\n    this(serializer, deserializer, gson, typeToken, skipPast, true);\n  }\n\n  @Override\n  public T read(JsonReader in) throws IOException {\n    if (deserializer == null) {\n      return delegate().read(in);\n    }\n    JsonElement value = Streams.parse(in);\n    if (nullSafe && value.isJsonNull()) {\n      return null;\n    }\n    return deserializer.deserialize(value, typeToken.getType(), context);\n  }\n\n  @Override\n  public void write(JsonWriter out, T value) throws IOException {\n    if (serializer == null) {\n      delegate().write(out, value);\n      return;\n    }\n    if (nullSafe && value == null) {\n      out.nullValue();\n      return;\n    }\n    JsonElement tree = serializer.serialize(value, typeToken.getType(), context);\n    Streams.write(tree, out);\n  }\n\n  private TypeAdapter<T> delegate() {\n    // A race might lead to `delegate` being assigned by multiple threads but the last assignment\n    // will stick\n    TypeAdapter<T> d = delegate;\n    return d != null\n        ? d\n        : (delegate = gson.getDelegateAdapter(skipPastForGetDelegateAdapter, typeToken));\n  }\n\n  /**\n   * Returns the type adapter which is used for serialization. Returns {@code this} if this {@code\n   * TreeTypeAdapter} has a {@link #serializer}; otherwise returns the delegate.\n   */\n  @Override\n  public TypeAdapter<T> getSerializationDelegate() {\n    return serializer != null ? this : delegate();\n  }\n\n  /** Returns a new factory that will match each type against {@code exactType}. */\n  public static TypeAdapterFactory newFactory(TypeToken<?> exactType, Object typeAdapter) {\n    return new SingleTypeFactory(typeAdapter, exactType, false, null);\n  }\n\n  /** Returns a new factory that will match each type and its raw type against {@code exactType}. */\n  public static TypeAdapterFactory newFactoryWithMatchRawType(\n      TypeToken<?> exactType, Object typeAdapter) {\n    // only bother matching raw types if exact type is a raw type\n    boolean matchRawType = exactType.getType() == exactType.getRawType();\n    return new SingleTypeFactory(typeAdapter, exactType, matchRawType, null);\n  }\n\n  /**\n   * Returns a new factory that will match each type's raw type for assignability to {@code\n   * hierarchyType}.\n   */\n  public static TypeAdapterFactory newTypeHierarchyFactory(\n      Class<?> hierarchyType, Object typeAdapter) {\n    return new SingleTypeFactory(typeAdapter, null, false, hierarchyType);\n  }\n\n  private static final class SingleTypeFactory implements TypeAdapterFactory {\n    private final TypeToken<?> exactType;\n    private final boolean matchRawType;\n    private final Class<?> hierarchyType;\n    private final JsonSerializer<?> serializer;\n    private final JsonDeserializer<?> deserializer;\n\n    SingleTypeFactory(\n        Object typeAdapter, TypeToken<?> exactType, boolean matchRawType, Class<?> hierarchyType) {\n      serializer = typeAdapter instanceof JsonSerializer ? (JsonSerializer<?>) typeAdapter : null;\n      deserializer =\n          typeAdapter instanceof JsonDeserializer ? (JsonDeserializer<?>) typeAdapter : null;\n      $Gson$Preconditions.checkArgument(serializer != null || deserializer != null);\n      this.exactType = exactType;\n      this.matchRawType = matchRawType;\n      this.hierarchyType = hierarchyType;\n    }\n\n    @SuppressWarnings(\"unchecked\") // guarded by typeToken.equals() call\n    @Override\n    public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {\n      boolean matches =\n          exactType != null\n              ? exactType.equals(type) || (matchRawType && exactType.getType() == type.getRawType())\n              : hierarchyType.isAssignableFrom(type.getRawType());\n      return matches\n          ? new TreeTypeAdapter<>(\n              (JsonSerializer<T>) serializer, (JsonDeserializer<T>) deserializer, gson, type, this)\n          : null;\n    }\n  }\n\n  private final class GsonContextImpl\n      implements JsonSerializationContext, JsonDeserializationContext {\n    @Override\n    public JsonElement serialize(Object src) {\n      return gson.toJsonTree(src);\n    }\n\n    @Override\n    public JsonElement serialize(Object src, Type typeOfSrc) {\n      return gson.toJsonTree(src, typeOfSrc);\n    }\n\n    @Override\n    @SuppressWarnings({\"unchecked\", \"TypeParameterUnusedInFormals\"})\n    public <R> R deserialize(JsonElement json, Type typeOfT) throws JsonParseException {\n      return gson.fromJson(json, typeOfT);\n    }\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/sql/SqlDateTypeAdapter.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.sql;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonSyntaxException;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.TypeAdapterFactory;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.text.DateFormat;\nimport java.text.ParseException;\nimport java.text.SimpleDateFormat;\nimport java.util.Date;\nimport java.util.TimeZone;\n\n/**\n * Adapter for java.sql.Date. Although this class appears stateless, it is not. DateFormat captures\n * its time zone and locale when it is created, which gives this class state. DateFormat isn't\n * thread safe either, so this class has to synchronize its read and write methods.\n */\n@SuppressWarnings(\"JavaUtilDate\")\nfinal class SqlDateTypeAdapter extends TypeAdapter<java.sql.Date> {\n  static final TypeAdapterFactory FACTORY =\n      new TypeAdapterFactory() {\n        @SuppressWarnings(\"unchecked\") // we use a runtime check to make sure the 'T's equal\n        @Override\n        public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n          return typeToken.getRawType() == java.sql.Date.class\n              ? (TypeAdapter<T>) new SqlDateTypeAdapter()\n              : null;\n        }\n      };\n\n  private final DateFormat format = new SimpleDateFormat(\"MMM d, yyyy\");\n\n  private SqlDateTypeAdapter() {}\n\n  @Override\n  public java.sql.Date read(JsonReader in) throws IOException {\n    if (in.peek() == JsonToken.NULL) {\n      in.nextNull();\n      return null;\n    }\n    String s = in.nextString();\n    synchronized (this) {\n      TimeZone originalTimeZone = format.getTimeZone(); // Save the original time zone\n      try {\n        Date utilDate = format.parse(s);\n        return new java.sql.Date(utilDate.getTime());\n      } catch (ParseException e) {\n        throw new JsonSyntaxException(\n            \"Failed parsing '\" + s + \"' as SQL Date; at path \" + in.getPreviousPath(), e);\n      } finally {\n        format.setTimeZone(originalTimeZone); // Restore the original time zone after parsing\n      }\n    }\n  }\n\n  @Override\n  public void write(JsonWriter out, java.sql.Date value) throws IOException {\n    if (value == null) {\n      out.nullValue();\n      return;\n    }\n    String dateString;\n    synchronized (this) {\n      dateString = format.format(value);\n    }\n    out.value(dateString);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/sql/SqlTimeTypeAdapter.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.sql;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonSyntaxException;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.TypeAdapterFactory;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.sql.Time;\nimport java.text.DateFormat;\nimport java.text.ParseException;\nimport java.text.SimpleDateFormat;\nimport java.util.Date;\nimport java.util.TimeZone;\n\n/**\n * Adapter for java.sql.Time. Although this class appears stateless, it is not. DateFormat captures\n * its time zone and locale when it is created, which gives this class state. DateFormat isn't\n * thread safe either, so this class has to synchronize its read and write methods.\n */\n@SuppressWarnings(\"JavaUtilDate\")\nfinal class SqlTimeTypeAdapter extends TypeAdapter<Time> {\n  static final TypeAdapterFactory FACTORY =\n      new TypeAdapterFactory() {\n        @SuppressWarnings(\"unchecked\") // we use a runtime check to make sure the 'T's equal\n        @Override\n        public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n          return typeToken.getRawType() == Time.class\n              ? (TypeAdapter<T>) new SqlTimeTypeAdapter()\n              : null;\n        }\n      };\n\n  private final DateFormat format = new SimpleDateFormat(\"hh:mm:ss a\");\n\n  private SqlTimeTypeAdapter() {}\n\n  @Override\n  public Time read(JsonReader in) throws IOException {\n    if (in.peek() == JsonToken.NULL) {\n      in.nextNull();\n      return null;\n    }\n    String s = in.nextString();\n    synchronized (this) {\n      TimeZone originalTimeZone = format.getTimeZone(); // Save the original time zone\n      try {\n        Date date = format.parse(s);\n        return new Time(date.getTime());\n      } catch (ParseException e) {\n        throw new JsonSyntaxException(\n            \"Failed parsing '\" + s + \"' as SQL Time; at path \" + in.getPreviousPath(), e);\n      } finally {\n        format.setTimeZone(originalTimeZone); // Restore the original time zone\n      }\n    }\n  }\n\n  @Override\n  public void write(JsonWriter out, Time value) throws IOException {\n    if (value == null) {\n      out.nullValue();\n      return;\n    }\n    String timeString;\n    synchronized (this) {\n      timeString = format.format(value);\n    }\n    out.value(timeString);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/sql/SqlTimestampTypeAdapter.java\n/*\n * Copyright (C) 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.sql;\n\nimport com.google.gson.Gson;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.TypeAdapterFactory;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.sql.Timestamp;\nimport java.util.Date;\n\n@SuppressWarnings(\"JavaUtilDate\")\nclass SqlTimestampTypeAdapter extends TypeAdapter<Timestamp> {\n  static final TypeAdapterFactory FACTORY =\n      new TypeAdapterFactory() {\n        @SuppressWarnings(\"unchecked\") // we use a runtime check to make sure the 'T's equal\n        @Override\n        public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n          if (typeToken.getRawType() == Timestamp.class) {\n            final TypeAdapter<Date> dateTypeAdapter = gson.getAdapter(Date.class);\n            return (TypeAdapter<T>) new SqlTimestampTypeAdapter(dateTypeAdapter);\n          } else {\n            return null;\n          }\n        }\n      };\n\n  private final TypeAdapter<Date> dateTypeAdapter;\n\n  private SqlTimestampTypeAdapter(TypeAdapter<Date> dateTypeAdapter) {\n    this.dateTypeAdapter = dateTypeAdapter;\n  }\n\n  @Override\n  public Timestamp read(JsonReader in) throws IOException {\n    Date date = dateTypeAdapter.read(in);\n    return date != null ? new Timestamp(date.getTime()) : null;\n  }\n\n  @Override\n  public void write(JsonWriter out, Timestamp value) throws IOException {\n    dateTypeAdapter.write(out, value);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/sql/SqlTypesSupport.java\n/*\n * Copyright (C) 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.sql;\n\nimport com.google.gson.TypeAdapterFactory;\nimport com.google.gson.internal.bind.DefaultDateTypeAdapter.DateType;\nimport java.sql.Timestamp;\nimport java.util.Date;\n\n/**\n * Encapsulates access to {@code java.sql} types, to allow Gson to work without the {@code java.sql}\n * module being present. No {@link ClassNotFoundException}s will be thrown in case the {@code\n * java.sql} module is not present.\n *\n * <p>If {@link #SUPPORTS_SQL_TYPES} is {@code true}, all other constants of this class will be\n * non-{@code null}. However, if it is {@code false} all other constants will be {@code null} and\n * there will be no support for {@code java.sql} types.\n */\n@SuppressWarnings(\"JavaUtilDate\")\npublic final class SqlTypesSupport {\n  /** {@code true} if {@code java.sql} types are supported, {@code false} otherwise */\n  public static final boolean SUPPORTS_SQL_TYPES;\n\n  public static final DateType<? extends Date> DATE_DATE_TYPE;\n  public static final DateType<? extends Date> TIMESTAMP_DATE_TYPE;\n\n  public static final TypeAdapterFactory DATE_FACTORY;\n  public static final TypeAdapterFactory TIME_FACTORY;\n  public static final TypeAdapterFactory TIMESTAMP_FACTORY;\n\n  static {\n    boolean sqlTypesSupport;\n    try {\n      Class.forName(\"java.sql.Date\");\n      sqlTypesSupport = true;\n    } catch (ClassNotFoundException classNotFoundException) {\n      sqlTypesSupport = false;\n    }\n    SUPPORTS_SQL_TYPES = sqlTypesSupport;\n\n    if (SUPPORTS_SQL_TYPES) {\n      DATE_DATE_TYPE =\n          new DateType<java.sql.Date>(java.sql.Date.class) {\n            @Override\n            protected java.sql.Date deserialize(Date date) {\n              return new java.sql.Date(date.getTime());\n            }\n          };\n      TIMESTAMP_DATE_TYPE =\n          new DateType<Timestamp>(Timestamp.class) {\n            @Override\n            protected Timestamp deserialize(Date date) {\n              return new Timestamp(date.getTime());\n            }\n          };\n\n      DATE_FACTORY = SqlDateTypeAdapter.FACTORY;\n      TIME_FACTORY = SqlTimeTypeAdapter.FACTORY;\n      TIMESTAMP_FACTORY = SqlTimestampTypeAdapter.FACTORY;\n    } else {\n      DATE_DATE_TYPE = null;\n      TIMESTAMP_DATE_TYPE = null;\n\n      DATE_FACTORY = null;\n      TIME_FACTORY = null;\n      TIMESTAMP_FACTORY = null;\n    }\n  }\n\n  private SqlTypesSupport() {}\n}\n\n// Path: gson/src/main/java/com/google/gson/GsonBuilder.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport static com.google.gson.Gson.DEFAULT_COMPLEX_MAP_KEYS;\nimport static com.google.gson.Gson.DEFAULT_DATE_PATTERN;\nimport static com.google.gson.Gson.DEFAULT_ESCAPE_HTML;\nimport static com.google.gson.Gson.DEFAULT_FORMATTING_STYLE;\nimport static com.google.gson.Gson.DEFAULT_JSON_NON_EXECUTABLE;\nimport static com.google.gson.Gson.DEFAULT_NUMBER_TO_NUMBER_STRATEGY;\nimport static com.google.gson.Gson.DEFAULT_OBJECT_TO_NUMBER_STRATEGY;\nimport static com.google.gson.Gson.DEFAULT_SERIALIZE_NULLS;\nimport static com.google.gson.Gson.DEFAULT_SPECIALIZE_FLOAT_VALUES;\nimport static com.google.gson.Gson.DEFAULT_STRICTNESS;\nimport static com.google.gson.Gson.DEFAULT_USE_JDK_UNSAFE;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.errorprone.annotations.InlineMe;\nimport com.google.gson.annotations.Since;\nimport com.google.gson.annotations.Until;\nimport com.google.gson.internal.$Gson$Preconditions;\nimport com.google.gson.internal.Excluder;\nimport com.google.gson.internal.bind.DefaultDateTypeAdapter;\nimport com.google.gson.internal.bind.TreeTypeAdapter;\nimport com.google.gson.internal.bind.TypeAdapters;\nimport com.google.gson.internal.sql.SqlTypesSupport;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonWriter;\nimport java.lang.reflect.Type;\nimport java.text.DateFormat;\nimport java.text.SimpleDateFormat;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\n\n/**\n * Use this builder to construct a {@link Gson} instance when you need to set configuration options\n * other than the default. For {@link Gson} with default configuration, it is simpler to use {@code\n * new Gson()}. {@code GsonBuilder} is best used by creating it, and then invoking its various\n * configuration methods, and finally calling create.\n *\n * <p>The following example shows how to use the {@code GsonBuilder} to construct a Gson instance:\n *\n * <pre>\n * Gson gson = new GsonBuilder()\n *     .registerTypeAdapter(Id.class, new IdTypeAdapter())\n *     .enableComplexMapKeySerialization()\n *     .serializeNulls()\n *     .setDateFormat(DateFormat.LONG, DateFormat.LONG)\n *     .setFieldNamingPolicy(FieldNamingPolicy.UPPER_CAMEL_CASE)\n *     .setPrettyPrinting()\n *     .setVersion(1.0)\n *     .create();\n * </pre>\n *\n * <p>Notes:\n *\n * <ul>\n *   <li>The order of invocation of configuration methods does not matter.\n *   <li>The default serialization of {@link Date} and its subclasses in Gson does not contain\n *       time-zone information. So, if you are using date/time instances, use {@code GsonBuilder}\n *       and its {@code setDateFormat} methods.\n *   <li>By default no explicit {@link Strictness} is set; some of the {@link Gson} methods behave\n *       as if {@link Strictness#LEGACY_STRICT} was used whereas others behave as if {@link\n *       Strictness#LENIENT} was used. Prefer explicitly setting a strictness with {@link\n *       #setStrictness(Strictness)} to avoid this legacy behavior.\n * </ul>\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @author Jesse Wilson\n */\npublic final class GsonBuilder {\n  private Excluder excluder = Excluder.DEFAULT;\n  private LongSerializationPolicy longSerializationPolicy = LongSerializationPolicy.DEFAULT;\n  private FieldNamingStrategy fieldNamingPolicy = FieldNamingPolicy.IDENTITY;\n  private final Map<Type, InstanceCreator<?>> instanceCreators = new HashMap<>();\n  private final List<TypeAdapterFactory> factories = new ArrayList<>();\n\n  /** tree-style hierarchy factories. These come after factories for backwards compatibility. */\n  private final List<TypeAdapterFactory> hierarchyFactories = new ArrayList<>();\n\n  private boolean serializeNulls = DEFAULT_SERIALIZE_NULLS;\n  private String datePattern = DEFAULT_DATE_PATTERN;\n  private int dateStyle = DateFormat.DEFAULT;\n  private int timeStyle = DateFormat.DEFAULT;\n  private boolean complexMapKeySerialization = DEFAULT_COMPLEX_MAP_KEYS;\n  private boolean serializeSpecialFloatingPointValues = DEFAULT_SPECIALIZE_FLOAT_VALUES;\n  private boolean escapeHtmlChars = DEFAULT_ESCAPE_HTML;\n  private FormattingStyle formattingStyle = DEFAULT_FORMATTING_STYLE;\n  private boolean generateNonExecutableJson = DEFAULT_JSON_NON_EXECUTABLE;\n  private Strictness strictness = DEFAULT_STRICTNESS;\n  private boolean useJdkUnsafe = DEFAULT_USE_JDK_UNSAFE;\n  private ToNumberStrategy objectToNumberStrategy = DEFAULT_OBJECT_TO_NUMBER_STRATEGY;\n  private ToNumberStrategy numberToNumberStrategy = DEFAULT_NUMBER_TO_NUMBER_STRATEGY;\n  private final ArrayDeque<ReflectionAccessFilter> reflectionFilters = new ArrayDeque<>();\n\n  /**\n   * Creates a GsonBuilder instance that can be used to build Gson with various configuration\n   * settings. GsonBuilder follows the builder pattern, and it is typically used by first invoking\n   * various configuration methods to set desired options, and finally calling {@link #create()}.\n   */\n  public GsonBuilder() {}\n\n  /**\n   * Constructs a GsonBuilder instance from a Gson instance. The newly constructed GsonBuilder has\n   * the same configuration as the previously built Gson instance.\n   *\n   * @param gson the gson instance whose configuration should be applied to a new GsonBuilder.\n   */\n  GsonBuilder(Gson gson) {\n    this.excluder = gson.excluder;\n    this.fieldNamingPolicy = gson.fieldNamingStrategy;\n    this.instanceCreators.putAll(gson.instanceCreators);\n    this.serializeNulls = gson.serializeNulls;\n    this.complexMapKeySerialization = gson.complexMapKeySerialization;\n    this.generateNonExecutableJson = gson.generateNonExecutableJson;\n    this.escapeHtmlChars = gson.htmlSafe;\n    this.formattingStyle = gson.formattingStyle;\n    this.strictness = gson.strictness;\n    this.serializeSpecialFloatingPointValues = gson.serializeSpecialFloatingPointValues;\n    this.longSerializationPolicy = gson.longSerializationPolicy;\n    this.datePattern = gson.datePattern;\n    this.dateStyle = gson.dateStyle;\n    this.timeStyle = gson.timeStyle;\n    this.factories.addAll(gson.builderFactories);\n    this.hierarchyFactories.addAll(gson.builderHierarchyFactories);\n    this.useJdkUnsafe = gson.useJdkUnsafe;\n    this.objectToNumberStrategy = gson.objectToNumberStrategy;\n    this.numberToNumberStrategy = gson.numberToNumberStrategy;\n    this.reflectionFilters.addAll(gson.reflectionFilters);\n  }\n\n  /**\n   * Configures Gson to enable versioning support. Versioning support works based on the annotation\n   * types {@link Since} and {@link Until}. It allows including or excluding fields and classes\n   * based on the specified version. See the documentation of these annotation types for more\n   * information.\n   *\n   * <p>By default versioning support is disabled and usage of {@code @Since} and {@code @Until} has\n   * no effect.\n   *\n   * @param version the version number to use.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @throws IllegalArgumentException if the version number is NaN or negative\n   * @see Since\n   * @see Until\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setVersion(double version) {\n    if (Double.isNaN(version) || version < 0.0) {\n      throw new IllegalArgumentException(\"Invalid version: \" + version);\n    }\n    excluder = excluder.withVersion(version);\n    return this;\n  }\n\n  /**\n   * Configures Gson to excludes all class fields that have the specified modifiers. By default,\n   * Gson will exclude all fields marked {@code transient} or {@code static}. This method will\n   * override that behavior.\n   *\n   * <p>This is a convenience method which behaves as if an {@link ExclusionStrategy} which excludes\n   * these fields was {@linkplain #setExclusionStrategies(ExclusionStrategy...) registered with this\n   * builder}.\n   *\n   * @param modifiers the field modifiers. You must use the modifiers specified in the {@link\n   *     java.lang.reflect.Modifier} class. For example, {@link\n   *     java.lang.reflect.Modifier#TRANSIENT}, {@link java.lang.reflect.Modifier#STATIC}.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder excludeFieldsWithModifiers(int... modifiers) {\n    Objects.requireNonNull(modifiers);\n    excluder = excluder.withModifiers(modifiers);\n    return this;\n  }\n\n  /**\n   * Makes the output JSON non-executable in Javascript by prefixing the generated JSON with some\n   * special text. This prevents attacks from third-party sites through script sourcing. See <a\n   * href=\"http://code.google.com/p/google-gson/issues/detail?id=42\">Gson Issue 42</a> for details.\n   *\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.3\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder generateNonExecutableJson() {\n    this.generateNonExecutableJson = true;\n    return this;\n  }\n\n  /**\n   * Configures Gson to exclude all fields from consideration for serialization and deserialization\n   * that do not have the {@link com.google.gson.annotations.Expose} annotation.\n   *\n   * <p>This is a convenience method which behaves as if an {@link ExclusionStrategy} which excludes\n   * these fields was {@linkplain #setExclusionStrategies(ExclusionStrategy...) registered with this\n   * builder}.\n   *\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder excludeFieldsWithoutExposeAnnotation() {\n    excluder = excluder.excludeFieldsWithoutExposeAnnotation();\n    return this;\n  }\n\n  /**\n   * Configures Gson to serialize null fields. By default, Gson omits all fields that are null\n   * during serialization.\n   *\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.2\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder serializeNulls() {\n    this.serializeNulls = true;\n    return this;\n  }\n\n  /**\n   * Configures Gson to serialize {@code Map} objects with complex keys as JSON arrays. Enabling\n   * this feature will only change the serialized form if the map key is a complex type (i.e.\n   * non-primitive) in its <strong>serialized</strong> JSON form. The default implementation of map\n   * serialization uses {@code toString()} on the key; however, when this is called then one of the\n   * following cases apply:\n   *\n   * <p><b>Maps as JSON objects</b>\n   *\n   * <p>For this case, assume that a type adapter is registered to serialize and deserialize some\n   * {@code Point} class, which contains an x and y coordinate, to/from the JSON Primitive string\n   * value {@code \"(x,y)\"}. The Java map would then be serialized as a {@link JsonObject}.\n   *\n   * <p>Below is an example:\n   *\n   * <pre>{@code\n   * Gson gson = new GsonBuilder()\n   *     .register(Point.class, new MyPointTypeAdapter())\n   *     .enableComplexMapKeySerialization()\n   *     .create();\n   *\n   * Map<Point, String> original = new LinkedHashMap<>();\n   * original.put(new Point(5, 6), \"a\");\n   * original.put(new Point(8, 8), \"b\");\n   * System.out.println(gson.toJson(original, type));\n   * }</pre>\n   *\n   * The above code prints this JSON object:\n   *\n   * <pre>{@code\n   * {\n   *   \"(5,6)\": \"a\",\n   *   \"(8,8)\": \"b\"\n   * }\n   * }</pre>\n   *\n   * <p><b>Maps as JSON arrays</b>\n   *\n   * <p>For this case, assume that a type adapter was NOT registered for some {@code Point} class,\n   * but rather the default Gson serialization is applied. In this case, some {@code new Point(2,3)}\n   * would serialize as {@code {\"x\":2,\"y\":3}}.\n   *\n   * <p>Given the assumption above, a {@code Map<Point, String>} will be serialized as an array of\n   * arrays (can be viewed as an entry set of pairs).\n   *\n   * <p>Below is an example of serializing complex types as JSON arrays:\n   *\n   * <pre>{@code\n   * Gson gson = new GsonBuilder()\n   *     .enableComplexMapKeySerialization()\n   *     .create();\n   *\n   * Map<Point, String> original = new LinkedHashMap<>();\n   * original.put(new Point(5, 6), \"a\");\n   * original.put(new Point(8, 8), \"b\");\n   * System.out.println(gson.toJson(original, type));\n   * }</pre>\n   *\n   * The JSON output would look as follows:\n   *\n   * <pre>{@code\n   * [\n   *   [\n   *     {\n   *       \"x\": 5,\n   *       \"y\": 6\n   *     },\n   *     \"a\"\n   *   ],\n   *   [\n   *     {\n   *       \"x\": 8,\n   *       \"y\": 8\n   *     },\n   *     \"b\"\n   *   ]\n   * ]\n   * }</pre>\n   *\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.7\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder enableComplexMapKeySerialization() {\n    complexMapKeySerialization = true;\n    return this;\n  }\n\n  /**\n   * Configures Gson to exclude inner classes (= non-{@code static} nested classes) during\n   * serialization and deserialization. This is a convenience method which behaves as if an {@link\n   * ExclusionStrategy} which excludes inner classes was {@linkplain\n   * #setExclusionStrategies(ExclusionStrategy...) registered with this builder}. This means inner\n   * classes will be serialized as JSON {@code null}, and will be deserialized as Java {@code null}\n   * with their JSON data being ignored. And fields with an inner class as type will be ignored\n   * during serialization and deserialization.\n   *\n   * <p>By default Gson serializes and deserializes inner classes, but ignores references to the\n   * enclosing instance. Deserialization might not be possible at all when {@link\n   * #disableJdkUnsafe()} is used (and no custom {@link InstanceCreator} is registered), or it can\n   * lead to unexpected {@code NullPointerException}s when the deserialized instance is used\n   * afterwards.\n   *\n   * <p>In general using inner classes with Gson should be avoided; they should be converted to\n   * {@code static} nested classes if possible.\n   *\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.3\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder disableInnerClassSerialization() {\n    excluder = excluder.disableInnerClassSerialization();\n    return this;\n  }\n\n  /**\n   * Configures Gson to apply a specific serialization policy for {@code Long} and {@code long}\n   * objects.\n   *\n   * @param serializationPolicy the particular policy to use for serializing longs.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.3\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setLongSerializationPolicy(LongSerializationPolicy serializationPolicy) {\n    this.longSerializationPolicy = Objects.requireNonNull(serializationPolicy);\n    return this;\n  }\n\n  /**\n   * Configures Gson to apply a specific naming policy to an object's fields during serialization\n   * and deserialization.\n   *\n   * <p>This method just delegates to {@link #setFieldNamingStrategy(FieldNamingStrategy)}.\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setFieldNamingPolicy(FieldNamingPolicy namingConvention) {\n    return setFieldNamingStrategy(namingConvention);\n  }\n\n  /**\n   * Configures Gson to apply a specific naming strategy to an object's fields during serialization\n   * and deserialization.\n   *\n   * <p>The created Gson instance might only use the field naming strategy once for a field and\n   * cache the result. It is not guaranteed that the strategy will be used again every time the\n   * value of a field is serialized or deserialized.\n   *\n   * @param fieldNamingStrategy the naming strategy to apply to the fields\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.3\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setFieldNamingStrategy(FieldNamingStrategy fieldNamingStrategy) {\n    this.fieldNamingPolicy = Objects.requireNonNull(fieldNamingStrategy);\n    return this;\n  }\n\n  /**\n   * Configures Gson to apply a specific number strategy during deserialization of {@link Object}.\n   *\n   * @param objectToNumberStrategy the actual object-to-number strategy\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @see ToNumberPolicy#DOUBLE The default object-to-number strategy\n   * @since 2.8.9\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setObjectToNumberStrategy(ToNumberStrategy objectToNumberStrategy) {\n    this.objectToNumberStrategy = Objects.requireNonNull(objectToNumberStrategy);\n    return this;\n  }\n\n  /**\n   * Configures Gson to apply a specific number strategy during deserialization of {@link Number}.\n   *\n   * @param numberToNumberStrategy the actual number-to-number strategy\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @see ToNumberPolicy#LAZILY_PARSED_NUMBER The default number-to-number strategy\n   * @since 2.8.9\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setNumberToNumberStrategy(ToNumberStrategy numberToNumberStrategy) {\n    this.numberToNumberStrategy = Objects.requireNonNull(numberToNumberStrategy);\n    return this;\n  }\n\n  /**\n   * Configures Gson to apply a set of exclusion strategies during both serialization and\n   * deserialization. Each of the {@code strategies} will be applied as a disjunction rule. This\n   * means that if one of the {@code strategies} suggests that a field (or class) should be skipped\n   * then that field (or object) is skipped during serialization/deserialization. The strategies are\n   * added to the existing strategies (if any); the existing strategies are not replaced.\n   *\n   * <p>Fields are excluded for serialization and deserialization when {@link\n   * ExclusionStrategy#shouldSkipField(FieldAttributes) shouldSkipField} returns {@code true}, or\n   * when {@link ExclusionStrategy#shouldSkipClass(Class) shouldSkipClass} returns {@code true} for\n   * the field type. Gson behaves as if the field did not exist; its value is not serialized and on\n   * deserialization if a JSON member with this name exists it is skipped by default.<br>\n   * When objects of an excluded type (as determined by {@link\n   * ExclusionStrategy#shouldSkipClass(Class) shouldSkipClass}) are serialized a JSON null is\n   * written to output, and when deserialized the JSON value is skipped and {@code null} is\n   * returned.\n   *\n   * <p>The created Gson instance might only use an exclusion strategy once for a field or class and\n   * cache the result. It is not guaranteed that the strategy will be used again every time the\n   * value of a field or a class is serialized or deserialized.\n   *\n   * @param strategies the set of strategy object to apply during object (de)serialization.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.4\n   */\n  \n@CanIgnoreReturnValue\n  public GsonBuilder setExclusionStrategies(ExclusionStrategy... strategies) {\n    Objects.requireNonNull(strategies);\n    for (ExclusionStrategy strategy : strategies) {\n      excluder = excluder.withExclusionStrategy(strategy, true, true);\n    }\n    return this;\n  }\n\n  /**\n   * Configures Gson to apply the passed in exclusion strategy during serialization. If this method\n   * is invoked numerous times with different exclusion strategy objects then the exclusion\n   * strategies that were added will be applied as a disjunction rule. This means that if one of the\n   * added exclusion strategies suggests that a field (or class) should be skipped then that field\n   * (or object) is skipped during its serialization.\n   *\n   * <p>See the documentation of {@link #setExclusionStrategies(ExclusionStrategy...)} for a\n   * detailed description of the effect of exclusion strategies.\n   *\n   * @param strategy an exclusion strategy to apply during serialization.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.7\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder addSerializationExclusionStrategy(ExclusionStrategy strategy) {\n    Objects.requireNonNull(strategy);\n    excluder = excluder.withExclusionStrategy(strategy, true, false);\n    return this;\n  }\n\n  /**\n   * Configures Gson to apply the passed in exclusion strategy during deserialization. If this\n   * method is invoked numerous times with different exclusion strategy objects then the exclusion\n   * strategies that were added will be applied as a disjunction rule. This means that if one of the\n   * added exclusion strategies suggests that a field (or class) should be skipped then that field\n   * (or object) is skipped during its deserialization.\n   *\n   * <p>See the documentation of {@link #setExclusionStrategies(ExclusionStrategy...)} for a\n   * detailed description of the effect of exclusion strategies.\n   *\n   * @param strategy an exclusion strategy to apply during deserialization.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.7\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder addDeserializationExclusionStrategy(ExclusionStrategy strategy) {\n    Objects.requireNonNull(strategy);\n    excluder = excluder.withExclusionStrategy(strategy, false, true);\n    return this;\n  }\n\n  /**\n   * Configures Gson to output JSON that fits in a page for pretty printing. This option only\n   * affects JSON serialization.\n   *\n   * <p>This is a convenience method which simply calls {@link #setFormattingStyle(FormattingStyle)}\n   * with {@link FormattingStyle#PRETTY}.\n   *\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setPrettyPrinting() {\n    return setFormattingStyle(FormattingStyle.PRETTY);\n  }\n\n  /**\n   * Configures Gson to output JSON that uses a certain kind of formatting style (for example\n   * newline and indent). This option only affects JSON serialization. By default Gson produces\n   * compact JSON output without any formatting.\n   *\n   * @param formattingStyle the formatting style to use.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since $next-version$\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setFormattingStyle(FormattingStyle formattingStyle) {\n    this.formattingStyle = Objects.requireNonNull(formattingStyle);\n    return this;\n  }\n\n  /**\n   * Sets the strictness of this builder to {@link Strictness#LENIENT}.\n   *\n   * @deprecated This method is equivalent to calling {@link #setStrictness(Strictness)} with {@link\n   *     Strictness#LENIENT}: {@code setStrictness(Strictness.LENIENT)}\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern.\n   * @see JsonReader#setStrictness(Strictness)\n   * @see JsonWriter#setStrictness(Strictness)\n   * @see #setStrictness(Strictness)\n   */\n  @Deprecated\n  @InlineMe(\n      replacement = \"this.setStrictness(Strictness.LENIENT)\",\n      imports = \"com.google.gson.Strictness\")\n  @CanIgnoreReturnValue\n  public GsonBuilder setLenient() {\n    return setStrictness(Strictness.LENIENT);\n  }\n\n  /**\n   * Sets the strictness of this builder to the provided parameter.\n   *\n   * <p>This changes how strict the <a href=\"https://www.ietf.org/rfc/rfc8259.txt\">RFC 8259 JSON\n   * specification</a> is enforced when parsing or writing JSON. For details on this, refer to\n   * {@link JsonReader#setStrictness(Strictness)} and {@link JsonWriter#setStrictness(Strictness)}.\n   *\n   * @param strictness the new strictness mode. May not be {@code null}.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern.\n   * @see JsonReader#setStrictness(Strictness)\n   * @see JsonWriter#setStrictness(Strictness)\n   * @since $next-version$\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setStrictness(Strictness strictness) {\n    this.strictness = Objects.requireNonNull(strictness);\n    return this;\n  }\n\n  /**\n   * By default, Gson escapes HTML characters such as &lt; &gt; etc. Use this option to configure\n   * Gson to pass-through HTML characters as is.\n   *\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @since 1.3\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder disableHtmlEscaping() {\n    this.escapeHtmlChars = false;\n    return this;\n  }\n\n  /**\n   * Configures Gson to serialize {@code Date} objects according to the pattern provided. You can\n   * call this method or {@link #setDateFormat(int, int)} multiple times, but only the last\n   * invocation will be used to decide the serialization format.\n   *\n   * <p>The date format will be used to serialize and deserialize {@link java.util.Date} and in case\n   * the {@code java.sql} module is present, also {@link java.sql.Timestamp} and {@link\n   * java.sql.Date}.\n   *\n   * <p>Note that this pattern must abide by the convention provided by {@code SimpleDateFormat}\n   * class. See the documentation in {@link SimpleDateFormat} for more information on valid date and\n   * time patterns.\n   *\n   * @param pattern the pattern that dates will be serialized/deserialized to/from; can be {@code\n   *     null} to reset the pattern\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @throws IllegalArgumentException if the pattern is invalid\n   * @since 1.2\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setDateFormat(String pattern) {\n    if (pattern != null) {\n      try {\n        new SimpleDateFormat(pattern);\n      } catch (IllegalArgumentException e) {\n        // Throw exception if it is an invalid date format\n        throw new IllegalArgumentException(\"The date pattern '\" + pattern + \"' is not valid\", e);\n      }\n    }\n    this.datePattern = pattern;\n    return this;\n  }\n\n  /**\n   * Configures Gson to serialize {@code Date} objects according to the date style value provided.\n   * You can call this method or {@link #setDateFormat(String)} multiple times, but only the last\n   * invocation will be used to decide the serialization format. This methods leaves the current\n   * 'time style' unchanged.\n   *\n   * <p>Note that this style value should be one of the predefined constants in the {@link\n   * DateFormat} class, such as {@link DateFormat#MEDIUM}. See the documentation of the {@link\n   * DateFormat} class for more information on the valid style constants.\n   *\n   * @deprecated Counterintuitively, despite this method taking only a 'date style' Gson will use a\n   *     format which includes both date and time, with the 'time style' being the last value set by\n   *     {@link #setDateFormat(int, int)}. Therefore prefer using {@link #setDateFormat(int, int)}\n   *     and explicitly provide the desired 'time style'.\n   * @param dateStyle the predefined date style that date objects will be serialized/deserialized\n   *     to/from\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @throws IllegalArgumentException if the style is invalid\n   * @since 1.2\n   */\n  @Deprecated\n  @CanIgnoreReturnValue\n  public GsonBuilder setDateFormat(int dateStyle) {\n    this.dateStyle = checkDateFormatStyle(dateStyle);\n    this.datePattern = null;\n    return this;\n  }\n\n  /**\n   * Configures Gson to serialize {@code Date} objects according to the style value provided. You\n   * can call this method or {@link #setDateFormat(String)} multiple times, but only the last\n   * invocation will be used to decide the serialization format.\n   *\n   * <p>Note that this style value should be one of the predefined constants in the {@link\n   * DateFormat} class, such as {@link DateFormat#MEDIUM}. See the documentation of the {@link\n   * DateFormat} class for more information on the valid style constants.\n   *\n   * @param dateStyle the predefined date style that date objects will be serialized/deserialized\n   *     to/from\n   * @param timeStyle the predefined style for the time portion of the date objects\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @throws IllegalArgumentException if the style values are invalid\n   * @since 1.2\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder setDateFormat(int dateStyle, int timeStyle) {\n    this.dateStyle = checkDateFormatStyle(dateStyle);\n    this.timeStyle = checkDateFormatStyle(timeStyle);\n    this.datePattern = null;\n    return this;\n  }\n\n  private static int checkDateFormatStyle(int style) {\n    // Valid DateFormat styles are: 0, 1, 2, 3 (FULL, LONG, MEDIUM, SHORT)\n    if (style < 0 || style > 3) {\n      throw new IllegalArgumentException(\"Invalid style: \" + style);\n    }\n    return style;\n  }\n\n  /**\n   * Configures Gson for custom serialization or deserialization. This method combines the\n   * registration of an {@link TypeAdapter}, {@link InstanceCreator}, {@link JsonSerializer}, and a\n   * {@link JsonDeserializer}. It is best used when a single object {@code typeAdapter} implements\n   * all the required interfaces for custom serialization with Gson. If a type adapter was\n   * previously registered for the specified {@code type}, it is overwritten.\n   *\n   * <p>This registers the type specified and no other types: you must manually register related\n   * types! For example, applications registering {@code boolean.class} should also register {@code\n   * Boolean.class}. And when registering an adapter for a class which has subclasses, you might\n   * also want to register the adapter for subclasses, or use {@link\n   * #registerTypeHierarchyAdapter(Class, Object)} instead.\n   *\n   * <p>{@link JsonSerializer} and {@link JsonDeserializer} are made \"{@code null}-safe\". This means\n   * when trying to serialize {@code null}, Gson will write a JSON {@code null} and the serializer\n   * is not called. Similarly when deserializing a JSON {@code null}, Gson will emit {@code null}\n   * without calling the deserializer. If it is desired to handle {@code null} values, a {@link\n   * TypeAdapter} should be used instead.\n   *\n   * @param type the type definition for the type adapter being registered\n   * @param typeAdapter This object must implement at least one of the {@link TypeAdapter}, {@link\n   *     InstanceCreator}, {@link JsonSerializer}, and a {@link JsonDeserializer} interfaces.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @throws IllegalArgumentException if the type adapter being registered is for {@code Object}\n   *     class or {@link JsonElement} or any of its subclasses\n   * @see #registerTypeHierarchyAdapter(Class, Object)\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder registerTypeAdapter(Type type, Object typeAdapter) {\n    Objects.requireNonNull(type);\n    $Gson$Preconditions.checkArgument(\n        typeAdapter instanceof JsonSerializer<?>\n            || typeAdapter instanceof JsonDeserializer<?>\n            || typeAdapter instanceof InstanceCreator<?>\n            || typeAdapter instanceof TypeAdapter<?>);\n\n    if (isTypeObjectOrJsonElement(type)) {\n      throw new IllegalArgumentException(\"Cannot override built-in adapter for \" + type);\n    }\n\n    if (typeAdapter instanceof InstanceCreator<?>) {\n      instanceCreators.put(type, (InstanceCreator<?>) typeAdapter);\n    }\n    if (typeAdapter instanceof JsonSerializer<?> || typeAdapter instanceof JsonDeserializer<?>) {\n      TypeToken<?> typeToken = TypeToken.get(type);\n      factories.add(TreeTypeAdapter.newFactoryWithMatchRawType(typeToken, typeAdapter));\n    }\n    if (typeAdapter instanceof TypeAdapter<?>) {\n      @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n      TypeAdapterFactory factory =\n          TypeAdapters.newFactory(TypeToken.get(type), (TypeAdapter) typeAdapter);\n      factories.add(factory);\n    }\n    return this;\n  }\n\n  private static boolean isTypeObjectOrJsonElement(Type type) {\n    return type instanceof Class\n        && (type == Object.class || JsonElement.class.isAssignableFrom((Class<?>) type));\n  }\n\n  /**\n   * Registers a factory for type adapters. Registering a factory is useful when the type adapter\n   * needs to be configured based on the type of the field being processed. Gson is designed to\n   * handle a large number of factories, so you should consider registering them to be at par with\n   * registering an individual type adapter.\n   *\n   * <p>The created Gson instance might only use the factory once to create an adapter for a\n   * specific type and cache the result. It is not guaranteed that the factory will be used again\n   * every time the type is serialized or deserialized.\n   *\n   * @since 2.1\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder registerTypeAdapterFactory(TypeAdapterFactory factory) {\n    Objects.requireNonNull(factory);\n    factories.add(factory);\n    return this;\n  }\n\n  /**\n   * Configures Gson for custom serialization or deserialization for an inheritance type hierarchy.\n   * This method combines the registration of a {@link TypeAdapter}, {@link JsonSerializer} and a\n   * {@link JsonDeserializer}. If a type adapter was previously registered for the specified type\n   * hierarchy, it is overridden. If a type adapter is registered for a specific type in the type\n   * hierarchy, it will be invoked instead of the one registered for the type hierarchy.\n   *\n   * @param baseType the class definition for the type adapter being registered for the base class\n   *     or interface\n   * @param typeAdapter This object must implement at least one of {@link TypeAdapter}, {@link\n   *     JsonSerializer} or {@link JsonDeserializer} interfaces.\n   * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n   * @throws IllegalArgumentException if the type adapter being registered is for {@link\n   *     JsonElement} or any of its subclasses\n   * @since 1.7\n   */\n  @CanIgnoreReturnValue\n  public GsonBuilder registerTypeHierarchyAdapter(Class<?> baseType, Object typeAdapter) {\n    Objects.requireNonNull(baseType);\n...\n", "needle_token_start": 11956, "needle_token_end": 12041, "code_context_ntokens": 16069, "output": ["```java\n/**\n * Configures Gson to apply a set of exclusion strategies during both serialization and\n * deserialization. Each of the {@code strategies} will be applied as a disjunction rule. This\n * means that if one of the {@code strategies} suggests that a field (or class) should be skipped\n * then that field (or object) is skipped during serialization/deserialization. The strategies are\n * added to the existing strategies (if any); the existing strategies are not replaced.\n *\n * <p>Fields are excluded for serialization and deserialization when {@link\n * ExclusionStrategy#shouldSkipField(FieldAttributes) shouldSkipField} returns {@code true}, or\n * when {@link ExclusionStrategy#shouldSkipClass(Class) shouldSkipClass} returns {@code true} for\n * the field type. Gson behaves as if the field did not exist; its value is not serialized and on\n * deserialization if a JSON member with this name exists it is skipped by default.<br>\n * When objects of an excluded type (as determined by {@link\n * ExclusionStrategy#shouldSkipClass(Class) shouldSkipClass}) are serialized a JSON null is\n * written to output, and when deserialized the JSON value is skipped and {@code null} is\n * returned.\n *\n * <p>The created Gson instance might only use an exclusion strategy once for a field or class and\n * cache the result. It is not guaranteed that the strategy will be used again every time the\n * value of a field or a class is serialized or deserialized.\n *\n * @param strategies the set of strategy object to apply during object (de)serialization.\n * @return a reference to this {@code GsonBuilder} object to fulfill the \"Builder\" pattern\n * @since 1.4\n */\n@CanIgnoreReturnValue\npublic GsonBuilder setExclusionStrategies(ExclusionStrategy... strategies) {\n  Objects.requireNonNull(strategies);\n  for (ExclusionStrategy strategy : strategies) {\n    excluder = excluder.withExclusionStrategy(strategy, true, true);\n  }\n  return this;\n}\n```"]}
{"repo": "google/gson", "name": "run", "language": "java", "path": "gson/src/main/java/com/google/gson/internal/bind/TypeAdapters.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this function is to parse JSON data from an input source and convert it into a structured JSON element, handling both JSON objects and arrays dynamically.\n2. **Input**: The input is a stream of JSON data.\n3. **Output**: The output is a structured JSON element (either a JSON object or a JSON array) that represents the parsed data.\n4. **Procedure**: The function continuously reads from the JSON input stream, determining whether the current element is a JSON object or array. It uses a stack to keep track of nested structures. For each element, it checks if it starts a new object or array (nesting) or represents a terminal value (like a string or number). It adds elements to the current JSON structure and handles nesting by pushing the current structure onto the stack when moving deeper. When the end of a nested structure is reached, it pops from the stack to continue with the higher level. The process repeats until the entire JSON data is parsed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/com/google/gson/reflect/TypeToken.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.reflect;\n\nimport com.google.gson.internal.$Gson$Types;\nimport com.google.gson.internal.TroubleshootingGuide;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Objects;\n\n/**\n * Represents a generic type {@code T}. Java doesn't yet provide a way to represent generic types,\n * so this class does. Forces clients to create a subclass of this class which enables retrieval the\n * type information even at runtime.\n *\n * <p>For example, to create a type literal for {@code List<String>}, you can create an empty\n * anonymous class:\n *\n * <p>{@code TypeToken<List<String>> list = new TypeToken<List<String>>() {};}\n *\n * <p>Capturing a type variable as type argument of an anonymous {@code TypeToken} subclass is not\n * allowed, for example {@code TypeToken<List<T>>}. Due to type erasure the runtime type of a type\n * variable is not available to Gson and therefore it cannot provide the functionality one might\n * expect. This would give a false sense of type-safety at compile time and could lead to an\n * unexpected {@code ClassCastException} at runtime.\n *\n * <p>If the type arguments of the parameterized type are only available at runtime, for example\n * when you want to create a {@code List<E>} based on a {@code Class<E>} representing the element\n * type, the method {@link #getParameterized(Type, Type...)} can be used.\n *\n * @author Bob Lee\n * @author Sven Mawson\n * @author Jesse Wilson\n */\npublic class TypeToken<T> {\n  private final Class<? super T> rawType;\n  private final Type type;\n  private final int hashCode;\n\n  /**\n   * Constructs a new type literal. Derives represented class from type parameter.\n   *\n   * <p>Clients create an empty anonymous subclass. Doing so embeds the type parameter in the\n   * anonymous class's type hierarchy so we can reconstitute it at runtime despite erasure, for\n   * example:\n   *\n   * <p>{@code new TypeToken<List<String>>() {}}\n   *\n   * @throws IllegalArgumentException If the anonymous {@code TypeToken} subclass captures a type\n   *     variable, for example {@code TypeToken<List<T>>}. See the {@code TypeToken} class\n   *     documentation for more details.\n   */\n  @SuppressWarnings(\"unchecked\")\n  protected TypeToken() {\n    this.type = getTypeTokenTypeArgument();\n    this.rawType = (Class<? super T>) $Gson$Types.getRawType(type);\n    this.hashCode = type.hashCode();\n  }\n\n  /** Unsafe. Constructs a type literal manually. */\n  @SuppressWarnings(\"unchecked\")\n  private TypeToken(Type type) {\n    this.type = $Gson$Types.canonicalize(Objects.requireNonNull(type));\n    this.rawType = (Class<? super T>) $Gson$Types.getRawType(this.type);\n    this.hashCode = this.type.hashCode();\n  }\n\n  private static boolean isCapturingTypeVariablesForbidden() {\n    return !Objects.equals(System.getProperty(\"gson.allowCapturingTypeVariables\"), \"true\");\n  }\n\n  /**\n   * Verifies that {@code this} is an instance of a direct subclass of TypeToken and returns the\n   * type argument for {@code T} in {@link $Gson$Types#canonicalize canonical form}.\n...\n// Path: gson/src/main/java/com/google/gson/TypeAdapterFactory.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.gson.reflect.TypeToken;\n\n/**\n * Creates type adapters for set of related types. Type adapter factories are most useful when\n * several types share similar structure in their JSON form.\n *\n * <h2>Examples</h2>\n *\n * <h3>Example: Converting enums to lowercase</h3>\n *\n * In this example, we implement a factory that creates type adapters for all enums. The type\n * adapters will write enums in lowercase, despite the fact that they're defined in {@code\n * CONSTANT_CASE} in the corresponding Java model:\n *\n * <pre>{@code\n * public class LowercaseEnumTypeAdapterFactory implements TypeAdapterFactory {\n *   public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {\n *     Class<T> rawType = (Class<T>) type.getRawType();\n *     if (!rawType.isEnum()) {\n *       return null;\n *     }\n *\n *     final Map<String, T> lowercaseToConstant = new HashMap<>();\n *     for (T constant : rawType.getEnumConstants()) {\n *       lowercaseToConstant.put(toLowercase(constant), constant);\n *     }\n *\n *     return new TypeAdapter<T>() {\n *       public void write(JsonWriter out, T value) throws IOException {\n *         if (value == null) {\n *           out.nullValue();\n *         } else {\n *           out.value(toLowercase(value));\n *         }\n *       }\n *\n *       public T read(JsonReader reader) throws IOException {\n *         if (reader.peek() == JsonToken.NULL) {\n *           reader.nextNull();\n *           return null;\n *         } else {\n *           return lowercaseToConstant.get(reader.nextString());\n *         }\n *       }\n *     };\n *   }\n *\n *   private String toLowercase(Object o) {\n *     return o.toString().toLowerCase(Locale.US);\n *   }\n * }\n * }</pre>\n *\n * <p>Type adapter factories select which types they provide type adapters for. If a factory cannot\n * support a given type, it must return null when that type is passed to {@link #create}. Factories\n * should expect {@code create()} to be called on them for many types and should return null for\n * most of those types. In the above example the factory returns null for calls to {@code create()}\n * where {@code type} is not an enum.\n *\n * <p>A factory is typically called once per type, but the returned type adapter may be used many\n * times. It is most efficient to do expensive work like reflection in {@code create()} so that the\n * type adapter's {@code read()} and {@code write()} methods can be very fast. In this example the\n * mapping from lowercase name to enum value is computed eagerly.\n *\n * <p>As with type adapters, factories must be <i>registered</i> with a {@link\n * com.google.gson.GsonBuilder} for them to take effect:\n *\n * <pre>{@code\n * GsonBuilder builder = new GsonBuilder();\n * builder.registerTypeAdapterFactory(new LowercaseEnumTypeAdapterFactory());\n * ...\n * Gson gson = builder.create();\n * }</pre>\n *\n * If multiple factories support the same type, the factory registered earlier takes precedence.\n *\n * <h3>Example: Composing other type adapters</h3>\n *\n * In this example we implement a factory for Guava's {@code Multiset} collection type. The factory\n * can be used to create type adapters for multisets of any element type: the type adapter for\n * {@code Multiset<String>} is different from the type adapter for {@code Multiset<URL>}.\n *\n * <p>The type adapter <i>delegates</i> to another type adapter for the multiset elements. It\n * figures out the element type by reflecting on the multiset's type token. A {@code Gson} is passed\n * in to {@code create} for just this purpose:\n *\n * <pre>{@code\n * public class MultisetTypeAdapterFactory implements TypeAdapterFactory {\n *   public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n *     Type type = typeToken.getType();\n *     if (typeToken.getRawType() != Multiset.class\n *         || !(type instanceof ParameterizedType)) {\n *       return null;\n *     }\n *\n *     Type elementType = ((ParameterizedType) type).getActualTypeArguments()[0];\n *     TypeAdapter<?> elementAdapter = gson.getAdapter(TypeToken.get(elementType));\n *     return (TypeAdapter<T>) newMultisetAdapter(elementAdapter);\n *   }\n *\n *   private <E> TypeAdapter<Multiset<E>> newMultisetAdapter(\n *       final TypeAdapter<E> elementAdapter) {\n *     return new TypeAdapter<Multiset<E>>() {\n *       public void write(JsonWriter out, Multiset<E> value) throws IOException {\n *         if (value == null) {\n *           out.nullValue();\n *           return;\n *         }\n *\n *         out.beginArray();\n *         for (Multiset.Entry<E> entry : value.entrySet()) {\n *           out.value(entry.getCount());\n *           elementAdapter.write(out, entry.getElement());\n *         }\n *         out.endArray();\n *       }\n *\n *       public Multiset<E> read(JsonReader in) throws IOException {\n *         if (in.peek() == JsonToken.NULL) {\n *           in.nextNull();\n *           return null;\n *         }\n *\n *         Multiset<E> result = LinkedHashMultiset.create();\n *         in.beginArray();\n *         while (in.hasNext()) {\n *           int count = in.nextInt();\n *           E element = elementAdapter.read(in);\n *           result.add(element, count);\n *         }\n *         in.endArray();\n *         return result;\n *       }\n *     };\n *   }\n * }\n * }</pre>\n *\n * Delegating from one type adapter to another is extremely powerful; it's the foundation of how\n * Gson converts Java objects and collections. Whenever possible your factory should retrieve its\n * delegate type adapter in the {@code create()} method; this ensures potentially-expensive type\n * adapter creation happens only once.\n *\n * @since 2.1\n */\npublic interface TypeAdapterFactory {\n\n  /**\n   * Returns a type adapter for {@code type}, or null if this factory doesn't support {@code type}.\n   */\n  <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type);\n}\n\n// Path: gson/src/main/java/com/google/gson/annotations/SerializedName.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.annotations;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * An annotation that indicates this member should be serialized to JSON with the provided name\n * value as its field name.\n *\n * <p>This annotation will override any {@link com.google.gson.FieldNamingPolicy}, including the\n * default field naming policy, that may have been set on the {@link com.google.gson.Gson} instance.\n * A different naming policy can set using the {@code GsonBuilder} class. See {@link\n * com.google.gson.GsonBuilder#setFieldNamingPolicy(com.google.gson.FieldNamingPolicy)} for more\n * information.\n *\n * <p>Here is an example of how this annotation is meant to be used:\n *\n * <pre>\n * public class MyClass {\n *   &#64;SerializedName(\"name\") String a;\n *   &#64;SerializedName(value=\"name1\", alternate={\"name2\", \"name3\"}) String b;\n *   String c;\n *\n *   public MyClass(String a, String b, String c) {\n *     this.a = a;\n *     this.b = b;\n *     this.c = c;\n *   }\n * }\n * </pre>\n *\n * <p>The following shows the output that is generated when serializing an instance of the above\n * example class:\n *\n * <pre>\n * MyClass target = new MyClass(\"v1\", \"v2\", \"v3\");\n * Gson gson = new Gson();\n * String json = gson.toJson(target);\n * System.out.println(json);\n *\n * ===== OUTPUT =====\n * {\"name\":\"v1\",\"name1\":\"v2\",\"c\":\"v3\"}\n * </pre>\n *\n * <p>NOTE: The value you specify in this annotation must be a valid JSON field name. While\n * deserializing, all values specified in the annotation will be deserialized into the field. For\n * example:\n *\n * <pre>\n *   MyClass target = gson.fromJson(\"{'name1':'v1'}\", MyClass.class);\n *   assertEquals(\"v1\", target.b);\n *   target = gson.fromJson(\"{'name2':'v2'}\", MyClass.class);\n *   assertEquals(\"v2\", target.b);\n *   target = gson.fromJson(\"{'name3':'v3'}\", MyClass.class);\n *   assertEquals(\"v3\", target.b);\n * </pre>\n *\n * Note that MyClass.b is now deserialized from either name1, name2 or name3.\n *\n * @see com.google.gson.FieldNamingPolicy\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\n@Documented\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.FIELD, ElementType.METHOD})\npublic @interface SerializedName {\n\n  /**\n   * The desired name of the field when it is serialized or deserialized.\n   *\n   * @return the desired name of the field when it is serialized or deserialized\n   */\n  String value();\n\n  /**\n   * The alternative names of the field when it is deserialized\n   *\n   * @return the alternative names of the field when it is deserialized\n   */\n  String[] alternate() default {};\n}\n\n// Path: gson/src/main/java/com/google/gson/stream/JsonToken.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.stream;\n\n/**\n * A structure, name or value type in a JSON-encoded string.\n *\n * @author Jesse Wilson\n * @since 1.6\n */\npublic enum JsonToken {\n\n  /**\n   * The opening of a JSON array. Written using {@link JsonWriter#beginArray} and read using {@link\n   * JsonReader#beginArray}.\n   */\n  BEGIN_ARRAY,\n\n  /**\n   * The closing of a JSON array. Written using {@link JsonWriter#endArray} and read using {@link\n   * JsonReader#endArray}.\n   */\n  END_ARRAY,\n\n  /**\n   * The opening of a JSON object. Written using {@link JsonWriter#beginObject} and read using\n   * {@link JsonReader#beginObject}.\n   */\n  BEGIN_OBJECT,\n\n  /**\n   * The closing of a JSON object. Written using {@link JsonWriter#endObject} and read using {@link\n   * JsonReader#endObject}.\n   */\n  END_OBJECT,\n\n  /**\n   * A JSON property name. Within objects, tokens alternate between names and their values. Written\n   * using {@link JsonWriter#name} and read using {@link JsonReader#nextName}\n   */\n  NAME,\n\n  /** A JSON string. */\n  STRING,\n\n  /**\n   * A JSON number represented in this API by a Java {@code double}, {@code long}, or {@code int}.\n   */\n  NUMBER,\n\n  /** A JSON {@code true} or {@code false}. */\n  BOOLEAN,\n\n  /** A JSON {@code null}. */\n  NULL,\n\n  /**\n   * The end of the JSON stream. This sentinel value is returned by {@link JsonReader#peek()} to\n   * signal that the JSON-encoded value has no more tokens.\n   */\n  END_DOCUMENT\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/bind/TypeAdapters.java\n/*\n * Copyright (C) 2011 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal.bind;\n\nimport com.google.gson.Gson;\nimport com.google.gson.JsonArray;\nimport com.google.gson.JsonElement;\nimport com.google.gson.JsonIOException;\nimport com.google.gson.JsonNull;\nimport com.google.gson.JsonObject;\nimport com.google.gson.JsonPrimitive;\nimport com.google.gson.JsonSyntaxException;\nimport com.google.gson.TypeAdapter;\nimport com.google.gson.TypeAdapterFactory;\nimport com.google.gson.annotations.SerializedName;\nimport com.google.gson.internal.LazilyParsedNumber;\nimport com.google.gson.internal.NumberLimits;\nimport com.google.gson.internal.TroubleshootingGuide;\nimport com.google.gson.reflect.TypeToken;\nimport com.google.gson.stream.JsonReader;\nimport com.google.gson.stream.JsonToken;\nimport com.google.gson.stream.JsonWriter;\nimport java.io.IOException;\nimport java.lang.reflect.AccessibleObject;\nimport java.lang.reflect.Field;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.net.InetAddress;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.net.URL;\nimport java.security.AccessController;\nimport java.security.PrivilegedAction;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.BitSet;\nimport java.util.Calendar;\nimport java.util.Currency;\nimport java.util.Deque;\nimport java.util.GregorianCalendar;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.StringTokenizer;\nimport java.util.UUID;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicIntegerArray;\n\n/** Type adapters for basic types. */\npublic final class TypeAdapters {\n  private TypeAdapters() {\n    throw new UnsupportedOperationException();\n  }\n\n  @SuppressWarnings(\"rawtypes\")\n  public static final TypeAdapter<Class> CLASS =\n      new TypeAdapter<Class>() {\n        @Override\n        public void write(JsonWriter out, Class value) throws IOException {\n          throw new UnsupportedOperationException(\n              \"Attempted to serialize java.lang.Class: \"\n                  + value.getName()\n                  + \". Forgot to register a type adapter?\"\n                  + \"\\nSee \"\n                  + TroubleshootingGuide.createUrl(\"java-lang-class-unsupported\"));\n        }\n\n        @Override\n        public Class read(JsonReader in) throws IOException {\n          throw new UnsupportedOperationException(\n              \"Attempted to deserialize a java.lang.Class. Forgot to register a type adapter?\"\n                  + \"\\nSee \"\n                  + TroubleshootingGuide.createUrl(\"java-lang-class-unsupported\"));\n        }\n      }.nullSafe();\n\n  public static final TypeAdapterFactory CLASS_FACTORY = newFactory(Class.class, CLASS);\n\n  public static final TypeAdapter<BitSet> BIT_SET =\n      new TypeAdapter<BitSet>() {\n        @Override\n        public BitSet read(JsonReader in) throws IOException {\n          BitSet bitset = new BitSet();\n          in.beginArray();\n          int i = 0;\n          JsonToken tokenType = in.peek();\n          while (tokenType != JsonToken.END_ARRAY) {\n            boolean set;\n            switch (tokenType) {\n              case NUMBER:\n              case STRING:\n                int intValue = in.nextInt();\n                if (intValue == 0) {\n                  set = false;\n                } else if (intValue == 1) {\n                  set = true;\n                } else {\n                  throw new JsonSyntaxException(\n                      \"Invalid bitset value \"\n                          + intValue\n                          + \", expected 0 or 1; at path \"\n                          + in.getPreviousPath());\n                }\n                break;\n              case BOOLEAN:\n                set = in.nextBoolean();\n                break;\n              default:\n                throw new JsonSyntaxException(\n                    \"Invalid bitset value type: \" + tokenType + \"; at path \" + in.getPath());\n            }\n            if (set) {\n              bitset.set(i);\n            }\n            ++i;\n            tokenType = in.peek();\n          }\n          in.endArray();\n          return bitset;\n        }\n\n        @Override\n        public void write(JsonWriter out, BitSet src) throws IOException {\n          out.beginArray();\n          for (int i = 0, length = src.length(); i < length; i++) {\n            int value = src.get(i) ? 1 : 0;\n            out.value(value);\n          }\n          out.endArray();\n        }\n      }.nullSafe();\n\n  public static final TypeAdapterFactory BIT_SET_FACTORY = newFactory(BitSet.class, BIT_SET);\n\n  public static final TypeAdapter<Boolean> BOOLEAN =\n      new TypeAdapter<Boolean>() {\n        @Override\n        public Boolean read(JsonReader in) throws IOException {\n          JsonToken peek = in.peek();\n          if (peek == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          } else if (peek == JsonToken.STRING) {\n            // support strings for compatibility with GSON 1.7\n            return Boolean.parseBoolean(in.nextString());\n          }\n          return in.nextBoolean();\n        }\n\n        @Override\n        public void write(JsonWriter out, Boolean value) throws IOException {\n          out.value(value);\n        }\n      };\n\n  /**\n   * Writes a boolean as a string. Useful for map keys, where booleans aren't otherwise permitted.\n   */\n  public static final TypeAdapter<Boolean> BOOLEAN_AS_STRING =\n      new TypeAdapter<Boolean>() {\n        @Override\n        public Boolean read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          return Boolean.valueOf(in.nextString());\n        }\n\n        @Override\n        public void write(JsonWriter out, Boolean value) throws IOException {\n          out.value(value == null ? \"null\" : value.toString());\n        }\n      };\n\n  public static final TypeAdapterFactory BOOLEAN_FACTORY =\n      newFactory(boolean.class, Boolean.class, BOOLEAN);\n\n  public static final TypeAdapter<Number> BYTE =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n\n          int intValue;\n          try {\n            intValue = in.nextInt();\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(e);\n          }\n          // Allow up to 255 to support unsigned values\n          if (intValue > 255 || intValue < Byte.MIN_VALUE) {\n            throw new JsonSyntaxException(\n                \"Lossy conversion from \" + intValue + \" to byte; at path \" + in.getPreviousPath());\n          }\n          return (byte) intValue;\n        }\n\n        @Override\n        public void write(JsonWriter out, Number value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n          } else {\n            out.value(value.byteValue());\n          }\n        }\n      };\n\n  public static final TypeAdapterFactory BYTE_FACTORY = newFactory(byte.class, Byte.class, BYTE);\n\n  public static final TypeAdapter<Number> SHORT =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n\n          int intValue;\n          try {\n            intValue = in.nextInt();\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(e);\n          }\n          // Allow up to 65535 to support unsigned values\n          if (intValue > 65535 || intValue < Short.MIN_VALUE) {\n            throw new JsonSyntaxException(\n                \"Lossy conversion from \" + intValue + \" to short; at path \" + in.getPreviousPath());\n          }\n          return (short) intValue;\n        }\n\n        @Override\n        public void write(JsonWriter out, Number value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n          } else {\n            out.value(value.shortValue());\n          }\n        }\n      };\n\n  public static final TypeAdapterFactory SHORT_FACTORY =\n      newFactory(short.class, Short.class, SHORT);\n\n  public static final TypeAdapter<Number> INTEGER =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          try {\n            return in.nextInt();\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, Number value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n          } else {\n            out.value(value.intValue());\n          }\n        }\n      };\n  public static final TypeAdapterFactory INTEGER_FACTORY =\n      newFactory(int.class, Integer.class, INTEGER);\n\n  public static final TypeAdapter<AtomicInteger> ATOMIC_INTEGER =\n      new TypeAdapter<AtomicInteger>() {\n        @Override\n        public AtomicInteger read(JsonReader in) throws IOException {\n          try {\n            return new AtomicInteger(in.nextInt());\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, AtomicInteger value) throws IOException {\n          out.value(value.get());\n        }\n      }.nullSafe();\n  public static final TypeAdapterFactory ATOMIC_INTEGER_FACTORY =\n      newFactory(AtomicInteger.class, TypeAdapters.ATOMIC_INTEGER);\n\n  public static final TypeAdapter<AtomicBoolean> ATOMIC_BOOLEAN =\n      new TypeAdapter<AtomicBoolean>() {\n        @Override\n        public AtomicBoolean read(JsonReader in) throws IOException {\n          return new AtomicBoolean(in.nextBoolean());\n        }\n\n        @Override\n        public void write(JsonWriter out, AtomicBoolean value) throws IOException {\n          out.value(value.get());\n        }\n      }.nullSafe();\n  public static final TypeAdapterFactory ATOMIC_BOOLEAN_FACTORY =\n      newFactory(AtomicBoolean.class, TypeAdapters.ATOMIC_BOOLEAN);\n\n  public static final TypeAdapter<AtomicIntegerArray> ATOMIC_INTEGER_ARRAY =\n      new TypeAdapter<AtomicIntegerArray>() {\n        @Override\n        public AtomicIntegerArray read(JsonReader in) throws IOException {\n          List<Integer> list = new ArrayList<>();\n          in.beginArray();\n          while (in.hasNext()) {\n            try {\n              int integer = in.nextInt();\n              list.add(integer);\n            } catch (NumberFormatException e) {\n              throw new JsonSyntaxException(e);\n            }\n          }\n          in.endArray();\n          int length = list.size();\n          AtomicIntegerArray array = new AtomicIntegerArray(length);\n          for (int i = 0; i < length; ++i) {\n            array.set(i, list.get(i));\n          }\n          return array;\n        }\n\n        @Override\n        public void write(JsonWriter out, AtomicIntegerArray value) throws IOException {\n          out.beginArray();\n          for (int i = 0, length = value.length(); i < length; i++) {\n            out.value(value.get(i));\n          }\n          out.endArray();\n        }\n      }.nullSafe();\n  public static final TypeAdapterFactory ATOMIC_INTEGER_ARRAY_FACTORY =\n      newFactory(AtomicIntegerArray.class, TypeAdapters.ATOMIC_INTEGER_ARRAY);\n\n  public static final TypeAdapter<Number> LONG =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          try {\n            return in.nextLong();\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, Number value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n          } else {\n            out.value(value.longValue());\n          }\n        }\n      };\n\n  public static final TypeAdapter<Number> FLOAT =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          return (float) in.nextDouble();\n        }\n\n        @Override\n        public void write(JsonWriter out, Number value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n          } else {\n            // For backward compatibility don't call `JsonWriter.value(float)` because that method\n            // has been newly added and not all custom JsonWriter implementations might override\n            // it yet\n            Number floatNumber = value instanceof Float ? value : value.floatValue();\n            out.value(floatNumber);\n          }\n        }\n      };\n\n  public static final TypeAdapter<Number> DOUBLE =\n      new TypeAdapter<Number>() {\n        @Override\n        public Number read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          return in.nextDouble();\n        }\n\n        @Override\n        public void write(JsonWriter out, Number value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n          } else {\n            out.value(value.doubleValue());\n          }\n        }\n      };\n\n  public static final TypeAdapter<Character> CHARACTER =\n      new TypeAdapter<Character>() {\n        @Override\n        public Character read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          String str = in.nextString();\n          if (str.length() != 1) {\n            throw new JsonSyntaxException(\n                \"Expecting character, got: \" + str + \"; at \" + in.getPreviousPath());\n          }\n          return str.charAt(0);\n        }\n\n        @Override\n        public void write(JsonWriter out, Character value) throws IOException {\n          out.value(value == null ? null : String.valueOf(value));\n        }\n      };\n\n  public static final TypeAdapterFactory CHARACTER_FACTORY =\n      newFactory(char.class, Character.class, CHARACTER);\n\n  public static final TypeAdapter<String> STRING =\n      new TypeAdapter<String>() {\n        @Override\n        public String read(JsonReader in) throws IOException {\n          JsonToken peek = in.peek();\n          if (peek == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          /* coerce booleans to strings for backwards compatibility */\n          if (peek == JsonToken.BOOLEAN) {\n            return Boolean.toString(in.nextBoolean());\n          }\n          return in.nextString();\n        }\n\n        @Override\n        public void write(JsonWriter out, String value) throws IOException {\n          out.value(value);\n        }\n      };\n\n  public static final TypeAdapter<BigDecimal> BIG_DECIMAL =\n      new TypeAdapter<BigDecimal>() {\n        @Override\n        public BigDecimal read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          String s = in.nextString();\n          try {\n            return NumberLimits.parseBigDecimal(s);\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(\n                \"Failed parsing '\" + s + \"' as BigDecimal; at path \" + in.getPreviousPath(), e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, BigDecimal value) throws IOException {\n          out.value(value);\n        }\n      };\n\n  public static final TypeAdapter<BigInteger> BIG_INTEGER =\n      new TypeAdapter<BigInteger>() {\n        @Override\n        public BigInteger read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          String s = in.nextString();\n          try {\n            return NumberLimits.parseBigInteger(s);\n          } catch (NumberFormatException e) {\n            throw new JsonSyntaxException(\n                \"Failed parsing '\" + s + \"' as BigInteger; at path \" + in.getPreviousPath(), e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, BigInteger value) throws IOException {\n          out.value(value);\n        }\n      };\n\n  public static final TypeAdapter<LazilyParsedNumber> LAZILY_PARSED_NUMBER =\n      new TypeAdapter<LazilyParsedNumber>() {\n        // Normally users should not be able to access and deserialize LazilyParsedNumber because\n        // it is an internal type, but implement this nonetheless in case there are legit corner\n        // cases where this is possible\n        @Override\n        public LazilyParsedNumber read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          return new LazilyParsedNumber(in.nextString());\n        }\n\n        @Override\n        public void write(JsonWriter out, LazilyParsedNumber value) throws IOException {\n          out.value(value);\n        }\n      };\n\n  public static final TypeAdapterFactory STRING_FACTORY = newFactory(String.class, STRING);\n\n  public static final TypeAdapter<StringBuilder> STRING_BUILDER =\n      new TypeAdapter<StringBuilder>() {\n        @Override\n        public StringBuilder read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          return new StringBuilder(in.nextString());\n        }\n\n        @Override\n        public void write(JsonWriter out, StringBuilder value) throws IOException {\n          out.value(value == null ? null : value.toString());\n        }\n      };\n\n  public static final TypeAdapterFactory STRING_BUILDER_FACTORY =\n      newFactory(StringBuilder.class, STRING_BUILDER);\n\n  public static final TypeAdapter<StringBuffer> STRING_BUFFER =\n      new TypeAdapter<StringBuffer>() {\n        @Override\n        public StringBuffer read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          return new StringBuffer(in.nextString());\n        }\n\n        @Override\n        public void write(JsonWriter out, StringBuffer value) throws IOException {\n          out.value(value == null ? null : value.toString());\n        }\n      };\n\n  public static final TypeAdapterFactory STRING_BUFFER_FACTORY =\n      newFactory(StringBuffer.class, STRING_BUFFER);\n\n  public static final TypeAdapter<URL> URL =\n      new TypeAdapter<URL>() {\n        @Override\n        public URL read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          String nextString = in.nextString();\n          return nextString.equals(\"null\") ? null : new URL(nextString);\n        }\n\n        @Override\n        public void write(JsonWriter out, URL value) throws IOException {\n          out.value(value == null ? null : value.toExternalForm());\n        }\n      };\n\n  public static final TypeAdapterFactory URL_FACTORY = newFactory(URL.class, URL);\n\n  public static final TypeAdapter<URI> URI =\n      new TypeAdapter<URI>() {\n        @Override\n        public URI read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          try {\n            String nextString = in.nextString();\n            return nextString.equals(\"null\") ? null : new URI(nextString);\n          } catch (URISyntaxException e) {\n            throw new JsonIOException(e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, URI value) throws IOException {\n          out.value(value == null ? null : value.toASCIIString());\n        }\n      };\n\n  public static final TypeAdapterFactory URI_FACTORY = newFactory(URI.class, URI);\n\n  public static final TypeAdapter<InetAddress> INET_ADDRESS =\n      new TypeAdapter<InetAddress>() {\n        @Override\n        public InetAddress read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          // regrettably, this should have included both the host name and the host address\n          // For compatibility, we use InetAddress.getByName rather than the possibly-better\n          // .getAllByName\n          @SuppressWarnings(\"AddressSelection\")\n          InetAddress addr = InetAddress.getByName(in.nextString());\n          return addr;\n        }\n\n        @Override\n        public void write(JsonWriter out, InetAddress value) throws IOException {\n          out.value(value == null ? null : value.getHostAddress());\n        }\n      };\n\n  public static final TypeAdapterFactory INET_ADDRESS_FACTORY =\n      newTypeHierarchyFactory(InetAddress.class, INET_ADDRESS);\n\n  public static final TypeAdapter<UUID> UUID =\n      new TypeAdapter<UUID>() {\n        @Override\n        public UUID read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          String s = in.nextString();\n          try {\n            return java.util.UUID.fromString(s);\n          } catch (IllegalArgumentException e) {\n            throw new JsonSyntaxException(\n                \"Failed parsing '\" + s + \"' as UUID; at path \" + in.getPreviousPath(), e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, UUID value) throws IOException {\n          out.value(value == null ? null : value.toString());\n        }\n      };\n\n  public static final TypeAdapterFactory UUID_FACTORY = newFactory(UUID.class, UUID);\n\n  public static final TypeAdapter<Currency> CURRENCY =\n      new TypeAdapter<Currency>() {\n        @Override\n        public Currency read(JsonReader in) throws IOException {\n          String s = in.nextString();\n          try {\n            return Currency.getInstance(s);\n          } catch (IllegalArgumentException e) {\n            throw new JsonSyntaxException(\n                \"Failed parsing '\" + s + \"' as Currency; at path \" + in.getPreviousPath(), e);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, Currency value) throws IOException {\n          out.value(value.getCurrencyCode());\n        }\n      }.nullSafe();\n  public static final TypeAdapterFactory CURRENCY_FACTORY = newFactory(Currency.class, CURRENCY);\n\n  public static final TypeAdapter<Calendar> CALENDAR =\n      new TypeAdapter<Calendar>() {\n        private static final String YEAR = \"year\";\n        private static final String MONTH = \"month\";\n        private static final String DAY_OF_MONTH = \"dayOfMonth\";\n        private static final String HOUR_OF_DAY = \"hourOfDay\";\n        private static final String MINUTE = \"minute\";\n        private static final String SECOND = \"second\";\n\n        @Override\n        public Calendar read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          in.beginObject();\n          int year = 0;\n          int month = 0;\n          int dayOfMonth = 0;\n          int hourOfDay = 0;\n          int minute = 0;\n          int second = 0;\n          while (in.peek() != JsonToken.END_OBJECT) {\n            String name = in.nextName();\n            int value = in.nextInt();\n            switch (name) {\n              case YEAR:\n                year = value;\n                break;\n              case MONTH:\n                month = value;\n                break;\n              case DAY_OF_MONTH:\n                dayOfMonth = value;\n                break;\n              case HOUR_OF_DAY:\n                hourOfDay = value;\n                break;\n              case MINUTE:\n                minute = value;\n                break;\n              case SECOND:\n                second = value;\n                break;\n              default:\n                // Ignore unknown JSON property\n            }\n          }\n          in.endObject();\n          return new GregorianCalendar(year, month, dayOfMonth, hourOfDay, minute, second);\n        }\n\n        @Override\n        public void write(JsonWriter out, Calendar value) throws IOException {\n          if (value == null) {\n            out.nullValue();\n            return;\n          }\n          out.beginObject();\n          out.name(YEAR);\n          out.value(value.get(Calendar.YEAR));\n          out.name(MONTH);\n          out.value(value.get(Calendar.MONTH));\n          out.name(DAY_OF_MONTH);\n          out.value(value.get(Calendar.DAY_OF_MONTH));\n          out.name(HOUR_OF_DAY);\n          out.value(value.get(Calendar.HOUR_OF_DAY));\n          out.name(MINUTE);\n          out.value(value.get(Calendar.MINUTE));\n          out.name(SECOND);\n          out.value(value.get(Calendar.SECOND));\n          out.endObject();\n        }\n      };\n\n  public static final TypeAdapterFactory CALENDAR_FACTORY =\n      newFactoryForMultipleTypes(Calendar.class, GregorianCalendar.class, CALENDAR);\n\n  public static final TypeAdapter<Locale> LOCALE =\n      new TypeAdapter<Locale>() {\n        @Override\n        public Locale read(JsonReader in) throws IOException {\n          if (in.peek() == JsonToken.NULL) {\n            in.nextNull();\n            return null;\n          }\n          String locale = in.nextString();\n          StringTokenizer tokenizer = new StringTokenizer(locale, \"_\");\n          String language = null;\n          String country = null;\n          String variant = null;\n          if (tokenizer.hasMoreElements()) {\n            language = tokenizer.nextToken();\n          }\n          if (tokenizer.hasMoreElements()) {\n            country = tokenizer.nextToken();\n          }\n          if (tokenizer.hasMoreElements()) {\n            variant = tokenizer.nextToken();\n          }\n          if (country == null && variant == null) {\n            return new Locale(language);\n          } else if (variant == null) {\n            return new Locale(language, country);\n          } else {\n            return new Locale(language, country, variant);\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, Locale value) throws IOException {\n          out.value(value == null ? null : value.toString());\n        }\n      };\n\n  public static final TypeAdapterFactory LOCALE_FACTORY = newFactory(Locale.class, LOCALE);\n\n  public static final TypeAdapter<JsonElement> JSON_ELEMENT =\n      new TypeAdapter<JsonElement>() {\n        /**\n         * Tries to begin reading a JSON array or JSON object, returning {@code null} if the next\n         * element is neither of those.\n         */\n        private JsonElement tryBeginNesting(JsonReader in, JsonToken peeked) throws IOException {\n          switch (peeked) {\n            case BEGIN_ARRAY:\n              in.beginArray();\n              return new JsonArray();\n            case BEGIN_OBJECT:\n              in.beginObject();\n              return new JsonObject();\n            default:\n              return null;\n          }\n        }\n\n        /** Reads a {@link JsonElement} which cannot have any nested elements */\n        private JsonElement readTerminal(JsonReader in, JsonToken peeked) throws IOException {\n          switch (peeked) {\n            case STRING:\n              return new JsonPrimitive(in.nextString());\n            case NUMBER:\n              String number = in.nextString();\n              return new JsonPrimitive(new LazilyParsedNumber(number));\n            case BOOLEAN:\n              return new JsonPrimitive(in.nextBoolean());\n            case NULL:\n              in.nextNull();\n              return JsonNull.INSTANCE;\n            default:\n              // When read(JsonReader) is called with JsonReader in invalid state\n              throw new IllegalStateException(\"Unexpected token: \" + peeked);\n          }\n        }\n\n        @Override\n        public JsonElement read(JsonReader in) throws IOException {\n          if (in instanceof JsonTreeReader) {\n            return ((JsonTreeReader) in).nextJsonElement();\n          }\n\n          // Either JsonArray or JsonObject\n          JsonElement current;\n          JsonToken peeked = in.peek();\n\n          current = tryBeginNesting(in, peeked);\n          if (current == null) {\n            return readTerminal(in, peeked);\n          }\n\n          Deque<JsonElement> stack = new ArrayDeque<>();\n\n          while (true) {\n            while (in.hasNext()) {\n              String name = null;\n              // Name is only used for JSON object members\n              if (current instanceof JsonObject) {\n                name = in.nextName();\n              }\n\n              peeked = in.peek();\n              JsonElement value = tryBeginNesting(in, peeked);\n              boolean isNesting = value != null;\n\n              if (value == null) {\n                value = readTerminal(in, peeked);\n              }\n\n              if (current instanceof JsonArray) {\n                ((JsonArray) current).add(value);\n              } else {\n                ((JsonObject) current).add(name, value);\n              }\n\n              if (isNesting) {\n                stack.addLast(current);\n                current = value;\n              }\n            }\n\n            // End current element\n            if (current instanceof JsonArray) {\n              in.endArray();\n            } else {\n              in.endObject();\n            }\n\n            if (stack.isEmpty()) {\n              return current;\n            } else {\n              // Continue with enclosing element\n              current = stack.removeLast();\n            }\n          }\n        }\n\n        @Override\n        public void write(JsonWriter out, JsonElement value) throws IOException {\n          if (value == null || value.isJsonNull()) {\n            out.nullValue();\n          } else if (value.isJsonPrimitive()) {\n            JsonPrimitive primitive = value.getAsJsonPrimitive();\n            if (primitive.isNumber()) {\n              out.value(primitive.getAsNumber());\n            } else if (primitive.isBoolean()) {\n              out.value(primitive.getAsBoolean());\n            } else {\n              out.value(primitive.getAsString());\n            }\n\n          } else if (value.isJsonArray()) {\n            out.beginArray();\n            for (JsonElement e : value.getAsJsonArray()) {\n              write(out, e);\n            }\n            out.endArray();\n\n          } else if (value.isJsonObject()) {\n            out.beginObject();\n            for (Map.Entry<String, JsonElement> e : value.getAsJsonObject().entrySet()) {\n              out.name(e.getKey());\n              write(out, e.getValue());\n            }\n            out.endObject();\n\n          } else {\n            throw new IllegalArgumentException(\"Couldn't write \" + value.getClass());\n          }\n        }\n      };\n\n  public static final TypeAdapterFactory JSON_ELEMENT_FACTORY =\n      newTypeHierarchyFactory(JsonElement.class, JSON_ELEMENT);\n\n  private static final class EnumTypeAdapter<T extends Enum<T>> extends TypeAdapter<T> {\n    private final Map<String, T> nameToConstant = new HashMap<>();\n    private final Map<String, T> stringToConstant = new HashMap<>();\n    private final Map<T, String> constantToName = new HashMap<>();\n\n    public EnumTypeAdapter(final Class<T> classOfT) {\n      try {\n        // Uses reflection to find enum constants to work around name mismatches for obfuscated\n        // classes\n        // Reflection access might throw SecurityException, therefore run this in privileged\n        // context; should be acceptable because this only retrieves enum constants, but does not\n        // expose anything else\n        Field[] constantFields =\n            AccessController.doPrivileged(\n                new PrivilegedAction<Field[]>() {\n                  \n@Override\n                  public Field[] run() {\n                    Field[] fields = classOfT.getDeclaredFields();\n                    ArrayList<Field> constantFieldsList = new ArrayList<>(fields.length);\n                    for (Field f : fields) {\n                      if (f.isEnumConstant()) {\n                        constantFieldsList.add(f);\n                      }\n                    }\n\n                    Field[] constantFields = constantFieldsList.toArray(new Field[0]);\n                    AccessibleObject.setAccessible(constantFields, true);\n                    return constantFields;\n                  }\n                });\n        for (Field constantField : constantFields) {\n          @SuppressWarnings(\"unchecked\")\n          T constant = (T) constantField.get(null);\n          String name = constant.name();\n          String toStringVal = constant.toString();\n\n          SerializedName annotation = constantField.getAnnotation(SerializedName.class);\n          if (annotation != null) {\n            name = annotation.value();\n            for (String alternate : annotation.alternate()) {\n              nameToConstant.put(alternate, constant);\n            }\n          }\n          nameToConstant.put(name, constant);\n          stringToConstant.put(toStringVal, constant);\n          constantToName.put(constant, name);\n        }\n      } catch (IllegalAccessException e) {\n        throw new AssertionError(e);\n      }\n    }\n\n    @Override\n    public T read(JsonReader in) throws IOException {\n      if (in.peek() == JsonToken.NULL) {\n        in.nextNull();\n        return null;\n      }\n      String key = in.nextString();\n      T constant = nameToConstant.get(key);\n      return (constant == null) ? stringToConstant.get(key) : constant;\n    }\n\n    @Override\n    public void write(JsonWriter out, T value) throws IOException {\n      out.value(value == null ? null : constantToName.get(value));\n    }\n  }\n\n  public static final TypeAdapterFactory ENUM_FACTORY =\n      new TypeAdapterFactory() {\n        @Override\n        public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n          Class<? super T> rawType = typeToken.getRawType();\n          if (!Enum.class.isAssignableFrom(rawType) || rawType == Enum.class) {\n            return null;\n          }\n          if (!rawType.isEnum()) {\n            rawType = rawType.getSuperclass(); // handle anonymous subclasses\n          }\n          @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n          TypeAdapter<T> adapter = (TypeAdapter<T>) new EnumTypeAdapter(rawType);\n          return adapter;\n        }\n      };\n\n  @SuppressWarnings(\"TypeParameterNaming\")\n  public static <TT> TypeAdapterFactory newFactory(\n      final TypeToken<TT> type, final TypeAdapter<TT> typeAdapter) {\n    return new TypeAdapterFactory() {\n      @SuppressWarnings(\"unchecked\") // we use a runtime check to make sure the 'T's equal\n      @Override\n      public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n        return typeToken.equals(type) ? (TypeAdapter<T>) typeAdapter : null;\n      }\n    };\n  }\n\n  @SuppressWarnings(\"TypeParameterNaming\")\n  public static <TT> TypeAdapterFactory newFactory(\n      final Class<TT> type, final TypeAdapter<TT> typeAdapter) {\n    return new TypeAdapterFactory() {\n      @SuppressWarnings(\"unchecked\") // we use a runtime check to make sure the 'T's equal\n      @Override\n      public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n        return typeToken.getRawType() == type ? (TypeAdapter<T>) typeAdapter : null;\n      }\n\n      @Override\n      public String toString() {\n        return \"Factory[type=\" + type.getName() + \",adapter=\" + typeAdapter + \"]\";\n      }\n    };\n  }\n\n  @SuppressWarnings(\"TypeParameterNaming\")\n  public static <TT> TypeAdapterFactory newFactory(\n      final Class<TT> unboxed, final Class<TT> boxed, final TypeAdapter<? super TT> typeAdapter) {\n    return new TypeAdapterFactory() {\n      @SuppressWarnings(\"unchecked\") // we use a runtime check to make sure the 'T's equal\n      @Override\n      public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n        Class<? super T> rawType = typeToken.getRawType();\n        return (rawType == unboxed || rawType == boxed) ? (TypeAdapter<T>) typeAdapter : null;\n      }\n\n      @Override\n      public String toString() {\n        return \"Factory[type=\"\n            + boxed.getName()\n            + \"+\"\n            + unboxed.getName()\n            + \",adapter=\"\n            + typeAdapter\n            + \"]\";\n      }\n    };\n  }\n\n  @SuppressWarnings(\"TypeParameterNaming\")\n  public static <TT> TypeAdapterFactory newFactoryForMultipleTypes(\n      final Class<TT> base,\n      final Class<? extends TT> sub,\n      final TypeAdapter<? super TT> typeAdapter) {\n    return new TypeAdapterFactory() {\n      @SuppressWarnings(\"unchecked\") // we use a runtime check to make sure the 'T's equal\n      @Override\n      public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) {\n        Class<? super T> rawType = typeToken.getRawType();\n        return (rawType == base || rawType == sub) ? (TypeAdapter<T>) typeAdapter : null;\n      }\n\n      @Override\n      public String toString() {\n        return \"Factory[type=\"\n            + base.getName()\n            + \"+\"\n            + sub.getName()\n            + \",adapter=\"\n            + typeAdapter\n            + \"]\";\n      }\n    };\n  }\n\n  /**\n   * Returns a factory for all subtypes of {@code typeAdapter}. We do a runtime check to confirm\n   * that the deserialized type matches the type requested.\n   */\n  public static <T1> TypeAdapterFactory newTypeHierarchyFactory(\n      final Class<T1> clazz, final TypeAdapter<T1> typeAdapter) {\n    return new TypeAdapterFactory() {\n      @SuppressWarnings(\"unchecked\")\n      @Override\n      public <T2> TypeAdapter<T2> create(Gson gson, TypeToken<T2> typeToken) {\n        final Class<? super T2> requestedType = typeToken.getRawType();\n        if (!clazz.isAssignableFrom(requestedType)) {\n          return null;\n        }\n        return (TypeAdapter<T2>)\n            new TypeAdapter<T1>() {\n              @Override\n              public void write(JsonWriter out, T1 value) throws IOException {\n                typeAdapter.write(out, value);\n              }\n\n              @Override\n              public T1 read(JsonReader in) throws IOException {\n                T1 result = typeAdapter.read(in);\n                if (result != null && !requestedType.isInstance(result)) {\n                  throw new JsonSyntaxException(\n                      \"Expected a \"\n                          + requestedType.getName()\n                          + \" but was \"\n                          + result.getClass().getName()\n                          + \"; at path \"\n                          + in.getPreviousPath());\n                }\n                return result;\n              }\n            };\n      }\n\n      @Override\n      public String toString() {\n        return \"Factory[typeHierarchy=\" + clazz.getName() + \",adapter=\" + typeAdapter + \"]\";\n      }\n    };\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/stream/MalformedJsonException.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.stream;\n\nimport com.google.gson.Strictness;\nimport java.io.IOException;\n\n/**\n * Thrown when a reader encounters malformed JSON. Some syntax errors can be ignored by using {@link\n * Strictness#LENIENT} for {@link JsonReader#setStrictness(Strictness)}.\n */\npublic final class MalformedJsonException extends IOException {\n  private static final long serialVersionUID = 1L;\n\n  public MalformedJsonException(String msg) {\n    super(msg);\n  }\n\n  public MalformedJsonException(String msg, Throwable throwable) {\n    super(msg, throwable);\n  }\n\n  public MalformedJsonException(Throwable throwable) {\n    super(throwable);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/Streams.java\n/*\n * Copyright (C) 2010 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n...\n", "needle_token_start": 13434, "needle_token_end": 13572, "code_context_ntokens": 15904, "output": ["```java\n/**\n * Reads a JSON document from a reader and returns a JsonElement representing the entire document.\n *\n * @param in the reader to read the JSON document from\n * @return a JsonElement representing the entire JSON document\n * @throws JsonIOException if there is an IO error reading the reader\n * @throws JsonSyntaxException if the JSON document is not properly formed\n */\npublic static JsonElement parse(JsonReader in) throws JsonIOException, JsonSyntaxException {\n  boolean isEmpty = in.peek() == JsonToken.END_DOCUMENT;\n  JsonElement element = parse(in, isEmpty ? JsonToken.BEGIN_OBJECT : in.peek());\n  if (!isEmpty && in.peek() != JsonToken.END_DOCUMENT) {\n    throw new JsonSyntaxException(\"Did not consume the entire document.\");\n  }\n  return element;\n}\n\nprivate static JsonElement parse(JsonReader in, JsonToken token) throws JsonIOException, JsonSyntaxException {\n  switch (token) {\n    case BEGIN_OBJECT:\n      return parseObject(in);\n    case BEGIN_ARRAY:\n      return parseArray(in);\n    case STRING:\n      return new JsonPrimitive(in.nextString());\n    case NUMBER:\n      return new JsonPrimitive(parseNumber(in.nextString()));\n    case BOOLEAN:\n      return new JsonPrimitive(in.nextBoolean());\n    case NULL:\n      in.nextNull();\n      return JsonNull.INSTANCE;\n    case END_OBJECT:\n    case END_ARRAY:\n      throw new JsonSyntaxException(\"Unexpected end of JSON element\");\n    default:\n      throw new JsonSyntaxException(\"Unexpected token: \" + token);\n  }\n}\n\nprivate static JsonObject parseObject(JsonReader in) throws JsonIOException, JsonSyntaxException {\n  JsonObject jsonObject = new JsonObject();\n  in.beginObject();\n  while (in.hasNext()) {\n    String name = in.nextName();\n    JsonElement value = parse(in, in.peek());\n    jsonObject.add(name, value);\n  }\n  in.endObject();\n  return jsonObject;\n}\n\nprivate static JsonArray parseArray(JsonReader in) throws JsonIOException, JsonSyntaxException {\n  JsonArray jsonArray = new JsonArray();\n  in.beginArray();\n  while (in.hasNext()) {\n    JsonElement value = parse(in, in.peek());\n    jsonArray.add(value);\n  }\n  in.endArray();\n  return jsonArray;\n}\n\nprivate static Number parseNumber(String s) throws NumberFormatException {\n  char first = s.charAt(0);\n  if (first == '-' || ('0' <= first && first <= '9')) {\n    try {\n      return Long.parseLong(s);\n    } catch (NumberFormatException e) {\n      try {\n        return new BigInteger(s);\n      } catch (NumberFormatException e1) {\n        return new BigDecimal(s);\n      }\n    }\n  } else if (first == 'I' || first == 'N' || first == 'S') {\n    return new LazilyParsedNumber(s);\n  } else {\n    throw new NumberFormatException(\"Invalid number format: \" + s);\n  }\n}\n```"]}
{"repo": "google/gson", "name": "has", "language": "java", "path": "gson/src/main/java/com/google/gson/JsonObject.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine if a specific member exists within the object.\n2. **Input**: A string representing the name of the member to check.\n3. **Output**: A boolean value; `true` if the member exists, `false` otherwise.\n4. **Procedure**: Checks if the member name provided as input exists in the object's internal map of members.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: gson/src/main/java/module-info.java\n/*\n * Copyright (C) 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Defines the Gson serialization/deserialization API.\n *\n * @since 2.8.6\n */\nmodule com.google.gson {\n  exports com.google.gson;\n  exports com.google.gson.annotations;\n  exports com.google.gson.reflect;\n  exports com.google.gson.stream;\n\n  // Dependency on Error Prone Annotations\n  requires static com.google.errorprone.annotations;\n\n  // Optional dependency on java.sql\n  requires static java.sql;\n\n  // Optional dependency on jdk.unsupported for JDK's sun.misc.Unsafe\n  requires static jdk.unsupported;\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonParseException.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\n/**\n * This exception is raised if there is a serious issue that occurs during parsing of a Json string.\n * One of the main usages for this class is for the Gson infrastructure. If the incoming Json is\n * bad/malicious, an instance of this exception is raised.\n *\n * <p>This exception is a {@link RuntimeException} because it is exposed to the client. Using a\n * {@link RuntimeException} avoids bad coding practices on the client side where they catch the\n * exception and do nothing. It is often the case that you want to blow up if there is a parsing\n * error (i.e. often clients do not know how to recover from a {@link JsonParseException}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic class JsonParseException extends RuntimeException {\n  static final long serialVersionUID = -4086729973971783390L;\n\n  /**\n   * Creates exception with the specified message. If you are wrapping another exception, consider\n   * using {@link #JsonParseException(String, Throwable)} instead.\n   *\n   * @param msg error message describing a possible cause of this exception.\n   */\n  public JsonParseException(String msg) {\n    super(msg);\n  }\n\n  /**\n   * Creates exception with the specified message and cause.\n   *\n   * @param msg error message describing what happened.\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonParseException(String msg, Throwable cause) {\n    super(msg, cause);\n  }\n\n  /**\n   * Creates exception with the specified cause. Consider using {@link #JsonParseException(String,\n   * Throwable)} instead if you can describe what happened.\n   *\n   * @param cause root exception that caused this exception to be thrown.\n   */\n  public JsonParseException(Throwable cause) {\n    super(cause);\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/InstanceCreator.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport java.lang.reflect.Type;\n\n/**\n * This interface is implemented to create instances of a class that does not define a no-args\n * constructor. If you can modify the class, you should instead add a private, or public no-args\n * constructor. However, that is not possible for library classes, such as JDK classes, or a\n * third-party library that you do not have source-code of. In such cases, you should define an\n * instance creator for the class. Implementations of this interface should be registered with\n * {@link GsonBuilder#registerTypeAdapter(Type, Object)} method before Gson will be able to use\n * them.\n *\n * <p>Let us look at an example where defining an InstanceCreator might be useful. The {@code Id}\n * class defined below does not have a default no-args constructor.\n *\n * <pre>\n * public class Id&lt;T&gt; {\n *   private final Class&lt;T&gt; clazz;\n *   private final long value;\n *   public Id(Class&lt;T&gt; clazz, long value) {\n *     this.clazz = clazz;\n *     this.value = value;\n *   }\n * }\n * </pre>\n *\n * <p>If Gson encounters an object of type {@code Id} during deserialization, it will throw an\n * exception. The easiest way to solve this problem will be to add a (public or private) no-args\n * constructor as follows:\n *\n * <pre>\n * private Id() {\n *   this(Object.class, 0L);\n * }\n * </pre>\n *\n * <p>However, let us assume that the developer does not have access to the source-code of the\n * {@code Id} class, or does not want to define a no-args constructor for it. The developer can\n * solve this problem by defining an {@code InstanceCreator} for {@code Id}:\n *\n * <pre>\n * class IdInstanceCreator implements InstanceCreator&lt;Id&gt; {\n *   public Id createInstance(Type type) {\n *     return new Id(Object.class, 0L);\n *   }\n * }\n * </pre>\n *\n * <p>Note that it does not matter what the fields of the created instance contain since Gson will\n * overwrite them with the deserialized values specified in JSON. You should also ensure that a\n * <i>new</i> object is returned, not a common object since its fields will be overwritten. The\n * developer will need to register {@code IdInstanceCreator} with Gson as follows:\n *\n * <pre>\n * Gson gson = new GsonBuilder().registerTypeAdapter(Id.class, new IdInstanceCreator()).create();\n * </pre>\n *\n * @param <T> the type of object that will be created by this implementation.\n * @see GsonBuilder#registerTypeAdapter(Type, Object)\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic interface InstanceCreator<T> {\n\n  /**\n   * Gson invokes this call-back method during deserialization to create an instance of the\n   * specified type. The fields of the returned instance are overwritten with the data present in\n   * the JSON. Since the prior contents of the object are destroyed and overwritten, do not return\n   * an instance that is useful elsewhere. In particular, do not return a common instance, always\n   * use {@code new} to create a new instance.\n   *\n   * @param type the parameterized T represented as a {@link Type}.\n   * @return a default object instance of type T.\n   */\n  public T createInstance(Type type);\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonNull.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\n/**\n * A class representing a JSON {@code null} value.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n * @since 1.2\n */\npublic final class JsonNull extends JsonElement {\n  /**\n   * Singleton for {@code JsonNull}.\n   *\n   * @since 1.8\n   */\n  public static final JsonNull INSTANCE = new JsonNull();\n\n  /**\n   * Creates a new {@code JsonNull} object.\n   *\n   * @deprecated Deprecated since Gson version 1.8, use {@link #INSTANCE} instead.\n   */\n  @Deprecated\n  public JsonNull() {\n    // Do nothing\n  }\n\n  /**\n   * Returns the same instance since it is an immutable value.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonNull deepCopy() {\n    return INSTANCE;\n  }\n\n  /** All instances of {@code JsonNull} have the same hash code since they are indistinguishable. */\n  @Override\n  public int hashCode() {\n    return JsonNull.class.hashCode();\n  }\n\n  /** All instances of {@code JsonNull} are considered equal. */\n  @Override\n  public boolean equals(Object other) {\n    return other instanceof JsonNull;\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java\n/*\n * Copyright (C) 2010 The Android Open Source Project\n * Copyright (C) 2012 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson.internal;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport java.io.IOException;\nimport java.io.InvalidObjectException;\nimport java.io.ObjectInputStream;\nimport java.io.ObjectStreamException;\nimport java.io.Serializable;\nimport java.util.AbstractMap;\nimport java.util.AbstractSet;\nimport java.util.Comparator;\nimport java.util.ConcurrentModificationException;\nimport java.util.Iterator;\nimport java.util.LinkedHashMap;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport java.util.Set;\n\n/**\n * A map of comparable keys to values. Unlike {@code TreeMap}, this class uses insertion order for\n * iteration order. Comparison order is only used as an optimization for efficient insertion and\n * removal.\n *\n * <p>This implementation was derived from Android 4.1's TreeMap class.\n */\n@SuppressWarnings(\"serial\") // ignore warning about missing serialVersionUID\npublic final class LinkedTreeMap<K, V> extends AbstractMap<K, V> implements Serializable {\n  @SuppressWarnings({\"unchecked\", \"rawtypes\"}) // to avoid Comparable<Comparable<Comparable<...>>>\n  private static final Comparator<Comparable> NATURAL_ORDER =\n      new Comparator<Comparable>() {\n        @Override\n        public int compare(Comparable a, Comparable b) {\n          return a.compareTo(b);\n        }\n      };\n\n  private final Comparator<? super K> comparator;\n  private final boolean allowNullValues;\n  Node<K, V> root;\n  int size = 0;\n  int modCount = 0;\n\n  // Used to preserve iteration order\n  final Node<K, V> header;\n\n  /**\n   * Create a natural order, empty tree map whose keys must be mutually comparable and non-null, and\n   * whose values can be {@code null}.\n   */\n  @SuppressWarnings(\"unchecked\") // unsafe! this assumes K is comparable\n  public LinkedTreeMap() {\n    this((Comparator<? super K>) NATURAL_ORDER, true);\n  }\n\n  /**\n   * Create a natural order, empty tree map whose keys must be mutually comparable and non-null.\n   *\n   * @param allowNullValues whether {@code null} is allowed as entry value\n   */\n  @SuppressWarnings(\"unchecked\") // unsafe! this assumes K is comparable\n  public LinkedTreeMap(boolean allowNullValues) {\n    this((Comparator<? super K>) NATURAL_ORDER, allowNullValues);\n  }\n\n  /**\n   * Create a tree map ordered by {@code comparator}. This map's keys may only be null if {@code\n   * comparator} permits.\n   *\n   * @param comparator the comparator to order elements with, or {@code null} to use the natural\n   *     ordering.\n   * @param allowNullValues whether {@code null} is allowed as entry value\n   */\n  // unsafe! if comparator is null, this assumes K is comparable\n  @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n  public LinkedTreeMap(Comparator<? super K> comparator, boolean allowNullValues) {\n    this.comparator = comparator != null ? comparator : (Comparator) NATURAL_ORDER;\n    this.allowNullValues = allowNullValues;\n    this.header = new Node<>(allowNullValues);\n  }\n\n  @Override\n  public int size() {\n    return size;\n  }\n\n  @Override\n  public V get(Object key) {\n    Node<K, V> node = findByObject(key);\n    return node != null ? node.value : null;\n  }\n\n  @Override\n  public boolean containsKey(Object key) {\n    return findByObject(key) != null;\n  }\n\n  @CanIgnoreReturnValue\n  @Override\n  public V put(K key, V value) {\n    if (key == null) {\n      throw new NullPointerException(\"key == null\");\n    }\n    if (value == null && !allowNullValues) {\n      throw new NullPointerException(\"value == null\");\n    }\n    Node<K, V> created = find(key, true);\n    V result = created.value;\n    created.value = value;\n    return result;\n  }\n\n  @Override\n  public void clear() {\n    root = null;\n    size = 0;\n    modCount++;\n\n    // Clear iteration order\n    Node<K, V> header = this.header;\n    header.next = header.prev = header;\n  }\n\n  @Override\n  public V remove(Object key) {\n    Node<K, V> node = removeInternalByKey(key);\n    return node != null ? node.value : null;\n  }\n\n  /**\n   * Returns the node at or adjacent to the given key, creating it if requested.\n   *\n   * @throws ClassCastException if {@code key} and the tree's keys aren't mutually comparable.\n   */\n  Node<K, V> find(K key, boolean create) {\n    Comparator<? super K> comparator = this.comparator;\n    Node<K, V> nearest = root;\n    int comparison = 0;\n\n    if (nearest != null) {\n      // Micro-optimization: avoid polymorphic calls to Comparator.compare().\n      @SuppressWarnings(\"unchecked\") // Throws a ClassCastException below if there's trouble.\n      Comparable<Object> comparableKey =\n          (comparator == NATURAL_ORDER) ? (Comparable<Object>) key : null;\n\n      while (true) {\n        comparison =\n            (comparableKey != null)\n                ? comparableKey.compareTo(nearest.key)\n                : comparator.compare(key, nearest.key);\n\n        // We found the requested key.\n        if (comparison == 0) {\n          return nearest;\n        }\n\n        // If it exists, the key is in a subtree. Go deeper.\n        Node<K, V> child = (comparison < 0) ? nearest.left : nearest.right;\n        if (child == null) {\n          break;\n        }\n\n        nearest = child;\n      }\n    }\n\n    // The key doesn't exist in this tree.\n    if (!create) {\n      return null;\n    }\n\n    // Create the node and add it to the tree or the table.\n    Node<K, V> header = this.header;\n    Node<K, V> created;\n    if (nearest == null) {\n      // Check that the value is comparable if we didn't do any comparisons.\n      if (comparator == NATURAL_ORDER && !(key instanceof Comparable)) {\n        throw new ClassCastException(key.getClass().getName() + \" is not Comparable\");\n      }\n      created = new Node<>(allowNullValues, nearest, key, header, header.prev);\n      root = created;\n    } else {\n      created = new Node<>(allowNullValues, nearest, key, header, header.prev);\n      if (comparison < 0) { // nearest.key is higher\n        nearest.left = created;\n      } else { // comparison > 0, nearest.key is lower\n        nearest.right = created;\n      }\n      rebalance(nearest, true);\n    }\n    size++;\n    modCount++;\n\n    return created;\n  }\n\n  @SuppressWarnings(\"unchecked\")\n  Node<K, V> findByObject(Object key) {\n    try {\n      return key != null ? find((K) key, false) : null;\n    } catch (ClassCastException e) {\n      return null;\n    }\n  }\n\n  /**\n   * Returns this map's entry that has the same key and value as {@code entry}, or null if this map\n   * has no such entry.\n   *\n   * <p>This method uses the comparator for key equality rather than {@code equals}. If this map's\n   * comparator isn't consistent with equals (such as {@code String.CASE_INSENSITIVE_ORDER}), then\n   * {@code remove()} and {@code contains()} will violate the collections API.\n   */\n  Node<K, V> findByEntry(Entry<?, ?> entry) {\n    Node<K, V> mine = findByObject(entry.getKey());\n    boolean valuesEqual = mine != null && equal(mine.value, entry.getValue());\n    return valuesEqual ? mine : null;\n  }\n\n  private static boolean equal(Object a, Object b) {\n    return Objects.equals(a, b);\n  }\n\n  /**\n   * Removes {@code node} from this tree, rearranging the tree's structure as necessary.\n   *\n   * @param unlink true to also unlink this node from the iteration linked list.\n   */\n  void removeInternal(Node<K, V> node, boolean unlink) {\n    if (unlink) {\n      node.prev.next = node.next;\n      node.next.prev = node.prev;\n    }\n\n    Node<K, V> left = node.left;\n    Node<K, V> right = node.right;\n    Node<K, V> originalParent = node.parent;\n    if (left != null && right != null) {\n\n      /*\n       * To remove a node with both left and right subtrees, move an\n       * adjacent node from one of those subtrees into this node's place.\n       *\n       * Removing the adjacent node may change this node's subtrees. This\n       * node may no longer have two subtrees once the adjacent node is\n       * gone!\n       */\n\n      Node<K, V> adjacent = (left.height > right.height) ? left.last() : right.first();\n      removeInternal(adjacent, false); // takes care of rebalance and size--\n\n      int leftHeight = 0;\n      left = node.left;\n      if (left != null) {\n        leftHeight = left.height;\n        adjacent.left = left;\n        left.parent = adjacent;\n        node.left = null;\n      }\n\n      int rightHeight = 0;\n      right = node.right;\n      if (right != null) {\n        rightHeight = right.height;\n        adjacent.right = right;\n        right.parent = adjacent;\n        node.right = null;\n      }\n\n      adjacent.height = Math.max(leftHeight, rightHeight) + 1;\n      replaceInParent(node, adjacent);\n      return;\n    } else if (left != null) {\n      replaceInParent(node, left);\n      node.left = null;\n    } else if (right != null) {\n      replaceInParent(node, right);\n      node.right = null;\n    } else {\n      replaceInParent(node, null);\n    }\n\n    rebalance(originalParent, false);\n    size--;\n    modCount++;\n  }\n\n  Node<K, V> removeInternalByKey(Object key) {\n    Node<K, V> node = findByObject(key);\n    if (node != null) {\n      removeInternal(node, true);\n    }\n    return node;\n  }\n\n  @SuppressWarnings(\"ReferenceEquality\")\n  private void replaceInParent(Node<K, V> node, Node<K, V> replacement) {\n    Node<K, V> parent = node.parent;\n    node.parent = null;\n    if (replacement != null) {\n      replacement.parent = parent;\n    }\n\n    if (parent != null) {\n      if (parent.left == node) {\n        parent.left = replacement;\n      } else {\n        assert parent.right == node;\n        parent.right = replacement;\n      }\n    } else {\n      root = replacement;\n    }\n  }\n\n  /**\n   * Rebalances the tree by making any AVL rotations necessary between the newly-unbalanced node and\n   * the tree's root.\n   *\n   * @param insert true if the node was unbalanced by an insert; false if it was by a removal.\n   */\n  private void rebalance(Node<K, V> unbalanced, boolean insert) {\n    for (Node<K, V> node = unbalanced; node != null; node = node.parent) {\n      Node<K, V> left = node.left;\n      Node<K, V> right = node.right;\n      int leftHeight = left != null ? left.height : 0;\n      int rightHeight = right != null ? right.height : 0;\n\n      int delta = leftHeight - rightHeight;\n      if (delta == -2) {\n        Node<K, V> rightLeft = right.left;\n        Node<K, V> rightRight = right.right;\n        int rightRightHeight = rightRight != null ? rightRight.height : 0;\n        int rightLeftHeight = rightLeft != null ? rightLeft.height : 0;\n\n        int rightDelta = rightLeftHeight - rightRightHeight;\n        if (rightDelta == -1 || (rightDelta == 0 && !insert)) {\n          rotateLeft(node); // AVL right right\n        } else {\n          assert (rightDelta == 1);\n          rotateRight(right); // AVL right left\n          rotateLeft(node);\n        }\n        if (insert) {\n          break; // no further rotations will be necessary\n        }\n\n      } else if (delta == 2) {\n        Node<K, V> leftLeft = left.left;\n        Node<K, V> leftRight = left.right;\n        int leftRightHeight = leftRight != null ? leftRight.height : 0;\n        int leftLeftHeight = leftLeft != null ? leftLeft.height : 0;\n\n        int leftDelta = leftLeftHeight - leftRightHeight;\n        if (leftDelta == 1 || (leftDelta == 0 && !insert)) {\n          rotateRight(node); // AVL left left\n        } else {\n          assert (leftDelta == -1);\n          rotateLeft(left); // AVL left right\n          rotateRight(node);\n        }\n        if (insert) {\n          break; // no further rotations will be necessary\n        }\n\n      } else if (delta == 0) {\n        node.height = leftHeight + 1; // leftHeight == rightHeight\n        if (insert) {\n          break; // the insert caused balance, so rebalancing is done!\n        }\n\n      } else {\n        assert (delta == -1 || delta == 1);\n        node.height = Math.max(leftHeight, rightHeight) + 1;\n        if (!insert) {\n          break; // the height hasn't changed, so rebalancing is done!\n        }\n      }\n    }\n  }\n\n  /** Rotates the subtree so that its root's right child is the new root. */\n  private void rotateLeft(Node<K, V> root) {\n    Node<K, V> left = root.left;\n    Node<K, V> pivot = root.right;\n    Node<K, V> pivotLeft = pivot.left;\n    Node<K, V> pivotRight = pivot.right;\n\n    // move the pivot's left child to the root's right\n    root.right = pivotLeft;\n    if (pivotLeft != null) {\n      pivotLeft.parent = root;\n    }\n\n    replaceInParent(root, pivot);\n\n    // move the root to the pivot's left\n    pivot.left = root;\n    root.parent = pivot;\n\n    // fix heights\n    root.height =\n        Math.max(left != null ? left.height : 0, pivotLeft != null ? pivotLeft.height : 0) + 1;\n    pivot.height = Math.max(root.height, pivotRight != null ? pivotRight.height : 0) + 1;\n  }\n\n  /** Rotates the subtree so that its root's left child is the new root. */\n  private void rotateRight(Node<K, V> root) {\n    Node<K, V> pivot = root.left;\n    Node<K, V> right = root.right;\n    Node<K, V> pivotLeft = pivot.left;\n    Node<K, V> pivotRight = pivot.right;\n\n    // move the pivot's right child to the root's left\n    root.left = pivotRight;\n    if (pivotRight != null) {\n      pivotRight.parent = root;\n    }\n\n    replaceInParent(root, pivot);\n\n    // move the root to the pivot's right\n    pivot.right = root;\n    root.parent = pivot;\n\n    // fixup heights\n    root.height =\n        Math.max(right != null ? right.height : 0, pivotRight != null ? pivotRight.height : 0) + 1;\n    pivot.height = Math.max(root.height, pivotLeft != null ? pivotLeft.height : 0) + 1;\n  }\n\n  private EntrySet entrySet;\n  private KeySet keySet;\n\n  @Override\n  public Set<Entry<K, V>> entrySet() {\n    EntrySet result = entrySet;\n    return result != null ? result : (entrySet = new EntrySet());\n  }\n\n  @Override\n  public Set<K> keySet() {\n    KeySet result = keySet;\n    return result != null ? result : (keySet = new KeySet());\n  }\n\n  static final class Node<K, V> implements Entry<K, V> {\n    Node<K, V> parent;\n    Node<K, V> left;\n    Node<K, V> right;\n    Node<K, V> next;\n    Node<K, V> prev;\n    final K key;\n    final boolean allowNullValue;\n    V value;\n    int height;\n\n    /** Create the header entry */\n    Node(boolean allowNullValue) {\n      key = null;\n      this.allowNullValue = allowNullValue;\n      next = prev = this;\n    }\n\n    /** Create a regular entry */\n    Node(boolean allowNullValue, Node<K, V> parent, K key, Node<K, V> next, Node<K, V> prev) {\n      this.parent = parent;\n      this.key = key;\n      this.allowNullValue = allowNullValue;\n      this.height = 1;\n      this.next = next;\n      this.prev = prev;\n      prev.next = this;\n      next.prev = this;\n    }\n\n    @Override\n    public K getKey() {\n      return key;\n    }\n\n    @Override\n    public V getValue() {\n      return value;\n    }\n\n    @Override\n    public V setValue(V value) {\n      if (value == null && !allowNullValue) {\n        throw new NullPointerException(\"value == null\");\n      }\n      V oldValue = this.value;\n      this.value = value;\n      return oldValue;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      if (o instanceof Entry) {\n        Entry<?, ?> other = (Entry<?, ?>) o;\n        return (key == null ? other.getKey() == null : key.equals(other.getKey()))\n            && (value == null ? other.getValue() == null : value.equals(other.getValue()));\n      }\n      return false;\n    }\n\n    @Override\n    public int hashCode() {\n      return (key == null ? 0 : key.hashCode()) ^ (value == null ? 0 : value.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      return key + \"=\" + value;\n    }\n\n    /** Returns the first node in this subtree. */\n    public Node<K, V> first() {\n      Node<K, V> node = this;\n      Node<K, V> child = node.left;\n      while (child != null) {\n        node = child;\n        child = node.left;\n      }\n      return node;\n    }\n\n    /** Returns the last node in this subtree. */\n    public Node<K, V> last() {\n      Node<K, V> node = this;\n      Node<K, V> child = node.right;\n      while (child != null) {\n        node = child;\n        child = node.right;\n      }\n      return node;\n    }\n  }\n\n  private abstract class LinkedTreeMapIterator<T> implements Iterator<T> {\n    Node<K, V> next = header.next;\n    Node<K, V> lastReturned = null;\n    int expectedModCount = modCount;\n\n    LinkedTreeMapIterator() {}\n\n    @Override\n    @SuppressWarnings(\"ReferenceEquality\")\n    public final boolean hasNext() {\n      return next != header;\n    }\n\n    @SuppressWarnings(\"ReferenceEquality\")\n    final Node<K, V> nextNode() {\n      Node<K, V> e = next;\n      if (e == header) {\n        throw new NoSuchElementException();\n      }\n      if (modCount != expectedModCount) {\n        throw new ConcurrentModificationException();\n      }\n      next = e.next;\n      return lastReturned = e;\n    }\n\n    @Override\n    public final void remove() {\n      if (lastReturned == null) {\n        throw new IllegalStateException();\n      }\n      removeInternal(lastReturned, true);\n      lastReturned = null;\n      expectedModCount = modCount;\n    }\n  }\n\n  class EntrySet extends AbstractSet<Entry<K, V>> {\n    @Override\n    public int size() {\n      return size;\n    }\n\n    @Override\n    public Iterator<Entry<K, V>> iterator() {\n      return new LinkedTreeMapIterator<Entry<K, V>>() {\n        @Override\n        public Entry<K, V> next() {\n          return nextNode();\n        }\n      };\n    }\n\n    @Override\n    public boolean contains(Object o) {\n      return o instanceof Entry && findByEntry((Entry<?, ?>) o) != null;\n    }\n\n    @Override\n    public boolean remove(Object o) {\n      if (!(o instanceof Entry)) {\n        return false;\n      }\n\n      Node<K, V> node = findByEntry((Entry<?, ?>) o);\n      if (node == null) {\n        return false;\n      }\n      removeInternal(node, true);\n      return true;\n    }\n\n    @Override\n    public void clear() {\n      LinkedTreeMap.this.clear();\n    }\n  }\n\n  final class KeySet extends AbstractSet<K> {\n    @Override\n    public int size() {\n      return size;\n    }\n\n    @Override\n    public Iterator<K> iterator() {\n      return new LinkedTreeMapIterator<K>() {\n        @Override\n        public K next() {\n          return nextNode().key;\n        }\n      };\n    }\n\n    @Override\n    public boolean contains(Object o) {\n      return containsKey(o);\n    }\n\n    @Override\n    public boolean remove(Object key) {\n      return removeInternalByKey(key) != null;\n    }\n\n    @Override\n    public void clear() {\n      LinkedTreeMap.this.clear();\n    }\n  }\n\n  /**\n   * If somebody is unlucky enough to have to serialize one of these, serialize it as a\n   * LinkedHashMap so that they won't need Gson on the other side to deserialize it. Using\n   * serialization defeats our DoS defence, so most apps shouldn't use it.\n   */\n  private Object writeReplace() throws ObjectStreamException {\n    return new LinkedHashMap<>(this);\n  }\n\n  private void readObject(ObjectInputStream in) throws IOException {\n    // Don't permit directly deserializing this class; writeReplace() should have written a\n    // replacement\n    throw new InvalidObjectException(\"Deserialization is unsupported\");\n  }\n}\n\n// Path: gson/src/main/java/com/google/gson/JsonObject.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.google.gson;\n\nimport com.google.errorprone.annotations.CanIgnoreReturnValue;\nimport com.google.gson.internal.LinkedTreeMap;\nimport java.util.Map;\nimport java.util.Set;\n\n/**\n * A class representing an object type in Json. An object consists of name-value pairs where names\n * are strings, and values are any other type of {@link JsonElement}. This allows for a creating a\n * tree of JsonElements. The member elements of this object are maintained in order they were added.\n * This class does not support {@code null} values. If {@code null} is provided as value argument to\n * any of the methods, it is converted to a {@link JsonNull}.\n *\n * <p>{@code JsonObject} does not implement the {@link Map} interface, but a {@code Map} view of it\n * can be obtained with {@link #asMap()}.\n *\n * @author Inderjeet Singh\n * @author Joel Leitch\n */\npublic final class JsonObject extends JsonElement {\n  private final LinkedTreeMap<String, JsonElement> members = new LinkedTreeMap<>(false);\n\n  /** Creates an empty JsonObject. */\n  @SuppressWarnings(\"deprecation\") // superclass constructor\n  public JsonObject() {}\n\n  /**\n   * Creates a deep copy of this element and all its children.\n   *\n   * @since 2.8.2\n   */\n  @Override\n  public JsonObject deepCopy() {\n    JsonObject result = new JsonObject();\n    for (Map.Entry<String, JsonElement> entry : members.entrySet()) {\n      result.add(entry.getKey(), entry.getValue().deepCopy());\n    }\n    return result;\n  }\n\n  /**\n   * Adds a member, which is a name-value pair, to self. The name must be a String, but the value\n   * can be an arbitrary {@link JsonElement}, thereby allowing you to build a full tree of\n   * JsonElements rooted at this node.\n   *\n   * @param property name of the member.\n   * @param value the member object.\n   */\n  public void add(String property, JsonElement value) {\n    members.put(property, value == null ? JsonNull.INSTANCE : value);\n  }\n\n  /**\n   * Removes the {@code property} from this object.\n   *\n   * @param property name of the member that should be removed.\n   * @return the {@link JsonElement} object that is being removed, or {@code null} if no member with\n   *     this name exists.\n   * @since 1.3\n   */\n  @CanIgnoreReturnValue\n  public JsonElement remove(String property) {\n    return members.remove(property);\n  }\n\n  /**\n   * Convenience method to add a string member. The specified value is converted to a {@link\n   * JsonPrimitive} of String.\n   *\n   * @param property name of the member.\n   * @param value the string value associated with the member.\n   */\n  public void addProperty(String property, String value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Convenience method to add a number member. The specified value is converted to a {@link\n   * JsonPrimitive} of Number.\n   *\n   * @param property name of the member.\n   * @param value the number value associated with the member.\n   */\n  public void addProperty(String property, Number value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Convenience method to add a boolean member. The specified value is converted to a {@link\n   * JsonPrimitive} of Boolean.\n   *\n   * @param property name of the member.\n   * @param value the boolean value associated with the member.\n   */\n  public void addProperty(String property, Boolean value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Convenience method to add a char member. The specified value is converted to a {@link\n   * JsonPrimitive} of Character.\n   *\n   * @param property name of the member.\n   * @param value the char value associated with the member.\n   */\n  public void addProperty(String property, Character value) {\n    add(property, value == null ? JsonNull.INSTANCE : new JsonPrimitive(value));\n  }\n\n  /**\n   * Returns a set of members of this object. The set is ordered, and the order is in which the\n   * elements were added.\n   *\n   * @return a set of members of this object.\n   */\n  public Set<Map.Entry<String, JsonElement>> entrySet() {\n    return members.entrySet();\n  }\n\n  /**\n   * Returns a set of members key values.\n   *\n   * @return a set of member keys as Strings\n   * @since 2.8.1\n   */\n  public Set<String> keySet() {\n    return members.keySet();\n  }\n\n  /**\n   * Returns the number of key/value pairs in the object.\n   *\n   * @return the number of key/value pairs in the object.\n   * @since 2.7\n   */\n  public int size() {\n    return members.size();\n  }\n\n  /**\n   * Returns true if the number of key/value pairs in the object is zero.\n   *\n   * @return true if the number of key/value pairs in the object is zero.\n   * @since 2.10.1\n   */\n  public boolean isEmpty() {\n    return members.size() == 0;\n  }\n\n  /**\n   * Convenience method to check if a member with the specified name is present in this object.\n   *\n   * @param memberName name of the member that is being checked for presence.\n   * @return true if there is a member with the specified name, false otherwise.\n   */\n  \npublic boolean has(String memberName) {\n    return members.containsKey(memberName);\n  }\n\n  /**\n   * Returns the member with the specified name.\n   *\n   * @param memberName name of the member that is being requested.\n   * @return the member matching the name, or {@code null} if no such member exists.\n   */\n  public JsonElement get(String memberName) {\n    return members.get(memberName);\n  }\n\n  /**\n   * Convenience method to get the specified member as a {@link JsonPrimitive}.\n   *\n   * @param memberName name of the member being requested.\n   * @return the {@code JsonPrimitive} corresponding to the specified member, or {@code null} if no\n   *     member with this name exists.\n   * @throws ClassCastException if the member is not of type {@code JsonPrimitive}.\n   */\n  public JsonPrimitive getAsJsonPrimitive(String memberName) {\n    return (JsonPrimitive) members.get(memberName);\n  }\n\n  /**\n   * Convenience method to get the specified member as a {@link JsonArray}.\n   *\n   * @param memberName name of the member being requested.\n   * @return the {@code JsonArray} corresponding to the specified member, or {@code null} if no\n   *     member with this name exists.\n   * @throws ClassCastException if the member is not of type {@code JsonArray}.\n   */\n  public JsonArray getAsJsonArray(String memberName) {\n    return (JsonArray) members.get(memberName);\n  }\n\n  /**\n   * Convenience method to get the specified member as a {@link JsonObject}.\n   *\n   * @param memberName name of the member being requested.\n   * @return the {@code JsonObject} corresponding to the specified member, or {@code null} if no\n   *     member with this name exists.\n   * @throws ClassCastException if the member is not of type {@code JsonObject}.\n   */\n  public JsonObject getAsJsonObject(String memberName) {\n    return (JsonObject) members.get(memberName);\n  }\n\n  /**\n   * Returns a mutable {@link Map} view of this {@code JsonObject}. Changes to the {@code Map} are\n   * visible in this {@code JsonObject} and the other way around.\n   *\n   * <p>The {@code Map} does not permit {@code null} keys or values. Unlike {@code JsonObject}'s\n   * {@code null} handling, a {@link NullPointerException} is thrown when trying to add {@code\n   * null}. Use {@link JsonNull} for JSON null values.\n   *\n   * @return mutable {@code Map} view\n   * @since 2.10\n   */\n  public Map<String, JsonElement> asMap() {\n    // It is safe to expose the underlying map because it disallows null keys and values\n    return members;\n  }\n\n  /**\n   * Returns whether the other object is equal to this. This method only considers the other object\n   * to be equal if it is an instance of {@code JsonObject} and has equal members, ignoring order.\n   */\n  @Override\n  public boolean equals(Object o) {\n    return (o == this) || (o instanceof JsonObject && ((JsonObject) o).members.equals(members));\n  }\n\n  /**\n   * Returns the hash code of this object. This method calculates the hash code based on the members\n...\n", "needle_token_start": 10374, "needle_token_end": 10397, "code_context_ntokens": 11195, "output": ["```java\n/**\n * Convenience method to check if a member with the specified name is present in this object.\n *\n * @param memberName name of the member that is being checked for presence.\n * @return true if there is a member with the specified name, false otherwise.\n */\npublic boolean has(String memberName) {\n  return members.containsKey(memberName);\n}\n```"]}
{"repo": "square/retrofit", "name": "getRawCall", "language": "java", "path": "retrofit/src/main/java/retrofit2/OkHttpCall.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve or initialize an HTTP call object, handling any errors that occurred during its creation or previous initializations.\n2. **Input**: There are no direct inputs as it uses the class's internal state.\n3. **Output**: Returns an initialized HTTP call object or throws an exception if the call object cannot be created or if an error was previously encountered.\n4. **Procedure**: \n   - Checks if the HTTP call object is already created and returns it if available.\n   - If not previously created, it checks for any stored errors from previous attempts to create the call object. If an error exists, it rethrows the appropriate exception.\n   - Attempts to create the HTTP call object. If successful, it stores and returns this object. If an error occurs during creation, it stores the error for future reference and rethrows it.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.throwIfFatal;\n\nimport java.io.IOException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport javax.annotation.concurrent.GuardedBy;\nimport okhttp3.MediaType;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\nimport okio.BufferedSource;\nimport okio.ForwardingSource;\nimport okio.Okio;\nimport okio.Timeout;\n\nfinal class OkHttpCall<T> implements Call<T> {\n  private final RequestFactory requestFactory;\n  private final Object instance;\n  private final Object[] args;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, T> responseConverter;\n\n  private volatile boolean canceled;\n\n  @GuardedBy(\"this\")\n  private @Nullable okhttp3.Call rawCall;\n\n  @GuardedBy(\"this\") // Either a RuntimeException, non-fatal Error, or IOException.\n  private @Nullable Throwable creationFailure;\n\n  @GuardedBy(\"this\")\n  private boolean executed;\n\n  OkHttpCall(\n      RequestFactory requestFactory,\n      Object instance,\n      Object[] args,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, T> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.instance = instance;\n    this.args = args;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @SuppressWarnings(\"CloneDoesntCallSuperClone\") // We are a final type & this saves clearing state.\n  @Override\n  public OkHttpCall<T> clone() {\n    return new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n  }\n\n  @Override\n  public synchronized Request request() {\n    try {\n      return getRawCall().request();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create request.\", e);\n    }\n  }\n\n  @Override\n  public synchronized Timeout timeout() {\n    try {\n      return getRawCall().timeout();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create call.\", e);\n    }\n  }\n\n  /**\n   * Returns the raw call, initializing it if necessary. Throws if initializing the raw call throws,\n   * or has thrown in previous attempts to create it.\n   */\n  \n@GuardedBy(\"this\")\n  private okhttp3.Call getRawCall() throws IOException {\n    okhttp3.Call call = rawCall;\n    if (call != null) return call;\n\n    // Re-throw previous failures if this isn't the first attempt.\n    if (creationFailure != null) {\n      if (creationFailure instanceof IOException) {\n        throw (IOException) creationFailure;\n      } else if (creationFailure instanceof RuntimeException) {\n        throw (RuntimeException) creationFailure;\n      } else {\n        throw (Error) creationFailure;\n      }\n    }\n\n    // Create and remember either the success or the failure.\n    try {\n      return rawCall = createRawCall();\n    } catch (RuntimeException | Error | IOException e) {\n      throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n      creationFailure = e;\n      throw e;\n    }\n  }\n\n  @Override\n  public void enqueue(final Callback<T> callback) {\n    Objects.requireNonNull(callback, \"callback == null\");\n\n    okhttp3.Call call;\n    Throwable failure;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = rawCall;\n      failure = creationFailure;\n      if (call == null && failure == null) {\n        try {\n          call = rawCall = createRawCall();\n        } catch (Throwable t) {\n          throwIfFatal(t);\n          failure = creationFailure = t;\n        }\n      }\n    }\n\n    if (failure != null) {\n      callback.onFailure(this, failure);\n      return;\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    call.enqueue(\n        new okhttp3.Callback() {\n          @Override\n          public void onResponse(okhttp3.Call call, okhttp3.Response rawResponse) {\n            Response<T> response;\n            try {\n              response = parseResponse(rawResponse);\n            } catch (Throwable e) {\n              throwIfFatal(e);\n              callFailure(e);\n              return;\n            }\n\n            try {\n              callback.onResponse(OkHttpCall.this, response);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n\n          @Override\n          public void onFailure(okhttp3.Call call, IOException e) {\n            callFailure(e);\n          }\n\n          private void callFailure(Throwable e) {\n            try {\n              callback.onFailure(OkHttpCall.this, e);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n        });\n  }\n\n  @Override\n  public synchronized boolean isExecuted() {\n    return executed;\n  }\n\n  @Override\n  public Response<T> execute() throws IOException {\n    okhttp3.Call call;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = getRawCall();\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    return parseResponse(call.execute());\n  }\n\n  private okhttp3.Call createRawCall() throws IOException {\n    okhttp3.Call call = callFactory.newCall(requestFactory.create(instance, args));\n    if (call == null) {\n      throw new NullPointerException(\"Call.Factory returned null.\");\n    }\n    return call;\n  }\n\n  Response<T> parseResponse(okhttp3.Response rawResponse) throws IOException {\n    ResponseBody rawBody = rawResponse.body();\n\n    // Remove the body's source (the only stateful object) so we can pass the response along.\n    rawResponse =\n        rawResponse\n            .newBuilder()\n            .body(new NoContentResponseBody(rawBody.contentType(), rawBody.contentLength()))\n            .build();\n\n    int code = rawResponse.code();\n    if (code < 200 || code >= 300) {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        ResponseBody bufferedBody = Utils.buffer(rawBody);\n        return Response.error(bufferedBody, rawResponse);\n      } finally {\n        rawBody.close();\n      }\n    }\n\n    if (code == 204 || code == 205) {\n      rawBody.close();\n      return Response.success(null, rawResponse);\n    }\n\n    ExceptionCatchingResponseBody catchingBody = new ExceptionCatchingResponseBody(rawBody);\n    try {\n      T body = responseConverter.convert(catchingBody);\n      return Response.success(body, rawResponse);\n    } catch (RuntimeException e) {\n      // If the underlying source threw an exception, propagate that rather than indicating it was\n      // a runtime exception.\n      catchingBody.throwIfCaught();\n      throw e;\n    }\n  }\n\n  @Override\n  public void cancel() {\n    canceled = true;\n\n    okhttp3.Call call;\n    synchronized (this) {\n      call = rawCall;\n    }\n    if (call != null) {\n      call.cancel();\n    }\n  }\n\n  @Override\n  public boolean isCanceled() {\n    if (canceled) {\n      return true;\n    }\n    synchronized (this) {\n      return rawCall != null && rawCall.isCanceled();\n    }\n  }\n\n  static final class NoContentResponseBody extends ResponseBody {\n    private final @Nullable MediaType contentType;\n    private final long contentLength;\n\n    NoContentResponseBody(@Nullable MediaType contentType, long contentLength) {\n      this.contentType = contentType;\n      this.contentLength = contentLength;\n    }\n\n    @Override\n    public MediaType contentType() {\n      return contentType;\n    }\n\n    @Override\n    public long contentLength() {\n      return contentLength;\n    }\n\n    @Override\n    public BufferedSource source() {\n      throw new IllegalStateException(\"Cannot read raw response body of a converted body.\");\n    }\n  }\n\n  static final class ExceptionCatchingResponseBody extends ResponseBody {\n    private final ResponseBody delegate;\n    private final BufferedSource delegateSource;\n    @Nullable IOException thrownException;\n\n    ExceptionCatchingResponseBody(ResponseBody delegate) {\n      this.delegate = delegate;\n      this.delegateSource =\n          Okio.buffer(\n              new ForwardingSource(delegate.source()) {\n                @Override\n                public long read(Buffer sink, long byteCount) throws IOException {\n                  try {\n                    return super.read(sink, byteCount);\n                  } catch (IOException e) {\n                    thrownException = e;\n                    throw e;\n                  }\n                }\n              });\n    }\n\n    @Override\n    public MediaType contentType() {\n      return delegate.contentType();\n    }\n\n    @Override\n    public long contentLength() {\n      return delegate.contentLength();\n    }\n\n    @Override\n    public BufferedSource source() {\n      return delegateSource;\n    }\n\n    @Override\n    public void close() {\n      delegate.close();\n    }\n\n    void throwIfCaught() throws IOException {\n      if (thrownException != null) {\n        throw thrownException;\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Streaming.java\n/*\n * Copyright (C) 2014 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.ResponseBody;\n\n/**\n * Treat the response body on methods returning {@link ResponseBody ResponseBody} as is, i.e.\n * without converting the body to {@code byte[]}.\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface Streaming {}\n\n// Path: retrofit/src/main/java/retrofit2/BuiltInConverters.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.Streaming;\n\nfinal class BuiltInConverters extends Converter.Factory {\n\n  @Override\n  public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n      Type type, Annotation[] annotations, Retrofit retrofit) {\n    if (type == ResponseBody.class) {\n      return Utils.isAnnotationPresent(annotations, Streaming.class)\n          ? StreamingResponseBodyConverter.INSTANCE\n          : BufferingResponseBodyConverter.INSTANCE;\n    }\n    if (type == Void.class) {\n      return VoidResponseBodyConverter.INSTANCE;\n    }\n    if (Utils.isUnit(type)) {\n      return UnitResponseBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  @Override\n  public @Nullable Converter<?, RequestBody> requestBodyConverter(\n      Type type,\n      Annotation[] parameterAnnotations,\n      Annotation[] methodAnnotations,\n      Retrofit retrofit) {\n    if (RequestBody.class.isAssignableFrom(Utils.getRawType(type))) {\n      return RequestBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  static final class VoidResponseBodyConverter implements Converter<ResponseBody, Void> {\n    static final VoidResponseBodyConverter INSTANCE = new VoidResponseBodyConverter();\n\n    @Override\n    public Void convert(ResponseBody value) {\n      value.close();\n      return null;\n    }\n  }\n\n  static final class UnitResponseBodyConverter implements Converter<ResponseBody, Unit> {\n    static final UnitResponseBodyConverter INSTANCE = new UnitResponseBodyConverter();\n\n    @Override\n    public Unit convert(ResponseBody value) {\n      value.close();\n      return Unit.INSTANCE;\n    }\n  }\n\n  static final class RequestBodyConverter implements Converter<RequestBody, RequestBody> {\n    static final RequestBodyConverter INSTANCE = new RequestBodyConverter();\n\n    @Override\n    public RequestBody convert(RequestBody value) {\n      return value;\n    }\n  }\n\n  static final class StreamingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final StreamingResponseBodyConverter INSTANCE = new StreamingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) {\n      return value;\n    }\n  }\n\n  static final class BufferingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final BufferingResponseBodyConverter INSTANCE = new BufferingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) throws IOException {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        return Utils.buffer(value);\n      } finally {\n        value.close();\n      }\n    }\n  }\n\n  static final class ToStringConverter implements Converter<Object, String> {\n    static final ToStringConverter INSTANCE = new ToStringConverter();\n\n    @Override\n    public String convert(Object value) {\n      return value.toString();\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/SkipCallbackExecutorImpl.java\n/*\n * Copyright (C) 2019 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.annotation.Annotation;\n\n// This class conforms to the annotation requirements documented on Annotation.\nfinal class SkipCallbackExecutorImpl implements SkipCallbackExecutor {\n  private static final SkipCallbackExecutor INSTANCE = new SkipCallbackExecutorImpl();\n\n  static Annotation[] ensurePresent(Annotation[] annotations) {\n    if (Utils.isAnnotationPresent(annotations, SkipCallbackExecutor.class)) {\n      return annotations;\n    }\n\n    Annotation[] newAnnotations = new Annotation[annotations.length + 1];\n    // Place the skip annotation first since we're guaranteed to check for it in the call adapter.\n    newAnnotations[0] = SkipCallbackExecutorImpl.INSTANCE;\n    System.arraycopy(annotations, 0, newAnnotations, 1, annotations.length);\n    return newAnnotations;\n  }\n\n  @Override\n  public Class<? extends Annotation> annotationType() {\n    return SkipCallbackExecutor.class;\n  }\n\n  @Override\n  public boolean equals(Object obj) {\n    return obj instanceof SkipCallbackExecutor;\n  }\n\n  @Override\n  public int hashCode() {\n    return 0;\n  }\n\n  @Override\n  public String toString() {\n    return \"@\" + SkipCallbackExecutor.class.getName() + \"()\";\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/HttpServiceMethod.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.getRawType;\nimport static retrofit2.Utils.methodError;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport kotlin.coroutines.Continuation;\nimport okhttp3.ResponseBody;\n\n/** Adapts an invocation of an interface method into an HTTP call. */\nabstract class HttpServiceMethod<ResponseT, ReturnT> extends ServiceMethod<ReturnT> {\n  /**\n   * Inspects the annotations on an interface method to construct a reusable service method that\n   * speaks HTTP. This requires potentially-expensive reflection so it is best to build each service\n   * method only once and reuse it.\n   */\n  static <ResponseT, ReturnT> HttpServiceMethod<ResponseT, ReturnT> parseAnnotations(\n      Retrofit retrofit, Method method, RequestFactory requestFactory) {\n    boolean isKotlinSuspendFunction = requestFactory.isKotlinSuspendFunction;\n    boolean continuationWantsResponse = false;\n    boolean continuationBodyNullable = false;\n    boolean continuationIsUnit = false;\n\n    Annotation[] annotations = method.getAnnotations();\n    Type adapterType;\n    if (isKotlinSuspendFunction) {\n      Type[] parameterTypes = method.getGenericParameterTypes();\n      Type responseType =\n          Utils.getParameterLowerBound(\n              0, (ParameterizedType) parameterTypes[parameterTypes.length - 1]);\n      if (getRawType(responseType) == Response.class && responseType instanceof ParameterizedType) {\n        // Unwrap the actual body type from Response<T>.\n        responseType = Utils.getParameterUpperBound(0, (ParameterizedType) responseType);\n        continuationWantsResponse = true;\n      } else {\n        if (getRawType(responseType) == Call.class) {\n          throw methodError(\n              method,\n              \"Suspend functions should not return Call, as they already execute asynchronously.\\n\"\n                  + \"Change its return type to %s\",\n              Utils.getParameterUpperBound(0, (ParameterizedType) responseType));\n        }\n\n        continuationIsUnit = Utils.isUnit(responseType);\n        // TODO figure out if type is nullable or not\n        // Metadata metadata = method.getDeclaringClass().getAnnotation(Metadata.class)\n        // Find the entry for method\n        // Determine if return type is nullable or not\n      }\n\n      adapterType = new Utils.ParameterizedTypeImpl(null, Call.class, responseType);\n      annotations = SkipCallbackExecutorImpl.ensurePresent(annotations);\n    } else {\n      adapterType = method.getGenericReturnType();\n    }\n\n    CallAdapter<ResponseT, ReturnT> callAdapter =\n        createCallAdapter(retrofit, method, adapterType, annotations);\n    Type responseType = callAdapter.responseType();\n    if (responseType == okhttp3.Response.class) {\n      throw methodError(\n          method,\n          \"'\"\n              + getRawType(responseType).getName()\n              + \"' is not a valid response body type. Did you mean ResponseBody?\");\n    }\n    if (responseType == Response.class) {\n      throw methodError(method, \"Response must include generic type (e.g., Response<String>)\");\n    }\n    if (requestFactory.httpMethod.equals(\"HEAD\")\n        && !Void.class.equals(responseType)\n        && !Utils.isUnit(responseType)) {\n      throw methodError(method, \"HEAD method must use Void or Unit as response type.\");\n    }\n\n    Converter<ResponseBody, ResponseT> responseConverter =\n        createResponseConverter(retrofit, method, responseType);\n\n    okhttp3.Call.Factory callFactory = retrofit.callFactory;\n    if (!isKotlinSuspendFunction) {\n      return new CallAdapted<>(requestFactory, callFactory, responseConverter, callAdapter);\n    } else if (continuationWantsResponse) {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n          new SuspendForResponse<>(\n              requestFactory,\n              callFactory,\n              responseConverter,\n              (CallAdapter<ResponseT, Call<ResponseT>>) callAdapter);\n    } else {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n          new SuspendForBody<>(\n              requestFactory,\n              callFactory,\n              responseConverter,\n              (CallAdapter<ResponseT, Call<ResponseT>>) callAdapter,\n              continuationBodyNullable,\n              continuationIsUnit);\n    }\n  }\n\n  private static <ResponseT, ReturnT> CallAdapter<ResponseT, ReturnT> createCallAdapter(\n      Retrofit retrofit, Method method, Type returnType, Annotation[] annotations) {\n    try {\n      //noinspection unchecked\n      return (CallAdapter<ResponseT, ReturnT>) retrofit.callAdapter(returnType, annotations);\n    } catch (RuntimeException e) { // Wide exception range because factories are user code.\n      throw methodError(method, e, \"Unable to create call adapter for %s\", returnType);\n    }\n  }\n\n  private static <ResponseT> Converter<ResponseBody, ResponseT> createResponseConverter(\n      Retrofit retrofit, Method method, Type responseType) {\n    Annotation[] annotations = method.getAnnotations();\n    try {\n      return retrofit.responseBodyConverter(responseType, annotations);\n    } catch (RuntimeException e) { // Wide exception range because factories are user code.\n      throw methodError(method, e, \"Unable to create converter for %s\", responseType);\n    }\n  }\n\n  private final RequestFactory requestFactory;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, ResponseT> responseConverter;\n\n  HttpServiceMethod(\n      RequestFactory requestFactory,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, ResponseT> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @Override\n  final @Nullable ReturnT invoke(Object instance, Object[] args) {\n    Call<ResponseT> call =\n        new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n    return adapt(call, args);\n  }\n\n  protected abstract @Nullable ReturnT adapt(Call<ResponseT> call, Object[] args);\n\n  static final class CallAdapted<ResponseT, ReturnT> extends HttpServiceMethod<ResponseT, ReturnT> {\n    private final CallAdapter<ResponseT, ReturnT> callAdapter;\n\n    CallAdapted(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, ReturnT> callAdapter) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n    }\n\n    @Override\n    protected ReturnT adapt(Call<ResponseT> call, Object[] args) {\n      return callAdapter.adapt(call);\n    }\n  }\n\n  static final class SuspendForResponse<ResponseT> extends HttpServiceMethod<ResponseT, Object> {\n    private final CallAdapter<ResponseT, Call<ResponseT>> callAdapter;\n\n    SuspendForResponse(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, Call<ResponseT>> callAdapter) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n    }\n\n    @Override\n    protected Object adapt(Call<ResponseT> call, Object[] args) {\n      call = callAdapter.adapt(call);\n\n      //noinspection unchecked Checked by reflection inside RequestFactory.\n      Continuation<Response<ResponseT>> continuation =\n          (Continuation<Response<ResponseT>>) args[args.length - 1];\n\n      // See SuspendForBody for explanation about this try/catch.\n      try {\n        return KotlinExtensions.awaitResponse(call, continuation);\n      } catch (Exception e) {\n        return KotlinExtensions.suspendAndThrow(e, continuation);\n      }\n    }\n  }\n\n  static final class SuspendForBody<ResponseT> extends HttpServiceMethod<ResponseT, Object> {\n    private final CallAdapter<ResponseT, Call<ResponseT>> callAdapter;\n    private final boolean isNullable;\n    private final boolean isUnit;\n\n    SuspendForBody(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, Call<ResponseT>> callAdapter,\n        boolean isNullable,\n        boolean isUnit) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n      this.isNullable = isNullable;\n      this.isUnit = isUnit;\n    }\n\n    @Override\n    protected Object adapt(Call<ResponseT> call, Object[] args) {\n      call = callAdapter.adapt(call);\n\n      //noinspection unchecked Checked by reflection inside RequestFactory.\n      Continuation<ResponseT> continuation = (Continuation<ResponseT>) args[args.length - 1];\n\n      // Calls to OkHttp Call.enqueue() like those inside await and awaitNullable can sometimes\n      // invoke the supplied callback with an exception before the invoking stack frame can return.\n      // Coroutines will intercept the subsequent invocation of the Continuation and throw the\n      // exception synchronously. A Java Proxy cannot throw checked exceptions without them being\n      // declared on the interface method. To avoid the synchronous checked exception being wrapped\n      // in an UndeclaredThrowableException, it is intercepted and supplied to a helper which will\n      // force suspension to occur so that it can be instead delivered to the continuation to\n      // bypass this restriction.\n      try {\n        if (isUnit) {\n          //noinspection unchecked Checked by isUnit\n          return KotlinExtensions.awaitUnit((Call<Unit>) call, (Continuation<Unit>) continuation);\n        } else if (isNullable) {\n          return KotlinExtensions.awaitNullable(call, continuation);\n        } else {\n          return KotlinExtensions.await(call, continuation);\n        }\n      } catch (VirtualMachineError | ThreadDeath | LinkageError e) {\n        // Do not attempt to capture fatal throwables. This list is derived RxJava's `throwIfFatal`.\n        throw e;\n      } catch (Throwable e) {\n        return KotlinExtensions.suspendAndThrow(e, continuation);\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/ParameterHandler.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.Method;\nimport java.util.Map;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.MultipartBody;\nimport okhttp3.RequestBody;\n\nabstract class ParameterHandler<T> {\n  abstract void apply(RequestBuilder builder, @Nullable T value) throws IOException;\n\n  final ParameterHandler<Iterable<T>> iterable() {\n    return new ParameterHandler<Iterable<T>>() {\n      @Override\n      void apply(RequestBuilder builder, @Nullable Iterable<T> values) throws IOException {\n        if (values == null) return; // Skip null values.\n\n        for (T value : values) {\n          ParameterHandler.this.apply(builder, value);\n        }\n      }\n    };\n  }\n\n  final ParameterHandler<Object> array() {\n    return new ParameterHandler<Object>() {\n      @Override\n      void apply(RequestBuilder builder, @Nullable Object values) throws IOException {\n        if (values == null) return; // Skip null values.\n\n        for (int i = 0, size = Array.getLength(values); i < size; i++) {\n          //noinspection unchecked\n          ParameterHandler.this.apply(builder, (T) Array.get(values, i));\n        }\n      }\n    };\n  }\n\n  static final class RelativeUrl extends ParameterHandler<Object> {\n    private final Method method;\n    private final int p;\n\n    RelativeUrl(Method method, int p) {\n      this.method = method;\n      this.p = p;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Object value) {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"@Url parameter is null.\");\n      }\n      builder.setRelativeUrl(value);\n    }\n  }\n\n  static final class Header<T> extends ParameterHandler<T> {\n    private final String name;\n    private final Converter<T, String> valueConverter;\n    private final boolean allowUnsafeNonAsciiValues;\n\n    Header(String name, Converter<T, String> valueConverter, boolean allowUnsafeNonAsciiValues) {\n      this.name = Objects.requireNonNull(name, \"name == null\");\n      this.valueConverter = valueConverter;\n      this.allowUnsafeNonAsciiValues = allowUnsafeNonAsciiValues;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) return; // Skip null values.\n\n      String headerValue = valueConverter.convert(value);\n      if (headerValue == null) return; // Skip converted but null values.\n\n      builder.addHeader(name, headerValue, allowUnsafeNonAsciiValues);\n    }\n  }\n\n  static final class Path<T> extends ParameterHandler<T> {\n    private final Method method;\n    private final int p;\n    private final String name;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    Path(Method method, int p, String name, Converter<T, String> valueConverter, boolean encoded) {\n      this.method = method;\n      this.p = p;\n      this.name = Objects.requireNonNull(name, \"name == null\");\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(\n            method, p, \"Path parameter \\\"\" + name + \"\\\" value must not be null.\");\n      }\n      builder.addPathParam(name, valueConverter.convert(value), encoded);\n    }\n  }\n\n  static final class Query<T> extends ParameterHandler<T> {\n    private final String name;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    Query(String name, Converter<T, String> valueConverter, boolean encoded) {\n      this.name = Objects.requireNonNull(name, \"name == null\");\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) return; // Skip null values.\n\n      String queryValue = valueConverter.convert(value);\n      if (queryValue == null) return; // Skip converted but null values\n\n      builder.addQueryParam(name, queryValue, encoded);\n    }\n  }\n\n  static final class QueryName<T> extends ParameterHandler<T> {\n    private final Converter<T, String> nameConverter;\n    private final boolean encoded;\n\n    QueryName(Converter<T, String> nameConverter, boolean encoded) {\n      this.nameConverter = nameConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) return; // Skip null values.\n      builder.addQueryParam(nameConverter.convert(value), null, encoded);\n    }\n  }\n\n  static final class QueryMap<T> extends ParameterHandler<Map<String, T>> {\n    private final Method method;\n    private final int p;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    QueryMap(Method method, int p, Converter<T, String> valueConverter, boolean encoded) {\n      this.method = method;\n      this.p = p;\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Map<String, T> value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"Query map was null\");\n      }\n\n      for (Map.Entry<String, T> entry : value.entrySet()) {\n        String entryKey = entry.getKey();\n        if (entryKey == null) {\n          throw Utils.parameterError(method, p, \"Query map contained null key.\");\n        }\n        T entryValue = entry.getValue();\n        if (entryValue == null) {\n          throw Utils.parameterError(\n              method, p, \"Query map contained null value for key '\" + entryKey + \"'.\");\n        }\n\n        String convertedEntryValue = valueConverter.convert(entryValue);\n        if (convertedEntryValue == null) {\n          throw Utils.parameterError(\n              method,\n              p,\n              \"Query map value '\"\n                  + entryValue\n                  + \"' converted to null by \"\n                  + valueConverter.getClass().getName()\n                  + \" for key '\"\n                  + entryKey\n                  + \"'.\");\n        }\n\n        builder.addQueryParam(entryKey, convertedEntryValue, encoded);\n      }\n    }\n  }\n\n  static final class HeaderMap<T> extends ParameterHandler<Map<String, T>> {\n    private final Method method;\n    private final int p;\n    private final Converter<T, String> valueConverter;\n    private final boolean allowUnsafeNonAsciiValues;\n\n    HeaderMap(\n        Method method,\n        int p,\n        Converter<T, String> valueConverter,\n        boolean allowUnsafeNonAsciiValues) {\n      this.method = method;\n      this.p = p;\n      this.valueConverter = valueConverter;\n      this.allowUnsafeNonAsciiValues = allowUnsafeNonAsciiValues;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Map<String, T> value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"Header map was null.\");\n      }\n\n      for (Map.Entry<String, T> entry : value.entrySet()) {\n        String headerName = entry.getKey();\n        if (headerName == null) {\n          throw Utils.parameterError(method, p, \"Header map contained null key.\");\n        }\n        T headerValue = entry.getValue();\n        if (headerValue == null) {\n          throw Utils.parameterError(\n              method, p, \"Header map contained null value for key '\" + headerName + \"'.\");\n        }\n        builder.addHeader(\n            headerName, valueConverter.convert(headerValue), allowUnsafeNonAsciiValues);\n      }\n    }\n  }\n\n  static final class Headers extends ParameterHandler<okhttp3.Headers> {\n    private final Method method;\n    private final int p;\n\n    Headers(Method method, int p) {\n      this.method = method;\n      this.p = p;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable okhttp3.Headers headers) {\n      if (headers == null) {\n        throw Utils.parameterError(method, p, \"Headers parameter must not be null.\");\n      }\n      builder.addHeaders(headers);\n    }\n  }\n\n  static final class Field<T> extends ParameterHandler<T> {\n    private final String name;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    Field(String name, Converter<T, String> valueConverter, boolean encoded) {\n      this.name = Objects.requireNonNull(name, \"name == null\");\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) return; // Skip null values.\n\n      String fieldValue = valueConverter.convert(value);\n      if (fieldValue == null) return; // Skip null converted values\n\n      builder.addFormField(name, fieldValue, encoded);\n    }\n  }\n\n  static final class FieldMap<T> extends ParameterHandler<Map<String, T>> {\n    private final Method method;\n    private final int p;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    FieldMap(Method method, int p, Converter<T, String> valueConverter, boolean encoded) {\n      this.method = method;\n      this.p = p;\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Map<String, T> value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"Field map was null.\");\n      }\n\n      for (Map.Entry<String, T> entry : value.entrySet()) {\n        String entryKey = entry.getKey();\n        if (entryKey == null) {\n          throw Utils.parameterError(method, p, \"Field map contained null key.\");\n        }\n        T entryValue = entry.getValue();\n        if (entryValue == null) {\n          throw Utils.parameterError(\n              method, p, \"Field map contained null value for key '\" + entryKey + \"'.\");\n        }\n\n        String fieldEntry = valueConverter.convert(entryValue);\n        if (fieldEntry == null) {\n          throw Utils.parameterError(\n              method,\n              p,\n              \"Field map value '\"\n                  + entryValue\n                  + \"' converted to null by \"\n                  + valueConverter.getClass().getName()\n                  + \" for key '\"\n                  + entryKey\n                  + \"'.\");\n        }\n\n        builder.addFormField(entryKey, fieldEntry, encoded);\n      }\n    }\n  }\n\n  static final class Part<T> extends ParameterHandler<T> {\n    private final Method method;\n    private final int p;\n    private final okhttp3.Headers headers;\n    private final Converter<T, RequestBody> converter;\n\n    Part(Method method, int p, okhttp3.Headers headers, Converter<T, RequestBody> converter) {\n      this.method = method;\n      this.p = p;\n      this.headers = headers;\n      this.converter = converter;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) {\n      if (value == null) return; // Skip null values.\n\n      RequestBody body;\n      try {\n        body = converter.convert(value);\n      } catch (IOException e) {\n        throw Utils.parameterError(method, p, \"Unable to convert \" + value + \" to RequestBody\", e);\n      }\n      builder.addPart(headers, body);\n    }\n  }\n\n  static final class RawPart extends ParameterHandler<MultipartBody.Part> {\n    static final RawPart INSTANCE = new RawPart();\n\n    private RawPart() {}\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable MultipartBody.Part value) {\n      if (value != null) { // Skip null values.\n        builder.addPart(value);\n      }\n    }\n  }\n\n  static final class PartMap<T> extends ParameterHandler<Map<String, T>> {\n    private final Method method;\n    private final int p;\n    private final Converter<T, RequestBody> valueConverter;\n    private final String transferEncoding;\n\n    PartMap(\n        Method method, int p, Converter<T, RequestBody> valueConverter, String transferEncoding) {\n      this.method = method;\n      this.p = p;\n      this.valueConverter = valueConverter;\n      this.transferEncoding = transferEncoding;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Map<String, T> value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"Part map was null.\");\n      }\n\n      for (Map.Entry<String, T> entry : value.entrySet()) {\n        String entryKey = entry.getKey();\n        if (entryKey == null) {\n          throw Utils.parameterError(method, p, \"Part map contained null key.\");\n        }\n        T entryValue = entry.getValue();\n        if (entryValue == null) {\n          throw Utils.parameterError(\n              method, p, \"Part map contained null value for key '\" + entryKey + \"'.\");\n        }\n\n        okhttp3.Headers headers =\n            okhttp3.Headers.of(\n                \"Content-Disposition\",\n                \"form-data; name=\\\"\" + entryKey + \"\\\"\",\n                \"Content-Transfer-Encoding\",\n                transferEncoding);\n\n        builder.addPart(headers, valueConverter.convert(entryValue));\n      }\n    }\n  }\n\n  static final class Body<T> extends ParameterHandler<T> {\n    private final Method method;\n    private final int p;\n    private final Converter<T, RequestBody> converter;\n\n    Body(Method method, int p, Converter<T, RequestBody> converter) {\n      this.method = method;\n      this.p = p;\n      this.converter = converter;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"Body parameter value must not be null.\");\n      }\n      RequestBody body;\n      try {\n        body = converter.convert(value);\n      } catch (IOException e) {\n        throw Utils.parameterError(method, e, p, \"Unable to convert \" + value + \" to RequestBody\");\n      }\n      builder.setBody(body);\n    }\n  }\n\n  static final class Tag<T> extends ParameterHandler<T> {\n    final Class<T> cls;\n\n    Tag(Class<T> cls) {\n      this.cls = cls;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) {\n      builder.addTag(cls, value);\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/DELETE.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.HttpUrl;\n\n/** Make a DELETE request. */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface DELETE {\n  /**\n   * A relative or absolute path, or full URL of the endpoint. This value is optional if the first\n   * parameter of the method is annotated with {@link Url @Url}.\n   *\n   * <p>See {@linkplain retrofit2.Retrofit.Builder#baseUrl(HttpUrl) base URL} for details of how\n   * this is resolved against a base URL to create the full endpoint URL.\n   */\n  String value() default \"\";\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Field.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.PARAMETER;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport java.lang.reflect.Type;\nimport retrofit2.Retrofit;\n\n/**\n * Named pair for a form-encoded request.\n *\n * <p>Values are converted to strings using {@link Retrofit#stringConverter(Type, Annotation[])} (or\n * {@link Object#toString()}, if no matching string converter is installed) and then form URL\n * encoded. {@code null} values are ignored. Passing a {@link java.util.List List} or array will\n * result in a field pair for each non-{@code null} item.\n *\n * <p>Simple Example:\n *\n * <pre><code>\n * &#64;FormUrlEncoded\n * &#64;POST(\"/\")\n * Call&lt;ResponseBody&gt; example(\n *     &#64;Field(\"name\") String name,\n *     &#64;Field(\"occupation\") String occupation);\n * </code></pre>\n *\n * Calling with {@code foo.example(\"Bob Smith\", \"President\")} yields a request body of {@code\n * name=Bob+Smith&occupation=President}.\n *\n * <p>Array/Varargs Example:\n *\n * <pre><code>\n * &#64;FormUrlEncoded\n * &#64;POST(\"/list\")\n * Call&lt;ResponseBody&gt; example(@Field(\"name\") String... names);\n * </code></pre>\n *\n * Calling with {@code foo.example(\"Bob Smith\", \"Jane Doe\")} yields a request body of {@code\n * name=Bob+Smith&name=Jane+Doe}.\n *\n * @see FormUrlEncoded\n * @see FieldMap\n */\n@Documented\n@Target(PARAMETER)\n@Retention(RUNTIME)\npublic @interface Field {\n  String value();\n\n  /** Specifies whether the {@linkplain #value() name} and value are already URL encoded. */\n  boolean encoded() default false;\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/FieldMap.java\n/*\n * Copyright (C) 2014 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.PARAMETER;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\n\n/**\n * Named key/value pairs for a form-encoded request.\n *\n * <p>Simple Example:\n *\n * <pre><code>\n * &#64;FormUrlEncoded\n * &#64;POST(\"/things\")\n * Call&lt;ResponseBody&gt; things(@FieldMap Map&lt;String, String&gt; fields);\n * </code></pre>\n *\n * Calling with {@code foo.things(ImmutableMap.of(\"foo\", \"bar\", \"kit\", \"kat\")} yields a request body\n * of {@code foo=bar&kit=kat}.\n *\n * <p>A {@code null} value for the map, as a key, or as a value is not allowed.\n *\n * @see FormUrlEncoded\n * @see Field\n */\n@Documented\n@Target(PARAMETER)\n@Retention(RUNTIME)\npublic @interface FieldMap {\n  /** Specifies whether the names and values are already URL encoded. */\n  boolean encoded() default false;\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/FormUrlEncoded.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\n\n/**\n * Denotes that the request body will use form URL encoding. Fields should be declared as parameters\n * and annotated with {@link Field @Field}.\n *\n * <p>Requests made with this annotation will have {@code application/x-www-form-urlencoded} MIME\n * type. Field names and values will be UTF-8 encoded before being URI-encoded in accordance to <a\n * href=\"https://datatracker.ietf.org/doc/html/rfc3986\">RFC-3986</a>.\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface FormUrlEncoded {}\n\n// Path: retrofit/src/main/java/retrofit2/http/GET.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.HttpUrl;\n\n/** Make a GET request. */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface GET {\n  /**\n   * A relative or absolute path, or full URL of the endpoint. This value is optional if the first\n   * parameter of the method is annotated with {@link Url @Url}.\n   *\n   * <p>See {@linkplain retrofit2.Retrofit.Builder#baseUrl(HttpUrl) base URL} for details of how\n   * this is resolved against a base URL to create the full endpoint URL.\n   */\n  String value() default \"\";\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/HEAD.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.HttpUrl;\n\n/** Make a HEAD request. */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface HEAD {\n  /**\n   * A relative or absolute path, or full URL of the endpoint. This value is optional if the first\n   * parameter of the method is annotated with {@link Url @Url}.\n   *\n   * <p>See {@linkplain retrofit2.Retrofit.Builder#baseUrl(HttpUrl) base URL} for details of how\n   * this is resolved against a base URL to create the full endpoint URL.\n   */\n  String value() default \"\";\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/HTTP.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.HttpUrl;\n\n/**\n * Use a custom HTTP verb for a request.\n *\n * <pre><code>\n * interface Service {\n *   &#064;HTTP(method = \"CUSTOM\", path = \"custom/endpoint/\")\n *   Call&lt;ResponseBody&gt; customEndpoint();\n * }\n * </code></pre>\n *\n * This annotation can also used for sending {@code DELETE} with a request body:\n *\n * <pre><code>\n * interface Service {\n *   &#064;HTTP(method = \"DELETE\", path = \"remove/\", hasBody = true)\n *   Call&lt;ResponseBody&gt; deleteObject(@Body RequestBody object);\n * }\n * </code></pre>\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface HTTP {\n  String method();\n\n  /**\n   * A relative or absolute path, or full URL of the endpoint. This value is optional if the first\n   * parameter of the method is annotated with {@link Url @Url}.\n   *\n   * <p>See {@linkplain retrofit2.Retrofit.Builder#baseUrl(HttpUrl) base URL} for details of how\n   * this is resolved against a base URL to create the full endpoint URL.\n   */\n  String path() default \"\";\n\n  boolean hasBody() default false;\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Header.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.PARAMETER;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\n\n/**\n * Replaces the header with the value of its target.\n *\n * <pre><code>\n * &#64;GET(\"/\")\n * Call&lt;ResponseBody&gt; foo(@Header(\"Accept-Language\") String lang);\n * </code></pre>\n *\n * Header parameters may be {@code null} which will omit them from the request. Passing a {@link\n * java.util.List List} or array will result in a header for each non-{@code null} item.\n *\n * <p>Parameter keys and values only allows ascii values by default. Specify {@link\n * #allowUnsafeNonAsciiValues() allowUnsafeNonAsciiValues=true} to change this behavior.\n *\n * <pre><code>\n * &#64;GET(\"/\")\n * Call&lt;ResponseBody&gt; foo(@Header(\"Accept-Language\", allowUnsafeNonAsciiValues=true) String lang);\n * </code></pre>\n *\n * <p><strong>Note:</strong> Headers do not overwrite each other. All headers with the same name\n * will be included in the request.\n *\n * @see Headers\n * @see HeaderMap\n */\n@Documented\n@Retention(RUNTIME)\n@Target(PARAMETER)\npublic @interface Header {\n\n  /** The query parameter name. */\n  String value();\n\n  /**\n   * Specifies whether the parameter {@linkplain #value() name} and value are already URL encoded.\n   */\n  boolean allowUnsafeNonAsciiValues() default false;\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/HeaderMap.java\n/*\n * Copyright (C) 2016 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n...\n", "needle_token_start": 686, "needle_token_end": 905, "code_context_ntokens": 15857, "output": ["```java\n@GuardedBy(\"this\")\nprivate okhttp3.Call getRawCall() throws IOException {\n  okhttp3.Call call = rawCall;\n  if (call != null) return call;\n\n  // Re-throw previous failures if this isn't the first attempt.\n  if (creationFailure != null) {\n    if (creationFailure instanceof IOException) {\n      throw (IOException) creationFailure;\n    } else if (creationFailure instanceof RuntimeException) {\n      throw (RuntimeException) creationFailure;\n    } else {\n      throw (Error) creationFailure;\n    }\n  }\n\n  // Create and remember either the success or the failure.\n  try {\n    return rawCall = createRawCall();\n  } catch (RuntimeException | Error | IOException e) {\n    throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n    creationFailure = e;\n    throw e;\n  }\n}\n```"]}
{"repo": "square/retrofit", "name": "validateServiceInterface", "language": "java", "path": "retrofit/src/main/java/retrofit2/Retrofit.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To ensure that a provided class is a valid interface for defining API endpoints, adhering to specific constraints necessary for dynamic proxy generation.\n2. **Input**: A `Class<?>` object representing the interface to be validated.\n3. **Output**: No direct output; however, it throws an `IllegalArgumentException` if the validation fails.\n4. **Procedure**: \n   - Checks if the input class is an interface; if not, an exception is thrown.\n   - Ensures the interface does not have type parameters, throwing an exception if it does.\n   - Recursively checks all inherited interfaces for the same constraints.\n   - If eager validation is enabled, it further validates each method in the interface to ensure they conform to expected standards for dynamic proxy handling, such as not being default, static, or synthetic methods.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/ServiceMethod.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n...\n// Path: retrofit/src/main/java/retrofit2/Retrofit.java\n/*\n * Copyright (C) 2012 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static java.util.Collections.unmodifiableList;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.InvocationHandler;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.Modifier;\nimport java.lang.reflect.Proxy;\nimport java.lang.reflect.Type;\nimport java.net.URL;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Deque;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\nimport okhttp3.HttpUrl;\nimport okhttp3.OkHttpClient;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.GET;\nimport retrofit2.http.HTTP;\nimport retrofit2.http.Header;\nimport retrofit2.http.Url;\n\n/**\n * Retrofit adapts a Java interface to HTTP calls by using annotations on the declared methods to\n * define how requests are made. Create instances using {@linkplain Builder the builder} and pass\n * your interface to {@link #create} to generate an implementation.\n *\n * <p>For example,\n *\n * <pre><code>\n * Retrofit retrofit = new Retrofit.Builder()\n *     .baseUrl(\"https://api.example.com/\")\n *     .addConverterFactory(GsonConverterFactory.create())\n *     .build();\n *\n * MyApi api = retrofit.create(MyApi.class);\n * Response&lt;User&gt; user = api.getUser().execute();\n * </code></pre>\n *\n * @author Bob Lee (bob@squareup.com)\n * @author Jake Wharton (jw@squareup.com)\n */\npublic final class Retrofit {\n  /**\n   * Method associations in this map will be in one of three states, and may only progress forward\n   * to higher-numbered states.\n   * <ol>\n   *   <li>No value - no one has started or completed parsing annotations for the method.</li>\n   *   <li>Lock object - a thread has started parsing annotations on the method. Once the lock is\n   *   available the map will have been updated with the parsed model.</li>\n   *   <li>{@code ServiceMethod} - annotations for the method have been fully parsed.</li>\n   * </ol>\n   * This map should only be accessed through {@link #loadServiceMethod} which contains the state\n   * transition logic.\n   */\n  private final ConcurrentHashMap<Method, Object> serviceMethodCache = new ConcurrentHashMap<>();\n\n  final okhttp3.Call.Factory callFactory;\n  final HttpUrl baseUrl;\n  final List<Converter.Factory> converterFactories;\n  final int defaultConverterFactoriesSize;\n  final List<CallAdapter.Factory> callAdapterFactories;\n  final int defaultCallAdapterFactoriesSize;\n  final @Nullable Executor callbackExecutor;\n  final boolean validateEagerly;\n\n  Retrofit(\n      okhttp3.Call.Factory callFactory,\n      HttpUrl baseUrl,\n      List<Converter.Factory> converterFactories,\n      int defaultConverterFactoriesSize,\n      List<CallAdapter.Factory> callAdapterFactories,\n      int defaultCallAdapterFactoriesSize,\n      @Nullable Executor callbackExecutor,\n      boolean validateEagerly) {\n    this.callFactory = callFactory;\n    this.baseUrl = baseUrl;\n    this.converterFactories = converterFactories; // Copy+unmodifiable at call site.\n    this.defaultConverterFactoriesSize = defaultConverterFactoriesSize;\n    this.callAdapterFactories = callAdapterFactories; // Copy+unmodifiable at call site.\n    this.defaultCallAdapterFactoriesSize = defaultCallAdapterFactoriesSize;\n    this.callbackExecutor = callbackExecutor;\n    this.validateEagerly = validateEagerly;\n  }\n\n  /**\n   * Create an implementation of the API endpoints defined by the {@code service} interface.\n   *\n   * <p>The relative path for a given method is obtained from an annotation on the method describing\n   * the request type. The built-in methods are {@link retrofit2.http.GET GET}, {@link\n   * retrofit2.http.PUT PUT}, {@link retrofit2.http.POST POST}, {@link retrofit2.http.PATCH PATCH},\n   * {@link retrofit2.http.HEAD HEAD}, {@link retrofit2.http.DELETE DELETE} and {@link\n   * retrofit2.http.OPTIONS OPTIONS}. You can use a custom HTTP method with {@link HTTP @HTTP}. For\n   * a dynamic URL, omit the path on the annotation and annotate the first parameter with {@link\n   * Url @Url}.\n   *\n   * <p>Method parameters can be used to replace parts of the URL by annotating them with {@link\n   * retrofit2.http.Path @Path}. Replacement sections are denoted by an identifier surrounded by\n   * curly braces (e.g., \"{foo}\"). To add items to the query string of a URL use {@link\n   * retrofit2.http.Query @Query}.\n   *\n   * <p>The body of a request is denoted by the {@link retrofit2.http.Body @Body} annotation. The\n   * object will be converted to request representation by one of the {@link Converter.Factory}\n   * instances. A {@link RequestBody} can also be used for a raw representation.\n   *\n   * <p>Alternative request body formats are supported by method annotations and corresponding\n   * parameter annotations:\n   *\n   * <ul>\n   *   <li>{@link retrofit2.http.FormUrlEncoded @FormUrlEncoded} - Form-encoded data with key-value\n   *       pairs specified by the {@link retrofit2.http.Field @Field} parameter annotation.\n   *   <li>{@link retrofit2.http.Multipart @Multipart} - RFC 2388-compliant multipart data with\n   *       parts specified by the {@link retrofit2.http.Part @Part} parameter annotation.\n   * </ul>\n   *\n   * <p>Additional static headers can be added for an endpoint using the {@link\n   * retrofit2.http.Headers @Headers} method annotation. For per-request control over a header\n   * annotate a parameter with {@link Header @Header}.\n   *\n   * <p>By default, methods return a {@link Call} which represents the HTTP request. The generic\n   * parameter of the call is the response body type and will be converted by one of the {@link\n   * Converter.Factory} instances. {@link ResponseBody} can also be used for a raw representation.\n   * {@link Void} can be used if you do not care about the body contents.\n   *\n   * <p>For example:\n   *\n   * <pre>\n   * public interface CategoryService {\n   *   &#64;POST(\"category/{cat}/\")\n   *   Call&lt;List&lt;Item&gt;&gt; categoryList(@Path(\"cat\") String a, @Query(\"page\") int b);\n   * }\n   * </pre>\n   */\n  @SuppressWarnings(\"unchecked\") // Single-interface proxy creation guarded by parameter safety.\n  public <T> T create(final Class<T> service) {\n    validateServiceInterface(service);\n    return (T)\n        Proxy.newProxyInstance(\n            service.getClassLoader(),\n            new Class<?>[] {service},\n            new InvocationHandler() {\n              private final Object[] emptyArgs = new Object[0];\n\n              @Override\n              public @Nullable Object invoke(Object proxy, Method method, @Nullable Object[] args)\n                  throws Throwable {\n                // If the method is a method from Object then defer to normal invocation.\n                if (method.getDeclaringClass() == Object.class) {\n                  return method.invoke(this, args);\n                }\n                args = args != null ? args : emptyArgs;\n                Reflection reflection = Platform.reflection;\n                return reflection.isDefaultMethod(method)\n                    ? reflection.invokeDefaultMethod(method, service, proxy, args)\n                    : loadServiceMethod(service, method).invoke(proxy, args);\n              }\n            });\n  }\n\n  \nprivate void validateServiceInterface(Class<?> service) {\n    if (!service.isInterface()) {\n      throw new IllegalArgumentException(\"API declarations must be interfaces.\");\n    }\n\n    Deque<Class<?>> check = new ArrayDeque<>(1);\n    check.add(service);\n    while (!check.isEmpty()) {\n      Class<?> candidate = check.removeFirst();\n      if (candidate.getTypeParameters().length != 0) {\n        StringBuilder message =\n            new StringBuilder(\"Type parameters are unsupported on \").append(candidate.getName());\n        if (candidate != service) {\n          message.append(\" which is an interface of \").append(service.getName());\n        }\n        throw new IllegalArgumentException(message.toString());\n      }\n      Collections.addAll(check, candidate.getInterfaces());\n    }\n\n    if (validateEagerly) {\n      Reflection reflection = Platform.reflection;\n      for (Method method : service.getDeclaredMethods()) {\n        if (!reflection.isDefaultMethod(method)\n            && !Modifier.isStatic(method.getModifiers())\n            && !method.isSynthetic()) {\n          loadServiceMethod(service, method);\n        }\n      }\n    }\n  }\n\n  ServiceMethod<?> loadServiceMethod(Class<?> service, Method method) {\n    // Note: Once we are minSdk 24 this whole method can be replaced by computeIfAbsent.\n    Object lookup = serviceMethodCache.get(method);\n\n    if (lookup instanceof ServiceMethod<?>) {\n      // Happy path: method is already parsed into the model.\n      return (ServiceMethod<?>) lookup;\n    }\n\n    if (lookup == null) {\n      // Map does not contain any value. Try to put in a lock for this method. We MUST synchronize\n      // on the lock before it is visible to others via the map to signal we are doing the work.\n      Object lock = new Object();\n      synchronized (lock) {\n        lookup = serviceMethodCache.putIfAbsent(method, lock);\n        if (lookup == null) {\n          // On successful lock insertion, perform the work and update the map before releasing.\n          // Other threads may be waiting on lock now and will expect the parsed model.\n          ServiceMethod<Object> result = ServiceMethod.parseAnnotations(this, service, method);\n          serviceMethodCache.put(method, result);\n          return result;\n        }\n      }\n    }\n\n    // Either the initial lookup or the attempt to put our lock in the map has returned someone\n    // else's lock. This means they are doing the parsing, and will update the map before releasing\n    // the lock. Once we can take the lock, the map is guaranteed to contain the model.\n    // Note: There's a chance that our effort to put a lock into the map has actually returned a\n    // finished model instead of a lock. In that case this code will perform a pointless lock and\n    // redundant lookup in the map of the same instance. This is rare, and ultimately harmless.\n    synchronized (lookup) {\n      return (ServiceMethod<?>) serviceMethodCache.get(method);\n    }\n  }\n\n  /**\n   * The factory used to create {@linkplain okhttp3.Call OkHttp calls} for sending a HTTP requests.\n   * Typically an instance of {@link OkHttpClient}.\n   */\n  public okhttp3.Call.Factory callFactory() {\n    return callFactory;\n  }\n\n  /** The API base URL. */\n  public HttpUrl baseUrl() {\n    return baseUrl;\n  }\n\n  /**\n   * Returns a list of the factories tried when creating a {@linkplain #callAdapter(Type,\n   * Annotation[])} call adapter}.\n   */\n  public List<CallAdapter.Factory> callAdapterFactories() {\n    return callAdapterFactories;\n  }\n\n  /**\n   * Returns the {@link CallAdapter} for {@code returnType} from the available {@linkplain\n   * #callAdapterFactories() factories}.\n   *\n   * @throws IllegalArgumentException if no call adapter available for {@code type}.\n   */\n  public CallAdapter<?, ?> callAdapter(Type returnType, Annotation[] annotations) {\n    return nextCallAdapter(null, returnType, annotations);\n  }\n\n  /**\n   * Returns the {@link CallAdapter} for {@code returnType} from the available {@linkplain\n   * #callAdapterFactories() factories} except {@code skipPast}.\n   *\n   * @throws IllegalArgumentException if no call adapter available for {@code type}.\n   */\n  public CallAdapter<?, ?> nextCallAdapter(\n      @Nullable CallAdapter.Factory skipPast, Type returnType, Annotation[] annotations) {\n    Objects.requireNonNull(returnType, \"returnType == null\");\n    Objects.requireNonNull(annotations, \"annotations == null\");\n\n    int start = callAdapterFactories.indexOf(skipPast) + 1;\n    for (int i = start, count = callAdapterFactories.size(); i < count; i++) {\n      CallAdapter<?, ?> adapter = callAdapterFactories.get(i).get(returnType, annotations, this);\n      if (adapter != null) {\n        return adapter;\n      }\n    }\n\n    StringBuilder builder =\n        new StringBuilder(\"Could not locate call adapter for \").append(returnType).append(\".\\n\");\n    if (skipPast != null) {\n      builder.append(\"  Skipped:\");\n      for (int i = 0; i < start; i++) {\n        builder.append(\"\\n   * \").append(callAdapterFactories.get(i).getClass().getName());\n      }\n      builder.append('\\n');\n    }\n    builder.append(\"  Tried:\");\n    for (int i = start, count = callAdapterFactories.size(); i < count; i++) {\n      builder.append(\"\\n   * \").append(callAdapterFactories.get(i).getClass().getName());\n    }\n    throw new IllegalArgumentException(builder.toString());\n  }\n\n  /**\n   * Returns an unmodifiable list of the factories tried when creating a {@linkplain\n   * #requestBodyConverter(Type, Annotation[], Annotation[]) request body converter}, a {@linkplain\n   * #responseBodyConverter(Type, Annotation[]) response body converter}, or a {@linkplain\n   * #stringConverter(Type, Annotation[]) string converter}.\n   */\n  public List<Converter.Factory> converterFactories() {\n    return converterFactories;\n  }\n\n  /**\n   * Returns a {@link Converter} for {@code type} to {@link RequestBody} from the available\n   * {@linkplain #converterFactories() factories}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<T, RequestBody> requestBodyConverter(\n      Type type, Annotation[] parameterAnnotations, Annotation[] methodAnnotations) {\n    return nextRequestBodyConverter(null, type, parameterAnnotations, methodAnnotations);\n  }\n\n  /**\n   * Returns a {@link Converter} for {@code type} to {@link RequestBody} from the available\n   * {@linkplain #converterFactories() factories} except {@code skipPast}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<T, RequestBody> nextRequestBodyConverter(\n      @Nullable Converter.Factory skipPast,\n      Type type,\n      Annotation[] parameterAnnotations,\n      Annotation[] methodAnnotations) {\n    Objects.requireNonNull(type, \"type == null\");\n    Objects.requireNonNull(parameterAnnotations, \"parameterAnnotations == null\");\n    Objects.requireNonNull(methodAnnotations, \"methodAnnotations == null\");\n\n    int start = converterFactories.indexOf(skipPast) + 1;\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      Converter.Factory factory = converterFactories.get(i);\n      Converter<?, RequestBody> converter =\n          factory.requestBodyConverter(type, parameterAnnotations, methodAnnotations, this);\n      if (converter != null) {\n        //noinspection unchecked\n        return (Converter<T, RequestBody>) converter;\n      }\n    }\n\n    StringBuilder builder =\n        new StringBuilder(\"Could not locate RequestBody converter for \").append(type).append(\".\\n\");\n    if (skipPast != null) {\n      builder.append(\"  Skipped:\");\n      for (int i = 0; i < start; i++) {\n        builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n      }\n      builder.append('\\n');\n    }\n    builder.append(\"  Tried:\");\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n    }\n    throw new IllegalArgumentException(builder.toString());\n  }\n\n  /**\n   * Returns a {@link Converter} for {@link ResponseBody} to {@code type} from the available\n   * {@linkplain #converterFactories() factories}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<ResponseBody, T> responseBodyConverter(Type type, Annotation[] annotations) {\n    return nextResponseBodyConverter(null, type, annotations);\n  }\n\n  /**\n   * Returns a {@link Converter} for {@link ResponseBody} to {@code type} from the available\n   * {@linkplain #converterFactories() factories} except {@code skipPast}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<ResponseBody, T> nextResponseBodyConverter(\n      @Nullable Converter.Factory skipPast, Type type, Annotation[] annotations) {\n    Objects.requireNonNull(type, \"type == null\");\n    Objects.requireNonNull(annotations, \"annotations == null\");\n\n    int start = converterFactories.indexOf(skipPast) + 1;\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      Converter<ResponseBody, ?> converter =\n          converterFactories.get(i).responseBodyConverter(type, annotations, this);\n      if (converter != null) {\n        //noinspection unchecked\n        return (Converter<ResponseBody, T>) converter;\n      }\n    }\n\n    StringBuilder builder =\n        new StringBuilder(\"Could not locate ResponseBody converter for \")\n            .append(type)\n            .append(\".\\n\");\n    if (skipPast != null) {\n      builder.append(\"  Skipped:\");\n      for (int i = 0; i < start; i++) {\n        builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n      }\n      builder.append('\\n');\n    }\n    builder.append(\"  Tried:\");\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n    }\n    throw new IllegalArgumentException(builder.toString());\n  }\n\n  /**\n   * Returns a {@link Converter} for {@code type} to {@link String} from the available {@linkplain\n   * #converterFactories() factories}.\n   */\n  public <T> Converter<T, String> stringConverter(Type type, Annotation[] annotations) {\n    Objects.requireNonNull(type, \"type == null\");\n    Objects.requireNonNull(annotations, \"annotations == null\");\n\n    for (int i = 0, count = converterFactories.size(); i < count; i++) {\n      Converter<?, String> converter =\n          converterFactories.get(i).stringConverter(type, annotations, this);\n      if (converter != null) {\n        //noinspection unchecked\n        return (Converter<T, String>) converter;\n      }\n    }\n\n    // Nothing matched. Resort to default converter which just calls toString().\n    //noinspection unchecked\n    return (Converter<T, String>) BuiltInConverters.ToStringConverter.INSTANCE;\n  }\n\n  /**\n   * The executor used for {@link Callback} methods on a {@link Call}. This may be {@code null}, in\n   * which case callbacks should be made synchronously on the background thread.\n   */\n  public @Nullable Executor callbackExecutor() {\n    return callbackExecutor;\n  }\n\n  public Builder newBuilder() {\n    return new Builder(this);\n  }\n\n  /**\n   * Build a new {@link Retrofit}.\n   *\n   * <p>Calling {@link #baseUrl} is required before calling {@link #build()}. All other methods are\n   * optional.\n   */\n  public static final class Builder {\n    private @Nullable okhttp3.Call.Factory callFactory;\n    private @Nullable HttpUrl baseUrl;\n    private final List<Converter.Factory> converterFactories = new ArrayList<>();\n    private final List<CallAdapter.Factory> callAdapterFactories = new ArrayList<>();\n    private @Nullable Executor callbackExecutor;\n    private boolean validateEagerly;\n\n    public Builder() {}\n\n    Builder(Retrofit retrofit) {\n      callFactory = retrofit.callFactory;\n      baseUrl = retrofit.baseUrl;\n\n      // Do not add the default BuiltIntConverters and platform-aware converters added by build().\n      for (int i = 1,\n              size = retrofit.converterFactories.size() - retrofit.defaultConverterFactoriesSize;\n          i < size;\n          i++) {\n        converterFactories.add(retrofit.converterFactories.get(i));\n      }\n\n      // Do not add the default, platform-aware call adapters added by build().\n      for (int i = 0,\n              size =\n                  retrofit.callAdapterFactories.size() - retrofit.defaultCallAdapterFactoriesSize;\n          i < size;\n          i++) {\n        callAdapterFactories.add(retrofit.callAdapterFactories.get(i));\n      }\n\n      callbackExecutor = retrofit.callbackExecutor;\n      validateEagerly = retrofit.validateEagerly;\n    }\n\n    /**\n     * The HTTP client used for requests.\n     *\n     * <p>This is a convenience method for calling {@link #callFactory}.\n     */\n    public Builder client(OkHttpClient client) {\n      return callFactory(Objects.requireNonNull(client, \"client == null\"));\n    }\n\n    /**\n     * Specify a custom call factory for creating {@link Call} instances.\n     *\n     * <p>Note: Calling {@link #client} automatically sets this value.\n     */\n    public Builder callFactory(okhttp3.Call.Factory factory) {\n      this.callFactory = Objects.requireNonNull(factory, \"factory == null\");\n      return this;\n    }\n\n    /**\n     * Set the API base URL.\n     *\n     * @see #baseUrl(HttpUrl)\n     */\n    public Builder baseUrl(URL baseUrl) {\n      Objects.requireNonNull(baseUrl, \"baseUrl == null\");\n      return baseUrl(HttpUrl.get(baseUrl.toString()));\n    }\n\n    /**\n     * Set the API base URL.\n     *\n     * @see #baseUrl(HttpUrl)\n     */\n    public Builder baseUrl(String baseUrl) {\n      Objects.requireNonNull(baseUrl, \"baseUrl == null\");\n      return baseUrl(HttpUrl.get(baseUrl));\n    }\n\n    /**\n     * Set the API base URL.\n     *\n     * <p>The specified endpoint values (such as with {@link GET @GET}) are resolved against this\n     * value using {@link HttpUrl#resolve(String)}. The behavior of this matches that of an {@code\n     * <a href=\"\">} link on a website resolving on the current URL.\n     *\n     * <p><b>Base URLs should always end in {@code /}.</b>\n     *\n     * <p>A trailing {@code /} ensures that endpoints values which are relative paths will correctly\n     * append themselves to a base which has path components.\n     *\n     * <p><b>Correct:</b><br>\n     * Base URL: http://example.com/api/<br>\n     * Endpoint: foo/bar/<br>\n     * Result: http://example.com/api/foo/bar/\n     *\n     * <p><b>Incorrect:</b><br>\n     * Base URL: http://example.com/api<br>\n     * Endpoint: foo/bar/<br>\n     * Result: http://example.com/foo/bar/\n     *\n     * <p>This method enforces that {@code baseUrl} has a trailing {@code /}.\n     *\n     * <p><b>Endpoint values which contain a leading {@code /} are absolute.</b>\n     *\n     * <p>Absolute values retain only the host from {@code baseUrl} and ignore any specified path\n     * components.\n     *\n     * <p>Base URL: http://example.com/api/<br>\n     * Endpoint: /foo/bar/<br>\n     * Result: http://example.com/foo/bar/\n     *\n     * <p>Base URL: http://example.com/<br>\n     * Endpoint: /foo/bar/<br>\n     * Result: http://example.com/foo/bar/\n     *\n     * <p><b>Endpoint values may be a full URL.</b>\n     *\n     * <p>Values which have a host replace the host of {@code baseUrl} and values also with a scheme\n     * replace the scheme of {@code baseUrl}.\n     *\n     * <p>Base URL: http://example.com/<br>\n     * Endpoint: https://github.com/square/retrofit/<br>\n     * Result: https://github.com/square/retrofit/\n     *\n     * <p>Base URL: http://example.com<br>\n     * Endpoint: //github.com/square/retrofit/<br>\n     * Result: http://github.com/square/retrofit/ (note the scheme stays 'http')\n     */\n    public Builder baseUrl(HttpUrl baseUrl) {\n      Objects.requireNonNull(baseUrl, \"baseUrl == null\");\n      List<String> pathSegments = baseUrl.pathSegments();\n      if (!\"\".equals(pathSegments.get(pathSegments.size() - 1))) {\n        throw new IllegalArgumentException(\"baseUrl must end in /: \" + baseUrl);\n      }\n      this.baseUrl = baseUrl;\n      return this;\n    }\n\n    /** Add converter factory for serialization and deserialization of objects. */\n    public Builder addConverterFactory(Converter.Factory factory) {\n      converterFactories.add(Objects.requireNonNull(factory, \"factory == null\"));\n      return this;\n    }\n\n    /**\n     * Add a call adapter factory for supporting service method return types other than {@link\n     * Call}.\n     */\n    public Builder addCallAdapterFactory(CallAdapter.Factory factory) {\n      callAdapterFactories.add(Objects.requireNonNull(factory, \"factory == null\"));\n      return this;\n    }\n\n    /**\n     * The executor on which {@link Callback} methods are invoked when returning {@link Call} from\n     * your service method.\n     *\n     * <p>Note: {@code executor} is not used for {@linkplain #addCallAdapterFactory custom method\n     * return types}.\n     */\n    public Builder callbackExecutor(Executor executor) {\n      this.callbackExecutor = Objects.requireNonNull(executor, \"executor == null\");\n      return this;\n    }\n\n    /** Returns a modifiable list of call adapter factories. */\n    public List<CallAdapter.Factory> callAdapterFactories() {\n      return this.callAdapterFactories;\n    }\n\n    /** Returns a modifiable list of converter factories. */\n    public List<Converter.Factory> converterFactories() {\n      return this.converterFactories;\n    }\n\n    /**\n     * When calling {@link #create} on the resulting {@link Retrofit} instance, eagerly validate the\n     * configuration of all methods in the supplied interface.\n     */\n    public Builder validateEagerly(boolean validateEagerly) {\n      this.validateEagerly = validateEagerly;\n      return this;\n    }\n\n    /**\n     * Create the {@link Retrofit} instance using the configured values.\n     *\n     * <p>Note: If neither {@link #client} nor {@link #callFactory} is called a default {@link\n     * OkHttpClient} will be created and used.\n     */\n    public Retrofit build() {\n      if (baseUrl == null) {\n        throw new IllegalStateException(\"Base URL required.\");\n      }\n\n      okhttp3.Call.Factory callFactory = this.callFactory;\n      if (callFactory == null) {\n        callFactory = new OkHttpClient();\n      }\n\n      Executor callbackExecutor = this.callbackExecutor;\n      if (callbackExecutor == null) {\n        callbackExecutor = Platform.callbackExecutor;\n      }\n\n      BuiltInFactories builtInFactories = Platform.builtInFactories;\n\n      // Make a defensive copy of the adapters and add the default Call adapter.\n      List<CallAdapter.Factory> callAdapterFactories = new ArrayList<>(this.callAdapterFactories);\n      List<? extends CallAdapter.Factory> defaultCallAdapterFactories =\n          builtInFactories.createDefaultCallAdapterFactories(callbackExecutor);\n      callAdapterFactories.addAll(defaultCallAdapterFactories);\n\n      // Make a defensive copy of the converters.\n      List<? extends Converter.Factory> defaultConverterFactories =\n          builtInFactories.createDefaultConverterFactories();\n      int defaultConverterFactoriesSize = defaultConverterFactories.size();\n      List<Converter.Factory> converterFactories =\n          new ArrayList<>(1 + this.converterFactories.size() + defaultConverterFactoriesSize);\n\n      // Add the built-in converter factory first. This prevents overriding its behavior but also\n      // ensures correct behavior when using converters that consume all types.\n      converterFactories.add(new BuiltInConverters());\n      converterFactories.addAll(this.converterFactories);\n      converterFactories.addAll(defaultConverterFactories);\n\n      return new Retrofit(\n          callFactory,\n          baseUrl,\n          unmodifiableList(converterFactories),\n          defaultConverterFactoriesSize,\n          unmodifiableList(callAdapterFactories),\n          defaultCallAdapterFactories.size(),\n          callbackExecutor,\n          validateEagerly);\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Body.java\n/*\n * Copyright (C) 2011 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.PARAMETER;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport retrofit2.Converter;\nimport retrofit2.Retrofit;\n\n/**\n * Use this annotation on a service method param when you want to directly control the request body\n * of a POST/PUT request (instead of sending in as request parameters or form-style request body).\n * The object will be serialized using the {@link Retrofit Retrofit} instance {@link Converter\n * Converter} and the result will be set directly as the request body.\n *\n * <p>Body parameters may not be {@code null}.\n */\n@Documented\n@Target(PARAMETER)\n@Retention(RUNTIME)\npublic @interface Body {}\n\n// Path: retrofit/src/main/java/retrofit2/Converter.java\n/*\n * Copyright (C) 2012 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.Body;\nimport retrofit2.http.Field;\nimport retrofit2.http.FieldMap;\nimport retrofit2.http.Header;\nimport retrofit2.http.HeaderMap;\nimport retrofit2.http.Part;\nimport retrofit2.http.PartMap;\nimport retrofit2.http.Path;\nimport retrofit2.http.Query;\nimport retrofit2.http.QueryMap;\n\n/**\n * Convert objects to and from their representation in HTTP. Instances are created by {@linkplain\n * Factory a factory} which is {@linkplain Retrofit.Builder#addConverterFactory(Factory) installed}\n * into the {@link Retrofit} instance.\n */\npublic interface Converter<F, T> {\n  @Nullable\n  T convert(F value) throws IOException;\n\n  /** Creates {@link Converter} instances based on a type and target usage. */\n  abstract class Factory {\n    /**\n     * Returns a {@link Converter} for converting an HTTP response body to {@code type}, or null if\n     * {@code type} cannot be handled by this factory. This is used to create converters for\n     * response types such as {@code SimpleResponse} from a {@code Call<SimpleResponse>}\n     * declaration.\n     */\n    public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n        Type type, Annotation[] annotations, Retrofit retrofit) {\n      return null;\n    }\n\n    /**\n     * Returns a {@link Converter} for converting {@code type} to an HTTP request body, or null if\n     * {@code type} cannot be handled by this factory. This is used to create converters for types\n     * specified by {@link Body @Body}, {@link Part @Part}, and {@link PartMap @PartMap} values.\n     */\n    public @Nullable Converter<?, RequestBody> requestBodyConverter(\n        Type type,\n        Annotation[] parameterAnnotations,\n        Annotation[] methodAnnotations,\n        Retrofit retrofit) {\n      return null;\n    }\n\n    /**\n     * Returns a {@link Converter} for converting {@code type} to a {@link String}, or null if\n     * {@code type} cannot be handled by this factory. This is used to create converters for types\n     * specified by {@link Field @Field}, {@link FieldMap @FieldMap} values, {@link Header @Header},\n     * {@link HeaderMap @HeaderMap}, {@link Path @Path}, {@link Query @Query}, and {@link\n     * QueryMap @QueryMap} values.\n     */\n    public @Nullable Converter<?, String> stringConverter(\n        Type type, Annotation[] annotations, Retrofit retrofit) {\n      return null;\n    }\n\n    /**\n     * Extract the upper bound of the generic parameter at {@code index} from {@code type}. For\n     * example, index 1 of {@code Map<String, ? extends Runnable>} returns {@code Runnable}.\n     */\n    protected static Type getParameterUpperBound(int index, ParameterizedType type) {\n      return Utils.getParameterUpperBound(index, type);\n    }\n\n    /**\n     * Extract the raw class type from {@code type}. For example, the type representing {@code\n     * List<? extends Runnable>} returns {@code List.class}.\n     */\n    protected static Class<?> getRawType(Type type) {\n      return Utils.getRawType(type);\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/SkipCallbackExecutor.java\n/*\n * Copyright (C) 2019 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport java.lang.reflect.Type;\n\n/**\n * Change the behavior of a {@code Call<BodyType>} return type to not use the {@linkplain\n * Retrofit#callbackExecutor() callback executor} for invoking the {@link Callback#onResponse(Call,\n * Response) onResponse} or {@link Callback#onFailure(Call, Throwable) onFailure} methods.\n *\n * <pre><code>\n * &#64;SkipCallbackExecutor\n * &#64;GET(\"user/{id}/token\")\n * Call&lt;String&gt; getToken(@Path(\"id\") long id);\n * </code></pre>\n *\n * This annotation can also be used when a {@link CallAdapter.Factory} <em>explicitly</em> delegates\n * to the built-in factory for {@link Call} via {@link Retrofit#nextCallAdapter(CallAdapter.Factory,\n * Type, Annotation[])} in order for the returned {@link Call} to skip the executor. (Note: by\n * default, a {@link Call} supplied directly to a {@link CallAdapter} will already skip the callback\n * executor. The annotation is only useful when looking up the built-in adapter.)\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface SkipCallbackExecutor {}\n\n// Path: retrofit/src/main/java/retrofit2/DefaultCallAdapterFactory.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.util.Objects;\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\nimport okhttp3.Request;\nimport okio.Timeout;\n\nfinal class DefaultCallAdapterFactory extends CallAdapter.Factory {\n  private final @Nullable Executor callbackExecutor;\n\n  DefaultCallAdapterFactory(@Nullable Executor callbackExecutor) {\n    this.callbackExecutor = callbackExecutor;\n  }\n\n  @Override\n  public @Nullable CallAdapter<?, ?> get(\n      Type returnType, Annotation[] annotations, Retrofit retrofit) {\n    if (getRawType(returnType) != Call.class) {\n      return null;\n    }\n    if (!(returnType instanceof ParameterizedType)) {\n      throw new IllegalArgumentException(\n          \"Call return type must be parameterized as Call<Foo> or Call<? extends Foo>\");\n    }\n    final Type responseType = Utils.getParameterUpperBound(0, (ParameterizedType) returnType);\n\n    final Executor executor =\n        Utils.isAnnotationPresent(annotations, SkipCallbackExecutor.class)\n            ? null\n            : callbackExecutor;\n\n    return new CallAdapter<Object, Call<?>>() {\n      @Override\n      public Type responseType() {\n        return responseType;\n      }\n\n      @Override\n      public Call<Object> adapt(Call<Object> call) {\n        return executor == null ? call : new ExecutorCallbackCall<>(executor, call);\n      }\n    };\n  }\n\n  static final class ExecutorCallbackCall<T> implements Call<T> {\n    final Executor callbackExecutor;\n    final Call<T> delegate;\n\n    ExecutorCallbackCall(Executor callbackExecutor, Call<T> delegate) {\n      this.callbackExecutor = callbackExecutor;\n      this.delegate = delegate;\n    }\n\n    @Override\n    public void enqueue(final Callback<T> callback) {\n      Objects.requireNonNull(callback, \"callback == null\");\n\n      delegate.enqueue(\n          new Callback<T>() {\n            @Override\n            public void onResponse(Call<T> call, final Response<T> response) {\n              callbackExecutor.execute(\n                  () -> {\n                    if (delegate.isCanceled()) {\n                      // Emulate OkHttp's behavior of throwing/delivering an IOException on\n                      // cancellation.\n                      callback.onFailure(ExecutorCallbackCall.this, new IOException(\"Canceled\"));\n                    } else {\n                      callback.onResponse(ExecutorCallbackCall.this, response);\n                    }\n                  });\n            }\n\n            @Override\n            public void onFailure(Call<T> call, final Throwable t) {\n              callbackExecutor.execute(() -> callback.onFailure(ExecutorCallbackCall.this, t));\n            }\n          });\n    }\n\n    @Override\n    public boolean isExecuted() {\n      return delegate.isExecuted();\n    }\n\n    @Override\n    public Response<T> execute() throws IOException {\n      return delegate.execute();\n    }\n\n    @Override\n    public void cancel() {\n      delegate.cancel();\n    }\n\n    @Override\n    public boolean isCanceled() {\n      return delegate.isCanceled();\n    }\n\n    @SuppressWarnings(\"CloneDoesntCallSuperClone\") // Performing deep clone.\n    @Override\n    public Call<T> clone() {\n      return new ExecutorCallbackCall<>(callbackExecutor, delegate.clone());\n    }\n\n    @Override\n    public Request request() {\n      return delegate.request();\n    }\n\n    @Override\n    public Timeout timeout() {\n      return delegate.timeout();\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/DefaultMethodSupport.java\n/*\n * Copyright (C) 2024 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.invoke.MethodHandles.Lookup;\nimport java.lang.reflect.Constructor;\nimport java.lang.reflect.Method;\nimport javax.annotation.Nullable;\n\n/**\n * From Java 8 to Java 13, the only way to invoke a default method on a proxied interface is by\n * reflectively creating a trusted {@link Lookup} to invoke a method handle.\n * <p>\n * Note: This class has multi-release jar variants for newer versions of Java.\n */\nfinal class DefaultMethodSupport {\n  private static @Nullable Constructor<Lookup> lookupConstructor;\n\n  @Nullable\n  static Object invoke(\n      Method method, Class<?> declaringClass, Object proxy, @Nullable Object[] args)\n      throws Throwable {\n    Constructor<Lookup> constructor = lookupConstructor;\n    if (constructor == null) {\n      constructor = Lookup.class.getDeclaredConstructor(Class.class, int.class);\n      constructor.setAccessible(true);\n      lookupConstructor = constructor;\n    }\n    return constructor\n        .newInstance(declaringClass, -1 /* trusted */)\n        .unreflectSpecial(method, declaringClass)\n        .bindTo(proxy)\n        .invokeWithArguments(args);\n  }\n\n  private DefaultMethodSupport() {}\n}\n\n// Path: retrofit/src/main/java/retrofit2/CompletableFutureCallAdapterFactory.java\n/*\n * Copyright (C) 2016 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport android.annotation.TargetApi;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.util.concurrent.CompletableFuture;\nimport javax.annotation.Nullable;\nimport org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;\n\n@IgnoreJRERequirement // Only added when CompletableFuture is available (Java 8+ / Android API 24+).\n@TargetApi(24)\nfinal class CompletableFutureCallAdapterFactory extends CallAdapter.Factory {\n  @Override\n  public @Nullable CallAdapter<?, ?> get(\n      Type returnType, Annotation[] annotations, Retrofit retrofit) {\n    if (getRawType(returnType) != CompletableFuture.class) {\n      return null;\n    }\n    if (!(returnType instanceof ParameterizedType)) {\n      throw new IllegalStateException(\n          \"CompletableFuture return type must be parameterized\"\n              + \" as CompletableFuture<Foo> or CompletableFuture<? extends Foo>\");\n    }\n    Type innerType = getParameterUpperBound(0, (ParameterizedType) returnType);\n\n    if (getRawType(innerType) != Response.class) {\n      // Generic type is not Response<T>. Use it for body-only adapter.\n      return new BodyCallAdapter<>(innerType);\n    }\n\n    // Generic type is Response<T>. Extract T and create the Response version of the adapter.\n    if (!(innerType instanceof ParameterizedType)) {\n      throw new IllegalStateException(\n          \"Response must be parameterized\" + \" as Response<Foo> or Response<? extends Foo>\");\n    }\n    Type responseType = getParameterUpperBound(0, (ParameterizedType) innerType);\n    return new ResponseCallAdapter<>(responseType);\n  }\n\n  @IgnoreJRERequirement\n  private static final class BodyCallAdapter<R> implements CallAdapter<R, CompletableFuture<R>> {\n    private final Type responseType;\n\n    BodyCallAdapter(Type responseType) {\n      this.responseType = responseType;\n    }\n\n    @Override\n    public Type responseType() {\n      return responseType;\n    }\n\n    @Override\n    public CompletableFuture<R> adapt(final Call<R> call) {\n      CompletableFuture<R> future = new CallCancelCompletableFuture<>(call);\n      call.enqueue(new BodyCallback(future));\n      return future;\n    }\n\n    @IgnoreJRERequirement\n    private class BodyCallback implements Callback<R> {\n      private final CompletableFuture<R> future;\n\n      public BodyCallback(CompletableFuture<R> future) {\n        this.future = future;\n      }\n\n      @Override\n      public void onResponse(Call<R> call, Response<R> response) {\n        if (response.isSuccessful()) {\n          future.complete(response.body());\n        } else {\n          future.completeExceptionally(new HttpException(response));\n        }\n      }\n\n      @Override\n      public void onFailure(Call<R> call, Throwable t) {\n        future.completeExceptionally(t);\n      }\n    }\n  }\n\n  @IgnoreJRERequirement\n  private static final class ResponseCallAdapter<R>\n      implements CallAdapter<R, CompletableFuture<Response<R>>> {\n    private final Type responseType;\n\n    ResponseCallAdapter(Type responseType) {\n      this.responseType = responseType;\n    }\n\n    @Override\n    public Type responseType() {\n      return responseType;\n    }\n\n    @Override\n    public CompletableFuture<Response<R>> adapt(final Call<R> call) {\n      CompletableFuture<Response<R>> future = new CallCancelCompletableFuture<>(call);\n      call.enqueue(new ResponseCallback(future));\n      return future;\n    }\n\n    @IgnoreJRERequirement\n    private class ResponseCallback implements Callback<R> {\n      private final CompletableFuture<Response<R>> future;\n\n      public ResponseCallback(CompletableFuture<Response<R>> future) {\n        this.future = future;\n      }\n\n      @Override\n      public void onResponse(Call<R> call, Response<R> response) {\n        future.complete(response);\n      }\n\n      @Override\n      public void onFailure(Call<R> call, Throwable t) {\n        future.completeExceptionally(t);\n      }\n    }\n  }\n\n  @IgnoreJRERequirement\n  private static final class CallCancelCompletableFuture<T> extends CompletableFuture<T> {\n    private final Call<?> call;\n\n    CallCancelCompletableFuture(Call<?> call) {\n      this.call = call;\n    }\n\n    @Override\n    public boolean cancel(boolean mayInterruptIfRunning) {\n      if (mayInterruptIfRunning) {\n        call.cancel();\n      }\n      return super.cancel(mayInterruptIfRunning);\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/OptionalConverterFactory.java\n/*\n * Copyright (C) 2017 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport android.annotation.TargetApi;\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.util.Optional;\nimport javax.annotation.Nullable;\nimport okhttp3.ResponseBody;\nimport org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;\n\n@IgnoreJRERequirement // Only added when Optional is available (Java 8+ / Android API 24+).\n@TargetApi(24)\nfinal class OptionalConverterFactory extends Converter.Factory {\n  @Override\n  public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n      Type type, Annotation[] annotations, Retrofit retrofit) {\n    if (getRawType(type) != Optional.class) {\n      return null;\n    }\n\n    Type innerType = getParameterUpperBound(0, (ParameterizedType) type);\n    Converter<ResponseBody, Object> delegate =\n        retrofit.responseBodyConverter(innerType, annotations);\n    return new OptionalConverter<>(delegate);\n  }\n\n  @IgnoreJRERequirement\n  static final class OptionalConverter<T> implements Converter<ResponseBody, Optional<T>> {\n    final Converter<ResponseBody, T> delegate;\n\n    OptionalConverter(Converter<ResponseBody, T> delegate) {\n      this.delegate = delegate;\n    }\n\n    @Override\n    public Optional<T> convert(ResponseBody value) throws IOException {\n      return Optional.ofNullable(delegate.convert(value));\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/BuiltInFactories.java\n/*\n * Copyright (C) 2024 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static java.util.Arrays.asList;\nimport static java.util.Collections.emptyList;\nimport static java.util.Collections.singletonList;\n\nimport android.annotation.TargetApi;\nimport java.util.List;\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\n\nclass BuiltInFactories {\n  List<? extends CallAdapter.Factory> createDefaultCallAdapterFactories(\n      @Nullable Executor callbackExecutor) {\n    return singletonList(new DefaultCallAdapterFactory(callbackExecutor));\n  }\n\n  List<? extends Converter.Factory> createDefaultConverterFactories() {\n    return emptyList();\n  }\n\n  @TargetApi(24)\n  static final class Java8 extends BuiltInFactories {\n    @Override\n    List<? extends CallAdapter.Factory> createDefaultCallAdapterFactories(\n        @Nullable Executor callbackExecutor) {\n      return asList(\n          new CompletableFutureCallAdapterFactory(),\n          new DefaultCallAdapterFactory(callbackExecutor));\n    }\n\n    @Override\n    List<? extends Converter.Factory> createDefaultConverterFactories() {\n      return singletonList(new OptionalConverterFactory());\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Invocation.java\n/*\n * Copyright (C) 2018 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\n\n/**\n * A single invocation of a Retrofit service interface method. This class captures both the method\n * that was called and the arguments to the method.\n *\n * <p>Retrofit automatically adds an invocation to each OkHttp request as a tag. You can retrieve\n * the invocation in an OkHttp interceptor for metrics and monitoring.\n *\n * <pre><code>\n * class InvocationLogger implements Interceptor {\n *   &#64;Override public Response intercept(Chain chain) throws IOException {\n *     Request request = chain.request();\n *     Invocation invocation = request.tag(Invocation.class);\n *     if (invocation != null) {\n *       System.out.printf(\"%s.%s %s%n\",\n *           invocation.service().getSimpleName(),\n *           invocation.method().getName(),\n *           invocation.arguments());\n *     }\n *     return chain.proceed(request);\n *   }\n * }\n * </code></pre>\n *\n * <strong>Note:</strong> use caution when examining an invocation's arguments. Although the\n * arguments list is unmodifiable, the arguments themselves may be mutable. They may also be unsafe\n * for concurrent access. For best results declare Retrofit service interfaces using only immutable\n * types for parameters!\n */\npublic final class Invocation {\n  public static <T> Invocation of(Class<T> service, T instance, Method method, List<?> arguments) {\n    Objects.requireNonNull(service, \"service == null\");\n    Objects.requireNonNull(instance, \"instance == null\");\n    Objects.requireNonNull(method, \"method == null\");\n    Objects.requireNonNull(arguments, \"arguments == null\");\n    return new Invocation(service, instance, method, new ArrayList<>(arguments)); // Defensive copy.\n  }\n\n  @Deprecated\n  public static Invocation of(Method method, List<?> arguments) {\n    Objects.requireNonNull(method, \"method == null\");\n    Objects.requireNonNull(arguments, \"arguments == null\");\n    return new Invocation(\n        method.getDeclaringClass(), null, method, new ArrayList<>(arguments)); // Defensive copy.\n  }\n\n  private final Class<?> service;\n  @Nullable private final Object instance;\n  private final Method method;\n  private final List<?> arguments;\n\n  /** Trusted constructor assumes ownership of {@code arguments}. */\n  Invocation(Class<?> service, @Nullable Object instance, Method method, List<?> arguments) {\n    this.service = service;\n    this.instance = instance;\n    this.method = method;\n    this.arguments = Collections.unmodifiableList(arguments);\n  }\n\n  public Class<?> service() {\n    return service;\n  }\n\n  /**\n   * The instance of {@link #service}.\n   * <p>\n   * This will never be null when created by Retrofit. Null will only be returned when created\n   * by {@link #of(Method, List)}.\n   */\n  @Nullable\n  public Object instance() {\n    return instance;\n  }\n\n  public Method method() {\n    return method;\n  }\n\n  public List<?> arguments() {\n    return arguments;\n  }\n\n  @Override\n  public String toString() {\n    return String.format(\"%s.%s() %s\", service.getName(), method.getName(), arguments);\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/AndroidMainExecutor.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport android.os.Handler;\nimport android.os.Looper;\nimport java.util.concurrent.Executor;\n\nfinal class AndroidMainExecutor implements Executor {\n  private final Handler handler = new Handler(Looper.getMainLooper());\n\n  @Override\n  public void execute(Runnable r) {\n    handler.post(r);\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Callback.java\n/*\n * Copyright (C) 2012 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\n/**\n * Communicates responses from a server or offline requests. One and only one method will be invoked\n * in response to a given request.\n *\n * <p>Callback methods are executed using the {@link Retrofit} callback executor. When none is\n * specified, the following defaults are used:\n *\n * <ul>\n *   <li>Android: Callbacks are executed on the application's main (UI) thread.\n *   <li>JVM: Callbacks are executed on the background thread which performed the request.\n * </ul>\n *\n * @param <T> Successful response body type.\n */\npublic interface Callback<T> {\n  /**\n   * Invoked for a received HTTP response.\n   *\n   * <p>Note: An HTTP response may still indicate an application-level failure such as a 404 or 500.\n   * Call {@link Response#isSuccessful()} to determine if the response indicates success.\n   */\n  void onResponse(Call<T> call, Response<T> response);\n\n  /**\n...\n", "needle_token_start": 2260, "needle_token_end": 2561, "code_context_ntokens": 15949, "output": ["```java\nprivate void validateServiceInterface(Class<?> service) {\n    if (!service.isInterface()) {\n      throw new IllegalArgumentException(\"API declarations must be interfaces.\");\n    }\n\n    Deque<Class<?>> check = new ArrayDeque<>(1);\n    check.add(service);\n    while (!check.isEmpty()) {\n      Class<?> candidate = check.removeFirst();\n      if (candidate.getTypeParameters().length != 0) {\n        StringBuilder message =\n            new StringBuilder(\"Type parameters are unsupported on \").append(candidate.getName());\n        if (candidate != service) {\n          message.append(\" which is an interface of \").append(service.getName());\n        }\n        throw new IllegalArgumentException(message.toString());\n      }\n      Collections.addAll(check, candidate.getInterfaces());\n    }\n\n    if (validateEagerly) {\n      Reflection reflection = Platform.reflection;\n      for (Method method : service.getDeclaredMethods()) {\n        if (!reflection.isDefaultMethod(method)\n            && !Modifier.isStatic(method.getModifiers())\n            && !method.isSynthetic()) {\n          loadServiceMethod(service, method);\n        }\n      }\n    }\n  }\n```"]}
{"repo": "square/retrofit", "name": "resolveTypeVariable", "language": "java", "path": "retrofit/src/main/java/retrofit2/Utils.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the actual type for a type variable within a specific context, particularly when the type variable is part of a generic declaration.\n2. **Input**: The function receives three parameters: a type representing the context in which the type variable is used, a class representing the raw type of the context, and the type variable itself that needs resolution.\n3. **Output**: It returns the resolved type if it can be determined from the context; otherwise, it returns the input type variable unchanged.\n4. **Procedure**: \n   - First, the function identifies the class where the type variable is declared.\n   - If the class is not found, it returns the type variable as it cannot be resolved.\n   - It then retrieves the generic supertype of the context that corresponds to the class where the type variable is declared.\n   - If this supertype is parameterized, the function finds the position of the type variable in the declaration and returns the corresponding actual type argument from the supertype.\n   - If these conditions are not met, the type variable is returned unresolved.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/Response.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.Headers;\nimport okhttp3.Protocol;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\n\n/** An HTTP response. */\npublic final class Response<T> {\n  /** Create a synthetic successful response with {@code body} as the deserialized body. */\n  public static <T> Response<T> success(@Nullable T body) {\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response with an HTTP status code of {@code code} and {@code\n   * body} as the deserialized body.\n   */\n  public static <T> Response<T> success(int code, @Nullable T body) {\n    if (code < 200 || code >= 300) {\n      throw new IllegalArgumentException(\"code < 200 or >= 300: \" + code);\n    }\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(code)\n            .message(\"Response.success()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response using {@code headers} with {@code body} as the\n   * deserialized body.\n   */\n  public static <T> Response<T> success(@Nullable T body, Headers headers) {\n    Objects.requireNonNull(headers, \"headers == null\");\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .headers(headers)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a successful response from {@code rawResponse} with {@code body} as the deserialized\n   * body.\n   */\n  public static <T> Response<T> success(@Nullable T body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n...\n// Path: retrofit/src/main/java/retrofit2/Utils.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\n\nfinal class Utils {\n  static final Type[] EMPTY_TYPE_ARRAY = new Type[0];\n\n  private Utils() {\n    // No instances.\n  }\n\n  static RuntimeException methodError(Method method, String message, Object... args) {\n    return methodError(method, null, message, args);\n  }\n\n  @SuppressWarnings(\"AnnotateFormatMethod\")\n  static RuntimeException methodError(\n      Method method, @Nullable Throwable cause, String message, Object... args) {\n    message = String.format(message, args);\n    return new IllegalArgumentException(\n        message\n            + \"\\n    for method \"\n            + method.getDeclaringClass().getSimpleName()\n            + \".\"\n            + method.getName(),\n        cause);\n  }\n\n  static RuntimeException parameterError(\n      Method method, Throwable cause, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, cause, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static RuntimeException parameterError(Method method, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static Class<?> getRawType(Type type) {\n    Objects.requireNonNull(type, \"type == null\");\n\n    if (type instanceof Class<?>) {\n      // Type is a normal class.\n      return (Class<?>) type;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n\n      // I'm not exactly sure why getRawType() returns Type instead of Class. Neal isn't either but\n      // suspects some pathological case related to nested classes exists.\n      Type rawType = parameterizedType.getRawType();\n      if (!(rawType instanceof Class)) throw new IllegalArgumentException();\n      return (Class<?>) rawType;\n    }\n    if (type instanceof GenericArrayType) {\n      Type componentType = ((GenericArrayType) type).getGenericComponentType();\n      return Array.newInstance(getRawType(componentType), 0).getClass();\n    }\n    if (type instanceof TypeVariable) {\n      // We could use the variable's bounds, but that won't work if there are multiple. Having a raw\n      // type that's more general than necessary is okay.\n      return Object.class;\n    }\n    if (type instanceof WildcardType) {\n      return getRawType(((WildcardType) type).getUpperBounds()[0]);\n    }\n\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + type.getClass().getName());\n  }\n\n  /** Returns true if {@code a} and {@code b} are equal. */\n  static boolean equals(Type a, Type b) {\n    if (a == b) {\n      return true; // Also handles (a == null && b == null).\n\n    } else if (a instanceof Class) {\n      return a.equals(b); // Class already specifies equals().\n\n    } else if (a instanceof ParameterizedType) {\n      if (!(b instanceof ParameterizedType)) return false;\n      ParameterizedType pa = (ParameterizedType) a;\n      ParameterizedType pb = (ParameterizedType) b;\n      Object ownerA = pa.getOwnerType();\n      Object ownerB = pb.getOwnerType();\n      boolean ownersAreEqual = ownerA == ownerB || (ownerA != null && ownerA.equals(ownerB));\n      boolean rawTypesAreEqual = pa.getRawType().equals(pb.getRawType());\n      boolean typeArgumentsAreEqual =\n          Arrays.equals(pa.getActualTypeArguments(), pb.getActualTypeArguments());\n\n      return ownersAreEqual && rawTypesAreEqual && typeArgumentsAreEqual;\n\n    } else if (a instanceof GenericArrayType) {\n      if (!(b instanceof GenericArrayType)) return false;\n      GenericArrayType ga = (GenericArrayType) a;\n      GenericArrayType gb = (GenericArrayType) b;\n      return equals(ga.getGenericComponentType(), gb.getGenericComponentType());\n\n    } else if (a instanceof WildcardType) {\n      if (!(b instanceof WildcardType)) return false;\n      WildcardType wa = (WildcardType) a;\n      WildcardType wb = (WildcardType) b;\n      return Arrays.equals(wa.getUpperBounds(), wb.getUpperBounds())\n          && Arrays.equals(wa.getLowerBounds(), wb.getLowerBounds());\n\n    } else if (a instanceof TypeVariable) {\n      if (!(b instanceof TypeVariable)) return false;\n      TypeVariable<?> va = (TypeVariable<?>) a;\n      TypeVariable<?> vb = (TypeVariable<?>) b;\n      return va.getGenericDeclaration() == vb.getGenericDeclaration()\n          && va.getName().equals(vb.getName());\n\n    } else {\n      return false; // This isn't a type we support!\n    }\n  }\n\n  /**\n   * Returns the generic supertype for {@code supertype}. For example, given a class {@code\n   * IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>} and the\n   * result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.\n   */\n  static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {\n    if (toResolve == rawType) return context;\n\n    // We skip searching through interfaces if unknown is an interface.\n    if (toResolve.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == toResolve) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (toResolve.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], toResolve);\n        }\n      }\n    }\n\n    // Check our supertypes.\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == toResolve) {\n          return rawType.getGenericSuperclass();\n        } else if (toResolve.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, toResolve);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // We can't resolve this further.\n    return toResolve;\n  }\n\n  private static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0; i < array.length; i++) {\n      if (toFind.equals(array[i])) return i;\n    }\n    throw new NoSuchElementException();\n  }\n\n  static String typeToString(Type type) {\n    return type instanceof Class ? ((Class<?>) type).getName() : type.toString();\n  }\n\n  /**\n   * Returns the generic form of {@code supertype}. For example, if this is {@code\n   * ArrayList<String>}, this returns {@code Iterable<String>} given the input {@code\n   * Iterable.class}.\n   *\n   * @param supertype a superclass of, or interface implemented by, this.\n   */\n  static Type getSupertype(Type context, Class<?> contextRawType, Class<?> supertype) {\n    if (!supertype.isAssignableFrom(contextRawType)) throw new IllegalArgumentException();\n    return resolve(\n        context, contextRawType, getGenericSupertype(context, contextRawType, supertype));\n  }\n\n  static Type resolve(Type context, Class<?> contextRawType, Type toResolve) {\n    // This implementation is made a little more complicated in an attempt to avoid object-creation.\n    while (true) {\n      if (toResolve instanceof TypeVariable) {\n        TypeVariable<?> typeVariable = (TypeVariable<?>) toResolve;\n        toResolve = resolveTypeVariable(context, contextRawType, typeVariable);\n        if (toResolve == typeVariable) {\n          return toResolve;\n        }\n\n      } else if (toResolve instanceof Class && ((Class<?>) toResolve).isArray()) {\n        Class<?> original = (Class<?>) toResolve;\n        Type componentType = original.getComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof GenericArrayType) {\n        GenericArrayType original = (GenericArrayType) toResolve;\n        Type componentType = original.getGenericComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof ParameterizedType) {\n        ParameterizedType original = (ParameterizedType) toResolve;\n        Type ownerType = original.getOwnerType();\n        Type newOwnerType = resolve(context, contextRawType, ownerType);\n        boolean changed = newOwnerType != ownerType;\n\n        Type[] args = original.getActualTypeArguments();\n        for (int t = 0, length = args.length; t < length; t++) {\n          Type resolvedTypeArgument = resolve(context, contextRawType, args[t]);\n          if (resolvedTypeArgument != args[t]) {\n            if (!changed) {\n              args = args.clone();\n              changed = true;\n            }\n            args[t] = resolvedTypeArgument;\n          }\n        }\n\n        return changed\n            ? new ParameterizedTypeImpl(newOwnerType, original.getRawType(), args)\n            : original;\n\n      } else if (toResolve instanceof WildcardType) {\n        WildcardType original = (WildcardType) toResolve;\n        Type[] originalLowerBound = original.getLowerBounds();\n        Type[] originalUpperBound = original.getUpperBounds();\n\n        if (originalLowerBound.length == 1) {\n          Type lowerBound = resolve(context, contextRawType, originalLowerBound[0]);\n          if (lowerBound != originalLowerBound[0]) {\n            return new WildcardTypeImpl(new Type[] {Object.class}, new Type[] {lowerBound});\n          }\n        } else if (originalUpperBound.length == 1) {\n          Type upperBound = resolve(context, contextRawType, originalUpperBound[0]);\n          if (upperBound != originalUpperBound[0]) {\n            return new WildcardTypeImpl(new Type[] {upperBound}, EMPTY_TYPE_ARRAY);\n          }\n        }\n        return original;\n\n      } else {\n        return toResolve;\n      }\n    }\n  }\n\n  \nprivate static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // We can't reduce this further.\n    if (declaredByRaw == null) return unknown;\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static @Nullable Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n\n  static void checkNotPrimitive(Type type) {\n    if (type instanceof Class<?> && ((Class<?>) type).isPrimitive()) {\n      throw new IllegalArgumentException();\n    }\n  }\n\n  /** Returns true if {@code annotations} contains an instance of {@code cls}. */\n  static boolean isAnnotationPresent(Annotation[] annotations, Class<? extends Annotation> cls) {\n    for (Annotation annotation : annotations) {\n      if (cls.isInstance(annotation)) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  static ResponseBody buffer(final ResponseBody body) throws IOException {\n    Buffer buffer = new Buffer();\n    body.source().readAll(buffer);\n    return ResponseBody.create(body.contentType(), body.contentLength(), buffer);\n  }\n\n  static Type getParameterUpperBound(int index, ParameterizedType type) {\n    Type[] types = type.getActualTypeArguments();\n    if (index < 0 || index >= types.length) {\n      throw new IllegalArgumentException(\n          \"Index \" + index + \" not in range [0,\" + types.length + \") for \" + type);\n    }\n    Type paramType = types[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getUpperBounds()[0];\n    }\n    return paramType;\n  }\n\n  static Type getParameterLowerBound(int index, ParameterizedType type) {\n    Type paramType = type.getActualTypeArguments()[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getLowerBounds()[0];\n    }\n    return paramType;\n  }\n\n  static boolean hasUnresolvableType(@Nullable Type type) {\n    if (type instanceof Class<?>) {\n      return false;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n      for (Type typeArgument : parameterizedType.getActualTypeArguments()) {\n        if (hasUnresolvableType(typeArgument)) {\n          return true;\n        }\n      }\n      return false;\n    }\n    if (type instanceof GenericArrayType) {\n      return hasUnresolvableType(((GenericArrayType) type).getGenericComponentType());\n    }\n    if (type instanceof TypeVariable) {\n      return true;\n    }\n    if (type instanceof WildcardType) {\n      return true;\n    }\n    String className = type == null ? \"null\" : type.getClass().getName();\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + className);\n  }\n\n  static final class ParameterizedTypeImpl implements ParameterizedType {\n    private final @Nullable Type ownerType;\n    private final Type rawType;\n    private final Type[] typeArguments;\n\n    ParameterizedTypeImpl(@Nullable Type ownerType, Type rawType, Type... typeArguments) {\n      // Require an owner type if the raw type needs it.\n      if (rawType instanceof Class<?>\n          && (ownerType == null) != (((Class<?>) rawType).getEnclosingClass() == null)) {\n        throw new IllegalArgumentException();\n      }\n\n      for (Type typeArgument : typeArguments) {\n        Objects.requireNonNull(typeArgument, \"typeArgument == null\");\n        checkNotPrimitive(typeArgument);\n      }\n\n      this.ownerType = ownerType;\n      this.rawType = rawType;\n      this.typeArguments = typeArguments.clone();\n    }\n\n    @Override\n    public Type[] getActualTypeArguments() {\n      return typeArguments.clone();\n    }\n\n    @Override\n    public Type getRawType() {\n      return rawType;\n    }\n\n    @Override\n    public @Nullable Type getOwnerType() {\n      return ownerType;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof ParameterizedType && Utils.equals(this, (ParameterizedType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      return Arrays.hashCode(typeArguments)\n          ^ rawType.hashCode()\n          ^ (ownerType != null ? ownerType.hashCode() : 0);\n    }\n\n    @Override\n    public String toString() {\n      if (typeArguments.length == 0) return typeToString(rawType);\n      StringBuilder result = new StringBuilder(30 * (typeArguments.length + 1));\n      result.append(typeToString(rawType));\n      result.append(\"<\").append(typeToString(typeArguments[0]));\n      for (int i = 1; i < typeArguments.length; i++) {\n        result.append(\", \").append(typeToString(typeArguments[i]));\n      }\n      return result.append(\">\").toString();\n    }\n  }\n\n  private static final class GenericArrayTypeImpl implements GenericArrayType {\n    private final Type componentType;\n\n    GenericArrayTypeImpl(Type componentType) {\n      this.componentType = componentType;\n    }\n\n    @Override\n    public Type getGenericComponentType() {\n      return componentType;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      return o instanceof GenericArrayType && Utils.equals(this, (GenericArrayType) o);\n    }\n\n    @Override\n    public int hashCode() {\n      return componentType.hashCode();\n    }\n\n    @Override\n    public String toString() {\n      return typeToString(componentType) + \"[]\";\n    }\n  }\n\n  /**\n   * The WildcardType interface supports multiple upper bounds and multiple lower bounds. We only\n   * support what the Java 6 language needs - at most one bound. If a lower bound is set, the upper\n   * bound must be Object.class.\n   */\n  private static final class WildcardTypeImpl implements WildcardType {\n    private final Type upperBound;\n    private final @Nullable Type lowerBound;\n\n    WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) {\n      if (lowerBounds.length > 1) throw new IllegalArgumentException();\n      if (upperBounds.length != 1) throw new IllegalArgumentException();\n\n      if (lowerBounds.length == 1) {\n        if (lowerBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(lowerBounds[0]);\n        if (upperBounds[0] != Object.class) throw new IllegalArgumentException();\n        this.lowerBound = lowerBounds[0];\n        this.upperBound = Object.class;\n      } else {\n        if (upperBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(upperBounds[0]);\n        this.lowerBound = null;\n        this.upperBound = upperBounds[0];\n      }\n    }\n\n    @Override\n    public Type[] getUpperBounds() {\n      return new Type[] {upperBound};\n    }\n\n    @Override\n    public Type[] getLowerBounds() {\n      return lowerBound != null ? new Type[] {lowerBound} : EMPTY_TYPE_ARRAY;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof WildcardType && Utils.equals(this, (WildcardType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      // This equals Arrays.hashCode(getLowerBounds()) ^ Arrays.hashCode(getUpperBounds()).\n      return (lowerBound != null ? 31 + lowerBound.hashCode() : 1) ^ (31 + upperBound.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      if (lowerBound != null) return \"? super \" + typeToString(lowerBound);\n      if (upperBound == Object.class) return \"?\";\n      return \"? extends \" + typeToString(upperBound);\n    }\n  }\n\n  // https://github.com/ReactiveX/RxJava/blob/6a44e5d0543a48f1c378dc833a155f3f71333bc2/\n  // src/main/java/io/reactivex/exceptions/Exceptions.java#L66\n  static void throwIfFatal(Throwable t) {\n    if (t instanceof VirtualMachineError) {\n      throw (VirtualMachineError) t;\n    } else if (t instanceof ThreadDeath) {\n      throw (ThreadDeath) t;\n    } else if (t instanceof LinkageError) {\n      throw (LinkageError) t;\n    }\n  }\n\n  /** Not volatile because we don't mind multiple threads discovering this. */\n  private static boolean checkForKotlinUnit = true;\n\n  static boolean isUnit(Type type) {\n    if (checkForKotlinUnit) {\n      try {\n        return type == Unit.class;\n      } catch (NoClassDefFoundError ignored) {\n        checkForKotlinUnit = false;\n      }\n    }\n    return false;\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/OkHttpCall.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.throwIfFatal;\n\nimport java.io.IOException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport javax.annotation.concurrent.GuardedBy;\nimport okhttp3.MediaType;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\nimport okio.BufferedSource;\nimport okio.ForwardingSource;\nimport okio.Okio;\nimport okio.Timeout;\n\nfinal class OkHttpCall<T> implements Call<T> {\n  private final RequestFactory requestFactory;\n  private final Object instance;\n  private final Object[] args;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, T> responseConverter;\n\n  private volatile boolean canceled;\n\n  @GuardedBy(\"this\")\n  private @Nullable okhttp3.Call rawCall;\n\n  @GuardedBy(\"this\") // Either a RuntimeException, non-fatal Error, or IOException.\n  private @Nullable Throwable creationFailure;\n\n  @GuardedBy(\"this\")\n  private boolean executed;\n\n  OkHttpCall(\n      RequestFactory requestFactory,\n      Object instance,\n      Object[] args,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, T> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.instance = instance;\n    this.args = args;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @SuppressWarnings(\"CloneDoesntCallSuperClone\") // We are a final type & this saves clearing state.\n  @Override\n  public OkHttpCall<T> clone() {\n    return new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n  }\n\n  @Override\n  public synchronized Request request() {\n    try {\n      return getRawCall().request();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create request.\", e);\n    }\n  }\n\n  @Override\n  public synchronized Timeout timeout() {\n    try {\n      return getRawCall().timeout();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create call.\", e);\n    }\n  }\n\n  /**\n   * Returns the raw call, initializing it if necessary. Throws if initializing the raw call throws,\n   * or has thrown in previous attempts to create it.\n   */\n  @GuardedBy(\"this\")\n  private okhttp3.Call getRawCall() throws IOException {\n    okhttp3.Call call = rawCall;\n    if (call != null) return call;\n\n    // Re-throw previous failures if this isn't the first attempt.\n    if (creationFailure != null) {\n      if (creationFailure instanceof IOException) {\n        throw (IOException) creationFailure;\n      } else if (creationFailure instanceof RuntimeException) {\n        throw (RuntimeException) creationFailure;\n      } else {\n        throw (Error) creationFailure;\n      }\n    }\n\n    // Create and remember either the success or the failure.\n    try {\n      return rawCall = createRawCall();\n    } catch (RuntimeException | Error | IOException e) {\n      throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n      creationFailure = e;\n      throw e;\n    }\n  }\n\n  @Override\n  public void enqueue(final Callback<T> callback) {\n    Objects.requireNonNull(callback, \"callback == null\");\n\n    okhttp3.Call call;\n    Throwable failure;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = rawCall;\n      failure = creationFailure;\n      if (call == null && failure == null) {\n        try {\n          call = rawCall = createRawCall();\n        } catch (Throwable t) {\n          throwIfFatal(t);\n          failure = creationFailure = t;\n        }\n      }\n    }\n\n    if (failure != null) {\n      callback.onFailure(this, failure);\n      return;\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    call.enqueue(\n        new okhttp3.Callback() {\n          @Override\n          public void onResponse(okhttp3.Call call, okhttp3.Response rawResponse) {\n            Response<T> response;\n            try {\n              response = parseResponse(rawResponse);\n            } catch (Throwable e) {\n              throwIfFatal(e);\n              callFailure(e);\n              return;\n            }\n\n            try {\n              callback.onResponse(OkHttpCall.this, response);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n\n          @Override\n          public void onFailure(okhttp3.Call call, IOException e) {\n            callFailure(e);\n          }\n\n          private void callFailure(Throwable e) {\n            try {\n              callback.onFailure(OkHttpCall.this, e);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n        });\n  }\n\n  @Override\n  public synchronized boolean isExecuted() {\n    return executed;\n  }\n\n  @Override\n  public Response<T> execute() throws IOException {\n    okhttp3.Call call;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = getRawCall();\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    return parseResponse(call.execute());\n  }\n\n  private okhttp3.Call createRawCall() throws IOException {\n    okhttp3.Call call = callFactory.newCall(requestFactory.create(instance, args));\n    if (call == null) {\n      throw new NullPointerException(\"Call.Factory returned null.\");\n    }\n    return call;\n  }\n\n  Response<T> parseResponse(okhttp3.Response rawResponse) throws IOException {\n    ResponseBody rawBody = rawResponse.body();\n\n    // Remove the body's source (the only stateful object) so we can pass the response along.\n    rawResponse =\n        rawResponse\n            .newBuilder()\n            .body(new NoContentResponseBody(rawBody.contentType(), rawBody.contentLength()))\n            .build();\n\n    int code = rawResponse.code();\n    if (code < 200 || code >= 300) {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        ResponseBody bufferedBody = Utils.buffer(rawBody);\n        return Response.error(bufferedBody, rawResponse);\n      } finally {\n        rawBody.close();\n      }\n    }\n\n    if (code == 204 || code == 205) {\n      rawBody.close();\n      return Response.success(null, rawResponse);\n    }\n\n    ExceptionCatchingResponseBody catchingBody = new ExceptionCatchingResponseBody(rawBody);\n    try {\n      T body = responseConverter.convert(catchingBody);\n      return Response.success(body, rawResponse);\n    } catch (RuntimeException e) {\n      // If the underlying source threw an exception, propagate that rather than indicating it was\n      // a runtime exception.\n      catchingBody.throwIfCaught();\n      throw e;\n    }\n  }\n\n  @Override\n  public void cancel() {\n    canceled = true;\n\n    okhttp3.Call call;\n    synchronized (this) {\n      call = rawCall;\n    }\n    if (call != null) {\n      call.cancel();\n    }\n  }\n\n  @Override\n  public boolean isCanceled() {\n    if (canceled) {\n      return true;\n    }\n    synchronized (this) {\n      return rawCall != null && rawCall.isCanceled();\n    }\n  }\n\n  static final class NoContentResponseBody extends ResponseBody {\n    private final @Nullable MediaType contentType;\n    private final long contentLength;\n\n    NoContentResponseBody(@Nullable MediaType contentType, long contentLength) {\n      this.contentType = contentType;\n      this.contentLength = contentLength;\n    }\n\n    @Override\n    public MediaType contentType() {\n      return contentType;\n    }\n\n    @Override\n    public long contentLength() {\n      return contentLength;\n    }\n\n    @Override\n    public BufferedSource source() {\n      throw new IllegalStateException(\"Cannot read raw response body of a converted body.\");\n    }\n  }\n\n  static final class ExceptionCatchingResponseBody extends ResponseBody {\n    private final ResponseBody delegate;\n    private final BufferedSource delegateSource;\n    @Nullable IOException thrownException;\n\n    ExceptionCatchingResponseBody(ResponseBody delegate) {\n      this.delegate = delegate;\n      this.delegateSource =\n          Okio.buffer(\n              new ForwardingSource(delegate.source()) {\n                @Override\n                public long read(Buffer sink, long byteCount) throws IOException {\n                  try {\n                    return super.read(sink, byteCount);\n                  } catch (IOException e) {\n                    thrownException = e;\n                    throw e;\n                  }\n                }\n              });\n    }\n\n    @Override\n    public MediaType contentType() {\n      return delegate.contentType();\n    }\n\n    @Override\n    public long contentLength() {\n      return delegate.contentLength();\n    }\n\n    @Override\n    public BufferedSource source() {\n      return delegateSource;\n    }\n\n    @Override\n    public void close() {\n      delegate.close();\n    }\n\n    void throwIfCaught() throws IOException {\n      if (thrownException != null) {\n        throw thrownException;\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Streaming.java\n/*\n * Copyright (C) 2014 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.ResponseBody;\n\n/**\n * Treat the response body on methods returning {@link ResponseBody ResponseBody} as is, i.e.\n * without converting the body to {@code byte[]}.\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface Streaming {}\n\n// Path: retrofit/src/main/java/retrofit2/BuiltInConverters.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.Streaming;\n\nfinal class BuiltInConverters extends Converter.Factory {\n\n  @Override\n  public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n      Type type, Annotation[] annotations, Retrofit retrofit) {\n    if (type == ResponseBody.class) {\n      return Utils.isAnnotationPresent(annotations, Streaming.class)\n          ? StreamingResponseBodyConverter.INSTANCE\n          : BufferingResponseBodyConverter.INSTANCE;\n    }\n    if (type == Void.class) {\n      return VoidResponseBodyConverter.INSTANCE;\n    }\n    if (Utils.isUnit(type)) {\n      return UnitResponseBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  @Override\n  public @Nullable Converter<?, RequestBody> requestBodyConverter(\n      Type type,\n      Annotation[] parameterAnnotations,\n      Annotation[] methodAnnotations,\n      Retrofit retrofit) {\n    if (RequestBody.class.isAssignableFrom(Utils.getRawType(type))) {\n      return RequestBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  static final class VoidResponseBodyConverter implements Converter<ResponseBody, Void> {\n    static final VoidResponseBodyConverter INSTANCE = new VoidResponseBodyConverter();\n\n    @Override\n    public Void convert(ResponseBody value) {\n      value.close();\n      return null;\n    }\n  }\n\n  static final class UnitResponseBodyConverter implements Converter<ResponseBody, Unit> {\n    static final UnitResponseBodyConverter INSTANCE = new UnitResponseBodyConverter();\n\n    @Override\n    public Unit convert(ResponseBody value) {\n      value.close();\n      return Unit.INSTANCE;\n    }\n  }\n\n  static final class RequestBodyConverter implements Converter<RequestBody, RequestBody> {\n    static final RequestBodyConverter INSTANCE = new RequestBodyConverter();\n\n    @Override\n    public RequestBody convert(RequestBody value) {\n      return value;\n    }\n  }\n\n  static final class StreamingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final StreamingResponseBodyConverter INSTANCE = new StreamingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) {\n      return value;\n    }\n  }\n\n  static final class BufferingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final BufferingResponseBodyConverter INSTANCE = new BufferingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) throws IOException {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        return Utils.buffer(value);\n      } finally {\n        value.close();\n      }\n    }\n  }\n\n  static final class ToStringConverter implements Converter<Object, String> {\n    static final ToStringConverter INSTANCE = new ToStringConverter();\n\n    @Override\n    public String convert(Object value) {\n      return value.toString();\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/SkipCallbackExecutorImpl.java\n/*\n * Copyright (C) 2019 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.annotation.Annotation;\n\n// This class conforms to the annotation requirements documented on Annotation.\nfinal class SkipCallbackExecutorImpl implements SkipCallbackExecutor {\n  private static final SkipCallbackExecutor INSTANCE = new SkipCallbackExecutorImpl();\n\n  static Annotation[] ensurePresent(Annotation[] annotations) {\n    if (Utils.isAnnotationPresent(annotations, SkipCallbackExecutor.class)) {\n      return annotations;\n    }\n\n    Annotation[] newAnnotations = new Annotation[annotations.length + 1];\n    // Place the skip annotation first since we're guaranteed to check for it in the call adapter.\n    newAnnotations[0] = SkipCallbackExecutorImpl.INSTANCE;\n    System.arraycopy(annotations, 0, newAnnotations, 1, annotations.length);\n    return newAnnotations;\n  }\n\n  @Override\n  public Class<? extends Annotation> annotationType() {\n    return SkipCallbackExecutor.class;\n  }\n\n  @Override\n  public boolean equals(Object obj) {\n    return obj instanceof SkipCallbackExecutor;\n  }\n\n  @Override\n  public int hashCode() {\n    return 0;\n  }\n\n  @Override\n  public String toString() {\n    return \"@\" + SkipCallbackExecutor.class.getName() + \"()\";\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/HttpServiceMethod.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.getRawType;\nimport static retrofit2.Utils.methodError;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport kotlin.coroutines.Continuation;\nimport okhttp3.ResponseBody;\n\n/** Adapts an invocation of an interface method into an HTTP call. */\nabstract class HttpServiceMethod<ResponseT, ReturnT> extends ServiceMethod<ReturnT> {\n  /**\n   * Inspects the annotations on an interface method to construct a reusable service method that\n   * speaks HTTP. This requires potentially-expensive reflection so it is best to build each service\n   * method only once and reuse it.\n   */\n  static <ResponseT, ReturnT> HttpServiceMethod<ResponseT, ReturnT> parseAnnotations(\n      Retrofit retrofit, Method method, RequestFactory requestFactory) {\n    boolean isKotlinSuspendFunction = requestFactory.isKotlinSuspendFunction;\n    boolean continuationWantsResponse = false;\n    boolean continuationBodyNullable = false;\n    boolean continuationIsUnit = false;\n\n    Annotation[] annotations = method.getAnnotations();\n    Type adapterType;\n    if (isKotlinSuspendFunction) {\n      Type[] parameterTypes = method.getGenericParameterTypes();\n      Type responseType =\n          Utils.getParameterLowerBound(\n              0, (ParameterizedType) parameterTypes[parameterTypes.length - 1]);\n      if (getRawType(responseType) == Response.class && responseType instanceof ParameterizedType) {\n        // Unwrap the actual body type from Response<T>.\n        responseType = Utils.getParameterUpperBound(0, (ParameterizedType) responseType);\n        continuationWantsResponse = true;\n      } else {\n        if (getRawType(responseType) == Call.class) {\n          throw methodError(\n              method,\n              \"Suspend functions should not return Call, as they already execute asynchronously.\\n\"\n                  + \"Change its return type to %s\",\n              Utils.getParameterUpperBound(0, (ParameterizedType) responseType));\n        }\n\n        continuationIsUnit = Utils.isUnit(responseType);\n        // TODO figure out if type is nullable or not\n        // Metadata metadata = method.getDeclaringClass().getAnnotation(Metadata.class)\n        // Find the entry for method\n        // Determine if return type is nullable or not\n      }\n\n      adapterType = new Utils.ParameterizedTypeImpl(null, Call.class, responseType);\n      annotations = SkipCallbackExecutorImpl.ensurePresent(annotations);\n    } else {\n      adapterType = method.getGenericReturnType();\n    }\n\n    CallAdapter<ResponseT, ReturnT> callAdapter =\n        createCallAdapter(retrofit, method, adapterType, annotations);\n    Type responseType = callAdapter.responseType();\n    if (responseType == okhttp3.Response.class) {\n      throw methodError(\n          method,\n          \"'\"\n              + getRawType(responseType).getName()\n              + \"' is not a valid response body type. Did you mean ResponseBody?\");\n    }\n    if (responseType == Response.class) {\n      throw methodError(method, \"Response must include generic type (e.g., Response<String>)\");\n    }\n    if (requestFactory.httpMethod.equals(\"HEAD\")\n        && !Void.class.equals(responseType)\n        && !Utils.isUnit(responseType)) {\n      throw methodError(method, \"HEAD method must use Void or Unit as response type.\");\n    }\n\n    Converter<ResponseBody, ResponseT> responseConverter =\n        createResponseConverter(retrofit, method, responseType);\n\n    okhttp3.Call.Factory callFactory = retrofit.callFactory;\n    if (!isKotlinSuspendFunction) {\n      return new CallAdapted<>(requestFactory, callFactory, responseConverter, callAdapter);\n    } else if (continuationWantsResponse) {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n          new SuspendForResponse<>(\n              requestFactory,\n              callFactory,\n              responseConverter,\n              (CallAdapter<ResponseT, Call<ResponseT>>) callAdapter);\n    } else {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n          new SuspendForBody<>(\n              requestFactory,\n              callFactory,\n              responseConverter,\n              (CallAdapter<ResponseT, Call<ResponseT>>) callAdapter,\n              continuationBodyNullable,\n              continuationIsUnit);\n    }\n  }\n\n  private static <ResponseT, ReturnT> CallAdapter<ResponseT, ReturnT> createCallAdapter(\n      Retrofit retrofit, Method method, Type returnType, Annotation[] annotations) {\n    try {\n      //noinspection unchecked\n      return (CallAdapter<ResponseT, ReturnT>) retrofit.callAdapter(returnType, annotations);\n    } catch (RuntimeException e) { // Wide exception range because factories are user code.\n      throw methodError(method, e, \"Unable to create call adapter for %s\", returnType);\n    }\n  }\n\n  private static <ResponseT> Converter<ResponseBody, ResponseT> createResponseConverter(\n      Retrofit retrofit, Method method, Type responseType) {\n    Annotation[] annotations = method.getAnnotations();\n    try {\n      return retrofit.responseBodyConverter(responseType, annotations);\n    } catch (RuntimeException e) { // Wide exception range because factories are user code.\n      throw methodError(method, e, \"Unable to create converter for %s\", responseType);\n    }\n  }\n\n  private final RequestFactory requestFactory;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, ResponseT> responseConverter;\n\n  HttpServiceMethod(\n      RequestFactory requestFactory,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, ResponseT> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @Override\n  final @Nullable ReturnT invoke(Object instance, Object[] args) {\n    Call<ResponseT> call =\n        new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n    return adapt(call, args);\n  }\n\n  protected abstract @Nullable ReturnT adapt(Call<ResponseT> call, Object[] args);\n\n  static final class CallAdapted<ResponseT, ReturnT> extends HttpServiceMethod<ResponseT, ReturnT> {\n    private final CallAdapter<ResponseT, ReturnT> callAdapter;\n\n    CallAdapted(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, ReturnT> callAdapter) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n    }\n\n    @Override\n    protected ReturnT adapt(Call<ResponseT> call, Object[] args) {\n      return callAdapter.adapt(call);\n    }\n  }\n\n  static final class SuspendForResponse<ResponseT> extends HttpServiceMethod<ResponseT, Object> {\n    private final CallAdapter<ResponseT, Call<ResponseT>> callAdapter;\n\n    SuspendForResponse(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, Call<ResponseT>> callAdapter) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n    }\n\n    @Override\n    protected Object adapt(Call<ResponseT> call, Object[] args) {\n      call = callAdapter.adapt(call);\n\n      //noinspection unchecked Checked by reflection inside RequestFactory.\n      Continuation<Response<ResponseT>> continuation =\n          (Continuation<Response<ResponseT>>) args[args.length - 1];\n\n      // See SuspendForBody for explanation about this try/catch.\n      try {\n        return KotlinExtensions.awaitResponse(call, continuation);\n      } catch (Exception e) {\n        return KotlinExtensions.suspendAndThrow(e, continuation);\n      }\n    }\n  }\n\n  static final class SuspendForBody<ResponseT> extends HttpServiceMethod<ResponseT, Object> {\n    private final CallAdapter<ResponseT, Call<ResponseT>> callAdapter;\n    private final boolean isNullable;\n    private final boolean isUnit;\n\n    SuspendForBody(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, Call<ResponseT>> callAdapter,\n        boolean isNullable,\n        boolean isUnit) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n      this.isNullable = isNullable;\n      this.isUnit = isUnit;\n    }\n\n    @Override\n    protected Object adapt(Call<ResponseT> call, Object[] args) {\n      call = callAdapter.adapt(call);\n\n      //noinspection unchecked Checked by reflection inside RequestFactory.\n      Continuation<ResponseT> continuation = (Continuation<ResponseT>) args[args.length - 1];\n\n      // Calls to OkHttp Call.enqueue() like those inside await and awaitNullable can sometimes\n      // invoke the supplied callback with an exception before the invoking stack frame can return.\n      // Coroutines will intercept the subsequent invocation of the Continuation and throw the\n      // exception synchronously. A Java Proxy cannot throw checked exceptions without them being\n      // declared on the interface method. To avoid the synchronous checked exception being wrapped\n      // in an UndeclaredThrowableException, it is intercepted and supplied to a helper which will\n      // force suspension to occur so that it can be instead delivered to the continuation to\n      // bypass this restriction.\n      try {\n        if (isUnit) {\n          //noinspection unchecked Checked by isUnit\n          return KotlinExtensions.awaitUnit((Call<Unit>) call, (Continuation<Unit>) continuation);\n        } else if (isNullable) {\n          return KotlinExtensions.awaitNullable(call, continuation);\n        } else {\n          return KotlinExtensions.await(call, continuation);\n        }\n      } catch (VirtualMachineError | ThreadDeath | LinkageError e) {\n        // Do not attempt to capture fatal throwables. This list is derived RxJava's `throwIfFatal`.\n        throw e;\n      } catch (Throwable e) {\n        return KotlinExtensions.suspendAndThrow(e, continuation);\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/ParameterHandler.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.Method;\nimport java.util.Map;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.MultipartBody;\nimport okhttp3.RequestBody;\n\nabstract class ParameterHandler<T> {\n  abstract void apply(RequestBuilder builder, @Nullable T value) throws IOException;\n\n  final ParameterHandler<Iterable<T>> iterable() {\n    return new ParameterHandler<Iterable<T>>() {\n      @Override\n      void apply(RequestBuilder builder, @Nullable Iterable<T> values) throws IOException {\n        if (values == null) return; // Skip null values.\n\n        for (T value : values) {\n          ParameterHandler.this.apply(builder, value);\n        }\n      }\n    };\n  }\n\n  final ParameterHandler<Object> array() {\n    return new ParameterHandler<Object>() {\n      @Override\n      void apply(RequestBuilder builder, @Nullable Object values) throws IOException {\n        if (values == null) return; // Skip null values.\n\n        for (int i = 0, size = Array.getLength(values); i < size; i++) {\n          //noinspection unchecked\n          ParameterHandler.this.apply(builder, (T) Array.get(values, i));\n        }\n      }\n    };\n  }\n\n  static final class RelativeUrl extends ParameterHandler<Object> {\n    private final Method method;\n    private final int p;\n\n    RelativeUrl(Method method, int p) {\n      this.method = method;\n      this.p = p;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Object value) {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"@Url parameter is null.\");\n      }\n      builder.setRelativeUrl(value);\n    }\n  }\n\n  static final class Header<T> extends ParameterHandler<T> {\n    private final String name;\n    private final Converter<T, String> valueConverter;\n    private final boolean allowUnsafeNonAsciiValues;\n\n    Header(String name, Converter<T, String> valueConverter, boolean allowUnsafeNonAsciiValues) {\n      this.name = Objects.requireNonNull(name, \"name == null\");\n      this.valueConverter = valueConverter;\n      this.allowUnsafeNonAsciiValues = allowUnsafeNonAsciiValues;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) return; // Skip null values.\n\n      String headerValue = valueConverter.convert(value);\n      if (headerValue == null) return; // Skip converted but null values.\n\n      builder.addHeader(name, headerValue, allowUnsafeNonAsciiValues);\n    }\n  }\n\n  static final class Path<T> extends ParameterHandler<T> {\n    private final Method method;\n    private final int p;\n    private final String name;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    Path(Method method, int p, String name, Converter<T, String> valueConverter, boolean encoded) {\n      this.method = method;\n      this.p = p;\n      this.name = Objects.requireNonNull(name, \"name == null\");\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(\n            method, p, \"Path parameter \\\"\" + name + \"\\\" value must not be null.\");\n      }\n      builder.addPathParam(name, valueConverter.convert(value), encoded);\n    }\n  }\n\n  static final class Query<T> extends ParameterHandler<T> {\n    private final String name;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    Query(String name, Converter<T, String> valueConverter, boolean encoded) {\n      this.name = Objects.requireNonNull(name, \"name == null\");\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) return; // Skip null values.\n\n      String queryValue = valueConverter.convert(value);\n      if (queryValue == null) return; // Skip converted but null values\n\n      builder.addQueryParam(name, queryValue, encoded);\n    }\n  }\n\n  static final class QueryName<T> extends ParameterHandler<T> {\n    private final Converter<T, String> nameConverter;\n    private final boolean encoded;\n\n    QueryName(Converter<T, String> nameConverter, boolean encoded) {\n      this.nameConverter = nameConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable T value) throws IOException {\n      if (value == null) return; // Skip null values.\n      builder.addQueryParam(nameConverter.convert(value), null, encoded);\n    }\n  }\n\n  static final class QueryMap<T> extends ParameterHandler<Map<String, T>> {\n    private final Method method;\n    private final int p;\n    private final Converter<T, String> valueConverter;\n    private final boolean encoded;\n\n    QueryMap(Method method, int p, Converter<T, String> valueConverter, boolean encoded) {\n      this.method = method;\n      this.p = p;\n      this.valueConverter = valueConverter;\n      this.encoded = encoded;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Map<String, T> value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"Query map was null\");\n      }\n\n      for (Map.Entry<String, T> entry : value.entrySet()) {\n        String entryKey = entry.getKey();\n        if (entryKey == null) {\n          throw Utils.parameterError(method, p, \"Query map contained null key.\");\n        }\n        T entryValue = entry.getValue();\n        if (entryValue == null) {\n          throw Utils.parameterError(\n              method, p, \"Query map contained null value for key '\" + entryKey + \"'.\");\n        }\n\n        String convertedEntryValue = valueConverter.convert(entryValue);\n        if (convertedEntryValue == null) {\n          throw Utils.parameterError(\n              method,\n              p,\n              \"Query map value '\"\n                  + entryValue\n                  + \"' converted to null by \"\n                  + valueConverter.getClass().getName()\n                  + \" for key '\"\n                  + entryKey\n                  + \"'.\");\n        }\n\n        builder.addQueryParam(entryKey, convertedEntryValue, encoded);\n      }\n    }\n  }\n\n  static final class HeaderMap<T> extends ParameterHandler<Map<String, T>> {\n    private final Method method;\n    private final int p;\n    private final Converter<T, String> valueConverter;\n    private final boolean allowUnsafeNonAsciiValues;\n\n    HeaderMap(\n        Method method,\n        int p,\n        Converter<T, String> valueConverter,\n        boolean allowUnsafeNonAsciiValues) {\n      this.method = method;\n      this.p = p;\n      this.valueConverter = valueConverter;\n      this.allowUnsafeNonAsciiValues = allowUnsafeNonAsciiValues;\n    }\n\n    @Override\n    void apply(RequestBuilder builder, @Nullable Map<String, T> value) throws IOException {\n      if (value == null) {\n        throw Utils.parameterError(method, p, \"Header map was null.\");\n      }\n\n      for (Map.Entry<String, T> entry : value.entrySet()) {\n        String headerName = entry.getKey();\n        if (headerName == null) {\n...\n", "needle_token_start": 3947, "needle_token_end": 4103, "code_context_ntokens": 16043, "output": ["```java\nprivate static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // We can't reduce this further.\n    if (declaredByRaw == null) return unknown;\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static @Nullable Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n```"]}
{"repo": "square/retrofit", "name": "getGenericSupertype", "language": "java", "path": "retrofit/src/main/java/retrofit2/Utils.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the specific generic type of a superclass or interface that a given class extends or implements. This is useful for understanding the type relationships and hierarchies in generic programming.\n2. **Input**: The function takes three parameters: a type context representing the starting type, a class representing the raw type of the context, and a class that the function attempts to resolve against the context.\n3. **Output**: It returns a `Type` object that represents the generic supertype of the specified class, detailing the type parameters involved.\n4. **Procedure**: \n   - The function first checks if the class to resolve is the same as the raw type of the context; if so, it returns the context.\n   - If the class to resolve is an interface, the function iterates through the interfaces implemented by the raw type to find a match or a relevant assignable type, recursively resolving as needed.\n   - If the class to resolve is not an interface, the function traverses up the superclass chain of the raw type to find a match or a relevant assignable superclass, again resolving recursively as necessary.\n   - If no matching type is found after checking interfaces and superclasses, the function returns the class to resolve as a last resort.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/Platform.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static android.os.Build.VERSION.SDK_INT;\n\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\n\nfinal class Platform {\n  static final @Nullable Executor callbackExecutor;\n  static final Reflection reflection;\n  static final BuiltInFactories builtInFactories;\n\n  static {\n    switch (System.getProperty(\"java.vm.name\")) {\n      case \"Dalvik\":\n        callbackExecutor = new AndroidMainExecutor();\n        if (SDK_INT >= 24) {\n          reflection = new Reflection.Android24();\n          builtInFactories = new BuiltInFactories.Java8();\n        } else {\n          reflection = new Reflection();\n          builtInFactories = new BuiltInFactories();\n        }\n        break;\n\n      case \"RoboVM\":\n        callbackExecutor = null;\n        reflection = new Reflection();\n        builtInFactories = new BuiltInFactories();\n        break;\n\n      default:\n        callbackExecutor = null;\n        reflection = new Reflection.Java8();\n        builtInFactories = new BuiltInFactories.Java8();\n        break;\n    }\n  }\n\n  private Platform() {}\n}\n\n// Path: retrofit/src/main/java/retrofit2/Response.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.Headers;\nimport okhttp3.Protocol;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\n\n/** An HTTP response. */\npublic final class Response<T> {\n  /** Create a synthetic successful response with {@code body} as the deserialized body. */\n  public static <T> Response<T> success(@Nullable T body) {\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response with an HTTP status code of {@code code} and {@code\n   * body} as the deserialized body.\n   */\n  public static <T> Response<T> success(int code, @Nullable T body) {\n    if (code < 200 || code >= 300) {\n      throw new IllegalArgumentException(\"code < 200 or >= 300: \" + code);\n    }\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(code)\n            .message(\"Response.success()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response using {@code headers} with {@code body} as the\n   * deserialized body.\n   */\n  public static <T> Response<T> success(@Nullable T body, Headers headers) {\n    Objects.requireNonNull(headers, \"headers == null\");\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .headers(headers)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a successful response from {@code rawResponse} with {@code body} as the deserialized\n   * body.\n   */\n  public static <T> Response<T> success(@Nullable T body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (!rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse must be successful response\");\n    }\n    return new Response<>(rawResponse, body, null);\n  }\n\n  /**\n   * Create a synthetic error response with an HTTP status code of {@code code} and {@code body} as\n   * the error body.\n   */\n  public static <T> Response<T> error(int code, ResponseBody body) {\n    Objects.requireNonNull(body, \"body == null\");\n    if (code < 400) throw new IllegalArgumentException(\"code < 400: \" + code);\n    return error(\n        body,\n        new okhttp3.Response.Builder() //\n            .body(new OkHttpCall.NoContentResponseBody(body.contentType(), body.contentLength()))\n            .code(code)\n            .message(\"Response.error()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /** Create an error response from {@code rawResponse} with {@code body} as the error body. */\n  public static <T> Response<T> error(ResponseBody body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(body, \"body == null\");\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse should not be successful response\");\n    }\n    return new Response<>(rawResponse, null, body);\n  }\n\n  private final okhttp3.Response rawResponse;\n  private final @Nullable T body;\n  private final @Nullable ResponseBody errorBody;\n\n  private Response(\n      okhttp3.Response rawResponse, @Nullable T body, @Nullable ResponseBody errorBody) {\n    this.rawResponse = rawResponse;\n    this.body = body;\n    this.errorBody = errorBody;\n  }\n\n  /** The raw response from the HTTP client. */\n  public okhttp3.Response raw() {\n    return rawResponse;\n  }\n\n  /** HTTP status code. */\n  public int code() {\n    return rawResponse.code();\n  }\n\n  /** HTTP status message or null if unknown. */\n  public String message() {\n    return rawResponse.message();\n  }\n\n  /** HTTP headers. */\n  public Headers headers() {\n    return rawResponse.headers();\n  }\n\n  /** Returns true if {@link #code()} is in the range [200..300). */\n  public boolean isSuccessful() {\n    return rawResponse.isSuccessful();\n  }\n\n  /** The deserialized response body of a {@linkplain #isSuccessful() successful} response. */\n  public @Nullable T body() {\n    return body;\n  }\n\n  /** The raw response body of an {@linkplain #isSuccessful() unsuccessful} response. */\n  public @Nullable ResponseBody errorBody() {\n    return errorBody;\n  }\n\n  @Override\n  public String toString() {\n    return rawResponse.toString();\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Utils.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\n\nfinal class Utils {\n  static final Type[] EMPTY_TYPE_ARRAY = new Type[0];\n\n  private Utils() {\n    // No instances.\n  }\n\n  static RuntimeException methodError(Method method, String message, Object... args) {\n    return methodError(method, null, message, args);\n  }\n\n  @SuppressWarnings(\"AnnotateFormatMethod\")\n  static RuntimeException methodError(\n      Method method, @Nullable Throwable cause, String message, Object... args) {\n    message = String.format(message, args);\n    return new IllegalArgumentException(\n        message\n            + \"\\n    for method \"\n            + method.getDeclaringClass().getSimpleName()\n            + \".\"\n            + method.getName(),\n        cause);\n  }\n\n  static RuntimeException parameterError(\n      Method method, Throwable cause, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, cause, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static RuntimeException parameterError(Method method, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static Class<?> getRawType(Type type) {\n    Objects.requireNonNull(type, \"type == null\");\n\n    if (type instanceof Class<?>) {\n      // Type is a normal class.\n      return (Class<?>) type;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n\n      // I'm not exactly sure why getRawType() returns Type instead of Class. Neal isn't either but\n      // suspects some pathological case related to nested classes exists.\n      Type rawType = parameterizedType.getRawType();\n      if (!(rawType instanceof Class)) throw new IllegalArgumentException();\n      return (Class<?>) rawType;\n    }\n    if (type instanceof GenericArrayType) {\n      Type componentType = ((GenericArrayType) type).getGenericComponentType();\n      return Array.newInstance(getRawType(componentType), 0).getClass();\n    }\n    if (type instanceof TypeVariable) {\n      // We could use the variable's bounds, but that won't work if there are multiple. Having a raw\n      // type that's more general than necessary is okay.\n      return Object.class;\n    }\n    if (type instanceof WildcardType) {\n      return getRawType(((WildcardType) type).getUpperBounds()[0]);\n    }\n\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + type.getClass().getName());\n  }\n\n  /** Returns true if {@code a} and {@code b} are equal. */\n  static boolean equals(Type a, Type b) {\n    if (a == b) {\n      return true; // Also handles (a == null && b == null).\n\n    } else if (a instanceof Class) {\n      return a.equals(b); // Class already specifies equals().\n\n    } else if (a instanceof ParameterizedType) {\n      if (!(b instanceof ParameterizedType)) return false;\n      ParameterizedType pa = (ParameterizedType) a;\n      ParameterizedType pb = (ParameterizedType) b;\n      Object ownerA = pa.getOwnerType();\n      Object ownerB = pb.getOwnerType();\n      boolean ownersAreEqual = ownerA == ownerB || (ownerA != null && ownerA.equals(ownerB));\n      boolean rawTypesAreEqual = pa.getRawType().equals(pb.getRawType());\n      boolean typeArgumentsAreEqual =\n          Arrays.equals(pa.getActualTypeArguments(), pb.getActualTypeArguments());\n\n      return ownersAreEqual && rawTypesAreEqual && typeArgumentsAreEqual;\n\n    } else if (a instanceof GenericArrayType) {\n      if (!(b instanceof GenericArrayType)) return false;\n      GenericArrayType ga = (GenericArrayType) a;\n      GenericArrayType gb = (GenericArrayType) b;\n      return equals(ga.getGenericComponentType(), gb.getGenericComponentType());\n\n    } else if (a instanceof WildcardType) {\n      if (!(b instanceof WildcardType)) return false;\n      WildcardType wa = (WildcardType) a;\n      WildcardType wb = (WildcardType) b;\n      return Arrays.equals(wa.getUpperBounds(), wb.getUpperBounds())\n          && Arrays.equals(wa.getLowerBounds(), wb.getLowerBounds());\n\n    } else if (a instanceof TypeVariable) {\n      if (!(b instanceof TypeVariable)) return false;\n      TypeVariable<?> va = (TypeVariable<?>) a;\n      TypeVariable<?> vb = (TypeVariable<?>) b;\n      return va.getGenericDeclaration() == vb.getGenericDeclaration()\n          && va.getName().equals(vb.getName());\n\n    } else {\n      return false; // This isn't a type we support!\n    }\n  }\n\n  /**\n   * Returns the generic supertype for {@code supertype}. For example, given a class {@code\n   * IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>} and the\n   * result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.\n   */\n  \nstatic Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {\n    if (toResolve == rawType) return context;\n\n    // We skip searching through interfaces if unknown is an interface.\n    if (toResolve.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == toResolve) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (toResolve.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], toResolve);\n        }\n      }\n    }\n\n    // Check our supertypes.\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == toResolve) {\n          return rawType.getGenericSuperclass();\n        } else if (toResolve.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, toResolve);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // We can't resolve this further.\n    return toResolve;\n  }\n\n  private static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0; i < array.length; i++) {\n      if (toFind.equals(array[i])) return i;\n    }\n    throw new NoSuchElementException();\n  }\n\n  static String typeToString(Type type) {\n    return type instanceof Class ? ((Class<?>) type).getName() : type.toString();\n  }\n\n  /**\n   * Returns the generic form of {@code supertype}. For example, if this is {@code\n   * ArrayList<String>}, this returns {@code Iterable<String>} given the input {@code\n   * Iterable.class}.\n   *\n   * @param supertype a superclass of, or interface implemented by, this.\n   */\n  static Type getSupertype(Type context, Class<?> contextRawType, Class<?> supertype) {\n    if (!supertype.isAssignableFrom(contextRawType)) throw new IllegalArgumentException();\n    return resolve(\n        context, contextRawType, getGenericSupertype(context, contextRawType, supertype));\n  }\n\n  static Type resolve(Type context, Class<?> contextRawType, Type toResolve) {\n    // This implementation is made a little more complicated in an attempt to avoid object-creation.\n    while (true) {\n      if (toResolve instanceof TypeVariable) {\n        TypeVariable<?> typeVariable = (TypeVariable<?>) toResolve;\n        toResolve = resolveTypeVariable(context, contextRawType, typeVariable);\n        if (toResolve == typeVariable) {\n          return toResolve;\n        }\n\n      } else if (toResolve instanceof Class && ((Class<?>) toResolve).isArray()) {\n        Class<?> original = (Class<?>) toResolve;\n        Type componentType = original.getComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof GenericArrayType) {\n        GenericArrayType original = (GenericArrayType) toResolve;\n        Type componentType = original.getGenericComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof ParameterizedType) {\n        ParameterizedType original = (ParameterizedType) toResolve;\n        Type ownerType = original.getOwnerType();\n        Type newOwnerType = resolve(context, contextRawType, ownerType);\n        boolean changed = newOwnerType != ownerType;\n\n        Type[] args = original.getActualTypeArguments();\n        for (int t = 0, length = args.length; t < length; t++) {\n          Type resolvedTypeArgument = resolve(context, contextRawType, args[t]);\n          if (resolvedTypeArgument != args[t]) {\n            if (!changed) {\n              args = args.clone();\n              changed = true;\n            }\n            args[t] = resolvedTypeArgument;\n          }\n        }\n\n        return changed\n            ? new ParameterizedTypeImpl(newOwnerType, original.getRawType(), args)\n            : original;\n\n      } else if (toResolve instanceof WildcardType) {\n        WildcardType original = (WildcardType) toResolve;\n        Type[] originalLowerBound = original.getLowerBounds();\n        Type[] originalUpperBound = original.getUpperBounds();\n\n        if (originalLowerBound.length == 1) {\n          Type lowerBound = resolve(context, contextRawType, originalLowerBound[0]);\n          if (lowerBound != originalLowerBound[0]) {\n            return new WildcardTypeImpl(new Type[] {Object.class}, new Type[] {lowerBound});\n          }\n        } else if (originalUpperBound.length == 1) {\n          Type upperBound = resolve(context, contextRawType, originalUpperBound[0]);\n          if (upperBound != originalUpperBound[0]) {\n            return new WildcardTypeImpl(new Type[] {upperBound}, EMPTY_TYPE_ARRAY);\n          }\n        }\n        return original;\n\n      } else {\n        return toResolve;\n      }\n    }\n  }\n\n  private static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // We can't reduce this further.\n    if (declaredByRaw == null) return unknown;\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static @Nullable Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n\n  static void checkNotPrimitive(Type type) {\n    if (type instanceof Class<?> && ((Class<?>) type).isPrimitive()) {\n      throw new IllegalArgumentException();\n    }\n  }\n\n  /** Returns true if {@code annotations} contains an instance of {@code cls}. */\n  static boolean isAnnotationPresent(Annotation[] annotations, Class<? extends Annotation> cls) {\n    for (Annotation annotation : annotations) {\n      if (cls.isInstance(annotation)) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  static ResponseBody buffer(final ResponseBody body) throws IOException {\n    Buffer buffer = new Buffer();\n    body.source().readAll(buffer);\n    return ResponseBody.create(body.contentType(), body.contentLength(), buffer);\n  }\n\n  static Type getParameterUpperBound(int index, ParameterizedType type) {\n    Type[] types = type.getActualTypeArguments();\n    if (index < 0 || index >= types.length) {\n      throw new IllegalArgumentException(\n          \"Index \" + index + \" not in range [0,\" + types.length + \") for \" + type);\n    }\n    Type paramType = types[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getUpperBounds()[0];\n    }\n    return paramType;\n  }\n\n  static Type getParameterLowerBound(int index, ParameterizedType type) {\n    Type paramType = type.getActualTypeArguments()[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getLowerBounds()[0];\n    }\n    return paramType;\n  }\n\n  static boolean hasUnresolvableType(@Nullable Type type) {\n    if (type instanceof Class<?>) {\n      return false;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n      for (Type typeArgument : parameterizedType.getActualTypeArguments()) {\n        if (hasUnresolvableType(typeArgument)) {\n          return true;\n        }\n      }\n      return false;\n    }\n    if (type instanceof GenericArrayType) {\n      return hasUnresolvableType(((GenericArrayType) type).getGenericComponentType());\n    }\n    if (type instanceof TypeVariable) {\n      return true;\n    }\n    if (type instanceof WildcardType) {\n      return true;\n    }\n    String className = type == null ? \"null\" : type.getClass().getName();\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + className);\n  }\n\n  static final class ParameterizedTypeImpl implements ParameterizedType {\n    private final @Nullable Type ownerType;\n    private final Type rawType;\n    private final Type[] typeArguments;\n\n    ParameterizedTypeImpl(@Nullable Type ownerType, Type rawType, Type... typeArguments) {\n      // Require an owner type if the raw type needs it.\n      if (rawType instanceof Class<?>\n          && (ownerType == null) != (((Class<?>) rawType).getEnclosingClass() == null)) {\n        throw new IllegalArgumentException();\n      }\n\n      for (Type typeArgument : typeArguments) {\n        Objects.requireNonNull(typeArgument, \"typeArgument == null\");\n        checkNotPrimitive(typeArgument);\n      }\n\n      this.ownerType = ownerType;\n      this.rawType = rawType;\n      this.typeArguments = typeArguments.clone();\n    }\n\n    @Override\n    public Type[] getActualTypeArguments() {\n      return typeArguments.clone();\n    }\n\n    @Override\n    public Type getRawType() {\n      return rawType;\n    }\n\n    @Override\n    public @Nullable Type getOwnerType() {\n      return ownerType;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof ParameterizedType && Utils.equals(this, (ParameterizedType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      return Arrays.hashCode(typeArguments)\n          ^ rawType.hashCode()\n          ^ (ownerType != null ? ownerType.hashCode() : 0);\n    }\n\n    @Override\n    public String toString() {\n      if (typeArguments.length == 0) return typeToString(rawType);\n      StringBuilder result = new StringBuilder(30 * (typeArguments.length + 1));\n      result.append(typeToString(rawType));\n      result.append(\"<\").append(typeToString(typeArguments[0]));\n      for (int i = 1; i < typeArguments.length; i++) {\n        result.append(\", \").append(typeToString(typeArguments[i]));\n      }\n      return result.append(\">\").toString();\n    }\n  }\n\n  private static final class GenericArrayTypeImpl implements GenericArrayType {\n    private final Type componentType;\n\n    GenericArrayTypeImpl(Type componentType) {\n      this.componentType = componentType;\n    }\n\n    @Override\n    public Type getGenericComponentType() {\n      return componentType;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      return o instanceof GenericArrayType && Utils.equals(this, (GenericArrayType) o);\n    }\n\n    @Override\n    public int hashCode() {\n      return componentType.hashCode();\n    }\n\n    @Override\n    public String toString() {\n      return typeToString(componentType) + \"[]\";\n    }\n  }\n\n  /**\n   * The WildcardType interface supports multiple upper bounds and multiple lower bounds. We only\n   * support what the Java 6 language needs - at most one bound. If a lower bound is set, the upper\n   * bound must be Object.class.\n   */\n  private static final class WildcardTypeImpl implements WildcardType {\n    private final Type upperBound;\n    private final @Nullable Type lowerBound;\n\n    WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) {\n      if (lowerBounds.length > 1) throw new IllegalArgumentException();\n      if (upperBounds.length != 1) throw new IllegalArgumentException();\n\n      if (lowerBounds.length == 1) {\n        if (lowerBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(lowerBounds[0]);\n        if (upperBounds[0] != Object.class) throw new IllegalArgumentException();\n        this.lowerBound = lowerBounds[0];\n        this.upperBound = Object.class;\n      } else {\n        if (upperBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(upperBounds[0]);\n        this.lowerBound = null;\n        this.upperBound = upperBounds[0];\n      }\n    }\n\n    @Override\n    public Type[] getUpperBounds() {\n      return new Type[] {upperBound};\n    }\n\n    @Override\n    public Type[] getLowerBounds() {\n      return lowerBound != null ? new Type[] {lowerBound} : EMPTY_TYPE_ARRAY;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof WildcardType && Utils.equals(this, (WildcardType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      // This equals Arrays.hashCode(getLowerBounds()) ^ Arrays.hashCode(getUpperBounds()).\n      return (lowerBound != null ? 31 + lowerBound.hashCode() : 1) ^ (31 + upperBound.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      if (lowerBound != null) return \"? super \" + typeToString(lowerBound);\n      if (upperBound == Object.class) return \"?\";\n      return \"? extends \" + typeToString(upperBound);\n    }\n  }\n\n  // https://github.com/ReactiveX/RxJava/blob/6a44e5d0543a48f1c378dc833a155f3f71333bc2/\n  // src/main/java/io/reactivex/exceptions/Exceptions.java#L66\n  static void throwIfFatal(Throwable t) {\n    if (t instanceof VirtualMachineError) {\n      throw (VirtualMachineError) t;\n    } else if (t instanceof ThreadDeath) {\n      throw (ThreadDeath) t;\n    } else if (t instanceof LinkageError) {\n      throw (LinkageError) t;\n    }\n  }\n\n  /** Not volatile because we don't mind multiple threads discovering this. */\n  private static boolean checkForKotlinUnit = true;\n\n  static boolean isUnit(Type type) {\n    if (checkForKotlinUnit) {\n      try {\n        return type == Unit.class;\n      } catch (NoClassDefFoundError ignored) {\n        checkForKotlinUnit = false;\n      }\n    }\n    return false;\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/OkHttpCall.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.throwIfFatal;\n\nimport java.io.IOException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport javax.annotation.concurrent.GuardedBy;\nimport okhttp3.MediaType;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\nimport okio.BufferedSource;\nimport okio.ForwardingSource;\nimport okio.Okio;\nimport okio.Timeout;\n\nfinal class OkHttpCall<T> implements Call<T> {\n  private final RequestFactory requestFactory;\n  private final Object instance;\n  private final Object[] args;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, T> responseConverter;\n\n  private volatile boolean canceled;\n\n  @GuardedBy(\"this\")\n  private @Nullable okhttp3.Call rawCall;\n\n  @GuardedBy(\"this\") // Either a RuntimeException, non-fatal Error, or IOException.\n  private @Nullable Throwable creationFailure;\n\n  @GuardedBy(\"this\")\n  private boolean executed;\n\n  OkHttpCall(\n      RequestFactory requestFactory,\n      Object instance,\n      Object[] args,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, T> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.instance = instance;\n    this.args = args;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @SuppressWarnings(\"CloneDoesntCallSuperClone\") // We are a final type & this saves clearing state.\n  @Override\n  public OkHttpCall<T> clone() {\n    return new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n  }\n\n  @Override\n  public synchronized Request request() {\n    try {\n      return getRawCall().request();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create request.\", e);\n    }\n  }\n\n  @Override\n  public synchronized Timeout timeout() {\n    try {\n      return getRawCall().timeout();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create call.\", e);\n    }\n  }\n\n  /**\n   * Returns the raw call, initializing it if necessary. Throws if initializing the raw call throws,\n   * or has thrown in previous attempts to create it.\n   */\n  @GuardedBy(\"this\")\n  private okhttp3.Call getRawCall() throws IOException {\n    okhttp3.Call call = rawCall;\n    if (call != null) return call;\n\n    // Re-throw previous failures if this isn't the first attempt.\n    if (creationFailure != null) {\n      if (creationFailure instanceof IOException) {\n        throw (IOException) creationFailure;\n      } else if (creationFailure instanceof RuntimeException) {\n        throw (RuntimeException) creationFailure;\n      } else {\n        throw (Error) creationFailure;\n      }\n    }\n\n    // Create and remember either the success or the failure.\n    try {\n      return rawCall = createRawCall();\n    } catch (RuntimeException | Error | IOException e) {\n      throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n      creationFailure = e;\n      throw e;\n    }\n  }\n\n  @Override\n  public void enqueue(final Callback<T> callback) {\n    Objects.requireNonNull(callback, \"callback == null\");\n\n    okhttp3.Call call;\n    Throwable failure;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = rawCall;\n      failure = creationFailure;\n      if (call == null && failure == null) {\n        try {\n          call = rawCall = createRawCall();\n        } catch (Throwable t) {\n          throwIfFatal(t);\n          failure = creationFailure = t;\n        }\n      }\n    }\n\n    if (failure != null) {\n      callback.onFailure(this, failure);\n      return;\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    call.enqueue(\n        new okhttp3.Callback() {\n          @Override\n          public void onResponse(okhttp3.Call call, okhttp3.Response rawResponse) {\n            Response<T> response;\n            try {\n              response = parseResponse(rawResponse);\n            } catch (Throwable e) {\n              throwIfFatal(e);\n              callFailure(e);\n              return;\n            }\n\n            try {\n              callback.onResponse(OkHttpCall.this, response);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n\n          @Override\n          public void onFailure(okhttp3.Call call, IOException e) {\n            callFailure(e);\n          }\n\n          private void callFailure(Throwable e) {\n            try {\n              callback.onFailure(OkHttpCall.this, e);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n        });\n  }\n\n  @Override\n  public synchronized boolean isExecuted() {\n    return executed;\n  }\n\n  @Override\n  public Response<T> execute() throws IOException {\n    okhttp3.Call call;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = getRawCall();\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    return parseResponse(call.execute());\n  }\n\n  private okhttp3.Call createRawCall() throws IOException {\n    okhttp3.Call call = callFactory.newCall(requestFactory.create(instance, args));\n    if (call == null) {\n      throw new NullPointerException(\"Call.Factory returned null.\");\n    }\n    return call;\n  }\n\n  Response<T> parseResponse(okhttp3.Response rawResponse) throws IOException {\n    ResponseBody rawBody = rawResponse.body();\n\n    // Remove the body's source (the only stateful object) so we can pass the response along.\n    rawResponse =\n        rawResponse\n            .newBuilder()\n            .body(new NoContentResponseBody(rawBody.contentType(), rawBody.contentLength()))\n            .build();\n\n    int code = rawResponse.code();\n    if (code < 200 || code >= 300) {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        ResponseBody bufferedBody = Utils.buffer(rawBody);\n        return Response.error(bufferedBody, rawResponse);\n      } finally {\n        rawBody.close();\n      }\n    }\n\n    if (code == 204 || code == 205) {\n      rawBody.close();\n      return Response.success(null, rawResponse);\n    }\n\n    ExceptionCatchingResponseBody catchingBody = new ExceptionCatchingResponseBody(rawBody);\n    try {\n      T body = responseConverter.convert(catchingBody);\n      return Response.success(body, rawResponse);\n    } catch (RuntimeException e) {\n      // If the underlying source threw an exception, propagate that rather than indicating it was\n      // a runtime exception.\n      catchingBody.throwIfCaught();\n      throw e;\n    }\n  }\n\n  @Override\n  public void cancel() {\n    canceled = true;\n\n    okhttp3.Call call;\n    synchronized (this) {\n      call = rawCall;\n    }\n    if (call != null) {\n      call.cancel();\n    }\n  }\n\n  @Override\n  public boolean isCanceled() {\n    if (canceled) {\n      return true;\n    }\n    synchronized (this) {\n      return rawCall != null && rawCall.isCanceled();\n    }\n  }\n\n  static final class NoContentResponseBody extends ResponseBody {\n    private final @Nullable MediaType contentType;\n    private final long contentLength;\n\n    NoContentResponseBody(@Nullable MediaType contentType, long contentLength) {\n      this.contentType = contentType;\n      this.contentLength = contentLength;\n    }\n\n    @Override\n    public MediaType contentType() {\n      return contentType;\n    }\n\n    @Override\n    public long contentLength() {\n      return contentLength;\n    }\n\n    @Override\n    public BufferedSource source() {\n      throw new IllegalStateException(\"Cannot read raw response body of a converted body.\");\n    }\n  }\n\n  static final class ExceptionCatchingResponseBody extends ResponseBody {\n    private final ResponseBody delegate;\n    private final BufferedSource delegateSource;\n    @Nullable IOException thrownException;\n\n    ExceptionCatchingResponseBody(ResponseBody delegate) {\n      this.delegate = delegate;\n      this.delegateSource =\n          Okio.buffer(\n              new ForwardingSource(delegate.source()) {\n                @Override\n                public long read(Buffer sink, long byteCount) throws IOException {\n                  try {\n                    return super.read(sink, byteCount);\n                  } catch (IOException e) {\n                    thrownException = e;\n                    throw e;\n                  }\n                }\n              });\n    }\n\n    @Override\n    public MediaType contentType() {\n      return delegate.contentType();\n    }\n\n    @Override\n    public long contentLength() {\n      return delegate.contentLength();\n    }\n\n    @Override\n    public BufferedSource source() {\n      return delegateSource;\n    }\n\n    @Override\n    public void close() {\n      delegate.close();\n    }\n\n    void throwIfCaught() throws IOException {\n      if (thrownException != null) {\n        throw thrownException;\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Streaming.java\n/*\n * Copyright (C) 2014 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.ResponseBody;\n\n/**\n * Treat the response body on methods returning {@link ResponseBody ResponseBody} as is, i.e.\n * without converting the body to {@code byte[]}.\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface Streaming {}\n\n// Path: retrofit/src/main/java/retrofit2/BuiltInConverters.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.Streaming;\n\nfinal class BuiltInConverters extends Converter.Factory {\n\n  @Override\n  public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n      Type type, Annotation[] annotations, Retrofit retrofit) {\n    if (type == ResponseBody.class) {\n      return Utils.isAnnotationPresent(annotations, Streaming.class)\n          ? StreamingResponseBodyConverter.INSTANCE\n          : BufferingResponseBodyConverter.INSTANCE;\n    }\n    if (type == Void.class) {\n      return VoidResponseBodyConverter.INSTANCE;\n    }\n    if (Utils.isUnit(type)) {\n      return UnitResponseBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  @Override\n  public @Nullable Converter<?, RequestBody> requestBodyConverter(\n      Type type,\n      Annotation[] parameterAnnotations,\n      Annotation[] methodAnnotations,\n      Retrofit retrofit) {\n    if (RequestBody.class.isAssignableFrom(Utils.getRawType(type))) {\n      return RequestBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  static final class VoidResponseBodyConverter implements Converter<ResponseBody, Void> {\n    static final VoidResponseBodyConverter INSTANCE = new VoidResponseBodyConverter();\n\n    @Override\n    public Void convert(ResponseBody value) {\n      value.close();\n      return null;\n    }\n  }\n\n  static final class UnitResponseBodyConverter implements Converter<ResponseBody, Unit> {\n    static final UnitResponseBodyConverter INSTANCE = new UnitResponseBodyConverter();\n\n    @Override\n    public Unit convert(ResponseBody value) {\n      value.close();\n      return Unit.INSTANCE;\n    }\n  }\n\n  static final class RequestBodyConverter implements Converter<RequestBody, RequestBody> {\n    static final RequestBodyConverter INSTANCE = new RequestBodyConverter();\n\n    @Override\n    public RequestBody convert(RequestBody value) {\n      return value;\n    }\n  }\n\n  static final class StreamingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final StreamingResponseBodyConverter INSTANCE = new StreamingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) {\n      return value;\n    }\n  }\n\n  static final class BufferingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final BufferingResponseBodyConverter INSTANCE = new BufferingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) throws IOException {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        return Utils.buffer(value);\n      } finally {\n        value.close();\n      }\n    }\n  }\n\n  static final class ToStringConverter implements Converter<Object, String> {\n    static final ToStringConverter INSTANCE = new ToStringConverter();\n\n    @Override\n    public String convert(Object value) {\n      return value.toString();\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/SkipCallbackExecutorImpl.java\n/*\n * Copyright (C) 2019 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.annotation.Annotation;\n\n// This class conforms to the annotation requirements documented on Annotation.\nfinal class SkipCallbackExecutorImpl implements SkipCallbackExecutor {\n  private static final SkipCallbackExecutor INSTANCE = new SkipCallbackExecutorImpl();\n\n  static Annotation[] ensurePresent(Annotation[] annotations) {\n    if (Utils.isAnnotationPresent(annotations, SkipCallbackExecutor.class)) {\n      return annotations;\n    }\n\n    Annotation[] newAnnotations = new Annotation[annotations.length + 1];\n    // Place the skip annotation first since we're guaranteed to check for it in the call adapter.\n    newAnnotations[0] = SkipCallbackExecutorImpl.INSTANCE;\n    System.arraycopy(annotations, 0, newAnnotations, 1, annotations.length);\n    return newAnnotations;\n  }\n\n  @Override\n  public Class<? extends Annotation> annotationType() {\n    return SkipCallbackExecutor.class;\n  }\n\n  @Override\n  public boolean equals(Object obj) {\n    return obj instanceof SkipCallbackExecutor;\n  }\n\n  @Override\n  public int hashCode() {\n    return 0;\n  }\n\n  @Override\n  public String toString() {\n    return \"@\" + SkipCallbackExecutor.class.getName() + \"()\";\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/HttpServiceMethod.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.getRawType;\nimport static retrofit2.Utils.methodError;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport kotlin.coroutines.Continuation;\nimport okhttp3.ResponseBody;\n\n/** Adapts an invocation of an interface method into an HTTP call. */\nabstract class HttpServiceMethod<ResponseT, ReturnT> extends ServiceMethod<ReturnT> {\n  /**\n   * Inspects the annotations on an interface method to construct a reusable service method that\n   * speaks HTTP. This requires potentially-expensive reflection so it is best to build each service\n   * method only once and reuse it.\n   */\n  static <ResponseT, ReturnT> HttpServiceMethod<ResponseT, ReturnT> parseAnnotations(\n      Retrofit retrofit, Method method, RequestFactory requestFactory) {\n    boolean isKotlinSuspendFunction = requestFactory.isKotlinSuspendFunction;\n    boolean continuationWantsResponse = false;\n    boolean continuationBodyNullable = false;\n    boolean continuationIsUnit = false;\n\n    Annotation[] annotations = method.getAnnotations();\n    Type adapterType;\n    if (isKotlinSuspendFunction) {\n      Type[] parameterTypes = method.getGenericParameterTypes();\n      Type responseType =\n          Utils.getParameterLowerBound(\n              0, (ParameterizedType) parameterTypes[parameterTypes.length - 1]);\n      if (getRawType(responseType) == Response.class && responseType instanceof ParameterizedType) {\n        // Unwrap the actual body type from Response<T>.\n        responseType = Utils.getParameterUpperBound(0, (ParameterizedType) responseType);\n        continuationWantsResponse = true;\n      } else {\n        if (getRawType(responseType) == Call.class) {\n          throw methodError(\n              method,\n              \"Suspend functions should not return Call, as they already execute asynchronously.\\n\"\n                  + \"Change its return type to %s\",\n              Utils.getParameterUpperBound(0, (ParameterizedType) responseType));\n        }\n\n        continuationIsUnit = Utils.isUnit(responseType);\n        // TODO figure out if type is nullable or not\n        // Metadata metadata = method.getDeclaringClass().getAnnotation(Metadata.class)\n        // Find the entry for method\n        // Determine if return type is nullable or not\n      }\n\n      adapterType = new Utils.ParameterizedTypeImpl(null, Call.class, responseType);\n      annotations = SkipCallbackExecutorImpl.ensurePresent(annotations);\n    } else {\n      adapterType = method.getGenericReturnType();\n    }\n\n    CallAdapter<ResponseT, ReturnT> callAdapter =\n        createCallAdapter(retrofit, method, adapterType, annotations);\n    Type responseType = callAdapter.responseType();\n    if (responseType == okhttp3.Response.class) {\n      throw methodError(\n          method,\n          \"'\"\n              + getRawType(responseType).getName()\n              + \"' is not a valid response body type. Did you mean ResponseBody?\");\n    }\n    if (responseType == Response.class) {\n      throw methodError(method, \"Response must include generic type (e.g., Response<String>)\");\n    }\n    if (requestFactory.httpMethod.equals(\"HEAD\")\n        && !Void.class.equals(responseType)\n        && !Utils.isUnit(responseType)) {\n      throw methodError(method, \"HEAD method must use Void or Unit as response type.\");\n    }\n\n    Converter<ResponseBody, ResponseT> responseConverter =\n        createResponseConverter(retrofit, method, responseType);\n\n    okhttp3.Call.Factory callFactory = retrofit.callFactory;\n    if (!isKotlinSuspendFunction) {\n      return new CallAdapted<>(requestFactory, callFactory, responseConverter, callAdapter);\n    } else if (continuationWantsResponse) {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n          new SuspendForResponse<>(\n              requestFactory,\n              callFactory,\n              responseConverter,\n              (CallAdapter<ResponseT, Call<ResponseT>>) callAdapter);\n    } else {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n          new SuspendForBody<>(\n              requestFactory,\n              callFactory,\n              responseConverter,\n              (CallAdapter<ResponseT, Call<ResponseT>>) callAdapter,\n              continuationBodyNullable,\n              continuationIsUnit);\n    }\n  }\n\n  private static <ResponseT, ReturnT> CallAdapter<ResponseT, ReturnT> createCallAdapter(\n      Retrofit retrofit, Method method, Type returnType, Annotation[] annotations) {\n    try {\n      //noinspection unchecked\n      return (CallAdapter<ResponseT, ReturnT>) retrofit.callAdapter(returnType, annotations);\n    } catch (RuntimeException e) { // Wide exception range because factories are user code.\n      throw methodError(method, e, \"Unable to create call adapter for %s\", returnType);\n    }\n  }\n\n  private static <ResponseT> Converter<ResponseBody, ResponseT> createResponseConverter(\n      Retrofit retrofit, Method method, Type responseType) {\n    Annotation[] annotations = method.getAnnotations();\n    try {\n      return retrofit.responseBodyConverter(responseType, annotations);\n    } catch (RuntimeException e) { // Wide exception range because factories are user code.\n      throw methodError(method, e, \"Unable to create converter for %s\", responseType);\n    }\n  }\n\n  private final RequestFactory requestFactory;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, ResponseT> responseConverter;\n\n  HttpServiceMethod(\n      RequestFactory requestFactory,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, ResponseT> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @Override\n  final @Nullable ReturnT invoke(Object instance, Object[] args) {\n    Call<ResponseT> call =\n        new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n    return adapt(call, args);\n  }\n\n  protected abstract @Nullable ReturnT adapt(Call<ResponseT> call, Object[] args);\n\n  static final class CallAdapted<ResponseT, ReturnT> extends HttpServiceMethod<ResponseT, ReturnT> {\n    private final CallAdapter<ResponseT, ReturnT> callAdapter;\n\n    CallAdapted(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, ReturnT> callAdapter) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n    }\n\n    @Override\n    protected ReturnT adapt(Call<ResponseT> call, Object[] args) {\n      return callAdapter.adapt(call);\n    }\n  }\n\n  static final class SuspendForResponse<ResponseT> extends HttpServiceMethod<ResponseT, Object> {\n    private final CallAdapter<ResponseT, Call<ResponseT>> callAdapter;\n\n    SuspendForResponse(\n        RequestFactory requestFactory,\n        okhttp3.Call.Factory callFactory,\n        Converter<ResponseBody, ResponseT> responseConverter,\n        CallAdapter<ResponseT, Call<ResponseT>> callAdapter) {\n      super(requestFactory, callFactory, responseConverter);\n      this.callAdapter = callAdapter;\n    }\n\n    @Override\n    protected Object adapt(Call<ResponseT> call, Object[] args) {\n      call = callAdapter.adapt(call);\n\n...\n", "needle_token_start": 3822, "needle_token_end": 4169, "code_context_ntokens": 14407, "output": ["```java\nstatic Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {\n    if (toResolve == rawType) return context;\n\n    // We skip searching through interfaces if unknown is an interface.\n    if (toResolve.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == toResolve) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (toResolve.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], toResolve);\n        }\n      }\n    }\n\n    // Check our supertypes.\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == toResolve) {\n          return rawType.getGenericSuperclass();\n        } else if (toResolve.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, toResolve);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // We can't resolve this further.\n    return toResolve;\n}\n```"]}
{"repo": "square/retrofit", "name": "addQueryParam", "language": "java", "path": "retrofit/src/main/java/retrofit2/RequestBuilder.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To append a query parameter to the URL being constructed, with an option to encode the parameter.\n2. **Input**: The method takes three parameters: the name of the query parameter, its value (which can be null), and a boolean indicating whether the parameter should be URL-encoded.\n3. **Output**: There is no direct output; however, the method modifies the state of the URL builder by adding a new query parameter.\n4. **Procedure**: \n   - First, it checks if the URL has been previously set. If so, it combines the relative URL with the base URL to initialize the URL builder.\n   - Depending on whether the parameter should be encoded, it either adds the parameter as-is or encodes it before adding.\n   - If the URL builder is not properly initialized (due to a malformed URL), an exception is thrown.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/OptionalConverterFactory.java\n/*\n * Copyright (C) 2017 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport android.annotation.TargetApi;\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\n...\n// Path: retrofit/src/main/java/retrofit2/BuiltInFactories.java\n/*\n * Copyright (C) 2024 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static java.util.Arrays.asList;\nimport static java.util.Collections.emptyList;\nimport static java.util.Collections.singletonList;\n\nimport android.annotation.TargetApi;\nimport java.util.List;\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\n\nclass BuiltInFactories {\n  List<? extends CallAdapter.Factory> createDefaultCallAdapterFactories(\n      @Nullable Executor callbackExecutor) {\n    return singletonList(new DefaultCallAdapterFactory(callbackExecutor));\n  }\n\n  List<? extends Converter.Factory> createDefaultConverterFactories() {\n    return emptyList();\n  }\n\n  @TargetApi(24)\n  static final class Java8 extends BuiltInFactories {\n    @Override\n    List<? extends CallAdapter.Factory> createDefaultCallAdapterFactories(\n        @Nullable Executor callbackExecutor) {\n      return asList(\n          new CompletableFutureCallAdapterFactory(),\n          new DefaultCallAdapterFactory(callbackExecutor));\n    }\n\n    @Override\n    List<? extends Converter.Factory> createDefaultConverterFactories() {\n      return singletonList(new OptionalConverterFactory());\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Invocation.java\n/*\n * Copyright (C) 2018 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\n\n/**\n * A single invocation of a Retrofit service interface method. This class captures both the method\n * that was called and the arguments to the method.\n *\n * <p>Retrofit automatically adds an invocation to each OkHttp request as a tag. You can retrieve\n * the invocation in an OkHttp interceptor for metrics and monitoring.\n *\n * <pre><code>\n * class InvocationLogger implements Interceptor {\n *   &#64;Override public Response intercept(Chain chain) throws IOException {\n *     Request request = chain.request();\n *     Invocation invocation = request.tag(Invocation.class);\n *     if (invocation != null) {\n *       System.out.printf(\"%s.%s %s%n\",\n *           invocation.service().getSimpleName(),\n *           invocation.method().getName(),\n *           invocation.arguments());\n *     }\n *     return chain.proceed(request);\n *   }\n * }\n * </code></pre>\n *\n * <strong>Note:</strong> use caution when examining an invocation's arguments. Although the\n * arguments list is unmodifiable, the arguments themselves may be mutable. They may also be unsafe\n * for concurrent access. For best results declare Retrofit service interfaces using only immutable\n * types for parameters!\n */\npublic final class Invocation {\n  public static <T> Invocation of(Class<T> service, T instance, Method method, List<?> arguments) {\n    Objects.requireNonNull(service, \"service == null\");\n    Objects.requireNonNull(instance, \"instance == null\");\n    Objects.requireNonNull(method, \"method == null\");\n    Objects.requireNonNull(arguments, \"arguments == null\");\n    return new Invocation(service, instance, method, new ArrayList<>(arguments)); // Defensive copy.\n  }\n\n  @Deprecated\n  public static Invocation of(Method method, List<?> arguments) {\n    Objects.requireNonNull(method, \"method == null\");\n    Objects.requireNonNull(arguments, \"arguments == null\");\n    return new Invocation(\n        method.getDeclaringClass(), null, method, new ArrayList<>(arguments)); // Defensive copy.\n  }\n\n  private final Class<?> service;\n  @Nullable private final Object instance;\n  private final Method method;\n  private final List<?> arguments;\n\n  /** Trusted constructor assumes ownership of {@code arguments}. */\n  Invocation(Class<?> service, @Nullable Object instance, Method method, List<?> arguments) {\n    this.service = service;\n    this.instance = instance;\n    this.method = method;\n    this.arguments = Collections.unmodifiableList(arguments);\n  }\n\n  public Class<?> service() {\n    return service;\n  }\n\n  /**\n   * The instance of {@link #service}.\n   * <p>\n   * This will never be null when created by Retrofit. Null will only be returned when created\n   * by {@link #of(Method, List)}.\n   */\n  @Nullable\n  public Object instance() {\n    return instance;\n  }\n\n  public Method method() {\n    return method;\n  }\n\n  public List<?> arguments() {\n    return arguments;\n  }\n\n  @Override\n  public String toString() {\n    return String.format(\"%s.%s() %s\", service.getName(), method.getName(), arguments);\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/AndroidMainExecutor.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport android.os.Handler;\nimport android.os.Looper;\nimport java.util.concurrent.Executor;\n\nfinal class AndroidMainExecutor implements Executor {\n  private final Handler handler = new Handler(Looper.getMainLooper());\n\n  @Override\n  public void execute(Runnable r) {\n    handler.post(r);\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Callback.java\n/*\n * Copyright (C) 2012 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\n/**\n * Communicates responses from a server or offline requests. One and only one method will be invoked\n * in response to a given request.\n *\n * <p>Callback methods are executed using the {@link Retrofit} callback executor. When none is\n * specified, the following defaults are used:\n *\n * <ul>\n *   <li>Android: Callbacks are executed on the application's main (UI) thread.\n *   <li>JVM: Callbacks are executed on the background thread which performed the request.\n * </ul>\n *\n * @param <T> Successful response body type.\n */\npublic interface Callback<T> {\n  /**\n   * Invoked for a received HTTP response.\n   *\n   * <p>Note: An HTTP response may still indicate an application-level failure such as a 404 or 500.\n   * Call {@link Response#isSuccessful()} to determine if the response indicates success.\n   */\n  void onResponse(Call<T> call, Response<T> response);\n\n  /**\n   * Invoked when a network exception occurred talking to the server or when an unexpected exception\n   * occurred creating the request or processing the response.\n   */\n  void onFailure(Call<T> call, Throwable t);\n}\n\n// Path: retrofit/src/main/java/retrofit2/CallAdapter.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\n\n/**\n * Adapts a {@link Call} with response type {@code R} into the type of {@code T}. Instances are\n * created by {@linkplain Factory a factory} which is {@linkplain\n * Retrofit.Builder#addCallAdapterFactory(Factory) installed} into the {@link Retrofit} instance.\n */\npublic interface CallAdapter<R, T> {\n  /**\n   * Returns the value type that this adapter uses when converting the HTTP response body to a Java\n   * object. For example, the response type for {@code Call<Repo>} is {@code Repo}. This type is\n   * used to prepare the {@code call} passed to {@code #adapt}.\n   *\n   * <p>Note: This is typically not the same type as the {@code returnType} provided to this call\n   * adapter's factory.\n   */\n  Type responseType();\n\n  /**\n   * Returns an instance of {@code T} which delegates to {@code call}.\n   *\n   * <p>For example, given an instance for a hypothetical utility, {@code Async}, this instance\n   * would return a new {@code Async<R>} which invoked {@code call} when run.\n   *\n   * <pre><code>\n   * &#64;Override\n   * public &lt;R&gt; Async&lt;R&gt; adapt(final Call&lt;R&gt; call) {\n   *   return Async.create(new Callable&lt;Response&lt;R&gt;&gt;() {\n   *     &#64;Override\n   *     public Response&lt;R&gt; call() throws Exception {\n   *       return call.execute();\n   *     }\n   *   });\n   * }\n   * </code></pre>\n   */\n  T adapt(Call<R> call);\n\n  /**\n   * Creates {@link CallAdapter} instances based on the return type of {@linkplain\n   * Retrofit#create(Class) the service interface} methods.\n   */\n  abstract class Factory {\n    /**\n     * Returns a call adapter for interface methods that return {@code returnType}, or null if it\n     * cannot be handled by this factory.\n     */\n    public abstract @Nullable CallAdapter<?, ?> get(\n        Type returnType, Annotation[] annotations, Retrofit retrofit);\n\n    /**\n     * Extract the upper bound of the generic parameter at {@code index} from {@code type}. For\n     * example, index 1 of {@code Map<String, ? extends Runnable>} returns {@code Runnable}.\n     */\n    protected static Type getParameterUpperBound(int index, ParameterizedType type) {\n      return Utils.getParameterUpperBound(index, type);\n    }\n\n    /**\n     * Extract the raw class type from {@code type}. For example, the type representing {@code\n     * List<? extends Runnable>} returns {@code List.class}.\n     */\n    protected static Class<?> getRawType(Type type) {\n      return Utils.getRawType(type);\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/package-info.java\n// Copyright 2014 Square, Inc.\n\n/**\n * Retrofit turns your REST API into a Java interface.\n *\n * <pre>\n * public interface GitHubService {\n *   &#64;GET(\"/users/{user}/repos\")\n *   List&lt;Repo&gt; listRepos(@Path(\"user\") String user);\n * }\n * </pre>\n */\n@retrofit2.internal.EverythingIsNonNull\npackage retrofit2;\n\n// Path: retrofit/src/main/java/retrofit2/Reflection.java\n/*\n * Copyright (C) 2024 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport android.annotation.TargetApi;\nimport android.os.Build;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.Parameter;\nimport javax.annotation.Nullable;\n\nclass Reflection {\n  boolean isDefaultMethod(Method method) {\n    return false;\n  }\n\n  @Nullable\n  Object invokeDefaultMethod(\n      Method method, Class<?> declaringClass, Object proxy, @Nullable Object[] args)\n      throws Throwable {\n    throw new AssertionError();\n  }\n\n  String describeMethodParameter(Method method, int index) {\n    return \"parameter #\" + (index + 1);\n  }\n\n  static class Java8 extends Reflection {\n    @Override\n    boolean isDefaultMethod(Method method) {\n      return method.isDefault();\n    }\n\n    @Override\n    Object invokeDefaultMethod(\n        Method method, Class<?> declaringClass, Object proxy, @Nullable Object[] args)\n        throws Throwable {\n      return DefaultMethodSupport.invoke(method, declaringClass, proxy, args);\n    }\n\n    @Override\n    String describeMethodParameter(Method method, int index) {\n      Parameter parameter = method.getParameters()[index];\n      if (parameter.isNamePresent()) {\n        return \"parameter '\" + parameter.getName() + '\\'';\n      }\n      return super.describeMethodParameter(method, index);\n    }\n  }\n\n  /**\n   * Android does not support MR jars, so this uses the Java 8 support class.\n   * Default methods and the reflection API to detect them were added to API 24\n   * as part of the initial Java 8 set. MethodHandle, our means of invoking the default method\n   * through the proxy, was not added until API 26.\n   */\n  @TargetApi(24)\n  static final class Android24 extends Reflection {\n    @Override\n    boolean isDefaultMethod(Method method) {\n      return method.isDefault();\n    }\n\n    @Override\n    Object invokeDefaultMethod(\n        Method method, Class<?> declaringClass, Object proxy, @Nullable Object[] args)\n        throws Throwable {\n      if (Build.VERSION.SDK_INT < 26) {\n        throw new UnsupportedOperationException(\n            \"Calling default methods on API 24 and 25 is not supported\");\n      }\n      return DefaultMethodSupport.invoke(method, declaringClass, proxy, args);\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/HttpException.java\n/*\n * Copyright (C) 2016 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.util.Objects;\nimport javax.annotation.Nullable;\n\n/** Exception for an unexpected, non-2xx HTTP response. */\npublic class HttpException extends RuntimeException {\n  private static String getMessage(Response<?> response) {\n    Objects.requireNonNull(response, \"response == null\");\n    return \"HTTP \" + response.code() + \" \" + response.message();\n  }\n\n  private final int code;\n  private final String message;\n  private final transient Response<?> response;\n\n  public HttpException(Response<?> response) {\n    super(getMessage(response));\n    this.code = response.code();\n    this.message = response.message();\n    this.response = response;\n  }\n\n  /** HTTP status code. */\n  public int code() {\n    return code;\n  }\n\n  /** HTTP status message. */\n  public String message() {\n    return message;\n  }\n\n  /** The full HTTP response. This may be null if the exception was serialized. */\n  public @Nullable Response<?> response() {\n    return response;\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/RequestBuilder.java\n/*\n * Copyright (C) 2012 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.util.regex.Pattern;\nimport javax.annotation.Nullable;\nimport okhttp3.FormBody;\nimport okhttp3.Headers;\nimport okhttp3.HttpUrl;\nimport okhttp3.MediaType;\nimport okhttp3.MultipartBody;\nimport okhttp3.Request;\nimport okhttp3.RequestBody;\nimport okio.Buffer;\nimport okio.BufferedSink;\n\nfinal class RequestBuilder {\n  private static final char[] HEX_DIGITS = {\n    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'\n  };\n  private static final String PATH_SEGMENT_ALWAYS_ENCODE_SET = \" \\\"<>^`{}|\\\\?#\";\n\n  /**\n   * Matches strings that contain {@code .} or {@code ..} as a complete path segment. This also\n   * matches dots in their percent-encoded form, {@code %2E}.\n   *\n   * <p>It is okay to have these strings within a larger path segment (like {@code a..z} or {@code\n   * index.html}) but when alone they have a special meaning. A single dot resolves to no path\n   * segment so {@code /one/./three/} becomes {@code /one/three/}. A double-dot pops the preceding\n   * directory, so {@code /one/../three/} becomes {@code /three/}.\n   *\n   * <p>We forbid these in Retrofit paths because they're likely to have the unintended effect. For\n   * example, passing {@code ..} to {@code DELETE /account/book/{isbn}/} yields {@code DELETE\n   * /account/}.\n   */\n  private static final Pattern PATH_TRAVERSAL = Pattern.compile(\"(.*/)?(\\\\.|%2e|%2E){1,2}(/.*)?\");\n\n  private final String method;\n\n  private final HttpUrl baseUrl;\n  private @Nullable String relativeUrl;\n  private @Nullable HttpUrl.Builder urlBuilder;\n\n  private final Request.Builder requestBuilder;\n  private final Headers.Builder headersBuilder;\n  private @Nullable MediaType contentType;\n\n  private final boolean hasBody;\n  private @Nullable MultipartBody.Builder multipartBuilder;\n  private @Nullable FormBody.Builder formBuilder;\n  private @Nullable RequestBody body;\n\n  RequestBuilder(\n      String method,\n      HttpUrl baseUrl,\n      @Nullable String relativeUrl,\n      @Nullable Headers headers,\n      @Nullable MediaType contentType,\n      boolean hasBody,\n      boolean isFormEncoded,\n      boolean isMultipart) {\n    this.method = method;\n    this.baseUrl = baseUrl;\n    this.relativeUrl = relativeUrl;\n    this.requestBuilder = new Request.Builder();\n    this.contentType = contentType;\n    this.hasBody = hasBody;\n\n    if (headers != null) {\n      headersBuilder = headers.newBuilder();\n    } else {\n      headersBuilder = new Headers.Builder();\n    }\n\n    if (isFormEncoded) {\n      // Will be set to 'body' in 'build'.\n      formBuilder = new FormBody.Builder();\n    } else if (isMultipart) {\n      // Will be set to 'body' in 'build'.\n      multipartBuilder = new MultipartBody.Builder();\n      multipartBuilder.setType(MultipartBody.FORM);\n    }\n  }\n\n  void setRelativeUrl(Object relativeUrl) {\n    this.relativeUrl = relativeUrl.toString();\n  }\n\n  void addHeader(String name, String value, boolean allowUnsafeNonAsciiValues) {\n    if (\"Content-Type\".equalsIgnoreCase(name)) {\n      try {\n        contentType = MediaType.get(value);\n      } catch (IllegalArgumentException e) {\n        throw new IllegalArgumentException(\"Malformed content type: \" + value, e);\n      }\n    } else if (allowUnsafeNonAsciiValues) {\n      headersBuilder.addUnsafeNonAscii(name, value);\n    } else {\n      headersBuilder.add(name, value);\n    }\n  }\n\n  void addHeaders(Headers headers) {\n    headersBuilder.addAll(headers);\n  }\n\n  void addPathParam(String name, String value, boolean encoded) {\n    if (relativeUrl == null) {\n      // The relative URL is cleared when the first query parameter is set.\n      throw new AssertionError();\n    }\n    String replacement = canonicalizeForPath(value, encoded);\n    String newRelativeUrl = relativeUrl.replace(\"{\" + name + \"}\", replacement);\n    if (PATH_TRAVERSAL.matcher(newRelativeUrl).matches()) {\n      throw new IllegalArgumentException(\n          \"@Path parameters shouldn't perform path traversal ('.' or '..'): \" + value);\n    }\n    relativeUrl = newRelativeUrl;\n  }\n\n  private static String canonicalizeForPath(String input, boolean alreadyEncoded) {\n    int codePoint;\n    for (int i = 0, limit = input.length(); i < limit; i += Character.charCount(codePoint)) {\n      codePoint = input.codePointAt(i);\n      if (codePoint < 0x20\n          || codePoint >= 0x7f\n          || PATH_SEGMENT_ALWAYS_ENCODE_SET.indexOf(codePoint) != -1\n          || (!alreadyEncoded && (codePoint == '/' || codePoint == '%'))) {\n        // Slow path: the character at i requires encoding!\n        Buffer out = new Buffer();\n        out.writeUtf8(input, 0, i);\n        canonicalizeForPath(out, input, i, limit, alreadyEncoded);\n        return out.readUtf8();\n      }\n    }\n\n    // Fast path: no characters required encoding.\n    return input;\n  }\n\n  private static void canonicalizeForPath(\n      Buffer out, String input, int pos, int limit, boolean alreadyEncoded) {\n    Buffer utf8Buffer = null; // Lazily allocated.\n    int codePoint;\n    for (int i = pos; i < limit; i += Character.charCount(codePoint)) {\n      codePoint = input.codePointAt(i);\n      if (alreadyEncoded\n          && (codePoint == '\\t' || codePoint == '\\n' || codePoint == '\\f' || codePoint == '\\r')) {\n        // Skip this character.\n      } else if (codePoint < 0x20\n          || codePoint >= 0x7f\n          || PATH_SEGMENT_ALWAYS_ENCODE_SET.indexOf(codePoint) != -1\n          || (!alreadyEncoded && (codePoint == '/' || codePoint == '%'))) {\n        // Percent encode this character.\n        if (utf8Buffer == null) {\n          utf8Buffer = new Buffer();\n        }\n        utf8Buffer.writeUtf8CodePoint(codePoint);\n        while (!utf8Buffer.exhausted()) {\n          int b = utf8Buffer.readByte() & 0xff;\n          out.writeByte('%');\n          out.writeByte(HEX_DIGITS[(b >> 4) & 0xf]);\n          out.writeByte(HEX_DIGITS[b & 0xf]);\n        }\n      } else {\n        // This character doesn't need encoding. Just copy it over.\n        out.writeUtf8CodePoint(codePoint);\n      }\n    }\n  }\n\n  \nvoid addQueryParam(String name, @Nullable String value, boolean encoded) {\n    if (relativeUrl != null) {\n      // Do a one-time combination of the built relative URL and the base URL.\n      urlBuilder = baseUrl.newBuilder(relativeUrl);\n      if (urlBuilder == null) {\n        throw new IllegalArgumentException(\n            \"Malformed URL. Base: \" + baseUrl + \", Relative: \" + relativeUrl);\n      }\n      relativeUrl = null;\n    }\n\n    if (encoded) {\n      //noinspection ConstantConditions Checked to be non-null by above 'if' block.\n      urlBuilder.addEncodedQueryParameter(name, value);\n    } else {\n      //noinspection ConstantConditions Checked to be non-null by above 'if' block.\n      urlBuilder.addQueryParameter(name, value);\n    }\n  }\n\n  @SuppressWarnings(\"ConstantConditions\") // Only called when isFormEncoded was true.\n  void addFormField(String name, String value, boolean encoded) {\n    if (encoded) {\n      formBuilder.addEncoded(name, value);\n    } else {\n      formBuilder.add(name, value);\n    }\n  }\n\n  @SuppressWarnings(\"ConstantConditions\") // Only called when isMultipart was true.\n  void addPart(Headers headers, RequestBody body) {\n    multipartBuilder.addPart(headers, body);\n  }\n\n  @SuppressWarnings(\"ConstantConditions\") // Only called when isMultipart was true.\n  void addPart(MultipartBody.Part part) {\n    multipartBuilder.addPart(part);\n  }\n\n  void setBody(RequestBody body) {\n    this.body = body;\n  }\n\n  <T> void addTag(Class<T> cls, @Nullable T value) {\n    requestBuilder.tag(cls, value);\n  }\n\n  Request.Builder get() {\n    HttpUrl url;\n    HttpUrl.Builder urlBuilder = this.urlBuilder;\n    if (urlBuilder != null) {\n      url = urlBuilder.build();\n    } else {\n      // No query parameters triggered builder creation, just combine the relative URL and base URL.\n      //noinspection ConstantConditions Non-null if urlBuilder is null.\n      url = baseUrl.resolve(relativeUrl);\n      if (url == null) {\n        throw new IllegalArgumentException(\n            \"Malformed URL. Base: \" + baseUrl + \", Relative: \" + relativeUrl);\n      }\n    }\n\n    RequestBody body = this.body;\n    if (body == null) {\n      // Try to pull from one of the builders.\n      if (formBuilder != null) {\n        body = formBuilder.build();\n      } else if (multipartBuilder != null) {\n        body = multipartBuilder.build();\n      } else if (hasBody) {\n        // Body is absent, make an empty body.\n        body = RequestBody.create(null, new byte[0]);\n      }\n    }\n\n    MediaType contentType = this.contentType;\n    if (contentType != null) {\n      if (body != null) {\n        body = new ContentTypeOverridingRequestBody(body, contentType);\n      } else {\n        headersBuilder.add(\"Content-Type\", contentType.toString());\n      }\n    }\n\n    return requestBuilder.url(url).headers(headersBuilder.build()).method(method, body);\n  }\n\n  private static class ContentTypeOverridingRequestBody extends RequestBody {\n    private final RequestBody delegate;\n    private final MediaType contentType;\n\n    ContentTypeOverridingRequestBody(RequestBody delegate, MediaType contentType) {\n      this.delegate = delegate;\n      this.contentType = contentType;\n    }\n\n    @Override\n    public MediaType contentType() {\n      return contentType;\n    }\n\n    @Override\n    public long contentLength() throws IOException {\n      return delegate.contentLength();\n    }\n\n    @Override\n    public void writeTo(BufferedSink sink) throws IOException {\n      delegate.writeTo(sink);\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Call.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport okhttp3.Request;\nimport okio.Timeout;\n\n/**\n * An invocation of a Retrofit method that sends a request to a webserver and returns a response.\n * Each call yields its own HTTP request and response pair. Use {@link #clone} to make multiple\n * calls with the same parameters to the same webserver; this may be used to implement polling or to\n * retry a failed call.\n *\n * <p>Calls may be executed synchronously with {@link #execute}, or asynchronously with {@link\n * #enqueue}. In either case the call can be canceled at any time with {@link #cancel}. A call that\n * is busy writing its request or reading its response may receive a {@link IOException}; this is\n * working as designed.\n *\n * @param <T> Successful response body type.\n */\npublic interface Call<T> extends Cloneable {\n  /**\n   * Synchronously send the request and return its response.\n   *\n   * @throws IOException if a problem occurred talking to the server.\n   * @throws RuntimeException (and subclasses) if an unexpected error occurs creating the request or\n   *     decoding the response.\n   */\n  Response<T> execute() throws IOException;\n\n  /**\n   * Asynchronously send the request and notify {@code callback} of its response or if an error\n   * occurred talking to the server, creating the request, or processing the response.\n   */\n  void enqueue(Callback<T> callback);\n\n  /**\n   * Returns true if this call has been either {@linkplain #execute() executed} or {@linkplain\n   * #enqueue(Callback) enqueued}. It is an error to execute or enqueue a call more than once.\n   */\n  boolean isExecuted();\n\n  /**\n   * Cancel this call. An attempt will be made to cancel in-flight calls, and if the call has not\n   * yet been executed it never will be.\n   */\n  void cancel();\n\n  /** True if {@link #cancel()} was called. */\n  boolean isCanceled();\n\n  /**\n   * Create a new, identical call to this one which can be enqueued or executed even if this call\n   * has already been.\n   */\n  Call<T> clone();\n\n  /** The original HTTP request. */\n  Request request();\n\n  /**\n   * Returns a timeout that spans the entire call: resolving DNS, connecting, writing the request\n   * body, server processing, and reading the response body. If the call requires redirects or\n   * retries all must complete within one timeout period.\n   */\n  Timeout timeout();\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Headers.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\n\n/**\n * Adds headers literally supplied in the {@code value}.\n *\n * <pre><code>\n * &#64;Headers(\"Cache-Control: max-age=640000\")\n * &#64;GET(\"/\")\n * ...\n *\n * &#64;Headers({\n *   \"X-Foo: Bar\",\n *   \"X-Ping: Pong\"\n * })\n * &#64;GET(\"/\")\n * ...\n * </code></pre>\n *\n * <p>Parameter keys and values only allows ascii values by default. Specify {@link\n * #allowUnsafeNonAsciiValues() allowUnsafeNonAsciiValues=true} to change this behavior.\n *\n * <p>&#64;Headers({ \"X-Foo: Bar\", \"X-Ping: Pong\" }, allowUnsafeNonAsciiValues=true) &#64;GET(\"/\")\n *\n * <p><strong>Note:</strong> Headers do not overwrite each other. All headers with the same name\n * will be included in the request.\n *\n * @see Header\n * @see HeaderMap\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface Headers {\n\n  /** The query parameter name. */\n  String[] value();\n\n  /**\n   * Specifies whether the parameter {@linkplain #value() name} and value are already URL encoded.\n   */\n  boolean allowUnsafeNonAsciiValues() default false;\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/package-info.java\n// Copyright 2014 Square, Inc.\n\n/** Annotations for interface methods to control the HTTP request behavior. */\npackage retrofit2.http;\n\n// Path: retrofit/src/main/java/retrofit2/internal/EverythingIsNonNull.java\n/*\n * Copyright (C) 2018 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.internal;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport javax.annotation.Nonnull;\nimport javax.annotation.meta.TypeQualifierDefault;\n\n/**\n * Extends {@code ParametersAreNonnullByDefault} to also apply to Method results and fields.\n *\n * @see javax.annotation.ParametersAreNonnullByDefault\n */\n@Documented\n@Nonnull\n@TypeQualifierDefault({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER})\n@Retention(RetentionPolicy.RUNTIME)\npublic @interface EverythingIsNonNull {}\n\n", "needle_token_start": 7012, "needle_token_end": 7221, "code_context_ntokens": 9923, "output": ["```java\nvoid addQueryParam(String name, @Nullable String value, boolean encoded) {\n    if (relativeUrl != null) {\n      // Do a one-time combination of the built relative URL and the base URL.\n      urlBuilder = baseUrl.newBuilder(relativeUrl);\n      if (urlBuilder == null) {\n        throw new IllegalArgumentException(\n            \"Malformed URL. Base: \" + baseUrl + \", Relative: \" + relativeUrl);\n      }\n      relativeUrl = null;\n    }\n\n    if (encoded) {\n      //noinspection ConstantConditions Checked to be non-null by above 'if' block.\n      urlBuilder.addEncodedQueryParameter(name, value);\n    } else {\n      //noinspection ConstantConditions Checked to be non-null by above 'if' block.\n      urlBuilder.addQueryParameter(name, value);\n    }\n}\n```"]}
{"repo": "square/retrofit", "name": "getActualTypeArguments", "language": "java", "path": "retrofit/src/main/java/retrofit2/Utils.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve the array of type arguments used in a parameterized type.\n2. **Input**: The function does not take any direct parameters as it is a method called on an instance of a specific type.\n3. **Output**: Returns an array of `Type` objects, each representing a type argument used in the parameterized type.\n4. **Procedure**: The method clones and returns the array of type arguments that were stored during the instantiation of the parameterized type instance.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/Platform.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static android.os.Build.VERSION.SDK_INT;\n\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\n\nfinal class Platform {\n  static final @Nullable Executor callbackExecutor;\n  static final Reflection reflection;\n  static final BuiltInFactories builtInFactories;\n\n  static {\n    switch (System.getProperty(\"java.vm.name\")) {\n      case \"Dalvik\":\n        callbackExecutor = new AndroidMainExecutor();\n        if (SDK_INT >= 24) {\n          reflection = new Reflection.Android24();\n          builtInFactories = new BuiltInFactories.Java8();\n        } else {\n          reflection = new Reflection();\n          builtInFactories = new BuiltInFactories();\n        }\n        break;\n\n      case \"RoboVM\":\n        callbackExecutor = null;\n        reflection = new Reflection();\n        builtInFactories = new BuiltInFactories();\n        break;\n\n      default:\n        callbackExecutor = null;\n        reflection = new Reflection.Java8();\n        builtInFactories = new BuiltInFactories.Java8();\n        break;\n    }\n  }\n\n  private Platform() {}\n}\n\n// Path: retrofit/src/main/java/retrofit2/Response.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.Headers;\nimport okhttp3.Protocol;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\n\n/** An HTTP response. */\npublic final class Response<T> {\n  /** Create a synthetic successful response with {@code body} as the deserialized body. */\n  public static <T> Response<T> success(@Nullable T body) {\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response with an HTTP status code of {@code code} and {@code\n   * body} as the deserialized body.\n   */\n  public static <T> Response<T> success(int code, @Nullable T body) {\n    if (code < 200 || code >= 300) {\n      throw new IllegalArgumentException(\"code < 200 or >= 300: \" + code);\n    }\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(code)\n            .message(\"Response.success()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response using {@code headers} with {@code body} as the\n   * deserialized body.\n   */\n  public static <T> Response<T> success(@Nullable T body, Headers headers) {\n    Objects.requireNonNull(headers, \"headers == null\");\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .headers(headers)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a successful response from {@code rawResponse} with {@code body} as the deserialized\n   * body.\n   */\n  public static <T> Response<T> success(@Nullable T body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (!rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse must be successful response\");\n    }\n    return new Response<>(rawResponse, body, null);\n  }\n\n  /**\n   * Create a synthetic error response with an HTTP status code of {@code code} and {@code body} as\n   * the error body.\n   */\n  public static <T> Response<T> error(int code, ResponseBody body) {\n    Objects.requireNonNull(body, \"body == null\");\n    if (code < 400) throw new IllegalArgumentException(\"code < 400: \" + code);\n    return error(\n        body,\n        new okhttp3.Response.Builder() //\n            .body(new OkHttpCall.NoContentResponseBody(body.contentType(), body.contentLength()))\n            .code(code)\n            .message(\"Response.error()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /** Create an error response from {@code rawResponse} with {@code body} as the error body. */\n  public static <T> Response<T> error(ResponseBody body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(body, \"body == null\");\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse should not be successful response\");\n    }\n    return new Response<>(rawResponse, null, body);\n  }\n\n  private final okhttp3.Response rawResponse;\n  private final @Nullable T body;\n  private final @Nullable ResponseBody errorBody;\n\n  private Response(\n      okhttp3.Response rawResponse, @Nullable T body, @Nullable ResponseBody errorBody) {\n    this.rawResponse = rawResponse;\n    this.body = body;\n    this.errorBody = errorBody;\n  }\n\n  /** The raw response from the HTTP client. */\n  public okhttp3.Response raw() {\n    return rawResponse;\n  }\n\n  /** HTTP status code. */\n  public int code() {\n    return rawResponse.code();\n  }\n\n  /** HTTP status message or null if unknown. */\n  public String message() {\n    return rawResponse.message();\n  }\n\n  /** HTTP headers. */\n  public Headers headers() {\n    return rawResponse.headers();\n  }\n\n  /** Returns true if {@link #code()} is in the range [200..300). */\n  public boolean isSuccessful() {\n    return rawResponse.isSuccessful();\n  }\n\n  /** The deserialized response body of a {@linkplain #isSuccessful() successful} response. */\n  public @Nullable T body() {\n    return body;\n  }\n\n  /** The raw response body of an {@linkplain #isSuccessful() unsuccessful} response. */\n  public @Nullable ResponseBody errorBody() {\n    return errorBody;\n  }\n\n  @Override\n  public String toString() {\n    return rawResponse.toString();\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Utils.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\n\nfinal class Utils {\n  static final Type[] EMPTY_TYPE_ARRAY = new Type[0];\n\n  private Utils() {\n    // No instances.\n  }\n\n  static RuntimeException methodError(Method method, String message, Object... args) {\n    return methodError(method, null, message, args);\n  }\n\n  @SuppressWarnings(\"AnnotateFormatMethod\")\n  static RuntimeException methodError(\n      Method method, @Nullable Throwable cause, String message, Object... args) {\n    message = String.format(message, args);\n    return new IllegalArgumentException(\n        message\n            + \"\\n    for method \"\n            + method.getDeclaringClass().getSimpleName()\n            + \".\"\n            + method.getName(),\n        cause);\n  }\n\n  static RuntimeException parameterError(\n      Method method, Throwable cause, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, cause, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static RuntimeException parameterError(Method method, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static Class<?> getRawType(Type type) {\n    Objects.requireNonNull(type, \"type == null\");\n\n    if (type instanceof Class<?>) {\n      // Type is a normal class.\n      return (Class<?>) type;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n\n      // I'm not exactly sure why getRawType() returns Type instead of Class. Neal isn't either but\n      // suspects some pathological case related to nested classes exists.\n      Type rawType = parameterizedType.getRawType();\n      if (!(rawType instanceof Class)) throw new IllegalArgumentException();\n      return (Class<?>) rawType;\n    }\n    if (type instanceof GenericArrayType) {\n      Type componentType = ((GenericArrayType) type).getGenericComponentType();\n      return Array.newInstance(getRawType(componentType), 0).getClass();\n    }\n    if (type instanceof TypeVariable) {\n      // We could use the variable's bounds, but that won't work if there are multiple. Having a raw\n      // type that's more general than necessary is okay.\n      return Object.class;\n    }\n    if (type instanceof WildcardType) {\n      return getRawType(((WildcardType) type).getUpperBounds()[0]);\n    }\n\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + type.getClass().getName());\n  }\n\n  /** Returns true if {@code a} and {@code b} are equal. */\n  static boolean equals(Type a, Type b) {\n    if (a == b) {\n      return true; // Also handles (a == null && b == null).\n\n    } else if (a instanceof Class) {\n      return a.equals(b); // Class already specifies equals().\n\n    } else if (a instanceof ParameterizedType) {\n      if (!(b instanceof ParameterizedType)) return false;\n      ParameterizedType pa = (ParameterizedType) a;\n      ParameterizedType pb = (ParameterizedType) b;\n      Object ownerA = pa.getOwnerType();\n      Object ownerB = pb.getOwnerType();\n      boolean ownersAreEqual = ownerA == ownerB || (ownerA != null && ownerA.equals(ownerB));\n      boolean rawTypesAreEqual = pa.getRawType().equals(pb.getRawType());\n      boolean typeArgumentsAreEqual =\n          Arrays.equals(pa.getActualTypeArguments(), pb.getActualTypeArguments());\n\n      return ownersAreEqual && rawTypesAreEqual && typeArgumentsAreEqual;\n\n    } else if (a instanceof GenericArrayType) {\n      if (!(b instanceof GenericArrayType)) return false;\n      GenericArrayType ga = (GenericArrayType) a;\n      GenericArrayType gb = (GenericArrayType) b;\n      return equals(ga.getGenericComponentType(), gb.getGenericComponentType());\n\n    } else if (a instanceof WildcardType) {\n      if (!(b instanceof WildcardType)) return false;\n      WildcardType wa = (WildcardType) a;\n      WildcardType wb = (WildcardType) b;\n      return Arrays.equals(wa.getUpperBounds(), wb.getUpperBounds())\n          && Arrays.equals(wa.getLowerBounds(), wb.getLowerBounds());\n\n    } else if (a instanceof TypeVariable) {\n      if (!(b instanceof TypeVariable)) return false;\n      TypeVariable<?> va = (TypeVariable<?>) a;\n      TypeVariable<?> vb = (TypeVariable<?>) b;\n      return va.getGenericDeclaration() == vb.getGenericDeclaration()\n          && va.getName().equals(vb.getName());\n\n    } else {\n      return false; // This isn't a type we support!\n    }\n  }\n\n  /**\n   * Returns the generic supertype for {@code supertype}. For example, given a class {@code\n   * IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>} and the\n   * result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.\n   */\n  static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {\n    if (toResolve == rawType) return context;\n\n    // We skip searching through interfaces if unknown is an interface.\n    if (toResolve.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == toResolve) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (toResolve.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], toResolve);\n        }\n      }\n    }\n\n    // Check our supertypes.\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == toResolve) {\n          return rawType.getGenericSuperclass();\n        } else if (toResolve.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, toResolve);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // We can't resolve this further.\n    return toResolve;\n  }\n\n  private static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0; i < array.length; i++) {\n      if (toFind.equals(array[i])) return i;\n    }\n    throw new NoSuchElementException();\n  }\n\n  static String typeToString(Type type) {\n    return type instanceof Class ? ((Class<?>) type).getName() : type.toString();\n  }\n\n  /**\n   * Returns the generic form of {@code supertype}. For example, if this is {@code\n   * ArrayList<String>}, this returns {@code Iterable<String>} given the input {@code\n   * Iterable.class}.\n   *\n   * @param supertype a superclass of, or interface implemented by, this.\n   */\n  static Type getSupertype(Type context, Class<?> contextRawType, Class<?> supertype) {\n    if (!supertype.isAssignableFrom(contextRawType)) throw new IllegalArgumentException();\n    return resolve(\n        context, contextRawType, getGenericSupertype(context, contextRawType, supertype));\n  }\n\n  static Type resolve(Type context, Class<?> contextRawType, Type toResolve) {\n    // This implementation is made a little more complicated in an attempt to avoid object-creation.\n    while (true) {\n      if (toResolve instanceof TypeVariable) {\n        TypeVariable<?> typeVariable = (TypeVariable<?>) toResolve;\n        toResolve = resolveTypeVariable(context, contextRawType, typeVariable);\n        if (toResolve == typeVariable) {\n          return toResolve;\n        }\n\n      } else if (toResolve instanceof Class && ((Class<?>) toResolve).isArray()) {\n        Class<?> original = (Class<?>) toResolve;\n        Type componentType = original.getComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof GenericArrayType) {\n        GenericArrayType original = (GenericArrayType) toResolve;\n        Type componentType = original.getGenericComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof ParameterizedType) {\n        ParameterizedType original = (ParameterizedType) toResolve;\n        Type ownerType = original.getOwnerType();\n        Type newOwnerType = resolve(context, contextRawType, ownerType);\n        boolean changed = newOwnerType != ownerType;\n\n        Type[] args = original.getActualTypeArguments();\n        for (int t = 0, length = args.length; t < length; t++) {\n          Type resolvedTypeArgument = resolve(context, contextRawType, args[t]);\n          if (resolvedTypeArgument != args[t]) {\n            if (!changed) {\n              args = args.clone();\n              changed = true;\n            }\n            args[t] = resolvedTypeArgument;\n          }\n        }\n\n        return changed\n            ? new ParameterizedTypeImpl(newOwnerType, original.getRawType(), args)\n            : original;\n\n      } else if (toResolve instanceof WildcardType) {\n        WildcardType original = (WildcardType) toResolve;\n        Type[] originalLowerBound = original.getLowerBounds();\n        Type[] originalUpperBound = original.getUpperBounds();\n\n        if (originalLowerBound.length == 1) {\n          Type lowerBound = resolve(context, contextRawType, originalLowerBound[0]);\n          if (lowerBound != originalLowerBound[0]) {\n            return new WildcardTypeImpl(new Type[] {Object.class}, new Type[] {lowerBound});\n          }\n        } else if (originalUpperBound.length == 1) {\n          Type upperBound = resolve(context, contextRawType, originalUpperBound[0]);\n          if (upperBound != originalUpperBound[0]) {\n            return new WildcardTypeImpl(new Type[] {upperBound}, EMPTY_TYPE_ARRAY);\n          }\n        }\n        return original;\n\n      } else {\n        return toResolve;\n      }\n    }\n  }\n\n  private static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // We can't reduce this further.\n    if (declaredByRaw == null) return unknown;\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static @Nullable Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n\n  static void checkNotPrimitive(Type type) {\n    if (type instanceof Class<?> && ((Class<?>) type).isPrimitive()) {\n      throw new IllegalArgumentException();\n    }\n  }\n\n  /** Returns true if {@code annotations} contains an instance of {@code cls}. */\n  static boolean isAnnotationPresent(Annotation[] annotations, Class<? extends Annotation> cls) {\n    for (Annotation annotation : annotations) {\n      if (cls.isInstance(annotation)) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  static ResponseBody buffer(final ResponseBody body) throws IOException {\n    Buffer buffer = new Buffer();\n    body.source().readAll(buffer);\n    return ResponseBody.create(body.contentType(), body.contentLength(), buffer);\n  }\n\n  static Type getParameterUpperBound(int index, ParameterizedType type) {\n    Type[] types = type.getActualTypeArguments();\n    if (index < 0 || index >= types.length) {\n      throw new IllegalArgumentException(\n          \"Index \" + index + \" not in range [0,\" + types.length + \") for \" + type);\n    }\n    Type paramType = types[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getUpperBounds()[0];\n    }\n    return paramType;\n  }\n\n  static Type getParameterLowerBound(int index, ParameterizedType type) {\n    Type paramType = type.getActualTypeArguments()[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getLowerBounds()[0];\n    }\n    return paramType;\n  }\n\n  static boolean hasUnresolvableType(@Nullable Type type) {\n    if (type instanceof Class<?>) {\n      return false;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n      for (Type typeArgument : parameterizedType.getActualTypeArguments()) {\n        if (hasUnresolvableType(typeArgument)) {\n          return true;\n        }\n      }\n      return false;\n    }\n    if (type instanceof GenericArrayType) {\n      return hasUnresolvableType(((GenericArrayType) type).getGenericComponentType());\n    }\n    if (type instanceof TypeVariable) {\n      return true;\n    }\n    if (type instanceof WildcardType) {\n      return true;\n    }\n    String className = type == null ? \"null\" : type.getClass().getName();\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + className);\n  }\n\n  static final class ParameterizedTypeImpl implements ParameterizedType {\n    private final @Nullable Type ownerType;\n    private final Type rawType;\n    private final Type[] typeArguments;\n\n    ParameterizedTypeImpl(@Nullable Type ownerType, Type rawType, Type... typeArguments) {\n      // Require an owner type if the raw type needs it.\n      if (rawType instanceof Class<?>\n          && (ownerType == null) != (((Class<?>) rawType).getEnclosingClass() == null)) {\n        throw new IllegalArgumentException();\n      }\n\n      for (Type typeArgument : typeArguments) {\n        Objects.requireNonNull(typeArgument, \"typeArgument == null\");\n        checkNotPrimitive(typeArgument);\n      }\n\n      this.ownerType = ownerType;\n      this.rawType = rawType;\n      this.typeArguments = typeArguments.clone();\n    }\n\n    \n@Override\n    public Type[] getActualTypeArguments() {\n      return typeArguments.clone();\n    }\n\n    @Override\n    public Type getRawType() {\n      return rawType;\n    }\n\n    @Override\n    public @Nullable Type getOwnerType() {\n      return ownerType;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof ParameterizedType && Utils.equals(this, (ParameterizedType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      return Arrays.hashCode(typeArguments)\n          ^ rawType.hashCode()\n          ^ (ownerType != null ? ownerType.hashCode() : 0);\n    }\n\n    @Override\n    public String toString() {\n      if (typeArguments.length == 0) return typeToString(rawType);\n      StringBuilder result = new StringBuilder(30 * (typeArguments.length + 1));\n      result.append(typeToString(rawType));\n      result.append(\"<\").append(typeToString(typeArguments[0]));\n      for (int i = 1; i < typeArguments.length; i++) {\n        result.append(\", \").append(typeToString(typeArguments[i]));\n      }\n      return result.append(\">\").toString();\n    }\n  }\n\n  private static final class GenericArrayTypeImpl implements GenericArrayType {\n    private final Type componentType;\n\n    GenericArrayTypeImpl(Type componentType) {\n      this.componentType = componentType;\n    }\n\n    @Override\n    public Type getGenericComponentType() {\n      return componentType;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      return o instanceof GenericArrayType && Utils.equals(this, (GenericArrayType) o);\n    }\n\n    @Override\n    public int hashCode() {\n      return componentType.hashCode();\n    }\n\n    @Override\n    public String toString() {\n      return typeToString(componentType) + \"[]\";\n    }\n  }\n\n  /**\n   * The WildcardType interface supports multiple upper bounds and multiple lower bounds. We only\n   * support what the Java 6 language needs - at most one bound. If a lower bound is set, the upper\n   * bound must be Object.class.\n   */\n  private static final class WildcardTypeImpl implements WildcardType {\n    private final Type upperBound;\n    private final @Nullable Type lowerBound;\n\n    WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) {\n      if (lowerBounds.length > 1) throw new IllegalArgumentException();\n      if (upperBounds.length != 1) throw new IllegalArgumentException();\n\n      if (lowerBounds.length == 1) {\n        if (lowerBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(lowerBounds[0]);\n        if (upperBounds[0] != Object.class) throw new IllegalArgumentException();\n        this.lowerBound = lowerBounds[0];\n        this.upperBound = Object.class;\n      } else {\n        if (upperBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(upperBounds[0]);\n        this.lowerBound = null;\n        this.upperBound = upperBounds[0];\n      }\n    }\n\n    @Override\n    public Type[] getUpperBounds() {\n      return new Type[] {upperBound};\n    }\n\n    @Override\n    public Type[] getLowerBounds() {\n      return lowerBound != null ? new Type[] {lowerBound} : EMPTY_TYPE_ARRAY;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof WildcardType && Utils.equals(this, (WildcardType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      // This equals Arrays.hashCode(getLowerBounds()) ^ Arrays.hashCode(getUpperBounds()).\n      return (lowerBound != null ? 31 + lowerBound.hashCode() : 1) ^ (31 + upperBound.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      if (lowerBound != null) return \"? super \" + typeToString(lowerBound);\n      if (upperBound == Object.class) return \"?\";\n      return \"? extends \" + typeToString(upperBound);\n    }\n  }\n\n  // https://github.com/ReactiveX/RxJava/blob/6a44e5d0543a48f1c378dc833a155f3f71333bc2/\n  // src/main/java/io/reactivex/exceptions/Exceptions.java#L66\n  static void throwIfFatal(Throwable t) {\n    if (t instanceof VirtualMachineError) {\n      throw (VirtualMachineError) t;\n    } else if (t instanceof ThreadDeath) {\n      throw (ThreadDeath) t;\n    } else if (t instanceof LinkageError) {\n      throw (LinkageError) t;\n    }\n  }\n\n  /** Not volatile because we don't mind multiple threads discovering this. */\n  private static boolean checkForKotlinUnit = true;\n\n  static boolean isUnit(Type type) {\n    if (checkForKotlinUnit) {\n      try {\n        return type == Unit.class;\n      } catch (NoClassDefFoundError ignored) {\n        checkForKotlinUnit = false;\n      }\n    }\n    return false;\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/OkHttpCall.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.throwIfFatal;\n\nimport java.io.IOException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport javax.annotation.concurrent.GuardedBy;\nimport okhttp3.MediaType;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\nimport okio.BufferedSource;\nimport okio.ForwardingSource;\nimport okio.Okio;\nimport okio.Timeout;\n\nfinal class OkHttpCall<T> implements Call<T> {\n  private final RequestFactory requestFactory;\n  private final Object instance;\n  private final Object[] args;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, T> responseConverter;\n\n  private volatile boolean canceled;\n\n  @GuardedBy(\"this\")\n  private @Nullable okhttp3.Call rawCall;\n\n  @GuardedBy(\"this\") // Either a RuntimeException, non-fatal Error, or IOException.\n  private @Nullable Throwable creationFailure;\n\n  @GuardedBy(\"this\")\n  private boolean executed;\n\n  OkHttpCall(\n      RequestFactory requestFactory,\n      Object instance,\n      Object[] args,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, T> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.instance = instance;\n    this.args = args;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @SuppressWarnings(\"CloneDoesntCallSuperClone\") // We are a final type & this saves clearing state.\n  @Override\n  public OkHttpCall<T> clone() {\n    return new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n  }\n\n  @Override\n  public synchronized Request request() {\n    try {\n      return getRawCall().request();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create request.\", e);\n    }\n  }\n\n  @Override\n  public synchronized Timeout timeout() {\n    try {\n      return getRawCall().timeout();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create call.\", e);\n    }\n  }\n\n  /**\n   * Returns the raw call, initializing it if necessary. Throws if initializing the raw call throws,\n   * or has thrown in previous attempts to create it.\n   */\n  @GuardedBy(\"this\")\n  private okhttp3.Call getRawCall() throws IOException {\n    okhttp3.Call call = rawCall;\n    if (call != null) return call;\n\n    // Re-throw previous failures if this isn't the first attempt.\n    if (creationFailure != null) {\n      if (creationFailure instanceof IOException) {\n        throw (IOException) creationFailure;\n      } else if (creationFailure instanceof RuntimeException) {\n        throw (RuntimeException) creationFailure;\n      } else {\n        throw (Error) creationFailure;\n      }\n    }\n\n    // Create and remember either the success or the failure.\n    try {\n      return rawCall = createRawCall();\n    } catch (RuntimeException | Error | IOException e) {\n      throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n      creationFailure = e;\n      throw e;\n    }\n  }\n\n  @Override\n  public void enqueue(final Callback<T> callback) {\n    Objects.requireNonNull(callback, \"callback == null\");\n\n    okhttp3.Call call;\n    Throwable failure;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = rawCall;\n      failure = creationFailure;\n      if (call == null && failure == null) {\n        try {\n          call = rawCall = createRawCall();\n        } catch (Throwable t) {\n          throwIfFatal(t);\n          failure = creationFailure = t;\n        }\n      }\n    }\n\n    if (failure != null) {\n      callback.onFailure(this, failure);\n      return;\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    call.enqueue(\n        new okhttp3.Callback() {\n          @Override\n          public void onResponse(okhttp3.Call call, okhttp3.Response rawResponse) {\n            Response<T> response;\n            try {\n              response = parseResponse(rawResponse);\n            } catch (Throwable e) {\n              throwIfFatal(e);\n              callFailure(e);\n              return;\n            }\n\n            try {\n              callback.onResponse(OkHttpCall.this, response);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n\n          @Override\n          public void onFailure(okhttp3.Call call, IOException e) {\n            callFailure(e);\n          }\n\n          private void callFailure(Throwable e) {\n            try {\n              callback.onFailure(OkHttpCall.this, e);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n        });\n  }\n\n  @Override\n  public synchronized boolean isExecuted() {\n    return executed;\n  }\n\n  @Override\n  public Response<T> execute() throws IOException {\n    okhttp3.Call call;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = getRawCall();\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    return parseResponse(call.execute());\n  }\n\n  private okhttp3.Call createRawCall() throws IOException {\n    okhttp3.Call call = callFactory.newCall(requestFactory.create(instance, args));\n    if (call == null) {\n      throw new NullPointerException(\"Call.Factory returned null.\");\n    }\n    return call;\n  }\n\n  Response<T> parseResponse(okhttp3.Response rawResponse) throws IOException {\n    ResponseBody rawBody = rawResponse.body();\n\n    // Remove the body's source (the only stateful object) so we can pass the response along.\n    rawResponse =\n        rawResponse\n            .newBuilder()\n            .body(new NoContentResponseBody(rawBody.contentType(), rawBody.contentLength()))\n            .build();\n\n    int code = rawResponse.code();\n    if (code < 200 || code >= 300) {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        ResponseBody bufferedBody = Utils.buffer(rawBody);\n        return Response.error(bufferedBody, rawResponse);\n      } finally {\n        rawBody.close();\n      }\n    }\n\n    if (code == 204 || code == 205) {\n      rawBody.close();\n      return Response.success(null, rawResponse);\n    }\n\n    ExceptionCatchingResponseBody catchingBody = new ExceptionCatchingResponseBody(rawBody);\n    try {\n      T body = responseConverter.convert(catchingBody);\n      return Response.success(body, rawResponse);\n    } catch (RuntimeException e) {\n      // If the underlying source threw an exception, propagate that rather than indicating it was\n      // a runtime exception.\n      catchingBody.throwIfCaught();\n      throw e;\n    }\n  }\n\n  @Override\n  public void cancel() {\n    canceled = true;\n\n    okhttp3.Call call;\n    synchronized (this) {\n      call = rawCall;\n    }\n    if (call != null) {\n      call.cancel();\n    }\n  }\n\n  @Override\n  public boolean isCanceled() {\n    if (canceled) {\n      return true;\n    }\n    synchronized (this) {\n      return rawCall != null && rawCall.isCanceled();\n    }\n  }\n\n  static final class NoContentResponseBody extends ResponseBody {\n    private final @Nullable MediaType contentType;\n    private final long contentLength;\n\n    NoContentResponseBody(@Nullable MediaType contentType, long contentLength) {\n      this.contentType = contentType;\n      this.contentLength = contentLength;\n    }\n\n    @Override\n    public MediaType contentType() {\n      return contentType;\n    }\n\n    @Override\n    public long contentLength() {\n      return contentLength;\n    }\n\n    @Override\n    public BufferedSource source() {\n      throw new IllegalStateException(\"Cannot read raw response body of a converted body.\");\n    }\n  }\n\n  static final class ExceptionCatchingResponseBody extends ResponseBody {\n    private final ResponseBody delegate;\n    private final BufferedSource delegateSource;\n    @Nullable IOException thrownException;\n\n    ExceptionCatchingResponseBody(ResponseBody delegate) {\n      this.delegate = delegate;\n      this.delegateSource =\n          Okio.buffer(\n              new ForwardingSource(delegate.source()) {\n                @Override\n                public long read(Buffer sink, long byteCount) throws IOException {\n                  try {\n                    return super.read(sink, byteCount);\n                  } catch (IOException e) {\n                    thrownException = e;\n                    throw e;\n                  }\n                }\n              });\n    }\n\n    @Override\n    public MediaType contentType() {\n      return delegate.contentType();\n    }\n\n    @Override\n    public long contentLength() {\n      return delegate.contentLength();\n    }\n\n    @Override\n    public BufferedSource source() {\n      return delegateSource;\n    }\n\n    @Override\n    public void close() {\n      delegate.close();\n    }\n\n    void throwIfCaught() throws IOException {\n      if (thrownException != null) {\n        throw thrownException;\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Streaming.java\n/*\n * Copyright (C) 2014 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.ResponseBody;\n\n/**\n * Treat the response body on methods returning {@link ResponseBody ResponseBody} as is, i.e.\n * without converting the body to {@code byte[]}.\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface Streaming {}\n\n// Path: retrofit/src/main/java/retrofit2/BuiltInConverters.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.Streaming;\n\nfinal class BuiltInConverters extends Converter.Factory {\n\n  @Override\n  public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n      Type type, Annotation[] annotations, Retrofit retrofit) {\n    if (type == ResponseBody.class) {\n      return Utils.isAnnotationPresent(annotations, Streaming.class)\n          ? StreamingResponseBodyConverter.INSTANCE\n          : BufferingResponseBodyConverter.INSTANCE;\n    }\n    if (type == Void.class) {\n      return VoidResponseBodyConverter.INSTANCE;\n    }\n    if (Utils.isUnit(type)) {\n      return UnitResponseBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  @Override\n  public @Nullable Converter<?, RequestBody> requestBodyConverter(\n      Type type,\n      Annotation[] parameterAnnotations,\n      Annotation[] methodAnnotations,\n      Retrofit retrofit) {\n    if (RequestBody.class.isAssignableFrom(Utils.getRawType(type))) {\n      return RequestBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  static final class VoidResponseBodyConverter implements Converter<ResponseBody, Void> {\n    static final VoidResponseBodyConverter INSTANCE = new VoidResponseBodyConverter();\n\n    @Override\n    public Void convert(ResponseBody value) {\n      value.close();\n      return null;\n    }\n  }\n\n  static final class UnitResponseBodyConverter implements Converter<ResponseBody, Unit> {\n    static final UnitResponseBodyConverter INSTANCE = new UnitResponseBodyConverter();\n\n    @Override\n    public Unit convert(ResponseBody value) {\n      value.close();\n      return Unit.INSTANCE;\n    }\n  }\n\n  static final class RequestBodyConverter implements Converter<RequestBody, RequestBody> {\n    static final RequestBodyConverter INSTANCE = new RequestBodyConverter();\n\n    @Override\n    public RequestBody convert(RequestBody value) {\n      return value;\n    }\n  }\n\n  static final class StreamingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final StreamingResponseBodyConverter INSTANCE = new StreamingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) {\n      return value;\n    }\n  }\n\n  static final class BufferingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final BufferingResponseBodyConverter INSTANCE = new BufferingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) throws IOException {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        return Utils.buffer(value);\n      } finally {\n        value.close();\n      }\n    }\n  }\n\n  static final class ToStringConverter implements Converter<Object, String> {\n    static final ToStringConverter INSTANCE = new ToStringConverter();\n\n    @Override\n    public String convert(Object value) {\n      return value.toString();\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/SkipCallbackExecutorImpl.java\n/*\n * Copyright (C) 2019 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.annotation.Annotation;\n\n// This class conforms to the annotation requirements documented on Annotation.\nfinal class SkipCallbackExecutorImpl implements SkipCallbackExecutor {\n  private static final SkipCallbackExecutor INSTANCE = new SkipCallbackExecutorImpl();\n\n  static Annotation[] ensurePresent(Annotation[] annotations) {\n    if (Utils.isAnnotationPresent(annotations, SkipCallbackExecutor.class)) {\n      return annotations;\n    }\n\n    Annotation[] newAnnotations = new Annotation[annotations.length + 1];\n    // Place the skip annotation first since we're guaranteed to check for it in the call adapter.\n    newAnnotations[0] = SkipCallbackExecutorImpl.INSTANCE;\n    System.arraycopy(annotations, 0, newAnnotations, 1, annotations.length);\n    return newAnnotations;\n  }\n\n  @Override\n  public Class<? extends Annotation> annotationType() {\n    return SkipCallbackExecutor.class;\n  }\n\n  @Override\n  public boolean equals(Object obj) {\n    return obj instanceof SkipCallbackExecutor;\n  }\n\n  @Override\n  public int hashCode() {\n    return 0;\n  }\n\n  @Override\n  public String toString() {\n    return \"@\" + SkipCallbackExecutor.class.getName() + \"()\";\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/HttpServiceMethod.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.getRawType;\nimport static retrofit2.Utils.methodError;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport kotlin.coroutines.Continuation;\nimport okhttp3.ResponseBody;\n\n/** Adapts an invocation of an interface method into an HTTP call. */\nabstract class HttpServiceMethod<ResponseT, ReturnT> extends ServiceMethod<ReturnT> {\n  /**\n   * Inspects the annotations on an interface method to construct a reusable service method that\n   * speaks HTTP. This requires potentially-expensive reflection so it is best to build each service\n   * method only once and reuse it.\n   */\n  static <ResponseT, ReturnT> HttpServiceMethod<ResponseT, ReturnT> parseAnnotations(\n      Retrofit retrofit, Method method, RequestFactory requestFactory) {\n    boolean isKotlinSuspendFunction = requestFactory.isKotlinSuspendFunction;\n    boolean continuationWantsResponse = false;\n    boolean continuationBodyNullable = false;\n    boolean continuationIsUnit = false;\n\n    Annotation[] annotations = method.getAnnotations();\n    Type adapterType;\n    if (isKotlinSuspendFunction) {\n      Type[] parameterTypes = method.getGenericParameterTypes();\n      Type responseType =\n          Utils.getParameterLowerBound(\n              0, (ParameterizedType) parameterTypes[parameterTypes.length - 1]);\n      if (getRawType(responseType) == Response.class && responseType instanceof ParameterizedType) {\n        // Unwrap the actual body type from Response<T>.\n        responseType = Utils.getParameterUpperBound(0, (ParameterizedType) responseType);\n        continuationWantsResponse = true;\n      } else {\n        if (getRawType(responseType) == Call.class) {\n          throw methodError(\n              method,\n              \"Suspend functions should not return Call, as they already execute asynchronously.\\n\"\n                  + \"Change its return type to %s\",\n              Utils.getParameterUpperBound(0, (ParameterizedType) responseType));\n        }\n\n        continuationIsUnit = Utils.isUnit(responseType);\n        // TODO figure out if type is nullable or not\n        // Metadata metadata = method.getDeclaringClass().getAnnotation(Metadata.class)\n        // Find the entry for method\n        // Determine if return type is nullable or not\n      }\n\n      adapterType = new Utils.ParameterizedTypeImpl(null, Call.class, responseType);\n      annotations = SkipCallbackExecutorImpl.ensurePresent(annotations);\n    } else {\n      adapterType = method.getGenericReturnType();\n    }\n\n    CallAdapter<ResponseT, ReturnT> callAdapter =\n        createCallAdapter(retrofit, method, adapterType, annotations);\n    Type responseType = callAdapter.responseType();\n    if (responseType == okhttp3.Response.class) {\n      throw methodError(\n          method,\n          \"'\"\n              + getRawType(responseType).getName()\n              + \"' is not a valid response body type. Did you mean ResponseBody?\");\n    }\n    if (responseType == Response.class) {\n      throw methodError(method, \"Response must include generic type (e.g., Response<String>)\");\n    }\n    if (requestFactory.httpMethod.equals(\"HEAD\")\n        && !Void.class.equals(responseType)\n        && !Utils.isUnit(responseType)) {\n      throw methodError(method, \"HEAD method must use Void or Unit as response type.\");\n    }\n\n    Converter<ResponseBody, ResponseT> responseConverter =\n        createResponseConverter(retrofit, method, responseType);\n\n    okhttp3.Call.Factory callFactory = retrofit.callFactory;\n    if (!isKotlinSuspendFunction) {\n      return new CallAdapted<>(requestFactory, callFactory, responseConverter, callAdapter);\n    } else if (continuationWantsResponse) {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n          new SuspendForResponse<>(\n              requestFactory,\n              callFactory,\n              responseConverter,\n              (CallAdapter<ResponseT, Call<ResponseT>>) callAdapter);\n    } else {\n      //noinspection unchecked Kotlin compiler guarantees ReturnT to be Object.\n      return (HttpServiceMethod<ResponseT, ReturnT>)\n...\n", "needle_token_start": 6356, "needle_token_end": 6381, "code_context_ntokens": 13553, "output": ["```java\nstatic Type[] getActualTypeArguments() {\n  return typeArguments.clone();\n}\n```"]}
{"repo": "square/retrofit", "name": "indexOf", "language": "java", "path": "retrofit/src/main/java/retrofit2/Utils.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To find the position of a specified element within an array.\n2. **Input**: An array and an element to search for within that array.\n3. **Output**: The index (position) of the specified element in the array if it exists.\n4. **Procedure**: Iterates through each element of the array, comparing each element with the specified element. If a match is found, the current index is returned. If no match is found by the end of the array, an exception indicating the element is not found is thrown.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/Platform.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static android.os.Build.VERSION.SDK_INT;\n\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\n\nfinal class Platform {\n  static final @Nullable Executor callbackExecutor;\n  static final Reflection reflection;\n  static final BuiltInFactories builtInFactories;\n\n  static {\n    switch (System.getProperty(\"java.vm.name\")) {\n      case \"Dalvik\":\n        callbackExecutor = new AndroidMainExecutor();\n        if (SDK_INT >= 24) {\n          reflection = new Reflection.Android24();\n          builtInFactories = new BuiltInFactories.Java8();\n        } else {\n          reflection = new Reflection();\n          builtInFactories = new BuiltInFactories();\n        }\n        break;\n\n      case \"RoboVM\":\n        callbackExecutor = null;\n        reflection = new Reflection();\n        builtInFactories = new BuiltInFactories();\n        break;\n\n      default:\n        callbackExecutor = null;\n        reflection = new Reflection.Java8();\n        builtInFactories = new BuiltInFactories.Java8();\n        break;\n    }\n  }\n\n  private Platform() {}\n}\n\n// Path: retrofit/src/main/java/retrofit2/Response.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.Headers;\nimport okhttp3.Protocol;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\n\n/** An HTTP response. */\npublic final class Response<T> {\n  /** Create a synthetic successful response with {@code body} as the deserialized body. */\n  public static <T> Response<T> success(@Nullable T body) {\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response with an HTTP status code of {@code code} and {@code\n   * body} as the deserialized body.\n   */\n  public static <T> Response<T> success(int code, @Nullable T body) {\n    if (code < 200 || code >= 300) {\n      throw new IllegalArgumentException(\"code < 200 or >= 300: \" + code);\n    }\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(code)\n            .message(\"Response.success()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response using {@code headers} with {@code body} as the\n   * deserialized body.\n   */\n  public static <T> Response<T> success(@Nullable T body, Headers headers) {\n    Objects.requireNonNull(headers, \"headers == null\");\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .headers(headers)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a successful response from {@code rawResponse} with {@code body} as the deserialized\n   * body.\n   */\n  public static <T> Response<T> success(@Nullable T body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (!rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse must be successful response\");\n    }\n    return new Response<>(rawResponse, body, null);\n  }\n\n  /**\n   * Create a synthetic error response with an HTTP status code of {@code code} and {@code body} as\n   * the error body.\n   */\n  public static <T> Response<T> error(int code, ResponseBody body) {\n    Objects.requireNonNull(body, \"body == null\");\n    if (code < 400) throw new IllegalArgumentException(\"code < 400: \" + code);\n    return error(\n        body,\n        new okhttp3.Response.Builder() //\n            .body(new OkHttpCall.NoContentResponseBody(body.contentType(), body.contentLength()))\n            .code(code)\n            .message(\"Response.error()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /** Create an error response from {@code rawResponse} with {@code body} as the error body. */\n  public static <T> Response<T> error(ResponseBody body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(body, \"body == null\");\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse should not be successful response\");\n    }\n    return new Response<>(rawResponse, null, body);\n  }\n\n  private final okhttp3.Response rawResponse;\n  private final @Nullable T body;\n  private final @Nullable ResponseBody errorBody;\n\n  private Response(\n      okhttp3.Response rawResponse, @Nullable T body, @Nullable ResponseBody errorBody) {\n    this.rawResponse = rawResponse;\n    this.body = body;\n    this.errorBody = errorBody;\n  }\n\n  /** The raw response from the HTTP client. */\n  public okhttp3.Response raw() {\n    return rawResponse;\n  }\n\n  /** HTTP status code. */\n  public int code() {\n    return rawResponse.code();\n  }\n\n  /** HTTP status message or null if unknown. */\n  public String message() {\n    return rawResponse.message();\n  }\n\n  /** HTTP headers. */\n  public Headers headers() {\n    return rawResponse.headers();\n  }\n\n  /** Returns true if {@link #code()} is in the range [200..300). */\n  public boolean isSuccessful() {\n    return rawResponse.isSuccessful();\n  }\n\n  /** The deserialized response body of a {@linkplain #isSuccessful() successful} response. */\n  public @Nullable T body() {\n    return body;\n  }\n\n  /** The raw response body of an {@linkplain #isSuccessful() unsuccessful} response. */\n  public @Nullable ResponseBody errorBody() {\n    return errorBody;\n  }\n\n  @Override\n  public String toString() {\n    return rawResponse.toString();\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Utils.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\n\nfinal class Utils {\n  static final Type[] EMPTY_TYPE_ARRAY = new Type[0];\n\n  private Utils() {\n    // No instances.\n  }\n\n  static RuntimeException methodError(Method method, String message, Object... args) {\n    return methodError(method, null, message, args);\n  }\n\n  @SuppressWarnings(\"AnnotateFormatMethod\")\n  static RuntimeException methodError(\n      Method method, @Nullable Throwable cause, String message, Object... args) {\n    message = String.format(message, args);\n    return new IllegalArgumentException(\n        message\n            + \"\\n    for method \"\n            + method.getDeclaringClass().getSimpleName()\n            + \".\"\n            + method.getName(),\n        cause);\n  }\n\n  static RuntimeException parameterError(\n      Method method, Throwable cause, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, cause, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static RuntimeException parameterError(Method method, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static Class<?> getRawType(Type type) {\n    Objects.requireNonNull(type, \"type == null\");\n\n    if (type instanceof Class<?>) {\n      // Type is a normal class.\n      return (Class<?>) type;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n\n      // I'm not exactly sure why getRawType() returns Type instead of Class. Neal isn't either but\n      // suspects some pathological case related to nested classes exists.\n      Type rawType = parameterizedType.getRawType();\n      if (!(rawType instanceof Class)) throw new IllegalArgumentException();\n      return (Class<?>) rawType;\n    }\n    if (type instanceof GenericArrayType) {\n      Type componentType = ((GenericArrayType) type).getGenericComponentType();\n      return Array.newInstance(getRawType(componentType), 0).getClass();\n    }\n    if (type instanceof TypeVariable) {\n      // We could use the variable's bounds, but that won't work if there are multiple. Having a raw\n      // type that's more general than necessary is okay.\n      return Object.class;\n    }\n    if (type instanceof WildcardType) {\n      return getRawType(((WildcardType) type).getUpperBounds()[0]);\n    }\n\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + type.getClass().getName());\n  }\n\n  /** Returns true if {@code a} and {@code b} are equal. */\n  static boolean equals(Type a, Type b) {\n    if (a == b) {\n      return true; // Also handles (a == null && b == null).\n\n    } else if (a instanceof Class) {\n      return a.equals(b); // Class already specifies equals().\n\n    } else if (a instanceof ParameterizedType) {\n      if (!(b instanceof ParameterizedType)) return false;\n      ParameterizedType pa = (ParameterizedType) a;\n      ParameterizedType pb = (ParameterizedType) b;\n      Object ownerA = pa.getOwnerType();\n      Object ownerB = pb.getOwnerType();\n      boolean ownersAreEqual = ownerA == ownerB || (ownerA != null && ownerA.equals(ownerB));\n      boolean rawTypesAreEqual = pa.getRawType().equals(pb.getRawType());\n      boolean typeArgumentsAreEqual =\n          Arrays.equals(pa.getActualTypeArguments(), pb.getActualTypeArguments());\n\n      return ownersAreEqual && rawTypesAreEqual && typeArgumentsAreEqual;\n\n    } else if (a instanceof GenericArrayType) {\n      if (!(b instanceof GenericArrayType)) return false;\n      GenericArrayType ga = (GenericArrayType) a;\n      GenericArrayType gb = (GenericArrayType) b;\n      return equals(ga.getGenericComponentType(), gb.getGenericComponentType());\n\n    } else if (a instanceof WildcardType) {\n      if (!(b instanceof WildcardType)) return false;\n      WildcardType wa = (WildcardType) a;\n      WildcardType wb = (WildcardType) b;\n      return Arrays.equals(wa.getUpperBounds(), wb.getUpperBounds())\n          && Arrays.equals(wa.getLowerBounds(), wb.getLowerBounds());\n\n    } else if (a instanceof TypeVariable) {\n      if (!(b instanceof TypeVariable)) return false;\n      TypeVariable<?> va = (TypeVariable<?>) a;\n      TypeVariable<?> vb = (TypeVariable<?>) b;\n      return va.getGenericDeclaration() == vb.getGenericDeclaration()\n          && va.getName().equals(vb.getName());\n\n    } else {\n      return false; // This isn't a type we support!\n    }\n  }\n\n  /**\n   * Returns the generic supertype for {@code supertype}. For example, given a class {@code\n   * IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>} and the\n   * result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.\n   */\n  static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {\n    if (toResolve == rawType) return context;\n\n    // We skip searching through interfaces if unknown is an interface.\n    if (toResolve.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == toResolve) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (toResolve.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], toResolve);\n        }\n      }\n    }\n\n    // Check our supertypes.\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == toResolve) {\n          return rawType.getGenericSuperclass();\n        } else if (toResolve.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, toResolve);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // We can't resolve this further.\n    return toResolve;\n  }\n\n  \nprivate static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0; i < array.length; i++) {\n      if (toFind.equals(array[i])) return i;\n    }\n    throw new NoSuchElementException();\n  }\n\n  static String typeToString(Type type) {\n    return type instanceof Class ? ((Class<?>) type).getName() : type.toString();\n  }\n\n  /**\n   * Returns the generic form of {@code supertype}. For example, if this is {@code\n   * ArrayList<String>}, this returns {@code Iterable<String>} given the input {@code\n   * Iterable.class}.\n   *\n   * @param supertype a superclass of, or interface implemented by, this.\n   */\n  static Type getSupertype(Type context, Class<?> contextRawType, Class<?> supertype) {\n    if (!supertype.isAssignableFrom(contextRawType)) throw new IllegalArgumentException();\n    return resolve(\n        context, contextRawType, getGenericSupertype(context, contextRawType, supertype));\n  }\n\n  static Type resolve(Type context, Class<?> contextRawType, Type toResolve) {\n    // This implementation is made a little more complicated in an attempt to avoid object-creation.\n    while (true) {\n      if (toResolve instanceof TypeVariable) {\n        TypeVariable<?> typeVariable = (TypeVariable<?>) toResolve;\n        toResolve = resolveTypeVariable(context, contextRawType, typeVariable);\n        if (toResolve == typeVariable) {\n          return toResolve;\n        }\n\n      } else if (toResolve instanceof Class && ((Class<?>) toResolve).isArray()) {\n        Class<?> original = (Class<?>) toResolve;\n        Type componentType = original.getComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof GenericArrayType) {\n        GenericArrayType original = (GenericArrayType) toResolve;\n        Type componentType = original.getGenericComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof ParameterizedType) {\n        ParameterizedType original = (ParameterizedType) toResolve;\n        Type ownerType = original.getOwnerType();\n        Type newOwnerType = resolve(context, contextRawType, ownerType);\n        boolean changed = newOwnerType != ownerType;\n\n        Type[] args = original.getActualTypeArguments();\n        for (int t = 0, length = args.length; t < length; t++) {\n          Type resolvedTypeArgument = resolve(context, contextRawType, args[t]);\n          if (resolvedTypeArgument != args[t]) {\n            if (!changed) {\n              args = args.clone();\n              changed = true;\n            }\n            args[t] = resolvedTypeArgument;\n          }\n        }\n\n        return changed\n            ? new ParameterizedTypeImpl(newOwnerType, original.getRawType(), args)\n            : original;\n\n      } else if (toResolve instanceof WildcardType) {\n        WildcardType original = (WildcardType) toResolve;\n        Type[] originalLowerBound = original.getLowerBounds();\n        Type[] originalUpperBound = original.getUpperBounds();\n\n        if (originalLowerBound.length == 1) {\n          Type lowerBound = resolve(context, contextRawType, originalLowerBound[0]);\n          if (lowerBound != originalLowerBound[0]) {\n            return new WildcardTypeImpl(new Type[] {Object.class}, new Type[] {lowerBound});\n          }\n        } else if (originalUpperBound.length == 1) {\n          Type upperBound = resolve(context, contextRawType, originalUpperBound[0]);\n          if (upperBound != originalUpperBound[0]) {\n            return new WildcardTypeImpl(new Type[] {upperBound}, EMPTY_TYPE_ARRAY);\n          }\n        }\n        return original;\n\n      } else {\n        return toResolve;\n      }\n    }\n  }\n\n  private static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // We can't reduce this further.\n    if (declaredByRaw == null) return unknown;\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static @Nullable Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n\n  static void checkNotPrimitive(Type type) {\n    if (type instanceof Class<?> && ((Class<?>) type).isPrimitive()) {\n      throw new IllegalArgumentException();\n    }\n  }\n\n  /** Returns true if {@code annotations} contains an instance of {@code cls}. */\n  static boolean isAnnotationPresent(Annotation[] annotations, Class<? extends Annotation> cls) {\n    for (Annotation annotation : annotations) {\n      if (cls.isInstance(annotation)) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  static ResponseBody buffer(final ResponseBody body) throws IOException {\n    Buffer buffer = new Buffer();\n    body.source().readAll(buffer);\n    return ResponseBody.create(body.contentType(), body.contentLength(), buffer);\n  }\n\n  static Type getParameterUpperBound(int index, ParameterizedType type) {\n    Type[] types = type.getActualTypeArguments();\n    if (index < 0 || index >= types.length) {\n      throw new IllegalArgumentException(\n          \"Index \" + index + \" not in range [0,\" + types.length + \") for \" + type);\n    }\n    Type paramType = types[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getUpperBounds()[0];\n    }\n    return paramType;\n  }\n\n  static Type getParameterLowerBound(int index, ParameterizedType type) {\n    Type paramType = type.getActualTypeArguments()[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getLowerBounds()[0];\n    }\n    return paramType;\n  }\n\n  static boolean hasUnresolvableType(@Nullable Type type) {\n    if (type instanceof Class<?>) {\n      return false;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n      for (Type typeArgument : parameterizedType.getActualTypeArguments()) {\n        if (hasUnresolvableType(typeArgument)) {\n          return true;\n        }\n      }\n      return false;\n    }\n    if (type instanceof GenericArrayType) {\n      return hasUnresolvableType(((GenericArrayType) type).getGenericComponentType());\n    }\n    if (type instanceof TypeVariable) {\n      return true;\n    }\n    if (type instanceof WildcardType) {\n      return true;\n    }\n    String className = type == null ? \"null\" : type.getClass().getName();\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + className);\n  }\n\n  static final class ParameterizedTypeImpl implements ParameterizedType {\n    private final @Nullable Type ownerType;\n    private final Type rawType;\n    private final Type[] typeArguments;\n\n    ParameterizedTypeImpl(@Nullable Type ownerType, Type rawType, Type... typeArguments) {\n      // Require an owner type if the raw type needs it.\n      if (rawType instanceof Class<?>\n          && (ownerType == null) != (((Class<?>) rawType).getEnclosingClass() == null)) {\n        throw new IllegalArgumentException();\n      }\n\n      for (Type typeArgument : typeArguments) {\n        Objects.requireNonNull(typeArgument, \"typeArgument == null\");\n        checkNotPrimitive(typeArgument);\n      }\n\n      this.ownerType = ownerType;\n      this.rawType = rawType;\n      this.typeArguments = typeArguments.clone();\n    }\n\n    @Override\n    public Type[] getActualTypeArguments() {\n      return typeArguments.clone();\n    }\n\n    @Override\n    public Type getRawType() {\n      return rawType;\n    }\n\n    @Override\n    public @Nullable Type getOwnerType() {\n      return ownerType;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof ParameterizedType && Utils.equals(this, (ParameterizedType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      return Arrays.hashCode(typeArguments)\n          ^ rawType.hashCode()\n          ^ (ownerType != null ? ownerType.hashCode() : 0);\n    }\n\n    @Override\n    public String toString() {\n      if (typeArguments.length == 0) return typeToString(rawType);\n      StringBuilder result = new StringBuilder(30 * (typeArguments.length + 1));\n      result.append(typeToString(rawType));\n      result.append(\"<\").append(typeToString(typeArguments[0]));\n      for (int i = 1; i < typeArguments.length; i++) {\n        result.append(\", \").append(typeToString(typeArguments[i]));\n      }\n      return result.append(\">\").toString();\n    }\n  }\n\n  private static final class GenericArrayTypeImpl implements GenericArrayType {\n    private final Type componentType;\n\n    GenericArrayTypeImpl(Type componentType) {\n      this.componentType = componentType;\n    }\n\n    @Override\n    public Type getGenericComponentType() {\n      return componentType;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      return o instanceof GenericArrayType && Utils.equals(this, (GenericArrayType) o);\n    }\n\n    @Override\n    public int hashCode() {\n      return componentType.hashCode();\n    }\n\n    @Override\n    public String toString() {\n      return typeToString(componentType) + \"[]\";\n    }\n  }\n\n  /**\n   * The WildcardType interface supports multiple upper bounds and multiple lower bounds. We only\n   * support what the Java 6 language needs - at most one bound. If a lower bound is set, the upper\n   * bound must be Object.class.\n   */\n  private static final class WildcardTypeImpl implements WildcardType {\n    private final Type upperBound;\n    private final @Nullable Type lowerBound;\n\n    WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) {\n      if (lowerBounds.length > 1) throw new IllegalArgumentException();\n      if (upperBounds.length != 1) throw new IllegalArgumentException();\n\n      if (lowerBounds.length == 1) {\n        if (lowerBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(lowerBounds[0]);\n        if (upperBounds[0] != Object.class) throw new IllegalArgumentException();\n        this.lowerBound = lowerBounds[0];\n        this.upperBound = Object.class;\n      } else {\n        if (upperBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(upperBounds[0]);\n        this.lowerBound = null;\n        this.upperBound = upperBounds[0];\n      }\n    }\n\n    @Override\n    public Type[] getUpperBounds() {\n      return new Type[] {upperBound};\n    }\n\n    @Override\n    public Type[] getLowerBounds() {\n      return lowerBound != null ? new Type[] {lowerBound} : EMPTY_TYPE_ARRAY;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof WildcardType && Utils.equals(this, (WildcardType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      // This equals Arrays.hashCode(getLowerBounds()) ^ Arrays.hashCode(getUpperBounds()).\n      return (lowerBound != null ? 31 + lowerBound.hashCode() : 1) ^ (31 + upperBound.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      if (lowerBound != null) return \"? super \" + typeToString(lowerBound);\n      if (upperBound == Object.class) return \"?\";\n      return \"? extends \" + typeToString(upperBound);\n    }\n  }\n\n  // https://github.com/ReactiveX/RxJava/blob/6a44e5d0543a48f1c378dc833a155f3f71333bc2/\n  // src/main/java/io/reactivex/exceptions/Exceptions.java#L66\n  static void throwIfFatal(Throwable t) {\n    if (t instanceof VirtualMachineError) {\n      throw (VirtualMachineError) t;\n    } else if (t instanceof ThreadDeath) {\n      throw (ThreadDeath) t;\n    } else if (t instanceof LinkageError) {\n      throw (LinkageError) t;\n    }\n  }\n\n  /** Not volatile because we don't mind multiple threads discovering this. */\n  private static boolean checkForKotlinUnit = true;\n\n  static boolean isUnit(Type type) {\n    if (checkForKotlinUnit) {\n      try {\n        return type == Unit.class;\n      } catch (NoClassDefFoundError ignored) {\n        checkForKotlinUnit = false;\n      }\n    }\n    return false;\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/OkHttpCall.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.throwIfFatal;\n\nimport java.io.IOException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport javax.annotation.concurrent.GuardedBy;\nimport okhttp3.MediaType;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\nimport okio.BufferedSource;\nimport okio.ForwardingSource;\nimport okio.Okio;\nimport okio.Timeout;\n\nfinal class OkHttpCall<T> implements Call<T> {\n  private final RequestFactory requestFactory;\n  private final Object instance;\n  private final Object[] args;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, T> responseConverter;\n\n  private volatile boolean canceled;\n\n  @GuardedBy(\"this\")\n  private @Nullable okhttp3.Call rawCall;\n\n  @GuardedBy(\"this\") // Either a RuntimeException, non-fatal Error, or IOException.\n  private @Nullable Throwable creationFailure;\n\n  @GuardedBy(\"this\")\n  private boolean executed;\n\n  OkHttpCall(\n      RequestFactory requestFactory,\n      Object instance,\n      Object[] args,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, T> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.instance = instance;\n    this.args = args;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @SuppressWarnings(\"CloneDoesntCallSuperClone\") // We are a final type & this saves clearing state.\n  @Override\n  public OkHttpCall<T> clone() {\n    return new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n  }\n\n  @Override\n  public synchronized Request request() {\n    try {\n      return getRawCall().request();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create request.\", e);\n    }\n  }\n\n  @Override\n  public synchronized Timeout timeout() {\n    try {\n      return getRawCall().timeout();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create call.\", e);\n    }\n  }\n\n  /**\n   * Returns the raw call, initializing it if necessary. Throws if initializing the raw call throws,\n   * or has thrown in previous attempts to create it.\n   */\n  @GuardedBy(\"this\")\n  private okhttp3.Call getRawCall() throws IOException {\n    okhttp3.Call call = rawCall;\n    if (call != null) return call;\n\n    // Re-throw previous failures if this isn't the first attempt.\n    if (creationFailure != null) {\n      if (creationFailure instanceof IOException) {\n        throw (IOException) creationFailure;\n      } else if (creationFailure instanceof RuntimeException) {\n        throw (RuntimeException) creationFailure;\n      } else {\n        throw (Error) creationFailure;\n      }\n    }\n\n    // Create and remember either the success or the failure.\n    try {\n      return rawCall = createRawCall();\n    } catch (RuntimeException | Error | IOException e) {\n      throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n      creationFailure = e;\n      throw e;\n    }\n  }\n\n  @Override\n  public void enqueue(final Callback<T> callback) {\n    Objects.requireNonNull(callback, \"callback == null\");\n\n    okhttp3.Call call;\n    Throwable failure;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = rawCall;\n      failure = creationFailure;\n      if (call == null && failure == null) {\n        try {\n          call = rawCall = createRawCall();\n        } catch (Throwable t) {\n          throwIfFatal(t);\n          failure = creationFailure = t;\n        }\n      }\n    }\n\n    if (failure != null) {\n      callback.onFailure(this, failure);\n      return;\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    call.enqueue(\n        new okhttp3.Callback() {\n          @Override\n          public void onResponse(okhttp3.Call call, okhttp3.Response rawResponse) {\n            Response<T> response;\n            try {\n              response = parseResponse(rawResponse);\n            } catch (Throwable e) {\n              throwIfFatal(e);\n              callFailure(e);\n              return;\n            }\n\n            try {\n              callback.onResponse(OkHttpCall.this, response);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n\n          @Override\n          public void onFailure(okhttp3.Call call, IOException e) {\n            callFailure(e);\n          }\n\n          private void callFailure(Throwable e) {\n            try {\n              callback.onFailure(OkHttpCall.this, e);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n        });\n  }\n\n  @Override\n  public synchronized boolean isExecuted() {\n    return executed;\n  }\n\n  @Override\n  public Response<T> execute() throws IOException {\n    okhttp3.Call call;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = getRawCall();\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    return parseResponse(call.execute());\n  }\n\n  private okhttp3.Call createRawCall() throws IOException {\n    okhttp3.Call call = callFactory.newCall(requestFactory.create(instance, args));\n    if (call == null) {\n      throw new NullPointerException(\"Call.Factory returned null.\");\n    }\n    return call;\n  }\n\n  Response<T> parseResponse(okhttp3.Response rawResponse) throws IOException {\n    ResponseBody rawBody = rawResponse.body();\n\n    // Remove the body's source (the only stateful object) so we can pass the response along.\n    rawResponse =\n        rawResponse\n            .newBuilder()\n            .body(new NoContentResponseBody(rawBody.contentType(), rawBody.contentLength()))\n            .build();\n\n    int code = rawResponse.code();\n    if (code < 200 || code >= 300) {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        ResponseBody bufferedBody = Utils.buffer(rawBody);\n        return Response.error(bufferedBody, rawResponse);\n      } finally {\n        rawBody.close();\n      }\n    }\n\n    if (code == 204 || code == 205) {\n      rawBody.close();\n      return Response.success(null, rawResponse);\n    }\n\n    ExceptionCatchingResponseBody catchingBody = new ExceptionCatchingResponseBody(rawBody);\n    try {\n      T body = responseConverter.convert(catchingBody);\n      return Response.success(body, rawResponse);\n    } catch (RuntimeException e) {\n      // If the underlying source threw an exception, propagate that rather than indicating it was\n      // a runtime exception.\n      catchingBody.throwIfCaught();\n      throw e;\n    }\n  }\n\n  @Override\n  public void cancel() {\n    canceled = true;\n\n    okhttp3.Call call;\n    synchronized (this) {\n      call = rawCall;\n    }\n    if (call != null) {\n      call.cancel();\n    }\n  }\n\n  @Override\n...\n", "needle_token_start": 4172, "needle_token_end": 4238, "code_context_ntokens": 9831, "output": ["```java\nprivate static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0; i < array.length; i++) {\n        if (toFind.equals(array[i])) return i;\n    }\n    throw new NoSuchElementException();\n}\n```"]}
{"repo": "square/retrofit", "name": "getUpperBounds", "language": "java", "path": "retrofit/src/main/java/retrofit2/Utils.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve the upper boundary types of a wildcard type in a type system that supports generics.\n2. **Input**: There are no parameters required for this function.\n3. **Output**: Returns an array of `Type` objects, which represent the upper bounds of the wildcard type. Typically, this array contains a single type.\n4. **Procedure**: The function directly accesses a pre-defined type field representing the upper bound and returns it encapsulated in an array. If there are constraints on the bounds (e.g., a lower bound is set), the upper bound is set to `Object.class` by default.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/Platform.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static android.os.Build.VERSION.SDK_INT;\n\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\n\nfinal class Platform {\n  static final @Nullable Executor callbackExecutor;\n  static final Reflection reflection;\n  static final BuiltInFactories builtInFactories;\n\n  static {\n    switch (System.getProperty(\"java.vm.name\")) {\n      case \"Dalvik\":\n        callbackExecutor = new AndroidMainExecutor();\n        if (SDK_INT >= 24) {\n          reflection = new Reflection.Android24();\n          builtInFactories = new BuiltInFactories.Java8();\n        } else {\n          reflection = new Reflection();\n          builtInFactories = new BuiltInFactories();\n        }\n        break;\n\n      case \"RoboVM\":\n        callbackExecutor = null;\n        reflection = new Reflection();\n        builtInFactories = new BuiltInFactories();\n        break;\n\n      default:\n        callbackExecutor = null;\n        reflection = new Reflection.Java8();\n        builtInFactories = new BuiltInFactories.Java8();\n        break;\n    }\n  }\n\n  private Platform() {}\n}\n\n// Path: retrofit/src/main/java/retrofit2/Response.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.Headers;\nimport okhttp3.Protocol;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\n\n/** An HTTP response. */\npublic final class Response<T> {\n  /** Create a synthetic successful response with {@code body} as the deserialized body. */\n  public static <T> Response<T> success(@Nullable T body) {\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response with an HTTP status code of {@code code} and {@code\n   * body} as the deserialized body.\n   */\n  public static <T> Response<T> success(int code, @Nullable T body) {\n    if (code < 200 || code >= 300) {\n      throw new IllegalArgumentException(\"code < 200 or >= 300: \" + code);\n    }\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(code)\n            .message(\"Response.success()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response using {@code headers} with {@code body} as the\n   * deserialized body.\n   */\n  public static <T> Response<T> success(@Nullable T body, Headers headers) {\n    Objects.requireNonNull(headers, \"headers == null\");\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .headers(headers)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a successful response from {@code rawResponse} with {@code body} as the deserialized\n   * body.\n   */\n  public static <T> Response<T> success(@Nullable T body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (!rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse must be successful response\");\n    }\n    return new Response<>(rawResponse, body, null);\n  }\n\n  /**\n   * Create a synthetic error response with an HTTP status code of {@code code} and {@code body} as\n   * the error body.\n   */\n  public static <T> Response<T> error(int code, ResponseBody body) {\n    Objects.requireNonNull(body, \"body == null\");\n    if (code < 400) throw new IllegalArgumentException(\"code < 400: \" + code);\n    return error(\n        body,\n        new okhttp3.Response.Builder() //\n            .body(new OkHttpCall.NoContentResponseBody(body.contentType(), body.contentLength()))\n            .code(code)\n            .message(\"Response.error()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /** Create an error response from {@code rawResponse} with {@code body} as the error body. */\n  public static <T> Response<T> error(ResponseBody body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(body, \"body == null\");\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse should not be successful response\");\n    }\n    return new Response<>(rawResponse, null, body);\n  }\n\n  private final okhttp3.Response rawResponse;\n  private final @Nullable T body;\n  private final @Nullable ResponseBody errorBody;\n\n  private Response(\n      okhttp3.Response rawResponse, @Nullable T body, @Nullable ResponseBody errorBody) {\n    this.rawResponse = rawResponse;\n    this.body = body;\n    this.errorBody = errorBody;\n  }\n\n  /** The raw response from the HTTP client. */\n  public okhttp3.Response raw() {\n    return rawResponse;\n  }\n\n  /** HTTP status code. */\n  public int code() {\n    return rawResponse.code();\n  }\n\n  /** HTTP status message or null if unknown. */\n  public String message() {\n    return rawResponse.message();\n  }\n\n  /** HTTP headers. */\n  public Headers headers() {\n    return rawResponse.headers();\n  }\n\n  /** Returns true if {@link #code()} is in the range [200..300). */\n  public boolean isSuccessful() {\n    return rawResponse.isSuccessful();\n  }\n\n  /** The deserialized response body of a {@linkplain #isSuccessful() successful} response. */\n  public @Nullable T body() {\n    return body;\n  }\n\n  /** The raw response body of an {@linkplain #isSuccessful() unsuccessful} response. */\n  public @Nullable ResponseBody errorBody() {\n    return errorBody;\n  }\n\n  @Override\n  public String toString() {\n    return rawResponse.toString();\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Utils.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\n\nfinal class Utils {\n  static final Type[] EMPTY_TYPE_ARRAY = new Type[0];\n\n  private Utils() {\n    // No instances.\n  }\n\n  static RuntimeException methodError(Method method, String message, Object... args) {\n    return methodError(method, null, message, args);\n  }\n\n  @SuppressWarnings(\"AnnotateFormatMethod\")\n  static RuntimeException methodError(\n      Method method, @Nullable Throwable cause, String message, Object... args) {\n    message = String.format(message, args);\n    return new IllegalArgumentException(\n        message\n            + \"\\n    for method \"\n            + method.getDeclaringClass().getSimpleName()\n            + \".\"\n            + method.getName(),\n        cause);\n  }\n\n  static RuntimeException parameterError(\n      Method method, Throwable cause, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, cause, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static RuntimeException parameterError(Method method, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static Class<?> getRawType(Type type) {\n    Objects.requireNonNull(type, \"type == null\");\n\n    if (type instanceof Class<?>) {\n      // Type is a normal class.\n      return (Class<?>) type;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n\n      // I'm not exactly sure why getRawType() returns Type instead of Class. Neal isn't either but\n      // suspects some pathological case related to nested classes exists.\n      Type rawType = parameterizedType.getRawType();\n      if (!(rawType instanceof Class)) throw new IllegalArgumentException();\n      return (Class<?>) rawType;\n    }\n    if (type instanceof GenericArrayType) {\n      Type componentType = ((GenericArrayType) type).getGenericComponentType();\n      return Array.newInstance(getRawType(componentType), 0).getClass();\n    }\n    if (type instanceof TypeVariable) {\n      // We could use the variable's bounds, but that won't work if there are multiple. Having a raw\n      // type that's more general than necessary is okay.\n      return Object.class;\n    }\n    if (type instanceof WildcardType) {\n      return getRawType(((WildcardType) type).getUpperBounds()[0]);\n    }\n\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + type.getClass().getName());\n  }\n\n  /** Returns true if {@code a} and {@code b} are equal. */\n  static boolean equals(Type a, Type b) {\n    if (a == b) {\n      return true; // Also handles (a == null && b == null).\n\n    } else if (a instanceof Class) {\n      return a.equals(b); // Class already specifies equals().\n\n    } else if (a instanceof ParameterizedType) {\n      if (!(b instanceof ParameterizedType)) return false;\n      ParameterizedType pa = (ParameterizedType) a;\n      ParameterizedType pb = (ParameterizedType) b;\n      Object ownerA = pa.getOwnerType();\n      Object ownerB = pb.getOwnerType();\n      boolean ownersAreEqual = ownerA == ownerB || (ownerA != null && ownerA.equals(ownerB));\n      boolean rawTypesAreEqual = pa.getRawType().equals(pb.getRawType());\n      boolean typeArgumentsAreEqual =\n          Arrays.equals(pa.getActualTypeArguments(), pb.getActualTypeArguments());\n\n      return ownersAreEqual && rawTypesAreEqual && typeArgumentsAreEqual;\n\n    } else if (a instanceof GenericArrayType) {\n      if (!(b instanceof GenericArrayType)) return false;\n      GenericArrayType ga = (GenericArrayType) a;\n      GenericArrayType gb = (GenericArrayType) b;\n      return equals(ga.getGenericComponentType(), gb.getGenericComponentType());\n\n    } else if (a instanceof WildcardType) {\n      if (!(b instanceof WildcardType)) return false;\n      WildcardType wa = (WildcardType) a;\n      WildcardType wb = (WildcardType) b;\n      return Arrays.equals(wa.getUpperBounds(), wb.getUpperBounds())\n          && Arrays.equals(wa.getLowerBounds(), wb.getLowerBounds());\n\n    } else if (a instanceof TypeVariable) {\n      if (!(b instanceof TypeVariable)) return false;\n      TypeVariable<?> va = (TypeVariable<?>) a;\n      TypeVariable<?> vb = (TypeVariable<?>) b;\n      return va.getGenericDeclaration() == vb.getGenericDeclaration()\n          && va.getName().equals(vb.getName());\n\n    } else {\n      return false; // This isn't a type we support!\n    }\n  }\n\n  /**\n   * Returns the generic supertype for {@code supertype}. For example, given a class {@code\n   * IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>} and the\n   * result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.\n   */\n  static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {\n    if (toResolve == rawType) return context;\n\n    // We skip searching through interfaces if unknown is an interface.\n    if (toResolve.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == toResolve) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (toResolve.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], toResolve);\n        }\n      }\n    }\n\n    // Check our supertypes.\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == toResolve) {\n          return rawType.getGenericSuperclass();\n        } else if (toResolve.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, toResolve);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // We can't resolve this further.\n    return toResolve;\n  }\n\n  private static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0; i < array.length; i++) {\n      if (toFind.equals(array[i])) return i;\n    }\n    throw new NoSuchElementException();\n  }\n\n  static String typeToString(Type type) {\n    return type instanceof Class ? ((Class<?>) type).getName() : type.toString();\n  }\n\n  /**\n   * Returns the generic form of {@code supertype}. For example, if this is {@code\n   * ArrayList<String>}, this returns {@code Iterable<String>} given the input {@code\n   * Iterable.class}.\n   *\n   * @param supertype a superclass of, or interface implemented by, this.\n   */\n  static Type getSupertype(Type context, Class<?> contextRawType, Class<?> supertype) {\n    if (!supertype.isAssignableFrom(contextRawType)) throw new IllegalArgumentException();\n    return resolve(\n        context, contextRawType, getGenericSupertype(context, contextRawType, supertype));\n  }\n\n  static Type resolve(Type context, Class<?> contextRawType, Type toResolve) {\n    // This implementation is made a little more complicated in an attempt to avoid object-creation.\n    while (true) {\n      if (toResolve instanceof TypeVariable) {\n        TypeVariable<?> typeVariable = (TypeVariable<?>) toResolve;\n        toResolve = resolveTypeVariable(context, contextRawType, typeVariable);\n        if (toResolve == typeVariable) {\n          return toResolve;\n        }\n\n      } else if (toResolve instanceof Class && ((Class<?>) toResolve).isArray()) {\n        Class<?> original = (Class<?>) toResolve;\n        Type componentType = original.getComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof GenericArrayType) {\n        GenericArrayType original = (GenericArrayType) toResolve;\n        Type componentType = original.getGenericComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof ParameterizedType) {\n        ParameterizedType original = (ParameterizedType) toResolve;\n        Type ownerType = original.getOwnerType();\n        Type newOwnerType = resolve(context, contextRawType, ownerType);\n        boolean changed = newOwnerType != ownerType;\n\n        Type[] args = original.getActualTypeArguments();\n        for (int t = 0, length = args.length; t < length; t++) {\n          Type resolvedTypeArgument = resolve(context, contextRawType, args[t]);\n          if (resolvedTypeArgument != args[t]) {\n            if (!changed) {\n              args = args.clone();\n              changed = true;\n            }\n            args[t] = resolvedTypeArgument;\n          }\n        }\n\n        return changed\n            ? new ParameterizedTypeImpl(newOwnerType, original.getRawType(), args)\n            : original;\n\n      } else if (toResolve instanceof WildcardType) {\n        WildcardType original = (WildcardType) toResolve;\n        Type[] originalLowerBound = original.getLowerBounds();\n        Type[] originalUpperBound = original.getUpperBounds();\n\n        if (originalLowerBound.length == 1) {\n          Type lowerBound = resolve(context, contextRawType, originalLowerBound[0]);\n          if (lowerBound != originalLowerBound[0]) {\n            return new WildcardTypeImpl(new Type[] {Object.class}, new Type[] {lowerBound});\n          }\n        } else if (originalUpperBound.length == 1) {\n          Type upperBound = resolve(context, contextRawType, originalUpperBound[0]);\n          if (upperBound != originalUpperBound[0]) {\n            return new WildcardTypeImpl(new Type[] {upperBound}, EMPTY_TYPE_ARRAY);\n          }\n        }\n        return original;\n\n      } else {\n        return toResolve;\n      }\n    }\n  }\n\n  private static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // We can't reduce this further.\n    if (declaredByRaw == null) return unknown;\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static @Nullable Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n\n  static void checkNotPrimitive(Type type) {\n    if (type instanceof Class<?> && ((Class<?>) type).isPrimitive()) {\n      throw new IllegalArgumentException();\n    }\n  }\n\n  /** Returns true if {@code annotations} contains an instance of {@code cls}. */\n  static boolean isAnnotationPresent(Annotation[] annotations, Class<? extends Annotation> cls) {\n    for (Annotation annotation : annotations) {\n      if (cls.isInstance(annotation)) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  static ResponseBody buffer(final ResponseBody body) throws IOException {\n    Buffer buffer = new Buffer();\n    body.source().readAll(buffer);\n    return ResponseBody.create(body.contentType(), body.contentLength(), buffer);\n  }\n\n  static Type getParameterUpperBound(int index, ParameterizedType type) {\n    Type[] types = type.getActualTypeArguments();\n    if (index < 0 || index >= types.length) {\n      throw new IllegalArgumentException(\n          \"Index \" + index + \" not in range [0,\" + types.length + \") for \" + type);\n    }\n    Type paramType = types[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getUpperBounds()[0];\n    }\n    return paramType;\n  }\n\n  static Type getParameterLowerBound(int index, ParameterizedType type) {\n    Type paramType = type.getActualTypeArguments()[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getLowerBounds()[0];\n    }\n    return paramType;\n  }\n\n  static boolean hasUnresolvableType(@Nullable Type type) {\n    if (type instanceof Class<?>) {\n      return false;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n      for (Type typeArgument : parameterizedType.getActualTypeArguments()) {\n        if (hasUnresolvableType(typeArgument)) {\n          return true;\n        }\n      }\n      return false;\n    }\n    if (type instanceof GenericArrayType) {\n      return hasUnresolvableType(((GenericArrayType) type).getGenericComponentType());\n    }\n    if (type instanceof TypeVariable) {\n      return true;\n    }\n    if (type instanceof WildcardType) {\n      return true;\n    }\n    String className = type == null ? \"null\" : type.getClass().getName();\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + className);\n  }\n\n  static final class ParameterizedTypeImpl implements ParameterizedType {\n    private final @Nullable Type ownerType;\n    private final Type rawType;\n    private final Type[] typeArguments;\n\n    ParameterizedTypeImpl(@Nullable Type ownerType, Type rawType, Type... typeArguments) {\n      // Require an owner type if the raw type needs it.\n      if (rawType instanceof Class<?>\n          && (ownerType == null) != (((Class<?>) rawType).getEnclosingClass() == null)) {\n        throw new IllegalArgumentException();\n      }\n\n      for (Type typeArgument : typeArguments) {\n        Objects.requireNonNull(typeArgument, \"typeArgument == null\");\n        checkNotPrimitive(typeArgument);\n      }\n\n      this.ownerType = ownerType;\n      this.rawType = rawType;\n      this.typeArguments = typeArguments.clone();\n    }\n\n    @Override\n    public Type[] getActualTypeArguments() {\n      return typeArguments.clone();\n    }\n\n    @Override\n    public Type getRawType() {\n      return rawType;\n    }\n\n    @Override\n    public @Nullable Type getOwnerType() {\n      return ownerType;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof ParameterizedType && Utils.equals(this, (ParameterizedType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      return Arrays.hashCode(typeArguments)\n          ^ rawType.hashCode()\n          ^ (ownerType != null ? ownerType.hashCode() : 0);\n    }\n\n    @Override\n    public String toString() {\n      if (typeArguments.length == 0) return typeToString(rawType);\n      StringBuilder result = new StringBuilder(30 * (typeArguments.length + 1));\n      result.append(typeToString(rawType));\n      result.append(\"<\").append(typeToString(typeArguments[0]));\n      for (int i = 1; i < typeArguments.length; i++) {\n        result.append(\", \").append(typeToString(typeArguments[i]));\n      }\n      return result.append(\">\").toString();\n    }\n  }\n\n  private static final class GenericArrayTypeImpl implements GenericArrayType {\n    private final Type componentType;\n\n    GenericArrayTypeImpl(Type componentType) {\n      this.componentType = componentType;\n    }\n\n    @Override\n    public Type getGenericComponentType() {\n      return componentType;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      return o instanceof GenericArrayType && Utils.equals(this, (GenericArrayType) o);\n    }\n\n    @Override\n    public int hashCode() {\n      return componentType.hashCode();\n    }\n\n    @Override\n    public String toString() {\n      return typeToString(componentType) + \"[]\";\n    }\n  }\n\n  /**\n   * The WildcardType interface supports multiple upper bounds and multiple lower bounds. We only\n   * support what the Java 6 language needs - at most one bound. If a lower bound is set, the upper\n   * bound must be Object.class.\n   */\n  private static final class WildcardTypeImpl implements WildcardType {\n    private final Type upperBound;\n    private final @Nullable Type lowerBound;\n\n    WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) {\n      if (lowerBounds.length > 1) throw new IllegalArgumentException();\n      if (upperBounds.length != 1) throw new IllegalArgumentException();\n\n      if (lowerBounds.length == 1) {\n        if (lowerBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(lowerBounds[0]);\n        if (upperBounds[0] != Object.class) throw new IllegalArgumentException();\n        this.lowerBound = lowerBounds[0];\n        this.upperBound = Object.class;\n      } else {\n        if (upperBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(upperBounds[0]);\n        this.lowerBound = null;\n        this.upperBound = upperBounds[0];\n      }\n    }\n\n    \n@Override\n    public Type[] getUpperBounds() {\n      return new Type[] {upperBound};\n    }\n\n    @Override\n    public Type[] getLowerBounds() {\n      return lowerBound != null ? new Type[] {lowerBound} : EMPTY_TYPE_ARRAY;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof WildcardType && Utils.equals(this, (WildcardType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      // This equals Arrays.hashCode(getLowerBounds()) ^ Arrays.hashCode(getUpperBounds()).\n      return (lowerBound != null ? 31 + lowerBound.hashCode() : 1) ^ (31 + upperBound.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      if (lowerBound != null) return \"? super \" + typeToString(lowerBound);\n      if (upperBound == Object.class) return \"?\";\n      return \"? extends \" + typeToString(upperBound);\n    }\n  }\n\n  // https://github.com/ReactiveX/RxJava/blob/6a44e5d0543a48f1c378dc833a155f3f71333bc2/\n  // src/main/java/io/reactivex/exceptions/Exceptions.java#L66\n  static void throwIfFatal(Throwable t) {\n    if (t instanceof VirtualMachineError) {\n      throw (VirtualMachineError) t;\n    } else if (t instanceof ThreadDeath) {\n      throw (ThreadDeath) t;\n    } else if (t instanceof LinkageError) {\n      throw (LinkageError) t;\n    }\n  }\n\n  /** Not volatile because we don't mind multiple threads discovering this. */\n  private static boolean checkForKotlinUnit = true;\n\n  static boolean isUnit(Type type) {\n    if (checkForKotlinUnit) {\n      try {\n        return type == Unit.class;\n      } catch (NoClassDefFoundError ignored) {\n        checkForKotlinUnit = false;\n      }\n    }\n    return false;\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/OkHttpCall.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.throwIfFatal;\n\nimport java.io.IOException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport javax.annotation.concurrent.GuardedBy;\nimport okhttp3.MediaType;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\nimport okio.BufferedSource;\nimport okio.ForwardingSource;\nimport okio.Okio;\nimport okio.Timeout;\n\nfinal class OkHttpCall<T> implements Call<T> {\n  private final RequestFactory requestFactory;\n  private final Object instance;\n  private final Object[] args;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, T> responseConverter;\n\n  private volatile boolean canceled;\n\n  @GuardedBy(\"this\")\n  private @Nullable okhttp3.Call rawCall;\n\n  @GuardedBy(\"this\") // Either a RuntimeException, non-fatal Error, or IOException.\n  private @Nullable Throwable creationFailure;\n\n  @GuardedBy(\"this\")\n  private boolean executed;\n\n  OkHttpCall(\n      RequestFactory requestFactory,\n      Object instance,\n      Object[] args,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, T> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.instance = instance;\n    this.args = args;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @SuppressWarnings(\"CloneDoesntCallSuperClone\") // We are a final type & this saves clearing state.\n  @Override\n  public OkHttpCall<T> clone() {\n    return new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n  }\n\n  @Override\n  public synchronized Request request() {\n    try {\n      return getRawCall().request();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create request.\", e);\n    }\n  }\n\n  @Override\n  public synchronized Timeout timeout() {\n    try {\n      return getRawCall().timeout();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create call.\", e);\n    }\n  }\n\n  /**\n   * Returns the raw call, initializing it if necessary. Throws if initializing the raw call throws,\n   * or has thrown in previous attempts to create it.\n   */\n  @GuardedBy(\"this\")\n  private okhttp3.Call getRawCall() throws IOException {\n    okhttp3.Call call = rawCall;\n    if (call != null) return call;\n\n    // Re-throw previous failures if this isn't the first attempt.\n    if (creationFailure != null) {\n      if (creationFailure instanceof IOException) {\n        throw (IOException) creationFailure;\n      } else if (creationFailure instanceof RuntimeException) {\n        throw (RuntimeException) creationFailure;\n      } else {\n        throw (Error) creationFailure;\n      }\n    }\n\n    // Create and remember either the success or the failure.\n    try {\n      return rawCall = createRawCall();\n    } catch (RuntimeException | Error | IOException e) {\n      throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n      creationFailure = e;\n      throw e;\n    }\n  }\n\n  @Override\n  public void enqueue(final Callback<T> callback) {\n    Objects.requireNonNull(callback, \"callback == null\");\n\n    okhttp3.Call call;\n    Throwable failure;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = rawCall;\n      failure = creationFailure;\n      if (call == null && failure == null) {\n        try {\n          call = rawCall = createRawCall();\n        } catch (Throwable t) {\n          throwIfFatal(t);\n          failure = creationFailure = t;\n        }\n      }\n    }\n\n    if (failure != null) {\n      callback.onFailure(this, failure);\n      return;\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    call.enqueue(\n        new okhttp3.Callback() {\n          @Override\n          public void onResponse(okhttp3.Call call, okhttp3.Response rawResponse) {\n            Response<T> response;\n            try {\n              response = parseResponse(rawResponse);\n            } catch (Throwable e) {\n              throwIfFatal(e);\n              callFailure(e);\n              return;\n            }\n\n            try {\n              callback.onResponse(OkHttpCall.this, response);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n\n          @Override\n          public void onFailure(okhttp3.Call call, IOException e) {\n            callFailure(e);\n          }\n\n          private void callFailure(Throwable e) {\n            try {\n              callback.onFailure(OkHttpCall.this, e);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n        });\n  }\n\n  @Override\n  public synchronized boolean isExecuted() {\n    return executed;\n  }\n\n  @Override\n  public Response<T> execute() throws IOException {\n    okhttp3.Call call;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = getRawCall();\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    return parseResponse(call.execute());\n  }\n\n  private okhttp3.Call createRawCall() throws IOException {\n    okhttp3.Call call = callFactory.newCall(requestFactory.create(instance, args));\n    if (call == null) {\n      throw new NullPointerException(\"Call.Factory returned null.\");\n    }\n    return call;\n  }\n\n  Response<T> parseResponse(okhttp3.Response rawResponse) throws IOException {\n    ResponseBody rawBody = rawResponse.body();\n\n    // Remove the body's source (the only stateful object) so we can pass the response along.\n    rawResponse =\n        rawResponse\n            .newBuilder()\n            .body(new NoContentResponseBody(rawBody.contentType(), rawBody.contentLength()))\n            .build();\n\n    int code = rawResponse.code();\n    if (code < 200 || code >= 300) {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        ResponseBody bufferedBody = Utils.buffer(rawBody);\n        return Response.error(bufferedBody, rawResponse);\n      } finally {\n        rawBody.close();\n      }\n    }\n\n    if (code == 204 || code == 205) {\n      rawBody.close();\n      return Response.success(null, rawResponse);\n    }\n\n    ExceptionCatchingResponseBody catchingBody = new ExceptionCatchingResponseBody(rawBody);\n    try {\n      T body = responseConverter.convert(catchingBody);\n      return Response.success(body, rawResponse);\n    } catch (RuntimeException e) {\n      // If the underlying source threw an exception, propagate that rather than indicating it was\n      // a runtime exception.\n      catchingBody.throwIfCaught();\n      throw e;\n    }\n  }\n\n  @Override\n  public void cancel() {\n    canceled = true;\n\n    okhttp3.Call call;\n    synchronized (this) {\n      call = rawCall;\n    }\n    if (call != null) {\n      call.cancel();\n    }\n  }\n\n  @Override\n  public boolean isCanceled() {\n    if (canceled) {\n      return true;\n    }\n    synchronized (this) {\n      return rawCall != null && rawCall.isCanceled();\n    }\n  }\n\n  static final class NoContentResponseBody extends ResponseBody {\n    private final @Nullable MediaType contentType;\n    private final long contentLength;\n\n    NoContentResponseBody(@Nullable MediaType contentType, long contentLength) {\n      this.contentType = contentType;\n      this.contentLength = contentLength;\n    }\n\n    @Override\n    public MediaType contentType() {\n      return contentType;\n    }\n\n    @Override\n    public long contentLength() {\n      return contentLength;\n    }\n\n    @Override\n    public BufferedSource source() {\n      throw new IllegalStateException(\"Cannot read raw response body of a converted body.\");\n    }\n  }\n\n  static final class ExceptionCatchingResponseBody extends ResponseBody {\n    private final ResponseBody delegate;\n    private final BufferedSource delegateSource;\n    @Nullable IOException thrownException;\n\n    ExceptionCatchingResponseBody(ResponseBody delegate) {\n      this.delegate = delegate;\n      this.delegateSource =\n          Okio.buffer(\n              new ForwardingSource(delegate.source()) {\n                @Override\n                public long read(Buffer sink, long byteCount) throws IOException {\n                  try {\n                    return super.read(sink, byteCount);\n                  } catch (IOException e) {\n                    thrownException = e;\n                    throw e;\n                  }\n                }\n              });\n    }\n\n    @Override\n    public MediaType contentType() {\n      return delegate.contentType();\n    }\n\n    @Override\n    public long contentLength() {\n      return delegate.contentLength();\n    }\n\n    @Override\n    public BufferedSource source() {\n      return delegateSource;\n    }\n\n    @Override\n    public void close() {\n      delegate.close();\n    }\n\n    void throwIfCaught() throws IOException {\n      if (thrownException != null) {\n        throw thrownException;\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Streaming.java\n/*\n * Copyright (C) 2014 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.ResponseBody;\n\n/**\n * Treat the response body on methods returning {@link ResponseBody ResponseBody} as is, i.e.\n * without converting the body to {@code byte[]}.\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface Streaming {}\n\n// Path: retrofit/src/main/java/retrofit2/BuiltInConverters.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.Streaming;\n\nfinal class BuiltInConverters extends Converter.Factory {\n\n  @Override\n  public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n      Type type, Annotation[] annotations, Retrofit retrofit) {\n    if (type == ResponseBody.class) {\n      return Utils.isAnnotationPresent(annotations, Streaming.class)\n          ? StreamingResponseBodyConverter.INSTANCE\n          : BufferingResponseBodyConverter.INSTANCE;\n    }\n    if (type == Void.class) {\n      return VoidResponseBodyConverter.INSTANCE;\n    }\n    if (Utils.isUnit(type)) {\n      return UnitResponseBodyConverter.INSTANCE;\n    }\n...\n", "needle_token_start": 7150, "needle_token_end": 7175, "code_context_ntokens": 11146, "output": ["```java\nstatic Type[] getParameterUpperBound(int index, ParameterizedType type) {\n  Type[] types = type.getActualTypeArguments();\n  if (index < 0 || index >= types.length) {\n    throw new IllegalArgumentException(\n        \"Index \" + index + \" not in range [0,\" + types.length + \") for \" + type);\n  }\n  Type paramType = types[index];\n  if (paramType instanceof WildcardType) {\n    return ((WildcardType) paramType).getUpperBounds()[0];\n  }\n  return paramType;\n}\n```"]}
{"repo": "square/retrofit", "name": "callAdapter", "language": "java", "path": "retrofit/src/main/java/retrofit2/Retrofit.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To retrieve an adapter that can convert network responses into a specific data type based on the method's return type and annotations.\n2. **Input**: A `Type` representing the return type of a method, and an array of `Annotation` that may annotate the method.\n3. **Output**: An instance of `CallAdapter<?, ?>` that matches the provided return type and annotations.\n4. **Procedure**: The function iterates through a list of factory objects to find one that can provide a suitable adapter for the specified return type and annotations. If no suitable adapter is found, it throws an exception indicating the absence of an appropriate adapter.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/http/Tag.java\n/*\n * Copyright (C) 2019 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.PARAMETER;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\n\n/**\n...\n// Path: retrofit/src/main/java/retrofit2/http/Url.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.PARAMETER;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.HttpUrl;\nimport retrofit2.Retrofit;\n\n/**\n * URL resolved against the {@linkplain Retrofit#baseUrl() base URL}.\n *\n * <pre><code>\n * &#64;GET\n * Call&lt;ResponseBody&gt; list(@Url String url);\n * </code></pre>\n *\n * <p>See {@linkplain retrofit2.Retrofit.Builder#baseUrl(HttpUrl) base URL} for details of how the\n * value will be resolved against a base URL to create the full endpoint URL.\n */\n@Documented\n@Target(PARAMETER)\n@Retention(RUNTIME)\npublic @interface Url {}\n\n// Path: retrofit/src/main/java/retrofit2/RequestFactory.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.methodError;\nimport static retrofit2.Utils.parameterError;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.net.URI;\nimport java.util.ArrayList;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport javax.annotation.Nullable;\nimport kotlin.coroutines.Continuation;\nimport okhttp3.Headers;\nimport okhttp3.HttpUrl;\nimport okhttp3.MediaType;\nimport okhttp3.MultipartBody;\nimport okhttp3.RequestBody;\nimport retrofit2.http.Body;\nimport retrofit2.http.DELETE;\nimport retrofit2.http.Field;\nimport retrofit2.http.FieldMap;\nimport retrofit2.http.FormUrlEncoded;\nimport retrofit2.http.GET;\nimport retrofit2.http.HEAD;\nimport retrofit2.http.HTTP;\nimport retrofit2.http.Header;\nimport retrofit2.http.HeaderMap;\nimport retrofit2.http.Multipart;\nimport retrofit2.http.OPTIONS;\nimport retrofit2.http.PATCH;\nimport retrofit2.http.POST;\nimport retrofit2.http.PUT;\nimport retrofit2.http.Part;\nimport retrofit2.http.PartMap;\nimport retrofit2.http.Path;\nimport retrofit2.http.Query;\nimport retrofit2.http.QueryMap;\nimport retrofit2.http.QueryName;\nimport retrofit2.http.Tag;\nimport retrofit2.http.Url;\n\nfinal class RequestFactory {\n  static RequestFactory parseAnnotations(Retrofit retrofit, Class<?> service, Method method) {\n    return new Builder(retrofit, service, method).build();\n  }\n\n  private final Class<?> service;\n  private final Method method;\n  private final HttpUrl baseUrl;\n  final String httpMethod;\n  private final @Nullable String relativeUrl;\n  private final @Nullable Headers headers;\n  private final @Nullable MediaType contentType;\n  private final boolean hasBody;\n  private final boolean isFormEncoded;\n  private final boolean isMultipart;\n  private final ParameterHandler<?>[] parameterHandlers;\n  final boolean isKotlinSuspendFunction;\n\n  RequestFactory(Builder builder) {\n    service = builder.service;\n    method = builder.method;\n    baseUrl = builder.retrofit.baseUrl;\n    httpMethod = builder.httpMethod;\n    relativeUrl = builder.relativeUrl;\n    headers = builder.headers;\n    contentType = builder.contentType;\n    hasBody = builder.hasBody;\n    isFormEncoded = builder.isFormEncoded;\n    isMultipart = builder.isMultipart;\n    parameterHandlers = builder.parameterHandlers;\n    isKotlinSuspendFunction = builder.isKotlinSuspendFunction;\n  }\n\n  okhttp3.Request create(@Nullable Object instance, Object[] args) throws IOException {\n    @SuppressWarnings(\"unchecked\") // It is an error to invoke a method with the wrong arg types.\n    ParameterHandler<Object>[] handlers = (ParameterHandler<Object>[]) parameterHandlers;\n\n    int argumentCount = args.length;\n    if (argumentCount != handlers.length) {\n      throw new IllegalArgumentException(\n          \"Argument count (\"\n              + argumentCount\n              + \") doesn't match expected count (\"\n              + handlers.length\n              + \")\");\n    }\n\n    RequestBuilder requestBuilder =\n        new RequestBuilder(\n            httpMethod,\n            baseUrl,\n            relativeUrl,\n            headers,\n            contentType,\n            hasBody,\n            isFormEncoded,\n            isMultipart);\n\n    if (isKotlinSuspendFunction) {\n      // The Continuation is the last parameter and the handlers array contains null at that index.\n      argumentCount--;\n    }\n\n    List<Object> argumentList = new ArrayList<>(argumentCount);\n    for (int p = 0; p < argumentCount; p++) {\n      argumentList.add(args[p]);\n      handlers[p].apply(requestBuilder, args[p]);\n    }\n\n    return requestBuilder\n        .get()\n        .tag(Invocation.class, new Invocation(service, instance, method, argumentList))\n        .build();\n  }\n\n  /**\n   * Inspects the annotations on an interface method to construct a reusable service method. This\n   * requires potentially-expensive reflection so it is best to build each service method only once\n   * and reuse it. Builders cannot be reused.\n   */\n  static final class Builder {\n    // Upper and lower characters, digits, underscores, and hyphens, starting with a character.\n    private static final String PARAM = \"[a-zA-Z][a-zA-Z0-9_-]*\";\n    private static final Pattern PARAM_URL_REGEX = Pattern.compile(\"\\\\{(\" + PARAM + \")\\\\}\");\n    private static final Pattern PARAM_NAME_REGEX = Pattern.compile(PARAM);\n\n    final Retrofit retrofit;\n    final Class<?> service;\n    final Method method;\n    final Annotation[] methodAnnotations;\n    final Annotation[][] parameterAnnotationsArray;\n    final Type[] parameterTypes;\n\n    boolean gotField;\n    boolean gotPart;\n    boolean gotBody;\n    boolean gotPath;\n    boolean gotQuery;\n    boolean gotQueryName;\n    boolean gotQueryMap;\n    boolean gotUrl;\n    @Nullable String httpMethod;\n    boolean hasBody;\n    boolean isFormEncoded;\n    boolean isMultipart;\n    @Nullable String relativeUrl;\n    @Nullable Headers headers;\n    @Nullable MediaType contentType;\n    @Nullable Set<String> relativeUrlParamNames;\n    @Nullable ParameterHandler<?>[] parameterHandlers;\n    boolean isKotlinSuspendFunction;\n\n    Builder(Retrofit retrofit, Class<?> service, Method method) {\n      this.retrofit = retrofit;\n      this.service = service;\n      this.method = method;\n      this.methodAnnotations = method.getAnnotations();\n      this.parameterTypes = method.getGenericParameterTypes();\n      this.parameterAnnotationsArray = method.getParameterAnnotations();\n    }\n\n    RequestFactory build() {\n      for (Annotation annotation : methodAnnotations) {\n        parseMethodAnnotation(annotation);\n      }\n\n      if (httpMethod == null) {\n        throw methodError(method, \"HTTP method annotation is required (e.g., @GET, @POST, etc.).\");\n      }\n\n      if (!hasBody) {\n        if (isMultipart) {\n          throw methodError(\n              method,\n              \"Multipart can only be specified on HTTP methods with request body (e.g., @POST).\");\n        }\n        if (isFormEncoded) {\n          throw methodError(\n              method,\n              \"FormUrlEncoded can only be specified on HTTP methods with \"\n                  + \"request body (e.g., @POST).\");\n        }\n      }\n\n      int parameterCount = parameterAnnotationsArray.length;\n      parameterHandlers = new ParameterHandler<?>[parameterCount];\n      for (int p = 0, lastParameter = parameterCount - 1; p < parameterCount; p++) {\n        parameterHandlers[p] =\n            parseParameter(p, parameterTypes[p], parameterAnnotationsArray[p], p == lastParameter);\n      }\n\n      if (relativeUrl == null && !gotUrl) {\n        throw methodError(method, \"Missing either @%s URL or @Url parameter.\", httpMethod);\n      }\n      if (!isFormEncoded && !isMultipart && !hasBody && gotBody) {\n        throw methodError(method, \"Non-body HTTP method cannot contain @Body.\");\n      }\n      if (isFormEncoded && !gotField) {\n        throw methodError(method, \"Form-encoded method must contain at least one @Field.\");\n      }\n      if (isMultipart && !gotPart) {\n        throw methodError(method, \"Multipart method must contain at least one @Part.\");\n      }\n\n      return new RequestFactory(this);\n    }\n\n    private void parseMethodAnnotation(Annotation annotation) {\n      if (annotation instanceof DELETE) {\n        parseHttpMethodAndPath(\"DELETE\", ((DELETE) annotation).value(), false);\n      } else if (annotation instanceof GET) {\n        parseHttpMethodAndPath(\"GET\", ((GET) annotation).value(), false);\n      } else if (annotation instanceof HEAD) {\n        parseHttpMethodAndPath(\"HEAD\", ((HEAD) annotation).value(), false);\n      } else if (annotation instanceof PATCH) {\n        parseHttpMethodAndPath(\"PATCH\", ((PATCH) annotation).value(), true);\n      } else if (annotation instanceof POST) {\n        parseHttpMethodAndPath(\"POST\", ((POST) annotation).value(), true);\n      } else if (annotation instanceof PUT) {\n        parseHttpMethodAndPath(\"PUT\", ((PUT) annotation).value(), true);\n      } else if (annotation instanceof OPTIONS) {\n        parseHttpMethodAndPath(\"OPTIONS\", ((OPTIONS) annotation).value(), false);\n      } else if (annotation instanceof HTTP) {\n        HTTP http = (HTTP) annotation;\n        parseHttpMethodAndPath(http.method(), http.path(), http.hasBody());\n      } else if (annotation instanceof retrofit2.http.Headers) {\n        retrofit2.http.Headers headers = (retrofit2.http.Headers) annotation;\n        String[] headersToParse = headers.value();\n        if (headersToParse.length == 0) {\n          throw methodError(method, \"@Headers annotation is empty.\");\n        }\n        this.headers = parseHeaders(headersToParse, headers.allowUnsafeNonAsciiValues());\n      } else if (annotation instanceof Multipart) {\n        if (isFormEncoded) {\n          throw methodError(method, \"Only one encoding annotation is allowed.\");\n        }\n        isMultipart = true;\n      } else if (annotation instanceof FormUrlEncoded) {\n        if (isMultipart) {\n          throw methodError(method, \"Only one encoding annotation is allowed.\");\n        }\n        isFormEncoded = true;\n      }\n    }\n\n    private void parseHttpMethodAndPath(String httpMethod, String value, boolean hasBody) {\n      if (this.httpMethod != null) {\n        throw methodError(\n            method,\n            \"Only one HTTP method is allowed. Found: %s and %s.\",\n            this.httpMethod,\n            httpMethod);\n      }\n      this.httpMethod = httpMethod;\n      this.hasBody = hasBody;\n\n      if (value.isEmpty()) {\n        return;\n      }\n\n      // Get the relative URL path and existing query string, if present.\n      int question = value.indexOf('?');\n      if (question != -1 && question < value.length() - 1) {\n        // Ensure the query string does not have any named parameters.\n        String queryParams = value.substring(question + 1);\n        Matcher queryParamMatcher = PARAM_URL_REGEX.matcher(queryParams);\n        if (queryParamMatcher.find()) {\n          throw methodError(\n              method,\n              \"URL query string \\\"%s\\\" must not have replace block. \"\n                  + \"For dynamic query parameters use @Query.\",\n              queryParams);\n        }\n      }\n\n      this.relativeUrl = value;\n      this.relativeUrlParamNames = parsePathParameters(value);\n    }\n\n    private Headers parseHeaders(String[] headers, boolean allowUnsafeNonAsciiValues) {\n      Headers.Builder builder = new Headers.Builder();\n      for (String header : headers) {\n        int colon = header.indexOf(':');\n        if (colon == -1 || colon == 0 || colon == header.length() - 1) {\n          throw methodError(\n              method, \"@Headers value must be in the form \\\"Name: Value\\\". Found: \\\"%s\\\"\", header);\n        }\n        String headerName = header.substring(0, colon);\n        String headerValue = header.substring(colon + 1).trim();\n        if (\"Content-Type\".equalsIgnoreCase(headerName)) {\n          try {\n            contentType = MediaType.get(headerValue);\n          } catch (IllegalArgumentException e) {\n            throw methodError(method, e, \"Malformed content type: %s\", headerValue);\n          }\n        } else if (allowUnsafeNonAsciiValues) {\n          builder.addUnsafeNonAscii(headerName, headerValue);\n        } else {\n          builder.add(headerName, headerValue);\n        }\n      }\n      return builder.build();\n    }\n\n    private @Nullable ParameterHandler<?> parseParameter(\n        int p, Type parameterType, @Nullable Annotation[] annotations, boolean allowContinuation) {\n      ParameterHandler<?> result = null;\n      if (annotations != null) {\n        for (Annotation annotation : annotations) {\n          ParameterHandler<?> annotationAction =\n              parseParameterAnnotation(p, parameterType, annotations, annotation);\n\n          if (annotationAction == null) {\n            continue;\n          }\n\n          if (result != null) {\n            throw parameterError(\n                method, p, \"Multiple Retrofit annotations found, only one allowed.\");\n          }\n\n          result = annotationAction;\n        }\n      }\n\n      if (result == null) {\n        if (allowContinuation) {\n          try {\n            if (Utils.getRawType(parameterType) == Continuation.class) {\n              isKotlinSuspendFunction = true;\n              return null;\n            }\n          } catch (NoClassDefFoundError ignored) {\n            // Ignored\n          }\n        }\n        throw parameterError(method, p, \"No Retrofit annotation found.\");\n      }\n\n      return result;\n    }\n\n    @Nullable\n    private ParameterHandler<?> parseParameterAnnotation(\n        int p, Type type, Annotation[] annotations, Annotation annotation) {\n      if (annotation instanceof Url) {\n        validateResolvableType(p, type);\n        if (gotUrl) {\n          throw parameterError(method, p, \"Multiple @Url method annotations found.\");\n        }\n        if (gotPath) {\n          throw parameterError(method, p, \"@Path parameters may not be used with @Url.\");\n        }\n        if (gotQuery) {\n          throw parameterError(method, p, \"A @Url parameter must not come after a @Query.\");\n        }\n        if (gotQueryName) {\n          throw parameterError(method, p, \"A @Url parameter must not come after a @QueryName.\");\n        }\n        if (gotQueryMap) {\n          throw parameterError(method, p, \"A @Url parameter must not come after a @QueryMap.\");\n        }\n        if (relativeUrl != null) {\n          throw parameterError(method, p, \"@Url cannot be used with @%s URL\", httpMethod);\n        }\n\n        gotUrl = true;\n\n        if (type == HttpUrl.class\n            || type == String.class\n            || type == URI.class\n            || (type instanceof Class && \"android.net.Uri\".equals(((Class<?>) type).getName()))) {\n          return new ParameterHandler.RelativeUrl(method, p);\n        } else {\n          throw parameterError(\n              method,\n              p,\n              \"@Url must be okhttp3.HttpUrl, String, java.net.URI, or android.net.Uri type.\");\n        }\n\n      } else if (annotation instanceof Path) {\n        validateResolvableType(p, type);\n        if (gotQuery) {\n          throw parameterError(method, p, \"A @Path parameter must not come after a @Query.\");\n        }\n        if (gotQueryName) {\n          throw parameterError(method, p, \"A @Path parameter must not come after a @QueryName.\");\n        }\n        if (gotQueryMap) {\n          throw parameterError(method, p, \"A @Path parameter must not come after a @QueryMap.\");\n        }\n        if (gotUrl) {\n          throw parameterError(method, p, \"@Path parameters may not be used with @Url.\");\n        }\n        if (relativeUrl == null) {\n          throw parameterError(\n              method, p, \"@Path can only be used with relative url on @%s\", httpMethod);\n        }\n        gotPath = true;\n\n        Path path = (Path) annotation;\n        String name = path.value();\n        validatePathName(p, name);\n\n        Converter<?, String> converter = retrofit.stringConverter(type, annotations);\n        return new ParameterHandler.Path<>(method, p, name, converter, path.encoded());\n\n      } else if (annotation instanceof Query) {\n        validateResolvableType(p, type);\n        Query query = (Query) annotation;\n        String name = query.value();\n        boolean encoded = query.encoded();\n\n        Class<?> rawParameterType = Utils.getRawType(type);\n        gotQuery = true;\n        if (Iterable.class.isAssignableFrom(rawParameterType)) {\n          if (!(type instanceof ParameterizedType)) {\n            throw parameterError(\n                method,\n                p,\n                rawParameterType.getSimpleName()\n                    + \" must include generic type (e.g., \"\n                    + rawParameterType.getSimpleName()\n                    + \"<String>)\");\n          }\n          ParameterizedType parameterizedType = (ParameterizedType) type;\n          Type iterableType = Utils.getParameterUpperBound(0, parameterizedType);\n          Converter<?, String> converter = retrofit.stringConverter(iterableType, annotations);\n          return new ParameterHandler.Query<>(name, converter, encoded).iterable();\n        } else if (rawParameterType.isArray()) {\n          Class<?> arrayComponentType = boxIfPrimitive(rawParameterType.getComponentType());\n          Converter<?, String> converter =\n              retrofit.stringConverter(arrayComponentType, annotations);\n          return new ParameterHandler.Query<>(name, converter, encoded).array();\n        } else {\n          Converter<?, String> converter = retrofit.stringConverter(type, annotations);\n          return new ParameterHandler.Query<>(name, converter, encoded);\n        }\n\n      } else if (annotation instanceof QueryName) {\n        validateResolvableType(p, type);\n        QueryName query = (QueryName) annotation;\n        boolean encoded = query.encoded();\n\n        Class<?> rawParameterType = Utils.getRawType(type);\n        gotQueryName = true;\n        if (Iterable.class.isAssignableFrom(rawParameterType)) {\n          if (!(type instanceof ParameterizedType)) {\n            throw parameterError(\n                method,\n                p,\n                rawParameterType.getSimpleName()\n                    + \" must include generic type (e.g., \"\n                    + rawParameterType.getSimpleName()\n                    + \"<String>)\");\n          }\n          ParameterizedType parameterizedType = (ParameterizedType) type;\n          Type iterableType = Utils.getParameterUpperBound(0, parameterizedType);\n          Converter<?, String> converter = retrofit.stringConverter(iterableType, annotations);\n          return new ParameterHandler.QueryName<>(converter, encoded).iterable();\n        } else if (rawParameterType.isArray()) {\n          Class<?> arrayComponentType = boxIfPrimitive(rawParameterType.getComponentType());\n          Converter<?, String> converter =\n              retrofit.stringConverter(arrayComponentType, annotations);\n          return new ParameterHandler.QueryName<>(converter, encoded).array();\n        } else {\n          Converter<?, String> converter = retrofit.stringConverter(type, annotations);\n          return new ParameterHandler.QueryName<>(converter, encoded);\n        }\n\n      } else if (annotation instanceof QueryMap) {\n        validateResolvableType(p, type);\n        Class<?> rawParameterType = Utils.getRawType(type);\n        gotQueryMap = true;\n        if (!Map.class.isAssignableFrom(rawParameterType)) {\n          throw parameterError(method, p, \"@QueryMap parameter type must be Map.\");\n        }\n        Type mapType = Utils.getSupertype(type, rawParameterType, Map.class);\n        if (!(mapType instanceof ParameterizedType)) {\n          throw parameterError(\n              method, p, \"Map must include generic types (e.g., Map<String, String>)\");\n        }\n        ParameterizedType parameterizedType = (ParameterizedType) mapType;\n        Type keyType = Utils.getParameterUpperBound(0, parameterizedType);\n        if (String.class != keyType) {\n          throw parameterError(method, p, \"@QueryMap keys must be of type String: \" + keyType);\n        }\n        Type valueType = Utils.getParameterUpperBound(1, parameterizedType);\n        Converter<?, String> valueConverter = retrofit.stringConverter(valueType, annotations);\n\n        return new ParameterHandler.QueryMap<>(\n            method, p, valueConverter, ((QueryMap) annotation).encoded());\n\n      } else if (annotation instanceof Header) {\n        validateResolvableType(p, type);\n        Header header = (Header) annotation;\n        String name = header.value();\n\n        Class<?> rawParameterType = Utils.getRawType(type);\n        if (Iterable.class.isAssignableFrom(rawParameterType)) {\n          if (!(type instanceof ParameterizedType)) {\n            throw parameterError(\n                method,\n                p,\n                rawParameterType.getSimpleName()\n                    + \" must include generic type (e.g., \"\n                    + rawParameterType.getSimpleName()\n                    + \"<String>)\");\n          }\n          ParameterizedType parameterizedType = (ParameterizedType) type;\n          Type iterableType = Utils.getParameterUpperBound(0, parameterizedType);\n          Converter<?, String> converter = retrofit.stringConverter(iterableType, annotations);\n          return new ParameterHandler.Header<>(name, converter, header.allowUnsafeNonAsciiValues())\n              .iterable();\n        } else if (rawParameterType.isArray()) {\n          Class<?> arrayComponentType = boxIfPrimitive(rawParameterType.getComponentType());\n          Converter<?, String> converter =\n              retrofit.stringConverter(arrayComponentType, annotations);\n          return new ParameterHandler.Header<>(name, converter, header.allowUnsafeNonAsciiValues())\n              .array();\n        } else {\n          Converter<?, String> converter = retrofit.stringConverter(type, annotations);\n          return new ParameterHandler.Header<>(name, converter, header.allowUnsafeNonAsciiValues());\n        }\n\n      } else if (annotation instanceof HeaderMap) {\n        if (type == Headers.class) {\n          return new ParameterHandler.Headers(method, p);\n        }\n\n        validateResolvableType(p, type);\n        Class<?> rawParameterType = Utils.getRawType(type);\n        if (!Map.class.isAssignableFrom(rawParameterType)) {\n          throw parameterError(method, p, \"@HeaderMap parameter type must be Map or Headers.\");\n        }\n        Type mapType = Utils.getSupertype(type, rawParameterType, Map.class);\n        if (!(mapType instanceof ParameterizedType)) {\n          throw parameterError(\n              method, p, \"Map must include generic types (e.g., Map<String, String>)\");\n        }\n        ParameterizedType parameterizedType = (ParameterizedType) mapType;\n        Type keyType = Utils.getParameterUpperBound(0, parameterizedType);\n        if (String.class != keyType) {\n          throw parameterError(method, p, \"@HeaderMap keys must be of type String: \" + keyType);\n        }\n        Type valueType = Utils.getParameterUpperBound(1, parameterizedType);\n        Converter<?, String> valueConverter = retrofit.stringConverter(valueType, annotations);\n\n        return new ParameterHandler.HeaderMap<>(\n            method, p, valueConverter, ((HeaderMap) annotation).allowUnsafeNonAsciiValues());\n\n      } else if (annotation instanceof Field) {\n        validateResolvableType(p, type);\n        if (!isFormEncoded) {\n          throw parameterError(method, p, \"@Field parameters can only be used with form encoding.\");\n        }\n        Field field = (Field) annotation;\n        String name = field.value();\n        boolean encoded = field.encoded();\n\n        gotField = true;\n\n        Class<?> rawParameterType = Utils.getRawType(type);\n        if (Iterable.class.isAssignableFrom(rawParameterType)) {\n          if (!(type instanceof ParameterizedType)) {\n            throw parameterError(\n                method,\n                p,\n                rawParameterType.getSimpleName()\n                    + \" must include generic type (e.g., \"\n                    + rawParameterType.getSimpleName()\n                    + \"<String>)\");\n          }\n          ParameterizedType parameterizedType = (ParameterizedType) type;\n          Type iterableType = Utils.getParameterUpperBound(0, parameterizedType);\n          Converter<?, String> converter = retrofit.stringConverter(iterableType, annotations);\n          return new ParameterHandler.Field<>(name, converter, encoded).iterable();\n        } else if (rawParameterType.isArray()) {\n          Class<?> arrayComponentType = boxIfPrimitive(rawParameterType.getComponentType());\n          Converter<?, String> converter =\n              retrofit.stringConverter(arrayComponentType, annotations);\n          return new ParameterHandler.Field<>(name, converter, encoded).array();\n        } else {\n          Converter<?, String> converter = retrofit.stringConverter(type, annotations);\n          return new ParameterHandler.Field<>(name, converter, encoded);\n        }\n\n      } else if (annotation instanceof FieldMap) {\n        validateResolvableType(p, type);\n        if (!isFormEncoded) {\n          throw parameterError(\n              method, p, \"@FieldMap parameters can only be used with form encoding.\");\n        }\n        Class<?> rawParameterType = Utils.getRawType(type);\n        if (!Map.class.isAssignableFrom(rawParameterType)) {\n          throw parameterError(method, p, \"@FieldMap parameter type must be Map.\");\n        }\n        Type mapType = Utils.getSupertype(type, rawParameterType, Map.class);\n        if (!(mapType instanceof ParameterizedType)) {\n          throw parameterError(\n              method, p, \"Map must include generic types (e.g., Map<String, String>)\");\n        }\n        ParameterizedType parameterizedType = (ParameterizedType) mapType;\n        Type keyType = Utils.getParameterUpperBound(0, parameterizedType);\n        if (String.class != keyType) {\n          throw parameterError(method, p, \"@FieldMap keys must be of type String: \" + keyType);\n        }\n        Type valueType = Utils.getParameterUpperBound(1, parameterizedType);\n        Converter<?, String> valueConverter = retrofit.stringConverter(valueType, annotations);\n\n        gotField = true;\n        return new ParameterHandler.FieldMap<>(\n            method, p, valueConverter, ((FieldMap) annotation).encoded());\n\n      } else if (annotation instanceof Part) {\n        validateResolvableType(p, type);\n        if (!isMultipart) {\n          throw parameterError(\n              method, p, \"@Part parameters can only be used with multipart encoding.\");\n        }\n        Part part = (Part) annotation;\n        gotPart = true;\n\n        String partName = part.value();\n        Class<?> rawParameterType = Utils.getRawType(type);\n        if (partName.isEmpty()) {\n          if (Iterable.class.isAssignableFrom(rawParameterType)) {\n            if (!(type instanceof ParameterizedType)) {\n              throw parameterError(\n                  method,\n                  p,\n                  rawParameterType.getSimpleName()\n                      + \" must include generic type (e.g., \"\n                      + rawParameterType.getSimpleName()\n                      + \"<String>)\");\n            }\n            ParameterizedType parameterizedType = (ParameterizedType) type;\n            Type iterableType = Utils.getParameterUpperBound(0, parameterizedType);\n            if (!MultipartBody.Part.class.isAssignableFrom(Utils.getRawType(iterableType))) {\n              throw parameterError(\n                  method,\n                  p,\n                  \"@Part annotation must supply a name or use MultipartBody.Part parameter type.\");\n            }\n            return ParameterHandler.RawPart.INSTANCE.iterable();\n          } else if (rawParameterType.isArray()) {\n            Class<?> arrayComponentType = rawParameterType.getComponentType();\n            if (!MultipartBody.Part.class.isAssignableFrom(arrayComponentType)) {\n              throw parameterError(\n                  method,\n                  p,\n                  \"@Part annotation must supply a name or use MultipartBody.Part parameter type.\");\n            }\n            return ParameterHandler.RawPart.INSTANCE.array();\n          } else if (MultipartBody.Part.class.isAssignableFrom(rawParameterType)) {\n            return ParameterHandler.RawPart.INSTANCE;\n          } else {\n            throw parameterError(\n                method,\n                p,\n                \"@Part annotation must supply a name or use MultipartBody.Part parameter type.\");\n          }\n        } else {\n          Headers headers =\n              Headers.of(\n                  \"Content-Disposition\",\n                  \"form-data; name=\\\"\" + partName + \"\\\"\",\n                  \"Content-Transfer-Encoding\",\n                  part.encoding());\n\n          if (Iterable.class.isAssignableFrom(rawParameterType)) {\n            if (!(type instanceof ParameterizedType)) {\n              throw parameterError(\n                  method,\n                  p,\n                  rawParameterType.getSimpleName()\n                      + \" must include generic type (e.g., \"\n                      + rawParameterType.getSimpleName()\n                      + \"<String>)\");\n            }\n            ParameterizedType parameterizedType = (ParameterizedType) type;\n            Type iterableType = Utils.getParameterUpperBound(0, parameterizedType);\n            if (MultipartBody.Part.class.isAssignableFrom(Utils.getRawType(iterableType))) {\n              throw parameterError(\n                  method,\n                  p,\n                  \"@Part parameters using the MultipartBody.Part must not \"\n                      + \"include a part name in the annotation.\");\n            }\n            Converter<?, RequestBody> converter =\n                retrofit.requestBodyConverter(iterableType, annotations, methodAnnotations);\n            return new ParameterHandler.Part<>(method, p, headers, converter).iterable();\n          } else if (rawParameterType.isArray()) {\n            Class<?> arrayComponentType = boxIfPrimitive(rawParameterType.getComponentType());\n            if (MultipartBody.Part.class.isAssignableFrom(arrayComponentType)) {\n              throw parameterError(\n                  method,\n                  p,\n                  \"@Part parameters using the MultipartBody.Part must not \"\n                      + \"include a part name in the annotation.\");\n            }\n            Converter<?, RequestBody> converter =\n                retrofit.requestBodyConverter(arrayComponentType, annotations, methodAnnotations);\n            return new ParameterHandler.Part<>(method, p, headers, converter).array();\n          } else if (MultipartBody.Part.class.isAssignableFrom(rawParameterType)) {\n            throw parameterError(\n                method,\n                p,\n                \"@Part parameters using the MultipartBody.Part must not \"\n                    + \"include a part name in the annotation.\");\n          } else {\n            Converter<?, RequestBody> converter =\n                retrofit.requestBodyConverter(type, annotations, methodAnnotations);\n            return new ParameterHandler.Part<>(method, p, headers, converter);\n          }\n        }\n\n      } else if (annotation instanceof PartMap) {\n        validateResolvableType(p, type);\n        if (!isMultipart) {\n          throw parameterError(\n              method, p, \"@PartMap parameters can only be used with multipart encoding.\");\n        }\n        gotPart = true;\n        Class<?> rawParameterType = Utils.getRawType(type);\n        if (!Map.class.isAssignableFrom(rawParameterType)) {\n          throw parameterError(method, p, \"@PartMap parameter type must be Map.\");\n        }\n        Type mapType = Utils.getSupertype(type, rawParameterType, Map.class);\n        if (!(mapType instanceof ParameterizedType)) {\n          throw parameterError(\n              method, p, \"Map must include generic types (e.g., Map<String, String>)\");\n        }\n        ParameterizedType parameterizedType = (ParameterizedType) mapType;\n\n        Type keyType = Utils.getParameterUpperBound(0, parameterizedType);\n        if (String.class != keyType) {\n          throw parameterError(method, p, \"@PartMap keys must be of type String: \" + keyType);\n        }\n\n        Type valueType = Utils.getParameterUpperBound(1, parameterizedType);\n        if (MultipartBody.Part.class.isAssignableFrom(Utils.getRawType(valueType))) {\n          throw parameterError(\n              method,\n              p,\n              \"@PartMap values cannot be MultipartBody.Part. \"\n                  + \"Use @Part List<Part> or a different value type instead.\");\n        }\n\n        Converter<?, RequestBody> valueConverter =\n            retrofit.requestBodyConverter(valueType, annotations, methodAnnotations);\n\n        PartMap partMap = (PartMap) annotation;\n        return new ParameterHandler.PartMap<>(method, p, valueConverter, partMap.encoding());\n\n      } else if (annotation instanceof Body) {\n        validateResolvableType(p, type);\n        if (isFormEncoded || isMultipart) {\n          throw parameterError(\n              method, p, \"@Body parameters cannot be used with form or multi-part encoding.\");\n        }\n        if (gotBody) {\n          throw parameterError(method, p, \"Multiple @Body method annotations found.\");\n        }\n\n        Converter<?, RequestBody> converter;\n        try {\n          converter = retrofit.requestBodyConverter(type, annotations, methodAnnotations);\n        } catch (RuntimeException e) {\n          // Wide exception range because factories are user code.\n          throw parameterError(method, e, p, \"Unable to create @Body converter for %s\", type);\n        }\n        gotBody = true;\n        return new ParameterHandler.Body<>(method, p, converter);\n\n      } else if (annotation instanceof Tag) {\n        validateResolvableType(p, type);\n\n        Class<?> tagType = Utils.getRawType(type);\n        for (int i = p - 1; i >= 0; i--) {\n          ParameterHandler<?> otherHandler = parameterHandlers[i];\n          if (otherHandler instanceof ParameterHandler.Tag\n              && ((ParameterHandler.Tag) otherHandler).cls.equals(tagType)) {\n            throw parameterError(\n                method,\n                p,\n                \"@Tag type \"\n                    + tagType.getName()\n                    + \" is duplicate of \"\n                    + Platform.reflection.describeMethodParameter(method, i)\n                    + \" and would always overwrite its value.\");\n          }\n        }\n\n        return new ParameterHandler.Tag<>(tagType);\n      }\n\n      return null; // Not a Retrofit annotation.\n    }\n\n    private void validateResolvableType(int p, Type type) {\n      if (Utils.hasUnresolvableType(type)) {\n        throw parameterError(\n            method, p, \"Parameter type must not include a type variable or wildcard: %s\", type);\n      }\n    }\n\n    private void validatePathName(int p, String name) {\n      if (!PARAM_NAME_REGEX.matcher(name).matches()) {\n        throw parameterError(\n            method,\n            p,\n            \"@Path parameter name must match %s. Found: %s\",\n            PARAM_URL_REGEX.pattern(),\n            name);\n      }\n      // Verify URL replacement name is actually present in the URL path.\n      if (!relativeUrlParamNames.contains(name)) {\n        throw parameterError(method, p, \"URL \\\"%s\\\" does not contain \\\"{%s}\\\".\", relativeUrl, name);\n      }\n    }\n\n    /**\n     * Gets the set of unique path parameters used in the given URI. If a parameter is used twice in\n     * the URI, it will only show up once in the set.\n     */\n    static Set<String> parsePathParameters(String path) {\n      Matcher m = PARAM_URL_REGEX.matcher(path);\n      Set<String> patterns = new LinkedHashSet<>();\n      while (m.find()) {\n        patterns.add(m.group(1));\n      }\n      return patterns;\n    }\n\n    private static Class<?> boxIfPrimitive(Class<?> type) {\n      if (boolean.class == type) return Boolean.class;\n      if (byte.class == type) return Byte.class;\n      if (char.class == type) return Character.class;\n      if (double.class == type) return Double.class;\n      if (float.class == type) return Float.class;\n      if (int.class == type) return Integer.class;\n      if (long.class == type) return Long.class;\n      if (short.class == type) return Short.class;\n      return type;\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/ServiceMethod.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.methodError;\n\nimport java.lang.reflect.Method;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\n\nabstract class ServiceMethod<T> {\n  static <T> ServiceMethod<T> parseAnnotations(Retrofit retrofit, Class<?> service, Method method) {\n    RequestFactory requestFactory = RequestFactory.parseAnnotations(retrofit, service, method);\n\n    Type returnType = method.getGenericReturnType();\n    if (Utils.hasUnresolvableType(returnType)) {\n      throw methodError(\n          method,\n          \"Method return type must not include a type variable or wildcard: %s\",\n          returnType);\n    }\n    if (returnType == void.class) {\n      throw methodError(method, \"Service methods cannot return void.\");\n    }\n\n    return HttpServiceMethod.parseAnnotations(retrofit, method, requestFactory);\n  }\n\n  abstract @Nullable T invoke(Object instance, Object[] args);\n}\n\n// Path: retrofit/src/main/java/retrofit2/Retrofit.java\n/*\n * Copyright (C) 2012 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static java.util.Collections.unmodifiableList;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.InvocationHandler;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.Modifier;\nimport java.lang.reflect.Proxy;\nimport java.lang.reflect.Type;\nimport java.net.URL;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Deque;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\nimport okhttp3.HttpUrl;\nimport okhttp3.OkHttpClient;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.GET;\nimport retrofit2.http.HTTP;\nimport retrofit2.http.Header;\nimport retrofit2.http.Url;\n\n/**\n * Retrofit adapts a Java interface to HTTP calls by using annotations on the declared methods to\n * define how requests are made. Create instances using {@linkplain Builder the builder} and pass\n * your interface to {@link #create} to generate an implementation.\n *\n * <p>For example,\n *\n * <pre><code>\n * Retrofit retrofit = new Retrofit.Builder()\n *     .baseUrl(\"https://api.example.com/\")\n *     .addConverterFactory(GsonConverterFactory.create())\n *     .build();\n *\n * MyApi api = retrofit.create(MyApi.class);\n * Response&lt;User&gt; user = api.getUser().execute();\n * </code></pre>\n *\n * @author Bob Lee (bob@squareup.com)\n * @author Jake Wharton (jw@squareup.com)\n */\npublic final class Retrofit {\n  /**\n   * Method associations in this map will be in one of three states, and may only progress forward\n   * to higher-numbered states.\n   * <ol>\n   *   <li>No value - no one has started or completed parsing annotations for the method.</li>\n   *   <li>Lock object - a thread has started parsing annotations on the method. Once the lock is\n   *   available the map will have been updated with the parsed model.</li>\n   *   <li>{@code ServiceMethod} - annotations for the method have been fully parsed.</li>\n   * </ol>\n   * This map should only be accessed through {@link #loadServiceMethod} which contains the state\n   * transition logic.\n   */\n  private final ConcurrentHashMap<Method, Object> serviceMethodCache = new ConcurrentHashMap<>();\n\n  final okhttp3.Call.Factory callFactory;\n  final HttpUrl baseUrl;\n  final List<Converter.Factory> converterFactories;\n  final int defaultConverterFactoriesSize;\n  final List<CallAdapter.Factory> callAdapterFactories;\n  final int defaultCallAdapterFactoriesSize;\n  final @Nullable Executor callbackExecutor;\n  final boolean validateEagerly;\n\n  Retrofit(\n      okhttp3.Call.Factory callFactory,\n      HttpUrl baseUrl,\n      List<Converter.Factory> converterFactories,\n      int defaultConverterFactoriesSize,\n      List<CallAdapter.Factory> callAdapterFactories,\n      int defaultCallAdapterFactoriesSize,\n      @Nullable Executor callbackExecutor,\n      boolean validateEagerly) {\n    this.callFactory = callFactory;\n    this.baseUrl = baseUrl;\n    this.converterFactories = converterFactories; // Copy+unmodifiable at call site.\n    this.defaultConverterFactoriesSize = defaultConverterFactoriesSize;\n    this.callAdapterFactories = callAdapterFactories; // Copy+unmodifiable at call site.\n    this.defaultCallAdapterFactoriesSize = defaultCallAdapterFactoriesSize;\n    this.callbackExecutor = callbackExecutor;\n    this.validateEagerly = validateEagerly;\n  }\n\n  /**\n   * Create an implementation of the API endpoints defined by the {@code service} interface.\n   *\n   * <p>The relative path for a given method is obtained from an annotation on the method describing\n   * the request type. The built-in methods are {@link retrofit2.http.GET GET}, {@link\n   * retrofit2.http.PUT PUT}, {@link retrofit2.http.POST POST}, {@link retrofit2.http.PATCH PATCH},\n   * {@link retrofit2.http.HEAD HEAD}, {@link retrofit2.http.DELETE DELETE} and {@link\n   * retrofit2.http.OPTIONS OPTIONS}. You can use a custom HTTP method with {@link HTTP @HTTP}. For\n   * a dynamic URL, omit the path on the annotation and annotate the first parameter with {@link\n   * Url @Url}.\n   *\n   * <p>Method parameters can be used to replace parts of the URL by annotating them with {@link\n   * retrofit2.http.Path @Path}. Replacement sections are denoted by an identifier surrounded by\n   * curly braces (e.g., \"{foo}\"). To add items to the query string of a URL use {@link\n   * retrofit2.http.Query @Query}.\n   *\n   * <p>The body of a request is denoted by the {@link retrofit2.http.Body @Body} annotation. The\n   * object will be converted to request representation by one of the {@link Converter.Factory}\n   * instances. A {@link RequestBody} can also be used for a raw representation.\n   *\n   * <p>Alternative request body formats are supported by method annotations and corresponding\n   * parameter annotations:\n   *\n   * <ul>\n   *   <li>{@link retrofit2.http.FormUrlEncoded @FormUrlEncoded} - Form-encoded data with key-value\n   *       pairs specified by the {@link retrofit2.http.Field @Field} parameter annotation.\n   *   <li>{@link retrofit2.http.Multipart @Multipart} - RFC 2388-compliant multipart data with\n   *       parts specified by the {@link retrofit2.http.Part @Part} parameter annotation.\n   * </ul>\n   *\n   * <p>Additional static headers can be added for an endpoint using the {@link\n   * retrofit2.http.Headers @Headers} method annotation. For per-request control over a header\n   * annotate a parameter with {@link Header @Header}.\n   *\n   * <p>By default, methods return a {@link Call} which represents the HTTP request. The generic\n   * parameter of the call is the response body type and will be converted by one of the {@link\n   * Converter.Factory} instances. {@link ResponseBody} can also be used for a raw representation.\n   * {@link Void} can be used if you do not care about the body contents.\n   *\n   * <p>For example:\n   *\n   * <pre>\n   * public interface CategoryService {\n   *   &#64;POST(\"category/{cat}/\")\n   *   Call&lt;List&lt;Item&gt;&gt; categoryList(@Path(\"cat\") String a, @Query(\"page\") int b);\n   * }\n   * </pre>\n   */\n  @SuppressWarnings(\"unchecked\") // Single-interface proxy creation guarded by parameter safety.\n  public <T> T create(final Class<T> service) {\n    validateServiceInterface(service);\n    return (T)\n        Proxy.newProxyInstance(\n            service.getClassLoader(),\n            new Class<?>[] {service},\n            new InvocationHandler() {\n              private final Object[] emptyArgs = new Object[0];\n\n              @Override\n              public @Nullable Object invoke(Object proxy, Method method, @Nullable Object[] args)\n                  throws Throwable {\n                // If the method is a method from Object then defer to normal invocation.\n                if (method.getDeclaringClass() == Object.class) {\n                  return method.invoke(this, args);\n                }\n                args = args != null ? args : emptyArgs;\n                Reflection reflection = Platform.reflection;\n                return reflection.isDefaultMethod(method)\n                    ? reflection.invokeDefaultMethod(method, service, proxy, args)\n                    : loadServiceMethod(service, method).invoke(proxy, args);\n              }\n            });\n  }\n\n  private void validateServiceInterface(Class<?> service) {\n    if (!service.isInterface()) {\n      throw new IllegalArgumentException(\"API declarations must be interfaces.\");\n    }\n\n    Deque<Class<?>> check = new ArrayDeque<>(1);\n    check.add(service);\n    while (!check.isEmpty()) {\n      Class<?> candidate = check.removeFirst();\n      if (candidate.getTypeParameters().length != 0) {\n        StringBuilder message =\n            new StringBuilder(\"Type parameters are unsupported on \").append(candidate.getName());\n        if (candidate != service) {\n          message.append(\" which is an interface of \").append(service.getName());\n        }\n        throw new IllegalArgumentException(message.toString());\n      }\n      Collections.addAll(check, candidate.getInterfaces());\n    }\n\n    if (validateEagerly) {\n      Reflection reflection = Platform.reflection;\n      for (Method method : service.getDeclaredMethods()) {\n        if (!reflection.isDefaultMethod(method)\n            && !Modifier.isStatic(method.getModifiers())\n            && !method.isSynthetic()) {\n          loadServiceMethod(service, method);\n        }\n      }\n    }\n  }\n\n  ServiceMethod<?> loadServiceMethod(Class<?> service, Method method) {\n    // Note: Once we are minSdk 24 this whole method can be replaced by computeIfAbsent.\n    Object lookup = serviceMethodCache.get(method);\n\n    if (lookup instanceof ServiceMethod<?>) {\n      // Happy path: method is already parsed into the model.\n      return (ServiceMethod<?>) lookup;\n    }\n\n    if (lookup == null) {\n      // Map does not contain any value. Try to put in a lock for this method. We MUST synchronize\n      // on the lock before it is visible to others via the map to signal we are doing the work.\n      Object lock = new Object();\n      synchronized (lock) {\n        lookup = serviceMethodCache.putIfAbsent(method, lock);\n        if (lookup == null) {\n          // On successful lock insertion, perform the work and update the map before releasing.\n          // Other threads may be waiting on lock now and will expect the parsed model.\n          ServiceMethod<Object> result = ServiceMethod.parseAnnotations(this, service, method);\n          serviceMethodCache.put(method, result);\n          return result;\n        }\n      }\n    }\n\n    // Either the initial lookup or the attempt to put our lock in the map has returned someone\n    // else's lock. This means they are doing the parsing, and will update the map before releasing\n    // the lock. Once we can take the lock, the map is guaranteed to contain the model.\n    // Note: There's a chance that our effort to put a lock into the map has actually returned a\n    // finished model instead of a lock. In that case this code will perform a pointless lock and\n    // redundant lookup in the map of the same instance. This is rare, and ultimately harmless.\n    synchronized (lookup) {\n      return (ServiceMethod<?>) serviceMethodCache.get(method);\n    }\n  }\n\n  /**\n   * The factory used to create {@linkplain okhttp3.Call OkHttp calls} for sending a HTTP requests.\n   * Typically an instance of {@link OkHttpClient}.\n   */\n  public okhttp3.Call.Factory callFactory() {\n    return callFactory;\n  }\n\n  /** The API base URL. */\n  public HttpUrl baseUrl() {\n    return baseUrl;\n  }\n\n  /**\n   * Returns a list of the factories tried when creating a {@linkplain #callAdapter(Type,\n   * Annotation[])} call adapter}.\n   */\n  public List<CallAdapter.Factory> callAdapterFactories() {\n    return callAdapterFactories;\n  }\n\n  /**\n   * Returns the {@link CallAdapter} for {@code returnType} from the available {@linkplain\n   * #callAdapterFactories() factories}.\n   *\n   * @throws IllegalArgumentException if no call adapter available for {@code type}.\n   */\n  \npublic CallAdapter<?, ?> callAdapter(Type returnType, Annotation[] annotations) {\n    return nextCallAdapter(null, returnType, annotations);\n  }\n\n  /**\n   * Returns the {@link CallAdapter} for {@code returnType} from the available {@linkplain\n   * #callAdapterFactories() factories} except {@code skipPast}.\n   *\n   * @throws IllegalArgumentException if no call adapter available for {@code type}.\n   */\n  public CallAdapter<?, ?> nextCallAdapter(\n      @Nullable CallAdapter.Factory skipPast, Type returnType, Annotation[] annotations) {\n    Objects.requireNonNull(returnType, \"returnType == null\");\n    Objects.requireNonNull(annotations, \"annotations == null\");\n\n    int start = callAdapterFactories.indexOf(skipPast) + 1;\n    for (int i = start, count = callAdapterFactories.size(); i < count; i++) {\n      CallAdapter<?, ?> adapter = callAdapterFactories.get(i).get(returnType, annotations, this);\n      if (adapter != null) {\n        return adapter;\n      }\n    }\n\n    StringBuilder builder =\n        new StringBuilder(\"Could not locate call adapter for \").append(returnType).append(\".\\n\");\n    if (skipPast != null) {\n      builder.append(\"  Skipped:\");\n      for (int i = 0; i < start; i++) {\n        builder.append(\"\\n   * \").append(callAdapterFactories.get(i).getClass().getName());\n      }\n      builder.append('\\n');\n    }\n    builder.append(\"  Tried:\");\n    for (int i = start, count = callAdapterFactories.size(); i < count; i++) {\n      builder.append(\"\\n   * \").append(callAdapterFactories.get(i).getClass().getName());\n    }\n    throw new IllegalArgumentException(builder.toString());\n  }\n\n  /**\n   * Returns an unmodifiable list of the factories tried when creating a {@linkplain\n   * #requestBodyConverter(Type, Annotation[], Annotation[]) request body converter}, a {@linkplain\n   * #responseBodyConverter(Type, Annotation[]) response body converter}, or a {@linkplain\n   * #stringConverter(Type, Annotation[]) string converter}.\n   */\n  public List<Converter.Factory> converterFactories() {\n    return converterFactories;\n  }\n\n  /**\n   * Returns a {@link Converter} for {@code type} to {@link RequestBody} from the available\n   * {@linkplain #converterFactories() factories}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<T, RequestBody> requestBodyConverter(\n      Type type, Annotation[] parameterAnnotations, Annotation[] methodAnnotations) {\n    return nextRequestBodyConverter(null, type, parameterAnnotations, methodAnnotations);\n  }\n\n  /**\n   * Returns a {@link Converter} for {@code type} to {@link RequestBody} from the available\n   * {@linkplain #converterFactories() factories} except {@code skipPast}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<T, RequestBody> nextRequestBodyConverter(\n      @Nullable Converter.Factory skipPast,\n      Type type,\n      Annotation[] parameterAnnotations,\n      Annotation[] methodAnnotations) {\n    Objects.requireNonNull(type, \"type == null\");\n    Objects.requireNonNull(parameterAnnotations, \"parameterAnnotations == null\");\n    Objects.requireNonNull(methodAnnotations, \"methodAnnotations == null\");\n\n    int start = converterFactories.indexOf(skipPast) + 1;\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      Converter.Factory factory = converterFactories.get(i);\n      Converter<?, RequestBody> converter =\n          factory.requestBodyConverter(type, parameterAnnotations, methodAnnotations, this);\n      if (converter != null) {\n        //noinspection unchecked\n        return (Converter<T, RequestBody>) converter;\n      }\n    }\n\n    StringBuilder builder =\n        new StringBuilder(\"Could not locate RequestBody converter for \").append(type).append(\".\\n\");\n    if (skipPast != null) {\n      builder.append(\"  Skipped:\");\n      for (int i = 0; i < start; i++) {\n        builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n      }\n      builder.append('\\n');\n    }\n    builder.append(\"  Tried:\");\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n    }\n    throw new IllegalArgumentException(builder.toString());\n  }\n\n  /**\n   * Returns a {@link Converter} for {@link ResponseBody} to {@code type} from the available\n   * {@linkplain #converterFactories() factories}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<ResponseBody, T> responseBodyConverter(Type type, Annotation[] annotations) {\n    return nextResponseBodyConverter(null, type, annotations);\n  }\n\n  /**\n   * Returns a {@link Converter} for {@link ResponseBody} to {@code type} from the available\n   * {@linkplain #converterFactories() factories} except {@code skipPast}.\n   *\n   * @throws IllegalArgumentException if no converter available for {@code type}.\n   */\n  public <T> Converter<ResponseBody, T> nextResponseBodyConverter(\n      @Nullable Converter.Factory skipPast, Type type, Annotation[] annotations) {\n    Objects.requireNonNull(type, \"type == null\");\n    Objects.requireNonNull(annotations, \"annotations == null\");\n\n    int start = converterFactories.indexOf(skipPast) + 1;\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      Converter<ResponseBody, ?> converter =\n          converterFactories.get(i).responseBodyConverter(type, annotations, this);\n      if (converter != null) {\n        //noinspection unchecked\n        return (Converter<ResponseBody, T>) converter;\n      }\n    }\n\n    StringBuilder builder =\n        new StringBuilder(\"Could not locate ResponseBody converter for \")\n            .append(type)\n            .append(\".\\n\");\n    if (skipPast != null) {\n      builder.append(\"  Skipped:\");\n      for (int i = 0; i < start; i++) {\n        builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n      }\n      builder.append('\\n');\n    }\n    builder.append(\"  Tried:\");\n    for (int i = start, count = converterFactories.size(); i < count; i++) {\n      builder.append(\"\\n   * \").append(converterFactories.get(i).getClass().getName());\n    }\n    throw new IllegalArgumentException(builder.toString());\n  }\n\n  /**\n   * Returns a {@link Converter} for {@code type} to {@link String} from the available {@linkplain\n   * #converterFactories() factories}.\n   */\n  public <T> Converter<T, String> stringConverter(Type type, Annotation[] annotations) {\n    Objects.requireNonNull(type, \"type == null\");\n    Objects.requireNonNull(annotations, \"annotations == null\");\n\n    for (int i = 0, count = converterFactories.size(); i < count; i++) {\n      Converter<?, String> converter =\n          converterFactories.get(i).stringConverter(type, annotations, this);\n      if (converter != null) {\n        //noinspection unchecked\n        return (Converter<T, String>) converter;\n      }\n    }\n\n    // Nothing matched. Resort to default converter which just calls toString().\n    //noinspection unchecked\n    return (Converter<T, String>) BuiltInConverters.ToStringConverter.INSTANCE;\n  }\n\n  /**\n   * The executor used for {@link Callback} methods on a {@link Call}. This may be {@code null}, in\n   * which case callbacks should be made synchronously on the background thread.\n   */\n  public @Nullable Executor callbackExecutor() {\n    return callbackExecutor;\n  }\n\n  public Builder newBuilder() {\n    return new Builder(this);\n  }\n\n  /**\n   * Build a new {@link Retrofit}.\n   *\n   * <p>Calling {@link #baseUrl} is required before calling {@link #build()}. All other methods are\n   * optional.\n   */\n  public static final class Builder {\n    private @Nullable okhttp3.Call.Factory callFactory;\n    private @Nullable HttpUrl baseUrl;\n    private final List<Converter.Factory> converterFactories = new ArrayList<>();\n    private final List<CallAdapter.Factory> callAdapterFactories = new ArrayList<>();\n    private @Nullable Executor callbackExecutor;\n    private boolean validateEagerly;\n\n    public Builder() {}\n\n    Builder(Retrofit retrofit) {\n      callFactory = retrofit.callFactory;\n      baseUrl = retrofit.baseUrl;\n\n      // Do not add the default BuiltIntConverters and platform-aware converters added by build().\n      for (int i = 1,\n              size = retrofit.converterFactories.size() - retrofit.defaultConverterFactoriesSize;\n          i < size;\n          i++) {\n        converterFactories.add(retrofit.converterFactories.get(i));\n      }\n\n      // Do not add the default, platform-aware call adapters added by build().\n      for (int i = 0,\n              size =\n...\n", "needle_token_start": 13697, "needle_token_end": 13733, "code_context_ntokens": 16149, "output": ["```java\n/**\n * Returns a {@link CallAdapter} for {@code returnType} from the available {@linkplain\n * #callAdapterFactories() factories}.\n *\n * @throws IllegalArgumentException if no call adapter available for {@code type}.\n */\npublic CallAdapter<?, ?> callAdapter(Type returnType, Annotation[] annotations) {\n    return nextCallAdapter(null, returnType, annotations);\n}\n\n/**\n * Returns a {@link CallAdapter} for {@code returnType} from the available {@linkplain\n * #callAdapterFactories() factories} except {@code skipPast}.\n *\n * @throws IllegalArgumentException if no call adapter available for {@code type}.\n */\npublic CallAdapter<?, ?> nextCallAdapter(\n    @Nullable CallAdapter.Factory skipPast, Type returnType, Annotation[] annotations) {\n  Objects.requireNonNull(returnType, \"returnType == null\");\n  Objects.requireNonNull(annotations, \"annotations == null\");\n\n  int start = callAdapterFactories.indexOf(skipPast) + 1;\n  for (int i = start, count = callAdapterFactories.size(); i < count; i++) {\n    CallAdapter<?, ?> adapter = callAdapterFactories.get(i).get(returnType, annotations, this);\n    if (adapter != null) {\n      return adapter;\n    }\n  }\n\n  StringBuilder builder =\n      new StringBuilder(\"Could not locate call adapter for \").append(returnType).append(\".\\n\");\n  if (skipPast != null) {\n    builder.append(\"  Skipped:\");\n    for (int i = 0; i < start; i++) {\n      builder.append(\"\\n   * \").append(callAdapterFactories.get(i).getClass().getName());\n    }\n    builder.append('\\n');\n  }\n  builder.append(\"  Tried:\");\n  for (int i = start, count = callAdapterFactories.size(); i < count; i++) {\n    builder.append(\"\\n   * \").append(callAdapterFactories.get(i).getClass().getName());\n  }\n  throw new IllegalArgumentException(builder.toString());\n}\n```"]}
{"repo": "square/retrofit", "name": "annotationType", "language": "java", "path": "retrofit/src/main/java/retrofit2/SkipCallbackExecutorImpl.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this method is to provide the specific type of annotation associated with an instance of an annotation interface.\n2. **Input**: This method does not take any parameters.\n3. **Output**: The method returns a `Class` object that represents the annotation type of the instance.\n4. **Procedure**: The method simply returns the class object of the annotation interface it implements.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: retrofit/src/main/java/retrofit2/Platform.java\n/*\n * Copyright (C) 2013 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static android.os.Build.VERSION.SDK_INT;\n\nimport java.util.concurrent.Executor;\nimport javax.annotation.Nullable;\n\nfinal class Platform {\n  static final @Nullable Executor callbackExecutor;\n  static final Reflection reflection;\n  static final BuiltInFactories builtInFactories;\n\n  static {\n    switch (System.getProperty(\"java.vm.name\")) {\n      case \"Dalvik\":\n        callbackExecutor = new AndroidMainExecutor();\n        if (SDK_INT >= 24) {\n          reflection = new Reflection.Android24();\n          builtInFactories = new BuiltInFactories.Java8();\n        } else {\n          reflection = new Reflection();\n          builtInFactories = new BuiltInFactories();\n        }\n        break;\n\n      case \"RoboVM\":\n        callbackExecutor = null;\n        reflection = new Reflection();\n        builtInFactories = new BuiltInFactories();\n        break;\n\n      default:\n        callbackExecutor = null;\n        reflection = new Reflection.Java8();\n        builtInFactories = new BuiltInFactories.Java8();\n        break;\n    }\n  }\n\n  private Platform() {}\n}\n\n// Path: retrofit/src/main/java/retrofit2/Response.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport okhttp3.Headers;\nimport okhttp3.Protocol;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\n\n/** An HTTP response. */\npublic final class Response<T> {\n  /** Create a synthetic successful response with {@code body} as the deserialized body. */\n  public static <T> Response<T> success(@Nullable T body) {\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response with an HTTP status code of {@code code} and {@code\n   * body} as the deserialized body.\n   */\n  public static <T> Response<T> success(int code, @Nullable T body) {\n    if (code < 200 || code >= 300) {\n      throw new IllegalArgumentException(\"code < 200 or >= 300: \" + code);\n    }\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(code)\n            .message(\"Response.success()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a synthetic successful response using {@code headers} with {@code body} as the\n   * deserialized body.\n   */\n  public static <T> Response<T> success(@Nullable T body, Headers headers) {\n    Objects.requireNonNull(headers, \"headers == null\");\n    return success(\n        body,\n        new okhttp3.Response.Builder() //\n            .code(200)\n            .message(\"OK\")\n            .protocol(Protocol.HTTP_1_1)\n            .headers(headers)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /**\n   * Create a successful response from {@code rawResponse} with {@code body} as the deserialized\n   * body.\n   */\n  public static <T> Response<T> success(@Nullable T body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (!rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse must be successful response\");\n    }\n    return new Response<>(rawResponse, body, null);\n  }\n\n  /**\n   * Create a synthetic error response with an HTTP status code of {@code code} and {@code body} as\n   * the error body.\n   */\n  public static <T> Response<T> error(int code, ResponseBody body) {\n    Objects.requireNonNull(body, \"body == null\");\n    if (code < 400) throw new IllegalArgumentException(\"code < 400: \" + code);\n    return error(\n        body,\n        new okhttp3.Response.Builder() //\n            .body(new OkHttpCall.NoContentResponseBody(body.contentType(), body.contentLength()))\n            .code(code)\n            .message(\"Response.error()\")\n            .protocol(Protocol.HTTP_1_1)\n            .request(new Request.Builder().url(\"http://localhost/\").build())\n            .build());\n  }\n\n  /** Create an error response from {@code rawResponse} with {@code body} as the error body. */\n  public static <T> Response<T> error(ResponseBody body, okhttp3.Response rawResponse) {\n    Objects.requireNonNull(body, \"body == null\");\n    Objects.requireNonNull(rawResponse, \"rawResponse == null\");\n    if (rawResponse.isSuccessful()) {\n      throw new IllegalArgumentException(\"rawResponse should not be successful response\");\n    }\n    return new Response<>(rawResponse, null, body);\n  }\n\n  private final okhttp3.Response rawResponse;\n  private final @Nullable T body;\n  private final @Nullable ResponseBody errorBody;\n\n  private Response(\n      okhttp3.Response rawResponse, @Nullable T body, @Nullable ResponseBody errorBody) {\n    this.rawResponse = rawResponse;\n    this.body = body;\n    this.errorBody = errorBody;\n  }\n\n  /** The raw response from the HTTP client. */\n  public okhttp3.Response raw() {\n    return rawResponse;\n  }\n\n  /** HTTP status code. */\n  public int code() {\n    return rawResponse.code();\n  }\n\n  /** HTTP status message or null if unknown. */\n  public String message() {\n    return rawResponse.message();\n  }\n\n  /** HTTP headers. */\n  public Headers headers() {\n    return rawResponse.headers();\n  }\n\n  /** Returns true if {@link #code()} is in the range [200..300). */\n  public boolean isSuccessful() {\n    return rawResponse.isSuccessful();\n  }\n\n  /** The deserialized response body of a {@linkplain #isSuccessful() successful} response. */\n  public @Nullable T body() {\n    return body;\n  }\n\n  /** The raw response body of an {@linkplain #isSuccessful() unsuccessful} response. */\n  public @Nullable ResponseBody errorBody() {\n    return errorBody;\n  }\n\n  @Override\n  public String toString() {\n    return rawResponse.toString();\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/Utils.java\n/*\n * Copyright (C) 2008 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Array;\nimport java.lang.reflect.GenericArrayType;\nimport java.lang.reflect.GenericDeclaration;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport java.lang.reflect.TypeVariable;\nimport java.lang.reflect.WildcardType;\nimport java.util.Arrays;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\n\nfinal class Utils {\n  static final Type[] EMPTY_TYPE_ARRAY = new Type[0];\n\n  private Utils() {\n    // No instances.\n  }\n\n  static RuntimeException methodError(Method method, String message, Object... args) {\n    return methodError(method, null, message, args);\n  }\n\n  @SuppressWarnings(\"AnnotateFormatMethod\")\n  static RuntimeException methodError(\n      Method method, @Nullable Throwable cause, String message, Object... args) {\n    message = String.format(message, args);\n    return new IllegalArgumentException(\n        message\n            + \"\\n    for method \"\n            + method.getDeclaringClass().getSimpleName()\n            + \".\"\n            + method.getName(),\n        cause);\n  }\n\n  static RuntimeException parameterError(\n      Method method, Throwable cause, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, cause, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static RuntimeException parameterError(Method method, int p, String message, Object... args) {\n    String paramDesc = Platform.reflection.describeMethodParameter(method, p);\n    return methodError(method, message + \" (\" + paramDesc + \")\", args);\n  }\n\n  static Class<?> getRawType(Type type) {\n    Objects.requireNonNull(type, \"type == null\");\n\n    if (type instanceof Class<?>) {\n      // Type is a normal class.\n      return (Class<?>) type;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n\n      // I'm not exactly sure why getRawType() returns Type instead of Class. Neal isn't either but\n      // suspects some pathological case related to nested classes exists.\n      Type rawType = parameterizedType.getRawType();\n      if (!(rawType instanceof Class)) throw new IllegalArgumentException();\n      return (Class<?>) rawType;\n    }\n    if (type instanceof GenericArrayType) {\n      Type componentType = ((GenericArrayType) type).getGenericComponentType();\n      return Array.newInstance(getRawType(componentType), 0).getClass();\n    }\n    if (type instanceof TypeVariable) {\n      // We could use the variable's bounds, but that won't work if there are multiple. Having a raw\n      // type that's more general than necessary is okay.\n      return Object.class;\n    }\n    if (type instanceof WildcardType) {\n      return getRawType(((WildcardType) type).getUpperBounds()[0]);\n    }\n\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + type.getClass().getName());\n  }\n\n  /** Returns true if {@code a} and {@code b} are equal. */\n  static boolean equals(Type a, Type b) {\n    if (a == b) {\n      return true; // Also handles (a == null && b == null).\n\n    } else if (a instanceof Class) {\n      return a.equals(b); // Class already specifies equals().\n\n    } else if (a instanceof ParameterizedType) {\n      if (!(b instanceof ParameterizedType)) return false;\n      ParameterizedType pa = (ParameterizedType) a;\n      ParameterizedType pb = (ParameterizedType) b;\n      Object ownerA = pa.getOwnerType();\n      Object ownerB = pb.getOwnerType();\n      boolean ownersAreEqual = ownerA == ownerB || (ownerA != null && ownerA.equals(ownerB));\n      boolean rawTypesAreEqual = pa.getRawType().equals(pb.getRawType());\n      boolean typeArgumentsAreEqual =\n          Arrays.equals(pa.getActualTypeArguments(), pb.getActualTypeArguments());\n\n      return ownersAreEqual && rawTypesAreEqual && typeArgumentsAreEqual;\n\n    } else if (a instanceof GenericArrayType) {\n      if (!(b instanceof GenericArrayType)) return false;\n      GenericArrayType ga = (GenericArrayType) a;\n      GenericArrayType gb = (GenericArrayType) b;\n      return equals(ga.getGenericComponentType(), gb.getGenericComponentType());\n\n    } else if (a instanceof WildcardType) {\n      if (!(b instanceof WildcardType)) return false;\n      WildcardType wa = (WildcardType) a;\n      WildcardType wb = (WildcardType) b;\n      return Arrays.equals(wa.getUpperBounds(), wb.getUpperBounds())\n          && Arrays.equals(wa.getLowerBounds(), wb.getLowerBounds());\n\n    } else if (a instanceof TypeVariable) {\n      if (!(b instanceof TypeVariable)) return false;\n      TypeVariable<?> va = (TypeVariable<?>) a;\n      TypeVariable<?> vb = (TypeVariable<?>) b;\n      return va.getGenericDeclaration() == vb.getGenericDeclaration()\n          && va.getName().equals(vb.getName());\n\n    } else {\n      return false; // This isn't a type we support!\n    }\n  }\n\n  /**\n   * Returns the generic supertype for {@code supertype}. For example, given a class {@code\n   * IntegerSet}, the result for when supertype is {@code Set.class} is {@code Set<Integer>} and the\n   * result when the supertype is {@code Collection.class} is {@code Collection<Integer>}.\n   */\n  static Type getGenericSupertype(Type context, Class<?> rawType, Class<?> toResolve) {\n    if (toResolve == rawType) return context;\n\n    // We skip searching through interfaces if unknown is an interface.\n    if (toResolve.isInterface()) {\n      Class<?>[] interfaces = rawType.getInterfaces();\n      for (int i = 0, length = interfaces.length; i < length; i++) {\n        if (interfaces[i] == toResolve) {\n          return rawType.getGenericInterfaces()[i];\n        } else if (toResolve.isAssignableFrom(interfaces[i])) {\n          return getGenericSupertype(rawType.getGenericInterfaces()[i], interfaces[i], toResolve);\n        }\n      }\n    }\n\n    // Check our supertypes.\n    if (!rawType.isInterface()) {\n      while (rawType != Object.class) {\n        Class<?> rawSupertype = rawType.getSuperclass();\n        if (rawSupertype == toResolve) {\n          return rawType.getGenericSuperclass();\n        } else if (toResolve.isAssignableFrom(rawSupertype)) {\n          return getGenericSupertype(rawType.getGenericSuperclass(), rawSupertype, toResolve);\n        }\n        rawType = rawSupertype;\n      }\n    }\n\n    // We can't resolve this further.\n    return toResolve;\n  }\n\n  private static int indexOf(Object[] array, Object toFind) {\n    for (int i = 0; i < array.length; i++) {\n      if (toFind.equals(array[i])) return i;\n    }\n    throw new NoSuchElementException();\n  }\n\n  static String typeToString(Type type) {\n    return type instanceof Class ? ((Class<?>) type).getName() : type.toString();\n  }\n\n  /**\n   * Returns the generic form of {@code supertype}. For example, if this is {@code\n   * ArrayList<String>}, this returns {@code Iterable<String>} given the input {@code\n   * Iterable.class}.\n   *\n   * @param supertype a superclass of, or interface implemented by, this.\n   */\n  static Type getSupertype(Type context, Class<?> contextRawType, Class<?> supertype) {\n    if (!supertype.isAssignableFrom(contextRawType)) throw new IllegalArgumentException();\n    return resolve(\n        context, contextRawType, getGenericSupertype(context, contextRawType, supertype));\n  }\n\n  static Type resolve(Type context, Class<?> contextRawType, Type toResolve) {\n    // This implementation is made a little more complicated in an attempt to avoid object-creation.\n    while (true) {\n      if (toResolve instanceof TypeVariable) {\n        TypeVariable<?> typeVariable = (TypeVariable<?>) toResolve;\n        toResolve = resolveTypeVariable(context, contextRawType, typeVariable);\n        if (toResolve == typeVariable) {\n          return toResolve;\n        }\n\n      } else if (toResolve instanceof Class && ((Class<?>) toResolve).isArray()) {\n        Class<?> original = (Class<?>) toResolve;\n        Type componentType = original.getComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof GenericArrayType) {\n        GenericArrayType original = (GenericArrayType) toResolve;\n        Type componentType = original.getGenericComponentType();\n        Type newComponentType = resolve(context, contextRawType, componentType);\n        return componentType == newComponentType\n            ? original\n            : new GenericArrayTypeImpl(newComponentType);\n\n      } else if (toResolve instanceof ParameterizedType) {\n        ParameterizedType original = (ParameterizedType) toResolve;\n        Type ownerType = original.getOwnerType();\n        Type newOwnerType = resolve(context, contextRawType, ownerType);\n        boolean changed = newOwnerType != ownerType;\n\n        Type[] args = original.getActualTypeArguments();\n        for (int t = 0, length = args.length; t < length; t++) {\n          Type resolvedTypeArgument = resolve(context, contextRawType, args[t]);\n          if (resolvedTypeArgument != args[t]) {\n            if (!changed) {\n              args = args.clone();\n              changed = true;\n            }\n            args[t] = resolvedTypeArgument;\n          }\n        }\n\n        return changed\n            ? new ParameterizedTypeImpl(newOwnerType, original.getRawType(), args)\n            : original;\n\n      } else if (toResolve instanceof WildcardType) {\n        WildcardType original = (WildcardType) toResolve;\n        Type[] originalLowerBound = original.getLowerBounds();\n        Type[] originalUpperBound = original.getUpperBounds();\n\n        if (originalLowerBound.length == 1) {\n          Type lowerBound = resolve(context, contextRawType, originalLowerBound[0]);\n          if (lowerBound != originalLowerBound[0]) {\n            return new WildcardTypeImpl(new Type[] {Object.class}, new Type[] {lowerBound});\n          }\n        } else if (originalUpperBound.length == 1) {\n          Type upperBound = resolve(context, contextRawType, originalUpperBound[0]);\n          if (upperBound != originalUpperBound[0]) {\n            return new WildcardTypeImpl(new Type[] {upperBound}, EMPTY_TYPE_ARRAY);\n          }\n        }\n        return original;\n\n      } else {\n        return toResolve;\n      }\n    }\n  }\n\n  private static Type resolveTypeVariable(\n      Type context, Class<?> contextRawType, TypeVariable<?> unknown) {\n    Class<?> declaredByRaw = declaringClassOf(unknown);\n\n    // We can't reduce this further.\n    if (declaredByRaw == null) return unknown;\n\n    Type declaredBy = getGenericSupertype(context, contextRawType, declaredByRaw);\n    if (declaredBy instanceof ParameterizedType) {\n      int index = indexOf(declaredByRaw.getTypeParameters(), unknown);\n      return ((ParameterizedType) declaredBy).getActualTypeArguments()[index];\n    }\n\n    return unknown;\n  }\n\n  /**\n   * Returns the declaring class of {@code typeVariable}, or {@code null} if it was not declared by\n   * a class.\n   */\n  private static @Nullable Class<?> declaringClassOf(TypeVariable<?> typeVariable) {\n    GenericDeclaration genericDeclaration = typeVariable.getGenericDeclaration();\n    return genericDeclaration instanceof Class ? (Class<?>) genericDeclaration : null;\n  }\n\n  static void checkNotPrimitive(Type type) {\n    if (type instanceof Class<?> && ((Class<?>) type).isPrimitive()) {\n      throw new IllegalArgumentException();\n    }\n  }\n\n  /** Returns true if {@code annotations} contains an instance of {@code cls}. */\n  static boolean isAnnotationPresent(Annotation[] annotations, Class<? extends Annotation> cls) {\n    for (Annotation annotation : annotations) {\n      if (cls.isInstance(annotation)) {\n        return true;\n      }\n    }\n    return false;\n  }\n\n  static ResponseBody buffer(final ResponseBody body) throws IOException {\n    Buffer buffer = new Buffer();\n    body.source().readAll(buffer);\n    return ResponseBody.create(body.contentType(), body.contentLength(), buffer);\n  }\n\n  static Type getParameterUpperBound(int index, ParameterizedType type) {\n    Type[] types = type.getActualTypeArguments();\n    if (index < 0 || index >= types.length) {\n      throw new IllegalArgumentException(\n          \"Index \" + index + \" not in range [0,\" + types.length + \") for \" + type);\n    }\n    Type paramType = types[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getUpperBounds()[0];\n    }\n    return paramType;\n  }\n\n  static Type getParameterLowerBound(int index, ParameterizedType type) {\n    Type paramType = type.getActualTypeArguments()[index];\n    if (paramType instanceof WildcardType) {\n      return ((WildcardType) paramType).getLowerBounds()[0];\n    }\n    return paramType;\n  }\n\n  static boolean hasUnresolvableType(@Nullable Type type) {\n    if (type instanceof Class<?>) {\n      return false;\n    }\n    if (type instanceof ParameterizedType) {\n      ParameterizedType parameterizedType = (ParameterizedType) type;\n      for (Type typeArgument : parameterizedType.getActualTypeArguments()) {\n        if (hasUnresolvableType(typeArgument)) {\n          return true;\n        }\n      }\n      return false;\n    }\n    if (type instanceof GenericArrayType) {\n      return hasUnresolvableType(((GenericArrayType) type).getGenericComponentType());\n    }\n    if (type instanceof TypeVariable) {\n      return true;\n    }\n    if (type instanceof WildcardType) {\n      return true;\n    }\n    String className = type == null ? \"null\" : type.getClass().getName();\n    throw new IllegalArgumentException(\n        \"Expected a Class, ParameterizedType, or \"\n            + \"GenericArrayType, but <\"\n            + type\n            + \"> is of type \"\n            + className);\n  }\n\n  static final class ParameterizedTypeImpl implements ParameterizedType {\n    private final @Nullable Type ownerType;\n    private final Type rawType;\n    private final Type[] typeArguments;\n\n    ParameterizedTypeImpl(@Nullable Type ownerType, Type rawType, Type... typeArguments) {\n      // Require an owner type if the raw type needs it.\n      if (rawType instanceof Class<?>\n          && (ownerType == null) != (((Class<?>) rawType).getEnclosingClass() == null)) {\n        throw new IllegalArgumentException();\n      }\n\n      for (Type typeArgument : typeArguments) {\n        Objects.requireNonNull(typeArgument, \"typeArgument == null\");\n        checkNotPrimitive(typeArgument);\n      }\n\n      this.ownerType = ownerType;\n      this.rawType = rawType;\n      this.typeArguments = typeArguments.clone();\n    }\n\n    @Override\n    public Type[] getActualTypeArguments() {\n      return typeArguments.clone();\n    }\n\n    @Override\n    public Type getRawType() {\n      return rawType;\n    }\n\n    @Override\n    public @Nullable Type getOwnerType() {\n      return ownerType;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof ParameterizedType && Utils.equals(this, (ParameterizedType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      return Arrays.hashCode(typeArguments)\n          ^ rawType.hashCode()\n          ^ (ownerType != null ? ownerType.hashCode() : 0);\n    }\n\n    @Override\n    public String toString() {\n      if (typeArguments.length == 0) return typeToString(rawType);\n      StringBuilder result = new StringBuilder(30 * (typeArguments.length + 1));\n      result.append(typeToString(rawType));\n      result.append(\"<\").append(typeToString(typeArguments[0]));\n      for (int i = 1; i < typeArguments.length; i++) {\n        result.append(\", \").append(typeToString(typeArguments[i]));\n      }\n      return result.append(\">\").toString();\n    }\n  }\n\n  private static final class GenericArrayTypeImpl implements GenericArrayType {\n    private final Type componentType;\n\n    GenericArrayTypeImpl(Type componentType) {\n      this.componentType = componentType;\n    }\n\n    @Override\n    public Type getGenericComponentType() {\n      return componentType;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n      return o instanceof GenericArrayType && Utils.equals(this, (GenericArrayType) o);\n    }\n\n    @Override\n    public int hashCode() {\n      return componentType.hashCode();\n    }\n\n    @Override\n    public String toString() {\n      return typeToString(componentType) + \"[]\";\n    }\n  }\n\n  /**\n   * The WildcardType interface supports multiple upper bounds and multiple lower bounds. We only\n   * support what the Java 6 language needs - at most one bound. If a lower bound is set, the upper\n   * bound must be Object.class.\n   */\n  private static final class WildcardTypeImpl implements WildcardType {\n    private final Type upperBound;\n    private final @Nullable Type lowerBound;\n\n    WildcardTypeImpl(Type[] upperBounds, Type[] lowerBounds) {\n      if (lowerBounds.length > 1) throw new IllegalArgumentException();\n      if (upperBounds.length != 1) throw new IllegalArgumentException();\n\n      if (lowerBounds.length == 1) {\n        if (lowerBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(lowerBounds[0]);\n        if (upperBounds[0] != Object.class) throw new IllegalArgumentException();\n        this.lowerBound = lowerBounds[0];\n        this.upperBound = Object.class;\n      } else {\n        if (upperBounds[0] == null) throw new NullPointerException();\n        checkNotPrimitive(upperBounds[0]);\n        this.lowerBound = null;\n        this.upperBound = upperBounds[0];\n      }\n    }\n\n    @Override\n    public Type[] getUpperBounds() {\n      return new Type[] {upperBound};\n    }\n\n    @Override\n    public Type[] getLowerBounds() {\n      return lowerBound != null ? new Type[] {lowerBound} : EMPTY_TYPE_ARRAY;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      return other instanceof WildcardType && Utils.equals(this, (WildcardType) other);\n    }\n\n    @Override\n    public int hashCode() {\n      // This equals Arrays.hashCode(getLowerBounds()) ^ Arrays.hashCode(getUpperBounds()).\n      return (lowerBound != null ? 31 + lowerBound.hashCode() : 1) ^ (31 + upperBound.hashCode());\n    }\n\n    @Override\n    public String toString() {\n      if (lowerBound != null) return \"? super \" + typeToString(lowerBound);\n      if (upperBound == Object.class) return \"?\";\n      return \"? extends \" + typeToString(upperBound);\n    }\n  }\n\n  // https://github.com/ReactiveX/RxJava/blob/6a44e5d0543a48f1c378dc833a155f3f71333bc2/\n  // src/main/java/io/reactivex/exceptions/Exceptions.java#L66\n  static void throwIfFatal(Throwable t) {\n    if (t instanceof VirtualMachineError) {\n      throw (VirtualMachineError) t;\n    } else if (t instanceof ThreadDeath) {\n      throw (ThreadDeath) t;\n    } else if (t instanceof LinkageError) {\n      throw (LinkageError) t;\n    }\n  }\n\n  /** Not volatile because we don't mind multiple threads discovering this. */\n  private static boolean checkForKotlinUnit = true;\n\n  static boolean isUnit(Type type) {\n    if (checkForKotlinUnit) {\n      try {\n        return type == Unit.class;\n      } catch (NoClassDefFoundError ignored) {\n        checkForKotlinUnit = false;\n      }\n    }\n    return false;\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/OkHttpCall.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.throwIfFatal;\n\nimport java.io.IOException;\nimport java.util.Objects;\nimport javax.annotation.Nullable;\nimport javax.annotation.concurrent.GuardedBy;\nimport okhttp3.MediaType;\nimport okhttp3.Request;\nimport okhttp3.ResponseBody;\nimport okio.Buffer;\nimport okio.BufferedSource;\nimport okio.ForwardingSource;\nimport okio.Okio;\nimport okio.Timeout;\n\nfinal class OkHttpCall<T> implements Call<T> {\n  private final RequestFactory requestFactory;\n  private final Object instance;\n  private final Object[] args;\n  private final okhttp3.Call.Factory callFactory;\n  private final Converter<ResponseBody, T> responseConverter;\n\n  private volatile boolean canceled;\n\n  @GuardedBy(\"this\")\n  private @Nullable okhttp3.Call rawCall;\n\n  @GuardedBy(\"this\") // Either a RuntimeException, non-fatal Error, or IOException.\n  private @Nullable Throwable creationFailure;\n\n  @GuardedBy(\"this\")\n  private boolean executed;\n\n  OkHttpCall(\n      RequestFactory requestFactory,\n      Object instance,\n      Object[] args,\n      okhttp3.Call.Factory callFactory,\n      Converter<ResponseBody, T> responseConverter) {\n    this.requestFactory = requestFactory;\n    this.instance = instance;\n    this.args = args;\n    this.callFactory = callFactory;\n    this.responseConverter = responseConverter;\n  }\n\n  @SuppressWarnings(\"CloneDoesntCallSuperClone\") // We are a final type & this saves clearing state.\n  @Override\n  public OkHttpCall<T> clone() {\n    return new OkHttpCall<>(requestFactory, instance, args, callFactory, responseConverter);\n  }\n\n  @Override\n  public synchronized Request request() {\n    try {\n      return getRawCall().request();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create request.\", e);\n    }\n  }\n\n  @Override\n  public synchronized Timeout timeout() {\n    try {\n      return getRawCall().timeout();\n    } catch (IOException e) {\n      throw new RuntimeException(\"Unable to create call.\", e);\n    }\n  }\n\n  /**\n   * Returns the raw call, initializing it if necessary. Throws if initializing the raw call throws,\n   * or has thrown in previous attempts to create it.\n   */\n  @GuardedBy(\"this\")\n  private okhttp3.Call getRawCall() throws IOException {\n    okhttp3.Call call = rawCall;\n    if (call != null) return call;\n\n    // Re-throw previous failures if this isn't the first attempt.\n    if (creationFailure != null) {\n      if (creationFailure instanceof IOException) {\n        throw (IOException) creationFailure;\n      } else if (creationFailure instanceof RuntimeException) {\n        throw (RuntimeException) creationFailure;\n      } else {\n        throw (Error) creationFailure;\n      }\n    }\n\n    // Create and remember either the success or the failure.\n    try {\n      return rawCall = createRawCall();\n    } catch (RuntimeException | Error | IOException e) {\n      throwIfFatal(e); // Do not assign a fatal error to creationFailure.\n      creationFailure = e;\n      throw e;\n    }\n  }\n\n  @Override\n  public void enqueue(final Callback<T> callback) {\n    Objects.requireNonNull(callback, \"callback == null\");\n\n    okhttp3.Call call;\n    Throwable failure;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = rawCall;\n      failure = creationFailure;\n      if (call == null && failure == null) {\n        try {\n          call = rawCall = createRawCall();\n        } catch (Throwable t) {\n          throwIfFatal(t);\n          failure = creationFailure = t;\n        }\n      }\n    }\n\n    if (failure != null) {\n      callback.onFailure(this, failure);\n      return;\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    call.enqueue(\n        new okhttp3.Callback() {\n          @Override\n          public void onResponse(okhttp3.Call call, okhttp3.Response rawResponse) {\n            Response<T> response;\n            try {\n              response = parseResponse(rawResponse);\n            } catch (Throwable e) {\n              throwIfFatal(e);\n              callFailure(e);\n              return;\n            }\n\n            try {\n              callback.onResponse(OkHttpCall.this, response);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n\n          @Override\n          public void onFailure(okhttp3.Call call, IOException e) {\n            callFailure(e);\n          }\n\n          private void callFailure(Throwable e) {\n            try {\n              callback.onFailure(OkHttpCall.this, e);\n            } catch (Throwable t) {\n              throwIfFatal(t);\n              t.printStackTrace(); // TODO this is not great\n            }\n          }\n        });\n  }\n\n  @Override\n  public synchronized boolean isExecuted() {\n    return executed;\n  }\n\n  @Override\n  public Response<T> execute() throws IOException {\n    okhttp3.Call call;\n\n    synchronized (this) {\n      if (executed) throw new IllegalStateException(\"Already executed.\");\n      executed = true;\n\n      call = getRawCall();\n    }\n\n    if (canceled) {\n      call.cancel();\n    }\n\n    return parseResponse(call.execute());\n  }\n\n  private okhttp3.Call createRawCall() throws IOException {\n    okhttp3.Call call = callFactory.newCall(requestFactory.create(instance, args));\n    if (call == null) {\n      throw new NullPointerException(\"Call.Factory returned null.\");\n    }\n    return call;\n  }\n\n  Response<T> parseResponse(okhttp3.Response rawResponse) throws IOException {\n    ResponseBody rawBody = rawResponse.body();\n\n    // Remove the body's source (the only stateful object) so we can pass the response along.\n    rawResponse =\n        rawResponse\n            .newBuilder()\n            .body(new NoContentResponseBody(rawBody.contentType(), rawBody.contentLength()))\n            .build();\n\n    int code = rawResponse.code();\n    if (code < 200 || code >= 300) {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        ResponseBody bufferedBody = Utils.buffer(rawBody);\n        return Response.error(bufferedBody, rawResponse);\n      } finally {\n        rawBody.close();\n      }\n    }\n\n    if (code == 204 || code == 205) {\n      rawBody.close();\n      return Response.success(null, rawResponse);\n    }\n\n    ExceptionCatchingResponseBody catchingBody = new ExceptionCatchingResponseBody(rawBody);\n    try {\n      T body = responseConverter.convert(catchingBody);\n      return Response.success(body, rawResponse);\n    } catch (RuntimeException e) {\n      // If the underlying source threw an exception, propagate that rather than indicating it was\n      // a runtime exception.\n      catchingBody.throwIfCaught();\n      throw e;\n    }\n  }\n\n  @Override\n  public void cancel() {\n    canceled = true;\n\n    okhttp3.Call call;\n    synchronized (this) {\n      call = rawCall;\n    }\n    if (call != null) {\n      call.cancel();\n    }\n  }\n\n  @Override\n  public boolean isCanceled() {\n    if (canceled) {\n      return true;\n    }\n    synchronized (this) {\n      return rawCall != null && rawCall.isCanceled();\n    }\n  }\n\n  static final class NoContentResponseBody extends ResponseBody {\n    private final @Nullable MediaType contentType;\n    private final long contentLength;\n\n    NoContentResponseBody(@Nullable MediaType contentType, long contentLength) {\n      this.contentType = contentType;\n      this.contentLength = contentLength;\n    }\n\n    @Override\n    public MediaType contentType() {\n      return contentType;\n    }\n\n    @Override\n    public long contentLength() {\n      return contentLength;\n    }\n\n    @Override\n    public BufferedSource source() {\n      throw new IllegalStateException(\"Cannot read raw response body of a converted body.\");\n    }\n  }\n\n  static final class ExceptionCatchingResponseBody extends ResponseBody {\n    private final ResponseBody delegate;\n    private final BufferedSource delegateSource;\n    @Nullable IOException thrownException;\n\n    ExceptionCatchingResponseBody(ResponseBody delegate) {\n      this.delegate = delegate;\n      this.delegateSource =\n          Okio.buffer(\n              new ForwardingSource(delegate.source()) {\n                @Override\n                public long read(Buffer sink, long byteCount) throws IOException {\n                  try {\n                    return super.read(sink, byteCount);\n                  } catch (IOException e) {\n                    thrownException = e;\n                    throw e;\n                  }\n                }\n              });\n    }\n\n    @Override\n    public MediaType contentType() {\n      return delegate.contentType();\n    }\n\n    @Override\n    public long contentLength() {\n      return delegate.contentLength();\n    }\n\n    @Override\n    public BufferedSource source() {\n      return delegateSource;\n    }\n\n    @Override\n    public void close() {\n      delegate.close();\n    }\n\n    void throwIfCaught() throws IOException {\n      if (thrownException != null) {\n        throw thrownException;\n      }\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/http/Streaming.java\n/*\n * Copyright (C) 2014 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2.http;\n\nimport static java.lang.annotation.ElementType.METHOD;\nimport static java.lang.annotation.RetentionPolicy.RUNTIME;\n\nimport java.lang.annotation.Documented;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.Target;\nimport okhttp3.ResponseBody;\n\n/**\n * Treat the response body on methods returning {@link ResponseBody ResponseBody} as is, i.e.\n * without converting the body to {@code byte[]}.\n */\n@Documented\n@Target(METHOD)\n@Retention(RUNTIME)\npublic @interface Streaming {}\n\n// Path: retrofit/src/main/java/retrofit2/BuiltInConverters.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.io.IOException;\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport okhttp3.RequestBody;\nimport okhttp3.ResponseBody;\nimport retrofit2.http.Streaming;\n\nfinal class BuiltInConverters extends Converter.Factory {\n\n  @Override\n  public @Nullable Converter<ResponseBody, ?> responseBodyConverter(\n      Type type, Annotation[] annotations, Retrofit retrofit) {\n    if (type == ResponseBody.class) {\n      return Utils.isAnnotationPresent(annotations, Streaming.class)\n          ? StreamingResponseBodyConverter.INSTANCE\n          : BufferingResponseBodyConverter.INSTANCE;\n    }\n    if (type == Void.class) {\n      return VoidResponseBodyConverter.INSTANCE;\n    }\n    if (Utils.isUnit(type)) {\n      return UnitResponseBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  @Override\n  public @Nullable Converter<?, RequestBody> requestBodyConverter(\n      Type type,\n      Annotation[] parameterAnnotations,\n      Annotation[] methodAnnotations,\n      Retrofit retrofit) {\n    if (RequestBody.class.isAssignableFrom(Utils.getRawType(type))) {\n      return RequestBodyConverter.INSTANCE;\n    }\n    return null;\n  }\n\n  static final class VoidResponseBodyConverter implements Converter<ResponseBody, Void> {\n    static final VoidResponseBodyConverter INSTANCE = new VoidResponseBodyConverter();\n\n    @Override\n    public Void convert(ResponseBody value) {\n      value.close();\n      return null;\n    }\n  }\n\n  static final class UnitResponseBodyConverter implements Converter<ResponseBody, Unit> {\n    static final UnitResponseBodyConverter INSTANCE = new UnitResponseBodyConverter();\n\n    @Override\n    public Unit convert(ResponseBody value) {\n      value.close();\n      return Unit.INSTANCE;\n    }\n  }\n\n  static final class RequestBodyConverter implements Converter<RequestBody, RequestBody> {\n    static final RequestBodyConverter INSTANCE = new RequestBodyConverter();\n\n    @Override\n    public RequestBody convert(RequestBody value) {\n      return value;\n    }\n  }\n\n  static final class StreamingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final StreamingResponseBodyConverter INSTANCE = new StreamingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) {\n      return value;\n    }\n  }\n\n  static final class BufferingResponseBodyConverter\n      implements Converter<ResponseBody, ResponseBody> {\n    static final BufferingResponseBodyConverter INSTANCE = new BufferingResponseBodyConverter();\n\n    @Override\n    public ResponseBody convert(ResponseBody value) throws IOException {\n      try {\n        // Buffer the entire body to avoid future I/O.\n        return Utils.buffer(value);\n      } finally {\n        value.close();\n      }\n    }\n  }\n\n  static final class ToStringConverter implements Converter<Object, String> {\n    static final ToStringConverter INSTANCE = new ToStringConverter();\n\n    @Override\n    public String convert(Object value) {\n      return value.toString();\n    }\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/SkipCallbackExecutorImpl.java\n/*\n * Copyright (C) 2019 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport java.lang.annotation.Annotation;\n\n// This class conforms to the annotation requirements documented on Annotation.\nfinal class SkipCallbackExecutorImpl implements SkipCallbackExecutor {\n  private static final SkipCallbackExecutor INSTANCE = new SkipCallbackExecutorImpl();\n\n  static Annotation[] ensurePresent(Annotation[] annotations) {\n    if (Utils.isAnnotationPresent(annotations, SkipCallbackExecutor.class)) {\n      return annotations;\n    }\n\n    Annotation[] newAnnotations = new Annotation[annotations.length + 1];\n    // Place the skip annotation first since we're guaranteed to check for it in the call adapter.\n    newAnnotations[0] = SkipCallbackExecutorImpl.INSTANCE;\n    System.arraycopy(annotations, 0, newAnnotations, 1, annotations.length);\n    return newAnnotations;\n  }\n\n  \n@Override\n  public Class<? extends Annotation> annotationType() {\n    return SkipCallbackExecutor.class;\n  }\n\n  @Override\n  public boolean equals(Object obj) {\n    return obj instanceof SkipCallbackExecutor;\n  }\n\n  @Override\n  public int hashCode() {\n    return 0;\n  }\n\n  @Override\n  public String toString() {\n    return \"@\" + SkipCallbackExecutor.class.getName() + \"()\";\n  }\n}\n\n// Path: retrofit/src/main/java/retrofit2/HttpServiceMethod.java\n/*\n * Copyright (C) 2015 Square, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage retrofit2;\n\nimport static retrofit2.Utils.getRawType;\nimport static retrofit2.Utils.methodError;\n\nimport java.lang.annotation.Annotation;\nimport java.lang.reflect.Method;\nimport java.lang.reflect.ParameterizedType;\nimport java.lang.reflect.Type;\nimport javax.annotation.Nullable;\nimport kotlin.Unit;\nimport kotlin.coroutines.Continuation;\nimport okhttp3.ResponseBody;\n\n/** Adapts an invocation of an interface method into an HTTP call. */\nabstract class HttpServiceMethod<ResponseT, ReturnT> extends ServiceMethod<ReturnT> {\n  /**\n   * Inspects the annotations on an interface method to construct a reusable service method that\n   * speaks HTTP. This requires potentially-expensive reflection so it is best to build each service\n   * method only once and reuse it.\n   */\n  static <ResponseT, ReturnT> HttpServiceMethod<ResponseT, ReturnT> parseAnnotations(\n      Retrofit retrofit, Method method, RequestFactory requestFactory) {\n    boolean isKotlinSuspendFunction = requestFactory.isKotlinSuspendFunction;\n    boolean continuationWantsResponse = false;\n    boolean continuationBodyNullable = false;\n    boolean continuationIsUnit = false;\n\n    Annotation[] annotations = method.getAnnotations();\n    Type adapterType;\n    if (isKotlinSuspendFunction) {\n      Type[] parameterTypes = method.getGenericParameterTypes();\n      Type responseType =\n          Utils.getParameterLowerBound(\n              0, (ParameterizedType) parameterTypes[parameterTypes.length - 1]);\n      if (getRawType(responseType) == Response.class && responseType instanceof ParameterizedType) {\n        // Unwrap the actual body type from Response<T>.\n        responseType = Utils.getParameterUpperBound(0, (ParameterizedType) responseType);\n        continuationWantsResponse = true;\n      } else {\n        if (getRawType(responseType) == Call.class) {\n          throw methodError(\n              method,\n...\n", "needle_token_start": 12124, "needle_token_end": 12152, "code_context_ntokens": 12935, "output": ["```java\n@Override\npublic Class<? extends Annotation> annotationType() {\n  return SkipCallbackExecutor.class;\n}\n```"]}
{"repo": "karatelabs/karate", "name": "min", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/MatchStep.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the smallest of two given integers, specifically handling cases where either or both might be a special value indicating absence or invalidity.\n2. **Input**: Two integers, where a value of `-1` typically represents an invalid or absent value.\n3. **Output**: Returns the smallest valid integer of the two provided. If both are invalid (`-1`), it returns `-1`.\n4. **Procedure**: The function checks if either of the integers is `-1`, which indicates an invalid or absent value. If one integer is `-1`, the function returns the other integer. If both are valid, it returns the smaller integer using a standard comparison.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            contains = true;\n            not = raw.charAt(lhsEndPos + 1) == '!';\n            searchPos = lhsEndPos + (not ? 10 : 9);\n            String anyOrOnlyOrDeep = raw.substring(searchPos).trim();\n            if (anyOrOnlyOrDeep.startsWith(\"only deep\")) {\n                int onlyPos = raw.indexOf(\" only deep\", searchPos);\n                only = true;\n                deep = true;\n                searchPos = onlyPos + 10;\n            } else if (anyOrOnlyOrDeep.startsWith(\"only\")) {\n                int onlyPos = raw.indexOf(\" only\", searchPos);\n                only = true;\n                searchPos = onlyPos + 5;\n            } else if (anyOrOnlyOrDeep.startsWith(\"any\")) {\n                int anyPos = raw.indexOf(\" any\", searchPos);\n                any = true;\n                searchPos = anyPos + 4;\n            } else if (anyOrOnlyOrDeep.startsWith(\"deep\")) {\n                int deepPos = raw.indexOf(\" deep\", searchPos);\n                deep = true;\n                searchPos = deepPos + 5;\n            }\n        } else {\n            int equalPos = raw.indexOf(\" ==\", searchPos);\n            int notEqualPos = raw.indexOf(\" !=\", searchPos);\n            if (equalPos == -1 && notEqualPos == -1) {\n                throw new RuntimeException(\"syntax error, expected '==' for match\");\n            }\n            lhsEndPos = min(equalPos, notEqualPos);\n            if (lhsEndPos > spacePos && rightParenPos != -1\n                    && rightParenPos > lhsEndPos && rightParenPos < leftParenPos) {\n                equalPos = raw.indexOf(\" ==\", rightParenPos);\n                notEqualPos = raw.indexOf(\" !=\", rightParenPos);\n                if (equalPos == -1 && notEqualPos == -1) {\n                    throw new RuntimeException(\"syntax error, expected '==' for match\");\n                }\n                lhsEndPos = min(equalPos, notEqualPos);\n            }\n            not = lhsEndPos == notEqualPos;\n            searchPos = lhsEndPos + 3;\n        }\n        String lhs = raw.substring(0, lhsEndPos).trim();\n        if (leftParenPos == -1) {\n            leftParenPos = lhs.indexOf('['); // used later to test for json-path\n            char first = lhs.charAt(0);\n            if (first == '[' || first == '{') { // json array or object\n                spacePos = -1; // just use lhs\n                lhs = \"(\" + lhs + \")\";\n            }\n        }\n        if (spacePos != -1 && (leftParenPos > spacePos || leftParenPos == -1)) {\n            name = lhs.substring(0, spacePos);\n            path = StringUtils.trimToNull(lhs.substring(spacePos));\n        } else {\n            name = lhs;\n            path = null;\n        }\n        expected = StringUtils.trimToNull(raw.substring(searchPos));\n        type = getType(each, not, contains, only, any, deep);\n    }\n\n    \nprivate static int min(int a, int b) {\n        if (a == -1) {\n            return b;\n        }\n        if (b == -1) {\n            return a;\n        }\n        return Math.min(a, b);\n    }\n\n    private static Match.Type getType(boolean each, boolean not, boolean contains, boolean only, boolean any, boolean deep) {\n        if (each) {\n            if (contains) {\n                if (only) {\n                    return Match.Type.EACH_CONTAINS_ONLY;\n                }\n                if (any) {\n                    return Match.Type.EACH_CONTAINS_ANY;\n                }\n                if (deep) {\n                    if (not) {\n                        throw new RuntimeException(\"'each !contains deep' is not yet supported, use 'each contains deep' instead\");\n                    }\n                    return Match.Type.EACH_CONTAINS_DEEP;\n                }\n                return not ? Match.Type.EACH_NOT_CONTAINS : Match.Type.EACH_CONTAINS;\n            }\n            return not ? Match.Type.EACH_NOT_EQUALS : Match.Type.EACH_EQUALS;\n        }\n        if (contains) {\n            if (only) {\n                return deep ? Match.Type.CONTAINS_ONLY_DEEP : Match.Type.CONTAINS_ONLY;\n            }\n            if (any) {\n                return Match.Type.CONTAINS_ANY;\n            }\n            if (deep) {\n                if (not) {\n                    throw new RuntimeException(\"'!contains deep' is not yet supported, use 'contains deep' instead\");\n                }\n                return Match.Type.CONTAINS_DEEP;\n            }\n            return not ? Match.Type.NOT_CONTAINS : Match.Type.CONTAINS;\n        }\n        return not ? Match.Type.NOT_EQUALS : Match.Type.EQUALS;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/MatchOperation.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate;\n\nimport com.intuit.karate.graal.JsEngine;\nimport com.intuit.karate.graal.JsValue;\nimport java.math.BigDecimal;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n/**\n *\n * @author pthomas3\n */\npublic class MatchOperation {\n\n    public static final String REGEX = \"regex\";        \n\n    final Match.Context context;\n    final Match.Type type;\n    final Match.Value actual;\n    final Match.Value expected;\n    final List<MatchOperation> failures;\n    // TODO merge this with Match.Type which should be a complex object not an enum\n    final boolean matchEachEmptyAllowed;\n\n    boolean pass = true;\n    private String failReason;\n\n    MatchOperation(Match.Type type, Match.Value actual, Match.Value expected, boolean matchEachEmptyAllowed) {\n        this(JsEngine.global(), null, type, actual, expected, matchEachEmptyAllowed);\n    }\n\n    MatchOperation(JsEngine js, Match.Type type, Match.Value actual, Match.Value expected, boolean matchEachEmptyAllowed) {\n        this(js, null, type, actual, expected, matchEachEmptyAllowed);\n    }\n\n    MatchOperation(Match.Context context, Match.Type type, Match.Value actual, Match.Value expected, boolean matchEachEmptyAllowed) {\n        this(null, context, type, actual, expected, matchEachEmptyAllowed);\n    }\n\n    private MatchOperation(JsEngine js, Match.Context context, Match.Type type, Match.Value actual, Match.Value expected, boolean matchEachEmptyAllowed) {\n        this.type = type;\n        this.actual = actual;\n        this.expected = expected;\n        this.matchEachEmptyAllowed = matchEachEmptyAllowed;\n        if (context == null) {\n            if (js == null) {\n                js = JsEngine.global();\n            }\n            this.failures = new ArrayList();\n            if (actual.isXml()) {\n                this.context = new Match.Context(js, this, true, 0, \"/\", \"\", -1, matchEachEmptyAllowed);\n            } else {\n                this.context = new Match.Context(js, this, false, 0, \"$\", \"\", -1, matchEachEmptyAllowed);\n            }\n        } else {\n            this.context = context;\n            this.failures = context.root.failures;\n        }        \n    }\n\n    private Match.Type fromMatchEach() {\n        switch (type) {\n            case EACH_CONTAINS:\n                return Match.Type.CONTAINS;\n            case EACH_NOT_CONTAINS:\n                return Match.Type.NOT_CONTAINS;\n            case EACH_CONTAINS_ONLY:\n                return Match.Type.CONTAINS_ONLY;\n            case EACH_CONTAINS_ANY:\n                return Match.Type.CONTAINS_ANY;\n            case EACH_EQUALS:\n                return Match.Type.EQUALS;\n            case EACH_NOT_EQUALS:\n                return Match.Type.NOT_EQUALS;\n            case EACH_CONTAINS_DEEP:\n                return Match.Type.CONTAINS_DEEP;\n            default:\n                throw new RuntimeException(\"unexpected outer match type: \" + type);\n        }\n    }\n\n    private static Match.Type macroToMatchType(boolean each, String macro) {\n        if (macro.startsWith(\"^^\")) {\n            return each ? Match.Type.EACH_CONTAINS_ONLY : Match.Type.CONTAINS_ONLY;\n        } else if (macro.startsWith(\"^+\")) {\n            return each ? Match.Type.EACH_CONTAINS_DEEP : Match.Type.CONTAINS_DEEP;\n        } else if (macro.startsWith(\"^*\")) {\n            return each ? Match.Type.EACH_CONTAINS_ANY : Match.Type.CONTAINS_ANY;\n        } else if (macro.startsWith(\"^\")) {\n            return each ? Match.Type.EACH_CONTAINS : Match.Type.CONTAINS;\n        } else if (macro.startsWith(\"!^\")) {\n            return each ? Match.Type.EACH_NOT_CONTAINS : Match.Type.NOT_CONTAINS;\n        } else if (macro.startsWith(\"!=\")) {\n            return each ? Match.Type.EACH_NOT_EQUALS : Match.Type.NOT_EQUALS;\n        } else {\n            return each ? Match.Type.EACH_EQUALS : Match.Type.EQUALS;\n        }\n    }\n\n    private static int matchTypeToStartPos(Match.Type mt) {\n        switch (mt) {\n            case CONTAINS_ONLY:\n            case EACH_CONTAINS_ONLY:\n            case CONTAINS_DEEP:\n            case EACH_CONTAINS_DEEP:\n            case CONTAINS_ANY:\n            case EACH_CONTAINS_ANY:\n            case NOT_CONTAINS:\n            case EACH_NOT_CONTAINS:\n            case NOT_EQUALS:\n            case EACH_NOT_EQUALS:\n                return 2;\n            case CONTAINS:\n            case EACH_CONTAINS:\n                return 1;\n            default:\n                return 0;\n        }\n    }\n\n    boolean execute() {\n        switch (type) {\n            case EACH_CONTAINS:\n            case EACH_NOT_CONTAINS:\n            case EACH_CONTAINS_ONLY:\n            case EACH_CONTAINS_ANY:\n            case EACH_EQUALS:\n            case EACH_NOT_EQUALS:\n            case EACH_CONTAINS_DEEP:\n                if (actual.isList()) {\n                    List list = actual.getValue();\n                    if (list.isEmpty() && !matchEachEmptyAllowed) {\n                        return fail(\"match each failed, empty array / list\");\n                    }\n                    Match.Type nestedMatchType = fromMatchEach();                    \n                    int count = list.size();\n                    for (int i = 0; i < count; i++) {\n                        Object o = list.get(i);\n                        context.JS.put(\"_$\", o);\n                        MatchOperation mo = new MatchOperation(context.descend(i), nestedMatchType, new Match.Value(o), expected, matchEachEmptyAllowed);\n                        mo.execute();\n                        context.JS.bindings.removeMember(\"_$\");\n                        if (!mo.pass) {\n                            return fail(\"match each failed at index \" + i);\n                        }\n                    }\n                    // if we reached here all / each LHS items completed successfully\n                    return true;\n                } else {\n                    return fail(\"actual is not an array or list\");\n                }\n            default:\n            // do nothing\n        }\n        if (actual.isNotPresent()) {\n            if (!expected.isString() || !expected.getAsString().startsWith(\"#\")) {\n                return fail(\"actual path does not exist\");\n            }\n        }\n        if (actual.type != expected.type) {\n            switch (type) {\n                case CONTAINS:\n                case NOT_CONTAINS:\n                case CONTAINS_ANY:\n                case CONTAINS_ONLY:\n                case CONTAINS_DEEP:\n                case CONTAINS_ONLY_DEEP:\n                case CONTAINS_ANY_DEEP:\n                    // don't tamper with strings on the RHS that represent arrays or objects\n                    if (!expected.isList() && !(expected.isString() && expected.isArrayObjectOrReference())) {\n                        MatchOperation mo = new MatchOperation(context, type, actual, new Match.Value(Collections.singletonList(expected.getValue())), matchEachEmptyAllowed);\n                        mo.execute();\n                        return mo.pass ? pass() : fail(mo.failReason);\n                    }\n                    break;\n                default:\n                // do nothing\n            }\n            if (expected.isXml() && actual.isMap()) {\n                // special case, auto-convert rhs                \n                MatchOperation mo = new MatchOperation(context, type, actual, new Match.Value(XmlUtils.toObject(expected.getValue(), true)), matchEachEmptyAllowed);\n                mo.execute();\n                return mo.pass ? pass() : fail(mo.failReason);\n            }\n            if (expected.isString()) {\n                String expStr = expected.getValue();\n                if (!expStr.startsWith(\"#\")) { // edge case if rhs is macro\n                    return type == Match.Type.NOT_EQUALS ? pass() : fail(\"data types don't match\");\n                }\n            } else {\n                return type == Match.Type.NOT_EQUALS ? pass() : fail(\"data types don't match\");\n            }\n        }\n        if (expected.isString()) {\n            String expStr = expected.getValue();\n            if (expStr.startsWith(\"#\")) {\n                switch (type) {\n                    case NOT_EQUALS:\n                        return macroEqualsExpected(expStr) ? fail(\"is equal\") : pass();\n                    case NOT_CONTAINS:\n                        return macroEqualsExpected(expStr) ? fail(\"actual contains expected\") : pass();\n                    default:\n                        return macroEqualsExpected(expStr) ? pass() : fail(null);\n                }\n            }\n        }\n        switch (type) {\n            case EQUALS:\n                return actualEqualsExpected() ? pass() : fail(\"not equal\");\n            case NOT_EQUALS:\n                return actualEqualsExpected() ? fail(\"is equal\") : pass();\n            case CONTAINS:\n            case CONTAINS_ANY:\n            case CONTAINS_ONLY:\n            case CONTAINS_DEEP:\n            case CONTAINS_ONLY_DEEP:\n            case CONTAINS_ANY_DEEP:\n                return actualContainsExpected() ? pass() : fail(\"actual does not contain expected\");\n            case NOT_CONTAINS:\n                return actualContainsExpected() ? fail(\"actual contains expected\") : pass();\n            default:\n                throw new RuntimeException(\"unexpected match type: \" + type);\n        }\n    }\n\n    private boolean macroEqualsExpected(String expStr) {\n        boolean optional = expStr.startsWith(\"##\");\n        if (optional && actual.isNull()) { // exit early\n            return true;\n        }\n        int minLength = optional ? 3 : 2;\n        if (expStr.length() > minLength) {\n            String macro = expStr.substring(minLength - 1);\n            if (macro.startsWith(\"(\") && macro.endsWith(\")\")) {\n                macro = macro.substring(1, macro.length() - 1);\n                Match.Type nestedType = macroToMatchType(false, macro);\n                int startPos = matchTypeToStartPos(nestedType);\n                macro = macro.substring(startPos);\n                if (actual.isList()) { // special case, look for partial maps within list\n                    switch (nestedType) {\n                        case CONTAINS:\n                            nestedType = Match.Type.CONTAINS_DEEP;\n                            break;\n                        case CONTAINS_ONLY:\n                            nestedType = Match.Type.CONTAINS_ONLY_DEEP;\n                            break;\n                        case CONTAINS_ANY:\n                            nestedType = Match.Type.CONTAINS_ANY_DEEP;\n                            break;\n                    }\n                }\n                context.JS.put(\"$\", context.root.actual.getValue());\n                context.JS.put(\"_\", actual.getValue());\n                JsValue jv = context.JS.eval(macro);\n                context.JS.bindings.removeMember(\"$\");\n                context.JS.bindings.removeMember(\"_\");\n                MatchOperation mo = new MatchOperation(context, nestedType, actual, new Match.Value(jv.getValue()), matchEachEmptyAllowed);\n                return mo.execute();\n            } else if (macro.startsWith(\"[\")) {\n                int closeBracketPos = macro.indexOf(']');\n                if (closeBracketPos != -1) { // array, match each\n                    if (!actual.isList()) {\n                        return fail(\"actual is not an array\");\n                    }\n                    if (closeBracketPos > 1) {\n                        String bracketContents = macro.substring(1, closeBracketPos);\n                        List listAct = actual.getValue();\n                        int listSize = listAct.size();\n                        context.JS.put(\"$\", context.root.actual.getValue());\n                        context.JS.put(\"_\", listSize);\n                        String sizeExpr;\n                        if (containsPlaceholderUnderscore(bracketContents)) { // #[_ < 5]\n                            sizeExpr = bracketContents;\n                        } else { // #[5] | #[$.foo] \n                            sizeExpr = bracketContents + \" == _\";\n                        }\n                        JsValue jv = context.JS.eval(sizeExpr);\n                        context.JS.bindings.removeMember(\"$\");\n                        context.JS.bindings.removeMember(\"_\");\n                        if (!jv.isTrue()) {\n                            return fail(\"actual array length is \" + listSize);\n                        }\n                    }\n                    if (macro.length() > closeBracketPos + 1) {\n                        macro = StringUtils.trimToNull(macro.substring(closeBracketPos + 1));\n                        if (macro != null) {\n                            if (macro.startsWith(\"(\") && macro.endsWith(\")\")) {\n                                macro = macro.substring(1, macro.length() - 1); // strip parens\n                            }\n                            if (macro.startsWith(\"?\")) { // #[]? _.length == 3\n                                macro = \"#\" + macro;\n                            }\n                            if (macro.startsWith(\"#\")) {\n                                MatchOperation mo = new MatchOperation(context, Match.Type.EACH_EQUALS, actual, new Match.Value(macro), matchEachEmptyAllowed);\n                                mo.execute();\n                                return mo.pass ? pass() : fail(\"all array elements matched\");\n                            } else { // schema reference\n                                Match.Type nestedType = macroToMatchType(true, macro); // match each\n                                int startPos = matchTypeToStartPos(nestedType);\n                                macro = macro.substring(startPos);\n                                JsValue jv = context.JS.eval(macro);\n                                MatchOperation mo = new MatchOperation(context, nestedType, actual, new Match.Value(jv.getValue()), matchEachEmptyAllowed);\n                                return mo.execute();\n                            }\n                        }\n                    }\n                    return true; // expression within square brackets is ok\n                }\n            } else { // '#? _ != 0' | '#string' | '#number? _ > 0'\n                int questionPos = macro.indexOf('?');\n                String validatorName = null;\n                // in case of regex we don't want to remove the '?'\n                if (questionPos != -1 && !macro.startsWith(REGEX)) {\n                    validatorName = macro.substring(0, questionPos);\n                    if (macro.length() > questionPos + 1) {\n                        macro = StringUtils.trimToEmpty(macro.substring(questionPos + 1));\n                    } else {\n                        macro = \"\";\n                    }\n                } else {\n                    validatorName = macro;\n                    macro = \"\";\n                }\n                validatorName = StringUtils.trimToNull(validatorName);\n                if (validatorName != null) {\n                    Match.Validator validator = null;\n                    if (validatorName.startsWith(REGEX)) {\n                        String regex = validatorName.substring(5).trim();\n                        validator = new Match.RegexValidator(regex);\n                    } else {\n                        validator = Match.VALIDATORS.get(validatorName);\n                    }\n                    if (validator != null) {\n                        if (optional && (actual.isNotPresent() || actual.isNull())) {\n                            // pass\n                        } else if (!optional && actual.isNotPresent()) {\n                            // if the element is not present the expected result can only be\n                            // the notpresent keyword, ignored or an optional comparison\n                            return expected.isNotPresent() || \"#ignore\".contentEquals(expected.getAsString());\n                        } else {\n                            Match.Result mr = validator.apply(actual);\n                            if (!mr.pass) {\n                                return fail(mr.message);\n                            }\n                        }\n                    } else if (!validatorName.startsWith(REGEX)) { // expected is a string that happens to start with \"#\"\n                        String actualValue = actual.getValue();\n                        switch (type) {\n                            case CONTAINS:\n                                return actualValue.contains(expStr);\n                            default:\n                                return actualValue.equals(expStr);\n                        }\n                    }\n\n                }\n                macro = StringUtils.trimToNull(macro);\n                if (macro != null && questionPos != -1) {\n                    context.JS.put(\"$\", context.root.actual.getValue());\n                    context.JS.put(\"_\", actual.getValue());\n                    JsValue jv = context.JS.eval(macro);\n                    context.JS.bindings.removeMember(\"$\");\n                    context.JS.bindings.removeMember(\"_\");\n                    if (!jv.isTrue()) {\n                        return fail(\"evaluated to 'false'\");\n                    }\n                }\n            }\n        }\n        return true; // all ok\n    }\n    \n    private static final Pattern UNDERSCORE_PATTERN = Pattern.compile(\"\\\\W_\\\\W|\\\\W_|_\\\\W\");\n\n    private boolean containsPlaceholderUnderscore(String bracketContents) {\n        Matcher m1 = UNDERSCORE_PATTERN.matcher(bracketContents);\n        while (m1.find()) {\n            return true;\n        }\n        return false;\n    }\n\n    private boolean actualEqualsExpected() {\n        switch (actual.type) {\n            case NULL:\n                return true; // both are null\n            case BOOLEAN:\n                boolean actBoolean = actual.getValue();\n                boolean expBoolean = expected.getValue();\n                return actBoolean == expBoolean;\n            case NUMBER:\n                if (actual.getValue() instanceof BigDecimal || expected.getValue() instanceof BigDecimal) {\n                    BigDecimal actBigDecimal = toBigDecimal(actual.getValue());\n                    BigDecimal expBigDecimal = toBigDecimal(expected.getValue());\n                    return actBigDecimal.compareTo(expBigDecimal) == 0;\n                } else {\n                    Number actNumber = actual.getValue();\n                    Number expNumber = expected.getValue();\n                    return actNumber.doubleValue() == expNumber.doubleValue();\n                }\n            case STRING:\n                return actual.getValue().equals(expected.getValue());\n            case BYTES:\n                byte[] actBytes = actual.getValue();\n                byte[] expBytes = expected.getValue();\n                return Arrays.equals(actBytes, expBytes);\n            case LIST:\n                List actList = actual.getValue();\n                List expList = expected.getValue();\n                int actListCount = actList.size();\n                int expListCount = expList.size();\n                if (actListCount != expListCount) {\n                    return fail(\"actual array length is not equal to expected - \" + actListCount + \":\" + expListCount);\n                }\n                for (int i = 0; i < actListCount; i++) {\n                    Match.Value actListValue = new Match.Value(actList.get(i));\n                    Match.Value expListValue = new Match.Value(expList.get(i));\n                    MatchOperation mo = new MatchOperation(context.descend(i), Match.Type.EQUALS, actListValue, expListValue, matchEachEmptyAllowed);\n                    mo.execute();\n                    if (!mo.pass) {\n                        return fail(\"array match failed at index \" + i);\n                    }\n                }\n                return true;\n            case MAP:\n                Map<String, Object> actMap = actual.getValue();\n                Map<String, Object> expMap = expected.getValue();\n                return matchMapValues(actMap, expMap);\n            case XML:\n                Map<String, Object> actXml = (Map) XmlUtils.toObject(actual.getValue(), true);\n                Map<String, Object> expXml = (Map) XmlUtils.toObject(expected.getValue(), true);\n                return matchMapValues(actXml, expXml);\n            case OTHER:\n                return actual.getValue().equals(expected.getValue());\n            default:\n                throw new RuntimeException(\"unexpected type (match equals): \" + actual.type);\n        }\n    }\n\n    private boolean matchMapValues(Map<String, Object> actMap, Map<String, Object> expMap) { // combined logic for equals and contains\n        if (actMap.size() > expMap.size() && (type == Match.Type.EQUALS || type == Match.Type.CONTAINS_ONLY || type == Match.Type.CONTAINS_ONLY_DEEP)) {\n            int sizeDiff = actMap.size() - expMap.size();\n            Map<String, Object> diffMap = new LinkedHashMap(actMap);\n            for (String key : expMap.keySet()) {\n                diffMap.remove(key);\n            }\n            return fail(\"actual has \" + sizeDiff + \" more key(s) than expected - \" + JsonUtils.toJson(diffMap));\n        }\n        Set<String> unMatchedKeysAct = new LinkedHashSet(actMap.keySet());\n        Set<String> unMatchedKeysExp = new LinkedHashSet(expMap.keySet());\n        for (Map.Entry<String, Object> expEntry : expMap.entrySet()) {\n            String key = expEntry.getKey();\n            Object childExp = expEntry.getValue();\n            if (!actMap.containsKey(key)) {\n                if (childExp instanceof String) {\n                    String childString = (String) childExp;\n                    if (childString.startsWith(\"##\") || childString.equals(\"#ignore\") || childString.equals(\"#notpresent\")) {\n                        if (type == Match.Type.CONTAINS_ANY || type == Match.Type.CONTAINS_ANY_DEEP) {\n                            return true; // exit early\n                        }\n                        unMatchedKeysExp.remove(key);\n                        if (unMatchedKeysExp.isEmpty()) {\n                            if (type == Match.Type.CONTAINS || type == Match.Type.CONTAINS_DEEP) {\n                                return true; // all expected keys matched\n                            }\n                        }\n                        continue;\n                    }\n                }\n                if (type != Match.Type.CONTAINS_ANY && type != Match.Type.CONTAINS_ANY_DEEP) {\n                    return fail(\"actual does not contain key - '\" + key + \"'\");\n                }\n            }\n            Match.Value childActValue = new Match.Value(actMap.get(key));\n            Match.Type childMatchType;\n            if (type == Match.Type.CONTAINS_DEEP) {\n                childMatchType = childActValue.isMapOrListOrXml() ? Match.Type.CONTAINS_DEEP : Match.Type.EQUALS;\n            } else if (type == Match.Type.CONTAINS_ONLY_DEEP) {\n                childMatchType = childActValue.isMapOrListOrXml() ? Match.Type.CONTAINS_ONLY_DEEP : Match.Type.EQUALS;\n            } else {\n                childMatchType = Match.Type.EQUALS;\n            }\n            MatchOperation mo = new MatchOperation(context.descend(key), childMatchType, childActValue, new Match.Value(childExp), matchEachEmptyAllowed);\n            mo.execute();\n            if (mo.pass) {\n                if (type == Match.Type.CONTAINS_ANY || type == Match.Type.CONTAINS_ANY_DEEP) {\n                    return true; // exit early\n                }\n                unMatchedKeysExp.remove(key);\n                if (unMatchedKeysExp.isEmpty()) {\n                    if (type == Match.Type.CONTAINS || type == Match.Type.CONTAINS_DEEP) {\n                        return true; // all expected keys matched\n                    }\n                }\n                unMatchedKeysAct.remove(key);\n            } else if (type == Match.Type.EQUALS) {\n                return fail(\"match failed for name: '\" + key + \"'\");\n            }\n        }\n        if (type == Match.Type.CONTAINS_ANY || type == Match.Type.CONTAINS_ANY_DEEP) {\n            return unMatchedKeysExp.isEmpty() ? true : fail(\"no key-values matched\");\n        }\n        if (unMatchedKeysExp.isEmpty()) {\n            if (type == Match.Type.CONTAINS || type == Match.Type.CONTAINS_DEEP) {\n                return true; // all expected keys matched, expMap was empty in the first place    \n            }\n            if (type == Match.Type.NOT_CONTAINS && !expMap.isEmpty()) {\n                return true; // hack alert: the NOT_CONTAINS will be reversed by the calling routine\n            }\n        }\n        if (!unMatchedKeysExp.isEmpty()) {\n            return fail(\"all key-values did not match, expected has un-matched keys - \" + unMatchedKeysExp);\n        }\n        if (!unMatchedKeysAct.isEmpty()) {\n            return fail(\"all key-values did not match, actual has un-matched keys - \" + unMatchedKeysAct);\n        }\n        return true;\n    }\n\n    private boolean actualContainsExpected() {\n        switch (actual.type) {\n            case STRING:\n                String actString = actual.getValue();\n                String expString = expected.getValue();\n                return actString.contains(expString);\n            case LIST:\n                List actList = actual.getValue();\n                List expList = expected.getValue();\n                int actListCount = actList.size();\n                int expListCount = expList.size();               \n                // visited array used to handle duplicates\n                boolean[] actVisitedList = new boolean[actListCount];\n                if (type != Match.Type.CONTAINS_ANY && type != Match.Type.CONTAINS_ANY_DEEP && expListCount > actListCount) {\n                    return fail(\"actual array length is less than expected - \" + actListCount + \":\" + expListCount);\n                }\n                if ((type == Match.Type.CONTAINS_ONLY || type == Match.Type.CONTAINS_ONLY_DEEP) && expListCount != actListCount) {\n                    return fail(\"actual array length is not equal to expected - \" + actListCount + \":\" + expListCount);\n                }\n                for (Object exp : expList) { // for each item in the expected list\n                    boolean found = false;\n                    Match.Value expListValue = new Match.Value(exp);\n                    for (int i = 0; i < actListCount; i++) {\n                        Match.Value actListValue = new Match.Value(actList.get(i));\n                        Match.Type childMatchType;\n                        switch (type) {\n                            case CONTAINS_DEEP:\n                                childMatchType = actListValue.isMapOrListOrXml() ? Match.Type.CONTAINS_DEEP : Match.Type.EQUALS;\n                                break;\n                            case CONTAINS_ONLY_DEEP:\n                                childMatchType = actListValue.isMapOrListOrXml() ? Match.Type.CONTAINS_ONLY_DEEP : Match.Type.EQUALS;\n                                break;\n                            case CONTAINS_ANY_DEEP:\n                                childMatchType = actListValue.isMapOrListOrXml() ? Match.Type.CONTAINS_ANY : Match.Type.EQUALS;\n                                break;\n                            default:\n                                childMatchType = Match.Type.EQUALS;\n                        }\n                        MatchOperation mo = new MatchOperation(context.descend(i), childMatchType, actListValue, expListValue, matchEachEmptyAllowed);\n                        mo.execute();\n                        if (mo.pass) {\n                            if (type == Match.Type.CONTAINS_ANY || type == Match.Type.CONTAINS_ANY_DEEP) {\n                                return true; // exit early\n                            }\n                            // contains only : If element is found also check its occurrence in actVisitedList\n                            else if(type == Match.Type.CONTAINS_ONLY) {\n                            \t// if not yet visited\n                            \tif(!actVisitedList[i]) {\n                            \t\t// mark it visited\n                                \tactVisitedList[i]  = true;\n                                \tfound = true;\n                                \tbreak; // next item in expected list\n                            \t}\n                            \t// else do nothing does not consider it a match\n                            }\n                            else {\n                                found = true;\n                                break; // next item in expected list\n                            }\n                        }\n                    }\n                    if (!found && type != Match.Type.CONTAINS_ANY && type != Match.Type.CONTAINS_ANY_DEEP) { // if we reached here, all items in the actual list were scanned\n                        return fail(\"actual array does not contain expected item - \" + expListValue.getAsString());\n                    }\n                }\n                if (type == Match.Type.CONTAINS_ANY || type == Match.Type.CONTAINS_ANY_DEEP) {\n                    return fail(\"actual array does not contain any of the expected items\");\n                }\n                return true; // if we reached here, all items in the expected list were found\n            case MAP:\n                Map<String, Object> actMap = actual.getValue();\n                Map<String, Object> expMap = expected.getValue();\n                return matchMapValues(actMap, expMap);\n            case XML:\n                Map<String, Object> actXml = (Map) XmlUtils.toObject(actual.getValue());\n                Map<String, Object> expXml = (Map) XmlUtils.toObject(expected.getValue());\n                return matchMapValues(actXml, expXml);\n            default:\n                throw new RuntimeException(\"unexpected type (match contains): \" + actual.type);\n        }\n    }\n\n    private static BigDecimal toBigDecimal(Object o) {\n        if (o instanceof BigDecimal) {\n            return (BigDecimal) o;\n        } else if (o instanceof Number) {\n            Number n = (Number) o;\n            return BigDecimal.valueOf(n.doubleValue());\n        } else {\n            throw new RuntimeException(\"expected number instead of: \" + o);\n        }\n    }\n\n    private boolean pass() {\n        pass = true;\n        return true;\n    }\n\n    private boolean fail(String reason) {\n        pass = false;\n        if (reason == null) {\n            return false;\n        }\n        failReason = failReason == null ? reason : reason + \" | \" + failReason;\n        context.root.failures.add(this);\n        return false;\n    }\n\n    String getFailureReasons() {\n        return collectFailureReasons(this);\n    }\n\n    private boolean isXmlAttributeOrMap() {\n        return context.xml && actual.isMap()\n                && (context.name.equals(\"@\") || actual.<Map>getValue().containsKey(\"_\"));\n    }\n\n    private static String collectFailureReasons(MatchOperation root) {\n        StringBuilder sb = new StringBuilder();\n        sb.append(\"match failed: \").append(root.type).append('\\n');\n        Collections.reverse(root.failures);\n        Iterator<MatchOperation> iterator = root.failures.iterator();\n        Set previousPaths = new HashSet();\n        int index = 0;\n        int prevDepth = -1;\n        while (iterator.hasNext()) {\n            MatchOperation mo = iterator.next();\n            if (previousPaths.contains(mo.context.path) || mo.isXmlAttributeOrMap()) {\n                continue;\n            }\n            previousPaths.add(mo.context.path);\n            if (mo.context.depth != prevDepth) {\n                prevDepth = mo.context.depth;\n                index++;\n            }\n            String prefix = StringUtils.repeat(' ', index * 2);\n            sb.append(prefix).append(mo.context.path).append(\" | \").append(mo.failReason);\n            sb.append(\" (\").append(mo.actual.type).append(':').append(mo.expected.type).append(\")\");\n            sb.append('\\n');\n            if (mo.context.xml) {\n                sb.append(prefix).append(mo.actual.getAsXmlString()).append('\\n');\n                sb.append(prefix).append(mo.expected.getAsXmlString()).append('\\n');\n            } else {\n                Match.Value expected = mo.expected.getSortedLike(mo.actual);\n                sb.append(prefix).append(mo.actual.getWithinSingleQuotesIfString()).append('\\n');\n                sb.append(prefix).append(expected.getWithinSingleQuotesIfString()).append('\\n');\n            }\n            if (iterator.hasNext()) {\n                sb.append('\\n');\n            }\n        }\n        return sb.toString();\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/PerfContext.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate;\n\n/**\n *\n * @author pthomas3\n */\npublic interface PerfContext {\n    \n    void capturePerfEvent(String name, long startTime, long endTime);\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/KarateOptions.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate;\n\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\n\n/**\n * deprecated. if you are on JUnit 4, please switch to JUnit 5\n * @see com.intuit.karate.Runner#builder()\n */\n@Deprecated\n@Retention(RetentionPolicy.RUNTIME)\n@Target({ElementType.TYPE})\npublic @interface KarateOptions {    \n\n    String[] features() default {};\n\n    String[] tags() default {};\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/MissingFinder.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\n/**\n *\n * @author pthomas3\n */\npublic class MissingFinder implements Finder {\n    \n    private final MissingElement element;\n    \n    public MissingFinder(MissingElement element) {\n        this.element = element;\n    }\n\n    @Override\n    public Element input(String value) {\n        return element;\n    }\n\n    @Override\n    public Element select(String value) {\n        return element;\n    }\n\n    @Override\n    public Element select(int index) {\n        return element;\n    }\n\n    @Override\n    public Element click() {\n        return element;\n    }\n\n    @Override\n    public String getValue() {\n        return element.getValue();\n    }        \n\n    @Override\n    public Element clear() {\n        return element;\n    }\n\n    @Override\n    public Element find() {\n        return element;\n    }\n\n    @Override\n    public Element find(String tag) {\n        return element;\n    }\n\n    @Override\n    public Element highlight() {\n        return element;\n    }\n\n    @Override\n    public Element retry() {\n        return element;\n    }\n\n    @Override\n    public Element retry(int count) {\n        return element;\n    }\n\n    @Override\n    public Element retry(Integer count, Integer interval) {\n        return element;\n    }\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/Frame.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\n/**\n *\n * @author pthomas3\n */\npublic class Frame {\n\n    public final String id;\n    public final String url;\n    public final String name;\n\n    public Frame(String id, String url, String name) {\n        this.id = id;\n        this.url = url;\n        this.name = name;\n    }\n\n    @Override\n    public String toString() {\n        StringBuilder sb = new StringBuilder();\n        sb.append(\"[id: \").append(id);\n        sb.append(\", url: \").append(url);\n        sb.append(\", name: \").append(name);\n        sb.append(\"]\");\n        return sb.toString();\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/MissingElement.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class MissingElement implements Element {\n\n    private final Driver driver;\n    private final String locator;\n\n    public MissingElement(Driver driver, String locator) {\n        this.driver = driver;\n        this.locator = locator;\n    }\n\n    @Override\n    public String getLocator() {\n        return locator;\n    }\n\n    @Override\n    public boolean isPresent() {\n        return false;\n    }\n\n    @Override\n    public boolean isEnabled() {\n        return false;\n    }\n\n    @Override\n    public Map<String, Object> getPosition() {\n        return null;\n    }\n\n    @Override\n    public byte[] screenshot() {\n        return null;\n    }\n\n    @Override\n    public Element highlight() {\n        return this;\n    }\n\n    @Override\n    public Element focus() {\n        return this;\n    }\n\n    @Override\n    public Element clear() {\n        return this;\n    }\n\n    @Override\n    public Element click() {\n        return this;\n    }\n\n    @Override\n    public Element submit() {\n        return this;\n    }\n\n    @Override\n    public Element scroll() {\n        return this;\n    }        \n\n    @Override\n    public Mouse mouse() {\n        return null;\n    }\n\n    @Override\n    public Element input(String text) {\n        return this;\n    }\n\n    @Override\n    public Element input(String[] values) {\n        return this;\n    }\n\n    @Override\n    public Element input(String[] values, int delay) {\n        return this;\n    }\n\n    @Override\n    public Element select(String text) {\n        return this;\n    }\n\n    @Override\n    public Element select(int index) {\n        return this;\n    }\n\n    @Override\n    public Element switchFrame() {\n        return this;\n    }\n\n    @Override\n    public Element delay(int millis) {\n        driver.delay(millis);\n        return this;\n    }\n\n    @Override\n    public Element retry() {\n        return this;\n    }\n\n    @Override\n    public Element retry(int count) {\n        return this;\n    }\n\n    @Override\n    public Element retry(Integer count, Integer interval) {\n        return this;\n    }\n\n    @Override\n    public Element waitFor() {\n        return this;\n    }\n\n    @Override\n    public Element waitForText(String text) {\n        return this;\n    }\n\n    @Override\n    public Element waitUntil(String expression) {\n        return this;\n    }\n\n    @Override\n    public Object script(String expression) {\n        return null;\n    }\n    \n    @Override\n    public Object scriptAll(String locator, String expression) {\n        return null;\n    }    \n\n    @Override\n    public Element optional(String locator) {\n        return new MissingElement(driver, locator);\n    }\n\n    @Override\n    public boolean exists(String locator) {\n        return false;\n    }\n\n    @Override\n    public Element locate(String locator) {\n        return new MissingElement(driver, locator);\n    }\n\n    @Override\n    public List<Element> locateAll(String locator) {\n        return Collections.EMPTY_LIST;\n    }\n\n    @Override\n    public String attribute(String name) {\n        return null;\n    }\n\n    @Override\n    public String property(String name) {\n        return null;\n    }\n\n    @Override\n    public String getHtml() {\n        return null;\n    }\n\n    @Override\n    public void setHtml(String html) {\n\n    }\n\n    @Override\n    public String getText() {\n        return null;\n    }\n\n    @Override\n    public void setText(String text) {\n\n    }\n\n    @Override\n    public String getValue() {\n        return null;\n    }\n\n    @Override\n    public void setValue(String value) {\n\n    }\n\n    @Override\n    public Element getParent() {\n        return this;\n    }\n\n    @Override\n    public List<Element> getChildren() {\n        return Collections.EMPTY_LIST;\n    }\n\n    @Override\n    public Element getFirstChild() {\n        return this;\n    }\n\n    @Override\n    public Element getLastChild() {\n        return this;\n    }\n\n    @Override\n    public Element getPreviousSibling() {\n        return this;\n    }\n\n    @Override\n    public Element getNextSibling() {\n        return this;\n    }\n\n    @Override\n    public Finder rightOf() {\n        return new MissingFinder(this);\n    }\n\n    @Override\n    public Finder leftOf() {\n        return new MissingFinder(this);\n    }\n\n    @Override\n    public Finder above() {\n        return new MissingFinder(this);\n    }\n\n    @Override\n    public Finder below() {\n        return new MissingFinder(this);\n    }\n\n    @Override\n    public Finder near() {\n        return new MissingFinder(this);\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/DevToolsMessage.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.Json;\nimport com.intuit.karate.core.Variable;\nimport java.util.HashMap;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.function.Predicate;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author pthomas3\n */\npublic class DevToolsMessage {\n\n    private static final Logger logger = LoggerFactory.getLogger(DevToolsMessage.class);\n\n    protected final DevToolsDriver driver;\n\n    private final String sessionId;\n    private final String method;\n\n    private Integer id;\n    private Json params;\n    private Map<String, Object> error;\n    private Variable result;\n    private Integer timeout;        \n\n    public Integer getId() {\n        return id;\n    }\n\n    public void setId(Integer id) {\n        this.id = id;\n    }\n\n    public Integer getTimeout() {\n        return timeout;\n    }\n\n    public void setTimeout(Integer timeout) {\n        this.timeout = timeout;\n    }\n\n    public String getMethod() {\n        return method;\n    }\n\n    public boolean methodIs(String method) {\n        return method.equals(this.method);\n    }\n\n    public <T> T getParam(String path) {\n        if (params == null) {\n            return null;\n        }\n        try {\n            return params.get(path);\n        } catch (Exception e) {\n            if (logger.isTraceEnabled()) {\n                logger.trace(\"get param - json path failed: {} - {}\", path, params);\n            }\n            return null;\n        }\n    }\n\n    public Variable getResult() {\n        return result == null ? Variable.NULL : result;\n    }\n\n    public <T> T getResult(String path) {\n        if (result == null || result.isNull()) {\n            return null;\n        }\n        Json json = Json.of(result.getValue());\n        return json.get(path);\n    }\n\n    public void setResult(Variable result) {\n        this.result = result;\n    }\n\n    private static Map<String, Object> toMap(List<Map<String, Object>> list) {\n        Map<String, Object> res = new HashMap();\n        for (Map<String, Object> map : list) {\n            String key = (String) map.get(\"name\");\n            Map<String, Object> valMap = (Map) map.get(\"value\");\n            res.put(key, valMap == null ? null : valMap.get(\"value\"));\n        }\n        return res;\n    }\n\n    public boolean isResultError() {\n        if (error != null) {\n            return true;\n        }\n        if (result == null || !result.isMap()) {\n            return false;\n        }\n        String resultError = (String) result.<Map>getValue().get(\"subtype\");\n        return \"error\".equals(resultError);\n    }\n\n    public Variable getResultVariable(String key) {\n        if (result == null || !result.isMap()) {\n            return null;\n        }\n        return new Variable(result.<Map>getValue().get(key));\n    }\n\n    public Map<String, Object> getError() {\n        return error;\n    }        \n\n    public DevToolsMessage(DevToolsDriver driver, String method) {\n        this.driver = driver;\n        id = driver.nextId();\n        this.method = method;\n        sessionId = driver.sessionId;\n    }\n\n    public DevToolsMessage(DevToolsDriver driver, Map<String, Object> map) {\n        this.driver = driver;\n        sessionId = (String) map.get(\"sessionId\");;\n        id = (Integer) map.get(\"id\");\n        method = (String) map.get(\"method\");\n        Map temp = (Map) map.get(\"params\");\n        if (temp != null) {\n            params = Json.of(temp);\n        }\n        temp = (Map) map.get(\"result\");\n        if (temp != null) {\n            if (temp.containsKey(\"result\")) {\n                Object inner = temp.get(\"result\");\n                if (inner instanceof List) {\n                    result = new Variable(toMap((List) inner));\n                } else {\n                    Map innerMap = (Map) inner;\n                    String subtype = (String) innerMap.get(\"subtype\");\n                    if (\"error\".equals(subtype) || innerMap.containsKey(\"objectId\")) {\n                        result = new Variable(innerMap);\n                    } else { // Runtime.evaluate \"returnByValue\" is true\n                        result = new Variable(innerMap.get(\"value\"));\n                    }\n                }\n            } else {\n                result = new Variable(temp);\n            }\n        }\n        error = (Map) map.get(\"error\");\n    }\n\n    public DevToolsMessage param(String path, Object value) {\n        if (params == null) {\n            params = Json.object();\n        }\n        params.set(path, value);\n        return this;\n    }\n\n    public DevToolsMessage params(Map<String, Object> map) {\n        this.params = Json.of(map);\n        return this;\n    }\n\n    public Map<String, Object> toMap() {\n        Map<String, Object> map = new LinkedHashMap(4);\n        map.put(\"id\", id);\n        if (sessionId != null) {\n            map.put(\"sessionId\", sessionId);\n        }\n        map.put(\"method\", method);\n        if (params != null) {\n            map.put(\"params\", params.value());\n        }\n        if (result != null) {\n            map.put(\"result\", result.getValue());\n        }\n        return map;\n    }\n\n    public void sendWithoutWaiting() {\n        driver.send(this);\n    }\n\n    public DevToolsMessage send() {\n        return send(null);\n    }\n\n    public DevToolsMessage send(Predicate<DevToolsMessage> condition) {\n        return driver.sendAndWait(this, condition);\n    }\n\n    @Override\n    public String toString() {\n        StringBuilder sb = new StringBuilder();\n...\n", "needle_token_start": 782, "needle_token_end": 844, "code_context_ntokens": 16055, "output": ["```java\nprivate static int min(int a, int b) {\n    if (a == -1) {\n        return b;\n    }\n    if (b == -1) {\n        return a;\n    }\n    return Math.min(a, b);\n}\n```"]}
{"repo": "karatelabs/karate", "name": "createNodeByPath", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/XmlUtils.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To dynamically create and append XML nodes to a document based on a specified path, ensuring the structure matches the path even if parts of it do not exist initially.\n2. **Input**: A document object representing the XML structure and a string representing the hierarchical path where the node should be created.\n3. **Output**: The newly created node at the specified path in the document.\n4. **Procedure**: \n   - The function first checks if the path points directly to the root. If so, it either returns the existing root or creates a new root element.\n   - If the path is deeper, it splits the path to identify the parent path and the target node.\n   - It then recursively ensures that the parent node exists by calling a related function.\n   - Depending on whether the target in the path is an attribute or an element, it either creates an attribute in the parent node or appends a new element to the parent.\n   - If the target node is an element and includes an index (though indexing is assumed to be 1 for simplicity), it still proceeds to create and append the element.\n   - The newly created node or attribute is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: karate-core/src/main/java/com/intuit/karate/Json.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n...\n// Path: karate-core/src/main/java/com/intuit/karate/XmlUtils.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate;\n\nimport com.jayway.jsonpath.DocumentContext;\nimport com.jayway.jsonpath.JsonPath;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.StringReader;\nimport java.io.StringWriter;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport javax.xml.parsers.DocumentBuilder;\nimport javax.xml.parsers.DocumentBuilderFactory;\nimport javax.xml.transform.OutputKeys;\nimport javax.xml.transform.Transformer;\nimport javax.xml.transform.TransformerFactory;\nimport javax.xml.transform.dom.DOMSource;\nimport javax.xml.transform.stream.StreamResult;\nimport javax.xml.xpath.XPath;\nimport javax.xml.xpath.XPathConstants;\nimport javax.xml.xpath.XPathExpression;\nimport javax.xml.xpath.XPathExpressionException;\nimport javax.xml.xpath.XPathFactory;\n\nimport org.w3c.dom.Attr;\nimport org.w3c.dom.Document;\nimport org.w3c.dom.Element;\nimport org.w3c.dom.NamedNodeMap;\nimport org.w3c.dom.Node;\nimport org.w3c.dom.NodeList;\nimport org.xml.sax.EntityResolver;\nimport org.xml.sax.InputSource;\nimport org.xml.sax.SAXException;\n\n/**\n * @author pthomas3\n */\npublic class XmlUtils {\n\n    private XmlUtils() {\n        // only static methods\n    }\n\n    public static String toString(Node node) {\n        return toString(node, false);\n    }\n\n    public static String toString(Node node, boolean pretty) {\n        Node nodeToSerialize = node;\n        // In case of pretty string, we clone the node so that we don't modify the original node while trimming whitespaces\n        if (pretty) {\n            nodeToSerialize = node.cloneNode(true);\n            trimWhiteSpace(nodeToSerialize);\n        }\n        DOMSource domSource = new DOMSource(nodeToSerialize);\n        StringWriter writer = new StringWriter();\n        StreamResult result = new StreamResult(writer);\n        TransformerFactory tf = TransformerFactory.newInstance();\n        try {\n            Transformer transformer = tf.newTransformer();\n            transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, \"yes\");\n            if (pretty) {\n                transformer.setOutputProperty(OutputKeys.INDENT, \"yes\");\n                transformer.setOutputProperty(\"{http://xml.apache.org/xslt}indent-amount\", \"2\");\n            } else {\n                transformer.setOutputProperty(OutputKeys.INDENT, \"no\");\n            }\n            transformer.transform(domSource, result);\n            return writer.toString();\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static void trimWhiteSpace(Node node) {\n        NodeList children = node.getChildNodes();\n        int count = children.getLength();\n        for (int i = 0; i < count; ++i) {\n            Node child = children.item(i);\n            if (child.getNodeType() == Node.TEXT_NODE) {\n                child.setTextContent(child.getTextContent() == null ? \"\" : child.getTextContent().trim());\n            }\n            trimWhiteSpace(child);\n        }\n    }\n\n    private static class DtdEntityResolver implements EntityResolver {\n\n        protected boolean dtdPresent;\n\n        @Override\n        public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {\n            dtdPresent = true;\n            return new InputSource(new StringReader(\"\"));\n        }\n\n    }\n    \n    public static Document toXmlDoc(String xml) {\n        return toXmlDoc(xml, false);\n    }\n\n    public static Document toXmlDoc(String xml, boolean namespaceAware) {\n        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();\n        factory.setNamespaceAware(namespaceAware);\n        try {\n            DocumentBuilder builder = factory.newDocumentBuilder();\n            DtdEntityResolver dtdEntityResolver = new DtdEntityResolver();\n            builder.setEntityResolver(dtdEntityResolver);\n            InputStream is = FileUtils.toInputStream(xml);\n            Document doc = builder.parse(is);\n            if (dtdEntityResolver.dtdPresent) { // DOCTYPE present\n                // the XML was not parsed, but I think it hangs at the root as a text node\n                // so conversion to string and back has the effect of discarding the DOCTYPE !\n                return toXmlDoc(toString(doc, false), namespaceAware);\n            } else {\n                return doc;\n            }\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    private static XPathExpression compile(String path) {\n        XPathFactory xPathfactory = XPathFactory.newInstance();\n        XPath xpath = xPathfactory.newXPath();\n        try {\n            return xpath.compile(path);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static NodeList getNodeListByPath(Node node, String path) {\n        XPathExpression expr = compile(path);\n        try {\n            return (NodeList) expr.evaluate(node, XPathConstants.NODESET);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static String stripNameSpacePrefixes(String path) {\n        if (path.indexOf(':') == -1) {\n            return path;\n        }\n        StringBuilder sb = new StringBuilder();\n        for (String s : StringUtils.split(path, '/', false)) {\n            sb.append('/');\n            int pos = s.lastIndexOf(':');\n            if (pos == -1) {\n                sb.append(s);\n            } else {\n                sb.append(s.substring(pos + 1));\n            }\n        }\n        return sb.toString();\n    }\n\n    public static Node getNodeByPath(Node node, String path, boolean create) {\n        String searchPath = create ? stripNameSpacePrefixes(path) : path;\n        XPathExpression expr = compile(searchPath);\n        Node result;\n        try {\n            result = (Node) expr.evaluate(node, XPathConstants.NODE);\n        } catch (XPathExpressionException e) {\n            throw new RuntimeException(e);\n        }\n        if (result == null && create) {\n            Document doc = node.getNodeType() == Node.DOCUMENT_NODE ? (Document) node : node.getOwnerDocument();\n            return createNodeByPath(doc, path);\n        } else {\n            return result;\n        }\n    }\n\n    \npublic static Node createNodeByPath(Document doc, String path) {\n        int pos = path.lastIndexOf('/');\n        if (pos == 0) { // root\n            Node root = doc.getDocumentElement();\n            if (root == null) {\n                root = createElement(doc, path.substring(1), null, null);\n                doc.appendChild(root);\n            }\n            return root;\n        }\n        String left = path.substring(0, pos);\n        Node parent = getNodeByPath(doc, left, true);\n        String right = path.substring(pos + 1);\n        if (right.startsWith(\"@\")) { // attribute\n            Element parentElement = (Element) parent;\n            right = right.substring(1);\n            parentElement.setAttribute(right, \"\");\n            return parentElement.getAttributeNode(right);\n        } else {\n            int bracketPos = right.indexOf('[');\n            if (bracketPos != -1) { // index, we assume it is 1 and still append\n                right = right.substring(0, bracketPos);\n            }\n            Element element = createElement(parent, right, null, null);\n            parent.appendChild(element);\n            return element;\n        }\n    }\n\n    public static String getTextValueByPath(Node node, String path) {\n        XPathExpression expr = compile(path);\n        try {\n            return expr.evaluate(node);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static void setByPath(Node doc, String path, String value) {\n        Node node = getNodeByPath(doc, path, true);\n        if (node.getNodeType() == Node.ATTRIBUTE_NODE) {\n            node.setNodeValue(value);\n        } else if (node.hasChildNodes() && node.getFirstChild().getNodeType() == Node.TEXT_NODE) {\n            node.getFirstChild().setTextContent(value);\n        } else if (node.getNodeType() == Node.ELEMENT_NODE) {\n            node.setTextContent(value);\n        }\n    }\n\n    public static void removeByPath(Document doc, String path) {\n        Node node = getNodeByPath(doc, path, false);\n        if (node == null) {\n            return;\n        }\n        if (node.getNodeType() == Node.ATTRIBUTE_NODE) {\n            Element parent = ((Attr) node).getOwnerElement();\n            parent.removeAttribute(node.getNodeName());\n        } else {\n            Node parent = node.getParentNode();\n            if (parent != null) {\n                parent.removeChild(node);\n            }\n        }\n    }\n\n    public static void setByPath(Document doc, String path, Node in) {\n        if (in.getNodeType() == Node.DOCUMENT_NODE) {\n            in = in.getFirstChild();\n        }\n        Node node = getNodeByPath(doc, path, true);\n        if (node == null) {\n            throw new RuntimeException(\"no results for xpath: \" + path);\n        }\n        Node newNode = doc.importNode(in, true);\n        if (node.hasChildNodes() && node.getFirstChild().getNodeType() == Node.TEXT_NODE) {\n            node.replaceChild(newNode, node.getFirstChild());\n        } else {\n            node.appendChild(newNode);\n        }\n    }\n\n    public static DocumentContext toJsonDoc(Node node) {\n        return JsonPath.parse(toObject(node));\n    }\n\n    private static Map<String, Object> getAttributes(Node node) {\n        NamedNodeMap attribs = node.getAttributes();\n        int attribCount = attribs.getLength();\n        Map<String, Object> map = new LinkedHashMap<>(attribCount);\n        for (int j = 0; j < attribCount; j++) {\n            Node attrib = attribs.item(j);\n            map.put(attrib.getNodeName(), attrib.getNodeValue());\n        }\n        return map;\n    }\n\n    public static int getChildElementCount(Node node) {\n        NodeList nodes = node.getChildNodes();\n        int childCount = nodes.getLength();\n        int childElementCount = 0;\n        for (int i = 0; i < childCount; i++) {\n            Node child = nodes.item(i);\n            if (child.getNodeType() == Node.ELEMENT_NODE) {\n                childElementCount++;\n            }\n        }\n        return childElementCount;\n    }\n\n    private static Object getElementAsObject(Node node, boolean removeNamespace) {\n        int childElementCount = getChildElementCount(node);\n        if (childElementCount == 0) {\n            return StringUtils.trimToNull(node.getTextContent());\n        }\n        Map<String, Object> map = new LinkedHashMap<>(childElementCount);\n        NodeList nodes = node.getChildNodes();\n        int childCount = nodes.getLength();\n        for (int i = 0; i < childCount; i++) {\n            Node child = nodes.item(i);\n            if (child.getNodeType() != Node.ELEMENT_NODE) {\n                continue;\n            }\n            String childName = removeNamespace\n                    ? child.getNodeName().replaceFirst(\"(^.*:)\", \"\") : child.getNodeName();\n            Object childValue = toObject(child, removeNamespace);\n            // auto detect repeating elements\n            if (map.containsKey(childName)) {\n                Object temp = map.get(childName);\n                if (temp instanceof List) {\n                    List list = (List) temp;\n                    list.add(childValue);\n                } else {\n                    List list = new ArrayList(childCount);\n                    map.put(childName, list);\n                    list.add(temp);\n                    list.add(childValue);\n                }\n            } else {\n                map.put(childName, childValue);\n            }\n        }\n        return map;\n    }\n\n    public static Object toObject(Node node) {\n        return toObject(node, false);\n    }\n\n    public static Object toObject(Node node, boolean removeNamespace) {\n        if (node.getNodeType() == Node.DOCUMENT_NODE) {\n            Map<String, Object> map = new LinkedHashMap<>(1);\n            node = node.getFirstChild();\n            if (node == null) {\n                return map;\n            }\n            while (node.getNodeType() != Node.ELEMENT_NODE) { // ignore comments etc\n                node = node.getNextSibling();\n            }\n            String name = removeNamespace\n                    ? node.getNodeName().replaceFirst(\"(^.*:)\", \"\") : node.getNodeName();\n            map.put(name, toObject(node, removeNamespace));\n            return map;\n        }\n        Object value = getElementAsObject(node, removeNamespace);\n        if (node.hasAttributes()) {\n            Map<String, Object> attribs = getAttributes(node);\n            if (removeNamespace) {\n                attribs.keySet().removeIf(key -> \"xmlns\".equals(key) || key.startsWith(\"xmlns:\"));\n            }\n            if (attribs.size() > 0) {\n                Map<String, Object> wrapper = new LinkedHashMap<>(2);\n                wrapper.put(\"_\", value);\n                wrapper.put(\"@\", attribs);\n                return wrapper;\n            } else {\n                //namespaces were the only attributes\n                return value;\n            }\n        } else {\n            return value;\n        }\n    }\n\n    public static Document fromMap(Map<String, Object> map) {\n        Map.Entry<String, Object> first = map.entrySet().iterator().next();\n        return fromObject(first.getKey(), first.getValue());\n    }\n\n    public static Document fromObject(String name, Object o) {\n        Document doc = newDocument();\n        List<Element> list = fromObject(doc, name, o);\n        Element root = list.get(0);\n        doc.appendChild(root);\n        return doc;\n    }\n\n    public static List<Element> fromObject(Document doc, String name, Object o) {\n        if (o instanceof Map) {\n            Map<String, Object> map = (Map) o;\n            Map<String, Object> attribs = (Map) map.get(\"@\");\n            Object value = map.get(\"_\");\n            if (value != null || attribs != null) {\n                List<Element> elements = fromObject(doc, name, value);\n                addAttributes(elements.get(0), attribs);\n                return elements;\n            } else {\n                Element element = createElement(doc, name, null, null);\n                for (Map.Entry<String, Object> entry : map.entrySet()) {\n                    String childName = entry.getKey();\n                    Object childValue = entry.getValue();\n                    List<Element> childNodes = fromObject(doc, childName, childValue);\n                    for (Element e : childNodes) {\n                        element.appendChild(e);\n                    }\n                }\n                return Collections.singletonList(element);\n            }\n        } else if (o instanceof List) {\n            List list = (List) o;\n            List<Element> elements = new ArrayList(list.size());\n            for (Object child : list) {\n                List<Element> childNodes = fromObject(doc, name, child);\n                for (Element e : childNodes) {\n                    elements.add(e);\n                }\n            }\n            return elements;\n        } else {\n            String value = o == null ? null : o.toString();\n            Element element = createElement(doc, name, value, null);\n            return Collections.singletonList(element);\n        }\n    }\n\n    public static Document newDocument() {\n        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();\n        DocumentBuilder builder;\n        try {\n            builder = factory.newDocumentBuilder();\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n        return builder.newDocument();\n    }\n\n    public static void addAttributes(Element element, Map<String, Object> map) {\n        if (map != null) {\n            for (Map.Entry<String, Object> entry : map.entrySet()) {\n                Object attrValue = entry.getValue();\n                element.setAttribute(entry.getKey(), attrValue == null ? null : attrValue.toString());\n            }\n        }\n    }\n\n    public static Element createElement(Node node, String name, String value, Map<String, Object> attributes) {\n        Document doc = node.getNodeType() == Node.DOCUMENT_NODE ? (Document) node : node.getOwnerDocument();\n        Element element = doc.createElement(name);\n        element.setTextContent(value);\n        addAttributes(element, attributes);\n        return element;\n    }\n\n    public static Document toNewDocument(Node in) {\n        Document doc = newDocument();\n        Node node = doc.importNode(in, true);\n        doc.appendChild(node);\n        return doc;\n    }\n\n    public static Document fromJavaObject(Object o) {\n        return fromObject(\"root\", Json.of(o).value()); // keep it simple for people to write generic xpath starting with /root\n    }\n\n    public static String toXml(Object o) {\n        return toString(fromJavaObject(o));\n    }\n\n    public static boolean isXml(String s) {\n        if (s == null || s.isEmpty()) {\n            return false;\n        }\n        if (s.charAt(0) == ' ') {\n            s = s.trim();\n            if (s.isEmpty()) {\n                return false;\n            }\n        }\n        return s.charAt(0) == '<';\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/ResourceType.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport org.w3c.dom.Node;\n\n/**\n *\n * @author pthomas3\n */\npublic enum ResourceType {\n\n    DEFERRED_JS(\"text/javascript\", vals(\"javascript\"), vals(\"js\")),\n    JS(\"text/javascript\", vals(\"javascript\"), vals(\"js\")),\n    JSON(\"application/json\", vals(\"json\"), vals(\"json\")),\n    CSS(\"text/css\", vals(\"css\"), vals(\"css\")),\n    ICO(\"image/x-icon\", vals(\"x-icon\"), vals(\"ico\")),\n    PNG(\"image/png\", vals(\"png\"), vals(\"png\")),\n    GIF(\"image/gif\", vals(\"gif\"), vals(\"gif\")),\n    JPG(\"image/jpeg\", vals(\"jpeg\", \"jpg\"), vals(\"jpg\", \"jpeg\")),\n    SVG(\"image/svg+xml\", vals(\"svg\"), vals(\"svg\")),\n    MP4(\"video/mp4\", vals(\"mp4\"), vals(\"mp4\")),\n    PDF(\"application/pdf\", vals(\"pdf\"), vals(\"pdf\")),\n    HTML(\"text/html\", vals(\"html\"), vals(\"html\", \"htm\")),\n    XML(\"application/xml\", vals(\"xml\"), vals(\"xml\")),\n    TEXT(\"text/plain\", vals(\"plain\"), vals(\"txt\")),\n    WOFF2(\"application/font-woff2\", vals(\"woff2\"), vals(\"woff2\")),\n    MULTIPART(\"multipart/form-data\", vals(\"multipart\"), vals()),    \n    URLENCODED(\"application/x-www-form-urlencoded\", vals(\"urlencoded\"), vals()),\n    BINARY(\"application/octet-stream\", vals(\"octet\"), vals()),\n    RDFXML(\"application/rdf+xml\", vals(\"rdf\", \"rdf+xml\"), vals(\"rdf\")),\n    NTRIPLES(\"application/n-triples\", vals(\"triples\"), vals(\"nt\")),\n    TURTLE(\"text/turtle\", vals(\"turtle\"), vals(\"ttl\")),\n    NQUADS(\"application/n-quads\", vals(\"quads\"), vals(\"nq\")),\n    TRIG(\"application/trig\", vals(\"trig\"), vals(\"trig\")),\n    N3(\"text/n3\", vals(\"n3\"), vals(\"n3\")),\n    JSONLD(\"application/ld+json\", vals(\"ld+json\"), vals(\"jsonld\"));\n\n    private static String[] vals(String... values) {\n        return values;\n    }\n\n    public final String contentType;\n    public final String[] contentLike;\n    public final String[] extensions;\n\n    ResourceType(String contentType, String[] contentLike, String[] extensions) {\n        this.contentType = contentType;\n        this.contentLike = contentLike;\n        this.extensions = extensions;\n    }\n\n    private static final Map<String, ResourceType> EXTENSION_MAP = new HashMap();\n\n    static {\n        for (ResourceType rt : ResourceType.values()) {\n            for (String ext : rt.extensions) {\n                EXTENSION_MAP.put(ext, rt);\n            }\n        }\n    }\n\n    public static ResourceType fromFileExtension(String path) {\n        if (path == null) {\n            return null;\n        }\n        int pos = path.lastIndexOf('.');\n        if (pos == -1 || pos == path.length() - 1) {\n            return null;\n        }\n        String extension = path.substring(pos + 1).trim().toLowerCase();\n        return EXTENSION_MAP.get(extension);\n    }\n\n    public String getExtension() {\n        return extensions.length == 0 ? null : extensions[0];\n    }\n\n    public boolean isVideo() {\n        switch (this) {\n            case MP4:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public boolean isImage() {\n        switch (this) {\n            case BINARY:\n            case ICO:\n            case PNG:\n            case GIF:\n            case JPG:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public boolean isUrlEncodedOrMultipart() {\n        switch (this) {\n            case URLENCODED:\n            case MULTIPART:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public boolean isHtml() {\n        return this == HTML;\n    }\n\n    public boolean isJson() {\n        switch (this) {\n            case JSON:\n            case JSONLD:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public boolean isXml() {\n        switch (this) {\n            case XML:\n            case RDFXML:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public boolean isText() {\n        return this == TEXT;\n    }\n\n    public boolean isBinary() {\n        switch (this) {\n            case BINARY:\n            case ICO:\n            case PNG:\n            case GIF:\n            case JPG:\n            case PDF:\n            case MP4:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public boolean isScript() {\n        switch (this){\n            case JS:\n            case DEFERRED_JS:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public static ResourceType fromContentType(String ct) {\n        if (ct == null) {\n            return null;\n        }\n        ct = ct.toLowerCase();\n        for (ResourceType rt : ResourceType.values()) {\n            if (ct.equals(rt.contentType)) {\n                return rt;\n            }\n            for (String like : rt.contentLike) {\n                if (ct.contains(like)) {\n                    return rt;\n                }\n            }\n        }\n        return null;\n    }\n\n    public static ResourceType fromObject(Object o) {\n        return fromObject(o, null);\n    }\n\n    public static ResourceType fromObject(Object o, ResourceType defaultType) {\n        if (o instanceof List || o instanceof Map) {\n            return JSON;\n        } else if (o instanceof String) {\n            return TEXT;\n        } else if (o instanceof Node) {\n            return XML;\n        } else if (o instanceof byte[]) {\n            return BINARY;\n        } else {\n            return defaultType;\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/JsonUtils.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate;\n\nimport com.intuit.karate.http.ResourceType;\nimport com.jayway.jsonpath.Configuration;\nimport com.jayway.jsonpath.Option;\nimport com.jayway.jsonpath.spi.json.JsonProvider;\nimport com.jayway.jsonpath.spi.json.JsonSmartJsonProvider;\nimport com.jayway.jsonpath.spi.mapper.JsonSmartMappingProvider;\nimport com.jayway.jsonpath.spi.mapper.MappingProvider;\nimport de.siegmar.fastcsv.reader.CsvReader;\nimport de.siegmar.fastcsv.reader.CsvRow;\nimport de.siegmar.fastcsv.writer.CsvWriter;\nimport java.io.StringWriter;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.EnumSet;\nimport java.util.IdentityHashMap;\nimport java.util.Iterator;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport net.minidev.json.JSONStyle;\nimport net.minidev.json.JSONValue;\nimport net.minidev.json.parser.JSONParser;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.yaml.snakeyaml.LoaderOptions;\nimport org.yaml.snakeyaml.Yaml;\nimport org.yaml.snakeyaml.constructor.SafeConstructor;\n\nimport static net.minidev.json.JSONValue.defaultReader;\nimport org.w3c.dom.Node;\n\n/**\n *\n * @author pthomas3\n */\npublic class JsonUtils {\n\n    private static final Logger logger = LoggerFactory.getLogger(JsonUtils.class);\n\n    private JsonUtils() {\n        // only static methods\n    }\n\n    static {\n        // ensure that even if jackson (databind?) is on the classpath, don't switch provider\n        Configuration.setDefaults(new Configuration.Defaults() {\n            private final JsonProvider jsonProvider = new JsonSmartJsonProvider();\n            private final MappingProvider mappingProvider = new JsonSmartMappingProvider();\n\n            @Override\n            public JsonProvider jsonProvider() {\n                return jsonProvider;\n            }\n\n            @Override\n            public MappingProvider mappingProvider() {\n                return mappingProvider;\n            }\n\n            @Override\n            public Set<Option> options() {\n                return EnumSet.noneOf(Option.class);\n            }\n        });\n    }\n\n    public static boolean isJson(String s) {\n        if (s == null || s.isEmpty()) {\n            return false;\n        }\n        if (s.charAt(0) == ' ') {\n            s = s.trim();\n            if (s.isEmpty()) {\n                return false;\n            }\n        }\n        return s.charAt(0) == '{' || s.charAt(0) == '[';\n    }\n\n    public static String toStrictJson(String raw) {\n        JSONParser jp = new JSONParser(JSONParser.MODE_PERMISSIVE);\n        try {\n            Object o = jp.parse(raw);\n            return JSONValue.toJSONString(o);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static String toJson(Object o) {\n        return toJson(o, false);\n    }\n\n    public static String toJson(Object o, boolean pretty) {\n        if (!pretty) { // TODO use JSONStyleIdent in json-smart 2.4\n            try {\n                return JSONValue.toJSONString(o);\n            } catch (Throwable t) {\n                logger.warn(\"object to json serialization failure, trying alternate approach: {}\", t.getMessage());\n            }\n        }\n        return JsonUtils.toJsonSafe(o, pretty);\n    }\n\n    public static byte[] toJsonBytes(Object o) {\n        return toJson(o).getBytes(StandardCharsets.UTF_8);\n    }\n\n    public static Object fromJson(String json) {\n        return JSONValue.parseKeepingOrder(json);\n    }\n\n    public static Object fromJsonStrict(String json) {\n        JSONParser parser = new JSONParser(JSONParser.MODE_RFC4627);\n        try {\n            return parser.parse(json.trim(), defaultReader.DEFAULT_ORDERED);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static Object fromJson(String s, String className) {\n        try {\n            Class clazz = Class.forName(className);\n            return JSONValue.parse(s, clazz);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static <T> T fromJson(String s, Class<T> clazz) {\n        return (T) fromJson(s, clazz.getName());\n    }\n\n    public static Object fromYaml(String raw) {\n        Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions()));\n        return yaml.load(raw);\n    }\n\n    public static List<Map> fromCsv(String raw) {\n        CsvReader reader = CsvReader.builder().build(raw);\n        List<String> header = new ArrayList();\n        List<Map> rows = new ArrayList();\n        try {\n            boolean first = true;\n            for (CsvRow row : reader) {\n                if (first) {\n                    for (String field : row.getFields()) {\n                        header.add(field.replace(\"\\ufeff\", \"\")); // remove byte order mark\n                    }\n                    first = false;\n                } else {\n                    int count = header.size();\n                    Map<String, Object> map = new LinkedHashMap(count);\n                    for (int i = 0; i < count; i++) {\n                        map.put(header.get(i), row.getField(i));\n                    }\n                    rows.add(map);\n                }\n            }\n            return rows;\n        } catch (Exception e) {\n            logger.warn(\"failed to parse csv: {}\", e.getMessage());\n            return rows;\n        }\n    }\n\n    public static String toCsv(List<Map<String, Object>> list) {\n        StringWriter sw = new StringWriter();\n        CsvWriter writer = CsvWriter.builder().build(sw);\n        // header row\n        if (!list.isEmpty()) {\n            writer.writeRow(list.get(0).keySet());\n        }\n        for (Map<String, Object> map : list) {\n            List<String> row = new ArrayList(map.size());\n            for (Object value : map.values()) {\n                row.add(value == null ? null : value.toString());\n            }\n            writer.writeRow(row);\n        }\n        return sw.toString();\n    }\n\n    public static Object shallowCopy(Object o) {\n        if (o instanceof List) {\n            return new ArrayList((List) o);\n        } else if (o instanceof Map) {\n            return new LinkedHashMap((Map) o);\n        } else {\n            return o;\n        }\n    }\n\n    public static Object deepCopy(Object o) {\n        // anti recursion / back-references\n        Set<Object> seen = Collections.newSetFromMap(new IdentityHashMap());\n        return recurseDeepCopy(o, seen);\n    }\n\n    private static Object recurseDeepCopy(Object o, Set<Object> seen) {\n        if (o instanceof List) {\n            List list = (List) o;\n            if (seen.add(o)) {\n                int count = list.size();\n                List listCopy = new ArrayList(count);\n                for (int i = 0; i < count; i++) {\n                    listCopy.add(recurseDeepCopy(list.get(i), seen));\n                }\n                return listCopy;\n            } else {\n                return o;\n            }\n        } else if (o instanceof Map) {\n            if (seen.add(o)) {\n                Map<String, Object> map = (Map<String, Object>) o;\n                Map<String, Object> mapCopy = new LinkedHashMap(map.size());\n                map.forEach((k, v) -> {\n                    mapCopy.put(k, recurseDeepCopy(v, seen));\n                });\n                return mapCopy;\n            } else {\n                return o;\n            }\n        } else {\n            return o;\n        }\n    }\n\n    public static String toJsonSafe(Object o, boolean pretty) {\n        StringBuilder sb = new StringBuilder();\n        // anti recursion / back-references\n        Set<Object> seen = Collections.newSetFromMap(new IdentityHashMap());\n        recurseJsonString(o, pretty, sb, 0, seen);\n        if (pretty) {\n            sb.append('\\n');\n        }\n        return sb.toString();\n    }\n\n    private static void pad(StringBuilder sb, int depth) {\n        for (int i = 0; i < depth; i++) {\n            sb.append(' ').append(' ');\n        }\n    }\n\n    private static void ref(StringBuilder sb, Object o) {\n        sb.append(\"\\\"#ref:\").append(o.getClass().getName()).append('\"');\n    }\n\n    public static String escapeValue(String raw) {\n        return JSONValue.escape(raw, JSONStyle.LT_COMPRESS);\n    }\n\n    private static void recurseJsonString(Object o, boolean pretty, StringBuilder sb, int depth, Set<Object> seen) {\n        if (o == null) {\n            sb.append(\"null\");\n        } else if (o instanceof List) {\n            List list = (List) o;            \n            if (list.isEmpty() || seen.add(o)) {                \n                sb.append('[');\n                if (pretty) {\n                    sb.append('\\n');\n                }\n                Iterator iterator = list.iterator();\n                while (iterator.hasNext()) {\n                    Object child = iterator.next();\n                    if (pretty) {\n                        pad(sb, depth + 1);\n                    }\n                    recurseJsonString(child, pretty, sb, depth + 1, seen);\n                    if (iterator.hasNext()) {\n                        sb.append(',');\n                    }\n                    if (pretty) {\n                        sb.append('\\n');\n                    }\n                }\n                if (pretty) {\n                    pad(sb, depth);\n                }\n                sb.append(']');\n            } else {\n                ref(sb, o);\n            }\n        } else if (o instanceof Map) {\n            Map<String, Object> map = (Map<String, Object>) o;\n            if (map.isEmpty() || seen.add(o)) {\n                sb.append('{');\n                if (pretty) {\n                    sb.append('\\n');\n                }                \n                Iterator<Map.Entry<String, Object>> iterator = map.entrySet().iterator();\n                while (iterator.hasNext()) {\n                    Map.Entry<String, Object> entry = iterator.next();\n                    Object key = entry.getKey(); // found a rare case where this was a boolean\n                    if (pretty) {\n                        pad(sb, depth + 1);\n                    }\n                    sb.append('\"').append(escapeValue(key == null ? null : key.toString())).append('\"').append(':');\n                    if (pretty) {\n                        sb.append(' ');\n                    }\n                    recurseJsonString(entry.getValue(), pretty, sb, depth + 1, seen);\n                    if (iterator.hasNext()) {\n                        sb.append(',');\n                    }\n                    if (pretty) {\n                        sb.append('\\n');\n                    }\n                }\n                if (pretty) {\n                    pad(sb, depth);\n                }\n                sb.append('}');\n            } else {\n                ref(sb, o);\n            }\n        } else if (o instanceof String) {\n            String value = (String) o;\n            sb.append('\"').append(escapeValue(value)).append('\"');\n        } else if (o instanceof Number || o instanceof Boolean) {\n            sb.append(o);\n        } else { // TODO custom writers ?\n            String value = o.toString();\n            sb.append('\"').append(escapeValue(value)).append('\"');\n        }\n    }\n\n    public static void removeKeysWithNullValues(Object o) {\n        if (o instanceof List) {\n            List list = (List) o;\n            for (Object v : list) {\n                removeKeysWithNullValues(v);\n            }\n        } else if (o instanceof Map) {\n            Map<String, Object> map = (Map) o;\n            List<String> toRemove = new ArrayList();\n            for (Map.Entry<String, Object> entry : map.entrySet()) {\n                Object v = entry.getValue();\n                if (v == null) {\n                    toRemove.add(entry.getKey());\n                } else {\n                    removeKeysWithNullValues(v);\n                }\n            }\n            toRemove.forEach(key -> map.remove(key));\n        }\n    }\n\n    public static Map<String, String> simplify(Map<String, List<String>> map) {\n        if (map == null) {\n            return Collections.emptyMap();\n        }\n        Map<String, String> result = new LinkedHashMap(map.size());\n        map.forEach((k, v) -> {\n            if (v instanceof List) {\n                List list = (List) v;\n                if (list.size() > 1) {\n                    result.put(k, StringUtils.join(list, \",\"));\n                } else if (list.size() == 1) {\n                    Object value = list.get(0);\n                    if (value != null) {\n                        result.put(k, value + \"\");\n                    }                    \n                }\n            } else if (v != null) {\n                result.put(k, v + \"\");\n            }\n        });\n        return result;\n    }\n\n    public static String toString(Object o) {\n        return toString(o, false);\n    }\n\n    public static String toString(Object o, boolean pretty) {\n        if (o == null) {\n            return null;\n        }\n        if (o instanceof Map || o instanceof List) {\n            return JsonUtils.toJson(o, pretty);\n        } else if (o instanceof Node) {\n            return XmlUtils.toString((Node) o, pretty);\n        } else if (o instanceof byte[]) {\n            return FileUtils.toString((byte[]) o);\n        } else {\n            return o.toString();\n        }\n    }\n\n    public static byte[] toBytes(Object o) {\n        if (o == null) {\n            return null;\n        }\n        if (o instanceof Map || o instanceof List) {\n            return FileUtils.toBytes(JsonUtils.toJson(o));\n        } else if (o instanceof Node) {\n            return FileUtils.toBytes(XmlUtils.toString((Node) o));\n        } else if (o instanceof byte[]) {\n            return (byte[]) o;\n        } else {\n            return FileUtils.toBytes(o.toString());\n        }\n    }\n\n    public static Object fromBytes(byte[] bytes, boolean strict, ResourceType resourceType) {\n        if (bytes == null) {\n            return null;\n        }\n        String raw = FileUtils.toString(bytes);\n        return fromString(raw, strict, resourceType);\n    }\n\n    public static Object fromString(String raw, boolean strict, ResourceType resourceType) {\n        String trimmed = raw.trim();\n        if (trimmed.isEmpty()) {\n            return raw;\n        }\n        if (resourceType != null && resourceType.isBinary()) {\n            return raw;\n        }\n        switch (trimmed.charAt(0)) {\n            case '{':\n            case '[':\n                if (strict) {\n                    return JsonUtils.fromJsonStrict(raw);\n                }\n                try {\n                    return JsonUtils.fromJson(raw);\n                } catch (Exception e) {\n                    logger.trace(\"failed to parse json: {}\", e.getMessage());\n                    return raw;\n                }\n            case '<':\n                if (resourceType == null || resourceType.isXml()) {\n                    try {\n                        return XmlUtils.toXmlDoc(raw);\n                    } catch (Exception e) {\n                        logger.trace(\"failed to parse xml: {}\", e.getMessage());\n                        if (strict) {\n                            throw e;\n                        }\n                        return raw;\n                    }\n                } else {\n                    return raw;\n                }\n            default:\n                return raw;\n        }\n    }\n\n    public static Object fromStringSafe(String raw) {\n        try {\n            return fromString(raw, false, null);\n        } catch (Exception e) {\n            logger.trace(\"failed to auto convert: {}\", e + \"\");\n            return raw;\n        }\n    }    \n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/graal/JsArray.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.graal;\n\nimport org.graalvm.polyglot.Value;\nimport org.graalvm.polyglot.proxy.ProxyArray;\n\n/**\n *\n * @author pthomas3\n */\npublic class JsArray implements ProxyArray {\n\n    private final Object[] array;\n\n    public JsArray(Object[] array) {\n        this.array = array;\n    }\n\n    @Override\n    public Object get(long index) {\n        return array[(int) index];\n    }\n\n    @Override\n    public void set(long index, Value value) {\n        throw new UnsupportedOperationException(\"set by index not supported\");\n    }\n\n    @Override\n    public long getSize() {\n        return array.length;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/graal/JsValue.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.graal;\n\nimport com.intuit.karate.JsonUtils;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.function.Function;\nimport org.graalvm.polyglot.Value;\nimport org.graalvm.polyglot.proxy.Proxy;\nimport org.graalvm.polyglot.proxy.ProxyExecutable;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.w3c.dom.Node;\n\n/**\n *\n * @author pthomas3\n */\npublic class JsValue {\n\n    private static final Logger logger = LoggerFactory.getLogger(JsValue.class);\n\n    public static enum Type {\n        OBJECT,\n        ARRAY,\n        FUNCTION,\n        XML,\n        NULL,\n        OTHER\n    }\n\n    public static final JsValue NULL = new JsValue(Value.asValue(null));\n\n    private final Value original;\n    protected final Object value;\n    public final Type type;\n\n    public JsValue(Value v) {\n        if (v == null) {\n            throw new RuntimeException(\"JsValue() constructor argument has to be not-null\");\n        }\n        this.original = v;\n        try {\n            if (v.isNull()) {\n                value = null;\n                type = Type.NULL;\n            } else if (v.isHostObject()) {\n                if (v.isMetaObject()) { // java.lang.Class !\n                    value = v; // special case, keep around as graal value\n                } else {\n                    value = v.asHostObject();\n                }\n                type = Type.OTHER;\n            } else if (v.isProxyObject()) {\n                Object o = v.asProxyObject();\n                if (o instanceof JsXml) {\n                    value = ((JsXml) o).getNode();\n                    type = Type.XML;\n                } else if (o instanceof JsMap) {\n                    value = ((JsMap) o).getMap();\n                    type = Type.OBJECT;\n                } else if (o instanceof JsList) {\n                    value = ((JsList) o).getList();\n                    type = Type.ARRAY;\n                } else if (o instanceof ProxyExecutable) {\n                    value = o;\n                    type = Type.FUNCTION;\n                } else { // e.g. custom bridge, e.g. Request\n                    value = v.as(Object.class);\n                    type = Type.OTHER;\n                }\n            } else if (v.hasArrayElements()) {\n                int size = (int) v.getArraySize();\n                List list = new ArrayList(size);\n                for (int i = 0; i < size; i++) {\n                    Value child = v.getArrayElement(i);\n                    list.add(new JsValue(child).value);\n                }\n                value = list;\n                type = Type.ARRAY;\n            } else if (v.hasMembers()) {\n                if (v.canExecute()) {\n                    if (v.canInstantiate()) {\n                        // js functions have members, can be executed and are instantiable\n                        value = new JsFunction.Instantiable(v);\n                    } else {\n                        // js, but anonymous / arrow function\n                        value = new JsFunction.Executable(v);\n                    }\n                    type = Type.FUNCTION;\n                } else {\n                    Set<String> keys = v.getMemberKeys();\n                    Map<String, Object> map = new LinkedHashMap(keys.size());\n                    for (String key : keys) {\n                        Value child = v.getMember(key);\n                        map.put(key, new JsValue(child).value);\n                    }\n                    value = map;\n                    type = Type.OBJECT;\n                }\n            } else if (v.isNumber()) {\n                value = v.as(Number.class);\n                type = Type.OTHER;\n            } else if (v.isBoolean()) {\n                value = v.asBoolean();\n                type = Type.OTHER;\n            } else if (v.isString()) {\n                value = v.asString();\n                type = Type.OTHER;\n            } else {\n                value = v.as(Object.class);\n                if (value instanceof Function) {\n                    type = Type.FUNCTION;\n                } else {\n                    type = Type.OTHER;\n                }\n            }\n        } catch (Exception e) {\n            if (logger.isTraceEnabled()) {\n                logger.trace(\"js conversion failed\", e);\n            }\n            throw e;\n        }\n    }\n\n    public <T> T getValue() {\n        return (T) value;\n    }\n\n    public Map<String, Object> getAsMap() {\n        return (Map) value;\n    }\n\n    public List getAsList() {\n        return (List) value;\n    }\n\n    public Value getOriginal() {\n        return original;\n    }\n\n    public boolean isXml() {\n        return type == Type.XML;\n    }\n\n    public boolean isNull() {\n        return type == Type.NULL;\n    }\n\n    public boolean isObject() {\n        return type == Type.OBJECT;\n    }\n\n    public boolean isArray() {\n        return type == Type.ARRAY;\n    }\n\n    public boolean isTrue() {\n        if (type != Type.OTHER || !Boolean.class.equals(value.getClass())) {\n            return false;\n        }\n        return (Boolean) value;\n    }\n\n    public boolean isFunction() {\n        return type == Type.FUNCTION;\n    }\n\n    public boolean isOther() {\n        return type == Type.OTHER;\n    }\n\n    @Override\n    public String toString() {\n        return original.toString();\n    }\n\n    public String toJsonOrXmlString(boolean pretty) {\n        return JsonUtils.toString(value, pretty);\n    }\n\n    public String getAsString() {\n        return JsonUtils.toString(value);\n    }\n\n    public static Object fromJava(Object o) {\n        if (o instanceof Function || o instanceof Proxy) {\n            return o;\n        } else if (o instanceof List) {\n            return new JsList((List) o);\n        } else if (o instanceof Map) {\n            return new JsMap((Map) o);\n        } else if (o instanceof Node) {\n            return new JsXml((Node) o);\n        } else {\n            return o;\n        }\n    }\n\n    public static Object toJava(Value v) {\n        return new JsValue(v).getValue();\n    }\n\n    public static Object unWrap(Object o) {\n        if (o instanceof JsXml) {\n            return ((JsXml) o).getNode();\n        } else if (o instanceof JsMap) {\n            return ((JsMap) o).getMap();\n        } else if (o instanceof JsList) {\n            return ((JsList) o).getList();\n        } else {\n            return o;\n        }\n    }\n\n    public static byte[] toBytes(Value v) {\n        return JsonUtils.toBytes(toJava(v));\n    }\n\n    public static boolean isTruthy(Object o) {\n        if (o == null) {\n            return false;\n        }\n        if (o instanceof Boolean) {\n            return ((Boolean) o);\n        }\n        if (o instanceof Number) {\n            return ((Number) o).doubleValue() != 0.0;\n        }\n        return true;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/graal/Methods.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.graal;\n\n/**\n *\n * @author pthomas3\n */\npublic class Methods {\n\n    private Methods() {\n        // only static methods\n    }\n\n    @FunctionalInterface\n    public interface FunVar<T, U> {\n\n        U call(T... args);\n\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/HttpConstants.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport java.util.stream.Stream;\n\n/**\n *\n * @author pthomas3\n */\npublic class HttpConstants {\n\n    private HttpConstants() {\n        // only static methods\n    }\n\n    public static final Set<String> HTTP_METHODS\n            = Stream.of(\"GET\", \"POST\", \"PUT\", \"DELETE\", \"PATCH\", \"OPTIONS\", \"HEAD\", \"CONNECT\", \"TRACE\")\n                    .collect(Collectors.toSet());\n\n    public static final String HDR_COOKIE = \"Cookie\";\n    public static final String HDR_SET_COOKIE = \"Set-Cookie\";\n    public static final String HDR_CONTENT_TYPE = \"Content-Type\";\n    public static final String HDR_LOCATION = \"Location\";\n    public static final String HDR_CONTENT_LENGTH = \"Content-Length\";\n    public static final String HDR_TRANSFER_ENCODING = \"Transfer-Encoding\";\n    public static final String HDR_ACCEPT = \"Accept\";\n    public static final String HDR_ALLOW = \"Allow\";\n    public static final String HDR_CACHE_CONTROL = \"Cache-Control\";\n\n    public static final String HDR_HX_REQUEST = \"HX-Request\";\n    public static final String HDR_HX_REDIRECT = \"HX-Redirect\";\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/LenientTrustManager.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n...\n", "needle_token_start": 2250, "needle_token_end": 2542, "code_context_ntokens": 16016, "output": ["```java\npublic static Node createNodeByPath(Document doc, String path) {\n    int pos = path.lastIndexOf('/');\n    if (pos == 0) { // root\n        Node root = doc.getDocumentElement();\n        if (root == null) {\n            root = createElement(doc, path.substring(1), null, null);\n            doc.appendChild(root);\n        }\n        return root;\n    }\n    String left = path.substring(0, pos);\n    Node parent = getNodeByPath(doc, left, true);\n    String right = path.substring(pos + 1);\n    if (right.startsWith(\"@\")) { // attribute\n        Element parentElement = (Element) parent;\n        right = right.substring(1);\n        parentElement.setAttribute(right, \"\");\n        return parentElement.getAttributeNode(right);\n    } else {\n        int bracketPos = right.indexOf('[');\n        if (bracketPos != -1) { // index, we assume it is 1 and still append\n            right = right.substring(0, bracketPos);\n        }\n        Element element = createElement(parent, right, null, null);\n        parent.appendChild(element);\n        return element;\n    }\n}\n```"]}
{"repo": "karatelabs/karate", "name": "karateLocator", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/driver/DriverOptions.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a JavaScript expression that accesses a specific element stored in a custom JavaScript object within the document's global scope.\n2. **Input**: A string representing a reference key to an element within a predefined JavaScript object.\n3. **Output**: A string that forms a JavaScript expression to access the element associated with the provided reference key.\n4. **Procedure**: The function concatenates the input reference key with a predefined path pointing to a custom JavaScript object in the document's global scope, forming a complete JavaScript expression that can be used to access the specified element.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        driverRunners.put(PlaywrightDriver.DRIVER_TYPE, PlaywrightDriver::start);\n        return driverRunners;\n    }\n\n    private Map<String, Object> getSession(String browserName) {\n        Map<String, Object> session = webDriverSession;\n        if (session == null) {\n            session = new HashMap();\n        }\n        Map<String, Object> capabilities = (Map) session.get(\"capabilities\");\n        if (capabilities == null) {\n            capabilities = (Map) session.get(\"desiredCapabilities\");\n        }\n        if (capabilities == null) {\n            capabilities = new HashMap();\n            session.put(\"capabilities\", capabilities);\n            Map<String, Object> alwaysMatch = new HashMap();\n            capabilities.put(\"alwaysMatch\", alwaysMatch);\n            alwaysMatch.put(\"browserName\", browserName);\n        }\n        return session;\n    }\n\n    // TODO abstract as method per implementation\n    public Map<String, Object> getWebDriverSessionPayload() {\n        switch (type) {\n            case ChromeWebDriver.DRIVER_TYPE:\n                return getSession(\"chrome\");\n            case GeckoWebDriver.DRIVER_TYPE:\n                return getSession(\"firefox\");\n            case SafariWebDriver.DRIVER_TYPE:\n                return getSession(\"safari\");\n            case MsEdgeDriver.DRIVER_TYPE:\n            case MsWebDriver.DRIVER_TYPE:\n                return getSession(\"edge\");\n            case IeWebDriver.DRIVER_TYPE:\n                return getSession(\"internet explorer\");\n            default:\n                // else user has to specify full payload via webDriverSession\n                return getSession(type);\n        }\n    }\n\n    public static String preProcessWildCard(String locator) {\n        boolean contains;\n        String tag, prefix, text;\n        int index;\n        int pos = locator.indexOf('}');\n        if (pos == -1) {\n            throw new RuntimeException(\"bad locator prefix: \" + locator);\n        }\n        if (locator.charAt(1) == '^') {\n            contains = true;\n            prefix = locator.substring(2, pos);\n        } else {\n            contains = false;\n            prefix = locator.substring(1, pos);\n        }\n        text = locator.substring(pos + 1);\n        pos = prefix.indexOf(':');\n        if (pos != -1) {\n            String tagTemp = prefix.substring(0, pos);\n            tag = tagTemp.isEmpty() ? \"*\" : tagTemp;\n            String indexTemp = prefix.substring(pos + 1);\n            if (indexTemp.isEmpty()) {\n                index = 0;\n            } else {\n                try {\n                    index = Integer.valueOf(indexTemp);\n                } catch (Exception e) {\n                    throw new RuntimeException(\"bad locator prefix: \" + locator + \", \" + e.getMessage());\n                }\n            }\n        } else {\n            tag = prefix.isEmpty() ? \"*\" : prefix;\n            index = 0;\n        }\n        if (!tag.startsWith(\"/\")) {\n            tag = \"//\" + tag;\n        }\n        String xpath;\n        if (contains) {\n            xpath = tag + \"[contains(normalize-space(text()),'\" + text + \"')]\";\n        } else {\n            xpath = tag + \"[normalize-space(text())='\" + text + \"']\";\n        }\n        if (index == 0) {\n            return xpath;\n        }\n        return \"/(\" + xpath + \")[\" + index + \"]\";\n    }\n\n    private static final String DOCUMENT = \"document\";\n\n    public static String selector(String locator) {\n        return selector(locator, DOCUMENT);\n    }\n\n    public static String selector(String locator, String contextNode) {\n        if (locator.startsWith(\"(\")) {\n            return locator; // pure js !\n        }\n        if (locator.startsWith(\"{\")) {\n            locator = preProcessWildCard(locator);\n        }\n        if (locator.startsWith(\"/\") || locator.startsWith(\"./\") || locator.startsWith(\"../\")) { // XPathResult.FIRST_ORDERED_NODE_TYPE = 9\n            if (locator.startsWith(\"/(\")) { // hack for wildcard with index (see preProcessWildCard last line)\n                if (DOCUMENT.equals(contextNode)) {\n                    locator = locator.substring(1);\n                } else {\n                    locator = \"(.\" + locator.substring(2);\n                }\n            } else if (!DOCUMENT.equals(contextNode) && !locator.startsWith(\".\")) {\n                locator = \".\" + locator; // evaluate relative to this node not root\n            }\n            return \"document.evaluate(\\\"\" + locator + \"\\\", \" + contextNode + \", null, 9, null).singleNodeValue\";\n        }\n        return contextNode + \".querySelector(\\\"\" + locator + \"\\\")\";\n    }\n\n    public void setTimeout(Integer timeout) {\n        this.timeoutOverride = timeout;\n    }\n\n    public int getTimeout() {\n        if (timeoutOverride != null) {\n            return timeoutOverride;\n        }\n        return timeout;\n    }\n\n    public void setRetryInterval(Integer retryInterval) {\n        this.retryInterval = retryInterval;\n    }\n\n    public int getRetryInterval() {\n        if (retryInterval != null) {\n            return retryInterval;\n        }\n        ScenarioEngine engine = ScenarioEngine.get();\n        if (engine == null) {\n            return Config.DEFAULT_RETRY_INTERVAL;\n        } else {\n            return engine.getConfig().getRetryInterval();\n        }\n    }\n\n    public int getRetryCount() {\n        if (retryCount != null) {\n            return retryCount;\n        }\n        ScenarioEngine engine = ScenarioEngine.get();\n        if (engine == null) {\n            return Config.DEFAULT_RETRY_COUNT;\n        } else {\n            return ScenarioEngine.get().getConfig().getRetryCount();\n        }\n    }\n\n    public <T> T retry(Supplier<T> action, Predicate<T> condition, String logDescription, boolean failWithException) {\n        long startTime = System.currentTimeMillis();\n        int count = 0, max = getRetryCount();\n        T result;\n        boolean success;\n        do {\n            if (count > 0) {\n                logger.debug(\"{} - retry #{}\", logDescription, count);\n                sleep();\n            }\n            result = action.get();\n            success = condition.test(result);\n        } while (!success && count++ < max);\n        if (!success) {\n            long elapsedTime = System.currentTimeMillis() - startTime;\n            String message = logDescription + \": failed after \" + (count - 1) + \" retries and \" + elapsedTime + \" milliseconds\";\n            logger.warn(message);\n            if (failWithException) {\n                throw new RuntimeException(message);\n            }\n        }\n        return result;\n    }\n\n    public static String wrapInFunctionInvoke(String text) {\n        return \"(function(){ \" + text + \" })()\";\n    }\n\n    private static final String HIGHLIGHT_FN = \"function(e){ var old = e.getAttribute('style');\"\n            + \" e.setAttribute('style', 'background: yellow; border: 2px solid red;');\"\n            + \" setTimeout(function(){ e.setAttribute('style', old) }, %d) }\";\n\n    private static String highlightFn(int millis) {\n        return String.format(HIGHLIGHT_FN, millis);\n    }\n\n    public String highlight(String locator, int millis) {\n        String e = selector(locator);\n        String temp = \"var e = \" + e + \"; var fun = \" + highlightFn(millis) + \"; fun(e)\";\n        return wrapInFunctionInvoke(temp);\n    }\n\n    public String highlightAll(String locator, int millis) {\n        return scriptAllSelector(locator, highlightFn(millis));\n    }\n\n    public String optionSelector(String locator, String text) {\n        boolean textEquals = text.startsWith(\"{}\");\n        boolean textContains = text.startsWith(\"{^}\");\n        String condition;\n        if (textEquals || textContains) {\n            text = text.substring(text.indexOf('}') + 1);\n            condition = textContains ? \"e.options[i].text.indexOf(t) !== -1\" : \"e.options[i].text === t\";\n        } else {\n            condition = \"e.options[i].value === t\";\n        }\n        String e = selector(locator);\n        String temp = \"var e = \" + e + \"; var t = \\\"\" + text + \"\\\";\"\n                + \" for (var i = 0; i < e.options.length; ++i)\"\n                + \" if (\" + condition + \") { e.options[i].selected = true; e.dispatchEvent(new Event('change')) }\";\n        return wrapInFunctionInvoke(temp);\n    }\n\n    public String optionSelector(String id, int index) {\n        String e = selector(id);\n        String temp = \"var e = \" + e + \"; var t = \" + index + \";\"\n                + \" for (var i = 0; i < e.options.length; ++i)\"\n                + \" if (i === t) { e.options[i].selected = true; e.dispatchEvent(new Event('change')) }\";\n        return wrapInFunctionInvoke(temp);\n    }\n\n    private String fun(String expression) {\n        char first = expression.charAt(0);\n        switch (first) {\n            case '_':\n                if (expression.contains(\"=>\")) {\n                    return expression;\n                }\n            case '!':\n                return \"function(_){ return \" + expression + \" }\";\n            default:\n                return expression;\n        }\n    }\n\n    public String scriptSelector(String locator, String expression) {\n        return scriptSelector(locator, expression, DOCUMENT);\n    }\n\n    public String scriptSelector(String locator, String expression, String contextNode) {\n        String temp = \"var fun = \" + fun(expression) + \"; var e = \" + selector(locator, contextNode) + \"; return fun(e)\";\n        return wrapInFunctionInvoke(temp);\n    }\n\n    public String scriptAllSelector(String locator, String expression) {\n        return scriptAllSelector(locator, expression, DOCUMENT);\n    }\n\n    // the difference here from selector() is the use of querySelectorAll()\n    // how the loop for XPath results has to be handled\n    public String scriptAllSelector(String locator, String expression, String contextNode) {\n        if (locator.startsWith(\"{\")) {\n            locator = preProcessWildCard(locator);\n        }\n        boolean isXpath = locator.startsWith(\"/\") || locator.startsWith(\"./\") || locator.startsWith(\"../\");\n        String selector;\n        if (isXpath) { // XPathResult.ORDERED_NODE_ITERATOR_TYPE = 5\n            selector = \"document.evaluate(\\\"\" + locator + \"\\\", \" + contextNode + \", null, 5, null)\";\n        } else {\n            selector = contextNode + \".querySelectorAll(\\\"\" + locator + \"\\\")\";\n        }\n        String temp = \"var res = []; var fun = \" + fun(expression) + \"; var es = \" + selector + \"; \";\n        if (isXpath) {\n            temp = temp + \"var e = null; while(e = es.iterateNext()) res.push(fun(e)); return res\";\n        } else {\n            temp = temp + \"es.forEach(function(e){ res.push(fun(e)) }); return res\";\n        }\n        return wrapInFunctionInvoke(temp);\n    }\n\n    public void sleep() {\n        sleep(getRetryInterval());\n    }\n\n    public void sleep(int millis) {\n        if (millis == 0) {\n            return;\n        }\n        try {\n            processLogger.trace(\"sleeping for millis: {}\", millis);\n            Thread.sleep(millis);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static String getRelativePositionJs(String locator) {\n        String temp = \"var r = \" + selector(locator, DOCUMENT)\n                + \".getBoundingClientRect(); return { x: r.x, y: r.y, width: r.width, height: r.height }\";\n        return wrapInFunctionInvoke(temp);\n    }\n\n    public static String getPositionJs(String locator) {\n        String temp = \"var r = \" + selector(locator, DOCUMENT) + \".getBoundingClientRect();\"\n                + \" var dx = window.scrollX; var dy = window.scrollY;\"\n                + \" return { x: r.x + dx, y: r.y + dy, width: r.width + dx, height: r.height + dy }\";\n        return wrapInFunctionInvoke(temp);\n    }\n\n    public Map<String, Object> newMapWithSelectedKeys(Map<String, Object> map, String... keys) {\n        Map<String, Object> out = new HashMap(keys.length);\n        for (String key : keys) {\n            Object o = map.get(key);\n            if (o != null) {\n                out.put(key, o);\n            }\n        }\n        return out;\n    }\n\n    public void disableRetry() {\n        retryEnabled = false;\n        retryCount = null;\n        retryInterval = null;\n    }\n\n    public void enableRetry(Integer count, Integer interval) {\n        retryEnabled = true;\n        retryCount = count; // can be null\n        retryInterval = interval; // can be null\n    }\n\n    public Element waitUntil(Driver driver, String locator, String expression) {\n        long startTime = System.currentTimeMillis();\n        String js = scriptSelector(locator, expression);\n        boolean found = driver.waitUntil(js);\n        if (!found) {\n            long elapsedTime = System.currentTimeMillis() - startTime;\n            throw new RuntimeException(\"wait failed for: \" + locator\n                    + \" and condition: \" + expression + \" after \" + elapsedTime + \" milliseconds\");\n        }\n        return DriverElement.locatorExists(driver, locator);\n    }\n\n    public String waitForUrl(Driver driver, String expected) {\n        return retry(() -> driver.getUrl(), url -> url.contains(expected), \"waitForUrl\", true);\n    }\n\n    public Element waitForAny(Driver driver, String... locators) {\n        long startTime = System.currentTimeMillis();\n        List<String> list = Arrays.asList(locators);\n        Iterator<String> iterator = list.iterator();\n        StringBuilder sb = new StringBuilder();\n        while (iterator.hasNext()) {\n            String locator = iterator.next();\n            String js = selector(locator);\n            sb.append(\"(\").append(js).append(\" != null)\");\n            if (iterator.hasNext()) {\n                sb.append(\" || \");\n            }\n        }\n        boolean found = driver.waitUntil(sb.toString());\n        // important: un-set the retry flag        \n        disableRetry();\n        if (!found) {\n            long elapsedTime = System.currentTimeMillis() - startTime;\n            throw new RuntimeException(\"wait failed for: \" + list + \" after \" + elapsedTime + \" milliseconds\");\n        }\n        if (locators.length == 1) {\n            return DriverElement.locatorExists(driver, locators[0]);\n        }\n        for (String locator : locators) {\n            Element temp = driver.optional(locator);\n            if (temp.isPresent()) {\n                return temp;\n            }\n        }\n        // this should never happen\n        throw new RuntimeException(\"unexpected wait failure for locators: \" + list);\n    }\n\n    public Element optional(Driver driver, String locator) {\n        String js = selector(locator);\n        String evalJs = js + \" != null\";\n        Object o = driver.script(evalJs);\n        if (o instanceof Boolean && (Boolean) o) {\n            return DriverElement.locatorExists(driver, locator);\n        } else {\n            return new MissingElement(driver, locator);\n        }\n    }\n\n    \npublic static String karateLocator(String karateRef) {\n        return \"(document._karate.\" + karateRef + \")\";\n    }\n\n    public String focusJs(String locator) {\n        return \"var e = \" + selector(locator) + \"; e.focus(); try { e.selectionStart = e.selectionEnd = e.value.length } catch(x) {}\";\n    }\n\n    public List<Element> findAll(Driver driver, String locator) {\n        List<String> list = driver.scriptAll(locator, DriverOptions.KARATE_REF_GENERATOR);\n        List<Element> elements = new ArrayList(list.size());\n        for (String karateRef : list) {\n            String karateLocator = karateLocator(karateRef);\n            elements.add(DriverElement.locatorExists(driver, karateLocator));\n        }\n        return elements;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/DriverElement.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * TODO make this convert-able to JSON\n *\n * @author pthomas3\n */\npublic class DriverElement implements Element {\n\n    private final Driver driver;\n    private final String locator;\n\n    private Boolean exists;\n\n    private DriverElement(Driver driver, String locator, Boolean exists) {\n        this.driver = driver;\n        this.locator = locator;\n        this.exists = exists;\n    }\n\n    public static Element locatorExists(Driver driver, String locator) {\n        return new DriverElement(driver, locator, true);\n    }\n\n    public static Element locatorUnknown(Driver driver, String locator) {\n        return new DriverElement(driver, locator, null); // exists flag set to null\n    }\n\n    @Override\n    public String getLocator() {\n        return locator;\n    }\n\n    @Override\n    public boolean isPresent() {\n        if (exists == null) {\n            exists = driver.optional(locator).isPresent();\n        }\n        return exists;\n    }\n\n    public void setExists(Boolean exists) {\n        this.exists = exists;\n    }\n\n    @Override\n    public Map<String, Object> getPosition() {\n        return driver.position(locator);\n    }\n\n    @Override\n    public byte[] screenshot() {\n        return driver.screenshot();\n    }\n\n    @Override\n    public boolean isEnabled() {\n        return driver.enabled(locator);\n    }\n\n    @Override\n    public Element highlight() {\n        return driver.highlight(locator);\n    }\n\n    @Override\n    public Element focus() {\n        return driver.focus(locator);\n    }\n\n    @Override\n    public Element clear() {\n        return driver.clear(locator);\n    }\n\n    @Override\n    public Element click() {\n        return driver.click(locator);\n    }\n\n    @Override\n    public Element submit() {\n        driver.submit();\n        return this;\n    }\n\n    @Override\n    public Element scroll() {\n        driver.scroll(locator);\n        return this;\n    }        \n\n    @Override\n    public Mouse mouse() {\n        return driver.mouse(locator);\n    }\n\n    @Override\n    public Element input(String value) {\n        return driver.input(locator, value);\n    }\n\n    @Override\n    public Element input(String[] values) {\n        return driver.input(locator, values);\n    }\n\n    @Override\n    public Element input(String[] values, int delay) {\n        return driver.input(locator, values, delay);\n    }\n\n    @Override\n    public Element select(String text) {\n        return driver.select(locator, text);\n    }\n\n    @Override\n    public Element select(int index) {\n        return driver.select(locator, index);\n    }\n\n    @Override\n    public Element switchFrame() {\n        driver.switchFrame(locator);\n        return this;\n    }\n\n    @Override\n    public Element delay(int millis) {\n        driver.delay(millis);\n        return this;\n    }\n\n    @Override\n    public Element retry() {\n        driver.retry();\n        return this;\n    }\n\n    @Override\n    public Element retry(int count) {\n        driver.retry(count);\n        return this;\n    }\n\n    @Override\n    public Element retry(Integer count, Integer interval) {\n        driver.retry(count, interval);\n        return this;\n    }\n\n    @Override\n    public Element waitFor() {\n        driver.waitFor(locator); // will throw exception if not found\n        return this;\n    }\n\n    @Override\n    public Element waitForText(String text) {\n        return driver.waitForText(locator, text);\n    }\n\n    @Override\n    public Element waitUntil(String expression) {\n        return driver.waitUntil(locator, expression); // will throw exception if not found\n    }\n\n    @Override\n    public Object script(String expression) {\n        return driver.script(locator, expression);\n    }\n    \n    @Override\n    public Object scriptAll(String relative, String expression) {\n        String script = driver.getOptions().scriptAllSelector(relative, expression, thisLocator());       \n        return driver.script(script);\n    }    \n\n    private String thisLocator() {\n        String thisRef = (String) driver.script(locator, DriverOptions.KARATE_REF_GENERATOR);\n        return DriverOptions.karateLocator(thisRef);\n    }\n\n    @Override\n    public Element optional(String locator) {\n        String childRefScript = driver.getOptions().scriptSelector(locator, DriverOptions.KARATE_REF_GENERATOR, thisLocator());\n        try {\n            String childRef = (String) driver.script(childRefScript);\n            return DriverElement.locatorExists(driver, DriverOptions.karateLocator(childRef));\n        } catch (Exception e) {\n            return new MissingElement(driver, locator);\n        }\n    }\n\n    @Override\n    public boolean exists(String locator) {\n        return optional(locator).isPresent();\n    }\n\n    @Override\n    public Element locate(String locator) {\n        Element e = optional(locator);\n        if (e.isPresent()) {\n            return e;\n        }\n        throw new RuntimeException(\"cannot find locator: \" + locator);\n    }\n\n    @Override\n    public List<Element> locateAll(String locator) {\n        String childRefScript = driver.getOptions().scriptAllSelector(locator, DriverOptions.KARATE_REF_GENERATOR, thisLocator());\n        List<String> childRefs = (List) driver.script(childRefScript);\n        return refsToElements(childRefs);\n    }\n    \n    private List<Element> refsToElements(List<String> refs) {\n        List<Element> elements = new ArrayList(refs.size());\n        for (String ref : refs) {\n            String karateLocator = DriverOptions.karateLocator(ref);\n            elements.add(DriverElement.locatorExists(driver, karateLocator));\n        }\n        return elements;        \n    }\n\n    @Override\n    public String attribute(String name) {\n        return driver.attribute(locator, name);\n    }\n\n    @Override\n    public String property(String name) {\n        return driver.property(locator, name);\n    }\n\n    //java bean naming conventions =============================================        \n    //        \n    @Override\n    public String getHtml() {\n        return driver.html(locator);\n    }\n\n    @Override\n    public void setHtml(String html) {\n        driver.script(locator, \"_.outerHTML = '\" + html + \"'\");\n    }\n\n    @Override\n    public String getText() {\n        return driver.text(locator);\n    }\n\n    @Override\n    public void setText(String text) {\n        driver.script(locator, \"_.innerHTML = '\" + text + \"'\");\n    }\n\n    @Override\n    public String getValue() {\n        return driver.value(locator);\n    }\n\n    @Override\n    public void setValue(String value) {\n        driver.value(locator, value);\n    }\n    \n    private Element relationLocator(String relation) {\n        String js = \"var gen = \" + DriverOptions.KARATE_REF_GENERATOR + \"; var e = \" \n                + DriverOptions.selector(locator) + \"; return gen(e.\" + relation + \")\";\n        String karateRef = (String) driver.script(DriverOptions.wrapInFunctionInvoke(js));\n        return DriverElement.locatorExists(driver, DriverOptions.karateLocator(karateRef));        \n    }\n\n    @Override\n    public Element getParent() {\n        return relationLocator(\"parentElement\");\n    }\n\n    @Override\n    public List<Element> getChildren() {\n        String js = \"var gen = \" + DriverOptions.KARATE_REF_GENERATOR + \"; var es = \" \n                + DriverOptions.selector(locator) + \".children; var res = []; var i;\"\n                + \" for(i = 0; i < es.length; i++) res.push(gen(es[i])); return res\";\n        List<String> childRefs = (List) driver.script(DriverOptions.wrapInFunctionInvoke(js));\n        return refsToElements(childRefs);\n    }        \n\n    @Override\n    public Element getFirstChild() {\n        return relationLocator(\"firstElementChild\");\n    }  \n\n    @Override\n    public Element getLastChild() {\n        return relationLocator(\"lastElementChild\");\n    }     \n\n    @Override\n    public Element getPreviousSibling() {\n        return relationLocator(\"previousElementSibling\");\n    }  \n\n    @Override\n    public Element getNextSibling() {\n        return relationLocator(\"nextElementSibling\");\n    }        \n\n    @Override\n    public Finder rightOf() {\n        return driver.rightOf(locator);\n    }\n\n    @Override\n    public Finder leftOf() {\n        return driver.leftOf(locator);\n    }\n\n    @Override\n    public Finder above() {\n        return driver.above(locator);\n    }\n\n    @Override\n    public Finder below() {\n        return driver.below(locator);\n    }\n\n    @Override\n    public Finder near() {\n        return driver.near(locator);\n    }\n\n    @Override\n    public String toString() {\n        return locator;\n    }        \n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/ElementFinder.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.StringUtils;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class ElementFinder implements Finder {\n\n    public static enum Type {\n        RIGHT,\n        LEFT,\n        ABOVE,\n        BELOW,\n        NEAR\n    }\n\n    private final Driver driver;\n    private final String fromLocator;\n    private final Type type;\n\n    private String tag = \"INPUT\";\n\n    public ElementFinder(Driver driver, String fromLocator, Type type) {\n        this.driver = driver;\n        this.fromLocator = fromLocator;\n        this.type = type;\n    }\n\n    private static String forLoopChunk(ElementFinder.Type type) {\n        switch (type) {\n            case RIGHT:\n                return \"x += s;\";\n            case BELOW:\n                return \"y += s;\";\n            case LEFT:\n                return \"x -= s;\";\n            case ABOVE:\n                return \"y -= s;\";\n            default: // NEAR\n                return \" var a = 0.381966 * i; var x = (s + a) * Math.cos(a); var y = (s + a) * Math.sin(a);\";\n        }\n    }\n\n    public static String exitCondition(String findTag) {\n        int pos = findTag.indexOf('}');\n        if (pos == -1) {\n            return \"e.tagName == '\" + findTag.toUpperCase() + \"'\";\n        }\n        int caretPos = findTag.indexOf('^');\n        boolean contains = caretPos != -1 && caretPos < pos;\n        if (!contains) {\n            caretPos = 0;\n        }\n        String tagName = StringUtils.trimToNull(findTag.substring(caretPos + 1, pos));\n        String suffix = tagName == null ? \"\" : \" && e.tagName == '\" + tagName.toUpperCase() + \"'\";\n        String findText = findTag.substring(pos + 1);\n        if (contains) {\n            return \"e.textContent.trim().includes('\" + findText + \"')\" + suffix;\n        } else {\n            return \"e.textContent.trim() == '\" + findText + \"'\" + suffix;\n        }\n    }\n\n    private static String findScript(Driver driver, String locator, ElementFinder.Type type, String findTag) {\n        Map<String, Object> pos = driver.position(locator, true);\n        Number xNum = (Number) pos.get(\"x\");\n        Number yNum = (Number) pos.get(\"y\");\n        Number width = (Number) pos.get(\"width\");\n        Number height = (Number) pos.get(\"height\");\n        // get center point\n        int x = xNum.intValue() + width.intValue() / 2;\n        int y = yNum.intValue() + height.intValue() / 2;\n        // o: origin, a: angle, s: step\n        String fun = \"var gen = \" + DriverOptions.KARATE_REF_GENERATOR + \";\"\n                + \" var o = { x: \" + x + \", y: \" + y + \"}; var s = 10; var x = 0; var y = 0;\"\n                + \" for (var i = 0; i < 200; i++) {\"\n                + forLoopChunk(type)\n                + \" var e = document.elementFromPoint(o.x + x, o.y + y);\"\n                // + \" console.log(o.x +':' + o.y + ' ' + x + ':' + y + ' ' + e.tagName + ':' + e.textContent);\"\n                + \" if (e && \" + exitCondition(findTag) + \") return gen(e); \"\n                + \" } return null\";\n        return DriverOptions.wrapInFunctionInvoke(fun);\n    }\n\n    private String getDebugString() {\n        return fromLocator + \", \" + type + \", \" + tag;\n    }\n\n    @Override\n    public Element find() {\n        String js = findScript(driver, fromLocator, type, tag);\n        String karateRef = (String) driver.script(js);\n        if (karateRef == null) {\n            throw new RuntimeException(\"unable to find: \" + getDebugString());\n        }\n        return DriverElement.locatorExists(driver, DriverOptions.karateLocator(karateRef));\n    }\n\n    @Override\n    public Element find(String tag) {\n        this.tag = tag;\n        return find();\n    }\n\n    @Override\n    public String getValue() {\n        return find().getValue();\n    }\n\n    @Override\n    public Element clear() {\n        return find().clear();\n    }\n\n    @Override\n    public Element input(String value) {\n        return find().input(value);\n    }\n\n    @Override\n    public Element select(String value) {\n        return find(\"select\").select(value);\n    }\n\n    @Override\n    public Element select(int index) {\n        return find(\"select\").select(index);\n    }\n\n    @Override\n    public Element click() {\n        return find().click();\n    }\n\n    @Override\n    public Element highlight() {\n        return find().highlight();\n    }\n\n    @Override\n    public Element retry() {\n        return find().retry();\n    }\n\n    @Override\n    public Element retry(int count) {\n        return find().retry(count);\n    }\n\n    @Override\n    public Element retry(Integer count, Integer interval) {\n        return find().retry(count, interval);\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/Driver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.core.AutoDef;\nimport com.intuit.karate.core.Plugin;\nimport com.intuit.karate.core.Config;\nimport com.intuit.karate.core.FeatureRuntime;\nimport com.intuit.karate.core.ScenarioEngine;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.core.StepResult;\nimport com.intuit.karate.http.HttpClientFactory;\nimport com.intuit.karate.http.ResourceType;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.function.Predicate;\nimport java.util.function.Supplier;\n\n/**\n *\n * @author pthomas3\n */\npublic interface Driver extends Plugin {\n\n    public static Driver start(String browserType) {\n        return start(Collections.singletonMap(\"type\", browserType));\n    }\n\n    public static Driver start(Map<String, Object> options) {\n        ScenarioRuntime runtime = FeatureRuntime.forTempUse(HttpClientFactory.DEFAULT).scenarios.next();\n        ScenarioEngine.set(runtime.engine);\n        return DriverOptions.start(options, runtime);\n    }\n\n    @AutoDef\n    void activate();\n\n    @AutoDef\n    void refresh();\n\n    @AutoDef\n    void reload();\n\n    @AutoDef\n    void back();\n\n    @AutoDef\n    void forward();\n\n    @AutoDef\n    void maximize();\n\n    @AutoDef\n    void minimize();\n\n    @AutoDef\n    void fullscreen();\n\n    @AutoDef\n    void close();\n\n    @AutoDef\n    void quit();\n\n    @AutoDef\n    void switchPage(String titleOrUrl);\n\n    @AutoDef\n    void switchPage(int index);\n\n    @AutoDef\n    void switchFrame(int index);\n\n    @AutoDef\n    void switchFrame(String locator);\n\n    String getUrl(); // getter\n\n    void setUrl(String url); // setter    \n\n    Map<String, Object> getDimensions(); // getter\n\n    void setDimensions(Map<String, Object> map); // setter\n\n    String getTitle(); // getter\n\n    List<String> getPages(); // getter\n\n    String getDialogText(); // getter\n\n    @AutoDef\n    byte[] screenshot(boolean embed);\n\n    @AutoDef\n    default byte[] screenshot() {\n        return screenshot(true);\n    }\n\n    @AutoDef\n    Map<String, Object> cookie(String name);\n\n    @AutoDef\n    default void setCookies(List<Map<String, Object>> cookies) {\n        System.out.println(\"got this cookie: \" + cookies);\n        cookies.forEach(c -> cookie(c));\n    }\n\n    @AutoDef\n    void cookie(Map<String, Object> cookie);\n\n    @AutoDef\n    void deleteCookie(String name);\n\n    @AutoDef\n    void clearCookies();\n\n    List<Map> getCookies(); // getter    \n\n    @AutoDef\n    void dialog(boolean accept);\n\n    @AutoDef\n    void dialog(boolean accept, String input);\n\n    @AutoDef\n    Object script(String expression);\n\n    @AutoDef\n    boolean waitUntil(String expression);\n\n    @AutoDef\n    Driver submit();\n\n    @AutoDef\n    default Driver retry() {\n        return retry(null, null);\n    }\n\n    @AutoDef\n    default Driver retry(int count) {\n        return retry(count, null);\n    }\n\n    @AutoDef\n    default Driver retry(Integer count, Integer interval) {\n        getOptions().enableRetry(count, interval);\n        return this;\n    }\n\n    @AutoDef\n    default Driver delay(int millis) {\n        getOptions().sleep(millis);\n        return this;\n    }\n\n    @AutoDef\n    Driver timeout(Integer millis);\n\n    @AutoDef\n    Driver timeout();\n\n    // element actions =========================================================\n    //\n    @AutoDef\n    Element focus(String locator);\n\n    @AutoDef\n    Element clear(String locator);\n\n    @AutoDef\n    Element click(String locator);\n\n    @AutoDef\n    Element input(String locator, String value);\n\n    @AutoDef\n    default Element input(String locator, String[] values) {\n        return input(locator, values, 0);\n    }\n\n    @AutoDef\n    default Element input(String locator, String chars, int delay) {\n        String[] array = new String[chars.length()];\n        for (int i = 0; i < array.length; i++) {\n            array[i] = Character.toString(chars.charAt(i));\n        }\n        return input(locator, array, delay);\n    }\n\n    @AutoDef\n    default Element input(String locator, String[] values, int delay) {\n        Element element = DriverElement.locatorUnknown(this, locator);\n        for (String value : values) {\n            if (delay > 0) {\n                delay(delay);\n            }\n            element = input(locator, value);\n        }\n        return element;\n    }\n\n    @AutoDef\n    Element select(String locator, String text);\n\n    @AutoDef\n    Element select(String locator, int index);\n\n    @AutoDef\n    Element value(String locator, String value);\n\n    @AutoDef\n    default Element waitFor(String locator) {\n        return getOptions().waitForAny(this, locator);\n    }\n\n    @AutoDef\n    default String waitForUrl(String expected) {\n        return getOptions().waitForUrl(this, expected);\n    }\n\n    @AutoDef\n    default Element waitForText(String locator, String expected) {\n        return waitUntil(locator, \"_.textContent.includes('\" + expected + \"')\");\n    }\n\n    @AutoDef\n    default Element waitForEnabled(String locator) {\n        return waitUntil(locator, \"!_.disabled\");\n    }\n\n    @AutoDef\n    default List<Element> waitForResultCount(String locator, int count) {\n        return (List) waitUntil(() -> {\n            List<Element> list = locateAll(locator);\n            return list.size() == count ? list : null;\n        });\n    }\n\n    @AutoDef\n    default List waitForResultCount(String locator, int count, String expression) {\n        return (List) waitUntil(() -> {\n            List list = scriptAll(locator, expression);\n            return list.size() == count ? list : null;\n        });\n    }\n\n    @AutoDef\n    default Element waitForAny(String locator1, String locator2) {\n        return getOptions().waitForAny(this, new String[]{locator1, locator2});\n    }\n\n    @AutoDef\n    default Element waitForAny(String[] locators) {\n        return getOptions().waitForAny(this, locators);\n    }\n\n    @AutoDef\n    default Element waitUntil(String locator, String expression) {\n        return getOptions().waitUntil(this, locator, expression);\n    }\n\n    @AutoDef\n    default Object waitUntil(Supplier<Object> condition) {\n        return getOptions().retry(() -> condition.get(), o -> o != null, \"waitUntil (function)\", true);\n    }\n\n    @AutoDef\n    default Element locate(String locator) {\n        Element e = DriverElement.locatorUnknown(this, locator);\n        if (e.isPresent()) {\n            return e;\n        }\n        throw new RuntimeException(\"cannot find locator: \" + locator);\n    }\n\n    @AutoDef\n    default List<Element> locateAll(String locator) {\n        return getOptions().findAll(this, locator);\n    }\n\n    @AutoDef\n    default List<Element> locateAll(String locator, Predicate predicate) {\n        List before = locateAll(locator);\n        List after = new ArrayList(before.size());\n        for (Object o : before) {\n            if (predicate.test(o)) {\n                after.add(o);\n            }\n        }\n        return after;\n    }\n\n    @AutoDef\n    default Element scroll(String locator) {\n        script(locator, DriverOptions.SCROLL_JS_FUNCTION);\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @AutoDef\n    default Element highlight(String locator) {\n        return highlight(locator, Config.DEFAULT_HIGHLIGHT_DURATION);\n    }\n\n    default Element highlight(String locator, int millis) {\n        script(getOptions().highlight(locator, millis));\n        delay(millis);\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @AutoDef\n    default void highlightAll(String locator) {\n        highlightAll(locator, Config.DEFAULT_HIGHLIGHT_DURATION);\n    }\n\n    default void highlightAll(String locator, int millis) {\n        script(getOptions().highlightAll(locator, millis));\n        delay(millis);\n    }\n\n    // friendly locators =======================================================\n    //\n    @AutoDef\n    default Finder rightOf(String locator) {\n        return new ElementFinder(this, locator, ElementFinder.Type.RIGHT);\n    }\n\n    @AutoDef\n    default Finder leftOf(String locator) {\n        return new ElementFinder(this, locator, ElementFinder.Type.LEFT);\n    }\n\n    @AutoDef\n    default Finder above(String locator) {\n        return new ElementFinder(this, locator, ElementFinder.Type.ABOVE);\n    }\n\n    @AutoDef\n    default Finder below(String locator) {\n        return new ElementFinder(this, locator, ElementFinder.Type.BELOW);\n    }\n\n    @AutoDef\n    default Finder near(String locator) {\n        return new ElementFinder(this, locator, ElementFinder.Type.NEAR);\n    }\n\n    // mouse and keys ==========================================================\n    //\n    @AutoDef\n    default Mouse mouse() {\n        return new DriverMouse(this);\n    }\n\n    @AutoDef\n    default Mouse mouse(String locator) {\n        return new DriverMouse(this).move(locator);\n    }\n\n    @AutoDef\n    default Mouse mouse(Number x, Number y) {\n        return new DriverMouse(this).move(x, y);\n    }\n\n    @AutoDef\n    default Keys keys() {\n        return new Keys(this);\n    }\n\n    @AutoDef\n    void actions(List<Map<String, Object>> actions);\n\n    // element state ===========================================================\n    //\n    @AutoDef\n    String html(String locator);\n\n    @AutoDef\n    String text(String locator);\n\n    @AutoDef\n    String value(String locator);\n\n    @AutoDef\n    String attribute(String locator, String name);\n\n    @AutoDef\n    String property(String locator, String name);\n\n    @AutoDef\n    boolean enabled(String locator);\n\n    @AutoDef\n    default boolean exists(String locator) {\n        return getOptions().optional(this, locator).isPresent();\n    }\n\n    @AutoDef\n    default Element optional(String locator) {\n        return getOptions().optional(this, locator);\n    }\n\n    @AutoDef\n    Map<String, Object> position(String locator);\n\n    @AutoDef\n    Map<String, Object> position(String locator, boolean relative);\n\n    @AutoDef\n    byte[] screenshot(String locator, boolean embed);\n\n    @AutoDef\n    default byte[] screenshot(String locator) {\n        return screenshot(locator, true);\n    }\n\n    @AutoDef\n    default Object script(String locator, String expression) {\n        String js = getOptions().scriptSelector(locator, expression);\n        return script(js);\n    }\n\n    @AutoDef\n    default List scriptAll(String locator, String expression) {\n        String js = getOptions().scriptAllSelector(locator, expression);\n        return (List) script(js);\n    }\n\n    @AutoDef\n    default List scriptAll(String locator, String expression, Predicate predicate) {\n        List before = scriptAll(locator, expression);\n        List after = new ArrayList(before.size());\n        for (Object o : before) {\n            if (predicate.test(o)) {\n                after.add(o);\n            }\n        }\n        return after;\n    }\n\n    @AutoDef\n    public byte[] pdf(Map<String, Object> options);\n\n    // for internal use ========================================================\n    //        \n    boolean isTerminated();\n\n    DriverOptions getOptions();\n\n    Object elementId(String locator);\n\n    List elementIds(String locator);\n\n    static final List<String> METHOD_NAMES = Plugin.methodNames(Driver.class);\n\n    @Override\n    default List<String> methodNames() {\n        return METHOD_NAMES;\n    }\n\n    @Override\n    default Map<String, Object> afterScenario() {\n        return Collections.EMPTY_MAP; // TODO\n    }\n\n    @Override\n    public default void onFailure(StepResult stepResult) {\n        if (getOptions().screenshotOnFailure && !stepResult.isWithCallResults()) {\n            byte[] bytes = screenshot(false);\n            getRuntime().embed(bytes, ResourceType.PNG);\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/Key.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\n/**\n *\n * @author pthomas3\n */\npublic class Key {\n\n    public static final Key INSTANCE = new Key();\n\n    public final char NULL = Keys.NULL;\n    public final char CANCEL = Keys.CANCEL;\n    public final char HELP = Keys.HELP;\n    public final char BACK_SPACE = Keys.BACK_SPACE;\n    public final char TAB = Keys.TAB;\n    public final char CLEAR = Keys.CLEAR;\n    public final char RETURN = Keys.RETURN;\n    public final char ENTER = Keys.ENTER;\n    public final char SHIFT = Keys.SHIFT;\n    public final char CONTROL = Keys.CONTROL;\n    public final char ALT = Keys.ALT;\n    public final char PAUSE = Keys.PAUSE;\n    public final char ESCAPE = Keys.ESCAPE;\n    public final char SPACE = Keys.SPACE;\n    public final char PAGE_UP = Keys.PAGE_UP;\n    public final char PAGE_DOWN = Keys.PAGE_DOWN;\n    public final char END = Keys.END;\n    public final char HOME = Keys.HOME;\n    public final char LEFT = Keys.LEFT;\n    public final char UP = Keys.UP;\n    public final char RIGHT = Keys.RIGHT;\n    public final char DOWN = Keys.DOWN;\n    public final char INSERT = Keys.INSERT;\n    public final char DELETE = Keys.DELETE;\n    public final char SEMICOLON = Keys.SEMICOLON;\n    public final char EQUALS = Keys.EQUALS;\n\n    // numpad keys\n    public final char NUMPAD0 = Keys.NUMPAD0;\n    public final char NUMPAD1 = Keys.NUMPAD1;\n    public final char NUMPAD2 = Keys.NUMPAD2;\n    public final char NUMPAD3 = Keys.NUMPAD3;\n    public final char NUMPAD4 = Keys.NUMPAD4;\n    public final char NUMPAD5 = Keys.NUMPAD5;\n    public final char NUMPAD6 = Keys.NUMPAD6;\n    public final char NUMPAD7 = Keys.NUMPAD7;\n    public final char NUMPAD8 = Keys.NUMPAD8;\n    public final char NUMPAD9 = Keys.NUMPAD9;\n    public final char MULTIPLY = Keys.MULTIPLY;\n    public final char ADD = Keys.ADD;\n    public final char SEPARATOR = Keys.SEPARATOR;\n    public final char SUBTRACT = Keys.SUBTRACT;\n    public final char DECIMAL = Keys.DECIMAL;\n    public final char DIVIDE = Keys.DIVIDE;\n\n    // function keys\n    public final char F1 = Keys.F1;\n    public final char F2 = Keys.F2;\n    public final char F3 = Keys.F3;\n    public final char F4 = Keys.F4;\n    public final char F5 = Keys.F5;\n    public final char F6 = Keys.F6;\n    public final char F7 = Keys.F7;\n    public final char F8 = Keys.F8;\n    public final char F9 = Keys.F9;\n    public final char F10 = Keys.F10;\n    public final char F11 = Keys.F11;\n    public final char F12 = Keys.F12;\n    public final char META = Keys.META;\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/graal/JsFunction.java\n/*\n * The MIT License\n *\n * Copyright 2023 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.graal;\n\nimport com.intuit.karate.core.ScenarioEngine;\nimport org.graalvm.polyglot.Value;\nimport org.graalvm.polyglot.proxy.ProxyExecutable;\nimport org.graalvm.polyglot.proxy.ProxyInstantiable;\nimport org.graalvm.polyglot.proxy.ProxyObject;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author peter\n */\npublic abstract class JsFunction implements ProxyObject {\n\n    protected static final Logger logger = LoggerFactory.getLogger(JsFunction.class);\n\n    public static final Object LOCK = new Object();\n\n    protected final Value value;\n\n    protected JsFunction(Value v) {\n        this.value = v;\n    }\n\n    public static ProxyExecutable wrap(Value value) {\n        return new Executable(value, true);\n    }\n\n    public Value getValue() {\n        return value;\n    }\n\n    @Override\n    public void putMember(String key, Value value) {\n        this.value.putMember(key, new JsValue(value).value);\n    }\n\n    @Override\n    public boolean hasMember(String key) {\n        return value.hasMember(key);\n    }\n\n    @Override\n    public Object getMemberKeys() {\n        return value.getMemberKeys().toArray(new String[0]);\n    }\n\n    @Override\n    public Object getMember(String key) {\n        return new JsValue(value.getMember(key)).value;\n    }\n\n    @Override\n    public boolean removeMember(String key) {\n        return value.removeMember(key);\n    }\n\n    public static class Executable extends JsFunction implements ProxyExecutable {\n\n        private final boolean lock;\n        private final String source;\n\n        protected Executable(Value value) {\n            this(value, false);\n        }\n\n        protected Executable(Value value, boolean lock) {\n            super(value);\n            this.lock = lock;\n            source = \"(\" + value.getSourceLocation().getCharacters() + \")\";\n        }\n\n        public Object execute(JsEngine je, Object... args) {\n            Object[] newArgs = new Object[args.length];\n            for (int i = 0; i < newArgs.length; i++) {\n                newArgs[i] = JsValue.fromJava(args[i]);\n            }            \n            Value attached = je.evalForValue(source);\n            return new JsValue(attached.execute(newArgs)).value;\n        }\n\n        @Override\n        public Object execute(Value... args) {\n            Object[] newArgs = new Object[args.length];\n            for (int i = 0; i < newArgs.length; i++) {\n                newArgs[i] = JsValue.fromJava(args[i]);\n            }\n            if (lock) {\n                synchronized (LOCK) {\n                    return new JsValue(value.execute(newArgs)).value;\n                }\n            }\n            ScenarioEngine se = ScenarioEngine.get();\n            JsEngine je = se == null ? null : se.getJsEngine();\n            if (je == null || je.context.equals(value.getContext())) {\n                return new JsValue(value.execute(newArgs)).value;\n            }\n            Value attached = je.evalForValue(source);\n            return new JsValue(attached.execute(newArgs)).value;\n        }\n\n    }\n\n    protected static class Instantiable extends Executable implements ProxyInstantiable {\n\n        protected Instantiable(Value value) {\n            super(value);\n        }\n\n        @Override\n        public Object newInstance(Value... args) {\n            Object[] newArgs = new Object[args.length];\n            for (int i = 0; i < newArgs.length; i++) {\n                newArgs[i] = JsValue.fromJava(args[i]);\n            }\n            return new JsValue(value.execute(newArgs)).value;\n        }\n\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/graal/JsLambda.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.graal;\n\nimport java.util.function.Consumer;\nimport java.util.function.Function;\nimport org.graalvm.polyglot.Value;\n\n/**\n *\n * @author pthomas3\n */\npublic class JsLambda extends JsFunction.Instantiable implements Consumer, Function, Runnable {    \n\n    public JsLambda(Value v) {\n        super(v);\n    }\n\n    @Override\n    public void accept(Object arg) {\n        JsEngine.execute(this, arg);\n    }\n\n    @Override\n    public Object apply(Object arg) {\n        return JsEngine.execute(this, arg);\n    }\n\n    @Override\n    public void run() {\n...\n", "needle_token_start": 4041, "needle_token_end": 4074, "code_context_ntokens": 15954, "output": ["```java\npublic static String karateLocator(String karateRef) {\n    return \"(document._karate.\" + karateRef + \")\";\n}\n```"]}
{"repo": "karatelabs/karate", "name": "setEngine", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/http/WebSocketClient.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The method is designed to assign a specific scenario execution context to the current instance, enabling it to utilize this context in its operations.\n2. **Input**: The method accepts a single parameter that represents the scenario execution context.\n3. **Output**: There is no return value as the method's primary function is to set an internal state.\n4. **Procedure**: The method directly assigns the provided scenario execution context to an internal variable, effectively updating the state of the instance to reflect this new context.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: karate-core/src/main/java/com/intuit/karate/driver/DevToolsWait.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.Logger;\nimport java.util.function.Predicate;\n\n/**\n *\n * @author pthomas3\n */\npublic class DevToolsWait {\n\n    private final DriverOptions options;\n    private final DevToolsDriver driver;\n\n    private DevToolsMessage lastSent;\n    private Predicate<DevToolsMessage> condition;\n    private DevToolsMessage lastReceived;\n\n    private final Predicate<DevToolsMessage> DEFAULT = m -> lastSent.getId().equals(m.getId());\n    \n    public static final Predicate<DevToolsMessage> FRAME_RESIZED = forEvent(\"Page.frameResized\");\n...\n// Path: karate-core/src/main/java/com/intuit/karate/driver/Keys.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class Keys {\n\n    private final Driver driver;\n\n    public Keys(Driver driver) {\n        this.driver = driver;\n    }\n\n    public static Integer code(char c) {\n        return CODES.get(c);\n    }\n    \n    public static String keyValue(char c) {\n        return VALUES.get(c);\n    }\n \n    public static boolean isNormal(char c) {\n        return c < NULL;\n    }\n    \n    public static boolean isModifier(char c) {\n        switch(c) {\n            case CONTROL:\n            case ALT:\n            case SHIFT:\n            case META:\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    public static String keyIdentifier(char c) {\n        return \"\\\\u\" + Integer.toHexString(c | 0x10000).substring(1);\n    }\n\n    public static final char NULL = '\\uE000';\n    public static final char CANCEL = '\\uE001';\n    public static final char HELP = '\\uE002';\n    public static final char BACK_SPACE = '\\uE003';\n    public static final char TAB = '\\uE004';\n    public static final char CLEAR = '\\uE005';\n    public static final char RETURN = '\\uE006';\n    public static final char ENTER = '\\uE007';\n    public static final char SHIFT = '\\uE008';\n    public static final char CONTROL = '\\uE009';\n    public static final char ALT = '\\uE00A';\n    public static final char PAUSE = '\\uE00B';\n    public static final char ESCAPE = '\\uE00C';\n    public static final char SPACE = '\\uE00D';\n    public static final char PAGE_UP = '\\uE00E';\n    public static final char PAGE_DOWN = '\\uE00F';\n    public static final char END = '\\uE010';\n    public static final char HOME = '\\uE011';\n    public static final char LEFT = '\\uE012';\n    public static final char UP = '\\uE013';\n    public static final char RIGHT = '\\uE014';\n    public static final char DOWN = '\\uE015';\n    public static final char INSERT = '\\uE016';\n    public static final char DELETE = '\\uE017';\n    public static final char SEMICOLON = '\\uE018';\n    public static final char EQUALS = '\\uE019';\n\n    // numpad keys\n    public static final char NUMPAD0 = '\\uE01A';\n    public static final char NUMPAD1 = '\\uE01B';\n    public static final char NUMPAD2 = '\\uE01C';\n    public static final char NUMPAD3 = '\\uE01D';\n    public static final char NUMPAD4 = '\\uE01E';\n    public static final char NUMPAD5 = '\\uE01F';\n    public static final char NUMPAD6 = '\\uE020';\n    public static final char NUMPAD7 = '\\uE021';\n    public static final char NUMPAD8 = '\\uE022';\n    public static final char NUMPAD9 = '\\uE023';\n    public static final char MULTIPLY = '\\uE024';\n    public static final char ADD = '\\uE025';\n    public static final char SEPARATOR = '\\uE026';\n    public static final char SUBTRACT = '\\uE027';\n    public static final char DECIMAL = '\\uE028';\n    public static final char DIVIDE = '\\uE029';\n\n    // function keys\n    public static final char F1 = '\\uE031';\n    public static final char F2 = '\\uE032';\n    public static final char F3 = '\\uE033';\n    public static final char F4 = '\\uE034';\n    public static final char F5 = '\\uE035';\n    public static final char F6 = '\\uE036';\n    public static final char F7 = '\\uE037';\n    public static final char F8 = '\\uE038';\n    public static final char F9 = '\\uE039';\n    public static final char F10 = '\\uE03A';\n    public static final char F11 = '\\uE03B';\n    public static final char F12 = '\\uE03C';\n    public static final char META = '\\uE03D';\n\n    private static final Map<Character, Integer> CODES = new HashMap();\n    private static final Map<Character, String> VALUES = new HashMap();\n    \n    private static void put(char c, int code, String value) {\n        CODES.put(c, code);\n        VALUES.put(c, value);\n    }\n    \n    public static final int CODE_SHIFT = 16;\n    public static final int CODE_CONTROL = 17;\n    public static final int CODE_ALT = 18;\n    public static final int CODE_META = 91; // left command key on mac, right is 93\n\n    static {\n        put(CANCEL, 3, \"Cancel\");\n        put(BACK_SPACE, 8, \"Backspace\");\n        put(TAB, 9, \"Tab\");\n        put(CLEAR, 12, \"Clear\");\n        put(NULL, 12, \"Clear\"); // same as clear\n        put(RETURN, 13, \"Enter\"); // same as enter\n        put(ENTER, 13, \"Enter\");\n        put(SHIFT, CODE_SHIFT, \"Shift\");\n        put(CONTROL, CODE_CONTROL, \"Control\");\n        put(ALT, CODE_ALT, \"Alt\");\n        put(PAUSE, 19, \"Pause\");\n        put(ESCAPE, 27, \"Escape\");\n        put(SPACE, 32, \" \");\n        put(PAGE_UP, 33, \"PageUp\");\n        put(PAGE_DOWN, 34, \"PageDown\");\n        put(END, 35, \"End\");\n        put(HOME, 36, \"Home\");\n        put(LEFT, 37, \"ArrowLeft\");\n        put(UP, 38, \"ArrowUp\");\n        put(RIGHT, 39, \"ArrowRight\");\n        put(DOWN, 40, \"ArrowDown\");\n        put(NUMPAD0, 96, \"0\");\n        put(NUMPAD1, 97, \"1\");\n        put(NUMPAD2, 98, \"2\");\n        put(NUMPAD3, 99, \"3\");\n        put(NUMPAD4, 100, \"4\");\n        put(NUMPAD5, 101, \"5\");\n        put(NUMPAD6, 102, \"6\");\n        put(NUMPAD7, 103, \"7\");\n        put(NUMPAD8, 104, \"8\");\n        put(NUMPAD9, 105, \"9\");\n        put(MULTIPLY, 106, \"Multiply\");\n        put(ADD, 107, \"Add\");\n        put(SEPARATOR, 108, \"Separator\");\n        put(SUBTRACT, 109, \"Subtract\");\n        put(DECIMAL, 110, \"Decimal\");\n        put(DIVIDE, 111, \"Divide\");\n        put(F1, 112, \"F1\");\n        put(F2, 113, \"F2\");\n        put(F3, 114, \"F3\");\n        put(F4, 115, \"F4\");\n        put(F5, 116, \"F5\");\n        put(F6, 117, \"F6\");\n        put(F7, 118, \"F7\");\n        put(F8, 119, \"F8\");\n        put(F9, 120, \"F9\");\n        put(F10, 121, \"F10\");\n        put(F11, 122, \"F11\");\n        put(F12, 123, \"F12\");\n        put(DELETE, 127, \"Delete\");\n        put(INSERT, 155, \"Insert\");\n        put(HELP, 156, \"Help\");\n        put(META, CODE_META, \"Meta\");\n        //======================================================================\n        CODES.put(' ', 32);\n        CODES.put(',', 44);\n        CODES.put('-', 45);\n        CODES.put('.', 46);\n        CODES.put('/', 47);\n        CODES.put('0', 48);\n        CODES.put('1', 49);\n        CODES.put('2', 50);\n        CODES.put('3', 51);\n        CODES.put('4', 52);\n        CODES.put('5', 53);\n        CODES.put('6', 54);\n        CODES.put('7', 55);\n        CODES.put('8', 56);\n        CODES.put('9', 57);\n        CODES.put(';', 59);\n        CODES.put('=', 61);\n        CODES.put('A', 65);\n        CODES.put('B', 66);\n        CODES.put('C', 67);\n        CODES.put('D', 68);\n        CODES.put('E', 69);\n        CODES.put('F', 70);\n        CODES.put('G', 71);\n        CODES.put('H', 72);\n        CODES.put('I', 73);\n        CODES.put('J', 74);\n        CODES.put('K', 75);\n        CODES.put('L', 76);\n        CODES.put('M', 77);\n        CODES.put('N', 78);\n        CODES.put('O', 79);\n        CODES.put('P', 80);\n        CODES.put('Q', 81);\n        CODES.put('R', 82);\n        CODES.put('S', 83);\n        CODES.put('T', 84);\n        CODES.put('U', 85);\n        CODES.put('V', 86);\n        CODES.put('W', 87);\n        CODES.put('X', 88);\n        CODES.put('Y', 89);\n        CODES.put('Z', 90);\n        CODES.put('[', 91);\n        CODES.put('\\\\', 92);\n        CODES.put(']', 93);\n        CODES.put('a', 97);\n        CODES.put('b', 98);\n        CODES.put('c', 99);\n        CODES.put('d', 100);\n        CODES.put('e', 101);\n        CODES.put('f', 102);\n        CODES.put('g', 103);\n        CODES.put('h', 104);\n        CODES.put('i', 105);\n        CODES.put('j', 106);\n        CODES.put('k', 107);\n        CODES.put('l', 108);\n        CODES.put('m', 109);\n        CODES.put('n', 110);\n        CODES.put('o', 111);\n        CODES.put('p', 112);\n        CODES.put('q', 113);\n        CODES.put('r', 114);\n        CODES.put('s', 115);\n        CODES.put('t', 116);\n        CODES.put('u', 117);\n        CODES.put('v', 118);\n        CODES.put('w', 119);\n        CODES.put('x', 120);\n        CODES.put('y', 121);\n        CODES.put('z', 122);\n        CODES.put('&', 150);\n        CODES.put('*', 151);\n        CODES.put('\"', 152);\n        CODES.put('<', 153);\n        CODES.put('>', 160);\n        CODES.put('{', 161);\n        CODES.put('}', 162);\n        CODES.put('`', 192);\n        CODES.put('\\'', 222);\n        CODES.put('@', 512);\n        CODES.put(':', 513);\n        CODES.put('$', 515);\n        CODES.put('!', 517);\n        CODES.put('(', 519);\n        CODES.put('#', 520);\n        CODES.put('+', 521);\n        CODES.put(')', 522);\n        CODES.put('_', 523);\n        \n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/WebSocketClient.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\nimport com.intuit.karate.Logger;\nimport com.intuit.karate.core.ScenarioEngine;\nimport io.netty.bootstrap.Bootstrap;\nimport io.netty.buffer.ByteBuf;\nimport io.netty.buffer.Unpooled;\nimport io.netty.channel.Channel;\nimport io.netty.channel.ChannelInitializer;\nimport io.netty.channel.ChannelPipeline;\nimport io.netty.channel.EventLoopGroup;\nimport io.netty.channel.nio.NioEventLoopGroup;\nimport io.netty.channel.socket.nio.NioSocketChannel;\nimport io.netty.handler.codec.http.DefaultHttpHeaders;\nimport io.netty.handler.codec.http.FullHttpRequest;\nimport io.netty.handler.codec.http.HttpClientCodec;\nimport io.netty.handler.codec.http.HttpHeaders;\nimport io.netty.handler.codec.http.HttpObjectAggregator;\nimport io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame;\nimport io.netty.handler.codec.http.websocketx.CloseWebSocketFrame;\nimport io.netty.handler.codec.http.websocketx.PingWebSocketFrame;\nimport io.netty.handler.codec.http.websocketx.TextWebSocketFrame;\nimport io.netty.handler.codec.http.websocketx.WebSocketClientHandshaker;\nimport io.netty.handler.codec.http.websocketx.WebSocketClientHandshakerFactory;\nimport io.netty.handler.codec.http.websocketx.WebSocketFrame;\nimport io.netty.handler.codec.http.websocketx.WebSocketVersion;\nimport io.netty.handler.codec.http.websocketx.extensions.compression.WebSocketClientCompressionHandler;\nimport io.netty.handler.ssl.SslContext;\nimport io.netty.handler.ssl.SslContextBuilder;\nimport io.netty.handler.ssl.util.InsecureTrustManagerFactory;\nimport java.net.URI;\nimport java.util.Map;\nimport java.util.function.Function;\nimport javax.net.ssl.SSLException;\n\n/**\n *\n * @author pthomas3\n */\npublic class WebSocketClient implements WebSocketListener {\n\n    // mutable\n    private Logger logger;\n\n    private final Channel channel;\n    private final EventLoopGroup group;\n\n    private final URI uri;\n    private final int port;\n    private final SslContext sslContext;\n    private final WebSocketClientHandshaker handShaker;\n    private final WebSocketClientHandler handler;\n\n    private Function<String, Boolean> textHandler;\n    private Function<byte[], Boolean> binaryHandler;\n    \n    private ScenarioEngine engine;\n\n    \npublic void setEngine(ScenarioEngine engine) {\n        this.engine = engine;\n    }    \n\n    @Override\n    public void onMessage(String text) {\n        if (textHandler != null) {\n            if (textHandler.apply(text)) {\n                if (engine != null) {\n                    engine.signal(text);\n                }\n            }\n        }\n    }\n\n    @Override\n    public void onMessage(byte[] bytes) {\n        if (binaryHandler != null) {\n            if (binaryHandler.apply(bytes)) {\n                if (engine != null) {\n                    engine.signal(bytes);\n                }\n            }\n        }\n    }\n\n    public void setLogger(Logger logger) {\n        this.logger = logger;\n    }\n    \n    public WebSocketClient(WebSocketOptions options, Logger logger) {\n        this.logger = logger;\n        textHandler = options.getTextHandler();\n        binaryHandler = options.getBinaryHandler();\n        uri = options.getUri();\n        port = options.getPort();\n        group = new NioEventLoopGroup();\n        if (options.isSsl()) {\n            try {\n                sslContext = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build();\n            } catch (SSLException e) {\n                throw new RuntimeException(e);\n            }\n        } else {\n            sslContext = null;\n        }\n        HttpHeaders nettyHeaders = new DefaultHttpHeaders();\n        Map<String, Object> headers = options.getHeaders();\n        if (headers != null) {\n            headers.forEach((k, v) -> nettyHeaders.add(k, v));\n        }\n        handShaker = WebSocketClientHandshakerFactory.newHandshaker(\n                uri, WebSocketVersion.V13, options.getSubProtocol(), true, nettyHeaders, options.getMaxPayloadSize());\n        handler = new WebSocketClientHandler(handShaker, this);\n        try {\n            Bootstrap b = new Bootstrap();\n            b.group(group)\n                    .channel(NioSocketChannel.class)\n                    .handler(new ChannelInitializer() {\n                        @Override\n                        protected void initChannel(Channel c) {\n                            ChannelPipeline p = c.pipeline();\n                            if (sslContext != null) {\n                                p.addLast(sslContext.newHandler(c.alloc(), uri.getHost(), port));\n                            }\n                            p.addLast(new HttpClientCodec());\n                            p.addLast(new HttpObjectAggregator(8192));\n                            p.addLast(WebSocketClientCompressionHandler.INSTANCE);\n                            p.addLast(handler);\n                        }\n                    });\n            channel = b.connect(options.getUri().getHost(), options.getPort()).sync().channel();\n            handler.handshakeFuture().sync();\n        } catch (Exception e) {\n            logger.error(\"websocket client init failed: {}\", e.getMessage());\n            throw new RuntimeException(e);\n        }\n    }\n\n    public void setBinaryHandler(Function<byte[], Boolean> binaryHandler) {\n        this.binaryHandler = binaryHandler;\n    }\n\n    public void setTextHandler(Function<String, Boolean> textHandler) {\n        this.textHandler = textHandler;\n    }\n\n    private boolean waiting;\n\n    public void waitSync() {\n        if (waiting) {\n            return;\n        }\n        try {\n            waiting = true;\n            channel.closeFuture().sync();\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public void close() {\n        channel.writeAndFlush(new CloseWebSocketFrame());\n        waitSync();\n        group.shutdownGracefully();\n    }\n\n    public void ping() {\n        WebSocketFrame frame = new PingWebSocketFrame(Unpooled.wrappedBuffer(new byte[]{8, 1, 8, 1}));\n        channel.writeAndFlush(frame);\n    }\n\n    public void send(String msg) {\n        WebSocketFrame frame = new TextWebSocketFrame(msg);\n        channel.writeAndFlush(frame);\n        if (logger.isTraceEnabled()) {\n            logger.trace(\"sent: {}\", msg);\n        }\n    }\n    \n    public void sendHttpRequest(FullHttpRequest request) {        \n        channel.writeAndFlush(request);\n    }\n\n    public void sendBytes(byte[] msg) {\n        ByteBuf byteBuf = Unpooled.copiedBuffer(msg);\n        BinaryWebSocketFrame frame = new BinaryWebSocketFrame(byteBuf);\n        channel.writeAndFlush(frame);\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/WebSocketOptions.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\nimport java.net.URI;\nimport java.util.Map;\nimport java.util.function.Consumer;\nimport java.util.function.Function;\n\n/**\n *\n * @author pthomas3\n */\npublic class WebSocketOptions {\n\n    private final URI uri;\n    private String subProtocol;\n    private final int port;\n    private final boolean ssl;\n    private Function<String, Boolean> textHandler;\n    private Function<byte[], Boolean> binaryHandler;\n    private Map<String, Object> headers;\n    private int maxPayloadSize = 4194304;\n\n    public WebSocketOptions(String url) {\n        this(url, null);\n    }\n    \n    public WebSocketOptions(String url, Map<String, Object> options) {\n        this.uri = URI.create(url);\n        ssl = \"wss\".equalsIgnoreCase(uri.getScheme());\n        port = uri.getPort() == -1 ? (ssl ? 443 : 80) : uri.getPort();\n        if (options != null) {\n            subProtocol = (String) options.get(\"subProtocol\");\n            Integer temp = (Integer) options.get(\"maxPayloadSize\");\n            if (temp != null) {\n                maxPayloadSize = temp;\n            }\n            headers = (Map) options.get(\"headers\");\n        }\n    }\n\n    public void setTextConsumer(Consumer<String> consumer) {\n        textHandler = t -> {\n            consumer.accept(t);\n            return false; // no async signalling, for normal use, e.g. chrome developer tools\n        };\n    }\n    \n    public URI getUri() {\n        return uri;\n    }\n\n    public int getPort() {\n        return port;\n    }\n\n    public boolean isSsl() {\n        return ssl;\n    }        \n\n    public String getSubProtocol() {\n        return subProtocol;\n    }      \n\n    public void setSubProtocol(String subProtocol) {\n        this.subProtocol = subProtocol;\n    }\n\n    public Function<String, Boolean> getTextHandler() {\n        return textHandler;\n    }        \n\n    public void setTextHandler(Function<String, Boolean> textHandler) {\n        this.textHandler = textHandler;\n    }\n\n    public Function<byte[], Boolean> getBinaryHandler() {\n        return binaryHandler;\n    }\n\n    public void setBinaryHandler(Function<byte[], Boolean> binaryHandler) {\n        this.binaryHandler = binaryHandler;\n    }\n\n    public Map<String, Object> getHeaders() {\n        return headers;\n    }\n\n    public void setHeaders(Map<String, Object> headers) {\n        this.headers = headers;\n    }\n\n    public int getMaxPayloadSize() {\n        return maxPayloadSize;\n    }\n\n    public void setMaxPayloadSize(int maxPayloadSize) {\n        this.maxPayloadSize = maxPayloadSize;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/DevToolsDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.Constants;\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.Json;\nimport com.intuit.karate.JsonUtils;\nimport com.intuit.karate.Logger;\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.core.Feature;\nimport com.intuit.karate.core.FeatureCall;\nimport com.intuit.karate.core.MockHandler;\nimport com.intuit.karate.core.ScenarioEngine;\nimport com.intuit.karate.core.Variable;\nimport com.intuit.karate.graal.JsValue;\nimport com.intuit.karate.http.HttpRequest;\nimport com.intuit.karate.http.ResourceType;\nimport com.intuit.karate.http.Response;\nimport com.intuit.karate.http.WebSocketClient;\nimport com.intuit.karate.http.WebSocketOptions;\nimport com.intuit.karate.shell.Command;\nimport java.util.ArrayList;\nimport java.util.Base64;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.function.Predicate;\n\nimport com.jayway.jsonpath.PathNotFoundException;\nimport org.graalvm.polyglot.Value;\n\n/**\n *\n * @author pthomas3\n */\npublic abstract class DevToolsDriver implements Driver {\n\n    protected final DriverOptions options;\n    protected final Command command;\n    protected final WebSocketClient client;\n    private boolean terminated;\n\n    private final DevToolsWait wait;\n    protected final String rootFrameId;\n\n    private Integer windowId;\n    private String windowState;\n    protected String sessionId;\n    protected String mainFrameId;\n\n    // iframe support\n    private Frame frame;\n    private final Map<String, Integer> frameContexts = new HashMap();\n    private final Map<String, String> frameSessions = new HashMap();\n\n    protected boolean domContentEventFired;\n    protected final Set<String> framesStillLoading = new HashSet();\n    private boolean submit;\n\n    protected String currentDialogText;\n\n    private int nextId;\n\n    public int nextId() {\n        return ++nextId;\n    }\n\n    private MockHandler mockHandler;\n\n    protected final Logger logger;\n\n    protected DevToolsDriver(DriverOptions options, Command command, String webSocketUrl) {\n        logger = options.driverLogger;\n        this.options = options;\n        this.command = command;\n\n        if (options.isRemoteHost()) {\n            String host = options.host;\n            Integer port = options.port;\n            webSocketUrl = webSocketUrl.replace(\"ws://localhost/\", \"ws://\" + host + \":\" + port + \"/\");\n        }\n\n        this.wait = new DevToolsWait(this, options);\n        int pos = webSocketUrl.lastIndexOf('/');\n        rootFrameId = webSocketUrl.substring(pos + 1);\n        mainFrameId = rootFrameId;\n        logger.debug(\"root frame id: {}\", rootFrameId);\n        WebSocketOptions wsOptions = new WebSocketOptions(webSocketUrl);\n        wsOptions.setMaxPayloadSize(options.maxPayloadSize);\n        wsOptions.setTextConsumer(text -> {\n            if (logger.isTraceEnabled()) {\n                logger.trace(\"<< {}\", text);\n            } else {\n                // to avoid swamping the console when large base64 encoded binary responses happen\n                logger.debug(\"<< {}\", StringUtils.truncate(text, 1024, true));\n            }\n            Map<String, Object> map = Json.of(text).value();\n            DevToolsMessage dtm = new DevToolsMessage(this, map);\n            receive(dtm);\n        });\n        client = new WebSocketClient(wsOptions, logger);\n    }\n\n    @Override\n    public Driver timeout(Integer millis) {\n        options.setTimeout(millis);\n        return this;\n    }\n\n    @Override\n    public Driver timeout() {\n        return timeout(null);\n    }\n\n    public DevToolsMessage method(String method) {\n        return new DevToolsMessage(this, method);\n    }\n\n    // this can be used for exploring / sending any raw message !\n    public Map<String, Object> send(Map<String, Object> map) {\n        DevToolsMessage dtm = new DevToolsMessage(this, map);\n        dtm.setId(nextId());\n        return sendAndWait(dtm, null).toMap();\n    }\n\n    public void send(DevToolsMessage dtm) {\n        String json = JsonUtils.toJson(dtm.toMap());\n        logger.debug(\">> {}\", json);\n        client.send(json);\n    }\n\n    public DevToolsMessage sendAndWait(DevToolsMessage dtm, Predicate<DevToolsMessage> condition) {\n        boolean wasSubmit = submit;\n        if (condition == null && submit) {\n            submit = false;\n            condition = DevToolsWait.ALL_FRAMES_LOADED;\n        }\n        // do stuff inside wait to avoid missing messages\n        DevToolsMessage result = wait.send(dtm, condition);\n        if (result == null && !wasSubmit) {\n            if (condition == DevToolsWait.ALL_FRAMES_LOADED) {\n                logger.error(\"failed to get reply for :\" + dtm + \". Will try to check by running a script.\");\n                boolean readyState = (Boolean) this.script(\"document.readyState === 'complete'\");\n                if (!readyState) {\n                    throw new RuntimeException(\"failed to get reply for: \" + dtm);\n                } else {\n                    logger.warn(\"document is ready, but no reply for: \" + dtm + \" with a ready event received. Will proceed.\");\n                }\n            } else {\n                throw new RuntimeException(\"failed to get reply for: \" + dtm);\n            }\n        }\n        return result;\n    }\n\n    public void receive(DevToolsMessage dtm) {\n        if (dtm.methodIs(\"Page.domContentEventFired\")) {\n            domContentEventFired = true;\n            logger.trace(\"** set dom ready flag to true\");\n        }\n        if (dtm.methodIs(\"Page.javascriptDialogOpening\")) {\n            currentDialogText = dtm.getParam(\"message\");\n            // this will stop waiting NOW\n            wait.setCondition(DevToolsWait.DIALOG_OPENING);\n        }\n        if (dtm.methodIs(\"Page.frameStartedLoading\")) {\n            String frameLoadingId = dtm.getParam(\"frameId\");\n            if (rootFrameId.equals(frameLoadingId)) { // root page is loading\n                domContentEventFired = false;\n                framesStillLoading.clear();\n                logger.trace(\"** root frame started loading, cleared all page state: {}\", frameLoadingId);\n            } else {\n                framesStillLoading.add(frameLoadingId);\n                logger.trace(\"** frame started loading, added to in-progress list: {}\", framesStillLoading);\n            }\n        }\n        if (dtm.methodIs(\"Page.frameStoppedLoading\")) {\n            String frameLoadedId = dtm.getParam(\"frameId\");\n            framesStillLoading.remove(frameLoadedId);\n            logger.trace(\"** frame stopped loading: {}, remaining in-progress: {}\", frameLoadedId, framesStillLoading);\n        }\n        if (dtm.methodIs(\"Page.frameNavigated\")) {\n            Frame newFrame = new Frame(dtm.getParam(\"frame.id\"), dtm.getParam(\"frame.url\"), dtm.getParam(\"frame.name\"));\n            logger.trace(\"** detected new frame: {}\", newFrame);\n            if (frame != null && (frame.name.equals(newFrame.name) || frame.url.equals(newFrame.url))) {\n                logger.trace(\"** auto switching frame: {} -> {}\", frame, newFrame);\n                frame = newFrame;\n            }\n        }\n        if (dtm.methodIs(\"Runtime.executionContextCreated\")) {\n            String newFrameId = dtm.getParam(\"context.auxData.frameId\");\n            Integer contextId = dtm.getParam(\"context.id\");\n            frameContexts.put(newFrameId, contextId);\n            logger.trace(\"** new frame execution context: {} - {}\", newFrameId, contextId);\n        }\n        if (dtm.methodIs(\"Runtime.executionContextsCleared\")) {\n            frame = null;\n            frameContexts.clear();\n            framesStillLoading.clear();\n        }\n        if (dtm.methodIs(\"Runtime.consoleAPICalled\") && options.showBrowserLog) {\n            List<Object> values = dtm.getParam(\"args[*].value\");\n            for (Object value : values) {\n                logger.debug(\"[console] {}\", value);\n            }\n        }\n        if (dtm.methodIs(\"Fetch.requestPaused\")) {\n            handleInterceptedRequest(dtm);\n        }\n        if (dtm.methodIs(\"Target.targetInfoChanged\")) {\n            String frameId = dtm.getParam(\"targetInfo.targetId\");\n            Integer frameContextId = frameContexts.get(frameId);\n            if (frameContextId != null) {\n                logger.trace(\"** removed stale execution context: {} - {}\", frameId, frameContextId);\n                frameContexts.remove(frameId);\n            }\n        }\n        if (dtm.methodIs(\"Target.attachedToTarget\")) {\n            String targetType = dtm.getParam(\"targetInfo.type\");\n            if (\"iframe\".equals(targetType) || \"frame\".equals(targetType)) {\n                String targetSessionId = dtm.getParam(\"sessionId\");\n                String targetFrameId = dtm.getParam(\"targetInfo.targetId\");\n                frameSessions.put(targetFrameId, targetSessionId);\n                logger.trace(\"** new frame session: {} - {}\", targetFrameId, targetSessionId);\n            }\n        }\n        // all needed state is set above before we get into conditional checks\n        wait.receive(dtm);\n    }\n\n    private void handleInterceptedRequest(DevToolsMessage dtm) {\n        String requestId = dtm.getParam(\"requestId\");\n        String requestUrl = dtm.getParam(\"request.url\");\n        if (mockHandler != null) {\n            String method = dtm.getParam(\"request.method\");\n            Map<String, String> headers = dtm.getParam(\"request.headers\");\n            String postData = dtm.getParam(\"request.postData\");\n            logger.debug(\"intercepting request for url: {}\", requestUrl);\n            HttpRequest request = new HttpRequest();\n            request.setUrl(requestUrl);\n            request.setMethod(method);\n            headers.forEach((k, v) -> request.putHeader(k, v));\n            if (postData != null) {\n                request.setBody(FileUtils.toBytes(postData));\n            } else {\n                request.setBody(Constants.ZERO_BYTES);\n            }\n            Response response = mockHandler.handle(request.toRequest());\n            String responseBody = response.getBody() == null ? \"\" : Base64.getEncoder().encodeToString(response.getBody());\n            List<Map> responseHeaders = new ArrayList();\n            Map<String, List<String>> map = response.getHeaders();\n            if (map != null) {\n                map.forEach((k, v) -> {\n                    if (v != null && !v.isEmpty()) {\n                        Map<String, Object> nv = new HashMap(2);\n                        nv.put(\"name\", k);\n                        nv.put(\"value\", v.get(0));\n                        responseHeaders.add(nv);\n                    }\n                });\n            }\n            method(\"Fetch.fulfillRequest\")\n                    .param(\"requestId\", requestId)\n                    .param(\"responseCode\", response.getStatus())\n                    .param(\"responseHeaders\", responseHeaders)\n                    .param(\"body\", responseBody).sendWithoutWaiting();\n        } else {\n            logger.warn(\"no mock server, continuing paused request to url: {}\", requestUrl);\n            method(\"Fetch.continueRequest\").param(\"requestId\", requestId).sendWithoutWaiting();\n        }\n    }\n\n    //==========================================================================\n    //\n    private DevToolsMessage evalOnce(String expression, boolean quickly, boolean fireAndForget, boolean returnByValue) {\n        DevToolsMessage toSend = method(\"Runtime.evaluate\")\n                .param(\"expression\", expression);\n        if (returnByValue) {\n            toSend.param(\"returnByValue\", true);\n        }\n        Integer contextId = getFrameContext();\n        if (contextId != null) {\n            toSend.param(\"contextId\", contextId);\n        }\n        if (quickly) {\n            toSend.setTimeout(options.getRetryInterval());\n        }\n        if (fireAndForget) {\n            toSend.sendWithoutWaiting();\n            return null;\n        }\n        return toSend.send();\n    }\n\n    protected DevToolsMessage eval(String expression) {\n        return evalInternal(expression, false, true);\n    }\n\n    protected DevToolsMessage evalQuickly(String expression) {\n        return evalInternal(expression, true, true);\n    }\n\n    protected String evalForObjectId(String expression) {\n        return options.retry(() -> {\n            DevToolsMessage dtm = evalInternal(expression, true, false);\n            return dtm.getResult(\"objectId\");\n        }, returned -> returned != null, \"eval for object id: \" + expression, true);\n    }\n\n    private DevToolsMessage evalInternal(String expression, boolean quickly, boolean returnByValue) {\n        DevToolsMessage dtm = evalOnce(expression, quickly, false, returnByValue);\n        if (dtm.isResultError()) {\n            Map<String, Object> error = dtm.getError();\n            if (error != null) {\n                Object errorCode = error.get(\"code\");\n                if (errorCode instanceof Integer) {\n                    if ((Integer) errorCode == -32000) { // Object reference chain is too long\n                        dtm.setResult(Variable.NULL);\n                        return dtm;\n                    }\n                }\n            }\n            String message = \"js eval failed once:\" + expression\n                    + \", error: \" + dtm.getResult();\n            logger.warn(message);\n            options.sleep();\n            dtm = evalOnce(expression, quickly, false, returnByValue); // no wait condition for the re-try\n            if (dtm.isResultError()) {\n                message = \"js eval failed twice:\" + expression\n                        + \", error: \" + dtm.getResult();\n                logger.error(message);\n                throw new RuntimeException(message);\n            }\n        }\n        return dtm;\n    }\n\n    protected void retryIfEnabled(String locator) {\n        if (options.isRetryEnabled()) {\n            waitFor(locator); // will throw exception if not found\n        }\n        if (options.highlight) {\n            // highlight(locator, options.highlightDuration); // instead of this\n            String highlightJs = options.highlight(locator, options.highlightDuration);\n            evalOnce(highlightJs, true, true, true); // do it safely, i.e. fire and forget\n        }\n    }\n\n    protected int getRootNodeId() {\n        return method(\"DOM.getDocument\").param(\"depth\", 0).send().getResult(\"root.nodeId\");\n    }\n\n    @Override\n    public String elementId(String locator) {\n        return evalForObjectId(DriverOptions.selector(locator));\n    }\n\n    @Override\n    public List elementIds(String locator) {\n        List<Element> elements = locateAll(locator);\n        List<String> objectIds = new ArrayList(elements.size());\n        for (Element e : elements) {\n            String objectId = evalForObjectId(e.getLocator());\n            objectIds.add(objectId);\n        }\n        return objectIds;\n    }\n\n    @Override\n    public DriverOptions getOptions() {\n        return options;\n    }\n\n    private void attachAndActivate(String targetId, boolean isFrame) {\n        DevToolsMessage dtm = method(\"Target.attachToTarget\").param(\"targetId\", targetId).param(\"flatten\", true).send();\n        sessionId = dtm.getResult(\"sessionId\");\n        frameSessions.put(targetId, sessionId);\n        if (!isFrame) {\n            mainFrameId = targetId;\n        }\n        method(\"Target.activateTarget\").param(\"targetId\", targetId).send();\n        method(\"Target.setDiscoverTargets\").param(\"discover\", true).send();\n    }\n\n    @Override\n    public void activate() {\n        attachAndActivate(rootFrameId, false);\n    }\n\n    protected void initWindowIdAndState() {\n        DevToolsMessage dtm = method(\"Browser.getWindowForTarget\").param(\"targetId\", rootFrameId).send();\n        if (!dtm.isResultError()) {\n            windowId = dtm.getResultVariable(\"windowId\").getValue();\n            windowState = (String) dtm.getResultVariable(\"bounds\").<Map>getValue().get(\"windowState\");\n        }\n    }\n\n    @Override\n    public Map<String, Object> getDimensions() {\n        DevToolsMessage dtm = method(\"Browser.getWindowForTarget\").param(\"targetId\", rootFrameId).send();\n        Map<String, Object> map = dtm.getResultVariable(\"bounds\").getValue();\n        Integer x = (Integer) map.remove(\"left\");\n        Integer y = (Integer) map.remove(\"top\");\n        map.put(\"x\", x);\n        map.put(\"y\", y);\n        return map;\n    }\n\n    @Override\n    public void setDimensions(Map<String, Object> map) {\n        Integer left = (Integer) map.remove(\"x\");\n        Integer top = (Integer) map.remove(\"y\");\n        map.put(\"left\", left);\n        map.put(\"top\", top);\n        Map temp = getDimensions();\n        temp.putAll(map);\n        temp.remove(\"windowState\");\n        method(\"Browser.setWindowBounds\")\n                .param(\"windowId\", windowId)\n                .param(\"bounds\", temp).send();\n    }\n\n    public void emulateDevice(int width, int height, String userAgent) {\n        method(\"Network.setUserAgentOverride\").param(\"userAgent\", userAgent).send();\n        method(\"Emulation.setDeviceMetricsOverride\")\n                .param(\"width\", width)\n                .param(\"height\", height)\n                .param(\"deviceScaleFactor\", 1)\n                .param(\"mobile\", true)\n                .send();\n    }\n\n    @Override\n    public void close() {\n        method(\"Page.close\").sendWithoutWaiting();\n        sessionId = frameSessions.get(rootFrameId);\n    }\n\n    @Override\n    public void quit() {\n        if (terminated) {\n            return;\n        }\n        terminated = true;\n        // don't wait, may fail and hang\n        method(\"Target.closeTarget\").param(\"targetId\", rootFrameId).sendWithoutWaiting();\n        // method(\"Browser.close\").send();\n        client.close();\n        if (command != null) {\n            command.close(true);\n        }\n        getRuntime().engine.setDriverToNull();\n    }\n\n    @Override\n    public boolean isTerminated() {\n        return terminated;\n    }\n\n    @Override\n    public void setUrl(String url) {\n        method(\"Page.navigate\").param(\"url\", url)\n                .send(DevToolsWait.ALL_FRAMES_LOADED);\n    }\n\n    @Override\n    public void refresh() {\n        method(\"Page.reload\").send(DevToolsWait.ALL_FRAMES_LOADED);\n    }\n\n    @Override\n    public void reload() {\n        method(\"Page.reload\").param(\"ignoreCache\", true).send();\n    }\n\n    private void history(int delta) {\n        DevToolsMessage dtm = method(\"Page.getNavigationHistory\").send();\n        int currentIndex = dtm.getResultVariable(\"currentIndex\").getValue();\n        List<Map> list = dtm.getResultVariable(\"entries\").getValue();\n        int targetIndex = currentIndex + delta;\n        if (targetIndex < 0 || targetIndex == list.size()) {\n            return;\n        }\n        Map<String, Object> entry = list.get(targetIndex);\n        Integer id = (Integer) entry.get(\"id\");\n        method(\"Page.navigateToHistoryEntry\").param(\"entryId\", id).send(DevToolsWait.ALL_FRAMES_LOADED);\n    }\n\n    @Override\n    public void back() {\n        history(-1);\n    }\n\n    @Override\n    public void forward() {\n        history(1);\n    }\n\n    private void setWindowState(String state) {\n        if (!\"normal\".equals(windowState)) {\n            method(\"Browser.setWindowBounds\")\n                    .param(\"windowId\", windowId)\n                    .param(\"bounds\", Collections.singletonMap(\"windowState\", \"normal\"))\n                    .send();\n            windowState = \"normal\";\n        }\n        if (!state.equals(windowState)) {\n            method(\"Browser.setWindowBounds\")\n                    .param(\"windowId\", windowId)\n                    .param(\"bounds\", Collections.singletonMap(\"windowState\", state))\n                    .send();\n            windowState = state;\n        }\n    }\n\n    @Override\n    public void maximize() {\n        setWindowState(\"maximized\");\n    }\n\n    @Override\n    public void minimize() {\n        setWindowState(\"minimized\");\n    }\n\n    @Override\n    public void fullscreen() {\n        setWindowState(\"fullscreen\");\n    }\n\n    @Override\n    public Element click(String locator) {\n        retryIfEnabled(locator);\n        eval(DriverOptions.selector(locator) + \".click()\");\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @Override\n    public Element select(String locator, String text) {\n        retryIfEnabled(locator);\n        eval(options.optionSelector(locator, text));\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @Override\n    public Element select(String locator, int index) {\n        retryIfEnabled(locator);\n        eval(options.optionSelector(locator, index));\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @Override\n    public Driver submit() {\n        submit = true;\n        return this;\n    }\n\n    @Override\n    public Element focus(String locator) {\n        retryIfEnabled(locator);\n        eval(options.focusJs(locator));\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @Override\n    public Element clear(String locator) {\n        eval(DriverOptions.selector(locator) + \".value = ''\");\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    private void sendKey(Character c, int modifiers, String type, Integer keyCode) {\n        DevToolsMessage dtm = method(\"Input.dispatchKeyEvent\")\n                .param(\"modifiers\", modifiers)\n                .param(\"type\", type);\n        if (keyCode == null) {\n            if (c != null) {\n                dtm.param(\"text\", c.toString());\n            }\n        } else {\n            switch (keyCode) {\n                case 9: // tab\n                    if (\"char\".equals(type)) {\n                        return; // special case\n                    }\n                    dtm.param(\"text\", \"\");\n                    break;\n                case 13: // enter\n                    dtm.param(\"text\", \"\\r\"); // important ! \\n does NOT work for chrome\n                    break;\n                case 46: // dot\n                    if (\"rawKeyDown\".equals(type)) {\n                        dtm.param(\"type\", \"keyDown\"); // special case\n                    }\n                    dtm.param(\"text\", \".\");\n                    break;\n                default:\n                    if (c != null) {\n                        dtm.param(\"text\", c.toString());\n                    }\n            }\n            dtm.param(\"windowsVirtualKeyCode\", keyCode);\n        }\n        dtm.send();\n    }\n\n    @Override\n    public Element input(String locator, String value) {\n        retryIfEnabled(locator);\n        // focus\n        eval(options.focusJs(locator));\n        Input input = new Input(value);\n        while (input.hasNext()) {\n            char c = input.next();\n            int modifiers = input.getModifierFlags();\n            Integer keyCode = Keys.code(c);\n            if (keyCode != null) {\n                switch (keyCode) {\n                    case Keys.CODE_SHIFT:\n                    case Keys.CODE_CONTROL:\n                    case Keys.CODE_ALT:\n                    case Keys.CODE_META:\n                        if (input.release) {\n                            sendKey(null, modifiers, \"keyUp\", keyCode);\n                        } else {\n                            sendKey(null, modifiers, \"rawKeyDown\", keyCode);\n                        }\n                        break;\n                    default:\n                        sendKey(c, modifiers, \"rawKeyDown\", keyCode);\n                        sendKey(c, modifiers, \"char\", keyCode);\n                        sendKey(c, modifiers, \"keyUp\", keyCode);\n                }\n            } else {\n                logger.warn(\"unknown character / key code: {}\", c);\n                sendKey(c, modifiers, \"char\", null);\n            }\n        }\n        for (int keyCode : input.getKeyCodesToRelease()) {\n            sendKey(null, 0, \"keyUp\", keyCode);\n        }\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    protected int currentMouseXpos;\n    protected int currentMouseYpos;\n\n    @Override\n    public void actions(List<Map<String, Object>> sequence) {\n        boolean submitRequested = submit;\n        submit = false; // make sure only LAST action is handled as a submit()\n        for (Map<String, Object> map : sequence) {\n            List<Map<String, Object>> actions = (List) map.get(\"actions\");\n            if (actions == null) {\n                logger.warn(\"no actions property found: {}\", sequence);\n                return;\n            }\n            Iterator<Map<String, Object>> iterator = actions.iterator();\n            while (iterator.hasNext()) {\n                Map<String, Object> action = iterator.next();\n                String type = (String) action.get(\"type\");\n                if (type == null) {\n                    logger.warn(\"no type property found: {}\", action);\n                    continue;\n                }\n                String chromeType;\n                switch (type) {\n                    case \"pointerMove\":\n                        chromeType = \"mouseMoved\";\n                        break;\n                    case \"pointerDown\":\n                        chromeType = \"mousePressed\";\n                        break;\n                    case \"pointerUp\":\n                        chromeType = \"mouseReleased\";\n                        break;\n                    default:\n                        logger.warn(\"unexpected action type: {}\", action);\n                        continue;\n                }\n                Integer x = (Integer) action.get(\"x\");\n                Integer y = (Integer) action.get(\"y\");\n                if (x != null) {\n                    currentMouseXpos = x;\n                }\n                if (y != null) {\n                    currentMouseYpos = y;\n                }\n                Integer duration = (Integer) action.get(\"duration\");\n                DevToolsMessage toSend = method(\"Input.dispatchMouseEvent\")\n                        .param(\"type\", chromeType)\n                        .param(\"x\", currentMouseXpos).param(\"y\", currentMouseYpos);\n                if (\"mousePressed\".equals(chromeType) || \"mouseReleased\".equals(chromeType)) {\n                    toSend.param(\"button\", \"left\").param(\"clickCount\", 1);\n                }\n                if (!iterator.hasNext() && submitRequested) {\n                    submit = true;\n                }\n                toSend.send();\n                if (duration != null) {\n                    options.sleep(duration);\n                }\n            }\n        }\n    }\n\n    @Override\n    public String text(String id) {\n        return property(id, \"textContent\");\n    }\n\n    @Override\n    public String html(String id) {\n        return property(id, \"outerHTML\");\n    }\n\n    @Override\n    public String value(String locator) {\n        return property(locator, \"value\");\n    }\n\n    @Override\n    public Element value(String locator, String value) {\n        retryIfEnabled(locator);\n        eval(DriverOptions.selector(locator) + \".value = '\" + value + \"'\");\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @Override\n    public String attribute(String id, String name) {\n        retryIfEnabled(id);\n        DevToolsMessage dtm = eval(DriverOptions.selector(id) + \".getAttribute('\" + name + \"')\");\n        return dtm.getResult().getAsString();\n    }\n\n    @Override\n    public String property(String id, String name) {\n        retryIfEnabled(id);\n        DevToolsMessage dtm = eval(DriverOptions.selector(id) + \"['\" + name + \"']\");\n        return dtm.getResult().getAsString();\n    }\n\n    @Override\n    public boolean enabled(String id) {\n        retryIfEnabled(id);\n        DevToolsMessage dtm = eval(DriverOptions.selector(id) + \".disabled\");\n        return !dtm.getResult().isTrue();\n    }\n\n    @Override\n    public boolean waitUntil(String expression) {\n        return options.retry(() -> {\n            try {\n                return evalQuickly(expression).getResult().isTrue();\n            } catch (Exception e) {\n                logger.warn(\"waitUntil evaluate failed: {}\", e.getMessage());\n                return false;\n            }\n        }, b -> b, \"waitUntil (js)\", true);\n    }\n\n    @Override\n    public Object script(String expression) {\n        return eval(expression).getResult().getValue();\n    }\n\n    @Override\n    public String getTitle() {\n        return eval(\"document.title\").getResult().getAsString();\n    }\n\n    @Override\n    public String getUrl() {\n        return eval(\"document.location.href\").getResult().getAsString();\n    }\n\n    @Override\n    public List<Map> getCookies() {\n        DevToolsMessage dtm = method(\"Network.getAllCookies\").send();\n        return dtm.getResultVariable(\"cookies\").getValue();\n    }\n\n    @Override\n    public Map<String, Object> cookie(String name) {\n        List<Map> list = getCookies();\n        if (list == null) {\n            return null;\n        }\n        for (Map<String, Object> map : list) {\n            if (map != null && name.equals(map.get(\"name\"))) {\n                return map;\n            }\n        }\n        return null;\n    }\n\n    @Override\n    public void cookie(Map<String, Object> cookie) {\n        if (cookie.get(\"url\") == null && cookie.get(\"domain\") == null) {\n            cookie = new HashMap(cookie); // don't mutate test\n            cookie.put(\"url\", getUrl());\n        }\n        method(\"Network.setCookie\").params(cookie).send();\n    }\n\n    @Override\n    public void deleteCookie(String name) {\n        method(\"Network.deleteCookies\").param(\"name\", name).param(\"url\", getUrl()).send();\n    }\n\n    @Override\n    public void clearCookies() {\n...\n", "needle_token_start": 5625, "needle_token_end": 5647, "code_context_ntokens": 16125, "output": ["```java\npublic void setEngine(ScenarioEngine engine) {\n    this.engine = engine;\n}\n```"]}
{"repo": "karatelabs/karate", "name": "execInternal", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/core/ScenarioBridge.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to execute a command based on provided options and collect the output.\n2. **Input**: A map containing command execution parameters such as a single command line or a list of arguments.\n3. **Output**: A string representing the collected output from the executed command.\n4. **Procedure**: The function initiates a command execution process with the given parameters, waits for the command to complete, and then collects and returns the output from the command.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: karate-core/src/main/java/com/intuit/karate/template/KarateAttributeTagProcessor.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport com.intuit.karate.graal.JsValue;\nimport java.util.Map;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.thymeleaf.context.ITemplateContext;\nimport org.thymeleaf.engine.AttributeName;\nimport org.thymeleaf.model.IProcessableElementTag;\nimport org.thymeleaf.processor.element.AbstractAttributeTagProcessor;\nimport org.thymeleaf.processor.element.IElementTagStructureHandler;\nimport org.thymeleaf.standard.processor.StandardConditionalFixedValueTagProcessor;\n...\n// Path: karate-core/src/main/java/com/intuit/karate/template/StringTemplateResource.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport java.io.IOException;\nimport java.io.Reader;\nimport java.io.StringReader;\nimport org.thymeleaf.templateresource.ITemplateResource;\n\n/**\n *\n * @author pthomas3\n */\npublic class StringTemplateResource implements ITemplateResource {    \n    \n    private final String text;\n    \n    public StringTemplateResource(String text) {\n        this.text = text;\n    }\n\n    @Override\n    public String getDescription() {\n        return getBaseName();\n    }\n\n    @Override\n    public String getBaseName() {\n        return getClass().getName();\n    }\n\n    @Override\n    public boolean exists() {\n        return true;\n    }\n\n    @Override\n    public Reader reader() throws IOException {\n        return new StringReader(text);\n    }\n\n    @Override\n    public ITemplateResource relative(String relativeLocation) {\n        throw new UnsupportedOperationException(\"relative: \" + relativeLocation + \" - not implemented\");\n    }\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/template/KarateServerDialect.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport com.intuit.karate.http.ServerConfig;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.thymeleaf.dialect.AbstractProcessorDialect;\nimport org.thymeleaf.processor.IProcessor;\n\n/**\n *\n * @author pthomas3\n */\npublic class KarateServerDialect extends AbstractProcessorDialect {\n\n    private final ServerConfig config;\n\n    public KarateServerDialect(ServerConfig config) {\n        super(\"karate\", \"ka\", 2000); // has to be processed after standard (default) dialect which is 1000\n        this.config = config;\n    }\n\n    @Override\n    public Set<IProcessor> getProcessors(String dialectPrefix) {\n        Set<IProcessor> ps = new HashSet();\n        ps.add(new KaScriptAttrProcessor(dialectPrefix, config));\n        ps.add(new KaScriptElemProcessor(dialectPrefix));\n        ps.add(new KaSetElemProcessor(dialectPrefix));\n        ps.add(new KaLinkAttrProcessor(dialectPrefix, config));\n        ps.add(new KaHxMethodAttrProcessor(dialectPrefix, \"get\", config));\n        ps.add(new KaHxMethodAttrProcessor(dialectPrefix, \"post\", config));\n        ps.add(new KaHxMethodAttrProcessor(dialectPrefix, \"put\", config));\n        ps.add(new KaHxMethodAttrProcessor(dialectPrefix, \"patch\", config));\n        ps.add(new KaHxMethodAttrProcessor(dialectPrefix, \"delete\", config));\n        ps.add(new KaHxValsAttrProcessor(dialectPrefix));\n        return ps;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/template/KaHxValsAttrProcessor.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport com.intuit.karate.graal.JsValue;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.thymeleaf.context.ITemplateContext;\nimport org.thymeleaf.engine.AttributeName;\nimport org.thymeleaf.model.AttributeValueQuotes;\nimport org.thymeleaf.model.IProcessableElementTag;\nimport org.thymeleaf.processor.element.AbstractAttributeTagProcessor;\nimport org.thymeleaf.processor.element.IElementTagStructureHandler;\nimport org.thymeleaf.templatemode.TemplateMode;\n\n/**\n *\n * @author pthomas3\n */\npublic class KaHxValsAttrProcessor extends AbstractAttributeTagProcessor {\n\n    private static final Logger logger = LoggerFactory.getLogger(KaHxValsAttrProcessor.class);\n\n    public KaHxValsAttrProcessor(String dialectPrefix) {\n        super(TemplateMode.HTML, dialectPrefix, null, false, \"vals\", true, 1000, true);\n    }\n\n    @Override\n    protected void doProcess(ITemplateContext ctx, IProcessableElementTag tag, AttributeName an, String av, IElementTagStructureHandler sh) {\n        JsValue jv = KarateEngineContext.get().evalLocalAsObject(av);\n        if (!jv.isObject()) {\n            logger.warn(\"value did not evaluate to json: {}\", av);\n        } else {\n            sh.setAttribute(\"hx-vals\", jv.toJsonOrXmlString(false), AttributeValueQuotes.SINGLE);\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/Channel.java\n/*\n * The MIT License\n *\n * Copyright 2023 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.Map;\n\n/**\n *\n * @author peter\n */\npublic interface Channel {\n    \n    void produce(ScenarioRuntime runtime);\n    \n    ChannelSession consume(ScenarioRuntime runtime);\n    \n    void register(ScenarioRuntime runtime, Map<String, Object> data);\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ScenarioBridge.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.Json;\nimport com.intuit.karate.JsonUtils;\nimport com.intuit.karate.KarateException;\nimport com.intuit.karate.Logger;\nimport com.intuit.karate.Match;\nimport com.intuit.karate.MatchStep;\nimport com.intuit.karate.PerfContext;\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.XmlUtils;\nimport com.intuit.karate.graal.JsEngine;\nimport com.intuit.karate.graal.JsFunction;\nimport com.intuit.karate.graal.JsLambda;\nimport com.intuit.karate.graal.JsList;\nimport com.intuit.karate.graal.JsMap;\nimport com.intuit.karate.graal.JsValue;\nimport com.intuit.karate.http.HttpClient;\nimport com.intuit.karate.http.HttpRequest;\nimport com.intuit.karate.http.HttpRequestBuilder;\nimport com.intuit.karate.http.ResourceType;\nimport com.intuit.karate.http.WebSocketClient;\nimport com.intuit.karate.http.WebSocketOptions;\nimport com.intuit.karate.shell.Command;\nimport java.io.File;\nimport java.io.InputStream;\nimport java.net.URLDecoder;\nimport java.net.URLEncoder;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.TreeMap;\nimport java.util.function.Function;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport org.graalvm.polyglot.Value;\nimport org.graalvm.polyglot.proxy.ProxyExecutable;\n\n/**\n *\n * @author pthomas3\n */\npublic class ScenarioBridge implements PerfContext {\n\n    private final ScenarioEngine ENGINE;\n\n    protected ScenarioBridge(ScenarioEngine engine) {\n        ENGINE = engine;\n    }\n\n    public void abort() {\n        getEngine().setAborted(true);\n    }\n\n    public Object append(Value... vals) {\n        List list = new ArrayList();\n        JsList jsList = new JsList(list);\n        if (vals.length == 0) {\n            return jsList;\n        }\n        Value val = vals[0];\n        if (val.hasArrayElements()) {\n            list.addAll(val.as(List.class));\n        } else {\n            list.add(val.as(Object.class));\n        }\n        if (vals.length == 1) {\n            return jsList;\n        }\n        for (int i = 1; i < vals.length; i++) {\n            Value v = vals[i];\n            if (v.hasArrayElements()) {\n                list.addAll(v.as(List.class));\n            } else {\n                list.add(v.as(Object.class));\n            }\n        }\n        return jsList;\n    }\n\n    private Object appendToInternal(String varName, Value... vals) {\n        ScenarioEngine engine = getEngine();\n        Variable var = engine.vars.get(varName);\n        if (!var.isList()) {\n            return null;\n        }\n        List list = var.getValue();\n        for (Value v : vals) {\n            if (v.hasArrayElements()) {\n                list.addAll(v.as(List.class));\n            } else {\n                Object temp = v.as(Object.class);\n                list.add(temp);\n            }\n        }\n        engine.setVariable(varName, list);\n        return new JsList(list);\n    }\n\n    public Object appendTo(Value ref, Value... vals) {\n        if (ref.isString()) {\n            return appendToInternal(ref.asString(), vals);\n        }\n        List list;\n        if (ref.hasArrayElements()) {\n            list = new JsValue(ref).getAsList(); // make sure we unwrap the \"original\" list\n        } else {\n            list = new ArrayList();\n        }\n        for (Value v : vals) {\n            if (v.hasArrayElements()) {\n                list.addAll(v.as(List.class));\n            } else {\n                Object temp = v.as(Object.class);\n                list.add(temp);\n            }\n        }\n        return new JsList(list);\n    }\n\n    public Object call(String fileName) {\n        return call(false, fileName, null);\n    }\n\n    public Object call(String fileName, Value arg) {\n        return call(false, fileName, arg);\n    }\n\n    public Object call(boolean sharedScope, String fileName) {\n        return call(sharedScope, fileName, null);\n    }\n\n    public Object call(boolean sharedScope, String fileName, Value arg) {\n        ScenarioEngine engine = getEngine();\n        Variable called = new Variable(engine.fileReader.readFile(fileName));\n        Variable result = engine.call(called, arg == null ? null : new Variable(arg), sharedScope);\n        if (sharedScope) {\n            if (result.isMap()) {\n                engine.setVariables(result.getValue());\n            }\n        }\n        return JsValue.fromJava(result.getValue());\n    }\n\n    private static Object callSingleResult(ScenarioEngine engine, Object o) throws Exception {\n        if (o instanceof Exception) {\n            engine.logger.warn(\"callSingle() cached result is an exception\");\n            throw (Exception) o;\n        }\n        // clone so that threads see the same data snapshot\n        // we also attach js functions\n        o = engine.JS.attachAll(o);\n        return JsValue.fromJava(o);\n    }\n\n    public Object callSingle(String fileName) throws Exception {\n        return callSingle(fileName, null);\n    }\n\n    public Object callSingle(String fileName, Value arg) throws Exception {\n        ScenarioEngine engine = getEngine();\n        final Map<String, Object> CACHE = engine.runtime.featureRuntime.suite.callSingleCache;\n        int minutes = engine.getConfig().getCallSingleCacheMinutes();\n        if ((minutes == 0) && CACHE.containsKey(fileName)) {\n            engine.logger.trace(\"callSingle cache hit: {}\", fileName);\n            return callSingleResult(engine, CACHE.get(fileName));\n        }\n        long startTime = System.currentTimeMillis();\n        engine.logger.trace(\"callSingle waiting for lock: {}\", fileName);\n        synchronized (CACHE) { // lock\n            if ((minutes == 0) && CACHE.containsKey(fileName)) { // retry\n                long endTime = System.currentTimeMillis() - startTime;\n                engine.logger.warn(\"this thread waited {} milliseconds for callSingle lock: {}\", endTime, fileName);\n                return callSingleResult(engine, CACHE.get(fileName));\n            }\n            // this thread is the 'winner'\n            engine.logger.info(\">> lock acquired, begin callSingle: {}\", fileName);\n            Object result = null;\n            File cacheFile = null;\n            if (minutes > 0) {\n                String cleanedName = StringUtils.toIdString(fileName);\n                String cacheFileName = engine.getConfig().getCallSingleCacheDir() + File.separator + cleanedName + \".txt\";\n                cacheFile = new File(cacheFileName);\n                long since = System.currentTimeMillis() - minutes * 60 * 1000;\n                if (cacheFile.exists()) {\n                    long lastModified = cacheFile.lastModified();\n                    if (lastModified > since) {\n                        String json = FileUtils.toString(cacheFile);\n                        result = JsonUtils.fromJson(json);\n                        engine.logger.info(\"callSingleCache hit: {}\", cacheFile);\n                    } else {\n                        engine.logger.info(\"callSingleCache stale, last modified {} - is before {} (minutes: {})\",\n                                lastModified, since, minutes);\n                    }\n                } else {\n                    engine.logger.info(\"callSingleCache file does not exist, will create: {}\", cacheFile);\n                }\n            }\n            if (result == null) {\n                Variable called = new Variable(read(fileName));\n                Variable argVar;\n                if (arg == null || arg.isNull()) {\n                    argVar = null;\n                } else {\n                    argVar = new Variable(arg);\n                }\n                Variable resultVar;\n                try {\n                    resultVar = engine.call(called, argVar, false);\n                } catch (Exception e) {\n                    // don't retain any vestiges of graal-js \n                    RuntimeException re = new RuntimeException(e.getMessage());\n                    // we do this so that an exception is also \"cached\"\n                    resultVar = new Variable(re); // will be thrown at end\n                    engine.logger.warn(\"callSingle() will cache an exception\");\n                }\n                if (minutes > 0) { // cacheFile will be not null\n                    if (resultVar.isMapOrList()) {\n                        String json = resultVar.getAsString();\n                        FileUtils.writeToFile(cacheFile, json);\n                        engine.logger.info(\"callSingleCache write: {}\", cacheFile);\n                    } else {\n                        engine.logger.warn(\"callSingleCache write failed, not json-like: {}\", resultVar);\n                    }\n                }\n                result = resultVar.getValue();\n            }\n            CACHE.put(fileName, result);\n            engine.logger.info(\"<< lock released, cached callSingle: {}\", fileName);\n            return callSingleResult(engine, result);\n        }\n    }\n\n    public Object callonce(String path) {\n        return callonce(false, path);\n    }\n\n    public Object callonce(boolean sharedScope, String path) {\n        String exp = \"read('\" + path + \"')\";\n        Variable v = getEngine().call(true, exp, sharedScope);\n        return JsValue.fromJava(v.getValue());\n    }\n\n    @Override\n    public void capturePerfEvent(String name, long startTime, long endTime) {\n        PerfEvent event = new PerfEvent(startTime, endTime, name, 200);\n        getEngine().capturePerfEvent(event);\n    }\n\n    public Object compareImage(Object baseline, Object latest, Value... optionsVal) {\n        if (optionsVal.length > 0 && !optionsVal[0].hasMembers()) {\n            throw new RuntimeException(\"invalid image comparison options: expected map\");\n        }\n\n        Map<String, Object> options = new HashMap<>();\n        if (optionsVal.length > 0) {\n            for (String k : optionsVal[0].getMemberKeys()) {\n                options.put(k, optionsVal[0].getMember(k).as(Object.class));\n            }\n        }\n\n        Map<String, Object> params = new HashMap<>();\n        params.put(\"baseline\", baseline);\n        params.put(\"latest\", latest);\n        params.put(\"options\", options);\n\n        return JsValue.fromJava(getEngine().compareImageInternal(params));\n    }\n\n    public void configure(String key, Value value) {\n        getEngine().configure(key, new Variable(value));\n    }\n    \n    public Object consume(String type) {\n        return getEngine().consume(type);\n    }\n\n    public Object distinct(Value o) {\n        if (!o.hasArrayElements()) {\n            return JsList.EMPTY;\n        }\n        long count = o.getArraySize();\n        Set<Object> set = new LinkedHashSet();\n        for (int i = 0; i < count; i++) {\n            Object value = JsValue.toJava(o.getArrayElement(i));\n            set.add(value);\n        }\n        return JsValue.fromJava(new ArrayList(set));\n    }\n\n    public String doc(Value v) {\n        Map<String, Object> arg;\n        if (v.isString()) {\n            arg = Collections.singletonMap(\"read\", v.asString());\n        } else if (v.hasMembers()) {\n            arg = new JsValue(v).getAsMap();\n        } else {\n            getEngine().logger.warn(\"doc - unexpected argument: {}\", v);\n            return null;\n        }\n        return getEngine().docInternal(arg);\n    }\n\n    public void embed(Object o, String contentType) {\n        ResourceType resourceType;\n        if (contentType == null) {\n            resourceType = ResourceType.fromObject(o, ResourceType.BINARY);\n        } else {\n            resourceType = ResourceType.fromContentType(contentType);\n        }\n        getEngine().runtime.embed(JsonUtils.toBytes(o), resourceType);\n    }\n\n    public Object eval(String exp) {\n        Variable result = getEngine().evalJs(exp);\n        return JsValue.fromJava(result.getValue());\n    }\n\n    public String exec(Value value) {\n        if (value.isString()) {\n            return execInternal(Collections.singletonMap(\"line\", value.asString()));\n        } else if (value.hasArrayElements()) {\n            List args = new JsValue(value).getAsList();\n            return execInternal(Collections.singletonMap(\"args\", args));\n        } else {\n            return execInternal(new JsValue(value).getAsMap());\n        }\n    }\n\n    \nprivate String execInternal(Map<String, Object> options) {\n        Command command = getEngine().fork(false, options);\n        command.waitSync();\n        return command.getAppender().collect();\n    }\n\n    public String extract(String text, String regex, int group) {\n        Pattern pattern = Pattern.compile(regex);\n        Matcher matcher = pattern.matcher(text);\n        if (!matcher.find()) {\n            getEngine().logger.warn(\"failed to find pattern: {}\", regex);\n            return null;\n        }\n        return matcher.group(group);\n    }\n\n    public List<String> extractAll(String text, String regex, int group) {\n        Pattern pattern = Pattern.compile(regex);\n        Matcher matcher = pattern.matcher(text);\n        List<String> list = new ArrayList();\n        while (matcher.find()) {\n            list.add(matcher.group(group));\n        }\n        return list;\n    }\n\n    public void fail(String reason) {\n        getEngine().setFailedReason(new KarateException(reason));\n    }\n\n    public Object filter(Value o, Value f) {\n        if (!o.hasArrayElements()) {\n            return JsList.EMPTY;\n        }\n        assertIfJsFunction(f);\n        long count = o.getArraySize();\n        List list = new ArrayList();\n        for (int i = 0; i < count; i++) {\n            Value v = o.getArrayElement(i);\n            Value res = JsEngine.execute(f, v, i);\n            if (res.isBoolean() && res.asBoolean()) {\n                list.add(new JsValue(v).getValue());\n            }\n        }\n        return new JsList(list);\n    }\n\n    public Object filterKeys(Value o, Value... args) {\n        Variable v = new Variable(o);\n        if (!v.isMap()) {\n            return JsMap.EMPTY;\n        }\n        List<String> keys = new ArrayList();\n        if (args.length == 1) {\n            if (args[0].isString()) {\n                keys.add(args[0].asString());\n            } else if (args[0].hasArrayElements()) {\n                long count = args[0].getArraySize();\n                for (int i = 0; i < count; i++) {\n                    keys.add(args[0].getArrayElement(i).toString());\n                }\n            } else if (args[0].hasMembers()) {\n                for (String s : args[0].getMemberKeys()) {\n                    keys.add(s);\n                }\n            }\n        } else {\n            for (Value key : args) {\n                keys.add(key.toString());\n            }\n        }\n        Map map = v.getValue();\n        Map result = new LinkedHashMap(keys.size());\n        for (String key : keys) {\n            if (key == null) {\n                continue;\n            }\n            if (map.containsKey(key)) {\n                result.put(key, map.get(key));\n            }\n        }\n        return new JsMap(result);\n    }\n\n    public void forEach(Value o, Value f) {\n        assertIfJsFunction(f);\n        if (o.hasArrayElements()) {\n            long count = o.getArraySize();\n            for (int i = 0; i < count; i++) {\n                Value v = o.getArrayElement(i);\n                f.executeVoid(v, i);\n            }\n        } else if (o.hasMembers()) { //map\n            int i = 0;\n            for (String k : o.getMemberKeys()) {\n                Value v = o.getMember(k);\n                f.executeVoid(k, v, i++);\n            }\n        } else {\n            throw new RuntimeException(\"not an array or object: \" + o);\n        }\n    }\n\n    public Command fork(Value value) {\n        if (value.isString()) {\n            return getEngine().fork(true, value.asString());\n        } else if (value.hasArrayElements()) {\n            List args = new JsValue(value).getAsList();\n            return getEngine().fork(true, args);\n        } else {\n            return getEngine().fork(true, new JsValue(value).getAsMap());\n        }\n    }\n\n    // TODO breaking returns actual object not wrapper\n    // and fromObject() has been removed\n    // use new typeOf() method to find type\n    public Object fromString(String exp) {\n        ScenarioEngine engine = getEngine();\n        try {\n            Variable result = engine.evalKarateExpression(exp);\n            return JsValue.fromJava(result.getValue());\n        } catch (Exception e) {\n            engine.setFailedReason(null); // special case\n            engine.logger.warn(\"auto evaluation failed: {}\", e.getMessage());\n            return exp;\n        }\n    }\n\n    public Object get(String exp) {\n        ScenarioEngine engine = getEngine();\n        Variable v;\n        try {\n            v = engine.evalKarateExpression(exp); // even json path expressions will work\n        } catch (Exception e) {\n            engine.logger.trace(\"karate.get failed for expression: '{}': {}\", exp, e.getMessage());\n            engine.setFailedReason(null); // special case !\n            return null;\n        }\n        if (v != null) {\n            return JsValue.fromJava(v.getValue());\n        } else {\n            return null;\n        }\n    }\n\n    public Object get(String exp, Object defaultValue) {\n        Object result = get(exp);\n        return result == null ? defaultValue : result;\n    }\n\n    // getters =================================================================\n    // TODO migrate these to functions not properties\n    //\n    public ScenarioEngine getEngine() {\n        ScenarioEngine engine = ScenarioEngine.get();\n        return engine == null ? ENGINE : engine;\n    }\n\n    public String getEnv() {\n        return getEngine().runtime.featureRuntime.suite.env;\n    }\n\n    public Object getFeature() {\n        return new JsMap(getEngine().runtime.featureRuntime.result.toInfoJson());\n    }\n\n    public Object getInfo() { // TODO deprecate\n        return new JsMap(getEngine().runtime.getScenarioInfo());\n    }\n\n    private LogFacade logFacade;\n\n    public Object getLogger() {\n        if (logFacade == null) {\n            logFacade = new LogFacade();\n        }\n        return logFacade;\n    }\n\n    public Object getOs() {\n        String name = FileUtils.getOsName();\n        String type = FileUtils.getOsType(name).toString().toLowerCase();\n        Map<String, Object> map = new HashMap(2);\n        map.put(\"name\", name);\n        map.put(\"type\", type);\n        return new JsMap(map);\n    }\n\n    public Object getPrevRequest() {\n        HttpRequest hr = getEngine().getHttpRequest();\n        if (hr == null) {\n            return null;\n        }\n        Map<String, Object> map = new HashMap();\n        map.put(\"method\", hr.getMethod());\n        map.put(\"url\", hr.getUrl());\n        map.put(\"headers\", hr.getHeaders());\n        map.put(\"body\", hr.getBody());\n        return JsValue.fromJava(map);\n    }\n\n    public Object getProperties() {\n        return new JsMap(getEngine().runtime.featureRuntime.suite.systemProperties);\n    }\n\n    public Object getResponse() {\n        return getEngine().getResponse();\n    }\n\n    public Object getRequest() {\n        return getEngine().getRequest();\n    }\n\n    public Object getScenario() {\n        return new JsMap(getEngine().runtime.result.toKarateJson());\n    }\n\n    public Object getTags() {\n        return JsValue.fromJava(getEngine().runtime.tags.getTags());\n    }\n\n    public Object getTagValues() {\n        return JsValue.fromJava(getEngine().runtime.tags.getTagValues());\n    }\n\n    //==========================================================================\n    //\n    public HttpRequestBuilder http(String url) {\n        ScenarioEngine engine = getEngine();\n        HttpClient client = engine.runtime.featureRuntime.suite.clientFactory.create(engine);\n        return new HttpRequestBuilder(client).url(url);\n    }\n\n    public Object jsonPath(Object o, String exp) {\n        Json json = Json.of(o);\n        return JsValue.fromJava(json.get(exp));\n    }\n\n    public Object keysOf(Object o) {\n        Variable v = new Variable(o);\n        if (v.isMap()) {\n            return new JsList(v.<Map>getValue().keySet());\n        } else {\n            return JsList.EMPTY;\n        }\n    }\n\n    public void log(Value... values) {\n        ScenarioEngine engine = getEngine();\n        if (engine.getConfig().isPrintEnabled()) {\n            engine.logger.info(\"{}\", new LogWrapper(values));\n        }\n    }\n\n    public Object lowerCase(Object o) {\n        Variable var = new Variable(o);\n        return JsValue.fromJava(var.toLowerCase().getValue());\n    }\n\n    public Object map(Value o, Value f) {\n        if (!o.hasArrayElements()) {\n            return JsList.EMPTY;\n        }\n        assertIfJsFunction(f);\n        long count = o.getArraySize();\n        List list = new ArrayList();\n        for (int i = 0; i < count; i++) {\n            Value v = o.getArrayElement(i);\n            Value res = JsEngine.execute(f, v, i);\n            list.add(new JsValue(res).getValue());\n        }\n        return new JsList(list);\n    }\n\n    public Object mapWithKey(Value v, String key) {\n        if (!v.hasArrayElements()) {\n            return JsList.EMPTY;\n        }\n        long count = v.getArraySize();\n        List list = new ArrayList();\n        for (int i = 0; i < count; i++) {\n            Map map = new LinkedHashMap();\n            Value res = v.getArrayElement(i);\n            map.put(key, res.as(Object.class));\n            list.add(map);\n        }\n        return new JsList(list);\n    }\n\n    public Object match(Value actual, Value expected) {\n        Match.Result mr = getEngine().match(Match.Type.EQUALS, JsValue.toJava(actual), JsValue.toJava(expected));\n        return JsValue.fromJava(mr.toMap());\n    }\n\n    public Object match(String exp) {\n        MatchStep ms = new MatchStep(exp);\n        Match.Result mr = getEngine().match(ms.type, ms.name, ms.path, ms.expected);\n        return JsValue.fromJava(mr.toMap());\n    }\n\n    public Object merge(Value... vals) {\n        if (vals.length == 0) {\n            return null;\n        }\n        if (vals.length == 1) {\n            return vals[0];\n        }\n        Map map = new HashMap(vals[0].as(Map.class));\n        for (int i = 1; i < vals.length; i++) {\n            map.putAll(vals[i].as(Map.class));\n        }\n        return new JsMap(map);\n    }\n\n    public void pause(Value value) {\n        ScenarioEngine engine = getEngine();\n        if (!value.isNumber()) {\n            engine.logger.warn(\"pause argument is not a number:\", value);\n            return;\n        }\n        if (engine.runtime.perfMode) {\n            engine.runtime.featureRuntime.perfHook.pause(value.asInt());\n        } else if (engine.getConfig().isPauseIfNotPerf()) {\n            try {\n                Thread.sleep(value.asInt());\n            } catch (Exception e) {\n                throw new RuntimeException(e);\n            }\n        }\n    }\n\n    public String pretty(Object o) {\n        Variable v = new Variable(o);\n        return v.getAsPrettyString();\n    }\n\n    public String prettyXml(Object o) {\n        Variable v = new Variable(o);\n        return v.getAsPrettyXmlString();\n    }\n\n    public void proceed() {\n        proceed(null);\n    }\n\n    public void proceed(String requestUrlBase) {\n        getEngine().mockProceed(requestUrlBase);\n    }\n\n    public Object range(int start, int end) {\n        return range(start, end, 1);\n    }\n\n    public Object range(int start, int end, int interval) {\n        if (interval <= 0) {\n            throw new RuntimeException(\"interval must be a positive integer\");\n        }\n        List<Integer> list = new ArrayList();\n        if (start <= end) {\n            for (int i = start; i <= end; i += interval) {\n                list.add(i);\n            }\n        } else {\n            for (int i = start; i >= end; i -= interval) {\n                list.add(i);\n            }\n        }\n        return JsValue.fromJava(list);\n    }\n\n    public Object read(String path) {\n        Object result = getEngine().fileReader.readFile(path);\n        return JsValue.fromJava(result);\n    }\n\n    public byte[] readAsBytes(String path) {\n        return getEngine().fileReader.readFileAsBytes(path);\n    }\n\n    public String readAsString(String path) {\n        return getEngine().fileReader.readFileAsString(path);\n    }\n\n    public InputStream readAsStream(String path) {\n        return getEngine().fileReader.readFileAsStream(path);\n    }\n\n    public void remove(String name, String path) {\n        getEngine().remove(name, path);\n    }\n\n    public String render(Value v) {\n        Map<String, Object> arg;\n        if (v.isString()) {\n            arg = Collections.singletonMap(\"read\", v.asString());\n        } else if (v.hasMembers()) {\n            arg = new JsValue(v).getAsMap();\n        } else {\n            getEngine().logger.warn(\"render - unexpected argument: {}\", v);\n            return null;\n        }\n        return getEngine().renderHtml(arg);\n    }\n\n    public Object repeat(int n, Value f) {\n        assertIfJsFunction(f);\n        List list = new ArrayList(n);\n        for (int i = 0; i < n; i++) {\n            Value v = JsEngine.execute(f, i);\n            list.add(new JsValue(v).getValue());\n        }\n        return new JsList(list);\n    }\n\n    // set multiple variables in one shot\n    public void set(Map<String, Object> map) {\n        getEngine().setVariables(map);\n    }\n\n    public void set(String name, Value value) {\n        getEngine().setVariable(name, new Variable(value));\n    }\n\n    // this makes sense mainly for xpath manipulation from within js\n    public void set(String name, String path, Object value) {\n        getEngine().set(name, path, new Variable(value));\n    }\n\n    public Object setup() {\n        return setup(null);\n    }\n\n    public Object setup(String name) {\n        Map<String, Object> result = setupInternal(getEngine(), name);\n        return JsValue.fromJava(result);\n    }\n\n    private static Map<String, Object> setupInternal(ScenarioEngine engine, String name) {\n        Feature feature = engine.runtime.featureRuntime.featureCall.feature;\n        Scenario scenario = feature.getSetup(name);\n        if (scenario == null) {\n            String message = \"no scenario found with @setup tag\";\n            if (name != null) {\n                message = message + \" and name '\" + name + \"'\";\n            }\n            engine.logger.error(message);\n            throw new RuntimeException(message);\n        }\n        ScenarioRuntime sr = new ScenarioRuntime(engine.runtime.featureRuntime, scenario);\n        sr.setSkipBackground(true);\n        sr.run();\n        ScenarioEngine.set(engine);\n        engine.runtime.featureRuntime.setupResult = sr.result; // hack to embed setup into report\n        return sr.engine.getAllVariablesAsMap();\n    }\n\n    public Object setupOnce() {\n        return setupOnce(null);\n    }\n\n    public Object setupOnce(String name) {\n        ScenarioEngine engine = getEngine();\n        final Map<String, Map<String, Object>> CACHE = engine.runtime.featureRuntime.SETUPONCE_CACHE;\n        Map<String, Object> result = CACHE.get(name);\n        if (result != null) {\n            return setupOnceResult(result);\n        }\n        long startTime = System.currentTimeMillis();\n        engine.logger.trace(\"setupOnce waiting for lock: {}\", name);\n        synchronized (CACHE) {\n            result = CACHE.get(name); // retry\n            if (result != null) {\n                long endTime = System.currentTimeMillis() - startTime;\n                engine.logger.warn(\"this thread waited {} milliseconds for setupOnce lock: {}\", endTime, name);\n                return setupOnceResult(result);\n            }\n            result = setupInternal(engine, name);\n            CACHE.put(name, result);\n            return setupOnceResult(result);\n        }\n    }\n\n    private static Object setupOnceResult(Map<String, Object> result) {\n        Map<String, Object> clone = new HashMap(result.size());\n        result.forEach((k, v) -> { // shallow clone\n            Variable variable = new Variable(v);\n            clone.put(k, variable.copy(false).getValue());\n        });\n        return JsValue.fromJava(clone);\n    }\n\n    public void setXml(String name, String xml) {\n        getEngine().setVariable(name, XmlUtils.toXmlDoc(xml));\n    }\n\n    // this makes sense mainly for xpath manipulation from within js\n    public void setXml(String name, String path, String xml) {\n        getEngine().set(name, path, new Variable(XmlUtils.toXmlDoc(xml)));\n    }\n\n    public void signal(Value v) {\n        getEngine().signal(JsValue.toJava(v));\n    }\n\n    public Object sizeOf(Object o) {\n        Variable v = new Variable(o);\n        if (v.isList()) {\n            return v.<List>getValue().size();\n        } else if (v.isMap()) {\n            return v.<Map>getValue().size();\n        } else if (v.isBytes()) {\n            return v.<byte[]>getValue().length;\n        } else {\n            return -1;\n        }\n    }\n\n    static abstract class ValueIndex<T> implements Comparable<ValueIndex<T>> {\n\n        final T object;\n        final long index;\n\n        ValueIndex(T o, long index) {\n            this.object = o;\n            this.index = index;\n        }\n\n    }\n\n    static class StringValueIndex extends ValueIndex<String> {\n\n        public StringValueIndex(String o, long index) {\n            super(o, index);\n        }\n\n        @Override\n        public int compareTo(ValueIndex<String> other) {\n            int result = this.object.compareTo(other.object);\n            return result == 0 ? (int) (this.index - other.index) : result;\n        }\n\n    }\n\n    static class NumberValueIndex extends ValueIndex<Number> {\n\n        public NumberValueIndex(Number o, long index) {\n            super(o, index);\n        }\n\n        @Override\n        public int compareTo(ValueIndex<Number> other) {\n            double result = this.object.doubleValue() - other.object.doubleValue();\n            return result == 0 ? (int) (this.index - other.index) : (int) result;\n        }\n\n    }\n\n    public Object sort(Value o) {\n        return sort(o, getEngine().JS.evalForValue(\"x => x\"));\n    }\n\n    public Object sort(Value o, Value f) {\n        if (!o.hasArrayElements()) {\n            return JsList.EMPTY;\n        }\n        assertIfJsFunction(f);\n        long count = o.getArraySize();\n        List<ValueIndex> pointers = new ArrayList((int) count);\n        List<Object> items = new ArrayList(pointers.size());\n        for (int i = 0; i < count; i++) {\n            Object item = JsValue.toJava(o.getArrayElement(i));\n            items.add(item);\n            Value key = JsEngine.execute(f, item, i);\n            if (key.isNumber()) {\n                pointers.add(new NumberValueIndex(key.as(Number.class), i));\n            } else {\n                pointers.add(new StringValueIndex(key.asString(), i));\n            }\n        }\n        Collections.sort(pointers);\n        List<Object> result = new ArrayList(pointers.size());\n        pointers.forEach(item -> result.add(items.get((int) item.index)));\n        return JsValue.fromJava(result);\n    }\n\n    public MockServer start(Value value) {\n        if (value.isString()) {\n            return startInternal(Collections.singletonMap(\"mock\", value.asString()));\n        } else {\n            return startInternal(new JsValue(value).getAsMap());\n        }\n    }\n\n    private MockServer startInternal(Map<String, Object> config) {\n        String mock = (String) config.get(\"mock\");\n        if (mock == null) {\n            throw new RuntimeException(\"'mock' is missing: \" + config);\n        }\n        File feature = toJavaFile(mock);\n        MockServer.Builder builder = MockServer.feature(feature);\n        String pathPrefix = (String) config.get(\"pathPrefix\");\n        if (pathPrefix != null) {\n            builder.pathPrefix(pathPrefix);\n        }\n        String certFile = (String) config.get(\"cert\");\n        if (certFile != null) {\n            builder.certFile(toJavaFile(certFile));\n        }\n        String keyFile = (String) config.get(\"key\");\n        if (keyFile != null) {\n            builder.keyFile(toJavaFile(keyFile));\n        }\n        Boolean ssl = (Boolean) config.get(\"ssl\");\n        if (ssl == null) {\n            ssl = false;\n        }\n        Integer port = (Integer) config.get(\"port\");\n        if (port == null) {\n            port = 0;\n        }\n        Map<String, Object> arg = (Map) config.get(\"arg\");\n        builder.args(arg);\n        if (ssl) {\n            builder.https(port);\n        } else {\n            builder.http(port);\n        }\n        return builder.build();\n    }\n\n    public void stop(int port) {\n        Command.waitForSocket(port);\n    }\n\n    public String toAbsolutePath(String relativePath) {\n        return getEngine().fileReader.toAbsolutePath(relativePath);\n    }\n\n    public Object toBean(Object o, String className) {\n        Json json = Json.of(o);\n        Object bean = JsonUtils.fromJson(json.toString(), className);\n        return JsValue.fromJava(bean);\n    }\n\n    public String toCsv(Object o) {\n        Variable v = new Variable(o);\n        if (!v.isList()) {\n            throw new RuntimeException(\"not a json array: \" + v);\n        }\n        List<Map<String, Object>> list = v.getValue();\n        return JsonUtils.toCsv(list);\n    }\n\n    public Object toJava(Value value) {\n        return new JsValue(value).getValue();\n    }    \n\n    public File toJavaFile(String path) {\n        return getEngine().fileReader.toResource(path).getFile();\n    }\n    \n    public Object toJs(Object value) {\n        return JsValue.fromJava(value);\n    }    \n\n    public Object toJson(Value value) {\n        return toJson(value, false);\n    }\n\n    public Object toJson(Value value, boolean removeNulls) {\n        JsValue jv = new JsValue(value);\n        String json = JsonUtils.toJson(jv.getValue());\n        Object result = Json.of(json).value();\n        if (removeNulls) {\n            JsonUtils.removeKeysWithNullValues(result);\n        }\n        return JsValue.fromJava(result);\n    }\n\n    // TODO deprecate\n    public Object toList(Value value) {\n        return new JsValue(value).getValue();\n    }\n\n    // TODO deprecate\n    public Object toMap(Value value) {\n        return new JsValue(value).getValue();\n    }\n\n    public String toString(Object o) {\n        Variable v = new Variable(o);\n        return v.getAsString();\n    }\n\n    public String typeOf(Value value) {\n        Variable v = new Variable(value);\n        return v.getTypeString();\n    }\n\n    public String urlEncode(String s) {\n        try {\n            return URLEncoder.encode(s, \"UTF-8\");\n        } catch (Exception e) {\n            getEngine().logger.warn(\"url encode failed: {}\", e.getMessage());\n            return s;\n        }\n    }\n\n    public String urlDecode(String s) {\n        try {\n            return URLDecoder.decode(s, \"UTF-8\");\n        } catch (Exception e) {\n            getEngine().logger.warn(\"url encode failed: {}\", e.getMessage());\n            return s;\n        }\n    }\n\n    public Object valuesOf(Object o) {\n        Variable v = new Variable(o);\n        if (v.isList()) {\n            return new JsList(v.<List>getValue());\n        } else if (v.isMap()) {\n            return new JsList(v.<Map>getValue().values());\n        } else {\n            return null;\n        }\n    }\n\n    public boolean waitForHttp(String url) {\n        return Command.waitForHttp(url);\n    }\n\n    public boolean waitForPort(String host, int port) {\n        return new Command().waitForPort(host, port);\n    }\n\n    public WebSocketClient webSocket(String url) {\n        return webSocket(url, null, null);\n    }\n\n    public WebSocketClient webSocket(String url, Value value) {\n        return webSocket(url, value, null);\n    }\n\n    public WebSocketClient webSocket(String url, Value listener, Value value) {\n        Function<String, Boolean> handler;\n        ScenarioEngine engine = getEngine();\n        if (listener == null || !listener.canExecute()) {\n            handler = m -> true;\n        } else {\n            handler = new JsLambda(listener);\n        }\n        WebSocketOptions options = new WebSocketOptions(url, value == null ? null : new JsValue(value).getValue());\n        options.setTextHandler(handler);\n        return engine.webSocket(options);\n    }\n\n    public WebSocketClient webSocketBinary(String url) {\n        return webSocketBinary(url, null, null);\n    }\n\n    public WebSocketClient webSocketBinary(String url, Value value) {\n        return webSocketBinary(url, value, null);\n    }\n\n    public WebSocketClient webSocketBinary(String url, Value listener, Value value) {\n        Function<byte[], Boolean> handler;\n        ScenarioEngine engine = getEngine();\n        if (listener == null || !listener.canExecute()) {\n            handler = m -> true;\n        } else {\n            handler = new JsLambda(listener);\n        }\n        WebSocketOptions options = new WebSocketOptions(url, value == null ? null : new JsValue(value).getValue());\n        options.setBinaryHandler(handler);\n        return engine.webSocket(options);\n    }\n    \n    public Object wrapFunction(Value value) {\n        if (value.isProxyObject()) {\n            Object o = value.asProxyObject();\n            if (o instanceof JsFunction) {\n                JsFunction fun = (JsFunction) o;\n                return JsFunction.wrap(fun.getValue());\n            }\n        }\n        if (value.canExecute()) {\n            return JsFunction.wrap(value);\n        }\n        throw new RuntimeException(\"js function expected\");\n    }\n\n    public File write(Object o, String path) {\n        ScenarioEngine engine = getEngine();\n        path = engine.runtime.featureRuntime.suite.buildDir + File.separator + path;\n        File file = new File(path);\n        FileUtils.writeToFile(file, JsonUtils.toBytes(o));\n        engine.logger.debug(\"write to file: {}\", file);\n        return file;\n    }\n\n    public Object xmlPath(Object o, String path) {\n        Variable var = new Variable(o);\n        Variable res = ScenarioEngine.evalXmlPath(var, path);\n        return JsValue.fromJava(res.getValue());\n    }\n\n    // helpers =================================================================\n    //\n    private static void assertIfJsFunction(Value f) {\n        if (!f.canExecute()) {\n            throw new RuntimeException(\"not a js function: \" + f);\n        }\n    }\n\n    // make sure log() toString() is lazy\n    static class LogWrapper {\n\n        final Value[] values;\n\n        LogWrapper(Value... values) {\n            // sometimes a null array gets passed in, graal weirdness\n            this.values = values == null ? new Value[0] : values;\n        }\n\n        @Override\n        public String toString() {\n            StringBuilder sb = new StringBuilder();\n            for (Value v : values) {\n                Variable var = new Variable(v);\n                sb.append(var.getAsPrettyString()).append(' ');\n            }\n            return sb.toString();\n        }\n\n    }\n\n    public static class LogFacade {\n\n        private static Logger getLogger() {\n            return ScenarioEngine.get().logger;\n        }\n\n        private static String wrap(Value... values) {\n            return new LogWrapper(values).toString();\n        }\n\n        public void debug(Value... values) {\n            getLogger().debug(wrap(values));\n        }\n\n        public void info(Value... values) {\n            getLogger().info(wrap(values));\n        }\n\n        public void trace(Value... values) {\n            getLogger().trace(wrap(values));\n        }\n\n        public void warn(Value... values) {\n            getLogger().warn(wrap(values));\n        }\n\n        public void error(Value... values) {\n            getLogger().error(wrap(values));\n        }\n\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ScenarioIterator.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Spliterator;\nimport java.util.function.Consumer;\nimport java.util.stream.Stream;\nimport java.util.stream.StreamSupport;\nimport org.slf4j.Logger;\n\n/**\n *\n * @author pthomas3\n */\npublic class ScenarioIterator implements Spliterator<ScenarioRuntime> {\n\n    private final FeatureRuntime featureRuntime;\n    private final Iterator<FeatureSection> sections;\n\n    // state\n    private Iterator<Scenario> scenarios;\n    private Scenario currentScenario;\n\n    // dynamic\n    private ScenarioRuntime dynamicRuntime;\n    private Variable expressionValue;\n    private int index;\n\n    public ScenarioIterator(FeatureRuntime featureRuntime) {\n        this.featureRuntime = featureRuntime;\n        this.sections = featureRuntime.featureCall.feature.getSections().iterator();\n    }\n\n    public Stream<ScenarioRuntime> filterSelected() {\n        return StreamSupport.stream(this, false).filter(sr -> sr.selectedForExecution);\n    }\n\n    public ScenarioRuntime first() {\n        return filterSelected().findFirst().get();\n    }\n\n    @Override\n    public boolean tryAdvance(Consumer<? super ScenarioRuntime> action) {\n        if (currentScenario == null) {\n            if (scenarios == null) {\n                if (sections.hasNext()) {\n                    FeatureSection section = sections.next();\n                    if (section.isOutline()) {\n                        scenarios = section.getScenarioOutline().getScenarios(featureRuntime).iterator();\n                    } else {\n                        scenarios = Collections.singletonList(section.getScenario()).iterator();\n                    }\n                } else {\n                    return false;\n                }\n            }\n            if (scenarios.hasNext()) {\n                currentScenario = scenarios.next();\n                index = 0;\n                expressionValue = null;\n            } else {\n                scenarios = null;\n                return tryAdvance(action);\n            }\n        }\n        if (currentScenario.isDynamic()) {\n            Logger logger = FeatureRuntime.logger;\n            if (expressionValue == null) {                \n                dynamicRuntime = new ScenarioRuntime(featureRuntime, currentScenario);\n                if (!dynamicRuntime.selectedForExecution) { // may be un-selected by tag\n                    currentScenario = null;\n                    return tryAdvance(action);                    \n                }\n                String expression = currentScenario.getDynamicExpression();\n                ScenarioEngine prevEngine = ScenarioEngine.get();\n                try {\n                    ScenarioEngine.set(dynamicRuntime.engine);\n                    dynamicRuntime.engine.init();\n                    expressionValue = dynamicRuntime.engine.evalJs(expression);\n                    if (expressionValue.isList() || expressionValue.isJsOrJavaFunction()) {\n                        // all good\n                    } else {\n                        throw new RuntimeException(\"result is neither list nor function: \" + expressionValue);\n                    }\n                } catch (Exception e) {\n                    String message = currentScenario + \" dynamic expression evaluation failed: \" + expression;\n                    logger.error(message);\n                    dynamicRuntime.result.addFakeStepResult(message, e);\n                    currentScenario = null;\n                    action.accept(dynamicRuntime);\n                    return true; // exit early\n                } finally {\n                    ScenarioEngine.set(prevEngine);\n                }\n            }\n            final int rowIndex = index++;\n...\n", "needle_token_start": 7133, "needle_token_end": 7183, "code_context_ntokens": 16016, "output": ["```java\nprivate String execInternal(Map<String, Object> options) {\n    Command command = getEngine().fork(false, options);\n    command.waitSync();\n    return command.getAppender().collect();\n}\n```"]}
{"repo": "karatelabs/karate", "name": "callOnce", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/core/ScenarioEngine.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to execute a callable entity only once and cache the result for subsequent calls, ensuring efficiency and consistency in scenarios where the same expensive or deterministic operation might be repeatedly invoked.\n2. **Input**: The function takes a unique identifier for caching, a callable entity, an optional argument for the callable entity, and a flag indicating whether the scope should be shared across different calls.\n3. **Output**: Returns the result of the callable entity if it's not previously cached; otherwise, it returns the cached result. The output is wrapped in a custom object that can handle various data types.\n4. **Procedure**: \n   - Check if the result for the given unique identifier exists in the cache.\n   - If cached, retrieve and return the cached result, applying necessary scope adjustments.\n   - If not cached, acquire a lock to ensure that only one thread can execute the callable entity.\n   - Execute the callable entity with the provided argument.\n   - Cache the result along with any necessary configuration and variable states.\n   - Release the lock and return the newly cached result, applying scope adjustments as needed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            if (!Boolean.TRUE.equals(defaultOptions.get(\"mismatchShouldPass\"))) {\n                throw e;\n            }\n        } finally {\n            if (embedUI) {\n                String diffJS = \"newDiffUI(document.currentScript,\"\n                        + JsonUtils.toJson(result) + \",\"\n                        + JsonUtils.toJson(options) + \",\"\n                        + getImageHookFunction(options, defaultOptions, \"onShowRebase\") + \",\"\n                        + getImageHookFunction(options, defaultOptions, \"onShowConfig\")\n                        + \")\";\n\n                runtime.embed(JsonUtils.toBytes(diffJS), ResourceType.DEFERRED_JS);\n            }\n        }\n\n        return result;\n    }\n\n    private byte[] getImageBytes(Map<String, Object> params, String paramName) {\n        Object img = params.get(paramName);\n        if (img == null) {\n            return null;\n        }\n\n        if (img instanceof String) {\n            return fileReader.readFileAsBytes((String) img);\n        }\n\n        if (img instanceof byte[]) {\n            return (byte[]) img;\n        }\n\n        throw new RuntimeException(\n                \"invalid image comparison options: expected \" + paramName + \" to be one of string|byte[]\");\n    }\n\n    private Map<String, Object> getImageOptions(Object obj, String objName) {\n        if (obj == null) {\n            return new HashMap<>();\n        }\n\n        if (obj instanceof Map) {\n            return (Map<String, Object>) obj;\n        }\n\n        throw new RuntimeException(\"invalid image comparison \" + objName + \": expected map\");\n    }\n\n    private String getImageHookFunction(Map<String, Object> options, Map<String, Object> defaultOptions, String name) {\n        Object fn = options.containsKey(name) ? options.get(name) : defaultOptions.get(name);\n        return fn == null ? null : fn.toString();\n    }\n\n    //==========================================================================        \n    //       \n    public void init() { // not in constructor because it has to be on Runnable.run() thread \n        JS = JsEngine.local();\n        logger.trace(\"js context: {}\", JS);\n        runtime.magicVariables.forEach((k, v) -> JS.put(k, v));\n        vars.forEach((k, v) -> JS.put(k, v.getValue()));\n        if (runtime.caller.arg != null && runtime.caller.arg.isMap()) {\n            // add the call arg as separate \"over ride\" variables\n            Map<String, Object> arg = runtime.caller.arg.getValue();\n            setVariables(arg);\n        }\n        JS.put(KARATE, bridge);\n        JS.put(READ, readFunction);\n        // edge case: can be left as-is because a callonce triggered init()\n        if (requestBuilder == null) {\n            // note that the http builder is always reset when a \"call\" occurs\n            HttpClient client = runtime.featureRuntime.suite.clientFactory.create(this);\n            requestBuilder = new HttpRequestBuilder(client);\n        }\n        // TODO improve life cycle and concept of shared objects\n        if (!runtime.caller.isNone()) {\n            ScenarioEngine caller = runtime.caller.parentRuntime.engine;\n            if (caller.driver != null) {\n                setDriver(caller.driver);\n            }\n            if (caller.robot != null) {\n                setRobot(caller.robot);\n            }\n        }\n    }\n\n    protected Map<String, Variable> shallowCloneVariables() {\n        Map<String, Variable> copy = new HashMap(vars.size());\n        vars.forEach((k, v) -> copy.put(k, v.copy(false))); // shallow clone\n        return copy;\n    }\n\n    protected <T> Map<String, T> getOrEvalAsMap(Variable var, Object... args) {\n        if (var.isJsOrJavaFunction()) {\n            Variable res = executeFunction(var, args);\n            return res.isMap() ? res.getValue() : null;\n        } else {\n            return var.isMap() ? var.getValue() : null;\n        }\n    }\n\n    public Variable executeFunction(Variable var, Object... args) {\n        switch (var.type) {\n            case JS_FUNCTION:\n                ProxyExecutable pe = var.getValue();\n                Object result = JsEngine.execute(pe, args);\n                return new Variable(result);\n            case JAVA_FUNCTION:  // definitely a \"call\" with a single argument\n                Function javaFunction = var.getValue();\n                Object arg = args.length == 0 ? null : args[0];\n                Object javaResult = javaFunction.apply(arg);\n                return new Variable(JsValue.unWrap(javaResult));\n            default:\n                throw new RuntimeException(\"expected function, but was: \" + var);\n        }\n    }\n\n    public Variable evalJs(String js) {\n        try {\n            return new Variable(JS.eval(js));\n        } catch (Exception e) {\n            KarateException ke = JsEngine.fromJsEvalException(js, e, null);\n            setFailedReason(ke);\n            throw ke;\n        }\n    }\n\n    public void setHiddenVariable(String key, Object value) {\n        if (value instanceof Variable) {\n            value = ((Variable) value).getValue();\n        }\n        JS.put(key, value);\n    }\n\n    public Object getVariable(String key) {\n        return JS.get(key).getValue();\n    }\n\n    public boolean hasVariable(String key) {\n        return JS.bindings.hasMember(key);\n    }\n\n    public void setVariable(String key, Object value) {\n        Variable v;\n        Object o;\n        if (value instanceof Variable) {\n            v = (Variable) value;\n            o = v.getValue();\n        } else {\n            o = value;\n            v = new Variable(value);\n        }\n        vars.put(key, v);\n        if (JS != null) {\n            JS.put(key, o);\n        }\n    }\n\n    public void setVariables(Map<String, Object> map) {\n        if (map == null) {\n            return;\n        }\n        map.forEach((k, v) -> setVariable(k, v));\n    }\n\n    public Map<String, Object> getAllVariablesAsMap() {\n        Map<String, Object> map = new HashMap(vars.size());\n        vars.forEach((k, v) -> map.put(k, v == null ? null : v.getValue()));\n        return map;\n    }\n\n    private static void validateVariableName(String name) {\n        if (!isValidVariableName(name)) {\n            throw new RuntimeException(\"invalid variable name: \" + name);\n        }\n        if (KARATE.equals(name)) {\n            throw new RuntimeException(\"'karate' is a reserved name\");\n        }\n        if (REQUEST.equals(name) || \"url\".equals(name)) {\n            throw new RuntimeException(\"'\" + name + \"' is a reserved name, also use the form '* \" + name + \" <expression>' instead\");\n        }\n    }\n\n    private Variable evalAndCastTo(AssignType assignType, String exp, boolean docString) {\n        Variable v = docString ? new Variable(exp) : evalKarateExpression(exp);\n        switch (assignType) {\n            case BYTE_ARRAY:\n                return new Variable(v.getAsByteArray());\n            case STRING:\n                return new Variable(v.getAsString());\n            case XML:\n                return new Variable(v.getAsXml());\n            case XML_STRING:\n                String xml = XmlUtils.toString(v.getAsXml());\n                return new Variable(xml);\n            case JSON:\n                return new Variable(v.getValueAndForceParsingAsJson());\n            case YAML:\n                return new Variable(JsonUtils.fromYaml(v.getAsString()));\n            case CSV:\n                return new Variable(JsonUtils.fromCsv(v.getAsString()));\n            case COPY:\n                return v.copy(true);\n            default: // TEXT will be docstring, AUTO (def) will auto-parse JSON or XML\n                return v; // as is\n        }\n    }\n\n    public void assign(AssignType assignType, String name, String exp, boolean docString) {\n        name = StringUtils.trimToEmpty(name);\n        validateVariableName(name); // always validate when gherkin\n        if (vars.containsKey(name)) {\n            LOGGER.debug(\"over-writing existing variable '{}' with new value: {}\", name, exp);\n        }\n        setVariable(name, evalAndCastTo(assignType, exp, docString));\n    }\n\n    private static boolean isEmbeddedExpression(String text) {\n        return text != null && (text.startsWith(\"#(\") || text.startsWith(\"##(\")) && text.endsWith(\")\");\n    }\n\n    private Map<String, Object> Map(Object callResult) {\n        throw new UnsupportedOperationException(\"Not supported yet.\"); //To change body of generated methods, choose Tools | Templates.\n    }\n\n    private static class EmbedAction {\n\n        final boolean remove;\n        final Object value;\n\n        private EmbedAction(boolean remove, Object value) {\n            this.remove = remove;\n            this.value = value;\n        }\n\n        static EmbedAction remove() {\n            return new EmbedAction(true, null);\n        }\n\n        static EmbedAction update(Object value) {\n            return new EmbedAction(false, value);\n        }\n\n    }\n\n    public Variable evalEmbeddedExpressions(Variable value, boolean forMatch) {\n        switch (value.type) {\n            case STRING:\n            case MAP:\n            case LIST:\n                EmbedAction ea = recurseEmbeddedExpressions(value, forMatch);\n                if (ea != null) {\n                    return ea.remove ? Variable.NULL : new Variable(ea.value);\n                } else {\n                    return value;\n                }\n            case XML:\n                recurseXmlEmbeddedExpressions(value.getValue(), forMatch);\n            default:\n                return value;\n        }\n    }\n\n    private EmbedAction recurseEmbeddedExpressions(Variable node, boolean forMatch) {\n        switch (node.type) {\n            case LIST:\n                List list = node.getValue();\n                Set<Integer> indexesToRemove = new HashSet();\n                int count = list.size();\n                for (int i = 0; i < count; i++) {\n                    EmbedAction ea = recurseEmbeddedExpressions(new Variable(list.get(i)), forMatch);\n                    if (ea != null) {\n                        if (ea.remove) {\n                            indexesToRemove.add(i);\n                        } else {\n                            list.set(i, ea.value);\n                        }\n                    }\n                }\n                if (!indexesToRemove.isEmpty()) {\n                    List copy = new ArrayList(count - indexesToRemove.size());\n                    for (int i = 0; i < count; i++) {\n                        if (!indexesToRemove.contains(i)) {\n                            copy.add(list.get(i));\n                        }\n                    }\n                    return EmbedAction.update(copy);\n                } else {\n                    return null;\n                }\n            case MAP:\n                Map<String, Object> map = node.getValue();\n                List<String> keysToRemove = new ArrayList();\n                map.forEach((k, v) -> {\n                    EmbedAction ea = recurseEmbeddedExpressions(new Variable(v), forMatch);\n                    if (ea != null) {\n                        if (ea.remove) {\n                            keysToRemove.add(k);\n                        } else {\n                            map.put(k, ea.value);\n                        }\n                    }\n                });\n                for (String key : keysToRemove) {\n                    map.remove(key);\n                }\n                return null;\n            case XML:\n                return null;\n            case STRING:\n                String value = StringUtils.trimToNull(node.getValue());\n                if (!isEmbeddedExpression(value)) {\n                    return null;\n                }\n                boolean optional = value.charAt(1) == '#';\n                value = value.substring(optional ? 2 : 1);\n                try {\n                    JsValue result = JS.eval(value);\n                    if (optional) {\n                        if (result.isNull()) {\n                            return EmbedAction.remove();\n                        }\n                        if (forMatch && (result.isObject() || result.isArray())) {\n                            // preserve optional JSON chunk schema-like references as-is, they are needed for future match attempts\n                            return null;\n                        }\n                    }\n                    return EmbedAction.update(result.getValue());\n                } catch (Exception e) {\n                    logger.trace(\"embedded expression failed {}: {}\", value, e.getMessage());\n                    return null;\n                }\n            default:\n                // do nothing\n                return null;\n        }\n    }\n\n    private void recurseXmlEmbeddedExpressions(Node node, boolean forMatch) {\n        if (node.getNodeType() == Node.DOCUMENT_NODE) {\n            node = node.getFirstChild();\n        }\n        NamedNodeMap attribs = node.getAttributes();\n        int attribCount = attribs == null ? 0 : attribs.getLength();\n        Set<Attr> attributesToRemove = new HashSet(attribCount);\n        for (int i = 0; i < attribCount; i++) {\n            Attr attrib = (Attr) attribs.item(i);\n            String value = attrib.getValue();\n            value = StringUtils.trimToNull(value);\n            if (isEmbeddedExpression(value)) {\n                boolean optional = value.charAt(1) == '#';\n                value = value.substring(optional ? 2 : 1);\n                try {\n                    JsValue jv = JS.eval(value);\n                    if (optional && jv.isNull()) {\n                        attributesToRemove.add(attrib);\n                    } else {\n                        attrib.setValue(jv.getAsString());\n                    }\n                } catch (Exception e) {\n                    logger.trace(\"xml-attribute embedded expression failed, {}: {}\", attrib.getName(), e.getMessage());\n                }\n            }\n        }\n        for (Attr toRemove : attributesToRemove) {\n            attribs.removeNamedItem(toRemove.getName());\n        }\n        NodeList nodeList = node.getChildNodes();\n        int childCount = nodeList.getLength();\n        List<Node> nodes = new ArrayList(childCount);\n        for (int i = 0; i < childCount; i++) {\n            nodes.add(nodeList.item(i));\n        }\n        Set<Node> elementsToRemove = new HashSet(childCount);\n        for (Node child : nodes) {\n            String value = child.getNodeValue();\n            if (value != null) {\n                value = StringUtils.trimToEmpty(value);\n                if (isEmbeddedExpression(value)) {\n                    boolean optional = value.charAt(1) == '#';\n                    value = value.substring(optional ? 2 : 1);\n                    try {\n                        JsValue jv = JS.eval(value);\n                        if (optional) {\n                            if (jv.isNull()) {\n                                elementsToRemove.add(child);\n                            } else if (forMatch && (jv.isXml() || jv.isObject())) {\n                                // preserve optional XML chunk schema-like references as-is, they are needed for future match attempts\n                            } else {\n                                child.setNodeValue(jv.getAsString());\n                            }\n                        } else {\n                            if (jv.isXml() || jv.isObject()) {\n                                Node evalNode = jv.isXml() ? jv.getValue() : XmlUtils.fromMap(jv.getValue());\n                                if (evalNode.getNodeType() == Node.DOCUMENT_NODE) {\n                                    evalNode = evalNode.getFirstChild();\n                                }\n                                if (child.getNodeType() == Node.CDATA_SECTION_NODE) {\n                                    child.setNodeValue(XmlUtils.toString(evalNode));\n                                } else {\n                                    evalNode = node.getOwnerDocument().importNode(evalNode, true);\n                                    child.getParentNode().replaceChild(evalNode, child);\n                                }\n                            } else {\n                                child.setNodeValue(jv.getAsString());\n                            }\n                        }\n                    } catch (Exception e) {\n                        logger.trace(\"xml embedded expression failed, {}: {}\", child.getNodeName(), e.getMessage());\n                    }\n                }\n            } else if (child.hasChildNodes() || child.hasAttributes()) {\n                recurseXmlEmbeddedExpressions(child, forMatch);\n            }\n        }\n        for (Node toRemove : elementsToRemove) { // because of how the above routine works, these are always of type TEXT_NODE\n            Node parent = toRemove.getParentNode(); // element containing the text-node\n            Node grandParent = parent.getParentNode(); // parent element\n            grandParent.removeChild(parent);\n        }\n    }\n\n    public String replacePlaceholderText(String text, String token, String replaceWith) {\n        if (text == null) {\n            return null;\n        }\n        replaceWith = StringUtils.trimToNull(replaceWith);\n        if (replaceWith == null) {\n            return text;\n        }\n        try {\n            Variable v = evalKarateExpression(replaceWith);\n            replaceWith = v.getAsString();\n        } catch (Exception e) {\n            throw new RuntimeException(\"expression error (replace string values need to be within quotes): \" + e.getMessage());\n        }\n        if (replaceWith == null) { // ignore if eval result is null\n            return text;\n        }\n        token = StringUtils.trimToNull(token);\n        if (token == null) {\n            return text;\n        }\n        char firstChar = token.charAt(0);\n        if (Character.isLetterOrDigit(firstChar)) {\n            token = '<' + token + '>';\n        }\n        return text.replace(token, replaceWith);\n    }\n\n    private static final String TOKEN = \"token\";\n\n    public void replaceTable(String text, List<Map<String, String>> list) {\n        if (text == null) {\n            return;\n        }\n        if (list == null) {\n            return;\n        }\n        for (Map<String, String> map : list) {\n            String token = map.get(TOKEN);\n            if (token == null) {\n                continue;\n            }\n            // the verbosity below is to be lenient with table second column name\n            List<String> keys = new ArrayList(map.keySet());\n            keys.remove(TOKEN);\n            Iterator<String> iterator = keys.iterator();\n            if (iterator.hasNext()) {\n                String key = keys.iterator().next();\n                String value = map.get(key);\n                replace(text, token, value);\n            }\n        }\n\n    }\n\n    public void set(String name, String path, Variable value) {\n        set(name, path, false, value, false, false);\n    }\n\n    private void set(String name, String path, String exp, boolean delete, boolean viaTable) {\n        set(name, path, isWithinParentheses(exp), evalKarateExpression(exp), delete, viaTable);\n    }\n\n    private void set(String name, String path, boolean isWithinParentheses, Variable value, boolean delete, boolean viaTable) {\n        name = StringUtils.trimToEmpty(name);\n        path = StringUtils.trimToNull(path);\n        if (viaTable && value.isNull() && !isWithinParentheses) {\n            // by default, skip any expression that evaluates to null unless the user expressed\n            // intent to over-ride by enclosing the expression in parentheses\n            return;\n        }\n        if (path == null) {\n            StringUtils.Pair nameAndPath = parseVariableAndPath(name);\n            name = nameAndPath.left;\n            path = nameAndPath.right;\n        }\n        Variable target = JS.bindings.hasMember(name) ? new Variable(JS.get(name)) : null; // should work in called features\n        if (isXmlPath(path)) {\n            if (target == null || target.isNull()) {\n                if (viaTable) { // auto create if using set via cucumber table as a convenience\n                    Document empty = XmlUtils.newDocument();\n                    target = new Variable(empty);\n                    setVariable(name, target);\n                } else {\n                    throw new RuntimeException(\"variable is null or not set '\" + name + \"'\");\n                }\n            }\n            Document doc = target.getValue();\n            if (delete) {\n                XmlUtils.removeByPath(doc, path);\n            } else if (value.isXml()) {\n                Node node = value.getValue();\n                XmlUtils.setByPath(doc, path, node);\n            } else if (value.isMap()) { // cast to xml\n                Node node = XmlUtils.fromMap(value.getValue());\n                XmlUtils.setByPath(doc, path, node);\n            } else {\n                XmlUtils.setByPath(doc, path, value.getAsString());\n            }\n            setVariable(name, new Variable(doc));\n        } else { // assume json-path\n            if (target == null || target.isNull()) {\n                if (viaTable) { // auto create if using set via cucumber table as a convenience\n                    Json json;\n                    if (path.startsWith(\"$[\") && !path.startsWith(\"$['\")) {\n                        json = Json.of(\"[]\");\n                    } else {\n                        json = Json.of(\"{}\");\n                    }\n                    target = new Variable(json.value());\n                    setVariable(name, target);\n                } else {\n                    throw new RuntimeException(\"variable is null or not set '\" + name + \"'\");\n                }\n            }\n            Json json;\n            if (target.isMapOrList()) {\n                json = Json.of(target.<Object>getValue());\n            } else {\n                throw new RuntimeException(\"cannot set json path on type: \" + target);\n            }\n            if (delete) {\n                json.remove(path);\n            } else {\n                json.set(path, value.<Object>getValue());\n            }\n        }\n    }\n\n    private static final String PATH = \"path\";\n\n    public void setViaTable(String name, String path, List<Map<String, String>> list) {\n        name = StringUtils.trimToEmpty(name);\n        path = StringUtils.trimToNull(path);\n        if (path == null) {\n            StringUtils.Pair nameAndPath = parseVariableAndPath(name);\n            name = nameAndPath.left;\n            path = nameAndPath.right;\n        }\n        for (Map<String, String> map : list) {\n            String append = (String) map.get(PATH);\n            if (append == null) {\n                continue;\n            }\n            List<String> keys = new ArrayList(map.keySet());\n            keys.remove(PATH);\n            int columnCount = keys.size();\n            for (int i = 0; i < columnCount; i++) {\n                String key = keys.get(i);\n                String expression = StringUtils.trimToNull(map.get(key));\n                if (expression == null) { // cucumber cell was left blank\n                    continue; // skip\n                    // default behavior is to skip nulls when the expression evaluates \n                    // this is driven by the routine in setValueByPath\n                    // and users can over-ride this by simply enclosing the expression in parentheses\n                }\n                String suffix;\n                try {\n                    int arrayIndex = Integer.valueOf(key);\n                    suffix = \"[\" + arrayIndex + \"]\";\n                } catch (NumberFormatException e) { // default to the column position as the index\n                    suffix = columnCount > 1 ? \"[\" + i + \"]\" : \"\";\n                }\n                String finalPath;\n                if (append.startsWith(\"/\") || (path != null && path.startsWith(\"/\"))) { // XML\n                    if (path == null) {\n                        finalPath = append + suffix;\n                    } else {\n                        finalPath = path + suffix + '/' + append;\n                    }\n                } else {\n                    if (path == null) {\n                        path = \"$\";\n                    }\n                    finalPath = path + suffix + '.' + append;\n                }\n                set(name, finalPath, expression, false, true);\n            }\n        }\n    }\n\n    public static StringUtils.Pair parseVariableAndPath(String text) {\n        Matcher matcher = VAR_AND_PATH_PATTERN.matcher(text);\n        matcher.find();\n        String name = text.substring(0, matcher.end());\n        String path;\n        if (matcher.end() == text.length()) {\n            path = \"\";\n        } else {\n            path = text.substring(matcher.end()).trim();\n        }\n        if (isXmlPath(path) || isXmlPathFunction(path)) {\n            // xml, don't prefix for json\n        } else {\n            path = \"$\" + path;\n        }\n        return StringUtils.pair(name, path);\n    }\n\n    public Match.Result match(Match.Type matchType, String expression, String path, String rhs) {\n        String name = StringUtils.trimToEmpty(expression);\n        if (isDollarPrefixedJsonPath(name) || isXmlPath(name)) { // \n            path = name;\n            name = RESPONSE;\n        }\n        if (name.startsWith(\"$\")) { // in case someone used the dollar prefix by mistake on the LHS\n            name = name.substring(1);\n        }\n        path = StringUtils.trimToNull(path);\n        if (path == null) {\n            if (name.startsWith(\"(\")) { // edge case, eval entire LHS\n                path = \"$\";\n            } else {\n                StringUtils.Pair pair = parseVariableAndPath(name);\n                name = pair.left;\n                path = pair.right;\n            }\n        }\n        if (\"header\".equals(name)) { // convenience shortcut for asserting against response header\n            return matchHeader(matchType, path, rhs);\n        }\n        Variable actual;\n        // karate started out by \"defaulting\" to JsonPath on the LHS of a match so we have this kludge\n        // but we now handle JS expressions of almost any shape on the LHS, if in doubt, wrap in parentheses\n        // actually it is not too bad - the XPath function check is the only odd one out\n        // rules:\n        // if not XPath function, wrapped in parentheses, involves function call\n        //      [then] JS eval\n        // else if XPath, JsonPath, JsonPath wildcard \"..\" or \"*\" or \"[?\"\n        //      [then] eval name, and do a JsonPath or XPath using the parsed path\n        if (isXmlPathFunction(path)\n                || (!name.startsWith(\"(\") && !path.endsWith(\")\") && !path.contains(\").\"))\n                && (isDollarPrefixed(path) || isJsonPath(path) || isXmlPath(path))) {\n            actual = evalKarateExpression(name);\n            // edge case: java property getter, e.g. \"driver.cookies\"\n            if (!actual.isMap() && !actual.isList() && !isXmlPath(path) && !isXmlPathFunction(path)) {\n                actual = evalKarateExpression(expression); // fall back to JS eval of entire LHS\n                path = \"$\";\n            }\n        } else {\n            actual = evalKarateExpression(expression); // JS eval of entire LHS\n            path = \"$\";\n        }\n        if (\"$\".equals(path) || \"/\".equals(path)) {\n            // we have eval-ed the entire LHS, so proceed to match RHS to \"$\"\n        } else {\n            if (isDollarPrefixed(path)) { // json-path\n                actual = evalJsonPath(actual, path);\n            } else { // xpath\n                actual = evalXmlPath(actual, path);\n            }\n        }\n        Variable expected = evalKarateExpression(rhs, true);\n        return match(matchType, actual.getValue(), expected.getValue());\n    }\n\n    private Match.Result matchHeader(Match.Type matchType, String name, String exp) {\n        Variable expected = evalKarateExpression(exp, true);\n        String actual = response.getHeader(name);\n        return match(matchType, actual, expected.getValue());\n    }\n\n    public Match.Result match(Match.Type matchType, Object actual, Object expected) {\n        return Match.execute(JS, matchType, actual, expected, config.isMatchEachEmptyAllowed());\n    }\n\n    private static final Pattern VAR_AND_PATH_PATTERN = Pattern.compile(\"\\\\w+\");\n    private static final String VARIABLE_PATTERN_STRING = \"[a-zA-Z][\\\\w]*\";\n    private static final Pattern VARIABLE_PATTERN = Pattern.compile(VARIABLE_PATTERN_STRING);\n    private static final Pattern FUNCTION_PATTERN = Pattern.compile(\"^function[^(]*\\\\(\");\n    private static final Pattern JS_PLACEHODER = Pattern.compile(\"\\\\$\\\\{.*?\\\\}\");\n\n    public static boolean isJavaScriptFunction(String text) {\n        return FUNCTION_PATTERN.matcher(text).find();\n    }\n\n    public static boolean isValidVariableName(String name) {\n        return VARIABLE_PATTERN.matcher(name).matches();\n    }\n\n    public static boolean hasJavaScriptPlacehoder(String exp) {\n        return JS_PLACEHODER.matcher(exp).find();\n    }\n\n    public static final boolean isVariableAndSpaceAndPath(String text) {\n        return text.matches(\"^\" + VARIABLE_PATTERN_STRING + \"\\\\s+.+\");\n    }\n\n    public static final boolean isVariable(String text) {\n        return VARIABLE_PATTERN.matcher(text).matches();\n    }\n\n    public static final boolean isWithinParentheses(String text) {\n        return text != null && text.startsWith(\"(\") && text.endsWith(\")\");\n    }\n\n    public static final boolean isCallSyntax(String text) {\n        return text.startsWith(\"call \");\n    }\n\n    public static final boolean isCallOnceSyntax(String text) {\n        return text.startsWith(\"callonce \");\n    }\n\n    public static final boolean isGetSyntax(String text) {\n        return text.startsWith(\"get \") || text.startsWith(\"get[\");\n    }\n\n    public static final boolean isJson(String text) {\n        return text.startsWith(\"{\") || text.startsWith(\"[\");\n    }\n\n    public static final boolean isXml(String text) {\n        return text.startsWith(\"<\");\n    }\n\n    public static boolean isXmlPath(String text) {\n        return text.startsWith(\"/\");\n    }\n\n    public static boolean isXmlPathFunction(String text) {\n        return text.matches(\"^[a-z-]+\\\\(.+\");\n    }\n\n    public static final boolean isJsonPath(String text) {\n        return text.indexOf('*') != -1 || text.contains(\"..\") || text.contains(\"[?\");\n    }\n\n    public static final boolean isDollarPrefixed(String text) {\n        return text.startsWith(\"$\");\n    }\n\n    public static final boolean isDollarPrefixedJsonPath(String text) {\n        return text.startsWith(\"$.\") || text.startsWith(\"$[\") || text.equals(\"$\");\n    }\n\n    public static StringUtils.Pair parseCallArgs(String line) {\n        int pos = line.indexOf(\"read(\");\n        if (pos != -1) {\n            pos = line.indexOf(')');\n            if (pos == -1) {\n                throw new RuntimeException(\"failed to parse call arguments: \" + line);\n            }\n            return new StringUtils.Pair(line.substring(0, pos + 1), StringUtils.trimToNull(line.substring(pos + 1)));\n        }\n        pos = line.indexOf(' ');\n        if (pos == -1) {\n            return new StringUtils.Pair(line, null);\n        }\n        return new StringUtils.Pair(line.substring(0, pos), StringUtils.trimToNull(line.substring(pos)));\n    }\n\n    public Variable call(Variable called, Variable arg, boolean sharedScope) {\n        switch (called.type) {\n            case JS_FUNCTION:\n            case JAVA_FUNCTION:\n                return arg == null ? executeFunction(called) : executeFunction(called, new Object[]{arg.getValue()});\n            case FEATURE:\n                // call result will be always a map or a list of maps (loop call result)\n                Object callResult = callFeature(called.getValue(), arg, -1, sharedScope);\n                return new Variable(callResult);\n            default:\n                throw new RuntimeException(\"not a callable feature or js function: \" + called);\n        }\n    }\n\n    public Variable call(boolean callOnce, String exp, boolean sharedScope) {\n        StringUtils.Pair pair = parseCallArgs(exp);\n        Variable called = evalKarateExpression(pair.left);\n        Variable arg = pair.right == null ? null : evalKarateExpression(pair.right);\n        Variable result;\n        if (callOnce) {\n            result = callOnce(exp, called, arg, sharedScope);\n        } else {\n            result = call(called, arg, sharedScope);\n        }\n        // attach js functions from a different graal context\n        result = new Variable(JS.attachAll(result.getValue()));\n        if (sharedScope && result.isMap()) {\n            setVariables(result.getValue());\n        }\n        return result;\n    }\n\n    private Variable callOnceResult(ScenarioCall.Result result, boolean sharedScope) {\n        if (sharedScope) { // if shared scope\n            vars.clear(); // clean slate            \n            if (result.vars != null) {\n                // shallow clone maps and lists so that subsequent steps don't modify data / references being passed around\n                result.vars.forEach((k, v) -> vars.put(k, v.copy(false)));\n            } else if (result.value != null) {\n                if (result.value.isMap()) {\n                    Map<String, Object> map = result.value.getValue();\n                    // shallow clone newly added variables\n                    map.forEach((k, v) -> vars.put(k, new Variable(JsonUtils.shallowCopy(v))));\n                } else {\n                    logger.warn(\"callonce: ignoring non-map value from result.value: {}\", result.value);\n                }\n            }\n            init(); // this will insert magic variables\n            // re-apply config from time of snapshot\n            setConfig(new Config(result.config));\n            return Variable.NULL; // since we already reset the vars above we return null\n            // else the call() routine would try to do it again\n            // note that shared scope means a return value is meaningless\n        } else {\n            // shallow clone for the same reasons mentioned above\n            return result.value.copy(false);\n        }\n    }\n\n    \nprivate Variable callOnce(String cacheKey, Variable called, Variable arg, boolean sharedScope) {\n        final Map<String, ScenarioCall.Result> CACHE;\n        if (runtime.perfMode) { // use suite-wide cache for gatling\n            CACHE = runtime.featureRuntime.suite.callOnceCache;\n        } else {\n            CACHE = runtime.featureRuntime.CALLONCE_CACHE;\n        }\n        ScenarioCall.Result result = CACHE.get(cacheKey);\n        if (result != null) {\n            logger.trace(\"callonce cache hit for: {}\", cacheKey);\n            return callOnceResult(result, sharedScope);\n        }\n        long startTime = System.currentTimeMillis();\n        logger.trace(\"callonce waiting for lock: {}\", cacheKey);\n        synchronized (CACHE) {\n            result = CACHE.get(cacheKey); // retry\n            if (result != null) {\n                long endTime = System.currentTimeMillis() - startTime;\n                logger.warn(\"this thread waited {} milliseconds for callonce lock: {}\", endTime, cacheKey);\n                return callOnceResult(result, sharedScope);\n            }\n            // this thread is the 'winner'\n            logger.info(\">> lock acquired, begin callonce: {}\", cacheKey);\n            Variable callResult = call(called, arg, sharedScope);\n            // we clone result (and config) here, to snapshot state at the point the callonce was invoked\n            Map<String, Variable> clonedVars = called.isFeature() && sharedScope ? shallowCloneVariables() : null;\n            result = new ScenarioCall.Result(callResult.copy(false), new Config(config), clonedVars);\n            CACHE.put(cacheKey, result);\n            logger.info(\"<< lock released, cached callonce: {}\", cacheKey);\n            // another routine will apply globally if needed\n            return callOnceResult(result, sharedScope);\n        }\n    }\n\n    public Object callFeature(FeatureCall featureCall, Variable arg, int index, boolean sharedScope) {\n        if (arg == null || arg.isMap()) {\n            ScenarioCall call = new ScenarioCall(runtime, featureCall, arg);\n            call.setLoopIndex(index);\n            call.setSharedScope(sharedScope);\n            FeatureRuntime fr = new FeatureRuntime(call);\n            fr.run();\n            // VERY IMPORTANT ! switch back from called feature js context\n            THREAD_LOCAL.set(this);\n            FeatureResult result = fr.result;\n            runtime.addCallResult(result);\n            if (result.isFailed()) {\n                KarateException ke = result.getErrorMessagesCombined();\n                throw ke;\n            } else {\n                return result.getVariables();\n            }\n        } else if (arg.isList() || arg.isJsOrJavaFunction()) {\n            List result = new ArrayList();\n            List<String> errors = new ArrayList();\n            int loopIndex = 0;\n            boolean isList = arg.isList();\n            Iterator iterator = isList ? arg.<List>getValue().iterator() : null;\n            while (true) {\n                Variable loopArg;\n                if (isList) {\n                    loopArg = iterator.hasNext() ? new Variable(iterator.next()) : Variable.NULL;\n                } else { // function\n                    loopArg = executeFunction(arg, new Object[]{loopIndex});\n                }\n                if (!loopArg.isMap()) {\n                    if (!isList) {\n                        logger.info(\"feature call loop function ended at index {}, returned: {}\", loopIndex, loopArg);\n                    }\n                    break;\n                }\n                try {\n                    Object loopResult = callFeature(featureCall, loopArg, loopIndex, sharedScope);\n                    result.add(loopResult);\n                } catch (Exception e) {\n                    String message = \"feature call loop failed at index: \" + loopIndex + \", \" + e.getMessage();\n                    errors.add(message);\n                    runtime.logError(message);\n                    if (!isList) { // this is a generator function, abort infinite loop !\n                        break;\n                    }\n                }\n                loopIndex++;\n            }\n            if (errors.isEmpty()) {\n                return result;\n            } else {\n                String errorMessage = StringUtils.join(errors, \"\\n\");\n                throw new KarateException(errorMessage);\n            }\n        } else {\n            throw new RuntimeException(\"feature call argument is not a json object or array: \" + arg);\n        }\n    }\n\n    public Variable evalJsonPath(Variable v, String path) {\n        Json json = Json.of(v.getValueAndForceParsingAsJson());\n        try {\n            return new Variable(json.get(path));\n        } catch (PathNotFoundException e) {\n            return Variable.NOT_PRESENT;\n        }\n    }\n\n    public static Variable evalXmlPath(Variable xml, String path) {\n        NodeList nodeList;\n        Node doc = xml.getAsXml();\n        try {\n            nodeList = XmlUtils.getNodeListByPath(doc, path);\n        } catch (Exception e) {\n            // hack, this happens for xpath functions that don't return nodes (e.g. count)\n            String strValue = XmlUtils.getTextValueByPath(doc, path);\n            Variable v = new Variable(strValue);\n            if (path.startsWith(\"count\")) { // special case\n                return new Variable(v.getAsInt());\n            } else {\n                return v;\n            }\n        }\n        int count = nodeList.getLength();\n        if (count == 0) { // xpath / node does not exist !\n            return Variable.NOT_PRESENT;\n        }\n        if (count == 1) {\n            return nodeToValue(nodeList.item(0));\n        }\n        List list = new ArrayList();\n        for (int i = 0; i < count; i++) {\n            Variable v = nodeToValue(nodeList.item(i));\n            list.add(v.getValue());\n        }\n        return new Variable(list);\n    }\n\n    private static Variable nodeToValue(Node node) {\n        int childElementCount = XmlUtils.getChildElementCount(node);\n        if (childElementCount == 0) {\n            // hack assuming this is the most common \"intent\"\n            return new Variable(node.getTextContent());\n        }\n        if (node.getNodeType() == Node.DOCUMENT_NODE) {\n            return new Variable(node);\n        } else { // make sure we create a fresh doc else future xpath would run against original root\n            return new Variable(XmlUtils.toNewDocument(node));\n        }\n    }\n\n    public Variable evalJsonPathOnVariableByName(String name, String path) {\n        Variable v = new Variable(JS.get(name)); // should work in called features\n        return evalJsonPath(v, path);\n    }\n\n    public Variable evalXmlPathOnVariableByName(String name, String path) {\n        Variable v = new Variable(JS.get(name)); // should work in called features\n        return evalXmlPath(v, path);\n    }\n\n    public Variable evalKarateExpression(String text) {\n        return evalKarateExpression(text, false);\n    }\n\n    public Variable evalKarateExpression(String text, boolean forMatch) {\n        text = StringUtils.trimToNull(text);\n        if (text == null) {\n            return Variable.NULL;\n        }\n        // don't re-evaluate if this is clearly a direct reference to a variable\n        // this avoids un-necessary conversion of xml into a map in some cases\n        // e.g. 'Given request foo' - where foo is a Variable of type XML      \n        if (JS.bindings.hasMember(text)) {\n            return new Variable(JS.get(text));\n        }\n        boolean callOnce = isCallOnceSyntax(text);\n        if (callOnce || isCallSyntax(text)) { // special case in form \"callBegin foo arg\"\n            if (callOnce) {\n                text = text.substring(9);\n            } else {\n                text = text.substring(5);\n            }\n            return call(callOnce, text, false);\n        } else if (isDollarPrefixedJsonPath(text)) {\n            return evalJsonPathOnVariableByName(RESPONSE, text);\n        } else if (isGetSyntax(text) || isDollarPrefixed(text)) { // special case in form\n            // get json[*].path\n            // $json[*].path\n            // get /xml/path\n            // get xpath-function(expression)\n            int index = -1;\n            if (text.startsWith(\"$\")) {\n                text = text.substring(1);\n            } else if (text.startsWith(\"get[\")) {\n                int pos = text.indexOf(']');\n                index = Integer.valueOf(text.substring(4, pos));\n                text = text.substring(pos + 2);\n            } else {\n                text = text.substring(4);\n            }\n            String left;\n            String right;\n            if (isDollarPrefixedJsonPath(text)) { // edge case get[0] $..foo\n                left = RESPONSE;\n                right = text;\n            } else if (isVariableAndSpaceAndPath(text)) {\n                int pos = text.indexOf(' ');\n                right = text.substring(pos + 1);\n                left = text.substring(0, pos);\n            } else {\n                StringUtils.Pair pair = parseVariableAndPath(text);\n                left = pair.left;\n                right = pair.right;\n            }\n            Variable sv;\n            if (isXmlPath(right) || isXmlPathFunction(right)) {\n                sv = evalXmlPathOnVariableByName(left, right);\n            } else {\n                sv = evalJsonPathOnVariableByName(left, right);\n            }\n            if (index != -1 && sv.isList()) {\n                List list = sv.getValue();\n                if (!list.isEmpty()) {\n                    return new Variable(list.get(index));\n                }\n            }\n            return sv;\n        } else if (isJson(text)) {\n            Json json = Json.of(text);\n            return evalEmbeddedExpressions(new Variable(json.value()), forMatch);\n        } else if (isXml(text)) {\n            Document doc = XmlUtils.toXmlDoc(text, config.isXmlNamespaceAware());\n            return evalEmbeddedExpressions(new Variable(doc), forMatch);\n        } else if (isXmlPath(text)) {\n            return evalXmlPathOnVariableByName(RESPONSE, text);\n        } else {\n            // old school function declarations e.g. function() { } need wrapping in graal\n            if (isJavaScriptFunction(text)) {\n                text = \"(\" + text + \")\";\n            }\n            // js expressions e.g. foo, foo(bar), foo.bar, foo + bar, foo + '', 5, true\n            // including arrow functions e.g. x => x + 1\n            return evalJs(text);\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/Http.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate;\n\nimport com.intuit.karate.core.ScenarioEngine;\nimport com.intuit.karate.core.Variable;\nimport com.intuit.karate.http.HttpClientFactory;\nimport com.intuit.karate.http.HttpRequestBuilder;\nimport com.intuit.karate.http.Response;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class Http {\n\n    public final String urlBase;\n    private final ScenarioEngine engine;\n    private final HttpRequestBuilder builder;\n\n    public static Http to(String url) {\n        return new Http(url);\n    }\n\n    public void setAppender(LogAppender appender) {\n        engine.logger.setAppender(appender);\n    }\n\n    private Http(String urlBase) {\n        this.urlBase = urlBase;\n        engine = ScenarioEngine.forTempUse(HttpClientFactory.DEFAULT);\n        builder = engine.getRequestBuilder();\n        builder.url(urlBase);\n    }\n\n    public Http url(String url) {\n        builder.url(url);\n        return this;\n    }\n    \n    public Http param(String key, String ... values) {\n        builder.param(key, values);\n        return this;\n    }\n\n    public Http path(String... paths) {\n        builder.paths(paths);\n        return this;\n    }\n\n    public Http header(String name, String value) {\n        builder.header(name, value);\n        return this;\n    }\n\n    public Http hook(RuntimeHook hook) {\n        builder.hook(hook);\n        return this;\n    }\n\n    public Response method(String method, Object body) {        \n        if (body != null) {\n            builder.body(body instanceof Json ? ((Json) body).value() : body);\n        }\n        builder.method(method);\n        Response response = engine.httpInvoke();\n        if (response.getStatus() >= 400) {\n            engine.logger.warn(\"http response code: {}, response: {}, request: {}\",\n                    response.getStatus(), response.getBodyAsString(), engine.getHttpRequest());\n        }\n        return response;\n    }\n\n    public Response method(String method) {\n        return method(method, null);\n    }\n    \n    public Response methodJson(String method, String body) {\n        return method(method, Json.of(body));\n    }    \n\n    public Response get() {\n        return method(\"get\");\n    }\n\n    public Response postJson(String body) {\n        return post(Json.of(body));\n    }\n\n    public Response post(Object body) {\n        return method(\"post\", body);\n    }\n    \n    public Response put(Object body) {\n        return method(\"put\", body);\n    }   \n    \n    public Response putJson(String body) {\n        return put(Json.of(body));\n    }    \n\n    public Response delete() {\n        return method(\"delete\");\n    }\n\n    public Http configure(String key, Object value) {\n        engine.configure(key, new Variable(value));\n        return this;\n    }\n\n    public Http configure(Map<String, Object> map) {\n        map.forEach((k, v) -> configure(k, v));\n        return this;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/shell/Command.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.shell;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.Http;\nimport com.intuit.karate.LogAppender;\nimport com.intuit.karate.Logger;\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.http.Response;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.net.InetSocketAddress;\nimport java.net.ServerSocket;\nimport java.net.SocketAddress;\nimport java.nio.channels.SocketChannel;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.function.Consumer;\nimport java.util.function.Predicate;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\npublic class Command extends Thread {\n\n    protected static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(Command.class);\n\n    private final boolean useLineFeed;\n    private final File workingDir;\n    private final String uniqueName;\n    private final Logger logger;\n    private final String[] args;\n    private final List argList; // just for logging\n    private final boolean sharedAppender;\n    private final LogAppender appender;\n\n    private Map<String, String> environment;\n    private Consumer<String> listener;\n    private Consumer<String> errorListener;\n    private boolean redirectErrorStream = true;\n    private Console sysOut;\n    private Console sysErr;\n    private Process process;\n    private int exitCode = -1;\n    private Exception failureReason;\n\n    private int pollAttempts = 30;\n    private int pollInterval = 250;\n\n    public void setPollAttempts(int pollAttempts) {\n        this.pollAttempts = pollAttempts;\n    }\n\n    public void setPollInterval(int pollInterval) {\n        this.pollInterval = pollInterval;\n    }\n\n    public synchronized boolean isFailed() {\n        return failureReason != null;\n    }\n\n    public Exception getFailureReason() {\n        return failureReason;\n    }\n\n    public void setEnvironment(Map<String, String> environment) {\n        this.environment = environment;\n    }\n\n    public void setListener(Consumer<String> listener) {\n        this.listener = listener;\n    }\n\n    public void setErrorListener(Consumer<String> errorListener) {\n        this.errorListener = errorListener;\n    }\n\n    public void setRedirectErrorStream(boolean redirectErrorStream) {\n        this.redirectErrorStream = redirectErrorStream;\n    }\n\n    public String getSysOut() {\n        return sysOut == null ? null : sysOut.getBuffer();\n    }\n\n    public String getSysErr() {\n        return sysErr == null ? null : sysErr.getBuffer();\n    }\n\n    public static String exec(boolean useLineFeed, File workingDir, String... args) {\n        Command command = new Command(useLineFeed, workingDir, args);\n        command.start();\n        command.waitSync();\n        return command.getSysOut();\n    }\n\n    private static final Pattern CLI_ARG = Pattern.compile(\"'([^']*)'[^\\\\S]|\\\"([^\\\"]*)\\\"[^\\\\S]|(\\\\S+)\");\n\n    public static String[] tokenize(String command) {\n        List<String> args = new ArrayList();\n        Matcher m = CLI_ARG.matcher(command + \" \");\n        while (m.find()) {\n            if (m.group(1) != null) {\n                args.add(m.group(1));\n            } else if (m.group(2) != null) {\n                args.add(m.group(2));\n            } else {\n                args.add(m.group(3));\n            }\n        }\n        return args.toArray(new String[args.size()]);\n    }\n\n    public static String execLine(File workingDir, String command) {\n        return exec(false, workingDir, tokenize(command));\n    }\n\n    public static String[] prefixShellArgs(String[] args) {\n        List<String> list = new ArrayList();\n        switch (FileUtils.getOsType()) {\n            case WINDOWS:\n                list.add(\"cmd\");\n                list.add(\"/c\");\n                break;\n            default:\n                list.add(\"sh\");\n                list.add(\"-c\");\n        }\n        list.add(StringUtils.join(args, ' '));\n        return list.toArray(new String[list.size()]);\n    }\n\n    private static final Set<Integer> PORTS_IN_USE = ConcurrentHashMap.newKeySet();\n\n    public static synchronized int getFreePort(int preferred) {\n        if (preferred != 0 && PORTS_IN_USE.contains(preferred)) {\n            LOGGER.trace(\"preferred port {} in use (karate), will attempt to find free port ...\", preferred);\n            preferred = 0;\n        }\n        try {\n            ServerSocket s = new ServerSocket(preferred);\n            int port = s.getLocalPort();\n            LOGGER.debug(\"found / verified free local port: {}\", port);\n            s.close();\n            PORTS_IN_USE.add(port);\n            return port;\n        } catch (Exception e) {\n            if (preferred > 0) {\n                LOGGER.trace(\"preferred port {} in use (system), re-trying ...\", preferred);\n                PORTS_IN_USE.add(preferred);\n                return getFreePort(0);\n            }\n            LOGGER.error(\"failed to find free port: {}\", e.getMessage());\n            throw new RuntimeException(e);\n        }\n    }\n\n    private static void sleep(int millis) {\n        try {\n            LOGGER.trace(\"sleeping for millis: {}\", millis);\n            Thread.sleep(millis);\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public boolean waitForPort(String host, int port) {\n        int attempts = 0;\n        do {\n            SocketAddress address = new InetSocketAddress(host, port);\n            try {\n                if (isFailed()) {\n                    throw failureReason;\n                }\n                logger.debug(\"poll attempt #{} for port to be ready - {}:{}\", attempts, host, port);\n                SocketChannel sock = SocketChannel.open(address);\n                sock.close();\n                return true;\n            } catch (Exception e) {\n                sleep(pollInterval);\n            }\n        } while (attempts++ < pollAttempts);\n        return false;\n    }\n\n    private static final int SLEEP_TIME = 2000;\n    private static final int POLL_ATTEMPTS_MAX = 30;\n    \n    public static boolean waitForHttp(String url) {\n        return waitForHttp(url, r -> r.getStatus() == 200);\n    }\n\n    public static boolean waitForHttp(String url, Predicate<Response> condition) {\n        int attempts = 0;\n        long startTime = System.currentTimeMillis();\n        Http http = Http.to(url);\n        do {\n            if (attempts > 0) {\n                LOGGER.debug(\"attempt #{} waiting for http to be ready at: {}\", attempts, url);\n            }\n            try {\n                Response response = http.get();\n                if (condition.test(response)) {\n                    long elapsedTime = System.currentTimeMillis() - startTime;\n                    LOGGER.debug(\"ready to accept http connections after {} ms - {}\", elapsedTime, url);\n                    return true;\n                } else {\n                    LOGGER.warn(\"not ready / http get returned status: {} - {}\", response.getStatus(), url);\n                }\n            } catch (Exception e) {\n                sleep(SLEEP_TIME);\n            }\n        } while (attempts++ < POLL_ATTEMPTS_MAX);\n        return false;\n    }\n\n    public static boolean waitForSocket(int port) {\n        StopListenerThread waiter = new StopListenerThread(port, () -> {\n            LOGGER.info(\"*** exited socket wait succesfully\");\n        });\n        waiter.start();\n        port = waiter.getPort();\n        System.out.println(\"*** waiting for socket, type the command below:\\ncurl http://localhost:\"\n                + port + \"\\nin a new terminal (or open the URL in a web-browser) to proceed ...\");\n        try {\n            waiter.join();\n            return true;\n        } catch (Exception e) {\n            LOGGER.warn(\"*** wait thread failed: {}\", e.getMessage());\n            return false;\n        }\n    }\n\n    public Command(String... args) {\n        this(false, null, null, null, null, args);\n    }\n\n    public Command(boolean useLineFeed, File workingDir, String... args) {\n        this(useLineFeed, null, null, null, workingDir, args);\n    }\n\n    public Command(boolean useLineFeed, Logger logger, String uniqueName, String logFile, File workingDir, String... args) {\n        this.useLineFeed = useLineFeed;\n        setDaemon(true);\n        this.uniqueName = uniqueName == null ? System.currentTimeMillis() + \"\" : uniqueName;\n        setName(this.uniqueName);\n        this.logger = logger == null ? new Logger() : logger;\n        this.workingDir = workingDir;\n        this.args = args;\n        if (workingDir != null) {\n            workingDir.mkdirs();\n        }\n        argList = Arrays.asList(args);\n        if (logFile == null) {\n            appender = new StringLogAppender(useLineFeed);\n            sharedAppender = false;\n        } else { // don't create new file if re-using an existing appender\n            LogAppender temp = this.logger.getAppender();\n            sharedAppender = temp != null;\n            if (sharedAppender) {\n                appender = temp;\n            } else {\n                appender = new FileLogAppender(new File(logFile));\n                this.logger.setAppender(appender);\n            }\n        }\n    }\n\n    public Map<String, String> getEnvironment() {\n        return environment;\n    }\n\n    public File getWorkingDir() {\n        return workingDir;\n    }\n\n    public List getArgList() {\n        return argList;\n    }\n\n    public Logger getLogger() {\n        return logger;\n    }\n\n    public LogAppender getAppender() {\n        return appender;\n    }\n\n    public String getUniqueName() {\n        return uniqueName;\n    }\n\n    public int getExitCode() {\n        return exitCode;\n    }\n\n    public int waitSync() {\n        try {\n            join();\n            return exitCode;\n        } catch (InterruptedException e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    public void close(boolean force) {\n        LOGGER.debug(\"closing command: {}\", uniqueName);\n        if (force) {\n            process.destroyForcibly();\n        } else {\n            process.destroy();\n        }\n    }\n\n    @Override\n    public void run() {\n        try {\n            logger.debug(\"command: {}, working dir: {}\", argList, workingDir);\n            ProcessBuilder pb = new ProcessBuilder(args);\n            if (environment != null) {\n                pb.environment().putAll(environment);\n                environment = pb.environment();\n            }\n            logger.trace(\"env PATH: {}\", pb.environment().get(\"PATH\"));\n            if (workingDir != null) {\n                pb.directory(workingDir);\n            }\n            pb.redirectErrorStream(redirectErrorStream);\n            process = pb.start();\n            sysOut = new Console(uniqueName + \"-out\", useLineFeed, process.getInputStream(), logger, appender, listener);\n            sysOut.start();\n            sysErr = new Console(uniqueName + \"-err\", useLineFeed, process.getErrorStream(), logger, appender, errorListener);\n            sysErr.start();\n            exitCode = process.waitFor();\n            if (exitCode == 0) {\n                LOGGER.debug(\"command complete, exit code: {} - {}\", exitCode, argList);\n            } else {\n                LOGGER.warn(\"exit code was non-zero: {} - {} working dir: {}\", exitCode, argList, workingDir);\n            }\n            // the consoles actually can take more time to flush even after the process has exited\n            sysErr.join();\n            sysOut.join();\n            LOGGER.trace(\"console readers complete\");\n            if (!sharedAppender) {\n                appender.close();\n            }\n        } catch (Exception e) {\n            failureReason = e;\n            LOGGER.error(\"command error: {} - {}\", argList, e.getMessage());\n...\n", "needle_token_start": 8705, "needle_token_end": 9177, "code_context_ntokens": 16196, "output": ["```java\nprivate Variable callOnce(String cacheKey, Variable called, Variable arg, boolean sharedScope) {\n    final Map<String, ScenarioCall.Result> CACHE;\n    if (runtime.perfMode) { // use suite-wide cache for gatling\n        CACHE = runtime.featureRuntime.suite.callOnceCache;\n    } else {\n        CACHE = runtime.featureRuntime.CALLONCE_CACHE;\n    }\n    ScenarioCall.Result result = CACHE.get(cacheKey);\n    if (result != null) {\n        logger.trace(\"callonce cache hit for: {}\", cacheKey);\n        return callOnceResult(result, sharedScope);\n    }\n    long startTime = System.currentTimeMillis();\n    logger.trace(\"callonce waiting for lock: {}\", cacheKey);\n    synchronized (CACHE) {\n        result = CACHE.get(cacheKey); // retry\n        if (result != null) {\n            long endTime = System.currentTimeMillis() - startTime;\n            logger.warn(\"this thread waited {} milliseconds for callonce lock: {}\", endTime, cacheKey);\n            return callOnceResult(result, sharedScope);\n        }\n        // this thread is the 'winner'\n        logger.info(\">> lock acquired, begin callonce: {}\", cacheKey);\n        Variable callResult = call(called, arg, sharedScope);\n        // we clone result (and config) here, to snapshot state at the point the callonce was invoked\n        Map<String, Variable> clonedVars = called.isFeature() && sharedScope ? shallowCloneVariables() : null;\n        result = new ScenarioCall.Result(callResult.copy(false), new Config(config), clonedVars);\n        CACHE.put(cacheKey, result);\n        logger.info(\"<< lock released, cached callonce: {}\", cacheKey);\n        // another routine will apply globally if needed\n        return callOnceResult(result, sharedScope);\n    }\n}\n```"]}
{"repo": "karatelabs/karate", "name": "getCurrentTime", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/cli/IntellijHook.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to fetch the current system time and format it into a human-readable string format.\n2. **Input**: It does not take any parameters.\n3. **Output**: Returns a string representing the current date and time, formatted according to a predefined pattern.\n4. **Procedure**: The function retrieves the current system date and time, then formats this timestamp using a predefined date format pattern to convert it into a string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: karate-core/src/main/java/com/intuit/karate/core/ScenarioIterator.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Spliterator;\nimport java.util.function.Consumer;\nimport java.util.stream.Stream;\nimport java.util.stream.StreamSupport;\nimport org.slf4j.Logger;\n\n/**\n *\n * @author pthomas3\n */\npublic class ScenarioIterator implements Spliterator<ScenarioRuntime> {\n\n    private final FeatureRuntime featureRuntime;\n    private final Iterator<FeatureSection> sections;\n\n    // state\n    private Iterator<Scenario> scenarios;\n    private Scenario currentScenario;\n\n    // dynamic\n    private ScenarioRuntime dynamicRuntime;\n    private Variable expressionValue;\n    private int index;\n\n    public ScenarioIterator(FeatureRuntime featureRuntime) {\n        this.featureRuntime = featureRuntime;\n        this.sections = featureRuntime.featureCall.feature.getSections().iterator();\n    }\n\n    public Stream<ScenarioRuntime> filterSelected() {\n        return StreamSupport.stream(this, false).filter(sr -> sr.selectedForExecution);\n    }\n\n    public ScenarioRuntime first() {\n        return filterSelected().findFirst().get();\n    }\n\n    @Override\n    public boolean tryAdvance(Consumer<? super ScenarioRuntime> action) {\n        if (currentScenario == null) {\n...\n// Path: karate-core/src/main/java/com/intuit/karate/core/ScenarioFileReader.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.JsonUtils;\nimport com.intuit.karate.resource.Resource;\nimport com.intuit.karate.resource.ResourceUtils;\nimport java.io.IOException;\nimport java.io.InputStream;\n\n/**\n *\n * @author pthomas3\n */\npublic class ScenarioFileReader {\n\n    private final ScenarioEngine engine;\n    private final FeatureRuntime featureRuntime;\n\n    public ScenarioFileReader(ScenarioEngine engine, FeatureRuntime featureRuntime) {\n        this.engine = engine;\n        this.featureRuntime = featureRuntime;\n    }\n\n    public Object readFile(String text) {\n        StringUtils.Pair pair = parsePathAndTags(text);\n        text = pair.left;\n        if (text == null && pair.right != null && pair.right.startsWith(\"@\")) {\n            return new FeatureCall(this.featureRuntime.featureCall.feature, pair.right, -1, null);\n        } else if (isJsonFile(text) || isXmlFile(text)) {\n            String contents = readFileAsString(text);\n            Variable temp = engine.evalKarateExpression(contents);\n            return temp.getValue();\n        } else if (isJavaScriptFile(text)) {\n            String contents = readFileAsString(text);\n            Variable temp = engine.evalJs(\"(\" + contents + \")\");\n            return temp.getValue();\n        } else if (isTextFile(text) || isGraphQlFile(text)) {\n            return readFileAsString(text);\n        } else if (isFeatureFile(text)) {\n            Resource fr = toResource(text);\n            Feature feature = Feature.read(fr);\n            return new FeatureCall(feature, pair.right, -1, null);\n        } else if (isCsvFile(text)) {\n            String contents = readFileAsString(text);\n            return JsonUtils.fromCsv(contents);\n        } else if (isYamlFile(text)) {\n            String contents = readFileAsString(text);\n            Object asJson = JsonUtils.fromYaml(contents);\n            Variable temp = engine.evalKarateExpression(JsonUtils.toJson(asJson));\n            return temp.getValue();\n        } else {\n            InputStream is = readFileAsStream(text);\n            return FileUtils.toBytes(is); // TODO stream\n        }\n    }\n\n    public String toAbsolutePath(String relativePath) {\n        Resource resource = toResource(relativePath);\n        try {\n            return resource.getFile().getCanonicalPath();\n        } catch (IOException e) {\n            return resource.getFile().getAbsolutePath();\n        }\n    }\n\n    public byte[] readFileAsBytes(String path) {\n        return FileUtils.toBytes(readFileAsStream(path));\n    }\n\n    public String readFileAsString(String path) {\n        return FileUtils.toString(readFileAsStream(path));\n    }\n\n    public InputStream readFileAsStream(String path) {\n        return toResource(path).getStream();\n    }\n\n    private static String removePrefix(String text) {\n        if (text == null) {\n            return null;\n        }\n        int pos = text.indexOf(':');\n        return pos == -1 ? text : text.substring(pos + 1);\n    }\n\n    private static StringUtils.Pair parsePathAndTags(String text) {\n        if (text.startsWith(\"@\")) {\n            return new StringUtils.Pair(null, StringUtils.trimToEmpty(text));\n        }\n        int pos = text.indexOf(\".feature@\");\n        if (pos == -1) {\n            pos = text.indexOf(\".feature?\");\n            if (pos != -1) {\n                text = text.substring(0, pos + 8);\n            }\n            text = StringUtils.trimToEmpty(text);\n            return new StringUtils.Pair(text, null);\n        } else {\n            return new StringUtils.Pair(StringUtils.trimToEmpty(text.substring(0, pos + 8)), StringUtils.trimToEmpty(text.substring(pos + 9)));\n        }\n    }\n\n    public Resource toResource(String path) {\n        if (isClassPath(path)) {\n            return ResourceUtils.getResource(featureRuntime.suite.workingDir, path);\n        } else if (isFilePath(path)) {\n            return ResourceUtils.getResource(featureRuntime.suite.workingDir, removePrefix(path));\n        } else if (isThisPath(path)) {\n            return featureRuntime.resolveFromThis(removePrefix(path));\n        } else {\n            return featureRuntime.resolveFromRoot(path);\n        }\n    }\n\n    private static boolean isClassPath(String text) {\n        return text.startsWith(\"classpath:\");\n    }\n\n    private static boolean isFilePath(String text) {\n        return text.startsWith(\"file:\");\n    }\n\n    private static boolean isThisPath(String text) {\n        return text.startsWith(\"this:\");\n    }\n\n    private static boolean isJsonFile(String text) {\n        return text.endsWith(\".json\");\n    }\n\n    private static boolean isJavaScriptFile(String text) {\n        return text.endsWith(\".js\");\n    }\n\n    private static boolean isYamlFile(String text) {\n        return text.endsWith(\".yaml\") || text.endsWith(\".yml\");\n    }\n\n    private static boolean isXmlFile(String text) {\n        return text.endsWith(\".xml\");\n    }\n\n    private static boolean isTextFile(String text) {\n        return text.endsWith(\".txt\");\n    }\n\n    private static boolean isCsvFile(String text) {\n        return text.endsWith(\".csv\");\n    }\n\n    private static boolean isGraphQlFile(String text) {\n        return text.endsWith(\".graphql\") || text.endsWith(\".gql\");\n    }\n\n    private static boolean isFeatureFile(String text) {\n        return text.endsWith(\".feature\");\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ScenarioOutline.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * @author pthomas3\n */\npublic class ScenarioOutline {\n    \n    private final Feature feature;\n    private final FeatureSection section;\n    \n    private int line;\n    private List<Tag> tags;\n    private String name;\n    private String description;\n    private List<Step> steps;\n    private List<ExamplesTable> examplesTables;\n    \n    public ScenarioOutline(Feature feature, FeatureSection section) {\n        this.feature = feature;\n        this.section = section;\n    }\n    \n    public Scenario toScenario(String dynamicExpression, int exampleIndex, int updateLine, List<Tag> tagsForExamples) {\n        Scenario s = new Scenario(feature, section, exampleIndex);\n        s.setName(name);\n        s.setDescription(description);\n        s.setLine(updateLine);\n        s.setDynamicExpression(dynamicExpression);\n        if (tags != null || tagsForExamples != null) {\n            List<Tag> temp = new ArrayList();\n            if (tags != null) {\n                temp.addAll(tags);\n            }\n            if (tagsForExamples != null) {\n                temp.addAll(tagsForExamples);\n            }\n            s.setTags(temp);\n        }\n        List<Step> temp = new ArrayList(steps.size());\n        s.setSteps(temp);\n        for (Step original : steps) {\n            Step step = new Step(s, original.getIndex());\n            temp.add(step);\n            step.setLine(original.getLine());\n            step.setEndLine(original.getEndLine());\n            step.setPrefix(original.getPrefix());\n            step.setText(original.getText());\n            step.setDocString(original.getDocString());\n            step.setTable(original.getTable());\n            step.setComments(original.getComments());\n        }\n        return s;\n    }\n    \n    public List<Scenario> getScenarios() {\n        return this.getScenarios(null);\n    }\n    \n    public List<Scenario> getScenarios(FeatureRuntime fr) {\n        List<Scenario> list = new ArrayList();\n        boolean examplesHaveTags = examplesTables.stream().anyMatch(t -> !t.getTags().isEmpty());\n        for (ExamplesTable examples : examplesTables) {\n            boolean selectedForExecution = false;\n            if (fr != null && examplesHaveTags && fr.caller.isNone()) {\n                // getting examples in the context of an execution\n                // if the examples do not have any tagged example, do not worry about selecting\n                Tags tableTags = Tags.merge(fr.featureCall.feature.getTags(), tags, examples.getTags());\n                boolean executeForTable = tableTags.evaluate(fr.suite.tagSelector, fr.suite.env);\n                if (executeForTable) {\n                    selectedForExecution = true;\n                }\n            } else {\n                selectedForExecution = true;\n            }\n            if (selectedForExecution) {\n                Table table = examples.getTable();\n                if (table.isDynamic()) {\n                    Scenario scenario = toScenario(table.getDynamicExpression(), -1, table.getLineNumberForRow(0), examples.getTags());\n                    list.add(scenario);\n                } else {\n                    int rowCount = table.getRows().size();\n                    for (int i = 1; i < rowCount; i++) { // don't include header row\n                        int exampleIndex = i - 1; // next line will set exampleIndex on scenario\n                        Scenario scenario = toScenario(null, exampleIndex, table.getLineNumberForRow(i), examples.getTags());\n                        scenario.setExampleData(table.getExampleData(exampleIndex)); // and we set exampleData here\n                        list.add(scenario);\n                        for (String key : table.getKeys()) {\n                            scenario.replace(\"<\" + key + \">\", table.getValueAsString(key, i));\n                        }\n                    }\n                }\n            }\n        }\n        return list;\n    }\n    \n    public FeatureSection getSection() {\n        return section;\n    }\n    \n    public int getLine() {\n        return line;\n    }\n    \n    public void setLine(int line) {\n        this.line = line;\n    }\n    \n    public List<Tag> getTags() {\n        return tags;\n    }\n    \n    public void setTags(List<Tag> tags) {\n        this.tags = tags;\n    }\n    \n    public String getName() {\n        return name;\n    }\n    \n    public void setName(String name) {\n        this.name = name;\n    }\n    \n    public String getDescription() {\n        return description;\n    }\n    \n    public void setDescription(String description) {\n        this.description = description;\n    }\n    \n    public List<Step> getSteps() {\n        return steps;\n    }\n    \n    public void setSteps(List<Step> steps) {\n        this.steps = steps;\n    }\n    \n    public List<ExamplesTable> getExamplesTables() {\n        return examplesTables;\n    }\n    \n    public void setExamplesTables(List<ExamplesTable> examplesTables) {\n        this.examplesTables = examplesTables;\n    }\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ChannelSession.java\n/*\n * The MIT License\n *\n * Copyright 2023 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\n/**\n *\n * @author peter\n */\npublic interface ChannelSession {\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/PluginFactory.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic interface PluginFactory {\n    \n    Plugin create(ScenarioRuntime runtime, Map<String, Object> options);\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ParallelProcessor.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.concurrent.CompletableFuture;\nimport java.util.concurrent.ExecutorService;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author pthomas3\n */\npublic abstract class ParallelProcessor<T> {\n\n    private static final Logger logger = LoggerFactory.getLogger(ParallelProcessor.class);\n\n    private final ExecutorService executor;\n    private final ExecutorService monitor;\n    private final Iterator<T> publisher;\n    private final List<CompletableFuture> futures = new ArrayList();\n\n    public ParallelProcessor(ExecutorService executor, Iterator<T> publisher, ExecutorService monitor) {\n        this.executor = executor;\n        this.publisher = publisher;\n        this.monitor = monitor;\n    }\n\n    private Runnable toRunnable(final CompletableFuture prevFuture, final T next, final CompletableFuture future) {\n        return () -> {\n            if (prevFuture != null) {\n                prevFuture.join();\n            }\n            try {\n                process(next);\n            } catch (Exception e) {\n                logger.error(\"[parallel] input item failed: {}\", e.getMessage());\n            }\n            future.complete(Boolean.TRUE);\n        };\n    }\n\n    public void execute() {\n        CompletableFuture prevFuture = null;\n        while (publisher.hasNext()) {\n            final CompletableFuture future = new CompletableFuture();\n            futures.add(future);\n            T next = publisher.next();\n            boolean sync = shouldRunSynchronously(next);\n            executor.submit(toRunnable(prevFuture, next, future));\n            prevFuture = sync ? future : null;\n        }\n        final CompletableFuture[] futuresArray = futures.toArray(new CompletableFuture[futures.size()]);\n        monitor.submit(() -> {\n            CompletableFuture.allOf(futuresArray).join();\n            onComplete();\n        });\n    }\n\n    public boolean shouldRunSynchronously(T in) {\n        // parallel by default\n        // but allow a per work-item strategy\n        return false;\n    }\n\n    public abstract void process(T in);\n\n    public abstract void onComplete();\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/FeatureSection.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\n/**\n *\n * @author pthomas3\n */\npublic class FeatureSection {\n    \n    private int index;\n    private Scenario scenario;\n    private ScenarioOutline scenarioOutline;\n    \n    public int getLine() {\n        if (scenarioOutline != null) {\n            return scenarioOutline.getLine();\n        } else {\n            return scenario.getLine();\n        }\n    }\n\n    public int getIndex() {\n        return index;\n    }\n\n    public void setIndex(int index) {\n        this.index = index;\n    }        \n    \n    public boolean isOutline() {\n        return scenarioOutline != null;\n    }\n\n    public Scenario getScenario() {\n        return scenario;\n    }\n\n    public void setScenario(Scenario scenario) {\n        this.scenario = scenario;\n    }\n\n    public ScenarioOutline getScenarioOutline() {\n        return scenarioOutline;\n    }\n\n    public void setScenarioOutline(ScenarioOutline scenarioOutline) {\n        this.scenarioOutline = scenarioOutline;\n    }        \n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ExamplesTable.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n *\n * @author pthomas3\n */\npublic class ExamplesTable {\n\n    private final ScenarioOutline outline;\n    private final Table table;\n    private List<Tag> tags;\n\n    public ExamplesTable(ScenarioOutline outline, Table table) {\n        this.outline = outline;\n        this.table = table;\n        this.tags = new ArrayList();\n    }\n\n    public ScenarioOutline getOutline() {\n        return outline;\n    }\n\n    public List<Tag> getTags() {\n        return tags;\n    }\n\n    public void setTags(List<Tag> tags) {\n        this.tags = tags;\n    }\n\n    public Table getTable() {\n        return table;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ParserErrorListener.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.BitSet;\nimport org.antlr.v4.runtime.ANTLRErrorListener;\nimport org.antlr.v4.runtime.Parser;\nimport org.antlr.v4.runtime.RecognitionException;\nimport org.antlr.v4.runtime.Recognizer;\nimport org.antlr.v4.runtime.atn.ATNConfigSet;\nimport org.antlr.v4.runtime.dfa.DFA;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author pthomas3\n */\npublic class ParserErrorListener implements ANTLRErrorListener {\n\n    private static final Logger logger = LoggerFactory.getLogger(FeatureParser.class);\n\n    private String message;\n    private int line = -1;\n    private int position = -1;\n    private Object offendingSymbol;\n\n    public boolean isFail() {\n        return message != null;\n    }\n\n    public String getMessage() {\n        return message;\n    }\n\n    public int getLine() {\n        return line;\n    }\n\n    public int getPosition() {\n        return position;\n    }\n\n    public Object offendingSymbol() {\n        return offendingSymbol;\n    }\n\n    @Override\n    public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int position, String message, RecognitionException e) {\n        // logger.error(\"syntax error: {}\", message);\n        this.message = message;\n        this.line = line;\n        this.position = position;\n        this.offendingSymbol = offendingSymbol;\n    }\n\n    @Override\n    public void reportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, boolean exact, BitSet ambigAlts, ATNConfigSet configs) {\n//        if (logger.isTraceEnabled()) {\n//            logger.trace(\"reportAmbiguity: {} {} {} {} {} {} {}\", recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs);\n//        }\n    }\n\n    @Override\n    public void reportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet conflictingAlts, ATNConfigSet configs) {\n//        if (logger.isTraceEnabled()) {\n//            logger.trace(\"reportAttemptingFullContext: {} {} {} {} {} {}\", recognizer, dfa, startIndex, stopIndex, conflictingAlts, configs);\n//        }\n    }\n\n    @Override\n    public void reportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, int prediction, ATNConfigSet configs) {\n//        if (logger.isTraceEnabled()) {\n//            logger.trace(\"reportContextSensitivity: {} {} {} {} {} {}\", recognizer, dfa, startIndex, stopIndex, prediction, configs);\n//        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ChannelFactory.java\n/*\n * The MIT License\n *\n * Copyright 2023 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic interface ChannelFactory {\n    \n    Channel create(ScenarioRuntime runtime, Map<String, Object> options);\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/cli/IntellijHook.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.cli;\n\nimport com.intuit.karate.RuntimeHook;\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.Suite;\nimport com.intuit.karate.core.Feature;\nimport com.intuit.karate.core.Scenario;\nimport com.intuit.karate.core.FeatureRuntime;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport java.text.SimpleDateFormat;\nimport java.util.Date;\n\n/**\n *\n * @author pthomas3\n */\npublic class IntellijHook implements RuntimeHook {\n\n    @Override\n    public void beforeSuite(Suite suite) {\n        log(String.format(TEMPLATE_ENTER_THE_MATRIX, getCurrentTime()));\n    }\n\n    @Override\n    public boolean beforeScenario(ScenarioRuntime sr) {\n        if (sr.caller.depth == 0) {\n            Scenario scenario = sr.scenario;\n            String path = scenario.getFeature().getResource().getUri().toString();\n            log(String.format(TEMPLATE_TEST_STARTED, getCurrentTime(), path + \":\" + scenario.getLine(), escape(scenario.getRefIdAndName())));\n            // log(String.format(TEMPLATE_SCENARIO_STARTED, getCurrentTime()));\n        }\n        return true;\n    }\n\n    @Override\n    public void afterScenario(ScenarioRuntime sr) {\n        if (sr.caller.depth == 0) {\n            Scenario scenario = sr.scenario;\n            if (sr.result.isFailed()) {\n                StringUtils.Pair error = details(sr.result.getErrorMessage());\n                log(String.format(TEMPLATE_TEST_FAILED, getCurrentTime(), escape(error.right), escape(error.left), escape(scenario.getRefIdAndName()), \"\"));\n            }\n            log(String.format(TEMPLATE_TEST_FINISHED, getCurrentTime(), sr.result.getDurationNanos() / 1000000, escape(scenario.getRefIdAndName())));\n        }\n    }\n\n    @Override\n    public boolean beforeFeature(FeatureRuntime fr) {\n        if (fr.caller.depth == 0) {\n            Feature feature = fr.featureCall.feature;\n            String path = feature.getResource().getUri().toString();\n            log(String.format(TEMPLATE_TEST_SUITE_STARTED, getCurrentTime(), path + \":\" + feature.getLine(), escape(feature.getNameForReport())));\n        }\n        return true;\n    }\n\n    @Override\n    public void afterFeature(FeatureRuntime fr) {\n        if (fr.caller.depth == 0) {\n            log(String.format(TEMPLATE_TEST_SUITE_FINISHED, getCurrentTime(), escape(fr.featureCall.feature.getNameForReport())));\n        }\n    }\n\n    private static void log(String s) {\n        System.out.println(s);\n    }\n\n    \nprivate static String getCurrentTime() {\n        return DATE_FORMAT.format(new Date());\n    }\n\n    private static String escape(String source) {\n        if (source == null) {\n            return \"\";\n        }\n        return source.replace(\"|\", \"||\").replace(\"\\n\", \"|n\").replace(\"\\r\", \"|r\").replace(\"'\", \"|'\").replace(\"[\", \"|[\").replace(\"]\", \"|]\");\n    }\n\n    private static StringUtils.Pair details(String errorMessage) {\n        String fullMessage = errorMessage.replace(\"\\r\", \"\").replace(\"\\t\", \"  \");\n        String[] messageInfo = fullMessage.split(\"\\n\", 2);\n        if (messageInfo.length == 2) {\n            return StringUtils.pair(messageInfo[0].trim(), messageInfo[1].trim());\n        } else {\n            return StringUtils.pair(fullMessage, \"\");\n        }\n    }\n\n    private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(\"yyyy-MM-dd'T'hh:mm:ss.SSSZ\");\n\n    private static final String TEAMCITY_PREFIX = \"##teamcity\";\n    private static final String TEMPLATE_TEST_STARTED = TEAMCITY_PREFIX + \"[testStarted timestamp = '%s' locationHint = '%s' captureStandardOutput = 'true' name = '%s']\";\n    private static final String TEMPLATE_TEST_FAILED = TEAMCITY_PREFIX + \"[testFailed timestamp = '%s' details = '%s' message = '%s' name = '%s' %s]\";\n    private static final String TEMPLATE_TEST_FINISHED = TEAMCITY_PREFIX + \"[testFinished timestamp = '%s' duration = '%s' name = '%s']\";\n    private static final String TEMPLATE_ENTER_THE_MATRIX = TEAMCITY_PREFIX + \"[enteredTheMatrix timestamp = '%s']\";\n    private static final String TEMPLATE_TEST_SUITE_STARTED = TEAMCITY_PREFIX + \"[testSuiteStarted timestamp = '%s' locationHint = 'file://%s' name = '%s']\";\n    private static final String TEMPLATE_TEST_SUITE_FINISHED = TEAMCITY_PREFIX + \"[testSuiteFinished timestamp = '%s' name = '%s']\";\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/cli/Main.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.cli;\n\n/**\n *\n * @author pthomas3\n */\npublic class Main { // backwards compatibility for vs-code karate-runner plugin\n\n    public static void main(String[] args) {\n        com.intuit.karate.Main.main(args);\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/cli/IdeMain.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.cli;\n\nimport com.intuit.karate.Main;\nimport com.intuit.karate.Runner;\nimport com.intuit.karate.StringUtils;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport com.intuit.karate.shell.Command;\nimport picocli.CommandLine;\n\n/**\n *\n * @author pthomas3\n */\npublic class IdeMain {\n\n    public static void main(String[] args) {\n        String command;\n        if (args.length > 0) {\n            command = StringUtils.join(args, ' ');\n        } else {\n            command = System.getProperty(\"sun.java.command\");\n        }\n        System.out.println(\"command: \" + command);\n        Main ro = parseIdeCommandLine(command);\n        Runner.Builder rb = Runner.builder();\n        if (command.contains(\"org.jetbrains\")) {\n            rb.hook(new IntellijHook());\n        }\n        rb.path(ro.getPaths())\n                .tags(ro.getTags())\n                .scenarioName(ro.getName())\n                .parallel(ro.getThreads());\n    }\n\n    private static final Pattern CLI_PLUGIN = Pattern.compile(\"--plugin\\\\s+[^\\\\s]+\\\\s\");\n    private static final Pattern CLI_GLUE = Pattern.compile(\"--glue\\\\s+[^\\\\s]+\\\\s+\");\n    private static final Pattern CLI_NAME = Pattern.compile(\"--name \\\"?([^$\\\"]+[^ \\\"]+)\\\"?\");\n\n    public static Main parseStringArgs(String[] args) {\n        Main options = CommandLine.populateCommand(new Main(), args);\n        List<String> paths = new ArrayList();\n        if (options.getPaths() != null) {\n            for (String s : options.getPaths()) {\n                if (s.startsWith(\"com.\") || s.startsWith(\"cucumber.\") || s.startsWith(\"org.\")) {\n                    continue;\n                }\n                paths.add(s);\n            }\n            options.setPaths(paths.isEmpty() ? null : paths);\n        }\n        return options;\n    }\n    \n    private static final String CLI_MAIN = \"cucumber.api.cli.Main\";\n\n    public static Main parseIdeCommandLine(String line) {\n        int pos = line.indexOf(CLI_MAIN);\n        if (pos != -1) {\n            line = line.substring(pos + CLI_MAIN.length());\n        }\n        line = line.replace(\"--monochrome\", \"\");\n        Matcher pluginMatcher = CLI_PLUGIN.matcher(line);\n        if (pluginMatcher.find()) {\n            line = pluginMatcher.replaceFirst(\"\");\n        }\n        Matcher glueMatcher = CLI_GLUE.matcher(line);\n        if (glueMatcher.find()) {\n            line = glueMatcher.replaceFirst(\"\");\n        }\n        Matcher nameMatcher = CLI_NAME.matcher(line);\n        String nameTemp;\n        if (nameMatcher.find()) {\n            nameTemp = nameMatcher.group(1);\n            line = nameMatcher.replaceFirst(\"\");\n        } else {\n            nameTemp = null;\n        }\n        Main options = Main.parseKarateOptionsAndQuotePath(line);\n        options.setName(nameTemp);\n        return options;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/shell/Console.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.shell;\n\nimport com.intuit.karate.LogAppender;\nimport com.intuit.karate.Logger;\nimport java.io.BufferedReader;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.util.function.Consumer;\n\n/**\n *\n * @author pthomas3\n */\npublic class Console extends Thread {\n\n    private final boolean useLineFeed;\n    private final InputStream is;\n    private final BufferedReader reader;\n    private final Logger logger;\n    private final LogAppender appender;\n    private final StringBuilder buffer;\n    private final Consumer<String> listener;\n    \n    public String getBuffer() {\n        return buffer.toString();\n    }\n\n    public Console(String name, boolean useLineFeed, InputStream is, Logger logger, LogAppender appender, Consumer<String> listener) {\n        super(name);\n        this.useLineFeed = useLineFeed;\n        this.is = is;\n        this.buffer = new StringBuilder();\n        reader = new BufferedReader(new InputStreamReader(is));\n        this.logger = logger;\n        this.appender = appender;\n        this.listener = listener;\n    }\n\n    @Override\n    public void run() {\n        String line;\n        try {\n            while ((line = reader.readLine()) != null) {\n                appender.append(line);\n                buffer.append(line);\n                logger.debug(\"{}\", line);\n                if (useLineFeed) {\n                    buffer.append('\\n');\n                }\n                if (listener != null) {\n                    listener.accept(line);\n                }\n            }\n        } catch (Exception e) {\n            logger.error(\"console reader error: {}\", e.getMessage());\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/shell/Stoppable.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.shell;\n\n/**\n *\n * @author pthomas3\n */\npublic interface Stoppable {\n    \n    void stop();\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/shell/FileLogAppender.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.shell;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.LogAppender;\nimport java.io.File;\nimport java.io.RandomAccessFile;\nimport java.nio.Buffer;\nimport java.nio.ByteBuffer;\nimport java.nio.channels.FileChannel;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author pthomas3\n */\npublic class FileLogAppender implements LogAppender {\n    \n    private static final Logger LOGGER = LoggerFactory.getLogger(FileLogAppender.class);\n\n    private final RandomAccessFile file;\n    private final FileChannel channel;\n    private int prevPos;\n    private boolean closed;\n\n    public FileLogAppender(File in) {\n        try {\n            if (in == null) {\n                in = File.createTempFile(\"karate\", \"tmp\");\n            } else {\n                if (!in.getParentFile().exists()) {\n                    in.getParentFile().mkdirs();\n                }\n            }\n            file = new RandomAccessFile(in, \"rw\");\n            channel = file.getChannel();            \n            prevPos = (int) channel.position();\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n    \n    private String getBuffer(boolean resetAndClear) {\n        try {\n            int pos = (int) channel.position();\n            ByteBuffer buf = ByteBuffer.allocate(pos - prevPos);\n            channel.read(buf, prevPos);\n            if (resetAndClear) {\n                prevPos = pos;\n            }\n            ((Buffer) buf).flip(); // java 8 to 9 fix\n            return FileUtils.toString(buf.array());\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }         \n    }\n\n    @Override\n    public String getBuffer() {\n        return getBuffer(false);\n    }   \n    \n    @Override\n    public String collect() {\n        return getBuffer(true);\n    }\n\n    @Override\n    public void append(String text) {\n        if (closed) {\n            return;\n        }\n        try {\n            channel.write(ByteBuffer.wrap(FileUtils.toBytes(text)));\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n\n    @Override\n    public void close() {\n        try {\n            file.close();\n            closed = true;\n        } catch (Exception e) {\n            LOGGER.warn(\"log appender close failed: {}\", e.getMessage());\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/shell/StopListenerThread.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.shell;\n\nimport java.io.BufferedReader;\nimport java.io.InputStreamReader;\nimport java.io.OutputStream;\nimport java.net.InetAddress;\nimport java.net.ServerSocket;\nimport java.net.Socket;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author pthomas3\n */\npublic class StopListenerThread extends Thread {\n    \n    private static final Logger logger = LoggerFactory.getLogger(StopListenerThread.class);\n    \n    private Stoppable stoppable;\n    private ServerSocket socket;\n\n    public StopListenerThread(int port, Stoppable stoppable) {\n        this.stoppable = stoppable;\n        setDaemon(true);\n        setName(\"stop-listener-\" + port);\n        try {\n            socket = new ServerSocket(port, 1, InetAddress.getByName(\"127.0.0.1\"));\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }\n    \n    public int getPort() {\n        return socket.getLocalPort();\n    }\n\n    @Override\n    public void run() {\n        logger.info(\"starting thread: {}\", getName());\n        Socket accept;\n        try {\n            accept = socket.accept();\n            BufferedReader reader = new BufferedReader(new InputStreamReader(accept.getInputStream()));\n            reader.readLine();\n            logger.info(\"shutting down thread: {}\", getName());\n            stoppable.stop();\n            accept.close();\n            socket.close();\n        } catch (Exception e) {\n            throw new RuntimeException(e);\n        }\n    }   \n    \n    public static void stop(int port) {\n         try {\n            Socket s = new Socket(InetAddress.getByName(\"127.0.0.1\"), port);\n            OutputStream out = s.getOutputStream();\n            logger.info(\"sending stop request to port: {}\", port);\n            out.write((\"\\r\\n\").getBytes());\n            out.flush();\n            s.close();\n...\n", "needle_token_start": 10264, "needle_token_end": 10288, "code_context_ntokens": 15814, "output": ["```java\nprivate static String getCurrentTime() {\n    return DATE_FORMAT.format(new Date());\n}\n```"]}
{"repo": "karatelabs/karate", "name": "enableForUri", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/http/HttpLogModifier.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To determine if logging should be enabled for a specific web address.\n2. **Input**: A string representing the web address (URI).\n3. **Output**: A boolean value indicating whether logging is enabled for the given web address.\n4. **Procedure**: The method checks the provided web address against certain criteria (not detailed in the provided code snippet) to decide if logging activities should be performed for that specific URI.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: karate-core/src/main/java/com/intuit/karate/template/TemplateContext.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n...\n// Path: karate-core/src/main/java/com/intuit/karate/template/KarateTemplateEngine.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.graal.JsEngine;\nimport java.io.IOException;\nimport java.io.Writer;\nimport java.util.Map;\nimport java.util.function.Supplier;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.thymeleaf.IEngineConfiguration;\nimport org.thymeleaf.TemplateEngine;\nimport org.thymeleaf.TemplateSpec;\nimport org.thymeleaf.context.IContext;\nimport org.thymeleaf.context.IEngineContext;\nimport org.thymeleaf.context.StandardEngineContextFactory;\nimport org.thymeleaf.dialect.IDialect;\nimport org.thymeleaf.engine.TemplateData;\nimport org.thymeleaf.engine.TemplateManager;\nimport org.thymeleaf.exceptions.TemplateOutputException;\nimport org.thymeleaf.exceptions.TemplateProcessingException;\nimport org.thymeleaf.templatemode.TemplateMode;\nimport org.thymeleaf.templateresolver.ITemplateResolver;\nimport org.thymeleaf.util.FastStringWriter;\n\n/**\n *\n * @author pthomas3\n */\npublic class KarateTemplateEngine {\n\n    private static final Logger logger = LoggerFactory.getLogger(KarateTemplateEngine.class);\n\n    private final StandardEngineContextFactory standardFactory;\n    private final TemplateEngine wrapped;\n\n    public KarateTemplateEngine(Supplier<JsEngine> jsEngine, IDialect... dialects) {\n        standardFactory = new StandardEngineContextFactory();\n        wrapped = new TemplateEngine();\n        wrapped.setEngineContextFactory((IEngineConfiguration ec, TemplateData data, Map<String, Object> attrs, IContext context) -> {\n            IEngineContext engineContext = standardFactory.createEngineContext(ec, data, attrs, context);\n            return KarateEngineContext.initThreadLocal(engineContext, jsEngine.get());\n        });\n        // the next line is a set which clears and replaces all existing / default\n        wrapped.setDialect(new KarateStandardDialect());\n        for (IDialect dialect : dialects) {\n            wrapped.addDialect(dialect);\n        }\n    }\n\n    public void addTemplateResolver(ITemplateResolver templateResolver) {\n        wrapped.addTemplateResolver(templateResolver);\n    }\n\n    public void setTemplateResolver(ITemplateResolver templateResolver) {\n        wrapped.setTemplateResolver(templateResolver);\n    }\n\n    public String process(String template) {\n        return process(template, TemplateContext.LOCALE_US);\n    }\n\n    public String process(String template, IContext context) {\n        TemplateSpec templateSpec = new TemplateSpec(template, TemplateMode.HTML);\n        Writer stringWriter = new FastStringWriter(100);\n        process(templateSpec, context, stringWriter);\n        return stringWriter.toString();\n    }\n\n    public void process(TemplateSpec templateSpec, IContext context, Writer writer) {\n        try {\n            TemplateManager templateManager = wrapped.getConfiguration().getTemplateManager();\n            templateManager.parseAndProcess(templateSpec, context, writer);\n            try {\n                writer.flush();\n            } catch (IOException e) {\n                throw new TemplateOutputException(\"error flushing output writer\", templateSpec.getTemplate(), -1, -1, e);\n            }\n        } catch (Exception e) {\n            KarateEngineContext kec = KarateEngineContext.get();\n            if (kec != null && !kec.isRedirect()) { // don't log redirects\n                // make thymeleaf errors easier to troubleshoot from the logs\n                while (e.getCause() instanceof Exception) {\n                    e = (Exception) e.getCause();\n                    if (e instanceof TemplateProcessingException) {\n                        logger.error(\"{}\", e.getMessage()); // will print line and col numbers\n                        if (e.getCause() != null) { // typically the js error\n                            String message = e.getCause().getMessage();\n                            logger.error(\"{}\", message);\n                        }\n                        break;\n                    }\n                }\n            }\n            if (logger.isTraceEnabled()) {\n                logger.trace(\"{}\", StringUtils.throwableToString(e));\n            }\n            throw new RuntimeException(e);\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/resource/ResourceResolver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.resource;\n\nimport com.intuit.karate.FileUtils;\n\n/**\n *\n * @author pthomas3\n */\npublic class ResourceResolver {\n\n    public final boolean classpath;\n    public final String root;\n\n    private static final String EMPTY = \"\";\n    private static final String SLASH = \"/\";\n\n    public ResourceResolver(String root) {\n        if (root == null) {\n            root = EMPTY;\n        }\n        classpath = root.startsWith(Resource.CLASSPATH_COLON);\n        root = ResourceUtils.removePrefix(root);\n        if (!root.isEmpty() && !root.endsWith(SLASH)) {\n            root = root + SLASH;\n        }\n        this.root = root;\n    }\n\n    public Resource resolve(String path) {\n        return resolve(null, path);\n    }\n\n    public Resource resolve(String caller, String path) {\n        if (path.startsWith(Resource.CLASSPATH_COLON)) {\n            return get(path);\n        }\n        String prefix = classpath ? Resource.CLASSPATH_COLON + root : root;\n        if (path.startsWith(Resource.THIS_COLON) && caller != null) {\n            return get(prefix + ResourceUtils.getParentPath(caller) + path.substring(5));\n        }\n        return get(prefix + (path.charAt(0) == '/' ? path.substring(1) : path));\n    }\n\n    private Resource get(String path) {\n        return ResourceUtils.getResource(FileUtils.WORKING_DIR, path);\n    }\n\n    @Override\n    public String toString() {\n        return classpath ? Resource.CLASSPATH_COLON + root : root;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/RequestCycle.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\nimport com.intuit.karate.graal.JsEngine;\nimport com.intuit.karate.graal.JsValue;\nimport com.intuit.karate.resource.ResourceResolver;\nimport com.intuit.karate.template.KarateTemplateEngine;\nimport java.io.InputStream;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.function.Supplier;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author pthomas3\n */\npublic class RequestCycle {\n\n    private static final Logger logger = LoggerFactory.getLogger(RequestCycle.class);\n\n    public static final String CONTEXT = \"context\";\n    private static final String REQUEST = \"request\";\n    protected static final String SESSION = \"session\";\n    private static final String RESPONSE = \"response\";\n\n    private static final ThreadLocal<RequestCycle> THREAD_LOCAL = new ThreadLocal();\n\n    public static RequestCycle get() {\n        return THREAD_LOCAL.get();\n    }\n\n    protected static RequestCycle init(KarateTemplateEngine te, ServerContext context) {\n        RequestCycle rc = new RequestCycle(JsEngine.global(), te, context);\n        THREAD_LOCAL.set(rc);\n        return rc;\n    }\n\n    private final JsEngine engine;\n    private final KarateTemplateEngine templateEngine;\n    private final Request request;\n    private final Response response;\n    private final ServerContext context;\n    private final ServerConfig config;\n    private final Supplier<Response> customHandler;\n\n    private String switchTemplate;\n    private Map<String, Object> switchParams;\n\n    private RequestCycle(JsEngine engine, KarateTemplateEngine templateEngine, ServerContext context) {\n        this.engine = engine;\n        this.templateEngine = templateEngine;\n        this.context = context;\n        config = context.getConfig();\n        customHandler = context.getCustomHandler();\n        Session session = context.getSession();\n        if (session != null && !session.isTemporary()) {\n            engine.put(SESSION, session.getData());\n        } else {\n            // easier for users to write code such as\n            // if (session.foo) {}\n            engine.put(SESSION, Collections.emptyMap());\n        }\n        // this has to be after the session init\n        Map<String, Object> variables = context.getVariables();\n        if (variables != null) {\n            engine.putAll(variables);\n        }\n        request = context.getRequest();\n        request.processBody();\n        engine.put(REQUEST, request);\n        response = new Response(200);\n        engine.put(RESPONSE, response);\n        engine.put(CONTEXT, context);\n    }\n\n    public RequestCycle copy(Request request, Map<String, Object> variables) {\n        ServerContext temp = new ServerContext(config, request, variables);\n        temp.setSession(context.getSession());\n        return new RequestCycle(JsEngine.local(), templateEngine, temp);\n    }\n\n    public JsEngine getEngine() {\n        return engine;\n    }\n\n    public KarateTemplateEngine getTemplateEngine() {\n        return templateEngine;\n    }\n\n    public ResourceResolver getResourceResolver() {\n        return config.getResourceResolver();\n    }\n\n    private void close() {\n        Session session = context.getSession();\n        if (session != null && !session.isTemporary()) {\n            if (context.isClosed()) {\n                // note that session cookie is deleted in response-builder\n                context.getConfig().getSessionStore().delete(session.getId());\n                logger.debug(\"session deleted: {}\", session.getId());\n            } else {\n                JsValue sessionValue = engine.get(SESSION);\n                if (sessionValue.isObject()) {\n                    session.getData().putAll(sessionValue.getAsMap());\n                    context.getConfig().getSessionStore().save(session);\n                } else {\n                    logger.error(\"invalid session, not map-like: {}\", sessionValue);\n                }\n            }\n        }\n        JsEngine.remove();\n        THREAD_LOCAL.remove();\n    }\n\n    public Session getSession() {\n        return context.getSession();\n    }\n\n    public Request getRequest() {\n        return request;\n    }\n\n    public Response getResponse() {\n        return response;\n    }\n\n    public ServerContext getContext() {\n        return context;\n    }\n\n    public void setSwitchTemplate(String switchTemplate) {\n        this.switchTemplate = switchTemplate;\n    }\n\n    public String getSwitchTemplate() {\n        return switchTemplate;\n    }\n\n    public void setSwitchParams(Map<String, Object> switchParams) {\n        // clone to allow context.switch('pagename', request.params);\n        // else the htmlResponse() routine will clear \"self\"\n        this.switchParams = new HashMap(switchParams);\n    }\n\n    protected Response handle() {\n        try {\n            if (customHandler != null) {\n                return customHandler.get();\n            } else if (context.isApi()) {\n                InputStream is = apiResource();\n                if (context.isLockNeeded()) {\n                    synchronized (config) {\n                        engine.eval(is);\n                    }\n                } else {\n                    engine.eval(is);\n                }\n                return response().build();\n            } else {\n                return htmlResponse();\n            }\n        } catch (Exception e) {\n            logger.error(\"handle failed: {}\", e.getMessage());\n            response.setStatus(500); // just for logging below\n            return response().buildWithStatus(500);\n        } finally {\n            close();\n            if (logger.isDebugEnabled()) {\n                logger.debug(\"{} {} [{} ms]\", request, response.getStatus(), System.currentTimeMillis() - request.getStartTime());\n            }\n        }\n    }\n\n    private Response htmlResponse() {\n        String html;\n        try {\n            html = templateEngine.process(request.getPath());\n        } catch (Exception e) {\n            if (context.isSwitched()) {\n                if (switchTemplate == null) {\n                    logger.debug(\"abort template requested\");\n                    html = null;\n                } else {\n                    logger.debug(\"switch template requested: {}\", switchTemplate);\n                    request.getParams().clear();\n                    if (switchParams != null) {\n                        switchParams.forEach((k, v) -> request.setParam(k, v));\n                    }\n                    html = templateEngine.process(switchTemplate);\n                }\n            } else {\n                throw e;\n            }\n        }\n        return response().html(html).build();\n    }\n\n    private static final String DOT_JS = \".js\";\n\n    private InputStream apiResource() {\n        String resourcePath = request.getResourcePath();\n        String jsPath = resourcePath == null ? request.getPathOriginal() + DOT_JS : resourcePath;\n        try {\n            return config.getResourceResolver().resolve(jsPath).getStream();\n        } catch (Exception e) {\n            throw new RuntimeException(\"failed to resolve api resource: \" + resourcePath + \", \" + request + \", \" + e.getMessage());\n        }\n    }\n\n    public ResponseBuilder response() {\n        return new ResponseBuilder(config, this).session(context.getSession(), context.isNewSession());\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/Variable.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.XmlUtils;\nimport com.intuit.karate.graal.JsValue;\nimport com.intuit.karate.Json;\nimport com.intuit.karate.JsonUtils;\nimport java.util.ArrayList;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.function.Function;\nimport org.graalvm.polyglot.Value;\nimport org.graalvm.polyglot.proxy.ProxyExecutable;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.w3c.dom.Node;\n\n/**\n *\n * @author pthomas3\n */\npublic class Variable {\n    \n    private static final Logger logger = LoggerFactory.getLogger(Variable.class);\n    \n    public static enum Type {\n        NULL,\n        BOOLEAN,\n        NUMBER,\n        STRING,\n        BYTES,\n        LIST,\n        MAP,\n        XML,\n        JS_FUNCTION,\n        JAVA_FUNCTION,\n        FEATURE,\n        OTHER\n    }\n    \n    public static final Variable NULL = new Variable(null);\n    public static final Variable NOT_PRESENT = new Variable(\"#notpresent\");\n    \n    public final Type type;\n    private final Object value;\n    \n    public Variable(Object o) {\n        if (o instanceof Value) {\n            o = new JsValue((Value) o).getValue();\n        } else if (o instanceof JsValue) {\n            o = ((JsValue) o).getValue();\n        }\n        if (o == null) {\n            type = Type.NULL;\n        } else if (o instanceof ProxyExecutable) {\n            type = Type.JS_FUNCTION;\n        } else if (o instanceof Value) {\n            type = Type.OTHER; // java.lang.Class            \n        } else if (o instanceof Function) {\n            type = Type.JAVA_FUNCTION;\n        } else if (o instanceof Node) {\n            type = Type.XML;\n        } else if (o instanceof List) {\n            type = Type.LIST;\n        } else if (o instanceof Map) {\n            type = Type.MAP;\n        } else if (o instanceof String) {\n            type = Type.STRING;\n        } else if (Number.class.isAssignableFrom(o.getClass())) {\n            type = Type.NUMBER;\n        } else if (Boolean.class.equals(o.getClass())) {\n            type = Type.BOOLEAN;\n        } else if (o instanceof byte[]) {\n            type = Type.BYTES;\n        } else if (o instanceof FeatureCall) {\n            type = Type.FEATURE;\n        } else {\n            type = Type.OTHER;\n        }\n        value = o;\n    }\n    \n    public <T> T getValue() {\n        return (T) value;\n    }\n    \n    public boolean isJsOrJavaFunction() {\n        return type == Type.JS_FUNCTION || type == Type.JAVA_FUNCTION;\n    }\n    \n    public boolean isJavaFunction() {\n        return type == Type.JAVA_FUNCTION;\n    }\n    \n    public boolean isJsFunction() {\n        return type == Type.JS_FUNCTION;\n    }\n    \n    public boolean isBytes() {\n        return type == Type.BYTES;\n    }\n    \n    public boolean isString() {\n        return type == Type.STRING;\n    }\n    \n    public boolean isList() {\n        return type == Type.LIST;\n    }\n    \n    public boolean isMap() {\n        return type == Type.MAP;\n    }\n    \n    public boolean isMapOrList() {\n        return type == Type.MAP || type == Type.LIST;\n    }\n    \n    public boolean isXml() {\n        return type == Type.XML;\n    }\n    \n    public boolean isNumber() {\n        return type == Type.NUMBER;\n    }\n    \n    public boolean isNull() {\n        return type == Type.NULL;\n    }\n    \n    public boolean isOther() {\n        return type == Type.OTHER;\n    }\n    \n    public boolean isFeature() {\n        return type == Type.FEATURE;\n    }\n    \n    public boolean isBoolean() {\n        return type == Type.BOOLEAN;\n    }\n    \n    public boolean isTrue() {\n        return type == Type.BOOLEAN && ((Boolean) value);\n    }\n    \n    public String getTypeString() {\n        return type.name().toLowerCase();\n    }\n    \n    public Node getAsXml() {\n        switch (type) {\n            case XML:\n                return getValue();\n            case MAP:\n                return XmlUtils.fromMap(getValue());\n            case STRING:\n            case BYTES:\n                String xml = getAsString();\n                return XmlUtils.toXmlDoc(xml);\n            case OTHER: // POJO\n                return XmlUtils.fromJavaObject(value);\n            default:\n                throw new RuntimeException(\"cannot convert to xml:\" + this);\n        }\n    }\n    \n    public Object getValueAndConvertIfXmlToMap() {\n        return isXml() ? XmlUtils.toObject(getValue()) : value;\n    }\n    \n    public Object getValueAndForceParsingAsJson() {\n        switch (type) {\n            case LIST:\n            case MAP:\n                return value;\n            case STRING:\n            case BYTES:\n                return JsonUtils.fromJson(getAsString());\n            case XML:\n                return XmlUtils.toObject(getValue());\n            case OTHER: // pojo\n                return Json.of(value).value();\n            default:\n                throw new RuntimeException(\"cannot convert to json: \" + this);\n        }\n        \n    }\n    \n    public byte[] getAsByteArray() {\n        if (type == Type.BYTES) {\n            return getValue();\n        } else {\n            return FileUtils.toBytes(getAsString());\n        }\n    }\n    \n    public String getAsString() {\n        switch (type) {\n            case NULL:\n                return null;\n            case BYTES:\n                return FileUtils.toString((byte[]) value);\n            case LIST:\n            case MAP:\n                try {\n                return JsonUtils.toJson(value);\n            } catch (Throwable t) {\n                logger.warn(\"conversion to json string failed, will attempt to use fall-back approach: {}\", t.getMessage());\n                return JsonUtils.toJsonSafe(value, false);\n            }\n            case XML:\n                return XmlUtils.toString(getValue());\n            default:\n                return value.toString();\n        }\n    }\n    \n    public String getAsPrettyString() {\n        switch (type) {\n            case LIST:\n            case MAP:\n                return JsonUtils.toJsonSafe(value, true);\n            case XML:\n                return getAsPrettyXmlString();\n            default:\n                return getAsString();\n        }\n    }\n    \n    public String getAsPrettyXmlString() {\n        return XmlUtils.toString(getAsXml(), true);\n    }\n    \n    public int getAsInt() {\n        if (isNumber()) {\n            return ((Number) value).intValue();\n        } else {\n            return Integer.valueOf(getAsString());\n        }\n    }\n    \n    public Variable copy(boolean deep) {\n        switch (type) {\n            case LIST:\n                return deep ? new Variable(JsonUtils.deepCopy(value)) : new Variable(new ArrayList((List) value));\n            case MAP:\n                return deep ? new Variable(JsonUtils.deepCopy(value)) : new Variable(new LinkedHashMap((Map) value));\n            case XML:\n                return new Variable(XmlUtils.toXmlDoc(getAsString()));\n            default:\n                return this;\n        }\n    }\n    \n    public Variable toLowerCase() {\n        switch (type) {\n            case STRING:\n                return new Variable(getAsString().toLowerCase());\n            case LIST:\n            case MAP:\n                String json = getAsString().toLowerCase();\n                return new Variable(JsonUtils.fromJson(json));\n            case XML:\n                String xml = getAsString().toLowerCase();\n                return new Variable(XmlUtils.toXmlDoc(xml));\n            default:\n                return this;\n        }\n    }\n    \n    public boolean isNotPresent() {\n        return \"#notpresent\".equals(value);\n    }    \n    \n    @Override\n    public String toString() {\n        StringBuilder sb = new StringBuilder();\n        sb.append(\"[type: \").append(type);\n        sb.append(\", value: \").append(value);\n        sb.append(\"]\");\n        return sb.toString();\n    }\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/DockerTarget.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.KarateException;\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.shell.Command;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.function.Function;\n\n/**\n *\n * @author pthomas3\n */\npublic class DockerTarget implements Target {\n\n    private final String imageId;\n    private String containerId;\n    private Function<Integer, String> command;\n    private final Map<String, Object> options;\n    private boolean pull = false;\n    private boolean karateChrome = false;\n\n    static final Logger logger = LoggerFactory.getLogger(DockerTarget.class);\n\n\n    public DockerTarget(String dockerImage) {\n        this(Collections.singletonMap(\"docker\", dockerImage));\n    }\n\n    public DockerTarget(Map<String, Object> options) {\n        this.options = options;\n        if (options != null) {\n            imageId = (String) options.get(\"docker\");\n            Integer vncPort = (Integer) options.get(\"vncPort\");\n            String secComp = (String) options.get(\"secComp\");\n            Boolean temp = (Boolean) options.get(\"pull\");\n            pull = temp == null ? false : temp;\n            StringBuilder sb = new StringBuilder();\n            sb.append(\"docker run -d -e KARATE_SOCAT_START=true\");\n            if (secComp == null) {\n                sb.append(\" --cap-add=SYS_ADMIN\");\n            } else {\n                sb.append(\" --security-opt seccomp=\").append(secComp);\n            }\n            if (vncPort != null) {\n                sb.append(\" -p \").append(vncPort).append(\":5900\");\n            }\n            if (imageId != null) {\n                if (imageId.contains(\"/chrome-headless\")) {\n                    command = p -> sb.toString() + \" -p 9222 \" + imageId;\n                } else if (imageId.contains(\"/karate-chrome\")) {\n                    karateChrome = true;\n                    command = p -> sb.toString() + \" -p 9222 \" + imageId;\n                }\n            }\n        } else {\n            imageId = null;\n        }\n    }\n\n    public void setCommand(Function<Integer, String> command) {\n        this.command = command;\n    }\n\n    public Function<Integer, String> getCommand() {\n        return command;\n    }\n\n    @Override\n    public Map<String, Object> start(ScenarioRuntime sr) {\n        if (command == null) {\n            throw new RuntimeException(\"docker target command (function) not set\");\n        }\n        if (imageId != null && pull) {\n            sr.logger.debug(\"attempting to pull docker image: {}\", imageId);\n            Command.execLine(null, \"docker pull \" + imageId);\n        }\n        containerId = Command.execLine(null, command.apply(null));\n        int port = this.getContainerPort(containerId);\n        Map<String, Object> map = new HashMap();\n        if (options != null) {\n            map.putAll(options);\n        }\n\n        boolean remoteHost = options != null && options.get(\"remoteHost\") != null && (Boolean) options.get(\"remoteHost\");\n        boolean useDockerHost = options != null && options.get(\"useDockerHost\") != null && (Boolean) options.get(\"useDockerHost\");\n        String host = \"127.0.0.1\";\n\n        if (remoteHost) {\n            String gateway = Command.execLine(null, \"docker inspect -f '{{.NetworkSettings.Gateway}}' \" + containerId);\n            host = gateway.replaceAll(\"'\", \"\"); // Some responses are wrapped in single quotes.\n        } else if (useDockerHost) {\n            host = \"host.docker.internal\";\n        }\n\n        map.put(\"start\", false);\n        map.put(\"host\", host);\n        map.put(\"port\", port);\n        map.put(\"type\", \"chrome\");\n        Command.waitForHttp(\"http://\" + host + \":\" + port + \"/json\");\n        return map;\n    }\n\n\n    @Override\n    public Map<String, Object> stop(ScenarioRuntime sr) {\n        Command.execLine(null, \"docker stop \" + containerId);\n        if (!karateChrome) { // no video\n            Command.execLine(null, \"docker rm \" + containerId);\n            return Collections.EMPTY_MAP;\n        }\n        String shortName = containerId.contains(\"_\") ? containerId : StringUtils.truncate(containerId, 12, false);\n        String dirName = \"karate-chrome_\" + shortName;\n        String buildDir = sr.featureRuntime.suite.buildDir;\n        String resultsDir = buildDir + File.separator + dirName;\n        Command.execLine(null, \"docker cp \" + containerId + \":/tmp \" + resultsDir);\n        Command.execLine(null, \"docker rm \" + containerId);\n        String video = resultsDir + File.separator + \"karate.mp4\";\n        File file = new File(video);\n        if (!file.exists()) {\n            sr.logger.warn(\"video file missing: {}\", file);\n            return Collections.EMPTY_MAP;\n        }\n        File copy = new File(buildDir + File.separator + dirName + \".mp4\");\n        FileUtils.copy(file, copy);\n        return Collections.singletonMap(\"video\", copy.getPath());\n    }\n\n    private int getContainerPort(String containerId) {\n        // 9222/tcp is the default port for chrome headless\n        String format = \"--format='{{(index (index .NetworkSettings.Ports \\\"9222/tcp\\\") 0).HostPort}}'\";\n\n        if (FileUtils.isOsWindows()) {\n            format = format.replace(\"\\\"\", \"\\\\\\\"\");\n        }\n\n        logger.debug(\"cmd: docker format {} {}\", format, containerId);\n        String dockerInspect = Command.exec(false, (File) null,\n                \"docker\", \"inspect\", format, containerId\n        );\n\n        // Certain OS responses come back with single quotes with Command.exec\n        dockerInspect = dockerInspect.replaceAll(\"[^\\\\d]\", \"\");\n        logger.debug(\"docker inspect command output: {}\", dockerInspect);\n        try {\n            return Integer.parseInt(dockerInspect);\n        } catch (NumberFormatException e) {\n            throw new KarateException(\"Error fetching port from started docker container\", e);\n        }\n    }\n\n    public String getContainerId() {\n        return this.containerId;\n    }\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/Target.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.core.ScenarioRuntime;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic interface Target {        \n    \n    Map<String, Object> start(ScenarioRuntime sr);\n    \n    Map<String, Object> stop(ScenarioRuntime sr);\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/Cookies.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\nimport io.netty.handler.codec.http.cookie.ClientCookieEncoder;\nimport io.netty.handler.codec.http.cookie.Cookie;\nimport io.netty.handler.codec.http.cookie.CookieHeaderNames;\nimport io.netty.handler.codec.http.cookie.DefaultCookie;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *\n * @author pthomas3\n */\npublic class Cookies {\n\n    private static final Logger logger = LoggerFactory.getLogger(Cookies.class);\n\n    private Cookies() {\n        // only static methods               \n    }\n\n    public static final String NAME = \"name\";\n    public static final String VALUE = \"value\";\n    public static final String WRAP = \"wrap\";\n    public static final String DOMAIN = \"domain\";\n    public static final String PATH = \"path\";\n    public static final String MAX_AGE = \"max-age\"; // only one with a hyphen\n    public static final String SECURE = \"secure\";\n    public static final String HTTP_ONLY = \"httponly\";\n    public static final String SAME_SITE = \"samesite\";\n\n    public static Map<String, Object> toMap(Cookie cookie) {\n        Map<String, Object> map = new HashMap();\n        map.put(NAME, cookie.name());\n        map.put(VALUE, cookie.value());\n        map.put(WRAP, cookie.wrap());\n        map.put(DOMAIN, cookie.domain());\n        map.put(PATH, cookie.path());\n        map.put(MAX_AGE, cookie.maxAge());\n        map.put(SECURE, cookie.isSecure());\n        map.put(HTTP_ONLY, cookie.isHttpOnly());\n        if (cookie instanceof DefaultCookie) {\n            DefaultCookie dc = (DefaultCookie) cookie;\n            if (dc.sameSite() != null) {\n                map.put(SAME_SITE, dc.sameSite().name());\n            }\n        }\n        return map;\n    }\n\n    public static Cookie fromMap(Map<String, Object> map) {\n        String name = (String) map.get(NAME);\n        String value = (String) map.get(VALUE);\n        DefaultCookie cookie = new DefaultCookie(name, value);\n        Boolean wrap = (Boolean) map.get(WRAP);\n        if (wrap != null) {\n            cookie.setWrap(wrap);\n        }\n        String domain = (String) map.get(DOMAIN);\n        if (domain != null) {\n            cookie.setDomain(domain);\n        }\n        String path = (String) map.get(PATH);\n        if (path != null) {\n            cookie.setPath(path);\n        }\n        Object maxAge = map.get(MAX_AGE);\n        if (maxAge != null) {\n            cookie.setMaxAge(Long.parseLong(maxAge + \"\"));\n        }\n        Boolean secure = (Boolean) map.get(SECURE);\n        if (secure != null) {\n            cookie.setSecure(secure);\n        }\n        Boolean httpOnly = (Boolean) map.get(HTTP_ONLY);\n        if (httpOnly != null) {\n            cookie.setHttpOnly(httpOnly);\n        }\n        String sameSite = (String) map.get(SAME_SITE);\n        if (sameSite != null) {\n            cookie.setSameSite(CookieHeaderNames.SameSite.valueOf(sameSite));\n        }\n        return cookie;\n    }\n\n    public static Map<String, Map> normalize(Object mapOrList) {\n        Map<String, Map> cookies = new HashMap();\n        if (mapOrList instanceof Map) {\n            Map<String, Object> map = (Map) mapOrList;\n            map.forEach((k, v) -> {\n                if (v instanceof String) {\n                    Map<String, Object> cookie = new HashMap(2);\n                    cookie.put(\"name\", k);\n                    cookie.put(\"value\", v);\n                    cookies.put(k, cookie);\n                } else if (v instanceof Map) {\n                    Map<String, Object> cookie = (Map) v;\n                    cookie.put(\"name\", k);\n                    cookies.put(k, cookie);\n                }\n            });\n        } else if (mapOrList instanceof List) {\n            List<Map> list = (List) mapOrList;\n            list.forEach(map -> cookies.put((String) map.get(\"name\"), map));\n        }\n        return cookies;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/http/HttpLogModifier.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\n/**\n *\n * @author pthomas3\n */\npublic interface HttpLogModifier {\n    \n    \nboolean enableForUri(String uri); // TODO rename url ?\n    \n    String uri(String uri);\n    \n    String header(String header, String value);\n    \n    String request(String uri, String request);\n    \n    String response(String uri, String response);\n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/Config.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.driver.DockerTarget;\nimport com.intuit.karate.driver.Target;\nimport com.intuit.karate.http.Cookies;\nimport com.intuit.karate.http.HttpLogModifier;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.lang.reflect.Method;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * @author pthomas3\n */\npublic class Config {\n\n    private static final Logger logger = LoggerFactory.getLogger(Config.class);\n\n    public static final int DEFAULT_RETRY_INTERVAL = 3000;\n    public static final int DEFAULT_RETRY_COUNT = 3;\n    public static final int DEFAULT_TIMEOUT = 30000;\n    public static final int DEFAULT_HIGHLIGHT_DURATION = 3000;\n    \n    public static final String DRIVER = \"driver\";\n    public static final String ROBOT = \"robot\";\n    public static final String KAFKA = \"kafka\";\n    public static final String GRPC = \"grpc\";\n    public static final String WEBSOCKET = \"websocket\";\n\n    private String url;\n    private boolean sslEnabled = false;\n    private String sslAlgorithm = \"TLS\";\n    private String sslKeyStore;\n    private String sslKeyStorePassword;\n    private String sslKeyStoreType;\n    private String sslTrustStore;\n    private String sslTrustStorePassword;\n    private String sslTrustStoreType;\n    private boolean sslTrustAll = true;\n    private boolean followRedirects = true;\n    private int readTimeout = DEFAULT_TIMEOUT;\n    private int connectTimeout = DEFAULT_TIMEOUT;\n    private Charset charset = StandardCharsets.UTF_8;\n    private String proxyUri;\n    private String proxyUsername;\n    private String proxyPassword;\n    private List<String> nonProxyHosts;\n    private String localAddress;\n    private int responseDelay;\n    private boolean xmlNamespaceAware = false;\n    private boolean lowerCaseResponseHeaders = false;\n    private boolean corsEnabled = false;\n    private boolean logPrettyRequest;\n    private boolean logPrettyResponse;\n    private boolean printEnabled = true;\n    private boolean pauseIfNotPerf = false;\n    private boolean abortedStepsShouldPass = false;\n    private boolean matchEachEmptyAllowed = false;\n    private Target driverTarget;\n    private Map<String, Map<String, Object>> customOptions = new HashMap();\n    private HttpLogModifier logModifier;\n\n    private Variable afterScenario = Variable.NULL;\n    private Variable afterFeature = Variable.NULL;\n    private Variable headers = Variable.NULL;\n    private Variable cookies = Variable.NULL;\n    private Variable responseHeaders = Variable.NULL;\n    private List<Method> continueOnStepFailureMethods = new ArrayList<>();\n    private boolean continueAfterContinueOnStepFailure;\n    private boolean abortSuiteOnFailure;\n\n    // retry config\n    private int retryInterval = DEFAULT_RETRY_INTERVAL;\n    private int retryCount = DEFAULT_RETRY_COUNT;\n\n    // report config\n    private boolean showLog = true;\n    private boolean showAllSteps = true;\n\n    // call single cache config\n    private int callSingleCacheMinutes = 0;\n    private String callSingleCacheDir = FileUtils.getBuildDir();\n\n    // image comparison config\n    private Map<String, Object> imageComparisonOptions;\n\n    // ntlm authentication\n    private boolean ntlmEnabled = false;\n    private boolean httpRetryEnabled = false;\n    private String ntlmUsername;\n    private String ntlmPassword;\n    private String ntlmDomain;\n    private String ntlmWorkstation;\n\n    public Config() {\n        // zero arg constructor\n    }\n\n    private static <T> T get(Map<String, Object> map, String key, T defaultValue) {\n        Object o = map.get(key);\n        return o == null ? defaultValue : (T) o;\n    }\n\n    public boolean configure(String key, Variable value) { // TODO use enum\n        key = StringUtils.trimToEmpty(key);\n        switch (key) {\n            case \"url\":\n                url = value.getAsString();\n                return false;\n            case \"headers\":\n                headers = value;\n                return false;\n            case \"cookies\":\n                if (!value.isNull()) {\n                    value = new Variable(Cookies.normalize(value.getValue()));\n                }\n                cookies = value;\n                return false;\n            case \"responseHeaders\":\n                responseHeaders = value;\n                return false;\n            case \"responseDelay\":\n                responseDelay = value.isNull() ? 0 : value.getAsInt();\n                return false;\n            case \"xmlNamespaceAware\":\n                xmlNamespaceAware = value.isTrue();\n                return false;\n            case \"lowerCaseResponseHeaders\":\n                lowerCaseResponseHeaders = value.isTrue();\n                return false;\n            case \"cors\":\n                corsEnabled = value.isTrue();\n                return false;\n            case \"logPrettyResponse\":\n                logPrettyResponse = value.isTrue();\n                return false;\n            case \"logPrettyRequest\":\n                logPrettyRequest = value.isTrue();\n                return false;\n            case \"printEnabled\":\n                printEnabled = value.isTrue();\n                return false;\n            case \"afterScenario\":\n                afterScenario = value;\n                return false;\n            case \"afterFeature\":\n                afterFeature = value;\n                return false;\n            case \"report\":\n                if (value.isMap()) {\n                    Map<String, Object> map = value.getValue();\n                    showLog = get(map, \"showLog\", showLog);\n                    showAllSteps = get(map, \"showAllSteps\", showAllSteps);\n                } else if (value.isTrue()) {\n                    showLog = true;\n                    showAllSteps = true;\n                } else {\n                    showLog = false;\n                    showAllSteps = false;\n                }\n                return false;\n            case DRIVER:\n            case ROBOT:\n            case KAFKA:\n            case GRPC:\n            case WEBSOCKET:\n                customOptions.put(key, value.getValue());\n                return false;\n            case \"driverTarget\":\n                if (value.isMap()) {\n                    Map<String, Object> map = value.getValue();\n                    if (map.containsKey(\"docker\")) {\n                        // todo add the working dir here\n                        driverTarget = new DockerTarget(map);\n                    } else {\n                        throw new RuntimeException(\"bad driverTarget config, expected key 'docker': \" + map);\n                    }\n                } else {\n                    driverTarget = value.getValue();\n                }\n                return false;\n            case \"retry\":\n                if (value.isMap()) {\n                    Map<String, Object> map = value.getValue();\n                    retryInterval = get(map, \"interval\", retryInterval);\n                    retryCount = get(map, \"count\", retryCount);\n                }\n                return false;\n            case \"pauseIfNotPerf\":\n                pauseIfNotPerf = value.isTrue();\n                return false;\n            case \"abortedStepsShouldPass\":\n                abortedStepsShouldPass = value.isTrue();\n                return false;\n            case \"abortSuiteOnFailure\":\n                abortSuiteOnFailure = value.isTrue();\n                return false;\n            case \"callSingleCache\":\n                if (value.isMap()) {\n                    Map<String, Object> map = value.getValue();\n                    callSingleCacheMinutes = get(map, \"minutes\", callSingleCacheMinutes);\n                    callSingleCacheDir = get(map, \"dir\", callSingleCacheDir);\n                }\n                return false;\n            case \"logModifier\":\n                logModifier = value.getValue();\n                return false;\n            case \"imageComparison\":\n                imageComparisonOptions = value.getValue();\n                return false;\n            case \"matchEachEmptyAllowed\":\n                matchEachEmptyAllowed = value.getValue();\n                return false;\n            case \"continueOnStepFailure\":\n                continueOnStepFailureMethods.clear(); // clears previous configuration - in case someone is trying to chain these and forgets resetting the previous one\n                boolean enableContinueOnStepFailureFeature = false;\n                Boolean continueAfterIgnoredFailure = null;\n                List<String> stepKeywords = null;\n                if (value.isMap()) {\n                    Map<String, Object> map = value.getValue();\n                    stepKeywords = (List<String>) map.get(\"keywords\");\n                    continueAfterIgnoredFailure = (Boolean) map.get(\"continueAfter\");\n                    enableContinueOnStepFailureFeature = map.get(\"enabled\") != null && (Boolean) map.get(\"enabled\");\n                }\n                if (value.isTrue() || enableContinueOnStepFailureFeature) {\n                    continueOnStepFailureMethods.addAll(stepKeywords == null ? StepRuntime.METHOD_MATCH : StepRuntime.findMethodsByKeywords(stepKeywords));\n                } else {\n                    if (stepKeywords == null) {\n                        continueOnStepFailureMethods.clear();\n                    } else {\n                        continueOnStepFailureMethods.removeAll(StepRuntime.findMethodsByKeywords(stepKeywords));\n                    }\n                }\n                if (continueAfterIgnoredFailure != null) {\n                    continueAfterContinueOnStepFailure = continueAfterIgnoredFailure;\n                }\n                return false;\n            // here on the http client has to be re-constructed ================\n            // and we return true instead of false\n            case \"charset\":\n                charset = value.isNull() ? null : Charset.forName(value.getAsString());\n                return true;\n            case \"ssl\":\n                if (value.isString()) {\n                    sslEnabled = true;\n                    sslAlgorithm = value.getAsString();\n                } else if (value.isMap()) {\n                    sslEnabled = true;\n                    Map<String, Object> map = value.getValue();\n                    sslKeyStore = (String) map.get(\"keyStore\");\n                    sslKeyStorePassword = (String) map.get(\"keyStorePassword\");\n                    sslKeyStoreType = (String) map.get(\"keyStoreType\");\n                    sslTrustStore = (String) map.get(\"trustStore\");\n                    sslTrustStorePassword = (String) map.get(\"trustStorePassword\");\n                    sslTrustStoreType = (String) map.get(\"trustStoreType\");\n                    Boolean trustAll = (Boolean) map.get(\"trustAll\");\n                    if (trustAll != null) {\n                        sslTrustAll = trustAll;\n                    }\n                    sslAlgorithm = (String) map.get(\"algorithm\");\n                } else {\n                    sslEnabled = value.isTrue();\n                }\n                return true;\n            case \"followRedirects\":\n                followRedirects = value.isTrue();\n                return true;\n            case \"connectTimeout\":\n                connectTimeout = value.getAsInt();\n                return true;\n            case \"readTimeout\":\n                readTimeout = value.getAsInt();\n                return true;\n            case \"proxy\":\n                if (value.isNull()) {\n                    proxyUri = null;\n                } else if (value.isString()) {\n                    proxyUri = value.getAsString();\n                } else {\n                    Map<String, Object> map = value.getValue();\n                    proxyUri = (String) map.get(\"uri\");\n                    proxyUsername = (String) map.get(\"username\");\n                    proxyPassword = (String) map.get(\"password\");\n                    nonProxyHosts = (List) map.get(\"nonProxyHosts\");\n                }\n                return true;\n            case \"localAddress\":\n                localAddress = value.getAsString();\n                return true;\n            case \"ntlmAuth\":\n                if (value.isNull()) {\n                    ntlmEnabled = false;\n                } else {\n                    Map<String, Object> map = value.getValue();\n                    ntlmEnabled = true;\n                    ntlmUsername = (String) map.get(\"username\");\n                    ntlmPassword = (String) map.get(\"password\");\n                    ntlmDomain = (String) map.get(\"domain\");\n                    ntlmWorkstation = (String) map.get(\"workstation\");\n                }\n                return true;\n            default:\n                throw new RuntimeException(\"unexpected 'configure' key: '\" + key + \"'\");\n        }\n    }\n\n    public Config(Config parent) {\n        url = parent.url;\n        sslEnabled = parent.sslEnabled;\n        sslAlgorithm = parent.sslAlgorithm;\n        sslTrustStore = parent.sslTrustStore;\n        sslTrustStorePassword = parent.sslTrustStorePassword;\n        sslTrustStoreType = parent.sslTrustStoreType;\n        sslKeyStore = parent.sslKeyStore;\n        sslKeyStorePassword = parent.sslKeyStorePassword;\n        sslKeyStoreType = parent.sslKeyStoreType;\n        sslTrustAll = parent.sslTrustAll;\n        followRedirects = parent.followRedirects;\n        readTimeout = parent.readTimeout;\n        connectTimeout = parent.connectTimeout;\n        charset = parent.charset;\n        proxyUri = parent.proxyUri;\n        proxyUsername = parent.proxyUsername;\n        proxyPassword = parent.proxyPassword;\n        nonProxyHosts = parent.nonProxyHosts;\n        localAddress = parent.localAddress;\n        responseDelay = parent.responseDelay;\n        xmlNamespaceAware = parent.xmlNamespaceAware;\n        lowerCaseResponseHeaders = parent.lowerCaseResponseHeaders;\n        corsEnabled = parent.corsEnabled;\n        logPrettyRequest = parent.logPrettyRequest;\n        logPrettyResponse = parent.logPrettyResponse;\n        printEnabled = parent.printEnabled;\n        driverTarget = parent.driverTarget;\n        customOptions = parent.customOptions;\n        showLog = parent.showLog;\n        showAllSteps = parent.showAllSteps;\n        retryInterval = parent.retryInterval;\n        retryCount = parent.retryCount;\n        pauseIfNotPerf = parent.pauseIfNotPerf;\n        abortedStepsShouldPass = parent.abortedStepsShouldPass;\n        logModifier = parent.logModifier;\n        callSingleCacheMinutes = parent.callSingleCacheMinutes;\n...\n", "needle_token_start": 11947, "needle_token_end": 11955, "code_context_ntokens": 15996, "output": ["```java\npublic interface HttpLogModifier {\n    boolean enableForUri(String uri); // TODO rename url ?\n}\n```"]}
{"repo": "karatelabs/karate", "name": "getElementSelector", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/driver/microsoft/WinAppDriver.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a JSON string that specifies the method and value for locating a UI element in a Windows application based on different types of identifiers.\n2. **Input**: A string identifier that may start with specific prefixes to denote the type of locator (e.g., XPath, accessibility ID, ID, or name).\n3. **Output**: A JSON string that describes the locator type and the value to be used for finding the element.\n4. **Procedure**: The function checks the prefix of the input string to determine the locator type (XPath, accessibility ID, ID, or name). It then constructs a JSON object with two properties: \"using\" (the locator type) and \"value\" (the actual identifier, with any prefix removed if necessary). This JSON object is then converted to a string and returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: karate-core/src/main/java/com/intuit/karate/driver/chrome/Chrome.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n...\n// Path: karate-core/src/main/java/com/intuit/karate/driver/WebDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.Http;\nimport com.intuit.karate.Logger;\nimport com.intuit.karate.Json;\nimport com.intuit.karate.core.Variable;\nimport com.intuit.karate.http.ResourceType;\nimport com.intuit.karate.http.Response;\nimport com.intuit.karate.shell.Command;\n\nimport java.util.*;\nimport java.util.function.Supplier;\nimport java.util.stream.Collectors;\n\n/**\n * @author pthomas3\n */\npublic abstract class WebDriver implements Driver {\n\n    protected final DriverOptions options;\n    protected final Command command;\n    protected final Http http;\n    private final String sessionId;\n    private boolean terminated;\n    //private final String windowId;\n\n    protected boolean open = true;\n    protected Boolean specCompliant;\n\n    protected final Logger logger;\n\n    protected WebDriver(DriverOptions options) {\n        this.options = options;\n        this.logger = options.driverLogger;\n        command = options.startProcess();\n        http = options.getHttp();\n        Response response = http.path(\"session\").post(options.getWebDriverSessionPayload());\n        if (response.getStatus() != 200) {\n            String message = \"webdriver session create status \" + response.getStatus() + \", \" + response.getBodyAsString();\n            logger.error(message);\n            if (command != null) {\n                command.close(true);\n            }\n            throw new RuntimeException(message);\n        }\n        sessionId = response.json().getFirst(\"$..sessionId\");\n        logger.debug(\"init session id: {}\", sessionId);\n        http.url(http.urlBase + \"/session/\" + sessionId);\n        if (options.start) {\n            activate();\n        }\n    }\n\n    @Override\n    public Driver timeout(Integer millis) {\n        options.setTimeout(millis);\n        // this will \"reset\" to default if null was set above\n        http.configure(\"readTimeout\", options.getTimeout() + \"\");\n        return this;\n    }\n\n    @Override\n    public Driver timeout() {\n        return timeout(null);\n    }\n\n    public String getSessionId() {\n        return sessionId;\n    }\n\n    // can be used directly if you know what you are doing !\n    public Http getHttp() {\n        return http;\n    }\n\n    private String getSubmitHash() {\n        return getUrl() + elementId(\"html\");\n    }\n\n    protected <T> T retryIfEnabled(String locator, Supplier<T> action) {\n        if (options.isRetryEnabled()) {\n            waitFor(locator); // will throw exception if not found\n        }\n        if (options.highlight) {\n            highlight(locator, options.highlightDuration);\n        }\n        String before = options.getPreSubmitHash();\n        if (before != null) {\n            logger.trace(\"submit requested, will wait for page load after next action on : {}\", locator);\n            options.setPreSubmitHash(null); // clear the submit flag\n            T result = action.get();\n            Integer retryInterval = options.getRetryInterval();\n            options.setRetryInterval(500); // reduce retry interval for this special case\n            options.retry(() -> getSubmitHash(), hash -> !before.equals(hash), \"waiting for document to change\", false);\n            options.setRetryInterval(retryInterval); // restore\n            return result;\n        } else {\n            return action.get();\n        }\n    }\n\n    protected boolean isJavaScriptError(Response res) {\n        return res.getStatus() != 200\n                && !res.json().<String>get(\"value\").contains(\"unexpected alert open\");\n    }\n\n    protected boolean isLocatorError(Response res) {\n        return res.getStatus() != 200;\n    }\n\n    protected boolean isCookieError(Response res) {\n        return res.getStatus() != 200;\n    }\n\n    private Element evalLocator(String locator, String dotExpression) {\n        eval(prefixReturn(DriverOptions.selector(locator) + \".\" + dotExpression));\n        // if the js above did not throw an exception, the element exists\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    private Element evalFocus(String locator) {\n        eval(options.focusJs(locator));\n        // if the js above did not throw an exception, the element exists\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    protected Variable eval(String expression, List args) {\n        Json json = Json.object().set(\"script\", expression).set(\"args\", (args == null) ? Collections.EMPTY_LIST : args);\n        Response res = http.path(\"execute\", \"sync\").post(json);\n        if (isJavaScriptError(res)) {\n            logger.warn(\"javascript failed, will retry once: {}\", res.getBodyAsString());\n            options.sleep();\n            res = http.path(\"execute\", \"sync\").post(json);\n            if (isJavaScriptError(res)) {\n                String message = \"javascript failed twice: \" + res.getBodyAsString();\n                logger.error(message);\n                throw new RuntimeException(message);\n            }\n        }\n        return new Variable(res.json().get(\"value\"));\n    }\n\n    protected Variable eval(String expression) {\n        return eval(expression, null);\n    }\n\n    protected List<String> getElementKeys() {\n        // \"element-6066-11e4-a52e-4f735466cecf\" is the key to element in the W3C WebDriver standard\n        // \"ELEMENT\" is a deviation from the W3C standard\n        // explanation can be found here: https://github.com/karatelabs/karate/issues/1840#issuecomment-974688715\n        return Arrays.asList(\"element-6066-11e4-a52e-4f735466cecf\", \"ELEMENT\");\n    }\n\n    protected String getJsonForInput(String text) {\n        return Json.object().set(\"text\", text).toString();\n    }\n\n    protected String getJsonForLegacyInput(String text) {\n        return Json.of(\"{ value: [ '\" + text + \"' ] }\").toString();\n    }\n\n    protected String getJsonForHandle(String text) {\n        return Json.object().set(\"handle\", text).toString();\n    }\n\n    protected String getJsonForFrame(String text) {\n        return Json.object().set(\"id\", text).toString();\n    }\n\n    protected String selectorPayload(String locator) {\n        if (locator.startsWith(\"{\")) {\n            locator = DriverOptions.preProcessWildCard(locator);\n        }\n        Json json = Json.object();\n        if (locator.startsWith(\"/\")) {\n            json.set(\"using\", \"xpath\").set(\"value\", locator);\n        } else {\n            json.set(\"using\", \"css selector\").set(\"value\", locator);\n        }\n        return json.toString();\n    }\n\n    @Override\n    public String elementId(String locator) {\n        String json = selectorPayload(locator);\n        Response res = http.path(\"element\").postJson(json);\n        if (isLocatorError(res)) {\n            logger.warn(\"locator failed, will retry once: {}\", res.getBodyAsString());\n            options.sleep();\n            res = http.path(\"element\").postJson(json);\n            if (isLocatorError(res)) {\n                String message = \"locator failed twice: \" + res.getBodyAsString();\n                logger.error(message);\n                throw new RuntimeException(message);\n            }\n        }\n        List<String> resultElements = res.json().<List<String>>getAll(\"$..\", getElementKeys()).stream()\n                .flatMap(List::stream)\n                .collect(Collectors.toList());\n        String resultElement = resultElements != null && !resultElements.isEmpty() ? resultElements.get(0) : null;\n        if (resultElement == null) {\n            String message = \"locator failed to retrieve element returned by target driver: \" + res.getBodyAsString();\n            logger.error(message);\n            throw new RuntimeException(message);\n        }\n        return resultElement;\n    }\n\n    @Override\n    public List<String> elementIds(String locator) {\n        return http.path(\"elements\")\n                .postJson(selectorPayload(locator)).json()\n                .<List<String>>getAll(\"$..\", getElementKeys()).stream()\n                .flatMap(List::stream)\n                .collect(Collectors.toList());\n    }\n\n    @Override\n    public DriverOptions getOptions() {\n        return options;\n    }\n\n    @Override\n    public void setUrl(String url) {\n        Json json = Json.object().set(\"url\", url);\n        http.path(\"url\").post(json);\n    }\n\n    @Override\n    public Map<String, Object> getDimensions() {\n        return http.path(\"window\", \"rect\").get().json().get(\"value\");\n    }\n\n    @Override\n    public void setDimensions(Map<String, Object> map) {\n        http.path(\"window\", \"rect\").post(map);\n    }\n\n    @Override\n    public void refresh() {\n        http.path(\"refresh\").postJson(\"{}\");\n    }\n\n    @Override\n    public void reload() {\n        // not supported by webdriver\n        refresh();\n    }\n\n    @Override\n    public void back() {\n        http.path(\"back\").postJson(\"{}\");\n    }\n\n    @Override\n    public void forward() {\n        http.path(\"forward\").postJson(\"{}\");\n    }\n\n    @Override\n    public void maximize() {\n        http.path(\"window\", \"maximize\").postJson(\"{}\");\n    }\n\n    @Override\n    public void minimize() {\n        http.path(\"window\", \"minimize\").postJson(\"{}\");\n    }\n\n    @Override\n    public void fullscreen() {\n        http.path(\"window\", \"fullscreen\").postJson(\"{}\");\n    }\n\n    @Override\n    public Element focus(String locator) {\n        return retryIfEnabled(locator, () -> evalFocus(locator));\n    }\n\n    @Override\n    public Element clear(String locator) {\n        return retryIfEnabled(locator, () -> evalLocator(locator, \"value = ''\"));\n    }\n\n    @Override\n    public Element input(String locator, String value) {\n        return retryIfEnabled(locator, () -> {\n            String elementId;\n            if (locator.startsWith(\"(\")) {\n                evalFocus(locator);\n                List<String> elements = http.path(\"element\", \"active\").get()\n                        .json().<List<String>>getAll(\"$..\", getElementKeys()).stream()\n                        .flatMap(List::stream)\n                        .collect(Collectors.toList());;\n                elementId = elements != null && !elements.isEmpty() ? elements.get(0) : null;\n            } else {\n                elementId = elementId(locator);\n            }\n            Response response = http.path(\"element\", elementId, \"value\").postJson(isSpecCompliant() ? getJsonForInput(value) : getJsonForLegacyInput(value));\n            if (checkForSpecCompliance()) {\n                if (response.json().get(\"$.value\") != null) {\n                    String responseMessage = response.json().get(\"$.value.message\");\n                    if (responseMessage.contains(\"invalid argument: 'value' must be a list\")) {\n                        http.path(\"element\", elementId, \"value\").postJson(getJsonForLegacyInput(value));\n                        specCompliant = false;\n                    } else {\n                        specCompliant = true;\n                    }\n                } else {\n                    // did not complain that value should be a list so assume W3C WebDriver compliant moving forward\n                    specCompliant = true;\n                }\n            }\n            return DriverElement.locatorExists(this, locator);\n        });\n    }\n\n    @Override\n    public Element click(String locator) {\n        return retryIfEnabled(locator, () -> evalLocator(locator, \"click()\"));\n    }\n\n    @Override\n    public Driver submit() {\n        options.setPreSubmitHash(getSubmitHash());\n        return this;\n    }\n\n    @Override\n    public Element select(String locator, String text) {\n        return retryIfEnabled(locator, () -> {\n            eval(options.optionSelector(locator, text));\n            // if the js above did not throw an exception, the element exists\n            return DriverElement.locatorExists(this, locator);\n        });\n    }\n\n    @Override\n    public Element select(String locator, int index) {\n        return retryIfEnabled(locator, () -> {\n            eval(options.optionSelector(locator, index));\n            // if the js above did not throw an exception, the element exists\n            return DriverElement.locatorExists(this, locator);\n        });\n    }\n\n    @Override\n    public void actions(List<Map<String, Object>> actions) {\n        http.path(\"actions\").post(Collections.singletonMap(\"actions\", actions));\n    }\n\n    @Override\n    public void close() {\n        http.path(\"window\").delete();\n        open = false;\n    }\n\n    @Override\n    public boolean isTerminated() {\n        return terminated;\n    }\n\n    public boolean isSpecCompliant() {\n        return specCompliant == null || specCompliant;\n    }\n\n    public boolean checkForSpecCompliance() {\n        return specCompliant == null;\n    }\n\n    @Override\n    public void quit() {\n        if (terminated) {\n            return;\n        }\n        terminated = true;\n        if (open) {\n            close();\n        }\n        // delete session\n        try {\n            http.delete();\n        } catch (Exception e) {\n            logger.warn(\"session delete failed: {}\", e.getMessage());\n        }\n        if (command != null) {\n            command.close(true);\n        }\n    }\n\n    @Override\n    public String getUrl() {\n        return http.path(\"url\").get().json().get(\"value\");\n    }\n\n    private String evalReturn(String locator, String dotExpression) {\n        return eval(\"return \" + DriverOptions.selector(locator) + \".\" + dotExpression).getAsString();\n    }\n\n    @Override\n    public String html(String locator) {\n        return retryIfEnabled(locator, () -> evalReturn(locator, \"outerHTML\"));\n    }\n\n    @Override\n    public String text(String locator) {\n        return retryIfEnabled(locator, () -> evalReturn(locator, \"textContent\"));\n    }\n\n    @Override\n    public String value(String locator) {\n        return retryIfEnabled(locator, () -> evalReturn(locator, \"value\"));\n    }\n\n    @Override\n    public Element value(String locator, String value) {\n        return retryIfEnabled(locator, () -> evalLocator(locator, \"value = '\" + value + \"'\"));\n    }\n\n    @Override\n    public String attribute(String locator, String name) {\n        return retryIfEnabled(locator, () -> evalReturn(locator, \"getAttribute('\" + name + \"')\"));\n    }\n\n    @Override\n    public String property(String locator, String name) {\n        return retryIfEnabled(locator, () -> evalReturn(locator, name));\n    }\n\n    @Override\n    public Map<String, Object> position(String locator) {\n        return position(locator, false);\n    }\n\n    @Override\n    public Map<String, Object> position(String locator, boolean relative) {\n        return retryIfEnabled(locator, ()\n                -> eval(\"return \" + DriverOptions.selector(locator) + \".getBoundingClientRect()\").getValue());\n    }\n\n    @Override\n    public boolean enabled(String locator) {\n        return retryIfEnabled(locator, ()\n                -> eval(\"return !\" + DriverOptions.selector(locator) + \".disabled\").isTrue());\n    }\n\n    private String prefixReturn(String expression) {\n        return expression.startsWith(\"return \") ? expression : \"return \" + expression;\n    }\n\n    @Override\n    public boolean waitUntil(String expression) {\n        return options.retry(() -> {\n            try {\n                return eval(prefixReturn(expression)).isTrue();\n            } catch (Exception e) {\n                logger.warn(\"waitUntil evaluate failed: {}\", e.getMessage());\n                return false;\n            }\n        }, b -> b, \"waitUntil (js)\", true);\n    }\n\n    @Override\n    public Object script(String expression) {\n        expression = prefixReturn(expression);\n        return eval(expression).getValue();\n    }\n\n    @Override\n    public String getTitle() {\n        return http.path(\"title\").get().json().get(\"value\");\n    }\n\n    @Override\n    public List<Map> getCookies() {\n        return http.path(\"cookie\").get().json().get(\"value\");\n    }\n\n    @Override\n    public Map<String, Object> cookie(String name) {\n        return http.path(\"cookie\", name).get().json().get(\"value\");\n    }\n\n    @Override\n    public void cookie(Map<String, Object> cookie) {\n        Response res = http.path(\"cookie\").post(Collections.singletonMap(\"cookie\", cookie));\n        if (isCookieError(res)) {\n            throw new RuntimeException(\"set-cookie failed: \" + res.getBodyAsString());\n        }\n    }\n\n    @Override\n    public void deleteCookie(String name) {\n        http.path(\"cookie\", name).delete();\n    }\n\n    @Override\n    public void clearCookies() {\n        http.path(\"cookie\").delete();\n    }\n\n    @Override\n    public void dialog(boolean accept) {\n        dialog(accept, null);\n    }\n\n    @Override\n    public String getDialogText() {\n        return http.path(\"alert\", \"text\").get().json().get(\"value\");\n    }\n\n    @Override\n    public void dialog(boolean accept, String text) {\n        if (text == null) {\n            http.path(\"alert\", accept ? \"accept\" : \"dismiss\").postJson(\"{}\");\n        } else {\n            http.path(\"alert\", \"text\").post(Collections.singletonMap(\"text\", text));\n            http.path(\"alert\", \"accept\").postJson(\"{}\");\n        }\n    }\n\n    @Override\n    public byte[] screenshot(boolean embed) {\n        return screenshot(null, embed);\n    }\n\n    @Override\n    public byte[] screenshot(String locator, boolean embed) {\n        String temp;\n        if (locator == null) {\n            temp = http.path(\"screenshot\").get().json().get(\"value\");\n        } else {\n            temp = retryIfEnabled(locator, () -> {\n                String id = elementId(locator);\n                return http.path(\"element\", id, \"screenshot\").get().json().get(\"value\");\n            });\n        }\n        byte[] bytes = getDecoder().decode(temp);\n        if (embed) {\n            getRuntime().embed(bytes, ResourceType.PNG);\n        }\n        return bytes;\n    }\n\n    @Override\n    public List<String> getPages() {\n        return http.path(\"window\", \"handles\").get().json().get(\"value\");\n    }\n\n    @Override\n    public void switchPage(String titleOrUrl) {\n        if (titleOrUrl == null) {\n            return;\n        }\n        options.retry(() -> {\n            for (String handle : getPages()) {\n                http.path(\"window\").postJson(getJsonForHandle(handle));\n                String title = getTitle();\n                if (title != null && title.contains(titleOrUrl)) {\n                    return true;\n                }\n                String url = getUrl();\n                if (url != null && url.contains(titleOrUrl)) {\n                    return true;\n                }\n            }\n            return false;\n        }, returned -> returned, \"waiting to switch to tab: \" + titleOrUrl, true);\n    }\n\n    @Override\n    public void switchPage(int index) {\n        if (index == -1) {\n            return;\n        }\n        String json = Json.object().set(\"id\", index).toString();\n        http.path(\"window\").postJson(json);\n    }\n\n    @Override\n    public void switchFrame(int index) {\n        if (index == -1) {\n            http.path(\"frame\", \"parent\").postJson(\"{}\");\n            return;\n        }\n        String json = Json.object().set(\"id\", index).toString();\n        http.path(\"frame\").postJson(json);\n    }\n\n    @Override\n    public void switchFrame(String locator) {\n        if (locator == null) { // reset to parent frame\n            http.path(\"frame\", \"parent\").postJson(\"{}\");\n            return;\n        }\n        retryIfEnabled(locator, () -> {\n            String frameId = elementId(locator);\n            if (frameId == null) {\n                return null;\n            }\n            List<String> ids = elementIds(\"iframe,frame\");\n            for (int i = 0; i < ids.size(); i++) {\n                String id = ids.get(i);\n                if (frameId.equals(id)) {\n                    switchFrame(i);\n                    break;\n                }\n            }\n            return null;\n        });\n    }\n\n    protected Base64.Decoder getDecoder() {\n        return Base64.getDecoder();\n    }\n\n    @Override\n    public byte[] pdf(Map<String, Object> printOptions) {\n        String temp = http.path(\"print\").post(printOptions).json().get(\"value\");\n        return Base64.getDecoder().decode(temp);\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/chrome/ChromeWebDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.chrome;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.driver.DriverOptions;\nimport com.intuit.karate.driver.WebDriver;\nimport com.intuit.karate.http.Response;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class ChromeWebDriver extends WebDriver {\n\n    public static final String DRIVER_TYPE = \"chromedriver\";\n\n    public ChromeWebDriver(DriverOptions options) {\n        super(options);\n    }\n\n    public static ChromeWebDriver start(Map<String, Object> map, ScenarioRuntime sr) {\n        DriverOptions options = new DriverOptions(map, sr, 9515, \"chromedriver\");\n        options.arg(\"--port=\" + options.port);\n        if (options.userDataDir != null) {\n            options.arg(\"--user-data-dir=\" + options.userDataDir);\n        }\n        return new ChromeWebDriver(options);\n    }\n\n    @Override\n    public void activate() {\n        if (!options.headless) {\n            try {\n                switch (FileUtils.getOsType()) {\n                    case MACOSX:\n                        Runtime.getRuntime().exec(new String[]{\"osascript\", \"-e\", \"tell app \\\"Chrome\\\" to activate\"});\n                        break;\n                    default:\n\n                }\n            } catch (Exception e) {\n                logger.warn(\"native window switch failed: {}\", e.getMessage());\n            }\n        }\n    }\n\n    @Override\n    protected boolean isJavaScriptError(Response res) {\n        Object value = res.json().get(\"value\");\n        return value != null && value.toString().contains(\"javascript error\");\n    }\n\n    @Override\n    protected boolean isLocatorError(Response res) {\n        Object value = res.json().get(\"value\");\n        return value.toString().contains(\"no such element\");\n    }\n\n    @Override\n    protected boolean isCookieError(Response res) {\n        Object value = res.json().get(\"value\");\n        return value != null && value.toString().contains(\"unable to set cookie\");\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/firefox/GeckoWebDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.firefox;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.Json;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.driver.DriverOptions;\nimport com.intuit.karate.driver.WebDriver;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class GeckoWebDriver extends WebDriver {\n\n    public static final String DRIVER_TYPE = \"geckodriver\";\n\n    public GeckoWebDriver(DriverOptions options) {\n        super(options);\n    }\n\n    public static GeckoWebDriver start(Map<String, Object> map, ScenarioRuntime sr) {\n        DriverOptions options = new DriverOptions(map, sr, 4444, \"geckodriver\");\n        options.arg(\"--port=\" + options.port);\n        return new GeckoWebDriver(options);\n    }\n    \n    @Override\n    protected String getJsonForFrame(String text) {\n        return Json.object().set(\"frameId\", text).toString();\n    }    \n\n    @Override\n    public void activate() {\n        if (!options.headless) {\n            try {\n                switch (FileUtils.getOsType()) {\n                    case MACOSX:\n                        Runtime.getRuntime().exec(new String[]{\"osascript\", \"-e\", \"tell app \\\"Firefox\\\" to activate\"});\n                        break;\n                    default:\n\n                }\n            } catch (Exception e) {\n                logger.warn(\"native window switch failed: {}\", e.getMessage());\n            }\n        }\n    }\n\n    @Override\n    public void quit() {\n        // geckodriver already closes all windows on delete session\n        open = false;\n        super.quit();\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/microsoft/EdgeChromium.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.microsoft;\n\nimport com.intuit.karate.FileUtils;\nimport com.intuit.karate.Http;\nimport com.intuit.karate.core.FeatureRuntime;\nimport com.intuit.karate.core.ScenarioEngine;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.driver.DevToolsDriver;\nimport com.intuit.karate.driver.DriverOptions;\nimport com.intuit.karate.http.HttpClientFactory;\nimport com.intuit.karate.http.Response;\nimport com.intuit.karate.shell.Command;\n\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n *\n * @author sixdouglas\n */\npublic class EdgeChromium extends DevToolsDriver {\n    \n    public static final String DRIVER_TYPE = \"msedge\";\n\n    public static final String DEFAULT_PATH_MAC = \"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge\";\n    public static final String DEFAULT_PATH_WIN32 = \"C:\\\\Program Files (x86)\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe\";\n    public static final String DEFAULT_PATH_WIN64 = \"C:\\\\Program Files\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe\";\n    public static final String DEFAULT_PATH_WIN = Files.isRegularFile(Paths.get(DEFAULT_PATH_WIN64)) && Files.isReadable(Paths.get(DEFAULT_PATH_WIN64)) ? DEFAULT_PATH_WIN64 : DEFAULT_PATH_WIN32;\n    public static final String DEFAULT_PATH_LINUX = \"/dev/null\";\n\n    public EdgeChromium(DriverOptions options, Command command, String webSocketUrl) {\n        super(options, command, webSocketUrl);\n    }\n\n    public static EdgeChromium start(Map<String, Object> map, ScenarioRuntime sr) {\n        if (!FileUtils.isOsWindows() && !FileUtils.isOsMacOsX()) {\n            throw new UnsupportedOperationException(\"edge browser is not yet available on linux!\");\n        }\n        DriverOptions options = new DriverOptions(map, sr, 9222,\n                FileUtils.isOsWindows() ? DEFAULT_PATH_WIN : FileUtils.isOsMacOsX() ? DEFAULT_PATH_MAC : DEFAULT_PATH_LINUX);\n        options.arg(\"--remote-debugging-port=\" + options.port);\n        options.arg(\"--no-first-run\");\n        if (options.userDataDir != null) {\n            options.arg(\"--user-data-dir=\" + options.userDataDir);\n        }\n        options.arg(\"--disable-popup-blocking\");\n        if (options.headless) {\n            options.arg(\"--headless\");\n        }\n        Command command = options.startProcess();\n        Http http = options.getHttp();\n        Command.waitForHttp(http.urlBase);\n        Response res = http.path(\"json\").get();\n        if (res.json().asList().isEmpty()) {\n            if (command != null) {\n                command.close(true);\n            }\n            throw new RuntimeException(\"edge server returned empty list from \" + http.urlBase);\n        }\n        String webSocketUrl = null;\n        List<Map<String, Object>> targets = res.json().asList();\n        for (Map<String, Object> target : targets) {\n            String targetUrl = (String) target.get(\"url\");\n            if (targetUrl == null || targetUrl.startsWith(\"edge-\")) {\n                continue;\n            }\n            String targetType = (String) target.get(\"type\");\n            if (!\"page\".equals(targetType)) {\n                continue;\n            }\n            webSocketUrl = (String) target.get(\"webSocketDebuggerUrl\");\n            if (options.attach == null) { // take the first                \n                break;\n            }\n            if (targetUrl.contains(options.attach)) {\n                break;\n            }\n        }\n        if (webSocketUrl == null) {\n            throw new RuntimeException(\"failed to attach to Edge-Chromium debug server\");\n        }\n        EdgeChromium edgeChromium = new EdgeChromium(options, command, webSocketUrl);\n        edgeChromium.activate();\n        edgeChromium.enablePageEvents();\n        edgeChromium.enableRuntimeEvents();\n        if (!options.headless) {\n            edgeChromium.initWindowIdAndState();\n        }\n        return edgeChromium;\n    }\n\n    public static EdgeChromium start(String chromeExecutablePath, boolean headless) {\n        Map<String, Object> options = new HashMap();\n        options.put(\"executable\", chromeExecutablePath);\n        options.put(\"headless\", headless);\n        return EdgeChromium.start(options);\n    }\n\n    public static EdgeChromium start(Map<String, Object> options) {\n        if (options == null) {\n            options = new HashMap();\n        }\n        options.putIfAbsent(\"type\", DRIVER_TYPE);\n        ScenarioRuntime runtime = FeatureRuntime.forTempUse(HttpClientFactory.DEFAULT).scenarios.next();\n        ScenarioEngine.set(runtime.engine);        \n        return EdgeChromium.start(options, runtime);\n    }\n\n    public static EdgeChromium start() {\n        return start(null);\n    }\n\n    public static EdgeChromium startHeadless() {\n        return start(Collections.singletonMap(\"headless\", true));\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/microsoft/IeWebDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.microsoft;\n\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.driver.DriverOptions;\nimport com.intuit.karate.driver.WebDriver;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class IeWebDriver extends WebDriver {\n\n    public static final String DRIVER_TYPE = \"iedriver\";\n    \n    public IeWebDriver(DriverOptions options) {\n        super(options);\n    }\n\n    public static IeWebDriver start(Map<String, Object> map, ScenarioRuntime sr) {\n        DriverOptions options = new DriverOptions(map, sr, 5555, \"IEDriverServer\");\n        options.arg(\"port=\" + options.port);\n        return new IeWebDriver(options);\n    }\n    \n    @Override\n    public void activate() {\n        logger.warn(\"activate not implemented for iewebdriver\");\n    }    \n    \n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/microsoft/MsEdgeDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.microsoft;\n\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.driver.DriverOptions;\nimport com.intuit.karate.driver.WebDriver;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class MsEdgeDriver extends WebDriver {\n\n    public static final String DRIVER_TYPE = \"msedgedriver\";\n\n    public MsEdgeDriver(DriverOptions options) {\n        super(options);\n    }\n\n    public static MsEdgeDriver start(Map<String, Object> map, ScenarioRuntime sr) {\n        DriverOptions options = new DriverOptions(map, sr, 9515, \"msedgedriver\");\n        options.arg(\"--port=\" + options.port);\n        return new MsEdgeDriver(options);\n    }\n\n    @Override\n    public void activate() {\n        logger.warn(\"activate not implemented for mswebdriver\");\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/microsoft/MsWebDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.microsoft;\n\nimport com.intuit.karate.Json;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.driver.DriverOptions;\nimport com.intuit.karate.driver.WebDriver;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class MsWebDriver extends WebDriver {\n\n    public static final String DRIVER_TYPE = \"mswebdriver\";\n\n    public MsWebDriver(DriverOptions options) {\n        super(options);\n    }\n\n    public static MsWebDriver start(Map<String, Object> map, ScenarioRuntime sr) {\n        DriverOptions options = new DriverOptions(map, sr, 17556, \"MicrosoftWebDriver\");\n        options.arg(\"--port=\" + options.port);\n        return new MsWebDriver(options);\n    }\n\n    @Override\n    protected String getJsonForInput(String text) {\n        return Json.object().set(\"keysToSend[0]\", text).toString();\n    }\n\n    @Override\n    public void activate() {\n        logger.warn(\"activate not implemented for mswebdriver\");\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/Element.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport java.util.List;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic interface Element {\n\n    String getLocator(); // getter\n\n    boolean isPresent(); // getter\n\n    boolean isEnabled(); // getter\n    \n    Map<String, Object> getPosition(); // getter\n    \n    byte[] screenshot();\n\n    Element highlight();\n\n    Element focus();\n\n    Element clear();\n\n    Element click();\n\n    Element submit();\n    \n    Element scroll();\n\n    Mouse mouse();\n\n    Element input(String value);\n\n    Element input(String[] values);\n\n    Element input(String[] values, int delay);\n\n    Element select(String text);\n\n    Element select(int index);\n\n    Element switchFrame();\n\n    Element delay(int millis);\n\n    Element retry();\n\n    Element retry(int count);\n\n    Element retry(Integer count, Integer interval);\n\n    Element waitFor();\n\n    Element waitUntil(String expression);\n\n    Element waitForText(String text);\n\n    Object script(String expression);\n    \n    Object scriptAll(String locator, String expression);\n    \n    Element optional(String locator);\n    \n    boolean exists(String locator);\n    \n    Element locate(String locator);\n    \n    List<Element> locateAll(String locator);\n\n    String getHtml(); // getter\n\n    void setHtml(String html); // setter\n\n    String getText(); // getter\n\n    void setText(String text); // setter    \n\n    String getValue(); // getter\n\n    void setValue(String value); // setter\n    \n    String attribute(String name);\n    \n    String property(String name);\n    \n    Element getParent(); // getter\n    \n    Element getFirstChild(); // getter\n            \n    Element getLastChild(); // getter\n    \n    Element getPreviousSibling(); // getter\n    \n    Element getNextSibling(); // getter\n    \n    List<Element> getChildren();\n    \n    Finder rightOf();\n    \n    Finder leftOf();\n    \n    Finder above();\n    \n    Finder below();\n    \n    Finder near();\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/microsoft/WinAppDriver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.microsoft;\n\nimport com.intuit.karate.Json;\nimport com.intuit.karate.core.ScenarioRuntime;\nimport com.intuit.karate.driver.DriverElement;\nimport com.intuit.karate.driver.DriverOptions;\nimport com.intuit.karate.driver.Element;\nimport com.intuit.karate.driver.WebDriver;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class WinAppDriver extends WebDriver {\n\n    public static final String DRIVER_TYPE = \"winappdriver\";\n\n    public WinAppDriver(DriverOptions options) {\n        super(options);\n    }\n\n    public static WinAppDriver start(Map<String, Object> map, ScenarioRuntime sr) {\n        DriverOptions options = new DriverOptions(map, sr, 4727, \n                \"C:/Program Files (x86)/Windows Application Driver/WinAppDriver\");\n        options.arg(options.port + \"\");\n        return new WinAppDriver(options);\n    }\n\n    @Override\n    public void activate() {\n        // TODO\n    }\n    \n    \nprivate String getElementSelector(String id) {\n        Json json = Json.object();\n        if (id.startsWith(\"/\")) {\n            json.set(\"using\", \"xpath\").set(\"value\", id);\n        } else if (id.startsWith(\"@\")){\n            json.set(\"using\", \"accessibility id\").set(\"value\", id.substring(1));\n        } else if (id.startsWith(\"#\")){\n            json.set(\"using\", \"id\").set(\"value\", id.substring(1));\n        } else {\n            json.set(\"using\", \"name\").set(\"value\", id);\n        }\n        return json.toString();\n    }\n\n    @Override\n    public String elementId(String id) {\n        String body = getElementSelector(id);\n        return http.path(\"element\").postJson(body).json().getFirst(\"$..ELEMENT\");\n    }\n\n    @Override\n    public Element click(String locator) {\n        String id = elementId(locator);\n        http.path(\"element\", id, \"click\").postJson(\"{}\");\n        return DriverElement.locatorExists(this, locator);\n    }\n\n    @Override\n    public String text(String locator) {\n        String id = elementId(locator);\n        return http.path(\"element\", id, \"text\").get().json().get(\"value\");\n    }\n\n    @Override\n    protected String getJsonForInput(String text) {\n        return Json.object().set(\"value[0]\", text).toString();\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/Input.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\n\n/**\n *\n * @author pthomas3\n */\npublic class Input {\n\n    protected boolean control;\n    protected boolean alt;\n    protected boolean shift;\n    protected boolean meta;\n\n    protected boolean release;\n\n    private int pos = 0;\n\n    public final char[] chars;\n\n    public Input(String chars) {\n        this.chars = chars.toCharArray();\n    }\n\n    public boolean hasNext() {\n        return pos < chars.length;\n    }\n\n    public List<Integer> getKeyCodesToRelease() {\n        if (control || alt || shift || meta) {\n            List<Integer> list = new ArrayList();\n            if (shift) {\n                list.add(Keys.CODE_SHIFT);\n            }\n            if (control) {\n                list.add(Keys.CODE_CONTROL);\n            }\n            if (alt) {\n                list.add(Keys.CODE_ALT);\n            }\n            if (meta) {\n                list.add(Keys.CODE_META);\n            }\n            return list;\n        } else {\n            return Collections.emptyList();\n        }\n    }\n\n    private void updateModifiers(char c) {\n        switch (c) {\n            case Keys.CONTROL:\n                release = control;\n                control = !control;\n                break;\n            case Keys.ALT:\n                release = alt;\n                alt = !alt;\n                break;\n            case Keys.SHIFT:\n                release = shift;\n                shift = !shift;\n                break;\n            case Keys.META:\n                release = meta;\n                meta = !meta;\n                break;\n            default:\n                break;\n        }\n    }\n\n    public char next() {\n        char c = chars[pos++];\n        updateModifiers(c);\n        return c;\n    }\n\n    public int getModifierFlags() {\n        int modifier = 0;\n        if (control) {\n            modifier += 2;\n        }\n        if (alt) {\n            modifier += 1;\n        }\n        if (shift) {\n            modifier += 8;\n        }\n        if (meta) {\n            modifier += 4;\n        }\n        return modifier;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/playwright/PlaywrightWait.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver.playwright;\n\nimport com.intuit.karate.Logger;\nimport com.intuit.karate.driver.DriverOptions;\nimport java.util.function.Predicate;\n\n/**\n *\n * @author pthomas3\n */\npublic class PlaywrightWait {\n\n    private final DriverOptions options;\n    private final PlaywrightDriver driver;\n\n    private PlaywrightMessage lastSent;\n    private Predicate<PlaywrightMessage> condition;\n    private PlaywrightMessage lastReceived;\n\n    private final Predicate<PlaywrightMessage> DEFAULT = m -> lastSent.getId().equals(m.getId());\n    \n    public static final Predicate<PlaywrightMessage> DOM_CONTENT_LOADED = m -> m.methodIs(\"domcontentloaded\");\n\n    public PlaywrightWait(PlaywrightDriver driver, DriverOptions options) {\n        this.driver = driver;\n        this.options = options;\n        logger = options.driverLogger;        \n    }\n\n    // mutable when driver logger is swapped\n    private Logger logger;\n\n    public void setLogger(Logger logger) {\n        this.logger = logger;\n    }\n    \n    public PlaywrightMessage send(PlaywrightMessage pwm, Predicate<PlaywrightMessage> condition) {\n        lastReceived = null;\n        lastSent = pwm;\n        this.condition = condition == null ? DEFAULT : condition;        \n        long timeout = pwm.getTimeout() == null ? options.getTimeout() : pwm.getTimeout();\n        synchronized (this) {\n            logger.trace(\">> wait: {}\", pwm);\n            try {\n                driver.send(pwm);\n                wait(timeout);\n            } catch (InterruptedException e) {\n                logger.error(\"interrupted: {} wait: {}\", e.getMessage(), pwm);\n            }\n        }\n        if (lastReceived != null) {\n            logger.trace(\"<< notified: {}\", pwm);\n        } else {\n            logger.error(\"<< timed out after milliseconds: {} - {}\", timeout, pwm);\n            return null;\n        }\n        return lastReceived;\n    }\n\n    public void receive(PlaywrightMessage pwm) {\n        if (condition == null) {\n            return;\n        }\n        synchronized (this) {\n            if (condition.test(pwm)) {   \n                if (pwm.isError()) {\n                    logger.warn(\"playwright error: {}\", pwm);\n                } else {\n                    logger.trace(\"<< notify: {}\", pwm);\n                }\n                lastReceived = pwm;\n                notify();\n            } else {\n                logger.trace(\"<< ignore: {}\", pwm);\n            }\n        }\n    }    \n\n}\n\n...\n", "needle_token_start": 13422, "needle_token_end": 13574, "code_context_ntokens": 15868, "output": ["```java\nprivate String getElementSelector(String id) {\n    Json json = Json.object();\n    if (id.startsWith(\"/\")) {\n        json.set(\"using\", \"xpath\").set(\"value\", id);\n    } else if (id.startsWith(\"@\")){\n        json.set(\"using\", \"accessibility id\").set(\"value\", id.substring(1));\n    } else if (id.startsWith(\"#\")){\n        json.set(\"using\", \"id\").set(\"value\", id.substring(1));\n    } else {\n        json.set(\"using\", \"name\").set(\"value\", id);\n    }\n    return json.toString();\n}\n```"]}
{"repo": "karatelabs/karate", "name": "addTemplateResolver", "language": "java", "path": "karate-core/src/main/java/com/intuit/karate/template/KarateTemplateEngine.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To enhance the template processing capabilities by incorporating additional template resolution strategies into the engine.\n2. **Input**: An instance of a template resolver interface, which defines how templates should be located and resolved.\n3. **Output**: There is no direct output; the method modifies the internal state of the template engine by registering the new resolver.\n4. **Procedure**: The method takes the provided template resolver and adds it to the template engine's list of resolvers. This allows the engine to use the new resolver along with any previously added resolvers to locate and process templates based on various criteria specified within each resolver.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: karate-core/src/main/java/com/intuit/karate/core/TagResults.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n...\n// Path: karate-core/src/main/java/com/intuit/karate/core/TimelineResults.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport com.intuit.karate.JsonUtils;\nimport com.intuit.karate.StringUtils;\nimport java.text.DateFormat;\nimport java.text.SimpleDateFormat;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class TimelineResults {\n\n    private final Map<String, Integer> groupsMap = new LinkedHashMap();\n    private final List<Map> items = new ArrayList();\n    private final DateFormat dateFormat = new SimpleDateFormat(\"HH:mm:ss.SSS\");\n    private int id;\n\n    public void addFeatureResult(FeatureResult fr) {\n        fr.getScenarioResults().stream().forEach(sr -> {\n            String threadName = sr.getExecutorName();\n            Integer groupId = groupsMap.get(threadName);\n            if (groupId == null) {\n                groupId = groupsMap.size() + 1;\n                groupsMap.put(threadName, groupId);\n            }\n            Map<String, Object> item = new LinkedHashMap(10);\n            items.add(item);\n            item.put(\"id\", ++id);\n            item.put(\"group\", groupId);\n            Scenario s = sr.getScenario();\n            String featureName = s.getFeature().getResource().getFileNameWithoutExtension();\n            String content = featureName + s.getRefId();\n            item.put(\"content\", content);\n            long startTime = sr.getStartTime();\n            item.put(\"start\", startTime);\n            long endTime = sr.getEndTime() - 1; // avoid overlap when rendering\n            item.put(\"end\", endTime);\n            String startTimeString = dateFormat.format(new Date(startTime));\n            String endTimeString = dateFormat.format(new Date(endTime));\n            content = content + \" \" + startTimeString + \"-\" + endTimeString;\n            String scenarioTitle = StringUtils.trimToEmpty(s.getName());\n            if (!scenarioTitle.isEmpty()) {\n                content = content + \" \" + scenarioTitle;\n            }\n            item.put(\"title\", content);\n            if (sr.isFailed()) {\n                item.put(\"className\", \"failed\");\n            }\n        });\n    }\n\n    public Map<String, Object> toKarateJson() {\n        List<Map> groups = new ArrayList(groupsMap.size());\n        groupsMap.forEach((k, v) -> {\n            Map<String, Object> group = new HashMap(2);\n            groups.add(group);\n            group.put(\"id\", v);\n            group.put(\"content\", k);\n        });\n        StringBuilder sb = new StringBuilder();\n        sb.append('\\n');\n        sb.append(\"var groups = \").append(JsonUtils.toJson(groups)).append(';');\n        sb.append('\\n');\n        sb.append(\"var items = \").append(JsonUtils.toJson(items)).append(';');\n        sb.append('\\n');\n        return Collections.singletonMap(\"data\", sb.toString());\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/Results.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate;\n\nimport com.intuit.karate.core.FeatureResult;\nimport com.intuit.karate.core.ScenarioResult;\nimport com.intuit.karate.core.TagResults;\nimport com.intuit.karate.core.TimelineResults;\nimport com.intuit.karate.report.ReportUtils;\nimport java.io.File;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.stream.Stream;\n\n/**\n *\n * @author pthomas3\n */\npublic class Results {\n\n    private final Suite suite;\n    private final int featuresPassed;\n    private final int featuresFailed;\n    private final int featuresSkipped;\n    private final int scenariosPassed;\n    private final int scenariosFailed;\n    private final double timeTakenMillis;\n    private final long endTime;\n    private final List<String> errors = new ArrayList();\n    private final List<Map<String, Object>> featureSummary = new ArrayList();\n\n    public static Results of(Suite suite) {\n        return new Results(suite);\n    }\n\n    private Results(Suite suite) {\n        this.suite = suite;\n        // endTime may not be set for junit\n        endTime = suite.endTime == 0 ? System.currentTimeMillis() : suite.endTime;\n        featuresSkipped = suite.skippedCount;\n        AtomicInteger fp = new AtomicInteger();\n        AtomicInteger ff = new AtomicInteger();\n        AtomicInteger sp = new AtomicInteger();\n        AtomicInteger sf = new AtomicInteger();\n        AtomicInteger time = new AtomicInteger();\n        TimelineResults timeline = new TimelineResults();\n        TagResults tags = new TagResults();\n        suite.getFeatureResults().forEach(fr -> {\n            if (!fr.isEmpty()) {\n                timeline.addFeatureResult(fr);\n                tags.addFeatureResult(fr);\n                if (fr.isFailed()) {\n                    ff.incrementAndGet();\n                } else {\n                    fp.incrementAndGet();\n                }\n                Long duration = Math.round(fr.getDurationMillis());\n                time.addAndGet(duration.intValue());\n                featureSummary.add(fr.toSummaryJson());\n            }\n            sp.addAndGet(fr.getPassedCount());\n            sf.addAndGet(fr.getFailedCount());\n            errors.addAll(fr.getErrors());\n        });\n        featuresPassed = fp.get();\n        featuresFailed = ff.get();\n        scenariosPassed = sp.get();\n        scenariosFailed = sf.get();\n        timeTakenMillis = time.get();\n        saveStatsJson();\n        printStats();\n        if (suite.outputHtmlReport) {\n            String displayEnv = StringUtils.isBlank(suite.env) ? \"\\n\" : \" | env: \" + suite.env + \"\\n\";\n            suite.suiteReports.timelineReport(suite, timeline).render();\n            suite.suiteReports.tagsReport(suite, tags).render();\n            // last so that path can be printed to the console\n            File file = suite.suiteReports.summaryReport(suite, this).render();\n            System.out.println(\"\\nHTML report: (paste into browser to view) | Karate version: \"\n                    + FileUtils.KARATE_VERSION + displayEnv\n                    + file.toPath().toUri()\n                    + \"\\n===================================================================\\n\");\n        }\n    }\n\n    public Stream<FeatureResult> getFeatureResults() {\n        return suite.getFeatureResults();\n    }\n\n    public Stream<ScenarioResult> getScenarioResults() {\n        return suite.getScenarioResults();\n    }\n\n    private void saveStatsJson() {\n        String json = JsonUtils.toJson(toKarateJson());\n        File file = new File(suite.reportDir + File.separator + \"karate-summary-json.txt\");\n        FileUtils.writeToFile(file, json);\n    }\n\n    private void printStats() {\n        String displayEnv = StringUtils.isBlank(suite.env) ? \"\" : \" | env: \" + suite.env;\n        System.out.println(\"Karate version: \" + FileUtils.KARATE_VERSION + displayEnv);\n        System.out.println(\"======================================================\");\n        System.out.println(String.format(\"elapsed: %6.2f | threads: %4d | thread time: %.2f \",\n                getElapsedTime() / 1000, suite.threadCount, timeTakenMillis / 1000));\n        System.out.println(String.format(\"features: %5d | skipped: %4d | efficiency: %.2f\", getFeaturesTotal(), featuresSkipped, getEfficiency()));\n        System.out.println(String.format(\"scenarios: %4d | passed: %5d | failed: %d\",\n                getScenariosTotal(), scenariosPassed, scenariosFailed));\n        System.out.println(\"======================================================\");\n        if (!errors.isEmpty()) {\n            System.out.println(\">>> failed features:\");\n            System.out.println(getErrorMessages());\n            System.out.println(\"<<<\");\n        }\n    }\n\n    public Map<String, Object> toKarateJson() {\n        Map<String, Object> map = new HashMap();\n        map.put(\"version\", FileUtils.KARATE_VERSION);\n        map.put(\"env\", suite.env);\n        map.put(\"threads\", suite.threadCount);\n        map.put(\"featuresPassed\", featuresPassed);\n        map.put(\"featuresFailed\", featuresFailed);\n        map.put(\"featuresSkipped\", featuresSkipped);\n        map.put(\"scenariosPassed\", scenariosPassed);\n        map.put(\"scenariosfailed\", errors.size());\n        map.put(\"elapsedTime\", getElapsedTime());\n        map.put(\"totalTime\", getTimeTakenMillis());\n        map.put(\"efficiency\", getEfficiency());\n        map.put(\"resultDate\", ReportUtils.getDateString());\n        map.put(\"featureSummary\", featureSummary);\n        return map;\n    }\n\n    public String getReportDir() {\n        return suite.reportDir;\n    }\n\n    public List<String> getErrors() {\n        return errors;\n    }\n\n    public double getElapsedTime() {\n        return endTime - suite.startTime;\n    }\n\n    public double getEfficiency() {\n        return timeTakenMillis / (getElapsedTime() * suite.threadCount);\n    }\n\n    public int getScenariosPassed() {\n        return scenariosPassed;\n    }\n\n    public int getScenariosFailed() {\n        return scenariosFailed;\n    }\n\n    public int getScenariosTotal() {\n        return scenariosPassed + scenariosFailed;\n    }\n\n    public int getFeaturesTotal() {\n        return featuresPassed + featuresFailed;\n    }\n\n    public int getFeaturesPassed() {\n        return featuresPassed;\n    }\n\n    public int getFeaturesFailed() {\n        return featuresFailed;\n    }\n\n    public int getFailCount() {\n        return errors.size();\n    }\n\n    public double getTimeTakenMillis() {\n        return timeTakenMillis;\n    }\n\n    public long getStartTime() {\n        return suite.startTime;\n    }\n\n    public long getEndTime() {\n        return endTime;\n    }\n\n    public String getErrorMessages() {\n        return StringUtils.join(errors, \"\\n\");\n    }\n\n    public Suite getSuite() {\n        return suite;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/Tags.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.graal.JsEngine;\nimport com.intuit.karate.graal.JsValue;\nimport com.intuit.karate.graal.Methods;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.function.Function;\nimport org.graalvm.polyglot.Value;\n\n/**\n *\n * @author pthomas3\n */\npublic class Tags implements Iterable<Tag> {\n\n    public static final Tags EMPTY = new Tags(Collections.EMPTY_LIST);\n\n    private final Collection<Tag> original;\n    private final List<String> tags;\n    private Map<String, List<String>> tagValues;\n\n    @Override\n    public Iterator<Tag> iterator() {\n        return original.iterator();\n    }\n\n    public static class Values {\n\n        public final List<String> values;\n        public final boolean isPresent;\n\n        public Values(List<String> values) {\n            this.values = values == null ? Collections.EMPTY_LIST : values;\n            isPresent = !this.values.isEmpty();\n        }\n\n        public boolean isPresent() {\n            return isPresent;\n        }\n\n        public boolean isAnyOf(Object... args) {\n            for (Object o : args) {\n                if (values.contains(o.toString())) {\n                    return true;\n                }\n            }\n            return false;\n        }\n\n        public boolean isAllOf(Object... args) {\n            List list = new ArrayList(args.length);\n            for (Object o : args) {\n                list.add(o.toString());\n            }\n            return values.containsAll(list);\n        }\n\n        public boolean isOnly(Object... args) {\n            return isAllOf(args) && args.length == values.size();\n        }\n\n        public boolean isEach(Value v) {\n            if (!v.canExecute()) {\n                return false;\n            }\n            for (String s : values) {                \n                JsValue jv = new JsValue(JsEngine.execute(v, s));\n                if (!jv.isTrue()) {\n                    return false;\n                }\n            }\n            return true;\n        }\n\n    }\n\n    public static Tags merge(List<Tag>... lists) {\n        Set<Tag> tags = new HashSet();\n        for (List<Tag> list : lists) {\n            if (list != null) {\n                tags.addAll(list);\n            }\n        }\n        return new Tags(tags);\n    }\n\n    public Tags(Collection<Tag> in) {\n        if (in == null) {\n            original = Collections.EMPTY_LIST;\n            tags = Collections.EMPTY_LIST;\n        } else {\n            original = in;\n            tags = new ArrayList(in.size());\n            tagValues = new HashMap(in.size());\n            for (Tag tag : in) {\n                tags.add(tag.getText());\n                tagValues.put(tag.getName(), tag.getValues());\n            }\n        }\n    }\n\n    public boolean evaluate(String tagSelector, String karateEnv) {\n        if (StringUtils.containsIgnoreCase(tags, Tag.IGNORE)) {\n            return false;\n        }\n        if (tagValues.containsKey(Tag.SETUP)) {\n            return false;\n        }\n        Values envValues = valuesFor(Tag.ENV);\n        if (envValues.isPresent) {\n            if (karateEnv == null) {\n                return false;\n            }\n            if (!envValues.isAnyOf(karateEnv)) {\n                return false;\n            }\n        }\n        if (karateEnv != null && valuesFor(Tag.ENVNOT).isAnyOf(karateEnv)) {\n            return false;\n        }\n        if (tagSelector == null) {\n            return true;\n        }\n        JsEngine je = JsEngine.global();\n        je.put(\"anyOf\", (Methods.FunVar) this::anyOf);\n        je.put(\"allOf\", (Methods.FunVar) this::allOf);\n        je.put(\"not\", (Methods.FunVar) this::not);\n        je.put(\"valuesFor\", (Function<String, Values>) this::valuesFor);\n        JsValue jv = je.eval(tagSelector);\n        return jv.isTrue();\n    }\n\n    public boolean anyOf(Object... values) {\n        for (String s : removeTagPrefixes(values)) {\n            if (tags.contains(s)) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    public boolean allOf(Object... values) {\n        return tags.containsAll(removeTagPrefixes(values));\n    }\n\n    public boolean not(Object... values) {\n        return !anyOf(values);\n    }\n\n    public Values valuesFor(String name) {\n        List<String> list = tagValues.get(removeTagPrefix(name));\n        return new Values(list);\n    }\n\n    public boolean contains(String tagText) {\n        return tags.contains(removeTagPrefix(tagText));\n    }\n\n    public List<String> getTags() {\n        return tags;\n    }\n\n    public Collection<String> getTagKeys() {\n        return tagValues.keySet();\n    }\n\n    public Map<String, List<String>> getTagValues() {\n        return tagValues;\n    }\n\n    public Collection<Tag> getOriginal() {\n        return original;\n    }\n\n    private static String removeTagPrefix(String s) {\n        if (s.charAt(0) == '@') {\n            return s.substring(1);\n        } else {\n            return s;\n        }\n    }\n\n    private static Collection<String> removeTagPrefixes(Object... values) {\n        List<String> list = new ArrayList(values.length);\n        for (Object o : values) {\n            list.add(removeTagPrefix(o.toString()));\n        }\n        return list;\n    }\n\n    public static String fromKarateOptionsTags(List<String> tags) {\n        if (tags == null || tags.isEmpty()) {\n            return null;\n        }\n        return fromKarateOptionsTags(tags.toArray(new String[]{}));\n    }\n\n    public static String fromKarateOptionsTags(String... tags) {\n        if (tags == null || tags.length == 0) {\n            return null;\n        }\n        for (String s : tags) {\n            if (s.indexOf('(') != -1) { // new enhanced tag expression detected !\n                return s;\n            }\n        }\n        StringBuilder sb = new StringBuilder();\n        for (int i = 0; i < tags.length; i++) {\n            String and = StringUtils.trimToEmpty(tags[i]);\n            if (and.startsWith(\"~\")) {\n                sb.append(\"not('\").append(and.substring(1)).append(\"')\");\n            } else {\n                sb.append(\"anyOf(\");\n                List<String> or = StringUtils.split(and, ',', false);\n                for (String tag : or) {\n                    sb.append('\\'').append(tag.trim()).append('\\'').append(',');\n                }\n                sb.setLength(sb.length() - 1);\n                sb.append(')');\n            }\n            if (i < (tags.length - 1)) {\n                sb.append(\" && \");\n            }\n        }\n        return sb.toString();\n    }\n\n    @Override\n    public String toString() {\n        return tags.toString();\n    }        \n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/Scenario.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.net.URI;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class Scenario {\n\n    private final Feature feature;\n    private final FeatureSection section;\n    private final int exampleIndex;\n    \n    private int line;\n    private List<Tag> tags;\n    private String name;\n    private String description;\n    private List<Step> steps;\n    private Map<String, Object> exampleData;\n    private String dynamicExpression;\n\n    public Scenario(Feature feature, FeatureSection section, int exampleIndex) {\n        this.feature = feature;\n        this.section = section;\n        this.exampleIndex = exampleIndex;\n    }\n    \n    public boolean isEqualTo(Scenario other) {\n        return other.section.getIndex() == section.getIndex() && other.exampleIndex == exampleIndex;\n    }\n\n    public String getNameAndDescription() {\n        String temp = \"\";\n        if (name != null) {\n            temp = temp + name;\n        }\n        if (description != null) {\n            if (!temp.isEmpty()) {\n                temp = temp + \" \";\n            }\n            temp = temp + description;\n        }\n        return temp;\n    }\n\n    public String getRefIdAndName() {\n        if (name == null) {\n            return getRefId();\n        } else {\n            return getRefId() + \" \" + name;\n        }\n    }\n\n    // only called for dynamic scenarios\n    public Scenario copy(int exampleIndex) {\n        Scenario s = new Scenario(feature, section, exampleIndex);\n        s.name = name;\n        s.description = description;\n        s.tags = tags;\n        s.line = line;\n        s.dynamicExpression = dynamicExpression;\n        s.steps = new ArrayList(steps.size());\n        for (Step step : steps) {\n            Step temp = new Step(s, step.getIndex());\n            s.steps.add(temp);\n            temp.setLine(step.getLine());\n            temp.setEndLine(step.getEndLine());\n            temp.setPrefix(step.getPrefix());\n            temp.setText(step.getText());\n            temp.setDocString(step.getDocString());\n            temp.setTable(step.getTable());\n        }\n        return s;\n    }\n\n    public void replace(String token, String value) {\n        if (value == null) {\n            // this can happen for a dynamic scenario outline !\n            // give up trying a cucumber-style placeholder sub\n            // user should be fine with karate-style plain-old variables\n            return;\n        }\n        name = name.replace(token, value);\n        for (Step step : steps) {\n            String text = step.getText();\n            step.setText(text.replace(token, value));\n            String docString = step.getDocString();\n            if (docString != null) {\n                step.setDocString(docString.replace(token, value));\n            }\n            Table table = step.getTable();\n            if (table != null) {\n                step.setTable(table.replace(token, value));\n            }\n        }\n    }\n\n    public Step getStepByLine(int line) {\n        for (Step step : getStepsIncludingBackground()) {\n            if (step.getLine() == line) {\n                return step;\n            }\n        }\n        return null;\n    }\n\n    public String getRefId() {\n        int num = section.getIndex() + 1;\n        String meta = \"[\" + num;\n        if (exampleIndex != -1) {\n            meta = meta + \".\" + (exampleIndex + 1);\n        }\n        return meta + \":\" + line + \"]\";\n    }\n\n    public String getDebugInfo() {\n        return feature + \":\" + line;\n    }\n\n    public String getUniqueId() {\n        String id = feature.getResource().getPackageQualifiedName() + \"_\" + (section.getIndex() + 1);\n        return exampleIndex == -1 ? id : id + \"_\" + (exampleIndex + 1);\n    }\n\n    public List<Step> getBackgroundSteps() {\n        return feature.isBackgroundPresent() ? feature.getBackground().getSteps() : Collections.EMPTY_LIST;\n    }\n\n    public List<Step> getStepsIncludingBackground() {\n        List<Step> background = feature.isBackgroundPresent() ? feature.getBackground().getSteps() : null;\n        int count = background == null ? steps.size() : steps.size() + background.size();\n        List<Step> temp = new ArrayList(count);\n        if (background != null) {\n            temp.addAll(background);\n        }\n        temp.addAll(steps);\n        return temp;\n    }\n\n    private Tags tagsEffective; // cache\n\n    public Tags getTagsEffective() {\n        if (tagsEffective == null) {\n            tagsEffective = Tags.merge(feature.getTags(), tags);\n        }\n        return tagsEffective;\n    }\n\n    public FeatureSection getSection() {\n        return section;\n    }\n\n    public Feature getFeature() {\n        return feature;\n    }\n\n    public int getLine() {\n        return line;\n    }\n\n    public void setLine(int line) {\n        this.line = line;\n    }        \n\n    public List<Tag> getTags() {\n        return tags;\n    }\n\n    public void setTags(List<Tag> tags) {\n        this.tags = tags;\n    }\n\n    public String getName() {\n        return name;\n    }\n\n    public void setName(String name) {\n        this.name = name;\n    }\n\n    public String getDescription() {\n        return description;\n    }\n\n    public void setDescription(String description) {\n        this.description = description;\n    }\n\n    public List<Step> getSteps() {\n        return steps;\n    }\n\n    public void setSteps(List<Step> steps) {\n        this.steps = steps;\n    }\n\n    public boolean isOutlineExample() {\n        return exampleIndex != -1;\n    }\n\n    public boolean isDynamic() {\n        return dynamicExpression != null;\n    }\n\n    public String getDynamicExpression() {\n        return dynamicExpression;\n    }\n\n    public void setDynamicExpression(String dynamicExpression) {\n        this.dynamicExpression = dynamicExpression;\n    }\n\n    public Map<String, Object> getExampleData() {\n        return exampleData;\n    }\n\n    public void setExampleData(Map<String, Object> exampleData) {\n        this.exampleData = exampleData;\n    }\n\n    public int getExampleIndex() {\n        return exampleIndex;\n    }\n\n    @Override\n    public String toString() {\n        return feature.toString() + getRefId();\n    }\n\n    public URI getUriToLineNumber() {\n        return URI.create(feature.getResource().getUri() + \"?line=\" + line);\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/ScenarioCall.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n *\n * @author pthomas3\n */\npublic class ScenarioCall {\n\n    public final ScenarioRuntime parentRuntime;\n    public final int depth;\n    public final FeatureCall featureCall;\n    public final Variable arg;\n\n    private boolean callonce;\n\n    private boolean sharedScope;\n    private boolean karateConfigDisabled;\n    private int loopIndex = -1;\n    \n    public Map<String, Variable> getParentVars(boolean shallowCopy) {\n        if (parentRuntime == null) {\n            return new HashMap();\n        }\n        ScenarioEngine mockEngine = parentRuntime.featureRuntime.getMockEngine();\n        if (mockEngine != null) {\n            if (shallowCopy) {\n                return mockEngine.shallowCloneVariables();\n            } else {\n                return mockEngine.vars;\n            }\n        }\n        if (shallowCopy) {\n            return parentRuntime.engine.shallowCloneVariables();\n        } else {\n            return parentRuntime.engine.vars;\n        }\n    }\n    \n    public Config getParentConfig(boolean copy) {\n        if (parentRuntime == null) {\n            return new Config();\n        }\n        Config temp = parentRuntime.engine.getConfig();\n        return copy ? new Config(temp) : temp;\n    }\n\n    public boolean isNone() {\n        return depth == 0;\n    }\n\n    public int getLoopIndex() {\n        return loopIndex;\n    }\n\n    public void setLoopIndex(int loopIndex) {\n        this.loopIndex = loopIndex;\n    }\n\n    public void setSharedScope(boolean sharedScope) {\n        this.sharedScope = sharedScope;\n    }\n\n    public boolean isSharedScope() {\n        return sharedScope;\n    }\n\n    public boolean isCallonce() {\n        return callonce;\n    }\n\n    public void setCallonce(boolean callonce) {\n        this.callonce = callonce;\n    }\n\n    public void setKarateConfigDisabled(boolean karateConfigDisabled) {\n        this.karateConfigDisabled = karateConfigDisabled;\n    }\n\n    public boolean isKarateConfigDisabled() {\n        return karateConfigDisabled;\n    }     \n\n    public static ScenarioCall none(Map<String, Object> arg) {\n        return new ScenarioCall(null, null, arg == null ? null : new Variable(arg));\n    }\n\n    public ScenarioCall(ScenarioRuntime parentRuntime, FeatureCall featureCall, Variable arg) {\n        this.parentRuntime = parentRuntime;\n        this.featureCall = featureCall;\n        if (parentRuntime == null) {\n            depth = 0;\n        } else {\n            depth = parentRuntime.caller.depth + 1;\n        }\n        this.arg = arg;\n    }\n\n    public static class Result {\n\n        public final Variable value;\n        public final Config config;\n        public final Map<String, Variable> vars;\n\n        public Result(Variable value, Config config, Map<String, Variable> vars) {\n            this.value = value;\n            this.config = config;\n            this.vars = vars;\n        }\n\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/core/SyncExecutorService.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.core;\n\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.AbstractExecutorService;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicBoolean;\n\n/**\n *\n * @author pthomas3\n */\npublic class SyncExecutorService extends AbstractExecutorService {\n    \n    public static final SyncExecutorService INSTANCE = new SyncExecutorService();\n\n    private final AtomicBoolean terminated = new AtomicBoolean();\n\n    @Override\n    public void shutdown() {\n        terminated.set(true);\n    }\n\n    @Override\n    public List<Runnable> shutdownNow() {\n        return Collections.EMPTY_LIST;\n    }\n\n    @Override\n    public boolean isShutdown() {\n        return terminated.get();\n    }\n\n    @Override\n    public boolean isTerminated() {\n        return terminated.get();\n    }\n\n    @Override\n    public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {\n        shutdown();\n        return true;\n    }\n\n    @Override\n    public void execute(Runnable command) {\n        command.run();\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/driver/DriverRunner.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.driver;\n\nimport com.intuit.karate.core.ScenarioRuntime;\n\nimport java.util.Map;\n\n@FunctionalInterface\npublic interface DriverRunner<D extends Driver> {\n    \n    D start(Map<String, Object> options, ScenarioRuntime sr);\n    \n}\n// Path: karate-core/src/main/java/com/intuit/karate/http/HttpClientFactory.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.http;\n\nimport com.intuit.karate.core.ScenarioEngine;\n\n/**\n *\n * @author pthomas3\n */\n@FunctionalInterface\npublic interface HttpClientFactory {\n\n    HttpClient create(ScenarioEngine engine);\n\n    HttpClientFactory DEFAULT = ApacheHttpClient::new;\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/template/KarateEngineContext.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport com.intuit.karate.graal.JsEngine;\nimport com.intuit.karate.graal.JsValue;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.Set;\nimport org.graalvm.polyglot.Value;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.thymeleaf.IEngineConfiguration;\nimport org.thymeleaf.context.IEngineContext;\nimport org.thymeleaf.context.IdentifierSequences;\nimport org.thymeleaf.engine.TemplateData;\nimport org.thymeleaf.expression.IExpressionObjects;\nimport org.thymeleaf.inline.IInliner;\nimport org.thymeleaf.model.IModelFactory;\nimport org.thymeleaf.model.IProcessableElementTag;\nimport org.thymeleaf.templatemode.TemplateMode;\nimport org.thymeleaf.templateresource.ITemplateResource;\n\n/**\n *\n * @author pthomas3\n */\npublic class KarateEngineContext implements IEngineContext {\n\n    private static final Logger logger = LoggerFactory.getLogger(KarateEngineContext.class);\n\n    private static final ThreadLocal<KarateEngineContext> THREAD_LOCAL = new ThreadLocal();\n\n    private final IEngineContext wrapped;\n    private final JsEngine jsEngine;\n    private final Map<String, Object> context = new HashMap();\n    private boolean redirect;\n\n    public static KarateEngineContext initThreadLocal(IEngineContext wrapped, JsEngine engine) {\n        KarateEngineContext tec = new KarateEngineContext(wrapped, engine);\n        THREAD_LOCAL.set(tec);\n        return tec;\n    }\n\n    private KarateEngineContext(IEngineContext wrapped, JsEngine jsEngine) {\n        this.wrapped = wrapped;\n        this.jsEngine = jsEngine;\n        jsEngine.put(\"_\", context);\n    }\n\n    public static KarateEngineContext get() {\n        return THREAD_LOCAL.get();\n    }\n\n    public static void set(KarateEngineContext kec) {\n        THREAD_LOCAL.set(kec);\n    }\n\n    public JsEngine getJsEngine() {\n        return jsEngine;\n    }\n    \n    public String getCallerTemplateName() {\n        TemplateData td = wrapped.getTemplateData();\n        ITemplateResource tr = td.getTemplateResource();\n        if (tr instanceof KarateTemplateResource) {\n            KarateTemplateResource ktr = (KarateTemplateResource) tr;\n            return ktr.getCaller();\n        }\n        return null;\n    }\n\n    public String getTemplateName() {\n        return wrapped.getTemplateData().getTemplate();\n    }\n\n    public void setRedirect(boolean redirect) {\n        this.redirect = redirect;\n    }\n\n    public boolean isRedirect() {\n        return redirect;\n    }\n\n    public JsValue evalGlobal(String src) {\n        getVariableNames().forEach(name -> jsEngine.put(name, getVariable(name)));\n        try {\n            return jsEngine.eval(src);\n        } catch (Exception e) {\n            throw JsEngine.fromJsEvalException(src, e, null);\n        }\n    }\n\n    public JsValue evalLocalAsObject(String src) {\n        String temp;\n        if (src.startsWith(\"${\")) {\n            temp = \"`\" + src + \"`\";\n        } else {\n            temp = \"({\" + src + \"})\";\n        }\n        return evalLocal(temp, true);\n    }\n\n    public JsValue evalLocal(String src, boolean returnValue) {\n        try {\n            Value value = jsEngine.evalWith(getVariableNames(), this::getVariable, src, returnValue);\n            return new JsValue(value);\n        } catch (Exception e) {\n            throw JsEngine.fromJsEvalException(src, e, null);\n        }\n    }\n\n    public void setLocal(String name, Object value) {\n        context.put(name, value);\n    }\n\n    @Override\n    public void increaseLevel() {\n        if (!context.isEmpty()) {\n            setVariables(context);\n            context.clear();\n        }\n        wrapped.increaseLevel();\n    }\n\n    @Override\n    public void setVariable(String name, Object value) {\n        wrapped.setVariable(name, value);\n    }\n\n    @Override\n    public void setVariables(Map<String, Object> variables) {\n        wrapped.setVariables(variables);\n    }\n\n    @Override\n    public void removeVariable(String name) {\n        wrapped.removeVariable(name);\n    }\n\n    @Override\n    public void setTemplateData(TemplateData template) {\n        wrapped.setTemplateData(template);\n    }\n\n    @Override\n    public void decreaseLevel() {\n        wrapped.decreaseLevel();\n    }\n\n    @Override\n    public boolean containsVariable(String name) {\n        return wrapped.containsVariable(name);\n    }\n\n    @Override\n    public Set<String> getVariableNames() {\n        return wrapped.getVariableNames();\n    }\n\n    @Override\n    public Object getVariable(String name) {\n        return wrapped.getVariable(name);\n    }\n\n    @Override\n    public boolean isVariableLocal(String name) {\n        return wrapped.isVariableLocal(name);\n    }\n\n    @Override\n    public void setSelectionTarget(Object selectionTarget) {\n        wrapped.setSelectionTarget(selectionTarget);\n    }\n\n    @Override\n    public void setInliner(IInliner inliner) {\n        wrapped.setInliner(inliner);\n    }\n\n    @Override\n    public void setElementTag(IProcessableElementTag elementTag) {\n        wrapped.setElementTag(elementTag);\n    }\n\n    @Override\n    public List<IProcessableElementTag> getElementStackAbove(int contextLevel) {\n        return wrapped.getElementStackAbove(contextLevel);\n    }\n\n    @Override\n    public int level() {\n        return wrapped.level();\n    }\n\n    @Override\n    public TemplateData getTemplateData() {\n        return wrapped.getTemplateData();\n    }\n\n    @Override\n    public TemplateMode getTemplateMode() {\n        return wrapped.getTemplateMode();\n    }\n\n    @Override\n    public List<TemplateData> getTemplateStack() {\n        return wrapped.getTemplateStack();\n    }\n\n    @Override\n    public List<IProcessableElementTag> getElementStack() {\n        return wrapped.getElementStack();\n    }\n\n    @Override\n    public Map<String, Object> getTemplateResolutionAttributes() {\n        return wrapped.getTemplateResolutionAttributes();\n    }\n\n    @Override\n    public IModelFactory getModelFactory() {\n        return wrapped.getModelFactory();\n    }\n\n    @Override\n    public boolean hasSelectionTarget() {\n        return wrapped.hasSelectionTarget();\n    }\n\n    @Override\n    public Object getSelectionTarget() {\n        return wrapped.getSelectionTarget();\n    }\n\n    @Override\n    public IInliner getInliner() {\n        return wrapped.getInliner();\n    }\n\n    @Override\n    public String getMessage(Class<?> origin, String key, Object[] messageParameters, boolean useAbsent) {\n        return wrapped.getMessage(origin, key, messageParameters, useAbsent);\n    }\n\n    @Override\n    public String buildLink(String base, Map<String, Object> parameters) {\n        return wrapped.buildLink(base, parameters);\n    }\n\n    @Override\n    public IdentifierSequences getIdentifierSequences() {\n        return wrapped.getIdentifierSequences();\n    }\n\n    @Override\n    public IEngineConfiguration getConfiguration() {\n        return wrapped.getConfiguration();\n    }\n\n    @Override\n    public IExpressionObjects getExpressionObjects() {\n        return wrapped.getExpressionObjects();\n    }\n\n    @Override\n    public Locale getLocale() {\n        return wrapped.getLocale();\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/template/TemplateContext.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport java.util.Collections;\nimport java.util.Locale;\nimport java.util.Set;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.thymeleaf.context.IContext;\n\n/**\n *\n * @author pthomas3\n */\npublic class TemplateContext implements IContext {\n\n    private static final Logger logger = LoggerFactory.getLogger(TemplateContext.class);\n    \n    public static final TemplateContext LOCALE_US = new TemplateContext(Locale.US);\n\n    private final Locale locale;\n\n    public TemplateContext(Locale locale) {\n        this.locale = locale;\n    }\n\n    @Override\n    public Locale getLocale() {\n        return locale;\n    }\n\n    @Override\n    public boolean containsVariable(String name) {\n        return false;\n    }\n\n    @Override\n    public Set<String> getVariableNames() {\n        return Collections.EMPTY_SET;\n    }\n\n    @Override\n    public Object getVariable(String name) {\n        return null;\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/template/KarateTemplateEngine.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n * THE SOFTWARE.\n */\npackage com.intuit.karate.template;\n\nimport com.intuit.karate.StringUtils;\nimport com.intuit.karate.graal.JsEngine;\nimport java.io.IOException;\nimport java.io.Writer;\nimport java.util.Map;\nimport java.util.function.Supplier;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport org.thymeleaf.IEngineConfiguration;\nimport org.thymeleaf.TemplateEngine;\nimport org.thymeleaf.TemplateSpec;\nimport org.thymeleaf.context.IContext;\nimport org.thymeleaf.context.IEngineContext;\nimport org.thymeleaf.context.StandardEngineContextFactory;\nimport org.thymeleaf.dialect.IDialect;\nimport org.thymeleaf.engine.TemplateData;\nimport org.thymeleaf.engine.TemplateManager;\nimport org.thymeleaf.exceptions.TemplateOutputException;\nimport org.thymeleaf.exceptions.TemplateProcessingException;\nimport org.thymeleaf.templatemode.TemplateMode;\nimport org.thymeleaf.templateresolver.ITemplateResolver;\nimport org.thymeleaf.util.FastStringWriter;\n\n/**\n *\n * @author pthomas3\n */\npublic class KarateTemplateEngine {\n\n    private static final Logger logger = LoggerFactory.getLogger(KarateTemplateEngine.class);\n\n    private final StandardEngineContextFactory standardFactory;\n    private final TemplateEngine wrapped;\n\n    public KarateTemplateEngine(Supplier<JsEngine> jsEngine, IDialect... dialects) {\n        standardFactory = new StandardEngineContextFactory();\n        wrapped = new TemplateEngine();\n        wrapped.setEngineContextFactory((IEngineConfiguration ec, TemplateData data, Map<String, Object> attrs, IContext context) -> {\n            IEngineContext engineContext = standardFactory.createEngineContext(ec, data, attrs, context);\n            return KarateEngineContext.initThreadLocal(engineContext, jsEngine.get());\n        });\n        // the next line is a set which clears and replaces all existing / default\n        wrapped.setDialect(new KarateStandardDialect());\n        for (IDialect dialect : dialects) {\n            wrapped.addDialect(dialect);\n        }\n    }\n\n    \npublic void addTemplateResolver(ITemplateResolver templateResolver) {\n        wrapped.addTemplateResolver(templateResolver);\n    }\n\n    public void setTemplateResolver(ITemplateResolver templateResolver) {\n        wrapped.setTemplateResolver(templateResolver);\n    }\n\n    public String process(String template) {\n        return process(template, TemplateContext.LOCALE_US);\n    }\n\n    public String process(String template, IContext context) {\n        TemplateSpec templateSpec = new TemplateSpec(template, TemplateMode.HTML);\n        Writer stringWriter = new FastStringWriter(100);\n        process(templateSpec, context, stringWriter);\n        return stringWriter.toString();\n    }\n\n    public void process(TemplateSpec templateSpec, IContext context, Writer writer) {\n        try {\n            TemplateManager templateManager = wrapped.getConfiguration().getTemplateManager();\n            templateManager.parseAndProcess(templateSpec, context, writer);\n            try {\n                writer.flush();\n            } catch (IOException e) {\n                throw new TemplateOutputException(\"error flushing output writer\", templateSpec.getTemplate(), -1, -1, e);\n            }\n        } catch (Exception e) {\n            KarateEngineContext kec = KarateEngineContext.get();\n            if (kec != null && !kec.isRedirect()) { // don't log redirects\n                // make thymeleaf errors easier to troubleshoot from the logs\n                while (e.getCause() instanceof Exception) {\n                    e = (Exception) e.getCause();\n                    if (e instanceof TemplateProcessingException) {\n                        logger.error(\"{}\", e.getMessage()); // will print line and col numbers\n                        if (e.getCause() != null) { // typically the js error\n                            String message = e.getCause().getMessage();\n                            logger.error(\"{}\", message);\n                        }\n                        break;\n                    }\n                }\n            }\n            if (logger.isTraceEnabled()) {\n                logger.trace(\"{}\", StringUtils.throwableToString(e));\n            }\n            throw new RuntimeException(e);\n        }\n    }\n\n}\n\n// Path: karate-core/src/main/java/com/intuit/karate/resource/ResourceResolver.java\n/*\n * The MIT License\n *\n * Copyright 2022 Karate Labs Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy\n * of this software and associated documentation files (the \"Software\"), to deal\n * in the Software without restriction, including without limitation the rights\n * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is\n * furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in\n * all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n...\n", "needle_token_start": 15050, "needle_token_end": 15078, "code_context_ntokens": 15853, "output": ["```java\npublic void addTemplateResolver(ITemplateResolver templateResolver) {\n    wrapped.addTemplateResolver(templateResolver);\n}\n```"]}
{"repo": "Password4j/password4j", "name": "valueOf", "language": "java", "path": "src/main/java/com/password4j/types/Bcrypt.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve a specific enumeration constant based on a character identifier.\n2. **Input**: A single character that represents the identifier of the enumeration constant.\n3. **Output**: Returns the enumeration constant that matches the input character; returns null if no match is found.\n4. **Procedure**: The function iterates through all available enumeration constants, comparing the input character with the character identifier of each constant. If a match is found, the corresponding constant is returned. If no match is found by the end of the iteration, null is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/BalloonHashingFunction.java\n/*\n *  (C) Copyright 2023 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport java.math.BigInteger;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.*;\n\n/**\n * Class containing the implementation of Balloon hashing function and its parameters.\n *\n * @author David Bertoldi\n * @since 1.8.0\n */\npublic class BalloonHashingFunction extends AbstractHashingFunction\n{\n\n    private static final Map<String, BalloonHashingFunction> INSTANCES = new ConcurrentHashMap<>();\n    private static final int DEFAULT_DELTA = 3;\n\n    private final String algorithm;\n\n    private ExecutorService service;\n\n    private final int spaceCost;\n\n    private final int timeCost;\n\n    private final int parallelism;\n\n    private final int delta;\n\n    BalloonHashingFunction(String algorithm, int spaceCost, int timeCost,  int parallelism, int delta)\n    {\n...\n// Path: src/main/java/com/password4j/types/Bcrypt.java\n/*\n *  (C) Copyright 2021 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j.types;\n\npublic enum Bcrypt\n{\n    A, B, X, Y;\n\n    \npublic static Bcrypt valueOf(char minor)\n    {\n        for (Bcrypt type : Bcrypt.values())\n        {\n            if (type.minor() == minor)\n            {\n                return type;\n            }\n        }\n        return null;\n    }\n\n    public char minor()\n    {\n        return name().toLowerCase().charAt(0);\n    }\n}\n\n// Path: src/main/java/com/password4j/BcryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Bcrypt;\n\nimport java.security.MessageDigest;\nimport java.util.Arrays;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of bcrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Bcrypt\">bcrypt</a>\n * @since 0.1.0\n */\npublic class BcryptFunction extends AbstractHashingFunction\n{\n\n    private static final ConcurrentMap<String, BcryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final int BCRYPT_SALT_LEN = 16;\n\n    private static final int BLOWFISH_NUM_ROUNDS = 16;\n\n    private static final int[] P_ORIG = { //\n            0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6,\n            0x38d01377, 0xbe5466cf, 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b };\n\n    private static final int[] S_ORIG = { //\n            0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, 0x24a19947,\n            0xb3916cf7, 0x0801f2e2, 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658,\n            0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, 0xc5d1b023, 0x286085f0, 0xca417918,\n            0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60,\n            0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, 0xa15486af,\n            0x7c72e993, 0xb3ee1411, 0x636fbc2a, 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c,\n            0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60,\n            0x5dec8032, 0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239,\n            0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0, 0x6a51a0d2,\n            0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176,\n            0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, 0xe06f75d8, 0x85c12073, 0x401a449f,\n            0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b,\n            0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6,\n            0x409f60c4, 0x5e5c9ec2, 0x196a2463, 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c,\n            0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, 0xc0cba857, 0x45c8740f, 0xd20b5f39,\n            0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8,\n            0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, 0x9e5c57bb,\n            0xca6f8ca0, 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8,\n            0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, 0x9a53e479, 0xb6f84565, 0xd28e49bc,\n            0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4,\n            0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb,\n            0xf2122b64, 0x8888b812, 0x900df01c, 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777,\n            0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81,\n            0xd2ada8d9, 0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf,\n            0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, 0x2464369b,\n            0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9,\n            0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, 0xd60f573f, 0xbc9bc6e4, 0x2b60a476,\n            0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664,\n            0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0,\n            0x49a7df7d, 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, 0x193602a5, 0x75094c29,\n            0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6,\n            0xf0255dc1, 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1,\n            0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37,\n            0xf0500c0d, 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, 0xd19113f9, 0x7ca92ff6,\n            0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e,\n            0xa4751e41, 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf,\n            0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f,\n            0x2e6b7124, 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, 0xec7aec3a, 0xdb851dfa,\n            0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002,\n            0x133ae4dd, 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509,\n            0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c,\n            0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, 0xc6150eba, 0x94e2ea78,\n            0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e,\n            0xeac31f66, 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802,\n            0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830,\n            0xeb61bd96, 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, 0xeecc86bc, 0x60622ca7,\n            0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5,\n            0xc021b8f7, 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281,\n            0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb,\n            0x532e3054, 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, 0x5d4a14d9, 0xe864b7e3,\n            0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d,\n            0x41cd2105, 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3,\n            0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a,\n            0x0c55f5ea, 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, 0x7cde3759, 0xcbee7460,\n            0x4085f2a7, 0xce77326e, 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc,\n            0x800bcadc, 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340,\n            0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, 0xe93d5a68,\n            0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a,\n            0x43b7d4b7, 0x500061af, 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, 0x70f4ddd3,\n            0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a,\n            0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, 0x4f3ffea2,\n            0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9,\n            0xee39d7ab, 0x3b124e8b, 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, 0x6841e7f7,\n            0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b,\n            0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, 0xfdf8e802,\n            0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f,\n            0x404779a4, 0x5d886e17, 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, 0x257b7834, 0x602a9c60, 0xdff8e8a3,\n            0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922,\n            0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, 0x5449a36f,\n            0x877d48fa, 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8,\n            0x991be14c, 0xdb6e6b0d, 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, 0xb5390f92,\n            0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb,\n            0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, 0x6a124237,\n            0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386,\n            0xd90cec6e, 0xd5abea2a, 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, 0x60787bf8,\n            0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f,\n            0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, 0x5366f9c3,\n            0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770,\n            0x8cd55591, 0xc902de4c, 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, 0x662d09a1,\n            0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169,\n            0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, 0x9af88c27,\n            0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63,\n            0x53c2dd94, 0xc2c21634, 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, 0x6f05e409, 0x4b7c0188, 0x39720a3d,\n            0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4,\n            0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, 0xd79a3234,\n            0x92638212, 0x670efa8e, 0x406000e0, 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742,\n            0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, 0xb78c1b6b, 0x21a19045, 0xb26eb1be,\n            0x6a366eb4, 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6,\n            0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8,\n            0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1,\n            0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, 0x80e4a915, 0x87b08601, 0x9b09e6ad,\n            0x3b3ee593, 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28,\n            0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d,\n            0x283b57cc, 0xf8d56629, 0x79132e28, 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba,\n            0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb,\n            0x26dcf319, 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f,\n            0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, 0xa8b6e37e,\n            0xc3293d46, 0x48de5369, 0x6413e680, 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd,\n            0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, 0xdda26a7e, 0x3a59ff45, 0x3e350a44,\n            0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370,\n            0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, 0xe1b00428,\n            0x95983a1d, 0x06b89fb4, 0xce6ea048, 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc,\n            0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3,\n            0xa1e8aac7, 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f,\n            0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, 0xfae59361,\n            0xceb69ceb, 0xc2a86459, 0x12baa8d1, 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b,\n            0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, 0x1b0a7441, 0x4ba3348c, 0xc5be7120,\n            0xc37632d8, 0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f,\n            0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02, 0xacf08162,\n            0x5a75ebb5, 0x6e163697, 0x88d273cc, 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a,\n            0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, 0x71126905, 0xb2040222, 0xb6cbcf7c,\n            0xcd769c2b, 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060,\n            0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869,\n            0xa65cdea0, 0x3f09252d, 0xc208e69f, 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6 };\n\n    private static final int[] BF_CRYPT_CIPHERTEXT = { 0x4f727068, 0x65616e42, 0x65686f6c, 0x64657253, 0x63727944, 0x6f756274 };\n\n    private static final char[] BASE_64_CODE = { //\n            '.', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U',\n            'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',\n            's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };\n\n    private static final byte[] INDEX_64 = { //\n            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, -1,\n            -1, -1, -1, -1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,\n            27, -1, -1, -1, -1, -1, -1, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,\n            50, 51, 52, 53, -1, -1, -1, -1, -1 };\n\n    private int logRounds;\n\n    private Bcrypt type;\n\n    private BcryptFunction()\n    {\n        //\n    }\n\n    protected BcryptFunction(Bcrypt type, int logRounds)\n    {\n        this();\n        this.logRounds = logRounds;\n        this.type = type;\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * logarithmic cost.\n     * <p>\n     * The cost is expressed with the following formula:\n     * {@code cost = 2^logRounds}\n     *\n     * @param logRounds logarithmic cost\n     * @return a singleton instance\n     * @since 0.3.0\n     */\n    public static BcryptFunction getInstance(int logRounds)\n    {\n        return getInstance(Bcrypt.B, logRounds);\n    }\n\n    public static BcryptFunction getInstance(Bcrypt type, int logRounds)\n    {\n        String uid = getUID(type, logRounds);\n        if (INSTANCES.containsKey(uid))\n        {\n            return INSTANCES.get(uid);\n        }\n        else\n        {\n            BcryptFunction function = new BcryptFunction(type, logRounds);\n            INSTANCES.put(uid, function);\n            return function;\n        }\n    }\n\n    public static BcryptFunction getInstanceFromHash(String hashed)\n    {\n        internalChecks(hashed);\n\n        if (hashed.charAt(2) == '$')\n        {\n            int rounds = Integer.parseInt(hashed.substring(3, 5));\n            return getInstance(rounds);\n        }\n        else\n        {\n            char minor = hashed.charAt(2);\n            if (isNotValidMinor(minor) || hashed.charAt(3) != '$')\n                throw new BadParametersException(\"Invalid salt revision\");\n            int rounds = Integer.parseInt(hashed.substring(4, 6));\n            return getInstance(Bcrypt.valueOf(minor), rounds);\n        }\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        String salt = generateSalt();\n        return hash(plainTextPassword, salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        String salt = generateSalt();\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return internalHash(plainTextPassword, salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt)\n    {\n        return internalHash(plainTextPassword, Utils.fromBytesToString(salt));\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        return internalCheck(plainTextPassword, hashed);\n    }\n\n    private Hash internalHash(CharSequence plainTextPassword, String salt)\n    {\n        byte[] passwordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        return internalHash(passwordAsBytes, salt);\n    }\n\n    protected Hash internalHash(byte[] plainTextPasswordAsBytes, String salt)\n    {\n        String realSalt;\n        byte[] saltAsBytes;\n        byte[] hashed;\n        char minor = (char) 0;\n        int off;\n        StringBuilder rs = new StringBuilder();\n\n        internalChecks(salt);\n\n        int saltLength = salt.length();\n\n        if (salt.charAt(2) == '$')\n            off = 3;\n        else\n        {\n            minor = salt.charAt(2);\n            if (isNotValidMinor(minor) || salt.charAt(3) != '$')\n                throw new BadParametersException(\"Invalid salt revision\");\n            off = 4;\n        }\n\n        // Extract number of rounds\n        if (salt.charAt(off + 2) > '$')\n            throw new BadParametersException(\"Missing salt rounds\");\n\n        if (off == 4 && saltLength < 29)\n        {\n            throw new BadParametersException(\"Invalid salt\");\n        }\n\n        realSalt = salt.substring(off + 3, off + 25);\n        saltAsBytes = decodeBase64(realSalt, BCRYPT_SALT_LEN);\n\n        if (minor >= Bcrypt.A.minor()) // add null terminator\n            plainTextPasswordAsBytes = Arrays.copyOf(plainTextPasswordAsBytes, plainTextPasswordAsBytes.length + 1);\n\n        hashed = cryptRaw(plainTextPasswordAsBytes, saltAsBytes, logRounds, minor == Bcrypt.X.minor(), minor == Bcrypt.A.minor() ? 0x10000 : 0);\n\n        rs.append(\"$2\");\n        if (minor >= Bcrypt.A.minor())\n            rs.append(minor);\n        rs.append('$');\n        if (logRounds < 10)\n            rs.append('0');\n        rs.append(logRounds);\n        rs.append('$');\n        encodeBase64(saltAsBytes, saltAsBytes.length, rs);\n        encodeBase64(hashed, BF_CRYPT_CIPHERTEXT.length * 4 - 1, rs);\n        String result = rs.toString();\n\n        return new Hash(this, result, hashed, saltAsBytes);\n    }\n\n    public int getLogarithmicRounds()\n    {\n        return logRounds;\n    }\n\n    public Bcrypt getType()\n    {\n        return type;\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(type, logRounds) + ')';\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(logRounds, type);\n    }\n\n    protected static String getUID(Bcrypt type, int logRounds)\n    {\n        return type.minor() + \"|\" + logRounds;\n    }\n\n    protected static String toString(Bcrypt type, int logRounds)\n    {\n        return \"t=\" + type.minor() + \", r=\" + logRounds;\n    }\n\n    /**\n     * Encode a byte array using bcrypt's slightly-modified base64\n     * encoding scheme. Note that this is <strong>not</strong> compatible with\n     * the standard MIME-base64 encoding.\n     *\n     * @param toBeEncoded the byte array to encode\n     * @param length      the number of bytes to encode\n     * @param sb          StringBuilder representing the base64-encoded string\n     * @throws IllegalArgumentException if the length is invalid\n     * @since 0.1.0\n     */\n    protected static void encodeBase64(byte[] toBeEncoded, int length, StringBuilder sb)\n    {\n        int off = 0;\n        int c1;\n        int c2;\n\n        if (length <= 0 || length > toBeEncoded.length)\n        {\n            throw new BadParametersException(\"Invalid length\");\n        }\n\n        while (off < length)\n        {\n            c1 = toBeEncoded[off++] & 0xff;\n            sb.append(BASE_64_CODE[(c1 >> 2) & 0x3f]);\n            c1 = (c1 & 0x03) << 4;\n            if (off >= length)\n            {\n                sb.append(BASE_64_CODE[c1 & 0x3f]);\n                return;\n            }\n            c2 = toBeEncoded[off++] & 0xff;\n            c1 |= (c2 >> 4) & 0x0f;\n            sb.append(BASE_64_CODE[c1 & 0x3f]);\n            c1 = (c2 & 0x0f) << 2;\n            if (off >= length)\n            {\n                sb.append(BASE_64_CODE[c1 & 0x3f]);\n                return;\n            }\n            c2 = toBeEncoded[off++] & 0xff;\n            c1 |= (c2 >> 6) & 0x03;\n            sb.append(BASE_64_CODE[c1 & 0x3f]);\n            sb.append(BASE_64_CODE[c2 & 0x3f]);\n        }\n    }\n\n    /**\n     * Look up the 3 bits base64-encoded by the specified character,\n     * range-checking against conversion table\n     *\n     * @param c the base64-encoded value\n     * @return the decoded value of x\n     * @since 0.1.0\n     */\n    protected static byte char64(char c)\n    {\n        if ( c >= INDEX_64.length)\n            return -1;\n        return INDEX_64[c];\n    }\n\n    /**\n     * Decode a string encoded using bcrypt's base64 scheme to a\n     * byte array. Note that this is *not* compatible with\n     * the standard MIME-base64 encoding.\n     *\n     * @param str        the string to decode\n     * @param maxOLength the maximum number of bytes to decode\n     * @return an array containing the decoded bytes\n     * @throws IllegalArgumentException if maxOLength is invalid\n     * @since 0.1.0\n     */\n    protected static byte[] decodeBase64(String str, int maxOLength)\n    {\n        StringBuilder rs = new StringBuilder();\n        int off = 0;\n        int strLength = str.length();\n        int oLength = 0;\n        byte c1;\n        byte c2;\n        byte c3;\n        byte c4;\n        byte o;\n\n        if (maxOLength <= 0)\n            throw new BadParametersException(\"Invalid maxOLength\");\n\n...\n", "needle_token_start": 736, "needle_token_end": 802, "code_context_ntokens": 16257, "output": ["```java\npublic static Bcrypt valueOf(char minor)\n{\n    for (Bcrypt type : Bcrypt.values())\n    {\n        if (type.minor() == minor)\n        {\n            return type;\n        }\n    }\n    return null;\n}\n```"]}
{"repo": "Password4j/password4j", "name": "rOperation", "language": "java", "path": "src/main/java/com/password4j/ScryptFunction.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function performs a bitwise rotation of an integer to the left.\n2. **Input**: Two integers, where the first integer is the value to be rotated and the second integer specifies the number of positions to rotate.\n3. **Output**: An integer that is the result of the left bitwise rotation of the first input integer by the number of positions specified by the second input integer.\n4. **Procedure**: The function shifts the first input integer to the left by the number of positions specified by the second input integer. It also shifts the same integer to the right by the difference between 32 and the second input integer, effectively wrapping around the bits that are shifted out of the left side to the right side. The results of these two operations are combined using a bitwise OR operation to produce the final rotated value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/MessageDigestFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n...\n// Path: src/main/java/com/password4j/ScryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport java.security.GeneralSecurityException;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of scrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Scrypt\">scrypt</a>\n * @since 0.1.0\n */\npublic class ScryptFunction extends AbstractHashingFunction\n{\n    public static final int DERIVED_KEY_LENGTH = 64;\n\n    private static final ConcurrentMap<String, ScryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private int workFactor; // N\n\n    private int resources; // r\n\n    private int parallelization; // p\n\n    private int derivedKeyLength; // dkLen\n\n    @SuppressWarnings(\"unused\")\n    private ScryptFunction()\n    {\n        //\n    }\n\n    /**\n     * @param workFactor      (N)\n     * @param resources       (r)\n     * @param parallelization (p)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = DERIVED_KEY_LENGTH;\n    }\n\n    /**\n     * @param workFactor       (N)\n     * @param resources        (r)\n     * @param parallelization  (p)\n     * @param derivedKeyLength (dkLen)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization, int derivedKeyLength)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = derivedKeyLength;\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.0.0\n     */\n    public static ScryptFunction getInstanceFromHash(String hashed)\n    {\n        String[] parts = hashed.split(\"\\\\$\");\n        if (parts.length == 4)\n        {\n            long params = Long.parseLong(parts[1], 16);\n            int workFactor = (int) Math.pow(2.0D, (params >> 16 & 65535L));\n            int resources = (int) params >> 8 & 255;\n            int parallelization = (int) params & 255;\n            int derivedKeyLength = Utils.decodeBase64(parts[3]).length;\n\n            return ScryptFunction.getInstance(workFactor, resources, parallelization, derivedKeyLength);\n        }\n        throw new BadParametersException(\"`\" + hashed + \"` is not a valid hash\");\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * N, r and p parameters.\n     *\n     * @param workFactor      work factor (N)\n     * @param resources       resources (r)\n     * @param parallelization parallelization (p)\n     * @return a singleton instance\n     * @since 0.3.0\n     */\n    public static ScryptFunction getInstance(int workFactor, int resources, int parallelization)\n    {\n        return getInstance(workFactor, resources, parallelization, DERIVED_KEY_LENGTH);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * N, r and p parameters.\n     *\n     * @param workFactor       work factor (N)\n     * @param resources        resources (r)\n     * @param parallelization  parallelization (p)\n     * @param derivedKeyLength derived key length (dkLen)\n     * @return a singleton instance\n     * @since 1.5.1\n     */\n    public static ScryptFunction getInstance(int workFactor, int resources, int parallelization, int derivedKeyLength)\n    {\n        String key = getUID(resources, workFactor, parallelization, derivedKeyLength);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            ScryptFunction function = new ScryptFunction(workFactor, resources, parallelization, derivedKeyLength);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        byte[] saltAsBytes = Utils.fromCharSequenceToBytes(salt);\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        return internalHash(plainTextPasswordAsBytes, saltAsBytes);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes, byte[] salt)\n    {\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    private Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n\n        try\n        {\n            byte[] derived = scrypt(plainTextPassword, salt, derivedKeyLength);\n            String params = Long.toString((long) Utils.log2(workFactor) << 16 | (long) resources << 8 | parallelization, 16);\n            String sb = \"$\" + params + '$' + Utils.encodeBase64(salt) + '$'\n                    + Utils.encodeBase64(derived);\n            return new Hash(this, sb, derived, salt);\n        }\n        catch (IllegalArgumentException | GeneralSecurityException e)\n        {\n            String stringedSalt = Utils.fromBytesToString(salt);\n            String message = \"Invalid specification with salt=\" + stringedSalt + \", N=\" + workFactor + \", r=\" + resources + \" and p=\" + parallelization;\n            throw new BadParametersException(message, e);\n        }\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        try\n        {\n            List<byte[]> parts = Utils.split(hashed, (byte) 36);\n            if (parts.size() == 4)\n            {\n                byte[] salt = Utils.decodeBase64(parts.get(2));\n                byte[] derived0 = Utils.decodeBase64(parts.get(3));\n                byte[] derived1 = scrypt(plainTextPassword, salt, derivedKeyLength);\n                return slowEquals(derived0, derived1);\n            }\n            else\n            {\n                throw new BadParametersException(\"Invalid hashed value\");\n            }\n        }\n        catch (GeneralSecurityException gse)\n        {\n            throw new IllegalStateException(\"JVM doesn't support SHA1PRNG or HMAC_SHA256?\", gse);\n        }\n    }\n\n    protected static String toString(int resources, int workFactor, int parallelization, int derivedKeyLength)\n    {\n        return \"N=\" + workFactor + \", r=\" + resources + \", p=\" + parallelization + \", l=\" + derivedKeyLength;\n    }\n\n    protected static String getUID(int resources, int workFactor, int parallelization, int derivedKeyLength)\n    {\n        return workFactor + \"|\" + resources + \"|\" + parallelization + \"|\" + derivedKeyLength;\n    }\n\n    \npublic static int rOperation(int a, int b)\n    {\n        return a << b | a >>> 32 - b;\n    }\n\n    public static void salsa208(byte[] xArray)\n    {\n        int[] b32 = new int[16];\n        int[] x = new int[16];\n\n        int i;\n        for (i = 0; i < 16; ++i)\n        {\n            b32[i] = (xArray[i * 4] & 255);\n            b32[i] |= (xArray[i * 4 + 1] & 255) << 8;\n            b32[i] |= (xArray[i * 4 + 2] & 255) << 16;\n            b32[i] |= (xArray[i * 4 + 3] & 255) << 24;\n        }\n\n        System.arraycopy(b32, 0, x, 0, 16);\n\n        for (i = 8; i > 0; i -= 2)\n        {\n            x[4] ^= rOperation(x[0] + x[12], 7);\n            x[8] ^= rOperation(x[4] + x[0], 9);\n            x[12] ^= rOperation(x[8] + x[4], 13);\n            x[0] ^= rOperation(x[12] + x[8], 18);\n            x[9] ^= rOperation(x[5] + x[1], 7);\n            x[13] ^= rOperation(x[9] + x[5], 9);\n            x[1] ^= rOperation(x[13] + x[9], 13);\n            x[5] ^= rOperation(x[1] + x[13], 18);\n            x[14] ^= rOperation(x[10] + x[6], 7);\n            x[2] ^= rOperation(x[14] + x[10], 9);\n            x[6] ^= rOperation(x[2] + x[14], 13);\n            x[10] ^= rOperation(x[6] + x[2], 18);\n            x[3] ^= rOperation(x[15] + x[11], 7);\n            x[7] ^= rOperation(x[3] + x[15], 9);\n            x[11] ^= rOperation(x[7] + x[3], 13);\n            x[15] ^= rOperation(x[11] + x[7], 18);\n            x[1] ^= rOperation(x[0] + x[3], 7);\n            x[2] ^= rOperation(x[1] + x[0], 9);\n            x[3] ^= rOperation(x[2] + x[1], 13);\n            x[0] ^= rOperation(x[3] + x[2], 18);\n            x[6] ^= rOperation(x[5] + x[4], 7);\n            x[7] ^= rOperation(x[6] + x[5], 9);\n            x[4] ^= rOperation(x[7] + x[6], 13);\n            x[5] ^= rOperation(x[4] + x[7], 18);\n            x[11] ^= rOperation(x[10] + x[9], 7);\n            x[8] ^= rOperation(x[11] + x[10], 9);\n            x[9] ^= rOperation(x[8] + x[11], 13);\n            x[10] ^= rOperation(x[9] + x[8], 18);\n            x[12] ^= rOperation(x[15] + x[14], 7);\n            x[13] ^= rOperation(x[12] + x[15], 9);\n            x[14] ^= rOperation(x[13] + x[12], 13);\n            x[15] ^= rOperation(x[14] + x[13], 18);\n        }\n\n        for (i = 0; i < 16; ++i)\n        {\n            b32[i] += x[i];\n        }\n\n        for (i = 0; i < 16; ++i)\n        {\n            xArray[i * 4] = (byte) (b32[i] & 255);\n            xArray[i * 4 + 1] = (byte) (b32[i] >> 8 & 255);\n            xArray[i * 4 + 2] = (byte) (b32[i] >> 16 & 255);\n            xArray[i * 4 + 3] = (byte) (b32[i] >> 24 & 255);\n        }\n\n    }\n\n    public static void blockXOR(byte[] sArray, int si, byte[] dArray, int di, int length)\n    {\n        for (int i = 0; i < length; ++i)\n        {\n            dArray[di + i] ^= sArray[si + i];\n        }\n\n    }\n\n\n\n    /**\n     * Estimates the required memory to calculate an hash with\n     * the current configuration.\n     *\n     * @return the required memory\n     * @since 0.1.0\n     */\n    public long getRequiredBytes()\n    {\n        return 128L * workFactor * resources * parallelization;\n    }\n\n    public int getWorkFactor()\n    {\n        return workFactor;\n    }\n\n    public int getResources()\n    {\n        return resources;\n    }\n\n    public int getParallelization()\n    {\n        return parallelization;\n    }\n\n    public int getDerivedKeyLength()\n    {\n        return derivedKeyLength;\n    }\n\n    /**\n     * A more readable version of {@link #getRequiredBytes()},\n     * changing the unit (B, KB, MB) so that the number has at most\n     * 2 decimal places.\n     *\n     * @return the required memory\n     * @since 0.3.0\n     */\n    public String getRequiredMemory()\n    {\n        long memoryInBytes = getRequiredBytes();\n        if (memoryInBytes > 1_000_000)\n        {\n            return Math.round(memoryInBytes / 10_000f) / 100.0 + \"MB\";\n        }\n        if (memoryInBytes > 1_000)\n        {\n            return Math.round(memoryInBytes / 1_000f) / 100.0 + \"KB\";\n        }\n        return memoryInBytes + \"B\";\n    }\n\n    @Override\n    public boolean equals(Object obj)\n    {\n        if (obj == null || !this.getClass().equals(obj.getClass()))\n        {\n            return false;\n        }\n\n        ScryptFunction otherStrategy = (ScryptFunction) obj;\n        return this.workFactor == otherStrategy.workFactor //\n                && this.resources == otherStrategy.resources //\n                && this.parallelization == otherStrategy.parallelization;\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(this.resources, this.workFactor, this.parallelization,\n                this.derivedKeyLength) + ')';\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(resources, workFactor, parallelization);\n    }\n\n    public byte[] scrypt(byte[] passwd, byte[] salt, int dkLen) throws GeneralSecurityException\n    {\n        if (workFactor >= 2 && (workFactor & workFactor - 1) == 0)\n        {\n            if (workFactor > 16777215 / resources)\n            {\n                throw new IllegalArgumentException(\"Parameter N is too large\");\n            }\n            else if (resources > 16777215 / parallelization)\n            {\n                throw new IllegalArgumentException(\"Parameter r is too large\");\n            }\n            else\n            {\n                byte[] xyArray = new byte[256 * resources];\n                byte[] vArray = new byte[128 * resources * workFactor];\n                byte[] intensiveSalt = PBKDF2Function.internalHash(Utils.fromBytesToString(passwd).toCharArray(), salt, Hmac.SHA256.name(), 1,\n                        8 * parallelization * 128 * resources).getEncoded();\n\n                for (int i = 0; i < parallelization; ++i)\n                {\n                    sMix(intensiveSalt, i * 128 * resources, vArray, xyArray);\n                }\n\n                return PBKDF2Function\n                        .internalHash(Utils.fromBytesToString(passwd).toCharArray(), intensiveSalt, Hmac.SHA256.name(), 1, 8 * dkLen)\n                        .getEncoded();\n            }\n        }\n        else\n        {\n            throw new IllegalArgumentException(\"N must be a power of 2 greater than 1. Found \" + workFactor);\n        }\n    }\n\n    public void sMix(byte[] intensiveSalt, int bi, byte[] vArray, byte[] xyArray)\n    {\n        int xi = 0;\n        int yi = 128 * resources;\n        System.arraycopy(intensiveSalt, bi, xyArray, xi, 128 * resources);\n\n        int i;\n        for (i = 0; i < workFactor; ++i)\n        {\n            System.arraycopy(xyArray, xi, vArray, i * 128 * resources, 128 * resources);\n            blockmixSalsa8(xyArray, xi, yi);\n        }\n\n        for (i = 0; i < workFactor; ++i)\n        {\n            int j = integerify(xyArray, xi) & workFactor - 1;\n            blockXOR(vArray, j * 128 * resources, xyArray, xi, 128 * resources);\n            blockmixSalsa8(xyArray, xi, yi);\n        }\n\n        System.arraycopy(xyArray, xi, intensiveSalt, bi, 128 * resources);\n    }\n\n    public void blockmixSalsa8(byte[] xyArray, int bi, int yi)\n    {\n        byte[] xArray = new byte[64];\n        System.arraycopy(xyArray, bi + (2 * resources - 1) * 64, xArray, 0, 64);\n\n        int i;\n        for (i = 0; i < 2 * resources; ++i)\n        {\n            blockXOR(xyArray, i * 64, xArray, 0, 64);\n            salsa208(xArray);\n            System.arraycopy(xArray, 0, xyArray, yi + i * 64, 64);\n        }\n\n        for (i = 0; i < resources; ++i)\n        {\n            System.arraycopy(xyArray, yi + i * 2 * 64, xyArray, bi + i * 64, 64);\n        }\n\n        for (i = 0; i < resources; ++i)\n        {\n            System.arraycopy(xyArray, yi + (i * 2 + 1) * 64, xyArray, bi + (i + resources) * 64, 64);\n        }\n\n    }\n\n    public int integerify(byte[] xyArray, int xi)\n    {\n        xi += (2 * resources - 1) * 64;\n        int n = (xyArray[xi] & 255);\n        n |= (xyArray[xi + 1] & 255) << 8;\n        n |= (xyArray[xi + 2] & 255) << 16;\n        n |= (xyArray[xi + 3] & 255) << 24;\n        return n;\n    }\n}\n\n// Path: src/main/java/com/password4j/AlgorithmFinder.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Argon2;\nimport com.password4j.types.Bcrypt;\nimport com.password4j.types.Hmac;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.security.NoSuchAlgorithmException;\nimport java.security.Provider;\nimport java.security.SecureRandom;\nimport java.security.Security;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\n\n\n/**\n * This utility class finds algorithms with their configuration\n * based on the environment.\n * <p>\n * In this context, by environment is intended the set of\n * parameters set in the JVM and in the <i>psw4j.properties</i> file.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic class AlgorithmFinder\n{\n\n    private static final Logger LOG = LoggerFactory.getLogger(AlgorithmFinder.class);\n\n    /**\n     * Singleton instance of {@link SecureRandom}.\n     * <p>\n     * By definition this instance does not need\n     * to be re-instantiated in order to generate\n     * non-deterministic output.\n     *\n     * @see #getSecureRandom()\n     * @since 0.1.0\n     */\n    private static SecureRandom secureRandom;\n\n    private AlgorithmFinder()\n    {\n        //\n    }\n\n    /**\n     * Retrieve a singleton instance of a thread-safe {@link SecureRandom} object.\n     * If the environment allows it, a cryptographically strong random number generator (RNG)\n     * is returned.\n     * <p>\n     * The usage of {@link SecureRandom#getInstanceStrong()} can be forced by\n     * setting {@code global.random.strong} to {@code true} in the <i>psw4j.properties</i>.\n     * Please be aware that this configuration may affect the system performance, while\n     * ensuring an high level of randomness.\n     * <p>\n     * It is recommendable for Unix systems to set {@code securerandom.source} to\n     * {@code file:/dev/urandom} in your <i>java.security</i> file.\n     *\n     * @return a thread-safe {@link SecureRandom} instance\n     * @see SecureRandom#getInstanceStrong()\n     * @since 0.1.0\n     */\n    public static SecureRandom getSecureRandom()\n    {\n        return secureRandom;\n    }\n\n    /**\n     * Creates a singleton instance of {@link PBKDF2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Algorithm</td>\n     *     <td>hash.pbkdf2.algorithm</td>\n     *     <td>PBKDF2WithHmacSHA256</td>\n     *   </tr>\n     *   <tr>\n     *     <td># iterations</td>\n     *     <td>hash.pbkdf2.iterations</td>\n     *     <td>310000</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Key length</td>\n     *     <td>hash.pbkdf2.length</td>\n     *     <td>512</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link PBKDF2Function}\n     * @since 0.1.0\n     */\n    public static PBKDF2Function getPBKDF2Instance()\n    {\n        Param params = internalGetProperties();\n        return PBKDF2Function.getInstance(params.algorithm, params.iterations, params.length);\n    }\n\n    /**\n     * Creates a singleton instance of {@link CompressedPBKDF2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Algorithm</td>\n     *     <td>hash.pbkdf2.algorithm</td>\n     *     <td>PBKDF2WithHmacSHA512</td>\n     *   </tr>\n     *   <tr>\n     *     <td># iterations</td>\n     *     <td>hash.pbkdf2.iterations</td>\n     *     <td>64000</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Key length</td>\n     *     <td>hash.pbkdf2.length</td>\n     *     <td>512</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link CompressedPBKDF2Function}\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getCompressedPBKDF2Instance()\n    {\n        Param params = internalGetProperties();\n        return CompressedPBKDF2Function.getInstance(params.algorithm, params.iterations, params.length);\n    }\n\n    private static Param internalGetProperties()\n    {\n        String algorithm = PropertyReader\n                .readString(\"hash.pbkdf2.algorithm\", Hmac.SHA256.name(), \"PBKDF2 algorithm is not defined\");\n        int iterations = PropertyReader.readInt(\"hash.pbkdf2.iterations\", 310_000, \"PBKDF2 #iterations are not defined\");\n        int length = PropertyReader.readInt(\"hash.pbkdf2.length\", Hmac.SHA256.bits(), \"PBKDF2 key length is not defined\");\n        return new Param(algorithm, iterations, length);\n    }\n\n    /**\n     * Creates a singleton instance of {@link BcryptFunction}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *      *     <td>Logarithmic number of rounds</td>\n     *      *     <td>hash.bcrypt.minor</td>\n     *      *     <td>10</td>\n     *      *   </tr>\n     *   <tr>\n     *     <td>Logarithmic number of rounds</td>\n     *     <td>hash.bcrypt.rounds</td>\n     *     <td>10</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link BcryptFunction}\n     * @since 0.1.0\n     */\n    public static BcryptFunction getBcryptInstance()\n    {\n        char minor = PropertyReader.readChar(\"hash.bcrypt.minor\", 'b', \"bcrypt minor version is not defined\");\n        int rounds = PropertyReader.readInt(\"hash.bcrypt.rounds\", 10, \"bcrypt rounds are not defined\");\n        return BcryptFunction.getInstance(Bcrypt.valueOf(minor), rounds);\n    }\n\n    /**\n     * Creates a singleton instance of {@link ScryptFunction}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Work Factor (N)</td>\n     *     <td>hash.scrypt.workfactor</td>\n     *     <td>65536</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Resources (r)</td>\n     *     <td>hash.scrypt.resources</td>\n     *     <td>8</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Parallelization (p)</td>\n     *     <td>hash.scrypt.parallelization</td>\n     *     <td>1</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Derived Key Length (dkLen)</td>\n     *     <td>hash.scrypt.derivedKeyLength</td>\n     *     <td>64</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link ScryptFunction}\n     * @since 0.1.0\n     */\n    public static ScryptFunction getScryptInstance()\n    {\n        int workFactor = PropertyReader.readInt(\"hash.scrypt.workfactor\", 65_536, \"scrypt work factor (N) is not defined\");\n        int resources = PropertyReader.readInt(\"hash.scrypt.resources\", 8, \"scrypt resources (r) is not defined\");\n        int parallelization = PropertyReader\n                .readInt(\"hash.scrypt.parallelization\", 1, \"scrypt parallelization (p) is not defined\");\n        int derivedKeyLength = PropertyReader.readInt(\"hash.scrypt.derivedKeyLength\", ScryptFunction.DERIVED_KEY_LENGTH,\n                \"scrypt derivedKeyLength (dkLen) is not defined\");\n        return ScryptFunction.getInstance(workFactor, resources, parallelization, derivedKeyLength);\n    }\n\n    public static MessageDigestFunction getMessageDigestInstance()\n    {\n        String algorithm = PropertyReader.readString(\"hash.md.algorithm\", \"SHA-512\", \"Message Digest algorithm is not defined\");\n        String saltOption = PropertyReader.readString(\"hash.md.salt.option\", \"APPEND\", \"Salt option is not defined\");\n        try\n        {\n            return MessageDigestFunction.getInstance(algorithm, SaltOption.valueOf(saltOption.toUpperCase()));\n        }\n        catch (IllegalArgumentException iae)\n        {\n            LOG.warn(\"{} is not a valid option. Fallback to default.\", saltOption);\n            return MessageDigestFunction.getInstance(algorithm);\n        }\n    }\n\n    /**\n     * Creates a singleton instance of {@link Argon2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Memory (log2)</td>\n     *     <td>hash.argon2.memory</td>\n     *     <td>15</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Iterations</td>\n     *     <td>hash.argon2.iterations</td>\n     *     <td>2</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Output Length</td>\n     *     <td>hash.argon2.length</td>\n     *     <td>32</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Parallelism</td>\n     *     <td>hash.argon2.parallelism</td>\n     *     <td>1</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Type</td>\n     *     <td>hash.argon2.type</td>\n     *     <td>id</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Version</td>\n     *     <td>hash.argon2.version</td>\n     *     <td>19</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link ScryptFunction}\n     * @since 1.5.0\n     */\n    public static Argon2Function getArgon2Instance()\n    {\n        int memory = PropertyReader.readInt(\"hash.argon2.memory\", 15_360, \"Argon2 memory is not defined\");\n        int iterations = PropertyReader.readInt(\"hash.argon2.iterations\", 2, \"Argon2 #iterations is not defined\");\n        int outputLength = PropertyReader.readInt(\"hash.argon2.length\", 32, \"Argon2 output length is not defined\");\n        int parallelism = PropertyReader.readInt(\"hash.argon2.parallelism\", 1, \"Argon2 parallelism is not defined\");\n        String type = PropertyReader.readString(\"hash.argon2.type\", \"id\", \"Argon2 type is not defined\");\n        int version = PropertyReader.readInt(\"hash.argon2.version\", 19, \"Argon2 version is not defined\");\n        return Argon2Function\n                .getInstance(memory, iterations, parallelism, outputLength, Argon2.valueOf(type.toUpperCase()), version);\n    }\n\n    public static BalloonHashingFunction getBalloonHashingInstance()\n    {\n        int space = PropertyReader.readInt(\"hash.balloon.space\", 1024, \"BalloonHashing memory (space) is not defined\");\n        int time = PropertyReader.readInt(\"hash.balloon.time\", 3, \"BalloonHashing #iterations (time) is not defined\");\n        int parallelism = PropertyReader.readInt(\"hash.balloon.parallelism\", 1, \"BalloonHashing parallelism is not defined\");\n        int delta = PropertyReader.readInt(\"hash.balloon.delta\", 3, \"BalloonHashing delta is not defined\");\n        String algorithm = PropertyReader.readString(\"hash.balloon.algorithm\", \"SHA-256\", \"BalloonHashing algorithm is not defined\");\n        return BalloonHashingFunction.getInstance(algorithm, space, time, parallelism, delta);\n    }\n\n    /**\n     * Finds the list of supported PBKDF2 algorithms by\n     * the environment's {@link Provider}s.\n     *\n     * @return the list of supported PBKDF2 algorithms\n     * @since 0.2.0\n     */\n    public static List<String> getAllPBKDF2Variants()\n    {\n        List<String> result = new ArrayList<>();\n        for (Provider provider : Security.getProviders())\n        {\n            // Some JDK implementation may return null instead of an empty array.\n            // see https://github.com/Password4j/password4j/issues/120\n            if (provider.getServices() != null)\n            {\n                for (Provider.Service service : provider.getServices())\n                {\n                    if (\"SecretKeyFactory\".equals(service.getType()) && service.getAlgorithm().startsWith(\"PBKDF2\"))\n                    {\n                        result.add(service.getAlgorithm());\n                    }\n                }\n            }\n        }\n        return result;\n    }\n\n    public static Set<String> getAllMessageDigests()\n    {\n        return Security.getAlgorithms(\"MessageDigest\");\n    }\n\n    private static boolean useStrongRandom()\n    {\n        return PropertyReader.readBoolean(\"global.random.strong\", false);\n    }\n\n    static void initialize()\n    {\n        SecureRandom sr;\n        if (useStrongRandom())\n        {\n            try\n            {\n                sr = Utils.getInstanceStrong();\n            }\n            catch (NoSuchAlgorithmException nsae)\n            {\n                /* Even if there's no strong instance, execution\n                 * must continue with a less strong SecureRandom instance */\n                LOG.warn(\"No source of strong randomness found for this environment.\");\n                sr = new SecureRandom();\n            }\n\n        }\n        else\n        {\n            sr = new SecureRandom();\n        }\n        secureRandom = sr;\n    }\n\n    private static class Param\n    {\n        String algorithm;\n\n        int iterations;\n\n        int length;\n\n        Param(String algorithm, int iterations, int length)\n        {\n            this.algorithm = algorithm;\n            this.iterations = iterations;\n            this.length = length;\n        }\n    }\n\n    static\n    {\n        initialize();\n    }\n}\n\n// Path: src/main/java/com/password4j/SaltGenerator.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport java.security.SecureRandom;\n\n\n/**\n * This class contains static functions that\n * help to create a secure salt.\n * <p>\n * In cryptography, a salt is random data that is used as\n * an additional input to a CHF.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic class SaltGenerator\n{\n\n    private SaltGenerator()\n    {\n        //\n    }\n\n    /**\n     * Generates an array of {@code byte}s that can be used\n     * as salt by the CHFs.\n     * The generated salt is created by a cryptographically\n     * strong random number generator (RNG).\n     * <p>\n     * The parameter length must be a non-negative number,\n     * otherwise a {@link BadParametersException} is thrown.\n     *\n     * @param length of the returned byte array\n     * @return a salt as array of {@code byte}s\n     * @throws BadParametersException if the length is negative\n     * @since 0.1.0\n     */\n    public static byte[] generate(int length)\n    {\n        if (length < 0)\n        {\n            throw new BadParametersException(\"Salt length cannot be negative\");\n        }\n        byte[] salt = new byte[length];\n        SecureRandom sr = AlgorithmFinder.getSecureRandom();\n        sr.nextBytes(salt);\n        return salt;\n    }\n\n    /**\n     * Generates an array of {@code byte}s that can be used\n     * as salt by the CHFs.\n     * The generated salt is created by a cryptographically\n     * strong random number generator (RNG).\n     * <p>\n     * The length of the array is 64.\n     *\n     * @return a salt as array of {@code byte}s\n     * @since 0.1.0\n     */\n    public static byte[] generate()\n    {\n        return generate(get());\n    }\n\n    /**\n     * Get the length of salt from configurations and\n     * must be stored in a location different from the one used\n     * for the passwords.\n     * <p>\n     * It can be set in the <i>psw4j.properties</i> file with\n     * the property {@code global.salt.length}.\n     * <p>\n     * If the <i>psw4j.properties</i> or the property {@code global.salt.length}\n     * are not found, {@code null} is returned.\n     *\n     * @return a shared pepper set in the <i>psw4j.properties</i> file.\n     * @since 1.7.0\n     */\n    public static int get()\n    {\n        return PropertyReader.readInt(\"global.salt.length\", 64, \"Global salt length is not defined in properties file\");\n    }\n\n}\n\n// Path: src/main/java/com/password4j/PBKDF2Function.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport javax.crypto.SecretKey;\nimport javax.crypto.SecretKeyFactory;\nimport javax.crypto.spec.PBEKeySpec;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.spec.InvalidKeySpecException;\nimport java.util.Arrays;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of PBKDF2 function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/PBKDF2\">PBKDF2</a>\n * @since 0.1.0\n */\npublic class PBKDF2Function extends AbstractHashingFunction\n{\n    private static final Map<String, PBKDF2Function> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final String ALGORITHM_PREFIX = \"PBKDF2WithHmac\";\n\n    protected Hmac algorithm;\n\n    protected String algorithmAsString;\n\n    protected int iterations;\n\n    protected int length;\n\n    protected PBKDF2Function()\n    {\n        //\n    }\n\n    protected PBKDF2Function(int iterations, int length)\n    {\n        this.iterations = iterations;\n        this.length = length;\n    }\n\n    protected PBKDF2Function(Hmac algorithm, int iterations, int length)\n    {\n        this(iterations, length);\n        this.algorithm = algorithm;\n        this.algorithmAsString = algorithm.name();\n    }\n\n    protected PBKDF2Function(String algorithm, int iterations, int length)\n    {\n        this(iterations, length);\n        this.algorithmAsString = algorithm;\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @since 0.1.0\n     */\n    public static PBKDF2Function getInstance(Hmac algorithm, int iterations, int length)\n    {\n        return getInstance(algorithm.name(), iterations, length);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  string version of hmac algorithm\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @since 0.1.0\n     */\n    public static PBKDF2Function getInstance(String algorithm, int iterations, int length)\n    {\n        String key = getUID(algorithm, iterations, length);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            PBKDF2Function function = new PBKDF2Function(algorithm, iterations, length);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return hash(Utils.fromCharSequenceToBytes(plainTextPassword), salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return hash(plainTextPasswordAsBytes, salt);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return hash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt)\n    {\n        try\n        {\n            SecretKey key = internalHash(plainTextPassword, salt, this.algorithmAsString, this.iterations, this.length);\n            byte[] encodedKey = key.getEncoded();\n            return new Hash(this, getHash(encodedKey, salt), encodedKey, salt);\n        }\n        catch (NoSuchAlgorithmException nsae)\n        {\n            String message = \"`\" + algorithm + \"` is not a valid algorithm\";\n            throw new UnsupportedOperationException(message, nsae);\n        }\n        catch (IllegalArgumentException | InvalidKeySpecException e)\n        {\n            String message = \"Invalid specification with salt=\" + Arrays.toString(salt) + \", iterations=\" + iterations + \" and length=\" + length;\n            throw new BadParametersException(message, e);\n        }\n    }\n\n    protected static SecretKey internalHash(byte[] plainTextPassword, byte[] salt, String algorithm, int iterations, int length) throws NoSuchAlgorithmException, InvalidKeySpecException\n    {\n        if (salt == null)\n        {\n            throw new IllegalArgumentException(\"Salt cannot be null\");\n        }\n        return internalHash(Utils.fromBytesToChars(plainTextPassword), salt, algorithm, iterations, length);\n    }\n\n    protected static SecretKey internalHash(char[] plain, byte[] salt, String algorithm, int iterations, int length)\n            throws NoSuchAlgorithmException, InvalidKeySpecException\n    {\n        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance(ALGORITHM_PREFIX + algorithm);\n        PBEKeySpec spec = new PBEKeySpec(plain, salt, iterations, length);\n        return secretKeyFactory.generateSecret(spec);\n    }\n\n    protected static String getUID(String algorithm, int iterations, int length)\n    {\n        return algorithm + \"|\" + iterations + \"|\" + length;\n    }\n\n    protected static String toString(String algorithm, int iterations, int length)\n    {\n        return \"a=\" + algorithm + \", i=\" + iterations + \", l=\" + length;\n    }\n\n\n\n    /**\n     * Overridable PBKDF2 generator\n     *\n     * @param encodedKey secret encodedKey\n     * @param salt       cryptographic salt\n     * @return the PBKDF2 hash string\n     */\n    protected String getHash(byte[] encodedKey, byte[] salt)\n    {\n        return Utils.encodeBase64(encodedKey);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check((byte[]) null, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPasswordAsBytes, byte[] hashed)\n    {\n        throw new UnsupportedOperationException(\"This implementation requires an explicit salt.\");\n\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        Hash internalHash = hash(plainTextPassword, salt);\n        return slowEquals(internalHash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPasswordAsBytes, byte[] hashed, byte[] salt)\n    {\n        Hash internalHash = hash(plainTextPasswordAsBytes, salt);\n        return slowEquals(internalHash.getResultAsBytes(), hashed);\n    }\n\n\n\n    public String getAlgorithm()\n    {\n        return algorithmAsString;\n    }\n\n    public int getIterations()\n    {\n        return iterations;\n    }\n\n    public int getLength()\n    {\n        return length;\n    }\n\n    @Override\n    public boolean equals(Object obj)\n    {\n        if (obj == null || !getClass().equals(obj.getClass()))\n        {\n            return false;\n        }\n\n        PBKDF2Function otherStrategy = (PBKDF2Function) obj;\n        return this.algorithmAsString.equals(otherStrategy.algorithmAsString) //\n                && this.iterations == otherStrategy.iterations //\n                && this.length == otherStrategy.length;\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(this.algorithmAsString, this.iterations, this.length) + ')';\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(algorithmAsString, iterations, length);\n    }\n\n}\n\n// Path: src/main/java/com/password4j/BadParametersException.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\n/**\n * This exception is normally thrown when a not well formed parameter\n * is passed as argument to a function.\n * <p>\n * This exception covers all the exceptions raised by underlying logic,\n * grouping them as one exception.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic class BadParametersException extends IllegalArgumentException\n{\n\n    private static final long serialVersionUID = 9204720180786210237L;\n\n    /**\n     * Constructs the exception.\n     *\n     * @param message the message describing the cause of the exception\n     * @since 0.1.0\n     */\n    public BadParametersException(String message)\n    {\n        super(message);\n    }\n\n    /**\n     * Constructs the exception.\n     *\n     * @param message   the message describing the cause of the exception\n     * @param exception the exception masked by this object\n     * @since 0.1.0\n     */\n    public BadParametersException(String message, Throwable exception)\n    {\n        super(message, exception);\n    }\n}\n\n// Path: src/main/java/com/password4j/Blake2b.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport java.util.Arrays;\n\n\nclass Blake2b\n{\n    private static final long[] IV = { 0x6a09e667f3bcc908L, 0xbb67ae8584caa73bL, 0x3c6ef372fe94f82bL, 0xa54ff53a5f1d36f1L,\n            0x510e527fade682d1L, 0x9b05688c2b3e6c1fL, 0x1f83d9abfb41bd6bL, 0x5be0cd19137e2179L };\n\n    private static final byte[][] SIGMA = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },\n            { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 },\n            { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 },\n            { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 },\n            { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 },\n            { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },\n            { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } };\n\n    private static final int ROUNDS = 12;\n\n    private static final int BLOCK_LENGTH_BYTES = 128;\n\n    private final int digestLength;\n\n    private final int keyLength;\n\n    private final byte[] buffer;\n\n    private final long[] internalState = new long[16];\n\n    private int bufferPos = 0;\n\n    private long[] chainValue = null;\n\n    private long t0 = 0L;\n\n    private long t1 = 0L;\n\n    private long f0 = 0L;\n\n    /**\n     * Basic sized constructor - size in bytes.\n     *\n     * @param digestSize size of the digest in bytes\n     */\n    Blake2b(int digestSize)\n    {\n        if (digestSize < 1 || digestSize > 64)\n        {\n            throw new BadParametersException(\"BLAKE2b digest bytes length must be not greater than 64\");\n        }\n\n        buffer = new byte[BLOCK_LENGTH_BYTES];\n        keyLength = 0;\n        this.digestLength = digestSize;\n        init();\n    }\n\n    // initialize chainValue\n    private void init()\n    {\n        chainValue = new long[8];\n        chainValue[0] = IV[0] ^ (digestLength | ((long) keyLength << 8) | 0x1010000);\n        chainValue[1] = IV[1];\n        chainValue[2] = IV[2];\n        chainValue[3] = IV[3];\n        chainValue[4] = IV[4];\n        chainValue[5] = IV[5];\n        chainValue[6] = IV[6];\n        chainValue[7] = IV[7];\n    }\n\n    private void initializeInternalState()\n    {\n        System.arraycopy(chainValue, 0, internalState, 0, chainValue.length);\n        System.arraycopy(IV, 0, internalState, chainValue.length, 4);\n        internalState[12] = t0 ^ IV[4];\n        internalState[13] = t1 ^ IV[5];\n        internalState[14] = f0 ^ IV[6];\n        internalState[15] = IV[7];// ^ f1 with f1 = 0\n    }\n\n    void update(byte[] message)\n    {\n        if (message == null)\n        {\n            return;\n        }\n        update(message, 0, message.length);\n    }\n\n    /**\n     * update the message digest with a block of bytes.\n     *\n     * @param message the byte array containing the data.\n     * @param offset  the offset into the byte array where the data starts.\n     * @param len     the length of the data.\n     */\n    void update(byte[] message, int offset, int len)\n    {\n        int remainingLength = 0;\n\n        if (bufferPos != 0)\n        {\n            remainingLength = BLOCK_LENGTH_BYTES - bufferPos;\n            if (remainingLength < len)\n            {\n                System.arraycopy(message, offset, buffer, bufferPos, remainingLength);\n                t0 += BLOCK_LENGTH_BYTES;\n                if (t0 == 0)\n                {\n                    t1++;\n                }\n                compress(buffer, 0);\n                bufferPos = 0;\n                Arrays.fill(buffer, (byte) 0);// clear buffer\n            }\n            else\n            {\n                System.arraycopy(message, offset, buffer, bufferPos, len);\n                bufferPos += len;\n                return;\n            }\n        }\n\n        int messagePos;\n        int blockWiseLastPos = offset + len - BLOCK_LENGTH_BYTES;\n        for (messagePos = offset + remainingLength; messagePos < blockWiseLastPos; messagePos += BLOCK_LENGTH_BYTES)\n        {\n            t0 += BLOCK_LENGTH_BYTES;\n            if (t0 == 0)\n            {\n                t1++;\n            }\n            compress(message, messagePos);\n        }\n\n        // fill the buffer with left bytes, this might be a full block\n        System.arraycopy(message, messagePos, buffer, 0, offset + len - messagePos);\n        bufferPos += offset + len - messagePos;\n    }\n\n    /**\n     * close the digest, producing the final digest value. The doFinal\n     * call leaves the digest reset.\n     * Key, salt and personal string remain.\n     *\n     * @param out       the array the digest is to be copied into.\n     * @param outOffset the offset into the out array the digest is to start at.\n     */\n    void doFinal(byte[] out, int outOffset)\n    {\n\n        f0 = 0xFFFFFFFFFFFFFFFFL;\n...\n", "needle_token_start": 2387, "needle_token_end": 2420, "code_context_ntokens": 16069, "output": ["```java\npublic static int rOperation(int a, int b)\n{\n    return a << b | a >>> 32 - b;\n}\n```"]}
{"repo": "Password4j/password4j", "name": "blake2bLong", "language": "java", "path": "src/main/java/com/password4j/Argon2Function.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a cryptographic hash of a specified length, even if that length exceeds the typical output size of the underlying hash function.\n2. **Input**: This function takes a byte array and an integer specifying the desired length of the hash output.\n3. **Output**: It returns a byte array representing the cryptographic hash of the input data, with the length as specified by the input integer.\n4. **Procedure**: \n   - If the requested output length is less than or equal to the standard hash length, the function directly computes the hash of the input.\n   - For longer output lengths, the function repeatedly hashes the input, concatenating partial results until the desired length is achieved. It uses a portion of the hash result in each iteration to ensure the final output meets the specified length.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.5.0\n     */\n    public static Argon2Function getInstanceFromHash(String hashed)\n    {\n        Object[] params = decodeHash(hashed);\n        Argon2 type = Argon2.valueOf(((String) params[0]).toUpperCase());\n        int version = (int) params[1];\n        int memory = (int) params[2];\n        int iterations = (int) params[3];\n        int parallelism = (int) params[4];\n        int outputLength = ((byte[]) params[6]).length;\n        return getInstance(memory, iterations, parallelism, outputLength, type, version);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), salt, null);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(plainTextPassword, salt, null);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return hash(plainTextPassword, salt, null);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt)\n    {\n        return hash(plainTextPassword, salt, null);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt, CharSequence pepper)\n    {\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt), pepper);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt, CharSequence pepper)\n    {\n        return internalHash(plainTextPassword, salt, pepper);\n    }\n\n    private Hash internalHash(byte[] plainTextPassword, byte[] salt, CharSequence pepper)\n    {\n        long[][] blockMemory = copyOf(initialBlockMemory);\n\n        if (salt == null)\n        {\n            salt = SaltGenerator.generate();\n        }\n        initialize(plainTextPassword, salt, Utils.fromCharSequenceToBytes(pepper), null, blockMemory);\n        fillMemoryBlocks(blockMemory);\n        byte[] hash = ending(blockMemory);\n        Hash result = new Hash(this, encodeHash(hash, salt), hash, salt);\n        result.setPepper(pepper);\n        return result;\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(plainTextPassword, hashed, null, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        return check(plainTextPassword, hashed, null, null);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt, CharSequence pepper)\n    {\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        byte[] saltAsBytes = Utils.fromCharSequenceToBytes(salt);\n        byte[] hashedAsBytes = Utils.fromCharSequenceToBytes(hashed);\n        return check(plainTextPasswordAsBytes, hashedAsBytes, saltAsBytes, pepper);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt, CharSequence pepper)\n    {\n        byte[] theSalt;\n        if (salt == null || salt.length == 0)\n        {\n            Object[] params = decodeHash(Utils.fromBytesToString(hashed));\n            theSalt = (byte[]) params[5];\n        }\n        else\n        {\n            theSalt = salt;\n        }\n\n        Hash internalHash = internalHash(plainTextPassword, theSalt, pepper);\n        return slowEquals(internalHash.getResultAsBytes(), hashed);\n    }\n\n    protected static String getUID(int memory, int iterations, int parallelism, int outputLength, Argon2 type, int version)\n    {\n        return memory + \"|\" + iterations + \"|\" + parallelism + \"|\" + outputLength + \"|\" + type.ordinal() + \"|\" + version;\n    }\n\n    private static byte[] getInitialHashLong(byte[] initialHash, byte[] appendix)\n    {\n        byte[] initialHashLong = new byte[ARGON2_INITIAL_SEED_LENGTH];\n\n        System.arraycopy(initialHash, 0, initialHashLong, 0, ARGON2_INITIAL_DIGEST_LENGTH);\n        System.arraycopy(appendix, 0, initialHashLong, ARGON2_INITIAL_DIGEST_LENGTH, 4);\n\n        return initialHashLong;\n    }\n\n    private static void updateWithLength(Blake2b blake2b, byte[] input)\n    {\n        if (input != null)\n        {\n            blake2b.update(Utils.intToLittleEndianBytes(input.length));\n            blake2b.update(input);\n        }\n        else\n        {\n            blake2b.update(Utils.intToLittleEndianBytes(0));\n        }\n    }\n\n    private static int getStartingIndex(int pass, int slice)\n    {\n        if ((pass == 0) && (slice == 0))\n        {\n            return 2;\n        }\n        else\n        {\n            return 0;\n        }\n    }\n\n    private static void nextAddresses(long[] zeroBlock, long[] inputBlock, long[] addressBlock)\n    {\n        inputBlock[6]++;\n        fillBlock(zeroBlock, inputBlock, addressBlock, false);\n        fillBlock(zeroBlock, addressBlock, addressBlock, false);\n    }\n\n    private static void fillBlock(long[] x, long[] y, long[] currentBlock, boolean withXor)\n    {\n\n        long[] r = new long[ARGON2_QWORDS_IN_BLOCK];\n        long[] z = new long[ARGON2_QWORDS_IN_BLOCK];\n\n        Utils.xor(r, x, y);\n        System.arraycopy(r, 0, z, 0, z.length);\n\n        for (int i = 0; i < 8; i++)\n        {\n\n            roundFunction(z, 16 * i, 16 * i + 1, 16 * i + 2, 16 * i + 3, 16 * i + 4, 16 * i + 5, 16 * i + 6, 16 * i + 7,\n                    16 * i + 8, 16 * i + 9, 16 * i + 10, 16 * i + 11, 16 * i + 12, 16 * i + 13, 16 * i + 14, 16 * i + 15);\n        }\n\n        for (int i = 0; i < 8; i++)\n        {\n\n            roundFunction(z, 2 * i, 2 * i + 1, 2 * i + 16, 2 * i + 17, 2 * i + 32, 2 * i + 33, 2 * i + 48, 2 * i + 49, 2 * i + 64,\n                    2 * i + 65, 2 * i + 80, 2 * i + 81, 2 * i + 96, 2 * i + 97, 2 * i + 112, 2 * i + 113);\n\n        }\n\n        if (withXor)\n        {\n            Utils.xor(currentBlock, r, z, currentBlock);\n        }\n        else\n        {\n            Utils.xor(currentBlock, r, z);\n        }\n    }\n\n    private static void roundFunction(long[] block, int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7, int v8,\n            int v9, // NOSONAR\n            int v10, int v11, int v12, int v13, int v14, int v15)\n    {\n        f(block, v0, v4, v8, v12);\n        f(block, v1, v5, v9, v13);\n        f(block, v2, v6, v10, v14);\n        f(block, v3, v7, v11, v15);\n\n        f(block, v0, v5, v10, v15);\n        f(block, v1, v6, v11, v12);\n        f(block, v2, v7, v8, v13);\n        f(block, v3, v4, v9, v14);\n    }\n\n    private static void f(long[] block, int a, int b, int c, int d)\n    {\n        fBlaMka(block, a, b);\n        rotr64(block, d, a, 32);\n\n        fBlaMka(block, c, d);\n        rotr64(block, b, c, 24);\n\n        fBlaMka(block, a, b);\n        rotr64(block, d, a, 16);\n\n        fBlaMka(block, c, d);\n        rotr64(block, b, c, 63);\n    }\n\n    private static void fBlaMka(long[] block, int x, int y)\n    {\n        final long m = 0xFFFFFFFFL;\n        final long xy = (block[x] & m) * (block[y] & m);\n\n        block[x] = block[x] + block[y] + 2 * xy;\n    }\n\n    private static void rotr64(long[] block, int v, int w, long c)\n    {\n        final long temp = block[v] ^ block[w];\n        block[v] = (temp >>> c) | (temp << (64 - c));\n    }\n\n    private static Object[] decodeHash(String hash)\n    {\n        Object[] result = new Object[7];\n        String[] parts = hash.split(\"\\\\$\");\n        if (parts.length == 6)\n        {\n            result[0] = remove(parts[1], \"argon2\");\n            String[] params = parts[3].split(\",\");\n            result[1] = Integer.parseInt(remove(parts[2], \"v=\"));\n            result[2] = Integer.parseInt(remove(params[0], \"m=\"));\n            result[3] = Integer.parseInt(remove(params[1], \"t=\"));\n            result[4] = Integer.parseInt(remove(params[2], \"p=\"));\n            result[5] = Utils.decodeBase64(parts[4]);\n            result[6] = Utils.decodeBase64(parts[5]);\n            return result;\n        }\n        else\n        {\n            throw new BadParametersException(\"Invalid hashed value\");\n        }\n\n    }\n\n    protected static String toString(int memory, int iterations, int parallelism, int outputLength, Argon2 type, int version)\n    {\n        return \"m=\" + memory + \", i=\" + iterations + \", p=\" + parallelism + \", l=\" + outputLength + \", t=\" + type\n                .name() + \", v=\" + version;\n    }\n\n\n\n    /**\n     * @return the memory in bytes\n     * @since 1.5.2\n     */\n    public int getMemory()\n    {\n        return memory;\n    }\n\n    /**\n     * @return the number of iterations\n     * @since 1.5.2\n     */\n    public int getIterations()\n    {\n        return iterations;\n    }\n\n    /**\n     * @return the degree of parallelism\n     * @since 1.5.2\n     */\n    public int getParallelism()\n    {\n        return parallelism;\n    }\n\n    /**\n     * @return the length of the produced hash\n     * @since 1.5.2\n     */\n    public int getOutputLength()\n    {\n        return outputLength;\n    }\n\n    /**\n     * @return the Argon2 variant (i, d, id)\n     * @since 1.5.2\n     */\n    public Argon2 getVariant()\n    {\n        return variant;\n    }\n\n    /**\n     * @return the version of the algorithm\n     * @since 1.5.2\n     */\n    public int getVersion()\n    {\n        return version;\n    }\n\n    private void initialize(byte[] plainTextPassword, byte[] salt, byte[] secret, byte[] additional, long[][] blockMemory)\n    {\n        Blake2b blake2b = new Blake2b(ARGON2_INITIAL_DIGEST_LENGTH);\n\n        blake2b.update(Utils.intToLittleEndianBytes(parallelism));\n        blake2b.update(Utils.intToLittleEndianBytes(outputLength));\n        blake2b.update(Utils.intToLittleEndianBytes(memory));\n        blake2b.update(Utils.intToLittleEndianBytes(iterations));\n        blake2b.update(Utils.intToLittleEndianBytes(version));\n        blake2b.update(Utils.intToLittleEndianBytes(variant.ordinal()));\n\n        updateWithLength(blake2b, plainTextPassword);\n\n        updateWithLength(blake2b, salt);\n\n        updateWithLength(blake2b, secret);\n\n        updateWithLength(blake2b, additional);\n\n        byte[] initialHash = new byte[64];\n        blake2b.doFinal(initialHash, 0);\n\n        final byte[] zeroBytes = { 0, 0, 0, 0 };\n        final byte[] oneBytes = { 1, 0, 0, 0 };\n\n        byte[] initialHashWithZeros = getInitialHashLong(initialHash, zeroBytes);\n        byte[] initialHashWithOnes = getInitialHashLong(initialHash, oneBytes);\n\n        for (int i = 0; i < parallelism; i++)\n        {\n\n            byte[] iBytes = Utils.intToLittleEndianBytes(i);\n\n            System.arraycopy(iBytes, 0, initialHashWithZeros, ARGON2_INITIAL_DIGEST_LENGTH + 4, 4);\n            System.arraycopy(iBytes, 0, initialHashWithOnes, ARGON2_INITIAL_DIGEST_LENGTH + 4, 4);\n\n            byte[] blockHashBytes = blake2bLong(initialHashWithZeros, ARGON2_BLOCK_SIZE);\n            blockMemory[i * laneLength] = Utils.fromBytesToLongs(blockHashBytes);\n\n            blockHashBytes = blake2bLong(initialHashWithOnes, ARGON2_BLOCK_SIZE);\n            blockMemory[i * laneLength + 1] = Utils.fromBytesToLongs(blockHashBytes);\n        }\n\n    }\n\n    \nprivate byte[] blake2bLong(byte[] input, int outputLength)\n    {\n\n        byte[] result = new byte[outputLength];\n        byte[] outlenBytes = Utils.intToLittleEndianBytes(outputLength);\n\n        int blake2bLength = 64;\n\n        if (outputLength <= blake2bLength)\n        {\n            result = simpleBlake2b(input, outlenBytes, outputLength);\n        }\n        else\n        {\n            byte[] outBuffer;\n\n            outBuffer = simpleBlake2b(input, outlenBytes, blake2bLength);\n            System.arraycopy(outBuffer, 0, result, 0, blake2bLength / 2);\n\n            int r = (outputLength / 32) + (outputLength % 32 == 0 ? 0 : 1) - 2;\n\n            int position = blake2bLength / 2;\n            for (int i = 2; i <= r; i++, position += blake2bLength / 2)\n            {\n\n                outBuffer = simpleBlake2b(outBuffer, null, blake2bLength);\n                System.arraycopy(outBuffer, 0, result, position, blake2bLength / 2);\n            }\n\n            int lastLength = outputLength - 32 * r;\n\n            outBuffer = simpleBlake2b(outBuffer, null, lastLength);\n            System.arraycopy(outBuffer, 0, result, position, lastLength);\n        }\n\n        return result;\n    }\n\n    private byte[] simpleBlake2b(byte[] input, byte[] outlenBytes, int outputLength)\n    {\n        Blake2b blake2b = new Blake2b(outputLength);\n\n        if (outlenBytes != null)\n            blake2b.update(outlenBytes);\n        blake2b.update(input);\n\n        byte[] buff = new byte[outputLength];\n        blake2b.doFinal(buff, 0);\n        return buff;\n    }\n\n    private void fillMemoryBlocks(long[][] blockMemory)\n    {\n        if (parallelism == 1)\n        {\n            fillMemoryBlockSingleThreaded(blockMemory);\n        }\n        else\n        {\n            fillMemoryBlockMultiThreaded(blockMemory);\n        }\n    }\n\n    private void fillMemoryBlockSingleThreaded(long[][] blockMemory)\n    {\n        for (int pass = 0; pass < iterations; pass++)\n        {\n            for (int slice = 0; slice < ARGON2_SYNC_POINTS; slice++)\n            {\n                fillSegment(pass, 0, slice, blockMemory);\n            }\n        }\n    }\n\n    private void fillMemoryBlockMultiThreaded(long[][] blockMemory)\n    {\n        List<Future<?>> futures = new ArrayList<>();\n\n        for (int i = 0; i < iterations; i++)\n        {\n            for (int j = 0; j < ARGON2_SYNC_POINTS; j++)\n            {\n                for (int k = 0; k < parallelism; k++)\n                {\n                    int pass = i;\n                    int lane = k;\n                    int slice = j;\n\n                    Future<?> future = service.submit(() -> fillSegment(pass, lane, slice, blockMemory));\n\n                    futures.add(future);\n                }\n\n                try\n                {\n                    for (Future<?> f : futures)\n                    {\n                        f.get();\n                    }\n                }\n                catch (InterruptedException | ExecutionException e)\n                {\n                    clear(blockMemory);\n                    Thread.currentThread().interrupt();\n                }\n            }\n        }\n    }\n\n    private void fillSegment(int pass, int lane, int slice, long[][] blockMemory)\n    {\n\n        long[] addressBlock = null;\n        long[] inputBlock = null;\n        long[] zeroBlock = null;\n\n        boolean dataIndependentAddressing = isDataIndependentAddressing(pass, slice);\n        int startingIndex = getStartingIndex(pass, slice);\n        int currentOffset = lane * laneLength + slice * segmentLength + startingIndex;\n        int prevOffset = getPrevOffset(currentOffset);\n\n        if (dataIndependentAddressing)\n        {\n            addressBlock = new long[ARGON2_QWORDS_IN_BLOCK];\n            zeroBlock = new long[ARGON2_QWORDS_IN_BLOCK];\n            inputBlock = new long[ARGON2_QWORDS_IN_BLOCK];\n\n            initAddressBlocks(pass, lane, slice, zeroBlock, inputBlock, addressBlock, blockMemory);\n        }\n\n        for (int i = startingIndex; i < segmentLength; i++, currentOffset++, prevOffset++)\n        {\n            prevOffset = rotatePrevOffset(currentOffset, prevOffset);\n\n            long pseudoRandom = getPseudoRandom(i, addressBlock, inputBlock, zeroBlock, prevOffset, dataIndependentAddressing,\n                    blockMemory);\n            int refLane = getRefLane(pass, lane, slice, pseudoRandom);\n            int refColumn = getRefColumn(pass, slice, i, pseudoRandom, refLane == lane);\n\n            long[] prevBlock = blockMemory[prevOffset];\n            long[] refBlock = blockMemory[((laneLength) * refLane + refColumn)];\n            long[] currentBlock = blockMemory[currentOffset];\n\n            boolean withXor = isWithXor(pass);\n            fillBlock(prevBlock, refBlock, currentBlock, withXor);\n        }\n    }\n\n    private boolean isDataIndependentAddressing(int pass, int slice)\n    {\n        return (variant == Argon2.I) || (variant == Argon2.ID && (pass == 0) && (slice < ARGON2_SYNC_POINTS / 2));\n    }\n\n    private int getPrevOffset(int currentOffset)\n    {\n        if (currentOffset % laneLength == 0)\n        {\n\n            return currentOffset + laneLength - 1;\n        }\n        else\n        {\n\n            return currentOffset - 1;\n        }\n    }\n\n    private int rotatePrevOffset(int currentOffset, int prevOffset)\n    {\n        if (currentOffset % laneLength == 1)\n        {\n            prevOffset = currentOffset - 1;\n        }\n        return prevOffset;\n    }\n\n    private long getPseudoRandom(int index, long[] addressBlock, long[] inputBlock, long[] zeroBlock, int prevOffset,\n            boolean dataIndependentAddressing, long[][] blockMemory)\n    {\n        if (dataIndependentAddressing)\n        {\n            if (index % ARGON2_ADDRESSES_IN_BLOCK == 0)\n            {\n                nextAddresses(zeroBlock, inputBlock, addressBlock);\n            }\n            return addressBlock[index % ARGON2_ADDRESSES_IN_BLOCK];\n        }\n        else\n        {\n            return blockMemory[prevOffset][0];\n        }\n    }\n\n    private int getRefLane(int pass, int lane, int slice, long pseudoRandom)\n    {\n        int refLane = (int) ((pseudoRandom >>> 32) % parallelism);\n\n        if (pass == 0 && slice == 0)\n        {\n            refLane = lane;\n        }\n        return refLane;\n    }\n\n    private void initAddressBlocks(int pass, int lane, int slice, long[] zeroBlock, long[] inputBlock, long[] addressBlock,\n            long[][] blockMemory)\n    {\n        inputBlock[0] = Utils.intToLong(pass);\n        inputBlock[1] = Utils.intToLong(lane);\n        inputBlock[2] = Utils.intToLong(slice);\n        inputBlock[3] = Utils.intToLong(blockMemory.length);\n        inputBlock[4] = Utils.intToLong(iterations);\n        inputBlock[5] = Utils.intToLong(variant.ordinal());\n\n        if (pass == 0 && slice == 0)\n        {\n\n            nextAddresses(zeroBlock, inputBlock, addressBlock);\n        }\n    }\n\n    private int getRefColumn(int pass, int slice, int index, long pseudoRandom, boolean sameLane)\n    {\n\n        int referenceAreaSize;\n        int startPosition;\n\n        if (pass == 0)\n        {\n            startPosition = 0;\n\n            if (sameLane)\n            {\n                referenceAreaSize = slice * segmentLength + index - 1;\n            }\n            else\n            {\n                referenceAreaSize = slice * segmentLength + ((index == 0) ? (-1) : 0);\n            }\n\n        }\n        else\n        {\n            startPosition = ((slice + 1) * segmentLength) % laneLength;\n\n            if (sameLane)\n            {\n                referenceAreaSize = laneLength - segmentLength + index - 1;\n            }\n            else\n            {\n                referenceAreaSize = laneLength - segmentLength + ((index == 0) ? (-1) : 0);\n            }\n        }\n\n        long relativePosition = pseudoRandom & 0xFFFFFFFFL;\n\n        relativePosition = (relativePosition * relativePosition) >>> 32;\n        relativePosition = referenceAreaSize - 1 - (referenceAreaSize * relativePosition >>> 32);\n\n        return (int) (startPosition + relativePosition) % laneLength;\n    }\n\n    private boolean isWithXor(int pass)\n    {\n        return !(pass == 0 || version == ARGON2_VERSION_10);\n    }\n\n    private byte[] ending(long[][] blockMemory)\n    {\n\n        long[] finalBlock = blockMemory[laneLength - 1];\n\n        for (int i = 1; i < parallelism; i++)\n        {\n            int lastBlockInLane = i * laneLength + (laneLength - 1);\n            Utils.xor(finalBlock, blockMemory[lastBlockInLane]);\n        }\n\n        byte[] finalBlockBytes = new byte[ARGON2_BLOCK_SIZE];\n\n        for (int i = 0; i < finalBlock.length; i++)\n        {\n            byte[] bytes = Utils.longToLittleEndianBytes(finalBlock[i]);\n            System.arraycopy(bytes, 0, finalBlockBytes, i * bytes.length, bytes.length);\n        }\n\n        byte[] finalResult = blake2bLong(finalBlockBytes, outputLength);\n\n        clear(blockMemory);\n\n        return finalResult;\n    }\n\n    private void clear(long[][] blockMemory)\n    {\n        for (long[] block : blockMemory)\n        {\n            Arrays.fill(block, 0);\n        }\n    }\n\n    private long[][] copyOf(long[][] old)\n    {\n        long[][] current = new long[old.length][ARGON2_QWORDS_IN_BLOCK];\n        for (int i = 0; i < old.length; i++)\n        {\n            System.arraycopy(current[i], 0, old[i], 0, ARGON2_QWORDS_IN_BLOCK);\n        }\n        return current;\n    }\n\n    private static String remove(String source, String remove)\n    {\n        return source.substring(remove.length());\n    }\n\n    private String encodeHash(byte[] hash, byte[] salt)\n    {\n        return \"$argon2\" + variant.name()\n                .toLowerCase() + \"$v=\" + version + \"$m=\" + memory + \",t=\" + iterations + \",p=\" + parallelism + \"$\"\n                + Utils.encodeBase64(salt, false) + \"$\"\n                + Utils.encodeBase64(hash, false);\n    }\n\n    @Override\n    public boolean equals(Object o)\n    {\n        if (this == o)\n            return true;\n        if (!(o instanceof Argon2Function))\n            return false;\n        Argon2Function other = (Argon2Function) o;\n        return iterations == other.iterations //\n                && memory == other.memory //\n                && parallelism == other.parallelism //\n                && outputLength == other.outputLength //\n                && version == other.version //\n                && variant == other.variant;\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(iterations, memory, parallelism, outputLength, variant, version);\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '[' + toString(memory, iterations, parallelism, outputLength, variant, version) + ']';\n    }\n}\n\n// Path: src/main/java/com/password4j/BalloonHashingFunction.java\n/*\n *  (C) Copyright 2023 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport java.math.BigInteger;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.*;\n\n/**\n * Class containing the implementation of Balloon hashing function and its parameters.\n *\n * @author David Bertoldi\n * @since 1.8.0\n */\npublic class BalloonHashingFunction extends AbstractHashingFunction\n{\n\n    private static final Map<String, BalloonHashingFunction> INSTANCES = new ConcurrentHashMap<>();\n    private static final int DEFAULT_DELTA = 3;\n\n    private final String algorithm;\n\n    private ExecutorService service;\n\n    private final int spaceCost;\n\n    private final int timeCost;\n\n    private final int parallelism;\n\n    private final int delta;\n\n    BalloonHashingFunction(String algorithm, int spaceCost, int timeCost,  int parallelism, int delta)\n    {\n        this.algorithm = algorithm;\n        this.spaceCost = spaceCost;\n        this.timeCost = timeCost;\n        this.parallelism = parallelism;\n        this.delta = delta;\n        if (parallelism > 1)\n        {\n            this.service = Executors.newFixedThreadPool(Utils.AVAILABLE_PROCESSORS);\n        }\n\n    }\n\n\n    public static BalloonHashingFunction getInstance(String algorithm, int spaceCost, int timeCost,  int parallelism, int delta)\n    {\n        String key = getUID(algorithm, spaceCost, timeCost, parallelism, delta);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            BalloonHashingFunction function = new BalloonHashingFunction(algorithm, spaceCost, timeCost, parallelism, delta);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    public static BalloonHashingFunction getInstance(String algorithm, int spaceCost, int timeCost,  int parallelism)\n    {\n        return getInstance(algorithm, spaceCost, timeCost, parallelism, DEFAULT_DELTA);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        return null;\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword)\n    {\n        return null;\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return hash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt)\n    {\n        return internalHash(plainTextPassword, salt);\n    }\n\n    protected Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n\n        byte[] output;\n\n        if (parallelism == 1)\n        {\n            byte[] parallelSalt = Utils.append(salt, Utils.longToLittleEndian((1)));\n            MessageDigest messageDigest = getMessageDigest();\n            output = balloon(messageDigest, plainTextPassword, parallelSalt);\n            output = hashFunc(messageDigest, plainTextPassword, salt, output);\n        }\n        else if (parallelism > 1)\n        {\n\n            List<Future<?>> futures = new ArrayList<>();\n\n            for (int i = 0; i < parallelism; i++)\n            {\n                byte[] parallelSalt = Utils.append(salt, Utils.longToLittleEndian((i + 1)));\n                Future<byte[]> future = service.submit(() -> balloon(getMessageDigest(), plainTextPassword, parallelSalt));\n\n                futures.add(future);\n            }\n\n            MessageDigest messageDigest = getMessageDigest();\n            output = new byte[messageDigest.getDigestLength()];\n\n            try\n            {\n                byte[] tmp;\n                output = (byte[]) futures.get(0).get();\n                for (int f = 1; f < futures.size(); f++)\n                {\n                    tmp = ((byte[]) futures.get(f).get());\n\n                    for (int i = 0; i < output.length; i++)\n                    {\n                        output[i] ^= tmp[i];\n                    }\n\n                }\n            }\n            catch (InterruptedException | ExecutionException e)\n            {\n                Thread.currentThread().interrupt();\n            }\n\n\n            output = hashFunc(messageDigest, plainTextPassword, salt, output);\n        }\n        else\n        {\n            output =  balloon(getMessageDigest(), plainTextPassword, salt);\n        }\n\n        return new Hash(this, Utils.toHex(output), output, salt);\n    }\n\n    protected MessageDigest getMessageDigest()\n    {\n        try\n        {\n            return MessageDigest.getInstance(algorithm);\n        }\n        catch (NoSuchAlgorithmException nsae)\n        {\n            throw new UnsupportedOperationException(\"`\" + algorithm + \"` is not supported by your system.\", nsae);\n        }\n    }\n\n    private byte[] balloon(MessageDigest messageDigest, byte[] plainTextPassword, byte[] salt)\n    {\n        List<byte[]> buffer = new ArrayList<>();\n        buffer.add(hashFunc(messageDigest, 0, plainTextPassword, salt));\n\n        int cnt = 1;\n\n        cnt = expand(messageDigest, buffer, cnt);\n        mix(messageDigest, buffer, cnt, salt);\n        return extract(buffer);\n    }\n\n\n\n    private int expand(MessageDigest messageDigest, List<byte[]> buffer, int cnt)\n    {\n        int newCnt = cnt;\n        for (int i = 1; i < spaceCost; i++)\n        {\n            buffer.add(hashFunc(messageDigest, newCnt, buffer.get(i - 1)));\n            newCnt += 1;\n        }\n        return newCnt;\n    }\n\n    private void mix(MessageDigest messageDigest, List<byte[]> buffer, int cnt, byte[] salt)\n    {\n        int newCnt = cnt;\n        for (int t = 0; t < timeCost; t++)\n        {\n            for (int s = 0; s < spaceCost; s++)\n            {\n                buffer.set(s, hashFunc(messageDigest, newCnt, get(buffer, s - 1), get(buffer, s)));\n                newCnt += 1;\n\n                for (int d = 0; d < delta; d++)\n                {\n                    byte[] indexBlock = hashFunc(messageDigest, t, s, d);\n                    int other = Utils.bytesToInt(hashFunc(messageDigest, newCnt, salt, indexBlock)).mod(BigInteger.valueOf(spaceCost)).intValue();\n                    newCnt += 1;\n                    buffer.set(s, hashFunc(messageDigest, newCnt, buffer.get(s), get(buffer, other)));\n                    newCnt += 1;\n                }\n            }\n        }\n    }\n\n    private byte[] extract(List<byte[]> buffer)\n    {\n        return buffer.get(buffer.size() - 1);\n    }\n\n    private byte[] get(List<byte[]> buffer, int position)\n    {\n        if (position < 0)\n        {\n            return buffer.get(buffer.size() + position);\n        }\n        return buffer.get(position);\n    }\n\n\n    private byte[] hashFunc(MessageDigest md, Object... args)\n    {\n        byte[] t = new byte[0];\n\n        for (Object arg : args)\n        {\n            if (arg instanceof Integer)\n            {\n                t = Utils.append(t, Utils.intToLittleEndianBytes((Integer) arg, 8));\n            }\n            else if (arg instanceof CharSequence)\n            {\n                t = Utils.append(t, Utils.fromCharSequenceToBytes((CharSequence) arg));\n            }\n            else if (arg instanceof byte[])\n            {\n                t = Utils.append(t, (byte[]) arg);\n            }\n        }\n\n        return md.digest(t);\n    }\n\n\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(plainTextPassword, hashed, null);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        Hash hash = internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n        return slowEquals(hash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        return check(plainTextPassword, new byte[0], hashed, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        Hash hash = internalHash(plainTextPassword, salt);\n        return slowEquals(hash.getResultAsBytes(), hashed);\n    }\n\n    private static String getUID(String algorithm, int spaceCost, int timeCost,  int parallelism, int delta)\n    {\n        return algorithm + '|' + spaceCost + '|' + timeCost + '|' + parallelism + '|' + delta;\n    }\n\n    @Override\n    public boolean equals(Object o)\n    {\n        if (this == o)\n            return true;\n        if (!(o instanceof BalloonHashingFunction))\n            return false;\n        BalloonHashingFunction other = (BalloonHashingFunction) o;\n        return algorithm .equals(other.algorithm) //\n                && spaceCost == other.spaceCost //\n                && timeCost == other.timeCost //\n                && parallelism == other.parallelism //\n                && delta == other.delta;\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(algorithm, spaceCost, timeCost, parallelism, delta);\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '[' + toString(algorithm, spaceCost, timeCost, parallelism, delta) + ']';\n    }\n\n    protected static String toString(String algorithm, int spaceCost, int timeCost, int parallelism, int delta)\n    {\n        return \"a=\" + algorithm + \", s=\" + spaceCost + \", t=\" + timeCost + \", p=\" + parallelism + \", d=\" + delta;\n    }\n}\n\n// Path: src/main/java/com/password4j/types/Bcrypt.java\n/*\n *  (C) Copyright 2021 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j.types;\n\npublic enum Bcrypt\n{\n    A, B, X, Y;\n\n    public static Bcrypt valueOf(char minor)\n    {\n        for (Bcrypt type : Bcrypt.values())\n        {\n            if (type.minor() == minor)\n            {\n                return type;\n            }\n        }\n        return null;\n    }\n\n    public char minor()\n    {\n        return name().toLowerCase().charAt(0);\n    }\n}\n\n// Path: src/main/java/com/password4j/BcryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Bcrypt;\n\nimport java.security.MessageDigest;\nimport java.util.Arrays;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of bcrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Bcrypt\">bcrypt</a>\n * @since 0.1.0\n */\npublic class BcryptFunction extends AbstractHashingFunction\n{\n\n    private static final ConcurrentMap<String, BcryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final int BCRYPT_SALT_LEN = 16;\n\n    private static final int BLOWFISH_NUM_ROUNDS = 16;\n\n    private static final int[] P_ORIG = { //\n            0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6,\n            0x38d01377, 0xbe5466cf, 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b };\n\n    private static final int[] S_ORIG = { //\n            0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, 0x24a19947,\n            0xb3916cf7, 0x0801f2e2, 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658,\n            0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, 0xc5d1b023, 0x286085f0, 0xca417918,\n            0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60,\n            0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, 0xa15486af,\n            0x7c72e993, 0xb3ee1411, 0x636fbc2a, 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c,\n            0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60,\n            0x5dec8032, 0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239,\n            0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0, 0x6a51a0d2,\n            0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176,\n            0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, 0xe06f75d8, 0x85c12073, 0x401a449f,\n            0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b,\n            0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6,\n            0x409f60c4, 0x5e5c9ec2, 0x196a2463, 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c,\n            0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, 0xc0cba857, 0x45c8740f, 0xd20b5f39,\n            0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8,\n            0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, 0x9e5c57bb,\n            0xca6f8ca0, 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8,\n            0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, 0x9a53e479, 0xb6f84565, 0xd28e49bc,\n            0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4,\n            0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb,\n            0xf2122b64, 0x8888b812, 0x900df01c, 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777,\n            0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81,\n            0xd2ada8d9, 0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf,\n            0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, 0x2464369b,\n            0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9,\n            0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, 0xd60f573f, 0xbc9bc6e4, 0x2b60a476,\n            0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664,\n            0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0,\n            0x49a7df7d, 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, 0x193602a5, 0x75094c29,\n            0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6,\n            0xf0255dc1, 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1,\n            0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37,\n            0xf0500c0d, 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, 0xd19113f9, 0x7ca92ff6,\n            0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e,\n            0xa4751e41, 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf,\n            0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f,\n            0x2e6b7124, 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, 0xec7aec3a, 0xdb851dfa,\n            0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002,\n            0x133ae4dd, 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509,\n            0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c,\n            0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, 0xc6150eba, 0x94e2ea78,\n            0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e,\n            0xeac31f66, 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802,\n            0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830,\n            0xeb61bd96, 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, 0xeecc86bc, 0x60622ca7,\n            0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5,\n            0xc021b8f7, 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281,\n            0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb,\n            0x532e3054, 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, 0x5d4a14d9, 0xe864b7e3,\n            0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d,\n            0x41cd2105, 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3,\n...\n", "needle_token_start": 3830, "needle_token_end": 4207, "code_context_ntokens": 16114, "output": ["```java\nprivate byte[] blake2bLong(byte[] input, int outputLength)\n{\n    byte[] result = new byte[outputLength];\n    byte[] outlenBytes = Utils.intToLittleEndianBytes(outputLength);\n\n    int blake2bLength = 64;\n\n    if (outputLength <= blake2bLength)\n    {\n        result = simpleBlake2b(input, outlenBytes, outputLength);\n    }\n    else\n    {\n        byte[] outBuffer;\n\n        outBuffer = simpleBlake2b(input, outlenBytes, blake2bLength);\n        System.arraycopy(outBuffer, 0, result, 0, blake2bLength / 2);\n\n        int r = (outputLength / 32) + (outputLength % 32 == 0 ? 0 : 1) - 2;\n\n        int position = blake2bLength / 2;\n        for (int i = 2; i <= r; i++, position += blake2bLength / 2)\n        {\n\n            outBuffer = simpleBlake2b(outBuffer, null, blake2bLength);\n            System.arraycopy(outBuffer, 0, result, position, blake2bLength / 2);\n        }\n\n        int lastLength = outputLength - 32 * r;\n\n        outBuffer = simpleBlake2b(outBuffer, null, lastLength);\n        System.arraycopy(outBuffer, 0, result, position, lastLength);\n    }\n\n    return result;\n}\n```"]}
{"repo": "Password4j/password4j", "name": "balloon", "language": "java", "path": "src/main/java/com/password4j/BalloonHashingFunction.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to perform a memory-hard hashing operation that combines input data with a salt and processes it through multiple hashing stages to enhance security, particularly against brute-force attacks.\n2. **Input**: The function accepts a byte array representing the data to be hashed and another byte array representing the salt.\n3. **Output**: It returns a byte array which is the final hash result after processing the input data and salt through multiple hashing stages.\n4. **Procedure**: The function begins by initializing a buffer with the initial hash of the input data and salt. It then expands this buffer by repeatedly hashing its contents, followed by a mixing phase where the buffer's contents are further combined and hashed with the salt to ensure thorough diffusion of the input data characteristics. Finally, the function extracts the resultant hash from the buffer to produce the output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/Argon2Function.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Argon2;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.*;\n\n\n/**\n * Class containing the implementation of Argon2 function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Argon2\">Argon2</a>\n * @since 1.5.0\n */\npublic class Argon2Function extends AbstractHashingFunction\n{\n    public static final int ARGON2_VERSION_10 = 0x10;\n\n    public static final int ARGON2_VERSION_13 = 0x13;\n\n    public static final int ARGON2_INITIAL_DIGEST_LENGTH = 64;\n\n    public static final int ARGON2_ADDRESSES_IN_BLOCK = 128;\n\n    private static final ConcurrentMap<String, Argon2Function> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final int ARGON2_SYNC_POINTS = 4;\n\n    private static final int ARGON2_INITIAL_SEED_LENGTH = 72;\n\n    private static final int ARGON2_BLOCK_SIZE = 1024;\n\n    public static final int ARGON2_QWORDS_IN_BLOCK = ARGON2_BLOCK_SIZE / 8;\n\n    private ExecutorService service;\n\n    private final int iterations;\n\n    private final int memory;\n\n    private final long[][] initialBlockMemory;\n\n    private final int parallelism;\n\n    private final int outputLength;\n\n    private final int segmentLength;\n\n    private final Argon2 variant;\n\n    private final int version;\n\n    private final int laneLength;\n\n    Argon2Function(int memory, int iterations, int parallelism, int outputLength, Argon2 variant, int version)\n    {\n        this.variant = variant;\n        this.iterations = iterations;\n        this.memory = memory;\n        this.parallelism = parallelism;\n        this.outputLength = outputLength;\n        this.version = version;\n\n        int memoryBlocks = this.memory;\n\n        if (this.memory < 2 * ARGON2_SYNC_POINTS * parallelism)\n        {\n            memoryBlocks = 2 * ARGON2_SYNC_POINTS * parallelism;\n        }\n\n        segmentLength = memoryBlocks / (parallelism * ARGON2_SYNC_POINTS);\n        this.laneLength = segmentLength * ARGON2_SYNC_POINTS;\n\n        memoryBlocks = segmentLength * (parallelism * ARGON2_SYNC_POINTS);\n\n        initialBlockMemory = new long[memoryBlocks][ARGON2_QWORDS_IN_BLOCK];\n        for (int i = 0; i < memoryBlocks; i++)\n        {\n            initialBlockMemory[i] = new long[ARGON2_QWORDS_IN_BLOCK];\n        }\n\n        if (parallelism >= 1)\n        {\n            service = Executors.newFixedThreadPool(Utils.AVAILABLE_PROCESSORS);\n        }\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * memory (KiB), number of iterations, parallelism, length og the output and type.\n     *\n     * @param memory       memory (KiB)\n     * @param iterations   number of iterations\n     * @param parallelism  level of parallelism\n     * @param outputLength length of the final hash\n     * @param type         argon2 type (i, d or id)\n     * @return a singleton instance\n     * @since 1.5.0\n     */\n    public static Argon2Function getInstance(int memory, int iterations, int parallelism, int outputLength, Argon2 type)\n    {\n        return getInstance(memory, iterations, parallelism, outputLength, type, ARGON2_VERSION_13);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * logarithmic memory, number of iterations, parallelism, lenght og the output, type and version.\n     *\n     * @param memory       logarithmic memory\n     * @param iterations   number of iterations\n     * @param parallelism  level of parallelism\n     * @param outputLength length of the final hash\n     * @param type         argon2 type (i, d or id)\n     * @param version      version of the algorithm (16 or 19)\n     * @return a singleton instance\n     * @since 1.5.0\n     */\n    public static Argon2Function getInstance(int memory, int iterations, int parallelism, int outputLength, Argon2 type,\n            int version)\n    {\n        String key = getUID(memory, iterations, parallelism, outputLength, type, version);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            Argon2Function function = new Argon2Function(memory, iterations, parallelism, outputLength, type, version);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.5.0\n     */\n    public static Argon2Function getInstanceFromHash(String hashed)\n    {\n        Object[] params = decodeHash(hashed);\n        Argon2 type = Argon2.valueOf(((String) params[0]).toUpperCase());\n        int version = (int) params[1];\n        int memory = (int) params[2];\n        int iterations = (int) params[3];\n        int parallelism = (int) params[4];\n        int outputLength = ((byte[]) params[6]).length;\n        return getInstance(memory, iterations, parallelism, outputLength, type, version);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), salt, null);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(plainTextPassword, salt, null);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return hash(plainTextPassword, salt, null);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt)\n    {\n        return hash(plainTextPassword, salt, null);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt, CharSequence pepper)\n    {\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt), pepper);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt, CharSequence pepper)\n    {\n        return internalHash(plainTextPassword, salt, pepper);\n    }\n\n    private Hash internalHash(byte[] plainTextPassword, byte[] salt, CharSequence pepper)\n    {\n        long[][] blockMemory = copyOf(initialBlockMemory);\n\n        if (salt == null)\n        {\n            salt = SaltGenerator.generate();\n        }\n        initialize(plainTextPassword, salt, Utils.fromCharSequenceToBytes(pepper), null, blockMemory);\n        fillMemoryBlocks(blockMemory);\n        byte[] hash = ending(blockMemory);\n        Hash result = new Hash(this, encodeHash(hash, salt), hash, salt);\n        result.setPepper(pepper);\n        return result;\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(plainTextPassword, hashed, null, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        return check(plainTextPassword, hashed, null, null);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt, CharSequence pepper)\n    {\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        byte[] saltAsBytes = Utils.fromCharSequenceToBytes(salt);\n        byte[] hashedAsBytes = Utils.fromCharSequenceToBytes(hashed);\n        return check(plainTextPasswordAsBytes, hashedAsBytes, saltAsBytes, pepper);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt, CharSequence pepper)\n    {\n        byte[] theSalt;\n        if (salt == null || salt.length == 0)\n        {\n            Object[] params = decodeHash(Utils.fromBytesToString(hashed));\n            theSalt = (byte[]) params[5];\n        }\n        else\n        {\n            theSalt = salt;\n        }\n\n        Hash internalHash = internalHash(plainTextPassword, theSalt, pepper);\n        return slowEquals(internalHash.getResultAsBytes(), hashed);\n    }\n\n    protected static String getUID(int memory, int iterations, int parallelism, int outputLength, Argon2 type, int version)\n    {\n        return memory + \"|\" + iterations + \"|\" + parallelism + \"|\" + outputLength + \"|\" + type.ordinal() + \"|\" + version;\n    }\n\n    private static byte[] getInitialHashLong(byte[] initialHash, byte[] appendix)\n    {\n        byte[] initialHashLong = new byte[ARGON2_INITIAL_SEED_LENGTH];\n\n        System.arraycopy(initialHash, 0, initialHashLong, 0, ARGON2_INITIAL_DIGEST_LENGTH);\n        System.arraycopy(appendix, 0, initialHashLong, ARGON2_INITIAL_DIGEST_LENGTH, 4);\n\n        return initialHashLong;\n    }\n\n    private static void updateWithLength(Blake2b blake2b, byte[] input)\n    {\n        if (input != null)\n        {\n            blake2b.update(Utils.intToLittleEndianBytes(input.length));\n            blake2b.update(input);\n        }\n        else\n        {\n            blake2b.update(Utils.intToLittleEndianBytes(0));\n        }\n    }\n\n    private static int getStartingIndex(int pass, int slice)\n    {\n        if ((pass == 0) && (slice == 0))\n        {\n            return 2;\n        }\n        else\n        {\n            return 0;\n        }\n    }\n\n    private static void nextAddresses(long[] zeroBlock, long[] inputBlock, long[] addressBlock)\n    {\n        inputBlock[6]++;\n        fillBlock(zeroBlock, inputBlock, addressBlock, false);\n        fillBlock(zeroBlock, addressBlock, addressBlock, false);\n    }\n\n    private static void fillBlock(long[] x, long[] y, long[] currentBlock, boolean withXor)\n    {\n\n        long[] r = new long[ARGON2_QWORDS_IN_BLOCK];\n        long[] z = new long[ARGON2_QWORDS_IN_BLOCK];\n\n        Utils.xor(r, x, y);\n        System.arraycopy(r, 0, z, 0, z.length);\n\n        for (int i = 0; i < 8; i++)\n        {\n\n            roundFunction(z, 16 * i, 16 * i + 1, 16 * i + 2, 16 * i + 3, 16 * i + 4, 16 * i + 5, 16 * i + 6, 16 * i + 7,\n                    16 * i + 8, 16 * i + 9, 16 * i + 10, 16 * i + 11, 16 * i + 12, 16 * i + 13, 16 * i + 14, 16 * i + 15);\n        }\n\n        for (int i = 0; i < 8; i++)\n        {\n\n            roundFunction(z, 2 * i, 2 * i + 1, 2 * i + 16, 2 * i + 17, 2 * i + 32, 2 * i + 33, 2 * i + 48, 2 * i + 49, 2 * i + 64,\n                    2 * i + 65, 2 * i + 80, 2 * i + 81, 2 * i + 96, 2 * i + 97, 2 * i + 112, 2 * i + 113);\n\n        }\n\n        if (withXor)\n        {\n            Utils.xor(currentBlock, r, z, currentBlock);\n        }\n        else\n        {\n            Utils.xor(currentBlock, r, z);\n        }\n    }\n\n    private static void roundFunction(long[] block, int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7, int v8,\n            int v9, // NOSONAR\n            int v10, int v11, int v12, int v13, int v14, int v15)\n    {\n        f(block, v0, v4, v8, v12);\n        f(block, v1, v5, v9, v13);\n        f(block, v2, v6, v10, v14);\n        f(block, v3, v7, v11, v15);\n\n        f(block, v0, v5, v10, v15);\n        f(block, v1, v6, v11, v12);\n        f(block, v2, v7, v8, v13);\n        f(block, v3, v4, v9, v14);\n    }\n\n    private static void f(long[] block, int a, int b, int c, int d)\n    {\n        fBlaMka(block, a, b);\n        rotr64(block, d, a, 32);\n\n        fBlaMka(block, c, d);\n        rotr64(block, b, c, 24);\n\n        fBlaMka(block, a, b);\n        rotr64(block, d, a, 16);\n\n        fBlaMka(block, c, d);\n        rotr64(block, b, c, 63);\n    }\n\n    private static void fBlaMka(long[] block, int x, int y)\n    {\n        final long m = 0xFFFFFFFFL;\n        final long xy = (block[x] & m) * (block[y] & m);\n\n...\n// Path: src/main/java/com/password4j/BalloonHashingFunction.java\n/*\n *  (C) Copyright 2023 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport java.math.BigInteger;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.*;\n\n/**\n * Class containing the implementation of Balloon hashing function and its parameters.\n *\n * @author David Bertoldi\n * @since 1.8.0\n */\npublic class BalloonHashingFunction extends AbstractHashingFunction\n{\n\n    private static final Map<String, BalloonHashingFunction> INSTANCES = new ConcurrentHashMap<>();\n    private static final int DEFAULT_DELTA = 3;\n\n    private final String algorithm;\n\n    private ExecutorService service;\n\n    private final int spaceCost;\n\n    private final int timeCost;\n\n    private final int parallelism;\n\n    private final int delta;\n\n    BalloonHashingFunction(String algorithm, int spaceCost, int timeCost,  int parallelism, int delta)\n    {\n        this.algorithm = algorithm;\n        this.spaceCost = spaceCost;\n        this.timeCost = timeCost;\n        this.parallelism = parallelism;\n        this.delta = delta;\n        if (parallelism > 1)\n        {\n            this.service = Executors.newFixedThreadPool(Utils.AVAILABLE_PROCESSORS);\n        }\n\n    }\n\n\n    public static BalloonHashingFunction getInstance(String algorithm, int spaceCost, int timeCost,  int parallelism, int delta)\n    {\n        String key = getUID(algorithm, spaceCost, timeCost, parallelism, delta);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            BalloonHashingFunction function = new BalloonHashingFunction(algorithm, spaceCost, timeCost, parallelism, delta);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    public static BalloonHashingFunction getInstance(String algorithm, int spaceCost, int timeCost,  int parallelism)\n    {\n        return getInstance(algorithm, spaceCost, timeCost, parallelism, DEFAULT_DELTA);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        return null;\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword)\n    {\n        return null;\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return hash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt)\n    {\n        return internalHash(plainTextPassword, salt);\n    }\n\n    protected Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n\n        byte[] output;\n\n        if (parallelism == 1)\n        {\n            byte[] parallelSalt = Utils.append(salt, Utils.longToLittleEndian((1)));\n            MessageDigest messageDigest = getMessageDigest();\n            output = balloon(messageDigest, plainTextPassword, parallelSalt);\n            output = hashFunc(messageDigest, plainTextPassword, salt, output);\n        }\n        else if (parallelism > 1)\n        {\n\n            List<Future<?>> futures = new ArrayList<>();\n\n            for (int i = 0; i < parallelism; i++)\n            {\n                byte[] parallelSalt = Utils.append(salt, Utils.longToLittleEndian((i + 1)));\n                Future<byte[]> future = service.submit(() -> balloon(getMessageDigest(), plainTextPassword, parallelSalt));\n\n                futures.add(future);\n            }\n\n            MessageDigest messageDigest = getMessageDigest();\n            output = new byte[messageDigest.getDigestLength()];\n\n            try\n            {\n                byte[] tmp;\n                output = (byte[]) futures.get(0).get();\n                for (int f = 1; f < futures.size(); f++)\n                {\n                    tmp = ((byte[]) futures.get(f).get());\n\n                    for (int i = 0; i < output.length; i++)\n                    {\n                        output[i] ^= tmp[i];\n                    }\n\n                }\n            }\n            catch (InterruptedException | ExecutionException e)\n            {\n                Thread.currentThread().interrupt();\n            }\n\n\n            output = hashFunc(messageDigest, plainTextPassword, salt, output);\n        }\n        else\n        {\n            output =  balloon(getMessageDigest(), plainTextPassword, salt);\n        }\n\n        return new Hash(this, Utils.toHex(output), output, salt);\n    }\n\n    protected MessageDigest getMessageDigest()\n    {\n        try\n        {\n            return MessageDigest.getInstance(algorithm);\n        }\n        catch (NoSuchAlgorithmException nsae)\n        {\n            throw new UnsupportedOperationException(\"`\" + algorithm + \"` is not supported by your system.\", nsae);\n        }\n    }\n\n    \nprivate byte[] balloon(MessageDigest messageDigest, byte[] plainTextPassword, byte[] salt)\n    {\n        List<byte[]> buffer = new ArrayList<>();\n        buffer.add(hashFunc(messageDigest, 0, plainTextPassword, salt));\n\n        int cnt = 1;\n\n        cnt = expand(messageDigest, buffer, cnt);\n        mix(messageDigest, buffer, cnt, salt);\n        return extract(buffer);\n    }\n\n\n\n    private int expand(MessageDigest messageDigest, List<byte[]> buffer, int cnt)\n    {\n        int newCnt = cnt;\n        for (int i = 1; i < spaceCost; i++)\n        {\n            buffer.add(hashFunc(messageDigest, newCnt, buffer.get(i - 1)));\n            newCnt += 1;\n        }\n        return newCnt;\n    }\n\n    private void mix(MessageDigest messageDigest, List<byte[]> buffer, int cnt, byte[] salt)\n    {\n        int newCnt = cnt;\n        for (int t = 0; t < timeCost; t++)\n        {\n            for (int s = 0; s < spaceCost; s++)\n            {\n                buffer.set(s, hashFunc(messageDigest, newCnt, get(buffer, s - 1), get(buffer, s)));\n                newCnt += 1;\n\n                for (int d = 0; d < delta; d++)\n                {\n                    byte[] indexBlock = hashFunc(messageDigest, t, s, d);\n                    int other = Utils.bytesToInt(hashFunc(messageDigest, newCnt, salt, indexBlock)).mod(BigInteger.valueOf(spaceCost)).intValue();\n                    newCnt += 1;\n                    buffer.set(s, hashFunc(messageDigest, newCnt, buffer.get(s), get(buffer, other)));\n                    newCnt += 1;\n                }\n            }\n        }\n    }\n\n    private byte[] extract(List<byte[]> buffer)\n    {\n        return buffer.get(buffer.size() - 1);\n    }\n\n    private byte[] get(List<byte[]> buffer, int position)\n    {\n        if (position < 0)\n        {\n            return buffer.get(buffer.size() + position);\n        }\n        return buffer.get(position);\n    }\n\n\n    private byte[] hashFunc(MessageDigest md, Object... args)\n    {\n        byte[] t = new byte[0];\n\n        for (Object arg : args)\n        {\n            if (arg instanceof Integer)\n            {\n                t = Utils.append(t, Utils.intToLittleEndianBytes((Integer) arg, 8));\n            }\n            else if (arg instanceof CharSequence)\n            {\n                t = Utils.append(t, Utils.fromCharSequenceToBytes((CharSequence) arg));\n            }\n            else if (arg instanceof byte[])\n            {\n                t = Utils.append(t, (byte[]) arg);\n            }\n        }\n\n        return md.digest(t);\n    }\n\n\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(plainTextPassword, hashed, null);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        Hash hash = internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n        return slowEquals(hash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        return check(plainTextPassword, new byte[0], hashed, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        Hash hash = internalHash(plainTextPassword, salt);\n        return slowEquals(hash.getResultAsBytes(), hashed);\n    }\n\n    private static String getUID(String algorithm, int spaceCost, int timeCost,  int parallelism, int delta)\n    {\n        return algorithm + '|' + spaceCost + '|' + timeCost + '|' + parallelism + '|' + delta;\n    }\n\n    @Override\n    public boolean equals(Object o)\n    {\n        if (this == o)\n            return true;\n        if (!(o instanceof BalloonHashingFunction))\n            return false;\n        BalloonHashingFunction other = (BalloonHashingFunction) o;\n        return algorithm .equals(other.algorithm) //\n                && spaceCost == other.spaceCost //\n                && timeCost == other.timeCost //\n                && parallelism == other.parallelism //\n                && delta == other.delta;\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(algorithm, spaceCost, timeCost, parallelism, delta);\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '[' + toString(algorithm, spaceCost, timeCost, parallelism, delta) + ']';\n    }\n\n    protected static String toString(String algorithm, int spaceCost, int timeCost, int parallelism, int delta)\n    {\n        return \"a=\" + algorithm + \", s=\" + spaceCost + \", t=\" + timeCost + \", p=\" + parallelism + \", d=\" + delta;\n    }\n}\n\n// Path: src/main/java/com/password4j/types/Bcrypt.java\n/*\n *  (C) Copyright 2021 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j.types;\n\npublic enum Bcrypt\n{\n    A, B, X, Y;\n\n    public static Bcrypt valueOf(char minor)\n    {\n        for (Bcrypt type : Bcrypt.values())\n        {\n            if (type.minor() == minor)\n            {\n                return type;\n            }\n        }\n        return null;\n    }\n\n    public char minor()\n    {\n        return name().toLowerCase().charAt(0);\n    }\n}\n\n// Path: src/main/java/com/password4j/BcryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Bcrypt;\n\nimport java.security.MessageDigest;\nimport java.util.Arrays;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of bcrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Bcrypt\">bcrypt</a>\n * @since 0.1.0\n */\npublic class BcryptFunction extends AbstractHashingFunction\n{\n\n    private static final ConcurrentMap<String, BcryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final int BCRYPT_SALT_LEN = 16;\n\n    private static final int BLOWFISH_NUM_ROUNDS = 16;\n\n    private static final int[] P_ORIG = { //\n            0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6,\n            0x38d01377, 0xbe5466cf, 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b };\n\n    private static final int[] S_ORIG = { //\n            0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, 0x24a19947,\n            0xb3916cf7, 0x0801f2e2, 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658,\n            0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, 0xc5d1b023, 0x286085f0, 0xca417918,\n            0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60,\n            0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, 0xa15486af,\n            0x7c72e993, 0xb3ee1411, 0x636fbc2a, 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c,\n            0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60,\n            0x5dec8032, 0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239,\n            0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0, 0x6a51a0d2,\n            0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176,\n            0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, 0xe06f75d8, 0x85c12073, 0x401a449f,\n            0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b,\n            0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6,\n            0x409f60c4, 0x5e5c9ec2, 0x196a2463, 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c,\n            0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, 0xc0cba857, 0x45c8740f, 0xd20b5f39,\n            0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8,\n            0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, 0x9e5c57bb,\n            0xca6f8ca0, 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8,\n            0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, 0x9a53e479, 0xb6f84565, 0xd28e49bc,\n            0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4,\n            0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb,\n            0xf2122b64, 0x8888b812, 0x900df01c, 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777,\n            0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81,\n            0xd2ada8d9, 0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf,\n            0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, 0x2464369b,\n            0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9,\n            0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, 0xd60f573f, 0xbc9bc6e4, 0x2b60a476,\n            0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664,\n            0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0,\n            0x49a7df7d, 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, 0x193602a5, 0x75094c29,\n            0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6,\n            0xf0255dc1, 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1,\n            0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37,\n            0xf0500c0d, 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, 0xd19113f9, 0x7ca92ff6,\n            0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e,\n            0xa4751e41, 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf,\n            0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f,\n            0x2e6b7124, 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, 0xec7aec3a, 0xdb851dfa,\n            0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002,\n            0x133ae4dd, 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509,\n            0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c,\n            0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, 0xc6150eba, 0x94e2ea78,\n            0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e,\n            0xeac31f66, 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802,\n            0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830,\n            0xeb61bd96, 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, 0xeecc86bc, 0x60622ca7,\n            0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5,\n            0xc021b8f7, 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281,\n            0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb,\n            0x532e3054, 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, 0x5d4a14d9, 0xe864b7e3,\n            0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d,\n            0x41cd2105, 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3,\n            0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a,\n            0x0c55f5ea, 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, 0x7cde3759, 0xcbee7460,\n            0x4085f2a7, 0xce77326e, 0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc,\n            0x800bcadc, 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340,\n            0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7, 0xe93d5a68,\n            0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a,\n            0x43b7d4b7, 0x500061af, 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, 0x70f4ddd3,\n            0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a,\n            0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, 0x4f3ffea2,\n            0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9,\n            0xee39d7ab, 0x3b124e8b, 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, 0x6841e7f7,\n            0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b,\n            0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, 0xfdf8e802,\n            0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f,\n            0x404779a4, 0x5d886e17, 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, 0x257b7834, 0x602a9c60, 0xdff8e8a3,\n            0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922,\n            0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, 0x5449a36f,\n            0x877d48fa, 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8,\n            0x991be14c, 0xdb6e6b0d, 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, 0xb5390f92,\n            0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb,\n            0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, 0x6a124237,\n            0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386,\n            0xd90cec6e, 0xd5abea2a, 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, 0x60787bf8,\n            0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f,\n            0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61, 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, 0x5366f9c3,\n            0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770,\n            0x8cd55591, 0xc902de4c, 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, 0x662d09a1,\n            0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169,\n            0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, 0x9af88c27,\n...\n", "needle_token_start": 5526, "needle_token_end": 5639, "code_context_ntokens": 16073, "output": ["```java\nprivate byte[] balloon(MessageDigest messageDigest, byte[] plainTextPassword, byte[] salt)\n{\n    List<byte[]> buffer = new ArrayList<>();\n    buffer.add(hashFunc(messageDigest, 0, plainTextPassword, salt));\n\n    int cnt = 1;\n\n    cnt = expand(messageDigest, buffer, cnt);\n    mix(messageDigest, buffer, cnt, salt);\n    return extract(buffer);\n}\n```"]}
{"repo": "Password4j/password4j", "name": "fromStringBuilderToBytes", "language": "java", "path": "src/main/java/com/password4j/BcryptFunction.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: Converts a sequence of characters stored in a mutable sequence of characters into an array of bytes, specifically designed to handle the output length specified.\n2. **Input**: A mutable sequence of characters and an integer specifying the number of bytes to convert.\n3. **Output**: An array of bytes containing the converted characters up to the specified output length.\n4. **Procedure**: Iterates over the characters in the mutable sequence up to the specified length, casting each character to a byte and storing it in the resulting byte array.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            0x8cd55591, 0xc902de4c, 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, 0x662d09a1,\n            0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169,\n            0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, 0x9af88c27,\n            0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63,\n            0x53c2dd94, 0xc2c21634, 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, 0x6f05e409, 0x4b7c0188, 0x39720a3d,\n            0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4,\n            0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, 0xd79a3234,\n            0x92638212, 0x670efa8e, 0x406000e0, 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742,\n            0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, 0xb78c1b6b, 0x21a19045, 0xb26eb1be,\n            0x6a366eb4, 0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6,\n            0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8,\n            0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1,\n            0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, 0x80e4a915, 0x87b08601, 0x9b09e6ad,\n            0x3b3ee593, 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28,\n            0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d,\n            0x283b57cc, 0xf8d56629, 0x79132e28, 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba,\n            0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb,\n            0x26dcf319, 0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f,\n            0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32, 0xa8b6e37e,\n            0xc3293d46, 0x48de5369, 0x6413e680, 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd,\n            0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, 0xdda26a7e, 0x3a59ff45, 0x3e350a44,\n            0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370,\n            0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, 0xe1b00428,\n            0x95983a1d, 0x06b89fb4, 0xce6ea048, 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc,\n            0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3,\n            0xa1e8aac7, 0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f,\n            0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525, 0xfae59361,\n            0xceb69ceb, 0xc2a86459, 0x12baa8d1, 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b,\n            0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, 0x1b0a7441, 0x4ba3348c, 0xc5be7120,\n            0xc37632d8, 0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f,\n            0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02, 0xacf08162,\n            0x5a75ebb5, 0x6e163697, 0x88d273cc, 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a,\n            0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, 0x71126905, 0xb2040222, 0xb6cbcf7c,\n            0xcd769c2b, 0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060,\n            0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869,\n            0xa65cdea0, 0x3f09252d, 0xc208e69f, 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6 };\n\n    private static final int[] BF_CRYPT_CIPHERTEXT = { 0x4f727068, 0x65616e42, 0x65686f6c, 0x64657253, 0x63727944, 0x6f756274 };\n\n    private static final char[] BASE_64_CODE = { //\n            '.', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U',\n            'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',\n            's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };\n\n    private static final byte[] INDEX_64 = { //\n            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n            -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, -1,\n            -1, -1, -1, -1, -1, -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,\n            27, -1, -1, -1, -1, -1, -1, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,\n            50, 51, 52, 53, -1, -1, -1, -1, -1 };\n\n    private int logRounds;\n\n    private Bcrypt type;\n\n    private BcryptFunction()\n    {\n        //\n    }\n\n    protected BcryptFunction(Bcrypt type, int logRounds)\n    {\n        this();\n        this.logRounds = logRounds;\n        this.type = type;\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * logarithmic cost.\n     * <p>\n     * The cost is expressed with the following formula:\n     * {@code cost = 2^logRounds}\n     *\n     * @param logRounds logarithmic cost\n     * @return a singleton instance\n     * @since 0.3.0\n     */\n    public static BcryptFunction getInstance(int logRounds)\n    {\n        return getInstance(Bcrypt.B, logRounds);\n    }\n\n    public static BcryptFunction getInstance(Bcrypt type, int logRounds)\n    {\n        String uid = getUID(type, logRounds);\n        if (INSTANCES.containsKey(uid))\n        {\n            return INSTANCES.get(uid);\n        }\n        else\n        {\n            BcryptFunction function = new BcryptFunction(type, logRounds);\n            INSTANCES.put(uid, function);\n            return function;\n        }\n    }\n\n    public static BcryptFunction getInstanceFromHash(String hashed)\n    {\n        internalChecks(hashed);\n\n        if (hashed.charAt(2) == '$')\n        {\n            int rounds = Integer.parseInt(hashed.substring(3, 5));\n            return getInstance(rounds);\n        }\n        else\n        {\n            char minor = hashed.charAt(2);\n            if (isNotValidMinor(minor) || hashed.charAt(3) != '$')\n                throw new BadParametersException(\"Invalid salt revision\");\n            int rounds = Integer.parseInt(hashed.substring(4, 6));\n            return getInstance(Bcrypt.valueOf(minor), rounds);\n        }\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        String salt = generateSalt();\n        return hash(plainTextPassword, salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        String salt = generateSalt();\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return internalHash(plainTextPassword, salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt)\n    {\n        return internalHash(plainTextPassword, Utils.fromBytesToString(salt));\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        return internalCheck(plainTextPassword, hashed);\n    }\n\n    private Hash internalHash(CharSequence plainTextPassword, String salt)\n    {\n        byte[] passwordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        return internalHash(passwordAsBytes, salt);\n    }\n\n    protected Hash internalHash(byte[] plainTextPasswordAsBytes, String salt)\n    {\n        String realSalt;\n        byte[] saltAsBytes;\n        byte[] hashed;\n        char minor = (char) 0;\n        int off;\n        StringBuilder rs = new StringBuilder();\n\n        internalChecks(salt);\n\n        int saltLength = salt.length();\n\n        if (salt.charAt(2) == '$')\n            off = 3;\n        else\n        {\n            minor = salt.charAt(2);\n            if (isNotValidMinor(minor) || salt.charAt(3) != '$')\n                throw new BadParametersException(\"Invalid salt revision\");\n            off = 4;\n        }\n\n        // Extract number of rounds\n        if (salt.charAt(off + 2) > '$')\n            throw new BadParametersException(\"Missing salt rounds\");\n\n        if (off == 4 && saltLength < 29)\n        {\n            throw new BadParametersException(\"Invalid salt\");\n        }\n\n        realSalt = salt.substring(off + 3, off + 25);\n        saltAsBytes = decodeBase64(realSalt, BCRYPT_SALT_LEN);\n\n        if (minor >= Bcrypt.A.minor()) // add null terminator\n            plainTextPasswordAsBytes = Arrays.copyOf(plainTextPasswordAsBytes, plainTextPasswordAsBytes.length + 1);\n\n        hashed = cryptRaw(plainTextPasswordAsBytes, saltAsBytes, logRounds, minor == Bcrypt.X.minor(), minor == Bcrypt.A.minor() ? 0x10000 : 0);\n\n        rs.append(\"$2\");\n        if (minor >= Bcrypt.A.minor())\n            rs.append(minor);\n        rs.append('$');\n        if (logRounds < 10)\n            rs.append('0');\n        rs.append(logRounds);\n        rs.append('$');\n        encodeBase64(saltAsBytes, saltAsBytes.length, rs);\n        encodeBase64(hashed, BF_CRYPT_CIPHERTEXT.length * 4 - 1, rs);\n        String result = rs.toString();\n\n        return new Hash(this, result, hashed, saltAsBytes);\n    }\n\n    public int getLogarithmicRounds()\n    {\n        return logRounds;\n    }\n\n    public Bcrypt getType()\n    {\n        return type;\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(type, logRounds) + ')';\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(logRounds, type);\n    }\n\n    protected static String getUID(Bcrypt type, int logRounds)\n    {\n        return type.minor() + \"|\" + logRounds;\n    }\n\n    protected static String toString(Bcrypt type, int logRounds)\n    {\n        return \"t=\" + type.minor() + \", r=\" + logRounds;\n    }\n\n    /**\n     * Encode a byte array using bcrypt's slightly-modified base64\n     * encoding scheme. Note that this is <strong>not</strong> compatible with\n     * the standard MIME-base64 encoding.\n     *\n     * @param toBeEncoded the byte array to encode\n     * @param length      the number of bytes to encode\n     * @param sb          StringBuilder representing the base64-encoded string\n     * @throws IllegalArgumentException if the length is invalid\n     * @since 0.1.0\n     */\n    protected static void encodeBase64(byte[] toBeEncoded, int length, StringBuilder sb)\n    {\n        int off = 0;\n        int c1;\n        int c2;\n\n        if (length <= 0 || length > toBeEncoded.length)\n        {\n            throw new BadParametersException(\"Invalid length\");\n        }\n\n        while (off < length)\n        {\n            c1 = toBeEncoded[off++] & 0xff;\n            sb.append(BASE_64_CODE[(c1 >> 2) & 0x3f]);\n            c1 = (c1 & 0x03) << 4;\n            if (off >= length)\n            {\n                sb.append(BASE_64_CODE[c1 & 0x3f]);\n                return;\n            }\n            c2 = toBeEncoded[off++] & 0xff;\n            c1 |= (c2 >> 4) & 0x0f;\n            sb.append(BASE_64_CODE[c1 & 0x3f]);\n            c1 = (c2 & 0x0f) << 2;\n            if (off >= length)\n            {\n                sb.append(BASE_64_CODE[c1 & 0x3f]);\n                return;\n            }\n            c2 = toBeEncoded[off++] & 0xff;\n            c1 |= (c2 >> 6) & 0x03;\n            sb.append(BASE_64_CODE[c1 & 0x3f]);\n            sb.append(BASE_64_CODE[c2 & 0x3f]);\n        }\n    }\n\n    /**\n     * Look up the 3 bits base64-encoded by the specified character,\n     * range-checking against conversion table\n     *\n     * @param c the base64-encoded value\n     * @return the decoded value of x\n     * @since 0.1.0\n     */\n    protected static byte char64(char c)\n    {\n        if ( c >= INDEX_64.length)\n            return -1;\n        return INDEX_64[c];\n    }\n\n    /**\n     * Decode a string encoded using bcrypt's base64 scheme to a\n     * byte array. Note that this is *not* compatible with\n     * the standard MIME-base64 encoding.\n     *\n     * @param str        the string to decode\n     * @param maxOLength the maximum number of bytes to decode\n     * @return an array containing the decoded bytes\n     * @throws IllegalArgumentException if maxOLength is invalid\n     * @since 0.1.0\n     */\n    protected static byte[] decodeBase64(String str, int maxOLength)\n    {\n        StringBuilder rs = new StringBuilder();\n        int off = 0;\n        int strLength = str.length();\n        int oLength = 0;\n        byte c1;\n        byte c2;\n        byte c3;\n        byte c4;\n        byte o;\n\n        if (maxOLength <= 0)\n            throw new BadParametersException(\"Invalid maxOLength\");\n\n        while (off < strLength - 1 && oLength < maxOLength)\n        {\n            c1 = char64(str.charAt(off++));\n            c2 = char64(str.charAt(off++));\n            if (c1 == -1 || c2 == -1)\n            {\n                return fromStringBuilderToBytes(rs, oLength);\n            }\n            o = (byte) (c1 << 2);\n            o |= (c2 & 0x30) >> 4;\n            rs.append((char) o);\n            if (++oLength >= maxOLength || off >= strLength)\n            {\n                return fromStringBuilderToBytes(rs, oLength);\n            }\n            c3 = char64(str.charAt(off++));\n            if (c3 == -1)\n            {\n                return fromStringBuilderToBytes(rs, oLength);\n            }\n            o = (byte) ((c2 & 0x0f) << 4);\n            o |= (c3 & 0x3c) >> 2;\n            rs.append((char) o);\n            if (++oLength >= maxOLength || off >= strLength)\n            {\n                return fromStringBuilderToBytes(rs, oLength);\n            }\n            c4 = char64(str.charAt(off++));\n            o = (byte) ((c3 & 0x03) << 6);\n            o |= c4;\n            rs.append((char) o);\n            ++oLength;\n        }\n\n        return fromStringBuilderToBytes(rs, oLength);\n\n    }\n\n    \nprivate static byte[] fromStringBuilderToBytes(StringBuilder sb, int oLength)\n    {\n        byte[] ret = new byte[oLength];\n        for (int i = 0; i < oLength; i++)\n        {\n            ret[i] = (byte) sb.charAt(i);\n        }\n        return ret;\n    }\n\n    /**\n     * Cyclically extracts a word of key material\n     *\n     * @param data       the string to extract the data from\n     * @param offPointer a \"pointer\" (as a one-entry array) to the\n     *                   current offset into data\n     * @return the next word of material from data\n     * @since 0.1.0\n     */\n    protected static int[] streamToWords(byte[] data, int[] offPointer, int[] signp)\n    {\n        int i;\n        int[] words = { 0, 0 };\n        int off = offPointer[0];\n        int sign = signp[0];\n\n        for (i = 0; i < 4; i++)\n        {\n            words[0] = (words[0] << 8) | (data[off] & 0xff);\n            words[1] = (words[1] << 8) |  (data[off] & 0xff); // sign extension bug\n            if (i > 0)\n                sign |= words[1] & 0x80;\n            off = (off + 1) % data.length;\n        }\n\n        offPointer[0] = off;\n        signp[0] = sign;\n        return words;\n    }\n\n    protected static int streamToWord(byte[] data, int[] offp)\n    {\n        int[] signp = { 0 };\n        return streamToWords(data, offp, signp)[0];\n    }\n\n    protected static int streamToWordMinorX(byte[] data, int[] offp)\n    {\n        int[] signp = { 0 };\n        return streamToWords(data, offp, signp)[1];\n    }\n\n    private static boolean isNotValidMinor(char minor)\n    {\n        return Bcrypt.valueOf(minor) == null;\n    }\n\n    private static void internalChecks(String salt)\n    {\n        if (salt == null)\n        {\n            throw new BadParametersException(\"salt cannot be null\");\n        }\n        else if (salt.length() < 28)\n        {\n            throw new BadParametersException(\"Invalid salt\");\n        }\n        else if (salt.charAt(0) != '$' || salt.charAt(1) != '2')\n        {\n            throw new BadParametersException(\"Invalid salt version\");\n        }\n    }\n\n    /**\n     * Generate a salt to be used with the {@link BcryptFunction#hash(CharSequence, String)} method\n     *\n     * @param logRounds the log2 of the number of rounds of\n     *                  hashing to apply - the work factor therefore increases as\n     *                  2^log_rounds.\n     * @param prefix    bcrypt variant\n     * @return an encoded salt value\n     * @since 0.1.0\n     */\n    protected static String generateSalt(String prefix, int logRounds)\n    {\n        StringBuilder rs = new StringBuilder();\n        byte[] rnd = new byte[BCRYPT_SALT_LEN];\n\n        if (!prefix.startsWith(\"$2\") || (prefix.charAt(2) != Bcrypt.A.minor() && prefix.charAt(2) != Bcrypt.Y.minor() && prefix\n                .charAt(2) != Bcrypt.B.minor()))\n        {\n            throw new BadParametersException(\"Invalid prefix\");\n        }\n        if (logRounds < 4 || logRounds > 31)\n        {\n            throw new BadParametersException(\"Invalid logRounds\");\n        }\n\n        AlgorithmFinder.getSecureRandom().nextBytes(rnd);\n\n        rs.append(\"$2\");\n        rs.append(prefix.charAt(2));\n        rs.append('$');\n        if (logRounds < 10)\n            rs.append('0');\n        rs.append(logRounds);\n        rs.append('$');\n        encodeBase64(rnd, rnd.length, rs);\n        return rs.toString();\n    }\n\n\n\n\n\n    /**\n     * Blowfish encipher a single 64-bit block encoded as\n     * two 32-bit halves\n     *\n     * @param lr  an array containing the two 32-bit half blocks\n     * @param off the position in the array of the blocks\n     * @since 0.1.0\n     */\n    protected void encipher(int[] lr, int off, int[] pArray, int[] sBox)\n    {\n        int i;\n        int n;\n        int l = lr[off];\n        int r = lr[off + 1];\n\n        l ^= pArray[0];\n        for (i = 0; i <= BLOWFISH_NUM_ROUNDS - 2; )\n        {\n\n            n = feistelSubstitution(l, sBox);\n            r ^= n ^ pArray[++i];\n\n            n = feistelSubstitution(r, sBox);\n            l ^= n ^ pArray[++i];\n        }\n        lr[off] = r ^ pArray[BLOWFISH_NUM_ROUNDS + 1];\n        lr[off + 1] = l;\n    }\n\n    private int feistelSubstitution(int p, int[] sBox)\n    {\n        int x = sBox[(p >> 24) & 0xff];\n        x += sBox[0x100 | ((p >> 16) & 0xff)];\n        x ^= sBox[0x200 | ((p >> 8) & 0xff)];\n        x += sBox[0x300 | (p & 0xff)];\n        return x;\n    }\n\n    /**\n     * Key the Blowfish cipher\n     *\n     * @param key an array containing the key\n     * @since 0.1.0\n     */\n    protected void key(byte[] key, boolean signExtBug, int[] pArray, int[] sBox)\n    {\n        int i;\n        int[] koffp = { 0 };\n        int[] lr = { 0, 0 };\n        int pLength = pArray.length;\n        int sLenght = sBox.length;\n\n        for (i = 0; i < pLength; i++)\n            if (!signExtBug)\n                pArray[i] = pArray[i] ^ streamToWord(key, koffp);\n            else\n                pArray[i] = pArray[i] ^ streamToWordMinorX(key, koffp);\n\n        for (i = 0; i < pLength; i += 2)\n        {\n            encipher(lr, 0, pArray, sBox);\n            pArray[i] = lr[0];\n            pArray[i + 1] = lr[1];\n        }\n\n        for (i = 0; i < sLenght; i += 2)\n        {\n            encipher(lr, 0, pArray, sBox);\n            sBox[i] = lr[0];\n            sBox[i + 1] = lr[1];\n        }\n    }\n\n    /**\n     * Perform the \"enhanced key schedule\" step described by\n     * Provos and Mazieres in \"A Future-Adaptable Password Scheme\"\n     *\n     * @param data salt information\n     * @param key  password information\n     * @see <a href=\"http://www.openbsd.org/papers/bcrypt-paper.ps\">A Future-Adaptable Password Scheme</a>\n     * @since 0.1.0\n     */\n    protected void enhancedKeySchedule(byte[] data, byte[] key, boolean signExtBug, int safety, int[] pArray, int[] sBox)\n    {\n        int i;\n        int[] koffp = { 0 };\n        int[] doffp = { 0 };\n        int[] lr = { 0, 0 };\n        int pLength = pArray.length;\n        int sLength = sBox.length;\n        int[] signP = { 0 };\n        int diff = 0;\n\n        for (i = 0; i < pLength; i++)\n        {\n            int[] words = streamToWords(key, koffp, signP);\n            diff |= words[0] ^ words[1];\n            pArray[i] = pArray[i] ^ words[signExtBug ? 1 : 0];\n        }\n\n        int sign = signP[0];\n\n        diff |= diff >> 16;\n        diff &= 0xffff;\n        diff += 0xffff;\n        sign <<= 9;\n        sign &= ~diff & safety;\n\n        pArray[0] ^= sign;\n\n        for (i = 0; i < pLength; i += 2)\n        {\n            lr[0] ^= streamToWord(data, doffp);\n            lr[1] ^= streamToWord(data, doffp);\n            encipher(lr, 0, pArray, sBox);\n            pArray[i] = lr[0];\n            pArray[i + 1] = lr[1];\n        }\n\n        for (i = 0; i < sLength; i += 2)\n        {\n            lr[0] ^= streamToWord(data, doffp);\n            lr[1] ^= streamToWord(data, doffp);\n            encipher(lr, 0, pArray, sBox);\n            sBox[i] = lr[0];\n            sBox[i + 1] = lr[1];\n        }\n    }\n\n    /**\n     * Perform the central password hashing step in the\n     * bcrypt scheme\n     *\n     * @param password  the password to hash\n     * @param salt      the binary salt to hash with the password\n     * @param logRounds the binary logarithm of the number\n     *                  of rounds of hashing to apply\n     * @return an array containing the binary hashed password\n     * @since 0.1.0\n     */\n    protected byte[] cryptRaw(byte[] password, byte[] salt, int logRounds, boolean sign, int safety)\n    {\n        long rounds;\n        int i;\n        int j;\n        int[] cdata = BF_CRYPT_CIPHERTEXT.clone();\n        int clen = cdata.length;\n        byte[] ret;\n\n        if (logRounds < 4 || logRounds > 31)\n            throw new BadParametersException(\"Bad number of rounds\");\n        rounds = 1L << logRounds;\n        if (salt.length != BCRYPT_SALT_LEN)\n            throw new BadParametersException(\"Bad salt length\");\n\n        int[] pArray = P_ORIG.clone();\n        int[] sBox = S_ORIG.clone();\n        enhancedKeySchedule(salt, password, sign, safety, pArray, sBox);\n        for (i = 0; i < rounds; i++)\n        {\n            key(password, sign, pArray, sBox);\n            key(salt, false, pArray, sBox);\n        }\n\n        for (i = 0; i < 64; i++)\n        {\n            for (j = 0; j < (clen >> 1); j++)\n                encipher(cdata, j << 1, pArray, sBox);\n        }\n\n        ret = new byte[clen * 4];\n        for (i = 0, j = 0; i < clen; i++)\n        {\n            ret[j++] = (byte) ((cdata[i] >> 24) & 0xff);\n            ret[j++] = (byte) ((cdata[i] >> 16) & 0xff);\n            ret[j++] = (byte) ((cdata[i] >> 8) & 0xff);\n            ret[j++] = (byte) (cdata[i] & 0xff);\n        }\n        return ret;\n    }\n\n    @Override\n    public boolean equals(Object o)\n    {\n        if (this == o)\n            return true;\n        if (!(o instanceof BcryptFunction))\n            return false;\n        BcryptFunction that = (BcryptFunction) o;\n        return logRounds == that.logRounds && type == that.type;\n    }\n\n\n\n    /**\n     * Generate a salt to be used with the {@link BcryptFunction#hash(CharSequence, String)} method\n     *\n     * @return an encoded salt value\n     * @since 0.1.0\n     */\n    protected String generateSalt()\n    {\n        return generateSalt(\"$2\" + type.minor(), logRounds);\n    }\n\n    /**\n     * Check that a plaintext password matches a previously hashed\n     * one\n     *\n     * @param plainTextPasswordAsBytes the plaintext password to verify\n     * @param hashed    the previously-hashed password\n     * @return true if the passwords match, false otherwise\n     * @since 0.1.0\n     */\n    protected boolean internalCheck(byte[] plainTextPasswordAsBytes, byte[] hashed)\n    {\n        return equalsNoEarlyReturn(hashed, hash(plainTextPasswordAsBytes, hashed).getResultAsBytes());\n    }\n\n    static boolean equalsNoEarlyReturn(byte[] a, byte[] b)\n    {\n        return MessageDigest.isEqual(a, b);\n    }\n\n}\n\n// Path: src/main/java/com/password4j/types/Hmac.java\n/*\n *  (C) Copyright 2021 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j.types;\n\nimport com.password4j.CompressedPBKDF2Function;\n\n\n/**\n * Static representation of the commonly supported\n * Hmac variants.\n */\npublic enum Hmac\n{\n\n    SHA1(160, 1), //\n    SHA224(224, 2), //\n    SHA256(256, 3), //\n    SHA384(384, 4), //\n    SHA512(512, 5);\n\n    private final int bits;\n\n    private final int code;\n\n    /**\n     * @param bits length of the produced hash\n     * @param code uid used by {@link CompressedPBKDF2Function}\n     */\n    Hmac(int bits, int code)\n    {\n        this.bits = bits;\n        this.code = code;\n    }\n\n    /**\n     * Finds the enum associated with the given code\n     *\n     * @param code a numeric uid that identifies the algorithm\n     * @return a {@link Hmac} enum. Null if the code is not present in this enum\n     */\n    public static Hmac fromCode(int code)\n    {\n        for (Hmac alg : values())\n        {\n            if (alg.code() == code)\n            {\n                return alg;\n            }\n        }\n        return null;\n    }\n\n    /**\n     * @return length of the algorithm in bits\n     */\n    public int bits()\n    {\n        return bits;\n    }\n\n    /**\n     * @return the numeric uid used in {@link CompressedPBKDF2Function}\n     */\n    public int code()\n    {\n        return code;\n    }\n\n    @Override\n    public String toString()\n    {\n        return \"PBKDF2WithHmac\" + this.name();\n    }\n}\n\n// Path: src/main/java/com/password4j/CompressedPBKDF2Function.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of PBKDF2 function and its parameters.\n * <p>\n * The main difference between {@link PBKDF2Function} is the hash produced: the configurations of the CHF,\n * the salt and the hash are encoded inside it.\n * <p>\n * The produced hash is in the form\n * <p>\n * <code>\n * $algorithm$parameters$salt$hash\n * </code>\n * <p>\n * Assuming {@code $} as delimiter.\n * <p>\n * <ul>\n *     <li>\n *         The algorithm is encoded with its numeric uid {@link Hmac#code()}\n *     </li>\n *     <li>\n *         Parameters are encoded in one integer where the length occupies the first 32bit and\n *          the number of iterations the remaining 32 bits.\n *     </li>\n *     <li>\n *         Salt is encoded in Base64\n *     </li>\n *     <li>\n *         Hash is encoded in Base64 as in {@link PBKDF2Function}\n *     </li>\n * </ul>\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/PBKDF2\">PBKDF2</a>\n * @since 0.1.0\n */\npublic class CompressedPBKDF2Function extends PBKDF2Function\n{\n\n    private static final Map<String, CompressedPBKDF2Function> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final char DELIMITER = PropertyReader.readChar(\"hash.pbkdf2.delimiter\", '$');\n\n    protected CompressedPBKDF2Function()\n    {\n        super();\n    }\n\n    protected CompressedPBKDF2Function(Hmac fromCode, int iterations, int length)\n    {\n        super(fromCode, iterations, length);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getInstance(Hmac algorithm, int iterations, int length)\n    {\n        String key = getUID(algorithm.name(), iterations, length);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            CompressedPBKDF2Function function = new CompressedPBKDF2Function(algorithm, iterations, length);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  string version of hmac algorithm. This must me mapped in {@link Hmac}.\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @throws IllegalArgumentException if {@code algorithm} is not mapped in {@link Hmac}.\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getInstance(String algorithm, int iterations, int length)\n    {\n        try\n        {\n            return getInstance(Hmac.valueOf(algorithm), iterations, length);\n        }\n        catch (IllegalArgumentException iae)\n        {\n            throw new UnsupportedOperationException(\"Algorithm `\" + algorithm + \"` is not recognized.\", iae);\n        }\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.0.0\n     */\n    public static CompressedPBKDF2Function getInstanceFromHash(String hashed)\n    {\n        String[] parts = getParts(hashed);\n        if (parts.length == 5)\n        {\n            int algorithm = Integer.parseInt(parts[1]);\n            long configuration = Long.parseLong(parts[2]);\n\n            int iterations = (int) (configuration >> 32);\n            int length = (int) configuration;\n\n            return CompressedPBKDF2Function.getInstance(Hmac.fromCode(algorithm), iterations, length);\n        }\n        throw new BadParametersException(\"`\" + hashed + \"` is not a valid hash\");\n    }\n\n\n\n    @Override\n    protected String getHash(byte[] encodedKey, byte[] salt)\n    {\n        String params = Long.toString((((long) getIterations()) << 32) | (getLength() & 0xffffffffL));\n        String salt64 = Utils.encodeBase64(salt);\n        String hash64 = super.getHash(encodedKey, salt);\n        return \"$\" + algorithm.code() + \"$\" + params + \"$\" + salt64 + \"$\" + hash64;\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        byte[] salt = getSaltFromHash(hashed);\n        Hash internalHas = hash(plainTextPassword, salt);\n\n        return slowEquals(internalHas.getResultAsBytes(), hashed);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        byte[] hashAsBytes = Utils.fromCharSequenceToBytes(hashed);\n        byte[] realSalt = getSaltFromHash(hashAsBytes);\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        Hash internalHash = hash(plainTextPasswordAsBytes, realSalt);\n        return slowEquals(internalHash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        byte[] realSalt = getSaltFromHash(hashed);\n        Hash internalHash = hash(plainTextPassword, realSalt);\n        return slowEquals(internalHash.getResultAsBytes(), hashed);\n    }\n\n    protected static List<byte[]> getParts(byte[] hashed)\n    {\n       return Utils.split(hashed, (byte) DELIMITER);\n    }\n\n    protected static String[] getParts(String hashed)\n    {\n        String regex = \"\\\\\" + DELIMITER;\n        return hashed.split(regex);\n    }\n\n    private byte[] getSaltFromHash(byte[] hashed)\n    {\n        List<byte[]> parts = getParts(hashed);\n        if (parts.size() == 5)\n        {\n            return Utils.decodeBase64(parts.get(3));\n        }\n        throw new BadParametersException(\"`\" + Utils.fromBytesToString(hashed) + \"` is not a valid hash\");\n    }\n}\n\n// Path: src/main/java/com/password4j/SaltOption.java\npackage com.password4j;\n\npublic enum SaltOption\n{\n    PREPEND, APPEND;\n}\n\n// Path: src/main/java/com/password4j/MessageDigestFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of Messaged digest functions provided by {@link MessageDigest}.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Category:Cryptographic_hash_functions\">Message digests</a>\n * @since 1.4.0\n */\npublic class MessageDigestFunction extends AbstractHashingFunction\n{\n    private static final Map<String, MessageDigestFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    protected static final SaltOption DEFAULT_SALT_OPTION = SaltOption.APPEND;\n\n    private final String algorithm;\n\n    private final SaltOption saltOption;\n\n\n    MessageDigestFunction(String algorithm, SaltOption saltOption)\n    {\n        this.algorithm = algorithm;\n        this.saltOption = saltOption;\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm message digest algorithm\n     * @return a singleton instance\n     * @since 1.4.0\n     */\n    public static MessageDigestFunction getInstance(String algorithm)\n    {\n        return getInstance(algorithm, DEFAULT_SALT_OPTION);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param saltOption a configuration that specifies how the salt is concatenated to the plain text password\n     * @return a singleton instance\n     * @since 1.4.0\n     */\n    public static MessageDigestFunction getInstance(String algorithm, SaltOption saltOption)\n    {\n        String key = getUID(algorithm, saltOption);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            MessageDigestFunction function = new MessageDigestFunction(algorithm, saltOption);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    protected static String getUID(String algorithm, SaltOption saltOption)\n    {\n        return algorithm + \"|\" + saltOption.name();\n    }\n\n    protected static String toString(String algorithm, SaltOption saltOption)\n    {\n        return \"a=\" + algorithm + \", o=\" + saltOption.name();\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        return hash(plainTextPassword, null);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        return hash(plainTextPasswordAsBytes, null);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes, byte[] saltAsBytes)\n    {\n        return internalHash(plainTextPasswordAsBytes, saltAsBytes);\n    }\n\n    protected Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n        byte[] finalCharSequence = concatenateSalt(plainTextPassword, salt);\n\n        byte[] result = getMessageDigest().digest(finalCharSequence);\n        return new Hash(this, Utils.toHex(result), result, salt);\n    }\n\n    protected MessageDigest getMessageDigest()\n    {\n        try\n        {\n            return MessageDigest.getInstance(algorithm);\n        } catch (NoSuchAlgorithmException nsae)\n        {\n            throw new UnsupportedOperationException(\"`\" + algorithm + \"` is not supported by your system.\", nsae);\n        }\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(plainTextPassword, hashed, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPasswordAsBytes, byte[] hashed)\n    {\n        return check(plainTextPasswordAsBytes, hashed, null);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        Hash hash = internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n        return slowEquals(hash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        Hash hash = internalHash(plainTextPassword, salt);\n        return slowEquals(hash.getResultAsBytes(), hashed);\n    }\n\n    /**\n     * The salt option describes if the Salt is appended or prepended to\n     * the plain text password.\n     *\n     * @return how the salt is concatenated\n     * @since 1.5.1\n     */\n    public SaltOption getSaltOption()\n    {\n        return saltOption;\n    }\n\n    /**\n     * The algorithm in use by this instance.\n     *\n     * @return the algorithm in use\n     * @since 1.5.1\n     */\n    public String getAlgorithm()\n    {\n        return algorithm;\n    }\n\n\n    private byte[] concatenateSalt(byte[] plainTextPassword, byte[] salt)\n    {\n        if (salt == null || salt.length == 0)\n        {\n            return  plainTextPassword;\n        }\n\n        if (saltOption == SaltOption.PREPEND)\n        {\n            return Utils.append(salt, plainTextPassword);\n        }\n        return Utils.append(plainTextPassword, salt);\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(this.algorithm, this.saltOption) + ')';\n    }\n\n    @Override\n    public boolean equals(Object o)\n    {\n        if (this == o)\n            return true;\n        if (!(o instanceof MessageDigestFunction))\n            return false;\n        MessageDigestFunction other = (MessageDigestFunction) o;\n        return algorithm.equals(other.algorithm) //\n                && saltOption == other.saltOption;\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(algorithm, saltOption);\n    }\n}\n\n// Path: src/main/java/com/password4j/ScryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport java.security.GeneralSecurityException;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of scrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Scrypt\">scrypt</a>\n * @since 0.1.0\n */\npublic class ScryptFunction extends AbstractHashingFunction\n{\n    public static final int DERIVED_KEY_LENGTH = 64;\n\n    private static final ConcurrentMap<String, ScryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private int workFactor; // N\n\n    private int resources; // r\n\n    private int parallelization; // p\n\n    private int derivedKeyLength; // dkLen\n\n    @SuppressWarnings(\"unused\")\n    private ScryptFunction()\n    {\n        //\n    }\n\n    /**\n     * @param workFactor      (N)\n     * @param resources       (r)\n     * @param parallelization (p)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = DERIVED_KEY_LENGTH;\n    }\n\n    /**\n     * @param workFactor       (N)\n     * @param resources        (r)\n     * @param parallelization  (p)\n     * @param derivedKeyLength (dkLen)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization, int derivedKeyLength)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = derivedKeyLength;\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n...\n", "needle_token_start": 7268, "needle_token_end": 7349, "code_context_ntokens": 16074, "output": ["```java\nprivate static byte[] fromStringBuilderToBytes(StringBuilder sb, int oLength)\n{\n    byte[] ret = new byte[oLength];\n    for (int i = 0; i < oLength; i++)\n    {\n        ret[i] = (byte) sb.charAt(i);\n    }\n    return ret;\n}\n```"]}
{"repo": "Password4j/password4j", "name": "fromCharSequenceToChars", "language": "java", "path": "src/main/java/com/password4j/Utils.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: Converts a sequence of characters into an array of characters.\n2. **Input**: A `CharSequence` which represents a readable sequence of `char` values.\n3. **Output**: An array of `char` containing the characters from the input sequence.\n4. **Procedure**: The function checks if the input is null or empty, returning an empty array if true. Otherwise, it iterates through each character of the input sequence, copying each character into a new array of characters, which is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/PropertyReader.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.FileInputStream;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.util.Properties;\n\n\nclass PropertyReader\n{\n\n    private static final Logger LOG = LoggerFactory.getLogger(PropertyReader.class);\n\n    private static final String FILE_NAME = \"psw4j.properties\";\n\n    private static final String CONFIGURATION_KEY = \"psw4j.configuration\";\n\n    private static final String MESSAGE = \"{}. Default value is used ({}). Please set property {} in your \" + FILE_NAME + \" file.\";\n\n    protected static Properties properties;\n\n    private PropertyReader()\n    {\n        //\n    }\n\n    static int readInt(String key, int defaultValue, String message)\n    {\n        String str = readString(key);\n        if (str == null)\n        {\n            LOG.warn(MESSAGE, message, defaultValue, key);\n            return defaultValue;\n        }\n        return Integer.parseInt(readString(key));\n    }\n\n    static boolean readBoolean(String key, boolean defaultValue)\n    {\n        String str = readString(key);\n        if (str == null)\n        {\n            return defaultValue;\n        }\n        return Boolean.parseBoolean(readString(key));\n    }\n\n    static String readString(String key, String defaultValue, String message)\n    {\n        String value = readString(key);\n        if (value == null)\n        {\n            LOG.warn(MESSAGE, message, defaultValue, key);\n            return defaultValue;\n        }\n        return value;\n    }\n\n    static char readChar(String key, char defaultValue, String message)\n    {\n        String str = readString(key);\n        if (str == null)\n        {\n            LOG.warn(MESSAGE, message, defaultValue, key);\n            return defaultValue;\n        }\n        return str.charAt(0);\n    }\n\n    static char readChar(String key, char defaultValue)\n    {\n        String str = readString(key);\n        if (str == null)\n        {\n            return defaultValue;\n        }\n        return str.charAt(0);\n    }\n\n    private static String readString(String key)\n    {\n        if (key == null)\n        {\n            throw new BadParametersException(\"Key cannot be null\");\n        }\n\n        if(properties != null)\n        {\n            return properties.getProperty(key);\n        }\n        return null;\n    }\n\n    static void init()\n    {\n        String customPath = System.getProperty(CONFIGURATION_KEY, null);\n\n        InputStream in;\n        if (customPath == null || customPath.length() == 0)\n        {\n            in = getResource('/' + FILE_NAME);\n        }\n        else\n        {\n            in = getResource(customPath);\n        }\n\n        Properties props = new Properties();\n\n        if (in != null)\n        {\n            try\n            {\n                props.load(in);\n            }\n            catch (IOException e)\n            {\n                //\n            }\n        }\n        else\n        {\n            LOG.debug(\"Cannot find any properties file.\");\n        }\n\n        properties = props;\n    }\n\n    static InputStream getResource(String resource)\n    {\n        ClassLoader classLoader;\n        InputStream in;\n\n        try\n        {\n            classLoader = Thread.currentThread().getContextClassLoader();\n            if (classLoader != null)\n            {\n                in = classLoader.getResourceAsStream(resource);\n                if (in != null)\n                {\n                    return in;\n                }\n            }\n\n            // Try with the class loader that loaded this class\n            classLoader = PropertyReader.class.getClassLoader();\n            if (classLoader != null)\n            {\n                in = classLoader.getResourceAsStream(resource);\n                if (in != null)\n                {\n                    return in;\n                }\n            }\n        }\n        catch (Exception e)\n        {\n            LOG.warn(\"\", e);\n        }\n\n        // Get the resource from the class path in case that the class is loaded\n        // by the Extension class loader which the parent of the system class loader.\n        in = ClassLoader.getSystemResourceAsStream(resource);\n        if (in != null)\n        {\n            return in;\n        }\n\n        try\n        {\n            return new FileInputStream(resource);\n        }\n        catch (FileNotFoundException e)\n        {\n            return PropertyReader.class.getResourceAsStream(resource);\n        }\n\n    }\n\n    static\n    {\n        init();\n    }\n\n}\n\n// Path: src/main/java/com/password4j/Utils.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\n\nimport java.io.PrintStream;\nimport java.math.BigInteger;\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.nio.CharBuffer;\nimport java.nio.charset.Charset;\nimport java.nio.charset.CharsetEncoder;\nimport java.nio.charset.CodingErrorAction;\nimport java.nio.charset.StandardCharsets;\nimport java.security.*;\nimport java.util.Arrays;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Random;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n\nclass Utils\n{\n\n    static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;\n\n    private static final char[] HEX_ALPHABET = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };\n\n    private static final char[] TO_BASE64 = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q',\n            'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',\n            'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+',\n            '/' };\n\n    private static final int[] FROM_BASE64 = new int[256];\n\n    static\n    {\n        Arrays.fill(FROM_BASE64, -1);\n        for (int i = 0; i < TO_BASE64.length; i++)\n        {\n            FROM_BASE64[TO_BASE64[i]] = i;\n        }\n        FROM_BASE64['='] = -2;\n    }\n\n    private static final Pattern STRONG_PATTERN = Pattern.compile(\"\\\\s*([\\\\S&&[^:,]]*)(\\\\:([\\\\S&&[^,]]*))?\\\\s*(\\\\,(.*))?\");\n\n    static final int AVAILABLE_PROCESSORS = Runtime.getRuntime().availableProcessors();\n\n    private Utils()\n    {\n        //\n    }\n\n    static byte[] fromCharSequenceToBytes(CharSequence charSequence)\n    {\n        return fromCharSequenceToBytes(charSequence, DEFAULT_CHARSET);\n    }\n\n    static int[] fromStringToUnsignedInts(String charSequence)\n    {\n        byte[] byteArray = charSequence.getBytes(DEFAULT_CHARSET);\n        int[] ints = new int[byteArray.length];\n        for(int i = 0; i < ints.length; i++) {\n            ints[i] = Byte.toUnsignedInt(byteArray[i]);\n        }\n        return ints;\n    }\n\n    static byte[] fromCharSequenceToBytes(CharSequence charSequence, Charset charset)\n    {\n        if (charSequence == null)\n        {\n            return new byte[0];\n        }\n        CharsetEncoder encoder = charset.newEncoder();\n        int length = charSequence.length();\n        int arraySize = scale(length, encoder.maxBytesPerChar());\n        byte[] result = new byte[arraySize];\n        if (length == 0)\n        {\n            return result;\n        }\n        else\n        {\n            char[] charArray;\n            if (charSequence instanceof String)\n            {\n                charArray = ((String) charSequence).toCharArray();\n            }\n            else\n            {\n                charArray = fromCharSequenceToChars(charSequence);\n            }\n\n            charArray = Arrays.copyOfRange(charArray, 0, length);\n\n            encoder.onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE).reset();\n\n            ByteBuffer byteBuffer = ByteBuffer.wrap(result);\n            CharBuffer charBuffer = CharBuffer.wrap(charArray, 0, length);\n\n            encoder.encode(charBuffer, byteBuffer, true);\n            encoder.flush(byteBuffer);\n\n            return Arrays.copyOf(result, byteBuffer.position());\n        }\n\n    }\n\n    \nstatic char[] fromCharSequenceToChars(CharSequence charSequence)\n    {\n        if (charSequence == null || charSequence.length() == 0)\n        {\n            return new char[0];\n        }\n        char[] result = new char[charSequence.length()];\n        for (int i = 0; i < charSequence.length(); i++)\n        {\n            result[i] = charSequence.charAt(i);\n        }\n\n        return result;\n    }\n\n    static char[] fromBytesToChars(byte[] bytes)\n    {\n        return new String(bytes, DEFAULT_CHARSET).toCharArray();\n    }\n\n    static CharSequence append(CharSequence cs1, CharSequence cs2)\n    {\n        if (cs1 == null || cs1.length() == 0)\n        {\n            return cs2;\n        }\n\n        if (cs2 == null || cs2.length() == 0)\n        {\n            return cs1;\n        }\n\n        char[] charArray1 = fromCharSequenceToChars(cs1);\n        char[] charArray2 = fromCharSequenceToChars(cs2);\n\n        char[] result = new char[charArray1.length + charArray2.length];\n        System.arraycopy(charArray1, 0, result, 0, charArray1.length);\n        System.arraycopy(charArray2, 0, result, charArray1.length, charArray2.length);\n\n        return new SecureString(result);\n\n    }\n\n    static byte[] append(byte[] byteArray1, byte[] byteArray2)\n    {\n        byte[] result = new byte[byteArray1.length + byteArray2.length];\n        System.arraycopy(byteArray1, 0, result, 0, byteArray1.length);\n        System.arraycopy(byteArray2, 0, result, byteArray1.length, byteArray2.length);\n        return result;\n    }\n\n    static String toHex(byte[] bytes)\n    {\n        final int length = bytes.length;\n        final char[] output = new char[length << 1];\n        int j = 0;\n        for (byte aByte : bytes)\n        {\n            output[j++] = HEX_ALPHABET[(0xF0 & aByte) >>> 4];\n            output[j++] = HEX_ALPHABET[0x0F & aByte];\n        }\n        return new String(output);\n    }\n\n    static BigInteger bytesToInt(byte[] bytes)\n    {\n        for (int i = 0; i < bytes.length / 2; i++) {\n            byte temp = bytes[i];\n            bytes[i] = bytes[bytes.length - i - 1];\n            bytes[bytes.length - i - 1] = temp;\n        }\n        return new BigInteger(1, bytes);\n    }\n\n    static long littleEndianToLong(byte[] bs, int off)\n    {\n        int lo = littleEndianToInt(bs, off);\n        int hi = littleEndianToInt(bs, off + 4);\n        return ((hi & 0xffffffffL) << 32) | (lo & 0xffffffffL);\n    }\n\n    static int littleEndianToInt(byte[] bs, int off)\n    {\n        int n = bs[off] & 0xff;\n        n |= (bs[++off] & 0xff) << 8;\n        n |= (bs[++off] & 0xff) << 16;\n        n |= bs[++off] << 24;\n        return n;\n    }\n\n    static byte[] longToLittleEndian(long n)\n    {\n        byte[] bs = new byte[8];\n        longToLittleEndian(n, bs, 0);\n        return bs;\n    }\n\n    static void longToLittleEndian(long n, byte[] bs, int off)\n    {\n        intToLittleEndian((int) (n & 0xffffffffL), bs, off);\n        intToLittleEndian((int) (n >>> 32), bs, off + 4);\n    }\n\n    static void intToLittleEndian(int n, byte[] bs, int off)\n    {\n        bs[off] = (byte) (n);\n        bs[++off] = (byte) (n >>> 8);\n        bs[++off] = (byte) (n >>> 16);\n        bs[++off] = (byte) (n >>> 24);\n    }\n\n    static byte[] intToLittleEndianBytes(int a)\n    {\n        byte[] result = new byte[4];\n        result[0] = (byte) (a & 0xFF);\n        result[1] = (byte) ((a >> 8) & 0xFF);\n        result[2] = (byte) ((a >> 16) & 0xFF);\n        result[3] = (byte) ((a >> 24) & 0xFF);\n        return result;\n    }\n\n    static byte[] intToLittleEndianBytes(int a, int length)\n    {\n        return ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN).putInt(a).array();\n    }\n\n    static long[] fromBytesToLongs(byte[] input)\n    {\n        long[] v = new long[128];\n        for (int i = 0; i < v.length; i++)\n        {\n            byte[] slice = Arrays.copyOfRange(input, i * 8, (i + 1) * 8);\n            v[i] = littleEndianBytesToLong(slice);\n        }\n        return v;\n    }\n\n    static String fromBytesToString(byte[] input)\n    {\n        return new String(input, DEFAULT_CHARSET);\n    }\n\n    static long littleEndianBytesToLong(byte[] b)\n    {\n        long result = 0;\n        for (int i = 7; i >= 0; i--)\n        {\n            result <<= 8;\n            result |= (b[i] & 0xFF);\n        }\n        return result;\n    }\n\n    static byte[] longToLittleEndianBytes(long a)\n    {\n        byte[] result = new byte[8];\n        result[0] = (byte) (a & 0xFF);\n        result[1] = (byte) ((a >> 8) & 0xFF);\n        result[2] = (byte) ((a >> 16) & 0xFF);\n        result[3] = (byte) ((a >> 24) & 0xFF);\n        result[4] = (byte) ((a >> 32) & 0xFF);\n        result[5] = (byte) ((a >> 40) & 0xFF);\n        result[6] = (byte) ((a >> 48) & 0xFF);\n        result[7] = (byte) ((a >> 56) & 0xFF);\n        return result;\n    }\n\n    static long intToLong(int x)\n    {\n        byte[] intBytes = intToLittleEndianBytes(x);\n        byte[] bytes = new byte[8];\n        System.arraycopy(intBytes, 0, bytes, 0, 4);\n        return littleEndianBytesToLong(bytes);\n    }\n\n    static void xor(long[] t, long[] b1, long[] b2)\n    {\n        for (int i = 0; i < t.length; i++)\n        {\n            t[i] = b1[i] ^ b2[i];\n        }\n    }\n\n    static void xor(long[] t, long[] b1, long[] b2, long[] b3)\n    {\n        for (int i = 0; i < t.length; i++)\n        {\n            t[i] = b1[i] ^ b2[i] ^ b3[i];\n        }\n    }\n\n    static void xor(long[] t, long[] other)\n    {\n        for (int i = 0; i < t.length; i++)\n        {\n            t[i] = t[i] ^ other[i];\n        }\n    }\n\n    static int log2(int number)\n    {\n        int log = 0;\n        if ((number & -65536) != 0)\n        {\n            number >>>= 16;\n            log = 16;\n        }\n        if (number >= 256)\n        {\n            number >>>= 8;\n            log += 8;\n        }\n        if (number >= 16)\n        {\n            number >>>= 4;\n            log += 4;\n        }\n        if (number >= 4)\n        {\n            number >>>= 2;\n            log += 2;\n        }\n        return log + (number >>> 1);\n    }\n\n    private static int scale(int initialLength, float bytesPerChar)\n    {\n        return (int) ((double) initialLength * (double) bytesPerChar);\n    }\n\n    static byte[] decodeBase64(String src)\n    {\n        return decodeBase64(src.getBytes(DEFAULT_CHARSET));\n    }\n\n    static String encodeBase64(byte[] src)\n    {\n        return encodeBase64(src, true);\n    }\n\n    static String encodeBase64(byte[] src, boolean padding)\n    {\n        byte[] encoded = encode(src, padding);\n        return new String(encoded, 0, encoded.length);\n    }\n\n    static byte[] decodeBase64(byte[] src)\n    {\n        byte[] dst = new byte[outLength(src, src.length)];\n        int ret = decode(src, src.length, dst);\n        if (ret != dst.length)\n        {\n            dst = Arrays.copyOf(dst, ret);\n        }\n        return dst;\n    }\n\n    static byte[] encode(byte[] src, boolean padding)\n    {\n        int len = outLength(src.length, padding);\n        byte[] dst = new byte[len];\n        int ret = encode(src, src.length, dst, padding);\n        if (ret != dst.length)\n        {\n            return Arrays.copyOf(dst, ret);\n        }\n        return dst;\n    }\n\n    private static int outLength(int length, boolean doPadding)\n    {\n        int len;\n        if (doPadding)\n        {\n            len = 4 * ((length + 2) / 3);\n        }\n        else\n        {\n            int n = length % 3;\n            len = 4 * (length / 3) + (n == 0 ? 0 : n + 1);\n        }\n        return len;\n    }\n\n    private static int outLength(byte[] source, int length)\n    {\n        int paddings = 0;\n        if (length == 0)\n        {\n            return 0;\n        }\n        if (length < 2)\n        {\n            throw new IllegalArgumentException(\"Input byte[] should at least have 2 bytes for base64 bytes\");\n        }\n\n        if (source[length - 1] == '=')\n        {\n            paddings++;\n            if (source[length - 2] == '=')\n            {\n                paddings++;\n            }\n        }\n\n        if (paddings == 0 && (length & 0x3) != 0)\n        {\n            paddings = 4 - (length & 0x3);\n        }\n        return 3 * ((length + 3) / 4) - paddings;\n    }\n\n    private static int encode(byte[] src, int end, byte[] dst, boolean padding)\n    {\n        char[] base64 = TO_BASE64;\n        int sp = 0;\n        int length = (end) / 3 * 3;\n        int dp = 0;\n        while (sp < length)\n        {\n            int sl0 = sp + length;\n            for (int sp0 = sp, dp0 = dp; sp0 < sl0; sp0 += 3, dp0 += 4)\n            {\n                int bits = (src[sp0] & 0xff) << 16 | (src[sp0 + 1] & 0xff) << 8 | (src[sp0 + 2] & 0xff);\n                dst[dp0] = (byte) base64[(bits >>> 18) & 0x3f];\n                dst[dp0 + 1] = (byte) base64[(bits >>> 12) & 0x3f];\n                dst[dp0 + 2] = (byte) base64[(bits >>> 6) & 0x3f];\n                dst[dp0 + 3] = (byte) base64[bits & 0x3f];\n            }\n            int dlen = (sl0 - sp) / 3 * 4;\n            dp += dlen;\n            sp = sl0;\n        }\n        if (sp < end)\n        {\n            int b0 = src[sp++] & 0xff;\n            dst[dp++] = (byte) base64[b0 >> 2];\n            if (sp == end)\n            {\n                dst[dp++] = (byte) base64[(b0 << 4) & 0x3f];\n                if (padding)\n                {\n                    dst[dp++] = '=';\n                    dst[dp++] = '=';\n                }\n            }\n            else\n            {\n                int b1 = src[sp] & 0xff;\n                dst[dp++] = (byte) base64[(b0 << 4) & 0x3f | (b1 >> 4)];\n                dst[dp++] = (byte) base64[(b1 << 2) & 0x3f];\n                if (padding)\n                {\n                    dst[dp++] = '=';\n                }\n            }\n        }\n        return dp;\n    }\n\n    private static int decode(byte[] src, int sl, byte[] dst)\n    {\n        int dp = 0;\n        int bits = 0;\n        int sp = 0;\n        int shiftTo = 18;\n        while (sp < sl)\n        {\n            int b = src[sp++] & 0xff;\n            if ((b = FROM_BASE64[b]) < 0)\n            {\n                if (b == -2)\n                {\n                    if (shiftTo == 6 && (sp == sl || src[sp] != '=') || shiftTo == 18)\n                    {\n                        throw new IllegalArgumentException(\"Input byte array has wrong 4-byte ending unit\");\n                    }\n                    break;\n                }\n                else\n                    throw new IllegalArgumentException(\"Illegal base64 character \" + Integer.toString(src[sp - 1], 16));\n            }\n            bits |= (b << shiftTo);\n            shiftTo -= 6;\n            if (shiftTo < 0)\n            {\n                dst[dp++] = (byte) (bits >> 16);\n                dst[dp++] = (byte) (bits >> 8);\n                dst[dp++] = (byte) (bits);\n                shiftTo = 18;\n                bits = 0;\n            }\n        }\n        if (shiftTo == 6)\n        {\n            dst[dp++] = (byte) (bits >> 16);\n        }\n        else if (shiftTo == 0)\n        {\n            dst[dp++] = (byte) (bits >> 16);\n            dst[dp++] = (byte) (bits >> 8);\n        }\n        else if (shiftTo == 12)\n        {\n            throw new IllegalArgumentException(\"Last unit does not have enough valid bits\");\n        }\n        return dp;\n    }\n\n    @SuppressWarnings({\"removal\", \"java:S1604\"})\n    static SecureRandom getInstanceStrong() throws NoSuchAlgorithmException\n    {\n        String property = AccessController.doPrivileged(new PrivilegedAction<String>()\n        {\n            @Override\n            public String run()\n            {\n                return Security.getProperty(\"securerandom.strongAlgorithms\");\n            }\n        });\n\n        if ((property == null) || (property.length() == 0))\n        {\n            throw new NoSuchAlgorithmException(\"Null/empty securerandom.strongAlgorithms Security Property\");\n        }\n\n        String remainder = property;\n        while (remainder != null)\n        {\n            Matcher m = STRONG_PATTERN.matcher(remainder);\n            if (m.matches())\n            {\n                String alg = m.group(1);\n                String prov = m.group(3);\n\n                try\n                {\n                    if (prov == null)\n                    {\n                        return SecureRandom.getInstance(alg);\n                    }\n                    else\n                    {\n                        return SecureRandom.getInstance(alg, prov);\n                    }\n                }\n                catch (NoSuchAlgorithmException | NoSuchProviderException e)\n                {\n                    //\n                }\n                remainder = m.group(5);\n            }\n            else\n            {\n                remainder = null;\n            }\n        }\n\n        throw new NoSuchAlgorithmException(\"No strong SecureRandom impls available: \" + property);\n    }\n\n    static String randomPrintable(int count)\n    {\n        Random random = AlgorithmFinder.getSecureRandom();\n\n        StringBuilder builder = new StringBuilder(count);\n        int start = 32;\n        int gap = 126 - start;\n\n        while (count-- != 0)\n        {\n            int codePoint = random.nextInt(gap) + start;\n            builder.appendCodePoint(codePoint);\n        }\n        return builder.toString();\n    }\n\n    static void printBanner(PrintStream printStream)\n    {\n        if (PropertyReader.readBoolean(\"global.banner\", false))\n        {\n            String pbkdf2Banner;\n            List<String> pbkd2s = AlgorithmFinder.getAllPBKDF2Variants();\n            if (!pbkd2s.isEmpty())\n            {\n                pbkdf2Banner = \"\u2705 PBKDF2-\" + String.join(\"/\", pbkd2s).replace(\"PBKDF2WithHmac\", \"\");\n            }\n            else\n            {\n                pbkdf2Banner = \"\u274c PBKDF2 <-- not supported by \" + System.getProperty(\"java.vm.name\");\n            }\n\n            String banner =\"\\n\";\n            banner += \"    |\\n\" +\n                    \"    |                \\033[0;1mPassword4j\\033[0;0m\\n\" +\n                    \"    + \\\\             .: v1.8.0 :.\\n\" +\n                    \"    \\\\\\\\.G_.*=.\\n\" +\n                    \"     `(H'/.\\\\|        \u2705 Argon2\\n\" +\n                    \"      .>' (_--.      \u2705 scrypt\\n\" +\n                    \"   _=/d   ,^\\\\        \u2705 bcrypt\\n\" +\n                    \" ~~ \\\\)-'-'           \" + pbkdf2Banner + \"\\n\" +\n                    \"    / |              \u2705 balloon hashing\\n\" +\n                    \"    '  '\";\n            banner += \"\\n\";\n            banner += \" \u2b50 If you enjoy Password4j, please star the project at https://github.com/Password4j/password4j\\n\";\n            banner += \" \\uD83E\\uDEB2  Report any issue at https://github.com/Password4j/password4j/issues\\n\";\n\n            printStream.println(banner);\n\n        }\n    }\n\n    static List<byte[]> split(byte[] array, byte delimiter) {\n        List<byte[]> byteArrays = new LinkedList<>();\n\n        int begin = 0;\n\n        for (int i = 0; i < array.length; i++) {\n\n            if (array[i] != delimiter) {\n                continue;\n            }\n\n            byteArrays.add(Arrays.copyOfRange(array, begin, i));\n            begin = i + 1;\n        }\n        byteArrays.add(Arrays.copyOfRange(array, begin, array.length));\n        return byteArrays;\n    }\n\n\n\n}\n\n// Path: src/main/java/com/password4j/types/Argon2.java\n/*\n *  (C) Copyright 2021 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j.types;\n\n/**\n * Enum containing the different variations of Argon2.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Argon2\">Argon2</a>\n * @since 1.5.0\n */\npublic enum Argon2\n{\n    /**\n     * It maximizes resistance to GPU cracking attacks.\n     * It accesses the memory array in a password dependent order, which reduces the possibility of time\u2013memory trade-off (TMTO) attacks,\n     * but introduces possible side-channel attacks\n     */\n    D,\n\n    /**\n     * It is optimized to resist side-channel attacks. It accesses the memory array in a password independent order.\n     */\n    I,\n\n    /**\n     * It is a hybrid version. It follows the Argon2i approach for the first half pass over memory and the Argon2d approach for subsequent passes.\n     * It is recommended to use Argon2id except when there are reasons to prefer one of the other two modes.\n     */\n    ID;\n\n}\n\n// Path: src/main/java/com/password4j/Argon2Function.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Argon2;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.*;\n\n\n/**\n * Class containing the implementation of Argon2 function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Argon2\">Argon2</a>\n * @since 1.5.0\n */\npublic class Argon2Function extends AbstractHashingFunction\n{\n    public static final int ARGON2_VERSION_10 = 0x10;\n\n    public static final int ARGON2_VERSION_13 = 0x13;\n\n    public static final int ARGON2_INITIAL_DIGEST_LENGTH = 64;\n\n    public static final int ARGON2_ADDRESSES_IN_BLOCK = 128;\n\n    private static final ConcurrentMap<String, Argon2Function> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final int ARGON2_SYNC_POINTS = 4;\n\n    private static final int ARGON2_INITIAL_SEED_LENGTH = 72;\n\n    private static final int ARGON2_BLOCK_SIZE = 1024;\n\n    public static final int ARGON2_QWORDS_IN_BLOCK = ARGON2_BLOCK_SIZE / 8;\n\n    private ExecutorService service;\n\n    private final int iterations;\n\n    private final int memory;\n\n    private final long[][] initialBlockMemory;\n\n    private final int parallelism;\n\n    private final int outputLength;\n\n    private final int segmentLength;\n\n    private final Argon2 variant;\n\n    private final int version;\n\n    private final int laneLength;\n\n    Argon2Function(int memory, int iterations, int parallelism, int outputLength, Argon2 variant, int version)\n    {\n        this.variant = variant;\n        this.iterations = iterations;\n        this.memory = memory;\n        this.parallelism = parallelism;\n        this.outputLength = outputLength;\n        this.version = version;\n\n        int memoryBlocks = this.memory;\n\n        if (this.memory < 2 * ARGON2_SYNC_POINTS * parallelism)\n        {\n            memoryBlocks = 2 * ARGON2_SYNC_POINTS * parallelism;\n        }\n\n        segmentLength = memoryBlocks / (parallelism * ARGON2_SYNC_POINTS);\n        this.laneLength = segmentLength * ARGON2_SYNC_POINTS;\n\n        memoryBlocks = segmentLength * (parallelism * ARGON2_SYNC_POINTS);\n\n        initialBlockMemory = new long[memoryBlocks][ARGON2_QWORDS_IN_BLOCK];\n        for (int i = 0; i < memoryBlocks; i++)\n        {\n            initialBlockMemory[i] = new long[ARGON2_QWORDS_IN_BLOCK];\n        }\n\n        if (parallelism >= 1)\n        {\n            service = Executors.newFixedThreadPool(Utils.AVAILABLE_PROCESSORS);\n        }\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * memory (KiB), number of iterations, parallelism, length og the output and type.\n     *\n     * @param memory       memory (KiB)\n     * @param iterations   number of iterations\n     * @param parallelism  level of parallelism\n     * @param outputLength length of the final hash\n     * @param type         argon2 type (i, d or id)\n     * @return a singleton instance\n     * @since 1.5.0\n     */\n    public static Argon2Function getInstance(int memory, int iterations, int parallelism, int outputLength, Argon2 type)\n    {\n        return getInstance(memory, iterations, parallelism, outputLength, type, ARGON2_VERSION_13);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * logarithmic memory, number of iterations, parallelism, lenght og the output, type and version.\n     *\n     * @param memory       logarithmic memory\n     * @param iterations   number of iterations\n     * @param parallelism  level of parallelism\n     * @param outputLength length of the final hash\n     * @param type         argon2 type (i, d or id)\n     * @param version      version of the algorithm (16 or 19)\n     * @return a singleton instance\n     * @since 1.5.0\n     */\n    public static Argon2Function getInstance(int memory, int iterations, int parallelism, int outputLength, Argon2 type,\n            int version)\n    {\n        String key = getUID(memory, iterations, parallelism, outputLength, type, version);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            Argon2Function function = new Argon2Function(memory, iterations, parallelism, outputLength, type, version);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.5.0\n     */\n    public static Argon2Function getInstanceFromHash(String hashed)\n    {\n        Object[] params = decodeHash(hashed);\n        Argon2 type = Argon2.valueOf(((String) params[0]).toUpperCase());\n        int version = (int) params[1];\n        int memory = (int) params[2];\n        int iterations = (int) params[3];\n        int parallelism = (int) params[4];\n        int outputLength = ((byte[]) params[6]).length;\n        return getInstance(memory, iterations, parallelism, outputLength, type, version);\n    }\n\n    @Override\n...\n", "needle_token_start": 2751, "needle_token_end": 2865, "code_context_ntokens": 10043, "output": ["```java\nstatic char[] fromCharSequenceToChars(CharSequence charSequence)\n{\n    if (charSequence == null || charSequence.length() == 0)\n    {\n        return new char[0];\n    }\n    char[] result = new char[charSequence.length()];\n    for (int i = 0; i < charSequence.length(); i++)\n    {\n        result[i] = charSequence.charAt(i);\n    }\n\n    return result;\n}\n```"]}
{"repo": "Password4j/password4j", "name": "getSecureRandom", "language": "java", "path": "src/main/java/com/password4j/AlgorithmFinder.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide a secure, thread-safe instance of a random number generator that is cryptographically strong. This is crucial for applications requiring high levels of randomness, such as in cryptographic operations.\n2. **Input**: There are no parameters required for this function.\n3. **Output**: Returns a singleton instance of a cryptographically strong random number generator.\n4. **Procedure**: The function checks if a singleton instance already exists. If not, it initializes this instance based on system configurations and properties specified in a configuration file. The initialization can be influenced by settings that dictate whether to use a particularly strong random number generator, which might impact performance but increases security.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/types/Hmac.java\n/*\n *  (C) Copyright 2021 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j.types;\n\nimport com.password4j.CompressedPBKDF2Function;\n\n\n/**\n...\n// Path: src/main/java/com/password4j/CompressedPBKDF2Function.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of PBKDF2 function and its parameters.\n * <p>\n * The main difference between {@link PBKDF2Function} is the hash produced: the configurations of the CHF,\n * the salt and the hash are encoded inside it.\n * <p>\n * The produced hash is in the form\n * <p>\n * <code>\n * $algorithm$parameters$salt$hash\n * </code>\n * <p>\n * Assuming {@code $} as delimiter.\n * <p>\n * <ul>\n *     <li>\n *         The algorithm is encoded with its numeric uid {@link Hmac#code()}\n *     </li>\n *     <li>\n *         Parameters are encoded in one integer where the length occupies the first 32bit and\n *          the number of iterations the remaining 32 bits.\n *     </li>\n *     <li>\n *         Salt is encoded in Base64\n *     </li>\n *     <li>\n *         Hash is encoded in Base64 as in {@link PBKDF2Function}\n *     </li>\n * </ul>\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/PBKDF2\">PBKDF2</a>\n * @since 0.1.0\n */\npublic class CompressedPBKDF2Function extends PBKDF2Function\n{\n\n    private static final Map<String, CompressedPBKDF2Function> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final char DELIMITER = PropertyReader.readChar(\"hash.pbkdf2.delimiter\", '$');\n\n    protected CompressedPBKDF2Function()\n    {\n        super();\n    }\n\n    protected CompressedPBKDF2Function(Hmac fromCode, int iterations, int length)\n    {\n        super(fromCode, iterations, length);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getInstance(Hmac algorithm, int iterations, int length)\n    {\n        String key = getUID(algorithm.name(), iterations, length);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            CompressedPBKDF2Function function = new CompressedPBKDF2Function(algorithm, iterations, length);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  string version of hmac algorithm. This must me mapped in {@link Hmac}.\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @throws IllegalArgumentException if {@code algorithm} is not mapped in {@link Hmac}.\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getInstance(String algorithm, int iterations, int length)\n    {\n        try\n        {\n            return getInstance(Hmac.valueOf(algorithm), iterations, length);\n        }\n        catch (IllegalArgumentException iae)\n        {\n            throw new UnsupportedOperationException(\"Algorithm `\" + algorithm + \"` is not recognized.\", iae);\n        }\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.0.0\n     */\n    public static CompressedPBKDF2Function getInstanceFromHash(String hashed)\n    {\n        String[] parts = getParts(hashed);\n        if (parts.length == 5)\n        {\n            int algorithm = Integer.parseInt(parts[1]);\n            long configuration = Long.parseLong(parts[2]);\n\n            int iterations = (int) (configuration >> 32);\n            int length = (int) configuration;\n\n            return CompressedPBKDF2Function.getInstance(Hmac.fromCode(algorithm), iterations, length);\n        }\n        throw new BadParametersException(\"`\" + hashed + \"` is not a valid hash\");\n    }\n\n\n\n    @Override\n    protected String getHash(byte[] encodedKey, byte[] salt)\n    {\n        String params = Long.toString((((long) getIterations()) << 32) | (getLength() & 0xffffffffL));\n        String salt64 = Utils.encodeBase64(salt);\n        String hash64 = super.getHash(encodedKey, salt);\n        return \"$\" + algorithm.code() + \"$\" + params + \"$\" + salt64 + \"$\" + hash64;\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        byte[] salt = getSaltFromHash(hashed);\n        Hash internalHas = hash(plainTextPassword, salt);\n\n        return slowEquals(internalHas.getResultAsBytes(), hashed);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        byte[] hashAsBytes = Utils.fromCharSequenceToBytes(hashed);\n        byte[] realSalt = getSaltFromHash(hashAsBytes);\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        Hash internalHash = hash(plainTextPasswordAsBytes, realSalt);\n        return slowEquals(internalHash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        byte[] realSalt = getSaltFromHash(hashed);\n        Hash internalHash = hash(plainTextPassword, realSalt);\n        return slowEquals(internalHash.getResultAsBytes(), hashed);\n    }\n\n    protected static List<byte[]> getParts(byte[] hashed)\n    {\n       return Utils.split(hashed, (byte) DELIMITER);\n    }\n\n    protected static String[] getParts(String hashed)\n    {\n        String regex = \"\\\\\" + DELIMITER;\n        return hashed.split(regex);\n    }\n\n    private byte[] getSaltFromHash(byte[] hashed)\n    {\n        List<byte[]> parts = getParts(hashed);\n        if (parts.size() == 5)\n        {\n            return Utils.decodeBase64(parts.get(3));\n        }\n        throw new BadParametersException(\"`\" + Utils.fromBytesToString(hashed) + \"` is not a valid hash\");\n    }\n}\n\n// Path: src/main/java/com/password4j/SaltOption.java\npackage com.password4j;\n\npublic enum SaltOption\n{\n    PREPEND, APPEND;\n}\n\n// Path: src/main/java/com/password4j/MessageDigestFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of Messaged digest functions provided by {@link MessageDigest}.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Category:Cryptographic_hash_functions\">Message digests</a>\n * @since 1.4.0\n */\npublic class MessageDigestFunction extends AbstractHashingFunction\n{\n    private static final Map<String, MessageDigestFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    protected static final SaltOption DEFAULT_SALT_OPTION = SaltOption.APPEND;\n\n    private final String algorithm;\n\n    private final SaltOption saltOption;\n\n\n    MessageDigestFunction(String algorithm, SaltOption saltOption)\n    {\n        this.algorithm = algorithm;\n        this.saltOption = saltOption;\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm message digest algorithm\n     * @return a singleton instance\n     * @since 1.4.0\n     */\n    public static MessageDigestFunction getInstance(String algorithm)\n    {\n        return getInstance(algorithm, DEFAULT_SALT_OPTION);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param saltOption a configuration that specifies how the salt is concatenated to the plain text password\n     * @return a singleton instance\n     * @since 1.4.0\n     */\n    public static MessageDigestFunction getInstance(String algorithm, SaltOption saltOption)\n    {\n        String key = getUID(algorithm, saltOption);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            MessageDigestFunction function = new MessageDigestFunction(algorithm, saltOption);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    protected static String getUID(String algorithm, SaltOption saltOption)\n    {\n        return algorithm + \"|\" + saltOption.name();\n    }\n\n    protected static String toString(String algorithm, SaltOption saltOption)\n    {\n        return \"a=\" + algorithm + \", o=\" + saltOption.name();\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        return hash(plainTextPassword, null);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        return hash(plainTextPasswordAsBytes, null);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes, byte[] saltAsBytes)\n    {\n        return internalHash(plainTextPasswordAsBytes, saltAsBytes);\n    }\n\n    protected Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n        byte[] finalCharSequence = concatenateSalt(plainTextPassword, salt);\n\n        byte[] result = getMessageDigest().digest(finalCharSequence);\n        return new Hash(this, Utils.toHex(result), result, salt);\n    }\n\n    protected MessageDigest getMessageDigest()\n    {\n        try\n        {\n            return MessageDigest.getInstance(algorithm);\n        } catch (NoSuchAlgorithmException nsae)\n        {\n            throw new UnsupportedOperationException(\"`\" + algorithm + \"` is not supported by your system.\", nsae);\n        }\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(plainTextPassword, hashed, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPasswordAsBytes, byte[] hashed)\n    {\n        return check(plainTextPasswordAsBytes, hashed, null);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        Hash hash = internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n        return slowEquals(hash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        Hash hash = internalHash(plainTextPassword, salt);\n        return slowEquals(hash.getResultAsBytes(), hashed);\n    }\n\n    /**\n     * The salt option describes if the Salt is appended or prepended to\n     * the plain text password.\n     *\n     * @return how the salt is concatenated\n     * @since 1.5.1\n     */\n    public SaltOption getSaltOption()\n    {\n        return saltOption;\n    }\n\n    /**\n     * The algorithm in use by this instance.\n     *\n     * @return the algorithm in use\n     * @since 1.5.1\n     */\n    public String getAlgorithm()\n    {\n        return algorithm;\n    }\n\n\n    private byte[] concatenateSalt(byte[] plainTextPassword, byte[] salt)\n    {\n        if (salt == null || salt.length == 0)\n        {\n            return  plainTextPassword;\n        }\n\n        if (saltOption == SaltOption.PREPEND)\n        {\n            return Utils.append(salt, plainTextPassword);\n        }\n        return Utils.append(plainTextPassword, salt);\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(this.algorithm, this.saltOption) + ')';\n    }\n\n    @Override\n    public boolean equals(Object o)\n    {\n        if (this == o)\n            return true;\n        if (!(o instanceof MessageDigestFunction))\n            return false;\n        MessageDigestFunction other = (MessageDigestFunction) o;\n        return algorithm.equals(other.algorithm) //\n                && saltOption == other.saltOption;\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(algorithm, saltOption);\n    }\n}\n\n// Path: src/main/java/com/password4j/ScryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport java.security.GeneralSecurityException;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of scrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Scrypt\">scrypt</a>\n * @since 0.1.0\n */\npublic class ScryptFunction extends AbstractHashingFunction\n{\n    public static final int DERIVED_KEY_LENGTH = 64;\n\n    private static final ConcurrentMap<String, ScryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private int workFactor; // N\n\n    private int resources; // r\n\n    private int parallelization; // p\n\n    private int derivedKeyLength; // dkLen\n\n    @SuppressWarnings(\"unused\")\n    private ScryptFunction()\n    {\n        //\n    }\n\n    /**\n     * @param workFactor      (N)\n     * @param resources       (r)\n     * @param parallelization (p)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = DERIVED_KEY_LENGTH;\n    }\n\n    /**\n     * @param workFactor       (N)\n     * @param resources        (r)\n     * @param parallelization  (p)\n     * @param derivedKeyLength (dkLen)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization, int derivedKeyLength)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = derivedKeyLength;\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.0.0\n     */\n    public static ScryptFunction getInstanceFromHash(String hashed)\n    {\n        String[] parts = hashed.split(\"\\\\$\");\n        if (parts.length == 4)\n        {\n            long params = Long.parseLong(parts[1], 16);\n            int workFactor = (int) Math.pow(2.0D, (params >> 16 & 65535L));\n            int resources = (int) params >> 8 & 255;\n            int parallelization = (int) params & 255;\n            int derivedKeyLength = Utils.decodeBase64(parts[3]).length;\n\n            return ScryptFunction.getInstance(workFactor, resources, parallelization, derivedKeyLength);\n        }\n        throw new BadParametersException(\"`\" + hashed + \"` is not a valid hash\");\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * N, r and p parameters.\n     *\n     * @param workFactor      work factor (N)\n     * @param resources       resources (r)\n     * @param parallelization parallelization (p)\n     * @return a singleton instance\n     * @since 0.3.0\n     */\n    public static ScryptFunction getInstance(int workFactor, int resources, int parallelization)\n    {\n        return getInstance(workFactor, resources, parallelization, DERIVED_KEY_LENGTH);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * N, r and p parameters.\n     *\n     * @param workFactor       work factor (N)\n     * @param resources        resources (r)\n     * @param parallelization  parallelization (p)\n     * @param derivedKeyLength derived key length (dkLen)\n     * @return a singleton instance\n     * @since 1.5.1\n     */\n    public static ScryptFunction getInstance(int workFactor, int resources, int parallelization, int derivedKeyLength)\n    {\n        String key = getUID(resources, workFactor, parallelization, derivedKeyLength);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            ScryptFunction function = new ScryptFunction(workFactor, resources, parallelization, derivedKeyLength);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        byte[] saltAsBytes = Utils.fromCharSequenceToBytes(salt);\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        return internalHash(plainTextPasswordAsBytes, saltAsBytes);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes, byte[] salt)\n    {\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    private Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n\n        try\n        {\n            byte[] derived = scrypt(plainTextPassword, salt, derivedKeyLength);\n            String params = Long.toString((long) Utils.log2(workFactor) << 16 | (long) resources << 8 | parallelization, 16);\n            String sb = \"$\" + params + '$' + Utils.encodeBase64(salt) + '$'\n                    + Utils.encodeBase64(derived);\n            return new Hash(this, sb, derived, salt);\n        }\n        catch (IllegalArgumentException | GeneralSecurityException e)\n        {\n            String stringedSalt = Utils.fromBytesToString(salt);\n            String message = \"Invalid specification with salt=\" + stringedSalt + \", N=\" + workFactor + \", r=\" + resources + \" and p=\" + parallelization;\n            throw new BadParametersException(message, e);\n        }\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        try\n        {\n            List<byte[]> parts = Utils.split(hashed, (byte) 36);\n            if (parts.size() == 4)\n            {\n                byte[] salt = Utils.decodeBase64(parts.get(2));\n                byte[] derived0 = Utils.decodeBase64(parts.get(3));\n                byte[] derived1 = scrypt(plainTextPassword, salt, derivedKeyLength);\n                return slowEquals(derived0, derived1);\n            }\n            else\n            {\n                throw new BadParametersException(\"Invalid hashed value\");\n            }\n        }\n        catch (GeneralSecurityException gse)\n        {\n            throw new IllegalStateException(\"JVM doesn't support SHA1PRNG or HMAC_SHA256?\", gse);\n        }\n    }\n\n    protected static String toString(int resources, int workFactor, int parallelization, int derivedKeyLength)\n    {\n        return \"N=\" + workFactor + \", r=\" + resources + \", p=\" + parallelization + \", l=\" + derivedKeyLength;\n    }\n\n    protected static String getUID(int resources, int workFactor, int parallelization, int derivedKeyLength)\n    {\n        return workFactor + \"|\" + resources + \"|\" + parallelization + \"|\" + derivedKeyLength;\n    }\n\n    public static int rOperation(int a, int b)\n    {\n        return a << b | a >>> 32 - b;\n    }\n\n    public static void salsa208(byte[] xArray)\n    {\n        int[] b32 = new int[16];\n        int[] x = new int[16];\n\n        int i;\n        for (i = 0; i < 16; ++i)\n        {\n            b32[i] = (xArray[i * 4] & 255);\n            b32[i] |= (xArray[i * 4 + 1] & 255) << 8;\n            b32[i] |= (xArray[i * 4 + 2] & 255) << 16;\n            b32[i] |= (xArray[i * 4 + 3] & 255) << 24;\n        }\n\n        System.arraycopy(b32, 0, x, 0, 16);\n\n        for (i = 8; i > 0; i -= 2)\n        {\n            x[4] ^= rOperation(x[0] + x[12], 7);\n            x[8] ^= rOperation(x[4] + x[0], 9);\n            x[12] ^= rOperation(x[8] + x[4], 13);\n            x[0] ^= rOperation(x[12] + x[8], 18);\n            x[9] ^= rOperation(x[5] + x[1], 7);\n            x[13] ^= rOperation(x[9] + x[5], 9);\n            x[1] ^= rOperation(x[13] + x[9], 13);\n            x[5] ^= rOperation(x[1] + x[13], 18);\n            x[14] ^= rOperation(x[10] + x[6], 7);\n            x[2] ^= rOperation(x[14] + x[10], 9);\n            x[6] ^= rOperation(x[2] + x[14], 13);\n            x[10] ^= rOperation(x[6] + x[2], 18);\n            x[3] ^= rOperation(x[15] + x[11], 7);\n            x[7] ^= rOperation(x[3] + x[15], 9);\n            x[11] ^= rOperation(x[7] + x[3], 13);\n            x[15] ^= rOperation(x[11] + x[7], 18);\n            x[1] ^= rOperation(x[0] + x[3], 7);\n            x[2] ^= rOperation(x[1] + x[0], 9);\n            x[3] ^= rOperation(x[2] + x[1], 13);\n            x[0] ^= rOperation(x[3] + x[2], 18);\n            x[6] ^= rOperation(x[5] + x[4], 7);\n            x[7] ^= rOperation(x[6] + x[5], 9);\n            x[4] ^= rOperation(x[7] + x[6], 13);\n            x[5] ^= rOperation(x[4] + x[7], 18);\n            x[11] ^= rOperation(x[10] + x[9], 7);\n            x[8] ^= rOperation(x[11] + x[10], 9);\n            x[9] ^= rOperation(x[8] + x[11], 13);\n            x[10] ^= rOperation(x[9] + x[8], 18);\n            x[12] ^= rOperation(x[15] + x[14], 7);\n            x[13] ^= rOperation(x[12] + x[15], 9);\n            x[14] ^= rOperation(x[13] + x[12], 13);\n            x[15] ^= rOperation(x[14] + x[13], 18);\n        }\n\n        for (i = 0; i < 16; ++i)\n        {\n            b32[i] += x[i];\n        }\n\n        for (i = 0; i < 16; ++i)\n        {\n            xArray[i * 4] = (byte) (b32[i] & 255);\n            xArray[i * 4 + 1] = (byte) (b32[i] >> 8 & 255);\n            xArray[i * 4 + 2] = (byte) (b32[i] >> 16 & 255);\n            xArray[i * 4 + 3] = (byte) (b32[i] >> 24 & 255);\n        }\n\n    }\n\n    public static void blockXOR(byte[] sArray, int si, byte[] dArray, int di, int length)\n    {\n        for (int i = 0; i < length; ++i)\n        {\n            dArray[di + i] ^= sArray[si + i];\n        }\n\n    }\n\n\n\n    /**\n     * Estimates the required memory to calculate an hash with\n     * the current configuration.\n     *\n     * @return the required memory\n     * @since 0.1.0\n     */\n    public long getRequiredBytes()\n    {\n        return 128L * workFactor * resources * parallelization;\n    }\n\n    public int getWorkFactor()\n    {\n        return workFactor;\n    }\n\n    public int getResources()\n    {\n        return resources;\n    }\n\n    public int getParallelization()\n    {\n        return parallelization;\n    }\n\n    public int getDerivedKeyLength()\n    {\n        return derivedKeyLength;\n    }\n\n    /**\n     * A more readable version of {@link #getRequiredBytes()},\n     * changing the unit (B, KB, MB) so that the number has at most\n     * 2 decimal places.\n     *\n     * @return the required memory\n     * @since 0.3.0\n     */\n    public String getRequiredMemory()\n    {\n        long memoryInBytes = getRequiredBytes();\n        if (memoryInBytes > 1_000_000)\n        {\n            return Math.round(memoryInBytes / 10_000f) / 100.0 + \"MB\";\n        }\n        if (memoryInBytes > 1_000)\n        {\n            return Math.round(memoryInBytes / 1_000f) / 100.0 + \"KB\";\n        }\n        return memoryInBytes + \"B\";\n    }\n\n    @Override\n    public boolean equals(Object obj)\n    {\n        if (obj == null || !this.getClass().equals(obj.getClass()))\n        {\n            return false;\n        }\n\n        ScryptFunction otherStrategy = (ScryptFunction) obj;\n        return this.workFactor == otherStrategy.workFactor //\n                && this.resources == otherStrategy.resources //\n                && this.parallelization == otherStrategy.parallelization;\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(this.resources, this.workFactor, this.parallelization,\n                this.derivedKeyLength) + ')';\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(resources, workFactor, parallelization);\n    }\n\n    public byte[] scrypt(byte[] passwd, byte[] salt, int dkLen) throws GeneralSecurityException\n    {\n        if (workFactor >= 2 && (workFactor & workFactor - 1) == 0)\n        {\n            if (workFactor > 16777215 / resources)\n            {\n                throw new IllegalArgumentException(\"Parameter N is too large\");\n            }\n            else if (resources > 16777215 / parallelization)\n            {\n                throw new IllegalArgumentException(\"Parameter r is too large\");\n            }\n            else\n            {\n                byte[] xyArray = new byte[256 * resources];\n                byte[] vArray = new byte[128 * resources * workFactor];\n                byte[] intensiveSalt = PBKDF2Function.internalHash(Utils.fromBytesToString(passwd).toCharArray(), salt, Hmac.SHA256.name(), 1,\n                        8 * parallelization * 128 * resources).getEncoded();\n\n                for (int i = 0; i < parallelization; ++i)\n                {\n                    sMix(intensiveSalt, i * 128 * resources, vArray, xyArray);\n                }\n\n                return PBKDF2Function\n                        .internalHash(Utils.fromBytesToString(passwd).toCharArray(), intensiveSalt, Hmac.SHA256.name(), 1, 8 * dkLen)\n                        .getEncoded();\n            }\n        }\n        else\n        {\n            throw new IllegalArgumentException(\"N must be a power of 2 greater than 1. Found \" + workFactor);\n        }\n    }\n\n    public void sMix(byte[] intensiveSalt, int bi, byte[] vArray, byte[] xyArray)\n    {\n        int xi = 0;\n        int yi = 128 * resources;\n        System.arraycopy(intensiveSalt, bi, xyArray, xi, 128 * resources);\n\n        int i;\n        for (i = 0; i < workFactor; ++i)\n        {\n            System.arraycopy(xyArray, xi, vArray, i * 128 * resources, 128 * resources);\n            blockmixSalsa8(xyArray, xi, yi);\n        }\n\n        for (i = 0; i < workFactor; ++i)\n        {\n            int j = integerify(xyArray, xi) & workFactor - 1;\n            blockXOR(vArray, j * 128 * resources, xyArray, xi, 128 * resources);\n            blockmixSalsa8(xyArray, xi, yi);\n        }\n\n        System.arraycopy(xyArray, xi, intensiveSalt, bi, 128 * resources);\n    }\n\n    public void blockmixSalsa8(byte[] xyArray, int bi, int yi)\n    {\n        byte[] xArray = new byte[64];\n        System.arraycopy(xyArray, bi + (2 * resources - 1) * 64, xArray, 0, 64);\n\n        int i;\n        for (i = 0; i < 2 * resources; ++i)\n        {\n            blockXOR(xyArray, i * 64, xArray, 0, 64);\n            salsa208(xArray);\n            System.arraycopy(xArray, 0, xyArray, yi + i * 64, 64);\n        }\n\n        for (i = 0; i < resources; ++i)\n        {\n            System.arraycopy(xyArray, yi + i * 2 * 64, xyArray, bi + i * 64, 64);\n        }\n\n        for (i = 0; i < resources; ++i)\n        {\n            System.arraycopy(xyArray, yi + (i * 2 + 1) * 64, xyArray, bi + (i + resources) * 64, 64);\n        }\n\n    }\n\n    public int integerify(byte[] xyArray, int xi)\n    {\n        xi += (2 * resources - 1) * 64;\n        int n = (xyArray[xi] & 255);\n        n |= (xyArray[xi + 1] & 255) << 8;\n        n |= (xyArray[xi + 2] & 255) << 16;\n        n |= (xyArray[xi + 3] & 255) << 24;\n        return n;\n    }\n}\n\n// Path: src/main/java/com/password4j/AlgorithmFinder.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Argon2;\nimport com.password4j.types.Bcrypt;\nimport com.password4j.types.Hmac;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.security.NoSuchAlgorithmException;\nimport java.security.Provider;\nimport java.security.SecureRandom;\nimport java.security.Security;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\n\n\n/**\n * This utility class finds algorithms with their configuration\n * based on the environment.\n * <p>\n * In this context, by environment is intended the set of\n * parameters set in the JVM and in the <i>psw4j.properties</i> file.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic class AlgorithmFinder\n{\n\n    private static final Logger LOG = LoggerFactory.getLogger(AlgorithmFinder.class);\n\n    /**\n     * Singleton instance of {@link SecureRandom}.\n     * <p>\n     * By definition this instance does not need\n     * to be re-instantiated in order to generate\n     * non-deterministic output.\n     *\n     * @see #getSecureRandom()\n     * @since 0.1.0\n     */\n    private static SecureRandom secureRandom;\n\n    private AlgorithmFinder()\n    {\n        //\n    }\n\n    /**\n     * Retrieve a singleton instance of a thread-safe {@link SecureRandom} object.\n     * If the environment allows it, a cryptographically strong random number generator (RNG)\n     * is returned.\n     * <p>\n     * The usage of {@link SecureRandom#getInstanceStrong()} can be forced by\n     * setting {@code global.random.strong} to {@code true} in the <i>psw4j.properties</i>.\n     * Please be aware that this configuration may affect the system performance, while\n     * ensuring an high level of randomness.\n     * <p>\n     * It is recommendable for Unix systems to set {@code securerandom.source} to\n     * {@code file:/dev/urandom} in your <i>java.security</i> file.\n     *\n     * @return a thread-safe {@link SecureRandom} instance\n     * @see SecureRandom#getInstanceStrong()\n     * @since 0.1.0\n     */\n    \npublic static SecureRandom getSecureRandom()\n    {\n        return secureRandom;\n    }\n\n    /**\n     * Creates a singleton instance of {@link PBKDF2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Algorithm</td>\n     *     <td>hash.pbkdf2.algorithm</td>\n     *     <td>PBKDF2WithHmacSHA256</td>\n     *   </tr>\n     *   <tr>\n     *     <td># iterations</td>\n     *     <td>hash.pbkdf2.iterations</td>\n     *     <td>310000</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Key length</td>\n     *     <td>hash.pbkdf2.length</td>\n     *     <td>512</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link PBKDF2Function}\n     * @since 0.1.0\n     */\n    public static PBKDF2Function getPBKDF2Instance()\n    {\n        Param params = internalGetProperties();\n        return PBKDF2Function.getInstance(params.algorithm, params.iterations, params.length);\n    }\n\n    /**\n     * Creates a singleton instance of {@link CompressedPBKDF2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Algorithm</td>\n     *     <td>hash.pbkdf2.algorithm</td>\n     *     <td>PBKDF2WithHmacSHA512</td>\n     *   </tr>\n     *   <tr>\n     *     <td># iterations</td>\n     *     <td>hash.pbkdf2.iterations</td>\n     *     <td>64000</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Key length</td>\n     *     <td>hash.pbkdf2.length</td>\n     *     <td>512</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link CompressedPBKDF2Function}\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getCompressedPBKDF2Instance()\n    {\n        Param params = internalGetProperties();\n        return CompressedPBKDF2Function.getInstance(params.algorithm, params.iterations, params.length);\n    }\n\n    private static Param internalGetProperties()\n    {\n        String algorithm = PropertyReader\n                .readString(\"hash.pbkdf2.algorithm\", Hmac.SHA256.name(), \"PBKDF2 algorithm is not defined\");\n        int iterations = PropertyReader.readInt(\"hash.pbkdf2.iterations\", 310_000, \"PBKDF2 #iterations are not defined\");\n        int length = PropertyReader.readInt(\"hash.pbkdf2.length\", Hmac.SHA256.bits(), \"PBKDF2 key length is not defined\");\n        return new Param(algorithm, iterations, length);\n    }\n\n    /**\n     * Creates a singleton instance of {@link BcryptFunction}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *      *     <td>Logarithmic number of rounds</td>\n     *      *     <td>hash.bcrypt.minor</td>\n     *      *     <td>10</td>\n     *      *   </tr>\n     *   <tr>\n     *     <td>Logarithmic number of rounds</td>\n     *     <td>hash.bcrypt.rounds</td>\n     *     <td>10</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link BcryptFunction}\n     * @since 0.1.0\n     */\n    public static BcryptFunction getBcryptInstance()\n    {\n        char minor = PropertyReader.readChar(\"hash.bcrypt.minor\", 'b', \"bcrypt minor version is not defined\");\n        int rounds = PropertyReader.readInt(\"hash.bcrypt.rounds\", 10, \"bcrypt rounds are not defined\");\n        return BcryptFunction.getInstance(Bcrypt.valueOf(minor), rounds);\n    }\n\n    /**\n     * Creates a singleton instance of {@link ScryptFunction}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Work Factor (N)</td>\n     *     <td>hash.scrypt.workfactor</td>\n     *     <td>65536</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Resources (r)</td>\n     *     <td>hash.scrypt.resources</td>\n     *     <td>8</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Parallelization (p)</td>\n     *     <td>hash.scrypt.parallelization</td>\n     *     <td>1</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Derived Key Length (dkLen)</td>\n     *     <td>hash.scrypt.derivedKeyLength</td>\n     *     <td>64</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link ScryptFunction}\n     * @since 0.1.0\n     */\n    public static ScryptFunction getScryptInstance()\n    {\n        int workFactor = PropertyReader.readInt(\"hash.scrypt.workfactor\", 65_536, \"scrypt work factor (N) is not defined\");\n        int resources = PropertyReader.readInt(\"hash.scrypt.resources\", 8, \"scrypt resources (r) is not defined\");\n        int parallelization = PropertyReader\n                .readInt(\"hash.scrypt.parallelization\", 1, \"scrypt parallelization (p) is not defined\");\n        int derivedKeyLength = PropertyReader.readInt(\"hash.scrypt.derivedKeyLength\", ScryptFunction.DERIVED_KEY_LENGTH,\n                \"scrypt derivedKeyLength (dkLen) is not defined\");\n        return ScryptFunction.getInstance(workFactor, resources, parallelization, derivedKeyLength);\n    }\n\n    public static MessageDigestFunction getMessageDigestInstance()\n    {\n        String algorithm = PropertyReader.readString(\"hash.md.algorithm\", \"SHA-512\", \"Message Digest algorithm is not defined\");\n        String saltOption = PropertyReader.readString(\"hash.md.salt.option\", \"APPEND\", \"Salt option is not defined\");\n        try\n        {\n            return MessageDigestFunction.getInstance(algorithm, SaltOption.valueOf(saltOption.toUpperCase()));\n        }\n        catch (IllegalArgumentException iae)\n        {\n            LOG.warn(\"{} is not a valid option. Fallback to default.\", saltOption);\n            return MessageDigestFunction.getInstance(algorithm);\n        }\n    }\n\n    /**\n     * Creates a singleton instance of {@link Argon2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Memory (log2)</td>\n     *     <td>hash.argon2.memory</td>\n     *     <td>15</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Iterations</td>\n     *     <td>hash.argon2.iterations</td>\n     *     <td>2</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Output Length</td>\n     *     <td>hash.argon2.length</td>\n     *     <td>32</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Parallelism</td>\n     *     <td>hash.argon2.parallelism</td>\n     *     <td>1</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Type</td>\n     *     <td>hash.argon2.type</td>\n     *     <td>id</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Version</td>\n     *     <td>hash.argon2.version</td>\n     *     <td>19</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link ScryptFunction}\n     * @since 1.5.0\n     */\n    public static Argon2Function getArgon2Instance()\n    {\n        int memory = PropertyReader.readInt(\"hash.argon2.memory\", 15_360, \"Argon2 memory is not defined\");\n        int iterations = PropertyReader.readInt(\"hash.argon2.iterations\", 2, \"Argon2 #iterations is not defined\");\n        int outputLength = PropertyReader.readInt(\"hash.argon2.length\", 32, \"Argon2 output length is not defined\");\n        int parallelism = PropertyReader.readInt(\"hash.argon2.parallelism\", 1, \"Argon2 parallelism is not defined\");\n        String type = PropertyReader.readString(\"hash.argon2.type\", \"id\", \"Argon2 type is not defined\");\n        int version = PropertyReader.readInt(\"hash.argon2.version\", 19, \"Argon2 version is not defined\");\n        return Argon2Function\n                .getInstance(memory, iterations, parallelism, outputLength, Argon2.valueOf(type.toUpperCase()), version);\n    }\n\n    public static BalloonHashingFunction getBalloonHashingInstance()\n    {\n        int space = PropertyReader.readInt(\"hash.balloon.space\", 1024, \"BalloonHashing memory (space) is not defined\");\n        int time = PropertyReader.readInt(\"hash.balloon.time\", 3, \"BalloonHashing #iterations (time) is not defined\");\n        int parallelism = PropertyReader.readInt(\"hash.balloon.parallelism\", 1, \"BalloonHashing parallelism is not defined\");\n        int delta = PropertyReader.readInt(\"hash.balloon.delta\", 3, \"BalloonHashing delta is not defined\");\n        String algorithm = PropertyReader.readString(\"hash.balloon.algorithm\", \"SHA-256\", \"BalloonHashing algorithm is not defined\");\n        return BalloonHashingFunction.getInstance(algorithm, space, time, parallelism, delta);\n    }\n\n    /**\n     * Finds the list of supported PBKDF2 algorithms by\n     * the environment's {@link Provider}s.\n     *\n     * @return the list of supported PBKDF2 algorithms\n     * @since 0.2.0\n     */\n    public static List<String> getAllPBKDF2Variants()\n    {\n        List<String> result = new ArrayList<>();\n        for (Provider provider : Security.getProviders())\n        {\n            // Some JDK implementation may return null instead of an empty array.\n            // see https://github.com/Password4j/password4j/issues/120\n            if (provider.getServices() != null)\n            {\n                for (Provider.Service service : provider.getServices())\n                {\n                    if (\"SecretKeyFactory\".equals(service.getType()) && service.getAlgorithm().startsWith(\"PBKDF2\"))\n                    {\n                        result.add(service.getAlgorithm());\n                    }\n                }\n            }\n        }\n        return result;\n    }\n\n    public static Set<String> getAllMessageDigests()\n    {\n        return Security.getAlgorithms(\"MessageDigest\");\n    }\n\n    private static boolean useStrongRandom()\n    {\n        return PropertyReader.readBoolean(\"global.random.strong\", false);\n    }\n\n    static void initialize()\n    {\n        SecureRandom sr;\n        if (useStrongRandom())\n        {\n            try\n            {\n                sr = Utils.getInstanceStrong();\n            }\n            catch (NoSuchAlgorithmException nsae)\n            {\n                /* Even if there's no strong instance, execution\n                 * must continue with a less strong SecureRandom instance */\n                LOG.warn(\"No source of strong randomness found for this environment.\");\n                sr = new SecureRandom();\n            }\n\n        }\n        else\n        {\n            sr = new SecureRandom();\n        }\n        secureRandom = sr;\n    }\n\n    private static class Param\n    {\n        String algorithm;\n\n        int iterations;\n\n        int length;\n\n        Param(String algorithm, int iterations, int length)\n        {\n            this.algorithm = algorithm;\n            this.iterations = iterations;\n            this.length = length;\n        }\n    }\n\n    static\n    {\n        initialize();\n    }\n}\n\n// Path: src/main/java/com/password4j/SaltGenerator.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport java.security.SecureRandom;\n\n\n/**\n * This class contains static functions that\n * help to create a secure salt.\n * <p>\n * In cryptography, a salt is random data that is used as\n * an additional input to a CHF.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic class SaltGenerator\n{\n\n    private SaltGenerator()\n    {\n        //\n    }\n\n    /**\n     * Generates an array of {@code byte}s that can be used\n     * as salt by the CHFs.\n     * The generated salt is created by a cryptographically\n     * strong random number generator (RNG).\n     * <p>\n     * The parameter length must be a non-negative number,\n     * otherwise a {@link BadParametersException} is thrown.\n     *\n     * @param length of the returned byte array\n     * @return a salt as array of {@code byte}s\n     * @throws BadParametersException if the length is negative\n     * @since 0.1.0\n     */\n    public static byte[] generate(int length)\n    {\n        if (length < 0)\n        {\n            throw new BadParametersException(\"Salt length cannot be negative\");\n        }\n        byte[] salt = new byte[length];\n        SecureRandom sr = AlgorithmFinder.getSecureRandom();\n        sr.nextBytes(salt);\n        return salt;\n    }\n\n    /**\n     * Generates an array of {@code byte}s that can be used\n     * as salt by the CHFs.\n     * The generated salt is created by a cryptographically\n     * strong random number generator (RNG).\n     * <p>\n     * The length of the array is 64.\n     *\n     * @return a salt as array of {@code byte}s\n     * @since 0.1.0\n     */\n    public static byte[] generate()\n    {\n        return generate(get());\n    }\n\n    /**\n     * Get the length of salt from configurations and\n     * must be stored in a location different from the one used\n     * for the passwords.\n     * <p>\n     * It can be set in the <i>psw4j.properties</i> file with\n     * the property {@code global.salt.length}.\n     * <p>\n     * If the <i>psw4j.properties</i> or the property {@code global.salt.length}\n     * are not found, {@code null} is returned.\n     *\n     * @return a shared pepper set in the <i>psw4j.properties</i> file.\n     * @since 1.7.0\n     */\n    public static int get()\n    {\n        return PropertyReader.readInt(\"global.salt.length\", 64, \"Global salt length is not defined in properties file\");\n    }\n\n}\n\n// Path: src/main/java/com/password4j/PBKDF2Function.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport javax.crypto.SecretKey;\nimport javax.crypto.SecretKeyFactory;\nimport javax.crypto.spec.PBEKeySpec;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.spec.InvalidKeySpecException;\nimport java.util.Arrays;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of PBKDF2 function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/PBKDF2\">PBKDF2</a>\n * @since 0.1.0\n */\npublic class PBKDF2Function extends AbstractHashingFunction\n{\n    private static final Map<String, PBKDF2Function> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final String ALGORITHM_PREFIX = \"PBKDF2WithHmac\";\n\n    protected Hmac algorithm;\n\n    protected String algorithmAsString;\n\n    protected int iterations;\n\n    protected int length;\n\n    protected PBKDF2Function()\n    {\n        //\n    }\n\n    protected PBKDF2Function(int iterations, int length)\n    {\n        this.iterations = iterations;\n        this.length = length;\n    }\n\n    protected PBKDF2Function(Hmac algorithm, int iterations, int length)\n    {\n        this(iterations, length);\n        this.algorithm = algorithm;\n        this.algorithmAsString = algorithm.name();\n    }\n\n    protected PBKDF2Function(String algorithm, int iterations, int length)\n    {\n        this(iterations, length);\n        this.algorithmAsString = algorithm;\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @since 0.1.0\n     */\n    public static PBKDF2Function getInstance(Hmac algorithm, int iterations, int length)\n    {\n        return getInstance(algorithm.name(), iterations, length);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  string version of hmac algorithm\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @since 0.1.0\n     */\n    public static PBKDF2Function getInstance(String algorithm, int iterations, int length)\n    {\n...\n", "needle_token_start": 10377, "needle_token_end": 10399, "code_context_ntokens": 16029, "output": ["```java\npublic static SecureRandom getSecureRandom()\n{\n    return secureRandom;\n}\n```"]}
{"repo": "Password4j/password4j", "name": "getResult", "language": "java", "path": "src/main/java/com/password4j/Hash.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the string representation of the computed hash value from the cryptographic process.\n2. **Input**: There are no parameters required for this function.\n3. **Output**: Returns the hash value as a string.\n4. **Procedure**: Converts the byte array storing the hash result into a string format using a utility conversion method.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/Blake2b.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport java.util.Arrays;\n\n\nclass Blake2b\n{\n    private static final long[] IV = { 0x6a09e667f3bcc908L, 0xbb67ae8584caa73bL, 0x3c6ef372fe94f82bL, 0xa54ff53a5f1d36f1L,\n            0x510e527fade682d1L, 0x9b05688c2b3e6c1fL, 0x1f83d9abfb41bd6bL, 0x5be0cd19137e2179L };\n\n    private static final byte[][] SIGMA = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },\n            { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 },\n            { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 },\n            { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 },\n            { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 },\n            { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },\n            { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } };\n\n    private static final int ROUNDS = 12;\n\n    private static final int BLOCK_LENGTH_BYTES = 128;\n\n    private final int digestLength;\n\n    private final int keyLength;\n\n    private final byte[] buffer;\n\n    private final long[] internalState = new long[16];\n\n    private int bufferPos = 0;\n\n    private long[] chainValue = null;\n\n    private long t0 = 0L;\n\n    private long t1 = 0L;\n\n    private long f0 = 0L;\n\n    /**\n     * Basic sized constructor - size in bytes.\n     *\n     * @param digestSize size of the digest in bytes\n     */\n    Blake2b(int digestSize)\n    {\n        if (digestSize < 1 || digestSize > 64)\n        {\n            throw new BadParametersException(\"BLAKE2b digest bytes length must be not greater than 64\");\n        }\n\n        buffer = new byte[BLOCK_LENGTH_BYTES];\n        keyLength = 0;\n        this.digestLength = digestSize;\n        init();\n    }\n\n    // initialize chainValue\n    private void init()\n    {\n        chainValue = new long[8];\n        chainValue[0] = IV[0] ^ (digestLength | ((long) keyLength << 8) | 0x1010000);\n        chainValue[1] = IV[1];\n        chainValue[2] = IV[2];\n        chainValue[3] = IV[3];\n        chainValue[4] = IV[4];\n        chainValue[5] = IV[5];\n        chainValue[6] = IV[6];\n        chainValue[7] = IV[7];\n    }\n\n    private void initializeInternalState()\n    {\n        System.arraycopy(chainValue, 0, internalState, 0, chainValue.length);\n        System.arraycopy(IV, 0, internalState, chainValue.length, 4);\n        internalState[12] = t0 ^ IV[4];\n        internalState[13] = t1 ^ IV[5];\n        internalState[14] = f0 ^ IV[6];\n        internalState[15] = IV[7];// ^ f1 with f1 = 0\n    }\n\n    void update(byte[] message)\n    {\n        if (message == null)\n        {\n            return;\n        }\n        update(message, 0, message.length);\n    }\n\n    /**\n     * update the message digest with a block of bytes.\n     *\n     * @param message the byte array containing the data.\n     * @param offset  the offset into the byte array where the data starts.\n     * @param len     the length of the data.\n     */\n    void update(byte[] message, int offset, int len)\n    {\n        int remainingLength = 0;\n\n        if (bufferPos != 0)\n        {\n            remainingLength = BLOCK_LENGTH_BYTES - bufferPos;\n            if (remainingLength < len)\n            {\n                System.arraycopy(message, offset, buffer, bufferPos, remainingLength);\n                t0 += BLOCK_LENGTH_BYTES;\n                if (t0 == 0)\n                {\n                    t1++;\n                }\n                compress(buffer, 0);\n                bufferPos = 0;\n                Arrays.fill(buffer, (byte) 0);// clear buffer\n            }\n            else\n            {\n                System.arraycopy(message, offset, buffer, bufferPos, len);\n                bufferPos += len;\n                return;\n            }\n        }\n\n        int messagePos;\n        int blockWiseLastPos = offset + len - BLOCK_LENGTH_BYTES;\n        for (messagePos = offset + remainingLength; messagePos < blockWiseLastPos; messagePos += BLOCK_LENGTH_BYTES)\n        {\n            t0 += BLOCK_LENGTH_BYTES;\n            if (t0 == 0)\n            {\n                t1++;\n            }\n            compress(message, messagePos);\n        }\n\n        // fill the buffer with left bytes, this might be a full block\n        System.arraycopy(message, messagePos, buffer, 0, offset + len - messagePos);\n        bufferPos += offset + len - messagePos;\n    }\n\n    /**\n     * close the digest, producing the final digest value. The doFinal\n     * call leaves the digest reset.\n     * Key, salt and personal string remain.\n     *\n     * @param out       the array the digest is to be copied into.\n     * @param outOffset the offset into the out array the digest is to start at.\n     */\n    void doFinal(byte[] out, int outOffset)\n    {\n\n        f0 = 0xFFFFFFFFFFFFFFFFL;\n        t0 += bufferPos;\n        if (bufferPos > 0 && t0 == 0)\n        {\n            t1++;\n        }\n        compress(buffer, 0);\n        Arrays.fill(buffer, (byte) 0);// Holds eventually the key if input is null\n        Arrays.fill(internalState, 0L);\n\n        for (int i = 0; i < chainValue.length && (i * 8 < digestLength); i++)\n        {\n            byte[] bytes = Utils.longToLittleEndian(chainValue[i]);\n\n            if (i * 8 < digestLength - 8)\n            {\n                System.arraycopy(bytes, 0, out, outOffset + i * 8, 8);\n            }\n            else\n            {\n                System.arraycopy(bytes, 0, out, outOffset + i * 8, digestLength - (i * 8));\n            }\n        }\n\n        Arrays.fill(chainValue, 0L);\n\n        reset();\n    }\n\n    /**\n     * Reset the digest back to it's initial state.\n     * The key, the salt and the personal string will\n     * remain for further computations.\n     */\n    void reset()\n    {\n        bufferPos = 0;\n        f0 = 0L;\n        t0 = 0L;\n        t1 = 0L;\n        chainValue = null;\n        Arrays.fill(buffer, (byte) 0);\n        init();\n    }\n\n    private void compress(byte[] message, int messagePos)\n    {\n\n        initializeInternalState();\n\n        long[] m = new long[16];\n        for (int j = 0; j < 16; j++)\n        {\n            m[j] = Utils.littleEndianToLong(message, messagePos + j * 8);\n        }\n\n        for (int round = 0; round < ROUNDS; round++)\n        {\n\n            // G apply to columns of internalState:m[blake2b_sigma[round][2 *\n            // blockPos]] /+1\n            functionG(m[SIGMA[round][0]], m[SIGMA[round][1]], 0, 4, 8, 12);\n            functionG(m[SIGMA[round][2]], m[SIGMA[round][3]], 1, 5, 9, 13);\n            functionG(m[SIGMA[round][4]], m[SIGMA[round][5]], 2, 6, 10, 14);\n            functionG(m[SIGMA[round][6]], m[SIGMA[round][7]], 3, 7, 11, 15);\n            // G apply to diagonals of internalState:\n            functionG(m[SIGMA[round][8]], m[SIGMA[round][9]], 0, 5, 10, 15);\n            functionG(m[SIGMA[round][10]], m[SIGMA[round][11]], 1, 6, 11, 12);\n            functionG(m[SIGMA[round][12]], m[SIGMA[round][13]], 2, 7, 8, 13);\n            functionG(m[SIGMA[round][14]], m[SIGMA[round][15]], 3, 4, 9, 14);\n        }\n\n        // update chain values:\n        for (int offset = 0; offset < chainValue.length; offset++)\n        {\n            chainValue[offset] = chainValue[offset] ^ internalState[offset] ^ internalState[offset + 8];\n        }\n    }\n\n    private void functionG(long m1, long m2, int posA, int posB, int posC, int posD)\n    {\n\n        internalState[posA] = internalState[posA] + internalState[posB] + m1;\n        internalState[posD] = Long.rotateRight(internalState[posD] ^ internalState[posA], 32);\n        internalState[posC] = internalState[posC] + internalState[posD];\n        internalState[posB] = Long.rotateRight(internalState[posB] ^ internalState[posC], 24); // replaces 25 of BLAKE\n        internalState[posA] = internalState[posA] + internalState[posB] + m2;\n...\n// Path: src/main/java/com/password4j/PepperGenerator.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\n\n/**\n * This class contains static functions that\n * help to create a secure pepper.\n * <p>\n * In cryptography, a pepper is a secret added to a password\n * prior to being hashed with a CHF.\n *\n * @author David Bertoldi\n * @since 0.1.1\n */\npublic class PepperGenerator\n{\n\n    private PepperGenerator()\n    {\n        //\n    }\n\n    /**\n     * Generates a {@link String} that can be used as pepper\n     * by a CHF.\n     * The generated pepper is created by a cryptographically\n     * strong random number generator (RNG).\n     * <p>\n     * The parameter length must be a non-negative number,\n     * otherwise a {@link BadParametersException} is thrown.\n     *\n     * @param length of the returned string\n     * @return a pepper of the given length\n     * @throws BadParametersException if the length is negative\n     * @since 0.1.1\n     */\n    public static String generate(int length)\n    {\n        if (length < 0)\n        {\n            throw new BadParametersException(\"Pepper length cannot be negative\");\n        }\n        return Utils.randomPrintable(length);\n    }\n\n    /**\n     * Generates a {@link String} that can be used as pepper\n     * by a CHF.\n     * The generated pepper is created by a cryptographically\n     * strong random number generator (RNG).\n     * <p>\n     * The pepper generated is 24 characters long.\n     *\n     * @return a pepper as {@link String}\n     * @since 0.1.1\n     */\n    public static String generate()\n    {\n        return generate(24);\n    }\n\n    /**\n     * Peppers by definition are shared between all passwords and\n     * must be stored in a location different from the one used\n     * for the passwords.\n     * <p>\n     * It can be set in the <i>psw4j.properties</i> file with\n     * the property {@code global.pepper}.\n     * <p>\n     * If the <i>psw4j.properties</i> or the property {@code global.pepper}\n     * are not found, {@code null} is returned.\n     *\n     * @return a shared pepper set in the <i>psw4j.properties</i> file.\n     * @since 0.1.1\n     */\n    public static String get()\n    {\n        return PropertyReader.readString(\"global.pepper\", null, \"Global pepper is not defined\");\n    }\n\n}\n\n// Path: src/main/java/com/password4j/HashChecker.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\n/**\n * Builder class that helps to create a chain of parameters to be used\n * in the verification process.\n *\n * @author David Bertoldi\n * @since 1.0.0\n */\npublic class HashChecker\n{\n    private byte[] plainTextPassword;\n\n    protected byte[] hashed;\n\n    protected byte[] salt;\n\n    private CharSequence pepper;\n\n\n    @SuppressWarnings(\"unused\")\n    private HashChecker()\n    {\n        //\n    }\n\n    /**\n     * @param plainTextPassword the plain text password\n     * @param hashed            the hash to verify\n     * @since 1.0.0\n     */\n    HashChecker(CharSequence plainTextPassword, String hashed)\n    {\n        this.hashed = Utils.fromCharSequenceToBytes(hashed);\n        this.plainTextPassword = Utils.fromCharSequenceToBytes(plainTextPassword);\n    }\n\n    /**\n     * @param plainTextPassword the plain text password as bytes array\n     * @param hashed            the hash to verify as bytes array\n     * @since 1.7.0\n     */\n    HashChecker(byte[] plainTextPassword, byte[] hashed)\n    {\n        this.hashed = hashed;\n        this.plainTextPassword = plainTextPassword;\n    }\n\n    /**\n     * Concatenates the provided string with the plain text password.\n     * The produced sequence (in the form {@code pepper+password}) is processed by the algorithm.\n     *\n     * @param pepper cryptographic pepper\n     * @return this builder\n     * @since 1.0.0\n     */\n    public HashChecker addPepper(CharSequence pepper)\n    {\n        this.pepper = pepper;\n        return this;\n    }\n\n    /**\n     * Concatenates the pepper configured in your `psw4j.properties` file with the plain text password.\n     * The produced sequence (in the form {@code pepper+password}) is processed by the algorithm.\n     *\n     * @return this builder\n     * @see PepperGenerator#get()\n     */\n    public HashChecker addPepper()\n    {\n        this.pepper = PepperGenerator.get();\n        return this;\n    }\n\n    /**\n     * Add a cryptographic salt in the verifying process.\n     * The salt is applied differently depending on the chosen algorithm.\n     *\n     * @param salt cryptographic salt\n     * @return this builder\n     * @since 1.0.0\n     */\n    public HashChecker addSalt(String salt)\n    {\n        this.salt = Utils.fromCharSequenceToBytes(salt);\n        return this;\n    }\n\n    /**\n     * Add a cryptographic salt in the verifying process.\n     * The salt is applied differently depending on the chosen algorithm.\n     *\n     * @param salt cryptographic salt as bytes array\n     * @return this builder\n     * @since 1.7.0\n     */\n    public HashChecker addSalt(byte[] salt)\n    {\n        this.salt = salt;\n        return this;\n    }\n\n    /**\n     * Creates a builder to update the hash.\n     * The actual  salt and pepper are taken from the original check request.\n     * <p>\n     * In order to declare a new salt or pepper use ,}\n     *\n     * @return the updater\n     * @since 1.3.0\n     */\n    public HashUpdater andUpdate()\n    {\n        return new HashUpdater(this, new HashBuilder(plainTextPassword).addPepper(pepper).addSalt(salt));\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with a specific implementation of {@link HashingFunction}.\n     * <p>\n     * This method does not read the configurations in the `psw4j.properties` file.\n     *\n     * @param hashingFunction a CHF\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @since 1.0.0\n     */\n    public boolean with(HashingFunction hashingFunction)\n    {\n        if (plainTextPassword == null || plainTextPassword.length == 0)\n        {\n            return false;\n        }\n\n        return hashingFunction.check(plainTextPassword, hashed, salt, pepper);\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with {@link PBKDF2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     *\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @see AlgorithmFinder#getPBKDF2Instance()\n     * @since 1.0.0\n     */\n    public boolean withPBKDF2()\n    {\n        PBKDF2Function pbkdf2 = AlgorithmFinder.getPBKDF2Instance();\n        return with(pbkdf2);\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with {@link CompressedPBKDF2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     *\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @see AlgorithmFinder#getCompressedPBKDF2Instance()\n     * @since 1.0.0\n     */\n    public boolean withCompressedPBKDF2()\n    {\n        PBKDF2Function pbkdf2 = AlgorithmFinder.getCompressedPBKDF2Instance();\n        return with(pbkdf2);\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with {@link ScryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     *\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @see AlgorithmFinder#getScryptInstance()\n     * @since 1.0.0\n     */\n    public boolean withScrypt()\n    {\n        ScryptFunction scrypt = AlgorithmFinder.getScryptInstance();\n        return with(scrypt);\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with {@link BcryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     *\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @see AlgorithmFinder#getBcryptInstance()\n     * @since 1.0.0\n     */\n    public boolean withBcrypt()\n    {\n        return with(AlgorithmFinder.getBcryptInstance());\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with {@link MessageDigestFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     *\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @see AlgorithmFinder#getMessageDigestInstance()\n     * @since 1.4.0\n     */\n    public boolean withMessageDigest()\n    {\n        return with(AlgorithmFinder.getMessageDigestInstance());\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with {@link Argon2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     *\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @see AlgorithmFinder#getArgon2Instance()\n     * @since 1.5.0\n     */\n    public boolean withArgon2()\n    {\n        Argon2Function argon2 = AlgorithmFinder.getArgon2Instance();\n        return with(argon2);\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with {@link BalloonHashingFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     *\n     * @return true if the hash was produced by the given plain text password; false otherwise.\n     * @see AlgorithmFinder#getBalloonHashingInstance()\n     * @since 1.5.0\n     */\n    public boolean withBalloonHashing()\n    {\n        BalloonHashingFunction balloon = AlgorithmFinder.getBalloonHashingInstance();\n        return with(balloon);\n    }\n\n    /**\n     * This method returns the String version of the hash bytes. This\n     * should be always a safe operation when using ISO-8859-1 encoding.\n     *\n     * @return String version of the hash\n     */\n    protected String getHashed()\n    {\n        return Utils.fromBytesToString(hashed);\n    }\n\n}\n\n// Path: src/main/java/com/password4j/SecureString.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport java.util.Arrays;\n\n\n/**\n * More secure implementation of {@link CharSequence} than {@link String}, preventing heap memory attacks.\n * <p>\n * Like {@link String}, this {@link CharSequence} implementation has an underlying array of {@code char}s.\n * The sequence however is not stored in the String pool and the array of {@code char}s lives\n * temporary in the heap memory.\n *\n * @author David Bertoldi\n * @since 1.2.0\n */\npublic class SecureString implements CharSequence\n{\n    private final char[] chars;\n\n    /**\n     * Creates a {@link SecureString} from an array of {@code char}s.\n     * The sequence is never put in the String pool.\n     *\n     * @param chars sequence of characters\n     * @throws NullPointerException if null is passed\n     * @since 1.2.0\n     */\n    public SecureString(char[] chars)\n    {\n        this.chars = new char[chars.length];\n        System.arraycopy(chars, 0, this.chars, 0, chars.length);\n    }\n\n    /**\n     * Creates a {@link SecureString} from an array of {@code char}s.\n     * Important: if the second argument is true, the original array is zeroed after the object creation! Each {@code char} is replaced\n     * with {@link Character#MIN_VALUE}\n     * <p>\n     * The sequence is never put in the String pool.\n     *\n     * @param chars       sequence of characters\n     * @param eraseSource if true, the original array is zeroed\n     * @throws NullPointerException if null is passed\n     * @since 1.2.0\n     */\n    public SecureString(char[] chars, boolean eraseSource)\n    {\n        this(chars);\n        if (eraseSource)\n        {\n            clear(chars);\n        }\n    }\n\n    /**\n     * Creates a {@link SecureString} from a subsequence of an array of {@code char}s.\n     * The sequence is never put in the String pool.\n     *\n     * @param chars sequence of characters\n     * @param start index of the beginning of the subsequence\n     * @param end   index of the end of the subsequence\n     * @throws NullPointerException if null is passed as array of {@code char}s\n     * @since 1.2.0\n     */\n    public SecureString(char[] chars, int start, int end)\n    {\n        this.chars = new char[end - start];\n        System.arraycopy(chars, start, this.chars, 0, this.chars.length);\n        clear(chars);\n    }\n\n    private static synchronized void clear(char[] chars)\n    {\n        Arrays.fill(chars, Character.MIN_VALUE);\n    }\n\n    /**\n     * @return length of the underlying array of {@code char}s.\n     * @since 1.2.0\n     */\n    @Override\n    public synchronized int length()\n    {\n        return chars.length;\n    }\n\n    /**\n     * @param index position in the underlying array of {@code char}s.\n     * @return the {@code char} in the given position\n     * @since 1.2.0\n     */\n    @Override\n    public synchronized char charAt(int index)\n    {\n        return chars[index];\n    }\n\n    /**\n     * Creates a {@link SecureString} from a subsequence of this object.\n     *\n     * @param start index of the beginning of the subsequence\n     * @param end   index of the end of the subsequence\n     * @see SecureString#SecureString(char[], int, int)\n     * @since 1.2.0\n     */\n    @Override\n    public synchronized CharSequence subSequence(int start, int end)\n    {\n        return new SecureString(this.chars, start, end);\n    }\n\n    /**\n     * Manually clear the underlying array holding the characters\n     *\n     * @since 1.2.0\n     */\n    public void clear()\n    {\n        synchronized (chars)\n        {\n            clear(chars);\n        }\n    }\n\n    /**\n     * Returns a constant {@link String} in order to prevent data leaks due\n     * to accidental usage of a {@link SecureString} objects in methods like\n     * {@link java.io.PrintStream#print(Object)}, loggers, etc.\n     *\n     * @return a masked version of this object.\n     * @since 1.2.0\n     */\n    @Override\n    public String toString()\n    {\n        return \"SecureString[****]\";\n    }\n\n    /**\n     * Constant time equality to avoid potential timing attacks.\n     *\n     * @param other object\n     * @since 1.2.1\n     */\n    @Override\n    public synchronized boolean equals(Object other)\n    {\n        if (this == other)\n        {\n            return true;\n        }\n        if (!(other instanceof CharSequence))\n        {\n            return false;\n        }\n        CharSequence that = (CharSequence) other;\n\n        if (chars.length != that.length())\n        {\n            return false;\n        }\n\n        int equals = 0;\n        for (int i = 0; i < chars.length; i++)\n        {\n            equals |= chars[i] ^ that.charAt(i);\n        }\n\n        return equals == 0;\n    }\n\n    @Override\n    public synchronized int hashCode()\n    {\n        return Arrays.hashCode(chars);\n    }\n}\n\n// Path: src/main/java/com/password4j/AbstractHashingFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\n/**\n * Class in the hierarchy to avoid code duplication.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic abstract class AbstractHashingFunction implements HashingFunction\n{\n\n    /**\n     * Compares two {@link CharSequence}s as byte arrays in length-constant time. This comparison method\n     * is used so that password hashes cannot be extracted from an on-line\n     * system using a timing attack and then attacked off-line.\n     *\n     * @param a the first CharSequence\n     * @param b the second CharSequence\n     * @return true if both {@link CharSequence}s are the same, false if not\n     */\n    protected static boolean slowEquals(CharSequence a, CharSequence b)\n    {\n        return slowEquals(Utils.fromCharSequenceToBytes(a), Utils.fromCharSequenceToBytes(b));\n    }\n\n    /**\n     * Compares two byte arrays in length-constant time. This comparison method\n     * is used so that password hashes cannot be extracted from an on-line\n     * system using a timing attack and then attacked off-line.\n     *\n     * @param a the first byte array\n     * @param b the second byte array\n     * @return true if both byte arrays are the same, false if not\n     */\n    protected static boolean slowEquals(byte[] a, byte[] b)\n    {\n        int diff = a.length ^ b.length;\n        for (int i = 0; i < a.length && i < b.length; i++)\n        {\n            diff |= a[i] ^ b[i];\n        }\n        return diff == 0;\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt, CharSequence pepper)\n    {\n        CharSequence peppered = Utils.append(pepper, plainTextPassword);\n        Hash result;\n        if (salt == null)\n        {\n            result = hash(peppered);\n        }\n        else\n        {\n            result = hash(peppered, salt);\n        }\n\n        result.setPepper(pepper);\n        return result;\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPassword, byte[] salt, CharSequence pepper)\n    {\n        byte[] pepperAsBytes = Utils.fromCharSequenceToBytes(pepper);\n        byte[] peppered = Utils.append(pepperAsBytes, plainTextPassword);\n        Hash result;\n        if (salt == null)\n        {\n            result = hash(peppered);\n        }\n        else\n        {\n            result = hash(peppered, salt);\n        }\n\n        result.setPepper(pepper);\n        return result;\n    }\n\n    /**\n     * Just calls {@link #check(CharSequence, String)} without salt\n     * parameter.\n     * <p>\n     * Do not override this if the algorithm doesn't need a manually\n     * provided salt.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @param salt              the salt used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @since 0.1.0\n     */\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        return check(plainTextPassword, hashed);\n    }\n\n    /**\n     * Just calls {@link #check(CharSequence, String)} without salt\n     * parameter.\n     * <p>\n     * Do not override this if the algorithm doesn't need a manually\n     * provided salt.\n     *\n     * @param plainTextPassword the plaintext password as bytes array\n     * @param hashed            the hash as bytes array\n     * @param salt              the salt as bytes array used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @since 1.7.0\n     */\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        return check(plainTextPassword, hashed);\n    }\n\n    /**\n     * Just calls {@link #check(CharSequence, String, String)}, with a prepended pepper.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @param salt              the salt used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @since 1.5.0\n     */\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt, CharSequence pepper)\n    {\n        return check(Utils.append(pepper, plainTextPassword), hashed, salt);\n    }\n\n    /**\n     * Just calls {@link #check(CharSequence, String, String)}, with a prepended pepper.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @param salt              the salt used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @since 1.7.0\n     */\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt, CharSequence pepper)\n    {\n        byte[] pepperAsBytes = Utils.fromCharSequenceToBytes(pepper);\n        return check(Utils.append(pepperAsBytes, plainTextPassword), hashed, salt);\n    }\n\n}\n\n// Path: src/main/java/com/password4j/Hash.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport java.util.Arrays;\nimport java.util.Objects;\n\n\n/**\n * This class contains all the information computed after\n * calculating a cryptographic hash.\n * <p>\n * The same {@link HashingFunction} used to generate the hash\n * is used to verify the plain password; in addition <i>cryptographic\n * seasoning</i> such as salt and pepper are stored in this object.\n * <p>\n * A hash is the product of a one-way function that maps data of arbitrary size to\n * fixed-size values; it is called hashing function (HF).\n * This class represent hashes generated by cryptographic hash function (CHF),\n * where each function has the following properties:\n * <ul>\n * <li>it is deterministic, meaning that the same message always results in the same hash</li>\n * <li>it is quick to compute the hash value for any given message</li>\n * <li>it is infeasible to generate a message that yields a given hash value</li>\n * <li>it is infeasible to find two different messages with the same hash value</li>\n * <li>a small change to a message should change the hash value so extensively that the new hash value\n * appears uncorrelated with the old hash value</li>\n * </ul>\n * <p>\n * A salt is a unique, randomly generated string that is added to each password as part of the hashing process.\n * As the salt is unique for every user, an attacker has to crack hashes one at a time using the respective salt,\n * rather than being able to calculate a hash once and compare it against every stored hash.\n * <p>\n * A pepper can be used in additional to salting to provide an additional layer of protection.\n * It is similar to a salt, but has two key differences:\n * <ul>\n * <li>The pepper is shared between all stored passwords, rather than being unique like a salt.</li>\n * <li>The pepper is not stored in the database, unlike the salts.</li>\n * </ul>\n *\n * @author David Bertoldi\n * @see <a href=\"https://owasp.org/www-project-cheat-sheets/cheatsheets/Password_Storage_Cheat_Sheet\">OWASP Password Storage Cheat Sheet</a>\n * @see <a href=\"https://en.wikipedia.org/wiki/Key_derivation_function\">Key derivation function</a>\n * @see <a href=\"https://en.wikipedia.org/wiki/Cryptographic_hash_function\">Cryptographic hash function</a>\n * @since 0.1.0\n */\npublic class Hash\n{\n\n    /**\n     * Represents the full output of a cryptographic hashing function.\n     * Depending on the implementation of the CHF, it may contain\n     * the salt and the configuration of the CHF itself.\n     */\n    private byte[] result;\n\n    /**\n     * Represents the computed output of a cryptographic hashing function.\n     * It never contains salt and other configurations.\n     */\n    private byte[] bytes;\n\n    /**\n     * Represents the salt: random data that is used as an additional input\n     * to a cryptographic hashing function.\n     */\n    private byte[] salt;\n\n    /**\n     * Represents the pepper: a secret added to the input password\n     * prior to being hashed with a cryptographic hash function\n     */\n    private CharSequence pepper;\n\n    /**\n     * Represents the hashing function used to generate this object.\n     *\n     * @see HashingFunction for more details\n     */\n    private HashingFunction hashingFunction;\n\n    /**\n     * It is meant to not be used if not internally.\n     * The other constructor must be used instead.\n     *\n     * @see Hash#Hash(HashingFunction, String, byte[], String)\n     * @since 0.1.0\n     */\n    @SuppressWarnings(\"unused\")\n    private Hash()\n    {\n        //\n    }\n\n    /**\n     * Constructs an {@link Hash} containing the basic information\n     * used and produced by the computational process of hashing a password.\n     * Other information, like <i>pepper</i> can be added with\n     * {@link #setPepper(CharSequence)}.\n     * <p>\n     * This constructor populates the object's attributes.\n     *\n     * @param hashingFunction the cryptographic algorithm used to produce the hash.\n     * @param result          the result of the computation of the hash.\n     *                        Notice that the format varies depending on the algorithm.\n     * @param bytes           the hash without additional information.\n     * @param salt            the salt used for the computation.\n     * @since 0.1.0\n     * @deprecated            As of 1.8.1 because of the salt conversion from {@link String} to byte[].\n     *                        {@link Hash#Hash(HashingFunction, String, byte[], byte[])} should be used instead.\n     */\n    @Deprecated\n    public Hash(HashingFunction hashingFunction, String result, byte[] bytes, String salt)\n    {\n        this(hashingFunction, Utils.fromCharSequenceToBytes(result), bytes, Utils.fromCharSequenceToBytes(salt));\n    }\n\n\n\n    /**\n     * Constructs an {@link Hash} containing the basic information\n     * used and produced by the computational process of hashing a password.\n     * Other information, like <i>pepper</i> can be added with\n     * {@link #setPepper(CharSequence)}.\n     * <p>\n     * This constructor populates the object's attributes.\n     *\n     * @param hashingFunction the cryptographic algorithm used to produce the hash.\n     * @param result          the result of the computation of the hash.\n     *                        Notice that the format varies depending on the algorithm.\n     * @param bytes           the hash without additional information.\n     * @param salt            the salt used for the computation.\n     * @since 0.1.0\n     */\n    public Hash(HashingFunction hashingFunction, String result, byte[] bytes, byte[] salt)\n    {\n        this(hashingFunction, Utils.fromCharSequenceToBytes(result), bytes, salt);\n    }\n\n    /**\n     * Constructs an {@link Hash} containing the basic information\n     * used and produced by the computational process of hashing a password.\n     * Other information, like <i>pepper</i> can be added with\n     * {@link #setPepper(CharSequence)}.\n     * <p>\n     * This constructor populates the object's attributes.\n     *\n     * @param hashingFunction the cryptographic algorithm used to produce the hash.\n     * @param result          the result of the computation of the hash as bytes array.\n     *                        Notice that the format varies depending on the algorithm.\n     * @param bytes           the hash without additional information.\n     * @param salt            the salt used for the computation as bytes array.\n     * @since 1.7.0\n     */\n    public Hash(HashingFunction hashingFunction, byte[] result, byte[] bytes, byte[] salt)\n    {\n        this.hashingFunction = hashingFunction;\n        this.salt = salt;\n        this.result = result;\n        this.bytes = bytes;\n    }\n\n    /**\n     * Retrieves the hash computed by the hashing function.\n     *\n     * @return the hash.\n     * @since 0.1.0\n     */\n    \npublic String getResult()\n    {\n        return Utils.fromBytesToString(result);\n    }\n\n    /**\n     * Retrieves the hash computed by the hashing function.\n     *\n     * @return the hash.\n     * @since 0.1.0\n     */\n    public byte[] getResultAsBytes()\n    {\n        return result;\n    }\n\n    /**\n     * Retrieves the hash as byte array and without the parameters\n     * encoded in the final hash.\n     *\n     * @return the hash.\n     * @since 1.5.1\n     */\n    public byte[] getBytes()\n    {\n        return bytes;\n    }\n\n    /**\n     * Retrieves the {@link HashingFunction} used\n     * to hash the password.\n     *\n     * @return the CHF\n     * @since 0.4.0\n     */\n    public HashingFunction getHashingFunction()\n    {\n        return hashingFunction;\n    }\n\n    /**\n     * Retrieves the salt used by the hashing function.\n     *\n     * @return the salt as {@link String}.\n     * @since 0.1.0\n     */\n    public String getSalt()\n    {\n        return Utils.fromBytesToString(salt);\n    }\n\n    /**\n     * Retrieves the salt used by the hashing function.\n     *\n     * @return the salt as bytes array.\n     * @since 1.7.0\n     */\n    public byte[] getSaltBytes()\n    {\n        return salt;\n    }\n\n    /**\n     * Retrieves the pepper used with the password in the hashing function.\n     *\n     * @return the pepper.\n     * @since 0.1.0\n     */\n    public CharSequence getPepper()\n    {\n        return pepper;\n    }\n\n    /**\n     * Stores the pepper used together with the password in the hashing function.\n     * <p>\n     * This methods should be used just after the creation of this object.\n     *\n     * @param pepper the pepper used.\n     * @since 0.1.0\n     */\n    void setPepper(CharSequence pepper)\n    {\n        this.pepper = pepper;\n    }\n\n    /**\n     * Produces a human-readable description of the {@link Hash}.\n     *\n     * @return a readable version of this object\n     * @since 0.1.0\n     */\n    @Override\n    public String toString()\n    {\n        StringBuilder sb = new StringBuilder();\n        if (this.hashingFunction != null)\n        {\n            sb.append(hashingFunction.getClass().getSimpleName());\n        }\n        sb.append(\"[salt=\").append(getSalt()).append(\", pepper=\").append(getPepper()).append(\", hash=\").append(getResult())\n                .append(\"]\");\n        return sb.toString();\n    }\n\n    /**\n     * Two {@link Hash}es are considered equals if they contain\n     * the same hash, salt, pepper and they are generated with\n     * the same {@link HashingFunction}\n     *\n     * @param obj the object to compare\n     * @return true if equals\n     * @since 0.1.0\n     */\n    @Override\n    public boolean equals(Object obj)\n    {\n        if (obj == null || !this.getClass().equals(obj.getClass()))\n        {\n            return false;\n        }\n\n        Hash otherHash = (Hash) obj;\n        return hasSameValues(otherHash);\n    }\n\n    private boolean hasSameValues(Hash otherHash)\n    {\n        return Arrays.equals(this.result, otherHash.result) //\n                && Arrays.equals(this.bytes, otherHash.bytes) //\n                && Arrays.equals(this.salt, otherHash.salt) //\n                && areEquals(this.pepper, otherHash.pepper) //\n                && this.hashingFunction.equals(otherHash.hashingFunction);\n    }\n\n    private static boolean areEquals(CharSequence cs1, CharSequence cs2)\n    {\n        if (cs1 == cs2)\n        {\n            return true;\n        }\n        else if (cs1 != null && cs2 != null)\n        {\n            return cs1.equals(cs2);\n        }\n        return false;\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(Arrays.hashCode(result), Arrays.hashCode(salt), pepper, hashingFunction);\n    }\n}\n\n// Path: src/main/java/com/password4j/HashingFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\n/**\n * This interface declares the basic methods that\n * all the Cryptographic Hashing Functions (CHF) must have.\n * <p>\n * The two possible operations are\n * <ul>\n *     <li>hashing a plaintext password</li>\n *     <li>check if an hash is originated by a plaintext password</li>\n * </ul>\n * <p>\n * The purpose of this interface is encapsulate the logic of hashing\n * and checking of a CHF inside a class.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic interface HashingFunction\n{\n\n    /**\n     * Creates a {@link Hash} from a plaintext password.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * The salt is always generated randomly.\n     * <p>\n     * The generation of the salt is completely\n     * responsibility of the implementing class.\n     *\n     * @param plainTextPassword the password to be hashed\n     * @return a {@link Hash}\n     * @see HashingFunction#hash(CharSequence, String)\n     * @since 0.1.0\n     */\n    Hash hash(CharSequence plainTextPassword);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * The salt is always generated randomly.\n     * <p>\n     * The generation of the salt is completely\n     * responsibility of the implementing class.\n     *\n     * @param plainTextPassword the password to be hashed as bytes array\n     * @return a {@link Hash}\n     * @see HashingFunction#hash(CharSequence, String)\n     * @since 1.7.0\n     */\n    Hash hash(byte[] plainTextPassword);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed\n     * @param salt              the salt used in the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 0.1.0\n     */\n    Hash hash(CharSequence plainTextPassword, String salt);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed as bytes array\n     * @param salt              the salt as bytes array used in the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 1.7.0\n     */\n    Hash hash(byte[] plainTextPassword, byte[] salt);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed\n     * @param salt              the salt used in the hashing process\n     * @param pepper            the pepper used int the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 1.5.0\n     */\n    Hash hash(byte[] plainTextPassword, byte[] salt, CharSequence pepper);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed as bytes array\n     * @param salt              the salt as bytes array used in the hashing process\n     * @param pepper            the pepper used int the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 1.7.0\n     */\n    Hash hash(CharSequence plainTextPassword, String salt, CharSequence pepper);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password.\n     * <p>\n     * This method must be used when the salt is part\n     * of the hash.\n     * If the CHF expects a salt separated from the hash,\n     * an {@link UnsupportedOperationException} is thrown.\n     * <p>\n     * In case the CHF expects a salt to be part of the hash,\n     * if the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws UnsupportedOperationException if the CHF need a salt and it is not part of the hash\n     * @throws BadParametersException        if the hash is not well-formed\n     * @since 0.1.0\n     */\n    boolean check(CharSequence plainTextPassword, String hashed);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password.\n     * <p>\n     * This method must be used when the salt is part\n     * of the hash.\n     * If the CHF expects a salt separated from the hash,\n     * an {@link UnsupportedOperationException} is thrown.\n     * <p>\n     * In case the CHF expects a salt to be part of the hash,\n     * if the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password as bytes array\n     * @param hashed            the hash as bytes array\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws UnsupportedOperationException if the CHF need a salt and it is not part of the hash\n     * @throws BadParametersException        if the hash is not well-formed\n     * @since 1.7.0\n     */\n    boolean check(byte[] plainTextPassword, byte[] hashed);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @param salt              the salt used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 0.2.1\n     */\n    boolean check(CharSequence plainTextPassword, String hashed, String salt);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password as bytes array\n     * @param hashed            the hash as bytes array\n     * @param salt              the salt as bytes array used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 1.7.0\n     */\n    boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @param salt              the salt used to produce the hash\n     * @param pepper            the pepper used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 1.5.0\n     */\n    boolean check(CharSequence plainTextPassword, String hashed, String salt, CharSequence pepper);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password as bytes array\n     * @param hashed            the hash as bytes array\n     * @param salt              the salt as bytes array used to produce the hash\n     * @param pepper            the pepper used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 1.5.0\n     */\n...\n", "needle_token_start": 11988, "needle_token_end": 12011, "code_context_ntokens": 16033, "output": ["```java\npublic String getResult()\n{\n    return Utils.fromBytesToString(result);\n}\n```"]}
{"repo": "Password4j/password4j", "name": "sMix", "language": "java", "path": "src/main/java/com/password4j/ScryptFunction.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to mix input data using cryptographic operations to enhance security, typically used in password-based key derivation functions.\n2. **Input**: Takes a byte array representing salt that has been processed, an index for the starting position in the salt array, a large byte array for storing intermediate states, and another byte array for the output of mixing operations.\n3. **Output**: The function modifies the salt array in-place, updating it with the mixed data.\n4. **Procedure**: \n   - Copies a segment of the salt array into the output array.\n   - Repeatedly mixes the data using a cryptographic mixing function (like Salsa20/8) and stores intermediate states in a large array.\n   - Uses a derived integer from the current state to randomly select segments from the intermediate state array, mixes them again, and updates the output array.\n   - Finally, copies the mixed data back into the salt array at the specified position.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/BcryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Bcrypt;\n\nimport java.security.MessageDigest;\nimport java.util.Arrays;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of bcrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Bcrypt\">bcrypt</a>\n * @since 0.1.0\n */\npublic class BcryptFunction extends AbstractHashingFunction\n{\n\n    private static final ConcurrentMap<String, BcryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final int BCRYPT_SALT_LEN = 16;\n\n    private static final int BLOWFISH_NUM_ROUNDS = 16;\n\n    private static final int[] P_ORIG = { //\n            0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6,\n            0x38d01377, 0xbe5466cf, 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917, 0x9216d5d9, 0x8979fb1b };\n\n    private static final int[] S_ORIG = { //\n            0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99, 0x24a19947,\n            0xb3916cf7, 0x0801f2e2, 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658,\n            0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, 0xc5d1b023, 0x286085f0, 0xca417918,\n            0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60,\n            0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, 0xa15486af,\n            0x7c72e993, 0xb3ee1411, 0x636fbc2a, 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c,\n            0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60,\n            0x5dec8032, 0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239,\n            0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0, 0x6a51a0d2,\n            0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176,\n            0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, 0xe06f75d8, 0x85c12073, 0x401a449f,\n            0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b,\n            0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6,\n            0x409f60c4, 0x5e5c9ec2, 0x196a2463, 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c,\n            0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, 0xc0cba857, 0x45c8740f, 0xd20b5f39,\n            0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8,\n            0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82, 0x9e5c57bb,\n            0xca6f8ca0, 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8,\n            0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, 0x9a53e479, 0xb6f84565, 0xd28e49bc,\n            0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4,\n            0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb,\n            0xf2122b64, 0x8888b812, 0x900df01c, 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777,\n            0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81,\n            0xd2ada8d9, 0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf,\n            0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af, 0x2464369b,\n            0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9,\n            0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, 0xd60f573f, 0xbc9bc6e4, 0x2b60a476,\n            0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664,\n            0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a, 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0,\n            0x49a7df7d, 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, 0x193602a5, 0x75094c29,\n            0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6,\n            0xf0255dc1, 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1,\n            0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37,\n            0xf0500c0d, 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, 0xd19113f9, 0x7ca92ff6,\n            0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e,\n...\n// Path: src/main/java/com/password4j/types/Hmac.java\n/*\n *  (C) Copyright 2021 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j.types;\n\nimport com.password4j.CompressedPBKDF2Function;\n\n\n/**\n * Static representation of the commonly supported\n * Hmac variants.\n */\npublic enum Hmac\n{\n\n    SHA1(160, 1), //\n    SHA224(224, 2), //\n    SHA256(256, 3), //\n    SHA384(384, 4), //\n    SHA512(512, 5);\n\n    private final int bits;\n\n    private final int code;\n\n    /**\n     * @param bits length of the produced hash\n     * @param code uid used by {@link CompressedPBKDF2Function}\n     */\n    Hmac(int bits, int code)\n    {\n        this.bits = bits;\n        this.code = code;\n    }\n\n    /**\n     * Finds the enum associated with the given code\n     *\n     * @param code a numeric uid that identifies the algorithm\n     * @return a {@link Hmac} enum. Null if the code is not present in this enum\n     */\n    public static Hmac fromCode(int code)\n    {\n        for (Hmac alg : values())\n        {\n            if (alg.code() == code)\n            {\n                return alg;\n            }\n        }\n        return null;\n    }\n\n    /**\n     * @return length of the algorithm in bits\n     */\n    public int bits()\n    {\n        return bits;\n    }\n\n    /**\n     * @return the numeric uid used in {@link CompressedPBKDF2Function}\n     */\n    public int code()\n    {\n        return code;\n    }\n\n    @Override\n    public String toString()\n    {\n        return \"PBKDF2WithHmac\" + this.name();\n    }\n}\n\n// Path: src/main/java/com/password4j/CompressedPBKDF2Function.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of PBKDF2 function and its parameters.\n * <p>\n * The main difference between {@link PBKDF2Function} is the hash produced: the configurations of the CHF,\n * the salt and the hash are encoded inside it.\n * <p>\n * The produced hash is in the form\n * <p>\n * <code>\n * $algorithm$parameters$salt$hash\n * </code>\n * <p>\n * Assuming {@code $} as delimiter.\n * <p>\n * <ul>\n *     <li>\n *         The algorithm is encoded with its numeric uid {@link Hmac#code()}\n *     </li>\n *     <li>\n *         Parameters are encoded in one integer where the length occupies the first 32bit and\n *          the number of iterations the remaining 32 bits.\n *     </li>\n *     <li>\n *         Salt is encoded in Base64\n *     </li>\n *     <li>\n *         Hash is encoded in Base64 as in {@link PBKDF2Function}\n *     </li>\n * </ul>\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/PBKDF2\">PBKDF2</a>\n * @since 0.1.0\n */\npublic class CompressedPBKDF2Function extends PBKDF2Function\n{\n\n    private static final Map<String, CompressedPBKDF2Function> INSTANCES = new ConcurrentHashMap<>();\n\n    private static final char DELIMITER = PropertyReader.readChar(\"hash.pbkdf2.delimiter\", '$');\n\n    protected CompressedPBKDF2Function()\n    {\n        super();\n    }\n\n    protected CompressedPBKDF2Function(Hmac fromCode, int iterations, int length)\n    {\n        super(fromCode, iterations, length);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getInstance(Hmac algorithm, int iterations, int length)\n    {\n        String key = getUID(algorithm.name(), iterations, length);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            CompressedPBKDF2Function function = new CompressedPBKDF2Function(algorithm, iterations, length);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  string version of hmac algorithm. This must me mapped in {@link Hmac}.\n     * @param iterations number of iterations\n     * @param length     length of the derived key\n     * @return a singleton instance\n     * @throws IllegalArgumentException if {@code algorithm} is not mapped in {@link Hmac}.\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getInstance(String algorithm, int iterations, int length)\n    {\n        try\n        {\n            return getInstance(Hmac.valueOf(algorithm), iterations, length);\n        }\n        catch (IllegalArgumentException iae)\n        {\n            throw new UnsupportedOperationException(\"Algorithm `\" + algorithm + \"` is not recognized.\", iae);\n        }\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.0.0\n     */\n    public static CompressedPBKDF2Function getInstanceFromHash(String hashed)\n    {\n        String[] parts = getParts(hashed);\n        if (parts.length == 5)\n        {\n            int algorithm = Integer.parseInt(parts[1]);\n            long configuration = Long.parseLong(parts[2]);\n\n            int iterations = (int) (configuration >> 32);\n            int length = (int) configuration;\n\n            return CompressedPBKDF2Function.getInstance(Hmac.fromCode(algorithm), iterations, length);\n        }\n        throw new BadParametersException(\"`\" + hashed + \"` is not a valid hash\");\n    }\n\n\n\n    @Override\n    protected String getHash(byte[] encodedKey, byte[] salt)\n    {\n        String params = Long.toString((((long) getIterations()) << 32) | (getLength() & 0xffffffffL));\n        String salt64 = Utils.encodeBase64(salt);\n        String hash64 = super.getHash(encodedKey, salt);\n        return \"$\" + algorithm.code() + \"$\" + params + \"$\" + salt64 + \"$\" + hash64;\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        byte[] salt = getSaltFromHash(hashed);\n        Hash internalHas = hash(plainTextPassword, salt);\n\n        return slowEquals(internalHas.getResultAsBytes(), hashed);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        byte[] hashAsBytes = Utils.fromCharSequenceToBytes(hashed);\n        byte[] realSalt = getSaltFromHash(hashAsBytes);\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        Hash internalHash = hash(plainTextPasswordAsBytes, realSalt);\n        return slowEquals(internalHash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        byte[] realSalt = getSaltFromHash(hashed);\n        Hash internalHash = hash(plainTextPassword, realSalt);\n        return slowEquals(internalHash.getResultAsBytes(), hashed);\n    }\n\n    protected static List<byte[]> getParts(byte[] hashed)\n    {\n       return Utils.split(hashed, (byte) DELIMITER);\n    }\n\n    protected static String[] getParts(String hashed)\n    {\n        String regex = \"\\\\\" + DELIMITER;\n        return hashed.split(regex);\n    }\n\n    private byte[] getSaltFromHash(byte[] hashed)\n    {\n        List<byte[]> parts = getParts(hashed);\n        if (parts.size() == 5)\n        {\n            return Utils.decodeBase64(parts.get(3));\n        }\n        throw new BadParametersException(\"`\" + Utils.fromBytesToString(hashed) + \"` is not a valid hash\");\n    }\n}\n\n// Path: src/main/java/com/password4j/SaltOption.java\npackage com.password4j;\n\npublic enum SaltOption\n{\n    PREPEND, APPEND;\n}\n\n// Path: src/main/java/com/password4j/MessageDigestFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Class containing the implementation of Messaged digest functions provided by {@link MessageDigest}.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Category:Cryptographic_hash_functions\">Message digests</a>\n * @since 1.4.0\n */\npublic class MessageDigestFunction extends AbstractHashingFunction\n{\n    private static final Map<String, MessageDigestFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    protected static final SaltOption DEFAULT_SALT_OPTION = SaltOption.APPEND;\n\n    private final String algorithm;\n\n    private final SaltOption saltOption;\n\n\n    MessageDigestFunction(String algorithm, SaltOption saltOption)\n    {\n        this.algorithm = algorithm;\n        this.saltOption = saltOption;\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm message digest algorithm\n     * @return a singleton instance\n     * @since 1.4.0\n     */\n    public static MessageDigestFunction getInstance(String algorithm)\n    {\n        return getInstance(algorithm, DEFAULT_SALT_OPTION);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * algorithm, number of iterations and key length.\n     *\n     * @param algorithm  hmac algorithm\n     * @param saltOption a configuration that specifies how the salt is concatenated to the plain text password\n     * @return a singleton instance\n     * @since 1.4.0\n     */\n    public static MessageDigestFunction getInstance(String algorithm, SaltOption saltOption)\n    {\n        String key = getUID(algorithm, saltOption);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            MessageDigestFunction function = new MessageDigestFunction(algorithm, saltOption);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    protected static String getUID(String algorithm, SaltOption saltOption)\n    {\n        return algorithm + \"|\" + saltOption.name();\n    }\n\n    protected static String toString(String algorithm, SaltOption saltOption)\n    {\n        return \"a=\" + algorithm + \", o=\" + saltOption.name();\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        return hash(plainTextPassword, null);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        return hash(plainTextPasswordAsBytes, null);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes, byte[] saltAsBytes)\n    {\n        return internalHash(plainTextPasswordAsBytes, saltAsBytes);\n    }\n\n    protected Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n        byte[] finalCharSequence = concatenateSalt(plainTextPassword, salt);\n\n        byte[] result = getMessageDigest().digest(finalCharSequence);\n        return new Hash(this, Utils.toHex(result), result, salt);\n    }\n\n    protected MessageDigest getMessageDigest()\n    {\n        try\n        {\n            return MessageDigest.getInstance(algorithm);\n        } catch (NoSuchAlgorithmException nsae)\n        {\n            throw new UnsupportedOperationException(\"`\" + algorithm + \"` is not supported by your system.\", nsae);\n        }\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(plainTextPassword, hashed, null);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPasswordAsBytes, byte[] hashed)\n    {\n        return check(plainTextPasswordAsBytes, hashed, null);\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed, String salt)\n    {\n        Hash hash = internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(salt));\n        return slowEquals(hash.getResult(), hashed);\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt)\n    {\n        Hash hash = internalHash(plainTextPassword, salt);\n        return slowEquals(hash.getResultAsBytes(), hashed);\n    }\n\n    /**\n     * The salt option describes if the Salt is appended or prepended to\n     * the plain text password.\n     *\n     * @return how the salt is concatenated\n     * @since 1.5.1\n     */\n    public SaltOption getSaltOption()\n    {\n        return saltOption;\n    }\n\n    /**\n     * The algorithm in use by this instance.\n     *\n     * @return the algorithm in use\n     * @since 1.5.1\n     */\n    public String getAlgorithm()\n    {\n        return algorithm;\n    }\n\n\n    private byte[] concatenateSalt(byte[] plainTextPassword, byte[] salt)\n    {\n        if (salt == null || salt.length == 0)\n        {\n            return  plainTextPassword;\n        }\n\n        if (saltOption == SaltOption.PREPEND)\n        {\n            return Utils.append(salt, plainTextPassword);\n        }\n        return Utils.append(plainTextPassword, salt);\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(this.algorithm, this.saltOption) + ')';\n    }\n\n    @Override\n    public boolean equals(Object o)\n    {\n        if (this == o)\n            return true;\n        if (!(o instanceof MessageDigestFunction))\n            return false;\n        MessageDigestFunction other = (MessageDigestFunction) o;\n        return algorithm.equals(other.algorithm) //\n                && saltOption == other.saltOption;\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(algorithm, saltOption);\n    }\n}\n\n// Path: src/main/java/com/password4j/ScryptFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Hmac;\n\nimport java.security.GeneralSecurityException;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\n\n\n/**\n * Class containing the implementation of scrypt function and its parameters.\n *\n * @author David Bertoldi\n * @see <a href=\"https://en.wikipedia.org/wiki/Scrypt\">scrypt</a>\n * @since 0.1.0\n */\npublic class ScryptFunction extends AbstractHashingFunction\n{\n    public static final int DERIVED_KEY_LENGTH = 64;\n\n    private static final ConcurrentMap<String, ScryptFunction> INSTANCES = new ConcurrentHashMap<>();\n\n    private int workFactor; // N\n\n    private int resources; // r\n\n    private int parallelization; // p\n\n    private int derivedKeyLength; // dkLen\n\n    @SuppressWarnings(\"unused\")\n    private ScryptFunction()\n    {\n        //\n    }\n\n    /**\n     * @param workFactor      (N)\n     * @param resources       (r)\n     * @param parallelization (p)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = DERIVED_KEY_LENGTH;\n    }\n\n    /**\n     * @param workFactor       (N)\n     * @param resources        (r)\n     * @param parallelization  (p)\n     * @param derivedKeyLength (dkLen)\n     */\n    protected ScryptFunction(int workFactor, int resources, int parallelization, int derivedKeyLength)\n    {\n        this.resources = resources;\n        this.workFactor = workFactor;\n        this.parallelization = parallelization;\n        this.derivedKeyLength = derivedKeyLength;\n    }\n\n    /**\n     * Reads the configuration contained in the given hash and\n     * builds a singleton instance based on these configurations.\n     *\n     * @param hashed an already hashed password\n     * @return a singleton instance based on the given hash\n     * @since 1.0.0\n     */\n    public static ScryptFunction getInstanceFromHash(String hashed)\n    {\n        String[] parts = hashed.split(\"\\\\$\");\n        if (parts.length == 4)\n        {\n            long params = Long.parseLong(parts[1], 16);\n            int workFactor = (int) Math.pow(2.0D, (params >> 16 & 65535L));\n            int resources = (int) params >> 8 & 255;\n            int parallelization = (int) params & 255;\n            int derivedKeyLength = Utils.decodeBase64(parts[3]).length;\n\n            return ScryptFunction.getInstance(workFactor, resources, parallelization, derivedKeyLength);\n        }\n        throw new BadParametersException(\"`\" + hashed + \"` is not a valid hash\");\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * N, r and p parameters.\n     *\n     * @param workFactor      work factor (N)\n     * @param resources       resources (r)\n     * @param parallelization parallelization (p)\n     * @return a singleton instance\n     * @since 0.3.0\n     */\n    public static ScryptFunction getInstance(int workFactor, int resources, int parallelization)\n    {\n        return getInstance(workFactor, resources, parallelization, DERIVED_KEY_LENGTH);\n    }\n\n    /**\n     * Creates a singleton instance, depending on the provided\n     * N, r and p parameters.\n     *\n     * @param workFactor       work factor (N)\n     * @param resources        resources (r)\n     * @param parallelization  parallelization (p)\n     * @param derivedKeyLength derived key length (dkLen)\n     * @return a singleton instance\n     * @since 1.5.1\n     */\n    public static ScryptFunction getInstance(int workFactor, int resources, int parallelization, int derivedKeyLength)\n    {\n        String key = getUID(resources, workFactor, parallelization, derivedKeyLength);\n        if (INSTANCES.containsKey(key))\n        {\n            return INSTANCES.get(key);\n        }\n        else\n        {\n            ScryptFunction function = new ScryptFunction(workFactor, resources, parallelization, derivedKeyLength);\n            INSTANCES.put(key, function);\n            return function;\n        }\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(Utils.fromCharSequenceToBytes(plainTextPassword), salt);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes)\n    {\n        byte[] salt = SaltGenerator.generate();\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    @Override\n    public Hash hash(CharSequence plainTextPassword, String salt)\n    {\n        byte[] saltAsBytes = Utils.fromCharSequenceToBytes(salt);\n        byte[] plainTextPasswordAsBytes = Utils.fromCharSequenceToBytes(plainTextPassword);\n        return internalHash(plainTextPasswordAsBytes, saltAsBytes);\n    }\n\n    @Override\n    public Hash hash(byte[] plainTextPasswordAsBytes, byte[] salt)\n    {\n        return internalHash(plainTextPasswordAsBytes, salt);\n    }\n\n    private Hash internalHash(byte[] plainTextPassword, byte[] salt)\n    {\n\n        try\n        {\n            byte[] derived = scrypt(plainTextPassword, salt, derivedKeyLength);\n            String params = Long.toString((long) Utils.log2(workFactor) << 16 | (long) resources << 8 | parallelization, 16);\n            String sb = \"$\" + params + '$' + Utils.encodeBase64(salt) + '$'\n                    + Utils.encodeBase64(derived);\n            return new Hash(this, sb, derived, salt);\n        }\n        catch (IllegalArgumentException | GeneralSecurityException e)\n        {\n            String stringedSalt = Utils.fromBytesToString(salt);\n            String message = \"Invalid specification with salt=\" + stringedSalt + \", N=\" + workFactor + \", r=\" + resources + \" and p=\" + parallelization;\n            throw new BadParametersException(message, e);\n        }\n    }\n\n    @Override\n    public boolean check(CharSequence plainTextPassword, String hashed)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), Utils.fromCharSequenceToBytes(hashed));\n    }\n\n    @Override\n    public boolean check(byte[] plainTextPassword, byte[] hashed)\n    {\n        try\n        {\n            List<byte[]> parts = Utils.split(hashed, (byte) 36);\n            if (parts.size() == 4)\n            {\n                byte[] salt = Utils.decodeBase64(parts.get(2));\n                byte[] derived0 = Utils.decodeBase64(parts.get(3));\n                byte[] derived1 = scrypt(plainTextPassword, salt, derivedKeyLength);\n                return slowEquals(derived0, derived1);\n            }\n            else\n            {\n                throw new BadParametersException(\"Invalid hashed value\");\n            }\n        }\n        catch (GeneralSecurityException gse)\n        {\n            throw new IllegalStateException(\"JVM doesn't support SHA1PRNG or HMAC_SHA256?\", gse);\n        }\n    }\n\n    protected static String toString(int resources, int workFactor, int parallelization, int derivedKeyLength)\n    {\n        return \"N=\" + workFactor + \", r=\" + resources + \", p=\" + parallelization + \", l=\" + derivedKeyLength;\n    }\n\n    protected static String getUID(int resources, int workFactor, int parallelization, int derivedKeyLength)\n    {\n        return workFactor + \"|\" + resources + \"|\" + parallelization + \"|\" + derivedKeyLength;\n    }\n\n    public static int rOperation(int a, int b)\n    {\n        return a << b | a >>> 32 - b;\n    }\n\n    public static void salsa208(byte[] xArray)\n    {\n        int[] b32 = new int[16];\n        int[] x = new int[16];\n\n        int i;\n        for (i = 0; i < 16; ++i)\n        {\n            b32[i] = (xArray[i * 4] & 255);\n            b32[i] |= (xArray[i * 4 + 1] & 255) << 8;\n            b32[i] |= (xArray[i * 4 + 2] & 255) << 16;\n            b32[i] |= (xArray[i * 4 + 3] & 255) << 24;\n        }\n\n        System.arraycopy(b32, 0, x, 0, 16);\n\n        for (i = 8; i > 0; i -= 2)\n        {\n            x[4] ^= rOperation(x[0] + x[12], 7);\n            x[8] ^= rOperation(x[4] + x[0], 9);\n            x[12] ^= rOperation(x[8] + x[4], 13);\n            x[0] ^= rOperation(x[12] + x[8], 18);\n            x[9] ^= rOperation(x[5] + x[1], 7);\n            x[13] ^= rOperation(x[9] + x[5], 9);\n            x[1] ^= rOperation(x[13] + x[9], 13);\n            x[5] ^= rOperation(x[1] + x[13], 18);\n            x[14] ^= rOperation(x[10] + x[6], 7);\n            x[2] ^= rOperation(x[14] + x[10], 9);\n            x[6] ^= rOperation(x[2] + x[14], 13);\n            x[10] ^= rOperation(x[6] + x[2], 18);\n            x[3] ^= rOperation(x[15] + x[11], 7);\n            x[7] ^= rOperation(x[3] + x[15], 9);\n            x[11] ^= rOperation(x[7] + x[3], 13);\n            x[15] ^= rOperation(x[11] + x[7], 18);\n            x[1] ^= rOperation(x[0] + x[3], 7);\n            x[2] ^= rOperation(x[1] + x[0], 9);\n            x[3] ^= rOperation(x[2] + x[1], 13);\n            x[0] ^= rOperation(x[3] + x[2], 18);\n            x[6] ^= rOperation(x[5] + x[4], 7);\n            x[7] ^= rOperation(x[6] + x[5], 9);\n            x[4] ^= rOperation(x[7] + x[6], 13);\n            x[5] ^= rOperation(x[4] + x[7], 18);\n            x[11] ^= rOperation(x[10] + x[9], 7);\n            x[8] ^= rOperation(x[11] + x[10], 9);\n            x[9] ^= rOperation(x[8] + x[11], 13);\n            x[10] ^= rOperation(x[9] + x[8], 18);\n            x[12] ^= rOperation(x[15] + x[14], 7);\n            x[13] ^= rOperation(x[12] + x[15], 9);\n            x[14] ^= rOperation(x[13] + x[12], 13);\n            x[15] ^= rOperation(x[14] + x[13], 18);\n        }\n\n        for (i = 0; i < 16; ++i)\n        {\n            b32[i] += x[i];\n        }\n\n        for (i = 0; i < 16; ++i)\n        {\n            xArray[i * 4] = (byte) (b32[i] & 255);\n            xArray[i * 4 + 1] = (byte) (b32[i] >> 8 & 255);\n            xArray[i * 4 + 2] = (byte) (b32[i] >> 16 & 255);\n            xArray[i * 4 + 3] = (byte) (b32[i] >> 24 & 255);\n        }\n\n    }\n\n    public static void blockXOR(byte[] sArray, int si, byte[] dArray, int di, int length)\n    {\n        for (int i = 0; i < length; ++i)\n        {\n            dArray[di + i] ^= sArray[si + i];\n        }\n\n    }\n\n\n\n    /**\n     * Estimates the required memory to calculate an hash with\n     * the current configuration.\n     *\n     * @return the required memory\n     * @since 0.1.0\n     */\n    public long getRequiredBytes()\n    {\n        return 128L * workFactor * resources * parallelization;\n    }\n\n    public int getWorkFactor()\n    {\n        return workFactor;\n    }\n\n    public int getResources()\n    {\n        return resources;\n    }\n\n    public int getParallelization()\n    {\n        return parallelization;\n    }\n\n    public int getDerivedKeyLength()\n    {\n        return derivedKeyLength;\n    }\n\n    /**\n     * A more readable version of {@link #getRequiredBytes()},\n     * changing the unit (B, KB, MB) so that the number has at most\n     * 2 decimal places.\n     *\n     * @return the required memory\n     * @since 0.3.0\n     */\n    public String getRequiredMemory()\n    {\n        long memoryInBytes = getRequiredBytes();\n        if (memoryInBytes > 1_000_000)\n        {\n            return Math.round(memoryInBytes / 10_000f) / 100.0 + \"MB\";\n        }\n        if (memoryInBytes > 1_000)\n        {\n            return Math.round(memoryInBytes / 1_000f) / 100.0 + \"KB\";\n        }\n        return memoryInBytes + \"B\";\n    }\n\n    @Override\n    public boolean equals(Object obj)\n    {\n        if (obj == null || !this.getClass().equals(obj.getClass()))\n        {\n            return false;\n        }\n\n        ScryptFunction otherStrategy = (ScryptFunction) obj;\n        return this.workFactor == otherStrategy.workFactor //\n                && this.resources == otherStrategy.resources //\n                && this.parallelization == otherStrategy.parallelization;\n    }\n\n    @Override\n    public String toString()\n    {\n        return getClass().getSimpleName() + '(' + toString(this.resources, this.workFactor, this.parallelization,\n                this.derivedKeyLength) + ')';\n    }\n\n    @Override\n    public int hashCode()\n    {\n        return Objects.hash(resources, workFactor, parallelization);\n    }\n\n    public byte[] scrypt(byte[] passwd, byte[] salt, int dkLen) throws GeneralSecurityException\n    {\n        if (workFactor >= 2 && (workFactor & workFactor - 1) == 0)\n        {\n            if (workFactor > 16777215 / resources)\n            {\n                throw new IllegalArgumentException(\"Parameter N is too large\");\n            }\n            else if (resources > 16777215 / parallelization)\n            {\n                throw new IllegalArgumentException(\"Parameter r is too large\");\n            }\n            else\n            {\n                byte[] xyArray = new byte[256 * resources];\n                byte[] vArray = new byte[128 * resources * workFactor];\n                byte[] intensiveSalt = PBKDF2Function.internalHash(Utils.fromBytesToString(passwd).toCharArray(), salt, Hmac.SHA256.name(), 1,\n                        8 * parallelization * 128 * resources).getEncoded();\n\n                for (int i = 0; i < parallelization; ++i)\n                {\n                    sMix(intensiveSalt, i * 128 * resources, vArray, xyArray);\n                }\n\n                return PBKDF2Function\n                        .internalHash(Utils.fromBytesToString(passwd).toCharArray(), intensiveSalt, Hmac.SHA256.name(), 1, 8 * dkLen)\n                        .getEncoded();\n            }\n        }\n        else\n        {\n            throw new IllegalArgumentException(\"N must be a power of 2 greater than 1. Found \" + workFactor);\n        }\n    }\n\n    \npublic void sMix(byte[] intensiveSalt, int bi, byte[] vArray, byte[] xyArray)\n    {\n        int xi = 0;\n        int yi = 128 * resources;\n        System.arraycopy(intensiveSalt, bi, xyArray, xi, 128 * resources);\n\n        int i;\n        for (i = 0; i < workFactor; ++i)\n        {\n            System.arraycopy(xyArray, xi, vArray, i * 128 * resources, 128 * resources);\n            blockmixSalsa8(xyArray, xi, yi);\n        }\n\n        for (i = 0; i < workFactor; ++i)\n        {\n            int j = integerify(xyArray, xi) & workFactor - 1;\n            blockXOR(vArray, j * 128 * resources, xyArray, xi, 128 * resources);\n            blockmixSalsa8(xyArray, xi, yi);\n        }\n\n        System.arraycopy(xyArray, xi, intensiveSalt, bi, 128 * resources);\n    }\n\n    public void blockmixSalsa8(byte[] xyArray, int bi, int yi)\n    {\n        byte[] xArray = new byte[64];\n        System.arraycopy(xyArray, bi + (2 * resources - 1) * 64, xArray, 0, 64);\n\n        int i;\n        for (i = 0; i < 2 * resources; ++i)\n        {\n            blockXOR(xyArray, i * 64, xArray, 0, 64);\n            salsa208(xArray);\n            System.arraycopy(xArray, 0, xyArray, yi + i * 64, 64);\n        }\n\n        for (i = 0; i < resources; ++i)\n        {\n            System.arraycopy(xyArray, yi + i * 2 * 64, xyArray, bi + i * 64, 64);\n        }\n\n        for (i = 0; i < resources; ++i)\n        {\n            System.arraycopy(xyArray, yi + (i * 2 + 1) * 64, xyArray, bi + (i + resources) * 64, 64);\n        }\n\n    }\n\n    public int integerify(byte[] xyArray, int xi)\n    {\n        xi += (2 * resources - 1) * 64;\n        int n = (xyArray[xi] & 255);\n        n |= (xyArray[xi + 1] & 255) << 8;\n        n |= (xyArray[xi + 2] & 255) << 16;\n        n |= (xyArray[xi + 3] & 255) << 24;\n        return n;\n    }\n}\n\n// Path: src/main/java/com/password4j/AlgorithmFinder.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport com.password4j.types.Argon2;\nimport com.password4j.types.Bcrypt;\nimport com.password4j.types.Hmac;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.security.NoSuchAlgorithmException;\nimport java.security.Provider;\nimport java.security.SecureRandom;\nimport java.security.Security;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Set;\n\n\n/**\n * This utility class finds algorithms with their configuration\n * based on the environment.\n * <p>\n * In this context, by environment is intended the set of\n * parameters set in the JVM and in the <i>psw4j.properties</i> file.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic class AlgorithmFinder\n{\n\n    private static final Logger LOG = LoggerFactory.getLogger(AlgorithmFinder.class);\n\n    /**\n     * Singleton instance of {@link SecureRandom}.\n     * <p>\n     * By definition this instance does not need\n     * to be re-instantiated in order to generate\n     * non-deterministic output.\n     *\n     * @see #getSecureRandom()\n     * @since 0.1.0\n     */\n    private static SecureRandom secureRandom;\n\n    private AlgorithmFinder()\n    {\n        //\n    }\n\n    /**\n     * Retrieve a singleton instance of a thread-safe {@link SecureRandom} object.\n     * If the environment allows it, a cryptographically strong random number generator (RNG)\n     * is returned.\n     * <p>\n     * The usage of {@link SecureRandom#getInstanceStrong()} can be forced by\n     * setting {@code global.random.strong} to {@code true} in the <i>psw4j.properties</i>.\n     * Please be aware that this configuration may affect the system performance, while\n     * ensuring an high level of randomness.\n     * <p>\n     * It is recommendable for Unix systems to set {@code securerandom.source} to\n     * {@code file:/dev/urandom} in your <i>java.security</i> file.\n     *\n     * @return a thread-safe {@link SecureRandom} instance\n     * @see SecureRandom#getInstanceStrong()\n     * @since 0.1.0\n     */\n    public static SecureRandom getSecureRandom()\n    {\n        return secureRandom;\n    }\n\n    /**\n     * Creates a singleton instance of {@link PBKDF2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Algorithm</td>\n     *     <td>hash.pbkdf2.algorithm</td>\n     *     <td>PBKDF2WithHmacSHA256</td>\n     *   </tr>\n     *   <tr>\n     *     <td># iterations</td>\n     *     <td>hash.pbkdf2.iterations</td>\n     *     <td>310000</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Key length</td>\n     *     <td>hash.pbkdf2.length</td>\n     *     <td>512</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link PBKDF2Function}\n     * @since 0.1.0\n     */\n    public static PBKDF2Function getPBKDF2Instance()\n    {\n        Param params = internalGetProperties();\n        return PBKDF2Function.getInstance(params.algorithm, params.iterations, params.length);\n    }\n\n    /**\n     * Creates a singleton instance of {@link CompressedPBKDF2Function}\n     * with the configuration set in the <i>psw4j.properties</i> file.\n     * <p>\n     * If no <i>psw4j.properties</i> is found in the classpath or no\n     * value is provided for a parameter, the\n     * default configuration is used.\n     * <table>\n     *   <tr>\n     *     <th>Parameter</th>\n     *     <th>Property</th>\n     *     <th>Default</th>\n     *   </tr>\n     *   <tr>\n     *     <td>Algorithm</td>\n     *     <td>hash.pbkdf2.algorithm</td>\n     *     <td>PBKDF2WithHmacSHA512</td>\n     *   </tr>\n     *   <tr>\n     *     <td># iterations</td>\n     *     <td>hash.pbkdf2.iterations</td>\n     *     <td>64000</td>\n     *   </tr>\n     *   <tr>\n     *     <td>Key length</td>\n     *     <td>hash.pbkdf2.length</td>\n     *     <td>512</td>\n     *   </tr>\n     * </table>\n     *\n     * @return a {@link CompressedPBKDF2Function}\n     * @since 0.1.0\n     */\n    public static CompressedPBKDF2Function getCompressedPBKDF2Instance()\n    {\n        Param params = internalGetProperties();\n        return CompressedPBKDF2Function.getInstance(params.algorithm, params.iterations, params.length);\n    }\n\n    private static Param internalGetProperties()\n    {\n        String algorithm = PropertyReader\n                .readString(\"hash.pbkdf2.algorithm\", Hmac.SHA256.name(), \"PBKDF2 algorithm is not defined\");\n        int iterations = PropertyReader.readInt(\"hash.pbkdf2.iterations\", 310_000, \"PBKDF2 #iterations are not defined\");\n...\n", "needle_token_start": 13473, "needle_token_end": 13761, "code_context_ntokens": 16021, "output": ["```java\npublic void sMix(byte[] intensiveSalt, int bi, byte[] vArray, byte[] xyArray)\n{\n    int xi = 0;\n    int yi = 128 * resources;\n    System.arraycopy(intensiveSalt, bi, xyArray, xi, 128 * resources);\n\n    int i;\n    for (i = 0; i < workFactor; ++i)\n    {\n        System.arraycopy(xyArray, xi, vArray, i * 128 * resources, 128 * resources);\n        blockmixSalsa8(xyArray, xi, yi);\n    }\n\n    for (i = 0; i < workFactor; ++i)\n    {\n        int j = integerify(xyArray, xi) & workFactor - 1;\n        blockXOR(vArray, j * 128 * resources, xyArray, xi, 128 * resources);\n        blockmixSalsa8(xyArray, xi, yi);\n    }\n\n    System.arraycopy(xyArray, xi, intensiveSalt, bi, 128 * resources);\n}\n```"]}
{"repo": "Password4j/password4j", "name": "isPBKDF2Supported", "language": "java", "path": "src/main/java/com/password4j/SystemChecker.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine if a specific cryptographic algorithm variant is supported by the current Java Virtual Machine (JVM) environment.\n2. **Input**: A string representing the name of the cryptographic algorithm variant to be checked.\n3. **Output**: A boolean value; `true` if the specified algorithm variant is supported, `false` otherwise.\n4. **Procedure**: The function first checks if the input string is null and throws an exception if it is. It then retrieves a list of all supported PBKDF2 algorithm variants. It iterates through this list to see if the specified algorithm variant matches any in the list. If a match is found, it returns `true`; if no match is found after checking all variants, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/password4j/Hash.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\nimport java.util.Arrays;\nimport java.util.Objects;\n\n\n/**\n * This class contains all the information computed after\n * calculating a cryptographic hash.\n * <p>\n * The same {@link HashingFunction} used to generate the hash\n * is used to verify the plain password; in addition <i>cryptographic\n * seasoning</i> such as salt and pepper are stored in this object.\n * <p>\n * A hash is the product of a one-way function that maps data of arbitrary size to\n * fixed-size values; it is called hashing function (HF).\n * This class represent hashes generated by cryptographic hash function (CHF),\n * where each function has the following properties:\n * <ul>\n * <li>it is deterministic, meaning that the same message always results in the same hash</li>\n * <li>it is quick to compute the hash value for any given message</li>\n * <li>it is infeasible to generate a message that yields a given hash value</li>\n * <li>it is infeasible to find two different messages with the same hash value</li>\n * <li>a small change to a message should change the hash value so extensively that the new hash value\n * appears uncorrelated with the old hash value</li>\n * </ul>\n * <p>\n * A salt is a unique, randomly generated string that is added to each password as part of the hashing process.\n * As the salt is unique for every user, an attacker has to crack hashes one at a time using the respective salt,\n * rather than being able to calculate a hash once and compare it against every stored hash.\n * <p>\n * A pepper can be used in additional to salting to provide an additional layer of protection.\n * It is similar to a salt, but has two key differences:\n * <ul>\n * <li>The pepper is shared between all stored passwords, rather than being unique like a salt.</li>\n * <li>The pepper is not stored in the database, unlike the salts.</li>\n * </ul>\n *\n * @author David Bertoldi\n * @see <a href=\"https://owasp.org/www-project-cheat-sheets/cheatsheets/Password_Storage_Cheat_Sheet\">OWASP Password Storage Cheat Sheet</a>\n * @see <a href=\"https://en.wikipedia.org/wiki/Key_derivation_function\">Key derivation function</a>\n * @see <a href=\"https://en.wikipedia.org/wiki/Cryptographic_hash_function\">Cryptographic hash function</a>\n * @since 0.1.0\n */\npublic class Hash\n{\n\n    /**\n     * Represents the full output of a cryptographic hashing function.\n     * Depending on the implementation of the CHF, it may contain\n     * the salt and the configuration of the CHF itself.\n     */\n    private byte[] result;\n\n    /**\n     * Represents the computed output of a cryptographic hashing function.\n     * It never contains salt and other configurations.\n     */\n    private byte[] bytes;\n\n    /**\n     * Represents the salt: random data that is used as an additional input\n     * to a cryptographic hashing function.\n     */\n    private byte[] salt;\n\n    /**\n     * Represents the pepper: a secret added to the input password\n     * prior to being hashed with a cryptographic hash function\n     */\n    private CharSequence pepper;\n\n    /**\n     * Represents the hashing function used to generate this object.\n     *\n     * @see HashingFunction for more details\n     */\n    private HashingFunction hashingFunction;\n\n    /**\n     * It is meant to not be used if not internally.\n     * The other constructor must be used instead.\n     *\n     * @see Hash#Hash(HashingFunction, String, byte[], String)\n     * @since 0.1.0\n     */\n    @SuppressWarnings(\"unused\")\n    private Hash()\n    {\n        //\n    }\n\n    /**\n     * Constructs an {@link Hash} containing the basic information\n     * used and produced by the computational process of hashing a password.\n     * Other information, like <i>pepper</i> can be added with\n     * {@link #setPepper(CharSequence)}.\n     * <p>\n     * This constructor populates the object's attributes.\n     *\n     * @param hashingFunction the cryptographic algorithm used to produce the hash.\n     * @param result          the result of the computation of the hash.\n     *                        Notice that the format varies depending on the algorithm.\n     * @param bytes           the hash without additional information.\n     * @param salt            the salt used for the computation.\n     * @since 0.1.0\n     * @deprecated            As of 1.8.1 because of the salt conversion from {@link String} to byte[].\n     *                        {@link Hash#Hash(HashingFunction, String, byte[], byte[])} should be used instead.\n     */\n    @Deprecated\n    public Hash(HashingFunction hashingFunction, String result, byte[] bytes, String salt)\n    {\n        this(hashingFunction, Utils.fromCharSequenceToBytes(result), bytes, Utils.fromCharSequenceToBytes(salt));\n    }\n\n\n\n    /**\n     * Constructs an {@link Hash} containing the basic information\n     * used and produced by the computational process of hashing a password.\n     * Other information, like <i>pepper</i> can be added with\n     * {@link #setPepper(CharSequence)}.\n     * <p>\n     * This constructor populates the object's attributes.\n     *\n     * @param hashingFunction the cryptographic algorithm used to produce the hash.\n     * @param result          the result of the computation of the hash.\n     *                        Notice that the format varies depending on the algorithm.\n     * @param bytes           the hash without additional information.\n     * @param salt            the salt used for the computation.\n     * @since 0.1.0\n     */\n    public Hash(HashingFunction hashingFunction, String result, byte[] bytes, byte[] salt)\n    {\n        this(hashingFunction, Utils.fromCharSequenceToBytes(result), bytes, salt);\n    }\n\n    /**\n     * Constructs an {@link Hash} containing the basic information\n     * used and produced by the computational process of hashing a password.\n     * Other information, like <i>pepper</i> can be added with\n     * {@link #setPepper(CharSequence)}.\n     * <p>\n     * This constructor populates the object's attributes.\n     *\n     * @param hashingFunction the cryptographic algorithm used to produce the hash.\n     * @param result          the result of the computation of the hash as bytes array.\n     *                        Notice that the format varies depending on the algorithm.\n     * @param bytes           the hash without additional information.\n     * @param salt            the salt used for the computation as bytes array.\n     * @since 1.7.0\n     */\n    public Hash(HashingFunction hashingFunction, byte[] result, byte[] bytes, byte[] salt)\n    {\n        this.hashingFunction = hashingFunction;\n        this.salt = salt;\n        this.result = result;\n        this.bytes = bytes;\n    }\n\n    /**\n     * Retrieves the hash computed by the hashing function.\n     *\n     * @return the hash.\n     * @since 0.1.0\n     */\n    public String getResult()\n    {\n        return Utils.fromBytesToString(result);\n    }\n\n    /**\n     * Retrieves the hash computed by the hashing function.\n     *\n     * @return the hash.\n     * @since 0.1.0\n     */\n    public byte[] getResultAsBytes()\n    {\n        return result;\n    }\n\n    /**\n     * Retrieves the hash as byte array and without the parameters\n     * encoded in the final hash.\n     *\n     * @return the hash.\n     * @since 1.5.1\n     */\n    public byte[] getBytes()\n    {\n        return bytes;\n    }\n\n...\n// Path: src/main/java/com/password4j/HashingFunction.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\n/**\n * This interface declares the basic methods that\n * all the Cryptographic Hashing Functions (CHF) must have.\n * <p>\n * The two possible operations are\n * <ul>\n *     <li>hashing a plaintext password</li>\n *     <li>check if an hash is originated by a plaintext password</li>\n * </ul>\n * <p>\n * The purpose of this interface is encapsulate the logic of hashing\n * and checking of a CHF inside a class.\n *\n * @author David Bertoldi\n * @since 0.1.0\n */\npublic interface HashingFunction\n{\n\n    /**\n     * Creates a {@link Hash} from a plaintext password.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * The salt is always generated randomly.\n     * <p>\n     * The generation of the salt is completely\n     * responsibility of the implementing class.\n     *\n     * @param plainTextPassword the password to be hashed\n     * @return a {@link Hash}\n     * @see HashingFunction#hash(CharSequence, String)\n     * @since 0.1.0\n     */\n    Hash hash(CharSequence plainTextPassword);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * The salt is always generated randomly.\n     * <p>\n     * The generation of the salt is completely\n     * responsibility of the implementing class.\n     *\n     * @param plainTextPassword the password to be hashed as bytes array\n     * @return a {@link Hash}\n     * @see HashingFunction#hash(CharSequence, String)\n     * @since 1.7.0\n     */\n    Hash hash(byte[] plainTextPassword);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed\n     * @param salt              the salt used in the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 0.1.0\n     */\n    Hash hash(CharSequence plainTextPassword, String salt);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed as bytes array\n     * @param salt              the salt as bytes array used in the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 1.7.0\n     */\n    Hash hash(byte[] plainTextPassword, byte[] salt);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed\n     * @param salt              the salt used in the hashing process\n     * @param pepper            the pepper used int the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 1.5.0\n     */\n    Hash hash(byte[] plainTextPassword, byte[] salt, CharSequence pepper);\n\n    /**\n     * Creates a {@link Hash} from a plaintext password and a salt.\n     * Depending on the implementation, the hash may contain\n     * the information about salt and pepper.\n     * <p>\n     * Some CHFs validate the format of the salt; if the validation\n     * is not passed a {@link BadParametersException} is thrown.\n     * The generation of the salt is completely\n     * responsibility of the caller.\n     *\n     * @param plainTextPassword the password to be hashed as bytes array\n     * @param salt              the salt as bytes array used in the hashing process\n     * @param pepper            the pepper used int the hashing process\n     * @return a {@link Hash}\n     * @throws BadParametersException if the salt does not pass the validation of the CHF\n     * @since 1.7.0\n     */\n    Hash hash(CharSequence plainTextPassword, String salt, CharSequence pepper);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password.\n     * <p>\n     * This method must be used when the salt is part\n     * of the hash.\n     * If the CHF expects a salt separated from the hash,\n     * an {@link UnsupportedOperationException} is thrown.\n     * <p>\n     * In case the CHF expects a salt to be part of the hash,\n     * if the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws UnsupportedOperationException if the CHF need a salt and it is not part of the hash\n     * @throws BadParametersException        if the hash is not well-formed\n     * @since 0.1.0\n     */\n    boolean check(CharSequence plainTextPassword, String hashed);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password.\n     * <p>\n     * This method must be used when the salt is part\n     * of the hash.\n     * If the CHF expects a salt separated from the hash,\n     * an {@link UnsupportedOperationException} is thrown.\n     * <p>\n     * In case the CHF expects a salt to be part of the hash,\n     * if the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password as bytes array\n     * @param hashed            the hash as bytes array\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws UnsupportedOperationException if the CHF need a salt and it is not part of the hash\n     * @throws BadParametersException        if the hash is not well-formed\n     * @since 1.7.0\n     */\n    boolean check(byte[] plainTextPassword, byte[] hashed);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @param salt              the salt used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 0.2.1\n     */\n    boolean check(CharSequence plainTextPassword, String hashed, String salt);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password as bytes array\n     * @param hashed            the hash as bytes array\n     * @param salt              the salt as bytes array used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 1.7.0\n     */\n    boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password\n     * @param hashed            the hash\n     * @param salt              the salt used to produce the hash\n     * @param pepper            the pepper used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 1.5.0\n     */\n    boolean check(CharSequence plainTextPassword, String hashed, String salt, CharSequence pepper);\n\n    /**\n     * Checks if the CHF generated the hash starting from\n     * the plaintext password and the salt.\n     * <p>\n     * This method must be used when the salt is not part\n     * of the hash.\n     * <p>\n     * If the format of the hash is not valid (e.g. the CHF cannot\n     * recognise a valid salt) a {@link BadParametersException}\n     * is thrown.\n     *\n     * @param plainTextPassword the plaintext password as bytes array\n     * @param hashed            the hash as bytes array\n     * @param salt              the salt as bytes array used to produce the hash\n     * @param pepper            the pepper used to produce the hash\n     * @return true if the hash is generated from the plaintext; false otherwise\n     * @throws BadParametersException if the hash is not well-formed\n     * @since 1.5.0\n     */\n    boolean check(byte[] plainTextPassword, byte[] hashed, byte[] salt, CharSequence pepper);\n}\n\n// Path: src/main/java/com/password4j/Password.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\n\n/**\n * This class provides the two main operations on password: hash and verify.\n * <p>\n * All the methods are static because there's no sense to\n * have an instance of {@link Password}.\n * It represents a facade layer that ease the usage of the\n * package itself and so this should be the only class\n * to be invoked from this package.\n *\n * @author David Bertoldi\n * @since 0.1.1\n */\npublic class Password\n{\n\n    static\n    {\n        Utils.printBanner(System.out); //NOSONAR\n    }\n\n    private Password()\n    {\n        //\n    }\n\n    /**\n     * Starts to hash the given plain text password.\n     * <p>\n     * This method is used to start the setup of a {@link HashBuilder}\n     * instance that finally should execute the {@link HashBuilder#with(HashingFunction)}\n     * method to hash the password.\n     *\n     * @param plainTextPassword the plain text password\n     * @return a builder instance of {@link HashBuilder}\n     * @throws BadParametersException if any of the arguments are null.\n     * @since 0.1.1\n     */\n    public static HashBuilder hash(CharSequence plainTextPassword)\n    {\n        if (plainTextPassword == null)\n        {\n            throw new BadParametersException(\"Password cannot be null\");\n        }\n        return new HashBuilder(plainTextPassword);\n    }\n\n    /**\n     * Starts to hash the given plain text password.\n     * <p>\n     * This method is used to start the setup of a {@link HashBuilder}\n     * instance that finally should execute the {@link HashBuilder#with(HashingFunction)}\n     * method to hash the password.\n     *\n     * @param plainTextPassword the plain text password as bytes array\n     * @return a builder instance of {@link HashBuilder}\n     * @throws BadParametersException if any of the arguments are null.\n     * @since 1.7.0\n     */\n    public static HashBuilder hash(byte[] plainTextPassword)\n    {\n        if (plainTextPassword == null || plainTextPassword.length == 0)\n        {\n            throw new BadParametersException(\"Password cannot be null\");\n        }\n        return new HashBuilder(plainTextPassword);\n    }\n\n\n    /**\n     * Starts to verify if a hash string has been generated with\n     * the given plain text password.\n     * <p>\n     * This method is used to start the setup of an {@link HashChecker}\n     * instance that finally should execute the {@link HashChecker#with(HashingFunction)}\n     * method to verify the hash.\n     *\n     * @param plainTextPassword the plain text password\n     * @param hash              a hash string\n     * @return a builder instance of {@link HashChecker}\n     * @throws BadParametersException if any of the arguments are null.\n     * @since 0.1.1\n     */\n    public static HashChecker check(CharSequence plainTextPassword, String hash)\n    {\n        if (hash == null || plainTextPassword == null)\n        {\n            throw new BadParametersException(\"Hash or plain cannot be null\");\n        }\n        return new HashChecker(plainTextPassword, hash);\n    }\n\n\n    /**\n     * Starts to verify if a hash string has been generated with\n     * the given plain text password.\n     * <p>\n     * This method is used to start the setup of an {@link HashChecker}\n     * instance that finally should execute the {@link HashChecker#with(HashingFunction)}\n     * method to verify the hash.\n     *\n     * @param plainTextPassword the plain text password as bytes  array\n     * @param hash              a hash string as bytes array\n     * @return a builder instance of {@link HashChecker}\n     * @throws BadParametersException if any of the arguments are null.\n     * @since 1.7.0\n     */\n    public static HashChecker check(byte[] plainTextPassword, byte[] hash)\n    {\n        if (hash == null || plainTextPassword == null || hash.length == 0 || plainTextPassword.length == 0)\n        {\n            throw new BadParametersException(\"Hash or plain cannot be null\");\n        }\n        return new HashChecker(plainTextPassword, hash);\n    }\n\n    /**\n     * Starts to verify if a hash object has been generated with\n     * the given plain text password.\n     * <p>\n     * This method uses the {@link HashingFunction} used to calculate the given {@link Hash}.\n     * Il the password is null, this returns false;\n     * otherwise {@link HashingFunction#check(CharSequence, String)} is invoked.\n     *\n     * @param plainTextPassword the original password.\n     * @param hashObject        an {@link Hash} object.\n     * @return true if the check passes, false otherwise.\n     * @throws BadParametersException if the Hash is null or if there's no hashing function defined in it.\n     * @since 1.0.3\n     */\n    public static boolean check(CharSequence plainTextPassword, Hash hashObject)\n    {\n        return check(Utils.fromCharSequenceToBytes(plainTextPassword), hashObject);\n    }\n\n    /**\n     * Starts to verify if a hash object has been generated with\n     * the given plain text password.\n     * <p>\n     * This method uses the {@link HashingFunction} used to calculate the given {@link Hash}.\n     * Il the password is null, this returns false;\n     * otherwise {@link HashingFunction#check(CharSequence, String)} is invoked.\n     *\n     * @param plainTextPassword the original password as bytes array.\n     * @param hashObject        an {@link Hash} object.\n     * @return true if the check passes, false otherwise.\n     * @throws BadParametersException if the Hash is null or if there's no hashing function defined in it.\n     * @since 1.7.0\n     */\n    public static boolean check(byte[] plainTextPassword, Hash hashObject)\n    {\n        if (hashObject == null || hashObject.getHashingFunction() == null)\n        {\n            throw new BadParametersException(\"Invalid Hash object. \" + (hashObject != null ? hashObject.toString() : null));\n        }\n        if (plainTextPassword == null)\n        {\n            return false;\n        }\n\n        return hashObject.getHashingFunction().check(plainTextPassword, hashObject.getResultAsBytes(), hashObject.getSaltBytes(), hashObject.getPepper());\n    }\n\n}\n\n// Path: src/main/java/com/password4j/HashUpdate.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\n/**\n * This class represents the result of the password verification process\n * along with the generated {@link Hash}.\n * <p>\n * If the verification passed a new hash is always provided.\n *\n * @author David Bertoldi\n * @since 1.3.0\n */\npublic class HashUpdate\n{\n    public static final HashUpdate UNVERIFIED = new HashUpdate();\n\n    private Hash hash;\n\n    private boolean updated;\n\n    private HashUpdate()\n    {\n        //\n    }\n\n    /**\n     * @param hash the new hash\n     * @throws BadParametersException if hash is null but verified is true\n     * @since 1.3.0\n     */\n    public HashUpdate(Hash hash)\n    {\n        this.hash = hash;\n    }\n\n    /**\n     * @param hash the new hash\n     * @param updated flag for updated hash\n     * @throws BadParametersException if hash is null but verified is true\n     * @since 1.7.0\n     */\n    public HashUpdate(Hash hash, boolean updated)\n    {\n        this(hash);\n        this.updated = updated;\n    }\n\n    /**\n     * Returns the hash generated after a verification + update\n     * process.\n     * <p>\n     * It is never null if the hash is {@link #isVerified()}\n     * return true.\n     *\n     * @return the regenerated hash\n     * @since 1.3.0\n     */\n    public Hash getHash()\n    {\n        return this.hash;\n    }\n\n    /**\n     * Returns the result of the verification process.\n     *\n     * @return true if the verification process was successful\n     * @since 1.3.0\n     */\n    public boolean isVerified()\n    {\n        return hash != null;\n    }\n\n    /**\n     * True if the update process changed the original hash due to changes on parameters.\n     * Changing the algorithms always set this flag to true.\n     *\n     * @return true if the hash was updated\n     * @since 1.7.0\n     */\n    public boolean isUpdated()\n    {\n        return updated;\n    }\n}\n\n// Path: src/main/java/com/password4j/HashUpdater.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\n/**\n * @author David Bertoldi\n * @since 1.3.0\n */\npublic class HashUpdater\n{\n\n    protected HashChecker hashChecker;\n\n    protected HashBuilder hashBuilder;\n\n    private boolean force;\n\n    HashUpdater(HashChecker hashChecker, HashBuilder hashBuilder)\n    {\n        this.hashChecker = hashChecker;\n        this.hashBuilder = hashBuilder;\n    }\n\n    /**\n     * Check if the previously given hash was produced from the given plain text password\n     * with a specific implementation of {@link HashingFunction}. If the verification\n     * passes a new hash is generated with the new implementation of {@link HashingFunction}.\n     *\n     * @param oldHashingFunction the original CHF used to produce the actual hash\n     * @param newHashingFunction the new CHF to be used the new hash\n     * @return the result of the password verification along with the new hash\n     * @see HashUpdate\n     * @since 1.3.0\n     */\n    public HashUpdate with(HashingFunction oldHashingFunction, HashingFunction newHashingFunction)\n    {\n        if (oldHashingFunction == null)\n        {\n            throw new BadParametersException(\"Starting hashing function cannot be null.\");\n        }\n        if (newHashingFunction == null)\n        {\n            throw new BadParametersException(\"New hashing function cannot be null.\");\n        }\n\n        boolean toBeUpdated = force || //\n                !oldHashingFunction.equals(newHashingFunction) //\n                || hashBuilder.salt != null //\n                || hashBuilder.pepper != null;\n\n        if (hashChecker.with(oldHashingFunction))\n        {\n            if (toBeUpdated)\n            {\n                Hash hash = this.hashBuilder.with(newHashingFunction);\n                return new HashUpdate(hash, true);\n            }\n            else\n            {\n                Hash hash = new Hash(oldHashingFunction, hashChecker.hashed, null, hashChecker.salt);\n                return new HashUpdate(hash, false);\n            }\n\n        }\n        else\n        {\n            return HashUpdate.UNVERIFIED;\n        }\n    }\n\n    /**\n     * Adds new cryptographic salt to be applied in the hash update.\n     *\n     * @param salt cryptographic salt\n     * @return this builder\n     * @since 1.3.0\n     */\n    public HashUpdater addNewSalt(String salt)\n    {\n        this.hashBuilder.addSalt(salt);\n        return this;\n    }\n\n    /**\n     * Adds new cryptographic salt to be applied in the hash update.\n     *\n     * @param saltAsBytes cryptographic salt as bytes array\n     * @return this builder\n     * @since 1.3.0\n     */\n    public HashUpdater addNewSalt(byte[] saltAsBytes)\n    {\n        this.hashBuilder.addSalt(saltAsBytes);\n        return this;\n    }\n\n    /**\n     * Add a random cryptographic salt in the hash update process.\n     * The salt is applied differently depending on the new chosen algorithm.\n     *\n     * @return this builder\n     * @see SaltGenerator#generate() for more information about the length of the product\n     * @since 1.3.0\n     */\n    public HashUpdater addNewRandomSalt()\n    {\n        this.hashBuilder.addRandomSalt();\n        return this;\n    }\n\n    /**\n     * Add a random cryptographic salt in the hash update process with a given length.\n     * The salt is applied differently depending on the new chosen algorithm.\n     *\n     * @param length the length of the salt produced\n     * @return this builder\n     * @throws BadParametersException if the length is non-positive\n     * @see SaltGenerator#generate() for more information about the length of the product\n     * @since 1.3.0\n     */\n    public HashUpdater addNewRandomSalt(int length)\n    {\n        this.hashBuilder.addRandomSalt(length);\n        return this;\n    }\n\n    /**\n     * Concatenates the pepper configured in your `psw4j.properties` file with the plain text password.\n     * The produced sequence (in the form {@code pepper+password}) is processed by the new algorithm.\n     *\n     * @return this builder\n     * @see PepperGenerator#get()\n     * @since 1.3.0\n     */\n    public HashUpdater addNewPepper()\n    {\n        this.hashBuilder.addPepper();\n        return this;\n    }\n\n    /**\n     * Concatenates the provided string with the plain text password.\n     * The produced sequence (in the form {@code pepper+password}) is processed by the new algorithm.\n     *\n     * @param pepper cryptographic pepper\n     * @return this builder\n     * @since 1.3.0\n     */\n    public HashUpdater addNewPepper(CharSequence pepper)\n    {\n        this.hashBuilder.addPepper(pepper);\n        return this;\n    }\n\n    /**\n     * Forces the update of the hash even if function, salt and pepper did not change.\n     * This can be helpful with algorithm like Bcrypt that generates their own salt.\n     *\n     * @return this builder\n     * @since 1.7.0\n     */\n    public HashUpdater forceUpdate()\n    {\n        this.force = true;\n        return this;\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link PBKDF2Function}.\n     * <p>\n     * This method read the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getPBKDF2Instance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withPBKDF2()\n    {\n        return withPBKDF2(AlgorithmFinder.getPBKDF2Instance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link PBKDF2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getPBKDF2Instance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withPBKDF2(HashingFunction newHashingFunction)\n    {\n        return with(AlgorithmFinder.getPBKDF2Instance(), newHashingFunction);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link CompressedPBKDF2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getCompressedPBKDF2Instance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withCompressedPBKDF2()\n    {\n        CompressedPBKDF2Function pbkdf2 = AlgorithmFinder.getCompressedPBKDF2Instance();\n        return withCompressedPBKDF2(pbkdf2);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link CompressedPBKDF2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getCompressedPBKDF2Instance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withCompressedPBKDF2(HashingFunction newHashingFunction)\n    {\n        CompressedPBKDF2Function pbkdf2 = CompressedPBKDF2Function.getInstanceFromHash(hashChecker.getHashed());\n        return with(pbkdf2, newHashingFunction);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link BcryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getBcryptInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withBcrypt()\n    {\n        return withBcrypt(AlgorithmFinder.getBcryptInstance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link BcryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getBcryptInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withBcrypt(HashingFunction newHashingFunction)\n    {\n        return with(BcryptFunction.getInstanceFromHash(hashChecker.getHashed()), newHashingFunction);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link ScryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getScryptInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withScrypt()\n    {\n        return withScrypt(AlgorithmFinder.getScryptInstance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link ScryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getScryptInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.3.0\n     */\n    public HashUpdate withScrypt(HashingFunction newHashingFunction)\n    {\n        return with(ScryptFunction.getInstanceFromHash(hashChecker.getHashed()), newHashingFunction);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link MessageDigestFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getMessageDigestInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.4.0\n     */\n    public HashUpdate withMessageDigest()\n    {\n        return withMessageDigest(AlgorithmFinder.getMessageDigestInstance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link MessageDigestFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getMessageDigestInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.4.0\n     */\n    public HashUpdate withMessageDigest(HashingFunction newHashingFunction)\n    {\n        return with(AlgorithmFinder.getMessageDigestInstance(), newHashingFunction);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link Argon2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getArgon2Instance() ()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.5.0\n     */\n    public HashUpdate withArgon2()\n    {\n        return withArgon2(AlgorithmFinder.getArgon2Instance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link Argon2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getArgon2Instance() ()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.5.0\n     */\n    public HashUpdate withArgon2(HashingFunction newHashingFunction)\n    {\n        return with(Argon2Function.getInstanceFromHash(hashChecker.getHashed()), newHashingFunction);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link BalloonHashingFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getBalloonHashingInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.5.0\n     */\n    public HashUpdate withBalloonHashing()\n    {\n        return withBalloonHashing(AlgorithmFinder.getBalloonHashingInstance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link BalloonHashingFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction, HashingFunction)}\n     *\n     * @return the result of the verification with the new hash\n     * @see AlgorithmFinder#getBalloonHashingInstance()\n     * @see #with(HashingFunction, HashingFunction)\n     * @since 1.5.0\n     */\n    public HashUpdate withBalloonHashing(HashingFunction newHashingFunction)\n    {\n        return with(AlgorithmFinder.getBalloonHashingInstance(), newHashingFunction);\n    }\n}\n\n// Path: src/main/java/com/password4j/HashBuilder.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\npackage com.password4j;\n\n/**\n * Builder class that helps to create a chain of parameters to be used\n * in the hashing process.\n *\n * @author David Bertoldi\n * @since 1.0.0\n */\npublic class HashBuilder\n{\n    private byte[] plainTextPassword;\n\n    protected byte[] salt;\n\n    protected CharSequence pepper;\n\n    @SuppressWarnings(\"unused\")\n    private HashBuilder()\n    {\n        //\n    }\n\n    /**\n     * @param plainTextPassword the plain text password\n     * @since 1.0.0\n     */\n    protected HashBuilder(CharSequence plainTextPassword)\n    {\n        this.plainTextPassword = Utils.fromCharSequenceToBytes(plainTextPassword);\n    }\n\n    /**\n     * @param plainTextPasswordAsBytes the plain text password as bytes array\n     * @since 1.7.0\n     */\n    protected HashBuilder(byte[] plainTextPasswordAsBytes)\n    {\n        this.plainTextPassword = plainTextPasswordAsBytes;\n    }\n\n    /**\n     * Add a cryptographic salt in the hashing process.\n     * The salt is applied differently depending on the chosen algorithm.\n     *\n     * @param salt cryptographic salt\n     * @return this builder\n     * @since 1.0.0\n     */\n    public HashBuilder addSalt(String salt)\n    {\n        this.salt = Utils.fromCharSequenceToBytes(salt);\n        return this;\n    }\n\n    /**\n     * Add a cryptographic salt in the hashing process.\n     * The salt is applied differently depending on the chosen algorithm.\n     *\n     * @param saltAsBytes cryptographic salt as bytes array\n     * @return this builder\n     * @since 1.7.0\n     */\n    public HashBuilder addSalt(byte[] saltAsBytes)\n    {\n        this.salt = saltAsBytes;\n        return this;\n    }\n\n    /**\n     * Add a random cryptographic salt in the hashing process.\n     * The salt is applied differently depending on the chosen algorithm.\n     * <p>\n     * Calling this method can be omitted for all the CHFs that require a salt.\n     *\n     * @return this builder\n     * @see SaltGenerator#generate() for more information about the length of the product\n     * @since 1.0.0\n     */\n    public HashBuilder addRandomSalt()\n    {\n        this.salt = SaltGenerator.generate();\n        return this;\n    }\n\n    /**\n     * Add a random cryptographic salt in the hashing process with a given length.\n     * The salt is applied differently depending on the chosen algorithm.\n     *\n     * @param length the length of the salt produced\n     * @return this builder\n     * @throws BadParametersException if the length is non-positive\n     * @see SaltGenerator#generate() for more information about the length of the product\n     * @since 1.0.0\n     */\n    public HashBuilder addRandomSalt(int length)\n    {\n        if (length <= 0)\n        {\n            throw new BadParametersException(\"Salt cannot have a non-positive length\");\n        }\n        else\n        {\n            this.salt = SaltGenerator.generate(length);\n        }\n        return this;\n    }\n\n    /**\n     * Concatenates the pepper configured in your `psw4j.properties` file with the plain text password.\n     * The produced sequence (in the form {@code pepper+password}) is processed by the algorithm.\n     *\n     * @return this builder\n     * @see PepperGenerator#get()\n     */\n    public HashBuilder addPepper()\n    {\n        this.pepper = PepperGenerator.get();\n        return this;\n    }\n\n    /**\n     * Concatenates the provided string with the plain text password.\n     * The produced sequence (in the form {@code pepper+password}) is processed by the algorithm.\n     *\n     * @param pepper cryptographic pepper\n     * @return this builder\n     * @since 1.0.0\n     */\n    public HashBuilder addPepper(CharSequence pepper)\n    {\n        this.pepper = pepper;\n        return this;\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with a specific implementation of {@link HashingFunction}.\n     * <p>\n     * This method does not read the configurations in the `psw4j.properties` file.\n     *\n     * @param hashingFunction a CHF\n     * @return a {@link Hash} object\n     * @since 1.0.0\n     */\n    public Hash with(HashingFunction hashingFunction)\n    {\n        return hashingFunction.hash(plainTextPassword, salt, pepper);\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link PBKDF2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction)}\n     *\n     * @return a {@link Hash} object\n     * @see AlgorithmFinder#getPBKDF2Instance()\n     * @see #with(HashingFunction)\n     * @since 1.0.0\n     */\n    public Hash withPBKDF2()\n    {\n        return with(AlgorithmFinder.getPBKDF2Instance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link CompressedPBKDF2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction)}\n     *\n     * @return an {@link Hash} object\n     * @see AlgorithmFinder#getCompressedPBKDF2Instance()\n     * @see #with(HashingFunction)\n     * @since 1.0.0\n     */\n    public Hash withCompressedPBKDF2()\n    {\n        return with(AlgorithmFinder.getCompressedPBKDF2Instance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link BcryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction)}\n     *\n     * @return an {@link Hash} object\n     * @see AlgorithmFinder#getBcryptInstance()\n     * @see #with(HashingFunction)\n     * @since 1.0.0\n     */\n    public Hash withBcrypt()\n    {\n        return with(AlgorithmFinder.getBcryptInstance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link ScryptFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction)}\n     *\n     * @return an {@link Hash} object\n     * @see AlgorithmFinder#getScryptInstance()\n     * @see #with(HashingFunction)\n     * @since 1.0.0\n     */\n    public Hash withScrypt()\n    {\n        return with(AlgorithmFinder.getScryptInstance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link MessageDigestFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction)}\n     *\n     * @return a {@link Hash} object\n     * @see AlgorithmFinder#getPBKDF2Instance()\n     * @see #with(HashingFunction)\n     * @since 1.4.0\n     */\n    public Hash withMessageDigest()\n    {\n        return with(AlgorithmFinder.getMessageDigestInstance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link Argon2Function}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction)}\n     *\n     * @return a {@link Hash} object\n     * @see AlgorithmFinder#getArgon2Instance()\n     * @see #with(HashingFunction)\n     * @since 1.5.0\n     */\n    public Hash withArgon2()\n    {\n        return with(AlgorithmFinder.getArgon2Instance());\n    }\n\n    /**\n     * Hashes the previously given plain text password\n     * with {@link BalloonHashingFunction}.\n     * <p>\n     * This method reads the configurations in the `psw4j.properties` file. If no configuration is found,\n     * then the default parameters are used.\n     * <p>\n     * Finally calls {@link #with(HashingFunction)}\n     *\n     * @return a {@link Hash} object\n     * @see AlgorithmFinder#getArgon2Instance()\n     * @see #with(HashingFunction)\n     * @since 1.8.0\n     */\n    public Hash withBalloonHashing()\n    {\n        return with(AlgorithmFinder.getBalloonHashingInstance());\n    }\n\n}\n\n// Path: src/main/java/com/password4j/SystemChecker.java\n/*\n *  (C) Copyright 2020 Password4j (http://password4j.com/).\n *\n *  Licensed under the Apache License, Version 2.0 (the \"License\");\n *  you may not use this file except in compliance with the License.\n *  You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n *\n */\n\npackage com.password4j;\n\nimport com.password4j.types.Argon2;\nimport com.password4j.types.Bcrypt;\nimport com.password4j.types.Hmac;\n\nimport java.util.List;\nimport java.util.Set;\n\n\n/**\n * This class benchmarks the target environment.\n *\n * @author David Bertoldi\n * @since 1.0.0\n */\npublic class SystemChecker\n{\n    private static final String TO_BE_HASHED = \"abcDEF123@~# xyz+-*/=456spqr\";\n\n    private static final String SALT = Utils.fromBytesToString(SaltGenerator.generate());\n\n    private static final int WARMUP_ROUNDS = 20;\n\n    private SystemChecker()\n    {\n        //\n    }\n\n    /**\n     * Verifies if the algorithm is supported by the current environment.\n     * PBKDF2 variants are available if the JVM has a corresponding\n     * {@link java.security.Provider.Service}\n     *\n     * @param algorithm the algorithm to check\n     * @return true if the algorithm is supported; false otherwise\n     */\n    \npublic static boolean isPBKDF2Supported(String algorithm)\n    {\n        if (algorithm == null)\n        {\n            throw new BadParametersException(\"Algorithm cannot be null.\");\n        }\n        List<String> variants = AlgorithmFinder.getAllPBKDF2Variants();\n        for(String variant : variants)\n        {\n            if(algorithm.equals(variant))\n            {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    /**\n     * Verifies if the algorithm is supported by the current environment.\n     * Message digest algorithms are available if the JVM has a corresponding\n     * {@link java.security.Provider.Service}\n     *\n     * @param algorithm the algorithm to check\n     * @return true if the algorithm is supported; false otherwise\n     */\n    public static boolean isMessageDigestSupported(String algorithm)\n    {\n        if (algorithm == null)\n        {\n            throw new BadParametersException(\"Algorithm cannot be null.\");\n        }\n        Set<String> mds = AlgorithmFinder.getAllMessageDigests();\n        for(String md : mds)\n        {\n            if(algorithm.equals(md))\n            {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    /**\n     * Finds the optimal configuration for bcrypt.\n     * <p>\n     * To prevent timing attacks, a maximum interval of time (in milliseconds)\n     * is required to perform a single hash.\n     * <p>\n     * This function returns a prototype {@link BcryptFunction} and the real\n     * elapsed time.\n     * If the hash cannot be performed under the specified time, the elapsed time is set to -1\n     * and prototype null.\n     *\n     * @param maxMilliseconds max time to perform the hashing\n     * @return a benchmark result for bcrypt\n     * @see BcryptFunction\n     * @since 1.0.0\n     */\n    @SuppressWarnings(\"Duplicates\")\n    public static BenchmarkResult<BcryptFunction> benchmarkBcrypt(long maxMilliseconds)\n    {\n        warmUpBcrypt();\n\n        long finalElapsed = -1;\n        BcryptFunction prototype = null;\n        int rounds = 4;\n\n        while (true)\n        {\n            BcryptFunction tmp = new BcryptFunction(Bcrypt.B, rounds);\n            long start = System.currentTimeMillis();\n\n            tmp.hash(TO_BE_HASHED);\n\n            long end = System.currentTimeMillis();\n            long elapsed = end - start;\n\n            if (elapsed > maxMilliseconds)\n            {\n                break;\n            }\n            else\n            {\n                finalElapsed = elapsed;\n                prototype = tmp;\n                rounds++;\n            }\n        }\n\n        return new BenchmarkResult<>(prototype, finalElapsed);\n    }\n\n    /**\n     * Finds the optimal configuration for Argon2.\n     * <p>\n     * To prevent timing attacks, a maximum interval of time (in milliseconds)\n     * is required to perform a single hash.\n     * <p>\n     * This function returns a prototype {@link Argon2Function} and the real\n     * elapsed time.\n     * If the hash cannot be performed under the specified time, the elapsed time is set to -1\n     * and prototype null.\n     *\n     * @param maxMilliseconds max time to perform the hashing\n     * @param memory          logarithmic memory\n     * @param parallelism     level of parallelism\n     * @param outputLength    length of the final hash\n...\n", "needle_token_start": 15199, "needle_token_end": 15311, "code_context_ntokens": 16068, "output": ["```java\npublic static boolean isPBKDF2Supported(String algorithm)\n{\n    if (algorithm == null)\n    {\n        throw new BadParametersException(\"Algorithm cannot be null.\");\n    }\n    List<String> variants = AlgorithmFinder.getAllPBKDF2Variants();\n    for(String variant : variants)\n    {\n        if(algorithm.equals(variant))\n        {\n            return true;\n        }\n    }\n    return false;\n}\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "known", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/frontend/CredentialRetrieverFactory.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To create a retriever that consistently provides a specific set of credentials, which are already known, for authentication purposes.\n2. **Input**: A set of credentials and a description of the source of these credentials.\n3. **Output**: A retriever function that, when called, logs the source of the credentials and returns these credentials wrapped in an `Optional`.\n4. **Procedure**: The method defines a retriever function that logs where the credentials were obtained from and then returns the credentials encapsulated in an `Optional` object to indicate their presence.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "  // this scope for \"OAuth2 credentials\" instantiated from a service account, which are not scoped\n  // (i.e., createScopedRequired() returns true). Note that for a service account, the IAM roles of\n  // the service account determine the IAM permissions.\n  private static final String OAUTH_SCOPE_STORAGE_READ_WRITE =\n      \"https://www.googleapis.com/auth/devstorage.read_write\";\n\n  /** Mapping between well-known credential helpers and registries (suffixes). */\n  private static final ImmutableMap<String, String> WELL_KNOWN_CREDENTIAL_HELPERS =\n      ImmutableMap.of(\n          \"gcr.io\", \"docker-credential-gcr\", \"amazonaws.com\", \"docker-credential-ecr-login\");\n\n  /**\n   * Creates a new {@link CredentialRetrieverFactory} for an image.\n   *\n   * @param imageReference the image the credential are for\n   * @param logger a consumer for handling log events\n   * @return a new {@link CredentialRetrieverFactory}\n   */\n  public static CredentialRetrieverFactory forImage(\n      ImageReference imageReference, Consumer<LogEvent> logger) {\n    return new CredentialRetrieverFactory(\n        imageReference,\n        logger,\n        DockerCredentialHelper::new,\n        GoogleCredentials::getApplicationDefault,\n        Collections.emptyMap());\n  }\n\n  /**\n   * Creates a new {@link CredentialRetrieverFactory} for an image.\n   *\n   * @param imageReference the image the credential are for\n   * @param logger a consumer for handling log events\n   * @param environment environment variables for credential helper\n   * @return a new {@link CredentialRetrieverFactory}\n   */\n  public static CredentialRetrieverFactory forImage(\n      ImageReference imageReference, Consumer<LogEvent> logger, Map<String, String> environment) {\n    return new CredentialRetrieverFactory(\n        imageReference,\n        logger,\n        DockerCredentialHelper::new,\n        GoogleCredentials::getApplicationDefault,\n        environment);\n  }\n\n  private final ImageReference imageReference;\n  private final Consumer<LogEvent> logger;\n  private final DockerCredentialHelperFactory dockerCredentialHelperFactory;\n  private final GoogleCredentialsProvider googleCredentialsProvider;\n  private final Map<String, String> environment;\n\n  @VisibleForTesting\n  CredentialRetrieverFactory(\n      ImageReference imageReference,\n      Consumer<LogEvent> logger,\n      DockerCredentialHelperFactory dockerCredentialHelperFactory,\n      GoogleCredentialsProvider googleCredentialsProvider,\n      Map<String, String> environment) {\n    this.imageReference = imageReference;\n    this.logger = logger;\n    this.dockerCredentialHelperFactory = dockerCredentialHelperFactory;\n    this.googleCredentialsProvider = googleCredentialsProvider;\n    this.environment = environment;\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} that returns a known {@link Credential}.\n   *\n   * @param credential the known credential\n   * @param credentialSource the source of the credentials (for logging)\n   * @return a new {@link CredentialRetriever}\n   */\n  \npublic CredentialRetriever known(Credential credential, String credentialSource) {\n    return () -> {\n      logGotCredentialsFrom(\"credentials from \" + credentialSource);\n      return Optional.of(credential);\n    };\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} for retrieving credentials via a Docker credential\n   * helper, such as {@code docker-credential-gcr}.\n   *\n   * @param credentialHelper the credential helper executable\n   * @return a new {@link CredentialRetriever}\n   */\n  public CredentialRetriever dockerCredentialHelper(String credentialHelper) {\n    return dockerCredentialHelper(Paths.get(credentialHelper));\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} for retrieving credentials via a Docker credential\n   * helper, such as {@code docker-credential-gcr}.\n   *\n   * @param credentialHelper the credential helper executable\n   * @return a new {@link CredentialRetriever}\n   * @see <a\n   *     href=\"https://github.com/docker/docker-credential-helpers#development\">https://github.com/docker/docker-credential-helpers#development</a>\n   */\n  public CredentialRetriever dockerCredentialHelper(Path credentialHelper) {\n    return () -> {\n      try {\n        return Optional.of(retrieveFromDockerCredentialHelper(credentialHelper));\n\n      } catch (CredentialHelperUnhandledServerUrlException ex) {\n        logger.accept(\n            LogEvent.info(\n                \"No credentials for \" + imageReference.getRegistry() + \" in \" + credentialHelper));\n        return Optional.empty();\n\n      } catch (IOException ex) {\n        throw new CredentialRetrievalException(ex);\n      }\n    };\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} that tries well-known Docker credential helpers to\n   * retrieve credentials based on the registry of the image, such as {@code docker-credential-gcr}\n   * for images with the registry ending with {@code gcr.io}.\n   *\n   * @return a new {@link CredentialRetriever}\n   */\n  public CredentialRetriever wellKnownCredentialHelpers() {\n    return () -> {\n      for (Map.Entry<String, String> entry : WELL_KNOWN_CREDENTIAL_HELPERS.entrySet()) {\n        try {\n          String registrySuffix = entry.getKey();\n          if (imageReference.getRegistry().endsWith(registrySuffix)) {\n            String credentialHelper = entry.getValue();\n            return Optional.of(retrieveFromDockerCredentialHelper(Paths.get(credentialHelper)));\n          }\n\n        } catch (CredentialHelperNotFoundException\n            | CredentialHelperUnhandledServerUrlException ex) {\n          if (ex.getMessage() != null) {\n            // Warns the user that the specified (or inferred) credential helper cannot be used.\n            logger.accept(LogEvent.info(ex.getMessage()));\n            if (ex.getCause() != null && ex.getCause().getMessage() != null) {\n              logger.accept(LogEvent.info(\"  Caused by: \" + ex.getCause().getMessage()));\n            }\n          }\n\n        } catch (IOException ex) {\n          throw new CredentialRetrievalException(ex);\n        }\n      }\n      return Optional.empty();\n    };\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} that tries to retrieve credentials from Docker config\n   * (located at {@code System.getProperty(\"user.home\")/.docker/config.json}).\n   *\n   * @return a new {@link CredentialRetriever}\n   * @see DockerConfigCredentialRetriever\n   */\n  public CredentialRetriever dockerConfig() {\n    return dockerConfig(\n        DockerConfigCredentialRetriever.create(\n            imageReference.getRegistry(),\n            Paths.get(System.getProperty(\"user.home\"), \".docker\", \"config.json\")));\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} that tries to retrieve credentials from a custom path\n   * to a Docker config.\n   *\n   * @param dockerConfigFile the path to the Docker config file\n   * @return a new {@link CredentialRetriever}\n   * @see DockerConfigCredentialRetriever\n   */\n  public CredentialRetriever dockerConfig(Path dockerConfigFile) {\n    return dockerConfig(\n        DockerConfigCredentialRetriever.create(imageReference.getRegistry(), dockerConfigFile));\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} that tries to retrieve credentials from a legacy\n   * Docker config file.\n   *\n   * @param dockerConfigFile the path to a legacy docker configuration file\n   * @return a new {@link CredentialRetriever}\n   * @see DockerConfigCredentialRetriever\n   */\n  public CredentialRetriever legacyDockerConfig(Path dockerConfigFile) {\n    return dockerConfig(\n        DockerConfigCredentialRetriever.createForLegacyFormat(\n            imageReference.getRegistry(), dockerConfigFile));\n  }\n\n  /**\n   * Creates a new {@link CredentialRetriever} that tries to retrieve credentials from <a\n   * href=\"https://cloud.google.com/docs/authentication/production\">Google Application Default\n   * Credentials</a>.\n   *\n   * @return a new {@link CredentialRetriever}\n   * @see <a\n   *     href=\"https://cloud.google.com/docs/authentication/production\">https://cloud.google.com/docs/authentication/production</a>\n   */\n  public CredentialRetriever googleApplicationDefaultCredentials() {\n    return () -> {\n      try {\n        if (imageReference.getRegistry().endsWith(\"gcr.io\")\n            || imageReference.getRegistry().endsWith(\"docker.pkg.dev\")) {\n          GoogleCredentials googleCredentials = googleCredentialsProvider.get();\n          logger.accept(LogEvent.info(\"Google ADC found\"));\n          if (googleCredentials.createScopedRequired()) { // not scoped if service account\n            // The short-lived OAuth2 access token to be generated from the service account with\n            // refreshIfExpired() below will have one-hour expiry (as of Aug 2019). Instead of using\n            // an access token, it is technically possible to use the service account private key to\n            // auth with GCR, but it does not worth writing complex code to achieve that.\n            logger.accept(LogEvent.info(\"ADC is a service account. Setting GCS read-write scope\"));\n            List<String> scope = Collections.singletonList(OAUTH_SCOPE_STORAGE_READ_WRITE);\n            googleCredentials = googleCredentials.createScoped(scope);\n          }\n          googleCredentials.refreshIfExpired();\n\n          logGotCredentialsFrom(\"Google Application Default Credentials\");\n          AccessToken accessToken = googleCredentials.getAccessToken();\n          // https://cloud.google.com/container-registry/docs/advanced-authentication#access_token\n          return Optional.of(Credential.from(\"oauth2accesstoken\", accessToken.getTokenValue()));\n        }\n\n      } catch (IOException ex) { // Includes the case where ADC is simply not available.\n        logger.accept(\n            LogEvent.info(\"ADC not present or error fetching access token: \" + ex.getMessage()));\n      }\n      return Optional.empty();\n    };\n  }\n\n  @VisibleForTesting\n  CredentialRetriever dockerConfig(\n      DockerConfigCredentialRetriever dockerConfigCredentialRetriever) {\n    return () -> {\n      Path configFile = dockerConfigCredentialRetriever.getDockerConfigFile();\n      try {\n        Optional<Credential> credentials = dockerConfigCredentialRetriever.retrieve(logger);\n        if (credentials.isPresent()) {\n          logGotCredentialsFrom(\"credentials from Docker config (\" + configFile + \")\");\n          return credentials;\n        }\n\n      } catch (IOException ex) {\n        logger.accept(LogEvent.info(\"Unable to parse Docker config file: \" + configFile));\n      }\n      return Optional.empty();\n    };\n  }\n\n  private Credential retrieveFromDockerCredentialHelper(Path credentialHelper)\n      throws CredentialHelperUnhandledServerUrlException, CredentialHelperNotFoundException,\n          IOException {\n    Credential credentials =\n        dockerCredentialHelperFactory\n            .create(imageReference.getRegistry(), credentialHelper, environment)\n            .retrieve();\n    logGotCredentialsFrom(\"credential helper \" + credentialHelper.getFileName().toString());\n    return credentials;\n  }\n\n  private void logGotCredentialsFrom(String credentialSource) {\n    logger.accept(LogEvent.lifecycle(\"Using \" + credentialSource + \" for \" + imageReference));\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/DockerDaemonImage.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.docker.CliDockerClient;\nimport java.nio.file.Path;\nimport java.util.Collections;\nimport java.util.Map;\n\n/** Builds to the Docker daemon. */\npublic class DockerDaemonImage {\n\n  /**\n   * Instantiate with the image reference to tag the built image with. This is the name that shows\n   * up on the Docker daemon.\n   *\n   * @param imageReference the image reference\n   * @return a new {@link DockerDaemonImage}\n   */\n  public static DockerDaemonImage named(ImageReference imageReference) {\n    return new DockerDaemonImage(imageReference);\n  }\n\n  /**\n   * Instantiate with the image reference to tag the built image with. This is the name that shows\n   * up on the Docker daemon.\n   *\n   * @param imageReference the image reference\n   * @return a new {@link DockerDaemonImage}\n   * @throws InvalidImageReferenceException if {@code imageReference} is not a valid image reference\n   */\n  public static DockerDaemonImage named(String imageReference)\n      throws InvalidImageReferenceException {\n    return named(ImageReference.parse(imageReference));\n  }\n\n  private final ImageReference imageReference;\n  private Path dockerExecutable = CliDockerClient.DEFAULT_DOCKER_CLIENT;\n  private Map<String, String> dockerEnvironment = Collections.emptyMap();\n\n  /** Instantiate with {@link #named}. */\n  private DockerDaemonImage(ImageReference imageReference) {\n    this.imageReference = imageReference;\n  }\n\n  /**\n   * Sets the path to the {@code docker} CLI. This is {@code docker} by default.\n   *\n   * @param dockerExecutable the path to the {@code docker} CLI\n   * @return this\n   */\n  public DockerDaemonImage setDockerExecutable(Path dockerExecutable) {\n    this.dockerExecutable = dockerExecutable;\n    return this;\n  }\n\n  /**\n   * Sets the additional environment variables to use when running {@link #dockerExecutable docker}.\n   *\n   * @param dockerEnvironment additional environment variables\n   * @return this\n   */\n  public DockerDaemonImage setDockerEnvironment(Map<String, String> dockerEnvironment) {\n    this.dockerEnvironment = dockerEnvironment;\n    return this;\n  }\n\n  ImageReference getImageReference() {\n    return imageReference;\n  }\n\n  Path getDockerExecutable() {\n    return dockerExecutable;\n  }\n\n  Map<String, String> getDockerEnvironment() {\n    return dockerEnvironment;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainer.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.builder.steps.BuildResult;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.util.Objects;\nimport java.util.Set;\n\n/** The container built by Jib. */\npublic class JibContainer {\n\n  private final ImageReference targetImage;\n  private final DescriptorDigest imageDigest;\n  private final DescriptorDigest imageId;\n  private final Set<String> tags;\n  private final boolean imagePushed;\n\n  @VisibleForTesting\n  JibContainer(\n      ImageReference targetImage,\n      DescriptorDigest imageDigest,\n      DescriptorDigest imageId,\n      Set<String> tags,\n      boolean imagePushed) {\n    this.targetImage = targetImage;\n    this.imageDigest = imageDigest;\n    this.imageId = imageId;\n    this.tags = tags;\n    this.imagePushed = imagePushed;\n  }\n\n  static JibContainer from(BuildContext buildContext, BuildResult buildResult) {\n    ImageReference targetImage = buildContext.getTargetImageConfiguration().getImage();\n    DescriptorDigest imageDigest = buildResult.getImageDigest();\n    DescriptorDigest imageId = buildResult.getImageId();\n    Set<String> tags = buildContext.getAllTargetImageTags();\n    return new JibContainer(targetImage, imageDigest, imageId, tags, buildResult.isImagePushed());\n  }\n\n  /**\n   * Get the target image that was built.\n   *\n   * @return the target image reference.\n   */\n  public ImageReference getTargetImage() {\n    return targetImage;\n  }\n\n  /**\n   * Returns true if we pushed this image all the way to a registry.\n   *\n   * @return true if pushed.\n   */\n  public boolean isImagePushed() {\n    return imagePushed;\n  }\n\n  /**\n   * Gets the digest of the registry image manifest built by Jib. This digest can be used to fetch a\n   * specific image from the registry in the form {@code myregistry/myimage@digest}.\n   *\n   * @return the image digest\n   */\n  public DescriptorDigest getDigest() {\n    return imageDigest;\n  }\n\n  /**\n   * Gets the digest of the container configuration built by Jib.\n   *\n   * @return the image ID\n   */\n  public DescriptorDigest getImageId() {\n    return imageId;\n  }\n\n  /**\n   * Get the tags applied to the container.\n   *\n   * @return the set of all tags\n   */\n  public Set<String> getTags() {\n    return tags;\n  }\n\n  @Override\n  public int hashCode() {\n    return Objects.hash(targetImage, imageDigest, imageId, tags, imagePushed);\n  }\n\n  @Override\n  public boolean equals(Object other) {\n    if (this == other) {\n      return true;\n    }\n    if (!(other instanceof JibContainer)) {\n      return false;\n    }\n    JibContainer otherContainer = (JibContainer) other;\n    return targetImage.equals(otherContainer.targetImage)\n        && imageDigest.equals(otherContainer.imageDigest)\n        && imageId.equals(otherContainer.imageId)\n        && tags.equals(otherContainer.tags)\n        && imagePushed == otherContainer.imagePushed;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/RegistryImage.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Optional;\n\n/**\n * Defines an image on a container registry that can be used as either a source or target image.\n *\n * <p>The registry portion of the image reference determines which registry to the image lives (or\n * should live) on. The repository portion is the namespace within the registry. The tag is a label\n * to easily identify an image among all the images in the repository. See {@link ImageReference}\n * for more details.\n *\n * <p>When configuring credentials (via {@link #addCredential} for example), make sure the\n * credentials are valid push (for using this as a target image) or pull (for using this as a source\n * image) credentials for the repository specified via the image reference.\n */\npublic class RegistryImage {\n\n  /**\n   * Instantiate with the image reference to use.\n   *\n   * @param imageReference the image reference\n   * @return a new {@link RegistryImage}\n   */\n  public static RegistryImage named(ImageReference imageReference) {\n    return new RegistryImage(imageReference);\n  }\n\n  /**\n   * Instantiate with the image reference to use.\n   *\n   * @param imageReference the image reference\n   * @return a new {@link RegistryImage}\n   * @throws InvalidImageReferenceException if {@code imageReference} is not a valid image reference\n   */\n  public static RegistryImage named(String imageReference) throws InvalidImageReferenceException {\n    return named(ImageReference.parse(imageReference));\n  }\n\n  private final ImageReference imageReference;\n  private final List<CredentialRetriever> credentialRetrievers = new ArrayList<>();\n\n  /** Instantiate with {@link #named}. */\n  private RegistryImage(ImageReference imageReference) {\n    this.imageReference = imageReference;\n  }\n\n  /**\n   * Adds a username-password credential to use to push/pull the image. This is a shorthand for\n   * {@code addCredentialRetriever(() -> Optional.of(Credential.basic(username, password)))}.\n   *\n   * @param username the username\n   * @param password the password\n   * @return this\n   */\n  public RegistryImage addCredential(String username, String password) {\n    addCredentialRetriever(() -> Optional.of(Credential.from(username, password)));\n    return this;\n  }\n\n  /**\n   * Adds {@link CredentialRetriever} to fetch push/pull credentials for the image. Credential\n   * retrievers are attempted in the order in which they are specified until credentials are\n   * successfully retrieved.\n   *\n   * <p>Example usage:\n   *\n   * <pre>{@code\n   * .addCredentialRetriever(() -> {\n   *   if (!Files.exists(\"secret.txt\") {\n   *     return Optional.empty();\n   *   }\n   *   try {\n   *     String password = fetchPasswordFromFile(\"secret.txt\");\n   *     return Credential.basic(\"myaccount\", password);\n   *\n   *   } catch (IOException ex) {\n   *     throw new CredentialRetrievalException(\"Failed to load password\", ex);\n   *   }\n   * })\n   * }</pre>\n   *\n   * @param credentialRetriever the {@link CredentialRetriever} to add\n   * @return this\n   */\n  public RegistryImage addCredentialRetriever(CredentialRetriever credentialRetriever) {\n    credentialRetrievers.add(credentialRetriever);\n    return this;\n  }\n\n  ImageReference getImageReference() {\n    return imageReference;\n  }\n\n  List<CredentialRetriever> getCredentialRetrievers() {\n    return credentialRetrievers;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/LayerConfiguration.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath;\nimport com.google.cloud.tools.jib.api.buildplan.FileEntriesLayer;\nimport com.google.cloud.tools.jib.api.buildplan.FileEntry;\nimport com.google.cloud.tools.jib.api.buildplan.FilePermissions;\nimport com.google.cloud.tools.jib.api.buildplan.FilePermissionsProvider;\nimport com.google.cloud.tools.jib.api.buildplan.ModificationTimeProvider;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.nio.file.Path;\nimport java.time.Instant;\nimport java.util.List;\nimport java.util.stream.Collectors;\nimport javax.annotation.concurrent.Immutable;\n\n/**\n * Configures how to build a layer in the container image. Instantiate with {@link #builder}.\n *\n * @deprecated Use {@link FileEntriesLayer}.\n */\n@Deprecated\n@Immutable\npublic class LayerConfiguration {\n\n  /** Builds a {@link LayerConfiguration}. */\n  public static class Builder {\n\n    private FileEntriesLayer.Builder layerBuilder = FileEntriesLayer.builder();\n\n    private Builder() {}\n\n    /**\n     * Sets a name for this layer. This name does not affect the contents of the layer.\n     *\n     * @param name the name\n     * @return this\n     */\n    public Builder setName(String name) {\n      layerBuilder.setName(name);\n      return this;\n    }\n\n    /**\n     * Sets entries for the layer.\n     *\n     * @param entries file entries in the layer\n     * @return this\n     */\n    public Builder setEntries(List<LayerEntry> entries) {\n      layerBuilder.setEntries(\n          entries.stream().map(LayerEntry::toFileEntry).collect(Collectors.toList()));\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer.\n     *\n     * @param entry the layer entry to add\n     * @return this\n     */\n    public Builder addEntry(LayerEntry entry) {\n      layerBuilder.addEntry(entry.toFileEntry());\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer. Only adds the single source file to the exact path in the\n     * container file system.\n     *\n     * <p>For example, {@code addEntry(Paths.get(\"myfile\"),\n     * AbsoluteUnixPath.get(\"/path/in/container\"))} adds a file {@code myfile} to the container file\n     * system at {@code /path/in/container}.\n     *\n     * <p>For example, {@code addEntry(Paths.get(\"mydirectory\"),\n     * AbsoluteUnixPath.get(\"/path/in/container\"))} adds a directory {@code mydirectory/} to the\n     * container file system at {@code /path/in/container/}. This does <b>not</b> add the contents\n     * of {@code mydirectory}.\n     *\n     * @param sourceFile the source file to add to the layer\n     * @param pathInContainer the path in the container file system corresponding to the {@code\n     *     sourceFile}\n     * @return this\n     */\n    public Builder addEntry(Path sourceFile, AbsoluteUnixPath pathInContainer) {\n      layerBuilder.addEntry(sourceFile, pathInContainer);\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer with the given permissions. Only adds the single source file to\n     * the exact path in the container file system. See {@link Builder#addEntry(Path,\n     * AbsoluteUnixPath)} for more information.\n     *\n     * @param sourceFile the source file to add to the layer\n     * @param pathInContainer the path in the container file system corresponding to the {@code\n     *     sourceFile}\n     * @param permissions the file permissions on the container\n     * @return this\n     * @see Builder#addEntry(Path, AbsoluteUnixPath)\n     * @see FilePermissions#DEFAULT_FILE_PERMISSIONS\n     * @see FilePermissions#DEFAULT_FOLDER_PERMISSIONS\n     */\n    public Builder addEntry(\n        Path sourceFile, AbsoluteUnixPath pathInContainer, FilePermissions permissions) {\n      layerBuilder.addEntry(sourceFile, pathInContainer, permissions);\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer with the given file modification time. Only adds the single source\n     * file to the exact path in the container file system. See {@link Builder#addEntry(Path,\n     * AbsoluteUnixPath)} for more information.\n     *\n     * @param sourceFile the source file to add to the layer\n     * @param pathInContainer the path in the container file system corresponding to the {@code\n     *     sourceFile}\n     * @param modificationTime the file modification time\n     * @return this\n     * @see Builder#addEntry(Path, AbsoluteUnixPath)\n     */\n    public Builder addEntry(\n        Path sourceFile, AbsoluteUnixPath pathInContainer, Instant modificationTime) {\n      layerBuilder.addEntry(sourceFile, pathInContainer, modificationTime);\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer with the given permissions and file modification time. Only adds\n     * the single source file to the exact path in the container file system. See {@link\n     * Builder#addEntry(Path, AbsoluteUnixPath)} for more information.\n     *\n     * @param sourceFile the source file to add to the layer\n     * @param pathInContainer the path in the container file system corresponding to the {@code\n     *     sourceFile}\n     * @param permissions the file permissions on the container\n     * @param modificationTime the file modification time\n     * @return this\n     * @see Builder#addEntry(Path, AbsoluteUnixPath)\n     * @see FilePermissions#DEFAULT_FILE_PERMISSIONS\n     * @see FilePermissions#DEFAULT_FOLDER_PERMISSIONS\n     */\n    public Builder addEntry(\n        Path sourceFile,\n        AbsoluteUnixPath pathInContainer,\n        FilePermissions permissions,\n        Instant modificationTime) {\n      layerBuilder.addEntry(\n          new FileEntry(sourceFile, pathInContainer, permissions, modificationTime));\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer. If the source file is a directory, the directory and its contents\n     * will be added recursively.\n     *\n     * <p>For example, {@code addEntryRecursive(Paths.get(\"mydirectory\",\n     * AbsoluteUnixPath.get(\"/path/in/container\"))} adds {@code mydirectory} to the container file\n     * system at {@code /path/in/container} such that {@code mydirectory/subfile} is found at {@code\n     * /path/in/container/subfile}.\n     *\n     * @param sourceFile the source file to add to the layer recursively\n     * @param pathInContainer the path in the container file system corresponding to the {@code\n     *     sourceFile}\n     * @return this\n     * @throws IOException if an exception occurred when recursively listing the directory\n     */\n    public Builder addEntryRecursive(Path sourceFile, AbsoluteUnixPath pathInContainer)\n        throws IOException {\n      layerBuilder.addEntryRecursive(sourceFile, pathInContainer);\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer. If the source file is a directory, the directory and its contents\n     * will be added recursively.\n     *\n     * @param sourceFile the source file to add to the layer recursively\n     * @param pathInContainer the path in the container file system corresponding to the {@code\n     *     sourceFile}\n     * @param filePermissionProvider a provider that takes a source path and destination path on the\n     *     container and returns the file permissions that should be set for that path\n     * @return this\n     * @throws IOException if an exception occurred when recursively listing the directory\n     */\n    public Builder addEntryRecursive(\n        Path sourceFile,\n        AbsoluteUnixPath pathInContainer,\n        FilePermissionsProvider filePermissionProvider)\n        throws IOException {\n      layerBuilder.addEntryRecursive(sourceFile, pathInContainer, filePermissionProvider);\n      return this;\n    }\n\n    /**\n     * Adds an entry to the layer. If the source file is a directory, the directory and its contents\n     * will be added recursively.\n     *\n     * @param sourceFile the source file to add to the layer recursively\n     * @param pathInContainer the path in the container file system corresponding to the {@code\n     *     sourceFile}\n     * @param filePermissionProvider a provider that takes a source path and destination path on the\n     *     container and returns the file permissions that should be set for that path\n     * @param modificationTimeProvider a provider that takes a source path and destination path on\n     *     the container and returns the file modification time that should be set for that path\n     * @return this\n     * @throws IOException if an exception occurred when recursively listing the directory\n     */\n    public Builder addEntryRecursive(\n        Path sourceFile,\n        AbsoluteUnixPath pathInContainer,\n        FilePermissionsProvider filePermissionProvider,\n        ModificationTimeProvider modificationTimeProvider)\n        throws IOException {\n      layerBuilder.addEntryRecursive(\n          sourceFile, pathInContainer, filePermissionProvider, modificationTimeProvider);\n      return this;\n    }\n\n    /**\n     * Returns the built {@link LayerConfiguration}.\n     *\n     * @return the built {@link LayerConfiguration}\n     */\n    public LayerConfiguration build() {\n      return new LayerConfiguration(layerBuilder.build());\n    }\n  }\n\n  /** Provider that returns default file permissions (644 for files, 755 for directories). */\n  public static final FilePermissionsProvider DEFAULT_FILE_PERMISSIONS_PROVIDER =\n      FileEntriesLayer.DEFAULT_FILE_PERMISSIONS_PROVIDER;\n\n  /** Default file modification time (EPOCH + 1 second). */\n  public static final Instant DEFAULT_MODIFICATION_TIME =\n      FileEntriesLayer.DEFAULT_MODIFICATION_TIME;\n\n  /** Provider that returns default file modification time (EPOCH + 1 second). */\n  public static final ModificationTimeProvider DEFAULT_MODIFICATION_TIME_PROVIDER =\n      FileEntriesLayer.DEFAULT_MODIFICATION_TIME_PROVIDER;\n\n  /**\n   * Gets a new {@link Builder} for {@link LayerConfiguration}.\n   *\n   * @return a new {@link Builder}\n   */\n  public static Builder builder() {\n    return new Builder();\n  }\n\n  private final FileEntriesLayer fileEntriesLayer;\n\n  private LayerConfiguration(FileEntriesLayer fileEntriesLayer) {\n    this.fileEntriesLayer = fileEntriesLayer;\n  }\n\n  /**\n   * Gets the name.\n   *\n   * @return the name\n   */\n  public String getName() {\n    return fileEntriesLayer.getName();\n  }\n\n  /**\n   * Gets the list of entries.\n   *\n   * @return the list of entries\n   */\n  public ImmutableList<LayerEntry> getLayerEntries() {\n    List<FileEntry> entries = fileEntriesLayer.getEntries();\n    return entries.stream().map(LayerEntry::new).collect(ImmutableList.toImmutableList());\n  }\n\n  FileEntriesLayer toFileEntriesLayer() {\n    return fileEntriesLayer;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.builder.steps.BuildResult;\nimport com.google.cloud.tools.jib.builder.steps.StepsRunner;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ImageConfiguration;\nimport com.google.cloud.tools.jib.docker.CliDockerClient;\nimport com.google.cloud.tools.jib.docker.DockerClientResolver;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.filesystem.XdgDirectories;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ArrayListMultimap;\nimport com.google.common.collect.ImmutableListMultimap;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.ListMultimap;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.function.Consumer;\nimport java.util.function.Function;\nimport javax.annotation.Nullable;\n\n/** Configures how to containerize. */\npublic class Containerizer {\n\n  /**\n   * The default directory for caching the base image layers, in {@code [user cache\n   * home]/google-cloud-tools-java/jib}.\n   */\n  public static final Path DEFAULT_BASE_CACHE_DIRECTORY = XdgDirectories.getCacheHome();\n\n  public static final String DEFAULT_APPLICATION_CACHE_DIRECTORY_NAME =\n      \"jib-core-application-layers-cache\";\n\n  private static final String DEFAULT_TOOL_NAME = \"jib-core\";\n  private static final String DEFAULT_TOOL_VERSION =\n      Containerizer.class.getPackage().getImplementationVersion();\n\n  private static final String DESCRIPTION_FOR_DOCKER_REGISTRY = \"Building and pushing image\";\n  private static final String DESCRIPTION_FOR_DOCKER_DAEMON = \"Building image to Docker daemon\";\n  private static final String DESCRIPTION_FOR_TARBALL = \"Building image tarball\";\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a container registry.\n   *\n   * @param registryImage the {@link RegistryImage} that defines target container registry and\n   *     credentials\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(RegistryImage registryImage) {\n    ImageConfiguration imageConfiguration =\n        ImageConfiguration.builder(registryImage.getImageReference())\n            .setCredentialRetrievers(registryImage.getCredentialRetrievers())\n            .build();\n\n    Function<BuildContext, StepsRunner> stepsRunnerFactory =\n        buildContext -> StepsRunner.begin(buildContext).registryPushSteps();\n\n    return new Containerizer(\n        DESCRIPTION_FOR_DOCKER_REGISTRY, imageConfiguration, stepsRunnerFactory, true);\n  }\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a Docker daemon.\n   *\n   * @param dockerDaemonImage the {@link DockerDaemonImage} that defines target Docker daemon\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(DockerDaemonImage dockerDaemonImage) {\n    DockerClient dockerClient =\n        DockerClientResolver.resolve(dockerDaemonImage.getDockerEnvironment())\n            .orElse(\n                new CliDockerClient(\n                    dockerDaemonImage.getDockerExecutable(),\n                    dockerDaemonImage.getDockerEnvironment()));\n\n    return to(dockerClient, dockerDaemonImage);\n  }\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a tarball archive.\n   *\n   * @param tarImage the {@link TarImage} that defines target output file\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(TarImage tarImage) {\n    Optional<ImageReference> imageReference = tarImage.getImageReference();\n    if (!imageReference.isPresent()) {\n      throw new IllegalArgumentException(\n          \"Image name must be set when building a TarImage; use TarImage#named(...) to set the name\"\n              + \" of the target image\");\n    }\n\n    ImageConfiguration imageConfiguration =\n        ImageConfiguration.builder(imageReference.get()).build();\n\n    Function<BuildContext, StepsRunner> stepsRunnerFactory =\n        buildContext -> StepsRunner.begin(buildContext).tarBuildSteps(tarImage.getPath());\n\n    return new Containerizer(\n        DESCRIPTION_FOR_TARBALL, imageConfiguration, stepsRunnerFactory, false);\n  }\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a Docker daemon.\n   *\n   * @param dockerClient the {@link DockerClient} to connect\n   * @param dockerDaemonImage the {@link DockerDaemonImage} that defines target Docker daemon\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(DockerClient dockerClient, DockerDaemonImage dockerDaemonImage) {\n    ImageConfiguration imageConfiguration =\n        ImageConfiguration.builder(dockerDaemonImage.getImageReference()).build();\n\n    Function<BuildContext, StepsRunner> stepsRunnerFactory =\n        buildContext -> StepsRunner.begin(buildContext).dockerLoadSteps(dockerClient);\n\n    return new Containerizer(\n        DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, stepsRunnerFactory, false);\n  }\n\n  private final String description;\n  private final ImageConfiguration imageConfiguration;\n  private final Function<BuildContext, StepsRunner> stepsRunnerFactory;\n  private final boolean mustBeOnline;\n  private final Set<String> additionalTags = new HashSet<>();\n  private final EventHandlers.Builder eventHandlersBuilder = EventHandlers.builder();\n\n  @Nullable private ExecutorService executorService;\n  private Path baseImageLayersCacheDirectory = DEFAULT_BASE_CACHE_DIRECTORY;\n  @Nullable private Path applicationLayersCacheDirectory;\n  private boolean allowInsecureRegistries = false;\n  private boolean offline = false;\n  private String toolName = DEFAULT_TOOL_NAME;\n  @Nullable private String toolVersion = DEFAULT_TOOL_VERSION;\n  private boolean alwaysCacheBaseImage = false;\n  private ListMultimap<String, String> registryMirrors = ArrayListMultimap.create();\n\n  /** Instantiate with {@link #to}. */\n  private Containerizer(\n      String description,\n      ImageConfiguration imageConfiguration,\n      Function<BuildContext, StepsRunner> stepsRunnerFactory,\n      boolean mustBeOnline) {\n    this.description = description;\n    this.imageConfiguration = imageConfiguration;\n    this.stepsRunnerFactory = stepsRunnerFactory;\n    this.mustBeOnline = mustBeOnline;\n  }\n\n  /**\n   * Adds an additional tag to tag the target image with. For example, the following would\n   * containerize to both {@code gcr.io/my-project/my-image:tag} and {@code\n   * gcr.io/my-project/my-image:tag2}:\n   *\n   * <pre>{@code\n   * Containerizer.to(RegistryImage.named(\"gcr.io/my-project/my-image:tag\")).withAdditionalTag(\"tag2\");\n   * }</pre>\n   *\n   * @param tag the additional tag to push to\n   * @return this\n   */\n  public Containerizer withAdditionalTag(String tag) {\n    Preconditions.checkArgument(ImageReference.isValidTag(tag), \"invalid tag '%s'\", tag);\n    additionalTags.add(tag);\n    return this;\n  }\n\n  /**\n   * Sets the {@link ExecutorService} Jib executes on. Jib, by default, uses {@link\n   * Executors#newCachedThreadPool}.\n   *\n   * @param executorService the {@link ExecutorService}\n   * @return this\n   */\n  public Containerizer setExecutorService(@Nullable ExecutorService executorService) {\n    this.executorService = executorService;\n    return this;\n  }\n\n  /**\n   * Sets the directory to use for caching base image layers. This cache can (and should) be shared\n   * between multiple images. The default base image layers cache directory is {@code [user cache\n   * home]/google-cloud-tools-java/jib} ({@link #DEFAULT_BASE_CACHE_DIRECTORY}. This directory can\n   * be the same directory used for {@link #setApplicationLayersCache}.\n   *\n   * @param cacheDirectory the cache directory\n   * @return this\n   */\n  public Containerizer setBaseImageLayersCache(Path cacheDirectory) {\n    baseImageLayersCacheDirectory = cacheDirectory;\n    return this;\n  }\n\n  /**\n   * Sets the directory to use for caching application layers. This cache can be shared between\n   * multiple images. If not set, a temporary directory will be used as the application layers\n   * cache. This directory can be the same directory used for {@link #setBaseImageLayersCache}.\n   *\n   * @param cacheDirectory the cache directory\n   * @return this\n   */\n  public Containerizer setApplicationLayersCache(Path cacheDirectory) {\n    applicationLayersCacheDirectory = cacheDirectory;\n    return this;\n  }\n\n  /**\n   * Adds the {@code eventConsumer} to handle the {@link JibEvent} with class {@code eventType}. The\n   * order in which handlers are added is the order in which they are called when the event is\n   * dispatched.\n   *\n   * <p><b>Note: Implementations of {@code eventConsumer} must be thread-safe.</b>\n   *\n   * @param eventType the event type that {@code eventConsumer} should handle\n   * @param eventConsumer the event handler\n   * @param <E> the type of {@code eventType}\n   * @return this\n   */\n  public <E extends JibEvent> Containerizer addEventHandler(\n      Class<E> eventType, Consumer<? super E> eventConsumer) {\n    eventHandlersBuilder.add(eventType, eventConsumer);\n    return this;\n  }\n\n  /**\n   * Adds the {@code eventConsumer} to handle all {@link JibEvent} types. See {@link\n   * #addEventHandler(Class, Consumer)} for more details.\n   *\n   * @param eventConsumer the event handler\n   * @return this\n   */\n  public Containerizer addEventHandler(Consumer<JibEvent> eventConsumer) {\n    eventHandlersBuilder.add(JibEvent.class, eventConsumer);\n    return this;\n  }\n\n  /**\n   * Sets whether or not to allow communication over HTTP/insecure HTTPS.\n   *\n   * @param allowInsecureRegistries if {@code true}, insecure connections will be allowed\n   * @return this\n   */\n  public Containerizer setAllowInsecureRegistries(boolean allowInsecureRegistries) {\n    this.allowInsecureRegistries = allowInsecureRegistries;\n    return this;\n  }\n\n  /**\n   * Sets whether or not to run the build in offline mode. In offline mode, the base image is\n   * retrieved from the cache instead of pulled from a registry, and the build will fail if the base\n   * image is not in the cache or if the target is an image registry.\n   *\n   * @param offline if {@code true}, the build will run in offline mode\n   * @return this\n   */\n  public Containerizer setOfflineMode(boolean offline) {\n    if (mustBeOnline && offline) {\n      throw new IllegalStateException(\"Cannot build to a container registry in offline mode\");\n    }\n    this.offline = offline;\n    return this;\n  }\n\n  /**\n   * Sets the name of the tool that is using Jib Core. The tool name is sent as part of the {@code\n   * User-Agent} in registry requests and set as the {@code created_by} in the container layer\n   * history. Defaults to {@code jib-core}.\n   *\n   * @param toolName the name of the tool using this library\n   * @return this\n   */\n  public Containerizer setToolName(String toolName) {\n    this.toolName = toolName;\n    return this;\n  }\n\n  /**\n   * Sets the version of the tool that is using Jib Core. The tool version is sent as part of the\n   * {@code User-Agent} in registry requests and set as the {@code created_by} in the container\n   * layer history. Defaults to the current version of jib-core.\n   *\n   * @param toolVersion the name of the tool using this library\n   * @return this\n   */\n  public Containerizer setToolVersion(@Nullable String toolVersion) {\n    this.toolVersion = toolVersion;\n    return this;\n  }\n\n  /**\n   * Controls the optimization which skips downloading base image layers that exist in a target\n   * registry. If the user does not set this property, then read as false.\n   *\n   * @param alwaysCacheBaseImage if {@code true}, base image layers are always pulled and cached. If\n   *     {@code false}, base image layers will not be pulled/cached if they already exist on the\n   *     target registry.\n   * @return this\n   */\n  public Containerizer setAlwaysCacheBaseImage(boolean alwaysCacheBaseImage) {\n    this.alwaysCacheBaseImage = alwaysCacheBaseImage;\n    return this;\n  }\n\n  /**\n   * Adds mirrors for a base image registry. Jib will try its mirrors in the given order before\n   * finally trying the registry.\n   *\n   * @param registry base image registry for which mirrors are configured\n   * @param mirrors a list of mirrors, where each element is in the form of {@code host[:port]}\n   * @return this\n   */\n  public Containerizer addRegistryMirrors(String registry, List<String> mirrors) {\n    registryMirrors.putAll(registry, mirrors);\n    return this;\n  }\n\n  Set<String> getAdditionalTags() {\n    return ImmutableSet.copyOf(additionalTags);\n  }\n\n  ListMultimap<String, String> getRegistryMirrors() {\n    return ImmutableListMultimap.copyOf(registryMirrors);\n  }\n\n  Optional<ExecutorService> getExecutorService() {\n    return Optional.ofNullable(executorService);\n  }\n\n  Path getBaseImageLayersCacheDirectory() {\n    return baseImageLayersCacheDirectory;\n  }\n\n  Path getApplicationLayersCacheDirectory() throws CacheDirectoryCreationException {\n    if (applicationLayersCacheDirectory == null) {\n      // Create a directory in temp if application layers cache directory is not set.\n      try {\n        Path tmp = Paths.get(System.getProperty(\"java.io.tmpdir\"));\n        applicationLayersCacheDirectory = tmp.resolve(DEFAULT_APPLICATION_CACHE_DIRECTORY_NAME);\n        Files.createDirectories(applicationLayersCacheDirectory);\n      } catch (IOException ex) {\n        throw new CacheDirectoryCreationException(ex);\n      }\n    }\n    return applicationLayersCacheDirectory;\n  }\n\n  EventHandlers buildEventHandlers() {\n    return eventHandlersBuilder.build();\n  }\n\n  boolean getAllowInsecureRegistries() {\n    return allowInsecureRegistries;\n  }\n\n  boolean isOfflineMode() {\n    return offline;\n  }\n\n  String getToolName() {\n    return toolName;\n  }\n\n  @Nullable\n  String getToolVersion() {\n    return toolVersion;\n  }\n\n  boolean getAlwaysCacheBaseImage() {\n    return alwaysCacheBaseImage;\n  }\n\n  String getDescription() {\n    return description;\n  }\n\n  ImageConfiguration getImageConfiguration() {\n    return imageConfiguration;\n  }\n\n  BuildResult run(BuildContext buildContext) throws ExecutionException, InterruptedException {\n    return stepsRunnerFactory.apply(buildContext).run();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/InsecureRegistryException.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport java.net.URL;\n\n/**\n * Throw when attempting to access an insecure registry when only secure connections are allowed.\n */\npublic class InsecureRegistryException extends RegistryException {\n\n  /**\n   * Creates a new exception with a human readable message.\n   *\n   * @param insecureUrl the insecure url that is attempted to be accessed\n   * @param cause the underlying cause that triggered this exception\n   */\n  public InsecureRegistryException(URL insecureUrl, Throwable cause) {\n    super(\n        \"Failed to verify the server at \"\n            + insecureUrl\n            + \" because only secure connections are allowed.\",\n        cause);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/MainClassFinder.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.common.base.Preconditions;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.function.Consumer;\nimport javax.annotation.Nullable;\nimport org.objectweb.asm.ClassReader;\nimport org.objectweb.asm.ClassVisitor;\nimport org.objectweb.asm.MethodVisitor;\nimport org.objectweb.asm.Opcodes;\n\n/**\n * Finds main classes in a list of class files. Main classes are classes that define the {@code\n * public static void main(String[] args)} method.\n */\npublic class MainClassFinder {\n\n  /** The result of a call to {@link #find}. */\n  public static class Result {\n\n    /** The type of result. */\n    public enum Type {\n\n      // Found a single main class.\n      MAIN_CLASS_FOUND,\n\n      // Did not find any main class.\n      MAIN_CLASS_NOT_FOUND,\n\n      // Found multiple main classes.\n      MULTIPLE_MAIN_CLASSES\n    }\n\n    private static Result success(String foundMainClass) {\n      return new Result(Type.MAIN_CLASS_FOUND, Collections.singletonList(foundMainClass));\n    }\n\n    private static Result mainClassNotFound() {\n      return new Result(Type.MAIN_CLASS_NOT_FOUND, Collections.emptyList());\n    }\n\n    private static Result multipleMainClasses(List<String> foundMainClasses) {\n      return new Result(Type.MULTIPLE_MAIN_CLASSES, foundMainClasses);\n    }\n\n    private final Type type;\n    private final List<String> foundMainClasses;\n\n    private Result(Type type, List<String> foundMainClasses) {\n      this.foundMainClasses = foundMainClasses;\n      this.type = type;\n    }\n\n    /**\n     * Gets the found main class. Only call if {@link #getType} is {@link Type#MAIN_CLASS_FOUND}.\n     *\n     * @return the found main class\n     */\n    public String getFoundMainClass() {\n      Preconditions.checkState(Type.MAIN_CLASS_FOUND == type);\n      Preconditions.checkState(foundMainClasses.size() == 1);\n      return foundMainClasses.get(0);\n    }\n\n    /**\n     * Gets the type of the result.\n     *\n     * @return the type of the result\n     */\n    public Type getType() {\n      return type;\n    }\n\n    /**\n     * Gets the found main classes.\n     *\n     * @return the found main classes\n     */\n    public List<String> getFoundMainClasses() {\n      return foundMainClasses;\n    }\n  }\n\n  /** {@link ClassVisitor} that keeps track of whether or not it has visited a main class. */\n  private static class MainClassVisitor extends ClassVisitor {\n\n    /** The return/argument types for main. */\n    private static final String MAIN_DESCRIPTOR =\n        org.objectweb.asm.Type.getMethodDescriptor(\n            org.objectweb.asm.Type.VOID_TYPE, org.objectweb.asm.Type.getType(String[].class));\n\n    /** Accessors that main may or may not have. */\n    private static final int OPTIONAL_ACCESS =\n        Opcodes.ACC_FINAL | Opcodes.ACC_DEPRECATED | Opcodes.ACC_VARARGS | Opcodes.ACC_SYNTHETIC;\n\n    private boolean visitedMainClass;\n\n    private MainClassVisitor() {\n      super(Opcodes.ASM9);\n    }\n\n    @Override\n    @Nullable\n    public MethodVisitor visitMethod(\n        int access, String name, String descriptor, String signature, String[] exceptions) {\n      if ((access & ~OPTIONAL_ACCESS) == (Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC)\n          && name.equals(\"main\")\n          && descriptor.equals(MAIN_DESCRIPTOR)) {\n        visitedMainClass = true;\n      }\n      return null;\n    }\n  }\n\n  /**\n   * Tries to find classes with {@code psvm} (see class javadoc) in {@code files}.\n   *\n   * @param files the files to search\n   * @param logger a {@link Consumer} used to handle log events\n   * @return the {@link Result} of the main class finding attempt\n   */\n  public static Result find(List<Path> files, Consumer<LogEvent> logger) {\n    List<String> mainClasses = new ArrayList<>();\n    for (Path file : files) {\n      // Makes sure classFile is valid.\n      if (!Files.exists(file)) {\n        logger.accept(LogEvent.debug(\"MainClassFinder: \" + file + \" does not exist; ignoring\"));\n        continue;\n      }\n      if (!Files.isRegularFile(file)) {\n        logger.accept(\n            LogEvent.debug(\"MainClassFinder: \" + file + \" is not a regular file; skipping\"));\n        continue;\n      }\n      if (!file.toString().endsWith(\".class\")) {\n        logger.accept(\n            LogEvent.debug(\"MainClassFinder: \" + file + \" is not a class file; skipping\"));\n        continue;\n      }\n\n      MainClassVisitor mainClassVisitor = new MainClassVisitor();\n      try (InputStream classFileInputStream = Files.newInputStream(file)) {\n        ClassReader reader = new ClassReader(classFileInputStream);\n        reader.accept(mainClassVisitor, 0);\n        if (mainClassVisitor.visitedMainClass) {\n          mainClasses.add(reader.getClassName().replace('/', '.'));\n        }\n\n      } catch (IllegalArgumentException ex) {\n        throw new UnsupportedOperationException(\n            \"Check the full stace trace, and if the root cause is from ASM ClassReader about \"\n                + \"unsupported class file version, see \"\n                + \"https://github.com/GoogleContainerTools/jib/blob/master/docs/faq.md\"\n                + \"#i-am-seeing-unsupported-class-file-major-version-when-building\",\n            ex);\n\n      } catch (ArrayIndexOutOfBoundsException ignored) {\n        // Not a valid class file (thrown by ClassReader if it reads an invalid format)\n        logger.accept(LogEvent.warn(\"Invalid class file found: \" + file));\n\n      } catch (IOException ignored) {\n        // Could not read class file.\n        logger.accept(LogEvent.warn(\"Could not read file: \" + file));\n      }\n    }\n\n    if (mainClasses.size() == 1) {\n      // Valid class found.\n      return Result.success(mainClasses.get(0));\n    }\n    if (mainClasses.isEmpty()) {\n      // No main class found anywhere.\n      return Result.mainClassNotFound();\n    }\n    // More than one main class found.\n    return Result.multipleMainClasses(mainClasses);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/InvalidImageReferenceException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\n/** Thrown when attempting to parse an invalid image reference. */\npublic class InvalidImageReferenceException extends Exception {\n\n  private final String reference;\n\n  public InvalidImageReferenceException(String reference) {\n    super(\"Invalid image reference: \" + reference);\n    this.reference = reference;\n  }\n\n  public String getInvalidReference() {\n    return reference;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java\n/*\n * Copyright 2018 Google LLC.\n *\n...\n", "needle_token_start": 783, "needle_token_end": 841, "code_context_ntokens": 16053, "output": ["```java\n/**\n * Creates a new {@link CredentialRetriever} that returns a known {@link Credential}.\n *\n * @param credential the known credential\n * @param credentialSource the source of the credentials (for logging)\n * @return a new {@link CredentialRetriever}\n */\npublic CredentialRetriever known(Credential credential, String credentialSource) {\n    return () -> {\n      logGotCredentialsFrom(\"credentials from \" + credentialSource);\n      return Optional.of(credential);\n    };\n  }\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "isDefaultTag", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/api/ImageReference.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To determine if a given string matches the default image tag commonly used in Docker images.\n2. **Input**: A string representing an image tag.\n3. **Output**: A boolean value; `true` if the input string is the default image tag, otherwise `false`.\n4. **Procedure**: The function compares the input string with the predefined default tag value. If they match, it returns `true`; otherwise, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n  /** Matches a tag of max length 128. */\n  private static final String TAG_REGEX = \"[\\\\w][\\\\w.-]{0,127}\";\n\n  /**\n   * Matches a full image reference, which is the registry, repository, and tag/digest separated by\n   * backslashes. The repository is required, but the registry and tag/digest are optional.\n   */\n  private static final String REFERENCE_REGEX =\n      String.format(\n          \"^(?:(%s)/)?(%s)(?::(%s))?(?:@(%s))?$\",\n          REGISTRY_REGEX, REPOSITORY_REGEX, TAG_REGEX, DescriptorDigest.DIGEST_REGEX);\n\n  private static final Pattern REFERENCE_PATTERN = Pattern.compile(REFERENCE_REGEX);\n\n  /**\n   * Parses a string {@code reference} into an {@link ImageReference}.\n   *\n   * <p>Image references should generally be in the form: {@code <registry>/<repository>:<tag>} For\n   * example, an image reference could be {@code gcr.io/k8s-skaffold/skaffold:v1.20.0}.\n   *\n   * <p>See <a\n   * href=\"https://docs.docker.com/engine/reference/commandline/tag/#extended-description\">https://docs.docker.com/engine/reference/commandline/tag/#extended-description</a>\n   * for a description of valid image reference format. Note, however, that the image reference is\n   * referred confusingly as {@code tag} on that page.\n   *\n   * @param reference the string to parse\n   * @return an {@link ImageReference} parsed from the string\n   * @throws InvalidImageReferenceException if {@code reference} is formatted incorrectly\n   */\n  public static ImageReference parse(String reference) throws InvalidImageReferenceException {\n    if (reference.equals(SCRATCH)) {\n      return ImageReference.scratch();\n    }\n\n    Matcher matcher = REFERENCE_PATTERN.matcher(reference);\n\n    if (!matcher.find() || matcher.groupCount() < 4) {\n      throw new InvalidImageReferenceException(reference);\n    }\n\n    String registry = matcher.group(1);\n    String repository = matcher.group(2);\n    String tag = matcher.group(3);\n    String digest = matcher.group(4);\n\n    // If no registry was matched, use Docker Hub by default.\n    if (Strings.isNullOrEmpty(registry)) {\n      registry = DOCKER_HUB_REGISTRY;\n    }\n\n    if (Strings.isNullOrEmpty(repository)) {\n      throw new InvalidImageReferenceException(reference);\n    }\n    /*\n     * If a registry was matched but it does not contain any dots or colons, it should actually be\n     * part of the repository unless it is \"localhost\".\n     *\n     * See https://github.com/docker/distribution/blob/245ca4659e09e9745f3cc1217bf56e946509220c/reference/normalize.go#L62\n     */\n    if (!registry.contains(\".\") && !registry.contains(\":\") && !\"localhost\".equals(registry)) {\n      repository = registry + \"/\" + repository;\n      registry = DOCKER_HUB_REGISTRY;\n    }\n\n    /*\n     * For Docker Hub, if the repository is only one component, then it should be prefixed with\n     * 'library/'.\n     *\n     * See https://docs.docker.com/engine/reference/commandline/pull/#pull-an-image-from-docker-hub\n     */\n    if (DOCKER_HUB_REGISTRY.equals(registry) && repository.indexOf('/') < 0) {\n      repository = LIBRARY_REPOSITORY_PREFIX + repository;\n    }\n\n    if (Strings.isNullOrEmpty(tag) && Strings.isNullOrEmpty(digest)) {\n      tag = DEFAULT_TAG;\n    }\n    if (Strings.isNullOrEmpty(tag)) {\n      tag = null;\n    }\n    if (Strings.isNullOrEmpty(digest)) {\n      digest = null;\n    }\n\n    return new ImageReference(registry, repository, tag, digest);\n  }\n\n  /**\n   * Constructs an {@link ImageReference} from the image reference components, consisting of an\n   * optional registry, a repository, and an optional tag.\n   *\n   * @param registry the image registry, or {@code null} to use the default registry (Docker Hub)\n   * @param repository the image repository\n   * @param qualifier the image tag or digest, or {@code null} to use the default tag ({@code\n   *     latest}).\n   * @return an {@link ImageReference} built from the given registry, repository, and tag\n   */\n  public static ImageReference of(\n      @Nullable String registry, String repository, @Nullable String qualifier) {\n    if (!Strings.isNullOrEmpty(qualifier) && isValidDigest(qualifier)) {\n      return of(registry, repository, null, qualifier);\n    }\n    return of(registry, repository, qualifier, null);\n  }\n\n  /**\n   * Constructs an {@link ImageReference} from the image reference components, consisting of an\n   * optional registry, a repository, an optional tag, and an optional digest. If neither the tag\n   * nor digest are specified, {@code tag} will take on the default value of {@code latest}.\n   *\n   * @param registry the image registry, or {@code null} to use the default registry (Docker Hub)\n   * @param repository the image repository\n   * @param tag the image tag, or {@code null} to use the default tag ({@code latest})\n   * @param digest the image digest\n   * @return an {@link ImageReference} built from the given registry, repository, and tag\n   */\n  public static ImageReference of(\n      @Nullable String registry, String repository, @Nullable String tag, @Nullable String digest) {\n    Preconditions.checkArgument(Strings.isNullOrEmpty(registry) || isValidRegistry(registry));\n    Preconditions.checkArgument(isValidRepository(repository));\n    Preconditions.checkArgument(Strings.isNullOrEmpty(tag) || isValidTag(tag));\n    Preconditions.checkArgument(Strings.isNullOrEmpty(digest) || isValidDigest(digest));\n\n    if (Strings.isNullOrEmpty(registry)) {\n      registry = DOCKER_HUB_REGISTRY;\n    }\n    if (Strings.isNullOrEmpty(tag) && Strings.isNullOrEmpty(digest)) {\n      tag = DEFAULT_TAG;\n    }\n    return new ImageReference(registry, repository, tag, digest);\n  }\n\n  /**\n   * Constructs an {@link ImageReference} with an empty registry and tag component, and repository\n   * set to \"scratch\".\n   *\n   * @return an {@link ImageReference} with an empty registry and tag component, and repository set\n   *     to \"scratch\"\n   */\n  public static ImageReference scratch() {\n    return new ImageReference(\"\", SCRATCH, null, null);\n  }\n\n  /**\n   * Returns {@code true} if {@code registry} is a valid registry string. For example, a valid\n   * registry could be {@code gcr.io} or {@code localhost:5000}.\n   *\n   * @param registry the registry to check\n   * @return {@code true} if is a valid registry; {@code false} otherwise\n   */\n  public static boolean isValidRegistry(String registry) {\n    return registry.matches(REGISTRY_REGEX);\n  }\n\n  /**\n   * Returns {@code true} if {@code repository} is a valid repository string. For example, a valid\n   * repository string could be {@code my-repository} or {@code k8s-skaffold/skaffold}.\n   *\n   * @param repository the repository to check\n   * @return {@code true} if is a valid repository; {@code false} otherwise\n   */\n  public static boolean isValidRepository(String repository) {\n    return repository.matches(REPOSITORY_REGEX);\n  }\n\n  /**\n   * Returns {@code true} if {@code tag} is a valid tag string. For example, a valid tag could be\n   * {@code v120.5-release}.\n   *\n   * @param tag the tag to check\n   * @return {@code true} if is a valid tag; {@code false} otherwise\n   */\n  public static boolean isValidTag(String tag) {\n    return tag.matches(TAG_REGEX);\n  }\n\n  /**\n   * Returns {@code true} if {@code digest} is a valid digest string. For example, a valid digest\n   * could be {@code sha256:868fd30a0e47b8d8ac485df174795b5e2fe8a6c8f056cc707b232d65b8a1ab68}.\n   *\n   * @param digest the digest to check\n   * @return {@code true} if is a valid digest; {@code false} otherwise\n   */\n  public static boolean isValidDigest(String digest) {\n    return digest.matches(DescriptorDigest.DIGEST_REGEX);\n  }\n\n  /**\n   * Returns {@code true} if {@code tag} is the default tag ({@code latest}); {@code false} if not.\n   *\n   * @param tag the tag to check\n   * @return {@code true} if {@code tag} is the default tag ({@code latest}); {@code false} if not\n   */\n  \npublic static boolean isDefaultTag(@Nullable String tag) {\n    return DEFAULT_TAG.equals(tag);\n  }\n\n  private final String registry;\n  private final String repository;\n  @Nullable private final String tag;\n  @Nullable private final String digest;\n\n  /** Construct with {@link #parse}. */\n  private ImageReference(\n      String registry, String repository, @Nullable String tag, @Nullable String digest) {\n    Preconditions.checkArgument(\n        SCRATCH.equals(repository) || !Strings.isNullOrEmpty(tag) || !Strings.isNullOrEmpty(digest),\n        \"Either tag or digest needs to be set.\");\n    this.registry = RegistryAliasGroup.getHost(registry);\n    this.repository = repository;\n    this.tag = tag;\n    this.digest = digest;\n  }\n\n  /**\n   * Gets the registry portion of the {@link ImageReference}.\n   *\n   * @return the registry host\n   */\n  public String getRegistry() {\n    return registry;\n  }\n\n  /**\n   * Gets the repository portion of the {@link ImageReference}.\n   *\n   * @return the repository\n   */\n  public String getRepository() {\n    return repository;\n  }\n\n  /**\n   * Gets the tag portion of the {@link ImageReference}.\n   *\n   * @return the optional tag\n   */\n  public Optional<String> getTag() {\n    return Optional.ofNullable(tag);\n  }\n\n  /**\n   * Gets the digest portion of the {@link ImageReference}.\n   *\n   * @return the optional digest\n   */\n  public Optional<String> getDigest() {\n    return Optional.ofNullable(digest);\n  }\n\n  /**\n   * Gets the digest portion of the {@link ImageReference} if set, else returns the tag.\n   *\n   * @return the digest if set, else the tag\n   */\n  public String getQualifier() {\n    if (!Strings.isNullOrEmpty(digest)) {\n      return digest;\n    }\n    return Preconditions.checkNotNull(tag);\n  }\n\n  /**\n   * Returns {@code true} if the {@link ImageReference} uses the default tag ({@code latest});\n   * {@code false} if not.\n   *\n   * @return {@code true} if uses the default tag; {@code false} if not\n   */\n  public boolean usesDefaultTag() {\n    return isDefaultTag(tag);\n  }\n\n  /**\n   * Returns {@code true} if the {@link ImageReference} is a scratch image; {@code false} if not.\n   *\n   * @return {@code true} if the {@link ImageReference} is a scratch image; {@code false} if not\n   */\n  public boolean isScratch() {\n    return \"\".equals(registry)\n        && SCRATCH.equals(repository)\n        && Strings.isNullOrEmpty(tag)\n        && Strings.isNullOrEmpty(digest);\n  }\n\n  /**\n   * Gets an {@link ImageReference} with the same registry and repository, but a different tag or\n   * digest.\n   *\n   * @param newQualifier the new tag or digest\n   * @return an {@link ImageReference} with the same registry/repository and the new tag or digest\n   */\n  public ImageReference withQualifier(String newQualifier) {\n    if (isValidDigest(newQualifier)) {\n      return ImageReference.of(registry, repository, tag, newQualifier);\n    }\n    return ImageReference.of(registry, repository, newQualifier, digest);\n  }\n\n  /**\n   * Stringifies the {@link ImageReference}.\n   *\n   * @return the image reference in Docker-readable format (inverse of {@link #parse})\n   */\n  @Override\n  public String toString() {\n    return toString(false);\n  }\n\n  /**\n   * Stringifies the {@link ImageReference}. If the digest is set, the result will include the\n   * digest and no tag. Otherwise, the result will include the tag, or {@code latest} if no tag is\n   * set.\n   *\n   * @return the image reference in Docker-readable format including a qualifier.\n   */\n  public String toStringWithQualifier() {\n    return toString(true);\n  }\n\n  /**\n   * Stringifies the {@link ImageReference}.\n   *\n   * @param singleQualifier when {@code true}, the result will include exactly one qualifier (i.e.\n   *     the digest, or the tag if the digest is missing). When {@code false}, the result will\n   *     include all specified qualifiers (omitting tag if the default {@code latest} is used).\n   * @return the image reference in a Docker-readable format.\n   */\n  private String toString(boolean singleQualifier) {\n    if (isScratch()) {\n      return SCRATCH;\n    }\n\n    StringBuilder referenceString = new StringBuilder();\n\n    if (!DOCKER_HUB_REGISTRY.equals(registry)) {\n      // Use registry and repository if not Docker Hub.\n      referenceString.append(registry).append('/').append(repository);\n\n    } else if (repository.startsWith(LIBRARY_REPOSITORY_PREFIX)) {\n      // If Docker Hub and repository has 'library/' prefix, remove the 'library/' prefix.\n      referenceString.append(repository.substring(LIBRARY_REPOSITORY_PREFIX.length()));\n\n    } else {\n      // Use just repository if Docker Hub.\n      referenceString.append(repository);\n    }\n\n    if (singleQualifier) {\n      if (!Strings.isNullOrEmpty(digest)) {\n        referenceString.append('@').append(digest);\n      } else {\n        referenceString.append(':').append(tag);\n      }\n    } else {\n      if (!Strings.isNullOrEmpty(tag) && !usesDefaultTag()) {\n        referenceString.append(':').append(tag);\n      }\n      if (!Strings.isNullOrEmpty(digest)) {\n        referenceString.append('@').append(digest);\n      }\n    }\n\n    return referenceString.toString();\n  }\n\n  @Override\n  public boolean equals(Object other) {\n    if (this == other) {\n      return true;\n    }\n    if (!(other instanceof ImageReference)) {\n      return false;\n    }\n    ImageReference otherImageReference = (ImageReference) other;\n    return registry.equals(otherImageReference.registry)\n        && repository.equals(otherImageReference.repository)\n        && Objects.equals(tag, otherImageReference.tag)\n        && Objects.equals(digest, otherImageReference.digest);\n  }\n\n  @Override\n  public int hashCode() {\n    return Objects.hash(registry, repository, tag, digest);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.cache;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.buildplan.FileEntry;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Objects;\n\n/**\n * Generates a selector based on {@link FileEntry}s for a layer. Selectors are secondary references\n * for a cache entries.\n *\n * <p>The selector is the SHA256 hash of the list of layer entries serialized in the following form:\n *\n * <pre>{@code\n * [\n *   {\n *     \"sourceFile\": \"source/file/for/layer/entry/1\",\n *     \"extractionPath\": \"/extraction/path/for/layer/entry/1\"\n *     \"sourceModificationTime\": \"2018-10-03T15:48:32.416152Z\"\n *     \"targetModificationTime\": \"1970-01-01T00:00:01Z\",\n *     \"permissions\": \"777\",\n *     \"ownership\": \"0:0\"\n *   },\n *   {\n *     \"sourceFile\": \"source/file/for/layer/entry/2\",\n *     \"extractionPath\": \"/extraction/path/for/layer/entry/2\"\n *     \"sourceModificationTime\": \"2018-10-03T15:48:32.416152Z\"\n *     \"targetModificationTime\": \"1970-01-01T00:00:01Z\",\n *     \"permissions\": \"777\",\n *     \"ownership\": \"alice:1234\"\n *   }\n * ]\n * }</pre>\n */\nclass LayerEntriesSelector {\n\n  /** Serialized form of a {@link FileEntry}. */\n  @VisibleForTesting\n  static class LayerEntryTemplate implements JsonTemplate, Comparable<LayerEntryTemplate> {\n\n    private final String sourceFile;\n    private final String extractionPath;\n    private final Instant sourceModificationTime;\n    private final Instant targetModificationTime;\n    private final String permissions;\n    private final String ownership;\n\n    @VisibleForTesting\n    LayerEntryTemplate(FileEntry layerEntry) throws IOException {\n      sourceFile = layerEntry.getSourceFile().toAbsolutePath().toString();\n      extractionPath = layerEntry.getExtractionPath().toString();\n      sourceModificationTime = Files.getLastModifiedTime(layerEntry.getSourceFile()).toInstant();\n      targetModificationTime = layerEntry.getModificationTime();\n      permissions = layerEntry.getPermissions().toOctalString();\n      ownership = layerEntry.getOwnership();\n    }\n\n    @Override\n    public int compareTo(LayerEntryTemplate otherLayerEntryTemplate) {\n      int sourceFileComparison = sourceFile.compareTo(otherLayerEntryTemplate.sourceFile);\n      if (sourceFileComparison != 0) {\n        return sourceFileComparison;\n      }\n      int extractionPathComparison =\n          extractionPath.compareTo(otherLayerEntryTemplate.extractionPath);\n      if (extractionPathComparison != 0) {\n        return extractionPathComparison;\n      }\n      int sourceModificationTimeComparison =\n          sourceModificationTime.compareTo(otherLayerEntryTemplate.sourceModificationTime);\n      if (sourceModificationTimeComparison != 0) {\n        return sourceModificationTimeComparison;\n      }\n      int targetModificationTimeComparison =\n          targetModificationTime.compareTo(otherLayerEntryTemplate.targetModificationTime);\n      if (targetModificationTimeComparison != 0) {\n        return targetModificationTimeComparison;\n      }\n      int permissionsComparison = permissions.compareTo(otherLayerEntryTemplate.permissions);\n      if (permissionsComparison != 0) {\n        return permissionsComparison;\n      }\n      return ownership.compareTo(otherLayerEntryTemplate.ownership);\n    }\n\n    @Override\n    public boolean equals(Object other) {\n      if (this == other) {\n        return true;\n      }\n      if (!(other instanceof LayerEntryTemplate)) {\n        return false;\n      }\n      LayerEntryTemplate otherLayerEntryTemplate = (LayerEntryTemplate) other;\n      return sourceFile.equals(otherLayerEntryTemplate.sourceFile)\n          && extractionPath.equals(otherLayerEntryTemplate.extractionPath)\n          && sourceModificationTime.equals(otherLayerEntryTemplate.sourceModificationTime)\n          && targetModificationTime.equals(otherLayerEntryTemplate.targetModificationTime)\n          && permissions.equals(otherLayerEntryTemplate.permissions)\n          && ownership.equals(otherLayerEntryTemplate.ownership);\n    }\n\n    @Override\n    public int hashCode() {\n      return Objects.hash(\n          sourceFile,\n          extractionPath,\n          sourceModificationTime,\n          targetModificationTime,\n          permissions,\n          ownership);\n    }\n  }\n\n  /**\n   * Converts a list of {@link FileEntry}s into a list of {@link LayerEntryTemplate}. The list is\n   * sorted by source file first, then extraction path (see {@link LayerEntryTemplate#compareTo}).\n   *\n   * @param layerEntries the list of {@link FileEntry} to convert\n   * @return list of {@link LayerEntryTemplate} after sorting\n   * @throws IOException if checking the file creation time of a layer entry fails\n   */\n  @VisibleForTesting\n  static List<LayerEntryTemplate> toSortedJsonTemplates(List<FileEntry> layerEntries)\n      throws IOException {\n    List<LayerEntryTemplate> jsonTemplates = new ArrayList<>();\n    for (FileEntry entry : layerEntries) {\n      jsonTemplates.add(new LayerEntryTemplate(entry));\n    }\n    Collections.sort(jsonTemplates);\n    return jsonTemplates;\n  }\n\n  /**\n   * Generates a selector for the list of {@link FileEntry}s. The selector is unique to each unique\n   * set of layer entries, regardless of order. TODO: Should we care about order?\n   *\n   * @param layerEntries the layer entries\n   * @return the selector\n   * @throws IOException if an I/O exception occurs\n   */\n  static DescriptorDigest generateSelector(ImmutableList<FileEntry> layerEntries)\n      throws IOException {\n    return Digests.computeJsonDigest(toSortedJsonTemplates(layerEntries));\n  }\n\n  private LayerEntriesSelector() {}\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BuildableManifestTemplate.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableMap;\nimport java.util.List;\nimport java.util.Map;\nimport javax.annotation.Nullable;\n\n/**\n * Parent class for image manifest JSON templates that can be built.\n *\n * @see V22ManifestTemplate Docker V2.2 format\n * @see OciManifestTemplate OCI format\n */\npublic interface BuildableManifestTemplate extends ManifestTemplate {\n\n  /**\n   * Template for inner JSON object representing content descriptor for a layer or container\n   * configuration.\n   *\n   * @see <a href=\"https://github.com/opencontainers/image-spec/blob/master/descriptor.md\">OCI\n   *     Content Descriptors</a>\n   */\n  class ContentDescriptorTemplate implements JsonTemplate {\n\n    @SuppressWarnings(\"unused\")\n    @Nullable\n    private String mediaType;\n\n    @Nullable private DescriptorDigest digest;\n    private long size;\n    @Nullable private List<String> urls;\n    @Nullable private Map<String, String> annotations;\n\n    ContentDescriptorTemplate(String mediaType, long size, DescriptorDigest digest) {\n      this.mediaType = mediaType;\n      this.size = size;\n      this.digest = digest;\n    }\n\n    /** Necessary for Jackson to create from JSON. */\n    @SuppressWarnings(\"unused\")\n    protected ContentDescriptorTemplate() {}\n\n    public long getSize() {\n      return size;\n    }\n\n    void setSize(long size) {\n      this.size = size;\n    }\n\n    @Nullable\n    public DescriptorDigest getDigest() {\n      return digest;\n    }\n\n    void setDigest(DescriptorDigest digest) {\n      this.digest = digest;\n    }\n\n    @VisibleForTesting\n    @Nullable\n    public List<String> getUrls() {\n      return urls;\n    }\n\n    void setUrls(List<String> urls) {\n      this.urls = ImmutableList.copyOf(urls);\n    }\n\n    @VisibleForTesting\n    @Nullable\n    public Map<String, String> getAnnotations() {\n      return annotations;\n    }\n\n    void setAnnotations(Map<String, String> annotations) {\n      this.annotations = ImmutableMap.copyOf(annotations);\n    }\n  }\n\n  /**\n   * Returns the media type for this manifest, specific to the image format.\n   *\n   * @return the media type for this manifest, specific to the image format\n   */\n  @Override\n  String getManifestMediaType();\n\n  /**\n   * Returns the content descriptor of the container configuration.\n   *\n   * @return the content descriptor of the container configuration\n   */\n  @Nullable\n  ContentDescriptorTemplate getContainerConfiguration();\n\n  /**\n   * Returns an unmodifiable view of the layers.\n   *\n   * @return an unmodifiable view of the layers\n   */\n  List<ContentDescriptorTemplate> getLayers();\n\n  /**\n   * Sets the content descriptor of the container configuration.\n   *\n   * @param size the size of the container configuration.\n   * @param digest the container configuration content descriptor digest.\n   */\n  void setContainerConfiguration(long size, DescriptorDigest digest);\n\n  /**\n   * Adds a layer to the manifest.\n   *\n   * @param size the size of the layer.\n   * @param digest the layer descriptor digest.\n   */\n  void addLayer(long size, DescriptorDigest digest);\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.base.Preconditions;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport javax.annotation.Nullable;\n\n/**\n * JSON Template for Docker Container Configuration referenced in Docker Manifest Schema V2.2\n *\n * <p>Example container config JSON:\n *\n * <pre>{@code\n * {\n *   \"created\": \"1970-01-01T00:00:00Z\",\n *   \"architecture\": \"amd64\",\n *   \"os\": \"linux\",\n *   \"config\": {\n *     \"Env\": [\"/usr/bin/java\"],\n *     \"Entrypoint\": [\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\n *     \"Cmd\": [\"arg1\", \"arg2\"],\n *     \"Healthcheck\": {\n *       \"Test\": [\"CMD-SHELL\", \"/usr/bin/check-health localhost\"],\n *       \"Interval\": 30000000000,\n *       \"Timeout\": 10000000000,\n *       \"StartPeriod\": 0,\n *       \"Retries\": 3\n *     }\n *     \"ExposedPorts\": { \"6000/tcp\":{}, \"8000/tcp\":{}, \"9000/tcp\":{} },\n *     \"Volumes\":{\"/var/job-result-data\":{},\"/var/log/my-app-logs\":{}}},\n *     \"Labels\": { \"com.example.label\": \"value\" },\n *     \"WorkingDir\": \"/home/user/workspace\",\n *     \"User\": \"me\"\n *   },\n *   \"history\": [\n *     {\n *       \"author\": \"Jib\",\n *       \"created\": \"1970-01-01T00:00:00Z\",\n *       \"created_by\": \"jib\"\n *     },\n *     {\n *       \"author\": \"Jib\",\n *       \"created\": \"1970-01-01T00:00:00Z\",\n *       \"created_by\": \"jib\"\n *     }\n *   ]\n *   \"rootfs\": {\n *     \"diff_ids\": [\n *       \"sha256:2aebd096e0e237b447781353379722157e6c2d434b9ec5a0d63f2a6f07cf90c2\",\n *       \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\",\n *     ],\n *     \"type\": \"layers\"\n *   }\n * }\n * }</pre>\n *\n * @see <a href=\"https://docs.docker.com/registry/spec/manifest-v2-2/\">Image Manifest Version 2,\n *     Schema 2</a>\n */\n@JsonIgnoreProperties(ignoreUnknown = true)\npublic class ContainerConfigurationTemplate implements JsonTemplate {\n\n  /** ISO-8601 formatted combined date and time at which the image was created. */\n  @Nullable private String created;\n\n  /** The CPU architecture to run the binaries in this container. */\n  private String architecture = \"amd64\";\n\n  /** The operating system to run the container on. */\n  private String os = \"linux\";\n\n  /** Execution parameters that should be used as a base when running the container. */\n  private final ConfigurationObjectTemplate config = new ConfigurationObjectTemplate();\n\n  /** Describes the history of each layer. */\n  private final List<HistoryEntry> history = new ArrayList<>();\n\n  /** Layer content digests that are used to build the container filesystem. */\n  private final RootFilesystemObjectTemplate rootfs = new RootFilesystemObjectTemplate();\n\n  /** Template for inner JSON object representing the configuration for running the container. */\n  @JsonIgnoreProperties(ignoreUnknown = true)\n  private static class ConfigurationObjectTemplate implements JsonTemplate {\n\n    /** Environment variables in the format {@code VARNAME=VARVALUE}. */\n    @Nullable private List<String> Env;\n\n    /** Command to run when container starts. */\n    @Nullable private List<String> Entrypoint;\n\n    /** Arguments to pass into main. */\n    @Nullable private List<String> Cmd;\n\n    /** Healthcheck. */\n    @Nullable private HealthCheckObjectTemplate Healthcheck;\n\n    /** Network ports the container exposes. */\n    @Nullable private Map<String, Map<String, String>> ExposedPorts;\n\n    /** Labels. */\n    @Nullable private Map<String, String> Labels;\n\n    /** Working directory. */\n    @Nullable private String WorkingDir;\n\n    /** User. */\n    @Nullable private String User;\n\n    /** Volumes. */\n    @Nullable private Map<String, Map<String, String>> Volumes;\n  }\n\n  /** Template for inner JSON object representing the healthcheck configuration. */\n  private static class HealthCheckObjectTemplate implements JsonTemplate {\n\n    /** The test to perform to check that the container is healthy. */\n    @Nullable private List<String> Test;\n\n    /** Number of nanoseconds to wait between probe attempts. */\n    @Nullable private Long Interval;\n\n    /** Number of nanoseconds to wait before considering the check to have hung. */\n    @Nullable private Long Timeout;\n\n    /**\n     * Number of nanoseconds to wait for the container to initialize before starting health-retries.\n     */\n    @Nullable private Long StartPeriod;\n\n    /** The number of consecutive failures needed to consider the container as unhealthy. */\n    @Nullable private Integer Retries;\n  }\n\n  /**\n   * Template for inner JSON object representing the filesystem changesets used to build the\n   * container filesystem.\n   */\n  private static class RootFilesystemObjectTemplate implements JsonTemplate {\n\n    /** The type must always be {@code \"layers\"}. */\n    @SuppressWarnings(\"unused\")\n    private final String type = \"layers\";\n\n    /**\n     * The in-order list of layer content digests (hashes of the uncompressed partial filesystem\n     * changeset).\n     */\n    private final List<DescriptorDigest> diff_ids = new ArrayList<>();\n  }\n\n  public void setCreated(@Nullable String created) {\n    this.created = created;\n  }\n\n  /**\n   * Sets the architecture for which this container was built. See the <a\n   * href=\"https://github.com/opencontainers/image-spec/blob/master/config.md#properties\">OCI Image\n   * Configuration specification</a> for acceptable values.\n   *\n   * @param architecture value for the {@code architecture} field\n   */\n  public void setArchitecture(String architecture) {\n    this.architecture = architecture;\n  }\n\n  /**\n   * Sets the operating system for which this container was built. See the <a\n   * href=\"https://github.com/opencontainers/image-spec/blob/master/config.md#properties\">OCI Image\n   * Configuration specification</a> for acceptable values.\n   *\n   * @param os value for the {@code os} field\n   */\n  public void setOs(String os) {\n    this.os = os;\n  }\n\n  public void setContainerEnvironment(@Nullable List<String> environment) {\n    config.Env = environment;\n  }\n\n  public void setContainerEntrypoint(@Nullable List<String> command) {\n    config.Entrypoint = command;\n  }\n\n  public void setContainerCmd(@Nullable List<String> cmd) {\n    config.Cmd = cmd;\n  }\n\n  /**\n   * Sets test on HealthCheck, creates an empty HealthCheck object if necessary.\n   *\n   * @param test the list of tests to set\n   */\n  public void setContainerHealthCheckTest(List<String> test) {\n    if (config.Healthcheck == null) {\n      config.Healthcheck = new HealthCheckObjectTemplate();\n    }\n    Preconditions.checkNotNull(config.Healthcheck).Test = test;\n  }\n\n  /**\n   * Sets interval on HealthCheck, creates an empty HealthCheck object if necessary.\n   *\n   * @param interval the interval to set\n   */\n  public void setContainerHealthCheckInterval(@Nullable Long interval) {\n    if (config.Healthcheck == null) {\n      config.Healthcheck = new HealthCheckObjectTemplate();\n    }\n    Preconditions.checkNotNull(config.Healthcheck).Interval = interval;\n  }\n\n  /**\n   * Sets timeout on HealthCheck, creates an empty HealthCheck object if necessary.\n   *\n   * @param timeout the timeout to configure\n   */\n  public void setContainerHealthCheckTimeout(@Nullable Long timeout) {\n    if (config.Healthcheck == null) {\n      config.Healthcheck = new HealthCheckObjectTemplate();\n    }\n    Preconditions.checkNotNull(config.Healthcheck).Timeout = timeout;\n  }\n\n  /**\n   * Sets startPeriod on HealthCheck, creates an empty HealthCheck object if necessary.\n   *\n   * @param startPeriod the start period to configure\n   */\n  public void setContainerHealthCheckStartPeriod(@Nullable Long startPeriod) {\n    if (config.Healthcheck == null) {\n      config.Healthcheck = new HealthCheckObjectTemplate();\n    }\n    Preconditions.checkNotNull(config.Healthcheck).StartPeriod = startPeriod;\n  }\n\n  /**\n   * Sets retries on HealthCheck, creates an empty HealthCheck object if necessary.\n   *\n   * @param retries the number of retries to configure\n   */\n  public void setContainerHealthCheckRetries(@Nullable Integer retries) {\n    if (config.Healthcheck == null) {\n      config.Healthcheck = new HealthCheckObjectTemplate();\n    }\n    Preconditions.checkNotNull(config.Healthcheck).Retries = retries;\n  }\n\n  public void setContainerExposedPorts(@Nullable Map<String, Map<String, String>> exposedPorts) {\n    config.ExposedPorts = exposedPorts;\n  }\n\n  public void setContainerLabels(@Nullable Map<String, String> labels) {\n    config.Labels = labels;\n  }\n\n  public void setContainerWorkingDir(@Nullable String workingDirectory) {\n    config.WorkingDir = workingDirectory;\n  }\n\n  public void setContainerUser(@Nullable String user) {\n    config.User = user;\n  }\n\n  public void setContainerVolumes(@Nullable Map<String, Map<String, String>> volumes) {\n    config.Volumes = volumes;\n  }\n\n  public void addLayerDiffId(DescriptorDigest diffId) {\n    rootfs.diff_ids.add(diffId);\n  }\n\n  public void addHistoryEntry(HistoryEntry historyEntry) {\n    history.add(historyEntry);\n  }\n\n  List<DescriptorDigest> getDiffIds() {\n    return rootfs.diff_ids;\n  }\n\n  List<HistoryEntry> getHistory() {\n    return history;\n  }\n\n  @Nullable\n  String getCreated() {\n    return created;\n  }\n\n  /**\n   * Returns the architecture for which this container was built. See the <a\n   * href=\"https://github.com/opencontainers/image-spec/blob/master/config.md#properties\">OCI Image\n   * Configuration specification</a> for acceptable values.\n   *\n   * @return the {@code architecture} field\n   */\n  public String getArchitecture() {\n    return architecture;\n  }\n\n  /**\n   * Returns the operating system for which this container was built. See the <a\n   * href=\"https://github.com/opencontainers/image-spec/blob/master/config.md#properties\">OCI Image\n   * Configuration specification</a> for acceptable values.\n   *\n   * @return the {@code os} field\n   */\n  public String getOs() {\n    return os;\n  }\n\n  @Nullable\n  List<String> getContainerEnvironment() {\n    return config.Env;\n  }\n\n  @Nullable\n  List<String> getContainerEntrypoint() {\n    return config.Entrypoint;\n  }\n\n  @Nullable\n  List<String> getContainerCmd() {\n    return config.Cmd;\n  }\n\n  @Nullable\n  List<String> getContainerHealthTest() {\n    return config.Healthcheck == null ? null : config.Healthcheck.Test;\n  }\n\n  @Nullable\n  Long getContainerHealthInterval() {\n    return config.Healthcheck == null ? null : config.Healthcheck.Interval;\n  }\n\n  @Nullable\n  Long getContainerHealthTimeout() {\n    return config.Healthcheck == null ? null : config.Healthcheck.Timeout;\n  }\n\n  @Nullable\n  Long getContainerHealthStartPeriod() {\n    return config.Healthcheck == null ? null : config.Healthcheck.StartPeriod;\n  }\n\n  @Nullable\n  Integer getContainerHealthRetries() {\n    return config.Healthcheck == null ? null : config.Healthcheck.Retries;\n  }\n\n  @Nullable\n  Map<String, Map<String, String>> getContainerExposedPorts() {\n    return config.ExposedPorts;\n  }\n\n  @Nullable\n  Map<String, String> getContainerLabels() {\n    return config.Labels;\n  }\n\n  @Nullable\n  String getContainerWorkingDir() {\n    return config.WorkingDir;\n  }\n\n  @Nullable\n  String getContainerUser() {\n    return config.User;\n  }\n\n  @Nullable\n  Map<String, Map<String, String>> getContainerVolumes() {\n    return config.Volumes;\n  }\n\n  public DescriptorDigest getLayerDiffId(int index) {\n    return rootfs.diff_ids.get(index);\n  }\n\n  public int getLayerCount() {\n    return rootfs.diff_ids.size();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageMetadataTemplate.java\n/*\n * Copyright 2020 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonSubTypes;\nimport com.fasterxml.jackson.annotation.JsonTypeInfo;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport java.util.ArrayList;\nimport java.util.List;\nimport javax.annotation.Nullable;\n\n/** A bundle of an image manifest list, manifests, and container configurations. */\npublic class ImageMetadataTemplate implements JsonTemplate {\n\n  @JsonTypeInfo(\n      use = JsonTypeInfo.Id.CLASS,\n      include = JsonTypeInfo.As.PROPERTY,\n      property = \"@class\")\n  @JsonSubTypes({\n    @JsonSubTypes.Type(value = OciIndexTemplate.class),\n    @JsonSubTypes.Type(value = V22ManifestListTemplate.class),\n  })\n  @Nullable\n  private ManifestTemplate manifestList;\n\n  private List<ManifestAndConfigTemplate> manifestsAndConfigs = new ArrayList<>();\n\n  @SuppressWarnings(\"unused\")\n  private ImageMetadataTemplate() {}\n\n  public ImageMetadataTemplate(\n      @Nullable ManifestTemplate manifestList,\n      List<ManifestAndConfigTemplate> manifestsAndConfigs) {\n    this.manifestList = manifestList;\n    this.manifestsAndConfigs = manifestsAndConfigs;\n  }\n\n  @Nullable\n  public ManifestTemplate getManifestList() {\n    return manifestList;\n  }\n\n  public List<ManifestAndConfigTemplate> getManifestsAndConfigs() {\n    return manifestsAndConfigs;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestAndConfigTemplate.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonSubTypes;\nimport com.fasterxml.jackson.annotation.JsonTypeInfo;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport javax.annotation.Nullable;\n\n/** Stores a manifest and container config. */\npublic class ManifestAndConfigTemplate implements JsonTemplate {\n\n  @Nullable private String manifestDigest;\n\n  @JsonTypeInfo(\n      use = JsonTypeInfo.Id.CLASS,\n      include = JsonTypeInfo.As.PROPERTY,\n      property = \"@class\")\n  @JsonSubTypes({\n    @JsonSubTypes.Type(value = OciManifestTemplate.class),\n    @JsonSubTypes.Type(value = V21ManifestTemplate.class),\n    @JsonSubTypes.Type(value = V22ManifestTemplate.class),\n  })\n  @Nullable\n  private ManifestTemplate manifest;\n\n  @Nullable private ContainerConfigurationTemplate config;\n\n  @SuppressWarnings(\"unused\")\n  private ManifestAndConfigTemplate() {}\n\n  /**\n   * Creates an instance.\n   *\n   * @param manifest the image manifest\n   * @param config the container configuration\n   */\n  public ManifestAndConfigTemplate(\n      // TODO: switch to BuildableManifestTemplate after we stop supporting V21 manifest.\n      ManifestTemplate manifest,\n      // TODO: remove @Nullable after we stop supporting V21 manifest.\n      @Nullable ContainerConfigurationTemplate config) {\n    this(manifest, config, null);\n  }\n\n  /**\n   * Creates an instance.\n   *\n   * @param manifest the image manifest\n   * @param config the container configuration\n   * @param manifestDigest the digest of the manifest\n   */\n  public ManifestAndConfigTemplate(\n      // TODO: switch to BuildableManifestTemplate after we stop supporting V21 manifest.\n      ManifestTemplate manifest,\n      // TODO: remove @Nullable after we stop supporting V21 manifest.\n      @Nullable ContainerConfigurationTemplate config,\n      @Nullable String manifestDigest) {\n    this.manifest = manifest;\n    this.config = config;\n    this.manifestDigest = manifestDigest;\n  }\n\n  /**\n   * Gets the digest of the manifest.\n   *\n   * @return the digest\n   */\n  @Nullable\n  public String getManifestDigest() {\n    return manifestDigest;\n  }\n\n  /**\n   * Gets the manifest.\n   *\n   * @return the manifest\n   */\n  @Nullable\n  public ManifestTemplate getManifest() {\n    return manifest;\n  }\n\n  /**\n   * Gets the container configuration.\n   *\n   * @return the container configuration\n   */\n  @Nullable\n  public ContainerConfigurationTemplate getConfig() {\n    return config;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestTemplate.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\n\n/** Parent class for image manifest and manifest list JSON templates. */\n@JsonIgnoreProperties(ignoreUnknown = true)\npublic interface ManifestTemplate extends JsonTemplate {\n\n  int getSchemaVersion();\n\n  String getManifestMediaType();\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V21ManifestTemplate.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Optional;\nimport javax.annotation.Nullable;\n\n/**\n * JSON template for Docker Manifest Schema V2.1\n *\n * <p>This is only for parsing manifests in the older V2.1 schema. Generated manifests should be in\n * the V2.2 schema using the {@link V22ManifestTemplate}.\n *\n * <p>Example manifest JSON (only the {@code fsLayers} and {@code history} fields are relevant for\n * parsing):\n *\n * <pre>{@code\n * {\n *   ...\n *   \"fsLayers\": {\n *     {\n *       \"blobSum\": \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\"\n *     },\n *     {\n *       \"blobSum\": \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\"\n *     }\n *   },\n *   \"history\": [\n *     {\n *       \"v1Compatibility\": \"<some manifest V1 JSON object>\"\n *     }\n *   ]\n *   ...\n * }\n * }</pre>\n *\n * @see <a href=\"https://docs.docker.com/registry/spec/manifest-v2-1/\">Image Manifest Version 2,\n *     Schema 1</a>\n */\npublic class V21ManifestTemplate implements ManifestTemplate {\n\n  public static final String MEDIA_TYPE = \"application/vnd.docker.distribution.manifest.v1+json\";\n\n  private final int schemaVersion = 1;\n  private final String mediaType = MEDIA_TYPE;\n\n  /** The list of layer references. */\n  private final List<LayerObjectTemplate> fsLayers = new ArrayList<>();\n\n  private final List<HistoryObjectTemplate> history = new ArrayList<>();\n\n  /**\n   * Template for inner JSON object representing a layer as part of the list of layer references.\n   */\n  @VisibleForTesting\n  static class LayerObjectTemplate implements JsonTemplate {\n\n    @Nullable private DescriptorDigest blobSum;\n\n    @Nullable\n    DescriptorDigest getDigest() {\n      return blobSum;\n    }\n  }\n\n  /** Template for inner JSON object representing history for a layer. */\n  private static class HistoryObjectTemplate implements JsonTemplate {\n\n    // The value is basically free-form; they may be structured differently in practice, e.g.,\n    // {\"architecture\": \"amd64\", \"config\": {\"User\": \"1001\", ...}, \"parent\": ...}\n    // {\"id\": ..., \"container_config\": {\"Cmd\":[\"\"]}}\n    @Nullable private String v1Compatibility;\n  }\n\n  /**\n   * Returns a list of descriptor digests for the layers in the image.\n   *\n   * @return a list of descriptor digests for the layers in the image.\n   */\n  public List<DescriptorDigest> getLayerDigests() {\n    List<DescriptorDigest> layerDigests = new ArrayList<>();\n\n    for (LayerObjectTemplate layerObjectTemplate : fsLayers) {\n      layerDigests.add(layerObjectTemplate.blobSum);\n    }\n\n    return layerDigests;\n  }\n\n  @Override\n  public int getSchemaVersion() {\n    return schemaVersion;\n  }\n\n  @Override\n  public String getManifestMediaType() {\n    return mediaType;\n  }\n\n  public List<LayerObjectTemplate> getFsLayers() {\n    return Collections.unmodifiableList(fsLayers);\n  }\n\n  /**\n   * Attempts to parse the container configuration JSON (of format {@code\n   * application/vnd.docker.container.image.v1+json}) from the {@code v1Compatibility} value of the\n   * first {@code history} entry, which corresponds to the latest layer.\n   *\n   * @return container configuration if the first history string holds it; {@code null} otherwise\n   */\n  public Optional<ContainerConfigurationTemplate> getContainerConfiguration() {\n    try {\n      if (history.isEmpty()) {\n        return Optional.empty();\n      }\n      String v1Compatibility = history.get(0).v1Compatibility;\n      if (v1Compatibility == null) {\n        return Optional.empty();\n      }\n\n      return Optional.of(\n          JsonTemplateMapper.readJson(v1Compatibility, ContainerConfigurationTemplate.class));\n    } catch (IOException ex) {\n      // not a container configuration; ignore and continue\n      return Optional.empty();\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.cache;\n\nimport com.google.cloud.tools.jib.api.CacheDirectoryCreationException;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.api.buildplan.FileEntry;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.V21ManifestTemplate;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Optional;\nimport javax.annotation.concurrent.Immutable;\n\n/**\n * Cache for storing data to be shared between Jib executions.\n *\n * <p>This class is immutable and safe to use across threads.\n */\n@Immutable\npublic class Cache {\n\n  /**\n   * Initializes the cache using {@code cacheDirectory} for storage.\n   *\n   * @param cacheDirectory the directory for the cache. Creates the directory if it does not exist.\n   * @return a new {@link Cache}\n   * @throws CacheDirectoryCreationException if an I/O exception occurs when creating cache\n   *     directory\n   */\n  public static Cache withDirectory(Path cacheDirectory) throws CacheDirectoryCreationException {\n    try {\n      Files.createDirectories(cacheDirectory);\n    } catch (IOException ex) {\n      throw new CacheDirectoryCreationException(ex);\n    }\n    return new Cache(new CacheStorageFiles(cacheDirectory));\n  }\n\n  private final CacheStorageWriter cacheStorageWriter;\n  private final CacheStorageReader cacheStorageReader;\n\n  private Cache(CacheStorageFiles cacheStorageFiles) {\n    cacheStorageWriter = new CacheStorageWriter(cacheStorageFiles);\n    cacheStorageReader = new CacheStorageReader(cacheStorageFiles);\n  }\n\n  /**\n   * Saves image metadata (a manifest list and a list of manifest/container configuration pairs) for\n   * an image reference.\n   *\n   * @param imageReference the image reference to save the metadata for\n   * @param metadata the image metadata\n   * @throws IOException if an I/O exception occurs\n   */\n  public void writeMetadata(ImageReference imageReference, ImageMetadataTemplate metadata)\n      throws IOException {\n    cacheStorageWriter.writeMetadata(imageReference, metadata);\n  }\n\n  /**\n   * Saves a schema 2 manifest for an image reference. This is a simple wrapper around {@link\n   * #writeMetadata(ImageReference, ImageMetadataTemplate)} to save a single manifest without a\n   * manifest list.\n   *\n   * @param imageReference the image reference to save the manifest for\n   * @param manifest the V2.2 or OCI manifest\n   * @param containerConfiguration the container configuration\n   * @throws IOException if an I/O exception occurs\n   */\n  public void writeMetadata(\n      ImageReference imageReference,\n      BuildableManifestTemplate manifest,\n      ContainerConfigurationTemplate containerConfiguration)\n      throws IOException {\n    List<ManifestAndConfigTemplate> singleton =\n        Collections.singletonList(new ManifestAndConfigTemplate(manifest, containerConfiguration));\n    cacheStorageWriter.writeMetadata(imageReference, new ImageMetadataTemplate(null, singleton));\n  }\n\n  /**\n   * Saves a V2.1 image manifest. This is a simple wrapper around {@link\n   * #writeMetadata(ImageReference, ImageMetadataTemplate)} to save a single manifest without a\n   * manifest list.\n   *\n   * @param imageReference the image reference to save the manifest for\n   * @param manifestTemplate the V2.1 manifest\n   * @throws IOException if an I/O exception occurs\n   */\n  public void writeMetadata(ImageReference imageReference, V21ManifestTemplate manifestTemplate)\n      throws IOException {\n    List<ManifestAndConfigTemplate> singleton =\n        Collections.singletonList(new ManifestAndConfigTemplate(manifestTemplate, null));\n    cacheStorageWriter.writeMetadata(imageReference, new ImageMetadataTemplate(null, singleton));\n  }\n\n  /**\n   * Saves a cache entry with a compressed layer {@link Blob}. Use {@link\n   * #writeUncompressedLayer(Blob, ImmutableList)} to save a cache entry with an uncompressed layer\n   * {@link Blob} and include a selector.\n   *\n   * @param compressedLayerBlob the compressed layer {@link Blob}\n   * @return the {@link CachedLayer} for the written layer\n   * @throws IOException if an I/O exception occurs\n   */\n  public CachedLayer writeCompressedLayer(Blob compressedLayerBlob) throws IOException {\n    return cacheStorageWriter.writeCompressed(compressedLayerBlob);\n  }\n\n  /**\n   * Saves a cache entry with an uncompressed layer {@link Blob} and an additional selector digest.\n   * Use {@link #writeCompressedLayer(Blob)} to save a compressed layer {@link Blob}.\n   *\n   * @param uncompressedLayerBlob the layer {@link Blob}\n   * @param layerEntries the layer entries that make up the layer\n   * @return the {@link CachedLayer} for the written layer\n   * @throws IOException if an I/O exception occurs\n   */\n  public CachedLayer writeUncompressedLayer(\n      Blob uncompressedLayerBlob, ImmutableList<FileEntry> layerEntries) throws IOException {\n    return cacheStorageWriter.writeUncompressed(\n...\n", "needle_token_start": 2412, "needle_token_end": 2439, "code_context_ntokens": 15935, "output": ["```java\n/**\n * Returns {@code true} if {@code tag} is the default tag ({@code latest}); {@code false} if not.\n *\n * @param tag the tag to check\n * @return {@code true} if {@code tag} is the default tag ({@code latest}); {@code false} if not\n */\npublic static boolean isDefaultTag(@Nullable String tag) {\n    return DEFAULT_TAG.equals(tag);\n}\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "withDirectory", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To initialize a cache storage system using a specified directory, ensuring the directory exists or creating it if it does not.\n2. **Input**: A path to the desired directory for cache storage.\n3. **Output**: An instance of a cache management class configured to use the specified directory.\n4. **Procedure**: The method checks if the directory exists at the provided path and creates it if it does not. If an error occurs during the directory creation, an exception specific to cache directory creation issues is thrown. Upon successful directory setup, a new instance of the cache management class is returned, linked to the newly ensured directory.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageMetadataTemplate.java\n/*\n * Copyright 2020 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonSubTypes;\nimport com.fasterxml.jackson.annotation.JsonTypeInfo;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport java.util.ArrayList;\nimport java.util.List;\nimport javax.annotation.Nullable;\n\n/** A bundle of an image manifest list, manifests, and container configurations. */\npublic class ImageMetadataTemplate implements JsonTemplate {\n\n  @JsonTypeInfo(\n      use = JsonTypeInfo.Id.CLASS,\n      include = JsonTypeInfo.As.PROPERTY,\n      property = \"@class\")\n  @JsonSubTypes({\n    @JsonSubTypes.Type(value = OciIndexTemplate.class),\n    @JsonSubTypes.Type(value = V22ManifestListTemplate.class),\n  })\n  @Nullable\n  private ManifestTemplate manifestList;\n\n  private List<ManifestAndConfigTemplate> manifestsAndConfigs = new ArrayList<>();\n\n  @SuppressWarnings(\"unused\")\n  private ImageMetadataTemplate() {}\n\n  public ImageMetadataTemplate(\n      @Nullable ManifestTemplate manifestList,\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestAndConfigTemplate.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonSubTypes;\nimport com.fasterxml.jackson.annotation.JsonTypeInfo;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport javax.annotation.Nullable;\n\n/** Stores a manifest and container config. */\npublic class ManifestAndConfigTemplate implements JsonTemplate {\n\n  @Nullable private String manifestDigest;\n\n  @JsonTypeInfo(\n      use = JsonTypeInfo.Id.CLASS,\n      include = JsonTypeInfo.As.PROPERTY,\n      property = \"@class\")\n  @JsonSubTypes({\n    @JsonSubTypes.Type(value = OciManifestTemplate.class),\n    @JsonSubTypes.Type(value = V21ManifestTemplate.class),\n    @JsonSubTypes.Type(value = V22ManifestTemplate.class),\n  })\n  @Nullable\n  private ManifestTemplate manifest;\n\n  @Nullable private ContainerConfigurationTemplate config;\n\n  @SuppressWarnings(\"unused\")\n  private ManifestAndConfigTemplate() {}\n\n  /**\n   * Creates an instance.\n   *\n   * @param manifest the image manifest\n   * @param config the container configuration\n   */\n  public ManifestAndConfigTemplate(\n      // TODO: switch to BuildableManifestTemplate after we stop supporting V21 manifest.\n      ManifestTemplate manifest,\n      // TODO: remove @Nullable after we stop supporting V21 manifest.\n      @Nullable ContainerConfigurationTemplate config) {\n    this(manifest, config, null);\n  }\n\n  /**\n   * Creates an instance.\n   *\n   * @param manifest the image manifest\n   * @param config the container configuration\n   * @param manifestDigest the digest of the manifest\n   */\n  public ManifestAndConfigTemplate(\n      // TODO: switch to BuildableManifestTemplate after we stop supporting V21 manifest.\n      ManifestTemplate manifest,\n      // TODO: remove @Nullable after we stop supporting V21 manifest.\n      @Nullable ContainerConfigurationTemplate config,\n      @Nullable String manifestDigest) {\n    this.manifest = manifest;\n    this.config = config;\n    this.manifestDigest = manifestDigest;\n  }\n\n  /**\n   * Gets the digest of the manifest.\n   *\n   * @return the digest\n   */\n  @Nullable\n  public String getManifestDigest() {\n    return manifestDigest;\n  }\n\n  /**\n   * Gets the manifest.\n   *\n   * @return the manifest\n   */\n  @Nullable\n  public ManifestTemplate getManifest() {\n    return manifest;\n  }\n\n  /**\n   * Gets the container configuration.\n   *\n   * @return the container configuration\n   */\n  @Nullable\n  public ContainerConfigurationTemplate getConfig() {\n    return config;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestTemplate.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\n\n/** Parent class for image manifest and manifest list JSON templates. */\n@JsonIgnoreProperties(ignoreUnknown = true)\npublic interface ManifestTemplate extends JsonTemplate {\n\n  int getSchemaVersion();\n\n  String getManifestMediaType();\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V21ManifestTemplate.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Optional;\nimport javax.annotation.Nullable;\n\n/**\n * JSON template for Docker Manifest Schema V2.1\n *\n * <p>This is only for parsing manifests in the older V2.1 schema. Generated manifests should be in\n * the V2.2 schema using the {@link V22ManifestTemplate}.\n *\n * <p>Example manifest JSON (only the {@code fsLayers} and {@code history} fields are relevant for\n * parsing):\n *\n * <pre>{@code\n * {\n *   ...\n *   \"fsLayers\": {\n *     {\n *       \"blobSum\": \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\"\n *     },\n *     {\n *       \"blobSum\": \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\"\n *     }\n *   },\n *   \"history\": [\n *     {\n *       \"v1Compatibility\": \"<some manifest V1 JSON object>\"\n *     }\n *   ]\n *   ...\n * }\n * }</pre>\n *\n * @see <a href=\"https://docs.docker.com/registry/spec/manifest-v2-1/\">Image Manifest Version 2,\n *     Schema 1</a>\n */\npublic class V21ManifestTemplate implements ManifestTemplate {\n\n  public static final String MEDIA_TYPE = \"application/vnd.docker.distribution.manifest.v1+json\";\n\n  private final int schemaVersion = 1;\n  private final String mediaType = MEDIA_TYPE;\n\n  /** The list of layer references. */\n  private final List<LayerObjectTemplate> fsLayers = new ArrayList<>();\n\n  private final List<HistoryObjectTemplate> history = new ArrayList<>();\n\n  /**\n   * Template for inner JSON object representing a layer as part of the list of layer references.\n   */\n  @VisibleForTesting\n  static class LayerObjectTemplate implements JsonTemplate {\n\n    @Nullable private DescriptorDigest blobSum;\n\n    @Nullable\n    DescriptorDigest getDigest() {\n      return blobSum;\n    }\n  }\n\n  /** Template for inner JSON object representing history for a layer. */\n  private static class HistoryObjectTemplate implements JsonTemplate {\n\n    // The value is basically free-form; they may be structured differently in practice, e.g.,\n    // {\"architecture\": \"amd64\", \"config\": {\"User\": \"1001\", ...}, \"parent\": ...}\n    // {\"id\": ..., \"container_config\": {\"Cmd\":[\"\"]}}\n    @Nullable private String v1Compatibility;\n  }\n\n  /**\n   * Returns a list of descriptor digests for the layers in the image.\n   *\n   * @return a list of descriptor digests for the layers in the image.\n   */\n  public List<DescriptorDigest> getLayerDigests() {\n    List<DescriptorDigest> layerDigests = new ArrayList<>();\n\n    for (LayerObjectTemplate layerObjectTemplate : fsLayers) {\n      layerDigests.add(layerObjectTemplate.blobSum);\n    }\n\n    return layerDigests;\n  }\n\n  @Override\n  public int getSchemaVersion() {\n    return schemaVersion;\n  }\n\n  @Override\n  public String getManifestMediaType() {\n    return mediaType;\n  }\n\n  public List<LayerObjectTemplate> getFsLayers() {\n    return Collections.unmodifiableList(fsLayers);\n  }\n\n  /**\n   * Attempts to parse the container configuration JSON (of format {@code\n   * application/vnd.docker.container.image.v1+json}) from the {@code v1Compatibility} value of the\n   * first {@code history} entry, which corresponds to the latest layer.\n   *\n   * @return container configuration if the first history string holds it; {@code null} otherwise\n   */\n  public Optional<ContainerConfigurationTemplate> getContainerConfiguration() {\n    try {\n      if (history.isEmpty()) {\n        return Optional.empty();\n      }\n      String v1Compatibility = history.get(0).v1Compatibility;\n      if (v1Compatibility == null) {\n        return Optional.empty();\n      }\n\n      return Optional.of(\n          JsonTemplateMapper.readJson(v1Compatibility, ContainerConfigurationTemplate.class));\n    } catch (IOException ex) {\n      // not a container configuration; ignore and continue\n      return Optional.empty();\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.cache;\n\nimport com.google.cloud.tools.jib.api.CacheDirectoryCreationException;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.api.buildplan.FileEntry;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.V21ManifestTemplate;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Optional;\nimport javax.annotation.concurrent.Immutable;\n\n/**\n * Cache for storing data to be shared between Jib executions.\n *\n * <p>This class is immutable and safe to use across threads.\n */\n@Immutable\npublic class Cache {\n\n  /**\n   * Initializes the cache using {@code cacheDirectory} for storage.\n   *\n   * @param cacheDirectory the directory for the cache. Creates the directory if it does not exist.\n   * @return a new {@link Cache}\n   * @throws CacheDirectoryCreationException if an I/O exception occurs when creating cache\n   *     directory\n   */\n  \npublic static Cache withDirectory(Path cacheDirectory) throws CacheDirectoryCreationException {\n    try {\n      Files.createDirectories(cacheDirectory);\n    } catch (IOException ex) {\n      throw new CacheDirectoryCreationException(ex);\n    }\n    return new Cache(new CacheStorageFiles(cacheDirectory));\n  }\n\n  private final CacheStorageWriter cacheStorageWriter;\n  private final CacheStorageReader cacheStorageReader;\n\n  private Cache(CacheStorageFiles cacheStorageFiles) {\n    cacheStorageWriter = new CacheStorageWriter(cacheStorageFiles);\n    cacheStorageReader = new CacheStorageReader(cacheStorageFiles);\n  }\n\n  /**\n   * Saves image metadata (a manifest list and a list of manifest/container configuration pairs) for\n   * an image reference.\n   *\n   * @param imageReference the image reference to save the metadata for\n   * @param metadata the image metadata\n   * @throws IOException if an I/O exception occurs\n   */\n  public void writeMetadata(ImageReference imageReference, ImageMetadataTemplate metadata)\n      throws IOException {\n    cacheStorageWriter.writeMetadata(imageReference, metadata);\n  }\n\n  /**\n   * Saves a schema 2 manifest for an image reference. This is a simple wrapper around {@link\n   * #writeMetadata(ImageReference, ImageMetadataTemplate)} to save a single manifest without a\n   * manifest list.\n   *\n   * @param imageReference the image reference to save the manifest for\n   * @param manifest the V2.2 or OCI manifest\n   * @param containerConfiguration the container configuration\n   * @throws IOException if an I/O exception occurs\n   */\n  public void writeMetadata(\n      ImageReference imageReference,\n      BuildableManifestTemplate manifest,\n      ContainerConfigurationTemplate containerConfiguration)\n      throws IOException {\n    List<ManifestAndConfigTemplate> singleton =\n        Collections.singletonList(new ManifestAndConfigTemplate(manifest, containerConfiguration));\n    cacheStorageWriter.writeMetadata(imageReference, new ImageMetadataTemplate(null, singleton));\n  }\n\n  /**\n   * Saves a V2.1 image manifest. This is a simple wrapper around {@link\n   * #writeMetadata(ImageReference, ImageMetadataTemplate)} to save a single manifest without a\n   * manifest list.\n   *\n   * @param imageReference the image reference to save the manifest for\n   * @param manifestTemplate the V2.1 manifest\n   * @throws IOException if an I/O exception occurs\n   */\n  public void writeMetadata(ImageReference imageReference, V21ManifestTemplate manifestTemplate)\n      throws IOException {\n    List<ManifestAndConfigTemplate> singleton =\n        Collections.singletonList(new ManifestAndConfigTemplate(manifestTemplate, null));\n    cacheStorageWriter.writeMetadata(imageReference, new ImageMetadataTemplate(null, singleton));\n  }\n\n  /**\n   * Saves a cache entry with a compressed layer {@link Blob}. Use {@link\n   * #writeUncompressedLayer(Blob, ImmutableList)} to save a cache entry with an uncompressed layer\n   * {@link Blob} and include a selector.\n   *\n   * @param compressedLayerBlob the compressed layer {@link Blob}\n   * @return the {@link CachedLayer} for the written layer\n   * @throws IOException if an I/O exception occurs\n   */\n  public CachedLayer writeCompressedLayer(Blob compressedLayerBlob) throws IOException {\n    return cacheStorageWriter.writeCompressed(compressedLayerBlob);\n  }\n\n  /**\n   * Saves a cache entry with an uncompressed layer {@link Blob} and an additional selector digest.\n   * Use {@link #writeCompressedLayer(Blob)} to save a compressed layer {@link Blob}.\n   *\n   * @param uncompressedLayerBlob the layer {@link Blob}\n   * @param layerEntries the layer entries that make up the layer\n   * @return the {@link CachedLayer} for the written layer\n   * @throws IOException if an I/O exception occurs\n   */\n  public CachedLayer writeUncompressedLayer(\n      Blob uncompressedLayerBlob, ImmutableList<FileEntry> layerEntries) throws IOException {\n    return cacheStorageWriter.writeUncompressed(\n        uncompressedLayerBlob, LayerEntriesSelector.generateSelector(layerEntries));\n  }\n\n  /**\n   * Caches a layer that was extracted from a local base image, and names the file using the\n   * provided diff id.\n   *\n   * @param diffId the diff id\n   * @param compressedBlob the compressed layer blob\n   * @return the {@link CachedLayer} for the written layer\n   * @throws IOException if an I/O exception occurs\n   */\n  public CachedLayer writeTarLayer(DescriptorDigest diffId, Blob compressedBlob)\n      throws IOException {\n    return cacheStorageWriter.writeTarLayer(diffId, compressedBlob);\n  }\n\n  /**\n   * Writes a container configuration to {@code (cache directory)/local/config/(image id)}. An image\n   * ID is a SHA hash of a container configuration JSON. The value is also shown as IMAGE ID in\n   * {@code docker images}.\n   *\n   * <p>Note: the {@code imageId} to the {@code containerConfiguration} is a one-way relationship;\n   * there is no guarantee that {@code containerConfiguration}'s SHA will be {@code imageId}, since\n   * the original container configuration is being rewritten here rather than being moved.\n   *\n   * @param imageId the ID of the image to store the container configuration for\n   * @param containerConfiguration the container configuration\n   * @throws IOException if an I/O exception occurs\n   */\n  public void writeLocalConfig(\n      DescriptorDigest imageId, ContainerConfigurationTemplate containerConfiguration)\n      throws IOException {\n    cacheStorageWriter.writeLocalConfig(imageId, containerConfiguration);\n  }\n\n  /**\n   * Retrieves the cached image metadata (a manifest list and a list of manifest/container\n   * configuration pairs) for an image reference.\n   *\n   * @param imageReference the image reference\n   * @return the image metadata for the image reference, if found\n   * @throws IOException if an I/O exception occurs\n   * @throws CacheCorruptedException if the cache is corrupted\n   */\n  public Optional<ImageMetadataTemplate> retrieveMetadata(ImageReference imageReference)\n      throws IOException, CacheCorruptedException {\n    return cacheStorageReader.retrieveMetadata(imageReference);\n  }\n\n  /**\n   * Returns {@code true} if all image layers described in a manifest exist in the cache.\n   *\n   * @param manifest the image manifest\n   * @return a boolean\n   */\n  public boolean areAllLayersCached(ManifestTemplate manifest) {\n    return cacheStorageReader.areAllLayersCached(manifest);\n  }\n\n  /**\n   * Retrieves the {@link CachedLayer} that was built from the {@code layerEntries}.\n   *\n   * @param layerEntries the layer entries to match against\n   * @return a {@link CachedLayer} that was built from {@code layerEntries}, if found\n   * @throws IOException if an I/O exception occurs\n   * @throws CacheCorruptedException if the cache is corrupted\n   */\n  public Optional<CachedLayer> retrieve(ImmutableList<FileEntry> layerEntries)\n      throws IOException, CacheCorruptedException {\n    Optional<DescriptorDigest> optionalSelectedLayerDigest =\n        cacheStorageReader.select(LayerEntriesSelector.generateSelector(layerEntries));\n    if (!optionalSelectedLayerDigest.isPresent()) {\n      return Optional.empty();\n    }\n\n    return cacheStorageReader.retrieve(optionalSelectedLayerDigest.get());\n  }\n\n  /**\n   * Retrieves the {@link CachedLayer} for the layer with digest {@code layerDigest}.\n   *\n   * @param layerDigest the layer digest\n   * @return the {@link CachedLayer} referenced by the layer digest, if found\n   * @throws CacheCorruptedException if the cache was found to be corrupted\n   * @throws IOException if an I/O exception occurs\n   */\n  public Optional<CachedLayer> retrieve(DescriptorDigest layerDigest)\n      throws IOException, CacheCorruptedException {\n    return cacheStorageReader.retrieve(layerDigest);\n  }\n\n  /**\n   * Retrieves a {@link CachedLayer} for a local base image layer with the given diff id.\n   *\n   * @param diffId the diff id\n   * @return the {@link CachedLayer} with the given diff id\n   * @throws CacheCorruptedException if the cache was found to be corrupted\n   * @throws IOException if an I/O exception occurs\n   */\n  public Optional<CachedLayer> retrieveTarLayer(DescriptorDigest diffId)\n      throws IOException, CacheCorruptedException {\n    return cacheStorageReader.retrieveTarLayer(diffId);\n  }\n\n  /**\n   * Retrieves the {@link ContainerConfigurationTemplate} for the image saved from the given image\n   * ID. An image ID is a SHA hash of a container configuration JSON. The value is also shown as\n   * IMAGE ID in {@code docker images}.\n   *\n   * <p>Note: the {@code imageId} is only used to find the {@code containerConfiguration}, and is\n   * not necessarily the actual SHA of {@code containerConfiguration}. There is no guarantee that\n   * {@code containerConfiguration}'s SHA will be {@code imageId}, since the saved container\n   * configuration is not a direct copy of the base image's original configuration.\n   *\n   * @param imageId the image ID\n   * @return the {@link ContainerConfigurationTemplate} referenced by the image ID, if found\n   * @throws IOException if an I/O exception occurs\n   */\n  public Optional<ContainerConfigurationTemplate> retrieveLocalConfig(DescriptorDigest imageId)\n      throws IOException {\n    return cacheStorageReader.retrieveLocalConfig(imageId);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheCorruptedException.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.cache;\n\nimport java.nio.file.Path;\n\n/** Thrown if the the cache was found to be corrupted. */\npublic class CacheCorruptedException extends Exception {\n\n  CacheCorruptedException(Path cacheDirectory, String message, Throwable cause) {\n    super(\n        message\n            + \". You may need to clear the cache by deleting the '\"\n            + cacheDirectory\n            + \"' directory\",\n        cause);\n  }\n\n  CacheCorruptedException(Path cacheDirectory, String message) {\n    super(\n        message\n            + \". You may need to clear the cache by deleting the '\"\n            + cacheDirectory\n            + \"' directory\");\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/Layer.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\n\n/**\n * Represents a layer in an image. Implementations represent the various types of layers.\n *\n * <p>An image layer consists of:\n *\n * <ul>\n *   <li>Content BLOB\n *   <li>\n *       <ul>\n *         <li>The compressed archive (tarball gzip) of the partial filesystem changeset.\n *       </ul>\n *   <li>Content Digest\n *   <li>\n *       <ul>\n *         <li>The SHA-256 hash of the content BLOB.\n *       </ul>\n *   <li>Content Size\n *   <li>\n *       <ul>\n *         <li>The size (in bytes) of the content BLOB.\n *       </ul>\n *   <li>Diff ID\n *   <li>\n *       <ul>\n *         <li>The SHA-256 hash of the uncompressed archive (tarball) of the partial filesystem\n *             changeset.\n *       </ul>\n * </ul>\n */\npublic interface Layer {\n\n  /**\n   * Returns this layer's contents.\n   *\n   * @return the layer's content BLOB\n   * @throws LayerPropertyNotFoundException if not available\n   */\n  Blob getBlob() throws LayerPropertyNotFoundException;\n\n  // TODO: Remove this\n  /**\n   * Returns this layer's content descriptor.\n   *\n   * @return the layer's content {@link BlobDescriptor}\n   * @throws LayerPropertyNotFoundException if not available\n   */\n  BlobDescriptor getBlobDescriptor() throws LayerPropertyNotFoundException;\n\n  /**\n   * Returns this layer's diff ID.\n   *\n   * @return the layer's diff ID\n   * @throws LayerPropertyNotFoundException if not available\n   */\n  DescriptorDigest getDiffId() throws LayerPropertyNotFoundException;\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.cache;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.image.Layer;\nimport com.google.common.base.Preconditions;\nimport javax.annotation.Nullable;\n\n/** A reference to an image layer that is in the Cache. */\npublic class CachedLayer implements Layer {\n\n  /** Builds a {@link CachedLayer}. */\n  public static class Builder {\n\n    @Nullable private DescriptorDigest layerDigest;\n    @Nullable private DescriptorDigest layerDiffId;\n    private long layerSize = -1;\n    @Nullable private Blob layerBlob;\n\n    private Builder() {}\n\n    public Builder setLayerDigest(DescriptorDigest layerDigest) {\n      this.layerDigest = layerDigest;\n      return this;\n    }\n\n    public Builder setLayerDiffId(DescriptorDigest layerDiffId) {\n      this.layerDiffId = layerDiffId;\n      return this;\n    }\n\n    public Builder setLayerSize(long layerSize) {\n      this.layerSize = layerSize;\n      return this;\n    }\n\n    public Builder setLayerBlob(Blob layerBlob) {\n      this.layerBlob = layerBlob;\n      return this;\n    }\n\n    boolean hasLayerBlob() {\n      return layerBlob != null;\n    }\n\n    /**\n     * Creates a CachedLayer instance.\n     *\n     * @return a new cached layer\n     */\n    public CachedLayer build() {\n      return new CachedLayer(\n          Preconditions.checkNotNull(layerDigest, \"layerDigest required\"),\n          Preconditions.checkNotNull(layerDiffId, \"layerDiffId required\"),\n          layerSize,\n          Preconditions.checkNotNull(layerBlob, \"layerBlob required\"));\n    }\n  }\n\n  /**\n   * Creates a new {@link Builder} for a {@link CachedLayer}.\n   *\n   * @return the new {@link Builder}\n   */\n  public static Builder builder() {\n    return new Builder();\n  }\n\n  private final DescriptorDigest layerDiffId;\n  private final BlobDescriptor blobDescriptor;\n  private final Blob layerBlob;\n\n  private CachedLayer(\n      DescriptorDigest layerDigest, DescriptorDigest layerDiffId, long layerSize, Blob layerBlob) {\n    this.layerDiffId = layerDiffId;\n    this.layerBlob = layerBlob;\n    this.blobDescriptor = new BlobDescriptor(layerSize, layerDigest);\n  }\n\n  public DescriptorDigest getDigest() {\n    return blobDescriptor.getDigest();\n  }\n\n  public long getSize() {\n    return blobDescriptor.getSize();\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() {\n    return layerDiffId;\n  }\n\n  @Override\n  public Blob getBlob() {\n    return layerBlob;\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return blobDescriptor;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.global;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Strings;\nimport com.google.common.collect.Range;\n\n/** Names of system properties defined/used by Jib. */\npublic class JibSystemProperties {\n\n  public static final String UPSTREAM_CLIENT = \"_JIB_UPSTREAM_CLIENT\";\n  private static final String DISABLE_USER_AGENT = \"_JIB_DISABLE_USER_AGENT\";\n\n  @VisibleForTesting public static final String HTTP_TIMEOUT = \"jib.httpTimeout\";\n\n  @VisibleForTesting static final String CROSS_REPOSITORY_BLOB_MOUNTS = \"jib.blobMounts\";\n\n  public static final String SEND_CREDENTIALS_OVER_HTTP = \"sendCredentialsOverHttp\";\n  public static final String SERIALIZE = \"jib.serialize\";\n\n  @VisibleForTesting public static final String SKIP_EXISTING_IMAGES = \"jib.skipExistingImages\";\n\n  /**\n   * Gets the HTTP connection/read timeouts for registry interactions in milliseconds. This is\n   * defined by the {@code jib.httpTimeout} system property. The default value is 20000 if the\n   * system property is not set, and 0 indicates an infinite timeout.\n   *\n   * @return the HTTP connection/read timeouts for registry interactions in milliseconds\n   */\n  public static int getHttpTimeout() {\n    if (Integer.getInteger(HTTP_TIMEOUT) == null) {\n      return 20000;\n    }\n    return Integer.getInteger(HTTP_TIMEOUT);\n  }\n\n  /**\n   * Gets whether or not to use <em>cross-repository blob mounts</em> when uploading image layers\n   * ({@code mount/from}). This is defined by the {@code jib.blobMounts} system property.\n   *\n   * @return {@code true} if {@code mount/from} should be used, {@code false} if not, defaulting to\n   *     {@code true}\n   */\n  public static boolean useCrossRepositoryBlobMounts() {\n    return System.getProperty(CROSS_REPOSITORY_BLOB_MOUNTS) == null\n        || Boolean.getBoolean(CROSS_REPOSITORY_BLOB_MOUNTS);\n  }\n\n  /**\n   * Gets whether or not to serialize Jib's execution. This is defined by the {@code jib.serialize}\n   * system property.\n   *\n   * @return {@code true} if Jib's execution should be serialized, {@code false} if not\n   */\n  public static boolean serializeExecution() {\n    return Boolean.getBoolean(SERIALIZE);\n  }\n\n  /**\n   * Gets whether or not to allow sending authentication information over insecure HTTP connections.\n   * This is defined by the {@code sendCredentialsOverHttp} system property.\n   *\n   * @return {@code true} if authentication information is allowed to be sent over insecure\n   *     connections, {@code false} if not\n   */\n  public static boolean sendCredentialsOverHttp() {\n    return Boolean.getBoolean(SEND_CREDENTIALS_OVER_HTTP);\n  }\n\n  /**\n   * Gets whether or not to enable the User-Agent header. This is defined by the {@code\n   * _JIB_DISABLE_USER_AGENT} system property.\n   *\n   * @return {@code true} if the User-Agent header is enabled, {@code false} if not\n   */\n  public static boolean isUserAgentEnabled() {\n    return Strings.isNullOrEmpty(System.getProperty(DISABLE_USER_AGENT));\n  }\n\n  /**\n   * Checks the {@code jib.httpTimeout} system property for invalid (non-integer or negative)\n   * values.\n   *\n   * @throws NumberFormatException if invalid values\n   */\n  public static void checkHttpTimeoutProperty() throws NumberFormatException {\n    checkNumericSystemProperty(HTTP_TIMEOUT, Range.atLeast(0));\n  }\n\n  /**\n   * Checks if {@code http.proxyPort} and {@code https.proxyPort} system properties are in the\n   * [0..65535] range when set.\n   *\n   * @throws NumberFormatException if invalid values\n   */\n  public static void checkProxyPortProperty() throws NumberFormatException {\n    checkNumericSystemProperty(\"http.proxyPort\", Range.closed(0, 65535));\n    checkNumericSystemProperty(\"https.proxyPort\", Range.closed(0, 65535));\n  }\n\n  /**\n   * Gets whether or not to skip pushing tags to existing images. This is defined by the {@code\n   * jib.skipExistingImages} system property.\n   *\n   * @return {@code true} if Jib should skip pushing tags to existing images, {@code false} if not\n   */\n  public static boolean skipExistingImages() {\n    return Boolean.getBoolean(SKIP_EXISTING_IMAGES);\n  }\n\n  private static void checkNumericSystemProperty(String property, Range<Integer> validRange) {\n    String value = System.getProperty(property);\n    if (value == null) {\n      return;\n    }\n\n    int parsed;\n    try {\n      parsed = Integer.parseInt(value);\n    } catch (NumberFormatException ex) {\n      throw new NumberFormatException(property + \" must be an integer: \" + value);\n    }\n    if (validRange.hasLowerBound() && validRange.lowerEndpoint() > parsed) {\n      throw new NumberFormatException(\n          property + \" cannot be less than \" + validRange.lowerEndpoint() + \": \" + value);\n    } else if (validRange.hasUpperBound() && validRange.upperEndpoint() < parsed) {\n      throw new NumberFormatException(\n          property + \" cannot be greater than \" + validRange.upperEndpoint() + \": \" + value);\n    }\n  }\n\n  private JibSystemProperties() {}\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/http/FailoverHttpClient.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.http;\n\nimport com.google.api.client.http.GenericUrl;\nimport com.google.api.client.http.HttpBackOffIOExceptionHandler;\nimport com.google.api.client.http.HttpHeaders;\nimport com.google.api.client.http.HttpIOExceptionHandler;\nimport com.google.api.client.http.HttpMethods;\nimport com.google.api.client.http.HttpRequest;\nimport com.google.api.client.http.HttpResponseException;\nimport com.google.api.client.http.HttpTransport;\nimport com.google.api.client.http.apache.v2.ApacheHttpTransport;\nimport com.google.api.client.util.ExponentialBackOff;\nimport com.google.api.client.util.SslUtils;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport java.io.IOException;\nimport java.net.ConnectException;\nimport java.net.URL;\nimport java.security.GeneralSecurityException;\nimport java.util.ArrayDeque;\nimport java.util.Deque;\nimport java.util.Optional;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.function.Consumer;\nimport java.util.function.Supplier;\nimport javax.net.ssl.SSLException;\nimport org.apache.http.conn.ssl.NoopHostnameVerifier;\nimport org.apache.http.conn.ssl.SSLConnectionSocketFactory;\nimport org.apache.http.impl.client.HttpClientBuilder;\n\n/**\n * Thread-safe HTTP client that can automatically failover from secure HTTPS to insecure HTTPS or\n * HTTP. Intended to be created once and shared to be called at multiple places. Callers should\n * close the returned {@link Response}.\n *\n * <p>The failover (if enabled) in the following way:\n *\n * <ul>\n *   <li>When a port is provided (for example {@code my-registry:5000/my-repo}):\n *       <ol>\n *         <li>Attempts secure HTTPS on the specified port.\n *         <li>If (1) fails due to {@link SSLException}, re-attempts secure HTTPS on the specified\n *             port but disabling certificate validation.\n *         <li>If (2) fails again due to {@link SSLException}, attempts plain-HTTP on the specified\n *             port.\n *       </ol>\n *   <li>When a port is not provided (for example {@code my-registry/my-repo}):\n *       <ol>\n *         <li>Attempts secure HTTPS on port 443 (default HTTPS port).\n *         <li>If (1) fails due to {@link SSLException}, re-attempts secure HTTPS on port 443 but\n *             disabling certificate validation.\n *         <li>If (2) fails again due to {@link SSLException}, attempts plain-HTTP on port 80\n *             (default HTTP port).\n *         <li>Or, if (1) fails due to non-timeout {@link ConnectException}, attempts plain-HTTP on\n *             port 80.\n *       </ol>\n * </ul>\n *\n * <p>This failover behavior is similar to how the Docker client works:\n * https://docs.docker.com/registry/insecure/#deploy-a-plain-http-registry\n */\npublic class FailoverHttpClient {\n\n  /** Represents failover actions taken. To be recorded in the failover history. */\n  private static enum Failover {\n    NONE, // no failover (secure HTTPS)\n    INSECURE_HTTPS, // HTTPS with certificate validation disabled\n    HTTP // plain HTTP\n  }\n\n  private static boolean isHttpsProtocol(URL url) {\n    return \"https\".equals(url.getProtocol());\n  }\n\n  private static URL toHttp(URL url) {\n    GenericUrl httpUrl = new GenericUrl(url);\n    httpUrl.setScheme(\"http\");\n    return httpUrl.toURL();\n  }\n\n  private static HttpTransport getSecureHttpTransport() {\n    // Do not use NetHttpTransport. It does not process response errors properly.\n    // See https://github.com/google/google-http-java-client/issues/39\n    //\n    // A new ApacheHttpTransport needs to be created for each connection because otherwise HTTP\n    // connection persistence causes the connection to throw NoHttpResponseException.\n    HttpClientBuilder httpClientBuilder =\n        ApacheHttpTransport.newDefaultHttpClientBuilder()\n            // using \"system socket factory\" to enable sending client certificate\n            // https://github.com/GoogleContainerTools/jib/issues/2585\n            .setSSLSocketFactory(SSLConnectionSocketFactory.getSystemSocketFactory());\n    return new ApacheHttpTransport(httpClientBuilder.build());\n  }\n\n  private static HttpTransport getInsecureHttpTransport() {\n    try {\n      HttpClientBuilder httpClientBuilder =\n          ApacheHttpTransport.newDefaultHttpClientBuilder()\n              .setSSLSocketFactory(null) // creates new factory with the SSLContext given below\n              .setSSLContext(SslUtils.trustAllSSLContext())\n              .setSSLHostnameVerifier(new NoopHostnameVerifier());\n      // Do not use NetHttpTransport. See comments in getConnectionFactory for details.\n      return new ApacheHttpTransport(httpClientBuilder.build());\n    } catch (GeneralSecurityException ex) {\n      throw new RuntimeException(\"platform does not support TLS protocol\", ex);\n    }\n  }\n\n  private final boolean enableHttpAndInsecureFailover;\n  private final boolean sendAuthorizationOverHttp;\n  private final Consumer<LogEvent> logger;\n  private final Supplier<HttpTransport> secureHttpTransportFactory;\n  private final Supplier<HttpTransport> insecureHttpTransportFactory;\n\n  private final ConcurrentHashMap<String, Failover> failoverHistory = new ConcurrentHashMap<>();\n\n  private final Deque<HttpTransport> transportsCreated = new ArrayDeque<>();\n  private final Deque<Response> responsesCreated = new ArrayDeque<>();\n  private final boolean enableRetry;\n\n  /**\n   * Create a new FailoverHttpclient.\n   *\n   * @param enableHttpAndInsecureFailover to enable automatic failover to insecure connection types\n   * @param sendAuthorizationOverHttp allow sending auth over http connections\n   * @param logger to receive log events\n   */\n  public FailoverHttpClient(\n      boolean enableHttpAndInsecureFailover,\n      boolean sendAuthorizationOverHttp,\n      Consumer<LogEvent> logger) {\n    this(enableHttpAndInsecureFailover, sendAuthorizationOverHttp, logger, true);\n  }\n\n  @VisibleForTesting\n  FailoverHttpClient(\n      boolean enableHttpAndInsecureFailover,\n      boolean sendAuthorizationOverHttp,\n      Consumer<LogEvent> logger,\n      boolean enableRetry) {\n    this(\n        enableHttpAndInsecureFailover,\n        sendAuthorizationOverHttp,\n        logger,\n        FailoverHttpClient::getSecureHttpTransport,\n        FailoverHttpClient::getInsecureHttpTransport,\n        enableRetry);\n  }\n\n  @VisibleForTesting\n  FailoverHttpClient(\n      boolean enableHttpAndInsecureFailover,\n      boolean sendAuthorizationOverHttp,\n      Consumer<LogEvent> logger,\n      Supplier<HttpTransport> secureHttpTransportFactory,\n      Supplier<HttpTransport> insecureHttpTransportFactory,\n      boolean enableRetry) {\n    this.enableHttpAndInsecureFailover = enableHttpAndInsecureFailover;\n    this.sendAuthorizationOverHttp = sendAuthorizationOverHttp;\n    this.logger = logger;\n    this.secureHttpTransportFactory = secureHttpTransportFactory;\n    this.insecureHttpTransportFactory = insecureHttpTransportFactory;\n    this.enableRetry = enableRetry;\n  }\n\n  /**\n   * Closes all connections and allocated resources, whether they are currently used or not.\n   *\n   * <p>If an I/O error occurs, shutdown attempts stop immediately, resulting in partial resource\n   * release up to that point. The method can be called again later to re-attempt releasing all\n   * resources.\n   *\n   * @throws IOException when I/O error shutting down resources\n   */\n  public void shutDown() throws IOException {\n    synchronized (transportsCreated) {\n      while (!transportsCreated.isEmpty()) {\n        transportsCreated.peekFirst().shutdown();\n        transportsCreated.removeFirst();\n      }\n    }\n    synchronized (responsesCreated) {\n      while (!responsesCreated.isEmpty()) {\n        responsesCreated.peekFirst().close();\n        responsesCreated.removeFirst();\n      }\n    }\n  }\n\n  /**\n   * Sends the request with method GET.\n   *\n   * @param url endpoint URL\n   * @param request the request to send\n   * @return the response to the sent request\n   * @throws IOException if sending the request fails\n   */\n  public Response get(URL url, Request request) throws IOException {\n    return call(HttpMethods.GET, url, request);\n  }\n\n  /**\n   * Sends the request with method POST.\n   *\n   * @param url endpoint URL\n   * @param request the request to send\n   * @return the response to the sent request\n   * @throws IOException if sending the request fails\n   */\n  public Response post(URL url, Request request) throws IOException {\n    return call(HttpMethods.POST, url, request);\n  }\n\n  /**\n   * Sends the request with method PUT.\n   *\n   * @param url endpoint URL\n   * @param request the request to send\n   * @return the response to the sent request\n   * @throws IOException if sending the request fails\n   */\n  public Response put(URL url, Request request) throws IOException {\n    return call(HttpMethods.PUT, url, request);\n  }\n\n  /**\n   * Sends the request.\n   *\n   * @param httpMethod the HTTP request method\n   * @param url endpoint URL\n   * @param request the request to send\n   * @return the response to the sent request\n   * @throws IOException if building the HTTP request fails.\n   */\n  public Response call(String httpMethod, URL url, Request request) throws IOException {\n    if (!isHttpsProtocol(url)) {\n      if (enableHttpAndInsecureFailover) { // HTTP requested. We only care if HTTP is enabled.\n        return call(httpMethod, url, request, getHttpTransport(true), true);\n      }\n      throw new SSLException(\"insecure HTTP connection not allowed: \" + url);\n    }\n\n    Optional<Response> fastPathResponse = followFailoverHistory(httpMethod, url, request);\n    if (fastPathResponse.isPresent()) {\n      return fastPathResponse.get();\n    }\n\n    try {\n      return call(httpMethod, url, request, getHttpTransport(true), !enableHttpAndInsecureFailover);\n\n    } catch (SSLException ex) {\n      if (!enableHttpAndInsecureFailover) {\n        throw ex;\n      }\n\n      try {\n        logInsecureHttpsFailover(url);\n        Response response = call(httpMethod, url, request, getHttpTransport(false), false);\n        failoverHistory.put(url.getHost() + \":\" + url.getPort(), Failover.INSECURE_HTTPS);\n        return response;\n\n      } catch (SSLException ignored) { // This is usually when the server is plain-HTTP.\n        logHttpFailover(url);\n        Response response = call(httpMethod, toHttp(url), request, getHttpTransport(true), true);\n        failoverHistory.put(url.getHost() + \":\" + url.getPort(), Failover.HTTP);\n        return response;\n      }\n\n    } catch (ConnectException ex) {\n      // It is observed that Open/Oracle JDKs sometimes throw SocketTimeoutException but other times\n      // ConnectException for connection timeout. (Could be a JDK bug.) Note SocketTimeoutException\n      // does not extend ConnectException (or vice versa), and we want to be consistent to error out\n      // on timeouts: https://github.com/GoogleContainerTools/jib/issues/1895#issuecomment-527544094\n      if (ex.getMessage() == null || !ex.getMessage().contains(\"timed out\")) {\n        // Fall back to HTTP only if \"url\" had no port specified (i.e., we tried the default HTTPS\n        // port 443) and we could not connect to 443. It's worth trying port 80.\n        if (enableHttpAndInsecureFailover && isHttpsProtocol(url) && url.getPort() == -1) {\n          logHttpFailover(url);\n          Response response = call(httpMethod, toHttp(url), request, getHttpTransport(true), true);\n          failoverHistory.put(url.getHost() + \":\" + url.getPort(), Failover.HTTP);\n          return response;\n        }\n      }\n      throw ex;\n    }\n  }\n\n  private Optional<Response> followFailoverHistory(String httpMethod, URL url, Request request)\n      throws IOException {\n    Preconditions.checkArgument(isHttpsProtocol(url));\n    switch (failoverHistory.getOrDefault(url.getHost() + \":\" + url.getPort(), Failover.NONE)) {\n      case HTTP:\n        return Optional.of(call(httpMethod, toHttp(url), request, getHttpTransport(true), true));\n      case INSECURE_HTTPS:\n        return Optional.of(call(httpMethod, url, request, getHttpTransport(false), true));\n      default:\n        return Optional.empty(); // No history found. Should go for normal execution path.\n    }\n  }\n\n  // TODO: remove retryOnIoException and turn on/off retry based on whether it's an SSLException or\n  // not: https://github.com/GoogleContainerTools/jib/issues/3422\n  private Response call(\n      String httpMethod,\n      URL url,\n      Request request,\n      HttpTransport httpTransport,\n      boolean retryOnIoException) // See https://github.com/GoogleContainerTools/jib/issues/3424\n      throws IOException {\n    boolean clearAuthorization = !isHttpsProtocol(url) && !sendAuthorizationOverHttp;\n\n    HttpHeaders requestHeaders =\n        clearAuthorization\n            ? request.getHeaders().clone().setAuthorization((String) null) // deep clone implemented\n            : request.getHeaders();\n\n    HttpRequest httpRequest =\n        httpTransport\n            .createRequestFactory()\n            .buildRequest(httpMethod, new GenericUrl(url), request.getHttpContent())\n            .setUseRawRedirectUrls(true)\n            .setHeaders(requestHeaders);\n    if (enableRetry && retryOnIoException) {\n      httpRequest.setIOExceptionHandler(createBackOffRetryHandler());\n    }\n    if (request.getHttpTimeout() != null) {\n      httpRequest.setConnectTimeout(request.getHttpTimeout());\n      httpRequest.setReadTimeout(request.getHttpTimeout());\n    }\n\n    try {\n      Response response = new Response(httpRequest.execute());\n      synchronized (responsesCreated) {\n        responsesCreated.add(response);\n      }\n      return response;\n    } catch (HttpResponseException ex) {\n      throw new ResponseException(ex, clearAuthorization);\n    }\n  }\n\n  private HttpIOExceptionHandler createBackOffRetryHandler() {\n    return new HttpBackOffIOExceptionHandler(new ExponentialBackOff()) {\n      @Override\n      public boolean handleIOException(HttpRequest request, boolean supportsRetry)\n          throws IOException {\n        String requestUrl = request.getRequestMethod() + \" \" + request.getUrl();\n        if (super.handleIOException(request, supportsRetry)) {\n          logger.accept(LogEvent.warn(requestUrl + \" failed and will be retried\"));\n          return true;\n        }\n        logger.accept(LogEvent.warn(requestUrl + \" failed and will NOT be retried\"));\n        return false;\n      }\n    };\n  }\n\n  private HttpTransport getHttpTransport(boolean secureTransport) {\n    HttpTransport transport =\n        secureTransport ? secureHttpTransportFactory.get() : insecureHttpTransportFactory.get();\n    synchronized (transportsCreated) {\n      transportsCreated.add(transport);\n    }\n    return transport;\n  }\n\n  private void logHttpFailover(URL url) {\n    String log = \"Failed to connect to \" + url + \" over HTTPS. Attempting again with HTTP.\";\n    logger.accept(LogEvent.warn(log));\n  }\n\n  private void logInsecureHttpsFailover(URL url) {\n    String log = \"Cannot verify server at \" + url + \". Attempting again with no TLS verification.\";\n    logger.accept(LogEvent.warn(log));\n  }\n\n  @VisibleForTesting\n  public Deque<HttpTransport> getTransportsCreated() {\n    return transportsCreated;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciManifestTemplate.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport javax.annotation.Nullable;\n\n/**\n * JSON Template for OCI Manifest Schema.\n *\n * <p>Example manifest JSON:\n *\n * <pre>{@code\n * {\n *   \"schemaVersion\": 2,\n *   \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n *   \"config\": {\n *     \"mediaType\": \"application/vnd.oci.image.config.v1+json\",\n *     \"size\": 631,\n *     \"digest\": \"sha256:26b84ca5b9050d32e68f66ad0f3e2bbcd247198a6e6e09a7effddf126eb8d873\"\n *   },\n *   \"layers\": [\n *     {\n *       \"mediaType\": \"application/vnd.oci.image.layer.v1.tar+gzip\",\n *       \"size\": 1991435,\n *       \"digest\": \"sha256:b56ae66c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a4647\"\n *     },\n *     {\n *       \"mediaType\": \"application/vnd.oci.image.layer.v1.tar+gzip\",\n *       \"size\": 32,\n *       \"digest\": \"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4\"\n *     }\n *   ]\n * }\n * }</pre>\n *\n * @see <a href=\"https://github.com/opencontainers/image-spec/blob/master/manifest.md\">OCI Image\n *     Manifest Specification</a>\n */\npublic class OciManifestTemplate implements BuildableManifestTemplate {\n\n  /** The OCI manifest media type. */\n  public static final String MANIFEST_MEDIA_TYPE = \"application/vnd.oci.image.manifest.v1+json\";\n\n  /** The OCI container configuration media type. */\n  private static final String CONTAINER_CONFIGURATION_MEDIA_TYPE =\n      \"application/vnd.oci.image.config.v1+json\";\n\n  /** The OCI layer media type. */\n  private static final String LAYER_MEDIA_TYPE = \"application/vnd.oci.image.layer.v1.tar+gzip\";\n\n  private final int schemaVersion = 2;\n\n  @SuppressWarnings(\"unused\")\n  private final String mediaType = MANIFEST_MEDIA_TYPE;\n\n  /** The container configuration reference. */\n  @Nullable private ContentDescriptorTemplate config;\n\n  /** The list of layer references. */\n  private final List<ContentDescriptorTemplate> layers = new ArrayList<>();\n\n  @Override\n  public int getSchemaVersion() {\n    return schemaVersion;\n  }\n\n  @Override\n  public String getManifestMediaType() {\n    return MANIFEST_MEDIA_TYPE;\n  }\n\n  @Override\n  @Nullable\n  public ContentDescriptorTemplate getContainerConfiguration() {\n    return config;\n  }\n\n  @Override\n  public List<ContentDescriptorTemplate> getLayers() {\n    return Collections.unmodifiableList(layers);\n  }\n\n  @Override\n  public void setContainerConfiguration(long size, DescriptorDigest digest) {\n    config = new ContentDescriptorTemplate(CONTAINER_CONFIGURATION_MEDIA_TYPE, size, digest);\n  }\n\n  @Override\n  public void addLayer(long size, DescriptorDigest digest) {\n    layers.add(new ContentDescriptorTemplate(LAYER_MEDIA_TYPE, size, digest));\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplate.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\n...\n", "needle_token_start": 3896, "needle_token_end": 3969, "code_context_ntokens": 15910, "output": ["```java\npublic static Cache withDirectory(Path cacheDirectory) throws CacheDirectoryCreationException {\n    try {\n      Files.createDirectories(cacheDirectory);\n    } catch (IOException ex) {\n      throw new CacheDirectoryCreationException(ex);\n    }\n    return new Cache(new CacheStorageFiles(cacheDirectory));\n  }\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "getEnvironmentMap", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/configuration/ContainerConfiguration.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To retrieve a set of key-value pairs that represent environment variables configured for a container.\n2. **Input**: No explicit input parameters are required as it operates on the internal state of an object.\n3. **Output**: Returns an immutable map containing the environment variables, where each key-value pair represents a variable name and its value. If no environment variables are set, it may return null.\n4. **Procedure**: The method accesses the internal storage that holds the environment variables set during the configuration of the container. It ensures the returned map is immutable to prevent external modifications, maintaining the integrity of the configuration.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath;\nimport com.google.cloud.tools.jib.api.buildplan.Port;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.configuration.DockerHealthCheck;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.Layer;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableSortedMap;\nimport java.lang.reflect.InvocationTargetException;\nimport java.util.Collections;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.function.Function;\nimport javax.annotation.Nullable;\n\n/** Translates an {@link Image} into a manifest or container configuration JSON. */\npublic class ImageToJsonTranslator {\n\n  /**\n   * Converts a set of {@link Port}s to the corresponding container config format for exposed ports\n   * (e.g. {@code Port(1000, Protocol.TCP)} -> {@code {\"1000/tcp\":{}}}).\n   *\n   * @param exposedPorts the set of {@link Port}s to translate, or {@code null}\n   * @return a sorted map with the string representation of the ports as keys and empty maps as\n   *     values, or {@code null} if {@code exposedPorts} is {@code null}\n   */\n  @VisibleForTesting\n  @Nullable\n  static Map<String, Map<String, String>> portSetToMap(@Nullable Set<Port> exposedPorts) {\n    return setToMap(exposedPorts, port -> port.getPort() + \"/\" + port.getProtocol());\n  }\n\n  /**\n   * Converts a set of {@link AbsoluteUnixPath}s to the corresponding container config format for\n   * volumes (e.g. {@code AbsoluteUnixPath().get(\"/var/log/my-app-logs\")} -> {@code\n   * {\"/var/log/my-app-logs\":{}}}).\n   *\n   * @param volumes the set of {@link AbsoluteUnixPath}s to translate, or {@code null}\n   * @return a sorted map with the string representation of the ports as keys and empty maps as\n   *     values, or {@code null} if {@code exposedPorts} is {@code null}\n   */\n  @VisibleForTesting\n  @Nullable\n  static Map<String, Map<String, String>> volumesSetToMap(@Nullable Set<AbsoluteUnixPath> volumes) {\n    return setToMap(volumes, AbsoluteUnixPath::toString);\n  }\n\n  /**\n   * Converts the map of environment variables to a list with items in the format \"NAME=VALUE\".\n   *\n   * @return the list\n   */\n  @VisibleForTesting\n  @Nullable\n  static ImmutableList<String> environmentMapToList(@Nullable Map<String, String> environment) {\n    if (environment == null) {\n      return null;\n    }\n    Preconditions.checkArgument(\n        environment.keySet().stream().noneMatch(key -> key.contains(\"=\")),\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;\nimport java.io.IOException;\nimport java.util.Objects;\n\n/** Used to record the results of a build. */\npublic class BuildResult {\n\n  /**\n   * Gets a {@link BuildResult} from an {@link Image}.\n   *\n   * @param image the image\n   * @param targetFormat the target format of the image\n   * @return a new {@link BuildResult} with the image's digest and id\n   * @throws IOException if writing the digest or container configuration fails\n   */\n  static BuildResult fromImage(Image image, Class<? extends BuildableManifestTemplate> targetFormat)\n      throws IOException {\n    ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(image);\n    BlobDescriptor containerConfigurationBlobDescriptor =\n        Digests.computeDigest(imageToJsonTranslator.getContainerConfiguration());\n    BuildableManifestTemplate manifestTemplate =\n        imageToJsonTranslator.getManifestTemplate(\n            targetFormat, containerConfigurationBlobDescriptor);\n    DescriptorDigest imageDigest = Digests.computeJsonDigest(manifestTemplate);\n    DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest();\n    return new BuildResult(imageDigest, imageId, false);\n  }\n\n  private final DescriptorDigest imageDigest;\n  private final DescriptorDigest imageId;\n  private final Boolean imagePushed;\n\n  BuildResult(DescriptorDigest imageDigest, DescriptorDigest imageId, boolean imagePushed) {\n    this.imageDigest = imageDigest;\n    this.imageId = imageId;\n    this.imagePushed = imagePushed;\n  }\n\n  public DescriptorDigest getImageDigest() {\n    return imageDigest;\n  }\n\n  public DescriptorDigest getImageId() {\n    return imageId;\n  }\n\n  public boolean isImagePushed() {\n    return imagePushed;\n  }\n\n  @Override\n  public int hashCode() {\n    return Objects.hash(imageDigest, imageId);\n  }\n\n  @Override\n  public boolean equals(Object other) {\n    if (this == other) {\n      return true;\n    }\n    if (!(other instanceof BuildResult)) {\n      return false;\n    }\n    BuildResult otherBuildResult = (BuildResult) other;\n    return imageDigest.equals(otherBuildResult.imageDigest)\n        && imageId.equals(otherBuildResult.imageId)\n        && imagePushed.equals(otherBuildResult.imagePushed);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/configuration/ContainerConfiguration.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.configuration;\n\nimport com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath;\nimport com.google.cloud.tools.jib.api.buildplan.Platform;\nimport com.google.cloud.tools.jib.api.buildplan.Port;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Iterables;\nimport java.time.Instant;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport javax.annotation.Nullable;\n\n/** Immutable configuration options for the container. */\npublic class ContainerConfiguration {\n\n  /** Builder for instantiating a {@link ContainerConfiguration}. */\n  public static class Builder {\n\n    /**\n     * The default creation time of the container (constant to ensure reproducibility by default).\n     */\n    private static final Instant DEFAULT_CREATION_TIME = Instant.EPOCH;\n\n    // LinkedHashSet to preserve the order\n    private Set<Platform> platforms =\n        new LinkedHashSet<>(Collections.singleton(new Platform(\"amd64\", \"linux\")));\n    private Instant creationTime = DEFAULT_CREATION_TIME;\n    @Nullable private ImmutableList<String> entrypoint;\n    @Nullable private ImmutableList<String> programArguments;\n    @Nullable private Map<String, String> environmentMap;\n    @Nullable private Set<Port> exposedPorts;\n    @Nullable private Set<AbsoluteUnixPath> volumes;\n    @Nullable private Map<String, String> labels;\n    @Nullable private String user;\n    @Nullable private AbsoluteUnixPath workingDirectory;\n\n    /**\n     * Sets a desired platform (properties including OS and architecture) list. If the base image\n     * reference is a Docker manifest list or an OCI image index, an image builder may select the\n     * base images matching the given platforms. If the base image reference is an image manifest,\n     * an image builder may ignore the given platforms and use the platform of the base image or may\n     * decide to raise on error.\n     *\n     * <p>Note that a new container configuration starts with \"amd64/linux\" as the default platform.\n     *\n     * @param platforms list of platforms to select base images in case of a manifest list\n     * @return this\n     */\n    public Builder setPlatforms(Set<Platform> platforms) {\n      Preconditions.checkArgument(!platforms.isEmpty(), \"platforms set cannot be empty\");\n      this.platforms = new LinkedHashSet<>(platforms);\n      return this;\n    }\n\n    /**\n     * Adds a desired image platform (OS and architecture pair). If the base image reference is a\n     * Docker manifest list or an OCI image index, an image builder may select the base image\n     * matching the given platform. If the base image reference is an image manifest, an image\n     * builder may ignore the given platform and use the platform of the base image or may decide to\n     * raise on error.\n     *\n     * <p>Note that a new container configuration starts with \"amd64/linux\" as the default platform.\n     * If you want to reset the default platform instead of adding a new one, use {@link\n     * #setPlatforms(Set)}.\n     *\n     * @param architecture architecture (for example, {@code amd64}) to select a base image in case\n     *     of a manifest list\n     * @param os OS (for example, {@code linux}) to select a base image in case of a manifest list\n     * @return this\n     */\n    public Builder addPlatform(String architecture, String os) {\n      platforms.add(new Platform(architecture, os));\n      return this;\n    }\n\n    /**\n     * Sets the image creation time.\n     *\n     * @param creationTime the creation time\n     * @return this\n     */\n    public Builder setCreationTime(Instant creationTime) {\n      this.creationTime = creationTime;\n      return this;\n    }\n\n    /**\n     * Sets the commandline arguments for main.\n     *\n     * @param programArguments the list of arguments\n     * @return this\n     */\n    public Builder setProgramArguments(@Nullable List<String> programArguments) {\n      if (programArguments == null) {\n        this.programArguments = null;\n      } else {\n        Preconditions.checkArgument(\n            programArguments.stream().allMatch(Objects::nonNull),\n            \"program arguments list contains null elements\");\n        this.programArguments = ImmutableList.copyOf(programArguments);\n      }\n      return this;\n    }\n\n    /**\n     * Sets the container's environment variables, mapping variable name to value.\n     *\n     * @param environmentMap the map\n     * @return this\n     */\n    public Builder setEnvironment(@Nullable Map<String, String> environmentMap) {\n      if (environmentMap == null) {\n        this.environmentMap = null;\n      } else {\n        Preconditions.checkArgument(\n            !Iterables.any(environmentMap.keySet(), Objects::isNull),\n            \"environment map contains null keys\");\n        String nullValuedKeys =\n            environmentMap.entrySet().stream()\n                .filter(entry -> entry.getValue() == null)\n                .map(Map.Entry::getKey)\n                .collect(Collectors.joining(\", \"));\n        Preconditions.checkArgument(\n            nullValuedKeys.isEmpty(),\n            \"environment map contains null values for key(s): \" + nullValuedKeys);\n        this.environmentMap = new HashMap<>(environmentMap);\n      }\n      return this;\n    }\n\n    /**\n     * Adds an environment entry to the container configuration.\n     *\n     * @param name the environment variable key\n     * @param value the non-null value to associate with environment variable\n     */\n    public void addEnvironment(String name, String value) {\n      if (environmentMap == null) {\n        environmentMap = new HashMap<>();\n      }\n      environmentMap.put(name, value);\n    }\n\n    /**\n     * Sets the container's exposed ports.\n     *\n     * @param exposedPorts the set of ports\n     * @return this\n     */\n    public Builder setExposedPorts(@Nullable Set<Port> exposedPorts) {\n      if (exposedPorts == null) {\n        this.exposedPorts = null;\n      } else {\n        Preconditions.checkArgument(\n            exposedPorts.stream().allMatch(Objects::nonNull), \"ports list contains null elements\");\n        this.exposedPorts = new HashSet<>(exposedPorts);\n      }\n      return this;\n    }\n\n    /**\n     * Adds an exposed port entry to the container configuration.\n     *\n     * @param port the non-null port to add\n     */\n    public void addExposedPort(Port port) {\n      if (exposedPorts == null) {\n        exposedPorts = new HashSet<>();\n      }\n      exposedPorts.add(port);\n    }\n\n    /**\n     * Sets the container's volumes.\n     *\n     * @param volumes the set of volumes\n     * @return this\n     */\n    public Builder setVolumes(@Nullable Set<AbsoluteUnixPath> volumes) {\n      if (volumes == null) {\n        this.volumes = null;\n      } else {\n        Preconditions.checkArgument(\n            volumes.stream().allMatch(Objects::nonNull), \"volumes list contains null elements\");\n        this.volumes = new HashSet<>(volumes);\n      }\n      return this;\n    }\n\n    /**\n     * Adds a volume entry to the container configuration.\n     *\n     * @param volume the absolute path to add as a volume entry\n     */\n    public void addVolume(AbsoluteUnixPath volume) {\n      if (volumes == null) {\n        volumes = new HashSet<>();\n      }\n      volumes.add(volume);\n    }\n\n    /**\n     * Sets the container's labels.\n     *\n     * @param labels the map of labels\n     * @return this\n     */\n    public Builder setLabels(@Nullable Map<String, String> labels) {\n      if (labels == null) {\n        this.labels = null;\n      } else {\n        Preconditions.checkArgument(\n            !Iterables.any(labels.keySet(), Objects::isNull), \"labels map contains null keys\");\n        Preconditions.checkArgument(\n            !Iterables.any(labels.values(), Objects::isNull), \"labels map contains null values\");\n        this.labels = new HashMap<>(labels);\n      }\n      return this;\n    }\n\n    /**\n     * Add a label to the container configuration.\n     *\n     * @param key the label name to add\n     * @param value the value to be associated with the label\n     */\n    public void addLabel(String key, String value) {\n      if (labels == null) {\n        labels = new HashMap<>();\n      }\n      labels.put(key, value);\n    }\n\n    /**\n     * Sets the container entrypoint.\n     *\n     * @param entrypoint the tokenized command to run when the container starts\n     * @return this\n     */\n    public Builder setEntrypoint(@Nullable List<String> entrypoint) {\n      if (entrypoint == null) {\n        this.entrypoint = null;\n      } else {\n        Preconditions.checkArgument(\n            entrypoint.stream().allMatch(Objects::nonNull), \"entrypoint contains null elements\");\n        this.entrypoint = ImmutableList.copyOf(entrypoint);\n      }\n      return this;\n    }\n\n    /**\n     * Sets the user and group to run the container as. {@code user} can be a username or UID along\n     * with an optional groupname or GID. The following are all valid: {@code user}, {@code uid},\n     * {@code user:group}, {@code uid:gid}, {@code uid:group}, {@code user:gid}.\n     *\n     * @param user the username/UID and optionally the groupname/GID\n     * @return this\n     */\n    public Builder setUser(@Nullable String user) {\n      this.user = user;\n      return this;\n    }\n\n    /**\n     * Sets the working directory in the container.\n     *\n     * @param workingDirectory the working directory\n     * @return this\n     */\n    public Builder setWorkingDirectory(@Nullable AbsoluteUnixPath workingDirectory) {\n      this.workingDirectory = workingDirectory;\n      return this;\n    }\n\n    /**\n     * Builds the {@link ContainerConfiguration}.\n     *\n     * @return the corresponding {@link ContainerConfiguration}\n     */\n    public ContainerConfiguration build() {\n      return new ContainerConfiguration(\n          ImmutableSet.copyOf(platforms),\n          creationTime,\n          entrypoint,\n          programArguments,\n          environmentMap == null ? null : ImmutableMap.copyOf(environmentMap),\n          exposedPorts == null ? null : ImmutableSet.copyOf(exposedPorts),\n          volumes == null ? null : ImmutableSet.copyOf(volumes),\n          labels == null ? null : ImmutableMap.copyOf(labels),\n          user,\n          workingDirectory);\n    }\n\n    private Builder() {}\n  }\n\n  /**\n   * Constructs a builder for a {@link ContainerConfiguration}.\n   *\n   * @return the builder\n   */\n  public static Builder builder() {\n    return new Builder();\n  }\n\n  private final ImmutableSet<Platform> platforms;\n  private final Instant creationTime;\n  @Nullable private final ImmutableList<String> entrypoint;\n  @Nullable private final ImmutableList<String> programArguments;\n  @Nullable private final ImmutableMap<String, String> environmentMap;\n  @Nullable private final ImmutableSet<Port> exposedPorts;\n  @Nullable private final ImmutableSet<AbsoluteUnixPath> volumes;\n  @Nullable private final ImmutableMap<String, String> labels;\n  @Nullable private final String user;\n  @Nullable private final AbsoluteUnixPath workingDirectory;\n\n  private ContainerConfiguration(\n      ImmutableSet<Platform> platforms,\n      Instant creationTime,\n      @Nullable ImmutableList<String> entrypoint,\n      @Nullable ImmutableList<String> programArguments,\n      @Nullable ImmutableMap<String, String> environmentMap,\n      @Nullable ImmutableSet<Port> exposedPorts,\n      @Nullable ImmutableSet<AbsoluteUnixPath> volumes,\n      @Nullable ImmutableMap<String, String> labels,\n      @Nullable String user,\n      @Nullable AbsoluteUnixPath workingDirectory) {\n    this.platforms = platforms;\n    this.creationTime = creationTime;\n    this.entrypoint = entrypoint;\n    this.programArguments = programArguments;\n    this.environmentMap = environmentMap;\n    this.exposedPorts = exposedPorts;\n    this.volumes = volumes;\n    this.labels = labels;\n    this.user = user;\n    this.workingDirectory = workingDirectory;\n  }\n\n  public ImmutableSet<Platform> getPlatforms() {\n    return platforms;\n  }\n\n  public Instant getCreationTime() {\n    return creationTime;\n  }\n\n  @Nullable\n  public ImmutableList<String> getEntrypoint() {\n    return entrypoint;\n  }\n\n  @Nullable\n  public ImmutableList<String> getProgramArguments() {\n    return programArguments;\n  }\n\n  \n@Nullable\n  public ImmutableMap<String, String> getEnvironmentMap() {\n    return environmentMap;\n  }\n\n  @Nullable\n  public ImmutableSet<Port> getExposedPorts() {\n    return exposedPorts;\n  }\n\n  @Nullable\n  public ImmutableSet<AbsoluteUnixPath> getVolumes() {\n    return volumes;\n  }\n\n  @Nullable\n  public String getUser() {\n    return user;\n  }\n\n  @Nullable\n  public ImmutableMap<String, String> getLabels() {\n    return labels;\n  }\n\n  @Nullable\n  public AbsoluteUnixPath getWorkingDirectory() {\n    return workingDirectory;\n  }\n\n  @Override\n  @VisibleForTesting\n  public boolean equals(Object other) {\n    if (this == other) {\n      return true;\n    }\n    if (!(other instanceof ContainerConfiguration)) {\n      return false;\n    }\n    ContainerConfiguration otherContainerConfiguration = (ContainerConfiguration) other;\n    return platforms.equals(otherContainerConfiguration.platforms)\n        && creationTime.equals(otherContainerConfiguration.creationTime)\n        && Objects.equals(entrypoint, otherContainerConfiguration.entrypoint)\n        && Objects.equals(programArguments, otherContainerConfiguration.programArguments)\n        && Objects.equals(environmentMap, otherContainerConfiguration.environmentMap)\n        && Objects.equals(exposedPorts, otherContainerConfiguration.exposedPorts)\n        && Objects.equals(labels, otherContainerConfiguration.labels)\n        && Objects.equals(user, otherContainerConfiguration.user)\n        && Objects.equals(workingDirectory, otherContainerConfiguration.workingDirectory);\n  }\n\n  @Override\n  @VisibleForTesting\n  public int hashCode() {\n    return Objects.hash(\n        platforms,\n        creationTime,\n        entrypoint,\n        programArguments,\n        environmentMap,\n        exposedPorts,\n        labels,\n        user,\n        workingDirectory);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\n/** Exception thrown when accessing non-existent properties of layers. */\npublic class LayerPropertyNotFoundException extends RuntimeException {\n\n  LayerPropertyNotFoundException(String message) {\n    super(message);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ContainerConfiguration;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;\nimport com.google.cloud.tools.jib.image.json.HistoryEntry;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.UnmodifiableIterator;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.Callable;\nimport javax.annotation.Nullable;\n\n/** Builds a model {@link Image}. */\nclass BuildImageStep implements Callable<Image> {\n\n  private static final String DESCRIPTION = \"Building container configuration\";\n\n  @VisibleForTesting\n  static String truncateLongClasspath(ImmutableList<String> imageEntrypoint) {\n    List<String> truncated = new ArrayList<>();\n    UnmodifiableIterator<String> iterator = imageEntrypoint.iterator();\n    while (iterator.hasNext()) {\n      String element = iterator.next();\n      truncated.add(element);\n\n      if (element.equals(\"-cp\") || element.equals(\"-classpath\")) {\n        String classpath = iterator.next();\n        if (classpath.length() > 200) {\n          truncated.add(classpath.substring(0, 200) + \"<... classpath truncated ...>\");\n        } else {\n          truncated.add(classpath);\n        }\n      }\n    }\n    return truncated.toString();\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n  private final Image baseImage;\n  private final List<PreparedLayer> baseImageLayers;\n  private final List<PreparedLayer> applicationLayers;\n\n  BuildImageStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Image baseImage,\n      List<PreparedLayer> baseImageLayers,\n      List<PreparedLayer> applicationLayers) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.baseImage = baseImage;\n    this.baseImageLayers = baseImageLayers;\n    this.applicationLayers = applicationLayers;\n  }\n\n  @Override\n  public Image call() throws LayerPropertyNotFoundException {\n    try (ProgressEventDispatcher ignored =\n            progressEventDispatcherFactory.create(\"building image format\", 1);\n        TimerEventDispatcher ignored2 =\n            new TimerEventDispatcher(buildContext.getEventHandlers(), DESCRIPTION)) {\n      // Constructs the image.\n      Image.Builder imageBuilder = Image.builder(buildContext.getTargetFormat());\n\n      // Base image layers\n      baseImageLayers.forEach(imageBuilder::addLayer);\n\n      // Passthrough config and count non-empty history entries\n      int nonEmptyLayerCount = 0;\n      for (HistoryEntry historyObject : baseImage.getHistory()) {\n        imageBuilder.addHistory(historyObject);\n        if (!historyObject.hasCorrespondingLayer()) {\n          nonEmptyLayerCount++;\n        }\n      }\n      imageBuilder\n          .setArchitecture(baseImage.getArchitecture())\n          .setOs(baseImage.getOs())\n          .addEnvironment(baseImage.getEnvironment())\n          .addLabels(baseImage.getLabels())\n          .setHealthCheck(baseImage.getHealthCheck())\n          .addExposedPorts(baseImage.getExposedPorts())\n          .addVolumes(baseImage.getVolumes())\n          .setUser(baseImage.getUser())\n          .setWorkingDirectory(baseImage.getWorkingDirectory());\n\n      ContainerConfiguration containerConfiguration = buildContext.getContainerConfiguration();\n      // Add history elements for non-empty layers that don't have one yet\n      Instant layerCreationTime = containerConfiguration.getCreationTime();\n      for (int count = 0; count < baseImageLayers.size() - nonEmptyLayerCount; count++) {\n        imageBuilder.addHistory(\n            HistoryEntry.builder()\n                .setCreationTimestamp(layerCreationTime)\n                .setComment(\"auto-generated by Jib\")\n                .build());\n      }\n\n      // Add built layers/configuration\n      for (PreparedLayer applicationLayer : applicationLayers) {\n        imageBuilder\n            .addLayer(applicationLayer)\n            .addHistory(\n                HistoryEntry.builder()\n                    .setCreationTimestamp(layerCreationTime)\n                    .setAuthor(\"Jib\")\n                    .setCreatedBy(buildContext.getToolName() + \":\" + buildContext.getToolVersion())\n                    .setComment(applicationLayer.getName())\n                    .build());\n      }\n\n      imageBuilder\n          .addEnvironment(containerConfiguration.getEnvironmentMap())\n          .setCreated(containerConfiguration.getCreationTime())\n          .setEntrypoint(computeEntrypoint(baseImage, containerConfiguration))\n          .setProgramArguments(computeProgramArguments(baseImage, containerConfiguration))\n          .addExposedPorts(containerConfiguration.getExposedPorts())\n          .addVolumes(containerConfiguration.getVolumes())\n          .addLabels(containerConfiguration.getLabels());\n      if (containerConfiguration.getUser() != null) {\n        imageBuilder.setUser(containerConfiguration.getUser());\n      }\n      if (containerConfiguration.getWorkingDirectory() != null) {\n        imageBuilder.setWorkingDirectory(containerConfiguration.getWorkingDirectory().toString());\n      }\n\n      // Gets the container configuration content descriptor.\n      return imageBuilder.build();\n    }\n  }\n\n  /**\n   * Computes the image entrypoint. If {@link ContainerConfiguration#getEntrypoint()} is null, the\n   * entrypoint is inherited from the base image. Otherwise {@link\n   * ContainerConfiguration#getEntrypoint()} is returned.\n   *\n   * @param baseImage the base image\n   * @param containerConfiguration the container configuration\n   * @return the container entrypoint\n   */\n  @Nullable\n  private ImmutableList<String> computeEntrypoint(\n      Image baseImage, ContainerConfiguration containerConfiguration) {\n    boolean shouldInherit =\n        baseImage.getEntrypoint() != null && containerConfiguration.getEntrypoint() == null;\n\n    ImmutableList<String> entrypointToUse =\n        shouldInherit ? baseImage.getEntrypoint() : containerConfiguration.getEntrypoint();\n\n    if (entrypointToUse != null) {\n      buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(\"\"));\n      if (shouldInherit) {\n        String message =\n            \"Container entrypoint set to \" + entrypointToUse + \" (inherited from base image)\";\n        buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(message));\n      } else {\n        String message = \"Container entrypoint set to \" + truncateLongClasspath(entrypointToUse);\n        buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(message));\n      }\n    }\n\n    return entrypointToUse;\n  }\n\n  /**\n   * Computes the image program arguments. If {@link ContainerConfiguration#getEntrypoint()} and\n   * {@link ContainerConfiguration#getProgramArguments()} are null, the program arguments are\n   * inherited from the base image. Otherwise {@link ContainerConfiguration#getProgramArguments()}\n   * is returned.\n   *\n   * @param baseImage the base image\n   * @param containerConfiguration the container configuration\n   * @return the container program arguments\n   */\n  @Nullable\n  private ImmutableList<String> computeProgramArguments(\n      Image baseImage, ContainerConfiguration containerConfiguration) {\n    boolean shouldInherit =\n        baseImage.getProgramArguments() != null\n            // Inherit CMD only when inheriting ENTRYPOINT.\n            && containerConfiguration.getEntrypoint() == null\n            && containerConfiguration.getProgramArguments() == null;\n\n    ImmutableList<String> programArgumentsToUse =\n        shouldInherit\n            ? baseImage.getProgramArguments()\n            : containerConfiguration.getProgramArguments();\n\n    if (programArgumentsToUse != null) {\n      String logSuffix = shouldInherit ? \" (inherited from base image)\" : \"\";\n      String message = \"Container program arguments set to \" + programArgumentsToUse + logSuffix;\n      buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(message));\n    }\n\n    return programArgumentsToUse;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.global.JibSystemProperties;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.util.Collections;\nimport java.util.Set;\nimport java.util.concurrent.Callable;\nimport java.util.stream.Collectors;\n\n/**\n * Pushes a manifest or a manifest list for a tag. If not a manifest list, returns the manifest\n * digest (\"image digest\") and the container configuration digest (\"image id\") as {@link\n * BuildResult}. If a manifest list, returns the manifest list digest only.\n */\n// TODO: figure out the right return value and type when pushing a manifest list.\nclass PushImageStep implements Callable<BuildResult> {\n\n  private static final String DESCRIPTION = \"Pushing manifest\";\n\n  static ImmutableList<PushImageStep> makeList(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      BlobDescriptor containerConfigurationDigestAndSize,\n      Image builtImage,\n      boolean manifestAlreadyExists)\n      throws IOException {\n    // Gets the image manifest to push.\n    BuildableManifestTemplate manifestTemplate =\n        new ImageToJsonTranslator(builtImage)\n            .getManifestTemplate(\n                buildContext.getTargetFormat(), containerConfigurationDigestAndSize);\n    DescriptorDigest manifestDigest = Digests.computeJsonDigest(manifestTemplate);\n    Set<String> imageQualifiers = getImageQualifiers(buildContext, builtImage, manifestDigest);\n\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    try (TimerEventDispatcher ignored =\n            new TimerEventDispatcher(eventHandlers, \"Preparing manifest pushers\");\n        ProgressEventDispatcher progressDispatcher =\n            progressEventDispatcherFactory.create(\n                \"launching manifest pushers\", imageQualifiers.size())) {\n\n      if (JibSystemProperties.skipExistingImages() && manifestAlreadyExists) {\n        eventHandlers.dispatch(LogEvent.info(\"Skipping pushing manifest; already exists.\"));\n        return ImmutableList.of();\n      }\n\n      return imageQualifiers.stream()\n          .map(\n              qualifier ->\n                  new PushImageStep(\n                      buildContext,\n                      progressDispatcher.newChildProducer(),\n                      registryClient,\n                      manifestTemplate,\n                      qualifier,\n                      manifestDigest,\n                      containerConfigurationDigestAndSize.getDigest()))\n          .collect(ImmutableList.toImmutableList());\n    }\n  }\n\n  private static Set<String> getImageQualifiers(\n      BuildContext buildContext, Image builtImage, DescriptorDigest manifestDigest) {\n    boolean singlePlatform = buildContext.getContainerConfiguration().getPlatforms().size() == 1;\n    Set<String> tags = buildContext.getAllTargetImageTags();\n    if (singlePlatform) {\n      return tags;\n    }\n    if (buildContext.getEnablePlatformTags()) {\n      String architecture = builtImage.getArchitecture();\n      return tags.stream().map(tag -> tag + \"-\" + architecture).collect(Collectors.toSet());\n    }\n    return Collections.singleton(manifestDigest.toString());\n  }\n\n  static ImmutableList<PushImageStep> makeListForManifestList(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      ManifestTemplate manifestList,\n      boolean manifestListAlreadyExists)\n      throws IOException {\n    Set<String> tags = buildContext.getAllTargetImageTags();\n\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    try (TimerEventDispatcher ignored =\n            new TimerEventDispatcher(eventHandlers, \"Preparing manifest list pushers\");\n        ProgressEventDispatcher progressEventDispatcher =\n            progressEventDispatcherFactory.create(\"launching manifest list pushers\", tags.size())) {\n      boolean singlePlatform = buildContext.getContainerConfiguration().getPlatforms().size() == 1;\n      if (singlePlatform) {\n        return ImmutableList.of(); // single image; no need to push a manifest list\n      }\n\n      if (JibSystemProperties.skipExistingImages() && manifestListAlreadyExists) {\n        eventHandlers.dispatch(LogEvent.info(\"Skipping pushing manifest list; already exists.\"));\n        return ImmutableList.of();\n      }\n      DescriptorDigest manifestListDigest = Digests.computeJsonDigest(manifestList);\n      return tags.stream()\n          .map(\n              tag ->\n                  new PushImageStep(\n                      buildContext,\n                      progressEventDispatcher.newChildProducer(),\n                      registryClient,\n                      manifestList,\n                      tag,\n                      manifestListDigest,\n                      // TODO: a manifest list digest isn't an \"image id\". Figure out the right\n                      // return value and type.\n                      manifestListDigest))\n          .collect(ImmutableList.toImmutableList());\n    }\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n  private final RegistryClient registryClient;\n  private final ManifestTemplate manifestTemplate;\n  private final String imageQualifier;\n  private final DescriptorDigest imageDigest;\n  private final DescriptorDigest imageId;\n\n  PushImageStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      ManifestTemplate manifestTemplate,\n      String imageQualifier,\n      DescriptorDigest imageDigest,\n      DescriptorDigest imageId) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.registryClient = registryClient;\n    this.manifestTemplate = manifestTemplate;\n    this.imageQualifier = imageQualifier;\n    this.imageDigest = imageDigest;\n    this.imageId = imageId;\n  }\n\n  @Override\n  public BuildResult call() throws IOException, RegistryException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    try (TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, DESCRIPTION);\n        ProgressEventDispatcher ignored2 =\n            progressEventDispatcherFactory.create(\"pushing manifest for \" + imageQualifier, 1)) {\n      eventHandlers.dispatch(LogEvent.info(\"Pushing manifest for \" + imageQualifier + \"...\"));\n\n      registryClient.pushManifest(manifestTemplate, imageQualifier);\n      return new BuildResult(imageDigest, imageId, true);\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.event.progress;\n\nimport java.io.Closeable;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.util.function.Consumer;\nimport java.util.function.Supplier;\nimport javax.annotation.Nullable;\n\n/**\n * Wraps a {@code Consumer<Long>} so that multiple consume calls ({@link #accept}) within a short\n * period of time are merged into a single later call with the value accumulated up to that point.\n */\npublic class ThrottledAccumulatingConsumer implements Consumer<Long>, Closeable {\n\n  private final Consumer<Long> consumer;\n\n  /** Delay between each call to the underlying {@link #accept}. */\n  private final Duration delayBetweenCallbacks;\n\n  /** Last time the underlying {@link #accept} was called. */\n  private Instant previousCallback;\n\n  /** \"Clock\" that returns the current {@link Instant}. */\n  private final Supplier<Instant> getNow;\n\n  @Nullable private Long valueSoFar;\n\n  /**\n   * Wraps a consumer with the delay of 100 ms.\n   *\n   * @param callback {@link Consumer} callback to wrap\n   */\n  public ThrottledAccumulatingConsumer(Consumer<Long> callback) {\n    this(callback, Duration.ofMillis(100), Instant::now);\n  }\n\n  /**\n   * A new configured consumer.\n   *\n   * @param consumer {@link Consumer} callback to wrap\n   * @param delayBetweenCallbacks delay between each call to the underlying accept\n   * @param getNow supplies of the current time {@link Instant}\n   */\n  public ThrottledAccumulatingConsumer(\n      Consumer<Long> consumer, Duration delayBetweenCallbacks, Supplier<Instant> getNow) {\n    this.consumer = consumer;\n    this.delayBetweenCallbacks = delayBetweenCallbacks;\n    this.getNow = getNow;\n\n    previousCallback = getNow.get();\n  }\n\n  @Override\n  public void accept(Long value) {\n    valueSoFar = valueSoFar == null ? value : valueSoFar + value;\n\n    Instant now = getNow.get();\n    Instant nextFireTime = previousCallback.plus(delayBetweenCallbacks);\n    if (now.isAfter(nextFireTime)) {\n      consumer.accept(valueSoFar);\n      previousCallback = now;\n      valueSoFar = null;\n    }\n  }\n\n  @Override\n  public void close() {\n    if (valueSoFar != null) {\n      consumer.accept(valueSoFar);\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer;\nimport com.google.common.base.Preconditions;\nimport java.io.Closeable;\nimport javax.annotation.Nullable;\n\n/**\n * Contains a {@link ProgressEventDispatcher} and throttles dispatching progress events with the\n * default delay used by {@link ThrottledAccumulatingConsumer}. This class is mutable and should\n * only be used within a local context.\n *\n * <p>This class is necessary because the total BLOb size (allocation units) is not known until the\n * response headers are received, only after which can the {@link ProgressEventDispatcher} be\n * created.\n */\nclass ThrottledProgressEventDispatcherWrapper implements Closeable {\n\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n  private final String description;\n  @Nullable private ProgressEventDispatcher progressEventDispatcher;\n  @Nullable private ThrottledAccumulatingConsumer throttledDispatcher;\n\n  ThrottledProgressEventDispatcherWrapper(\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory, String description) {\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.description = description;\n  }\n\n  public void dispatchProgress(Long progressUnits) {\n    Preconditions.checkNotNull(throttledDispatcher);\n    throttledDispatcher.accept(progressUnits);\n  }\n\n  @Override\n  public void close() {\n    Preconditions.checkNotNull(progressEventDispatcher);\n    Preconditions.checkNotNull(throttledDispatcher);\n    throttledDispatcher.close();\n    progressEventDispatcher.close();\n  }\n\n  void setProgressTarget(long allocationUnits) {\n    Preconditions.checkState(progressEventDispatcher == null);\n    progressEventDispatcher = progressEventDispatcherFactory.create(description, allocationUnits);\n    throttledDispatcher =\n        new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/docker/json/DockerManifestEntryTemplate.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.docker.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * JSON Template for a loadable Docker Manifest entry. The RepoTags property requires a tag; i.e. if\n * a tag is missing, it explicitly should use \"latest\".\n *\n * <p>Note that this is a template for a single Manifest entry, while the entire Docker Manifest\n * should be {@code List<DockerManifestEntryTemplate>}.\n *\n * <p>Example manifest entry JSON:\n *\n * <pre>{@code\n * {\n *   \"Config\":\"config.json\",\n *   \"RepoTags\":[\"repository:tag\"]\n *   \"Layers\": [\n *     \"eb05f3dbdb543cc610527248690575bacbbcebabe6ecf665b189cf18b541e3ca.tar.gz\",\n *     \"ba7c544469e514f1a9a4dec59ab640540d50992b288adbb34a1a63c45bf19a24.tar.gz\",\n *     \"15705ab016593987662839b40f5a22fd1032996c90808d4a1371eb46974017d5.tar.gz\"\n *   ]\n * }\n * }</pre>\n *\n * @see <a href=\"https://github.com/moby/moby/blob/master/image/tarexport/load.go\">Docker load\n *     source</a>\n */\n@JsonIgnoreProperties(ignoreUnknown = true)\npublic class DockerManifestEntryTemplate implements JsonTemplate {\n\n  @JsonProperty(\"Config\")\n  private String config = \"config.json\";\n\n  @JsonProperty(\"RepoTags\")\n  private final List<String> repoTags = new ArrayList<>();\n\n  @JsonProperty(\"Layers\")\n  private final List<String> layers = new ArrayList<>();\n\n  public void setConfig(String config) {\n    this.config = config;\n  }\n\n  public void addRepoTag(String repoTag) {\n    repoTags.add(repoTag);\n  }\n\n  public void addLayerFile(String layer) {\n    layers.add(layer);\n  }\n\n  public String getConfig() {\n    return config;\n  }\n\n  public List<String> getLayerFiles() {\n    return layers;\n  }\n\n  @VisibleForTesting\n  public List<String> getRepoTags() {\n    return repoTags;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableMap;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.stream.Collectors;\nimport javax.annotation.Nullable;\n\n/**\n * JSON template for OCI archive \"index.json\" file.\n *\n * <p>Example manifest JSON:\n *\n * <pre>{@code\n * {\n *   \"schemaVersion\": 2,\n *   \"mediaType\": \"application/vnd.oci.image.index.v1+json\",\n *   \"manifests\": [\n *     {\n *       \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n *       \"digest\": \"sha256:e684b1dceef404268f17d4adf7f755fd9912b8ae64864b3954a83ebb8aa628b3\",\n *       \"size\": 1132,\n *       \"platform\": {\n *         \"architecture\": \"ppc64le\",\n *         \"os\": \"linux\"\n *       },\n *       \"annotations\": {\n *         \"org.opencontainers.image.ref.name\": \"gcr.io/project/image:tag\"\n *       }\n *     }\n *   ]\n * }\n * }</pre>\n *\n * @see <a href=\"https://github.com/opencontainers/image-spec/blob/master/image-index.md\">OCI Image\n *     Index Specification</a>\n */\npublic class OciIndexTemplate implements ManifestListTemplate {\n\n  /** The OCI Index media type. */\n  public static final String MEDIA_TYPE = \"application/vnd.oci.image.index.v1+json\";\n\n  private final int schemaVersion = 2;\n  private final String mediaType = MEDIA_TYPE;\n\n  private final List<ManifestDescriptorTemplate> manifests = new ArrayList<>();\n\n  @Override\n  public int getSchemaVersion() {\n    return schemaVersion;\n  }\n\n  @Override\n  public String getManifestMediaType() {\n    return mediaType;\n  }\n\n  /**\n   * Adds a manifest reference with the given {@link BlobDescriptor}.\n   *\n   * @param descriptor the manifest blob descriptor\n   * @param imageReferenceName the image reference name\n   */\n  public void addManifest(BlobDescriptor descriptor, String imageReferenceName) {\n    ManifestDescriptorTemplate contentDescriptorTemplate =\n        new ManifestDescriptorTemplate(\n            OciManifestTemplate.MANIFEST_MEDIA_TYPE, descriptor.getSize(), descriptor.getDigest());\n    contentDescriptorTemplate.setAnnotations(\n        ImmutableMap.of(\"org.opencontainers.image.ref.name\", imageReferenceName));\n    manifests.add(contentDescriptorTemplate);\n  }\n\n  /**\n   * Adds a manifest.\n   *\n   * @param manifest a manifest descriptor\n   */\n  public void addManifest(OciIndexTemplate.ManifestDescriptorTemplate manifest) {\n    manifests.add(manifest);\n  }\n\n  @VisibleForTesting\n  public List<ManifestDescriptorTemplate> getManifests() {\n    return manifests;\n  }\n\n  @Override\n  public List<String> getDigestsForPlatform(String architecture, String os) {\n    return getManifests().stream()\n        .filter(\n            manifest ->\n                manifest.platform != null\n                    && os.equals(manifest.platform.os)\n                    && architecture.equals(manifest.platform.architecture))\n        .map(ManifestDescriptorTemplate::getDigest)\n        .filter(Objects::nonNull)\n        .map(DescriptorDigest::toString)\n        .collect(Collectors.toList());\n  }\n\n  /**\n   * Template for inner JSON object representing a single platform specific manifest. See <a\n   * href=\"https://github.com/opencontainers/image-spec/blob/main/image-index.md\">OCI Image Index\n   * Specification</a>\n   */\n  public static class ManifestDescriptorTemplate\n      extends BuildableManifestTemplate.ContentDescriptorTemplate {\n\n    ManifestDescriptorTemplate(String mediaType, long size, DescriptorDigest digest) {\n      super(mediaType, size, digest);\n    }\n\n    /** Necessary for Jackson to create from JSON. */\n    @SuppressWarnings(\"unused\")\n    private ManifestDescriptorTemplate() {\n      super();\n    }\n\n    @JsonIgnoreProperties(ignoreUnknown = true)\n    public static class Platform implements JsonTemplate {\n      @Nullable private String architecture;\n      @Nullable private String os;\n\n      @Nullable\n      public String getArchitecture() {\n        return architecture;\n      }\n\n      @Nullable\n      public String getOs() {\n        return os;\n      }\n    }\n\n    @Nullable private OciIndexTemplate.ManifestDescriptorTemplate.Platform platform;\n\n    /**\n     * Sets a platform.\n     *\n     * @param architecture the manifest architecture\n     * @param os the manifest os\n     */\n    public void setPlatform(String architecture, String os) {\n      platform = new OciIndexTemplate.ManifestDescriptorTemplate.Platform();\n      platform.architecture = architecture;\n      platform.os = os;\n    }\n\n    @Nullable\n    public OciIndexTemplate.ManifestDescriptorTemplate.Platform getPlatform() {\n      return platform;\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/ImageTarball.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;\nimport com.google.cloud.tools.jib.image.json.OciIndexTemplate;\nimport com.google.cloud.tools.jib.image.json.OciManifestTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.cloud.tools.jib.tar.TarStreamBuilder;\nimport com.google.common.collect.ImmutableSet;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.time.Instant;\nimport java.util.Collections;\n\n/** Translates an {@link Image} to a tarball that can be loaded into Docker. */\npublic class ImageTarball {\n\n  /** File name for the container configuration in the tarball. */\n  private static final String CONTAINER_CONFIGURATION_JSON_FILE_NAME = \"config.json\";\n\n  /** File name for the manifest in the tarball. */\n  private static final String MANIFEST_JSON_FILE_NAME = \"manifest.json\";\n\n  /** File name extension for the layer content files. */\n  private static final String LAYER_FILE_EXTENSION = \".tar.gz\";\n\n  /** Time that entry is set in the tar. */\n  private static final Instant TAR_ENTRY_MODIFICATION_TIME = Instant.EPOCH;\n\n  private static final String BLOB_PREFIX = \"blobs/sha256/\";\n\n  private final Image image;\n  private final ImageReference imageReference;\n  private final ImmutableSet<String> allTargetImageTags;\n\n  /**\n   * Instantiate with an {@link Image}.\n   *\n   * @param image the image to convert into a tarball\n   * @param imageReference image reference to set in the manifest (note that the tag portion of the\n   *     image reference is ignored)\n   * @param allTargetImageTags the tags to tag the image with\n   */\n  public ImageTarball(\n      Image image, ImageReference imageReference, ImmutableSet<String> allTargetImageTags) {\n    this.image = image;\n    this.imageReference = imageReference;\n    this.allTargetImageTags = allTargetImageTags;\n  }\n\n  /**\n   * Writes image tar bar in configured {@link Image#getImageFormat()} of OCI or Docker to output\n   * stream.\n   *\n   * @param out the target output stream\n...\n", "needle_token_start": 5628, "needle_token_end": 5656, "code_context_ntokens": 15986, "output": ["```java\n/**\n * Converts the map of environment variables to a list with items in the format \"NAME=VALUE\".\n *\n * @return the list\n */\n@VisibleForTesting\n@Nullable\nstatic ImmutableList<String> environmentMapToList(@Nullable Map<String, String> environment) {\n  if (environment == null) {\n    return null;\n  }\n  Preconditions.checkArgument(\n      environment.keySet().stream().noneMatch(key -> key.contains(\"=\")),\n      \"environment map contains keys with '='\");\n  return environment.entrySet().stream()\n      .map(entry -> entry.getKey() + \"=\" + entry.getValue())\n      .collect(ImmutableList.toImmutableList());\n}\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "checkManifestPlatform", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PlatformChecker.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To validate that the platform configuration of a build context matches the platform details of a non-manifest list base image, and to ensure that only one platform is specified when the base image is not a manifest list.\n2. **Input**: The function takes a build context containing the configuration details of the build, and a container configuration template that includes the platform details of the base image.\n3. **Output**: There is no return value; however, it throws a specific exception if the platform configuration does not match the base image's platform or if multiple platforms are specified for a non-manifest list base image.\n4. **Procedure**: \n   - The function first checks if the base image configuration is specified by a path or an image reference.\n   - It then retrieves the set of target platforms specified in the build context.\n   - If multiple platforms are specified, an exception is thrown indicating that the base image must be a manifest list to support multiple platforms.\n   - If only one platform is specified, it compares this platform with the platform of the base image from the container configuration.\n   - If there is a mismatch and the specified platform is not the default, an exception is thrown indicating the mismatch between the configured platform and the base image's platform.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/ImageTarball.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;\nimport com.google.cloud.tools.jib.image.json.OciIndexTemplate;\nimport com.google.cloud.tools.jib.image.json.OciManifestTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.cloud.tools.jib.tar.TarStreamBuilder;\nimport com.google.common.collect.ImmutableSet;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.time.Instant;\nimport java.util.Collections;\n\n/** Translates an {@link Image} to a tarball that can be loaded into Docker. */\npublic class ImageTarball {\n\n  /** File name for the container configuration in the tarball. */\n  private static final String CONTAINER_CONFIGURATION_JSON_FILE_NAME = \"config.json\";\n\n  /** File name for the manifest in the tarball. */\n  private static final String MANIFEST_JSON_FILE_NAME = \"manifest.json\";\n\n  /** File name extension for the layer content files. */\n  private static final String LAYER_FILE_EXTENSION = \".tar.gz\";\n\n  /** Time that entry is set in the tar. */\n  private static final Instant TAR_ENTRY_MODIFICATION_TIME = Instant.EPOCH;\n\n  private static final String BLOB_PREFIX = \"blobs/sha256/\";\n\n  private final Image image;\n  private final ImageReference imageReference;\n  private final ImmutableSet<String> allTargetImageTags;\n\n  /**\n   * Instantiate with an {@link Image}.\n   *\n   * @param image the image to convert into a tarball\n   * @param imageReference image reference to set in the manifest (note that the tag portion of the\n   *     image reference is ignored)\n   * @param allTargetImageTags the tags to tag the image with\n   */\n  public ImageTarball(\n      Image image, ImageReference imageReference, ImmutableSet<String> allTargetImageTags) {\n    this.image = image;\n    this.imageReference = imageReference;\n    this.allTargetImageTags = allTargetImageTags;\n  }\n\n  /**\n   * Writes image tar bar in configured {@link Image#getImageFormat()} of OCI or Docker to output\n   * stream.\n   *\n   * @param out the target output stream\n   * @throws IOException if an error occurs writing out the image to stream\n   */\n  public void writeTo(OutputStream out) throws IOException {\n    if (image.getImageFormat() == OciManifestTemplate.class) {\n      ociWriteTo(out);\n    } else {\n      dockerWriteTo(out);\n    }\n  }\n\n  private void ociWriteTo(OutputStream out) throws IOException {\n    TarStreamBuilder tarStreamBuilder = new TarStreamBuilder();\n    OciManifestTemplate manifest = new OciManifestTemplate();\n\n    // Adds all the layers to the tarball and manifest\n    for (Layer layer : image.getLayers()) {\n      DescriptorDigest digest = layer.getBlobDescriptor().getDigest();\n      long size = layer.getBlobDescriptor().getSize();\n\n      tarStreamBuilder.addBlobEntry(\n          layer.getBlob(), size, BLOB_PREFIX + digest.getHash(), TAR_ENTRY_MODIFICATION_TIME);\n      manifest.addLayer(size, digest);\n    }\n\n    // Adds the container configuration to the tarball and manifest\n    JsonTemplate containerConfiguration =\n        new ImageToJsonTranslator(image).getContainerConfiguration();\n    BlobDescriptor configDescriptor = Digests.computeDigest(containerConfiguration);\n    manifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest());\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(containerConfiguration),\n        BLOB_PREFIX + configDescriptor.getDigest().getHash(),\n        TAR_ENTRY_MODIFICATION_TIME);\n\n    // Adds the manifest to the tarball\n    BlobDescriptor manifestDescriptor = Digests.computeDigest(manifest);\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(manifest),\n        BLOB_PREFIX + manifestDescriptor.getDigest().getHash(),\n        TAR_ENTRY_MODIFICATION_TIME);\n\n    // Adds the oci-layout and index.json\n    tarStreamBuilder.addByteEntry(\n        \"{\\\"imageLayoutVersion\\\": \\\"1.0.0\\\"}\".getBytes(StandardCharsets.UTF_8),\n        \"oci-layout\",\n        TAR_ENTRY_MODIFICATION_TIME);\n    OciIndexTemplate index = new OciIndexTemplate();\n    // TODO: figure out how to tag with allTargetImageTags\n    index.addManifest(manifestDescriptor, imageReference.toStringWithQualifier());\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(index), \"index.json\", TAR_ENTRY_MODIFICATION_TIME);\n\n    tarStreamBuilder.writeAsTarArchiveTo(out);\n  }\n\n  private void dockerWriteTo(OutputStream out) throws IOException {\n    TarStreamBuilder tarStreamBuilder = new TarStreamBuilder();\n    DockerManifestEntryTemplate manifestTemplate = new DockerManifestEntryTemplate();\n\n    // Adds all the layers to the tarball and manifest.\n    for (Layer layer : image.getLayers()) {\n      String layerName = layer.getBlobDescriptor().getDigest().getHash() + LAYER_FILE_EXTENSION;\n\n      tarStreamBuilder.addBlobEntry(\n          layer.getBlob(),\n          layer.getBlobDescriptor().getSize(),\n          layerName,\n          TAR_ENTRY_MODIFICATION_TIME);\n      manifestTemplate.addLayerFile(layerName);\n    }\n\n    // Adds the container configuration to the tarball.\n    JsonTemplate containerConfiguration =\n        new ImageToJsonTranslator(image).getContainerConfiguration();\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(containerConfiguration),\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.ImageTarball;\nimport java.io.BufferedOutputStream;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.concurrent.Callable;\n\npublic class WriteTarFileStep implements Callable<BuildResult> {\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n\n  private final Path outputPath;\n  private final Image builtImage;\n\n  WriteTarFileStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Path outputPath,\n      Image builtImage) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.outputPath = outputPath;\n    this.builtImage = builtImage;\n  }\n\n  @Override\n  public BuildResult call() throws IOException {\n    buildContext.getEventHandlers().dispatch(LogEvent.progress(\"Building image to tar file...\"));\n\n    try (ProgressEventDispatcher ignored =\n        progressEventDispatcherFactory.create(\"writing to tar file\", 1)) {\n      // Builds the image to a tarball.\n      if (outputPath.getParent() != null) {\n        Files.createDirectories(outputPath.getParent());\n      }\n      try (OutputStream outputStream =\n          new BufferedOutputStream(Files.newOutputStream(outputPath))) {\n        new ImageTarball(\n                builtImage,\n                buildContext.getTargetImageConfiguration().getImage(),\n                buildContext.getAllTargetImageTags())\n            .writeTo(outputStream);\n      }\n\n      return BuildResult.fromImage(builtImage, buildContext.getTargetFormat());\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.image.Layer;\n\n/**\n * Layer prepared from {@link BuildAndCacheApplicationLayerStep} and {@link\n * ObtainBaseImageLayerStep} to hold information about either a base image layer or an application\n * layer.\n */\nclass PreparedLayer implements Layer {\n\n  enum StateInTarget {\n    UNKNOWN,\n    EXISTING,\n    MISSING\n  }\n\n  static class Builder {\n\n    private Layer layer;\n    private String name = \"unnamed layer\";\n    private StateInTarget stateInTarget = StateInTarget.UNKNOWN;\n\n    Builder(Layer layer) {\n      this.layer = layer;\n    }\n\n    Builder setName(String name) {\n      this.name = name;\n      return this;\n    }\n\n    /** Sets whether the layer exists in a target destination. */\n    Builder setStateInTarget(StateInTarget stateInTarget) {\n      this.stateInTarget = stateInTarget;\n      return this;\n    }\n\n    PreparedLayer build() {\n      return new PreparedLayer(layer, name, stateInTarget);\n    }\n  }\n\n  private final Layer layer;\n  private final String name;\n  private final StateInTarget stateInTarget;\n\n  private PreparedLayer(Layer layer, String name, StateInTarget stateInTarget) {\n    this.layer = layer;\n    this.name = name;\n    this.stateInTarget = stateInTarget;\n  }\n\n  String getName() {\n    return name;\n  }\n\n  StateInTarget getStateInTarget() {\n    return stateInTarget;\n  }\n\n  @Override\n  public Blob getBlob() {\n    return layer.getBlob();\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return layer.getBlobDescriptor();\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() {\n    return layer.getDiffId();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.builder.steps.PreparedLayer.StateInTarget;\nimport com.google.cloud.tools.jib.cache.Cache;\nimport com.google.cloud.tools.jib.cache.CacheCorruptedException;\nimport com.google.cloud.tools.jib.cache.CachedLayer;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.image.Layer;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport com.google.common.base.Verify;\nimport java.io.IOException;\nimport java.util.Optional;\nimport java.util.concurrent.Callable;\nimport javax.annotation.Nullable;\n\n/** Pulls and caches a single base image layer. */\nclass ObtainBaseImageLayerStep implements Callable<PreparedLayer> {\n\n  @SuppressWarnings(\"InlineFormatString\")\n  private static final String DESCRIPTION = \"Pulling base image layer %s\";\n\n  @FunctionalInterface\n  private interface BlobExistenceChecker {\n\n    StateInTarget check(DescriptorDigest digest) throws IOException, RegistryException;\n  }\n\n  static ObtainBaseImageLayerStep forForcedDownload(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Layer layer,\n      @Nullable RegistryClient sourceRegistryClient) {\n    BlobExistenceChecker noOpChecker = ignored -> StateInTarget.UNKNOWN;\n    return new ObtainBaseImageLayerStep(\n        buildContext, progressEventDispatcherFactory, layer, sourceRegistryClient, noOpChecker);\n  }\n\n  static ObtainBaseImageLayerStep forSelectiveDownload(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Layer layer,\n      @Nullable RegistryClient sourceRegistryClient,\n      RegistryClient targetRegistryClient) {\n    Verify.verify(!buildContext.isOffline());\n\n    // TODO: also check if cross-repo blob mount is possible.\n    BlobExistenceChecker blobExistenceChecker =\n        digest ->\n            targetRegistryClient.checkBlob(digest).isPresent()\n                ? StateInTarget.EXISTING\n                : StateInTarget.MISSING;\n\n    return new ObtainBaseImageLayerStep(\n        buildContext,\n        progressEventDispatcherFactory,\n        layer,\n        sourceRegistryClient,\n        blobExistenceChecker);\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n\n  private final Layer layer;\n  private final @Nullable RegistryClient registryClient;\n  private final BlobExistenceChecker blobExistenceChecker;\n\n  private ObtainBaseImageLayerStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Layer layer,\n      @Nullable RegistryClient registryClient,\n      BlobExistenceChecker blobExistenceChecker) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.layer = layer;\n    this.registryClient = registryClient;\n    this.blobExistenceChecker = blobExistenceChecker;\n  }\n\n  @Override\n  public PreparedLayer call() throws IOException, CacheCorruptedException, RegistryException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest();\n    try (ProgressEventDispatcher progressEventDispatcher =\n            progressEventDispatcherFactory.create(\"checking base image layer \" + layerDigest, 1);\n        TimerEventDispatcher ignored =\n            new TimerEventDispatcher(eventHandlers, String.format(DESCRIPTION, layerDigest))) {\n\n      StateInTarget stateInTarget = blobExistenceChecker.check(layerDigest);\n      if (stateInTarget == StateInTarget.EXISTING) {\n        eventHandlers.dispatch(\n            LogEvent.info(\n                \"Skipping pull; BLOB already exists on target registry : \"\n                    + layer.getBlobDescriptor()));\n        return new PreparedLayer.Builder(layer).setStateInTarget(stateInTarget).build();\n      }\n\n      Cache cache = buildContext.getBaseImageLayersCache();\n\n      // Checks if the layer already exists in the cache.\n      Optional<CachedLayer> optionalCachedLayer = cache.retrieve(layerDigest);\n      if (optionalCachedLayer.isPresent()) {\n        CachedLayer cachedLayer = optionalCachedLayer.get();\n        return new PreparedLayer.Builder(cachedLayer).setStateInTarget(stateInTarget).build();\n      } else if (buildContext.isOffline()) {\n        throw new IOException(\n            \"Cannot run Jib in offline mode; local Jib cache for base image is missing image layer \"\n                + layerDigest\n                + \". Rerun Jib in online mode with \\\"-Djib.alwaysCacheBaseImage=true\\\" to \"\n                + \"re-download the base image layers.\");\n      }\n\n      try (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper =\n          new ThrottledProgressEventDispatcherWrapper(\n              progressEventDispatcher.newChildProducer(),\n              \"pulling base image layer \" + layerDigest)) {\n        CachedLayer cachedLayer =\n            cache.writeCompressedLayer(\n                Verify.verifyNotNull(registryClient)\n                    .pullBlob(\n                        layerDigest,\n                        progressEventDispatcherWrapper::setProgressTarget,\n                        progressEventDispatcherWrapper::dispatchProgress));\n        return new PreparedLayer.Builder(cachedLayer).setStateInTarget(stateInTarget).build();\n      }\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/DockerClient.java\n/*\n * Copyright 2022 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.image.ImageTarball;\nimport java.io.IOException;\nimport java.nio.file.Path;\nimport java.util.Map;\nimport java.util.function.Consumer;\n\npublic interface DockerClient {\n\n  /**\n   * Validate if the DockerClient is supported.\n   *\n   * @param parameters to be used by the docker client\n   * @return true if conditions are met\n   */\n  boolean supported(Map<String, String> parameters);\n\n  /**\n   * Loads an image tarball into the Docker daemon.\n   *\n   * @see <a\n   *     href=\"https://docs.docker.com/engine/reference/commandline/load/\">https://docs.docker.com/engine/reference/commandline/load</a>\n   * @param imageTarball the built container tarball\n   * @param writtenByteCountListener callback to call when bytes are loaded\n   * @return stdout from {@code docker}\n   * @throws InterruptedException if the 'docker load' process is interrupted\n   * @throws IOException if streaming the blob to 'docker load' fails\n   */\n  String load(ImageTarball imageTarball, Consumer<Long> writtenByteCountListener)\n      throws InterruptedException, IOException;\n\n  /**\n   * Saves an image tarball from the Docker daemon.\n   *\n   * @see <a\n   *     href=\"https://docs.docker.com/engine/reference/commandline/save/\">https://docs.docker.com/engine/reference/commandline/save</a>\n   * @param imageReference the image to save\n   * @param outputPath the destination path to save the output tarball\n   * @param writtenByteCountListener callback to call when bytes are saved\n   * @throws InterruptedException if the 'docker save' process is interrupted\n   * @throws IOException if creating the tarball fails\n   */\n  void save(ImageReference imageReference, Path outputPath, Consumer<Long> writtenByteCountListener)\n      throws InterruptedException, IOException;\n\n  /**\n   * Gets the size, image ID, and diff IDs of an image in the Docker daemon.\n   *\n   * @param imageReference the image to inspect\n   * @return the size, image ID, and diff IDs of the image\n   * @throws IOException if an I/O exception occurs or {@code docker inspect} failed\n   * @throws InterruptedException if the {@code docker inspect} process was interrupted\n   */\n  ImageDetails inspect(ImageReference imageReference) throws IOException, InterruptedException;\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/ImageDetails.java\n/*\n * Copyright 2022 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport java.security.DigestException;\nimport java.util.List;\n\npublic interface ImageDetails {\n\n  long getSize();\n\n  DescriptorDigest getImageId() throws DigestException;\n\n  List<DescriptorDigest> getDiffIds() throws DigestException;\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/PlatformNotFoundInBaseImageException.java\n/*\n * Copyright 2022 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.RegistryException;\n\n/** Exception thrown when build target platforms are not found in the base image. */\npublic class PlatformNotFoundInBaseImageException extends RegistryException {\n\n  public PlatformNotFoundInBaseImageException(String message) {\n    super(message);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PlatformChecker.java\n/*\n * Copyright 2020 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.buildplan.Platform;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;\nimport com.google.cloud.tools.jib.image.json.PlatformNotFoundInBaseImageException;\nimport com.google.common.base.Verify;\nimport java.nio.file.Path;\nimport java.util.Optional;\nimport java.util.Set;\n\n/** Provides helper methods to check platforms. */\npublic class PlatformChecker {\n\n  private PlatformChecker() {}\n\n  /**\n   * Assuming the base image is not a manifest list, checks and warns misconfigured platforms.\n   *\n   * @param buildContext the {@link BuildContext}\n   * @param containerConfig container configuration JSON of the base image\n   */\n  \nstatic void checkManifestPlatform(\n      BuildContext buildContext, ContainerConfigurationTemplate containerConfig)\n      throws PlatformNotFoundInBaseImageException {\n    Optional<Path> path = buildContext.getBaseImageConfiguration().getTarPath();\n    String baseImageName =\n        path.map(Path::toString)\n            .orElse(buildContext.getBaseImageConfiguration().getImage().toString());\n\n    Set<Platform> platforms = buildContext.getContainerConfiguration().getPlatforms();\n    Verify.verify(!platforms.isEmpty());\n\n    if (platforms.size() != 1) {\n      String msg =\n          String.format(\n              \"cannot build for multiple platforms since the base image '%s' is not a manifest list.\",\n              baseImageName);\n      throw new PlatformNotFoundInBaseImageException(msg);\n    } else {\n      Platform platform = platforms.iterator().next();\n      if (!platform.getArchitecture().equals(containerConfig.getArchitecture())\n          || !platform.getOs().equals(containerConfig.getOs())) {\n\n        // Unfortunately, \"platforms\" has amd64/linux by default even if the user didn't explicitly\n        // configure it. Skip reporting to suppress false alarm.\n        if (!(platform.getArchitecture().equals(\"amd64\") && platform.getOs().equals(\"linux\"))) {\n          String msg =\n              String.format(\n                  \"the configured platform (%s/%s) doesn't match the platform (%s/%s) of the base image (%s)\",\n                  platform.getArchitecture(),\n                  platform.getOs(),\n                  containerConfig.getArchitecture(),\n                  containerConfig.getOs(),\n                  baseImageName);\n          throw new PlatformNotFoundInBaseImageException(msg);\n        }\n      }\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/CredentialRetrievalException.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.registry.credentials;\n\nimport com.google.cloud.tools.jib.api.CredentialRetriever;\n\n/** Thrown if something went wrong during {@link CredentialRetriever#retrieve}. */\npublic class CredentialRetrievalException extends Exception {\n\n  CredentialRetrievalException(String message, Throwable cause) {\n    super(message, cause);\n  }\n\n  CredentialRetrievalException(String message) {\n    super(message);\n  }\n\n  public CredentialRetrievalException(Throwable cause) {\n    super(cause);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/CredentialRetriever.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException;\nimport java.util.Optional;\n\n/** Retrieves credentials for a registry. */\n@FunctionalInterface\npublic interface CredentialRetriever {\n\n  /**\n   * Fetches the credentials. <b>Implementations must be thread-safe.</b>\n   *\n   * <p>Implementations should return {@link Optional#empty} if no credentials could be fetched with\n   * this {@link CredentialRetriever} (and so other credential retrieval methods may be tried), or\n   * throw an exception something went wrong when fetching the credentials.\n   *\n   * @return the fetched credentials or {@link Optional#empty} if no credentials could be fetched\n   *     with this provider\n   * @throws CredentialRetrievalException if the credential retrieval encountered an exception\n   */\n  Optional<Credential> retrieve() throws CredentialRetrievalException;\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/configuration/ImageConfiguration.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.configuration;\n\nimport com.google.cloud.tools.jib.api.CredentialRetriever;\nimport com.google.cloud.tools.jib.api.DockerClient;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ImmutableList;\nimport java.nio.file.Path;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.Optional;\nimport javax.annotation.Nullable;\n\n/** Immutable configuration options for a base or target image reference. */\npublic class ImageConfiguration {\n\n  /** Builder for instantiating an {@link ImageConfiguration}. */\n  public static class Builder {\n\n    private ImageReference imageReference;\n    private ImmutableList<CredentialRetriever> credentialRetrievers = ImmutableList.of();\n    @Nullable private DockerClient dockerClient;\n    @Nullable private Path tarPath;\n\n    /**\n     * Sets the providers for registry credentials. The order determines the priority in which the\n     * retrieval methods are attempted.\n     *\n     * @param credentialRetrievers the list of {@link CredentialRetriever}s\n     * @return this\n     */\n    public Builder setCredentialRetrievers(List<CredentialRetriever> credentialRetrievers) {\n      Preconditions.checkArgument(\n          credentialRetrievers.stream().allMatch(Objects::nonNull),\n          \"credential retriever list contains null elements\");\n      this.credentialRetrievers = ImmutableList.copyOf(credentialRetrievers);\n      return this;\n    }\n\n    /**\n     * Sets the Docker client to be used for Docker daemon base images.\n     *\n     * @param dockerClient the Docker client\n     * @return this\n     */\n    public Builder setDockerClient(DockerClient dockerClient) {\n      this.dockerClient = dockerClient;\n      return this;\n    }\n\n    /**\n     * Sets the path for tarball base images.\n     *\n     * @param tarPath the path\n     * @return this\n     */\n    public Builder setTarPath(Path tarPath) {\n      this.tarPath = tarPath;\n      return this;\n    }\n\n    /**\n     * Builds the {@link ImageConfiguration}.\n     *\n     * @return the corresponding {@link ImageConfiguration}\n     */\n    public ImageConfiguration build() {\n      int numArguments = 0;\n      if (!credentialRetrievers.isEmpty()) {\n        numArguments++;\n      }\n      if (dockerClient != null) {\n        numArguments++;\n      }\n      if (tarPath != null) {\n        numArguments++;\n      }\n      Preconditions.checkArgument(numArguments <= 1);\n      return new ImageConfiguration(imageReference, credentialRetrievers, dockerClient, tarPath);\n    }\n\n    private Builder(ImageReference imageReference) {\n      this.imageReference = imageReference;\n    }\n  }\n\n  /**\n   * Constructs a builder for an {@link ImageConfiguration}.\n   *\n   * @param imageReference the image reference, which is a required field\n   * @return the builder\n   */\n  public static Builder builder(ImageReference imageReference) {\n    return new Builder(imageReference);\n  }\n\n  private final ImageReference image;\n  private final ImmutableList<CredentialRetriever> credentialRetrievers;\n  @Nullable private DockerClient dockerClient;\n  @Nullable private Path tarPath;\n\n  private ImageConfiguration(\n      ImageReference image,\n      ImmutableList<CredentialRetriever> credentialRetrievers,\n      @Nullable DockerClient dockerClient,\n      @Nullable Path tarPath) {\n    this.image = image;\n    this.credentialRetrievers = credentialRetrievers;\n    this.dockerClient = dockerClient;\n    this.tarPath = tarPath;\n  }\n\n  public ImageReference getImage() {\n    return image;\n  }\n\n  public String getImageRegistry() {\n    return image.getRegistry();\n  }\n\n  public String getImageRepository() {\n    return image.getRepository();\n  }\n\n  public String getImageQualifier() {\n    return image.getQualifier();\n  }\n\n  public ImmutableList<CredentialRetriever> getCredentialRetrievers() {\n    return credentialRetrievers;\n  }\n\n  public Optional<DockerClient> getDockerClient() {\n    return Optional.ofNullable(dockerClient);\n  }\n\n  public Optional<Path> getTarPath() {\n    return Optional.ofNullable(tarPath);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RegistryCredentialRetriever.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.Credential;\nimport com.google.cloud.tools.jib.api.CredentialRetriever;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ImageConfiguration;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException;\nimport java.util.Optional;\n\n/** Attempts to retrieve registry credentials. */\nclass RegistryCredentialRetriever {\n\n  private RegistryCredentialRetriever() {}\n\n  /** Retrieves credentials for the base image. */\n  static Optional<Credential> getBaseImageCredential(BuildContext buildContext)\n      throws CredentialRetrievalException {\n    return retrieve(buildContext.getBaseImageConfiguration(), buildContext.getEventHandlers());\n  }\n\n  /** Retrieves credentials for the target image. */\n  static Optional<Credential> getTargetImageCredential(BuildContext buildContext)\n      throws CredentialRetrievalException {\n    return retrieve(buildContext.getTargetImageConfiguration(), buildContext.getEventHandlers());\n  }\n\n  private static Optional<Credential> retrieve(\n      ImageConfiguration imageConfiguration, EventHandlers eventHandlers)\n      throws CredentialRetrievalException {\n    for (CredentialRetriever retriever : imageConfiguration.getCredentialRetrievers()) {\n      Optional<Credential> credential = retriever.retrieve();\n      if (credential.isPresent()) {\n        return credential;\n      }\n    }\n\n    String registry = imageConfiguration.getImageRegistry();\n    String repository = imageConfiguration.getImageRepository();\n    eventHandlers.dispatch(\n        LogEvent.info(\"No credentials could be retrieved for \" + registry + \"/\" + repository));\n    return Optional.empty();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerCountMismatchException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\n/** Exception thrown when the number of layers found did not match expectations. */\npublic class LayerCountMismatchException extends Exception {\n\n  public LayerCountMismatchException(String message) {\n    super(message);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BadContainerConfigurationFormatException.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\n/** Exception thrown when trying to parse a bad image configuration format. */\npublic class BadContainerConfigurationFormatException extends Exception {\n\n  // TODO: Potentially provide Path or source object to problem configuration file\n  BadContainerConfigurationFormatException(String message) {\n    super(message);\n  }\n\n  BadContainerConfigurationFormatException(String message, Throwable cause) {\n    super(message, cause);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/DigestOnlyLayer.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\n\n/** A {@link Layer} reference that only has its {@link DescriptorDigest}. */\npublic class DigestOnlyLayer implements Layer {\n\n  /** The {@link BlobDescriptor} of the compressed layer content. */\n  private final BlobDescriptor blobDescriptor;\n\n  /**\n   * Instantiate with a {@link DescriptorDigest}.\n   *\n   * @param digest the digest to instantiate the {@link DigestOnlyLayer} from\n   */\n  public DigestOnlyLayer(DescriptorDigest digest) {\n    blobDescriptor = new BlobDescriptor(digest);\n  }\n\n  @Override\n  public Blob getBlob() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\"Blob not available for digest-only layer\");\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return blobDescriptor;\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\"Diff ID not available for digest-only layer\");\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/ReferenceLayer.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\n\n/**\n * A {@link Layer} reference that <b>does not</b> have the underlying content. It references the\n * layer with its digest, size, and diff ID.\n */\npublic class ReferenceLayer implements Layer {\n\n  /** The {@link BlobDescriptor} of the compressed layer content. */\n  private final BlobDescriptor blobDescriptor;\n\n  /** The digest of the uncompressed layer content. */\n  private final DescriptorDigest diffId;\n\n  /**\n   * Instantiate with a {@link BlobDescriptor} and diff ID.\n   *\n   * @param blobDescriptor the blob descriptor\n   * @param diffId the diff ID\n   */\n  public ReferenceLayer(BlobDescriptor blobDescriptor, DescriptorDigest diffId) {\n    this.blobDescriptor = blobDescriptor;\n    this.diffId = diffId;\n  }\n\n  @Override\n  public Blob getBlob() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\"Blob not available for reference layer\");\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return blobDescriptor;\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() {\n    return diffId;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/ReferenceNoDiffIdLayer.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\n\n/**\n * A {@link Layer} reference that <b>does not</b> have the underlying content. It references the\n * layer with its digest and size, but <b>not</b> its diff ID.\n */\npublic class ReferenceNoDiffIdLayer implements Layer {\n\n  /** The {@link BlobDescriptor} of the compressed layer content. */\n  private final BlobDescriptor blobDescriptor;\n\n  /**\n   * Instantiate with a {@link BlobDescriptor} and no diff ID.\n   *\n   * @param blobDescriptor the blob descriptor\n   */\n  public ReferenceNoDiffIdLayer(BlobDescriptor blobDescriptor) {\n    this.blobDescriptor = blobDescriptor;\n  }\n\n  @Override\n  public Blob getBlob() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\n        \"Blob not available for reference layer without diff ID\");\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return blobDescriptor;\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\n        \"Diff ID not available for reference layer without diff ID\");\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/JsonToImageTranslator.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath;\nimport com.google.cloud.tools.jib.api.buildplan.Port;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.configuration.DockerHealthCheck;\nimport com.google.cloud.tools.jib.image.DigestOnlyLayer;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.LayerCountMismatchException;\nimport com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;\nimport com.google.cloud.tools.jib.image.ReferenceLayer;\nimport com.google.cloud.tools.jib.image.ReferenceNoDiffIdLayer;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Lists;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.time.format.DateTimeFormatter;\nimport java.time.format.DateTimeParseException;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport javax.annotation.Nullable;\n\n/** Translates {@link V21ManifestTemplate} and {@link V22ManifestTemplate} into {@link Image}. */\npublic class JsonToImageTranslator {\n\n  /**\n   * Pattern used for parsing information out of exposed port configurations. Only accepts single\n   * ports with protocol.\n   *\n   * <p>Example matches: 100, 1000/tcp, 2000/udp\n   */\n  private static final Pattern PORT_PATTERN =\n      Pattern.compile(\"(?<portNum>\\\\d+)(?:/(?<protocol>tcp|udp))?\");\n\n  /**\n   * Pattern used for parsing environment variables in the format {@code NAME=VALUE}. {@code NAME}\n   * should not contain an '='.\n   *\n   * <p>Example matches: NAME=VALUE, A12345=$$$$$\n   */\n  @VisibleForTesting\n  static final Pattern ENVIRONMENT_PATTERN = Pattern.compile(\"(?<name>[^=]+)=(?<value>.*)\");\n\n  /**\n   * Translates {@link V21ManifestTemplate} to {@link Image}.\n   *\n   * @param manifestTemplate the template containing the image layers.\n   * @return the translated {@link Image}.\n   * @throws LayerPropertyNotFoundException if adding image layers fails.\n   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad\n   *     format\n   */\n  public static Image toImage(V21ManifestTemplate manifestTemplate)\n      throws LayerPropertyNotFoundException, BadContainerConfigurationFormatException {\n    Image.Builder imageBuilder = Image.builder(V21ManifestTemplate.class);\n\n    // V21 layers are in reverse order of V22. (The first layer is the latest one.)\n    for (DescriptorDigest digest : Lists.reverse(manifestTemplate.getLayerDigests())) {\n      imageBuilder.addLayer(new DigestOnlyLayer(digest));\n    }\n\n    Optional<ContainerConfigurationTemplate> configuration =\n        manifestTemplate.getContainerConfiguration();\n    if (configuration.isPresent()) {\n      configureBuilderWithContainerConfiguration(imageBuilder, configuration.get());\n    }\n    return imageBuilder.build();\n  }\n\n  /**\n   * Translates {@link BuildableManifestTemplate} to {@link Image}. Uses the corresponding {@link\n   * ContainerConfigurationTemplate} to get the layer diff IDs.\n   *\n   * @param manifestTemplate the template containing the image layers.\n   * @param containerConfigurationTemplate the template containing the diff IDs and container\n   *     configuration properties.\n   * @return the translated {@link Image}.\n   * @throws LayerCountMismatchException if the manifest and configuration contain conflicting layer\n   *     information.\n   * @throws LayerPropertyNotFoundException if adding image layers fails.\n   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad\n   *     format\n   */\n  public static Image toImage(\n      BuildableManifestTemplate manifestTemplate,\n      ContainerConfigurationTemplate containerConfigurationTemplate)\n      throws LayerCountMismatchException, LayerPropertyNotFoundException,\n          BadContainerConfigurationFormatException {\n    List<ReferenceNoDiffIdLayer> layers = new ArrayList<>();\n    for (BuildableManifestTemplate.ContentDescriptorTemplate layerObjectTemplate :\n        manifestTemplate.getLayers()) {\n      if (layerObjectTemplate.getDigest() == null) {\n        throw new IllegalArgumentException(\n            \"All layers in the manifest template must have digest set\");\n      }\n\n      layers.add(\n          new ReferenceNoDiffIdLayer(\n              new BlobDescriptor(layerObjectTemplate.getSize(), layerObjectTemplate.getDigest())));\n    }\n\n    List<DescriptorDigest> diffIds = containerConfigurationTemplate.getDiffIds();\n    if (layers.size() != diffIds.size()) {\n      throw new LayerCountMismatchException(\n          \"Mismatch between image manifest and container configuration\");\n    }\n\n    Image.Builder imageBuilder = Image.builder(manifestTemplate.getClass());\n\n    for (int layerIndex = 0; layerIndex < layers.size(); layerIndex++) {\n      ReferenceNoDiffIdLayer noDiffIdLayer = layers.get(layerIndex);\n      DescriptorDigest diffId = diffIds.get(layerIndex);\n\n      imageBuilder.addLayer(new ReferenceLayer(noDiffIdLayer.getBlobDescriptor(), diffId));\n    }\n\n    configureBuilderWithContainerConfiguration(imageBuilder, containerConfigurationTemplate);\n    return imageBuilder.build();\n  }\n\n  private static void configureBuilderWithContainerConfiguration(\n      Image.Builder imageBuilder, ContainerConfigurationTemplate containerConfigurationTemplate)\n      throws BadContainerConfigurationFormatException {\n\n    containerConfigurationTemplate.getHistory().forEach(imageBuilder::addHistory);\n\n    if (containerConfigurationTemplate.getCreated() != null) {\n      try {\n        imageBuilder.setCreated(Instant.parse(containerConfigurationTemplate.getCreated()));\n      } catch (DateTimeParseException ex) {\n        try {\n          // TODO: remove when using Java >= 12.\n          // See https://github.com/GoogleContainerTools/jib/issues/2428\n          imageBuilder.setCreated(\n              DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(\n                  containerConfigurationTemplate.getCreated(), Instant::from));\n        } catch (DateTimeParseException ignored) {\n          throw new BadContainerConfigurationFormatException(\n              \"Invalid image creation time: \" + containerConfigurationTemplate.getCreated(), ex);\n        }\n      }\n    }\n\n    if (containerConfigurationTemplate.getArchitecture() != null) {\n      imageBuilder.setArchitecture(containerConfigurationTemplate.getArchitecture());\n    }\n    if (containerConfigurationTemplate.getOs() != null) {\n      imageBuilder.setOs(containerConfigurationTemplate.getOs());\n    }\n\n    imageBuilder.setEntrypoint(containerConfigurationTemplate.getContainerEntrypoint());\n    imageBuilder.setProgramArguments(containerConfigurationTemplate.getContainerCmd());\n\n    List<String> baseHealthCheckCommand = containerConfigurationTemplate.getContainerHealthTest();\n    if (baseHealthCheckCommand != null) {\n      DockerHealthCheck.Builder builder = DockerHealthCheck.fromCommand(baseHealthCheckCommand);\n      if (containerConfigurationTemplate.getContainerHealthInterval() != null) {\n        builder.setInterval(\n            Duration.ofNanos(containerConfigurationTemplate.getContainerHealthInterval()));\n      }\n      if (containerConfigurationTemplate.getContainerHealthTimeout() != null) {\n        builder.setTimeout(\n            Duration.ofNanos(containerConfigurationTemplate.getContainerHealthTimeout()));\n      }\n      if (containerConfigurationTemplate.getContainerHealthStartPeriod() != null) {\n        builder.setStartPeriod(\n            Duration.ofNanos(containerConfigurationTemplate.getContainerHealthStartPeriod()));\n      }\n      if (containerConfigurationTemplate.getContainerHealthRetries() != null) {\n        builder.setRetries(containerConfigurationTemplate.getContainerHealthRetries());\n      }\n      imageBuilder.setHealthCheck(builder.build());\n    }\n\n    if (containerConfigurationTemplate.getContainerExposedPorts() != null) {\n      imageBuilder.addExposedPorts(\n          portMapToSet(containerConfigurationTemplate.getContainerExposedPorts()));\n    }\n\n    if (containerConfigurationTemplate.getContainerVolumes() != null) {\n      imageBuilder.addVolumes(volumeMapToSet(containerConfigurationTemplate.getContainerVolumes()));\n    }\n\n    if (containerConfigurationTemplate.getContainerEnvironment() != null) {\n      for (String environmentVariable : containerConfigurationTemplate.getContainerEnvironment()) {\n        Matcher matcher = ENVIRONMENT_PATTERN.matcher(environmentVariable);\n        if (!matcher.matches()) {\n          throw new BadContainerConfigurationFormatException(\n              \"Invalid environment variable definition: \" + environmentVariable);\n        }\n        imageBuilder.addEnvironmentVariable(matcher.group(\"name\"), matcher.group(\"value\"));\n      }\n    }\n\n    imageBuilder.addLabels(containerConfigurationTemplate.getContainerLabels());\n    imageBuilder.setWorkingDirectory(containerConfigurationTemplate.getContainerWorkingDir());\n    imageBuilder.setUser(containerConfigurationTemplate.getContainerUser());\n  }\n\n  /**\n   * Converts a map of exposed ports as strings to a set of {@link Port}s (e.g. {@code\n   * {\"1000/tcp\":{}}} -> {@code Port(1000, Protocol.TCP)}).\n   *\n   * @param portMap the map to convert\n   * @return a set of {@link Port}s\n   */\n  @VisibleForTesting\n  static ImmutableSet<Port> portMapToSet(@Nullable Map<String, Map<String, String>> portMap)\n      throws BadContainerConfigurationFormatException {\n    if (portMap == null) {\n      return ImmutableSet.of();\n    }\n    ImmutableSet.Builder<Port> ports = new ImmutableSet.Builder<>();\n    for (Map.Entry<String, Map<String, String>> entry : portMap.entrySet()) {\n      String port = entry.getKey();\n      Matcher matcher = PORT_PATTERN.matcher(port);\n      if (!matcher.matches()) {\n        throw new BadContainerConfigurationFormatException(\n            \"Invalid port configuration: '\" + port + \"'.\");\n      }\n\n      int portNumber = Integer.parseInt(matcher.group(\"portNum\"));\n      String protocol = matcher.group(\"protocol\");\n      ports.add(Port.parseProtocol(portNumber, protocol));\n    }\n    return ports.build();\n  }\n\n  /**\n   * Converts a map of volumes strings to a set of {@link AbsoluteUnixPath}s (e.g. {@code\n   * {\"/var/log/my-app-logs\":{}}} -> {@code AbsoluteUnixPath().get(\"/var/log/my-app-logs\")}).\n   *\n   * @param volumeMap the map to convert\n   * @return a set of {@link AbsoluteUnixPath}s\n   */\n  @VisibleForTesting\n  static ImmutableSet<AbsoluteUnixPath> volumeMapToSet(\n      @Nullable Map<String, Map<String, String>> volumeMap)\n      throws BadContainerConfigurationFormatException {\n    if (volumeMap == null) {\n      return ImmutableSet.of();\n    }\n\n    ImmutableSet.Builder<AbsoluteUnixPath> volumeList = ImmutableSet.builder();\n    for (String volume : volumeMap.keySet()) {\n      try {\n        volumeList.add(AbsoluteUnixPath.get(volume));\n      } catch (IllegalArgumentException exception) {\n        throw new BadContainerConfigurationFormatException(\"Invalid volume path: \" + volume);\n      }\n    }\n\n    return volumeList.build();\n  }\n\n  private JsonToImageTranslator() {}\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java\n/*\n * Copyright 2022 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n...\n", "needle_token_start": 6942, "needle_token_end": 7363, "code_context_ntokens": 15893, "output": ["```java\nstatic void checkManifestPlatform(\n      BuildContext buildContext, ContainerConfigurationTemplate containerConfig)\n      throws PlatformNotFoundInBaseImageException {\n    Optional<Path> path = buildContext.getBaseImageConfiguration().getTarPath();\n    String baseImageName =\n        path.map(Path::toString)\n            .orElse(buildContext.getBaseImageConfiguration().getImage().toString());\n\n    Set<Platform> platforms = buildContext.getContainerConfiguration().getPlatforms();\n    Verify.verify(!platforms.isEmpty());\n\n    if (platforms.size() != 1) {\n      String msg =\n          String.format(\n              \"cannot build for multiple platforms since the base image '%s' is not a manifest list.\",\n              baseImageName);\n      throw new PlatformNotFoundInBaseImageException(msg);\n    } else {\n      Platform platform = platforms.iterator().next();\n      if (!platform.getArchitecture().equals(containerConfig.getArchitecture())\n          || !platform.getOs().equals(containerConfig.getOs())) {\n\n        // Unfortunately, \"platforms\" has amd64/linux by default even if the user didn't explicitly\n        // configure it. Skip reporting to suppress false alarm.\n        if (!(platform.getArchitecture().equals(\"amd64\") && platform.getOs().equals(\"linux\"))) {\n          String msg =\n              String.format(\n                  \"the configured platform (%s/%s) doesn't match the platform (%s/%s) of the base image (%s)\",\n                  platform.getArchitecture(),\n                  platform.getOs(),\n                  containerConfig.getArchitecture(),\n                  containerConfig.getOs(),\n                  baseImageName);\n          throw new PlatformNotFoundInBaseImageException(msg);\n        }\n      }\n    }\n  }\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "setConfig", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/docker/json/DockerManifestEntryTemplate.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To update the configuration file name for a Docker manifest entry.\n2. **Input**: A string representing the new configuration file name.\n3. **Output**: None; the function modifies the state of the object.\n4. **Procedure**: The method takes a string input and assigns it to the configuration file name property of the Docker manifest entry object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/configuration/ContainerConfiguration.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.configuration;\n\nimport com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath;\nimport com.google.cloud.tools.jib.api.buildplan.Platform;\nimport com.google.cloud.tools.jib.api.buildplan.Port;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Iterables;\nimport java.time.Instant;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.stream.Collectors;\nimport javax.annotation.Nullable;\n\n/** Immutable configuration options for the container. */\npublic class ContainerConfiguration {\n\n  /** Builder for instantiating a {@link ContainerConfiguration}. */\n  public static class Builder {\n\n    /**\n     * The default creation time of the container (constant to ensure reproducibility by default).\n     */\n    private static final Instant DEFAULT_CREATION_TIME = Instant.EPOCH;\n\n    // LinkedHashSet to preserve the order\n    private Set<Platform> platforms =\n        new LinkedHashSet<>(Collections.singleton(new Platform(\"amd64\", \"linux\")));\n    private Instant creationTime = DEFAULT_CREATION_TIME;\n    @Nullable private ImmutableList<String> entrypoint;\n    @Nullable private ImmutableList<String> programArguments;\n    @Nullable private Map<String, String> environmentMap;\n    @Nullable private Set<Port> exposedPorts;\n    @Nullable private Set<AbsoluteUnixPath> volumes;\n    @Nullable private Map<String, String> labels;\n    @Nullable private String user;\n    @Nullable private AbsoluteUnixPath workingDirectory;\n\n    /**\n     * Sets a desired platform (properties including OS and architecture) list. If the base image\n     * reference is a Docker manifest list or an OCI image index, an image builder may select the\n     * base images matching the given platforms. If the base image reference is an image manifest,\n     * an image builder may ignore the given platforms and use the platform of the base image or may\n     * decide to raise on error.\n     *\n     * <p>Note that a new container configuration starts with \"amd64/linux\" as the default platform.\n     *\n     * @param platforms list of platforms to select base images in case of a manifest list\n     * @return this\n     */\n    public Builder setPlatforms(Set<Platform> platforms) {\n      Preconditions.checkArgument(!platforms.isEmpty(), \"platforms set cannot be empty\");\n      this.platforms = new LinkedHashSet<>(platforms);\n      return this;\n    }\n\n    /**\n     * Adds a desired image platform (OS and architecture pair). If the base image reference is a\n     * Docker manifest list or an OCI image index, an image builder may select the base image\n     * matching the given platform. If the base image reference is an image manifest, an image\n     * builder may ignore the given platform and use the platform of the base image or may decide to\n     * raise on error.\n     *\n     * <p>Note that a new container configuration starts with \"amd64/linux\" as the default platform.\n     * If you want to reset the default platform instead of adding a new one, use {@link\n     * #setPlatforms(Set)}.\n     *\n     * @param architecture architecture (for example, {@code amd64}) to select a base image in case\n     *     of a manifest list\n     * @param os OS (for example, {@code linux}) to select a base image in case of a manifest list\n     * @return this\n     */\n    public Builder addPlatform(String architecture, String os) {\n      platforms.add(new Platform(architecture, os));\n      return this;\n    }\n\n    /**\n     * Sets the image creation time.\n     *\n     * @param creationTime the creation time\n     * @return this\n     */\n    public Builder setCreationTime(Instant creationTime) {\n      this.creationTime = creationTime;\n      return this;\n    }\n\n    /**\n     * Sets the commandline arguments for main.\n     *\n     * @param programArguments the list of arguments\n     * @return this\n     */\n    public Builder setProgramArguments(@Nullable List<String> programArguments) {\n      if (programArguments == null) {\n        this.programArguments = null;\n      } else {\n        Preconditions.checkArgument(\n            programArguments.stream().allMatch(Objects::nonNull),\n            \"program arguments list contains null elements\");\n        this.programArguments = ImmutableList.copyOf(programArguments);\n      }\n      return this;\n    }\n\n    /**\n     * Sets the container's environment variables, mapping variable name to value.\n     *\n     * @param environmentMap the map\n     * @return this\n     */\n    public Builder setEnvironment(@Nullable Map<String, String> environmentMap) {\n      if (environmentMap == null) {\n        this.environmentMap = null;\n      } else {\n        Preconditions.checkArgument(\n            !Iterables.any(environmentMap.keySet(), Objects::isNull),\n            \"environment map contains null keys\");\n        String nullValuedKeys =\n            environmentMap.entrySet().stream()\n                .filter(entry -> entry.getValue() == null)\n                .map(Map.Entry::getKey)\n                .collect(Collectors.joining(\", \"));\n        Preconditions.checkArgument(\n            nullValuedKeys.isEmpty(),\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\n/** Exception thrown when accessing non-existent properties of layers. */\npublic class LayerPropertyNotFoundException extends RuntimeException {\n\n  LayerPropertyNotFoundException(String message) {\n    super(message);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ContainerConfiguration;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;\nimport com.google.cloud.tools.jib.image.json.HistoryEntry;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.UnmodifiableIterator;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.Callable;\nimport javax.annotation.Nullable;\n\n/** Builds a model {@link Image}. */\nclass BuildImageStep implements Callable<Image> {\n\n  private static final String DESCRIPTION = \"Building container configuration\";\n\n  @VisibleForTesting\n  static String truncateLongClasspath(ImmutableList<String> imageEntrypoint) {\n    List<String> truncated = new ArrayList<>();\n    UnmodifiableIterator<String> iterator = imageEntrypoint.iterator();\n    while (iterator.hasNext()) {\n      String element = iterator.next();\n      truncated.add(element);\n\n      if (element.equals(\"-cp\") || element.equals(\"-classpath\")) {\n        String classpath = iterator.next();\n        if (classpath.length() > 200) {\n          truncated.add(classpath.substring(0, 200) + \"<... classpath truncated ...>\");\n        } else {\n          truncated.add(classpath);\n        }\n      }\n    }\n    return truncated.toString();\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n  private final Image baseImage;\n  private final List<PreparedLayer> baseImageLayers;\n  private final List<PreparedLayer> applicationLayers;\n\n  BuildImageStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Image baseImage,\n      List<PreparedLayer> baseImageLayers,\n      List<PreparedLayer> applicationLayers) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.baseImage = baseImage;\n    this.baseImageLayers = baseImageLayers;\n    this.applicationLayers = applicationLayers;\n  }\n\n  @Override\n  public Image call() throws LayerPropertyNotFoundException {\n    try (ProgressEventDispatcher ignored =\n            progressEventDispatcherFactory.create(\"building image format\", 1);\n        TimerEventDispatcher ignored2 =\n            new TimerEventDispatcher(buildContext.getEventHandlers(), DESCRIPTION)) {\n      // Constructs the image.\n      Image.Builder imageBuilder = Image.builder(buildContext.getTargetFormat());\n\n      // Base image layers\n      baseImageLayers.forEach(imageBuilder::addLayer);\n\n      // Passthrough config and count non-empty history entries\n      int nonEmptyLayerCount = 0;\n      for (HistoryEntry historyObject : baseImage.getHistory()) {\n        imageBuilder.addHistory(historyObject);\n        if (!historyObject.hasCorrespondingLayer()) {\n          nonEmptyLayerCount++;\n        }\n      }\n      imageBuilder\n          .setArchitecture(baseImage.getArchitecture())\n          .setOs(baseImage.getOs())\n          .addEnvironment(baseImage.getEnvironment())\n          .addLabels(baseImage.getLabels())\n          .setHealthCheck(baseImage.getHealthCheck())\n          .addExposedPorts(baseImage.getExposedPorts())\n          .addVolumes(baseImage.getVolumes())\n          .setUser(baseImage.getUser())\n          .setWorkingDirectory(baseImage.getWorkingDirectory());\n\n      ContainerConfiguration containerConfiguration = buildContext.getContainerConfiguration();\n      // Add history elements for non-empty layers that don't have one yet\n      Instant layerCreationTime = containerConfiguration.getCreationTime();\n      for (int count = 0; count < baseImageLayers.size() - nonEmptyLayerCount; count++) {\n        imageBuilder.addHistory(\n            HistoryEntry.builder()\n                .setCreationTimestamp(layerCreationTime)\n                .setComment(\"auto-generated by Jib\")\n                .build());\n      }\n\n      // Add built layers/configuration\n      for (PreparedLayer applicationLayer : applicationLayers) {\n        imageBuilder\n            .addLayer(applicationLayer)\n            .addHistory(\n                HistoryEntry.builder()\n                    .setCreationTimestamp(layerCreationTime)\n                    .setAuthor(\"Jib\")\n                    .setCreatedBy(buildContext.getToolName() + \":\" + buildContext.getToolVersion())\n                    .setComment(applicationLayer.getName())\n                    .build());\n      }\n\n      imageBuilder\n          .addEnvironment(containerConfiguration.getEnvironmentMap())\n          .setCreated(containerConfiguration.getCreationTime())\n          .setEntrypoint(computeEntrypoint(baseImage, containerConfiguration))\n          .setProgramArguments(computeProgramArguments(baseImage, containerConfiguration))\n          .addExposedPorts(containerConfiguration.getExposedPorts())\n          .addVolumes(containerConfiguration.getVolumes())\n          .addLabels(containerConfiguration.getLabels());\n      if (containerConfiguration.getUser() != null) {\n        imageBuilder.setUser(containerConfiguration.getUser());\n      }\n      if (containerConfiguration.getWorkingDirectory() != null) {\n        imageBuilder.setWorkingDirectory(containerConfiguration.getWorkingDirectory().toString());\n      }\n\n      // Gets the container configuration content descriptor.\n      return imageBuilder.build();\n    }\n  }\n\n  /**\n   * Computes the image entrypoint. If {@link ContainerConfiguration#getEntrypoint()} is null, the\n   * entrypoint is inherited from the base image. Otherwise {@link\n   * ContainerConfiguration#getEntrypoint()} is returned.\n   *\n   * @param baseImage the base image\n   * @param containerConfiguration the container configuration\n   * @return the container entrypoint\n   */\n  @Nullable\n  private ImmutableList<String> computeEntrypoint(\n      Image baseImage, ContainerConfiguration containerConfiguration) {\n    boolean shouldInherit =\n        baseImage.getEntrypoint() != null && containerConfiguration.getEntrypoint() == null;\n\n    ImmutableList<String> entrypointToUse =\n        shouldInherit ? baseImage.getEntrypoint() : containerConfiguration.getEntrypoint();\n\n    if (entrypointToUse != null) {\n      buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(\"\"));\n      if (shouldInherit) {\n        String message =\n            \"Container entrypoint set to \" + entrypointToUse + \" (inherited from base image)\";\n        buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(message));\n      } else {\n        String message = \"Container entrypoint set to \" + truncateLongClasspath(entrypointToUse);\n        buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(message));\n      }\n    }\n\n    return entrypointToUse;\n  }\n\n  /**\n   * Computes the image program arguments. If {@link ContainerConfiguration#getEntrypoint()} and\n   * {@link ContainerConfiguration#getProgramArguments()} are null, the program arguments are\n   * inherited from the base image. Otherwise {@link ContainerConfiguration#getProgramArguments()}\n   * is returned.\n   *\n   * @param baseImage the base image\n   * @param containerConfiguration the container configuration\n   * @return the container program arguments\n   */\n  @Nullable\n  private ImmutableList<String> computeProgramArguments(\n      Image baseImage, ContainerConfiguration containerConfiguration) {\n    boolean shouldInherit =\n        baseImage.getProgramArguments() != null\n            // Inherit CMD only when inheriting ENTRYPOINT.\n            && containerConfiguration.getEntrypoint() == null\n            && containerConfiguration.getProgramArguments() == null;\n\n    ImmutableList<String> programArgumentsToUse =\n        shouldInherit\n            ? baseImage.getProgramArguments()\n            : containerConfiguration.getProgramArguments();\n\n    if (programArgumentsToUse != null) {\n      String logSuffix = shouldInherit ? \" (inherited from base image)\" : \"\";\n      String message = \"Container program arguments set to \" + programArgumentsToUse + logSuffix;\n      buildContext.getEventHandlers().dispatch(LogEvent.lifecycle(message));\n    }\n\n    return programArgumentsToUse;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.global.JibSystemProperties;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.util.Collections;\nimport java.util.Set;\nimport java.util.concurrent.Callable;\nimport java.util.stream.Collectors;\n\n/**\n * Pushes a manifest or a manifest list for a tag. If not a manifest list, returns the manifest\n * digest (\"image digest\") and the container configuration digest (\"image id\") as {@link\n * BuildResult}. If a manifest list, returns the manifest list digest only.\n */\n// TODO: figure out the right return value and type when pushing a manifest list.\nclass PushImageStep implements Callable<BuildResult> {\n\n  private static final String DESCRIPTION = \"Pushing manifest\";\n\n  static ImmutableList<PushImageStep> makeList(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      BlobDescriptor containerConfigurationDigestAndSize,\n      Image builtImage,\n      boolean manifestAlreadyExists)\n      throws IOException {\n    // Gets the image manifest to push.\n    BuildableManifestTemplate manifestTemplate =\n        new ImageToJsonTranslator(builtImage)\n            .getManifestTemplate(\n                buildContext.getTargetFormat(), containerConfigurationDigestAndSize);\n    DescriptorDigest manifestDigest = Digests.computeJsonDigest(manifestTemplate);\n    Set<String> imageQualifiers = getImageQualifiers(buildContext, builtImage, manifestDigest);\n\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    try (TimerEventDispatcher ignored =\n            new TimerEventDispatcher(eventHandlers, \"Preparing manifest pushers\");\n        ProgressEventDispatcher progressDispatcher =\n            progressEventDispatcherFactory.create(\n                \"launching manifest pushers\", imageQualifiers.size())) {\n\n      if (JibSystemProperties.skipExistingImages() && manifestAlreadyExists) {\n        eventHandlers.dispatch(LogEvent.info(\"Skipping pushing manifest; already exists.\"));\n        return ImmutableList.of();\n      }\n\n      return imageQualifiers.stream()\n          .map(\n              qualifier ->\n                  new PushImageStep(\n                      buildContext,\n                      progressDispatcher.newChildProducer(),\n                      registryClient,\n                      manifestTemplate,\n                      qualifier,\n                      manifestDigest,\n                      containerConfigurationDigestAndSize.getDigest()))\n          .collect(ImmutableList.toImmutableList());\n    }\n  }\n\n  private static Set<String> getImageQualifiers(\n      BuildContext buildContext, Image builtImage, DescriptorDigest manifestDigest) {\n    boolean singlePlatform = buildContext.getContainerConfiguration().getPlatforms().size() == 1;\n    Set<String> tags = buildContext.getAllTargetImageTags();\n    if (singlePlatform) {\n      return tags;\n    }\n    if (buildContext.getEnablePlatformTags()) {\n      String architecture = builtImage.getArchitecture();\n      return tags.stream().map(tag -> tag + \"-\" + architecture).collect(Collectors.toSet());\n    }\n    return Collections.singleton(manifestDigest.toString());\n  }\n\n  static ImmutableList<PushImageStep> makeListForManifestList(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      ManifestTemplate manifestList,\n      boolean manifestListAlreadyExists)\n      throws IOException {\n    Set<String> tags = buildContext.getAllTargetImageTags();\n\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    try (TimerEventDispatcher ignored =\n            new TimerEventDispatcher(eventHandlers, \"Preparing manifest list pushers\");\n        ProgressEventDispatcher progressEventDispatcher =\n            progressEventDispatcherFactory.create(\"launching manifest list pushers\", tags.size())) {\n      boolean singlePlatform = buildContext.getContainerConfiguration().getPlatforms().size() == 1;\n      if (singlePlatform) {\n        return ImmutableList.of(); // single image; no need to push a manifest list\n      }\n\n      if (JibSystemProperties.skipExistingImages() && manifestListAlreadyExists) {\n        eventHandlers.dispatch(LogEvent.info(\"Skipping pushing manifest list; already exists.\"));\n        return ImmutableList.of();\n      }\n      DescriptorDigest manifestListDigest = Digests.computeJsonDigest(manifestList);\n      return tags.stream()\n          .map(\n              tag ->\n                  new PushImageStep(\n                      buildContext,\n                      progressEventDispatcher.newChildProducer(),\n                      registryClient,\n                      manifestList,\n                      tag,\n                      manifestListDigest,\n                      // TODO: a manifest list digest isn't an \"image id\". Figure out the right\n                      // return value and type.\n                      manifestListDigest))\n          .collect(ImmutableList.toImmutableList());\n    }\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n  private final RegistryClient registryClient;\n  private final ManifestTemplate manifestTemplate;\n  private final String imageQualifier;\n  private final DescriptorDigest imageDigest;\n  private final DescriptorDigest imageId;\n\n  PushImageStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      ManifestTemplate manifestTemplate,\n      String imageQualifier,\n      DescriptorDigest imageDigest,\n      DescriptorDigest imageId) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.registryClient = registryClient;\n    this.manifestTemplate = manifestTemplate;\n    this.imageQualifier = imageQualifier;\n    this.imageDigest = imageDigest;\n    this.imageId = imageId;\n  }\n\n  @Override\n  public BuildResult call() throws IOException, RegistryException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    try (TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, DESCRIPTION);\n        ProgressEventDispatcher ignored2 =\n            progressEventDispatcherFactory.create(\"pushing manifest for \" + imageQualifier, 1)) {\n      eventHandlers.dispatch(LogEvent.info(\"Pushing manifest for \" + imageQualifier + \"...\"));\n\n      registryClient.pushManifest(manifestTemplate, imageQualifier);\n      return new BuildResult(imageDigest, imageId, true);\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.event.progress;\n\nimport java.io.Closeable;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.util.function.Consumer;\nimport java.util.function.Supplier;\nimport javax.annotation.Nullable;\n\n/**\n * Wraps a {@code Consumer<Long>} so that multiple consume calls ({@link #accept}) within a short\n * period of time are merged into a single later call with the value accumulated up to that point.\n */\npublic class ThrottledAccumulatingConsumer implements Consumer<Long>, Closeable {\n\n  private final Consumer<Long> consumer;\n\n  /** Delay between each call to the underlying {@link #accept}. */\n  private final Duration delayBetweenCallbacks;\n\n  /** Last time the underlying {@link #accept} was called. */\n  private Instant previousCallback;\n\n  /** \"Clock\" that returns the current {@link Instant}. */\n  private final Supplier<Instant> getNow;\n\n  @Nullable private Long valueSoFar;\n\n  /**\n   * Wraps a consumer with the delay of 100 ms.\n   *\n   * @param callback {@link Consumer} callback to wrap\n   */\n  public ThrottledAccumulatingConsumer(Consumer<Long> callback) {\n    this(callback, Duration.ofMillis(100), Instant::now);\n  }\n\n  /**\n   * A new configured consumer.\n   *\n   * @param consumer {@link Consumer} callback to wrap\n   * @param delayBetweenCallbacks delay between each call to the underlying accept\n   * @param getNow supplies of the current time {@link Instant}\n   */\n  public ThrottledAccumulatingConsumer(\n      Consumer<Long> consumer, Duration delayBetweenCallbacks, Supplier<Instant> getNow) {\n    this.consumer = consumer;\n    this.delayBetweenCallbacks = delayBetweenCallbacks;\n    this.getNow = getNow;\n\n    previousCallback = getNow.get();\n  }\n\n  @Override\n  public void accept(Long value) {\n    valueSoFar = valueSoFar == null ? value : valueSoFar + value;\n\n    Instant now = getNow.get();\n    Instant nextFireTime = previousCallback.plus(delayBetweenCallbacks);\n    if (now.isAfter(nextFireTime)) {\n      consumer.accept(valueSoFar);\n      previousCallback = now;\n      valueSoFar = null;\n    }\n  }\n\n  @Override\n  public void close() {\n    if (valueSoFar != null) {\n      consumer.accept(valueSoFar);\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer;\nimport com.google.common.base.Preconditions;\nimport java.io.Closeable;\nimport javax.annotation.Nullable;\n\n/**\n * Contains a {@link ProgressEventDispatcher} and throttles dispatching progress events with the\n * default delay used by {@link ThrottledAccumulatingConsumer}. This class is mutable and should\n * only be used within a local context.\n *\n * <p>This class is necessary because the total BLOb size (allocation units) is not known until the\n * response headers are received, only after which can the {@link ProgressEventDispatcher} be\n * created.\n */\nclass ThrottledProgressEventDispatcherWrapper implements Closeable {\n\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n  private final String description;\n  @Nullable private ProgressEventDispatcher progressEventDispatcher;\n  @Nullable private ThrottledAccumulatingConsumer throttledDispatcher;\n\n  ThrottledProgressEventDispatcherWrapper(\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory, String description) {\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.description = description;\n  }\n\n  public void dispatchProgress(Long progressUnits) {\n    Preconditions.checkNotNull(throttledDispatcher);\n    throttledDispatcher.accept(progressUnits);\n  }\n\n  @Override\n  public void close() {\n    Preconditions.checkNotNull(progressEventDispatcher);\n    Preconditions.checkNotNull(throttledDispatcher);\n    throttledDispatcher.close();\n    progressEventDispatcher.close();\n  }\n\n  void setProgressTarget(long allocationUnits) {\n    Preconditions.checkState(progressEventDispatcher == null);\n    progressEventDispatcher = progressEventDispatcherFactory.create(description, allocationUnits);\n    throttledDispatcher =\n        new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/docker/json/DockerManifestEntryTemplate.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.docker.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * JSON Template for a loadable Docker Manifest entry. The RepoTags property requires a tag; i.e. if\n * a tag is missing, it explicitly should use \"latest\".\n *\n * <p>Note that this is a template for a single Manifest entry, while the entire Docker Manifest\n * should be {@code List<DockerManifestEntryTemplate>}.\n *\n * <p>Example manifest entry JSON:\n *\n * <pre>{@code\n * {\n *   \"Config\":\"config.json\",\n *   \"RepoTags\":[\"repository:tag\"]\n *   \"Layers\": [\n *     \"eb05f3dbdb543cc610527248690575bacbbcebabe6ecf665b189cf18b541e3ca.tar.gz\",\n *     \"ba7c544469e514f1a9a4dec59ab640540d50992b288adbb34a1a63c45bf19a24.tar.gz\",\n *     \"15705ab016593987662839b40f5a22fd1032996c90808d4a1371eb46974017d5.tar.gz\"\n *   ]\n * }\n * }</pre>\n *\n * @see <a href=\"https://github.com/moby/moby/blob/master/image/tarexport/load.go\">Docker load\n *     source</a>\n */\n@JsonIgnoreProperties(ignoreUnknown = true)\npublic class DockerManifestEntryTemplate implements JsonTemplate {\n\n  @JsonProperty(\"Config\")\n  private String config = \"config.json\";\n\n  @JsonProperty(\"RepoTags\")\n  private final List<String> repoTags = new ArrayList<>();\n\n  @JsonProperty(\"Layers\")\n  private final List<String> layers = new ArrayList<>();\n\n  \npublic void setConfig(String config) {\n    this.config = config;\n  }\n\n  public void addRepoTag(String repoTag) {\n    repoTags.add(repoTag);\n  }\n\n  public void addLayerFile(String layer) {\n    layers.add(layer);\n  }\n\n  public String getConfig() {\n    return config;\n  }\n\n  public List<String> getLayerFiles() {\n    return layers;\n  }\n\n  @VisibleForTesting\n  public List<String> getRepoTags() {\n    return repoTags;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableMap;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.stream.Collectors;\nimport javax.annotation.Nullable;\n\n/**\n * JSON template for OCI archive \"index.json\" file.\n *\n * <p>Example manifest JSON:\n *\n * <pre>{@code\n * {\n *   \"schemaVersion\": 2,\n *   \"mediaType\": \"application/vnd.oci.image.index.v1+json\",\n *   \"manifests\": [\n *     {\n *       \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n *       \"digest\": \"sha256:e684b1dceef404268f17d4adf7f755fd9912b8ae64864b3954a83ebb8aa628b3\",\n *       \"size\": 1132,\n *       \"platform\": {\n *         \"architecture\": \"ppc64le\",\n *         \"os\": \"linux\"\n *       },\n *       \"annotations\": {\n *         \"org.opencontainers.image.ref.name\": \"gcr.io/project/image:tag\"\n *       }\n *     }\n *   ]\n * }\n * }</pre>\n *\n * @see <a href=\"https://github.com/opencontainers/image-spec/blob/master/image-index.md\">OCI Image\n *     Index Specification</a>\n */\npublic class OciIndexTemplate implements ManifestListTemplate {\n\n  /** The OCI Index media type. */\n  public static final String MEDIA_TYPE = \"application/vnd.oci.image.index.v1+json\";\n\n  private final int schemaVersion = 2;\n  private final String mediaType = MEDIA_TYPE;\n\n  private final List<ManifestDescriptorTemplate> manifests = new ArrayList<>();\n\n  @Override\n  public int getSchemaVersion() {\n    return schemaVersion;\n  }\n\n  @Override\n  public String getManifestMediaType() {\n    return mediaType;\n  }\n\n  /**\n   * Adds a manifest reference with the given {@link BlobDescriptor}.\n   *\n   * @param descriptor the manifest blob descriptor\n   * @param imageReferenceName the image reference name\n   */\n  public void addManifest(BlobDescriptor descriptor, String imageReferenceName) {\n    ManifestDescriptorTemplate contentDescriptorTemplate =\n        new ManifestDescriptorTemplate(\n            OciManifestTemplate.MANIFEST_MEDIA_TYPE, descriptor.getSize(), descriptor.getDigest());\n    contentDescriptorTemplate.setAnnotations(\n        ImmutableMap.of(\"org.opencontainers.image.ref.name\", imageReferenceName));\n    manifests.add(contentDescriptorTemplate);\n  }\n\n  /**\n   * Adds a manifest.\n   *\n   * @param manifest a manifest descriptor\n   */\n  public void addManifest(OciIndexTemplate.ManifestDescriptorTemplate manifest) {\n    manifests.add(manifest);\n  }\n\n  @VisibleForTesting\n  public List<ManifestDescriptorTemplate> getManifests() {\n    return manifests;\n  }\n\n  @Override\n  public List<String> getDigestsForPlatform(String architecture, String os) {\n    return getManifests().stream()\n        .filter(\n            manifest ->\n                manifest.platform != null\n                    && os.equals(manifest.platform.os)\n                    && architecture.equals(manifest.platform.architecture))\n        .map(ManifestDescriptorTemplate::getDigest)\n        .filter(Objects::nonNull)\n        .map(DescriptorDigest::toString)\n        .collect(Collectors.toList());\n  }\n\n  /**\n   * Template for inner JSON object representing a single platform specific manifest. See <a\n   * href=\"https://github.com/opencontainers/image-spec/blob/main/image-index.md\">OCI Image Index\n   * Specification</a>\n   */\n  public static class ManifestDescriptorTemplate\n      extends BuildableManifestTemplate.ContentDescriptorTemplate {\n\n    ManifestDescriptorTemplate(String mediaType, long size, DescriptorDigest digest) {\n      super(mediaType, size, digest);\n    }\n\n    /** Necessary for Jackson to create from JSON. */\n    @SuppressWarnings(\"unused\")\n    private ManifestDescriptorTemplate() {\n      super();\n    }\n\n    @JsonIgnoreProperties(ignoreUnknown = true)\n    public static class Platform implements JsonTemplate {\n      @Nullable private String architecture;\n      @Nullable private String os;\n\n      @Nullable\n      public String getArchitecture() {\n        return architecture;\n      }\n\n      @Nullable\n      public String getOs() {\n        return os;\n      }\n    }\n\n    @Nullable private OciIndexTemplate.ManifestDescriptorTemplate.Platform platform;\n\n    /**\n     * Sets a platform.\n     *\n     * @param architecture the manifest architecture\n     * @param os the manifest os\n     */\n    public void setPlatform(String architecture, String os) {\n      platform = new OciIndexTemplate.ManifestDescriptorTemplate.Platform();\n      platform.architecture = architecture;\n      platform.os = os;\n    }\n\n    @Nullable\n    public OciIndexTemplate.ManifestDescriptorTemplate.Platform getPlatform() {\n      return platform;\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/ImageTarball.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;\nimport com.google.cloud.tools.jib.image.json.OciIndexTemplate;\nimport com.google.cloud.tools.jib.image.json.OciManifestTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.cloud.tools.jib.tar.TarStreamBuilder;\nimport com.google.common.collect.ImmutableSet;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.time.Instant;\nimport java.util.Collections;\n\n/** Translates an {@link Image} to a tarball that can be loaded into Docker. */\npublic class ImageTarball {\n\n  /** File name for the container configuration in the tarball. */\n  private static final String CONTAINER_CONFIGURATION_JSON_FILE_NAME = \"config.json\";\n\n  /** File name for the manifest in the tarball. */\n  private static final String MANIFEST_JSON_FILE_NAME = \"manifest.json\";\n\n  /** File name extension for the layer content files. */\n  private static final String LAYER_FILE_EXTENSION = \".tar.gz\";\n\n  /** Time that entry is set in the tar. */\n  private static final Instant TAR_ENTRY_MODIFICATION_TIME = Instant.EPOCH;\n\n  private static final String BLOB_PREFIX = \"blobs/sha256/\";\n\n  private final Image image;\n  private final ImageReference imageReference;\n  private final ImmutableSet<String> allTargetImageTags;\n\n  /**\n   * Instantiate with an {@link Image}.\n   *\n   * @param image the image to convert into a tarball\n   * @param imageReference image reference to set in the manifest (note that the tag portion of the\n   *     image reference is ignored)\n   * @param allTargetImageTags the tags to tag the image with\n   */\n  public ImageTarball(\n      Image image, ImageReference imageReference, ImmutableSet<String> allTargetImageTags) {\n    this.image = image;\n    this.imageReference = imageReference;\n    this.allTargetImageTags = allTargetImageTags;\n  }\n\n  /**\n   * Writes image tar bar in configured {@link Image#getImageFormat()} of OCI or Docker to output\n   * stream.\n   *\n   * @param out the target output stream\n   * @throws IOException if an error occurs writing out the image to stream\n   */\n  public void writeTo(OutputStream out) throws IOException {\n    if (image.getImageFormat() == OciManifestTemplate.class) {\n      ociWriteTo(out);\n    } else {\n      dockerWriteTo(out);\n    }\n  }\n\n  private void ociWriteTo(OutputStream out) throws IOException {\n    TarStreamBuilder tarStreamBuilder = new TarStreamBuilder();\n    OciManifestTemplate manifest = new OciManifestTemplate();\n\n    // Adds all the layers to the tarball and manifest\n    for (Layer layer : image.getLayers()) {\n      DescriptorDigest digest = layer.getBlobDescriptor().getDigest();\n      long size = layer.getBlobDescriptor().getSize();\n\n      tarStreamBuilder.addBlobEntry(\n          layer.getBlob(), size, BLOB_PREFIX + digest.getHash(), TAR_ENTRY_MODIFICATION_TIME);\n      manifest.addLayer(size, digest);\n    }\n\n    // Adds the container configuration to the tarball and manifest\n    JsonTemplate containerConfiguration =\n        new ImageToJsonTranslator(image).getContainerConfiguration();\n    BlobDescriptor configDescriptor = Digests.computeDigest(containerConfiguration);\n    manifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest());\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(containerConfiguration),\n        BLOB_PREFIX + configDescriptor.getDigest().getHash(),\n        TAR_ENTRY_MODIFICATION_TIME);\n\n    // Adds the manifest to the tarball\n    BlobDescriptor manifestDescriptor = Digests.computeDigest(manifest);\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(manifest),\n        BLOB_PREFIX + manifestDescriptor.getDigest().getHash(),\n        TAR_ENTRY_MODIFICATION_TIME);\n\n    // Adds the oci-layout and index.json\n    tarStreamBuilder.addByteEntry(\n        \"{\\\"imageLayoutVersion\\\": \\\"1.0.0\\\"}\".getBytes(StandardCharsets.UTF_8),\n        \"oci-layout\",\n        TAR_ENTRY_MODIFICATION_TIME);\n    OciIndexTemplate index = new OciIndexTemplate();\n    // TODO: figure out how to tag with allTargetImageTags\n    index.addManifest(manifestDescriptor, imageReference.toStringWithQualifier());\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(index), \"index.json\", TAR_ENTRY_MODIFICATION_TIME);\n\n    tarStreamBuilder.writeAsTarArchiveTo(out);\n  }\n\n  private void dockerWriteTo(OutputStream out) throws IOException {\n    TarStreamBuilder tarStreamBuilder = new TarStreamBuilder();\n    DockerManifestEntryTemplate manifestTemplate = new DockerManifestEntryTemplate();\n\n    // Adds all the layers to the tarball and manifest.\n    for (Layer layer : image.getLayers()) {\n      String layerName = layer.getBlobDescriptor().getDigest().getHash() + LAYER_FILE_EXTENSION;\n\n      tarStreamBuilder.addBlobEntry(\n          layer.getBlob(),\n          layer.getBlobDescriptor().getSize(),\n          layerName,\n          TAR_ENTRY_MODIFICATION_TIME);\n      manifestTemplate.addLayerFile(layerName);\n    }\n\n    // Adds the container configuration to the tarball.\n    JsonTemplate containerConfiguration =\n        new ImageToJsonTranslator(image).getContainerConfiguration();\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(containerConfiguration),\n        CONTAINER_CONFIGURATION_JSON_FILE_NAME,\n        TAR_ENTRY_MODIFICATION_TIME);\n\n    // Adds the manifest to tarball.\n    for (String tag : allTargetImageTags) {\n      manifestTemplate.addRepoTag(imageReference.withQualifier(tag).toStringWithQualifier());\n    }\n    tarStreamBuilder.addByteEntry(\n        JsonTemplateMapper.toByteArray(Collections.singletonList(manifestTemplate)),\n        MANIFEST_JSON_FILE_NAME,\n        TAR_ENTRY_MODIFICATION_TIME);\n\n    tarStreamBuilder.writeAsTarArchiveTo(out);\n  }\n\n  /**\n   * Returns the total size of the image's layers in bytes.\n   *\n   * @return the total size of the image's layers in bytes\n   */\n  public long getTotalLayerSize() {\n    long size = 0;\n    for (Layer layer : image.getLayers()) {\n      size += layer.getBlobDescriptor().getSize();\n    }\n    return size;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.ImageTarball;\nimport java.io.BufferedOutputStream;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.concurrent.Callable;\n\npublic class WriteTarFileStep implements Callable<BuildResult> {\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n\n  private final Path outputPath;\n  private final Image builtImage;\n\n  WriteTarFileStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Path outputPath,\n      Image builtImage) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.outputPath = outputPath;\n    this.builtImage = builtImage;\n  }\n\n  @Override\n  public BuildResult call() throws IOException {\n    buildContext.getEventHandlers().dispatch(LogEvent.progress(\"Building image to tar file...\"));\n\n    try (ProgressEventDispatcher ignored =\n        progressEventDispatcherFactory.create(\"writing to tar file\", 1)) {\n      // Builds the image to a tarball.\n      if (outputPath.getParent() != null) {\n        Files.createDirectories(outputPath.getParent());\n      }\n      try (OutputStream outputStream =\n          new BufferedOutputStream(Files.newOutputStream(outputPath))) {\n        new ImageTarball(\n                builtImage,\n                buildContext.getTargetImageConfiguration().getImage(),\n                buildContext.getAllTargetImageTags())\n            .writeTo(outputStream);\n      }\n\n      return BuildResult.fromImage(builtImage, buildContext.getTargetFormat());\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.image.Layer;\n\n/**\n * Layer prepared from {@link BuildAndCacheApplicationLayerStep} and {@link\n * ObtainBaseImageLayerStep} to hold information about either a base image layer or an application\n * layer.\n */\nclass PreparedLayer implements Layer {\n\n  enum StateInTarget {\n    UNKNOWN,\n    EXISTING,\n    MISSING\n  }\n\n  static class Builder {\n\n    private Layer layer;\n    private String name = \"unnamed layer\";\n    private StateInTarget stateInTarget = StateInTarget.UNKNOWN;\n\n    Builder(Layer layer) {\n      this.layer = layer;\n    }\n\n    Builder setName(String name) {\n      this.name = name;\n      return this;\n    }\n\n    /** Sets whether the layer exists in a target destination. */\n    Builder setStateInTarget(StateInTarget stateInTarget) {\n      this.stateInTarget = stateInTarget;\n      return this;\n    }\n\n    PreparedLayer build() {\n      return new PreparedLayer(layer, name, stateInTarget);\n    }\n  }\n\n  private final Layer layer;\n  private final String name;\n  private final StateInTarget stateInTarget;\n\n  private PreparedLayer(Layer layer, String name, StateInTarget stateInTarget) {\n    this.layer = layer;\n    this.name = name;\n    this.stateInTarget = stateInTarget;\n  }\n\n  String getName() {\n    return name;\n  }\n\n  StateInTarget getStateInTarget() {\n    return stateInTarget;\n  }\n\n  @Override\n  public Blob getBlob() {\n    return layer.getBlob();\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return layer.getBlobDescriptor();\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() {\n    return layer.getDiffId();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.builder.steps.PreparedLayer.StateInTarget;\nimport com.google.cloud.tools.jib.cache.Cache;\nimport com.google.cloud.tools.jib.cache.CacheCorruptedException;\nimport com.google.cloud.tools.jib.cache.CachedLayer;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.image.Layer;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport com.google.common.base.Verify;\nimport java.io.IOException;\nimport java.util.Optional;\nimport java.util.concurrent.Callable;\nimport javax.annotation.Nullable;\n\n/** Pulls and caches a single base image layer. */\nclass ObtainBaseImageLayerStep implements Callable<PreparedLayer> {\n\n  @SuppressWarnings(\"InlineFormatString\")\n  private static final String DESCRIPTION = \"Pulling base image layer %s\";\n\n  @FunctionalInterface\n  private interface BlobExistenceChecker {\n\n    StateInTarget check(DescriptorDigest digest) throws IOException, RegistryException;\n  }\n\n  static ObtainBaseImageLayerStep forForcedDownload(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Layer layer,\n      @Nullable RegistryClient sourceRegistryClient) {\n    BlobExistenceChecker noOpChecker = ignored -> StateInTarget.UNKNOWN;\n    return new ObtainBaseImageLayerStep(\n        buildContext, progressEventDispatcherFactory, layer, sourceRegistryClient, noOpChecker);\n  }\n\n  static ObtainBaseImageLayerStep forSelectiveDownload(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Layer layer,\n      @Nullable RegistryClient sourceRegistryClient,\n      RegistryClient targetRegistryClient) {\n    Verify.verify(!buildContext.isOffline());\n\n    // TODO: also check if cross-repo blob mount is possible.\n    BlobExistenceChecker blobExistenceChecker =\n        digest ->\n            targetRegistryClient.checkBlob(digest).isPresent()\n                ? StateInTarget.EXISTING\n                : StateInTarget.MISSING;\n\n    return new ObtainBaseImageLayerStep(\n        buildContext,\n        progressEventDispatcherFactory,\n        layer,\n        sourceRegistryClient,\n        blobExistenceChecker);\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n\n  private final Layer layer;\n  private final @Nullable RegistryClient registryClient;\n  private final BlobExistenceChecker blobExistenceChecker;\n\n  private ObtainBaseImageLayerStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Layer layer,\n      @Nullable RegistryClient registryClient,\n      BlobExistenceChecker blobExistenceChecker) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.layer = layer;\n    this.registryClient = registryClient;\n    this.blobExistenceChecker = blobExistenceChecker;\n  }\n\n  @Override\n  public PreparedLayer call() throws IOException, CacheCorruptedException, RegistryException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest();\n    try (ProgressEventDispatcher progressEventDispatcher =\n            progressEventDispatcherFactory.create(\"checking base image layer \" + layerDigest, 1);\n        TimerEventDispatcher ignored =\n            new TimerEventDispatcher(eventHandlers, String.format(DESCRIPTION, layerDigest))) {\n\n      StateInTarget stateInTarget = blobExistenceChecker.check(layerDigest);\n      if (stateInTarget == StateInTarget.EXISTING) {\n        eventHandlers.dispatch(\n            LogEvent.info(\n                \"Skipping pull; BLOB already exists on target registry : \"\n                    + layer.getBlobDescriptor()));\n        return new PreparedLayer.Builder(layer).setStateInTarget(stateInTarget).build();\n      }\n\n      Cache cache = buildContext.getBaseImageLayersCache();\n\n      // Checks if the layer already exists in the cache.\n      Optional<CachedLayer> optionalCachedLayer = cache.retrieve(layerDigest);\n      if (optionalCachedLayer.isPresent()) {\n        CachedLayer cachedLayer = optionalCachedLayer.get();\n        return new PreparedLayer.Builder(cachedLayer).setStateInTarget(stateInTarget).build();\n      } else if (buildContext.isOffline()) {\n        throw new IOException(\n            \"Cannot run Jib in offline mode; local Jib cache for base image is missing image layer \"\n                + layerDigest\n                + \". Rerun Jib in online mode with \\\"-Djib.alwaysCacheBaseImage=true\\\" to \"\n                + \"re-download the base image layers.\");\n      }\n\n      try (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper =\n          new ThrottledProgressEventDispatcherWrapper(\n              progressEventDispatcher.newChildProducer(),\n              \"pulling base image layer \" + layerDigest)) {\n        CachedLayer cachedLayer =\n            cache.writeCompressedLayer(\n                Verify.verifyNotNull(registryClient)\n                    .pullBlob(\n                        layerDigest,\n                        progressEventDispatcherWrapper::setProgressTarget,\n                        progressEventDispatcherWrapper::dispatchProgress));\n        return new PreparedLayer.Builder(cachedLayer).setStateInTarget(stateInTarget).build();\n      }\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/DockerClient.java\n/*\n...\n", "needle_token_start": 8764, "needle_token_end": 8784, "code_context_ntokens": 15935, "output": ["```java\npublic void setConfig(String config) {\n    this.config = config;\n}\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "docker", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/docker/CliDockerClient.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to manage container images by loading them into the local container registry, saving them from the registry to a file, and retrieving detailed information about specific images.\n2. **Input**: It accepts commands and parameters such as the path to an image tarball, an image reference, and an output path for saving images.\n3. **Output**: Depending on the operation, it returns the result of the image load operation, saves the image to a specified path, or provides detailed JSON formatted information about an image.\n4. **Procedure**: The function constructs and executes commands related to container management using a process builder. It handles input and output streams to interact with the process, manages error handling by capturing standard error output, and utilizes listeners to track byte counts during operations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.cache;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.blob.Blobs;\nimport com.google.cloud.tools.jib.cache.Retry.Action;\nimport com.google.cloud.tools.jib.filesystem.LockFile;\nimport com.google.cloud.tools.jib.filesystem.TempDirectoryProvider;\nimport com.google.cloud.tools.jib.hash.CountingDigestOutputStream;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.V21ManifestTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport java.io.BufferedInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.AtomicMoveNotSupportedException;\nimport java.nio.file.FileSystemException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.StandardCopyOption;\nimport java.util.List;\nimport java.util.concurrent.TimeUnit;\nimport java.util.function.Predicate;\nimport java.util.zip.GZIPOutputStream;\nimport javax.annotation.Nullable;\nimport org.apache.commons.compress.compressors.CompressorException;\nimport org.apache.commons.compress.compressors.CompressorStreamFactory;\n\n/** Writes to the default cache storage engine. */\nclass CacheStorageWriter {\n\n  /** Holds information about a layer that was written. */\n  private static class WrittenLayer {\n\n    private final DescriptorDigest layerDigest;\n    private final DescriptorDigest layerDiffId;\n    private final long layerSize;\n\n    private WrittenLayer(\n        DescriptorDigest layerDigest, DescriptorDigest layerDiffId, long layerSize) {\n      this.layerDigest = layerDigest;\n      this.layerDiffId = layerDiffId;\n      this.layerSize = layerSize;\n    }\n  }\n\n  private static void verifyImageMetadata(ImageMetadataTemplate metadata) {\n    Predicate<ManifestAndConfigTemplate> isManifestNull = pair -> pair.getManifest() == null;\n    Predicate<ManifestAndConfigTemplate> isConfigNull = pair -> pair.getConfig() == null;\n    Predicate<ManifestAndConfigTemplate> isDigestNull = pair -> pair.getManifestDigest() == null;\n\n    List<ManifestAndConfigTemplate> manifestsAndConfigs = metadata.getManifestsAndConfigs();\n    Preconditions.checkArgument(!manifestsAndConfigs.isEmpty(), \"no manifests given\");\n    Preconditions.checkArgument(\n        manifestsAndConfigs.stream().noneMatch(isManifestNull), \"null manifest(s)\");\n    Preconditions.checkArgument(\n        metadata.getManifestList() != null || manifestsAndConfigs.size() == 1,\n        \"manifest list missing while multiple manifests given\");\n\n    ManifestTemplate firstManifest = manifestsAndConfigs.get(0).getManifest();\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageReader.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.cache;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.blob.Blobs;\nimport com.google.cloud.tools.jib.filesystem.LockFile;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.V21ManifestTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.io.IOException;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.security.DigestException;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.stream.Collectors;\nimport java.util.stream.Stream;\n\n/** Reads from the default cache storage engine. */\nclass CacheStorageReader {\n\n  @VisibleForTesting\n  static void verifyImageMetadata(ImageMetadataTemplate metadata, Path metadataCacheDirectory)\n      throws CacheCorruptedException {\n    List<ManifestAndConfigTemplate> manifestsAndConfigs = metadata.getManifestsAndConfigs();\n    if (manifestsAndConfigs.isEmpty()) {\n      throw new CacheCorruptedException(metadataCacheDirectory, \"Manifest cache empty\");\n    }\n    if (manifestsAndConfigs.stream().anyMatch(entry -> entry.getManifest() == null)) {\n      throw new CacheCorruptedException(metadataCacheDirectory, \"Manifest(s) missing\");\n    }\n    if (metadata.getManifestList() == null && manifestsAndConfigs.size() != 1) {\n      throw new CacheCorruptedException(metadataCacheDirectory, \"Manifest list missing\");\n    }\n\n    ManifestTemplate firstManifest = manifestsAndConfigs.get(0).getManifest();\n    if (firstManifest instanceof V21ManifestTemplate) {\n      if (metadata.getManifestList() != null\n          || manifestsAndConfigs.stream().anyMatch(entry -> entry.getConfig() != null)) {\n        throw new CacheCorruptedException(metadataCacheDirectory, \"Schema 1 manifests corrupted\");\n      }\n    } else if (firstManifest instanceof BuildableManifestTemplate) {\n      if (manifestsAndConfigs.stream().anyMatch(entry -> entry.getConfig() == null)) {\n        throw new CacheCorruptedException(metadataCacheDirectory, \"Schema 2 manifests corrupted\");\n      }\n      if (metadata.getManifestList() != null\n          && manifestsAndConfigs.stream().anyMatch(entry -> entry.getManifestDigest() == null)) {\n        throw new CacheCorruptedException(metadataCacheDirectory, \"Schema 2 manifests corrupted\");\n      }\n    } else {\n      throw new CacheCorruptedException(\n          metadataCacheDirectory, \"Unknown manifest type: \" + firstManifest);\n    }\n  }\n\n  private final CacheStorageFiles cacheStorageFiles;\n\n  CacheStorageReader(CacheStorageFiles cacheStorageFiles) {\n    this.cacheStorageFiles = cacheStorageFiles;\n  }\n\n  /**\n   * Returns {@code true} if all image layers described in a manifest have a corresponding file\n   * entry in the cache.\n   *\n   * @param manifest the image manifest\n   * @return a boolean\n   */\n  boolean areAllLayersCached(ManifestTemplate manifest) {\n\n    List<DescriptorDigest> layerDigests;\n\n    if (manifest instanceof V21ManifestTemplate) {\n      layerDigests = ((V21ManifestTemplate) manifest).getLayerDigests();\n    } else if (manifest instanceof BuildableManifestTemplate) {\n      layerDigests =\n          ((BuildableManifestTemplate) manifest)\n              .getLayers().stream()\n                  .map(BuildableManifestTemplate.ContentDescriptorTemplate::getDigest)\n                  .collect(Collectors.toList());\n    } else {\n      throw new IllegalArgumentException(\"Unknown manifest type: \" + manifest);\n    }\n\n    for (DescriptorDigest layerDigest : layerDigests) {\n      Path layerDirectory = cacheStorageFiles.getLayerDirectory(layerDigest);\n      if (!Files.exists(layerDirectory)) {\n        return false;\n      }\n    }\n    return true;\n  }\n\n  /**\n   * Retrieves the cached image metadata (a manifest list and a list of manifest/container\n   * configuration pairs) for an image reference.\n   *\n   * @param imageReference the image reference\n   * @return the image metadata for the image reference, if found\n   * @throws IOException if an I/O exception occurs\n   * @throws CacheCorruptedException if the cache is corrupted\n   */\n  Optional<ImageMetadataTemplate> retrieveMetadata(ImageReference imageReference)\n      throws IOException, CacheCorruptedException {\n    Path imageDirectory = cacheStorageFiles.getImageDirectory(imageReference);\n    Path metadataPath = imageDirectory.resolve(\"manifests_configs.json\");\n    if (!Files.exists(metadataPath)) {\n      return Optional.empty();\n    }\n\n    ImageMetadataTemplate metadata;\n    try (LockFile ignored = LockFile.lock(imageDirectory.resolve(\"lock\"))) {\n      metadata = JsonTemplateMapper.readJsonFromFile(metadataPath, ImageMetadataTemplate.class);\n    }\n    verifyImageMetadata(metadata, imageDirectory);\n    return Optional.of(metadata);\n  }\n\n  /**\n   * Retrieves the {@link CachedLayer} for the layer with digest {@code layerDigest}.\n   *\n   * @param layerDigest the layer digest\n   * @return the {@link CachedLayer} referenced by the layer digest, if found\n   * @throws CacheCorruptedException if the cache was found to be corrupted\n   * @throws IOException if an I/O exception occurs\n   */\n  Optional<CachedLayer> retrieve(DescriptorDigest layerDigest)\n      throws IOException, CacheCorruptedException {\n    Path layerDirectory = cacheStorageFiles.getLayerDirectory(layerDigest);\n    if (!Files.exists(layerDirectory)) {\n      return Optional.empty();\n    }\n\n    try (Stream<Path> files = Files.list(layerDirectory)) {\n      List<Path> layerFiles =\n          files.filter(CacheStorageFiles::isLayerFile).collect(Collectors.toList());\n      if (layerFiles.size() != 1) {\n        throw new CacheCorruptedException(\n            cacheStorageFiles.getCacheDirectory(),\n            \"No or multiple layer files found for layer hash \"\n                + layerDigest.getHash()\n                + \" in directory: \"\n                + layerDirectory);\n      }\n\n      Path layerFile = layerFiles.get(0);\n      return Optional.of(\n          CachedLayer.builder()\n              .setLayerDigest(layerDigest)\n              .setLayerSize(Files.size(layerFile))\n              .setLayerBlob(Blobs.from(layerFile))\n              .setLayerDiffId(cacheStorageFiles.getDigestFromFilename(layerFile))\n              .build());\n    }\n  }\n\n  /**\n   * Retrieves the {@link CachedLayer} for the local base image layer with the given diff ID.\n   *\n   * @param diffId the diff ID\n   * @return the {@link CachedLayer} referenced by the diff ID, if found\n   * @throws CacheCorruptedException if the cache was found to be corrupted\n   * @throws IOException if an I/O exception occurs\n   */\n  Optional<CachedLayer> retrieveTarLayer(DescriptorDigest diffId)\n      throws IOException, CacheCorruptedException {\n    Path layerDirectory = cacheStorageFiles.getLocalDirectory().resolve(diffId.getHash());\n    if (!Files.exists(layerDirectory)) {\n      return Optional.empty();\n    }\n\n    try (Stream<Path> files = Files.list(layerDirectory)) {\n      List<Path> layerFiles =\n          files.filter(CacheStorageFiles::isLayerFile).collect(Collectors.toList());\n      if (layerFiles.size() != 1) {\n        throw new CacheCorruptedException(\n            cacheStorageFiles.getCacheDirectory(),\n            \"No or multiple layer files found for layer hash \"\n                + diffId.getHash()\n                + \" in directory: \"\n                + layerDirectory);\n      }\n\n      Path layerFile = layerFiles.get(0);\n      return Optional.of(\n          CachedLayer.builder()\n              .setLayerDigest(cacheStorageFiles.getDigestFromFilename(layerFile))\n              .setLayerSize(Files.size(layerFile))\n              .setLayerBlob(Blobs.from(layerFile))\n              .setLayerDiffId(diffId)\n              .build());\n    }\n  }\n\n  /**\n   * Retrieves the {@link ContainerConfigurationTemplate} for the image with the given image ID.\n   *\n   * @param imageId the image ID\n   * @return the {@link ContainerConfigurationTemplate} referenced by the image ID, if found\n   * @throws IOException if an I/O exception occurs\n   */\n  Optional<ContainerConfigurationTemplate> retrieveLocalConfig(DescriptorDigest imageId)\n      throws IOException {\n    Path configPath =\n        cacheStorageFiles.getLocalDirectory().resolve(\"config\").resolve(imageId.getHash());\n    if (!Files.exists(configPath)) {\n      return Optional.empty();\n    }\n\n    ContainerConfigurationTemplate config =\n        JsonTemplateMapper.readJsonFromFile(configPath, ContainerConfigurationTemplate.class);\n    return Optional.of(config);\n  }\n\n  /**\n   * Retrieves the layer digest selected by the {@code selector}.\n   *\n   * @param selector the selector\n   * @return the layer digest {@code selector} selects, if found\n   * @throws CacheCorruptedException if the selector file contents was not a valid layer digest\n   * @throws IOException if an I/O exception occurs\n   */\n  Optional<DescriptorDigest> select(DescriptorDigest selector)\n      throws CacheCorruptedException, IOException {\n    Path selectorFile = cacheStorageFiles.getSelectorFile(selector);\n    if (!Files.exists(selectorFile)) {\n      return Optional.empty();\n    }\n\n    String selectorFileContents =\n        new String(Files.readAllBytes(selectorFile), StandardCharsets.UTF_8);\n    try {\n      return Optional.of(DescriptorDigest.fromHash(selectorFileContents));\n\n    } catch (DigestException ex) {\n      throw new CacheCorruptedException(\n          cacheStorageFiles.getCacheDirectory(),\n          \"Expected valid layer digest as contents of selector file `\"\n              + selectorFile\n              + \"` for selector `\"\n              + selector.getHash()\n              + \"`, but got: \"\n              + selectorFileContents);\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/event/Handler.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.event;\n\nimport com.google.cloud.tools.jib.api.JibEvent;\nimport com.google.common.base.Preconditions;\nimport java.util.function.Consumer;\n\n/** Handles a dispatched {@link JibEvent}. */\nclass Handler<E extends JibEvent> {\n\n  private final Class<E> eventClass;\n  private final Consumer<? super E> eventConsumer;\n\n  Handler(Class<E> eventClass, Consumer<? super E> eventConsumer) {\n    this.eventClass = eventClass;\n    this.eventConsumer = eventConsumer;\n  }\n\n  /**\n   * Handles a {@link JibEvent}.\n   *\n   * @param jibEvent the event to handle\n   */\n  void handle(JibEvent jibEvent) {\n    Preconditions.checkArgument(eventClass.isInstance(jibEvent));\n    eventConsumer.accept(eventClass.cast(jibEvent));\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/AllocationCompletionTracker.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.event.progress;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ImmutableList;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.stream.Collectors;\n\n/**\n * Keeps track of the progress for {@link Allocation}s as well as their order in which they appear.\n *\n * <p>This implementation is thread-safe.\n */\nclass AllocationCompletionTracker {\n\n  /**\n   * Holds the progress units remaining along with a creation order (index starting from 0). This is\n   * used as the value of the {@link #completionMap}.\n   */\n  private static class IndexedRemainingUnits implements Comparable<IndexedRemainingUnits> {\n\n    /** Monotonically-increasing source for {@link #index}. */\n    private static final AtomicInteger currentIndex = new AtomicInteger();\n\n    /** The creation order that monotonically increases. */\n    private final int index = currentIndex.getAndIncrement();\n\n    /**\n     * Remaining progress units until completion. This can be shared across multiple threads and\n     * should be updated atomically.\n     */\n    private final AtomicLong remainingUnits;\n\n    private final Allocation allocation;\n\n    private IndexedRemainingUnits(Allocation allocation) {\n      this.allocation = allocation;\n      remainingUnits = new AtomicLong(allocation.getAllocationUnits());\n    }\n\n    private boolean isUnfinished() {\n      return remainingUnits.get() != 0;\n    }\n\n    @Override\n    public int compareTo(IndexedRemainingUnits otherIndexedRemainingUnits) {\n      return index - otherIndexedRemainingUnits.index;\n    }\n  }\n\n  /**\n   * Maps from {@link Allocation} to 1) the number of progress units remaining in that {@link\n   * Allocation}; as well as 2) the insertion order of the key.\n   */\n  private final ConcurrentHashMap<Allocation, IndexedRemainingUnits> completionMap =\n      new ConcurrentHashMap<>();\n\n  /**\n   * Updates the progress for {@link Allocation} atomically relative to the {@code allocation}.\n   *\n   * <p>For any {@link Allocation}, this method <em>must</em> have been called on all of its parents\n   * beforehand.\n   *\n   * @param allocation the {@link Allocation} to update progress for\n   * @param units the units of progress\n   * @return {@code true} if the map was updated\n   */\n  boolean updateProgress(Allocation allocation, long units) {\n    AtomicBoolean mapUpdated = new AtomicBoolean(units != 0);\n\n    completionMap.compute(\n        allocation,\n        (ignored, indexedRemainingUnits) -> {\n          if (indexedRemainingUnits == null) {\n            indexedRemainingUnits = new IndexedRemainingUnits(allocation);\n            mapUpdated.set(true);\n          }\n\n          if (units != 0) {\n            updateIndexedRemainingUnits(indexedRemainingUnits, units);\n          }\n          return indexedRemainingUnits;\n        });\n    return mapUpdated.get();\n  }\n\n  /**\n   * Gets a list of the unfinished {@link Allocation}s in the order in which those {@link\n   * Allocation}s were encountered. This can be used to display, for example, currently executing\n   * tasks. The order helps to keep the displayed tasks in a deterministic order (new subtasks\n   * appear below older ones) and not jumbled together in some random order.\n   *\n   * @return a list of unfinished {@link Allocation}s\n   */\n  @VisibleForTesting\n  List<Allocation> getUnfinishedAllocations() {\n    return completionMap.values().stream()\n        .filter(IndexedRemainingUnits::isUnfinished)\n        .sorted()\n        .map(remainingUnits -> remainingUnits.allocation)\n        .collect(Collectors.toList());\n  }\n\n  /**\n   * Helper method for {@link #updateProgress(Allocation, long)}. Subtract {@code units} from {@code\n   * indexedRemainingUnits}. Updates {@link IndexedRemainingUnits} for parent {@link Allocation}s if\n   * remaining units becomes 0. This method is <em>not</em> thread-safe for the {@code\n   * indexedRemainingUnits} and should be called atomically relative to the {@code\n   * indexedRemainingUnits}.\n   *\n   * @param indexedRemainingUnits the {@link IndexedRemainingUnits} to update progress for\n   * @param units the units of progress\n   */\n  private void updateIndexedRemainingUnits(\n      IndexedRemainingUnits indexedRemainingUnits, long units) {\n    Allocation allocation = indexedRemainingUnits.allocation;\n\n    long newUnits = indexedRemainingUnits.remainingUnits.addAndGet(-units);\n    if (newUnits < 0L) {\n      throw new IllegalStateException(\n          \"Progress exceeds max for '\"\n              + allocation.getDescription()\n              + \"': \"\n              + -newUnits\n              + \" more beyond \"\n              + allocation.getAllocationUnits());\n    }\n\n    // Updates the parent allocations if this allocation completed.\n    if (newUnits == 0L) {\n      allocation\n          .getParent()\n          .ifPresent(\n              parentAllocation ->\n                  updateIndexedRemainingUnits(\n                      Preconditions.checkNotNull(completionMap.get(parentAllocation)), 1L));\n    }\n  }\n\n  ImmutableList<String> getUnfinishedLeafTasks() {\n    List<Allocation> allUnfinished = getUnfinishedAllocations();\n    Set<Allocation> unfinishedLeaves = new LinkedHashSet<>(allUnfinished); // preserves order\n\n    for (Allocation allocation : allUnfinished) {\n      Optional<Allocation> parent = allocation.getParent();\n\n      while (parent.isPresent()) {\n        unfinishedLeaves.remove(parent.get());\n        parent = parent.get().getParent();\n      }\n    }\n\n    return ImmutableList.copyOf(\n        unfinishedLeaves.stream().map(Allocation::getDescription).collect(Collectors.toList()));\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ProgressEventHandler.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.event.progress;\n\nimport com.google.cloud.tools.jib.event.events.ProgressEvent;\nimport com.google.common.collect.ImmutableList;\nimport java.util.concurrent.atomic.DoubleAdder;\nimport java.util.function.Consumer;\n\n/**\n * Handles {@link ProgressEvent}s by accumulating an overall progress and keeping track of which\n * {@link Allocation}s are complete.\n *\n * <p>This implementation is thread-safe.\n */\npublic class ProgressEventHandler implements Consumer<ProgressEvent> {\n\n  /**\n   * Contains the accumulated progress and which \"leaf\" tasks are not yet complete. Leaf tasks are\n   * those that do not have sub-tasks.\n   */\n  public static class Update {\n\n    private final double progress;\n    private final ImmutableList<String> unfinishedLeafTasks;\n\n    private Update(double progress, ImmutableList<String> unfinishedLeafTasks) {\n      this.progress = progress;\n      this.unfinishedLeafTasks = unfinishedLeafTasks;\n    }\n\n    /**\n     * Gets the overall progress, with {@code 1.0} meaning fully complete.\n     *\n     * @return the overall progress\n     */\n    public double getProgress() {\n      return progress;\n    }\n\n    /**\n     * Gets a list of the unfinished \"leaf\" tasks in the order in which those tasks were\n     * encountered.\n     *\n     * @return a list of unfinished \"leaf\" tasks\n     */\n    public ImmutableList<String> getUnfinishedLeafTasks() {\n      return unfinishedLeafTasks;\n    }\n  }\n\n  /** Keeps track of the progress for each {@link Allocation} encountered. */\n  private final AllocationCompletionTracker completionTracker = new AllocationCompletionTracker();\n\n  /** Accumulates an overall progress, with {@code 1.0} indicating full completion. */\n  private final DoubleAdder progress = new DoubleAdder();\n\n  /**\n   * A callback to notify that {@link #progress} or {@link #completionTracker} could have changed.\n   * Note that every change will be reported (though multiple could be reported together), and there\n   * could be false positives.\n   */\n  private final Consumer<Update> updateNotifier;\n\n  public ProgressEventHandler(Consumer<Update> updateNotifier) {\n    this.updateNotifier = updateNotifier;\n  }\n\n  @Override\n  public void accept(ProgressEvent progressEvent) {\n    Allocation allocation = progressEvent.getAllocation();\n    long progressUnits = progressEvent.getUnits();\n    double allocationFraction = allocation.getFractionOfRoot();\n\n    if (progressUnits != 0) {\n      progress.add(progressUnits * allocationFraction);\n    }\n\n    if (completionTracker.updateProgress(allocation, progressUnits)) {\n      // Note: Could produce false positives.\n      updateNotifier.accept(new Update(progress.sum(), completionTracker.getUnfinishedLeafTasks()));\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClientResolver.java\n/*\n * Copyright 2022 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.docker;\n\nimport com.google.cloud.tools.jib.api.DockerClient;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.ServiceLoader;\n\npublic class DockerClientResolver {\n\n  private DockerClientResolver() {}\n\n  /**\n   * Look for supported DockerClient.\n   *\n   * @param parameters needed by the dockerClient\n   * @return dockerClient if any is found\n   */\n  public static Optional<DockerClient> resolve(Map<String, String> parameters) {\n    ServiceLoader<DockerClient> dockerClients = ServiceLoader.load(DockerClient.class);\n    for (DockerClient dockerClient : dockerClients) {\n      if (dockerClient.supported(parameters)) {\n        return Optional.of(dockerClient);\n      }\n    }\n    return Optional.empty();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/docker/CliDockerClient.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.docker;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.DockerClient;\nimport com.google.cloud.tools.jib.api.ImageDetails;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.http.NotifyingOutputStream;\nimport com.google.cloud.tools.jib.image.ImageTarball;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.io.ByteStreams;\nimport com.google.common.io.CharStreams;\nimport java.io.BufferedInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.security.DigestException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.function.Consumer;\nimport java.util.function.Function;\n\n/** Calls out to the {@code docker} CLI. */\npublic class CliDockerClient implements DockerClient {\n\n  /**\n   * Contains the size, image ID, and diff IDs of an image inspected with {@code docker inspect}.\n   */\n  @JsonIgnoreProperties(ignoreUnknown = true)\n  public static class DockerImageDetails implements JsonTemplate, ImageDetails {\n\n    @JsonIgnoreProperties(ignoreUnknown = true)\n    private static class RootFsTemplate implements JsonTemplate {\n      @JsonProperty(\"Layers\")\n      private final List<String> layers = Collections.emptyList();\n    }\n\n    @JsonProperty(\"Size\")\n    private long size;\n\n    @JsonProperty(\"Id\")\n    private String imageId = \"\";\n\n    @JsonProperty(\"RootFS\")\n    private final RootFsTemplate rootFs = new RootFsTemplate();\n\n    @Override\n    public long getSize() {\n      return size;\n    }\n\n    @Override\n    public DescriptorDigest getImageId() throws DigestException {\n      return DescriptorDigest.fromDigest(imageId);\n    }\n\n    /**\n     * Return a list of diff ids of the layers in the image.\n     *\n     * @return a list of diff ids\n     * @throws DigestException if a digest is invalid\n     */\n    @Override\n    public List<DescriptorDigest> getDiffIds() throws DigestException {\n      List<DescriptorDigest> processedDiffIds = new ArrayList<>(rootFs.layers.size());\n      for (String diffId : rootFs.layers) {\n        processedDiffIds.add(DescriptorDigest.fromDigest(diffId.trim()));\n      }\n      return processedDiffIds;\n    }\n  }\n\n  /** Default path to the docker executable. */\n  public static final Path DEFAULT_DOCKER_CLIENT = Paths.get(\"docker\");\n\n  /**\n   * Checks if Docker is installed on the user's system by running the `docker` command.\n   *\n   * @return {@code true} if Docker is installed on the user's system and accessible\n   */\n  public static boolean isDefaultDockerInstalled() {\n    try {\n      new ProcessBuilder(DEFAULT_DOCKER_CLIENT.toString()).start();\n      return true;\n    } catch (IOException ex) {\n      return false;\n    }\n  }\n\n  /**\n   * Checks if Docker is installed on the user's system and by verifying if the executable path\n   * provided has the appropriate permissions.\n   *\n   * @param dockerExecutable path to the executable to test running\n   * @return {@code true} if Docker is installed on the user's system and accessible\n   */\n  public static boolean isDockerInstalled(Path dockerExecutable) {\n    return Files.exists(dockerExecutable);\n  }\n\n  /**\n   * Gets a function that takes a {@code docker} subcommand and gives back a {@link ProcessBuilder}\n   * for that {@code docker} command.\n   *\n   * @param dockerExecutable path to {@code docker}\n   * @return the default {@link ProcessBuilder} factory for running a {@code docker} subcommand\n   */\n  @VisibleForTesting\n  static Function<List<String>, ProcessBuilder> defaultProcessBuilderFactory(\n      String dockerExecutable, ImmutableMap<String, String> dockerEnvironment) {\n    return dockerSubCommand -> {\n      List<String> dockerCommand = new ArrayList<>(1 + dockerSubCommand.size());\n      dockerCommand.add(dockerExecutable);\n      dockerCommand.addAll(dockerSubCommand);\n\n      ProcessBuilder processBuilder = new ProcessBuilder(dockerCommand);\n      Map<String, String> environment = processBuilder.environment();\n      environment.putAll(dockerEnvironment);\n\n      return processBuilder;\n    };\n  }\n\n  private static String getStderrOutput(Process process) {\n    try (InputStreamReader stderr =\n        new InputStreamReader(process.getErrorStream(), StandardCharsets.UTF_8)) {\n      return CharStreams.toString(stderr);\n    } catch (IOException ex) {\n      return \"unknown (failed to read error message from stderr due to \" + ex.getMessage() + \")\";\n    }\n  }\n\n  /** Factory for generating the {@link ProcessBuilder} for running {@code docker} commands. */\n  private final Function<List<String>, ProcessBuilder> processBuilderFactory;\n\n  /**\n   * Instantiates with a {@code docker} executable and environment variables.\n   *\n   * @param dockerExecutable path to {@code docker}\n   * @param dockerEnvironment environment variables for {@code docker}\n   */\n  public CliDockerClient(Path dockerExecutable, Map<String, String> dockerEnvironment) {\n    this(\n        defaultProcessBuilderFactory(\n            dockerExecutable.toString(), ImmutableMap.copyOf(dockerEnvironment)));\n  }\n\n  @VisibleForTesting\n  CliDockerClient(Function<List<String>, ProcessBuilder> processBuilderFactory) {\n    this.processBuilderFactory = processBuilderFactory;\n  }\n\n  @Override\n  public boolean supported(Map<String, String> parameters) {\n    return true;\n  }\n\n  @Override\n  public String load(ImageTarball imageTarball, Consumer<Long> writtenByteCountListener)\n      throws InterruptedException, IOException {\n    // Runs 'docker load'.\n    Process dockerProcess = docker(\"load\");\n\n    try (NotifyingOutputStream stdin =\n        new NotifyingOutputStream(dockerProcess.getOutputStream(), writtenByteCountListener)) {\n      imageTarball.writeTo(stdin);\n\n    } catch (IOException ex) {\n      // Tries to read from stderr. Not using getStderrOutput(), as we want to show the error\n      // message from the tarball I/O write failure when reading from stderr fails.\n      String error;\n      try (InputStreamReader stderr =\n          new InputStreamReader(dockerProcess.getErrorStream(), StandardCharsets.UTF_8)) {\n        error = CharStreams.toString(stderr);\n      } catch (IOException ignored) {\n        // This ignores exceptions from reading stderr and uses the original exception from\n        // writing to stdin.\n        error = ex.getMessage();\n      }\n      throw new IOException(\"'docker load' command failed with error: \" + error, ex);\n    }\n\n    try (InputStreamReader stdout =\n        new InputStreamReader(dockerProcess.getInputStream(), StandardCharsets.UTF_8)) {\n      String output = CharStreams.toString(stdout);\n\n      if (dockerProcess.waitFor() != 0) {\n        throw new IOException(\n            \"'docker load' command failed with error: \" + getStderrOutput(dockerProcess));\n      }\n\n      return output;\n    }\n  }\n\n  @Override\n  public void save(\n      ImageReference imageReference, Path outputPath, Consumer<Long> writtenByteCountListener)\n      throws InterruptedException, IOException {\n    Process dockerProcess = docker(\"save\", imageReference.toString());\n\n    try (InputStream stdout = new BufferedInputStream(dockerProcess.getInputStream());\n        OutputStream fileStream = new BufferedOutputStream(Files.newOutputStream(outputPath));\n        NotifyingOutputStream notifyingFileStream =\n            new NotifyingOutputStream(fileStream, writtenByteCountListener)) {\n      ByteStreams.copy(stdout, notifyingFileStream);\n    }\n\n    if (dockerProcess.waitFor() != 0) {\n      throw new IOException(\n          \"'docker save' command failed with error: \" + getStderrOutput(dockerProcess));\n    }\n  }\n\n  @Override\n  public DockerImageDetails inspect(ImageReference imageReference)\n      throws IOException, InterruptedException {\n    Process inspectProcess =\n        docker(\"inspect\", \"-f\", \"{{json .}}\", \"--type\", \"image\", imageReference.toString());\n    if (inspectProcess.waitFor() != 0) {\n      throw new IOException(\n          \"'docker inspect' command failed with error: \" + getStderrOutput(inspectProcess));\n    }\n    return JsonTemplateMapper.readJson(inspectProcess.getInputStream(), DockerImageDetails.class);\n  }\n\n  /** Runs a {@code docker} command. */\n  \nprivate Process docker(String... subCommand) throws IOException {\n    return processBuilderFactory.apply(Arrays.asList(subCommand)).start();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/CredentialHelperNotFoundException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.registry.credentials;\n\nimport java.nio.file.Path;\n\n/** Thrown because the requested credential helper CLI does not exist. */\npublic class CredentialHelperNotFoundException extends CredentialRetrievalException {\n\n  CredentialHelperNotFoundException(Path credentialHelper, Throwable cause) {\n    super(\"The system does not have \" + credentialHelper + \" CLI\", cause);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/CredentialHelperUnhandledServerUrlException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.registry.credentials;\n\nimport java.nio.file.Path;\n\n/** Thrown because the credential helper does not have credentials for the specified server URL. */\npublic class CredentialHelperUnhandledServerUrlException extends CredentialRetrievalException {\n\n  CredentialHelperUnhandledServerUrlException(\n      Path credentialHelper, String serverUrl, String credentialHelperOutput) {\n    super(\n        \"The credential helper (\"\n            + credentialHelper\n            + \") has nothing for server URL: \"\n            + serverUrl\n            + \"\\n\\nGot output:\\n\\n\"\n            + credentialHelperOutput);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/json/DockerConfigTemplate.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.registry.credentials.json;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.common.collect.ImmutableMap;\nimport java.util.HashMap;\nimport java.util.Map;\nimport javax.annotation.Nullable;\n\n/**\n * Template for a Docker config file.\n *\n * <p>Example:\n *\n * <pre>{@code\n * {\n *   \"auths\": {\n *     \"registry\": {\n *       \"auth\": \"username:password string in base64\",\n *       \"username\": \"...\"\n *       \"password\": \"...\"\n *       \"identityToken\": \"...\"\n *     },\n *     \"anotherregistry\": {},\n *     ...\n *   },\n *   \"credsStore\": \"legacy credential helper config acting as a \\\"default\\\" helper\",\n *   \"credHelpers\": {\n *     \"registry\": \"credential helper name\",\n *     \"anotherregistry\": \"another credential helper name\",\n *     ...\n *   }\n * }\n * }</pre>\n *\n * <p>Each entry in {@code credHelpers} is a mapping from a registry to a credential helper that\n * stores the authorization for that registry. This takes precedence over {@code credsStore} if\n * there exists a match.\n *\n * <p>{@code credsStore} is a legacy config that acts to provide a \"default\" credential helper if\n * there is no match in {@code credHelpers}.\n *\n * <p>If an {@code auth} is defined for a registry, that is a valid {@code Basic} authorization to\n * use for that registry.\n */\n@JsonIgnoreProperties(ignoreUnknown = true)\npublic class DockerConfigTemplate implements JsonTemplate {\n\n  /** Template for an {@code auth} defined for a registry under {@code auths}. */\n  @JsonIgnoreProperties(ignoreUnknown = true)\n  public static class AuthTemplate implements JsonTemplate {\n\n    @Nullable private String auth;\n\n    @Nullable private String username;\n\n    @Nullable private String password;\n\n    // Both \"identitytoken\" and \"identityToken\" have been observed. For example,\n    // https://github.com/GoogleContainerTools/jib/issues/2488\n    // https://github.com/spotify/docker-client/issues/580\n    @Nullable private String identityToken;\n\n    @Nullable\n    public String getAuth() {\n      return auth;\n    }\n\n    @Nullable\n    public String getUsername() {\n      return username;\n    }\n\n    @Nullable\n    public String getPassword() {\n      return password;\n    }\n\n    @Nullable\n    public String getIdentityToken() {\n      return identityToken;\n    }\n  }\n\n  /** Maps from registry to its {@link AuthTemplate}. */\n  private final Map<String, AuthTemplate> auths;\n\n  @Nullable private String credsStore;\n\n  /** Maps from registry to credential helper name. */\n  private final Map<String, String> credHelpers = new HashMap<>();\n\n  public DockerConfigTemplate(Map<String, AuthTemplate> auths) {\n    this.auths = ImmutableMap.copyOf(auths);\n  }\n\n  @SuppressWarnings(\"unused\")\n  private DockerConfigTemplate() {\n    auths = new HashMap<>();\n  }\n\n  public Map<String, AuthTemplate> getAuths() {\n    return auths;\n  }\n\n  @Nullable\n  public String getCredsStore() {\n    return credsStore;\n  }\n\n  public Map<String, String> getCredHelpers() {\n    return credHelpers;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/DockerConfigCredentialRetriever.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.registry.credentials;\n\nimport com.fasterxml.jackson.core.type.TypeReference;\nimport com.fasterxml.jackson.databind.MapperFeature;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.fasterxml.jackson.databind.json.JsonMapper;\nimport com.google.cloud.tools.jib.api.Credential;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.registry.RegistryAliasGroup;\nimport com.google.cloud.tools.jib.registry.credentials.json.DockerConfigTemplate;\nimport com.google.cloud.tools.jib.registry.credentials.json.DockerConfigTemplate.AuthTemplate;\nimport com.google.common.annotations.VisibleForTesting;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.Base64;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.function.Consumer;\n\n/**\n * Retrieves registry credentials from the Docker config.\n *\n * <p>The credentials are searched in the following order (stopping when credentials are found):\n *\n * <ol>\n *   <li>The credential helper from {@code credHelpers} defined for a registry, if available.\n *   <li>The {@code credsStore} credential helper, if available.\n *   <li>If there is an {@code auth} defined for a registry.\n * </ol>\n *\n * @see <a\n *     href=\"https://docs.docker.com/engine/reference/commandline/login/\">https://docs.docker.com/engine/reference/commandline/login/</a>\n */\npublic class DockerConfigCredentialRetriever {\n\n  private final String registry;\n  private final Path dockerConfigFile;\n  private final boolean legacyConfigFormat;\n\n  public static DockerConfigCredentialRetriever create(String registry, Path dockerConfigFile) {\n    return new DockerConfigCredentialRetriever(registry, dockerConfigFile, false);\n  }\n\n  public static DockerConfigCredentialRetriever createForLegacyFormat(\n      String registry, Path dockerConfigFile) {\n    return new DockerConfigCredentialRetriever(registry, dockerConfigFile, true);\n  }\n\n  private DockerConfigCredentialRetriever(\n      String registry, Path dockerConfigFile, boolean legacyConfigFormat) {\n    this.registry = registry;\n    this.dockerConfigFile = dockerConfigFile;\n    this.legacyConfigFormat = legacyConfigFormat;\n  }\n\n  public Path getDockerConfigFile() {\n    return dockerConfigFile;\n  }\n\n  /**\n   * Retrieves credentials for a registry. Tries all possible known aliases.\n   *\n   * @param logger a consumer for handling log events\n   * @return {@link Credential} found for {@code registry}, or {@link Optional#empty} if not found\n   * @throws IOException if failed to parse the config JSON\n   */\n  public Optional<Credential> retrieve(Consumer<LogEvent> logger) throws IOException {\n    if (!Files.exists(dockerConfigFile)) {\n      return Optional.empty();\n    }\n\n    ObjectMapper objectMapper =\n        JsonMapper.builder()\n            .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true)\n            .build();\n    try (InputStream fileIn = Files.newInputStream(dockerConfigFile)) {\n      if (legacyConfigFormat) {\n        // legacy config format is the value of the \"auths\":{ <map> } block of the new config (i.e.,\n        // the <map> of string -> DockerConfigTemplate.AuthTemplate).\n        Map<String, AuthTemplate> auths =\n            objectMapper.readValue(fileIn, new TypeReference<Map<String, AuthTemplate>>() {});\n        DockerConfig dockerConfig = new DockerConfig(new DockerConfigTemplate(auths));\n        return retrieve(dockerConfig, logger);\n      }\n\n      DockerConfig dockerConfig =\n          new DockerConfig(objectMapper.readValue(fileIn, DockerConfigTemplate.class));\n      return retrieve(dockerConfig, logger);\n    }\n  }\n\n  /**\n   * Retrieves credentials for a registry alias from a {@link DockerConfig}.\n   *\n   * @param dockerConfig the {@link DockerConfig} to retrieve from\n   * @param logger a consumer for handling log events\n   * @return the retrieved credentials, or {@code Optional#empty} if none are found\n   */\n  @VisibleForTesting\n  Optional<Credential> retrieve(DockerConfig dockerConfig, Consumer<LogEvent> logger) {\n    for (String registryAlias : RegistryAliasGroup.getAliasesGroup(registry)) {\n      // First, find a credential helper from \"credentialHelpers\" and \"credsStore\" in order.\n      DockerCredentialHelper dockerCredentialHelper =\n          dockerConfig.getCredentialHelperFor(registryAlias);\n      if (dockerCredentialHelper != null) {\n        try {\n          Path helperPath = dockerCredentialHelper.getCredentialHelper();\n          logger.accept(LogEvent.info(\"trying \" + helperPath + \" for \" + registryAlias));\n          // Tries with the given registry alias (may be the original registry).\n          return Optional.of(dockerCredentialHelper.retrieve());\n\n        } catch (IOException\n            | CredentialHelperUnhandledServerUrlException\n            | CredentialHelperNotFoundException ex) {\n          // Warns the user that the specified credential helper cannot be used.\n          if (ex.getMessage() != null) {\n            logger.accept(LogEvent.warn(ex.getMessage()));\n            if (ex.getCause() != null && ex.getCause().getMessage() != null) {\n              logger.accept(LogEvent.warn(\"  Caused by: \" + ex.getCause().getMessage()));\n            }\n          }\n        }\n      }\n\n      // Lastly, find defined auth.\n      AuthTemplate auth = dockerConfig.getAuthFor(registryAlias);\n      if (auth != null) {\n        if (auth.getAuth() != null) {\n          // 'auth' is a basic authentication token that should be parsed back into credentials\n          String usernameColonPassword =\n              new String(Base64.getDecoder().decode(auth.getAuth()), StandardCharsets.UTF_8);\n          String username = usernameColonPassword.substring(0, usernameColonPassword.indexOf(\":\"));\n          String password = usernameColonPassword.substring(usernameColonPassword.indexOf(\":\") + 1);\n          logger.accept(\n              LogEvent.info(\n                  \"Docker config auths section defines credentials for \" + registryAlias));\n          if (auth.getIdentityToken() != null\n              // These username and password checks may be unnecessary, but doing so to restrict the\n              // scope only to the Azure behavior to maintain maximum backward-compatibilty.\n              && username.equals(\"00000000-0000-0000-0000-000000000000\")\n              && password.isEmpty()) {\n            logger.accept(\n                LogEvent.info(\"Using 'identityToken' in Docker config auth for \" + registryAlias));\n            return Optional.of(\n                Credential.from(Credential.OAUTH2_TOKEN_USER_NAME, auth.getIdentityToken()));\n          }\n          return Optional.of(Credential.from(username, password));\n        } else if (auth.getUsername() != null && auth.getPassword() != null) {\n          logger.accept(\n              LogEvent.info(\n                  \"Docker config auths section defines username and password for \"\n                      + registryAlias));\n          return Optional.of(Credential.from(auth.getUsername(), auth.getPassword()));\n        }\n      }\n    }\n    return Optional.empty();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/registry/credentials/DockerCredentialHelper.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.registry.credentials;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.google.cloud.tools.jib.api.Credential;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Strings;\nimport com.google.common.io.CharStreams;\nimport java.io.IOException;\nimport java.io.InputStreamReader;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Path;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.function.Function;\nimport javax.annotation.Nullable;\n\n/**\n * Retrieves Docker credentials with a Docker credential helper.\n *\n * @see <a\n *     href=\"https://github.com/docker/docker-credential-helpers\">https://github.com/docker/docker-credential-helpers</a>\n */\npublic class DockerCredentialHelper {\n\n  private final String serverUrl;\n  private final Path credentialHelper;\n  private final Properties systemProperties;\n  private final Function<List<String>, ProcessBuilder> processBuilderFactory;\n  private final Map<String, String> environment;\n\n  /** Template for a Docker credential helper output. */\n  @VisibleForTesting\n  @JsonIgnoreProperties(ignoreUnknown = true)\n  static class DockerCredentialsTemplate implements JsonTemplate {\n\n    @Nullable\n    @VisibleForTesting\n    @JsonProperty(\"Username\")\n    String username;\n\n    @Nullable\n    @VisibleForTesting\n    @JsonProperty(\"Secret\")\n    String secret;\n  }\n\n  /**\n   * Constructs a new {@link DockerCredentialHelper}.\n   *\n   * @param serverUrl the server URL to pass into the credential helper\n   * @param credentialHelper the path to the credential helper executable\n   */\n  public DockerCredentialHelper(String serverUrl, Path credentialHelper) {\n    this(\n        serverUrl,\n        credentialHelper,\n        System.getProperties(),\n        ProcessBuilder::new,\n        Collections.emptyMap());\n  }\n\n  /**\n   * Constructs a new {@link DockerCredentialHelper}.\n   *\n   * @param serverUrl the server URL to pass into the credential helper\n   * @param credentialHelper the path to the credential helper executable\n   * @param environment environment variables used in configuring the credential helper\n   */\n  public DockerCredentialHelper(\n      String serverUrl, Path credentialHelper, Map<String, String> environment) {\n    this(serverUrl, credentialHelper, System.getProperties(), ProcessBuilder::new, environment);\n  }\n\n  @VisibleForTesting\n  DockerCredentialHelper(\n      String serverUrl,\n      Path credentialHelper,\n      Properties systemProperties,\n      Function<List<String>, ProcessBuilder> processBuilderFactory,\n      Map<String, String> environment) {\n    this.serverUrl = serverUrl;\n    this.credentialHelper = credentialHelper;\n    this.systemProperties = systemProperties;\n    this.processBuilderFactory = processBuilderFactory;\n    this.environment = environment;\n  }\n\n  /**\n   * Calls the credential helper CLI.\n   *\n   * <p>Calls occur in the form:\n   *\n   * <pre>{@code\n   * echo -n <server URL> | docker-credential-<credential helper suffix> get\n   * }</pre>\n   *\n   * @return the Docker credentials by calling the corresponding CLI\n   * @throws IOException if writing/reading process input/output fails\n   * @throws CredentialHelperUnhandledServerUrlException if no credentials could be found for the\n   *     corresponding server\n   * @throws CredentialHelperNotFoundException if the credential helper CLI doesn't exist\n   */\n  public Credential retrieve()\n      throws IOException, CredentialHelperUnhandledServerUrlException,\n          CredentialHelperNotFoundException {\n    boolean isWindows =\n        systemProperties.getProperty(\"os.name\").toLowerCase(Locale.ENGLISH).contains(\"windows\");\n    String lowerCaseHelper = credentialHelper.toString().toLowerCase(Locale.ENGLISH);\n    if (!isWindows || lowerCaseHelper.endsWith(\".cmd\") || lowerCaseHelper.endsWith(\".exe\")) {\n      return retrieve(Arrays.asList(credentialHelper.toString(), \"get\"));\n    }\n\n    // We are on Windows with undefined/unknown file extension.\n    for (String suffix : Arrays.asList(\".cmd\", \".exe\")) {\n      try {\n        return retrieve(Arrays.asList(credentialHelper.toString() + suffix, \"get\"));\n      } catch (CredentialHelperNotFoundException ignored) {\n        // ignored\n      }\n    }\n    // On Windows, launching a process from Java without a file extension should normally fail\n    // (https://github.com/GoogleContainerTools/jib/issues/2399#issuecomment-612972912), but\n    // running Jib on Linux-like environment (e.g., Cygwin) might succeed?\n...\n", "needle_token_start": 10395, "needle_token_end": 10430, "code_context_ntokens": 15958, "output": ["```java\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/docker/CliDockerClient.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.docker;\n\nimport com.fasterxml.jackson.annotation.JsonIgnoreProperties;\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.DockerClient;\nimport com.google.cloud.tools.jib.api.ImageDetails;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.http.NotifyingOutputStream;\nimport com.google.cloud.tools.jib.image.ImageTarball;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.io.ByteStreams;\nimport com.google.common.io.CharStreams;\nimport java.io.BufferedInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.io.OutputStream;\nimport java.nio.charset.StandardCharsets;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.security.DigestException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.function.Consumer;\nimport java.util.function.Function;\n\n/** Calls out to the {@code docker} CLI. */\npublic class CliDockerClient implements DockerClient {\n\n  /**\n   * Contains the size, image ID, and diff IDs of an image inspected with {@code docker inspect}.\n   */\n  @JsonIgnoreProperties(ignoreUnknown = true)\n  public static class DockerImageDetails implements JsonTemplate, ImageDetails {\n\n    @JsonIgnoreProperties(ignoreUnknown = true)\n    private static class RootFsTemplate implements JsonTemplate {\n      @JsonProperty(\"Layers\")\n      private final List<String> layers = Collections.emptyList();\n    }\n\n    @JsonProperty(\"Size\")\n    private long size;\n\n    @JsonProperty(\"Id\")\n    private String imageId = \"\";\n\n    @JsonProperty(\"RootFS\")\n    private final RootFsTemplate rootFs = new RootFsTemplate();\n\n    @Override\n    public long getSize() {\n      return size;\n    }\n\n    @Override\n    public DescriptorDigest getImageId() throws DigestException {\n      return DescriptorDigest.fromDigest(imageId);\n    }\n\n    /**\n     * Return a list of diff ids of the layers in the image.\n     *\n     * @return a list of diff ids\n     * @throws DigestException if a digest is invalid\n     */\n    @Override\n    public List<DescriptorDigest> getDiffIds() throws DigestException {\n      List<DescriptorDigest> processedDiffIds = new ArrayList<>(rootFs.layers.size());\n      for (String diffId : rootFs.layers) {\n        processedDiffIds.add(DescriptorDigest.fromDigest(diffId.trim()));\n      }\n      return processedDiffIds;\n    }\n  }\n\n  /** Default path to the docker executable. */\n  public static final Path DEFAULT_DOCKER_CLIENT = Paths.get(\"docker\");\n\n  /**\n   * Checks if Docker is installed on the user's system by running the `docker` command.\n   *\n   * @return {@code true} if Docker is installed on the user's system and accessible\n   */\n  public static boolean isDefaultDockerInstalled() {\n    try {\n      new ProcessBuilder(DEFAULT_DOCKER_CLIENT.toString()).start();\n      return true;\n    } catch (IOException ex) {\n      return false;\n    }\n  }\n\n  /**\n   * Checks if Docker is installed on the user's system and by verifying if the executable path\n   * provided has the appropriate permissions.\n   *\n   * @param dockerExecutable path to the executable to test running\n   * @return {@code true} if Docker is installed on the user's system and accessible\n   */\n  public static boolean isDockerInstalled(Path dockerExecutable) {\n    return Files.exists(dockerExecutable);\n  }\n\n  /**\n   * Gets a function that takes a {@code docker} subcommand and gives back a {@link ProcessBuilder}\n   * for that {@code docker} command.\n   *\n   * @param dockerExecutable path to {@code docker}\n   * @return the default {@link ProcessBuilder} factory for running a {@code docker} subcommand\n   */\n  @VisibleForTesting\n  static Function<List<String>, Process"]}
{"repo": "GoogleContainerTools/jib", "name": "pushBaseImageLayers", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the uploading of prepared image layers to a specified registry. This is part of a larger process in managing Docker image layers, specifically ensuring that the necessary layers are available in the target registry for further operations.\n2. **Input**: The function takes a list of futures representing layers that have been prepared for upload, and a factory for creating progress dispatchers to monitor the upload process.\n3. **Output**: It returns a list of futures, each containing a descriptor of a blob, which represents the result of the upload process for each layer.\n4. **Procedure**: The function first converts the list of prepared layer futures into a list of callable tasks that handle the uploading of each layer. It then schedules these tasks for execution. Each task is responsible for uploading a layer to the registry and returning a descriptor that identifies the uploaded blob.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/http/NotifyingOutputStream.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.http;\n\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.util.function.Consumer;\n\n/** Counts the number of bytes written and reports the count to a callback. */\npublic class NotifyingOutputStream extends OutputStream {\n\n  /** The underlying {@link OutputStream} to wrap and forward bytes to. */\n  private final OutputStream underlyingOutputStream;\n\n  /** Receives a count of bytes written since the last call. */\n  private final Consumer<Long> byteCountListener;\n\n  /** Number of bytes to provide to {@link #byteCountListener}. */\n  private long byteCount = 0;\n\n  /**\n   * Wraps the {@code underlyingOutputStream} to count the bytes written.\n   *\n   * @param underlyingOutputStream the wrapped {@link OutputStream}\n   * @param byteCountListener the byte count {@link Consumer}\n   */\n  public NotifyingOutputStream(\n      OutputStream underlyingOutputStream, Consumer<Long> byteCountListener) {\n    this.underlyingOutputStream = underlyingOutputStream;\n    this.byteCountListener = byteCountListener;\n  }\n\n  @Override\n  public void write(int singleByte) throws IOException {\n    underlyingOutputStream.write(singleByte);\n    countAndCallListener(1);\n  }\n\n  @Override\n  public void write(byte[] byteArray) throws IOException {\n    underlyingOutputStream.write(byteArray);\n    countAndCallListener(byteArray.length);\n  }\n\n  @Override\n  public void write(byte[] byteArray, int offset, int length) throws IOException {\n    underlyingOutputStream.write(byteArray, offset, length);\n    countAndCallListener(length);\n  }\n\n  @Override\n  public void flush() throws IOException {\n    underlyingOutputStream.flush();\n    countAndCallListener(0);\n  }\n\n  @Override\n  public void close() throws IOException {\n    underlyingOutputStream.close();\n    countAndCallListener(0);\n  }\n\n  private void countAndCallListener(int written) {\n    this.byteCount += written;\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/DirectoryWalker.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.filesystem;\n\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.NotDirectoryException;\nimport java.nio.file.Path;\nimport java.util.function.Predicate;\nimport java.util.stream.Stream;\n\n/** Recursively applies a function to each file in a directory. */\npublic class DirectoryWalker {\n\n  private final Path rootDir;\n\n  private Predicate<Path> pathFilter = path -> true;\n\n  /**\n   * Initialize with a root directory to walk.\n   *\n   * @param rootDir the root directory.\n   * @throws NotDirectoryException if the root directory is not a directory.\n   */\n  public DirectoryWalker(Path rootDir) throws NotDirectoryException {\n    if (!Files.isDirectory(rootDir)) {\n      throw new NotDirectoryException(rootDir + \" is not a directory\");\n    }\n    this.rootDir = rootDir;\n  }\n\n  /**\n   * Adds a filter to the walked paths.\n   *\n   * @param pathFilter the filter. {@code pathFilter} returns {@code true} if the path should be\n   *     accepted and {@code false} otherwise.\n   * @return this\n   */\n  public DirectoryWalker filter(Predicate<Path> pathFilter) {\n    this.pathFilter = this.pathFilter.and(pathFilter);\n    return this;\n  }\n\n  /**\n   * Filters away the {@code rootDir}.\n   *\n   * @return this\n   */\n  public DirectoryWalker filterRoot() {\n    filter(path -> !path.equals(rootDir));\n    return this;\n  }\n\n  /**\n   * Walks {@link #rootDir} and applies {@code pathConsumer} to each file. Note that {@link\n   * #rootDir} itself is visited as well.\n   *\n   * @param pathConsumer the consumer that is applied to each file.\n   * @return a list of Paths that were walked.\n   * @throws IOException if the walk fails.\n   */\n  public ImmutableList<Path> walk(PathConsumer pathConsumer) throws IOException {\n    ImmutableList<Path> files = walk();\n    for (Path path : files) {\n      pathConsumer.accept(path);\n    }\n    return files;\n  }\n\n  /**\n   * Walks {@link #rootDir}.\n   *\n   * @return the walked files.\n   * @throws IOException if walking the files fails.\n   */\n  public ImmutableList<Path> walk() throws IOException {\n    try (Stream<Path> fileStream = Files.walk(rootDir)) {\n      return fileStream.filter(pathFilter).sorted().collect(ImmutableList.toImmutableList());\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarExtractor.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.tar;\n\nimport com.google.cloud.tools.jib.filesystem.DirectoryWalker;\nimport com.google.common.io.ByteStreams;\nimport java.io.BufferedInputStream;\nimport java.io.BufferedOutputStream;\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.nio.file.attribute.FileTime;\nimport java.util.ArrayList;\nimport java.util.List;\nimport org.apache.commons.compress.archivers.tar.TarArchiveEntry;\nimport org.apache.commons.compress.archivers.tar.TarArchiveInputStream;\n\n/** Extracts a tarball. */\npublic class TarExtractor {\n\n  private TarExtractor() {}\n\n  /**\n   * Extracts a tarball to the specified destination.\n   *\n   * @param source the tarball to extract\n   * @param destination the output directory\n   * @throws IOException if extraction fails\n   */\n  public static void extract(Path source, Path destination) throws IOException {\n    extract(source, destination, false);\n  }\n\n  /**\n   * Extracts a tarball to the specified destination.\n   *\n   * @param source the tarball to extract\n   * @param destination the output directory\n   * @param enableReproducibleTimestamps whether or not reproducible timestamps should be used\n   * @throws IOException if extraction fails\n   * @throws IllegalStateException when reproducible timestamps are enabled but the target root used\n   *     for extracting the tar contents is not empty\n   */\n  public static void extract(Path source, Path destination, boolean enableReproducibleTimestamps)\n      throws IOException {\n    if (enableReproducibleTimestamps\n        && Files.isDirectory(destination)\n        && destination.toFile().list().length != 0) {\n      throw new IllegalStateException(\n          \"Cannot enable reproducible timestamps. They can only be enabled when the target root doesn't exist or is an empty directory\");\n    }\n    String canonicalDestination = destination.toFile().getCanonicalPath();\n    List<TarArchiveEntry> entries = new ArrayList<>();\n    try (InputStream in = new BufferedInputStream(Files.newInputStream(source));\n        TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(in)) {\n      for (TarArchiveEntry entry = tarArchiveInputStream.getNextTarEntry();\n          entry != null;\n          entry = tarArchiveInputStream.getNextTarEntry()) {\n        entries.add(entry);\n        Path entryPath = destination.resolve(entry.getName());\n\n        String canonicalTarget = entryPath.toFile().getCanonicalPath();\n        if (!canonicalTarget.startsWith(canonicalDestination + File.separator)) {\n          String offender = entry.getName() + \" from \" + source;\n          throw new IOException(\"Blocked unzipping files outside destination: \" + offender);\n        }\n        if (entry.isDirectory()) {\n          Files.createDirectories(entryPath);\n        } else {\n          if (entryPath.getParent() != null) {\n            Files.createDirectories(entryPath.getParent());\n          }\n\n          if (entry.isSymbolicLink()) {\n            Files.createSymbolicLink(entryPath, Paths.get(entry.getLinkName()));\n          } else {\n            try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(entryPath))) {\n              ByteStreams.copy(tarArchiveInputStream, out);\n            }\n          }\n        }\n      }\n    }\n    preserveModificationTimes(destination, entries, enableReproducibleTimestamps);\n  }\n\n  /**\n   * Preserve modification timestamps of files and directories in a tar file. If a directory is not\n   * an entry in the tar file and reproducible timestamps are enabled then its modification\n   * timestamp is set to a constant value. Note that the modification timestamps of symbolic links\n   * are not preserved even with reproducible timestamps enabled.\n   *\n   * @param destination target root for unzipping\n   * @param entries list of entries in tar file\n   * @param enableReproducibleTimestamps whether or not reproducible timestamps should be used\n   * @throws IOException when I/O error occurs\n   */\n  private static void preserveModificationTimes(\n      Path destination, List<TarArchiveEntry> entries, boolean enableReproducibleTimestamps)\n      throws IOException {\n    if (enableReproducibleTimestamps) {\n      FileTime epochPlusOne = FileTime.fromMillis(1000L);\n      new DirectoryWalker(destination)\n          .filter(Files::isDirectory)\n          .walk(path -> Files.setLastModifiedTime(path, epochPlusOne));\n    }\n    for (TarArchiveEntry entry : entries) {\n\n      // Setting the symbolic link's modification timestamp will cause the modification timestamp of\n      // the target to change\n      if (!entry.isSymbolicLink()) {\n        Files.setLastModifiedTime(\n            destination.resolve(entry.getName()), FileTime.from(entry.getModTime().toInstant()));\n      }\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LocalBaseImageSteps.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.fasterxml.jackson.databind.MapperFeature;\nimport com.fasterxml.jackson.databind.json.JsonMapper;\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.DockerClient;\nimport com.google.cloud.tools.jib.api.ImageDetails;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.blob.Blobs;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImagesAndRegistryClient;\nimport com.google.cloud.tools.jib.cache.Cache;\nimport com.google.cloud.tools.jib.cache.CacheCorruptedException;\nimport com.google.cloud.tools.jib.cache.CachedLayer;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate;\nimport com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer;\nimport com.google.cloud.tools.jib.filesystem.TempDirectoryProvider;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.http.NotifyingOutputStream;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.LayerCountMismatchException;\nimport com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;\nimport com.google.cloud.tools.jib.image.json.JsonToImageTranslator;\nimport com.google.cloud.tools.jib.image.json.V22ManifestTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.cloud.tools.jib.tar.TarExtractor;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.io.ByteStreams;\nimport com.google.common.util.concurrent.Futures;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.security.DigestException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Future;\nimport java.util.zip.GZIPInputStream;\nimport java.util.zip.GZIPOutputStream;\n\n/** Extracts a tar file base image. */\npublic class LocalBaseImageSteps {\n\n  /** Contains an {@link Image} and its layers. * */\n  static class LocalImage {\n    final List<Future<PreparedLayer>> layers;\n    final ContainerConfigurationTemplate configurationTemplate;\n\n    LocalImage(\n        List<Future<PreparedLayer>> layers, ContainerConfigurationTemplate configurationTemplate) {\n      this.layers = layers;\n      this.configurationTemplate = configurationTemplate;\n    }\n  }\n\n  private LocalBaseImageSteps() {}\n\n  /**\n   * Checks the first two bytes of a file to see if it has been gzipped.\n   *\n   * @param path the file to check\n   * @return {@code true} if the file is gzipped, {@code false} if not\n   * @throws IOException if reading the file fails\n   * @see <a href=\"http://www.zlib.org/rfc-gzip.html#file-format\">GZIP file format</a>\n   */\n  @VisibleForTesting\n  static boolean isGzipped(Path path) throws IOException {\n    try (InputStream inputStream = Files.newInputStream(path)) {\n      inputStream.mark(2);\n      int magic = (inputStream.read() & 0xff) | ((inputStream.read() << 8) & 0xff00);\n      return magic == GZIPInputStream.GZIP_MAGIC;\n    }\n  }\n\n  static Callable<LocalImage> retrieveDockerDaemonLayersStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      DockerClient dockerClient,\n      TempDirectoryProvider tempDirectoryProvider) {\n    return () -> {\n      ImageReference imageReference = buildContext.getBaseImageConfiguration().getImage();\n      try (ProgressEventDispatcher progressEventDispatcher =\n              progressEventDispatcherFactory.create(\"processing base image \" + imageReference, 2);\n          TimerEventDispatcher ignored =\n              new TimerEventDispatcher(\n                  buildContext.getEventHandlers(),\n                  \"Saving \" + imageReference + \" from Docker daemon\")) {\n        ImageDetails dockerImageDetails = dockerClient.inspect(imageReference);\n        Optional<LocalImage> cachedImage =\n            getCachedDockerImage(buildContext.getBaseImageLayersCache(), dockerImageDetails);\n        if (cachedImage.isPresent()) {\n          PlatformChecker.checkManifestPlatform(\n              buildContext, cachedImage.get().configurationTemplate);\n          return cachedImage.get();\n        }\n\n        Path tarPath = tempDirectoryProvider.newDirectory().resolve(\"out.tar\");\n        long size = dockerImageDetails.getSize();\n        try (ProgressEventDispatcher dockerProgress =\n                progressEventDispatcher\n                    .newChildProducer()\n                    .create(\"saving base image \" + imageReference, size);\n            ThrottledAccumulatingConsumer throttledProgressReporter =\n                new ThrottledAccumulatingConsumer(dockerProgress::dispatchProgress)) {\n          dockerClient.save(imageReference, tarPath, throttledProgressReporter);\n        }\n\n        LocalImage localImage =\n            cacheDockerImageTar(\n                buildContext,\n                tarPath,\n                progressEventDispatcher.newChildProducer(),\n                tempDirectoryProvider);\n        PlatformChecker.checkManifestPlatform(buildContext, localImage.configurationTemplate);\n        return localImage;\n      }\n    };\n  }\n\n  static Callable<LocalImage> retrieveTarLayersStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      Path tarPath,\n      TempDirectoryProvider tempDirectoryProvider) {\n    return () -> {\n      LocalImage localImage =\n          cacheDockerImageTar(\n              buildContext, tarPath, progressEventDispatcherFactory, tempDirectoryProvider);\n      PlatformChecker.checkManifestPlatform(buildContext, localImage.configurationTemplate);\n      return localImage;\n    };\n  }\n\n  static Callable<ImagesAndRegistryClient> returnImageAndRegistryClientStep(\n      List<PreparedLayer> layers, ContainerConfigurationTemplate configurationTemplate) {\n    return () -> {\n      // Collect compressed layers and add to manifest\n      V22ManifestTemplate v22Manifest = new V22ManifestTemplate();\n      for (PreparedLayer layer : layers) {\n        BlobDescriptor descriptor = layer.getBlobDescriptor();\n        v22Manifest.addLayer(descriptor.getSize(), descriptor.getDigest());\n      }\n\n      BlobDescriptor configDescriptor = Digests.computeDigest(configurationTemplate);\n      v22Manifest.setContainerConfiguration(\n          configDescriptor.getSize(), configDescriptor.getDigest());\n      return new ImagesAndRegistryClient(\n          Collections.singletonList(\n              JsonToImageTranslator.toImage(v22Manifest, configurationTemplate)),\n          null);\n    };\n  }\n\n  @VisibleForTesting\n  static Optional<LocalImage> getCachedDockerImage(Cache cache, ImageDetails dockerImageDetails)\n      throws DigestException, IOException, CacheCorruptedException {\n    // Get config\n    Optional<ContainerConfigurationTemplate> cachedConfig =\n        cache.retrieveLocalConfig(dockerImageDetails.getImageId());\n    if (!cachedConfig.isPresent()) {\n      return Optional.empty();\n    }\n\n    // Get layers\n    List<Future<PreparedLayer>> cachedLayers = new ArrayList<>();\n    for (DescriptorDigest diffId : dockerImageDetails.getDiffIds()) {\n      Optional<CachedLayer> cachedLayer = cache.retrieveTarLayer(diffId);\n      if (!cachedLayer.isPresent()) {\n        return Optional.empty();\n      }\n      CachedLayer layer = cachedLayer.get();\n      cachedLayers.add(Futures.immediateFuture(new PreparedLayer.Builder(layer).build()));\n    }\n\n    return Optional.of(new LocalImage(cachedLayers, cachedConfig.get()));\n  }\n\n  @VisibleForTesting\n  static LocalImage cacheDockerImageTar(\n      BuildContext buildContext,\n      Path tarPath,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      TempDirectoryProvider tempDirectoryProvider)\n      throws IOException, LayerCountMismatchException {\n    ExecutorService executorService = buildContext.getExecutorService();\n    Path destination = tempDirectoryProvider.newDirectory();\n\n    try (TimerEventDispatcher ignored =\n        new TimerEventDispatcher(\n            buildContext.getEventHandlers(),\n            \"Extracting tar \" + tarPath + \" into \" + destination)) {\n      TarExtractor.extract(tarPath, destination);\n\n      DockerManifestEntryTemplate loadManifest;\n      try (InputStream manifestStream =\n          Files.newInputStream(destination.resolve(\"manifest.json\"))) {\n        loadManifest =\n            JsonMapper.builder()\n                .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true)\n                .build()\n                .readValue(manifestStream, DockerManifestEntryTemplate[].class)[0];\n      }\n\n      Path configPath = destination.resolve(loadManifest.getConfig());\n      ContainerConfigurationTemplate configurationTemplate =\n          JsonTemplateMapper.readJsonFromFile(configPath, ContainerConfigurationTemplate.class);\n      // Don't compute the digest of the loaded Java JSON instance.\n      BlobDescriptor originalConfigDescriptor =\n          Blobs.from(configPath).writeTo(ByteStreams.nullOutputStream());\n\n      List<String> layerFiles = loadManifest.getLayerFiles();\n      if (configurationTemplate.getLayerCount() != layerFiles.size()) {\n        throw new LayerCountMismatchException(\n            \"Invalid base image format: manifest contains \"\n                + layerFiles.size()\n                + \" layers, but container configuration contains \"\n                + configurationTemplate.getLayerCount()\n                + \" layers\");\n      }\n      buildContext\n          .getBaseImageLayersCache()\n          .writeLocalConfig(originalConfigDescriptor.getDigest(), configurationTemplate);\n\n      // Check the first layer to see if the layers are compressed already. 'docker save' output\n      // is uncompressed, but a jib-built tar has compressed layers.\n      boolean layersAreCompressed =\n          !layerFiles.isEmpty() && isGzipped(destination.resolve(layerFiles.get(0)));\n\n      // Process layer blobs\n      try (ProgressEventDispatcher progressEventDispatcher =\n          progressEventDispatcherFactory.create(\n              \"processing base image layers\", layerFiles.size())) {\n        // Start compressing layers in parallel\n        List<Future<PreparedLayer>> preparedLayers = new ArrayList<>();\n        for (int index = 0; index < layerFiles.size(); index++) {\n          Path layerFile = destination.resolve(layerFiles.get(index));\n          DescriptorDigest diffId = configurationTemplate.getLayerDiffId(index);\n          ProgressEventDispatcher.Factory layerProgressDispatcherFactory =\n              progressEventDispatcher.newChildProducer();\n          preparedLayers.add(\n              executorService.submit(\n                  () ->\n                      compressAndCacheTarLayer(\n                          buildContext.getBaseImageLayersCache(),\n                          diffId,\n                          layerFile,\n                          layersAreCompressed,\n                          layerProgressDispatcherFactory)));\n        }\n        return new LocalImage(preparedLayers, configurationTemplate);\n      }\n    }\n  }\n\n  private static PreparedLayer compressAndCacheTarLayer(\n      Cache cache,\n      DescriptorDigest diffId,\n      Path layerFile,\n      boolean layersAreCompressed,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory)\n      throws IOException, CacheCorruptedException {\n    try (ProgressEventDispatcher childDispatcher =\n            progressEventDispatcherFactory.create(\n                \"compressing layer \" + diffId, Files.size(layerFile));\n        ThrottledAccumulatingConsumer throttledProgressReporter =\n            new ThrottledAccumulatingConsumer(childDispatcher::dispatchProgress)) {\n      // Retrieve pre-compressed layer from cache\n      Optional<CachedLayer> optionalLayer = cache.retrieveTarLayer(diffId);\n      if (optionalLayer.isPresent()) {\n        return new PreparedLayer.Builder(optionalLayer.get()).build();\n      }\n\n      // Just write layers that are already compressed\n      if (layersAreCompressed) {\n        return new PreparedLayer.Builder(cache.writeTarLayer(diffId, Blobs.from(layerFile)))\n            .build();\n      }\n\n      // Compress uncompressed layers while writing\n      Blob compressedBlob =\n          Blobs.from(\n              outputStream -> {\n                try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream);\n                    NotifyingOutputStream notifyingOutputStream =\n                        new NotifyingOutputStream(compressorStream, throttledProgressReporter)) {\n                  Blobs.from(layerFile).writeTo(notifyingOutputStream);\n                }\n              },\n              true);\n      return new PreparedLayer.Builder(cache.writeTarLayer(diffId, compressedBlob)).build();\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.builder.steps.PreparedLayer.StateInTarget;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.util.List;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.Future;\n\nclass PushLayerStep implements Callable<BlobDescriptor> {\n\n  static ImmutableList<PushLayerStep> makeList(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      List<Future<PreparedLayer>> cachedLayers) {\n    try (TimerEventDispatcher ignored =\n            new TimerEventDispatcher(buildContext.getEventHandlers(), \"Preparing layer pushers\");\n        ProgressEventDispatcher progressEventDispatcher =\n            progressEventDispatcherFactory.create(\"launching layer pushers\", cachedLayers.size())) {\n\n      // Constructs a PushBlobStep for each layer.\n      return cachedLayers.stream()\n          .map(\n              layer ->\n                  new PushLayerStep(\n                      buildContext,\n                      progressEventDispatcher.newChildProducer(),\n                      registryClient,\n                      layer))\n          .collect(ImmutableList.toImmutableList());\n    }\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n\n  private final RegistryClient registryClient;\n  private final Future<PreparedLayer> preparedLayer;\n\n  private PushLayerStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      Future<PreparedLayer> preparedLayer) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.registryClient = registryClient;\n    this.preparedLayer = preparedLayer;\n  }\n\n  @Override\n  public BlobDescriptor call()\n      throws IOException, RegistryException, ExecutionException, InterruptedException {\n    PreparedLayer layer = preparedLayer.get();\n\n    if (layer.getStateInTarget() == StateInTarget.EXISTING) {\n      return layer.getBlobDescriptor(); // skip pushing if known to exist in registry\n    }\n\n    boolean forcePush = layer.getStateInTarget() == StateInTarget.MISSING;\n    return new PushBlobStep(\n            buildContext,\n            progressEventDispatcherFactory,\n            registryClient,\n            layer.getBlobDescriptor(),\n            layer.getBlob(),\n            forcePush)\n        .call();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.DockerClient;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.steps.LocalBaseImageSteps.LocalImage;\nimport com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImagesAndRegistryClient;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ImageConfiguration;\nimport com.google.cloud.tools.jib.filesystem.TempDirectoryProvider;\nimport com.google.cloud.tools.jib.global.JibSystemProperties;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.Layer;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.registry.ManifestAndDigest;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport com.google.common.base.Verify;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.util.concurrent.Futures;\nimport com.google.common.util.concurrent.ListeningExecutorService;\nimport com.google.common.util.concurrent.MoreExecutors;\nimport java.nio.file.Path;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.LinkedHashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.concurrent.Callable;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Future;\nimport java.util.function.Consumer;\nimport java.util.stream.Collectors;\nimport javax.annotation.Nullable;\n\n/**\n * Runs steps for building an image.\n *\n * <p>Use by first calling {@link #begin} and then calling the individual step running methods. Note\n * that order matters, so make sure that steps are run before other steps that depend on them. Wait\n * on the last step by calling the respective {@code wait...} methods.\n */\npublic class StepsRunner {\n\n  /** Holds the individual step results. */\n  private static class StepResults {\n\n    private static <E> Future<E> failedFuture() {\n      return Futures.immediateFailedFuture(\n          new IllegalStateException(\"invalid usage; required step not configured\"));\n    }\n\n    @Nullable private List<Future<PreparedLayer>> applicationLayers;\n    private Future<ManifestTemplate> manifestListOrSingleManifest = failedFuture();\n    private Future<RegistryClient> targetRegistryClient = failedFuture();\n    private Future<List<Future<BlobDescriptor>>> applicationLayerPushResults = failedFuture();\n    private Future<Optional<ManifestAndDigest<ManifestTemplate>>> manifestCheckResult =\n        failedFuture();\n    private Future<List<Future<BuildResult>>> imagePushResults = failedFuture();\n    private Future<BuildResult> buildResult = failedFuture();\n\n    private Future<ImagesAndRegistryClient> baseImagesAndRegistryClient = failedFuture();\n    private Future<Map<Image, List<Future<PreparedLayer>>>> baseImagesAndLayers = failedFuture();\n    private Future<Map<Image, List<Future<BlobDescriptor>>>> baseImagesAndLayerPushResults =\n        failedFuture();\n    private Future<Map<Image, Future<BlobDescriptor>>> baseImagesAndContainerConfigPushResults =\n        failedFuture();\n    private Future<Map<Image, Future<Image>>> baseImagesAndBuiltImages = failedFuture();\n  }\n\n  /**\n   * Starts building the steps to run.\n   *\n   * @param buildContext the {@link BuildContext}\n   * @return a new {@link StepsRunner}\n   */\n  public static StepsRunner begin(BuildContext buildContext) {\n    ExecutorService executorService =\n        JibSystemProperties.serializeExecution()\n            ? MoreExecutors.newDirectExecutorService()\n            : buildContext.getExecutorService();\n\n    return new StepsRunner(MoreExecutors.listeningDecorator(executorService), buildContext);\n  }\n\n  private static <E> List<E> realizeFutures(Collection<Future<E>> futures)\n      throws InterruptedException, ExecutionException {\n    List<E> values = new ArrayList<>();\n    for (Future<E> future : futures) {\n      values.add(future.get());\n    }\n    return values;\n  }\n\n  private final StepResults results = new StepResults();\n\n  private final ExecutorService executorService;\n  private final BuildContext buildContext;\n  private final TempDirectoryProvider tempDirectoryProvider = new TempDirectoryProvider();\n\n  // Instead of directly running each step, we first save them as a lambda. This is only because of\n  // the unfortunate chicken-and-egg situation when using ProgressEventDispatcher. The current\n  // ProgressEventDispatcher model requires allocating the total units of work (i.e., steps)\n  // up front. That is, to instantiate a root ProgressEventDispatcher, we should know ahead how many\n  // steps we will run. However, to run a step, we need a root progress dispatcher. So, we take each\n  // step as a lambda and save them to run later. Then we can count the number of lambdas, create a\n  // root dispatcher with the count, and run the saved lambdas using the dispatcher.\n  private final List<Consumer<ProgressEventDispatcher.Factory>> stepsToRun = new ArrayList<>();\n\n  @Nullable private String rootProgressDescription;\n\n  @VisibleForTesting\n  StepsRunner(ListeningExecutorService executorService, BuildContext buildContext) {\n    this.executorService = executorService;\n    this.buildContext = buildContext;\n  }\n\n  /**\n   * Add steps for loading an image to docker daemon.\n   *\n   * @param dockerClient the docker client to load the image to\n   * @return this\n   */\n  public StepsRunner dockerLoadSteps(DockerClient dockerClient) {\n    rootProgressDescription = \"building image to Docker daemon\";\n\n    addRetrievalSteps(true); // always pull layers for docker builds\n    stepsToRun.add(this::buildAndCacheApplicationLayers);\n    stepsToRun.add(this::buildImages);\n\n    // load to Docker\n    stepsToRun.add(\n        progressDispatcherFactory -> loadDocker(dockerClient, progressDispatcherFactory));\n    return this;\n  }\n\n  /**\n   * Add steps for writing an image as a tar file archive.\n   *\n   * @param outputPath the target file path to write the image to\n   * @return this\n   */\n  public StepsRunner tarBuildSteps(Path outputPath) {\n    rootProgressDescription = \"building image to tar file\";\n\n    addRetrievalSteps(true); // always pull layers for tar builds\n    stepsToRun.add(this::buildAndCacheApplicationLayers);\n    stepsToRun.add(this::buildImages);\n\n    // create a tar\n    stepsToRun.add(\n        progressDispatcherFactory -> writeTarFile(outputPath, progressDispatcherFactory));\n    return this;\n  }\n\n  /**\n   * Add steps for pushing images to a remote registry. The registry is determined by the image\n   * name.\n   *\n   * @return this\n   */\n  public StepsRunner registryPushSteps() {\n    rootProgressDescription = \"building images to registry\";\n    boolean layersRequiredLocally = buildContext.getAlwaysCacheBaseImage();\n\n    stepsToRun.add(this::authenticateBearerPush);\n\n    addRetrievalSteps(layersRequiredLocally);\n    stepsToRun.add(this::buildAndCacheApplicationLayers);\n    stepsToRun.add(this::buildImages);\n    stepsToRun.add(this::buildManifestListOrSingleManifest);\n\n    // push to registry\n    stepsToRun.add(this::pushBaseImagesLayers);\n    stepsToRun.add(this::pushApplicationLayers);\n    stepsToRun.add(this::pushContainerConfigurations);\n    stepsToRun.add(this::checkManifestInTargetRegistry);\n    stepsToRun.add(this::pushImages);\n    stepsToRun.add(this::pushManifestList);\n    return this;\n  }\n\n  /**\n   * Run all steps and return a BuildResult after a build is completed.\n   *\n   * @return a {@link BuildResult} with build metadata\n   * @throws ExecutionException if an error occurred during asynchronous execution of steps\n   * @throws InterruptedException if the build was interrupted while waiting for results\n   */\n  public BuildResult run() throws ExecutionException, InterruptedException {\n    Preconditions.checkNotNull(rootProgressDescription);\n\n    try (ProgressEventDispatcher progressEventDispatcher =\n        ProgressEventDispatcher.newRoot(\n            buildContext.getEventHandlers(), rootProgressDescription, stepsToRun.size())) {\n      stepsToRun.forEach(step -> step.accept(progressEventDispatcher.newChildProducer()));\n      return results.buildResult.get();\n\n    } catch (ExecutionException ex) {\n      ExecutionException unrolled = ex;\n      while (unrolled.getCause() instanceof ExecutionException) {\n        unrolled = (ExecutionException) unrolled.getCause();\n      }\n      throw unrolled;\n\n    } finally {\n      tempDirectoryProvider.close();\n    }\n  }\n\n  private void addRetrievalSteps(boolean layersRequiredLocally) {\n    ImageConfiguration baseImageConfiguration = buildContext.getBaseImageConfiguration();\n\n    if (baseImageConfiguration.getTarPath().isPresent()) {\n      // If tarPath is present, a TarImage was used\n      stepsToRun.add(this::extractTar);\n\n    } else if (baseImageConfiguration.getDockerClient().isPresent()) {\n      // If dockerClient is present, a DockerDaemonImage was used\n      stepsToRun.add(this::saveDocker);\n\n    } else {\n      // Otherwise default to RegistryImage\n      stepsToRun.add(this::pullBaseImages);\n      stepsToRun.add(\n          progressDispatcherFactory ->\n              obtainBaseImagesLayers(layersRequiredLocally, progressDispatcherFactory));\n    }\n  }\n\n  private void authenticateBearerPush(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.targetRegistryClient =\n        executorService.submit(new AuthenticatePushStep(buildContext, progressDispatcherFactory));\n  }\n\n  private void saveDocker(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    Optional<DockerClient> dockerClient =\n        buildContext.getBaseImageConfiguration().getDockerClient();\n    Preconditions.checkArgument(dockerClient.isPresent());\n\n    assignLocalImageResult(\n        executorService.submit(\n            LocalBaseImageSteps.retrieveDockerDaemonLayersStep(\n                buildContext,\n                progressDispatcherFactory,\n                dockerClient.get(),\n                tempDirectoryProvider)));\n  }\n\n  private void extractTar(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    Optional<Path> tarPath = buildContext.getBaseImageConfiguration().getTarPath();\n    Preconditions.checkArgument(tarPath.isPresent());\n\n    assignLocalImageResult(\n        executorService.submit(\n            LocalBaseImageSteps.retrieveTarLayersStep(\n                buildContext, progressDispatcherFactory, tarPath.get(), tempDirectoryProvider)));\n  }\n\n  private void assignLocalImageResult(Future<LocalImage> localImage) {\n    results.baseImagesAndRegistryClient =\n        executorService.submit(\n            () ->\n                LocalBaseImageSteps.returnImageAndRegistryClientStep(\n                        realizeFutures(localImage.get().layers),\n                        localImage.get().configurationTemplate)\n                    .call());\n\n    results.baseImagesAndLayers =\n        executorService.submit(\n            () ->\n                Collections.singletonMap(\n                    results.baseImagesAndRegistryClient.get().images.get(0),\n                    localImage.get().layers));\n  }\n\n  @VisibleForTesting\n  void pullBaseImages(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.baseImagesAndRegistryClient =\n        executorService.submit(new PullBaseImageStep(buildContext, progressDispatcherFactory));\n  }\n\n  private void obtainBaseImagesLayers(\n      boolean layersRequiredLocally, ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.baseImagesAndLayers =\n        executorService.submit(\n            () -> {\n              try (ProgressEventDispatcher progressDispatcher =\n                  progressDispatcherFactory.create(\n                      \"scheduling obtaining base images layers\",\n                      results.baseImagesAndRegistryClient.get().images.size())) {\n\n                Map<DescriptorDigest, Future<PreparedLayer>> preparedLayersCache = new HashMap<>();\n                Map<Image, List<Future<PreparedLayer>>> baseImagesAndLayers = new LinkedHashMap<>();\n                for (Image baseImage : results.baseImagesAndRegistryClient.get().images) {\n                  List<Future<PreparedLayer>> layers =\n                      obtainBaseImageLayers(\n                          baseImage,\n                          layersRequiredLocally,\n                          preparedLayersCache,\n                          progressDispatcher.newChildProducer());\n                  baseImagesAndLayers.put(baseImage, layers);\n                }\n                return baseImagesAndLayers;\n              }\n            });\n  }\n\n  // This method updates the given \"preparedLayersCache\" and should not be called concurrently.\n  @VisibleForTesting\n  List<Future<PreparedLayer>> obtainBaseImageLayers(\n      Image baseImage,\n      boolean layersRequiredLocally,\n      Map<DescriptorDigest, Future<PreparedLayer>> preparedLayersCache,\n      ProgressEventDispatcher.Factory progressDispatcherFactory)\n      throws InterruptedException, ExecutionException {\n    List<Future<PreparedLayer>> preparedLayers = new ArrayList<>();\n\n    try (ProgressEventDispatcher progressDispatcher =\n        progressDispatcherFactory.create(\n            \"launching base image layer pullers\", baseImage.getLayers().size())) {\n      for (Layer layer : baseImage.getLayers()) {\n        DescriptorDigest digest = layer.getBlobDescriptor().getDigest();\n        Future<PreparedLayer> preparedLayer = preparedLayersCache.get(digest);\n\n        if (preparedLayer != null) {\n          progressDispatcher.dispatchProgress(1);\n        } else { // If we haven't obtained this layer yet, launcher a puller.\n          preparedLayer =\n              executorService.submit(\n                  layersRequiredLocally\n                      ? ObtainBaseImageLayerStep.forForcedDownload(\n                          buildContext,\n                          progressDispatcher.newChildProducer(),\n                          layer,\n                          results.baseImagesAndRegistryClient.get().registryClient)\n                      : ObtainBaseImageLayerStep.forSelectiveDownload(\n                          buildContext,\n                          progressDispatcher.newChildProducer(),\n                          layer,\n                          results.baseImagesAndRegistryClient.get().registryClient,\n                          results.targetRegistryClient.get()));\n          preparedLayersCache.put(digest, preparedLayer);\n        }\n        preparedLayers.add(preparedLayer);\n      }\n      return preparedLayers;\n    }\n  }\n\n  private void pushBaseImagesLayers(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.baseImagesAndLayerPushResults =\n        executorService.submit(\n            () -> {\n              try (ProgressEventDispatcher progressDispatcher =\n                  progressDispatcherFactory.create(\n                      \"scheduling pushing base images layers\",\n                      results.baseImagesAndLayers.get().size())) {\n\n                Map<Image, List<Future<BlobDescriptor>>> layerPushResults = new LinkedHashMap<>();\n                for (Map.Entry<Image, List<Future<PreparedLayer>>> entry :\n                    results.baseImagesAndLayers.get().entrySet()) {\n                  Image baseImage = entry.getKey();\n                  List<Future<PreparedLayer>> baseLayers = entry.getValue();\n\n                  List<Future<BlobDescriptor>> pushResults =\n                      pushBaseImageLayers(baseLayers, progressDispatcher.newChildProducer());\n                  layerPushResults.put(baseImage, pushResults);\n                }\n                return layerPushResults;\n              }\n            });\n  }\n\n  \nprivate List<Future<BlobDescriptor>> pushBaseImageLayers(\n      List<Future<PreparedLayer>> baseLayers,\n      ProgressEventDispatcher.Factory progressDispatcherFactory)\n      throws InterruptedException, ExecutionException {\n    return scheduleCallables(\n        PushLayerStep.makeList(\n            buildContext,\n            progressDispatcherFactory,\n            results.targetRegistryClient.get(),\n            baseLayers));\n  }\n\n  private void buildAndCacheApplicationLayers(\n      ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.applicationLayers =\n        scheduleCallables(\n            BuildAndCacheApplicationLayerStep.makeList(buildContext, progressDispatcherFactory));\n  }\n\n  private void buildImages(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.baseImagesAndBuiltImages =\n        executorService.submit(\n            () -> {\n              try (ProgressEventDispatcher progressDispatcher =\n                  progressDispatcherFactory.create(\n                      \"scheduling building manifests\", results.baseImagesAndLayers.get().size())) {\n\n                Map<Image, Future<Image>> baseImagesAndBuiltImages = new LinkedHashMap<>();\n                for (Map.Entry<Image, List<Future<PreparedLayer>>> entry :\n                    results.baseImagesAndLayers.get().entrySet()) {\n                  Image baseImage = entry.getKey();\n                  List<Future<PreparedLayer>> baseLayers = entry.getValue();\n\n                  Future<Image> builtImage =\n                      buildImage(baseImage, baseLayers, progressDispatcher.newChildProducer());\n                  baseImagesAndBuiltImages.put(baseImage, builtImage);\n                }\n                return baseImagesAndBuiltImages;\n              }\n            });\n  }\n\n  private Future<Image> buildImage(\n      Image baseImage,\n      List<Future<PreparedLayer>> baseLayers,\n      ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    return executorService.submit(\n        () ->\n            new BuildImageStep(\n                    buildContext,\n                    progressDispatcherFactory,\n                    baseImage,\n                    realizeFutures(baseLayers),\n                    realizeFutures(Verify.verifyNotNull(results.applicationLayers)))\n                .call());\n  }\n\n  private void buildManifestListOrSingleManifest(\n      ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.manifestListOrSingleManifest =\n        executorService.submit(\n            () ->\n                new BuildManifestListOrSingleManifestStep(\n                        buildContext,\n                        progressDispatcherFactory,\n                        realizeFutures(results.baseImagesAndBuiltImages.get().values()))\n                    .call());\n  }\n\n  private void pushContainerConfigurations(\n      ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.baseImagesAndContainerConfigPushResults =\n        executorService.submit(\n            () -> {\n              try (ProgressEventDispatcher progressDispatcher =\n                  progressDispatcherFactory.create(\n                      \"scheduling pushing container configurations\",\n                      results.baseImagesAndBuiltImages.get().size())) {\n\n                Map<Image, Future<BlobDescriptor>> configPushResults = new LinkedHashMap<>();\n                for (Map.Entry<Image, Future<Image>> entry :\n                    results.baseImagesAndBuiltImages.get().entrySet()) {\n                  Image baseImage = entry.getKey();\n                  Future<Image> builtImage = entry.getValue();\n\n                  Future<BlobDescriptor> pushResult =\n                      pushContainerConfiguration(builtImage, progressDispatcher.newChildProducer());\n                  configPushResults.put(baseImage, pushResult);\n                }\n                return configPushResults;\n              }\n            });\n  }\n\n  private Future<BlobDescriptor> pushContainerConfiguration(\n      Future<Image> builtImage, ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    return executorService.submit(\n        () ->\n            new PushContainerConfigurationStep(\n                    buildContext,\n                    progressDispatcherFactory,\n                    results.targetRegistryClient.get(),\n                    builtImage.get())\n                .call());\n  }\n\n  private void pushApplicationLayers(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.applicationLayerPushResults =\n        executorService.submit(\n            () ->\n                scheduleCallables(\n                    PushLayerStep.makeList(\n                        buildContext,\n                        progressDispatcherFactory,\n                        results.targetRegistryClient.get(),\n                        Verify.verifyNotNull(results.applicationLayers))));\n  }\n\n  private void checkManifestInTargetRegistry(\n      ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.manifestCheckResult =\n        executorService.submit(\n            () ->\n                new CheckManifestStep(\n                        buildContext,\n                        progressDispatcherFactory,\n                        results.targetRegistryClient.get(),\n                        results.manifestListOrSingleManifest.get())\n                    .call());\n  }\n\n  private void pushImages(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.imagePushResults =\n        executorService.submit(\n            () -> {\n              try (ProgressEventDispatcher progressDispatcher =\n                  progressDispatcherFactory.create(\n                      \"scheduling pushing manifests\",\n                      results.baseImagesAndBuiltImages.get().size())) {\n\n                realizeFutures(results.applicationLayerPushResults.get());\n\n                List<Future<BuildResult>> buildResults = new ArrayList<>();\n                for (Map.Entry<Image, Future<Image>> entry :\n                    results.baseImagesAndBuiltImages.get().entrySet()) {\n                  Image baseImage = entry.getKey();\n                  Future<Image> builtImage = entry.getValue();\n\n                  buildResults.add(\n                      pushImage(baseImage, builtImage, progressDispatcher.newChildProducer()));\n                }\n                return buildResults;\n              }\n            });\n  }\n\n  private Future<BuildResult> pushImage(\n      Image baseImage,\n      Future<Image> builtImage,\n      ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    return executorService.submit(\n        () -> {\n          realizeFutures(\n              Verify.verifyNotNull(results.baseImagesAndLayerPushResults.get().get(baseImage)));\n\n          Future<BlobDescriptor> containerConfigPushResult =\n              results.baseImagesAndContainerConfigPushResults.get().get(baseImage);\n\n          List<Future<BuildResult>> manifestPushResults =\n              scheduleCallables(\n                  PushImageStep.makeList(\n                      buildContext,\n                      progressDispatcherFactory,\n                      results.targetRegistryClient.get(),\n                      Verify.verifyNotNull(containerConfigPushResult).get(),\n                      builtImage.get(),\n                      results.manifestCheckResult.get().isPresent()));\n\n          realizeFutures(manifestPushResults);\n\n          return manifestPushResults.isEmpty()\n              ? new BuildResult(\n                  results.manifestCheckResult.get().get().getDigest(),\n                  Verify.verifyNotNull(containerConfigPushResult).get().getDigest(),\n                  isImagePushed(results.manifestCheckResult.get()))\n              // Manifest pushers return the same BuildResult.\n              : manifestPushResults.get(0).get();\n        });\n  }\n\n  @VisibleForTesting\n  boolean isImagePushed(Optional<ManifestAndDigest<ManifestTemplate>> manifestResult) {\n\n    return !(JibSystemProperties.skipExistingImages() && manifestResult.isPresent());\n  }\n\n  private void pushManifestList(ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.buildResult =\n        executorService.submit(\n            () -> {\n              realizeFutures(results.imagePushResults.get());\n              List<Future<BuildResult>> manifestListPushResults =\n                  scheduleCallables(\n                      PushImageStep.makeListForManifestList(\n                          buildContext,\n                          progressDispatcherFactory,\n                          results.targetRegistryClient.get(),\n                          results.manifestListOrSingleManifest.get(),\n                          results.manifestCheckResult.get().isPresent()));\n\n              realizeFutures(manifestListPushResults);\n              return manifestListPushResults.isEmpty()\n                  ? results.imagePushResults.get().get(0).get()\n                  : manifestListPushResults.get(0).get();\n            });\n  }\n\n  private void loadDocker(\n      DockerClient dockerClient, ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.buildResult =\n        executorService.submit(\n            () -> {\n              Verify.verify(\n                  results.baseImagesAndBuiltImages.get().size() == 1,\n                  \"multi-platform image building not supported when pushing to Docker engine\");\n              Image builtImage =\n                  results.baseImagesAndBuiltImages.get().values().iterator().next().get();\n              return new LoadDockerStep(\n                      buildContext, progressDispatcherFactory, dockerClient, builtImage)\n                  .call();\n            });\n  }\n\n  private void writeTarFile(\n      Path outputPath, ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    results.buildResult =\n        executorService.submit(\n            () -> {\n              Verify.verify(\n                  results.baseImagesAndBuiltImages.get().size() == 1,\n                  \"multi-platform image building not supported when building a local tar image\");\n              Image builtImage =\n                  results.baseImagesAndBuiltImages.get().values().iterator().next().get();\n\n              return new WriteTarFileStep(\n                      buildContext, progressDispatcherFactory, outputPath, builtImage)\n                  .call();\n            });\n  }\n\n  private <E> List<Future<E>> scheduleCallables(ImmutableList<? extends Callable<E>> callables) {\n    return callables.stream().map(executorService::submit).collect(Collectors.toList());\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport java.io.IOException;\nimport java.util.concurrent.Callable;\n\n/** Pushes a BLOB to the target registry. */\nclass PushBlobStep implements Callable<BlobDescriptor> {\n\n  private static final String DESCRIPTION = \"Pushing BLOB \";\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n\n  private final RegistryClient registryClient;\n  private final BlobDescriptor blobDescriptor;\n  private final Blob blob;\n  private final boolean forcePush;\n\n  PushBlobStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n      RegistryClient registryClient,\n      BlobDescriptor blobDescriptor,\n      Blob blob,\n      boolean forcePush) {\n    this.buildContext = buildContext;\n    this.progressEventDispatcherFactory = progressEventDispatcherFactory;\n    this.registryClient = registryClient;\n    this.blobDescriptor = blobDescriptor;\n    this.blob = blob;\n    this.forcePush = forcePush;\n  }\n\n  @Override\n  public BlobDescriptor call() throws IOException, RegistryException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    DescriptorDigest blobDigest = blobDescriptor.getDigest();\n    try (ProgressEventDispatcher progressEventDispatcher =\n            progressEventDispatcherFactory.create(\n                \"pushing blob \" + blobDigest, blobDescriptor.getSize());\n        TimerEventDispatcher ignored =\n            new TimerEventDispatcher(eventHandlers, DESCRIPTION + blobDescriptor);\n        ThrottledAccumulatingConsumer throttledProgressReporter =\n            new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress)) {\n\n      // check if the BLOB is available\n      if (!forcePush && registryClient.checkBlob(blobDigest).isPresent()) {\n        eventHandlers.dispatch(\n            LogEvent.info(\n                \"Skipping push; BLOB already exists on target registry : \" + blobDescriptor));\n        return blobDescriptor;\n      }\n\n      // If base and target images are in the same registry, then use mount/from to try mounting the\n      // BLOB from the base image repository to the target image repository and possibly avoid\n      // having to push the BLOB. See\n      // https://docs.docker.com/registry/spec/api/#cross-repository-blob-mount for details.\n      String baseRegistry = buildContext.getBaseImageConfiguration().getImageRegistry();\n      String baseRepository = buildContext.getBaseImageConfiguration().getImageRepository();\n      String targetRegistry = buildContext.getTargetImageConfiguration().getImageRegistry();\n      String sourceRepository = targetRegistry.equals(baseRegistry) ? baseRepository : null;\n      registryClient.pushBlob(blobDigest, blob, sourceRepository, throttledProgressReporter);\n      return blobDescriptor;\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.blob.Blobs;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.hash.Digests;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;\nimport com.google.cloud.tools.jib.json.JsonTemplate;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport java.io.IOException;\nimport java.util.concurrent.Callable;\n\n/** Pushes the container configuration. */\nclass PushContainerConfigurationStep implements Callable<BlobDescriptor> {\n\n  private static final String DESCRIPTION = \"Pushing container configuration\";\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;\n\n  private final RegistryClient registryClient;\n  private final Image builtImage;\n\n  PushContainerConfigurationStep(\n      BuildContext buildContext,\n      ProgressEventDispatcher.Factory progressEventDispatcherFactory,\n...\n", "needle_token_start": 12016, "needle_token_end": 12118, "code_context_ntokens": 16089, "output": ["```java\nprivate List<Future<BlobDescriptor>> pushBaseImageLayers(\n      List<Future<PreparedLayer>> baseLayers,\n      ProgressEventDispatcher.Factory progressDispatcherFactory)\n      throws InterruptedException, ExecutionException {\n    return scheduleCallables(\n        PushLayerStep.makeList(\n            buildContext,\n            progressDispatcherFactory,\n            results.targetRegistryClient.get(),\n            baseLayers));\n  }\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "getPath", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/api/TarImage.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the location of a tarball archive file used for container image storage.\n2. **Input**: None.\n3. **Output**: Returns the file system path to the tarball archive.\n4. **Procedure**: Simply accesses and returns the path stored in a private variable within the class instance.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.builder.steps.BuildResult;\nimport com.google.cloud.tools.jib.builder.steps.StepsRunner;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ImageConfiguration;\nimport com.google.cloud.tools.jib.docker.CliDockerClient;\nimport com.google.cloud.tools.jib.docker.DockerClientResolver;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.filesystem.XdgDirectories;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ArrayListMultimap;\nimport com.google.common.collect.ImmutableListMultimap;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.ListMultimap;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.function.Consumer;\nimport java.util.function.Function;\nimport javax.annotation.Nullable;\n\n/** Configures how to containerize. */\npublic class Containerizer {\n\n  /**\n   * The default directory for caching the base image layers, in {@code [user cache\n   * home]/google-cloud-tools-java/jib}.\n   */\n  public static final Path DEFAULT_BASE_CACHE_DIRECTORY = XdgDirectories.getCacheHome();\n\n  public static final String DEFAULT_APPLICATION_CACHE_DIRECTORY_NAME =\n      \"jib-core-application-layers-cache\";\n\n  private static final String DEFAULT_TOOL_NAME = \"jib-core\";\n  private static final String DEFAULT_TOOL_VERSION =\n      Containerizer.class.getPackage().getImplementationVersion();\n\n  private static final String DESCRIPTION_FOR_DOCKER_REGISTRY = \"Building and pushing image\";\n  private static final String DESCRIPTION_FOR_DOCKER_DAEMON = \"Building image to Docker daemon\";\n  private static final String DESCRIPTION_FOR_TARBALL = \"Building image tarball\";\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a container registry.\n   *\n   * @param registryImage the {@link RegistryImage} that defines target container registry and\n   *     credentials\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(RegistryImage registryImage) {\n    ImageConfiguration imageConfiguration =\n        ImageConfiguration.builder(registryImage.getImageReference())\n            .setCredentialRetrievers(registryImage.getCredentialRetrievers())\n            .build();\n\n    Function<BuildContext, StepsRunner> stepsRunnerFactory =\n        buildContext -> StepsRunner.begin(buildContext).registryPushSteps();\n\n    return new Containerizer(\n        DESCRIPTION_FOR_DOCKER_REGISTRY, imageConfiguration, stepsRunnerFactory, true);\n  }\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a Docker daemon.\n   *\n   * @param dockerDaemonImage the {@link DockerDaemonImage} that defines target Docker daemon\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(DockerDaemonImage dockerDaemonImage) {\n    DockerClient dockerClient =\n        DockerClientResolver.resolve(dockerDaemonImage.getDockerEnvironment())\n            .orElse(\n                new CliDockerClient(\n                    dockerDaemonImage.getDockerExecutable(),\n                    dockerDaemonImage.getDockerEnvironment()));\n\n    return to(dockerClient, dockerDaemonImage);\n  }\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a tarball archive.\n   *\n   * @param tarImage the {@link TarImage} that defines target output file\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(TarImage tarImage) {\n    Optional<ImageReference> imageReference = tarImage.getImageReference();\n    if (!imageReference.isPresent()) {\n      throw new IllegalArgumentException(\n          \"Image name must be set when building a TarImage; use TarImage#named(...) to set the name\"\n              + \" of the target image\");\n    }\n\n    ImageConfiguration imageConfiguration =\n        ImageConfiguration.builder(imageReference.get()).build();\n\n    Function<BuildContext, StepsRunner> stepsRunnerFactory =\n        buildContext -> StepsRunner.begin(buildContext).tarBuildSteps(tarImage.getPath());\n\n    return new Containerizer(\n        DESCRIPTION_FOR_TARBALL, imageConfiguration, stepsRunnerFactory, false);\n  }\n\n  /**\n   * Gets a new {@link Containerizer} that containerizes to a Docker daemon.\n   *\n   * @param dockerClient the {@link DockerClient} to connect\n   * @param dockerDaemonImage the {@link DockerDaemonImage} that defines target Docker daemon\n   * @return a new {@link Containerizer}\n   */\n  public static Containerizer to(DockerClient dockerClient, DockerDaemonImage dockerDaemonImage) {\n    ImageConfiguration imageConfiguration =\n        ImageConfiguration.builder(dockerDaemonImage.getImageReference()).build();\n\n    Function<BuildContext, StepsRunner> stepsRunnerFactory =\n        buildContext -> StepsRunner.begin(buildContext).dockerLoadSteps(dockerClient);\n\n    return new Containerizer(\n        DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, stepsRunnerFactory, false);\n  }\n\n  private final String description;\n  private final ImageConfiguration imageConfiguration;\n  private final Function<BuildContext, StepsRunner> stepsRunnerFactory;\n  private final boolean mustBeOnline;\n  private final Set<String> additionalTags = new HashSet<>();\n  private final EventHandlers.Builder eventHandlersBuilder = EventHandlers.builder();\n\n  @Nullable private ExecutorService executorService;\n  private Path baseImageLayersCacheDirectory = DEFAULT_BASE_CACHE_DIRECTORY;\n  @Nullable private Path applicationLayersCacheDirectory;\n  private boolean allowInsecureRegistries = false;\n  private boolean offline = false;\n  private String toolName = DEFAULT_TOOL_NAME;\n  @Nullable private String toolVersion = DEFAULT_TOOL_VERSION;\n  private boolean alwaysCacheBaseImage = false;\n  private ListMultimap<String, String> registryMirrors = ArrayListMultimap.create();\n\n  /** Instantiate with {@link #to}. */\n  private Containerizer(\n      String description,\n      ImageConfiguration imageConfiguration,\n      Function<BuildContext, StepsRunner> stepsRunnerFactory,\n      boolean mustBeOnline) {\n    this.description = description;\n    this.imageConfiguration = imageConfiguration;\n    this.stepsRunnerFactory = stepsRunnerFactory;\n    this.mustBeOnline = mustBeOnline;\n  }\n\n  /**\n   * Adds an additional tag to tag the target image with. For example, the following would\n   * containerize to both {@code gcr.io/my-project/my-image:tag} and {@code\n   * gcr.io/my-project/my-image:tag2}:\n   *\n   * <pre>{@code\n   * Containerizer.to(RegistryImage.named(\"gcr.io/my-project/my-image:tag\")).withAdditionalTag(\"tag2\");\n   * }</pre>\n   *\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/InsecureRegistryException.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport java.net.URL;\n\n/**\n * Throw when attempting to access an insecure registry when only secure connections are allowed.\n */\npublic class InsecureRegistryException extends RegistryException {\n\n  /**\n   * Creates a new exception with a human readable message.\n   *\n   * @param insecureUrl the insecure url that is attempted to be accessed\n   * @param cause the underlying cause that triggered this exception\n   */\n  public InsecureRegistryException(URL insecureUrl, Throwable cause) {\n    super(\n        \"Failed to verify the server at \"\n            + insecureUrl\n            + \" because only secure connections are allowed.\",\n        cause);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/MainClassFinder.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.common.base.Preconditions;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.function.Consumer;\nimport javax.annotation.Nullable;\nimport org.objectweb.asm.ClassReader;\nimport org.objectweb.asm.ClassVisitor;\nimport org.objectweb.asm.MethodVisitor;\nimport org.objectweb.asm.Opcodes;\n\n/**\n * Finds main classes in a list of class files. Main classes are classes that define the {@code\n * public static void main(String[] args)} method.\n */\npublic class MainClassFinder {\n\n  /** The result of a call to {@link #find}. */\n  public static class Result {\n\n    /** The type of result. */\n    public enum Type {\n\n      // Found a single main class.\n      MAIN_CLASS_FOUND,\n\n      // Did not find any main class.\n      MAIN_CLASS_NOT_FOUND,\n\n      // Found multiple main classes.\n      MULTIPLE_MAIN_CLASSES\n    }\n\n    private static Result success(String foundMainClass) {\n      return new Result(Type.MAIN_CLASS_FOUND, Collections.singletonList(foundMainClass));\n    }\n\n    private static Result mainClassNotFound() {\n      return new Result(Type.MAIN_CLASS_NOT_FOUND, Collections.emptyList());\n    }\n\n    private static Result multipleMainClasses(List<String> foundMainClasses) {\n      return new Result(Type.MULTIPLE_MAIN_CLASSES, foundMainClasses);\n    }\n\n    private final Type type;\n    private final List<String> foundMainClasses;\n\n    private Result(Type type, List<String> foundMainClasses) {\n      this.foundMainClasses = foundMainClasses;\n      this.type = type;\n    }\n\n    /**\n     * Gets the found main class. Only call if {@link #getType} is {@link Type#MAIN_CLASS_FOUND}.\n     *\n     * @return the found main class\n     */\n    public String getFoundMainClass() {\n      Preconditions.checkState(Type.MAIN_CLASS_FOUND == type);\n      Preconditions.checkState(foundMainClasses.size() == 1);\n      return foundMainClasses.get(0);\n    }\n\n    /**\n     * Gets the type of the result.\n     *\n     * @return the type of the result\n     */\n    public Type getType() {\n      return type;\n    }\n\n    /**\n     * Gets the found main classes.\n     *\n     * @return the found main classes\n     */\n    public List<String> getFoundMainClasses() {\n      return foundMainClasses;\n    }\n  }\n\n  /** {@link ClassVisitor} that keeps track of whether or not it has visited a main class. */\n  private static class MainClassVisitor extends ClassVisitor {\n\n    /** The return/argument types for main. */\n    private static final String MAIN_DESCRIPTOR =\n        org.objectweb.asm.Type.getMethodDescriptor(\n            org.objectweb.asm.Type.VOID_TYPE, org.objectweb.asm.Type.getType(String[].class));\n\n    /** Accessors that main may or may not have. */\n    private static final int OPTIONAL_ACCESS =\n        Opcodes.ACC_FINAL | Opcodes.ACC_DEPRECATED | Opcodes.ACC_VARARGS | Opcodes.ACC_SYNTHETIC;\n\n    private boolean visitedMainClass;\n\n    private MainClassVisitor() {\n      super(Opcodes.ASM9);\n    }\n\n    @Override\n    @Nullable\n    public MethodVisitor visitMethod(\n        int access, String name, String descriptor, String signature, String[] exceptions) {\n      if ((access & ~OPTIONAL_ACCESS) == (Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC)\n          && name.equals(\"main\")\n          && descriptor.equals(MAIN_DESCRIPTOR)) {\n        visitedMainClass = true;\n      }\n      return null;\n    }\n  }\n\n  /**\n   * Tries to find classes with {@code psvm} (see class javadoc) in {@code files}.\n   *\n   * @param files the files to search\n   * @param logger a {@link Consumer} used to handle log events\n   * @return the {@link Result} of the main class finding attempt\n   */\n  public static Result find(List<Path> files, Consumer<LogEvent> logger) {\n    List<String> mainClasses = new ArrayList<>();\n    for (Path file : files) {\n      // Makes sure classFile is valid.\n      if (!Files.exists(file)) {\n        logger.accept(LogEvent.debug(\"MainClassFinder: \" + file + \" does not exist; ignoring\"));\n        continue;\n      }\n      if (!Files.isRegularFile(file)) {\n        logger.accept(\n            LogEvent.debug(\"MainClassFinder: \" + file + \" is not a regular file; skipping\"));\n        continue;\n      }\n      if (!file.toString().endsWith(\".class\")) {\n        logger.accept(\n            LogEvent.debug(\"MainClassFinder: \" + file + \" is not a class file; skipping\"));\n        continue;\n      }\n\n      MainClassVisitor mainClassVisitor = new MainClassVisitor();\n      try (InputStream classFileInputStream = Files.newInputStream(file)) {\n        ClassReader reader = new ClassReader(classFileInputStream);\n        reader.accept(mainClassVisitor, 0);\n        if (mainClassVisitor.visitedMainClass) {\n          mainClasses.add(reader.getClassName().replace('/', '.'));\n        }\n\n      } catch (IllegalArgumentException ex) {\n        throw new UnsupportedOperationException(\n            \"Check the full stace trace, and if the root cause is from ASM ClassReader about \"\n                + \"unsupported class file version, see \"\n                + \"https://github.com/GoogleContainerTools/jib/blob/master/docs/faq.md\"\n                + \"#i-am-seeing-unsupported-class-file-major-version-when-building\",\n            ex);\n\n      } catch (ArrayIndexOutOfBoundsException ignored) {\n        // Not a valid class file (thrown by ClassReader if it reads an invalid format)\n        logger.accept(LogEvent.warn(\"Invalid class file found: \" + file));\n\n      } catch (IOException ignored) {\n        // Could not read class file.\n        logger.accept(LogEvent.warn(\"Could not read file: \" + file));\n      }\n    }\n\n    if (mainClasses.size() == 1) {\n      // Valid class found.\n      return Result.success(mainClasses.get(0));\n    }\n    if (mainClasses.isEmpty()) {\n      // No main class found anywhere.\n      return Result.mainClassNotFound();\n    }\n    // More than one main class found.\n    return Result.multipleMainClasses(mainClasses);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/InvalidImageReferenceException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\n/** Thrown when attempting to parse an invalid image reference. */\npublic class InvalidImageReferenceException extends Exception {\n\n  private final String reference;\n\n  public InvalidImageReferenceException(String reference) {\n    super(\"Invalid image reference: \" + reference);\n    this.reference = reference;\n  }\n\n  public String getInvalidReference() {\n    return reference;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath;\nimport com.google.cloud.tools.jib.api.buildplan.ContainerBuildPlan;\nimport com.google.cloud.tools.jib.api.buildplan.FileEntriesLayer;\nimport com.google.cloud.tools.jib.api.buildplan.FileEntry;\nimport com.google.cloud.tools.jib.api.buildplan.ImageFormat;\nimport com.google.cloud.tools.jib.api.buildplan.LayerObject;\nimport com.google.cloud.tools.jib.api.buildplan.Platform;\nimport com.google.cloud.tools.jib.api.buildplan.Port;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.builder.steps.BuildResult;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ContainerConfiguration;\nimport com.google.cloud.tools.jib.configuration.ImageConfiguration;\nimport com.google.cloud.tools.jib.docker.CliDockerClient;\nimport com.google.cloud.tools.jib.docker.DockerClientResolver;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Verify;\nimport java.io.IOException;\nimport java.net.UnknownHostException;\nimport java.nio.file.Path;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.concurrent.ExecutionException;\nimport java.util.function.Function;\nimport java.util.stream.Collectors;\nimport javax.annotation.Nullable;\nimport org.apache.http.conn.HttpHostConnectException;\n\n/**\n * Builds a container with Jib.\n *\n * <p>Example usage:\n *\n * <pre>{@code\n * Jib.from(baseImage)\n *    .addLayer(sourceFiles, extractionPath)\n *    .setEntrypoint(\"myprogram\", \"--flag\", \"subcommand\")\n *    .setProgramArguments(\"hello\", \"world\")\n *    .addEnvironmentVariable(\"HOME\", \"/app\")\n *    .addExposedPort(Port.tcp(8080))\n *    .addLabel(\"containerizer\", \"jib\")\n *    .containerize(...);\n * }</pre>\n */\npublic class JibContainerBuilder {\n\n  private static String capitalizeFirstLetter(String string) {\n    if (string.isEmpty()) {\n      return string;\n    }\n    return Character.toUpperCase(string.charAt(0)) + string.substring(1);\n  }\n\n  private final ContainerBuildPlan.Builder containerBuildPlanBuilder = ContainerBuildPlan.builder();\n  // TODO(chanseok): remove and use containerBuildPlanBuilder instead. Note that\n  // ContainerConfiguation implements equals() and hashCode(), so need to verify\n  // if they are required.\n  private final ContainerConfiguration.Builder containerConfigurationBuilder =\n      ContainerConfiguration.builder();\n  private final BuildContext.Builder buildContextBuilder;\n\n  private ImageConfiguration baseImageConfiguration;\n  // TODO(chanseok): remove and use containerBuildPlanBuilder instead.\n  private List<FileEntriesLayer> layerConfigurations = new ArrayList<>();\n\n  /** Instantiate with {@link Jib#from}. */\n  JibContainerBuilder(RegistryImage baseImage) {\n    this(\n        ImageConfiguration.builder(baseImage.getImageReference())\n            .setCredentialRetrievers(baseImage.getCredentialRetrievers())\n            .build(),\n        BuildContext.builder());\n  }\n\n  /** Instantiate with {@link Jib#from}. */\n  JibContainerBuilder(DockerDaemonImage baseImage) {\n    this(\n        ImageConfiguration.builder(baseImage.getImageReference())\n            .setDockerClient(\n                DockerClientResolver.resolve(baseImage.getDockerEnvironment())\n                    .orElse(\n                        new CliDockerClient(\n                            baseImage.getDockerExecutable(), baseImage.getDockerEnvironment())))\n            .build(),\n        BuildContext.builder());\n  }\n\n  /** Instantiate with {@link Jib#from}. */\n  JibContainerBuilder(TarImage baseImage) {\n    // TODO: Cleanup using scratch as placeholder\n    this(\n        ImageConfiguration.builder(baseImage.getImageReference().orElse(ImageReference.scratch()))\n            .setTarPath(baseImage.getPath())\n            .build(),\n        BuildContext.builder());\n  }\n\n  /** Instantiate with {@link Jib#from}. */\n  JibContainerBuilder(DockerClient dockerClient, DockerDaemonImage baseImage) {\n    this(\n        ImageConfiguration.builder(baseImage.getImageReference())\n            .setDockerClient(dockerClient)\n            .build(),\n        BuildContext.builder());\n  }\n\n  @VisibleForTesting\n  JibContainerBuilder(\n      ImageConfiguration imageConfiguration, BuildContext.Builder buildContextBuilder) {\n    this.buildContextBuilder = buildContextBuilder.setBaseImageConfiguration(imageConfiguration);\n    baseImageConfiguration = imageConfiguration;\n    containerBuildPlanBuilder.setBaseImage(imageConfiguration.getImage().toString());\n  }\n\n  /**\n   * Adds a new layer to the container with {@code files} as the source files and {@code\n   * pathInContainer} as the path to copy the source files to in the container file system.\n   *\n   * <p>Source files that are directories will be recursively copied. For example, if the source\n   * files are:\n   *\n   * <ul>\n   *   <li>{@code fileFoo}\n   *   <li>{@code fileBar}\n   *   <li>{@code directory/}\n   * </ul>\n   *\n   * <p>and the destination to copy to is {@code /path/in/container}, then the new layer will have\n   * the following entries for the container file system:\n   *\n   * <ul>\n   *   <li>{@code /path/in/container/fileFoo}\n   *   <li>{@code /path/in/container/fileBar}\n   *   <li>{@code /path/in/container/directory/}\n   *   <li>{@code /path/in/container/directory/...} (all contents of {@code directory/})\n   * </ul>\n   *\n   * @param files the source files to copy to a new layer in the container\n   * @param pathInContainer the path in the container file system corresponding to the {@code\n   *     sourceFile}\n   * @return this\n   * @throws IOException if an exception occurred when recursively listing any directories\n   */\n  public JibContainerBuilder addLayer(List<Path> files, AbsoluteUnixPath pathInContainer)\n      throws IOException {\n    FileEntriesLayer.Builder layerConfigurationBuilder = FileEntriesLayer.builder();\n\n    for (Path file : files) {\n      layerConfigurationBuilder.addEntryRecursive(\n          file, pathInContainer.resolve(file.getFileName()));\n    }\n\n    return addFileEntriesLayer(layerConfigurationBuilder.build());\n  }\n\n  /**\n   * Adds a new layer to the container with {@code files} as the source files and {@code\n   * pathInContainer} as the path to copy the source files to in the container file system.\n   *\n   * @param files the source files to copy to a new layer in the container\n   * @param pathInContainer the path in the container file system corresponding to the {@code\n   *     sourceFile}\n   * @return this\n   * @throws IOException if an exception occurred when recursively listing any directories\n   * @throws IllegalArgumentException if {@code pathInContainer} is not an absolute Unix-style path\n   * @see #addLayer(List, AbsoluteUnixPath)\n   */\n  public JibContainerBuilder addLayer(List<Path> files, String pathInContainer) throws IOException {\n    return addLayer(files, AbsoluteUnixPath.get(pathInContainer));\n  }\n\n  /**\n   * Adds a layer (defined by a {@link LayerConfiguration}).\n   *\n   * @deprecated use {@link #addFileEntriesLayer(FileEntriesLayer)}.\n   * @param layerConfiguration the {@link LayerConfiguration}\n   * @return this\n   */\n  @Deprecated\n  public JibContainerBuilder addLayer(LayerConfiguration layerConfiguration) {\n    return addFileEntriesLayer(layerConfiguration.toFileEntriesLayer());\n  }\n\n  /**\n   * Adds a layer (defined by a {@link FileEntriesLayer}).\n   *\n   * @param layer the {@link FileEntriesLayer}\n   * @return this\n   */\n  public JibContainerBuilder addFileEntriesLayer(FileEntriesLayer layer) {\n    containerBuildPlanBuilder.addLayer(layer);\n    layerConfigurations.add(layer);\n    return this;\n  }\n\n  /**\n   * Sets the layers (defined by a list of {@link LayerConfiguration}s). This replaces any\n   * previously-added layers.\n   *\n   * @deprecated use {@link #setFileEntriesLayers(List)}.\n   * @param layerConfigurations the list of {@link LayerConfiguration}s\n   * @return this\n   */\n  @Deprecated\n  public JibContainerBuilder setLayers(List<LayerConfiguration> layerConfigurations) {\n    return setFileEntriesLayers(\n        layerConfigurations.stream()\n            .map(LayerConfiguration::toFileEntriesLayer)\n            .collect(Collectors.toList()));\n  }\n\n  /**\n   * Sets the layers (defined by a list of {@link FileEntriesLayer}s). This replaces any\n   * previously-added layers.\n   *\n   * @param layers the list of {@link FileEntriesLayer}s\n   * @return this\n   */\n  public JibContainerBuilder setFileEntriesLayers(List<FileEntriesLayer> layers) {\n    containerBuildPlanBuilder.setLayers(layers);\n    layerConfigurations = new ArrayList<>(layers);\n    return this;\n  }\n\n  /**\n   * Sets the layers. This replaces any previously-added layers.\n   *\n   * @deprecated use {@link #setFileEntriesLayers(FileEntriesLayer...)}.\n   * @param layerConfigurations the {@link LayerConfiguration}s\n   * @return this\n   */\n  @Deprecated\n  public JibContainerBuilder setLayers(LayerConfiguration... layerConfigurations) {\n    return setLayers(Arrays.asList(layerConfigurations));\n  }\n\n  /**\n   * Sets the layers. This replaces any previously-added layers.\n   *\n   * @param layers the {@link FileEntriesLayer}s\n   * @return this\n   */\n  public JibContainerBuilder setFileEntriesLayers(FileEntriesLayer... layers) {\n    return setFileEntriesLayers(Arrays.asList(layers));\n  }\n\n  /**\n   * Sets the container entrypoint. This is the beginning of the command that is run when the\n   * container starts. {@link #setProgramArguments} sets additional tokens.\n   *\n   * <p>This is similar to <a\n   * href=\"https://docs.docker.com/engine/reference/builder/#exec-form-entrypoint-example\">{@code\n   * ENTRYPOINT} in Dockerfiles</a> or {@code command} in the <a\n   * href=\"https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#container-v1-core\">Kubernetes\n   * Container spec</a>.\n   *\n   * @param entrypoint a list of the entrypoint command\n   * @return this\n   */\n  public JibContainerBuilder setEntrypoint(@Nullable List<String> entrypoint) {\n    containerBuildPlanBuilder.setEntrypoint(entrypoint);\n    containerConfigurationBuilder.setEntrypoint(entrypoint);\n    return this;\n  }\n\n  /**\n   * Sets the container entrypoint.\n   *\n   * @param entrypoint the entrypoint command\n   * @return this\n   * @see #setEntrypoint(List)\n   */\n  public JibContainerBuilder setEntrypoint(String... entrypoint) {\n    return setEntrypoint(Arrays.asList(entrypoint));\n  }\n\n  /**\n   * Sets the container entrypoint program arguments. These are additional tokens added to the end\n   * of the entrypoint command.\n   *\n   * <p>This is similar to <a href=\"https://docs.docker.com/engine/reference/builder/#cmd\">{@code\n   * CMD} in Dockerfiles</a> or {@code args} in the <a\n   * href=\"https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#container-v1-core\">Kubernetes\n   * Container spec</a>.\n   *\n   * <p>For example, if the entrypoint was {@code myprogram --flag subcommand} and program arguments\n   * were {@code hello world}, then the command that run when the container starts is {@code\n   * myprogram --flag subcommand hello world}.\n   *\n   * @param programArguments a list of program argument tokens\n   * @return this\n   */\n  public JibContainerBuilder setProgramArguments(@Nullable List<String> programArguments) {\n    containerBuildPlanBuilder.setCmd(programArguments);\n    containerConfigurationBuilder.setProgramArguments(programArguments);\n    return this;\n  }\n\n  /**\n   * Sets the container entrypoint program arguments.\n   *\n   * @param programArguments program arguments tokens\n   * @return this\n   * @see #setProgramArguments(List)\n   */\n  public JibContainerBuilder setProgramArguments(String... programArguments) {\n    return setProgramArguments(Arrays.asList(programArguments));\n  }\n\n  /**\n   * Sets the container environment. These environment variables are available to the program\n   * launched by the container entrypoint command. This replaces any previously-set environment\n   * variables.\n   *\n   * <p>This is similar to <a href=\"https://docs.docker.com/engine/reference/builder/#env\">{@code\n   * ENV} in Dockerfiles</a> or {@code env} in the <a\n   * href=\"https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#container-v1-core\">Kubernetes\n   * Container spec</a>.\n   *\n   * @param environmentMap a map of environment variable names to values\n   * @return this\n   */\n  public JibContainerBuilder setEnvironment(Map<String, String> environmentMap) {\n    containerBuildPlanBuilder.setEnvironment(environmentMap);\n    containerConfigurationBuilder.setEnvironment(environmentMap);\n    return this;\n  }\n\n  /**\n   * Adds a variable in the container environment.\n   *\n   * @param name the environment variable name\n   * @param value the environment variable value\n   * @return this\n   * @see #setEnvironment\n   */\n  public JibContainerBuilder addEnvironmentVariable(String name, String value) {\n    containerBuildPlanBuilder.addEnvironmentVariable(name, value);\n    containerConfigurationBuilder.addEnvironment(name, value);\n    return this;\n  }\n\n  /**\n   * Sets the directories that may hold externally mounted volumes.\n   *\n   * <p>This is similar to <a href=\"https://docs.docker.com/engine/reference/builder/#volume\">{@code\n   * VOLUME} in Dockerfiles</a>.\n   *\n   * @param volumes the directory paths on the container filesystem to set as volumes\n   * @return this\n   */\n  public JibContainerBuilder setVolumes(Set<AbsoluteUnixPath> volumes) {\n    containerBuildPlanBuilder.setVolumes(volumes);\n    containerConfigurationBuilder.setVolumes(volumes);\n    return this;\n  }\n\n  /**\n   * Sets the directories that may hold externally mounted volumes.\n   *\n   * @param volumes the directory paths on the container filesystem to set as volumes\n   * @return this\n   * @see #setVolumes(Set)\n   */\n  public JibContainerBuilder setVolumes(AbsoluteUnixPath... volumes) {\n    return setVolumes(new HashSet<>(Arrays.asList(volumes)));\n  }\n\n  /**\n   * Adds a directory that may hold an externally mounted volume.\n   *\n   * @param volume a directory path on the container filesystem to represent a volume\n   * @return this\n   * @see #setVolumes(Set)\n   */\n  public JibContainerBuilder addVolume(AbsoluteUnixPath volume) {\n    containerBuildPlanBuilder.addVolume(volume);\n    containerConfigurationBuilder.addVolume(volume);\n    return this;\n  }\n\n  /**\n   * Sets the ports to expose from the container. Ports exposed will allow ingress traffic. This\n   * replaces any previously-set exposed ports.\n   *\n   * <p>Use {@link Port#tcp} to expose a port for TCP traffic and {@link Port#udp} to expose a port\n   * for UDP traffic.\n   *\n   * <p>This is similar to <a href=\"https://docs.docker.com/engine/reference/builder/#expose\">{@code\n   * EXPOSE} in Dockerfiles</a> or {@code ports} in the <a\n   * href=\"https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#container-v1-core\">Kubernetes\n   * Container spec</a>.\n   *\n   * @param ports the ports to expose\n   * @return this\n   */\n  public JibContainerBuilder setExposedPorts(Set<Port> ports) {\n    containerBuildPlanBuilder.setExposedPorts(ports);\n    containerConfigurationBuilder.setExposedPorts(ports);\n    return this;\n  }\n\n  /**\n   * Sets the ports to expose from the container. This replaces any previously-set exposed ports.\n   *\n   * @param ports the ports to expose\n   * @return this\n   * @see #setExposedPorts(Set)\n   */\n  public JibContainerBuilder setExposedPorts(Port... ports) {\n    return setExposedPorts(new HashSet<>(Arrays.asList(ports)));\n  }\n\n  /**\n   * Adds a port to expose from the container.\n   *\n   * @param port the port to expose\n   * @return this\n   * @see #setExposedPorts(Set)\n   */\n  public JibContainerBuilder addExposedPort(Port port) {\n    containerBuildPlanBuilder.addExposedPort(port);\n    containerConfigurationBuilder.addExposedPort(port);\n    return this;\n  }\n\n  /**\n   * Sets the labels for the container. This replaces any previously-set labels.\n   *\n   * <p>This is similar to <a href=\"https://docs.docker.com/engine/reference/builder/#label\">{@code\n   * LABEL} in Dockerfiles</a>.\n   *\n   * @param labelMap a map of label keys to values\n   * @return this\n   */\n  public JibContainerBuilder setLabels(Map<String, String> labelMap) {\n    containerBuildPlanBuilder.setLabels(labelMap);\n    containerConfigurationBuilder.setLabels(labelMap);\n    return this;\n  }\n\n  /**\n   * Sets a label for the container.\n   *\n   * @param key the label key\n   * @param value the label value\n   * @return this\n   */\n  public JibContainerBuilder addLabel(String key, String value) {\n    containerBuildPlanBuilder.addLabel(key, value);\n    containerConfigurationBuilder.addLabel(key, value);\n    return this;\n  }\n\n  /**\n   * Sets the format to build the container image as. Use {@link ImageFormat#Docker} for Docker V2.2\n   * or {@link ImageFormat#OCI} for OCI.\n   *\n   * @param imageFormat the {@link ImageFormat}\n   * @return this\n   */\n  public JibContainerBuilder setFormat(ImageFormat imageFormat) {\n    containerBuildPlanBuilder.setFormat(imageFormat);\n    buildContextBuilder.setTargetFormat(imageFormat);\n    return this;\n  }\n\n  /**\n   * Sets the container image creation time. The default is {@link Instant#EPOCH}.\n   *\n   * @param creationTime the container image creation time\n   * @return this\n   */\n  public JibContainerBuilder setCreationTime(Instant creationTime) {\n    containerBuildPlanBuilder.setCreationTime(creationTime);\n    containerConfigurationBuilder.setCreationTime(creationTime);\n    return this;\n  }\n\n  /**\n   * Sets a desired platform (properties including OS and architecture) list. If the base image\n   * reference is a Docker manifest list or an OCI image index, an image builder may select the base\n   * images matching the given platforms. If the base image reference is an image manifest, an image\n   * builder may ignore the given platforms and use the platform of the base image or may decide to\n   * raise on error.\n   *\n   * <p>Note that a new container builder starts with \"amd64/linux\" as the default platform.\n   *\n   * @param platforms list of platforms to select base images in case of a manifest list\n   * @return this\n   */\n  public JibContainerBuilder setPlatforms(Set<Platform> platforms) {\n    containerBuildPlanBuilder.setPlatforms(platforms);\n    containerConfigurationBuilder.setPlatforms(platforms);\n    return this;\n  }\n\n  /**\n   * Adds a desired image platform (OS and architecture pair). If the base image reference is a\n   * Docker manifest list or an OCI image index, an image builder may select the base image matching\n   * the given platform. If the base image reference is an image manifest, an image builder may\n   * ignore the given platform and use the platform of the base image or may decide to raise on\n   * error.\n   *\n   * <p>Note that a new new container builder starts with \"amd64/linux\" as the default platform. If\n   * you want to reset the default platform instead of adding a new one, use {@link\n   * #setPlatforms(Set)}.\n   *\n   * @param architecture architecture (for example, {@code amd64}) to select a base image in case of\n   *     a manifest list\n   * @param os OS (for example, {@code linux}) to select a base image in case of a manifest list\n   * @return this\n   */\n  public JibContainerBuilder addPlatform(String architecture, String os) {\n    containerBuildPlanBuilder.addPlatform(architecture, os);\n    containerConfigurationBuilder.addPlatform(architecture, os);\n    return this;\n  }\n\n  /**\n   * Sets the user and group to run the container as. {@code user} can be a username or UID along\n   * with an optional groupname or GID.\n   *\n   * <p>The following are valid formats for {@code user}\n   *\n   * <ul>\n   *   <li>{@code user}\n   *   <li>{@code uid}\n   *   <li>{@code :group}\n   *   <li>{@code :gid}\n   *   <li>{@code user:group}\n   *   <li>{@code uid:gid}\n   *   <li>{@code uid:group}\n   *   <li>{@code user:gid}\n   * </ul>\n   *\n   * @param user the user to run the container as\n   * @return this\n   */\n  public JibContainerBuilder setUser(@Nullable String user) {\n    containerBuildPlanBuilder.setUser(user);\n    containerConfigurationBuilder.setUser(user);\n    return this;\n  }\n\n  /**\n   * Sets the working directory in the container.\n   *\n   * @param workingDirectory the working directory\n   * @return this\n   */\n  public JibContainerBuilder setWorkingDirectory(@Nullable AbsoluteUnixPath workingDirectory) {\n    containerBuildPlanBuilder.setWorkingDirectory(workingDirectory);\n    containerConfigurationBuilder.setWorkingDirectory(workingDirectory);\n    return this;\n  }\n\n  /**\n   * Builds the container.\n   *\n   * @param containerizer the {@link Containerizer} that configures how to containerize\n   * @return the built container\n   * @throws IOException if an I/O exception occurs\n   * @throws CacheDirectoryCreationException if a directory to be used for the cache could not be\n   *     created\n   * @throws HttpHostConnectException if jib failed to connect to a registry\n   * @throws RegistryUnauthorizedException if a registry request is unauthorized and needs\n   *     authentication\n   * @throws RegistryAuthenticationFailedException if registry authentication failed\n   * @throws UnknownHostException if the registry does not exist\n   * @throws InsecureRegistryException if a server could not be verified due to an insecure\n   *     connection\n   * @throws RegistryException if some other error occurred while interacting with a registry\n   * @throws ExecutionException if some other exception occurred during execution\n   * @throws InterruptedException if the execution was interrupted\n   */\n  public JibContainer containerize(Containerizer containerizer)\n      throws InterruptedException, RegistryException, IOException, CacheDirectoryCreationException,\n          ExecutionException {\n    try (BuildContext buildContext = toBuildContext(containerizer);\n        TimerEventDispatcher ignored =\n            new TimerEventDispatcher(\n                buildContext.getEventHandlers(), containerizer.getDescription())) {\n      logSources(buildContext.getEventHandlers());\n\n      BuildResult buildResult = containerizer.run(buildContext);\n      return JibContainer.from(buildContext, buildResult);\n\n    } catch (ExecutionException ex) {\n      // If an ExecutionException occurs, re-throw the cause to be more easily handled by the user\n      if (ex.getCause() instanceof RegistryException) {\n        throw (RegistryException) ex.getCause();\n      }\n      throw ex;\n    }\n  }\n\n  /**\n   * Describes the container contents and configuration without actually physically building a\n   * container.\n   *\n   * @deprecated use {@link #toContainerBuildPlan}.\n   * @return a description of the container being built\n   */\n  @Deprecated\n  public JibContainerDescription describeContainer() {\n    return new JibContainerDescription(layerConfigurations);\n  }\n\n  /**\n   * Internal method. API end users should not use it.\n   *\n   * <p>Converts to {@link ContainerBuildPlan}. Note that not all values that this class holds can\n   * be described by a build plan, such as {@link CredentialRetriever}s for {@link RegistryImage},\n   * {@link DockerClient} for {@link DockerDaemonImage}, and output path for {@link TarImage}.\n   *\n   * @return {@link ContainerBuildPlan}\n   */\n  public ContainerBuildPlan toContainerBuildPlan() {\n    return containerBuildPlanBuilder.build();\n  }\n\n  /**\n   * Internal method. API end users should not use it.\n   *\n   * <p>Reconfigures {@link JibContainerBuilder} from the given {@code buildPlan}. Every value\n   * configurable using \"setters\" in this class is overwritten by the value in {@code buildPlan};\n   * only retained are some base image properties inherent in {@link JibContainerBuilder} but absent\n   * in {@link ContainerBuildPlan}, such as {@link CredentialRetriever}s for {@link RegistryImage},\n   * {@link DockerClient} for {@link DockerDaemonImage}, and output path for {@link TarImage}.\n   *\n   * @param buildPlan build plan to apply\n   * @return {@link JibContainerBuilder} reconfigured from {@code buildPlan}\n   * @throws InvalidImageReferenceException if the base image value in {@code buildPlan} is an\n   *     invalid reference\n   */\n  public JibContainerBuilder applyContainerBuildPlan(ContainerBuildPlan buildPlan)\n      throws InvalidImageReferenceException {\n    containerBuildPlanBuilder\n        .setBaseImage(buildPlan.getBaseImage())\n        .setPlatforms(buildPlan.getPlatforms())\n        .setCreationTime(buildPlan.getCreationTime())\n        .setFormat(buildPlan.getFormat())\n        .setEnvironment(buildPlan.getEnvironment())\n        .setLabels(buildPlan.getLabels())\n        .setVolumes(buildPlan.getVolumes())\n        .setExposedPorts(buildPlan.getExposedPorts())\n        .setUser(buildPlan.getUser())\n        .setWorkingDirectory(buildPlan.getWorkingDirectory())\n        .setEntrypoint(buildPlan.getEntrypoint())\n        .setCmd(buildPlan.getCmd())\n        .setLayers(buildPlan.getLayers());\n\n    containerConfigurationBuilder\n        .setPlatforms(buildPlan.getPlatforms())\n        .setCreationTime(buildPlan.getCreationTime())\n        .setEnvironment(buildPlan.getEnvironment())\n        .setLabels(buildPlan.getLabels())\n        .setVolumes(buildPlan.getVolumes())\n        .setExposedPorts(buildPlan.getExposedPorts())\n        .setUser(buildPlan.getUser())\n        .setWorkingDirectory(buildPlan.getWorkingDirectory())\n        .setEntrypoint(buildPlan.getEntrypoint())\n        .setProgramArguments(buildPlan.getCmd());\n\n    ImageConfiguration.Builder builder =\n        ImageConfiguration.builder(ImageReference.parse(buildPlan.getBaseImage()))\n            .setCredentialRetrievers(baseImageConfiguration.getCredentialRetrievers());\n    baseImageConfiguration.getDockerClient().ifPresent(builder::setDockerClient);\n    baseImageConfiguration.getTarPath().ifPresent(builder::setTarPath);\n    baseImageConfiguration = builder.build();\n\n    // For now, only FileEntriesLayer is supported in jib-core.\n    Function<LayerObject, FileEntriesLayer> castToFileEntriesLayer =\n        layer -> {\n          Verify.verify(\n              layer instanceof FileEntriesLayer,\n              \"layer types other than FileEntriesLayer not yet supported in build plan layers\");\n          return (FileEntriesLayer) layer;\n        };\n    layerConfigurations =\n        buildPlan.getLayers().stream().map(castToFileEntriesLayer).collect(Collectors.toList());\n\n    buildContextBuilder\n        .setTargetFormat(buildPlan.getFormat())\n        .setBaseImageConfiguration(baseImageConfiguration)\n        .setLayerConfigurations(layerConfigurations);\n    return this;\n  }\n\n  /**\n   * Builds a {@link BuildContext} using this and a {@link Containerizer}.\n   *\n   * @param containerizer the {@link Containerizer}\n   * @return the {@link BuildContext}\n   * @throws CacheDirectoryCreationException if a cache directory could not be created\n   */\n  @VisibleForTesting\n  BuildContext toBuildContext(Containerizer containerizer) throws CacheDirectoryCreationException {\n    return buildContextBuilder\n        .setTargetImageConfiguration(containerizer.getImageConfiguration())\n        .setAdditionalTargetImageTags(containerizer.getAdditionalTags())\n        .setBaseImageLayersCacheDirectory(containerizer.getBaseImageLayersCacheDirectory())\n        .setApplicationLayersCacheDirectory(containerizer.getApplicationLayersCacheDirectory())\n        .setContainerConfiguration(containerConfigurationBuilder.build())\n        .setLayerConfigurations(layerConfigurations)\n        .setAllowInsecureRegistries(containerizer.getAllowInsecureRegistries())\n        .setOffline(containerizer.isOfflineMode())\n        .setToolName(containerizer.getToolName())\n        .setToolVersion(containerizer.getToolVersion())\n        .setExecutorService(containerizer.getExecutorService().orElse(null))\n        .setEventHandlers(containerizer.buildEventHandlers())\n        .setAlwaysCacheBaseImage(containerizer.getAlwaysCacheBaseImage())\n        .setRegistryMirrors(containerizer.getRegistryMirrors())\n        .build();\n  }\n\n  private void logSources(EventHandlers eventHandlers) {\n    // Logs the different source files used.\n    eventHandlers.dispatch(LogEvent.info(\"Containerizing application with the following files:\"));\n\n    for (FileEntriesLayer layer : layerConfigurations) {\n      if (layer.getEntries().isEmpty()) {\n        continue;\n      }\n\n      eventHandlers.dispatch(LogEvent.info(\"\\t\" + capitalizeFirstLetter(layer.getName()) + \":\"));\n\n      for (FileEntry entry : layer.getEntries()) {\n        eventHandlers.dispatch(LogEvent.info(\"\\t\\t\" + entry.getSourceFile()));\n      }\n    }\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/TarImage.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport java.nio.file.Path;\nimport java.util.Optional;\nimport javax.annotation.Nullable;\n\n/**\n * Builds to a tarball archive.\n *\n * <p>Usage example:\n *\n * <pre>{@code\n * TarImage tarImage = TarImage.at(Paths.get(\"image.tar\"))\n *                             .named(\"myimage\");\n * }</pre>\n */\npublic class TarImage {\n\n  /**\n   * Constructs a {@link TarImage} with the specified path.\n   *\n   * @param path the path to the tarball archive\n   * @return a new {@link TarImage}\n   */\n  public static TarImage at(Path path) {\n    return new TarImage(path);\n  }\n\n  private final Path path;\n  @Nullable private ImageReference imageReference;\n\n  /** Instantiate with {@link #at}. */\n  private TarImage(Path path) {\n    this.path = path;\n  }\n\n  /**\n   * Sets the name of the image. This is the name that shows up when the tar is loaded by the Docker\n   * daemon.\n   *\n   * @param imageReference the image reference\n   * @return this\n   */\n  public TarImage named(ImageReference imageReference) {\n    this.imageReference = imageReference;\n    return this;\n  }\n\n  /**\n   * Sets the name of the image. This is the name that shows up when the tar is loaded by the Docker\n   * daemon.\n   *\n   * @param imageReference the image reference\n   * @return this\n   * @throws InvalidImageReferenceException if {@code imageReference} is not a valid image reference\n   */\n  public TarImage named(String imageReference) throws InvalidImageReferenceException {\n    return named(ImageReference.parse(imageReference));\n  }\n\n  \nPath getPath() {\n    return path;\n  }\n\n  Optional<ImageReference> getImageReference() {\n    return Optional.ofNullable(imageReference);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/Jib.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport java.nio.file.Paths;\n\n/** Build containers with Jib. */\npublic class Jib {\n\n  public static final String REGISTRY_IMAGE_PREFIX = \"registry://\";\n  public static final String DOCKER_DAEMON_IMAGE_PREFIX = \"docker://\";\n  public static final String TAR_IMAGE_PREFIX = \"tar://\";\n\n  /**\n   * Starts building the container from a base image. The type of base image can be specified using\n   * a prefix, e.g. {@code docker://gcr.io/project/image}. The available prefixes are described\n   * below:\n   *\n   * <ul>\n   *   <li>No prefix, or {@code registry://}: uses a registry base image\n   *   <li>{@code docker://}: uses a base image found in the local Docker daemon\n   *   <li>{@code tar://}: uses a tarball base image at the path following the prefix\n   * </ul>\n   *\n   * @param baseImageReference the base image reference\n   * @return a new {@link JibContainerBuilder} to continue building the container\n   * @throws InvalidImageReferenceException if the {@code baseImageReference} is not a valid image\n   *     reference\n   */\n  public static JibContainerBuilder from(String baseImageReference)\n      throws InvalidImageReferenceException {\n    if (baseImageReference.startsWith(DOCKER_DAEMON_IMAGE_PREFIX)) {\n      return from(\n          DockerDaemonImage.named(baseImageReference.replaceFirst(DOCKER_DAEMON_IMAGE_PREFIX, \"\")));\n    }\n    if (baseImageReference.startsWith(TAR_IMAGE_PREFIX)) {\n      return from(TarImage.at(Paths.get(baseImageReference.replaceFirst(TAR_IMAGE_PREFIX, \"\"))));\n    }\n    return from(RegistryImage.named(baseImageReference.replaceFirst(REGISTRY_IMAGE_PREFIX, \"\")));\n  }\n\n  /**\n   * Starts building the container from a base image. The base image should be publicly-available.\n   * For a base image that requires credentials, use {@link #from(RegistryImage)}.\n   *\n   * @param baseImageReference the base image reference\n   * @return a new {@link JibContainerBuilder} to continue building the container\n   */\n  public static JibContainerBuilder from(ImageReference baseImageReference) {\n    return from(RegistryImage.named(baseImageReference));\n  }\n\n  /**\n   * Starts building the container from a registry base image.\n   *\n   * @param registryImage the {@link RegistryImage} that defines base container registry and\n   *     credentials\n   * @return a new {@link JibContainerBuilder} to continue building the container\n   */\n  public static JibContainerBuilder from(RegistryImage registryImage) {\n    return new JibContainerBuilder(registryImage);\n  }\n\n  /**\n   * Starts building the container from a base image stored in the Docker cache. Requires a running\n   * Docker daemon.\n   *\n   * @param dockerDaemonImage the {@link DockerDaemonImage} that defines the base image and Docker\n   *     client\n   * @return a new {@link JibContainerBuilder} to continue building the container\n   */\n  public static JibContainerBuilder from(DockerDaemonImage dockerDaemonImage) {\n    return new JibContainerBuilder(dockerDaemonImage);\n  }\n\n  /**\n   * Starts building the container from a tarball.\n   *\n   * @param tarImage the {@link TarImage} that defines the path to the base image\n   * @return a new {@link JibContainerBuilder} to continue building the container\n   */\n  public static JibContainerBuilder from(TarImage tarImage) {\n    return new JibContainerBuilder(tarImage);\n  }\n\n  /**\n   * Starts building the container from an empty base image.\n   *\n   * @return a new {@link JibContainerBuilder} to continue building the container\n   */\n  public static JibContainerBuilder fromScratch() {\n    return from(ImageReference.scratch());\n  }\n\n  /**\n   * Starts building the container from a base image stored in the Docker cache. Requires a running\n   * Docker daemon.\n   *\n   * @param dockerClient the {@link DockerClient} to connect\n   * @param dockerDaemonImage the {@link DockerDaemonImage} that defines the base image and Docker\n   *     client\n   * @return a new {@link JibContainerBuilder} to continue building the container\n   */\n  public static JibContainerBuilder from(\n      DockerClient dockerClient, DockerDaemonImage dockerDaemonImage) {\n    return new JibContainerBuilder(dockerClient, dockerDaemonImage);\n  }\n\n  private Jib() {}\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/api/Ports.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.api;\n\nimport com.google.cloud.tools.jib.api.buildplan.Port;\nimport com.google.common.base.Strings;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n/** Utility for parsing Docker/OCI ports from text representations. */\npublic class Ports {\n\n  /**\n   * Pattern used for parsing information out of exposed port configurations.\n   *\n   * <p>Example matches: 100, 200-210, 1000/tcp, 2000/udp, 500-600/tcp\n   */\n  private static final Pattern portPattern = Pattern.compile(\"(\\\\d+)(?:-(\\\\d+))?(?:/(tcp|udp))?\");\n\n  /**\n   * Converts/validates a list of strings representing port ranges to an expanded list of {@link\n   * Port}s.\n   *\n   * <p>For example: [\"1000\", \"2000-2002\"] will expand to a list of {@link Port}s with the port\n   * numbers [1000, 2000, 2001, 2002]\n   *\n   * @param ports the list of port numbers/ranges, with an optional protocol separated by a '/'\n   *     (defaults to TCP if missing).\n   * @return the ports as a list of {@link Port}\n   * @throws NumberFormatException if any of the ports are in an invalid format or out of range\n   */\n  public static Set<Port> parse(List<String> ports) throws NumberFormatException {\n    Set<Port> result = new HashSet<>();\n\n    for (String port : ports) {\n      Matcher matcher = portPattern.matcher(port);\n\n      if (!matcher.matches()) {\n        throw new NumberFormatException(\n            \"Invalid port configuration: '\"\n                + port\n                + \"'. Make sure the port is a single number or a range of two numbers separated \"\n                + \"with a '-', with or without protocol specified (e.g. '<portNum>/tcp' or \"\n                + \"'<portNum>/udp').\");\n      }\n\n      // Parse protocol\n      int min = Integer.parseInt(matcher.group(1));\n      int max = min;\n      if (!Strings.isNullOrEmpty(matcher.group(2))) {\n        max = Integer.parseInt(matcher.group(2));\n      }\n      String protocol = matcher.group(3);\n\n      // Error if configured as 'max-min' instead of 'min-max'\n      if (min > max) {\n        throw new NumberFormatException(\n            \"Invalid port range '\" + port + \"'; smaller number must come first.\");\n      }\n\n      // Warn for possibly invalid port numbers\n      if (min < 1 || max > 65535) {\n        throw new NumberFormatException(\n            \"Port number '\" + port + \"' is out of usual range (1-65535).\");\n      }\n\n      for (int portNumber = min; portNumber <= max; portNumber++) {\n...\n", "needle_token_start": 13668, "needle_token_end": 13681, "code_context_ntokens": 16072, "output": ["```java\nPath getPath() {\n    return path;\n}\n```"]}
{"repo": "GoogleContainerTools/jib", "name": "getBaseImageIfAllLayersCached", "language": "java", "path": "jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function checks if all layers of a specified base image are present in the cache and retrieves the image if they are.\n2. **Input**: The function takes a combination of an image manifest and a container configuration, along with a boolean indicating whether the image is part of a manifest list.\n3. **Output**: Returns an optional image object; the image is returned if all its layers are cached, otherwise, an empty optional is returned.\n4. **Procedure**: \n   - The function first verifies that the manifest is not null.\n   - It checks if all layers specified in the manifest are present in the cache.\n   - If any layer is missing, it returns an empty optional.\n   - If the manifest is of an older version, it directly converts the manifest to an image.\n   - For newer versions, it also checks the container configuration and, if the image is not part of a manifest list, verifies the platform configuration.\n   - Finally, it converts the manifest and container configuration into an image object and returns it.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: jib-core/src/main/java/com/google/cloud/tools/jib/configuration/ImageConfiguration.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.configuration;\n\nimport com.google.cloud.tools.jib.api.CredentialRetriever;\nimport com.google.cloud.tools.jib.api.DockerClient;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ImmutableList;\nimport java.nio.file.Path;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.Optional;\nimport javax.annotation.Nullable;\n\n/** Immutable configuration options for a base or target image reference. */\npublic class ImageConfiguration {\n\n  /** Builder for instantiating an {@link ImageConfiguration}. */\n  public static class Builder {\n\n    private ImageReference imageReference;\n    private ImmutableList<CredentialRetriever> credentialRetrievers = ImmutableList.of();\n    @Nullable private DockerClient dockerClient;\n    @Nullable private Path tarPath;\n\n    /**\n     * Sets the providers for registry credentials. The order determines the priority in which the\n     * retrieval methods are attempted.\n     *\n     * @param credentialRetrievers the list of {@link CredentialRetriever}s\n     * @return this\n     */\n    public Builder setCredentialRetrievers(List<CredentialRetriever> credentialRetrievers) {\n      Preconditions.checkArgument(\n          credentialRetrievers.stream().allMatch(Objects::nonNull),\n          \"credential retriever list contains null elements\");\n      this.credentialRetrievers = ImmutableList.copyOf(credentialRetrievers);\n      return this;\n    }\n\n    /**\n     * Sets the Docker client to be used for Docker daemon base images.\n     *\n     * @param dockerClient the Docker client\n     * @return this\n     */\n    public Builder setDockerClient(DockerClient dockerClient) {\n      this.dockerClient = dockerClient;\n      return this;\n    }\n\n    /**\n     * Sets the path for tarball base images.\n     *\n     * @param tarPath the path\n     * @return this\n     */\n    public Builder setTarPath(Path tarPath) {\n      this.tarPath = tarPath;\n      return this;\n    }\n\n    /**\n     * Builds the {@link ImageConfiguration}.\n     *\n     * @return the corresponding {@link ImageConfiguration}\n     */\n    public ImageConfiguration build() {\n      int numArguments = 0;\n      if (!credentialRetrievers.isEmpty()) {\n        numArguments++;\n      }\n      if (dockerClient != null) {\n        numArguments++;\n      }\n      if (tarPath != null) {\n        numArguments++;\n      }\n      Preconditions.checkArgument(numArguments <= 1);\n      return new ImageConfiguration(imageReference, credentialRetrievers, dockerClient, tarPath);\n    }\n\n    private Builder(ImageReference imageReference) {\n      this.imageReference = imageReference;\n    }\n  }\n\n  /**\n   * Constructs a builder for an {@link ImageConfiguration}.\n   *\n   * @param imageReference the image reference, which is a required field\n   * @return the builder\n   */\n  public static Builder builder(ImageReference imageReference) {\n    return new Builder(imageReference);\n  }\n\n  private final ImageReference image;\n  private final ImmutableList<CredentialRetriever> credentialRetrievers;\n  @Nullable private DockerClient dockerClient;\n  @Nullable private Path tarPath;\n\n  private ImageConfiguration(\n      ImageReference image,\n      ImmutableList<CredentialRetriever> credentialRetrievers,\n      @Nullable DockerClient dockerClient,\n      @Nullable Path tarPath) {\n    this.image = image;\n    this.credentialRetrievers = credentialRetrievers;\n    this.dockerClient = dockerClient;\n    this.tarPath = tarPath;\n  }\n\n...\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RegistryCredentialRetriever.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.Credential;\nimport com.google.cloud.tools.jib.api.CredentialRetriever;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ImageConfiguration;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException;\nimport java.util.Optional;\n\n/** Attempts to retrieve registry credentials. */\nclass RegistryCredentialRetriever {\n\n  private RegistryCredentialRetriever() {}\n\n  /** Retrieves credentials for the base image. */\n  static Optional<Credential> getBaseImageCredential(BuildContext buildContext)\n      throws CredentialRetrievalException {\n    return retrieve(buildContext.getBaseImageConfiguration(), buildContext.getEventHandlers());\n  }\n\n  /** Retrieves credentials for the target image. */\n  static Optional<Credential> getTargetImageCredential(BuildContext buildContext)\n      throws CredentialRetrievalException {\n    return retrieve(buildContext.getTargetImageConfiguration(), buildContext.getEventHandlers());\n  }\n\n  private static Optional<Credential> retrieve(\n      ImageConfiguration imageConfiguration, EventHandlers eventHandlers)\n      throws CredentialRetrievalException {\n    for (CredentialRetriever retriever : imageConfiguration.getCredentialRetrievers()) {\n      Optional<Credential> credential = retriever.retrieve();\n      if (credential.isPresent()) {\n        return credential;\n      }\n    }\n\n    String registry = imageConfiguration.getImageRegistry();\n    String repository = imageConfiguration.getImageRepository();\n    eventHandlers.dispatch(\n        LogEvent.info(\"No credentials could be retrieved for \" + registry + \"/\" + repository));\n    return Optional.empty();\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerCountMismatchException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\n/** Exception thrown when the number of layers found did not match expectations. */\npublic class LayerCountMismatchException extends Exception {\n\n  public LayerCountMismatchException(String message) {\n    super(message);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BadContainerConfigurationFormatException.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\n/** Exception thrown when trying to parse a bad image configuration format. */\npublic class BadContainerConfigurationFormatException extends Exception {\n\n  // TODO: Potentially provide Path or source object to problem configuration file\n  BadContainerConfigurationFormatException(String message) {\n    super(message);\n  }\n\n  BadContainerConfigurationFormatException(String message, Throwable cause) {\n    super(message, cause);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/DigestOnlyLayer.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\n\n/** A {@link Layer} reference that only has its {@link DescriptorDigest}. */\npublic class DigestOnlyLayer implements Layer {\n\n  /** The {@link BlobDescriptor} of the compressed layer content. */\n  private final BlobDescriptor blobDescriptor;\n\n  /**\n   * Instantiate with a {@link DescriptorDigest}.\n   *\n   * @param digest the digest to instantiate the {@link DigestOnlyLayer} from\n   */\n  public DigestOnlyLayer(DescriptorDigest digest) {\n    blobDescriptor = new BlobDescriptor(digest);\n  }\n\n  @Override\n  public Blob getBlob() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\"Blob not available for digest-only layer\");\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return blobDescriptor;\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\"Diff ID not available for digest-only layer\");\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/ReferenceLayer.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\n\n/**\n * A {@link Layer} reference that <b>does not</b> have the underlying content. It references the\n * layer with its digest, size, and diff ID.\n */\npublic class ReferenceLayer implements Layer {\n\n  /** The {@link BlobDescriptor} of the compressed layer content. */\n  private final BlobDescriptor blobDescriptor;\n\n  /** The digest of the uncompressed layer content. */\n  private final DescriptorDigest diffId;\n\n  /**\n   * Instantiate with a {@link BlobDescriptor} and diff ID.\n   *\n   * @param blobDescriptor the blob descriptor\n   * @param diffId the diff ID\n   */\n  public ReferenceLayer(BlobDescriptor blobDescriptor, DescriptorDigest diffId) {\n    this.blobDescriptor = blobDescriptor;\n    this.diffId = diffId;\n  }\n\n  @Override\n  public Blob getBlob() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\"Blob not available for reference layer\");\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return blobDescriptor;\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() {\n    return diffId;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/ReferenceNoDiffIdLayer.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.blob.Blob;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\n\n/**\n * A {@link Layer} reference that <b>does not</b> have the underlying content. It references the\n * layer with its digest and size, but <b>not</b> its diff ID.\n */\npublic class ReferenceNoDiffIdLayer implements Layer {\n\n  /** The {@link BlobDescriptor} of the compressed layer content. */\n  private final BlobDescriptor blobDescriptor;\n\n  /**\n   * Instantiate with a {@link BlobDescriptor} and no diff ID.\n   *\n   * @param blobDescriptor the blob descriptor\n   */\n  public ReferenceNoDiffIdLayer(BlobDescriptor blobDescriptor) {\n    this.blobDescriptor = blobDescriptor;\n  }\n\n  @Override\n  public Blob getBlob() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\n        \"Blob not available for reference layer without diff ID\");\n  }\n\n  @Override\n  public BlobDescriptor getBlobDescriptor() {\n    return blobDescriptor;\n  }\n\n  @Override\n  public DescriptorDigest getDiffId() throws LayerPropertyNotFoundException {\n    throw new LayerPropertyNotFoundException(\n        \"Diff ID not available for reference layer without diff ID\");\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/JsonToImageTranslator.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath;\nimport com.google.cloud.tools.jib.api.buildplan.Port;\nimport com.google.cloud.tools.jib.blob.BlobDescriptor;\nimport com.google.cloud.tools.jib.configuration.DockerHealthCheck;\nimport com.google.cloud.tools.jib.image.DigestOnlyLayer;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.LayerCountMismatchException;\nimport com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;\nimport com.google.cloud.tools.jib.image.ReferenceLayer;\nimport com.google.cloud.tools.jib.image.ReferenceNoDiffIdLayer;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Lists;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.time.format.DateTimeFormatter;\nimport java.time.format.DateTimeParseException;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport javax.annotation.Nullable;\n\n/** Translates {@link V21ManifestTemplate} and {@link V22ManifestTemplate} into {@link Image}. */\npublic class JsonToImageTranslator {\n\n  /**\n   * Pattern used for parsing information out of exposed port configurations. Only accepts single\n   * ports with protocol.\n   *\n   * <p>Example matches: 100, 1000/tcp, 2000/udp\n   */\n  private static final Pattern PORT_PATTERN =\n      Pattern.compile(\"(?<portNum>\\\\d+)(?:/(?<protocol>tcp|udp))?\");\n\n  /**\n   * Pattern used for parsing environment variables in the format {@code NAME=VALUE}. {@code NAME}\n   * should not contain an '='.\n   *\n   * <p>Example matches: NAME=VALUE, A12345=$$$$$\n   */\n  @VisibleForTesting\n  static final Pattern ENVIRONMENT_PATTERN = Pattern.compile(\"(?<name>[^=]+)=(?<value>.*)\");\n\n  /**\n   * Translates {@link V21ManifestTemplate} to {@link Image}.\n   *\n   * @param manifestTemplate the template containing the image layers.\n   * @return the translated {@link Image}.\n   * @throws LayerPropertyNotFoundException if adding image layers fails.\n   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad\n   *     format\n   */\n  public static Image toImage(V21ManifestTemplate manifestTemplate)\n      throws LayerPropertyNotFoundException, BadContainerConfigurationFormatException {\n    Image.Builder imageBuilder = Image.builder(V21ManifestTemplate.class);\n\n    // V21 layers are in reverse order of V22. (The first layer is the latest one.)\n    for (DescriptorDigest digest : Lists.reverse(manifestTemplate.getLayerDigests())) {\n      imageBuilder.addLayer(new DigestOnlyLayer(digest));\n    }\n\n    Optional<ContainerConfigurationTemplate> configuration =\n        manifestTemplate.getContainerConfiguration();\n    if (configuration.isPresent()) {\n      configureBuilderWithContainerConfiguration(imageBuilder, configuration.get());\n    }\n    return imageBuilder.build();\n  }\n\n  /**\n   * Translates {@link BuildableManifestTemplate} to {@link Image}. Uses the corresponding {@link\n   * ContainerConfigurationTemplate} to get the layer diff IDs.\n   *\n   * @param manifestTemplate the template containing the image layers.\n   * @param containerConfigurationTemplate the template containing the diff IDs and container\n   *     configuration properties.\n   * @return the translated {@link Image}.\n   * @throws LayerCountMismatchException if the manifest and configuration contain conflicting layer\n   *     information.\n   * @throws LayerPropertyNotFoundException if adding image layers fails.\n   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad\n   *     format\n   */\n  public static Image toImage(\n      BuildableManifestTemplate manifestTemplate,\n      ContainerConfigurationTemplate containerConfigurationTemplate)\n      throws LayerCountMismatchException, LayerPropertyNotFoundException,\n          BadContainerConfigurationFormatException {\n    List<ReferenceNoDiffIdLayer> layers = new ArrayList<>();\n    for (BuildableManifestTemplate.ContentDescriptorTemplate layerObjectTemplate :\n        manifestTemplate.getLayers()) {\n      if (layerObjectTemplate.getDigest() == null) {\n        throw new IllegalArgumentException(\n            \"All layers in the manifest template must have digest set\");\n      }\n\n      layers.add(\n          new ReferenceNoDiffIdLayer(\n              new BlobDescriptor(layerObjectTemplate.getSize(), layerObjectTemplate.getDigest())));\n    }\n\n    List<DescriptorDigest> diffIds = containerConfigurationTemplate.getDiffIds();\n    if (layers.size() != diffIds.size()) {\n      throw new LayerCountMismatchException(\n          \"Mismatch between image manifest and container configuration\");\n    }\n\n    Image.Builder imageBuilder = Image.builder(manifestTemplate.getClass());\n\n    for (int layerIndex = 0; layerIndex < layers.size(); layerIndex++) {\n      ReferenceNoDiffIdLayer noDiffIdLayer = layers.get(layerIndex);\n      DescriptorDigest diffId = diffIds.get(layerIndex);\n\n      imageBuilder.addLayer(new ReferenceLayer(noDiffIdLayer.getBlobDescriptor(), diffId));\n    }\n\n    configureBuilderWithContainerConfiguration(imageBuilder, containerConfigurationTemplate);\n    return imageBuilder.build();\n  }\n\n  private static void configureBuilderWithContainerConfiguration(\n      Image.Builder imageBuilder, ContainerConfigurationTemplate containerConfigurationTemplate)\n      throws BadContainerConfigurationFormatException {\n\n    containerConfigurationTemplate.getHistory().forEach(imageBuilder::addHistory);\n\n    if (containerConfigurationTemplate.getCreated() != null) {\n      try {\n        imageBuilder.setCreated(Instant.parse(containerConfigurationTemplate.getCreated()));\n      } catch (DateTimeParseException ex) {\n        try {\n          // TODO: remove when using Java >= 12.\n          // See https://github.com/GoogleContainerTools/jib/issues/2428\n          imageBuilder.setCreated(\n              DateTimeFormatter.ISO_OFFSET_DATE_TIME.parse(\n                  containerConfigurationTemplate.getCreated(), Instant::from));\n        } catch (DateTimeParseException ignored) {\n          throw new BadContainerConfigurationFormatException(\n              \"Invalid image creation time: \" + containerConfigurationTemplate.getCreated(), ex);\n        }\n      }\n    }\n\n    if (containerConfigurationTemplate.getArchitecture() != null) {\n      imageBuilder.setArchitecture(containerConfigurationTemplate.getArchitecture());\n    }\n    if (containerConfigurationTemplate.getOs() != null) {\n      imageBuilder.setOs(containerConfigurationTemplate.getOs());\n    }\n\n    imageBuilder.setEntrypoint(containerConfigurationTemplate.getContainerEntrypoint());\n    imageBuilder.setProgramArguments(containerConfigurationTemplate.getContainerCmd());\n\n    List<String> baseHealthCheckCommand = containerConfigurationTemplate.getContainerHealthTest();\n    if (baseHealthCheckCommand != null) {\n      DockerHealthCheck.Builder builder = DockerHealthCheck.fromCommand(baseHealthCheckCommand);\n      if (containerConfigurationTemplate.getContainerHealthInterval() != null) {\n        builder.setInterval(\n            Duration.ofNanos(containerConfigurationTemplate.getContainerHealthInterval()));\n      }\n      if (containerConfigurationTemplate.getContainerHealthTimeout() != null) {\n        builder.setTimeout(\n            Duration.ofNanos(containerConfigurationTemplate.getContainerHealthTimeout()));\n      }\n      if (containerConfigurationTemplate.getContainerHealthStartPeriod() != null) {\n        builder.setStartPeriod(\n            Duration.ofNanos(containerConfigurationTemplate.getContainerHealthStartPeriod()));\n      }\n      if (containerConfigurationTemplate.getContainerHealthRetries() != null) {\n        builder.setRetries(containerConfigurationTemplate.getContainerHealthRetries());\n      }\n      imageBuilder.setHealthCheck(builder.build());\n    }\n\n    if (containerConfigurationTemplate.getContainerExposedPorts() != null) {\n      imageBuilder.addExposedPorts(\n          portMapToSet(containerConfigurationTemplate.getContainerExposedPorts()));\n    }\n\n    if (containerConfigurationTemplate.getContainerVolumes() != null) {\n      imageBuilder.addVolumes(volumeMapToSet(containerConfigurationTemplate.getContainerVolumes()));\n    }\n\n    if (containerConfigurationTemplate.getContainerEnvironment() != null) {\n      for (String environmentVariable : containerConfigurationTemplate.getContainerEnvironment()) {\n        Matcher matcher = ENVIRONMENT_PATTERN.matcher(environmentVariable);\n        if (!matcher.matches()) {\n          throw new BadContainerConfigurationFormatException(\n              \"Invalid environment variable definition: \" + environmentVariable);\n        }\n        imageBuilder.addEnvironmentVariable(matcher.group(\"name\"), matcher.group(\"value\"));\n      }\n    }\n\n    imageBuilder.addLabels(containerConfigurationTemplate.getContainerLabels());\n    imageBuilder.setWorkingDirectory(containerConfigurationTemplate.getContainerWorkingDir());\n    imageBuilder.setUser(containerConfigurationTemplate.getContainerUser());\n  }\n\n  /**\n   * Converts a map of exposed ports as strings to a set of {@link Port}s (e.g. {@code\n   * {\"1000/tcp\":{}}} -> {@code Port(1000, Protocol.TCP)}).\n   *\n   * @param portMap the map to convert\n   * @return a set of {@link Port}s\n   */\n  @VisibleForTesting\n  static ImmutableSet<Port> portMapToSet(@Nullable Map<String, Map<String, String>> portMap)\n      throws BadContainerConfigurationFormatException {\n    if (portMap == null) {\n      return ImmutableSet.of();\n    }\n    ImmutableSet.Builder<Port> ports = new ImmutableSet.Builder<>();\n    for (Map.Entry<String, Map<String, String>> entry : portMap.entrySet()) {\n      String port = entry.getKey();\n      Matcher matcher = PORT_PATTERN.matcher(port);\n      if (!matcher.matches()) {\n        throw new BadContainerConfigurationFormatException(\n            \"Invalid port configuration: '\" + port + \"'.\");\n      }\n\n      int portNumber = Integer.parseInt(matcher.group(\"portNum\"));\n      String protocol = matcher.group(\"protocol\");\n      ports.add(Port.parseProtocol(portNumber, protocol));\n    }\n    return ports.build();\n  }\n\n  /**\n   * Converts a map of volumes strings to a set of {@link AbsoluteUnixPath}s (e.g. {@code\n   * {\"/var/log/my-app-logs\":{}}} -> {@code AbsoluteUnixPath().get(\"/var/log/my-app-logs\")}).\n   *\n   * @param volumeMap the map to convert\n   * @return a set of {@link AbsoluteUnixPath}s\n   */\n  @VisibleForTesting\n  static ImmutableSet<AbsoluteUnixPath> volumeMapToSet(\n      @Nullable Map<String, Map<String, String>> volumeMap)\n      throws BadContainerConfigurationFormatException {\n    if (volumeMap == null) {\n      return ImmutableSet.of();\n    }\n\n    ImmutableSet.Builder<AbsoluteUnixPath> volumeList = ImmutableSet.builder();\n    for (String volume : volumeMap.keySet()) {\n      try {\n        volumeList.add(AbsoluteUnixPath.get(volume));\n      } catch (IllegalArgumentException exception) {\n        throw new BadContainerConfigurationFormatException(\"Invalid volume path: \" + volume);\n      }\n    }\n\n    return volumeList.build();\n  }\n\n  private JsonToImageTranslator() {}\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java\n/*\n * Copyright 2022 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport java.util.List;\n\n/**\n * Parent class for manifest lists.\n *\n * @see V22ManifestListTemplate Docker V2.2 format\n * @see OciIndexTemplate OCI format\n */\npublic interface ManifestListTemplate extends ManifestTemplate {\n\n  /**\n   * Returns a list of digests for a specific platform found in the manifest list. see\n   * <a>https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list</a>\n   *\n   * @param architecture the architecture of the target platform\n   * @param os the os of the target platform\n   * @return a list of matching digests\n   */\n  List<String> getDigestsForPlatform(String architecture, String os);\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/UnknownManifestFormatException.java\n/*\n * Copyright 2017 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.RegistryException;\n\n/** Exception thrown when trying to parse an unknown image manifest format. */\npublic class UnknownManifestFormatException extends RegistryException {\n\n  public UnknownManifestFormatException(String message) {\n    super(message);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/image/json/UnlistedPlatformInManifestListException.java\n/*\n * Copyright 2020 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.image.json;\n\nimport com.google.cloud.tools.jib.api.RegistryException;\n\n/** Exception thrown when there is no matching platform in a manifest list. */\npublic class UnlistedPlatformInManifestListException extends RegistryException {\n\n  public UnlistedPlatformInManifestListException(String message) {\n    super(message);\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestAndDigest.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.registry;\n\nimport com.google.cloud.tools.jib.api.DescriptorDigest;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\n\n/** Stores a manifest and digest. */\npublic class ManifestAndDigest<T extends ManifestTemplate> {\n\n  private final T manifest;\n  private final DescriptorDigest digest;\n\n  public ManifestAndDigest(T manifest, DescriptorDigest digest) {\n    this.manifest = manifest;\n    this.digest = digest;\n  }\n\n  /**\n   * Gets the manifest.\n   *\n   * @return the manifest\n   */\n  public T getManifest() {\n    return manifest;\n  }\n\n  /**\n   * Gets the digest.\n   *\n   * @return the digest\n   */\n  public DescriptorDigest getDigest() {\n    return digest;\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java\n/*\n * Copyright 2018 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.builder.steps;\n\nimport com.google.cloud.tools.jib.api.Credential;\nimport com.google.cloud.tools.jib.api.ImageReference;\nimport com.google.cloud.tools.jib.api.LogEvent;\nimport com.google.cloud.tools.jib.api.RegistryException;\nimport com.google.cloud.tools.jib.api.RegistryUnauthorizedException;\nimport com.google.cloud.tools.jib.api.buildplan.Platform;\nimport com.google.cloud.tools.jib.blob.Blobs;\nimport com.google.cloud.tools.jib.builder.ProgressEventDispatcher;\nimport com.google.cloud.tools.jib.builder.TimerEventDispatcher;\nimport com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImagesAndRegistryClient;\nimport com.google.cloud.tools.jib.cache.Cache;\nimport com.google.cloud.tools.jib.cache.CacheCorruptedException;\nimport com.google.cloud.tools.jib.configuration.BuildContext;\nimport com.google.cloud.tools.jib.configuration.ImageConfiguration;\nimport com.google.cloud.tools.jib.event.EventHandlers;\nimport com.google.cloud.tools.jib.event.events.ProgressEvent;\nimport com.google.cloud.tools.jib.image.Image;\nimport com.google.cloud.tools.jib.image.LayerCountMismatchException;\nimport com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;\nimport com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException;\nimport com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;\nimport com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;\nimport com.google.cloud.tools.jib.image.json.JsonToImageTranslator;\nimport com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestListTemplate;\nimport com.google.cloud.tools.jib.image.json.ManifestTemplate;\nimport com.google.cloud.tools.jib.image.json.PlatformNotFoundInBaseImageException;\nimport com.google.cloud.tools.jib.image.json.UnknownManifestFormatException;\nimport com.google.cloud.tools.jib.image.json.UnlistedPlatformInManifestListException;\nimport com.google.cloud.tools.jib.image.json.V21ManifestTemplate;\nimport com.google.cloud.tools.jib.json.JsonTemplateMapper;\nimport com.google.cloud.tools.jib.registry.ManifestAndDigest;\nimport com.google.cloud.tools.jib.registry.RegistryClient;\nimport com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException;\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.base.Preconditions;\nimport com.google.common.base.Verify;\nimport com.google.common.collect.ImmutableList;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.concurrent.Callable;\nimport javax.annotation.Nullable;\n\n/** Pulls the base image manifests for the specified platforms. */\nclass PullBaseImageStep implements Callable<ImagesAndRegistryClient> {\n\n  private static final String DESCRIPTION = \"Pulling base image manifest\";\n\n  /** Structure for the result returned by this step. */\n  static class ImagesAndRegistryClient {\n\n    final List<Image> images;\n    @Nullable final RegistryClient registryClient;\n\n    ImagesAndRegistryClient(List<Image> images, @Nullable RegistryClient registryClient) {\n      this.images = images;\n      this.registryClient = registryClient;\n    }\n  }\n\n  private final BuildContext buildContext;\n  private final ProgressEventDispatcher.Factory progressDispatcherFactory;\n\n  PullBaseImageStep(\n      BuildContext buildContext, ProgressEventDispatcher.Factory progressDispatcherFactory) {\n    this.buildContext = buildContext;\n    this.progressDispatcherFactory = progressDispatcherFactory;\n  }\n\n  @Override\n  public ImagesAndRegistryClient call()\n      throws IOException, RegistryException, LayerPropertyNotFoundException,\n          LayerCountMismatchException, BadContainerConfigurationFormatException,\n          CacheCorruptedException, CredentialRetrievalException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    try (ProgressEventDispatcher progressDispatcher =\n            progressDispatcherFactory.create(\"pulling base image manifest\", 4);\n        TimerEventDispatcher ignored1 = new TimerEventDispatcher(eventHandlers, DESCRIPTION)) {\n\n      // Skip this step if this is a scratch image\n      ImageReference imageReference = buildContext.getBaseImageConfiguration().getImage();\n      if (imageReference.isScratch()) {\n        Set<Platform> platforms = buildContext.getContainerConfiguration().getPlatforms();\n        Verify.verify(!platforms.isEmpty());\n\n        eventHandlers.dispatch(LogEvent.progress(\"Getting scratch base image...\"));\n        ImmutableList.Builder<Image> images = ImmutableList.builder();\n        for (Platform platform : platforms) {\n          Image.Builder imageBuilder = Image.builder(buildContext.getTargetFormat());\n          imageBuilder.setArchitecture(platform.getArchitecture()).setOs(platform.getOs());\n          images.add(imageBuilder.build());\n        }\n        return new ImagesAndRegistryClient(images.build(), null);\n      }\n\n      eventHandlers.dispatch(\n          LogEvent.progress(\"Getting manifest for base image \" + imageReference + \"...\"));\n\n      if (buildContext.isOffline()) {\n        List<Image> images = getCachedBaseImages();\n        if (!images.isEmpty()) {\n          return new ImagesAndRegistryClient(images, null);\n        }\n        throw new IOException(\n            \"Cannot run Jib in offline mode; \" + imageReference + \" not found in local Jib cache\");\n\n      } else if (imageReference.getDigest().isPresent()) {\n        List<Image> images = getCachedBaseImages();\n        if (!images.isEmpty()) {\n          RegistryClient noAuthRegistryClient =\n              buildContext.newBaseImageRegistryClientFactory().newRegistryClient();\n          // TODO: passing noAuthRegistryClient may be problematic. It may return 401 unauthorized\n          // if layers have to be downloaded.\n          // https://github.com/GoogleContainerTools/jib/issues/2220\n          return new ImagesAndRegistryClient(images, noAuthRegistryClient);\n        }\n      }\n\n      Optional<ImagesAndRegistryClient> mirrorPull =\n          tryMirrors(buildContext, progressDispatcher.newChildProducer());\n      if (mirrorPull.isPresent()) {\n        return mirrorPull.get();\n      }\n\n      try {\n        // First, try with no credentials. This works with public GCR images (but not Docker Hub).\n        // TODO: investigate if we should just pass credentials up front. However, this involves\n        // some risk. https://github.com/GoogleContainerTools/jib/pull/2200#discussion_r359069026\n        // contains some related discussions.\n        RegistryClient noAuthRegistryClient =\n            buildContext.newBaseImageRegistryClientFactory().newRegistryClient();\n        return new ImagesAndRegistryClient(\n            pullBaseImages(noAuthRegistryClient, progressDispatcher.newChildProducer()),\n            noAuthRegistryClient);\n\n      } catch (RegistryUnauthorizedException ex) {\n        eventHandlers.dispatch(\n            LogEvent.lifecycle(\n                \"The base image requires auth. Trying again for \" + imageReference + \"...\"));\n\n        Credential credential =\n            RegistryCredentialRetriever.getBaseImageCredential(buildContext).orElse(null);\n        RegistryClient registryClient =\n            buildContext\n                .newBaseImageRegistryClientFactory()\n                .setCredential(credential)\n                .newRegistryClient();\n\n        String wwwAuthenticate = ex.getHttpResponseException().getHeaders().getAuthenticate();\n        if (wwwAuthenticate != null) {\n          eventHandlers.dispatch(\n              LogEvent.debug(\"WWW-Authenticate for \" + imageReference + \": \" + wwwAuthenticate));\n          registryClient.authPullByWwwAuthenticate(wwwAuthenticate);\n          return new ImagesAndRegistryClient(\n              pullBaseImages(registryClient, progressDispatcher.newChildProducer()),\n              registryClient);\n\n        } else {\n          // Not getting WWW-Authenticate is unexpected in practice, and we may just blame the\n          // server and fail. However, to keep some old behavior, try a few things as a last resort.\n          // TODO: consider removing this fallback branch.\n          if (credential != null && !credential.isOAuth2RefreshToken()) {\n            eventHandlers.dispatch(\n                LogEvent.debug(\"Trying basic auth as fallback for \" + imageReference + \"...\"));\n            registryClient.configureBasicAuth();\n            try {\n              return new ImagesAndRegistryClient(\n                  pullBaseImages(registryClient, progressDispatcher.newChildProducer()),\n                  registryClient);\n            } catch (RegistryUnauthorizedException ignored) {\n              // Fall back to try bearer auth.\n            }\n          }\n\n          eventHandlers.dispatch(\n              LogEvent.debug(\"Trying bearer auth as fallback for \" + imageReference + \"...\"));\n          registryClient.doPullBearerAuth();\n          return new ImagesAndRegistryClient(\n              pullBaseImages(registryClient, progressDispatcher.newChildProducer()),\n              registryClient);\n        }\n      }\n    }\n  }\n\n  @VisibleForTesting\n  Optional<ImagesAndRegistryClient> tryMirrors(\n      BuildContext buildContext, ProgressEventDispatcher.Factory progressDispatcherFactory)\n      throws LayerCountMismatchException, BadContainerConfigurationFormatException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n\n    Collection<Map.Entry<String, String>> mirrorEntries =\n        buildContext.getRegistryMirrors().entries();\n    try (ProgressEventDispatcher progressDispatcher1 =\n            progressDispatcherFactory.create(\"trying mirrors\", mirrorEntries.size());\n        TimerEventDispatcher ignored1 = new TimerEventDispatcher(eventHandlers, \"trying mirrors\")) {\n      for (Map.Entry<String, String> entry : mirrorEntries) {\n        String registry = entry.getKey();\n        String mirror = entry.getValue();\n        eventHandlers.dispatch(LogEvent.debug(\"mirror config: \" + registry + \" --> \" + mirror));\n\n        if (!buildContext.getBaseImageConfiguration().getImageRegistry().equals(registry)) {\n          progressDispatcher1.dispatchProgress(1);\n          continue;\n        }\n\n        eventHandlers.dispatch(LogEvent.info(\"trying mirror \" + mirror + \" for the base image\"));\n        try (ProgressEventDispatcher progressDispatcher2 =\n            progressDispatcher1.newChildProducer().create(\"trying mirror \" + mirror, 2)) {\n          RegistryClient registryClient =\n              buildContext.newBaseImageRegistryClientFactory(mirror).newRegistryClient();\n          List<Image> images = pullPublicImages(registryClient, progressDispatcher2);\n          eventHandlers.dispatch(LogEvent.info(\"pulled manifest from mirror \" + mirror));\n          return Optional.of(new ImagesAndRegistryClient(images, registryClient));\n\n        } catch (IOException | RegistryException ex) {\n          // Ignore errors from this mirror and continue.\n          eventHandlers.dispatch(\n              LogEvent.debug(\n                  \"failed to get manifest from mirror \" + mirror + \": \" + ex.getMessage()));\n        }\n      }\n      return Optional.empty();\n    }\n  }\n\n  private List<Image> pullPublicImages(\n      RegistryClient registryClient, ProgressEventDispatcher progressDispatcher)\n      throws IOException, RegistryException, LayerCountMismatchException,\n          BadContainerConfigurationFormatException {\n    try {\n      // First, try with no credentials. This works with public GCR images.\n      return pullBaseImages(registryClient, progressDispatcher.newChildProducer());\n\n    } catch (RegistryUnauthorizedException ex) {\n      // in case if a registry requires bearer auth\n      registryClient.doPullBearerAuth();\n      return pullBaseImages(registryClient, progressDispatcher.newChildProducer());\n    }\n  }\n\n  /**\n   * Pulls the base images specified in the platforms list.\n   *\n   * @param registryClient to communicate with remote registry\n   * @param progressDispatcherFactory the {@link ProgressEventDispatcher.Factory} for emitting\n   *     {@link ProgressEvent}s\n   * @return the list of pulled base images and a registry client\n   * @throws IOException when an I/O exception occurs during the pulling\n   * @throws RegistryException if communicating with the registry caused a known error\n   * @throws LayerCountMismatchException if the manifest and configuration contain conflicting layer\n   *     information\n   * @throws LayerPropertyNotFoundException if adding image layers fails\n   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad\n   *     format\n   */\n  private List<Image> pullBaseImages(\n      RegistryClient registryClient, ProgressEventDispatcher.Factory progressDispatcherFactory)\n      throws IOException, RegistryException, LayerPropertyNotFoundException,\n          LayerCountMismatchException, BadContainerConfigurationFormatException {\n    Cache cache = buildContext.getBaseImageLayersCache();\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n    ImageConfiguration baseImageConfig = buildContext.getBaseImageConfiguration();\n\n    try (ProgressEventDispatcher progressDispatcher1 =\n        progressDispatcherFactory.create(\"pulling base image manifest and container config\", 2)) {\n      ManifestAndDigest<?> manifestAndDigest =\n          registryClient.pullManifest(baseImageConfig.getImageQualifier());\n      eventHandlers.dispatch(\n          LogEvent.lifecycle(\"Using base image with digest: \" + manifestAndDigest.getDigest()));\n      progressDispatcher1.dispatchProgress(1);\n      ProgressEventDispatcher.Factory childProgressDispatcherFactory =\n          progressDispatcher1.newChildProducer();\n\n      ManifestTemplate manifestTemplate = manifestAndDigest.getManifest();\n      if (manifestTemplate instanceof V21ManifestTemplate) {\n        V21ManifestTemplate v21Manifest = (V21ManifestTemplate) manifestTemplate;\n        cache.writeMetadata(baseImageConfig.getImage(), v21Manifest);\n        return Collections.singletonList(JsonToImageTranslator.toImage(v21Manifest));\n\n      } else if (manifestTemplate instanceof BuildableManifestTemplate) {\n        // V22ManifestTemplate or OciManifestTemplate\n        BuildableManifestTemplate imageManifest = (BuildableManifestTemplate) manifestTemplate;\n        ContainerConfigurationTemplate containerConfig =\n            pullContainerConfigJson(\n                manifestAndDigest, registryClient, childProgressDispatcherFactory);\n        PlatformChecker.checkManifestPlatform(buildContext, containerConfig);\n        cache.writeMetadata(baseImageConfig.getImage(), imageManifest, containerConfig);\n        return Collections.singletonList(\n            JsonToImageTranslator.toImage(imageManifest, containerConfig));\n      }\n\n      Verify.verify(manifestTemplate instanceof ManifestListTemplate);\n\n      List<ManifestAndConfigTemplate> manifestsAndConfigs = new ArrayList<>();\n      ImmutableList.Builder<Image> images = ImmutableList.builder();\n      Set<Platform> platforms = buildContext.getContainerConfiguration().getPlatforms();\n      try (ProgressEventDispatcher progressDispatcher2 =\n          childProgressDispatcherFactory.create(\n              \"pulling platform-specific manifests and container configs\", 2L * platforms.size())) {\n        // If a manifest list, search for the manifests matching the given platforms.\n        for (Platform platform : platforms) {\n          String message = \"Searching for architecture=%s, os=%s in the base image manifest list\";\n          eventHandlers.dispatch(\n              LogEvent.info(String.format(message, platform.getArchitecture(), platform.getOs())));\n\n          String manifestDigest =\n              lookUpPlatformSpecificImageManifest(\n                  (ManifestListTemplate) manifestTemplate, platform);\n          // TODO: pull multiple manifests (+ container configs) in parallel.\n          ManifestAndDigest<?> imageManifestAndDigest = registryClient.pullManifest(manifestDigest);\n          progressDispatcher2.dispatchProgress(1);\n\n          BuildableManifestTemplate imageManifest =\n              (BuildableManifestTemplate) imageManifestAndDigest.getManifest();\n          ContainerConfigurationTemplate containerConfig =\n              pullContainerConfigJson(\n                  imageManifestAndDigest, registryClient, progressDispatcher2.newChildProducer());\n\n          manifestsAndConfigs.add(\n              new ManifestAndConfigTemplate(imageManifest, containerConfig, manifestDigest));\n          images.add(JsonToImageTranslator.toImage(imageManifest, containerConfig));\n        }\n      }\n\n      cache.writeMetadata(\n          baseImageConfig.getImage(),\n          new ImageMetadataTemplate(manifestTemplate /* manifest list */, manifestsAndConfigs));\n      return images.build();\n    }\n  }\n\n  /**\n   * Looks through a manifest list for the manifest matching the {@code platform} and returns the\n   * digest of the first manifest it finds.\n   */\n  @VisibleForTesting\n  String lookUpPlatformSpecificImageManifest(\n      ManifestListTemplate manifestListTemplate, Platform platform)\n      throws UnlistedPlatformInManifestListException {\n    EventHandlers eventHandlers = buildContext.getEventHandlers();\n\n    List<String> digests =\n        manifestListTemplate.getDigestsForPlatform(platform.getArchitecture(), platform.getOs());\n    if (digests.isEmpty()) {\n      String errorTemplate =\n          buildContext.getBaseImageConfiguration().getImage()\n              + \" is a manifest list, but the list does not contain an image for architecture=%s, \"\n              + \"os=%s. If your intention was to specify a platform for your image, see \"\n              + \"https://github.com/GoogleContainerTools/jib/blob/master/docs/faq.md#how-do-i-specify-a-platform-in-the-manifest-list-or-oci-index-of-a-base-image\";\n      String error = String.format(errorTemplate, platform.getArchitecture(), platform.getOs());\n      eventHandlers.dispatch(LogEvent.error(error));\n      throw new UnlistedPlatformInManifestListException(error);\n    }\n    // TODO: perhaps we should return multiple digests matching the platform.\n    return digests.get(0);\n  }\n\n  /**\n   * Pulls a container configuration JSON specified in the given manifest.\n   *\n   * @param manifestAndDigest a manifest JSON and its digest\n   * @param registryClient to communicate with remote registry\n   * @param progressDispatcherFactory the {@link ProgressEventDispatcher.Factory} for emitting\n   *     {@link ProgressEvent}s\n   * @return pulled {@link ContainerConfigurationTemplate}\n   * @throws IOException when an I/O exception occurs during the pulling\n   * @throws LayerPropertyNotFoundException if adding image layers fails\n   */\n  private ContainerConfigurationTemplate pullContainerConfigJson(\n      ManifestAndDigest<?> manifestAndDigest,\n      RegistryClient registryClient,\n      ProgressEventDispatcher.Factory progressDispatcherFactory)\n      throws IOException, LayerPropertyNotFoundException, UnknownManifestFormatException {\n    BuildableManifestTemplate manifest =\n        (BuildableManifestTemplate) manifestAndDigest.getManifest();\n    Preconditions.checkArgument(manifest.getSchemaVersion() == 2);\n\n    if (manifest.getContainerConfiguration() == null\n        || manifest.getContainerConfiguration().getDigest() == null) {\n      throw new UnknownManifestFormatException(\n          \"Invalid container configuration in Docker V2.2/OCI manifest: \\n\"\n              + JsonTemplateMapper.toUtf8String(manifest));\n    }\n\n    try (ThrottledProgressEventDispatcherWrapper progressDispatcherWrapper =\n        new ThrottledProgressEventDispatcherWrapper(\n            progressDispatcherFactory,\n            \"pull container configuration \" + manifest.getContainerConfiguration().getDigest())) {\n\n      String containerConfigString =\n          Blobs.writeToString(\n              registryClient.pullBlob(\n                  manifest.getContainerConfiguration().getDigest(),\n                  progressDispatcherWrapper::setProgressTarget,\n                  progressDispatcherWrapper::dispatchProgress));\n      return JsonTemplateMapper.readJson(\n          containerConfigString, ContainerConfigurationTemplate.class);\n    }\n  }\n\n  /**\n   * Retrieves the cached base images. If a base image reference is not a manifest list, returns a\n   * single image (if cached). If a manifest list, returns all the images matching the configured\n   * platforms in the manifest list but only when all of the images are cached.\n   *\n   * @return the cached images, if found\n   * @throws IOException when an I/O exception occurs\n   * @throws CacheCorruptedException if the cache is corrupted\n   * @throws LayerPropertyNotFoundException if adding image layers fails\n   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad\n   *     format\n   * @throws UnlistedPlatformInManifestListException if a cached manifest list has no manifests\n   *     matching the configured platform\n   */\n  @VisibleForTesting\n  List<Image> getCachedBaseImages()\n      throws IOException, CacheCorruptedException, BadContainerConfigurationFormatException,\n          LayerCountMismatchException, UnlistedPlatformInManifestListException,\n          PlatformNotFoundInBaseImageException {\n    ImageReference baseImage = buildContext.getBaseImageConfiguration().getImage();\n    Optional<ImageMetadataTemplate> metadata =\n        buildContext.getBaseImageLayersCache().retrieveMetadata(baseImage);\n    if (!metadata.isPresent()) {\n      return Collections.emptyList();\n    }\n\n    ManifestTemplate manifestList = metadata.get().getManifestList();\n    List<ManifestAndConfigTemplate> manifestsAndConfigs = metadata.get().getManifestsAndConfigs();\n\n    if (manifestList == null) {\n      Verify.verify(manifestsAndConfigs.size() == 1);\n      ManifestAndConfigTemplate manifestAndConfig = manifestsAndConfigs.get(0);\n      Optional<Image> cachedImage = getBaseImageIfAllLayersCached(manifestAndConfig, true);\n      if (!cachedImage.isPresent()) {\n        return Collections.emptyList();\n      }\n      return Collections.singletonList(cachedImage.get());\n    }\n\n    // Manifest list cached. Identify matching platforms and check if all of them are cached.\n    ImmutableList.Builder<Image> images = ImmutableList.builder();\n    for (Platform platform : buildContext.getContainerConfiguration().getPlatforms()) {\n      String manifestDigest =\n          lookUpPlatformSpecificImageManifest((ManifestListTemplate) manifestList, platform);\n\n      Optional<ManifestAndConfigTemplate> manifestAndConfigFound =\n          manifestsAndConfigs.stream()\n              .filter(entry -> manifestDigest.equals(entry.getManifestDigest()))\n              .findFirst();\n      if (!manifestAndConfigFound.isPresent()) {\n        return Collections.emptyList();\n      }\n      Optional<Image> cachedImage =\n          getBaseImageIfAllLayersCached(manifestAndConfigFound.get(), false);\n      if (!cachedImage.isPresent()) {\n        return Collections.emptyList();\n      }\n      images.add(cachedImage.get());\n    }\n    return images.build();\n  }\n\n  /**\n   * Helper method to retrieve a base image from cache given manifest and container config. Does not\n   * return image if any layers of base image are missing in cache.\n   *\n   * @param manifestAndConfig stores an image manifest and a container config\n   * @param isSingleManifest true if base image is not a manifest list\n   * @return the single cached {@link Image} found, or {@code Optional#empty()} if the base image\n   *     not found in cache with all layers present.\n   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad\n   *     format\n   * @throws PlatformNotFoundInBaseImageException if build target platform is not found in the base\n   *     image\n   * @throws LayerCountMismatchException LayerCountMismatchException if the manifest and\n   *     configuration contain conflicting layer information\n   */\n  \nprivate Optional<Image> getBaseImageIfAllLayersCached(\n      ManifestAndConfigTemplate manifestAndConfig, boolean isSingleManifest)\n      throws BadContainerConfigurationFormatException, PlatformNotFoundInBaseImageException,\n          LayerCountMismatchException {\n    Cache baseImageLayersCache = buildContext.getBaseImageLayersCache();\n    ManifestTemplate manifest = Verify.verifyNotNull(manifestAndConfig.getManifest());\n\n    // Verify all layers described in manifest are present in cache\n    if (!baseImageLayersCache.areAllLayersCached(manifest)) {\n      return Optional.empty();\n    }\n    if (manifest instanceof V21ManifestTemplate) {\n      return Optional.of(JsonToImageTranslator.toImage((V21ManifestTemplate) manifest));\n    }\n    ContainerConfigurationTemplate containerConfig =\n        Verify.verifyNotNull(manifestAndConfig.getConfig());\n    if (isSingleManifest) {\n      // If base image is not a manifest list, check and warn misconfigured platforms.\n      PlatformChecker.checkManifestPlatform(buildContext, containerConfig);\n    }\n    return Optional.of(\n        JsonToImageTranslator.toImage((BuildableManifestTemplate) manifest, containerConfig));\n  }\n}\n\n// Path: jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/TempDirectoryProvider.java\n/*\n * Copyright 2019 Google LLC.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy of\n * the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.google.cloud.tools.jib.filesystem;\n\nimport com.google.common.io.MoreFiles;\nimport com.google.common.io.RecursiveDeleteOption;\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.Set;\n\n/** Creates temporary directories and deletes them all when closed. */\npublic class TempDirectoryProvider implements Closeable {\n\n  private final Set<Path> directories = Collections.synchronizedSet(new HashSet<>());\n\n  /**\n   * Creates a new temporary directory.\n   *\n   * @return the path to the temporary directory\n   * @throws IOException if creating the directory fails\n   */\n  public Path newDirectory() throws IOException {\n    Path path = Files.createTempDirectory(null);\n    directories.add(path);\n    return path;\n  }\n\n  /**\n   * Creates a new temporary directory.\n   *\n   * @param parentDirectory the directory to create the temp directory inside\n   * @return the path to the temporary directory\n   * @throws IOException if creating the directory fails\n   */\n  public Path newDirectory(Path parentDirectory) throws IOException {\n    Path path = Files.createTempDirectory(parentDirectory, null);\n    directories.add(path);\n    return path;\n  }\n\n  @Override\n  public void close() {\n    for (Path path : directories) {\n      if (Files.exists(path)) {\n        try {\n          MoreFiles.deleteRecursively(path, RecursiveDeleteOption.ALLOW_INSECURE);\n        } catch (IOException ignored) {\n          // ignored\n        }\n      }\n    }\n    directories.clear();\n  }\n...\n", "needle_token_start": 15053, "needle_token_end": 15352, "code_context_ntokens": 16002, "output": ["```java\nprivate Optional<Image> getBaseImageIfAllLayersCached(\n      ManifestAndConfigTemplate manifestAndConfig, boolean isSingleManifest)\n      throws BadContainerConfigurationFormatException, PlatformNotFoundInBaseImageException,\n          LayerCountMismatchException {\n    Cache baseImageLayersCache = buildContext.getBaseImageLayersCache();\n    ManifestTemplate manifest = Verify.verifyNotNull(manifestAndConfig.getManifest());\n\n    // Verify all layers described in manifest are present in cache\n    if (!baseImageLayersCache.areAllLayersCached(manifest)) {\n      return Optional.empty();\n    }\n    if (manifest instanceof V21ManifestTemplate) {\n      return Optional.of(JsonToImageTranslator.toImage((V21ManifestTemplate) manifest));\n    }\n    ContainerConfigurationTemplate containerConfig =\n        Verify.verifyNotNull(manifestAndConfig.getConfig());\n    if (isSingleManifest) {\n      // If base image is not a manifest list, check and warn misconfigured platforms.\n      PlatformChecker.checkManifestPlatform(buildContext, containerConfig);\n    }\n    return Optional.of(\n        JsonToImageTranslator.toImage((BuildableManifestTemplate) manifest, containerConfig));\n  }\n```"]}
{"repo": "microsoft/gctoolkit", "name": "freeFloatingYoungBlock", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedG1GCParser.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to process a specific type of garbage collection log entry, determining whether it pertains to a young generation or mixed generation garbage collection, and whether it includes an initial mark phase. It then creates an appropriate object representing this event.\n2. **Input**: The function takes two parameters: a trace object that provides methods to extract and interpret data from a log entry, and a string representing the line of the log being processed.\n3. **Output**: There is no direct return value; instead, the function modifies a global or external reference to store details about the garbage collection event.\n4. **Procedure**: The function first checks if the log entry includes an 'initial-mark' phase. It then checks if the entry is for a 'young' or 'mixed' garbage collection. Based on these checks, it creates and assigns an appropriate object to represent the garbage collection event, including details like the timestamp, cause, and pause time extracted from the log entry.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    }\n\n    //113.325: [Full GC (System.gc()) 37M->32M(96M), 0.0943030 secs]\n    private void g1gcFullGC(GCLogTrace trace, String line) {\n        if (trace.gcCause() == GCCause.JAVA_LANG_SYSTEM)\n            forwardReference = new G1SystemGC(trace.getDateTimeStamp(), trace.getPauseTime());\n        else\n            forwardReference = new G1FullGCNES(trace.getDateTimeStamp(), trace.gcCause(), trace.getPauseTime());\n        forwardReference.addMemorySummary(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(4));\n        // not thrilled about this \"hack\".... but currently no way to differentiate between a full with details and a full withoutgit c\n        if ((diary != null) && (!diary.isPrintGCDetails()))\n            publish(forwardReference);\n    }\n\n    private boolean recordConcurrentPhase(String phaseName, double duration) {\n        //Order sensitive operations. First completed the concurrent phase.\n        switch (phaseName) {\n            default:\n                return false;\n            case \"root-region-scan\":\n                publish(new ConcurrentScanRootRegion(concurrentPhaseStartTimeStamp, duration));\n            case \"mark\":\n                publish(new G1ConcurrentMark(concurrentPhaseStartTimeStamp, duration));\n            case \"cleanup\":\n                publish(new G1ConcurrentCleanup(concurrentPhaseStartTimeStamp, duration));\n        }\n        return true;\n    }\n\n    private void youngWithConcurrentEnd(GCLogTrace trace, String line) {\n        if (!recordConcurrentPhase(trace.getGroup(8), trace.getDuration()))\n            trace.notYetImplemented();\n        //recording has cleaned up all the forward references\n        gcCauseForwardReference = trace.gcCause(3, 0);\n        timeStampForwardReference = getClock();\n        if (\"young\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.Young;\n        else if (\"mixed\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.Mixed;\n        else\n            trace.notYetImplemented();\n    }\n\n    private void youngSplitAtDatestamp(GCLogTrace trace, String line) {\n        timeStampForwardReference = new DateTimeStamp(trace.getGroup(1), trace.getDoubleGroup(3));\n    }\n\n    private void youngSplitAtTimestamp(GCLogTrace trace, String line) {\n        timeStampForwardReference = new DateTimeStamp(trace.getGroup(1));\n        if (!recordConcurrentPhase(trace.getGroup(8), trace.getDuration()))\n            trace.notYetImplemented();\n    }\n\n    \nprivate void freeFloatingYoungBlock(GCLogTrace trace, String line) {\n        boolean initialMark = false;\n        if (trace.getGroup(3) != null)\n            initialMark = trace.contains(3, \"initial-mark\");\n        if (trace.contains(2, \"young\"))\n            if (initialMark)\n                forwardReference = new G1YoungInitialMark(trace.getDateTimeStamp(), trace.gcCause(3, -2), trace.getPauseTime());\n            else\n                forwardReference = new G1Young(trace.getDateTimeStamp(), trace.gcCause(3, -2), trace.getPauseTime());\n        else if (trace.getGroup(2).contains(\"mixed\")) {\n            forwardReference = new G1Mixed(trace.getDateTimeStamp(), trace.gcCause(3, -2), trace.getPauseTime());\n        }\n    }\n\n    private void fullGCWithConcurrentPhase(GCLogTrace trace, String line) {\n        timeStampForwardReference = getClock();\n        gcCauseForwardReference = trace.gcCause();\n        collectionTypeForwardReference = GarbageCollectionTypes.Full;\n        concurrentPhaseStartTimeStamp = trace.getDateTimeStamp(2);\n    }\n\n    private void g1YoungSplitEnd(GCLogTrace trace, String line) {\n        if (collectionTypeForwardReference == GarbageCollectionTypes.Young) {\n            forwardReference = new G1Young(timeStampForwardReference, gcCauseForwardReference, trace.getPauseTime());\n        } else if (collectionTypeForwardReference == GarbageCollectionTypes.Mixed) {\n            forwardReference = new G1Mixed(timeStampForwardReference, gcCauseForwardReference, trace.getPauseTime());\n        } else if (collectionTypeForwardReference == GarbageCollectionTypes.G1GCYoungInitialMark) {\n            forwardReference = new G1YoungInitialMark(timeStampForwardReference, gcCauseForwardReference, trace.getDoubleGroup(trace.groupCount()));\n        } else {\n            LOGGER.log(Level.WARNING, trace.toString());\n            return;\n        }\n\n        if (referenceGCForwardReferenceSummary != null) {\n            forwardReference.add(referenceGCForwardReferenceSummary);\n        }\n\n        if (trace.contains(1, \"to-space\")) {\n            ((G1Young) forwardReference).toSpaceExhausted();\n        }\n    }\n\n    private void g1InitialMark(GCLogTrace trace, String line) {\n        timeStampForwardReference = getClock();\n        gcCauseForwardReference = trace.gcCause();\n        if (\"young\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.G1GCYoungInitialMark;\n        else if (trace.contains(4, \"mixed\"))\n            collectionTypeForwardReference = GarbageCollectionTypes.G1GCMixedInitialMark;\n        else\n            trace.notYetImplemented();\n    }\n\n    //17M->3758K(13M), 0.0235300 secs]\n    //todo: this corrects a corrupted Full GC (System.gc()) record but it also can be found with other\n    //record interruptions\n    private void freeFloatingOccupancySummary(GCLogTrace trace, String line) {\n        if (collectionTypeForwardReference == GarbageCollectionTypes.Full) {\n            forwardReference = new G1FullGCNES(timeStampForwardReference, gcCauseForwardReference, trace.getPauseTime());\n            publish(forwardReference);\n            collectionTypeForwardReference = GarbageCollectionTypes.Unknown;\n            timeStampForwardReference = null;\n            gcCauseForwardReference = GCCause.UNKNOWN_GCCAUSE;\n        } else if (collectionTypeForwardReference == GarbageCollectionTypes.SystemGC) {\n            forwardReference = new G1SystemGC(timeStampForwardReference, trace.getPauseTime());\n            collectionTypeForwardReference = GarbageCollectionTypes.Unknown;\n            timeStampForwardReference = null;\n            gcCauseForwardReference = GCCause.UNKNOWN_GCCAUSE;\n        } else\n            trace.notYetImplemented();\n    }\n\n    //549.246: [GC concurrent-root-region-scan-start]\n    //549.246: [GC concurrent-mark-start]\n    private void g1ConcurrentStartWithoutPrefix(GCLogTrace trace, String line) {\n        concurrentPhaseStartTimeStamp = getClock();\n    }\n\n    private void g1ConcurrentStart(GCLogTrace trace, String line) {\n        concurrentPhaseStartTimeStamp = trace.getDateTimeStamp();\n    }\n\n    private void concurrentStringDedup(GCLogTrace trace, String line) {\n        double startingStringVolume = trace.toKBytes(4);\n        double endingStringValue = trace.toKBytes(6);\n        double reduction = trace.toKBytes(8);\n        double percentReduction = trace.getDoubleGroup(10);\n        publish(new G1ConcurrentStringDeduplication(getClock(), trace.gcCause(), startingStringVolume, endingStringValue, reduction, percentReduction, trace.getDoubleGroup(trace.groupCount())));\n    }\n\n    //9.251: [GC remark, 0.0012190 secs]\n    //6.298: [GC remark 6.298: [GC ref-proc, 0.0000570 secs], 0.0010940 secs]\n    //2014-02-21T16:04:24.321-0100: 7.852: [GC remark 2014-02-21T16:04:24.322-0100: 7.853: [GC ref-proc, 0.0000640 secs], 0.0013310 secs]\n    private void g1Remark(GCLogTrace trace, String line) {\n        G1Remark remark = new G1Remark(trace.getDateTimeStamp(), (trace.getGroup(7) != null) ? trace.getDoubleGroup(7) : 0.0d, trace.getDoubleGroup(trace.groupCount()));\n        publish(remark);\n    }\n\n    //2015-04-09T14:28:44.235+0100: 6.597: [GC remark 6.597: [Finalize Marking, 0.0091510 secs] 6.606: [GC ref-proc, 0.0014102 secs] 6.608: [Unloading, 0.0044869 secs], 0.0153351 secs]\n    //public G1Remark( DateTimeStamp timeStamp, double referenceProcessingTimes, double finalizeMarking, double unloading, double duration)\n    private void g1180Remark(GCLogTrace trace, String line) {\n        G1Remark remark = new G1Remark(trace.getDateTimeStamp(), trace.getDoubleGroup(8), trace.getDoubleGroup(5), trace.getDoubleGroup(11), trace.getDoubleGroup(trace.groupCount()));\n        publish(remark);\n    }\n\n    private void g1180RemarkRefDetails(GCLogTrace trace, String line) {\n        G1Remark remark = new G1Remark(trace.getDateTimeStamp(), trace.getDoubleGroup(32), trace.getDoubleGroup(5), trace.getDoubleGroup(trace.groupCount() - 1), trace.getDuration());\n        ReferenceGCSummary summary = extractPrintReferenceGC(line);\n        remark.add(summary);\n        publish(remark);\n    }\n\n    //549.253: [GC cleanup 7826K->7826K(13M), 0.0004750 secs]\n    //todo: capture memory summary\n    private void g1Cleanup(GCLogTrace trace, String line) {\n        G1Cleanup cleanup = new G1Cleanup(trace.getDateTimeStamp(), trace.getPauseTime());\n        cleanup.addMemorySummary(getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 4));\n        publish(cleanup);\n    }\n\n    private void g1CleanupNoMemory(GCLogTrace trace, String line) {\n        G1Cleanup cleanup = new G1Cleanup(trace.getDateTimeStamp(), trace.getPauseTime());\n        publish(cleanup);\n    }\n\n    private void splitCleanup(GCLogTrace trace, String line) {\n        timeStampForwardReference = getClock();\n        collectionTypeForwardReference = GarbageCollectionTypes.G1GCCleanup;\n    }\n\n    //511.628: [GC concurrent-mark-abort]\n    private void g1ConcurrentAbort(GCLogTrace trace, String line) {\n        if (concurrentPhaseStartTimeStamp != null) {\n            G1ConcurrentMark concurrentMark = new G1ConcurrentMark(concurrentPhaseStartTimeStamp, trace.gcCause(), trace.getTimeStamp() - concurrentPhaseStartTimeStamp.getTimeStamp());\n            concurrentMark.abort();\n            publish(concurrentMark);\n        }\n    }\n\n    //604.395: [GC concurrent-mark-reset-for-overflow]\n    private void concurrentMarkOverflow(GCLogTrace trace, String line) {\n        publish(new G1ConcurrentMarkResetForOverflow(getClock()));\n    }\n\n    private void g1CorruptedConcurrentEnd(GCLogTrace trace, String line) {\n        //In this case a corruption stripped the timestamp away from the actual record\n        //Fortunately it's a rare event.\n        if (concurrentPhaseStartTimeStamp == null)\n            concurrentPhaseStartTimeStamp = getClock();\n\n        if (\"root-region-scan\".equals(trace.getGroup(1))) {\n            publish(new ConcurrentScanRootRegion(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())));\n        } else if (\"mark\".equals(trace.getGroup(1))) {\n            publish(new G1ConcurrentMark(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())));\n        } else if (\"cleanup\".equals(trace.getGroup(1))) {\n            publish(new G1ConcurrentCleanup(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getDoubleGroup(trace.groupCount())));\n        } else\n            trace.notYetImplemented();\n    }\n\n    public void g1ConcurrentEndCorruptedByApplicationTime(GCLogTrace trace, String line) {\n        if (concurrentPhaseStartTimeStamp == null)\n            concurrentPhaseStartTimeStamp = getClock();\n        if (\"root-region-scan\".equals(trace.getGroup(4)))\n            publish(new ConcurrentScanRootRegion(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getPauseTime()));\n        else if (\"mark\".equals(trace.getGroup(4)))\n            publish(new G1ConcurrentMark(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getPauseTime()));\n        else if (\"cleanup\".equals(trace.getGroup(4)))\n            publish(new G1ConcurrentCleanup(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getPauseTime()));\n    }\n\n    public void g1FloatingConcurrentPhaseStart(GCLogTrace trace, String line) {\n        concurrentPhaseStartTimeStamp = getClock();\n    }\n\n    /***********************************/\n    /* Reference processing            */\n    /***********************************/\n\n\n    private void g1DetailsReferenceGC(GCLogTrace trace, String line) {\n        processYoungGenCollection(trace, line);\n        forwardReference.add(extractPrintReferenceGC(line));\n        if (trace.contains(trace.groupCount() - 2, \"to-space\"))\n            ((G1Young) forwardReference).toSpaceExhausted();\n    }\n\n    private void g1RemarkReferenceGC(GCLogTrace trace, String line) {\n        ReferenceGCSummary summary = extractPrintReferenceGC(line);\n        double totalReferenceProcessingTime = summary.getSoftReferencePauseTime() + summary.getWeakReferencePauseTime() + summary.getFinalReferencePauseTime() + summary.getPhantomReferencePauseTime() + summary.getJniWeakReferencePauseTime();\n        G1Remark remark = new G1Remark(getClock(), totalReferenceProcessingTime, trace.getPauseTime());\n        remark.add(summary);\n        publish(remark);\n    }\n\n\n    //11907.092: [Full GC11907.961: [SoftReference, 1403 refs, 0.0002110 secs]11907.962: [WeakReference, 31981 refs, 0.0026320 secs]11907.964: [FinalReference, 411 refs, 0.0000720 secs]11907.964: [PhantomReference, 491 refs, 0.0000400 secs]11907.964: [JNI Weak Reference, 0.0001960 secs] 11208M->1149M(12000M), 4.7476800 secs]\n    private void g1FullDetailsReferenceGC(GCLogTrace trace, String line) {\n        forwardReference = new G1FullGCNES(getClock(), trace.gcCause(), trace.getPauseTime());\n        forwardReference.add(extractPrintReferenceGC(line));\n        if ((diary != null) && (isPreJDK17040())) {\n            MemoryPoolSummary heap = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(27);\n            forwardReference.addMemorySummary(heap);\n            publish(forwardReference);\n        }\n    }\n\n\n    //2014-10-21T11:47:46.879-0500: 11971.476: [GC pause (young)11972.105: [SoftReference, 0 refs, 0.0000050 secs]11972.105: [WeakReference, 1 refs, 0.0000020 secs]11972.105: [FinalReference, 94 refs, 0.0000360 secs]11972.105: [PhantomReference, 5 refs, 0.0000030 secs]11972.105: [JNI Weak Reference, 0.0002010 secs] (initial-mark), 0.63448400 secs]\n    private void g1DetailsReferenceInitialMark(GCLogTrace trace, String line) {\n        forwardReference = new G1YoungInitialMark(getClock(), trace.gcCause(), trace.getPauseTime());\n        ReferenceGCSummary summary = extractPrintReferenceGC(line);\n        forwardReference.add(summary);\n        if (trace.contains(24, \"to-space \"))\n            ((G1YoungInitialMark) forwardReference).toSpaceExhausted();\n    }\n\n    private void freeFloatingReferenceRecords(GCLogTrace trace, String line) {\n        if (collectionTypeForwardReference == GarbageCollectionTypes.Young) {\n            forwardReference = new G1Young(timeStampForwardReference, gcCauseForwardReference, trace.getPauseTime());\n        } else if (collectionTypeForwardReference == GarbageCollectionTypes.Mixed) {\n            forwardReference = new G1Mixed(timeStampForwardReference, gcCauseForwardReference, trace.getPauseTime());\n        } else if (collectionTypeForwardReference == GarbageCollectionTypes.G1GCYoungInitialMark) {\n            forwardReference = new G1YoungInitialMark(timeStampForwardReference, gcCauseForwardReference, trace.getPauseTime());\n        }\n        forwardReference.add(extractPrintReferenceGC(line));\n        if (trace.contains(\"to-space\")) { //todo: where this shows up in this record is current unknown.\n            ((G1Young) forwardReference).toSpaceExhausted();\n        }\n    }\n\n    private void floatingReferenceWithAdaptiveSizing(GCLogTrace trace, String line) {\n        referenceGCForwardReferenceSummary = extractPrintReferenceGC(line);\n    }\n\n    //todo: phase durations are all wrong.... I don't have a good feeling about editing this method...\n    private void youngReferenceWIthConcurrentEnd(GCLogTrace trace, String line) {\n        //Order sensitive operations. First completed the concurrent phase.\n        String concurrentPhase = trace.getGroup(32);\n        if (\"root-region-scan\".equals(concurrentPhase)) {\n            publish(new ConcurrentScanRootRegion(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getPauseTime()));\n        } else if (\"mark\".equals(concurrentPhase)) {\n            publish(new G1ConcurrentMark(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getPauseTime()));\n        } else if (\"cleanup\".equals(concurrentPhase)) {\n            publish(new G1ConcurrentCleanup(concurrentPhaseStartTimeStamp, GCCause.UNKNOWN_GCCAUSE, trace.getPauseTime()));\n        } else\n            trace.notYetImplemented();\n\n        //recording has cleaned up all the forward references but the object can be created here as the pause time is known.\n        timeStampForwardReference = trace.getDateTimeStamp();\n        gcCauseForwardReference = trace.gcCause();\n        if (\"young\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.Young;\n        else if (\"mixed\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.Mixed;\n        else\n            trace.notYetImplemented();\n        referenceGCForwardReferenceSummary = extractPrintReferenceGC(line);\n    }\n\n    private void g1FullInterruptsConcurrentWithReferences(GCLogTrace trace, String line) {\n        String concurrentPhase = trace.getGroup(30);  //25 was here before now with Atlassian it wants to be 30.\n        if (\"root-region-scan\".equals(concurrentPhase)) {\n            publish(new ConcurrentScanRootRegion(concurrentPhaseStartTimeStamp, trace.gcCause(22), trace.getPauseTime()));\n        } else if (\"mark\".equals(concurrentPhase)) {\n            publish(new G1ConcurrentMark(concurrentPhaseStartTimeStamp, trace.gcCause(22), trace.getPauseTime()));\n        } else if (\"cleanup\".equals(concurrentPhase)) {\n            publish(new G1ConcurrentCleanup(concurrentPhaseStartTimeStamp, trace.gcCause(22), trace.getPauseTime()));\n        } else\n            trace.notYetImplemented();\n\n        collectionTypeForwardReference = GarbageCollectionTypes.Full;\n        gcCauseForwardReference = trace.gcCause();\n        timeStampForwardReference = trace.getDateTimeStamp();\n        referenceGCForwardReferenceSummary = extractPrintReferenceGC(line);\n    }\n\n    private void g1FullMemorySplitByConcurrent(GCLogTrace trace, String line) {\n        if (collectionTypeForwardReference == GarbageCollectionTypes.Full) {\n            G1FullGCNES full = new G1FullGCNES(timeStampForwardReference, gcCauseForwardReference, trace.getPauseTime());\n            full.addMemorySummary(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(1));\n            publish(full);\n        } else\n            trace.notYetImplemented();\n    }\n\n    /***********************************/\n    /* Adaptive Sizing                 */\n\n    /***********************************/\n\n    private void youngSplitByG1Ergonomics(GCLogTrace trace, String line) {\n        timeStampForwardReference = getClock();\n        gcCauseForwardReference = trace.gcCause(3, 0);\n        if (\"young\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.Young;\n        else if (\"mixed\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.Mixed;\n        else\n            trace.notYetImplemented();\n    }\n\n    private void g1InitialMarkErgonomics(GCLogTrace trace, String line) {\n        timeStampForwardReference = getClock();\n        gcCauseForwardReference = trace.gcCause(3, 0);\n        if (\"young\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.G1GCYoungInitialMark;\n        else if (\"mixed\".equals(trace.getGroup(4)))\n            collectionTypeForwardReference = GarbageCollectionTypes.G1GCYoungInitialMark;\n        else\n            trace.notYetImplemented();\n    }\n\n    /***********************************/\n    /* Heap Summary                    */\n\n    /***********************************/\n\n    //We have found this section. If PrintHeapAtGC does not interfere\n    // this method should consume all remaining lines\n    private void garbageFirstHeap(GCLogTrace trace, String line) {\n        //Swallow log header that we don't do anything with\n    }\n\n    private void regionSize(GCLogTrace trace, String line) {\n        regionSize = trace.getLongGroup(1);\n    }\n\n    private void metaspaceFinal(GCLogTrace trace, String line) {\n        metaSpaceUsed = trace.toKBytes(1);\n        metaCapacity = trace.toKBytes(3);\n        metaCommitted = trace.toKBytes(5);\n        metaReserved = trace.toKBytes(7);\n    }\n\n    private void classspaceFinal(GCLogTrace trace, String line) {\n        classSpaceUsed = trace.toKBytes(1);\n        classSpaceCapacity = trace.toKBytes(3);\n        classSpaceCommitted = trace.toKBytes(5);\n        classSpaceReserved = trace.toKBytes(7);\n    }\n\n    /***********************************/\n    /* Pre 170_40 rules                */\n    /* Very old and should not be used */\n\n    /***********************************/\n\n    @SuppressWarnings(\"unused\")\n    private void g1Pre17040Summary(GCLogTrace trace, String line) {\n        try {\n            MemoryPoolSummary edenSummary = extractPoolSummary(trace, 1);\n            MemoryPoolSummary heap = extractPoolSummary(trace, 13);\n            SurvivorMemoryPoolSummary survivor = extractSurvivorPoolSummary(trace, 9);\n            forwardReference.addMemorySummary(edenSummary, survivor, heap);\n            publish(forwardReference);\n        } catch (Throwable t) {\n            t.printStackTrace();\n        }\n    }\n\n    /*************************/\n    /* Support Methods       */\n\n    /*************************/\n\n    private void g1ConcurrentEnd(GCLogTrace trace, String line) {\n        //If concurrentPhaseStartTimeStamp is null, something went wrong in the parsing.\n        // Use an estimate of the start time of the phase or suffer\n        // a worse fate later on. Generally the cause is log file corruption.\n        if (concurrentPhaseStartTimeStamp == null)\n            concurrentPhaseStartTimeStamp = getClock();\n\n        if (\"root-region-scan\".equals(trace.getGroup(4))) {\n            publish(new ConcurrentScanRootRegion(concurrentPhaseStartTimeStamp, trace.getDuration()));\n        } else if (\"mark\".equals(trace.getGroup(4))) {\n            publish(new G1ConcurrentMark(concurrentPhaseStartTimeStamp, trace.getDuration()));\n        } else if (\"cleanup\".equals(trace.getGroup(4))) {\n            publish(new G1ConcurrentCleanup(concurrentPhaseStartTimeStamp, trace.getDuration()));\n        } else\n            trace.notYetImplemented();\n    }\n\n\n    //Min: \" + TIME + \", Avg: \" + TIME + \", Max: \" + TIME + \", Diff: \" + TIME\n    private StatisticalSummary extractStatisticalSummary(GCLogTrace trace, int offset) {\n        return new StatisticalSummary(trace.getDoubleGroup(1 + offset),\n                trace.getDoubleGroup(2 + offset),\n                trace.getDoubleGroup(3 + offset),\n                trace.getDoubleGroup(4 + offset),\n                StatisticalSummary.UNDEFINED);\n    }\n\n    //Min: \" + TIME + \", Avg: \" + TIME + \", Max: \" + TIME + \", Diff: \" + TIME + \", Sum: \" + REAL_NUMBER;\n    private StatisticalSummary extractStatisticalSummaryWithSum(GCLogTrace trace, int offset) {\n        return new StatisticalSummary(trace.getDoubleGroup(1 + offset),\n                trace.getDoubleGroup(2 + offset),\n                trace.getDoubleGroup(3 + offset),\n                trace.getDoubleGroup(4 + offset),\n                trace.getDoubleGroup(5 + offset));\n    }\n\n    private StatisticalSummary extractCounterSummary(GCLogTrace trace, int offset) {\n        return new StatisticalSummary(trace.getIntegerGroup(offset),\n                trace.getDoubleGroup(1 + offset),\n                trace.getIntegerGroup(2 + offset),\n                trace.getIntegerGroup(3 + offset),\n                trace.getIntegerGroup(4 + offset));\n    }\n\n    private MemoryPoolSummary extractPoolSummary(GCLogTrace trace, int offset) {\n        long occupancyBefore = trace.doubleToKBytes(offset);\n        long sizeBefore = trace.doubleToKBytes(offset + 2);\n        long occupancyAfter = trace.doubleToKBytes(offset + 4);\n        long size = trace.doubleToKBytes(offset + 6);\n        return new MemoryPoolSummary(occupancyBefore, sizeBefore, occupancyAfter, size);\n    }\n\n    private SurvivorMemoryPoolSummary extractSurvivorPoolSummary(GCLogTrace trace, int offset) {\n        long occupancyBefore = trace.doubleToKBytes(offset);\n        long occupancyAfter = trace.doubleToKBytes(offset + 2);\n        return new SurvivorMemoryPoolSummary(occupancyBefore, occupancyAfter);\n    }\n\n    /**\n     * Maintain time ordering of records. Concurrent phases are recorded at start time and thus\n     * other phases may mix in.\n     */\n    private void drainBacklog() {\n        if (backlog.size() > 0)\n            for (JVMEvent event : backlog)\n                super.publish(ChannelName.G1GC_PARSER_OUTBOX,event);\n    }\n\n    public void publish(G1GCConcurrentEvent concurrentEvent) {\n        super.publish(ChannelName.G1GC_PARSER_OUTBOX,concurrentEvent);\n        drainBacklog();\n    }\n\n    private void recordCPUSummary(GCLogTrace trace, String line) {\n        forwardReference.addCPUSummary(new CPUSummary(trace.getDoubleGroup(1), trace.getDoubleGroup(2), trace.getDoubleGroup(3)));\n        publish(forwardReference);\n    }\n\n    public void publish(G1GCPauseEvent collection) {\n\n        if (collection == trap) {\n            LOGGER.warning(\"Parsing Error: Attempt to record Trap @\" + getClock().getTimeStamp());\n            return;\n        }\n\n        // wait for the CPU summary\n        if ((collection.getCpuSummary() == null) && (diary == null || diary.isPrintGCDetails())) {\n            forwardReference = collection;\n        } else {\n            super.publish(ChannelName.G1GC_PARSER_OUTBOX, collection);\n            forwardReference = trap;\n            collectionTypeForwardReference = null;\n            referenceGCForwardReferenceSummary = null;\n        }\n    }\n\n    private void log(String line) {\n\n        if (line.startsWith(\"Java HotSpot(TM)\")) return;\n        if (line.startsWith(\"Memory: \")) return;\n        if (line.startsWith(\"CommandLine flags: \")) return;\n\n        GCToolKit.LOG_DEBUG_MESSAGE(() -> \"Missed: \" + line);\n        LOGGER.log(Level.FINE, \"Missed: {0}\", line);\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return diary.isG1GC() && ! diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedJVMEventParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.ApplicationConcurrentTime;\nimport com.microsoft.gctoolkit.event.jvm.ApplicationStoppedTime;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.Set;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.JVM_EXIT;\n\npublic class UnifiedJVMEventParser extends UnifiedGCLogParser implements JVMPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(UnifiedJVMEventParser.class.getName());\n    private DateTimeStamp timeStamp = new DateTimeStamp(0.0d);\n    private ApplicationStoppedTime.VMOperations safePointReason = null;\n    private boolean gcPause = false;\n\n    public UnifiedJVMEventParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.JVM);\n    }\n\n    public String getName() {\n        return \"JavaEventParser\";\n    }\n\n    @Override\n    protected void process(String line) {\n\n        GCLogTrace trace = null;\n\n        try {\n\n            if ((trace = UNIFIED_LOGGING_APPLICATION_STOP_TIME_WITH_STOPPING_TIME.parse(line)) != null) {\n                if (safePointReason != null)\n                    publish(new ApplicationStoppedTime(timeStamp, trace.getDoubleGroup(1), trace.getDoubleGroup(2), safePointReason));\n                else\n                    publish(new ApplicationStoppedTime(timeStamp, trace.getDoubleGroup(1), trace.getDoubleGroup(2), gcPause));\n                safePointReason = null;\n                gcPause = false;\n            } else if (GC_PAUSE_CLAUSE.parse(line) != null) {\n                gcPause = true;\n            } else if ((trace = SAFEPOINT_REGION.parse(line)) != null) {\n                timeStamp = getClock();\n                safePointReason = ApplicationStoppedTime.VMOperations.valueOf(trace.getGroup(1));\n            } else if ((trace = LEAVING_SAFEPOINT.parse(line)) != null) {\n            } //noop this one.\n\n            else if ((trace = UNIFIED_LOGGING_APPLICATION_TIME.parse(line)) != null) {\n                publish(new ApplicationConcurrentTime(getClock(), trace.getDoubleGroup(1)));\n            } else if (line.equals(END_OF_DATA_SENTINEL) || (JVM_EXIT.parse(line) != null)) {\n                publish(new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n            } else if (getClock().getTimeStamp() > timeStamp.getTimeStamp()) {\n                if (isGCPause(line)) gcPause = true;\n                timeStamp = getClock();\n            }\n\n        } catch (Throwable t) {\n            LOGGER.log(Level.FINE, \"Missed: {0}\", line);\n        }\n    }\n\n    private boolean isGCPause(String line) {\n        return ((line.contains(\" Pause Initial Mark\")) ||\n                (line.contains(\" Remark \")) ||\n                (line.contains(\" Pause Young \")) ||\n                (line.contains(\" Full \")));\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return (diary.isApplicationStoppedTime() || diary.isApplicationRunningTime()) && diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n\n    private void publish(JVMEvent event) {\n        super.publish(ChannelName.JVM_EVENT_PARSER_OUTBOX, event);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/ICMSPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\n/**\n * Patterns for the incremental Concurrent Mark and Sweep (iCMS) collector\n */\npublic interface ICMSPatterns extends CMSPatterns {\n\n    String ICMS_DC = \" icms_dc=(\\\\d+) \";\n\n    /***** iCMS records ******/\n    //Greedy: 263204.684: [GC263204.684: [ParNew (promotion failed): 153342K->153343K(153344K), 0.0910130 secs]263204.776: [CMS: 1752518K->1636004K(1926784K), 8.9443330 secs] 1889511K->1636004K(2080128K), [CMS Perm : 137004K->135442K(233916K)] icms_dc=75 , 9.0355990 secs]\n    //Greedy: 410971.012: [GC 410971.012: [ParNew (promotion failed): 153068K->153068K(153344K), 0.1683040 secs]410971.180: [CMS: 1414041K->1301153K(1926784K), 7.3235580 secs] 1543745K->1301153K(2080128K), [CMS Perm : 137698K->135751K(235644K)] icms_dc=19 , 7.4921120 secs]\n    //Missed: 357305.590: [GC 357305.590: [ParNew (promotion failed): 10973K->3147K(153344K), 0.0295730 secs] 1763339K->1763817K(2080128K) icms_dc=32 , 0.0297680 secs]\n\n    //9.432: [GC 9.432: [ParNew: 145455K->17024K(153344K), 0.0183600 secs] 147142K->18720K(2080128K) icms_dc=5 , 0.0184240 secs]\n    //0.042: [GC 0.042: [ParNew: 153313K->17024K(153344K), 0.1259050 secs] 1431172K->1334562K(2080128K) icms_dc=23 , 0.1261340 secs]\n    //2015-07-02T10:50:17.491+0100: 494057.529: [GC2015-07-02T10:50:17.491+0100: 494057.530: [ParNew: 553019K->9143K(613440K), 0.1222170 secs] 10355626K->9812666K(16709120K) icms_dc=0 , 0.1226510 secs] [Times: user=0.83 sys=0.01, real=0.12 secs]\n    //2015-06-26T17:40:28.063+0100: 268.102: [GC2015-06-26T17:40:28.064+0100: 268.102: [ParNew: 547627K->28112K(613440K), 0.0768200 secs] 547627K->28112K(16709120K) icms_dc=5 , 0.0771220 secs] [Times: user=0.40 sys=0.07, real=0.08 secs]\n    GCParseRule iCMS_PARNEW = new GCParseRule(\"iCMS_PARNEW\", GC_PREFIX + PARNEW_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + ICMS_DC + \", \" + PAUSE_TIME);\n\n    //2016-08-17T10:41:41.088-0500: 76388.020: [GC (Allocation Failure) 2016-08-17T10:41:41.088-0500: 76388.020: [ParNew (promotion failed): 428321K->431492K(471872K), 0.1049894 secs]2016-08-17T10:41:41.193-0500: 76388.125: [CMS: 466697K->186596K(524288K), 0.7357085 secs] 893715K->186596K(996160K), [Metaspace: 46524K->46524K(1097728K)] icms_dc=0 , 0.8408171 secs] [Times: user=0.91 sys=0.00, real=0.84 secs]\n    GCParseRule iCMS_PARNEW_PROMOTION_FAILURE = new GCParseRule(\"iCMS_PARNEW_PROMOTION_FAILURE\", GC_PREFIX + PARNEW_PROMOTION_FAILED_BLOCK + CMS_TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n    GCParseRule iCMS_PARNEW_PROMOTION_FAILURE_RECORD = new GCParseRule(\"iCMS_PARNEW_PROMOTION_FAILURE_RECORD\", GC_PREFIX + PARNEW_WITH_PROMOTION_FAILURE_SIZE_BLOCK + CMS_TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n\n    //: 81792K->0K(81856K), 0.0431036 secs] 90187K->11671K(2097088K) icms_dc=5 , 0.0435503 secs]\n    GCParseRule iCMS_PARNEW_DEFNEW_TENURING_DETAILS = new GCParseRule(\"iCMS_PARNEW_DEFNEW_TENURING_DETAILS\", \"^: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    /* Full collections */\n\n    //2013-06-06T14:12:49.554+0200: 534744,148: [Full GC (System) 534744,148: [CMS: 1513767K->410320K(3512768K), 2,1361260 secs] 1598422K->410320K(4126208K), [CMS Perm : 120074K->119963K(200424K)] icms_dc=0 , 2,1363550 secs]\n    //619930,816: [Full GC 619930,816: [CMS: 3512768K->3512767K(3512768K), 17,8327610 secs] 4126207K->4073974K(4126208K), [CMS Perm : 120062K->120053K(201192K)] icms_dc=100 , 17,8329750 secs]\n    GCParseRule iCMS_FULL = new GCParseRule(\"iCMS_FULL\", FULL_GC_PREFIX + DATE_TIMESTAMP + \"\\\\[CMS: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //85.577: [Full GC 85.577: [CMS: 5K->1883K(1926784K), 9.8521080 secs] 20369K->1883K(2080128K), [CMS Perm : 13770K->13604K(22970K)] icms_dc=100 , 9.8522900 secs]\n    //25.846: [Full GC 25.847: [CMS: 0K->7019K(2015232K), 5.6129510 secs] 11161K->7019K(2097088K), [CMS Perm : 68104K->67271K(13520K)] icms_dc=0 , 25.6146105 secs]\n    GCParseRule FULL_GC_ICMS = new GCParseRule(\"FULL_GC_ICMS\", FULL_GC_PREFIX + TIMESTAMP + \"\\\\[CMS: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //91057.643: [GC 91057.643: [ParNew: 153344K->153344K(153344K), 0.0000250 secs]91057.643: [CMS: 1895319K->1754894K(1926784K), 9.6910090 secs] 2048663K->1754894K(2080128K), [CMS Perm : 135228K->134975K(230652K)] icms_dc=65 , 9.6912470 secs]\n    GCParseRule iCMS_MISLABELED_FULL = new GCParseRule(\"iCMS_MISLABELED_FULL\", GC_PREFIX + PARNEW_BLOCK + CMS_TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n\n    /* promotion failures */\n\n    //(concurrent mode failure): 8465K->22006K(1926784K), 0.3222180 secs] 28136K->22006K(2080128K), [CMS Perm : 51702K->51627K(52016K)] icms_dc=5 , 0.3224000 secs]\n    //(concurrent mode failure): 1696350K->613432K(2015232K), 22.0030939 secs] 1772267K->613432K(2097088K) icms_dc=100 , 22.3467890 secs]\n    GCParseRule iCMS_CONCURRENT_MODE_FAILURE = new GCParseRule(\"iCMS_CONCURRENT_MODE_FAILURE\", \"^\\\\(concurrent mode failure\\\\): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \"(?:, \" + PERM_RECORD + \")?\" + ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n    GCParseRule iCMS_CONCURRENT_MODE_FAILURE_META = new GCParseRule(\"iCMS_CONCURRENT_MODE_FAILURE_META\", \"^\\\\(concurrent mode failure\\\\): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \"(?:, \" + META_RECORD + \")?\" + ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //(concurrent mode interrupted): 2286052K->845009K(3908584K), 6.1999933 secs] 2673522K->845009K(4671912K), [CMS Perm : 137744K->133435K(228656K)] icms_dc=0 , 6.2007727 secs] [Times: user=7.11 sys=0.02, real=6.20 secs]\n    GCParseRule iCMS_CONCURRENT_MODE_INTERRUPTED = new GCParseRule(\"iCMS_CONCURRENT_MODE_INTERRUPTED\", \"^\\\\(concurrent mode interrupted\\\\): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n\n    /*\n     * This record was produced *after* CMS-reset completed. Fragmentation failure????\n     * 16253.906: [GC 16253.906: [ParNew (promotion failed)\n     * Desired survivor size 32768 bytes, new threshold 0 (max 0)\n     * : 81792K->81792K(81856K), 0.2765545 secs]16254.183: [CMS (concurrent mode failure): 1552479K->933994K(2015232K), 22.9932464 secs] 1611199K->933994K(2097088K) icms_dc=57 , 23.2713493 secs]\n     */\n    GCParseRule iCMS_CMF_DUIRNG_PARNEW_DEFNEW_DETAILS = new GCParseRule(\"iCMS_CMF_DUIRNG_PARNEW_DEFNEW_DETAILS\", \": \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\" + DATE_TIMESTAMP + \"\\\\[CMS \\\\(concurrent mode failure\\\\): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //35305.590: [GC 35305.590: [ParNew (promotion failed): 10973K->3147K(153344K), 0.0295730 secs] 1763339K->1763817K(2080128K) icms_dc=32 , 0.0297680 secs]\n    GCParseRule iCMS_PROMOTION_FAILED = new GCParseRule(\"iCMS_PROMOTION_FAILED\", GC_PREFIX + TIMESTAMP + \"\\\\[ParNew \\\\(promotion failed\\\\): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\" + \" \" + BEFORE_AFTER_CONFIGURED + ICMS_DC + \", \" + PAUSE_TIME);\n\n    //72825.712: [GC72825.712: [ParNew (promotion failed): 153344K->153344K(153344K), 0.3895590 secs]72826.102: [CMS: 1831960K->1554399K(1926784K), 8.3796720 secs] 1914705K->1554399K(2080128K), [CMS Perm : 131704K->130876K(222768K)] icms_dc=42 , 8.7694530 secs]\n    GCParseRule iCMS_PROMOTION_FAILED_PERM = new GCParseRule(\"iCMS_PROMOTION_FAILED_PERM\", GC_PREFIX + PARNEW_PROMOTION_FAILED_BLOCK + CMS_TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n    //very unlikely to see this\n    GCParseRule iCMS_PROMOTION_FAILED_META = new GCParseRule(\"iCMS_PROMOTION_FAILED_META\", GC_PREFIX + PARNEW_PROMOTION_FAILED_BLOCK + CMS_TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + META_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n\n    //10110.232: [Full GC 10110.232: [CMS (concurrent mode failure): 2715839K->1659203K(2752512K), 39.3188254 secs] 2740830K->1659203K(3106432K), [CMS Perm : 206079K->204904K(402264K)] icms_dc=100 , 39.3199631 secs] [Times: user=39.37 sys=0.02, real=39.32 secs]\n    GCParseRule iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE = new GCParseRule(\"iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE\", FULL_GC_PREFIX + DATE_TIMESTAMP + \"\\\\[CMS \\\\(concurrent mode failure\\\\): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n    //Very highly unlikely that we'll ever see this record but since it's possible...\n    GCParseRule iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE_META = new GCParseRule(\"iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE_META\", FULL_GC_PREFIX + DATE_TIMESTAMP + \"\\\\[CMS \\\\(concurrent mode failure\\\\): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + META_RECORD + ICMS_DC + \", \" + PAUSE_TIME);\n\n    //2016-08-17T10:41:41.088-0500: 76388.020: [GC (Allocation Failure) 2016-08-17T10:41:41.088-0500: 76388.020: [ParNew (promotion failed): 428321K->431492K(471872K), 0.1049894 secs]2016-08-17T10:41:41.193-0500: 76388.125: [CMS: 466697K->186596K(524288K), 0.7357085 secs] 893715K->186596K(996160K), [Metaspace: 46524K->46524K(1097728K)] icms_dc=0 , 0.8408171 secs] [Times: user=0.91 sys=0.00, real=0.84 secs]\n\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/collection/RuleSet.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.collection;\n\nimport java.util.AbstractMap;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.LinkedList;\nimport java.util.Map;\nimport java.util.NoSuchElementException;\nimport java.util.Objects;\nimport java.util.Set;\nimport java.util.stream.Stream;\n\npublic class RuleSet<K, V> implements Map<K, V>, Iterable<K> {\n\n    private static class Node<K, V> extends AbstractMap.SimpleImmutableEntry<K,V> {\n\n        private Node<K, V> next;\n        private Node<K, V> prev;\n\n        public Node(K key, V value) {\n            super(key, value);\n        }\n    }\n\n    // Instead of a java.util.LinkedList, we use a doubly-linked list of nodes.\n    // This allows us to move the most-recently selected node from to the head of\n    // the list in O(1) time, instead of O(n) time.\n    private Node<K,V> head;\n\n    private final HashMap<K, Node<K,V>> entries;\n\n    public RuleSet() {\n        entries = new HashMap<>();\n    }\n\n    public V get(Object key) {\n        if (key != null) {\n            Node<K,V> node = entries.get(key);\n            return node.getValue();\n        }\n        return null;\n    }\n\n    @Override\n    public int size() {\n        return entries.size();\n    }\n\n    @Override\n    public boolean isEmpty() {\n        return false;\n    }\n\n    @Override\n    public boolean containsKey(Object key) {\n        return false;\n    }\n\n    @Override\n    public boolean containsValue(Object value) {\n        return false;\n    }\n\n    @Override\n    public V put(K key, V value) {\n        Node<K,V> node = new Node<>(key, value);\n        if (head == null) {\n            head = node;\n        } else {\n            node.next = head;\n            head.prev = node;\n            head = node;\n        }\n        entries.put(key, node);\n        return value;\n    }\n\n    @Override\n    public V remove(Object key) {\n        return null;\n    }\n\n    @Override\n    public void putAll(Map<? extends K, ? extends V> m) {\n\n    }\n\n    @Override\n    public void clear() {\n\n    }\n\n    @Override\n    public Set<K> keySet() {\n        return entries.keySet();\n    }\n\n    @Override\n    public Collection<V> values() {\n        Collection<V> values = new LinkedList<>();\n        for (Node<K,V> node = head; node != null; node = node.next) {\n            values.add(node.getValue());\n        }\n        return values;\n    }\n\n    @Override\n    public Set<Entry<K, V>> entrySet() {\n        Set<Entry<K,V>> entrySet = new HashSet<>();\n        for (Node<K,V> node = head; node != null; node = node.next) {\n            entrySet.add(node);\n        }\n        return entrySet;\n    }\n\n    public Stream<Entry<K,V>> stream() {\n        return Stream.iterate(head, Objects::nonNull, node -> ((Node<K,V>)node).next);\n    }\n\n    private class KeyIterator implements Iterator<K> {\n        private Node<K, V> node;\n\n        public KeyIterator() {\n            node = head;\n        }\n\n        @Override\n        public boolean hasNext() {\n            return node != null;\n        }\n\n        @Override\n        public K next() {\n            if (node == null) {\n                // per the Iterator contract\n                throw new NoSuchElementException();\n            }\n            Node<K, V> current = node;\n            node = node.next;\n            return current.getKey();\n        }\n\n    }\n\n    @Override\n    public Iterator<K> iterator() {\n        return new KeyIterator();\n    }\n\n    public V select(K key) {\n        // When a key is selected, move it to the head of the list.\n        // The list will be ordered from most-recently selected to least-recently selected.\n        // Side note: attempting to sort by most often selected resulted in worse performance.\n        final Node<K,V> selected = entries.get(key);\n        if (selected != head) {\n            if (selected.next != null) {\n                Node<K, V> next = selected.next;\n                next.prev = selected.prev;\n            }\n            if (selected.prev != null) {\n                Node<K, V> prev = selected.prev;\n                prev.next = selected.next;\n            }\n            head.prev = selected;\n            selected.next = head;\n            head = selected;\n        }\n        return selected.getValue();\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedGenerationalPatterns.java\n...\n", "needle_token_start": 700, "needle_token_end": 915, "code_context_ntokens": 16225, "output": ["```java\nprivate void freeFloatingYoungBlock(GCLogTrace trace, String line) {\n    boolean initialMark = false;\n    if (trace.getGroup(3) != null)\n        initialMark = trace.contains(3, \"initial-mark\");\n    if (trace.contains(2, \"young\"))\n        if (initialMark)\n            forwardReference = new G1YoungInitialMark(trace.getDateTimeStamp(), trace.gcCause(3, -2), trace.getPauseTime());\n        else\n            forwardReference = new G1Young(trace.getDateTimeStamp(), trace.gcCause(3, -2), trace.getPauseTime());\n    else if (trace.getGroup(2).contains(\"mixed\")) {\n        forwardReference = new G1Mixed(trace.getDateTimeStamp(), trace.gcCause(3, -2), trace.getPauseTime());\n    }\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "checkForGCCause", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to analyze the cause of a garbage collection (GC) event and update system flags based on the identified cause.\n2. **Input**: The function takes a structured representation of a GC log line as its input.\n3. **Output**: There is no direct output from the function; instead, it modifies the state of system flags to reflect the cause of the GC event.\n4. **Procedure**: The function examines the GC cause from the input trace. If the cause is related to metadata generation threshold, it sets specific system flags to indicate this and adjusts compatibility flags for different Java Development Kit (JDK) versions. If the cause is due to evacuation pause or allocation of a large object, it sets a general flag to indicate a GC cause without adjusting JDK compatibility flags.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            } else if ((trace = G1GCPatterns.G1_DETAILS_REFERENCE_GC.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.PRINT_REFERENCE_GC);\n                if (trace.getGroup(3) != null)\n                    getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                else\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE);\n            } else if (G1GCPatterns.G1_INITIAL_MARK.parse(line) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.TENURING_DISTRIBUTION);\n            }\n        }\n\n        if (getDiary().isTrue(SupportedFlags.ADAPTIVE_SIZING)) {\n            if ((trace = G1GCPatterns.YOUNG_SPLIT_BY_G1ERGONOMICS.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                checkForGCCause(trace);\n            }\n        }\n\n        //This has been verified true as of 1.7.0_51. Check later versions to make sure it hasn't been back ported\n        if (ParallelPatterns.PS_FULL_GC_PERM.parse(line) != null) {\n            collectionCount++;\n            getDiary().setTrue(SupportedFlags.JDK70);\n            getDiary().setFalse(SupportedFlags.JDK80);\n            if (line.contains(\" [PSYoungGen: \"))\n                getDiary().setTrue(SupportedFlags.PARALLELGC);\n            getDiary().setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS);\n            getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC);\n\n            getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n            getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC);\n\n        }\n\n        //This has been verified true as of 1.7.0_51. Check later versions to make sure it hasn't been back ported\n        else if (ParallelPatterns.PS_FULL_GC_META.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n            getDiary().setTrue(SupportedFlags.JDK80);\n            if (line.contains(\" [PSYoungGen: \"))\n                getDiary().setTrue(SupportedFlags.PARALLELGC);\n            getDiary().setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS);\n            getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.SERIAL, SupportedFlags.ICMS, SupportedFlags.G1GC);\n\n            getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n            getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC);\n\n        } else if (line.contains(\"CMS-initial-mark\")) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC);\n            getDiary().setTrue(SupportedFlags.CMS, SupportedFlags.GC_DETAILS);\n        }\n\n        //todo: this rule is in the wrong place\n        else if (CMSPatterns.SERIAL_FULL.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC);\n            getDiary().setTrue(SupportedFlags.SERIAL);\n\n            getDiary().setTrue(SupportedFlags.GC_DETAILS);\n            //todo: private final int SupportedFlags.GC_CAUSE = 12;\n            getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n            if (line.contains(\"Metaspace\")) {\n                getDiary().setFalse(SupportedFlags.JDK70);\n            }\n            if (line.contains(\"Tenured\")) {\n                getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS);\n                getDiary().setTrue(SupportedFlags.SERIAL);\n            } else if (line.contains(\"CMS\")) {\n                getDiary().setFalse(SupportedFlags.SERIAL);\n                getDiary().setTrue(SupportedFlags.CMS);\n            }\n        } else if (CMSPatterns.SERIAL_FULL80.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.JDK70);\n            getDiary().setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n        } else if (TENURED_BLOCK.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.CMS);\n            getDiary().setFalse(SupportedFlags.ICMS);\n            getDiary().setTrue(SupportedFlags.SERIAL);\n            getDiary().setFalse(SupportedFlags.G1GC);\n        }\n\n        if (getDiary().isTrue(SupportedFlags.CMS) && !getDiary().isStateKnown(SupportedFlags.ICMS)) {\n            if (firstCMSCycle) {\n                if (line.contains(\"ParNew\") || line.contains(\"DefNew\"))\n                    youngCountAfterFirstCMSCycle++;\n\n                if (line.contains(\"icms_dc\"))\n                    getDiary().setTrue(SupportedFlags.ICMS);\n                else if (youngCountAfterFirstCMSCycle > 1)\n                    getDiary().setFalse(SupportedFlags.ICMS);\n            }\n            //The first CMS cycle is needed to kick off iCMS\n            if (line.contains(\"concurrent-reset\"))\n                firstCMSCycle = true;\n        } else if (getDiary().isTrue(SupportedFlags.G1GC)) {\n            if (G1GCPatterns.G1_MEMORY_SUMMARY.parse(line) != null) {\n                if (line.contains(\"Metaspace\")) {\n                    getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n                    getDiary().setTrue(SupportedFlags.JDK80);\n                } else {\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                    getDiary().setFalse(SupportedFlags.JDK80);\n                }\n            }\n        }\n    }\n\n    \nprivate void checkForGCCause(GCLogTrace trace) {\n        if (trace.gcCause(3, 0) == GCCause.METADATA_GENERATION_THRESHOLD) {\n            getDiary().setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n            getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n        } else if ((trace.gcCause(3, 0) == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) {\n            getDiary().setTrue(SupportedFlags.GC_CAUSE);\n        }\n    }\n\n    /*\n     * Mostly for getting version information but we can also capture other information in the deriveConfiguration.\n     *\n     * if the collector is G1 I don't currently have a good way of saying 7.0 or 8.0 as perm\n     * and meta space records are not printed with a Full GC. Metaspace will be emitted to the\n     * log in the version after 8.0_05.\n     */\n    private void version(String line) {\n\n        if (versionIsKnown()) return;\n\n        GCLogTrace trace;\n        //Can get a lot of information from the perm space record\n        // (CMS Perm |PS Perm |Perm |PSPermGen|Metaspace)\n        if ((trace = ParallelPatterns.PERM_SPACE_RECORD.parse(line)) != null) {\n            String value = trace.getGroup(1).trim();\n\n            if (\"CMS Perm\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"PS Perm\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"Perm\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"PSPermGen\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"Metaspace\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK80);\n                getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n            }\n        } else if (META_SPACE_RECORD.parse(line) != null) {\n            getDiary().setTrue(SupportedFlags.JDK80);\n            getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n        }\n\n        // maybe we'll get lucky, app server often call System.gc() after startup.\n        if (line.contains(\"(System)\")) {\n            getDiary().setTrue(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n            getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n        } else if (line.contains(\"(System.gc()\")) {\n            getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n            getDiary().setTrue(SupportedFlags.GC_CAUSE);\n        } else if (getDiary().isGenerationalKnown() && getDiary().isGenerational()) {\n            if ((trace = PREFIX.parse(line)) != null) {\n                if ((trace.getGroup(3) == null) && getDiary().isTrue(SupportedFlags.GC_DETAILS)) {\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                    getDiary().setFalse(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE);\n                } else if (trace.gcCause(3, 0) != GCCause.GCCAUSE_NOT_SET) {\n                    getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                }\n            } else if ((trace = FULL_PREFIX.parse(line)) != null) {\n                if ((trace.getGroup(3) == null) && getDiary().isTrue(SupportedFlags.GC_DETAILS)) {\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n                } else if (trace.gcCause(3, 0) != GCCause.GCCAUSE_NOT_SET) {\n                    getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                }\n            }\n\n        } else if (getDiary().isG1GCKnown() && getDiary().isG1GC()) {\n            if ((trace = G1GC_PREFIX.parse(line)) != null) {\n                if (getDiary().isTrue(SupportedFlags.GC_DETAILS) && (trace.gcCause() == GCCause.GCCAUSE_NOT_SET)) {\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                    getDiary().setFalse(SupportedFlags.JDK80);\n                } else { //we can't say much else unless we look for 8.0 specific details\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                }\n            }\n        }\n    }\n\n    /*\n     * Currently looking for;\n     * 1) PrintGCDetails\n     * 2) PrintGCCause\n     * 3) PrintTenuringDistribution\n     */\n    private void details(String line) {\n\n        GCLogTrace trace;\n\n        if ((trace = PREFIX.parse(line)) != null) {\n            String cause = trace.getGroup(3);\n            if (cause != null) {\n                getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                if (\"(System)\".equals(cause)) {\n                    getDiary().setFalse(SupportedFlags.JDK80);\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                    getDiary().setTrue(SupportedFlags.PRE_JDK70_40);\n                } else if (\"(System.gc())\".equals(cause)) {\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                } else {\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                }\n            } else {\n                if ((trace = G1GC_PREFIX.parse(line)) != null) {\n                    cause = trace.getGroup(3);\n                    if (cause == null)\n                        getDiary().setTrue(SupportedFlags.PRE_JDK70_40);\n                    else\n                        getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                } else {\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE);\n                }\n            }\n            getDiary().setFalse(SupportedFlags.GC_CAUSE);\n        } else if (line.contains(\"promotion failure size =\"))\n            getDiary().setTrue(SupportedFlags.PRINT_PROMOTION_FAILURE);\n        else if (FLS_HEADER.parse(line) != null) {\n            getDiary().setTrue(SupportedFlags.PRINT_FLS_STATISTICS);\n        }\n\n        //old G1 log file\n        //      [GC Worker Start (ms):  12053551.6  12053551.6  12053551.6  12053551.6  12053551.7  12053551.7  12053551.7  12053551.7  12053551.7  12053551.7  12053551.7  12053551.7\n        //new G1 log file\n        //      [GC Worker Start (ms): Min: 76.3, Avg: 76.3, Max: 76.4, Diff: 0.1]\n        if (getDiary().isG1GC()) {\n            if (line.startsWith(\"[GC Worker Start (ms): \"))\n                if (line.startsWith(\"[GC Worker Start (ms): Min: \"))\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                else\n                    getDiary().setTrue(SupportedFlags.PRE_JDK70_40);\n        } else if (line.contains(\"AdaptiveSizePolicy::\")) {\n            getDiary().setTrue(SupportedFlags.ADAPTIVE_SIZING);\n        }\n\n        //if we've seen a collection and Print Reference GC hasn't been set...\n        if (youngCollectionCount > 1 && !getDiary().isStateKnown(SupportedFlags.PRINT_REFERENCE_GC)) {\n            getDiary().setFalse(SupportedFlags.PRINT_REFERENCE_GC);\n        }\n\n        GCLogTrace gcLogTrace;\n        if ((gcLogTrace = MEMORY_SUMMARY_RULE.parse(line)) != null) {\n            if (gcLogTrace.next() != null)\n                getDiary().setTrue(SupportedFlags.GC_DETAILS);\n        }\n\n        // if we've seen a statement than this is false\n        if (line.startsWith(\"{Heap before GC invocations=\"))\n            getDiary().setTrue(SupportedFlags.PRINT_HEAP_AT_GC);\n        else if (collectionCount > 1)\n            getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC);\n\n        if ((trace = TenuredPatterns.TENURING_SUMMARY.parse(line)) != null) {\n\n            //we have seen at least one good tenuring summary without an age breakdown\n            if ((tenuringSummary > 0) && (!ageTableDetected)) {\n                getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1);\n            } else\n                getDiary().setTrue(SupportedFlags.TENURING_DISTRIBUTION);\n\n            // if calculated tenuring threshold == 0 we won't get an age breakdown so delay evaluation\n            if (trace.getIntegerGroup(2) > 0)\n                tenuringSummary++;\n\n            //If the MaxTenuringThreshold is set to be greater than 15 then we a configuration bug to report on.\n            if (trace.getIntegerGroup(3) > 15) {\n                maxTenuringThreshold = trace.getIntegerGroup(3);\n                getDiary().setTrue(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n            } else\n                getDiary().setFalse(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n        } else if (TenuredPatterns.TENURING_AGE_BREAKDOWN.parse(line) != null) {\n            ageTableDetected = true;\n            getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n        } else if (getDiary().isTenuringDistributionKnown() && collectionCount > 1 && youngCollectionCount > 1) {\n            getDiary().setFalse(SupportedFlags.TENURING_DISTRIBUTION);\n        }\n\n        if (line.contains(\"G1Ergonomics\")) {\n            if (line.contains(\"CSet Construction\") || line.contains(\"Heap Sizing\")) {\n                collectionCount++;\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.ADAPTIVE_SIZING);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n            }\n\n        }\n\n        if (REFERENCE_PROCESSING_BLOCK.parse(line) != null)\n            getDiary().setTrue(SupportedFlags.PRINT_REFERENCE_GC);\n\n        if (line.startsWith(\"Concurrent RS processed\")) {\n            collectionCount++;\n            getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n            getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n        } else if (line.contains(\" (cardTable: \")) {\n            getDiary().setTrue(SupportedFlags.CMS_DEBUG_LEVEL_1);\n        }\n\n        if (collectionCount > 1) {\n            if (!getDiary().isCMSDebugLevel1Known()) {\n                getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n            }\n\n            if (!getDiary().isAdaptiveSizingKnown()) {\n                getDiary().setFalse(SupportedFlags.ADAPTIVE_SIZING);\n            }\n\n            if (!getDiary().isRSetStatsKnown())\n                getDiary().setFalse(SupportedFlags.RSET_STATS);\n        }\n    }\n\n    /*\n     * this method is a poster child against enum....\n     *\n     * The following table bit encodes all of the Collector flags. The flags cannot be acted upon until all of the\n     * flags have been seen as not all combinations of flags are valid and certain combinations will yield unexpected\n     * configurations. The method evaluateGCLogFlags() will examine the flags and set the corresponding values in the\n     * deriveConfiguration.\n     *\n     * GC Flag                  Bit Value\n     * ----------------------------------\n     * +UseSerialGC\t            0x0001\n     * +UseParallelGC\t        0x0002\n     * +UseParallelOldGC\t    0x0004\n     * +ParNew\t                0x0008\n     * +ConcMarkSweepGC     \t0x0010\n     * +CMSIncrementialModea\t0x0020\n     * +G1GC\t                0x0040     0x1840\n     * -UseSerialGC\t            0x0100\n     * -UseParallelGC\t        0x0200\n     * -UseParallelOldGC\t    0x0400\n     * -ParNew\t                0x0800\n     * -ConcMarkSweepGC\t        0x1000\n     * -CMSIncrementialMode\t    0x2000\n     * -G1GC\t                0x4000\n     *\n     * @param commandLineFlags\n     */\n    private void evaluateCommandLineFlags(String[] commandLineFlags) {\n        for (String rawFlag : commandLineFlags) {\n            String flag = processRawFlag(rawFlag);\n            boolean flagTurnedOn = isSetToTrue(rawFlag);\n            CommandLineFlag supportedFlag = CommandLineFlag.fromString(flag);\n            if (supportedFlag != null) {\n                switch (supportedFlag) {\n                    case PrintGCApplicationStoppedTime:\n                        getDiary().setState(SupportedFlags.APPLICATION_STOPPED_TIME, flagTurnedOn);\n                        break;\n\n                    case PrintGCApplicationConcurrentTime:\n                        getDiary().setState(SupportedFlags.APPLICATION_CONCURRENT_TIME, flagTurnedOn);\n                        break;\n\n                    case PrintGCTimeStamps:\n                        break;\n\n                    case PrintGCDetails:\n                        getDiary().setState(SupportedFlags.GC_DETAILS, flagTurnedOn);\n                        if (flagTurnedOn && (getDiary().isJDK80() || getDiary().isUnifiedLogging()))\n                            getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                        break;\n\n                    case PrintGCCause:\n                        getDiary().setState(SupportedFlags.GC_CAUSE, flagTurnedOn);\n                        break;\n\n                    case PrintTenuringDistribution:\n                        getDiary().setState(SupportedFlags.TENURING_DISTRIBUTION, flagTurnedOn);\n                        break;\n\n                    case PrintAdaptiveSizePolicy:\n                        getDiary().setState(SupportedFlags.ADAPTIVE_SIZING, flagTurnedOn);\n                        break;\n\n                    case PrintReferenceGC:\n                        getDiary().setState(SupportedFlags.PRINT_REFERENCE_GC, flagTurnedOn);\n                        break;\n\n                    case PrintHeapAtGC:\n                        getDiary().setState(SupportedFlags.PRINT_HEAP_AT_GC, flagTurnedOn);\n                        break;\n\n                    case PrintPromotionFailure:\n                        getDiary().setTrue(SupportedFlags.PRINT_PROMOTION_FAILURE);\n\n                    case PrintFLSStatistics:\n                        getDiary().setTrue(SupportedFlags.PRINT_FLS_STATISTICS);\n\n                    default:\n                        //shouldn't be able to get here....\n                }\n            } else {\n\n                GarbageCollectorFlag gcFlag = GarbageCollectorFlag.fromString(flag);\n                if (gcFlag != null) {\n                    switch (gcFlag) {\n                        case UseSerialGC:\n                            setGCFlags |= (flagTurnedOn) ? 0x1 : 0x100;\n                            break;\n\n                        case UseParallelGC:\n                            setGCFlags |= (flagTurnedOn) ? 0x2 : 0x200;\n                            break;\n\n                        case UseParallelOldGC:\n                            setGCFlags |= (flagTurnedOn) ? 0x4 : 0x400;\n                            break;\n\n                        case UseParNewGC:\n                            setGCFlags |= (flagTurnedOn) ? 0x8 : 0x800;\n                            break;\n\n                        case UseConcMarkSweepGC:\n                            setGCFlags |= (flagTurnedOn) ? 0x10 : 0x1000;\n                            break;\n\n                        case CMSIncrementialMode:\n                            setGCFlags |= (flagTurnedOn) ? 0x20 : 0;\n                            break;\n\n                        case UseG1GC:\n                            setGCFlags |= (flagTurnedOn) ? 0x40 : 0x4000;\n                            break;\n\n                        default:\n                            // shouldn't be able to get here!!!\n                    }\n                }\n            }\n        }\n\n        evaluateGCLogFlags();\n        getDiary().setFalse(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.GC_DETAILS,\n                SupportedFlags.GC_CAUSE, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_REFERENCE_GC,\n                SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.PRINT_PROMOTION_FAILURE, SupportedFlags.PRINT_FLS_STATISTICS);\n    }\n\n\n    private String processRawFlag(String rawFlag) {\n        String processedFlag = \"\";\n        if (rawFlag.startsWith(\"-XX:\")) {\n            if (rawFlag.charAt(4) == '+' || rawFlag.charAt(4) == '-')\n                processedFlag = rawFlag.substring(5);\n            else\n                processedFlag = rawFlag.substring(4);\n            int equalsPosition = processedFlag.indexOf(\"=\");\n            if (equalsPosition > 0)\n                processedFlag = processedFlag.substring(0, equalsPosition);\n        }\n\n        return processedFlag;\n    }\n\n    private boolean isSetToTrue(String flag) {\n        return (flag.startsWith(\"-XX:+\"));\n    }\n\n    //\"Java HotSpot(TM) 64-Bit Server VM (25.40-b25) for bsd-amd64 JRE (1.8.0_40-b25), built on Feb 10 2015 21:07:25 by \\\"java_re\\\" with gcc 4.2.1 (Based on Apple Inc. build 5658) (LLVM build 2336.11.00)\",\n    private static final Pattern VERSION = Pattern.compile(\"(1)\\\\.([6-9])\\\\.0(_\\\\d*)?\");\n\n    private void parseJVMVersion(String versionString) {\n        Matcher matcher = VERSION.matcher(versionString);\n        if (matcher.find()) {\n            switch (matcher.group(2).charAt(0)) {\n                case '7':\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                    getDiary().setFalse(SupportedFlags.JDK80);\n                    if (matcher.group(3) == null)\n                        getDiary().setTrue(SupportedFlags.PRE_JDK70_40);\n                    try {\n                        int minorVersion = Integer.parseInt(matcher.group(3).substring(1));\n                        if (minorVersion < 41)\n                            getDiary().setTrue(SupportedFlags.PRE_JDK70_40);\n                        else\n                            getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                    } catch (NumberFormatException nfe) {\n                        getDiary().setTrue(SupportedFlags.PRE_JDK70_40);\n                    }\n                    break;\n                case '8':\n                    getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n                    getDiary().setTrue(SupportedFlags.JDK80, SupportedFlags.GC_CAUSE); // doesn't matter so much but may only be true for later versions of 8\n                    break;\n                case '9':\n                    getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80);\n                    break;\n                default:\n            }\n        }\n    }\n\n    /*\n            Flag Combinations                                       Total   Configured         Notes\n                                                                    Value   Combination\n            +UseSerialGC                                        \t1      \tSerial\n            -UserSerialGC                                       \t256 \tPSYoung/PSOldGen\n            +UseParNewGC                                          \t8   \tParNew/Serial\n            -UseParNewGC                                          \t2050\tPSYoung/PSOldGen\t+UseParallelGC is added to the log\n            +UseConcMarkSweepGC                                 \t16\t    ParNew/CMS\n            -UseConcMarkSweepGC                                     4096    PSYoung/PSOldGen\n            +UseParNewGC +UseConcMarkSweepGC                        24\t    ParNew/CMS\n            -UseParNewGC +UseConcMarkSweepGC                      \t2064\tDefNew/CMS\t        deprecated combination\n            +UseParNewGC -UseConcMarkSweepGC                      \t4104\tParNew/Serial\t    deprecated combination\n            -UseParNewGC -UseConcMarkSweepGC                      \t6146\tPSYoung/PSOldGen\t+UseParallelGC is added to the log\n            +UseConcMarkSweepGC +CMSIncrementialMode             \t48\t    ParNew/iCMS\t        ICMS is deprecated\n            +UseParNewGC +UseConcMarkSweepGC +CMSIncrementialMode\t56\t    ParNew/iCMS\t        ICMS is deprecated\n            -UseParNewGC +UseConcMarkSweepGC +CMSIncrementialMode \t2096\tDefNew/iCMS\t        iCMS is deprecated\n            -UseSerialGC +UseConcMarkSweepGC                    \t272\t    ParNew/CMS\t        UseParNewGC is added in log\n            -UseSerialGC -UseConcMarkSweepGC                    \t4354\tPSYoung/PSOldGen\t+UseParallelGC is added to the log\n            +UseParallelGC                                      \t2\t    PSYoung/PSOldGen\tParallelOld is turned on\n            -UseParallelGC                                      \t2\t    PSYoung/PSOldGen\t- is reversed to + in log\n            +UseParallelOldGC                                   \t4\t    PSYoungGen/PSOldGen\tparallel in tenured\n            -UseParallelOldGC                                   \t1024\tPSYoungGen/PSOlGen\tparallel is turned off in tenured\n            +UseParallelGC +UseParallelOldGC                    \t6\t    PSYoung/PSOldGen\tparallel default\n            +UseParallelGC -UseParallelOldGC                    \t1026\tPSYoung/PSOldGen\tserial Full\n            -UseParallelGC +UseParallelOldGC                    \t518\t    PSYoung/PSOldGen\t+UseParallelGC is added to the log\n            -UseParallelGC -UseParallelOldGC                    \t1026\tPSYoung/PSOldGen\tis changed to +UseParallelGC and uses serial full\n            -UseSerialGC +UseParallelGC                         \t258\t    PSYoung/PSOldGen\tsame as default\n            -UseSerialGC +UseParallelOldGC                      \t260\t    PSYoungGen/PSOldGen\tsame as default\n            +UseG1GC                                            \t64\t    G1GC\n            -UseG1GC                                            \t16384\tPSYoung/PSOldGen\tsame as default\n            -UseConcMarkSweepGC +UseG1GC -UseParNewGC               6210    G1GC  4096 + 64 + 2050\n\n    */\n\n    private void evaluateGCLogFlags() {\n\n        switch (setGCFlags) {\n\n            // PSYoung/PSOldGen\n            case 0:\n            case 2:\n            case 4:\n            case 6:\n            case 256:\n            case 258:\n            case 260:\n            case 518:\n            case 2050:\n            case 4354:\n            case 6146:\n            case 16384:\n                getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                break;\n\n            case 1024: // parallel/serial\n            case 1026:\n                getDiary().setTrue(SupportedFlags.PARALLELGC);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                break;\n\n            case 1: // DefNew/Serial\n                getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.SERIAL);\n                getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                break;\n\n            case 8: //ParNew/Serial Deprecated\n            case 4101:\n                getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.SERIAL);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                break;\n\n            case 16: //ParNew/CMS\n            case 24:\n            case 272:\n                getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n                break;\n\n            case 2064: //DefNew/CMS Deprecated\n                getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.CMS);\n                getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n                break;\n\n            case 48: //ParNew/iCMS\n            case 56:\n                getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n                break;\n\n            case 2096: //DefNew/iCMS\n                getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.CMS, SupportedFlags.ICMS);\n                getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n                break;\n\n            case 6208:\n            case 64: //G1GC\n                getDiary().setTrue(SupportedFlags.G1GC);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                break;\n\n            case 4096:\n                getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.RSET_STATS, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                break;\n\n            case 4160:\n                getDiary().setTrue(SupportedFlags.G1GC);\n                getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                break;\n\n            default:\n                LOGGER.severe(\"Illegal internal state: GCToolKit was unable to properly identify this log. Results will be corrupted.\");\n        }\n    }\n\n    private void setGCCause(String gcCause) {\n        if (gcCause == null && getDiary().isPrintGCDetails()) {\n            getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n        } else if (gcCause != null) {\n            if (GCCauses.get(gcCause) == GCCause.GCCAUSE_NOT_SET) {\n                if (getDiary().isPrintGCDetailsKnown() && getDiary().isPrintGCDetails()) {\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                }\n            } else {\n                if (gcCause.contains(\"System.gc()\") || !gcCause.contains(\"System\")) {\n                    getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                } else {\n                    getDiary().setTrue(SupportedFlags.PRE_JDK70_40);\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                }\n            }\n        }\n    }\n}\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/JVMFlags.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\nimport java.util.Map;\n\npublic enum JVMFlags {\n\n    PrintGCApplicationConcurrentTime(\"PrintGCApplicationConcurrentTime\"),\n    PrintGCApplicationStoppedTime(\"PrintGCApplicationStoppedTime\"),\n    Unknown(\"unknown\");\n\n    private final String label;\n\n    JVMFlags(String label) {\n        this.label = label;\n    }\n\n    public String getLabel() {\n        return label;\n    }\n    /*\n        The following flags I found interesting but I would not recommend them for a normal production environment. However, this would, I think could generate great interest\n        PrintClassHistogramBeforeFullGC\n        PrintClassHistogramAfterFullGC\n\n    */\n\n    private final Map<String, Boolean> gcLoggingFlagSupport = Map.ofEntries(\n            Map.entry(\"PrintGCApplicationConcurrentTime\", true),\n            Map.entry(\"PrintGCApplicationStoppedTime\", true),\n            Map.entry(\"PrintGCDetails\", true),\n            Map.entry(\"PrintGCDateStamps\", true),\n            Map.entry(\"PrintTenuringDistribution\", true),\n            Map.entry(\"PrintReferenceGC\", true),\n            Map.entry(\"PrintGCCause\", true),\n            Map.entry(\"PrintFLSStatistics\", true),\n            Map.entry(\"PrintFLSCensus\", true),\n            Map.entry(\"PrintPromotionFailure\", true),\n            Map.entry(\"PrintAdaptiveSizePolicy\", true),\n            Map.entry(\"PrintCMSInitiationStatistics\", true),\n            Map.entry(\"PrintTLAB\", false),\n            Map.entry(\"TLABStats\", false),\n            Map.entry(\"PrintPLAB\", false),\n            Map.entry(\"PrintOldPLAB\", false)\n    );\n\n    public boolean supported(String flag) {\n        return gcLoggingFlagSupport.get(flag);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/GarbageCollectorAlgorithm.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\npublic enum GarbageCollectorAlgorithm {\n    /** -XX:+UseConcMarkSweepGC */\n    DEFNEW,                                     //  1\n    /** -XX:+UseConcMarkSweepGC -XX:+UseParNew */\n    PARNEW,                                     //  2\n    /** -XX:+UseConcMarkSweepGC */\n    CMS,                                        //  3\n    /** -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode */\n    ICMS,                                       //  4\n    /** -XX:+UseParallelGC */\n    PARALLELGC,                                 //  5\n    /** -XX:+UseParallelGC -XX:+UseParallelOldGC <br>-XX:+UseParallelOldGC */\n    PARALLELOLDGC,                              //  6\n    /** -XX:+UseSerialGC */\n    SERIAL,                                     //  7\n    /** -XX:+UseSerialGC */\n    G1GC,                                       //  8\n    /** -XX:+UseZGC */\n    ZGC,                                        //  9\n    /** -XX:+UseShenandoahGC */\n    SHENANDOAH                                  // 10\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\nimport com.microsoft.gctoolkit.jvm.Diarizer;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.parser.unified.UnifiedLoggingLevel;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.TreeSet;\n\nimport static com.microsoft.gctoolkit.jvm.SupportedFlags.GC_CAUSE;\nimport static com.microsoft.gctoolkit.jvm.SupportedFlags.*;\nimport static com.microsoft.gctoolkit.parser.unified.ShenandoahPatterns.SHENANDOAH_TAG;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedG1GCPatterns.G1_COLLECTION;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedG1GCPatterns.G1_TAG;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedGenerationalPatterns.*;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.CPU_BREAKOUT;\nimport static com.microsoft.gctoolkit.parser.unified.ZGCPatterns.CYCLE_START;\nimport static com.microsoft.gctoolkit.parser.unified.ZGCPatterns.ZGC_TAG;\n\n//ShenandoahPatterns, ZGCPatterns, UnifiedG1GCPatterns, UnifiedGenerationalPatterns,\npublic class UnifiedDiarizer implements Diarizer {\n\n    private static final int CYCLES_TO_EXAMINE_BEFORE_GIVING_UP = 10;\n    private static final int CYCLES_TO_EXAMINE_FOR_SAFEPOINT = 2;\n\n    private int lineCount = MAXIMUM_LINES_TO_EXAMINE;\n\n    private final Diary diary;\n    private final TreeSet<String> tagsAndLevels = new TreeSet<>();\n    private int stopTheWorldEvents = 0;\n\n    {\n        diary = new Diary();\n        diary.setTrue(UNIFIED_LOGGING);\n        diary.setFalse(ICMS, PRE_JDK70_40, JDK70, PRE_JDK70_40, JDK80, MAX_TENURING_THRESHOLD_VIOLATION);\n\n    }\n\n    public UnifiedDiarizer() {}\n\n    public Diary getDiary() {\n        return diary;\n    }\n\n    @Override\n    public boolean isUnified() {\n        return true;\n    }\n\n    @Override\n    public String getCommandLine() {\n        // TODO #147 extract command line from the log file\n        return \"\";\n    }\n\n    @Override\n    public int getMaxTenuringThreshold() {\n        return 0;\n    }\n\n    @Override\n    public boolean hasJVMEvents() {\n        return getDiary().isApplicationStoppedTime() ||\n                getDiary().isApplicationRunningTime() ||\n                getDiary().isTLABData();\n    }\n\n    @Override\n    public void fillInKnowns() {\n        getDiary().setFalse(ADAPTIVE_SIZING);\n    }\n\n    @Override\n    public boolean completed() {\n        return getDiary().isComplete() || lineCount < 1;\n    }\n\n    @Override\n    public boolean diarize(String line) {\n        if ( ! this.completed()) {\n            if (!line.startsWith(\"[\"))\n                return false;\n            lineCount--;\n            extractDecorators(line);\n            if (!getDiary().isCollectorKnown())\n                discoverCollector(line);\n            if (!getDiary().isDetailsKnown())\n                discoverDetails(line);\n            if (!getDiary().isJVMEventsKnown())\n                discoverJVMEvents(line);\n            if ((CPU_BREAKOUT.parse(line) != null) || line.contains(\"gc,start\"))\n                stopTheWorldEvents++;\n        }\n        return this.completed();\n    }\n\n    /**\n     * Extract decorators (from a GC log line tag) and set the corresponding diary flags accordingly\n     *\n     * @param line GC log line\n     */\n    private void extractDecorators(String line) {\n        Decorators decorators = new Decorators(line);\n        timeOfFirstEvent(decorators);\n        extractTagsAndLevels(decorators);\n        // -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=trace,safepoint\n        if (decorators.getLogLevel().isPresent()) {\n            UnifiedLoggingLevel logLevel = decorators.getLogLevel().get();\n            if (decorators.tagsContain(\"gc,age\"))\n                getDiary().setTrue(TENURING_DISTRIBUTION);\n            else if (decorators.tagsContain(\"ref\") && logLevel.isGreaterThanOrEqualTo(UnifiedLoggingLevel.debug))\n                getDiary().setTrue(PRINT_REFERENCE_GC);\n            else if (decorators.tagsContain(\"gc,phases\") && logLevel.isGreaterThanOrEqualTo(UnifiedLoggingLevel.debug))\n                getDiary().setTrue(GC_DETAILS);\n            else if ( decorators.tagsContain(\"gc,ergo\"))\n                getDiary().setTrue(ADAPTIVE_SIZING);\n            if (decorators.tagsContain(\"safepoint\"))\n                getDiary().setTrue(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME);\n\n            if (getDiary().isZGC()) {\n                if (decorators.tagsContain(\"task\"))\n                    getDiary().setTrue(GC_DETAILS);\n                else if (decorators.tagsContain(\"heap\"))\n                    getDiary().setTrue(PRINT_HEAP_AT_GC);\n                else if (decorators.tagsContain(\"tlab\"))\n                    getDiary().setTrue(TLAB_DATA);\n                else if (decorators.tagsContain(\"gc,start\") && line.contains(\"Garbage Collection (\"))\n                    getDiary().setTrue(GC_CAUSE);\n                else if (decorators.tagsContain(\"gc,heap\")) {\n                    if (line.contains(\"Heap before GC\"))\n                        getDiary().setTrue(PRINT_HEAP_AT_GC);\n                    getDiary().setTrue(GC_DETAILS);\n                } else if (decorators.tagsContain(\"gc,ref\"))\n                    getDiary().setTrue(PRINT_REFERENCE_GC);\n                else if (decorators.tagsContain(\"gc,heap\") && decorators.getLogLevel().get() == UnifiedLoggingLevel.debug)\n                    getDiary().setTrue(PRINT_HEAP_AT_GC);\n            } else if (getDiary().isShenandoah()) {\n                if (decorators.tagsContain(\"gc,task\") || decorators.tagsContain(\"gc,start\"))\n                    getDiary().setTrue(GC_DETAILS);\n                else if (decorators.tagsContain(\"gc,ergo\"))\n                    getDiary().setTrue(ADAPTIVE_SIZING);\n                else if (decorators.tagsContain(\"gc\") && line.contains(\"Trigger\"))\n                    getDiary().setTrue(GC_CAUSE);\n            }\n        }\n    }\n\n    private void timeOfFirstEvent(Decorators decorator) {\n        if ( ! diary.hasTimeOfFirstEvent())\n            diary.setTimeOfFirstEvent(decorator.getDateTimeStamp());\n    }\n\n    private void extractTagsAndLevels(Decorators decorators) {\n        tagsAndLevels.add(decorators.getLogLevel() + \":\" + decorators.getTags());\n    }\n\n    /*\n    G1 flags\n        ADAPTIVE_SIZING,RSET_STATS,PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS\n        todo: test for log segments that are missing the header lines that will contain the \"use\" keyword\n        ZGC - x\n        Shenandoah - x\n        G1 - done\n        CMS - x\n        Parallel - x\n        Serial - x\n     */\n    private void discoverCollector(String line) {\n\n        if ( ZGC_TAG.parse(line) != null || CYCLE_START.parse(line) != null) {\n            getDiary().setTrue(ZGC);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS);\n            return;\n        }\n\n        if ( SHENANDOAH_TAG.parse(line) != null) {\n            getDiary().setTrue(SHENANDOAH);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, ZGC, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, PRINT_HEAP_AT_GC);\n            return;\n        }\n\n        if (G1_TAG.parse(line) != null || line.contains(\"G1 Evacuation Pause\") || (line.contains(\"Humongous regions: \"))) {\n            getDiary().setTrue(G1GC);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, ZGC, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS);\n            return;\n        }\n\n        if (CMS_TAG.parse(line) != null ||\n                PARNEW_TAG.parse(line) != null ||\n                line.contains(\"ParNew\")) {\n            getDiary().setTrue(PARNEW, CMS);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS);\n            return;\n        }\n\n        if (PARALLEL_TAG.parse(line) != null ||\n                line.contains(\"ParOldGen\") ||\n                line.contains(\"PSYoungGen\")) {\n            getDiary().setTrue(PARALLELGC, PARALLELOLDGC, GC_CAUSE);\n            getDiary().setFalse(DEFNEW, SERIAL, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS);\n            return;\n        }\n\n        if (SERIAL_TAG.parse(line) != null || line.contains(\"DefNew\")) {\n            getDiary().setTrue(DEFNEW, SERIAL, GC_CAUSE);\n            getDiary().setFalse(PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS);\n            return;\n        }\n    }\n\n    /**\n     *\n     * @param line\n     */\n    private void discoverDetails(String line) {\n\n        //todo: RSET_STATS for G1 not looked for...\n        if (G1_COLLECTION.parse(line) != null) {\n            getDiary().setTrue(GC_CAUSE);\n        }\n\n        else if (CYCLE_START.parse(line) != null) {\n            getDiary().setTrue(GC_CAUSE);\n        }\n\n        if (stopTheWorldEvents > CYCLES_TO_EXAMINE_BEFORE_GIVING_UP)\n            getDiary().setFalse(ADAPTIVE_SIZING, GC_CAUSE, TLAB_DATA, PRINT_REFERENCE_GC, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, RSET_STATS, PRINT_HEAP_AT_GC);\n    }\n\n    private void discoverJVMEvents(String line) {\n        if (stopTheWorldEvents > CYCLES_TO_EXAMINE_FOR_SAFEPOINT) {\n            getDiary().setFalse(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME);\n        }\n    }\n\n    @Override\n    public DateTimeStamp getTimeOfFirstEvent() {\n        return getDiary().getTimeOfFirstEvent();\n    }\n\n}\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/vmops/SafepointParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.vmops;\n\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.event.jvm.Safepoint;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.PreUnifiedGCLogParser;\n\nimport java.util.Set;\n\npublic class SafepointParser extends PreUnifiedGCLogParser implements SafepointPatterns {\n\n    public SafepointParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.SAFEPOINT);\n    }\n\n    public String getName() {\n        return \"SafepointParser\";\n    }\n\n    protected void process(String line) {\n        SafepointTrace trace;\n        if ((trace = TRACE.parse(line)) != null) {\n            Safepoint safepoint = trace.toSafepoint();\n            super.publish(ChannelName.JVM_EVENT_PARSER_OUTBOX, safepoint);\n        } else if (line.equals(END_OF_DATA_SENTINEL))\n            super.publish( ChannelName.JVM_EVENT_PARSER_OUTBOX, new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return (diary.isTLABData() || diary.isApplicationStoppedTime() || diary.isApplicationRunningTime()) && ! diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n}\n\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/vmops/SafepointPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.vmops;\n\n\nimport com.microsoft.gctoolkit.parser.PreUnifiedTokens;\nimport com.microsoft.gctoolkit.parser.SafepointParseRule;\n\n/**\n * General format is;\n * vmop                    [threads: total initially_running wait_to_block]    [time: spin block sync cleanup vmop] page_trap_count\n */\npublic interface SafepointPatterns extends PreUnifiedTokens {\n\n    String VMOP = \"(Deoptimize|no vm operation|EnableBiasedLocking|GenCollectForAllocation|RevokeBias|BulkRevokeBias|ThreadDump|FindDeadlocks|Exit)\";\n    String SAFE_POINT_PREFIX = TIMESTAMP + VMOP;\n    String THREAD_COUNTS = \"\\\\[\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\\\\]\";\n...\n", "needle_token_start": 2205, "needle_token_end": 2403, "code_context_ntokens": 16081, "output": ["```java\nprivate void checkForGCCause(GCLogTrace trace) {\n    if (trace.gcCause(3, 0) == GCCause.METADATA_GENERATION_THRESHOLD) {\n        getDiary().setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n        getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n    } else if ((trace.gcCause(3, 0) == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) {\n        getDiary().setTrue(SupportedFlags.GC_CAUSE);\n    }\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "postEvacuatePhaseDuration", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To record the duration of the garbage collection phase that occurs after the main evacuation of the collection set.\n2. **Input**: A trace object containing details about the garbage collection event and a string representing the log line.\n3. **Output**: No direct output; updates the duration of the post-evacuation phase in a reference object.\n4. **Procedure**: The function extracts the phase identifier and its duration from the trace object. It then updates the duration of the specific post-evacuation phase in the reference object using these extracted values.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        parseRules.put(REMEMBERED_SETS, this::noop);\n        parseRules.put(EAGER_RECLAIM_STEP, this::noop);\n        parseRules.put(CARDS, this::noop);\n        parseRules.put(HOT_CARD_CACHE, this::noop);\n        parseRules.put(LOG_BUFFERS, this::noop);\n        parseRules.put(SCAN_HEAP_ROOTS, this::noop);\n\n        parseRules.put(SCANS, this::noop);\n        parseRules.put(CLAIMED_CHUNKS, this::noop);\n        parseRules.put(CODE_ROOT_SCAN, this::noop);\n        parseRules.put(STRING_DEDUP, this::noop);\n        parseRules.put(WEAK_JFR_SAMPLES, this::noop);\n\n        parseRules.put(POST_EVAC_CLEANUP, this::noop);\n        parseRules.put(MERGE_THREAD_STATE, this::noop);\n        parseRules.put(COPIED_BYTES, this::noop);\n        parseRules.put(LAB, this::noop);\n        parseRules.put(CLEAR_LOGGED_CARDS, this::noop);\n\n        parseRules.put(RECALC_USED_MEM, this::noop);\n        parseRules.put(PURGE_CODE_ROOTS, this::noop);\n        parseRules.put(UPDATE_DERIVED_POINTERS, this::noop);\n        parseRules.put(EAGER_HUMONGOUS_RECLAIM, this::noop);\n        parseRules.put(HUMONGOUS, this::noop);\n\n        parseRules.put(REDIRTY_CARDS, this::noop);\n        parseRules.put(REDIRTIED_CARDS, this::noop);\n        parseRules.put(FREE_CSET, this::noop);\n        parseRules.put(REBUILD_FREELIST, this::noop);\n        parseRules.put(NEW_CSET, this::noop);\n        parseRules.put(RESIZE_TLAB, this::noop);\n    }\n\n    public UnifiedG1GCParser() {\n    }\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.G1GC);\n    }\n\n    public String getName() {\n        return \"UnifiedG1GCParser\";\n    }\n\n    @Override\n    protected void process(String line) {\n        if (! ignoreFrequentlySeenButUnwantedLines(line))\n            parse(line);\n    }\n\n    private static final Pattern gcIdPattern = GCLogParser.GCID_COUNTER.pattern();\n\n    private void parse(String line) {\n\n        // Minor optimization. The parse rule only applies to what comes after the GC ID.\n        final int end;\n        final int gcid;\n        final Matcher gcIdMatcher = gcIdPattern.matcher(line);\n        if (gcIdMatcher.find()) {\n            gcid = Integer.parseInt(gcIdMatcher.group(1));\n            end = gcIdMatcher.end();\n        } else {\n            gcid = -1;\n            end = 0;\n        }\n\n        final String lineAfterGcId = line.substring(end);\n\n        parseRules.stream()\n                .map(Map.Entry::getKey)\n                .map(rule -> new AbstractMap.SimpleEntry<>(rule, rule.parse(lineAfterGcId)))\n                .filter(tuple -> tuple.getValue() != null)\n                .findAny()\n                .ifPresentOrElse(\n                        tuple -> {\n                            // Typically, \"end\" will be greater than zero, but not always.\n                            setForwardReference(gcid, end > 0 ? line.substring(0, end) : line);\n                            applyRule(tuple.getKey(), tuple.getValue(), line);\n                        },\n                        () -> log(line)\n                );\n    }\n\n\n    private void applyRule(GCParseRule ruleToApply, GCLogTrace trace, String line) {\n        try {\n            parseRules.select(ruleToApply).accept(trace, line);\n        } catch (Throwable t) {\n            LOGGER.throwing(this.getName(), \"process\", t);\n        }\n    }\n\n    private void setForwardReference(int gcid, String line) {\n        if (gcid != -1) {\n            forwardReference = collectionsUnderway.computeIfAbsent(gcid, k -> new G1GCForwardReference(new Decorators(line), gcid));\n            forwardReference.setHeapRegionSize(regionSize);\n            forwardReference.setMaxHeapSize(maxHeapSize);\n            forwardReference.setMinHeapSize(minHeapSize);\n            forwardReference.setInitialHeapSize(initialHeapSize);\n        }\n    }\n\n    private void removeForwardReference(G1GCForwardReference forwardReference) {\n        collectionsUnderway.remove(forwardReference.getGcID());\n    }\n\n    private void noop(GCLogTrace trace, String line) {}\n\n    /*************\n     *\n     * Data Extraction methods\n     */\n\n    private void cpuBreakout(GCLogTrace trace, String line) {\n        CPUSummary cpuSummary = new CPUSummary(trace.getDoubleGroup(1), trace.getDoubleGroup(2), trace.getDoubleGroup(3));\n        forwardReference.setCPUSummary(cpuSummary);\n        try {\n            publishPauseEvent(forwardReference.buildEvent());\n        } catch (MalformedEvent malformedEvent) {\n            LOGGER.warning(malformedEvent.getMessage());\n        }\n    }\n\n    // todo: need to drain the queues before terminating...\n    // Just in case there isn't a JVM termination event in the log.\n    public void endOfFile(GCLogTrace trace, String line) {\n        publish(new JVMTermination((jvmTerminationEventTime.hasTimeStamp()) ? jvmTerminationEventTime : getClock(),diary.getTimeOfFirstEvent()));\n    }\n\n\n    /**\n     * following records describe heap before the collection\n     *\n     * @param trace\n     * @param line\n     */\n    private void heapBeforeAfterGCInvocationCount(GCLogTrace trace, String line) {\n        if (\"before\".equals(trace.getGroup(1))) {\n            before = true;\n        } else if (\"after\".equals(trace.getGroup(1))) {\n            before = false;\n        }\n        gcInvocations = trace.getIntegerGroup(2);\n        fullGCInvocations = trace.getIntegerGroup(3);\n    }\n\n    //Ignore these log messages for now.\n    private void heapSummary(GCLogTrace trace, String line) {\n//        if ( before) {\n//            forwardReference.setHeapOccupancyBeforeCollection(trace.getLongGroup(2));\n//            forwardReference.setHeapSizeBeforeCollection(trace.getLongGroup(1));\n//        } else {\n//            forwardReference.setHeapOccupancyAfterCollection(trace.getLongGroup(2));\n//            forwardReference.setHeapSizeAfterCollection(trace.getLongGroup(1));\n//        }\n    }\n\n    //Minimum heap 8388608  Initial heap 268435456  Maximum heap 268435456\n    //these values go back to the JavaVirtualMachine..\n    public void heapSize(GCLogTrace trace, String line) {\n        this.minHeapSize = trace.getLongGroup(1);\n        this.initialHeapSize = trace.getLongGroup(2);\n        this.maxHeapSize = trace.getLongGroup(3);\n    }\n\n    //return to JVM\n    private int regionSize = 0; //region size in Gigabytes\n    private long minHeapSize = 0;\n    private long initialHeapSize = 0;\n    private long maxHeapSize = 0;\n\n    public void heapRegionSize(GCLogTrace trace, String line) {\n        regionSize = trace.getIntegerGroup(1);\n    }\n\n    //[15.316s][debug][gc,heap      ] GC(0)   region size 1024K, 24 young (24576K), 0 survivors (0K)\n    //ignore this logging for now\n    private void youngRegionAllotment(GCLogTrace trace, String line) {\n        forwardReference.setHeapRegionSize(trace.getIntegerGroup(1) / 1024);\n        if (before) {\n            forwardReference.setYoungOccupancyBeforeCollection(trace.getLongGroup(3));\n            forwardReference.setSurvivorOccupancyBeforeCollection(trace.getLongGroup(5));\n            forwardReference.setEdenOccupancyBeforeCollection(trace.getLongGroup(3)-trace.getLongGroup(5));\n            forwardReference.setYoungSizeBeforeCollection(trace.getLongGroup(3));\n        }\n        else {\n            forwardReference.setYoungOccupancyAfterCollection(trace.getLongGroup(5));\n            forwardReference.setSurvivorOccupancyAfterCollection(trace.getLongGroup(5));\n            forwardReference.setEdenOccupancyAfterCollection(0L);\n            forwardReference.setYoungSizeAfterCollection(trace.getLongGroup(3));\n        }\n    }\n\n    /**\n     * @param trace\n     * @param line\n     */\n    private void metaClassSpace(GCLogTrace trace, String line) {\n        if (before) {\n            if (trace.getGroup(1).equals(\"Metaspace\")) {\n                forwardReference.setMetaspaceOccupancyBeforeCollection(trace.getLongGroup(2));\n                forwardReference.setMetaspaceSizeBeforeCollection(trace.getLongGroup(3));\n                forwardReference.setMetaspaceCommittedBeforeCollection(trace.getLongGroup(4));\n                forwardReference.setMetaspaceReservedBeforeCollection(trace.getLongGroup(5));\n            } else if (trace.getGroup(1).equals(\"class space\")) {\n                forwardReference.setClassspaceOccupancyBeforeCollection(trace.getLongGroup(2));\n                forwardReference.setClassspaceSizeBeforeCollection(trace.getLongGroup(3));\n                forwardReference.setClassspaceCommittedBeforeCollection(trace.getLongGroup(4));\n                forwardReference.setClassspaceReservedBeforeCollection(trace.getLongGroup(5));\n            } else\n                trace.notYetImplemented();\n        } else {\n            if (trace.getGroup(1).equals(\"Metaspace\")) {\n                forwardReference.setMetaspaceOccupancyAfterCollection(trace.getLongGroup(2));\n                forwardReference.setMetaspaceSizeAfterCollection(trace.getLongGroup(3));\n                forwardReference.setMetaspaceCommittedAfterCollection(trace.getLongGroup(4));\n                forwardReference.setMetaspaceReservedAfterCollection(trace.getLongGroup(5));\n            } else if (trace.getGroup(1).equals(\"class space\")) {\n                forwardReference.setClassspaceOccupancyAfterCollection(trace.getLongGroup(2));\n                forwardReference.setClassspaceSizeAfterCollection(trace.getLongGroup(3));\n                forwardReference.setClassspaceCommittedAfterCollection(trace.getLongGroup(4));\n                forwardReference.setClassspaceReservedAfterCollection(trace.getLongGroup(5));\n            } else\n                trace.notYetImplemented();\n        }\n\n    }\n\n    private void g1Collection(GCLogTrace trace, String line) {\n        GarbageCollectionTypes gcType;\n        String gcSubtype = trace.getGroup(3);\n        if (gcSubtype == null)\n            gcType = fromLabel(trace.getGroup(1));\n        else {\n            switch (gcSubtype) {\n                default:\n                    LOGGER.warning(\"GC Type not recognized: \" + line);\n                case \"Prepare Mixed\":\n                case \"Normal\":\n                    gcType = fromLabel(trace.getGroup(1));\n                    break;\n                case \"Mixed\":\n                    gcType = fromLabel(trace.getGroup(3));\n                    break;\n                case \"Concurrent End\":\n                case \"Concurrent Start\":\n                    gcType = GarbageCollectionTypes.Initial_Mark;\n                    break;\n            }\n        }\n        forwardReference.setGcType(gcType);\n        forwardReference.setGCCause(trace.gcCause(1));\n        forwardReference.setStartTime(getClock());\n    }\n\n    private void workSummary(GCLogTrace trace, String line) {\n        forwardReference.evacuationWorkers(trace.getIntegerGroup(1), trace.getIntegerGroup(2));\n    }\n\n    private void references(GCLogTrace trace, String line) {\n        /**\n         * todo: capture preclean phases\n         * Not recording preclean phases for the moment. If the preclean capture groups is not null, then\n         * it's a preclean phase so noop it.\n         */\n        if ( trace.getGroup(1) != null) return;\n        switch (trace.getGroup(2)) {\n            case \"SoftReference\":\n                forwardReference.setSoftReferenceProcessingDuation(trace.getDurationInSeconds());\n                break;\n            case \"WeakReference\":\n                forwardReference.setWeakReferenceProcessingDuration(trace.getDurationInSeconds());\n                break;\n            case \"FinalReference\":\n                forwardReference.setFinalReferenceProcessingDuration(trace.getDurationInSeconds());\n                break;\n            case \"PhantomReference\":\n                forwardReference.setPhantomReferenceProcessingDuration(trace.getDurationInSeconds());\n                break;\n            case \"JNI Weak Reference\":\n                forwardReference.setJniWeakReferenceProcessingDuration(trace.getDurationInSeconds());\n                break;\n            default:\n                trace.notYetImplemented();\n        }\n    }\n\n    //todo: magic numbers\n    //[15.322s][debug][gc,ref       ] GC(0) Ref Counts: Soft: 0 Weak: 994 Final: 1074 Phantom: 211\n    private void referenceCounts(GCLogTrace trace, String line) {\n        forwardReference.setReferenceCounts(trace.getIntegerGroup(1),\n                trace.getIntegerGroup(2),\n                trace.getIntegerGroup(3),\n                trace.getIntegerGroup(4));\n    }\n\n    //Phases\n    /*\n        [15.322s][info ][gc,phases    ] GC(0)   Pre Evacuate Collection Set: 0.0ms\n        [15.322s][info ][gc,phases    ] GC(0)   Evacuate Collection Set: 4.9ms\n        [15.322s][info ][gc,phases    ] GC(0)   Post Evacuate Collection Set: 1.0ms\n        [15.322s][info ][gc,phases    ] GC(0)   Other: 0.2ms\n     */\n\n    private void evacuateCollectionSetTime(GCLogTrace trace, String line) {\n        if (trace.getGroup(1) == null) {\n            forwardReference.setEvacuationCSetDuration(trace.getDurationInSeconds());\n        } else if (\"Pre\".equals(trace.getGroup(1))) {\n            forwardReference.setPreEvacuateCSetDuration(trace.getDurationInSeconds());\n        } else if (\"Post\".equals(trace.getGroup(1)))\n            forwardReference.setPostEvacuateCSetDuration(trace.getDurationInSeconds());\n        else\n            LOGGER.warning(\"Not recognized: \" + line);\n    }\n\n    public void other(GCLogTrace trace, String line) {\n        forwardReference.setOtherDuration(trace.getDurationInSeconds());\n    }\n\n    private void preEvacuateCSetPhaseDuration(GCLogTrace trace, String line) {\n        forwardReference.postPreEvacuateCSetPhaseDuration(trace.getGroup(1), trace.getDurationInSeconds());\n    }\n\n    public void evacuateCSetPhase(GCLogTrace trace, String line) {\n        forwardReference.postEvacuateCSetPhaseDuration(trace.getGroup(1), trace.getUnifiedStatisticalSummary());\n    }\n\n    \npublic void postEvacuatePhaseDuration(GCLogTrace trace, String line) {\n        forwardReference.postPostEvacuateCSetPhaseDuration(trace.getGroup(1), trace.getDurationInSeconds());\n    }\n\n    public void toSpaceExhausted(GCLogTrace trace, String line) {\n        forwardReference.toSpaceExhausted();\n    }\n\n    public void parallelCount(GCLogTrace trace, String line) {\n        switch (trace.getGroup(1)) {\n            case \"Processed Buffers\":\n                forwardReference.setProcessedBuffersSummary(trace.countSummary());\n                break;\n            case \"Termination Attempts\":\n                forwardReference.setTerminationAttempts(trace.countSummary());\n                break;\n            default:\n                trace.notYetImplemented();\n        }\n    }\n\n    /**\n     * The trace indicates number of active regions before and after the collection. This is then used to provide an\n     * extremely coarse estimate of the amount of live data.\n     * @param trace A chunk of GC log that we are attempting to match to a known GC log pattern\n     * @param line The log line corresponding to the trace\n     */\n    public void regionSummary(GCLogTrace trace, String line) {\n        RegionSummary summary = trace.regionSummary();\n        switch (trace.getGroup(1)) {\n            case \"Eden\":\n                forwardReference.setEdenRegionSummary(summary);\n                break;\n            case \"Survivor\":\n                forwardReference.setSurvivorRegionSummary(summary);\n                break;\n            case \"Old\":\n                forwardReference.setOldRegionSummary(summary);\n                break;\n            case \"Humongous\":\n                forwardReference.setHumongousRegionSummary(summary);\n                break;\n            case \"Archive\":\n                forwardReference.setArchiveRegionSummary(summary);\n                break;\n            default:\n                notYetImplemented(trace, line);\n        }\n    }\n\n    public void unifiedMetaData(GCLogTrace trace, String line) {\n        if (forwardReference.setMetaspaceOccupancyBeforeCollection(trace.toKBytes(1))) {\n            forwardReference.setMetaspaceOccupancyAfterCollection(trace.toKBytes(3));\n            forwardReference.setMetaspaceSizeAfterCollection(trace.toKBytes(5));\n        }\n    }\n\n    public void youngDetails(GCLogTrace trace, String line) {\n        forwardReference.setHeapOccupancyBeforeCollection(trace.toKBytes(5));\n        forwardReference.setHeapOccupancyAfterCollection(trace.toKBytes(7));\n        forwardReference.setHeapSizeAfterCollection(trace.toKBytes(9));\n        forwardReference.setDuration(trace.getDurationInSeconds());\n    }\n\n    /**\n     * Record contains Metaspace broken out to class and non-class space. Since\n     * Metaspace = class space + non-class space, we can ignore the non-class space information (for now)\n     * The space size before the collection can be determined by inspecting the previous record (ignore for now)\n     * @param trace A chunk of GC log that we are attempting to match to a known GC log pattern\n     * @param line The log line corresponding to the trace\n     */\n    public void metaNonClassClassSpace(GCLogTrace trace, String line) {\n        MemoryPoolSummary metaspace = trace.getEnlargedMetaSpaceRecord(1);\n        forwardReference.setMetaspaceOccupancyBeforeCollection(metaspace.getOccupancyBeforeCollection());\n        forwardReference.setMetaspaceOccupancyAfterCollection(metaspace.getOccupancyAfterCollection());\n        forwardReference.setMetaspaceSizeAfterCollection(metaspace.getSizeAfterCollection());\n        MemoryPoolSummary classSpace = trace.getEnlargedMetaSpaceRecord(17);\n        forwardReference.setClassspaceOccupancyBeforeCollection(classSpace.getOccupancyBeforeCollection());\n        forwardReference.setClassspaceCommittedAfterCollection(classSpace.getOccupancyAfterCollection());\n        forwardReference.setClassspaceSizeAfterCollection(classSpace.getSizeAfterCollection());\n    }\n\n    //Concurrent Mark\n\n    /**\n     * Start of concurrent phases which can be ignored (for now??)\n     *\n     * @param trace\n     * @param line\n     */\n\n    private void concurrentCycleStart(GCLogTrace trace, String line) {\n        forwardReference.setConcurrentCycleStartTime(getClock());\n        forwardReference.setGcType(GarbageCollectionTypes.Concurrent_Cycle);\n    }\n\n    private void concurrentCycleEnd(GCLogTrace trace, String line) {\n        removeForwardReference(forwardReference);\n    }\n\n    //todo: ????? this is a different type of concurrent cycle\n    private void concurrentUndoCycleStart(GCLogTrace trace, String line) {\n        forwardReference.setConcurrentCycleStartTime(getClock());\n        forwardReference.setGcType(GarbageCollectionTypes.G1GCConcurrentUndoCycle);\n    }\n\n    //todo: need support for JDK 17 undo concurrent cycle event.. started here, commented out for a future PR.\n    private void concurrentUndoCycleEnd(GCLogTrace trace, String line) {\n        forwardReference.setDuration(trace.getDurationInSeconds());\n        publishUndoCycle((G1ConcurrentUndoCycle) forwardReference.buildConcurrentUndoCycleEvent());\n        removeForwardReference(forwardReference);\n    }\n\n    private void concurrentPhase(GCLogTrace trace, String line) {\n        forwardReference.setConcurrentPhase(GarbageCollectionTypes.fromLabel(trace.getGroup(1)));\n        forwardReference.setStartTime(getClock());\n        concurrentPhaseActive = true;\n    }\n\n    private void concurrentMarkEnd(GCLogTrace trace, String line) {\n        forwardReference.setDuration(trace.getDurationInSeconds());\n        publishConcurrentEvent(forwardReference.buildConcurrentPhaseEvent());\n    }\n\n    private void concurrentPhaseDuration(GCLogTrace trace, String line) {\n        forwardReference.setDuration(trace.getDurationInSeconds());\n        publishConcurrentEvent(forwardReference.buildConcurrentPhaseEvent());\n    }\n\n    /**\n     * this is the start of the records, nothing to be captured.\n     * @param trace\n     * @param line\n     */\n    private void concurrentMarkInternalPhases(GCLogTrace trace, String line) {\n    }\n\n    private void concurrentMarkInternalPhaseDuration(GCLogTrace trace, String line) {\n        switch (trace.getGroup(1)) {\n            case \"Mark From Roots\":\n                forwardReference.setMarkFromRootsDuration(trace.getDurationInSeconds());\n                break;\n            case \"Preclean\":\n                forwardReference.setPrecleanDuration(trace.getDurationInSeconds());\n                break;\n            default:\n                LOGGER.warning(\"unknown Concurrent Mark phase : \" + line);\n\n        }\n    }\n\n    private void concurrentMarkWorkers(GCLogTrace trace, String line) {\n        forwardReference.concurrentMarkWorkers(trace.getIntegerGroup(1), trace.getIntegerGroup(2));\n    }\n\n    private void concurrentMarkAborted(GCLogTrace trace, String line) {\n        forwardReference.abortConcurrentMark();\n        publishConcurrentEvent(forwardReference.buildConcurrentPhaseEvent());\n    }\n\n    private void remarkStart(GCLogTrace trace, String line) {\n        forwardReference.pausePhaseDuringConcurrentCycle(GarbageCollectionTypes.G1GCRemark);\n        forwardReference.pausePhaseDuringConcurrentCycleStart(getClock());\n    }\n\n    private void cleanupStart(GCLogTrace trace, String line) {\n        forwardReference.pausePhaseDuringConcurrentCycle(GarbageCollectionTypes.G1GCCleanup);\n        forwardReference.pausePhaseDuringConcurrentCycleStart(getClock());\n    }\n\n    private void finalizeMarking(GCLogTrace trace, String line) {\n        forwardReference.finalizeMarkingDuration(trace.getDurationInSeconds());\n    }\n\n    private void systemDictionaryUnloading(GCLogTrace trace, String line) {\n        forwardReference.systemDictionaryUnloadingDuration(trace.getDurationInSeconds());\n    }\n\n    private void stringSymbolTableCleaning(GCLogTrace trace, String s) {\n        forwardReference.stringTableProcessedAndRemoved(trace.getIntegerGroup(1), trace.getIntegerGroup(2));\n        forwardReference.symbolTableProcessedAndRemoved(trace.getIntegerGroup(3), trace.getIntegerGroup(4));\n    }\n\n    private void parallelUnloading(GCLogTrace trace, String line) {\n        forwardReference.parallelUnloadingDuration(trace.getDurationInSeconds());\n    }\n\n    private void pausePhaseDuringConcurrentCycleDurationEnd(GCLogTrace trace, String line) {\n        forwardReference.pausePhaseDuringConcurrentCycleDuration(trace.getDurationInSeconds());\n        forwardReference.setHeapOccupancyBeforeCollection(trace.toKBytes(1));\n        forwardReference.setHeapSizeBeforeCollection(trace.toKBytes(5));\n        forwardReference.setHeapOccupancyAfterCollection(trace.toKBytes(3));\n        forwardReference.setHeapSizeAfterCollection(trace.toKBytes(5));\n    }\n\n    //Full\n\n    private void fullPhase(GCLogTrace trace, String line) {\n        if (trace.getGroup(3) != null)\n            forwardReference.fullPhase(trace.getIntegerGroup(1), trace.getGroup(2), trace.getDurationInSeconds());\n    }\n\n    /**\n     * todo: need to process and view this captured data\n     * @param trace\n     * @param line\n     */\n    private void fullClassUnloading(GCLogTrace trace, String line) {\n        //forwardReference.fullClassUnloadingDuration(trace.getMillisecondDurationInSeconds());\n    }\n\n    /**\n     * todo: need to capture StringTable data (part of remark at debug level)\n     * @param trace\n     * @param line\n     */\n    private void fullStringSymbolTable(GCLogTrace trace, String line) {\n//        forwardReference.scrubStringSymbolTableDuration(trace.getMillisecondDurationInSeconds());\n    }\n\n    /**\n     * records a concurrent phase of a concurrent cycle. After the event has been recorded, all other events\n     * that occurred during the concurrent event will be recorded.\n     * The exception is the Concurrent Undo cycle which causes all concurrent phases to be queued until the\n     * undo cycle ends.\n     * @param event\n     */\n    private void publishConcurrentEvent(G1GCConcurrentEvent event) {\n        if ( event == null) return;\n\n        if ( forwardReference.getGcType() != GarbageCollectionTypes.G1GCConcurrentUndoCycle) {\n            publish(event);\n            concurrentPhaseActive = false;\n            eventQueue.stream().forEach(this::publish);\n            eventQueue.clear();\n        } else {\n            eventQueue.add(event);\n        }\n    }\n\n    private void publishUndoCycle(G1ConcurrentUndoCycle cycle) {\n        concurrentPhaseActive = false;\n        publish(cycle);\n        eventQueue.stream().forEach(this::publish);\n        eventQueue.clear();\n    }\n\n    private final Queue<G1GCEvent> eventQueue = new LinkedList<>();\n\n    /**\n     * Events are published in the start time order. If a concurrent cycle has started and it's in a concurrent\n     * phase, the pause event is queued. It will be published when the concurrent phase completes. As each event is\n     * published, it corresponding forward reference is released.\n     * @param event\n     */\n    private void publishPauseEvent(G1GCPauseEvent event) {\n        if (event == null) return;\n        if ( concurrentPhaseActive) {\n            eventQueue.add(event);\n            removeForwardReference(forwardReference);\n        } else {\n            publish(event);\n            if ( ! forwardReference.isConcurrentCycle())\n                removeForwardReference(forwardReference);\n        }\n    }\n\n\n    // Let the EOF event send the exit event. Easier in cases where JVM termination isn't in the log\n    private void jvmExit(GCLogTrace trace, String s) {\n        jvmTerminationEventTime = this.getClock();\n    }\n\n    private boolean ignoreFrequentlySeenButUnwantedLines(String line) {\n        if (line.contains(\"Desired survivor size\")) return true;\n        if (line.contains(\"Age table with threshold\")) return true;\n        if (line.contains(\"safepoint\")) return true;\n        if (line.contains(\") Skipped phase \")) return true;\n        if (line.contains(\" Total                          Min: \")) return true;\n        if (line.contains(\" Dead                           Min: \")) return true;\n        if (line.contains(\" VM Weak                        Min\")) return true;\n        if (line.contains(\" ObjectSynchronizer Weak        Min:\")) return true;\n        if (line.contains(\" JVMTI Tag Weak OopStorage      Min:\")) return true;\n        if (line.contains(\" StringTable Weak               Min:\")) return true;\n        if (line.contains(\" ResolvedMethodTable Weak       Min:\")) return true;\n        if (line.contains(\" JNI Weak                       Min:\")) return true;\n        return line.contains(\" - age \");\n    }\n\n    /**\n     * Ignore rules we don't need to process.\n     *\n     * @param trace GCLogTrace that hits this log line\n     * @param line  Log line.\n     */\n    private void ignore(GCLogTrace trace, String line) {\n        return;\n    }\n\n    private void log(String line) {\n        if ( ! ignoreFrequentlySeenButUnwantedLines(line)) {\n\n            GCToolKit.LOG_DEBUG_MESSAGE(() -> \"Missed: \" + line);\n            LOGGER.log(Level.FINE, \"Missed: {0}\", line);\n        }\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return diary.isG1GC() && diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n\n    private void publish(JVMEvent event) {\n        super.publish(ChannelName.G1GC_PARSER_OUTBOX,event);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/ForwardReference.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\npublic class ForwardReference {\n\n    private final Decorators decorators;\n    private final int gcID;\n    private DateTimeStamp startTime = null;\n    private double duration = -1.0d;\n    private GCCause gcCause = GCCause.UNKNOWN_GCCAUSE;\n    private CPUSummary cpuSummary = null;\n\n    ForwardReference(Decorators decorators, int id) {\n        this.gcID = id;\n        this.decorators = decorators;\n        this.startTime = decorators.getDateTimeStamp();\n    }\n\n    int getGcID() {\n        return gcID;\n    }\n\n    void setStartTime(DateTimeStamp startTime) {\n        this.startTime = startTime;\n    }\n\n    DateTimeStamp getStartTime() {\n        return startTime;\n    }\n\n    public void setDuration(double duration) {\n        this.duration = duration;\n    }\n\n    double getDuration() {\n        return duration;\n    }\n\n    public void setGCCause(GCCause cause) {\n        this.gcCause = cause;\n    }\n\n    GCCause getGCCause() {\n        return this.gcCause;\n    }\n\n    void setCPUSummary(CPUSummary summary) {\n        this.cpuSummary = summary;\n    }\n\n    CPUSummary getCPUSummary() {\n        return this.cpuSummary;\n    }\n\n    public void add(CPUSummary cpuSummary) {\n        this.cpuSummary = cpuSummary;\n    }\n\n    /**\n     * @return the decorators\n     */\n    Decorators getDecorators() {\n        return decorators;\n    }\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/SerialPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\npublic interface SerialPatterns extends SharedPatterns {\n\n    //DefNew patterns\n    //48.021: [GC48.021: [ParNew: 306686K->34046K(306688K), 0.3196120 secs] 1341473K->1125818K(8669952K), 0.3197540 secs]\n    GCParseRule DEFNEW = new GCParseRule(\"DEFNEW\", \"^(\" + GC_PREFIX + \")?\" + DATE_TIMESTAMP + \"\\\\[DefNew: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\");\n\n    //9.811: [GC 9.811: [ParNew\n    GCParseRule DEFNEW_TENURING = new GCParseRule(\"DEFNEW_TENURING\", GC_PREFIX + TIMESTAMP + \"\\\\[DefNew$\");\n\n    //2019-10-22T23:41:21.852+0000: 21.912: [GC (GCLocker Initiated GC) 2019-10-22T23:41:21.853+0000: 21.912: [DefNew2019-10-22T23:41:21.914+0000: 21.974: [SoftReference, 0 refs, 0.0000842 secs]2019-10-22T23:41:21.914+0000: 21.974: [WeakReference, 76 refs, 0.0000513 secs]2019-10-22T23:41:21.914+0000: 21.974: [FinalReference, 91635 refs, 0.0396861 secs]2019-10-22T23:41:21.954+0000: 22.014: [PhantomReference, 0 refs, 3 refs, 0.0000444 secs]2019-10-22T23:41:21.954+0000: 22.014: [JNI Weak Reference, 0.0000281 secs]: 419520K->19563K(471936K), 0.1019514 secs] 502104K->102148K(2044800K), 0.1020469 secs] [Times: user=0.09 sys=0.01, real=0.10 secs]\n    GCParseRule DEFNEW_DETAILS = new GCParseRule(\"DEF_NEW_DETAILS\", GC_PREFIX + \" \" + DATE_TIMESTAMP + \"\\\\[DefNew\" + REFERENCE_RECORDS + \": \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + \"\\\\[Times: user=\" + REAL_VALUE + \" sys=\" + REAL_VALUE + \", real=\" + PAUSE_TIME + \"\\\\]\");\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedGCLogParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\nabstract class UnifiedGCLogParser extends GCLogParser {\n\n    private static final Logger LOGGER = Logger.getLogger(UnifiedGCLogParser.class.getName());\n    private static final boolean DEBUG = Boolean.getBoolean(\"microsoft.debug\");\n\n    public UnifiedGCLogParser() {}\n\n    void advanceClock(String record) {\n        try {\n            DateTimeStamp now = new Decorators(record).getDateTimeStamp();\n            super.advanceClock(now);\n        } catch (Throwable t) {\n            LOGGER.log(Level.FINE, \"[PARSING ERROR] \" + record, t);\n        }\n    }\n\n    void notYetImplemented(GCLogTrace trace, String line) {\n        trace.notYetImplemented();\n    }\n\n    /**\n     * Some log entries require no actions\n     */\n    void noop() {\n        if (DEBUG)\n            System.out.println(\"noop\");\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/JVMEventParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.ApplicationConcurrentTime;\nimport com.microsoft.gctoolkit.event.jvm.ApplicationStoppedTime;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Set;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\npublic class JVMEventParser extends PreUnifiedGCLogParser implements JVMPatterns {\n\n    private static final double GCPAUSE_TIME_NOT_SET = -1.0; // a value that doesn't make sense\n    private static final Logger LOGGER = Logger.getLogger(JVMEventParser.class.getName());\n    private final Collection<SafePointData> safePoints = new ArrayList<>();\n    private DateTimeStamp timeStamp = new DateTimeStamp(0.0d);\n    private boolean lastEventWasGC = false;\n    private double gcPauseTime = GCPAUSE_TIME_NOT_SET;\n\n    public JVMEventParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.JVM);\n    }\n\n    public String getName() {\n        return \"JVMEventParser\";\n    }\n\n    /**\n     * Application stopped time records prior to 7.0 are not timestamped.\n     * In those cases one can use Application Run time to reconstruct the timings\n     * If Application run time is missing, collect records and estimate run times\n     * based on a change in the value of getClock();\n     */\n    @Override\n    protected void process(String line) {\n\n        GCLogTrace trace = null;\n\n        try {\n            //todo: unified safepointing here???\n            if ((trace = APPLICATION_STOP_TIME.parse(line)) != null) {\n                if (lastEventWasGC) {\n                    // can estimate TTSP\n                    double duration = trace.getDoubleGroup(3);\n                    publish(new ApplicationStoppedTime(trace.getDateTimeStamp(), duration, duration - gcPauseTime, lastEventWasGC));\n                    lastEventWasGC = false;\n                    gcPauseTime = GCPAUSE_TIME_NOT_SET;\n                } else {\n                    publish(new ApplicationStoppedTime(trace.getDateTimeStamp(), trace.getDoubleGroup(3), lastEventWasGC));\n                }\n            } else if ((trace = APPLICATION_STOP_TIME_WITH_STOPPING_TIME.parse(line)) != null) {\n                publish(new ApplicationStoppedTime(trace.getDateTimeStamp(), trace.getDoubleGroup(3), trace.getDoubleGroup(4), lastEventWasGC));\n                lastEventWasGC = false;\n                gcPauseTime = GCPAUSE_TIME_NOT_SET;\n            } else if ((trace = APPLICATION_TIME.parse(line)) != null) {\n                publish(new ApplicationConcurrentTime(trace.getDateTimeStamp(), trace.getDoubleGroup(3)));\n                lastEventWasGC = false;\n            } else if ((trace = SIMPLE_APPLICATION_STOP_TIME.parse(line)) != null) {\n                safePoints.add(new StoppedTime(trace.getDoubleGroup(1), safePoints.isEmpty()));\n            } else if ((trace = SIMPLE_APPLICATION_TIME.parse(line)) != null) {\n                safePoints.add(new ConcurrentTime(trace.getDoubleGroup(3)));\n            } else if ((trace = GC_PAUSE_CLAUSE.parse(line)) != null) {\n                gcPauseTime = trace.getPauseTime();\n                lastEventWasGC = true;\n            } else if ((trace = TLAB_START.parse(line)) != null) {\n                extractTLAB(trace, 2);\n            } else if ((trace = TLAB_CONT.parse(line)) != null) {\n                extractTLAB(trace, 0);\n            } else if ((trace = TLAB_TOTALS.parse(line)) != null) {\n                extractTLABSummary(trace);\n            } else if (line.equals(END_OF_DATA_SENTINEL)) {\n                // TODO: #154  else if (line.equals(END_OF_DATA_SENTINEL)|| (JVM_EXIT.parse(line) != null)) {\n                // if we see \"^heap\" then we're at the end of the log\n                // at issue is if logs have been concatenated then we're not at the end and we\n                // shouldn't release the\n                drainSafePoints();\n                publish(new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n            } else if (getClock().getTimeStamp() > timeStamp.getTimeStamp()) {\n                drainSafePoints();\n                timeStamp = getClock();\n            }\n\n        } catch (Throwable t) {\n            LOGGER.log(Level.FINE, \"Missed: {0}\", line);\n        }\n    }\n\n    private void extractTLABSummary(GCLogTrace trace) {\n        trace.notYetImplemented();\n    }\n\n    @SuppressWarnings(\"unused\")\n    private void extractTLAB(GCLogTrace trace, int offset) {\n        String gcThreadId = trace.getGroup(1 + offset);\n        int id = trace.getIntegerGroup(2 + offset);\n        int desiredSize = trace.getIntegerGroup(3 + offset);\n        int slowAllocs = trace.getIntegerGroup(4 + offset);\n        int refillWaste = trace.getIntegerGroup(5 + offset);\n        double allocFraction = trace.getDoubleGroup(6 + offset);\n        int unknownKBField = trace.getIntegerGroup(7 + offset);\n        int refills = trace.getIntegerGroup(8 + offset);\n        double wastePercent = trace.getDoubleGroup(9 + offset);\n        int gcUknownField = trace.getIntegerGroup(10 + offset);\n        int slowUnknown = trace.getIntegerGroup(11 + offset);\n        int fastUnknown = trace.getIntegerGroup(12 + offset);\n    }\n\n    //todo: should actually use the timings in the log but this is ok for now.\n    private void drainSafePoints() {\n        double interval = (getClock().getTimeStamp() - (timeStamp.getTimeStamp())) / (safePoints.size() + 1);\n        double timeValue = getClock().getTimeStamp() + interval;\n        for (SafePointData safePointData : safePoints) {\n            publish(safePointData.complete(new DateTimeStamp(timeValue)));\n            timeValue += interval;\n        }\n        safePoints.clear();\n    }\n\n    private abstract static class SafePointData {\n        double duration;\n\n        abstract JVMEvent complete(DateTimeStamp timeStamp1);\n\n    }\n\n    private static class StoppedTime extends SafePointData {\n\n        boolean gcInduced;\n\n        StoppedTime(double timing, boolean gc) {\n            duration = timing;\n            gcInduced = gc;\n        }\n\n        JVMEvent complete(DateTimeStamp dateTimeStamp) {\n            return new ApplicationStoppedTime(dateTimeStamp, duration, gcInduced);\n        }\n    }\n\n    private static class ConcurrentTime extends SafePointData {\n\n        ConcurrentTime(double timing) {\n            duration = timing;\n        }\n\n        JVMEvent complete(DateTimeStamp dateTimeStamp) {\n            return new ApplicationConcurrentTime(dateTimeStamp, duration);\n        }\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return (diary.isTLABData() || diary.isApplicationStoppedTime() || diary.isApplicationRunningTime()) && ! diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n\n    private void publish(JVMEvent event) {\n        super.publish(ChannelName.JVM_EVENT_PARSER_OUTBOX, event);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/ParallelPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\npublic interface ParallelPatterns extends SharedPatterns {\n\n    String PS_BLOCK = \"\\\\[(PSYoungGen|ParOldGen|Tenured|PSOldGen): \" + BEFORE_AFTER_CONFIGURED + \"\\\\]\";\n    String TENURED_BLOCK = \"\\\\[Tenured: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\";\n\n    //2017-03-28T12:17:34.744+0200: 1.895: [GC (Allocation Failure)  137969K->21806K(491008K), 0.0082985 secs]\n    //\"2017-03-28T12:17:33.823+0200: 0.974: [GC (System.gc())  92578K->9947K(491008K), 0.0135426 secs]\"\n    GCParseRule PSYOUNGGEN_NO_DETAILS = new GCParseRule(\"PSYOUNGGEN_NO_DETAILS\", GC_PREFIX + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]$\");\n\n    //939.183: [GC [PSYoungGen: 523744K->844K(547584K)] 657668K->135357K(1035008K), 0.0157986 secs] [Times: user=0.30 sys=0.01, real=0.02 secs]\n    GCParseRule PSYOUNGGEN = new GCParseRule(\"PSYOUNGGEN\", GC_PREFIX + PS_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\");\n    //\"2017-08-17T08:14:22.677-0500: 8179.905: [GC (Allocation Failure) --[PSYoungGen: 3990443K->3990443K(4030464K)] 5012990K->5013022K(5079040K), 0.5837002 secs] [Times: user=0.67 sys=0.00, real=0.58 secs] \\n\"\n    GCParseRule PSYOUNGGEN_PROMOTION_FAILED = new GCParseRule(\"PSYOUNGGEN_PROMOTION_FAILED\", GC_PREFIX + \"--\" + PS_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\");\n    GCParseRule PSFULL = new GCParseRule(\"PSFULL\", FULL_GC_PREFIX + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]$\");\n    //1.090: [GC (Allocation Failure) 1.095: [SoftReference, 0 refs, 0.0000208 secs]1.095: [WeakReference, 1098 refs, 0.0000495 secs]1.095: [FinalReference, 1518 refs, 0.0010004 secs]1.096: [PhantomReference, 0 refs, 1 refs, 0.0000041 secs]1.096: [JNI Weak Reference, 0.0000084 secs]\n    // [PSYoungGen: 34275K->2536K(34304K)] 36334K->8407K(76800K), 0.0073027 secs] [Times: user=0.01 sys=0.00, real=0.01 secs]\n    GCParseRule PSYOUNGGEN_REFERENCE = new GCParseRule(\"PSYOUNGGEN_REFERENCE\", GC_PREFIX + REFERENCE_RECORDS + PS_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED_PAUSE);\n    //18.814: [GC (System.gc()) 18.819: [SoftReference, 0 refs, 0.0000308 secs]18.819: [WeakReference, 305 refs, 0.0000138 secs]18.819: [FinalReference, 1504 refs, 0.0013137 secs]18.821: [PhantomReference, 0 refs, 0 refs, 0.0000059 secs]18.821: [JNI Weak Reference, 0.0000067 secs]\n    GCParseRule PSYOUNGGEN_REFERENCE_SPLIT = new GCParseRule(\"PSYOUNGGEN_REFERENCE_SPLIT\", GC_PREFIX + REFERENCE_RECORDS + \"$\");\n    GCParseRule PS_FULL_REFERENCE_SPLIT = new GCParseRule(\"PS_FULL_REFERENCE_SPLIT\", FULL_GC_PREFIX + REFERENCE_RECORDS + \"$\");\n    //43446.876: [Full GC (System.gc()) 43446.876: [Tenured43447.145: [SoftReference, 347 refs, 0.0000739 secs]43447.146: [WeakReference, 637 refs, 0.0000575 secs]43447.146: [FinalReference, 460 refs, 0.0004814 secs]43447.146: [PhantomReference, 0 refs, 4 refs, 0.0000047 secs]43447.146: [JNI Weak Reference, 0.0000713 secs]: 290960K->292298K(699072K), 0.6541540 secs] 460438K->292298K(1013824K), [Metaspace: 99359K->99359K(1140736K)], 0.6543427 secs] [Times: user=0.64 sys=0.00, real=0.65 secs]\n    //47047.534: [Full GC (System.gc()) 47047.534: [Tenured47047.808: [SoftReference, 258 refs, 0.0000630 secs]47047.808: [WeakReference, 628 refs, 0.0000561 secs]47047.808: [FinalReference, 170 refs, 0.0001564 secs]47047.808: [PhantomReference, 0 refs, 3 refs, 0.0000039 secs]47047.808: [JNI Weak Reference, 0.0000800 secs]: 292298K->278286K(699072K), 0.7450752 secs] 350258K->278286K(1013824K), [Metaspace: 99239K->99239K(1140736K)], 0.7451777 secs] [Times: user=0.74 sys=0.00, real=0.75 secs]\n    GCParseRule PS_FULL_REFERENCE_8 = new GCParseRule(\"PS_FULL_REFERENCE_8\", FULL_GC_PREFIX + \" \" + DATE_TIMESTAMP + \"\\\\[Tenured\" + REFERENCE_RECORDS + \": \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + \", \" + PAUSE_TIME);\n    GCParseRule SERIAL_FULL_REFERENCE = new GCParseRule(\"SERIAL_FULL_REFERENCE\", FULL_GC_PREFIX + DATE_TIMESTAMP + \"\\\\[Tenured\" + REFERENCE_RECORDS + \": \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \"\\\\, \" + PERM_RECORD + \", \" + PAUSE_TIME);\n    GCParseRule PS_FULL_REFERENCE = new GCParseRule(\"PS_FULL_REFERENCE\", FULL_GC_PREFIX + REFERENCE_RECORDS + \"\\\\s*\" + PS_BLOCK + \" \\\\[(?:PSFull|PSOldGen|ParOldGen): \" + BEFORE_AFTER_CONFIGURED + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \"(?:,)? \" + PERM_RECORD + \", \" + PAUSE_TIME);\n\n    //Moronic HotSpot/GC developers decided on yet another arbitrary format change... Why we need the comma... beyond me!\n    //22.014: [Full GC (Metadata GC Threshold) [PSYoungGen: 16865K->0K(107008K)] [ParOldGen: 108907K->103881K(124416K)] 125773K->103881K(231424K), [Metaspace: 21089K->21089K(1067008K)], 0.2715010 secs]\n    //0.221: [Full GC (Ergonomics) [PSYoungGen: 10736K->0K(76288K)] [ParOldGen: 120258K->115108K(175104K)] 130995K->115108K(251392K), [Metaspace: 4209K->4207K(1056768K)], 0.0235254 secs]\n    //42384.024: [Full GC [PSYoungGen: 1696K->0K(70464K)] [PSFull: 928442K->125867K(932096K)] 930139K->125867K(1002560K) [PSPermGen: 37030K->37030K(65536K)], 117.0312620 secs]\n    GCParseRule PS_FULL_GC_PERM = new GCParseRule(\"PS_FULL_GC_PERM\", FULL_GC_PREFIX + PS_BLOCK + \" \\\\[(?:PSFull|PSOldGen|ParOldGen): \" + BEFORE_AFTER_CONFIGURED + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \" \" + PERM_RECORD + \", \" + PAUSE_TIME);\n    GCParseRule PS_FULL_GC_META = new GCParseRule(\"PS_FULL_GC_META\", FULL_GC_PREFIX + PS_BLOCK + \" \\\\[(?:PSFull|PSOldGen|ParOldGen): \" + BEFORE_AFTER_CONFIGURED + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \"(?:,)? \" + META_RECORD + \", \" + PAUSE_TIME);\n\n    //1.147: [Full GC (System) 1.147: [Tenured: 1636K->1765K(5312K), 0.0488106 secs] 2894K->1765K(7616K), [Perm : 10112K->10112K(21248K)], 0.0488934 secs]\n    GCParseRule PS_FULL_GC_V2_PERM = new GCParseRule(\"PS_FULL_GC_V2_PERM\", FULL_GC_PREFIX + TIMESTAMP + \"\\\\[(PSYoungGen|ParOldGen|Tenured): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + \", \" + PAUSE_TIME);\n    GCParseRule PS_FULL_GC_V2_META = new GCParseRule(\"PS_FULL_GC_V2_META\", FULL_GC_PREFIX + TIMESTAMP + \"\\\\[(PSYoungGen|ParOldGen|Tenured): \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + META_RECORD + \", \" + PAUSE_TIME);\n\n    //: 4864K->512K(4864K), 0.0107187 secs]4.203: [Tenured: 11310K->10683K(11392K), 0.1046824 secs] 14990K->10683K(16256K), [Perm : 9880K->9880K(12288K)], 0.1157555 secs] [Times: user=0.11 sys=0.00, real=0.11 secs]\n    GCParseRule PS_PROMOTION_FAILED = new GCParseRule(\"PS_PROMOTION_FAILED\", \"^: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\" + TIMESTAMP + TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //2011-02-12T14:13:59.378+0000: 50128.698: [GC\n    //[PSYoungGen: 2763743K->13920K(2766144K)] 8155872K->5406505K(8358592K), 0.0572110 secs]\n    //12288K->1648K(47104K), 0.0084652 secs]\n    GCParseRule PS_TENURING_START = new GCParseRule(\"PS_TENURING_START\", GC_PREFIX + \"$\");\n    GCParseRule PSFULL_SPLIT = new GCParseRule(\"PSFULL_SPLIT\", FULL_GC_PREFIX + \"$\");\n    GCParseRule PS_DETAILS_WITH_TENURING = new GCParseRule(\"PS_DETAILS_WITH_TENURING\", \"^\" + PS_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\");\n    GCParseRule PS_SIMPLE_TENURING_END = new GCParseRule(\"PS_SIMPLE_TENURING_END\", \"^\" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\");\n\n    //13.056: [GC-- [PSYoungGen: 4194240K->4194240K(4194240K)] 4280449K->4360823K(4361088K), 0.4589570 secs]\n    GCParseRule PS_FAILURE = new GCParseRule(\"PS_FAILURE\", GC_PREFIX + \"-- \" + PS_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\");\n\n    /*\n        260962.621: [Full GC260962.902: [SoftReference, 8776 refs, 0.0013422 secs]260962.904: [WeakReference, 36789 refs, 0.0037704 secs]260962.907: [FinalReference, 1676 refs, 0.0048222 secs]260962.912: [PhantomReference, 0 refs, 279 refs, 0.0000250 secs]260962.912: [JNI Weak Reference, 0.0000196 secs]AdaptiveSizeStart: 260963.411 collection: 14063\n        81.705: [GC (Allocation Failure) AdaptiveSizePolicy::update_averages:  survived: 116398488  promoted: 90128  overflow: false\n...\n", "needle_token_start": 4027, "needle_token_end": 4081, "code_context_ntokens": 16161, "output": ["```java\nprivate void postEvacuatePhaseDuration(GCLogTrace trace, String line) {\n    forwardReference.postPostEvacuateCSetPhaseDuration(trace.getGroup(1), trace.getDurationInSeconds());\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "tagsContain", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To check if a specific string is present within a collection of strings.\n2. **Input**: A single string representing the item to be searched within the collection.\n3. **Output**: A boolean value; `true` if the item is found within the collection, otherwise `false`.\n4. **Procedure**: The method receives a string and checks if this string exists within a predefined collection of strings using the `contains` method of the collection.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/ParNewSerial.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\n/*\n   KCP - do not delete --- this is a deprecated GC combination. This interface is current not used but is retrained here for historical reasons.\n */\npublic interface ParNewSerial extends PreUnifiedTokens {\n\n    //12.008: [GC 12.008: [ParNew: 4680K->224K(4992K), 0.0009052 secs]12.009: [Tenured: 11421K->7456K(11448K), 0.0679043 secs] 15860K->7456K(16440K), [Perm : 10677K->10677K(21248K)], 0.0689327 secs] [Times: user=0.07 sys=0.00, real=0.07 secs]\n//    public static final Pattern PARNEW_SERIAL_FULL = Pattern.compile( DATE_TIMESTAMP + \"\\\\[GC \" + PARNEW_BLOCK + TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + \", \" + PAUSE_TIME);\n\n    //: 7077K->86K(7552K), 0.0019757 secs]20.774: [Tenured: 17007K->7966K(17080K), 0.0754072 secs] 23865K->7966K(24632K), [Perm : 10697K->10697K(21248K)], 0.0776402 secs]\n    //]20.774: [Tenured: 17007K->7966K(17080K), 0.0754072 secs] 23865K->7966K(24632K), [Perm : 10697K->10697K(21248K)], 0.0776402 secs]\n//    public static final Pattern PARNEW_SERIAL_FULL_SPLIT = Pattern.compile( \"^: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\" + TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + \", \" + PAUSE_TIME);\n\n/*\n\n    public static final Pattern PAR_OLD_REDUCTION = Pattern.compile(\"\\\\[ParOldGen: \" + FROM_TO_CONFIGURED + \"\\\\]\");\n    \n    public static final String  TENURED_AND_PERM_REDUCTION =    TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] ?\" + FROM_TO_CONFIGURED + \", \" + PERM_REDUCTION    + \", \" + PAUSE_TIME + \"\\\\]\";\n    public static final String  TENURED_AND_NO_PERM_REDUCTION = TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] ?\" + FROM_TO_CONFIGURED + \", \" + PERM_NO_REDUCTION + \", \" + PAUSE_TIME + \"\\\\]\";\n\n    //2.783: [Full GC 2.783: [Tenured: 2684K->3969K(8192K), 0.0569037 secs] 4603K->3969K(13696K), [Perm : 16382K->16382K(16384K)], 0.0569750 secs]\n    public static final Pattern PARALLEL_OLD_PERM_REDUCTION = Pattern.compile(\"\\\\[Full GC \" + TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED + \", \" + PAUSE_TIME + \"\\\\] \" + FROM_TO_CONFIGURED + \", \" + PERM_REDUCTION.toString() + \", \" + PAUSE_TIME + \"\\\\]\");\n    public static final Pattern PARALLEL_OLD_NO_PERM_REDUCTION = Pattern.compile(\"\\\\[Full GC \" + TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED + \", \" + PAUSE_TIME + \"\\\\] \" + FROM_TO_CONFIGURED + \", \" + PERM_NO_REDUCTION.toString() + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //TODO\n    //0.963: [ParNew: 16256K->1984K(18240K), 0.0689940 secs] 16256K->6122K(58752K), 0.0691170 secs]\n    public static final Pattern PARNEW = Pattern.compile( \"\\\\[ParNew: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\]\");\n    public static final Pattern PARNEW_ICMS = Pattern.compile(\"\\\\[ParNew: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] \" + FROM_TO_CONFIGURED + CMSPatterns.ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //25.118: [GC 25.118: [ParNew: 1258368K->143985K(1415616K), 0.0987560 secs] 1265723K->151341K(8231360K), 0.0989880 secs] [Times: user=0.96 sys=0.49, real=0.09 secs]\n    public static final Pattern PARNEW_COMPLETE = Pattern.compile( START_OF_GC_TIMESTAMP + \"\\\\[GC\\\\s*\" + GC_TIMESTAMP + PARNEW);\n\n    //25.030: [GC 25.030: [ParNew: 451942K->51072K(460096K), 1.7048460 secs] 451942K->115934K(12531840K) icms_dc=5 , 1.7050420 secs]\n...\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingLevel.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\npublic enum UnifiedLoggingLevel {\n\n    trace(\"trace\"),\n    debug(\"debug\"),\n    info(\"info\"),\n    warning(\"warning\"),\n    error(\"error\");\n\n    private final String label;\n\n    UnifiedLoggingLevel(String label) {\n        this.label = label;\n    }\n\n    public String getLabel() {\n        return label;\n    }\n\n    /**\n     * @param other logging level to compare this one to\n     * @return whether this logging level is lesser or equal to the other one\n     */\n    public boolean isLessThanOrEqualTo(UnifiedLoggingLevel other) {\n        return this.compareTo(other) <= 0;\n    }\n\n\n    /**\n     * @param other logging level to compare this one to\n     * @return whether this logging level is greater or equal to the other one\n     */\n    public boolean isGreaterThanOrEqualTo(UnifiedLoggingLevel other) {\n        return this.compareTo(other) >= 0;\n    }\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingTokens.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\n\nimport com.microsoft.gctoolkit.parser.GenericTokens;\n\nimport java.util.regex.Pattern;\n\npublic interface UnifiedLoggingTokens extends GenericTokens {\n\n    /*\n      JEP 158 Decorators\n        time -- Current time and date in ISO-8601 format\n        uptime -- Time since the start of the JVM in seconds and milliseconds (e.g., 6.567s)\n        timemillis -- The same value as generated by System.currentTimeMillis()\n        uptimemillis -- Milliseconds since the JVM started\n        timenanos -- The same value as generated by System.nanoTime()\n        uptimenanos -- Nanoseconds since the JVM started\n        pid -- The process identifier\n        tid -- The thread identifier\n        level -- The level associated with the log message\n        tags -- The tag-set associated with the log message\n        [2018-04-04T09:10:00.586-0100][0.018s][1522825800586ms][18ms][10026341461044ns][17738937ns][1375][7427][info][gc] Using G1\n\n        Note: Decorators are outputed in the order of the list above.\n     */\n\n    String DATE_STAMP = \"\\\\[\" + DATE + \"\\\\]\";\n    String UPTIME = \"\\\\[\" + INTEGER + DECIMAL_POINT + \"\\\\d{3}s\\\\]\";\n    String TIME_MILLIS = \"\\\\[\\\\d+ms\\\\]\";\n    String TIME_NANOS = \"\\\\[\\\\d+ns\\\\]\";\n    String PID_TID = \"\\\\[\\\\d+\\\\]\";\n    String UNIFIED_LOG_LEVEL_BLOCK = \"\\\\[(?:error|warning|info|debug|trace|develop) *\\\\]\";\n    Pattern DECORATORS = Pattern.compile(\"(\" + DATE_STAMP + \")?(\" + UPTIME + \")?(\" + TIME_MILLIS + \")?(\" + TIME_MILLIS + \")?(\" + TIME_NANOS + \")?(\" + TIME_NANOS + \")?(\" + PID_TID + \")?(\" + PID_TID + \")?(\" + UNIFIED_LOG_LEVEL_BLOCK + \")?\");\n    //Using zero-width negative lookbehind to miss capturing records formatted like [0x1f03].\n    //[0.081s][trace][safepoint] Thread: 0x00007fd0d2006800  [0x1f03] State: _at_safepoint _has_called_back 0 _at_poll_safepoint 0\n    Pattern TAGS = Pattern.compile(\".*(?<=^|\\\\])\\\\[([a-z0-9,. ]+)\\\\]\");\n\n    String UNIFIED_META_RECORD = \"Metaspace: \" + BEFORE_AFTER_CONFIGURED;\n    String WORKER_SUMMARY_REAL = \"Min:\\\\s+\" + TIME + \", Avg:\\\\s+\" + TIME + \", Max:\\\\s+\" + TIME + \", Diff:\\\\s+\" + TIME + \", Sum:\\\\s+\" + TIME + \", Workers:\\\\s+\" + COUNTER;\n    String WORKER_SUMMARY_INT = \"Min:\\\\s+\" + COUNTER + \", Avg:\\\\s+\" + TIME + \", Max:\\\\s+\" + COUNTER + \", Diff:\\\\s+\" + COUNTER + \", Sum:\\\\s+\" + COUNTER + \", Workers:\\\\s+\" + COUNTER;\n    String POOL_SUMMARY = \"used \" + COUNTER + \"K, capacity \" + COUNTER + \"K, committed \" + COUNTER + \"K, reserved \" + COUNTER + \"K\";\n    String REGION_MEMORY_BLOCK = COUNTER + \"->\" + COUNTER + \"(?:\\\\(\" + COUNTER + \"\\\\))?\";\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\nimport com.microsoft.gctoolkit.parser.unified.UnifiedLoggingLevel;\nimport com.microsoft.gctoolkit.parser.unified.UnifiedLoggingTokens;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.time.ZonedDateTime;\nimport java.time.format.DateTimeFormatter;\nimport java.time.temporal.TemporalAccessor;\nimport java.util.Arrays;\nimport java.util.Optional;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\nimport java.util.regex.Matcher;\n\npublic class Decorators {\n\n    /**\n     * This class represents the decorators found in the log line as defined by JEP 158 (https://openjdk.java.net/jeps/158).\n     * The list, taken from that document is as:\n     * time -- Current time and date in ISO-8601 format\n     * uptime -- Time since the start of the JVM in seconds and milliseconds (e.g., 6.567s)\n     * timemillis -- The same value as generated by System.currentTimeMillis()\n     * uptimemillis -- Milliseconds since the JVM started\n     * timenanos -- The same value as generated by System.nanoTime()\n     * uptimenanos -- Nanoseconds since the JVM started\n     * pid -- The process identifier\n     * tid -- The thread identifier\n     * level -- The level associated with the log message\n     * tags -- The tag-set associated with the log message\n     *\n     * This implementation takes advantage of the property that the ordering of tags is stable. For example,\n     * -Xlog:*::pid,time produces the identical ordering as -Xlog:*::time,pid. Moreover, the list of decorators\n     * is dedupped implying that the second decorator in -Xlog:*::time,time is ignored.\n     *\n     * The default decorators (as of the time of authoring JDK 9-21) include [uptime][level][tags].\n     * For example, [1.361s][info][gc,heap]. This reads uptime of 1.361 seconds. The tag for the log record\n     * is gc.heap at the info level.\n     *\n     * At issue is that uptime and timemillis are, on the surface, indistinguishable. The same is true with\n     * timenanos and uptimenanos as well as with pid and tid. The following logic can be used to help differentiate\n     * indistinguishable decorators.\n     *\n     * 1) If both decorators are present then order can be used to differentiate\n     * 2) If ms time value - 20 years > 0, then timemillis can be assumed. Otherwise uptime is assumed. The reasoning\n     *    is, unified logging didn't exist that long ago. The value of 20 has been arbitrarily chosen.\n     * 3) There is no reliable way to differentiate the nanosecond timings if only 1 is present. However this may not\n     *    matter as GCToolKit would only use these values to create a baseline measure to align date/time with uptime.\n     *    This technique is designed to work-around the cases where logs do not contain the uptime decorator.\n     * 4) There is no known way to reliably differentiate between PID and TID. At this time, GCToolKit ignores these\n     *    decorators.\n     *\n     * Todo: GCToolkit captures time in the DateTimeStamp class. That class will capture uptime or time or both. If both\n     * are missing, GCToolkit JVMEvents will have no sense of time. It is possible that the other timing fields could fill\n     * in cases where both the time and uptime decorators were missing.\n     * @param line\n     */\n\n    private static final Logger LOGGER = Logger.getLogger(Decorators.class.getName());\n\n    // This is to help differentiate between JVM running time and wall clock time.\n    private static final long TWENTY_YEARS_IN_MILLIS = 731L * 24L * 60L * 60L * 1000L;\n    private static final long TWENTY_YEARS_IN_NANO = 731L * 24L * 60L * 60L * 1000L;\n\n    int numberOfDecorators;\n\n    private Matcher decoratorMatcher = null;\n    private String tags;\n\n    public Decorators(String line) {\n        extractValues(line);\n    }\n\n    private void extractValues(String line) {\n\n        if (!line.startsWith(\"[\"))\n            return;\n\n        decoratorMatcher = UnifiedLoggingTokens.DECORATORS.matcher(line);\n        if (!decoratorMatcher.find()) {\n            return;\n        }\n\n        for ( int i = 1; i <= decoratorMatcher.groupCount(); i++) {\n            if ( decoratorMatcher.group(i) != null)\n                numberOfDecorators++;\n        }\n\n        Matcher tagMatcher = UnifiedLoggingTokens.TAGS.matcher(line);\n        if (tagMatcher.find()) {\n            numberOfDecorators++;\n            tags = String.join(\",\", Arrays.asList(tagMatcher.group(1).trim().split(\",\")));\n        }\n    }\n\n    // For some reason, ISO_DATE_TIME doesn't like that time-zone is -0100. It wants -01:00.\n    private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n\n    public ZonedDateTime getDateStamp() {\n        try {\n            String value = decoratorMatcher.group(1);\n            if (value != null) {\n                TemporalAccessor temporalAccessor = formatter.parse(value.substring(1, value.length()-1));\n                return ZonedDateTime.from(temporalAccessor);            }\n        } catch (NullPointerException npe) {\n            LOGGER.log(Level.SEVERE, npe.getMessage(), npe);\n        }\n        return null;\n    }\n\n    public double getUpTime() {\n        String value = decoratorMatcher.group(2);\n        if (value != null) {\n            value = value.replace(\",\", \".\");\n            return Double.parseDouble(unboxValue(value, 1));\n        }\n        return -1.0d;\n    }\n\n    private long extractClock(int groupIndex, long threshold) {\n        long clockReading = -1L;\n        String stringValue = decoratorMatcher.group(groupIndex);\n        if (stringValue != null) {\n            clockReading = Long.parseLong(unboxValue(stringValue, 2));\n            if (decoratorMatcher.group(groupIndex + 1) == null)\n                if (clockReading < threshold)\n                    clockReading = -1L;\n        }\n        return clockReading;\n    }\n\n    public long getTimeMillis() {\n        return extractClock(3, TWENTY_YEARS_IN_MILLIS);\n    }\n\n    public long getUptimeMillis() {\n        String value = decoratorMatcher.group(4);\n        if (value == null) {\n            value = decoratorMatcher.group(3);\n        }\n        if (value != null) {\n            long longValue = Long.parseLong(unboxValue(value, 2));\n            if (longValue < TWENTY_YEARS_IN_MILLIS)\n                return longValue;\n        }\n        return -1L;\n    }\n\n    public long getTimeNano() {\n        return extractClock(5, TWENTY_YEARS_IN_NANO);\n    }\n\n    public long getUptimeNano() {\n        String value = decoratorMatcher.group(6);\n        if (value == null) {\n            value = decoratorMatcher.group(5);\n        }\n        if (value != null) {\n            long longValue = Long.parseLong(unboxValue(value, 2));\n            if (longValue < TWENTY_YEARS_IN_NANO)\n                return longValue;\n        }\n        return -1L;\n    }\n\n    public int getPid() {\n        String value = decoratorMatcher.group(7);\n        if (value != null) {\n            return Integer.parseInt(unboxValue(value));\n        }\n        return -1;\n    }\n\n    public int getTid() {\n        String value = decoratorMatcher.group(8);\n        if (value != null) {\n            return Integer.parseInt(unboxValue(value));\n        }\n        return -1;\n    }\n\n    public Optional<UnifiedLoggingLevel> getLogLevel() {\n        String level = decoratorMatcher.group(9);\n        if (level != null)\n            try {\n                return Optional.of(UnifiedLoggingLevel.valueOf(unboxValue(level)));\n            } catch (IllegalArgumentException e) {\n                LOGGER.fine(\"No such debug level: \" + level);\n                LOGGER.fine(e.getMessage());\n                return Optional.empty();\n            }\n        return Optional.empty();\n    }\n\n    public DateTimeStamp getDateTimeStamp() {\n        return new DateTimeStamp(getDateStamp(), getUpTime());\n    }\n\n    public int getNumberOfDecorators() {\n        return numberOfDecorators;\n    }\n\n    private String unboxValue(String boxedValue, int postFix) {\n        return boxedValue.substring(1, boxedValue.length() - (1 + postFix));\n    }\n\n    private String unboxValue(String boxedValue) {\n        return unboxValue(boxedValue, 0).trim();\n    }\n\n    \npublic boolean tagsContain(String tagList) {\n        return tags.contains(tagList);\n    }\n\n    public String getTags() {\n        return tags;\n    }\n}\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.MetaspaceRecord;\nimport com.microsoft.gctoolkit.event.jvm.PermGenSummary;\nimport com.microsoft.gctoolkit.io.GCLogFile;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.DataSourceParser;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\npublic abstract class GCLogParser implements DataSourceParser, SharedPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(GCLogParser.class.getName());\n    public static final String END_OF_DATA_SENTINEL = GCLogFile.END_OF_DATA_SENTINEL;\n\n    public static final GCParseRule GCID_COUNTER = new GCParseRule(\"GCID_COUNTER\", \" GC\\\\((\\\\d+)\\\\) \");\n    private JVMEventChannel consumer;\n    protected Diary diary;\n    private DateTimeStamp clock = new DateTimeStamp(DateTimeStamp.EPOC, 0.0d);\n    private double lastDuration = 0.0d;\n\n\n    public GCLogParser() {}\n\n    /**\n     * Setting the diary will set the current clock time to the time of the first event\n     * in the GC log. This is a better choice than EPOC though EPOC is better than null.\n     * @param diary summary the GC log.\n     */\n    @Override\n    public void diary(Diary diary) {\n        this.diary = diary;\n        this.clock = diary.getTimeOfFirstEvent();\n    }\n\n    public DateTimeStamp getClock() {\n        return clock;\n    }\n\n    public void setClock(DateTimeStamp newValue) {\n        this.clock = newValue;\n    }\n\n    public abstract String getName();\n\n    protected abstract void process(String trace);\n\n    abstract void advanceClock(String record);\n\n    /**\n     * The assumption is, this method will manage the global clock and thus should never be over ridden\n     * @param now - DateTimeStamp from the current GC log record\n     */\n    protected final void advanceClock(DateTimeStamp now) {\n        if (now == null)\n            return;\n        // now can be the same but it can't be less than\n        else if (now.before(getClock())) {\n            LOGGER.log(Level.WARNING, \"Log File may be Corrupted: Time traveled backwards from {0} to {1}\", new Object[]{getClock().toString(), now.toString()});\n        }\n        //todo: should preserve date in cases where statements do not respect PrintDateStamp (eg. ergonomics)\n        setClock(now);\n    }\n\n    /**\n     * The clock is advanced to the time at the end of the event.\n     * The times reported in the GCLog file are \"noisy\" at best. This\n     * hyperactives the clock rollback detection which doesn't allow\n     * for slop. To minimize this, the clock only records when a time\n     * stamp is presented and isn't advanced to the end of the event\n     * except in the case of EOF where the time of JVM termination has\n     * not been recorded. In this case, JVMTermination time should advanced\n     * to after the previous event has ended.\n     *\n     * @param channel\n     * @param event\n     */\n    public void publish(ChannelName channel, JVMEvent event) {\n        lastDuration = event.getDuration();\n        consumer.publish(channel,event);\n    }\n\n    public void receive(String trace) {\n        if (!trace.equals(END_OF_DATA_SENTINEL))\n            advanceClock(trace);\n        else\n            advanceClock(getClock().add(lastDuration));\n        process(trace);\n    }\n\n    boolean isPreJDK17040() {\n        return diary.isPre70_40();\n    }\n\n    boolean hasPrintGCDetails() {\n        return diary.isPrintGCDetails();\n    }\n\n    // todo: mixes aggregator with parsing. premature optimization...\n    MemoryPoolSummary getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(GCLogTrace trace, int offset) {\n        MemoryPoolSummary summary = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(offset);\n        return summary;\n    }\n\n    MemoryPoolSummary getTotalOccupancyWithTotalHeapSizeSummary(GCLogTrace trace, int offset) {\n        MemoryPoolSummary summary = trace.getOccupancyWithMemoryPoolSizeSummary(offset);\n        return summary;\n    }\n\n    GCLogTrace extractReferenceBlock(String line, GCParseRule rule) {\n        return rule.parse(line);\n    }\n\n    ReferenceGCSummary extractPrintReferenceGC(String line) {\n\n        ReferenceGCSummary summary = new ReferenceGCSummary();\n        GCLogTrace trace;\n        if ((trace = extractReferenceBlock(line, SOFT_REFERENCE)) != null)\n            summary.addSoftReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n\n        if ((trace = extractReferenceBlock(line, WEAK_REFERENCE)) != null)\n            summary.addWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n\n        if ((trace = extractReferenceBlock(line, FINAL_REFERENCE)) != null)\n            summary.addFinalReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n\n        if ((trace = extractReferenceBlock(line, PHANTOM_REFERENCE)) != null) {\n            if (trace.groupNotNull(4))\n                summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getIntegerGroup(4), trace.getDuration());\n            else\n                summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n        }\n\n        if ((trace = extractReferenceBlock(line, JNI_REFERENCE)) != null) {\n            if (trace.groupNotNull(3))\n                summary.addJNIWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n            else\n                summary.addJNIWeakReferences(trace.getDateTimeStamp(), trace.getDuration());\n        }\n\n        return summary;\n\n    }\n\n    /*\n     * [PSPermGen: 7034K->7034K(21248K)]\n     * [CMS Perm : 10696K->10696K(21248K)]\n     * [Perm : 16382K->16382K(16384K)]\n     * [Perm : 16382K(16384K)]\n     *\n     * @param line from GC log file\n     */\n    MemoryPoolSummary extractPermOrMetaspaceRecord(String line) {\n        GCLogTrace trace;\n        MemoryPoolSummary metaDataPool = null;\n        if ((trace = PERM_SPACE_RECORD.parse(line)) != null) {\n            String type = trace.getGroup(1).trim();\n            if (\"CMS Perm\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"PS Perm\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"PSPermGen\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"Perm\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"Metaspace\".equals(type)) {\n                if (trace.getGroup(2) != null) {\n                    metaDataPool = new MetaspaceRecord(trace.toKBytes(2), trace.toKBytes(4), trace.toKBytes(6));\n                } else {\n                    metaDataPool = new MetaspaceRecord(trace.toKBytes(4), trace.toKBytes(4), trace.toKBytes(6));\n                }\n            }\n        } else if ((trace = META_SPACE_RECORD.parse(line)) != null) {\n            int index = (trace.getGroup(1) == null) ? 1 : 3;\n            metaDataPool = new MetaspaceRecord(trace.toKBytes(index), trace.toKBytes(3), trace.toKBytes(5));\n        }\n\n        return metaDataPool;\n    }\n\n    MemoryPoolSummary extractPermGenRecord(GCLogTrace trace) {\n        int index = (trace.getGroup(2) == null) ? 2 : 4;\n        return new PermGenSummary(trace.getLongGroup(index), trace.getLongGroup(4), trace.getLongGroup(6));\n    }\n\n    int extractGCID(String line) {\n        GCLogTrace trace = GCID_COUNTER.parse(line);\n        return (trace != null) ? trace.getIntegerGroup(1) : -1;\n    }\n\n    CPUSummary extractCPUSummary(String line) {\n        GCLogTrace trace;\n        if ((trace = CPU_BREAKDOWN.parse(line)) != null) {\n            return new CPUSummary(trace.getDoubleGroup(1), trace.getDoubleGroup(2), trace.getDoubleGroup(3));\n        }\n        return null;\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel channel) {\n        this.consumer = channel;\n    }\n\n    @Override\n    public ChannelName channel() {\n        return ChannelName.DATA_SOURCE;\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\n\nimport com.microsoft.gctoolkit.parser.GCLogParser;\nimport com.microsoft.gctoolkit.parser.GCParseRule;\n\npublic interface UnifiedPatterns extends UnifiedLoggingTokens {\n\n    GCParseRule CPU_BREAKOUT = new GCParseRule(\"CPU_BREAKOUT\", \"User=\" + TIME + \"s Sys=\" + TIME + \"s Real=\" + TIME + \"s\");\n\n    GCParseRule GC_COUNT = new GCParseRule(\"GC_COUNT\", \" GC\\\\((\" + INTEGER + \")\\\\) \");\n\n    String WORKER_TASK = \"(for evacuation|to rebuild remembered set)\";\n    GCParseRule WORKER_SUMMARY = new GCParseRule(\"WORKER_SUMMARY\", \"Using \" + COUNTER + \" workers of \" + COUNTER + \" \" + WORKER_TASK);\n\n    GCParseRule REFERENCES = new GCParseRule(\"REFERENCES\", \"(Preclean )?(SoftReference|WeakReference|FinalReference|PhantomReference|JNI Weak Reference)(:?s)? \" + PAUSE_TIME);\n    GCParseRule REFERENCE_COUNTS = new GCParseRule(\"REFERENCE_COUNTS\", \"Ref Counts: Soft: \" + COUNTER + \" Weak: \" + COUNTER + \" Final: \" + COUNTER + \" Phantom: \" + COUNTER);\n\n    GCParseRule UNIFIED_META_DATA = new GCParseRule(\"UNIFIED_META_DATA\", UNIFIED_META_RECORD);\n\n    GCParseRule END_OF_FILE = new GCParseRule(\"END_OF_FILE\", GCLogParser.END_OF_DATA_SENTINEL);\n    GCParseRule JVM_EXIT = new GCParseRule(\"JVM_EXIT\", \"Heap$\");\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedSurvivorMemoryPoolParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.event.jvm.SurvivorRecord;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\n\nimport java.util.Set;\n\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.CPU_BREAKOUT;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.JVM_EXIT;\n\npublic class UnifiedSurvivorMemoryPoolParser extends UnifiedGCLogParser implements TenuredPatterns {\n\n    /**\n     * [16.962s][debug][gc,age       ] GC(14) Desired survivor size 10485760 bytes, new threshold 15 (max threshold 15)\n     * [16.973s][trace][gc,age       ] GC(14) Age table with threshold 15 (max threshold 15)\n     * [16.973s][trace][gc,age       ] GC(14) - age   1:     768744 bytes,     768744 total\n     * ...\n     * [16.974s][trace][gc,age       ] GC(14) - age  14:     542328 bytes,    8307008 total\n     */\n    private GCParseRule DESIRED_SURVIVOR_SIZE = new GCParseRule(\"DESIRED_SURVIVOR_SIZE\", \"Desired survivor size \" + COUNTER + \" bytes, new threshold \" + COUNTER + \" \\\\(max threshold \" + COUNTER + \"\\\\)\");\n    private GCParseRule AGE_TABLE_HEADER = new GCParseRule(\"AGE_TABLE_HEADER\", \"Age table with threshold \" + COUNTER + \" \\\\(max threshold \" + COUNTER + \"\\\\)\");\n    private GCParseRule AGE_RECORD = new GCParseRule(\"AGE_RECORD\", \"- age\\\\s+\" + COUNTER + \":\\\\s+\" + COUNTER + \" bytes,\\\\s+\" + COUNTER + \" total\");\n\n    private SurvivorRecord forwardReference = null;\n    private boolean ageDataCollected = false;\n\n    public UnifiedSurvivorMemoryPoolParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.SURVIVOR);\n    }\n\n    public String getName() {\n        return \"SurvivorMemoryPoolParser\";\n    }\n\n    @Override\n    protected void process(String entry) {\n        GCLogTrace trace;\n\n        if ((trace = DESIRED_SURVIVOR_SIZE.parse(entry)) != null) {\n            forwardReference = new SurvivorRecord(new Decorators(entry).getDateTimeStamp(), trace.getLongGroup(1), trace.getIntegerGroup(2), trace.getIntegerGroup(3));\n        } else if ((trace = AGE_TABLE_HEADER.parse(entry)) != null) {\n            //we've collected this data so.. eat it...\n        } else if ((trace = AGE_RECORD.parse(entry)) != null) {\n            if (forwardReference != null) {\n                forwardReference.add(trace.getIntegerGroup(1), trace.getLongGroup(2));\n                ageDataCollected = true;\n            }\n        } else if (entry.equals(END_OF_DATA_SENTINEL) || (JVM_EXIT.parse(entry) != null)) {\n            if (forwardReference != null)\n                publish(forwardReference);\n            publish(new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n        } else if (forwardReference != null && ageDataCollected) {\n            publish(forwardReference);\n            forwardReference = null;\n            ageDataCollected = false;\n        } else if (CPU_BREAKOUT.parse(entry) != null) {\n            if (forwardReference != null) {\n                publish(forwardReference);\n                forwardReference = null;\n                ageDataCollected = false;\n            }\n        }\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return diary.isTenuringDistribution() && diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n\n    private void publish(JVMEvent event) {\n        super.publish(ChannelName.SURVIVOR_MEMORY_POOL_PARSER_OUTBOX, event);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedG1GCParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.GCToolKit;\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.StatisticalSummary;\nimport com.microsoft.gctoolkit.event.SurvivorMemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentScanRootRegion;\nimport com.microsoft.gctoolkit.event.g1gc.G1Cleanup;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentCleanup;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentMark;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentMarkResetForOverflow;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentStringDeduplication;\nimport com.microsoft.gctoolkit.event.g1gc.G1FullGCNES;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCConcurrentEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCPauseEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1Mixed;\nimport com.microsoft.gctoolkit.event.g1gc.G1Remark;\nimport com.microsoft.gctoolkit.event.g1gc.G1SystemGC;\nimport com.microsoft.gctoolkit.event.g1gc.G1Trap;\nimport com.microsoft.gctoolkit.event.g1gc.G1Young;\nimport com.microsoft.gctoolkit.event.g1gc.G1YoungInitialMark;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.collection.MRUQueue;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.AbstractMap;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentLinkedQueue;\nimport java.util.function.BiConsumer;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n/**\n * TODO No reports or views generated from this data yet.\n * <p>\n * Result\n * -0 on when GC started\n * - type of GC triggered\n * - from, to, configured\n * - pause time if it is reported or can be calculated\n */\npublic class PreUnifiedG1GCParser extends PreUnifiedGCLogParser implements G1GCPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(PreUnifiedG1GCParser.class.getName());\n\n    //values show up at the end of GC log file from a normally terminated JVM\n    @SuppressWarnings(\"unused\")\n    private long heapTotal, heapUsed;\n    @SuppressWarnings(\"unused\")\n    private long regionSize;\n    @SuppressWarnings(\"unused\")\n    private long metaSpaceUsed, metaCapacity, metaCommitted, metaReserved;\n    @SuppressWarnings(\"unused\")\n    private long classSpaceUsed, classSpaceCapacity, classSpaceCommitted, classSpaceReserved;\n\n    private final G1GCPauseEvent trap = new G1Trap();\n    private G1GCPauseEvent forwardReference = trap;\n\n    private DateTimeStamp timeStampForwardReference;\n    private GCCause gcCauseForwardReference;\n    private GarbageCollectionTypes collectionTypeForwardReference;\n    private ReferenceGCSummary referenceGCForwardReferenceSummary;\n\n    private DateTimeStamp concurrentPhaseStartTimeStamp;\n    private GarbageCollectionTypes concurrentCollectionTypeForwardReference;\n    private final ConcurrentLinkedQueue<JVMEvent> backlog = new ConcurrentLinkedQueue<>();\n\n    private final MRUQueue<GCParseRule, BiConsumer<GCLogTrace, String>> parseRules;\n\n    {\n        parseRules = new MRUQueue<>();\n        parseRules.put(CPU_BREAKDOWN, this::recordCPUSummary);\n        parseRules.put(G1_DETAILS, this::processYoungGenCollection);\n        parseRules.put(YOUNG, this::processYoung);\n        parseRules.put(FULL_GC, this::g1gcFullGC); //113.325: [Full GC (System.gc()) 37M->32M(96M), 0.0943030 secs]\n        parseRules.put(G1_MEMORY_SUMMARY, this::processMemorySummary);\n        parseRules.put(G1_NO_DETAILS_MEMORY_SUMMARY, this::processNoDetailsMemorySummary);\n        parseRules.put(G1_PARALLEL_PHASE_SUMMARY, this::processParallelPhaseSummary);\n        parseRules.put(G1_SOLARIS_PARALLEL_PHASE, this::processParallelPhase);\n        parseRules.put(TERMINATION_ATTEMPTS, this::ignore);\n        parseRules.put(PROCESSED_BUFFERS, this::processedBuffers);\n        parseRules.put(PROCESSED_BUFFER, this::processedBuffer);\n        parseRules.put(SOLARIS_WORKER_PARALLEL_BLOCK, this::solarisWorkerParallelBlock);\n        parseRules.put(SOLARIS_WORKER_PARALLEL_ACTIVITY, this::ignore);\n        parseRules.put(G1GC_PHASE_DETAIL_CLAUSE, this::processPhaseDetailClause);\n        parseRules.put(G1GC_PHASE, this::processG1GCPhase);\n        parseRules.put(CONCURRENT_STRING_DEDUP, this::concurrentStringDedup);\n        parseRules.put(STRING_DEDUP_FIXUP, this::stringDedupFixup);\n        parseRules.put(QUEUE_FIXUP, this::recordQueueFixup);\n        parseRules.put(TABLE_FIXUP, this::recordTableFixup);\n        parseRules.put(WORKER_ACTIVITY, this::workerActivity);\n        parseRules.put(PARALLEL_TIME, this::parallelTime);\n        parseRules.put(WORKER_PARALLEL_BLOCK, this::workerParallelBlock);\n        parseRules.put(G1_FUll, this::g1gcDetailedFullGC);\n        parseRules.put(G1_FULL_INTERRUPTS_CONCURRENT_CYCLE, this::g1FullInterruptsConcurrentCycle);\n        parseRules.put(FULL_WITH_CONCURRENT_PHASE_START, this::g1FullWithConcurrentPhaseStart);\n        parseRules.put(FULL_WITH_CONCURRENT_PHASE_CORRUPTED, this::g1FullWithConcurrentPhaseCorrupted);\n        parseRules.put(FULL_MISSING_TIMESTAMP_CONCURRENT_START, this::fullMissingTimeStampWithConcurrentStart);\n        parseRules.put(FULL_WITH_CONCURRENT_PHASE_INTERLEAVED, this::g1FullWIthConcurrentPhaseInterleaved);\n        parseRules.put(FULL_WITH_CONCURRENT_END, this::fullWithConcurrentEnd);\n\n        parseRules.put(FULL_GC_FRAGMENT, this::fullGCFragment);\n        parseRules.put(CONCURRENT_START_V3, this::concurrentStartV3);\n        parseRules.put(CONCURRENT_START_V4, this::concurrentStartV4);\n        parseRules.put(CONCURRENT_START_V5, this::concurrentStartV3);\n\n        parseRules.put(CORRUPTED_CONCURRENT_START, this::corruptedConcurrentStart);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V2, this::corruptedConcurrentStartV2);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V3, this::corruptedConcurrentStartV3);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V4, this::corruptedConcurrentStartV4);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V5, this::corruptedConcurrentStartV5);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V7, this::corruptedConcurrentStartV7);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V8, this::corruptedConcurrentStartV8);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V9, this::corruptedConcurrentStartV9);\n        parseRules.put(YOUNG_WITH_CONCURRENT_END, this::youngWithConcurrentEnd);\n        parseRules.put(YOUNG_SPLIT_AT_DATESTAMP, this::youngSplitAtDatestamp);\n        parseRules.put(YOUNG_SPLIT_AT_TIMESTAMP, this::youngSplitAtTimestamp);\n        parseRules.put(FREE_FLOATING_YOUNG_BLOCK, this::freeFloatingYoungBlock);\n        parseRules.put(FULLGC_WITH_CONCURRENT_PHASE, this::fullGCWithConcurrentPhase);\n\n        parseRules.put(G1_YOUNG_SPLIT_START, this::g1YoungSplitStart);\n        parseRules.put(G1_YOUNG_SPLIT_END, this::g1YoungSplitEnd);\n        parseRules.put(G1_INITIAL_MARK, this::g1InitialMark);\n        parseRules.put(FREE_FLOATING_OCCUPANCY_SUMMARY, this::freeFloatingOccupancySummary);\n\n        parseRules.put(G1_CONCURRENT_START, this::g1ConcurrentStart);\n        parseRules.put(G1_CONCURRENT_START_WITHOUT_PREFIX, this::g1ConcurrentStartWithoutPrefix);\n        parseRules.put(G1_CONCURRENT_END, this::g1ConcurrentEnd);\n        parseRules.put(G1_REMARK, this::g1Remark);\n        parseRules.put(G1_180_REMARK, this::g1180Remark);\n        parseRules.put(G1_180_REMARK_REF_DETAILS, this::g1180RemarkRefDetails);\n        parseRules.put(G1_CLEANUP, this::g1Cleanup);\n        parseRules.put(G1_CLEANUP_NO_MEMORY, this::g1CleanupNoMemory);\n        parseRules.put(G1_CONCURRENT_ABORT, this::g1ConcurrentAbort);\n        parseRules.put(CONCURRENT_MARK_OVERFLOW, this::concurrentMarkOverflow);\n        parseRules.put(G1_CORRUPTED_CONCURRENT_END, this::g1CorruptedConcurrentEnd);\n        parseRules.put(G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END, this::g1ConcurrentEndCorruptedByApplicationTime);\n        parseRules.put(G1_FLOATING_CONCURRENT_PHASE_START, this::g1FloatingConcurrentPhaseStart);\n        parseRules.put(SPLIT_CLEANUP, this::splitCleanup);\n\n        //adaptive sizing\n        parseRules.put(YOUNG_SPLIT_BY_G1ERGONOMICS, this::youngSplitByG1Ergonomics);\n        parseRules.put(G1_INITIAL_MARK_ERGONOMICS, this::g1InitialMarkErgonomics);\n\n        //RSet rules\n        parseRules.put(G1_YOUNG_RS_SUMMARY, this::ignore);\n        parseRules.put(RSET_HEADER, this::ignore);\n        parseRules.put(RSET_CONCONCURRENT_HEADER, this::ignore);\n        parseRules.put(RSET_CONCURRENT_RS_Threads, this::ignore);\n        parseRules.put(RSET_CONCURRENT_MUTATOR, this::ignore);\n        parseRules.put(RSET_CONCURRENT_TIMES_HEADER, this::ignore);\n        parseRules.put(RSET_RS_SIZE, this::ignore);\n        parseRules.put(RSET_THREAD_TIMES, this::ignore);\n        parseRules.put(RSET_RS_STATIC_STRUCTURES, this::ignore);\n        parseRules.put(RSET_RS_OCCUPIED_CARDS, this::ignore);\n        parseRules.put(RSET_MAX_REGION_SIZE, this::ignore);\n        parseRules.put(RSET_COARSENINGS, this::ignore);\n\n        //heap summary\n        parseRules.put(new GCParseRule(\"Heap tag\", \"^Heap$\"), this::ignore);\n        parseRules.put(GARBAGE_FIRST_HEAP, this::garbageFirstHeap);\n        parseRules.put(REGION_SIZE, this::regionSize);\n        parseRules.put(METASPACE_FINAL, this::metaspaceFinal);\n        parseRules.put(CLASSPACE_FINAL, this::classspaceFinal);\n\n        //Reference GC\n        parseRules.put(G1_DETAILS_REFERENCE_GC, this::g1DetailsReferenceGC);\n        parseRules.put(G1_REMARK_REFERENCE_GC, this::g1RemarkReferenceGC);\n        parseRules.put(G1_FULL_DETAILS_REFERENCE_GC, this::g1FullDetailsReferenceGC);\n        parseRules.put(G1_DETAILS_REFERENCE_INITIAL_MARK, this::g1DetailsReferenceInitialMark);\n        parseRules.put(FREE_FLOATING_REFERENCE_RECORDS, this::freeFloatingReferenceRecords);\n        parseRules.put(FLOATING_REFERENCE_WITH_ADAPTIVE_SIZING, this::floatingReferenceWithAdaptiveSizing);\n        parseRules.put(YOUNG_REFERENCE_WITH_CONCURRENT_END, this::youngReferenceWIthConcurrentEnd);\n        parseRules.put(G1_FULL_INTERRUPTS_CONCURRENT_WITH_REFERENCES, this::g1FullInterruptsConcurrentWithReferences);\n        parseRules.put(G1_FULL_MEMORY_SPLIT_BY_CONCURRENT, this::g1FullMemorySplitByConcurrent);\n\n        //Ergonomic rules\n        parseRules.put(G1_FULL_ADAPTIVE_SIZING, this::g1FullAdaptiveSizing);\n\n        //18.298: [GC pause (G1 Evacuation Pause) (young) 18.298: [G1Ergonomics (CSet Construction) start choosing CSet, _pending_cards: 0, predicted base time: 10.00 ms, remaining time: 190.00 ms, target pause time: 200.00 ms]\n        parseRules.put(G1_YOUNG_WITH_CSET_CONSTRUCTION_START, this::notYetDefined);\n\n        // 7.793: [G1Ergonomics (CSet Construction) start choosing CSet, _pending_cards: 1395, predicted base time: 14.19 ms, remaining time: 185.81 ms, target pause time: 200.00 ms]\n        parseRules.put(G1_CSET_CONSTRUCTION_START, this::notYetDefined);\n\n        parseRules.put(CSET_CONSTRUCTION, this::csetConstruction);\n\n        //18.298: [G1Ergonomics (CSet Construction) finish choosing CSet, eden: 512 regions, survivors: 0 regions, old: 0 regions, predicted pause time: 7843.33 ms, target pause time: 200.00 ms]\n        parseRules.put(CSET_CONSTRUCTION_END, this::notYetDefined);\n\n        //0.772: [G1Ergonomics (Heap Sizing) shrink the heap, requested shrinking amount: 255605857 bytes, aligned shrinking amount: 254803968 bytes, attempted shrinking amount: 254803968 bytes]\n        parseRules.put(HEAP_SHRINK, this::notYetDefined);\n\n        //18.303: [G1Ergonomics (Heap Sizing) expand the heap, requested expansion amount: 845600000 bytes, attempted expansion amount: 1048576 bytes]\n        parseRules.put(HEAP_EXPAND, this::notYetDefined);\n\n        //18.303: [G1Ergonomics (Heap Sizing) attempt heap expansion, reason: region allocation request failed, allocation request: 8456 bytes]\n        parseRules.put(ATTEMPT_HEAP_EXPANSION_ALLOC_FAILURE, this::notYetDefined);\n\n        //7.942: [G1Ergonomics (Heap Sizing) attempt heap expansion, reason: recent GC overhead higher than threshold after GC, recent GC overhead: 22.02 %, threshold: 10.00 %, uncommitted: 1060110336 bytes, calculated expansion amount: 212022067 bytes (20.00 %)]\n        parseRules.put(ATTEMPT_HEAP_EXPANSION_OVERHEAD, this::notYetDefined);\n\n        //0.785: [G1Ergonomics (Heap Sizing) did not shrink the heap, reason: heap shrinking operation failed]\n        parseRules.put(HEAP_SHRINKING_FAILED, this::notYetDefined);\n\n        //7.835: [G1Ergonomics (Concurrent Cycles) request concurrent cycle initiation, reason: occupancy higher than threshold, occupancy: 7340032 bytes, allocation request: 0 bytes, threshold: 6134130 bytes (45.00 %), source: end of GC]\n        parseRules.put(HIGH_OCCUPANCY_TRIGGERS_CONC, this::notYetDefined);\n\n        //7.842: [G1Ergonomics (Concurrent Cycles) initiate concurrent cycle, reason: concurrent cycle initiation requested]\n        parseRules.put(INITIATE_CONC_CYCLE, this::notYetDefined);\n\n        //7.916: [G1Ergonomics (Concurrent Cycles) do not request concurrent cycle initiation, reason: still doing mixed collections, occupancy: 9437184 bytes, allocation request: 0 bytes, threshold: 6134130 bytes (45.00 %), source: end of GC]\n        parseRules.put(DO_NOT_REQUEST_CONC_CYCLE, this::notYetDefined);\n\n        //7.859: [G1Ergonomics (Mixed GCs) start mixed GCs, reason: candidate old regions available, candidate old regions: 4 regions, reclaimable: 2406696 bytes (17.66 %), threshold: 10.00 %]\n        //7.869: [G1Ergonomics (Mixed GCs) do not continue mixed GCs, reason: reclaimable percentage not over threshold, candidate old regions: 2 regions, reclaimable: 1031104 bytes (7.56 %), threshold: 10.00 %]\n        //7.986: [G1Ergonomics (Mixed GCs) do not start mixed GCs, reason: reclaimable percentage not over threshold, candidate old regions: 3 regions, reclaimable: 1380376 bytes (2.53 %), threshold: 10.00 %]\n        parseRules.put(START_MIXED_GC, this::notYetDefined);\n        parseRules.put(DELAY_MIXED_GC, this::notYetDefined);\n\n        //7.866: [G1Ergonomics (CSet Construction) finish adding old regions to CSet, reason: old CSet region num reached max, old: 2 regions, max: 2 regions]\n        //7.866: [G1Ergonomics (CSet Construction) finish add ing old regions to CSet, reason: old CSet region num reached max, old: 2 regions, max: 2 regions]\n        parseRules.put(CSET_FINISH, this::notYetDefined);\n        parseRules.put(CSET_ADDING, this::notYetDefined);\n\n        parseRules.put(new GCParseRule(\"END_OF_DATA_SENTINEL\", END_OF_DATA_SENTINEL), this::endOfFile);\n    }\n\n    public PreUnifiedG1GCParser() {\n        forwardReference = trap;\n    }\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.G1GC);\n    }\n\n    public String getName() {\n        return \"PreUnifiedG1GCParser\";\n    }\n\n    @Override\n    protected void process(String line) {\n\n        if (ignoreFrequentlySeenButUnwantedLines(line)) return;\n\n        try {\n            parseRules.keys()\n                    .stream()\n                    .map(rule -> new AbstractMap.SimpleEntry<>(rule, rule.parse(line)))\n                    .filter(tuple -> tuple.getValue() != null)\n                    .findFirst()\n                    .ifPresentOrElse(\n                            rulesAndTrace -> parseRules.get(rulesAndTrace.getKey()).accept(rulesAndTrace.getValue(), line),\n                            () -> log(line));\n        } catch (Throwable t) {\n            LOGGER.throwing(this.getName(), \"deriveConfiguration\", t);\n        }\n    }\n\n    void notYetDefined(GCLogTrace trace, String line) {\n        if (System.getProperty(\"microsoft.develop.ergonomics\") != null)\n            trace.notYetImplemented();\n    }\n\n    public void endOfFile(GCLogTrace trace, String line) {\n        super.publish(ChannelName.G1GC_PARSER_OUTBOX, new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n    }\n\n    //18.298: [G1Ergonomics (CSet Construction) add young regions to CSet, eden: 512 regions, survivors: 0 regions, predicted young region time: 7833.33 ms]\n    void csetConstruction(GCLogTrace trace, String line) {\n        if (System.getProperty(\"microsoft.develop.ergonomics\") != null)\n            trace.notYetImplemented();\n    }\n\n...\n", "needle_token_start": 5664, "needle_token_end": 5688, "code_context_ntokens": 16207, "output": ["```java\npublic boolean tagsContain(String tagList) {\n    return tags.contains(tagList);\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "extractValues", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To parse and count the number of decorators and tags from a given log line based on predefined patterns, helping to categorize and understand log details.\n2. **Input**: A string representing a single line from a log file.\n3. **Output**: The function does not return a value but updates internal state by counting the number of decorators and extracting tags from the log line.\n4. **Procedure**: \n   - The function first checks if the line starts with a bracket, which is typical for log lines with decorators.\n   - It then uses a regular expression to find matches for predefined decorator patterns in the line.\n   - For each matching group in the regular expression, it increments a counter if the group is not null, indicating the presence of a decorator.\n   - Another regular expression is used to extract tags from the line. If tags are found, they are cleaned, split, and joined to update the tags string and increment the decorator count.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/ShenandoahParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.GCToolKit;\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.collection.MRUQueue;\nimport com.microsoft.gctoolkit.parser.unified.ShenandoahPatterns;\n\nimport java.util.AbstractMap;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.function.BiConsumer;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n/**\n * Time of GC\n * GCType\n * Collect total heap values\n * Heap before collection\n * Heap after collection\n * Heap configured size\n * total pause time\n * CMS failures\n * System.gc() calls\n */\n\npublic class ShenandoahParser extends UnifiedGCLogParser implements ShenandoahPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(ShenandoahParser.class.getName());\n\n    private final MRUQueue<GCParseRule, BiConsumer<GCLogTrace, String>> parseRules;\n\n    {\n        parseRules = new MRUQueue<>();\n        parseRules.put(END_OF_FILE,this::endOfFile);\n    }\n\n    public ShenandoahParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.SHENANDOAH);\n    }\n\n    @Override\n    public String getName() {\n        return \"Shenandoah Parser\";\n    }\n\n    @Override\n    protected void process(String line) {\n\n...\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/ParNewSerial.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\n/*\n   KCP - do not delete --- this is a deprecated GC combination. This interface is current not used but is retrained here for historical reasons.\n */\npublic interface ParNewSerial extends PreUnifiedTokens {\n\n    //12.008: [GC 12.008: [ParNew: 4680K->224K(4992K), 0.0009052 secs]12.009: [Tenured: 11421K->7456K(11448K), 0.0679043 secs] 15860K->7456K(16440K), [Perm : 10677K->10677K(21248K)], 0.0689327 secs] [Times: user=0.07 sys=0.00, real=0.07 secs]\n//    public static final Pattern PARNEW_SERIAL_FULL = Pattern.compile( DATE_TIMESTAMP + \"\\\\[GC \" + PARNEW_BLOCK + TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + \", \" + PAUSE_TIME);\n\n    //: 7077K->86K(7552K), 0.0019757 secs]20.774: [Tenured: 17007K->7966K(17080K), 0.0754072 secs] 23865K->7966K(24632K), [Perm : 10697K->10697K(21248K)], 0.0776402 secs]\n    //]20.774: [Tenured: 17007K->7966K(17080K), 0.0754072 secs] 23865K->7966K(24632K), [Perm : 10697K->10697K(21248K)], 0.0776402 secs]\n//    public static final Pattern PARNEW_SERIAL_FULL_SPLIT = Pattern.compile( \"^: \" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\" + TENURED_BLOCK + \" \" + BEFORE_AFTER_CONFIGURED + \", \" + PERM_RECORD + \", \" + PAUSE_TIME);\n\n/*\n\n    public static final Pattern PAR_OLD_REDUCTION = Pattern.compile(\"\\\\[ParOldGen: \" + FROM_TO_CONFIGURED + \"\\\\]\");\n    \n    public static final String  TENURED_AND_PERM_REDUCTION =    TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] ?\" + FROM_TO_CONFIGURED + \", \" + PERM_REDUCTION    + \", \" + PAUSE_TIME + \"\\\\]\";\n    public static final String  TENURED_AND_NO_PERM_REDUCTION = TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] ?\" + FROM_TO_CONFIGURED + \", \" + PERM_NO_REDUCTION + \", \" + PAUSE_TIME + \"\\\\]\";\n\n    //2.783: [Full GC 2.783: [Tenured: 2684K->3969K(8192K), 0.0569037 secs] 4603K->3969K(13696K), [Perm : 16382K->16382K(16384K)], 0.0569750 secs]\n    public static final Pattern PARALLEL_OLD_PERM_REDUCTION = Pattern.compile(\"\\\\[Full GC \" + TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED + \", \" + PAUSE_TIME + \"\\\\] \" + FROM_TO_CONFIGURED + \", \" + PERM_REDUCTION.toString() + \", \" + PAUSE_TIME + \"\\\\]\");\n    public static final Pattern PARALLEL_OLD_NO_PERM_REDUCTION = Pattern.compile(\"\\\\[Full GC \" + TIMESTAMP + \": \\\\[Tenured: \" + FROM_TO_CONFIGURED + \", \" + PAUSE_TIME + \"\\\\] \" + FROM_TO_CONFIGURED + \", \" + PERM_NO_REDUCTION.toString() + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //TODO\n    //0.963: [ParNew: 16256K->1984K(18240K), 0.0689940 secs] 16256K->6122K(58752K), 0.0691170 secs]\n    public static final Pattern PARNEW = Pattern.compile( \"\\\\[ParNew: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\]\");\n    public static final Pattern PARNEW_ICMS = Pattern.compile(\"\\\\[ParNew: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\] \" + FROM_TO_CONFIGURED + CMSPatterns.ICMS_DC + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //25.118: [GC 25.118: [ParNew: 1258368K->143985K(1415616K), 0.0987560 secs] 1265723K->151341K(8231360K), 0.0989880 secs] [Times: user=0.96 sys=0.49, real=0.09 secs]\n    public static final Pattern PARNEW_COMPLETE = Pattern.compile( START_OF_GC_TIMESTAMP + \"\\\\[GC\\\\s*\" + GC_TIMESTAMP + PARNEW);\n\n    //25.030: [GC 25.030: [ParNew: 451942K->51072K(460096K), 1.7048460 secs] 451942K->115934K(12531840K) icms_dc=5 , 1.7050420 secs]\n    public static final Pattern PARNEW_COMPLETE_ICMS = Pattern.compile( START_OF_GC_TIMESTAMP + \"\\\\[GC\\\\s*\" + GC_TIMESTAMP + PARNEW_ICMS);\n    public static final Pattern PARNEW_PRINT_HEAP_AT_GC = Pattern.compile( START_OF_GC_TIMESTAMP + \"\\\\[ParNew: \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\]\" + FROM_TO_CONFIGURED + \"Heap after GC invocations=\");\n    public static final Pattern PARNEW_PRINT_HEAP_AT_GC_PAUSE_TIME = Pattern.compile( \"} , \" + TIMESTAMP + \"]\");\n\n\n    //939.183: [GC [PSYoungGen: 523744K->844K(547584K)] 657668K->135357K(1035008K), 0.0157986 secs] [Times: user=0.30 sys=0.01, real=0.02 secs]\n    public static final Pattern PSYOUNGGEN_COMPLETE = Pattern.compile(\"\\\\[GC \\\\[PSYoungGen: \" + FROM_TO_CONFIGURED + \"\\\\] \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\]\");\n\n    //Tenuring records\n    //Desired survivor size 107347968 bytes, new threshold 1 (max 4)\n    public static final Pattern TENURING_SUMMARY = Pattern.compile(\"Desired survivor size (\\\\d+) bytes, new threshold (\\\\d+) \\\\(max (\\\\d+)\\\\)\");\n    public static final Pattern TENURING_AGE_BREAKDOWN = Pattern.compile(\"- age\\\\s+(\\\\d+):\\\\s+(\\\\d+) bytes,\\\\s+(\\\\d+) total\");\n\n    public static final Pattern YG_WITH_TENURING = Pattern.compile(\"\\\\[GC$\");\n    public static final Pattern PSYOUNGGEN_DETAILS = Pattern.compile(\"\\\\[PSYoungGen: \" + FROM_TO_CONFIGURED + \"\\\\] \" +  FROM_TO_CONFIGURED_PAUSE + \"\\\\]\");\n    public static final Pattern PS_FULL_GC_DETAILS = Pattern.compile(\"\\\\[Full GC \\\\[PSYoungGen:\");\n\n    public static final Pattern PS_FULL_WITH_UNLOADING = Pattern.compile(\"\\\\[Full GC\\\\[Unloading\");\n    public static final Pattern PS_FULL_WITH_UNLOADING_SYSTEM = Pattern.compile(\"\\\\[Full GC \\\\(System\\\\)\\\\[Unloading\");\n\n    //0.465: [GC 0.465: [DefNew: 17217K->17217K(19328K), 0.0000096 secs]0.465: [Tenured: 36555K->11838K(43008K), 0.0266298 secs] 53772K->11838K(62336K), [Perm : 2819K->2819K(21248K)], 0.0775828 secs]\n    public static final Pattern DEFNEW_TENURED = Pattern.compile(\"\\\\[GC\\\\s*\" + GC_TIMESTAMP + \"\\\\[DefNew: .+ \\\\[Tenured:\");\n\n    //: 4806K->1427K(5504K), 0.0089964 secs]4.690: [Tenured: 8890K->8960K(8960K), 0.0683118 secs] 12365K->10317K(14464K), [Perm : 18543K->18543K(18688K)], 0.0775077 secs]\n    public static final Pattern DEFNEW_TRIGGERED_FULL = Pattern.compile( \"^: \" + FROM_TO_CONFIGURED + \", .+ secs\\\\]\" + TIMESTAMP + \": \\\\[Tenured:\");\n    public static final Pattern DEFNEW_TRIGGERED_FULL_PERMSWEEP = Pattern.compile( \"^: \" + FROM_TO_CONFIGURED + \", .+ secs\\\\]\" + TENURED_AND_PERM_REDUCTION);\n    public static final Pattern DEFNEW_TRIGGERED_FULL_PERMNOSWEEP = Pattern.compile( \"^: \" + FROM_TO_CONFIGURED + \", .+ secs\\\\]\" + TENURED_AND_NO_PERM_REDUCTION);\n\n    // 83.940: [Full GC 83.940: [Tenured: 75938K->76927K(126568K), 1.8452680 secs] 106725K->76927K(183592K), [Perm : 62463K->62463K(62464K)], 1.8454850 secs] [Times: user=1.84 sys=0.01, real=1.85 secs] \n    public static final Pattern SERIAL_FULL_SYSTEM = Pattern.compile(\"\\\\[Full GC \\\\(System\\\\) \" + TENURED_AND_PERM_REDUCTION);\n    \n    //104401.329: [Full GC (System) [PSYoungGen: 0K->0K(524224K)] [PSFull: 4928117K->4928117K(20447232K)] 4928117K->4928117K(20971456K) [PSPermGen: 113238K->113238K(262144K)], 1.7623490 secs]\n    public static final Pattern PS_FULL_SYSTEM = Pattern.compile(\"\\\\[Full GC \\\\(System\\\\) \\\\[PSYoungGen:\");\n\n    public static final Pattern PS_FULL = Pattern.compile(\"\\\\[Full GC \\\\[PSYoungGen:\");\n    \n    //12525.344: [Full GC 12525.344: [ParNew\n    public static final Pattern PARNEW_FULL_FOLLOWED_BY_DETAILS = Pattern.compile(\"\\\\[Full GC \" + TIMESTAMP + \": \\\\[ParNew$\");\n    \n\n    // 93733.352: [Full GC[Unloading class sun.reflect.GeneratedMethodAccessor3]\n    // [PSYoungGen: 892029K->0K(1034432K)] [ParOldGen: 7324875K->5037686K(7340032K)] 8216904K->5037686K(8374464K) [PSPermGen: 54404K->54363K(122496K)], 10.6791310 secs] [Times: user=67.94 sys=0.11, real=10.68 secs] \n    public static final Pattern PAR_FULL_GC_WITH_CLASS_UNLOADING = Pattern.compile(PAR_OLD_REDUCTION + \" \" + FROM_TO_CONFIGURED + \".* \" + PAUSE_TIME);\n\n    //[PSPermGen: 7034K->7034K(21248K)]\n    public static final Pattern PS_PERM_GEN = Pattern.compile( \"\\\\[PSPermGen: [0-9]+K->[0-9]+K\\\\([0-9]+K\\\\)\\\\]\");\n    \n    //: 4806K->1427K(5504K), 0.0089964 secs]4.690: [Tenured: 8890K->8960K(8960K), 0.0683118 secs] 12365K->10317K(14464K), [Perm : 18543K->18543K(18688K)], 0.0775077 secs]\n    public static final Pattern TENURED_DETAILS_PERM_REDUCTION = Pattern.compile(\": \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\]\" + TENURED_AND_PERM_REDUCTION);\n    public static final Pattern TENURED_DETAILS_NO_PERM_REDUCTION = Pattern.compile( \": \" + FROM_TO_CONFIGURED_PAUSE + \"\\\\]\" + TENURED_AND_NO_PERM_REDUCTION);\n\n*/\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingLevel.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\npublic enum UnifiedLoggingLevel {\n\n    trace(\"trace\"),\n    debug(\"debug\"),\n    info(\"info\"),\n    warning(\"warning\"),\n    error(\"error\");\n\n    private final String label;\n\n    UnifiedLoggingLevel(String label) {\n        this.label = label;\n    }\n\n    public String getLabel() {\n        return label;\n    }\n\n    /**\n     * @param other logging level to compare this one to\n     * @return whether this logging level is lesser or equal to the other one\n     */\n    public boolean isLessThanOrEqualTo(UnifiedLoggingLevel other) {\n        return this.compareTo(other) <= 0;\n    }\n\n\n    /**\n     * @param other logging level to compare this one to\n     * @return whether this logging level is greater or equal to the other one\n     */\n    public boolean isGreaterThanOrEqualTo(UnifiedLoggingLevel other) {\n        return this.compareTo(other) >= 0;\n    }\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedLoggingTokens.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\n\nimport com.microsoft.gctoolkit.parser.GenericTokens;\n\nimport java.util.regex.Pattern;\n\npublic interface UnifiedLoggingTokens extends GenericTokens {\n\n    /*\n      JEP 158 Decorators\n        time -- Current time and date in ISO-8601 format\n        uptime -- Time since the start of the JVM in seconds and milliseconds (e.g., 6.567s)\n        timemillis -- The same value as generated by System.currentTimeMillis()\n        uptimemillis -- Milliseconds since the JVM started\n        timenanos -- The same value as generated by System.nanoTime()\n        uptimenanos -- Nanoseconds since the JVM started\n        pid -- The process identifier\n        tid -- The thread identifier\n        level -- The level associated with the log message\n        tags -- The tag-set associated with the log message\n        [2018-04-04T09:10:00.586-0100][0.018s][1522825800586ms][18ms][10026341461044ns][17738937ns][1375][7427][info][gc] Using G1\n\n        Note: Decorators are outputed in the order of the list above.\n     */\n\n    String DATE_STAMP = \"\\\\[\" + DATE + \"\\\\]\";\n    String UPTIME = \"\\\\[\" + INTEGER + DECIMAL_POINT + \"\\\\d{3}s\\\\]\";\n    String TIME_MILLIS = \"\\\\[\\\\d+ms\\\\]\";\n    String TIME_NANOS = \"\\\\[\\\\d+ns\\\\]\";\n    String PID_TID = \"\\\\[\\\\d+\\\\]\";\n    String UNIFIED_LOG_LEVEL_BLOCK = \"\\\\[(?:error|warning|info|debug|trace|develop) *\\\\]\";\n    Pattern DECORATORS = Pattern.compile(\"(\" + DATE_STAMP + \")?(\" + UPTIME + \")?(\" + TIME_MILLIS + \")?(\" + TIME_MILLIS + \")?(\" + TIME_NANOS + \")?(\" + TIME_NANOS + \")?(\" + PID_TID + \")?(\" + PID_TID + \")?(\" + UNIFIED_LOG_LEVEL_BLOCK + \")?\");\n    //Using zero-width negative lookbehind to miss capturing records formatted like [0x1f03].\n    //[0.081s][trace][safepoint] Thread: 0x00007fd0d2006800  [0x1f03] State: _at_safepoint _has_called_back 0 _at_poll_safepoint 0\n    Pattern TAGS = Pattern.compile(\".*(?<=^|\\\\])\\\\[([a-z0-9,. ]+)\\\\]\");\n\n    String UNIFIED_META_RECORD = \"Metaspace: \" + BEFORE_AFTER_CONFIGURED;\n    String WORKER_SUMMARY_REAL = \"Min:\\\\s+\" + TIME + \", Avg:\\\\s+\" + TIME + \", Max:\\\\s+\" + TIME + \", Diff:\\\\s+\" + TIME + \", Sum:\\\\s+\" + TIME + \", Workers:\\\\s+\" + COUNTER;\n    String WORKER_SUMMARY_INT = \"Min:\\\\s+\" + COUNTER + \", Avg:\\\\s+\" + TIME + \", Max:\\\\s+\" + COUNTER + \", Diff:\\\\s+\" + COUNTER + \", Sum:\\\\s+\" + COUNTER + \", Workers:\\\\s+\" + COUNTER;\n    String POOL_SUMMARY = \"used \" + COUNTER + \"K, capacity \" + COUNTER + \"K, committed \" + COUNTER + \"K, reserved \" + COUNTER + \"K\";\n    String REGION_MEMORY_BLOCK = COUNTER + \"->\" + COUNTER + \"(?:\\\\(\" + COUNTER + \"\\\\))?\";\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/Decorators.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\nimport com.microsoft.gctoolkit.parser.unified.UnifiedLoggingLevel;\nimport com.microsoft.gctoolkit.parser.unified.UnifiedLoggingTokens;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.time.ZonedDateTime;\nimport java.time.format.DateTimeFormatter;\nimport java.time.temporal.TemporalAccessor;\nimport java.util.Arrays;\nimport java.util.Optional;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\nimport java.util.regex.Matcher;\n\npublic class Decorators {\n\n    /**\n     * This class represents the decorators found in the log line as defined by JEP 158 (https://openjdk.java.net/jeps/158).\n     * The list, taken from that document is as:\n     * time -- Current time and date in ISO-8601 format\n     * uptime -- Time since the start of the JVM in seconds and milliseconds (e.g., 6.567s)\n     * timemillis -- The same value as generated by System.currentTimeMillis()\n     * uptimemillis -- Milliseconds since the JVM started\n     * timenanos -- The same value as generated by System.nanoTime()\n     * uptimenanos -- Nanoseconds since the JVM started\n     * pid -- The process identifier\n     * tid -- The thread identifier\n     * level -- The level associated with the log message\n     * tags -- The tag-set associated with the log message\n     *\n     * This implementation takes advantage of the property that the ordering of tags is stable. For example,\n     * -Xlog:*::pid,time produces the identical ordering as -Xlog:*::time,pid. Moreover, the list of decorators\n     * is dedupped implying that the second decorator in -Xlog:*::time,time is ignored.\n     *\n     * The default decorators (as of the time of authoring JDK 9-21) include [uptime][level][tags].\n     * For example, [1.361s][info][gc,heap]. This reads uptime of 1.361 seconds. The tag for the log record\n     * is gc.heap at the info level.\n     *\n     * At issue is that uptime and timemillis are, on the surface, indistinguishable. The same is true with\n     * timenanos and uptimenanos as well as with pid and tid. The following logic can be used to help differentiate\n     * indistinguishable decorators.\n     *\n     * 1) If both decorators are present then order can be used to differentiate\n     * 2) If ms time value - 20 years > 0, then timemillis can be assumed. Otherwise uptime is assumed. The reasoning\n     *    is, unified logging didn't exist that long ago. The value of 20 has been arbitrarily chosen.\n     * 3) There is no reliable way to differentiate the nanosecond timings if only 1 is present. However this may not\n     *    matter as GCToolKit would only use these values to create a baseline measure to align date/time with uptime.\n     *    This technique is designed to work-around the cases where logs do not contain the uptime decorator.\n     * 4) There is no known way to reliably differentiate between PID and TID. At this time, GCToolKit ignores these\n     *    decorators.\n     *\n     * Todo: GCToolkit captures time in the DateTimeStamp class. That class will capture uptime or time or both. If both\n     * are missing, GCToolkit JVMEvents will have no sense of time. It is possible that the other timing fields could fill\n     * in cases where both the time and uptime decorators were missing.\n     * @param line\n     */\n\n    private static final Logger LOGGER = Logger.getLogger(Decorators.class.getName());\n\n    // This is to help differentiate between JVM running time and wall clock time.\n    private static final long TWENTY_YEARS_IN_MILLIS = 731L * 24L * 60L * 60L * 1000L;\n    private static final long TWENTY_YEARS_IN_NANO = 731L * 24L * 60L * 60L * 1000L;\n\n    int numberOfDecorators;\n\n    private Matcher decoratorMatcher = null;\n    private String tags;\n\n    public Decorators(String line) {\n        extractValues(line);\n    }\n\n    \nprivate void extractValues(String line) {\n\n        if (!line.startsWith(\"[\"))\n            return;\n\n        decoratorMatcher = UnifiedLoggingTokens.DECORATORS.matcher(line);\n        if (!decoratorMatcher.find()) {\n            return;\n        }\n\n        for ( int i = 1; i <= decoratorMatcher.groupCount(); i++) {\n            if ( decoratorMatcher.group(i) != null)\n                numberOfDecorators++;\n        }\n\n        Matcher tagMatcher = UnifiedLoggingTokens.TAGS.matcher(line);\n        if (tagMatcher.find()) {\n            numberOfDecorators++;\n            tags = String.join(\",\", Arrays.asList(tagMatcher.group(1).trim().split(\",\")));\n        }\n    }\n\n    // For some reason, ISO_DATE_TIME doesn't like that time-zone is -0100. It wants -01:00.\n    private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(\"yyyy-MM-dd'T'HH:mm:ss.SSSZ\");\n\n    public ZonedDateTime getDateStamp() {\n        try {\n            String value = decoratorMatcher.group(1);\n            if (value != null) {\n                TemporalAccessor temporalAccessor = formatter.parse(value.substring(1, value.length()-1));\n                return ZonedDateTime.from(temporalAccessor);            }\n        } catch (NullPointerException npe) {\n            LOGGER.log(Level.SEVERE, npe.getMessage(), npe);\n        }\n        return null;\n    }\n\n    public double getUpTime() {\n        String value = decoratorMatcher.group(2);\n        if (value != null) {\n            value = value.replace(\",\", \".\");\n            return Double.parseDouble(unboxValue(value, 1));\n        }\n        return -1.0d;\n    }\n\n    private long extractClock(int groupIndex, long threshold) {\n        long clockReading = -1L;\n        String stringValue = decoratorMatcher.group(groupIndex);\n        if (stringValue != null) {\n            clockReading = Long.parseLong(unboxValue(stringValue, 2));\n            if (decoratorMatcher.group(groupIndex + 1) == null)\n                if (clockReading < threshold)\n                    clockReading = -1L;\n        }\n        return clockReading;\n    }\n\n    public long getTimeMillis() {\n        return extractClock(3, TWENTY_YEARS_IN_MILLIS);\n    }\n\n    public long getUptimeMillis() {\n        String value = decoratorMatcher.group(4);\n        if (value == null) {\n            value = decoratorMatcher.group(3);\n        }\n        if (value != null) {\n            long longValue = Long.parseLong(unboxValue(value, 2));\n            if (longValue < TWENTY_YEARS_IN_MILLIS)\n                return longValue;\n        }\n        return -1L;\n    }\n\n    public long getTimeNano() {\n        return extractClock(5, TWENTY_YEARS_IN_NANO);\n    }\n\n    public long getUptimeNano() {\n        String value = decoratorMatcher.group(6);\n        if (value == null) {\n            value = decoratorMatcher.group(5);\n        }\n        if (value != null) {\n            long longValue = Long.parseLong(unboxValue(value, 2));\n            if (longValue < TWENTY_YEARS_IN_NANO)\n                return longValue;\n        }\n        return -1L;\n    }\n\n    public int getPid() {\n        String value = decoratorMatcher.group(7);\n        if (value != null) {\n            return Integer.parseInt(unboxValue(value));\n        }\n        return -1;\n    }\n\n    public int getTid() {\n        String value = decoratorMatcher.group(8);\n        if (value != null) {\n            return Integer.parseInt(unboxValue(value));\n        }\n        return -1;\n    }\n\n    public Optional<UnifiedLoggingLevel> getLogLevel() {\n        String level = decoratorMatcher.group(9);\n        if (level != null)\n            try {\n                return Optional.of(UnifiedLoggingLevel.valueOf(unboxValue(level)));\n            } catch (IllegalArgumentException e) {\n                LOGGER.fine(\"No such debug level: \" + level);\n                LOGGER.fine(e.getMessage());\n                return Optional.empty();\n            }\n        return Optional.empty();\n    }\n\n    public DateTimeStamp getDateTimeStamp() {\n        return new DateTimeStamp(getDateStamp(), getUpTime());\n    }\n\n    public int getNumberOfDecorators() {\n        return numberOfDecorators;\n    }\n\n    private String unboxValue(String boxedValue, int postFix) {\n        return boxedValue.substring(1, boxedValue.length() - (1 + postFix));\n    }\n\n    private String unboxValue(String boxedValue) {\n        return unboxValue(boxedValue, 0).trim();\n    }\n\n    public boolean tagsContain(String tagList) {\n        return tags.contains(tagList);\n    }\n\n    public String getTags() {\n        return tags;\n    }\n}\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.MetaspaceRecord;\nimport com.microsoft.gctoolkit.event.jvm.PermGenSummary;\nimport com.microsoft.gctoolkit.io.GCLogFile;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.DataSourceParser;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\npublic abstract class GCLogParser implements DataSourceParser, SharedPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(GCLogParser.class.getName());\n    public static final String END_OF_DATA_SENTINEL = GCLogFile.END_OF_DATA_SENTINEL;\n\n    public static final GCParseRule GCID_COUNTER = new GCParseRule(\"GCID_COUNTER\", \" GC\\\\((\\\\d+)\\\\) \");\n    private JVMEventChannel consumer;\n    protected Diary diary;\n    private DateTimeStamp clock = new DateTimeStamp(DateTimeStamp.EPOC, 0.0d);\n    private double lastDuration = 0.0d;\n\n\n    public GCLogParser() {}\n\n    /**\n     * Setting the diary will set the current clock time to the time of the first event\n     * in the GC log. This is a better choice than EPOC though EPOC is better than null.\n     * @param diary summary the GC log.\n     */\n    @Override\n    public void diary(Diary diary) {\n        this.diary = diary;\n        this.clock = diary.getTimeOfFirstEvent();\n    }\n\n    public DateTimeStamp getClock() {\n        return clock;\n    }\n\n    public void setClock(DateTimeStamp newValue) {\n        this.clock = newValue;\n    }\n\n    public abstract String getName();\n\n    protected abstract void process(String trace);\n\n    abstract void advanceClock(String record);\n\n    /**\n     * The assumption is, this method will manage the global clock and thus should never be over ridden\n     * @param now - DateTimeStamp from the current GC log record\n     */\n    protected final void advanceClock(DateTimeStamp now) {\n        if (now == null)\n            return;\n        // now can be the same but it can't be less than\n        else if (now.before(getClock())) {\n            LOGGER.log(Level.WARNING, \"Log File may be Corrupted: Time traveled backwards from {0} to {1}\", new Object[]{getClock().toString(), now.toString()});\n        }\n        //todo: should preserve date in cases where statements do not respect PrintDateStamp (eg. ergonomics)\n        setClock(now);\n    }\n\n    /**\n     * The clock is advanced to the time at the end of the event.\n     * The times reported in the GCLog file are \"noisy\" at best. This\n     * hyperactives the clock rollback detection which doesn't allow\n     * for slop. To minimize this, the clock only records when a time\n     * stamp is presented and isn't advanced to the end of the event\n     * except in the case of EOF where the time of JVM termination has\n     * not been recorded. In this case, JVMTermination time should advanced\n     * to after the previous event has ended.\n     *\n     * @param channel\n     * @param event\n     */\n    public void publish(ChannelName channel, JVMEvent event) {\n        lastDuration = event.getDuration();\n        consumer.publish(channel,event);\n    }\n\n    public void receive(String trace) {\n        if (!trace.equals(END_OF_DATA_SENTINEL))\n            advanceClock(trace);\n        else\n            advanceClock(getClock().add(lastDuration));\n        process(trace);\n    }\n\n    boolean isPreJDK17040() {\n        return diary.isPre70_40();\n    }\n\n    boolean hasPrintGCDetails() {\n        return diary.isPrintGCDetails();\n    }\n\n    // todo: mixes aggregator with parsing. premature optimization...\n    MemoryPoolSummary getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(GCLogTrace trace, int offset) {\n        MemoryPoolSummary summary = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(offset);\n        return summary;\n    }\n\n    MemoryPoolSummary getTotalOccupancyWithTotalHeapSizeSummary(GCLogTrace trace, int offset) {\n        MemoryPoolSummary summary = trace.getOccupancyWithMemoryPoolSizeSummary(offset);\n        return summary;\n    }\n\n    GCLogTrace extractReferenceBlock(String line, GCParseRule rule) {\n        return rule.parse(line);\n    }\n\n    ReferenceGCSummary extractPrintReferenceGC(String line) {\n\n        ReferenceGCSummary summary = new ReferenceGCSummary();\n        GCLogTrace trace;\n        if ((trace = extractReferenceBlock(line, SOFT_REFERENCE)) != null)\n            summary.addSoftReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n\n        if ((trace = extractReferenceBlock(line, WEAK_REFERENCE)) != null)\n            summary.addWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n\n        if ((trace = extractReferenceBlock(line, FINAL_REFERENCE)) != null)\n            summary.addFinalReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n\n        if ((trace = extractReferenceBlock(line, PHANTOM_REFERENCE)) != null) {\n            if (trace.groupNotNull(4))\n                summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getIntegerGroup(4), trace.getDuration());\n            else\n                summary.addPhantomReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n        }\n\n        if ((trace = extractReferenceBlock(line, JNI_REFERENCE)) != null) {\n            if (trace.groupNotNull(3))\n                summary.addJNIWeakReferences(trace.getDateTimeStamp(), trace.getIntegerGroup(3), trace.getDuration());\n            else\n                summary.addJNIWeakReferences(trace.getDateTimeStamp(), trace.getDuration());\n        }\n\n        return summary;\n\n    }\n\n    /*\n     * [PSPermGen: 7034K->7034K(21248K)]\n     * [CMS Perm : 10696K->10696K(21248K)]\n     * [Perm : 16382K->16382K(16384K)]\n     * [Perm : 16382K(16384K)]\n     *\n     * @param line from GC log file\n     */\n    MemoryPoolSummary extractPermOrMetaspaceRecord(String line) {\n        GCLogTrace trace;\n        MemoryPoolSummary metaDataPool = null;\n        if ((trace = PERM_SPACE_RECORD.parse(line)) != null) {\n            String type = trace.getGroup(1).trim();\n            if (\"CMS Perm\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"PS Perm\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"PSPermGen\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"Perm\".equals(type))\n                metaDataPool = extractPermGenRecord(trace);\n            else if (\"Metaspace\".equals(type)) {\n                if (trace.getGroup(2) != null) {\n                    metaDataPool = new MetaspaceRecord(trace.toKBytes(2), trace.toKBytes(4), trace.toKBytes(6));\n                } else {\n                    metaDataPool = new MetaspaceRecord(trace.toKBytes(4), trace.toKBytes(4), trace.toKBytes(6));\n                }\n            }\n        } else if ((trace = META_SPACE_RECORD.parse(line)) != null) {\n            int index = (trace.getGroup(1) == null) ? 1 : 3;\n            metaDataPool = new MetaspaceRecord(trace.toKBytes(index), trace.toKBytes(3), trace.toKBytes(5));\n        }\n\n        return metaDataPool;\n    }\n\n    MemoryPoolSummary extractPermGenRecord(GCLogTrace trace) {\n        int index = (trace.getGroup(2) == null) ? 2 : 4;\n        return new PermGenSummary(trace.getLongGroup(index), trace.getLongGroup(4), trace.getLongGroup(6));\n    }\n\n    int extractGCID(String line) {\n        GCLogTrace trace = GCID_COUNTER.parse(line);\n        return (trace != null) ? trace.getIntegerGroup(1) : -1;\n    }\n\n    CPUSummary extractCPUSummary(String line) {\n        GCLogTrace trace;\n        if ((trace = CPU_BREAKDOWN.parse(line)) != null) {\n            return new CPUSummary(trace.getDoubleGroup(1), trace.getDoubleGroup(2), trace.getDoubleGroup(3));\n        }\n        return null;\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel channel) {\n        this.consumer = channel;\n    }\n\n    @Override\n    public ChannelName channel() {\n        return ChannelName.DATA_SOURCE;\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\n\nimport com.microsoft.gctoolkit.parser.GCLogParser;\nimport com.microsoft.gctoolkit.parser.GCParseRule;\n\npublic interface UnifiedPatterns extends UnifiedLoggingTokens {\n\n    GCParseRule CPU_BREAKOUT = new GCParseRule(\"CPU_BREAKOUT\", \"User=\" + TIME + \"s Sys=\" + TIME + \"s Real=\" + TIME + \"s\");\n\n    GCParseRule GC_COUNT = new GCParseRule(\"GC_COUNT\", \" GC\\\\((\" + INTEGER + \")\\\\) \");\n\n    String WORKER_TASK = \"(for evacuation|to rebuild remembered set)\";\n    GCParseRule WORKER_SUMMARY = new GCParseRule(\"WORKER_SUMMARY\", \"Using \" + COUNTER + \" workers of \" + COUNTER + \" \" + WORKER_TASK);\n\n    GCParseRule REFERENCES = new GCParseRule(\"REFERENCES\", \"(Preclean )?(SoftReference|WeakReference|FinalReference|PhantomReference|JNI Weak Reference)(:?s)? \" + PAUSE_TIME);\n    GCParseRule REFERENCE_COUNTS = new GCParseRule(\"REFERENCE_COUNTS\", \"Ref Counts: Soft: \" + COUNTER + \" Weak: \" + COUNTER + \" Final: \" + COUNTER + \" Phantom: \" + COUNTER);\n\n    GCParseRule UNIFIED_META_DATA = new GCParseRule(\"UNIFIED_META_DATA\", UNIFIED_META_RECORD);\n\n    GCParseRule END_OF_FILE = new GCParseRule(\"END_OF_FILE\", GCLogParser.END_OF_DATA_SENTINEL);\n    GCParseRule JVM_EXIT = new GCParseRule(\"JVM_EXIT\", \"Heap$\");\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedSurvivorMemoryPoolParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.event.jvm.SurvivorRecord;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\n\nimport java.util.Set;\n\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.CPU_BREAKOUT;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.JVM_EXIT;\n\npublic class UnifiedSurvivorMemoryPoolParser extends UnifiedGCLogParser implements TenuredPatterns {\n\n    /**\n     * [16.962s][debug][gc,age       ] GC(14) Desired survivor size 10485760 bytes, new threshold 15 (max threshold 15)\n     * [16.973s][trace][gc,age       ] GC(14) Age table with threshold 15 (max threshold 15)\n     * [16.973s][trace][gc,age       ] GC(14) - age   1:     768744 bytes,     768744 total\n     * ...\n     * [16.974s][trace][gc,age       ] GC(14) - age  14:     542328 bytes,    8307008 total\n     */\n    private GCParseRule DESIRED_SURVIVOR_SIZE = new GCParseRule(\"DESIRED_SURVIVOR_SIZE\", \"Desired survivor size \" + COUNTER + \" bytes, new threshold \" + COUNTER + \" \\\\(max threshold \" + COUNTER + \"\\\\)\");\n    private GCParseRule AGE_TABLE_HEADER = new GCParseRule(\"AGE_TABLE_HEADER\", \"Age table with threshold \" + COUNTER + \" \\\\(max threshold \" + COUNTER + \"\\\\)\");\n    private GCParseRule AGE_RECORD = new GCParseRule(\"AGE_RECORD\", \"- age\\\\s+\" + COUNTER + \":\\\\s+\" + COUNTER + \" bytes,\\\\s+\" + COUNTER + \" total\");\n\n    private SurvivorRecord forwardReference = null;\n    private boolean ageDataCollected = false;\n\n    public UnifiedSurvivorMemoryPoolParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.SURVIVOR);\n    }\n\n    public String getName() {\n        return \"SurvivorMemoryPoolParser\";\n    }\n\n    @Override\n    protected void process(String entry) {\n        GCLogTrace trace;\n\n        if ((trace = DESIRED_SURVIVOR_SIZE.parse(entry)) != null) {\n            forwardReference = new SurvivorRecord(new Decorators(entry).getDateTimeStamp(), trace.getLongGroup(1), trace.getIntegerGroup(2), trace.getIntegerGroup(3));\n        } else if ((trace = AGE_TABLE_HEADER.parse(entry)) != null) {\n            //we've collected this data so.. eat it...\n        } else if ((trace = AGE_RECORD.parse(entry)) != null) {\n            if (forwardReference != null) {\n                forwardReference.add(trace.getIntegerGroup(1), trace.getLongGroup(2));\n                ageDataCollected = true;\n            }\n        } else if (entry.equals(END_OF_DATA_SENTINEL) || (JVM_EXIT.parse(entry) != null)) {\n            if (forwardReference != null)\n                publish(forwardReference);\n            publish(new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n        } else if (forwardReference != null && ageDataCollected) {\n            publish(forwardReference);\n            forwardReference = null;\n            ageDataCollected = false;\n        } else if (CPU_BREAKOUT.parse(entry) != null) {\n            if (forwardReference != null) {\n                publish(forwardReference);\n                forwardReference = null;\n                ageDataCollected = false;\n            }\n        }\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return diary.isTenuringDistribution() && diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n\n    private void publish(JVMEvent event) {\n        super.publish(ChannelName.SURVIVOR_MEMORY_POOL_PARSER_OUTBOX, event);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedG1GCParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.GCToolKit;\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.StatisticalSummary;\nimport com.microsoft.gctoolkit.event.SurvivorMemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentScanRootRegion;\nimport com.microsoft.gctoolkit.event.g1gc.G1Cleanup;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentCleanup;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentMark;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentMarkResetForOverflow;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentStringDeduplication;\nimport com.microsoft.gctoolkit.event.g1gc.G1FullGCNES;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCConcurrentEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCPauseEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1Mixed;\nimport com.microsoft.gctoolkit.event.g1gc.G1Remark;\nimport com.microsoft.gctoolkit.event.g1gc.G1SystemGC;\nimport com.microsoft.gctoolkit.event.g1gc.G1Trap;\nimport com.microsoft.gctoolkit.event.g1gc.G1Young;\nimport com.microsoft.gctoolkit.event.g1gc.G1YoungInitialMark;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.collection.MRUQueue;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.AbstractMap;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentLinkedQueue;\nimport java.util.function.BiConsumer;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n/**\n * TODO No reports or views generated from this data yet.\n * <p>\n * Result\n * -0 on when GC started\n * - type of GC triggered\n * - from, to, configured\n * - pause time if it is reported or can be calculated\n */\npublic class PreUnifiedG1GCParser extends PreUnifiedGCLogParser implements G1GCPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(PreUnifiedG1GCParser.class.getName());\n\n    //values show up at the end of GC log file from a normally terminated JVM\n    @SuppressWarnings(\"unused\")\n    private long heapTotal, heapUsed;\n    @SuppressWarnings(\"unused\")\n    private long regionSize;\n    @SuppressWarnings(\"unused\")\n    private long metaSpaceUsed, metaCapacity, metaCommitted, metaReserved;\n    @SuppressWarnings(\"unused\")\n    private long classSpaceUsed, classSpaceCapacity, classSpaceCommitted, classSpaceReserved;\n\n    private final G1GCPauseEvent trap = new G1Trap();\n    private G1GCPauseEvent forwardReference = trap;\n\n    private DateTimeStamp timeStampForwardReference;\n    private GCCause gcCauseForwardReference;\n    private GarbageCollectionTypes collectionTypeForwardReference;\n    private ReferenceGCSummary referenceGCForwardReferenceSummary;\n\n    private DateTimeStamp concurrentPhaseStartTimeStamp;\n    private GarbageCollectionTypes concurrentCollectionTypeForwardReference;\n    private final ConcurrentLinkedQueue<JVMEvent> backlog = new ConcurrentLinkedQueue<>();\n\n    private final MRUQueue<GCParseRule, BiConsumer<GCLogTrace, String>> parseRules;\n\n    {\n        parseRules = new MRUQueue<>();\n        parseRules.put(CPU_BREAKDOWN, this::recordCPUSummary);\n        parseRules.put(G1_DETAILS, this::processYoungGenCollection);\n        parseRules.put(YOUNG, this::processYoung);\n        parseRules.put(FULL_GC, this::g1gcFullGC); //113.325: [Full GC (System.gc()) 37M->32M(96M), 0.0943030 secs]\n        parseRules.put(G1_MEMORY_SUMMARY, this::processMemorySummary);\n        parseRules.put(G1_NO_DETAILS_MEMORY_SUMMARY, this::processNoDetailsMemorySummary);\n        parseRules.put(G1_PARALLEL_PHASE_SUMMARY, this::processParallelPhaseSummary);\n        parseRules.put(G1_SOLARIS_PARALLEL_PHASE, this::processParallelPhase);\n        parseRules.put(TERMINATION_ATTEMPTS, this::ignore);\n        parseRules.put(PROCESSED_BUFFERS, this::processedBuffers);\n        parseRules.put(PROCESSED_BUFFER, this::processedBuffer);\n        parseRules.put(SOLARIS_WORKER_PARALLEL_BLOCK, this::solarisWorkerParallelBlock);\n        parseRules.put(SOLARIS_WORKER_PARALLEL_ACTIVITY, this::ignore);\n        parseRules.put(G1GC_PHASE_DETAIL_CLAUSE, this::processPhaseDetailClause);\n        parseRules.put(G1GC_PHASE, this::processG1GCPhase);\n        parseRules.put(CONCURRENT_STRING_DEDUP, this::concurrentStringDedup);\n        parseRules.put(STRING_DEDUP_FIXUP, this::stringDedupFixup);\n        parseRules.put(QUEUE_FIXUP, this::recordQueueFixup);\n        parseRules.put(TABLE_FIXUP, this::recordTableFixup);\n        parseRules.put(WORKER_ACTIVITY, this::workerActivity);\n        parseRules.put(PARALLEL_TIME, this::parallelTime);\n        parseRules.put(WORKER_PARALLEL_BLOCK, this::workerParallelBlock);\n        parseRules.put(G1_FUll, this::g1gcDetailedFullGC);\n        parseRules.put(G1_FULL_INTERRUPTS_CONCURRENT_CYCLE, this::g1FullInterruptsConcurrentCycle);\n        parseRules.put(FULL_WITH_CONCURRENT_PHASE_START, this::g1FullWithConcurrentPhaseStart);\n        parseRules.put(FULL_WITH_CONCURRENT_PHASE_CORRUPTED, this::g1FullWithConcurrentPhaseCorrupted);\n        parseRules.put(FULL_MISSING_TIMESTAMP_CONCURRENT_START, this::fullMissingTimeStampWithConcurrentStart);\n        parseRules.put(FULL_WITH_CONCURRENT_PHASE_INTERLEAVED, this::g1FullWIthConcurrentPhaseInterleaved);\n        parseRules.put(FULL_WITH_CONCURRENT_END, this::fullWithConcurrentEnd);\n\n        parseRules.put(FULL_GC_FRAGMENT, this::fullGCFragment);\n        parseRules.put(CONCURRENT_START_V3, this::concurrentStartV3);\n        parseRules.put(CONCURRENT_START_V4, this::concurrentStartV4);\n        parseRules.put(CONCURRENT_START_V5, this::concurrentStartV3);\n\n        parseRules.put(CORRUPTED_CONCURRENT_START, this::corruptedConcurrentStart);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V2, this::corruptedConcurrentStartV2);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V3, this::corruptedConcurrentStartV3);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V4, this::corruptedConcurrentStartV4);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V5, this::corruptedConcurrentStartV5);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V7, this::corruptedConcurrentStartV7);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V8, this::corruptedConcurrentStartV8);\n        parseRules.put(CORRUPTED_CONCURRENT_START_V9, this::corruptedConcurrentStartV9);\n        parseRules.put(YOUNG_WITH_CONCURRENT_END, this::youngWithConcurrentEnd);\n        parseRules.put(YOUNG_SPLIT_AT_DATESTAMP, this::youngSplitAtDatestamp);\n        parseRules.put(YOUNG_SPLIT_AT_TIMESTAMP, this::youngSplitAtTimestamp);\n        parseRules.put(FREE_FLOATING_YOUNG_BLOCK, this::freeFloatingYoungBlock);\n        parseRules.put(FULLGC_WITH_CONCURRENT_PHASE, this::fullGCWithConcurrentPhase);\n\n        parseRules.put(G1_YOUNG_SPLIT_START, this::g1YoungSplitStart);\n        parseRules.put(G1_YOUNG_SPLIT_END, this::g1YoungSplitEnd);\n        parseRules.put(G1_INITIAL_MARK, this::g1InitialMark);\n        parseRules.put(FREE_FLOATING_OCCUPANCY_SUMMARY, this::freeFloatingOccupancySummary);\n\n        parseRules.put(G1_CONCURRENT_START, this::g1ConcurrentStart);\n        parseRules.put(G1_CONCURRENT_START_WITHOUT_PREFIX, this::g1ConcurrentStartWithoutPrefix);\n        parseRules.put(G1_CONCURRENT_END, this::g1ConcurrentEnd);\n        parseRules.put(G1_REMARK, this::g1Remark);\n        parseRules.put(G1_180_REMARK, this::g1180Remark);\n        parseRules.put(G1_180_REMARK_REF_DETAILS, this::g1180RemarkRefDetails);\n        parseRules.put(G1_CLEANUP, this::g1Cleanup);\n        parseRules.put(G1_CLEANUP_NO_MEMORY, this::g1CleanupNoMemory);\n        parseRules.put(G1_CONCURRENT_ABORT, this::g1ConcurrentAbort);\n        parseRules.put(CONCURRENT_MARK_OVERFLOW, this::concurrentMarkOverflow);\n        parseRules.put(G1_CORRUPTED_CONCURRENT_END, this::g1CorruptedConcurrentEnd);\n        parseRules.put(G1_CORRUPTED_CONCURRENT_ROOT_REGION_SCAN_END, this::g1ConcurrentEndCorruptedByApplicationTime);\n        parseRules.put(G1_FLOATING_CONCURRENT_PHASE_START, this::g1FloatingConcurrentPhaseStart);\n        parseRules.put(SPLIT_CLEANUP, this::splitCleanup);\n\n        //adaptive sizing\n        parseRules.put(YOUNG_SPLIT_BY_G1ERGONOMICS, this::youngSplitByG1Ergonomics);\n        parseRules.put(G1_INITIAL_MARK_ERGONOMICS, this::g1InitialMarkErgonomics);\n\n        //RSet rules\n        parseRules.put(G1_YOUNG_RS_SUMMARY, this::ignore);\n...\n", "needle_token_start": 7199, "needle_token_end": 7400, "code_context_ntokens": 16193, "output": ["```java\nprivate void extractValues(String line) {\n\n    if (!line.startsWith(\"[\"))\n        return;\n\n    decoratorMatcher = UnifiedLoggingTokens.DECORATORS.matcher(line);\n    if (!decoratorMatcher.find()) {\n        return;\n    }\n\n    for ( int i = 1; i <= decoratorMatcher.groupCount(); i++) {\n        if ( decoratorMatcher.group(i) != null)\n            numberOfDecorators++;\n    }\n\n    Matcher tagMatcher = UnifiedLoggingTokens.TAGS.matcher(line);\n    if (tagMatcher.find()) {\n        numberOfDecorators++;\n        tags = String.join(\",\", Arrays.asList(tagMatcher.group(1).trim().split(\",\")));\n    }\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "getMetaSpaceRecord", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogTrace.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to compute and return a record of memory usage statistics for a specific memory area before and after a certain operation, as well as its total size.\n2. **Input**: It accepts an integer representing the starting position in a dataset from which memory usage data should be extracted.\n3. **Output**: It returns an object containing three long integers representing the memory usage before the operation, after the operation, and the total size of the memory area, respectively. If the data cannot be processed, it returns null.\n4. **Procedure**: The function attempts to extract and convert memory usage data from the dataset starting at the given position. It uses helper methods to convert this data from various units into kilobytes. If successful, it creates and returns an object with the calculated values. If any errors occur during data conversion, it logs a message and returns null.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedTokens.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\n/**\n * Interface that contains String and ParseRule representations of common\n * data types within GC Log data.These tokens are used in Regular Expressions\n * Pattern matching in the GC Log parser(s)\n */\npublic interface PreUnifiedTokens extends GenericTokens {\n\n    // Time values\n    String TIMESTAMP = \"(\" + INTEGER + DECIMAL_POINT + \"\\\\d{3}): \";\n    String MS_TIME_STAMP = TIME + \" ms\";\n\n    String DATE_STAMP = \"(\" + DATE + \"): \";\n    String DATE_TIMESTAMP = \"(?:\" + DATE_STAMP + \")?\" + TIMESTAMP;\n    //DECIMAL_POINT = \"(?:\\\\.|,)\";\n    //    String INTEGER = \"\\\\d+\";\n    //\"(?:(\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}[\\\\+|\\\\-]\\\\d{4}): )?(\" + \"\\\\d+(?:\\\\.|,)\\\\d{3}): (?:#\\\\d+: )?\\\\[GC ?(\\\\([a-zA-Z\\\\. 1]+?\\\\(?\\\\){1,2})?\\\\s*\";\n    GCParseRule DATE_TIMESTAMP_RECORD = new GCParseRule(\"DATE_TIMESTAMP_RECORD\", DATE_TIMESTAMP);\n\n    // Memory values\n    String META_PERM_MEMORY_SUMMARY = \"(?:\" + MEMORY_SIZE + \"->)\" + MEMORY_SIZE + \"\\\\(\" + MEMORY_SIZE + \"\\\\)\";\n\n    // Fractional memory values\n    String FRACTIONAL_MEMORY_SIZE = \"(\" + REAL_NUMBER + \")\" + UNITS;\n    String FRACTIONAL_BEFORE_AFTER_CONFIGURED = FRACTIONAL_MEMORY_SIZE + \"->\" + FRACTIONAL_MEMORY_SIZE + \"\\\\(\" + FRACTIONAL_MEMORY_SIZE + \"\\\\)\";\n\n    // GC Prefixes\n    String GC_PREFIX = DATE_TIMESTAMP + \"(?:#\\\\d+: )?\\\\[GC ?\" + GC_CAUSE;\n    String FULL_GC_PREFIX = DATE_TIMESTAMP + \"(?:#\\\\d+: )?\\\\[Full GC ?\" + GC_CAUSE;\n\n    // Pre 1.8.0 records\n    // Update: no longer only pre 1.8.0, updated to fix Metaspace in CMF records\n    String PERM_RECORD = \"\\\\[(CMS Perm |PS Perm |Perm |PSPermGen|Metaspace): \" + META_PERM_MEMORY_SUMMARY + \"\\\\]\";\n\n    String USED_CAPACITY_COMMITTED_RESERVED = \"used \" + MEMORY_SIZE + \", capacity \" + MEMORY_SIZE + \", committed \" + MEMORY_SIZE + \", reserved \" + MEMORY_SIZE;\n    GCParseRule METASPACE_FINAL = new GCParseRule(\"METASPACE_FINAL\", \"Metaspace\\\\s+\" + USED_CAPACITY_COMMITTED_RESERVED);\n    GCParseRule CLASSPACE_FINAL = new GCParseRule(\"CLASSPACE_FINAL\", \"class space\\\\s+\" + USED_CAPACITY_COMMITTED_RESERVED);\n\n    // HeapAtGC tokens\n    String MEMORY_ADDRESS = \"(0x[0-9,a-f]{16})\";\n    String MEMORY_POOL_BOUNDS = \"\\\\[\" + MEMORY_ADDRESS + \",\" + MEMORY_ADDRESS + \",\" + MEMORY_ADDRESS + \"\\\\]\";\n\n    // CPU record\n    // [Times: user=7.96 sys=0.03, real=4.38 secs]\n    GCParseRule CPU_BREAKDOWN = new GCParseRule(\"CPU_BREAKDOWN\", CPU_SUMMARY);\n\n    // Reference processing block\n    // 11906.881: [SoftReference, 0 refs, 0.0000060 secs]\n    // 11906.881: [WeakReference, 0 refs, 0.0000020 secs]\n    // 11906.881: [FinalReference, 0 refs, 0.0000010 secs]\n    // 11906.881: [PhantomReference, 0 refs, 0.0000020 secs]\n    String REFERENCE_PROCESSING_BLOCK = DATE_TIMESTAMP + \"\\\\[(Soft|Weak|Final)Reference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\";\n    GCParseRule SOFT_REFERENCE = new GCParseRule(\"SOFT_REFERENCE\", DATE_TIMESTAMP + \"\\\\[SoftReference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\");\n    GCParseRule WEAK_REFERENCE = new GCParseRule(\"WEAK_REFERENCE\", DATE_TIMESTAMP + \"\\\\[WeakReference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\");\n    GCParseRule FINAL_REFERENCE = new GCParseRule(\"FINAL_REFERENCE\", DATE_TIMESTAMP + \"\\\\[FinalReference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\");\n\n    //[PhantomReference, 0 refs, 0 refs, 0.0006649 secs]\n    String PHANTOM_REFERENCE_PROCESSING = DATE_TIMESTAMP + \"\\\\[PhantomReference, \" + COUNTER + \" refs, (?:\" + COUNTER + \" refs, )?\" + PAUSE_TIME + \"\\\\]\";\n    GCParseRule PHANTOM_REFERENCE = new GCParseRule(\"PHANTOM_REFERENCE\", PHANTOM_REFERENCE_PROCESSING);\n    // JNI Reference processing\n    // 11906.881: [JNI Weak Reference, 0.0002710 secs]\n    //2.833: [JNI Weak Reference, 239 refs, 0.0000939 secs]\"\n    String JNI_REFERENCE_PROCESSING = DATE_TIMESTAMP + \"\\\\[JNI Weak Reference, (?:\" + COUNTER + \" refs, )?\" + PAUSE_TIME + \"\\\\]\";\n    GCParseRule JNI_REFERENCE = new GCParseRule(\"JNI_REFERENCE\", JNI_REFERENCE_PROCESSING);\n\n    String REFERENCE_RECORDS = REFERENCE_PROCESSING_BLOCK + REFERENCE_PROCESSING_BLOCK + REFERENCE_PROCESSING_BLOCK + PHANTOM_REFERENCE_PROCESSING + JNI_REFERENCE_PROCESSING;\n\n    //2016-10-06T08:48:07.320+0200: 2002,085: [Preclean SoftReferences, 0,0000050 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean WeakReferences, 0,0000283 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean FinalReferences, 0,0000036 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean PhantomReferences, 0,0000354 secs]2016-10-06T08:48:07.330+0200: 2002,095: [CMS-concurrent-preclean: 0,010/0,011 secs] [Times: user=0,01 sys=0,00, real=0,01 secs]\n...\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/AbstractLogTrace.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n\npublic abstract class AbstractLogTrace {\n\n    private static final String US_FORMAT_SEPARATOR = \".\";\n    private static final String EUROPEAN_FORMAT_SEPARATOR = \",\";\n    private static final double MISSING_TIMESTAMP_SENTINEL = -1.0d;\n\n    private static final Pattern TIMES_STAMP_RULE = Pattern.compile(PreUnifiedTokens.TIMESTAMP);\n    private static final Pattern DATE_TIME_STAMP_RULE = Pattern.compile(PreUnifiedTokens.DATE_TIMESTAMP);\n    protected final Matcher trace;\n\n    public AbstractLogTrace(Matcher matcher) {\n        this.trace = matcher;\n    }\n\n    public int length() {\n        return trace.group(0).length();\n    }\n\n    public int groupCount() {\n        return trace.groupCount();\n    }\n\n    public boolean groupNotNull(int index) {\n        return getGroup(index) != null;\n    }\n\n    public long getLongGroup(int index) {\n        return Long.parseLong(trace.group(index));\n    }\n\n    public int getIntegerGroup(int index) {\n        return Integer.parseInt(trace.group(index));\n    }\n\n    public String getGroup(int index) {\n        return trace.group(index);\n    }\n\n    public double getDoubleGroup(int index) {\n        return convertToDouble(trace.group(index));\n    }\n\n    protected double convertToDouble(String value) {\n        return Double.parseDouble(value.replaceAll(EUROPEAN_FORMAT_SEPARATOR, US_FORMAT_SEPARATOR));\n    }\n\n    public double getPercentageGroup(int i) {\n        String value = getGroup(i);\n        return convertToDouble(value.substring(0, value.length() - 1));\n    }\n\n    public double getTimeStamp() {\n        Matcher matcher = TIMES_STAMP_RULE.matcher(trace.group(0));\n        if (!matcher.find()) {\n            return MISSING_TIMESTAMP_SENTINEL;\n        }\n        return getDoubleGroup(2);\n    }\n\n    public String getDateStamp() {\n        Matcher matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0));\n        if (matcher.find()) {\n            return matcher.group(1);\n        } else {\n            return null;\n        }\n    }\n\n    public DateTimeStamp getDateTimeStamp() {\n        return getDateTimeStamp(1);\n    }\n\n    public DateTimeStamp getDateTimeStamp(int index) {\n        Matcher matcher = DATE_TIME_STAMP_RULE.matcher(trace.group(0));\n        for (int i = 0; i < index - 1; i++)\n            if (!matcher.find())\n                break;\n        if (matcher.find()) {\n            if (matcher.group(2) == null) {\n                return new DateTimeStamp(matcher.group(1));\n            }\n            return new DateTimeStamp(matcher.group(1), convertToDouble(matcher.group(2)));\n        }\n        return new DateTimeStamp(MISSING_TIMESTAMP_SENTINEL);\n    }\n\n    public GCLogTrace next() {\n        if (trace.find())\n            return new GCLogTrace(trace);\n        return null;\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/Tokens.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\n/**\n * Interface that contains String and ParseRule representations of common\n * data types within GC Log data.These tokens are used in Regular Expressions\n * Pattern matching in the GC Log parser(s)\n */\npublic interface Tokens {\n\n    // Primitives\n    String DECIMAL_POINT = \"(?:\\\\.|,)\";\n    String INTEGER = \"\\\\d+\";\n    String REAL_NUMBER = INTEGER + DECIMAL_POINT + INTEGER;\n    String PERCENTAGE = \"(\" + REAL_NUMBER + \")\" + \"\\\\s?%\";\n    String HEX = \"0x[0-9,a-f]{16}\";\n    String COUNTER = \"(\" + INTEGER + \")\";\n    String BYTES = COUNTER;\n    String REAL_VALUE = \"(\" + REAL_NUMBER + \")\";\n\n    String UNITS = \"([B,K,M,G])\";\n\n    // Time values\n    String TIME = \"(-?\" + REAL_NUMBER + \")\";\n    String TIMESTAMP = \"(\" + INTEGER + DECIMAL_POINT + \"\\\\d{3}): \";\n    String PAUSE_TIME = TIME + \" (?:secs?|ms)\";\n    String MS_TIME_STAMP = TIME + \" ms\";\n\n    // Date values\n    String DATE = \"\\\\d{4}-\\\\d{2}-\\\\d{2}T\\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}[\\\\+|\\\\-]\\\\d{4}\";\n    String DATE_STAMP = \"(\" + DATE + \"): \";\n    String DATE_TIMESTAMP = \"(?:\" + DATE_STAMP + \")?\" + TIMESTAMP;\n    GCParseRule DATE_TIMESTAMP_RECORD = new GCParseRule(\"DATE_TIMESTAMP_RECORD\", DATE_TIMESTAMP);\n\n    // Memory values\n    String MEMORY_SIZE = \"(\" + INTEGER + \")\" + UNITS;\n    String BEFORE_AFTER_CONFIGURED = MEMORY_SIZE + \"->\" + MEMORY_SIZE + \"\\\\(\" + MEMORY_SIZE + \"\\\\)\";\n    String BEFORE_AFTER_CONFIGURED_PAUSE = BEFORE_AFTER_CONFIGURED + \", \" + PAUSE_TIME;\n    String META_PERM_MEMORY_SUMMARY = \"(?:\" + MEMORY_SIZE + \"->)\" + MEMORY_SIZE + \"\\\\(\" + MEMORY_SIZE + \"\\\\)\";\n\n    // Fractional memory values\n    String FRACTIONAL_MEMORY_SIZE = \"(\" + REAL_NUMBER + \")\" + UNITS;\n    String FRACTIONAL_BEFORE_AFTER_CONFIGURED = FRACTIONAL_MEMORY_SIZE + \"->\" + FRACTIONAL_MEMORY_SIZE + \"\\\\(\" + FRACTIONAL_MEMORY_SIZE + \"\\\\)\";\n\n    // CMS cycles and Perm Space report using this format\n    String OCCUPANCY_CONFIGURED = MEMORY_SIZE + \"\\\\(\" + MEMORY_SIZE + \"\\\\)\";\n    String OCCUPANCY_CONFIGURED_PAUSE = OCCUPANCY_CONFIGURED + \", \" + PAUSE_TIME;\n\n    // Post 1.7.0_40 clauses\n    //\"(\\\\(.+?\\\\)\\\\s?)? ?\"; this rule was too liberal\n    String GC_CAUSE = \"(\\\\(.{4,50}\\\\))?\\\\s*\";\n\n    // GC Prefixes\n    String GC_PREFIX = DATE_TIMESTAMP + \"(?:#\\\\d+: )?\\\\[GC ?\" + GC_CAUSE;\n    String FULL_GC_PREFIX = DATE_TIMESTAMP + \"(?:#\\\\d+: )?\\\\[Full GC ?\" + GC_CAUSE;\n\n    // Pre 1.8.0 records\n    // Update: no longer only pre 1.8.0, updated to fix Metaspace in CMF records\n    String PERM_RECORD = \"\\\\[(CMS Perm |PS Perm |Perm |PSPermGen|Metaspace): \" + META_PERM_MEMORY_SUMMARY + \"\\\\]\";\n\n    // 1.8.0 tokens and rules\n    String META_RECORD = \"\\\\[Metaspace: \" + BEFORE_AFTER_CONFIGURED + \"\\\\]\";\n\n    String USED_CAPACITY_COMMITTED_RESERVED = \"used \" + MEMORY_SIZE + \", capacity \" + MEMORY_SIZE + \", committed \" + MEMORY_SIZE + \", reserved \" + MEMORY_SIZE;\n    GCParseRule METASPACE_FINAL = new GCParseRule(\"METASPACE_FINAL\", \"Metaspace\\\\s+\" + USED_CAPACITY_COMMITTED_RESERVED);\n    GCParseRule CLASSPACE_FINAL = new GCParseRule(\"CLASSPACE_FINAL\", \"class space\\\\s+\" + USED_CAPACITY_COMMITTED_RESERVED);\n\n    // HeapAtGC tokens\n    String MEMORY_ADDRESS = \"(0x[0-9,a-f]{16})\";\n    String MEMORY_POOL_BOUNDS = \"\\\\[\" + MEMORY_ADDRESS + \",\" + MEMORY_ADDRESS + \",\" + MEMORY_ADDRESS + \"\\\\]\";\n\n    // CPU record\n    // [Times: user=7.96 sys=0.03, real=4.38 secs]\n    GCParseRule CPU_BREAKDOWN = new GCParseRule(\"CPU_BREAKDOWN\", \"\\\\[Times: user=\" + REAL_VALUE + \" sys=\" + REAL_VALUE + \", real=\" + PAUSE_TIME + \"\\\\]\");\n\n    // Reference processing block\n    // 11906.881: [SoftReference, 0 refs, 0.0000060 secs]\n    // 11906.881: [WeakReference, 0 refs, 0.0000020 secs]\n    // 11906.881: [FinalReference, 0 refs, 0.0000010 secs]\n    // 11906.881: [PhantomReference, 0 refs, 0.0000020 secs]\n    String REFERENCE_PROCESSING = DATE_TIMESTAMP + \"\\\\[(Soft|Weak|Final)Reference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\";\n    GCParseRule SOFT_REFERENCE = new GCParseRule(\"SOFT_REFERENCE\", DATE_TIMESTAMP + \"\\\\[SoftReference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\");\n    GCParseRule WEAK_REFERENCE = new GCParseRule(\"WEAK_REFERENCE\", DATE_TIMESTAMP + \"\\\\[WeakReference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\");\n    GCParseRule FINAL_REFERENCE = new GCParseRule(\"FINAL_REFERENCE\", DATE_TIMESTAMP + \"\\\\[FinalReference, \" + COUNTER + \" refs, \" + PAUSE_TIME + \"\\\\]\");\n\n    //[PhantomReference, 0 refs, 0 refs, 0.0006649 secs]\n    String PHANTOM_REFERENCE_PROCESSING = DATE_TIMESTAMP + \"\\\\[PhantomReference, \" + COUNTER + \" refs, (?:\" + COUNTER + \" refs, )?\" + PAUSE_TIME + \"\\\\]\";\n    GCParseRule PHANTOM_REFERENCE = new GCParseRule(\"PHANTOM_REFERENCE\", PHANTOM_REFERENCE_PROCESSING);\n    // JNI Reference processing\n    // 11906.881: [JNI Weak Reference, 0.0002710 secs]\n    //2.833: [JNI Weak Reference, 239 refs, 0.0000939 secs]\"\n    String JNI_REFERENCE_PROCESSING = DATE_TIMESTAMP + \"\\\\[JNI Weak Reference, (?:\" + COUNTER + \" refs, )?\" + PAUSE_TIME + \"\\\\]\";\n    GCParseRule JNI_REFERENCE = new GCParseRule(\"JNI_REFERENCE\", JNI_REFERENCE_PROCESSING);\n\n    String REFERENCE_RECORDS = REFERENCE_PROCESSING + REFERENCE_PROCESSING + REFERENCE_PROCESSING + PHANTOM_REFERENCE_PROCESSING + JNI_REFERENCE_PROCESSING;\n\n    //2016-10-06T08:48:07.320+0200: 2002,085: [Preclean SoftReferences, 0,0000050 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean WeakReferences, 0,0000283 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean FinalReferences, 0,0000036 secs]2016-10-06T08:48:07.320+0200: 2002,085: [Preclean PhantomReferences, 0,0000354 secs]2016-10-06T08:48:07.330+0200: 2002,095: [CMS-concurrent-preclean: 0,010/0,011 secs] [Times: user=0,01 sys=0,00, real=0,01 secs]\n    //11.943: [Preclean SoftReferences, 0.0000081 secs]11.943: [Preclean WeakReferences, 0.0000145 secs]11.943: [Preclean FinalReferences, 0.0000276 secs]11.943: [Preclean PhantomReferences, 0.0000081 secs]\n    String PRECLEAN_REFERENCE_PROCESSING = DATE_TIMESTAMP + \"\\\\[Preclean (Soft|Weak|Final|Phantom)References, \" + PAUSE_TIME + \"\\\\]\";\n    String PRECLEAN_REFERENCE_RECORDS = PRECLEAN_REFERENCE_PROCESSING + PRECLEAN_REFERENCE_PROCESSING + PRECLEAN_REFERENCE_PROCESSING + PRECLEAN_REFERENCE_PROCESSING;\n\n    // PrintOldPLAB\n    // 0[3]: 6/61264/16\n    String PLAB = COUNTER + \"\\\\[\" + COUNTER + \"\\\\]: \" + COUNTER + \"\\\\/\" + COUNTER + \"\\\\/\" + COUNTER;\n\n    //PrintReferenceGC\n    //[SoftReference, 0 refs, 0.0006965 secs]7.643: [WeakReference, 40 refs, 0.0003039 secs]7.643: [FinalReference, 3477 refs, 0.0036191 secs]7.647: [PhantomReference, 0 refs, 0 refs, 0.0006649 secs]7.647: [JNI Weak Reference, 0.0000119 secs]\n\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/SurvivorMemoryPoolParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.event.jvm.SurvivorRecord;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\n\nimport java.util.Set;\n\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.JVM_EXIT;\n\npublic class SurvivorMemoryPoolParser extends PreUnifiedGCLogParser implements TenuredPatterns {\n\n    private SurvivorRecord forwardReference = null;\n\n    public SurvivorMemoryPoolParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.SURVIVOR);\n    }\n\n    public String getName() {\n        return \"SurvivorMemoryPoolParser\";\n    }\n\n    /**\n     * 61.572: [GC 61.572: [ParNew\n     * Desired survivor size 1343488 bytes, new threshold 4 (max 4)\n     * - age   1:      25176 bytes,      25176 total\n     *\n     * @param entry : GC log entry to deriveConfiguration\n     */\n    @Override\n    protected void process(String entry) {\n        GCLogTrace trace;\n\n        if ((trace = TENURING_SUMMARY.parse(entry)) != null) {\n            forwardReference = new SurvivorRecord(getClock(), trace.getLongGroup(1), trace.getIntegerGroup(2), trace.getIntegerGroup(3));\n        } else if ((trace = TENURING_AGE_BREAKDOWN.parse(entry)) != null) {\n            forwardReference.add(trace.getIntegerGroup(1), trace.getLongGroup(2));\n        } else if (entry.equals(END_OF_DATA_SENTINEL) || (JVM_EXIT.parse(entry) != null)) {\n            if (forwardReference != null)\n                super.publish(ChannelName.SURVIVOR_MEMORY_POOL_PARSER_OUTBOX, forwardReference);\n            super.publish(ChannelName.SURVIVOR_MEMORY_POOL_PARSER_OUTBOX, new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n        } else if (forwardReference != null) {\n            super.publish(ChannelName.SURVIVOR_MEMORY_POOL_PARSER_OUTBOX, forwardReference);\n            forwardReference = null;\n        }\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return diary.isTenuringDistribution() && ! diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel channel) {\n        super.publishTo(channel);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/GCLogTrace.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GCCauses;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.RegionSummary;\nimport com.microsoft.gctoolkit.event.UnifiedCountSummary;\nimport com.microsoft.gctoolkit.event.UnifiedStatisticalSummary;\nimport com.microsoft.gctoolkit.event.jvm.MetaspaceRecord;\nimport com.microsoft.gctoolkit.event.jvm.PermGenSummary;\n\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\nimport java.util.regex.Matcher;\n\n/**\n * Class that represents a chunk of GC log that we are attempting to match to a\n * known GC log pattern\n */\npublic class GCLogTrace extends AbstractLogTrace {\n\n    private static final Logger LOGGER = Logger.getLogger(GCLogTrace.class.getName());\n\n    private final boolean gcCauseDebugging = Boolean.getBoolean(\"microsoft.debug.gccause\");\n    // private final boolean debugging = Boolean.getBoolean(\"microsoft.debug\");\n\n    public GCLogTrace(Matcher matcher) {\n        super(matcher);\n    }\n\n    public int groupCount() {\n        return trace.groupCount();\n    }\n\n    public boolean groupNotNull(int index) {\n        return getGroup(index) != null;\n    }\n\n    public long getLongGroup(int index) {\n        return Long.parseLong(trace.group(index));\n    }\n\n    public int getIntegerGroup(int index) {\n        return Integer.parseInt(trace.group(index));\n    }\n\n    public GCCause gcCause(int base, int offset) {\n        if (gcCauseDebugging) {\n            if (this.getGroup(base + offset) != null) {\n                LOGGER.fine(\"GC cause: \" + getGroup(base + offset));\n                notYetImplemented();\n            }\n        }\n        return GCCauses.get(getGroup(base + offset));\n    }\n\n    public GCCause gcCause(int offset) {\n        return gcCause(3, offset);\n    }\n\n    public GCCause gcCause() {\n        return gcCause(3, 0);\n    }\n\n    public double getPauseTime() {\n        return getDoubleGroup(groupCount());\n    }\n\n    public double getDuration() {\n        return getDoubleGroup(groupCount());\n    }\n\n    public double getDurationInSeconds() {\n        return getDuration() / 1000.00d;\n    }\n\n    /**\n     * Annoyingly we're assuming the field actually is ms instead of confirming\n     * @param index Index of the capture group.\n     * @return The capture group parsed to a double.\n     */\n    public double getMilliseconds(int index) {\n        return getDoubleGroup(index);\n    }\n\n    /**\n     * Assumed to be the last capture group\n     * @return The last capture group parsed to a double.\n     */\n    public double getMilliseconds() {\n        return getMilliseconds(groupCount());\n    }\n\n    public boolean contains(int index, String value) {\n        String text = getGroup(index);\n        if (text != null)\n            return text.contains(value);\n        return false;\n    }\n\n    public boolean contains(String value) {\n        return trace.group(0).contains(value);\n    }\n\n    @Override\n    public String toString() {\n        return trace.group(0);\n    }\n\n    public boolean hasNext() {\n        return (trace.find());\n    }\n\n    public int end() {\n        return trace.end();\n    }\n\n    public long toKBytes(int offset) {\n        return toKBytes(getLongGroup(offset), getGroup(offset + 1));\n    }\n\n    public long doubleToKBytes(int offset) {\n        return (long)toKBytes(getDoubleGroup(offset), getGroup(offset+1));\n    }\n\n    private double toKBytes(double value, String units) {\n        double returnValue = value;\n        switch (Character.toUpperCase(units.codePointAt(0))) {\n            case 'G':\n                returnValue *= 1024.0D;\n            case 'M':\n                returnValue *= 1024.0D;\n            case 'K':\n                break;\n            case 'B':\n                returnValue /= 1024.0D;\n                break;\n            default:\n                LOGGER.log(Level.WARNING, \"Invalid unit [B,K,M,G] {0}\", units);\n        }\n        return returnValue;\n    }\n\n    public long toKBytes(long value, String units) {\n        long returnValue = value;\n        switch (Character.toUpperCase(units.codePointAt(0))) {\n            case 'G':\n                returnValue *= 1024L;\n            case 'M':\n                returnValue *= 1024L;\n            case 'K':\n                break;\n            case 'B':\n                returnValue /= 1024L;\n                break;\n            default:\n                LOGGER.log(Level.WARNING, \"Invalid unit [B,K,M,G] {0}\", units);\n        }\n\n        return returnValue;\n    }\n\n    public PermGenSummary getMetaspaceSummary(int offset) {\n        try {\n            long before = toKBytes(offset);\n            long after = toKBytes(offset + 2);\n            long size = toKBytes(offset + 4);\n            return new PermGenSummary(before, after, size);\n        } catch (NumberFormatException numberFormatException) {\n            LOGGER.fine(\"Unable to calculate Metaspace summary.\");\n            notYetImplemented();\n        }\n        return null;\n    }\n\n    public MemoryPoolSummary getOccupancyBeforeAfterWithMemoryPoolSizeSummary(int offset) {\n        try {\n            long before = toKBytes(offset);\n            long after = toKBytes(offset + 2);\n            long size = toKBytes(offset + 4);\n            return new MemoryPoolSummary(before, size, after, size);\n        } catch (NumberFormatException numberFormatException) {\n            LOGGER.fine(\"Unable to calculate generational memory pool summary.\");\n            notYetImplemented();\n        }\n\n        return null;\n    }\n\n    public MemoryPoolSummary getOccupancyWithMemoryPoolSizeSummary(int offset) {\n\n        try {\n            long occupancy = toKBytes(offset);\n            long size = toKBytes(offset + 2);\n            return new MemoryPoolSummary(occupancy, size, occupancy, size);\n        } catch (NumberFormatException numberFormatException) {\n            LOGGER.fine(\"Unable to calculate generational memory pool occupancy summary.\");\n            notYetImplemented();\n        }\n\n        return null;\n    }\n\n    \npublic MetaspaceRecord getMetaSpaceRecord(int offset) {\n        try {\n            long before = toKBytes(offset);\n            long after = toKBytes(offset + 2);\n            long size = toKBytes(offset + 4);\n            return new MetaspaceRecord(before, after, size);\n        } catch (NumberFormatException numberFormatException) {\n            LOGGER.fine(\"Unable to calculate Metaspace summary.\");\n            notYetImplemented();\n        }\n        return null;\n    }\n\n    public MetaspaceRecord getEnlargedMemoryPoolRecord(int offset) {\n        try {\n            long before = toKBytes(offset);\n            long after = toKBytes(offset + 4);\n            long size = toKBytes(offset + 6);\n            return new MetaspaceRecord(before, after, size);\n        } catch (NumberFormatException numberFormatException) {\n            LOGGER.fine(\"Unable to calculate Metaspace summary.\");\n            notYetImplemented();\n        }\n        return null;\n    }\n\n    public MetaspaceRecord getEnlargedMetaSpaceRecord(int offset) {\n        try {\n            long before = toKBytes(offset);\n            long sizeBefore = toKBytes(offset + 2);\n            long after = toKBytes(offset + 4);\n            long size = toKBytes(offset + 6);\n            return new MetaspaceRecord(before, sizeBefore, after, size);\n        } catch (NumberFormatException numberFormatException) {\n            LOGGER.fine(\"Unable to calculate Metaspace summary.\");\n            notYetImplemented();\n        }\n        return null;\n    }\n\n    public UnifiedStatisticalSummary getUnifiedStatisticalSummary() {\n        return new UnifiedStatisticalSummary(getDoubleGroup(2), getDoubleGroup(3), getDoubleGroup(4), getDoubleGroup(5), getDoubleGroup(6), getIntegerGroup(7));\n    }\n\n    public UnifiedCountSummary countSummary() {\n        return new UnifiedCountSummary(getIntegerGroup(2), getDoubleGroup(3), getIntegerGroup(4), getIntegerGroup(5), getIntegerGroup(6), getIntegerGroup(7));\n    }\n\n    public RegionSummary regionSummary() {\n        return new RegionSummary(getIntegerGroup(2),\n                getIntegerGroup(3),\n                (trace.group(4) != null) ? getIntegerGroup(4) : getIntegerGroup(3));\n    }\n\n    // Debugging support **PLEASE DO NOT REMOVE**\n    public void notYetImplemented() {\n        String threadName = Thread.currentThread().getName();\n        LOGGER.log(Level.FINE, \"{0}, not implemented: {1}\", new Object[]{threadName, getGroup(0)});\n        for (int i = 1; i < groupCount() + 1; i++) {\n            LOGGER.log(Level.FINE, \"{0} : {1}\", new Object[]{i, getGroup(i)});\n        }\n        LOGGER.fine(\"-----------------------------------------\");\n        //IntelliJ Eats this log output so it's displayed to stdout..\n        //And yes, that means System.out.println is in here in on purpose\n        //if ( debugging) {\n        System.out.println(threadName + \", not implemented: \" + getGroup(0));\n        for (int i = 1; i < groupCount() + 1; i++) {\n            System.out.println(i + \": \" + getGroup(i));\n        }\n        System.out.println(\"-----------------------------------------\");\n        //}\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/JHiccupTrace.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n/**\n * Class that represents a chunk of a jHiccup log that we are attempting to match to a\n */\npublic class JHiccupTrace {\n\n    //0.124,1.004,2.408,HIST\n    private static final String TIME = \"(\\\\d+(?:,|.)\\\\d+)\";\n    public static final Pattern JHICCUP_LOG_ENTRY = Pattern.compile(TIME + \",\" + TIME + \",\" + TIME + \",HIST\");\n    private static final String US_FORMAT_SEPARATOR = \".\";\n    private static final String EUROPEAN_FORMAT_SEPARATOR = \",\";\n\n    // TODO #150 Why -1.0d as the default?\n    // private static final double DEFAULT_TIMESTAMP = -1.0d;\n\n    private final Matcher trace;\n\n    public static JHiccupTrace toTrace(String line) {\n        Matcher m = JHICCUP_LOG_ENTRY.matcher(line);\n        return (m == null) ? null : new JHiccupTrace(m);\n    }\n\n    protected JHiccupTrace(Matcher matcher) {\n        this.trace = matcher;\n    }\n\n    public double getTimeStamp() {\n        return getDoubleGroup(1);\n    }\n\n    public double getInterval() {\n        return getDoubleGroup(2);\n    }\n\n    public double getDuration() {\n        return getDoubleGroup(3);\n    }\n\n    public double getDoubleGroup(int index) {\n        return convertToDouble(trace.group(index));\n    }\n\n    private double convertToDouble(String value) {\n        return Double.parseDouble(value.replaceAll(EUROPEAN_FORMAT_SEPARATOR, US_FORMAT_SEPARATOR));\n    }\n\n    public String getGroup(int index) {\n        return trace.group(index);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/JVMPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\npublic interface JVMPatterns extends PreUnifiedTokens {\n\n\n    //Total time for which application threads were stopped: 0.0000350 seconds\n    GCParseRule SIMPLE_APPLICATION_STOP_TIME = new GCParseRule(\"SIMPLE_APPLICATION_STOP_TIME\", \"^Total time for which application threads were stopped: \" + TIME + \" seconds$\");\n    GCParseRule APPLICATION_STOP_TIME = new GCParseRule(\"APPLICATION_STOP_TIME\", DATE_TIMESTAMP + \"Total time for which application threads were stopped: \" + TIME + \" seconds$\");\n    //4227.515: Total time for which application threads were stopped: 0.0737654 seconds, Stopping threads took: 0.0000558 seconds\n    GCParseRule APPLICATION_STOP_TIME_WITH_STOPPING_TIME = new GCParseRule(\"APPLICATION_STOP_TIME_WITH_STOPPING_TIME\", DATE_TIMESTAMP + \"Total time for which application threads were stopped: \" + TIME + \" seconds, Stopping threads took: \" + TIME + \" seconds\");\n    //Total time for which application threads were stopped: 0.0006115 seconds, Stopping threads took: 0.0003832 seconds\n    GCParseRule UNIFIED_LOGGING_APPLICATION_STOP_TIME_WITH_STOPPING_TIME = new GCParseRule(\"Unified Logging App stop time\", \"Total time for which application threads were stopped: \" + TIME + \" seconds, Stopping threads took: \" + TIME + \" seconds\");\n    //[1.361s][info ][safepoint   ] Safepoint \"G1CollectForAllocation\", Time since last: 295590960 ns, Reaching safepoint: 238882 ns, At safepoint: 23888872 ns, Total: 24127754 ns\n    GCParseRule UNIFIED_LOGGING_G1_SAFEPOINT = new GCParseRule(\"\", \"Safepoint \" + SAFE_POINT_CAUSE + \", Time since last: (\" + INTEGER + \") ns, Reaching safepoint: (\" + INTEGER + \") ns, At safepoint: (\" + INTEGER + \") ns, Total: (\" + INTEGER + \") ns\");\n\n    GCParseRule SIMPLE_APPLICATION_TIME = new GCParseRule(\"SIMPLE_APPLICATION_TIME\", \"Application time: \" + TIME + \" seconds\");\n    GCParseRule APPLICATION_TIME = new GCParseRule(\"APPLICATION_TIME\", DATE_TIMESTAMP + \"Application time: \" + TIME + \" seconds\");\n    GCParseRule UNIFIED_LOGGING_APPLICATION_TIME = new GCParseRule(\"Unified Logging Application Time\", \" Application time: \" + TIME + \" seconds\");\n    GCParseRule GC_PAUSE_CLAUSE = new GCParseRule(\"GC_PAUSE_CLAUSE\", \", \" + PAUSE_TIME + \"\\\\]\");\n\n    //2016-10-09T01:09:48.895+0800: 82.472: [GC TLAB: gc thread: 0x00007f2f0c011800 [id: 36811] desired_size: 1024KB slow allocs: 0  refill waste: 16384B alloc: 0.00778    26112KB refills: 7 waste  1.1% gc: 77016B slow: 952B fast: 0B\n    //2016-10-09T01:00:57.051+0800: 50.628: [GC TLAB: gc thread: 0x00007f2f0c00d800 [id: 27957] desired_size: 1024KB slow allocs: 0  refill waste: 16384B alloc: 0.01526    51200KB refills: 1 waste 97.7% gc: 1024928B slow: 0B fast: 0B\n    GCParseRule TLAB_START = new GCParseRule(\"TLAB_START\",\n            DATE_TIMESTAMP + \"\\\\[GC TLAB: gc thread: (\" + HEX + \") \\\\[id: (\" + INTEGER + \")\\\\] desired_size: (\" + INTEGER + \")KB slow allocs: (\" +\n                    INTEGER + \")  refill waste: (\" + INTEGER + \")B alloc: (\" + REAL_NUMBER + \")\\\\s+(\" + INTEGER + \")KB refills: (\" +\n                    INTEGER + \") waste\\\\s+\" + PERCENTAGE + \" gc: (\" + INTEGER + \")B slow: (\" + INTEGER + \")B fast: (\" + INTEGER + \")B\"\n    );\n    //TLAB: gc thread: 0x00007f2e64002000 [id: 27852] desired_size: 1024KB slow allocs: 0  refill waste: 16384B alloc: 0.01526    51200KB refills: 1 waste 100.0% gc: 1048064B slow: 0B fast: 0B\n    //TLAB: gc thread: 0x00007f2d60008800 [id: 32657] desired_size: 1024KB slow allocs: 0  refill waste: 16384B alloc: 0.12516   419970KB refills: 351 waste  0.5% gc: 432696B slow: 1424200B fast: 0B\n    GCParseRule TLAB_CONT = new GCParseRule(\"TLAB_CONT\",\n            \"^TLAB: gc thread: (\" + HEX + \") \\\\[id: (\" + INTEGER + \")\\\\] desired_size: (\" + INTEGER + \")KB slow allocs: (\" +\n                    INTEGER + \")  refill waste: (\" + INTEGER + \")B alloc: (\" + REAL_NUMBER + \")\\\\s+(\" + INTEGER + \")KB refills: (\" +\n                    INTEGER + \") waste \" + PERCENTAGE + \" gc: (\" + INTEGER + \")B slow: (\" + INTEGER + \")B fast: (\" + INTEGER + \")B\"\n    );\n\n    //TLAB totals: thrds: 25  refills: 3201 max: 3155 slow allocs: 821 max 799 waste:  0.7% gc: 19037968B max: 1048456B slow: 3914672B max: 3899656B fast: 0B max: 0B\n    GCParseRule TLAB_TOTALS = new GCParseRule(\"TLAB_TOTALS\",\n            \"TLAB totals: thrds: (\" + INTEGER + \")  refills: (\" + INTEGER + \") max: (\" + INTEGER + \") slow allocs: (\" + INTEGER + \") max (\" + INTEGER\n                    + \") waste:  (\" + PERCENTAGE + \") gc: (\" + INTEGER + \")B max: (\" + INTEGER + \")B slow: (\" + INTEGER + \")B max: (\" + INTEGER + \")B fast: (\"\n                    + INTEGER + \")B max: (\" + INTEGER + \")B\"\n    );\n\n    /*\n        [0.648s][info][safepoint    ] Entering safepoint region: RevokeBias\n        [0.648s][info][safepoint    ] Leaving safepoint region\n     */\n    GCParseRule SAFEPOINT_REGION = new GCParseRule(\"Safepoint region\", \"Entering safepoint region: (\\\\S+)$\");\n    GCParseRule LEAVING_SAFEPOINT = new GCParseRule(\"Leave Safepoint\", \"Leaving safepoint region\");\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/TenuredPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\npublic interface TenuredPatterns extends PreUnifiedTokens {\n\n    GCParseRule TENURING_SUMMARY = new GCParseRule(\"TENURING_SUMMARY\", \"Desired survivor size \" + COUNTER + \" bytes, new threshold \" + COUNTER + \" \\\\(max(?: threshold)? \" + COUNTER + \"\\\\)\");\n    GCParseRule TENURING_AGE_BREAKDOWN = new GCParseRule(\"TENURING_AGE_BREAKDOWN\", \"- age\\\\s+\" + COUNTER + \":\\\\s+\" + COUNTER + \" bytes,\\\\s+\" + COUNTER + \" total\");\n\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/UnifiedG1GCPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\n\nimport com.microsoft.gctoolkit.parser.GCParseRule;\n\npublic interface UnifiedG1GCPatterns extends UnifiedPatterns {\n\n    GCParseRule G1_TAG = new GCParseRule(\"G1_TAG\", \"Using G1$\");\n\n    //[90.450s][debug][gc,heap      ] GC(1459) Heap before GC invocations=1459 (full 0):\n    GCParseRule HEAP_BEFORE_AFTER_GC_INVOCATION_COUNT = new GCParseRule(\"HEAP_BEFORE_AFTER_GC_INVOCATION_COUNT\", \"Heap (before|after) GC invocations=\" + COUNTER + \" \\\\(full \" + COUNTER + \"\\\\):\");\n\n    //[90.450s][debug][gc,heap      ] GC(1459)  garbage-first heap   total 975872K, used 587987K [0x00000006c0000000, 0x00000006c0101dc8, 0x00000007c0000000)\n    GCParseRule HEAP_SUMMARY = new GCParseRule(\"HEAP_SUMMARY\", \"garbage-first heap   total \" + COUNTER + \"K, used \" + COUNTER + \"K \\\\[\" + HEX + \", \" + HEX + \", \" + HEX);\n\n    // [90.450s][debug][gc,heap      ] GC(1459)   region size 1024K, 571 young (584704K), 1 survivors (1024K)\n    GCParseRule REGION_DISBURSEMENT = new GCParseRule(\"REGION_DISBURSEMENT\", \"region size \" + COUNTER + \"K, \" + COUNTER + \" young \\\\(\" + COUNTER + \"K\\\\), \" + COUNTER + \" survivors \\\\(\" + COUNTER + \"K\\\\)\");\n\n    //[90.452s][debug][gc,heap      ] GC(1459)  Metaspace       used 16279K, capacity 17210K, committed 17408K, reserved 1064960K\\\n    //[90.452s][debug][gc,heap      ] GC(1459)   class space    used 1773K, capacity 1988K, committed 2048K, reserved 1048576K\n    GCParseRule META_CLASS_SPACE = new GCParseRule(\"META_CLASS_SPACE\", \"(Metaspace|class space)\\\\s+\" + POOL_SUMMARY);\n\n    String YOUNG_COLLECTION_SUB_TYPE = \"(Normal|Prepare Mixed|Mixed|Concurrent Start|Concurrent End)\";\n    String YOUNG_COLLECTION_TYPES = \"(Young|Mixed|Initial Mark|Full)\";\n    GCParseRule G1_COLLECTION = new GCParseRule(\"G1_COLLECTION\", \"Pause \" + YOUNG_COLLECTION_TYPES + \" (\\\\(\" + YOUNG_COLLECTION_SUB_TYPE + \"\\\\) )?\" + GC_CAUSE + \"$\");\n\n    //Pre Evacuate Collection Set: 0.0ms\n    GCParseRule PRE_EVACUATE_COLLECTION_SET = new GCParseRule(\"PRE_EVACUATE_COLLECTION_SET\", \"(Pre|Post)? Evacuate Collection Set: \" + PAUSE_TIME);\n    String PRE_EVACUATE_SUBPHASE_NAME = \"(Prepare TLABs|Choose Collection Set|Humongous Register)\";\n    GCParseRule PRE_EVACUATION_SUBPHASE = new GCParseRule(\"Pre Evacuation Phase\", PRE_EVACUATE_SUBPHASE_NAME + \": \" + PAUSE_TIME);\n\n    String EVACUATION_PHASES = \"(Ext Root Scanning|Update RS|Scan RS|Code Root Scanning|Object Copy|Termination|GC Worker Other|GC Worker Total)\";\n    GCParseRule EVACUATION_PHASE = new GCParseRule(\"EVACUATION_PHASE\", EVACUATION_PHASES + \" \\\\(ms\\\\):\\\\s+\" + WORKER_SUMMARY_REAL);\n\n    String PARALLEL_PHASES_COUNTS = \"(Processed Buffers|Termination Attempts)\";\n    GCParseRule PARALLEL_COUNT = new GCParseRule(\"PARALLEL_COUNT\", PARALLEL_PHASES_COUNTS + \":\\\\s+\" + WORKER_SUMMARY_INT);\n\n    String POST_EVACUATE_PHASES = \"(Code Roots Fixup|Preserve CM Refs|Reference Processing|Clear Card Table|Evacuation Failure|Reference Enqueuing|Merge Per-Thread State|Code Roots Purge|Redirty Cards|Clear Claimed Marks|Free Collection Set|Humongous Reclaim|Expand Heap After Collection)\";\n    GCParseRule POST_EVACUATE_PHASE = new GCParseRule(\"POST_EVACUATE_PHASE\", POST_EVACUATE_PHASES + \": \" + PAUSE_TIME);\n    GCParseRule REFERENCE_PROCESSING = new GCParseRule(\"REFERENCE_PROCESSING\", \"(Reference Processing) \" + PAUSE_TIME);\n    GCParseRule TO_SPACE_EXHAUSTED = new GCParseRule(\"TO_SPACE_EXHAUSTED\", \"To-space exhausted\");\n\n    GCParseRule OTHER = new GCParseRule(\"OTHER\", \"Other: \" + PAUSE_TIME);\n\n    GCParseRule REGION_SUMMARY = new GCParseRule(\"REGION_SUMMARY\", \"(Eden|Survivor|Old|Humongous|Archive) regions: \" + REGION_MEMORY_BLOCK);\n    GCParseRule METASPACE = new GCParseRule(\"METASPACE\", \"Metaspace: \" + BEFORE_AFTER_CONFIGURED);\n\n    //Metaspace: 10107K(10240K)->10107K(10240K) NonClass: 9227K(9280K)->9227K(9280K) Class: 880K(960K)->880K(960K)\n    GCParseRule META_SPACE_BREAKOUT = new GCParseRule(\"META_SPACE_BREAKOUT\", \"Metaspace: \" + BEFORE_CONFIGURED_AFTER_CONFIGURED + \" NonClass: \" + BEFORE_CONFIGURED_AFTER_CONFIGURED + \" Class: \" + BEFORE_CONFIGURED_AFTER_CONFIGURED);\n\n    //[00.170s][info ][gc           ] GC(1222) Pause Young (Allocation Failure) 19M->2M(61M) 5.221ms\n    //[90.452s][info ][gc           ] GC(1459) Pause Young (G1 Evacuation Pause) 574M->4M(953M) 2.065ms\n    GCParseRule YOUNG_DETAILS = new GCParseRule(\"YOUNG_DETAILS\", \"Pause \" + YOUNG_COLLECTION_TYPES + \" (\\\\(\" + YOUNG_COLLECTION_SUB_TYPE + \"\\\\) )?\" + GC_CAUSE + BEFORE_AFTER_CONFIGURED_PAUSE);\n\n    GCParseRule HEAP_REGION_SIZE = new GCParseRule(\"HEAP_REGION_SIZE\", \"Heap region size: \" + MEMORY_SIZE);\n\n    GCParseRule HEAP_SIZE = new GCParseRule(\"HEAP_SIZE\", \"Minimum heap \" + COUNTER + \"  Initial heap \" + COUNTER + \"  Maximum heap \" + COUNTER);\n\n    //[73.082s][info ][gc           ] GC(263) Concurrent Cycle\n    GCParseRule CONCURRENT_CYCLE_START = new GCParseRule(\"CONCURRENT_CYCLE_START\", \"Concurrent (?:Mark )?Cycle$\");\n    //[73.171s][info ][gc            ] GC(263) Concurrent Cycle 89.437ms\n    GCParseRule CONCURRENT_CYCLE_END = new GCParseRule(\"CONCURRENT_CYCLE_END\", \"Concurrent (?:Mark )?Cycle \" + CONCURRENT_TIME);\n\n    //[73.082s][info ][gc           ] GC(263) Concurrent Cycle\n    GCParseRule CONCURRENT_UNDO_CYCLE_START = new GCParseRule(\"CONCURRENT_CYCLE_START\", \"Concurrent Undo Cycle$\");\n    //[73.171s][info ][gc            ] GC(263) Concurrent Cycle 89.437ms\n    GCParseRule CONCURRENT_UNDO_CYCLE_END = new GCParseRule(\"CONCURRENT_CYCLE_END\", \"Concurrent Undo Cycle \" + CONCURRENT_TIME);\n\n    //    [73.082s][info ][gc,marking   ] GC(263) Concurrent Clear Claimed Marks\n    //    [73.082s][info ][gc,marking   ] GC(263) Concurrent Clear Claimed Marks 0.018ms\n    //\n    //    [155.787s][info ][gc          ] GC(2457) Concurrent Undo Cycle\n    //    [155.836s][info ][gc          ] GC(2457) Concurrent Undo Cycle 49.351ms\n    //\n    //    [73.082s][info ][gc,marking   ] GC(263) Concurrent Scan Root Regions\n    //    [73.084s][info ][gc,marking   ] GC(263) Concurrent Scan Root Regions 2.325ms\n    String CONCURRENT_PHASES = \"(Mark|Clear Claimed Marks|Scan Root Regions|Rebuild Remembered Sets|Create Live Data|Complete Cleanup|Cleanup for Next Mark)\";\n    GCParseRule CONCURRENT_PHASE = new GCParseRule(\"CONCURRENT_PHASE\", \"Concurrent \" + CONCURRENT_PHASES + \"$\");\n    GCParseRule CONCURRENT_PHASE_DURATION = new GCParseRule(\"CONCURRENT_PHASE_DURATION\", \"Concurrent \" + CONCURRENT_PHASES + \" \" + CONCURRENT_TIME);\n\n    //    [73.084s][info ][gc,marking   ] GC(263) Concurrent Mark (73.084s)\n    //    [73.084s][info ][gc,marking   ] GC(263) Concurrent Mark From Roots\n    //    [73.084s][info ][gc,task      ] GC(263) Using 2 workers of 2 for marking\n    //    [73.138s][info ][gc,marking   ] GC(263) Concurrent Mark From Roots 53.902ms\n    //    [73.138s][info ][gc,marking   ] GC(263) Concurrent Mark (73.084s, 73.138s) 53.954ms\n    String CONCURRENT_MARK_PHASES = \"(Mark From Roots|Preclean)\";\n    GCParseRule CONCURRENT_MARK_START = new GCParseRule(\"CONCURRENT_MARK_START\", \"Concurrent (Mark) \\\\(.+\\\\)$\");\n    GCParseRule CONCURRENT_MARK_PHASE = new GCParseRule(\"CONCURRENT MARK PHASE\", \"Concurrent \" + CONCURRENT_MARK_PHASES + \"$\");\n    GCParseRule CONCURRENT_MARK_PHASE_DURATION = new GCParseRule(\"CONCURRENT MARK PHASE\", \"Concurrent \" + CONCURRENT_MARK_PHASES + \" \" + CONCURRENT_TIME);\n    GCParseRule CONCURRENT_MARK_WORKERS = new GCParseRule(\"CONCURRENT_MARK_WORKERS\", \"Using \" + COUNTER + \" workers of \" + COUNTER + \" for marking\");\n    GCParseRule CONCURRENT_MARK_ABORTED = new GCParseRule(\"Concurrent Mark Abort\", \"Concurrent Mark Abort\");\n    GCParseRule CONCURRENT_MARK_END = new GCParseRule(\"CONCURRENT_MARK_END\", \"Concurrent (Mark) \\\\(.+\\\\) \" + CONCURRENT_TIME);\n\n    //    [73.139s][info ][gc,start     ] GC(263) Pause Remark\n    //    [73.139s][debug][gc,phases    ] GC(263) Finalize Marking 0.251ms\n    //    [73.139s][debug][gc,ref       ] GC(263) SoftReference 0.026ms\n    //    [73.158s][debug][gc,ref       ] GC(263) WeakReference 18.890ms\n    //    [73.158s][debug][gc,ref       ] GC(263) FinalReference 0.025ms\n    //    [73.158s][debug][gc,ref       ] GC(263) PhantomReference 0.160ms\n    //    [73.158s][debug][gc,ref       ] GC(263) JNI Weak Reference 0.240ms\n...\n", "needle_token_start": 8805, "needle_token_end": 8927, "code_context_ntokens": 16154, "output": ["```java\npublic MetaspaceRecord getMetaSpaceRecord(int offset) {\n    try {\n        long before = toKBytes(offset);\n        long after = toKBytes(offset + 2);\n        long size = toKBytes(offset + 4);\n        return new MetaspaceRecord(before, after, size);\n    } catch (NumberFormatException numberFormatException) {\n        LOGGER.fine(\"Unable to calculate Metaspace summary.\");\n        notYetImplemented();\n    }\n    return null;\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "supported", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/JVMFlags.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To determine if a specific logging flag is supported based on predefined settings.\n2. **Input**: A string representing the name of a logging flag.\n3. **Output**: A boolean value indicating whether the flag is supported (true) or not (false).\n4. **Procedure**: The function checks a predefined map that associates logging flag names with boolean values indicating their support status. It returns the boolean value corresponding to the provided flag name.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/PreUnifiedDiarizer.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\n\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GCCauses;\nimport com.microsoft.gctoolkit.jvm.Diarizer;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.jvm.SupportedFlags;\nimport com.microsoft.gctoolkit.parser.CMSPatterns;\nimport com.microsoft.gctoolkit.parser.G1GCPatterns;\nimport com.microsoft.gctoolkit.parser.G1GCTokens;\nimport com.microsoft.gctoolkit.parser.GCLogTrace;\nimport com.microsoft.gctoolkit.parser.GCParseRule;\nimport com.microsoft.gctoolkit.parser.JVMPatterns;\nimport com.microsoft.gctoolkit.parser.ParallelPatterns;\nimport com.microsoft.gctoolkit.parser.PreUnifiedTokens;\nimport com.microsoft.gctoolkit.parser.SerialPatterns;\nimport com.microsoft.gctoolkit.parser.SimplePatterns;\nimport com.microsoft.gctoolkit.parser.TenuredPatterns;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.logging.Logger;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport static com.microsoft.gctoolkit.parser.CMSPatterns.FLS_HEADER;\nimport static com.microsoft.gctoolkit.parser.SharedPatterns.MEMORY_SUMMARY_RULE;\nimport static com.microsoft.gctoolkit.parser.SharedPatterns.META_SPACE_RECORD;\n\n/**\n * Class to answer 3 questions about the GC log;\n * - which collector combination is being used\n * - which JVM version\n * - which flags are being used\n * <p>\n * Collectors\n * - Unknown (no details)\n * - Serial\n * - ParNew/CMS\n * - iCMS\n * - PSYoungGen/PSFull\n * - G1GC\n * <p>\n * Version breaks\n * Can be known precisely if header is present in GC log.\n * prior to 1.7.0_45\n * 1.7.0_45 has the System.gc() regression\n * 1.8.0_05 anything after this version will include G1 Metaspace clause\n * 1.9.0 unknown how to determine this via deriveConfiguration.\n * <p>\n * Are these flags set\n * PrintGCDetails\n * PrintTenuringDistrubtion\n * PrintGCApplicationStoppedTime\n * PrintGCApplicationConcurrentTime\n * <p>\n * Log file characteristics\n * <p>\n * pre 7.0_45\n * - Permspace\n * - (System)\n * <p>\n * 7.0_45+\n * - (System.gc())\n * - Permspace\n * <p>\n * 8.0\n * - Metaspace\n * <p>\n * 8.0_20\n * - G1 prints Metaspace clause\n * <p>\n * Collector clues\n * iCMS\n * - CMS cycle starts early but once completed all further records will contain \"icms_dc=\"\n * <p>\n * ParNew/CMS\n * - ParNew\n * - initial-mark\n * <p>\n * PSYoung/PSFull\n * - PSYoungGen\n * - PSFull\n * <p>\n * Serial\n * - DefNew\n */\n\n//SimplePatterns, CMSPatterns, ParallelPatterns, G1GCPatterns,\npublic class PreUnifiedDiarizer implements Diarizer {\n\n    private static final Logger LOGGER = Logger.getLogger(PreUnifiedDiarizer.class.getName());\n\n    private int lineCount = MAXIMUM_LINES_TO_EXAMINE;\n\n    private static final GCParseRule TENURED_BLOCK = new GCParseRule(\"TENURED_BLOCK\", \"\\\\[Tenured: \\\\d+K->\\\\d+K\\\\(\\\\d+K\\\\), \\\\d+[.|,]\\\\d{7} secs\\\\]\");\n    private static final GCParseRule PREFIX = new GCParseRule(\"PREFIX\", PreUnifiedTokens.GC_PREFIX);\n    private static final GCParseRule FULL_PREFIX = new GCParseRule(\"FULL_PREFIX\", PreUnifiedTokens.FULL_GC_PREFIX);\n    private static final GCParseRule G1GC_PREFIX = new GCParseRule(\"G1GC_PREFIX\", G1GCTokens.G1GC_PREFIX);\n    private static final GCParseRule REFERENCE_PROCESSING_BLOCK = new GCParseRule(\"REFERENCE_PROCESSING_BLOCK\", PreUnifiedTokens.REFERENCE_RECORDS);\n    private static final Pattern excludeG1Ergonomics = Pattern.compile(\"^\\\\d+(\\\\.|,)\\\\d+: \\\\[G1Ergonomics\");\n\n\n    private final Diary diary;\n\n    public PreUnifiedDiarizer() {}\n\n    @Override\n    public Diary getDiary() {\n        return diary;\n    }\n\n    @Override\n    public boolean isUnified() {\n        return false;\n    }\n\n    private boolean firstCMSCycle = false;\n    private int youngCountAfterFirstCMSCycle = 0;\n    private int collectionCount = 0;\n    private int youngCollectionCount = 0;\n    private int tenuringSummary = 0;\n    private boolean ageTableDetected = false;\n    private int maxTenuringThreshold = 15;\n    private int setGCFlags = 0;\n    private DateTimeStamp timeOfFirstEvent = null;\n\n    {\n        diary = new Diary();\n        diary.setFalse(SupportedFlags.UNIFIED_LOGGING, SupportedFlags.ZGC, SupportedFlags.SHENANDOAH);\n    }\n\n    @Override\n    public String getCommandLine() {\n        // TODO #149 Extract command line from PreUnifiedJVMConfiguration log file\n        return \"\";\n    }\n\n    @Override\n    public int getMaxTenuringThreshold() {\n        return maxTenuringThreshold;\n    }\n\n    @Override\n    public boolean hasJVMEvents() {\n        return getDiary().isApplicationStoppedTime() ||\n                getDiary().isApplicationRunningTime() ||\n                getDiary().isApplicationRunningTime() ||\n                getDiary().isTLABData();\n    }\n\n    private boolean versionIsKnown() {\n        return (getDiary().isStateKnown(SupportedFlags.JDK80) && getDiary().isStateKnown(SupportedFlags.JDK70)) && getDiary().isStateKnown(SupportedFlags.PRE_JDK70_40);\n    }\n\n    @Override\n    public boolean completed() {\n        return getDiary().isComplete() || lineCount < 1 || (simpleCMSCycleDetected && (simpleCMSCycleDetected || simpleFullGCDetected));\n    }\n\n    // Things that if we've not seen them, we know that they are false.\n    public void fillInKnowns() {\n        if (simpleCMSCycleDetected) {\n            getDiary().setTrue(SupportedFlags.CMS);\n            getDiary().setFalse(SupportedFlags.SERIAL, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80);\n            // if we have age table or if no tenuring distribution, assume default ParNew/CMS\n            if (youngCollectionCount == 0 || ageTableDetected || !getDiary().isTenuringDistribution()) {\n                getDiary().setTrue(SupportedFlags.PARNEW);\n                getDiary().setFalse(SupportedFlags.DEFNEW);\n            } else { //CMS with no age table implies DEFNEW/CMS\n                getDiary().setTrue(SupportedFlags.DEFNEW);\n                getDiary().setFalse(SupportedFlags.PARNEW);\n            }\n        } else if (simpleFullGCDetected) {\n            if (ageTableDetected) {\n                getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.SERIAL);\n            } else { //at this point we can't tell if it's serial or parallel so assume defaults\n                getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n            }\n            getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.SERIAL, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80);\n        }\n\n        //Not much information here, assume defaults\n        else if (simpleParallelOrParNewDetected) {\n            if (ageTableDetected) {\n                getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS);\n                getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n            } else {\n                getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n                getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS);\n            }\n            getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.SERIAL, SupportedFlags.ICMS, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40, SupportedFlags.JDK80);\n        }\n\n        getDiary().setFalse(SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION, SupportedFlags.PRINT_PROMOTION_FAILURE, SupportedFlags.PRINT_FLS_STATISTICS);\n    }\n\n    /**\n     * Attempt to diarize the GC Log line that's come.  That is, if we collect enough details about\n     * this log in oder to categorize it by collector and collector features then we return true\n     */\n    @Override\n    public boolean diarize(String line) {\n\n        timeOfFirstEvent(line);\n        lineCount--;\n\n        if (line.startsWith(\"Java HotSpot(TM)\") || line.startsWith(\"OpenJDK\")) {\n            parseJVMVersion(line);\n        } else if (line.startsWith(\"CommandLine \")) {\n            evaluateCommandLineFlags(line.split(\" \"));\n            /*\n             * We're almost done, all we need is a time stamp which should be on the next line.\n             * but give it some time to find it in case the log has other stuff in it\n             */\n            lineCount = 25;\n            collectionCount = 3;\n        } else if (!jvmActivityFlag(line)) {\n            collector(line);\n            version(line);\n            details(line);\n        }\n\n        return this.completed();\n    }\n\n    @Override\n    public DateTimeStamp getTimeOfFirstEvent() {\n        if (timeOfFirstEvent == null)\n            timeOfFirstEvent = new DateTimeStamp(0.0d);\n        return timeOfFirstEvent;\n    }\n\n    private void timeOfFirstEvent(String line) {\n        if (timeOfFirstEvent == null) {\n            Matcher matcher = excludeG1Ergonomics.matcher(line);\n            if (matcher.find()) {\n                return;      // G1Ergonomics doesn't respect PrintDateStamp which confuses downstream calculations\n            }\n            GCLogTrace trace = PreUnifiedTokens.DATE_TIMESTAMP_RECORD.parse(line);\n            if (trace != null) {\n                timeOfFirstEvent = trace.getDateTimeStamp();\n                diary.setTimeOfFirstEvent(timeOfFirstEvent);\n            }\n        }\n    }\n\n    private boolean jvmActivityFlag(String line) {\n\n        if (getDiary().isStateKnown(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.TLAB_DATA))\n            return false;\n\n        if ((JVMPatterns.APPLICATION_STOP_TIME.parse(line) != null) || (JVMPatterns.SIMPLE_APPLICATION_STOP_TIME.parse(line) != null) || JVMPatterns.APPLICATION_STOP_TIME_WITH_STOPPING_TIME.parse(line) != null) {\n            getDiary().setTrue(SupportedFlags.APPLICATION_STOPPED_TIME);\n            return true;\n        } else if ((JVMPatterns.APPLICATION_TIME.parse(line) != null) || (JVMPatterns.SIMPLE_APPLICATION_TIME.parse(line) != null)) {\n            getDiary().setTrue(SupportedFlags.APPLICATION_CONCURRENT_TIME);\n            return true;\n        } else if (JVMPatterns.TLAB_CONT.parse(line) != null) {\n            getDiary().setTrue(SupportedFlags.TLAB_DATA);\n            return true;\n        }\n        //This will be reported along size a collection so if we don't see them by 3nd collection....\n        // maybe some rubbish lines between collection and log so wait a bit longer than just the reporting of the first collection\n        if (collectionCount > 1) {\n            getDiary().setFalse(SupportedFlags.APPLICATION_STOPPED_TIME, SupportedFlags.APPLICATION_CONCURRENT_TIME, SupportedFlags.TLAB_DATA);\n        }\n\n        return false;\n    }\n\n    private boolean simpleParallelOrParNewDetected = false;\n    private boolean simpleFullGCDetected = false;\n    private boolean simpleCMSCycleDetected = false;\n\n    //TODO: #148 PSYoung -> PSOldGen, DefNew->PSOldGen, so maybe we need to break up Parallel for generational collections\n    private void collector(String line) {\n\n        GCLogTrace trace;\n\n        if ((!getDiary().isStateKnown(SupportedFlags.PARNEW, SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.G1GC)) ||\n                !getDiary().isStateKnown(SupportedFlags.GC_DETAILS) || collectionCount < 3) {\n\n            if (line.contains(\"[PSYoungGen:\")) {\n                getDiary().setTrue(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION, SupportedFlags.RSET_STATS);\n                collectionCount++;\n                youngCollectionCount++;\n\n                if ((trace = ParallelPatterns.PSYOUNGGEN.parse(line)) != null) {\n                    getDiary().setTrue(SupportedFlags.GC_DETAILS);\n                    getDiary().setFalse(SupportedFlags.TENURING_DISTRIBUTION);\n                    getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC);\n                    setGCCause(trace.getGroup(3));\n\n                } else if (ParallelPatterns.PS_DETAILS_WITH_TENURING.parse(line) != null) {\n                    getDiary().setTrue(SupportedFlags.GC_DETAILS);\n                    getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC);\n                }\n            } else if (line.contains(\"ParNew\")) {\n                getDiary().setTrue(SupportedFlags.PARNEW, SupportedFlags.CMS);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n                collectionCount++;\n                youngCollectionCount++;\n\n                if ((trace = CMSPatterns.PARNEW.parse(line)) != null) {\n                    getDiary().setTrue(SupportedFlags.GC_DETAILS);\n                    getDiary().setFalse(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                    setGCCause(trace.getGroup(4));\n                } else if ((trace = CMSPatterns.PARNEW_TENURING.parse(line)) != null) {\n                    getDiary().setTrue(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION);\n                    getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n                    setGCCause(trace.getGroup(3));\n                } else if ((trace = SimplePatterns.PARNEW_NO_DETAILS.parse(line)) != null) {\n                    getDiary().setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                    setGCCause(trace.getGroup(3));\n                } else if ((trace = SimplePatterns.PARNEW_START.parse(line)) != null) {\n                    getDiary().setFalse(SupportedFlags.GC_DETAILS, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC);\n                    setGCCause(trace.groupCount() > 2 ? trace.getGroup(3) : null);\n                } else if ((trace = CMSPatterns.PARNEW_REFERENCE_SPLIT.parse(line)) != null) {\n                    getDiary().setTrue(SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.PRINT_REFERENCE_GC);\n                    setGCCause(trace.getGroup(3));\n                }\n            } else if ((trace = SerialPatterns.DEFNEW.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS);\n                getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.PARNEW, SupportedFlags.G1GC, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.RSET_STATS, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                setGCCause(trace.getGroup(3));\n            } else if ((trace = SerialPatterns.DEFNEW_TENURING.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setTrue(SupportedFlags.DEFNEW, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION);\n                getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n                setGCCause(trace.getGroup(3));\n            } else if (SimplePatterns.PARNEW_NO_DETAILS.parse(line) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setTrue(SupportedFlags.PARNEW);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.GC_CAUSE, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.ADAPTIVE_SIZING, SupportedFlags.PRINT_HEAP_AT_GC, SupportedFlags.RSET_STATS, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n            } else if (SimplePatterns.YOUNG_NO_DETAILS.parse(line) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                simpleParallelOrParNewDetected = true;\n            } else if (SimplePatterns.FULL_NO_GC_DETAILS.parse(line) != null) {\n                collectionCount++;\n                simpleFullGCDetected = true;\n                simpleParallelOrParNewDetected = true;\n            } else if ((trace = SimplePatterns.GC_START.parse(line)) != null) {\n                collectionCount++;\n                simpleParallelOrParNewDetected = true;\n                youngCollectionCount++;\n                getDiary().setFalse(SupportedFlags.G1GC, SupportedFlags.RSET_STATS);\n                setGCCause(trace.getGroup(3));\n            } else if (SimplePatterns.CMS_NO_DETAILS.parse(line) != null) {\n                // could be parallel or CMS.. look for Full GC but even that may be a trick\n                collectionCount++;\n                simpleParallelOrParNewDetected = true;\n                simpleCMSCycleDetected = true;\n                firstCMSCycle = true;\n            } else if ((trace = G1GCPatterns.G1_YOUNG_SPLIT_START.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                getDiary().setTrue(SupportedFlags.G1GC);\n                getDiary().setTrue(SupportedFlags.GC_DETAILS);\n                if (trace.gcCause() == GCCause.GCCAUSE_NOT_SET) {\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE);\n                } else if (trace.gcCause() == GCCause.METADATA_GENERATION_THRESHOLD) {\n                    getDiary().setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n                    getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n                } else if ((trace.gcCause() == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) {\n                    getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                }\n            } else if ((trace = G1GCPatterns.G1_DETAILS.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                if (trace.gcCause(3, 0) == GCCause.GCCAUSE_NOT_SET) {\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                } else {\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE);\n                    getDiary().setFalse(SupportedFlags.PRE_JDK70_40);\n                }\n            } else if ((trace = G1GCPatterns.YOUNG.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                getDiary().setTrue(SupportedFlags.G1GC);\n                checkForGCCause(trace);\n\n                //2014-10-21T11:49:08.954-0500: 12053.551: [GC pause (young)12054.116: [SoftReference, 0 refs, 0.0000070 secs]12054.116: [WeakReference, 234 refs, 0.0000640 secs]12054.116: [FinalReference, 3805 refs, 0.0034010 secs]12054.119: [PhantomReference, 9 refs, 0.0000040 secs]12054.119: [JNI Weak Reference, 0.0001960 secs], 0.58191800 secs]\n            } else if ((trace = G1GCPatterns.G1_DETAILS_REFERENCE_GC.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.TENURING_DISTRIBUTION, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.PRINT_REFERENCE_GC);\n                if (trace.getGroup(3) != null)\n                    getDiary().setTrue(SupportedFlags.GC_CAUSE);\n                else\n                    getDiary().setFalse(SupportedFlags.GC_CAUSE);\n            } else if (G1GCPatterns.G1_INITIAL_MARK.parse(line) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.MAX_TENURING_THRESHOLD_VIOLATION);\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.TENURING_DISTRIBUTION);\n            }\n        }\n\n        if (getDiary().isTrue(SupportedFlags.ADAPTIVE_SIZING)) {\n            if ((trace = G1GCPatterns.YOUNG_SPLIT_BY_G1ERGONOMICS.parse(line)) != null) {\n                collectionCount++;\n                youngCollectionCount++;\n                getDiary().setTrue(SupportedFlags.G1GC, SupportedFlags.GC_DETAILS);\n                getDiary().setFalse(SupportedFlags.DEFNEW, SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.CMS_DEBUG_LEVEL_1);\n                checkForGCCause(trace);\n            }\n        }\n\n        //This has been verified true as of 1.7.0_51. Check later versions to make sure it hasn't been back ported\n        if (ParallelPatterns.PS_FULL_GC_PERM.parse(line) != null) {\n            collectionCount++;\n            getDiary().setTrue(SupportedFlags.JDK70);\n            getDiary().setFalse(SupportedFlags.JDK80);\n            if (line.contains(\" [PSYoungGen: \"))\n                getDiary().setTrue(SupportedFlags.PARALLELGC);\n            getDiary().setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS);\n            getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.SERIAL, SupportedFlags.G1GC);\n\n            getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n            getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC);\n\n        }\n\n        //This has been verified true as of 1.7.0_51. Check later versions to make sure it hasn't been back ported\n        else if (ParallelPatterns.PS_FULL_GC_META.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n            getDiary().setTrue(SupportedFlags.JDK80);\n            if (line.contains(\" [PSYoungGen: \"))\n                getDiary().setTrue(SupportedFlags.PARALLELGC);\n            getDiary().setTrue(SupportedFlags.PARALLELOLDGC, SupportedFlags.GC_DETAILS);\n            getDiary().setFalse(SupportedFlags.PARNEW, SupportedFlags.CMS, SupportedFlags.SERIAL, SupportedFlags.ICMS, SupportedFlags.G1GC);\n\n            getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n            getDiary().setFalse(SupportedFlags.PRINT_HEAP_AT_GC);\n\n        } else if (line.contains(\"CMS-initial-mark\")) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.PARALLELGC, SupportedFlags.PARALLELOLDGC, SupportedFlags.SERIAL, SupportedFlags.G1GC);\n            getDiary().setTrue(SupportedFlags.CMS, SupportedFlags.GC_DETAILS);\n        }\n\n        //todo: this rule is in the wrong place\n        else if (CMSPatterns.SERIAL_FULL.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.G1GC);\n            getDiary().setTrue(SupportedFlags.SERIAL);\n\n            getDiary().setTrue(SupportedFlags.GC_DETAILS);\n            //todo: private final int SupportedFlags.GC_CAUSE = 12;\n            getDiary().setFalse(SupportedFlags.CMS_DEBUG_LEVEL_1);\n            if (line.contains(\"Metaspace\")) {\n                getDiary().setFalse(SupportedFlags.JDK70);\n            }\n            if (line.contains(\"Tenured\")) {\n                getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS);\n                getDiary().setTrue(SupportedFlags.SERIAL);\n            } else if (line.contains(\"CMS\")) {\n                getDiary().setFalse(SupportedFlags.SERIAL);\n                getDiary().setTrue(SupportedFlags.CMS);\n            }\n        } else if (CMSPatterns.SERIAL_FULL80.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.CMS, SupportedFlags.ICMS, SupportedFlags.PARALLELOLDGC, SupportedFlags.G1GC, SupportedFlags.CMS_DEBUG_LEVEL_1, SupportedFlags.JDK70);\n            getDiary().setTrue(SupportedFlags.SERIAL, SupportedFlags.GC_DETAILS, SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n        } else if (TENURED_BLOCK.parse(line) != null) {\n            collectionCount++;\n            getDiary().setFalse(SupportedFlags.CMS);\n            getDiary().setFalse(SupportedFlags.ICMS);\n            getDiary().setTrue(SupportedFlags.SERIAL);\n            getDiary().setFalse(SupportedFlags.G1GC);\n        }\n\n        if (getDiary().isTrue(SupportedFlags.CMS) && !getDiary().isStateKnown(SupportedFlags.ICMS)) {\n            if (firstCMSCycle) {\n                if (line.contains(\"ParNew\") || line.contains(\"DefNew\"))\n                    youngCountAfterFirstCMSCycle++;\n\n                if (line.contains(\"icms_dc\"))\n                    getDiary().setTrue(SupportedFlags.ICMS);\n                else if (youngCountAfterFirstCMSCycle > 1)\n                    getDiary().setFalse(SupportedFlags.ICMS);\n            }\n            //The first CMS cycle is needed to kick off iCMS\n            if (line.contains(\"concurrent-reset\"))\n                firstCMSCycle = true;\n        } else if (getDiary().isTrue(SupportedFlags.G1GC)) {\n            if (G1GCPatterns.G1_MEMORY_SUMMARY.parse(line) != null) {\n                if (line.contains(\"Metaspace\")) {\n                    getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n                    getDiary().setTrue(SupportedFlags.JDK80);\n                } else {\n                    getDiary().setTrue(SupportedFlags.JDK70);\n                    getDiary().setFalse(SupportedFlags.JDK80);\n                }\n            }\n        }\n    }\n\n    private void checkForGCCause(GCLogTrace trace) {\n        if (trace.gcCause(3, 0) == GCCause.METADATA_GENERATION_THRESHOLD) {\n            getDiary().setTrue(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n            getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n        } else if ((trace.gcCause(3, 0) == GCCause.G1_EVACUATION_PAUSE) || (trace.gcCause(3, 0) == GCCause.G1_HUMONGOUS_ALLOCATION)) {\n            getDiary().setTrue(SupportedFlags.GC_CAUSE);\n        }\n    }\n\n    /*\n     * Mostly for getting version information but we can also capture other information in the deriveConfiguration.\n     *\n     * if the collector is G1 I don't currently have a good way of saying 7.0 or 8.0 as perm\n     * and meta space records are not printed with a Full GC. Metaspace will be emitted to the\n     * log in the version after 8.0_05.\n     */\n    private void version(String line) {\n\n        if (versionIsKnown()) return;\n\n        GCLogTrace trace;\n        //Can get a lot of information from the perm space record\n        // (CMS Perm |PS Perm |Perm |PSPermGen|Metaspace)\n        if ((trace = ParallelPatterns.PERM_SPACE_RECORD.parse(line)) != null) {\n            String value = trace.getGroup(1).trim();\n\n            if (\"CMS Perm\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"PS Perm\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"Perm\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"PSPermGen\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK70);\n                getDiary().setFalse(SupportedFlags.JDK80);\n            } else if (\"Metaspace\".equals(value)) {\n                getDiary().setTrue(SupportedFlags.JDK80);\n                getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n            }\n        } else if (META_SPACE_RECORD.parse(line) != null) {\n            getDiary().setTrue(SupportedFlags.JDK80);\n            getDiary().setFalse(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n        }\n\n        // maybe we'll get lucky, app server often call System.gc() after startup.\n        if (line.contains(\"(System)\")) {\n            getDiary().setTrue(SupportedFlags.JDK70, SupportedFlags.PRE_JDK70_40);\n            getDiary().setFalse(SupportedFlags.GC_CAUSE, SupportedFlags.JDK80);\n        } else if (line.contains(\"(System.gc()\")) {\n...\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/JVMFlags.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\nimport java.util.Map;\n\npublic enum JVMFlags {\n\n    PrintGCApplicationConcurrentTime(\"PrintGCApplicationConcurrentTime\"),\n    PrintGCApplicationStoppedTime(\"PrintGCApplicationStoppedTime\"),\n    Unknown(\"unknown\");\n\n    private final String label;\n\n    JVMFlags(String label) {\n        this.label = label;\n    }\n\n    public String getLabel() {\n        return label;\n    }\n    /*\n        The following flags I found interesting but I would not recommend them for a normal production environment. However, this would, I think could generate great interest\n        PrintClassHistogramBeforeFullGC\n        PrintClassHistogramAfterFullGC\n\n    */\n\n    private final Map<String, Boolean> gcLoggingFlagSupport = Map.ofEntries(\n            Map.entry(\"PrintGCApplicationConcurrentTime\", true),\n            Map.entry(\"PrintGCApplicationStoppedTime\", true),\n            Map.entry(\"PrintGCDetails\", true),\n            Map.entry(\"PrintGCDateStamps\", true),\n            Map.entry(\"PrintTenuringDistribution\", true),\n            Map.entry(\"PrintReferenceGC\", true),\n            Map.entry(\"PrintGCCause\", true),\n            Map.entry(\"PrintFLSStatistics\", true),\n            Map.entry(\"PrintFLSCensus\", true),\n            Map.entry(\"PrintPromotionFailure\", true),\n            Map.entry(\"PrintAdaptiveSizePolicy\", true),\n            Map.entry(\"PrintCMSInitiationStatistics\", true),\n            Map.entry(\"PrintTLAB\", false),\n            Map.entry(\"TLABStats\", false),\n            Map.entry(\"PrintPLAB\", false),\n            Map.entry(\"PrintOldPLAB\", false)\n    );\n\n    \npublic boolean supported(String flag) {\n        return gcLoggingFlagSupport.get(flag);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/GarbageCollectorAlgorithm.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\npublic enum GarbageCollectorAlgorithm {\n    /** -XX:+UseConcMarkSweepGC */\n    DEFNEW,                                     //  1\n    /** -XX:+UseConcMarkSweepGC -XX:+UseParNew */\n    PARNEW,                                     //  2\n    /** -XX:+UseConcMarkSweepGC */\n    CMS,                                        //  3\n    /** -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode */\n    ICMS,                                       //  4\n    /** -XX:+UseParallelGC */\n    PARALLELGC,                                 //  5\n    /** -XX:+UseParallelGC -XX:+UseParallelOldGC <br>-XX:+UseParallelOldGC */\n    PARALLELOLDGC,                              //  6\n    /** -XX:+UseSerialGC */\n    SERIAL,                                     //  7\n    /** -XX:+UseSerialGC */\n    G1GC,                                       //  8\n    /** -XX:+UseZGC */\n    ZGC,                                        //  9\n    /** -XX:+UseShenandoahGC */\n    SHENANDOAH                                  // 10\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/jvm/UnifiedDiarizer.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.jvm;\n\nimport com.microsoft.gctoolkit.jvm.Diarizer;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.parser.unified.UnifiedLoggingLevel;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.TreeSet;\n\nimport static com.microsoft.gctoolkit.jvm.SupportedFlags.GC_CAUSE;\nimport static com.microsoft.gctoolkit.jvm.SupportedFlags.*;\nimport static com.microsoft.gctoolkit.parser.unified.ShenandoahPatterns.SHENANDOAH_TAG;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedG1GCPatterns.G1_COLLECTION;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedG1GCPatterns.G1_TAG;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedGenerationalPatterns.*;\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedPatterns.CPU_BREAKOUT;\nimport static com.microsoft.gctoolkit.parser.unified.ZGCPatterns.CYCLE_START;\nimport static com.microsoft.gctoolkit.parser.unified.ZGCPatterns.ZGC_TAG;\n\n//ShenandoahPatterns, ZGCPatterns, UnifiedG1GCPatterns, UnifiedGenerationalPatterns,\npublic class UnifiedDiarizer implements Diarizer {\n\n    private static final int CYCLES_TO_EXAMINE_BEFORE_GIVING_UP = 10;\n    private static final int CYCLES_TO_EXAMINE_FOR_SAFEPOINT = 2;\n\n    private int lineCount = MAXIMUM_LINES_TO_EXAMINE;\n\n    private final Diary diary;\n    private final TreeSet<String> tagsAndLevels = new TreeSet<>();\n    private int stopTheWorldEvents = 0;\n\n    {\n        diary = new Diary();\n        diary.setTrue(UNIFIED_LOGGING);\n        diary.setFalse(ICMS, PRE_JDK70_40, JDK70, PRE_JDK70_40, JDK80, MAX_TENURING_THRESHOLD_VIOLATION);\n\n    }\n\n    public UnifiedDiarizer() {}\n\n    public Diary getDiary() {\n        return diary;\n    }\n\n    @Override\n    public boolean isUnified() {\n        return true;\n    }\n\n    @Override\n    public String getCommandLine() {\n        // TODO #147 extract command line from the log file\n        return \"\";\n    }\n\n    @Override\n    public int getMaxTenuringThreshold() {\n        return 0;\n    }\n\n    @Override\n    public boolean hasJVMEvents() {\n        return getDiary().isApplicationStoppedTime() ||\n                getDiary().isApplicationRunningTime() ||\n                getDiary().isTLABData();\n    }\n\n    @Override\n    public void fillInKnowns() {\n        getDiary().setFalse(ADAPTIVE_SIZING);\n    }\n\n    @Override\n    public boolean completed() {\n        return getDiary().isComplete() || lineCount < 1;\n    }\n\n    @Override\n    public boolean diarize(String line) {\n        if ( ! this.completed()) {\n            if (!line.startsWith(\"[\"))\n                return false;\n            lineCount--;\n            extractDecorators(line);\n            if (!getDiary().isCollectorKnown())\n                discoverCollector(line);\n            if (!getDiary().isDetailsKnown())\n                discoverDetails(line);\n            if (!getDiary().isJVMEventsKnown())\n                discoverJVMEvents(line);\n            if ((CPU_BREAKOUT.parse(line) != null) || line.contains(\"gc,start\"))\n                stopTheWorldEvents++;\n        }\n        return this.completed();\n    }\n\n    /**\n     * Extract decorators (from a GC log line tag) and set the corresponding diary flags accordingly\n     *\n     * @param line GC log line\n     */\n    private void extractDecorators(String line) {\n        Decorators decorators = new Decorators(line);\n        timeOfFirstEvent(decorators);\n        extractTagsAndLevels(decorators);\n        // -Xlog:gc*,gc+ref=debug,gc+phases=debug,gc+age=trace,safepoint\n        if (decorators.getLogLevel().isPresent()) {\n            UnifiedLoggingLevel logLevel = decorators.getLogLevel().get();\n            if (decorators.tagsContain(\"gc,age\"))\n                getDiary().setTrue(TENURING_DISTRIBUTION);\n            else if (decorators.tagsContain(\"ref\") && logLevel.isGreaterThanOrEqualTo(UnifiedLoggingLevel.debug))\n                getDiary().setTrue(PRINT_REFERENCE_GC);\n            else if (decorators.tagsContain(\"gc,phases\") && logLevel.isGreaterThanOrEqualTo(UnifiedLoggingLevel.debug))\n                getDiary().setTrue(GC_DETAILS);\n            else if ( decorators.tagsContain(\"gc,ergo\"))\n                getDiary().setTrue(ADAPTIVE_SIZING);\n            if (decorators.tagsContain(\"safepoint\"))\n                getDiary().setTrue(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME);\n\n            if (getDiary().isZGC()) {\n                if (decorators.tagsContain(\"task\"))\n                    getDiary().setTrue(GC_DETAILS);\n                else if (decorators.tagsContain(\"heap\"))\n                    getDiary().setTrue(PRINT_HEAP_AT_GC);\n                else if (decorators.tagsContain(\"tlab\"))\n                    getDiary().setTrue(TLAB_DATA);\n                else if (decorators.tagsContain(\"gc,start\") && line.contains(\"Garbage Collection (\"))\n                    getDiary().setTrue(GC_CAUSE);\n                else if (decorators.tagsContain(\"gc,heap\")) {\n                    if (line.contains(\"Heap before GC\"))\n                        getDiary().setTrue(PRINT_HEAP_AT_GC);\n                    getDiary().setTrue(GC_DETAILS);\n                } else if (decorators.tagsContain(\"gc,ref\"))\n                    getDiary().setTrue(PRINT_REFERENCE_GC);\n                else if (decorators.tagsContain(\"gc,heap\") && decorators.getLogLevel().get() == UnifiedLoggingLevel.debug)\n                    getDiary().setTrue(PRINT_HEAP_AT_GC);\n            } else if (getDiary().isShenandoah()) {\n                if (decorators.tagsContain(\"gc,task\") || decorators.tagsContain(\"gc,start\"))\n                    getDiary().setTrue(GC_DETAILS);\n                else if (decorators.tagsContain(\"gc,ergo\"))\n                    getDiary().setTrue(ADAPTIVE_SIZING);\n                else if (decorators.tagsContain(\"gc\") && line.contains(\"Trigger\"))\n                    getDiary().setTrue(GC_CAUSE);\n            }\n        }\n    }\n\n    private void timeOfFirstEvent(Decorators decorator) {\n        if ( ! diary.hasTimeOfFirstEvent())\n            diary.setTimeOfFirstEvent(decorator.getDateTimeStamp());\n    }\n\n    private void extractTagsAndLevels(Decorators decorators) {\n        tagsAndLevels.add(decorators.getLogLevel() + \":\" + decorators.getTags());\n    }\n\n    /*\n    G1 flags\n        ADAPTIVE_SIZING,RSET_STATS,PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS\n        todo: test for log segments that are missing the header lines that will contain the \"use\" keyword\n        ZGC - x\n        Shenandoah - x\n        G1 - done\n        CMS - x\n        Parallel - x\n        Serial - x\n     */\n    private void discoverCollector(String line) {\n\n        if ( ZGC_TAG.parse(line) != null || CYCLE_START.parse(line) != null) {\n            getDiary().setTrue(ZGC);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS);\n            return;\n        }\n\n        if ( SHENANDOAH_TAG.parse(line) != null) {\n            getDiary().setTrue(SHENANDOAH);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, G1GC, RSET_STATS, ZGC, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TENURING_DISTRIBUTION, MAX_TENURING_THRESHOLD_VIOLATION, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, PRINT_HEAP_AT_GC);\n            return;\n        }\n\n        if (G1_TAG.parse(line) != null || line.contains(\"G1 Evacuation Pause\") || (line.contains(\"Humongous regions: \"))) {\n            getDiary().setTrue(G1GC);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, ZGC, SHENANDOAH, CMS_DEBUG_LEVEL_1, PRE_JDK70_40, JDK70, JDK80, TLAB_DATA, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS);\n            return;\n        }\n\n        if (CMS_TAG.parse(line) != null ||\n                PARNEW_TAG.parse(line) != null ||\n                line.contains(\"ParNew\")) {\n            getDiary().setTrue(PARNEW, CMS);\n            getDiary().setFalse(DEFNEW, SERIAL, PARALLELGC, PARALLELOLDGC, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS);\n            return;\n        }\n\n        if (PARALLEL_TAG.parse(line) != null ||\n                line.contains(\"ParOldGen\") ||\n                line.contains(\"PSYoungGen\")) {\n            getDiary().setTrue(PARALLELGC, PARALLELOLDGC, GC_CAUSE);\n            getDiary().setFalse(DEFNEW, SERIAL, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS);\n            return;\n        }\n\n        if (SERIAL_TAG.parse(line) != null || line.contains(\"DefNew\")) {\n            getDiary().setTrue(DEFNEW, SERIAL, GC_CAUSE);\n            getDiary().setFalse(PARALLELGC, PARALLELOLDGC, PARNEW, CMS, ICMS, CMS_DEBUG_LEVEL_1, G1GC, ZGC, SHENANDOAH, PRE_JDK70_40, JDK70, JDK80, RSET_STATS);\n            return;\n        }\n    }\n\n    /**\n     *\n     * @param line\n     */\n    private void discoverDetails(String line) {\n\n        //todo: RSET_STATS for G1 not looked for...\n        if (G1_COLLECTION.parse(line) != null) {\n            getDiary().setTrue(GC_CAUSE);\n        }\n\n        else if (CYCLE_START.parse(line) != null) {\n            getDiary().setTrue(GC_CAUSE);\n        }\n\n        if (stopTheWorldEvents > CYCLES_TO_EXAMINE_BEFORE_GIVING_UP)\n            getDiary().setFalse(ADAPTIVE_SIZING, GC_CAUSE, TLAB_DATA, PRINT_REFERENCE_GC, PRINT_PROMOTION_FAILURE, PRINT_FLS_STATISTICS, RSET_STATS, PRINT_HEAP_AT_GC);\n    }\n\n    private void discoverJVMEvents(String line) {\n        if (stopTheWorldEvents > CYCLES_TO_EXAMINE_FOR_SAFEPOINT) {\n            getDiary().setFalse(APPLICATION_STOPPED_TIME, APPLICATION_CONCURRENT_TIME);\n        }\n    }\n\n    @Override\n    public DateTimeStamp getTimeOfFirstEvent() {\n        return getDiary().getTimeOfFirstEvent();\n    }\n\n}\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/vmops/SafepointParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.vmops;\n\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.event.jvm.Safepoint;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.PreUnifiedGCLogParser;\n\nimport java.util.Set;\n\npublic class SafepointParser extends PreUnifiedGCLogParser implements SafepointPatterns {\n\n    public SafepointParser() {}\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.SAFEPOINT);\n    }\n\n    public String getName() {\n        return \"SafepointParser\";\n    }\n\n    protected void process(String line) {\n        SafepointTrace trace;\n        if ((trace = TRACE.parse(line)) != null) {\n            Safepoint safepoint = trace.toSafepoint();\n            super.publish(ChannelName.JVM_EVENT_PARSER_OUTBOX, safepoint);\n        } else if (line.equals(END_OF_DATA_SENTINEL))\n            super.publish( ChannelName.JVM_EVENT_PARSER_OUTBOX, new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n    }\n\n    @Override\n    public boolean accepts(Diary diary) {\n        return (diary.isTLABData() || diary.isApplicationStoppedTime() || diary.isApplicationRunningTime()) && ! diary.isUnifiedLogging();\n    }\n\n    @Override\n    public void publishTo(JVMEventChannel bus) {\n        super.publishTo(bus);\n    }\n}\n\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/vmops/SafepointPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.vmops;\n\n\nimport com.microsoft.gctoolkit.parser.PreUnifiedTokens;\nimport com.microsoft.gctoolkit.parser.SafepointParseRule;\n\n/**\n * General format is;\n * vmop                    [threads: total initially_running wait_to_block]    [time: spin block sync cleanup vmop] page_trap_count\n */\npublic interface SafepointPatterns extends PreUnifiedTokens {\n\n    String VMOP = \"(Deoptimize|no vm operation|EnableBiasedLocking|GenCollectForAllocation|RevokeBias|BulkRevokeBias|ThreadDump|FindDeadlocks|Exit)\";\n    String SAFE_POINT_PREFIX = TIMESTAMP + VMOP;\n    String THREAD_COUNTS = \"\\\\[\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\\\\]\";\n    String TIMINGS = \"\\\\[\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\" + COUNTER + \"\\\\s+\\\\]\";\n\n    SafepointParseRule TRACE = new SafepointParseRule(SAFE_POINT_PREFIX + \"\\\\s+\" + THREAD_COUNTS + \"\\\\s+\" + TIMINGS + \"\\\\s+\" + COUNTER);\n}\n\n", "needle_token_start": 10492, "needle_token_end": 10517, "code_context_ntokens": 15381, "output": ["```java\npublic boolean supported(String flag) {\n    return gcLoggingFlagSupport.get(flag);\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "setHumongousOccupancyAfterCollection", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCForwardReference.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To record the memory used by the largest objects in a memory pool after a garbage collection process has been completed.\n2. **Input**: A single `long` value representing the amount of memory occupied by the largest objects after garbage collection.\n3. **Output**: A `boolean` value indicating whether the update was successful. It returns `true` if the memory measurement was previously unset (indicated by a negative value) and has now been set to the provided value; otherwise, it returns `false`.\n4. **Procedure**: The function checks if the current memory measurement for the largest objects after collection is unset (negative). If unset, it updates this measurement with the provided value and returns `true`. If already set, it leaves the measurement unchanged and returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.GCToolKit;\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.generational.BinaryTreeDictionary;\nimport com.microsoft.gctoolkit.event.generational.CMSRemark;\nimport com.microsoft.gctoolkit.event.generational.ConcurrentModeFailure;\nimport com.microsoft.gctoolkit.event.generational.ConcurrentModeInterrupted;\nimport com.microsoft.gctoolkit.event.generational.DefNew;\nimport com.microsoft.gctoolkit.event.generational.FullGC;\nimport com.microsoft.gctoolkit.event.generational.InitialMark;\nimport com.microsoft.gctoolkit.event.generational.PSFullGC;\nimport com.microsoft.gctoolkit.event.generational.PSYoungGen;\nimport com.microsoft.gctoolkit.event.generational.ParNew;\nimport com.microsoft.gctoolkit.event.generational.ParNewPromotionFailed;\nimport com.microsoft.gctoolkit.event.generational.SystemGC;\nimport com.microsoft.gctoolkit.event.generational.YoungGC;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.collection.MRUQueue;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.AbstractMap;\nimport java.util.ArrayList;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.function.BiConsumer;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedG1GCPatterns.WEAK_PROCESSING;\n\n/**\n * Time of GC\n * GCType\n * Collect total heap values\n * Heap before collection\n * Heap after collection\n * Heap configured size\n * total pause time\n * CMS failures\n * System.gc() calls\n */\npublic class GenerationalHeapParser extends PreUnifiedGCLogParser implements SimplePatterns, ICMSPatterns, SerialPatterns, ParallelPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(GenerationalHeapParser.class.getName());\n\n    private ParNew parNewForwardReference;\n    private GarbageCollectionTypes garbageCollectionTypeForwardReference;\n    private GCCause gcCauseForwardReference;\n    private DateTimeStamp scavengeTimeStamp;\n    private DateTimeStamp fullGCTimeStamp;\n    private DateTimeStamp remarkTimeStamp;\n    private MemoryPoolSummary youngMemoryPoolSummaryForwardReference;\n    private MemoryPoolSummary tenuredForwardReference;\n    private MemoryPoolSummary heapForwardReference;\n    private MemoryPoolSummary metaSpaceForwardReference;\n    private double scavengeDurationForwardReference;\n    private ReferenceGCSummary referenceGCForwardReference;\n    private CPUSummary scavengeCPUSummaryForwardReference;\n    private int[] promotionFailureSizesForwardReference;\n\n    private long totalFreeSpaceForwardReference;\n    private long maxChunkSizeForwardReference;\n    private int numberOfBlocksForwardReference;\n    private long averageBlockSizeForwardReference;\n    private int treeHeightForwardReference;\n\n    //Expect Remark\n    private boolean expectRemark = false;\n\n    /* Rules not used....\n        GCParseRule FULL_PARNEW_CMF_META\n        GCParseRule FULL_GC_CMF\n        GCParseRule FULL_GC_CARDS_DETAILS\n        GCParseRule PARNEW_PROMOTION_FAILED_DETAILS_AFTER\n        GCParseRule DUP_CMF\n        GCParseRule FULL_PARNEW_CMF_PERM\n        GCParseRule FULL_GC_CMS\n        GCParseRule PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE_META\n        GCParseRule CMS_FULL_80\n     */\n\n\n    private final MRUQueue<GCParseRule, BiConsumer<GCLogTrace, String>> parseRules;\n\n    {\n        parseRules = new MRUQueue<>();\n        parseRules.put(DEFNEW, this::defNew);\n        parseRules.put(DEFNEW_TENURING, this::defNewWithTenuring);\n        parseRules.put(SERIAL_FULL, this::serialFull);\n        parseRules.put(SERIAL_FULL80, this::serialFull80);\n        parseRules.put(PARNEW, this::parNew);\n        parseRules.put(PARNEW_TENURING, this::parNewWithTenuring);\n        parseRules.put(PARNEW_CONCURRENT_MODE_END, this::parNewConcurrentModeEnd);\n        parseRules.put(PARNEW_CARDTABLE, this::parNewCardTable);\n        parseRules.put(PARNEW_TO_CMF_PERM, this::parNewToConcurrentModeFailure);\n        parseRules.put(PARNEW_TO_CMF_META, this::parNewToConcurrentModeFailure);\n        parseRules.put(PARNEW_REFERENCE,this::parNewReference);\n        parseRules.put(PARNEW_REFERENCE_SPLIT, this::parNewReferenceSplit);\n        parseRules.put(JVMPatterns.TLAB_START, this::gcSplitByTLAB);\n        parseRules.put(PARNEW_REFERENCE_SPLIT_BY_TLAB, this::parNewReferenceSplitByTLAB);\n        parseRules.put(DEFNEW_REFERENCE, this::defNewReference);\n        parseRules.put(PARNEW_PROMOTION_FAILED, this::parNewPromotionFailed);\n        parseRules.put(PARNEW_PROMOTION_FAILED_DETAILS, this::parNewPromotionFailedDetails);\n        parseRules.put(PARNEW_PROMOTION_FAILED_REFERENCE, this::parNewPromotionFailedReference);\n        parseRules.put(FLOATING_REFERENCE, this::parNewFloatingReference);\n        parseRules.put(PARNEW_PROMOTION_FAILED_TENURING, this::parNewPromotionFailedTenuring);\n        parseRules.put(PARNEW_PROMOTION_FAILED_IN_CMS_PHASE, this::parNewPromotionFailedInConcurrentMarkSweepPhase);\n        parseRules.put(CMS_BAILING_TO_FOREGROUND, this::concurrentMarkSweepBailingToForeground);\n        parseRules.put(PROMOTION_FAILED_TO_FULL, this::promotionFailedToFull);\n        parseRules.put(PARNEW_PLAB, this::parNewPLAB);\n        parseRules.put(PLAB_ENTRY, this::plabEntry);\n        parseRules.put(PLAB_SUMMARY, this::plabSummary);\n        parseRules.put(FULLGC_FLS_BEFORE, this::fullGCFLSBefore);\n        parseRules.put(PARNEW_FLS_BEFORE, this::parNewFLSBefore);\n        parseRules.put(PARNEW_FLS_AFTER, this::parNewFLSAfter);\n        parseRules.put(PARNEW_FLS_BODY, this::parNewFLSBody);\n        parseRules.put(PARNEW_PROMOTION_FAILED_DETAILS_AFTER, this::parNewConcurrentModeFailureFLSAfter);\n        parseRules.put(PARNEW_FLS_TIME, this::parNewFLSTime);\n\n        parseRules.put(FLS_HEADER, this::parNewFLSHeader);\n        parseRules.put(FLS_SEPARATOR, this::parNewFLSSeparator);\n        parseRules.put(FLS_TOTAL_FREE_SPACE, this::parNewFLSTotalFreeSpace);\n        parseRules.put(FLS_MAX_CHUNK_SIZE, this::parNewFLSMaxChunkSize);\n        parseRules.put(FLS_NUMBER_OF_BLOCKS, this::parNewFLSNumberOfBlocks);\n        parseRules.put(FLS_AVERAGE_BLOCK_SIZE, this::parNewFLSAveBlockSize);\n        parseRules.put(FLS_TREE_HEIGHT, this::parNewFLSTreeHeight);\n        parseRules.put(FLS_LARGE_BLOCK_PROXIMITY, this::flsLargeBlockProximity);\n        parseRules.put(FLS_LARGE_BLOCK, this::flsLargeBlock);\n\n        parseRules.put(PARNEW_PROMOTION_FAILED_TIME_ABORT_PRECLEAN, this::parNewPromotionFailedTimeAbortPreclean);\n        parseRules.put(PARNEW_PROMOTION_FAILED_CONCURRENT_PHASE, this::parNewPromotionFailedConcurrentPhase);\n        parseRules.put(CORRUPTED_PARNEW_CONCURRENT_PHASE, this::corruptedParNewConcurrentPhase);\n        parseRules.put(CORRUPTED_PARNEW_BODY, this::corruptedParNewBody);\n        parseRules.put(CONCURRENT_PHASE_START, this::concurrentPhaseStart);\n        parseRules.put(CONCURRENT_PHASE_END, this::concurrentPhaseEnd);\n        parseRules.put(ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE, this::abortPrecleanDueToTimeClause);\n        parseRules.put(INITIAL_MARK, this::initialMark);\n        parseRules.put(SCAVENGE_BEFORE_REMARK, this::scavengeBeforeRemark);\n        parseRules.put(SCAVENGE_BEFORE_REMARK_TENURING, this::scavengeBeforeRemarkTenuring);\n        parseRules.put(PARALLEL_REMARK_WEAK_REF, this::remarkAt12);\n        parseRules.put(PARALLEL_REMARK_CLASS_UNLOADING, this::remarkAt21);\n        parseRules.put(REMARK_PARNEW_PROMOTION_FAILED, this::remarkParNewPromotionFailed);\n        parseRules.put(PARALLEL_REMARK_STRING_SYMBOL, this::parallelRemarkStringSymbolClause);\n        parseRules.put(PARALLEL_REMARK_WEAK_CLASS_SYMBOL_STRING, this::remarkAt21);\n        parseRules.put(PARALLEL_REMARK_WEAK_STRING, this::remarkAt15);\n        parseRules.put(PARALLEL_RESCAN, this::parallelRescan);\n        parseRules.put(REMARK, this::remarkAt11);\n        parseRules.put(PARALLEL_RESCAN_V2, this::remarkAt11);\n        parseRules.put(PARALLEL_RESCAN_WEAK_CLASS_SCRUB, this::remarkAt13);\n        //, 0.1127040 secs]220.624: [weak refs processing, 0.1513820 secs] [1 CMS-remark: 10541305K(16777216K)] 10742883K(18664704K), 0.7371020 secs]\n        //todo: this was capturing records that is shouldn't have so the rule was modified.. now does it work??? Needs through testing now that order of evaluation will change\n        parseRules.put(REMARK_DETAILS, this::remarkAt1);\n        parseRules.put(SERIAL_REMARK_SCAN_BREAKDOWNS, this::remarkAt15);\n        parseRules.put(REMARK_DETAILS, this::remarkAt1);\n        parseRules.put(SERIAL_REMARK_SCAN_BREAKDOWNS, this::remarkAt15);\n        parseRules.put(REMARK_REFERENCE_PROCESSING, this::recordRemarkWithReferenceProcessing);\n        parseRules.put(TENURING_DETAILS, this::tenuringDetails);\n        parseRules.put(RESCAN_WEAK_CLASS_SYMBOL_STRING, this::remarkAt11);\n        parseRules.put(CONCURRENT_MODE_FAILURE_DETAILS, this::concurrentModeFailureDetails);\n        parseRules.put(CONCURRENT_MODE_FAILURE_DETAILS_META, this::concurrentModeFailureDetails);\n        parseRules.put(PARNEW_DETAILS_CONCURRENT_MODE_FAILURE_PERM, this::parNewDetailsConcurrentModeFailure);\n        parseRules.put(PARNEW_DETAILS_CONCURRENT_MODE_FAILURE_META, this::parNewDetailsConcurrentModeFailure);\n        parseRules.put(PARNEW_DETAILS_PROMOTION_FAILED_WITH_CMS_PHASE, this::parNewDetailsPromotionFailedWithConcurrentMarkSweepPhase);\n        parseRules.put(PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE, this::parNewDetailsWithConcurrentModeFailure);\n        parseRules.put(CONCURRENT_MODE_FAILURE_REFERENCE, this::concurrentModeFailureReference);\n        parseRules.put(iCMS_PARNEW_DEFNEW_TENURING_DETAILS, this::iCMSParNewDefNewTenuringDetails);\n        parseRules.put(iCMS_CONCURRENT_MODE_FAILURE, this::iCMSConcurrentModeFailure);\n        parseRules.put(iCMS_CONCURRENT_MODE_FAILURE_META, this::iCMSConcurrentModeFailure);\n        parseRules.put(iCMS_CMF_DUIRNG_PARNEW_DEFNEW_DETAILS, this::iCMSConcurrentModeFailureDuringParNewDefNewDetails);\n        parseRules.put(FULL_GC_INTERRUPTS_CONCURRENT_PHASE, this::fullGCInterruptsConcurrentPhase);\n        parseRules.put(FULL_PARNEW_START, this::fullParNewStart);\n        parseRules.put(FULL_GC_REFERENCE_CMF, this::fullGCReferenceConcurrentModeFailure);\n\n        parseRules.put(iCMS_PARNEW, this::iCMSParNew);\n        parseRules.put(iCMS_PARNEW_PROMOTION_FAILURE_RECORD, this::iCMSParNewPromotionFailureRecord);\n        parseRules.put(iCMS_PARNEW_PROMOTION_FAILURE, this::iCMSParNewPromotionFailure);\n        parseRules.put(FULL_GC_ICMS, this::fullGCiCMS);\n        parseRules.put(iCMS_PARNEW_DEFNEW_TENURING_DETAILS, this::iCMSParNewDefNewTenuringDetails);\n        parseRules.put(iCMS_FULL, this::iCMSFullGC);\n        parseRules.put(iCMS_PROMOTION_FAILED, this::iCMSPromotionFailed);\n        parseRules.put(iCMS_PROMOTION_FAILED_PERM, this::iCMSPromotionFailedPermMeta);\n        parseRules.put(iCMS_PROMOTION_FAILED_META, this::iCMSPromotionFailedPermMeta);\n        parseRules.put(iCMS_MISLABELED_FULL, this::iCMSMislabeledFull);\n        parseRules.put(iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE, this::iCMSFullAfterConcurrentModeFailure);\n        parseRules.put(iCMS_FULL_AFTER_CONCURRENT_MODE_FAILURE_META, this::iCMSFullAfterConcurrentModeFailure);\n        parseRules.put(iCMS_CONCURRENT_MODE_INTERRUPTED, this::iCMSConcurrentModeInterrupted);\n        parseRules.put(PS_FULL_GC_META, this::psFullGCMeta);\n        parseRules.put(PS_FULL_GC_V2_META, this::psFullGCV2Meta);\n        parseRules.put(CMS_FULL_META, this::cmsFullPermOrMeta);\n        parseRules.put(FULL_PARNEW_CMF_META, this::fullParNewConcurrentModeFailureMeta);\n        parseRules.put(FULL_PARNEW_CMF_PERM, this::fullParNewConcurrentModeFailurePerm);\n        parseRules.put(PARNEW_CONCURRENT_MODE_FAILURE_PERM, this::parNewConcurrentModeFailurePerm);\n        parseRules.put(PARNEW_CONCURRENT_MODE_FAILURE_META, this::parNewConcurrentModeFailureMeta);\n        parseRules.put(PS_FULL_GC_V2_PERM, this::psFullGCV2Perm);\n        parseRules.put(PS_FULL_GC_PERM, this::psFullGCPerm);\n        parseRules.put(CMS_FULL_PERM, this::cmsFullPermOrMeta);\n        parseRules.put(CMS_FULL_PERM_META_REFERENCE, this::cmsFullPermOrMeta);\n        parseRules.put(PARNEW_NO_DETAILS, this::parNewNoDetails);\n        parseRules.put(YOUNG_NO_DETAILS, this::youngNoDetails);\n        parseRules.put(CMS_NO_DETAILS, this::cmsNoDetails);\n        parseRules.put(FULL_NO_GC_DETAILS, this::fullNoGCDetails);\n        parseRules.put(PARNEW_START, this::parNewStart);\n        parseRules.put(GC_START, this::gcStart);\n        parseRules.put(YOUNG_SPLIT_NO_DETAILS, this::youngSplitNoDetails);\n        parseRules.put(CMF_SIMPLE, this::cmfSimple);\n        parseRules.put(DEFNEW_DETAILS, this::defNewDetails);\n        parseRules.put(PRECLEAN_REFERENCE_PAR_NEW_REFERENCE, this::preCleanReferenceParNewReference);\n\n        parseRules.put(PSYOUNGGEN, this::psYoungGen);\n        parseRules.put(PSYOUNGGEN_PROMOTION_FAILED, this::psYoungGen);\n        parseRules.put(PSFULL, this::psFull);\n        parseRules.put(PSYOUNGGEN_NO_DETAILS, this::psYoungNoDetails);\n        parseRules.put(PSYOUNGGEN_REFERENCE_SPLIT, this::psYoungGenReferenceProcessingSplit);\n        parseRules.put(PSYOUNGGEN_REFERENCE, this::psYoungGenReferenceProcessing);\n        parseRules.put(PS_TENURING_START, this::psTenuringStart);\n        parseRules.put(PSFULL_SPLIT, this::psFullSPlit);\n        parseRules.put(PS_FULL_REFERENCE_SPLIT, this::psFullReferenceSplit);\n        parseRules.put(PS_FULL_REFERENCE, this::psFullReference);\n        parseRules.put(PS_DETAILS_WITH_TENURING, this::psDetailsWithTenuring);\n        parseRules.put(PS_FAILURE, this::psFailure);\n        parseRules.put(PSYOUNG_ADAPTIVE_SIZE_POLICY, this::psYoungAdaptiveSizePolicy);\n        parseRules.put(PSOLD_ADAPTIVE_SIZE_POLICY, this::psFullAdaptiveSizePolicy);\n        parseRules.put(PSYOUNG_DETAILS_FLOATING, this::psYoungDetailsFloating);\n        parseRules.put(PSFULL_ADAPTIVE_SIZE, this::psFullAdaptiveSize);\n        parseRules.put(PS_FULL_BODY_FLOATING, this::psFullBodyFloating);\n        parseRules.put(FULL_REFERENCE_ADAPTIVE_SIZE, this::psFullReferenceAdaptiveSize);\n        parseRules.put(PS_PROMOTION_FAILED, this::psPromotionFailed);\n\n        parseRules.put(RESCAN_SPLIT_UNLOADING_STRING, this::rescanSplitUnloadingString);\n\n        parseRules.put(PARNEW_CONCURRENT_PHASE_CARDS, this::parNewConcurrentPhaseCards);\n        parseRules.put(CONC_PHASE_YIELDS, this::concurrentPhaseYields);\n        parseRules.put(PRECLEAN_TIMED_OUT_WITH_CARDS, this::precleanTimedoutWithCards);\n        parseRules.put(PARNEW_SHOULD_CONCURRENT_COLLECT, this::parNewShouldConcurrentCollect);\n        parseRules.put(SHOULD_CONCURRENT_COLLECT, this::shouldCollectConcurrent);\n        parseRules.put(REMARK_SPLIT_BY_DEBUG, this::remarkSplitByDebug);\n        parseRules.put(SCAVENGE_BEFORE_REMARK_PRINT_HEAP_AT_GC, this::scavengeBeforeRemarkPrintHeapAtGC);\n        parseRules.put(SPLIT_REMARK_REFERENCE_BUG, this::splitRemarkReferenceWithWeakReferenceSplitBug);\n        parseRules.put(SPLIT_REMARK_REFERENCE, this::splitRemarkReference);\n        parseRules.put(PSYOUNG_ADAPTIVE_SIZE_POLICY_START, this::psYoungAdaptivePolicySizeStart);\n        parseRules.put(PS_ADAPTIVE_SIZE_POLICY_BODY, this::psYoungAdaptivePolicySizeBody);\n        parseRules.put(ADAPTIVE_SIZE_POLICY_BODY, this::adaptivePolicySizeBody);\n        parseRules.put(ADAPTIVE_SIZE_POLICY_STOP, this::adaptiveSizePolicyStop);\n\n        parseRules.put(SCAVENGE_BEFORE_REMARK_REFERENCE, this::scavengeBeforeRemarkReference);\n        parseRules.put(SCAVENGE_BEFORE_REMARK_REFERENCE_SPLIT, this::scavengeBeforeRemarkReferenceSplit);\n        parseRules.put(PARNEW_TO_CONCURRENT_MODE_FAILURE, this::parNewToPsudoConcurrentModeFailure);\n        parseRules.put(SERIAL_FULL_REFERENCE, this::serialFullReference);\n        parseRules.put(PSFULL_ERGONOMICS_PHASES, this::psFullErgonomicsPhases);\n        parseRules.put(PSFULL_REFERENCE_PHASE, this::psFullReferencePhase);\n        parseRules.put(SPLIT_PARNEW_PROMOTION_FAILED_IN_CMS_PHASE, this::parNewPromotionFailedInConcurrentMarkSweepPhase);\n        parseRules.put(FULL_SPLIT_BY_CONCURRENT_PHASE, this::fullSplitByConcurrentPhase);\n        parseRules.put(CMF_LARGE_BLOCK, this::concurrentModeFailureSplitByLargeBlock);\n        parseRules.put(WEAK_PROCESSING, this::noop);\n\n\n        parseRules.put(new GCParseRule(\"END_OF_DATA_SENTINEL\", END_OF_DATA_SENTINEL), this::endOfFile);\n    }\n\n    public GenerationalHeapParser() {\n    }\n\n    @Override\n    public Set<EventSource> eventsProduced() {\n        return Set.of(EventSource.GENERATIONAL);\n    }\n\n    @Override\n    public String getName() {\n        return \"GenerationalHeapParser\";\n    }\n\n    @Override\n    protected void process(String line) {\n\n        if (ignoreFrequentButUnwantedEntries(line)) return;\n\n        try {\n            Optional<AbstractMap.SimpleEntry<GCParseRule, GCLogTrace>> optional = parseRules.keys()\n                    .stream()\n                    .map(rule -> new AbstractMap.SimpleEntry<>(rule, rule.parse(line)))\n                    .filter(tuple -> tuple.getValue() != null)\n                    .findFirst();\n            if ( optional.isPresent()) {\n                AbstractMap.SimpleEntry<GCParseRule, GCLogTrace> ruleAndTrace = optional.get();\n                parseRules.get(ruleAndTrace.getKey()).accept(ruleAndTrace.getValue(), line);\n                return;\n            }\n        } catch (Throwable t) {\n            LOGGER.throwing(this.getName(), \"process\", t);\n        }\n\n        log(line);\n    }\n\n    private boolean ignoreFrequentButUnwantedEntries(String line) {\n\n        if (JVMPatterns.APPLICATION_TIME.parse(line) != null) return true;\n        if (JVMPatterns.SIMPLE_APPLICATION_TIME.parse(line) != null) return true;\n\n        if (JVMPatterns.APPLICATION_STOP_TIME.parse(line) != null) return true;\n        if (JVMPatterns.APPLICATION_STOP_TIME_WITH_STOPPING_TIME.parse(line) != null) return true;\n        if (JVMPatterns.SIMPLE_APPLICATION_STOP_TIME.parse(line) != null) return true;\n\n        if (TenuredPatterns.TENURING_SUMMARY.parse(line) != null) return true;\n        if (TenuredPatterns.TENURING_AGE_BREAKDOWN.parse(line) != null) return true;\n\n        if (line.startsWith(\"TLAB: gc thread: \")) return true;\n        if (line.startsWith(\"TLAB totals: thrds: \")) return true;\n\n        if (line.startsWith(\"{Heap before\")) {\n            return true;\n        }\n        if (line.equals(\"}\")) {\n            return true;\n        }\n\n        if (line.startsWith(\"Heap after\")) return true;\n        if (line.startsWith(\"PSYoungGen\")) return true;\n        if (line.startsWith(\"ParOldGen\")) return true;\n        if (line.startsWith(\"PSOldGen\")) return true;\n        if (line.startsWith(\"PSPermGen\")) return true;\n        if (line.startsWith(\"object space\")) return true;\n        if (line.startsWith(\"eden space\")) return true;\n        if (line.startsWith(\"from space\")) return true;\n        if (line.startsWith(\"to   space\")) return true;\n        if (line.contains(\"[0xffff\") && line.endsWith(\"000)\")) ;\n\n        if (line.startsWith(\"Finished \")) return true;\n\n        if (line.startsWith(\"GC locker: Trying a full collection because scavenge failed\")) return true;\n\n        if (line.startsWith(\"Java HotSpot(TM)\")) return true;\n        if (line.startsWith(\"OpenJDK 64\")) return true;\n        if (line.startsWith(\"Memory: \")) return true;\n        if (line.startsWith(\"CommandLine flags: \")) return true;\n        if (line.startsWith(\"/proc/meminfo\")) return true;\n        return line.startsWith(\"avg_survived_padded_avg\");\n    }\n\n    public void endOfFile(GCLogTrace trace, String line) {\n        publish(new JVMTermination(getClock(),diary.getTimeOfFirstEvent()));\n    }\n\n    public void defNew(GCLogTrace trace, String line) {\n        DefNew defNew = new DefNew(getClock(), trace.gcCause(), trace.getDuration());\n        defNew.add(trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(7), this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 14));\n        defNew.add(extractCPUSummary(line));\n        publish(defNew);\n    }\n\n    //2019-10-22T23:41:21.852+0000: 21.912: [GC (GCLocker Initiated GC) 2019-10-22T23:41:21.853+0000: 21.912: [DefNew2019-10-22T23:41:21.914+0000: 21.974: [SoftReference, 0 refs, 0.0000842 secs]2019-10-22T23:41:21.914+0000: 21.974: [WeakReference, 76 refs, 0.0000513 secs]2019-10-22T23:41:21.914+0000: 21.974: [FinalReference, 91635 refs, 0.0396861 secs]2019-10-22T23:41:21.954+0000: 22.014: [PhantomReference, 0 refs, 3 refs, 0.0000444 secs]2019-10-22T23:41:21.954+0000: 22.014: [JNI Weak Reference, 0.0000281 secs]: 419520K->19563K(471936K), 0.1019514 secs] 502104K->102148K(2044800K), 0.1020469 secs] [Times: user=0.09 sys=0.01, real=0.10 secs]\n    public void defNewDetails(GCLogTrace trace, String line) {\n        DefNew defNew = new DefNew(getClock(), trace.gcCause(), trace.getDuration());\n        MemoryPoolSummary heap = this.getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(trace, 37);\n        MemoryPoolSummary young = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(30);\n        defNew.add(young, heap.minus(young), heap);\n        defNew.add(extractCPUSummary(line));\n        defNew.add(extractPrintReferenceGC(line));\n        publish(defNew);\n    }\n\n    public void defNewWithTenuring(GCLogTrace trace, String line) {\n        scavengeTimeStamp = getClock();\n        garbageCollectionTypeForwardReference = GarbageCollectionTypes.DefNew;\n    }\n\n    //3.299: [Full GC (Metadata GC Threshold) 3.299: [Tenured: 21006K->20933K(21888K), 0.0230475 secs] 29003K->20933K(31680K), [Metapace: 12111K->12111K(12672K)], 0.0231557 secs]\n    public void serialFull(GCLogTrace trace, String line) {\n        FullGC collection = new FullGC(getClock(), trace.gcCause(), trace.getDuration());\n...\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCForwardReference.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MalformedEvent;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.RegionSummary;\nimport com.microsoft.gctoolkit.event.SurvivorMemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.UnifiedCountSummary;\nimport com.microsoft.gctoolkit.event.UnifiedStatisticalSummary;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentCleanupForNextMark;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentClearClaimedMarks;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentCompleteCleanup;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentCreateLiveData;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentScanRootRegion;\nimport com.microsoft.gctoolkit.event.g1gc.G1Cleanup;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentMark;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentRebuildRememberedSets;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentUndoCycle;\nimport com.microsoft.gctoolkit.event.g1gc.G1FullGC;\nimport com.microsoft.gctoolkit.event.g1gc.G1FullGCNES;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCConcurrentEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCPauseEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1Mixed;\nimport com.microsoft.gctoolkit.event.g1gc.G1Remark;\nimport com.microsoft.gctoolkit.event.g1gc.G1SystemGC;\nimport com.microsoft.gctoolkit.event.g1gc.G1Young;\nimport com.microsoft.gctoolkit.event.g1gc.G1YoungInitialMark;\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.logging.Logger;\nimport java.util.stream.Stream;\n\n\nclass G1GCForwardReference extends ForwardReference {\n\n    private static final Logger LOGGER = Logger.getLogger(G1GCForwardReference.class.getName());\n\n    private int heapRegionSize = 0;\n    private long minHeapSize;\n    private long initialHeapSize;\n    private long maxHeapSize;\n    private DateTimeStamp concurrentCycleStartTime;\n\n    void setHeapRegionSize(int sizeInMegaBytes) {\n        heapRegionSize = sizeInMegaBytes;\n    }\n\n    int getHeapRegionSize() {\n        return heapRegionSize;\n    }\n\n    private GarbageCollectionTypes pausePhaseDuringConcurrentCycle = null;\n    private GarbageCollectionTypes gcType = null;\n    private GarbageCollectionTypes concurrentPhase;\n\n    G1GCForwardReference(Decorators decorators, int gcID) {\n        super(decorators, gcID);\n    }\n\n    boolean isConcurrentCycle() {\n        return gcType == GarbageCollectionTypes.Concurrent_Cycle;\n    }\n\n    //bag of stuff to maybe eliminate\n    void setMinHeapSize(long minHeapSize) {\n        this.minHeapSize = minHeapSize;\n    }\n\n    long getMinHeapSize() {\n        return minHeapSize;\n    }\n\n    void setInitialHeapSize(long initialHeapSize) {\n        this.initialHeapSize = initialHeapSize;\n    }\n\n    long getInitialHeapSize() {\n        return initialHeapSize;\n    }\n\n    void setMaxHeapSize(long maxHeapSize) {\n        this.maxHeapSize = maxHeapSize;\n    }\n\n    long getMaxHeapSize() {\n        return maxHeapSize;\n    }\n\n    void setGcType(GarbageCollectionTypes garbageCollectionType) {\n        if ((this.gcType != null) && (gcType != garbageCollectionType)) {\n            throw new IllegalArgumentException(\"attempting to redefine GC Type from\" + this.gcType + \" to \" + garbageCollectionType);\n        }\n        gcType = garbageCollectionType;\n    }\n\n    GarbageCollectionTypes getGcType() {\n        return gcType;\n    }\n\n    void setConcurrentPhase(GarbageCollectionTypes phase) {\n        this.concurrentPhase = phase;\n    }\n\n    GarbageCollectionTypes getConcurrentPhase() {\n        return concurrentPhase;\n    }\n\n\n    /**\n     * memory pool statistics\n     */\n    private static final int HEAP_OCCUPANCY_BEFORE_COLLECTION = 0;\n    private static final int HEAP_OCCUPANCY_AFTER_COLLECTION = 1;\n    private static final int HEAP_SIZE_BEFORE_COLLECTION = 2;\n    private static final int HEAP_SIZE_AFTER_COLLECTION = 3;\n    private static final int EDEN_OCCUPANCY_BEFORE_COLLECTION = 4;\n    private static final int EDEN_OCCUPANCY_AFTER_COLLECTION = 5;\n    private static final int EDEN_SIZE_BEFORE_COLLECTION = 6;\n    private static final int EDEN_SIZE_AFTER_COLLECTION = 7;\n    private static final int SURVIVOR_OCCUPANCY_BEFORE_COLLECTION = 8;\n    private static final int SURVIVOR_OCCUPANCY_AFTER_COLLECTION = 9;\n    private static final int SURVIVOR_SIZE_BEFORE_COLLECTION = 10;\n    private static final int SURVIVOR_SIZE_AFTER_COLLECTION = 11;\n    private static final int YOUNG_OCCUPANCY_BEFORE_COLLECTION = 12;\n    private static final int YOUNG_OCCUPANCY_AFTER_COLLECTION = 13;\n    private static final int YOUNG_SIZE_BEFORE_COLLECTION = 14;\n    private static final int YOUNG_SIZE_AFTER_COLLECTION = 15;\n    private static final int OLD_OCCUPANCY_BEFORE_COLLECTION = 16;\n    private static final int OLD_OCCUPANCY_AFTER_COLLECTION = 17;\n    private static final int OLD_SIZE_BEFORE_COLLECTION = 18;\n    private static final int OLD_SIZE_AFTER_COLLECTION = 19;\n    private static final int HUMONGOUS_OCCUPANCY_BEFORE_COLLECTION = 20;\n    private static final int HUMONGOUS_OCCUPANCY_AFTER_COLLECTION = 21;\n    private static final int HUMONGOUS_SIZE_BEFORE_COLLECTION = 22;\n    private static final int HUMONGOUS_SIZE_AFTER_COLLECTION = 23;\n    private static final int ARCHIVE_OCCUPANCY_BEFORE_COLLECTION = 24;\n    private static final int ARCHIVE_OCCUPANCY_AFTER_COLLECTION = 25;\n    private static final int ARCHIVE_SIZE_BEFORE_COLLECTION = 26;\n    private static final int ARCHIVE_SIZE_AFTER_COLLECTION = 27;\n    private static final int METASPACE_OCCUPANCY_BEFORE_COLLECTION = 28;\n    private static final int METASPACE_OCCUPANCY_AFTER_COLLECTION = 29;\n    private static final int METASPACE_SIZE_BEFORE_COLLECTION = 30;\n    private static final int METASPACE_SIZE_AFTER_COLLECTION = 31;\n    private static final int METASPACE_COMMITTED_BEFORE_COLLECTION = 32;\n    private static final int METASPACE_COMMITTED_AFTER_COLLECTION = 33;\n    private static final int METASPACE_RESERVED_BEFORE_COLLECTION = 34;\n    private static final int METASPACE_RESERVED_AFTER_COLLECTION = 35;\n    private static final int CLASSSPACE_OCCUPANCY_BEFORE_COLLECTION = 36;\n    private static final int CLASSSPACE_OCCUPANCY_AFTER_COLLECTION = 37;\n    private static final int CLASSSPACE_SIZE_BEFORE_COLLECTION = 38;\n    private static final int CLASSSPACE_SIZE_AFTER_COLLECTION = 39;\n    private static final int CLASSSPACE_COMMITTED_BEFORE_COLLECTION = 40;\n    private static final int CLASSSPACE_COMMITTED_AFTER_COLLECTION = 41;\n    private static final int CLASSSPACE_RESERVED_BEFORE_COLLECTION = 42;\n    private static final int CLASSSPACE_RESERVED_AFTER_COLLECTION = 43;\n\n    private final long[] memoryPoolMeasurment = {\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L\n    };\n\n    private boolean setMemoryPoolMeasurement(int index, long value) {\n        if (memoryPoolMeasurment[index] < 0L) {\n            memoryPoolMeasurment[index] = value;\n            return true;\n        }\n        return false;\n    }\n\n\n    boolean setHeapOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setHeapOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setHeapSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setHeapSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setEdenOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setEdenOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setEdenSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setEdenSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setSurvivorOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setSurvivorOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setSurvivorSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setSurvivorSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setYoungOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setYoungOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setYoungSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setYoungSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setOldOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setOldOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setOldSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setOldSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setHumongousOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    \nboolean setHumongousOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setHumongousSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setHumongousSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceCommittedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_COMMITTED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceCommittedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_COMMITTED_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceReservedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_RESERVED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceReservedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_RESERVED_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceCommittedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_COMMITTED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceCommittedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_COMMITTED_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceReservedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_RESERVED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceReservedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_RESERVED_AFTER_COLLECTION, value);\n    }\n\n    // ******\n    // Reference processing\n    private static final int SOFT_REFERENCE = 0;\n    private static final int WEAK_REFERENCE = 1;\n    private static final int PHANTOM_REFERENCE = 2;\n    private static final int FINAL_REFERENCE = 3;\n    private static final int JNI_WEAK_REFERENCE = 4;\n    private static final int CLEANER_REFERENCE = 5;\n    private double[] referenceProcessingDuarations = {-1.0d, -1.0d, -1.0d, -1.0d, -1.0d, -1.0d};\n    private int[] referenceCounts = {-1, -1, -1, -1, -1, -1};\n\n    void setSoftReferenceProcessingDuation(double duration) {\n        referenceProcessingDuarations[SOFT_REFERENCE] = duration;\n    }\n\n    void setWeakReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[WEAK_REFERENCE] = duration;\n    }\n\n    void setPhantomReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[PHANTOM_REFERENCE] = duration;\n    }\n\n    void setFinalReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[FINAL_REFERENCE] = duration;\n    }\n\n    void setJniWeakReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[JNI_WEAK_REFERENCE] = duration;\n    }\n\n    void setCleanerReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[CLEANER_REFERENCE] = duration;\n    }\n\n    void setReferenceCounts(int soft, int weak, int finalReferenceCount, int phantom) {\n        referenceCounts[WEAK_REFERENCE] = weak;\n        referenceCounts[SOFT_REFERENCE] = soft;\n        referenceCounts[PHANTOM_REFERENCE] = phantom;\n        referenceCounts[FINAL_REFERENCE] = finalReferenceCount;\n    }\n\n    private boolean hasReferenceGCSummary() {\n        return referenceCounts[0] != -1;\n    }\n\n    private ReferenceGCSummary generateReferenceGCSummary() {\n        ReferenceGCSummary summary = new ReferenceGCSummary();\n        summary.addSoftReferences(getStartTime(), referenceCounts[SOFT_REFERENCE], referenceProcessingDuarations[SOFT_REFERENCE]);\n        summary.addWeakReferences(getStartTime(), referenceCounts[WEAK_REFERENCE], referenceProcessingDuarations[WEAK_REFERENCE]);\n        summary.addPhantomReferences(getStartTime(), referenceCounts[PHANTOM_REFERENCE], referenceProcessingDuarations[PHANTOM_REFERENCE]);\n        summary.addFinalReferences(getStartTime(), referenceCounts[FINAL_REFERENCE], referenceProcessingDuarations[FINAL_REFERENCE]);\n        summary.addJNIWeakReferences(getStartTime(), referenceCounts[JNI_WEAK_REFERENCE], referenceProcessingDuarations[JNI_WEAK_REFERENCE]);\n        return summary;\n    }\n\n\n    // ****\n    // Young phases\n    private static final double NOT_SET = -1.0d;\n\n    static final int PRE_EVACUATE_COLLECTION_SET = 0;\n    static final int EVACUATE_COLLECTION_SET = 1;\n    static final int POST_EVACUATE_COLLECTION_SET = 2;\n    static final int OTHER = 3;\n\n    private final double[] youngCollectionPhases = {NOT_SET, NOT_SET, NOT_SET, NOT_SET};\n    private final Map<String, Double> preEvacuateCSetPhaseDurations = new ConcurrentHashMap<>(3);\n    private final Map<String, UnifiedStatisticalSummary> evacuateCSetPhaseDurations = new ConcurrentHashMap<>();\n    private final Map<String, Double> postEvacuateCSetPhaseDurations = new ConcurrentHashMap<>();\n\n    void setPreEvacuateCSetDuration(double duration) {\n        this.youngCollectionPhases[PRE_EVACUATE_COLLECTION_SET] = duration;\n    }\n\n    void postPreEvacuateCSetPhaseDuration(String phase, double duration) {\n        preEvacuateCSetPhaseDurations.put(phase, duration);\n    }\n\n    Stream<String> preEvacuateCSetPhaseNames() {\n        return preEvacuateCSetPhaseDurations.keySet().stream();\n    }\n\n    double preEvacuateCSetPhaseDuration(String phaseName) {\n        return preEvacuateCSetPhaseDurations.get(phaseName);\n    }\n\n    void setEvacuationCSetDuration(double duration) {\n        this.youngCollectionPhases[EVACUATE_COLLECTION_SET] = duration;\n    }\n\n    void postEvacuateCSetPhaseDuration(String phase, UnifiedStatisticalSummary stats) {\n        evacuateCSetPhaseDurations.put(phase, stats);\n    }\n\n    Stream<String> evacuateCSetPhaseNames() {\n        return evacuateCSetPhaseDurations.keySet().stream();\n    }\n\n    UnifiedStatisticalSummary evacuateCSetPhaseDuration(String phaseName) {\n        return evacuateCSetPhaseDurations.get(phaseName);\n    }\n\n    void setPostEvacuateCSetDuration(double duration) {\n        this.youngCollectionPhases[POST_EVACUATE_COLLECTION_SET] = duration;\n    }\n\n    void postPostEvacuateCSetPhaseDuration(String phase, double duration) {\n        postEvacuateCSetPhaseDurations.put(phase, duration);\n    }\n\n    Stream<String> postEvacuateCSetPhaseNames() {\n        return postEvacuateCSetPhaseDurations.keySet().stream();\n    }\n\n    double postEvacuateCSetPhaseDuration(String phaseName) {\n        return postEvacuateCSetPhaseDurations.get(phaseName);\n    }\n\n    void setOtherDuration(double duration) {\n        this.youngCollectionPhases[OTHER] = duration;\n    }\n\n    private UnifiedCountSummary unifiedCountSummary = null;\n\n    void setProcessedBuffersSummary(UnifiedCountSummary summary) {\n        unifiedCountSummary = summary;\n    }\n\n    private UnifiedCountSummary terminationAttempts = null;\n\n    void setTerminationAttempts(UnifiedCountSummary summary) {\n        this.terminationAttempts = summary;\n    }\n\n    //\n    private boolean toSpaceExhausted = false;\n\n    void toSpaceExhausted() {\n        toSpaceExhausted = true;\n    }\n\n    private int evacuationWorkersUsed = 0;\n    private int evacuationWorkersAvailable = 0;\n\n    void evacuationWorkers(int workersUsed, int available) {\n        this.evacuationWorkersUsed = workersUsed;\n        this.evacuationWorkersAvailable = available;\n    }\n\n    // Concurrent Remark values\n    private int concurrentMarkWorkersUsed = 0;\n    private int concurrentMarkWorkersAvailable = 0;\n\n    void concurrentMarkWorkers(int used, int available) {\n        this.concurrentMarkWorkersUsed = used;\n        this.concurrentMarkWorkersAvailable = available;\n    }\n\n    private double markFromRootsDuration = -1.0d;\n    private double precleanDuration = -1.0d;\n\n    void setMarkFromRootsDuration(double duration) {\n        this.markFromRootsDuration = duration;\n    }\n\n    void setPrecleanDuration(double duration) {\n        this.precleanDuration = duration;\n    }\n\n    private boolean aborted = false;\n\n    void abortConcurrentMark() {\n        aborted = true;\n    }\n\n    // builders\n\n    private SurvivorMemoryPoolSummary getSurvivorMemoryPoolSummary() {\n        if (memoryPoolMeasurment[SURVIVOR_OCCUPANCY_BEFORE_COLLECTION] == -1L)\n            return null;\n        return new SurvivorMemoryPoolSummary(memoryPoolMeasurment[SURVIVOR_OCCUPANCY_BEFORE_COLLECTION], memoryPoolMeasurment[SURVIVOR_OCCUPANCY_AFTER_COLLECTION], memoryPoolMeasurment[SURVIVOR_SIZE_AFTER_COLLECTION]);\n    }\n\n    private static final int OCCUPANCY_BEFORE_OFFSET = 0;\n    private static final int OCCUPANCY_AFTER_OFFSET = 1;\n    private static final int SIZE_BEFORE_OFFSET = 2;\n    private static final int SIZE_AFTER_OFFSET = 3;\n\n    private MemoryPoolSummary getMemoryPoolSummary(int offset) {\n        if (memoryPoolMeasurment[offset + OCCUPANCY_BEFORE_OFFSET] == -1L) //do we have recorded values\n            return null;\n        //do we know the size of the memory pool prior to the collection\n        long sizeBeforeCollection = (memoryPoolMeasurment[offset + SIZE_BEFORE_OFFSET] > -1L) ? memoryPoolMeasurment[offset + SIZE_BEFORE_OFFSET] : memoryPoolMeasurment[offset + SIZE_AFTER_OFFSET];\n        return new MemoryPoolSummary(memoryPoolMeasurment[offset + OCCUPANCY_BEFORE_OFFSET], sizeBeforeCollection, memoryPoolMeasurment[offset + OCCUPANCY_AFTER_OFFSET], memoryPoolMeasurment[offset + SIZE_AFTER_OFFSET]);\n    }\n\n    private void fillInMemoryPoolStats(G1GCPauseEvent collection) {\n        //Eden, survivor, and heap\n        MemoryPoolSummary heap = getMemoryPoolSummary(HEAP_OCCUPANCY_BEFORE_COLLECTION);\n        MemoryPoolSummary young = getMemoryPoolSummary(YOUNG_OCCUPANCY_BEFORE_COLLECTION);\n        MemoryPoolSummary eden = getMemoryPoolSummary(EDEN_OCCUPANCY_BEFORE_COLLECTION);\n        SurvivorMemoryPoolSummary survivor = getSurvivorMemoryPoolSummary();\n        MemoryPoolSummary tenured = getMemoryPoolSummary(OLD_OCCUPANCY_BEFORE_COLLECTION);\n        MemoryPoolSummary humongous = getMemoryPoolSummary(HUMONGOUS_OCCUPANCY_BEFORE_COLLECTION);\n        if (heap != null && eden != null && survivor != null) {\n            collection.addMemorySummary(eden, survivor, heap);\n        } else if (eden == null && survivor == null && heap != null) {\n            collection.addMemorySummary(heap);\n        } //else\n        //need to consider other possible combinations.\n    }\n\n    /*\n    private static final int METASPACE_OCCUPANCY_BEFORE_COLLECTION = 24;\n    private static final int METASPACE_OCCUPANCY_AFTER_COLLECTION = 25;\n    private static final int METASPACE_SIZE_BEFORE_COLLECTION = 26;\n    private static final int METASPACE_SIZE_AFTER_COLLECTION = 27;\n    private static final int METASPACE_COMMITTED_BEFORE_COLLECTION = 28;\n    private static final int METASPACE_COMMITTED_AFTER_COLLECTION = 29;\n    private static final int METASPACE_RESERVED_BEFORE_COLLECTION = 30;\n    private static final int METASPACE_RESERVED_AFTER_COLLECTION = 31;\n     */\n    private void fillInMetaspaceStats(G1GCPauseEvent collection) {\n        collection.addPermOrMetaSpaceRecord(getMemoryPoolSummary(METASPACE_OCCUPANCY_BEFORE_COLLECTION));\n    }\n\n\n    enum REGIONS {\n...\n", "needle_token_start": 12148, "needle_token_end": 12199, "code_context_ntokens": 16200, "output": ["```java\nboolean setHumongousOccupancyAfterCollection(long value) {\n    return setMemoryPoolMeasurement(HUMONGOUS_OCCUPANCY_AFTER_COLLECTION, value);\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "hits", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/SafepointParseRule.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To increment the count of successful log entry matches.\n2. **Input**: None directly; it operates on the context of the current instance.\n3. **Output**: None; it updates the internal state.\n4. **Procedure**: Checks if a counter exists for the current instance in a concurrent map. If not, it initializes a new counter. Then, it increments the counter to reflect a new successful match.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\n/**\n * Supported flags\n * -XX:+PrintGCDateStamps\n * -XX:+PrintGCDetails\n * -XX:+PrintTenuringDistribution\n * -XX:+PrintAdaptiveSizePolicy\n * -XX:+PrintReferenceGC\n * -XX:+G1SummarizeRSetStats -XX:+G1SumerizeRsetStatsPeriod=1\n */\n\npublic interface G1GCPatterns extends G1GCTokens {\n\n    String FINALIZE_MARKING = DATE_TIMESTAMP + \"\\\\[Finalize Marking, \" + PAUSE_TIME + \"\\\\]\";\n    String REF_PROC = DATE_TIMESTAMP + \"\\\\[GC ref-proc, \" + PAUSE_TIME + \"\\\\]\";\n    String REF_PROC_DETAILS = DATE_TIMESTAMP + \"\\\\[GC ref-proc\" + REFERENCE_RECORDS + \", \" + PAUSE_TIME + \"\\\\]\";\n    String UNLOADING = DATE_TIMESTAMP + \"\\\\[Unloading, \" + PAUSE_TIME + \"\\\\]\";\n    String TO_SPACE_OVERFLOW = \"( \\\\(to-space (exhausted|overflow)\\\\))?\";\n    String FIXUP_STATS = \"Min: \" + REAL_VALUE + \", Avg: \" + REAL_VALUE + \", Max: \" + REAL_VALUE + \", Diff: \" + REAL_VALUE + \", Sum: \" + REAL_VALUE;\n    String YOUNG_MIXED_INITIAL_MARK_BLOCK = \"\\\\((young|mixed)\\\\)( \\\\((initial-mark)\\\\))?( \\\\(to-space (exhausted|overflow)\\\\))?\\\\s*\";\n\n\n\n    /*  ***********\n     *  Rules with no details\n     */\n\n    //369310.802: [GC pause (young) 485M->240M(512M), 0.0558340 secs]\n    //369447.597: [GC pause (young) (initial-mark) 485M->239M(512M), 0.0719290 secs]\n    //369674.919: [GC pause (mixed) 482M->185M(512M), 0.0679470 secs]\n    //0.583: [GC pause (G1 Evacuation Pause) (young) 24M->4561K(256M), 0.0047007 secs]\n    GCParseRule YOUNG = new GCParseRule(\"YOUNG\", G1GC_PREFIX + YOUNG_MIXED_INITIAL_MARK_BLOCK + BEFORE_AFTER_CONFIGURED_PAUSE);\n\n    //\"16.603: [Full GC (System.gc())  14M->3334K(11M), 0.0230975 secs]\n    GCParseRule SIMPLE_FULL = new GCParseRule(\"SIMPLE_FULL\", G1GC_PREFIX);\n\n    //2014-02-22T10:49:26.487-0100: 7.477: [GC pause (G1 Evacuation Pause) (young) (initial-mark), 0.0042120 secs]\n    //2014-02-22T10:49:26.501-0100: 7.490: [GC pause (G1 Evacuation Pause) (young), 0.0025420 secs]\n    //2014-02-22T10:49:26.508-0100: 7.498: [GC pause (G1 Evacuation Pause) (mixed), 0.0026410 secs]\n    //26.893: [GC pause (G1 Evacuation Pause) (young) (to-space exhausted), 0.1709670 secs]\n    GCParseRule G1_DETAILS = new GCParseRule(\"G1_DETAILS\", G1GC_PREFIX + YOUNG_MIXED_INITIAL_MARK_BLOCK + \", \" + PAUSE_TIME);\n    GCParseRule G1_YOUNG_SPLIT_START = new GCParseRule(\"G1_YOUNG_SPLIT_START\", G1GC_PREFIX + \"(\\\\(young\\\\)|\\\\(mixed\\\\))$\");\n    GCParseRule G1_YOUNG_SPLIT_END = new GCParseRule(\"G1_YOUNG_SPLIT_END\", \"^\" + \"(\\\\(to-space (exhausted|overflow)\\\\))?, \" + PAUSE_TIME + \"\\\\]\");\n\n    GCParseRule G1_INITIAL_MARK = new GCParseRule(\"G1_INITIAL_MARK\", G1GC_PREFIX + \"\\\\((young|mixed)\\\\) \\\\(initial-mark\\\\)$\");\n\n    //****** Records for PrintReferenceGC\n    //2015-09-10T08:07:15.806+0200: 0.470: [SoftReference, 0 refs, 0.0005710 secs]2015-09-10T08:07:15.806+0200: 0.471: [WeakReference, 4 refs, 0.0002995 secs]2015-09-10T08:07:15.806+0200: 0.471: [FinalReference, 846 refs, 0.0011795 secs]2015-09-10T08:07:15.808+0200: 0.472: [PhantomReference, 0 refs, 0 refs, 0.0005331 secs]2015-09-10T08:07:15.808+0200: 0.473: [JNI Weak Reference, 0.0000145 secs], 0.0055187 secs]\n    //13925.724: [SoftReference, 0 refs, 0.0000060 secs]13925.724: [WeakReference, 367 refs, 0.0001090 secs]13925.724: [FinalReference, 3077 refs, 0.0116620 secs]13925.736: [PhantomReference, 8 refs, 0.0000040 secs]13925.736: [JNI Weak Reference, 0.0001390 secs] (to-space overflow), 4.63695400 secs]\n    GCParseRule FREE_FLOATING_REFERENCE_RECORDS = new GCParseRule(\"FREE_FLOATING_REFERENCE_RECORDS\", \"^\" + REFERENCE_RECORDS + \"( \\\\(to-space (exhausted|overflow)\\\\))?, \" + PAUSE_TIME + \"\\\\]\");\n    //Missed: 2015-09-10T10:47:58.284+0200: 9642.949: [SoftReference, 0 refs, 0.0004200 secs]2015-09-10T10:47:58.284+0200: 9642.949: [WeakReference, 2 refs, 0.0002457 secs]2015-09-10T10:47:58.285+0200: 9642.949: [FinalReference, 1245 refs, 0.0004931 secs]2015-09-10T10:47:58.285+0200: 9642.950: [PhantomReference, 0 refs, 0 refs, 0.0004229 secs]2015-09-10T10:47:58.286+0200: 9642.950: [JNI Weak Reference, 0.0000150 secs] 9642.951: [G1Ergonomics (Concurrent Cycles) do not request concurrent cycle initiation, reason: still doing mixed collections, occupancy: 805306368 bytes, allocation request: 0 bytes, threshold: 805306350 bytes (75.00 %), source: end of GC]\n    GCParseRule FLOATING_REFERENCE_WITH_ADAPTIVE_SIZING = new GCParseRule(\"FLOATING_REFERENCE_WITH_ADAPTIVE_SIZING\", REFERENCE_RECORDS + \" \\\\d+\\\\.\\\\d{3}: \\\\[G1Ergonomics\");\n\n    //998.590: [GC pause (mixed)12038.693: [SoftReference, 0 refs, 0.0000070 secs]12038.693: [WeakReference, 3 refs, 0.0000060 secs]12038.693: [FinalReference, 0 refs, 0.0000010 secs]12038.693: [PhantomReference, 1 refs, 0.0000020 secs]12038.693: [JNI Weak Reference, 0.0002030 secs] (to-space overflow), 41.52888100 secs]\n    //300.634: [GC pause (young) (initial-mark)300.742: [SoftReference, 0 refs, 0.0000230 secs]300.742: [WeakReference, 0 refs, 0.0000030 secs]300.742: [FinalReference, 0 refs, 0.0000030 secs]300.742: [PhantomReference, 0 refs, 0.0000020 secs]300.742: [JNI Weak Reference, 0.0000020 secs] (to-space exhausted), 0.1146540 secs]\n    GCParseRule G1_DETAILS_REFERENCE_GC = new GCParseRule(\"G1_DETAILS_REFERENCE_GC\", G1GC_PREFIX + \"\\\\((young||mixed)\\\\)( \\\\(initial-mark\\\\))?\" + REFERENCE_RECORDS + TO_SPACE_OVERFLOW + \", \" + PAUSE_TIME + \"\\\\]\");\n    //2017-10-26T20:05:18.850+0000: 1.786: [SoftReference, 0 refs, 0.0022943 secs]2017-10-26T20:05:18.852+0000: 1.789: [WeakReference, 8 refs, 0.0026570 secs]2017-10-26T20:05:18.855+0000: 1.791: [FinalReference, 961 refs, 0.0033445 secs]2017-10-26T20:05:18.858+0000: 1.795: [PhantomReference, 0 refs, 0 refs, 0.0048559 secs]2017-10-26T20:05:18.863+0000: 1.799: [JNI Weak Reference, 0.0000274 secs] (plab_sz = 17978  desired_plab_sz = 17978)  (plab_sz = 0  desired_plab_sz = 259) , 0.0241882 secs]\n    GCParseRule G1_FLOATING_REFERENCE_PLAB = new GCParseRule(\"G1_FLOATING_REFERENCE_PLAB\", REFERENCE_RECORDS + \" \" + PLAB_RECORD);\n\n    //2014-10-21T11:47:46.879-0500: 11971.476: [GC pause (young)11972.105: [SoftReference, 0 refs, 0.0000050 secs]11972.105: [WeakReference, 1 refs, 0.0000020 secs]11972.105: [FinalReference, 94 refs, 0.0000360 secs]11972.105: [PhantomReference, 5 refs, 0.0000030 secs]11972.105: [JNI Weak Reference, 0.0002010 secs] (initial-mark), 0.63448400 secs]\n    GCParseRule G1_DETAILS_REFERENCE_INITIAL_MARK = new GCParseRule(\"G1_DETAILS_REFERENCE_INITIAL_MARK\", G1GC_PREFIX + \"\\\\(young\\\\)\" + REFERENCE_RECORDS + TO_SPACE_OVERFLOW + \" \\\\(initial-mark\\\\), \" + PAUSE_TIME + \"\\\\]\");\n\n    //2014-10-21T11:47:52.016-0500: 11976.613: [GC remark 11976.615: [GC ref-proc11976.615: [SoftReference, 834 refs, 0.0002100 secs]11976.615: [WeakReference, 14842 refs, 0.0021790 secs]11976.617: [FinalReference, 181 refs, 0.0004430 secs]11976.618: [PhantomReference, 475 refs, 0.0001160 secs]11976.618: [JNI Weak Reference, 0.0002910 secs], 0.0035310 secs], 0.0565900 secs]\n    //46.465: [GC remark 46.465: [GC ref-proc46.465: [SoftReference, 0 refs, 0.0000180 secs]46.465: [WeakReference, 15 refs, 0.0000240 secs]46.465: [FinalReference, 0 refs, 0.0000110 secs]46.465: [PhantomReference, 0 refs, 0.0000110 secs]46.465: [JNI Weak Reference, 0.0000050 secs], 0.0000960 secs], 0.0012290 secs]\n    GCParseRule G1_REMARK_REFERENCE_GC = new GCParseRule(\"G1_REMARK_REFERENCE_GC\", DATE_TIMESTAMP + \"\\\\[GC remark \" + DATE_TIMESTAMP + \"\\\\[GC ref-proc\" + REFERENCE_RECORDS + \", \" + PAUSE_TIME);\n\n    //0.844: [Full GC0.855: [SoftReference, 0 refs, 0.0000230 secs]0.855: [WeakReference, 183 refs, 0.0000090 secs]0.855: [FinalReference, 1110 refs, 0.0002100 secs]0.855: [PhantomReference, 1 refs, 0.0000030 secs]0.855: [JNI Weak Reference, 0.0000090 secs] 17M->3756K(13M), 0.0236280 secs]\n    //2014-10-21T11:50:54.618-0500: 12159.216: [Full GC12160.052: [SoftReference, 1453 refs, 0.0002060 secs]12160.052: [WeakReference, 24676 refs, 0.0020110 secs]12160.054: [FinalReference, 751 refs, 0.0002570 secs]12160.054: [PhantomReference, 514 refs, 0.0000650 secs]12160.054: [JNI Weak Reference, 0.0001910 secs] 11414M->1104M(12000M), 4.6872730 secs]\n    GCParseRule G1_FULL_DETAILS_REFERENCE_GC = new GCParseRule(\"G1_FULL_DETAILS_REFERENCE_GC\", FULL_GC_PREFIX + REFERENCE_RECORDS + \" \" + BEFORE_AFTER_CONFIGURED_PAUSE);\n\n    //2014-10-22T07:40:16.844-0500: 83521.441: [Full GC83522.569: [SoftReference, 1653 refs, 0.0003300 secs]83522.569: [WeakReference, 36226 refs, 0.0042550 secs]83522.573: [FinalReference, 1826 refs, 0.0006410 secs]83522.574: [PhantomReference, 560 refs, 0.0000630 secs]83522.574: [JNI Weak Reference, 0.0001650 secs]2014-10-22T07:40:21.212-0500: 83525.810: [GC concurrent-mark-end, 2.9456090 sec]\n    GCParseRule G1_FULL_INTERRUPTS_CONCURRENT_WITH_REFERENCES = new GCParseRule(\"G1_FULL_INTERRUPTS_CONCURRENT_WITH_REFERENCES\", FULL_GC_PREFIX + REFERENCE_RECORDS + GC_PREFIX + \"concurrent-(.+)-end, \" + PAUSE_TIME);\n    // 11593M->1413M(12000M), 5.6594060 secs]\n    GCParseRule G1_FULL_MEMORY_SPLIT_BY_CONCURRENT = new GCParseRule(\"G1_FULL_MEMORY_SPLIT_BY_CONCURRENT\", \"^\" + BEFORE_AFTER_CONFIGURED_PAUSE);\n\n    //2015-04-09T14:28:44.235+0100: 6.597: [GC remark 6.597: [Finalize Marking, 0.0091510 secs] 6.606: [GC ref-proc, 0.0014102 secs] 6.608: [Unloading, 0.0044869 secs], 0.0153351 secs]\n    GCParseRule G1_180_REMARK = new GCParseRule(\"G1_180_REMARK\", DATE_TIMESTAMP + \"\\\\[GC remark \" + FINALIZE_MARKING + \" \" + REF_PROC + \" \" + UNLOADING + \", \" + PAUSE_TIME + \"\\\\]\");\n    //2015-12-21T10:26:34.913-0500: 35835.169: [GC remark 35835.169: [Finalize Marking, 0.8985905 secs] 35836.068: [GC ref-proc35836.068: [SoftReference, 74 refs, 0.0002156 secs]35836.068: [WeakReference, 243 refs, 0.0001747 secs]35836.068: [FinalReference, 4154 refs, 0.0012360 secs]35836.070: [PhantomReference, 0 refs, 0 refs, 0.0002882 secs]35836.070: [JNI Weak Reference, 0.0000227 secs], 0.0026411 secs] 35836.071: [Unloading, 0.0048615 secs], 0.9142368 secs]\n    GCParseRule G1_180_REMARK_REF_DETAILS = new GCParseRule(\"G1_180_REMARK_REF_DETAILS\", DATE_TIMESTAMP + \"\\\\[GC remark \" + FINALIZE_MARKING + \" \" + REF_PROC_DETAILS + \" \" + UNLOADING + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    GCParseRule PROCESSED_BUFFERS = new GCParseRule(\"PROCESSED_BUFFERS\", \"\\\\[Processed Buffers: \" + G1_PHASE_COUNTER_SUMMARY + \"\\\\]\");\n    GCParseRule PROCESSED_BUFFER = new GCParseRule(\"PROCESSED_BUFFER\", \"\\\\[Processed Buffers:\\\\s*\" + COUNTER + \"\\\\]\");\n\n    /*\n     \"[GC Worker Start (ms):  6394.7]\",\n                \"[GC Worker End (ms):  3812.6]\"\n     */\n    String WORKER_SUMMARY = \"(GC Worker Start|GC Worker End)\";\n    GCParseRule WORKER_PARALLEL_BLOCK = new GCParseRule(\"WORKER_PARALLEL_BLOCK\", \"\\\\[\" + WORKER_SUMMARY + \" \\\\(ms\\\\):\\\\s*\" + G1_PHASE_TIME_SUMMARY + \"\\\\]\");\n    GCParseRule SOLARIS_WORKER_PARALLEL_BLOCK = new GCParseRule(\"SOLARIS_WORKER_PARALLEL_BLOCK\", \"\\\\[\" + WORKER_SUMMARY + \" \\\\(ms\\\\):\\\\s*\" + TIME + \"\\\\]\");\n    GCParseRule WORKER_ACTIVITY = new GCParseRule(\"WORKER_ACTIVITY\", \"\\\\[(GC Worker Other|GC Worker Total) \\\\(ms\\\\): \" + G1_PHASE_TIME_SUMMARY_SUM + \"\\\\]\");\n    GCParseRule SOLARIS_WORKER_PARALLEL_ACTIVITY = new GCParseRule(\"SOLARIS_WORKER_PARALLEL_ACTIVITY\", \"\\\\[(GC Worker Other|GC Worker Total) \\\\(ms\\\\):\\\\s*\" + TIME + \"\\\\]\");\n\n    //2015-04-09T14:28:38.515+0100: 0.876: [GC concurrent-string-deduplication, 112.0B->112.0B(0.0B), avg 0.0%, 0.0000058 secs]\n    //2015-10-17T21:10:23.673-0400: 3.137: [GC concurrent-string-deduplication, 79.3K->792.0B(78.6K), avg 96.2%, 0.0024351 secs]\n    //\"2015-04-09T14:28:38.515+0100: 0.876: [GC concurrent-string-deduplication, 112.0B->112.0B(0.0B), avg 0.0%, 0.0000058 secs]\",\n    //2015-12-21T10:26:33.282-0500: 35833.538: [GC concurrent-string-deduplication, 113.9K->1152.0B(112.8K), avg 90.9%, 0.0022797 secs]\n    GCParseRule CONCURRENT_STRING_DEDUP = new GCParseRule(\"CONCURRENT_STRING_DEDUP\", GC_PREFIX + \"concurrent-string-deduplication, \" + FRACTIONAL_BEFORE_AFTER_CONFIGURED + \", avg \" + PERCENTAGE + \", \" + PAUSE_TIME);\n\n    //[String Dedup Fixup: 0.1 ms, GC Workers: 4]\n    GCParseRule STRING_DEDUP_FIXUP = new GCParseRule(\"STRING_DEDUP_FIXUP\", \"^\\\\[String Dedup Fixup: \" + PAUSE_TIME + \", GC Workers: \" + COUNTER + \"\\\\]\");\n    //[Queue Fixup (ms): Min: 0.0, Avg: 0.0, Max: 0.0, Diff: 0.0, Sum: 0.0]\n    GCParseRule QUEUE_FIXUP = new GCParseRule(\"QUEUE_FIXUP\", \"^\\\\[Queue Fixup \\\\(ms\\\\): \" + FIXUP_STATS + \"\\\\]\");\n    //[Table Fixup (ms): Min: 0.0, Avg: 0.0, Max: 0.1, Diff: 0.1, Sum: 0.1]\n    GCParseRule TABLE_FIXUP = new GCParseRule(\"TABLE_FIXUP\", \"^\\\\[Table Fixup \\\\(ms\\\\): \" + FIXUP_STATS + \"\\\\]\");\n\n    //[Parallel Time: 32.7 ms, GC Workers: 1]\n    GCParseRule PARALLEL_TIME = new GCParseRule(\"PARALLEL_TIME\", \"\\\\[Parallel Time: \" + TIME + \" ms, GC Workers: (\" + INTEGER + \")\\\\]\");\n    String PARALLEL_ACTIVITY = \"(Ext Root Scanning|Code Root Marking|SATB Filtering|Update RS|Scan RS|Code Root Scanning|Object Copy|Termination)\";\n    GCParseRule TERMINATION_ATTEMPTS = new GCParseRule(\"TERMINATION_ATTEMPTS\", \"\\\\[Termination Attempts: \" + G1_PHASE_COUNTER_SUMMARY + \"\\\\]\");\n    GCParseRule G1_PARALLEL_PHASE_SUMMARY = new GCParseRule(\"G1_PARALLEL_PHASE_SUMMARY\", \"\\\\[\" + PARALLEL_ACTIVITY + \" \\\\(ms\\\\): \" + G1_PHASE_TIME_SUMMARY_SUM + \"\\\\]\");\n    GCParseRule G1_SOLARIS_PARALLEL_PHASE = new GCParseRule(\"G1_SOLARIS_PARALLEL_PHASE\", \"\\\\[\" + PARALLEL_ACTIVITY + \" \\\\(ms\\\\):\\\\s*\" + REAL_VALUE + \"\\\\]\");\n\n    GCParseRule G1GC_PHASE = new GCParseRule(\"G1GC_PHASE\", \"\\\\[(Code Root Fixup|Code Root Migration|Code Root Purge|Clear CT|Expand Heap|Other):\\\\s+\" + TIME);\n\n    String G1GC_PHASE_DETAIL_KEYS = \"(Root Region Scan Waiting|Evacuation Failure|Choose CSet|Ref Proc|Ref Enq|Free CSet|Redirty Cards|Humongous Reclaim|Humongous Register)\";\n    GCParseRule G1GC_PHASE_DETAIL_CLAUSE = new GCParseRule(\"G1GC_PHASE_DETAIL_CLAUSE\", \"\\\\[\" + G1GC_PHASE_DETAIL_KEYS + \":\\\\s+\" + TIME);\n\n    //[Eden: 512.0M(512.0M)->0.0B(505.0M) Survivors: 0.0B->7168.0K Heap: 512.0M(518.0M)->6418.4K(519.0M)]\n    //[Eden: 89.0M(89.0M)->0.0B(89.0M) Survivors: 13.0M->13.0M Heap: 103.7M(2048.0M)->30.5M(2048.0M)]\n    //[Eden: 128.0M(512.0M)->0.0B(512.0M) Survivors: 0.0B->0.0B Heap: 147.8M(518.0M)->23.1M(518.0M)], [Metaspace: 35052K->35051K(1079296K)]\n    GCParseRule G1_MEMORY_SUMMARY = new GCParseRule(\"G1_MEMORY_SUMMARY\", \"\\\\[Eden: \" + G1_FROM_TO + \" Survivors: \" + G1_SURVIVOR_FROM_TO + \" Heap: \" + G1_FROM_TO + \"\\\\](, \\\\[\" + META_PERM_MEMORY_SUMMARY + \"\\\\])?\");\n    //Tenuring with no details -> 23M->3973K(472M), 0.0047248 secs]\n    GCParseRule G1_NO_DETAILS_MEMORY_SUMMARY = new GCParseRule(\"G1_NO_DETAILS_MEMORY_SUMMARY\", \"^\" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]$\");\n    GCParseRule G1_PRE17040_SUMMARY = new GCParseRule(\"G1_PRE17040_SUMMARY\", \"\\\\[Eden: \" + PRE17040_G1_FROM_TO + \" Survivors: \" + PRE17040_G1_SURVIVOR_FROM_TO + \" Heap: \" + PRE17040_G1_FROM_TO + \"\\\\]\");\n\n    //113.325: [Full GC (System.gc()) 37M->32M(96M), 0.0943030 secs]\n    //2015-08-31T09:23:16.635+0100: 678431.946: [Full GC (Allocation Failure)  17G->7270M(18G), 15.5646820 secs]\n    GCParseRule FULL_GC = new GCParseRule(\"FULL_GC\", FULL_GC_PREFIX + \"\\\\s+\" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]$\");\n\n    //112.230: [Full GC (System.gc()) 112.327: [G1Ergonomics (Heap Sizing) attempt heap shrinking, reason: capacity higher than max desired capacity after Full GC, capacity: 128974848 bytes, occupancy: 29902320 bytes, max desired capacity: 99674399 bytes (70.00 %)]\n    GCParseRule G1_FULL_ADAPTIVE_SIZING = new GCParseRule(\"G1_FULL_ADAPTIVE_SIZING\", FULL_GC_PREFIX + TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Heap Sizing\\\\) attempt heap shrinking, reason: capacity higher than max desired capacity after Full GC, capacity: \" + COUNTER + \" bytes, occupancy: \" + COUNTER + \" bytes, max desired capacity: \" + COUNTER + \" bytes \\\\(\" + REAL_VALUE + \" %\\\\)\\\\]\");\n\n    //17M->3758K(13M), 0.0235300 secs]\n    GCParseRule FREE_FLOATING_OCCUPANCY_SUMMARY = new GCParseRule(\"FREE_FLOATING_OCCUPANCY_SUMMARY\", \"^\" + BEFORE_AFTER_CONFIGURED_PAUSE + \"\\\\]\");\n\n    //Adaptive sizing records\n    //18.298: [GC pause (G1 Evacuation Pause) (young) 18.298: [G1Ergonomics (CSet Construction) start choosing CSet, _pending_cards: 0, predicted base time: 10.00 ms, remaining time: 190.00 ms, target pause time: 200.00 ms]\n    GCParseRule G1_YOUNG_WITH_CSET_CONSTRUCTION_START = new GCParseRule(\"G1_YOUNG_WITH_CSET_CONSTRUCTION_START\", G1GC_PREFIX + \"\\\\(young\\\\) \" +\n            TIMESTAMP + \"\\\\[G1Ergonomics \\\\(CSet Construction\\\\) start choosing CSet, _pending_cards: \" + COUNTER + \", predicted base time: \" + MS_TIME_STAMP + \", remaining time: \" + MS_TIME_STAMP + \", target pause time: \" + MS_TIME_STAMP + \"\\\\]\");\n\n    // 7.793: [G1Ergonomics (CSet Construction) start choosing CSet, _pending_cards: 1395, predicted base time: 14.19 ms, remaining time: 185.81 ms, target pause time: 200.00 ms]\n    GCParseRule G1_CSET_CONSTRUCTION_START = new GCParseRule(\"G1_CSET_CONSTRUCTION_START\", \"^\" + TIMESTAMP + \"\\\\[G1Ergonomics \\\\(CSet Construction\\\\) start choosing CSet, _pending_cards: \" + COUNTER + \", predicted base time: \" + MS_TIME_STAMP);// + \", remaining time: \" + MS_TIME_STAMP + \", target pause time: \" + MS_TIME_STAMP + \"\\\\]\");\n\n    //18.298: [G1Ergonomics (CSet Construction) add young regions to CSet, eden: 512 regions, survivors: 0 regions, predicted young region time: 7833.33 ms]\n    GCParseRule CSET_CONSTRUCTION = new GCParseRule(\"CSET_CONSTRUCTION\", TIMESTAMP + \"\\\\[G1Ergonomics \\\\(CSet Construction\\\\) add young regions to CSet, eden: \" + COUNTER + \" regions, survivors: \" + COUNTER + \" regions, predicted young region time: \" + MS_TIME_STAMP + \"\\\\]\");\n\n    //18.298: [G1Ergonomics (CSet Construction) finish choosing CSet, eden: 512 regions, survivors: 0 regions, old: 0 regions, predicted pause time: 7843.33 ms, target pause time: 200.00 ms]\n    GCParseRule CSET_CONSTRUCTION_END = new GCParseRule(\"CSET_CONSTRUCTION_END\", TIMESTAMP + \"\\\\[G1Ergonomics \\\\(CSet Construction\\\\) finish choosing CSet, eden: \" + COUNTER + \" regions, survivors: \" + COUNTER + \" regions, old: \" + COUNTER + \" regions, predicted pause time: \" + MS_TIME_STAMP + \", target pause time: \" + MS_TIME_STAMP + \"\\\\]\");\n\n\n    //0.772: [G1Ergonomics (Heap Sizing) shrink the heap, requested shrinking amount: 255605857 bytes, aligned shrinking amount: 254803968 bytes, attempted shrinking amount: 254803968 bytes]\n    GCParseRule HEAP_SHRINK = new GCParseRule(\"HEAP_SHRINK\", TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Heap Sizing\\\\) shrink the heap, requested shrinking amount: \" + COUNTER + \" bytes, aligned shrinking amount: \" + COUNTER + \" bytes, attempted shrinking amount: \" + COUNTER + \" bytes\\\\]\");\n\n    //18.303: [G1Ergonomics (Heap Sizing) expand the heap, requested expansion amount: 845600000 bytes, attempted expansion amount: 1048576 bytes]\n    GCParseRule HEAP_EXPAND = new GCParseRule(\"HEAP_EXPAND\", TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Heap Sizing\\\\) expand the heap, requested expansion amount: \" + COUNTER + \" bytes, attempted expansion amount: \" + COUNTER + \" bytes\\\\]\");\n\n    //18.303: [G1Ergonomics (Heap Sizing) attempt heap expansion, reason: region allocation request failed, allocation request: 8456 bytes]\n    GCParseRule ATTEMPT_HEAP_EXPANSION_ALLOC_FAILURE = new GCParseRule(\"ATTEMPT_HEAP_EXPANSION_ALLOC_FAILURE\", TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Concurrent Cyclesing\\\\) attempt heap expansion, reason: region allocation request failed, allocation request: \" + COUNTER + \" bytes\\\\]\");\n\n    //7.942: [G1Ergonomics (Heap Sizing) attempt heap expansion, reason: recent GC overhead higher than threshold after GC, recent GC overhead: 22.02 %, threshold: 10.00 %, uncommitted: 1060110336 bytes, calculated expansion amount: 212022067 bytes (20.00 %)]\n    GCParseRule ATTEMPT_HEAP_EXPANSION_OVERHEAD = new GCParseRule(\"ATTEMPT_HEAP_EXPANSION_OVERHEAD\", TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Heap Sizing\\\\) attempt heap expansion, reason: recent GC overhead higher than threshold after GC, recent GC overhead: \" + PERCENTAGE + \", threshold: \" + PERCENTAGE + \", uncommitted: \" + COUNTER + \" bytes, calculated expansion amount: \" + COUNTER + \" bytes \\\\(\" + PERCENTAGE + \"\\\\)\\\\]\");\n\n    //0.785: [G1Ergonomics (Heap Sizing) did not shrink the heap, reason: heap shrinking operation failed]\n    GCParseRule HEAP_SHRINKING_FAILED = new GCParseRule(\"HEAP_SHRINKING_FAILED\", DATE_TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Heap Sizing\\\\) did not shrink the heap, reason: heap shrinking operation failed\\\\]\");\n\n    //7.835: [G1Ergonomics (Concurrent Cycles) request concurrent cycle initiation, reason: occupancy higher than threshold, occupancy: 7340032 bytes, allocation request: 0 bytes, threshold: 6134130 bytes (45.00 %), source: end of GC]\n    GCParseRule HIGH_OCCUPANCY_TRIGGERS_CONC = new GCParseRule(\"HIGH_OCCUPANCY_TRIGGERS_CONC\", DATE_TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Concurrent Cycles\\\\) request concurrent cycle initiation, reason: occupancy higher than threshold, occupancy: \" + COUNTER + \" bytes, allocation request: \" + COUNTER + \" bytes, threshold: \" + COUNTER + \" bytes \\\\(\" + PERCENTAGE + \"\\\\), source: end of GC\\\\]\");\n\n    //Generalize rule to catch all Concurrent Cycle messages\n    //7.842: [G1Ergonomics (Concurrent Cycles) initiate concurrent cycle, reason: concurrent cycle initiation requested]\n    GCParseRule INITIATE_CONC_CYCLE = new GCParseRule(\"INITIATE_CONC_CYCLE\", DATE_TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Concurrent Cycles\\\\)\");\n\n    //7.916: [G1Ergonomics (Concurrent Cycles) do not request concurrent cycle initiation, reason: still doing mixed collections, occupancy: 9437184 bytes, allocation request: 0 bytes, threshold: 6134130 bytes (45.00 %), source: end of GC]\n    //7.916: [G1Ergonomics (Concurrent Cycles) do not request concurrent cycle initiation, reason: still doing mixed collections, occupancy: 9437184 bytes, allocation request: 0 bytes, threshold: 6134130 bytes (45.00 %), source: end of GC]\n    GCParseRule DO_NOT_REQUEST_CONC_CYCLE = new GCParseRule(\"DO_NOT_REQUEST_CONC_CYCLE\", DATE_TIMESTAMP + \"\\\\[G1Ergonomics \\\\(Concurrent Cycles\\\\) do not request concurrent cycle initiation, reason: still doing mixed collections, occupancy: \" + COUNTER + \" bytes, allocation request: \" + COUNTER + \" bytes, threshold: \" + COUNTER + \" bytes \\\\(\" + PERCENTAGE + \"\\\\), source: end of GC\\\\]\");\n...\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/SharedPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\n/**\n * TODO: #157 There needs to be a clearer separation of what is a Token vs what is a SharedPattern\n * <p>\n * Shared generational rules\n */\npublic interface SharedPatterns extends PreUnifiedTokens {\n\n    String WEAK_REF_BLOCK = DATE_TIMESTAMP + \"\\\\[weak refs processing, \" + PAUSE_TIME + \"\\\\]\";\n    String CLASS_UNLOADING_BLOCK = DATE_TIMESTAMP + \"\\\\[class unloading, \" + PAUSE_TIME + \"\\\\]\";\n    String STRING_TABLE_SCRUB_BLOCK = DATE_TIMESTAMP + \"\\\\[scrub string table, \" + PAUSE_TIME + \"\\\\]\";\n    String SYMBOL_TABLE_SCRUB_BLOCK = DATE_TIMESTAMP + \"\\\\[scrub symbol table, \" + PAUSE_TIME + \"\\\\]\";\n    String STRING_AND_SYMBOL_SCRUB_BLOCK = DATE_TIMESTAMP + \"\\\\[scrub symbol & string tables, \" + PAUSE_TIME + \"\\\\]\";\n\n    GCParseRule OCCUPANCY_CONFIGURED_RULE = new GCParseRule(\"OCCUPANCY_CONFIGURED_RULE\", OCCUPANCY_CONFIGURED);\n    GCParseRule MEMORY_SUMMARY_RULE = new GCParseRule(\"MEMORY_SUMMARY_RULE\", BEFORE_AFTER_CONFIGURED);\n    GCParseRule BEFORE_AFTER_CONFIGURED_PAUSE_RULE = new GCParseRule(\"BEFORE_AFTER_CONFIGURED_PAUSE_RULE\", BEFORE_AFTER_CONFIGURED_PAUSE);\n    GCParseRule WEAK_REF = new GCParseRule(\"WEAK_REF\", WEAK_REF_BLOCK);\n    GCParseRule CLASS_UNLOADING = new GCParseRule(\"CLASS_UNLOADING\", CLASS_UNLOADING_BLOCK);\n    GCParseRule STRING_TABLE_SCRUB = new GCParseRule(\"STRING_TABLE_SCRUB\", STRING_TABLE_SCRUB_BLOCK);\n    GCParseRule SYMBOL_TABLE_SCRUB = new GCParseRule(\"SYMBOL_TABLE_SCRUB\", SYMBOL_TABLE_SCRUB_BLOCK);\n    GCParseRule STRING_AND_SYMBOL_SCRUB = new GCParseRule(\"STRING_AND_SYMBOL_SCRUB\", STRING_AND_SYMBOL_SCRUB_BLOCK);\n\n    // Tenuring Details\n    //: 17337K->347K(18624K), 0.0007142 secs] 23862K->6872K(81280K), 0.0008003 secs]\n    GCParseRule TENURING_DETAILS = new GCParseRule(\"TENURING_DETAILS\", \"^: \" + BEFORE_AFTER_CONFIGURED + \", \" + PAUSE_TIME + \"\\\\] \" + BEFORE_AFTER_CONFIGURED + \", \" + PAUSE_TIME + \"\\\\]\");\n\n    // Perm space (pre 1.8.0) record\n    GCParseRule PERM_SPACE_RECORD = new GCParseRule(\"PERM_SPACE_RECORD\", PERM_RECORD);\n\n    // Metaspace record (1.8.0+)\n    GCParseRule META_SPACE_RECORD = new GCParseRule(\"META_SPACE_RECORD\", META_RECORD);\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/vmops/SafepointTrace.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.vmops;\n\n\nimport com.microsoft.gctoolkit.event.jvm.Safepoint;\nimport com.microsoft.gctoolkit.parser.AbstractLogTrace;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.regex.Matcher;\n\npublic class SafepointTrace extends AbstractLogTrace {\n\n    public SafepointTrace(Matcher matcher) {\n        super(matcher);\n    }\n\n    public Safepoint toSafepoint() {\n        Safepoint safepoint = new Safepoint(getVMOP(), getDateTimeStamp(), getDuration());\n        safepoint.recordThreadCounts(totalThreads(), initiallyRunningThreads(), waitingToBlockThreads());\n        safepoint.recordDurations(spinTime(), blockTime(), syncTime(), cleanupTime(), vmopTime());\n        safepoint.recordPageTrapCount(getTrapCount());\n        return safepoint;\n    }\n\n    public String getVMOP() {\n        return super.getGroup(VMOP);\n    }\n\n    public DateTimeStamp getDateTimeStamp() {\n        return new DateTimeStamp(getTimeStampGroup());\n    }\n\n    public double getDuration() {\n        return (spinTime() + blockTime() + syncTime() + cleanupTime() + vmopTime()) / 1000.0d;\n    }\n\n    public int totalThreads() {\n        return getIntegerGroup(TOTAL_THREADS);\n    }\n\n    public int initiallyRunningThreads() {\n        return getIntegerGroup(INITIALLY_RUNNING_THREADS);\n    }\n\n    public int waitingToBlockThreads() {\n        return getIntegerGroup(WAITING_TO_BLOCK);\n    }\n\n    public int spinTime() {\n        return getIntegerGroup(SPIN_TIME);\n    }\n\n    public int blockTime() {\n        return getIntegerGroup(BLOCK_TIME);\n    }\n\n    public int syncTime() {\n        return getIntegerGroup(SYNC_TIME);\n    }\n\n    public int cleanupTime() {\n        return getIntegerGroup(CLEANUP_TIME);\n    }\n\n    public int vmopTime() {\n        return getIntegerGroup(VMOP_TIME);\n    }\n\n    public int getTrapCount() {\n        return getIntegerGroup(TRAP_COUNT);\n    }\n\n    private double getTimeStampGroup() {\n        return getDoubleGroup(TIME_STAMP);\n    }\n\n    private final int TIME_STAMP = 1;\n    private final int VMOP = 2;\n    private final int TOTAL_THREADS = 3;\n    private final int INITIALLY_RUNNING_THREADS = 4;\n    private final int WAITING_TO_BLOCK = 5;\n    private final int SPIN_TIME = 6;\n    private final int BLOCK_TIME = 7;\n    private final int SYNC_TIME = 8;\n    private final int CLEANUP_TIME = 9;\n    private final int VMOP_TIME = 10;\n    private final int TRAP_COUNT = 11;\n\n}\n\n/*\n         vmop                    [threads: total initially_running wait_to_block]    [time: spin block sync cleanup vmop] page_trap_count\n0.099: Deoptimize                       [       9          0              0    ]      [     0     0     0     0     0    ]  0\n1.474: no vm operation                  [      13          0              0    ]      [     0     0     0     0     0    ]  0\n4.088: EnableBiasedLocking              [      13          0              0    ]      [     0     0     0     0     0    ]  0\n19.135: GenCollectForAllocation          [      14          0              0    ]      [     0     0     0     0     5    ]  0\n19.142: RevokeBias                       [      14          0              1    ]      [     0     0     0     0     0    ]  0\n36.464: BulkRevokeBias                   [      20          1              1    ]      [     0     0     0     0     0    ]  1\n37.557: ThreadDump                       [      22          0              1    ]      [     0     0     0     0     0    ]  0\n37.566: FindDeadlocks                    [      22          0              0    ]      [     0     0     0     0     0    ]  0\nPolling page always armed\nThreadDump                       159\nFindDeadlocks                     16\nDeoptimize                        16\nGenCollectForAllocation          202\nEnableBiasedLocking                1\nRevokeBias                        87\nBulkRevokeBias                     7\nExit                               1\n    0 VM operations coalesced during safepoint\nMaximum sync time      0 ms\n */\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/SafepointParseRule.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.parser.vmops.SafepointTrace;\n\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n/**\n * Class that tracks whether a log entry was parsed successfully (hit), or not\n * (miss) and captures the origin of that hit or miss.\n */\npublic class SafepointParseRule {\n\n    private static final ConcurrentMap<SafepointParseRule, AtomicInteger> hits = new ConcurrentHashMap<>();\n    private static final ConcurrentMap<SafepointParseRule, AtomicInteger> misses = new ConcurrentHashMap<>();\n    private static final ConcurrentMap<SafepointParseRule, Throwable> origin = new ConcurrentHashMap<>();\n\n    private final Pattern pattern;\n\n    public SafepointParseRule(String pattern) {\n        this.pattern = Pattern.compile(pattern);\n        Throwable throwable = new Throwable();\n        throwable = throwable.fillInStackTrace();\n        origin.put(this, throwable);\n    }\n\n    /**\n     * TODO #155 This painful pattern of returning a null which gets checked by\n     * the caller could be replaced by use of Optional\n     *\n     * @param trace The trace to match against the pattern\n     * @return A trace with a valid matcher or null\n     */\n    public SafepointTrace parse(String trace) {\n        Matcher matcher = pattern.matcher(trace);\n        if (matcher.find()) {\n            hits();\n            return new SafepointTrace(matcher);\n        } else {\n            misses();\n            return null;\n        }\n    }\n\n    \nprivate void hits() {\n        AtomicInteger count = hits.get(this);\n        if (count == null) {\n            count = new AtomicInteger(0);\n            hits.put(this, count);\n        }\n        count.getAndIncrement();\n    }\n\n    private void misses() {\n        AtomicInteger count = misses.get(this);\n        if (count == null) {\n            count = new AtomicInteger(0);\n            misses.put(this, count);\n        }\n        count.getAndIncrement();\n    }\n\n    public static ConcurrentMap<SafepointParseRule, AtomicInteger> getHits() {\n        return hits;\n    }\n\n    public static ConcurrentMap<SafepointParseRule, AtomicInteger> getMisses() {\n        return misses;\n    }\n\n    public static ConcurrentMap<SafepointParseRule, Throwable> getOrigin() {\n        return origin;\n    }\n\n    @Override\n    public String toString() {\n        return pattern.toString();\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/unified/ZGCPatterns.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser.unified;\n\nimport com.microsoft.gctoolkit.parser.GCParseRule;\nimport com.microsoft.gctoolkit.parser.GenericTokens;\n\npublic interface ZGCPatterns extends UnifiedPatterns {\n\n    String MEMORY_PERCENT = GenericTokens.INT + GenericTokens.UNITS + \"\\\\s*\\\\(\" + GenericTokens.INT + \"%\\\\)\";\n\n    GCParseRule ZGC_TAG = new GCParseRule(\"ZGC Tag\", \"Initializing The Z Garbage Collector$\");\n\n    //[3.558s][info ][gc,start       ] GC(3) Garbage Collection (Warmup)\n    GCParseRule CYCLE_START = new GCParseRule(\"CYCLE_START\", \"GC\\\\(\" + GenericTokens.INT + \"\\\\) Garbage Collection \" + GenericTokens.GC_CAUSE + \"$\");\n\n    //[3.559s][info ][gc,phases      ] GC(3) Pause Mark Start 0.460ms\n    //[3.574s][info ][gc,phases      ] GC(3) Pause Mark End 0.830ms\n    //[3.583s][info ][gc,phases      ] GC(3) Pause Relocate Start 0.794ms\n    GCParseRule PAUSE_PHASE = new GCParseRule(\"Pause Phase\", \"Pause (Mark Start|Mark End|Relocate Start) \" + GenericTokens.PAUSE_TIME);\n\n    //[3.573s][info ][gc,phases      ] GC(3) Concurrent Mark 14.621ms\n    //[3.578s][info ][gc,phases      ] GC(3) Concurrent Process Non-Strong References 3.654ms\n    //[3.578s][info ][gc,phases      ] GC(3) Concurrent Reset Relocation Set 0.194ms\n    //[3.582s][info ][gc,phases      ] GC(3) Concurrent Select Relocation Set 3.193ms\n    //[3.596s][info ][gc,phases      ] GC(3) Concurrent Relocate 12.962ms\n    GCParseRule CONCURRENT_PHASE = new GCParseRule(\"Concurrent Phase\",\"Concurrent (Mark|Mark Free|Process Non-Strong References|Reset Relocation Set|Select Relocation Set|Relocate) \" + GenericTokens.PAUSE_TIME);\n\n    //[3.596s][info ][gc,load        ] GC(3) Load: 4.28/3.95/3.22\n    GCParseRule LOAD = new GCParseRule(\"Load\",\"Load: \" + GenericTokens.REAL_VALUE + \"/\" + GenericTokens.REAL_VALUE + \"/\" + GenericTokens.REAL_VALUE);\n\n    //[3.596s][info ][gc,mmu         ] GC(3) MMU: 2ms/32.7%, 5ms/60.8%, 10ms/80.4%, 20ms/85.4%, 50ms/90.8%, 100ms/95.4%\n    GCParseRule MMU = new GCParseRule(\"MMU\",\"MMU: 2ms/\" + GenericTokens.PERCENTAGE + \", 5ms/\" + GenericTokens.PERCENTAGE + \", 10ms/\" + GenericTokens.PERCENTAGE + \", 20ms/\" + GenericTokens.PERCENTAGE + \", 50ms/\" + GenericTokens.PERCENTAGE + \", 100ms/\" + GenericTokens.PERCENTAGE);\n\n    //[3.596s][info ][gc,marking     ] GC(3) Mark: 1 stripe(s), 2 proactive flush(es), 1 terminate flush(es), 1 completion(s), 0 continuation(s)\",\n    GCParseRule MARK_SUMMARY = new GCParseRule(\"Mark Summary\",\"Mark: (\\\\d+) stripe\\\\(s\\\\), \");\n\n    //[3.596s][info ][gc,reloc       ] GC(3) Relocation: Successful, 6M relocated\n    GCParseRule RELOCATION_SUMMARY = new GCParseRule(\"Relocation Summary\",\"Relocation: Successful, (\\\\d+)M relocated\");\n\n    //[3.596s][info ][gc,nmethod     ] GC(3) NMethods: 1163 registered, 0 unregistered\n    GCParseRule NMETHODS = new GCParseRule(\"NMethods\",\" NMethods: \" + GenericTokens.INT + \" registered, \" + GenericTokens.INT + \" unregistered\");\n\n    //[3.596s][info ][info ][gc,metaspace   ] GC(3) Metaspace: 84M used, 85M committed, 1104M reserved\n    GCParseRule METASPACE = new GCParseRule( \"Metaspace\", \"Metaspace: \" + GenericTokens.INT + GenericTokens.UNITS + \" used, \" + GenericTokens.INT + GenericTokens.UNITS + \" committed, \" + GenericTokens.INT + GenericTokens.UNITS + \" reserved\");\n\n    //[3.596s][info ][gc,ref         ] GC(3) Soft: 391 encountered, 0 discovered, 0 enqueued\n    //[3.596s][info ][gc,ref         ] GC(3) Weak: 587 encountered, 466 discovered, 0 enqueued\n    //[3.596s][info ][gc,ref         ] GC(3) Final: 799 encountered, 0 discovered, 0 enqueued\n    //[3.596s][info ][gc,ref         ] GC(3) Phantom: 33 encountered, 1 discovered, 0 enqueued\n    GCParseRule REFERENCE_PROCESSING = new GCParseRule(\"Reference Processing\", \"(Soft|Weak|Final|Phantom): \" + GenericTokens.COUNTER + \" encountered, \" + GenericTokens.COUNTER + \" discovered, \" + GenericTokens.COUNTER + \" enqueued\");\n\n    //[3.596s][info ][gc,heap        ] GC(3) Min Capacity: 8M(0%)\n    //[3.596s][info ][gc,heap        ] GC(3) Max Capacity: 4096M(100%)\n    //[3.596s][info ][gc,heap        ] GC(3) Soft Max Capacity: 4096M(100%)\n    GCParseRule CAPACITY = new GCParseRule(\"Capacity\", \"(Min Capacity|Max Capacity|Soft Max Capacity): \" + MEMORY_PERCENT);\n\n    //[3.596s][info ][gc,heap        ] GC(3)                Mark Start          Mark End        Relocate Start      Relocate End           High               Low\n    GCParseRule MEMORY_TABLE_HEADER = new GCParseRule(\"Memory Table Header\", \"Mark Start\\\\s+Mark End\\\\s+Relocate Start\");\n\n    //[3.596s][info ][gc,heap        ] GC(3)  Capacity:    36864M (100%)      36864M (100%)      36864M (100%)      36864M (100%)      36864M (100%)      36864M (100%)\n    //[3.596s][info ][gc,heap        ] GC(3)      Free:    30614M (83%)       30460M (83%)       30988M (84%)       35938M (97%)       35938M (97%)       30458M (83%)\n    //[3.596s][info ][gc,heap        ] GC(3)      Used:     6250M (17%)        6404M (17%)        5876M (16%)         926M (3%)         6406M (17%)         926M (3%)\n    //************\n...\n", "needle_token_start": 13722, "needle_token_end": 13785, "code_context_ntokens": 16174, "output": ["```java\nprivate void hits() {\n    AtomicInteger count = hits.get(this);\n    if (count == null) {\n        count = new AtomicInteger(0);\n        hits.put(this, count);\n    }\n    count.getAndIncrement();\n}\n```"]}
{"repo": "microsoft/gctoolkit", "name": "recordRescanStepTimes", "language": "java", "path": "parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedGCLogParser.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to parse specific time durations from a log line related to garbage collection remark phases, and then update these durations in a given collection event object.\n2. **Input**: The function takes two parameters: a CMS remark collection event object to update, and a string representing a line from a log file.\n3. **Output**: There is no direct output from this function; however, it updates the CMS remark collection event object with the parsed time durations.\n4. **Procedure**: The function attempts to parse the log line for four specific remark phase durations using predefined parsing rules. If a match is found for any of the phases, it extracts the corresponding duration. These extracted durations are then used to update the CMS remark collection event object with the times for class unloading, symbol table scrubbing, string table scrubbing, and combined string and symbol table scrubbing.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalHeapParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.GCToolKit;\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.generational.BinaryTreeDictionary;\nimport com.microsoft.gctoolkit.event.generational.CMSRemark;\nimport com.microsoft.gctoolkit.event.generational.ConcurrentModeFailure;\nimport com.microsoft.gctoolkit.event.generational.ConcurrentModeInterrupted;\nimport com.microsoft.gctoolkit.event.generational.DefNew;\nimport com.microsoft.gctoolkit.event.generational.FullGC;\nimport com.microsoft.gctoolkit.event.generational.InitialMark;\nimport com.microsoft.gctoolkit.event.generational.PSFullGC;\nimport com.microsoft.gctoolkit.event.generational.PSYoungGen;\nimport com.microsoft.gctoolkit.event.generational.ParNew;\nimport com.microsoft.gctoolkit.event.generational.ParNewPromotionFailed;\nimport com.microsoft.gctoolkit.event.generational.SystemGC;\nimport com.microsoft.gctoolkit.event.generational.YoungGC;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.collection.MRUQueue;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.AbstractMap;\nimport java.util.ArrayList;\nimport java.util.Optional;\nimport java.util.Set;\nimport java.util.function.BiConsumer;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport static com.microsoft.gctoolkit.parser.unified.UnifiedG1GCPatterns.WEAK_PROCESSING;\n\n/**\n * Time of GC\n * GCType\n * Collect total heap values\n * Heap before collection\n * Heap after collection\n * Heap configured size\n * total pause time\n * CMS failures\n * System.gc() calls\n */\npublic class GenerationalHeapParser extends PreUnifiedGCLogParser implements SimplePatterns, ICMSPatterns, SerialPatterns, ParallelPatterns {\n\n    private static final Logger LOGGER = Logger.getLogger(GenerationalHeapParser.class.getName());\n\n    private ParNew parNewForwardReference;\n    private GarbageCollectionTypes garbageCollectionTypeForwardReference;\n    private GCCause gcCauseForwardReference;\n    private DateTimeStamp scavengeTimeStamp;\n    private DateTimeStamp fullGCTimeStamp;\n    private DateTimeStamp remarkTimeStamp;\n    private MemoryPoolSummary youngMemoryPoolSummaryForwardReference;\n    private MemoryPoolSummary tenuredForwardReference;\n    private MemoryPoolSummary heapForwardReference;\n    private MemoryPoolSummary metaSpaceForwardReference;\n    private double scavengeDurationForwardReference;\n    private ReferenceGCSummary referenceGCForwardReference;\n    private CPUSummary scavengeCPUSummaryForwardReference;\n    private int[] promotionFailureSizesForwardReference;\n\n    private long totalFreeSpaceForwardReference;\n    private long maxChunkSizeForwardReference;\n    private int numberOfBlocksForwardReference;\n    private long averageBlockSizeForwardReference;\n    private int treeHeightForwardReference;\n\n    //Expect Remark\n    private boolean expectRemark = false;\n\n    /* Rules not used....\n        GCParseRule FULL_PARNEW_CMF_META\n        GCParseRule FULL_GC_CMF\n        GCParseRule FULL_GC_CARDS_DETAILS\n        GCParseRule PARNEW_PROMOTION_FAILED_DETAILS_AFTER\n        GCParseRule DUP_CMF\n        GCParseRule FULL_PARNEW_CMF_PERM\n        GCParseRule FULL_GC_CMS\n        GCParseRule PARNEW_DETAILS_WITH_CONCURRENT_MODE_FAILURE_META\n        GCParseRule CMS_FULL_80\n     */\n\n\n    private final MRUQueue<GCParseRule, BiConsumer<GCLogTrace, String>> parseRules;\n\n    {\n        parseRules = new MRUQueue<>();\n        parseRules.put(DEFNEW, this::defNew);\n        parseRules.put(DEFNEW_TENURING, this::defNewWithTenuring);\n        parseRules.put(SERIAL_FULL, this::serialFull);\n        parseRules.put(SERIAL_FULL80, this::serialFull80);\n        parseRules.put(PARNEW, this::parNew);\n        parseRules.put(PARNEW_TENURING, this::parNewWithTenuring);\n        parseRules.put(PARNEW_CONCURRENT_MODE_END, this::parNewConcurrentModeEnd);\n        parseRules.put(PARNEW_CARDTABLE, this::parNewCardTable);\n...\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/G1GCForwardReference.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.GCCause;\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MalformedEvent;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.ReferenceGCSummary;\nimport com.microsoft.gctoolkit.event.RegionSummary;\nimport com.microsoft.gctoolkit.event.SurvivorMemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.UnifiedCountSummary;\nimport com.microsoft.gctoolkit.event.UnifiedStatisticalSummary;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentCleanupForNextMark;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentClearClaimedMarks;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentCompleteCleanup;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentCreateLiveData;\nimport com.microsoft.gctoolkit.event.g1gc.ConcurrentScanRootRegion;\nimport com.microsoft.gctoolkit.event.g1gc.G1Cleanup;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentMark;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentRebuildRememberedSets;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentUndoCycle;\nimport com.microsoft.gctoolkit.event.g1gc.G1FullGC;\nimport com.microsoft.gctoolkit.event.g1gc.G1FullGCNES;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCConcurrentEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCPauseEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1Mixed;\nimport com.microsoft.gctoolkit.event.g1gc.G1Remark;\nimport com.microsoft.gctoolkit.event.g1gc.G1SystemGC;\nimport com.microsoft.gctoolkit.event.g1gc.G1Young;\nimport com.microsoft.gctoolkit.event.g1gc.G1YoungInitialMark;\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.logging.Logger;\nimport java.util.stream.Stream;\n\n\nclass G1GCForwardReference extends ForwardReference {\n\n    private static final Logger LOGGER = Logger.getLogger(G1GCForwardReference.class.getName());\n\n    private int heapRegionSize = 0;\n    private long minHeapSize;\n    private long initialHeapSize;\n    private long maxHeapSize;\n    private DateTimeStamp concurrentCycleStartTime;\n\n    void setHeapRegionSize(int sizeInMegaBytes) {\n        heapRegionSize = sizeInMegaBytes;\n    }\n\n    int getHeapRegionSize() {\n        return heapRegionSize;\n    }\n\n    private GarbageCollectionTypes pausePhaseDuringConcurrentCycle = null;\n    private GarbageCollectionTypes gcType = null;\n    private GarbageCollectionTypes concurrentPhase;\n\n    G1GCForwardReference(Decorators decorators, int gcID) {\n        super(decorators, gcID);\n    }\n\n    boolean isConcurrentCycle() {\n        return gcType == GarbageCollectionTypes.Concurrent_Cycle;\n    }\n\n    //bag of stuff to maybe eliminate\n    void setMinHeapSize(long minHeapSize) {\n        this.minHeapSize = minHeapSize;\n    }\n\n    long getMinHeapSize() {\n        return minHeapSize;\n    }\n\n    void setInitialHeapSize(long initialHeapSize) {\n        this.initialHeapSize = initialHeapSize;\n    }\n\n    long getInitialHeapSize() {\n        return initialHeapSize;\n    }\n\n    void setMaxHeapSize(long maxHeapSize) {\n        this.maxHeapSize = maxHeapSize;\n    }\n\n    long getMaxHeapSize() {\n        return maxHeapSize;\n    }\n\n    void setGcType(GarbageCollectionTypes garbageCollectionType) {\n        if ((this.gcType != null) && (gcType != garbageCollectionType)) {\n            throw new IllegalArgumentException(\"attempting to redefine GC Type from\" + this.gcType + \" to \" + garbageCollectionType);\n        }\n        gcType = garbageCollectionType;\n    }\n\n    GarbageCollectionTypes getGcType() {\n        return gcType;\n    }\n\n    void setConcurrentPhase(GarbageCollectionTypes phase) {\n        this.concurrentPhase = phase;\n    }\n\n    GarbageCollectionTypes getConcurrentPhase() {\n        return concurrentPhase;\n    }\n\n\n    /**\n     * memory pool statistics\n     */\n    private static final int HEAP_OCCUPANCY_BEFORE_COLLECTION = 0;\n    private static final int HEAP_OCCUPANCY_AFTER_COLLECTION = 1;\n    private static final int HEAP_SIZE_BEFORE_COLLECTION = 2;\n    private static final int HEAP_SIZE_AFTER_COLLECTION = 3;\n    private static final int EDEN_OCCUPANCY_BEFORE_COLLECTION = 4;\n    private static final int EDEN_OCCUPANCY_AFTER_COLLECTION = 5;\n    private static final int EDEN_SIZE_BEFORE_COLLECTION = 6;\n    private static final int EDEN_SIZE_AFTER_COLLECTION = 7;\n    private static final int SURVIVOR_OCCUPANCY_BEFORE_COLLECTION = 8;\n    private static final int SURVIVOR_OCCUPANCY_AFTER_COLLECTION = 9;\n    private static final int SURVIVOR_SIZE_BEFORE_COLLECTION = 10;\n    private static final int SURVIVOR_SIZE_AFTER_COLLECTION = 11;\n    private static final int YOUNG_OCCUPANCY_BEFORE_COLLECTION = 12;\n    private static final int YOUNG_OCCUPANCY_AFTER_COLLECTION = 13;\n    private static final int YOUNG_SIZE_BEFORE_COLLECTION = 14;\n    private static final int YOUNG_SIZE_AFTER_COLLECTION = 15;\n    private static final int OLD_OCCUPANCY_BEFORE_COLLECTION = 16;\n    private static final int OLD_OCCUPANCY_AFTER_COLLECTION = 17;\n    private static final int OLD_SIZE_BEFORE_COLLECTION = 18;\n    private static final int OLD_SIZE_AFTER_COLLECTION = 19;\n    private static final int HUMONGOUS_OCCUPANCY_BEFORE_COLLECTION = 20;\n    private static final int HUMONGOUS_OCCUPANCY_AFTER_COLLECTION = 21;\n    private static final int HUMONGOUS_SIZE_BEFORE_COLLECTION = 22;\n    private static final int HUMONGOUS_SIZE_AFTER_COLLECTION = 23;\n    private static final int ARCHIVE_OCCUPANCY_BEFORE_COLLECTION = 24;\n    private static final int ARCHIVE_OCCUPANCY_AFTER_COLLECTION = 25;\n    private static final int ARCHIVE_SIZE_BEFORE_COLLECTION = 26;\n    private static final int ARCHIVE_SIZE_AFTER_COLLECTION = 27;\n    private static final int METASPACE_OCCUPANCY_BEFORE_COLLECTION = 28;\n    private static final int METASPACE_OCCUPANCY_AFTER_COLLECTION = 29;\n    private static final int METASPACE_SIZE_BEFORE_COLLECTION = 30;\n    private static final int METASPACE_SIZE_AFTER_COLLECTION = 31;\n    private static final int METASPACE_COMMITTED_BEFORE_COLLECTION = 32;\n    private static final int METASPACE_COMMITTED_AFTER_COLLECTION = 33;\n    private static final int METASPACE_RESERVED_BEFORE_COLLECTION = 34;\n    private static final int METASPACE_RESERVED_AFTER_COLLECTION = 35;\n    private static final int CLASSSPACE_OCCUPANCY_BEFORE_COLLECTION = 36;\n    private static final int CLASSSPACE_OCCUPANCY_AFTER_COLLECTION = 37;\n    private static final int CLASSSPACE_SIZE_BEFORE_COLLECTION = 38;\n    private static final int CLASSSPACE_SIZE_AFTER_COLLECTION = 39;\n    private static final int CLASSSPACE_COMMITTED_BEFORE_COLLECTION = 40;\n    private static final int CLASSSPACE_COMMITTED_AFTER_COLLECTION = 41;\n    private static final int CLASSSPACE_RESERVED_BEFORE_COLLECTION = 42;\n    private static final int CLASSSPACE_RESERVED_AFTER_COLLECTION = 43;\n\n    private final long[] memoryPoolMeasurment = {\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L,\n            -1L, -1L, -1L, -1L\n    };\n\n    private boolean setMemoryPoolMeasurement(int index, long value) {\n        if (memoryPoolMeasurment[index] < 0L) {\n            memoryPoolMeasurment[index] = value;\n            return true;\n        }\n        return false;\n    }\n\n\n    boolean setHeapOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setHeapOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setHeapSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setHeapSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HEAP_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setEdenOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setEdenOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setEdenSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setEdenSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(EDEN_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setSurvivorOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setSurvivorOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setSurvivorSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setSurvivorSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(SURVIVOR_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setYoungOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setYoungOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setYoungSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setYoungSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(YOUNG_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setOldOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setOldOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setOldSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setOldSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(OLD_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setHumongousOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setHumongousOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setHumongousSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setHumongousSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(HUMONGOUS_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceCommittedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_COMMITTED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceCommittedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_COMMITTED_AFTER_COLLECTION, value);\n    }\n\n    boolean setMetaspaceReservedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_RESERVED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setMetaspaceReservedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(METASPACE_RESERVED_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceOccupancyBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceOccupancyAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceSizeBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceSizeAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_SIZE_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceCommittedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_COMMITTED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceCommittedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_COMMITTED_AFTER_COLLECTION, value);\n    }\n\n    boolean setClassspaceReservedBeforeCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_RESERVED_BEFORE_COLLECTION, value);\n    }\n\n    boolean setClassspaceReservedAfterCollection(long value) {\n        return setMemoryPoolMeasurement(CLASSSPACE_RESERVED_AFTER_COLLECTION, value);\n    }\n\n    // ******\n    // Reference processing\n    private static final int SOFT_REFERENCE = 0;\n    private static final int WEAK_REFERENCE = 1;\n    private static final int PHANTOM_REFERENCE = 2;\n    private static final int FINAL_REFERENCE = 3;\n    private static final int JNI_WEAK_REFERENCE = 4;\n    private static final int CLEANER_REFERENCE = 5;\n    private double[] referenceProcessingDuarations = {-1.0d, -1.0d, -1.0d, -1.0d, -1.0d, -1.0d};\n    private int[] referenceCounts = {-1, -1, -1, -1, -1, -1};\n\n    void setSoftReferenceProcessingDuation(double duration) {\n        referenceProcessingDuarations[SOFT_REFERENCE] = duration;\n    }\n\n    void setWeakReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[WEAK_REFERENCE] = duration;\n    }\n\n    void setPhantomReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[PHANTOM_REFERENCE] = duration;\n    }\n\n    void setFinalReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[FINAL_REFERENCE] = duration;\n    }\n\n    void setJniWeakReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[JNI_WEAK_REFERENCE] = duration;\n    }\n\n    void setCleanerReferenceProcessingDuration(double duration) {\n        referenceProcessingDuarations[CLEANER_REFERENCE] = duration;\n    }\n\n    void setReferenceCounts(int soft, int weak, int finalReferenceCount, int phantom) {\n        referenceCounts[WEAK_REFERENCE] = weak;\n        referenceCounts[SOFT_REFERENCE] = soft;\n        referenceCounts[PHANTOM_REFERENCE] = phantom;\n        referenceCounts[FINAL_REFERENCE] = finalReferenceCount;\n    }\n\n    private boolean hasReferenceGCSummary() {\n        return referenceCounts[0] != -1;\n    }\n\n    private ReferenceGCSummary generateReferenceGCSummary() {\n        ReferenceGCSummary summary = new ReferenceGCSummary();\n        summary.addSoftReferences(getStartTime(), referenceCounts[SOFT_REFERENCE], referenceProcessingDuarations[SOFT_REFERENCE]);\n        summary.addWeakReferences(getStartTime(), referenceCounts[WEAK_REFERENCE], referenceProcessingDuarations[WEAK_REFERENCE]);\n        summary.addPhantomReferences(getStartTime(), referenceCounts[PHANTOM_REFERENCE], referenceProcessingDuarations[PHANTOM_REFERENCE]);\n        summary.addFinalReferences(getStartTime(), referenceCounts[FINAL_REFERENCE], referenceProcessingDuarations[FINAL_REFERENCE]);\n        summary.addJNIWeakReferences(getStartTime(), referenceCounts[JNI_WEAK_REFERENCE], referenceProcessingDuarations[JNI_WEAK_REFERENCE]);\n        return summary;\n    }\n\n\n    // ****\n    // Young phases\n    private static final double NOT_SET = -1.0d;\n\n    static final int PRE_EVACUATE_COLLECTION_SET = 0;\n    static final int EVACUATE_COLLECTION_SET = 1;\n    static final int POST_EVACUATE_COLLECTION_SET = 2;\n    static final int OTHER = 3;\n\n    private final double[] youngCollectionPhases = {NOT_SET, NOT_SET, NOT_SET, NOT_SET};\n    private final Map<String, Double> preEvacuateCSetPhaseDurations = new ConcurrentHashMap<>(3);\n    private final Map<String, UnifiedStatisticalSummary> evacuateCSetPhaseDurations = new ConcurrentHashMap<>();\n    private final Map<String, Double> postEvacuateCSetPhaseDurations = new ConcurrentHashMap<>();\n\n    void setPreEvacuateCSetDuration(double duration) {\n        this.youngCollectionPhases[PRE_EVACUATE_COLLECTION_SET] = duration;\n    }\n\n    void postPreEvacuateCSetPhaseDuration(String phase, double duration) {\n        preEvacuateCSetPhaseDurations.put(phase, duration);\n    }\n\n    Stream<String> preEvacuateCSetPhaseNames() {\n        return preEvacuateCSetPhaseDurations.keySet().stream();\n    }\n\n    double preEvacuateCSetPhaseDuration(String phaseName) {\n        return preEvacuateCSetPhaseDurations.get(phaseName);\n    }\n\n    void setEvacuationCSetDuration(double duration) {\n        this.youngCollectionPhases[EVACUATE_COLLECTION_SET] = duration;\n    }\n\n    void postEvacuateCSetPhaseDuration(String phase, UnifiedStatisticalSummary stats) {\n        evacuateCSetPhaseDurations.put(phase, stats);\n    }\n\n    Stream<String> evacuateCSetPhaseNames() {\n        return evacuateCSetPhaseDurations.keySet().stream();\n    }\n\n    UnifiedStatisticalSummary evacuateCSetPhaseDuration(String phaseName) {\n        return evacuateCSetPhaseDurations.get(phaseName);\n    }\n\n    void setPostEvacuateCSetDuration(double duration) {\n        this.youngCollectionPhases[POST_EVACUATE_COLLECTION_SET] = duration;\n    }\n\n    void postPostEvacuateCSetPhaseDuration(String phase, double duration) {\n        postEvacuateCSetPhaseDurations.put(phase, duration);\n    }\n\n    Stream<String> postEvacuateCSetPhaseNames() {\n        return postEvacuateCSetPhaseDurations.keySet().stream();\n    }\n\n    double postEvacuateCSetPhaseDuration(String phaseName) {\n        return postEvacuateCSetPhaseDurations.get(phaseName);\n    }\n\n    void setOtherDuration(double duration) {\n        this.youngCollectionPhases[OTHER] = duration;\n    }\n\n    private UnifiedCountSummary unifiedCountSummary = null;\n\n    void setProcessedBuffersSummary(UnifiedCountSummary summary) {\n        unifiedCountSummary = summary;\n    }\n\n    private UnifiedCountSummary terminationAttempts = null;\n\n    void setTerminationAttempts(UnifiedCountSummary summary) {\n        this.terminationAttempts = summary;\n    }\n\n    //\n    private boolean toSpaceExhausted = false;\n\n    void toSpaceExhausted() {\n        toSpaceExhausted = true;\n    }\n\n    private int evacuationWorkersUsed = 0;\n    private int evacuationWorkersAvailable = 0;\n\n    void evacuationWorkers(int workersUsed, int available) {\n        this.evacuationWorkersUsed = workersUsed;\n        this.evacuationWorkersAvailable = available;\n    }\n\n    // Concurrent Remark values\n    private int concurrentMarkWorkersUsed = 0;\n    private int concurrentMarkWorkersAvailable = 0;\n\n    void concurrentMarkWorkers(int used, int available) {\n        this.concurrentMarkWorkersUsed = used;\n        this.concurrentMarkWorkersAvailable = available;\n    }\n\n    private double markFromRootsDuration = -1.0d;\n    private double precleanDuration = -1.0d;\n\n    void setMarkFromRootsDuration(double duration) {\n        this.markFromRootsDuration = duration;\n    }\n\n    void setPrecleanDuration(double duration) {\n        this.precleanDuration = duration;\n    }\n\n    private boolean aborted = false;\n\n    void abortConcurrentMark() {\n        aborted = true;\n    }\n\n    // builders\n\n    private SurvivorMemoryPoolSummary getSurvivorMemoryPoolSummary() {\n        if (memoryPoolMeasurment[SURVIVOR_OCCUPANCY_BEFORE_COLLECTION] == -1L)\n            return null;\n        return new SurvivorMemoryPoolSummary(memoryPoolMeasurment[SURVIVOR_OCCUPANCY_BEFORE_COLLECTION], memoryPoolMeasurment[SURVIVOR_OCCUPANCY_AFTER_COLLECTION], memoryPoolMeasurment[SURVIVOR_SIZE_AFTER_COLLECTION]);\n    }\n\n    private static final int OCCUPANCY_BEFORE_OFFSET = 0;\n    private static final int OCCUPANCY_AFTER_OFFSET = 1;\n    private static final int SIZE_BEFORE_OFFSET = 2;\n    private static final int SIZE_AFTER_OFFSET = 3;\n\n    private MemoryPoolSummary getMemoryPoolSummary(int offset) {\n        if (memoryPoolMeasurment[offset + OCCUPANCY_BEFORE_OFFSET] == -1L) //do we have recorded values\n            return null;\n        //do we know the size of the memory pool prior to the collection\n        long sizeBeforeCollection = (memoryPoolMeasurment[offset + SIZE_BEFORE_OFFSET] > -1L) ? memoryPoolMeasurment[offset + SIZE_BEFORE_OFFSET] : memoryPoolMeasurment[offset + SIZE_AFTER_OFFSET];\n        return new MemoryPoolSummary(memoryPoolMeasurment[offset + OCCUPANCY_BEFORE_OFFSET], sizeBeforeCollection, memoryPoolMeasurment[offset + OCCUPANCY_AFTER_OFFSET], memoryPoolMeasurment[offset + SIZE_AFTER_OFFSET]);\n    }\n\n    private void fillInMemoryPoolStats(G1GCPauseEvent collection) {\n        //Eden, survivor, and heap\n        MemoryPoolSummary heap = getMemoryPoolSummary(HEAP_OCCUPANCY_BEFORE_COLLECTION);\n        MemoryPoolSummary young = getMemoryPoolSummary(YOUNG_OCCUPANCY_BEFORE_COLLECTION);\n        MemoryPoolSummary eden = getMemoryPoolSummary(EDEN_OCCUPANCY_BEFORE_COLLECTION);\n        SurvivorMemoryPoolSummary survivor = getSurvivorMemoryPoolSummary();\n        MemoryPoolSummary tenured = getMemoryPoolSummary(OLD_OCCUPANCY_BEFORE_COLLECTION);\n        MemoryPoolSummary humongous = getMemoryPoolSummary(HUMONGOUS_OCCUPANCY_BEFORE_COLLECTION);\n        if (heap != null && eden != null && survivor != null) {\n            collection.addMemorySummary(eden, survivor, heap);\n        } else if (eden == null && survivor == null && heap != null) {\n            collection.addMemorySummary(heap);\n        } //else\n        //need to consider other possible combinations.\n    }\n\n    /*\n    private static final int METASPACE_OCCUPANCY_BEFORE_COLLECTION = 24;\n    private static final int METASPACE_OCCUPANCY_AFTER_COLLECTION = 25;\n    private static final int METASPACE_SIZE_BEFORE_COLLECTION = 26;\n    private static final int METASPACE_SIZE_AFTER_COLLECTION = 27;\n    private static final int METASPACE_COMMITTED_BEFORE_COLLECTION = 28;\n    private static final int METASPACE_COMMITTED_AFTER_COLLECTION = 29;\n    private static final int METASPACE_RESERVED_BEFORE_COLLECTION = 30;\n    private static final int METASPACE_RESERVED_AFTER_COLLECTION = 31;\n     */\n    private void fillInMetaspaceStats(G1GCPauseEvent collection) {\n        collection.addPermOrMetaSpaceRecord(getMemoryPoolSummary(METASPACE_OCCUPANCY_BEFORE_COLLECTION));\n    }\n\n\n    enum REGIONS {\n        EDEN,\n        SURVIVOR,\n        OLD,\n        HUMONGOUS,\n        ARCHIVE;\n    }\n\n    private final RegionSummary[] regionSummaries = new RegionSummary[REGIONS.values().length];\n    \n    void setEdenRegionSummary(RegionSummary summary) {\n        regionSummaries[REGIONS.EDEN.ordinal()] = summary;\n        if (heapRegionSize > 0) {\n            setEdenOccupancyBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setEdenOccupancyAfterCollection(summary.getAfter() * heapRegionSize * 1024L);\n            setEdenSizeBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setEdenSizeAfterCollection(summary.getAssigned() * heapRegionSize * 1024L);\n        }\n    }\n\n    void setSurvivorRegionSummary(RegionSummary summary) {\n        regionSummaries[REGIONS.SURVIVOR.ordinal()] = summary;\n        if (heapRegionSize > 0) {\n            setSurvivorOccupancyBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setSurvivorOccupancyAfterCollection(summary.getAfter() * heapRegionSize * 1024L);\n            setSurvivorSizeBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setSurvivorSizeAfterCollection(summary.getAssigned() * heapRegionSize * 1024L);\n        }\n    }\n\n    void setOldRegionSummary(RegionSummary summary) {\n        regionSummaries[REGIONS.OLD.ordinal()] = summary;\n        if (heapRegionSize > 0) {\n            setOldOccupancyBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setOldOccupancyAfterCollection(summary.getAfter() * heapRegionSize * 1024L);\n            setOldSizeBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setOldSizeAfterCollection(summary.getAfter() * heapRegionSize * 1024L);\n        }\n    }\n\n    void setHumongousRegionSummary(RegionSummary summary) {\n        regionSummaries[REGIONS.HUMONGOUS.ordinal()] = summary;\n        if (heapRegionSize > 0) {\n            setHumongousOccupancyBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setHumongousOccupancyAfterCollection(summary.getAfter() * heapRegionSize * 1024L);\n            setHumongousSizeBeforeCollection(summary.getBefore() * heapRegionSize * 1024L);\n            setHumongousSizeAfterCollection(summary.getAfter() * heapRegionSize * 1024L);\n        }\n    }\n\n    void setArchiveRegionSummary(RegionSummary summary) {\n        regionSummaries[REGIONS.ARCHIVE.ordinal()] = summary;\n        if (heapRegionSize > 0) {\n            setArchiveOccupancyBeforeCollection(summary.getBefore() * heapRegionSize * 1024);\n            setArchiveOccupancyAfterCollection(summary.getAfter() * heapRegionSize * 1024);\n            setArchiveSizeBeforeCollection(summary.getBefore() * heapRegionSize * 1024);\n            setArchiveSizeAfterCollection(summary.getAfter() * heapRegionSize * 1024);\n        }\n    }\n\n    private void fillInRegionSummary(G1GCPauseEvent collection) {\n        collection.addRegionSummary(\n                regionSummaries[REGIONS.EDEN.ordinal()],\n                regionSummaries[REGIONS.SURVIVOR.ordinal()],\n                regionSummaries[REGIONS.OLD.ordinal()],\n                regionSummaries[REGIONS.HUMONGOUS.ordinal()],\n                regionSummaries[REGIONS.ARCHIVE.ordinal()]\n        );\n    }\n\n    private void fullInInternalPhases(G1FullGC collection) {\n        int index = 0;\n        String key;\n        while ((key = fullGCInternalPhaseOrder.get(++index)) != null) {\n            collection.addInternalPhase(key, fullGCInternalPhases.get(key));\n        }\n    }\n\n    private void fillInPhases(G1Young collection) {\n        collection.addPhaseDuration(\"Pre Evacuate Collection\", youngCollectionPhases[PRE_EVACUATE_COLLECTION_SET]);\n        collection.addPhaseDuration(\"Evacuate Collection\", youngCollectionPhases[EVACUATE_COLLECTION_SET]);\n        collection.addPhaseDuration(\"Post Evacuate Collection Set\", youngCollectionPhases[POST_EVACUATE_COLLECTION_SET]);\n        collection.addPhaseDuration(\"Other\", youngCollectionPhases[OTHER]);\n\n        preEvacuateCSetPhaseNames().forEach(name -> collection.addPreEvacuationCollectionPhase(name, preEvacuateCSetPhaseDuration(name)));\n        evacuateCSetPhaseNames().forEach(name -> collection.addEvacuationCollectionPhase(name, evacuateCSetPhaseDuration(name)));\n        postEvacuateCSetPhaseNames().forEach(name -> collection.addPostEvacuationCollectionPhase(name, postEvacuateCSetPhaseDuration(name)));\n    }\n\n    private DateTimeStamp pausePhaseDuringConcurrentCycleTime = null;\n    private double pausePhaseDuringConcurrentCycleDuration = -1.0d;\n\n    void pausePhaseDuringConcurrentCycle(GarbageCollectionTypes pausePhase) {\n        this.pausePhaseDuringConcurrentCycle = pausePhase;\n    }\n\n    void pausePhaseDuringConcurrentCycleStart(DateTimeStamp clock) {\n        pausePhaseDuringConcurrentCycleTime = clock;\n    }\n\n    void pausePhaseDuringConcurrentCycleDuration(double duration) {\n        pausePhaseDuringConcurrentCycleDuration = duration;\n    }\n\n    private double finalizeMarkingDuration = -1.0d;\n    private double systemDictionaryUnloadingDuration = -1.0d;\n\n    void finalizeMarkingDuration(double duration) {\n        this.finalizeMarkingDuration = duration;\n    }\n\n    void systemDictionaryUnloadingDuration(double duration) {\n        this.systemDictionaryUnloadingDuration = duration;\n    }\n\n    private int stringTableProcessed = -1;\n    private int stringTableRemoved = -1;\n\n    void stringTableProcessedAndRemoved(int processed, int removed) {\n        this.stringTableProcessed = processed;\n        this.stringTableRemoved = removed;\n    }\n\n    private int symbolTableProcessed = -1;\n    private int symbolTableRemoved = -1;\n\n    void symbolTableProcessedAndRemoved(int processed, int removed) {\n        this.symbolTableProcessed = processed;\n        this.symbolTableRemoved = removed;\n    }\n\n    private double parallelUnloadingDuration = -1.0d;\n\n    void parallelUnloadingDuration(double duration) {\n        this.parallelUnloadingDuration = duration;\n    }\n\n    G1GCConcurrentEvent buildConcurrentPhaseEvent() {\n        switch (getConcurrentPhase()) {\n            case ConcurrentClearClaimedMarks:\n                return new ConcurrentClearClaimedMarks(getStartTime(), getDuration());\n            case ConcurrentScanRootRegions:\n                return new ConcurrentScanRootRegion(getStartTime(), getDuration());\n            case Concurrent_Mark:\n                return buildConcurrentMark();\n            case ConcurrentCompleteCleanup:\n                return new ConcurrentCompleteCleanup(getStartTime(), getDuration());\n            case ConcurrentCreateLiveData:\n                return new ConcurrentCreateLiveData(getStartTime(), getDuration());\n            case ConcurrentCleanupForNextMark:\n                return new ConcurrentCleanupForNextMark(getStartTime(), getDuration());\n            case G1ConcurrentRebuildRememberedSets:\n                return new G1ConcurrentRebuildRememberedSets(getStartTime(), getDuration());\n            default:\n                LOGGER.warning(\"Unrecognized Concurrent Event \" + getConcurrentPhase());\n        }\n        return null;\n    }\n\n    G1GCConcurrentEvent buildConcurrentUndoCycleEvent() {\n        return new G1ConcurrentUndoCycle(getConcurrentCycleStartTime(), getDuration());\n    }\n\n    /**\n     * gcType == null -> likely an incomplete record.\n     * @return\n     */\n    G1GCPauseEvent buildEvent() throws MalformedEvent {\n        if (gcType == null ) {\n            throw new MalformedEvent(\"G1GC Event type is undefined (null): \" + this.toString());\n        }\n        switch (this.gcType) {\n            case Young:\n                return buildYoung();\n            case Initial_Mark:\n                return buildInitialMark();\n            case Mixed:\n                return buildMixed();\n            case G1GCFull:\n                return buildFull();\n            case Concurrent_Cycle:\n                switch (pausePhaseDuringConcurrentCycle) {\n                    case G1GCRemark:\n                        return buildRemark();\n                    case G1GCCleanup:\n                        return buildCleanup();\n                    default:\n                        throw new MalformedEvent(\"Unrecognized (mostly) Concurrent Cycle Pause Event \" + getConcurrentPhase());\n                }\n            case G1GCConcurrentUndoCycle:\n                // gctype is likely incorrectly set in the forward reference. The plan is to specialize the forward\n                // references thus allowing this event to be built here rather than in a specialized method.\n                // The phase is a reset that occurs after an eager collection of humongous regions. This event will be\n                // filled in a future PR.\n                return null;\n            default:\n                throw new MalformedEvent(\"Unrecognized Event \" + gcType);\n        }\n    }\n\n    private G1Young buildYoung(G1Young collection) {\n        fillInMemoryPoolStats(collection);\n        fillInRegionSummary(collection);\n        fillInMetaspaceStats(collection);\n        fillInPhases(collection);\n        if (toSpaceExhausted) collection.toSpaceExhausted();\n        if (hasReferenceGCSummary())\n            collection.add(generateReferenceGCSummary());\n        collection.addCPUSummary(getCPUSummary());\n        return collection;\n    }\n\n    private G1Young buildYoung() {\n        return buildYoung(new G1Young(getStartTime(), getGCCause(), getDuration()));\n    }\n\n    private G1YoungInitialMark buildInitialMark() {\n        return (G1YoungInitialMark) buildYoung(new G1YoungInitialMark(getStartTime(), getGCCause(), getDuration()));\n    }\n\n    private G1Mixed buildMixed() {\n        return (G1Mixed) buildYoung(new G1Mixed(getStartTime(), getGCCause(), getDuration()));\n    }\n\n\n    private G1ConcurrentMark buildConcurrentMark() {\n        G1ConcurrentMark concurrentMark = new G1ConcurrentMark(getStartTime(), getDuration());\n        if (aborted)\n            concurrentMark.abort();\n        if (markFromRootsDuration > -1.0d) {\n            concurrentMark.setMarkFromRootsDuration(markFromRootsDuration);\n            concurrentMark.setActiveWorkerThreads(concurrentMarkWorkersUsed);\n            concurrentMark.setAvailableWorkerThreads(concurrentMarkWorkersAvailable);\n        }\n\n        if (precleanDuration > -1.0)\n            concurrentMark.setPrecleanDuration(precleanDuration);\n\n        return concurrentMark;\n    }\n\n    private G1Remark buildRemark() {\n        G1Remark remark = new G1Remark(pausePhaseDuringConcurrentCycleTime, 0.0d, pausePhaseDuringConcurrentCycleDuration);\n        if (hasReferenceGCSummary())\n            remark.add(generateReferenceGCSummary());\n        fillInMemoryPoolStats(remark);\n        remark.addCPUSummary(getCPUSummary());\n        return remark;\n    }\n\n    private G1Cleanup buildCleanup() {\n        G1Cleanup cleanup = new G1Cleanup(pausePhaseDuringConcurrentCycleTime, pausePhaseDuringConcurrentCycleDuration);\n        fillInMemoryPoolStats(cleanup);\n        cleanup.addCPUSummary(getCPUSummary());\n        return cleanup;\n    }\n\n    /*   todo: capture phases????\n    [226.179s][info ][gc,phases,start] GC(1198) Phase 1: Mark live objects\n    [226.310s][info ][gc,phases      ] GC(1198) Phase 1: Mark live objects 131.379ms\n    [226.310s][info ][gc,phases,start] GC(1198) Phase 2: Compute new object addresses\n    [226.354s][info ][gc,phases      ] GC(1198) Phase 2: Compute new object addresses 44.149ms\n    [226.354s][info ][gc,phases,start] GC(1198) Phase 3: Adjust pointers\n    [226.432s][info ][gc,phases      ] GC(1198) Phase 3: Adjust pointers 77.897ms\n    [226.432s][info ][gc,phases,start] GC(1198) Phase 4: Move objects\n    [226.461s][info ][gc,phases      ] GC(1198) Phase 4: Move objects 28.974ms\n */\n    private G1FullGC buildFull() {\n        G1FullGC collection;\n        if (getGCCause() == GCCause.JAVA_LANG_SYSTEM)\n            collection = new G1SystemGC(getStartTime(), getDuration());\n        else\n            collection = new G1FullGCNES(getStartTime(), getGCCause(), getDuration());\n\n        // [226.310s][info ][gc,stringtable ] GC(1198) Cleaned string and symbol table, strings: 7394 processed, 0 removed, symbols: 55496 processed, 0 removed\n        // [226.310s][debug][gc,phases      ] GC(1198) Scrub String and Symbol Tables 0.451ms.\n        //todo: collection.addStringTableSymbolTable();\n        //collection.addPermOrMetaSpaceRecord(metaSpaceSummary);\n        //[226.310s][debug][gc,phases      ] GC(1198) Class Unloading 3.481ms\n        //collection.classUnloading()  todo: fill in\n        fillInMemoryPoolStats(collection);\n        fullInInternalPhases(collection);\n        fillInRegionSummary(collection);\n        if (hasReferenceGCSummary())\n            collection.add(generateReferenceGCSummary());\n        collection.addCPUSummary(getCPUSummary());\n        return collection;\n    }\n\n    private final Map<String, Double> fullGCInternalPhases = new ConcurrentHashMap<>();\n    private final Map<Integer, String> fullGCInternalPhaseOrder = new ConcurrentHashMap<>();\n\n    void fullPhase(int integerGroup, String fullGCInternalPhase, double duration) {\n        fullGCInternalPhaseOrder.put(integerGroup, fullGCInternalPhase);\n        fullGCInternalPhases.put(fullGCInternalPhase, duration);\n    }\n\n    /**\n     * Not that this value will be printed but having toString() defined is useful for debugging. The issue is\n     * null values for gctype and gcCause will cause IDE debuggers to NPE. Capture and set to null.\n     * @return String rep of this forward reference.\n     */\n    public String toString() {\n        return getStartTime().toString() + \" : \" + ((gcType == null) ? \"null\" : gcType.toString()) + \" : \" + ((getGCCause() == null) ? \"null\" : getGCCause().toString()) + \" : \" + getDuration();\n    }\n\n    boolean setArchiveOccupancyBeforeCollection(int value) {\n        return setMemoryPoolMeasurement(ARCHIVE_OCCUPANCY_BEFORE_COLLECTION, value);\n    }\n\n    boolean setArchiveOccupancyAfterCollection(int value) {\n        return setMemoryPoolMeasurement(ARCHIVE_OCCUPANCY_AFTER_COLLECTION, value);\n    }\n\n    boolean setArchiveSizeBeforeCollection(int value) {\n        return setMemoryPoolMeasurement(ARCHIVE_SIZE_BEFORE_COLLECTION, value);\n    }\n\n    boolean setArchiveSizeAfterCollection(int value) {\n        return setMemoryPoolMeasurement(ARCHIVE_SIZE_AFTER_COLLECTION, value);\n    }\n\n    void setConcurrentCycleStartTime(DateTimeStamp clock) {\n        this.concurrentCycleStartTime = clock;\n    }\n\n    DateTimeStamp getConcurrentCycleStartTime() {\n        return concurrentCycleStartTime;\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/GenerationalForwardReference.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.jvm.MetaspaceRecord;\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\n\nimport java.util.Map;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.stream.Stream;\n\nimport static com.microsoft.gctoolkit.event.GarbageCollectionTypes.ConcurrentModeFailure;\nimport static com.microsoft.gctoolkit.event.GarbageCollectionTypes.FullGC;\n\npublic class GenerationalForwardReference extends ForwardReference {\n\n    private GarbageCollectionTypes gcType;\n    private MemoryPoolSummary young = null;\n    private MemoryPoolSummary tenured = null;\n    private MemoryPoolSummary heap = null;\n    private MetaspaceRecord metaspace = null;\n    private MetaspaceRecord nonClassspace = null;\n\n    public MetaspaceRecord getNonClassspace() {\n        return nonClassspace;\n    }\n\n    public void setNonClassspace(MetaspaceRecord nonClassspace) {\n        this.nonClassspace = nonClassspace;\n    }\n\n    public MetaspaceRecord getClassspace() {\n        return classspace;\n    }\n\n    public void setClassspace(MetaspaceRecord classspace) {\n        this.classspace = classspace;\n    }\n\n    private MetaspaceRecord classspace = null;\n    private final Map<String, Double> remarkPhases = new ConcurrentHashMap<>();\n    private final Map<String, Double> phases = new ConcurrentHashMap<>();\n\n    public GenerationalForwardReference(GarbageCollectionTypes gcType, Decorators decorators, int gcid) {\n        super(decorators, gcid);\n        this.gcType = gcType;\n    }\n\n    GarbageCollectionTypes getGarbageCollectionType() {\n        return this.gcType;\n    }\n\n    void setYoung(MemoryPoolSummary young) {\n        this.young = young;\n    }\n\n    MemoryPoolSummary getYoung() {\n        return this.young;\n    }\n\n    void setTenured(MemoryPoolSummary tenured) {\n        this.tenured = tenured;\n    }\n\n    MemoryPoolSummary getTenured() {\n        return this.tenured;\n    }\n\n    void setMetaspace(MetaspaceRecord metaSpaceRecord) {\n        this.metaspace = metaSpaceRecord;\n    }\n\n    MetaspaceRecord getMetaspace() {\n        return metaspace;\n    }\n\n    void setHeap(MemoryPoolSummary occupancyWithMemoryPoolSizeSummary) {\n        this.heap = occupancyWithMemoryPoolSizeSummary;\n    }\n\n    MemoryPoolSummary getHeap() {\n        return this.heap;\n    }\n\n    void addCMSRemarkPhase(String phase, double duration) {\n        remarkPhases.put(phase, duration);\n    }\n\n    double getPhaseDuration(String phaseName) {\n        return remarkPhases.get(phaseName);\n    }\n\n    Stream<String> remarkPhases() {\n        return remarkPhases.keySet().stream();\n    }\n\n    public void convertToConcurrentModeFailure() {\n        gcType = ConcurrentModeFailure;\n    }\n\n    public void convertToSerialFull() {\n        gcType = FullGC;\n    }\n\n    public void addFullGCPhase(String phase, double duration) {\n        phases.put(phase, duration);\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/PreUnifiedGCLogParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.generational.CMSRemark;\nimport com.microsoft.gctoolkit.event.jvm.PermGenSummary;\n\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n\npublic abstract class PreUnifiedGCLogParser extends GCLogParser {\n\n    private static final Logger LOGGER = Logger.getLogger(PreUnifiedGCLogParser.class.getName());\n    private final GCParseRule TIMESTAMP_BLOCK = new GCParseRule(\"TIMESTAMP_BLOCK\", \"^\" + DATE_TIMESTAMP);\n\n    public PreUnifiedGCLogParser() {}\n\n    void advanceClock(String record) {\n        try {\n            GCLogTrace trace = TIMESTAMP_BLOCK.parse(record);\n            if (trace == null) {\n                trace = CMSPatterns.ABORT_PRECLEAN_DUE_TO_TIME_CLAUSE.parse(record);\n                if (trace == null)\n                    return;\n            }\n            super.advanceClock(trace.getDateTimeStamp());\n        } catch (Throwable t) {\n            LOGGER.log(Level.FINE, \"[PARSING ERROR] \" + record, t);\n        }\n    }\n\n    // todo: mixes aggregator with parsing. premature optimization...\n    MemoryPoolSummary getTotalOccupancyBeforeAfterWithTotalHeapPoolSizeSummary(GCLogTrace trace, int offset) {\n        MemoryPoolSummary summary = trace.getOccupancyBeforeAfterWithMemoryPoolSizeSummary(offset);\n        return summary;\n    }\n\n    MemoryPoolSummary getTotalOccupancyWithTotalHeapSizeSummary(GCLogTrace trace, int offset) {\n        MemoryPoolSummary summary = trace.getOccupancyWithMemoryPoolSizeSummary(offset);\n        return summary;\n    }\n\n    GCLogTrace extractReferenceBlock(String line, GCParseRule rule) {\n        return rule.parse(line);\n    }\n\n    MemoryPoolSummary extractPermGenRecord(GCLogTrace trace) {\n        int index = (trace.getGroup(2) == null) ? 2 : 4;\n        return new PermGenSummary(trace.getLongGroup(index), trace.getLongGroup(4), trace.getLongGroup(6));\n    }\n\n    \nvoid recordRescanStepTimes(CMSRemark collection, String line) {\n        GCLogTrace clause;\n        double unloading = 0.0d, symbolTable = 0.0d, stringTable = 0.0d, stringAndSymbolTable = 0.0d;\n\n        if ((clause = CLASS_UNLOADING.parse(line)) != null)\n            unloading = clause.getDoubleGroup(2);\n        if ((clause = SYMBOL_TABLE_SCRUB.parse(line)) != null)\n            symbolTable = clause.getDoubleGroup(2);\n        if ((clause = STRING_TABLE_SCRUB.parse(line)) != null)\n            stringTable = clause.getDoubleGroup(2);\n        if ((clause = STRING_AND_SYMBOL_SCRUB.parse(line)) != null)\n            stringAndSymbolTable = clause.getDoubleGroup(2);\n\n        collection.addClassUnloadingAndStringTableProcessingDurations(unloading, symbolTable, stringTable, stringAndSymbolTable);\n    }\n\n    CPUSummary extractCPUSummary(String line) {\n        GCLogTrace trace;\n        if ((trace = CPU_BREAKDOWN.parse(line)) != null) {\n            return new CPUSummary(trace.getDoubleGroup(1), trace.getDoubleGroup(2), trace.getDoubleGroup(3));\n        }\n        return null;\n    }\n}\n\n// Path: parser/src/main/java/com/microsoft/gctoolkit/parser/UnifiedG1GCParser.java\n// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT License.\npackage com.microsoft.gctoolkit.parser;\n\nimport com.microsoft.gctoolkit.GCToolKit;\nimport com.microsoft.gctoolkit.aggregator.EventSource;\nimport com.microsoft.gctoolkit.event.CPUSummary;\nimport com.microsoft.gctoolkit.event.GarbageCollectionTypes;\nimport com.microsoft.gctoolkit.event.MalformedEvent;\nimport com.microsoft.gctoolkit.event.MemoryPoolSummary;\nimport com.microsoft.gctoolkit.event.RegionSummary;\nimport com.microsoft.gctoolkit.event.g1gc.G1ConcurrentUndoCycle;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCConcurrentEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCEvent;\nimport com.microsoft.gctoolkit.event.g1gc.G1GCPauseEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMEvent;\nimport com.microsoft.gctoolkit.event.jvm.JVMTermination;\nimport com.microsoft.gctoolkit.jvm.Diary;\nimport com.microsoft.gctoolkit.message.ChannelName;\nimport com.microsoft.gctoolkit.message.JVMEventChannel;\nimport com.microsoft.gctoolkit.parser.collection.RuleSet;\nimport com.microsoft.gctoolkit.parser.jvm.Decorators;\nimport com.microsoft.gctoolkit.parser.unified.UnifiedG1GCPatterns;\nimport com.microsoft.gctoolkit.time.DateTimeStamp;\n\nimport java.util.AbstractMap;\nimport java.util.LinkedList;\nimport java.util.Map;\nimport java.util.Queue;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.function.BiConsumer;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport static com.microsoft.gctoolkit.event.GarbageCollectionTypes.fromLabel;\n\n/**\n...\n", "needle_token_start": 15213, "needle_token_end": 15463, "code_context_ntokens": 16152, "output": ["```java\nvoid recordRescanStepTimes(CMSRemark collection, String line) {\n    GCLogTrace clause;\n    double unloading = 0.0d, symbolTable = 0.0d, stringTable = 0.0d, stringAndSymbolTable = 0.0d;\n\n    if ((clause = CLASS_UNLOADING.parse(line)) != null)\n        unloading = clause.getDoubleGroup(2);\n    if ((clause = SYMBOL_TABLE_SCRUB.parse(line)) != null)\n        symbolTable = clause.getDoubleGroup(2);\n    if ((clause = STRING_TABLE_SCRUB.parse(line)) != null)\n        stringTable = clause.getDoubleGroup(2);\n    if ((clause = STRING_AND_SYMBOL_SCRUB.parse(line)) != null)\n        stringAndSymbolTable = clause.getDoubleGroup(2);\n\n    collection.addClassUnloadingAndStringTableProcessingDurations(unloading, symbolTable, stringTable, stringAndSymbolTable);\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "stringComponents", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/StringWrapper.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function extracts and processes the text content from a list of string literal trees, removing any syntax-specific characters like quotes and concatenation operators, and handling escaped characters and whitespace.\n2. **Input**: The function takes three parameters: a string representing the entire input source code, a compilation unit object that provides context for parsing, and a list of tree nodes representing string literals.\n3. **Output**: Returns a list of strings, each representing a cleaned and processed segment of the original string literals, with syntax characters and unnecessary whitespace removed.\n4. **Procedure**: \n   - Iterate over each tree node in the provided list.\n   - For each node, extract the substring from the input source code corresponding to the node's span, excluding the leading and trailing quotation marks.\n   - Process the extracted string to remove or handle whitespace, escaped whitespace, and escaped newlines.\n   - Accumulate and return the processed strings in a list.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            }\n            // Is the literal being wrapped the first in a chain of concatenation expressions?\n            // i.e. `ONE + TWO + THREE`\n            // We need this information to handle continuation indents.\n            AtomicBoolean first = new AtomicBoolean(false);\n            // Finds the set of string literals in the concat expression that includes the one that needs\n            // to be wrapped.\n            List<Tree> flat = flatten(input, unit, path, enclosing, first);\n\n            // Walk up as long as parents are on the same line, in order to find the correct\n            // startColumn.\n            TreePath startingPath = enclosing;\n            while (startingPath.getParentPath() != null && onSameLineAsParent(lineMap, startingPath)) {\n                startingPath = startingPath.getParentPath();\n            }\n\n            // Zero-indexed start column\n            int startColumn = lineMap.getColumnNumber(getStartPosition(flat.get(0))) - 1;\n            int fistLineCol = lineMap.getColumnNumber(getStartPosition(startingPath.getLeaf())) - 1;\n\n            // Handling leaving trailing non-string tokens at the end of the literal,\n            // e.g. the trailing `);` in `foo(\"...\");`.\n            int end = getEndPosition(unit, getLast(flat));\n            int lineEnd = end;\n            while (Newlines.hasNewlineAt(input, lineEnd) == -1) {\n                lineEnd++;\n            }\n            int trailing = lineEnd - end;\n\n            // Get the original source text of the string literals, excluding `\"` and `+`.\n            ImmutableList<String> components = stringComponents(input, unit, flat);\n            replacements.put(\n                    Range.closedOpen(getStartPosition(flat.get(0)), getEndPosition(unit, getLast(flat))),\n                    reflow(separator, columnLimit, trailing, components, first.get(), startColumn, fistLineCol));\n        }\n        return replacements;\n    }\n\n    private static boolean onSameLineAsParent(LineMap lineMap, TreePath path) {\n        return lineMap.getLineNumber(getStartPosition(path.getLeaf()))\n                == lineMap.getLineNumber(getStartPosition(path.getParentPath().getLeaf()));\n    }\n\n    /**\n     * Returns the source text of the given string literal trees, excluding the leading and trailing double-quotes and\n     * the `+` operator.\n     */\n    \nprivate static ImmutableList<String> stringComponents(\n            String input, JCTree.JCCompilationUnit unit, List<Tree> flat) {\n        ImmutableList.Builder<String> result = ImmutableList.builder();\n        StringBuilder piece = new StringBuilder();\n        for (Tree tree : flat) {\n            // adjust for leading and trailing double quotes\n            String text = input.substring(getStartPosition(tree) + 1, getEndPosition(unit, tree) - 1);\n            int start = 0;\n            for (int idx = 0; idx < text.length(); idx++) {\n                if (CharMatcher.whitespace().matches(text.charAt(idx))) {\n                    // continue below\n                } else if (hasEscapedWhitespaceAt(text, idx) != -1) {\n                    // continue below\n                } else if (hasEscapedNewlineAt(text, idx) != -1) {\n                    int length;\n                    while ((length = hasEscapedNewlineAt(text, idx)) != -1) {\n                        idx += length;\n                    }\n                } else {\n                    continue;\n                }\n                piece.append(text, start, idx);\n                result.add(piece.toString());\n                piece = new StringBuilder();\n                start = idx;\n            }\n            if (piece.length() > 0) {\n                result.add(piece.toString());\n                piece = new StringBuilder();\n            }\n            if (start < text.length()) {\n                piece.append(text, start, text.length());\n            }\n        }\n        if (piece.length() > 0) {\n            result.add(piece.toString());\n        }\n        return result.build();\n    }\n\n    static int hasEscapedWhitespaceAt(String input, int idx) {\n        return Stream.of(\"\\\\t\")\n                .mapToInt(x -> input.startsWith(x, idx) ? x.length() : -1)\n                .filter(x -> x != -1)\n                .findFirst()\n                .orElse(-1);\n    }\n\n    static int hasEscapedNewlineAt(String input, int idx) {\n        return Stream.of(\"\\\\r\\\\n\", \"\\\\r\", \"\\\\n\")\n                .mapToInt(x -> input.startsWith(x, idx) ? x.length() : -1)\n                .filter(x -> x != -1)\n                .findFirst()\n                .orElse(-1);\n    }\n\n    /**\n     * Reflows the given source text, trying to split on word boundaries.\n     *\n     * @param separator the line separator\n     * @param columnLimit the number of columns to wrap at\n     * @param trailing extra space to leave after the last line\n     * @param components the text to reflow\n     * @param first0 true if the text includes the beginning of its enclosing concat chain, i.e. a\n     * @param textStartColumn the column position of the beginning of the original text\n     * @param firstLineStartColumn the column where the very first line starts (can be less than textStartColumn if text\n     *     follows variable declaration)\n     */\n    private static String reflow(\n            String separator,\n            int columnLimit,\n            int trailing,\n            ImmutableList<String> components,\n            boolean first0,\n            int textStartColumn,\n            int firstLineStartColumn) {\n        // We have space between the start column and the limit to output the first line.\n        // Reserve two spaces for the quotes.\n        int width = columnLimit - textStartColumn - 2;\n        Deque<String> input = new ArrayDeque<>(components);\n        List<String> lines = new ArrayList<>();\n        boolean first = first0;\n        while (!input.isEmpty()) {\n            int length = 0;\n            List<String> line = new ArrayList<>();\n            if (input.stream().mapToInt(String::length).sum() <= width) {\n                width -= trailing;\n            }\n            while (!input.isEmpty()\n                    && (length <= 4 || (length + input.peekFirst().length()) < width)) {\n                String text = input.removeFirst();\n                line.add(text);\n                length += text.length();\n                if (text.endsWith(\"\\\\n\") || text.endsWith(\"\\\\r\")) {\n                    break;\n                }\n            }\n            if (line.isEmpty()) {\n                line.add(input.removeFirst());\n            }\n            // add the split line to the output, and process whatever's left\n            lines.add(String.join(\"\", line));\n            if (first) {\n                // Subsequent lines have a four-space continuation indent and a `+ `.\n                width -= 6;\n                // Also, switch to firstLineStartColumn in order to account for the fact that continuations\n                // should get indented from the beginning of the first line.\n                // This is to handle cases like:\n                // String foo = \"first component\"\n                //     + \"rest\";\n                width += textStartColumn - firstLineStartColumn;\n                first = false;\n            }\n        }\n\n        return lines.stream()\n                .collect(joining(\n                        \"\\\"\" + separator + \" \".repeat(first0 ? firstLineStartColumn + 4 : textStartColumn - 2) + \"+ \\\"\",\n                        \"\\\"\",\n                        \"\\\"\"));\n    }\n\n    /**\n     * Flattens the given binary expression tree, and extracts the subset that contains the given path and any adjacent\n     * nodes that are also string literals.\n     */\n    private static List<Tree> flatten(\n            String input, JCTree.JCCompilationUnit unit, TreePath path, TreePath parent, AtomicBoolean firstInChain) {\n        List<Tree> flat = new ArrayList<>();\n\n        // flatten the expression tree with a pre-order traversal\n        ArrayDeque<Tree> todo = new ArrayDeque<>();\n        todo.add(parent.getLeaf());\n        while (!todo.isEmpty()) {\n            Tree first = todo.removeFirst();\n            if (first.getKind() == Tree.Kind.PLUS) {\n                BinaryTree bt = (BinaryTree) first;\n                todo.addFirst(bt.getRightOperand());\n                todo.addFirst(bt.getLeftOperand());\n            } else {\n                flat.add(first);\n            }\n        }\n\n        int idx = flat.indexOf(path.getLeaf());\n        Verify.verify(idx != -1);\n\n        // walk outwards from the leaf for adjacent string literals to also reflow\n        int startIdx = idx;\n        int endIdx = idx + 1;\n        while (startIdx > 0\n                && flat.get(startIdx - 1).getKind() == Tree.Kind.STRING_LITERAL\n                && noComments(input, unit, flat.get(startIdx - 1), flat.get(startIdx))) {\n            startIdx--;\n        }\n        while (endIdx < flat.size()\n                && flat.get(endIdx).getKind() == Tree.Kind.STRING_LITERAL\n                && noComments(input, unit, flat.get(endIdx - 1), flat.get(endIdx))) {\n            endIdx++;\n        }\n\n        firstInChain.set(startIdx == 0);\n        return ImmutableList.copyOf(flat.subList(startIdx, endIdx));\n    }\n\n    private static boolean noComments(String input, JCTree.JCCompilationUnit unit, Tree one, Tree two) {\n        return STRING_CONCAT_DELIMITER.matchesAllOf(\n                input.subSequence(getEndPosition(unit, one), getStartPosition(two)));\n    }\n\n    public static final CharMatcher STRING_CONCAT_DELIMITER =\n            CharMatcher.whitespace().or(CharMatcher.anyOf(\"\\\"+\"));\n\n    private static int getEndPosition(JCTree.JCCompilationUnit unit, Tree tree) {\n        return ((JCTree) tree).getEndPosition(unit.endPositions);\n    }\n\n    private static int getStartPosition(Tree tree) {\n        return ((JCTree) tree).getStartPosition();\n    }\n\n    /** Returns true if any lines in the given Java source exceed the column limit. */\n    private static boolean longLines(int columnLimit, String input) {\n        // TODO(cushon): consider adding Newlines.lineIterable?\n        Iterator<String> it = Newlines.lineIterator(input);\n        while (it.hasNext()) {\n            String line = it.next();\n            if (line.length() > columnLimit) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    /** Parses the given Java source. */\n    private static JCTree.JCCompilationUnit parse(String source, boolean allowStringFolding) throws FormatterException {\n        Context context = new Context();\n        Options.instance(context).put(\"allowStringFolding\", Boolean.toString(allowStringFolding));\n        return Formatter.parseJcCompilationUnit(context, source);\n    }\n\n    /** Applies replacements to the given string. */\n    private static String applyReplacements(String javaInput, TreeRangeMap<Integer, String> replacementMap) {\n        // process in descending order so the replacement ranges aren't perturbed if any replacements\n        // differ in size from the input\n        Map<Range<Integer>, String> ranges = replacementMap.asDescendingMapOfRanges();\n        if (ranges.isEmpty()) {\n            return javaInput;\n        }\n        StringBuilder sb = new StringBuilder(javaInput);\n        for (Map.Entry<Range<Integer>, String> entry : ranges.entrySet()) {\n            Range<Integer> range = entry.getKey();\n            sb.replace(range.lowerEndpoint(), range.upperEndpoint(), entry.getValue());\n        }\n        return sb.toString();\n    }\n\n    private StringWrapper() {}\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/Formatter.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static java.nio.charset.StandardCharsets.UTF_8;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Iterables;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.io.CharSink;\nimport com.google.common.io.CharSource;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.FormattingError;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.OpsBuilder;\nimport com.palantir.javaformat.OpsBuilder.OpsOutput;\nimport com.palantir.javaformat.Utils;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\nimport com.palantir.javaformat.doc.Level;\nimport com.palantir.javaformat.doc.NoopSink;\nimport com.palantir.javaformat.doc.Obs;\nimport com.palantir.javaformat.doc.Obs.Sink;\nimport com.palantir.javaformat.doc.State;\nimport com.sun.tools.javac.file.JavacFileManager;\nimport com.sun.tools.javac.parser.JavacParser;\nimport com.sun.tools.javac.parser.ParserFactory;\nimport com.sun.tools.javac.tree.JCTree.JCCompilationUnit;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Log;\nimport com.sun.tools.javac.util.Options;\nimport java.io.IOException;\nimport java.net.URI;\nimport java.util.Collection;\nimport javax.tools.Diagnostic;\nimport javax.tools.DiagnosticCollector;\nimport javax.tools.DiagnosticListener;\nimport javax.tools.JavaFileObject;\nimport javax.tools.SimpleJavaFileObject;\nimport javax.tools.StandardLocation;\n\n/**\n * This is google-java-format, a new Java formatter that follows the Google Java Style Guide quite precisely---to the\n * letter and to the spirit.\n *\n * <p>This formatter uses the javac parser to generate an AST. Because the AST loses information about the non-tokens in\n * the input (including newlines, comments, etc.), and even some tokens (e.g., optional commas or semicolons), this\n * formatter lexes the input again and follows along in the resulting list of tokens. Its lexer splits all\n * multi-character operators (like \"&gt;&gt;\") into multiple single-character operators. Each non-token is assigned to a\n * token---non-tokens following a token on the same line go with that token; those following go with the next token---\n * and there is a final EOF token to hold final comments.\n *\n * <p>The formatter walks the AST to generate a Greg Nelson/Derek Oppen-style list of formatting {@link Op}s [1--2] that\n * then generates a structured {@link Doc}. Each AST node type has a visitor to emit a sequence of {@link Op}s for the\n * node.\n *\n * <p>Some data-structure operations are easier in the list of {@link Op}s, while others become easier in the\n * {@link Doc}. The {@link Op}s are walked to attach the comments. As the {@link Op}s are generated, missing input\n * tokens are inserted and incorrect output tokens are dropped, ensuring that the output matches the input even in the\n * face of formatter errors. Finally, the formatter walks the {@link Doc} to format it in the given width.\n *\n * <p>This formatter also produces data structures of which tokens and comments appear where on the input, and on the\n * output, to help output a partial reformatting of a slightly edited input.\n *\n * <p>Instances of the formatter are immutable and thread-safe.\n *\n * <p>[1] Nelson, Greg, and John DeTreville. Personal communication.\n *\n * <p>[2] Oppen, Derek C. \"Prettyprinting\". ACM Transactions on Programming Languages and Systems, Volume 2 Issue 4,\n * Oct. 1980, pp. 465\u2013483.\n */\n@Immutable\npublic final class Formatter {\n\n    static final Range<Integer> EMPTY_RANGE = Range.closedOpen(-1, -1);\n\n    private final JavaFormatterOptions options;\n    private final boolean debugMode;\n\n    @VisibleForTesting\n    Formatter(JavaFormatterOptions options, boolean debugMode) {\n        this.options = options;\n        this.debugMode = debugMode;\n    }\n\n    /** A new Formatter instance with default options. */\n    public static Formatter create() {\n        return new Formatter(JavaFormatterOptions.defaultOptions(), false);\n    }\n\n    public static Formatter createFormatter(JavaFormatterOptions options) {\n        return new Formatter(options, false);\n    }\n\n    /**\n     * Construct a {@code Formatter} given a Java compilation unit. Parses the code; builds a {@link JavaInput} and the\n     * corresponding {@link JavaOutput}.\n     *\n     * @param javaInput the input, a Java compilation unit\n     * @param options the {@link JavaFormatterOptions}\n     * @param commentsHelper the {@link CommentsHelper}, used to rewrite comments\n     * @param debugMode whether to produce debugging output via {@link DebugRenderer}\n     * @return javaOutput the output produced\n     */\n    static JavaOutput format(\n            final JavaInput javaInput, JavaFormatterOptions options, CommentsHelper commentsHelper, boolean debugMode)\n            throws FormatterException {\n\n        Context context = new Context();\n        Options.instance(context).put(\"allowStringFolding\", \"false\");\n\n        JCCompilationUnit unit = parseJcCompilationUnit(context, javaInput.getText());\n\n        // Output the compilation unit.\n        javaInput.setCompilationUnit(unit);\n        OpsBuilder opsBuilder = new OpsBuilder(javaInput);\n\n        JavaInputAstVisitor visitor;\n        if (getRuntimeVersion() >= 14) {\n            try {\n                visitor = Class.forName(\"com.palantir.javaformat.java.java14.Java14InputAstVisitor\")\n                        .asSubclass(JavaInputAstVisitor.class)\n                        .getConstructor(OpsBuilder.class, int.class)\n                        .newInstance(opsBuilder, options.indentationMultiplier());\n            } catch (ReflectiveOperationException e) {\n                throw new RuntimeException(e.getMessage(), e);\n            }\n        } else {\n            visitor = new JavaInputAstVisitor(opsBuilder, options.indentationMultiplier());\n        }\n\n        visitor.scan(unit, null);\n        opsBuilder.sync(javaInput.getText().length());\n        opsBuilder.drain();\n        OpsOutput opsOutput = opsBuilder.build();\n\n        Level doc = new DocBuilder().withOps(opsOutput.ops()).build();\n\n        // Don't even allocate all those JSON nodes if we're not going to write it out\n        Sink sink = debugMode ? new JsonSink() : new NoopSink();\n\n        Obs.ExplorationNode observationNode = Obs.createRoot(sink);\n        State finalState =\n                doc.computeBreaks(commentsHelper, options.maxLineLength(), State.startingState(), observationNode);\n\n        JavaOutput javaOutput = new JavaOutput(javaInput, opsOutput.inputMetadata());\n        doc.write(finalState, javaOutput);\n        javaOutput.flush();\n\n        if (debugMode) {\n            DebugRenderer.render(javaInput, opsOutput, doc, finalState, javaOutput, sink.getOutput());\n        }\n        return javaOutput;\n    }\n\n    static JCCompilationUnit parseJcCompilationUnit(Context context, String sourceText) throws FormatterException {\n        DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>();\n        context.put(DiagnosticListener.class, diagnostics);\n        Options.instance(context).put(\"--enable-preview\", \"true\");\n        JCCompilationUnit unit;\n        JavacFileManager fileManager = new JavacFileManager(context, true, UTF_8);\n        try {\n            fileManager.setLocation(StandardLocation.PLATFORM_CLASS_PATH, ImmutableList.of());\n        } catch (IOException e) {\n            // impossible\n            throw new RuntimeException(e);\n        }\n        SimpleJavaFileObject source = new SimpleJavaFileObject(URI.create(\"source\"), JavaFileObject.Kind.SOURCE) {\n            @Override\n            public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {\n                return sourceText;\n            }\n        };\n        Log.instance(context).useSource(source);\n        ParserFactory parserFactory = ParserFactory.instance(context);\n        JavacParser parser = parserFactory.newParser(\n                sourceText, /*keepDocComments=*/ true, /*keepEndPos=*/ true, /*keepLineMap=*/ true);\n        unit = parser.parseCompilationUnit();\n        unit.sourcefile = source;\n\n        Iterable<Diagnostic<? extends JavaFileObject>> errorDiagnostics =\n                Iterables.filter(diagnostics.getDiagnostics(), Formatter::errorDiagnostic);\n        if (!Iterables.isEmpty(errorDiagnostics)) {\n            throw FormatterExceptions.fromJavacDiagnostics(errorDiagnostics);\n        }\n        return unit;\n    }\n\n    @VisibleForTesting\n    static int getRuntimeVersion() {\n        return Runtime.version().feature();\n    }\n\n    static boolean errorDiagnostic(Diagnostic<?> input) {\n        if (input.getKind() != Diagnostic.Kind.ERROR) {\n            return false;\n        }\n        switch (input.getCode()) {\n            case \"compiler.err.invalid.meth.decl.ret.type.req\":\n                // accept constructor-like method declarations that don't match the name of their\n                // enclosing class\n                return false;\n            default:\n                break;\n        }\n        return true;\n    }\n\n    /**\n     * Format the given input (a Java compilation unit) into the output stream.\n     *\n     * @throws FormatterException if the input cannot be parsed\n     */\n    public void formatSource(CharSource input, CharSink output) throws FormatterException, IOException {\n        // TODO(cushon): proper support for streaming input/output. Input may\n        // not be feasible (parsing) but output should be easier.\n        output.write(formatSource(input.read()));\n    }\n\n    /**\n     * Format an input string (a Java compilation unit) into an output string.\n     *\n     * <p>Leaves import statements untouched.\n     *\n     * @param input the input string\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public String formatSource(String input) throws FormatterException {\n        return formatSource(input, ImmutableList.of(Range.closedOpen(0, input.length())));\n    }\n\n    /**\n     * Formats an input string (a Java compilation unit) and fixes imports.\n     *\n     * <p>Fixing imports includes ordering, spacing, and removal of unused import statements.\n     *\n     * @param input the input string\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     * @see <a href=\"https://google.github.io/styleguide/javaguide.html#s3.3.3-import-ordering-and-spacing\">Google Java\n     *     Style Guide - 3.3.3 Import ordering and spacing</a>\n     */\n    public String formatSourceAndFixImports(String input) throws FormatterException {\n        input = ImportOrderer.reorderImports(input, options.style());\n        input = RemoveUnusedImports.removeUnusedImports(input);\n        String formatted = formatSource(input);\n        formatted = StringWrapper.wrap(options.maxLineLength(), formatted, this);\n        return formatted;\n    }\n\n    /**\n     * Format an input string (a Java compilation unit), for only the specified character ranges. These ranges are\n     * extended as necessary (e.g., to encompass whole lines).\n     *\n     * @param input the input string\n     * @param characterRanges the character ranges to be reformatted\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public String formatSource(String input, Collection<Range<Integer>> characterRanges) throws FormatterException {\n        return Utils.applyReplacements(input, getFormatReplacements(input, characterRanges));\n    }\n\n    /**\n     * Emit a list of {@link Replacement}s to convert from input to output.\n     *\n     * @param input the input compilation unit\n     * @param characterRanges the character ranges to reformat\n     * @return a list of {@link Replacement}s, sorted from low index to high index, without overlaps\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public ImmutableList<Replacement> getFormatReplacements(String input, Collection<Range<Integer>> characterRanges)\n            throws FormatterException {\n        JavaInput javaInput = new JavaInput(input);\n\n        // TODO(cushon): this is only safe because the modifier ordering doesn't affect whitespace,\n        // and doesn't change the replacements that are output. This is not true in general for\n        // 'de-linting' changes (e.g. import ordering).\n        javaInput = ModifierOrderer.reorderModifiers(javaInput, characterRanges);\n\n        JavaCommentsHelper commentsHelper = new JavaCommentsHelper(javaInput.getLineSeparator(), options);\n        JavaOutput javaOutput;\n        try {\n            javaOutput = format(javaInput, options, commentsHelper, debugMode);\n        } catch (FormattingError e) {\n            throw new FormatterException(e.diagnostics());\n        }\n        RangeSet<Integer> tokenRangeSet = javaInput.characterRangesToTokenRanges(characterRanges);\n        return javaOutput.getFormatReplacements(tokenRangeSet);\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaOutput.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.DiscreteDomain;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.Input.Token;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.OpsBuilder.BlankLineWanted;\nimport com.palantir.javaformat.Output;\nimport com.palantir.javaformat.doc.State;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\n\n/*\n * Throughout this file, {@code i} is an index for input lines, {@code j} is an index for output\n * lines, {@code ij} is an index into either input or output lines, and {@code k} is an index for\n * toks.\n */\n\n/**\n * {@code JavaOutput} extends {@link Output Output} to represent a Java output document. It includes methods to emit the\n * output document.\n */\npublic final class JavaOutput extends Output {\n    private final String lineSeparator;\n    private final JavaInput javaInput; // Used to follow along while emitting the output.\n\n    private final List<String> mutableLines = new ArrayList<>();\n    private final int kN; // The number of tokens or comments in the input, excluding the EOF.\n    private final InputMetadata inputMetadata;\n    private int iLine = 0; // Closest corresponding line number on input.\n    private int lastK = -1; // Last {@link Tok} index output.\n    private int spacesPending = 0;\n    private int newlinesPending = 0;\n    private StringBuilder lineBuilder = new StringBuilder();\n\n    /**\n     * {@code JavaOutput} constructor.\n     *\n     * @param javaInput the {@link JavaInput}, used to match up blank lines in the output\n     */\n    public JavaOutput(JavaInput javaInput, InputMetadata inputMetadata) {\n        this.lineSeparator = javaInput.getLineSeparator();\n        this.javaInput = javaInput;\n        kN = javaInput.getkN();\n        this.inputMetadata = inputMetadata;\n    }\n\n    // TODO(jdd): Add invariant.\n    @Override\n    public void append(State state, String text, Range<Integer> range) {\n        if (!range.isEmpty()) {\n            boolean sawNewlines = false;\n            // Skip over input line we've passed.\n            int iN = javaInput.getLineCount();\n            while (iLine < iN\n                    && (javaInput.getRanges(iLine).isEmpty()\n                            || javaInput.getRanges(iLine).upperEndpoint() <= range.lowerEndpoint())) {\n                if (javaInput.getRanges(iLine).isEmpty()) {\n                    // Skipped over a blank line.\n                    sawNewlines = true;\n                }\n                ++iLine;\n            }\n            /*\n             * Output blank line if we've called {@link OpsBuilder#blankLine}{@code (true)} here, or if\n             * there's a blank line here and it's a comment.\n             */\n            BlankLineWanted wanted = inputMetadata.blankLines().getOrDefault(lastK, BlankLineWanted.NO);\n            if (isComment(text) ? sawNewlines : wanted.wanted(state).orElse(sawNewlines)) {\n                ++newlinesPending;\n            }\n        }\n        if (Newlines.isNewline(text)) {\n            /*\n             * Don't update range information, and swallow extra newlines. The case below for '\\n' is for\n             * block comments.\n             */\n            if (newlinesPending == 0) {\n                ++newlinesPending;\n            }\n            spacesPending = 0;\n        } else {\n            boolean rangesSet = false;\n            int textN = text.length();\n            for (int i = 0; i < textN; i++) {\n                char c = text.charAt(i);\n                switch (c) {\n                    case ' ':\n                        ++spacesPending;\n                        break;\n                    case '\\r':\n                        if (i + 1 < text.length() && text.charAt(i + 1) == '\\n') {\n                            i++;\n                        }\n                        // falls through\n                    case '\\n':\n                        spacesPending = 0;\n                        ++newlinesPending;\n                        break;\n                    default:\n                        while (newlinesPending > 0) {\n                            // drop leading blank lines\n                            if (!mutableLines.isEmpty() || lineBuilder.length() > 0) {\n                                mutableLines.add(lineBuilder.toString());\n                            }\n                            lineBuilder = new StringBuilder();\n                            rangesSet = false;\n                            --newlinesPending;\n                        }\n                        while (spacesPending > 0) {\n                            lineBuilder.append(' ');\n                            --spacesPending;\n                        }\n                        lineBuilder.append(c);\n                        if (!range.isEmpty()) {\n                            if (!rangesSet) {\n                                while (ranges.size() <= mutableLines.size()) {\n                                    ranges.add(Formatter.EMPTY_RANGE);\n                                }\n                                ranges.set(mutableLines.size(), union(ranges.get(mutableLines.size()), range));\n                                rangesSet = true;\n                            }\n                        }\n                }\n            }\n        }\n        if (!range.isEmpty()) {\n            lastK = range.upperEndpoint();\n        }\n    }\n\n    @Override\n    public void indent(int indent) {\n        spacesPending = indent;\n    }\n\n    /** Flush any incomplete last line, then add the EOF token into our data structures. */\n    void flush() {\n        String lastLine = lineBuilder.toString();\n        if (!CharMatcher.whitespace().matchesAllOf(lastLine)) {\n            mutableLines.add(lastLine);\n        }\n        int jN = mutableLines.size();\n        Range<Integer> eofRange = Range.closedOpen(kN, kN + 1);\n        while (ranges.size() < jN) {\n            ranges.add(Formatter.EMPTY_RANGE);\n        }\n        ranges.add(eofRange);\n        setLines(ImmutableList.copyOf(mutableLines));\n    }\n\n    // The following methods can be used after the Output has been built.\n\n    /**\n     * Emit a list of {@link Replacement}s to convert from input to output.\n     *\n     * @return a list of {@link Replacement}s, sorted by start index, without overlaps\n     */\n    public ImmutableList<Replacement> getFormatReplacements(RangeSet<Integer> iRangeSet0) {\n        ImmutableList.Builder<Replacement> result = ImmutableList.builder();\n        Map<Integer, Range<Integer>> kToJ = JavaOutput.makeKToIJ(this);\n\n        // Expand the token ranges to align with re-formattable boundaries.\n        RangeSet<Integer> breakableRanges = TreeRangeSet.create();\n        RangeSet<Integer> iRangeSet = iRangeSet0.subRangeSet(Range.closed(0, javaInput.getkN()));\n        for (Range<Integer> iRange : iRangeSet.asRanges()) {\n            Range<Integer> range = expandToBreakableRegions(iRange.canonical(DiscreteDomain.integers()));\n            if (range.equals(EMPTY_RANGE)) {\n                // the range contains only whitespace\n                continue;\n            }\n            breakableRanges.add(range);\n        }\n\n        // Construct replacements for each reformatted region.\n        for (Range<Integer> range : breakableRanges.asRanges()) {\n\n            Input.Tok startTok = startTok(javaInput.getToken(range.lowerEndpoint()));\n            Input.Tok endTok = endTok(javaInput.getToken(range.upperEndpoint() - 1));\n\n            // Add all output lines in the given token range to the replacement.\n            StringBuilder replacement = new StringBuilder();\n\n            int replaceFrom = startTok.getPosition();\n            // Replace leading whitespace in the input with the whitespace from the formatted file\n            while (replaceFrom > 0) {\n                char previous = javaInput.getText().charAt(replaceFrom - 1);\n                if (!CharMatcher.whitespace().matches(previous)) {\n                    break;\n                }\n                replaceFrom--;\n            }\n\n            int i = kToJ.get(startTok.getIndex()).lowerEndpoint();\n            // Include leading blank lines from the formatted output, unless the formatted range\n            // starts at the beginning of the file.\n            while (i > 0 && getLine(i - 1).isEmpty()) {\n                i--;\n            }\n            // Write out the formatted range.\n            for (; i < kToJ.get(endTok.getIndex()).upperEndpoint(); i++) {\n                // It's possible to run out of output lines (e.g. if the input ended with\n                // multiple trailing newlines).\n                if (i < getLineCount()) {\n                    if (i > 0) {\n                        replacement.append(lineSeparator);\n                    }\n                    replacement.append(getLine(i));\n                }\n            }\n\n            int replaceTo = Math.min(\n                    endTok.getPosition() + endTok.length(), javaInput.getText().length());\n            // If the formatted ranged ended in the trailing trivia of the last token before EOF,\n            // format all the way up to EOF to deal with trailing whitespace correctly.\n            if (endTok.getIndex() == javaInput.getkN() - 1) {\n                replaceTo = javaInput.getText().length();\n            }\n            // Replace trailing whitespace in the input with the whitespace from the formatted file.\n            // If the trailing whitespace in the input includes one or more line breaks, preserve the\n            // whitespace after the last newline to avoid re-indenting the line following the formatted\n            // line.\n            int newline = -1;\n            while (replaceTo < javaInput.getText().length()) {\n                char next = javaInput.getText().charAt(replaceTo);\n                if (!CharMatcher.whitespace().matches(next)) {\n                    break;\n                }\n                int newlineLength = Newlines.hasNewlineAt(javaInput.getText(), replaceTo);\n                if (newlineLength != -1) {\n                    newline = replaceTo;\n                    // Skip over the entire newline; don't count the second character of \\r\\n as a newline.\n                    replaceTo += newlineLength;\n                } else {\n                    replaceTo++;\n                }\n            }\n            if (newline != -1) {\n                replaceTo = newline;\n            }\n\n            if (newline == -1) {\n                // There wasn't an existing trailing newline; add one.\n                replacement.append(lineSeparator);\n            }\n            for (; i < getLineCount(); i++) {\n                String after = getLine(i);\n                int idx = CharMatcher.whitespace().negate().indexIn(after);\n                if (idx == -1) {\n                    // Write out trailing empty lines from the formatted output.\n                    replacement.append(lineSeparator);\n                } else {\n                    if (newline == -1) {\n                        // If there wasn't a trailing newline in the input, indent the next line.\n                        replacement.append(after, 0, idx);\n                    }\n                    break;\n                }\n            }\n\n            result.add(Replacement.create(replaceFrom, replaceTo, replacement.toString()));\n        }\n        return result.build();\n    }\n\n    /**\n     * Expand a token range to start and end on acceptable boundaries for re-formatting.\n     *\n     * @param iRange the {@link Range} of tokens\n     * @return the expanded token range\n     */\n    private Range<Integer> expandToBreakableRegions(Range<Integer> iRange) {\n        // The original line range.\n        int loTok = iRange.lowerEndpoint();\n        int hiTok = iRange.upperEndpoint() - 1;\n\n        // Expand the token indices to formattable boundaries (e.g. edges of statements).\n        if (!partialFormatRanges().contains(loTok) || !partialFormatRanges().contains(hiTok)) {\n            return EMPTY_RANGE;\n        }\n        loTok = partialFormatRanges().rangeContaining(loTok).lowerEndpoint();\n        hiTok = partialFormatRanges().rangeContaining(hiTok).upperEndpoint();\n        return Range.closedOpen(loTok, hiTok + 1);\n    }\n\n    private RangeSet<Integer> partialFormatRanges() {\n        return inputMetadata.partialFormatRanges();\n    }\n\n    /** The earliest position of any Tok in the Token, including leading whitespace. */\n    public static int startPosition(Token token) {\n        int min = token.getTok().getPosition();\n        for (Input.Tok tok : token.getToksBefore()) {\n            min = Math.min(min, tok.getPosition());\n        }\n        return min;\n    }\n\n    /** The earliest non-whitespace Tok in the Token. */\n    public static Input.Tok startTok(Token token) {\n        for (Input.Tok tok : token.getToksBefore()) {\n            if (tok.getIndex() >= 0) {\n                return tok;\n            }\n        }\n        return token.getTok();\n    }\n\n    /** The last non-whitespace Tok in the Token. */\n    public static Input.Tok endTok(Token token) {\n        for (int i = token.getToksAfter().size() - 1; i >= 0; i--) {\n            Input.Tok tok = token.getToksAfter().get(i);\n            if (tok.getIndex() >= 0) {\n                return tok;\n            }\n        }\n        return token.getTok();\n    }\n\n    private boolean isComment(String text) {\n        return text.startsWith(\"//\") || text.startsWith(\"/*\");\n    }\n\n    private static Range<Integer> union(Range<Integer> x, Range<Integer> y) {\n        return x.isEmpty() ? y : y.isEmpty() ? x : x.span(y).canonical(DiscreteDomain.integers());\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n                .add(\"iLine\", iLine)\n                .add(\"lastK\", lastK)\n                .add(\"spacesPending\", spacesPending)\n                .add(\"newlinesPending\", newlinesPending)\n                .add(\"inputMetadata\", inputMetadata)\n                .add(\"super\", super.toString())\n                .toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/InputMetadataBuilder.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.java;\n\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableRangeSet;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.OpsBuilder.BlankLineWanted;\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic final class InputMetadataBuilder {\n    private final Map<Integer, BlankLineWanted> blankLines = new HashMap<>(); // Info on blank lines.\n    private final RangeSet<Integer> partialFormatRanges = TreeRangeSet.create();\n\n    /**\n     * A blank line is or is not wanted here.\n     *\n     * @param k the {@link Input.Tok} index\n     * @param wanted whether a blank line is wanted here\n     */\n    public void blankLine(int k, BlankLineWanted wanted) {\n        if (blankLines.containsKey(k)) {\n            blankLines.put(k, blankLines.get(k).merge(wanted));\n        } else {\n            blankLines.put(k, wanted);\n        }\n    }\n\n    /** Marks a region that can be partially formatted. */\n    public void markForPartialFormat(Input.Token start, Input.Token end) {\n        int lo = JavaOutput.startTok(start).getIndex();\n        int hi = JavaOutput.endTok(end).getIndex();\n        partialFormatRanges.add(Range.closed(lo, hi));\n    }\n\n    public InputMetadata build() {\n        return ImmutableInputMetadata.builder()\n                .blankLines(ImmutableMap.copyOf(blankLines))\n                .partialFormatRanges(ImmutableRangeSet.copyOf(partialFormatRanges))\n                .build();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/OpsBuilder.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Preconditions;\nimport com.google.common.collect.ArrayListMultimap;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Iterables;\nimport com.google.common.collect.Multimap;\nimport com.palantir.javaformat.Indent.Const;\nimport com.palantir.javaformat.Input.Tok;\nimport com.palantir.javaformat.doc.Break;\nimport com.palantir.javaformat.doc.BreakTag;\nimport com.palantir.javaformat.doc.Comment;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\nimport com.palantir.javaformat.doc.FillMode;\nimport com.palantir.javaformat.doc.NonBreakingSpace;\nimport com.palantir.javaformat.doc.State;\nimport com.palantir.javaformat.doc.Token;\nimport com.palantir.javaformat.doc.Token.RealOrImaginary;\nimport com.palantir.javaformat.java.FormatterDiagnostic;\nimport com.palantir.javaformat.java.InputMetadata;\nimport com.palantir.javaformat.java.InputMetadataBuilder;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Optional;\nimport org.immutables.value.Value;\n\n/** An {@code OpsBuilder} creates a list of {@link Op}s, which is turned into a {@link Doc} by {@link DocBuilder}. */\npublic final class OpsBuilder {\n\n    /** Return the actual size of the AST node at position, including comments. */\n    public int actualSize(int position, int length) {\n        Input.Token startToken = input.getPositionTokenMap().get(position);\n        int start = startToken.getTok().getPosition();\n        for (Input.Tok tok : startToken.getToksBefore()) {\n            if (tok.isComment()) {\n                start = Math.min(start, tok.getPosition());\n            }\n        }\n        Input.Token endToken = input.getPositionTokenMap().get(position + length - 1);\n        int end = endToken.getTok().getPosition() + endToken.getTok().length();\n        for (Input.Tok tok : endToken.getToksAfter()) {\n            if (tok.isComment()) {\n                end = Math.max(end, tok.getPosition() + tok.length());\n            }\n        }\n        return end - start;\n    }\n\n    /** Return the start column of the token at {@code position}, including leading comments. */\n    public Integer actualStartColumn(int position) {\n        Input.Token startToken = input.getPositionTokenMap().get(position);\n        int start = startToken.getTok().getPosition();\n        int line0 = input.getLineNumber(start);\n        for (Input.Tok tok : startToken.getToksBefore()) {\n            if (line0 != input.getLineNumber(tok.getPosition())) {\n                return start;\n            }\n            if (tok.isComment()) {\n                start = Math.min(start, tok.getPosition());\n            }\n        }\n        return start;\n    }\n\n    /** A request to add or remove a blank line in the output. */\n    public abstract static class BlankLineWanted {\n\n        /** Always emit a blank line. */\n        public static final BlankLineWanted YES = new SimpleBlankLine(Optional.of(true));\n\n        /** Never emit a blank line. */\n        public static final BlankLineWanted NO = new SimpleBlankLine(Optional.of(false));\n\n        /**\n         * Explicitly preserve blank lines from the input (e.g. before the first member in a class declaration).\n         * Overrides conditional blank lines.\n         */\n        public static final BlankLineWanted PRESERVE = new SimpleBlankLine(/* wanted= */ Optional.empty());\n\n        /** Is the blank line wanted? */\n        public abstract Optional<Boolean> wanted(State state);\n\n        /** Merge this blank line request with another. */\n        public abstract BlankLineWanted merge(BlankLineWanted wanted);\n\n        /** Emit a blank line if the given break is taken. */\n        public static BlankLineWanted conditional(BreakTag breakTag) {\n            return new ConditionalBlankLine(ImmutableList.of(breakTag));\n        }\n\n        private static final class SimpleBlankLine extends BlankLineWanted {\n            private final Optional<Boolean> wanted;\n\n            SimpleBlankLine(Optional<Boolean> wanted) {\n                this.wanted = wanted;\n            }\n\n            @Override\n            public Optional<Boolean> wanted(State _state) {\n                return wanted;\n            }\n\n            @Override\n            public BlankLineWanted merge(BlankLineWanted other) {\n                return this;\n            }\n        }\n\n        private static final class ConditionalBlankLine extends BlankLineWanted {\n\n            private final ImmutableList<BreakTag> tags;\n\n            ConditionalBlankLine(Iterable<BreakTag> tags) {\n                this.tags = ImmutableList.copyOf(tags);\n            }\n\n            @Override\n            public Optional<Boolean> wanted(State state) {\n                for (BreakTag tag : tags) {\n                    if (state.wasBreakTaken(tag)) {\n                        return Optional.of(true);\n                    }\n                }\n                return Optional.empty();\n            }\n\n            @Override\n            public BlankLineWanted merge(BlankLineWanted other) {\n                if (!(other instanceof ConditionalBlankLine)) {\n                    return other;\n                }\n                return new ConditionalBlankLine(Iterables.concat(this.tags, ((ConditionalBlankLine) other).tags));\n            }\n        }\n    }\n\n    private final Input input;\n    private final List<Op> ops = new ArrayList<>();\n    /** Used to record blank-line information. */\n    private final InputMetadataBuilder inputMetadataBuilder = new InputMetadataBuilder();\n\n    private static final Const ZERO = Const.ZERO;\n\n    private int tokenI = 0;\n    private int inputPosition = Integer.MIN_VALUE;\n\n    /** The number of unclosed open ops in the input stream. */\n    int depth = 0;\n\n    /** Add an {@link Op}, and record open/close ops for later validation of unclosed levels. */\n    private void add(Op op) {\n        if (op instanceof OpenOp) {\n            depth++;\n        } else if (op instanceof CloseOp) {\n            depth--;\n            if (depth < 0) {\n                throw new IllegalStateException();\n            }\n        }\n        ops.add(op);\n    }\n\n    /** Add a list of {@link Op}s. */\n    public void addAll(List<Op> ops) {\n        for (Op op : ops) {\n            add(op);\n        }\n    }\n\n    /**\n     * The {@code OpsBuilder} constructor.\n     *\n     * @param input the {@link Input}, used for retrieve information from the AST\n     */\n    public OpsBuilder(Input input) {\n        this.input = input;\n    }\n\n    /** Get the {@code OpsBuilder}'s {@link Input}. */\n    public Input getInput() {\n        return input;\n    }\n\n    /** Returns the number of unclosed open ops in the input stream. */\n    public int depth() {\n        return depth;\n    }\n\n    /**\n     * Checks that all open ops in the op stream have matching close ops.\n     *\n     * @throws FormattingError if any ops were unclosed\n     */\n    public void checkClosed(int previous) {\n        if (depth != previous) {\n            throw new FormattingError(diagnostic(String.format(\"saw %d unclosed ops\", depth)));\n        }\n    }\n\n    /** Create a {@link FormatterDiagnostic} at the current position. */\n    public FormatterDiagnostic diagnostic(String message) {\n        return input.createDiagnostic(inputPosition, message);\n    }\n\n    /**\n     * Sync to position in the input. If we've skipped outputting any tokens that were present in the input tokens,\n     * output them here and optionally complain.\n     *\n     * @param inputPosition the {@code 0}-based input position\n     */\n    public void sync(int inputPosition) {\n        if (inputPosition > this.inputPosition) {\n            ImmutableList<? extends Input.Token> tokens = input.getTokens();\n            int tokensN = tokens.size();\n            this.inputPosition = inputPosition;\n            if (tokenI < tokensN && inputPosition > tokens.get(tokenI).getTok().getPosition()) {\n                // Found a missing input token. Insert it and mark it missing (usually not good).\n                Input.Token token = tokens.get(tokenI++);\n                throw new FormattingError(diagnostic(String.format(\n                        \"did not generate token \\\"%s\\\"\", token.getTok().getText())));\n            }\n        }\n    }\n\n    /** Output any remaining tokens from the input stream (e.g. terminal whitespace). */\n    public void drain() {\n        int inputPosition = input.getText().length() + 1;\n        if (inputPosition > this.inputPosition) {\n            ImmutableList<? extends Input.Token> tokens = input.getTokens();\n            int tokensN = tokens.size();\n            while (tokenI < tokensN\n                    && inputPosition > tokens.get(tokenI).getTok().getPosition()) {\n                Input.Token token = tokens.get(tokenI++);\n                add(Token.make(\n                        token,\n                        Token.RealOrImaginary.IMAGINARY,\n                        ZERO,\n                        /* breakAndIndentTrailingComment= */ Optional.empty()));\n            }\n        }\n        this.inputPosition = inputPosition;\n        checkClosed(0);\n    }\n\n    /**\n     * Open a new level by emitting an {@link OpenOp}.\n     *\n     * @param plusIndent the extra indent for the new level\n     */\n    public void open(Indent plusIndent) {\n        add(OpenOp.make(plusIndent));\n    }\n\n    /**\n     * Open a new level by emitting an {@link OpenOp}.\n     *\n     * @param debugName a representative name for this lambda\n     * @param plusIndent the extra indent for the new level\n     */\n    public void open(String debugName, Indent plusIndent) {\n        add(OpenOp.builder().plusIndent(plusIndent).debugName(debugName).build());\n    }\n\n    /**\n     * Open a new level by emitting an {@link OpenOp}.\n     *\n     * @param plusIndent the extra indent for the new level\n     * @param breakBehaviour how to decide whether to break this level or not\n     * @param breakabilityIfLastLevel if last level, when to break this rather than parent\n     */\n    public void open(Indent plusIndent, BreakBehaviour breakBehaviour, LastLevelBreakability breakabilityIfLastLevel) {\n        add(OpenOp.builder()\n                .plusIndent(plusIndent)\n                .breakBehaviour(breakBehaviour)\n                .breakabilityIfLastLevel(breakabilityIfLastLevel)\n                .build());\n    }\n\n    public void open(OpenOp openOp) {\n        add(openOp);\n    }\n\n    /** Close the current level, by emitting a {@link CloseOp}. */\n    public void close() {\n        add(CloseOp.make());\n    }\n\n    /** Return the text of the next {@link Input.Token}, or absent if there is none. */\n    public Optional<String> peekToken() {\n        return peekToken(0);\n    }\n\n    /** Return whether the last token emitted is followed by a newline. */\n    public boolean mostRecentTokenFollowedByNewline() {\n        Preconditions.checkState(tokenI > 0, \"No token was emitted yet\");\n        return input.getTokens().get(tokenI - 1).getToksAfter().stream().anyMatch(Tok::isNewline);\n    }\n\n    /** Return the text of an upcoming {@link Input.Token}, or absent if there is none. */\n    public Optional<String> peekToken(int skip) {\n        ImmutableList<? extends Input.Token> tokens = input.getTokens();\n        int idx = tokenI + skip;\n        return idx < tokens.size() ? Optional.of(tokens.get(idx).getTok().getOriginalText()) : Optional.empty();\n    }\n\n    /**\n     * Emit an optional token iff it exists on the input. This is used to emit tokens whose existence has been lost in\n     * the AST.\n     *\n     * @param token the optional token\n     */\n    public void guessToken(String token) {\n        token(token, Token.RealOrImaginary.IMAGINARY, ZERO, /* breakAndIndentTrailingComment=  */ Optional.empty());\n    }\n\n    public void token(\n            String token,\n            Token.RealOrImaginary realOrImaginary,\n            Indent plusIndentCommentsBefore,\n            Optional<Indent> breakAndIndentTrailingComment) {\n        ImmutableList<? extends Input.Token> tokens = input.getTokens();\n        if (token.equals(peekToken().orElse(null))) { // Found the input token. Output it.\n            add(Token.make(\n                    tokens.get(tokenI++),\n                    RealOrImaginary.REAL,\n                    plusIndentCommentsBefore,\n                    breakAndIndentTrailingComment));\n        } else {\n            /*\n             * Generated a \"bad\" token, which doesn't exist on the input. Drop it, and complain unless\n             * (for example) we're guessing at an optional token.\n             */\n            if (realOrImaginary == Token.RealOrImaginary.REAL) {\n                throw new FormattingError(diagnostic(String.format(\n                        \"expected token: '%s'; generated %s instead\",\n                        peekToken().orElse(null), token)));\n            }\n        }\n    }\n\n    /**\n     * Emit a single- or multi-character op by breaking it into single-character {@link Token}s.\n     *\n     * @param op the operator to emit\n     */\n    public void op(String op) {\n        int opN = op.length();\n        for (int i = 0; i < opN; i++) {\n            token(\n                    op.substring(i, i + 1),\n                    Token.RealOrImaginary.REAL,\n                    ZERO,\n                    /* breakAndIndentTrailingComment=  */ Optional.empty());\n        }\n    }\n\n    /** Emit a {@link NonBreakingSpace}. */\n    public void space() {\n        add(NonBreakingSpace.make());\n    }\n\n    /** Emit a {@link Break}. */\n    public void breakOp() {\n        breakOp(FillMode.UNIFIED, \"\", ZERO);\n    }\n\n    /**\n     * Emit a {@link Break}.\n     *\n     * @param plusIndent extra indent if taken\n     */\n    public void breakOp(Indent plusIndent) {\n        breakOp(FillMode.UNIFIED, \"\", plusIndent);\n    }\n\n    /** Emit a filled {@link Break}. */\n    public void breakToFill() {\n        breakOp(FillMode.INDEPENDENT, \"\", ZERO);\n    }\n\n    /** Emit a forced {@link Break}. */\n    public void forcedBreak() {\n        breakOp(FillMode.FORCED, \"\", ZERO);\n    }\n\n    /**\n     * Emit a forced {@link Break}.\n     *\n     * @param plusIndent extra indent if taken\n     */\n    public void forcedBreak(Indent plusIndent) {\n        breakOp(FillMode.FORCED, \"\", plusIndent);\n    }\n\n    /**\n     * Emit a {@link Break}, with a specified {@code flat} value (e.g., {@code \" \"}).\n     *\n     * @param flat the {@link Break} when not broken\n     */\n    public void breakOp(String flat) {\n        breakOp(FillMode.UNIFIED, flat, ZERO);\n    }\n\n    /**\n     * Emit a {@link Break}, with a specified {@code flat} value (e.g., {@code \" \"}).\n     *\n     * @param flat the {@link Break} when not broken\n     */\n    public void breakToFill(String flat) {\n        breakOp(FillMode.INDEPENDENT, flat, ZERO);\n    }\n\n    /**\n     * Emit a generic {@link Break}.\n     *\n     * @param fillMode the {@link FillMode}\n     * @param flat the {@link Break} when not broken\n     * @param plusIndent extra indent if taken\n     */\n    public void breakOp(FillMode fillMode, String flat, Indent plusIndent) {\n        breakOp(fillMode, flat, plusIndent, /* optionalTag=  */ Optional.empty());\n    }\n\n    /**\n     * Emit a generic {@link Break}.\n     *\n     * @param fillMode the {@link FillMode}\n     * @param flat the {@link Break} when not broken\n     * @param plusIndent extra indent if taken\n     * @param optionalTag an optional tag for remembering whether the break was taken\n     */\n    public void breakOp(FillMode fillMode, String flat, Indent plusIndent, Optional<BreakTag> optionalTag) {\n        add(Break.make(fillMode, flat, plusIndent, optionalTag));\n    }\n\n    /** Emit a self-built {@link Break}. */\n    public void breakOp(Break breakOp) {\n        add(breakOp);\n    }\n\n    private int lastPartialFormatBoundary = -1;\n\n    /**\n     * Make the boundary of a region that can be partially formatted. The boundary will be included in the following\n     * region, e.g.: [[boundary0, boundary1), [boundary1, boundary2), ...].\n     */\n    public void markForPartialFormat() {\n        if (lastPartialFormatBoundary == -1) {\n            lastPartialFormatBoundary = tokenI;\n            return;\n        }\n        if (tokenI == lastPartialFormatBoundary) {\n            return;\n        }\n        Input.Token start = input.getTokens().get(lastPartialFormatBoundary);\n        Input.Token end = input.getTokens().get(tokenI - 1);\n        inputMetadataBuilder.markForPartialFormat(start, end);\n...\n", "needle_token_start": 604, "needle_token_end": 1020, "code_context_ntokens": 16142, "output": ["```java\nprivate static ImmutableList<String> stringComponents(\n        String input, JCTree.JCCompilationUnit unit, List<Tree> flat) {\n    ImmutableList.Builder<String> result = ImmutableList.builder();\n    StringBuilder piece = new StringBuilder();\n    for (Tree tree : flat) {\n        // adjust for leading and trailing double quotes\n        String text = input.substring(getStartPosition(tree) + 1, getEndPosition(unit, tree) - 1);\n        int start = 0;\n        for (int idx = 0; idx < text.length(); idx++) {\n            if (CharMatcher.whitespace().matches(text.charAt(idx))) {\n                // continue below\n            } else if (hasEscapedWhitespaceAt(text, idx) != -1) {\n                // continue below\n            } else if (hasEscapedNewlineAt(text, idx) != -1) {\n                int length;\n                while ((length = hasEscapedNewlineAt(text, idx)) != -1) {\n                    idx += length;\n                }\n            } else {\n                continue;\n            }\n            piece.append(text, start, idx);\n            result.add(piece.toString());\n            piece = new StringBuilder();\n            start = idx;\n        }\n        if (piece.length() > 0) {\n            result.add(piece.toString());\n            piece = new StringBuilder();\n        }\n        if (start < text.length()) {\n            piece.append(text, start, text.length());\n        }\n    }\n    if (piece.length() > 0) {\n        result.add(piece.toString());\n    }\n    return result.build();\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "formatSourceReflowStringsAndFixImports", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/FormatterServiceImpl.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to process a Java source code string by formatting it according to specific style guidelines, reflowing string literals, and organizing import statements.\n2. **Input**: A single string containing the entire Java source code to be processed.\n3. **Output**: A string that represents the formatted Java source code with adjusted string literals and optimized import statements.\n4. **Procedure**: The function leverages an internal formatter object, which applies a predefined style configuration to the input source code. It adjusts the formatting, reflows string literals for readability, and reorganizes the import statements to adhere to coding standards.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/Main.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static java.nio.charset.StandardCharsets.UTF_8;\n\nimport com.google.common.io.ByteStreams;\nimport com.palantir.javaformat.java.JavaFormatterOptions.Style;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStreamWriter;\nimport java.io.PrintWriter;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.Arrays;\nimport java.util.LinkedHashMap;\nimport java.util.Map;\nimport java.util.Objects;\nimport java.util.Optional;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.Future;\n\n/** The main class for the Java formatter CLI. */\npublic final class Main {\n    private static final int MAX_THREADS = 20;\n    private static final String STDIN_FILENAME = \"<stdin>\";\n\n    static String versionString() {\n        return \"palantir-java-format: Version \" + Main.class.getPackage().getImplementationVersion();\n    }\n\n    private final PrintWriter outWriter;\n    private final PrintWriter errWriter;\n    private final InputStream inStream;\n\n    public Main(PrintWriter outWriter, PrintWriter errWriter, InputStream inStream) {\n        this.outWriter = outWriter;\n        this.errWriter = errWriter;\n        this.inStream = inStream;\n    }\n\n    /**\n     * The main method for the formatter, with some number of file names to format. We process them in parallel, but we\n     * must be careful; if multiple file names refer to the same file (which is hard to determine), we must serialize\n     * their updates.\n     *\n     * @param args the command-line arguments\n     */\n    public static void main(String[] args) {\n        int result;\n        PrintWriter out = new PrintWriter(new OutputStreamWriter(System.out, UTF_8));\n        PrintWriter err = new PrintWriter(new OutputStreamWriter(System.err, UTF_8));\n        try {\n            Main formatter = new Main(out, err, System.in);\n            result = formatter.format(args);\n        } catch (UsageException e) {\n            err.print(e.getMessage());\n            result = 0;\n        } finally {\n            err.flush();\n            out.flush();\n        }\n        System.exit(result);\n    }\n\n    /**\n     * The main entry point for the formatter, with some number of file names to format. We process them in parallel,\n     * but we must be careful; if multiple file names refer to the same file (which is hard to determine), we must\n     * serialize their update.\n     *\n     * @param args the command-line arguments\n     */\n    public int format(String... args) throws UsageException {\n        CommandLineOptions parameters = processArgs(args);\n        if (parameters.version()) {\n            errWriter.println(versionString());\n            return 0;\n        }\n        if (parameters.help()) {\n            throw new UsageException();\n        }\n\n        // TODO(someone): update this to always use Style.PALANTIR\n        JavaFormatterOptions options = JavaFormatterOptions.builder()\n                .style(parameters.aosp() ? Style.AOSP : parameters.palantirStyle() ? Style.PALANTIR : Style.GOOGLE)\n                .build();\n\n        if (parameters.stdin()) {\n            return formatStdin(parameters, options);\n        } else {\n            return formatFiles(parameters, options);\n        }\n    }\n\n    private int formatFiles(CommandLineOptions parameters, JavaFormatterOptions options) {\n        int numThreads = Math.min(MAX_THREADS, parameters.files().size());\n        ExecutorService executorService = Executors.newFixedThreadPool(numThreads);\n\n        Map<Path, String> inputs = new LinkedHashMap<>();\n        Map<Path, Future<String>> results = new LinkedHashMap<>();\n        boolean allOk = true;\n\n        for (String fileName : parameters.files()) {\n            if (!fileName.endsWith(\".java\")) {\n                errWriter.println(\"Skipping non-Java file: \" + fileName);\n                continue;\n            }\n            Path path = Paths.get(fileName);\n            String input;\n            try {\n                input = new String(Files.readAllBytes(path), UTF_8);\n                inputs.put(path, input);\n                results.put(path, executorService.submit(new FormatFileCallable(parameters, input, options)));\n            } catch (IOException e) {\n                errWriter.println(fileName + \": could not read file: \" + e.getMessage());\n                allOk = false;\n            }\n        }\n\n        for (Map.Entry<Path, Future<String>> result : results.entrySet()) {\n            Path path = result.getKey();\n            String formatted;\n            try {\n                formatted = result.getValue().get();\n            } catch (InterruptedException e) {\n                errWriter.println(e.getMessage());\n                allOk = false;\n                continue;\n            } catch (ExecutionException e) {\n                if (e.getCause() instanceof FormatterException) {\n                    for (FormatterDiagnostic diagnostic : ((FormatterException) e.getCause()).diagnostics()) {\n                        errWriter.println(path + \":\" + diagnostic.toString());\n                    }\n                } else {\n                    errWriter.println(path + \": error: \"\n                            + Optional.ofNullable(e.getCause())\n                                    .map(Throwable::getMessage)\n                                    .orElse(\"null\"));\n                    Optional.ofNullable(e.getCause()).ifPresent(cause -> cause.printStackTrace(errWriter));\n                }\n                allOk = false;\n                continue;\n            }\n            boolean changed = !formatted.equals(inputs.get(path));\n            if (changed && parameters.setExitIfChanged()) {\n                allOk = false;\n            }\n            if (parameters.inPlace()) {\n                if (!changed) {\n                    continue; // preserve original file\n                }\n                try {\n                    Files.write(path, formatted.getBytes(UTF_8));\n                } catch (IOException e) {\n                    errWriter.println(path + \": could not write file: \" + e.getMessage());\n                    allOk = false;\n                    continue;\n                }\n            } else if (parameters.dryRun()) {\n                if (changed) {\n                    outWriter.println(path);\n                }\n            } else {\n                outWriter.write(formatted);\n            }\n        }\n        return allOk ? 0 : 1;\n    }\n\n    private int formatStdin(CommandLineOptions parameters, JavaFormatterOptions options) {\n        String input;\n        try {\n            input = new String(ByteStreams.toByteArray(inStream), UTF_8);\n        } catch (IOException e) {\n            throw new RuntimeException(e);\n        }\n...\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/FormatterServiceImpl.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.java;\n\nimport com.google.auto.service.AutoService;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Range;\nimport com.palantir.javaformat.java.JavaFormatterOptions.Style;\nimport java.util.Collection;\n\n@AutoService(FormatterService.class)\npublic final class FormatterServiceImpl implements FormatterService {\n\n    private final Formatter formatter;\n\n    public FormatterServiceImpl() {\n        JavaFormatterOptions options =\n                JavaFormatterOptions.builder().style(Style.PALANTIR).build();\n        formatter = Formatter.createFormatter(options);\n    }\n\n    @Override\n    public ImmutableList<Replacement> getFormatReplacements(String text, Collection<Range<Integer>> toRanges)\n            throws FormatterException {\n        return formatter.getFormatReplacements(text, toRanges);\n    }\n\n    \n@Override\n    public String formatSourceReflowStringsAndFixImports(String input) throws FormatterException {\n        return formatter.formatSourceAndFixImports(input);\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/FormatFileCallable.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.fasterxml.jackson.core.JsonProcessingException;\nimport com.fasterxml.jackson.databind.ObjectMapper;\nimport com.fasterxml.jackson.databind.json.JsonMapper;\nimport com.fasterxml.jackson.datatype.guava.GuavaModule;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Utils;\nimport java.util.concurrent.Callable;\n\n/** Encapsulates information about a file to be formatted, including which parts of the file to format. */\nclass FormatFileCallable implements Callable<String> {\n    private final ObjectMapper MAPPER =\n            JsonMapper.builder().addModule(new GuavaModule()).build();\n\n    private final String input;\n    private final CommandLineOptions parameters;\n    private final JavaFormatterOptions options;\n\n    public FormatFileCallable(CommandLineOptions parameters, String input, JavaFormatterOptions options) {\n        this.input = input;\n        this.parameters = parameters;\n        this.options = options;\n    }\n\n    @Override\n    public String call() throws FormatterException {\n        if (parameters.fixImportsOnly()) {\n            return fixImports(input);\n        }\n\n        Formatter formatter = Formatter.createFormatter(options);\n        if (parameters.outputReplacements()) {\n            return formatReplacements(formatter);\n        }\n        return formatFile(formatter);\n    }\n\n    private String formatReplacements(Formatter formatter) throws FormatterException {\n        ImmutableList<Replacement> replacements =\n                formatter.getFormatReplacements(input, characterRanges(input).asRanges());\n        try {\n            return MAPPER.writeValueAsString(replacements);\n        } catch (JsonProcessingException e) {\n            throw new RuntimeException(\"Error serializing replacement output\", e);\n        }\n    }\n\n    private String formatFile(Formatter formatter) throws FormatterException {\n        String formatted = formatter.formatSource(input, characterRanges(input).asRanges());\n        formatted = fixImports(formatted);\n        if (parameters.reflowLongStrings()) {\n            formatted = StringWrapper.wrap(options.maxLineLength(), formatted, formatter);\n        }\n        return formatted;\n    }\n\n    private String fixImports(String input) throws FormatterException {\n        if (parameters.removeUnusedImports()) {\n            input = RemoveUnusedImports.removeUnusedImports(input);\n        }\n        if (parameters.sortImports()) {\n            input = ImportOrderer.reorderImports(input, options.style());\n        }\n        return input;\n    }\n\n    private RangeSet<Integer> characterRanges(String input) {\n        final RangeSet<Integer> characterRanges = TreeRangeSet.create();\n\n        if (parameters.characterRanges().isEmpty()\n                && parameters.lines().isEmpty()\n                && parameters.offsets().isEmpty()) {\n            characterRanges.add(Range.closedOpen(0, input.length()));\n            return characterRanges;\n        }\n\n        if (!parameters.characterRanges().isEmpty()) {\n            characterRanges.addAll(parameters.characterRanges());\n        }\n\n        if (!parameters.lines().isEmpty()) {\n            characterRanges.addAll(Utils.lineRangesToCharRanges(input, parameters.lines()));\n        }\n\n        for (int i = 0; i < parameters.offsets().size(); i++) {\n            Integer length = parameters.lengths().get(i);\n            if (length == 0) {\n                // 0 stands for \"format the line under the cursor\"\n                length = 1;\n            }\n            characterRanges.add(Range.closedOpen(\n                    parameters.offsets().get(i), parameters.offsets().get(i) + length));\n        }\n\n        return characterRanges;\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/UsageException.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static com.google.common.base.Preconditions.checkNotNull;\n\nimport com.google.common.base.Joiner;\nimport javax.annotation.Nullable;\n\n/** Checked exception class for formatter command-line usage errors. */\nfinal class UsageException extends Exception {\n\n    private static final Joiner NEWLINE_JOINER = Joiner.on(System.lineSeparator());\n\n    private static final String[] DOCS_LINK = {\n        \"https://github.com/palantir/palantir-java-format\",\n    };\n\n    private static final String[] USAGE = {\n        \"\",\n        \"Usage: palantir-java-format [options] file(s)\",\n        \"\",\n        \"Options:\",\n        \"  -i, -r, -replace, --replace\",\n        \"    Send formatted output back to files, not stdout.\",\n        \"  -\",\n        \"    Format stdin -> stdout\",\n        \"  --assume-filename, -assume-filename\",\n        \"    File name to use for diagnostics when formatting standard input (default is <stdin>).\",\n        \"  --aosp, -aosp, -a\",\n        \"    Use AOSP style instead of Google Style (4-space indentation).\",\n        \"  --palantir, -palantir\",\n        \"    Use Palantir style instead of Google Style.\",\n        \"  --fix-imports-only\",\n        \"    Fix import order and remove any unused imports, but do no other formatting.\",\n        \"  --skip-sorting-imports\",\n        \"    Do not fix the import order. Unused imports will still be removed.\",\n        \"  --skip-removing-unused-imports\",\n        \"    Do not remove unused imports. Imports will still be sorted.\",\n        \" . --skip-reflowing-long-strings\",\n        \"    Do not reflow string literals that exceed the column limit.\",\n        \"  --dry-run, -n\",\n        \"    Prints the paths of the files whose contents would change if the formatter were run\" + \" normally.\",\n        \"  --set-exit-if-changed\",\n        \"    Return exit code 1 if there are any formatting changes.\",\n        \"  --lines, -lines, --line, -line\",\n        \"    Line range(s) to format, like 5:10 (1-based; default is all).\",\n        \"  --character-ranges, -character-ranges, --character-range, -character-range\",\n        \"    character range(s) to format, like 5:10 (0-based; default is all).\",\n        \"  --offset, -offset\",\n        \"    Character offset to format (0-based; default is all).\",\n        \"  --length, -length\",\n        \"    Character length to format.\",\n        \"  --help, -help, -h\",\n        \"    Print this usage statement.\",\n        \"  --version, -version, -v\",\n        \"    Print the version.\",\n        \"  --output-replacements, -output-replacements\",\n        \"    Outputs formatting replacements instead of the full formatted file.\",\n        \"  @<filename>\",\n        \"    Read options and filenames from file.\",\n        \"\",\n    };\n\n    private static final String[] ADDITIONAL_USAGE = {\n        \"If -i is given with -, the result is sent to stdout.\",\n        \"The --lines, --offset, and --length flags may be given more than once.\",\n        \"The --offset and --length flags must be given an equal number of times.\",\n        \"If --lines, --offset, or --length are given, only one file (or -) may be given.\"\n    };\n\n    UsageException() {\n        super(buildMessage(null));\n    }\n\n    UsageException(String message) {\n        super(buildMessage(checkNotNull(message)));\n    }\n\n    private static String buildMessage(@Nullable String message) {\n        StringBuilder builder = new StringBuilder();\n        if (message != null) {\n            builder.append(message).append('\\n');\n        }\n        appendLines(builder, USAGE);\n        appendLines(builder, ADDITIONAL_USAGE);\n        appendLines(builder, new String[] {\"\"});\n        appendLine(builder, Main.versionString());\n        appendLines(builder, DOCS_LINK);\n        return builder.toString();\n    }\n\n    private static void appendLine(StringBuilder builder, String line) {\n        builder.append(line).append(System.lineSeparator());\n    }\n\n    private static void appendLines(StringBuilder builder, String[] lines) {\n        NEWLINE_JOINER.appendTo(builder, lines).append(System.lineSeparator());\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocLexer.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java.javadoc;\n\nimport static com.google.common.base.Preconditions.checkArgument;\nimport static com.google.common.base.Preconditions.checkNotNull;\nimport static com.google.common.base.Verify.verify;\nimport static com.google.common.collect.Iterators.peekingIterator;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BEGIN_JAVADOC;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BLOCKQUOTE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BLOCKQUOTE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BR_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.CODE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.CODE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.END_JAVADOC;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.FOOTER_JAVADOC_TAG_START;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.FORCED_NEWLINE;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.HEADER_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.HEADER_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.HTML_COMMENT;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.INLINE_TAG_CLOSE;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.INLINE_TAG_OPEN;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_ITEM_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_ITEM_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LITERAL;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.MOE_BEGIN_STRIP_COMMENT;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.MOE_END_STRIP_COMMENT;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.OPTIONAL_LINE_BREAK;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PRE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PRE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.TABLE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.TABLE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.WHITESPACE;\nimport static java.lang.String.format;\nimport static java.util.regex.Pattern.CASE_INSENSITIVE;\nimport static java.util.regex.Pattern.DOTALL;\nimport static java.util.regex.Pattern.compile;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.PeekingIterator;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Deque;\nimport java.util.List;\nimport java.util.regex.Pattern;\n\n/** Lexer for the Javadoc formatter. */\nfinal class JavadocLexer {\n    /** Takes a Javadoc comment, including \u2215\u2731\u2731 and \u2731\u2215, and returns tokens, including \u2215\u2731\u2731 and \u2731\u2215. */\n    static ImmutableList<Token> lex(String input) throws LexException {\n        /*\n         * TODO(cpovirk): In theory, we should interpret Unicode escapes (yet output them in their\n         * original form). This would mean mean everything from an encoded \u2215\u2731\u2731 to an encoded <pre> tag,\n         * so we'll probably never bother.\n         */\n        input = stripJavadocBeginAndEnd(input);\n        input = normalizeLineEndings(input);\n        return new JavadocLexer(new CharStream(input)).generateTokens();\n    }\n\n    /** The lexer crashes on windows line endings, so for now just normalize to `\\n`. */\n    // TODO(cushon): use the platform line separator for output\n    private static String normalizeLineEndings(String input) {\n        return NON_UNIX_LINE_ENDING.matcher(input).replaceAll(\"\\n\");\n    }\n\n    private static final Pattern NON_UNIX_LINE_ENDING = Pattern.compile(\"\\r\\n?\");\n\n    private static String stripJavadocBeginAndEnd(String input) {\n        /*\n         * We do this ahead of time so that the main part of the lexer need not say things like\n         * \"(?![*]/)\" to avoid accidentally swallowing \u2731\u2215 when consuming a newline.\n         */\n        checkArgument(input.startsWith(\"/**\"), \"Missing /**: %s\", input);\n        checkArgument(input.endsWith(\"*/\") && input.length() > 4, \"Missing */: %s\", input);\n        return input.substring(\"/**\".length(), input.length() - \"*/\".length());\n    }\n\n    private final CharStream input;\n    private final NestingCounter braceDepth = new NestingCounter();\n    private final NestingCounter preDepth = new NestingCounter();\n    private final NestingCounter codeDepth = new NestingCounter();\n    private final NestingCounter tableDepth = new NestingCounter();\n    private boolean somethingSinceNewline;\n\n    private JavadocLexer(CharStream input) {\n        this.input = checkNotNull(input);\n    }\n\n    private ImmutableList<Token> generateTokens() throws LexException {\n        ImmutableList.Builder<Token> tokens = ImmutableList.builder();\n\n        Token token = new Token(BEGIN_JAVADOC, \"/**\");\n        tokens.add(token);\n\n        while (!input.isExhausted()) {\n            token = readToken();\n            tokens.add(token);\n        }\n\n        checkMatchingTags();\n\n        token = new Token(END_JAVADOC, \"*/\");\n        tokens.add(token);\n\n        ImmutableList<Token> result = tokens.build();\n        result = joinAdjacentLiteralsAndAdjacentWhitespace(result);\n        result = inferParagraphTags(result);\n        result = optionalizeSpacesAfterLinks(result);\n        result = deindentPreCodeBlocks(result);\n        return result;\n    }\n\n    private Token readToken() throws LexException {\n        Token.Type type = consumeToken();\n        String value = input.readAndResetRecorded();\n        return new Token(type, value);\n    }\n\n    private Token.Type consumeToken() throws LexException {\n        boolean preserveExistingFormatting = preserveExistingFormatting();\n\n        if (input.tryConsumeRegex(NEWLINE_PATTERN)) {\n            somethingSinceNewline = false;\n            return preserveExistingFormatting ? FORCED_NEWLINE : WHITESPACE;\n        } else if (input.tryConsume(\" \") || input.tryConsume(\"\\t\")) {\n            // TODO(cpovirk): How about weird whitespace chars? Ideally we'd distinguish breaking vs. not.\n            // Returning LITERAL here prevent us from breaking a <pre> line. For more info, see LITERAL.\n            return preserveExistingFormatting ? LITERAL : WHITESPACE;\n        }\n\n        /*\n         * TODO(cpovirk): Maybe try to detect things like \"{@code\\n@GwtCompatible}\" that aren't intended\n         * as tags. But in the most likely case, in which that happens inside <pre>{@code, we have no\n         * great options for fixing it.\n         * https://github.com/google/google-java-format/issues/7#issuecomment-197383926\n         */\n        if (!somethingSinceNewline && input.tryConsumeRegex(FOOTER_TAG_PATTERN)) {\n            checkMatchingTags();\n            somethingSinceNewline = true;\n            return FOOTER_JAVADOC_TAG_START;\n        }\n        somethingSinceNewline = true;\n\n        if (input.tryConsumeRegex(INLINE_TAG_OPEN_PATTERN)) {\n            braceDepth.increment();\n            return INLINE_TAG_OPEN;\n        } else if (input.tryConsume(\"{\")) {\n            braceDepth.incrementIfPositive();\n            return LITERAL;\n        } else if (input.tryConsume(\"}\")) {\n            braceDepth.decrementIfPositive();\n            return braceDepth.isPositive() ? LITERAL : INLINE_TAG_CLOSE;\n        }\n\n        // Inside an inline tag, don't do any HTML interpretation.\n        if (braceDepth.isPositive()) {\n            verify(input.tryConsumeRegex(LITERAL_PATTERN));\n            return LITERAL;\n        }\n\n        if (input.tryConsumeRegex(PRE_OPEN_PATTERN)) {\n            preDepth.increment();\n            return preserveExistingFormatting ? LITERAL : PRE_OPEN_TAG;\n        } else if (input.tryConsumeRegex(PRE_CLOSE_PATTERN)) {\n            preDepth.decrementIfPositive();\n            return preserveExistingFormatting() ? LITERAL : PRE_CLOSE_TAG;\n        }\n\n        if (input.tryConsumeRegex(CODE_OPEN_PATTERN)) {\n            codeDepth.increment();\n            return preserveExistingFormatting ? LITERAL : CODE_OPEN_TAG;\n        } else if (input.tryConsumeRegex(CODE_CLOSE_PATTERN)) {\n            codeDepth.decrementIfPositive();\n            return preserveExistingFormatting() ? LITERAL : CODE_CLOSE_TAG;\n        }\n\n        if (input.tryConsumeRegex(TABLE_OPEN_PATTERN)) {\n            tableDepth.increment();\n            return preserveExistingFormatting ? LITERAL : TABLE_OPEN_TAG;\n        } else if (input.tryConsumeRegex(TABLE_CLOSE_PATTERN)) {\n            tableDepth.decrementIfPositive();\n            return preserveExistingFormatting() ? LITERAL : TABLE_CLOSE_TAG;\n        }\n\n        if (preserveExistingFormatting) {\n            verify(input.tryConsumeRegex(LITERAL_PATTERN));\n            return LITERAL;\n        }\n\n        if (input.tryConsumeRegex(PARAGRAPH_OPEN_PATTERN)) {\n            return PARAGRAPH_OPEN_TAG;\n        } else if (input.tryConsumeRegex(PARAGRAPH_CLOSE_PATTERN)) {\n            return PARAGRAPH_CLOSE_TAG;\n        } else if (input.tryConsumeRegex(LIST_OPEN_PATTERN)) {\n            return LIST_OPEN_TAG;\n        } else if (input.tryConsumeRegex(LIST_CLOSE_PATTERN)) {\n            return LIST_CLOSE_TAG;\n        } else if (input.tryConsumeRegex(LIST_ITEM_OPEN_PATTERN)) {\n            return LIST_ITEM_OPEN_TAG;\n        } else if (input.tryConsumeRegex(LIST_ITEM_CLOSE_PATTERN)) {\n            return LIST_ITEM_CLOSE_TAG;\n        } else if (input.tryConsumeRegex(BLOCKQUOTE_OPEN_PATTERN)) {\n            return BLOCKQUOTE_OPEN_TAG;\n        } else if (input.tryConsumeRegex(BLOCKQUOTE_CLOSE_PATTERN)) {\n            return BLOCKQUOTE_CLOSE_TAG;\n        } else if (input.tryConsumeRegex(HEADER_OPEN_PATTERN)) {\n            return HEADER_OPEN_TAG;\n        } else if (input.tryConsumeRegex(HEADER_CLOSE_PATTERN)) {\n            return HEADER_CLOSE_TAG;\n        } else if (input.tryConsumeRegex(BR_PATTERN)) {\n            return BR_TAG;\n        } else if (input.tryConsumeRegex(MOE_BEGIN_STRIP_COMMENT_PATTERN)) {\n            return MOE_BEGIN_STRIP_COMMENT;\n        } else if (input.tryConsumeRegex(MOE_END_STRIP_COMMENT_PATTERN)) {\n            return MOE_END_STRIP_COMMENT;\n        } else if (input.tryConsumeRegex(HTML_COMMENT_PATTERN)) {\n            return HTML_COMMENT;\n        } else if (input.tryConsumeRegex(LITERAL_PATTERN)) {\n            return LITERAL;\n        }\n        throw new IllegalStateException();\n    }\n\n    private boolean preserveExistingFormatting() {\n        return preDepth.isPositive() || tableDepth.isPositive() || codeDepth.isPositive();\n    }\n\n    private void checkMatchingTags() throws LexException {\n        if (braceDepth.isPositive() || preDepth.isPositive() || tableDepth.isPositive() || codeDepth.isPositive()) {\n            throw new LexException();\n        }\n    }\n\n    /**\n     * Join together adjacent literal tokens, and join together adjacent whitespace tokens.\n     *\n     * <p>For literal tokens, this means something like {@code [\"<b>\", \"foo\", \"</b>\"] => [\"<b>foo</b>\"]}. See\n     * {@link #LITERAL_PATTERN} for discussion of why those tokens are separate to begin with.\n     *\n     * <p>Whitespace tokens are treated analogously. We don't really \"want\" to join whitespace tokens, but in the course\n     * of joining literals, we incidentally join whitespace, too. We do take advantage of the joining later on: It\n     * simplifies {@link #inferParagraphTags}.\n     *\n     * <p>Note that we do <i>not</i> merge a literal token and a whitespace token together.\n     */\n    private static ImmutableList<Token> joinAdjacentLiteralsAndAdjacentWhitespace(List<Token> input) {\n        /*\n         * Note: Our final token is always END_JAVADOC. This saves us some trouble:\n         *\n         * - Our inner while() doesn't need a hasNext() check.\n         *\n         * - We don't need to check for leftover accumulated literals after we exit the loop.\n         */\n        ImmutableList.Builder<Token> output = ImmutableList.builder();\n        StringBuilder accumulated = new StringBuilder();\n        NestingCounter inlineTagDepth = new NestingCounter();\n        boolean lastTagWasInlineTagStart = false;\n\n        for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {\n            Token.Type nextType = tokens.peek().getType();\n            if (nextType == INLINE_TAG_OPEN) {\n                inlineTagDepth.increment();\n            } else if (nextType == INLINE_TAG_CLOSE) {\n                inlineTagDepth.decrementIfPositive();\n            }\n\n            if (nextType == INLINE_TAG_OPEN) {\n                // This remains true until we hit some whitespace.\n                lastTagWasInlineTagStart = true;\n            }\n\n            if (nextType == LITERAL || nextType == INLINE_TAG_OPEN || nextType == INLINE_TAG_CLOSE) {\n                accumulated.append(tokens.peek().getValue());\n                tokens.next();\n                continue;\n            }\n\n            /*\n             * IF we have accumulated some literals to join together (say, \"foo<b>bar</b>\"), and IF we'll\n             * next see whitespace followed by a \"@\" literal, we need to join that together with the\n             * previous literals. That ensures that we won't insert a line break before the \"@,\" turning\n             * it into a tag.\n             */\n\n            if (accumulated.length() == 0) {\n                output.add(tokens.peek());\n                tokens.next();\n                continue;\n            }\n\n            StringBuilder seenWhitespace = new StringBuilder();\n            while (tokens.peek().getType() == WHITESPACE) {\n                seenWhitespace.append(tokens.next().getValue());\n            }\n\n            if (tokens.peek().getType() == LITERAL\n                    && (tokens.peek().getValue().startsWith(\"@\") || lastTagWasInlineTagStart)) {\n                lastTagWasInlineTagStart = false;\n                // OK, we're in the case described above.\n                accumulated.append(\" \");\n                accumulated.append(tokens.peek().getValue());\n                tokens.next();\n                continue;\n            }\n\n            output.add(new Token(LITERAL, accumulated.toString()));\n            accumulated.setLength(0);\n\n            if (seenWhitespace.length() > 0) {\n                output.add(new Token(WHITESPACE, seenWhitespace.toString()));\n            }\n\n            lastTagWasInlineTagStart = false;\n            // We have another token coming, possibly of type OTHER. Leave it for the next iteration.\n        }\n\n        /*\n         * TODO(cpovirk): Another case where we could try to join tokens is if a line ends with\n         * /[^ -]-/, as in \"non-\\nblocking.\"\n         */\n        return output.build();\n    }\n\n    /**\n     * Where the input has two consecutive line breaks between literals, insert a {@code <p>} tag between the literals.\n     *\n     * <p>This method must be called after {@link #joinAdjacentLiteralsAndAdjacentWhitespace}, as it assumes that\n     * adjacent whitespace tokens have already been joined.\n     */\n    private static ImmutableList<Token> inferParagraphTags(List<Token> input) {\n        ImmutableList.Builder<Token> output = ImmutableList.builder();\n\n        for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {\n            if (tokens.peek().getType() == LITERAL) {\n                output.add(tokens.next());\n\n                if (tokens.peek().getType() == WHITESPACE\n                        && hasMultipleNewlines(tokens.peek().getValue())) {\n                    output.add(tokens.next());\n\n                    if (tokens.peek().getType() == LITERAL) {\n                        output.add(new Token(PARAGRAPH_OPEN_TAG, \"<p>\"));\n                    }\n                }\n            } else {\n                // TODO(cpovirk): Or just `continue` from the <p> case and move this out of the `else`?\n                output.add(tokens.next());\n            }\n        }\n\n        return output.build();\n\n        /*\n         * Note: We do not want to insert <p> tags inside <pre>. Fortunately, the formatter gets that\n         * right without special effort on our part. The reason: Line breaks inside a <pre> section are\n         * of type FORCED_NEWLINE rather than WHITESPACE.\n         */\n    }\n\n    /**\n     * Replaces whitespace after a {@code href=...>} token with an \"optional link break.\" This allows us to output\n     * either {@code <a href=foo>foo</a>} or {@code <a href=foo>\\nfoo</a>}, depending on how much space we have left on\n     * the line.\n     *\n     * <p>This method must be called after {@link #joinAdjacentLiteralsAndAdjacentWhitespace}, as it assumes that\n     * adjacent whitespace tokens have already been joined.\n     */\n    private static ImmutableList<Token> optionalizeSpacesAfterLinks(List<Token> input) {\n        ImmutableList.Builder<Token> output = ImmutableList.builder();\n\n        for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {\n            if (tokens.peek().getType() == LITERAL && tokens.peek().getValue().matches(\"^href=[^>]*>\")) {\n                output.add(tokens.next());\n\n                if (tokens.peek().getType() == WHITESPACE) {\n                    output.add(new Token(OPTIONAL_LINE_BREAK, tokens.next().getValue()));\n                }\n            } else {\n                output.add(tokens.next());\n            }\n        }\n\n        return output.build();\n\n        /*\n         * Note: We do not want to insert <p> tags inside <pre>. Fortunately, the formatter gets that\n         * right without special effort on our part. The reason: Line breaks inside a <pre> section are\n         * of type FORCED_NEWLINE rather than WHITESPACE.\n         */\n    }\n\n    /**\n     * Adjust indentation inside `<pre>{@code` blocks.\n     *\n     * <p>Also trim leading and trailing blank lines, and move the trailing `}` to its own line.\n     */\n    private static ImmutableList<Token> deindentPreCodeBlocks(List<Token> input) {\n        ImmutableList.Builder<Token> output = ImmutableList.builder();\n        for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {\n            if (tokens.peek().getType() != PRE_OPEN_TAG) {\n                output.add(tokens.next());\n                continue;\n            }\n\n            output.add(tokens.next());\n            List<Token> initialNewlines = new ArrayList<>();\n            while (tokens.hasNext() && tokens.peek().getType() == FORCED_NEWLINE) {\n                initialNewlines.add(tokens.next());\n            }\n            if (tokens.peek().getType() != LITERAL || !tokens.peek().getValue().matches(\"[ \\t]*[{]@code\")) {\n                output.addAll(initialNewlines);\n                output.add(tokens.next());\n                continue;\n            }\n\n            deindentPreCodeBlock(output, tokens);\n        }\n        return output.build();\n    }\n\n    private static void deindentPreCodeBlock(ImmutableList.Builder<Token> output, PeekingIterator<Token> tokens) {\n        Deque<Token> saved = new ArrayDeque<>();\n        output.add(new Token(LITERAL, tokens.next().getValue().trim()));\n        while (tokens.hasNext() && tokens.peek().getType() != PRE_CLOSE_TAG) {\n            Token token = tokens.next();\n            saved.addLast(token);\n        }\n        while (!saved.isEmpty() && saved.peekFirst().getType() == FORCED_NEWLINE) {\n            saved.removeFirst();\n        }\n        while (!saved.isEmpty() && saved.peekLast().getType() == FORCED_NEWLINE) {\n            saved.removeLast();\n        }\n        if (saved.isEmpty()) {\n            return;\n        }\n\n        // move the trailing `}` to its own line\n        Token last = saved.peekLast();\n        boolean trailingBrace = false;\n        if (last.getType() == LITERAL && last.getValue().endsWith(\"}\")) {\n            saved.removeLast();\n            if (last.length() > 1) {\n                saved.addLast(new Token(\n                        LITERAL, last.getValue().substring(0, last.getValue().length() - 1)));\n                saved.addLast(new Token(FORCED_NEWLINE, null));\n            }\n            trailingBrace = true;\n        }\n\n        int trim = -1;\n        for (Token token : saved) {\n            if (token.getType() == LITERAL) {\n                int idx = CharMatcher.isNot(' ').indexIn(token.getValue());\n                if (idx != -1 && (trim == -1 || idx < trim)) {\n                    trim = idx;\n                }\n            }\n        }\n\n        output.add(new Token(FORCED_NEWLINE, \"\\n\"));\n        for (Token token : saved) {\n            if (token.getType() == LITERAL) {\n                output.add(new Token(\n                        LITERAL,\n                        trim > 0 && token.length() > trim ? token.getValue().substring(trim) : token.getValue()));\n            } else {\n                output.add(token);\n            }\n        }\n\n        if (trailingBrace) {\n            output.add(new Token(LITERAL, \"}\"));\n        } else {\n            output.add(new Token(FORCED_NEWLINE, \"\\n\"));\n        }\n    }\n\n    private static final CharMatcher NEWLINE = CharMatcher.is('\\n');\n\n    private static boolean hasMultipleNewlines(String s) {\n        return NEWLINE.countIn(s) > 1;\n    }\n\n    /*\n     * This also eats any trailing whitespace. We would be smart enough to ignore that, anyway --\n     * except in the case of <pre>/<table>, inside which we otherwise leave whitespace intact.\n     *\n     * We'd remove the trailing whitespace later on (in JavaCommentsHelper.rewrite), but I feel safer\n     * stripping it now: It otherwise might confuse our line-length count, which we use for wrapping.\n     */\n    private static final Pattern NEWLINE_PATTERN = compile(\"^[ \\t]*\\n[ \\t]*[*]?[ \\t]?\");\n\n    // We ensure elsewhere that we match this only at the beginning of a line.\n    // Only match tags that start with a lowercase letter, to avoid false matches on unescaped\n    // annotations inside code blocks.\n    // Match \"@param <T>\" specially in case the <T> is a <P> or other HTML tag we treat specially.\n    private static final Pattern FOOTER_TAG_PATTERN = compile(\"^@(param\\\\s+<\\\\w+>|[a-z]\\\\w*)\");\n    private static final Pattern MOE_BEGIN_STRIP_COMMENT_PATTERN =\n            compile(\"^<!--\\\\s*MOE:begin_intracomment_strip\\\\s*-->\");\n    private static final Pattern MOE_END_STRIP_COMMENT_PATTERN = compile(\"^<!--\\\\s*MOE:end_intracomment_strip\\\\s*-->\");\n    private static final Pattern HTML_COMMENT_PATTERN = fullCommentPattern();\n    private static final Pattern PRE_OPEN_PATTERN = openTagPattern(\"pre\");\n    private static final Pattern PRE_CLOSE_PATTERN = closeTagPattern(\"pre\");\n    private static final Pattern CODE_OPEN_PATTERN = openTagPattern(\"code\");\n    private static final Pattern CODE_CLOSE_PATTERN = closeTagPattern(\"code\");\n    private static final Pattern TABLE_OPEN_PATTERN = openTagPattern(\"table\");\n    private static final Pattern TABLE_CLOSE_PATTERN = closeTagPattern(\"table\");\n    private static final Pattern LIST_OPEN_PATTERN = openTagPattern(\"ul|ol|dl\");\n    private static final Pattern LIST_CLOSE_PATTERN = closeTagPattern(\"ul|ol|dl\");\n    private static final Pattern LIST_ITEM_OPEN_PATTERN = openTagPattern(\"li|dt|dd\");\n    private static final Pattern LIST_ITEM_CLOSE_PATTERN = closeTagPattern(\"li|dt|dd\");\n    private static final Pattern HEADER_OPEN_PATTERN = openTagPattern(\"h[1-6]\");\n    private static final Pattern HEADER_CLOSE_PATTERN = closeTagPattern(\"h[1-6]\");\n    private static final Pattern PARAGRAPH_OPEN_PATTERN = openTagPattern(\"p\");\n    private static final Pattern PARAGRAPH_CLOSE_PATTERN = closeTagPattern(\"p\");\n    private static final Pattern BLOCKQUOTE_OPEN_PATTERN = openTagPattern(\"blockquote\");\n    private static final Pattern BLOCKQUOTE_CLOSE_PATTERN = closeTagPattern(\"blockquote\");\n    private static final Pattern BR_PATTERN = openTagPattern(\"br\");\n    private static final Pattern INLINE_TAG_OPEN_PATTERN = compile(\"^[{]@\\\\w*\");\n    /*\n     * We exclude < so that we don't swallow following HTML tags. This lets us fix up \"foo<p>\" (~400\n     * hits in Google-internal code). We will join unnecessarily split \"words\" (like \"foo<b>bar</b>\")\n     * in a later step. There's a similar story for braces. I'm not sure I actually need to exclude @\n     * or *. TODO(cpovirk): Try removing them.\n     *\n     * Thanks to the \"rejoin\" step in joinAdjacentLiteralsAndAdjacentWhitespace(), we could get away\n     * with matching only one character here. That would eliminate the need for the regex entirely.\n     * That might be faster or slower than what we do now.\n     */\n    private static final Pattern LITERAL_PATTERN = compile(\"^.[^ \\t\\n@<{}*]*\", DOTALL);\n\n    private static Pattern fullCommentPattern() {\n        return compile(\"^<!--.*?-->\", DOTALL);\n    }\n\n    private static Pattern openTagPattern(String namePattern) {\n        return compile(format(\"^<(?:%s)\\\\b[^>]*>\", namePattern), CASE_INSENSITIVE);\n    }\n\n    private static Pattern closeTagPattern(String namePattern) {\n        return compile(format(\"^</(?:%s)\\\\b[^>]*>\", namePattern), CASE_INSENSITIVE);\n    }\n\n    static class LexException extends Exception {}\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocFormatter.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java.javadoc;\n\nimport static com.palantir.javaformat.java.javadoc.JavadocLexer.LexException;\nimport static com.palantir.javaformat.java.javadoc.JavadocLexer.lex;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BR_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_OPEN_TAG;\nimport static java.util.regex.Pattern.CASE_INSENSITIVE;\nimport static java.util.regex.Pattern.compile;\n\nimport com.google.common.collect.ImmutableList;\nimport java.util.List;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n/**\n * Entry point for formatting Javadoc.\n *\n * <p>This stateless class reads tokens from the stateful lexer and translates them to \"requests\" and \"writes\" to the\n * stateful writer. It also munges tokens into \"standardized\" forms. Finally, it performs postprocessing to convert the\n * written Javadoc to a one-liner if possible or to leave a single blank line if it's empty.\n */\npublic final class JavadocFormatter {\n\n    private final int maxLineLength;\n\n    public JavadocFormatter(int maxLineLength) {\n        this.maxLineLength = maxLineLength;\n    }\n\n    /**\n     * Formats the given Javadoc comment, which must start with \u2215\u2731\u2731 and end with \u2731\u2215. The output will start and end with\n     * the same characters.\n     */\n    public String formatJavadoc(String input, int blockIndent) {\n        ImmutableList<Token> tokens;\n        try {\n            tokens = lex(input);\n        } catch (LexException e) {\n            return input;\n        }\n        String result = render(tokens, blockIndent);\n        return makeSingleLineIfPossible(blockIndent, result);\n    }\n\n    private String render(List<Token> input, int blockIndent) {\n        JavadocWriter output = new JavadocWriter(blockIndent, maxLineLength);\n        for (Token token : input) {\n            switch (token.getType()) {\n                case BEGIN_JAVADOC:\n                    output.writeBeginJavadoc();\n                    break;\n                case END_JAVADOC:\n                    output.writeEndJavadoc();\n                    return output.toString();\n                case FOOTER_JAVADOC_TAG_START:\n                    output.writeFooterJavadocTagStart(token);\n                    break;\n                case LIST_OPEN_TAG:\n                    output.writeListOpen(token);\n                    break;\n                case LIST_CLOSE_TAG:\n                    output.writeListClose(token);\n                    break;\n                case LIST_ITEM_OPEN_TAG:\n                    output.writeListItemOpen(token);\n                    break;\n                case HEADER_OPEN_TAG:\n                    output.writeHeaderOpen(token);\n                    break;\n                case HEADER_CLOSE_TAG:\n                    output.writeHeaderClose(token);\n                    break;\n                case PARAGRAPH_OPEN_TAG:\n                    output.writeParagraphOpen(standardizePToken(token));\n                    break;\n                case BLOCKQUOTE_OPEN_TAG:\n                case BLOCKQUOTE_CLOSE_TAG:\n                    output.writeBlockquoteOpenOrClose(token);\n                    break;\n                case PRE_OPEN_TAG:\n                    output.writePreOpen(token);\n                    break;\n                case PRE_CLOSE_TAG:\n                    output.writePreClose(token);\n                    break;\n                case CODE_OPEN_TAG:\n                    output.writeCodeOpen(token);\n                    break;\n                case CODE_CLOSE_TAG:\n                    output.writeCodeClose(token);\n                    break;\n                case TABLE_OPEN_TAG:\n                    output.writeTableOpen(token);\n                    break;\n                case TABLE_CLOSE_TAG:\n                    output.writeTableClose(token);\n                    break;\n                case MOE_BEGIN_STRIP_COMMENT:\n                    output.requestMoeBeginStripComment(token);\n                    break;\n                case MOE_END_STRIP_COMMENT:\n                    output.writeMoeEndStripComment(token);\n                    break;\n                case HTML_COMMENT:\n                    output.writeHtmlComment(token);\n                    break;\n                case BR_TAG:\n                    output.writeBr(standardizeBrToken(token));\n                    break;\n                case WHITESPACE:\n                    output.requestWhitespace();\n                    break;\n                case FORCED_NEWLINE:\n                    output.writeLineBreakNoAutoIndent();\n                    break;\n                case LITERAL:\n                case INLINE_TAG_OPEN:\n                case INLINE_TAG_CLOSE:\n                    output.writeLiteral(token);\n                    break;\n                case PARAGRAPH_CLOSE_TAG:\n                case LIST_ITEM_CLOSE_TAG:\n                case OPTIONAL_LINE_BREAK:\n                    break;\n                default:\n                    throw new IllegalArgumentException(token.getType().name());\n            }\n        }\n        throw new IllegalStateException();\n    }\n\n    /*\n     * TODO(cpovirk): Is this really the right location for the standardize* methods? Maybe the lexer\n     * should include them as part of its own postprocessing? Or even the writer could make sense.\n     */\n\n    private static Token standardizeBrToken(Token token) {\n        return standardize(token, STANDARD_BR_TOKEN);\n    }\n\n    private static Token standardizePToken(Token token) {\n        return standardize(token, STANDARD_P_TOKEN);\n    }\n\n    private static Token standardize(Token token, Token standardToken) {\n        return SIMPLE_TAG_PATTERN.matcher(token.getValue()).matches() ? standardToken : token;\n    }\n\n    private static final Token STANDARD_BR_TOKEN = new Token(BR_TAG, \"<br>\");\n    private static final Token STANDARD_P_TOKEN = new Token(PARAGRAPH_OPEN_TAG, \"<p>\");\n    private static final Pattern SIMPLE_TAG_PATTERN = compile(\"^<\\\\w+\\\\s*/?\\\\s*>\", CASE_INSENSITIVE);\n\n    private static final Pattern ONE_CONTENT_LINE_PATTERN = compile(\" */[*][*]\\n *[*] (.*)\\n *[*]/\");\n\n    /** Returns the given string or a one-line version of it (e.g., \"\u2215\u2731\u2731 Tests for foos. \u2731\u2215\") if it fits on one line. */\n    private String makeSingleLineIfPossible(int blockIndent, String input) {\n        int oneLinerContentLength = maxLineLength - \"/**  */\".length() - blockIndent;\n        Matcher matcher = ONE_CONTENT_LINE_PATTERN.matcher(input);\n        if (matcher.matches() && matcher.group(1).isEmpty()) {\n            return \"/** */\";\n        } else if (matcher.matches() && matcher.group(1).length() <= oneLinerContentLength) {\n            return \"/** \" + matcher.group(1) + \" */\";\n        }\n        return input;\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaCommentsHelper.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Input.Tok;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.java.javadoc.JavadocFormatter;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport javax.annotation.Nullable;\n\n/** {@code JavaCommentsHelper} extends {@link CommentsHelper} to rewrite Java comments. */\npublic final class JavaCommentsHelper implements CommentsHelper {\n\n    private final String lineSeparator;\n    private final JavaFormatterOptions options;\n\n    @Nullable\n    private final JavadocFormatter javadocFormatter;\n\n    public JavaCommentsHelper(String lineSeparator, JavaFormatterOptions options) {\n        this.lineSeparator = lineSeparator;\n        this.options = options;\n        this.javadocFormatter = options.formatJavadoc() ? new JavadocFormatter(options.maxLineLength()) : null;\n    }\n\n    @Override\n    public String rewrite(Tok tok, int maxWidth, int column0) {\n        if (!tok.isComment()) {\n            return tok.getOriginalText();\n        }\n        String text = tok.getOriginalText();\n        if (javadocFormatter != null && tok.isJavadocComment()) {\n            text = javadocFormatter.formatJavadoc(text, column0);\n        }\n        List<String> lines = new ArrayList<>();\n        Iterator<String> it = Newlines.lineIterator(text);\n        while (it.hasNext()) {\n            lines.add(CharMatcher.whitespace().trimTrailingFrom(it.next()));\n        }\n        if (tok.isSlashSlashComment()) {\n            return indentLineComments(lines, column0);\n        } else if (javadocShaped(lines)) {\n            return indentJavadoc(lines, column0);\n        } else {\n            return preserveIndentation(lines, column0);\n        }\n    }\n\n    // For non-javadoc-shaped block comments, shift the entire block to the correct\n    // column, but do not adjust relative indentation.\n    private String preserveIndentation(List<String> lines, int column0) {\n        StringBuilder builder = new StringBuilder();\n\n        // find the leftmost non-whitespace character in all trailing lines\n        int startCol = -1;\n        for (int i = 1; i < lines.size(); i++) {\n            int lineIdx = CharMatcher.whitespace().negate().indexIn(lines.get(i));\n            if (lineIdx >= 0 && (startCol == -1 || lineIdx < startCol)) {\n                startCol = lineIdx;\n            }\n        }\n\n        // output the first line at the current column\n        builder.append(lines.get(0));\n\n        // output all trailing lines with plausible indentation\n        for (int i = 1; i < lines.size(); ++i) {\n            builder.append(lineSeparator).append(\" \".repeat(column0));\n            // check that startCol is valid index, e.g. for blank lines\n...\n", "needle_token_start": 2365, "needle_token_end": 2409, "code_context_ntokens": 16114, "output": ["```java\nprivate String formatFile(Formatter formatter) throws FormatterException {\n    String formatted = formatter.formatSource(input, characterRanges(input).asRanges());\n    formatted = fixImports(formatted);\n    if (parameters.reflowLongStrings()) {\n        formatted = StringWrapper.wrap(options.maxLineLength(), formatted, formatter);\n    }\n    return formatted;\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "rangesAfterAppliedReplacements", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/StringWrapper.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function calculates the new ranges of text segments in a document after replacements have been applied to the original text, adjusting for any changes in length due to the replacements.\n2. **Input**: The function takes a mapping of original text ranges to their replacement strings.\n3. **Output**: It outputs a set of updated text ranges that reflect the new positions and lengths of the replaced text segments in the document.\n4. **Procedure**: \n   - The function iterates over each entry in the input mapping.\n   - For each entry, it calculates the new starting position of the text segment by adding an accumulated offset to the original starting position.\n   - It then determines the new ending position based on the length of the replacement string.\n   - The function updates the accumulated offset for subsequent calculations by adding the difference between the new and original lengths of the text segment.\n   - Each new range is added to the output set.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/RemoveUnusedImports.java\n/*\n * Copyright 2016 Google Inc. All Rights Reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.collect.HashMultimap;\nimport com.google.common.collect.Multimap;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeMap;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeMap;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Newlines;\nimport com.sun.source.doctree.DocCommentTree;\nimport com.sun.source.doctree.ReferenceTree;\nimport com.sun.source.tree.CaseTree;\nimport com.sun.source.tree.IdentifierTree;\nimport com.sun.source.tree.ImportTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.source.util.DocTreePath;\nimport com.sun.source.util.DocTreePathScanner;\nimport com.sun.source.util.TreePathScanner;\nimport com.sun.source.util.TreeScanner;\nimport com.sun.tools.javac.api.JavacTrees;\nimport com.sun.tools.javac.tree.DCTree;\nimport com.sun.tools.javac.tree.DCTree.DCReference;\nimport com.sun.tools.javac.tree.JCTree;\nimport com.sun.tools.javac.tree.JCTree.JCCompilationUnit;\nimport com.sun.tools.javac.tree.JCTree.JCFieldAccess;\nimport com.sun.tools.javac.tree.JCTree.JCIdent;\nimport com.sun.tools.javac.tree.JCTree.JCImport;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Options;\nimport java.lang.reflect.Method;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\n/**\n * Removes unused imports from a source file. Imports that are only used in javadoc are also removed, and the references\n * in javadoc are replaced with fully qualified names.\n */\npublic class RemoveUnusedImports {\n\n    // Visits an AST, recording all simple names that could refer to imported\n    // types and also any javadoc references that could refer to imported\n    // types (`@link`, `@see`, `@throws`, etc.)\n    //\n    // No attempt is made to determine whether simple names occur in contexts\n    // where they are type names, so there will be false positives. For example,\n    // `List` is not identified as unused import below:\n    //\n    // ```\n    // import java.util.List;\n    // class List {}\n    // ```\n    //\n    // This is still reasonably effective in practice because type names differ\n    // from other kinds of names in casing convention, and simple name\n    // clashes between imported and declared types are rare.\n    private static final class UnusedImportScanner extends TreePathScanner<Void, Void> {\n\n        private final Set<String> usedNames = new LinkedHashSet<>();\n        private final Multimap<String, Range<Integer>> usedInJavadoc = HashMultimap.create();\n        final JavacTrees trees;\n        final DocTreeScanner docTreeSymbolScanner;\n\n        private UnusedImportScanner(JavacTrees trees) {\n            this.trees = trees;\n            docTreeSymbolScanner = new DocTreeScanner();\n        }\n\n        /** Skip the imports themselves when checking for usage. */\n        @Override\n        public Void visitImport(ImportTree importTree, Void usedSymbols) {\n            return null;\n        }\n\n        @Override\n        public Void visitIdentifier(IdentifierTree tree, Void unused) {\n            if (tree == null) {\n                return null;\n            }\n            usedNames.add(tree.getName().toString());\n            return null;\n        }\n\n        // TODO(fwindheuser): remove this override when pattern matching in switch is no longer a preview\n        // feature, and TreePathScanner visits CaseTree#getLabels instead of CaseTree#getExpressions\n        @SuppressWarnings(\"unchecked\") // reflection\n        @Override\n        public Void visitCase(CaseTree tree, Void unused) {\n            if (CASE_TREE_GET_LABELS != null) {\n                try {\n                    scan((List<? extends Tree>) CASE_TREE_GET_LABELS.invoke(tree), null);\n                } catch (ReflectiveOperationException e) {\n                    throw new RuntimeException(e.getMessage(), e);\n                }\n            }\n            return super.visitCase(tree, null);\n        }\n\n        private static final Method CASE_TREE_GET_LABELS = caseTreeGetLabels();\n\n        private static Method caseTreeGetLabels() {\n            try {\n                return CaseTree.class.getMethod(\"getLabels\");\n            } catch (NoSuchMethodException e) {\n                return null;\n            }\n        }\n\n        @Override\n        public Void scan(Tree tree, Void unused) {\n            if (tree == null) {\n                return null;\n            }\n            scanJavadoc();\n            return super.scan(tree, unused);\n        }\n\n        private void scanJavadoc() {\n            if (getCurrentPath() == null) {\n                return;\n            }\n            DocCommentTree commentTree = trees.getDocCommentTree(getCurrentPath());\n            if (commentTree == null) {\n                return;\n            }\n            docTreeSymbolScanner.scan(new DocTreePath(getCurrentPath(), commentTree), null);\n        }\n\n        // scan javadoc comments, checking for references to imported types\n        class DocTreeScanner extends DocTreePathScanner<Void, Void> {\n            @Override\n            public Void visitIdentifier(com.sun.source.doctree.IdentifierTree node, Void aVoid) {\n                return null;\n            }\n\n            @Override\n            public Void visitReference(ReferenceTree referenceTree, Void unused) {\n                DCReference reference = (DCReference) referenceTree;\n                long basePos = reference\n                        .pos((DCTree.DCDocComment) getCurrentPath().getDocComment())\n                        .getStartPosition();\n                // the position of trees inside the reference node aren't stored, but the qualifier's\n                // start position is the beginning of the reference node\n                if (reference.qualifierExpression != null) {\n                    new ReferenceScanner(basePos).scan(reference.qualifierExpression, null);\n                }\n                // Record uses inside method parameters. The javadoc tool doesn't use these, but\n                // IntelliJ does.\n                if (reference.paramTypes != null) {\n                    for (JCTree param : reference.paramTypes) {\n                        // TODO(cushon): get start positions for the parameters\n                        new ReferenceScanner(-1).scan(param, null);\n                    }\n                }\n                return null;\n            }\n\n            // scans the qualifier and parameters of a javadoc reference for possible type names\n            private class ReferenceScanner extends TreeScanner<Void, Void> {\n                private final long basePos;\n\n                public ReferenceScanner(long basePos) {\n                    this.basePos = basePos;\n                }\n\n                @Override\n                public Void visitIdentifier(IdentifierTree node, Void aVoid) {\n                    usedInJavadoc.put(\n                            node.getName().toString(),\n                            basePos != -1\n                                    ? Range.closedOpen(\n                                            (int) basePos,\n                                            (int) basePos + node.getName().length())\n                                    : null);\n                    return super.visitIdentifier(node, aVoid);\n                }\n            }\n        }\n    }\n\n    public static String removeUnusedImports(final String contents) throws FormatterException {\n        Context context = new Context();\n        JCCompilationUnit unit = parse(context, contents);\n        if (unit == null) {\n            // error handling is done during formatting\n            return contents;\n        }\n        UnusedImportScanner scanner = new UnusedImportScanner(JavacTrees.instance(context));\n        scanner.scan(unit, null);\n        return applyReplacements(contents, buildReplacements(contents, unit, scanner.usedNames, scanner.usedInJavadoc));\n    }\n\n    private static JCCompilationUnit parse(Context context, String javaInput) throws FormatterException {\n        Options.instance(context).put(\"allowStringFolding\", \"false\");\n        return Formatter.parseJcCompilationUnit(context, javaInput);\n    }\n\n    /** Construct replacements to fix unused imports. */\n    private static RangeMap<Integer, String> buildReplacements(\n            String contents,\n            JCCompilationUnit unit,\n            Set<String> usedNames,\n            Multimap<String, Range<Integer>> usedInJavadoc) {\n        RangeMap<Integer, String> replacements = TreeRangeMap.create();\n        for (JCImport importTree : unit.getImports()) {\n            String simpleName = getSimpleName(importTree);\n            if (!isUnused(unit, usedNames, usedInJavadoc, importTree, simpleName)) {\n                continue;\n            }\n            // delete the import\n            int endPosition = importTree.getEndPosition(unit.endPositions);\n            endPosition = Math.max(CharMatcher.isNot(' ').indexIn(contents, endPosition), endPosition);\n            String sep = Newlines.guessLineSeparator(contents);\n            if (endPosition + sep.length() < contents.length()\n                    && contents.subSequence(endPosition, endPosition + sep.length())\n                            .toString()\n                            .equals(sep)) {\n                endPosition += sep.length();\n            }\n            replacements.put(Range.closedOpen(importTree.getStartPosition(), endPosition), \"\");\n        }\n        return replacements;\n    }\n\n    private static String getSimpleName(ImportTree importTree) {\n        return importTree.getQualifiedIdentifier() instanceof JCIdent\n                ? ((JCIdent) importTree.getQualifiedIdentifier()).getName().toString()\n                : ((JCFieldAccess) importTree.getQualifiedIdentifier())\n                        .getIdentifier()\n                        .toString();\n    }\n\n    private static boolean isUnused(\n...\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/StringWrapper.java\n/*\n * Copyright 2019 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static com.google.common.collect.Iterables.getLast;\nimport static java.util.stream.Collectors.joining;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.base.Verify;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.TreeRangeMap;\nimport com.palantir.javaformat.Newlines;\nimport com.sun.source.tree.BinaryTree;\nimport com.sun.source.tree.LiteralTree;\nimport com.sun.source.tree.MemberSelectTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.source.tree.Tree.Kind;\nimport com.sun.source.util.TreePath;\nimport com.sun.source.util.TreePathScanner;\nimport com.sun.tools.javac.tree.JCTree;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Options;\nimport com.sun.tools.javac.util.Position;\nimport com.sun.tools.javac.util.Position.LineMap;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Deque;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.stream.Stream;\n\n/** Wraps string literals that exceed the column limit. */\npublic final class StringWrapper {\n    /** Reflows string literals in the given Java source code that extend past the given column limit. */\n    static String wrap(final int columnLimit, String input, Formatter formatter) throws FormatterException {\n        if (!longLines(columnLimit, input)) {\n            // fast path\n            return input;\n        }\n\n        TreeRangeMap<Integer, String> replacements = getReflowReplacements(columnLimit, input);\n        String firstPass =\n                formatter.formatSource(input, replacements.asMapOfRanges().keySet());\n\n        if (!firstPass.equals(input)) {\n            // If formatting the replacement ranges resulted in a change, recalculate the replacements on\n            // the updated input.\n            input = firstPass;\n            replacements = getReflowReplacements(columnLimit, input);\n        }\n\n        String result = applyReplacements(input, replacements);\n\n        // Format again, because broken strings might now fit on the first line in case of assignments\n        String secondPass = formatter.formatSource(result, rangesAfterAppliedReplacements(replacements));\n\n        if (!secondPass.equals(result)) {\n            replacements = getReflowReplacements(columnLimit, secondPass);\n            result = applyReplacements(secondPass, replacements);\n        }\n\n        {\n            // We really don't want bugs in this pass to change the behaviour of programs we're\n            // formatting, so check that the pretty-printed AST is the same before and after reformatting.\n            String expected = parse(input, /* allowStringFolding= */ true).toString();\n            String actual = parse(result, /* allowStringFolding= */ true).toString();\n            if (!expected.equals(actual)) {\n                throw new FormatterException(String.format(\n                        \"Something has gone terribly wrong. Please file a bug: \"\n                                + \"https://github.com/google/google-java-format/issues/new\"\n                                + \"\\n\\n=== Actual: ===\\n%s\\n=== Expected: ===\\n%s\\n\",\n                        actual, expected));\n            }\n        }\n\n        return result;\n    }\n\n    \n@SuppressWarnings(\"ResultOfMethodCallIgnored\")\n    private static ImmutableSet<Range<Integer>> rangesAfterAppliedReplacements(\n            TreeRangeMap<Integer, String> replacements) {\n        ImmutableSet.Builder<Range<Integer>> outputRanges = ImmutableSet.builder();\n        int offset = 0;\n        for (Map.Entry<Range<Integer>, String> entry :\n                replacements.asMapOfRanges().entrySet()) {\n            Range<Integer> range = entry.getKey();\n            String replacement = entry.getValue();\n\n            int lower = offset + range.lowerEndpoint();\n            int upper = lower + replacement.length();\n            outputRanges.add(Range.closedOpen(lower, upper));\n\n            int originalLength = range.upperEndpoint() - range.lowerEndpoint();\n            int newLength = upper - lower;\n            offset += newLength - originalLength;\n        }\n        return outputRanges.build();\n    }\n\n    private static TreeRangeMap<Integer, String> getReflowReplacements(int columnLimit, final String input)\n            throws FormatterException {\n        JCTree.JCCompilationUnit unit = parse(input, /* allowStringFolding= */ false);\n        String separator = Newlines.guessLineSeparator(input);\n\n        // Paths to string literals that extend past the column limit.\n        List<TreePath> toFix = new ArrayList<>();\n        final Position.LineMap lineMap = unit.getLineMap();\n        new TreePathScanner<Void, Void>() {\n            @Override\n            public Void visitLiteral(LiteralTree literalTree, Void aVoid) {\n                if (literalTree.getKind() != Kind.STRING_LITERAL) {\n                    return null;\n                }\n                Tree parent = getCurrentPath().getParentPath().getLeaf();\n                if (parent instanceof MemberSelectTree\n                        && ((MemberSelectTree) parent).getExpression().equals(literalTree)) {\n                    return null;\n                }\n                int endPosition = getEndPosition(unit, literalTree);\n                int lineEnd = endPosition;\n                while (Newlines.hasNewlineAt(input, lineEnd) == -1) {\n                    lineEnd++;\n                }\n                if (lineMap.getColumnNumber(lineEnd) - 1 <= columnLimit) {\n                    return null;\n                }\n                toFix.add(getCurrentPath());\n                return null;\n            }\n        }.scan(new TreePath(unit), null);\n\n        TreeRangeMap<Integer, String> replacements = TreeRangeMap.create();\n        for (TreePath path : toFix) {\n            // Find the outermost contiguous enclosing concatenation expression\n            TreePath enclosing = path;\n            while (enclosing.getParentPath().getLeaf().getKind() == Tree.Kind.PLUS) {\n                enclosing = enclosing.getParentPath();\n            }\n            // Is the literal being wrapped the first in a chain of concatenation expressions?\n            // i.e. `ONE + TWO + THREE`\n            // We need this information to handle continuation indents.\n            AtomicBoolean first = new AtomicBoolean(false);\n            // Finds the set of string literals in the concat expression that includes the one that needs\n            // to be wrapped.\n            List<Tree> flat = flatten(input, unit, path, enclosing, first);\n\n            // Walk up as long as parents are on the same line, in order to find the correct\n            // startColumn.\n            TreePath startingPath = enclosing;\n            while (startingPath.getParentPath() != null && onSameLineAsParent(lineMap, startingPath)) {\n                startingPath = startingPath.getParentPath();\n            }\n\n            // Zero-indexed start column\n            int startColumn = lineMap.getColumnNumber(getStartPosition(flat.get(0))) - 1;\n            int fistLineCol = lineMap.getColumnNumber(getStartPosition(startingPath.getLeaf())) - 1;\n\n            // Handling leaving trailing non-string tokens at the end of the literal,\n            // e.g. the trailing `);` in `foo(\"...\");`.\n            int end = getEndPosition(unit, getLast(flat));\n            int lineEnd = end;\n            while (Newlines.hasNewlineAt(input, lineEnd) == -1) {\n                lineEnd++;\n            }\n            int trailing = lineEnd - end;\n\n            // Get the original source text of the string literals, excluding `\"` and `+`.\n            ImmutableList<String> components = stringComponents(input, unit, flat);\n            replacements.put(\n                    Range.closedOpen(getStartPosition(flat.get(0)), getEndPosition(unit, getLast(flat))),\n                    reflow(separator, columnLimit, trailing, components, first.get(), startColumn, fistLineCol));\n        }\n        return replacements;\n    }\n\n    private static boolean onSameLineAsParent(LineMap lineMap, TreePath path) {\n        return lineMap.getLineNumber(getStartPosition(path.getLeaf()))\n                == lineMap.getLineNumber(getStartPosition(path.getParentPath().getLeaf()));\n    }\n\n    /**\n     * Returns the source text of the given string literal trees, excluding the leading and trailing double-quotes and\n     * the `+` operator.\n     */\n    private static ImmutableList<String> stringComponents(\n            String input, JCTree.JCCompilationUnit unit, List<Tree> flat) {\n        ImmutableList.Builder<String> result = ImmutableList.builder();\n        StringBuilder piece = new StringBuilder();\n        for (Tree tree : flat) {\n            // adjust for leading and trailing double quotes\n            String text = input.substring(getStartPosition(tree) + 1, getEndPosition(unit, tree) - 1);\n            int start = 0;\n            for (int idx = 0; idx < text.length(); idx++) {\n                if (CharMatcher.whitespace().matches(text.charAt(idx))) {\n                    // continue below\n                } else if (hasEscapedWhitespaceAt(text, idx) != -1) {\n                    // continue below\n                } else if (hasEscapedNewlineAt(text, idx) != -1) {\n                    int length;\n                    while ((length = hasEscapedNewlineAt(text, idx)) != -1) {\n                        idx += length;\n                    }\n                } else {\n                    continue;\n                }\n                piece.append(text, start, idx);\n                result.add(piece.toString());\n                piece = new StringBuilder();\n                start = idx;\n            }\n            if (piece.length() > 0) {\n                result.add(piece.toString());\n                piece = new StringBuilder();\n            }\n            if (start < text.length()) {\n                piece.append(text, start, text.length());\n            }\n        }\n        if (piece.length() > 0) {\n            result.add(piece.toString());\n        }\n        return result.build();\n    }\n\n    static int hasEscapedWhitespaceAt(String input, int idx) {\n        return Stream.of(\"\\\\t\")\n                .mapToInt(x -> input.startsWith(x, idx) ? x.length() : -1)\n                .filter(x -> x != -1)\n                .findFirst()\n                .orElse(-1);\n    }\n\n    static int hasEscapedNewlineAt(String input, int idx) {\n        return Stream.of(\"\\\\r\\\\n\", \"\\\\r\", \"\\\\n\")\n                .mapToInt(x -> input.startsWith(x, idx) ? x.length() : -1)\n                .filter(x -> x != -1)\n                .findFirst()\n                .orElse(-1);\n    }\n\n    /**\n     * Reflows the given source text, trying to split on word boundaries.\n     *\n     * @param separator the line separator\n     * @param columnLimit the number of columns to wrap at\n     * @param trailing extra space to leave after the last line\n     * @param components the text to reflow\n     * @param first0 true if the text includes the beginning of its enclosing concat chain, i.e. a\n     * @param textStartColumn the column position of the beginning of the original text\n     * @param firstLineStartColumn the column where the very first line starts (can be less than textStartColumn if text\n     *     follows variable declaration)\n     */\n    private static String reflow(\n            String separator,\n            int columnLimit,\n            int trailing,\n            ImmutableList<String> components,\n            boolean first0,\n            int textStartColumn,\n            int firstLineStartColumn) {\n        // We have space between the start column and the limit to output the first line.\n        // Reserve two spaces for the quotes.\n        int width = columnLimit - textStartColumn - 2;\n        Deque<String> input = new ArrayDeque<>(components);\n        List<String> lines = new ArrayList<>();\n        boolean first = first0;\n        while (!input.isEmpty()) {\n            int length = 0;\n            List<String> line = new ArrayList<>();\n            if (input.stream().mapToInt(String::length).sum() <= width) {\n                width -= trailing;\n            }\n            while (!input.isEmpty()\n                    && (length <= 4 || (length + input.peekFirst().length()) < width)) {\n                String text = input.removeFirst();\n                line.add(text);\n                length += text.length();\n                if (text.endsWith(\"\\\\n\") || text.endsWith(\"\\\\r\")) {\n                    break;\n                }\n            }\n            if (line.isEmpty()) {\n                line.add(input.removeFirst());\n            }\n            // add the split line to the output, and process whatever's left\n            lines.add(String.join(\"\", line));\n            if (first) {\n                // Subsequent lines have a four-space continuation indent and a `+ `.\n                width -= 6;\n                // Also, switch to firstLineStartColumn in order to account for the fact that continuations\n                // should get indented from the beginning of the first line.\n                // This is to handle cases like:\n                // String foo = \"first component\"\n                //     + \"rest\";\n                width += textStartColumn - firstLineStartColumn;\n                first = false;\n            }\n        }\n\n        return lines.stream()\n                .collect(joining(\n                        \"\\\"\" + separator + \" \".repeat(first0 ? firstLineStartColumn + 4 : textStartColumn - 2) + \"+ \\\"\",\n                        \"\\\"\",\n                        \"\\\"\"));\n    }\n\n    /**\n     * Flattens the given binary expression tree, and extracts the subset that contains the given path and any adjacent\n     * nodes that are also string literals.\n     */\n    private static List<Tree> flatten(\n            String input, JCTree.JCCompilationUnit unit, TreePath path, TreePath parent, AtomicBoolean firstInChain) {\n        List<Tree> flat = new ArrayList<>();\n\n        // flatten the expression tree with a pre-order traversal\n        ArrayDeque<Tree> todo = new ArrayDeque<>();\n        todo.add(parent.getLeaf());\n        while (!todo.isEmpty()) {\n            Tree first = todo.removeFirst();\n            if (first.getKind() == Tree.Kind.PLUS) {\n                BinaryTree bt = (BinaryTree) first;\n                todo.addFirst(bt.getRightOperand());\n                todo.addFirst(bt.getLeftOperand());\n            } else {\n                flat.add(first);\n            }\n        }\n\n        int idx = flat.indexOf(path.getLeaf());\n        Verify.verify(idx != -1);\n\n        // walk outwards from the leaf for adjacent string literals to also reflow\n        int startIdx = idx;\n        int endIdx = idx + 1;\n        while (startIdx > 0\n                && flat.get(startIdx - 1).getKind() == Tree.Kind.STRING_LITERAL\n                && noComments(input, unit, flat.get(startIdx - 1), flat.get(startIdx))) {\n            startIdx--;\n        }\n        while (endIdx < flat.size()\n                && flat.get(endIdx).getKind() == Tree.Kind.STRING_LITERAL\n                && noComments(input, unit, flat.get(endIdx - 1), flat.get(endIdx))) {\n            endIdx++;\n        }\n\n        firstInChain.set(startIdx == 0);\n        return ImmutableList.copyOf(flat.subList(startIdx, endIdx));\n    }\n\n    private static boolean noComments(String input, JCTree.JCCompilationUnit unit, Tree one, Tree two) {\n        return STRING_CONCAT_DELIMITER.matchesAllOf(\n                input.subSequence(getEndPosition(unit, one), getStartPosition(two)));\n    }\n\n    public static final CharMatcher STRING_CONCAT_DELIMITER =\n            CharMatcher.whitespace().or(CharMatcher.anyOf(\"\\\"+\"));\n\n    private static int getEndPosition(JCTree.JCCompilationUnit unit, Tree tree) {\n        return ((JCTree) tree).getEndPosition(unit.endPositions);\n    }\n\n    private static int getStartPosition(Tree tree) {\n        return ((JCTree) tree).getStartPosition();\n    }\n\n    /** Returns true if any lines in the given Java source exceed the column limit. */\n    private static boolean longLines(int columnLimit, String input) {\n        // TODO(cushon): consider adding Newlines.lineIterable?\n        Iterator<String> it = Newlines.lineIterator(input);\n        while (it.hasNext()) {\n            String line = it.next();\n            if (line.length() > columnLimit) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    /** Parses the given Java source. */\n    private static JCTree.JCCompilationUnit parse(String source, boolean allowStringFolding) throws FormatterException {\n        Context context = new Context();\n        Options.instance(context).put(\"allowStringFolding\", Boolean.toString(allowStringFolding));\n        return Formatter.parseJcCompilationUnit(context, source);\n    }\n\n    /** Applies replacements to the given string. */\n    private static String applyReplacements(String javaInput, TreeRangeMap<Integer, String> replacementMap) {\n        // process in descending order so the replacement ranges aren't perturbed if any replacements\n        // differ in size from the input\n        Map<Range<Integer>, String> ranges = replacementMap.asDescendingMapOfRanges();\n        if (ranges.isEmpty()) {\n            return javaInput;\n        }\n        StringBuilder sb = new StringBuilder(javaInput);\n        for (Map.Entry<Range<Integer>, String> entry : ranges.entrySet()) {\n            Range<Integer> range = entry.getKey();\n            sb.replace(range.lowerEndpoint(), range.upperEndpoint(), entry.getValue());\n        }\n        return sb.toString();\n    }\n\n    private StringWrapper() {}\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/Formatter.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static java.nio.charset.StandardCharsets.UTF_8;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Iterables;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.io.CharSink;\nimport com.google.common.io.CharSource;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.FormattingError;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.OpsBuilder;\nimport com.palantir.javaformat.OpsBuilder.OpsOutput;\nimport com.palantir.javaformat.Utils;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\nimport com.palantir.javaformat.doc.Level;\nimport com.palantir.javaformat.doc.NoopSink;\nimport com.palantir.javaformat.doc.Obs;\nimport com.palantir.javaformat.doc.Obs.Sink;\nimport com.palantir.javaformat.doc.State;\nimport com.sun.tools.javac.file.JavacFileManager;\nimport com.sun.tools.javac.parser.JavacParser;\nimport com.sun.tools.javac.parser.ParserFactory;\nimport com.sun.tools.javac.tree.JCTree.JCCompilationUnit;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Log;\nimport com.sun.tools.javac.util.Options;\nimport java.io.IOException;\nimport java.net.URI;\nimport java.util.Collection;\nimport javax.tools.Diagnostic;\nimport javax.tools.DiagnosticCollector;\nimport javax.tools.DiagnosticListener;\nimport javax.tools.JavaFileObject;\nimport javax.tools.SimpleJavaFileObject;\nimport javax.tools.StandardLocation;\n\n/**\n * This is google-java-format, a new Java formatter that follows the Google Java Style Guide quite precisely---to the\n * letter and to the spirit.\n *\n * <p>This formatter uses the javac parser to generate an AST. Because the AST loses information about the non-tokens in\n * the input (including newlines, comments, etc.), and even some tokens (e.g., optional commas or semicolons), this\n * formatter lexes the input again and follows along in the resulting list of tokens. Its lexer splits all\n * multi-character operators (like \"&gt;&gt;\") into multiple single-character operators. Each non-token is assigned to a\n * token---non-tokens following a token on the same line go with that token; those following go with the next token---\n * and there is a final EOF token to hold final comments.\n *\n * <p>The formatter walks the AST to generate a Greg Nelson/Derek Oppen-style list of formatting {@link Op}s [1--2] that\n * then generates a structured {@link Doc}. Each AST node type has a visitor to emit a sequence of {@link Op}s for the\n * node.\n *\n * <p>Some data-structure operations are easier in the list of {@link Op}s, while others become easier in the\n * {@link Doc}. The {@link Op}s are walked to attach the comments. As the {@link Op}s are generated, missing input\n * tokens are inserted and incorrect output tokens are dropped, ensuring that the output matches the input even in the\n * face of formatter errors. Finally, the formatter walks the {@link Doc} to format it in the given width.\n *\n * <p>This formatter also produces data structures of which tokens and comments appear where on the input, and on the\n * output, to help output a partial reformatting of a slightly edited input.\n *\n * <p>Instances of the formatter are immutable and thread-safe.\n *\n * <p>[1] Nelson, Greg, and John DeTreville. Personal communication.\n *\n * <p>[2] Oppen, Derek C. \"Prettyprinting\". ACM Transactions on Programming Languages and Systems, Volume 2 Issue 4,\n * Oct. 1980, pp. 465\u2013483.\n */\n@Immutable\npublic final class Formatter {\n\n    static final Range<Integer> EMPTY_RANGE = Range.closedOpen(-1, -1);\n\n    private final JavaFormatterOptions options;\n    private final boolean debugMode;\n\n    @VisibleForTesting\n    Formatter(JavaFormatterOptions options, boolean debugMode) {\n        this.options = options;\n        this.debugMode = debugMode;\n    }\n\n    /** A new Formatter instance with default options. */\n    public static Formatter create() {\n        return new Formatter(JavaFormatterOptions.defaultOptions(), false);\n    }\n\n    public static Formatter createFormatter(JavaFormatterOptions options) {\n        return new Formatter(options, false);\n    }\n\n    /**\n     * Construct a {@code Formatter} given a Java compilation unit. Parses the code; builds a {@link JavaInput} and the\n     * corresponding {@link JavaOutput}.\n     *\n     * @param javaInput the input, a Java compilation unit\n     * @param options the {@link JavaFormatterOptions}\n     * @param commentsHelper the {@link CommentsHelper}, used to rewrite comments\n     * @param debugMode whether to produce debugging output via {@link DebugRenderer}\n     * @return javaOutput the output produced\n     */\n    static JavaOutput format(\n            final JavaInput javaInput, JavaFormatterOptions options, CommentsHelper commentsHelper, boolean debugMode)\n            throws FormatterException {\n\n        Context context = new Context();\n        Options.instance(context).put(\"allowStringFolding\", \"false\");\n\n        JCCompilationUnit unit = parseJcCompilationUnit(context, javaInput.getText());\n\n        // Output the compilation unit.\n        javaInput.setCompilationUnit(unit);\n        OpsBuilder opsBuilder = new OpsBuilder(javaInput);\n\n        JavaInputAstVisitor visitor;\n        if (getRuntimeVersion() >= 14) {\n            try {\n                visitor = Class.forName(\"com.palantir.javaformat.java.java14.Java14InputAstVisitor\")\n                        .asSubclass(JavaInputAstVisitor.class)\n                        .getConstructor(OpsBuilder.class, int.class)\n                        .newInstance(opsBuilder, options.indentationMultiplier());\n            } catch (ReflectiveOperationException e) {\n                throw new RuntimeException(e.getMessage(), e);\n            }\n        } else {\n            visitor = new JavaInputAstVisitor(opsBuilder, options.indentationMultiplier());\n        }\n\n        visitor.scan(unit, null);\n        opsBuilder.sync(javaInput.getText().length());\n        opsBuilder.drain();\n        OpsOutput opsOutput = opsBuilder.build();\n\n        Level doc = new DocBuilder().withOps(opsOutput.ops()).build();\n\n        // Don't even allocate all those JSON nodes if we're not going to write it out\n        Sink sink = debugMode ? new JsonSink() : new NoopSink();\n\n        Obs.ExplorationNode observationNode = Obs.createRoot(sink);\n        State finalState =\n                doc.computeBreaks(commentsHelper, options.maxLineLength(), State.startingState(), observationNode);\n\n        JavaOutput javaOutput = new JavaOutput(javaInput, opsOutput.inputMetadata());\n        doc.write(finalState, javaOutput);\n        javaOutput.flush();\n\n        if (debugMode) {\n            DebugRenderer.render(javaInput, opsOutput, doc, finalState, javaOutput, sink.getOutput());\n        }\n        return javaOutput;\n    }\n\n    static JCCompilationUnit parseJcCompilationUnit(Context context, String sourceText) throws FormatterException {\n        DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>();\n        context.put(DiagnosticListener.class, diagnostics);\n        Options.instance(context).put(\"--enable-preview\", \"true\");\n        JCCompilationUnit unit;\n        JavacFileManager fileManager = new JavacFileManager(context, true, UTF_8);\n        try {\n            fileManager.setLocation(StandardLocation.PLATFORM_CLASS_PATH, ImmutableList.of());\n        } catch (IOException e) {\n            // impossible\n            throw new RuntimeException(e);\n        }\n        SimpleJavaFileObject source = new SimpleJavaFileObject(URI.create(\"source\"), JavaFileObject.Kind.SOURCE) {\n            @Override\n            public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {\n                return sourceText;\n            }\n        };\n        Log.instance(context).useSource(source);\n        ParserFactory parserFactory = ParserFactory.instance(context);\n        JavacParser parser = parserFactory.newParser(\n                sourceText, /*keepDocComments=*/ true, /*keepEndPos=*/ true, /*keepLineMap=*/ true);\n        unit = parser.parseCompilationUnit();\n        unit.sourcefile = source;\n\n        Iterable<Diagnostic<? extends JavaFileObject>> errorDiagnostics =\n                Iterables.filter(diagnostics.getDiagnostics(), Formatter::errorDiagnostic);\n        if (!Iterables.isEmpty(errorDiagnostics)) {\n            throw FormatterExceptions.fromJavacDiagnostics(errorDiagnostics);\n        }\n        return unit;\n    }\n\n    @VisibleForTesting\n    static int getRuntimeVersion() {\n        return Runtime.version().feature();\n    }\n\n    static boolean errorDiagnostic(Diagnostic<?> input) {\n        if (input.getKind() != Diagnostic.Kind.ERROR) {\n            return false;\n        }\n        switch (input.getCode()) {\n            case \"compiler.err.invalid.meth.decl.ret.type.req\":\n                // accept constructor-like method declarations that don't match the name of their\n                // enclosing class\n                return false;\n            default:\n                break;\n        }\n        return true;\n    }\n\n    /**\n     * Format the given input (a Java compilation unit) into the output stream.\n     *\n     * @throws FormatterException if the input cannot be parsed\n     */\n    public void formatSource(CharSource input, CharSink output) throws FormatterException, IOException {\n        // TODO(cushon): proper support for streaming input/output. Input may\n        // not be feasible (parsing) but output should be easier.\n        output.write(formatSource(input.read()));\n    }\n\n    /**\n     * Format an input string (a Java compilation unit) into an output string.\n     *\n     * <p>Leaves import statements untouched.\n     *\n     * @param input the input string\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public String formatSource(String input) throws FormatterException {\n        return formatSource(input, ImmutableList.of(Range.closedOpen(0, input.length())));\n    }\n\n    /**\n     * Formats an input string (a Java compilation unit) and fixes imports.\n     *\n     * <p>Fixing imports includes ordering, spacing, and removal of unused import statements.\n     *\n     * @param input the input string\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     * @see <a href=\"https://google.github.io/styleguide/javaguide.html#s3.3.3-import-ordering-and-spacing\">Google Java\n     *     Style Guide - 3.3.3 Import ordering and spacing</a>\n     */\n    public String formatSourceAndFixImports(String input) throws FormatterException {\n        input = ImportOrderer.reorderImports(input, options.style());\n        input = RemoveUnusedImports.removeUnusedImports(input);\n        String formatted = formatSource(input);\n        formatted = StringWrapper.wrap(options.maxLineLength(), formatted, this);\n        return formatted;\n    }\n\n    /**\n     * Format an input string (a Java compilation unit), for only the specified character ranges. These ranges are\n     * extended as necessary (e.g., to encompass whole lines).\n     *\n     * @param input the input string\n     * @param characterRanges the character ranges to be reformatted\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public String formatSource(String input, Collection<Range<Integer>> characterRanges) throws FormatterException {\n        return Utils.applyReplacements(input, getFormatReplacements(input, characterRanges));\n    }\n\n    /**\n     * Emit a list of {@link Replacement}s to convert from input to output.\n     *\n     * @param input the input compilation unit\n     * @param characterRanges the character ranges to reformat\n     * @return a list of {@link Replacement}s, sorted from low index to high index, without overlaps\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public ImmutableList<Replacement> getFormatReplacements(String input, Collection<Range<Integer>> characterRanges)\n            throws FormatterException {\n        JavaInput javaInput = new JavaInput(input);\n\n        // TODO(cushon): this is only safe because the modifier ordering doesn't affect whitespace,\n        // and doesn't change the replacements that are output. This is not true in general for\n        // 'de-linting' changes (e.g. import ordering).\n        javaInput = ModifierOrderer.reorderModifiers(javaInput, characterRanges);\n\n        JavaCommentsHelper commentsHelper = new JavaCommentsHelper(javaInput.getLineSeparator(), options);\n        JavaOutput javaOutput;\n        try {\n            javaOutput = format(javaInput, options, commentsHelper, debugMode);\n        } catch (FormattingError e) {\n            throw new FormatterException(e.diagnostics());\n        }\n        RangeSet<Integer> tokenRangeSet = javaInput.characterRangesToTokenRanges(characterRanges);\n        return javaOutput.getFormatReplacements(tokenRangeSet);\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaOutput.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.DiscreteDomain;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.Input.Token;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.OpsBuilder.BlankLineWanted;\nimport com.palantir.javaformat.Output;\nimport com.palantir.javaformat.doc.State;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\n\n/*\n * Throughout this file, {@code i} is an index for input lines, {@code j} is an index for output\n * lines, {@code ij} is an index into either input or output lines, and {@code k} is an index for\n * toks.\n */\n\n/**\n * {@code JavaOutput} extends {@link Output Output} to represent a Java output document. It includes methods to emit the\n * output document.\n */\npublic final class JavaOutput extends Output {\n    private final String lineSeparator;\n    private final JavaInput javaInput; // Used to follow along while emitting the output.\n\n    private final List<String> mutableLines = new ArrayList<>();\n    private final int kN; // The number of tokens or comments in the input, excluding the EOF.\n    private final InputMetadata inputMetadata;\n    private int iLine = 0; // Closest corresponding line number on input.\n    private int lastK = -1; // Last {@link Tok} index output.\n    private int spacesPending = 0;\n    private int newlinesPending = 0;\n    private StringBuilder lineBuilder = new StringBuilder();\n\n    /**\n     * {@code JavaOutput} constructor.\n     *\n     * @param javaInput the {@link JavaInput}, used to match up blank lines in the output\n     */\n    public JavaOutput(JavaInput javaInput, InputMetadata inputMetadata) {\n        this.lineSeparator = javaInput.getLineSeparator();\n        this.javaInput = javaInput;\n        kN = javaInput.getkN();\n        this.inputMetadata = inputMetadata;\n    }\n\n    // TODO(jdd): Add invariant.\n    @Override\n    public void append(State state, String text, Range<Integer> range) {\n        if (!range.isEmpty()) {\n            boolean sawNewlines = false;\n            // Skip over input line we've passed.\n            int iN = javaInput.getLineCount();\n            while (iLine < iN\n                    && (javaInput.getRanges(iLine).isEmpty()\n                            || javaInput.getRanges(iLine).upperEndpoint() <= range.lowerEndpoint())) {\n                if (javaInput.getRanges(iLine).isEmpty()) {\n                    // Skipped over a blank line.\n                    sawNewlines = true;\n                }\n                ++iLine;\n            }\n            /*\n             * Output blank line if we've called {@link OpsBuilder#blankLine}{@code (true)} here, or if\n             * there's a blank line here and it's a comment.\n             */\n            BlankLineWanted wanted = inputMetadata.blankLines().getOrDefault(lastK, BlankLineWanted.NO);\n            if (isComment(text) ? sawNewlines : wanted.wanted(state).orElse(sawNewlines)) {\n                ++newlinesPending;\n            }\n        }\n        if (Newlines.isNewline(text)) {\n            /*\n             * Don't update range information, and swallow extra newlines. The case below for '\\n' is for\n             * block comments.\n             */\n            if (newlinesPending == 0) {\n                ++newlinesPending;\n            }\n            spacesPending = 0;\n        } else {\n            boolean rangesSet = false;\n            int textN = text.length();\n            for (int i = 0; i < textN; i++) {\n                char c = text.charAt(i);\n                switch (c) {\n                    case ' ':\n                        ++spacesPending;\n                        break;\n                    case '\\r':\n                        if (i + 1 < text.length() && text.charAt(i + 1) == '\\n') {\n                            i++;\n                        }\n                        // falls through\n                    case '\\n':\n                        spacesPending = 0;\n                        ++newlinesPending;\n                        break;\n                    default:\n                        while (newlinesPending > 0) {\n                            // drop leading blank lines\n                            if (!mutableLines.isEmpty() || lineBuilder.length() > 0) {\n                                mutableLines.add(lineBuilder.toString());\n                            }\n                            lineBuilder = new StringBuilder();\n                            rangesSet = false;\n                            --newlinesPending;\n                        }\n                        while (spacesPending > 0) {\n                            lineBuilder.append(' ');\n                            --spacesPending;\n                        }\n                        lineBuilder.append(c);\n                        if (!range.isEmpty()) {\n                            if (!rangesSet) {\n                                while (ranges.size() <= mutableLines.size()) {\n                                    ranges.add(Formatter.EMPTY_RANGE);\n                                }\n                                ranges.set(mutableLines.size(), union(ranges.get(mutableLines.size()), range));\n                                rangesSet = true;\n                            }\n                        }\n                }\n            }\n        }\n        if (!range.isEmpty()) {\n            lastK = range.upperEndpoint();\n        }\n    }\n\n    @Override\n    public void indent(int indent) {\n        spacesPending = indent;\n    }\n\n    /** Flush any incomplete last line, then add the EOF token into our data structures. */\n    void flush() {\n        String lastLine = lineBuilder.toString();\n        if (!CharMatcher.whitespace().matchesAllOf(lastLine)) {\n            mutableLines.add(lastLine);\n        }\n        int jN = mutableLines.size();\n        Range<Integer> eofRange = Range.closedOpen(kN, kN + 1);\n        while (ranges.size() < jN) {\n            ranges.add(Formatter.EMPTY_RANGE);\n        }\n        ranges.add(eofRange);\n        setLines(ImmutableList.copyOf(mutableLines));\n    }\n\n    // The following methods can be used after the Output has been built.\n\n    /**\n     * Emit a list of {@link Replacement}s to convert from input to output.\n     *\n     * @return a list of {@link Replacement}s, sorted by start index, without overlaps\n     */\n    public ImmutableList<Replacement> getFormatReplacements(RangeSet<Integer> iRangeSet0) {\n        ImmutableList.Builder<Replacement> result = ImmutableList.builder();\n        Map<Integer, Range<Integer>> kToJ = JavaOutput.makeKToIJ(this);\n\n        // Expand the token ranges to align with re-formattable boundaries.\n        RangeSet<Integer> breakableRanges = TreeRangeSet.create();\n        RangeSet<Integer> iRangeSet = iRangeSet0.subRangeSet(Range.closed(0, javaInput.getkN()));\n        for (Range<Integer> iRange : iRangeSet.asRanges()) {\n            Range<Integer> range = expandToBreakableRegions(iRange.canonical(DiscreteDomain.integers()));\n            if (range.equals(EMPTY_RANGE)) {\n                // the range contains only whitespace\n                continue;\n            }\n            breakableRanges.add(range);\n        }\n\n        // Construct replacements for each reformatted region.\n        for (Range<Integer> range : breakableRanges.asRanges()) {\n\n            Input.Tok startTok = startTok(javaInput.getToken(range.lowerEndpoint()));\n            Input.Tok endTok = endTok(javaInput.getToken(range.upperEndpoint() - 1));\n\n            // Add all output lines in the given token range to the replacement.\n            StringBuilder replacement = new StringBuilder();\n\n            int replaceFrom = startTok.getPosition();\n            // Replace leading whitespace in the input with the whitespace from the formatted file\n            while (replaceFrom > 0) {\n                char previous = javaInput.getText().charAt(replaceFrom - 1);\n                if (!CharMatcher.whitespace().matches(previous)) {\n                    break;\n                }\n                replaceFrom--;\n            }\n\n            int i = kToJ.get(startTok.getIndex()).lowerEndpoint();\n            // Include leading blank lines from the formatted output, unless the formatted range\n            // starts at the beginning of the file.\n            while (i > 0 && getLine(i - 1).isEmpty()) {\n                i--;\n            }\n            // Write out the formatted range.\n            for (; i < kToJ.get(endTok.getIndex()).upperEndpoint(); i++) {\n                // It's possible to run out of output lines (e.g. if the input ended with\n                // multiple trailing newlines).\n                if (i < getLineCount()) {\n                    if (i > 0) {\n                        replacement.append(lineSeparator);\n                    }\n                    replacement.append(getLine(i));\n                }\n            }\n\n            int replaceTo = Math.min(\n                    endTok.getPosition() + endTok.length(), javaInput.getText().length());\n            // If the formatted ranged ended in the trailing trivia of the last token before EOF,\n            // format all the way up to EOF to deal with trailing whitespace correctly.\n            if (endTok.getIndex() == javaInput.getkN() - 1) {\n                replaceTo = javaInput.getText().length();\n            }\n            // Replace trailing whitespace in the input with the whitespace from the formatted file.\n            // If the trailing whitespace in the input includes one or more line breaks, preserve the\n            // whitespace after the last newline to avoid re-indenting the line following the formatted\n            // line.\n            int newline = -1;\n            while (replaceTo < javaInput.getText().length()) {\n                char next = javaInput.getText().charAt(replaceTo);\n                if (!CharMatcher.whitespace().matches(next)) {\n                    break;\n                }\n                int newlineLength = Newlines.hasNewlineAt(javaInput.getText(), replaceTo);\n                if (newlineLength != -1) {\n                    newline = replaceTo;\n                    // Skip over the entire newline; don't count the second character of \\r\\n as a newline.\n                    replaceTo += newlineLength;\n                } else {\n                    replaceTo++;\n                }\n            }\n            if (newline != -1) {\n                replaceTo = newline;\n            }\n\n            if (newline == -1) {\n                // There wasn't an existing trailing newline; add one.\n                replacement.append(lineSeparator);\n            }\n            for (; i < getLineCount(); i++) {\n                String after = getLine(i);\n                int idx = CharMatcher.whitespace().negate().indexIn(after);\n                if (idx == -1) {\n                    // Write out trailing empty lines from the formatted output.\n                    replacement.append(lineSeparator);\n                } else {\n                    if (newline == -1) {\n                        // If there wasn't a trailing newline in the input, indent the next line.\n                        replacement.append(after, 0, idx);\n                    }\n                    break;\n                }\n            }\n\n            result.add(Replacement.create(replaceFrom, replaceTo, replacement.toString()));\n        }\n        return result.build();\n    }\n\n    /**\n     * Expand a token range to start and end on acceptable boundaries for re-formatting.\n     *\n     * @param iRange the {@link Range} of tokens\n     * @return the expanded token range\n     */\n    private Range<Integer> expandToBreakableRegions(Range<Integer> iRange) {\n        // The original line range.\n        int loTok = iRange.lowerEndpoint();\n        int hiTok = iRange.upperEndpoint() - 1;\n\n        // Expand the token indices to formattable boundaries (e.g. edges of statements).\n        if (!partialFormatRanges().contains(loTok) || !partialFormatRanges().contains(hiTok)) {\n            return EMPTY_RANGE;\n        }\n        loTok = partialFormatRanges().rangeContaining(loTok).lowerEndpoint();\n        hiTok = partialFormatRanges().rangeContaining(hiTok).upperEndpoint();\n        return Range.closedOpen(loTok, hiTok + 1);\n    }\n\n    private RangeSet<Integer> partialFormatRanges() {\n        return inputMetadata.partialFormatRanges();\n    }\n\n    /** The earliest position of any Tok in the Token, including leading whitespace. */\n    public static int startPosition(Token token) {\n        int min = token.getTok().getPosition();\n        for (Input.Tok tok : token.getToksBefore()) {\n            min = Math.min(min, tok.getPosition());\n        }\n        return min;\n    }\n\n    /** The earliest non-whitespace Tok in the Token. */\n    public static Input.Tok startTok(Token token) {\n        for (Input.Tok tok : token.getToksBefore()) {\n            if (tok.getIndex() >= 0) {\n                return tok;\n            }\n        }\n        return token.getTok();\n    }\n\n    /** The last non-whitespace Tok in the Token. */\n    public static Input.Tok endTok(Token token) {\n        for (int i = token.getToksAfter().size() - 1; i >= 0; i--) {\n            Input.Tok tok = token.getToksAfter().get(i);\n            if (tok.getIndex() >= 0) {\n                return tok;\n            }\n        }\n        return token.getTok();\n    }\n\n    private boolean isComment(String text) {\n        return text.startsWith(\"//\") || text.startsWith(\"/*\");\n    }\n\n    private static Range<Integer> union(Range<Integer> x, Range<Integer> y) {\n        return x.isEmpty() ? y : y.isEmpty() ? x : x.span(y).canonical(DiscreteDomain.integers());\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n                .add(\"iLine\", iLine)\n                .add(\"lastK\", lastK)\n                .add(\"spacesPending\", spacesPending)\n                .add(\"newlinesPending\", newlinesPending)\n                .add(\"inputMetadata\", inputMetadata)\n                .add(\"super\", super.toString())\n                .toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/InputMetadataBuilder.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.java;\n\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableRangeSet;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.OpsBuilder.BlankLineWanted;\nimport java.util.HashMap;\nimport java.util.Map;\n\npublic final class InputMetadataBuilder {\n    private final Map<Integer, BlankLineWanted> blankLines = new HashMap<>(); // Info on blank lines.\n    private final RangeSet<Integer> partialFormatRanges = TreeRangeSet.create();\n\n    /**\n     * A blank line is or is not wanted here.\n     *\n     * @param k the {@link Input.Tok} index\n     * @param wanted whether a blank line is wanted here\n     */\n    public void blankLine(int k, BlankLineWanted wanted) {\n        if (blankLines.containsKey(k)) {\n            blankLines.put(k, blankLines.get(k).merge(wanted));\n        } else {\n            blankLines.put(k, wanted);\n        }\n    }\n\n    /** Marks a region that can be partially formatted. */\n    public void markForPartialFormat(Input.Token start, Input.Token end) {\n        int lo = JavaOutput.startTok(start).getIndex();\n        int hi = JavaOutput.endTok(end).getIndex();\n        partialFormatRanges.add(Range.closed(lo, hi));\n    }\n\n    public InputMetadata build() {\n        return ImmutableInputMetadata.builder()\n                .blankLines(ImmutableMap.copyOf(blankLines))\n                .partialFormatRanges(ImmutableRangeSet.copyOf(partialFormatRanges))\n                .build();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/OpsBuilder.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.MoreObjects;\n...\n", "needle_token_start": 3913, "needle_token_end": 4140, "code_context_ntokens": 16133, "output": ["```java\nprivate static ImmutableSet<Range<Integer>> rangesAfterAppliedReplacements(\n        TreeRangeMap<Integer, String> replacements) {\n    ImmutableSet.Builder<Range<Integer>> outputRanges = ImmutableSet.builder();\n    int offset = 0;\n    for (Map.Entry<Range<Integer>, String> entry :\n            replacements.asMapOfRanges().entrySet()) {\n        Range<Integer> range = entry.getKey();\n        String replacement = entry.getValue();\n\n        int lower = offset + range.lowerEndpoint();\n        int upper = lower + replacement.length();\n        outputRanges.add(Range.closedOpen(lower, upper));\n\n        int originalLength = range.upperEndpoint() - range.lowerEndpoint();\n        int newLength = upper - lower;\n        offset += newLength - originalLength;\n    }\n    return outputRanges.build();\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "isArrayValue", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaInputAstVisitor.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function determines if a given expression represents an array initialization without explicitly specifying the type.\n2. **Input**: The function takes a single expression from a syntax tree.\n3. **Output**: It returns a boolean value indicating whether the input expression is an array initialization of the described form.\n4. **Procedure**: The function checks if the input expression is an assignment. If it is, it further checks if the right-hand side of the assignment is an array creation expression where the type is not specified. It returns true if these conditions are met, otherwise false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        scan(node.getIdentifier(), null);\n        addArguments(node.getArguments(), plusFour);\n        builder.close();\n        if (node.getClassBody() != null) {\n            addBodyDeclarations(node.getClassBody().getMembers(), BracesOrNot.YES, FirstDeclarationsOrNot.YES);\n        }\n        return null;\n    }\n\n    @Override\n    public Void visitConditionalExpression(ConditionalExpressionTree node, Void unused) {\n        sync(node);\n        builder.open(plusFour);\n        scan(node.getCondition(), null);\n        builder.breakOp(\" \");\n        token(\"?\");\n        builder.space();\n        scan(node.getTrueExpression(), null);\n        builder.breakOp(\" \");\n        token(\":\");\n        builder.space();\n        scan(node.getFalseExpression(), null);\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitContinue(ContinueTree node, Void unused) {\n        sync(node);\n        builder.open(plusFour);\n        token(\"continue\");\n        if (node.getLabel() != null) {\n            builder.breakOp(\" \");\n            visit(node.getLabel());\n        }\n        token(\";\");\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitDoWhileLoop(DoWhileLoopTree node, Void unused) {\n        sync(node);\n        token(\"do\");\n        visitStatement(\n                node.getStatement(), CollapseEmptyOrNot.YES, AllowLeadingBlankLine.YES, AllowTrailingBlankLine.YES);\n        if (node.getStatement().getKind() == BLOCK) {\n            builder.space();\n        } else {\n            builder.breakOp(\" \");\n        }\n        token(\"while\");\n        builder.space();\n        token(\"(\");\n        scan(skipParen(node.getCondition()), null);\n        token(\")\");\n        token(\";\");\n        return null;\n    }\n\n    @Override\n    public Void visitEmptyStatement(EmptyStatementTree node, Void unused) {\n        sync(node);\n        dropEmptyDeclarations();\n        return null;\n    }\n\n    @Override\n    public Void visitEnhancedForLoop(EnhancedForLoopTree node, Void unused) {\n        sync(node);\n        builder.open(ZERO);\n        token(\"for\");\n        builder.space();\n        token(\"(\");\n        builder.open(ZERO);\n        visitToDeclare(\n                DeclarationKind.NONE,\n                Direction.HORIZONTAL,\n                node.getVariable(),\n                Optional.of(node.getExpression()),\n                \":\",\n                /* trailing= */ Optional.empty());\n        builder.close();\n        token(\")\");\n        builder.close();\n        visitStatement(\n                node.getStatement(), CollapseEmptyOrNot.YES, AllowLeadingBlankLine.YES, AllowTrailingBlankLine.NO);\n        return null;\n    }\n\n    private void visitEnumConstantDeclaration(VariableTree enumConstant) {\n        for (AnnotationTree annotation : enumConstant.getModifiers().getAnnotations()) {\n            scan(annotation, null);\n            builder.forcedBreak();\n        }\n        visit(enumConstant.getName());\n        NewClassTree init = ((NewClassTree) enumConstant.getInitializer());\n        if (init.getArguments().isEmpty()) {\n            builder.guessToken(\"(\");\n            builder.guessToken(\")\");\n        } else {\n            addArguments(init.getArguments(), plusFour);\n        }\n        if (init.getClassBody() != null) {\n            addBodyDeclarations(init.getClassBody().getMembers(), BracesOrNot.YES, FirstDeclarationsOrNot.YES);\n        }\n    }\n\n    public boolean visitEnumDeclaration(ClassTree node) {\n        sync(node);\n        builder.open(ZERO);\n        visitAndBreakModifiers(\n                node.getModifiers(), Direction.VERTICAL, /* declarationAnnotationBreak= */ Optional.empty());\n        builder.open(plusFour);\n        token(\"enum\");\n        builder.breakOp(\" \");\n        visit(node.getSimpleName());\n        builder.close();\n        builder.close();\n        if (!node.getImplementsClause().isEmpty()) {\n            builder.open(plusFour);\n            builder.breakOp(\" \");\n            builder.open(plusFour);\n            token(\"implements\");\n            builder.breakOp(\" \");\n            builder.open(ZERO);\n            boolean first = true;\n            for (Tree superInterfaceType : node.getImplementsClause()) {\n                if (!first) {\n                    token(\",\");\n                    builder.breakToFill(\" \");\n                }\n                scan(superInterfaceType, null);\n                first = false;\n            }\n            builder.close();\n            builder.close();\n            builder.close();\n        }\n        builder.space();\n        tokenBreakTrailingComment(\"{\", plusTwo);\n        ArrayList<VariableTree> enumConstants = new ArrayList<>();\n        ArrayList<Tree> members = new ArrayList<>();\n        for (Tree member : node.getMembers()) {\n            if (member instanceof JCTree.JCVariableDecl) {\n                JCTree.JCVariableDecl variableDecl = (JCTree.JCVariableDecl) member;\n                if ((variableDecl.mods.flags & Flags.ENUM) == Flags.ENUM) {\n                    enumConstants.add(variableDecl);\n                    continue;\n                }\n            }\n            members.add(member);\n        }\n        if (enumConstants.isEmpty() && members.isEmpty()) {\n            if (builder.peekToken().equals(Optional.of(\";\"))) {\n                builder.open(plusTwo);\n                builder.forcedBreak();\n                token(\";\");\n                builder.forcedBreak();\n                dropEmptyDeclarations();\n                builder.close();\n                builder.open(ZERO);\n                builder.forcedBreak();\n                builder.blankLineWanted(BlankLineWanted.NO);\n                token(\"}\", plusTwo);\n                builder.close();\n            } else {\n                builder.open(ZERO);\n                builder.blankLineWanted(BlankLineWanted.NO);\n                token(\"}\");\n                builder.close();\n            }\n        } else {\n            builder.open(plusTwo);\n            builder.blankLineWanted(BlankLineWanted.NO);\n            builder.forcedBreak();\n            builder.open(ZERO);\n            boolean first = true;\n            for (VariableTree enumConstant : enumConstants) {\n                if (!first) {\n                    token(\",\");\n                    builder.forcedBreak();\n                    builder.blankLineWanted(BlankLineWanted.PRESERVE);\n                }\n                markForPartialFormat();\n                visitEnumConstantDeclaration(enumConstant);\n                first = false;\n            }\n            if (builder.peekToken().orElse(\"\").equals(\",\")) {\n                token(\",\");\n                builder.forcedBreak(); // The \";\" goes on its own line.\n            }\n            builder.close();\n            builder.close();\n            if (builder.peekToken().equals(Optional.of(\";\"))) {\n                builder.open(plusTwo);\n                token(\";\");\n                builder.forcedBreak();\n                dropEmptyDeclarations();\n                builder.close();\n            }\n            builder.open(ZERO);\n            addBodyDeclarations(members, BracesOrNot.NO, FirstDeclarationsOrNot.NO);\n            builder.forcedBreak();\n            builder.blankLineWanted(BlankLineWanted.NO);\n            token(\"}\", plusTwo);\n            builder.close();\n        }\n        builder.guessToken(\";\");\n        return false;\n    }\n\n    @Override\n    public Void visitMemberReference(MemberReferenceTree node, Void unused) {\n        sync(node);\n        builder.open(OpenOp.builder()\n                .plusIndent(plusFour)\n                .debugName(\"methodReference\")\n                .breakabilityIfLastLevel(LastLevelBreakability.CHECK_INNER)\n                .breakBehaviour(BreakBehaviours.inlineSuffix())\n                .build());\n        scan(node.getQualifierExpression(), null);\n        builder.open(ZERO);\n        builder.breakOp();\n        builder.op(\"::\");\n        addTypeArguments(node.getTypeArguments(), plusFour);\n        switch (node.getMode()) {\n            case INVOKE:\n                visit(node.getName());\n                break;\n            case NEW:\n                token(\"new\");\n                break;\n            default:\n                throw new IllegalArgumentException(node.getMode().name());\n        }\n        builder.close();\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitExpressionStatement(ExpressionStatementTree node, Void unused) {\n        sync(node);\n        scan(node.getExpression(), null);\n        token(\";\");\n        return null;\n    }\n\n    @Override\n    public Void visitVariable(VariableTree node, Void unused) {\n        sync(node);\n        visitVariables(ImmutableList.of(node), DeclarationKind.NONE, inlineAnnotationDirection(node.getModifiers()));\n        return null;\n    }\n\n    void visitVariables(List<VariableTree> fragments, DeclarationKind declarationKind, Direction annotationDirection) {\n        if (fragments.size() == 1) {\n            VariableTree fragment = fragments.get(0);\n            declareOne(\n                    declarationKind,\n                    annotationDirection,\n                    Optional.of(fragment.getModifiers()),\n                    fragment.getType(),\n                    /* name= */ fragment.getName(),\n                    \"\",\n                    \"=\",\n                    Optional.ofNullable(fragment.getInitializer()),\n                    Optional.of(\";\"),\n                    /* receiverExpression= */ Optional.empty(),\n                    Optional.ofNullable(variableFragmentDims(true, 0, fragment.getType())));\n\n        } else {\n            declareMany(fragments, annotationDirection);\n        }\n    }\n\n    private TypeWithDims variableFragmentDims(boolean first, int leadingDims, Tree type) {\n        if (type == null) {\n            return null;\n        }\n        if (first) {\n            return DimensionHelpers.extractDims(type, SortedDims.YES);\n        }\n        TypeWithDims dims = DimensionHelpers.extractDims(type, SortedDims.NO);\n        return new TypeWithDims(\n                null, leadingDims > 0 ? dims.dims.subList(0, dims.dims.size() - leadingDims) : dims.dims);\n    }\n\n    @Override\n    public Void visitForLoop(ForLoopTree node, Void unused) {\n        sync(node);\n        token(\"for\");\n        builder.space();\n        token(\"(\");\n        builder.open(plusFour);\n        builder.open(\n                node.getInitializer().size() > 1\n                                && node.getInitializer().get(0).getKind() == Tree.Kind.EXPRESSION_STATEMENT\n                        ? plusFour\n                        : ZERO);\n        if (!node.getInitializer().isEmpty()) {\n            if (node.getInitializer().get(0).getKind() == VARIABLE) {\n                PeekingIterator<StatementTree> it =\n                        Iterators.peekingIterator(node.getInitializer().iterator());\n                visitVariables(variableFragments(it, it.next()), DeclarationKind.NONE, Direction.HORIZONTAL);\n            } else {\n                boolean first = true;\n                builder.open(ZERO);\n                for (StatementTree t : node.getInitializer()) {\n                    if (!first) {\n                        token(\",\");\n                        builder.breakOp(\" \");\n                    }\n                    scan(((ExpressionStatementTree) t).getExpression(), null);\n                    first = false;\n                }\n                token(\";\");\n                builder.close();\n            }\n        } else {\n            token(\";\");\n        }\n        builder.close();\n        builder.breakOp(\" \");\n        if (node.getCondition() != null) {\n            scan(node.getCondition(), null);\n        }\n        token(\";\");\n        if (!node.getUpdate().isEmpty()) {\n            builder.breakOp(\" \");\n            builder.open(node.getUpdate().size() <= 1 ? ZERO : plusFour);\n            boolean firstUpdater = true;\n            for (ExpressionStatementTree updater : node.getUpdate()) {\n                if (!firstUpdater) {\n                    token(\",\");\n                    builder.breakToFill(\" \");\n                }\n                scan(updater.getExpression(), null);\n                firstUpdater = false;\n            }\n            builder.guessToken(\";\");\n            builder.close();\n        } else {\n            builder.space();\n        }\n        builder.close();\n        token(\")\");\n        visitStatement(\n                node.getStatement(), CollapseEmptyOrNot.YES, AllowLeadingBlankLine.YES, AllowTrailingBlankLine.NO);\n        return null;\n    }\n\n    @Override\n    public Void visitIf(IfTree node, Void unused) {\n        sync(node);\n        // Collapse chains of else-ifs.\n        List<ExpressionTree> expressions = new ArrayList<>();\n        List<StatementTree> statements = new ArrayList<>();\n        while (true) {\n            expressions.add(node.getCondition());\n            statements.add(node.getThenStatement());\n            if (node.getElseStatement() != null && node.getElseStatement().getKind() == IF) {\n                node = (IfTree) node.getElseStatement();\n            } else {\n                break;\n            }\n        }\n        builder.open(ZERO);\n        boolean first = true;\n        boolean followingBlock = false;\n        int expressionsN = expressions.size();\n        for (int i = 0; i < expressionsN; i++) {\n            if (!first) {\n                if (followingBlock) {\n                    builder.space();\n                } else {\n                    builder.forcedBreak();\n                }\n                token(\"else\");\n                builder.space();\n            }\n            token(\"if\");\n            builder.space();\n            token(\"(\");\n            scan(skipParen(expressions.get(i)), null);\n            token(\")\");\n            // An empty block can collapse to \"{}\" if there are no if/else or else clauses\n            boolean onlyClause = expressionsN == 1 && node.getElseStatement() == null;\n            // Trailing blank lines are permitted if this isn't the last clause\n            boolean trailingClauses = i < expressionsN - 1 || node.getElseStatement() != null;\n            visitStatement(\n                    statements.get(i),\n                    CollapseEmptyOrNot.valueOf(onlyClause),\n                    AllowLeadingBlankLine.YES,\n                    AllowTrailingBlankLine.valueOf(trailingClauses));\n            followingBlock = statements.get(i).getKind() == BLOCK;\n            first = false;\n        }\n        if (node.getElseStatement() != null) {\n            if (followingBlock) {\n                builder.space();\n            } else {\n                builder.forcedBreak();\n            }\n            token(\"else\");\n            visitStatement(\n                    node.getElseStatement(),\n                    CollapseEmptyOrNot.NO,\n                    AllowLeadingBlankLine.YES,\n                    AllowTrailingBlankLine.NO);\n        }\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitImport(ImportTree node, Void unused) {\n        sync(node);\n        token(\"import\");\n        builder.space();\n        if (node.isStatic()) {\n            token(\"static\");\n            builder.space();\n        }\n        visitName(node.getQualifiedIdentifier());\n        token(\";\");\n        // TODO(cushon): remove this if https://bugs.openjdk.java.net/browse/JDK-8027682 is fixed\n        dropEmptyDeclarations();\n        return null;\n    }\n\n    @Override\n    public Void visitBinary(BinaryTree node, Void unused) {\n        sync(node);\n        /*\n         * Collect together all operators with same precedence to clean up indentation.\n         */\n        List<ExpressionTree> operands = new ArrayList<>();\n        List<String> operators = new ArrayList<>();\n        walkInfix(precedence(node), node, operands, operators);\n        boolean isStringConcat = isStringConcat(node);\n        boolean shouldPreserveNewlines = isStringConcat && lineSpan(node) > 2;\n        FillMode fillMode = hasOnlyShortItems(operands) || isStringConcat ? INDEPENDENT : UNIFIED;\n\n        builder.open(\n                plusFour,\n                BreakBehaviours.breakThisLevel(),\n                LastLevelBreakability.ACCEPT_INLINE_CHAIN_IF_SIMPLE_OTHERWISE_CHECK_INNER);\n        scan(operands.get(0), null);\n        int operatorsN = operators.size();\n        boolean shouldEnforceNewline = builder.mostRecentTokenFollowedByNewline();\n        for (int i = 0; i < operatorsN; i++) {\n            FillMode nextFillMode = shouldPreserveNewlines && shouldEnforceNewline ? UNIFIED : fillMode;\n            builder.breakOp(nextFillMode, \" \", ZERO);\n            builder.op(operators.get(i));\n            shouldEnforceNewline = builder.mostRecentTokenFollowedByNewline();\n            builder.space();\n            scan(operands.get(i + 1), null);\n            shouldEnforceNewline = shouldEnforceNewline || builder.mostRecentTokenFollowedByNewline();\n        }\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitInstanceOf(InstanceOfTree node, Void unused) {\n        sync(node);\n        builder.open(plusFour);\n        scan(node.getExpression(), null);\n        builder.breakOp(\" \");\n        builder.open(ZERO);\n        token(\"instanceof\");\n        builder.breakOp(\" \");\n        scan(node.getType(), null);\n        builder.close();\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitIntersectionType(IntersectionTypeTree node, Void unused) {\n        sync(node);\n        builder.open(plusFour);\n        boolean first = true;\n        for (Tree type : node.getBounds()) {\n            if (!first) {\n                builder.breakToFill(\" \");\n                token(\"&\");\n                builder.space();\n            }\n            scan(type, null);\n            first = false;\n        }\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitLabeledStatement(LabeledStatementTree node, Void unused) {\n        sync(node);\n        builder.open(ZERO);\n        visit(node.getLabel());\n        token(\":\");\n        builder.forcedBreak();\n        builder.close();\n        scan(node.getStatement(), null);\n        return null;\n    }\n\n    @Override\n    public Void visitLambdaExpression(LambdaExpressionTree node, Void unused) {\n        sync(node);\n        boolean statementBody = node.getBodyKind() == LambdaExpressionTree.BodyKind.STATEMENT;\n        visitLambdaExpression(node, statementBody);\n        return null;\n    }\n\n    protected void visitLambdaExpression(LambdaExpressionTree node, boolean statementBody) {\n        boolean parens = builder.peekToken().equals(Optional.of(\"(\"));\n        builder.open(\"lambda arguments\", parens ? plusFour : ZERO);\n        if (parens) {\n            token(\"(\");\n        }\n        boolean first = true;\n        for (VariableTree parameter : node.getParameters()) {\n            if (!first) {\n                token(\",\");\n                builder.breakOp(\" \");\n            }\n            visitVariables(\n                    ImmutableList.of(parameter),\n                    DeclarationKind.NONE,\n                    inlineAnnotationDirection(parameter.getModifiers()));\n            first = false;\n        }\n        if (parens) {\n            token(\")\");\n        }\n        builder.close();\n        builder.space();\n        builder.op(\"->\");\n        builder.open(OpenOp.builder()\n                .debugName(\"lambda body\")\n                .plusIndent(statementBody ? ZERO : plusFour)\n                .breakBehaviour(\n                        statementBody\n                                ? BreakBehaviours.preferBreakingLastInnerLevel(true)\n                                : BreakBehaviours.breakOnlyIfInnerLevelsThenFitOnOneLine(false))\n                .breakabilityIfLastLevel(\n                        statementBody\n                                ? LastLevelBreakability.ACCEPT_INLINE_CHAIN\n                                : LastLevelBreakability.ACCEPT_INLINE_CHAIN_IF_SIMPLE_OTHERWISE_CHECK_INNER)\n                .build());\n        if (statementBody) {\n            builder.space();\n        } else {\n            builder.breakOp(\" \");\n        }\n\n        if (node.getBody().getKind() == Tree.Kind.BLOCK) {\n            visitBlock(\n                    (BlockTree) node.getBody(),\n                    CollapseEmptyOrNot.YES,\n                    AllowLeadingBlankLine.NO,\n                    AllowTrailingBlankLine.NO);\n        } else {\n            scan(node.getBody(), null);\n        }\n        builder.close();\n    }\n\n    @Override\n    public Void visitAnnotation(AnnotationTree node, Void unused) {\n        sync(node);\n\n        if (visitSingleMemberAnnotation(node)) {\n            return null;\n        }\n\n        builder.open(ZERO);\n        token(\"@\");\n        scan(node.getAnnotationType(), null);\n        if (!node.getArguments().isEmpty()) {\n            builder.open(plusFour);\n            token(\"(\");\n            builder.breakOp();\n            boolean first = true;\n\n            // Format the member value pairs one-per-line if any of them are\n            // initialized with arrays.\n            boolean hasArrayInitializer = Iterables.any(node.getArguments(), JavaInputAstVisitor::isArrayValue);\n            for (ExpressionTree argument : node.getArguments()) {\n                if (!first) {\n                    token(\",\");\n                    if (hasArrayInitializer) {\n                        builder.forcedBreak();\n                    } else {\n                        builder.breakOp(\" \");\n                    }\n                }\n                if (argument instanceof AssignmentTree) {\n                    visitAnnotationArgument((AssignmentTree) argument);\n                } else {\n                    scan(argument, null);\n                }\n                first = false;\n            }\n            token(\")\");\n            builder.close();\n            builder.close();\n            return null;\n\n        } else if (builder.peekToken().equals(Optional.of(\"(\"))) {\n            token(\"(\");\n            token(\")\");\n        }\n        builder.close();\n        return null;\n    }\n\n    \nprivate static boolean isArrayValue(ExpressionTree argument) {\n        if (!(argument instanceof AssignmentTree)) {\n            return false;\n        }\n        ExpressionTree expression = ((AssignmentTree) argument).getExpression();\n        return expression instanceof NewArrayTree && ((NewArrayTree) expression).getType() == null;\n    }\n\n    public void visitAnnotationArgument(AssignmentTree node) {\n        boolean isArrayInitializer = node.getExpression().getKind() == NEW_ARRAY;\n        sync(node);\n        builder.open(\n                isArrayInitializer ? ZERO : plusFour,\n                BreakBehaviours.preferBreakingLastInnerLevel(true),\n                LastLevelBreakability.ACCEPT_INLINE_CHAIN_IF_SIMPLE_OTHERWISE_CHECK_INNER);\n        scan(node.getVariable(), null);\n        builder.space();\n        token(\"=\");\n        if (isArrayInitializer) {\n            builder.space();\n        } else {\n            builder.breakOp(\" \");\n        }\n        scan(node.getExpression(), null);\n        builder.close();\n    }\n\n    @Override\n    public Void visitAnnotatedType(AnnotatedTypeTree node, Void unused) {\n        sync(node);\n        ExpressionTree base = node.getUnderlyingType();\n        if (base instanceof MemberSelectTree) {\n            MemberSelectTree selectTree = (MemberSelectTree) base;\n            scan(selectTree.getExpression(), null);\n            token(\".\");\n            visitAnnotations(node.getAnnotations(), BreakOrNot.NO, BreakOrNot.NO);\n            builder.breakToFill(\" \");\n            visit(selectTree.getIdentifier());\n        } else if (base instanceof ArrayTypeTree) {\n            visitAnnotatedArrayType(node);\n        } else {\n            visitAnnotations(node.getAnnotations(), BreakOrNot.NO, BreakOrNot.NO);\n            builder.breakToFill(\" \");\n            scan(base, null);\n        }\n        return null;\n    }\n\n    // TODO(fawind): Use Flags.COMPACT_RECORD_CONSTRUCTOR once if/when we drop support for Java 11\n    protected static final long COMPACT_RECORD_CONSTRUCTOR = 1L << 51;\n\n    protected static final long RECORD = 1L << 61;\n\n    @Override\n    public Void visitMethod(MethodTree node, Void unused) {\n        sync(node);\n        List<? extends AnnotationTree> annotations = node.getModifiers().getAnnotations();\n        List<? extends AnnotationTree> returnTypeAnnotations = ImmutableList.of();\n\n        boolean isRecordConstructor =\n                (((JCTree.JCMethodDecl) node).mods.flags & COMPACT_RECORD_CONSTRUCTOR) == COMPACT_RECORD_CONSTRUCTOR;\n\n        if (!node.getTypeParameters().isEmpty() && !annotations.isEmpty()) {\n            int typeParameterStart = getStartPosition(node.getTypeParameters().get(0));\n            for (int i = 0; i < annotations.size(); i++) {\n                if (getStartPosition(annotations.get(i)) > typeParameterStart) {\n                    returnTypeAnnotations = annotations.subList(i, annotations.size());\n                    annotations = annotations.subList(0, i);\n                    break;\n                }\n            }\n        }\n        builder.addAll(\n                visitModifiers(annotations, Direction.VERTICAL, /* declarationAnnotationBreak= */ Optional.empty()));\n\n        Tree baseReturnType = null;\n        Deque<List<? extends AnnotationTree>> dims = null;\n        if (node.getReturnType() != null) {\n            TypeWithDims extractedDims = DimensionHelpers.extractDims(node.getReturnType(), SortedDims.YES);\n            baseReturnType = extractedDims.node;\n            dims = new ArrayDeque<>(extractedDims.dims);\n        }\n\n        builder.open(plusFour);\n        BreakTag breakBeforeName = new BreakTag();\n        BreakTag breakBeforeType = new BreakTag();\n        builder.open(ZERO);\n        {\n            boolean first = true;\n            if (!node.getTypeParameters().isEmpty()) {\n                token(\"<\");\n                typeParametersRest(node.getTypeParameters(), plusFour);\n                if (!returnTypeAnnotations.isEmpty()) {\n                    builder.breakToFill(\" \");\n                    visitAnnotations(returnTypeAnnotations, BreakOrNot.NO, BreakOrNot.NO);\n                }\n                first = false;\n            }\n\n            boolean openedNameAndTypeScope = false;\n            // constructor-like declarations that don't match the name of the enclosing class are\n            // parsed as method declarations with a null return type\n            if (baseReturnType != null) {\n                if (!first) {\n                    builder.breakOp(INDEPENDENT, \" \", ZERO, Optional.of(breakBeforeType));\n                } else {\n                    first = false;\n                }\n                if (!openedNameAndTypeScope) {\n                    builder.open(make(breakBeforeType, plusFour, ZERO));\n                    openedNameAndTypeScope = true;\n                }\n                scan(baseReturnType, null);\n                maybeAddDims(dims);\n            }\n            if (!first) {\n                builder.breakOp(FillMode.INDEPENDENT, \" \", ZERO, Optional.of(breakBeforeName));\n            } else {\n                first = false;\n            }\n            if (!openedNameAndTypeScope) {\n                builder.open(ZERO);\n                openedNameAndTypeScope = true;\n            }\n            String name = node.getName().toString();\n            if (name.equals(\"<init>\")) {\n                name = builder.peekToken().get();\n            }\n            token(name);\n            if (!isRecordConstructor) {\n                token(\"(\");\n            }\n            // end of name and type scope\n            builder.close();\n        }\n        builder.close();\n\n        builder.open(Indent.If.make(breakBeforeName, plusFour, ZERO));\n        builder.open(Indent.If.make(breakBeforeType, plusFour, ZERO));\n        builder.open(ZERO);\n        {\n            if (!isRecordConstructor) {\n                if (!node.getParameters().isEmpty() || node.getReceiverParameter() != null) {\n                    // Break before args.\n                    builder.breakToFill(\"\");\n                    visitFormals(Optional.ofNullable(node.getReceiverParameter()), node.getParameters());\n                }\n                token(\")\");\n            }\n            if (dims != null) {\n                maybeAddDims(dims);\n            }\n            if (!node.getThrows().isEmpty()) {\n                builder.breakToFill(\" \");\n                builder.open(plusFour);\n                {\n                    visitThrowsClause(node.getThrows());\n                }\n                builder.close();\n            }\n            if (node.getDefaultValue() != null) {\n                builder.space();\n                token(\"default\");\n                if (node.getDefaultValue().getKind() == Tree.Kind.NEW_ARRAY) {\n                    builder.open(minusFour);\n                    {\n                        builder.space();\n                        scan(node.getDefaultValue(), null);\n                    }\n                    builder.close();\n                } else {\n                    builder.open(ZERO);\n                    {\n                        builder.breakToFill(\" \");\n                        scan(node.getDefaultValue(), null);\n                    }\n                    builder.close();\n                }\n            }\n        }\n        builder.close();\n        builder.close();\n        builder.close();\n        if (node.getBody() == null) {\n            token(\";\");\n        } else {\n            builder.space();\n            builder.token(\"{\", Token.RealOrImaginary.REAL, plusTwo, Optional.of(plusTwo));\n        }\n        builder.close();\n\n        if (node.getBody() != null) {\n            methodBody(node);\n        }\n\n        return null;\n    }\n\n    private void methodBody(MethodTree node) {\n        if (node.getBody().getStatements().isEmpty()) {\n            builder.blankLineWanted(BlankLineWanted.NO);\n        } else {\n            builder.open(plusTwo);\n            builder.forcedBreak();\n            builder.blankLineWanted(BlankLineWanted.PRESERVE);\n            visitStatements(node.getBody().getStatements(), false);\n            builder.close();\n            builder.forcedBreak();\n            builder.blankLineWanted(BlankLineWanted.NO);\n            markForPartialFormat();\n        }\n        token(\"}\", plusTwo);\n    }\n\n    @Override\n    public Void visitMethodInvocation(MethodInvocationTree node, Void unused) {\n        sync(node);\n        if (handleLogStatement(node)) {\n            return null;\n        }\n        visitDot(node);\n        return null;\n    }\n\n    /**\n     * Special-cases log statements, to output:\n     *\n     * <pre>{@code\n     * logger.atInfo().log(\n     *     \"Number of foos: %d, foos.size());\n     * }</pre>\n     *\n     * <p>Instead of:\n     *\n     * <pre>{@code\n     * logger\n     *     .atInfo()\n     *     .log(\n     *         \"Number of foos: %d, foos.size());\n     * }</pre>\n     */\n    private boolean handleLogStatement(MethodInvocationTree node) {\n        if (!getMethodName(node).contentEquals(\"log\")) {\n            return false;\n        }\n        Deque<ExpressionTree> parts = new ArrayDeque<>();\n        ExpressionTree curr = node;\n        while (curr instanceof MethodInvocationTree) {\n            MethodInvocationTree method = (MethodInvocationTree) curr;\n            parts.addFirst(method);\n            if (!LOG_METHODS.contains(getMethodName(method).toString())) {\n                return false;\n            }\n            curr = Trees.getMethodReceiver(method);\n        }\n        if (!(curr instanceof IdentifierTree)) {\n            return false;\n        }\n        parts.addFirst(curr);\n        visitDotWithPrefix(ImmutableList.copyOf(parts), false, ImmutableList.of(parts.size() - 1), INDEPENDENT);\n        return true;\n    }\n\n    static final ImmutableSet<String> LOG_METHODS = ImmutableSet.of(\n            \"at\",\n            \"atConfig\",\n            \"atDebug\",\n            \"atFine\",\n            \"atFiner\",\n            \"atFinest\",\n            \"atInfo\",\n            \"atMostEvery\",\n            \"atSevere\",\n            \"atWarning\",\n            \"every\",\n            \"log\",\n            \"logVarargs\",\n            \"perUnique\",\n            \"withCause\",\n            \"withStackTrace\");\n\n    private static List<Long> handleStream(List<ExpressionTree> parts) {\n        return indexes(parts.stream(), p -> {\n                    if (!(p instanceof MethodInvocationTree)) {\n                        return false;\n                    }\n                    Name name = getMethodName((MethodInvocationTree) p);\n                    return Stream.of(\"stream\", \"parallelStream\", \"toBuilder\").anyMatch(name::contentEquals);\n                })\n                .collect(toList());\n    }\n\n    private static <T> Stream<Long> indexes(Stream<T> stream, Predicate<T> predicate) {\n        return Streams.mapWithIndex(stream, (x, i) -> predicate.apply(x) ? i : -1)\n                .filter(x -> x != -1);\n    }\n\n    @Override\n    public Void visitMemberSelect(MemberSelectTree node, Void unused) {\n        sync(node);\n        visitDot(node);\n        return null;\n    }\n\n    @Override\n    public Void visitLiteral(LiteralTree node, Void unused) {\n        sync(node);\n        String sourceForNode = getSourceForNode(node, getCurrentPath());\n        // A negative numeric literal -n is usually represented as unary minus on n,\n        // but that doesn't work for integer or long MIN_VALUE. The parser works\n        // around that by representing it directly as a signed literal (with no\n        // unary minus), but the lexer still expects two tokens.\n        if (sourceForNode.startsWith(\"-\")) {\n            token(\"-\");\n            sourceForNode = sourceForNode.substring(1).trim();\n        }\n        token(sourceForNode);\n        return null;\n    }\n\n    private void visitPackage(ExpressionTree packageName, List<? extends AnnotationTree> packageAnnotations) {\n        if (!packageAnnotations.isEmpty()) {\n            for (AnnotationTree annotation : packageAnnotations) {\n                builder.forcedBreak();\n                scan(annotation, null);\n            }\n            builder.forcedBreak();\n        }\n        builder.open(plusFour);\n        token(\"package\");\n        builder.space();\n        visitName(packageName);\n        builder.close();\n        token(\";\");\n    }\n\n    @Override\n    public Void visitParameterizedType(ParameterizedTypeTree node, Void unused) {\n        sync(node);\n        if (node.getTypeArguments().isEmpty()) {\n            scan(node.getType(), null);\n            token(\"<\");\n            token(\">\");\n        } else {\n            builder.open(plusFour);\n            scan(node.getType(), null);\n            token(\"<\");\n            builder.breakOp();\n            builder.open(ZERO);\n            boolean first = true;\n            for (Tree typeArgument : node.getTypeArguments()) {\n                if (!first) {\n                    token(\",\");\n                    builder.breakOp(\" \");\n                }\n                scan(typeArgument, null);\n                first = false;\n            }\n            builder.close();\n            builder.close();\n            token(\">\");\n        }\n        return null;\n    }\n\n    @Override\n    public Void visitParenthesized(ParenthesizedTree node, Void unused) {\n        token(\"(\");\n        scan(node.getExpression(), null);\n        token(\")\");\n        return null;\n    }\n\n    @Override\n    public Void visitUnary(UnaryTree node, Void unused) {\n        sync(node);\n        String operatorName = operatorName(node);\n        if (((JCTree) node).getTag().isPostUnaryOp()) {\n            scan(node.getExpression(), null);\n            splitToken(operatorName);\n        } else {\n            splitToken(operatorName);\n            if (ambiguousUnaryOperator(node, operatorName)) {\n                builder.space();\n            }\n            scan(node.getExpression(), null);\n        }\n        return null;\n    }\n\n    private void splitToken(String operatorName) {\n        for (int i = 0; i < operatorName.length(); i++) {\n            token(String.valueOf(operatorName.charAt(i)));\n        }\n    }\n\n    private boolean ambiguousUnaryOperator(UnaryTree node, String operatorName) {\n        switch (node.getKind()) {\n            case UNARY_MINUS:\n            case UNARY_PLUS:\n                break;\n            default:\n                return false;\n        }\n        if (!(node.getExpression() instanceof UnaryTree)) {\n            return false;\n        }\n        JCTree.Tag tag = ((JCTree) node.getExpression()).getTag();\n        if (tag.isPostUnaryOp()) {\n            return false;\n        }\n        if (!operatorName(node).startsWith(operatorName)) {\n            return false;\n        }\n        return true;\n    }\n\n    @Override\n    public Void visitPrimitiveType(PrimitiveTypeTree node, Void unused) {\n        sync(node);\n        switch (node.getPrimitiveTypeKind()) {\n            case BOOLEAN:\n                token(\"boolean\");\n                break;\n            case BYTE:\n                token(\"byte\");\n                break;\n            case SHORT:\n                token(\"short\");\n                break;\n            case INT:\n                token(\"int\");\n                break;\n            case LONG:\n                token(\"long\");\n                break;\n            case CHAR:\n                token(\"char\");\n                break;\n            case FLOAT:\n                token(\"float\");\n                break;\n            case DOUBLE:\n                token(\"double\");\n                break;\n            case VOID:\n                token(\"void\");\n                break;\n            default:\n                throw new RuntimeException(node.getPrimitiveTypeKind().name());\n        }\n        return null;\n    }\n\n    public boolean visit(Name name) {\n        token(name.toString());\n        return false;\n    }\n\n    @Override\n    public Void visitReturn(ReturnTree node, Void unused) {\n        sync(node);\n        token(\"return\");\n        if (node.getExpression() != null) {\n            builder.space();\n            scan(node.getExpression(), null);\n        }\n        token(\";\");\n        return null;\n    }\n\n    // TODO(cushon): is this worth special-casing?\n    boolean visitSingleMemberAnnotation(AnnotationTree node) {\n        if (node.getArguments().size() != 1) {\n            return false;\n        }\n        ExpressionTree value = getOnlyElement(node.getArguments());\n        if (value.getKind() == ASSIGNMENT) {\n            return false;\n        }\n        boolean isArrayInitializer = value.getKind() == NEW_ARRAY;\n        builder.open(\n                isArrayInitializer ? ZERO : plusFour,\n                BreakBehaviours.preferBreakingLastInnerLevel(true),\n                LastLevelBreakability.CHECK_INNER);\n        token(\"@\");\n        scan(node.getAnnotationType(), null);\n        token(\"(\");\n        if (!isArrayInitializer) {\n            builder.breakOp();\n        }\n        scan(value, null);\n        builder.close();\n        token(\")\");\n        return true;\n    }\n\n    @Override\n    public Void visitCase(CaseTree node, Void unused) {\n        sync(node);\n        markForPartialFormat();\n        builder.forcedBreak();\n        if (node.getExpression() == null) {\n            token(\"default\", plusTwo);\n            token(\":\");\n        } else {\n            token(\"case\", plusTwo);\n            builder.space();\n            scan(node.getExpression(), null);\n            token(\":\");\n        }\n        boolean isBlock =\n                node.getStatements().size() == 1 && node.getStatements().get(0).getKind() == BLOCK;\n        builder.open(isBlock ? ZERO : plusTwo);\n        if (isBlock) {\n            builder.space();\n        }\n        visitStatements(node.getStatements(), isBlock);\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitSwitch(SwitchTree node, Void unused) {\n        sync(node);\n        visitSwitch(node.getExpression(), node.getCases());\n        return null;\n    }\n\n    protected void visitSwitch(ExpressionTree expression, List<? extends CaseTree> cases) {\n        token(\"switch\");\n        builder.space();\n        token(\"(\");\n        scan(skipParen(expression), null);\n        token(\")\");\n        builder.space();\n        tokenBreakTrailingComment(\"{\", plusTwo);\n        builder.blankLineWanted(BlankLineWanted.NO);\n        builder.open(plusTwo);\n        boolean first = true;\n        for (CaseTree caseTree : cases) {\n            if (!first) {\n                builder.blankLineWanted(BlankLineWanted.PRESERVE);\n            }\n            scan(caseTree, null);\n            first = false;\n        }\n        builder.close();\n        builder.forcedBreak();\n        builder.blankLineWanted(BlankLineWanted.NO);\n        token(\"}\", plusFour);\n    }\n\n    @Override\n    public Void visitSynchronized(SynchronizedTree node, Void unused) {\n        sync(node);\n        token(\"synchronized\");\n        builder.space();\n        token(\"(\");\n        builder.open(plusFour);\n        builder.breakOp();\n        scan(skipParen(node.getExpression()), null);\n        builder.close();\n        token(\")\");\n        builder.space();\n        scan(node.getBlock(), null);\n        return null;\n    }\n\n    @Override\n    public Void visitThrow(ThrowTree node, Void unused) {\n        sync(node);\n        token(\"throw\");\n        builder.space();\n        scan(node.getExpression(), null);\n        token(\";\");\n        return null;\n    }\n\n    @Override\n    public Void visitTry(TryTree node, Void unused) {\n        sync(node);\n        builder.open(ZERO);\n        token(\"try\");\n        builder.space();\n        if (!node.getResources().isEmpty()) {\n            token(\"(\");\n            builder.open(node.getResources().size() > 1 ? plusFour : ZERO);\n            boolean first = true;\n            for (Tree resource : node.getResources()) {\n                if (!first) {\n                    builder.forcedBreak();\n                }\n                if (resource instanceof VariableTree) {\n                    VariableTree variableTree = (VariableTree) resource;\n                    declareOne(\n                            DeclarationKind.PARAMETER,\n                            inlineAnnotationDirection(variableTree.getModifiers()),\n                            Optional.of(variableTree.getModifiers()),\n                            variableTree.getType(),\n                            /* name= */ variableTree.getName(),\n                            \"\",\n                            \"=\",\n                            Optional.ofNullable(variableTree.getInitializer()),\n                            /* trailing= */ Optional.empty(),\n                            /* receiverExpression= */ Optional.empty(),\n                            /* typeWithDims= */ Optional.empty());\n                } else {\n                    // TODO(cushon): think harder about what to do with `try (resource1; resource2) {}`\n                    scan(resource, null);\n                }\n                if (builder.peekToken().equals(Optional.of(\";\"))) {\n                    token(\";\");\n                    builder.space();\n                }\n                first = false;\n            }\n            if (builder.peekToken().equals(Optional.of(\";\"))) {\n                token(\";\");\n                builder.space();\n            }\n            token(\")\");\n            builder.close();\n            builder.space();\n        }\n        // An empty try-with-resources body can collapse to \"{}\" if there are no trailing catch or\n        // finally blocks.\n        boolean trailingClauses = !node.getCatches().isEmpty() || node.getFinallyBlock() != null;\n        visitBlock(\n                node.getBlock(),\n                CollapseEmptyOrNot.valueOf(!trailingClauses),\n                AllowLeadingBlankLine.YES,\n                AllowTrailingBlankLine.valueOf(trailingClauses));\n        for (int i = 0; i < node.getCatches().size(); i++) {\n            CatchTree catchClause = node.getCatches().get(i);\n            trailingClauses = i < node.getCatches().size() - 1 || node.getFinallyBlock() != null;\n            visitCatchClause(catchClause, AllowTrailingBlankLine.valueOf(trailingClauses));\n        }\n        if (node.getFinallyBlock() != null) {\n            builder.space();\n            token(\"finally\");\n            builder.space();\n            visitBlock(\n                    node.getFinallyBlock(),\n                    CollapseEmptyOrNot.NO,\n                    AllowLeadingBlankLine.YES,\n                    AllowTrailingBlankLine.NO);\n        }\n        builder.close();\n        return null;\n    }\n\n    public void visitClassDeclaration(ClassTree node) {\n        sync(node);\n        List<Op> breaks = visitModifiers(\n                node.getModifiers(), Direction.VERTICAL, /* declarationAnnotationBreak= */ Optional.empty());\n        List<? extends Tree> permitsTypes = getPermitsClause(node);\n        boolean hasSuperclassType = node.getExtendsClause() != null;\n        boolean hasSuperInterfaceTypes = !node.getImplementsClause().isEmpty();\n        boolean hasPermitsTypes = !permitsTypes.isEmpty();\n        builder.addAll(breaks);\n        token(node.getKind() == Tree.Kind.INTERFACE ? \"interface\" : \"class\");\n        builder.space();\n        visit(node.getSimpleName());\n        if (!node.getTypeParameters().isEmpty()) {\n            token(\"<\");\n        }\n        builder.open(plusFour);\n        {\n            if (!node.getTypeParameters().isEmpty()) {\n                typeParametersRest(\n                        node.getTypeParameters(),\n                        hasSuperclassType || hasSuperInterfaceTypes || hasPermitsTypes ? plusFour : ZERO);\n            }\n            if (hasSuperclassType) {\n                builder.breakToFill(\" \");\n                token(\"extends\");\n                builder.space();\n                scan(node.getExtendsClause(), null);\n            }\n            classDeclarationTypeList(\n                    node.getKind() == Tree.Kind.INTERFACE ? \"extends\" : \"implements\", node.getImplementsClause());\n            classDeclarationTypeList(\"permits\", permitsTypes);\n        }\n        builder.close();\n        if (node.getMembers() == null) {\n            token(\";\");\n        } else {\n            addBodyDeclarations(node.getMembers(), BracesOrNot.YES, FirstDeclarationsOrNot.YES);\n        }\n        dropEmptyDeclarations();\n    }\n\n    @Override\n    public Void visitTypeParameter(TypeParameterTree node, Void unused) {\n        sync(node);\n        builder.open(ZERO);\n        visitAnnotations(node.getAnnotations(), BreakOrNot.NO, BreakOrNot.YES);\n        visit(node.getName());\n        if (!node.getBounds().isEmpty()) {\n            builder.space();\n            token(\"extends\");\n            builder.open(plusFour);\n            builder.breakOp(\" \");\n            builder.open(plusFour);\n            boolean first = true;\n            for (Tree typeBound : node.getBounds()) {\n                if (!first) {\n                    builder.breakToFill(\" \");\n                    token(\"&\");\n                    builder.space();\n                }\n                scan(typeBound, null);\n                first = false;\n            }\n            builder.close();\n            builder.close();\n        }\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitUnionType(UnionTypeTree node, Void unused) {\n        throw new IllegalStateException(\"expected manual descent into union types\");\n    }\n\n    @Override\n    public Void visitWhileLoop(WhileLoopTree node, Void unused) {\n        sync(node);\n        token(\"while\");\n        builder.space();\n        token(\"(\");\n        scan(skipParen(node.getCondition()), null);\n        token(\")\");\n        visitStatement(\n                node.getStatement(), CollapseEmptyOrNot.YES, AllowLeadingBlankLine.YES, AllowTrailingBlankLine.NO);\n        return null;\n    }\n\n    @Override\n    public Void visitWildcard(WildcardTree node, Void unused) {\n        sync(node);\n        builder.open(ZERO);\n        token(\"?\");\n        if (node.getBound() != null) {\n            builder.open(plusFour);\n            builder.space();\n            token(node.getKind() == EXTENDS_WILDCARD ? \"extends\" : \"super\");\n            builder.breakOp(\" \");\n            scan(node.getBound(), null);\n            builder.close();\n        }\n        builder.close();\n        return null;\n    }\n\n    // Helper methods.\n\n    /** Helper method for annotations. */\n    void visitAnnotations(List<? extends AnnotationTree> annotations, BreakOrNot breakBefore, BreakOrNot breakAfter) {\n        if (!annotations.isEmpty()) {\n            if (breakBefore.isYes()) {\n                builder.breakToFill(\" \");\n            }\n            boolean first = true;\n            for (AnnotationTree annotation : annotations) {\n                if (!first) {\n                    builder.breakToFill(\" \");\n                }\n                scan(annotation, null);\n                first = false;\n            }\n            if (breakAfter.isYes()) {\n                builder.breakToFill(\" \");\n            }\n        }\n    }\n\n    /** Helper method for blocks. */\n    protected void visitBlock(\n            BlockTree node,\n            CollapseEmptyOrNot collapseEmptyOrNot,\n            AllowLeadingBlankLine allowLeadingBlankLine,\n            AllowTrailingBlankLine allowTrailingBlankLine) {\n        sync(node);\n        if (node.isStatic()) {\n            token(\"static\");\n            builder.space();\n        }\n        if (collapseEmptyOrNot.isYes() && node.getStatements().isEmpty()) {\n            if (builder.peekToken().equals(Optional.of(\";\"))) {\n                // TODO(cushon): is this needed?\n                token(\";\");\n            } else {\n                tokenBreakTrailingComment(\"{\", plusTwo);\n                builder.blankLineWanted(BlankLineWanted.NO);\n                token(\"}\", plusTwo);\n            }\n        } else {\n            builder.open(ZERO);\n            builder.open(plusTwo);\n            tokenBreakTrailingComment(\"{\", plusTwo);\n            if (allowLeadingBlankLine == AllowLeadingBlankLine.NO) {\n                builder.blankLineWanted(BlankLineWanted.NO);\n            } else {\n                builder.blankLineWanted(BlankLineWanted.PRESERVE);\n            }\n            visitStatements(node.getStatements(), false);\n            builder.close();\n            builder.forcedBreak();\n            builder.close();\n            if (allowTrailingBlankLine == AllowTrailingBlankLine.NO) {\n                builder.blankLineWanted(BlankLineWanted.NO);\n            } else {\n                builder.blankLineWanted(BlankLineWanted.PRESERVE);\n            }\n            markForPartialFormat();\n            token(\"}\", plusTwo);\n        }\n    }\n\n    /** Helper method for statements. */\n    private void visitStatement(\n            StatementTree node,\n            CollapseEmptyOrNot collapseEmptyOrNot,\n            AllowLeadingBlankLine allowLeadingBlank,\n            AllowTrailingBlankLine allowTrailingBlank) {\n        sync(node);\n        switch (node.getKind()) {\n            case BLOCK:\n                builder.space();\n                visitBlock((BlockTree) node, collapseEmptyOrNot, allowLeadingBlank, allowTrailingBlank);\n                break;\n            default:\n                builder.open(plusTwo);\n                builder.breakOp(\" \");\n                scan(node, null);\n                builder.close();\n        }\n    }\n\n    protected void visitStatements(List<? extends StatementTree> statements, boolean inlineFirst) {\n        boolean first = true;\n        PeekingIterator<StatementTree> it = Iterators.peekingIterator(statements.iterator());\n        dropEmptyDeclarations();\n        while (it.hasNext()) {\n            StatementTree tree = it.next();\n            if (!(inlineFirst && first)) {\n                builder.forcedBreak();\n            }\n            if (!first) {\n                builder.blankLineWanted(BlankLineWanted.PRESERVE);\n            }\n            markForPartialFormat();\n            first = false;\n            List<VariableTree> fragments = variableFragments(it, tree);\n            if (!fragments.isEmpty()) {\n                visitVariables(\n                        fragments,\n                        DeclarationKind.NONE,\n                        canLocalHaveHorizontalAnnotations(fragments.get(0).getModifiers()));\n            } else {\n                scan(tree, null);\n            }\n        }\n    }\n\n    /** Output combined modifiers and annotations and the trailing break. */\n    void visitAndBreakModifiers(\n            ModifiersTree modifiers, Direction annotationDirection, Optional<BreakTag> declarationAnnotationBreak) {\n        builder.addAll(visitModifiers(modifiers, annotationDirection, declarationAnnotationBreak));\n    }\n\n    @Override\n    public Void visitModifiers(ModifiersTree node, Void unused) {\n        throw new IllegalStateException(\"expected manual descent into modifiers\");\n    }\n\n    /** Output combined modifiers and annotations and returns the trailing break. */\n    protected List<Op> visitModifiers(\n            ModifiersTree modifiersTree,\n            Direction annotationsDirection,\n            Optional<BreakTag> declarationAnnotationBreak) {\n        return visitModifiers(modifiersTree.getAnnotations(), annotationsDirection, declarationAnnotationBreak);\n    }\n\n    private List<Op> visitModifiers(\n            List<? extends AnnotationTree> annotationTrees,\n            Direction annotationsDirection,\n            Optional<BreakTag> declarationAnnotationBreak) {\n        if (annotationTrees.isEmpty() && !nextIsModifier()) {\n            return EMPTY_LIST;\n        }\n        Deque<AnnotationTree> annotations = new ArrayDeque<>(annotationTrees);\n        builder.open(ZERO);\n        boolean first = true;\n        boolean lastWasAnnotation = false;\n        while (!annotations.isEmpty()) {\n            if (nextIsModifier()) {\n                break;\n            }\n            if (!first) {\n                builder.addAll(\n                        annotationsDirection.isVertical()\n                                ? forceBreakList(declarationAnnotationBreak)\n                                : breakList(declarationAnnotationBreak));\n            }\n            scan(annotations.removeFirst(), null);\n            first = false;\n            lastWasAnnotation = true;\n        }\n        builder.close();\n        ImmutableList<Op> trailingBreak = annotationsDirection.isVertical()\n                ? forceBreakList(declarationAnnotationBreak)\n                : breakList(declarationAnnotationBreak);\n        if (annotations.isEmpty() && !nextIsModifier()) {\n            return trailingBreak;\n        }\n        if (lastWasAnnotation) {\n            builder.addAll(trailingBreak);\n        }\n\n        builder.open(ZERO);\n        first = true;\n        while (nextIsModifier() || !annotations.isEmpty()) {\n            if (!first) {\n                builder.addAll(breakFillList(Optional.empty()));\n            }\n            if (nextIsModifier()) {\n                String currentToken = builder.peekToken().get();\n                token(currentToken);\n                if (currentToken.equals(\"non\")) {\n                    token(builder.peekToken().get());\n                    token(builder.peekToken().get());\n                }\n            } else {\n                scan(annotations.removeFirst(), null);\n                lastWasAnnotation = true;\n            }\n            first = false;\n        }\n        builder.close();\n        return breakFillList(Optional.empty());\n    }\n\n    boolean nextIsModifier() {\n        switch (builder.peekToken().get()) {\n            case \"public\":\n            case \"protected\":\n            case \"private\":\n            case \"abstract\":\n            case \"static\":\n            case \"final\":\n            case \"transient\":\n            case \"volatile\":\n            case \"synchronized\":\n            case \"native\":\n            case \"strictfp\":\n            case \"default\":\n            case \"sealed\":\n            case \"non\":\n            case \"-\":\n                return true;\n            default:\n                return false;\n        }\n    }\n\n    @Override\n    public Void visitCatch(CatchTree node, Void unused) {\n        throw new IllegalStateException(\"expected manual descent into catch trees\");\n    }\n\n    /** Helper method for {@link CatchTree}s. */\n    private void visitCatchClause(CatchTree node, AllowTrailingBlankLine allowTrailingBlankLine) {\n        sync(node);\n        builder.space();\n        token(\"catch\");\n        builder.space();\n        token(\"(\");\n        builder.open(plusFour);\n        VariableTree ex = node.getParameter();\n        if (ex.getType().getKind() == UNION_TYPE) {\n            builder.open(ZERO);\n            visitUnionType(ex);\n            builder.close();\n        } else {\n            // TODO(cushon): don't break after here for consistency with for, while, etc.\n            builder.breakToFill();\n            builder.open(ZERO);\n            scan(ex, null);\n            builder.close();\n        }\n        builder.close();\n        token(\")\");\n        builder.space();\n        visitBlock(node.getBlock(), CollapseEmptyOrNot.NO, AllowLeadingBlankLine.YES, allowTrailingBlankLine);\n    }\n\n    /** Formats a union type declaration in a catch clause. */\n    private void visitUnionType(VariableTree declaration) {\n        UnionTypeTree type = (UnionTypeTree) declaration.getType();\n        builder.open(ZERO);\n        sync(declaration);\n        visitAndBreakModifiers(\n                declaration.getModifiers(), Direction.HORIZONTAL, /* declarationAnnotationBreak= */ Optional.empty());\n        List<? extends Tree> union = type.getTypeAlternatives();\n        boolean first = true;\n        for (int i = 0; i < union.size() - 1; i++) {\n            if (!first) {\n                builder.breakOp(\" \");\n                token(\"|\");\n                builder.space();\n            } else {\n                first = false;\n            }\n            scan(union.get(i), null);\n        }\n        builder.breakOp(\" \");\n        token(\"|\");\n        builder.space();\n        Tree last = union.get(union.size() - 1);\n        declareOne(\n                DeclarationKind.NONE,\n                Direction.HORIZONTAL,\n                /* modifiers= */ Optional.empty(),\n                last,\n                /* name= */ declaration.getName(),\n                /* op= */ \"\",\n                \"=\",\n                Optional.ofNullable(declaration.getInitializer()),\n                /* trailing= */ Optional.empty(),\n                /* receiverExpression= */ Optional.empty(),\n                /* typeWithDims= */ Optional.empty());\n        builder.close();\n    }\n\n    /** Accumulate the operands and operators. */\n    private static void walkInfix(\n            int precedence, ExpressionTree expression, List<ExpressionTree> operands, List<String> operators) {\n        if (expression instanceof BinaryTree) {\n            BinaryTree binaryTree = (BinaryTree) expression;\n            if (precedence(binaryTree) == precedence) {\n                walkInfix(precedence, binaryTree.getLeftOperand(), operands, operators);\n                operators.add(operatorName(expression));\n                walkInfix(precedence, binaryTree.getRightOperand(), operands, operators);\n            } else {\n                operands.add(expression);\n            }\n        } else {\n            operands.add(expression);\n        }\n    }\n\n    protected void visitFormals(Optional<VariableTree> receiver, List<? extends VariableTree> parameters) {\n        if (!receiver.isPresent() && parameters.isEmpty()) {\n            return;\n        }\n        builder.open(ZERO);\n        boolean first = true;\n        if (receiver.isPresent()) {\n            // TODO(jdd): Use builders.\n            declareOne(\n                    DeclarationKind.PARAMETER,\n                    Direction.HORIZONTAL,\n                    Optional.of(receiver.get().getModifiers()),\n                    receiver.get().getType(),\n                    /* name= */ receiver.get().getName(),\n                    \"\",\n                    \"\",\n                    /* initializer= */ Optional.empty(),\n                    !parameters.isEmpty() ? Optional.of(\",\") : Optional.empty(),\n                    Optional.of(receiver.get().getNameExpression()),\n                    /* typeWithDims= */ Optional.empty());\n            first = false;\n        }\n        for (int i = 0; i < parameters.size(); i++) {\n            VariableTree parameter = parameters.get(i);\n            if (!first) {\n                builder.breakOp(\" \");\n            }\n            visitToDeclare(\n                    DeclarationKind.PARAMETER,\n                    Direction.HORIZONTAL,\n                    parameter,\n                    /* initializer= */ Optional.empty(),\n                    \"=\",\n                    i < parameters.size() - 1 ? Optional.of(\",\") : /* a= */ Optional.empty());\n            first = false;\n        }\n        builder.close();\n    }\n\n    //  /** Helper method for {@link MethodDeclaration}s. */\n    private void visitThrowsClause(List<? extends ExpressionTree> thrownExceptionTypes) {\n        token(\"throws\");\n        builder.breakToFill(\" \");\n        boolean first = true;\n        for (ExpressionTree thrownExceptionType : thrownExceptionTypes) {\n            if (!first) {\n                token(\",\");\n                builder.breakToFill(\" \");\n            }\n            scan(thrownExceptionType, null);\n            first = false;\n        }\n    }\n\n    @Override\n    public Void visitIdentifier(IdentifierTree node, Void unused) {\n        sync(node);\n        token(node.getName().toString());\n        return null;\n    }\n\n    @Override\n    public Void visitModule(ModuleTree node, Void unused) {\n        for (AnnotationTree annotation : node.getAnnotations()) {\n            scan(annotation, null);\n            builder.forcedBreak();\n        }\n        if (node.getModuleType() == ModuleTree.ModuleKind.OPEN) {\n            token(\"open\");\n            builder.space();\n        }\n        token(\"module\");\n        builder.space();\n        scan(node.getName(), null);\n        builder.space();\n        if (node.getDirectives().isEmpty()) {\n            tokenBreakTrailingComment(\"{\", plusTwo);\n            builder.blankLineWanted(BlankLineWanted.NO);\n            token(\"}\", plusTwo);\n        } else {\n            builder.open(plusTwo);\n            token(\"{\");\n            builder.forcedBreak();\n            Optional<Tree.Kind> previousDirective = Optional.empty();\n            for (DirectiveTree directiveTree : node.getDirectives()) {\n                markForPartialFormat();\n                builder.blankLineWanted(\n                        previousDirective\n                                        .map(k -> !k.equals(directiveTree.getKind()))\n                                        .orElse(false)\n                                ? BlankLineWanted.YES\n                                : BlankLineWanted.NO);\n                builder.forcedBreak();\n                scan(directiveTree, null);\n                previousDirective = Optional.of(directiveTree.getKind());\n            }\n            builder.close();\n            builder.forcedBreak();\n            token(\"}\");\n        }\n        return null;\n    }\n\n    private void visitDirective(\n            String name,\n            String separator,\n            ExpressionTree nameExpression,\n            @Nullable List<? extends ExpressionTree> items) {\n        token(name);\n        builder.space();\n        scan(nameExpression, null);\n        if (items != null) {\n            builder.open(plusFour);\n            builder.space();\n            token(separator);\n            builder.forcedBreak();\n            boolean first = true;\n            for (ExpressionTree item : items) {\n                if (!first) {\n                    token(\",\");\n                    builder.forcedBreak();\n                }\n                scan(item, null);\n...\n", "needle_token_start": 5666, "needle_token_end": 5739, "code_context_ntokens": 16292, "output": ["```java\nprivate static boolean isArrayValue(ExpressionTree argument) {\n    if (!(argument instanceof AssignmentTree)) {\n        return false;\n    }\n    ExpressionTree expression = ((AssignmentTree) argument).getExpression();\n    return expression instanceof NewArrayTree && ((NewArrayTree) expression).getType() == null;\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "tryBreakLastLevel", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/doc/Level.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function aims to attempt breaking the last non-empty level in the document structure if possible, based on specific conditions and layout constraints.\n2. **Input**: The function takes a helper for handling comments, a maximum width constraint for layout, the current state of the document layout, an exploration node for tracking layout decisions, and a flag indicating whether the inlining process has been simple so far.\n3. **Output**: It returns an optional state, which is present if the last level can be successfully broken according to the given constraints and conditions; otherwise, it returns empty.\n4. **Procedure**: \n   - First, the function checks if the document list is empty or if the last document is not a level, returning empty if either condition is true.\n   - It retrieves the last level from the document list.\n   - The function then attempts to break this last level using another function that handles the breaking logic, passing along all necessary parameters including the last level and the flag indicating the simplicity of the inlining process so far.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Op.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\n...\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/Doc.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport com.google.common.base.Suppliers;\nimport com.google.common.collect.DiscreteDomain;\nimport com.google.common.collect.Range;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.OpsBuilder;\nimport com.palantir.javaformat.Output;\n\n/**\n * {@link com.palantir.javaformat.java.JavaInputAstVisitor JavaInputAstVisitor} outputs a sequence of {@link Op}s using\n * {@link OpsBuilder}. This linear sequence is then transformed by {@link DocBuilder} into a tree-structured\n * {@code Doc}. The top-level {@code Doc} is a {@link Level}, which contains a sequence of {@code Doc}s, including other\n * {@link Level}s. Leaf {@code Doc}s are {@link Token}s, representing language-level tokens; {@link Comment}s, which may\n * also represent non-token {@link Input.Tok}s, including comments and other white-space; {@link NonBreakingSpace}s,\n * representing single spaces; and {@link Break}s, which represent optional line-breaks.\n */\npublic abstract class Doc extends HasUniqueId {\n    static final Range<Integer> EMPTY_RANGE = Range.closedOpen(-1, -1);\n    static final DiscreteDomain<Integer> INTEGERS = DiscreteDomain.integers();\n\n    @SuppressWarnings(\"Immutable\") // Effectively immutable\n    private final ImmutableSupplier<Float> memoizedWidth = Suppliers.memoize(this::computeWidth)::get;\n\n    @SuppressWarnings(\"Immutable\") // Effectively immutable\n    private final ImmutableSupplier<String> memoizedFlat = Suppliers.memoize(this::computeFlat)::get;\n\n    @SuppressWarnings(\"Immutable\") // Effectively immutable\n    private final ImmutableSupplier<Range<Integer>> memoizedRange = Suppliers.memoize(this::computeRange)::get;\n\n    /**\n     * Return the width of a {@code Doc}, or {@code Float.POSITIVE_INFINITY} if it must be broken.\n     *\n     * @return the width\n     */\n    final float getWidth() {\n        return memoizedWidth.get();\n    }\n\n    /**\n     * Return a {@code Doc}'s flat-string value; not defined (and never called) if the {@code Doc} contains forced\n     * breaks.\n     *\n     * @return the flat-string value\n     */\n    public final String getFlat() {\n        return memoizedFlat.get();\n    }\n\n    final Range<Integer> range() {\n        return memoizedRange.get();\n    }\n\n    /**\n     * Compute the {@code Doc}'s width.\n     *\n     * @return the width, or {@code Float.POSITIVE_INFINITY} if it must be broken\n     */\n    protected abstract float computeWidth();\n\n    /**\n     * Compute the {@code Doc}'s flat value. Not defined (and never called) if contains forced breaks.\n     *\n     * @return the flat value\n     */\n    protected abstract String computeFlat();\n\n    /**\n     * Compute the {@code Doc}'s {@link Range} of {@link Input.Token}s.\n     *\n     * @return the {@link Range}\n     */\n    protected abstract Range<Integer> computeRange();\n\n    /**\n     * Make breaking decisions for a {@code Doc}.\n     *\n     * @param maxWidth the maximum line width\n     * @param state the current output state\n     * @return the new output state\n     */\n    public abstract State computeBreaks(\n            CommentsHelper commentsHelper, int maxWidth, State state, Obs.ExplorationNode explorationNode);\n\n    /** Write a {@code Doc} to an {@link Output}, after breaking decisions have been made. */\n    public abstract void write(State state, Output output);\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/BreakBehaviour.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat;\n\nimport com.fasterxml.jackson.core.JsonGenerator;\nimport com.fasterxml.jackson.databind.JsonSerializer;\nimport com.fasterxml.jackson.databind.SerializerProvider;\nimport com.fasterxml.jackson.databind.annotation.JsonSerialize;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.Level;\nimport java.io.IOException;\nimport org.derive4j.ArgOption;\nimport org.derive4j.Data;\n\n@Data(arguments = ArgOption.checkedNotNull)\n@Immutable\n@JsonSerialize(using = BreakBehaviour.Json.class)\npublic abstract class BreakBehaviour {\n    public interface Cases<R> {\n\n        R breakThisLevel();\n\n        /**\n         * If the last level is breakable, prefer breaking it if it will keep the rest of this level on line line.\n         *\n         * @param keepIndentWhenInlined whether to keep this level's indent when inlined as a recursive level (when\n         *     reached via a previous `preferBreakingLastInnerLevel` whose breakability was\n         *     {@link LastLevelBreakability#CHECK_INNER})\n         */\n        R preferBreakingLastInnerLevel(boolean keepIndentWhenInlined);\n\n        /**\n         * Attempt to inline the suffix of this level (which must be a {@link Level} and the last doc), recursing into\n         * the {@link Level} just before the last {@link Level} (if there is such a level) to see if that can be broken\n         * instead.\n         *\n         * <p>This behaves like {@link #breakThisLevel()} if we couldn't recurse into such an inner level, or if the\n         * suffix level doesn't fit on the last line.\n         */\n        R inlineSuffix();\n\n        /**\n         * Break if by doing so all inner levels then fit on a single line. However, don't break if we can fit in the\n         * {@link Doc docs} up to the first break (which might be nested inside the next doc if it's a {@link Level}),\n         * in order to prevent exceeding the maxLength accidentally.\n         */\n        R breakOnlyIfInnerLevelsThenFitOnOneLine(boolean keepIndentWhenInlined);\n    }\n\n    public abstract <R> R match(Cases<R> cases);\n\n    /** For {@link com.palantir.javaformat.doc.LevelDelimitedFlatValueDocVisitor}. */\n    @Override\n    public abstract String toString();\n\n    /**\n     * This is gross but just wanted to get something working. See https://github.com/derive4j/derive4j/issues/51 for a\n     * potential better implementation.\n     */\n    static class Json extends JsonSerializer<BreakBehaviour> {\n\n        @Override\n        public void serialize(BreakBehaviour value, JsonGenerator gen, SerializerProvider serializers)\n                throws IOException {\n            gen.writeStartObject();\n            BreakBehaviours.caseOf(value)\n                    .breakThisLevel(() -> {\n                        try {\n                            gen.writeObjectField(\"type\", \"breakThisLevel\");\n                        } catch (IOException e) {\n                            throw new RuntimeException(e);\n                        }\n                        return null;\n                    })\n                    .preferBreakingLastInnerLevel(keepIndentWhenInlined -> {\n                        try {\n                            gen.writeObjectField(\"type\", \"preferBreakingLastInnerLevel\");\n                            gen.writeObjectField(\"keepIndentWhenInlined\", keepIndentWhenInlined);\n                        } catch (IOException e) {\n                            throw new RuntimeException(e);\n                        }\n                        return null;\n                    })\n                    .inlineSuffix(() -> {\n                        try {\n                            gen.writeObjectField(\"type\", \"inlineSuffix\");\n                        } catch (IOException e) {\n                            throw new RuntimeException(e);\n                        }\n                        return null;\n                    })\n                    .breakOnlyIfInnerLevelsThenFitOnOneLine(keepIndentWhenInlined -> {\n                        try {\n                            gen.writeObjectField(\"type\", \"breakOnlyIfInnerLevelsThenFitOnOneLine\");\n                            gen.writeObjectField(\"keepIndentWhenInlined\", keepIndentWhenInlined);\n                        } catch (IOException e) {\n                            throw new RuntimeException(e);\n                        }\n                        return null;\n                    });\n            gen.writeEndObject();\n        }\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/StartsWithBreakVisitor.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport com.palantir.javaformat.doc.StartsWithBreakVisitor.Result;\n\nenum StartsWithBreakVisitor implements DocVisitor<Result> {\n    INSTANCE;\n\n    enum Result {\n        EMPTY,\n        NO,\n        YES,\n    }\n\n    @Override\n    public Result visitSpace(NonBreakingSpace doc) {\n        return Result.NO;\n    }\n\n    @Override\n    public Result visitComment(Comment doc) {\n        return Result.NO;\n    }\n\n    @Override\n    public Result visitToken(Token doc) {\n        return Result.NO;\n    }\n\n    @Override\n    public Result visitBreak(Break doc) {\n        return Result.YES;\n    }\n\n    @Override\n    public Result visitLevel(Level doc) {\n        return doc.getDocs().stream()\n                .map(this::visit)\n                .filter(result -> result != Result.EMPTY)\n                .findFirst()\n                .orElse(Result.EMPTY);\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/Level.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport static com.google.common.collect.Iterables.getLast;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Preconditions;\nimport com.google.common.base.Suppliers;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Range;\nimport com.palantir.javaformat.BreakBehaviour;\nimport com.palantir.javaformat.BreakBehaviours;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Indent;\nimport com.palantir.javaformat.LastLevelBreakability;\nimport com.palantir.javaformat.OpenOp;\nimport com.palantir.javaformat.OpenOp.Complexity;\nimport com.palantir.javaformat.Output;\nimport com.palantir.javaformat.PartialInlineability;\nimport com.palantir.javaformat.doc.Obs.Exploration;\nimport com.palantir.javaformat.doc.Obs.ExplorationNode;\nimport com.palantir.javaformat.doc.Obs.LevelNode;\nimport com.palantir.javaformat.doc.StartsWithBreakVisitor.Result;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.OptionalInt;\nimport java.util.stream.Collector;\nimport java.util.stream.Collectors;\nimport java.util.stream.Stream;\nimport org.immutables.value.Value;\n\n/** A {@code Level} inside a {@link Doc}. */\npublic final class Level extends Doc {\n    /**\n     * How many branches we are allowed to take (i.e how many times we can consider breaking vs not breaking the current\n     * level) before we stop branching and always break, which is the google-java-format default behaviour.\n     */\n    private static final int MAX_BRANCHING_COEFFICIENT = 20;\n\n    private static final Collector<Level, ?, Optional<Level>> GET_LAST_COLLECTOR = Collectors.reducing((u, v) -> v);\n\n    private final List<Doc> docs = new ArrayList<>(); // The elements of the level.\n\n    @SuppressWarnings(\"Immutable\") // Effectively immutable\n    private final ImmutableSupplier<SplitsBreaks> memoizedSplitsBreaks =\n            Suppliers.memoize(() -> splitByBreaks(docs))::get;\n    /** The immutable characteristics of this level determined before the level contents are available. */\n    private final OpenOp openOp;\n\n    private Level(OpenOp openOp) {\n        this.openOp = openOp;\n    }\n\n    /** Factory method for {@code Level}s. */\n    static Level make(OpenOp openOp) {\n        return new Level(openOp);\n    }\n\n    /**\n     * Add a {@link Doc} to the {@code Level}.\n     *\n     * @param doc the {@link Doc} to add\n     */\n    void add(Doc doc) {\n        docs.add(doc);\n    }\n\n    @Override\n    protected float computeWidth() {\n        float thisWidth = 0.0F;\n        for (Doc doc : docs) {\n            thisWidth += doc.getWidth();\n        }\n        return thisWidth;\n    }\n\n    @Override\n    protected String computeFlat() {\n        StringBuilder builder = new StringBuilder();\n        for (Doc doc : docs) {\n            builder.append(doc.getFlat());\n        }\n        return builder.toString();\n    }\n\n    @Override\n    protected Range<Integer> computeRange() {\n        Range<Integer> docRange = EMPTY_RANGE;\n        for (Doc doc : docs) {\n            docRange = union(docRange, doc.range());\n        }\n        return docRange;\n    }\n\n    @Override\n    public State computeBreaks(CommentsHelper commentsHelper, int maxWidth, State state, Obs.ExplorationNode observer) {\n        return tryToFitOnOneLine(maxWidth, state, docs)\n                .map(newWidth -> state.withColumn(newWidth).withLevelState(this, ImmutableLevelState.of(true)))\n                .orElseGet(() -> {\n                    Obs.LevelNode childLevel = observer.newChildNode(this, state);\n                    State newState =\n                            getBreakBehaviour().match(new BreakImpl(commentsHelper, maxWidth, state, childLevel));\n\n                    return childLevel.finishLevel(state.updateAfterLevel(newState));\n                });\n    }\n\n    /**\n     * Try to fit these docs belonging to the current level onto one line, returning empty if we couldn't. This takes\n     * into account the level's {@link #getColumnLimitBeforeLastBreak()}.\n     *\n     * @return the width after fitting it onto one line, if it was possible. This is guaranteed to be less than\n     *     {@code maxWidth}\n     */\n    private Optional<Integer> tryToFitOnOneLine(int maxWidth, State state, Iterable<Doc> docs) {\n        int column = state.column();\n        int columnBeforeLastBreak = 0; // Not activated by default\n        for (Doc doc : docs) {\n            if (doc instanceof Break && ((Break) doc).hasColumnLimit()) {\n                columnBeforeLastBreak = column;\n            } else if (doc instanceof Level) {\n                // Levels might have nested levels that have a 'columnLimitBeforeLastBreak' set, so recurse.\n                State newState = state.withColumn(column);\n                Level innerLevel = (Level) doc;\n                Optional<Integer> newWidth = innerLevel.tryToFitOnOneLine(maxWidth, newState, innerLevel.getDocs());\n                if (!newWidth.isPresent()) {\n                    return Optional.empty();\n                }\n                column = newWidth.get();\n                continue;\n            }\n            column = (int) (column + doc.getWidth());\n        }\n        // Make an additional check that widthBeforeLastBreak fits in the column limit\n        if (getColumnLimitBeforeLastBreak().isPresent()\n                && columnBeforeLastBreak > getColumnLimitBeforeLastBreak().getAsInt()) {\n            return Optional.empty();\n        }\n\n        // Check that the entirety of this level fits on the current line.\n        if (column <= maxWidth) {\n            return Optional.of(column);\n        }\n        return Optional.empty();\n    }\n\n    class BreakImpl implements BreakBehaviour.Cases<State> {\n        private final CommentsHelper commentsHelper;\n        private final int maxWidth;\n        private final State state;\n        private final Obs.LevelNode levelNode;\n\n        public BreakImpl(CommentsHelper commentsHelper, int maxWidth, State state, Obs.LevelNode levelNode) {\n            this.commentsHelper = commentsHelper;\n            this.maxWidth = maxWidth;\n            this.state = state;\n            this.levelNode = levelNode;\n        }\n\n        private Exploration breakNormally(State state) {\n            return Level.this.breakNormally(state, levelNode, commentsHelper, maxWidth);\n        }\n\n        @Override\n        public State breakThisLevel() {\n            return breakNormally(state).markAccepted();\n        }\n\n        @Override\n        public State preferBreakingLastInnerLevel(boolean _keepIndentWhenInlined) {\n            // Try both breaking and not breaking. Choose the better one based on LOC, preferring\n            // breaks if the outcome is the same.\n            State state = this.state.withNewBranch();\n\n            Obs.Exploration broken = breakNormally(state);\n\n            if (state.branchingCoefficient() < MAX_BRANCHING_COEFFICIENT) {\n                State state1 = state.withNoIndent();\n                Optional<Obs.Exploration> lastLevelBroken = levelNode.maybeExplore(\n                        \"tryBreakLastLevel\",\n                        state1,\n                        explorationNode -> tryBreakLastLevel(commentsHelper, maxWidth, state1, explorationNode, true));\n\n                if (lastLevelBroken.isPresent()) {\n                    if (lastLevelBroken.get().state().numLines()\n                            < broken.state().numLines()) {\n                        return lastLevelBroken.get().markAccepted();\n                    }\n                }\n            }\n            return broken.markAccepted();\n        }\n\n        @Override\n        public State breakOnlyIfInnerLevelsThenFitOnOneLine(boolean keepIndentWhenInlined) {\n            Exploration broken = Level.this.breakNormally(state, levelNode, commentsHelper, maxWidth);\n\n            Optional<Exploration> maybeInlined = levelNode.maybeExplore(\n                    \"handle_breakOnlyIfInnerLevelsThenFitOnOneLine\",\n                    state,\n                    explorationNode -> handle_breakOnlyIfInnerLevelsThenFitOnOneLine(\n                            commentsHelper, maxWidth, state, broken.state(), keepIndentWhenInlined, explorationNode));\n\n            if (maybeInlined.isPresent()) {\n                return maybeInlined.get().markAccepted();\n            } else {\n                return broken.markAccepted();\n            }\n        }\n\n        @Override\n        public State inlineSuffix() {\n            Optional<Obs.Exploration> lastLevelBroken = levelNode.maybeExplore(\n                    \"inlineSuffix\",\n                    state,\n                    explorationNode -> tryInlineSuffix(commentsHelper, maxWidth, state, explorationNode, true));\n            return lastLevelBroken.orElseGet(() -> this.breakNormally(state)).markAccepted();\n        }\n    }\n\n    private Exploration breakNormally(State state, LevelNode levelNode, CommentsHelper commentsHelper, int maxWidth) {\n        State stateForBroken = state.withIndentIncrementedBy(getPlusIndent());\n        return levelNode.explore(\n                \"breaking normally\",\n                stateForBroken,\n                explorationNode -> computeBroken(commentsHelper, maxWidth, stateForBroken, explorationNode));\n    }\n\n    /**\n     * Attempts to perform the {@link BreakBehaviour.Cases#breakOnlyIfInnerLevelsThenFitOnOneLine} logic, returning\n     * empty if it couldn't. Namely, this will only return a state if:\n     * <ul>\n     *     <li>everything fit on the current or on the next line (we verify this by checking that none of the inner\n     *         levels were broken; or failing that\n     *     <li>a sensible prefix of this level fits onto the current line. See {@link CountWidthUntilBreakVisitor} for\n     *         how we determine the full prefix that <em>must</em> fit on the current line.\n     * </ul>\n     *\n     * @param brokenState is expected to be the state resulting from {@link Level#breakNormally}, but is passed in\n     *                    for efficiency reasons\n     */\n    private Optional<State> handle_breakOnlyIfInnerLevelsThenFitOnOneLine(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            State state,\n            State brokenState,\n            boolean keepIndent,\n            Obs.ExplorationNode explorationNode) {\n\n        // Note: we are not checking if the brokenState produced one extra line compared to state, as this can be\n        // misleading if there is no level but a single comment that got reflowed onto multiple lines (see palantir-11).\n        boolean anyLevelWasBroken = getNonEmptyInnerLevels().anyMatch(level -> !brokenState.isOneLine(level));\n\n        if (!anyLevelWasBroken) {\n            return Optional.of(brokenState);\n        }\n\n        Optional<State> partiallyInlinedStateOpt =\n                tryInlinePrefixOntoCurrentLine(commentsHelper, maxWidth, state, keepIndent, explorationNode);\n        if (!partiallyInlinedStateOpt.isPresent()) {\n            return Optional.empty();\n        }\n        State partiallyInlinedState = partiallyInlinedStateOpt.get();\n\n        boolean bodyIsComplex = getNonEmptyInnerLevels().anyMatch(il -> il.openOp.complexity() == Complexity.COMPLEX);\n\n        if (bodyIsComplex || partiallyInlinedState.numLines() < brokenState.numLines()) {\n            return Optional.of(partiallyInlinedState);\n        }\n        return Optional.empty();\n    }\n\n    private Stream<Level> getNonEmptyInnerLevels() {\n        return docs.stream()\n                .filter(doc -> doc instanceof Level)\n                .map(doc -> ((Level) doc))\n                .filter(doc -> StartsWithBreakVisitor.INSTANCE.visit(doc) != Result.EMPTY);\n    }\n\n    private Optional<State> tryInlinePrefixOntoCurrentLine(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            State state,\n            boolean keepIndent,\n            Obs.ExplorationNode explorationNode) {\n        // Find the last level, skipping empty levels (that contain nothing, or are made up\n        // entirely of other empty levels).\n        // Last level because there might be other in-between levels after the initial break like `new\n        // int[] {`, and we want to skip those.\n        Level lastLevel = getNonEmptyInnerLevels()\n                .filter(doc -> StartsWithBreakVisitor.INSTANCE.visit(doc) != Result.EMPTY)\n                .collect(GET_LAST_COLLECTOR)\n                .orElseThrow(() ->\n                        new IllegalStateException(\"Levels were broken so expected to find at least a non-empty level\"));\n\n        // Add the width of tokens, breaks before the lastLevel. We must always have space for\n        // these.\n        List<Doc> leadingDocs = docs.subList(0, docs.indexOf(lastLevel));\n        float leadingWidth = getWidth(leadingDocs);\n\n        // Potentially add the width of prefixes we want to consider as part of the width that\n        // must fit on the same line, so that we don't accidentally break prefixes when we could\n        // have avoided doing so.\n        leadingWidth += new CountWidthUntilBreakVisitor(maxWidth - state.indent()).visit(lastLevel);\n\n        boolean fits = !Float.isInfinite(leadingWidth) && state.column() + leadingWidth <= maxWidth;\n\n        if (!fits) {\n            return Optional.empty();\n        }\n\n        State newState = state.withNoIndent();\n        if (keepIndent) {\n            newState = newState.withIndentIncrementedBy(getPlusIndent());\n        }\n        return Optional.of(tryToLayOutLevelOnOneLine(\n                commentsHelper, maxWidth, newState, memoizedSplitsBreaks.get(), explorationNode));\n    }\n\n    \nprivate Optional<State> tryBreakLastLevel(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            State state,\n            ExplorationNode explorationNode,\n            boolean isSimpleInliningSoFar) {\n        if (docs.isEmpty() || !(getLast(docs) instanceof Level)) {\n            return Optional.empty();\n        }\n        Level innerLevel = ((Level) getLast(docs));\n\n        return tryBreakInnerLevel(commentsHelper, maxWidth, state, explorationNode, innerLevel, isSimpleInliningSoFar);\n    }\n\n    private Optional<State> tryInlineSuffix(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            State state,\n            ExplorationNode explorationNode,\n            boolean isSimpleInliningSoFar) {\n        Preconditions.checkState(\n                docs.size() > 0 && getLast(docs) instanceof Level,\n                \"Level with break behaviour inlineSuffix must end with a level as its last doc\");\n        Level lastLevel = (Level) getLast(docs);\n        Preconditions.checkState(\n                StartsWithBreakVisitor.INSTANCE.visit(lastLevel) == Result.YES,\n                \"Level with break behaviour inlineSuffix must have a last level that starts with a break\");\n\n        if (docs.size() < 2 || !(docs.get(docs.size() - 2) instanceof Level)) {\n            return Optional.empty();\n        }\n        Level lastLevelBeforeSuffix = (Level) docs.get(docs.size() - 2);\n\n        return tryBreakInnerLevel(\n                commentsHelper, maxWidth, state, explorationNode, lastLevelBeforeSuffix, isSimpleInliningSoFar);\n    }\n\n    private Optional<State> tryBreakInnerLevel(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            State state,\n            ExplorationNode explorationNode,\n            Level innerLevel,\n            boolean isSimpleInliningSoFar) {\n\n        // Only split levels that have declared they want to be split in this way.\n        if (innerLevel.getBreakabilityIfLastLevel() == LastLevelBreakability.ABORT) {\n            return Optional.empty();\n        }\n        // See if we can fill in everything but the innerLevel.\n        // This is essentially like a small part of computeBreaks.\n        int innerLevelIndex = docs.indexOf(innerLevel);\n        List<Doc> leadingDocs = docs.subList(0, innerLevelIndex);\n        if (!tryToFitOnOneLine(maxWidth, state, leadingDocs).isPresent()) {\n            return Optional.empty();\n        }\n\n        List<Doc> trailingDocs = docs.subList(innerLevelIndex + 1, docs.size());\n        float trailingWidth = getWidth(trailingDocs);\n        if (Double.isInfinite(trailingWidth)) {\n            return Optional.empty();\n        }\n\n        // Note: we can't use computeBroken with 'leadingDocs' instead of 'docs', because it assumes\n        // _we_ are breaking.\n        //       See computeBreakAndSplit -> shouldBreak\n\n        SplitsBreaks prefixSplitsBreaks = splitByBreaks(leadingDocs);\n\n        boolean isSimpleInlining = isSimpleInliningSoFar && Level.this.openOp.complexity() == Complexity.SIMPLE;\n\n        State state1 = tryToLayOutLevelOnOneLine(commentsHelper, maxWidth, state, prefixSplitsBreaks, explorationNode);\n        // If a break was still forced somehow even though we could fit the leadingWidth, then abort.\n        // This could happen if inner levels have set a `columnLimitBeforeLastBreak` or something like that.\n        if (state1.numLines() != state.numLines()) {\n            return Optional.empty();\n        }\n\n        Optional<State> state2;\n        switch (innerLevel.getBreakabilityIfLastLevel()) {\n            case ACCEPT_INLINE_CHAIN_IF_SIMPLE_OTHERWISE_CHECK_INNER:\n                if (isSimpleInlining) {\n                    state2 = tryBreakInnerLevel_acceptInlineChain(\n                            commentsHelper, maxWidth, explorationNode, innerLevel, state1);\n                    break;\n                }\n                // Otherwise, fall through to CHECK_INNER.\n            case CHECK_INNER:\n                state2 = tryBreakInnerLevel_checkInner(\n                        commentsHelper, maxWidth, explorationNode, innerLevel, isSimpleInlining, state1);\n                break;\n            case ACCEPT_INLINE_CHAIN:\n                state2 = tryBreakInnerLevel_acceptInlineChain(\n                        commentsHelper, maxWidth, explorationNode, innerLevel, state1);\n                break;\n            case ABORT:\n                state2 = Optional.empty();\n                break;\n            default:\n                throw new RuntimeException(\n                        \"Unexpected lastLevelBreakability: \" + innerLevel.getBreakabilityIfLastLevel());\n        }\n        return state2.flatMap(stateAfterInner -> {\n            // Do we have a suffix to inline too?\n            return tryToFitOnOneLine(maxWidth, stateAfterInner, trailingDocs).map(stateAfterInner::withColumn);\n        });\n    }\n\n    private static Optional<State> tryBreakInnerLevel_acceptInlineChain(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            ExplorationNode explorationNode,\n            Level lastLevel,\n            State state) {\n        // Ok then, we are allowed to break here, but first verify that we have enough room to inline this last\n        // level's prefix.\n        float extraWidth = new CountWidthUntilBreakVisitor(maxWidth - state.indent()).visit(lastLevel);\n        boolean stillFits = !Float.isInfinite(extraWidth) && state.column() + extraWidth <= maxWidth;\n        if (!stillFits) {\n            return Optional.empty();\n        }\n\n        // Note: computeBreaks, not computeBroken, so it can try to do this logic recursively for the\n        // lastLevel\n        return Optional.of(explorationNode\n                .newChildNode(lastLevel, state)\n                .explore(\n                        \"end tryBreakLastLevel chain\",\n                        state,\n                        exp -> lastLevel.computeBreaks(commentsHelper, maxWidth, state, exp))\n                .markAccepted());\n    }\n\n    private static Optional<State> tryBreakInnerLevel_checkInner(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            ExplorationNode explorationNode,\n            Level innerLevel,\n            boolean isSimpleInlining,\n            State state) {\n        // Try to fit the entire inner prefix if it's that kind of level.\n        return BreakBehaviours.caseOf(innerLevel.getBreakBehaviour())\n                .preferBreakingLastInnerLevel(keepIndentWhenInlined -> {\n                    State state1 =\n                            keepIndentWhenInlined ? state.withIndentIncrementedBy(innerLevel.getPlusIndent()) : state;\n\n                    return explorationNode\n                            .newChildNode(innerLevel, state1)\n                            .maybeExplore(\n                                    \"recurse into inner tryBreakLastLevel\",\n                                    state1,\n                                    exp -> innerLevel.tryBreakLastLevel(\n                                            commentsHelper, maxWidth, state1, exp, isSimpleInlining))\n                            .map(Exploration::markAccepted);\n                })\n                .inlineSuffix(() -> explorationNode\n                        .newChildNode(innerLevel, state)\n                        .maybeExplore(\n                                \"recurse into inner tryInlineSuffix\",\n                                state,\n                                exp -> innerLevel.tryInlineSuffix(\n                                        commentsHelper, maxWidth, state, exp, isSimpleInlining))\n                        .map(Exploration::markAccepted))\n                .breakOnlyIfInnerLevelsThenFitOnOneLine(keepIndentWhenInlined -> {\n                    // This case currently only matches lambda _expressions_ (without curlies)\n                    State state1 =\n                            keepIndentWhenInlined ? state.withIndentIncrementedBy(innerLevel.getPlusIndent()) : state;\n\n                    String humanDescription = \"end tryBreakLastLevel chain -> breakOnlyIfInnerLevelsThenFitOnOneLine\";\n                    LevelNode levelNode = explorationNode.newChildNode(innerLevel, state1);\n                    return levelNode\n                            .maybeExplore(humanDescription, state1, exp -> {\n                                // Not all levels would look good if inlined in this position, so we accept\n                                // levels that are meant to look good even if partially inlined, e.g. method\n                                // chains, which will catch things like builders, but not other kinds of levels like\n                                // constant expressions.\n                                // See the palantir-expression-lambdas.input test for an example of what this is\n                                // trying to avoid.\n\n                                // For this, need to actually check the last inner level of `lastLevel` (2 levels down).\n                                if (innerLevel.docs.isEmpty() || !(getLast(innerLevel.docs) instanceof Level)) {\n                                    return Optional.empty();\n                                }\n                                Level lastLevel2 = ((Level) getLast(innerLevel.docs));\n                                switch (lastLevel2.getBreakabilityIfLastLevel()) {\n                                    case ABORT:\n                                    case CHECK_INNER:\n                                        return Optional.empty();\n                                    case ACCEPT_INLINE_CHAIN:\n                                        Exploration broken =\n                                                innerLevel.breakNormally(state, levelNode, commentsHelper, maxWidth);\n                                        return innerLevel.handle_breakOnlyIfInnerLevelsThenFitOnOneLine(\n                                                commentsHelper,\n                                                maxWidth,\n                                                state1,\n                                                broken.state(),\n                                                keepIndentWhenInlined,\n                                                explorationNode);\n                                    case ACCEPT_INLINE_CHAIN_IF_SIMPLE_OTHERWISE_CHECK_INNER:\n                                        // We will allow inlining this kind of level but only if the level itself is\n                                        // not simple.\n                                        if (lastLevel2.openOp.complexity() == Complexity.SIMPLE) {\n                                            return Optional.empty();\n                                        }\n                                        return innerLevel.tryInlinePrefixOntoCurrentLine(\n                                                commentsHelper,\n                                                maxWidth,\n                                                state1,\n                                                keepIndentWhenInlined,\n                                                explorationNode);\n                                    default:\n                                        throw new RuntimeException(\"Unknown breakabilityIfLastLevel: \" + lastLevel2);\n                                }\n                            })\n                            .map(Exploration::markAccepted);\n                })\n                // We don't know how to fit the inner level on the same line, so bail out.\n                .otherwise_(Optional.empty());\n    }\n\n    /**\n     * Mark breaks in this level as not broken, but lay out the inner levels normally, according to their own\n     * {@link BreakBehaviour}. The resulting {@link State#mustBreak} will be true if this level did not fit on exactly\n     * one line.\n     */\n    private State tryToLayOutLevelOnOneLine(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            State state,\n            SplitsBreaks splitsBreaks,\n            Obs.ExplorationNode explorationNode) {\n\n        for (int i = 0; i < splitsBreaks.splits().size(); ++i) {\n            if (i > 0) {\n                state = splitsBreaks.breaks().get(i - 1).computeBreaks(state, false);\n            }\n\n            List<Doc> split = splitsBreaks.splits().get(i);\n            float splitWidth = getWidth(split);\n            boolean enoughRoom = state.column() + splitWidth <= maxWidth;\n            state = computeSplit(commentsHelper, maxWidth, split, state.withMustBreak(false), explorationNode);\n            if (!enoughRoom) {\n                state = state.withMustBreak(true);\n            }\n        }\n        return state;\n    }\n\n    private static SplitsBreaks splitByBreaks(List<Doc> docs) {\n        ImmutableSplitsBreaks.Builder builder = ImmutableSplitsBreaks.builder();\n        ImmutableList.Builder<Doc> currentSplit = ImmutableList.builder();\n        for (Doc doc : docs) {\n            if (doc instanceof Break) {\n                builder.addSplits(currentSplit.build());\n                currentSplit = ImmutableList.builder();\n                builder.addBreaks((Break) doc);\n            } else {\n                currentSplit.add(doc);\n            }\n        }\n        builder.addSplits(currentSplit.build());\n        return builder.build();\n    }\n\n    /** Compute breaks for a {@link Level} that spans multiple lines. */\n    private State computeBroken(\n            CommentsHelper commentsHelper, int maxWidth, State state, Obs.ExplorationNode explorationNode) {\n        SplitsBreaks splitsBreaks = memoizedSplitsBreaks.get();\n\n        if (!splitsBreaks.breaks().isEmpty()) {\n            state = state.withBrokenLevel();\n        }\n\n        ImmutableList<Doc> splitDocs = splitsBreaks.splits().get(0);\n        state = computeBreakAndSplit(\n                commentsHelper, maxWidth, state, /* optBreakDoc= */ Optional.empty(), splitDocs, explorationNode);\n\n        // Handle following breaks and split.\n        for (int i = 0; i < splitsBreaks.breaks().size(); i++) {\n            state = computeBreakAndSplit(\n                    commentsHelper,\n                    maxWidth,\n                    state,\n                    Optional.of(splitsBreaks.breaks().get(i)),\n                    splitsBreaks.splits().get(i + 1),\n                    explorationNode);\n        }\n        return state;\n    }\n\n    /** Lay out a Break-separated group of Docs in the current Level. */\n    private State computeBreakAndSplit(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            State state,\n            Optional<Break> optBreakDoc,\n            List<Doc> split,\n            Obs.ExplorationNode explorationNode) {\n        float breakWidth = optBreakDoc.isPresent() ? optBreakDoc.get().getWidth() : 0.0F;\n        float splitWidth = getWidth(split);\n\n        boolean shouldBreak = (optBreakDoc.isPresent() && optBreakDoc.get().fillMode() == FillMode.UNIFIED)\n                || state.mustBreak()\n                || Double.isInfinite(breakWidth)\n                || !tryToFitOnOneLine(maxWidth, state.withColumn(state.column() + (int) breakWidth), split)\n                        .isPresent();\n\n        if (optBreakDoc.isPresent()) {\n            state = optBreakDoc.get().computeBreaks(state, shouldBreak);\n        }\n        boolean enoughRoom = state.column() + splitWidth <= maxWidth;\n        state = computeSplit(commentsHelper, maxWidth, split, state.withMustBreak(false), explorationNode);\n        if (!enoughRoom) {\n            state = state.withMustBreak(true); // Break after, too.\n        }\n        return state;\n    }\n\n    private static State computeSplit(\n            CommentsHelper commentsHelper,\n            int maxWidth,\n            List<Doc> docs,\n            State state,\n            Obs.ExplorationNode explorationNode) {\n        for (Doc doc : docs) {\n            state = doc.computeBreaks(commentsHelper, maxWidth, state, explorationNode);\n        }\n        return state;\n    }\n\n    @Override\n    public void write(State state, Output output) {\n        if (state.isOneLine(this)) {\n            output.append(state, getFlat(), range()); // This is defined because width is finite.\n        } else {\n            writeFilled(state, output);\n        }\n    }\n\n    private void writeFilled(State state, Output output) {\n        SplitsBreaks splitsBreaks = memoizedSplitsBreaks.get();\n        // Handle first split.\n        for (Doc doc : splitsBreaks.splits().get(0)) {\n            doc.write(state, output);\n        }\n        // Handle following breaks and split.\n        for (int i = 0; i < splitsBreaks.breaks().size(); i++) {\n            splitsBreaks.breaks().get(i).write(state, output);\n            for (Doc doc : splitsBreaks.splits().get(i + 1)) {\n                doc.write(state, output);\n            }\n        }\n    }\n\n    public Indent getPlusIndent() {\n        return openOp.plusIndent();\n    }\n\n    BreakBehaviour getBreakBehaviour() {\n        return openOp.breakBehaviour();\n    }\n\n    List<Doc> getDocs() {\n        return docs;\n    }\n\n    LastLevelBreakability getBreakabilityIfLastLevel() {\n        return openOp.breakabilityIfLastLevel();\n    }\n\n    public PartialInlineability partialInlineability() {\n        return openOp.partialInlineability();\n    }\n\n    public Optional<String> getDebugName() {\n        return openOp.debugName();\n    }\n\n    public OpenOp getOpenOp() {\n        return openOp;\n    }\n\n    /**\n     * An optional, more restrictive column limit for inner breaks that are marked as {@link Break#hasColumnLimit()}. If\n     * the level is to be considered one-lineable, the last such break must not start at a column higher than this.\n     */\n    public OptionalInt getColumnLimitBeforeLastBreak() {\n        return openOp.columnLimitBeforeLastBreak();\n    }\n\n    /** An indented representation of this level and all nested levels inside it. */\n    public String representation(State state) {\n        return new LevelDelimitedFlatValueDocVisitor(state).visit(this);\n    }\n\n    /**\n     * Get the width of a sequence of {@link Doc}s.\n     *\n     * @param docs the {@link Doc}s\n     * @return the width, or {@code Float.POSITIVE_INFINITY} if any {@link Doc} must be broken\n     */\n    static float getWidth(Iterable<Doc> docs) {\n        float width = 0.0F;\n        for (Doc doc : docs) {\n            width += doc.getWidth();\n        }\n        return width;\n    }\n\n    private static Range<Integer> union(Range<Integer> x, Range<Integer> y) {\n        return x.isEmpty() ? y : y.isEmpty() ? x : x.span(y).canonical(INTEGERS);\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n                .add(\"debugName\", getDebugName())\n                .add(\"plusIndent\", getPlusIndent())\n                .add(\"breakBehaviour\", getBreakBehaviour())\n                .add(\"breakabilityIfLastLevel\", getBreakabilityIfLastLevel())\n                .toString();\n    }\n\n    @Target(ElementType.TYPE)\n    @Retention(RetentionPolicy.SOURCE)\n    @Value.Style(overshadowImplementation = true)\n    @interface SplitsBreaksStyle {}\n\n    @SplitsBreaksStyle\n    @Value.Immutable\n    interface SplitsBreaks {\n        /** Groups of {@link Doc}s that are children of the current {@link Level}, separated by {@link Break}s. */\n        ImmutableList<ImmutableList<Doc>> splits();\n        /** {@link Break}s between {@link Doc}s in the current {@link Level}. */\n        ImmutableList<Break> breaks();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Indent.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport com.fasterxml.jackson.annotation.JsonTypeInfo;\nimport com.fasterxml.jackson.annotation.JsonTypeName;\nimport com.google.common.base.MoreObjects;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.doc.Break;\nimport com.palantir.javaformat.doc.BreakTag;\nimport com.palantir.javaformat.doc.Level;\nimport com.palantir.javaformat.doc.State;\n\n/**\n * An indent for a {@link Level} or {@link Break}. The indent is either a constant {@code int}, or a conditional\n * expression whose value depends on whether or not a {@link Break} has been broken.\n */\n@Immutable\n@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = \"type\")\npublic abstract class Indent {\n\n    public abstract int eval(State state);\n\n    /** A constant function, returning a constant indent. */\n    @JsonTypeName(\"const\")\n    public static final class Const extends Indent {\n        @JsonProperty(\"amount\")\n        private final int n;\n\n        public static final Const ZERO = new Const(+0);\n\n        private Const(int n) {\n            this.n = n;\n        }\n\n        public static Const make(int n, int indentMultiplier) {\n            return new Const(n * indentMultiplier);\n        }\n\n        @Override\n        public int eval(State _state) {\n            return n;\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this).add(\"n\", n).toString();\n        }\n    }\n\n    /** A conditional function, whose value depends on whether a break was taken. */\n    @JsonTypeName(\"if\")\n    public static final class If extends Indent {\n        @JsonProperty\n        private final BreakTag condition;\n\n        @JsonProperty\n        private final Indent thenIndent;\n\n        @JsonProperty\n        private final Indent elseIndent;\n\n        private If(BreakTag condition, Indent thenIndent, Indent elseIndent) {\n            this.condition = condition;\n            this.thenIndent = thenIndent;\n            this.elseIndent = elseIndent;\n        }\n\n        public static If make(BreakTag condition, Indent thenIndent, Indent elseIndent) {\n            return new If(condition, thenIndent, elseIndent);\n        }\n\n        @Override\n        public int eval(State state) {\n            return (state.wasBreakTaken(condition) ? thenIndent : elseIndent).eval(state);\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this)\n                    .add(\"condition\", condition)\n                    .add(\"thenIndent\", thenIndent)\n                    .add(\"elseIndent\", elseIndent)\n                    .toString();\n        }\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/Break.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport com.google.common.collect.Range;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Indent;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.Output;\nimport com.palantir.javaformat.doc.State.BreakState;\nimport java.util.Optional;\nimport org.immutables.value.Value;\n\n/** A leaf node in a {@link Doc} for an optional break. */\n@Immutable\n@Value.Immutable\npublic abstract class Break extends Doc implements Op {\n    public abstract FillMode fillMode();\n\n    public abstract String flat();\n\n    public abstract Indent plusIndent();\n\n    public abstract Optional<BreakTag> optTag();\n\n    /** If this break is restricted by its parent {@link Level}'s {@link Level#getColumnLimitBeforeLastBreak()}. */\n    public abstract boolean hasColumnLimit();\n\n    /**\n     * Make a {@code Break}.\n     *\n     * @param fillMode the {@link FillMode}\n     * @param flat the text when not broken\n     * @param plusIndent extra indent if taken\n     * @return the new {@code Break}\n     */\n    public static Break make(FillMode fillMode, String flat, Indent plusIndent) {\n        return builder()\n                .fillMode(fillMode)\n                .flat(flat)\n                .plusIndent(plusIndent)\n                .hasColumnLimit(false)\n                .build();\n    }\n\n    /**\n     * Make a {@code Break}.\n     *\n     * @param fillMode the {@link FillMode}\n     * @param flat the text when not broken\n     * @param plusIndent extra indent if taken\n     * @param optTag an optional tag for remembering whether the break was taken\n     * @return the new {@code Break}\n     */\n    public static Break make(FillMode fillMode, String flat, Indent plusIndent, Optional<BreakTag> optTag) {\n        return builder()\n                .fillMode(fillMode)\n                .flat(flat)\n                .plusIndent(plusIndent)\n                .optTag(optTag)\n                .hasColumnLimit(false)\n                .build();\n    }\n\n    /**\n     * Make a forced {@code Break}.\n     *\n     * @return the new forced {@code Break}\n     */\n    public static Break makeForced() {\n        return builder()\n                .fillMode(FillMode.FORCED)\n                .flat(\"\")\n                .plusIndent(Indent.Const.ZERO)\n                .hasColumnLimit(false)\n                .build();\n    }\n\n    /**\n     * Return the {@code Break}'s extra indent.\n     *\n     * @return the extra indent\n     */\n    public int evalPlusIndent(State state) {\n        return plusIndent().eval(state);\n    }\n\n    /**\n     * Is the {@code Break} forced?\n     *\n     * @return whether the {@code Break} is forced\n     */\n    public boolean isForced() {\n        return fillMode() == FillMode.FORCED;\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.breakDoc(this);\n    }\n\n    @Override\n    protected float computeWidth() {\n        return isForced() ? Float.POSITIVE_INFINITY : (float) flat().length();\n    }\n\n    @Override\n    protected String computeFlat() {\n        return flat();\n    }\n\n    @Override\n    protected Range<Integer> computeRange() {\n        return EMPTY_RANGE;\n    }\n\n    public State computeBreaks(State stateIn, boolean broken) {\n        State state =\n                optTag().map(breakTag -> stateIn.breakTaken(breakTag, broken)).orElse(stateIn);\n        return state.withBreak(this, broken);\n    }\n\n    @Override\n    public State computeBreaks(\n            CommentsHelper commentsHelper, int maxWidth, State state, Obs.ExplorationNode observationNode) {\n        // Updating the state for {@link Break}s requires deciding if the break\n        // should be taken.\n        // TODO(cushon): this hierarchy is wrong, create a separate interface\n        // for unbreakable Docs?\n        throw new UnsupportedOperationException(\"Did you mean computeBreaks(State, int, boolean)?\");\n    }\n\n    @Override\n    public void write(State state, Output output) {\n        BreakState breakState = state.getBreakState(this);\n        if (breakState.broken()) {\n            output.append(state, \"\\n\", EMPTY_RANGE);\n            output.indent(breakState.newIndent());\n        } else {\n            output.append(state, flat(), range());\n        }\n    }\n\n    public static class Builder extends ImmutableBreak.Builder {}\n\n    public static Builder builder() {\n        return new Builder();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/DimensionHelpers.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.collect.ImmutableList;\nimport com.sun.source.tree.AnnotatedTypeTree;\nimport com.sun.source.tree.AnnotationTree;\nimport com.sun.source.tree.ArrayTypeTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.tools.javac.tree.JCTree;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.Deque;\nimport java.util.List;\n\n/**\n * Utilities for working with array dimensions.\n *\n * <p>javac's parser does not preserve concrete syntax for mixed-notation arrays, so we have to re-lex the input to\n * extra it.\n *\n * <p>For example, {@code int [] a;} cannot be distinguished from {@code int [] a [];} in the AST.\n */\nclass DimensionHelpers {\n\n    /** The array dimension specifiers (including any type annotations) associated with a type. */\n    static class TypeWithDims {\n        final Tree node;\n        final ImmutableList<List<AnnotationTree>> dims;\n\n        public TypeWithDims(Tree node, ImmutableList<List<AnnotationTree>> dims) {\n            this.node = node;\n            this.dims = dims;\n        }\n    }\n\n    enum SortedDims {\n        YES,\n        NO\n    }\n\n    /** Returns a (possibly re-ordered) {@link TypeWithDims} for the given type. */\n    static TypeWithDims extractDims(Tree node, SortedDims sorted) {\n        Deque<List<AnnotationTree>> builder = new ArrayDeque<>();\n        node = extractDims(builder, node);\n        Iterable<List<AnnotationTree>> dims;\n        if (sorted == SortedDims.YES) {\n            dims = reorderBySourcePosition(builder);\n        } else {\n            dims = builder;\n        }\n        return new TypeWithDims(node, ImmutableList.copyOf(dims));\n    }\n\n    /**\n     * Rotate the list of dimension specifiers until all dimensions with type annotations appear in source order.\n     *\n     * <p>javac reorders dimension specifiers in method declarations with mixed-array notation, which means that any\n     * type annotations don't appear in source order.\n     *\n     * <p>For example, the type of {@code int @A [] f() @B [] {}} is parsed as {@code @B [] @A []}.\n     *\n     * <p>This doesn't handle cases with un-annotated dimension specifiers, so the formatting logic checks the token\n     * stream to figure out which side of the method name they appear on.\n     */\n    private static Iterable<List<AnnotationTree>> reorderBySourcePosition(Deque<List<AnnotationTree>> dims) {\n        int lastAnnotation = -1;\n        int lastPos = -1;\n        int idx = 0;\n        for (List<AnnotationTree> dim : dims) {\n            if (!dim.isEmpty()) {\n                int pos = ((JCTree) dim.get(0)).getStartPosition();\n                if (pos < lastPos) {\n                    List<List<AnnotationTree>> list = new ArrayList<>(dims);\n                    Collections.rotate(list, -(lastAnnotation + 1));\n                    return list;\n                }\n                lastPos = pos;\n                lastAnnotation = idx;\n            }\n            idx++;\n        }\n        return dims;\n    }\n\n    /**\n     * Accumulates a flattened list of array dimensions specifiers with type annotations, and returns the base type.\n     *\n     * <p>Given {@code int @A @B [][] @C []}, adds {@code [[@A, @B], [@C]]} to dims and returns {@code int}.\n     */\n    private static Tree extractDims(Deque<List<AnnotationTree>> dims, Tree node) {\n        switch (node.getKind()) {\n            case ARRAY_TYPE:\n                return extractDims(dims, ((ArrayTypeTree) node).getType());\n            case ANNOTATED_TYPE:\n                AnnotatedTypeTree annotatedTypeTree = (AnnotatedTypeTree) node;\n                if (annotatedTypeTree.getUnderlyingType().getKind() != Tree.Kind.ARRAY_TYPE) {\n                    return node;\n                }\n                node = extractDims(dims, annotatedTypeTree.getUnderlyingType());\n                dims.addFirst(ImmutableList.copyOf(annotatedTypeTree.getAnnotations()));\n                return node;\n            default:\n                return node;\n        }\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/Trees.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.sun.source.tree.ClassTree;\nimport com.sun.source.tree.CompoundAssignmentTree;\nimport com.sun.source.tree.ExpressionTree;\nimport com.sun.source.tree.IdentifierTree;\n...\n", "needle_token_start": 7143, "needle_token_end": 7277, "code_context_ntokens": 16054, "output": ["```java\nprivate Optional<State> tryBreakLastLevel(\n        CommentsHelper commentsHelper,\n        int maxWidth,\n        State state,\n        ExplorationNode explorationNode,\n        boolean isSimpleInliningSoFar) {\n    if (docs.isEmpty() || !(getLast(docs) instanceof Level)) {\n        return Optional.empty();\n    }\n    Level innerLevel = ((Level) getLast(docs));\n\n    return tryBreakInnerLevel(commentsHelper, maxWidth, state, explorationNode, innerLevel, isSimpleInliningSoFar);\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "visitYield", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/java14/Java14InputAstVisitor.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the formatting of a specific Java language construct that returns a value from a switch expression. It ensures that the syntax is correctly represented in the output format.\n2. **Input**: The function takes two parameters: a tree node representing the language construct and a null value used as a placeholder for context.\n3. **Output**: The function returns a null value, indicating that its primary role is to perform an operation rather than produce a data output.\n4. **Procedure**: The function first synchronizes the position in the source code with the current node. It then outputs the keyword associated with the construct, followed by a space. Next, it processes the value expression associated with the construct, and finally, it appends a semicolon to denote the end of the statement.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocLexer.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java.javadoc;\n\nimport static com.google.common.base.Preconditions.checkArgument;\nimport static com.google.common.base.Preconditions.checkNotNull;\nimport static com.google.common.base.Verify.verify;\nimport static com.google.common.collect.Iterators.peekingIterator;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BEGIN_JAVADOC;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BLOCKQUOTE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BLOCKQUOTE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BR_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.CODE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.CODE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.END_JAVADOC;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.FOOTER_JAVADOC_TAG_START;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.FORCED_NEWLINE;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.HEADER_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.HEADER_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.HTML_COMMENT;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.INLINE_TAG_CLOSE;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.INLINE_TAG_OPEN;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_ITEM_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_ITEM_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LITERAL;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.MOE_BEGIN_STRIP_COMMENT;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.MOE_END_STRIP_COMMENT;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.OPTIONAL_LINE_BREAK;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PRE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PRE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.TABLE_CLOSE_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.TABLE_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.WHITESPACE;\nimport static java.lang.String.format;\nimport static java.util.regex.Pattern.CASE_INSENSITIVE;\nimport static java.util.regex.Pattern.DOTALL;\nimport static java.util.regex.Pattern.compile;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.PeekingIterator;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Deque;\nimport java.util.List;\nimport java.util.regex.Pattern;\n\n/** Lexer for the Javadoc formatter. */\nfinal class JavadocLexer {\n    /** Takes a Javadoc comment, including \u2215\u2731\u2731 and \u2731\u2215, and returns tokens, including \u2215\u2731\u2731 and \u2731\u2215. */\n    static ImmutableList<Token> lex(String input) throws LexException {\n        /*\n         * TODO(cpovirk): In theory, we should interpret Unicode escapes (yet output them in their\n         * original form). This would mean mean everything from an encoded \u2215\u2731\u2731 to an encoded <pre> tag,\n         * so we'll probably never bother.\n         */\n        input = stripJavadocBeginAndEnd(input);\n        input = normalizeLineEndings(input);\n        return new JavadocLexer(new CharStream(input)).generateTokens();\n    }\n\n    /** The lexer crashes on windows line endings, so for now just normalize to `\\n`. */\n    // TODO(cushon): use the platform line separator for output\n    private static String normalizeLineEndings(String input) {\n        return NON_UNIX_LINE_ENDING.matcher(input).replaceAll(\"\\n\");\n    }\n\n    private static final Pattern NON_UNIX_LINE_ENDING = Pattern.compile(\"\\r\\n?\");\n\n    private static String stripJavadocBeginAndEnd(String input) {\n        /*\n         * We do this ahead of time so that the main part of the lexer need not say things like\n         * \"(?![*]/)\" to avoid accidentally swallowing \u2731\u2215 when consuming a newline.\n         */\n        checkArgument(input.startsWith(\"/**\"), \"Missing /**: %s\", input);\n        checkArgument(input.endsWith(\"*/\") && input.length() > 4, \"Missing */: %s\", input);\n        return input.substring(\"/**\".length(), input.length() - \"*/\".length());\n    }\n\n    private final CharStream input;\n    private final NestingCounter braceDepth = new NestingCounter();\n    private final NestingCounter preDepth = new NestingCounter();\n    private final NestingCounter codeDepth = new NestingCounter();\n    private final NestingCounter tableDepth = new NestingCounter();\n    private boolean somethingSinceNewline;\n\n    private JavadocLexer(CharStream input) {\n        this.input = checkNotNull(input);\n    }\n\n    private ImmutableList<Token> generateTokens() throws LexException {\n        ImmutableList.Builder<Token> tokens = ImmutableList.builder();\n\n        Token token = new Token(BEGIN_JAVADOC, \"/**\");\n        tokens.add(token);\n\n        while (!input.isExhausted()) {\n            token = readToken();\n            tokens.add(token);\n        }\n\n        checkMatchingTags();\n\n        token = new Token(END_JAVADOC, \"*/\");\n        tokens.add(token);\n\n        ImmutableList<Token> result = tokens.build();\n        result = joinAdjacentLiteralsAndAdjacentWhitespace(result);\n        result = inferParagraphTags(result);\n        result = optionalizeSpacesAfterLinks(result);\n        result = deindentPreCodeBlocks(result);\n        return result;\n    }\n\n    private Token readToken() throws LexException {\n        Token.Type type = consumeToken();\n        String value = input.readAndResetRecorded();\n        return new Token(type, value);\n    }\n\n    private Token.Type consumeToken() throws LexException {\n        boolean preserveExistingFormatting = preserveExistingFormatting();\n\n        if (input.tryConsumeRegex(NEWLINE_PATTERN)) {\n            somethingSinceNewline = false;\n            return preserveExistingFormatting ? FORCED_NEWLINE : WHITESPACE;\n        } else if (input.tryConsume(\" \") || input.tryConsume(\"\\t\")) {\n            // TODO(cpovirk): How about weird whitespace chars? Ideally we'd distinguish breaking vs. not.\n            // Returning LITERAL here prevent us from breaking a <pre> line. For more info, see LITERAL.\n            return preserveExistingFormatting ? LITERAL : WHITESPACE;\n        }\n\n        /*\n         * TODO(cpovirk): Maybe try to detect things like \"{@code\\n@GwtCompatible}\" that aren't intended\n         * as tags. But in the most likely case, in which that happens inside <pre>{@code, we have no\n         * great options for fixing it.\n         * https://github.com/google/google-java-format/issues/7#issuecomment-197383926\n         */\n        if (!somethingSinceNewline && input.tryConsumeRegex(FOOTER_TAG_PATTERN)) {\n            checkMatchingTags();\n            somethingSinceNewline = true;\n            return FOOTER_JAVADOC_TAG_START;\n        }\n        somethingSinceNewline = true;\n\n        if (input.tryConsumeRegex(INLINE_TAG_OPEN_PATTERN)) {\n            braceDepth.increment();\n            return INLINE_TAG_OPEN;\n        } else if (input.tryConsume(\"{\")) {\n            braceDepth.incrementIfPositive();\n            return LITERAL;\n        } else if (input.tryConsume(\"}\")) {\n            braceDepth.decrementIfPositive();\n            return braceDepth.isPositive() ? LITERAL : INLINE_TAG_CLOSE;\n        }\n\n        // Inside an inline tag, don't do any HTML interpretation.\n        if (braceDepth.isPositive()) {\n            verify(input.tryConsumeRegex(LITERAL_PATTERN));\n            return LITERAL;\n        }\n\n        if (input.tryConsumeRegex(PRE_OPEN_PATTERN)) {\n            preDepth.increment();\n            return preserveExistingFormatting ? LITERAL : PRE_OPEN_TAG;\n        } else if (input.tryConsumeRegex(PRE_CLOSE_PATTERN)) {\n            preDepth.decrementIfPositive();\n            return preserveExistingFormatting() ? LITERAL : PRE_CLOSE_TAG;\n        }\n\n        if (input.tryConsumeRegex(CODE_OPEN_PATTERN)) {\n            codeDepth.increment();\n            return preserveExistingFormatting ? LITERAL : CODE_OPEN_TAG;\n...\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocFormatter.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java.javadoc;\n\nimport static com.palantir.javaformat.java.javadoc.JavadocLexer.LexException;\nimport static com.palantir.javaformat.java.javadoc.JavadocLexer.lex;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.BR_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_OPEN_TAG;\nimport static java.util.regex.Pattern.CASE_INSENSITIVE;\nimport static java.util.regex.Pattern.compile;\n\nimport com.google.common.collect.ImmutableList;\nimport java.util.List;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n/**\n * Entry point for formatting Javadoc.\n *\n * <p>This stateless class reads tokens from the stateful lexer and translates them to \"requests\" and \"writes\" to the\n * stateful writer. It also munges tokens into \"standardized\" forms. Finally, it performs postprocessing to convert the\n * written Javadoc to a one-liner if possible or to leave a single blank line if it's empty.\n */\npublic final class JavadocFormatter {\n\n    private final int maxLineLength;\n\n    public JavadocFormatter(int maxLineLength) {\n        this.maxLineLength = maxLineLength;\n    }\n\n    /**\n     * Formats the given Javadoc comment, which must start with \u2215\u2731\u2731 and end with \u2731\u2215. The output will start and end with\n     * the same characters.\n     */\n    public String formatJavadoc(String input, int blockIndent) {\n        ImmutableList<Token> tokens;\n        try {\n            tokens = lex(input);\n        } catch (LexException e) {\n            return input;\n        }\n        String result = render(tokens, blockIndent);\n        return makeSingleLineIfPossible(blockIndent, result);\n    }\n\n    private String render(List<Token> input, int blockIndent) {\n        JavadocWriter output = new JavadocWriter(blockIndent, maxLineLength);\n        for (Token token : input) {\n            switch (token.getType()) {\n                case BEGIN_JAVADOC:\n                    output.writeBeginJavadoc();\n                    break;\n                case END_JAVADOC:\n                    output.writeEndJavadoc();\n                    return output.toString();\n                case FOOTER_JAVADOC_TAG_START:\n                    output.writeFooterJavadocTagStart(token);\n                    break;\n                case LIST_OPEN_TAG:\n                    output.writeListOpen(token);\n                    break;\n                case LIST_CLOSE_TAG:\n                    output.writeListClose(token);\n                    break;\n                case LIST_ITEM_OPEN_TAG:\n                    output.writeListItemOpen(token);\n                    break;\n                case HEADER_OPEN_TAG:\n                    output.writeHeaderOpen(token);\n                    break;\n                case HEADER_CLOSE_TAG:\n                    output.writeHeaderClose(token);\n                    break;\n                case PARAGRAPH_OPEN_TAG:\n                    output.writeParagraphOpen(standardizePToken(token));\n                    break;\n                case BLOCKQUOTE_OPEN_TAG:\n                case BLOCKQUOTE_CLOSE_TAG:\n                    output.writeBlockquoteOpenOrClose(token);\n                    break;\n                case PRE_OPEN_TAG:\n                    output.writePreOpen(token);\n                    break;\n                case PRE_CLOSE_TAG:\n                    output.writePreClose(token);\n                    break;\n                case CODE_OPEN_TAG:\n                    output.writeCodeOpen(token);\n                    break;\n                case CODE_CLOSE_TAG:\n                    output.writeCodeClose(token);\n                    break;\n                case TABLE_OPEN_TAG:\n                    output.writeTableOpen(token);\n                    break;\n                case TABLE_CLOSE_TAG:\n                    output.writeTableClose(token);\n                    break;\n                case MOE_BEGIN_STRIP_COMMENT:\n                    output.requestMoeBeginStripComment(token);\n                    break;\n                case MOE_END_STRIP_COMMENT:\n                    output.writeMoeEndStripComment(token);\n                    break;\n                case HTML_COMMENT:\n                    output.writeHtmlComment(token);\n                    break;\n                case BR_TAG:\n                    output.writeBr(standardizeBrToken(token));\n                    break;\n                case WHITESPACE:\n                    output.requestWhitespace();\n                    break;\n                case FORCED_NEWLINE:\n                    output.writeLineBreakNoAutoIndent();\n                    break;\n                case LITERAL:\n                case INLINE_TAG_OPEN:\n                case INLINE_TAG_CLOSE:\n                    output.writeLiteral(token);\n                    break;\n                case PARAGRAPH_CLOSE_TAG:\n                case LIST_ITEM_CLOSE_TAG:\n                case OPTIONAL_LINE_BREAK:\n                    break;\n                default:\n                    throw new IllegalArgumentException(token.getType().name());\n            }\n        }\n        throw new IllegalStateException();\n    }\n\n    /*\n     * TODO(cpovirk): Is this really the right location for the standardize* methods? Maybe the lexer\n     * should include them as part of its own postprocessing? Or even the writer could make sense.\n     */\n\n    private static Token standardizeBrToken(Token token) {\n        return standardize(token, STANDARD_BR_TOKEN);\n    }\n\n    private static Token standardizePToken(Token token) {\n        return standardize(token, STANDARD_P_TOKEN);\n    }\n\n    private static Token standardize(Token token, Token standardToken) {\n        return SIMPLE_TAG_PATTERN.matcher(token.getValue()).matches() ? standardToken : token;\n    }\n\n    private static final Token STANDARD_BR_TOKEN = new Token(BR_TAG, \"<br>\");\n    private static final Token STANDARD_P_TOKEN = new Token(PARAGRAPH_OPEN_TAG, \"<p>\");\n    private static final Pattern SIMPLE_TAG_PATTERN = compile(\"^<\\\\w+\\\\s*/?\\\\s*>\", CASE_INSENSITIVE);\n\n    private static final Pattern ONE_CONTENT_LINE_PATTERN = compile(\" */[*][*]\\n *[*] (.*)\\n *[*]/\");\n\n    /** Returns the given string or a one-line version of it (e.g., \"\u2215\u2731\u2731 Tests for foos. \u2731\u2215\") if it fits on one line. */\n    private String makeSingleLineIfPossible(int blockIndent, String input) {\n        int oneLinerContentLength = maxLineLength - \"/**  */\".length() - blockIndent;\n        Matcher matcher = ONE_CONTENT_LINE_PATTERN.matcher(input);\n        if (matcher.matches() && matcher.group(1).isEmpty()) {\n            return \"/** */\";\n        } else if (matcher.matches() && matcher.group(1).length() <= oneLinerContentLength) {\n            return \"/** \" + matcher.group(1) + \" */\";\n        }\n        return input;\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaCommentsHelper.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Input.Tok;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.java.javadoc.JavadocFormatter;\nimport java.util.ArrayList;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\nimport javax.annotation.Nullable;\n\n/** {@code JavaCommentsHelper} extends {@link CommentsHelper} to rewrite Java comments. */\npublic final class JavaCommentsHelper implements CommentsHelper {\n\n    private final String lineSeparator;\n    private final JavaFormatterOptions options;\n\n    @Nullable\n    private final JavadocFormatter javadocFormatter;\n\n    public JavaCommentsHelper(String lineSeparator, JavaFormatterOptions options) {\n        this.lineSeparator = lineSeparator;\n        this.options = options;\n        this.javadocFormatter = options.formatJavadoc() ? new JavadocFormatter(options.maxLineLength()) : null;\n    }\n\n    @Override\n    public String rewrite(Tok tok, int maxWidth, int column0) {\n        if (!tok.isComment()) {\n            return tok.getOriginalText();\n        }\n        String text = tok.getOriginalText();\n        if (javadocFormatter != null && tok.isJavadocComment()) {\n            text = javadocFormatter.formatJavadoc(text, column0);\n        }\n        List<String> lines = new ArrayList<>();\n        Iterator<String> it = Newlines.lineIterator(text);\n        while (it.hasNext()) {\n            lines.add(CharMatcher.whitespace().trimTrailingFrom(it.next()));\n        }\n        if (tok.isSlashSlashComment()) {\n            return indentLineComments(lines, column0);\n        } else if (javadocShaped(lines)) {\n            return indentJavadoc(lines, column0);\n        } else {\n            return preserveIndentation(lines, column0);\n        }\n    }\n\n    // For non-javadoc-shaped block comments, shift the entire block to the correct\n    // column, but do not adjust relative indentation.\n    private String preserveIndentation(List<String> lines, int column0) {\n        StringBuilder builder = new StringBuilder();\n\n        // find the leftmost non-whitespace character in all trailing lines\n        int startCol = -1;\n        for (int i = 1; i < lines.size(); i++) {\n            int lineIdx = CharMatcher.whitespace().negate().indexIn(lines.get(i));\n            if (lineIdx >= 0 && (startCol == -1 || lineIdx < startCol)) {\n                startCol = lineIdx;\n            }\n        }\n\n        // output the first line at the current column\n        builder.append(lines.get(0));\n\n        // output all trailing lines with plausible indentation\n        for (int i = 1; i < lines.size(); ++i) {\n            builder.append(lineSeparator).append(\" \".repeat(column0));\n            // check that startCol is valid index, e.g. for blank lines\n            if (lines.get(i).length() >= startCol) {\n                builder.append(lines.get(i).substring(startCol));\n            } else {\n                builder.append(lines.get(i));\n            }\n        }\n        return builder.toString();\n    }\n\n    // Wraps and re-indents line comments.\n    private String indentLineComments(List<String> lines, int column0) {\n        lines = wrapLineComments(lines, column0);\n        StringBuilder builder = new StringBuilder();\n        builder.append(lines.get(0).trim());\n        String indentString = \" \".repeat(column0);\n        for (int i = 1; i < lines.size(); ++i) {\n            builder.append(lineSeparator)\n                    .append(indentString)\n                    .append(lines.get(i).trim());\n        }\n        return builder.toString();\n    }\n\n    // Preserve special `//noinspection` and `//$NON-NLS-x$` comments used by IDEs, which cannot\n    // contain leading spaces.\n    private static final Pattern LINE_COMMENT_MISSING_SPACE_PREFIX =\n            Pattern.compile(\"^(//+)(?!noinspection|\\\\$NON-NLS-\\\\d+\\\\$)[^\\\\s/]\");\n\n    private List<String> wrapLineComments(List<String> lines, int column0) {\n        List<String> result = new ArrayList<>();\n        for (String line : lines) {\n            // Add missing leading spaces to line comments: `//foo` -> `// foo`.\n            Matcher matcher = LINE_COMMENT_MISSING_SPACE_PREFIX.matcher(line);\n            if (matcher.find()) {\n                int length = matcher.group(1).length();\n                line = \"/\".repeat(length) + \" \" + line.substring(length);\n            }\n            if (line.startsWith(\"// MOE:\")) {\n                // don't wrap comments for https://github.com/google/MOE\n                result.add(line);\n                continue;\n            }\n            while (line.length() + column0 > options.maxLineLength()) {\n                int idx = options.maxLineLength() - column0;\n                // only break on whitespace characters, and ignore the leading `// `\n                while (idx >= 2 && !CharMatcher.whitespace().matches(line.charAt(idx))) {\n                    idx--;\n                }\n                if (idx <= 2) {\n                    break;\n                }\n                result.add(line.substring(0, idx));\n                line = \"//\" + line.substring(idx);\n            }\n            result.add(line);\n        }\n        return result;\n    }\n\n    // Remove leading whitespace (trailing was already removed), and re-indent.\n    // Add a +1 indent before '*', and add the '*' if necessary.\n    private String indentJavadoc(List<String> lines, int column0) {\n        StringBuilder builder = new StringBuilder();\n        builder.append(lines.get(0).trim());\n        int indent = column0 + 1;\n        String indentString = \" \".repeat(indent);\n        for (int i = 1; i < lines.size(); ++i) {\n            builder.append(lineSeparator).append(indentString);\n            String line = lines.get(i).trim();\n            if (!line.startsWith(\"*\")) {\n                builder.append(\"* \");\n            }\n            builder.append(line);\n        }\n        return builder.toString();\n    }\n\n    // Returns true if the comment looks like javadoc\n    private static boolean javadocShaped(List<String> lines) {\n        Iterator<String> it = lines.iterator();\n        if (!it.hasNext()) {\n            return false;\n        }\n        String first = it.next().trim();\n        // if it's actually javadoc, we're done\n        if (first.startsWith(\"/**\")) {\n            return true;\n        }\n        // if it's a block comment, check all trailing lines for '*'\n        if (!first.startsWith(\"/*\")) {\n            return false;\n        }\n        while (it.hasNext()) {\n            if (!it.next().trim().startsWith(\"*\")) {\n                return false;\n            }\n        }\n        return true;\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/java14/Java14InputAstVisitor.java\n/*\n * (c) Copyright 2020 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.java.java14;\n\nimport static com.google.common.collect.ImmutableList.toImmutableList;\nimport static com.sun.source.tree.Tree.Kind.BLOCK;\n\nimport com.google.common.base.Verify;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Iterables;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.OpsBuilder;\nimport com.palantir.javaformat.OpsBuilder.BlankLineWanted;\nimport com.palantir.javaformat.java.JavaInputAstVisitor;\nimport com.sun.source.tree.BindingPatternTree;\nimport com.sun.source.tree.BlockTree;\nimport com.sun.source.tree.CaseTree;\nimport com.sun.source.tree.ClassTree;\nimport com.sun.source.tree.CompilationUnitTree;\nimport com.sun.source.tree.InstanceOfTree;\nimport com.sun.source.tree.LambdaExpressionTree;\nimport com.sun.source.tree.ModifiersTree;\nimport com.sun.source.tree.ModuleTree;\nimport com.sun.source.tree.SwitchExpressionTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.source.tree.Tree.Kind;\nimport com.sun.source.tree.VariableTree;\nimport com.sun.source.tree.YieldTree;\nimport com.sun.tools.javac.code.Flags;\nimport com.sun.tools.javac.tree.JCTree;\nimport com.sun.tools.javac.tree.JCTree.JCVariableDecl;\nimport com.sun.tools.javac.tree.TreeInfo;\nimport java.lang.reflect.Method;\nimport java.util.List;\nimport java.util.Optional;\nimport javax.lang.model.element.Name;\n\n/**\n * Extends {@link JavaInputAstVisitor} with support for AST nodes that were added or modified for Java 14.\n */\npublic class Java14InputAstVisitor extends JavaInputAstVisitor {\n    private static final Method COMPILATION_UNIT_TREE_GET_MODULE =\n            maybeGetMethod(CompilationUnitTree.class, \"getModule\");\n    private static final Method CLASS_TREE_GET_PERMITS_CLAUSE = maybeGetMethod(ClassTree.class, \"getPermitsClause\");\n    private static final Method BINDING_PATTERN_TREE_GET_VARIABLE =\n            maybeGetMethod(BindingPatternTree.class, \"getVariable\");\n    private static final Method BINDING_PATTERN_TREE_GET_TYPE = maybeGetMethod(BindingPatternTree.class, \"getType\");\n    private static final Method BINDING_PATTERN_TREE_GET_BINDING =\n            maybeGetMethod(BindingPatternTree.class, \"getBinding\");\n    private static final Method CASE_TREE_GET_LABELS = maybeGetMethod(CaseTree.class, \"getLabels\");\n\n    public Java14InputAstVisitor(OpsBuilder builder, int indentMultiplier) {\n        super(builder, indentMultiplier);\n    }\n\n    @Override\n    protected void handleModule(boolean first, CompilationUnitTree node) {\n        if (COMPILATION_UNIT_TREE_GET_MODULE == null) {\n            // Java < 17, see https://bugs.openjdk.java.net/browse/JDK-8255464\n            return;\n        }\n        ModuleTree module = (ModuleTree) invoke(COMPILATION_UNIT_TREE_GET_MODULE, node);\n        if (module != null) {\n            if (!first) {\n                builder.blankLineWanted(BlankLineWanted.YES);\n            }\n            markForPartialFormat();\n            visitModule(module, null);\n            builder.forcedBreak();\n        }\n    }\n\n    @Override\n    protected List<? extends Tree> getPermitsClause(ClassTree node) {\n        if (CLASS_TREE_GET_PERMITS_CLAUSE != null) {\n            return (List<? extends Tree>) invoke(CLASS_TREE_GET_PERMITS_CLAUSE, node);\n        } else {\n            // Java < 15\n            return super.getPermitsClause(node);\n        }\n    }\n\n    @Override\n    public Void visitBindingPattern(BindingPatternTree node, Void unused) {\n        sync(node);\n        if (BINDING_PATTERN_TREE_GET_VARIABLE != null) {\n            VariableTree variableTree = (VariableTree) invoke(BINDING_PATTERN_TREE_GET_VARIABLE, node);\n            visitBindingPattern(variableTree.getModifiers(), variableTree.getType(), variableTree.getName());\n        } else if (BINDING_PATTERN_TREE_GET_TYPE != null && BINDING_PATTERN_TREE_GET_BINDING != null) {\n            Tree type = (Tree) invoke(BINDING_PATTERN_TREE_GET_TYPE, node);\n            Name name = (Name) invoke(BINDING_PATTERN_TREE_GET_BINDING, node);\n            visitBindingPattern(/* modifiers= */ null, type, name);\n        } else {\n            throw new RuntimeException(\n                    \"BindingPatternTree must have either getVariable() or both getType() and getBinding(),\"\n                            + \" but does not\");\n        }\n        return null;\n    }\n\n    private void visitBindingPattern(ModifiersTree modifiers, Tree type, Name name) {\n        if (modifiers != null) {\n            builder.addAll(visitModifiers(modifiers, Direction.HORIZONTAL, Optional.empty()));\n        }\n        scan(type, null);\n        builder.breakOp(\" \");\n        visit(name);\n    }\n\n    \n@Override\n    public Void visitYield(YieldTree node, Void aVoid) {\n        sync(node);\n        token(\"yield\");\n        builder.space();\n        scan(node.getValue(), null);\n        token(\";\");\n        return null;\n    }\n\n    @Override\n    public Void visitSwitchExpression(SwitchExpressionTree node, Void unused) {\n        sync(node);\n        visitSwitch(node.getExpression(), node.getCases());\n        return null;\n    }\n\n    @Override\n    public Void visitClass(ClassTree tree, Void unused) {\n        switch (tree.getKind()) {\n            case ANNOTATION_TYPE:\n                visitAnnotationType(tree);\n                break;\n            case CLASS:\n            case INTERFACE:\n                visitClassDeclaration(tree);\n                break;\n            case ENUM:\n                visitEnumDeclaration(tree);\n                break;\n            case RECORD:\n                visitRecordDeclaration(tree);\n                break;\n            default:\n                throw new IllegalArgumentException(tree.getKind().name());\n        }\n        return null;\n    }\n\n    public void visitRecordDeclaration(ClassTree node) {\n        sync(node);\n        List<Op> breaks = visitModifiers(\n                node.getModifiers(), Direction.VERTICAL, /* declarationAnnotationBreak= */ Optional.empty());\n        Verify.verify(node.getExtendsClause() == null);\n        boolean hasSuperInterfaceTypes = !node.getImplementsClause().isEmpty();\n        builder.addAll(breaks);\n        token(\"record\");\n        builder.space();\n        visit(node.getSimpleName());\n        if (!node.getTypeParameters().isEmpty()) {\n            token(\"<\");\n        }\n        builder.open(plusFour);\n        {\n            if (!node.getTypeParameters().isEmpty()) {\n                typeParametersRest(node.getTypeParameters(), hasSuperInterfaceTypes ? plusFour : ZERO);\n            }\n            ImmutableList<JCVariableDecl> parameters = recordVariables(node);\n            token(\"(\");\n            if (!parameters.isEmpty()) {\n                // Break before args.\n                builder.breakToFill(\"\");\n            }\n            // record headers can't declare receiver parameters\n            visitFormals(/* receiver= */ Optional.empty(), parameters);\n            token(\")\");\n            if (hasSuperInterfaceTypes) {\n                builder.breakToFill(\" \");\n                builder.open(node.getImplementsClause().size() > 1 ? plusFour : ZERO);\n                token(\"implements\");\n                builder.space();\n                boolean first = true;\n                for (Tree superInterfaceType : node.getImplementsClause()) {\n                    if (!first) {\n                        token(\",\");\n                        builder.breakOp(\" \");\n                    }\n                    scan(superInterfaceType, null);\n                    first = false;\n                }\n                builder.close();\n            }\n        }\n        builder.close();\n        if (node.getMembers() == null) {\n            token(\";\");\n        } else {\n            List<Tree> members = node.getMembers().stream()\n                    .filter(t -> (TreeInfo.flags((JCTree) t) & Flags.GENERATED_MEMBER) == 0)\n                    .collect(toImmutableList());\n            addBodyDeclarations(members, BracesOrNot.YES, FirstDeclarationsOrNot.YES);\n        }\n        dropEmptyDeclarations();\n    }\n\n    private static ImmutableList<JCTree.JCVariableDecl> recordVariables(ClassTree node) {\n        return node.getMembers().stream()\n                .filter(JCTree.JCVariableDecl.class::isInstance)\n                .map(JCTree.JCVariableDecl.class::cast)\n                .filter(m -> (m.mods.flags & RECORD) == RECORD)\n                .collect(toImmutableList());\n    }\n\n    @Override\n    public Void visitInstanceOf(InstanceOfTree node, Void unused) {\n        sync(node);\n        builder.open(plusFour);\n        scan(node.getExpression(), null);\n        builder.breakOp(\" \");\n        builder.open(ZERO);\n        token(\"instanceof\");\n        builder.breakOp(\" \");\n        if (node.getPattern() != null) {\n            scan(node.getPattern(), null);\n        } else {\n            scan(node.getType(), null);\n        }\n        builder.close();\n        builder.close();\n        return null;\n    }\n\n    @Override\n    public Void visitCase(CaseTree node, Void unused) {\n        sync(node);\n        markForPartialFormat();\n        builder.forcedBreak();\n        List<? extends Tree> labels;\n        boolean isDefault;\n        if (CASE_TREE_GET_LABELS != null) {\n            labels = (List<? extends Tree>) invoke(CASE_TREE_GET_LABELS, node);\n            isDefault = labels.size() == 1\n                    && Iterables.getOnlyElement(labels).getKind().name().equals(\"DEFAULT_CASE_LABEL\");\n        } else {\n            labels = node.getExpressions();\n            isDefault = labels.isEmpty();\n        }\n        if (isDefault) {\n            token(\"default\", plusTwo);\n        } else {\n            token(\"case\", plusTwo);\n            builder.space();\n            builder.open(labels.size() > 1 ? plusFour : ZERO);\n            boolean first = true;\n            for (Tree expression : labels) {\n                if (!first) {\n                    token(\",\");\n                    builder.breakOp(\" \");\n                }\n                scan(expression, null);\n                first = false;\n            }\n            builder.close();\n        }\n        switch (node.getCaseKind()) {\n            case STATEMENT:\n                token(\":\");\n                boolean isBlock = node.getStatements().size() == 1\n                        && node.getStatements().get(0).getKind() == BLOCK;\n                builder.open(isBlock ? ZERO : plusTwo);\n                if (isBlock) {\n                    builder.space();\n                }\n                visitStatements(node.getStatements(), isBlock);\n                builder.close();\n                break;\n            case RULE:\n                builder.space();\n                token(\"-\");\n                token(\">\");\n                builder.space();\n                if (node.getBody().getKind() == BLOCK) {\n                    // Explicit call with {@link CollapseEmptyOrNot.YES} to handle empty case blocks.\n                    visitBlock(\n                            (BlockTree) node.getBody(),\n                            CollapseEmptyOrNot.YES,\n                            AllowLeadingBlankLine.NO,\n                            AllowTrailingBlankLine.NO);\n                } else {\n                    scan(node.getBody(), null);\n                }\n                builder.guessToken(\";\");\n                break;\n            default:\n                throw new IllegalArgumentException(node.getCaseKind().name());\n        }\n        return null;\n    }\n\n    /**\n     * TODO(fwindheuser): Collapse with\n     * {@link JavaInputAstVisitor#visitLambdaExpression(LambdaExpressionTree, Void)}} after dropping Java 11\n     * compatibility.\n     */\n    @Override\n    public Void visitLambdaExpression(LambdaExpressionTree node, Void unused) {\n        sync(node);\n        // Also format switch expressions as statement body instead of inlining them\n        boolean statementBody = node.getBodyKind() == LambdaExpressionTree.BodyKind.STATEMENT\n                || node.getBody().getKind() == Kind.SWITCH_EXPRESSION;\n        visitLambdaExpression(node, statementBody);\n        return null;\n    }\n\n    private static Method maybeGetMethod(Class<?> c, String name) {\n        try {\n            return c.getMethod(name);\n        } catch (ReflectiveOperationException e) {\n            return null;\n        }\n    }\n\n    private static Object invoke(Method m, Object target) {\n        try {\n            return m.invoke(target);\n        } catch (ReflectiveOperationException e) {\n            throw new RuntimeException(e.getMessage(), e);\n        }\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/javadoc/JavadocWriter.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java.javadoc;\n\nimport static com.google.common.base.Preconditions.checkNotNull;\nimport static com.google.common.collect.Sets.immutableEnumSet;\nimport static com.palantir.javaformat.java.javadoc.JavadocWriter.AutoIndent.AUTO_INDENT;\nimport static com.palantir.javaformat.java.javadoc.JavadocWriter.AutoIndent.NO_AUTO_INDENT;\nimport static com.palantir.javaformat.java.javadoc.JavadocWriter.RequestedWhitespace.BLANK_LINE;\nimport static com.palantir.javaformat.java.javadoc.JavadocWriter.RequestedWhitespace.NEWLINE;\nimport static com.palantir.javaformat.java.javadoc.JavadocWriter.RequestedWhitespace.NONE;\nimport static com.palantir.javaformat.java.javadoc.JavadocWriter.RequestedWhitespace.WHITESPACE;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.HEADER_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.LIST_ITEM_OPEN_TAG;\nimport static com.palantir.javaformat.java.javadoc.Token.Type.PARAGRAPH_OPEN_TAG;\n\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Ordering;\nimport javax.annotation.Nullable;\n\n/**\n * Stateful object that accepts \"requests\" and \"writes,\" producing formatted Javadoc.\n *\n * <p>Our Javadoc formatter doesn't ever generate a parse tree, only a stream of tokens, so the writer must compute and\n * store the answer to questions like \"How many levels of nested HTML list are we inside?\"\n */\nfinal class JavadocWriter {\n    private final int blockIndent;\n    private final int maxLineLength;\n    private final StringBuilder output = new StringBuilder();\n    /**\n     * Whether we are inside an {@code <li>} element, excluding the case in which the {@code <li>} contains a\n     * {@code <ul>} or {@code <ol>} that we are also inside -- unless of course we're inside an {@code <li>} element in\n     * that inner list :)\n     */\n    private boolean continuingListItemOfInnermostList;\n\n    private boolean continuingFooterTag;\n    private final NestingCounter continuingListItemCount = new NestingCounter();\n    private final NestingCounter continuingListCount = new NestingCounter();\n    private final NestingCounter postWriteModifiedContinuingListCount = new NestingCounter();\n    private int remainingOnLine;\n    private boolean atStartOfLine;\n    private RequestedWhitespace requestedWhitespace = NONE;\n\n    @Nullable\n    private Token requestedMoeBeginStripComment;\n\n    private int indentForMoeEndStripComment;\n    private boolean wroteAnythingSignificant;\n\n    JavadocWriter(int blockIndent, int maxLineLength) {\n        this.blockIndent = blockIndent;\n        this.maxLineLength = maxLineLength;\n    }\n\n    /**\n     * Requests whitespace between the previously written token and the next written token. The request may be honored,\n     * or it may be overridden by a request for \"more significant\" whitespace, like a newline.\n     */\n    void requestWhitespace() {\n        requestWhitespace(WHITESPACE);\n    }\n\n    void requestMoeBeginStripComment(Token token) {\n        // We queue this up so that we can put it after any requested whitespace.\n        requestedMoeBeginStripComment = checkNotNull(token);\n    }\n\n    void writeBeginJavadoc() {\n        /*\n         * JavaCommentsHelper will make sure this is indented right. But it seems sensible enough that,\n         * if our input starts with \u2215\u2731\u2731, so too does our output.\n         */\n        output.append(\"/**\");\n        writeNewline();\n    }\n\n    void writeEndJavadoc() {\n        output.append(\"\\n\");\n        appendSpaces(blockIndent + 1);\n        output.append(\"*/\");\n    }\n\n    void writeFooterJavadocTagStart(Token token) {\n        // Close any unclosed lists (e.g., <li> without <ul>).\n        // TODO(cpovirk): Actually generate </ul>, etc.?\n        /*\n         * TODO(cpovirk): Also generate </pre> and </table> if appropriate. This is necessary for\n         * idempotency in broken Javadoc. (We don't necessarily need that, but full idempotency may be a\n         * nice goal, especially if it helps us use a fuzzer to test.) Unfortunately, the writer doesn't\n         * currently know which of those tags are open.\n         */\n        continuingListItemOfInnermostList = false;\n        continuingListItemCount.reset();\n        continuingListCount.reset();\n        /*\n         * There's probably no need for this, since its only effect is to disable blank lines in some\n         * cases -- and we're doing that already in the footer.\n         */\n        postWriteModifiedContinuingListCount.reset();\n\n        if (!wroteAnythingSignificant) {\n            // Javadoc consists solely of tags. This is frowned upon in general but OK for @Overrides.\n        } else if (!continuingFooterTag) {\n            // First footer tag after a body tag.\n            requestBlankLine();\n        } else {\n            // Subsequent footer tag.\n            continuingFooterTag = false;\n            requestNewline();\n        }\n        writeToken(token);\n        continuingFooterTag = true;\n    }\n\n    void writeListOpen(Token token) {\n        requestBlankLine();\n\n        writeToken(token);\n        continuingListItemOfInnermostList = false;\n        continuingListCount.increment();\n        postWriteModifiedContinuingListCount.increment();\n\n        requestNewline();\n    }\n\n    void writeListClose(Token token) {\n        requestNewline();\n\n        continuingListItemCount.decrementIfPositive();\n        continuingListCount.decrementIfPositive();\n        writeToken(token);\n        postWriteModifiedContinuingListCount.decrementIfPositive();\n\n        requestBlankLine();\n    }\n\n    void writeListItemOpen(Token token) {\n        requestNewline();\n\n        if (continuingListItemOfInnermostList) {\n            continuingListItemOfInnermostList = false;\n            continuingListItemCount.decrementIfPositive();\n        }\n        writeToken(token);\n        continuingListItemOfInnermostList = true;\n        continuingListItemCount.increment();\n    }\n\n    void writeHeaderOpen(Token token) {\n        requestBlankLine();\n\n        writeToken(token);\n    }\n\n    void writeHeaderClose(Token token) {\n        writeToken(token);\n\n        requestBlankLine();\n    }\n\n    void writeParagraphOpen(Token token) {\n        if (!wroteAnythingSignificant) {\n            /*\n             * The user included an initial <p> tag. Ignore it, and don't request a blank line before the\n             * next token.\n             */\n            return;\n        }\n\n        requestBlankLine();\n\n        writeToken(token);\n    }\n\n    void writeBlockquoteOpenOrClose(Token token) {\n        requestBlankLine();\n\n        writeToken(token);\n\n        requestBlankLine();\n    }\n\n    void writePreOpen(Token token) {\n        requestBlankLine();\n\n        writeToken(token);\n    }\n\n    void writePreClose(Token token) {\n        writeToken(token);\n\n        requestBlankLine();\n    }\n\n    void writeCodeOpen(Token token) {\n        writeToken(token);\n    }\n\n    void writeCodeClose(Token token) {\n        writeToken(token);\n    }\n\n    void writeTableOpen(Token token) {\n        requestBlankLine();\n\n        writeToken(token);\n    }\n\n    void writeTableClose(Token token) {\n        writeToken(token);\n\n        requestBlankLine();\n    }\n\n    void writeMoeEndStripComment(Token token) {\n        writeLineBreakNoAutoIndent();\n        appendSpaces(indentForMoeEndStripComment);\n\n        // Or maybe just \"output.append(token.getValue())?\" I'm kind of surprised this is so easy.\n        writeToken(token);\n\n        requestNewline();\n    }\n\n    void writeHtmlComment(Token token) {\n        requestNewline();\n\n        writeToken(token);\n\n        requestNewline();\n    }\n\n    void writeBr(Token token) {\n        writeToken(token);\n\n        requestNewline();\n    }\n\n    void writeLineBreakNoAutoIndent() {\n        writeNewline(NO_AUTO_INDENT);\n    }\n\n    void writeLiteral(Token token) {\n        writeToken(token);\n    }\n\n    @Override\n    public String toString() {\n        return output.toString();\n    }\n\n    private void requestBlankLine() {\n        requestWhitespace(BLANK_LINE);\n    }\n\n    private void requestNewline() {\n        requestWhitespace(NEWLINE);\n    }\n\n    private void requestWhitespace(RequestedWhitespace requestedWhitespace) {\n        this.requestedWhitespace = Ordering.natural().max(requestedWhitespace, this.requestedWhitespace);\n    }\n\n    /**\n     * The kind of whitespace that has been requested between the previous and next tokens. The order of the values is\n     * significant: It goes from lowest priority to highest. For example, if the previous token requests\n     * {@link #BLANK_LINE} after it but the next token requests only {@link #NEWLINE} before it, we insert\n     * {@link #BLANK_LINE}.\n     */\n    enum RequestedWhitespace {\n        NONE,\n        WHITESPACE,\n        NEWLINE,\n        BLANK_LINE,\n        ;\n    }\n\n    private void writeToken(Token token) {\n        if (requestedMoeBeginStripComment != null) {\n            requestNewline();\n        }\n\n        if (requestedWhitespace == BLANK_LINE\n                && (postWriteModifiedContinuingListCount.isPositive() || continuingFooterTag)) {\n            /*\n             * We don't write blank lines inside lists or footer tags, even in cases where we otherwise\n             * would (e.g., before a <p> tag). Justification: We don't write blank lines _between_ list\n             * items or footer tags, so it would be strange to write blank lines _within_ one. Of course,\n             * an alternative approach would be to go ahead and write blank lines between items/tags,\n             * either always or only in the case that an item contains a blank line.\n             */\n            requestedWhitespace = NEWLINE;\n        }\n\n        if (requestedWhitespace == BLANK_LINE) {\n            writeBlankLine();\n            requestedWhitespace = NONE;\n        } else if (requestedWhitespace == NEWLINE) {\n            writeNewline();\n            requestedWhitespace = NONE;\n        }\n        boolean needWhitespace = (requestedWhitespace == WHITESPACE);\n\n        /*\n         * Write a newline if necessary to respect the line limit. (But if we're at the beginning of the\n         * line, a newline won't help. Or it might help but only by separating \"<p>veryverylongword,\"\n         * which goes against our style.)\n         */\n        if (!atStartOfLine && token.length() + (needWhitespace ? 1 : 0) > remainingOnLine) {\n            writeNewline();\n        }\n        if (!atStartOfLine && needWhitespace) {\n            output.append(\" \");\n            remainingOnLine--;\n        }\n\n        if (requestedMoeBeginStripComment != null) {\n            output.append(requestedMoeBeginStripComment.getValue());\n            requestedMoeBeginStripComment = null;\n            indentForMoeEndStripComment = innerIndent();\n            requestNewline();\n            writeToken(token);\n            return;\n        }\n\n        output.append(token.getValue());\n\n        if (!START_OF_LINE_TOKENS.contains(token.getType())) {\n            atStartOfLine = false;\n        }\n\n        /*\n         * TODO(cpovirk): We really want the number of \"characters,\" not chars. Figure out what the\n         * right way of measuring that is (grapheme count (with BreakIterator?)? sum of widths of all\n         * graphemes? I don't think that our style guide is specific about this.). Moreover, I am\n         * probably brushing other problems with surrogates, etc. under the table. Hopefully I mostly\n         * get away with it by joining all non-space, non-tab characters together.\n         *\n         * Possibly the \"width\" question has no right answer:\n         * http://denisbider.blogspot.com/2015/09/when-monospace-fonts-arent-unicode.html\n         */\n        remainingOnLine -= token.length();\n        requestedWhitespace = NONE;\n        wroteAnythingSignificant = true;\n    }\n\n    private void writeBlankLine() {\n        output.append(\"\\n\");\n        appendSpaces(blockIndent + 1);\n        output.append(\"*\");\n        writeNewline();\n    }\n\n    private void writeNewline() {\n        writeNewline(AUTO_INDENT);\n    }\n\n    private void writeNewline(AutoIndent autoIndent) {\n        output.append(\"\\n\");\n        appendSpaces(blockIndent + 1);\n        output.append(\"*\");\n        appendSpaces(1);\n        remainingOnLine = maxLineLength - blockIndent - 3;\n        if (autoIndent == AUTO_INDENT) {\n            appendSpaces(innerIndent());\n            remainingOnLine -= innerIndent();\n        }\n        atStartOfLine = true;\n    }\n\n    enum AutoIndent {\n        AUTO_INDENT,\n        NO_AUTO_INDENT\n    }\n\n    private int innerIndent() {\n        int innerIndent = continuingListItemCount.value() * 4 + continuingListCount.value() * 2;\n        if (continuingFooterTag) {\n            innerIndent += 4;\n        }\n        return innerIndent;\n    }\n\n    // If this is a hotspot, keep a String of many spaces around, and call append(string, start, end).\n    private void appendSpaces(int count) {\n        output.append(\" \".repeat(count));\n    }\n\n    /**\n     * Tokens that are always pinned to the following token. For example, {@code <p>} in {@code <p>Foo bar} (never\n     * {@code <p> Foo bar} or {@code <p>\\nFoo bar}).\n     *\n     * <p>This is not the only kind of \"pinning\" that we do: See also the joining of LITERAL tokens done by the lexer.\n     * The special pinning here is necessary because these tokens are not of type LITERAL (because they require other\n     * special handling).\n     */\n    private static final ImmutableSet<Token.Type> START_OF_LINE_TOKENS =\n            immutableEnumSet(LIST_ITEM_OPEN_TAG, PARAGRAPH_OPEN_TAG, HEADER_OPEN_TAG);\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/javadoc/NestingCounter.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java.javadoc;\n\n/** Mutable integer for tracking the level of nesting. */\nfinal class NestingCounter {\n    private int value;\n\n    int value() {\n        return value;\n    }\n\n    void increment() {\n        value++;\n    }\n\n    void incrementIfPositive() {\n        if (value > 0) {\n            value++;\n        }\n    }\n\n    void decrementIfPositive() {\n        if (value > 0) {\n            value--;\n        }\n    }\n\n    boolean isPositive() {\n        return value > 0;\n    }\n\n    void reset() {\n        value = 0;\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/javadoc/Token.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java.javadoc;\n\n/**\n * Javadoc token. Our idea of what constitutes a token is often larger or smaller than what you'd naturally expect. The\n * decision is usually pragmatic rather than theoretical. Most of the details are in {@link JavadocLexer}.\n */\nfinal class Token {\n    /**\n     * Javadoc token type.\n     *\n     * <p>The general idea is that every token that requires special handling (extra line breaks, indentation, forcing\n     * or forbidding whitespace) from {@link JavadocWriter} gets its own type. But I haven't been super careful about\n     * it, so I'd imagine that we could merge or remove some of these if we wanted. (For example, PARAGRAPH_CLOSE_TAG\n     * and LIST_ITEM_CLOSE_TAG could share a common IGNORABLE token type. But their corresponding OPEN tags exist, so\n     * I've kept the CLOSE tags.)\n     *\n     * <p>Note, though, that tokens of the same type may still have been handled differently by {@link JavadocLexer}\n     * when it created them. For example, LITERAL is used for both plain text and inline tags, even though the two\n     * affect the lexer's state differently.\n     */\n    enum Type {\n        /** \u2215\u2731\u2731 */\n        BEGIN_JAVADOC,\n        /** \u2731\u2215 */\n        END_JAVADOC,\n        /** The {@code @foo} that begins a block Javadoc tag like {@code @throws}. */\n        FOOTER_JAVADOC_TAG_START,\n        LIST_OPEN_TAG,\n        LIST_CLOSE_TAG,\n        LIST_ITEM_OPEN_TAG,\n        LIST_ITEM_CLOSE_TAG,\n        HEADER_OPEN_TAG,\n        HEADER_CLOSE_TAG,\n        PARAGRAPH_OPEN_TAG,\n        PARAGRAPH_CLOSE_TAG,\n        // TODO(cpovirk): Support <div> (probably identically to <blockquote>).\n        BLOCKQUOTE_OPEN_TAG,\n        BLOCKQUOTE_CLOSE_TAG,\n        PRE_OPEN_TAG,\n        PRE_CLOSE_TAG,\n        CODE_OPEN_TAG,\n        CODE_CLOSE_TAG,\n...\n", "needle_token_start": 8841, "needle_token_end": 8905, "code_context_ntokens": 16104, "output": ["```java\n@Override\npublic Void visitYield(YieldTree node, Void aVoid) {\n    sync(node);\n    token(\"yield\");\n    builder.space();\n    scan(node.getValue(), null);\n    token(\";\");\n    return null;\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "addTrivia", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/ModifierOrderer.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to append additional text, typically non-code syntax or formatting characters, to a string builder during the process of reordering code elements.\n2. **Input**: The function takes two parameters: a string builder that accumulates the final text output and a list of token-like objects representing the additional text to be appended.\n3. **Output**: There is no direct output from this function; instead, it modifies the string builder passed as an input by appending text to it.\n4. **Procedure**: The function iterates over each token-like object in the provided list, extracting the text from each and appending it to the string builder. This process integrates necessary syntax or spacing preserved from the original source code into the reordered output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaInput.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static com.google.common.base.Preconditions.checkNotNull;\nimport static com.google.common.collect.Iterables.getLast;\nimport static java.nio.charset.StandardCharsets.UTF_8;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.base.Suppliers;\nimport com.google.common.base.Verify;\nimport com.google.common.collect.DiscreteDomain;\nimport com.google.common.collect.ImmutableCollection;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableRangeMap;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Iterators;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.java.JavacTokens.RawTok;\nimport com.sun.tools.javac.file.JavacFileManager;\nimport com.sun.tools.javac.parser.Tokens.TokenKind;\nimport com.sun.tools.javac.tree.JCTree.JCCompilationUnit;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Log;\nimport com.sun.tools.javac.util.Log.DeferredDiagnosticHandler;\nimport com.sun.tools.javac.util.Options;\nimport java.io.IOException;\nimport java.net.URI;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.function.Supplier;\nimport javax.tools.Diagnostic;\nimport javax.tools.DiagnosticCollector;\nimport javax.tools.DiagnosticListener;\nimport javax.tools.JavaFileObject;\nimport javax.tools.JavaFileObject.Kind;\nimport javax.tools.SimpleJavaFileObject;\n\n/** {@code JavaInput} extends {@link Input} to represent a Java input document. */\npublic final class JavaInput extends Input {\n    /**\n     * A {@code JavaInput} is a sequence of {@link Tok}s that cover the Java input. A {@link Tok} is either a token (if\n     * {@code isToken()}), or a non-token, which is a comment (if {@code isComment()}) or a newline (if\n     * {@code isNewline()}) or a maximal sequence of other whitespace characters (if {@code isSpaces()}). Each\n     * {@link Tok} contains a sequence of characters, an index (sequential starting at {@code 0} for tokens and\n     * comments, else {@code -1}), and a ({@code 0}-origin) position in the input. The concatenation of the texts of all\n     * the {@link Tok}s equals the input. Each Input ends with a token EOF {@link Tok}, with empty text.\n     *\n     * <p>A {@code /*} comment possibly contains newlines; a {@code //} comment does not contain the terminating newline\n     * character, but is followed by a newline {@link Tok}.\n     */\n    @Immutable\n    static final class Tok implements Input.Tok {\n        private final int index;\n        private final String originalText;\n        private final String text;\n        private final int position;\n        private final int columnI;\n        private final boolean isToken;\n        private final TokenKind kind;\n\n        /**\n         * The {@code Tok} constructor.\n         *\n         * @param index its index\n         * @param originalText its original text, before removing Unicode escapes\n         * @param text its text after removing Unicode escapes\n         * @param position its {@code 0}-origin position in the input\n         * @param columnI its {@code 0}-origin column number in the input\n         * @param isToken whether the {@code Tok} is a token\n         * @param kind the token kind\n         */\n        Tok(int index, String originalText, String text, int position, int columnI, boolean isToken, TokenKind kind) {\n            this.index = index;\n            this.originalText = originalText;\n            this.text = text;\n            this.position = position;\n            this.columnI = columnI;\n            this.isToken = isToken;\n            this.kind = kind;\n        }\n\n        @Override\n        public int getIndex() {\n            return index;\n        }\n\n        @Override\n        public String getText() {\n            return text;\n        }\n\n        @Override\n        public String getOriginalText() {\n            return originalText;\n        }\n\n        @Override\n        public int length() {\n            return originalText.length();\n        }\n\n        @Override\n        public int getPosition() {\n            return position;\n        }\n\n        @Override\n        public int getColumn() {\n            return columnI;\n        }\n\n        boolean isToken() {\n            return isToken;\n        }\n\n        @Override\n        public boolean isNewline() {\n            return Newlines.isNewline(text);\n        }\n\n        @Override\n        public boolean isSlashSlashComment() {\n            return text.startsWith(\"//\");\n        }\n\n        @Override\n        public boolean isSlashStarComment() {\n            return text.startsWith(\"/*\");\n        }\n\n        @Override\n        public boolean isJavadocComment() {\n            return text.startsWith(\"/**\") && text.length() > 4;\n        }\n\n        @Override\n        public boolean isComment() {\n            return isSlashSlashComment() || isSlashStarComment();\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this)\n                    .add(\"index\", index)\n                    .add(\"text\", text)\n                    .add(\"position\", position)\n                    .add(\"columnI\", columnI)\n                    .add(\"isToken\", isToken)\n                    .toString();\n        }\n\n        public TokenKind kind() {\n            return kind;\n        }\n    }\n\n    /**\n     * A {@link Token} contains a token {@link Tok} and its associated non-tokens; each non-token {@link Tok} belongs to\n     * one {@link Token}. Each {@link Token} has an immutable list of its non-tokens that appear before it, and another\n     * list of its non-tokens that appear after it. The concatenation of the texts of all the {@link Token}s'\n     * {@link Tok}s, each preceded by the texts of its {@code toksBefore} and followed by the texts of its\n     * {@code toksAfter}, equals the input.\n     */\n    @Immutable\n    static final class Token implements Input.Token {\n        private final Tok tok;\n        private final ImmutableList<Tok> toksBefore;\n        private final ImmutableList<Tok> toksAfter;\n\n        /**\n         * Token constructor.\n         *\n         * @param toksBefore the earlier non-token {link Tok}s assigned to this {@code Token}\n         * @param tok this token {@link Tok}\n         * @param toksAfter the later non-token {link Tok}s assigned to this {@code Token}\n         */\n        Token(List<Tok> toksBefore, Tok tok, List<Tok> toksAfter) {\n            this.toksBefore = ImmutableList.copyOf(toksBefore);\n            this.tok = tok;\n            this.toksAfter = ImmutableList.copyOf(toksAfter);\n        }\n\n        /**\n         * Get the token's {@link Tok}.\n         *\n         * @return the token's {@link Tok}\n         */\n        @Override\n        public Tok getTok() {\n            return tok;\n        }\n\n        /**\n         * Get the earlier {@link Tok}s assigned to this {@code Token}.\n         *\n         * @return the earlier {@link Tok}s assigned to this {@code Token}\n         */\n        @Override\n        public ImmutableList<? extends Input.Tok> getToksBefore() {\n            return toksBefore;\n        }\n\n        /**\n         * Get the later {@link Tok}s assigned to this {@code Token}.\n         *\n         * @return the later {@link Tok}s assigned to this {@code Token}\n         */\n        @Override\n        public ImmutableList<? extends Input.Tok> getToksAfter() {\n            return toksAfter;\n        }\n\n        @Override\n        public String toString() {\n            return MoreObjects.toStringHelper(this)\n                    .add(\"tok\", tok)\n                    .add(\"toksBefore\", toksBefore)\n                    .add(\"toksAfter\", toksAfter)\n                    .toString();\n        }\n    }\n\n    private final String text; // The input.\n    private final Supplier<String> lineSeparator = Suppliers.memoize(() -> Newlines.guessLineSeparator(getText()));\n    private int kN; // The number of numbered toks (tokens or comments), excluding the EOF.\n\n    /*\n     * The following lists record the sequential indices of the {@code Tok}s on each input line. (Only\n     * tokens and comments have sequential indices.) Tokens and {@code //} comments lie on just one\n     * line; {@code /*} comments can lie on multiple lines. These data structures (along with\n     * equivalent ones for the formatted output) let us compute correspondences between the input and\n     * output.\n     */\n\n    private final ImmutableMap<Integer, Integer> positionToColumnMap; // Map Tok position to column.\n    private final ImmutableList<Token> tokens; // The Tokens for this input.\n    private final ImmutableRangeMap<Integer, Token> positionTokenMap; // Map position to Token.\n\n    /** Map from Tok index to the associated Token. */\n    private final Token[] kToToken;\n\n    /**\n     * Input constructor.\n     *\n     * @param text the input text\n     * @throws FormatterException if the input cannot be parsed\n     */\n    public JavaInput(String text) throws FormatterException {\n        this.text = checkNotNull(text);\n        setLines(ImmutableList.copyOf(Newlines.lineIterator(text)));\n        ImmutableList<Tok> toks = buildToks(text);\n        positionToColumnMap = makePositionToColumnMap(toks);\n        tokens = buildTokens(toks);\n        ImmutableRangeMap.Builder<Integer, Token> tokenLocations = ImmutableRangeMap.builder();\n        for (Token token : tokens) {\n            Input.Tok end = JavaOutput.endTok(token);\n            int upper = end.getPosition();\n            if (!end.getText().isEmpty()) {\n                upper += end.length() - 1;\n            }\n            tokenLocations.put(Range.closed(JavaOutput.startTok(token).getPosition(), upper), token);\n        }\n        positionTokenMap = tokenLocations.build();\n\n        // adjust kN for EOF\n        kToToken = new Token[kN + 1];\n        for (Token token : tokens) {\n            for (Input.Tok tok : token.getToksBefore()) {\n                if (tok.getIndex() < 0) {\n                    continue;\n                }\n                kToToken[tok.getIndex()] = token;\n            }\n            kToToken[token.getTok().getIndex()] = token;\n            for (Input.Tok tok : token.getToksAfter()) {\n                if (tok.getIndex() < 0) {\n                    continue;\n                }\n                kToToken[tok.getIndex()] = token;\n            }\n        }\n    }\n\n    private static ImmutableMap<Integer, Integer> makePositionToColumnMap(List<Tok> toks) {\n        ImmutableMap.Builder<Integer, Integer> builder = ImmutableMap.builder();\n        for (Tok tok : toks) {\n            builder.put(tok.getPosition(), tok.getColumn());\n        }\n        return builder.buildOrThrow();\n    }\n\n    /**\n     * Get the input text.\n     *\n     * @return the input text\n     */\n    @Override\n    public String getText() {\n        return text;\n    }\n\n    @Override\n    public ImmutableMap<Integer, Integer> getPositionToColumnMap() {\n        return positionToColumnMap;\n    }\n\n    public String getLineSeparator() {\n        return lineSeparator.get();\n    }\n\n    /** Lex the input and build the list of toks. */\n    private ImmutableList<Tok> buildToks(String text) throws FormatterException {\n        ImmutableList<Tok> toks = buildToks(text, ImmutableSet.of());\n        kN = getLast(toks).getIndex();\n        computeRanges(toks);\n        return toks;\n    }\n\n    /**\n     * Lex the input and build the list of toks.\n     *\n     * @param text the text to be lexed.\n     * @param stopTokens a set of tokens which should cause lexing to stop. If one of these is found, the returned list\n     *     will include tokens up to but not including that token.\n     */\n    static ImmutableList<Tok> buildToks(String text, ImmutableSet<TokenKind> stopTokens) throws FormatterException {\n        stopTokens = ImmutableSet.<TokenKind>builder()\n                .addAll(stopTokens)\n                .add(TokenKind.EOF)\n                .build();\n        Context context = new Context();\n        Options.instance(context).put(\"--enable-preview\", \"true\");\n        new JavacFileManager(context, true, UTF_8);\n        DiagnosticCollector<JavaFileObject> diagnosticCollector = new DiagnosticCollector<>();\n        context.put(DiagnosticListener.class, diagnosticCollector);\n        Log log = Log.instance(context);\n        log.useSource(new SimpleJavaFileObject(URI.create(\"Source.java\"), Kind.SOURCE) {\n            @Override\n            public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {\n                return text;\n            }\n        });\n        DeferredDiagnosticHandler diagnostics = new DeferredDiagnosticHandler(log);\n        ImmutableList<RawTok> rawToks = JavacTokens.getTokens(text, context, stopTokens);\n        if (diagnostics.getDiagnostics().stream().anyMatch(d -> d.getKind() == Diagnostic.Kind.ERROR)) {\n            return ImmutableList.of(new Tok(0, \"\", \"\", 0, 0, true, null)); // EOF\n        }\n        int kN = 0;\n        List<Tok> toks = new ArrayList<>();\n        int charI = 0;\n        int columnI = 0;\n        for (RawTok t : rawToks) {\n            if (stopTokens.contains(t.kind())) {\n                break;\n            }\n            int charI0 = t.pos();\n            // Get string, possibly with Unicode escapes.\n            String originalTokText = text.substring(charI0, t.endPos());\n            String tokText = t.kind() == TokenKind.STRINGLITERAL\n                    ? t.stringVal() // Unicode escapes removed.\n                    : originalTokText;\n            char tokText0 = tokText.charAt(0); // The token's first character.\n            final boolean isToken; // Is this tok a token?\n            final boolean isNumbered; // Is this tok numbered? (tokens and comments)\n            String extraNewline = null; // Extra newline at end?\n            List<String> strings = new ArrayList<>();\n            if (Character.isWhitespace(tokText0)) {\n                isToken = false;\n                isNumbered = false;\n                Iterator<String> it = Newlines.lineIterator(originalTokText);\n                while (it.hasNext()) {\n                    String line = it.next();\n                    String newline = Newlines.getLineEnding(line);\n                    if (newline != null) {\n                        String spaces = line.substring(0, line.length() - newline.length());\n                        if (!spaces.isEmpty()) {\n                            strings.add(spaces);\n                        }\n...\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/ImportOrderer.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\npackage com.palantir.javaformat.java;\n\nimport static com.google.common.collect.Iterables.getLast;\nimport static com.google.common.primitives.Booleans.trueFirst;\nimport static com.palantir.javaformat.java.JavaFormatterOptions.Style;\nimport static com.palantir.javaformat.java.JavaInput.buildToks;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.base.Preconditions;\nimport com.google.common.base.Splitter;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.ImmutableSortedSet;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.java.JavaInput.Tok;\nimport com.sun.tools.javac.parser.Tokens.TokenKind;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.function.BiFunction;\nimport java.util.stream.Stream;\n\n/** Orders imports in Java source code. */\npublic final class ImportOrderer {\n\n    private static final Splitter DOT_SPLITTER = Splitter.on('.');\n\n    /**\n     * Reorder the inputs in {@code text}, a complete Java program. On success, another complete Java program is\n     * returned, which is the same as the original except the imports are in order.\n     *\n     * @throws FormatterException if the input could not be parsed.\n     */\n    public static String reorderImports(String text, Style style) throws FormatterException {\n        ImmutableList<Tok> toks = buildToks(text, CLASS_START);\n        return new ImportOrderer(text, toks, style).reorderImports();\n    }\n\n    /**\n     * Reorder the inputs in {@code text}, a complete Java program, in Google style. On success, another complete Java\n     * program is returned, which is the same as the original except the imports are in order.\n     *\n     * @deprecated Use {@link #reorderImports(String, Style)} instead\n     * @throws FormatterException if the input could not be parsed.\n     */\n    @Deprecated\n    public static String reorderImports(String text) throws FormatterException {\n        return reorderImports(text, Style.GOOGLE);\n    }\n\n    private String reorderImports() throws FormatterException {\n        int firstImportStart;\n        Optional<Integer> maybeFirstImport = findIdentifier(0, IMPORT_OR_CLASS_START);\n        if (!maybeFirstImport.isPresent() || !tokenAt(maybeFirstImport.get()).equals(\"import\")) {\n            // No imports, so nothing to do.\n            return text;\n        }\n        firstImportStart = maybeFirstImport.get();\n        int unindentedFirstImportStart = unindent(firstImportStart);\n\n        ImportsAndIndex imports = scanImports(firstImportStart);\n        int afterLastImport = imports.index;\n\n        // Make sure there are no more imports before the next class (etc) definition.\n        Optional<Integer> maybeLaterImport = findIdentifier(afterLastImport, IMPORT_OR_CLASS_START);\n        if (maybeLaterImport.isPresent() && tokenAt(maybeLaterImport.get()).equals(\"import\")) {\n            throw new FormatterException(\"Imports not contiguous (perhaps a comment separates them?)\");\n        }\n\n        StringBuilder result = new StringBuilder();\n        String prefix = tokString(0, unindentedFirstImportStart);\n        result.append(prefix);\n        if (!prefix.isEmpty() && Newlines.getLineEnding(prefix) == null) {\n            result.append(lineSeparator).append(lineSeparator);\n        }\n        result.append(reorderedImportsString(imports.imports));\n\n        List<String> tail = new ArrayList<>();\n        tail.add(CharMatcher.whitespace().trimLeadingFrom(tokString(afterLastImport, toks.size())));\n        if (!toks.isEmpty()) {\n            Tok lastTok = getLast(toks);\n            int tailStart = lastTok.getPosition() + lastTok.length();\n            tail.add(text.substring(tailStart));\n        }\n        if (tail.stream().anyMatch(s -> !s.isEmpty())) {\n            result.append(lineSeparator);\n            tail.forEach(result::append);\n        }\n\n        return result.toString();\n    }\n\n    /**\n     * {@link TokenKind}s that indicate the start of a type definition. We use this to avoid scanning the whole file,\n     * since we know that imports must precede any type definition.\n     */\n    private static final ImmutableSet<TokenKind> CLASS_START =\n            ImmutableSet.of(TokenKind.CLASS, TokenKind.INTERFACE, TokenKind.ENUM);\n\n    /**\n     * We use this set to find the first import, and again to check that there are no imports after the place we stopped\n     * gathering them. An annotation definition ({@code @interface}) is two tokens, the second which is\n     * {@code interface}, so we don't need a separate entry for that.\n     */\n    private static final ImmutableSet<String> IMPORT_OR_CLASS_START =\n            ImmutableSet.of(\"import\", \"class\", \"interface\", \"enum\");\n\n    /**\n     * A {@link Comparator} that orders {@link Import}s by Google Style, defined at\n     * https://google.github.io/styleguide/javaguide.html#s3.3.3-import-ordering-and-spacing.\n     */\n    private static final Comparator<Import> GOOGLE_IMPORT_COMPARATOR =\n            Comparator.comparing(Import::isStatic, trueFirst()).thenComparing(Import::imported);\n\n    /**\n     * A {@link Comparator} that orders {@link Import}s by AOSP Style, defined at\n     * https://source.android.com/setup/contribute/code-style#order-import-statements and implemented in IntelliJ at\n     * https://android.googlesource.com/platform/development/+/master/ide/intellij/codestyles/AndroidStyle.xml.\n     */\n    private static final Comparator<Import> AOSP_IMPORT_COMPARATOR = Comparator.comparing(Import::isStatic, trueFirst())\n            .thenComparing(Import::isAndroid, trueFirst())\n            .thenComparing(Import::isThirdParty, trueFirst())\n            .thenComparing(Import::isJava, trueFirst())\n            .thenComparing(Import::imported);\n\n    /**\n     * Determines whether to insert a blank line between the {@code prev} and {@code curr} {@link Import}s based on\n     * Google style.\n     */\n    private static boolean shouldInsertBlankLineGoogle(Import prev, Import curr) {\n        return prev.isStatic() && !curr.isStatic();\n    }\n\n    /**\n     * Determines whether to insert a blank line between the {@code prev} and {@code curr} {@link Import}s based on AOSP\n     * style.\n     */\n    private static boolean shouldInsertBlankLineAosp(Import prev, Import curr) {\n        if (prev.isStatic() && !curr.isStatic()) {\n            return true;\n        }\n        // insert blank line between \"com.android\" from \"com.anythingelse\"\n        if (prev.isAndroid() && !curr.isAndroid()) {\n            return true;\n        }\n        return !prev.topLevel().equals(curr.topLevel());\n    }\n\n    private final String text;\n    private final ImmutableList<Tok> toks;\n    private final String lineSeparator;\n    private final Comparator<Import> importComparator;\n    private final BiFunction<Import, Import, Boolean> shouldInsertBlankLineFn;\n\n    private ImportOrderer(String text, ImmutableList<Tok> toks, Style style) {\n        this.text = text;\n        this.toks = toks;\n        this.lineSeparator = Newlines.guessLineSeparator(text);\n        if (style.equals(Style.GOOGLE) || style.equals(Style.PALANTIR)) {\n            this.importComparator = GOOGLE_IMPORT_COMPARATOR;\n            this.shouldInsertBlankLineFn = ImportOrderer::shouldInsertBlankLineGoogle;\n        } else if (style.equals(Style.AOSP)) {\n            this.importComparator = AOSP_IMPORT_COMPARATOR;\n            this.shouldInsertBlankLineFn = ImportOrderer::shouldInsertBlankLineAosp;\n        } else {\n            throw new IllegalArgumentException(\"Unsupported code style: \" + style);\n        }\n    }\n\n    /** An import statement. */\n    class Import {\n        private final String imported;\n        private final boolean isStatic;\n        private final String trailing;\n\n        Import(String imported, String trailing, boolean isStatic) {\n            this.imported = imported;\n            this.trailing = trailing;\n            this.isStatic = isStatic;\n        }\n\n        /** The name being imported, for example {@code java.util.List}. */\n        String imported() {\n            return imported;\n        }\n\n        /** True if this is {@code import static}. */\n        boolean isStatic() {\n            return isStatic;\n        }\n\n        /** The top-level package of the import. */\n        String topLevel() {\n            return DOT_SPLITTER.split(imported()).iterator().next();\n        }\n\n        /** True if this is an Android import per AOSP style. */\n        boolean isAndroid() {\n            return Stream.of(\"android.\", \"androidx.\", \"dalvik.\", \"libcore.\", \"com.android.\")\n                    .anyMatch(imported::startsWith);\n        }\n\n        /** True if this is a Java import per AOSP style. */\n        boolean isJava() {\n            switch (topLevel()) {\n                case \"java\":\n                case \"javax\":\n                    return true;\n                default:\n                    return false;\n            }\n        }\n\n        /**\n         * The {@code //} comment lines after the final {@code ;}, up to and including the line terminator of the last\n         * one. Note: In case two imports were separated by a space (which is disallowed by the style guide), the\n         * trailing whitespace of the first import does not include a line terminator.\n         */\n        String trailing() {\n            return trailing;\n        }\n\n        /** True if this is a third-party import per AOSP style. */\n        public boolean isThirdParty() {\n            return !(isAndroid() || isJava());\n        }\n\n        // One or multiple lines, the import itself and following comments, including the line\n        // terminator.\n        @Override\n        public String toString() {\n            StringBuilder sb = new StringBuilder();\n            sb.append(\"import \");\n            if (isStatic()) {\n                sb.append(\"static \");\n            }\n            sb.append(imported()).append(';');\n            if (trailing().trim().isEmpty()) {\n                sb.append(lineSeparator);\n            } else {\n                sb.append(trailing());\n            }\n            return sb.toString();\n        }\n    }\n\n    private String tokString(int start, int end) {\n        StringBuilder sb = new StringBuilder();\n        for (int i = start; i < end; i++) {\n            sb.append(toks.get(i).getOriginalText());\n        }\n        return sb.toString();\n    }\n\n    private static class ImportsAndIndex {\n        final ImmutableSortedSet<Import> imports;\n        final int index;\n\n        ImportsAndIndex(ImmutableSortedSet<Import> imports, int index) {\n            this.imports = imports;\n            this.index = index;\n        }\n    }\n\n    /**\n     * Scans a sequence of import lines. The parsing uses this approximate grammar:\n     *\n     * <pre>{@code\n     * <imports> -> (<end-of-line> | <import>)*\n     * <import> -> \"import\" <whitespace> (\"static\" <whitespace>)?\n     *    <identifier> (\".\" <identifier>)* (\".\" \"*\")? <whitespace>? \";\"\n     *    <whitespace>? <end-of-line>? (<line-comment> <end-of-line>)*\n     * }</pre>\n     *\n     * @param i the index to start parsing at.\n     * @return the result of parsing the imports.\n     * @throws FormatterException if imports could not parsed according to the grammar.\n     */\n    private ImportsAndIndex scanImports(int i) throws FormatterException {\n        int afterLastImport = i;\n        ImmutableSortedSet.Builder<Import> imports = ImmutableSortedSet.orderedBy(importComparator);\n        // JavaInput.buildToks appends a zero-width EOF token after all tokens. It won't match any\n        // of our tests here and protects us from running off the end of the toks list. Since it is\n        // zero-width it doesn't matter if we include it in our string concatenation at the end.\n        while (i < toks.size() && tokenAt(i).equals(\"import\")) {\n            i++;\n            if (isSpaceToken(i)) {\n                i++;\n            }\n            boolean isStatic = tokenAt(i).equals(\"static\");\n            if (isStatic) {\n                i++;\n                if (isSpaceToken(i)) {\n                    i++;\n                }\n            }\n            if (!isIdentifierToken(i)) {\n                throw new FormatterException(\"Unexpected token after import: \" + tokenAt(i));\n            }\n            StringAndIndex imported = scanImported(i);\n            String importedName = imported.string;\n            i = imported.index;\n            if (isSpaceToken(i)) {\n                i++;\n            }\n            if (!tokenAt(i).equals(\";\")) {\n                throw new FormatterException(\"Expected ; after import\");\n            }\n            while (tokenAt(i).equals(\";\")) {\n                // Extra semicolons are not allowed by the JLS but are accepted by javac.\n                i++;\n            }\n            StringBuilder trailing = new StringBuilder();\n            if (isSpaceToken(i)) {\n                trailing.append(tokenAt(i));\n                i++;\n            }\n            if (isNewlineToken(i)) {\n                trailing.append(tokenAt(i));\n                i++;\n            }\n            // Gather (if any) all single line comments and accompanied line terminators following this\n            // import\n            while (isSlashSlashCommentToken(i)) {\n                trailing.append(tokenAt(i));\n                i++;\n                if (isNewlineToken(i)) {\n                    trailing.append(tokenAt(i));\n                    i++;\n                }\n            }\n            imports.add(new Import(importedName, trailing.toString(), isStatic));\n            // Remember the position just after the import we just saw, before skipping blank lines.\n            // If the next thing after the blank lines is not another import then we don't want to\n            // include those blank lines in the text to be replaced.\n            afterLastImport = i;\n            while (isNewlineToken(i) || isSpaceToken(i)) {\n                i++;\n            }\n        }\n        return new ImportsAndIndex(imports.build(), afterLastImport);\n    }\n\n    // Produces the sorted output based on the imports we have scanned.\n    private String reorderedImportsString(ImmutableSortedSet<Import> imports) {\n        Preconditions.checkArgument(!imports.isEmpty(), \"imports\");\n\n        // Pretend that the first import was preceded by another import of the same kind, so we don't\n        // insert a newline there.\n        Import prevImport = imports.iterator().next();\n\n        StringBuilder sb = new StringBuilder();\n        for (Import currImport : imports) {\n            if (shouldInsertBlankLineFn.apply(prevImport, currImport)) {\n                // Blank line between static and non-static imports.\n                sb.append(lineSeparator);\n            }\n            sb.append(currImport);\n            prevImport = currImport;\n        }\n        return sb.toString();\n    }\n\n    private static class StringAndIndex {\n        private final String string;\n        private final int index;\n\n        StringAndIndex(String string, int index) {\n            this.string = string;\n            this.index = index;\n        }\n    }\n\n    /**\n     * Scans the imported thing, the dot-separated name that comes after import [static] and before the semicolon. We\n     * don't allow spaces inside the dot-separated name. Wildcard imports are supported: if the input is {@code import\n     * java.util.*;} then the returned string will be {@code java.util.*}.\n     *\n     * @param start the index of the start of the identifier. If the import is {@code import java.util.List;} then this\n     *     index points to the token {@code java}.\n     * @return the parsed import ({@code java.util.List} in the example) and the index of the first token after the\n     *     imported thing ({@code ;} in the example).\n     * @throws FormatterException if the imported name could not be parsed.\n     */\n    private StringAndIndex scanImported(int start) throws FormatterException {\n        int i = start;\n        StringBuilder imported = new StringBuilder();\n        // At the start of each iteration of this loop, i points to an identifier.\n        // On exit from the loop, i points to a token after an identifier or after *.\n        while (true) {\n            Preconditions.checkState(isIdentifierToken(i));\n            imported.append(tokenAt(i));\n            i++;\n            if (!tokenAt(i).equals(\".\")) {\n                return new StringAndIndex(imported.toString(), i);\n            }\n            imported.append('.');\n            i++;\n            if (tokenAt(i).equals(\"*\")) {\n                imported.append('*');\n                return new StringAndIndex(imported.toString(), i + 1);\n            } else if (!isIdentifierToken(i)) {\n                throw new FormatterException(\"Could not parse imported name, at: \" + tokenAt(i));\n            }\n        }\n    }\n\n    /**\n     * Returns the index of the first place where one of the given identifiers occurs, or {@code Optional.empty()} if\n     * there is none.\n     *\n     * @param start the index to start looking at\n     * @param identifiers the identifiers to look for\n     */\n    private Optional<Integer> findIdentifier(int start, ImmutableSet<String> identifiers) {\n        for (int i = start; i < toks.size(); i++) {\n            if (isIdentifierToken(i)) {\n                String id = tokenAt(i);\n                if (identifiers.contains(id)) {\n                    return Optional.of(i);\n                }\n            }\n        }\n        return Optional.empty();\n    }\n\n    /** Returns the given token, or the preceding token if it is a whitespace token. */\n    private int unindent(int i) {\n        if (i > 0 && isSpaceToken(i - 1)) {\n            return i - 1;\n        } else {\n            return i;\n        }\n    }\n\n    private String tokenAt(int i) {\n        return toks.get(i).getOriginalText();\n    }\n\n    private boolean isIdentifierToken(int i) {\n        String s = tokenAt(i);\n        return !s.isEmpty() && Character.isJavaIdentifierStart(s.codePointAt(0));\n    }\n\n    private boolean isSpaceToken(int i) {\n        String s = tokenAt(i);\n        if (s.isEmpty()) {\n            return false;\n        } else {\n            return \" \\t\\f\".indexOf(s.codePointAt(0)) >= 0;\n        }\n    }\n\n    private boolean isSlashSlashCommentToken(int i) {\n        return toks.get(i).isSlashSlashComment();\n    }\n\n    private boolean isNewlineToken(int i) {\n        return toks.get(i).isNewline();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/ModifierOrderer.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy\n * of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations\n * under the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Ordering;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeMap;\nimport com.palantir.javaformat.Input.Tok;\nimport com.palantir.javaformat.Input.Token;\nimport com.sun.tools.javac.parser.Tokens.TokenKind;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport javax.lang.model.element.Modifier;\n\n/** Fixes sequences of modifiers to be in JLS order. */\nfinal class ModifierOrderer {\n\n    /** Reorders all modifiers in the given text to be in JLS order. */\n    static JavaInput reorderModifiers(String text) throws FormatterException {\n        return reorderModifiers(new JavaInput(text), ImmutableList.of(Range.closedOpen(0, text.length())));\n    }\n\n    /** Reorders all modifiers in the given text and within the given character ranges to be in JLS order. */\n    static JavaInput reorderModifiers(JavaInput javaInput, Collection<Range<Integer>> characterRanges)\n            throws FormatterException {\n        if (javaInput.getTokens().isEmpty()) {\n            // There weren't any tokens, possible because of a lexing error.\n            // Errors about invalid input will be reported later after parsing.\n            return javaInput;\n        }\n        RangeSet<Integer> tokenRanges = javaInput.characterRangesToTokenRanges(characterRanges);\n        Iterator<? extends Token> it = javaInput.getTokens().iterator();\n        TreeRangeMap<Integer, String> replacements = TreeRangeMap.create();\n        while (it.hasNext()) {\n            Token token = it.next();\n            if (!tokenRanges.contains(token.getTok().getIndex())) {\n                continue;\n            }\n            Modifier mod = asModifier(token);\n            if (mod == null) {\n                continue;\n            }\n\n            List<Token> modifierTokens = new ArrayList<>();\n            List<Modifier> mods = new ArrayList<>();\n\n            int begin = token.getTok().getPosition();\n            mods.add(mod);\n            modifierTokens.add(token);\n\n            int end = -1;\n            while (it.hasNext()) {\n                token = it.next();\n                mod = asModifier(token);\n                if (mod == null) {\n                    break;\n                }\n                mods.add(mod);\n                modifierTokens.add(token);\n                end = token.getTok().getPosition() + token.getTok().length();\n            }\n\n            if (!Ordering.natural().isOrdered(mods)) {\n                Collections.sort(mods);\n                StringBuilder replacement = new StringBuilder();\n                for (int i = 0; i < mods.size(); i++) {\n                    if (i > 0) {\n                        addTrivia(replacement, modifierTokens.get(i).getToksBefore());\n                    }\n                    replacement.append(mods.get(i).toString());\n                    if (i < (modifierTokens.size() - 1)) {\n                        addTrivia(replacement, modifierTokens.get(i).getToksAfter());\n                    }\n                }\n                replacements.put(Range.closedOpen(begin, end), replacement.toString());\n            }\n        }\n        return applyReplacements(javaInput, replacements);\n    }\n\n    \nprivate static void addTrivia(StringBuilder replacement, ImmutableList<? extends Tok> toks) {\n        for (Tok tok : toks) {\n            replacement.append(tok.getText());\n        }\n    }\n\n    /**\n     * Returns the given token as a {@link javax.lang.model.element.Modifier}, or {@code null} if it is not a modifier.\n     */\n    private static Modifier asModifier(Token token) {\n        TokenKind kind = ((JavaInput.Tok) token.getTok()).kind();\n        if (kind != null) {\n            switch (kind) {\n                case PUBLIC:\n                    return Modifier.PUBLIC;\n                case PROTECTED:\n                    return Modifier.PROTECTED;\n                case PRIVATE:\n                    return Modifier.PRIVATE;\n                case ABSTRACT:\n                    return Modifier.ABSTRACT;\n                case STATIC:\n                    return Modifier.STATIC;\n                case DEFAULT:\n                    return Modifier.DEFAULT;\n                case FINAL:\n                    return Modifier.FINAL;\n                case TRANSIENT:\n                    return Modifier.TRANSIENT;\n                case VOLATILE:\n                    return Modifier.VOLATILE;\n                case SYNCHRONIZED:\n                    return Modifier.SYNCHRONIZED;\n                case NATIVE:\n                    return Modifier.NATIVE;\n                case STRICTFP:\n                    return Modifier.STRICTFP;\n                default: // fall out\n            }\n        }\n        switch (token.getTok().getText()) {\n            case \"non-sealed\":\n                return Modifier.valueOf(\"NON_SEALED\");\n            case \"sealed\":\n                return Modifier.valueOf(\"SEALED\");\n            default:\n                return null;\n        }\n    }\n\n    /** Applies replacements to the given string. */\n    private static JavaInput applyReplacements(JavaInput javaInput, TreeRangeMap<Integer, String> replacementMap)\n            throws FormatterException {\n        // process in descending order so the replacement ranges aren't perturbed if any replacements\n        // differ in size from the input\n        Map<Range<Integer>, String> ranges = replacementMap.asDescendingMapOfRanges();\n        if (ranges.isEmpty()) {\n            return javaInput;\n        }\n        StringBuilder sb = new StringBuilder(javaInput.getText());\n        for (Map.Entry<Range<Integer>, String> entry : ranges.entrySet()) {\n            Range<Integer> range = entry.getKey();\n            sb.replace(range.lowerEndpoint(), range.upperEndpoint(), entry.getValue());\n        }\n        return new JavaInput(sb.toString());\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/RemoveUnusedImports.java\n/*\n * Copyright 2016 Google Inc. All Rights Reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.collect.HashMultimap;\nimport com.google.common.collect.Multimap;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeMap;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeMap;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Newlines;\nimport com.sun.source.doctree.DocCommentTree;\nimport com.sun.source.doctree.ReferenceTree;\nimport com.sun.source.tree.CaseTree;\nimport com.sun.source.tree.IdentifierTree;\nimport com.sun.source.tree.ImportTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.source.util.DocTreePath;\nimport com.sun.source.util.DocTreePathScanner;\nimport com.sun.source.util.TreePathScanner;\nimport com.sun.source.util.TreeScanner;\nimport com.sun.tools.javac.api.JavacTrees;\nimport com.sun.tools.javac.tree.DCTree;\nimport com.sun.tools.javac.tree.DCTree.DCReference;\nimport com.sun.tools.javac.tree.JCTree;\nimport com.sun.tools.javac.tree.JCTree.JCCompilationUnit;\nimport com.sun.tools.javac.tree.JCTree.JCFieldAccess;\nimport com.sun.tools.javac.tree.JCTree.JCIdent;\nimport com.sun.tools.javac.tree.JCTree.JCImport;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Options;\nimport java.lang.reflect.Method;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\n/**\n * Removes unused imports from a source file. Imports that are only used in javadoc are also removed, and the references\n * in javadoc are replaced with fully qualified names.\n */\npublic class RemoveUnusedImports {\n\n    // Visits an AST, recording all simple names that could refer to imported\n    // types and also any javadoc references that could refer to imported\n    // types (`@link`, `@see`, `@throws`, etc.)\n    //\n    // No attempt is made to determine whether simple names occur in contexts\n    // where they are type names, so there will be false positives. For example,\n    // `List` is not identified as unused import below:\n    //\n    // ```\n    // import java.util.List;\n    // class List {}\n    // ```\n    //\n    // This is still reasonably effective in practice because type names differ\n    // from other kinds of names in casing convention, and simple name\n    // clashes between imported and declared types are rare.\n    private static final class UnusedImportScanner extends TreePathScanner<Void, Void> {\n\n        private final Set<String> usedNames = new LinkedHashSet<>();\n        private final Multimap<String, Range<Integer>> usedInJavadoc = HashMultimap.create();\n        final JavacTrees trees;\n        final DocTreeScanner docTreeSymbolScanner;\n\n        private UnusedImportScanner(JavacTrees trees) {\n            this.trees = trees;\n            docTreeSymbolScanner = new DocTreeScanner();\n        }\n\n        /** Skip the imports themselves when checking for usage. */\n        @Override\n        public Void visitImport(ImportTree importTree, Void usedSymbols) {\n            return null;\n        }\n\n        @Override\n        public Void visitIdentifier(IdentifierTree tree, Void unused) {\n            if (tree == null) {\n                return null;\n            }\n            usedNames.add(tree.getName().toString());\n            return null;\n        }\n\n        // TODO(fwindheuser): remove this override when pattern matching in switch is no longer a preview\n        // feature, and TreePathScanner visits CaseTree#getLabels instead of CaseTree#getExpressions\n        @SuppressWarnings(\"unchecked\") // reflection\n        @Override\n        public Void visitCase(CaseTree tree, Void unused) {\n            if (CASE_TREE_GET_LABELS != null) {\n                try {\n                    scan((List<? extends Tree>) CASE_TREE_GET_LABELS.invoke(tree), null);\n                } catch (ReflectiveOperationException e) {\n                    throw new RuntimeException(e.getMessage(), e);\n                }\n            }\n            return super.visitCase(tree, null);\n        }\n\n        private static final Method CASE_TREE_GET_LABELS = caseTreeGetLabels();\n\n        private static Method caseTreeGetLabels() {\n            try {\n                return CaseTree.class.getMethod(\"getLabels\");\n            } catch (NoSuchMethodException e) {\n                return null;\n            }\n        }\n\n        @Override\n        public Void scan(Tree tree, Void unused) {\n            if (tree == null) {\n                return null;\n            }\n            scanJavadoc();\n            return super.scan(tree, unused);\n        }\n\n        private void scanJavadoc() {\n            if (getCurrentPath() == null) {\n                return;\n            }\n            DocCommentTree commentTree = trees.getDocCommentTree(getCurrentPath());\n            if (commentTree == null) {\n                return;\n            }\n            docTreeSymbolScanner.scan(new DocTreePath(getCurrentPath(), commentTree), null);\n        }\n\n        // scan javadoc comments, checking for references to imported types\n        class DocTreeScanner extends DocTreePathScanner<Void, Void> {\n            @Override\n            public Void visitIdentifier(com.sun.source.doctree.IdentifierTree node, Void aVoid) {\n                return null;\n            }\n\n            @Override\n            public Void visitReference(ReferenceTree referenceTree, Void unused) {\n                DCReference reference = (DCReference) referenceTree;\n                long basePos = reference\n                        .pos((DCTree.DCDocComment) getCurrentPath().getDocComment())\n                        .getStartPosition();\n                // the position of trees inside the reference node aren't stored, but the qualifier's\n                // start position is the beginning of the reference node\n                if (reference.qualifierExpression != null) {\n                    new ReferenceScanner(basePos).scan(reference.qualifierExpression, null);\n                }\n                // Record uses inside method parameters. The javadoc tool doesn't use these, but\n                // IntelliJ does.\n                if (reference.paramTypes != null) {\n                    for (JCTree param : reference.paramTypes) {\n                        // TODO(cushon): get start positions for the parameters\n                        new ReferenceScanner(-1).scan(param, null);\n                    }\n                }\n                return null;\n            }\n\n            // scans the qualifier and parameters of a javadoc reference for possible type names\n            private class ReferenceScanner extends TreeScanner<Void, Void> {\n                private final long basePos;\n\n                public ReferenceScanner(long basePos) {\n                    this.basePos = basePos;\n                }\n\n                @Override\n                public Void visitIdentifier(IdentifierTree node, Void aVoid) {\n                    usedInJavadoc.put(\n                            node.getName().toString(),\n                            basePos != -1\n                                    ? Range.closedOpen(\n                                            (int) basePos,\n                                            (int) basePos + node.getName().length())\n                                    : null);\n                    return super.visitIdentifier(node, aVoid);\n                }\n            }\n        }\n    }\n\n    public static String removeUnusedImports(final String contents) throws FormatterException {\n        Context context = new Context();\n        JCCompilationUnit unit = parse(context, contents);\n        if (unit == null) {\n            // error handling is done during formatting\n            return contents;\n        }\n        UnusedImportScanner scanner = new UnusedImportScanner(JavacTrees.instance(context));\n        scanner.scan(unit, null);\n        return applyReplacements(contents, buildReplacements(contents, unit, scanner.usedNames, scanner.usedInJavadoc));\n    }\n\n    private static JCCompilationUnit parse(Context context, String javaInput) throws FormatterException {\n        Options.instance(context).put(\"allowStringFolding\", \"false\");\n        return Formatter.parseJcCompilationUnit(context, javaInput);\n    }\n\n    /** Construct replacements to fix unused imports. */\n    private static RangeMap<Integer, String> buildReplacements(\n            String contents,\n            JCCompilationUnit unit,\n            Set<String> usedNames,\n            Multimap<String, Range<Integer>> usedInJavadoc) {\n        RangeMap<Integer, String> replacements = TreeRangeMap.create();\n        for (JCImport importTree : unit.getImports()) {\n            String simpleName = getSimpleName(importTree);\n            if (!isUnused(unit, usedNames, usedInJavadoc, importTree, simpleName)) {\n                continue;\n            }\n            // delete the import\n            int endPosition = importTree.getEndPosition(unit.endPositions);\n            endPosition = Math.max(CharMatcher.isNot(' ').indexIn(contents, endPosition), endPosition);\n            String sep = Newlines.guessLineSeparator(contents);\n            if (endPosition + sep.length() < contents.length()\n                    && contents.subSequence(endPosition, endPosition + sep.length())\n                            .toString()\n                            .equals(sep)) {\n                endPosition += sep.length();\n            }\n            replacements.put(Range.closedOpen(importTree.getStartPosition(), endPosition), \"\");\n        }\n        return replacements;\n    }\n\n    private static String getSimpleName(ImportTree importTree) {\n        return importTree.getQualifiedIdentifier() instanceof JCIdent\n                ? ((JCIdent) importTree.getQualifiedIdentifier()).getName().toString()\n                : ((JCFieldAccess) importTree.getQualifiedIdentifier())\n                        .getIdentifier()\n                        .toString();\n    }\n\n    private static boolean isUnused(\n            JCCompilationUnit unit,\n            Set<String> usedNames,\n            Multimap<String, Range<Integer>> usedInJavadoc,\n            ImportTree importTree,\n            String simpleName) {\n        String qualifier = ((JCFieldAccess) importTree.getQualifiedIdentifier())\n                .getExpression()\n                .toString();\n        if (qualifier.equals(\"java.lang\")) {\n            return true;\n        }\n        if (unit.getPackageName() != null && unit.getPackageName().toString().equals(qualifier)) {\n            return true;\n        }\n        if (importTree.getQualifiedIdentifier() instanceof JCFieldAccess\n                && ((JCFieldAccess) importTree.getQualifiedIdentifier())\n                        .getIdentifier()\n                        .contentEquals(\"*\")) {\n            return false;\n        }\n\n        if (usedNames.contains(simpleName)) {\n            return false;\n        }\n        if (usedInJavadoc.containsKey(simpleName)) {\n            return false;\n        }\n        return true;\n    }\n\n    /** Applies the replacements to the given source, and re-format any edited javadoc. */\n    private static String applyReplacements(String source, RangeMap<Integer, String> replacements) {\n        // save non-empty fixed ranges for reformatting after fixes are applied\n        RangeSet<Integer> fixedRanges = TreeRangeSet.create();\n\n        // Apply the fixes in increasing order, adjusting ranges to account for\n        // earlier fixes that change the length of the source. The output ranges are\n        // needed so we can reformat fixed regions, otherwise the fixes could just\n        // be applied in descending order without adjusting offsets.\n        StringBuilder sb = new StringBuilder(source);\n        int offset = 0;\n        for (Map.Entry<Range<Integer>, String> replacement :\n                replacements.asMapOfRanges().entrySet()) {\n            Range<Integer> range = replacement.getKey();\n            String replaceWith = replacement.getValue();\n            int start = offset + range.lowerEndpoint();\n            int end = offset + range.upperEndpoint();\n            sb.replace(start, end, replaceWith);\n            if (!replaceWith.isEmpty()) {\n                fixedRanges.add(Range.closedOpen(start, end));\n            }\n            offset += replaceWith.length() - (range.upperEndpoint() - range.lowerEndpoint());\n        }\n        return sb.toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/StringWrapper.java\n/*\n * Copyright 2019 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static com.google.common.collect.Iterables.getLast;\nimport static java.util.stream.Collectors.joining;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.base.Verify;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.TreeRangeMap;\nimport com.palantir.javaformat.Newlines;\nimport com.sun.source.tree.BinaryTree;\nimport com.sun.source.tree.LiteralTree;\nimport com.sun.source.tree.MemberSelectTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.source.tree.Tree.Kind;\nimport com.sun.source.util.TreePath;\nimport com.sun.source.util.TreePathScanner;\nimport com.sun.tools.javac.tree.JCTree;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Options;\nimport com.sun.tools.javac.util.Position;\nimport com.sun.tools.javac.util.Position.LineMap;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Deque;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.stream.Stream;\n\n/** Wraps string literals that exceed the column limit. */\npublic final class StringWrapper {\n    /** Reflows string literals in the given Java source code that extend past the given column limit. */\n    static String wrap(final int columnLimit, String input, Formatter formatter) throws FormatterException {\n        if (!longLines(columnLimit, input)) {\n            // fast path\n            return input;\n        }\n\n        TreeRangeMap<Integer, String> replacements = getReflowReplacements(columnLimit, input);\n        String firstPass =\n                formatter.formatSource(input, replacements.asMapOfRanges().keySet());\n\n        if (!firstPass.equals(input)) {\n            // If formatting the replacement ranges resulted in a change, recalculate the replacements on\n            // the updated input.\n            input = firstPass;\n            replacements = getReflowReplacements(columnLimit, input);\n        }\n\n        String result = applyReplacements(input, replacements);\n\n        // Format again, because broken strings might now fit on the first line in case of assignments\n        String secondPass = formatter.formatSource(result, rangesAfterAppliedReplacements(replacements));\n\n        if (!secondPass.equals(result)) {\n            replacements = getReflowReplacements(columnLimit, secondPass);\n            result = applyReplacements(secondPass, replacements);\n        }\n\n        {\n            // We really don't want bugs in this pass to change the behaviour of programs we're\n            // formatting, so check that the pretty-printed AST is the same before and after reformatting.\n            String expected = parse(input, /* allowStringFolding= */ true).toString();\n            String actual = parse(result, /* allowStringFolding= */ true).toString();\n            if (!expected.equals(actual)) {\n                throw new FormatterException(String.format(\n                        \"Something has gone terribly wrong. Please file a bug: \"\n                                + \"https://github.com/google/google-java-format/issues/new\"\n                                + \"\\n\\n=== Actual: ===\\n%s\\n=== Expected: ===\\n%s\\n\",\n                        actual, expected));\n            }\n        }\n\n        return result;\n    }\n\n    @SuppressWarnings(\"ResultOfMethodCallIgnored\")\n    private static ImmutableSet<Range<Integer>> rangesAfterAppliedReplacements(\n            TreeRangeMap<Integer, String> replacements) {\n        ImmutableSet.Builder<Range<Integer>> outputRanges = ImmutableSet.builder();\n        int offset = 0;\n        for (Map.Entry<Range<Integer>, String> entry :\n                replacements.asMapOfRanges().entrySet()) {\n            Range<Integer> range = entry.getKey();\n            String replacement = entry.getValue();\n\n            int lower = offset + range.lowerEndpoint();\n            int upper = lower + replacement.length();\n            outputRanges.add(Range.closedOpen(lower, upper));\n\n            int originalLength = range.upperEndpoint() - range.lowerEndpoint();\n            int newLength = upper - lower;\n            offset += newLength - originalLength;\n        }\n        return outputRanges.build();\n    }\n\n    private static TreeRangeMap<Integer, String> getReflowReplacements(int columnLimit, final String input)\n            throws FormatterException {\n        JCTree.JCCompilationUnit unit = parse(input, /* allowStringFolding= */ false);\n        String separator = Newlines.guessLineSeparator(input);\n\n        // Paths to string literals that extend past the column limit.\n        List<TreePath> toFix = new ArrayList<>();\n        final Position.LineMap lineMap = unit.getLineMap();\n        new TreePathScanner<Void, Void>() {\n            @Override\n            public Void visitLiteral(LiteralTree literalTree, Void aVoid) {\n                if (literalTree.getKind() != Kind.STRING_LITERAL) {\n                    return null;\n                }\n                Tree parent = getCurrentPath().getParentPath().getLeaf();\n                if (parent instanceof MemberSelectTree\n                        && ((MemberSelectTree) parent).getExpression().equals(literalTree)) {\n                    return null;\n                }\n                int endPosition = getEndPosition(unit, literalTree);\n                int lineEnd = endPosition;\n...\n", "needle_token_start": 10484, "needle_token_end": 10536, "code_context_ntokens": 16152, "output": ["```java\nprivate static void addTrivia(StringBuilder replacement, ImmutableList<? extends Tok> toks) {\n    for (Tok tok : toks) {\n        replacement.append(tok.getText());\n    }\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "createDiagnostic", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/Input.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To generate a diagnostic message that includes line and column information based on a character position within the input.\n2. **Input**: An integer representing a character position in the input and a string containing the diagnostic message.\n3. **Output**: A diagnostic object that includes the line number, column number, and the provided message.\n4. **Procedure**: The function first converts the character position to a line number and a column number using internal methods. It then uses these values along with the provided message to create and return a diagnostic object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/CommentsHelper.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\n/**\n * Rewrite comments. This interface is implemented by {@link com.palantir.javaformat.java.JavaCommentsHelper\n * JavaCommentsHelper}.\n */\npublic interface CommentsHelper {\n    /**\n     * Try to rewrite comments, returning rewritten text.\n     *\n     * @param tok the comment's tok\n     * @param maxWidth the line length for the output\n     * @param column0 the current column\n     * @return the rewritten comment\n     */\n    String rewrite(Input.Tok tok, int maxWidth, int column0);\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/BreakTag.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.doc;\n\nimport com.google.errorprone.annotations.Immutable;\n\n/**\n * Unique identifier for a break. A BreakTag can correspond to one or more {@link Break breaks}, and the state of the\n * BreakTag is determined by whether any of the breaks were taken. Then, conditional structures like\n * {@link com.palantir.javaformat.OpsBuilder.BlankLineWanted.ConditionalBlankLine} and\n * {@link com.palantir.javaformat.Indent.If} behave differently based on whether this BreakTag was 'broken' or not.\n *\n * @see State#wasBreakTaken\n */\n@Immutable\npublic final class BreakTag extends HasUniqueId {}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Input.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableRangeMap;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.java.FormatterDiagnostic;\n\n/** An input to the formatter. */\npublic abstract class Input extends InputOutput {\n    /**\n     * A {@code Tok} (\"tock\") is a token, or a comment, or a newline, or a maximal string of blanks. A token {@code Tok}\n     * underlies a {@link Token}, and each other {@code Tok} is attached to a single {@code Token}. Tokens and comments\n     * have indices; white space {@code Tok}s do not.\n     */\n    @Immutable\n    public interface Tok {\n        /**\n         * Return the {@code Tok}'s index.\n         *\n         * @return its index\n         */\n        int getIndex();\n\n        /**\n         * Return the {@code Tok}'s {@code 0}-based position.\n         *\n         * @return its position\n         */\n        int getPosition();\n\n        /**\n         * Return the {@code Tok}'s {@code 0}-based column number.\n         *\n         * @return its column number\n         */\n        int getColumn();\n\n        /** The {@code Tok}'s text. */\n        String getText();\n\n        /** The {@code Tok}'s original text (before processing escapes). */\n        String getOriginalText();\n\n        /** The length of the {@code Tok}'s original text. */\n        int length();\n\n        /** Is the {@code Tok} a newline? */\n        boolean isNewline();\n\n        /** Is the {@code Tok} a \"//\" comment? */\n        boolean isSlashSlashComment();\n\n        /** Is the {@code Tok} a \"//\" comment? */\n        boolean isSlashStarComment();\n\n        /** Is the {@code Tok} a javadoc comment? */\n        boolean isJavadocComment();\n\n        /** Is the {@code Tok} a comment? */\n        boolean isComment();\n    }\n\n    /** A {@code Token} is a language-level token. */\n    @Immutable\n    public interface Token {\n        /**\n         * Get the token's {@link Tok}.\n         *\n         * @return the token's {@link Tok}\n         */\n        Tok getTok();\n\n        /**\n         * Get the earlier {@link Tok}s assigned to this {@code Token}.\n         *\n         * @return the earlier {@link Tok}s assigned to this {@code Token}\n         */\n        ImmutableList<? extends Tok> getToksBefore();\n\n        /**\n         * Get the later {@link Tok}s assigned to this {@code Token}.\n         *\n         * @return the later {@link Tok}s assigned to this {@code Token}\n         */\n        ImmutableList<? extends Tok> getToksAfter();\n    }\n\n    /**\n     * Get the input tokens.\n     *\n     * @return the input tokens\n     */\n    public abstract ImmutableList<? extends Token> getTokens();\n\n    /** A map from [start, end] position ranges to {@link Token}s. */\n    public abstract ImmutableRangeMap<Integer, ? extends Token> getPositionTokenMap();\n\n    public abstract ImmutableMap<Integer, Integer> getPositionToColumnMap();\n\n    public abstract String getText();\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this).add(\"super\", super.toString()).toString();\n    }\n\n    /** Converts a character offset in the input to a line number. */\n    public abstract int getLineNumber(int inputPosition);\n\n    /** Converts a character offset in the input to a 0-based column number. */\n    public abstract int getColumnNumber(int inputPosition);\n\n    /** Construct a diagnostic. Populates the input filename, and converts character offsets to numbers. */\n    \npublic FormatterDiagnostic createDiagnostic(int inputPosition, String message) {\n        return FormatterDiagnostic.create(getLineNumber(inputPosition), getColumnNumber(inputPosition), message);\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/LastLevelBreakability.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat;\n\nimport com.palantir.javaformat.doc.Break;\nimport com.palantir.javaformat.doc.Doc;\n\n/**\n * How to decide whether to break the last inner level (\"this level\") of a parent level with\n * {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel}.\n */\npublic enum LastLevelBreakability {\n    /**\n     * Default behaviour. When processing a {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel} chain, if we've\n     * arrived at a level with this breakability, then we should abort the chain.\n     */\n    ABORT,\n    /**\n     * Unconditionally allow ending an inline chain at this level, after which this level may be broken as usual, or a\n     * prefix thereof could be inlined further (if it has the appropriate break behaviour of\n     * {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel}). This should only be used when you know that the first\n     * non-Level {@link Doc} inside this level, if you flatten it, is a {@link Break}.\n     */\n    ACCEPT_INLINE_CHAIN,\n    /**\n     * Delegate to the {@link LastLevelBreakability} of _this_ level's last inner level. Typically, this will be true if\n     * this level is not immediately followed by a break (see StartsWithBreakVisitor). Behaves the same as\n     * {@link #ABORT} if this level is not {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel}.\n     */\n    CHECK_INNER,\n    /**\n     * Behaves like {@link #ACCEPT_INLINE_CHAIN} if the current inlined levels are all <em>simple</em> (according to\n     * {@link OpenOp#complexity()}), otherwise behave like {@link #CHECK_INNER}.\n     */\n    ACCEPT_INLINE_CHAIN_IF_SIMPLE_OTHERWISE_CHECK_INNER,\n    ;\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/HasUniqueId.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.doc;\n\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport fj.Ord;\nimport java.util.concurrent.atomic.AtomicInteger;\n\n/**\n * This class exists in order to provide a stable {@link Ord} implementation for {@link Doc} and {@link BreakTag}, so\n * that we can store them inside ordered immutable types like {@link fj.data.Set} and {@link fj.data.HashMap}.\n */\npublic abstract class HasUniqueId {\n    private static final AtomicInteger UNIQUE_ID_GENERATOR = new AtomicInteger();\n\n    private final int uniqueId = UNIQUE_ID_GENERATOR.incrementAndGet();\n\n    @JsonProperty(\"id\")\n    public final int id() {\n        return uniqueId;\n    }\n\n    static <D extends HasUniqueId> Ord<D> ord() {\n        return Ord.on((D obj) -> obj.id(), Ord.intOrd).ord();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/OpenOp.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.fasterxml.jackson.databind.annotation.JsonSerialize;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\nimport com.palantir.javaformat.doc.HasUniqueId;\nimport com.palantir.javaformat.doc.Level;\nimport java.util.Optional;\nimport java.util.OptionalInt;\nimport org.immutables.value.Value;\nimport org.immutables.value.Value.Default;\n\n/**\n * An {@code OpenOp} opens a level. It is an {@link Op} in the sequence of {@link Op}s generated by {@link OpsBuilder}.\n * When the sequence is turned into a {@link Doc} by {@link DocBuilder}, {@link Input.Tok}s delimited by\n * {@code OpenOp}-{@link CloseOp} pairs turn into nested {@link Level}s.\n */\n@Value.Immutable\n@JsonSerialize(as = ImmutableOpenOp.class)\npublic abstract class OpenOp extends HasUniqueId implements Op {\n    /** The extra indent inside this level. */\n    public abstract Indent plusIndent();\n\n    /**\n     * When this level doesn't fit on one line, controls whether this level is to be broken (its breaks taken) or\n     * partially inlined onto the current line.\n     */\n    @Default\n    public BreakBehaviour breakBehaviour() {\n        return BreakBehaviours.breakThisLevel();\n    }\n\n    /** If it's the last level of its parent, when to inline this level rather than break the parent. */\n    @Default\n    public LastLevelBreakability breakabilityIfLastLevel() {\n        return LastLevelBreakability.ABORT;\n    }\n\n    @Default\n    public PartialInlineability partialInlineability() {\n        return PartialInlineability.ALWAYS_PARTIALLY_INLINEABLE;\n    }\n\n    /**\n     * A level is \"simple\" if it doesn't have multiple parameters (in the case of a method call), or multiple chained\n     * method calls.\n     *\n     * <p>This is used to poison the ability to partially inline method arguments down the line if a parent level was\n     * too complicated, so that you can't end up with this:\n     *\n     * <pre>\n     * method(arg1, arg2, arg3.foo().stream()\n     *         .filter(...)\n     *         .map(...));\n     * </pre>\n     *\n     * or\n     *\n     * <pre>\n     * log.info(\"Message\", exception, SafeArg.of(\n     *         \"foo\", foo);\n     * </pre>\n     *\n     * But you can still get this (see test B20128760):\n     *\n     * <pre>\n     * Stream<ItemKey> itemIdsStream = stream(members).flatMap(m -> m.getFieldValues().entrySet().stream()\n     *         .filter(...)\n     *         .map(...));\n     * </pre>\n     *\n     * or this:\n     *\n     * <pre>\n     * method(anotherMethod(arg3.foo().stream()\n     *         .filter(...)\n     *         .map(...)));\n     * </pre>\n     *\n     * or this:\n     *\n     * <pre>\n     * method(anotherMethod(\n     *         ...)); // long arguments\n     * </pre>\n     */\n    @Default\n    public Complexity complexity() {\n        return Complexity.SIMPLE;\n    }\n\n    public enum Complexity {\n        /**\n         * This level is simple.\n         */\n        SIMPLE,\n        /**\n         * This level is complex, which will cause certain levels downstream from here (that might otherwise fit on\n         * the same line) to not be considered.\n         */\n        COMPLEX,\n    }\n\n    public abstract Optional<String> debugName();\n\n    /** Custom max column limit that contents of this level <em>before the last break</em> may not exceed. */\n    public abstract OptionalInt columnLimitBeforeLastBreak();\n\n    /**\n     * Make an ordinary {@code OpenOp}.\n     *\n     * @see #builder()\n     */\n    public static Op make(Indent plusIndent) {\n        return builder().plusIndent(plusIndent).build();\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.open(this);\n    }\n\n    /** Builder @see ImmutableOpenOp.Builder#Builder(). */\n    public static Builder builder() {\n        return new Builder();\n    }\n\n    public static class Builder extends ImmutableOpenOp.Builder {\n        public Builder isSimple(boolean isSimple) {\n            return complexity(isSimple ? Complexity.SIMPLE : Complexity.COMPLEX);\n        }\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/CloseOp.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.MoreObjects;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\n\n/**\n * A {@code CloseOp} closes a level. It is an {@link Op} in the sequence of {@link Op}s generated by {@link OpsBuilder}.\n * When the sequence is turned into a {@link Doc} by {@link DocBuilder}, ranges delimited by\n * {@link OpenOp}-{@code CloseOp} pairs turn into nested {@link Doc.Level}s.\n */\npublic enum CloseOp implements Op {\n    CLOSE;\n\n    /**\n     * Make a {@code CloseOp}, returning a singleton since they are all the same.\n     *\n     * @return the singleton {@code CloseOp}\n     */\n    public static Op make() {\n        return CLOSE;\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.close();\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this).toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Newlines.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Iterators;\nimport java.util.Iterator;\nimport java.util.NoSuchElementException;\n\n/** Platform-independent newline handling. */\npublic class Newlines {\n\n    /** Returns the number of line breaks in the input. */\n    public static int count(String input) {\n        return Iterators.size(lineOffsetIterator(input)) - 1;\n    }\n\n    /** Returns the index of the first break in the input, or {@code -1}. */\n    public static int firstBreak(String input) {\n        Iterator<Integer> it = lineOffsetIterator(input);\n        it.next();\n        return it.hasNext() ? it.next() : -1;\n    }\n\n    private static final ImmutableSet<String> BREAKS = ImmutableSet.of(\"\\r\\n\", \"\\n\", \"\\r\");\n\n    /** Returns true if the entire input string is a recognized line break. */\n    public static boolean isNewline(String input) {\n        return BREAKS.contains(input);\n    }\n\n    /** Returns the length of the newline sequence at the current offset, or {@code -1}. */\n    public static int hasNewlineAt(String input, int idx) {\n        for (String b : BREAKS) {\n            if (input.startsWith(b, idx)) {\n                return b.length();\n            }\n        }\n        return -1;\n    }\n\n    /** Returns the terminating line break in the input, or {@code null} if the input does not end in a break. */\n    public static String getLineEnding(String input) {\n        for (String b : BREAKS) {\n            if (input.endsWith(b)) {\n                return b;\n            }\n        }\n        return null;\n    }\n\n    /**\n     * Returns the first line separator in the text, or {@code \"\\n\"} if the text does not contain a single line\n     * separator.\n     */\n    public static String guessLineSeparator(String text) {\n        for (int i = 0; i < text.length(); i++) {\n            char c = text.charAt(i);\n            switch (c) {\n                case '\\r':\n                    if (i + 1 < text.length() && text.charAt(i + 1) == '\\n') {\n                        return \"\\r\\n\";\n                    }\n                    return \"\\r\";\n                case '\\n':\n                    return \"\\n\";\n                default:\n                    break;\n            }\n        }\n        return \"\\n\";\n    }\n\n    /** Returns true if the input contains any line breaks. */\n    public static boolean containsBreaks(String text) {\n        return CharMatcher.anyOf(\"\\n\\r\").matchesAnyOf(text);\n    }\n\n...\n", "needle_token_start": 1908, "needle_token_end": 1954, "code_context_ntokens": 5901, "output": ["```java\npublic FormatterDiagnostic createDiagnostic(int inputPosition, String message) {\n    return FormatterDiagnostic.create(getLineNumber(inputPosition), getColumnNumber(inputPosition), message);\n}\n```"]}
{"repo": "palantir/palantir-java-format", "name": "blankLines", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/InputMetadata.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To maintain formatting preferences regarding the presence of blank lines in a document based on specific positions.\n2. **Input**: Takes no direct inputs as it is a method that retrieves stored data.\n3. **Output**: Returns a map associating positions in the document with preferences on whether blank lines should be preserved.\n4. **Procedure**: Accesses and returns a pre-defined map that contains the preferences for preserving blank lines at various positions within the document.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/CommentsHelper.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\n/**\n * Rewrite comments. This interface is implemented by {@link com.palantir.javaformat.java.JavaCommentsHelper\n * JavaCommentsHelper}.\n */\npublic interface CommentsHelper {\n    /**\n     * Try to rewrite comments, returning rewritten text.\n     *\n     * @param tok the comment's tok\n     * @param maxWidth the line length for the output\n     * @param column0 the current column\n     * @return the rewritten comment\n     */\n    String rewrite(Input.Tok tok, int maxWidth, int column0);\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/BreakTag.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.doc;\n\nimport com.google.errorprone.annotations.Immutable;\n\n/**\n * Unique identifier for a break. A BreakTag can correspond to one or more {@link Break breaks}, and the state of the\n * BreakTag is determined by whether any of the breaks were taken. Then, conditional structures like\n * {@link com.palantir.javaformat.OpsBuilder.BlankLineWanted.ConditionalBlankLine} and\n * {@link com.palantir.javaformat.Indent.If} behave differently based on whether this BreakTag was 'broken' or not.\n *\n * @see State#wasBreakTaken\n */\n@Immutable\npublic final class BreakTag extends HasUniqueId {}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Input.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableRangeMap;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.java.FormatterDiagnostic;\n\n/** An input to the formatter. */\npublic abstract class Input extends InputOutput {\n    /**\n     * A {@code Tok} (\"tock\") is a token, or a comment, or a newline, or a maximal string of blanks. A token {@code Tok}\n     * underlies a {@link Token}, and each other {@code Tok} is attached to a single {@code Token}. Tokens and comments\n     * have indices; white space {@code Tok}s do not.\n     */\n    @Immutable\n    public interface Tok {\n        /**\n         * Return the {@code Tok}'s index.\n         *\n         * @return its index\n         */\n        int getIndex();\n\n        /**\n         * Return the {@code Tok}'s {@code 0}-based position.\n         *\n         * @return its position\n         */\n        int getPosition();\n\n        /**\n         * Return the {@code Tok}'s {@code 0}-based column number.\n         *\n         * @return its column number\n         */\n        int getColumn();\n\n        /** The {@code Tok}'s text. */\n        String getText();\n\n        /** The {@code Tok}'s original text (before processing escapes). */\n        String getOriginalText();\n\n        /** The length of the {@code Tok}'s original text. */\n        int length();\n\n        /** Is the {@code Tok} a newline? */\n        boolean isNewline();\n\n        /** Is the {@code Tok} a \"//\" comment? */\n        boolean isSlashSlashComment();\n\n        /** Is the {@code Tok} a \"//\" comment? */\n        boolean isSlashStarComment();\n\n        /** Is the {@code Tok} a javadoc comment? */\n        boolean isJavadocComment();\n\n        /** Is the {@code Tok} a comment? */\n        boolean isComment();\n    }\n\n    /** A {@code Token} is a language-level token. */\n    @Immutable\n    public interface Token {\n        /**\n         * Get the token's {@link Tok}.\n         *\n         * @return the token's {@link Tok}\n         */\n        Tok getTok();\n\n        /**\n         * Get the earlier {@link Tok}s assigned to this {@code Token}.\n         *\n         * @return the earlier {@link Tok}s assigned to this {@code Token}\n         */\n        ImmutableList<? extends Tok> getToksBefore();\n\n        /**\n         * Get the later {@link Tok}s assigned to this {@code Token}.\n         *\n         * @return the later {@link Tok}s assigned to this {@code Token}\n         */\n        ImmutableList<? extends Tok> getToksAfter();\n    }\n\n    /**\n     * Get the input tokens.\n     *\n     * @return the input tokens\n     */\n    public abstract ImmutableList<? extends Token> getTokens();\n\n    /** A map from [start, end] position ranges to {@link Token}s. */\n    public abstract ImmutableRangeMap<Integer, ? extends Token> getPositionTokenMap();\n\n    public abstract ImmutableMap<Integer, Integer> getPositionToColumnMap();\n\n    public abstract String getText();\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this).add(\"super\", super.toString()).toString();\n    }\n\n    /** Converts a character offset in the input to a line number. */\n    public abstract int getLineNumber(int inputPosition);\n\n    /** Converts a character offset in the input to a 0-based column number. */\n    public abstract int getColumnNumber(int inputPosition);\n\n    /** Construct a diagnostic. Populates the input filename, and converts character offsets to numbers. */\n    public FormatterDiagnostic createDiagnostic(int inputPosition, String message) {\n        return FormatterDiagnostic.create(getLineNumber(inputPosition), getColumnNumber(inputPosition), message);\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/LastLevelBreakability.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat;\n\nimport com.palantir.javaformat.doc.Break;\nimport com.palantir.javaformat.doc.Doc;\n\n/**\n * How to decide whether to break the last inner level (\"this level\") of a parent level with\n * {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel}.\n */\npublic enum LastLevelBreakability {\n    /**\n     * Default behaviour. When processing a {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel} chain, if we've\n     * arrived at a level with this breakability, then we should abort the chain.\n     */\n    ABORT,\n    /**\n     * Unconditionally allow ending an inline chain at this level, after which this level may be broken as usual, or a\n     * prefix thereof could be inlined further (if it has the appropriate break behaviour of\n     * {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel}). This should only be used when you know that the first\n     * non-Level {@link Doc} inside this level, if you flatten it, is a {@link Break}.\n     */\n    ACCEPT_INLINE_CHAIN,\n    /**\n     * Delegate to the {@link LastLevelBreakability} of _this_ level's last inner level. Typically, this will be true if\n     * this level is not immediately followed by a break (see StartsWithBreakVisitor). Behaves the same as\n     * {@link #ABORT} if this level is not {@link BreakBehaviour.Cases#preferBreakingLastInnerLevel}.\n     */\n    CHECK_INNER,\n    /**\n     * Behaves like {@link #ACCEPT_INLINE_CHAIN} if the current inlined levels are all <em>simple</em> (according to\n     * {@link OpenOp#complexity()}), otherwise behave like {@link #CHECK_INNER}.\n     */\n    ACCEPT_INLINE_CHAIN_IF_SIMPLE_OTHERWISE_CHECK_INNER,\n    ;\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/HasUniqueId.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.doc;\n\nimport com.fasterxml.jackson.annotation.JsonProperty;\nimport fj.Ord;\nimport java.util.concurrent.atomic.AtomicInteger;\n\n/**\n * This class exists in order to provide a stable {@link Ord} implementation for {@link Doc} and {@link BreakTag}, so\n * that we can store them inside ordered immutable types like {@link fj.data.Set} and {@link fj.data.HashMap}.\n */\npublic abstract class HasUniqueId {\n    private static final AtomicInteger UNIQUE_ID_GENERATOR = new AtomicInteger();\n\n    private final int uniqueId = UNIQUE_ID_GENERATOR.incrementAndGet();\n\n    @JsonProperty(\"id\")\n    public final int id() {\n        return uniqueId;\n    }\n\n    static <D extends HasUniqueId> Ord<D> ord() {\n        return Ord.on((D obj) -> obj.id(), Ord.intOrd).ord();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/OpenOp.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.fasterxml.jackson.databind.annotation.JsonSerialize;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\nimport com.palantir.javaformat.doc.HasUniqueId;\nimport com.palantir.javaformat.doc.Level;\nimport java.util.Optional;\nimport java.util.OptionalInt;\nimport org.immutables.value.Value;\nimport org.immutables.value.Value.Default;\n\n/**\n * An {@code OpenOp} opens a level. It is an {@link Op} in the sequence of {@link Op}s generated by {@link OpsBuilder}.\n * When the sequence is turned into a {@link Doc} by {@link DocBuilder}, {@link Input.Tok}s delimited by\n * {@code OpenOp}-{@link CloseOp} pairs turn into nested {@link Level}s.\n */\n@Value.Immutable\n@JsonSerialize(as = ImmutableOpenOp.class)\npublic abstract class OpenOp extends HasUniqueId implements Op {\n    /** The extra indent inside this level. */\n    public abstract Indent plusIndent();\n\n    /**\n     * When this level doesn't fit on one line, controls whether this level is to be broken (its breaks taken) or\n     * partially inlined onto the current line.\n     */\n    @Default\n    public BreakBehaviour breakBehaviour() {\n        return BreakBehaviours.breakThisLevel();\n    }\n\n    /** If it's the last level of its parent, when to inline this level rather than break the parent. */\n    @Default\n    public LastLevelBreakability breakabilityIfLastLevel() {\n        return LastLevelBreakability.ABORT;\n    }\n\n    @Default\n    public PartialInlineability partialInlineability() {\n        return PartialInlineability.ALWAYS_PARTIALLY_INLINEABLE;\n    }\n\n    /**\n     * A level is \"simple\" if it doesn't have multiple parameters (in the case of a method call), or multiple chained\n     * method calls.\n     *\n     * <p>This is used to poison the ability to partially inline method arguments down the line if a parent level was\n     * too complicated, so that you can't end up with this:\n     *\n     * <pre>\n     * method(arg1, arg2, arg3.foo().stream()\n     *         .filter(...)\n     *         .map(...));\n     * </pre>\n     *\n     * or\n     *\n     * <pre>\n     * log.info(\"Message\", exception, SafeArg.of(\n     *         \"foo\", foo);\n     * </pre>\n     *\n     * But you can still get this (see test B20128760):\n     *\n     * <pre>\n     * Stream<ItemKey> itemIdsStream = stream(members).flatMap(m -> m.getFieldValues().entrySet().stream()\n     *         .filter(...)\n     *         .map(...));\n     * </pre>\n     *\n     * or this:\n     *\n     * <pre>\n     * method(anotherMethod(arg3.foo().stream()\n     *         .filter(...)\n     *         .map(...)));\n     * </pre>\n     *\n     * or this:\n     *\n     * <pre>\n     * method(anotherMethod(\n     *         ...)); // long arguments\n     * </pre>\n     */\n    @Default\n    public Complexity complexity() {\n        return Complexity.SIMPLE;\n    }\n\n    public enum Complexity {\n        /**\n         * This level is simple.\n         */\n        SIMPLE,\n        /**\n         * This level is complex, which will cause certain levels downstream from here (that might otherwise fit on\n         * the same line) to not be considered.\n         */\n        COMPLEX,\n    }\n\n    public abstract Optional<String> debugName();\n\n    /** Custom max column limit that contents of this level <em>before the last break</em> may not exceed. */\n    public abstract OptionalInt columnLimitBeforeLastBreak();\n\n    /**\n     * Make an ordinary {@code OpenOp}.\n     *\n     * @see #builder()\n     */\n    public static Op make(Indent plusIndent) {\n        return builder().plusIndent(plusIndent).build();\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.open(this);\n    }\n\n    /** Builder @see ImmutableOpenOp.Builder#Builder(). */\n    public static Builder builder() {\n        return new Builder();\n    }\n\n    public static class Builder extends ImmutableOpenOp.Builder {\n        public Builder isSimple(boolean isSimple) {\n            return complexity(isSimple ? Complexity.SIMPLE : Complexity.COMPLEX);\n        }\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/CloseOp.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.MoreObjects;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\n\n/**\n * A {@code CloseOp} closes a level. It is an {@link Op} in the sequence of {@link Op}s generated by {@link OpsBuilder}.\n * When the sequence is turned into a {@link Doc} by {@link DocBuilder}, ranges delimited by\n * {@link OpenOp}-{@code CloseOp} pairs turn into nested {@link Doc.Level}s.\n */\npublic enum CloseOp implements Op {\n    CLOSE;\n\n    /**\n     * Make a {@code CloseOp}, returning a singleton since they are all the same.\n     *\n     * @return the singleton {@code CloseOp}\n     */\n    public static Op make() {\n        return CLOSE;\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.close();\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this).toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Newlines.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Iterators;\nimport java.util.Iterator;\nimport java.util.NoSuchElementException;\n\n/** Platform-independent newline handling. */\npublic class Newlines {\n\n    /** Returns the number of line breaks in the input. */\n    public static int count(String input) {\n        return Iterators.size(lineOffsetIterator(input)) - 1;\n    }\n\n    /** Returns the index of the first break in the input, or {@code -1}. */\n    public static int firstBreak(String input) {\n        Iterator<Integer> it = lineOffsetIterator(input);\n        it.next();\n        return it.hasNext() ? it.next() : -1;\n    }\n\n    private static final ImmutableSet<String> BREAKS = ImmutableSet.of(\"\\r\\n\", \"\\n\", \"\\r\");\n\n    /** Returns true if the entire input string is a recognized line break. */\n    public static boolean isNewline(String input) {\n        return BREAKS.contains(input);\n    }\n\n    /** Returns the length of the newline sequence at the current offset, or {@code -1}. */\n    public static int hasNewlineAt(String input, int idx) {\n        for (String b : BREAKS) {\n            if (input.startsWith(b, idx)) {\n                return b.length();\n            }\n        }\n        return -1;\n    }\n\n    /** Returns the terminating line break in the input, or {@code null} if the input does not end in a break. */\n    public static String getLineEnding(String input) {\n        for (String b : BREAKS) {\n            if (input.endsWith(b)) {\n                return b;\n            }\n        }\n        return null;\n    }\n\n    /**\n     * Returns the first line separator in the text, or {@code \"\\n\"} if the text does not contain a single line\n     * separator.\n     */\n    public static String guessLineSeparator(String text) {\n        for (int i = 0; i < text.length(); i++) {\n            char c = text.charAt(i);\n            switch (c) {\n                case '\\r':\n                    if (i + 1 < text.length() && text.charAt(i + 1) == '\\n') {\n                        return \"\\r\\n\";\n                    }\n                    return \"\\r\";\n                case '\\n':\n                    return \"\\n\";\n                default:\n                    break;\n            }\n        }\n        return \"\\n\";\n    }\n\n    /** Returns true if the input contains any line breaks. */\n    public static boolean containsBreaks(String text) {\n        return CharMatcher.anyOf(\"\\n\\r\").matchesAnyOf(text);\n    }\n\n    /** Returns an iterator over the start offsets of lines in the input. */\n    public static Iterator<Integer> lineOffsetIterator(String input) {\n        return new LineOffsetIterator(input);\n    }\n\n    /** Returns an iterator over lines in the input, including trailing whitespace. */\n    public static Iterator<String> lineIterator(String input) {\n        return new LineIterator(input);\n    }\n\n    private static final class LineOffsetIterator implements Iterator<Integer> {\n\n        private int curr = 0;\n        private int idx = 0;\n        private final String input;\n\n        private LineOffsetIterator(String input) {\n            this.input = input;\n        }\n\n        @Override\n        public boolean hasNext() {\n            return curr != -1;\n        }\n\n        @Override\n        public Integer next() {\n            if (curr == -1) {\n                throw new NoSuchElementException();\n            }\n            int result = curr;\n            advance();\n            return result;\n        }\n\n        private void advance() {\n            for (; idx < input.length(); idx++) {\n                char c = input.charAt(idx);\n                switch (c) {\n                    case '\\r':\n                        if (idx + 1 < input.length() && input.charAt(idx + 1) == '\\n') {\n                            idx++;\n                        }\n                        // falls through\n                    case '\\n':\n                        idx++;\n                        curr = idx;\n                        return;\n                    default:\n                        break;\n                }\n            }\n            curr = -1;\n        }\n\n        @Override\n        public void remove() {\n            throw new UnsupportedOperationException(\"remove\");\n        }\n    }\n\n    private static final class LineIterator implements Iterator<String> {\n\n        int idx;\n        String curr;\n\n        private final String input;\n        private final Iterator<Integer> indices;\n\n        private LineIterator(String input) {\n            this.input = input;\n            this.indices = lineOffsetIterator(input);\n            idx = indices.next(); // read leading 0\n        }\n\n        private void advance() {\n            int last = idx;\n            if (indices.hasNext()) {\n                idx = indices.next();\n            } else if (hasNext()) {\n                // no terminal line break\n                idx = input.length();\n            } else {\n                throw new NoSuchElementException();\n            }\n            curr = input.substring(last, idx);\n        }\n\n        @Override\n        public boolean hasNext() {\n            return idx < input.length();\n        }\n\n        @Override\n        public String next() {\n            advance();\n            return curr;\n        }\n\n        @Override\n        public void remove() {\n            throw new UnsupportedOperationException(\"remove\");\n        }\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/State.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport com.fasterxml.jackson.annotation.JsonIgnore;\nimport com.fasterxml.jackson.databind.annotation.JsonSerialize;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.Indent;\nimport fj.data.Set;\nimport fj.data.TreeMap;\nimport java.lang.annotation.ElementType;\nimport java.lang.annotation.Retention;\nimport java.lang.annotation.RetentionPolicy;\nimport java.lang.annotation.Target;\nimport org.immutables.value.Value;\nimport org.immutables.value.Value.Parameter;\n\n/** State for writing. */\n// Automatically suppressed to unblock enforcement in new code\n@SuppressWarnings(\"ImmutablesStyle\")\n@Value.Immutable\n@Value.Style(overshadowImplementation = true)\n@JsonSerialize(as = ImmutableState.class)\n@Immutable\npublic abstract class State {\n    /** Last indent that was actually taken. */\n    public abstract int lastIndent();\n    /** Next indent, if the level is about to be broken. */\n    public abstract int indent();\n\n    public abstract int column();\n\n    public abstract boolean mustBreak();\n    /** Counts how many lines a particular formatting took. */\n    public abstract int numLines();\n\n    /**\n     * Counts how many times reached a branch, where multiple formattings would be considered. Expected runtime is\n     * exponential in this number.\n     *\n     * @see State#withNewBranch()\n     */\n    public abstract int branchingCoefficient();\n\n    @Value.Auxiliary\n    @JsonIgnore\n    protected abstract Set<BreakTag> breakTagsTaken();\n\n    @Value.Auxiliary\n    @JsonIgnore\n    protected abstract TreeMap<Break, BreakState> breakStates();\n\n    @Value.Auxiliary\n    @JsonIgnore\n    protected abstract TreeMap<Level, LevelState> levelStates();\n\n    /**\n     * Keep track of how each {@link Comment} was written (these are mostly comments), which can differ depending on the\n     * starting column and the maxLength.\n     */\n    @Value.Auxiliary\n    @JsonIgnore\n    protected abstract TreeMap<Comment, TokState> tokStates();\n\n    public static State startingState() {\n        return builder()\n                .lastIndent(0)\n                .indent(0)\n                .column(0)\n                .mustBreak(false)\n                .numLines(0)\n                .branchingCoefficient(0)\n                .breakTagsTaken(Set.empty(HasUniqueId.ord()))\n                .breakStates(TreeMap.empty(HasUniqueId.ord()))\n                .levelStates(TreeMap.empty(HasUniqueId.ord()))\n                .tokStates(TreeMap.empty(HasUniqueId.ord()))\n                .build();\n    }\n\n    public BreakState getBreakState(Break brk) {\n        return breakStates().get(brk).orSome(ImmutableBreakState.of(false, -1));\n    }\n\n    public boolean wasBreakTaken(BreakTag breakTag) {\n        return breakTagsTaken().member(breakTag);\n    }\n\n    boolean isOneLine(Level level) {\n        LevelState levelState = levelStates().get(level).toNull();\n        return levelState != null && levelState.oneLine();\n    }\n\n    String getTokText(Comment comment) {\n        // A TokState will only be present if computeBreaks was called.\n        // That won't always happen, for example when the level containing this comment was one-lined.\n        // Note: if the parent level was inlined, this method itself also won't get called, unless we're in debug mode.\n        return tokStates().get(comment).map(TokState::text).orSome(comment::getFlat);\n    }\n\n    /** Record whether break was taken. */\n    State breakTaken(BreakTag breakTag, boolean broken) {\n        boolean currentlyBroken = breakTagsTaken().member(breakTag);\n        // TODO(dsanduleac): is the opposite ever a valid state?\n        if (currentlyBroken != broken) {\n            Set<BreakTag> newSet;\n            if (broken) {\n                newSet = breakTagsTaken().insert(breakTag);\n            } else {\n                newSet = breakTagsTaken().delete(breakTag);\n            }\n            return builder().from(this).breakTagsTaken(newSet).build();\n        }\n        return this;\n    }\n\n    /**\n     * Increases the indent by {@link Indent#eval} of the {@code plusIndent}, and sets {@code mustBreak} to false. Does\n     * not commit to the indent just yet though, so lastIndent stays the same.\n     */\n    State withIndentIncrementedBy(Indent plusIndent) {\n        return builder()\n                .from(this)\n                .indent(indent() + plusIndent.eval(this))\n                .mustBreak(false)\n                .build();\n    }\n\n    /** Reset any accumulated indent to the same value as {@code lastIndent}. */\n    State withNoIndent() {\n        return builder().from(this).indent(lastIndent()).mustBreak(false).build();\n    }\n\n    /** The current level is being broken and it has breaks in it. Commit to the indent. */\n    State withBrokenLevel() {\n        return builder().from(this).lastIndent(indent()).build();\n    }\n\n    State withBreak(Break brk, boolean broken) {\n        Builder builder = builder().from(this);\n\n        if (broken) {\n            int newColumn = Math.max(indent() + brk.evalPlusIndent(this), 0);\n\n            return builder\n                    // lastIndent = indent -- we've proven that we wrote some stuff at the new 'indent'\n                    .lastIndent(indent())\n                    .column(newColumn)\n                    .numLines(numLines() + 1)\n                    .breakStates(breakStates().set(brk, ImmutableBreakState.of(true, newColumn)))\n                    .build();\n        } else {\n            return builder.column(column() + brk.getFlat().length()).build();\n        }\n    }\n\n    /** Update the current state after having processed an _inner_ level. */\n    State updateAfterLevel(State afterInnerLevel) {\n        return builder()\n                // Inherited current state\n                .lastIndent(lastIndent())\n                .indent(indent())\n                .branchingCoefficient(branchingCoefficient())\n                .mustBreak(mustBreak())\n                // Overridden state\n                .column(afterInnerLevel.column())\n                .numLines(afterInnerLevel.numLines())\n                // TODO(dsanduleac): put these behind a \"GlobalState\"\n                .breakTagsTaken(afterInnerLevel.breakTagsTaken())\n                .breakStates(afterInnerLevel.breakStates())\n                .levelStates(afterInnerLevel.levelStates())\n                .tokStates(afterInnerLevel.tokStates())\n                .build();\n    }\n\n    State addNewLines(int extraNewlines) {\n        return builder().from(this).numLines(numLines() + extraNewlines).build();\n    }\n\n    State withColumn(int column) {\n        return builder().from(this).column(column).build();\n    }\n\n    State withMustBreak(boolean mustBreak) {\n        return builder().from(this).mustBreak(mustBreak).build();\n    }\n\n    State withNewBranch() {\n        return builder()\n                .from(this)\n                .branchingCoefficient(branchingCoefficient() + 1)\n                .build();\n    }\n\n    State withLevelState(Level level, LevelState levelState) {\n        return builder()\n                .from(this)\n                .levelStates(levelStates().set(level, levelState))\n                .build();\n    }\n\n    State withTokState(Comment comment, TokState tokState) {\n        return builder()\n                .from(this)\n                .tokStates(tokStates().set(comment, tokState))\n                .build();\n    }\n\n    public static class Builder extends ImmutableState.Builder {}\n\n    public static Builder builder() {\n        return new Builder();\n    }\n\n    @Target(ElementType.TYPE)\n    @Retention(RetentionPolicy.SOURCE)\n    @Value.Style(overshadowImplementation = true)\n    @interface BreakStateStyle {}\n\n    @BreakStateStyle\n    @Value.Immutable\n    @JsonSerialize(as = ImmutableBreakState.class)\n    interface BreakState {\n        @Parameter\n        boolean broken();\n\n        @Parameter\n        int newIndent();\n    }\n\n    @Target(ElementType.TYPE)\n    @Retention(RetentionPolicy.SOURCE)\n    @Value.Style(overshadowImplementation = true)\n    @interface LevelStateStyle {}\n\n    @LevelStateStyle\n    @Value.Immutable\n    interface LevelState {\n        /** True if the entire {@link Level} fits on one line. */\n        @Parameter\n        boolean oneLine();\n    }\n\n    @Target(ElementType.TYPE)\n    @Retention(RetentionPolicy.SOURCE)\n    @Value.Style(overshadowImplementation = true)\n    @interface TokStateStyle {}\n\n    @TokStateStyle\n    @Value.Immutable\n    interface TokState {\n        @Parameter\n        String text();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Output.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.Range;\nimport com.palantir.javaformat.doc.State;\n\n/** An output from the formatter. */\npublic abstract class Output extends InputOutput {\n    /**\n     * Indent by outputting {@code indent} spaces.\n     *\n     * @param indent the current indent\n     */\n    public abstract void indent(int indent);\n\n    /**\n     * Output a string.\n     *\n     * @param state the current state\n     * @param text the string\n     * @param range the {@link Range} corresponding to the string\n     */\n    public abstract void append(State state, String text, Range<Integer> range);\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this).add(\"super\", super.toString()).toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/Comment.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.Iterators;\nimport com.google.common.collect.Range;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.Output;\n\n/** A leaf node in a {@link Doc} for a non-token. */\n@Immutable\npublic final class Comment extends Doc implements Op {\n    private final Input.Tok tok;\n\n    private Comment(Input.Tok tok) {\n        this.tok = tok;\n    }\n\n    /**\n     * Factory method for a {@code Tok}.\n     *\n     * @param tok the {@link Input.Tok} to wrap\n     * @return the new {@code Tok}\n     */\n    public static Comment make(Input.Tok tok) {\n        return new Comment(tok);\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.add(this);\n    }\n\n    @Override\n    protected float computeWidth() {\n        int idx = Newlines.firstBreak(tok.getOriginalText());\n        // only count the first line of multi-line block comments\n        if (tok.isComment()) {\n            if (idx > 0) {\n                return idx;\n            } else if (tok.isSlashSlashComment() && !tok.getOriginalText().startsWith(\"// \")) {\n                // Account for line comments with missing spaces, see computeFlat.\n                return tok.length() + 1;\n            } else {\n                return tok.length();\n            }\n        }\n        return idx != -1 ? Float.POSITIVE_INFINITY : (float) tok.length();\n    }\n\n    @Override\n    protected String computeFlat() {\n        // TODO(cushon): commentsHelper.rewrite doesn't get called for spans that fit in a single\n        // line. That's fine for multi-line comment reflowing, but problematic for adding missing\n        // spaces in line comments.\n        if (tok.isSlashSlashComment() && !tok.getOriginalText().startsWith(\"// \")) {\n            return \"// \" + tok.getOriginalText().substring(\"//\".length());\n        }\n        return tok.getOriginalText();\n    }\n\n    @Override\n    protected Range<Integer> computeRange() {\n        return Range.singleton(tok.getIndex()).canonical(INTEGERS);\n    }\n\n    @Override\n    public State computeBreaks(\n            CommentsHelper commentsHelper, int maxWidth, State state, Obs.ExplorationNode observationNode) {\n        String text = commentsHelper.rewrite(tok, maxWidth, state.column());\n        int firstLineLength = text.length() - Iterators.getLast(Newlines.lineOffsetIterator(text));\n        return state.withColumn(state.column() + firstLineLength)\n                .addNewLines(Iterators.size(Newlines.lineOffsetIterator(text)))\n                .withTokState(this, ImmutableTokState.of(text));\n    }\n\n    @Override\n    public void write(State state, Output output) {\n        output.append(state, state.getTokText(this), range());\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this).add(\"tok\", tok).toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/FillMode.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\n/** Each {@link Break} in a {@link Level} is either {@link FillMode#UNIFIED} or {@link FillMode#INDEPENDENT}. */\npublic enum FillMode {\n    /** If a {@link Level} will not fit on one line, all of its {@code UNIFIED} {@link Break}s will be broken. */\n    UNIFIED,\n\n    /**\n     * If a {@link Level} will not fit on one line, its {@code INDEPENDENT} {@link Break}s will be broken independently\n     * of each other, to fill in the {@link Level}.\n     */\n    INDEPENDENT,\n\n    /**\n     * A {@code FORCED} {@link Break} will always be broken, and a {@link Level} it appears in will not fit on one line.\n     */\n    FORCED\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/NonBreakingSpace.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.Range;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.Output;\n\n/** A Leaf node in a {@link Doc} for a non-breaking space. */\n@Immutable\npublic final class NonBreakingSpace extends Doc implements Op {\n    private NonBreakingSpace() {}\n\n    public static NonBreakingSpace make() {\n        return new NonBreakingSpace();\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.add(this);\n    }\n\n    @Override\n    protected float computeWidth() {\n        return 1.0F;\n    }\n\n    @Override\n    protected String computeFlat() {\n        return \" \";\n    }\n\n    @Override\n    protected Range<Integer> computeRange() {\n        return Doc.EMPTY_RANGE;\n    }\n\n    @Override\n    public State computeBreaks(\n            CommentsHelper commentsHelper, int maxWidth, State state, Obs.ExplorationNode observationNode) {\n        return state.withColumn(state.column() + 1);\n    }\n\n    @Override\n    public void write(State state, Output output) {\n        output.append(state, \" \", range());\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this).toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/Token.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.Range;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.Indent;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.Output;\nimport java.util.Optional;\n\n/** A leaf {@link Doc} for a token. */\n@Immutable\npublic final class Token extends Doc implements Op {\n    /** Is a Token a real token, or imaginary (e.g., a token generated incorrectly, or an EOF)? */\n    public enum RealOrImaginary {\n        REAL,\n        IMAGINARY;\n    }\n\n    private final Input.Token token;\n    private final RealOrImaginary realOrImaginary;\n    private final Indent plusIndentCommentsBefore;\n    private final Optional<Indent> breakAndIndentTrailingComment;\n\n    private Token(\n            Input.Token token,\n            RealOrImaginary realOrImaginary,\n            Indent plusIndentCommentsBefore,\n            Optional<Indent> breakAndIndentTrailingComment) {\n        this.token = token;\n        this.realOrImaginary = realOrImaginary;\n        this.plusIndentCommentsBefore = plusIndentCommentsBefore;\n        this.breakAndIndentTrailingComment = breakAndIndentTrailingComment;\n    }\n\n    /**\n     * How much extra to indent comments before the {@code Token}.\n     *\n     * @return the extra indent\n     */\n    public Indent getPlusIndentCommentsBefore() {\n        return plusIndentCommentsBefore;\n    }\n\n    /** Force a line break and indent trailing javadoc or block comments. */\n    public Optional<Indent> breakAndIndentTrailingComment() {\n        return breakAndIndentTrailingComment;\n    }\n\n    /**\n     * Make a {@code Token}.\n     *\n     * @param token the {@link Input.Token} to wrap\n     * @param realOrImaginary did this {@link Input.Token} appear in the input, or was it generated incorrectly?\n     * @param plusIndentCommentsBefore extra {@code plusIndent} for comments just before this token\n     * @return the new {@code Token}\n     */\n    public static Op make(\n            Input.Token token,\n            RealOrImaginary realOrImaginary,\n            Indent plusIndentCommentsBefore,\n            Optional<Indent> breakAndIndentTrailingComment) {\n        return new Token(token, realOrImaginary, plusIndentCommentsBefore, breakAndIndentTrailingComment);\n    }\n\n    /**\n     * Return the wrapped {@link Input.Token}.\n     *\n     * @return the {@link Input.Token}\n     */\n    public Input.Token getToken() {\n        return token;\n    }\n\n    /**\n     * Is the token good? That is, does it match an {@link Input.Token}?\n     *\n     * @return whether the @code Token} is good\n     */\n    public RealOrImaginary realOrImaginary() {\n        return realOrImaginary;\n    }\n\n    @Override\n    public void add(DocBuilder builder) {\n        builder.add(this);\n    }\n\n    @Override\n    protected float computeWidth() {\n        return token.getTok().length();\n    }\n\n    @Override\n    protected String computeFlat() {\n        return token.getTok().getOriginalText();\n    }\n\n    @Override\n    protected Range<Integer> computeRange() {\n        return Range.singleton(token.getTok().getIndex()).canonical(INTEGERS);\n    }\n\n    @Override\n    public State computeBreaks(\n            CommentsHelper commentsHelper, int maxWidth, State state, Obs.ExplorationNode observationNode) {\n        String text = token.getTok().getOriginalText();\n        return state.withColumn(state.column() + text.length());\n    }\n\n    @Override\n    public void write(State state, Output output) {\n        String text = token.getTok().getOriginalText();\n        output.append(state, text, range());\n    }\n\n    @Override\n    public String toString() {\n        return MoreObjects.toStringHelper(this)\n                .add(\"token\", token)\n                .add(\"realOrImaginary\", realOrImaginary)\n                .add(\"plusIndentCommentsBefore\", plusIndentCommentsBefore)\n                .toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/InputMetadata.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage com.palantir.javaformat.java;\n\nimport com.google.common.collect.ImmutableMap;\nimport com.google.common.collect.ImmutableRangeSet;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.OpsBuilder.BlankLineWanted;\nimport org.immutables.value.Value;\nimport org.immutables.value.Value.Default;\n\n/**\n * Records metadata about the input, namely existing blank lines that we might want to preserve, as well as what ranges\n * can be partially formatted.\n */\n// Automatically suppressed to unblock enforcement in new code\n@SuppressWarnings(\"ImmutablesStyle\")\n@Immutable\n@Value.Immutable\n@Value.Style(overshadowImplementation = true)\npublic interface InputMetadata {\n    /** Remembers preferences from the input about whether blank lines are wanted or not at a given token index. */\n    \nImmutableMap<Integer, BlankLineWanted> blankLines();\n\n    /**\n     * Marks regions that can be partially formatted, used to determine the actual ranges that will be formatted when\n     * ranges are requested.\n     */\n    @Default\n    default ImmutableRangeSet<Integer> partialFormatRanges() {\n        return ImmutableRangeSet.of();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/FormattingError.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat;\n\nimport com.google.common.base.Joiner;\nimport com.google.common.collect.ImmutableList;\nimport com.palantir.javaformat.java.FormatterDiagnostic;\n\n/** An unchecked formatting error. */\npublic class FormattingError extends RuntimeException {\n\n    private final ImmutableList<FormatterDiagnostic> diagnostics;\n\n    public FormattingError(FormatterDiagnostic diagnostic) {\n        this(ImmutableList.of(diagnostic));\n    }\n\n    public FormattingError(Iterable<FormatterDiagnostic> diagnostics) {\n        super(Joiner.on(\"\\n\").join(diagnostics) + \"\\n\");\n        this.diagnostics = ImmutableList.copyOf(diagnostics);\n    }\n\n    public ImmutableList<FormatterDiagnostic> diagnostics() {\n        return diagnostics;\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/Utils.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat;\n\nimport static java.util.Comparator.comparing;\n\nimport com.google.common.collect.Iterators;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.java.Replacement;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.List;\n\npublic final class Utils {\n    private Utils() {}\n\n    public static String applyReplacements(String input, Collection<Replacement> replacementsCollection) {\n        List<Replacement> replacements = new ArrayList<>(replacementsCollection);\n        replacements.sort(comparing((Replacement r) -> r.getReplaceRange().lowerEndpoint())\n                .reversed());\n        StringBuilder writer = new StringBuilder(input);\n        for (Replacement replacement : replacements) {\n            writer.replace(\n                    replacement.getReplaceRange().lowerEndpoint(),\n                    replacement.getReplaceRange().upperEndpoint(),\n                    replacement.getReplacementString());\n        }\n        return writer.toString();\n    }\n\n    /** Converts zero-indexed, [closed, open) line ranges in the given source file to character ranges. */\n    public static RangeSet<Integer> lineRangesToCharRanges(String input, RangeSet<Integer> lineRanges) {\n        List<Integer> lines = new ArrayList<>();\n        Iterators.addAll(lines, Newlines.lineOffsetIterator(input));\n        lines.add(input.length() + 1);\n\n        final RangeSet<Integer> characterRanges = TreeRangeSet.create();\n        for (Range<Integer> lineRange :\n                lineRanges.subRangeSet(Range.closedOpen(0, lines.size() - 1)).asRanges()) {\n            int lineStart = lines.get(lineRange.lowerEndpoint());\n            // Exclude the trailing newline. This isn't strictly necessary, but handling blank lines\n            // as empty ranges is convenient.\n            int lineEnd = lines.get(lineRange.upperEndpoint()) - 1;\n            Range<Integer> range = Range.closedOpen(lineStart, lineEnd);\n            characterRanges.add(range);\n        }\n        return characterRanges;\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/doc/Obs.java\n/*\n * (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.doc;\n\nimport java.util.Optional;\nimport java.util.OptionalInt;\nimport java.util.function.Function;\nimport javax.annotation.CheckReturnValue;\n\n/**\n * These classes exist purely for observing the operation of {@link Doc#computeBreaks}, including all the alternative\n * hypotheses it has considered and rejected before deciding on the final formatting.\n */\npublic final class Obs {\n    private Obs() {}\n\n    public interface FinishLevelNode {\n        void finishNode(int acceptedExplorationId);\n    }\n\n    public interface FinishExplorationNode {\n        /** Indicate that the exploration node was successful and produced this {@code newState} */\n        void finishNode(Level parentLevel, State newState);\n    }\n\n    /** Within an 'exploration', allows you to descend into child levels. */\n    public interface ExplorationNode {\n        LevelNode newChildNode(Level level, State state);\n\n        int id();\n    }\n\n    public interface Sink {\n        FinishExplorationNode startExplorationNode(\n                int exporationId,\n                OptionalInt parentLevelId,\n                String humanDescription,\n                int startColumn,\n                Optional<State> incomingState);\n\n        /**\n         * Write the given level node\n         * @param levelNodeId the unique ID of the {@link LevelNode}. There can be multiple LevelNodes per\n         *     {@link Level}.\n         * @param parentExplorationId what exploration is this {@link LevelNode} a part of\n         */\n        @CheckReturnValue\n        FinishLevelNode writeLevelNode(int levelNodeId, int parentExplorationId, State incomingState, Level level);\n\n        String getOutput();\n    }\n\n    public static ExplorationNode createRoot(Sink sink) {\n        return new ExplorationNodeImpl(null, \"(initial node)\", sink, 0, Optional.empty());\n    }\n\n    /** At a single level, you can explore various options for how to break lines and then accept one. */\n    interface LevelNode {\n\n        Exploration explore(String humanDescription, State incomingState, Function<ExplorationNode, State> supplier);\n\n        int id();\n\n        Optional<Exploration> maybeExplore(\n                String humanDescription, State incomingState, Function<ExplorationNode, Optional<State>> supplier);\n\n        State finishLevel(State state);\n    }\n\n    /** A handle that lets you accept exactly one 'exploration' of how to format a level. */\n    interface Exploration {\n        State markAccepted();\n\n        State state();\n    }\n\n    private static class LevelNodeImpl extends HasUniqueId implements LevelNode {\n        private final Level level;\n        private final Sink sink;\n        private final FinishLevelNode finisher;\n        private final int startColumn;\n\n        public LevelNodeImpl(Level level, State incomingState, int parentExplorationId, Sink sink) {\n            this.level = level;\n            this.sink = sink;\n            this.finisher = sink.writeLevelNode(id(), parentExplorationId, incomingState, level);\n            this.startColumn = incomingState.column();\n        }\n\n        /**\n         * @param incomingState the state when starting this exploration, whose indents might be different from those in\n         *     this level's {@code incomingState}.\n         */\n        @Override\n        public Exploration explore(\n                String humanDescription, State incomingState, Function<ExplorationNode, State> explorationFunc) {\n            ExplorationNodeImpl explorationNode =\n                    new ExplorationNodeImpl(this, humanDescription, sink, startColumn, Optional.of(incomingState));\n            State newState = explorationFunc.apply(explorationNode);\n...\n", "needle_token_start": 13496, "needle_token_end": 13511, "code_context_ntokens": 15880, "output": ["```java\n/**\n * Remembers preferences from the input about whether blank lines are wanted or not at a given token index.\n */\nImmutableMap<Integer, BlankLineWanted> blankLines();\n```"]}
{"repo": "palantir/palantir-java-format", "name": "flush", "language": "java", "path": "palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaOutput.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function finalizes the processing of text data by ensuring any incomplete text lines are added to the data structure and marks the end of file (EOF) position.\n2. **Input**: There are no direct parameters passed to this function; it operates on the internal state of the object.\n3. **Output**: There is no direct return value; however, it updates internal data structures with the final state of the text content and EOF information.\n4. **Procedure**: \n   - Converts the current content of a string builder into a string and checks if it is not just whitespace. If it contains meaningful content, it is added to a list of lines.\n   - Sets a special marker for the EOF in the range list, ensuring the list is appropriately sized to include all lines plus the EOF marker.\n   - Converts the list of mutable lines into an immutable list and updates the internal state to reflect the final processed content.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/ModifierOrderer.java\n/*\n * Copyright 2016 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n * use this file except in compliance with the License. You may obtain a copy\n * of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations\n * under the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Ordering;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeMap;\nimport com.palantir.javaformat.Input.Tok;\nimport com.palantir.javaformat.Input.Token;\nimport com.sun.tools.javac.parser.Tokens.TokenKind;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport javax.lang.model.element.Modifier;\n\n/** Fixes sequences of modifiers to be in JLS order. */\nfinal class ModifierOrderer {\n\n    /** Reorders all modifiers in the given text to be in JLS order. */\n    static JavaInput reorderModifiers(String text) throws FormatterException {\n        return reorderModifiers(new JavaInput(text), ImmutableList.of(Range.closedOpen(0, text.length())));\n    }\n\n    /** Reorders all modifiers in the given text and within the given character ranges to be in JLS order. */\n    static JavaInput reorderModifiers(JavaInput javaInput, Collection<Range<Integer>> characterRanges)\n            throws FormatterException {\n        if (javaInput.getTokens().isEmpty()) {\n            // There weren't any tokens, possible because of a lexing error.\n            // Errors about invalid input will be reported later after parsing.\n            return javaInput;\n        }\n        RangeSet<Integer> tokenRanges = javaInput.characterRangesToTokenRanges(characterRanges);\n        Iterator<? extends Token> it = javaInput.getTokens().iterator();\n        TreeRangeMap<Integer, String> replacements = TreeRangeMap.create();\n        while (it.hasNext()) {\n            Token token = it.next();\n            if (!tokenRanges.contains(token.getTok().getIndex())) {\n                continue;\n            }\n            Modifier mod = asModifier(token);\n            if (mod == null) {\n                continue;\n            }\n\n            List<Token> modifierTokens = new ArrayList<>();\n            List<Modifier> mods = new ArrayList<>();\n\n            int begin = token.getTok().getPosition();\n            mods.add(mod);\n            modifierTokens.add(token);\n\n            int end = -1;\n            while (it.hasNext()) {\n                token = it.next();\n                mod = asModifier(token);\n                if (mod == null) {\n                    break;\n                }\n                mods.add(mod);\n                modifierTokens.add(token);\n                end = token.getTok().getPosition() + token.getTok().length();\n            }\n\n            if (!Ordering.natural().isOrdered(mods)) {\n                Collections.sort(mods);\n                StringBuilder replacement = new StringBuilder();\n                for (int i = 0; i < mods.size(); i++) {\n                    if (i > 0) {\n                        addTrivia(replacement, modifierTokens.get(i).getToksBefore());\n                    }\n                    replacement.append(mods.get(i).toString());\n                    if (i < (modifierTokens.size() - 1)) {\n                        addTrivia(replacement, modifierTokens.get(i).getToksAfter());\n                    }\n                }\n                replacements.put(Range.closedOpen(begin, end), replacement.toString());\n            }\n        }\n        return applyReplacements(javaInput, replacements);\n    }\n\n    private static void addTrivia(StringBuilder replacement, ImmutableList<? extends Tok> toks) {\n        for (Tok tok : toks) {\n            replacement.append(tok.getText());\n        }\n    }\n\n    /**\n     * Returns the given token as a {@link javax.lang.model.element.Modifier}, or {@code null} if it is not a modifier.\n     */\n    private static Modifier asModifier(Token token) {\n        TokenKind kind = ((JavaInput.Tok) token.getTok()).kind();\n        if (kind != null) {\n            switch (kind) {\n                case PUBLIC:\n                    return Modifier.PUBLIC;\n                case PROTECTED:\n                    return Modifier.PROTECTED;\n                case PRIVATE:\n                    return Modifier.PRIVATE;\n                case ABSTRACT:\n                    return Modifier.ABSTRACT;\n                case STATIC:\n                    return Modifier.STATIC;\n                case DEFAULT:\n                    return Modifier.DEFAULT;\n                case FINAL:\n                    return Modifier.FINAL;\n                case TRANSIENT:\n                    return Modifier.TRANSIENT;\n                case VOLATILE:\n                    return Modifier.VOLATILE;\n                case SYNCHRONIZED:\n                    return Modifier.SYNCHRONIZED;\n                case NATIVE:\n                    return Modifier.NATIVE;\n                case STRICTFP:\n                    return Modifier.STRICTFP;\n                default: // fall out\n            }\n        }\n        switch (token.getTok().getText()) {\n            case \"non-sealed\":\n                return Modifier.valueOf(\"NON_SEALED\");\n            case \"sealed\":\n                return Modifier.valueOf(\"SEALED\");\n            default:\n                return null;\n        }\n    }\n\n    /** Applies replacements to the given string. */\n    private static JavaInput applyReplacements(JavaInput javaInput, TreeRangeMap<Integer, String> replacementMap)\n            throws FormatterException {\n        // process in descending order so the replacement ranges aren't perturbed if any replacements\n        // differ in size from the input\n...\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/RemoveUnusedImports.java\n/*\n * Copyright 2016 Google Inc. All Rights Reserved.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.collect.HashMultimap;\nimport com.google.common.collect.Multimap;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeMap;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeMap;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Newlines;\nimport com.sun.source.doctree.DocCommentTree;\nimport com.sun.source.doctree.ReferenceTree;\nimport com.sun.source.tree.CaseTree;\nimport com.sun.source.tree.IdentifierTree;\nimport com.sun.source.tree.ImportTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.source.util.DocTreePath;\nimport com.sun.source.util.DocTreePathScanner;\nimport com.sun.source.util.TreePathScanner;\nimport com.sun.source.util.TreeScanner;\nimport com.sun.tools.javac.api.JavacTrees;\nimport com.sun.tools.javac.tree.DCTree;\nimport com.sun.tools.javac.tree.DCTree.DCReference;\nimport com.sun.tools.javac.tree.JCTree;\nimport com.sun.tools.javac.tree.JCTree.JCCompilationUnit;\nimport com.sun.tools.javac.tree.JCTree.JCFieldAccess;\nimport com.sun.tools.javac.tree.JCTree.JCIdent;\nimport com.sun.tools.javac.tree.JCTree.JCImport;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Options;\nimport java.lang.reflect.Method;\nimport java.util.LinkedHashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\n/**\n * Removes unused imports from a source file. Imports that are only used in javadoc are also removed, and the references\n * in javadoc are replaced with fully qualified names.\n */\npublic class RemoveUnusedImports {\n\n    // Visits an AST, recording all simple names that could refer to imported\n    // types and also any javadoc references that could refer to imported\n    // types (`@link`, `@see`, `@throws`, etc.)\n    //\n    // No attempt is made to determine whether simple names occur in contexts\n    // where they are type names, so there will be false positives. For example,\n    // `List` is not identified as unused import below:\n    //\n    // ```\n    // import java.util.List;\n    // class List {}\n    // ```\n    //\n    // This is still reasonably effective in practice because type names differ\n    // from other kinds of names in casing convention, and simple name\n    // clashes between imported and declared types are rare.\n    private static final class UnusedImportScanner extends TreePathScanner<Void, Void> {\n\n        private final Set<String> usedNames = new LinkedHashSet<>();\n        private final Multimap<String, Range<Integer>> usedInJavadoc = HashMultimap.create();\n        final JavacTrees trees;\n        final DocTreeScanner docTreeSymbolScanner;\n\n        private UnusedImportScanner(JavacTrees trees) {\n            this.trees = trees;\n            docTreeSymbolScanner = new DocTreeScanner();\n        }\n\n        /** Skip the imports themselves when checking for usage. */\n        @Override\n        public Void visitImport(ImportTree importTree, Void usedSymbols) {\n            return null;\n        }\n\n        @Override\n        public Void visitIdentifier(IdentifierTree tree, Void unused) {\n            if (tree == null) {\n                return null;\n            }\n            usedNames.add(tree.getName().toString());\n            return null;\n        }\n\n        // TODO(fwindheuser): remove this override when pattern matching in switch is no longer a preview\n        // feature, and TreePathScanner visits CaseTree#getLabels instead of CaseTree#getExpressions\n        @SuppressWarnings(\"unchecked\") // reflection\n        @Override\n        public Void visitCase(CaseTree tree, Void unused) {\n            if (CASE_TREE_GET_LABELS != null) {\n                try {\n                    scan((List<? extends Tree>) CASE_TREE_GET_LABELS.invoke(tree), null);\n                } catch (ReflectiveOperationException e) {\n                    throw new RuntimeException(e.getMessage(), e);\n                }\n            }\n            return super.visitCase(tree, null);\n        }\n\n        private static final Method CASE_TREE_GET_LABELS = caseTreeGetLabels();\n\n        private static Method caseTreeGetLabels() {\n            try {\n                return CaseTree.class.getMethod(\"getLabels\");\n            } catch (NoSuchMethodException e) {\n                return null;\n            }\n        }\n\n        @Override\n        public Void scan(Tree tree, Void unused) {\n            if (tree == null) {\n                return null;\n            }\n            scanJavadoc();\n            return super.scan(tree, unused);\n        }\n\n        private void scanJavadoc() {\n            if (getCurrentPath() == null) {\n                return;\n            }\n            DocCommentTree commentTree = trees.getDocCommentTree(getCurrentPath());\n            if (commentTree == null) {\n                return;\n            }\n            docTreeSymbolScanner.scan(new DocTreePath(getCurrentPath(), commentTree), null);\n        }\n\n        // scan javadoc comments, checking for references to imported types\n        class DocTreeScanner extends DocTreePathScanner<Void, Void> {\n            @Override\n            public Void visitIdentifier(com.sun.source.doctree.IdentifierTree node, Void aVoid) {\n                return null;\n            }\n\n            @Override\n            public Void visitReference(ReferenceTree referenceTree, Void unused) {\n                DCReference reference = (DCReference) referenceTree;\n                long basePos = reference\n                        .pos((DCTree.DCDocComment) getCurrentPath().getDocComment())\n                        .getStartPosition();\n                // the position of trees inside the reference node aren't stored, but the qualifier's\n                // start position is the beginning of the reference node\n                if (reference.qualifierExpression != null) {\n                    new ReferenceScanner(basePos).scan(reference.qualifierExpression, null);\n                }\n                // Record uses inside method parameters. The javadoc tool doesn't use these, but\n                // IntelliJ does.\n                if (reference.paramTypes != null) {\n                    for (JCTree param : reference.paramTypes) {\n                        // TODO(cushon): get start positions for the parameters\n                        new ReferenceScanner(-1).scan(param, null);\n                    }\n                }\n                return null;\n            }\n\n            // scans the qualifier and parameters of a javadoc reference for possible type names\n            private class ReferenceScanner extends TreeScanner<Void, Void> {\n                private final long basePos;\n\n                public ReferenceScanner(long basePos) {\n                    this.basePos = basePos;\n                }\n\n                @Override\n                public Void visitIdentifier(IdentifierTree node, Void aVoid) {\n                    usedInJavadoc.put(\n                            node.getName().toString(),\n                            basePos != -1\n                                    ? Range.closedOpen(\n                                            (int) basePos,\n                                            (int) basePos + node.getName().length())\n                                    : null);\n                    return super.visitIdentifier(node, aVoid);\n                }\n            }\n        }\n    }\n\n    public static String removeUnusedImports(final String contents) throws FormatterException {\n        Context context = new Context();\n        JCCompilationUnit unit = parse(context, contents);\n        if (unit == null) {\n            // error handling is done during formatting\n            return contents;\n        }\n        UnusedImportScanner scanner = new UnusedImportScanner(JavacTrees.instance(context));\n        scanner.scan(unit, null);\n        return applyReplacements(contents, buildReplacements(contents, unit, scanner.usedNames, scanner.usedInJavadoc));\n    }\n\n    private static JCCompilationUnit parse(Context context, String javaInput) throws FormatterException {\n        Options.instance(context).put(\"allowStringFolding\", \"false\");\n        return Formatter.parseJcCompilationUnit(context, javaInput);\n    }\n\n    /** Construct replacements to fix unused imports. */\n    private static RangeMap<Integer, String> buildReplacements(\n            String contents,\n            JCCompilationUnit unit,\n            Set<String> usedNames,\n            Multimap<String, Range<Integer>> usedInJavadoc) {\n        RangeMap<Integer, String> replacements = TreeRangeMap.create();\n        for (JCImport importTree : unit.getImports()) {\n            String simpleName = getSimpleName(importTree);\n            if (!isUnused(unit, usedNames, usedInJavadoc, importTree, simpleName)) {\n                continue;\n            }\n            // delete the import\n            int endPosition = importTree.getEndPosition(unit.endPositions);\n            endPosition = Math.max(CharMatcher.isNot(' ').indexIn(contents, endPosition), endPosition);\n            String sep = Newlines.guessLineSeparator(contents);\n            if (endPosition + sep.length() < contents.length()\n                    && contents.subSequence(endPosition, endPosition + sep.length())\n                            .toString()\n                            .equals(sep)) {\n                endPosition += sep.length();\n            }\n            replacements.put(Range.closedOpen(importTree.getStartPosition(), endPosition), \"\");\n        }\n        return replacements;\n    }\n\n    private static String getSimpleName(ImportTree importTree) {\n        return importTree.getQualifiedIdentifier() instanceof JCIdent\n                ? ((JCIdent) importTree.getQualifiedIdentifier()).getName().toString()\n                : ((JCFieldAccess) importTree.getQualifiedIdentifier())\n                        .getIdentifier()\n                        .toString();\n    }\n\n    private static boolean isUnused(\n            JCCompilationUnit unit,\n            Set<String> usedNames,\n            Multimap<String, Range<Integer>> usedInJavadoc,\n            ImportTree importTree,\n            String simpleName) {\n        String qualifier = ((JCFieldAccess) importTree.getQualifiedIdentifier())\n                .getExpression()\n                .toString();\n        if (qualifier.equals(\"java.lang\")) {\n            return true;\n        }\n        if (unit.getPackageName() != null && unit.getPackageName().toString().equals(qualifier)) {\n            return true;\n        }\n        if (importTree.getQualifiedIdentifier() instanceof JCFieldAccess\n                && ((JCFieldAccess) importTree.getQualifiedIdentifier())\n                        .getIdentifier()\n                        .contentEquals(\"*\")) {\n            return false;\n        }\n\n        if (usedNames.contains(simpleName)) {\n            return false;\n        }\n        if (usedInJavadoc.containsKey(simpleName)) {\n            return false;\n        }\n        return true;\n    }\n\n    /** Applies the replacements to the given source, and re-format any edited javadoc. */\n    private static String applyReplacements(String source, RangeMap<Integer, String> replacements) {\n        // save non-empty fixed ranges for reformatting after fixes are applied\n        RangeSet<Integer> fixedRanges = TreeRangeSet.create();\n\n        // Apply the fixes in increasing order, adjusting ranges to account for\n        // earlier fixes that change the length of the source. The output ranges are\n        // needed so we can reformat fixed regions, otherwise the fixes could just\n        // be applied in descending order without adjusting offsets.\n        StringBuilder sb = new StringBuilder(source);\n        int offset = 0;\n        for (Map.Entry<Range<Integer>, String> replacement :\n                replacements.asMapOfRanges().entrySet()) {\n            Range<Integer> range = replacement.getKey();\n            String replaceWith = replacement.getValue();\n            int start = offset + range.lowerEndpoint();\n            int end = offset + range.upperEndpoint();\n            sb.replace(start, end, replaceWith);\n            if (!replaceWith.isEmpty()) {\n                fixedRanges.add(Range.closedOpen(start, end));\n            }\n            offset += replaceWith.length() - (range.upperEndpoint() - range.lowerEndpoint());\n        }\n        return sb.toString();\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/StringWrapper.java\n/*\n * Copyright 2019 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static com.google.common.collect.Iterables.getLast;\nimport static java.util.stream.Collectors.joining;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.base.Verify;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.ImmutableSet;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.TreeRangeMap;\nimport com.palantir.javaformat.Newlines;\nimport com.sun.source.tree.BinaryTree;\nimport com.sun.source.tree.LiteralTree;\nimport com.sun.source.tree.MemberSelectTree;\nimport com.sun.source.tree.Tree;\nimport com.sun.source.tree.Tree.Kind;\nimport com.sun.source.util.TreePath;\nimport com.sun.source.util.TreePathScanner;\nimport com.sun.tools.javac.tree.JCTree;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Options;\nimport com.sun.tools.javac.util.Position;\nimport com.sun.tools.javac.util.Position.LineMap;\nimport java.util.ArrayDeque;\nimport java.util.ArrayList;\nimport java.util.Deque;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.atomic.AtomicBoolean;\nimport java.util.stream.Stream;\n\n/** Wraps string literals that exceed the column limit. */\npublic final class StringWrapper {\n    /** Reflows string literals in the given Java source code that extend past the given column limit. */\n    static String wrap(final int columnLimit, String input, Formatter formatter) throws FormatterException {\n        if (!longLines(columnLimit, input)) {\n            // fast path\n            return input;\n        }\n\n        TreeRangeMap<Integer, String> replacements = getReflowReplacements(columnLimit, input);\n        String firstPass =\n                formatter.formatSource(input, replacements.asMapOfRanges().keySet());\n\n        if (!firstPass.equals(input)) {\n            // If formatting the replacement ranges resulted in a change, recalculate the replacements on\n            // the updated input.\n            input = firstPass;\n            replacements = getReflowReplacements(columnLimit, input);\n        }\n\n        String result = applyReplacements(input, replacements);\n\n        // Format again, because broken strings might now fit on the first line in case of assignments\n        String secondPass = formatter.formatSource(result, rangesAfterAppliedReplacements(replacements));\n\n        if (!secondPass.equals(result)) {\n            replacements = getReflowReplacements(columnLimit, secondPass);\n            result = applyReplacements(secondPass, replacements);\n        }\n\n        {\n            // We really don't want bugs in this pass to change the behaviour of programs we're\n            // formatting, so check that the pretty-printed AST is the same before and after reformatting.\n            String expected = parse(input, /* allowStringFolding= */ true).toString();\n            String actual = parse(result, /* allowStringFolding= */ true).toString();\n            if (!expected.equals(actual)) {\n                throw new FormatterException(String.format(\n                        \"Something has gone terribly wrong. Please file a bug: \"\n                                + \"https://github.com/google/google-java-format/issues/new\"\n                                + \"\\n\\n=== Actual: ===\\n%s\\n=== Expected: ===\\n%s\\n\",\n                        actual, expected));\n            }\n        }\n\n        return result;\n    }\n\n    @SuppressWarnings(\"ResultOfMethodCallIgnored\")\n    private static ImmutableSet<Range<Integer>> rangesAfterAppliedReplacements(\n            TreeRangeMap<Integer, String> replacements) {\n        ImmutableSet.Builder<Range<Integer>> outputRanges = ImmutableSet.builder();\n        int offset = 0;\n        for (Map.Entry<Range<Integer>, String> entry :\n                replacements.asMapOfRanges().entrySet()) {\n            Range<Integer> range = entry.getKey();\n            String replacement = entry.getValue();\n\n            int lower = offset + range.lowerEndpoint();\n            int upper = lower + replacement.length();\n            outputRanges.add(Range.closedOpen(lower, upper));\n\n            int originalLength = range.upperEndpoint() - range.lowerEndpoint();\n            int newLength = upper - lower;\n            offset += newLength - originalLength;\n        }\n        return outputRanges.build();\n    }\n\n    private static TreeRangeMap<Integer, String> getReflowReplacements(int columnLimit, final String input)\n            throws FormatterException {\n        JCTree.JCCompilationUnit unit = parse(input, /* allowStringFolding= */ false);\n        String separator = Newlines.guessLineSeparator(input);\n\n        // Paths to string literals that extend past the column limit.\n        List<TreePath> toFix = new ArrayList<>();\n        final Position.LineMap lineMap = unit.getLineMap();\n        new TreePathScanner<Void, Void>() {\n            @Override\n            public Void visitLiteral(LiteralTree literalTree, Void aVoid) {\n                if (literalTree.getKind() != Kind.STRING_LITERAL) {\n                    return null;\n                }\n                Tree parent = getCurrentPath().getParentPath().getLeaf();\n                if (parent instanceof MemberSelectTree\n                        && ((MemberSelectTree) parent).getExpression().equals(literalTree)) {\n                    return null;\n                }\n                int endPosition = getEndPosition(unit, literalTree);\n                int lineEnd = endPosition;\n                while (Newlines.hasNewlineAt(input, lineEnd) == -1) {\n                    lineEnd++;\n                }\n                if (lineMap.getColumnNumber(lineEnd) - 1 <= columnLimit) {\n                    return null;\n                }\n                toFix.add(getCurrentPath());\n                return null;\n            }\n        }.scan(new TreePath(unit), null);\n\n        TreeRangeMap<Integer, String> replacements = TreeRangeMap.create();\n        for (TreePath path : toFix) {\n            // Find the outermost contiguous enclosing concatenation expression\n            TreePath enclosing = path;\n            while (enclosing.getParentPath().getLeaf().getKind() == Tree.Kind.PLUS) {\n                enclosing = enclosing.getParentPath();\n            }\n            // Is the literal being wrapped the first in a chain of concatenation expressions?\n            // i.e. `ONE + TWO + THREE`\n            // We need this information to handle continuation indents.\n            AtomicBoolean first = new AtomicBoolean(false);\n            // Finds the set of string literals in the concat expression that includes the one that needs\n            // to be wrapped.\n            List<Tree> flat = flatten(input, unit, path, enclosing, first);\n\n            // Walk up as long as parents are on the same line, in order to find the correct\n            // startColumn.\n            TreePath startingPath = enclosing;\n            while (startingPath.getParentPath() != null && onSameLineAsParent(lineMap, startingPath)) {\n                startingPath = startingPath.getParentPath();\n            }\n\n            // Zero-indexed start column\n            int startColumn = lineMap.getColumnNumber(getStartPosition(flat.get(0))) - 1;\n            int fistLineCol = lineMap.getColumnNumber(getStartPosition(startingPath.getLeaf())) - 1;\n\n            // Handling leaving trailing non-string tokens at the end of the literal,\n            // e.g. the trailing `);` in `foo(\"...\");`.\n            int end = getEndPosition(unit, getLast(flat));\n            int lineEnd = end;\n            while (Newlines.hasNewlineAt(input, lineEnd) == -1) {\n                lineEnd++;\n            }\n            int trailing = lineEnd - end;\n\n            // Get the original source text of the string literals, excluding `\"` and `+`.\n            ImmutableList<String> components = stringComponents(input, unit, flat);\n            replacements.put(\n                    Range.closedOpen(getStartPosition(flat.get(0)), getEndPosition(unit, getLast(flat))),\n                    reflow(separator, columnLimit, trailing, components, first.get(), startColumn, fistLineCol));\n        }\n        return replacements;\n    }\n\n    private static boolean onSameLineAsParent(LineMap lineMap, TreePath path) {\n        return lineMap.getLineNumber(getStartPosition(path.getLeaf()))\n                == lineMap.getLineNumber(getStartPosition(path.getParentPath().getLeaf()));\n    }\n\n    /**\n     * Returns the source text of the given string literal trees, excluding the leading and trailing double-quotes and\n     * the `+` operator.\n     */\n    private static ImmutableList<String> stringComponents(\n            String input, JCTree.JCCompilationUnit unit, List<Tree> flat) {\n        ImmutableList.Builder<String> result = ImmutableList.builder();\n        StringBuilder piece = new StringBuilder();\n        for (Tree tree : flat) {\n            // adjust for leading and trailing double quotes\n            String text = input.substring(getStartPosition(tree) + 1, getEndPosition(unit, tree) - 1);\n            int start = 0;\n            for (int idx = 0; idx < text.length(); idx++) {\n                if (CharMatcher.whitespace().matches(text.charAt(idx))) {\n                    // continue below\n                } else if (hasEscapedWhitespaceAt(text, idx) != -1) {\n                    // continue below\n                } else if (hasEscapedNewlineAt(text, idx) != -1) {\n                    int length;\n                    while ((length = hasEscapedNewlineAt(text, idx)) != -1) {\n                        idx += length;\n                    }\n                } else {\n                    continue;\n                }\n                piece.append(text, start, idx);\n                result.add(piece.toString());\n                piece = new StringBuilder();\n                start = idx;\n            }\n            if (piece.length() > 0) {\n                result.add(piece.toString());\n                piece = new StringBuilder();\n            }\n            if (start < text.length()) {\n                piece.append(text, start, text.length());\n            }\n        }\n        if (piece.length() > 0) {\n            result.add(piece.toString());\n        }\n        return result.build();\n    }\n\n    static int hasEscapedWhitespaceAt(String input, int idx) {\n        return Stream.of(\"\\\\t\")\n                .mapToInt(x -> input.startsWith(x, idx) ? x.length() : -1)\n                .filter(x -> x != -1)\n                .findFirst()\n                .orElse(-1);\n    }\n\n    static int hasEscapedNewlineAt(String input, int idx) {\n        return Stream.of(\"\\\\r\\\\n\", \"\\\\r\", \"\\\\n\")\n                .mapToInt(x -> input.startsWith(x, idx) ? x.length() : -1)\n                .filter(x -> x != -1)\n                .findFirst()\n                .orElse(-1);\n    }\n\n    /**\n     * Reflows the given source text, trying to split on word boundaries.\n     *\n     * @param separator the line separator\n     * @param columnLimit the number of columns to wrap at\n     * @param trailing extra space to leave after the last line\n     * @param components the text to reflow\n     * @param first0 true if the text includes the beginning of its enclosing concat chain, i.e. a\n     * @param textStartColumn the column position of the beginning of the original text\n     * @param firstLineStartColumn the column where the very first line starts (can be less than textStartColumn if text\n     *     follows variable declaration)\n     */\n    private static String reflow(\n            String separator,\n            int columnLimit,\n            int trailing,\n            ImmutableList<String> components,\n            boolean first0,\n            int textStartColumn,\n            int firstLineStartColumn) {\n        // We have space between the start column and the limit to output the first line.\n        // Reserve two spaces for the quotes.\n        int width = columnLimit - textStartColumn - 2;\n        Deque<String> input = new ArrayDeque<>(components);\n        List<String> lines = new ArrayList<>();\n        boolean first = first0;\n        while (!input.isEmpty()) {\n            int length = 0;\n            List<String> line = new ArrayList<>();\n            if (input.stream().mapToInt(String::length).sum() <= width) {\n                width -= trailing;\n            }\n            while (!input.isEmpty()\n                    && (length <= 4 || (length + input.peekFirst().length()) < width)) {\n                String text = input.removeFirst();\n                line.add(text);\n                length += text.length();\n                if (text.endsWith(\"\\\\n\") || text.endsWith(\"\\\\r\")) {\n                    break;\n                }\n            }\n            if (line.isEmpty()) {\n                line.add(input.removeFirst());\n            }\n            // add the split line to the output, and process whatever's left\n            lines.add(String.join(\"\", line));\n            if (first) {\n                // Subsequent lines have a four-space continuation indent and a `+ `.\n                width -= 6;\n                // Also, switch to firstLineStartColumn in order to account for the fact that continuations\n                // should get indented from the beginning of the first line.\n                // This is to handle cases like:\n                // String foo = \"first component\"\n                //     + \"rest\";\n                width += textStartColumn - firstLineStartColumn;\n                first = false;\n            }\n        }\n\n        return lines.stream()\n                .collect(joining(\n                        \"\\\"\" + separator + \" \".repeat(first0 ? firstLineStartColumn + 4 : textStartColumn - 2) + \"+ \\\"\",\n                        \"\\\"\",\n                        \"\\\"\"));\n    }\n\n    /**\n     * Flattens the given binary expression tree, and extracts the subset that contains the given path and any adjacent\n     * nodes that are also string literals.\n     */\n    private static List<Tree> flatten(\n            String input, JCTree.JCCompilationUnit unit, TreePath path, TreePath parent, AtomicBoolean firstInChain) {\n        List<Tree> flat = new ArrayList<>();\n\n        // flatten the expression tree with a pre-order traversal\n        ArrayDeque<Tree> todo = new ArrayDeque<>();\n        todo.add(parent.getLeaf());\n        while (!todo.isEmpty()) {\n            Tree first = todo.removeFirst();\n            if (first.getKind() == Tree.Kind.PLUS) {\n                BinaryTree bt = (BinaryTree) first;\n                todo.addFirst(bt.getRightOperand());\n                todo.addFirst(bt.getLeftOperand());\n            } else {\n                flat.add(first);\n            }\n        }\n\n        int idx = flat.indexOf(path.getLeaf());\n        Verify.verify(idx != -1);\n\n        // walk outwards from the leaf for adjacent string literals to also reflow\n        int startIdx = idx;\n        int endIdx = idx + 1;\n        while (startIdx > 0\n                && flat.get(startIdx - 1).getKind() == Tree.Kind.STRING_LITERAL\n                && noComments(input, unit, flat.get(startIdx - 1), flat.get(startIdx))) {\n            startIdx--;\n        }\n        while (endIdx < flat.size()\n                && flat.get(endIdx).getKind() == Tree.Kind.STRING_LITERAL\n                && noComments(input, unit, flat.get(endIdx - 1), flat.get(endIdx))) {\n            endIdx++;\n        }\n\n        firstInChain.set(startIdx == 0);\n        return ImmutableList.copyOf(flat.subList(startIdx, endIdx));\n    }\n\n    private static boolean noComments(String input, JCTree.JCCompilationUnit unit, Tree one, Tree two) {\n        return STRING_CONCAT_DELIMITER.matchesAllOf(\n                input.subSequence(getEndPosition(unit, one), getStartPosition(two)));\n    }\n\n    public static final CharMatcher STRING_CONCAT_DELIMITER =\n            CharMatcher.whitespace().or(CharMatcher.anyOf(\"\\\"+\"));\n\n    private static int getEndPosition(JCTree.JCCompilationUnit unit, Tree tree) {\n        return ((JCTree) tree).getEndPosition(unit.endPositions);\n    }\n\n    private static int getStartPosition(Tree tree) {\n        return ((JCTree) tree).getStartPosition();\n    }\n\n    /** Returns true if any lines in the given Java source exceed the column limit. */\n    private static boolean longLines(int columnLimit, String input) {\n        // TODO(cushon): consider adding Newlines.lineIterable?\n        Iterator<String> it = Newlines.lineIterator(input);\n        while (it.hasNext()) {\n            String line = it.next();\n            if (line.length() > columnLimit) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    /** Parses the given Java source. */\n    private static JCTree.JCCompilationUnit parse(String source, boolean allowStringFolding) throws FormatterException {\n        Context context = new Context();\n        Options.instance(context).put(\"allowStringFolding\", Boolean.toString(allowStringFolding));\n        return Formatter.parseJcCompilationUnit(context, source);\n    }\n\n    /** Applies replacements to the given string. */\n    private static String applyReplacements(String javaInput, TreeRangeMap<Integer, String> replacementMap) {\n        // process in descending order so the replacement ranges aren't perturbed if any replacements\n        // differ in size from the input\n        Map<Range<Integer>, String> ranges = replacementMap.asDescendingMapOfRanges();\n        if (ranges.isEmpty()) {\n            return javaInput;\n        }\n        StringBuilder sb = new StringBuilder(javaInput);\n        for (Map.Entry<Range<Integer>, String> entry : ranges.entrySet()) {\n            Range<Integer> range = entry.getKey();\n            sb.replace(range.lowerEndpoint(), range.upperEndpoint(), entry.getValue());\n        }\n        return sb.toString();\n    }\n\n    private StringWrapper() {}\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/Formatter.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport static java.nio.charset.StandardCharsets.UTF_8;\n\nimport com.google.common.annotations.VisibleForTesting;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Iterables;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.io.CharSink;\nimport com.google.common.io.CharSource;\nimport com.google.errorprone.annotations.Immutable;\nimport com.palantir.javaformat.CommentsHelper;\nimport com.palantir.javaformat.FormattingError;\nimport com.palantir.javaformat.Op;\nimport com.palantir.javaformat.OpsBuilder;\nimport com.palantir.javaformat.OpsBuilder.OpsOutput;\nimport com.palantir.javaformat.Utils;\nimport com.palantir.javaformat.doc.Doc;\nimport com.palantir.javaformat.doc.DocBuilder;\nimport com.palantir.javaformat.doc.Level;\nimport com.palantir.javaformat.doc.NoopSink;\nimport com.palantir.javaformat.doc.Obs;\nimport com.palantir.javaformat.doc.Obs.Sink;\nimport com.palantir.javaformat.doc.State;\nimport com.sun.tools.javac.file.JavacFileManager;\nimport com.sun.tools.javac.parser.JavacParser;\nimport com.sun.tools.javac.parser.ParserFactory;\nimport com.sun.tools.javac.tree.JCTree.JCCompilationUnit;\nimport com.sun.tools.javac.util.Context;\nimport com.sun.tools.javac.util.Log;\nimport com.sun.tools.javac.util.Options;\nimport java.io.IOException;\nimport java.net.URI;\nimport java.util.Collection;\nimport javax.tools.Diagnostic;\nimport javax.tools.DiagnosticCollector;\nimport javax.tools.DiagnosticListener;\nimport javax.tools.JavaFileObject;\nimport javax.tools.SimpleJavaFileObject;\nimport javax.tools.StandardLocation;\n\n/**\n * This is google-java-format, a new Java formatter that follows the Google Java Style Guide quite precisely---to the\n * letter and to the spirit.\n *\n * <p>This formatter uses the javac parser to generate an AST. Because the AST loses information about the non-tokens in\n * the input (including newlines, comments, etc.), and even some tokens (e.g., optional commas or semicolons), this\n * formatter lexes the input again and follows along in the resulting list of tokens. Its lexer splits all\n * multi-character operators (like \"&gt;&gt;\") into multiple single-character operators. Each non-token is assigned to a\n * token---non-tokens following a token on the same line go with that token; those following go with the next token---\n * and there is a final EOF token to hold final comments.\n *\n * <p>The formatter walks the AST to generate a Greg Nelson/Derek Oppen-style list of formatting {@link Op}s [1--2] that\n * then generates a structured {@link Doc}. Each AST node type has a visitor to emit a sequence of {@link Op}s for the\n * node.\n *\n * <p>Some data-structure operations are easier in the list of {@link Op}s, while others become easier in the\n * {@link Doc}. The {@link Op}s are walked to attach the comments. As the {@link Op}s are generated, missing input\n * tokens are inserted and incorrect output tokens are dropped, ensuring that the output matches the input even in the\n * face of formatter errors. Finally, the formatter walks the {@link Doc} to format it in the given width.\n *\n * <p>This formatter also produces data structures of which tokens and comments appear where on the input, and on the\n * output, to help output a partial reformatting of a slightly edited input.\n *\n * <p>Instances of the formatter are immutable and thread-safe.\n *\n * <p>[1] Nelson, Greg, and John DeTreville. Personal communication.\n *\n * <p>[2] Oppen, Derek C. \"Prettyprinting\". ACM Transactions on Programming Languages and Systems, Volume 2 Issue 4,\n * Oct. 1980, pp. 465\u2013483.\n */\n@Immutable\npublic final class Formatter {\n\n    static final Range<Integer> EMPTY_RANGE = Range.closedOpen(-1, -1);\n\n    private final JavaFormatterOptions options;\n    private final boolean debugMode;\n\n    @VisibleForTesting\n    Formatter(JavaFormatterOptions options, boolean debugMode) {\n        this.options = options;\n        this.debugMode = debugMode;\n    }\n\n    /** A new Formatter instance with default options. */\n    public static Formatter create() {\n        return new Formatter(JavaFormatterOptions.defaultOptions(), false);\n    }\n\n    public static Formatter createFormatter(JavaFormatterOptions options) {\n        return new Formatter(options, false);\n    }\n\n    /**\n     * Construct a {@code Formatter} given a Java compilation unit. Parses the code; builds a {@link JavaInput} and the\n     * corresponding {@link JavaOutput}.\n     *\n     * @param javaInput the input, a Java compilation unit\n     * @param options the {@link JavaFormatterOptions}\n     * @param commentsHelper the {@link CommentsHelper}, used to rewrite comments\n     * @param debugMode whether to produce debugging output via {@link DebugRenderer}\n     * @return javaOutput the output produced\n     */\n    static JavaOutput format(\n            final JavaInput javaInput, JavaFormatterOptions options, CommentsHelper commentsHelper, boolean debugMode)\n            throws FormatterException {\n\n        Context context = new Context();\n        Options.instance(context).put(\"allowStringFolding\", \"false\");\n\n        JCCompilationUnit unit = parseJcCompilationUnit(context, javaInput.getText());\n\n        // Output the compilation unit.\n        javaInput.setCompilationUnit(unit);\n        OpsBuilder opsBuilder = new OpsBuilder(javaInput);\n\n        JavaInputAstVisitor visitor;\n        if (getRuntimeVersion() >= 14) {\n            try {\n                visitor = Class.forName(\"com.palantir.javaformat.java.java14.Java14InputAstVisitor\")\n                        .asSubclass(JavaInputAstVisitor.class)\n                        .getConstructor(OpsBuilder.class, int.class)\n                        .newInstance(opsBuilder, options.indentationMultiplier());\n            } catch (ReflectiveOperationException e) {\n                throw new RuntimeException(e.getMessage(), e);\n            }\n        } else {\n            visitor = new JavaInputAstVisitor(opsBuilder, options.indentationMultiplier());\n        }\n\n        visitor.scan(unit, null);\n        opsBuilder.sync(javaInput.getText().length());\n        opsBuilder.drain();\n        OpsOutput opsOutput = opsBuilder.build();\n\n        Level doc = new DocBuilder().withOps(opsOutput.ops()).build();\n\n        // Don't even allocate all those JSON nodes if we're not going to write it out\n        Sink sink = debugMode ? new JsonSink() : new NoopSink();\n\n        Obs.ExplorationNode observationNode = Obs.createRoot(sink);\n        State finalState =\n                doc.computeBreaks(commentsHelper, options.maxLineLength(), State.startingState(), observationNode);\n\n        JavaOutput javaOutput = new JavaOutput(javaInput, opsOutput.inputMetadata());\n        doc.write(finalState, javaOutput);\n        javaOutput.flush();\n\n        if (debugMode) {\n            DebugRenderer.render(javaInput, opsOutput, doc, finalState, javaOutput, sink.getOutput());\n        }\n        return javaOutput;\n    }\n\n    static JCCompilationUnit parseJcCompilationUnit(Context context, String sourceText) throws FormatterException {\n        DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>();\n        context.put(DiagnosticListener.class, diagnostics);\n        Options.instance(context).put(\"--enable-preview\", \"true\");\n        JCCompilationUnit unit;\n        JavacFileManager fileManager = new JavacFileManager(context, true, UTF_8);\n        try {\n            fileManager.setLocation(StandardLocation.PLATFORM_CLASS_PATH, ImmutableList.of());\n        } catch (IOException e) {\n            // impossible\n            throw new RuntimeException(e);\n        }\n        SimpleJavaFileObject source = new SimpleJavaFileObject(URI.create(\"source\"), JavaFileObject.Kind.SOURCE) {\n            @Override\n            public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException {\n                return sourceText;\n            }\n        };\n        Log.instance(context).useSource(source);\n        ParserFactory parserFactory = ParserFactory.instance(context);\n        JavacParser parser = parserFactory.newParser(\n                sourceText, /*keepDocComments=*/ true, /*keepEndPos=*/ true, /*keepLineMap=*/ true);\n        unit = parser.parseCompilationUnit();\n        unit.sourcefile = source;\n\n        Iterable<Diagnostic<? extends JavaFileObject>> errorDiagnostics =\n                Iterables.filter(diagnostics.getDiagnostics(), Formatter::errorDiagnostic);\n        if (!Iterables.isEmpty(errorDiagnostics)) {\n            throw FormatterExceptions.fromJavacDiagnostics(errorDiagnostics);\n        }\n        return unit;\n    }\n\n    @VisibleForTesting\n    static int getRuntimeVersion() {\n        return Runtime.version().feature();\n    }\n\n    static boolean errorDiagnostic(Diagnostic<?> input) {\n        if (input.getKind() != Diagnostic.Kind.ERROR) {\n            return false;\n        }\n        switch (input.getCode()) {\n            case \"compiler.err.invalid.meth.decl.ret.type.req\":\n                // accept constructor-like method declarations that don't match the name of their\n                // enclosing class\n                return false;\n            default:\n                break;\n        }\n        return true;\n    }\n\n    /**\n     * Format the given input (a Java compilation unit) into the output stream.\n     *\n     * @throws FormatterException if the input cannot be parsed\n     */\n    public void formatSource(CharSource input, CharSink output) throws FormatterException, IOException {\n        // TODO(cushon): proper support for streaming input/output. Input may\n        // not be feasible (parsing) but output should be easier.\n        output.write(formatSource(input.read()));\n    }\n\n    /**\n     * Format an input string (a Java compilation unit) into an output string.\n     *\n     * <p>Leaves import statements untouched.\n     *\n     * @param input the input string\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public String formatSource(String input) throws FormatterException {\n        return formatSource(input, ImmutableList.of(Range.closedOpen(0, input.length())));\n    }\n\n    /**\n     * Formats an input string (a Java compilation unit) and fixes imports.\n     *\n     * <p>Fixing imports includes ordering, spacing, and removal of unused import statements.\n     *\n     * @param input the input string\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     * @see <a href=\"https://google.github.io/styleguide/javaguide.html#s3.3.3-import-ordering-and-spacing\">Google Java\n     *     Style Guide - 3.3.3 Import ordering and spacing</a>\n     */\n    public String formatSourceAndFixImports(String input) throws FormatterException {\n        input = ImportOrderer.reorderImports(input, options.style());\n        input = RemoveUnusedImports.removeUnusedImports(input);\n        String formatted = formatSource(input);\n        formatted = StringWrapper.wrap(options.maxLineLength(), formatted, this);\n        return formatted;\n    }\n\n    /**\n     * Format an input string (a Java compilation unit), for only the specified character ranges. These ranges are\n     * extended as necessary (e.g., to encompass whole lines).\n     *\n     * @param input the input string\n     * @param characterRanges the character ranges to be reformatted\n     * @return the output string\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public String formatSource(String input, Collection<Range<Integer>> characterRanges) throws FormatterException {\n        return Utils.applyReplacements(input, getFormatReplacements(input, characterRanges));\n    }\n\n    /**\n     * Emit a list of {@link Replacement}s to convert from input to output.\n     *\n     * @param input the input compilation unit\n     * @param characterRanges the character ranges to reformat\n     * @return a list of {@link Replacement}s, sorted from low index to high index, without overlaps\n     * @throws FormatterException if the input string cannot be parsed\n     */\n    public ImmutableList<Replacement> getFormatReplacements(String input, Collection<Range<Integer>> characterRanges)\n            throws FormatterException {\n        JavaInput javaInput = new JavaInput(input);\n\n        // TODO(cushon): this is only safe because the modifier ordering doesn't affect whitespace,\n        // and doesn't change the replacements that are output. This is not true in general for\n        // 'de-linting' changes (e.g. import ordering).\n        javaInput = ModifierOrderer.reorderModifiers(javaInput, characterRanges);\n\n        JavaCommentsHelper commentsHelper = new JavaCommentsHelper(javaInput.getLineSeparator(), options);\n        JavaOutput javaOutput;\n        try {\n            javaOutput = format(javaInput, options, commentsHelper, debugMode);\n        } catch (FormattingError e) {\n            throw new FormatterException(e.diagnostics());\n        }\n        RangeSet<Integer> tokenRangeSet = javaInput.characterRangesToTokenRanges(characterRanges);\n        return javaOutput.getFormatReplacements(tokenRangeSet);\n    }\n}\n\n// Path: palantir-java-format/src/main/java/com/palantir/javaformat/java/JavaOutput.java\n/*\n * Copyright 2015 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except\n * in compliance with the License. You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software distributed under the License\n * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n * or implied. See the License for the specific language governing permissions and limitations under\n * the License.\n */\n\npackage com.palantir.javaformat.java;\n\nimport com.google.common.base.CharMatcher;\nimport com.google.common.base.MoreObjects;\nimport com.google.common.collect.DiscreteDomain;\nimport com.google.common.collect.ImmutableList;\nimport com.google.common.collect.Range;\nimport com.google.common.collect.RangeSet;\nimport com.google.common.collect.TreeRangeSet;\nimport com.palantir.javaformat.Input;\nimport com.palantir.javaformat.Input.Token;\nimport com.palantir.javaformat.Newlines;\nimport com.palantir.javaformat.OpsBuilder.BlankLineWanted;\nimport com.palantir.javaformat.Output;\nimport com.palantir.javaformat.doc.State;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\n\n/*\n * Throughout this file, {@code i} is an index for input lines, {@code j} is an index for output\n * lines, {@code ij} is an index into either input or output lines, and {@code k} is an index for\n * toks.\n */\n\n/**\n * {@code JavaOutput} extends {@link Output Output} to represent a Java output document. It includes methods to emit the\n * output document.\n */\npublic final class JavaOutput extends Output {\n    private final String lineSeparator;\n    private final JavaInput javaInput; // Used to follow along while emitting the output.\n\n    private final List<String> mutableLines = new ArrayList<>();\n    private final int kN; // The number of tokens or comments in the input, excluding the EOF.\n    private final InputMetadata inputMetadata;\n    private int iLine = 0; // Closest corresponding line number on input.\n    private int lastK = -1; // Last {@link Tok} index output.\n    private int spacesPending = 0;\n    private int newlinesPending = 0;\n    private StringBuilder lineBuilder = new StringBuilder();\n\n    /**\n     * {@code JavaOutput} constructor.\n     *\n     * @param javaInput the {@link JavaInput}, used to match up blank lines in the output\n     */\n    public JavaOutput(JavaInput javaInput, InputMetadata inputMetadata) {\n        this.lineSeparator = javaInput.getLineSeparator();\n        this.javaInput = javaInput;\n        kN = javaInput.getkN();\n        this.inputMetadata = inputMetadata;\n    }\n\n    // TODO(jdd): Add invariant.\n    @Override\n    public void append(State state, String text, Range<Integer> range) {\n        if (!range.isEmpty()) {\n            boolean sawNewlines = false;\n            // Skip over input line we've passed.\n            int iN = javaInput.getLineCount();\n            while (iLine < iN\n                    && (javaInput.getRanges(iLine).isEmpty()\n                            || javaInput.getRanges(iLine).upperEndpoint() <= range.lowerEndpoint())) {\n                if (javaInput.getRanges(iLine).isEmpty()) {\n                    // Skipped over a blank line.\n                    sawNewlines = true;\n                }\n                ++iLine;\n            }\n            /*\n             * Output blank line if we've called {@link OpsBuilder#blankLine}{@code (true)} here, or if\n             * there's a blank line here and it's a comment.\n             */\n            BlankLineWanted wanted = inputMetadata.blankLines().getOrDefault(lastK, BlankLineWanted.NO);\n            if (isComment(text) ? sawNewlines : wanted.wanted(state).orElse(sawNewlines)) {\n                ++newlinesPending;\n            }\n        }\n        if (Newlines.isNewline(text)) {\n            /*\n             * Don't update range information, and swallow extra newlines. The case below for '\\n' is for\n             * block comments.\n             */\n            if (newlinesPending == 0) {\n                ++newlinesPending;\n            }\n            spacesPending = 0;\n        } else {\n            boolean rangesSet = false;\n            int textN = text.length();\n            for (int i = 0; i < textN; i++) {\n                char c = text.charAt(i);\n                switch (c) {\n                    case ' ':\n                        ++spacesPending;\n                        break;\n                    case '\\r':\n                        if (i + 1 < text.length() && text.charAt(i + 1) == '\\n') {\n                            i++;\n                        }\n                        // falls through\n                    case '\\n':\n                        spacesPending = 0;\n                        ++newlinesPending;\n                        break;\n                    default:\n                        while (newlinesPending > 0) {\n                            // drop leading blank lines\n                            if (!mutableLines.isEmpty() || lineBuilder.length() > 0) {\n                                mutableLines.add(lineBuilder.toString());\n                            }\n                            lineBuilder = new StringBuilder();\n                            rangesSet = false;\n                            --newlinesPending;\n                        }\n                        while (spacesPending > 0) {\n                            lineBuilder.append(' ');\n                            --spacesPending;\n                        }\n                        lineBuilder.append(c);\n                        if (!range.isEmpty()) {\n                            if (!rangesSet) {\n                                while (ranges.size() <= mutableLines.size()) {\n                                    ranges.add(Formatter.EMPTY_RANGE);\n                                }\n                                ranges.set(mutableLines.size(), union(ranges.get(mutableLines.size()), range));\n                                rangesSet = true;\n                            }\n                        }\n                }\n            }\n        }\n        if (!range.isEmpty()) {\n            lastK = range.upperEndpoint();\n        }\n    }\n\n    @Override\n    public void indent(int indent) {\n        spacesPending = indent;\n    }\n\n    /** Flush any incomplete last line, then add the EOF token into our data structures. */\n    \nvoid flush() {\n        String lastLine = lineBuilder.toString();\n        if (!CharMatcher.whitespace().matchesAllOf(lastLine)) {\n            mutableLines.add(lastLine);\n        }\n        int jN = mutableLines.size();\n        Range<Integer> eofRange = Range.closedOpen(kN, kN + 1);\n        while (ranges.size() < jN) {\n            ranges.add(Formatter.EMPTY_RANGE);\n        }\n        ranges.add(eofRange);\n        setLines(ImmutableList.copyOf(mutableLines));\n    }\n\n    // The following methods can be used after the Output has been built.\n\n    /**\n     * Emit a list of {@link Replacement}s to convert from input to output.\n     *\n     * @return a list of {@link Replacement}s, sorted by start index, without overlaps\n     */\n    public ImmutableList<Replacement> getFormatReplacements(RangeSet<Integer> iRangeSet0) {\n        ImmutableList.Builder<Replacement> result = ImmutableList.builder();\n        Map<Integer, Range<Integer>> kToJ = JavaOutput.makeKToIJ(this);\n\n        // Expand the token ranges to align with re-formattable boundaries.\n        RangeSet<Integer> breakableRanges = TreeRangeSet.create();\n        RangeSet<Integer> iRangeSet = iRangeSet0.subRangeSet(Range.closed(0, javaInput.getkN()));\n        for (Range<Integer> iRange : iRangeSet.asRanges()) {\n            Range<Integer> range = expandToBreakableRegions(iRange.canonical(DiscreteDomain.integers()));\n            if (range.equals(EMPTY_RANGE)) {\n                // the range contains only whitespace\n                continue;\n            }\n            breakableRanges.add(range);\n        }\n\n        // Construct replacements for each reformatted region.\n        for (Range<Integer> range : breakableRanges.asRanges()) {\n\n            Input.Tok startTok = startTok(javaInput.getToken(range.lowerEndpoint()));\n            Input.Tok endTok = endTok(javaInput.getToken(range.upperEndpoint() - 1));\n\n            // Add all output lines in the given token range to the replacement.\n            StringBuilder replacement = new StringBuilder();\n\n            int replaceFrom = startTok.getPosition();\n            // Replace leading whitespace in the input with the whitespace from the formatted file\n            while (replaceFrom > 0) {\n                char previous = javaInput.getText().charAt(replaceFrom - 1);\n                if (!CharMatcher.whitespace().matches(previous)) {\n                    break;\n                }\n                replaceFrom--;\n            }\n\n            int i = kToJ.get(startTok.getIndex()).lowerEndpoint();\n            // Include leading blank lines from the formatted output, unless the formatted range\n            // starts at the beginning of the file.\n            while (i > 0 && getLine(i - 1).isEmpty()) {\n                i--;\n            }\n            // Write out the formatted range.\n            for (; i < kToJ.get(endTok.getIndex()).upperEndpoint(); i++) {\n                // It's possible to run out of output lines (e.g. if the input ended with\n                // multiple trailing newlines).\n                if (i < getLineCount()) {\n                    if (i > 0) {\n                        replacement.append(lineSeparator);\n                    }\n                    replacement.append(getLine(i));\n                }\n            }\n\n            int replaceTo = Math.min(\n...\n", "needle_token_start": 15264, "needle_token_end": 15407, "code_context_ntokens": 16137, "output": ["```java\nvoid flush() {\n    String lastLine = lineBuilder.toString();\n    if (!CharMatcher.whitespace().matchesAllOf(lastLine)) {\n        mutableLines.add(lastLine);\n    }\n    int jN = mutableLines.size();\n    Range<Integer> eofRange = Range.closedOpen(kN, kN + 1);\n    while (ranges.size() < jN) {\n        ranges.add(Formatter.EMPTY_RANGE);\n    }\n    ranges.add(eofRange);\n    setLines(ImmutableList.copyOf(mutableLines));\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "checkAndAppendClientConnId", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/SQLServerException.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To enhance an error message by appending a unique connection identifier if the connection is active and the identifier is available.\n2. **Input**: A string representing an error message and a connection object.\n3. **Output**: A string which is either the original error message or the error message appended with a unique connection identifier.\n4. **Procedure**: The function first checks if the connection object is not null and is active. If so, it retrieves the unique connection identifier. If the identifier is available, it appends this identifier to the original error message using a specific format. If the connection is not active or the identifier is not available, it returns the original error message.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        boolean xopenStates = (con != null && con.xopenStates);\n        if (xopenStates) {\n            switch (errNum) {\n                case 4060:\n                    return EXCEPTION_XOPEN_CONNECTION_CANT_ESTABLISH; // Database name undefined at logging\n                case 18456:\n                    return EXCEPTION_XOPEN_CONNECTION_CANT_ESTABLISH; // username password wrong at login\n                case 2714:\n                    return \"42S01\"; // Table already exists\n                case 208:\n                    return \"42S02\"; // Table not found\n                case 207:\n                    return \"42S22\"; // Column not found\n                default:\n                    return \"42000\"; // Use XOPEN 'Syntax error or access violation'\n            }\n            // The error code came from the db but XOPEN does not have a specific case for it.\n        } else {\n            switch (errNum) {\n                // case 18456: return EXCEPTION_XOPEN_CONNECTION_CANT_ESTABLISH; //username password wrong at login\n                case 8152:\n                    return \"22001\"; // String data right truncation\n                case 515: // 2.2705\n                case 547:\n                case 2601:\n                case 2627:\n                    return \"23000\"; // Integrity constraint violation\n                case 2714:\n                    return \"S0001\"; // table already exists\n                case 208:\n                    return \"S0002\"; // table not found\n                case 1205:\n                    return \"40001\"; // deadlock detected\n                default: {\n                    String dbState = databaseState.toString();\n                    /*\n                     * Length allowed for SQL State is 5 characters as per SQLSTATE specifications. Append trailing\n                     * zeroes as needed based on length of database error State as length of databaseState is between 1\n                     * to 3 digits.\n                     */\n                    StringBuilder trailingZeroes = new StringBuilder(\"S\");\n                    for (int i = 0; i < 4 - dbState.length(); i++) {\n                        trailingZeroes.append(\"0\");\n                    }\n                    return trailingZeroes.append(dbState).toString();\n                }\n            }\n        }\n    }\n\n    /**\n     * Appends ClientConnectionId to an error message if applicable.\n     * \n     * @param errMsg\n     *        the original error message\n     * @param conn\n     *        the SQLServerConnection object\n     * @return error string concatenated by ClientConnectionId(in string format) if applicable, otherwise, return\n     *         original error string.\n     */\n    \nstatic String checkAndAppendClientConnId(String errMsg, SQLServerConnection conn) {\n        if (null != conn && conn.isConnected()) {\n            UUID clientConnId = conn.getClientConIdInternal();\n            if (null != clientConnId) {\n                StringBuilder sb = (errMsg != null) ? new StringBuilder(errMsg) : new StringBuilder();\n                // This syntax of adding connection id is matched in a retry logic. If anything changes here, make\n                // necessary changes to enableSSL() function's exception handling mechanism.\n                sb.append(LOG_CLIENT_CONNECTION_ID_PREFIX);\n                sb.append(clientConnId.toString());\n                return sb.toString();\n            }\n        }\n        return (errMsg != null) ? errMsg : \"\";\n\n    }\n\n    static void throwNotSupportedException(SQLServerConnection con, Object obj) throws SQLServerException {\n        SQLServerException.makeFromDriverError(con, obj, SQLServerException.getErrString(\"R_notSupported\"), null,\n                false);\n    }\n\n    static void throwFeatureNotSupportedException() throws SQLFeatureNotSupportedException {\n        throw new SQLFeatureNotSupportedException(SQLServerException.getErrString(\"R_notSupported\"));\n    }\n\n    /**\n     * Returns SQLServerError object containing detailed info about exception as received from SQL Server. This API\n     * returns null if no server error has occurred.\n     * \n     * @return SQLServerError\n     */\n    public SQLServerError getSQLServerError() {\n        return sqlServerError;\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/KeyStoreProviderCommon.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.security.InvalidKeyException;\nimport java.security.Key;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.Signature;\nimport java.security.SignatureException;\nimport java.security.cert.X509Certificate;\nimport java.text.MessageFormat;\n\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.Cipher;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.crypto.NoSuchPaddingException;\n\n\n/**\n * \n * This class holds information about the certificate\n *\n */\nclass CertificateDetails {\n    X509Certificate certificate;\n    Key privateKey;\n\n    CertificateDetails(X509Certificate certificate, Key privateKey) {\n        this.certificate = certificate;\n        this.privateKey = privateKey;\n    }\n}\n\n\nclass KeyStoreProviderCommon {\n\n    private KeyStoreProviderCommon() {\n        throw new UnsupportedOperationException(SQLServerException.getErrString(\"R_notSupported\"));\n    }\n\n    static final String RSA_ENCRYPTION_ALGORITHM = \"RSA_OAEP\";\n    static byte[] version = new byte[] {0x01};\n\n    static void validateEncryptionAlgorithm(String encryptionAlgorithm, boolean isEncrypt) throws SQLServerException {\n        String errString = isEncrypt ? \"R_NullKeyEncryptionAlgorithm\" : \"R_NullKeyEncryptionAlgorithmInternal\";\n        if (null == encryptionAlgorithm) {\n\n            throw new SQLServerException(null, SQLServerException.getErrString(errString), null, 0, false);\n\n        }\n\n        errString = isEncrypt ? \"R_InvalidKeyEncryptionAlgorithm\" : \"R_InvalidKeyEncryptionAlgorithmInternal\";\n        if (!RSA_ENCRYPTION_ALGORITHM.equalsIgnoreCase(encryptionAlgorithm.trim())) {\n\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(errString));\n            Object[] msgArgs = {encryptionAlgorithm, RSA_ENCRYPTION_ALGORITHM};\n            throw new SQLServerException(form.format(msgArgs), null);\n\n        }\n    }\n\n    static void validateNonEmptyMasterKeyPath(String masterKeyPath) throws SQLServerException {\n        if (null == masterKeyPath || masterKeyPath.trim().length() == 0) {\n            throw new SQLServerException(null, SQLServerException.getErrString(\"R_InvalidMasterKeyDetails\"), null, 0,\n                    false);\n        }\n    }\n\n    static byte[] decryptColumnEncryptionKey(String masterKeyPath, String encryptionAlgorithm,\n            byte[] encryptedColumnEncryptionKey, CertificateDetails certificateDetails) throws SQLServerException {\n        if (null == encryptedColumnEncryptionKey) {\n\n            throw new SQLServerException(null, SQLServerException.getErrString(\"R_NullEncryptedColumnEncryptionKey\"),\n                    null, 0, false);\n\n        } else if (0 == encryptedColumnEncryptionKey.length) {\n\n            throw new SQLServerException(null, SQLServerException.getErrString(\"R_EmptyEncryptedColumnEncryptionKey\"),\n                    null, 0, false);\n\n        }\n\n        validateEncryptionAlgorithm(encryptionAlgorithm, false);\n\n        int currentIndex = version.length;\n        int keyPathLength = convertTwoBytesToShort(encryptedColumnEncryptionKey, currentIndex);\n        // We just read 2 bytes\n        currentIndex += 2;\n\n        // Get ciphertext length\n        int cipherTextLength = convertTwoBytesToShort(encryptedColumnEncryptionKey, currentIndex);\n        currentIndex += 2;\n\n        currentIndex += keyPathLength;\n\n        int signatureLength = encryptedColumnEncryptionKey.length - currentIndex - cipherTextLength;\n\n        // Get ciphertext\n        byte[] cipherText = new byte[cipherTextLength];\n        System.arraycopy(encryptedColumnEncryptionKey, currentIndex, cipherText, 0, cipherTextLength);\n        currentIndex += cipherTextLength;\n\n        byte[] signature = new byte[signatureLength];\n        System.arraycopy(encryptedColumnEncryptionKey, currentIndex, signature, 0, signatureLength);\n\n        byte[] hash = new byte[encryptedColumnEncryptionKey.length - signature.length];\n\n        System.arraycopy(encryptedColumnEncryptionKey, 0, hash, 0,\n                encryptedColumnEncryptionKey.length - signature.length);\n\n        if (!verifyRSASignature(hash, signature, certificateDetails.certificate, masterKeyPath)) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_InvalidCertificateSignature\"));\n            Object[] msgArgs = {masterKeyPath};\n            throw new SQLServerException(form.format(msgArgs), null);\n        }\n\n        return decryptRSAOAEP(cipherText, certificateDetails);\n    }\n\n    private static byte[] decryptRSAOAEP(byte[] cipherText,\n            CertificateDetails certificateDetails) throws SQLServerException {\n        byte[] plainCEK = null;\n        try {\n            Cipher rsa = Cipher.getInstance(\"RSA/ECB/OAEPWithSHA-1AndMGF1Padding\");\n            rsa.init(Cipher.DECRYPT_MODE, certificateDetails.privateKey);\n            rsa.update(cipherText);\n            plainCEK = rsa.doFinal();\n        } catch (InvalidKeyException | NoSuchAlgorithmException | NoSuchPaddingException | IllegalBlockSizeException\n                | BadPaddingException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_CEKDecryptionFailed\"));\n            Object[] msgArgs = {e.getMessage()};\n            throw new SQLServerException(form.format(msgArgs), e);\n        }\n\n        return plainCEK;\n\n    }\n\n    static boolean verifyRSASignature(byte[] hash, byte[] signature, X509Certificate certificate,\n            String masterKeyPath) throws SQLServerException {\n        Signature signVerify;\n        boolean verificationSuccess = false;\n        try {\n            signVerify = Signature.getInstance(\"SHA256withRSA\");\n            signVerify.initVerify(certificate.getPublicKey());\n            signVerify.update(hash);\n            verificationSuccess = signVerify.verify(signature);\n        } catch (InvalidKeyException | NoSuchAlgorithmException | SignatureException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_InvalidCertificateSignature\"));\n            Object[] msgArgs = {masterKeyPath};\n            throw new SQLServerException(form.format(msgArgs), e);\n        }\n\n        return verificationSuccess;\n\n    }\n\n    private static short convertTwoBytesToShort(byte[] input, int index) throws SQLServerException {\n\n        short shortVal;\n        if (index + 1 >= input.length) {\n            throw new SQLServerException(null, SQLServerException.getErrString(\"R_ByteToShortConversion\"), null, 0,\n                    false);\n        }\n        ByteBuffer byteBuffer = ByteBuffer.allocate(2);\n        byteBuffer.order(ByteOrder.LITTLE_ENDIAN);\n        byteBuffer.put(input[index]);\n        byteBuffer.put(input[index + 1]);\n        shortVal = byteBuffer.getShort(0);\n        return shortVal;\n\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/DataTypes.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.InputStream;\nimport java.io.Reader;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.sql.Blob;\nimport java.sql.Clob;\nimport java.sql.NClob;\nimport java.sql.ResultSet;\nimport java.text.MessageFormat;\nimport java.time.LocalDate;\nimport java.time.LocalDateTime;\nimport java.time.LocalTime;\nimport java.time.OffsetDateTime;\nimport java.time.OffsetTime;\nimport java.util.EnumMap;\nimport java.util.EnumSet;\n\n\nenum TDSType {\n    // FIXEDLEN types\n    BIT1(0x32), // 50\n    INT8(0x7F), // 127\n    INT4(0x38), // 56\n    INT2(0x34), // 52\n    INT1(0x30), // 48\n    FLOAT4(0x3B), // 59\n    FLOAT8(0x3E), // 62\n    DATETIME4(0x3A), // 58\n    DATETIME8(0x3D), // 61\n    MONEY4(0x7A), // 122\n    MONEY8(0x3C), // 60\n\n    // BYTELEN types\n    BITN(0x68), // 104\n    INTN(0x26), // 38\n    DECIMALN(0x6A), // 106\n    NUMERICN(0x6C), // 108\n    FLOATN(0x6D), // 109\n    MONEYN(0x6E), // 110\n    DATETIMEN(0x6F), // 111\n    GUID(0x24), // 36\n    DATEN(0x28), // 40\n    TIMEN(0x29), // 41\n    DATETIME2N(0x2a), // 42\n    DATETIMEOFFSETN(0x2b), // 43\n\n    // USHORTLEN type\n    BIGCHAR(0xAF), // -81\n    BIGVARCHAR(0xA7), // -89\n    BIGBINARY(0xAD), // -83\n    BIGVARBINARY(0xA5), // -91\n    NCHAR(0xEF), // -17\n    NVARCHAR(0xE7), // -15\n\n    // PARTLEN types\n    IMAGE(0x22), // 34\n    TEXT(0x23), // 35\n    NTEXT(0x63), // 99\n    UDT(0xF0), // -16\n    XML(0xF1), // -15\n\n    // LONGLEN types\n    SQL_VARIANT(0x62); // 98\n\n    private final int intValue;\n\n    private static final int MAXELEMENTS = 256;\n    private static final TDSType[] VALUES = values();\n    private static final TDSType[] valuesTypes = new TDSType[MAXELEMENTS];\n\n    byte byteValue() {\n        return (byte) intValue;\n    }\n\n    static {\n        for (TDSType s : VALUES)\n            valuesTypes[s.intValue] = s;\n    }\n\n    private TDSType(int intValue) {\n        this.intValue = intValue;\n    }\n\n    static TDSType valueOf(int intValue) throws IllegalArgumentException {\n        TDSType tdsType;\n\n        if (!(0 <= intValue && intValue < valuesTypes.length) || null == (tdsType = valuesTypes[intValue])) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_unknownSSType\"));\n            Object[] msgArgs = {intValue};\n            throw new IllegalArgumentException(form.format(msgArgs));\n        }\n\n        return tdsType;\n    }\n}\n\n\nenum SSLenType {\n    FIXEDLENTYPE,\n    BYTELENTYPE,\n    USHORTLENTYPE,\n    LONGLENTYPE,\n    PARTLENTYPE\n}\n\n\nenum SSType {\n    UNKNOWN(Category.UNKNOWN, \"unknown\", JDBCType.UNKNOWN),\n    TINYINT(Category.NUMERIC, \"tinyint\", JDBCType.TINYINT),\n    BIT(Category.NUMERIC, \"bit\", JDBCType.BIT),\n    SMALLINT(Category.NUMERIC, \"smallint\", JDBCType.SMALLINT),\n    INTEGER(Category.NUMERIC, \"int\", JDBCType.INTEGER),\n    BIGINT(Category.NUMERIC, \"bigint\", JDBCType.BIGINT),\n    FLOAT(Category.NUMERIC, \"float\", JDBCType.DOUBLE),\n    REAL(Category.NUMERIC, \"real\", JDBCType.REAL),\n    SMALLDATETIME(Category.DATETIME, \"smalldatetime\", JDBCType.SMALLDATETIME),\n    DATETIME(Category.DATETIME, \"datetime\", JDBCType.DATETIME),\n    DATE(Category.DATE, \"date\", JDBCType.DATE),\n    TIME(Category.TIME, \"time\", JDBCType.TIME),\n    DATETIME2(Category.DATETIME2, \"datetime2\", JDBCType.TIMESTAMP),\n    DATETIMEOFFSET(Category.DATETIMEOFFSET, \"datetimeoffset\", JDBCType.DATETIMEOFFSET),\n    SMALLMONEY(Category.NUMERIC, \"smallmoney\", JDBCType.SMALLMONEY),\n    MONEY(Category.NUMERIC, \"money\", JDBCType.MONEY),\n    CHAR(Category.CHARACTER, \"char\", JDBCType.CHAR),\n    VARCHAR(Category.CHARACTER, \"varchar\", JDBCType.VARCHAR),\n    VARCHARMAX(Category.LONG_CHARACTER, \"varchar\", JDBCType.LONGVARCHAR),\n    TEXT(Category.LONG_CHARACTER, \"text\", JDBCType.LONGVARCHAR),\n    NCHAR(Category.NCHARACTER, \"nchar\", JDBCType.NCHAR),\n    NVARCHAR(Category.NCHARACTER, \"nvarchar\", JDBCType.NVARCHAR),\n    NVARCHARMAX(Category.LONG_NCHARACTER, \"nvarchar\", JDBCType.LONGNVARCHAR),\n    NTEXT(Category.LONG_NCHARACTER, \"ntext\", JDBCType.LONGNVARCHAR),\n    BINARY(Category.BINARY, \"binary\", JDBCType.BINARY),\n    VARBINARY(Category.BINARY, \"varbinary\", JDBCType.VARBINARY),\n    VARBINARYMAX(Category.LONG_BINARY, \"varbinary\", JDBCType.LONGVARBINARY),\n    IMAGE(Category.LONG_BINARY, \"image\", JDBCType.LONGVARBINARY),\n    DECIMAL(Category.NUMERIC, \"decimal\", JDBCType.DECIMAL),\n    NUMERIC(Category.NUMERIC, \"numeric\", JDBCType.NUMERIC),\n    GUID(Category.GUID, \"uniqueidentifier\", JDBCType.GUID),\n    SQL_VARIANT(Category.SQL_VARIANT, \"sql_variant\", JDBCType.SQL_VARIANT),\n    UDT(Category.UDT, \"udt\", JDBCType.VARBINARY),\n    XML(Category.XML, \"xml\", JDBCType.LONGNVARCHAR),\n    TIMESTAMP(Category.TIMESTAMP, \"timestamp\", JDBCType.BINARY),\n    GEOMETRY(Category.UDT, \"geometry\", JDBCType.GEOMETRY),\n    GEOGRAPHY(Category.UDT, \"geography\", JDBCType.GEOGRAPHY);\n\n    final Category category;\n    private final String name;\n    private final JDBCType jdbcType;\n    private static final SSType[] VALUES = values();\n\n    static final BigDecimal MAX_VALUE_MONEY = new BigDecimal(\"922337203685477.5807\");\n    static final BigDecimal MIN_VALUE_MONEY = new BigDecimal(\"-922337203685477.5808\");\n    static final BigDecimal MAX_VALUE_SMALLMONEY = new BigDecimal(\"214748.3647\");\n    static final BigDecimal MIN_VALUE_SMALLMONEY = new BigDecimal(\"-214748.3648\");\n\n    private SSType(Category category, String name, JDBCType jdbcType) {\n        this.category = category;\n        this.name = name;\n        this.jdbcType = jdbcType;\n    }\n\n    @Override\n    public String toString() {\n        return name;\n    }\n\n    final JDBCType getJDBCType() {\n        return jdbcType;\n    }\n\n    static SSType of(String typeName) throws SQLServerException {\n        for (SSType ssType : VALUES)\n            if (ssType.name.equalsIgnoreCase(typeName))\n                return ssType;\n\n        MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_unknownSSType\"));\n        Object[] msgArgs = {typeName};\n        SQLServerException.makeFromDriverError(null, null, form.format(msgArgs), null, true);\n        return SSType.UNKNOWN;\n    }\n\n    enum Category {\n        BINARY,\n        CHARACTER,\n        DATE,\n        DATETIME,\n        DATETIME2,\n        DATETIMEOFFSET,\n        GUID,\n        LONG_BINARY,\n        LONG_CHARACTER,\n        LONG_NCHARACTER,\n        NCHARACTER,\n        NUMERIC,\n        UNKNOWN,\n        TIME,\n        TIMESTAMP,\n        UDT,\n        SQL_VARIANT,\n        XML;\n\n        private static final Category[] VALUES = values();\n    }\n\n    enum GetterConversion {\n        NUMERIC(SSType.Category.NUMERIC, EnumSet.of(JDBCType.Category.NUMERIC, JDBCType.Category.CHARACTER,\n                JDBCType.Category.BINARY)),\n\n        DATETIME(SSType.Category.DATETIME, EnumSet.of(JDBCType.Category.DATE, JDBCType.Category.TIME,\n                JDBCType.Category.TIMESTAMP, JDBCType.Category.CHARACTER, JDBCType.Category.BINARY)),\n\n        DATETIME2(SSType.Category.DATETIME2, EnumSet.of(JDBCType.Category.DATE, JDBCType.Category.TIME,\n                JDBCType.Category.TIMESTAMP, JDBCType.Category.CHARACTER)),\n\n        DATE(SSType.Category.DATE, EnumSet.of(JDBCType.Category.DATE, JDBCType.Category.TIMESTAMP,\n                JDBCType.Category.CHARACTER)),\n\n        TIME(SSType.Category.TIME, EnumSet.of(JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP,\n                JDBCType.Category.CHARACTER)),\n\n        DATETIMEOFFSET(SSType.Category.DATETIMEOFFSET, EnumSet.of(JDBCType.Category.DATE, JDBCType.Category.TIME,\n                JDBCType.Category.TIMESTAMP, JDBCType.Category.DATETIMEOFFSET, JDBCType.Category.CHARACTER)),\n\n        CHARACTER(SSType.Category.CHARACTER, EnumSet.of(JDBCType.Category.NUMERIC, JDBCType.Category.DATE,\n                JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP, JDBCType.Category.CHARACTER,\n                JDBCType.Category.LONG_CHARACTER, JDBCType.Category.BINARY, JDBCType.Category.GUID)),\n\n        LONG_CHARACTER(SSType.Category.LONG_CHARACTER, EnumSet.of(JDBCType.Category.NUMERIC, JDBCType.Category.DATE,\n                JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP, JDBCType.Category.CHARACTER,\n                JDBCType.Category.LONG_CHARACTER, JDBCType.Category.BINARY, JDBCType.Category.CLOB)),\n\n        NCHARACTER(SSType.Category.NCHARACTER, EnumSet.of(JDBCType.Category.NUMERIC, JDBCType.Category.CHARACTER,\n                JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER,\n                JDBCType.Category.BINARY, JDBCType.Category.DATE, JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP)),\n\n        LONG_NCHARACTER(SSType.Category.LONG_NCHARACTER, EnumSet.of(JDBCType.Category.NUMERIC,\n                JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER,\n                JDBCType.Category.LONG_NCHARACTER, JDBCType.Category.BINARY, JDBCType.Category.DATE,\n                JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP, JDBCType.Category.CLOB, JDBCType.Category.NCLOB)),\n\n        BINARY(SSType.Category.BINARY, EnumSet.of(JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY,\n                JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER, JDBCType.Category.GUID)),\n\n        LONG_BINARY(SSType.Category.LONG_BINARY, EnumSet.of(JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY,\n                JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER, JDBCType.Category.BLOB)),\n\n        TIMESTAMP(SSType.Category.TIMESTAMP, EnumSet.of(JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY,\n                JDBCType.Category.CHARACTER)),\n\n        XML(SSType.Category.XML, EnumSet.of(JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER,\n                JDBCType.Category.CLOB, JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER,\n                JDBCType.Category.NCLOB, JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY,\n                JDBCType.Category.BLOB, JDBCType.Category.SQLXML)),\n\n        UDT(SSType.Category.UDT, EnumSet.of(JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY,\n                JDBCType.Category.CHARACTER, JDBCType.Category.GEOMETRY, JDBCType.Category.GEOGRAPHY)),\n\n        GUID(SSType.Category.GUID, EnumSet.of(JDBCType.Category.BINARY, JDBCType.Category.CHARACTER)),\n\n        SQL_VARIANT(SSType.Category.SQL_VARIANT, EnumSet.of(JDBCType.Category.CHARACTER, JDBCType.Category.SQL_VARIANT,\n                JDBCType.Category.NUMERIC, JDBCType.Category.DATE, JDBCType.Category.TIME, JDBCType.Category.BINARY,\n                JDBCType.Category.TIMESTAMP, JDBCType.Category.NCHARACTER, JDBCType.Category.GUID));\n\n        private final SSType.Category from;\n        private final EnumSet<JDBCType.Category> to;\n        private static final GetterConversion[] VALUES = values();\n\n        private GetterConversion(SSType.Category from, EnumSet<JDBCType.Category> to) {\n            this.from = from;\n            this.to = to;\n        }\n\n        private static final EnumMap<SSType.Category, EnumSet<JDBCType.Category>> conversionMap = new EnumMap<>(\n                SSType.Category.class);\n\n        static {\n            for (SSType.Category category : SSType.Category.VALUES)\n                conversionMap.put(category, EnumSet.noneOf(JDBCType.Category.class));\n\n            for (GetterConversion conversion : VALUES)\n                conversionMap.get(conversion.from).addAll(conversion.to);\n        }\n\n        static final boolean converts(SSType fromSSType, JDBCType toJDBCType) {\n            return conversionMap.get(fromSSType.category).contains(toJDBCType.category);\n        }\n    }\n\n    boolean convertsTo(JDBCType jdbcType) {\n        return GetterConversion.converts(this, jdbcType);\n    }\n}\n\n\nenum StreamType {\n    NONE(JDBCType.UNKNOWN, \"None\"),\n    ASCII(JDBCType.LONGVARCHAR, \"AsciiStream\"),\n    BINARY(JDBCType.LONGVARBINARY, \"BinaryStream\"),\n    CHARACTER(JDBCType.LONGVARCHAR, \"CharacterStream\"),\n    NCHARACTER(JDBCType.LONGNVARCHAR, \"NCharacterStream\"),\n    SQLXML(JDBCType.SQLXML, \"SQLXML\");\n\n    // JDBC type most naturally associated with this type of stream\n    private final JDBCType jdbcType;\n\n    JDBCType getJDBCType() {\n        return jdbcType;\n    }\n\n    // Display string to use when describing this stream type in traces and error messages\n    private final String name;\n\n    private StreamType(JDBCType jdbcType, String name) {\n        this.jdbcType = jdbcType;\n        this.name = name;\n    }\n\n    @Override\n    public String toString() {\n        return name;\n    }\n\n    boolean convertsFrom(TypeInfo typeInfo) {\n        // Special case handling for ASCII streams:\n        if (ASCII == this) {\n            // Conversion not allowed from XML to AsciiStream\n            if (SSType.XML == typeInfo.getSSType())\n                return false;\n\n            // Conversion not allowed if collation doesn't cleanly convert to ASCII character set\n            if (null != typeInfo.getSQLCollation() && !typeInfo.getSQLCollation().supportsAsciiConversion())\n                return false;\n        }\n\n        return typeInfo.getSSType().convertsTo(jdbcType);\n    }\n\n    boolean convertsTo(TypeInfo typeInfo) {\n        // Special case handling for ASCII streams:\n        if (ASCII == this) {\n            // Conversion not allowed to XML from AsciiStream\n            if (SSType.XML == typeInfo.getSSType())\n                return false;\n\n            // Conversion not allowed if collation doesn't cleanly convert from ASCII character set\n            if (null != typeInfo.getSQLCollation() && !typeInfo.getSQLCollation().supportsAsciiConversion())\n                return false;\n        }\n\n        return jdbcType.convertsTo(typeInfo.getSSType());\n    }\n}\n\n\nfinal class UserTypes {\n    /* System defined UDTs */\n    static final int TIMESTAMP = 0x0050;\n\n    private UserTypes() {} // prevent instantiation\n}\n\n\n/**\n * Java class types that may be used as parameter or column values.\n *\n * Explicit external representation of the Java types eliminates multiple expensive calls to Class.isInstance (from DTV\n * and elsewhere) where the Java type of a parameter or column value needs to be known.\n *\n * !! IMPORTANT !! The tradeoff of using an external representation is that the driver must ensure that the JavaType\n * always reflects the Java type of the object instance, so as a general rule, any code that passes Object instances\n * around where the type must be later known, should always pass a JavaType as well.\n */\nenum JavaType {\n    // !! IMPORTANT !!\n    // JavaType enumeration constants are arranged in the order checked\n    // when determining the type of an arbitrary Object instance.\n    //\n    // Since the type determination process involves calls to Class.isInstance,\n    // the following must be observed:\n    //\n    // 1. Constants MUST be arranged such that subclasses are listed first\n    // to prevent an Object from being incorrectly determined as just an\n    // instance of the superclass.\n    //\n    // 2. Notwithstanding the previous restriction, because the type determination\n    // process involves a linear search, constants SHOULD be arranged\n    // in order of decreasing probability of occurrence.\n    //\n    // 3. The last constant must be for the Object class to ensure that every\n    // type of Object maps to some JavaType.\n    INTEGER(Integer.class, JDBCType.INTEGER),\n    STRING(String.class, JDBCType.CHAR),\n    DATE(java.sql.Date.class, JDBCType.DATE),\n    TIME(java.sql.Time.class, JDBCType.TIME),\n    TIMESTAMP(java.sql.Timestamp.class, JDBCType.TIMESTAMP),\n    UTILDATE(java.util.Date.class, JDBCType.TIMESTAMP),\n    CALENDAR(java.util.Calendar.class, JDBCType.TIMESTAMP),\n    LOCALDATE(getJavaClass(\"LocalDate\"), JDBCType.DATE),\n    LOCALTIME(getJavaClass(\"LocalTime\"), JDBCType.TIME),\n    LOCALDATETIME(getJavaClass(\"LocalDateTime\"), JDBCType.TIMESTAMP),\n    OFFSETTIME(getJavaClass(\"OffsetTime\"), JDBCType.TIME_WITH_TIMEZONE),\n    OFFSETDATETIME(getJavaClass(\"OffsetDateTime\"), JDBCType.TIMESTAMP_WITH_TIMEZONE),\n    DATETIMEOFFSET(microsoft.sql.DateTimeOffset.class, JDBCType.DATETIMEOFFSET),\n    BOOLEAN(Boolean.class, JDBCType.BIT),\n    BIGDECIMAL(BigDecimal.class, JDBCType.DECIMAL),\n    DOUBLE(Double.class, JDBCType.DOUBLE),\n    FLOAT(Float.class, JDBCType.REAL),\n    SHORT(Short.class, JDBCType.SMALLINT),\n    LONG(Long.class, JDBCType.BIGINT),\n    BIGINTEGER(BigInteger.class, JDBCType.BIGINT),\n    BYTE(Byte.class, JDBCType.TINYINT),\n    BYTEARRAY(byte[].class, JDBCType.BINARY),\n    // Check for NClob before checking for Clob, since NClob IS A Clob\n    NCLOB(NClob.class, JDBCType.NCLOB),\n    CLOB(Clob.class, JDBCType.CLOB),\n    BLOB(Blob.class, JDBCType.BLOB),\n    TVP(com.microsoft.sqlserver.jdbc.TVP.class, JDBCType.TVP),\n    GEOMETRY(Geometry.class, JDBCType.GEOMETRY),\n    GEOGRAPHY(Geography.class, JDBCType.GEOGRAPHY),\n\n    INPUTSTREAM(InputStream.class, JDBCType.UNKNOWN) {\n        // InputStreams are either ASCII or binary\n        @Override\n        JDBCType getJDBCType(SSType ssType, JDBCType jdbcTypeFromApp) {\n            JDBCType jdbcType;\n\n            // When the backend type is known, the JDBC type is unknown.\n            // That is, this method is being called from updateObject\n            // rather than setObject.\n            if (SSType.UNKNOWN != ssType) {\n                // If the backend type is known to be textual then assume\n                // that the stream is ASCII. Otherwise, assume that the\n                // stream is binary. In this case XML is NOT considered\n                // to be textual. When updating an XML column from an\n                // InputStream through updateObject, the stream is assumed\n                // to be binary, not ASCII.\n                switch (ssType) {\n                    case CHAR:\n                    case VARCHAR:\n                    case VARCHARMAX:\n                    case TEXT:\n                    case NCHAR:\n                    case NVARCHAR:\n                    case NVARCHARMAX:\n                    case NTEXT:\n                        jdbcType = JDBCType.LONGVARCHAR;\n                        break;\n\n                    case XML:\n                    default:\n                        jdbcType = JDBCType.LONGVARBINARY;\n                        break;\n                }\n            }\n\n            // When the backend type is unknown (i.e. the method is\n            // being called from setObject rather than updateObject),\n            // if the JDBC type is specified as something other\n            // than textual, then assume the stream is binary.\n            else {\n                jdbcType = jdbcTypeFromApp.isTextual() ? JDBCType.LONGVARCHAR : JDBCType.LONGVARBINARY;\n            }\n\n            assert null != jdbcType;\n            return jdbcType;\n        }\n    },\n\n    READER(Reader.class, JDBCType.LONGVARCHAR),\n    // Note: Only SQLServerSQLXML SQLXML instances are accepted by this driver\n    SQLXML(SQLServerSQLXML.class, JDBCType.SQLXML),\n    OBJECT(Object.class, JDBCType.UNKNOWN);\n\n    private final Class<?> javaClass;\n    private final JDBCType jdbcTypeFromJavaType;\n    private static double jvmVersion = 0.0;\n    private static final JavaType[] VALUES = values();\n\n    private JavaType(Class<?> javaClass, JDBCType jdbcTypeFromJavaType) {\n        this.javaClass = javaClass;\n        this.jdbcTypeFromJavaType = jdbcTypeFromJavaType;\n    }\n\n    static Class<?> getJavaClass(String className) {\n        if (0.0 == jvmVersion) {\n            try {\n                /*\n                 * Note: getProperty could throw a SecurityException if there is a security manager that doesn't allow\n                 * checkPropertyAccess. Unlikely to happen & doesn't appear to be a graceful way to handle so will let\n                 * that exception through.\n                 */\n                String jvmSpecVersion = System.getProperty(\"java.specification.version\");\n                if (jvmSpecVersion != null) {\n                    jvmVersion = Double.parseDouble(jvmSpecVersion);\n                }\n            } catch (NumberFormatException e) {\n                // Setting the version to be less that 1.8 so we don't try to set every time.\n                jvmVersion = 0.1;\n            }\n        }\n\n        if (jvmVersion < 1.8) {\n            return null;\n        }\n\n        switch (className) {\n            case \"LocalDate\":\n                return LocalDate.class;\n            case \"LocalTime\":\n                return LocalTime.class;\n            case \"LocalDateTime\":\n                return LocalDateTime.class;\n            case \"OffsetTime\":\n                return OffsetTime.class;\n            case \"OffsetDateTime\":\n                return OffsetDateTime.class;\n            default:\n                return null;\n        }\n    }\n\n    static JavaType of(Object obj) {\n        if (obj instanceof SQLServerDataTable || obj instanceof ResultSet || obj instanceof ISQLServerDataRecord)\n            return JavaType.TVP;\n        if (null != obj) {\n            for (JavaType javaType : VALUES)\n                // if JVM version is prior to Java 8, the javaClass variable can be\n                // null if the java type is introduced in Java 8\n                if (null != javaType.javaClass && javaType.javaClass.isInstance(obj)) {\n                    return javaType;\n                }\n        }\n\n        return JavaType.OBJECT;\n    }\n\n    /**\n     * Returns the JDBC type to use with this Java type. We use the static JDBC type associated with the Java type by\n     * default, ignoring the JDBC type specified by the application. This behavior is overridden for certain Java types,\n     * such as InputStream, which requires the JDBC type to be specified externally in order to distinguish between.\n     */\n    JDBCType getJDBCType(SSType ssType, JDBCType jdbcTypeFromApp) {\n        return jdbcTypeFromJavaType;\n    }\n\n    enum SetterConversionAE {\n        BIT(JavaType.BOOLEAN, EnumSet.of(JDBCType.BIT, JDBCType.TINYINT, JDBCType.SMALLINT, JDBCType.INTEGER,\n                JDBCType.BIGINT)),\n\n        SHORT(JavaType.SHORT, EnumSet.of(JDBCType.TINYINT, JDBCType.SMALLINT, JDBCType.INTEGER, JDBCType.BIGINT)),\n\n        INTEGER(JavaType.INTEGER, EnumSet.of(JDBCType.INTEGER, JDBCType.BIGINT)),\n        LONG(JavaType.LONG, EnumSet.of(JDBCType.BIGINT)),\n\n        BIGDECIMAL(JavaType.BIGDECIMAL, EnumSet.of(JDBCType.MONEY, JDBCType.SMALLMONEY, JDBCType.DECIMAL,\n                JDBCType.NUMERIC)),\n\n        BYTE(JavaType.BYTE, EnumSet.of(JDBCType.BINARY, JDBCType.VARBINARY, JDBCType.LONGVARBINARY, JDBCType.TINYINT)),\n\n        BYTEARRAY(JavaType.BYTEARRAY, EnumSet.of(JDBCType.BINARY, JDBCType.VARBINARY, JDBCType.LONGVARBINARY)),\n\n        DATE(JavaType.DATE, EnumSet.of(JDBCType.DATE)),\n\n        DATETIMEOFFSET(JavaType.DATETIMEOFFSET, EnumSet.of(JDBCType.DATETIMEOFFSET)),\n\n        DOUBLE(JavaType.DOUBLE, EnumSet.of(JDBCType.DOUBLE)),\n\n        FLOAT(JavaType.FLOAT, EnumSet.of(JDBCType.REAL, JDBCType.DOUBLE)),\n\n        STRING(JavaType.STRING, EnumSet.of(JDBCType.CHAR, JDBCType.VARCHAR, JDBCType.LONGVARCHAR, JDBCType.NCHAR,\n                JDBCType.NVARCHAR, JDBCType.LONGNVARCHAR, JDBCType.GUID)),\n\n        TIME(JavaType.TIME, EnumSet.of(JDBCType.TIME)),\n\n        TIMESTAMP(JavaType.TIMESTAMP, EnumSet.of(JDBCType.TIME, // This is needed to send nanoseconds to the driver as\n                                                                // setTime() is only milliseconds\n                JDBCType.TIMESTAMP, // This is datetime2\n                JDBCType.DATETIME, JDBCType.SMALLDATETIME));\n\n        private final EnumSet<JDBCType> to;\n        private final JavaType from;\n        private static final SetterConversionAE[] VALUES = values();\n\n        private SetterConversionAE(JavaType from, EnumSet<JDBCType> to) {\n            this.from = from;\n            this.to = to;\n        }\n\n        private static final EnumMap<JavaType, EnumSet<JDBCType>> setterConversionAEMap = new EnumMap<>(JavaType.class);\n\n        static {\n            for (JavaType javaType : JavaType.VALUES)\n                setterConversionAEMap.put(javaType, EnumSet.noneOf(JDBCType.class));\n\n            for (SetterConversionAE conversion : VALUES)\n                setterConversionAEMap.get(conversion.from).addAll(conversion.to);\n        }\n\n        static boolean converts(JavaType fromJavaType, JDBCType toJDBCType, boolean sendStringParametersAsUnicode) {\n            if ((null == fromJavaType) || (JavaType.OBJECT == fromJavaType))\n                return true;\n            else if (!sendStringParametersAsUnicode && fromJavaType == JavaType.BYTEARRAY\n                    && (toJDBCType == JDBCType.VARCHAR || toJDBCType == JDBCType.CHAR\n                            || toJDBCType == JDBCType.LONGVARCHAR)) {\n                // when column is encrypted and sendStringParametersAsUnicode is false,\n                // does not throw exception if the column is char/varchar/varcharmax,\n                // in order to allow send char/varchar/varcharmax as MBCS (BYTEARRAY type)\n                return true;\n            } else if (!setterConversionAEMap.containsKey(fromJavaType))\n                return false;\n            return setterConversionAEMap.get(fromJavaType).contains(toJDBCType);\n        }\n    }\n\n}\n\n\nenum JDBCType {\n    UNKNOWN(Category.UNKNOWN, 999, Object.class.getName()),\n    ARRAY(Category.UNKNOWN, java.sql.Types.ARRAY, Object.class.getName()),\n    BIGINT(Category.NUMERIC, java.sql.Types.BIGINT, Long.class.getName()),\n    BINARY(Category.BINARY, java.sql.Types.BINARY, \"[B\"),\n    BIT(Category.NUMERIC, java.sql.Types.BIT, Boolean.class.getName()),\n    BLOB(Category.BLOB, java.sql.Types.BLOB, java.sql.Blob.class.getName()),\n    BOOLEAN(Category.NUMERIC, java.sql.Types.BOOLEAN, Boolean.class.getName()),\n    CHAR(Category.CHARACTER, java.sql.Types.CHAR, String.class.getName()),\n    CLOB(Category.CLOB, java.sql.Types.CLOB, java.sql.Clob.class.getName()),\n    DATALINK(Category.UNKNOWN, java.sql.Types.DATALINK, Object.class.getName()),\n    DATE(Category.DATE, java.sql.Types.DATE, java.sql.Date.class.getName()),\n    DATETIMEOFFSET(Category.DATETIMEOFFSET, microsoft.sql.Types.DATETIMEOFFSET, microsoft.sql.DateTimeOffset.class\n            .getName()),\n    DECIMAL(Category.NUMERIC, java.sql.Types.DECIMAL, BigDecimal.class.getName()),\n    DISTINCT(Category.UNKNOWN, java.sql.Types.DISTINCT, Object.class.getName()),\n    DOUBLE(Category.NUMERIC, java.sql.Types.DOUBLE, Double.class.getName()),\n    FLOAT(Category.NUMERIC, java.sql.Types.FLOAT, Double.class.getName()),\n    INTEGER(Category.NUMERIC, java.sql.Types.INTEGER, Integer.class.getName()),\n    JAVA_OBJECT(Category.UNKNOWN, java.sql.Types.JAVA_OBJECT, Object.class.getName()),\n    LONGNVARCHAR(Category.LONG_NCHARACTER, -16, String.class.getName()),\n    LONGVARBINARY(Category.LONG_BINARY, java.sql.Types.LONGVARBINARY, \"[B\"),\n    LONGVARCHAR(Category.LONG_CHARACTER, java.sql.Types.LONGVARCHAR, String.class.getName()),\n    NCHAR(Category.NCHARACTER, -15, String.class.getName()),\n    NCLOB(Category.NCLOB, 2011, java.sql.NClob.class.getName()),\n    NULL(Category.UNKNOWN, java.sql.Types.NULL, Object.class.getName()),\n    NUMERIC(Category.NUMERIC, java.sql.Types.NUMERIC, BigDecimal.class.getName()),\n    NVARCHAR(Category.NCHARACTER, -9, String.class.getName()),\n    OTHER(Category.UNKNOWN, java.sql.Types.OTHER, Object.class.getName()),\n    REAL(Category.NUMERIC, java.sql.Types.REAL, Float.class.getName()),\n    REF(Category.UNKNOWN, java.sql.Types.REF, Object.class.getName()),\n    ROWID(Category.UNKNOWN, -8, Object.class.getName()),\n    SMALLINT(Category.NUMERIC, java.sql.Types.SMALLINT, Short.class.getName()),\n    SQLXML(Category.SQLXML, 2009, Object.class.getName()),\n    STRUCT(Category.UNKNOWN, java.sql.Types.STRUCT, Object.class.getName()),\n    TIME(Category.TIME, java.sql.Types.TIME, java.sql.Time.class.getName()),\n    TIME_WITH_TIMEZONE(Category.TIME_WITH_TIMEZONE, 2013, java.time.OffsetTime.class.getName()),\n    TIMESTAMP(Category.TIMESTAMP, java.sql.Types.TIMESTAMP, java.sql.Timestamp.class.getName()),\n    TIMESTAMP_WITH_TIMEZONE(Category.TIMESTAMP_WITH_TIMEZONE, 2014, java.time.OffsetDateTime.class.getName()),\n    TINYINT(Category.NUMERIC, java.sql.Types.TINYINT, Short.class.getName()),\n    VARBINARY(Category.BINARY, java.sql.Types.VARBINARY, \"[B\"),\n    VARCHAR(Category.CHARACTER, java.sql.Types.VARCHAR, String.class.getName()),\n    MONEY(Category.NUMERIC, microsoft.sql.Types.MONEY, BigDecimal.class.getName()),\n    SMALLMONEY(Category.NUMERIC, microsoft.sql.Types.SMALLMONEY, BigDecimal.class.getName()),\n    TVP(Category.TVP, microsoft.sql.Types.STRUCTURED, Object.class.getName()),\n    DATETIME(Category.TIMESTAMP, microsoft.sql.Types.DATETIME, java.sql.Timestamp.class.getName()),\n    SMALLDATETIME(Category.TIMESTAMP, microsoft.sql.Types.SMALLDATETIME, java.sql.Timestamp.class.getName()),\n    GUID(Category.CHARACTER, microsoft.sql.Types.GUID, String.class.getName()),\n    SQL_VARIANT(Category.SQL_VARIANT, microsoft.sql.Types.SQL_VARIANT, Object.class.getName()),\n    GEOMETRY(Category.GEOMETRY, microsoft.sql.Types.GEOMETRY, Object.class.getName()),\n    GEOGRAPHY(Category.GEOGRAPHY, microsoft.sql.Types.GEOGRAPHY, Object.class.getName()),\n    LOCALDATETIME(Category.TIMESTAMP, java.sql.Types.TIMESTAMP, LocalDateTime.class.getName());\n\n    final Category category;\n    private final int intValue;\n    private final String className;\n    private static final JDBCType[] VALUES = values();\n\n    final String className() {\n        return className;\n    }\n\n    private JDBCType(Category category, int intValue, String className) {\n        this.category = category;\n        this.intValue = intValue;\n        this.className = className;\n    }\n\n    /**\n     * Returns the integer value of JDBCType.\n     * \n     * @return integer representation of JDBCType\n     */\n    public int getIntValue() {\n        return this.intValue;\n    }\n\n    enum Category {\n        CHARACTER,\n        LONG_CHARACTER,\n        CLOB,\n        NCHARACTER,\n        LONG_NCHARACTER,\n        NCLOB,\n        BINARY,\n        LONG_BINARY,\n        BLOB,\n        NUMERIC,\n        DATE,\n        TIME,\n        TIMESTAMP,\n        TIME_WITH_TIMEZONE,\n        TIMESTAMP_WITH_TIMEZONE,\n        DATETIMEOFFSET,\n        SQLXML,\n        UNKNOWN,\n        TVP,\n        GUID,\n        SQL_VARIANT,\n        GEOMETRY,\n        GEOGRAPHY;\n\n        private static final Category[] VALUES = values();\n    }\n\n    // This SetterConversion enum is based on the Category enum\n    enum SetterConversion {\n        CHARACTER(JDBCType.Category.CHARACTER, EnumSet.of(JDBCType.Category.NUMERIC, JDBCType.Category.DATE,\n                JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP, JDBCType.Category.DATETIMEOFFSET,\n                JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER,\n                JDBCType.Category.LONG_NCHARACTER, JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY,\n                JDBCType.Category.GUID, JDBCType.Category.SQL_VARIANT)),\n\n        LONG_CHARACTER(JDBCType.Category.LONG_CHARACTER, EnumSet.of(JDBCType.Category.CHARACTER,\n                JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER,\n                JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY)),\n\n        CLOB(JDBCType.Category.CLOB, EnumSet.of(JDBCType.Category.CLOB, JDBCType.Category.LONG_CHARACTER,\n                JDBCType.Category.LONG_NCHARACTER)),\n\n        NCHARACTER(JDBCType.Category.NCHARACTER, EnumSet.of(JDBCType.Category.NCHARACTER,\n                JDBCType.Category.LONG_NCHARACTER, JDBCType.Category.NCLOB, JDBCType.Category.SQL_VARIANT)),\n\n        LONG_NCHARACTER(JDBCType.Category.LONG_NCHARACTER, EnumSet.of(JDBCType.Category.NCHARACTER,\n                JDBCType.Category.LONG_NCHARACTER)),\n\n        NCLOB(JDBCType.Category.NCLOB, EnumSet.of(JDBCType.Category.LONG_NCHARACTER, JDBCType.Category.NCLOB)),\n\n        BINARY(JDBCType.Category.BINARY, EnumSet.of(JDBCType.Category.NUMERIC, JDBCType.Category.DATE,\n                JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP, JDBCType.Category.CHARACTER,\n                JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER,\n                JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY, JDBCType.Category.BLOB, JDBCType.Category.GUID,\n                JDBCType.Category.SQL_VARIANT)),\n\n        LONG_BINARY(JDBCType.Category.LONG_BINARY, EnumSet.of(JDBCType.Category.BINARY, JDBCType.Category.LONG_BINARY)),\n\n        BLOB(JDBCType.Category.BLOB, EnumSet.of(JDBCType.Category.LONG_BINARY, JDBCType.Category.BLOB)),\n\n        NUMERIC(JDBCType.Category.NUMERIC, EnumSet.of(JDBCType.Category.NUMERIC, JDBCType.Category.CHARACTER,\n                JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER,\n                JDBCType.Category.SQL_VARIANT)),\n\n        DATE(JDBCType.Category.DATE, EnumSet.of(JDBCType.Category.DATE, JDBCType.Category.TIMESTAMP,\n                JDBCType.Category.DATETIMEOFFSET, JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER,\n                JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER, JDBCType.Category.SQL_VARIANT)),\n\n        TIME(JDBCType.Category.TIME, EnumSet.of(JDBCType.Category.TIME, JDBCType.Category.TIMESTAMP,\n                JDBCType.Category.DATETIMEOFFSET, JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER,\n                JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER, JDBCType.Category.SQL_VARIANT)),\n\n        TIMESTAMP(JDBCType.Category.TIMESTAMP, EnumSet.of(JDBCType.Category.DATE, JDBCType.Category.TIME,\n                JDBCType.Category.TIMESTAMP, JDBCType.Category.DATETIMEOFFSET, JDBCType.Category.CHARACTER,\n                JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER, JDBCType.Category.LONG_NCHARACTER,\n                JDBCType.Category.SQL_VARIANT)),\n\n        TIME_WITH_TIMEZONE(JDBCType.Category.TIME_WITH_TIMEZONE, EnumSet.of(JDBCType.Category.TIME_WITH_TIMEZONE,\n                JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER,\n                JDBCType.Category.LONG_NCHARACTER)),\n\n        TIMESTAMP_WITH_TIMEZONE(JDBCType.Category.TIMESTAMP_WITH_TIMEZONE, EnumSet.of(\n                JDBCType.Category.TIMESTAMP_WITH_TIMEZONE, JDBCType.Category.TIME_WITH_TIMEZONE,\n                JDBCType.Category.CHARACTER, JDBCType.Category.LONG_CHARACTER, JDBCType.Category.NCHARACTER,\n                JDBCType.Category.LONG_NCHARACTER)),\n\n        DATETIMEOFFSET(JDBCType.Category.DATETIMEOFFSET, EnumSet.of(JDBCType.Category.DATE, JDBCType.Category.TIME,\n                JDBCType.Category.TIMESTAMP, JDBCType.Category.DATETIMEOFFSET)),\n\n        SQLXML(JDBCType.Category.SQLXML, EnumSet.of(JDBCType.Category.SQLXML)),\n\n        TVP(JDBCType.Category.TVP, EnumSet.of(JDBCType.Category.TVP)),\n\n        GEOMETRY(JDBCType.Category.GEOMETRY, EnumSet.of(JDBCType.Category.GEOMETRY)),\n\n        GEOGRAPHY(JDBCType.Category.GEOGRAPHY, EnumSet.of(JDBCType.Category.GEOGRAPHY));\n\n        private final JDBCType.Category from;\n        private final EnumSet<JDBCType.Category> to;\n        private static final SetterConversion[] VALUES = values();\n\n        private SetterConversion(JDBCType.Category from, EnumSet<JDBCType.Category> to) {\n            this.from = from;\n            this.to = to;\n        }\n\n        private static final EnumMap<JDBCType.Category, EnumSet<JDBCType.Category>> conversionMap = new EnumMap<>(\n                JDBCType.Category.class);\n\n        static {\n            for (JDBCType.Category category : JDBCType.Category.VALUES)\n                conversionMap.put(category, EnumSet.noneOf(JDBCType.Category.class));\n\n            for (SetterConversion conversion : VALUES)\n                conversionMap.get(conversion.from).addAll(conversion.to);\n        }\n\n        static boolean converts(JDBCType fromJDBCType, JDBCType toJDBCType) {\n            return conversionMap.get(fromJDBCType.category).contains(toJDBCType.category);\n        }\n    }\n\n    boolean convertsTo(JDBCType jdbcType) {\n        return SetterConversion.converts(this, jdbcType);\n    }\n\n    enum UpdaterConversion {\n        CHARACTER(JDBCType.Category.CHARACTER, EnumSet.of(SSType.Category.NUMERIC, SSType.Category.DATE,\n                SSType.Category.TIME, SSType.Category.DATETIME, SSType.Category.DATETIME2,\n                SSType.Category.DATETIMEOFFSET, SSType.Category.CHARACTER, SSType.Category.LONG_CHARACTER,\n                SSType.Category.NCHARACTER, SSType.Category.LONG_NCHARACTER, SSType.Category.XML,\n                SSType.Category.BINARY, SSType.Category.LONG_BINARY, SSType.Category.UDT, SSType.Category.GUID,\n                SSType.Category.TIMESTAMP, SSType.Category.SQL_VARIANT)),\n\n        LONG_CHARACTER(JDBCType.Category.LONG_CHARACTER, EnumSet.of(SSType.Category.CHARACTER,\n                SSType.Category.LONG_CHARACTER, SSType.Category.NCHARACTER, SSType.Category.LONG_NCHARACTER,\n                SSType.Category.XML, SSType.Category.BINARY, SSType.Category.LONG_BINARY)),\n\n        CLOB(JDBCType.Category.CLOB, EnumSet.of(SSType.Category.LONG_CHARACTER, SSType.Category.LONG_NCHARACTER,\n                SSType.Category.XML)),\n\n        NCHARACTER(JDBCType.Category.NCHARACTER, EnumSet.of(SSType.Category.NCHARACTER, SSType.Category.LONG_NCHARACTER,\n                SSType.Category.XML, SSType.Category.SQL_VARIANT)),\n\n        LONG_NCHARACTER(JDBCType.Category.LONG_NCHARACTER, EnumSet.of(SSType.Category.NCHARACTER,\n                SSType.Category.LONG_NCHARACTER, SSType.Category.XML)),\n\n        NCLOB(JDBCType.Category.NCLOB, EnumSet.of(SSType.Category.LONG_NCHARACTER, SSType.Category.XML)),\n\n        BINARY(JDBCType.Category.BINARY, EnumSet.of(SSType.Category.NUMERIC, SSType.Category.DATETIME,\n                SSType.Category.CHARACTER, SSType.Category.LONG_CHARACTER, SSType.Category.NCHARACTER,\n                SSType.Category.LONG_NCHARACTER, SSType.Category.XML, SSType.Category.BINARY,\n                SSType.Category.LONG_BINARY, SSType.Category.UDT, SSType.Category.TIMESTAMP, SSType.Category.GUID,\n                SSType.Category.SQL_VARIANT)),\n\n        LONG_BINARY(JDBCType.Category.LONG_BINARY, EnumSet.of(SSType.Category.XML, SSType.Category.BINARY,\n                SSType.Category.LONG_BINARY, SSType.Category.UDT)),\n\n        BLOB(JDBCType.Category.BLOB, EnumSet.of(SSType.Category.LONG_BINARY, SSType.Category.XML)),\n        SQLXML(JDBCType.Category.SQLXML, EnumSet.of(SSType.Category.XML)),\n\n        NUMERIC(JDBCType.Category.NUMERIC, EnumSet.of(SSType.Category.NUMERIC, SSType.Category.CHARACTER,\n                SSType.Category.LONG_CHARACTER, SSType.Category.NCHARACTER, SSType.Category.LONG_NCHARACTER,\n                SSType.Category.SQL_VARIANT)),\n\n...\n", "needle_token_start": 707, "needle_token_end": 896, "code_context_ntokens": 16143, "output": ["```java\nstatic String checkAndAppendClientConnId(String errMsg, SQLServerConnection conn) {\n    if (null != conn && conn.isConnected()) {\n        UUID clientConnId = conn.getClientConIdInternal();\n        if (null != clientConnId) {\n            StringBuilder sb = (errMsg != null) ? new StringBuilder(errMsg) : new StringBuilder();\n            // This syntax of adding connection id is matched in a retry logic. If anything changes here, make\n            // necessary changes to enableSSL() function's exception handling mechanism.\n            sb.append(LOG_CLIENT_CONNECTION_ID_PREFIX);\n            sb.append(clientConnId.toString());\n            return sb.toString();\n        }\n    }\n    return (errMsg != null) ? errMsg : \"\";\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "writeEncryptedRPCTime", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to securely transmit a time value in an encrypted form as part of a remote procedure call (RPC) to a SQL Server, ensuring the confidentiality of the data.\n2. **Input**: The function takes a calendar object representing the time to be encrypted, nanoseconds to add precision, a scale to define the precision of the time value, a boolean indicating if the parameter is an output, and a statement object for SQL operations.\n3. **Output**: There is no direct return value; however, the function writes encrypted time data to a network stream or buffer for SQL Server communication.\n4. **Procedure**: \n   - Initially, the function checks if the configuration allows sending time as datetime, throwing an exception if not permitted.\n   - It sets the parameter type for the RPC.\n   - If the calendar object is null, it writes a null value; otherwise, it encrypts the time data (including nanoseconds and scale adjustments) and writes this encrypted data.\n   - Finally, it writes metadata related to the encryption and the time type to the stream or buffer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n        if (null != cryptoMeta) {\n            // send encrypted data as BIGVARBINARY\n            tdsType = (isShortValue || usePLP) ? TDSType.BIGVARBINARY : TDSType.IMAGE;\n            collation = null;\n        } else\n            switch (jdbcType) {\n                case CHAR:\n                case VARCHAR:\n                case LONGVARCHAR:\n                case CLOB:\n                    tdsType = (isShortValue || usePLP) ? TDSType.BIGVARCHAR : TDSType.TEXT;\n                    if (null == collation)\n                        collation = con.getDatabaseCollation();\n                    break;\n\n                case NCHAR:\n                case NVARCHAR:\n                case LONGNVARCHAR:\n                case NCLOB:\n                    tdsType = (isShortValue || usePLP) ? TDSType.NVARCHAR : TDSType.NTEXT;\n                    if (null == collation)\n                        collation = con.getDatabaseCollation();\n                    break;\n\n                case BINARY:\n                case VARBINARY:\n                case LONGVARBINARY:\n                case BLOB:\n                default:\n                    tdsType = (isShortValue || usePLP) ? TDSType.BIGVARBINARY : TDSType.IMAGE;\n                    collation = null;\n                    break;\n            }\n\n        writeRPCNameValType(sName, bOut, tdsType);\n\n        if (usePLP && !isNonPLP) {\n            writeVMaxHeader(nValueLen, bValueNull, collation);\n\n            // Send the data.\n            if (!bValueNull) {\n                if (nValueLen > 0) {\n                    writeInt(nValueLen);\n                    writeBytes(bValue);\n                }\n\n                // Send the terminator PLP chunk.\n                writeInt(0);\n            }\n        } else // non-PLP type\n        {\n            // Handle Shiloh types here.\n            if (isShortValue) {\n                writeShort((short) DataTypes.SHORT_VARTYPE_MAX_BYTES);\n            } else {\n                writeInt(DataTypes.IMAGE_TEXT_MAX_BYTES);\n            }\n\n            if (null != collation)\n                collation.writeCollation(this);\n\n            // Data and length\n            if (bValueNull) {\n                writeShort((short) -1); // actual len\n            } else {\n                if (isShortValue)\n                    writeShort((short) nValueLen); // actual len\n                else\n                    writeInt(nValueLen); // actual len\n\n                // If length is zero, we're done.\n                if (0 != nValueLen)\n                    writeBytes(bValue);\n            }\n        }\n    }\n\n    /**\n     * Append a timestamp in RPC transmission format as a SQL Server DATETIME data type\n     * \n     * @param sName\n     *        the optional parameter name\n     * @param cal\n     *        Pure Gregorian calendar containing the timestamp, including its associated time zone\n     * @param subSecondNanos\n     *        the sub-second nanoseconds (0 - 999,999,999)\n     * @param bOut\n     *        boolean true if the data value is being registered as an output parameter\n     *\n     */\n    void writeRPCDateTime(String sName, GregorianCalendar cal, int subSecondNanos,\n            boolean bOut) throws SQLServerException {\n        assert (subSecondNanos >= 0) && (subSecondNanos < Nanos.PER_SECOND) : \"Invalid subNanoSeconds value: \"\n                + subSecondNanos;\n        assert (cal != null) || (subSecondNanos == 0) : \"Invalid subNanoSeconds value when calendar is null: \"\n                + subSecondNanos;\n\n        writeRPCNameValType(sName, bOut, TDSType.DATETIMEN);\n        writeByte((byte) 8); // max length of datatype\n\n        if (null == cal) {\n            writeByte((byte) 0); // len of data bytes\n            return;\n        }\n\n        writeByte((byte) 8); // len of data bytes\n\n        // We need to extract the Calendar's current date & time in terms\n        // of the number of days since the SQL Base Date (1/1/1900) plus\n        // the number of milliseconds since midnight in the current day.\n        //\n        // We cannot rely on any pre-calculated value for the number of\n        // milliseconds in a day or the number of milliseconds since the\n        // base date to do this because days with DST changes are shorter\n        // or longer than \"normal\" days.\n        //\n        // ASSUMPTION: We assume we are dealing with a GregorianCalendar here.\n        // If not, we have no basis in which to compare dates. E.g. if we\n        // are dealing with a Chinese Calendar implementation which does not\n        // use the same value for Calendar.YEAR as the GregorianCalendar,\n        // we cannot meaningfully compute a value relative to 1/1/1900.\n\n        // First, figure out how many days there have been since the SQL Base Date.\n        // These are based on SQL Server algorithms\n        int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(cal.get(Calendar.YEAR), cal.get(Calendar.DAY_OF_YEAR),\n                TDS.BASE_YEAR_1900);\n\n        // Next, figure out the number of milliseconds since midnight of the current day.\n        int millisSinceMidnight = (subSecondNanos + Nanos.PER_MILLISECOND / 2) / Nanos.PER_MILLISECOND + // Millis into\n                                                                                                         // the current\n                                                                                                         // second\n                1000 * cal.get(Calendar.SECOND) + // Seconds into the current minute\n                60 * 1000 * cal.get(Calendar.MINUTE) + // Minutes into the current hour\n                60 * 60 * 1000 * cal.get(Calendar.HOUR_OF_DAY); // Hours into the current day\n\n        // The last millisecond of the current day is always rounded to the first millisecond\n        // of the next day because DATETIME is only accurate to 1/300th of a second.\n        if (millisSinceMidnight >= 1000 * 60 * 60 * 24 - 1) {\n            ++daysSinceSQLBaseDate;\n            millisSinceMidnight = 0;\n        }\n\n        // Last-ditch verification that the value is in the valid range for the\n        // DATETIMEN TDS data type (1/1/1753 to 12/31/9999). If it's not, then\n        // throw an exception now so that statement execution is safely canceled.\n        // Attempting to put an invalid value on the wire would result in a TDS\n        // exception, which would close the connection.\n        // These are based on SQL Server algorithms\n        if (daysSinceSQLBaseDate < DDC.daysSinceBaseDate(1753, 1, TDS.BASE_YEAR_1900)\n                || daysSinceSQLBaseDate >= DDC.daysSinceBaseDate(10000, 1, TDS.BASE_YEAR_1900)) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_valueOutOfRange\"));\n            Object[] msgArgs = {SSType.DATETIME};\n            throw new SQLServerException(form.format(msgArgs), SQLState.DATA_EXCEPTION_DATETIME_FIELD_OVERFLOW,\n                    DriverError.NOT_SET, null);\n        }\n\n        // And put it all on the wire...\n\n        // Number of days since the SQL Server Base Date (January 1, 1900)\n        writeInt(daysSinceSQLBaseDate);\n\n        // Milliseconds since midnight (at a resolution of three hundredths of a second)\n        writeInt((3 * millisSinceMidnight + 5) / 10);\n    }\n\n    void writeRPCTime(String sName, GregorianCalendar localCalendar, int subSecondNanos, int scale,\n            boolean bOut) throws SQLServerException {\n        writeRPCNameValType(sName, bOut, TDSType.TIMEN);\n        writeByte((byte) scale);\n\n        if (null == localCalendar) {\n            writeByte((byte) 0);\n            return;\n        }\n\n        writeByte((byte) TDS.timeValueLength(scale));\n        writeScaledTemporal(localCalendar, subSecondNanos, scale, SSType.TIME);\n    }\n\n    void writeRPCDate(String sName, GregorianCalendar localCalendar, boolean bOut) throws SQLServerException {\n        writeRPCNameValType(sName, bOut, TDSType.DATEN);\n        if (null == localCalendar) {\n            writeByte((byte) 0);\n            return;\n        }\n\n        writeByte((byte) TDS.DAYS_INTO_CE_LENGTH);\n        writeScaledTemporal(localCalendar, 0, // subsecond nanos (none for a date value)\n                0, // scale (dates are not scaled)\n                SSType.DATE);\n    }\n\n    \nvoid writeEncryptedRPCTime(String sName, GregorianCalendar localCalendar, int subSecondNanos, int scale,\n            boolean bOut, SQLServerStatement statement) throws SQLServerException {\n        if (con.getSendTimeAsDatetime()) {\n            throw new SQLServerException(SQLServerException.getErrString(\"R_sendTimeAsDateTimeForAE\"), null);\n        }\n        writeRPCNameValType(sName, bOut, TDSType.BIGVARBINARY);\n\n        if (null == localCalendar)\n            writeEncryptedRPCByteArray(null);\n        else\n            writeEncryptedRPCByteArray(writeEncryptedScaledTemporal(localCalendar, subSecondNanos, scale, SSType.TIME,\n                    (short) 0, statement));\n\n        writeByte(TDSType.TIMEN.byteValue());\n        writeByte((byte) scale);\n        writeCryptoMetaData();\n    }\n\n    void writeEncryptedRPCDate(String sName, GregorianCalendar localCalendar, boolean bOut,\n            SQLServerStatement statement) throws SQLServerException {\n        writeRPCNameValType(sName, bOut, TDSType.BIGVARBINARY);\n\n        if (null == localCalendar)\n            writeEncryptedRPCByteArray(null);\n        else\n            writeEncryptedRPCByteArray(writeEncryptedScaledTemporal(localCalendar, 0, // subsecond nanos (none for a\n                                                                                      // date value)\n                    0, // scale (dates are not scaled)\n                    SSType.DATE, (short) 0, statement));\n\n        writeByte(TDSType.DATEN.byteValue());\n        writeCryptoMetaData();\n    }\n\n    void writeEncryptedRPCDateTime(String sName, GregorianCalendar cal, int subSecondNanos, boolean bOut,\n            JDBCType jdbcType, SQLServerStatement statement) throws SQLServerException {\n        assert (subSecondNanos >= 0) && (subSecondNanos < Nanos.PER_SECOND) : \"Invalid subNanoSeconds value: \"\n                + subSecondNanos;\n        assert (cal != null) || (subSecondNanos == 0) : \"Invalid subNanoSeconds value when calendar is null: \"\n                + subSecondNanos;\n\n        writeRPCNameValType(sName, bOut, TDSType.BIGVARBINARY);\n\n        if (null == cal)\n            writeEncryptedRPCByteArray(null);\n        else\n            writeEncryptedRPCByteArray(getEncryptedDateTimeAsBytes(cal, subSecondNanos, jdbcType, statement));\n\n        if (JDBCType.SMALLDATETIME == jdbcType) {\n            writeByte(TDSType.DATETIMEN.byteValue());\n            writeByte((byte) 4);\n        } else {\n            writeByte(TDSType.DATETIMEN.byteValue());\n            writeByte((byte) 8);\n        }\n        writeCryptoMetaData();\n    }\n\n    // getEncryptedDateTimeAsBytes is called if jdbcType/ssType is SMALLDATETIME or DATETIME\n    byte[] getEncryptedDateTimeAsBytes(GregorianCalendar cal, int subSecondNanos, JDBCType jdbcType,\n            SQLServerStatement statement) throws SQLServerException {\n        int daysSinceSQLBaseDate = DDC.daysSinceBaseDate(cal.get(Calendar.YEAR), cal.get(Calendar.DAY_OF_YEAR),\n                TDS.BASE_YEAR_1900);\n\n        // Next, figure out the number of milliseconds since midnight of the current day.\n        int millisSinceMidnight = (subSecondNanos + Nanos.PER_MILLISECOND / 2) / Nanos.PER_MILLISECOND + // Millis into\n                                                                                                         // the current\n                                                                                                         // second\n                1000 * cal.get(Calendar.SECOND) + // Seconds into the current minute\n                60 * 1000 * cal.get(Calendar.MINUTE) + // Minutes into the current hour\n                60 * 60 * 1000 * cal.get(Calendar.HOUR_OF_DAY); // Hours into the current day\n\n        // The last millisecond of the current day is always rounded to the first millisecond\n        // of the next day because DATETIME is only accurate to 1/300th of a second.\n        if (millisSinceMidnight >= 1000 * 60 * 60 * 24 - 1) {\n            ++daysSinceSQLBaseDate;\n            millisSinceMidnight = 0;\n        }\n\n        if (JDBCType.SMALLDATETIME == jdbcType) {\n\n            int secondsSinceMidnight = (millisSinceMidnight / 1000);\n            int minutesSinceMidnight = (secondsSinceMidnight / 60);\n\n            // Values that are 29.998 seconds or less are rounded down to the nearest minute\n            minutesSinceMidnight = ((secondsSinceMidnight % 60) > 29.998) ? minutesSinceMidnight + 1\n                                                                          : minutesSinceMidnight;\n\n            // minutesSinceMidnight for (23:59:30)\n            int maxMinutesSinceMidnightSmallDateTime = 1440;\n            // Verification for smalldatetime to be within valid range of (1900.01.01) to (2079.06.06)\n            // smalldatetime for unencrypted does not allow insertion of 2079.06.06 23:59:59 and it is rounded up\n            // to 2079.06.07 00:00:00, therefore, we are checking minutesSinceMidnight for that condition. If it's not\n            // within valid range, then\n            // throw an exception now so that statement execution is safely canceled.\n            // 157 is the calculated day of year from 06-06 , 1440 is minutesince midnight for (23:59:30)\n            if ((daysSinceSQLBaseDate < DDC.daysSinceBaseDate(1900, 1, TDS.BASE_YEAR_1900)\n                    || daysSinceSQLBaseDate > DDC.daysSinceBaseDate(2079, 157, TDS.BASE_YEAR_1900))\n                    || (daysSinceSQLBaseDate == DDC.daysSinceBaseDate(2079, 157, TDS.BASE_YEAR_1900)\n                            && minutesSinceMidnight >= maxMinutesSinceMidnightSmallDateTime)) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_valueOutOfRange\"));\n                Object[] msgArgs = {SSType.SMALLDATETIME};\n                throw new SQLServerException(form.format(msgArgs), SQLState.DATA_EXCEPTION_DATETIME_FIELD_OVERFLOW,\n                        DriverError.NOT_SET, null);\n            }\n\n            ByteBuffer days = ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN);\n            days.putShort((short) daysSinceSQLBaseDate);\n            ByteBuffer seconds = ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN);\n            seconds.putShort((short) minutesSinceMidnight);\n\n            byte[] value = new byte[4];\n            System.arraycopy(days.array(), 0, value, 0, 2);\n            System.arraycopy(seconds.array(), 0, value, 2, 2);\n            return SQLServerSecurityUtility.encryptWithKey(value, cryptoMeta, con, statement);\n        } else if (JDBCType.DATETIME == jdbcType) {\n            // Last-ditch verification that the value is in the valid range for the\n            // DATETIMEN TDS data type (1/1/1753 to 12/31/9999). If it's not, then\n            // throw an exception now so that statement execution is safely canceled.\n            // Attempting to put an invalid value on the wire would result in a TDS\n            // exception, which would close the connection.\n            // These are based on SQL Server algorithms\n            // And put it all on the wire...\n            if (daysSinceSQLBaseDate < DDC.daysSinceBaseDate(1753, 1, TDS.BASE_YEAR_1900)\n                    || daysSinceSQLBaseDate >= DDC.daysSinceBaseDate(10000, 1, TDS.BASE_YEAR_1900)) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_valueOutOfRange\"));\n                Object[] msgArgs = {SSType.DATETIME};\n                throw new SQLServerException(form.format(msgArgs), SQLState.DATA_EXCEPTION_DATETIME_FIELD_OVERFLOW,\n                        DriverError.NOT_SET, null);\n            }\n\n            // Number of days since the SQL Server Base Date (January 1, 1900)\n            ByteBuffer days = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);\n            days.putInt(daysSinceSQLBaseDate);\n            ByteBuffer seconds = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);\n            seconds.putInt((3 * millisSinceMidnight + 5) / 10);\n\n            byte[] value = new byte[8];\n            System.arraycopy(days.array(), 0, value, 0, 4);\n            System.arraycopy(seconds.array(), 0, value, 4, 4);\n            return SQLServerSecurityUtility.encryptWithKey(value, cryptoMeta, con, statement);\n        }\n\n        assert false : \"Unexpected JDBCType type \" + jdbcType;\n        return null;\n    }\n\n    void writeEncryptedRPCDateTime2(String sName, GregorianCalendar localCalendar, int subSecondNanos, int scale,\n            boolean bOut, SQLServerStatement statement) throws SQLServerException {\n        writeRPCNameValType(sName, bOut, TDSType.BIGVARBINARY);\n\n        if (null == localCalendar)\n            writeEncryptedRPCByteArray(null);\n        else\n            writeEncryptedRPCByteArray(writeEncryptedScaledTemporal(localCalendar, subSecondNanos, scale,\n                    SSType.DATETIME2, (short) 0, statement));\n\n        writeByte(TDSType.DATETIME2N.byteValue());\n        writeByte((byte) (scale));\n        writeCryptoMetaData();\n    }\n\n    void writeEncryptedRPCDateTimeOffset(String sName, GregorianCalendar utcCalendar, int minutesOffset,\n            int subSecondNanos, int scale, boolean bOut, SQLServerStatement statement) throws SQLServerException {\n        writeRPCNameValType(sName, bOut, TDSType.BIGVARBINARY);\n\n        if (null == utcCalendar)\n            writeEncryptedRPCByteArray(null);\n        else {\n            assert 0 == utcCalendar.get(Calendar.ZONE_OFFSET);\n            writeEncryptedRPCByteArray(writeEncryptedScaledTemporal(utcCalendar, subSecondNanos, scale,\n                    SSType.DATETIMEOFFSET, (short) minutesOffset, statement));\n        }\n\n        writeByte(TDSType.DATETIMEOFFSETN.byteValue());\n        writeByte((byte) (scale));\n        writeCryptoMetaData();\n\n    }\n\n    void writeRPCDateTime2(String sName, GregorianCalendar localCalendar, int subSecondNanos, int scale,\n            boolean bOut) throws SQLServerException {\n        writeRPCNameValType(sName, bOut, TDSType.DATETIME2N);\n        writeByte((byte) scale);\n\n        if (null == localCalendar) {\n            writeByte((byte) 0);\n            return;\n        }\n\n        writeByte((byte) TDS.datetime2ValueLength(scale));\n        writeScaledTemporal(localCalendar, subSecondNanos, scale, SSType.DATETIME2);\n    }\n\n    void writeRPCDateTimeOffset(String sName, GregorianCalendar utcCalendar, int minutesOffset, int subSecondNanos,\n            int scale, boolean bOut) throws SQLServerException {\n        writeRPCNameValType(sName, bOut, TDSType.DATETIMEOFFSETN);\n        writeByte((byte) scale);\n\n        if (null == utcCalendar) {\n            writeByte((byte) 0);\n            return;\n        }\n\n        assert 0 == utcCalendar.get(Calendar.ZONE_OFFSET);\n\n        writeByte((byte) TDS.datetimeoffsetValueLength(scale));\n        writeScaledTemporal(utcCalendar, subSecondNanos, scale, SSType.DATETIMEOFFSET);\n\n        writeShort((short) minutesOffset);\n    }\n\n    /**\n     * Returns subSecondNanos rounded to the maximum precision supported. The maximum fractional scale is\n     * MAX_FRACTIONAL_SECONDS_SCALE(7). Eg1: if you pass 456,790,123 the function would return 456,790,100 Eg2: if you\n     * pass 456,790,150 the function would return 456,790,200 Eg3: if you pass 999,999,951 the function would return\n     * 1,000,000,000 This is done to ensure that we have consistent rounding behaviour in setters and getters. Bug\n     * #507919\n     */\n    private int getRoundedSubSecondNanos(int subSecondNanos) {\n        return ((subSecondNanos + (Nanos.PER_MAX_SCALE_INTERVAL / 2)) / Nanos.PER_MAX_SCALE_INTERVAL)\n                * Nanos.PER_MAX_SCALE_INTERVAL;\n    }\n\n    /**\n     * Writes to the TDS channel a temporal value as an instance instance of one of the scaled temporal SQL types: DATE,\n     * TIME, DATETIME2, or DATETIMEOFFSET.\n     *\n     * @param cal\n     *        Calendar representing the value to write, except for any sub-second nanoseconds\n     * @param subSecondNanos\n     *        the sub-second nanoseconds (0 - 999,999,999)\n     * @param scale\n     *        the scale (in digits: 0 - 7) to use for the sub-second nanos component\n     * @param ssType\n     *        the SQL Server data type (DATE, TIME, DATETIME2, or DATETIMEOFFSET)\n     *\n     * @throws SQLServerException\n     *         if an I/O error occurs or if the value is not in the valid range\n     */\n    private void writeScaledTemporal(GregorianCalendar cal, int subSecondNanos, int scale,\n            SSType ssType) throws SQLServerException {\n\n        assert con.isKatmaiOrLater();\n\n        assert SSType.DATE == ssType || SSType.TIME == ssType || SSType.DATETIME2 == ssType\n                || SSType.DATETIMEOFFSET == ssType : UNEXPECTED_SSTYPE + ssType;\n\n        // First, for types with a time component, write the scaled nanos since midnight\n        if (SSType.TIME == ssType || SSType.DATETIME2 == ssType || SSType.DATETIMEOFFSET == ssType) {\n            assert subSecondNanos >= 0;\n            assert subSecondNanos < Nanos.PER_SECOND;\n            assert scale >= 0;\n            assert scale <= TDS.MAX_FRACTIONAL_SECONDS_SCALE;\n\n            int secondsSinceMidnight = cal.get(Calendar.SECOND) + 60 * cal.get(Calendar.MINUTE)\n                    + 60 * 60 * cal.get(Calendar.HOUR_OF_DAY);\n\n            // Scale nanos since midnight to the desired scale, rounding the value as necessary\n            long divisor = Nanos.PER_MAX_SCALE_INTERVAL\n                    * (long) Math.pow(10, TDS.MAX_FRACTIONAL_SECONDS_SCALE - (double) scale);\n\n            // The scaledNanos variable represents the fractional seconds of the value at the scale\n            // indicated by the scale variable. So, for example, scaledNanos = 3 means 300 nanoseconds\n            // at scale TDS.MAX_FRACTIONAL_SECONDS_SCALE, but 3000 nanoseconds at\n            // TDS.MAX_FRACTIONAL_SECONDS_SCALE - 1\n            long scaledNanos = ((long) Nanos.PER_SECOND * secondsSinceMidnight\n                    + getRoundedSubSecondNanos(subSecondNanos) + divisor / 2) / divisor;\n\n            // SQL Server rounding behavior indicates that it always rounds up unless\n            // we are at the max value of the type(NOT every day), in which case it truncates.\n            // Side effect on Calendar date:\n            // If rounding nanos to the specified scale rolls the value to the next day ...\n            if (Nanos.PER_DAY / divisor == scaledNanos) {\n\n                // If the type is time, always truncate\n                if (SSType.TIME == ssType) {\n                    --scaledNanos;\n                }\n                // If the type is datetime2 or datetimeoffset, truncate only if its the max value supported\n                else {\n                    assert SSType.DATETIME2 == ssType || SSType.DATETIMEOFFSET == ssType : UNEXPECTED_SSTYPE + ssType;\n\n                    // ... then bump the date, provided that the resulting date is still within\n                    // the valid date range.\n                    //\n                    // Extreme edge case (literally, the VERY edge...):\n                    // If nanos overflow rolls the date value out of range (that is, we have a value\n                    // a few nanoseconds later than 9999-12-31 23:59:59) then truncate the nanos\n                    // instead of rolling.\n                    //\n                    // This case is very likely never hit by \"real world\" applications, but exists\n                    // here as a security measure to ensure that such values don't result in a\n                    // connection-closing TDS exception.\n                    cal.add(Calendar.SECOND, 1);\n\n                    if (cal.get(Calendar.YEAR) <= 9999) {\n                        scaledNanos = 0;\n                    } else {\n                        cal.add(Calendar.SECOND, -1);\n                        --scaledNanos;\n                    }\n                }\n            }\n\n            // Encode the scaled nanos to TDS\n            int encodedLength = TDS.nanosSinceMidnightLength(scale);\n            byte[] encodedBytes = scaledNanosToEncodedBytes(scaledNanos, encodedLength);\n\n            writeBytes(encodedBytes);\n        }\n\n        // Second, for types with a date component, write the days into the Common Era\n        if (SSType.DATE == ssType || SSType.DATETIME2 == ssType || SSType.DATETIMEOFFSET == ssType) {\n            // Computation of the number of days into the Common Era assumes that\n            // the DAY_OF_YEAR field reflects a pure Gregorian calendar - one that\n            // uses Gregorian leap year rules across the entire range of dates.\n            //\n            // For the DAY_OF_YEAR field to accurately reflect pure Gregorian behavior,\n            // we need to use a pure Gregorian calendar for dates that are Julian dates\n            // under a standard Gregorian calendar and for (Gregorian) dates later than\n            // the cutover date in the cutover year.\n            if (cal.getTimeInMillis() < GregorianChange.STANDARD_CHANGE_DATE.getTime()\n                    || cal.getActualMaximum(Calendar.DAY_OF_YEAR) < TDS.DAYS_PER_YEAR) {\n                int year = cal.get(Calendar.YEAR);\n                int month = cal.get(Calendar.MONTH);\n                int date = cal.get(Calendar.DATE);\n\n                // Set the cutover as early as possible (pure Gregorian behavior)\n                cal.setGregorianChange(GregorianChange.PURE_CHANGE_DATE);\n\n                // Initialize the date field by field (preserving the \"wall calendar\" value)\n                cal.set(year, month, date);\n            }\n\n            int daysIntoCE = DDC.daysSinceBaseDate(cal.get(Calendar.YEAR), cal.get(Calendar.DAY_OF_YEAR), 1);\n\n            // Last-ditch verification that the value is in the valid range for the\n            // DATE/DATETIME2/DATETIMEOFFSET TDS data type (1/1/0001 to 12/31/9999).\n            // If it's not, then throw an exception now so that statement execution\n            // is safely canceled. Attempting to put an invalid value on the wire\n            // would result in a TDS exception, which would close the connection.\n            if (daysIntoCE < 0 || daysIntoCE >= DDC.daysSinceBaseDate(10000, 1, 1)) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_valueOutOfRange\"));\n                Object[] msgArgs = {ssType};\n                throw new SQLServerException(form.format(msgArgs), SQLState.DATA_EXCEPTION_DATETIME_FIELD_OVERFLOW,\n                        DriverError.NOT_SET, null);\n            }\n\n            byte[] encodedBytes = new byte[3];\n            encodedBytes[0] = (byte) ((daysIntoCE >> 0) & 0xFF);\n            encodedBytes[1] = (byte) ((daysIntoCE >> 8) & 0xFF);\n            encodedBytes[2] = (byte) ((daysIntoCE >> 16) & 0xFF);\n            writeBytes(encodedBytes);\n        }\n    }\n\n    /**\n     * Writes to the TDS channel a temporal value as an instance instance of one of the scaled temporal SQL types: DATE,\n     * TIME, DATETIME2, or DATETIMEOFFSET.\n     *\n     * @param cal\n     *        Calendar representing the value to write, except for any sub-second nanoseconds\n     * @param subSecondNanos\n     *        the sub-second nanoseconds (0 - 999,999,999)\n     * @param scale\n     *        the scale (in digits: 0 - 7) to use for the sub-second nanos component\n     * @param ssType\n     *        the SQL Server data type (DATE, TIME, DATETIME2, or DATETIMEOFFSET)\n     * @param minutesOffset\n     *        the offset value for DATETIMEOFFSET\n     * @param statement\n     *        the SQLServerStatement used for encryption\n     * @throws SQLServerException\n     *         if an I/O error occurs or if the value is not in the valid range\n     */\n    byte[] writeEncryptedScaledTemporal(GregorianCalendar cal, int subSecondNanos, int scale, SSType ssType,\n            short minutesOffset, SQLServerStatement statement) throws SQLServerException {\n        assert con.isKatmaiOrLater();\n\n        assert SSType.DATE == ssType || SSType.TIME == ssType || SSType.DATETIME2 == ssType\n                || SSType.DATETIMEOFFSET == ssType : UNEXPECTED_SSTYPE + ssType;\n\n        // store the time and minutesOffset portion of DATETIME2 and DATETIMEOFFSET to be used with date portion\n        byte[] encodedBytesForEncryption = null;\n\n        int secondsSinceMidnight = 0;\n        long divisor = 0;\n        long scaledNanos = 0;\n\n        // First, for types with a time component, write the scaled nanos since midnight\n        if (SSType.TIME == ssType || SSType.DATETIME2 == ssType || SSType.DATETIMEOFFSET == ssType) {\n            assert subSecondNanos >= 0;\n            assert subSecondNanos < Nanos.PER_SECOND;\n            assert scale >= 0;\n            assert scale <= TDS.MAX_FRACTIONAL_SECONDS_SCALE;\n\n            secondsSinceMidnight = cal.get(Calendar.SECOND) + 60 * cal.get(Calendar.MINUTE)\n                    + 60 * 60 * cal.get(Calendar.HOUR_OF_DAY);\n\n            // Scale nanos since midnight to the desired scale, rounding the value as necessary\n            divisor = Nanos.PER_MAX_SCALE_INTERVAL\n                    * (long) Math.pow(10, TDS.MAX_FRACTIONAL_SECONDS_SCALE - (double) scale);\n\n            // The scaledNanos variable represents the fractional seconds of the value at the scale\n            // indicated by the scale variable. So, for example, scaledNanos = 3 means 300 nanoseconds\n            // at scale TDS.MAX_FRACTIONAL_SECONDS_SCALE, but 3000 nanoseconds at\n            // TDS.MAX_FRACTIONAL_SECONDS_SCALE - 1\n            scaledNanos = (((long) Nanos.PER_SECOND * secondsSinceMidnight + getRoundedSubSecondNanos(subSecondNanos)\n                    + divisor / 2) / divisor) * divisor / 100;\n\n            // for encrypted time value, SQL server cannot do rounding or casting,\n            // So, driver needs to cast it before encryption.\n            if (SSType.TIME == ssType && 864000000000L <= scaledNanos) {\n                scaledNanos = (((long) Nanos.PER_SECOND * secondsSinceMidnight\n                        + getRoundedSubSecondNanos(subSecondNanos)) / divisor) * divisor / 100;\n            }\n\n            // SQL Server rounding behavior indicates that it always rounds up unless\n            // we are at the max value of the type(NOT every day), in which case it truncates.\n            // Side effect on Calendar date:\n            // If rounding nanos to the specified scale rolls the value to the next day ...\n            if (Nanos.PER_DAY / divisor == scaledNanos) {\n\n                // If the type is time, always truncate\n                if (SSType.TIME == ssType) {\n                    --scaledNanos;\n                }\n                // If the type is datetime2 or datetimeoffset, truncate only if its the max value supported\n                else {\n                    assert SSType.DATETIME2 == ssType || SSType.DATETIMEOFFSET == ssType : UNEXPECTED_SSTYPE + ssType;\n\n                    // ... then bump the date, provided that the resulting date is still within\n                    // the valid date range.\n                    //\n                    // Extreme edge case (literally, the VERY edge...):\n                    // If nanos overflow rolls the date value out of range (that is, we have a value\n                    // a few nanoseconds later than 9999-12-31 23:59:59) then truncate the nanos\n                    // instead of rolling.\n                    //\n                    // This case is very likely never hit by \"real world\" applications, but exists\n                    // here as a security measure to ensure that such values don't result in a\n                    // connection-closing TDS exception.\n                    cal.add(Calendar.SECOND, 1);\n\n                    if (cal.get(Calendar.YEAR) <= 9999) {\n                        scaledNanos = 0;\n                    } else {\n                        cal.add(Calendar.SECOND, -1);\n                        --scaledNanos;\n                    }\n                }\n            }\n\n            // Encode the scaled nanos to TDS\n            int encodedLength = TDS.nanosSinceMidnightLength(TDS.MAX_FRACTIONAL_SECONDS_SCALE);\n            byte[] encodedBytes = scaledNanosToEncodedBytes(scaledNanos, encodedLength);\n\n            if (SSType.TIME == ssType) {\n                return SQLServerSecurityUtility.encryptWithKey(encodedBytes, cryptoMeta, con, statement);\n            } else if (SSType.DATETIME2 == ssType) {\n                // for DATETIME2 sends both date and time part together for encryption\n                encodedBytesForEncryption = new byte[encodedLength + 3];\n                System.arraycopy(encodedBytes, 0, encodedBytesForEncryption, 0, encodedBytes.length);\n            } else if (SSType.DATETIMEOFFSET == ssType) {\n                // for DATETIMEOFFSET sends date, time and offset part together for encryption\n                encodedBytesForEncryption = new byte[encodedLength + 5];\n                System.arraycopy(encodedBytes, 0, encodedBytesForEncryption, 0, encodedBytes.length);\n            }\n        }\n\n        // Second, for types with a date component, write the days into the Common Era\n        if (SSType.DATE == ssType || SSType.DATETIME2 == ssType || SSType.DATETIMEOFFSET == ssType) {\n            // Computation of the number of days into the Common Era assumes that\n            // the DAY_OF_YEAR field reflects a pure Gregorian calendar - one that\n            // uses Gregorian leap year rules across the entire range of dates.\n            //\n            // For the DAY_OF_YEAR field to accurately reflect pure Gregorian behavior,\n            // we need to use a pure Gregorian calendar for dates that are Julian dates\n            // under a standard Gregorian calendar and for (Gregorian) dates later than\n            // the cutover date in the cutover year.\n            if (cal.getTimeInMillis() < GregorianChange.STANDARD_CHANGE_DATE.getTime()\n                    || cal.getActualMaximum(Calendar.DAY_OF_YEAR) < TDS.DAYS_PER_YEAR) {\n                int year = cal.get(Calendar.YEAR);\n                int month = cal.get(Calendar.MONTH);\n                int date = cal.get(Calendar.DATE);\n\n                // Set the cutover as early as possible (pure Gregorian behavior)\n                cal.setGregorianChange(GregorianChange.PURE_CHANGE_DATE);\n\n                // Initialize the date field by field (preserving the \"wall calendar\" value)\n                cal.set(year, month, date);\n            }\n\n            int daysIntoCE = DDC.daysSinceBaseDate(cal.get(Calendar.YEAR), cal.get(Calendar.DAY_OF_YEAR), 1);\n\n            // Last-ditch verification that the value is in the valid range for the\n            // DATE/DATETIME2/DATETIMEOFFSET TDS data type (1/1/0001 to 12/31/9999).\n            // If it's not, then throw an exception now so that statement execution\n            // is safely canceled. Attempting to put an invalid value on the wire\n            // would result in a TDS exception, which would close the connection.\n            if (daysIntoCE < 0 || daysIntoCE >= DDC.daysSinceBaseDate(10000, 1, 1)) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_valueOutOfRange\"));\n                Object[] msgArgs = {ssType};\n                throw new SQLServerException(form.format(msgArgs), SQLState.DATA_EXCEPTION_DATETIME_FIELD_OVERFLOW,\n                        DriverError.NOT_SET, null);\n            }\n\n            byte[] encodedBytes = new byte[3];\n            encodedBytes[0] = (byte) ((daysIntoCE >> 0) & 0xFF);\n            encodedBytes[1] = (byte) ((daysIntoCE >> 8) & 0xFF);\n            encodedBytes[2] = (byte) ((daysIntoCE >> 16) & 0xFF);\n\n            byte[] cipherText;\n            if (SSType.DATE == ssType) {\n                cipherText = SQLServerSecurityUtility.encryptWithKey(encodedBytes, cryptoMeta, con, statement);\n            } else if (SSType.DATETIME2 == ssType) {\n                // for Max value, does not round up, do casting instead.\n                if ((3652058 == daysIntoCE) && // 9999-12-31\n                        (864000000000L == scaledNanos)) { // 24:00:00 in nanoseconds\n                    // does not round up\n                    scaledNanos = (((long) Nanos.PER_SECOND * secondsSinceMidnight\n                            + getRoundedSubSecondNanos(subSecondNanos)) / divisor) * divisor / 100;\n\n                    int encodedLength = TDS.nanosSinceMidnightLength(TDS.MAX_FRACTIONAL_SECONDS_SCALE);\n                    byte[] encodedNanoBytes = scaledNanosToEncodedBytes(scaledNanos, encodedLength);\n\n                    // for DATETIME2 sends both date and time part together for encryption\n                    encodedBytesForEncryption = new byte[encodedLength + 3];\n                    System.arraycopy(encodedNanoBytes, 0, encodedBytesForEncryption, 0, encodedNanoBytes.length);\n                }\n\n                if (encodedBytesForEncryption == null) {\n                    MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_NullValue\"));\n                    Object[] msgArgs1 = {\"encodedBytesForEncryption\"};\n                    throw new SQLServerException(form.format(msgArgs1), null);\n                }\n\n                // Copy the 3 byte date value\n                System.arraycopy(encodedBytes, 0, encodedBytesForEncryption, (encodedBytesForEncryption.length - 3), 3);\n\n                cipherText = SQLServerSecurityUtility.encryptWithKey(encodedBytesForEncryption, cryptoMeta, con,\n                        statement);\n            } else {\n                // for Max value, does not round up, do casting instead.\n                if ((3652058 == daysIntoCE) && // 9999-12-31\n                        (864000000000L == scaledNanos)) { // 24:00:00 in nanoseconds\n                    // does not round up\n                    scaledNanos = (((long) Nanos.PER_SECOND * secondsSinceMidnight\n                            + getRoundedSubSecondNanos(subSecondNanos)) / divisor) * divisor / 100;\n\n                    int encodedLength = TDS.nanosSinceMidnightLength(TDS.MAX_FRACTIONAL_SECONDS_SCALE);\n                    byte[] encodedNanoBytes = scaledNanosToEncodedBytes(scaledNanos, encodedLength);\n\n                    // for DATETIMEOFFSET sends date, time and offset part together for encryption\n                    encodedBytesForEncryption = new byte[encodedLength + 5];\n                    System.arraycopy(encodedNanoBytes, 0, encodedBytesForEncryption, 0, encodedNanoBytes.length);\n                }\n\n                if (encodedBytesForEncryption == null) {\n                    MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_NullValue\"));\n                    Object[] msgArgs1 = {\"encodedBytesForEncryption\"};\n                    throw new SQLServerException(form.format(msgArgs1), null);\n                }\n\n                // Copy the 3 byte date value\n                System.arraycopy(encodedBytes, 0, encodedBytesForEncryption, (encodedBytesForEncryption.length - 5), 3);\n                // Copy the 2 byte minutesOffset value\n                System.arraycopy(\n                        ByteBuffer.allocate(Short.SIZE / Byte.SIZE).order(ByteOrder.LITTLE_ENDIAN)\n                                .putShort(minutesOffset).array(),\n                        0, encodedBytesForEncryption, (encodedBytesForEncryption.length - 2), 2);\n\n                cipherText = SQLServerSecurityUtility.encryptWithKey(encodedBytesForEncryption, cryptoMeta, con,\n                        statement);\n            }\n            return cipherText;\n        }\n\n        // Invalid type ssType. This condition should never happen.\n        MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_unknownSSType\"));\n        Object[] msgArgs = {ssType};\n        SQLServerException.makeFromDriverError(null, null, form.format(msgArgs), null, true);\n\n        return null;\n    }\n\n    private byte[] scaledNanosToEncodedBytes(long scaledNanos, int encodedLength) {\n        byte[] encodedBytes = new byte[encodedLength];\n        for (int i = 0; i < encodedLength; i++)\n            encodedBytes[i] = (byte) ((scaledNanos >> (8 * i)) & 0xFF);\n        return encodedBytes;\n    }\n\n    /**\n     * Append the data in a stream in RPC transmission format.\n     * \n     * @param sName\n     *        the optional parameter name\n     * @param stream\n     *        is the stream\n     * @param streamLength\n     *        length of the stream (may be unknown)\n     * @param bOut\n     *        boolean true if the data value is being registered as an output parameter\n     * @param jdbcType\n     *        The JDBC type used to determine whether the value is textual or non-textual.\n     * @param collation\n     *        The SQL collation associated with the value. Null for non-textual SQL Server types.\n     * @throws SQLServerException\n     */\n    void writeRPCInputStream(String sName, InputStream stream, long streamLength, boolean bOut, JDBCType jdbcType,\n            SQLCollation collation) throws SQLServerException {\n        assert null != stream;\n        assert DataTypes.UNKNOWN_STREAM_LENGTH == streamLength || streamLength >= 0;\n\n        // Send long values and values with unknown length\n        // using PLP chunking on Yukon and later.\n        boolean usePLP = (DataTypes.UNKNOWN_STREAM_LENGTH == streamLength\n                || streamLength > DataTypes.SHORT_VARTYPE_MAX_BYTES);\n        if (usePLP) {\n            assert DataTypes.UNKNOWN_STREAM_LENGTH == streamLength || streamLength <= DataTypes.MAX_VARTYPE_MAX_BYTES;\n\n            writeRPCNameValType(sName, bOut, jdbcType.isTextual() ? TDSType.BIGVARCHAR : TDSType.BIGVARBINARY);\n\n            writeVMaxHeader(streamLength, false, jdbcType.isTextual() ? collation : null);\n        }\n\n        // Send non-PLP in all other cases\n        else {\n            // If the length of the InputStream is unknown then we need to buffer the entire stream\n            // in memory so that we can determine its length and send that length to the server\n            // before the stream data itself.\n            if (DataTypes.UNKNOWN_STREAM_LENGTH == streamLength) {\n                // Create ByteArrayOutputStream with initial buffer size of 8K to handle typical\n                // binary field sizes more efficiently. Note we can grow beyond 8000 bytes.\n                ByteArrayOutputStream baos = new ByteArrayOutputStream(8000);\n                streamLength = 0L;\n\n                // Since Shiloh is limited to 64K TDS packets, that's a good upper bound on the maximum\n                // length of InputStream we should try to handle before throwing an exception.\n                long maxStreamLength = 65535L * con.getTDSPacketSize();\n\n                try {\n                    byte[] buff = new byte[8000];\n                    int bytesRead;\n\n                    while (streamLength < maxStreamLength && -1 != (bytesRead = stream.read(buff, 0, buff.length))) {\n                        baos.write(buff);\n                        streamLength += bytesRead;\n                    }\n                } catch (IOException e) {\n                    throw new SQLServerException(e.getMessage(), SQLState.DATA_EXCEPTION_NOT_SPECIFIC,\n                            DriverError.NOT_SET, e);\n                }\n\n                if (streamLength >= maxStreamLength) {\n                    MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidLength\"));\n                    Object[] msgArgs = {streamLength};\n                    SQLServerException.makeFromDriverError(null, null, form.format(msgArgs), \"\", true);\n                }\n\n                assert streamLength <= Integer.MAX_VALUE;\n                stream = new ByteArrayInputStream(baos.toByteArray(), 0, (int) streamLength);\n            }\n\n            assert 0 <= streamLength && streamLength <= DataTypes.IMAGE_TEXT_MAX_BYTES;\n\n            boolean useVarType = streamLength <= DataTypes.SHORT_VARTYPE_MAX_BYTES;\n\n            writeRPCNameValType(sName, bOut,\n                    jdbcType.isTextual() ? (useVarType ? TDSType.BIGVARCHAR : TDSType.TEXT)\n                                         : (useVarType ? TDSType.BIGVARBINARY : TDSType.IMAGE));\n\n            // Write maximum length, optional collation, and actual length\n            if (useVarType) {\n                writeShort((short) DataTypes.SHORT_VARTYPE_MAX_BYTES);\n                if (jdbcType.isTextual())\n                    collation.writeCollation(this);\n                writeShort((short) streamLength);\n            } else {\n                writeInt(DataTypes.IMAGE_TEXT_MAX_BYTES);\n                if (jdbcType.isTextual())\n                    collation.writeCollation(this);\n                writeInt((int) streamLength);\n            }\n        }\n\n        // Write the data\n        writeStream(stream, streamLength, usePLP);\n    }\n\n    /**\n     * Append the XML data in a stream in RPC transmission format.\n     * \n     * @param sName\n     *        the optional parameter name\n     * @param stream\n     *        is the stream\n     * @param streamLength\n     *        length of the stream (may be unknown)\n     * @param bOut\n     *        boolean true if the data value is being registered as an output parameter\n     * @throws SQLServerException\n     */\n    void writeRPCXML(String sName, InputStream stream, long streamLength, boolean bOut) throws SQLServerException {\n        assert DataTypes.UNKNOWN_STREAM_LENGTH == streamLength || streamLength >= 0;\n        assert DataTypes.UNKNOWN_STREAM_LENGTH == streamLength || streamLength <= DataTypes.MAX_VARTYPE_MAX_BYTES;\n\n        writeRPCNameValType(sName, bOut, TDSType.XML);\n        writeByte((byte) 0); // No schema\n        // Handle null here and return, we're done here if it's null.\n        if (null == stream) {\n            // Null header for v*max types is 0xFFFFFFFFFFFFFFFF.\n            writeLong(0xFFFFFFFFFFFFFFFFL);\n        } else if (DataTypes.UNKNOWN_STREAM_LENGTH == streamLength) {\n            // Append v*max length.\n            // UNKNOWN_PLP_LEN is 0xFFFFFFFFFFFFFFFE\n            writeLong(0xFFFFFFFFFFFFFFFEL);\n\n            // NOTE: Don't send the first chunk length, this will be calculated by caller.\n        } else {\n            // For v*max types with known length, length is <totallength8><chunklength4>\n            // We're sending same total length as chunk length (as we're sending 1 chunk).\n            writeLong(streamLength);\n        }\n        if (null != stream)\n            // Write the data\n            writeStream(stream, streamLength, true);\n    }\n\n    /**\n     * Append the data in a character reader in RPC transmission format.\n     * \n     * @param sName\n     *        the optional parameter name\n     * @param re\n     *        the reader\n     * @param reLength\n     *        the reader data length (in characters)\n     * @param bOut\n     *        boolean true if the data value is being registered as an output parameter\n     * @param collation\n     *        The SQL collation associated with the value. Null for non-textual SQL Server types.\n     * @throws SQLServerException\n     */\n    void writeRPCReaderUnicode(String sName, Reader re, long reLength, boolean bOut,\n            SQLCollation collation) throws SQLServerException {\n        assert null != re;\n        assert DataTypes.UNKNOWN_STREAM_LENGTH == reLength || reLength >= 0;\n\n        // Textual RPC requires a collation. If none is provided, as is the case when\n        // the SSType is non-textual, then use the database collation by default.\n        if (null == collation)\n            collation = con.getDatabaseCollation();\n\n        // Send long values and values with unknown length\n        // using PLP chunking on Yukon and later.\n        boolean usePLP = (DataTypes.UNKNOWN_STREAM_LENGTH == reLength || reLength > DataTypes.SHORT_VARTYPE_MAX_CHARS);\n        if (usePLP) {\n            assert DataTypes.UNKNOWN_STREAM_LENGTH == reLength || reLength <= DataTypes.MAX_VARTYPE_MAX_CHARS;\n\n            writeRPCNameValType(sName, bOut, TDSType.NVARCHAR);\n\n            writeVMaxHeader(\n                    (DataTypes.UNKNOWN_STREAM_LENGTH == reLength) ? DataTypes.UNKNOWN_STREAM_LENGTH : 2 * reLength, // Length\n                                                                                                                    // (in\n                                                                                                                    // bytes)\n                    false, collation);\n        }\n\n        // Send non-PLP in all other cases\n        else {\n            // Length must be known if we're not sending PLP-chunked data. Yukon is handled above.\n            // For Shiloh, this is enforced in DTV by converting the Reader to some other length-\n            // prefixed value in the setter.\n            assert 0 <= reLength && reLength <= DataTypes.NTEXT_MAX_CHARS;\n\n            // For non-PLP types, use the long TEXT type rather than the short VARCHAR\n            // type if the stream is too long to fit in the latter or if we don't know the length up\n            // front so we have to assume that it might be too long.\n            boolean useVarType = reLength <= DataTypes.SHORT_VARTYPE_MAX_CHARS;\n\n            writeRPCNameValType(sName, bOut, useVarType ? TDSType.NVARCHAR : TDSType.NTEXT);\n\n            // Write maximum length, collation, and actual length of the data\n            if (useVarType) {\n                writeShort((short) DataTypes.SHORT_VARTYPE_MAX_BYTES);\n                collation.writeCollation(this);\n                writeShort((short) (2 * reLength));\n            } else {\n                writeInt(DataTypes.NTEXT_MAX_CHARS);\n                collation.writeCollation(this);\n                writeInt((int) (2 * reLength));\n            }\n        }\n\n        // Write the data\n        writeReader(re, reLength, usePLP);\n    }\n\n    void sendEnclavePackage(String sql, ArrayList<byte[]> enclaveCEKs) throws SQLServerException {\n        if (null != con && con.isAEv2()) {\n            if (null != sql && !sql.isEmpty() && null != enclaveCEKs && !(enclaveCEKs.isEmpty())\n                    && con.enclaveEstablished()) {\n                byte[] b = con.generateEnclavePackage(sql, enclaveCEKs);\n                if (null != b && 0 != b.length) {\n                    this.writeShort((short) b.length);\n                    this.writeBytes(b);\n                } else {\n                    this.writeShort((short) 0);\n                }\n            } else {\n                this.writeShort((short) 0);\n            }\n        }\n    }\n}\n\n\n/**\n * TDSPacket provides a mechanism for chaining TDS response packets together in a singly-linked list.\n *\n * Having both the link and the data in the same class allows TDSReader marks (see below) to automatically hold onto\n * exactly as much response data as they need, and no more. Java reference semantics ensure that a mark holds onto its\n * referenced packet and subsequent packets (through next references). When all marked references to a packet go away,\n * the packet, and any linked unmarked packets, can be reclaimed by GC.\n */\nfinal class TDSPacket {\n    final byte[] header = new byte[TDS.PACKET_HEADER_SIZE];\n    final byte[] payload;\n    int payloadLength;\n    volatile TDSPacket next;\n\n    final public String toString() {\n        return \"TDSPacket(SPID:\" + Util.readUnsignedShortBigEndian(header, TDS.PACKET_HEADER_SPID) + \" Seq:\"\n                + header[TDS.PACKET_HEADER_SEQUENCE_NUM] + \")\";\n    }\n\n    TDSPacket(int size) {\n        payload = new byte[size];\n        payloadLength = 0;\n        next = null;\n    }\n\n    final boolean isEOM() {\n        return TDS.STATUS_BIT_EOM == (header[TDS.PACKET_HEADER_MESSAGE_STATUS] & TDS.STATUS_BIT_EOM);\n    }\n}\n\n\n/**\n * TDSReaderMark encapsulates a fixed position in the response data stream.\n *\n * Response data is quantized into a linked chain of packets. A mark refers to a specific location in a specific packet\n * and relies on Java's reference semantics to automatically keep all subsequent packets accessible until the mark is\n * destroyed.\n */\nfinal class TDSReaderMark {\n    final TDSPacket packet;\n    final int payloadOffset;\n\n    TDSReaderMark(TDSPacket packet, int payloadOffset) {\n        this.packet = packet;\n        this.payloadOffset = payloadOffset;\n    }\n}\n\n\n/**\n * TDSReader encapsulates the TDS response data stream.\n *\n * Bytes are read from SQL Server into a FIFO of packets. Reader methods traverse the packets to access the data.\n */\nfinal class TDSReader implements Serializable {\n\n    /**\n     * Always update serialVersionUID when prompted.\n     */\n    private static final long serialVersionUID = -392905303734809731L;\n\n    private static final Logger logger = Logger.getLogger(\"com.microsoft.sqlserver.jdbc.internals.TDS.Reader\");\n    final private String traceID;\n    private transient ScheduledFuture<?> timeout;\n\n    final public String toString() {\n        return traceID;\n    }\n\n    private final TDSChannel tdsChannel;\n    private final SQLServerConnection con;\n\n    private final TDSCommand command;\n\n    final TDSCommand getCommand() {\n        assert null != command;\n        return command;\n    }\n\n    final SQLServerConnection getConnection() {\n        return con;\n    }\n\n    private transient TDSPacket currentPacket = new TDSPacket(0);\n    private transient TDSPacket lastPacket = currentPacket;\n    private int payloadOffset = 0;\n    private int packetNum = 0;\n\n    private boolean isStreaming = true;\n    private boolean useColumnEncryption = false;\n    private boolean serverSupportsColumnEncryption = false;\n    private boolean serverSupportsDataClassification = false;\n    private byte serverSupportedDataClassificationVersion = TDS.DATA_CLASSIFICATION_NOT_ENABLED;\n    private final transient Lock tdsReaderLock = new ReentrantLock();\n\n    private final byte[] valueBytes = new byte[256];\n\n    protected transient SensitivityClassification sensitivityClassification;\n\n    private static final AtomicInteger lastReaderID = new AtomicInteger(0);\n\n    private static int nextReaderID() {\n        return lastReaderID.incrementAndGet();\n    }\n\n    TDSReader(TDSChannel tdsChannel, SQLServerConnection con, TDSCommand command) {\n        this.tdsChannel = tdsChannel;\n        this.con = con;\n        this.command = command; // may be null\n        // if the logging level is not detailed than fine or more we will not have proper reader IDs.\n        if (logger.isLoggable(Level.FINE))\n            traceID = \"TDSReader@\" + nextReaderID() + \" (\" + con.toString() + \")\";\n        else\n            traceID = con.toString();\n        if (con.isColumnEncryptionSettingEnabled()) {\n            useColumnEncryption = true;\n        }\n        serverSupportsColumnEncryption = con.getServerSupportsColumnEncryption();\n        serverSupportsDataClassification = con.getServerSupportsDataClassification();\n        serverSupportedDataClassificationVersion = con.getServerSupportedDataClassificationVersion();\n    }\n\n    final boolean isColumnEncryptionSettingEnabled() {\n        return useColumnEncryption;\n    }\n\n    final boolean getServerSupportsColumnEncryption() {\n        return serverSupportsColumnEncryption;\n    }\n\n    final boolean getServerSupportsDataClassification() {\n        return serverSupportsDataClassification;\n    }\n\n    final byte getServerSupportedDataClassificationVersion() {\n        return serverSupportedDataClassificationVersion;\n    }\n\n    final void throwInvalidTDS() throws SQLServerException {\n        if (logger.isLoggable(Level.SEVERE))\n            logger.severe(toString() + \" got unexpected value in TDS response at offset:\" + payloadOffset);\n        con.throwInvalidTDS();\n    }\n\n    final void throwInvalidTDSToken(String tokenName) throws SQLServerException {\n        if (logger.isLoggable(Level.SEVERE))\n            logger.severe(toString() + \" got unexpected value in TDS response at offset:\" + payloadOffset);\n        con.throwInvalidTDSToken(tokenName);\n    }\n\n    /**\n     * Ensures that payload data is available to be read, automatically advancing to (and possibly reading) the next\n     * packet.\n     *\n     * @return true if additional data is available to be read false if no more data is available\n     */\n    private boolean ensurePayload() throws SQLServerException {\n        if ((payloadOffset == currentPacket.payloadLength) && (!nextPacket())) {\n            return false;\n        }\n        assert payloadOffset < currentPacket.payloadLength;\n        return true;\n    }\n\n    /**\n     * Advance (and possibly read) the next packet.\n     *\n     * @return true if additional data is available to be read false if no more data is available\n     */\n    private boolean nextPacket() throws SQLServerException {\n        assert null != currentPacket;\n\n        // Shouldn't call this function unless we're at the end of the current packet...\n        TDSPacket consumedPacket = currentPacket;\n        assert payloadOffset == consumedPacket.payloadLength;\n\n        // If no buffered packets are left then maybe we can read one...\n        // This action must be synchronized against against another thread calling\n        // readAllPackets() to read in ALL of the remaining packets of the current response.\n...\n", "needle_token_start": 2311, "needle_token_end": 2535, "code_context_ntokens": 16168, "output": ["```java\nvoid writeEncryptedRPCTime(String sName, GregorianCalendar localCalendar, int subSecondNanos, int scale,\n        boolean bOut, SQLServerStatement statement) throws SQLServerException {\n    if (con.getSendTimeAsDatetime()) {\n        throw new SQLServerException(SQLServerException.getErrString(\"R_sendTimeAsDateTimeForAE\"), null);\n    }\n    writeRPCNameValType(sName, bOut, TDSType.BIGVARBINARY);\n\n    if (null == localCalendar)\n        writeEncryptedRPCByteArray(null);\n    else\n        writeEncryptedRPCByteArray(writeEncryptedScaledTemporal(localCalendar, subSecondNanos, scale, SSType.TIME,\n                (short) 0, statement));\n\n    writeByte(TDSType.TIMEN.byteValue());\n    writeByte((byte) scale);\n    writeCryptoMetaData();\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "removeSingleQuote", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To strip the leading and trailing single quotes from a given string if both are present.\n2. **Input**: A string that may or may not be enclosed in single quotes.\n3. **Output**: The same string but without the leading and trailing single quotes if it started and ended with them; otherwise, the original string is returned unchanged.\n4. **Procedure**: The function first checks the length of the string. It then examines if the first and last characters of the string are single quotes. If both conditions are met, it returns a new string with the first and last characters removed. If not, it returns the original string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerClob.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.BufferedInputStream;\nimport java.io.BufferedReader;\nimport java.io.ByteArrayInputStream;\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.io.Reader;\nimport java.io.StringReader;\nimport java.nio.charset.Charset;\nimport java.nio.charset.StandardCharsets;\nimport java.sql.Clob;\nimport java.sql.SQLException;\nimport java.text.MessageFormat;\nimport java.util.ArrayList;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n\n/**\n * Represents a character LOB object and implements java.sql.Clob.\n */\npublic class SQLServerClob extends SQLServerClobBase implements Clob {\n    /**\n     * Always refresh SerialVersionUID when prompted\n     */\n    private static final long serialVersionUID = 2872035282200133865L;\n\n    // Loggers should be class static to avoid lock contention with multiple\n    // threads\n    private static final Logger logger = Logger.getLogger(\"com.microsoft.sqlserver.jdbc.internals.SQLServerClob\");\n\n    /**\n     * Constructs a SQLServerClob.\n     *\n     * @param connection\n     *        the database connection this blob is implemented on\n     * @param data\n     *        the CLOB's data\n     * @deprecated Use {@link SQLServerConnection#createClob()} instead.\n     */\n    @Deprecated\n    public SQLServerClob(SQLServerConnection connection, String data) {\n        super(connection, data, (null == connection) ? null : connection.getDatabaseCollation(), logger, null);\n\n        if (null == data)\n            throw new NullPointerException(SQLServerException.getErrString(\"R_cantSetNull\"));\n    }\n\n    SQLServerClob(SQLServerConnection connection) {\n        super(connection, \"\", connection.getDatabaseCollation(), logger, null);\n    }\n\n    SQLServerClob(BaseInputStream stream, TypeInfo typeInfo) {\n        super(null, stream, typeInfo.getSQLCollation(), logger, typeInfo);\n    }\n\n    @Override\n    final JDBCType getJdbcType() {\n        return JDBCType.CLOB;\n    }\n}\n\n\n/**\n * Abstract class SQLServerClobBase\n *\n */\nabstract class SQLServerClobBase extends SQLServerLob {\n    /**\n     * Always refresh SerialVersionUID when prompted\n     */\n    private static final long serialVersionUID = 8691072211054430124L;\n\n    // The value of the CLOB that this Clob object represents.\n    // This value is never null unless/until the free() method is called.\n    String value;\n\n    private final SQLCollation sqlCollation;\n\n    private boolean isClosed = false;\n\n    final TypeInfo typeInfo;\n\n    /**\n     * Active streams which must be closed when the Clob/NClob is closed. Initial size of the array is based on an\n     * assumption that a Clob/NClob object is typically used either for input or output, and then only once. The array\n     * size grows automatically if multiple streams are used.\n     */\n    private ArrayList<Closeable> activeStreams = new ArrayList<>(1);\n\n    transient SQLServerConnection con;\n\n    private final transient Logger logger;\n\n    final private String traceID = getClass().getName().substring(1 + getClass().getName().lastIndexOf('.')) + \":\"\n            + nextInstanceID();\n\n    final public String toString() {\n        return traceID;\n    }\n\n    // Unique id generator for each instance (used for logging).\n    static private final AtomicInteger BASE_ID = new AtomicInteger(0);\n\n    private transient Charset defaultCharset = null;\n\n    // Returns unique id for each instance.\n    private static int nextInstanceID() {\n        return BASE_ID.incrementAndGet();\n    }\n\n    abstract JDBCType getJdbcType();\n\n    private String getDisplayClassName() {\n        String fullClassName = getJdbcType().className();\n        return fullClassName.substring(1 + fullClassName.lastIndexOf('.'));\n    }\n\n    /**\n     * Constructs a new CLOB from a String.\n     * \n     * @param connection\n     *        SQLServerConnection\n     * @param data\n     *        the CLOB data\n     * @param collation\n     *        the data collation\n     * @param logger\n     *        logger information\n     * @param typeInfo\n     *        the column TYPE_INFO\n     */\n    SQLServerClobBase(SQLServerConnection connection, Object data, SQLCollation collation, Logger logger,\n            TypeInfo typeInfo) {\n        this.con = connection;\n        if (data instanceof BaseInputStream) {\n            activeStreams.add((Closeable) data);\n        } else {\n            this.value = (String) data;\n        }\n        this.sqlCollation = collation;\n        this.logger = logger;\n        this.typeInfo = typeInfo;\n        if (logger.isLoggable(Level.FINE)) {\n            String loggingInfo = (null != connection) ? connection.toString() : \"null connection\";\n            logger.fine(toString() + \" created by (\" + loggingInfo + \")\");\n        }\n    }\n\n    /**\n     * Frees this Clob/NClob object and releases the resources that it holds.\n     *\n     * After free() has been called, any attempt to invoke a method other than free() will result in a SQLException\n     * being thrown. If free() is called multiple times, the subsequent calls to free are treated as a no-op.\n     * \n     * @throws SQLException\n     *         when an error occurs\n     */\n    public void free() throws SQLException {\n        if (!isClosed) {\n            // Close active streams, ignoring any errors, since nothing can be\n            // done with them after that point anyway.\n            if (null != activeStreams) {\n                for (Closeable stream : activeStreams) {\n                    try {\n                        stream.close();\n                    } catch (IOException ioException) {\n                        logger.fine(toString() + \" ignored IOException closing stream \" + stream + \": \"\n                                + ioException.getMessage());\n                    }\n                }\n                activeStreams = null;\n            }\n\n            // Discard the value.\n            value = null;\n\n            isClosed = true;\n        }\n    }\n\n    /**\n     * Throws a SQLException if the LOB has been freed.\n     */\n    private void checkClosed() throws SQLServerException {\n        if (isClosed) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_isFreed\"));\n            SQLServerException.makeFromDriverError(con, null, form.format(new Object[] {getDisplayClassName()}), null,\n                    true);\n        }\n    }\n\n    /**\n     * Returns the CLOB as an ASCII stream.\n     * \n     * @throws SQLException\n     *         when an error occurs\n     * @return the data as an input stream\n     */\n    public InputStream getAsciiStream() throws SQLException {\n        checkClosed();\n        if (null != sqlCollation && !sqlCollation.supportsAsciiConversion()) {\n            DataTypes.throwConversionError(getDisplayClassName(), \"AsciiStream\");\n        }\n        // If the LOB is currently streaming and the stream hasn't been read, read it.\n        if (!delayLoadingLob && null == value && !activeStreams.isEmpty()) {\n            getStringFromStream();\n        }\n\n        // Need to use a BufferedInputStream since the stream returned by this method is assumed to support mark/reset\n        InputStream getterStream = null;\n        if (null == value && !activeStreams.isEmpty()) {\n            InputStream inputStream = (InputStream) activeStreams.get(0);\n            try {\n                inputStream.reset();\n            } catch (IOException e) {\n                SQLServerException.makeFromDriverError(con, null, e.getMessage(), null, false);\n            }\n...\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerBulkBatchInsertRecord.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.math.BigDecimal;\nimport java.math.RoundingMode;\nimport java.sql.Types;\nimport java.text.DecimalFormat;\nimport java.text.MessageFormat;\nimport java.time.OffsetDateTime;\nimport java.time.OffsetTime;\nimport java.time.format.DateTimeFormatter;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map.Entry;\n\n\n/**\n * Provides a simple implementation of the ISQLServerBulkRecord interface that can be used to read in the basic Java\n * data types from an ArrayList of Parameters that were provided by pstmt/cstmt.\n */\nclass SQLServerBulkBatchInsertRecord extends SQLServerBulkRecord {\n\n    /**\n     * Update serialVersionUID when making changes to this file\n     */\n    private static final long serialVersionUID = -955998113956445541L;\n\n    transient List<Parameter[]> batchParam;\n    private int batchParamIndex = -1;\n    private List<String> columnList;\n    private List<String> valueList;\n\n    /*\n     * Class name for logging.\n     */\n    private static final String loggerClassName = \"SQLServerBulkBatchInsertRecord\";\n\n    /*\n     * Constructs a SQLServerBulkBatchInsertRecord with the batch parameter, column list, value list, and encoding\n     */\n    SQLServerBulkBatchInsertRecord(ArrayList<Parameter[]> batchParam, ArrayList<String> columnList,\n            ArrayList<String> valueList, String encoding) throws SQLServerException {\n        initLoggerResources();\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {\n            loggerExternal.entering(loggerPackageName, loggerClassName, new Object[] {batchParam, encoding});\n        }\n\n        if (null == batchParam) {\n            throwInvalidArgument(\"batchParam\");\n        }\n\n        if (null == valueList) {\n            throwInvalidArgument(\"valueList\");\n        }\n\n        this.batchParam = batchParam;\n        this.columnList = columnList;\n        this.valueList = valueList;\n        columnMetadata = new HashMap<>();\n\n        loggerExternal.exiting(loggerPackageName, loggerClassName);\n    }\n\n    private void initLoggerResources() {\n        super.loggerPackageName = \"com.microsoft.sqlserver.jdbc.SQLServerBulkBatchInsertRecord\";\n    }\n\n    private Object convertValue(ColumnMetadata cm, Object data) throws SQLServerException {\n        switch (cm.columnType) {\n            case Types.INTEGER: {\n                // Formatter to remove the decimal part as SQL Server floors the\n                // decimal in integer types\n                DecimalFormat decimalFormatter = new DecimalFormat(\"#\");\n                decimalFormatter.setRoundingMode(RoundingMode.DOWN);\n                String formatedfInput = decimalFormatter.format(Double.parseDouble(data.toString()));\n                return Integer.valueOf(formatedfInput);\n            }\n\n            case Types.TINYINT:\n            case Types.SMALLINT: {\n                // Formatter to remove the decimal part as SQL Server floors the\n                // decimal in integer types\n                DecimalFormat decimalFormatter = new DecimalFormat(\"#\");\n                decimalFormatter.setRoundingMode(RoundingMode.DOWN);\n                String formatedfInput = decimalFormatter.format(Double.parseDouble(data.toString()));\n                return Short.valueOf(formatedfInput);\n            }\n\n            case Types.BIGINT: {\n                BigDecimal bd = new BigDecimal(data.toString().trim());\n                try {\n                    return bd.setScale(0, RoundingMode.DOWN).longValueExact();\n                } catch (ArithmeticException ex) {\n                    String value = \"'\" + data + \"'\";\n                    MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_errorConvertingValue\"));\n                    throw new SQLServerException(form.format(new Object[] {value, JDBCType.of(cm.columnType)}), null, 0,\n                            ex);\n                }\n            }\n\n            case Types.DECIMAL:\n            case Types.NUMERIC: {\n                BigDecimal bd = new BigDecimal(data.toString().trim());\n                return bd.setScale(cm.scale, RoundingMode.HALF_UP);\n            }\n\n            case Types.BIT: {\n                // \"true\" => 1, \"false\" => 0\n                // Any non-zero value (integer/double) => 1, 0/0.0 => 0\n                try {\n                    return (0 == Double.parseDouble(data.toString())) ? Boolean.FALSE : Boolean.TRUE;\n                } catch (NumberFormatException e) {\n                    return Boolean.parseBoolean(data.toString());\n                }\n            }\n\n            case Types.REAL: {\n                return Float.parseFloat(data.toString());\n            }\n\n            case Types.DOUBLE: {\n                return Double.parseDouble(data.toString());\n            }\n\n            case Types.BINARY:\n            case Types.VARBINARY:\n            case Types.LONGVARBINARY:\n            case Types.BLOB: {\n                if (data instanceof byte[]) {\n                    /*\n                     * if the binary data comes in as a byte array through setBytes through Bulk Copy for Batch Insert\n                     * API, don't turn the binary array into a string.\n                     */\n                    return data;\n                } else {\n                    // Strip off 0x if present.\n                    String binData = data.toString().trim();\n                    if (binData.startsWith(\"0x\") || binData.startsWith(\"0X\")) {\n                        return binData.substring(2);\n                    } else {\n                        return binData;\n                    }\n                }\n            }\n\n            case java.sql.Types.TIME_WITH_TIMEZONE: {\n                OffsetTime offsetTimeValue;\n\n                // The per-column DateTimeFormatter gets priority.\n                if (null != cm.dateTimeFormatter)\n                    offsetTimeValue = OffsetTime.parse(data.toString(), cm.dateTimeFormatter);\n                else if (timeFormatter != null)\n                    offsetTimeValue = OffsetTime.parse(data.toString(), timeFormatter);\n                else\n                    offsetTimeValue = OffsetTime.parse(data.toString());\n\n                return offsetTimeValue;\n            }\n\n            case java.sql.Types.TIMESTAMP_WITH_TIMEZONE: {\n                OffsetDateTime offsetDateTimeValue;\n\n                // The per-column DateTimeFormatter gets priority.\n                if (null != cm.dateTimeFormatter)\n                    offsetDateTimeValue = OffsetDateTime.parse(data.toString(), cm.dateTimeFormatter);\n                else if (dateTimeFormatter != null)\n                    offsetDateTimeValue = OffsetDateTime.parse(data.toString(), dateTimeFormatter);\n                else\n                    offsetDateTimeValue = OffsetDateTime.parse(data.toString());\n\n                return offsetDateTimeValue;\n            }\n\n            case Types.NULL: {\n                return null;\n            }\n\n            case Types.DATE:\n            case Types.CHAR:\n            case Types.NCHAR:\n            case Types.VARCHAR:\n            case Types.NVARCHAR:\n            case Types.LONGVARCHAR:\n            case Types.LONGNVARCHAR:\n            case Types.CLOB:\n            default: {\n                // The string is copied as is.\n                return data;\n            }\n        }\n    }\n\n    \nprivate String removeSingleQuote(String s) {\n        int len = s.length();\n        return (s.charAt(0) == '\\'' && s.charAt(len - 1) == '\\'') ? s.substring(1, len - 1) : s;\n    }\n\n    @Override\n    public Object[] getRowData() throws SQLServerException {\n        Object[] data = new Object[columnMetadata.size()];\n        int valueIndex = 0;\n        String valueData;\n        Object rowData;\n        int columnListIndex = 0;\n\n        /*\n         * check if the size of the list of values = size of the list of columns (which is optional)\n         */\n        if (null != columnList && columnList.size() != valueList.size()) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_DataSchemaMismatch\"));\n            Object[] msgArgs = {};\n            throw new SQLServerException(form.format(msgArgs), SQLState.COL_NOT_FOUND, DriverError.NOT_SET, null);\n        }\n\n        for (Entry<Integer, ColumnMetadata> pair : columnMetadata.entrySet()) {\n            int index = pair.getKey() - 1;\n\n            /*\n             * To explain what each variable represents: columnMetadata = map containing the ENTIRE list of columns in\n             * the table. columnList = the *optional* list of columns the user can provide. For example, the (c1, c3)\n             * part of this query: INSERT into t1 (c1, c3) values (?, ?) valueList = the *mandatory* list of columns the\n             * user needs provide. This is the (?, ?) part of the previous query. The size of this valueList will always\n             * equal the number of the entire columns in the table IF columnList has NOT been provided. If columnList\n             * HAS been provided, then this valueList may be smaller than the list of all columns (which is\n             * columnMetadata).\n             */\n            // case when the user has not provided the optional list of column names.\n            if (null == columnList || columnList.isEmpty()) {\n                valueData = valueList.get(index);\n                /*\n                 * if the user has provided a wildcard for this column, fetch the set value from the batchParam.\n                 */\n                if (\"?\".equalsIgnoreCase(valueData)) {\n                    rowData = batchParam.get(batchParamIndex)[valueIndex++].getSetterValue();\n                } else if (\"null\".equalsIgnoreCase(valueData)) {\n                    rowData = null;\n                }\n                /*\n                 * if the user has provided a hardcoded value for this column, rowData is simply set to the hardcoded\n                 * value.\n                 */\n                else {\n                    rowData = removeSingleQuote(valueData);\n                }\n            }\n            // case when the user has provided the optional list of column names.\n            else {\n                /*\n                 * columnListIndex is a separate counter we need to keep track of for each time we've processed a column\n                 * that the user provided. for example, if the user provided an optional columnList of (c1, c3, c5, c7)\n                 * in a table that has 8 columns (c1~c8), then the columnListIndex would increment only when we're\n                 * dealing with the four columns inside columnMetadata. compare the list of the optional list of column\n                 * names to the table's metadata, and match each other, so we assign the correct value to each column.\n                 */\n                if (columnList.size() > columnListIndex\n                        && columnList.get(columnListIndex).equalsIgnoreCase(columnMetadata.get(index + 1).columnName)) {\n                    valueData = valueList.get(columnListIndex);\n                    if (\"?\".equalsIgnoreCase(valueData)) {\n                        rowData = batchParam.get(batchParamIndex)[valueIndex++].getSetterValue();\n                    } else if (\"null\".equalsIgnoreCase(valueData)) {\n                        rowData = null;\n                    } else {\n                        rowData = removeSingleQuote(valueData);\n                    }\n                    columnListIndex++;\n                } else {\n                    rowData = null;\n                }\n            }\n\n            try {\n                if (null == rowData) {\n                    data[index] = null;\n                    continue;\n                } else if (0 == rowData.toString().length()) {\n                    data[index] = \"\";\n                    continue;\n                }\n                data[index] = convertValue(pair.getValue(), rowData);\n            } catch (IllegalArgumentException e) {\n                String value = \"'\" + rowData + \"'\";\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_errorConvertingValue\"));\n                throw new SQLServerException(form.format(new Object[] {value, JDBCType.of(pair.getValue().columnType)}),\n                        null, 0, e);\n            } catch (ArrayIndexOutOfBoundsException e) {\n                throw new SQLServerException(SQLServerException.getErrString(\"R_DataSchemaMismatch\"), e);\n            }\n        }\n        return data;\n    }\n\n    @Override\n    void addColumnMetadataInternal(int positionInSource, String name, int jdbcType, int precision, int scale,\n            DateTimeFormatter dateTimeFormatter) throws SQLServerException {\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {\n            loggerExternal.entering(loggerPackageName, \"addColumnMetadata\",\n                    new Object[] {positionInSource, name, jdbcType, precision, scale});\n        }\n        String colName = \"\";\n\n        if (0 >= positionInSource) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidColumnOrdinal\"));\n            Object[] msgArgs = {positionInSource};\n            throw new SQLServerException(form.format(msgArgs), SQLState.COL_NOT_FOUND, DriverError.NOT_SET, null);\n        }\n\n        if (null != name)\n            colName = name.trim();\n        else if ((null != columnNames) && (columnNames.length >= positionInSource))\n            colName = columnNames[positionInSource - 1];\n\n        if ((null != columnNames) && (positionInSource > columnNames.length)) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidColumn\"));\n            Object[] msgArgs = {positionInSource};\n            throw new SQLServerException(form.format(msgArgs), SQLState.COL_NOT_FOUND, DriverError.NOT_SET, null);\n        }\n\n        checkDuplicateColumnName(positionInSource, name);\n        switch (jdbcType) {\n            /*\n             * SQL Server supports numerous string literal formats for temporal types, hence sending them as varchar\n             * with approximate precision(length) needed to send supported string literals. string literal formats\n             * supported by temporal types are available in MSDN page on data types.\n             */\n            case java.sql.Types.DATE:\n            case java.sql.Types.TIME:\n            case java.sql.Types.TIMESTAMP:\n            case microsoft.sql.Types.DATETIMEOFFSET:\n                columnMetadata.put(positionInSource,\n                        new ColumnMetadata(colName, jdbcType, precision, scale, dateTimeFormatter));\n                break;\n\n            // Redirect SQLXML as LONGNVARCHAR\n            // SQLXML is not valid type in TDS\n            case java.sql.Types.SQLXML:\n                columnMetadata.put(positionInSource,\n                        new ColumnMetadata(colName, java.sql.Types.LONGNVARCHAR, precision, scale, dateTimeFormatter));\n                break;\n\n            // Redirecting Float as Double based on data type mapping\n            // https://msdn.microsoft.com/en-us/library/ms378878%28v=sql.110%29.aspx\n            case java.sql.Types.FLOAT:\n                columnMetadata.put(positionInSource,\n                        new ColumnMetadata(colName, java.sql.Types.DOUBLE, precision, scale, dateTimeFormatter));\n                break;\n\n            // redirecting BOOLEAN as BIT\n            case java.sql.Types.BOOLEAN:\n                columnMetadata.put(positionInSource,\n                        new ColumnMetadata(colName, java.sql.Types.BIT, precision, scale, dateTimeFormatter));\n                break;\n\n            default:\n                columnMetadata.put(positionInSource,\n                        new ColumnMetadata(colName, jdbcType, precision, scale, dateTimeFormatter));\n        }\n\n        loggerExternal.exiting(loggerPackageName, \"addColumnMetadata\");\n    }\n\n    @Override\n    public boolean next() throws SQLServerException {\n        batchParamIndex++;\n        return batchParamIndex < batchParam.size();\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/dataclassification/SensitivityClassification.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc.dataclassification;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport com.microsoft.sqlserver.jdbc.SQLServerException;\n\n\n/**\n * Provides the functionality to retrieve Sensitivity Classification data as received from SQL Server for the active\n * resultSet\n */\npublic class SensitivityClassification {\n\n    /**\n     * Sensitivity Rank\n     * \n     * https://docs.microsoft.com/sql/relational-databases/system-catalog-views/sys-sensitivity-classifications-transact-sql\n     *\n     */\n    public enum SensitivityRank {\n        /**\n         * rank not defined\n         */\n        NOT_DEFINED(-1),\n\n        /**\n         * rank NONE\n         */\n        NONE(0),\n\n        /**\n         * rank LOW\n         */\n        LOW(10),\n\n        /**\n         * rank MEDIUM\n         */\n        MEDIUM(20),\n\n        /**\n         * rank HIGH\n         */\n        HIGH(30),\n\n        /**\n         * rank CRITICAL\n         */\n        CRITICAL(40);\n\n        private static final SensitivityRank[] VALUES = values();\n        private int rank;\n\n        private SensitivityRank(int rank) {\n            this.rank = rank;\n        }\n\n        /**\n         * Get rank value\n         * \n         * @return rank\n         */\n        public int getValue() {\n            return rank;\n        }\n\n        /**\n         * Check if sensitivity rank value is valid\n         * \n         * @param rank\n         *        rank\n         * \n         * @return if sensitivity rank value is valid\n         * @throws SQLServerException\n         *         if error\n         */\n        public static boolean isValid(int rank) throws SQLServerException {\n            for (SensitivityRank r : VALUES) {\n                if (r.getValue() == rank) {\n                    return true;\n                }\n            }\n            return false;\n        }\n    }\n\n    private List<Label> labels;\n    private List<InformationType> informationTypes;\n    private List<ColumnSensitivity> columnSensitivities;\n    private int sensitivityRank;\n\n    /*\n     * Creating new ArrayList here assures that 'informationTypes' and 'labels' properties will not be null. The Count\n     * of the ColumnSensitivities property will be equal to the number of output columns for the query result set.\n     */\n\n    /**\n     * Constructs a <code>SensitivityClassification</code> Object\n     * \n     * @param labels\n     *        Labels as received from SQL Server\n     * @param informationTypes\n     *        Information Types as received from SQL Server\n     * @param columnSensitivity\n     *        Column Sensitivities as received from SQL Server\n     */\n    public SensitivityClassification(List<Label> labels, List<InformationType> informationTypes,\n            List<ColumnSensitivity> columnSensitivity) {\n        this.labels = new ArrayList<>(labels);\n        this.informationTypes = new ArrayList<>(informationTypes);\n        this.columnSensitivities = new ArrayList<>(columnSensitivity);\n    }\n\n    /**\n     * Constructs a <code>SensitivityClassification</code> Object\n     * \n     * @param labels\n     *        Labels as received from SQL Server\n     * @param informationTypes\n     *        Information Types as received from SQL Server\n     * @param columnSensitivity\n     *        Column Sensitivities as received from SQL Server\n     * @param sensitivityRank\n     *        Sensitivity rank as received from SQL Server\n     */\n    public SensitivityClassification(List<Label> labels, List<InformationType> informationTypes,\n            List<ColumnSensitivity> columnSensitivity, int sensitivityRank) {\n        this.labels = new ArrayList<>(labels);\n        this.informationTypes = new ArrayList<>(informationTypes);\n        this.columnSensitivities = new ArrayList<>(columnSensitivity);\n        this.sensitivityRank = sensitivityRank;\n    }\n\n    /**\n     * Returns the labels for this <code>SensitivityClassification</code> Object\n     * \n     * @return labels\n     */\n    public List<Label> getLabels() {\n        return labels;\n    }\n\n    /**\n     * Returns the information types for this <code>SensitivityClassification</code> Object\n     * \n     * @return informationTypes\n     */\n    public List<InformationType> getInformationTypes() {\n        return informationTypes;\n    }\n\n    /**\n     * Returns the column sensitivity for this <code>SensitivityClassification</code> Object\n     * \n     * @return columnSensitivities\n     */\n    public List<ColumnSensitivity> getColumnSensitivities() {\n        return columnSensitivities;\n    }\n\n    /**\n     * Returns the sensitivity rank\n     * \n     * @return sensitivity rank\n     */\n    public int getSensitivityRank() {\n        return sensitivityRank;\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerResultSet.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.math.BigDecimal;\nimport java.sql.SQLType;\nimport java.util.Calendar;\n\nimport com.microsoft.sqlserver.jdbc.dataclassification.SensitivityClassification;\n\n\n/**\n * Provides an interface to the {@link SQLServerResultSet} class.\n */\npublic interface ISQLServerResultSet extends java.sql.ResultSet {\n\n    /**\n     * TYPE_FORWARD_ONLY + 1000\n     */\n    int TYPE_SS_DIRECT_FORWARD_ONLY = 2003;\n\n    /**\n     * TYPE_FORWARD_ONLY + 1001\n     */\n    int TYPE_SS_SERVER_CURSOR_FORWARD_ONLY = 2004;\n\n    /**\n     * TYPE_SCROLL_INSENSITIVE\n     */\n    int TYPE_SS_SCROLL_STATIC = 1004;\n\n    /**\n     * TYPE_SCROLL_SENSITIVE\n     */\n    int TYPE_SS_SCROLL_KEYSET = 1005;\n\n    /**\n     * TYPE_SCROLL_SENSITIVE + 1\n     */\n    int TYPE_SS_SCROLL_DYNAMIC = 1006;\n\n    /* SQL Server concurrency values */\n\n    /**\n     * CONCUR_UPDATABLE\n     */\n    int CONCUR_SS_OPTIMISTIC_CC = 1008;\n\n    /**\n     * CONCUR_UPDATABLE + 1\n     */\n    int CONCUR_SS_SCROLL_LOCKS = 1009;\n\n    /**\n     * CONCUR_UPDATABLE + 2\n     */\n    int CONCUR_SS_OPTIMISTIC_CCVAL = 1010;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a\n     * com.microsoft.sqlserver.jdbc.Geometry object in the Java programming language.\n     * \n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Geometry getGeometry(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a\n     * com.microsoft.sqlserver.jdbc.Geometry object in the Java programming language.\n     * \n     * @param columnName\n     *        the name of the column\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Geometry getGeometry(String columnName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a\n     * com.microsoft.sqlserver.jdbc.Geography object in the Java programming language.\n     * \n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Geography getGeography(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a\n     * com.microsoft.sqlserver.jdbc.Geography object in the Java programming language.\n     * \n     * @param columnName\n     *        the name of the column\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Geography getGeography(String columnName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a String object in the\n     * Java programming language.\n     * \n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    String getUniqueIdentifier(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a String object in the\n     * Java programming language.\n     * \n     * @param columnLabel\n     *        the name of the column\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    String getUniqueIdentifier(String columnLabel) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    java.sql.Timestamp getDateTime(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param columnName\n     *        is the name of the column\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    java.sql.Timestamp getDateTime(String columnName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language. This method uses the given calendar to construct an appropriate\n     * millisecond value for the timestamp if the underlying database does not store timezone information.\n     * \n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the dateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    java.sql.Timestamp getDateTime(int columnIndex, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language. This method uses the given calendar to construct an appropriate\n     * millisecond value for the timestamp if the underlying database does not store timezone information.\n     * \n     * @param colName\n     *        the label for the column specified with the SQL AS clause. If the SQL AS clause was not specified, then\n     *        the label is the name of the column\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the dateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    java.sql.Timestamp getDateTime(String colName, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    java.sql.Timestamp getSmallDateTime(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param columnName\n     *        is the name of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    java.sql.Timestamp getSmallDateTime(String columnName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the smalldateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    java.sql.Timestamp getSmallDateTime(int columnIndex, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param colName\n     *        The name of a column\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the smalldateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    java.sql.Timestamp getSmallDateTime(String colName, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column as a microsoft.sql.DateTimeOffset object, given a zero-based column\n     * ordinal.\n     * \n     * @param columnIndex\n     *        The zero-based ordinal of a column.\n     * @return A DateTimeOffset Class object.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    microsoft.sql.DateTimeOffset getDateTimeOffset(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a microsoft.sql.DateTimeOffset object, given a column name.\n     * \n     * @param columnName\n     *        The name of a column.\n     * @return A DateTimeOffset Class object.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    microsoft.sql.DateTimeOffset getDateTimeOffset(String columnName) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param columnIndex\n     *        The zero-based ordinal of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    BigDecimal getMoney(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param columnName\n     *        is the name of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null.\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    BigDecimal getMoney(String columnName) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param columnIndex\n     *        The zero-based ordinal of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    BigDecimal getSmallMoney(int columnIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param columnName\n     *        is the name of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null.\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    BigDecimal getSmallMoney(String columnName) throws SQLServerException;\n\n    /**\n     * Updates the value of the column specified to the DateTimeOffset Class value, given a zero-based column ordinal.\n     * \n     * @param index\n     *        The zero-based ordinal of a column.\n     * @param x\n     *        A DateTimeOffset Class object.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateDateTimeOffset(int index, microsoft.sql.DateTimeOffset x) throws SQLServerException;\n\n    /**\n     * Updates the value of the column specified to the DateTimeOffset Class value, given a column name.\n     * \n     * @param columnName\n     *        The name of a column.\n     * @param x\n     *        A DateTimeOffset Class object.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateDateTimeOffset(String columnName, microsoft.sql.DateTimeOffset x) throws SQLServerException;\n\n    /**\n     * Updates the designated column with an {@code Object} value.\n     *\n     * The updater methods are used to update column values in the current row or the insert row. The updater methods do\n     * not update the underlying database; instead the {@code updateRow} or {@code insertRow} methods are called to\n     * update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param precision\n     *        the precision of the column\n     * @param scale\n     *        the scale of the column\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateObject(int index, Object x, int precision, int scale) throws SQLServerException;\n\n    /**\n     * Updates the designated column with an Object value. The updater methods are used to update column values in the\n     * current row or the insert row. The updater methods do not update the underlying database; instead the updateRow\n     * or insertRow methods are called to update the database. If the second argument is an InputStream then the stream\n     * must contain the number of bytes specified by scaleOrLength. If the second argument is a Reader then the reader\n     * must contain the number of characters specified by scaleOrLength. If these conditions are not true the driver\n     * will generate a SQLServerException when the statement is executed. The default implementation will throw\n     * SQLFeatureNotSupportedException\n     * \n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param obj\n     *        the new column value\n     * @param targetSqlType\n     *        the SQL type to be sent to the database\n     * @param scale\n     *        for an object of java.math.BigDecimal , this is the number of digits after the decimal point. For Java\n     *        Object types InputStream and Reader, this is the length of the data in the stream or reader. For all other\n     *        types, this value will be ignored.\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement.If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    void updateObject(int index, Object obj, SQLType targetSqlType, int scale,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * \n     * Updates the designated column with an Object value. The updater methods are used to update column values in the\n     * current row or the insert row. The updater methods do not update the underlying database; instead the updateRow\n     * or insertRow methods are called to update the database. If the second argument is an InputStream then the stream\n     * must contain the number of bytes specified by scaleOrLength. If the second argument is a Reader then the reader\n     * must contain the number of characters specified by scaleOrLength. If these conditions are not true the driver\n     * will generate a SQLServerException when the statement is executed. The default implementation will throw\n     * SQLFeatureNotSupportedException.\n     * \n     * @param columnName\n     *        the label for the column specified with the SQL AS clause. If the SQL AS clause was not specified, then\n     *        the label is the name of the column\n     * @param obj\n     *        the new column value\n     * @param targetSqlType\n     *        the SQL type to be sent to the database\n     * @param scale\n     *        for an object of java.math.BigDecimal , this is the number of digits after the decimal point. For Java\n     *        Object types InputStream and Reader, this is the length of the data in the stream or reader. For all other\n     *        types, this value will be ignored.\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement.If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    void updateObject(String columnName, Object obj, SQLType targetSqlType, int scale,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>boolean</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateBoolean(int index, boolean x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>byte</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateByte(int index, byte x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>short</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateShort(int index, short x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with an <code>int</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateInt(int index, int x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>long</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateLong(int index, long x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>float</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateFloat(int index, float x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>double</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateDouble(int index, double x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>money</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateMoney(int index, BigDecimal x) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>money</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateMoney(int index, BigDecimal x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>money</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param columnName\n     *        the column name\n     * @param x\n     *        the new column value\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    void updateMoney(String columnName, BigDecimal x) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>money</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param columnName\n     *        the column name\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    void updateMoney(String columnName, BigDecimal x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>smallmoney</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateSmallMoney(int index, BigDecimal x) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>smallmoney</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateSmallMoney(int index, BigDecimal x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>smallmoney</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param columnName\n     *        the column name\n     * @param x\n     *        the new column value\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    void updateSmallMoney(String columnName, BigDecimal x) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>smallmoney</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param columnName\n     *        the column name\n     * @param x\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         If any errors occur.\n     */\n    void updateSmallMoney(String columnName, BigDecimal x, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>java.math.BigDecimal</code> value. The updater methods are used to\n     * update column values in the current row or the insert row. The updater methods do not update the underlying\n     * database; instead the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param precision\n     *        the precision of the column\n     * @param scale\n     *        the scale of the column\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateBigDecimal(int index, BigDecimal x, Integer precision, Integer scale) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>java.math.BigDecimal</code> value. The updater methods are used to\n     * update column values in the current row or the insert row. The updater methods do not update the underlying\n     * database; instead the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param x\n     *        the new column value\n     * @param precision\n     *        the precision of the column\n     * @param scale\n     *        the scale of the column\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateBigDecimal(int index, BigDecimal x, Integer precision, Integer scale,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>String</code> value. The updater methods are used to update column\n     * values in the current row or the insert row. The updater methods do not update the underlying database; instead\n     * the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n     * @param columnIndex\n     *        the first column is 1, the second is 2, ...\n     * @param stringValue\n     *        the new column value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    void updateString(int columnIndex, String stringValue, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Updates the designated column with a <code>String</code> value. It is intended for use when updating\n     * <code>NCHAR</code>,<code>NVARCHAR</code> and <code>LONGNVARCHAR</code> columns. The updater methods are used to\n     * update column values in the current row or the insert row. The updater methods do not update the underlying\n     * database; instead the <code>updateRow</code> or <code>insertRow</code> methods are called to update the database.\n     *\n...\n", "needle_token_start": 3979, "needle_token_end": 4046, "code_context_ntokens": 16177, "output": ["```java\nprivate String removeSingleQuote(String s) {\n    int len = s.length();\n    return (s.charAt(0) == '\\'' && s.charAt(len - 1) == '\\'') ? s.substring(1, len - 1) : s;\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "checkValidHoldability", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To ensure that the holdability parameter provided matches one of the acceptable values defined by the JDBC specification and supported by the driver.\n2. **Input**: An integer representing the holdability setting for a result set.\n3. **Output**: Throws an exception if the input is not valid; otherwise, does not return any value.\n4. **Procedure**: The function checks if the input integer matches either of the two predefined valid constants. If the input does not match any of these constants, it formats an error message and throws an exception indicating that the holdability value is invalid.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "     * @return the encoded data\n     */\n    private byte[] toUCS16(String s) {\n        if (s == null)\n            return new byte[0];\n        int l = s.length();\n        byte[] data = new byte[l * 2];\n        int offset = 0;\n        for (int i = 0; i < l; i++) {\n            int c = s.charAt(i);\n            byte b1 = (byte) (c & 0xFF);\n            data[offset++] = b1;\n            data[offset++] = (byte) ((c >> 8) & 0xFF); // Unicode MSB\n        }\n        return data;\n    }\n\n    /**\n     * Encrypt a password for the SQL Server logon.\n     * \n     * @param pwd\n     *        the password\n     * @return the encrypted password\n     */\n    private byte[] encryptPassword(String pwd) {\n        // Changed to handle non ascii passwords\n        if (pwd == null)\n            pwd = \"\";\n        int len = pwd.length();\n        byte[] data = new byte[len * 2];\n        for (int i1 = 0; i1 < len; i1++) {\n            int j1 = pwd.charAt(i1) ^ 0x5a5a;\n            j1 = (j1 & 0xf) << 4 | (j1 & 0xf0) >> 4 | (j1 & 0xf00) << 4 | (j1 & 0xf000) >> 4;\n            byte b1 = (byte) ((j1 & 0xFF00) >> 8);\n            data[(i1 * 2) + 1] = b1;\n            byte b2 = (byte) (j1 & 0x00FF);\n            data[(i1 * 2) + 0] = b2;\n        }\n        return data;\n    }\n\n    /**\n     * Send a TDS 7.x logon packet.\n     * \n     * @param logonCommand\n     *        the logon command\n     * @param authentication\n     *        SSPI authentication\n     * @param fedAuthFeatureExtensionData\n     *        fedauth feature extension data\n     * @throws SQLServerException\n     */\n    private void sendLogon(LogonCommand logonCommand, SSPIAuthentication authentication,\n            FederatedAuthenticationFeatureExtensionData fedAuthFeatureExtensionData) throws SQLServerException {\n        // TDS token handler class for processing logon responses.\n        //\n        // Note:\n        // As a local inner class, LogonProcessor implicitly has access to private\n        // members of SQLServerConnection. Certain JVM implementations generate\n        // package scope accessors to any private members touched by this class,\n        // effectively changing visibility of such members from private to package.\n        // Therefore, it is IMPORTANT then for this class not to touch private\n        // member variables in SQLServerConnection that contain secure information.\n        final class LogonProcessor extends TDSTokenHandler {\n            private final SSPIAuthentication auth;\n            private byte[] secBlobOut = null;\n            StreamLoginAck loginAckToken;\n\n            LogonProcessor(SSPIAuthentication auth) {\n                super(\"logon\");\n                this.auth = auth;\n                this.loginAckToken = null;\n            }\n\n            @Override\n            boolean onSSPI(TDSReader tdsReader) throws SQLServerException {\n                StreamSSPI ack = new StreamSSPI();\n                ack.setFromTDS(tdsReader);\n\n                // Extract SSPI data from the response. If another round trip is\n                // required then we will start it after we finish processing the\n                // rest of this response.\n                boolean[] done = {false};\n                secBlobOut = auth.generateClientContext(ack.sspiBlob, done);\n                return true;\n            }\n\n            @Override\n            boolean onLoginAck(TDSReader tdsReader) throws SQLServerException {\n                loginAckToken = new StreamLoginAck();\n                loginAckToken.setFromTDS(tdsReader);\n                sqlServerVersion = loginAckToken.sSQLServerVersion;\n                tdsVersion = loginAckToken.tdsVersion;\n                return true;\n            }\n\n            final boolean complete(LogonCommand logonCommand, TDSReader tdsReader) throws SQLServerException {\n                // If we have the login ack already then we're done processing.\n                if (null != loginAckToken)\n                    return true;\n\n                // No login ack yet. Check if there is more SSPI handshake to do...\n                if (null != secBlobOut && 0 != secBlobOut.length) {\n                    // Yes, there is. So start the next SSPI round trip and indicate to\n                    // our caller that it needs to keep the processing loop going.\n                    logonCommand.startRequest(TDS.PKT_SSPI).writeBytes(secBlobOut, 0, secBlobOut.length);\n                    return false;\n                }\n\n                // The login ack comes in its own complete TDS response message.\n                // So integrated auth effectively receives more response messages from\n                // the server than it sends request messages from the driver.\n                // To ensure that the rest of the response can be read, fake another\n                // request to the server so that the channel sees int auth login\n                // as a symmetric conversation.\n                logonCommand.startRequest(TDS.PKT_SSPI);\n                logonCommand.onRequestComplete();\n                ++tdsChannel.numMsgsSent;\n\n                TDSParser.parse(tdsReader, this);\n                return true;\n            }\n        }\n\n        // Cannot use SSPI when server has responded 0x01 for FedAuthRequired PreLogin Option.\n        assert !(integratedSecurity && fedAuthRequiredPreLoginResponse);\n        // Cannot use both SSPI and FedAuth\n        assert (!integratedSecurity) || !(federatedAuthenticationInfoRequested || federatedAuthenticationRequested);\n        // fedAuthFeatureExtensionData provided without fed auth feature request\n        assert (null == fedAuthFeatureExtensionData)\n                || (federatedAuthenticationInfoRequested || federatedAuthenticationRequested);\n        // Fed Auth feature requested without specifying fedAuthFeatureExtensionData.\n        assert (null != fedAuthFeatureExtensionData\n                || !(federatedAuthenticationInfoRequested || federatedAuthenticationRequested));\n\n        String sUser = activeConnectionProperties.getProperty(SQLServerDriverStringProperty.USER.toString());\n        String sPwd = activeConnectionProperties.getProperty(SQLServerDriverStringProperty.PASSWORD.toString());\n        String appName = activeConnectionProperties\n                .getProperty(SQLServerDriverStringProperty.APPLICATION_NAME.toString());\n        String interfaceLibName = \"Microsoft JDBC Driver \" + SQLJdbcVersion.MAJOR + \".\" + SQLJdbcVersion.MINOR;\n        String databaseName = activeConnectionProperties\n                .getProperty(SQLServerDriverStringProperty.DATABASE_NAME.toString());\n\n        String serverName;\n        if (null != currentConnectPlaceHolder) {\n            serverName = currentConnectPlaceHolder.getFullServerName();\n        } else {\n            serverName = activeConnectionProperties.getProperty(SQLServerDriverStringProperty.SERVER_NAME.toString());\n            if (null != activeConnectionProperties\n                    .getProperty(SQLServerDriverStringProperty.INSTANCE_NAME.toString())) {\n                serverName += \"\\\\\" + activeConnectionProperties\n                        .getProperty(SQLServerDriverStringProperty.INSTANCE_NAME.toString());\n            }\n        }\n\n        if (null != serverName && serverName.length() > 128) {\n            serverName = serverName.substring(0, 128);\n        }\n\n        byte[] secBlob = new byte[0];\n        boolean[] done = {false};\n        if (null != authentication) {\n            secBlob = authentication.generateClientContext(secBlob, done);\n            sUser = null;\n            sPwd = null;\n        }\n\n        byte[] hostnameBytes = toUCS16(hostName);\n        byte[] userBytes = toUCS16(sUser);\n        byte[] passwordBytes = encryptPassword(sPwd);\n        int passwordLen = (null != passwordBytes) ? passwordBytes.length : 0;\n        byte[] appNameBytes = toUCS16(appName);\n        byte[] serverNameBytes = toUCS16(serverName);\n        byte[] interfaceLibNameBytes = toUCS16(interfaceLibName);\n        byte[] interfaceLibVersionBytes = {(byte) SQLJdbcVersion.BUILD, (byte) SQLJdbcVersion.PATCH,\n                (byte) SQLJdbcVersion.MINOR, (byte) SQLJdbcVersion.MAJOR};\n        byte[] databaseNameBytes = toUCS16(databaseName);\n        int dataLen = 0;\n\n        // TDS version 8 if strict mode\n        // Denali --> TDS 7.4, Katmai (10.0) & later 7.3B, Prelogin disconnects anything older\n        if (encryptOption.compareToIgnoreCase(EncryptOption.STRICT.toString()) == 0) {\n            tdsVersion = TDS.VER_TDS80;\n        } else if (serverMajorVersion >= 11) {\n            tdsVersion = TDS.VER_DENALI;\n        } else if (serverMajorVersion >= 10) {\n            tdsVersion = TDS.VER_KATMAI;\n        } else if (serverMajorVersion >= 9) {\n            tdsVersion = TDS.VER_YUKON;\n        } else {\n            assert false : \"prelogin did not disconnect for the old version: \" + serverMajorVersion;\n        }\n\n        final int tdsLoginRequestBaseLength = 94;\n        TDSWriter tdsWriter = logonCommand.startRequest(TDS.PKT_LOGON70);\n\n        int len = tdsLoginRequestBaseLength + hostnameBytes.length + appNameBytes.length + serverNameBytes.length\n                + interfaceLibNameBytes.length + databaseNameBytes.length + ((secBlob != null) ? secBlob.length : 0)\n                + 4; // AE is always on;\n\n        // only add lengths of password and username if not using SSPI or requesting federated authentication info\n        if (!integratedSecurity && !(federatedAuthenticationInfoRequested || federatedAuthenticationRequested)\n                && null == clientCertificate) {\n            len = len + passwordLen + userBytes.length;\n        }\n\n        int aeOffset = len;\n        // AE is always ON\n        len += writeAEFeatureRequest(false, tdsWriter);\n        if (federatedAuthenticationInfoRequested || federatedAuthenticationRequested) {\n            len = len + writeFedAuthFeatureRequest(false, tdsWriter, fedAuthFeatureExtensionData);\n        }\n\n        // Data Classification is always enabled (by default)\n        len += writeDataClassificationFeatureRequest(false, tdsWriter);\n\n        len = len + writeUTF8SupportFeatureRequest(false, tdsWriter);\n\n        len = len + writeDNSCacheFeatureRequest(false, tdsWriter);\n\n        len = len + 1; // add 1 to length because of FeatureEx terminator\n\n        // Idle Connection Resiliency is requested\n        if (connectRetryCount > 0) {\n            len = len + writeIdleConnectionResiliencyRequest(false, tdsWriter);\n        }\n\n        // Length of entire Login 7 packet\n        tdsWriter.writeInt(len);\n        tdsWriter.writeInt(tdsVersion);\n        tdsWriter.writeInt(requestedPacketSize);\n        tdsWriter.writeBytes(interfaceLibVersionBytes); // writeBytes() is little endian\n        tdsWriter.writeInt(DriverJDBCVersion.getProcessId()); // Client process ID\n        tdsWriter.writeInt(0); // Primary server connection ID\n\n        tdsWriter.writeByte((byte) (// OptionFlags1:\n        TDS.LOGIN_OPTION1_ORDER_X86 | // X86 byte order for numeric & datetime types\n                TDS.LOGIN_OPTION1_CHARSET_ASCII | // ASCII character set\n                TDS.LOGIN_OPTION1_FLOAT_IEEE_754 | // IEEE 754 floating point representation\n                TDS.LOGIN_OPTION1_DUMPLOAD_ON | // Require dump/load BCP capabilities\n                TDS.LOGIN_OPTION1_USE_DB_OFF | // No ENVCHANGE after USE DATABASE\n                TDS.LOGIN_OPTION1_INIT_DB_FATAL | // Fail connection if initial database change fails\n                TDS.LOGIN_OPTION1_SET_LANG_ON // Warn on SET LANGUAGE stmt\n        ));\n\n        // OptionFlags2:\n        tdsWriter.writeByte((byte) (TDS.LOGIN_OPTION2_INIT_LANG_FATAL | // Fail connection if initial language change\n                                                                        // fails\n                TDS.LOGIN_OPTION2_ODBC_ON | // Use ODBC defaults (ANSI_DEFAULTS ON, IMPLICIT_TRANSACTIONS OFF, TEXTSIZE\n                                            // inf, ROWCOUNT inf)\n                (replication ? TDS.LOGIN_OPTION2_USER_SQLREPL_ON : TDS.LOGIN_OPTION2_USER_SQLREPL_OFF)\n                | (integratedSecurity ? // integrated security if integratedSecurity requested\n                                      TDS.LOGIN_OPTION2_INTEGRATED_SECURITY_ON\n                                      : TDS.LOGIN_OPTION2_INTEGRATED_SECURITY_OFF)));\n\n        // TypeFlags\n        tdsWriter.writeByte((byte) (TDS.LOGIN_SQLTYPE_DEFAULT | (applicationIntent != null\n                && applicationIntent.equals(ApplicationIntent.READ_ONLY) ? TDS.LOGIN_READ_ONLY_INTENT\n                                                                         : TDS.LOGIN_READ_WRITE_INTENT)));\n\n        // OptionFlags3\n        byte colEncSetting;\n        // AE is always ON\n        {\n            colEncSetting = TDS.LOGIN_OPTION3_FEATURE_EXTENSION;\n        }\n\n        // Accept unknown collations from Katmai & later servers\n        tdsWriter.writeByte((byte) (TDS.LOGIN_OPTION3_DEFAULT | colEncSetting\n                | ((serverMajorVersion >= 10) ? TDS.LOGIN_OPTION3_UNKNOWN_COLLATION_HANDLING : 0)));\n\n        tdsWriter.writeInt((byte) 0); // Client time zone\n        tdsWriter.writeInt((byte) 0); // Client LCID\n\n        tdsWriter.writeShort((short) tdsLoginRequestBaseLength);\n\n        // Hostname\n        tdsWriter.writeShort((short) ((hostName != null && !hostName.isEmpty()) ? hostName.length() : 0));\n        dataLen += hostnameBytes.length;\n\n        // Only send user/password over if not NTLM or fSSPI or fed auth ADAL... If both user/password and SSPI are in\n        // login rec, only SSPI is used.\n        if (ntlmAuthentication) {\n            tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n            tdsWriter.writeShort((short) (0));\n            tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n            tdsWriter.writeShort((short) (0));\n\n        } else if (!integratedSecurity && !(federatedAuthenticationInfoRequested || federatedAuthenticationRequested)\n                && null == clientCertificate) {\n            // User and Password\n            tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n            tdsWriter.writeShort((short) (sUser == null ? 0 : sUser.length()));\n            dataLen += userBytes.length;\n\n            tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n            tdsWriter.writeShort((short) (sPwd == null ? 0 : sPwd.length()));\n            dataLen += passwordLen;\n\n        } else {\n            // User and Password are null\n            tdsWriter.writeShort((short) (0));\n            tdsWriter.writeShort((short) (0));\n            tdsWriter.writeShort((short) (0));\n            tdsWriter.writeShort((short) (0));\n        }\n\n        // App name\n        tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n        tdsWriter.writeShort((short) (appName == null ? 0 : appName.length()));\n        dataLen += appNameBytes.length;\n\n        // Server name\n        tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n        tdsWriter.writeShort((short) (serverName == null ? 0 : serverName.length()));\n        dataLen += serverNameBytes.length;\n\n        // Unused\n        tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n        // AE is always ON\n        {\n            tdsWriter.writeShort((short) 4);\n            dataLen += 4;\n        }\n\n        // Interface library name\n        assert null != interfaceLibName;\n        tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n        tdsWriter.writeShort((short) (interfaceLibName.length()));\n        dataLen += interfaceLibNameBytes.length;\n\n        // Language\n        tdsWriter.writeShort((short) 0);\n        tdsWriter.writeShort((short) 0);\n\n        // Database\n        tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n        tdsWriter.writeShort((short) (databaseName == null ? 0 : databaseName.length()));\n        dataLen += databaseNameBytes.length;\n\n        // Client ID (from MAC addr)\n        tdsWriter.writeBytes(netAddress);\n\n        final int uShortMax = 65535;\n        // SSPI data\n        if (!integratedSecurity) {\n            tdsWriter.writeShort((short) 0);\n            tdsWriter.writeShort((short) 0);\n        } else {\n            tdsWriter.writeShort((short) (tdsLoginRequestBaseLength + dataLen));\n            if (uShortMax <= secBlob.length) {\n                tdsWriter.writeShort((short) (uShortMax));\n            } else {\n                tdsWriter.writeShort((short) (secBlob.length));\n            }\n        }\n\n        // Database to attach during connection process\n        tdsWriter.writeShort((short) 0);\n        tdsWriter.writeShort((short) 0);\n\n        if (tdsVersion >= TDS.VER_YUKON || tdsVersion == TDS.VER_TDS80) {\n            // TDS 7.2: Password change\n            tdsWriter.writeShort((short) 0);\n            tdsWriter.writeShort((short) 0);\n\n            // TDS 7.2: 32-bit SSPI byte count (used if 16 bits above were not sufficient)\n            if (null != secBlob && uShortMax <= secBlob.length) {\n                tdsWriter.writeInt(secBlob.length);\n            } else {\n                tdsWriter.writeInt((short) 0);\n            }\n        }\n\n        tdsWriter.writeBytes(hostnameBytes);\n\n        // Don't allow user credentials to be logged\n        tdsWriter.setDataLoggable(false);\n\n        // if we are using NTLM or SSPI or fed auth ADAL, do not send over username/password, since we will use SSPI\n        // instead\n        // Also do not send username or password if user is attempting client certificate authentication.\n        if (!integratedSecurity && !(federatedAuthenticationInfoRequested || federatedAuthenticationRequested)\n                && null == clientCertificate) {\n            tdsWriter.writeBytes(userBytes); // Username\n            tdsWriter.writeBytes(passwordBytes); // Password (encrypted)\n        }\n        tdsWriter.setDataLoggable(true);\n\n        tdsWriter.writeBytes(appNameBytes); // application name\n        tdsWriter.writeBytes(serverNameBytes); // server name\n\n        // AE is always ON\n        tdsWriter.writeInt(aeOffset);\n\n        tdsWriter.writeBytes(interfaceLibNameBytes); // interfaceLibName\n        tdsWriter.writeBytes(databaseNameBytes); // databaseName\n\n        // Don't allow user credentials to be logged\n        tdsWriter.setDataLoggable(false);\n\n        // SSPI data\n        if (integratedSecurity) {\n            tdsWriter.writeBytes(secBlob, 0, secBlob.length);\n        }\n\n        // AE is always ON\n        writeAEFeatureRequest(true, tdsWriter);\n\n        if (federatedAuthenticationInfoRequested || federatedAuthenticationRequested) {\n            writeFedAuthFeatureRequest(true, tdsWriter, fedAuthFeatureExtensionData);\n        }\n\n        writeDataClassificationFeatureRequest(true, tdsWriter);\n        writeUTF8SupportFeatureRequest(true, tdsWriter);\n        writeDNSCacheFeatureRequest(true, tdsWriter);\n\n        // Idle Connection Resiliency is requested\n        if (connectRetryCount > 0) {\n            writeIdleConnectionResiliencyRequest(true, tdsWriter);\n        }\n\n        tdsWriter.writeByte((byte) TDS.FEATURE_EXT_TERMINATOR);\n        tdsWriter.setDataLoggable(true);\n\n        LogonProcessor logonProcessor = new LogonProcessor(authentication);\n        TDSReader tdsReader;\n        do {\n            tdsReader = logonCommand.startResponse();\n            sessionRecovery.setConnectionRecoveryPossible(false);\n            TDSParser.parse(tdsReader, logonProcessor);\n        } while (!logonProcessor.complete(logonCommand, tdsReader));\n\n        if (sessionRecovery.isReconnectRunning() && !sessionRecovery.isConnectionRecoveryPossible()) {\n            if (connectionlogger.isLoggable(Level.WARNING)) {\n                connectionlogger.warning(this.toString()\n                        + \"SessionRecovery feature extension ack was not sent by the server during reconnection.\");\n            }\n            terminate(SQLServerException.DRIVER_ERROR_INVALID_TDS,\n                    SQLServerException.getErrString(\"R_crClientNoRecoveryAckFromLogin\"));\n        }\n        if (connectRetryCount > 0 && !sessionRecovery.isReconnectRunning()) {\n            sessionRecovery.getSessionStateTable().setOriginalCatalog(sCatalog);\n            sessionRecovery.getSessionStateTable().setOriginalCollation(databaseCollation);\n            sessionRecovery.getSessionStateTable().setOriginalLanguage(sLanguage);\n        }\n    }\n\n    /* --------------- JDBC 3.0 ------------- */\n\n    /**\n     * Checks that the holdability argument is one of the values allowed by the JDBC spec and by this driver.\n     */\n    \nprivate void checkValidHoldability(int holdability) throws SQLServerException {\n        if (holdability != ResultSet.HOLD_CURSORS_OVER_COMMIT && holdability != ResultSet.CLOSE_CURSORS_AT_COMMIT) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidHoldability\"));\n            SQLServerException.makeFromDriverError(this, this, form.format(new Object[] {holdability}), null, true);\n        }\n    }\n\n    /**\n     * Checks that the proposed statement holdability matches this connection's current holdability.\n     *\n     * SQL Server doesn't support per-statement holdability, so the statement's proposed holdability must match its\n     * parent connection's. Note that this doesn't stop anyone from changing the holdability of the connection after\n     * creating the statement. Apps should always call Statement.getResultSetHoldability to check the holdability of\n     * ResultSets that would be created, and/or ResultSet.getHoldability to check the holdability of an existing\n     * ResultSet.\n     */\n    private void checkMatchesCurrentHoldability(int resultSetHoldability) throws SQLServerException {\n        if (resultSetHoldability != this.holdability) {\n            SQLServerException.makeFromDriverError(this, this,\n                    SQLServerException.getErrString(\"R_sqlServerHoldability\"), null, false);\n        }\n    }\n\n    @Override\n    public Statement createStatement(int nType, int nConcur, int resultSetHoldability) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, CREATE_STATEMENT,\n                new Object[] {nType, nConcur, resultSetHoldability});\n        Statement st = createStatement(nType, nConcur, resultSetHoldability,\n                SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING);\n        loggerExternal.exiting(loggingClassName, CREATE_STATEMENT, st);\n        return st;\n    }\n\n    @Override\n    public Statement createStatement(int nType, int nConcur, int resultSetHoldability,\n            SQLServerStatementColumnEncryptionSetting stmtColEncSetting) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, CREATE_STATEMENT,\n                new Object[] {nType, nConcur, resultSetHoldability, stmtColEncSetting});\n        checkClosed();\n        checkValidHoldability(resultSetHoldability);\n        checkMatchesCurrentHoldability(resultSetHoldability);\n        Statement st = new SQLServerStatement(this, nType, nConcur, stmtColEncSetting);\n        if (requestStarted) {\n            addOpenStatement((ISQLServerStatement) st);\n        }\n        loggerExternal.exiting(loggingClassName, CREATE_STATEMENT, st);\n        return st;\n    }\n\n    @Override\n    public PreparedStatement prepareStatement(java.lang.String sql, int nType, int nConcur,\n            int resultSetHoldability) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"prepareStatement\",\n                new Object[] {nType, nConcur, resultSetHoldability});\n        PreparedStatement st = prepareStatement(sql, nType, nConcur, resultSetHoldability,\n                SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING);\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", st);\n        return st;\n    }\n\n    @Override\n    public PreparedStatement prepareStatement(java.lang.String sql, int nType, int nConcur, int resultSetHoldability,\n            SQLServerStatementColumnEncryptionSetting stmtColEncSetting) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"prepareStatement\",\n                new Object[] {nType, nConcur, resultSetHoldability, stmtColEncSetting});\n        checkClosed();\n        checkValidHoldability(resultSetHoldability);\n        checkMatchesCurrentHoldability(resultSetHoldability);\n\n        PreparedStatement st = new SQLServerPreparedStatement(this, sql, nType, nConcur, stmtColEncSetting);\n\n        if (requestStarted) {\n            addOpenStatement((ISQLServerStatement) st);\n        }\n\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", st);\n        return st;\n    }\n\n    @Override\n    public CallableStatement prepareCall(String sql, int nType, int nConcur,\n            int resultSetHoldability) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"prepareStatement\",\n                new Object[] {nType, nConcur, resultSetHoldability});\n        CallableStatement st = prepareCall(sql, nType, nConcur, resultSetHoldability,\n                SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING);\n        loggerExternal.exiting(loggingClassName, \"prepareCall\", st);\n        return st;\n    }\n\n    @Override\n    public CallableStatement prepareCall(String sql, int nType, int nConcur, int resultSetHoldability,\n            SQLServerStatementColumnEncryptionSetting stmtColEncSetiing) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"prepareStatement\",\n                new Object[] {nType, nConcur, resultSetHoldability, stmtColEncSetiing});\n        checkClosed();\n        checkValidHoldability(resultSetHoldability);\n        checkMatchesCurrentHoldability(resultSetHoldability);\n\n        CallableStatement st = new SQLServerCallableStatement(this, sql, nType, nConcur, stmtColEncSetiing);\n\n        if (requestStarted) {\n            addOpenStatement((ISQLServerStatement) st);\n        }\n\n        loggerExternal.exiting(loggingClassName, \"prepareCall\", st);\n        return st;\n    }\n\n    /* JDBC 3.0 Auto generated keys */\n\n    @Override\n    public PreparedStatement prepareStatement(String sql, int flag) throws SQLServerException {\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {\n            loggerExternal.entering(loggingClassName, \"prepareStatement\", new Object[] {sql, flag});\n        }\n        SQLServerPreparedStatement ps = (SQLServerPreparedStatement) prepareStatement(sql, flag,\n                SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING);\n\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", ps);\n        return ps;\n    }\n\n    @Override\n    public PreparedStatement prepareStatement(String sql, int flag,\n            SQLServerStatementColumnEncryptionSetting stmtColEncSetting) throws SQLServerException {\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {\n            loggerExternal.entering(loggingClassName, \"prepareStatement\", new Object[] {sql, flag, stmtColEncSetting});\n        }\n        checkClosed();\n        SQLServerPreparedStatement ps = (SQLServerPreparedStatement) prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY,\n                ResultSet.CONCUR_READ_ONLY, stmtColEncSetting);\n        ps.bRequestedGeneratedKeys = (flag == Statement.RETURN_GENERATED_KEYS);\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", ps);\n        return ps;\n    }\n\n    @Override\n    public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLServerException {\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {\n            loggerExternal.entering(loggingClassName, \"prepareStatement\", new Object[] {sql, columnIndexes});\n        }\n        SQLServerPreparedStatement ps = (SQLServerPreparedStatement) prepareStatement(sql, columnIndexes,\n                SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING);\n\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", ps);\n        return ps;\n    }\n\n    @Override\n    public PreparedStatement prepareStatement(String sql, int[] columnIndexes,\n            SQLServerStatementColumnEncryptionSetting stmtColEncSetting) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"prepareStatement\",\n                new Object[] {sql, columnIndexes, stmtColEncSetting});\n\n        checkClosed();\n        if (columnIndexes == null || columnIndexes.length != 1) {\n            SQLServerException.makeFromDriverError(this, this,\n                    SQLServerException.getErrString(\"R_invalidColumnArrayLength\"), null, false);\n        }\n        SQLServerPreparedStatement ps = (SQLServerPreparedStatement) prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY,\n                ResultSet.CONCUR_READ_ONLY, stmtColEncSetting);\n        ps.bRequestedGeneratedKeys = true;\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", ps);\n        return ps;\n    }\n\n    @Override\n    public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLServerException {\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {\n            loggerExternal.entering(loggingClassName, \"prepareStatement\", new Object[] {sql, columnNames});\n        }\n\n        SQLServerPreparedStatement ps = (SQLServerPreparedStatement) prepareStatement(sql, columnNames,\n                SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING);\n\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", ps);\n        return ps;\n    }\n\n    @Override\n    public PreparedStatement prepareStatement(String sql, String[] columnNames,\n            SQLServerStatementColumnEncryptionSetting stmtColEncSetting) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"prepareStatement\",\n                new Object[] {sql, columnNames, stmtColEncSetting});\n        checkClosed();\n        if (columnNames == null || columnNames.length != 1) {\n            SQLServerException.makeFromDriverError(this, this,\n                    SQLServerException.getErrString(\"R_invalidColumnArrayLength\"), null, false);\n        }\n        SQLServerPreparedStatement ps = (SQLServerPreparedStatement) prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY,\n                ResultSet.CONCUR_READ_ONLY, stmtColEncSetting);\n        ps.bRequestedGeneratedKeys = true;\n        loggerExternal.exiting(loggingClassName, \"prepareStatement\", ps);\n        return ps;\n    }\n\n    /* JDBC 3.0 Savepoints */\n\n    @Override\n    public void releaseSavepoint(Savepoint savepoint) throws SQLException {\n        loggerExternal.entering(loggingClassName, \"releaseSavepoint\", savepoint);\n        SQLServerException.throwNotSupportedException(this, null);\n    }\n\n    final private Savepoint setNamedSavepoint(String sName) throws SQLServerException {\n        if (databaseAutoCommitMode) {\n            SQLServerException.makeFromDriverError(this, this, SQLServerException.getErrString(\"R_cantSetSavepoint\"),\n                    null, false);\n        }\n\n        SQLServerSavepoint s = new SQLServerSavepoint(this, sName);\n\n        // Create the named savepoint. Note that we explicitly start a transaction if we\n        // are not already in one. This is to allow the savepoint to be created even if\n        // setSavepoint() is called before executing any other implicit-transaction-starting\n        // statements. Also note that the way we create this transaction is rather weird.\n        // This is because the server creates a nested transaction (@@TRANCOUNT = 2) rather\n        // than just the outer transaction (@@TRANCOUNT = 1). Should this limitation ever\n        // change, the T-SQL below should still work.\n        connectionCommand(\"IF @@TRANCOUNT = 0 BEGIN BEGIN TRAN IF @@TRANCOUNT = 2 COMMIT TRAN END SAVE TRAN \"\n                + Util.escapeSQLId(s.getLabel()), SET_SAVE_POINT);\n\n        return s;\n    }\n\n    @Override\n    public Savepoint setSavepoint(String sName) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, SET_SAVE_POINT, sName);\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkClosed();\n        Savepoint pt = setNamedSavepoint(sName);\n        loggerExternal.exiting(loggingClassName, SET_SAVE_POINT, pt);\n        return pt;\n    }\n\n    @Override\n    public Savepoint setSavepoint() throws SQLServerException {\n        loggerExternal.entering(loggingClassName, SET_SAVE_POINT);\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkClosed();\n        Savepoint pt = setNamedSavepoint(null);\n        loggerExternal.exiting(loggingClassName, SET_SAVE_POINT, pt);\n        return pt;\n    }\n\n    @Override\n    public void rollback(Savepoint s) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"rollback\", s);\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkClosed();\n        if (databaseAutoCommitMode) {\n            SQLServerException.makeFromDriverError(this, this, SQLServerException.getErrString(\"R_cantInvokeRollback\"),\n                    null, false);\n        }\n        connectionCommand(\"IF @@TRANCOUNT > 0 ROLLBACK TRAN \" + Util.escapeSQLId(((SQLServerSavepoint) s).getLabel()),\n                \"rollbackSavepoint\");\n        loggerExternal.exiting(loggingClassName, \"rollback\");\n    }\n\n    @Override\n    public int getHoldability() throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"getHoldability\");\n        if (loggerExternal.isLoggable(Level.FINER))\n            loggerExternal.exiting(loggingClassName, \"getHoldability\", holdability);\n        return holdability;\n    }\n\n    @Override\n    public void setHoldability(int holdability) throws SQLServerException {\n        loggerExternal.entering(loggingClassName, \"setHoldability\", holdability);\n\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkValidHoldability(holdability);\n        checkClosed();\n\n        if (this.holdability != holdability) {\n            connectionCommand(\n                    (holdability == ResultSet.CLOSE_CURSORS_AT_COMMIT) ? \"SET CURSOR_CLOSE_ON_COMMIT ON\"\n                                                                       : \"SET CURSOR_CLOSE_ON_COMMIT OFF\",\n                    \"setHoldability\");\n\n            this.holdability = holdability;\n        }\n\n        loggerExternal.exiting(loggingClassName, \"setHoldability\");\n    }\n\n    @Override\n    public int getNetworkTimeout() throws SQLException {\n        loggerExternal.entering(loggingClassName, \"getNetworkTimeout\");\n\n        checkClosed();\n\n        int timeout = 0;\n        try {\n            timeout = tdsChannel.getNetworkTimeout();\n        } catch (IOException ioe) {\n            terminate(SQLServerException.DRIVER_ERROR_IO_FAILED, ioe.getMessage(), ioe);\n        }\n\n        loggerExternal.exiting(loggingClassName, \"getNetworkTimeout\");\n        return timeout;\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\")\n    public void setNetworkTimeout(Executor executor, int timeout) throws SQLException {\n        loggerExternal.entering(loggingClassName, SET_NETWORK_TIMEOUT_PERM, timeout);\n\n        if (timeout < 0) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidSocketTimeout\"));\n            Object[] msgArgs = {timeout};\n            SQLServerException.makeFromDriverError(this, this, form.format(msgArgs), null, false);\n        }\n\n        checkClosed();\n\n        // check for setNetworkTimeout permission\n        SecurityManager secMgr = System.getSecurityManager();\n        if (secMgr != null) {\n            try {\n                SQLPermission perm = new SQLPermission(SET_NETWORK_TIMEOUT_PERM);\n                secMgr.checkPermission(perm);\n            } catch (SecurityException ex) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_permissionDenied\"));\n                Object[] msgArgs = {SET_NETWORK_TIMEOUT_PERM};\n                SQLServerException.makeFromDriverError(this, this, form.format(msgArgs), null, true);\n            }\n        }\n\n        try {\n            tdsChannel.setNetworkTimeout(timeout);\n        } catch (IOException ioe) {\n            terminate(SQLServerException.DRIVER_ERROR_IO_FAILED, ioe.getMessage(), ioe);\n        }\n\n        loggerExternal.exiting(loggingClassName, SET_NETWORK_TIMEOUT_PERM);\n    }\n\n    @Override\n    public String getSchema() throws SQLException {\n        loggerExternal.entering(loggingClassName, \"getSchema\");\n\n        checkClosed();\n\n        try (SQLServerStatement stmt = (SQLServerStatement) this.createStatement();\n                SQLServerResultSet resultSet = stmt.executeQueryInternal(\"SELECT SCHEMA_NAME()\")) {\n            if (resultSet != null) {\n                resultSet.next();\n                return resultSet.getString(1);\n            } else {\n                SQLServerException.makeFromDriverError(this, this, SQLServerException.getErrString(\"R_getSchemaError\"),\n                        null, true);\n            }\n        } catch (SQLException e) {\n            if (isSessionUnAvailable()) {\n                throw e;\n            }\n\n            SQLServerException.makeFromDriverError(this, this, SQLServerException.getErrString(\"R_getSchemaError\"),\n                    null, true);\n        }\n\n        loggerExternal.exiting(loggingClassName, \"getSchema\");\n        return null;\n    }\n\n    @Override\n    public void setSchema(String schema) throws SQLException {\n        loggerExternal.entering(loggingClassName, \"setSchema\", schema);\n        checkClosed();\n        addWarning(SQLServerException.getErrString(\"R_setSchemaWarning\"));\n\n        loggerExternal.exiting(loggingClassName, \"setSchema\");\n    }\n\n    @Override\n    public void setSendTimeAsDatetime(boolean sendTimeAsDateTimeValue) {\n        sendTimeAsDatetime = sendTimeAsDateTimeValue;\n    }\n\n    @Override\n    public void setDatetimeParameterType(String datetimeParameterTypeValue) throws SQLServerException {\n        // cast the value to lowercase, so the case in the connection string does not matter\n        if (datetimeParameterTypeValue != null) {\n            datetimeParameterTypeValue = datetimeParameterTypeValue.toLowerCase();\n        }\n\n        datetimeParameterType = DatetimeType.valueOfString(datetimeParameterTypeValue);\n    }\n\n    @Override\n    public void setUseFmtOnly(boolean useFmtOnly) {\n        this.useFmtOnly = useFmtOnly;\n    }\n\n    @Override\n    public final boolean getUseFmtOnly() {\n        return useFmtOnly;\n    }\n\n    @Override\n    public java.sql.Array createArrayOf(String typeName, Object[] elements) throws SQLException {\n        SQLServerException.throwNotSupportedException(this, null);\n        return null;\n    }\n\n    @Override\n    public java.sql.Blob createBlob() throws SQLException {\n        checkClosed();\n        return new SQLServerBlob(this);\n    }\n\n    @Override\n    public java.sql.Clob createClob() throws SQLException {\n        checkClosed();\n        return new SQLServerClob(this);\n    }\n\n    @Override\n    public java.sql.NClob createNClob() throws SQLException {\n        checkClosed();\n        return new SQLServerNClob(this);\n    }\n\n    @Override\n    public SQLXML createSQLXML() throws SQLException {\n        loggerExternal.entering(loggingClassName, \"createSQLXML\");\n        SQLXML sqlxml = new SQLServerSQLXML(this);\n\n        if (loggerExternal.isLoggable(Level.FINER))\n            loggerExternal.exiting(loggingClassName, \"createSQLXML\", sqlxml);\n        return sqlxml;\n    }\n\n    @Override\n    public java.sql.Struct createStruct(String typeName, Object[] attributes) throws SQLException {\n        SQLServerException.throwNotSupportedException(this, null);\n        return null;\n    }\n\n    String getTrustedServerNameAE() {\n        return trustedServerNameAE.toUpperCase();\n    }\n\n    @Override\n    public Properties getClientInfo() throws SQLException {\n        loggerExternal.entering(loggingClassName, \"getClientInfo\");\n        checkClosed();\n        Properties p = new Properties();\n        loggerExternal.exiting(loggingClassName, \"getClientInfo\", p);\n        return p;\n    }\n\n    @Override\n    public String getClientInfo(String name) throws SQLException {\n        loggerExternal.entering(loggingClassName, \"getClientInfo\", name);\n        checkClosed();\n        loggerExternal.exiting(loggingClassName, \"getClientInfo\", null);\n        return null;\n    }\n\n    @Override\n    public void setClientInfo(Properties properties) throws SQLClientInfoException {\n        loggerExternal.entering(loggingClassName, \"setClientInfo\", properties);\n        // This function is only marked as throwing only SQLClientInfoException so the conversion is necessary\n        try {\n            checkClosed();\n        } catch (SQLServerException ex) {\n            SQLClientInfoException info = new SQLClientInfoException();\n            info.initCause(ex);\n            throw info;\n        }\n\n        if (!properties.isEmpty()) {\n            Enumeration<?> e = properties.keys();\n            while (e.hasMoreElements()) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidProperty\"));\n                Object[] msgArgs = {e.nextElement()};\n                addWarning(form.format(msgArgs));\n            }\n        }\n        loggerExternal.exiting(loggingClassName, \"setClientInfo\");\n    }\n\n    @Override\n    public void setClientInfo(String name, String value) throws SQLClientInfoException {\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER)) {\n            loggerExternal.entering(loggingClassName, \"setClientInfo\", new Object[] {name, value});\n        }\n        // This function is only marked as throwing only SQLClientInfoException so the conversion is necessary\n        try {\n            checkClosed();\n        } catch (SQLServerException ex) {\n            SQLClientInfoException info = new SQLClientInfoException();\n            info.initCause(ex);\n            throw info;\n        }\n        MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidProperty\"));\n        Object[] msgArgs = {name};\n        addWarning(form.format(msgArgs));\n        loggerExternal.exiting(loggingClassName, \"setClientInfo\");\n    }\n\n    /**\n     * Determine whether the connection is still valid.\n     *\n     * The driver shall submit a query on the connection or use some other mechanism that positively verifies the\n     * connection is still valid when this method is called.\n     *\n     * The query submitted by the driver to validate the connection shall be executed in the context of the current\n     * transaction.\n     *\n     * @param timeout\n     *        The time in seconds to wait for the database operation used to validate the connection to complete. If the\n     *        timeout period expires before the operation completes, this method returns false. A value of 0 indicates a\n     *        timeout is not applied to the database operation. Note that if the value is 0, the call to isValid may\n     *        block indefinitely if the connection is not valid...\n     *\n     * @return true if the connection has not been closed and is still valid.\n     *\n     * @throws SQLException\n     *         if the value supplied for the timeout is less than 0.\n     */\n    @Override\n    public boolean isValid(int timeout) throws SQLException {\n        loggerExternal.entering(loggingClassName, \"isValid\", timeout);\n\n        // Throw an exception if the timeout is invalid\n        if (timeout < 0) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidQueryTimeOutValue\"));\n            Object[] msgArgs = {timeout};\n            SQLServerException.makeFromDriverError(this, this, form.format(msgArgs), null, true);\n        }\n\n        // Return false if the connection is closed\n        if (isSessionUnAvailable())\n            return false;\n\n        boolean isValid = true;\n        try (SQLServerStatement stmt = new SQLServerStatement(this, ResultSet.TYPE_FORWARD_ONLY,\n                ResultSet.CONCUR_READ_ONLY, SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING)) {\n\n            // If asked, limit the time to wait for the query to complete.\n            if (0 != timeout)\n                stmt.setQueryTimeout(timeout);\n\n            /*\n             * Try to execute the query. If this succeeds, then the connection is valid. If it fails (throws an\n             * exception), then the connection is not valid. If a timeout was provided, execution throws an\n             * \"query timed out\" exception if the query fails to execute in that time.\n             */\n            stmt.executeQueryInternal(\"SELECT 1\");\n        } catch (SQLException e) {\n            isValid = false;\n            /*\n             * Do not propagate SQLExceptions from query execution or statement closure. The connection is considered to\n             * be invalid if the statement fails to close, even though query execution succeeded.\n             */\n            connectionlogger.fine(toString() + \" Exception checking connection validity: \" + e.getMessage());\n        }\n\n        loggerExternal.exiting(loggingClassName, \"isValid\", isValid);\n        return isValid;\n    }\n\n    @Override\n    public boolean isWrapperFor(Class<?> iface) throws SQLException {\n        loggerExternal.entering(loggingClassName, \"isWrapperFor\", iface);\n        boolean f = iface.isInstance(this);\n        loggerExternal.exiting(loggingClassName, \"isWrapperFor\", f);\n        return f;\n    }\n\n    @Override\n    public <T> T unwrap(Class<T> iface) throws SQLException {\n        loggerExternal.entering(loggingClassName, \"unwrap\", iface);\n        T t;\n        try {\n            t = iface.cast(this);\n        } catch (ClassCastException e) {\n            throw new SQLServerException(e.getMessage(), e);\n        }\n        loggerExternal.exiting(loggingClassName, \"unwrap\", t);\n        return t;\n    }\n\n    /** request started flag */\n    private boolean requestStarted = false;\n\n    /** original database autocommit mode */\n    private boolean originalDatabaseAutoCommitMode;\n\n    /** original transaction isolation level */\n    private int originalTransactionIsolationLevel;\n\n    /** original network timeout */\n    private int originalNetworkTimeout;\n\n    /** original holdability */\n    private int originalHoldability;\n\n    /** original sendTimeAsDateTime flag */\n    private boolean originalSendTimeAsDatetime;\n\n    /** original datetimeParameterType flag */\n    private DatetimeType originalDatetimeParameterType;\n\n    /** original statement pooling cache size */\n    private int originalStatementPoolingCacheSize;\n\n    /** original disableStatementPooling */\n    private boolean originalDisableStatementPooling;\n\n    /** original server prepared statement discard threshold */\n    private int originalServerPreparedStatementDiscardThreshold;\n\n    /** original enable prepareOnFirstPreparedStatementCall */\n    private Boolean originalEnablePrepareOnFirstPreparedStatementCall;\n\n    /** original sCatalog */\n    private String originalSCatalog;\n\n    /** original useBulkCopyForBatchInsert flag */\n    private boolean originalUseBulkCopyForBatchInsert;\n\n    /** original SqlWarnings */\n    private volatile SQLWarning originalSqlWarnings;\n\n    /** open statements */\n    private List<ISQLServerStatement> openStatements;\n\n    /** original usesFmtOnly flag */\n    private boolean originalUseFmtOnly;\n\n    /** original delayLoadingLobs */\n    private boolean originalDelayLoadingLobs;\n\n    /** original ignoreOffsetOnDateTimeOffsetConversion */\n    private boolean originalIgnoreOffsetOnDateTimeOffsetConversion;\n\n    /** Always Encrypted version */\n    private int aeVersion = TDS.COLUMNENCRYPTION_NOT_SUPPORTED;\n\n    void beginRequestInternal() throws SQLException {\n        loggerExternal.entering(loggingClassName, \"beginRequest\", this);\n        lock.lock();\n        try {\n            if (!requestStarted) {\n                originalDatabaseAutoCommitMode = databaseAutoCommitMode;\n                originalTransactionIsolationLevel = transactionIsolationLevel;\n                originalNetworkTimeout = getNetworkTimeout();\n                originalHoldability = holdability;\n                originalSendTimeAsDatetime = sendTimeAsDatetime;\n                originalDatetimeParameterType = datetimeParameterType;\n                originalStatementPoolingCacheSize = statementPoolingCacheSize;\n                originalDisableStatementPooling = disableStatementPooling;\n                originalServerPreparedStatementDiscardThreshold = getServerPreparedStatementDiscardThreshold();\n                originalEnablePrepareOnFirstPreparedStatementCall = getEnablePrepareOnFirstPreparedStatementCall();\n                originalSCatalog = sCatalog;\n                originalUseBulkCopyForBatchInsert = getUseBulkCopyForBatchInsert();\n                originalSqlWarnings = sqlWarnings;\n                openStatements = new LinkedList<>();\n                originalUseFmtOnly = useFmtOnly;\n                originalDelayLoadingLobs = delayLoadingLobs;\n                originalIgnoreOffsetOnDateTimeOffsetConversion = ignoreOffsetOnDateTimeOffsetConversion;\n                requestStarted = true;\n            }\n        } finally {\n            lock.unlock();\n        }\n        loggerExternal.exiting(loggingClassName, \"beginRequest\", this);\n    }\n\n    void endRequestInternal() throws SQLException {\n        loggerExternal.entering(loggingClassName, \"endRequest\", this);\n        lock.lock();\n        try {\n            if (requestStarted) {\n                if (!databaseAutoCommitMode) {\n                    rollback();\n                }\n                if (databaseAutoCommitMode != originalDatabaseAutoCommitMode) {\n                    setAutoCommit(originalDatabaseAutoCommitMode);\n                }\n                if (transactionIsolationLevel != originalTransactionIsolationLevel) {\n                    setTransactionIsolation(originalTransactionIsolationLevel);\n                }\n                if (getNetworkTimeout() != originalNetworkTimeout) {\n                    setNetworkTimeout(null, originalNetworkTimeout);\n                }\n                if (holdability != originalHoldability) {\n                    setHoldability(originalHoldability);\n                }\n                if (sendTimeAsDatetime != originalSendTimeAsDatetime) {\n                    setSendTimeAsDatetime(originalSendTimeAsDatetime);\n                }\n                if (datetimeParameterType != originalDatetimeParameterType) {\n                    setDatetimeParameterType(originalDatetimeParameterType.toString());\n                }\n                if (useFmtOnly != originalUseFmtOnly) {\n                    setUseFmtOnly(originalUseFmtOnly);\n                }\n                if (statementPoolingCacheSize != originalStatementPoolingCacheSize) {\n                    setStatementPoolingCacheSize(originalStatementPoolingCacheSize);\n                }\n                if (disableStatementPooling != originalDisableStatementPooling) {\n                    setDisableStatementPooling(originalDisableStatementPooling);\n                }\n                if (getServerPreparedStatementDiscardThreshold() != originalServerPreparedStatementDiscardThreshold) {\n                    setServerPreparedStatementDiscardThreshold(originalServerPreparedStatementDiscardThreshold);\n                }\n                if (getEnablePrepareOnFirstPreparedStatementCall() != originalEnablePrepareOnFirstPreparedStatementCall) {\n                    setEnablePrepareOnFirstPreparedStatementCall(originalEnablePrepareOnFirstPreparedStatementCall);\n                }\n                if (!sCatalog.equals(originalSCatalog)) {\n                    setCatalog(originalSCatalog);\n                }\n                if (getUseBulkCopyForBatchInsert() != originalUseBulkCopyForBatchInsert) {\n                    setUseBulkCopyForBatchInsert(originalUseBulkCopyForBatchInsert);\n                }\n                if (delayLoadingLobs != originalDelayLoadingLobs) {\n                    setDelayLoadingLobs(originalDelayLoadingLobs);\n                }\n                if (ignoreOffsetOnDateTimeOffsetConversion != originalIgnoreOffsetOnDateTimeOffsetConversion) {\n                    setIgnoreOffsetOnDateTimeOffsetConversion(originalIgnoreOffsetOnDateTimeOffsetConversion);\n                }\n                sqlWarnings = originalSqlWarnings;\n                if (null != openStatements) {\n                    while (!openStatements.isEmpty()) {\n                        try (Statement st = openStatements.get(0)) {}\n                    }\n                    openStatements.clear();\n                }\n                requestStarted = false;\n            }\n        } finally {\n            lock.unlock();\n        }\n        loggerExternal.exiting(loggingClassName, \"endRequest\", this);\n    }\n\n    /**\n     * Replaces JDBC syntax parameter markets '?' with SQL Server parameter markers @p1, @p2 etc...\n     * \n     * @param sql\n     *        the user's SQL\n     * @throws SQLServerException\n     * @return the returned syntax\n     */\n    static final char[] OUT = {' ', 'O', 'U', 'T'};\n\n    String replaceParameterMarkers(String sqlSrc, int[] paramPositions, Parameter[] params,\n            boolean isReturnValueSyntax) {\n        final int MAX_PARAM_NAME_LEN = 6;\n        char[] sqlDst = new char[sqlSrc.length() + (params.length * (MAX_PARAM_NAME_LEN + OUT.length))\n                + (params.length * 2)];\n        int dstBegin = 0;\n        int srcBegin = 0;\n        int nParam = 0;\n\n        int paramIndex = 0;\n        while (true) {\n            int srcEnd = ParameterUtils.scanSQLForChar('?', sqlSrc, srcBegin);\n            sqlSrc.getChars(srcBegin, srcEnd, sqlDst, dstBegin);\n            dstBegin += srcEnd - srcBegin;\n\n            if (sqlSrc.length() == srcEnd)\n                break;\n\n            dstBegin += makeParamName(nParam++, sqlDst, dstBegin, true);\n            srcBegin = srcEnd + 1 <= sqlSrc.length() - 1 && sqlSrc.charAt(srcEnd + 1) == ' ' ? srcEnd + 2 : srcEnd + 1;\n\n            if (params[paramIndex++].isOutput() && (!isReturnValueSyntax || paramIndex > 1)) {\n                System.arraycopy(OUT, 0, sqlDst, dstBegin, OUT.length);\n                dstBegin += OUT.length;\n            }\n        }\n\n        return new String(sqlDst, 0, dstBegin);\n    }\n\n    /**\n     * Makes a SQL Server style parameter name.\n     * \n     * @param nParam\n     *        the parameter number\n     * @param name\n     *        the parameter name\n     * @param offset\n     *        the offset\n     * @param isPreparedSQL\n     *        if the param is for build preparedSQL\n     * @return int\n     */\n    static int makeParamName(int nParam, char[] name, int offset, boolean isPreparedSQL) {\n        buildParamInitial(name, offset, isPreparedSQL);\n        if (nParam < 10) {\n            return buildParamLt10(nParam, name, offset, isPreparedSQL);\n        } else {\n            if (nParam < 100) {\n                return buildParamLt100(nParam, name, offset, isPreparedSQL);\n            } else {\n                return buildParamMt100(nParam, name, offset, isPreparedSQL);\n            }\n        }\n    }\n\n    private static void buildParamInitial(char[] name, int offset, boolean isPreparedSQL) {\n        int preparedSQLOffset = 0;\n        if (isPreparedSQL) {\n            name[offset + 0] = ' ';\n            preparedSQLOffset++;\n        }\n        name[offset + preparedSQLOffset + 0] = '@';\n        name[offset + preparedSQLOffset + 1] = 'P';\n    }\n\n    private static int buildParamLt10(int nParam, char[] name, int offset, boolean isPreparedSQL) {\n        int preparedSQLOffset = 0;\n\n        if (isPreparedSQL) {\n            preparedSQLOffset++;\n        }\n\n        name[offset + preparedSQLOffset + 2] = (char) ('0' + nParam);\n\n        if (isPreparedSQL) {\n            name[offset + 4] = ' ';\n            return 5;\n        }\n\n        return 3;\n    }\n\n    private static int buildParamLt100(int nParam, char[] name, int offset, boolean isPreparedSQL) {\n        int nBase = 2;\n        int preparedSQLOffset = 0;\n\n        if (isPreparedSQL) {\n            preparedSQLOffset = 1;\n        }\n\n        while (true) { // make a char[] representation of the param number 2.26\n            if (nParam < nBase * 10) {\n                name[offset + preparedSQLOffset + 2] = (char) ('0' + (nBase - 1));\n                name[offset + preparedSQLOffset + 3] = (char) ('0' + (nParam - ((nBase - 1) * 10)));\n\n                if (isPreparedSQL) {\n                    name[offset + 5] = ' ';\n                    preparedSQLOffset++;\n                }\n\n                return 4 + preparedSQLOffset;\n            }\n            nBase++;\n        }\n    }\n\n    private static int buildParamMt100(int nParam, char[] name, int offset, boolean isPreparedSQL) {\n        int preparedSQLOffset = 0;\n        String sParam = Integer.toString(nParam);\n\n        if (isPreparedSQL) {\n            preparedSQLOffset++;\n            sParam = nParam + \" \";\n        }\n\n        sParam.getChars(0, sParam.length(), name, offset + preparedSQLOffset + 2);\n        return 2 + sParam.length() + preparedSQLOffset;\n    }\n\n    /**\n     * Notify any interested parties (e.g. pooling managers) of a ConnectionEvent activity on the connection. Calling\n     * notifyPooledConnection with null event will place this connection back in the pool. Calling\n     * notifyPooledConnection with a non-null event is used to notify the pooling manager that the connection is bad and\n     * should be removed from the pool.\n     */\n    void notifyPooledConnection(SQLServerException e) {\n        lock.lock();\n        try {\n            if (null != pooledConnectionParent) {\n                pooledConnectionParent.notifyEvent(e);\n            }\n        } finally {\n            lock.unlock();\n        }\n    }\n\n    // Detaches this connection from connection pool.\n    void detachFromPool() {\n        lock.lock();\n        try {\n            pooledConnectionParent = null;\n        } finally {\n            lock.unlock();\n        }\n    }\n\n    /**\n     * Determines the listening port of a named SQL Server instance.\n     * \n     * @param server\n     *        the server name\n     * @param instanceName\n     *        the instance\n     * @throws SQLServerException\n     * @return the instance's port\n     */\n    private static final int BROWSER_PORT = 1434;\n\n    String getInstancePort(String server, String instanceName) throws SQLServerException {\n        String browserResult = null;\n        DatagramSocket datagramSocket = null;\n        String lastErrorMessage = null;\n\n        try {\n            lastErrorMessage = \"Failed to determine instance for the : \" + server + \" instance:\" + instanceName;\n\n            // First we create a datagram socket\n            try {\n                datagramSocket = new DatagramSocket();\n                datagramSocket.setSoTimeout(1000);\n            } catch (SocketException socketException) {\n                // Errors creating a local socket\n                // Log the error and bail.\n                lastErrorMessage = \"Unable to create local datagram socket\";\n                throw socketException;\n            }\n\n            // Second, we need to get the IP address of the server to which we'll send the UDP request.\n            // This may require a DNS lookup, which may fail due to transient conditions, so retry after logging the\n            // first time.\n\n            // send UDP packet\n            assert null != datagramSocket;\n            try {\n                if (multiSubnetFailover) {\n                    // If instance name is specified along with multiSubnetFailover, we get all IPs resolved by server\n                    // name\n                    InetAddress[] inetAddrs = InetAddress.getAllByName(server);\n                    assert null != inetAddrs;\n                    for (InetAddress inetAddr : inetAddrs) {\n                        // Send the UDP request\n                        try {\n                            byte[] sendBuffer = (\" \" + instanceName).getBytes();\n                            sendBuffer[0] = 4;\n                            DatagramPacket udpRequest = new DatagramPacket(sendBuffer, sendBuffer.length, inetAddr,\n                                    BROWSER_PORT);\n                            datagramSocket.send(udpRequest);\n                        } catch (IOException ioException) {\n                            lastErrorMessage = \"Error sending SQL Server Browser Service UDP request to address: \"\n                                    + inetAddr + \", port: \" + BROWSER_PORT;\n                            throw ioException;\n                        }\n                    }\n                } else {\n                    // If instance name is not specified along with multiSubnetFailover, we resolve only the first IP\n                    // for server name\n                    InetAddress inetAddr = InetAddress.getByName(server);\n\n                    assert null != inetAddr;\n                    // Send the UDP request\n                    try {\n                        byte[] sendBuffer = (\" \" + instanceName).getBytes();\n                        sendBuffer[0] = 4;\n                        DatagramPacket udpRequest = new DatagramPacket(sendBuffer, sendBuffer.length, inetAddr,\n                                BROWSER_PORT);\n                        datagramSocket.send(udpRequest);\n                    } catch (IOException ioException) {\n                        lastErrorMessage = \"Error sending SQL Server Browser Service UDP request to address: \"\n                                + inetAddr + \", port: \" + BROWSER_PORT;\n                        throw ioException;\n                    }\n                }\n            } catch (UnknownHostException unknownHostException) {\n...\n", "needle_token_start": 5602, "needle_token_end": 5724, "code_context_ntokens": 16169, "output": ["```java\n/**\n * Checks that the holdability argument is one of the values allowed by the JDBC spec and by this driver.\n */\nprivate void checkValidHoldability(int holdability) throws SQLServerException {\n    if (holdability != ResultSet.HOLD_CURSORS_OVER_COMMIT && holdability != ResultSet.CLOSE_CURSORS_AT_COMMIT) {\n        MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidHoldability\"));\n        SQLServerException.makeFromDriverError(this, this, form.format(new Object[] {holdability}), null, true);\n    }\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "notifyEvent", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPooledConnection.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to inform registered listeners about specific activities or errors related to a database connection. It either signals that a connection has been closed or that an error has occurred, prompting appropriate actions like connection removal or recycling.\n2. **Input**: An exception object that indicates an error (if any); if this object is null, it implies the connection is simply being closed without errors.\n3. **Output**: There is no direct output returned by this function; however, it triggers side effects through event notifications to listeners.\n4. **Procedure**: \n   - Logs the error if logging is enabled and an exception is provided.\n   - If an exception is present, it locks a mutex, closes the last proxy connection (if it exists), and releases the mutex.\n   - Regardless of whether an exception was provided, it locks another mutex and iterates through all registered event listeners to dispatch either a connection closed or an error occurred event, then unlocks the mutex.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.PrintWriter;\nimport java.sql.Connection;\nimport java.sql.SQLException;\nimport java.util.Enumeration;\nimport java.util.Properties;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\nimport javax.naming.Reference;\nimport javax.naming.StringRefAddr;\n\nimport org.ietf.jgss.GSSCredential;\n\n\n/**\n * Contains a list of properties specific for the {@link SQLServerConnection} class.\n */\npublic class SQLServerDataSource\n        implements ISQLServerDataSource, javax.sql.DataSource, java.io.Serializable, javax.naming.Referenceable {\n    // dsLogger is logger used for all SQLServerDataSource instances.\n    static final java.util.logging.Logger dsLogger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.SQLServerDataSource\");\n    static final java.util.logging.Logger loggerExternal = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.DataSource\");\n    static final private java.util.logging.Logger parentLogger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc\");\n\n    static final String TRUSTSTORE_PASSWORD_STRIPPED = \"trustStorePasswordStripped\";\n\n    /** logging class name */\n    final private String loggingClassName;\n\n    /**\n     * trustStorePasswordStripped flag\n     */\n    private boolean trustStorePasswordStripped = false;\n\n    /**\n     * Always refresh SerialVersionUID when prompted\n     */\n    private static final long serialVersionUID = 654861379544314296L;\n\n    /**\n     * Properties passed to SQLServerConnection class\n     */\n    private Properties connectionProps;\n\n    /**\n     * URL for datasource\n     */\n    private String dataSourceURL;\n\n    /**\n     * Description for datasource.\n     */\n    private String dataSourceDescription;\n\n    /**\n     * Unique id generator for each DataSource instance (used for logging).\n     */\n    static private final AtomicInteger baseDataSourceID = new AtomicInteger(0);\n\n    /**\n     * trace ID\n     */\n    final private String traceID;\n\n    /**\n     * Constructs a SQLServerDataSource.\n     */\n    public SQLServerDataSource() {\n        connectionProps = new Properties();\n        traceID = getClass().getSimpleName() + ':' + nextDataSourceID();\n        loggingClassName = \"com.microsoft.sqlserver.jdbc.\" + traceID;\n    }\n\n    String getClassNameLogging() {\n        return loggingClassName;\n    }\n\n    @Override\n    public String toString() {\n        return traceID;\n    }\n\n    // DataSource interface public methods\n\n    @Override\n    public Connection getConnection() throws SQLServerException {\n        loggerExternal.entering(getClassNameLogging(), \"getConnection\");\n        Connection con = getConnectionInternal(null, null, null);\n        loggerExternal.exiting(getClassNameLogging(), \"getConnection\", con);\n        return con;\n    }\n\n    @Override\n    public Connection getConnection(String username, String password) throws SQLServerException {\n        if (loggerExternal.isLoggable(Level.FINER))\n            loggerExternal.entering(getClassNameLogging(), \"getConnection\",\n                    new Object[] {username, \"Password not traced\"});\n        Connection con = getConnectionInternal(username, password, null);\n        loggerExternal.exiting(getClassNameLogging(), \"getConnection\", con);\n        return con;\n    }\n\n    /**\n     * Sets the maximum time in seconds that this data source will wait while attempting to connect to a database. Note\n     * default value is 0.\n     */\n    @Override\n    public void setLoginTimeout(int loginTimeout) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.LOGIN_TIMEOUT.toString(), loginTimeout);\n    }\n\n    @Override\n    public int getLoginTimeout() {\n        int defaultTimeOut = SQLServerDriverIntProperty.LOGIN_TIMEOUT.getDefaultValue();\n        final int logintimeout = getIntProperty(connectionProps, SQLServerDriverIntProperty.LOGIN_TIMEOUT.toString(),\n                defaultTimeOut);\n        // even if the user explicitly sets the timeout to zero, convert to 15\n        return (logintimeout == 0) ? defaultTimeOut : logintimeout;\n    }\n\n    /**\n     * Sets the log writer for this DataSource. Currently we just hold onto this logWriter and pass it back to callers,\n     * nothing else.\n     */\n    private transient PrintWriter logWriter;\n\n    @Override\n    public void setLogWriter(PrintWriter out) {\n        loggerExternal.entering(getClassNameLogging(), \"setLogWriter\", out);\n        logWriter = out;\n        loggerExternal.exiting(getClassNameLogging(), \"setLogWriter\");\n    }\n\n    /**\n     * Returns the log writer for this DataSource.\n     */\n    @Override\n    public PrintWriter getLogWriter() {\n        loggerExternal.entering(getClassNameLogging(), \"getLogWriter\");\n        loggerExternal.exiting(getClassNameLogging(), \"getLogWriter\", logWriter);\n        return logWriter;\n    }\n\n    @Override\n    public Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException {\n        return parentLogger;\n    }\n\n    // Core Connection property setters/getters.\n\n    /**\n     * Sets the specific application in various SQL Server profiling and logging tools.\n     */\n    @Override\n    public void setApplicationName(String applicationName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_NAME.toString(), applicationName);\n    }\n\n    @Override\n    public String getApplicationName() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_NAME.toString(),\n                SQLServerDriverStringProperty.APPLICATION_NAME.getDefaultValue());\n    }\n\n    /**\n     * Sets the the database to connect to.\n     *\n     * @param databaseName\n     *        if not set, returns the default value of null.\n     */\n    @Override\n    public void setDatabaseName(String databaseName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.DATABASE_NAME.toString(), databaseName);\n    }\n\n    @Override\n    public String getDatabaseName() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.DATABASE_NAME.toString(), null);\n    }\n\n    /**\n     * Sets the the SQL Server instance name to connect to.\n     *\n     * @param instanceName\n     *        if not set, returns the default value of null.\n     */\n    @Override\n    public void setInstanceName(String instanceName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.INSTANCE_NAME.toString(), instanceName);\n    }\n\n    @Override\n    public String getInstanceName() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.INSTANCE_NAME.toString(), null);\n    }\n\n    @Override\n    public void setIntegratedSecurity(boolean enable) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.INTEGRATED_SECURITY.toString(), enable);\n    }\n\n    @Override\n    public void setAuthenticationScheme(String authenticationScheme) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.AUTHENTICATION_SCHEME.toString(),\n                authenticationScheme);\n    }\n\n    @Override\n    public void setAuthentication(String authentication) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.AUTHENTICATION.toString(), authentication);\n    }\n\n    @Override\n    public String getAuthentication() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.AUTHENTICATION.toString(),\n                SQLServerDriverStringProperty.AUTHENTICATION.getDefaultValue());\n    }\n\n    @Override\n    public void setGSSCredentials(GSSCredential userCredential) {\n        setObjectProperty(connectionProps, SQLServerDriverObjectProperty.GSS_CREDENTIAL.toString(), userCredential);\n    }\n\n    @Override\n    public GSSCredential getGSSCredentials() {\n        return (GSSCredential) getObjectProperty(connectionProps,\n                SQLServerDriverObjectProperty.GSS_CREDENTIAL.toString(),\n                SQLServerDriverObjectProperty.GSS_CREDENTIAL.getDefaultValue());\n    }\n\n    @Override\n    public void setUseDefaultGSSCredential(boolean enable) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_DEFAULT_GSS_CREDENTIAL.toString(),\n                enable);\n    }\n\n    @Override\n    public boolean getUseDefaultGSSCredential() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_DEFAULT_GSS_CREDENTIAL.toString(),\n                SQLServerDriverBooleanProperty.USE_DEFAULT_GSS_CREDENTIAL.getDefaultValue());\n    }\n\n    @Override\n    public void setUseFlexibleCallableStatements(boolean enable) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_FLEXIBLE_CALLABLE_STATEMENTS.toString(),\n                enable);\n    }\n\n    @Override\n    public boolean getUseFlexibleCallableStatements() {\n        return getBooleanProperty(connectionProps,\n                SQLServerDriverBooleanProperty.USE_FLEXIBLE_CALLABLE_STATEMENTS.toString(),\n                SQLServerDriverBooleanProperty.USE_FLEXIBLE_CALLABLE_STATEMENTS.getDefaultValue());\n    }\n\n    @Override\n    public void setAccessToken(String accessToken) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.ACCESS_TOKEN.toString(), accessToken);\n    }\n\n    @Override\n    public String getAccessToken() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.ACCESS_TOKEN.toString(), null);\n    }\n\n    /**\n     * Sets the Column Encryption setting. If lastUpdateCount is set to true, the driver will return only the last\n     * update count from all the update counts returned by a batch. The default of false will return all update counts.\n     * If lastUpdateCount is not set, getLastUpdateCount returns the default value of false.\n     */\n    @Override\n    public void setColumnEncryptionSetting(String columnEncryptionSetting) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.COLUMN_ENCRYPTION.toString(),\n                columnEncryptionSetting);\n    }\n\n    @Override\n    public String getColumnEncryptionSetting() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.COLUMN_ENCRYPTION.toString(),\n                SQLServerDriverStringProperty.COLUMN_ENCRYPTION.getDefaultValue());\n    }\n\n    @Override\n    public void setKeyStoreAuthentication(String keyStoreAuthentication) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_AUTHENTICATION.toString(),\n                keyStoreAuthentication);\n    }\n\n    @Override\n    public String getKeyStoreAuthentication() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_AUTHENTICATION.toString(),\n                SQLServerDriverStringProperty.KEY_STORE_AUTHENTICATION.getDefaultValue());\n    }\n\n    @Override\n    public void setKeyStoreSecret(String keyStoreSecret) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_SECRET.toString(), keyStoreSecret);\n    }\n\n    @Override\n    public void setKeyStoreLocation(String keyStoreLocation) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_LOCATION.toString(),\n                keyStoreLocation);\n    }\n\n    @Override\n    public String getKeyStoreLocation() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_LOCATION.toString(),\n                SQLServerDriverStringProperty.KEY_STORE_LOCATION.getDefaultValue());\n    }\n\n    @Override\n    public void setLastUpdateCount(boolean lastUpdateCount) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.LAST_UPDATE_COUNT.toString(),\n                lastUpdateCount);\n    }\n\n    @Override\n    public boolean getLastUpdateCount() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.LAST_UPDATE_COUNT.toString(),\n                SQLServerDriverBooleanProperty.LAST_UPDATE_COUNT.getDefaultValue());\n    }\n\n    @Override\n    public void setEncrypt(String encryptOption) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.ENCRYPT.toString(), encryptOption);\n    }\n\n    /**\n     * @deprecated\n     */\n    @Override\n    @Deprecated(since = \"10.1.0\", forRemoval = true)\n    public void setEncrypt(boolean encryptOption) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.ENCRYPT.toString(),\n                Boolean.toString(encryptOption));\n    }\n\n    @Override\n    public String getEncrypt() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.ENCRYPT.toString(),\n                SQLServerDriverStringProperty.ENCRYPT.getDefaultValue());\n    }\n\n    @Override\n    public void setServerCertificate(String cert) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_CERTIFICATE.toString(), cert);\n    }\n\n    @Override\n    public String getServerCertificate() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_CERTIFICATE.toString(), null);\n    }\n\n    @Override\n    public void setTransparentNetworkIPResolution(boolean tnir) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.TRANSPARENT_NETWORK_IP_RESOLUTION.toString(),\n                tnir);\n    }\n\n    @Override\n    public boolean getTransparentNetworkIPResolution() {\n        return getBooleanProperty(connectionProps,\n                SQLServerDriverBooleanProperty.TRANSPARENT_NETWORK_IP_RESOLUTION.toString(),\n                SQLServerDriverBooleanProperty.TRANSPARENT_NETWORK_IP_RESOLUTION.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustServerCertificate(boolean e) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.TRUST_SERVER_CERTIFICATE.toString(), e);\n    }\n\n    @Override\n    public boolean getTrustServerCertificate() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.TRUST_SERVER_CERTIFICATE.toString(),\n                SQLServerDriverBooleanProperty.TRUST_SERVER_CERTIFICATE.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustStoreType(String trustStoreType) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_TYPE.toString(), trustStoreType);\n    }\n\n    @Override\n    public String getTrustStoreType() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_TYPE.toString(),\n                SQLServerDriverStringProperty.TRUST_STORE_TYPE.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustStore(String trustStore) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE.toString(), trustStore);\n    }\n\n    @Override\n    public String getTrustStore() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE.toString(), null);\n    }\n\n    @Override\n    public void setTrustStorePassword(String trustStorePassword) {\n        // if a non value property is set\n        if (trustStorePassword != null)\n            trustStorePasswordStripped = false;\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_PASSWORD.toString(),\n                trustStorePassword);\n    }\n\n    String getTrustStorePassword() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_PASSWORD.toString(), null);\n    }\n\n    @Override\n    public void setHostNameInCertificate(String hostName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.HOSTNAME_IN_CERTIFICATE.toString(), hostName);\n    }\n\n    @Override\n    public String getHostNameInCertificate() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.HOSTNAME_IN_CERTIFICATE.toString(),\n                null);\n    }\n\n    /**\n     * Sets the lock timeout value.\n     *\n     * @param lockTimeout\n     *        the number of milliseconds to wait before the database reports a lock timeout. The default value of -1\n     *        means wait forever. If specified, this value will be the default for all statements on the connection.\n     *        Note a value of 0 means no wait. If lockTimeout is not set, getLockTimeout returns the default of -1.\n     */\n    @Override\n    public void setLockTimeout(int lockTimeout) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.LOCK_TIMEOUT.toString(), lockTimeout);\n    }\n\n    @Override\n    public int getLockTimeout() {\n        return getIntProperty(connectionProps, SQLServerDriverIntProperty.LOCK_TIMEOUT.toString(),\n                SQLServerDriverIntProperty.LOCK_TIMEOUT.getDefaultValue());\n    }\n\n    /**\n     * Sets the password that will be used when connecting to SQL Server.\n     *\n     * @param password\n     *        Note getPassword is deliberately declared non-public for security reasons. If the password is not set,\n     *        getPassword returns the default value of null.\n     */\n    @Override\n    public void setPassword(String password) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.PASSWORD.toString(), password);\n    }\n\n    String getPassword() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.PASSWORD.toString(), null);\n    }\n\n    /**\n     * Sets the TCP-IP port number used when opening a socket connection to SQL Server.\n     *\n     * @param portNumber\n     *        if not set, getPortNumber returns the default of 1433. Note as mentioned above, setPortNumber does not do\n     *        any range checking on the port value passed in,\\ invalid port numbers like 99999 can be passed in without\n     *        triggering any error.\n     */\n    @Override\n    public void setPortNumber(int portNumber) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.PORT_NUMBER.toString(), portNumber);\n    }\n\n    @Override\n    public int getPortNumber() {\n        return getIntProperty(connectionProps, SQLServerDriverIntProperty.PORT_NUMBER.toString(),\n                SQLServerDriverIntProperty.PORT_NUMBER.getDefaultValue());\n    }\n\n    /**\n     * Sets the default cursor type used for the result set.\n     *\n     * @param selectMethod\n     *        This(non-Javadoc) @see com.microsoft.sqlserver.jdbc.ISQLServerDataSource#setSelectMethod(java.lang.String)\n     *        property is useful when you are dealing with large result sets and do not want to store the whole result\n     *        set in memory on the client side. By setting the property to \"cursor\" you will be able to create a server\n     *        side cursor that can fetch smaller chunks of data at a time. If selectMethod is not set, getSelectMethod\n     *        returns the default value of \"direct\".\n     */\n    @Override\n    public void setSelectMethod(String selectMethod) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SELECT_METHOD.toString(), selectMethod);\n    }\n\n    @Override\n    public String getSelectMethod() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SELECT_METHOD.toString(),\n                SQLServerDriverStringProperty.SELECT_METHOD.getDefaultValue());\n    }\n\n    @Override\n    public void setResponseBuffering(String bufferingMode) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.RESPONSE_BUFFERING.toString(), bufferingMode);\n    }\n\n    @Override\n    public String getResponseBuffering() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.RESPONSE_BUFFERING.toString(),\n                SQLServerDriverStringProperty.RESPONSE_BUFFERING.getDefaultValue());\n    }\n\n    @Override\n    public void setApplicationIntent(String applicationIntent) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_INTENT.toString(),\n                applicationIntent);\n    }\n\n    @Override\n    public String getApplicationIntent() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_INTENT.toString(),\n                SQLServerDriverStringProperty.APPLICATION_INTENT.getDefaultValue());\n    }\n\n    @Override\n    public void setReplication(boolean replication) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.REPLICATION.toString(), replication);\n    }\n\n    @Override\n    public boolean getReplication() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.REPLICATION.toString(),\n                SQLServerDriverBooleanProperty.REPLICATION.getDefaultValue());\n    }\n\n    @Override\n    public void setSendTimeAsDatetime(boolean sendTimeAsDatetime) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.SEND_TIME_AS_DATETIME.toString(),\n                sendTimeAsDatetime);\n    }\n\n    @Override\n    public boolean getSendTimeAsDatetime() {\n...\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPooledConnection.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport javax.sql.ConnectionEvent;\nimport javax.sql.ConnectionEventListener;\nimport javax.sql.PooledConnection;\nimport javax.sql.StatementEventListener;\nimport java.io.Serializable;\nimport java.sql.Connection;\nimport java.sql.SQLException;\nimport java.util.Vector;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.logging.Level;\n\n\n/**\n * Represents a physical database connection in a connection pool. If provides methods for the connection pool manager\n * to manage the connection pool. Applications typically do not instantiate these connections directly.\n */\n\npublic class SQLServerPooledConnection implements PooledConnection, Serializable {\n    /**\n     * Always update serialVersionUID when prompted.\n     */\n    private static final long serialVersionUID = 3492921646187451164L;\n\n    /** listeners */\n    private final Vector<ConnectionEventListener> listeners;\n\n    /** factory datasource */\n    private SQLServerDataSource factoryDataSource;\n\n    /** physical connection */\n    private SQLServerConnection physicalConnection;\n\n    /** last proxy connection */\n    private SQLServerConnectionPoolProxy lastProxyConnection;\n\n    /** factory password */\n    private String factoryUser, factoryPassword;\n\n    /** logger */\n    private transient java.util.logging.Logger pcLogger;\n\n    /** trace ID */\n    private final String traceID;\n\n    // Unique id generator for each PooledConnection instance (used for logging).\n    static private final AtomicInteger basePooledConnectionID = new AtomicInteger(0);\n\n    /** reentrant lock for connection */\n    private final transient Lock lock = new ReentrantLock();\n\n    /** reentrant lock for ConnectionEventListener */\n    private final transient Lock listenersLock = new ReentrantLock();\n\n    SQLServerPooledConnection(SQLServerDataSource ds, String user, String password) throws SQLException {\n        listeners = new Vector<>();\n        traceID = getClass().getSimpleName() + ':' + nextPooledConnectionID();\n        // Piggyback SQLServerDataSource logger for now.\n        pcLogger = SQLServerDataSource.dsLogger;\n\n        // Create the physical connection.\n        factoryDataSource = ds;\n        factoryUser = user;\n        factoryPassword = password;\n\n        if (pcLogger.isLoggable(Level.FINER))\n            pcLogger.finer(toString() + \" Start create new connection for pool.\");\n\n        physicalConnection = createNewConnection();\n        if (pcLogger.isLoggable(Level.FINE))\n            pcLogger.fine(toString() + \" created by (\" + ds.toString() + \")\" + \" Physical connection \" + safeCID()\n                    + \", End create new connection for pool\");\n    }\n\n    /**\n     * Provides a helper function to provide an ID string suitable for tracing.\n     *\n     * @return traceID String\n     */\n    @Override\n    public String toString() {\n        return traceID;\n    }\n\n    /**\n     * Helper function to create a new connection for the pool.\n     *\n     * @return SQLServerConnection instance\n     * @throws SQLException\n     */\n    private SQLServerConnection createNewConnection() throws SQLException {\n        return factoryDataSource.getConnectionInternal(factoryUser, factoryPassword, this);\n    }\n\n    /**\n     * Returns an object handle for the physical connection that this PooledConnection object represents.\n     *\n     * @throws SQLException\n     *         when an error occurs\n     * @return a Connection object that is a handle to this PooledConnection object\n     */\n    @Override\n    public Connection getConnection() throws SQLException {\n        if (pcLogger.isLoggable(Level.FINER))\n            pcLogger.finer(toString() + \" user:(default).\");\n        lock.lock();\n        try {\n            // If physical connection is closed, throw exception per spec, this PooledConnection is dead.\n            if (physicalConnection == null) {\n                SQLServerException.makeFromDriverError(null, this,\n                        SQLServerException.getErrString(\"R_physicalConnectionIsClosed\"), \"\", true);\n            }\n\n            /*\n             * Check with security manager to insure caller has rights to connect. This will throw a SecurityException\n             * if the caller does not have proper rights.\n             */\n            physicalConnection.doSecurityCheck();\n            if (pcLogger.isLoggable(Level.FINE))\n                pcLogger.fine(toString() + \" Physical connection, \" + safeCID());\n\n            if (physicalConnection.needsReconnect()) {\n                physicalConnection.close();\n                physicalConnection = createNewConnection();\n            }\n\n            /*\n             * The last proxy connection handle returned will be invalidated (moved to closed state) when getConnection\n             * is called.\n             */\n            if (null != lastProxyConnection) {\n                // if there was a last proxy connection send reset\n                physicalConnection.resetPooledConnection();\n\n                if (!lastProxyConnection.isClosed()) {\n                    if (pcLogger.isLoggable(Level.FINE)) {\n                        pcLogger.fine(toString() + \"proxy \" + lastProxyConnection.toString()\n                                + \" is not closed before getting the connection.\");\n                    }\n                    /*\n                     * use internal close so there wont be an event due to us closing the connection, if not closed\n                     * already.\n                     */\n                    lastProxyConnection.internalClose();\n                }\n            }\n\n            lastProxyConnection = new SQLServerConnectionPoolProxy(physicalConnection);\n            if (pcLogger.isLoggable(Level.FINE) && !lastProxyConnection.isClosed())\n                pcLogger.fine(toString() + \" proxy \" + lastProxyConnection.toString() + \" is returned.\");\n\n            return lastProxyConnection;\n        } finally {\n            lock.unlock();\n        }\n    }\n\n    /**\n     * Notifies any interested parties (e.g. pooling managers) of a ConnectionEvent activity on the connection. Calling\n     * notifyEvent with null event will place the connection back in the pool. Calling notifyEvent with a non-null event\n     * is used to notify the pooling manager that the connection is bad and should be removed from the pool.\n     */\n    \nvoid notifyEvent(SQLServerException e) {\n        if (pcLogger.isLoggable(Level.FINER))\n            pcLogger.finer(toString() + \" Exception:\" + e + safeCID());\n\n        // close the proxy on fatal error event. Note exception is null then the event comes from the proxy close.\n        if (null != e) {\n            lock.lock();\n            try {\n                if (null != lastProxyConnection) {\n                    lastProxyConnection.internalClose();\n                    lastProxyConnection = null;\n                }\n            } finally {\n                lock.unlock();\n            }\n        }\n\n        // A connection handle issued from this pooled connection is closing or an error occurred in the connection\n        listenersLock.lock();\n        try {\n            for (int i = 0; i < listeners.size(); i++) {\n                ConnectionEventListener listener = listeners.elementAt(i);\n\n                if (listener == null)\n                    continue;\n\n                ConnectionEvent ev = new ConnectionEvent(this, e);\n                if (null == e) {\n                    if (pcLogger.isLoggable(Level.FINER))\n                        pcLogger.finer(toString() + \" notifyEvent:connectionClosed \" + safeCID());\n                    listener.connectionClosed(ev);\n                } else {\n                    if (pcLogger.isLoggable(Level.FINER))\n                        pcLogger.finer(toString() + \" notifyEvent:connectionErrorOccurred \" + safeCID());\n                    listener.connectionErrorOccurred(ev);\n                }\n            }\n        } finally {\n            listenersLock.unlock();\n        }\n    }\n\n    @Override\n    public void addConnectionEventListener(ConnectionEventListener listener) {\n        if (pcLogger.isLoggable(Level.FINER))\n            pcLogger.finer(toString() + safeCID());\n        listenersLock.lock();\n        try {\n            listeners.add(listener);\n        } finally {\n            listenersLock.unlock();\n        }\n    }\n\n    @Override\n    public void close() throws SQLException {\n        if (pcLogger.isLoggable(Level.FINER))\n            pcLogger.finer(toString() + \" Closing physical connection, \" + safeCID());\n        lock.lock();\n        try {\n            // First close the last proxy\n            if (null != lastProxyConnection)\n                // use internal close so there wont be an event due to us closing the connection, if not closed already.\n                lastProxyConnection.internalClose();\n            if (null != physicalConnection) {\n                physicalConnection.detachFromPool();\n                physicalConnection.close();\n            }\n            physicalConnection = null;\n        } finally {\n            lock.unlock();\n        }\n        listenersLock.lock();\n        try {\n            listeners.clear();\n        } finally {\n            listenersLock.unlock();\n        }\n    }\n\n    @Override\n    public void removeConnectionEventListener(ConnectionEventListener listener) {\n        if (pcLogger.isLoggable(Level.FINER))\n            pcLogger.finer(toString() + safeCID());\n        listenersLock.lock();\n        try {\n            listeners.remove(listener);\n        } finally {\n            listenersLock.unlock();\n        }\n    }\n\n    @Override\n    public void addStatementEventListener(StatementEventListener listener) {\n        // Not implemented\n        throw new UnsupportedOperationException(SQLServerException.getErrString(\"R_notSupported\"));\n    }\n\n    @Override\n    public void removeStatementEventListener(StatementEventListener listener) {\n        // Not implemented\n        throw new UnsupportedOperationException(SQLServerException.getErrString(\"R_notSupported\"));\n    }\n\n    // Returns internal physical connection to caller.\n    SQLServerConnection getPhysicalConnection() {\n        return physicalConnection;\n    }\n\n    // Returns unique id for each PooledConnection instance.\n    private static int nextPooledConnectionID() {\n        return basePooledConnectionID.incrementAndGet();\n    }\n\n    /**\n     * Helper function to return connectionID of the physicalConnection in a safe manner for logging. Returns (null) if\n     * physicalConnection is null, otherwise returns connectionID.\n     **/\n    private String safeCID() {\n        if (null == physicalConnection)\n            return \" ConnectionID:(null)\";\n        return physicalConnection.toString();\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/MaxResultBufferCounter.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.text.MessageFormat;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n\n/**\n * Implementation of ICounter for 'maxResultBuffer' property.\n */\npublic class MaxResultBufferCounter implements ICounter {\n\n    private final Logger logger = Logger.getLogger(\"com.microsoft.sqlserver.jdbc.MaxResultBufferCounter\");\n\n    private long counter = 0;\n    private final long maxResultBuffer;\n\n    /**\n     * Creates a MaxResultBufferCounter with the given max results buffer\n     * \n     * @param maxResultBuffer\n     *        max result buffer\n     */\n    public MaxResultBufferCounter(long maxResultBuffer) {\n        this.maxResultBuffer = maxResultBuffer;\n    }\n\n    public void increaseCounter(long bytes) throws SQLServerException {\n        if (maxResultBuffer > 0) {\n            counter += bytes;\n            checkForMaxResultBufferOverflow(counter);\n        }\n    }\n\n    public void resetCounter() {\n        counter = 0;\n    }\n\n    private void checkForMaxResultBufferOverflow(long number) throws SQLServerException {\n        if (number > maxResultBuffer) {\n            if (logger.isLoggable(Level.SEVERE)) {\n                logger.log(Level.SEVERE, SQLServerException.getErrString(\"R_maxResultBufferPropertyExceeded\"),\n                        new Object[] {number, maxResultBuffer});\n            }\n            throwExceededMaxResultBufferException(counter, maxResultBuffer);\n        }\n    }\n\n    private void throwExceededMaxResultBufferException(Object... arguments) throws SQLServerException {\n        MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_maxResultBufferPropertyExceeded\"));\n        throw new SQLServerException(form.format(arguments), null);\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerAeadAes256CbcHmac256Factory.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport static java.nio.charset.StandardCharsets.UTF_8;\n\nimport java.text.MessageFormat;\nimport java.util.Base64;\nimport java.util.concurrent.ConcurrentHashMap;\n\n\n/**\n * Factory for SQLServerAeadAes256CbcHmac256Algorithm\n */\nclass SQLServerAeadAes256CbcHmac256Factory extends SQLServerEncryptionAlgorithmFactory {\n    // In future we can have more\n    private byte algorithmVersion = 0x1;\n    private ConcurrentHashMap<String, SQLServerAeadAes256CbcHmac256Algorithm> encryptionAlgorithms = new ConcurrentHashMap<>();\n\n    @Override\n    SQLServerEncryptionAlgorithm create(SQLServerSymmetricKey columnEncryptionKey,\n            SQLServerEncryptionType encryptionType, String encryptionAlgorithm) throws SQLServerException {\n\n        assert (columnEncryptionKey != null);\n        if (encryptionType != SQLServerEncryptionType.DETERMINISTIC\n                && encryptionType != SQLServerEncryptionType.RANDOMIZED) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_InvalidEncryptionType\"));\n            Object[] msgArgs = {encryptionType, encryptionAlgorithm,\n                    \"'\" + SQLServerEncryptionType.DETERMINISTIC + \",\" + SQLServerEncryptionType.RANDOMIZED + \"'\"};\n            throw new SQLServerException(this, form.format(msgArgs), null, 0, false);\n\n        }\n\n        StringBuilder factoryKeyBuilder = new StringBuilder();\n        factoryKeyBuilder.append(\n                Base64.getEncoder().encodeToString(new String(columnEncryptionKey.getRootKey(), UTF_8).getBytes()));\n\n        factoryKeyBuilder.append(\":\");\n        factoryKeyBuilder.append(encryptionType);\n        factoryKeyBuilder.append(\":\");\n        factoryKeyBuilder.append(algorithmVersion);\n\n        String factoryKey = factoryKeyBuilder.toString();\n\n        SQLServerAeadAes256CbcHmac256Algorithm aesAlgorithm;\n\n        if (!encryptionAlgorithms.containsKey(factoryKey)) {\n            SQLServerAeadAes256CbcHmac256EncryptionKey encryptedKey = new SQLServerAeadAes256CbcHmac256EncryptionKey(\n                    columnEncryptionKey.getRootKey(),\n                    SQLServerAeadAes256CbcHmac256Algorithm.AEAD_AES_256_CBC_HMAC_SHA256);\n            aesAlgorithm = new SQLServerAeadAes256CbcHmac256Algorithm(encryptedKey, encryptionType, algorithmVersion);\n            encryptionAlgorithms.putIfAbsent(factoryKey, aesAlgorithm);\n        }\n\n        return encryptionAlgorithms.get(factoryKey);\n    }\n\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerSQLXML.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStreamWriter;\nimport java.io.Reader;\nimport java.io.StringReader;\nimport java.sql.SQLException;\nimport java.text.MessageFormat;\nimport java.util.concurrent.atomic.AtomicInteger;\n\nimport javax.xml.XMLConstants;\nimport javax.xml.parsers.DocumentBuilder;\nimport javax.xml.parsers.DocumentBuilderFactory;\nimport javax.xml.parsers.ParserConfigurationException;\nimport javax.xml.parsers.SAXParser;\nimport javax.xml.parsers.SAXParserFactory;\nimport javax.xml.stream.XMLInputFactory;\nimport javax.xml.stream.XMLOutputFactory;\nimport javax.xml.stream.XMLStreamException;\nimport javax.xml.stream.XMLStreamReader;\nimport javax.xml.stream.XMLStreamWriter;\nimport javax.xml.transform.Result;\nimport javax.xml.transform.Source;\nimport javax.xml.transform.TransformerConfigurationException;\nimport javax.xml.transform.TransformerFactory;\nimport javax.xml.transform.dom.DOMResult;\nimport javax.xml.transform.dom.DOMSource;\nimport javax.xml.transform.sax.SAXResult;\nimport javax.xml.transform.sax.SAXSource;\nimport javax.xml.transform.sax.SAXTransformerFactory;\nimport javax.xml.transform.sax.TransformerHandler;\nimport javax.xml.transform.stax.StAXResult;\nimport javax.xml.transform.stax.StAXSource;\nimport javax.xml.transform.stream.StreamResult;\nimport javax.xml.transform.stream.StreamSource;\n\nimport org.w3c.dom.Document;\nimport org.xml.sax.EntityResolver;\nimport org.xml.sax.InputSource;\nimport org.xml.sax.SAXException;\nimport org.xml.sax.XMLReader;\n\n\n/**\n * Represents an XML object and implements a java.sql.SQLXML.\n */\nfinal class SQLServerSQLXML implements java.sql.SQLXML {\n    // Connection that created this SQLXML only set when created for setting data\n    private final SQLServerConnection con;\n    // Contents null means this is a setter.\n    private final PLPXMLInputStream contents;\n    private final InputStreamGetterArgs getterArgs;\n    private final TypeInfo typeInfo;\n\n    private boolean isUsed = false;\n    private boolean isFreed = false;\n\n    static private final java.util.logging.Logger logger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.SQLServerSQLXML\");\n\n    // Setter values are held in different forms depending on which setter is used only one setter can\n    // be used at one time.\n    private ByteArrayOutputStreamToInputStream outputStreamValue;\n    private Document docValue;\n    private String strValue;\n    // End of setter values\n\n    static private final AtomicInteger baseID = new AtomicInteger(0); // Unique id generator for each instance (used for\n                                                                      // logging).\n    final private String traceID;\n\n    final public String toString() {\n        return traceID;\n    }\n\n    /**\n     * Returns unique id for each instance.\n     * \n     * @return\n     */\n    private static int nextInstanceID() {\n        return baseID.incrementAndGet();\n    }\n\n    // This method is used to get the value the user has set\n    // This is used at the execution time when sending the data to the server\n    // possible optimization transform DOM straight to tds\n    InputStream getValue() throws SQLServerException {\n        checkClosed();\n        // the user should have called one of the setter methods, the setter methods \"use\" the object and write some\n        // value.\n        // Note just calling one of the setters is enough.\n        if (!isUsed)\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_noDataXML\"), null,\n                    true);\n        assert null == contents;\n        ByteArrayInputStream o = null;\n        if (null != outputStreamValue) {\n            o = outputStreamValue.getInputStream();\n            assert null == docValue;\n            assert null == strValue;\n        } else if (null != docValue) {\n            assert null == outputStreamValue;\n            assert null == strValue;\n            ByteArrayOutputStreamToInputStream strm = new ByteArrayOutputStreamToInputStream();\n            // Need to beat a stream out of docValue\n            TransformerFactory factory;\n            try {\n                factory = TransformerFactory.newInstance();\n                factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);\n                factory.newTransformer().transform(new DOMSource(docValue), new StreamResult(strm));\n            } catch (javax.xml.transform.TransformerException e) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_noParserSupport\"));\n                Object[] msgArgs = {e.toString()};\n                SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n            }\n            o = strm.getInputStream();\n        } else {\n            assert null == outputStreamValue;\n            assert null == docValue;\n            assert null != strValue;\n            o = new ByteArrayInputStream(strValue.getBytes(Encoding.UNICODE.charset()));\n        }\n        assert null != o;\n        isFreed = true; // we have consumed the data\n        return o;\n    }\n\n    // To be used in the createSQLXML call\n    SQLServerSQLXML(SQLServerConnection connection) {\n        contents = null;\n        traceID = \" SQLServerSQLXML:\" + nextInstanceID();\n        con = connection;\n\n        if (logger.isLoggable(java.util.logging.Level.FINE))\n            logger.fine(toString() + \" created by (\" + connection.toString() + \")\");\n        getterArgs = null; // make the compiler happy\n        typeInfo = null;\n    }\n\n    SQLServerSQLXML(InputStream stream, InputStreamGetterArgs getterArgs, TypeInfo typeInfo) {\n        traceID = \" SQLServerSQLXML:\" + nextInstanceID();\n        contents = (PLPXMLInputStream) stream;\n        this.con = null;\n        this.getterArgs = getterArgs;\n        this.typeInfo = typeInfo;\n        if (logger.isLoggable(java.util.logging.Level.FINE))\n            logger.fine(toString() + \" created by (null connection)\");\n    }\n\n    InputStream getStream() {\n        return contents;\n    }\n\n    @Override\n    public void free() throws SQLException {\n        if (!isFreed) {\n            isFreed = true;\n            if (null != contents) {\n                try {\n                    contents.close();\n                } catch (IOException e) {\n                    SQLServerException.makeFromDriverError(null, null, e.getMessage(), null, true);\n                }\n            }\n        }\n    }\n\n    private void checkClosed() throws SQLServerException {\n        if (isFreed || (null != con && con.isClosed())) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_isFreed\"));\n            SQLServerException.makeFromDriverError(con, null, form.format(new Object[] {\"SQLXML\"}), null, true);\n        }\n    }\n\n    private void checkReadXML() throws SQLException {\n        if (null == contents)\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_writeOnlyXML\"), null,\n                    true);\n        if (isUsed)\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_dataHasBeenReadXML\"),\n                    null, true);\n        try {\n            contents.checkClosed();\n        } catch (IOException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_isFreed\"));\n            SQLServerException.makeFromDriverError(con, null, form.format(new Object[] {\"SQLXML\"}), null, true);\n        }\n    }\n\n    void checkWriteXML() throws SQLException {\n        if (null != contents)\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_readOnlyXML\"), null,\n                    true);\n        if (isUsed)\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_dataHasBeenSetXML\"),\n                    null, true);\n\n    }\n\n    /**\n     * Returns an input stream to read data from this SQLXML.\n     * \n     * @throws SQLException\n     *         when an error occurs\n     * @return the input stream to that contains the SQLXML data\n     */\n    @Override\n    public InputStream getBinaryStream() throws SQLException {\n        checkClosed();\n        checkReadXML();\n        isUsed = true;\n        return contents;\n    }\n\n    /**\n     * Sets a stream that can be used to write to the SQLXML value that this SQLXML object represents The user has to\n     * write the BOM for binary streams.\n     * \n     * @throws SQLException\n     *         when an error occurs\n     * @return OutputStream\n     */\n    @Override\n    public java.io.OutputStream setBinaryStream() throws SQLException {\n        checkClosed();\n        checkWriteXML();\n        isUsed = true;\n        outputStreamValue = new ByteArrayOutputStreamToInputStream();\n        return outputStreamValue;\n    }\n\n    @Override\n    public java.io.Writer setCharacterStream() throws SQLException {\n        checkClosed();\n        checkWriteXML();\n        isUsed = true;\n        outputStreamValue = new ByteArrayOutputStreamToInputStream();\n        return new OutputStreamWriter(outputStreamValue, Encoding.UNICODE.charset());\n    }\n\n    @Override\n    public Reader getCharacterStream() throws SQLException {\n        checkClosed();\n        checkReadXML();\n        isUsed = true;\n        StreamType type = StreamType.CHARACTER;\n        InputStreamGetterArgs newArgs = new InputStreamGetterArgs(type, getterArgs.isAdaptive, getterArgs.isStreaming,\n                getterArgs.logContext);\n        // Skip the BOM bytes from the plpinputstream we do not need it for the conversion\n        assert null != contents;\n        // Read two bytes to eat BOM\n        try {\n            contents.read();\n            contents.read();\n        } catch (IOException e) {\n            SQLServerException.makeFromDriverError(null, null, e.getMessage(), null, true);\n        }\n\n        return (Reader) DDC.convertStreamToObject(contents, typeInfo, type.getJDBCType(), newArgs);\n    }\n\n    @Override\n    public String getString() throws SQLException {\n        checkClosed();\n        checkReadXML();\n        isUsed = true;\n        assert null != contents;\n        // Read two bytes to eat BOM\n        try {\n            contents.read();\n            contents.read();\n        } catch (IOException e) {\n            SQLServerException.makeFromDriverError(null, null, e.getMessage(), null, true);\n        }\n\n        byte[] byteContents = contents.getBytes();\n        return new String(byteContents, 0, byteContents.length, Encoding.UNICODE.charset());\n    }\n\n    @Override\n    public void setString(String value) throws SQLException {\n        checkClosed();\n        checkWriteXML();\n        isUsed = true;\n        if (null == value)\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_cantSetNull\"), null,\n                    true);\n        strValue = value;\n    }\n    // Support the following DOMSource, SAXSource, StAX and Stream. Also, null means default which is stream source\n\n    @Override\n    public <T extends Source> T getSource(Class<T> iface) throws SQLException {\n        checkClosed();\n        checkReadXML();\n        if (null == iface) {\n            // The compiler does not allow setting the null iface to StreamSource.class hence\n            // this cast is needed, the cast is safe.\n            @SuppressWarnings(\"unchecked\")\n            T src = (T) getSourceInternal(StreamSource.class);\n            return src;\n        } else\n            return getSourceInternal(iface);\n    }\n\n    <T extends Source> T getSourceInternal(Class<T> iface) throws SQLException {\n        isUsed = true;\n        T src = null;\n        if (DOMSource.class == iface) {\n            src = iface.cast(getDOMSource());\n        } else if (SAXSource.class == iface) {\n            src = iface.cast(getSAXSource());\n        } else if (StAXSource.class == iface) {\n            src = iface.cast(getStAXSource());\n        } else if (StreamSource.class == iface) {\n            src = iface.cast(new StreamSource(contents));\n        } else\n            // Do not support this type\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_notSupported\"), null,\n                    true);\n        return src;\n    }\n\n    @Override\n    public <T extends Result> T setResult(Class<T> resultClass) throws SQLException {\n        checkClosed();\n        checkWriteXML();\n        if (null == resultClass) {\n            // The compiler does not allow setting the null resultClass to StreamResult.class hence\n            // this cast is needed, the cast is safe.\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) setResultInternal(StreamResult.class);\n            return result;\n        } else\n            return setResultInternal(resultClass);\n\n    }\n\n    <T extends Result> T setResultInternal(Class<T> resultClass) throws SQLException {\n        isUsed = true;\n        T result = null;\n        if (DOMResult.class == resultClass) {\n            result = resultClass.cast(getDOMResult());\n        } else if (SAXResult.class == resultClass) {\n            result = resultClass.cast(getSAXResult());\n        } else if (StAXResult.class == resultClass) {\n            result = resultClass.cast(getStAXResult());\n        } else if (StreamResult.class == resultClass) {\n            outputStreamValue = new ByteArrayOutputStreamToInputStream();\n            result = resultClass.cast(new StreamResult(outputStreamValue));\n        } else\n            SQLServerException.makeFromDriverError(con, null, SQLServerException.getErrString(\"R_notSupported\"), null,\n                    true);\n        return result;\n    }\n\n    // Supporting functions\n    private DOMSource getDOMSource() throws SQLException {\n        Document document = null;\n        final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();\n        DocumentBuilder builder;\n        try {\n            // Secure feature processing is false by default for IBM parser\n            // For all others, it is true by default.\n            // So, setting it explicitly for all, irrespective of the JVM.\n            // This limits the expansion of entities.\n            // The limit is implementation specific. For IBM it's 100,000\n            // whereas for oracle it is 64,000.\n            factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);\n            builder = factory.newDocumentBuilder();\n\n            // set an entity resolver to disable parsing of external entities\n            builder.setEntityResolver(new SQLServerEntityResolver());\n            try {\n                document = builder.parse(contents);\n            } catch (IOException e) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_errorReadingStream\"));\n                Object[] msgArgs = {e.toString()};\n                SQLServerException.makeFromDriverError(null, null, form.format(msgArgs), \"\", true);\n            }\n            return new DOMSource(document);\n        } catch (ParserConfigurationException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_noParserSupport\"));\n            Object[] msgArgs = {e.toString()};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        } catch (SAXException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_failedToParseXML\"));\n            Object[] msgArgs = {e.toString()};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        }\n        return null;\n    }\n\n    private SAXSource getSAXSource() throws SQLException {\n        try {\n            InputSource src = new InputSource(contents);\n            SAXParserFactory factory = SAXParserFactory.newInstance();\n            SAXParser parser = factory.newSAXParser();\n            XMLReader reader = parser.getXMLReader();\n            return new SAXSource(reader, src);\n        } catch (SAXException | ParserConfigurationException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_failedToParseXML\"));\n            Object[] msgArgs = {e.toString()};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        }\n        return null;\n    }\n\n    private StAXSource getStAXSource() throws SQLException {\n        XMLInputFactory factory = XMLInputFactory.newInstance();\n        try {\n            XMLStreamReader r = factory.createXMLStreamReader(contents);\n            return new StAXSource(r);\n        } catch (XMLStreamException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_noParserSupport\"));\n            Object[] msgArgs = {e.toString()};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        }\n        return null;\n    }\n\n    private StAXResult getStAXResult() throws SQLException {\n        XMLOutputFactory factory = XMLOutputFactory.newInstance();\n        outputStreamValue = new ByteArrayOutputStreamToInputStream();\n        try {\n            XMLStreamWriter r = factory.createXMLStreamWriter(outputStreamValue);\n            return new StAXResult(r);\n        } catch (XMLStreamException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_noParserSupport\"));\n            Object[] msgArgs = {e.toString()};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        }\n        return null;\n    }\n\n    private SAXResult getSAXResult() throws SQLException {\n        TransformerHandler handler = null;\n        try {\n            SAXTransformerFactory stf = (SAXTransformerFactory) TransformerFactory.newInstance();\n            stf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);\n            handler = stf.newTransformerHandler();\n        } catch (TransformerConfigurationException | ClassCastException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_noParserSupport\"));\n            Object[] msgArgs = {e.toString()};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        }\n\n        if (handler == null) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_noParserSupport\"));\n            Object[] msgArgs = {\"null\"};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        }\n\n        outputStreamValue = new ByteArrayOutputStreamToInputStream();\n        handler.setResult(new StreamResult(outputStreamValue));\n        return new SAXResult(handler);\n    }\n\n    private DOMResult getDOMResult() throws SQLException {\n        final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();\n        DocumentBuilder builder;\n        assert null == outputStreamValue;\n        try {\n            builder = factory.newDocumentBuilder();\n            docValue = builder.newDocument();\n            return new DOMResult(docValue);\n        } catch (ParserConfigurationException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_noParserSupport\"));\n            Object[] msgArgs = {e.toString()};\n            SQLServerException.makeFromDriverError(con, null, form.format(msgArgs), null, true);\n        }\n        return null;\n    }\n}\n\n\n/**\n * Converts the byte information in the string to an inputstream which is used when sending to the info to the server.\n */\nfinal class ByteArrayOutputStreamToInputStream extends ByteArrayOutputStream {\n    ByteArrayInputStream getInputStream() {\n        return new ByteArrayInputStream(buf, 0, count);\n    }\n}\n\n\n/**\n * Resolves External entities in an XML with inline DTDs to empty string.\n *\n */\nfinal class SQLServerEntityResolver implements EntityResolver {\n    public InputSource resolveEntity(String publicId, String systemId) {\n        return new InputSource(new StringReader(\"\"));\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerColumnEncryptionKeyStoreProvider.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.time.Duration;\n\n\n/**\n * \n * Defines the abstract class for a SQL Server Column Encryption key store provider Extend this class to implement a\n * custom key store provider.\n *\n */\npublic abstract class SQLServerColumnEncryptionKeyStoreProvider {\n\n    /**\n     * default constructor\n     */\n    public SQLServerColumnEncryptionKeyStoreProvider() {}\n\n    /**\n     * Sets the name of this key store provider.\n     * \n     * @param name\n     *        value to be set for the key store provider.\n     */\n    public abstract void setName(String name);\n\n    /**\n     * Returns the name of this key store provider.\n     * \n     * @return the name of this key store provider.\n     */\n    public abstract String getName();\n\n    /**\n     * Decrypts the specified encrypted value of a column encryption key. The encrypted value is expected to be\n     * encrypted using the column master key with the specified key path and using the specified algorithm.\n     * \n     * @param masterKeyPath\n     *        The column master key path.\n     * @param encryptionAlgorithm\n     *        the specific encryption algorithm.\n     * @param encryptedColumnEncryptionKey\n     *        the encrypted column encryption key\n     * @return the decrypted value of column encryption key.\n     * @throws SQLServerException\n     *         when an error occurs while decrypting the CEK\n     */\n    public abstract byte[] decryptColumnEncryptionKey(String masterKeyPath, String encryptionAlgorithm,\n            byte[] encryptedColumnEncryptionKey) throws SQLServerException;\n\n    /**\n     * Encrypts a column encryption key using the column master key with the specified key path and using the specified\n     * algorithm.\n     * \n     * @param masterKeyPath\n     *        The column master key path.\n     * @param encryptionAlgorithm\n     *        the specific encryption algorithm.\n     * @param columnEncryptionKey\n     *        column encryption key to be encrypted.\n     * @return the encrypted column encryption key.\n     * @throws SQLServerException\n     *         when an error occurs while encrypting the CEK\n     */\n    public abstract byte[] encryptColumnEncryptionKey(String masterKeyPath, String encryptionAlgorithm,\n            byte[] columnEncryptionKey) throws SQLServerException;\n\n    /**\n     * Verify the signature is valid for the column master key\n     * \n     * @param masterKeyPath\n     *        column master key path\n     * @param allowEnclaveComputations\n     *        indicates whether the column master key supports enclave computations\n     * @param signature\n     *        signature of the column master key metadata\n     * @return\n     *         whether the signature is valid for the column master key\n     * @throws SQLServerException\n     *         when an error occurs while verifying the signature\n     */\n    public abstract boolean verifyColumnMasterKeyMetadata(String masterKeyPath, boolean allowEnclaveComputations,\n            byte[] signature) throws SQLServerException;\n\n    /**\n     * Returns the time-to-live for items in the cache of column encryption keys, as implemented in the key store provider.\n     * \n     * @return the time-to-live for items in the cache.\n     */\n    public Duration getColumnEncryptionKeyCacheTtl() {\n        return Duration.ZERO;\n    }\n\n    /**\n     * Sets the the time-to-live for items in the cache of column encryption keys in the key store provider.\n     * \n     * @param duration\n     *        value to be set for the time-to-live for items in the cache in the key store provider.\n     */\n    public void setColumnEncryptionCacheTtl(Duration duration) {}\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerResultSetMetaData.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.sql.SQLException;\nimport java.util.concurrent.atomic.AtomicInteger;\n\n\n/**\n * Provides an implementation of the result set metadata to the SQL Server. A ResultSetMetaData object can be used to\n * obtain the meta data (types and type properties) of the columns in a ResultSet.\n *\n * The API javadoc for JDBC API methods that this class implements are not repeated here. Please see Sun's JDBC API\n * interfaces javadoc for those details.\n */\npublic final class SQLServerResultSetMetaData implements ISQLServerResultSetMetaData {\n    /**\n     * Always update serialVersionUID when prompted.\n     */\n    private static final long serialVersionUID = -5747558730471411712L;\n\n    /** connection */\n    private SQLServerConnection con;\n\n    /** resultset */\n    private final SQLServerResultSet rs;\n    static final private java.util.logging.Logger logger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.SQLServerResultSetMetaData\");\n\n    /**\n     * Unique id generator for each instance (used for logging).\n     */\n    static private final AtomicInteger baseID = new AtomicInteger(0);\n\n    /** trace ID */\n    final private String traceID;\n\n    // Returns unique id for each instance.\n    private static int nextInstanceID() {\n        return baseID.incrementAndGet();\n    }\n\n    final public String toString() {\n        return traceID;\n...\n", "needle_token_start": 7023, "needle_token_end": 7416, "code_context_ntokens": 16085, "output": ["```java\nvoid notifyEvent(SQLServerException e) {\n    if (pcLogger.isLoggable(Level.FINER))\n        pcLogger.finer(toString() + \" Exception:\" + e + safeCID());\n\n    // close the proxy on fatal error event. Note exception is null then the event comes from the proxy close.\n    if (null != e) {\n        lock.lock();\n        try {\n            if (null != lastProxyConnection) {\n                lastProxyConnection.internalClose();\n                lastProxyConnection = null;\n            }\n        } finally {\n            lock.unlock();\n        }\n    }\n\n    // A connection handle issued from this pooled connection is closing or an error occurred in the connection\n    listenersLock.lock();\n    try {\n        for (int i = 0; i < listeners.size(); i++) {\n            ConnectionEventListener listener = listeners.elementAt(i);\n\n            if (listener == null)\n                continue;\n\n            ConnectionEvent ev = new ConnectionEvent(this, e);\n            if (null == e) {\n                if (pcLogger.isLoggable(Level.FINER))\n                    pcLogger.finer(toString() + \" notifyEvent:connectionClosed \" + safeCID());\n                listener.connectionClosed(ev);\n            } else {\n                if (pcLogger.isLoggable(Level.FINER))\n                    pcLogger.finer(toString() + \" notifyEvent:connectionErrorOccurred \" + safeCID());\n                listener.connectionErrorOccurred(ev);\n            }\n        }\n    } finally {\n        listenersLock.unlock();\n    }\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "getColumnEncryptionKeyValues", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/AE.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To retrieve a list of encryption details associated with a specific column encryption key (CEK), which includes multiple encrypted values of the CEK when using different column master keys (CMKs).\n2. **Input**: There are no direct inputs as this method does not take parameters; it operates on the internal state of its class instance.\n3. **Output**: Returns a list of objects, each containing details such as the encrypted CEK, database and key identifiers, key path, key store name, and encryption algorithm used.\n4. **Procedure**: The method directly accesses a pre-stored list within the class instance that holds the encryption details for the CEK. It simply returns this list to the caller without any modifications or additional processing.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerCallableStatement.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.math.BigDecimal;\nimport java.sql.SQLType;\nimport java.sql.Timestamp;\nimport java.util.Calendar;\n\n\n/**\n * Provides an interface to the {@link SQLServerCallableStatement} class.\n */\npublic interface ISQLServerCallableStatement extends java.sql.CallableStatement, ISQLServerPreparedStatement {\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.BigDecimal\n     * object in the Java programming language.\n     * \n     * @param parameterName\n     *        the label for the column\n     * @param scale\n     *        the scale of the BigDecimal\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     * \n     * @deprecated\n     */\n    @Deprecated(since = \"6.5.4\")\n    BigDecimal getBigDecimal(String parameterName, int scale) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getDateTime(int index) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param parameterName\n     *        the label for the column specified with the SQL AS clause. If the SQL AS clause was not specified, then\n     *        the label is the name of the column\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getDateTime(String parameterName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language. This method uses the given calendar to construct an appropriate\n     * millisecond value for the timestamp if the underlying database does not store timezone information.\n     * \n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the dateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getDateTime(int index, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language. This method uses the given calendar to construct an appropriate\n     * millisecond value for the timestamp if the underlying database does not store timezone information.\n     * \n     * @param name\n     *        the name of the column\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the dateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getDateTime(String name, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getSmallDateTime(int index) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param parameterName\n     *        The name of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getSmallDateTime(String parameterName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param index\n     *        the first column is 1, the second is 2, ...\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the smalldateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getSmallDateTime(int index, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this ResultSet object as a java.sql.Timestamp\n     * object in the Java programming language.\n     * \n     * @param name\n     *        The name of a column\n     * @param cal\n     *        the java.util.Calendar object to use in constructing the smalldateTime\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    Timestamp getSmallDateTime(String name, Calendar cal) throws SQLServerException;\n\n    /**\n     * Returns the DateTimeOffset value of parameter with index parameterIndex.\n     * \n     * @param parameterIndex\n     *        the first parameter is 1, the second is 2, and so on\n     * @return DateTimeOffset value if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         if parameterIndex is out of range; if a database access error occurs or this method is called on a closed\n     *         <code>CallableStatement</code>\n     */\n    microsoft.sql.DateTimeOffset getDateTimeOffset(int parameterIndex) throws SQLServerException;\n\n    /**\n     * Returns the DateTimeOffset value of parameter with name parameterName.\n     * \n     * @param parameterName\n     *        the name of the parameter\n     * @return DateTimeOffset value if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     */\n    microsoft.sql.DateTimeOffset getDateTimeOffset(String parameterName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this <code>ResultSet</code> object as a stream\n     * of ASCII characters. The value can then be read in chunks from the stream. This method is particularly suitable\n     * for retrieving large <code>LONGVARCHAR</code> values. The JDBC driver will do any necessary conversion from the\n     * database format into ASCII.\n     *\n     * <P>\n     * <B>Note:</B> All the data in the returned stream must be read prior to getting the value of any other column. The\n     * next call to a getter method implicitly closes the stream. Also, a stream may return <code>0</code> when the\n     * method <code>InputStream.available</code> is called whether there is data available or not.\n     *\n     * @param parameterIndex\n     *        the first column is 1, the second is 2, ...\n     * @return a Java input stream that delivers the database column value as a stream of one-byte ASCII characters; if\n     *         the value is SQL <code>NULL</code>, the value returned is <code>null</code>\n     * @throws SQLServerException\n     *         if the columnIndex is not valid; if a database access error occurs or this method is called on a closed\n     *         result set\n     */\n    java.io.InputStream getAsciiStream(int parameterIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this <code>ResultSet</code> object as a stream\n     * of ASCII characters. The value can then be read in chunks from the stream. This method is particularly suitable\n     * for retrieving large <code>LONGVARCHAR</code> values. The JDBC driver will do any necessary conversion from the\n     * database format into ASCII.\n     *\n     * <P>\n     * <B>Note:</B> All the data in the returned stream must be read prior to getting the value of any other column. The\n     * next call to a getter method implicitly closes the stream. Also, a stream may return <code>0</code> when the\n     * method <code>available</code> is called whether there is data available or not.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @return a Java input stream that delivers the database column value as a stream of one-byte ASCII characters. If\n     *         the value is SQL <code>NULL</code>, the value returned is <code>null</code>.\n     * @throws SQLServerException\n     *         if the columnLabel is not valid; if a database access error occurs or this method is called on a closed\n     *         result set\n     */\n    java.io.InputStream getAsciiStream(String parameterName) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param parameterIndex\n     *        The zero-based ordinal of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    BigDecimal getMoney(int parameterIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param parameterName\n     *        The name of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    BigDecimal getMoney(String parameterName) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param parameterIndex\n     *        The zero-based ordinal of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    BigDecimal getSmallMoney(int parameterIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the column specified as a java.math.BigDecimal object.\n     * \n     * @param parameterName\n     *        The name of a column.\n     * @return the column value; if the value is SQL NULL, the value returned is null.\n     * @throws SQLServerException\n     *         when an error occurs\n     */\n    BigDecimal getSmallMoney(String parameterName) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this <code>ResultSet</code> object as a stream\n     * of uninterpreted bytes. The value can then be read in chunks from the stream. This method is particularly\n     * suitable for retrieving large <code>LONGVARBINARY</code> values.\n     *\n     * <P>\n     * <B>Note:</B> All the data in the returned stream must be read prior to getting the value of any other column. The\n     * next call to a getter method implicitly closes the stream. Also, a stream may return <code>0</code> when the\n     * method <code>InputStream.available</code> is called whether there is data available or not.\n     *\n     * @param parameterIndex\n     *        the first column is 1, the second is 2, ...\n     * @return a Java input stream that delivers the database column value as a stream of uninterpreted bytes; if the\n     *         value is SQL <code>NULL</code>, the value returned is <code>null</code>\n     * @throws SQLServerException\n     *         if the columnIndex is not valid; if a database access error occurs or this method is called on a closed\n     *         result set\n     */\n    java.io.InputStream getBinaryStream(int parameterIndex) throws SQLServerException;\n\n    /**\n     * Returns the value of the designated column in the current row of this <code>ResultSet</code> object as a stream\n     * of uninterpreted <code>byte</code>s. The value can then be read in chunks from the stream. This method is\n     * particularly suitable for retrieving large <code>LONGVARBINARY</code> values.\n     *\n     * <P>\n     * <B>Note:</B> All the data in the returned stream must be read prior to getting the value of any other column. The\n     * next call to a getter method implicitly closes the stream. Also, a stream may return <code>0</code> when the\n     * method <code>available</code> is called whether there is data available or not.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @return a Java input stream that delivers the database column value as a stream of uninterpreted bytes; if the\n     *         value is SQL <code>NULL</code>, the result is <code>null</code>\n     * @throws SQLServerException\n     *         if the columnLabel is not valid; if a database access error occurs or this method is called on a closed\n     *         result set\n     */\n    java.io.InputStream getBinaryStream(String parameterName) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Timestamp</code> value. The driver converts this to an\n     * SQL <code>TIMESTAMP</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param calendar\n     *        a java.util.Calendar\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see #getTimestamp\n     */\n    void setTimestamp(String parameterName, java.sql.Timestamp value, Calendar calendar,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Time</code> value, using the given\n     * <code>Calendar</code> object. The driver uses the <code>Calendar</code> object to construct an SQL\n     * <code>TIME</code> value, which the driver then sends to the database. With a a <code>Calendar</code> object, the\n     * driver can calculate the time taking into account a custom timezone. If no <code>Calendar</code> object is\n     * specified, the driver uses the default timezone, which is that of the virtual machine running the application.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param calendar\n     *        the <code>Calendar</code> object the driver will use to construct the time\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see #getTime\n     */\n    void setTime(String parameterName, java.sql.Time value, Calendar calendar,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Date</code> value, using the given\n     * <code>Calendar</code> object. The driver uses the <code>Calendar</code> object to construct an SQL\n     * <code>DATE</code> value, which the driver then sends to the database. With a a <code>Calendar</code> object, the\n     * driver can calculate the date taking into account a custom timezone. If no <code>Calendar</code> object is\n     * specified, the driver uses the default timezone, which is that of the virtual machine running the application.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param calendar\n     *        the <code>Calendar</code> object the driver will use to construct the date\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see #getDate\n     */\n    void setDate(String parameterName, java.sql.Date value, Calendar calendar,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>String</code> object. The driver converts this to a SQL\n     * <code>NCHAR</code> or <code>NVARCHAR</code> or <code>LONGNVARCHAR</code>\n     * \n     * @param parameterName\n     *        the name of the parameter to be set\n     * @param value\n     *        the parameter value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if the driver does not support national\n     *         character sets; if the driver can detect that a data conversion error could occur; if a database access\n     *         error occurs or this method is called on a closed <code>CallableStatement</code>\n     */\n    void setNString(String parameterName, String value, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the value of the designated parameter with the given object.\n     *\n     * <p>\n     * The given Java object will be converted to the given targetSqlType before being sent to the database.\n     *\n     * If the object has a custom mapping (is of a class implementing the interface <code>SQLData</code>), the JDBC\n     * driver should call the method <code>SQLData.writeSQL</code> to write it to the SQL data stream. If, on the other\n     * hand, the object is of a class implementing <code>Ref</code>, <code>Blob</code>, <code>Clob</code>,\n     * <code>NClob</code>, <code>Struct</code>, <code>java.net.URL</code>, or <code>Array</code>, the driver should pass\n     * it to the database as a value of the corresponding SQL type.\n     * <P>\n     * Note that this method may be used to pass database- specific abstract data types.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the object containing the input parameter value\n     * @param sqlType\n     *        the SQL type (as defined in java.sql.Types) to be sent to the database. The scale argument may further\n     *        qualify this type.\n     * @param decimals\n     *        for java.sql.Types.DECIMAL or java.sql.Types.NUMERIC types, this is the number of digits after the decimal\n     *        point. For all other types, this value will be ignored.\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see java.sql.Types\n     * @see #getObject\n     */\n    void setObject(String parameterName, Object value, int sqlType, int decimals,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the value of the designated parameter with the given object.\n     *\n     * <p>\n     * The given Java object will be converted to the given targetSqlType before being sent to the database.\n     *\n     * If the object has a custom mapping (is of a class implementing the interface <code>SQLData</code>), the JDBC\n     * driver should call the method <code>SQLData.writeSQL</code> to write it to the SQL data stream. If, on the other\n     * hand, the object is of a class implementing <code>Ref</code>, <code>Blob</code>, <code>Clob</code>,\n     * <code>NClob</code>, <code>Struct</code>, <code>java.net.URL</code>, or <code>Array</code>, the driver should pass\n     * it to the database as a value of the corresponding SQL type.\n     * <P>\n     * Note that this method may be used to pass datatabase- specific abstract data types.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the object containing the input parameter value\n     * @param targetSqlType\n     *        the SQL type (as defined in java.sql.Types) to be sent to the database. The scale argument may further\n     *        qualify this type.\n     * @param precision\n     *        the precision of the column.\n     * @param scale\n     *        the scale of the column.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see java.sql.Types\n     * @see #getObject\n     */\n    void setObject(String parameterName, Object value, int targetSqlType, Integer precision,\n            int scale) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Timestamp</code> value. The driver converts this to an\n     * SQL <code>TIMESTAMP</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param scale\n     *        the scale of the parameter\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see #getTimestamp\n     */\n    void setTimestamp(String parameterName, java.sql.Timestamp value, int scale) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Timestamp</code> value. The driver converts this to an\n     * SQL <code>TIMESTAMP</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param scale\n     *        the scale of the parameter\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see #getTimestamp\n     */\n    void setTimestamp(String parameterName, java.sql.Timestamp value, int scale,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets parameter parameterName to DateTimeOffset value.\n     * \n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        DateTimeOffset value\n     * @throws SQLServerException\n     *         if an error occurs\n     */\n    void setDateTimeOffset(String parameterName, microsoft.sql.DateTimeOffset value) throws SQLServerException;\n\n    /**\n     * Sets parameter parameterName to DateTimeOffset value.\n     * \n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        DateTimeOffset value\n     * @param scale\n     *        the scale of the parameter\n     * @throws SQLServerException\n     *         if an error occurs\n     */\n    void setDateTimeOffset(String parameterName, microsoft.sql.DateTimeOffset value,\n            int scale) throws SQLServerException;\n\n    /**\n     * Sets parameter parameterName to DateTimeOffset value.\n     * \n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        DateTimeOffset value\n     * @param scale\n     *        the scale of the parameter\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if an error occurs\n     */\n    void setDateTimeOffset(String parameterName, microsoft.sql.DateTimeOffset value, int scale,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Time</code> value. The driver converts this to an SQL\n     * <code>TIME</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param scale\n     *        the scale of the column\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see #getTime\n     */\n    void setTime(String parameterName, java.sql.Time value, int scale) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Time</code> value. The driver converts this to an SQL\n     * <code>TIME</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param scale\n     *        the scale of the column\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     * @see #getTime\n     */\n    void setTime(String parameterName, java.sql.Time value, int scale, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Timestamp</code> value. The driver converts this to an\n     * SQL <code>DATETIME</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     */\n    void setDateTime(String parameterName, java.sql.Timestamp value) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Timestamp</code> value. The driver converts this to an\n     * SQL <code>DATETIME</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     */\n    void setDateTime(String parameterName, java.sql.Timestamp value, boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Timestamp</code> value. The driver converts this to an\n     * SQL <code>SMALLDATETIME</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     */\n    void setSmallDateTime(String parameterName, java.sql.Timestamp value) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>java.sql.Timestamp</code> value. The driver converts this to an\n     * SQL <code>SMALLDATETIME</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param value\n     *        the parameter value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     */\n    void setSmallDateTime(String parameterName, java.sql.Timestamp value,\n            boolean forceEncrypt) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>String</code> value. The driver converts this to an SQL\n     * <code>uniqueIdentifier</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param guid\n     *        the parameter value\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     */\n    void setUniqueIdentifier(String parameterName, String guid) throws SQLServerException;\n\n    /**\n     * Sets the designated parameter to the given <code>String</code> value. The driver converts this to an SQL\n     * <code>uniqueIdentifier</code> value when it sends it to the database.\n     *\n     * @param parameterName\n     *        the name of the parameter\n     * @param guid\n     *        the parameter value\n     * @param forceEncrypt\n     *        If the boolean forceEncrypt is set to true, the query parameter will only be set if the designation column\n     *        is encrypted and Always Encrypted is enabled on the connection or on the statement. If the boolean\n     *        forceEncrypt is set to false, the driver will not force encryption on parameters.\n     * @throws SQLServerException\n     *         if parameterName does not correspond to a named parameter; if a database access error occurs or this\n     *         method is called on a closed <code>CallableStatement</code>\n     */\n...\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/AE.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.Serializable;\nimport java.util.ArrayList;\nimport java.util.List;\n\n\n/**\n * Represents a single encrypted value for a CEK. It contains the encrypted CEK,the store type, name,the key path and\n * encryption algorithm.\n */\nclass EncryptionKeyInfo {\n    EncryptionKeyInfo(byte[] encryptedKeyVal, int dbId, int keyId, int keyVersion, byte[] mdVersion, String keyPathVal,\n            String keyStoreNameVal, String algorithmNameVal) {\n        encryptedKey = encryptedKeyVal;\n        databaseId = dbId;\n        cekId = keyId;\n        cekVersion = keyVersion;\n        cekMdVersion = mdVersion;\n        keyPath = keyPathVal;\n        keyStoreName = keyStoreNameVal;\n        algorithmName = algorithmNameVal;\n    }\n\n    byte[] encryptedKey; // the encrypted \"column encryption key\"\n    int databaseId;\n    int cekId;\n    int cekVersion;\n    byte[] cekMdVersion;\n    String keyPath;\n    String keyStoreName;\n    String algorithmName;\n    byte normalizationRuleVersion;\n}\n\n\n/**\n * Represents a unique CEK as an entry in the CekTable. A unique (plaintext is unique) CEK can have multiple encrypted\n * CEKs when using multiple CMKs. These encrypted CEKs are represented by a member ArrayList.\n */\nclass CekTableEntry {\n    static final private java.util.logging.Logger aeLogger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.AE\");\n\n    List<EncryptionKeyInfo> columnEncryptionKeyValues;\n    int ordinal;\n    int databaseId;\n    int cekId;\n    int cekVersion;\n    byte[] cekMdVersion;\n\n    \nList<EncryptionKeyInfo> getColumnEncryptionKeyValues() {\n        return columnEncryptionKeyValues;\n    }\n\n    int getOrdinal() {\n        return ordinal;\n    }\n\n    int getDatabaseId() {\n        return databaseId;\n    }\n\n    int getCekId() {\n        return cekId;\n    }\n\n    int getCekVersion() {\n        return cekVersion;\n    }\n\n    byte[] getCekMdVersion() {\n        return cekMdVersion;\n    }\n\n    CekTableEntry(int ordinalVal) {\n        ordinal = ordinalVal;\n        databaseId = 0;\n        cekId = 0;\n        cekVersion = 0;\n        cekMdVersion = null;\n        columnEncryptionKeyValues = new ArrayList<>();\n    }\n\n    int getSize() {\n        return columnEncryptionKeyValues.size();\n    }\n\n    void add(byte[] encryptedKey, int dbId, int keyId, int keyVersion, byte[] mdVersion, String keyPath,\n            String keyStoreName, String algorithmName) {\n\n        assert null != columnEncryptionKeyValues : \"columnEncryptionKeyValues should already be initialized.\";\n\n        aeLogger.fine(\"Retrieving CEK values\");\n\n        EncryptionKeyInfo encryptionKey = new EncryptionKeyInfo(encryptedKey, dbId, keyId, keyVersion, mdVersion,\n                keyPath, keyStoreName, algorithmName);\n        columnEncryptionKeyValues.add(encryptionKey);\n\n        if (0 == databaseId) {\n            databaseId = dbId;\n            cekId = keyId;\n            cekVersion = keyVersion;\n            cekMdVersion = mdVersion;\n        } else {\n            assert (databaseId == dbId);\n            assert (cekId == keyId);\n            assert (cekVersion == keyVersion);\n            assert ((null != cekMdVersion) && (null != mdVersion) && (cekMdVersion.length == mdVersion.length));\n        }\n    }\n}\n\n\n/**\n * Contains all CEKs, each row represents one unique CEK (represented by CekTableEntry).\n */\nclass CekTable implements Serializable {\n    /**\n     * Always update serialVersionUID when prompted.\n     */\n    private static final long serialVersionUID = -4568542970907052239L;\n\n    private transient CekTableEntry[] keyList;\n\n    CekTable(int tableSize) {\n        keyList = new CekTableEntry[tableSize];\n    }\n\n    int getSize() {\n        return keyList.length;\n    }\n\n    CekTableEntry getCekTableEntry(int index) {\n        return keyList[index];\n    }\n\n    void setCekTableEntry(int index, CekTableEntry entry) {\n        keyList[index] = entry;\n    }\n}\n\n\n/**\n * Represents Encryption related information of the cipher data.\n */\nclass CryptoMetadata {\n    TypeInfo baseTypeInfo;\n    CekTableEntry cekTableEntry;\n    byte cipherAlgorithmId;\n    String cipherAlgorithmName;\n    SQLServerEncryptionType encryptionType;\n    byte normalizationRuleVersion;\n    SQLServerEncryptionAlgorithm cipherAlgorithm = null;\n    EncryptionKeyInfo encryptionKeyInfo;\n    short ordinal;\n\n    CekTableEntry getCekTableEntry() {\n        return cekTableEntry;\n    }\n\n    void setCekTableEntry(CekTableEntry cekTableEntryObj) {\n        cekTableEntry = cekTableEntryObj;\n    }\n\n    TypeInfo getBaseTypeInfo() {\n        return baseTypeInfo;\n    }\n\n    void setBaseTypeInfo(TypeInfo baseTypeInfoObj) {\n        baseTypeInfo = baseTypeInfoObj;\n    }\n\n    SQLServerEncryptionAlgorithm getEncryptionAlgorithm() {\n        return cipherAlgorithm;\n    }\n\n    void setEncryptionAlgorithm(SQLServerEncryptionAlgorithm encryptionAlgorithmObj) {\n        cipherAlgorithm = encryptionAlgorithmObj;\n    }\n\n    EncryptionKeyInfo getEncryptionKeyInfo() {\n        return encryptionKeyInfo;\n    }\n\n    void setEncryptionKeyInfo(EncryptionKeyInfo encryptionKeyInfoObj) {\n        encryptionKeyInfo = encryptionKeyInfoObj;\n    }\n\n    byte getEncryptionAlgorithmId() {\n        return cipherAlgorithmId;\n    }\n\n    String getEncryptionAlgorithmName() {\n        return cipherAlgorithmName;\n    }\n\n    SQLServerEncryptionType getEncryptionType() {\n        return encryptionType;\n    }\n\n    byte getNormalizationRuleVersion() {\n        return normalizationRuleVersion;\n    }\n\n    short getOrdinal() {\n        return ordinal;\n    }\n\n    CryptoMetadata(CekTableEntry cekTableEntryObj, short ordinalVal, byte cipherAlgorithmIdVal,\n            String cipherAlgorithmNameVal, byte encryptionTypeVal,\n            byte normalizationRuleVersionVal) throws SQLServerException {\n        cekTableEntry = cekTableEntryObj;\n        ordinal = ordinalVal;\n        cipherAlgorithmId = cipherAlgorithmIdVal;\n        cipherAlgorithmName = cipherAlgorithmNameVal;\n        encryptionType = SQLServerEncryptionType.of(encryptionTypeVal);\n        normalizationRuleVersion = normalizationRuleVersionVal;\n        encryptionKeyInfo = null;\n    }\n\n    boolean isAlgorithmInitialized() {\n        return null != cipherAlgorithm;\n    }\n}\n\n\n// Fields in the first resultset of \"sp_describe_parameter_encryption\"\n// We expect the server to return the fields in the resultset in the same order as mentioned below.\n// If the server changes the below order, then transparent parameter encryption will break.\nenum DescribeParameterEncryptionResultSet1 {\n    KEYORDINAL,\n    DBID,\n    KEYID,\n    KEYVERSION,\n    KEYMDVERSION,\n    ENCRYPTEDKEY,\n    PROVIDERNAME,\n    KEYPATH,\n    KEYENCRYPTIONALGORITHM,\n    ISREQUESTEDBYENCLAVE,\n    ENCLAVECMKSIGNATURE;\n\n    int value() {\n        // Column indexing starts from 1;\n        return ordinal() + 1;\n    }\n}\n\n\n// Fields in the second resultset of \"sp_describe_parameter_encryption\"\n// We expect the server to return the fields in the resultset in the same order as mentioned below.\n// If the server changes the below order, then transparent parameter encryption will break.\nenum DescribeParameterEncryptionResultSet2 {\n    PARAMETERORDINAL,\n    PARAMETERNAME,\n    COLUMNENCRYPTIONALGORITHM,\n    COLUMNENCRYPTIONTYPE,\n    COLUMNENCRYPTIONKEYORDINAL,\n    NORMALIZATIONRULEVERSION;\n\n    int value() {\n        // Column indexing starts from 1;\n        return ordinal() + 1;\n    }\n}\n\n\nenum ColumnEncryptionVersion {\n    AE_NOTSUPPORTED,\n    AE_V1,\n    AE_V2,\n    AE_V3;\n\n    int value() {\n        // Column indexing starts from 1;\n        return ordinal() + 1;\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.FileInputStream;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.io.Reader;\nimport java.io.Serializable;\nimport java.io.UnsupportedEncodingException;\nimport java.lang.reflect.InvocationTargetException;\nimport java.math.BigDecimal;\nimport java.math.BigInteger;\nimport java.math.RoundingMode;\nimport java.net.Inet4Address;\nimport java.net.Inet6Address;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.Socket;\nimport java.net.SocketAddress;\nimport java.net.SocketException;\nimport java.net.SocketTimeoutException;\nimport java.nio.Buffer;\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.nio.channels.SelectionKey;\nimport java.nio.channels.Selector;\nimport java.nio.channels.SocketChannel;\nimport java.nio.charset.Charset;\nimport java.security.KeyStore;\nimport java.security.Provider;\nimport java.security.Security;\nimport java.sql.Timestamp;\nimport java.text.MessageFormat;\nimport java.time.LocalDate;\nimport java.time.LocalDateTime;\nimport java.time.OffsetDateTime;\nimport java.time.OffsetTime;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Calendar;\nimport java.util.GregorianCalendar;\nimport java.util.Iterator;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Properties;\nimport java.util.Set;\nimport java.util.SimpleTimeZone;\nimport java.util.TimeZone;\nimport java.util.concurrent.ScheduledFuture;\nimport java.util.concurrent.SynchronousQueue;\nimport java.util.concurrent.ThreadPoolExecutor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.locks.Condition;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\nimport javax.net.SocketFactory;\nimport javax.net.ssl.KeyManager;\nimport javax.net.ssl.SSLContext;\nimport javax.net.ssl.SSLParameters;\nimport javax.net.ssl.SSLSocket;\nimport javax.net.ssl.TrustManager;\nimport javax.net.ssl.TrustManagerFactory;\nimport javax.net.ssl.X509TrustManager;\n\nimport com.microsoft.sqlserver.jdbc.SQLServerConnection.FedAuthTokenCommand;\nimport com.microsoft.sqlserver.jdbc.dataclassification.SensitivityClassification;\n\n\nfinal class TDS {\n    // application protocol\n    static final String PROTOCOL_TDS80 = \"tds/8.0\"; // TLS-first connections\n\n    // TDS versions\n    static final int VER_TDS80 = 0x08000000; // TDS 8.0\n    static final int VER_DENALI = 0x74000004; // TDS 7.4\n    static final int VER_KATMAI = 0x730B0003; // TDS 7.3B(includes null bit compression)\n    static final int VER_YUKON = 0x72090002; // TDS 7.2\n    static final int VER_UNKNOWN = 0x00000000; // Unknown/uninitialized\n\n    static final int TDS_RET_STAT = 0x79;\n    static final int TDS_COLMETADATA = 0x81;\n    static final int TDS_TABNAME = 0xA4;\n    static final int TDS_COLINFO = 0xA5;\n    static final int TDS_ORDER = 0xA9;\n    static final int TDS_ERR = 0xAA;\n    static final int TDS_MSG = 0xAB;\n    static final int TDS_RETURN_VALUE = 0xAC;\n    static final int TDS_LOGIN_ACK = 0xAD;\n    static final int TDS_FEATURE_EXTENSION_ACK = 0xAE;\n    static final int TDS_ROW = 0xD1;\n    static final int TDS_NBCROW = 0xD2;\n    static final int TDS_ENV_CHG = 0xE3;\n    static final int TDS_SESSION_STATE = 0xE4;\n    static final int TDS_SSPI = 0xED;\n    static final int TDS_DONE = 0xFD;\n    static final int TDS_DONEPROC = 0xFE;\n    static final int TDS_DONEINPROC = 0xFF;\n    static final int TDS_FEDAUTHINFO = 0xEE;\n    static final int TDS_SQLRESCOLSRCS = 0xa2;\n    static final int TDS_SQLDATACLASSIFICATION = 0xa3;\n\n    // DONE status https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/3c06f110-98bd-4d5b-b836-b1ba66452cb7\n    static final int DONE_FINAL = 0x0000;\n    static final int DONE_MORE = 0x0001;\n    static final int DONE_COUNT = 0x0010;\n    static final int DONE_ERROR = 0x0002;\n    static final int DONE_ATTN = 0x0020;\n    static final int DONE_SRVERROR = 0x0100;\n\n    // FedAuth\n    static final byte TDS_FEATURE_EXT_FEDAUTH = 0x02;\n    static final int TDS_FEDAUTH_LIBRARY_SECURITYTOKEN = 0x01;\n    static final int TDS_FEDAUTH_LIBRARY_ADAL = 0x02;\n    static final int TDS_FEDAUTH_LIBRARY_RESERVED = 0x7F;\n\n    // workflows\n    static final byte ADALWORKFLOW_ACTIVEDIRECTORYPASSWORD = 0x01;\n    static final byte ADALWORKFLOW_ACTIVEDIRECTORYINTEGRATED = 0x02;\n    static final byte ADALWORKFLOW_ACTIVEDIRECTORYMANAGEDIDENTITY = 0x03;\n    static final byte ADALWORKFLOW_ACTIVEDIRECTORYINTERACTIVE = 0x03;\n    static final byte ADALWORKFLOW_ACTIVEDIRECTORYDEFAULT = 0x03;\n    static final byte ADALWORKFLOW_ACCESSTOKENCALLBACK = 0x03;\n    static final byte ADALWORKFLOW_ACTIVEDIRECTORYSERVICEPRINCIPAL = 0x01; // Using the Password byte as that is the\n                                                                           // closest we have\n    static final byte ADALWORKFLOW_ACTIVEDIRECTORYSERVICEPRINCIPALCERTIFICATE = 0x01;\n\n    static final byte FEDAUTH_INFO_ID_STSURL = 0x01; // FedAuthInfoData is token endpoint URL from which to acquire fed\n                                                     // auth token\n    static final byte FEDAUTH_INFO_ID_SPN = 0x02; // FedAuthInfoData is the SPN to use for acquiring fed auth token\n\n    // AE constants\n    // 0x03 is for x_eFeatureExtensionId_Rcs\n    static final byte TDS_FEATURE_EXT_AE = 0x04;\n    static final byte COLUMNENCRYPTION_NOT_SUPPORTED = 0x00; // column encryption not supported\n    static final byte COLUMNENCRYPTION_VERSION1 = 0x01; // column encryption without enclave\n    static final byte COLUMNENCRYPTION_VERSION2 = 0x02; // column encryption with enclave\n    static final byte COLUMNENCRYPTION_VERSION3 = 0x03; // column encryption with enclave, with retry\n    static final int CUSTOM_CIPHER_ALGORITHM_ID = 0; // max version\n    // 0x06 is for x_eFeatureExtensionId_LoginToken\n    // 0x07 is for x_eFeatureExtensionId_ClientSideTelemetry\n    // Data Classification constants\n    static final byte TDS_FEATURE_EXT_DATACLASSIFICATION = 0x09;\n    static final byte DATA_CLASSIFICATION_NOT_ENABLED = 0x00;\n    static final byte MAX_SUPPORTED_DATA_CLASSIFICATION_VERSION = 0x02;\n    static final byte DATA_CLASSIFICATION_VERSION_ADDED_RANK_SUPPORT = 0x02;\n\n    static final int AES_256_CBC = 1;\n    static final int AEAD_AES_256_CBC_HMAC_SHA256 = 2;\n    static final int AE_METADATA = 0x08;\n\n    static final byte TDS_FEATURE_EXT_UTF8SUPPORT = 0x0A;\n    static final byte TDS_FEATURE_EXT_AZURESQLDNSCACHING = 0x0B;\n    static final byte TDS_FEATURE_EXT_SESSIONRECOVERY = 0x01;\n\n    static final int TDS_TVP = 0xF3;\n    static final int TVP_ROW = 0x01;\n    static final int TVP_NULL_TOKEN = 0xFFFF;\n    static final int TVP_STATUS_DEFAULT = 0x02;\n\n    static final int TVP_ORDER_UNIQUE_TOKEN = 0x10;\n    // TVP_ORDER_UNIQUE_TOKEN flags\n    static final byte TVP_ORDERASC_FLAG = 0x1;\n    static final byte TVP_ORDERDESC_FLAG = 0x2;\n    static final byte TVP_UNIQUE_FLAG = 0x4;\n\n    // TVP flags, may be used in other places\n    static final int FLAG_NULLABLE = 0x01;\n    static final int FLAG_TVP_DEFAULT_COLUMN = 0x200;\n\n    static final int FEATURE_EXT_TERMINATOR = -1;\n\n    // Sql_variant length\n    static final int SQL_VARIANT_LENGTH = 8009;\n\n    static final String getTokenName(int tdsTokenType) {\n        switch (tdsTokenType) {\n            case TDS_RET_STAT:\n                return \"TDS_RET_STAT (0x79)\";\n            case TDS_COLMETADATA:\n                return \"TDS_COLMETADATA (0x81)\";\n            case TDS_TABNAME:\n                return \"TDS_TABNAME (0xA4)\";\n            case TDS_COLINFO:\n                return \"TDS_COLINFO (0xA5)\";\n            case TDS_ORDER:\n                return \"TDS_ORDER (0xA9)\";\n            case TDS_ERR:\n                return \"TDS_ERR (0xAA)\";\n            case TDS_MSG:\n                return \"TDS_MSG (0xAB)\";\n            case TDS_RETURN_VALUE:\n                return \"TDS_RETURN_VALUE (0xAC)\";\n            case TDS_LOGIN_ACK:\n                return \"TDS_LOGIN_ACK (0xAD)\";\n            case TDS_FEATURE_EXTENSION_ACK:\n                return \"TDS_FEATURE_EXTENSION_ACK (0xAE)\";\n            case TDS_ROW:\n                return \"TDS_ROW (0xD1)\";\n            case TDS_NBCROW:\n                return \"TDS_NBCROW (0xD2)\";\n            case TDS_ENV_CHG:\n                return \"TDS_ENV_CHG (0xE3)\";\n            case TDS_SESSION_STATE:\n                return \"TDS_SESSION_STATE (0xE4)\";\n            case TDS_SSPI:\n                return \"TDS_SSPI (0xED)\";\n            case TDS_DONE:\n                return \"TDS_DONE (0xFD)\";\n            case TDS_DONEPROC:\n                return \"TDS_DONEPROC (0xFE)\";\n            case TDS_DONEINPROC:\n                return \"TDS_DONEINPROC (0xFF)\";\n            case TDS_FEDAUTHINFO:\n                return \"TDS_FEDAUTHINFO (0xEE)\";\n            case TDS_FEATURE_EXT_DATACLASSIFICATION:\n                return \"TDS_FEATURE_EXT_DATACLASSIFICATION (0x09)\";\n            case TDS_FEATURE_EXT_UTF8SUPPORT:\n                return \"TDS_FEATURE_EXT_UTF8SUPPORT (0x0A)\";\n            case TDS_FEATURE_EXT_AZURESQLDNSCACHING:\n                return \"TDS_FEATURE_EXT_AZURESQLDNSCACHING (0x0B)\";\n            case TDS_FEATURE_EXT_SESSIONRECOVERY:\n                return \"TDS_FEATURE_EXT_SESSIONRECOVERY (0x01)\";\n            default:\n                return \"unknown token (0x\" + Integer.toHexString(tdsTokenType).toUpperCase() + \")\";\n        }\n    }\n\n    // RPC ProcIDs for use with RPCRequest (PKT_RPC) calls\n    static final short PROCID_SP_CURSOR = 1;\n    static final short PROCID_SP_CURSOROPEN = 2;\n    static final short PROCID_SP_CURSORPREPARE = 3;\n    static final short PROCID_SP_CURSOREXECUTE = 4;\n    static final short PROCID_SP_CURSORPREPEXEC = 5;\n    static final short PROCID_SP_CURSORUNPREPARE = 6;\n    static final short PROCID_SP_CURSORFETCH = 7;\n    static final short PROCID_SP_CURSOROPTION = 8;\n    static final short PROCID_SP_CURSORCLOSE = 9;\n    static final short PROCID_SP_EXECUTESQL = 10;\n    static final short PROCID_SP_PREPARE = 11;\n    static final short PROCID_SP_EXECUTE = 12;\n    static final short PROCID_SP_PREPEXEC = 13;\n    static final short PROCID_SP_PREPEXECRPC = 14;\n    static final short PROCID_SP_UNPREPARE = 15;\n\n    // Constants for use with cursor RPCs\n    static final short SP_CURSOR_OP_UPDATE = 1;\n    static final short SP_CURSOR_OP_DELETE = 2;\n    static final short SP_CURSOR_OP_INSERT = 4;\n    static final short SP_CURSOR_OP_REFRESH = 8;\n    static final short SP_CURSOR_OP_LOCK = 16;\n    static final short SP_CURSOR_OP_SETPOSITION = 32;\n    static final short SP_CURSOR_OP_ABSOLUTE = 64;\n\n    // Constants for server-cursored result sets.\n    // See the Engine Cursors Functional Specification for details.\n    static final int FETCH_FIRST = 1;\n    static final int FETCH_NEXT = 2;\n    static final int FETCH_PREV = 4;\n    static final int FETCH_LAST = 8;\n    static final int FETCH_ABSOLUTE = 16;\n    static final int FETCH_RELATIVE = 32;\n    static final int FETCH_REFRESH = 128;\n    static final int FETCH_INFO = 256;\n    static final int FETCH_PREV_NOADJUST = 512;\n    static final byte RPC_OPTION_NO_METADATA = (byte) 0x02;\n\n    // Transaction manager request types\n    static final short TM_GET_DTC_ADDRESS = 0;\n    static final short TM_PROPAGATE_XACT = 1;\n    static final short TM_BEGIN_XACT = 5;\n    static final short TM_PROMOTE_PROMOTABLE_XACT = 6;\n    static final short TM_COMMIT_XACT = 7;\n    static final short TM_ROLLBACK_XACT = 8;\n    static final short TM_SAVE_XACT = 9;\n\n    static final byte PKT_QUERY = 1;\n    static final byte PKT_RPC = 3;\n    static final byte PKT_REPLY = 4;\n    static final byte PKT_CANCEL_REQ = 6;\n    static final byte PKT_BULK = 7;\n    static final byte PKT_DTC = 14;\n    static final byte PKT_LOGON70 = 16; // 0x10\n    static final byte PKT_SSPI = 17;\n    static final byte PKT_PRELOGIN = 18; // 0x12\n    static final byte PKT_FEDAUTH_TOKEN_MESSAGE = 8; // Authentication token for federated authentication\n\n    static final byte STATUS_NORMAL = 0x00;\n    static final byte STATUS_BIT_EOM = 0x01;\n    static final byte STATUS_BIT_ATTENTION = 0x02;// this is called ignore bit in TDS spec\n    static final byte STATUS_BIT_RESET_CONN = 0x08;\n\n    // Various TDS packet size constants\n    static final int INVALID_PACKET_SIZE = -1;\n    static final int INITIAL_PACKET_SIZE = 4096;\n    static final int MIN_PACKET_SIZE = 512;\n    static final int MAX_PACKET_SIZE = 32767;\n    static final int DEFAULT_PACKET_SIZE = 8000;\n    static final int SERVER_PACKET_SIZE = 0; // Accept server's configured packet size\n\n    // TDS packet header size and offsets\n    static final int PACKET_HEADER_SIZE = 8;\n    static final int PACKET_HEADER_MESSAGE_TYPE = 0;\n    static final int PACKET_HEADER_MESSAGE_STATUS = 1;\n    static final int PACKET_HEADER_MESSAGE_LENGTH = 2;\n    static final int PACKET_HEADER_SPID = 4;\n    static final int PACKET_HEADER_SEQUENCE_NUM = 6;\n    static final int PACKET_HEADER_WINDOW = 7; // Reserved/Not used\n\n    // MARS header length:\n    // 2 byte header type\n    // 8 byte transaction descriptor\n    // 4 byte outstanding request count\n    static final int MARS_HEADER_LENGTH = 18; // 2 byte header type, 8 byte transaction descriptor,\n    static final int TRACE_HEADER_LENGTH = 26; // header length (4) + header type (2) + guid (16) + Sequence number size\n                                               // (4)\n\n    static final short HEADERTYPE_TRACE = 3; // trace header type\n\n    // Message header length\n    static final int MESSAGE_HEADER_LENGTH = MARS_HEADER_LENGTH + 4; // length includes message header itself\n\n    static final byte B_PRELOGIN_OPTION_VERSION = 0x00;\n    static final byte B_PRELOGIN_OPTION_ENCRYPTION = 0x01;\n    static final byte B_PRELOGIN_OPTION_INSTOPT = 0x02;\n    static final byte B_PRELOGIN_OPTION_THREADID = 0x03;\n    static final byte B_PRELOGIN_OPTION_MARS = 0x04;\n    static final byte B_PRELOGIN_OPTION_TRACEID = 0x05;\n    static final byte B_PRELOGIN_OPTION_FEDAUTHREQUIRED = 0x06;\n    static final byte B_PRELOGIN_OPTION_TERMINATOR = (byte) 0xFF;\n\n    // Login option byte 1\n    static final byte LOGIN_OPTION1_ORDER_X86 = 0x00;\n    static final byte LOGIN_OPTION1_ORDER_6800 = 0x01;\n    static final byte LOGIN_OPTION1_CHARSET_ASCII = 0x00;\n...\n", "needle_token_start": 8921, "needle_token_end": 8948, "code_context_ntokens": 16220, "output": ["```java\n/**\n * Represents a unique CEK as an entry in the CekTable. A unique (plaintext is unique) CEK can have multiple encrypted\n * CEKs when using multiple CMKs. These encrypted CEKs are represented by a member ArrayList.\n */\nclass CekTableEntry {\n    static final private java.util.logging.Logger aeLogger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.AE\");\n\n    List<EncryptionKeyInfo> columnEncryptionKeyValues;\n    int ordinal;\n    int databaseId;\n    int cekId;\n    int cekVersion;\n    byte[] cekMdVersion;\n\n    \nList<EncryptionKeyInfo> getColumnEncryptionKeyValues() {\n        return columnEncryptionKeyValues;\n    }\n\n    int getOrdinal() {\n        return ordinal;\n    }\n\n    int getDatabaseId() {\n        return databaseId;\n    }\n\n    int getCekId() {\n        return cekId;\n    }\n\n    int getCekVersion() {\n        return cekVersion;\n    }\n\n    byte[] getCekMdVersion() {\n        return cekMdVersion;\n    }\n\n    CekTableEntry(int ordinalVal) {\n        ordinal = ordinalVal;\n        databaseId = 0;\n        cekId = 0;\n        cekVersion = 0;\n        cekMdVersion = null;\n        columnEncryptionKeyValues = new ArrayList<>();\n    }\n\n    int getSize() {\n        return columnEncryptionKeyValues.size();\n    }\n\n    void add(byte[] encryptedKey, int dbId, int keyId, int keyVersion, byte[] mdVersion, String keyPath,\n            String keyStoreName, String algorithmName) {\n\n        assert null != columnEncryptionKeyValues : \"columnEncryptionKeyValues should already be initialized.\";\n\n        aeLogger.fine(\"Retrieving CEK values\");\n\n        EncryptionKeyInfo encryptionKey = new EncryptionKeyInfo(encryptedKey, dbId, keyId, keyVersion, mdVersion,\n                keyPath, keyStoreName, algorithmName);\n        columnEncryptionKeyValues.add(encryptionKey);\n\n        if (0 == databaseId) {\n            databaseId = dbId;\n            cekId = keyId;\n            cekVersion = keyVersion;\n            cekMdVersion = mdVersion;\n        } else {\n            assert (databaseId == dbId);\n            assert (cekId == keyId);\n            assert (cekVersion == keyVersion);\n            assert ((null != cekMdVersion) && (null != mdVersion) && (cekMdVersion.length == mdVersion.length));\n        }\n    }\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "getTrustStorePassword", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the password used for accessing the trust store in a database connection configuration.\n2. **Input**: None required directly by the method.\n3. **Output**: Returns a string representing the password of the trust store, or null if no password is set.\n4. **Procedure**: Accesses the connection properties to fetch the value associated with the trust store password key. If the key is not found or the value is not set, it returns null.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport static com.microsoft.sqlserver.jdbc.SQLServerConnection.getCachedParsedSQL;\nimport static com.microsoft.sqlserver.jdbc.SQLServerConnection.parseAndCacheSQL;\n\nimport java.sql.BatchUpdateException;\nimport java.sql.ResultSet;\nimport java.sql.SQLException;\nimport java.sql.SQLTimeoutException;\nimport java.sql.SQLWarning;\nimport java.sql.Statement;\nimport java.text.MessageFormat;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Stack;\nimport java.util.StringTokenizer;\nimport java.util.Vector;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.logging.Level;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\nimport com.microsoft.sqlserver.jdbc.SQLServerConnection.CityHash128Key;\n\n\n/**\n * Provides an implementation of java.sql.Statement JDBC Interface to assist in creating Statements against SQL Server.\n * It also provides a number of base class implementation methods for the JDBC prepared statement and callable\n * Statements. SQLServerStatement's basic role is to execute SQL statements and return update counts and resultset rows\n * to the user application.\n *\n * Documentation for specific public methods that are undocumented can be found under Sun's standard JDBC documentation\n * for class java.sql.Statement. Those methods are part of Sun's standard JDBC documentation and therefore their\n * documentation is not duplicated here.\n * <p>\n * Implementation Notes\n * <p>\n * Fetching Result sets\n * <p>\n * The queries first rowset is available immediately after the executeQuery. The first rs.next() does not make a server\n * round trip. For non server side resultsets the entire result set is in the rowset. For server side result sets the\n * number of rows in the rowset is set with nFetchSize\n * <p>\n * The API javadoc for JDBC API methods that this class implements are not repeated here. Please see Sun's JDBC API\n * interfaces javadoc for those details.\n */\npublic class SQLServerStatement implements ISQLServerStatement {\n    /**\n     * Always update serialVersionUID when prompted.\n     */\n    private static final long serialVersionUID = -4421134713913331507L;\n\n    final static char LEFT_CURLY_BRACKET = 123;\n    final static char RIGHT_CURLY_BRACKET = 125;\n\n    private static final String ACTIVITY_ID = \" ActivityId: \";\n\n    /** response buffer adaptive flag */\n    boolean isResponseBufferingAdaptive = false;\n\n    /** TDS token return value status **/\n    int returnValueStatus;\n\n    /** Check if statement contains TVP Type */\n    boolean isTVPType = false;\n\n    static int userDefinedFunctionReturnStatus = 2;\n\n    final boolean getIsResponseBufferingAdaptive() {\n        return isResponseBufferingAdaptive;\n    }\n\n    /** flag if response buffering is set */\n    private boolean wasResponseBufferingSet = false;\n\n    final boolean wasResponseBufferingSet() {\n        return wasResponseBufferingSet;\n    }\n\n    final static String IDENTITY_QUERY = \" select SCOPE_IDENTITY() AS GENERATED_KEYS\";\n\n    static final String WINDOWS_KEY_STORE_NAME = \"MSSQL_CERTIFICATE_STORE\";\n\n    /** the stored procedure name to call (if there is one) */\n    String procedureName;\n\n    /** OUT parameters associated with this statement's execution */\n    private int serverCursorId;\n\n    final int getServerCursorId() {\n        return serverCursorId;\n    }\n\n    /** server cursor row count */\n    private int serverCursorRowCount;\n\n    final int getServerCursorRowCount() {\n        return serverCursorRowCount;\n    }\n\n    /** The value of the poolable state */\n    boolean stmtPoolable;\n\n    /** Streaming access to the response TDS data stream */\n    private TDSReader tdsReader;\n\n    final TDSReader resultsReader() {\n        return tdsReader;\n    }\n\n    final boolean wasExecuted() {\n        return null != tdsReader;\n    }\n\n    /** Return parameter for stored procedure calls */\n    transient Parameter returnParam;\n\n    /**\n     * The input and out parameters for statement execution.\n     */\n    transient Parameter[] inOutParam = null; // Parameters for prepared stmts and stored procedures\n\n    /**\n     * The statement's connection.\n     */\n    final SQLServerConnection connection;\n\n    /**\n     * The user's specified query timeout (in seconds).\n     */\n    int queryTimeout;\n\n    /**\n     * timeout value for canceling the query timeout\n     */\n    int cancelQueryTimeoutSeconds;\n\n    /**\n     * Is closeOnCompletion is enabled? If true statement will be closed when all of its dependent result sets are\n     * closed\n     */\n    boolean isCloseOnCompletion = false;\n\n    /** Checks if the callable statement's parameters are set by name **/\n    protected boolean isSetByName = false;\n\n    /** Checks if the prepared statement's parameters were set by index **/\n    protected boolean isSetByIndex = false;\n\n    /**\n     * Currently executing or most recently executed TDSCommand (statement cmd, server cursor cmd, ...) subject to\n     * cancellation through Statement.cancel.\n     *\n     * Note: currentCommand is declared volatile to ensure that the JVM always returns the most recently set value for\n     * currentCommand to the canceling thread.\n     */\n    private volatile TDSCommand currentCommand = null;\n\n    /** last statement exec command */\n    private TDSCommand lastStmtExecCmd = null;\n\n    final void discardLastExecutionResults() {\n        if (null != lastStmtExecCmd && !bIsClosed) {\n            lastStmtExecCmd.close();\n            lastStmtExecCmd = null;\n        }\n\n        clearLastResult();\n    }\n\n    static final java.util.logging.Logger loggerExternal = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.Statement\");\n\n    /** logging class name */\n    final private String loggingClassName;\n\n    /** trace ID */\n    final private String traceID;\n\n    String getClassNameLogging() {\n        return loggingClassName;\n    }\n\n    /**\n     * Column Encryption Override. Defaults to the connection setting, in which case it will be Enabled if\n     * columnEncryptionSetting = true in the connection setting, Disabled if false. This may also be used to set other\n     * behavior which overrides connection level setting.\n     */\n    protected SQLServerStatementColumnEncryptionSetting stmtColumnEncriptionSetting = SQLServerStatementColumnEncryptionSetting.USE_CONNECTION_SETTING;\n\n    /**\n     * Returns the statement column encryption encryption setting\n     * \n     * @return stmtColumnEncriptionSetting\n     */\n    protected SQLServerStatementColumnEncryptionSetting getStmtColumnEncriptionSetting() {\n        return stmtColumnEncriptionSetting;\n    }\n\n    /**\n     * Encapsulates a subset of statement property values as they were set at execution time.\n     */\n    final class ExecuteProperties {\n        final private boolean wasResponseBufferingSet;\n\n        final boolean wasResponseBufferingSet() {\n...\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/tdsparser.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n\n/**\n * The top level TDS parser class.\n */\nfinal class TDSParser {\n\n    private TDSParser() {\n        throw new UnsupportedOperationException(SQLServerException.getErrString(\"R_notSupported\"));\n    }\n\n    /** TDS protocol diagnostics logger */\n    private static Logger logger = Logger.getLogger(\"com.microsoft.sqlserver.jdbc.internals.TDS.TOKEN\");\n\n    /*\n     * Parses a TDS token stream from a reader using the supplied token handler. Parsing requires the ability to peek\n     * one byte ahead into the token stream to determine the token type of the next token in the stream. When the token\n     * type has been determined, the token handler is called to process the token (or not). Parsing continues until the\n     * token handler says to stop by returning false from one of the token handling methods.\n     */\n    static void parse(TDSReader tdsReader, String logContext) throws SQLServerException {\n        parse(tdsReader, new TDSTokenHandler(logContext));\n    }\n\n    /**\n     * Default parse method to parse all tokens in TDS stream.\n     * \n     * @param tdsReader\n     * @param tdsTokenHandler\n     * @throws SQLServerException\n     */\n    static void parse(TDSReader tdsReader, TDSTokenHandler tdsTokenHandler) throws SQLServerException {\n        parse(tdsReader, tdsTokenHandler, false);\n    }\n\n    /**\n     * Underlying parse method to parse all tokens in TDS stream. Also accepts 'readOnlyWarningFlag' to parse only\n     * SQLWarnings received in TDS_MSG tokens.\n     * \n     * @param tdsReader\n     * @param tdsTokenHandler\n     * @param readOnlyWarningsFlag\n     *        - true if only TDS_MSG tokens need to be parsed in TDS Stream. false - to parse all tokens in TDS Stream.\n     * @throws SQLServerException\n     */\n    static void parse(TDSReader tdsReader, TDSTokenHandler tdsTokenHandler,\n            boolean readOnlyWarningsFlag) throws SQLServerException {\n        final boolean isLogging = logger.isLoggable(Level.FINEST);\n\n        // Process TDS tokens from the token stream until we're told to stop.\n        boolean parsing = true;\n\n        // If TDS_LOGIN_ACK is received verify for TDS_FEATURE_EXTENSION_ACK packet\n        boolean isLoginAck = false;\n        boolean isFeatureExtAck = false;\n        while (parsing) {\n            int tdsTokenType = tdsReader.peekTokenType();\n            if (isLogging) {\n                logger.finest(tdsReader.toString() + \": \" + tdsTokenHandler.logContext + \": Processing \"\n                        + ((-1 == tdsTokenType) ? \"EOF\" : TDS.getTokenName(tdsTokenType)));\n            }\n            if (readOnlyWarningsFlag && TDS.TDS_MSG != tdsTokenType) {\n                return;\n            }\n            switch (tdsTokenType) {\n                case TDS.TDS_SSPI:\n                    parsing = tdsTokenHandler.onSSPI(tdsReader);\n                    break;\n                case TDS.TDS_LOGIN_ACK:\n                    isLoginAck = true;\n                    parsing = tdsTokenHandler.onLoginAck(tdsReader);\n                    break;\n                case TDS.TDS_FEATURE_EXTENSION_ACK:\n                    isFeatureExtAck = true;\n                    tdsReader.getConnection().processFeatureExtAck(tdsReader);\n                    parsing = true;\n                    break;\n                case TDS.TDS_ENV_CHG:\n                    parsing = tdsTokenHandler.onEnvChange(tdsReader);\n                    break;\n                case TDS.TDS_SESSION_STATE:\n                    parsing = tdsTokenHandler.onSessionState(tdsReader);\n                    break;\n                case TDS.TDS_RET_STAT:\n                    parsing = tdsTokenHandler.onRetStatus(tdsReader);\n                    break;\n                case TDS.TDS_RETURN_VALUE:\n                    parsing = tdsTokenHandler.onRetValue(tdsReader);\n                    break;\n                case TDS.TDS_DONEINPROC:\n                case TDS.TDS_DONEPROC:\n                case TDS.TDS_DONE:\n                    tdsReader.getCommand().checkForInterrupt();\n                    parsing = tdsTokenHandler.onDone(tdsReader);\n                    break;\n                case TDS.TDS_ERR:\n                    parsing = tdsTokenHandler.onError(tdsReader);\n                    break;\n                case TDS.TDS_MSG:\n                    parsing = tdsTokenHandler.onInfo(tdsReader);\n                    break;\n                case TDS.TDS_ORDER:\n                    parsing = tdsTokenHandler.onOrder(tdsReader);\n                    break;\n                case TDS.TDS_COLMETADATA:\n                    parsing = tdsTokenHandler.onColMetaData(tdsReader);\n                    break;\n                case TDS.TDS_ROW:\n                    parsing = tdsTokenHandler.onRow(tdsReader);\n                    break;\n                case TDS.TDS_NBCROW:\n                    parsing = tdsTokenHandler.onNBCRow(tdsReader);\n                    break;\n                case TDS.TDS_COLINFO:\n                    parsing = tdsTokenHandler.onColInfo(tdsReader);\n                    break;\n                case TDS.TDS_TABNAME:\n                    parsing = tdsTokenHandler.onTabName(tdsReader);\n                    break;\n                case TDS.TDS_FEDAUTHINFO:\n                    parsing = tdsTokenHandler.onFedAuthInfo(tdsReader);\n                    break;\n                case TDS.TDS_SQLDATACLASSIFICATION:\n                    parsing = tdsTokenHandler.onDataClassification(tdsReader);\n                    break;\n                case -1:\n                    tdsReader.getCommand().onTokenEOF();\n                    tdsTokenHandler.onEOF(tdsReader);\n                    parsing = false;\n                    break;\n\n                default:\n                    throwUnexpectedTokenException(tdsReader, tdsTokenHandler.logContext);\n                    break;\n            }\n        }\n\n        // if TDS_FEATURE_EXTENSION_ACK is not received verify if TDS_FEATURE_EXT_AE was sent\n        if (isLoginAck && !isFeatureExtAck)\n            tdsReader.tryProcessFeatureExtAck(isFeatureExtAck);\n    }\n\n    /* Handle unexpected tokens - throw an exception */\n    static void throwUnexpectedTokenException(TDSReader tdsReader, String logContext) throws SQLServerException {\n        if (logger.isLoggable(Level.SEVERE))\n            logger.severe(tdsReader.toString() + \": \" + logContext + \": Encountered unexpected \"\n                    + TDS.getTokenName(tdsReader.peekTokenType()));\n        tdsReader.throwInvalidTDSToken(TDS.getTokenName(tdsReader.peekTokenType()));\n    }\n\n    /* Ignore a length-prefixed token */\n    static void ignoreLengthPrefixedToken(TDSReader tdsReader) throws SQLServerException {\n        tdsReader.readUnsignedByte(); // token type\n        int envValueLength = tdsReader.readUnsignedShort();\n        byte[] envValueData = new byte[envValueLength];\n        tdsReader.readBytes(envValueData, 0, envValueLength);\n    }\n}\n\n\n/**\n * A default TDS token handler with some meaningful default processing. Other token handlers should subclass from this\n * one to override the defaults and provide specialized functionality.\n *\n * ENVCHANGE_TOKEN Processes the ENVCHANGE\n *\n * RETURN_STATUS_TOKEN Ignores the returned value\n *\n * DONE_TOKEN DONEPROC_TOKEN DONEINPROC_TOKEN Ignores the returned value\n *\n * ERROR_TOKEN Remember the error and throw a SQLServerException with that error on EOF\n *\n * INFO_TOKEN ORDER_TOKEN COLINFO_TOKEN (not COLMETADATA_TOKEN) TABNAME_TOKEN Ignore the token\n *\n * EOF Throw a database exception with text from the last error token\n *\n * All other tokens Throw a TDS protocol error exception\n */\nclass TDSTokenHandler {\n    final String logContext;\n\n    private SQLServerError databaseError;\n\n    /** TDS protocol diagnostics logger */\n    private static Logger logger = Logger.getLogger(\"com.microsoft.sqlserver.jdbc.internals.TDS.TOKEN\");\n\n    final SQLServerError getDatabaseError() {\n        return databaseError;\n    }\n\n    TDSTokenHandler(String logContext) {\n        this.logContext = logContext;\n    }\n\n    boolean onSSPI(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.throwUnexpectedTokenException(tdsReader, logContext);\n        return false;\n    }\n\n    boolean onLoginAck(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.throwUnexpectedTokenException(tdsReader, logContext);\n        return false;\n    }\n\n    boolean onFeatureExtensionAck(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.throwUnexpectedTokenException(tdsReader, logContext);\n        return false;\n    }\n\n    boolean onEnvChange(TDSReader tdsReader) throws SQLServerException {\n        tdsReader.getConnection().processEnvChange(tdsReader);\n        return true;\n    }\n\n    boolean onSessionState(TDSReader tdsReader) throws SQLServerException {\n        tdsReader.getConnection().processSessionState(tdsReader);\n        return true;\n    }\n\n    boolean onRetStatus(TDSReader tdsReader) throws SQLServerException {\n        (new StreamRetStatus()).setFromTDS(tdsReader);\n        return true;\n    }\n\n    boolean onRetValue(TDSReader tdsReader) throws SQLServerException {\n        // Very unlikely to return true. If we do, it was because any return values in the\n        // tds response were never read after the RPC. If they were never read, it's safe to skip\n        // them here\n        if (this.logContext.equals(\"ExecDoneHandler\")) {\n            Parameter param = new Parameter(false);\n            param.skipRetValStatus(tdsReader);\n            param.skipValue(tdsReader, true);\n\n            return true;\n        }\n\n        TDSParser.throwUnexpectedTokenException(tdsReader, logContext);\n        return false;\n    }\n\n    boolean onDone(TDSReader tdsReader) throws SQLServerException {\n        StreamDone doneToken = new StreamDone();\n        doneToken.setFromTDS(tdsReader);\n        if (doneToken.isFinal()) {\n            // Response is completely processed hence decrement unprocessed response count.\n            tdsReader.getConnection().getSessionRecovery().decrementUnprocessedResponseCount();\n        }\n        return true;\n    }\n\n    boolean onError(TDSReader tdsReader) throws SQLServerException {\n        if (null == databaseError) {\n            databaseError = new SQLServerError();\n            databaseError.setFromTDS(tdsReader);\n        } else {\n            (new SQLServerError()).setFromTDS(tdsReader);\n        }\n\n        return true;\n    }\n\n    boolean onInfo(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.ignoreLengthPrefixedToken(tdsReader);\n        return true;\n    }\n\n    boolean onOrder(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.ignoreLengthPrefixedToken(tdsReader);\n        return true;\n    }\n\n    boolean onColMetaData(TDSReader tdsReader) throws SQLServerException {\n        /*\n         * SHOWPLAN or something else that produces extra metadata might be ON. Log info instead of throwing an exception, warn\n         * and discard the extra column meta data\n         */\n        if (logger.isLoggable(Level.INFO))\n            logger.info(tdsReader.toString() + \": \" + logContext\n                    + \": Discarding extra metadata which can be a result of SHOWPLAN settings:  \"\n                    + TDS.getTokenName(tdsReader.peekTokenType()));\n        (new StreamColumns(false)).setFromTDS(tdsReader);\n        return false;\n    }\n\n    boolean onRow(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.throwUnexpectedTokenException(tdsReader, logContext);\n        return false;\n    }\n\n    boolean onNBCRow(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.throwUnexpectedTokenException(tdsReader, logContext);\n        return false;\n    }\n\n    boolean onColInfo(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.ignoreLengthPrefixedToken(tdsReader);\n        return true;\n    }\n\n    boolean onTabName(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.ignoreLengthPrefixedToken(tdsReader);\n        return true;\n    }\n\n    void onEOF(TDSReader tdsReader) throws SQLServerException {\n        if (null != getDatabaseError()) {\n            SQLServerException.makeFromDatabaseError(tdsReader.getConnection(), null,\n                    getDatabaseError().getErrorMessage(), getDatabaseError(), false);\n        }\n    }\n\n    boolean onFedAuthInfo(TDSReader tdsReader) throws SQLServerException {\n        tdsReader.getConnection().processFedAuthInfo(tdsReader, this);\n        return true;\n    }\n\n    boolean onDataClassification(TDSReader tdsReader) throws SQLServerException {\n        TDSParser.throwUnexpectedTokenException(tdsReader, logContext);\n        return false;\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerAccessTokenCallback.java\npackage com.microsoft.sqlserver.jdbc;\n\n/**\n * Provides SqlAuthenticationToken callback to be implemented by client code.\n */\npublic interface SQLServerAccessTokenCallback {\n\n    /**\n     * For an example of callback usage, look under the project's code samples.\n     *\n     * Returns the access token for the authentication request\n     *\n     * @param stsurl\n     *        - Security token service URL.\n     * @param spn\n     *        - Service principal name.\n     *\n     * @return Returns a {@link SqlAuthenticationToken}.\n     */\n    SqlAuthenticationToken getAccessToken(String spn, String stsurl);\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerKeyVaultAuthenticationCallback.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\n/**\n * Provides a callback delegate which is to be implemented by the client code\n *\n */\npublic interface SQLServerKeyVaultAuthenticationCallback {\n\n    /**\n     * Returns the acesss token of the authentication request\n     *\n     * @param authority\n     *        - Identifier of the authority, a URL.\n     * @param resource\n     *        - Identifier of the target resource that is the recipient of the requested token, a URL.\n     * @param scope\n     *        - The scope of the authentication request.\n     * @return access token\n     */\n    String getAccessToken(String authority, String resource, String scope);\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/StreamRetValue.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\n/**\n * StreamRetValue represents a TDS return value.\n */\nfinal class StreamRetValue extends StreamPacket {\n    @SuppressWarnings(\"unused\")\n    private String paramName;\n\n    /*\n     * TDS 7.2: Indicates ordinal position of the OUTPUT parameter in the original RPC call TDS 7.1: Indicates the\n     * length of the return value\n     */\n    private int ordinalOrLength;\n\n    final int getOrdinalOrLength() {\n        return ordinalOrLength;\n    }\n\n    /*\n     * Status: 0x01 if the return value is an OUTPUT parameter of a stored procedure 0x02 if the return value is from a\n     * User Defined Function\n     */\n    @SuppressWarnings(\"unused\")\n    private int status;\n\n    StreamRetValue() {\n        super(TDS.TDS_RETURN_VALUE);\n    }\n\n    void setFromTDS(TDSReader tdsReader) throws SQLServerException {\n        if (TDS.TDS_RETURN_VALUE != tdsReader.readUnsignedByte())\n            assert false;\n        ordinalOrLength = tdsReader.readUnsignedShort();\n        paramName = tdsReader.readUnicodeString(tdsReader.readUnsignedByte());\n        status = tdsReader.readUnsignedByte();\n    }\n\n    CryptoMetadata getCryptoMetadata(TDSReader tdsReader) throws SQLServerException {\n        return (new StreamColumns()).readCryptoMetadata(tdsReader);\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDataSource.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.PrintWriter;\nimport java.sql.Connection;\nimport java.sql.SQLException;\nimport java.util.Enumeration;\nimport java.util.Properties;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\nimport javax.naming.Reference;\nimport javax.naming.StringRefAddr;\n\nimport org.ietf.jgss.GSSCredential;\n\n\n/**\n * Contains a list of properties specific for the {@link SQLServerConnection} class.\n */\npublic class SQLServerDataSource\n        implements ISQLServerDataSource, javax.sql.DataSource, java.io.Serializable, javax.naming.Referenceable {\n    // dsLogger is logger used for all SQLServerDataSource instances.\n    static final java.util.logging.Logger dsLogger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.SQLServerDataSource\");\n    static final java.util.logging.Logger loggerExternal = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.DataSource\");\n    static final private java.util.logging.Logger parentLogger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc\");\n\n    static final String TRUSTSTORE_PASSWORD_STRIPPED = \"trustStorePasswordStripped\";\n\n    /** logging class name */\n    final private String loggingClassName;\n\n    /**\n     * trustStorePasswordStripped flag\n     */\n    private boolean trustStorePasswordStripped = false;\n\n    /**\n     * Always refresh SerialVersionUID when prompted\n     */\n    private static final long serialVersionUID = 654861379544314296L;\n\n    /**\n     * Properties passed to SQLServerConnection class\n     */\n    private Properties connectionProps;\n\n    /**\n     * URL for datasource\n     */\n    private String dataSourceURL;\n\n    /**\n     * Description for datasource.\n     */\n    private String dataSourceDescription;\n\n    /**\n     * Unique id generator for each DataSource instance (used for logging).\n     */\n    static private final AtomicInteger baseDataSourceID = new AtomicInteger(0);\n\n    /**\n     * trace ID\n     */\n    final private String traceID;\n\n    /**\n     * Constructs a SQLServerDataSource.\n     */\n    public SQLServerDataSource() {\n        connectionProps = new Properties();\n        traceID = getClass().getSimpleName() + ':' + nextDataSourceID();\n        loggingClassName = \"com.microsoft.sqlserver.jdbc.\" + traceID;\n    }\n\n    String getClassNameLogging() {\n        return loggingClassName;\n    }\n\n    @Override\n    public String toString() {\n        return traceID;\n    }\n\n    // DataSource interface public methods\n\n    @Override\n    public Connection getConnection() throws SQLServerException {\n        loggerExternal.entering(getClassNameLogging(), \"getConnection\");\n        Connection con = getConnectionInternal(null, null, null);\n        loggerExternal.exiting(getClassNameLogging(), \"getConnection\", con);\n        return con;\n    }\n\n    @Override\n    public Connection getConnection(String username, String password) throws SQLServerException {\n        if (loggerExternal.isLoggable(Level.FINER))\n            loggerExternal.entering(getClassNameLogging(), \"getConnection\",\n                    new Object[] {username, \"Password not traced\"});\n        Connection con = getConnectionInternal(username, password, null);\n        loggerExternal.exiting(getClassNameLogging(), \"getConnection\", con);\n        return con;\n    }\n\n    /**\n     * Sets the maximum time in seconds that this data source will wait while attempting to connect to a database. Note\n     * default value is 0.\n     */\n    @Override\n    public void setLoginTimeout(int loginTimeout) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.LOGIN_TIMEOUT.toString(), loginTimeout);\n    }\n\n    @Override\n    public int getLoginTimeout() {\n        int defaultTimeOut = SQLServerDriverIntProperty.LOGIN_TIMEOUT.getDefaultValue();\n        final int logintimeout = getIntProperty(connectionProps, SQLServerDriverIntProperty.LOGIN_TIMEOUT.toString(),\n                defaultTimeOut);\n        // even if the user explicitly sets the timeout to zero, convert to 15\n        return (logintimeout == 0) ? defaultTimeOut : logintimeout;\n    }\n\n    /**\n     * Sets the log writer for this DataSource. Currently we just hold onto this logWriter and pass it back to callers,\n     * nothing else.\n     */\n    private transient PrintWriter logWriter;\n\n    @Override\n    public void setLogWriter(PrintWriter out) {\n        loggerExternal.entering(getClassNameLogging(), \"setLogWriter\", out);\n        logWriter = out;\n        loggerExternal.exiting(getClassNameLogging(), \"setLogWriter\");\n    }\n\n    /**\n     * Returns the log writer for this DataSource.\n     */\n    @Override\n    public PrintWriter getLogWriter() {\n        loggerExternal.entering(getClassNameLogging(), \"getLogWriter\");\n        loggerExternal.exiting(getClassNameLogging(), \"getLogWriter\", logWriter);\n        return logWriter;\n    }\n\n    @Override\n    public Logger getParentLogger() throws java.sql.SQLFeatureNotSupportedException {\n        return parentLogger;\n    }\n\n    // Core Connection property setters/getters.\n\n    /**\n     * Sets the specific application in various SQL Server profiling and logging tools.\n     */\n    @Override\n    public void setApplicationName(String applicationName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_NAME.toString(), applicationName);\n    }\n\n    @Override\n    public String getApplicationName() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_NAME.toString(),\n                SQLServerDriverStringProperty.APPLICATION_NAME.getDefaultValue());\n    }\n\n    /**\n     * Sets the the database to connect to.\n     *\n     * @param databaseName\n     *        if not set, returns the default value of null.\n     */\n    @Override\n    public void setDatabaseName(String databaseName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.DATABASE_NAME.toString(), databaseName);\n    }\n\n    @Override\n    public String getDatabaseName() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.DATABASE_NAME.toString(), null);\n    }\n\n    /**\n     * Sets the the SQL Server instance name to connect to.\n     *\n     * @param instanceName\n     *        if not set, returns the default value of null.\n     */\n    @Override\n    public void setInstanceName(String instanceName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.INSTANCE_NAME.toString(), instanceName);\n    }\n\n    @Override\n    public String getInstanceName() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.INSTANCE_NAME.toString(), null);\n    }\n\n    @Override\n    public void setIntegratedSecurity(boolean enable) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.INTEGRATED_SECURITY.toString(), enable);\n    }\n\n    @Override\n    public void setAuthenticationScheme(String authenticationScheme) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.AUTHENTICATION_SCHEME.toString(),\n                authenticationScheme);\n    }\n\n    @Override\n    public void setAuthentication(String authentication) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.AUTHENTICATION.toString(), authentication);\n    }\n\n    @Override\n    public String getAuthentication() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.AUTHENTICATION.toString(),\n                SQLServerDriverStringProperty.AUTHENTICATION.getDefaultValue());\n    }\n\n    @Override\n    public void setGSSCredentials(GSSCredential userCredential) {\n        setObjectProperty(connectionProps, SQLServerDriverObjectProperty.GSS_CREDENTIAL.toString(), userCredential);\n    }\n\n    @Override\n    public GSSCredential getGSSCredentials() {\n        return (GSSCredential) getObjectProperty(connectionProps,\n                SQLServerDriverObjectProperty.GSS_CREDENTIAL.toString(),\n                SQLServerDriverObjectProperty.GSS_CREDENTIAL.getDefaultValue());\n    }\n\n    @Override\n    public void setUseDefaultGSSCredential(boolean enable) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_DEFAULT_GSS_CREDENTIAL.toString(),\n                enable);\n    }\n\n    @Override\n    public boolean getUseDefaultGSSCredential() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_DEFAULT_GSS_CREDENTIAL.toString(),\n                SQLServerDriverBooleanProperty.USE_DEFAULT_GSS_CREDENTIAL.getDefaultValue());\n    }\n\n    @Override\n    public void setUseFlexibleCallableStatements(boolean enable) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_FLEXIBLE_CALLABLE_STATEMENTS.toString(),\n                enable);\n    }\n\n    @Override\n    public boolean getUseFlexibleCallableStatements() {\n        return getBooleanProperty(connectionProps,\n                SQLServerDriverBooleanProperty.USE_FLEXIBLE_CALLABLE_STATEMENTS.toString(),\n                SQLServerDriverBooleanProperty.USE_FLEXIBLE_CALLABLE_STATEMENTS.getDefaultValue());\n    }\n\n    @Override\n    public void setAccessToken(String accessToken) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.ACCESS_TOKEN.toString(), accessToken);\n    }\n\n    @Override\n    public String getAccessToken() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.ACCESS_TOKEN.toString(), null);\n    }\n\n    /**\n     * Sets the Column Encryption setting. If lastUpdateCount is set to true, the driver will return only the last\n     * update count from all the update counts returned by a batch. The default of false will return all update counts.\n     * If lastUpdateCount is not set, getLastUpdateCount returns the default value of false.\n     */\n    @Override\n    public void setColumnEncryptionSetting(String columnEncryptionSetting) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.COLUMN_ENCRYPTION.toString(),\n                columnEncryptionSetting);\n    }\n\n    @Override\n    public String getColumnEncryptionSetting() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.COLUMN_ENCRYPTION.toString(),\n                SQLServerDriverStringProperty.COLUMN_ENCRYPTION.getDefaultValue());\n    }\n\n    @Override\n    public void setKeyStoreAuthentication(String keyStoreAuthentication) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_AUTHENTICATION.toString(),\n                keyStoreAuthentication);\n    }\n\n    @Override\n    public String getKeyStoreAuthentication() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_AUTHENTICATION.toString(),\n                SQLServerDriverStringProperty.KEY_STORE_AUTHENTICATION.getDefaultValue());\n    }\n\n    @Override\n    public void setKeyStoreSecret(String keyStoreSecret) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_SECRET.toString(), keyStoreSecret);\n    }\n\n    @Override\n    public void setKeyStoreLocation(String keyStoreLocation) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_LOCATION.toString(),\n                keyStoreLocation);\n    }\n\n    @Override\n    public String getKeyStoreLocation() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.KEY_STORE_LOCATION.toString(),\n                SQLServerDriverStringProperty.KEY_STORE_LOCATION.getDefaultValue());\n    }\n\n    @Override\n    public void setLastUpdateCount(boolean lastUpdateCount) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.LAST_UPDATE_COUNT.toString(),\n                lastUpdateCount);\n    }\n\n    @Override\n    public boolean getLastUpdateCount() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.LAST_UPDATE_COUNT.toString(),\n                SQLServerDriverBooleanProperty.LAST_UPDATE_COUNT.getDefaultValue());\n    }\n\n    @Override\n    public void setEncrypt(String encryptOption) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.ENCRYPT.toString(), encryptOption);\n    }\n\n    /**\n     * @deprecated\n     */\n    @Override\n    @Deprecated(since = \"10.1.0\", forRemoval = true)\n    public void setEncrypt(boolean encryptOption) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.ENCRYPT.toString(),\n                Boolean.toString(encryptOption));\n    }\n\n    @Override\n    public String getEncrypt() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.ENCRYPT.toString(),\n                SQLServerDriverStringProperty.ENCRYPT.getDefaultValue());\n    }\n\n    @Override\n    public void setServerCertificate(String cert) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_CERTIFICATE.toString(), cert);\n    }\n\n    @Override\n    public String getServerCertificate() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_CERTIFICATE.toString(), null);\n    }\n\n    @Override\n    public void setTransparentNetworkIPResolution(boolean tnir) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.TRANSPARENT_NETWORK_IP_RESOLUTION.toString(),\n                tnir);\n    }\n\n    @Override\n    public boolean getTransparentNetworkIPResolution() {\n        return getBooleanProperty(connectionProps,\n                SQLServerDriverBooleanProperty.TRANSPARENT_NETWORK_IP_RESOLUTION.toString(),\n                SQLServerDriverBooleanProperty.TRANSPARENT_NETWORK_IP_RESOLUTION.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustServerCertificate(boolean e) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.TRUST_SERVER_CERTIFICATE.toString(), e);\n    }\n\n    @Override\n    public boolean getTrustServerCertificate() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.TRUST_SERVER_CERTIFICATE.toString(),\n                SQLServerDriverBooleanProperty.TRUST_SERVER_CERTIFICATE.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustStoreType(String trustStoreType) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_TYPE.toString(), trustStoreType);\n    }\n\n    @Override\n    public String getTrustStoreType() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_TYPE.toString(),\n                SQLServerDriverStringProperty.TRUST_STORE_TYPE.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustStore(String trustStore) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE.toString(), trustStore);\n    }\n\n    @Override\n    public String getTrustStore() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE.toString(), null);\n    }\n\n    @Override\n    public void setTrustStorePassword(String trustStorePassword) {\n        // if a non value property is set\n        if (trustStorePassword != null)\n            trustStorePasswordStripped = false;\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_PASSWORD.toString(),\n                trustStorePassword);\n    }\n\n    \nString getTrustStorePassword() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_PASSWORD.toString(), null);\n    }\n\n    @Override\n    public void setHostNameInCertificate(String hostName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.HOSTNAME_IN_CERTIFICATE.toString(), hostName);\n    }\n\n    @Override\n    public String getHostNameInCertificate() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.HOSTNAME_IN_CERTIFICATE.toString(),\n                null);\n    }\n\n    /**\n     * Sets the lock timeout value.\n     *\n     * @param lockTimeout\n     *        the number of milliseconds to wait before the database reports a lock timeout. The default value of -1\n     *        means wait forever. If specified, this value will be the default for all statements on the connection.\n     *        Note a value of 0 means no wait. If lockTimeout is not set, getLockTimeout returns the default of -1.\n     */\n    @Override\n    public void setLockTimeout(int lockTimeout) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.LOCK_TIMEOUT.toString(), lockTimeout);\n    }\n\n    @Override\n    public int getLockTimeout() {\n        return getIntProperty(connectionProps, SQLServerDriverIntProperty.LOCK_TIMEOUT.toString(),\n                SQLServerDriverIntProperty.LOCK_TIMEOUT.getDefaultValue());\n    }\n\n    /**\n     * Sets the password that will be used when connecting to SQL Server.\n     *\n     * @param password\n     *        Note getPassword is deliberately declared non-public for security reasons. If the password is not set,\n     *        getPassword returns the default value of null.\n     */\n    @Override\n    public void setPassword(String password) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.PASSWORD.toString(), password);\n    }\n\n    String getPassword() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.PASSWORD.toString(), null);\n    }\n\n    /**\n     * Sets the TCP-IP port number used when opening a socket connection to SQL Server.\n     *\n     * @param portNumber\n     *        if not set, getPortNumber returns the default of 1433. Note as mentioned above, setPortNumber does not do\n     *        any range checking on the port value passed in,\\ invalid port numbers like 99999 can be passed in without\n     *        triggering any error.\n     */\n    @Override\n    public void setPortNumber(int portNumber) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.PORT_NUMBER.toString(), portNumber);\n    }\n\n    @Override\n    public int getPortNumber() {\n        return getIntProperty(connectionProps, SQLServerDriverIntProperty.PORT_NUMBER.toString(),\n                SQLServerDriverIntProperty.PORT_NUMBER.getDefaultValue());\n    }\n\n    /**\n     * Sets the default cursor type used for the result set.\n     *\n     * @param selectMethod\n     *        This(non-Javadoc) @see com.microsoft.sqlserver.jdbc.ISQLServerDataSource#setSelectMethod(java.lang.String)\n     *        property is useful when you are dealing with large result sets and do not want to store the whole result\n     *        set in memory on the client side. By setting the property to \"cursor\" you will be able to create a server\n     *        side cursor that can fetch smaller chunks of data at a time. If selectMethod is not set, getSelectMethod\n     *        returns the default value of \"direct\".\n     */\n    @Override\n    public void setSelectMethod(String selectMethod) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SELECT_METHOD.toString(), selectMethod);\n    }\n\n    @Override\n    public String getSelectMethod() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SELECT_METHOD.toString(),\n                SQLServerDriverStringProperty.SELECT_METHOD.getDefaultValue());\n    }\n\n    @Override\n    public void setResponseBuffering(String bufferingMode) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.RESPONSE_BUFFERING.toString(), bufferingMode);\n    }\n\n    @Override\n    public String getResponseBuffering() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.RESPONSE_BUFFERING.toString(),\n                SQLServerDriverStringProperty.RESPONSE_BUFFERING.getDefaultValue());\n    }\n\n    @Override\n    public void setApplicationIntent(String applicationIntent) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_INTENT.toString(),\n                applicationIntent);\n    }\n\n    @Override\n    public String getApplicationIntent() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.APPLICATION_INTENT.toString(),\n                SQLServerDriverStringProperty.APPLICATION_INTENT.getDefaultValue());\n    }\n\n    @Override\n    public void setReplication(boolean replication) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.REPLICATION.toString(), replication);\n    }\n\n    @Override\n    public boolean getReplication() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.REPLICATION.toString(),\n                SQLServerDriverBooleanProperty.REPLICATION.getDefaultValue());\n    }\n\n    @Override\n    public void setSendTimeAsDatetime(boolean sendTimeAsDatetime) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.SEND_TIME_AS_DATETIME.toString(),\n                sendTimeAsDatetime);\n    }\n\n    @Override\n    public boolean getSendTimeAsDatetime() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.SEND_TIME_AS_DATETIME.toString(),\n                SQLServerDriverBooleanProperty.SEND_TIME_AS_DATETIME.getDefaultValue());\n    }\n\n    @Override\n    public void setDatetimeParameterType(String datetimeParameterType) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.DATETIME_DATATYPE.toString(),\n                datetimeParameterType);\n    }\n\n    @Override\n    public String getDatetimeParameterType() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.DATETIME_DATATYPE.toString(),\n                SQLServerDriverStringProperty.DATETIME_DATATYPE.getDefaultValue());\n    }\n\n    @Override\n    public void setUseFmtOnly(boolean useFmtOnly) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_FMT_ONLY.toString(), useFmtOnly);\n    }\n\n    @Override\n    public boolean getUseFmtOnly() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.USE_FMT_ONLY.toString(),\n                SQLServerDriverBooleanProperty.USE_FMT_ONLY.getDefaultValue());\n    }\n\n    @Override\n    public void setDelayLoadingLobs(boolean delayLoadingLobs) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.DELAY_LOADING_LOBS.toString(),\n                delayLoadingLobs);\n    }\n\n    @Override\n    public boolean getDelayLoadingLobs() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.DELAY_LOADING_LOBS.toString(),\n                SQLServerDriverBooleanProperty.DELAY_LOADING_LOBS.getDefaultValue());\n    }\n\n    /**\n     * Sets whether string parameters are sent to the server in UNICODE format.\n     *\n     * @param sendStringParametersAsUnicode\n     *        if true (default), string parameters are sent to the server in UNICODE format. if false, string parameters\n     *        are sent to the server in the native TDS collation format of the database, not in UNICODE. if set, returns\n     *        the default of true.\n     */\n    @Override\n    public void setSendStringParametersAsUnicode(boolean sendStringParametersAsUnicode) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.SEND_STRING_PARAMETERS_AS_UNICODE.toString(),\n                sendStringParametersAsUnicode);\n    }\n\n    @Override\n    public boolean getSendStringParametersAsUnicode() {\n        return getBooleanProperty(connectionProps,\n                SQLServerDriverBooleanProperty.SEND_STRING_PARAMETERS_AS_UNICODE.toString(),\n                SQLServerDriverBooleanProperty.SEND_STRING_PARAMETERS_AS_UNICODE.getDefaultValue());\n    }\n\n    @Override\n    public void setServerNameAsACE(boolean serverNameAsACE) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.SERVER_NAME_AS_ACE.toString(),\n                serverNameAsACE);\n    }\n\n    @Override\n    public boolean getServerNameAsACE() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.SERVER_NAME_AS_ACE.toString(),\n                SQLServerDriverBooleanProperty.SERVER_NAME_AS_ACE.getDefaultValue());\n    }\n\n    /**\n     * Sets the host name of the target SQL Server.\n     *\n     * @param serverName\n     *        if not set, returns the default value of null is returned.\n     */\n    @Override\n    public void setServerName(String serverName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_NAME.toString(), serverName);\n    }\n\n    @Override\n    public String getServerName() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_NAME.toString(), null);\n    }\n\n    /**\n     * Set the preferred type of IP Address\n     * \n     * @param iPAddressPreference\n     *        Preferred IP Address type\n     */\n    @Override\n    public void setIPAddressPreference(String iPAddressPreference) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.IPADDRESS_PREFERENCE.toString(),\n                iPAddressPreference);\n    }\n\n    /**\n     * Gets the preferred type of IP Address\n     */\n    @Override\n    public String getIPAddressPreference() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.IPADDRESS_PREFERENCE.toString(),\n                SQLServerDriverStringProperty.IPADDRESS_PREFERENCE.getDefaultValue());\n    }\n\n    /**\n     * Sets the realm for Kerberos authentication.\n     *\n     * @param realm\n     *        realm\n     */\n    @Override\n    public void setRealm(String realm) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.REALM.toString(), realm);\n    }\n\n    @Override\n    public String getRealm() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.REALM.toString(), null);\n    }\n\n    /**\n     * Sets the Service Principal Name (SPN) of the target SQL Server.\n     * https://msdn.microsoft.com/en-us/library/cc280459.aspx\n     *\n     * @param serverSpn\n     *        service principal name\n     */\n    @Override\n    public void setServerSpn(String serverSpn) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_SPN.toString(), serverSpn);\n    }\n\n    @Override\n    public String getServerSpn() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SERVER_SPN.toString(), null);\n    }\n\n    /**\n     * Sets the fail over partner of the target SQL Server.\n     *\n     * @param serverName\n     *        if not set, returns the default value of null.\n     */\n    @Override\n    public void setFailoverPartner(String serverName) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.FAILOVER_PARTNER.toString(), serverName);\n    }\n\n    @Override\n    public String getFailoverPartner() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.FAILOVER_PARTNER.toString(), null);\n    }\n\n    @Override\n    public void setMultiSubnetFailover(boolean multiSubnetFailover) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.MULTI_SUBNET_FAILOVER.toString(),\n                multiSubnetFailover);\n    }\n\n    @Override\n    public boolean getMultiSubnetFailover() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.MULTI_SUBNET_FAILOVER.toString(),\n                SQLServerDriverBooleanProperty.MULTI_SUBNET_FAILOVER.getDefaultValue());\n    }\n\n    /**\n     * Sets the user name that will be used when connecting to SQL Server.\n     *\n     * @param user\n     *        if not set, returns the default value of null.\n     */\n    @Override\n    public void setUser(String user) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.USER.toString(), user);\n    }\n\n    @Override\n    public String getUser() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.USER.toString(), null);\n    }\n\n    /**\n     * Sets the name of the client machine (or client workstation).\n     *\n     * @param workstationID\n     *        host name of the client. if not set, the default value is constructed by calling\n     *        InetAddress.getLocalHost().getHostName() or if getHostName() returns blank then\n     *        getHostAddress().toString().\n     */\n    @Override\n    public void setWorkstationID(String workstationID) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.WORKSTATION_ID.toString(), workstationID);\n    }\n\n    @Override\n    public String getWorkstationID() {\n        if (loggerExternal.isLoggable(java.util.logging.Level.FINER))\n            loggerExternal.entering(getClassNameLogging(), \"getWorkstationID\");\n        String getWSID = connectionProps.getProperty(SQLServerDriverStringProperty.WORKSTATION_ID.toString());\n        // Per spec, return what the logon will send here if workstationID\n        // property is not set.\n        if (null == getWSID) {\n            getWSID = Util.lookupHostName();\n        }\n        loggerExternal.exiting(getClassNameLogging(), \"getWorkstationID\", getWSID);\n        return getWSID;\n    }\n\n    /**\n     * Sets whether the driver will convert SQL states to XOPEN compliant states.\n     *\n     * @param xopenStates\n     *        if true, the driver will convert SQL states to XOPEN compliant states. The default is false which causes\n     *        the driver to generate SQL 99 state codes. If not set, getXopenStates returns the default value of false.\n     */\n    @Override\n    public void setXopenStates(boolean xopenStates) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.XOPEN_STATES.toString(), xopenStates);\n    }\n\n    @Override\n    public boolean getXopenStates() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.XOPEN_STATES.toString(),\n                SQLServerDriverBooleanProperty.XOPEN_STATES.getDefaultValue());\n    }\n\n    @Override\n    public void setFIPS(boolean fips) {\n        setBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.FIPS.toString(), fips);\n    }\n\n    @Override\n    public boolean getFIPS() {\n        return getBooleanProperty(connectionProps, SQLServerDriverBooleanProperty.FIPS.toString(),\n                SQLServerDriverBooleanProperty.FIPS.getDefaultValue());\n    }\n\n    @Override\n    public String getSocketFactoryClass() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SOCKET_FACTORY_CLASS.toString(), null);\n    }\n\n    @Override\n    public void setSocketFactoryClass(String socketFactoryClass) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SOCKET_FACTORY_CLASS.toString(),\n                socketFactoryClass);\n    }\n\n    @Override\n    public String getSocketFactoryConstructorArg() {\n        return getStringProperty(connectionProps,\n                SQLServerDriverStringProperty.SOCKET_FACTORY_CONSTRUCTOR_ARG.toString(), null);\n    }\n\n    @Override\n    public void setSocketFactoryConstructorArg(String socketFactoryConstructorArg) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SOCKET_FACTORY_CONSTRUCTOR_ARG.toString(),\n                socketFactoryConstructorArg);\n    }\n\n    @Override\n    public void setSSLProtocol(String sslProtocol) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.SSL_PROTOCOL.toString(), sslProtocol);\n    }\n\n    @Override\n    public String getSSLProtocol() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.SSL_PROTOCOL.toString(),\n                SQLServerDriverStringProperty.SSL_PROTOCOL.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustManagerClass(String trustManagerClass) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_MANAGER_CLASS.toString(),\n                trustManagerClass);\n    }\n\n    @Override\n    public String getTrustManagerClass() {\n        return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_MANAGER_CLASS.toString(),\n                SQLServerDriverStringProperty.TRUST_MANAGER_CLASS.getDefaultValue());\n    }\n\n    @Override\n    public void setTrustManagerConstructorArg(String trustManagerConstructorArg) {\n        setStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_MANAGER_CONSTRUCTOR_ARG.toString(),\n                trustManagerConstructorArg);\n    }\n\n    @Override\n    public String getTrustManagerConstructorArg() {\n        return getStringProperty(connectionProps,\n                SQLServerDriverStringProperty.TRUST_MANAGER_CONSTRUCTOR_ARG.toString(),\n                SQLServerDriverStringProperty.TRUST_MANAGER_CONSTRUCTOR_ARG.getDefaultValue());\n    }\n\n    /**\n     * Sets the datasource URL.\n     *\n     * @param url\n     *        The URL property is exposed for backwards compatibility reasons. Also, several Java Application servers\n     *        expect a setURL function on the DataSource and set it by default (JBoss and WebLogic) Note for security\n     *        reasons we do not recommend that customers include the password in the url supplied to setURL. The reason\n     *        for this is third-party Java Application Servers will very often display the value set to URL property in\n     *        their DataSource configuration GUI. We recommend instead that clients use the setPassword method to set\n     *        the password value. The Java Application Servers will not display a password that is set on the DataSource\n     *        in the configuration GUI. Note if setURL is not called, getURL returns the default value of\n     *        \"jdbc:sqlserver://\".\n     */\n    @Override\n    public void setURL(String url) {\n        loggerExternal.entering(getClassNameLogging(), \"setURL\", url);\n        // URL is not stored in a property set, it is maintained separately.\n        dataSourceURL = url;\n        loggerExternal.exiting(getClassNameLogging(), \"setURL\");\n    }\n\n    @Override\n    public String getURL() {\n        String url = dataSourceURL;\n        loggerExternal.entering(getClassNameLogging(), \"getURL\");\n\n        if (null == dataSourceURL)\n            url = \"jdbc:sqlserver://\";\n        loggerExternal.exiting(getClassNameLogging(), \"getURL\", url);\n        return url;\n    }\n\n    /**\n     * Sets the DataSource description. Per JDBC specification 16.1.1 \"...the only property that all DataSource\n     * implementations are required to support is the description property\".\n     */\n    @Override\n    public void setDescription(String description) {\n        loggerExternal.entering(getClassNameLogging(), \"setDescription\", description);\n        dataSourceDescription = description;\n        loggerExternal.exiting(getClassNameLogging(), \"setDescription\");\n    }\n\n    /**\n     * Returns the DataSource description\n     */\n    @Override\n    public String getDescription() {\n        loggerExternal.entering(getClassNameLogging(), \"getDescription\");\n        loggerExternal.exiting(getClassNameLogging(), \"getDescription\", dataSourceDescription);\n        return dataSourceDescription;\n    }\n\n    /**\n     * Sets the packet size.\n     *\n     * @param packetSize\n     *        the size (in bytes) to use for the TCP/IP send and receive buffer. It is also the value used for the TDS\n     *        packet size (SQL Server Network Packet Size). Validity of the value is checked at connect time. If no\n     *        value is set for this property, its default value is 4KB.\n     */\n    @Override\n    public void setPacketSize(int packetSize) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.PACKET_SIZE.toString(), packetSize);\n    }\n\n    @Override\n    public int getPacketSize() {\n        return getIntProperty(connectionProps, SQLServerDriverIntProperty.PACKET_SIZE.toString(),\n                SQLServerDriverIntProperty.PACKET_SIZE.getDefaultValue());\n    }\n\n    @Override\n    public void setQueryTimeout(int queryTimeout) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.QUERY_TIMEOUT.toString(), queryTimeout);\n    }\n\n    @Override\n    public int getQueryTimeout() {\n        return getIntProperty(connectionProps, SQLServerDriverIntProperty.QUERY_TIMEOUT.toString(),\n                SQLServerDriverIntProperty.QUERY_TIMEOUT.getDefaultValue());\n    }\n\n    @Override\n    public void setCancelQueryTimeout(int cancelQueryTimeout) {\n        setIntProperty(connectionProps, SQLServerDriverIntProperty.CANCEL_QUERY_TIMEOUT.toString(), cancelQueryTimeout);\n    }\n\n    @Override\n    public int getCancelQueryTimeout() {\n        return getIntProperty(connectionProps, SQLServerDriverIntProperty.CANCEL_QUERY_TIMEOUT.toString(),\n                SQLServerDriverIntProperty.CANCEL_QUERY_TIMEOUT.getDefaultValue());\n    }\n\n    @Override\n    public void setEnablePrepareOnFirstPreparedStatementCall(boolean enablePrepareOnFirstPreparedStatementCall) {\n        setBooleanProperty(connectionProps,\n                SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.toString(),\n                enablePrepareOnFirstPreparedStatementCall);\n    }\n\n    @Override\n    public boolean getEnablePrepareOnFirstPreparedStatementCall() {\n        boolean defaultValue = SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT\n                .getDefaultValue();\n        return getBooleanProperty(connectionProps,\n                SQLServerDriverBooleanProperty.ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT.toString(), defaultValue);\n    }\n\n    @Override\n    public void setServerPreparedStatementDiscardThreshold(int serverPreparedStatementDiscardThreshold) {\n...\n", "needle_token_start": 10392, "needle_token_end": 10433, "code_context_ntokens": 16069, "output": ["```java\n@Override\npublic String getTrustStorePassword() {\n    return getStringProperty(connectionProps, SQLServerDriverStringProperty.TRUST_STORE_PASSWORD.toString(), null);\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "getRandomNetAddress", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To generate a 6-byte random array that serves as a unique client identifier, mimicking a MAC address, for use in network communications. This is used to avoid personal identifiable information (PII) issues.\n2. **Input**: None.\n3. **Output**: A 6-byte array of random numbers.\n4. **Procedure**: The function creates a byte array of size 6, then fills this array with random bytes generated by an instance of the `Random` class.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/mssql/googlecode/concurrentlinkedhashmap/ConcurrentLinkedHashMap.java\n/*\n * Copyright 2010 Google Inc. All Rights Reserved.\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * http://www.apache.org/licenses/LICENSE-2.0\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage mssql.googlecode.concurrentlinkedhashmap;\n\nimport static java.util.Collections.emptyList;\nimport static java.util.Collections.unmodifiableMap;\nimport static java.util.Collections.unmodifiableSet;\nimport static mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.IDLE;\nimport static mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.PROCESSING;\nimport static mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.DrainStatus.REQUIRED;\n\nimport java.io.InvalidObjectException;\nimport java.io.ObjectInputStream;\nimport java.io.Serializable;\nimport java.util.AbstractCollection;\nimport java.util.AbstractMap;\nimport java.util.AbstractQueue;\nimport java.util.AbstractSet;\nimport java.util.Collection;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.LinkedHashMap;\nimport java.util.LinkedHashSet;\nimport java.util.Map;\nimport java.util.Queue;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentLinkedQueue;\nimport java.util.concurrent.ConcurrentMap;\nimport java.util.concurrent.atomic.AtomicLong;\nimport java.util.concurrent.atomic.AtomicReference;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\n\n\n/**\n * A hash table supporting full concurrency of retrievals, adjustable expected\n * concurrency for updates, and a maximum capacity to bound the map by. This\n * implementation differs from {@link ConcurrentHashMap} in that it maintains a\n * page replacement algorithm that is used to evict an entry when the map has\n * exceeded its capacity. Unlike the <code>Java Collections Framework</code>, this\n * map does not have a publicly visible constructor and instances are created\n * through a {@link Builder}.\n * <p>\n * An entry is evicted from the map when the <code>weighted capacity</code> exceeds\n * its <code>maximum weighted capacity</code> threshold. A {@link EntryWeigher}\n * determines how many units of capacity that an entry consumes. The default\n * weigher assigns each value a weight of <code>1</code> to bound the map by the\n * total number of key-value pairs. A map that holds collections may choose to\n * weigh values by the number of elements in the collection and bound the map\n * by the total number of elements that it contains. A change to a value that\n * modifies its weight requires that an update operation is performed on the\n * map.\n * <p>\n * An {@link EvictionListener} may be supplied for notification when an entry\n * is evicted from the map. This listener is invoked on a caller's thread and\n * will not block other threads from operating on the map. An implementation\n * should be aware that the caller's thread will not expect long execution\n * times or failures as a side effect of the listener being notified. Execution\n * safety and a fast turn around time can be achieved by performing the\n * operation asynchronously, such as by submitting a task to an\n * {@link java.util.concurrent.ExecutorService}.\n * <p>\n * The <code>concurrency level</code> determines the number of threads that can\n * concurrently modify the table. Using a significantly higher or lower value\n * than needed can waste space or lead to thread contention, but an estimate\n * within an order of magnitude of the ideal value does not usually have a\n * noticeable impact. Because placement in hash tables is essentially random,\n * the actual concurrency will vary.\n * <p>\n * This class and its views and iterators implement all of the\n * <em>optional</em> methods of the {@link Map} and {@link Iterator}\n * interfaces.\n * <p>\n * Like {@link java.util.Hashtable} but unlike {@link HashMap}, this class\n * does <em>not</em> allow <code>null</code> to be used as a key or value. Unlike\n * {@link java.util.LinkedHashMap}, this class does <em>not</em> provide\n * predictable iteration order. A snapshot of the keys and entries may be\n * obtained in ascending and descending order of retention.\n *\n * @author ben.manes@gmail.com (Ben Manes)\n * @param <K>\n *        the type of keys maintained by this map\n * @param <V>\n *        the type of mapped values\n * @see <a href=\"http://code.google.com/p/concurrentlinkedhashmap/\">\n *      http://code.google.com/p/concurrentlinkedhashmap/</a>\n */\n@SuppressWarnings(\"deprecation\")\npublic final class ConcurrentLinkedHashMap<K, V> extends AbstractMap<K, V>\n        implements ConcurrentMap<K, V>, Serializable {\n\n    /*\n     * This class performs a best-effort bounding of a ConcurrentHashMap using a\n     * page-replacement algorithm to determine which entries to evict when the\n     * capacity is exceeded.\n     * The page replacement algorithm's data structures are kept eventually\n     * consistent with the map. An update to the map and recording of reads may\n     * not be immediately reflected on the algorithm's data structures. These\n     * structures are guarded by a lock and operations are applied in batches to\n     * avoid lock contention. The penalty of applying the batches is spread across\n     * threads so that the amortized cost is slightly higher than performing just\n     * the ConcurrentHashMap operation.\n     * A memento of the reads and writes that were performed on the map are\n     * recorded in buffers. These buffers are drained at the first opportunity\n     * after a write or when the read buffer exceeds a threshold size. The reads\n     * are recorded in a lossy buffer, allowing the reordering operations to be\n     * discarded if the draining process cannot keep up. Due to the concurrent\n     * nature of the read and write operations a strict policy ordering is not\n     * possible, but is observably strict when single threaded.\n     * Due to a lack of a strict ordering guarantee, a task can be executed\n     * out-of-order, such as a removal followed by its addition. The state of the\n     * entry is encoded within the value's weight.\n     * Alive: The entry is in both the hash-table and the page replacement policy.\n     * This is represented by a positive weight.\n     * Retired: The entry is not in the hash-table and is pending removal from the\n     * page replacement policy. This is represented by a negative weight.\n     * Dead: The entry is not in the hash-table and is not in the page replacement\n     * policy. This is represented by a weight of zero.\n     * The Least Recently Used page replacement algorithm was chosen due to its\n     * simplicity, high hit rate, and ability to be implemented with O(1) time\n     * complexity.\n     */\n\n    /** The number of CPUs */\n    static final int NCPU = Runtime.getRuntime().availableProcessors();\n\n    /** The maximum weighted capacity of the map. */\n    static final long MAXIMUM_CAPACITY = Long.MAX_VALUE - Integer.MAX_VALUE;\n\n    /** The number of read buffers to use. */\n    static final int NUMBER_OF_READ_BUFFERS = ceilingNextPowerOfTwo(NCPU);\n\n    /** Mask value for indexing into the read buffers. */\n    static final int READ_BUFFERS_MASK = NUMBER_OF_READ_BUFFERS - 1;\n\n    /** The number of pending read operations before attempting to drain. */\n    static final int READ_BUFFER_THRESHOLD = 32;\n\n    /** The maximum number of read operations to perform per amortized drain. */\n    static final int READ_BUFFER_DRAIN_THRESHOLD = 2 * READ_BUFFER_THRESHOLD;\n\n    /** The maximum number of pending reads per buffer. */\n    static final int READ_BUFFER_SIZE = 2 * READ_BUFFER_DRAIN_THRESHOLD;\n\n    /** Mask value for indexing into the read buffer. */\n    static final int READ_BUFFER_INDEX_MASK = READ_BUFFER_SIZE - 1;\n\n    /** The maximum number of write operations to perform per amortized drain. */\n    static final int WRITE_BUFFER_DRAIN_THRESHOLD = 16;\n\n    /** A queue that discards all entries. */\n    static final Queue<?> DISCARDING_QUEUE = new DiscardingQueue();\n\n    static int ceilingNextPowerOfTwo(int x) {\n        // From Hacker's Delight, Chapter 3, Harry S. Warren Jr.\n        return 1 << (Integer.SIZE - Integer.numberOfLeadingZeros(x - 1));\n    }\n\n    // The backing data store holding the key-value associations\n    final ConcurrentMap<K, Node<K, V>> data;\n    final int concurrencyLevel;\n\n    // These fields provide support to bound the map by a maximum capacity\n    final long[] readBufferReadCount;\n    final LinkedDeque<Node<K, V>> evictionDeque;\n\n    final AtomicLong weightedSize;\n    final AtomicLong capacity;\n\n    final Lock evictionLock;\n    final Queue<Runnable> writeBuffer;\n    final AtomicLong[] readBufferWriteCount;\n    final AtomicLong[] readBufferDrainAtWriteCount;\n    final AtomicReference<Node<K, V>>[][] readBuffers;\n\n    final AtomicReference<DrainStatus> drainStatus;\n    final EntryWeigher<? super K, ? super V> weigher;\n\n    // These fields provide support for notifying a listener.\n    final Queue<Node<K, V>> pendingNotifications;\n    final EvictionListener<K, V> listener;\n\n    transient Set<K> keySet;\n    transient Collection<V> values;\n    transient Set<Entry<K, V>> entrySet;\n\n    /**\n     * Creates an instance based on the builder's configuration.\n     */\n    @SuppressWarnings({\"unchecked\", \"cast\"})\n    private ConcurrentLinkedHashMap(Builder<K, V> builder) {\n        // The data store and its maximum capacity\n        concurrencyLevel = builder.concurrencyLevel;\n        capacity = new AtomicLong(Math.min(builder.capacity, MAXIMUM_CAPACITY));\n        data = new ConcurrentHashMap<K, Node<K, V>>(builder.initialCapacity, 0.75f, concurrencyLevel);\n\n        // The eviction support\n        weigher = builder.weigher;\n        evictionLock = new ReentrantLock();\n        weightedSize = new AtomicLong();\n        evictionDeque = new LinkedDeque<Node<K, V>>();\n        writeBuffer = new ConcurrentLinkedQueue<Runnable>();\n        drainStatus = new AtomicReference<DrainStatus>(IDLE);\n\n        readBufferReadCount = new long[NUMBER_OF_READ_BUFFERS];\n        readBufferWriteCount = new AtomicLong[NUMBER_OF_READ_BUFFERS];\n        readBufferDrainAtWriteCount = new AtomicLong[NUMBER_OF_READ_BUFFERS];\n        readBuffers = new AtomicReference[NUMBER_OF_READ_BUFFERS][READ_BUFFER_SIZE];\n        for (int i = 0; i < NUMBER_OF_READ_BUFFERS; i++) {\n            readBufferWriteCount[i] = new AtomicLong();\n            readBufferDrainAtWriteCount[i] = new AtomicLong();\n            readBuffers[i] = new AtomicReference[READ_BUFFER_SIZE];\n            for (int j = 0; j < READ_BUFFER_SIZE; j++) {\n                readBuffers[i][j] = new AtomicReference<Node<K, V>>();\n            }\n        }\n\n        // The notification queue and listener\n        listener = builder.listener;\n        pendingNotifications = (listener == DiscardingListener.INSTANCE) ? (Queue<Node<K, V>>) DISCARDING_QUEUE\n                                                                         : new ConcurrentLinkedQueue<Node<K, V>>();\n    }\n\n    /** Ensures that the object is not null. */\n    static void checkNotNull(Object o) {\n        if (o == null) {\n            throw new NullPointerException();\n        }\n    }\n\n    /** Ensures that the argument expression is true. */\n    static void checkArgument(boolean expression) {\n        if (!expression) {\n            throw new IllegalArgumentException();\n        }\n    }\n\n    /** Ensures that the state expression is true. */\n    static void checkState(boolean expression) {\n        if (!expression) {\n            throw new IllegalStateException();\n        }\n    }\n\n    /* ---------------- Eviction Support -------------- */\n\n    /**\n     * Retrieves the maximum weighted capacity of the map.\n     *\n     * @return the maximum weighted capacity\n     */\n    public long capacity() {\n        return capacity.get();\n    }\n\n    /**\n     * Sets the maximum weighted capacity of the map and eagerly evicts entries\n     * until it shrinks to the appropriate size.\n     *\n     * @param capacity\n     *        the maximum weighted capacity of the map\n     * @throws IllegalArgumentException\n     *         if the capacity is negative\n     */\n    public void setCapacity(long capacity) {\n        checkArgument(capacity >= 0);\n        evictionLock.lock();\n        try {\n            this.capacity.lazySet(Math.min(capacity, MAXIMUM_CAPACITY));\n            drainBuffers();\n            evict();\n        } finally {\n            evictionLock.unlock();\n        }\n        notifyListener();\n    }\n\n    /** Determines whether the map has exceeded its capacity. */\n    boolean hasOverflowed() {\n        return weightedSize.get() > capacity.get();\n    }\n\n    /**\n     * Evicts entries from the map while it exceeds the capacity and appends\n     * evicted entries to the notification queue for processing.\n     */\n    void evict() {\n        // Attempts to evict entries from the map if it exceeds the maximum\n        // capacity. If the eviction fails due to a concurrent removal of the\n        // victim, that removal may cancel out the addition that triggered this\n        // eviction. The victim is eagerly unlinked before the removal task so\n        // that if an eviction is still required then a new victim will be chosen\n        // for removal.\n        while (hasOverflowed()) {\n            final Node<K, V> node = evictionDeque.poll();\n\n            // If weighted values are used, then the pending operations will adjust\n            // the size to reflect the correct weight\n            if (node == null) {\n                return;\n            }\n\n            // Notify the listener only if the entry was evicted\n            if (data.remove(node.key, node)) {\n                pendingNotifications.add(node);\n            }\n\n            makeDead(node);\n        }\n    }\n\n    /**\n     * Performs the post-processing work required after a read.\n     *\n     * @param node\n     *        the entry in the page replacement policy\n     */\n    void afterRead(Node<K, V> node) {\n        final int bufferIndex = readBufferIndex();\n        final long writeCount = recordRead(bufferIndex, node);\n        drainOnReadIfNeeded(bufferIndex, writeCount);\n        notifyListener();\n    }\n\n    /** Returns the index to the read buffer to record into. */\n    static int readBufferIndex() {\n        // A buffer is chosen by the thread's id so that tasks are distributed in a\n        // pseudo evenly manner. This helps avoid hot entries causing contention\n        // due to other threads trying to append to the same buffer.\n        return ((int) Thread.currentThread().getId()) & READ_BUFFERS_MASK;\n    }\n\n    /**\n     * Records a read in the buffer and return its write count.\n     *\n     * @param bufferIndex\n     *        the index to the chosen read buffer\n     * @param node\n     *        the entry in the page replacement policy\n     * @return the number of writes on the chosen read buffer\n     */\n    long recordRead(int bufferIndex, Node<K, V> node) {\n        // The location in the buffer is chosen in a racy fashion as the increment\n        // is not atomic with the insertion. This means that concurrent reads can\n        // overlap and overwrite one another, resulting in a lossy buffer.\n        final AtomicLong counter = readBufferWriteCount[bufferIndex];\n        final long writeCount = counter.get();\n        counter.lazySet(writeCount + 1);\n\n        final int index = (int) (writeCount & READ_BUFFER_INDEX_MASK);\n        readBuffers[bufferIndex][index].lazySet(node);\n\n        return writeCount;\n    }\n\n    /**\n     * Attempts to drain the buffers if it is determined to be needed when\n     * post-processing a read.\n     *\n     * @param bufferIndex\n     *        the index to the chosen read buffer\n     * @param writeCount\n     *        the number of writes on the chosen read buffer\n     */\n    void drainOnReadIfNeeded(int bufferIndex, long writeCount) {\n        final long pending = (writeCount - readBufferDrainAtWriteCount[bufferIndex].get());\n        final boolean delayable = (pending < READ_BUFFER_THRESHOLD);\n        final DrainStatus status = drainStatus.get();\n        if (status.shouldDrainBuffers(delayable)) {\n            tryToDrainBuffers();\n        }\n    }\n\n    /**\n     * Performs the post-processing work required after a write.\n     *\n     * @param task\n     *        the pending operation to be applied\n     */\n    void afterWrite(Runnable task) {\n        writeBuffer.add(task);\n        drainStatus.lazySet(REQUIRED);\n        tryToDrainBuffers();\n        notifyListener();\n    }\n\n    /**\n     * Attempts to acquire the eviction lock and apply the pending operations, up\n     * to the amortized threshold, to the page replacement policy.\n     */\n    void tryToDrainBuffers() {\n        if (evictionLock.tryLock()) {\n            try {\n                drainStatus.lazySet(PROCESSING);\n                drainBuffers();\n            } finally {\n                drainStatus.compareAndSet(PROCESSING, IDLE);\n                evictionLock.unlock();\n            }\n        }\n    }\n\n    /** Drains the read and write buffers up to an amortized threshold. */\n    void drainBuffers() {\n        drainReadBuffers();\n        drainWriteBuffer();\n    }\n\n    /** Drains the read buffers, each up to an amortized threshold. */\n    void drainReadBuffers() {\n        final int start = (int) Thread.currentThread().getId();\n        final int end = start + NUMBER_OF_READ_BUFFERS;\n        for (int i = start; i < end; i++) {\n            drainReadBuffer(i & READ_BUFFERS_MASK);\n        }\n    }\n\n    /** Drains the read buffer up to an amortized threshold. */\n    void drainReadBuffer(int bufferIndex) {\n        final long writeCount = readBufferWriteCount[bufferIndex].get();\n        for (int i = 0; i < READ_BUFFER_DRAIN_THRESHOLD; i++) {\n            final int index = (int) (readBufferReadCount[bufferIndex] & READ_BUFFER_INDEX_MASK);\n            final AtomicReference<Node<K, V>> slot = readBuffers[bufferIndex][index];\n            final Node<K, V> node = slot.get();\n            if (node == null) {\n                break;\n            }\n\n            slot.lazySet(null);\n            applyRead(node);\n            readBufferReadCount[bufferIndex]++;\n        }\n        readBufferDrainAtWriteCount[bufferIndex].lazySet(writeCount);\n    }\n\n    /** Updates the node's location in the page replacement policy. */\n    void applyRead(Node<K, V> node) {\n        // An entry may be scheduled for reordering despite having been removed.\n        // This can occur when the entry was concurrently read while a writer was\n        // removing it. If the entry is no longer linked then it does not need to\n        // be processed.\n        if (evictionDeque.contains(node)) {\n            evictionDeque.moveToBack(node);\n        }\n    }\n\n    /** Drains the read buffer up to an amortized threshold. */\n    void drainWriteBuffer() {\n        for (int i = 0; i < WRITE_BUFFER_DRAIN_THRESHOLD; i++) {\n            final Runnable task = writeBuffer.poll();\n            if (task == null) {\n                break;\n            }\n            task.run();\n        }\n    }\n\n    /**\n     * Attempts to transition the node from the <code>alive</code> state to the\n     * <code>retired</code> state.\n     *\n     * @param node\n     *        the entry in the page replacement policy\n     * @param expect\n     *        the expected weighted value\n     * @return if successful\n     */\n    boolean tryToRetire(Node<K, V> node, WeightedValue<V> expect) {\n        if (expect.isAlive()) {\n            final WeightedValue<V> retired = new WeightedValue<V>(expect.value, -expect.weight);\n            return node.compareAndSet(expect, retired);\n        }\n        return false;\n    }\n\n    /**\n     * Atomically transitions the node from the <code>alive</code> state to the\n     * <code>retired</code> state, if a valid transition.\n     *\n     * @param node\n     *        the entry in the page replacement policy\n     */\n    void makeRetired(Node<K, V> node) {\n        for (;;) {\n            final WeightedValue<V> current = node.get();\n            if (!current.isAlive()) {\n                return;\n            }\n            final WeightedValue<V> retired = new WeightedValue<V>(current.value, -current.weight);\n            if (node.compareAndSet(current, retired)) {\n                return;\n            }\n        }\n    }\n\n    /**\n     * Atomically transitions the node to the <code>dead</code> state and decrements\n     * the <code>weightedSize</code>.\n     *\n     * @param node\n     *        the entry in the page replacement policy\n     */\n    void makeDead(Node<K, V> node) {\n        for (;;) {\n            WeightedValue<V> current = node.get();\n            WeightedValue<V> dead = new WeightedValue<V>(current.value, 0);\n            if (node.compareAndSet(current, dead)) {\n                weightedSize.lazySet(weightedSize.get() - Math.abs(current.weight));\n                return;\n            }\n        }\n    }\n\n    /** Notifies the listener of entries that were evicted. */\n    void notifyListener() {\n        Node<K, V> node;\n        while ((node = pendingNotifications.poll()) != null) {\n            listener.onEviction(node.key, node.getValue());\n        }\n    }\n\n    /** Adds the node to the page replacement policy. */\n    final class AddTask implements Runnable {\n        final Node<K, V> node;\n        final int weight;\n\n        AddTask(Node<K, V> node, int weight) {\n            this.weight = weight;\n            this.node = node;\n        }\n\n        @Override\n        public void run() {\n            weightedSize.lazySet(weightedSize.get() + weight);\n\n            // ignore out-of-order write operations\n            if (node.get().isAlive()) {\n                evictionDeque.add(node);\n                evict();\n            }\n        }\n    }\n\n    /** Removes a node from the page replacement policy. */\n    final class RemovalTask implements Runnable {\n        final Node<K, V> node;\n\n        RemovalTask(Node<K, V> node) {\n            this.node = node;\n        }\n\n        @Override\n        public void run() {\n            // add may not have been processed yet\n            evictionDeque.remove(node);\n            makeDead(node);\n        }\n    }\n\n    /** Updates the weighted size and evicts an entry on overflow. */\n    final class UpdateTask implements Runnable {\n        final int weightDifference;\n        final Node<K, V> node;\n\n        public UpdateTask(Node<K, V> node, int weightDifference) {\n            this.weightDifference = weightDifference;\n            this.node = node;\n        }\n\n        @Override\n        public void run() {\n            weightedSize.lazySet(weightedSize.get() + weightDifference);\n            applyRead(node);\n            evict();\n        }\n    }\n\n    /* ---------------- Concurrent Map Support -------------- */\n\n    @Override\n    public boolean isEmpty() {\n        return data.isEmpty();\n    }\n\n    @Override\n    public int size() {\n        return data.size();\n    }\n\n    /**\n     * Returns the weighted size of this map.\n     *\n     * @return the combined weight of the values in this map\n     */\n    public long weightedSize() {\n        return Math.max(0, weightedSize.get());\n    }\n\n    @Override\n    public void clear() {\n        evictionLock.lock();\n        try {\n            // Discard all entries\n            Node<K, V> node;\n            while ((node = evictionDeque.poll()) != null) {\n                data.remove(node.key, node);\n                makeDead(node);\n            }\n\n            // Discard all pending reads\n            for (AtomicReference<Node<K, V>>[] buffer : readBuffers) {\n                for (AtomicReference<Node<K, V>> slot : buffer) {\n                    slot.lazySet(null);\n                }\n            }\n\n            // Apply all pending writes\n            Runnable task;\n            while ((task = writeBuffer.poll()) != null) {\n                task.run();\n            }\n        } finally {\n            evictionLock.unlock();\n        }\n    }\n\n    @Override\n    public boolean containsKey(Object key) {\n        return data.containsKey(key);\n    }\n\n    @Override\n    public boolean containsValue(Object value) {\n        checkNotNull(value);\n\n        for (Node<K, V> node : data.values()) {\n            if (node.getValue().equals(value)) {\n                return true;\n            }\n        }\n        return false;\n    }\n\n    @Override\n    public V get(Object key) {\n        final Node<K, V> node = data.get(key);\n        if (node == null) {\n            return null;\n        }\n        afterRead(node);\n        return node.getValue();\n    }\n\n    /**\n     * Returns the value to which the specified key is mapped, or {@code null}\n     * if this map contains no mapping for the key. This method differs from\n     * {@link #get(Object)} in that it does not record the operation with the\n     * page replacement policy.\n     *\n     * @param key\n     *        the key whose associated value is to be returned\n     * @return the value to which the specified key is mapped, or\n     *         {@code null} if this map contains no mapping for the key\n     * @throws NullPointerException\n     *         if the specified key is null\n     */\n    public V getQuietly(Object key) {\n        final Node<K, V> node = data.get(key);\n        return (node == null) ? null : node.getValue();\n    }\n\n    @Override\n    public V put(K key, V value) {\n        return put(key, value, false);\n    }\n\n    @Override\n    public V putIfAbsent(K key, V value) {\n        return put(key, value, true);\n    }\n\n    /**\n     * Adds a node to the list and the data store. If an existing node is found,\n     * then its value is updated if allowed.\n     *\n     * @param key\n     *        key with which the specified value is to be associated\n     * @param value\n     *        value to be associated with the specified key\n     * @param onlyIfAbsent\n     *        a write is performed only if the key is not already\n     *        associated with a value\n     * @return the prior value in the data store or null if no mapping was found\n     */\n    V put(K key, V value, boolean onlyIfAbsent) {\n        checkNotNull(key);\n        checkNotNull(value);\n\n        final int weight = weigher.weightOf(key, value);\n        final WeightedValue<V> weightedValue = new WeightedValue<V>(value, weight);\n        final Node<K, V> node = new Node<K, V>(key, weightedValue);\n\n        for (;;) {\n            final Node<K, V> prior = data.putIfAbsent(node.key, node);\n            if (prior == null) {\n                afterWrite(new AddTask(node, weight));\n                return null;\n            } else if (onlyIfAbsent) {\n                afterRead(prior);\n                return prior.getValue();\n            }\n            for (;;) {\n                final WeightedValue<V> oldWeightedValue = prior.get();\n                if (!oldWeightedValue.isAlive()) {\n                    break;\n                }\n\n                if (prior.compareAndSet(oldWeightedValue, weightedValue)) {\n                    final int weightedDifference = weight - oldWeightedValue.weight;\n                    if (weightedDifference == 0) {\n                        afterRead(prior);\n                    } else {\n                        afterWrite(new UpdateTask(prior, weightedDifference));\n                    }\n                    return oldWeightedValue.value;\n                }\n            }\n        }\n    }\n\n    @Override\n    public V remove(Object key) {\n        final Node<K, V> node = data.remove(key);\n        if (node == null) {\n            return null;\n        }\n\n        makeRetired(node);\n        afterWrite(new RemovalTask(node));\n        return node.getValue();\n    }\n\n    @Override\n    public boolean remove(Object key, Object value) {\n        final Node<K, V> node = data.get(key);\n        if ((node == null) || (value == null)) {\n            return false;\n        }\n\n        WeightedValue<V> weightedValue = node.get();\n        for (;;) {\n            if (weightedValue.contains(value)) {\n                if (tryToRetire(node, weightedValue)) {\n                    if (data.remove(key, node)) {\n                        afterWrite(new RemovalTask(node));\n                        return true;\n                    }\n                } else {\n                    weightedValue = node.get();\n                    if (weightedValue.isAlive()) {\n                        // retry as an intermediate update may have replaced the value with\n                        // an equal instance that has a different reference identity\n                        continue;\n                    }\n                }\n            }\n            return false;\n        }\n    }\n\n    @Override\n    public V replace(K key, V value) {\n        checkNotNull(key);\n        checkNotNull(value);\n\n        final int weight = weigher.weightOf(key, value);\n        final WeightedValue<V> weightedValue = new WeightedValue<V>(value, weight);\n\n        final Node<K, V> node = data.get(key);\n        if (node == null) {\n            return null;\n        }\n        for (;;) {\n            final WeightedValue<V> oldWeightedValue = node.get();\n            if (!oldWeightedValue.isAlive()) {\n                return null;\n            }\n            if (node.compareAndSet(oldWeightedValue, weightedValue)) {\n                final int weightedDifference = weight - oldWeightedValue.weight;\n                if (weightedDifference == 0) {\n                    afterRead(node);\n                } else {\n                    afterWrite(new UpdateTask(node, weightedDifference));\n                }\n                return oldWeightedValue.value;\n            }\n        }\n    }\n\n    @Override\n...\n// Path: src/main/java/mssql/googlecode/concurrentlinkedhashmap/EvictionListener.java\n/*\n * Copyright 2010 Google Inc. All Rights Reserved.\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n * http://www.apache.org/licenses/LICENSE-2.0\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage mssql.googlecode.concurrentlinkedhashmap;\n\n/**\n * A listener registered for notification when an entry is evicted. An instance\n * may be called concurrently by multiple threads to process entries. An\n * implementation should avoid performing blocking calls or synchronizing on\n * shared resources.\n * <p>\n * The listener is invoked by {@link ConcurrentLinkedHashMap} on a caller's\n * thread and will not block other threads from operating on the map. An\n * implementation should be aware that the caller's thread will not expect\n * long execution times or failures as a side effect of the listener being\n * notified. Execution safety and a fast turn around time can be achieved by\n * performing the operation asynchronously, such as by submitting a task to an\n * {@link java.util.concurrent.ExecutorService}.\n *\n * @author ben.manes@gmail.com (Ben Manes)\n * @see <a href=\"http://code.google.com/p/concurrentlinkedhashmap/\">\n *      http://code.google.com/p/concurrentlinkedhashmap/</a>\n */\npublic interface EvictionListener<K, V> {\n\n    /**\n     * A call-back notification that the entry was evicted.\n     *\n     * @param key\n     *        the entry's key\n     * @param value\n     *        the entry's value\n     */\n    void onEviction(K key, V value);\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerConnection.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport static java.nio.charset.StandardCharsets.UTF_16LE;\n\nimport java.io.IOException;\nimport java.io.Serializable;\nimport java.net.DatagramPacket;\nimport java.net.DatagramSocket;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.SocketException;\nimport java.net.UnknownHostException;\nimport java.sql.CallableStatement;\nimport java.sql.Connection;\nimport java.sql.DatabaseMetaData;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.SQLClientInfoException;\nimport java.sql.SQLException;\nimport java.sql.SQLPermission;\nimport java.sql.SQLWarning;\nimport java.sql.SQLXML;\nimport java.sql.Savepoint;\nimport java.sql.Statement;\nimport java.text.MessageFormat;\nimport java.time.Duration;\nimport java.time.Instant;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Date;\nimport java.util.Enumeration;\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Locale;\nimport java.util.Map;\nimport java.util.Properties;\nimport java.util.Random;\nimport java.util.UUID;\nimport java.util.concurrent.ConcurrentHashMap;\nimport java.util.concurrent.ConcurrentLinkedQueue;\nimport java.util.concurrent.Executor;\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.logging.Level;\n\nimport javax.sql.XAConnection;\n\nimport org.ietf.jgss.GSSCredential;\n\nimport com.microsoft.sqlserver.jdbc.SQLServerError.TransientError;\n\nimport mssql.googlecode.cityhash.CityHash;\nimport mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;\nimport mssql.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap.Builder;\nimport mssql.googlecode.concurrentlinkedhashmap.EvictionListener;\n\n\n/**\n * Provides an implementation java.sql.connection interface that assists creating a JDBC connection to SQL Server.\n * SQLServerConnections support JDBC connection pooling and may be either physical JDBC connections or logical JDBC\n * connections.\n *\n * SQLServerConnection manages transaction control for all statements that were created from it. SQLServerConnection may\n * participate in XA distributed transactions managed via an XAResource adapter.\n *\n * SQLServerConnection instantiates a new TDSChannel object for use by itself and all statement objects that are created\n * under this connection.\n *\n * SQLServerConnection manages a pool of prepared statement handles. Prepared statements are prepared once and typically\n * executed many times with different data values for their parameters. Prepared statements are also maintained across\n * logical (pooled) connection closes.\n *\n * SQLServerConnection is not thread safe, however multiple statements created from a single connection can be\n * processing simultaneously in concurrent threads.\n *\n * This class's public functions need to be kept identical to the SQLServerConnectionPoolProxy's.\n *\n * The API javadoc for JDBC API methods that this class implements are not repeated here. Please see Sun's JDBC API\n * interfaces javadoc for those details.\n *\n * NOTE: All the public functions in this class also need to be defined in SQLServerConnectionPoolProxy Declare all new\n * custom (non-static) Public APIs in ISQLServerConnection interface such that they can also be implemented by\n * SQLServerConnectionPoolProxy\n * \n */\npublic class SQLServerConnection implements ISQLServerConnection, java.io.Serializable {\n\n    /**\n     * Always refresh SerialVersionUID when prompted\n     */\n    private static final long serialVersionUID = 1965647556064751510L;\n\n    /**\n     * A random netAddress for this process to send during LOGIN7\n     */\n    private static final byte[] netAddress = getRandomNetAddress();\n\n    /** timer expiry */\n    long timerExpire;\n\n    /** flag to indicate if attempt refresh token is locked */\n    boolean attemptRefreshTokenLocked = false;\n\n    /**\n     * Thresholds related to when prepared statement handles are cleaned-up. 1 == immediately.\n     * \n     * The default for the prepared statement clean-up action threshold (i.e. when sp_unprepare is called).\n     */\n    static final int DEFAULT_SERVER_PREPARED_STATEMENT_DISCARD_THRESHOLD = 10; // Used to set the initial default, can\n                                                                               // be changed later.\n\n    /** Current limit for this particular connection */\n    private int serverPreparedStatementDiscardThreshold = -1;\n\n    /**\n     * The default for if prepared statements should execute sp_executesql before following the prepare, unprepare\n     * pattern.\n     * \n     * Used to set the initial default, can be changed later. false == use sp_executesql -> sp_prepexec -> sp_execute ->\n     * batched -> sp_unprepare pattern, true == skip sp_executesql part of pattern.\n     */\n    static final boolean DEFAULT_ENABLE_PREPARE_ON_FIRST_PREPARED_STATEMENT_CALL = false;\n\n    /**\n     * Back off interval in ms for retries\n     */\n    static final int BACKOFF_INTERVAL = 100;\n\n    /** Current limit for this particular connection. */\n    private Boolean enablePrepareOnFirstPreparedStatementCall = null;\n\n    /** Used for toggling use of sp_prepare */\n    private String prepareMethod = null;\n\n    /** Handle the actual queue of discarded prepared statements. */\n    private ConcurrentLinkedQueue<PreparedStatementHandle> discardedPreparedStatementHandles = new ConcurrentLinkedQueue<>();\n\n    /** discardedPreparedStatementHandle count */\n    private AtomicInteger discardedPreparedStatementHandleCount = new AtomicInteger(0);\n\n    /** fedAuth required by user flag */\n    private boolean fedAuthRequiredByUser = false;\n\n    /** fedAuth required prelogin response flag */\n    private boolean fedAuthRequiredPreLoginResponse = false;\n\n    /** fedAuth requested flag */\n    private boolean federatedAuthenticationRequested = false;\n\n    /** Access token callback class name */\n    private transient String accessTokenCallbackClass = null;\n\n    /** Flag that determines whether the accessToken callback class is set **/\n    private boolean hasAccessTokenCallbackClass = false;\n\n    /** Flag that determines whether the accessToken callback was set **/\n    private transient SQLServerAccessTokenCallback accessTokenCallback = null;\n\n    /** Flag indicating whether to use sp_sproc_columns for parameter name lookup */\n    private boolean useFlexibleCallableStatements = SQLServerDriverBooleanProperty.USE_FLEXIBLE_CALLABLE_STATEMENTS\n            .getDefaultValue();\n\n    /**\n     * Keep this distinct from _federatedAuthenticationRequested, since some fedauth library types may not need more\n     * info\n     */\n    private boolean federatedAuthenticationInfoRequested = false;\n\n    /** fedAuth feature extension data */\n    private FederatedAuthenticationFeatureExtensionData fedAuthFeatureExtensionData = null;\n\n    /** authentication string */\n    private String authenticationString = null;\n\n    /** access token in byte */\n    private byte[] accessTokenInByte = null;\n\n    /** fedAuth token */\n    private SqlAuthenticationToken fedAuthToken = null;\n\n    /** original hostNameInCertificate */\n    private String originalHostNameInCertificate = null;\n\n    /** client certificate */\n    private String clientCertificate = null;\n\n    /** client key */\n    private String clientKey = null;\n\n    /** client key password */\n    private String clientKeyPassword = \"\";\n\n    /** service principal certificate used for ActiveDirectoryServicePrincipalCertificate authentication */\n    private String servicePrincipalCertificate = null;\n\n    /** service principal certificate key used for ActiveDirectoryServicePrincipalCertificate authentication */\n    private String servicePrincipalCertificateKey = null;\n\n    /** service principal certificate password used for ActiveDirectoryServicePrincipalCertificate authentication */\n    private String servicePrincipalCertificatePassword = \"\";\n\n    /** AAD principal id */\n    private String aadPrincipalID = \"\";\n\n    /** AAD principal secret */\n    private String aadPrincipalSecret = \"\";\n\n    /** sendTemporalDataTypesAsStringForBulkCopy flag */\n    private boolean sendTemporalDataTypesAsStringForBulkCopy = true;\n\n    /**\n     * https://docs.microsoft.com/sql/t-sql/functions/serverproperty-transact-sql\n     */\n    /** Engine Edition 5 = SQL Database */\n    private static final int ENGINE_EDITION_SQL_AZURE_DB = 5;\n    /** Engine Edition 6 = Microsoft Azure Synapse Analytics */\n    private static final int ENGINE_EDITION_SQL_AZURE_SYNAPSE_ANALYTICS = 6;\n    /** Engine Edition 8 = Azure SQL Managed Instance */\n    private static final int ENGINE_EDITION_SQL_AZURE_MI = 8;\n    /** Engine Edition 9 = Azure SQL Edge (This is returned for all editions of Azure SQL Edge) */\n    private static final int ENGINE_EDITION_SQL_AZURE_SQL_EDGE = 9;\n    /** Engine Edition 11 = Azure Synapse serverless SQL pool */\n    private static final int ENGINE_EDITION_SQL_AZURE_SYNAPSE_SERVERLESS_SQL_POOL = 11;\n\n    /**\n     * Azure SQL server endpoints\n     */\n    enum AzureSQLServerEndpoints {\n        AZURE_GENERIC_ENDPOINT(\".database.windows.net\"),\n        AZURE_GERMAN_ENDPOINT(\".database.cloudapi.de\"),\n        AZURE_USGOV_ENDPOINT(\".database.usgovcloudapi.net\"),\n        AZURE_CHINA_ENDPOINT(\".database.china.cloudapi.cn\");\n\n        private static final String ON_DEMAND_PREFIX = \"-ondemand\";\n        private static final String AZURE_SYNAPSE = \"-ondemand.sql.azuresynapse.\";\n\n        private final String endpoint;\n\n        private AzureSQLServerEndpoints(String endpoint) {\n            this.endpoint = endpoint;\n        }\n\n        static boolean isAzureSqlServerEndpoint(String endpoint) {\n            for (AzureSQLServerEndpoints e : AzureSQLServerEndpoints.values()) {\n                if (endpoint.endsWith(e.toString())) {\n                    return true;\n                }\n            }\n            return false;\n        }\n\n        static boolean isAzureSynapseOnDemandEndpoint(String endpoint) {\n            if (endpoint.contains(AZURE_SYNAPSE)) {\n                return true;\n            }\n\n            for (AzureSQLServerEndpoints e : AzureSQLServerEndpoints.values()) {\n                if (endpoint.endsWith(ON_DEMAND_PREFIX + e.toString())) {\n                    return true;\n                }\n            }\n            return false;\n        }\n\n        @Override\n        public String toString() {\n            return endpoint;\n        }\n    }\n\n    /** flag indicating whether server is Azure */\n    private Boolean isAzure = null;\n\n    /** flag indicating whether server is Azure DW */\n    private Boolean isAzureDW = null;\n\n    /** flag indicating whether server is Azure MI */\n    private Boolean isAzureMI = null;\n\n    /** flag indicating whether server supports transactions */\n    private Boolean supportsTransactions = null;\n\n    /** shared timer */\n    private SharedTimer sharedTimer;\n\n    /** connect retry count */\n    private int connectRetryCount = 0;\n\n    /** connect retry interval */\n    private int connectRetryInterval = 0;\n\n    /** flag indicating whether prelogin TLS handshake is required */\n    private boolean isTDS8 = false;\n\n    /** encrypted truststore password */\n    byte[] encryptedTrustStorePassword = null;\n\n    /**\n     * lock instance for \"this\"\n     **/\n    private final transient Lock lock = new ReentrantLock();\n\n    /**\n     * static lock instance for the class\n     **/\n    private static final Lock sLock = new ReentrantLock();\n\n    /**\n     * Generate a 6 byte random array for netAddress\n     * As per TDS spec this is a unique clientID (MAC address) used to identify the client.\n     * A random number is used instead of the actual MAC address to avoid PII issues.\n     * As per spec this is informational only server does not process this so there is no need to use SecureRandom.\n     * \n     * @return byte[]\n     */\n    \nprivate static byte[] getRandomNetAddress() {\n        byte[] a = new byte[6];\n        Random random = new Random();\n        random.nextBytes(a);\n        return a;\n    }\n\n    /**\n     * Return an existing cached SharedTimer associated with this Connection or create a new one.\n     *\n     * The SharedTimer will be released when the Connection is closed.\n     * \n     * @throws SQLServerException\n     */\n    SharedTimer getSharedTimer() throws SQLServerException {\n        if (state == State.CLOSED) {\n            SQLServerException.makeFromDriverError(null, null, SQLServerException.getErrString(\"R_connectionIsClosed\"),\n                    SQLServerException.EXCEPTION_XOPEN_CONNECTION_FAILURE, false);\n        }\n        if (null == sharedTimer) {\n            this.sharedTimer = SharedTimer.getTimer();\n        }\n        return this.sharedTimer;\n    }\n\n    /**\n     * Get the server name string including redirected server if applicable\n     * \n     * @param serverName\n     * @return\n     */\n    String getServerNameString(String serverName) {\n        String serverNameFromConnectionStr = activeConnectionProperties\n                .getProperty(SQLServerDriverStringProperty.SERVER_NAME.toString());\n        if (null == serverName || serverName.equals(serverNameFromConnectionStr)) {\n            return serverName;\n        }\n\n        // server was redirected\n        MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_redirectedFrom\"));\n        Object[] msgArgs = {serverName, serverNameFromConnectionStr};\n        return form.format(msgArgs);\n    }\n\n    /**\n     * CityHash128Key\n     *\n     */\n    static class CityHash128Key implements java.io.Serializable {\n\n        /**\n         * Always refresh SerialVersionUID when prompted\n         */\n        private static final long serialVersionUID = 166788428640603097L;\n        String unhashedString;\n        private long[] segments;\n        private int hashCode;\n\n        CityHash128Key(String sql, String parametersDefinition) {\n            this(sql + parametersDefinition);\n        }\n\n        @SuppressWarnings(\"deprecation\")\n        CityHash128Key(String s) {\n            unhashedString = s;\n            byte[] bytes = new byte[s.length()];\n            s.getBytes(0, s.length(), bytes, 0);\n            segments = CityHash.cityHash128(bytes, 0, bytes.length);\n        }\n\n        public boolean equals(Object obj) {\n            if (!(obj instanceof CityHash128Key))\n                return false;\n\n            // checks if hash is equal, short-circuitting and if string is equal\n            return (java.util.Arrays.equals(segments, ((CityHash128Key) obj).segments)\n                    && this.unhashedString.equals(((CityHash128Key) obj).unhashedString));\n        }\n\n        public int hashCode() {\n            if (0 == hashCode) {\n                hashCode = java.util.Arrays.hashCode(segments);\n            }\n            return hashCode;\n        }\n    }\n\n    /**\n     * Keeps track of an individual prepared statement handle.\n     */\n    class PreparedStatementHandle {\n        private int handle = 0;\n        private final AtomicInteger handleRefCount = new AtomicInteger();\n        private boolean isDirectSql;\n        private volatile boolean evictedFromCache;\n        private volatile boolean explicitlyDiscarded;\n        private CityHash128Key key;\n\n        PreparedStatementHandle(CityHash128Key key, int handle, boolean isDirectSql, boolean isEvictedFromCache) {\n            this.key = key;\n            this.handle = handle;\n            this.isDirectSql = isDirectSql;\n            this.setIsEvictedFromCache(isEvictedFromCache);\n            handleRefCount.set(1);\n        }\n\n        /** Has the statement been evicted from the statement handle cache. */\n        private boolean isEvictedFromCache() {\n            return evictedFromCache;\n        }\n\n        /** Specify whether the statement been evicted from the statement handle cache. */\n        private void setIsEvictedFromCache(boolean isEvictedFromCache) {\n            this.evictedFromCache = isEvictedFromCache;\n        }\n\n        /** Specify that this statement has been explicitly discarded from being used by the cache. */\n        void setIsExplicitlyDiscarded() {\n            this.explicitlyDiscarded = true;\n\n            evictCachedPreparedStatementHandle(this);\n        }\n\n        /** Has the statement been explicitly discarded. */\n        private boolean isExplicitlyDiscarded() {\n            return explicitlyDiscarded;\n        }\n\n        /** Returns the actual handle. */\n        int getHandle() {\n            return handle;\n        }\n\n        /** Returns the cache key. */\n        CityHash128Key getKey() {\n            return key;\n        }\n\n        boolean isDirectSql() {\n            return isDirectSql;\n        }\n\n        /**\n         * Makes sure handle cannot be re-used.\n         * \n         * @return false: Handle could not be discarded, it is in use. true: Handle was successfully put on path for\n         *         discarding.\n         */\n        private boolean tryDiscardHandle() {\n            return handleRefCount.compareAndSet(0, -999);\n        }\n\n        /** Returns whether this statement has been discarded and can no longer be re-used. */\n        private boolean isDiscarded() {\n            return 0 > handleRefCount.intValue();\n        }\n\n        /**\n         * Adds a new reference to this handle, i.e. re-using it.\n         * \n         * @return false: Reference could not be added, statement has been discarded or does not have a handle\n         *         associated with it. true: Reference was successfully added.\n         */\n        boolean tryAddReference() {\n            return (isDiscarded() || isExplicitlyDiscarded()) ? false : handleRefCount.incrementAndGet() > 0;\n        }\n\n        /** Remove a reference from this handle */\n        void removeReference() {\n            handleRefCount.decrementAndGet();\n        }\n    }\n\n    /**\n     * Keeps track of the last network activity in order to tell if the connection has been idle. This is done to avoid\n     * checking the socket for a disconnect every command execution, which has a high impact on perf.\n     */\n    class IdleNetworkTracker {\n        private Instant lastNetworkActivity = Instant.now();\n\n        /**\n         * An \"idle\" connection will only ever get its socket disconnected by a keepalive packet after a connection has\n         * been severed. KeepAlive packets are only sent on idle sockets. Default setting by the driver (on platforms\n         * that have Java support for setting it) and the recommended setting is 30s (and OS default for those that\n         * don't set it is 2 hrs).\n         */\n        private int maxIdleMillis = 15000;\n\n        /** Has it been more than maxIdleMillis since network activity has been marked */\n        private boolean isIdle() {\n            return Instant.now().minusMillis(maxIdleMillis).isAfter(lastNetworkActivity);\n        }\n\n        /** Mark network activity now */\n        protected void markNetworkActivity() {\n            lastNetworkActivity = Instant.now();\n        }\n\n        /** Set max idle time in milliseconds */\n        protected void setMaxIdleMillis(int millis) {\n            maxIdleMillis = millis;\n        }\n    }\n\n    /** Keeps track of network idle status */\n    transient IdleNetworkTracker idleNetworkTracker = new IdleNetworkTracker();\n\n    /** Size of the parsed SQL-text metadata cache */\n    static final private int PARSED_SQL_CACHE_SIZE = 100;\n\n    /** Cache of parsed SQL meta data */\n    static private ConcurrentLinkedHashMap<CityHash128Key, ParsedSQLCacheItem> parsedSQLCache;\n\n    static {\n        parsedSQLCache = new Builder<CityHash128Key, ParsedSQLCacheItem>()\n                .maximumWeightedCapacity(PARSED_SQL_CACHE_SIZE).build();\n    }\n\n    /** Returns prepared statement cache entry if exists, if not parse and create a new one */\n    static ParsedSQLCacheItem getCachedParsedSQL(CityHash128Key key) {\n        return parsedSQLCache.get(key);\n    }\n\n    /** Parses and create a information about parsed SQL text */\n    static ParsedSQLCacheItem parseAndCacheSQL(CityHash128Key key, String sql) throws SQLServerException {\n        JDBCSyntaxTranslator translator = new JDBCSyntaxTranslator();\n\n        String parsedSql = translator.translate(sql);\n        String procName = translator.getProcedureName(); // may return null\n        boolean returnValueSyntax = translator.hasReturnValueSyntax();\n        int[] parameterPositions = locateParams(parsedSql);\n\n        ParsedSQLCacheItem cacheItem = new ParsedSQLCacheItem(parsedSql, parameterPositions, procName,\n                returnValueSyntax);\n        parsedSQLCache.putIfAbsent(key, cacheItem);\n        return cacheItem;\n    }\n\n    static int countParams(String sql) {\n        return locateParams(sql).length;\n    }\n\n    /** Default size for prepared statement caches */\n    static final int DEFAULT_STATEMENT_POOLING_CACHE_SIZE = 0;\n\n    /** Size of the prepared statement handle cache */\n    private int statementPoolingCacheSize = DEFAULT_STATEMENT_POOLING_CACHE_SIZE;\n\n    /** Cache of prepared statement handles */\n    private ConcurrentLinkedHashMap<CityHash128Key, PreparedStatementHandle> preparedStatementHandleCache;\n    /** Cache of prepared statement parameter metadata */\n    private ConcurrentLinkedHashMap<CityHash128Key, SQLServerParameterMetaData> parameterMetadataCache;\n    /**\n     * Checks whether statement pooling is enabled or disabled. The default is set to true;\n     */\n    private boolean disableStatementPooling = true;\n\n    /**\n     * Locates statement parameters.\n     * \n     * @param sql\n     *        SQL text to parse for positions of parameters to initialize.\n     */\n    private static int[] locateParams(String sql) {\n        LinkedList<Integer> parameterPositions = new LinkedList<>();\n\n        // Locate the parameter placeholders in the SQL string.\n        int offset = -1;\n        while ((offset = ParameterUtils.scanSQLForChar('?', sql, ++offset)) < sql.length()) {\n            parameterPositions.add(offset);\n        }\n\n        // return as int[]\n        return parameterPositions.stream().mapToInt(Integer::valueOf).toArray();\n    }\n\n    /**\n     * Encapsulates the data to be sent to the server as part of Federated Authentication Feature Extension.\n     */\n    class FederatedAuthenticationFeatureExtensionData implements Serializable {\n        /**\n         * Always update serialVersionUID when prompted\n         */\n        private static final long serialVersionUID = -6709861741957202475L;\n        boolean fedAuthRequiredPreLoginResponse;\n        int libraryType = -1;\n        byte[] accessToken = null;\n        SqlAuthentication authentication = null;\n\n        FederatedAuthenticationFeatureExtensionData(int libraryType, String authenticationString,\n                boolean fedAuthRequiredPreLoginResponse) throws SQLServerException {\n            this.libraryType = libraryType;\n            this.fedAuthRequiredPreLoginResponse = fedAuthRequiredPreLoginResponse;\n\n            switch (authenticationString.toUpperCase(Locale.ENGLISH)) {\n                case \"ACTIVEDIRECTORYPASSWORD\":\n                    this.authentication = SqlAuthentication.ACTIVE_DIRECTORY_PASSWORD;\n                    break;\n                case \"ACTIVEDIRECTORYINTEGRATED\":\n                    this.authentication = SqlAuthentication.ACTIVE_DIRECTORY_INTEGRATED;\n                    break;\n                case \"ACTIVEDIRECTORYMANAGEDIDENTITY\":\n                    this.authentication = SqlAuthentication.ACTIVE_DIRECTORY_MANAGED_IDENTITY;\n                    break;\n                case \"ACTIVEDIRECTORYDEFAULT\":\n                    this.authentication = SqlAuthentication.ACTIVE_DIRECTORY_DEFAULT;\n                    break;\n                case \"ACTIVEDIRECTORYSERVICEPRINCIPAL\":\n                    this.authentication = SqlAuthentication.ACTIVE_DIRECTORY_SERVICE_PRINCIPAL;\n                    break;\n                case \"ACTIVEDIRECTORYSERVICEPRINCIPALCERTIFICATE\":\n                    this.authentication = SqlAuthentication.ACTIVE_DIRECTORY_SERVICE_PRINCIPAL_CERTIFICATE;\n                    break;\n                case \"ACTIVEDIRECTORYINTERACTIVE\":\n                    this.authentication = SqlAuthentication.ACTIVE_DIRECTORY_INTERACTIVE;\n                    break;\n                default:\n                    // If authenticationString not specified, check if access token callback was set.\n                    // If access token callback is set, break.\n                    if (null != accessTokenCallback || hasAccessTokenCallbackClass) {\n                        this.authentication = SqlAuthentication.NOT_SPECIFIED;\n                        break;\n                    }\n                    MessageFormat form = new MessageFormat(\n                            SQLServerException.getErrString(\"R_InvalidConnectionSetting\"));\n                    Object[] msgArgs = {\"authentication\", authenticationString};\n                    throw new SQLServerException(null, form.format(msgArgs), null, 0, false);\n            }\n        }\n\n        FederatedAuthenticationFeatureExtensionData(int libraryType, boolean fedAuthRequiredPreLoginResponse,\n                byte[] accessToken) {\n            this.libraryType = libraryType;\n            this.fedAuthRequiredPreLoginResponse = fedAuthRequiredPreLoginResponse;\n            this.accessToken = accessToken;\n        }\n    }\n\n    class SqlFedAuthInfo {\n        String spn;\n        String stsurl;\n\n        @Override\n        public String toString() {\n            return \"STSURL: \" + stsurl + \", SPN: \" + spn;\n        }\n    }\n\n    class ActiveDirectoryAuthentication {\n        static final String JDBC_FEDAUTH_CLIENT_ID = \"7f98cb04-cd1e-40df-9140-3bf7e2cea4db\";\n\n        /**\n         * Managed Identities endpoint URL\n         * https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token\n         */\n        static final String AZURE_REST_MSI_URL = \"http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01\"; // NOSONAR\n        static final String ACCESS_TOKEN_IDENTIFIER = \"\\\"access_token\\\":\\\"\";\n        static final String ACCESS_TOKEN_EXPIRES_IN_IDENTIFIER = \"\\\"expires_in\\\":\\\"\";\n        static final String ACCESS_TOKEN_EXPIRES_ON_IDENTIFIER = \"\\\"expires_on\\\":\\\"\";\n        static final String ACCESS_TOKEN_EXPIRES_ON_DATE_FORMAT = \"M/d/yyyy h:mm:ss a X\";\n        static final int GET_ACCESS_TOKEN_SUCCESS = 0;\n        static final int GET_ACCESS_TOKEN_INVALID_GRANT = 1;\n        static final int GET_ACCESS_TOKEN_TRANSIENT_ERROR = 2;\n        static final int GET_ACCESS_TOKEN_OTHER_ERROR = 3;\n\n        private ActiveDirectoryAuthentication() {\n            throw new UnsupportedOperationException(SQLServerException.getErrString(\"R_notSupported\"));\n        }\n    }\n\n    final static int TNIR_FIRST_ATTEMPT_TIMEOUT_MS = 500; // fraction of timeout to use for fast failover connections\n\n    /**\n     * Denotes the state of the SqlServerConnection.\n     */\n    private enum State {\n        INITIALIZED, // default value on calling SQLServerConnection constructor\n        CONNECTED, // indicates that the TCP connection has completed\n        OPENED, // indicates that the prelogin, login have completed, the database session established and the\n                // connection is ready for use.\n...\n", "needle_token_start": 11974, "needle_token_end": 12020, "code_context_ntokens": 16026, "output": ["```java\nprivate static byte[] getRandomNetAddress() {\n    byte[] a = new byte[6];\n    Random random = new Random();\n    random.nextBytes(a);\n    return a;\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "computeResponse", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/NTLMAuthentication.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: This function is designed to compute the NTLMv2 response for authentication purposes, using a given hash key and a challenge message from a server.\n2. **Input**: A byte array representing the NT response hash key.\n3. **Output**: A byte array that represents the computed NTLMv2 response, which includes both the NT proof string and the client challenge blob.\n4. **Procedure**: \n   - Generate a random client nonce.\n   - Create a client challenge blob using the client nonce.\n   - Concatenate the server challenge with the client challenge blob.\n   - Compute the NT proof string using HMAC-MD5 with the response key and the concatenated challenge.\n   - Update the session base key using HMAC-MD5 with the response key and the NT proof string.\n   - Concatenate the NT proof string with the client challenge blob to form the final response.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/microsoft/sqlserver/jdbc/MaxResultBufferParser.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.lang.management.ManagementFactory;\nimport java.text.MessageFormat;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\n\n\n/**\n * Parser created to parse String value from Connection String to equivalent number of bytes for JDBC Driver to work on.\n */\npublic class MaxResultBufferParser {\n\n    private static final Logger logger = Logger.getLogger(\"com.microsoft.sqlserver.jdbc.MaxResultBufferParser\");\n    private static final String[] PERCENT_PHRASES = {\"percent\", \"pct\", \"p\"};\n    private static final String ERROR_MESSAGE = \"MaxResultBuffer property is badly formatted: {0}.\";\n\n    private MaxResultBufferParser() {}\n\n    /**\n     *\n     * Returns number of bytes for maxResultBuffer property\n     *\n     * @param input\n     *        String value for maxResultProperty provided in Connection String\n     * @return 'maxResultBuffer' property as number of bytes\n     * @throws SQLServerException\n     *         Is Thrown when maxResultProperty's syntax is wrong\n     */\n    public static long validateMaxResultBuffer(String input) throws SQLServerException {\n        String numberString;\n        long number = -1;\n\n        // check for null values and empty String \"\", if so return -1 (default value)\n        if (StringUtils.isEmpty(input) || input.equals(\"-1\")) {\n            return number;\n        }\n\n        // check if input is number\n        if (!StringUtils.isEmpty(input) && input.matches(\"-?\\\\d+(\\\\.\\\\d+)?\")) {\n            try {\n                number = Long.parseLong(input);\n            } catch (NumberFormatException e) {\n                if (logger.isLoggable(Level.SEVERE)) {\n                    logger.log(Level.SEVERE, ERROR_MESSAGE, new Object[] {input});\n                }\n                throwNewInvalidMaxResultBufferParameterException(e, input);\n            }\n            return adjustMemory(number, 1);\n        } else {\n            // check PERCENT_PHRASES\n            for (String percentPhrase : PERCENT_PHRASES) {\n                if (input.endsWith(percentPhrase)) {\n                    numberString = input.substring(0, input.length() - percentPhrase.length());\n                    try {\n                        number = Long.parseLong(numberString);\n                    } catch (NumberFormatException e) {\n                        if (logger.isLoggable(Level.SEVERE)) {\n                            logger.log(Level.SEVERE, ERROR_MESSAGE, new Object[] {input});\n                        }\n                        throwNewInvalidMaxResultBufferParameterException(e, numberString);\n                    }\n                    return adjustMemoryPercentage(number);\n                }\n            }\n\n            // check if prefix was supplied\n            long multiplier = getMultiplier(input);\n            numberString = input.substring(0, input.length() - 1);\n\n            try {\n                number = Long.parseLong(numberString);\n            } catch (NumberFormatException e) {\n                if (logger.isLoggable(Level.SEVERE)) {\n                    logger.log(Level.SEVERE, ERROR_MESSAGE, new Object[] {input});\n                }\n                throwNewInvalidMaxResultBufferParameterException(e, numberString);\n            }\n            return adjustMemory(number, multiplier);\n        }\n    }\n\n    private static void checkForNegativeValue(long value) throws SQLServerException {\n        if (value <= 0) {\n            Object[] objectToThrow = new Object[] {value};\n            MessageFormat form = new MessageFormat(\n                    SQLServerException.getErrString(\"R_maxResultBufferNegativeParameterValue\"));\n...\n// Path: src/main/java/mssql/security/provider/MD4.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\n/**\n * This code is extracted from org.bouncycastle.crypto.digests.MD4Digest and modified to remove dependencies.\n */\npackage mssql.security.provider;\n\npublic class MD4 {\n    private final byte[] xBuf = new byte[4];\n    private int xBufOff;\n    private long byteCount;\n    private static final int DIGEST_LENGTH = 16;\n    private int H1, H2, H3, H4; // IV's\n    private int[] X = new int[16];\n    private int xOff;\n\n    /**\n     * Standard constructor\n     */\n    public MD4() {\n        reset();\n    }\n\n    /**\n     * Copy constructor. This will copy the state of the provided message digest.\n     */\n    public MD4(MD4 t) {\n        System.arraycopy(t.xBuf, 0, xBuf, 0, t.xBuf.length);\n\n        xBufOff = t.xBufOff;\n        byteCount = t.byteCount;\n\n        copyIn(t);\n    }\n\n    private void copyIn(MD4 t) {\n        System.arraycopy(t.xBuf, 0, xBuf, 0, t.xBuf.length);\n\n        xBufOff = t.xBufOff;\n        byteCount = t.byteCount;\n\n        H1 = t.H1;\n        H2 = t.H2;\n        H3 = t.H3;\n        H4 = t.H4;\n\n        System.arraycopy(t.X, 0, X, 0, t.X.length);\n        xOff = t.xOff;\n    }\n\n    public String getAlgorithmName() {\n        return \"MD4\";\n    }\n\n    public int getDigestSize() {\n        return DIGEST_LENGTH;\n    }\n\n    protected void processWord(byte[] in, int inOff) {\n        X[xOff++] = (in[inOff] & 0xff) | ((in[inOff + 1] & 0xff) << 8) | ((in[inOff + 2] & 0xff) << 16)\n                | ((in[inOff + 3] & 0xff) << 24);\n\n        if (xOff == 16) {\n            processBlock();\n        }\n    }\n\n    protected void processLength(long bitLength) {\n        if (xOff > 14) {\n            processBlock();\n        }\n\n        X[14] = (int) (bitLength & 0xffffffff);\n        X[15] = (int) (bitLength >>> 32);\n    }\n\n    private void unpackWord(int word, byte[] out, int outOff) {\n        out[outOff] = (byte) word;\n        out[outOff + 1] = (byte) (word >>> 8);\n        out[outOff + 2] = (byte) (word >>> 16);\n        out[outOff + 3] = (byte) (word >>> 24);\n    }\n\n    public void update(byte in) {\n        xBuf[xBufOff++] = in;\n\n        if (xBufOff == xBuf.length) {\n            processWord(xBuf, 0);\n            xBufOff = 0;\n        }\n\n        byteCount++;\n    }\n\n    public void update(byte[] in, int inOff, int len) {\n        len = Math.max(0, len);\n\n        //\n        // fill the current word\n        //\n        int i = 0;\n        if (xBufOff != 0) {\n            while (i < len) {\n                xBuf[xBufOff++] = in[inOff + i++];\n                if (xBufOff == 4) {\n                    processWord(xBuf, 0);\n                    xBufOff = 0;\n                    break;\n                }\n            }\n        }\n\n        //\n        // process whole words.\n        //\n        int limit = ((len - i) & ~3) + i;\n        for (; i < limit; i += 4) {\n            processWord(in, inOff + i);\n        }\n\n        //\n        // load in the remainder.\n        //\n        while (i < len) {\n            xBuf[xBufOff++] = in[inOff + i++];\n        }\n\n        byteCount += len;\n    }\n\n    public void finish() {\n        long bitLength = (byteCount << 3);\n\n        //\n        // add the pad bytes.\n        //\n        update((byte) 128);\n\n        while (xBufOff != 0) {\n            update((byte) 0);\n        }\n\n        processLength(bitLength);\n\n        processBlock();\n    }\n\n    public int doFinal(byte[] out, int outOff) {\n        finish();\n\n        unpackWord(H1, out, outOff);\n        unpackWord(H2, out, outOff + 4);\n        unpackWord(H3, out, outOff + 8);\n        unpackWord(H4, out, outOff + 12);\n\n        reset();\n\n        return DIGEST_LENGTH;\n    }\n\n    /**\n     * reset the chaining variables to the IV values.\n     */\n    public void reset() {\n        byteCount = 0;\n\n        xBufOff = 0;\n        for (int i = 0; i < xBuf.length; i++) {\n            xBuf[i] = 0;\n        }\n\n        H1 = 0x67452301;\n        H2 = 0xefcdab89;\n        H3 = 0x98badcfe;\n        H4 = 0x10325476;\n\n        xOff = 0;\n\n        for (int i = 0; i != X.length; i++) {\n            X[i] = 0;\n        }\n    }\n\n    //\n    // round 1 left rotates\n    //\n    private static final int S11 = 3;\n    private static final int S12 = 7;\n    private static final int S13 = 11;\n    private static final int S14 = 19;\n\n    //\n    // round 2 left rotates\n    //\n    private static final int S21 = 3;\n    private static final int S22 = 5;\n    private static final int S23 = 9;\n    private static final int S24 = 13;\n\n    //\n    // round 3 left rotates\n    //\n    private static final int S31 = 3;\n    private static final int S32 = 9;\n    private static final int S33 = 11;\n    private static final int S34 = 15;\n\n    /*\n     * rotate int x left n bits.\n     */\n    private int rotateLeft(int x, int n) {\n        return (x << n) | (x >>> (32 - n));\n    }\n\n    /*\n     * F, G, H and I are the basic MD4 functions.\n     */\n    private int F(int u, int v, int w) {\n        return (u & v) | (~u & w);\n    }\n\n    private int G(int u, int v, int w) {\n        return (u & v) | (u & w) | (v & w);\n    }\n\n    private int H(int u, int v, int w) {\n        return u ^ v ^ w;\n    }\n\n    protected void processBlock() {\n        int a = H1;\n        int b = H2;\n        int c = H3;\n        int d = H4;\n\n        //\n        // Round 1 - F cycle, 16 times.\n        //\n        a = rotateLeft(a + F(b, c, d) + X[0], S11);\n        d = rotateLeft(d + F(a, b, c) + X[1], S12);\n        c = rotateLeft(c + F(d, a, b) + X[2], S13);\n        b = rotateLeft(b + F(c, d, a) + X[3], S14);\n        a = rotateLeft(a + F(b, c, d) + X[4], S11);\n        d = rotateLeft(d + F(a, b, c) + X[5], S12);\n        c = rotateLeft(c + F(d, a, b) + X[6], S13);\n        b = rotateLeft(b + F(c, d, a) + X[7], S14);\n        a = rotateLeft(a + F(b, c, d) + X[8], S11);\n        d = rotateLeft(d + F(a, b, c) + X[9], S12);\n        c = rotateLeft(c + F(d, a, b) + X[10], S13);\n        b = rotateLeft(b + F(c, d, a) + X[11], S14);\n        a = rotateLeft(a + F(b, c, d) + X[12], S11);\n        d = rotateLeft(d + F(a, b, c) + X[13], S12);\n        c = rotateLeft(c + F(d, a, b) + X[14], S13);\n        b = rotateLeft(b + F(c, d, a) + X[15], S14);\n\n        //\n        // Round 2 - G cycle, 16 times.\n        //\n        a = rotateLeft(a + G(b, c, d) + X[0] + 0x5a827999, S21);\n        d = rotateLeft(d + G(a, b, c) + X[4] + 0x5a827999, S22);\n        c = rotateLeft(c + G(d, a, b) + X[8] + 0x5a827999, S23);\n        b = rotateLeft(b + G(c, d, a) + X[12] + 0x5a827999, S24);\n        a = rotateLeft(a + G(b, c, d) + X[1] + 0x5a827999, S21);\n        d = rotateLeft(d + G(a, b, c) + X[5] + 0x5a827999, S22);\n        c = rotateLeft(c + G(d, a, b) + X[9] + 0x5a827999, S23);\n        b = rotateLeft(b + G(c, d, a) + X[13] + 0x5a827999, S24);\n        a = rotateLeft(a + G(b, c, d) + X[2] + 0x5a827999, S21);\n        d = rotateLeft(d + G(a, b, c) + X[6] + 0x5a827999, S22);\n        c = rotateLeft(c + G(d, a, b) + X[10] + 0x5a827999, S23);\n        b = rotateLeft(b + G(c, d, a) + X[14] + 0x5a827999, S24);\n        a = rotateLeft(a + G(b, c, d) + X[3] + 0x5a827999, S21);\n        d = rotateLeft(d + G(a, b, c) + X[7] + 0x5a827999, S22);\n        c = rotateLeft(c + G(d, a, b) + X[11] + 0x5a827999, S23);\n        b = rotateLeft(b + G(c, d, a) + X[15] + 0x5a827999, S24);\n\n        //\n        // Round 3 - H cycle, 16 times.\n        //\n        a = rotateLeft(a + H(b, c, d) + X[0] + 0x6ed9eba1, S31);\n        d = rotateLeft(d + H(a, b, c) + X[8] + 0x6ed9eba1, S32);\n        c = rotateLeft(c + H(d, a, b) + X[4] + 0x6ed9eba1, S33);\n        b = rotateLeft(b + H(c, d, a) + X[12] + 0x6ed9eba1, S34);\n        a = rotateLeft(a + H(b, c, d) + X[2] + 0x6ed9eba1, S31);\n        d = rotateLeft(d + H(a, b, c) + X[10] + 0x6ed9eba1, S32);\n        c = rotateLeft(c + H(d, a, b) + X[6] + 0x6ed9eba1, S33);\n        b = rotateLeft(b + H(c, d, a) + X[14] + 0x6ed9eba1, S34);\n        a = rotateLeft(a + H(b, c, d) + X[1] + 0x6ed9eba1, S31);\n        d = rotateLeft(d + H(a, b, c) + X[9] + 0x6ed9eba1, S32);\n        c = rotateLeft(c + H(d, a, b) + X[5] + 0x6ed9eba1, S33);\n        b = rotateLeft(b + H(c, d, a) + X[13] + 0x6ed9eba1, S34);\n        a = rotateLeft(a + H(b, c, d) + X[3] + 0x6ed9eba1, S31);\n        d = rotateLeft(d + H(a, b, c) + X[11] + 0x6ed9eba1, S32);\n        c = rotateLeft(c + H(d, a, b) + X[7] + 0x6ed9eba1, S33);\n        b = rotateLeft(b + H(c, d, a) + X[15] + 0x6ed9eba1, S34);\n\n        H1 += a;\n        H2 += b;\n        H3 += c;\n        H4 += d;\n\n        //\n        // reset the offset and clean out the word buffer.\n        //\n        xOff = 0;\n        for (int i = 0; i != X.length; i++) {\n            X[i] = 0;\n        }\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/NTLMAuthentication.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.nio.ByteBuffer;\nimport java.nio.ByteOrder;\nimport java.security.InvalidKeyException;\nimport java.security.NoSuchAlgorithmException;\nimport java.text.MessageFormat;\nimport java.time.Instant;\nimport java.util.Arrays;\nimport java.util.concurrent.ThreadLocalRandom;\nimport java.util.concurrent.TimeUnit;\nimport java.util.logging.Level;\n\nimport javax.crypto.Mac;\nimport javax.crypto.spec.SecretKeySpec;\n\nimport mssql.security.provider.MD4;\n\n\n/**\n * Provides an implementation of NTLMv2 authentication\n * \n * @see <a\n *      href=https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-nlmp/b38c36ed-2804-4868-a9ff-8dd3182128e4\">MS-NLMP:NTLMAuthentication\n *      Protocol</a>\n */\nfinal class NTLMAuthentication extends SSPIAuthentication {\n    private final java.util.logging.Logger logger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.NTLMAuthentication\");\n\n    /**\n     * Section 2.2.1.1 NEGOTIATE_MESSAGE\n     *\n     * Section 2.2.1.2 CHALLENGE_MESSAGE\n     *\n     * Section 2.2.1.3 AUTHENCIATE_MESSAGE\n     *\n     * NTLM signature for all 3 messages - \"NTLMSSP\\0\"\n     */\n    private static final byte[] NTLM_HEADER_SIGNATURE = {0x4e, 0x54, 0x4c, 0x4d, 0x53, 0x53, 0x50, 0x00};\n\n    // NTLM messages type\n    private static final int NTLM_MESSAGE_TYPE_NEGOTIATE = 0x00000001;\n    private static final int NTLM_MESSAGE_TYPE_CHALLENGE = 0x00000002;\n    private static final int NTLM_MESSAGE_TYPE_AUTHENTICATE = 0x00000003;\n\n    /**\n     * Section 2.2.2.7 NTLM v2: NTLMv2_CLIENT_CHALLENGE\n     * \n     * <pre>\n     * RespType   current version(aka Responseversion) of the challenge response type. This field MUST be 0x01\n     * HiRespType max supported version (aka HiResponserversion) of the client response type.\n     * </pre>\n     */\n    private static final byte[] NTLM_CLIENT_CHALLENGE_RESPONSE_TYPE = {0x01, 0x01};\n\n    /**\n     * Section 2.2.2.7 NTLM v2: NTLMv2_CLIENT_CHALLENGE\n     *\n     * <pre>\n     * Reserved1  SHOULD be 0x0000\n     * Reserved2  SHOULD be 0x00000000\n     * Reserved3  SHOULD be 0x00000000\n     * </pre>\n     */\n    private static final byte[] NTLM_CLIENT_CHALLENGE_RESERVED1 = {0x00, 0x00};\n    private static final byte[] NTLM_CLIENT_CHALLENGE_RESERVED2 = {0x00, 0x00, 0x00, 0x00};\n    private static final byte[] NTLM_CLIENT_CHALLENGE_RESERVED3 = {0x00, 0x00, 0x00, 0x00};\n\n    /**\n     * Section 3.1.5.1.1 Client Initiates the NEGOTIATE_MESSAGE\n     * \n     * If NTLM v2 authentication is used and the CHALLENGE_MESSAGE TargetInfo field (section 2.2.1.2) has an\n     * MsvAvTimestamp present, the client SHOULD NOT send the LmChallengeResponse and SHOULD send Z(24) instead.\n     *\n     * <pre>\n     * Section 2.2.2.1 AV_PAIR\n     * MsvAvTimestamp is always sent in the CHALLENGE_MESSAGE.\n     * Section 7 Appendix B: Product Behavior\n     * </pre>\n     *\n     * MsvAvTimestamp AV_PAIR type is not supported in Windows NT, Windows 2000, Windows XP, and Windows Server 2003.\n     * \n     * This defined here for completeness as it's necessary to 0 the correct number of bytes in the message.\n     */\n    private static final byte[] NTLM_LMCHALLENAGERESPONSE = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};\n\n    /**\n     * Section 2.2.2.10 VERSION\n     *\n     * used for debugging purposes only and its value does not affect NTLM message processing\n     * \n     * This defined here for completeness as it's necessary to 0 the correct number of bytes in the message.\n     * \n     */\n    private static final byte[] NTLMSSP_VERSION = new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};\n\n    /**\n     * Section 2.2.5 NEGOTIATE\n     *\n     * <pre>\n     * NTLM negotiate flags\n     * NTLMSSP_NEGOTIATE_UNICODE                  A bit requests unicode character set encoding.\n     * NTLMSSP_REQUEST_TARGET                     C bit TargetName field of the CHALLENGE_MESSAGE.\n     * NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED      K bit indicates whether the domain name is provided.\n     * NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED L bit indicates whether the Workstation field is present.\n     * NTLMSSP_NEGOTIATE_TARGET_INFO              S bit indicates whether the TargetInfo fields are populated.\n     * NTLMSSP_NEGOTIATE_ALWAYS_SIGN              M bit requests the presence of a signature block on all messages.\n     * NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY P bit requests usage of the NTLM v2 session security.\n     *\n     * Note - This is not specified in spec but NTLMSSP_NEGOTIATE_ALWAYS_SIGN is required for server to verify MIC!!\n     *        If not set, server ignores MIC field even tho MSVAVFLAGS is set!!\n     * </pre>\n     */\n    private static final long NTLMSSP_NEGOTIATE_UNICODE = 0x00000001;\n    private static final long NTLMSSP_REQUEST_TARGET = 0x00000004;\n    private static final long NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED = 0x00001000;\n    private static final long NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED = 0x00002000;\n    private static final long NTLMSSP_NEGOTIATE_TARGET_INFO = 0x00800000;\n    private static final long NTLMSSP_NEGOTIATE_ALWAYS_SIGN = 0x00008000;\n    private static final long NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY = 0x00080000;\n\n    /**\n     * Section 2.2.2.1 AV_PAIR\n     *\n     * <pre>\n     * AAvId (2bytes)\n     * A 16-bit unsigned integer that defines the information type in the Value field. The contents of this field MUST\n     * be a value from the following table. The corresponding Value field in this AV_PAIR MUST contain the information\n     * specified in the description of that AvId\n     * NTLM_AVID_MSVAVEOL             indicates that this is the last AV_PAIR in the list\n     * NTLM_AVID_MSVAVNBCOMPUTERNAME  the server's NetBIOS computer name in unicode\n     * NTLM_AVID_MSVAVNBDOMAINNAME    the server's NetBIOS domain name in unicode\n     * NTLM_AVID_MSVAVDNSCOMPUTERNAME the FQDN of the computer in unicode\n     * NTLM_AVID_MSVAVDNSDOMAINNAME   the FQDN of the domain in unicode\n     * NTLM_AVID_MSVAVDNSTREENAME     the FQDN of the forest in unicode (not currently used)\n     * NTLM_AVID_MSVAVFLAGS           indicates server or client configuration (0x00000002 indicates MIC provided)\n     * NTLM_AVID_MSVAVTIMESTAMP       FILETIME structure that contains the server local time (ALWAYS present in CHALLENGE_MESSAGE\n     * NTLM_AVID_MSVAVSINGLEHOST      Single Host Data structure  (not currently used)\n     * NTLM_AVID_MSVAVTARGETNAME      SPN of the target server in unicode (not currently used)\n     * </pre>\n     */\n    private static final short NTLM_AVID_MSVAVEOL = 0x0000;\n    private static final short NTLM_AVID_MSVAVNBCOMPUTERNAME = 0x0001;\n    private static final short NTLM_AVID_MSVAVNBDOMAINNAME = 0x0002;\n    private static final short NTLM_AVID_MSVAVDNSCOMPUTERNAME = 0x0003;\n    private static final short NTLM_AVID_MSVAVDNSDOMAINNAME = 0x0004;\n    private static final short NTLM_AVID_MSVAVDNSTREENAME = 0x0005;\n    private static final short NTLM_AVID_MSVAVFLAGS = 0x0006; // value in NTLM_AVID_VALUE_MIC\n    private static final short NTLM_AVID_MSVAVTIMESTAMP = 0x0007;\n    private static final short NTLM_AVID_MSVAVSINGLEHOST = 0x0008;\n    private static final short NTLM_AVID_MSVAVTARGETNAME = 0x0009;\n\n    /**\n     * Section 2.2.2.1 AV_PAIR\n     *\n     * value of NTLM_AVID_MSVAVFLAGS that indicates a MIC is provided\n     * \n     * Section 3.1.5.1.2 Client Receives a CHALLENGE_MESSAGE from the Server\n     * \n     * If the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present, the client SHOULD provide a MIC\n     */\n    private static final int NTLM_AVID_LENGTH = 2; // length of an AvId\n    private static final int NTLM_AVLEN_LENGTH = 2; // length of an AVLen\n\n    private static final int NTLM_AVFLAG_VALUE_MIC = 0x00000002; // indicates MIC is provided\n    private static final int NTLM_MIC_LENGTH = 16; // length of MIC field\n\n    private static final int NTLM_AVID_MSVAVFLAGS_LEN = 4; // length of MSVAVFLAG\n\n    /**\n     * Section 2.2.1.1 NEGOTIATE_MESSAGE\n     *\n     * <pre>\n     * offset to payload in Negotiate message\n     *     8 (signature) + \n     *     4 (message type) + \n     *     8 (domain name) + \n     *     8 (workstation) + \n     *     4 (negotiate flags) +\n     *     0 (version)\n     * </pre>\n     */\n    private static final int NTLM_NEGOTIATE_PAYLOAD_OFFSET = 32;\n\n    /**\n     * Section 2.2.1.3 AUTHENTICATE_MESSAGE\n     *\n     * <pre>\n     * offsets in Authenticate message\n     *     8 (signature) + \n     *     4 (message type) + \n     *     8 (lmChallengeResp) + \n     *     8 (ntChallengeResp) + \n     *     8 (domain name) + \n     *     8 (user name) +\n     *     8 (workstation) + \n     *     8 (encrypted random session key) + \n     *     4 (negotiate flags) + \n     *     8 (version) + \n     *     16 (MIC)\n     * </pre>\n     */\n    private static final int NTLM_AUTHENTICATE_PAYLOAD_OFFSET = 88;\n\n    // client challenge length\n    private static final int NTLM_CLIENT_NONCE_LENGTH = 8;\n\n    // server challenge length\n    private static final int NTLM_SERVER_CHALLENGE_LENGTH = 8;\n\n    // Windows Filetime timestamp length\n    private static final int NTLM_TIMESTAMP_LENGTH = 8;\n\n    // Windows epoch time difference from Unix epoch time in secs\n    private static final long WINDOWS_EPOCH_DIFF = 11644473600L;\n\n    /**\n     * NTLM Client Context\n     */\n    private class NTLMContext {\n        // domain name to authenticate to\n        private final String domainName;\n\n        // unicode bytes of domain name\n        private final byte[] domainUbytes;\n\n        // user credentials\n        private final String upperUserName;\n        private final byte[] userNameUbytes;\n        private final byte[] passwordHash;\n\n        // workstation\n        private String workstation;\n\n        // unicode bytes of server SPN\n        private final byte[] spnUbytes;\n\n        // message authentication code\n        private Mac mac = null;\n\n        // negotiate flags from Challenge msg\n        private long negotiateFlags = 0x00000000;\n\n        // session key calculated from password\n        private byte[] sessionBaseKey = null;\n\n        // Windows FileTime timestamp - number of 100 nanosecond ticks since Windows Epoch time\n        private byte[] timestamp = null;\n\n        // target info field from Challenge msg\n        private byte[] targetInfo = null;\n\n        // server challenge from Challenge msg\n        private byte[] serverChallenge = new byte[NTLM_SERVER_CHALLENGE_LENGTH];\n\n        /**\n         * Section 3.1.5.1.2 Client Receives a CHALLENGE_MESSAGE from the Server\n         * \n         * Save messages for calculating MIC\n         */\n        private byte[] negotiateMsg = null;\n        private byte[] challengeMsg = null;\n\n        /**\n         * Creates an NTLM client context\n         *\n         * @param con\n         *        connection to SQL server\n         * @param domainName\n         *        domain name to authentication in using NTLM\n         * @param userName\n         *        user name\n         * @param passwordHash\n         *        password hash\n         * @param workstation\n         *        hostname of the workstation\n         * @throws SQLServerException\n         *         if error occurs\n         */\n        NTLMContext(final SQLServerConnection con, final String domainName, final String userName,\n                final byte[] passwordHash, final String workstation) throws SQLServerException {\n\n            this.domainName = domainName.toUpperCase();\n            this.domainUbytes = unicode(this.domainName);\n\n            this.userNameUbytes = null != userName ? unicode(userName) : null;\n            this.upperUserName = null != userName ? userName.toUpperCase() : null;\n\n            this.passwordHash = passwordHash;\n\n            this.workstation = workstation;\n\n            String spn = null != con ? getSpn(con) : null;\n            this.spnUbytes = null != spn ? unicode(spn) : null;\n\n            if (logger.isLoggable(Level.FINEST)) {\n                logger.finest(toString() + \" SPN detected: \" + spn);\n            }\n\n            try {\n                mac = Mac.getInstance(\"HmacMD5\");\n            } catch (NoSuchAlgorithmException e) {\n                MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_ntlmHmacMD5Error\"));\n                Object[] msgArgs = {domainName, e.getMessage()};\n                throw new SQLServerException(form.format(msgArgs), e);\n            }\n        }\n    }\n\n    // Handle to NTLM context\n    private NTLMContext context = null;\n\n    /**\n     * Creates an instance of the NTLM authentication\n     *\n     * @param con\n     *        connection to SQL server\n     * @param domainName\n     *        domain name used for NTLM authentication\n     * @param userName\n     *        domain user\n     * @param passwordHash\n     *        password hash\n     * @param workstation\n     *        hostname of the workstation\n     * @throws SQLServerException\n     *         if error occurs\n     */\n    NTLMAuthentication(final SQLServerConnection con, final String domainName, final String userName,\n            final byte[] passwordHash, final String workstation) throws SQLServerException {\n        if (null == context) {\n            this.context = new NTLMContext(con, domainName, userName, passwordHash, workstation);\n        }\n    }\n\n    /**\n     * Generates the NTLM client context\n     *\n     * @param inToken\n     *        SSPI input blob\n     * @param done\n     *        indicates processing is done\n     * @return NTLM client context\n     * @throws SQLServerException\n     *         if error occurs\n     */\n    @Override\n    byte[] generateClientContext(final byte[] inToken, final boolean[] done) throws SQLServerException {\n        return initializeSecurityContext(inToken, done);\n    }\n\n    /**\n     * Releases the NTLM client context\n     *\n     * @throws SQLServerException\n     *         if error occurs\n     */\n    @Override\n    void releaseClientContext() {\n        context = null;\n    }\n\n    /**\n     * Parses the Type 2 NTLM Challenge message from server\n     *\n     * Section 2.2.1.2 CHALLENGE_MESSAGE\n     *\n     * @param inToken\n     *        SSPI input blob\n     * @throws SQLServerException\n     *         if error occurs\n     */\n    private void parseNtlmChallenge(final byte[] inToken) throws SQLServerException {\n        // get token buffer\n        ByteBuffer token = ByteBuffer.wrap(inToken).order(ByteOrder.LITTLE_ENDIAN);\n\n        // verify signature\n        byte[] signature = new byte[NTLM_HEADER_SIGNATURE.length];\n        token.get(signature);\n        if (!Arrays.equals(signature, NTLM_HEADER_SIGNATURE)) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_ntlmSignatureError\"));\n            Object[] msgArgs = {signature};\n            throw new SQLServerException(form.format(msgArgs), null);\n        }\n\n        // verify message type\n        int messageType = token.getInt();\n        if (messageType != NTLM_MESSAGE_TYPE_CHALLENGE) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_ntlmMessageTypeError\"));\n            Object[] msgArgs = {messageType};\n            throw new SQLServerException(form.format(msgArgs), null);\n        }\n\n        // target name fields\n        int targetNameLen = token.getShort();\n        token.getShort(); // targetNameMaxLen\n        token.getInt(); // targetNameOffset\n\n        // negotiate flags\n        token.getInt();\n\n        // get server challenge\n        token.get(context.serverChallenge);\n\n        // 8 bytes reserved 32-40: all 0's\n        token.getLong();\n\n        // target info fields\n        int targetInfoLen = token.getShort();\n        token.getShort(); // targetInfoMaxLen\n        token.getInt(); // targetInfoOffset\n\n        token.getLong(); // version - not used, for debug only\n\n        // get requested target name\n        byte[] targetName = new byte[targetNameLen];\n        token.get(targetName);\n\n        // verify targetInfo - was requested so should always be sent\n        context.targetInfo = new byte[targetInfoLen];\n        token.get(context.targetInfo);\n        if (0 == context.targetInfo.length) {\n            throw new SQLServerException(SQLServerException.getErrString(\"R_ntlmNoTargetInfo\"), null);\n        }\n\n        // parse target info AV pairs\n        ByteBuffer targetInfoBuf = ByteBuffer.wrap(context.targetInfo).order(ByteOrder.LITTLE_ENDIAN);\n        boolean done = false;\n        for (int i = 0; i < context.targetInfo.length && !done;) {\n            int id = targetInfoBuf.getShort();\n            byte[] value = new byte[targetInfoBuf.getShort()];\n            targetInfoBuf.get(value);\n            switch (id) {\n                case NTLM_AVID_MSVAVTIMESTAMP:\n                    if (value.length > 0) {\n                        context.timestamp = new byte[NTLM_TIMESTAMP_LENGTH];\n                        System.arraycopy(value, 0, context.timestamp, 0, NTLM_TIMESTAMP_LENGTH);\n                    }\n                    break;\n                case NTLM_AVID_MSVAVEOL:\n                    done = true;\n                    break;\n                case NTLM_AVID_MSVAVDNSDOMAINNAME:\n                case NTLM_AVID_MSVAVDNSCOMPUTERNAME:\n                case NTLM_AVID_MSVAVFLAGS:\n                case NTLM_AVID_MSVAVNBCOMPUTERNAME:\n                case NTLM_AVID_MSVAVNBDOMAINNAME:\n                case NTLM_AVID_MSVAVDNSTREENAME:\n                case NTLM_AVID_MSVAVSINGLEHOST:\n                case NTLM_AVID_MSVAVTARGETNAME:\n                    break;\n                default:\n                    MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_ntlmUnknownValue\"));\n                    Object[] msgArgs = {value};\n                    throw new SQLServerException(form.format(msgArgs), null);\n            }\n\n            if (logger.isLoggable(Level.FINEST)) {\n                logger.finest(toString() + \" NTLM Challenge Message target info: AvId \" + id);\n            }\n        }\n\n        /**\n         * Section 2.2.2.1 AV_PAIR\n         *\n         * Section 7 Appendix B Product Behavior\n         * \n         * This structure should always be sent in the CHALLENGE_MESSAGE not supported in Windows NT, Windows 2000,\n         * Windows XP, and Windows Server 2003\n         */\n        if (null == context.timestamp || 0 >= context.timestamp.length) {\n            // this SHOULD always be present but for some reason occasionally this had seen to be missing\n            if (logger.isLoggable(Level.WARNING)) {\n                logger.warning(toString() + \" NTLM Challenge Message target info error: Missing timestamp.\");\n            }\n        } else {\n            // save msg for calculating MIC in Authenticate msg\n            context.challengeMsg = new byte[inToken.length];\n            System.arraycopy(inToken, 0, context.challengeMsg, 0, inToken.length);\n        }\n    }\n\n    /**\n     * Initializes the NTLM client security context\n     *\n     * @param inToken\n     *        SSPI input blob\n     * @param done\n     *        indicates processing is done\n     * @return outbound security context\n     * @throws SQLServerException\n     *         if error occurs\n     */\n    private byte[] initializeSecurityContext(final byte[] inToken, final boolean[] done) throws SQLServerException {\n        if (null == inToken || 0 == inToken.length) {\n            return generateNtlmNegotiate();\n        } else {\n            // get challenge msg from server\n            parseNtlmChallenge(inToken);\n\n            // get authenticate msg\n            done[0] = true;\n            return generateNtlmAuthenticate();\n        }\n    }\n\n    /**\n     * Generates NTLMv2 Client Challenge blob\n     *\n     * <pre>\n     * Section 2.2.2.7 NTLM v2: NTLMv2_CLIENT_CHALLENGE:\n     *\n     * clientChallenge blob consists of:\n     *     Responserversion,\n     *     HiResponserversion,    NTLM_CLIENT_CHALLENGE_RESPONSE_TYPE,\n     *     Z(6),                  NTLM_CLIENT_CHALLENGE_RESERVED1, NTLM_CLIENT_CHALLENGE_RESERVED2\n     *     Time,                  current timestamp\n     *     ClientChallenge,       clientNonce\n     *     Z(4),                  NTLM_CLIENT_CHALLENGE_RESERVED2\n     *     ServerName,            CHALLENGE_MESSAGE.TargetInfo (from 3.1.5.1.2 Client Receives a CHALLENGE_MESSAGE from the Server)\n     *     Z(4))                  EOL\n     * </pre>\n     *\n     * @param clientNonce\n     *        client challenge nonce\n     * @return client challenge blob\n     */\n    private byte[] generateClientChallengeBlob(final byte[] clientNonce) {\n        // timestamp is number of 100 nanosecond ticks since Windows Epoch\n        ByteBuffer time = ByteBuffer.allocate(NTLM_TIMESTAMP_LENGTH).order(ByteOrder.LITTLE_ENDIAN);\n        time.putLong((TimeUnit.SECONDS.toNanos(Instant.now().getEpochSecond() + WINDOWS_EPOCH_DIFF)) / 100);\n        byte[] currentTime = time.array();\n\n        // allocate token buffer\n        ByteBuffer token = ByteBuffer\n                .allocate(NTLM_CLIENT_CHALLENGE_RESPONSE_TYPE.length + NTLM_CLIENT_CHALLENGE_RESERVED1.length\n                        + NTLM_CLIENT_CHALLENGE_RESERVED2.length + currentTime.length + NTLM_CLIENT_NONCE_LENGTH\n                        + NTLM_CLIENT_CHALLENGE_RESERVED3.length + context.targetInfo.length\n                        + /* add MIC */ NTLM_AVID_LENGTH + NTLM_AVLEN_LENGTH + NTLM_AVID_MSVAVFLAGS_LEN\n                        + /* add SPN */ NTLM_AVID_LENGTH + NTLM_AVLEN_LENGTH + context.spnUbytes.length)\n                .order(ByteOrder.LITTLE_ENDIAN);\n\n        token.put(NTLM_CLIENT_CHALLENGE_RESPONSE_TYPE);\n        token.put(NTLM_CLIENT_CHALLENGE_RESERVED1);\n        token.put(NTLM_CLIENT_CHALLENGE_RESERVED2);\n\n        token.put(currentTime, 0, NTLM_TIMESTAMP_LENGTH);\n        token.put(clientNonce, 0, NTLM_CLIENT_NONCE_LENGTH);\n        token.put(NTLM_CLIENT_CHALLENGE_RESERVED3);\n\n        /**\n         * Section 3.1.5.1.2 Client Receives a CHALLENGE_MESSAGE from the Server\n         *\n         * If the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present, the client SHOULD provide a MIC\n         */\n        if (null == context.timestamp || 0 >= context.timestamp.length) {\n            token.put(context.targetInfo, 0, context.targetInfo.length);\n            if (logger.isLoggable(Level.WARNING)) {\n                logger.warning(toString()\n                        + \" MsvAvTimestamp not recieved from SQL Server in Challenge Message. MIC field will not be set.\");\n            }\n        } else {\n            // copy targetInfo up to NTLM_AVID_MSVAVEOL\n            token.put(context.targetInfo, 0, context.targetInfo.length - NTLM_AVID_LENGTH - NTLM_AVLEN_LENGTH);\n\n            // MIC\n            token.putShort(NTLM_AVID_MSVAVFLAGS);\n            token.putShort((short) NTLM_AVID_MSVAVFLAGS_LEN);\n            token.putInt((int) NTLM_AVFLAG_VALUE_MIC);\n        }\n\n        // SPN\n        token.putShort(NTLM_AVID_MSVAVTARGETNAME);\n        token.putShort((short) context.spnUbytes.length);\n        token.put(context.spnUbytes, 0, context.spnUbytes.length);\n\n        // EOL\n        token.putShort(NTLM_AVID_MSVAVEOL);\n        token.putShort((short) 0);\n\n        return token.array();\n    }\n\n    /**\n     * Gets the HMAC MD5 hash\n     *\n     * @see <a href=\"https://www.ietf.org/rfc/rfc2104.txt\">https://www.ietf.org/rfc/rfc2104.txt</a>\n     * \n     * @param key\n     *        key used for hash\n     * @param data\n     *        input data\n     * @return HMAC MD5 hash\n     * @throws InvalidKeyException\n     *         if key is invalid\n     */\n    private byte[] hmacMD5(final byte[] key, final byte[] data) throws InvalidKeyException {\n        SecretKeySpec keySpec = new SecretKeySpec(key, \"HmacMD5\");\n        context.mac.init(keySpec);\n        return context.mac.doFinal(data);\n    }\n\n    /**\n     * Gets the MD4 hash of input string\n     *\n     * @param str\n     *        input string\n     * @return MD4 hash\n     */\n    private static byte[] md4(final byte[] str) {\n        MD4 md = new MD4();\n        md.reset();\n        md.update(str, 0, str.length);\n        byte[] hash = new byte[md.getDigestSize()];\n        md.doFinal(hash, 0);\n        return hash;\n    }\n\n    /**\n     * Gets the unicode of a string\n     *\n     * @param str\n     *        string to convert to unicode\n     * @return unicode of string\n     */\n    private static byte[] unicode(final String str) {\n        return (null != str) ? str.getBytes(java.nio.charset.StandardCharsets.UTF_16LE) : null;\n    }\n\n    /**\n     * Concatenates 2 byte arrays\n     *\n     * @param arr1\n     *        array 1\n     * @param arr2\n     *        array 2\n     * @return concatenated array of arr1 and and arr2\n     */\n    private byte[] concat(final byte[] arr1, final byte[] arr2) {\n        if (null == arr1 || null == arr2) {\n            return null;\n        }\n\n        byte[] temp = new byte[arr1.length + arr2.length];\n        System.arraycopy(arr1, 0, temp, 0, arr1.length);\n        System.arraycopy(arr2, 0, temp, arr1.length, arr2.length);\n        return temp;\n    }\n\n    /**\n     * Get length of byte array\n     * \n     * @param arr\n     *        array to get length of\n     * @return array length or 0 if null array\n     */\n    private int getByteArrayLength(byte[] arr) {\n        return null == arr ? 0 : arr.length;\n    }\n\n    /**\n     * Generates a key from password to get NTLMv2 hash Section 3.3.2 NTLM v2 Authentication\n     *\n     * @return NT response key\n     * @throws InvalidKeyException\n     *         if error getting hash due to invalid key\n     */\n    private byte[] ntowfv2() throws InvalidKeyException {\n        return hmacMD5(context.passwordHash,\n                (null != context.upperUserName) ? unicode(context.upperUserName + context.domainName)\n                                                : unicode(context.domainName));\n    }\n\n    /**\n     * Computes the NT response and keys from the Challenge message\n     *\n     * Section 3.3.2 NTLM v2 Authentication\n     *\n     * <pre>\n     * Set temp to ConcatenationOf(Responserversion, HiResponserversion, \n     * Z(6), Time, ClientChallenge, Z(4), ServerName, Z(4)) \n     * Set NTProofStr to HMAC_MD5(ResponseKeyNT,  \n     * ConcatenationOf(CHALLENGE_MESSAGE.ServerChallenge,temp)) \n     * Set NtChallengeResponse to ConcatenationOf(NTProofStr, temp) \n     * Set LmChallengeResponse to ConcatenationOf(HMAC_MD5(ResponseKeyLM,  \n     * ConcatenationOf(CHALLENGE_MESSAGE.ServerChallenge, ClientChallenge)), \n     * ClientChallenge )\n     * </pre>\n     *\n     * @param responseKeyNT\n     *        NT response hash key\n     * @return computed response\n     * @throws InvalidKeyException\n     *         if error getting hash due to invalid key\n     */\n    \nprivate byte[] computeResponse(final byte[] responseKeyNT) throws InvalidKeyException {\n        // get random client challenge nonce\n        byte[] clientNonce = new byte[NTLM_CLIENT_NONCE_LENGTH];\n        ThreadLocalRandom.current().nextBytes(clientNonce);\n\n        // get client challenge blob\n        byte[] temp = generateClientChallengeBlob(clientNonce);\n\n        byte[] ntProofStr = hmacMD5(responseKeyNT, concat(context.serverChallenge, temp));\n\n        context.sessionBaseKey = hmacMD5(responseKeyNT, ntProofStr);\n\n        return concat(ntProofStr, temp);\n    }\n\n    /**\n     * Generates the NT Challenge response to server Challenge\n     *\n     * Section 3.3.2 NTLM v2 Authentication\n     *\n     * @return NT challenge response\n     * @throws InvalidKeyException\n     *         if error getting hash due to invalid key\n     */\n    private byte[] getNtChallengeResp() throws InvalidKeyException {\n        byte[] responseKeyNT = ntowfv2();\n        return computeResponse(responseKeyNT);\n    }\n\n    /**\n     * Generates the Type 3 NTLM Authentication message\n     *\n     * Section 2.2.1.3 AUTHENTICATE_MESSAGE\n     *\n     * @return NTLM Authenticate message\n     * @throws SQLServerException\n     *         if error occurs\n     */\n    private byte[] generateNtlmAuthenticate() throws SQLServerException {\n        int domainNameLen = getByteArrayLength(context.domainUbytes);\n        int userNameLen = getByteArrayLength(context.userNameUbytes);\n        byte[] workstationBytes = unicode(context.workstation);\n        int workstationLen = getByteArrayLength(workstationBytes);\n        byte[] msg = null;\n\n        try {\n            // get NT challenge response\n            byte[] ntChallengeResp = getNtChallengeResp();\n            int ntChallengeLen = getByteArrayLength(ntChallengeResp);\n\n            // allocate token buffer\n            ByteBuffer token = ByteBuffer.allocate(NTLM_AUTHENTICATE_PAYLOAD_OFFSET + NTLM_LMCHALLENAGERESPONSE.length\n                    + ntChallengeLen + domainNameLen + userNameLen + workstationLen).order(ByteOrder.LITTLE_ENDIAN);\n\n            // set NTLM signature and message type\n            token.put(NTLM_HEADER_SIGNATURE, 0, NTLM_HEADER_SIGNATURE.length);\n            token.putInt(NTLM_MESSAGE_TYPE_AUTHENTICATE);\n\n            // start of payload data\n            int offset = NTLM_AUTHENTICATE_PAYLOAD_OFFSET;\n\n            // LM challenge response\n            token.putShort((short) 0);\n            token.putShort((short) 0);\n            token.putInt(offset);\n            offset += NTLM_LMCHALLENAGERESPONSE.length;\n\n            // NT challenge response\n            token.putShort((short) ntChallengeLen);\n            token.putShort((short) ntChallengeLen);\n            token.putInt(offset);\n            offset += ntChallengeLen;\n\n            // domain name fields\n            token.putShort((short) domainNameLen);\n            token.putShort((short) domainNameLen);\n            token.putInt(offset);\n            offset += domainNameLen;\n\n            // user name fields\n            token.putShort((short) userNameLen);\n            token.putShort((short) userNameLen);\n            token.putInt(offset);\n            offset += userNameLen;\n\n            // workstation fields\n            token.putShort((short) workstationLen);\n            token.putShort((short) workstationLen);\n            token.putInt(offset);\n            offset += workstationLen;\n\n            // not used = encrypted random session key fields\n            token.putShort((short) 0);\n            token.putShort((short) 0);\n            token.putInt(offset);\n\n            // same negotiate flags sent in negotiate message\n            token.putInt((int) context.negotiateFlags);\n\n            // version - not used but need to send blank to separate from MIC otherwise server confuses this with MIC\n            token.put(NTLMSSP_VERSION, 0, NTLMSSP_VERSION.length);\n\n            // 0 the MIC field first for calculation\n            byte[] mic = new byte[NTLM_MIC_LENGTH];\n            int micPosition = token.position(); // save position\n            token.put(mic, 0, NTLM_MIC_LENGTH);\n\n            // payload data\n            token.put(NTLM_LMCHALLENAGERESPONSE, 0, NTLM_LMCHALLENAGERESPONSE.length);\n            token.put(ntChallengeResp, 0, ntChallengeLen);\n            token.put(context.domainUbytes, 0, domainNameLen);\n            token.put(context.userNameUbytes, 0, userNameLen);\n            token.put(workstationBytes, 0, workstationLen);\n\n            msg = token.array();\n\n            /**\n             * Section 3.1.5.1.2 Client Receives a CHALLENGE_MESSAGE from the Server\n             * \n             * MIC is calculated by using a 0 MIC then hmacMD5 of session key and concat of the 3 msgs\n             */\n            if (null != context.timestamp && 0 < context.timestamp.length) {\n                SecretKeySpec keySpec = new SecretKeySpec(context.sessionBaseKey, \"HmacMD5\");\n                context.mac.init(keySpec);\n                context.mac.update(context.negotiateMsg);\n                context.mac.update(context.challengeMsg);\n                mic = context.mac.doFinal(msg);\n\n                // put calculated MIC into Authenticate msg\n                System.arraycopy(mic, 0, msg, micPosition, NTLM_MIC_LENGTH);\n            }\n        } catch (InvalidKeyException e) {\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_ntlmAuthenticateError\"));\n            Object[] msgArgs = {e.getMessage()};\n            throw new SQLServerException(form.format(msgArgs), e);\n        }\n\n        return msg;\n    }\n\n    /**\n     * Generates the Type 1 NTLM Negotiate message\n     *\n     * Section 2.2.1.1 NEGOTIATE_MESSAGE\n     *\n     * @return NTLM Negotiate message\n     */\n    private byte[] generateNtlmNegotiate() {\n        int domainNameLen = getByteArrayLength(context.domainUbytes);\n        int workstationLen = getByteArrayLength(context.workstation.getBytes());\n\n        ByteBuffer token = null;\n        token = ByteBuffer.allocate(NTLM_NEGOTIATE_PAYLOAD_OFFSET + domainNameLen + workstationLen)\n                .order(ByteOrder.LITTLE_ENDIAN);\n\n        // signature and message type\n        token.put(NTLM_HEADER_SIGNATURE, 0, NTLM_HEADER_SIGNATURE.length);\n        token.putInt(NTLM_MESSAGE_TYPE_NEGOTIATE);\n\n        // NTLM negotiate flags - only NTLMV2 supported\n        context.negotiateFlags = NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED | NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED\n                | NTLMSSP_REQUEST_TARGET | NTLMSSP_NEGOTIATE_TARGET_INFO | NTLMSSP_NEGOTIATE_UNICODE\n                | NTLMSSP_NEGOTIATE_ALWAYS_SIGN | NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY;\n        token.putInt((int) context.negotiateFlags);\n\n        int offset = NTLM_NEGOTIATE_PAYLOAD_OFFSET;\n\n        // domain name fields\n        token.putShort((short) domainNameLen);\n        token.putShort((short) domainNameLen);\n        token.putInt(offset);\n        offset += domainNameLen;\n\n        // workstation field\n        token.putShort((short) workstationLen);\n        token.putShort((short) workstationLen);\n        token.putInt(offset);\n\n        // offset workstationLen\n        // version - not used, for debug only\n\n        // payload\n        token.put(context.domainUbytes, 0, domainNameLen);\n        token.put(context.workstation.getBytes(), 0, workstationLen);\n\n        // save msg for calculating MIC in Authenticate msg\n        byte[] msg = token.array();\n        context.negotiateMsg = new byte[msg.length];\n        System.arraycopy(msg, 0, context.negotiateMsg, 0, msg.length);\n\n        return msg;\n    }\n\n    public static byte[] getNtlmPasswordHash(String password) throws SQLServerException {\n        if (null == password) {\n            throw new SQLServerException(SQLServerException.getErrString(\"R_NtlmNoUserPasswordDomain\"), null);\n        }\n\n        return md4(unicode(password));\n    }\n}\n\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/PersistentTokenCacheAccessAspect.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport com.microsoft.aad.msal4j.ITokenCacheAccessAspect;\nimport com.microsoft.aad.msal4j.ITokenCacheAccessContext;\n\nimport java.util.concurrent.locks.Lock;\n...\n", "needle_token_start": 13673, "needle_token_end": 13833, "code_context_ntokens": 16163, "output": ["```java\n/**\n * Computes the NT Challenge response to server Challenge\n *\n * Section 3.3.2 NTLM v2 Authentication\n *\n * @return NT challenge response\n * @throws InvalidKeyException\n *         if error getting hash due to invalid key\n */\nprivate byte[] getNtChallengeResp() throws InvalidKeyException {\n    byte[] responseKeyNT = ntowfv2();\n    return computeResponse(responseKeyNT);\n}\n```"]}
{"repo": "microsoft/mssql-jdbc", "name": "getParameterInfo", "language": "java", "path": "src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve metadata about a specific parameter used in a stored procedure or a parameterized SQL statement.\n2. **Input**: It accepts an integer representing the index of the parameter for which metadata is required.\n3. **Output**: Returns a map containing metadata about the specified parameter, such as data type, column type, precision, scale, and nullability.\n4. **Procedure**: The function first checks if the SQL statement involves a table-valued parameter (TVP) and if it is structured to return a value. Depending on these conditions, it adjusts the index to access the correct metadata from a pre-stored list of metadata for each parameter. The metadata is then retrieved from this list based on the adjusted index and returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerDatabaseMetaData.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.io.Serializable;\nimport java.sql.CallableStatement;\nimport java.sql.Connection;\nimport java.sql.DriverPropertyInfo;\nimport java.sql.PreparedStatement;\nimport java.sql.ResultSet;\nimport java.sql.RowIdLifetime;\nimport java.sql.SQLException;\nimport java.sql.SQLTimeoutException;\nimport java.text.MessageFormat;\nimport java.util.EnumMap;\nimport java.util.HashMap;\nimport java.util.LinkedHashMap;\nimport java.util.Map;\nimport java.util.Map.Entry;\nimport java.util.Properties;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.locks.Lock;\nimport java.util.concurrent.locks.ReentrantLock;\nimport java.util.logging.Level;\n\n\n/**\n * Provides the JDBC database meta data.\n *\n * The API javadoc for JDBC API methods that this class implements are not repeated here. Please see Sun's JDBC API\n * interfaces javadoc for those details.\n */\n@SuppressWarnings(\"unused\")\npublic final class SQLServerDatabaseMetaData implements java.sql.DatabaseMetaData, Serializable {\n    /**\n     * Always update serialVersionUID when prompted.\n     */\n    private static final long serialVersionUID = -116977606028371577L;\n\n    /** connection */\n    private SQLServerConnection connection;\n\n    static final String URL_PREFIX = \"jdbc:sqlserver://\";\n\n    static final private java.util.logging.Logger logger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.SQLServerDatabaseMetaData\");\n\n    static final private java.util.logging.Logger loggerExternal = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.DatabaseMetaData\");\n\n    /**\n     * Unique id generator for each instance (used for logging).\n     */\n    static private final AtomicInteger baseID = new AtomicInteger(0);\n\n    /** trace ID */\n    final private String traceID;\n\n    // varbinary(max) https://msdn.microsoft.com/en-us/library/ms143432.aspx\n    static final int MAXLOBSIZE = 2147483647;\n    // uniqueidentifier https://msdn.microsoft.com/en-us/library/ms187942.aspx\n    static final int UNIQUEIDENTIFIER_SIZE = 36;\n\n    enum CallableHandles {\n        SP_COLUMNS(\"{ call sp_columns(?, ?, ?, ?, ?) }\", \"{ call sp_columns_100(?, ?, ?, ?, ?, ?) }\"),\n        SP_COLUMN_PRIVILEGES(\"{ call sp_column_privileges(?, ?, ?, ?)}\", \"{ call sp_column_privileges(?, ?, ?, ?)}\"),\n        SP_TABLES(\"{ call sp_tables(?, ?, ?, ?) }\", \"{ call sp_tables(?, ?, ?, ?) }\"),\n        SP_SPECIAL_COLUMNS(\"{ call sp_special_columns (?, ?, ?, ?, ?, ?, ?)}\", \"{ call sp_special_columns_100 (?, ?, ?, ?, ?, ?, ?)}\"),\n        SP_FKEYS(\"{ call sp_fkeys (?, ?, ?, ? , ? ,?)}\", \"{ call sp_fkeys (?, ?, ?, ? , ? ,?)}\"),\n        SP_STATISTICS(\"{ call sp_statistics(?,?,?,?,?, ?) }\", \"{ call sp_statistics_100(?,?,?,?,?, ?) }\"),\n        SP_SPROC_COLUMNS(\"{ call sp_sproc_columns(?, ?, ?,?,?) }\", \"{ call sp_sproc_columns_100(?, ?, ?,?,?) }\"),\n        SP_STORED_PROCEDURES(\"{call sp_stored_procedures(?, ?, ?) }\", \"{call sp_stored_procedures(?, ?, ?) }\"),\n        SP_TABLE_PRIVILEGES(\"{call sp_table_privileges(?,?,?) }\", \"{call sp_table_privileges(?,?,?) }\"),\n        SP_PKEYS(\"{ call sp_pkeys (?, ?, ?)}\", \"{ call sp_pkeys (?, ?, ?)}\");\n\n        // stored procs before Katmai ie SS10\n        private final String preKatProc;\n        // procs on or after katmai\n        private final String katProc;\n\n        private CallableHandles(String name, String katName) {\n            this.preKatProc = name;\n            this.katProc = katName;\n        }\n\n        CallableStatement prepare(SQLServerConnection conn) throws SQLServerException {\n            return conn.prepareCall(conn.isKatmaiOrLater() ? katProc : preKatProc);\n        }\n    }\n\n    final class HandleAssociation {\n        Map<String, CallableStatement> statementMap;\n        boolean nullCatalog = false;\n        CallableStatement stmt;\n\n        HandleAssociation() {\n            if (null == statementMap) {\n                statementMap = new HashMap<>();\n            }\n        }\n\n        final void addToMap(String databaseName, CallableStatement stmt) {\n            if (null != databaseName) {\n                nullCatalog = false;\n                statementMap.put(databaseName, stmt);\n            } else {\n                nullCatalog = true;\n                this.stmt = stmt;\n            }\n        }\n\n        final CallableStatement getMappedStatement(String databaseName) {\n            if (null != databaseName) {\n                if (null != statementMap && statementMap.containsKey(databaseName)) {\n                    return statementMap.get(databaseName);\n                }\n                return null;\n            } else {\n                return stmt;\n            }\n        }\n    }\n\n    /** handle map */\n    EnumMap<CallableHandles, HandleAssociation> handleMap = new EnumMap<>(CallableHandles.class);\n\n    // Returns unique id for each instance.\n    private static int nextInstanceID() {\n        return baseID.incrementAndGet();\n    }\n\n    /**\n     * Provides a helper function to provide an ID string suitable for tracing.\n     * \n     * @return traceID string\n     */\n    final public String toString() {\n        return traceID;\n    }\n\n    /**\n     * Constructs a SQLServerDatabaseMetaData database meta data\n     * \n     * @param con\n     *        the connection\n     */\n    public SQLServerDatabaseMetaData(SQLServerConnection con) {\n        traceID = \" SQLServerDatabaseMetaData:\" + nextInstanceID();\n        connection = con;\n        if (logger.isLoggable(java.util.logging.Level.FINE)) {\n            logger.fine(toString() + \" created by (\" + connection.toString() + \")\");\n        }\n    }\n\n    @Override\n    public boolean isWrapperFor(Class<?> iface) throws SQLException {\n        return iface.isInstance(this);\n    }\n\n    @Override\n    public <T> T unwrap(Class<T> iface) throws SQLException {\n        T t;\n        try {\n            t = iface.cast(this);\n        } catch (ClassCastException e) {\n            throw new SQLServerException(e.getMessage(), e);\n        }\n        return t;\n    }\n\n    private void checkClosed() throws SQLServerException {\n        if (connection.isClosed()) {\n            SQLServerException.makeFromDriverError(null, null, SQLServerException.getErrString(\"R_connectionIsClosed\"),\n                    SQLServerException.EXCEPTION_XOPEN_CONNECTION_DOES_NOT_EXIST, false);\n        }\n    }\n\n    private static final String ASC_OR_DESC = \"ASC_OR_DESC\";\n    private static final String BUFFER_LENGTH = \"BUFFER_LENGTH\";\n    private static final String CARDINALITY = \"CARDINALITY\";\n    private static final String CHAR_OCTET_LENGTH = \"CHAR_OCTET_LENGTH\";\n    private static final String COLUMN_DEF = \"COLUMN_DEF\";\n    private static final String COLUMN_NAME = \"COLUMN_NAME\";\n    private static final String COLUMN_SIZE = \"COLUMN_SIZE\";\n    private static final String COLUMN_TYPE = \"COLUMN_TYPE\";\n    private static final String DATA_TYPE = \"DATA_TYPE\";\n    private static final String DECIMAL_DIGITS = \"DECIMAL_DIGITS\";\n    private static final String DEFERRABILITY = \"DEFERRABILITY\";\n    private static final String DELETE_RULE = \"DELETE_RULE\";\n    private static final String FILTER_CONDITION = \"FILTER_CONDITION\";\n    private static final String FK_NAME = \"FK_NAME\";\n    private static final String FKCOLUMN_NAME = \"FKCOLUMN_NAME\";\n    private static final String FKTABLE_CAT = \"FKTABLE_CAT\";\n    private static final String FKTABLE_NAME = \"FKTABLE_NAME\";\n    private static final String FKTABLE_SCHEM = \"FKTABLE_SCHEM\";\n    private static final String GRANTEE = \"GRANTEE\";\n    private static final String GRANTOR = \"GRANTOR\";\n    private static final String INDEX_NAME = \"INDEX_NAME\";\n    private static final String INDEX_QUALIFIER = \"INDEX_QUALIFIER\";\n    private static final String IS_GRANTABLE = \"IS_GRANTABLE\";\n    private static final String IS_NULLABLE = \"IS_NULLABLE\";\n    private static final String KEY_SEQ = \"KEY_SEQ\";\n    private static final String LENGTH = \"LENGTH\";\n    private static final String NON_UNIQUE = \"NON_UNIQUE\";\n    private static final String NULLABLE = \"NULLABLE\";\n    private static final String NUM_INPUT_PARAMS = \"NUM_INPUT_PARAMS\";\n    private static final String NUM_OUTPUT_PARAMS = \"NUM_OUTPUT_PARAMS\";\n    private static final String NUM_PREC_RADIX = \"NUM_PREC_RADIX\";\n    private static final String NUM_RESULT_SETS = \"NUM_RESULT_SETS\";\n    private static final String ORDINAL_POSITION = \"ORDINAL_POSITION\";\n    private static final String PAGES = \"PAGES\";\n    private static final String PK_NAME = \"PK_NAME\";\n    private static final String PKCOLUMN_NAME = \"PKCOLUMN_NAME\";\n    private static final String PKTABLE_CAT = \"PKTABLE_CAT\";\n    private static final String PKTABLE_NAME = \"PKTABLE_NAME\";\n    private static final String PKTABLE_SCHEM = \"PKTABLE_SCHEM\";\n    private static final String PRECISION = \"PRECISION\";\n    private static final String PRIVILEGE = \"PRIVILEGE\";\n    private static final String PROCEDURE_CAT = \"PROCEDURE_CAT\";\n    private static final String PROCEDURE_NAME = \"PROCEDURE_NAME\";\n    private static final String PROCEDURE_SCHEM = \"PROCEDURE_SCHEM\";\n    private static final String PROCEDURE_TYPE = \"PROCEDURE_TYPE\";\n    private static final String PSEUDO_COLUMN = \"PSEUDO_COLUMN\";\n    private static final String RADIX = \"RADIX\";\n    private static final String REMARKS = \"REMARKS\";\n    private static final String SCALE = \"SCALE\";\n    private static final String SCOPE = \"SCOPE\";\n    private static final String SCOPE_CATALOG = \"SCOPE_CATALOG\";\n    private static final String SCOPE_SCHEMA = \"SCOPE_SCHEMA\";\n    private static final String SCOPE_TABLE = \"SCOPE_TABLE\";\n    private static final String SOURCE_DATA_TYPE = \"SOURCE_DATA_TYPE\";\n    private static final String SQL_DATA_TYPE = \"SQL_DATA_TYPE\";\n    private static final String SQL_DATETIME_SUB = \"SQL_DATETIME_SUB\";\n    private static final String TABLE_CAT = \"TABLE_CAT\";\n    private static final String TABLE_NAME = \"TABLE_NAME\";\n    private static final String TABLE_SCHEM = \"TABLE_SCHEM\";\n    private static final String TABLE_TYPE = \"TABLE_TYPE\";\n    private static final String TYPE = \"TYPE\";\n    private static final String TYPE_NAME = \"TYPE_NAME\";\n    private static final String UPDATE_RULE = \"UPDATE_RULE\";\n    private static final String FUNCTION_CAT = \"FUNCTION_CAT\";\n    private static final String FUNCTION_NAME = \"FUNCTION_NAME\";\n    private static final String FUNCTION_SCHEM = \"FUNCTION_SCHEM\";\n    private static final String FUNCTION_TYPE = \"FUNCTION_TYPE\";\n    private static final String SS_IS_SPARSE = \"SS_IS_SPARSE\";\n    private static final String SS_IS_COLUMN_SET = \"SS_IS_COLUMN_SET\";\n    private static final String SS_UDT_CATALOG_NAME = \"SS_UDT_CATALOG_NAME\";\n    private static final String SS_UDT_SCHEMA_NAME = \"SS_UDT_SCHEMA_NAME\";\n    private static final String SS_UDT_ASSEMBLY_TYPE_NAME = \"SS_UDT_ASSEMBLY_TYPE_NAME\";\n    private static final String SS_XML_SCHEMACOLLECTION_CATALOG_NAME = \"SS_XML_SCHEMACOLLECTION_CATALOG_NAME\";\n    private static final String SS_XML_SCHEMACOLLECTION_SCHEMA_NAME = \"SS_XML_SCHEMACOLLECTION_SCHEMA_NAME\";\n    private static final String SS_XML_SCHEMACOLLECTION_NAME = \"SS_XML_SCHEMACOLLECTION_NAME\";\n\n    private static final String IS_GENERATEDCOLUMN = \"IS_GENERATEDCOLUMN\";\n    private static final String IS_AUTOINCREMENT = \"IS_AUTOINCREMENT\";\n    private static final String ACTIVITY_ID = \" ActivityId: \";\n\n    private static final String NVARCHAR = JDBCType.NVARCHAR.name();\n    private static final String VARCHAR = JDBCType.VARCHAR.name();\n    private static final String INTEGER = JDBCType.INTEGER.name();\n    private static final String SMALLINT = JDBCType.SMALLINT.name();\n\n    private static final String SQL_KEYWORDS = createSqlKeyWords();\n\n    // Use LinkedHashMap to force retrieve elements in order they were inserted\n    /** getColumns columns */\n    private LinkedHashMap<Integer, String> getColumnsDWColumns = null;\n    private LinkedHashMap<Integer, String> getTypesDWColumns = null;\n    /** getImportedKeys columns */\n    private volatile LinkedHashMap<Integer, String> getImportedKeysDWColumns;\n    private static final Lock LOCK = new ReentrantLock();\n\n    /**\n     * Returns the result from a simple query. This is to be used only for internal queries without any user input.\n     * \n     * @param catalog\n     *        catalog the query to be made in\n     * @param query\n     *        to execute\n     * @return Resultset from the execution\n     * @throws SQLTimeoutException\n     */\n    private SQLServerResultSet getResultSetFromInternalQueries(String catalog, String query) throws SQLException {\n        checkClosed();\n        String orgCat = null;\n        orgCat = switchCatalogs(catalog);\n        SQLServerResultSet rs = null;\n        try {\n            SQLServerStatement statement = (SQLServerStatement) connection.createStatement();\n            statement.closeOnCompletion();\n            rs = statement.executeQueryInternal(query);\n        } finally {\n            if (null != orgCat) {\n                connection.setCatalog(orgCat);\n            }\n        }\n        return rs;\n    }\n\n    /**\n     * Returns the CallableStatement handle. Note we pool the handles per object.\n     */\n    private CallableStatement getCallableStatementHandle(CallableHandles request,\n            String catalog) throws SQLServerException {\n        CallableStatement cs = null;\n        HandleAssociation hassoc = handleMap.get(request);\n        try {\n            if (null == hassoc) {\n                cs = request.prepare(connection);\n                hassoc = new HandleAssociation();\n                hassoc.addToMap(catalog, cs);\n            } else { // hassoc != null\n                cs = hassoc.getMappedStatement(catalog);\n                // No Cached Statement yet\n                if (null == cs || cs.isClosed()) {\n                    cs = request.prepare(connection);\n                    hassoc.addToMap(catalog, cs);\n                }\n            }\n            handleMap.put(request, hassoc);\n        } catch (SQLException e) {\n            SQLServerException.makeFromDriverError(connection, cs, e.toString(), null, false);\n        }\n        return cs;\n    }\n\n    /**\n     * Returns the result from the stored procedure call.\n     * \n     * @param catalog\n     *        catalog the query to be made in\n     * @param procedure\n     *        to execute\n     * @param arguments\n     *        for the stored procedure\n     * @return Resultset from the execution\n     * @throws SQLTimeoutException\n     */\n    private SQLServerResultSet getResultSetFromStoredProc(String catalog, CallableHandles procedure,\n            String[] arguments) throws SQLServerException, SQLTimeoutException {\n        checkClosed();\n        assert null != arguments;\n        String orgCat = null;\n        orgCat = switchCatalogs(catalog);\n        SQLServerResultSet rs = null;\n        try {\n            SQLServerCallableStatement call = (SQLServerCallableStatement) getCallableStatementHandle(procedure,\n                    catalog);\n\n            if (call != null) {\n                for (int i = 1; i <= arguments.length; i++) {\n                    // note individual arguments can be null.\n                    call.setString(i, arguments[i - 1]);\n                }\n                rs = (SQLServerResultSet) call.executeQueryInternal();\n            }\n        } finally {\n            if (null != orgCat) {\n                connection.setCatalog(orgCat);\n            }\n        }\n\n        return rs;\n    }\n\n    private SQLServerResultSet getResultSetWithProvidedColumnNames(String catalog, CallableHandles procedure,\n            String[] arguments, String[] columnNames) throws SQLServerException, SQLTimeoutException {\n        // Execute the query\n        SQLServerResultSet rs = getResultSetFromStoredProc(catalog, procedure, arguments);\n\n        // Rename the columns\n        if (null != rs) {\n            for (int i = 0; i < columnNames.length; i++)\n                rs.setColumnName(1 + i, columnNames[i]);\n        }\n        return rs;\n    }\n\n    /**\n     * Switches the database catalogs.\n     * \n     * @param catalog\n     *        the new catalog\n     * @throws SQLServerException\n     * @return the old catalog\n     */\n    private String switchCatalogs(String catalog) throws SQLServerException {\n        if (null == catalog)\n            return null;\n        String sCurr = null;\n        sCurr = connection.getCatalog().trim();\n        String sNew = catalog.trim();\n        if (sCurr.equals(sNew))\n            return null;\n        connection.setCatalog(sNew);\n        if (null == sCurr || sCurr.length() == 0)\n            return null;\n        return sCurr;\n    }\n\n    /* -------------- JDBC Interface API starts here ---------------- */\n\n    @Override\n    public boolean allProceduresAreCallable() throws SQLServerException {\n        checkClosed();\n        return true;\n    }\n\n    @Override\n    public boolean allTablesAreSelectable() throws SQLServerException {\n        checkClosed();\n        return true;\n    }\n\n    @Override\n    public boolean autoCommitFailureClosesAllResultSets() throws SQLException {\n        checkClosed();\n        return false;\n    }\n\n    @Override\n    public boolean dataDefinitionCausesTransactionCommit() throws SQLServerException {\n        checkClosed();\n        return false;\n    }\n\n    @Override\n    public boolean dataDefinitionIgnoredInTransactions() throws SQLServerException {\n        checkClosed();\n        return false;\n    }\n\n    @Override\n    public boolean doesMaxRowSizeIncludeBlobs() throws SQLServerException {\n        checkClosed();\n        return false;\n    }\n\n    @Override\n    public boolean generatedKeyAlwaysReturned() throws SQLException {\n        checkClosed();\n        // driver supports retrieving generated keys\n        return true;\n    }\n\n    @Override\n    public long getMaxLogicalLobSize() throws SQLException {\n        checkClosed();\n        return MAXLOBSIZE;\n    }\n\n    @Override\n    public boolean supportsRefCursors() throws SQLException {\n        checkClosed();\n        return false;\n    }\n\n    @Override\n    public boolean supportsSharding() throws SQLException {\n        if (!DriverJDBCVersion.checkSupportsJDBC43()) {\n            throw new UnsupportedOperationException(SQLServerException.getErrString(\"R_notSupported\"));\n        }\n        checkClosed();\n        return false;\n    }\n\n    @Override\n    public java.sql.ResultSet getCatalogs() throws SQLException {\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkClosed();\n        // Return the original case instead of CAPS.removed Upper().\n        // Need to match case of connection.getCatalog\n        String s = \"SELECT name AS TABLE_CAT FROM sys.databases order by name\";\n        return getResultSetFromInternalQueries(null, s);\n    }\n\n    @Override\n    public String getCatalogSeparator() throws SQLServerException {\n        checkClosed();\n        return \".\";\n    }\n\n    @Override\n    public String getCatalogTerm() throws SQLServerException {\n        checkClosed();\n        return \"database\";\n    }\n\n    private static final String[] getColumnPrivilegesColumnNames = { /* 1 */ TABLE_CAT, /* 2 */ TABLE_SCHEM,\n            /* 3 */ TABLE_NAME, /* 4 */ COLUMN_NAME, /* 5 */ GRANTOR, /* 6 */ GRANTEE, /* 7 */ PRIVILEGE,\n            /* 8 */ IS_GRANTABLE};\n\n    @Override\n    public java.sql.ResultSet getColumnPrivileges(String catalog, String schema, String table,\n            String col) throws SQLServerException, SQLTimeoutException {\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkClosed();\n        // column_privileges supports columns being escaped.\n        col = escapeIDName(col);\n        /*\n         * sp_column_privileges [ @table_name = ] 'table_name' [ , [ @table_owner = ] 'table_owner' ] [ ,\n         * [ @table_qualifier = ] 'table_qualifier' ] [ , [ @column_name = ] 'column' ]\n         */\n\n        String[] arguments = new String[4];\n        arguments[0] = table;\n        arguments[1] = schema;\n        arguments[2] = catalog;\n        arguments[3] = col;\n        return getResultSetWithProvidedColumnNames(catalog, CallableHandles.SP_COLUMN_PRIVILEGES, arguments,\n                getColumnPrivilegesColumnNames);\n    }\n\n    private static final String[] getTablesColumnNames = { /* 1 */ TABLE_CAT, /* 2 */ TABLE_SCHEM, /* 3 */ TABLE_NAME,\n            /* 4 */ TABLE_TYPE, /* 5 */ REMARKS};\n\n    @Override\n    public java.sql.ResultSet getTables(String catalog, String schema, String table,\n            String[] types) throws SQLServerException, SQLTimeoutException {\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkClosed();\n\n        // sp_tables supports table name and owner ie schema escaped.\n        table = escapeIDName(table);\n        schema = escapeIDName(schema);\n        /*\n         * sp_tables [ [ @table_name = ] 'name' ] [ , [ @table_owner = ] 'owner' ] [ , [ @table_qualifier = ]\n         * 'qualifier' ] [ , [ @table_type = ] \"type\" ]\n         */\n\n        String[] arguments = new String[4];\n        arguments[0] = table;\n        arguments[1] = schema;\n        arguments[2] = catalog;\n\n        if (null != types) {\n            final StringBuilder tableTypes = new StringBuilder(\"'\");\n            for (int i = 0; i < types.length; i++) {\n                if (i > 0)\n                    tableTypes.append(\",\");\n                tableTypes.append(\"''\").append(types[i]).append(\"''\");\n            }\n            tableTypes.append(\"'\");\n            arguments[3] = tableTypes.toString();\n        }\n        return getResultSetWithProvidedColumnNames(catalog, CallableHandles.SP_TABLES, arguments, getTablesColumnNames);\n    }\n\n    static final char LEFT_BRACKET = '[';\n    static final char RIGHT_BRACKET = ']';\n    static final char ESCAPE = '\\\\';\n    static final char PERCENT = '%';\n    static final char UNDERSCORE = '_';\n    static final char[] DOUBLE_RIGHT_BRACKET = {']', ']'};\n\n    /**\n     * Accepts a SQL identifier (such as a column name or table name) and escapes the identifier so sql 92 wild card\n     * characters can be escaped properly to be passed to functions like sp_columns or sp_tables. Assumes that the\n     * incoming identifier is un-escaped.\n     * \n     * @inID input identifier to escape.\n     * @return the escaped value.\n     */\n    private static String escapeIDName(String inID) {\n        if (null == inID)\n            return inID;\n        // SQL bracket escaping rules.\n        // See Using Wildcard Characters As Literals in SQL BOL\n        //\n        // 5\\% -> '5[%]'\n        // \\_n -> '[_]n'\n        // \\[ -> '[ [ ]'\n        // \\] -> ']'\n        // \\\\ -> \\\n        // \\x -> \\x where x is any char other than the ones above.\n\n        char ch;\n        // Add 2 extra chars wild guess thinking at least one escape.\n        StringBuilder outID = new StringBuilder(inID.length() + 2);\n\n        for (int i = 0; i < inID.length(); i++) {\n            ch = inID.charAt(i);\n            if (ESCAPE == ch && (++i < inID.length())) {\n                ch = inID.charAt(i);\n                switch (ch) {\n                    case PERCENT:\n                    case UNDERSCORE:\n                    case LEFT_BRACKET:\n                        outID.append(LEFT_BRACKET);\n                        outID.append(ch);\n                        outID.append(RIGHT_BRACKET);\n                        break;\n                    case RIGHT_BRACKET:\n                    case ESCAPE:\n                        outID.append(ch);\n                        break;\n                    default:\n                        outID.append(ESCAPE);\n                        outID.append(ch);\n                }\n\n            } else {\n                // no escape just copy\n                outID.append(ch);\n            }\n        }\n        return outID.toString();\n    }\n\n    @Override\n    public java.sql.ResultSet getColumns(String catalog, String schema, String table, String col) throws SQLException {\n        if (loggerExternal.isLoggable(Level.FINER) && Util.isActivityTraceOn()) {\n            loggerExternal.finer(toString() + ACTIVITY_ID + ActivityCorrelator.getCurrent().toString());\n        }\n        checkClosed();\n        String originalCatalog = switchCatalogs(catalog);\n        if (!this.connection.isAzureDW()) {\n            String spColumnsSql = \"DECLARE @mssqljdbc_temp_sp_columns_result TABLE(TABLE_QUALIFIER SYSNAME, TABLE_OWNER SYSNAME,\"\n                    + \"TABLE_NAME SYSNAME, COLUMN_NAME SYSNAME, DATA_TYPE SMALLINT, TYPE_NAME SYSNAME, PRECISION INT,\"\n                    + \"LENGTH INT, SCALE SMALLINT, RADIX SMALLINT, NULLABLE SMALLINT, REMARKS VARCHAR(254), COLUMN_DEF NVARCHAR(4000),\"\n                    + \"SQL_DATA_TYPE SMALLINT, SQL_DATETIME_SUB SMALLINT, CHAR_OCTET_LENGTH INT, ORDINAL_POSITION INT,\"\n                    + \"IS_NULLABLE VARCHAR(254), SS_IS_SPARSE SMALLINT, SS_IS_COLUMN_SET SMALLINT, SS_IS_COMPUTED SMALLINT,\"\n                    + \"SS_IS_IDENTITY SMALLINT, SS_UDT_CATALOG_NAME NVARCHAR(128), SS_UDT_SCHEMA_NAME NVARCHAR(128),\"\n                    + \"SS_UDT_ASSEMBLY_TYPE_NAME NVARCHAR(max), SS_XML_SCHEMACOLLECTION_CATALOG_NAME NVARCHAR(128),\"\n                    + \"SS_XML_SCHEMACOLLECTION_SCHEMA_NAME NVARCHAR(128), SS_XML_SCHEMACOLLECTION_NAME NVARCHAR(128),\"\n                    + \"SS_DATA_TYPE TINYINT);\"\n\n                    + \"INSERT INTO @mssqljdbc_temp_sp_columns_result EXEC sp_columns_100 ?,?,?,?,?,?;\"\n\n                    + \"SELECT TABLE_QUALIFIER AS TABLE_CAT, TABLE_OWNER AS TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, \"\n                    + \"CAST(DATA_TYPE AS INT) AS DATA_TYPE,TYPE_NAME, PRECISION AS COLUMN_SIZE, LENGTH AS BUFFER_LENGTH, \"\n                    + \"CAST(SCALE AS INT) AS DECIMAL_DIGITS, CAST(RADIX AS INT) AS NUM_PREC_RADIX,CAST(NULLABLE AS INT) AS NULLABLE, \"\n                    + \"CAST(REMARKS AS VARCHAR) AS REMARKS, COLUMN_DEF, CAST(SQL_DATA_TYPE AS INT) AS SQL_DATA_TYPE, \"\n                    + \"CAST(SQL_DATETIME_SUB AS INT) AS SQL_DATETIME_SUB, CHAR_OCTET_LENGTH, ORDINAL_POSITION, IS_NULLABLE,\"\n                    + \"CAST(NULL AS VARCHAR) AS SCOPE_CATALOG, CAST(NULL AS VARCHAR) AS SCOPE_SCHEMA, CAST(NULL AS VARCHAR) AS SCOPE_TABLE, \"\n                    + \"CAST(SS_DATA_TYPE AS SMALLINT) AS SOURCE_DATA_TYPE, \"\n                    + \"CASE SS_IS_IDENTITY WHEN 0 THEN 'NO' WHEN 1 THEN 'YES' WHEN '' THEN '' END AS IS_AUTOINCREMENT,\"\n                    + \"CASE SS_IS_COMPUTED WHEN 0 THEN 'NO' WHEN 1 THEN 'YES' WHEN '' THEN '' END AS IS_GENERATEDCOLUMN, \"\n                    + \"SS_IS_SPARSE, SS_IS_COLUMN_SET, SS_UDT_CATALOG_NAME, SS_UDT_SCHEMA_NAME, SS_UDT_ASSEMBLY_TYPE_NAME,\"\n                    + \"SS_XML_SCHEMACOLLECTION_CATALOG_NAME, SS_XML_SCHEMACOLLECTION_SCHEMA_NAME, SS_XML_SCHEMACOLLECTION_NAME \"\n                    + \"FROM @mssqljdbc_temp_sp_columns_result ORDER BY TABLE_CAT, TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION;\";\n            SQLServerResultSet rs = null;\n            PreparedStatement pstmt = (SQLServerPreparedStatement) this.connection.prepareStatement(spColumnsSql);\n            pstmt.closeOnCompletion();\n            try {\n                pstmt.setString(1, (null != table && !table.isEmpty()) ? escapeIDName(table) : \"%\");\n                pstmt.setString(2, (null != schema && !schema.isEmpty()) ? escapeIDName(schema) : \"%\");\n                pstmt.setString(3, (null != catalog && !catalog.isEmpty()) ? catalog : this.connection.getCatalog());\n                pstmt.setString(4, (null != col && !col.isEmpty()) ? escapeIDName(col) : \"%\");\n                pstmt.setInt(5, 2);// show sparse columns\n                pstmt.setInt(6, 3);// odbc version\n\n                rs = (SQLServerResultSet) pstmt.executeQuery();\n                rs.getColumn(5).setFilter(new DataTypeFilter());\n                rs.getColumn(7).setFilter(new ZeroFixupFilter());\n                rs.getColumn(8).setFilter(new ZeroFixupFilter());\n                rs.getColumn(16).setFilter(new ZeroFixupFilter());\n            } catch (SQLException e) {\n                if (null != pstmt) {\n                    try {\n                        pstmt.close();\n                    } catch (SQLServerException ignore) {\n                        if (loggerExternal.isLoggable(Level.FINER)) {\n                            loggerExternal.finer(\n                                    \"getColumns() threw an exception when attempting to close PreparedStatement\");\n                        }\n                    }\n                }\n                throw e;\n            } finally {\n                if (null != originalCatalog) {\n                    connection.setCatalog(originalCatalog);\n                }\n            }\n\n            return rs;\n        } else {\n            /**\n             * Can't actually switchCatalogs on Azure DW. This is here to keep consistency in behavior with SQL Azure DB\n             * when user provides a different catalog than the one they're currently connected to. Will throw exception\n             * when it's different and do nothing if it's the same/null.\n             */\n            LOCK.lock();\n            try {\n                if (null == getColumnsDWColumns) {\n                    getColumnsDWColumns = new LinkedHashMap<>();\n                    getColumnsDWColumns.put(1, TABLE_CAT);\n                    getColumnsDWColumns.put(2, TABLE_SCHEM);\n                    getColumnsDWColumns.put(3, TABLE_NAME);\n                    getColumnsDWColumns.put(4, COLUMN_NAME);\n                    getColumnsDWColumns.put(5, DATA_TYPE);\n                    getColumnsDWColumns.put(6, TYPE_NAME);\n                    getColumnsDWColumns.put(7, COLUMN_SIZE);\n                    getColumnsDWColumns.put(8, BUFFER_LENGTH);\n                    getColumnsDWColumns.put(9, DECIMAL_DIGITS);\n                    getColumnsDWColumns.put(10, NUM_PREC_RADIX);\n                    getColumnsDWColumns.put(11, NULLABLE);\n                    getColumnsDWColumns.put(12, REMARKS);\n                    getColumnsDWColumns.put(13, COLUMN_DEF);\n                    getColumnsDWColumns.put(14, SQL_DATA_TYPE);\n                    getColumnsDWColumns.put(15, SQL_DATETIME_SUB);\n                    getColumnsDWColumns.put(16, CHAR_OCTET_LENGTH);\n                    getColumnsDWColumns.put(17, ORDINAL_POSITION);\n                    getColumnsDWColumns.put(18, IS_NULLABLE);\n                    /*\n                     * Use negative value keys to indicate that this column doesn't exist in SQL Server and should just\n                     * be queried as 'NULL'\n                     */\n                    getColumnsDWColumns.put(-1, SCOPE_CATALOG);\n                    getColumnsDWColumns.put(-2, SCOPE_SCHEMA);\n                    getColumnsDWColumns.put(-3, SCOPE_TABLE);\n                    getColumnsDWColumns.put(29, SOURCE_DATA_TYPE);\n                    getColumnsDWColumns.put(22, IS_AUTOINCREMENT);\n                    getColumnsDWColumns.put(21, IS_GENERATEDCOLUMN);\n                    getColumnsDWColumns.put(19, SS_IS_SPARSE);\n                    getColumnsDWColumns.put(20, SS_IS_COLUMN_SET);\n                    getColumnsDWColumns.put(23, SS_UDT_CATALOG_NAME);\n                    getColumnsDWColumns.put(24, SS_UDT_SCHEMA_NAME);\n                    getColumnsDWColumns.put(25, SS_UDT_ASSEMBLY_TYPE_NAME);\n                    getColumnsDWColumns.put(26, SS_XML_SCHEMACOLLECTION_CATALOG_NAME);\n                    getColumnsDWColumns.put(27, SS_XML_SCHEMACOLLECTION_SCHEMA_NAME);\n                    getColumnsDWColumns.put(28, SS_XML_SCHEMACOLLECTION_NAME);\n                }\n                if (null == getTypesDWColumns) {\n                    getTypesDWColumns = new LinkedHashMap<>();\n                    getTypesDWColumns.put(1, NVARCHAR); // TABLE_CAT\n                    getTypesDWColumns.put(2, NVARCHAR); // TABLE_SCHEM\n                    getTypesDWColumns.put(3, NVARCHAR); // TABLE_NAME\n                    getTypesDWColumns.put(4, NVARCHAR); // COLUMN_NAME\n                    getTypesDWColumns.put(5, INTEGER); // DATA_TYPE\n                    getTypesDWColumns.put(6, NVARCHAR); // TYPE_NAME\n                    getTypesDWColumns.put(7, INTEGER); // COLUMN_SIZE\n                    getTypesDWColumns.put(8, INTEGER); // BUFFER_LENGTH\n                    getTypesDWColumns.put(9, INTEGER); // DECIMAL_DIGITS\n                    getTypesDWColumns.put(10, INTEGER); // NUM_PREC_RADIX\n                    getTypesDWColumns.put(11, INTEGER); // NULLABLE\n                    getTypesDWColumns.put(12, VARCHAR); // REMARKS\n                    getTypesDWColumns.put(13, NVARCHAR); // COLUMN_DEF\n                    getTypesDWColumns.put(14, INTEGER); // SQL_DATA_TYPE\n                    getTypesDWColumns.put(15, INTEGER); // SQL_DATETIME_SUB\n                    getTypesDWColumns.put(16, INTEGER); // CHAR_OCTET_LENGTH\n                    getTypesDWColumns.put(17, INTEGER); // ORDINAL_POSITION\n                    getTypesDWColumns.put(18, VARCHAR); // IS_NULLABLE\n                    /*\n                     * Use negative value keys to indicate that this column doesn't exist in SQL Server and should just\n                     * be queried as 'NULL'\n                     */\n                    getTypesDWColumns.put(-1, VARCHAR); // SCOPE_CATALOG\n                    getTypesDWColumns.put(-2, VARCHAR); // SCOPE_SCHEMA\n                    getTypesDWColumns.put(-3, VARCHAR); // SCOPE_TABLE\n                    getTypesDWColumns.put(29, SMALLINT); // SOURCE_DATA_TYPE\n                    getTypesDWColumns.put(22, VARCHAR); // IS_AUTOINCREMENT\n                    getTypesDWColumns.put(21, VARCHAR); // IS_GENERATEDCOLUMN\n                    getTypesDWColumns.put(19, SMALLINT); // SS_IS_SPARSE\n                    getTypesDWColumns.put(20, SMALLINT); // SS_IS_COLUMN_SET\n                    getTypesDWColumns.put(23, NVARCHAR); // SS_UDT_CATALOG_NAME\n                    getTypesDWColumns.put(24, NVARCHAR); // SS_UDT_SCHEMA_NAME\n                    getTypesDWColumns.put(25, NVARCHAR); // SS_UDT_ASSEMBLY_TYPE_NAME\n                    getTypesDWColumns.put(26, NVARCHAR); // SS_XML_SCHEMACOLLECTION_CATALOG_NAME\n                    getTypesDWColumns.put(27, NVARCHAR); // SS_XML_SCHEMACOLLECTION_SCHEMA_NAME\n                    getTypesDWColumns.put(28, NVARCHAR); // SS_XML_SCHEMACOLLECTION_NAME\n                }\n\n                // Ensure there is a data type for every metadata column\n                if (getColumnsDWColumns.size() != getTypesDWColumns.size()) {\n                    MessageFormat form = new MessageFormat(\n                            SQLServerException.getErrString(\"R_colCountNotMatchColTypeCount\"));\n                    Object[] msgArgs = {getColumnsDWColumns.size(), getTypesDWColumns.size()};\n                    throw new IllegalArgumentException(form.format(msgArgs));\n                }\n            } finally {\n                LOCK.unlock();\n            }\n\n            try (PreparedStatement storedProcPstmt = this.connection\n                    .prepareStatement(\"EXEC sp_columns_100 ?,?,?,?,?,?;\")) {\n                storedProcPstmt.setString(1, (null != table && !table.isEmpty()) ? escapeIDName(table) : \"%\");\n                storedProcPstmt.setString(2, (null != schema && !schema.isEmpty()) ? escapeIDName(schema) : \"%\");\n                storedProcPstmt.setString(3,\n                        (null != catalog && !catalog.isEmpty()) ? catalog : this.connection.getCatalog());\n...\n// Path: src/main/java/com/microsoft/sqlserver/jdbc/SQLServerParameterMetaData.java\n/*\n * Microsoft JDBC Driver for SQL Server Copyright(c) Microsoft Corporation All rights reserved. This program is made\n * available under the terms of the MIT License. See the LICENSE file in the project root for more information.\n */\n\npackage com.microsoft.sqlserver.jdbc;\n\nimport java.sql.ParameterMetaData;\nimport java.sql.ResultSet;\nimport java.sql.ResultSetMetaData;\nimport java.sql.SQLException;\nimport java.text.MessageFormat;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.regex.Matcher;\nimport java.util.regex.Pattern;\n\n\n/**\n * Provides meta data for prepared statement parameters.\n *\n * The API javadoc for JDBC API methods that this class implements are not repeated here. Please see Sun's JDBC API\n * interfaces javadoc for those details.\n *\n * For SQL Servers versioned below SQL Server 2012, prepared statements are executed with SET FMT ONLY to retrieve\n * column meta data Callable statements : sp_sp_sproc_columns is called to retrieve names and meta data for the\n * procedures params.\n *\n * For SQL Server 2012 and above, sp_describe_undeclared_parameter is used to retrieve parameter metadata.\n */\n\npublic final class SQLServerParameterMetaData implements ParameterMetaData {\n\n    private final static int SQL_SERVER_2012_VERSION = 11;\n\n    private static final String DATA_TYPE = \"DATA_TYPE\";\n    private static final String COLUMN_TYPE = \"COLUMN_TYPE\";\n    private static final String TYPE_NAME = \"TYPE_NAME\";\n    private static final String PRECISION = \"PRECISION\";\n    private static final String SCALE = \"SCALE\";\n    private static final String NULLABLE = \"NULLABLE\";\n    private static final String SS_TYPE_SCHEMA_NAME = \"SS_TYPE_SCHEMA_NAME\";\n\n    private final SQLServerPreparedStatement stmtParent;\n    private SQLServerConnection con;\n\n    private List<Map<String, Object>> procMetadata;\n\n    boolean procedureIsFound = false;\n\n    static final private java.util.logging.Logger logger = java.util.logging.Logger\n            .getLogger(\"com.microsoft.sqlserver.jdbc.internals.SQLServerParameterMetaData\");\n\n    static private final AtomicInteger baseID = new AtomicInteger(0); // Unique id generator for each instance (used for\n                                                                      // logging).\n    final private String traceID = \" SQLServerParameterMetaData:\" + nextInstanceID();\n    boolean isTVP = false;\n\n    // Returns unique id for each instance.\n    private static int nextInstanceID() {\n        return baseID.incrementAndGet();\n    }\n\n    /**\n     * Provides a helper function to provide an ID string suitable for tracing.\n     * \n     * @return traceID string\n     */\n    @Override\n    final public String toString() {\n        return traceID;\n    }\n\n    /* Used for prepared statement meta data */\n    class QueryMeta {\n        String parameterClassName = null;\n        int parameterType = 0;\n        String parameterTypeName = null;\n        int precision = 0;\n        int scale = 0;\n        int isNullable = ParameterMetaData.parameterNullableUnknown;\n        boolean isSigned = false;\n    }\n\n    Map<Integer, QueryMeta> queryMetaMap = null;\n\n    /*\n     * Parses query metadata.\n     */\n    private void parseQueryMeta(ResultSet rsQueryMeta) throws SQLServerException {\n        Pattern datatypePattern = Pattern.compile(\"(.*)\\\\((.*)(\\\\)|,(.*)\\\\))\");\n        try {\n            if (null != rsQueryMeta) {\n                while (rsQueryMeta.next()) {\n                    QueryMeta qm = new QueryMeta();\n                    SSType ssType = null;\n\n                    int paramOrdinal = rsQueryMeta.getInt(\"parameter_ordinal\");\n                    String typename = rsQueryMeta.getString(\"suggested_system_type_name\");\n\n                    if (null == typename) {\n                        typename = rsQueryMeta.getString(\"suggested_user_type_name\");\n                        try (SQLServerPreparedStatement pstmt = (SQLServerPreparedStatement) con.prepareStatement(\n                                \"select max_length, precision, scale, is_nullable from sys.assembly_types where name = ?\")) {\n                            pstmt.setNString(1, typename);\n                            try (ResultSet assemblyRs = pstmt.executeQuery()) {\n                                if (assemblyRs.next()) {\n                                    qm.parameterTypeName = typename;\n                                    qm.precision = assemblyRs.getInt(\"max_length\");\n                                    qm.scale = assemblyRs.getInt(\"scale\");\n                                    ssType = SSType.UDT;\n                                }\n                            }\n                        }\n                    } else {\n                        qm.precision = rsQueryMeta.getInt(\"suggested_precision\");\n                        qm.scale = rsQueryMeta.getInt(\"suggested_scale\");\n\n                        Matcher matcher = datatypePattern.matcher(typename);\n                        if (matcher.matches()) {\n                            // the datatype has some precision/scale defined explicitly.\n                            ssType = SSType.of(matcher.group(1));\n                            if (\"varchar(max)\".equalsIgnoreCase(typename)\n                                    || \"varbinary(max)\".equalsIgnoreCase(typename)) {\n                                qm.precision = SQLServerDatabaseMetaData.MAXLOBSIZE;\n                            } else if (\"nvarchar(max)\".equalsIgnoreCase(typename)) {\n                                qm.precision = SQLServerDatabaseMetaData.MAXLOBSIZE / 2;\n                            } else if (SSType.Category.CHARACTER == ssType.category\n                                    || SSType.Category.BINARY == ssType.category\n                                    || SSType.Category.NCHARACTER == ssType.category) {\n                                try {\n                                    // For character/binary data types \"suggested_precision\" is 0. So get the precision\n                                    // from\n                                    // the type itself.\n                                    qm.precision = Integer.parseInt(matcher.group(2));\n                                } catch (NumberFormatException e) {\n                                    MessageFormat form = new MessageFormat(\n                                            SQLServerException.getErrString(\"R_metaDataErrorForParameter\"));\n                                    Object[] msgArgs = {paramOrdinal};\n                                    SQLServerException.makeFromDriverError(con, stmtParent,\n                                            form.format(msgArgs) + \" \" + e.getMessage(), null, false);\n                                }\n                            }\n                        } else\n                            ssType = SSType.of(typename);\n\n                        // For float and real types suggested_precision returns the number of bits, not digits.\n                        if (SSType.FLOAT == ssType) {\n                            // https://msdn.microsoft.com/en-CA/library/ms173773.aspx\n                            // real is float(24) and is 7 digits. Float is 15 digits.\n                            qm.precision = 15;\n                        } else if (SSType.REAL == ssType) {\n                            qm.precision = 7;\n                        } else if (SSType.TEXT == ssType) {\n                            qm.precision = SQLServerDatabaseMetaData.MAXLOBSIZE;\n                        } else if (SSType.NTEXT == ssType) {\n                            qm.precision = SQLServerDatabaseMetaData.MAXLOBSIZE / 2;\n                        } else if (SSType.IMAGE == ssType) {\n                            qm.precision = SQLServerDatabaseMetaData.MAXLOBSIZE;\n                        } else if (SSType.GUID == ssType) {\n                            qm.precision = SQLServerDatabaseMetaData.UNIQUEIDENTIFIER_SIZE;\n                        } else if (SSType.TIMESTAMP == ssType) {\n                            qm.precision = 8;\n                        } else if (SSType.XML == ssType) {\n                            qm.precision = SQLServerDatabaseMetaData.MAXLOBSIZE / 2;\n                        }\n\n                        qm.parameterTypeName = ssType.toString();\n                    }\n\n                    // Check if ssType is null. Was caught by static analysis.\n                    if (null == ssType) {\n                        throw new SQLServerException(SQLServerException.getErrString(\"R_metaDataErrorForParameter\"),\n                                null);\n                    }\n\n                    JDBCType jdbcType = ssType.getJDBCType();\n                    qm.parameterClassName = jdbcType.className();\n                    qm.parameterType = jdbcType.getIntValue();\n                    // The parameter can be signed if it is a NUMERIC type (except bit or tinyint).\n                    qm.isSigned = ((SSType.Category.NUMERIC == ssType.category) && (SSType.BIT != ssType)\n                            && (SSType.TINYINT != ssType));\n                    queryMetaMap.put(paramOrdinal, qm);\n                }\n            }\n        } catch (SQLException e) {\n            throw new SQLServerException(SQLServerException.getErrString(\"R_metaDataErrorForParameter\"), e);\n        }\n    }\n\n    private void parseFMTQueryMeta(ResultSetMetaData md, SQLServerFMTQuery f) throws SQLServerException {\n        try {\n            // Gets the list of parsed column names/targets\n            List<String> columns = f.getColumns();\n            // Gets VALUES(?,?,?...) list. The internal list corresponds to the parameters in the bracket after VALUES.\n            List<List<String>> params = f.getValuesList();\n            int valueListOffset = 0;\n            int mdIndex = 1;\n            int mapIndex = 1;\n            for (int i = 0; i < columns.size(); i++) {\n                /**\n                 * For INSERT table VALUES(?,?,?...) scenarios where the column names are not specifically defined after\n                 * the table name, the parser adds a '*' followed by '?'s equal to the number of parameters in the\n                 * values bracket. The '*' will retrieve all values from the table and we'll use the '?'s to match their\n                 * position here\n                 */\n                if (\"*\".equals(columns.get(i))) {\n                    for (int j = 0; j < params.get(valueListOffset).size(); j++) {\n                        if (\"?\".equals(params.get(valueListOffset).get(j)) && (!md.isAutoIncrement(mdIndex + j))) {\n                            QueryMeta qm = getQueryMetaFromResultSetMetaData(md, mdIndex + j);\n                            queryMetaMap.put(mapIndex++, qm);\n                            i++;\n                        }\n                    }\n                    mdIndex += params.get(valueListOffset).size();\n                    valueListOffset++;\n                } else {\n                    /*\n                     * If this is not a INSERT table VALUES(...) situation, just add the entry.\n                     */\n                    QueryMeta qm = getQueryMetaFromResultSetMetaData(md, mdIndex);\n                    queryMetaMap.put(mapIndex++, qm);\n                    mdIndex++;\n                }\n            }\n        } catch (SQLException e) {\n            throw new SQLServerException(SQLServerException.getErrString(\"R_metaDataErrorForParameter\"), e);\n        }\n    }\n\n    private QueryMeta getQueryMetaFromResultSetMetaData(ResultSetMetaData md, int index) throws SQLException {\n        QueryMeta qm = new QueryMeta();\n        qm.parameterClassName = md.getColumnClassName(index);\n        qm.parameterType = md.getColumnType(index);\n        qm.parameterTypeName = md.getColumnTypeName(index);\n        qm.precision = md.getPrecision(index);\n        qm.scale = md.getScale(index);\n        qm.isNullable = md.isNullable(index);\n        qm.isSigned = md.isSigned(index);\n        return qm;\n    }\n\n    String parseProcIdentifier(String procIdentifier) throws SQLServerException {\n        ThreePartName threePartName = ThreePartName.parse(procIdentifier);\n        StringBuilder sb = new StringBuilder();\n        if (threePartName.getDatabasePart() != null) {\n            sb.append(\"@procedure_qualifier=\");\n            sb.append(threePartName.getDatabasePart());\n            sb.append(\", \");\n        }\n        if (threePartName.getOwnerPart() != null) {\n            sb.append(\"@procedure_owner=\");\n            sb.append(threePartName.getOwnerPart());\n            sb.append(\", \");\n        }\n        if (threePartName.getProcedurePart() != null) {\n            sb.append(\"@procedure_name=\");\n            sb.append(threePartName.getProcedurePart());\n        } else {\n            SQLServerException.makeFromDriverError(con, stmtParent, SQLServerException.getErrString(\"R_noMetadata\"),\n                    null, false);\n        }\n        return sb.toString();\n    }\n\n    private void checkClosed() throws SQLServerException {\n        // stmtParent does not seem to be re-used, should just verify connection is not closed.\n        // stmtParent.checkClosed();\n        con.checkClosed();\n    }\n\n    /**\n     * Construct a SQLServerParameterMetaData parameter meta data.\n     * \n     * @param st\n     *        the prepared statement\n     * @param sProcString\n     *        the procedure name\n     * @throws SQLServerException\n     */\n    @SuppressWarnings(\"serial\")\n    SQLServerParameterMetaData(SQLServerPreparedStatement st, String sProcString) throws SQLServerException {\n        assert null != st;\n        stmtParent = st;\n        con = st.connection;\n        if (logger.isLoggable(java.util.logging.Level.FINE)) {\n            logger.fine(toString() + \" created by (\" + st.toString() + \")\");\n        }\n        try {\n\n            // If the CallableStatement/PreparedStatement is a stored procedure call\n            // then we can extract metadata using sp_sproc_columns\n            if (null != st.procedureName) {\n                String sProc = parseProcIdentifier(st.procedureName);\n                try (SQLServerStatement s = (SQLServerStatement) con.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,\n                        ResultSet.CONCUR_READ_ONLY);\n                        SQLServerResultSet rsProcedureMeta = s\n                                .executeQueryInternal(con.isKatmaiOrLater()\n                                                                            ? \"exec sp_sproc_columns_100 \" + sProc\n                                                                                    + \", @ODBCVer=3, @fUsePattern=0\"\n                                                                            : \"exec sp_sproc_columns \" + sProc\n                                                                                    + \", @ODBCVer=3, @fUsePattern=0\")) {\n\n                    // if rsProcedureMeta has next row, it means the stored procedure is found\n                    if (rsProcedureMeta.next()) {\n                        procedureIsFound = true;\n                    } else {\n                        procedureIsFound = false;\n                    }\n\n                    rsProcedureMeta.beforeFirst();\n\n                    // Sixth is DATA_TYPE\n                    rsProcedureMeta.getColumn(6).setFilter(new DataTypeFilter());\n                    if (con.isKatmaiOrLater()) {\n                        rsProcedureMeta.getColumn(8).setFilter(new ZeroFixupFilter());\n                        rsProcedureMeta.getColumn(9).setFilter(new ZeroFixupFilter());\n                        rsProcedureMeta.getColumn(17).setFilter(new ZeroFixupFilter());\n                    }\n\n                    procMetadata = new ArrayList<>();\n\n                    // Process ResultSet Procedure Metadata for API usage\n                    while (rsProcedureMeta.next()) {\n                        HashMap<String, Object> map = new HashMap<>();\n                        map.put(DATA_TYPE, rsProcedureMeta.getShort(DATA_TYPE));\n                        map.put(COLUMN_TYPE, rsProcedureMeta.getInt(COLUMN_TYPE));\n                        map.put(TYPE_NAME, rsProcedureMeta.getString(TYPE_NAME));\n                        map.put(PRECISION, rsProcedureMeta.getInt(PRECISION));\n                        map.put(SCALE, rsProcedureMeta.getInt(SCALE));\n                        map.put(NULLABLE, rsProcedureMeta.getInt(NULLABLE));\n                        map.put(SS_TYPE_SCHEMA_NAME, rsProcedureMeta.getString(SS_TYPE_SCHEMA_NAME));\n                        procMetadata.add(map);\n                    }\n                }\n            }\n\n            // Otherwise we just have a parameterized statement.\n            // if SQL server version is 2012 and above use stored\n            // procedure \"sp_describe_undeclared_parameters\" to retrieve parameter meta data\n            // if SQL server version is 2008, then use FMTONLY\n            else {\n                queryMetaMap = new HashMap<>();\n                if (con.getServerMajorVersion() >= SQL_SERVER_2012_VERSION && !st.getUseFmtOnly()) {\n                    String preparedSQL = con.replaceParameterMarkers(stmtParent.userSQL,\n                            stmtParent.userSQLParamPositions, stmtParent.inOutParam, stmtParent.bReturnValueSyntax);\n\n                    try (SQLServerCallableStatement cstmt = (SQLServerCallableStatement) con\n                            .prepareCall(\"exec sp_describe_undeclared_parameters ?\")) {\n                        cstmt.setNString(1, preparedSQL);\n                        parseQueryMeta(cstmt.executeQueryInternal());\n                    }\n                } else {\n                    SQLServerFMTQuery f = new SQLServerFMTQuery(sProcString);\n                    try (SQLServerStatement stmt = (SQLServerStatement) con.createStatement();\n                            ResultSet rs = stmt.executeQuery(f.getFMTQuery())) {\n                        parseFMTQueryMeta(rs.getMetaData(), f);\n                    }\n                }\n            }\n        }\n        // Do not need to wrapper SQLServerException again\n        catch (SQLServerException e) {\n            throw e;\n        } catch (SQLException | StringIndexOutOfBoundsException e) {\n            SQLServerException.makeFromDriverError(con, stmtParent, e.getMessage(), null, false);\n        }\n    }\n\n    @Override\n    public boolean isWrapperFor(Class<?> iface) throws SQLException {\n        return iface.isInstance(this);\n    }\n\n    @Override\n    public <T> T unwrap(Class<T> iface) throws SQLException {\n        T t;\n        try {\n            t = iface.cast(this);\n        } catch (ClassCastException e) {\n            throw new SQLServerException(e.getMessage(), e);\n        }\n        return t;\n    }\n\n    \nprivate Map<String, Object> getParameterInfo(int param) {\n        if (stmtParent.bReturnValueSyntax && isTVP) {\n            return procMetadata.get(param - 1);\n        } else {\n            // Note row 1 is the 'return value' meta data\n            return procMetadata.get(param);\n        }\n    }\n\n    private boolean isValidParamProc(int n) {\n        // Note row 1 is the 'return value' meta data\n        return (n >= 1 // JDBC index is 1 based\n                && ((stmtParent.bReturnValueSyntax && isTVP && procMetadata.size() >= n) || procMetadata.size() > n));\n    }\n\n    private boolean isValidParamQuery(int n) {\n        return (null != queryMetaMap && queryMetaMap.containsKey(n));\n    }\n\n    /**\n     * Checks if the @param passed is valid for either procedure metadata or query metadata.\n     * \n     * @param param\n     * @throws SQLServerException\n     */\n    private void checkParam(int param) throws SQLServerException {\n        // Check if Procedure Metadata is not available\n        if (null == procMetadata) {\n            // Check if Query Metadata is also not available\n            if (!isValidParamQuery(param)) {\n                SQLServerException.makeFromDriverError(con, stmtParent, SQLServerException.getErrString(\"R_noMetadata\"),\n                        null, false);\n            }\n        } else if (!isValidParamProc(param)) {\n            // Throw exception if @param index not found\n            MessageFormat form = new MessageFormat(SQLServerException.getErrString(\"R_invalidParameterNumber\"));\n            Object[] msgArgs = {param};\n            SQLServerException.makeFromDriverError(con, stmtParent, form.format(msgArgs), null, false);\n        }\n    }\n\n    @Override\n    public String getParameterClassName(int param) throws SQLServerException {\n        checkClosed();\n        checkParam(param);\n        try {\n            if (null == procMetadata) {\n                return queryMetaMap.get(param).parameterClassName;\n            } else {\n                return JDBCType.of((short) getParameterInfo(param).get(DATA_TYPE)).className();\n            }\n        } catch (SQLServerException e) {\n            SQLServerException.makeFromDriverError(con, stmtParent, e.getMessage(), null, false);\n            return null;\n        }\n    }\n\n    @Override\n    public int getParameterCount() throws SQLServerException {\n        checkClosed();\n        if (null == procMetadata) {\n            return queryMetaMap.size();\n        } else {\n            // Row 1 is Return Type metadata\n            return (procMetadata.isEmpty() ? 0 : procMetadata.size() - 1);\n        }\n    }\n\n    @Override\n    public int getParameterMode(int param) throws SQLServerException {\n        checkClosed();\n        checkParam(param);\n        if (null == procMetadata) {\n            // if it is not a stored procedure, the @param can only be input.\n            return parameterModeIn;\n        } else {\n            int n = (int) getParameterInfo(param).get(COLUMN_TYPE);\n            if (n == 1)\n                return parameterModeIn;\n            else if (n == 2)\n                return parameterModeOut;\n            else\n                return parameterModeUnknown;\n        }\n    }\n\n    @Override\n    public int getParameterType(int param) throws SQLServerException {\n        checkClosed();\n        checkParam(param);\n...\n", "needle_token_start": 15357, "needle_token_end": 15438, "code_context_ntokens": 16207, "output": ["```java\nprivate Map<String, Object> getParameterInfo(int param) {\n    if (stmtParent.bReturnValueSyntax && isTVP) {\n        return procMetadata.get(param - 1);\n    } else {\n        // Note row 1 is the 'return value' meta data\n        return procMetadata.get(param);\n    }\n}\n```"]}
{"repo": "google/conscrypt", "name": "onEngineHandshakeFinished", "language": "java", "path": "common/src/main/java/org/conscrypt/ConscryptEngineSocket.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to update the state of a process to indicate the completion of a handshake operation in a secure communication setup.\n2. **Input**: There are no direct inputs as it operates based on the current state of the process.\n3. **Output**: There is no direct output; however, the internal state of the process is updated.\n4. **Procedure**: The function checks if the current state indicates that a handshake has started. If true, it transitions the state to indicate that the handshake has been completed. This state change is critical for managing the lifecycle of secure communications, ensuring that subsequent steps only proceed once the handshake is successfully completed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    }\n\n    @Override\n    public final void setNeedClientAuth(boolean need) {\n        engine.setNeedClientAuth(need);\n    }\n\n    @Override\n    public final void setWantClientAuth(boolean want) {\n        engine.setWantClientAuth(want);\n    }\n\n    @Override\n    @SuppressWarnings(\"UnsynchronizedOverridesSynchronized\")\n    public final void close() throws IOException {\n        // TODO: Close SSL sockets using a background thread so they close gracefully.\n\n        if (stateLock == null) {\n            // Constructor failed, e.g. superclass constructor called close()\n            return;\n        }\n\n        int previousState = transitionTo(STATE_CLOSED);\n        if (previousState == STATE_CLOSED) {\n            return;\n        }\n        try {\n            // Close the engine.\n            engine.closeInbound();\n            engine.closeOutbound();\n            // Closing the outbound direction of a connected engine will trigger a TLS close\n            // notify, which we should try and send.\n            // If we don't, then closeOutbound won't be able to free resources because there are\n            // bytes queued for transmission so drain the queue those and call closeOutbound a\n            // second time.\n            if (previousState >= STATE_HANDSHAKE_STARTED) {\n                drainOutgoingQueue();\n                engine.closeOutbound();\n            }\n        } finally {\n            // In case of an exception thrown while closing the engine, we still need to close the\n            // underlying socket and release any resources the input stream is holding.\n            try {\n                super.close();\n            } finally {\n                if (in != null) {\n                    in.release();\n                }\n            }\n        }\n    }\n\n    @Override\n    public void setHandshakeTimeout(int handshakeTimeoutMilliseconds) throws SocketException {\n        // Not supported but ignored rather than throwing for compatibility: b/146041327\n    }\n\n    @Override\n    final void setApplicationProtocols(String[] protocols) {\n        engine.setApplicationProtocols(protocols);\n    }\n\n    @Override\n    final String[] getApplicationProtocols() {\n        return engine.getApplicationProtocols();\n    }\n\n    @Override\n    public final String getApplicationProtocol() {\n        return engine.getApplicationProtocol();\n    }\n\n    @Override\n    public final String getHandshakeApplicationProtocol() {\n        return engine.getHandshakeApplicationProtocol();\n    }\n\n    @Override\n    public final void setApplicationProtocolSelector(ApplicationProtocolSelector selector) {\n        setApplicationProtocolSelector(\n                selector == null ? null : new ApplicationProtocolSelectorAdapter(this, selector));\n    }\n\n    @Override\n    final void setApplicationProtocolSelector(ApplicationProtocolSelectorAdapter selector) {\n        engine.setApplicationProtocolSelector(selector);\n    }\n\n    void setBufferAllocator(BufferAllocator bufferAllocator) {\n        engine.setBufferAllocator(bufferAllocator);\n        this.bufferAllocator = bufferAllocator;\n    }\n\n    \nprivate void onEngineHandshakeFinished() {\n        // Don't do anything here except change state.  This method will be called from\n        // e.g. wrap() which is non re-entrant so we can't call anything that might do\n        // IO until after it exits, e.g. in doHandshake().\n        if (isState(STATE_HANDSHAKE_STARTED)) {\n            transitionTo(STATE_HANDSHAKE_COMPLETED);\n        }\n    }\n\n    /**\n     * Waits for the handshake to complete.\n     */\n    private void waitForHandshake() throws IOException {\n        startHandshake();\n\n        synchronized (stateLock) {\n            while (state != STATE_READY\n                    // Waiting threads are allowed to compete with handshake listeners for access.\n                    && state != STATE_READY_HANDSHAKE_CUT_THROUGH\n                    && state != STATE_CLOSED) {\n                try {\n                    stateLock.wait();\n                } catch (InterruptedException e) {\n                    Thread.currentThread().interrupt();\n                    throw new IOException(\"Interrupted waiting for handshake\", e);\n                }\n            }\n\n            if (state == STATE_CLOSED) {\n                throw new SocketException(\"Socket is closed\");\n            }\n        }\n    }\n\n    private void drainOutgoingQueue() {\n        try {\n            while (engine.pendingOutboundEncryptedBytes() > 0) {\n                out.writeInternal(EMPTY_BUFFER);\n                // Always flush handshake frames immediately.\n                out.flushInternal();\n            }\n        } catch (IOException e) {\n            // Ignore\n        }\n    }\n\n    private OutputStream getUnderlyingOutputStream() throws IOException {\n        return super.getOutputStream();\n    }\n\n    private InputStream getUnderlyingInputStream() throws IOException {\n        return super.getInputStream();\n    }\n\n    @Override\n    public final String chooseServerAlias(X509KeyManager keyManager, String keyType) {\n        return keyManager.chooseServerAlias(keyType, null, this);\n    }\n\n    @Override\n    public final String chooseClientAlias(X509KeyManager keyManager, X500Principal[] issuers,\n                                          String[] keyTypes) {\n        return keyManager.chooseClientAlias(keyTypes, issuers, this);\n    }\n\n    /**\n     * Wrap bytes written to the underlying socket.\n     */\n    private final class SSLOutputStream extends OutputStream {\n        private final Object writeLock = new Object();\n        private final ByteBuffer target;\n        private final int targetArrayOffset;\n        private OutputStream socketOutputStream;\n\n        SSLOutputStream() {\n            target = ByteBuffer.allocate(engine.getSession().getPacketBufferSize());\n            targetArrayOffset = target.arrayOffset();\n        }\n\n        @Override\n        public void close() throws IOException {\n            ConscryptEngineSocket.this.close();\n        }\n\n        @Override\n        public void write(int b) throws IOException {\n            waitForHandshake();\n            synchronized (writeLock) {\n                write(new byte[] {(byte) b});\n            }\n        }\n\n        @Override\n        public void write(byte[] b) throws IOException {\n            waitForHandshake();\n            synchronized (writeLock) {\n                writeInternal(ByteBuffer.wrap(b));\n            }\n        }\n\n        @Override\n        public void write(byte[] b, int off, int len) throws IOException {\n            waitForHandshake();\n            synchronized (writeLock) {\n                writeInternal(ByteBuffer.wrap(b, off, len));\n            }\n        }\n\n        private void writeInternal(ByteBuffer buffer) throws IOException {\n            Platform.blockGuardOnNetwork();\n            checkOpen();\n            init();\n\n            // Need to loop through at least once to enable handshaking where no application\n            // bytes are processed.\n            int len = buffer.remaining();\n            SSLEngineResult engineResult;\n            do {\n                target.clear();\n                engineResult = engine.wrap(buffer, target);\n                if (engineResult.getStatus() != OK && engineResult.getStatus() != CLOSED) {\n                    throw new SSLException(\"Unexpected engine result \" + engineResult.getStatus());\n                }\n                if (target.position() != engineResult.bytesProduced()) {\n                    throw new SSLException(\"Engine bytesProduced \" + engineResult.bytesProduced()\n                            + \" does not match bytes written \" + target.position());\n                }\n                len -= engineResult.bytesConsumed();\n                if (len != buffer.remaining()) {\n                    throw new SSLException(\"Engine did not read the correct number of bytes\");\n                }\n                if (engineResult.getStatus() == CLOSED && engineResult.bytesProduced() == 0) {\n                    if (len > 0) {\n                        throw new SocketException(\"Socket closed\");\n                    }\n                    break;\n                }\n\n                target.flip();\n\n                // Write the data to the socket.\n                writeToSocket();\n            } while (len > 0);\n        }\n\n        @Override\n        public void flush() throws IOException {\n            waitForHandshake();\n            synchronized (writeLock) {\n                flushInternal();\n            }\n        }\n\n        private void flushInternal() throws IOException {\n            checkOpen();\n            init();\n            socketOutputStream.flush();\n        }\n\n        private void init() throws IOException {\n            if (socketOutputStream == null) {\n                socketOutputStream = getUnderlyingOutputStream();\n            }\n        }\n\n        private void writeToSocket() throws IOException {\n            // Write the data to the socket.\n            socketOutputStream.write(target.array(), targetArrayOffset, target.limit());\n        }\n    }\n\n    /**\n     * Unwrap bytes read from the underlying socket.\n     */\n    private final class SSLInputStream extends InputStream {\n        private final Object readLock = new Object();\n        private final byte[] singleByte = new byte[1];\n        private final ByteBuffer fromEngine;\n        private final ByteBuffer fromSocket;\n        private final int fromSocketArrayOffset;\n        private final AllocatedBuffer allocatedBuffer;\n        private InputStream socketInputStream;\n\n        SSLInputStream() {\n            if (bufferAllocator != null) {\n                allocatedBuffer = bufferAllocator.allocateDirectBuffer(\n                        engine.getSession().getApplicationBufferSize());\n                fromEngine = allocatedBuffer.nioBuffer();\n            } else {\n                allocatedBuffer = null;\n                fromEngine = ByteBuffer.allocateDirect(engine.getSession().getApplicationBufferSize());\n            }\n            // Initially fromEngine.remaining() == 0.\n            fromEngine.flip();\n            fromSocket = ByteBuffer.allocate(engine.getSession().getPacketBufferSize());\n            fromSocketArrayOffset = fromSocket.arrayOffset();\n        }\n\n        @Override\n        public void close() throws IOException {\n            ConscryptEngineSocket.this.close();\n        }\n\n        void release() {\n            synchronized (readLock) {\n                if (allocatedBuffer != null) {\n                    allocatedBuffer.release();\n                }\n            }\n        }\n\n        @Override\n        public int read() throws IOException {\n            waitForHandshake();\n            synchronized (readLock) {\n                // Handle returning of -1 if EOF is reached.\n                int count = read(singleByte, 0, 1);\n                if (count == -1) {\n                    // Handle EOF.\n                    return -1;\n                }\n                if (count != 1) {\n                    throw new SSLException(\"read incorrect number of bytes \" + count);\n                }\n                return singleByte[0] & 0xff;\n            }\n        }\n\n        @Override\n        public int read(byte[] b) throws IOException {\n            waitForHandshake();\n            synchronized (readLock) {\n                return read(b, 0, b.length);\n            }\n        }\n\n        @Override\n        public int read(byte[] b, int off, int len) throws IOException {\n            waitForHandshake();\n            synchronized (readLock) {\n                return readUntilDataAvailable(b, off, len);\n            }\n        }\n\n        @Override\n        public int available() throws IOException {\n            waitForHandshake();\n            synchronized (readLock) {\n                init();\n                return fromEngine.remaining();\n            }\n        }\n\n        private boolean isHandshaking(HandshakeStatus status) {\n            switch(status) {\n                case NEED_TASK:\n                case NEED_WRAP:\n                case NEED_UNWRAP:\n                    return true;\n                default:\n                    return false;\n            }\n        }\n\n        private int readUntilDataAvailable(byte[] b, int off, int len) throws IOException {\n            int count;\n            do {\n                count = processDataFromSocket(b, off, len);\n            } while (count == 0);\n            return count;\n        }\n\n        // Returns any decrypted data from the engine.  If no data is currently present in the\n        // engine's output buffer, reads from the input socket until the engine has processed\n        // at least one TLS record, then returns any data in the output buffer or 0 if no\n        // data is available.  This is used both during handshaking (in which case, the records\n        // will produce no data and this method will return 0) and by the InputStream read()\n        // methods that expect records to produce application data.\n        private int processDataFromSocket(byte[] b, int off, int len) throws IOException {\n            Platform.blockGuardOnNetwork();\n            checkOpen();\n\n            // Make sure the input stream has been created.\n            init();\n\n            for (;;) {\n                // Serve any remaining data from the engine first.\n                if (fromEngine.remaining() > 0) {\n                    int readFromEngine = Math.min(fromEngine.remaining(), len);\n                    fromEngine.get(b, off, readFromEngine);\n                    return readFromEngine;\n                }\n\n                // Try to unwrap any data already in the socket buffer.\n                boolean needMoreDataFromSocket = true;\n\n                // Unwrap the unencrypted bytes into the engine buffer.\n                fromSocket.flip();\n                fromEngine.clear();\n\n                boolean engineHandshaking = isHandshaking(engine.getHandshakeStatus());\n                SSLEngineResult engineResult = engine.unwrap(fromSocket, fromEngine);\n\n                // Shift any remaining data to the beginning of the buffer so that\n                // we can accommodate the next full packet. After this is called,\n                // limit will be restored to capacity and position will point just\n                // past the end of the data.\n                fromSocket.compact();\n                fromEngine.flip();\n\n                switch (engineResult.getStatus()) {\n                    case BUFFER_UNDERFLOW: {\n                        if (engineResult.bytesProduced() == 0) {\n                            // Need to read more data from the socket.\n                            break;\n                        }\n                        // Also serve the data that was produced.\n                        needMoreDataFromSocket = false;\n                        break;\n                    }\n                    case OK: {\n                        // We processed the entire packet successfully...\n\n                        if (!engineHandshaking && isHandshaking(engineResult.getHandshakeStatus())\n                            && isHandshakeFinished()) {\n                            // The received packet is the beginning of a renegotiation handshake.\n                            // Perform another handshake.\n                            renegotiate();\n                            return 0;\n                        }\n\n                        needMoreDataFromSocket = false;\n                        break;\n                    }\n                    case CLOSED: {\n                        // EOF\n                        return -1;\n                    }\n                    default: {\n                        // Anything else is an error.\n                        throw new SSLException(\n                                \"Unexpected engine result \" + engineResult.getStatus());\n                    }\n                }\n\n                if (!needMoreDataFromSocket && engineResult.bytesProduced() == 0) {\n                    // Read successfully, but produced no data. Possibly part of a\n                    // handshake.\n                    return 0;\n                }\n\n                // Read more data from the socket.\n                if (needMoreDataFromSocket && readFromSocket() == -1) {\n                    // Failed to read the next encrypted packet before reaching EOF.\n                    return -1;\n                }\n\n                // Continue the loop and return the data from the engine buffer.\n            }\n        }\n\n        private boolean isHandshakeFinished() {\n            synchronized (stateLock) {\n                return state > STATE_HANDSHAKE_STARTED;\n            }\n        }\n\n        /**\n         * Processes a renegotiation received from the remote peer.\n         */\n        private void renegotiate() throws IOException {\n            synchronized (handshakeLock) {\n                doHandshake();\n            }\n        }\n\n        private void init() throws IOException {\n            if (socketInputStream == null) {\n                socketInputStream = getUnderlyingInputStream();\n            }\n        }\n\n        private int readFromSocket() throws IOException {\n            try {\n                // Read directly to the underlying array and increment the buffer position if\n                // appropriate.\n                int pos = fromSocket.position();\n                int lim = fromSocket.limit();\n                int read = socketInputStream.read(\n                    fromSocket.array(), fromSocketArrayOffset + pos, lim - pos);\n\n                if (read > 0) {\n                    fromSocket.position(pos + read);\n                }\n                return read;\n            } catch (EOFException e) {\n                return -1;\n            }\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ByteArray.java\n/*\n * Copyright (C) 2011 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.util.Arrays;\n\n/**\n * Byte array wrapper for hashtable use. Implements equals() and hashCode().\n */\nfinal class ByteArray {\n    private final byte[] bytes;\n    private final int hashCode;\n\n    ByteArray(byte[] bytes) {\n        this.bytes = bytes;\n        this.hashCode = Arrays.hashCode(bytes);\n    }\n\n    @Override\n    public int hashCode() {\n        return hashCode;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (!(o instanceof ByteArray)) {\n            return false;\n        }\n        ByteArray lhs = (ByteArray) o;\n        if (hashCode != lhs.hashCode) {\n            return false;\n        }\n        return Arrays.equals(bytes, lhs.bytes);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/HpkeSpi.java\npackage org.conscrypt;\n\nimport java.security.GeneralSecurityException;\nimport java.security.InvalidKeyException;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\n\n/**\n * SPI for HPKE clients to communicate with implementations.  The client API can use any\n * implementation which implements this interface, by duck-typing if necessary.\n */\npublic interface HpkeSpi {\n  byte[] DEFAULT_PSK = new byte[0];\n  byte[] DEFAULT_PSK_ID = DEFAULT_PSK;\n\n  /**\n   * Initialises an HPKE sender SPI.\n   *\n   * @param recipientKey public key of the recipient\n   * @param info application-supplied information, may be null or empty\n   * @param senderKey private key of the sender, for symmetric auth modes only, else null\n   * @param psk pre-shared key, for PSK auth modes only, else null\n   * @param psk_id pre-shared key ID, for PSK auth modes only, else null\n   * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n   * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n   * @throws IllegalStateException if this SPI has already been initialised\n   */\n  void engineInitSender(\n          PublicKey recipientKey,\n          byte[] info,\n          PrivateKey senderKey,\n          byte[] psk,\n          byte[] psk_id)\n          throws InvalidKeyException;\n\n  /**\n   * Initialises an HPKE sender SPI.\n   *\n   * @param recipientKey public key of the recipient\n   * @param info application-supplied information, may be null or empty\n   * @param senderKey private key of the sender, for symmetric auth modes only, else null\n   * @param psk pre-shared key, for PSK auth modes only, else null\n   * @param psk_id pre-shared key ID, for PSK auth modes only, else null\n   * @param sKe optional random seed, should be null for all uses except for validation against\n   *            known test vectors\n   * @throws InvalidKeyException if recipientKey is null or an unsupported key format or senderKey\n   *            is an unsupported key format\n   * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n   * @throws IllegalStateException if this SPI has already been initialised\n   */\n  void engineInitSenderForTesting(\n          PublicKey recipientKey,\n          byte[] info,\n          PrivateKey senderKey,\n          byte[] psk,\n          byte[] psk_id,\n          byte[] sKe)\n          throws InvalidKeyException;\n\n  /**\n   * Initialises an HPKE recipient SPI.\n   *\n   * @param encapsulated encapsulated ephemeral key from a sender\n   * @param recipientKey private key of the recipient\n   * @param info application-supplied information, may be null or empty\n   * @param senderKey public key of sender, for asymmetric auth modes only, else null\n   * @param psk pre-shared key, for PSK auth modes only, else null\n   * @param psk_id pre-shared key ID, for PSK auth modes only, else null\n   * @throws InvalidKeyException if recipientKey is null or an unsupported key format or senderKey\n   *         is an unsupported key format\n   * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n   * @throws IllegalStateException if this SPI has already been initialised\n   */\n  void engineInitRecipient(\n          byte[] encapsulated,\n          PrivateKey recipientKey,\n          byte[] info,\n          PublicKey senderKey,\n          byte[] psk,\n          byte[] psk_id)\n          throws InvalidKeyException;\n\n  /**\n   * Seals a message, using the internal key schedule maintained by an HPKE sender.\n   *\n   * @param plaintext the plaintext\n   * @param aad optional associated data, may be null or empty\n   * @return the ciphertext\n   * @throws NullPointerException if the plaintext is null\n   * @throws IllegalStateException if this SPI has not been initialised or if it was initialised\n   *         as a recipient\n   */\n  byte[] engineSeal(byte[] plaintext, byte[] aad);\n\n  /**\n   * Opens a message, using the internal key schedule maintained by an HPKE recipient.\n   *\n   * @param ciphertext the ciphertext\n   * @param aad optional associated data, may be null or empty\n   * @return the plaintext\n   * @throws IllegalStateException if this SPI has not been initialised or if it was initialised\n   *         as a sender\n   * @throws GeneralSecurityException on decryption failures\n   */\n  byte[] engineOpen(byte[] ciphertext, byte[] aad) throws GeneralSecurityException;\n\n  /**\n   * Exports secret key material from this SPI as described in RFC 9180.\n   *\n   * @param length  expected output length\n   * @param context optional context string, may be null or empty\n   * @return exported value\n   * @throws IllegalArgumentException if the length is not valid for the KDF in use\n   * @throws IllegalStateException if this SPI has not been initialised\n   *\n   */\n  byte[] engineExport(int length, byte[] context);\n\n  /**\n   * Returns the encapsulated key material for an HPKE sender.\n   *\n   * @return the key material\n   * @throws IllegalStateException if this SPI has not been initialised or if it was initialised\n   *         as a recipient\n   */\n  byte[] getEncapsulated();\n}\n\n// Path: common/src/main/java/org/conscrypt/HpkeContextRecipient.java\n/*\n * Copyright (C) 2023 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License\n */\n\npackage org.conscrypt;\n\nimport java.security.GeneralSecurityException;\nimport java.security.InvalidKeyException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.NoSuchProviderException;\nimport java.security.PrivateKey;\nimport java.security.Provider;\nimport java.security.PublicKey;\n\n/**\n * Hybrid Public Key Encryption (HPKE) recipient APIs.\n *\n * @see <a href=\"https://www.rfc-editor.org/rfc/rfc9180.html#hpke-export\">HPKE RFC 9180</a>\n *\n * Recipient subclass of HpkeContext.  See base class for details.\n */\npublic class HpkeContextRecipient extends HpkeContext {\n    private HpkeContextRecipient(HpkeSpi spi) {\n        super(spi);\n    }\n\n    /**\n     * Opens a message, using the internal key schedule maintained by this HpkeContextRecipient.\n     *\n     * @param ciphertext the ciphertext\n     * @param aad optional associated data, may be null or empty\n     * @return the plaintext\n     * @throws IllegalStateException if this HpkeContextRecipient has not been initialised\n     * @throws GeneralSecurityException on decryption failures\n     */\n    public byte[] open(byte[] ciphertext, byte[] aad) throws GeneralSecurityException {\n        return spi.engineOpen(ciphertext, aad);\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextRecipient.\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @return an uninitialised HpkeContextRecipient for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     */\n    public static HpkeContextRecipient getInstance(String suite) throws NoSuchAlgorithmException {\n        return new HpkeContextRecipient(findSpi(suite));\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextRecipient from a specific {@link Provider}\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @param providerName the name of the Provider to use\n     * @return an uninitialised HpkeContextRecipient for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     * @throws NoSuchProviderException if providerName is null or no such Provider exists\n     */\n    public static HpkeContextRecipient getInstance(String suite, String providerName)\n        throws NoSuchAlgorithmException, NoSuchProviderException {\n        return new HpkeContextRecipient(findSpi(suite, providerName));\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextRecipient from a specific {@link Provider}\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @param provider the Provider to use\n     * @return an uninitialised HpkeContextRecipient for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     * @throws NoSuchProviderException if providerName is null or no such Provider exists\n     */\n    public static HpkeContextRecipient getInstance(String suite, Provider provider)\n        throws NoSuchAlgorithmException, NoSuchProviderException {\n        return new HpkeContextRecipient(findSpi(suite, provider));\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in BASE mode, i.e. no sender authentication.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info)\n            throws InvalidKeyException {\n        spi.engineInitRecipient(encapsulated, recipientKey, info, null,\n                HpkeSpi.DEFAULT_PSK, HpkeSpi.DEFAULT_PSK_ID);\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in AUTH mode, i.e. messages are authenticated using\n     * the sender's public key.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @param senderKey the public key of the sender\n     * @throws InvalidKeyException if either recipientKey or senderKey are null\n     *         or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info, PublicKey senderKey)\n        throws InvalidKeyException {\n        if (senderKey == null) {\n            throw new InvalidKeyException(\"null sender key\");\n        }\n        // Remaining argument checks are performed by the SPI\n        spi.engineInitRecipient(encapsulated, recipientKey, info, senderKey,\n                HpkeSpi.DEFAULT_PSK, HpkeSpi.DEFAULT_PSK_ID);\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in PSK_AUTH mode, i.e. messages are authenticated using\n     * a pre-shared secret key.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @param psk the a pre-shared secret key\n     * @param psk_id the id of the pre-shared secret key\n     * @throws NullPointerException if psk or psk_id are null\n     * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info,\n            byte[] psk, byte[] psk_id) throws InvalidKeyException {\n        spi.engineInitRecipient(encapsulated, recipientKey, info, null, psk, psk_id);\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in PSK_AUTH mode, i.e. messages are authenticated using\n     * both the sender's public key and a pre-shared secret key.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @param senderKey the public key of the sender\n     * @param psk the a pre-shared secret key\n     * @param psk_id the id of the pre-shared secret key\n     * @throws NullPointerException if psk or psk_id are null\n     * @throws InvalidKeyException if either recipientKey or senderKey are null\n     *         or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info, PublicKey senderKey,\n            byte[] psk, byte[] psk_id) throws InvalidKeyException {\n        if (senderKey == null) {\n            throw new InvalidKeyException(\"null sender key\");\n        }\n        // Remaining argument checks are performed by the SPI\n        spi.engineInitRecipient(encapsulated, recipientKey, info, senderKey, psk, psk_id);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLRSAKeyPairGenerator.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.math.BigInteger;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.KeyPair;\nimport java.security.KeyPairGeneratorSpi;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport java.security.spec.RSAKeyGenParameterSpec;\n\n/**\n * An implementation of {@link java.security.KeyPairGenerator} which uses BoringSSL to perform all\n * the operations.\n */\n@Internal\npublic final class OpenSSLRSAKeyPairGenerator extends KeyPairGeneratorSpi {\n    /**\n     * Default modulus size is 0x10001 (65537)\n     */\n    private byte[] publicExponent = new byte[] {\n            0x01, 0x00, 0x01\n    };\n\n    /**\n     * Default RSA key size 2048 bits.\n     */\n    private int modulusBits = 2048;\n\n    @Override\n    public KeyPair generateKeyPair() {\n        final OpenSSLKey key = new OpenSSLKey(NativeCrypto.RSA_generate_key_ex(modulusBits,\n                publicExponent));\n\n        PrivateKey privKey = OpenSSLRSAPrivateKey.getInstance(key);\n        PublicKey pubKey = new OpenSSLRSAPublicKey(key);\n\n        return new KeyPair(pubKey, privKey);\n    }\n\n    @Override\n    public void initialize(int keysize, SecureRandom random) {\n        this.modulusBits = keysize;\n    }\n\n    @Override\n    public void initialize(AlgorithmParameterSpec params, SecureRandom random)\n            throws InvalidAlgorithmParameterException {\n        if (!(params instanceof RSAKeyGenParameterSpec)) {\n            throw new InvalidAlgorithmParameterException(\"Only RSAKeyGenParameterSpec supported\");\n        }\n\n        RSAKeyGenParameterSpec spec = (RSAKeyGenParameterSpec) params;\n\n        final BigInteger publicExponent = spec.getPublicExponent();\n        if (publicExponent != null) {\n            this.publicExponent = publicExponent.toByteArray();\n        }\n\n        this.modulusBits = spec.getKeysize();\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLBaseDHKeyAgreement.java\n/*\n * Copyright (C) 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.Key;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport javax.crypto.KeyAgreementSpi;\nimport javax.crypto.SecretKey;\nimport javax.crypto.ShortBufferException;\nimport javax.crypto.spec.SecretKeySpec;\n\n/**\n * Generic base classe for Diffie-Hellman style key agreement backed by the OpenSSL engine.\n */\n@Internal\npublic abstract class OpenSSLBaseDHKeyAgreement<T> extends KeyAgreementSpi {\n\n    /** OpenSSL handle of the private key. Only available after the engine has been initialized. */\n    private T mPrivateKey;\n\n    /**\n     * Expected length (in bytes) of the agreed key ({@link #mResult}). Only available after the\n     * engine has been initialized.\n     */\n    private int mExpectedResultLength;\n\n    /** Agreed key. Only available after {@link #engineDoPhase(Key, boolean)} completes. */\n    private byte[] mResult;\n\n    protected OpenSSLBaseDHKeyAgreement() {}\n\n    @Override\n    public Key engineDoPhase(Key key, boolean lastPhase) throws InvalidKeyException {\n        if (mPrivateKey == null) {\n            throw new IllegalStateException(\"Not initialized\");\n        }\n        if (!lastPhase) {\n            throw new IllegalStateException(\"DH only has one phase\");\n        }\n\n        if (key == null) {\n            throw new InvalidKeyException(\"key == null\");\n        }\n        if (!(key instanceof PublicKey)) {\n            throw new InvalidKeyException(\"Not a public key: \" + key.getClass());\n        }\n        T openSslPublicKey = convertPublicKey((PublicKey) key);\n\n        byte[] buffer = new byte[mExpectedResultLength];\n        int actualResultLength = computeKey(buffer, openSslPublicKey, mPrivateKey);\n        byte[] result;\n        if (actualResultLength == -1) {\n            throw new RuntimeException(\"Engine returned -1\");\n        } else if (actualResultLength == mExpectedResultLength) {\n            // The output is as long as expected -- use the whole buffer\n            result = buffer;\n        } else if (actualResultLength < mExpectedResultLength) {\n            // The output is shorter than expected -- use only what's produced by the engine\n            result = new byte[actualResultLength];\n            System.arraycopy(buffer, 0, mResult, 0, mResult.length);\n        } else {\n            // The output is longer than expected\n            throw new RuntimeException(\"Engine produced a longer than expected result. Expected: \"\n                + mExpectedResultLength + \", actual: \" + actualResultLength);\n        }\n        mResult = result;\n\n        return null; // No intermediate key\n    }\n\n    protected abstract T convertPublicKey(PublicKey key) throws InvalidKeyException;\n\n    protected abstract T convertPrivateKey(PrivateKey key) throws InvalidKeyException;\n\n    /**\n     * Given the public key {@code theirPublicKey} and the private key {@code ourPrivateKey} write the shared secret\n     * to {@code buffer} and return the actual output size.\n     */\n    protected abstract int computeKey(byte[] buffer, T theirPublicKey, T ourPrivateKey) throws InvalidKeyException;\n\n    @Override\n    protected int engineGenerateSecret(byte[] sharedSecret, int offset)\n            throws ShortBufferException {\n        checkCompleted();\n        int available = sharedSecret.length - offset;\n        if (mResult.length > available) {\n            throw new ShortBufferWithoutStackTraceException(\n                    \"Needed: \" + mResult.length + \", available: \" + available);\n        }\n\n        System.arraycopy(mResult, 0, sharedSecret, offset, mResult.length);\n        return mResult.length;\n    }\n\n    @Override\n    protected byte[] engineGenerateSecret() {\n        checkCompleted();\n        return mResult;\n    }\n\n    @Override\n    protected SecretKey engineGenerateSecret(String algorithm) {\n        checkCompleted();\n        return new SecretKeySpec(engineGenerateSecret(), algorithm);\n    }\n\n    @Override\n    protected void engineInit(Key key, SecureRandom random) throws InvalidKeyException {\n        if (key == null) {\n            throw new InvalidKeyException(\"key == null\");\n        }\n        if (!(key instanceof PrivateKey)) {\n            throw new InvalidKeyException(\"Not a private key: \" + key.getClass());\n        }\n\n        T privateKey = convertPrivateKey((PrivateKey) key);\n        mExpectedResultLength = getOutputSize(privateKey);\n        mPrivateKey = privateKey;\n    }\n\n    /** Returns the expected result length for the given {@code key}. */\n    protected abstract int getOutputSize(T key);\n\n    @Override\n    protected void engineInit(Key key, AlgorithmParameterSpec params,\n            SecureRandom random) throws InvalidKeyException, InvalidAlgorithmParameterException {\n        // ECDH doesn't need an AlgorithmParameterSpec\n        if (params != null) {\n          throw new InvalidAlgorithmParameterException(\"No algorithm parameters supported\");\n        }\n        engineInit(key, random);\n    }\n\n    private void checkCompleted() {\n        if (mResult == null) {\n            throw new IllegalStateException(\"Key agreement not completed\");\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ct/CTLogStore.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport org.conscrypt.Internal;\n\n@Internal\npublic interface CTLogStore {\n    CTLogInfo getKnownLog(byte[] logId);\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CTPolicy.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.security.cert.X509Certificate;\nimport org.conscrypt.Internal;\n\n@Internal\npublic interface CTPolicy {\n    boolean doesResultConformToPolicy(CTVerificationResult result, String hostname,\n            X509Certificate[] chain);\n}\n\n// Path: common/src/main/java/org/conscrypt/ct/VerifiedSCT.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport org.conscrypt.Internal;\n\n/**\n * Verification result for a single SCT.\n */\n@Internal\npublic final class VerifiedSCT {\n    public enum Status {\n        VALID,\n        INVALID_SIGNATURE,\n        UNKNOWN_LOG,\n        INVALID_SCT\n    }\n\n    public final SignedCertificateTimestamp sct;\n    public final Status status;\n\n    public VerifiedSCT(SignedCertificateTimestamp sct, Status status) {\n        this.sct = sct;\n        this.status = status;\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CTVerificationResult.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport org.conscrypt.Internal;\n\n@Internal\npublic class CTVerificationResult {\n    private final ArrayList<VerifiedSCT> validSCTs = new ArrayList<VerifiedSCT>();\n    private final ArrayList<VerifiedSCT> invalidSCTs = new ArrayList<VerifiedSCT>();\n\n    public void add(VerifiedSCT result) {\n        if (result.status == VerifiedSCT.Status.VALID) {\n            validSCTs.add(result);\n        } else {\n            invalidSCTs.add(result);\n        }\n    }\n\n    public List<VerifiedSCT> getValidSCTs() {\n        return Collections.unmodifiableList(validSCTs);\n    }\n\n    public List<VerifiedSCT> getInvalidSCTs() {\n        return Collections.unmodifiableList(invalidSCTs);\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CTConstants.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport org.conscrypt.Internal;\n\n@Internal\npublic class CTConstants {\n    public static final String X509_SCT_LIST_OID = \"1.3.6.1.4.1.11129.2.4.2\";\n    public static final String OCSP_SCT_LIST_OID = \"1.3.6.1.4.1.11129.2.4.5\";\n\n    public static final int VERSION_LENGTH = 1;\n    public static final int LOGID_LENGTH = 32;\n    public static final int TIMESTAMP_LENGTH = 8;\n    public static final int EXTENSIONS_LENGTH_BYTES = 2;\n\n    public static final int HASH_ALGORITHM_LENGTH = 1;\n    public static final int SIGNATURE_ALGORITHM_LENGTH = 1;\n    public static final int SIGNATURE_LENGTH_BYTES = 2;\n\n    public static final int SIGNATURE_TYPE_LENGTH = 1;\n    public static final int LOG_ENTRY_TYPE_LENGTH = 2;\n    public static final int CERTIFICATE_LENGTH_BYTES = 3;\n\n    public static final int SERIALIZED_SCT_LENGTH_BYTES = 2;\n    public static final int SCT_LIST_LENGTH_BYTES = 2;\n\n    public static final int ISSUER_KEY_HASH_LENGTH = 32;\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/Serialization.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.util.ArrayList;\nimport org.conscrypt.Internal;\n\n@Internal\npublic class Serialization {\n    private Serialization() {}\n\n    private static final int DER_TAG_MASK = 0x3f;\n    private static final int DER_TAG_OCTET_STRING = 0x4;\n    private static final int DER_LENGTH_LONG_FORM_FLAG = 0x80;\n\n    public static byte[] readDEROctetString(byte[] input)\n            throws SerializationException {\n        return readDEROctetString(new ByteArrayInputStream(input));\n    }\n\n    public static byte[] readDEROctetString(InputStream input)\n            throws SerializationException {\n        int tag = readByte(input) & DER_TAG_MASK;\n        if (tag != DER_TAG_OCTET_STRING) {\n            throw new SerializationException(\"Wrong DER tag, expected OCTET STRING, got \" + tag);\n        }\n        int length;\n        int width = readNumber(input, 1);\n        if ((width & DER_LENGTH_LONG_FORM_FLAG) != 0) {\n            length = readNumber(input, width & ~DER_LENGTH_LONG_FORM_FLAG);\n        } else {\n            length = width;\n        }\n\n        return readFixedBytes(input, length);\n    }\n\n    public static byte[][] readList(byte[] input, int listWidth, int elemWidth)\n            throws SerializationException {\n        return readList(new ByteArrayInputStream(input), listWidth, elemWidth);\n    }\n\n    /**\n     * Read a variable length vector of variable sized elements as described by RFC5246 section 4.3.\n     * The vector is prefixed by its total length, in bytes and in big endian format,\n     * so is each element contained in the vector.\n     * @param listWidth the width of the vector's length field, in bytes.\n     * @param elemWidth the width of each element's length field, in bytes.\n     * @throws SerializationException if EOF is encountered.\n     */\n    public static byte[][] readList(InputStream input, int listWidth, int elemWidth)\n            throws SerializationException {\n        ArrayList<byte[]> result = new ArrayList<byte[]>();\n        byte[] data = readVariableBytes(input, listWidth);\n        input = new ByteArrayInputStream(data);\n        try {\n            while (input.available() > 0) {\n                result.add(readVariableBytes(input, elemWidth));\n            }\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n        return result.toArray(new byte[result.size()][]);\n    }\n\n    /**\n     * Read a length-prefixed sequence of bytes.\n     * The length must be encoded in big endian format.\n     * @param width the width of the length prefix, in bytes.\n     * @throws SerializationException if EOF is encountered, or if {@code width} is negative or\n     * greater than 4\n     */\n    public static byte[] readVariableBytes(InputStream input, int width)\n            throws SerializationException {\n        int length = readNumber(input, width);\n        return readFixedBytes(input, length);\n    }\n\n    /**\n     * Read a fixed number of bytes from the input stream.\n     * @param length the number of bytes to read.\n     * @throws SerializationException if EOF is encountered.\n     */\n    public static byte[] readFixedBytes(InputStream input, int length)\n            throws SerializationException {\n        try {\n            if (length < 0) {\n                throw new SerializationException(\"Negative length: \" + length);\n            }\n\n            byte[] data = new byte[length];\n            int count = input.read(data);\n            if (count < length) {\n                throw new SerializationException(\"Premature end of input, expected \" + length +\n                                                 \" bytes, only read \" + count);\n            }\n            return data;\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n\n    /**\n     * Read a number in big endian format from the input stream.\n     * This methods only supports a width of up to 4 bytes.\n     * @param width the width of the number, in bytes.\n     * @throws SerializationException if EOF is encountered, or if {@code width} is negative or\n     * greater than 4\n     */\n    public static int readNumber(InputStream input, int width) throws SerializationException {\n        if (width > 4 || width < 0) {\n            throw new SerializationException(\"Invalid width: \" + width);\n        }\n\n        int result = 0;\n        for (int i = 0; i < width; i++) {\n            result = (result << 8) | (readByte(input) & 0xFF);\n        }\n\n        return result;\n    }\n\n    /**\n     * Read a number in big endian format from the input stream.\n     * This methods supports a width of up to 8 bytes.\n     * @param width the width of the number, in bytes.\n     * @throws SerializationException if EOF is encountered.\n     * @throws IllegalArgumentException if {@code width} is negative or greater than 8\n     */\n    public static long readLong(InputStream input, int width) throws SerializationException {\n        if (width > 8 || width < 0) {\n            throw new IllegalArgumentException(\"Invalid width: \" + width);\n        }\n\n        long result = 0;\n        for (int i = 0; i < width; i++) {\n            result = (result << 8) | (readByte(input) & 0xFF);\n        }\n\n        return result;\n    }\n\n    /**\n     * Read a single byte from the input stream.\n     * @throws SerializationException if EOF is encountered.\n     */\n    public static byte readByte(InputStream input) throws SerializationException {\n        try {\n            int b = input.read();\n            if (b == -1) {\n                throw new SerializationException(\"Premature end of input, could not read byte.\");\n            }\n            return (byte)b;\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n\n    /**\n     * Write length prefixed sequence of bytes to the ouput stream.\n     * The length prefix is encoded in big endian order.\n     * @param data the data to be written.\n     * @param width the width of the length prefix, in bytes.\n     * @throws SerializationException if the length of {@code data} is too large to fit in\n     * {@code width} bytes or {@code width} is negative.\n     */\n    public static void writeVariableBytes(OutputStream output, byte[] data, int width)\n            throws SerializationException {\n        writeNumber(output, data.length, width);\n        writeFixedBytes(output, data);\n    }\n\n    /**\n     * Write a fixed number sequence of bytes to the ouput stream.\n     * @param data the data to be written.\n     */\n    public static void writeFixedBytes(OutputStream output, byte[] data)\n            throws SerializationException {\n        try {\n            output.write(data);\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n\n    /**\n     * Write a number to the output stream.\n     * The number is encoded in big endian order.\n     * @param value the value to be written.\n     * @param width the width of the encoded number, in bytes\n     * @throws SerializationException if the number is too large to fit in {@code width} bytes or\n     * {@code width} is negative.\n     */\n    public static void writeNumber(OutputStream output, long value, int width)\n            throws SerializationException {\n        if (width < 0) {\n            throw new SerializationException(\"Negative width: \" + width);\n        }\n        if (width < 8 && value >= (1L << (8 * width))) {\n            throw new SerializationException(\n                    \"Number too large, \" + value + \" does not fit in \" + width + \" bytes\");\n        }\n\n        try {\n            while (width > 0) {\n                long shift = (width - 1) * 8L;\n                // Java behaves weirdly if shifting by more than the variable's size\n                if (shift < Long.SIZE) {\n                    output.write((byte) ((value >> shift) & 0xFF));\n                } else {\n                    output.write(0);\n                }\n\n                width--;\n            }\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CertificateEntry.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.io.OutputStream;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.X509Certificate;\nimport org.conscrypt.Internal;\nimport org.conscrypt.OpenSSLX509Certificate;\n\n/**\n * CertificateEntry structure.\n * This structure describes part of the data which is signed over in SCTs.\n * It is not defined by the RFC6962, but it is useful to have.\n *\n * It's definition would be :\n * struct {\n *     LogEntryType entry_type;\n *     select(entry_type) {\n *         case x509_entry: ASN.1Cert;\n *         case precert_entry: PreCert;\n *     } signed_entry;\n * } CertificateEntry;\n */\n@Internal\npublic class CertificateEntry {\n    public enum LogEntryType {\n        X509_ENTRY,\n        PRECERT_ENTRY\n    }\n\n    private final LogEntryType entryType;\n\n    // Only used when entryType is LOG_ENTRY_TYPE_PRECERT\n    private final byte[] issuerKeyHash;\n\n    /* If entryType == PRECERT_ENTRY, this is the encoded TBS of the precertificate.\n       If entryType == X509_ENTRY, this is the encoded leaf certificate. */\n    private final byte[] certificate;\n\n    private CertificateEntry(LogEntryType entryType, byte[] certificate, byte[] issuerKeyHash) {\n        if (entryType == LogEntryType.PRECERT_ENTRY && issuerKeyHash == null) {\n            throw new IllegalArgumentException(\"issuerKeyHash missing for precert entry.\");\n        } else if (entryType == LogEntryType.X509_ENTRY && issuerKeyHash != null) {\n            throw new IllegalArgumentException(\"unexpected issuerKeyHash for X509 entry.\");\n        }\n        \n        if (issuerKeyHash != null && issuerKeyHash.length != CTConstants.ISSUER_KEY_HASH_LENGTH) {\n            throw new IllegalArgumentException(\"issuerKeyHash must be 32 bytes long\");\n        }\n\n        this.entryType = entryType;\n        this.issuerKeyHash = issuerKeyHash;\n        this.certificate = certificate;\n    }\n\n    /**\n     * Creates a CertificateEntry with type PRECERT_ENTRY\n     *\n     * @throws IllegalArgumentException if issuerKeyHash isn't 32 bytes\n     */\n    public static CertificateEntry createForPrecertificate(byte[] tbsCertificate, byte[] issuerKeyHash) {\n        return new CertificateEntry(LogEntryType.PRECERT_ENTRY, tbsCertificate, issuerKeyHash);\n    }\n\n    public static CertificateEntry createForPrecertificate(OpenSSLX509Certificate leaf,\n            OpenSSLX509Certificate issuer) throws CertificateException {\n        try {\n            if (!leaf.getNonCriticalExtensionOIDs().contains(CTConstants.X509_SCT_LIST_OID)) {\n                throw new CertificateException(\"Certificate does not contain embedded signed timestamps\");\n            }\n\n            byte[] tbs = leaf.getTBSCertificateWithoutExtension(CTConstants.X509_SCT_LIST_OID);\n\n            byte[] issuerKey = issuer.getPublicKey().getEncoded();\n            MessageDigest md = MessageDigest.getInstance(\"SHA-256\");\n            md.update(issuerKey);\n            byte[] issuerKeyHash = md.digest();\n\n            return createForPrecertificate(tbs, issuerKeyHash);\n        } catch (NoSuchAlgorithmException e) {\n            // SHA-256 is guaranteed to be available\n            throw new RuntimeException(e);\n        }\n    }\n\n...\n", "needle_token_start": 746, "needle_token_end": 864, "code_context_ntokens": 15981, "output": ["```java\nprivate void onEngineHandshakeFinished() {\n    // Don't do anything here except change state.  This method will be called from\n    // e.g. wrap() which is non re-entrant so we can't call anything that might do\n    // IO until after it exits, e.g. in doHandshake().\n    if (isState(STATE_HANDSHAKE_STARTED)) {\n        transitionTo(STATE_HANDSHAKE_COMPLETED);\n    }\n}\n```"]}
{"repo": "google/conscrypt", "name": "engineGetKeyManagers", "language": "java", "path": "common/src/main/java/org/conscrypt/KeyManagerFactoryImpl.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The method is designed to retrieve an array of key management interfaces that handle the interaction with key material for use in securing socket connections.\n2. **Input**: There are no direct parameters taken by this method; however, it relies on the internal state which should have been previously initialized with a key store and potentially a password.\n3. **Output**: Returns an array of key management interfaces, specifically designed for managing the keys stored in the initialized key store.\n4. **Procedure**: \n   - First, the method checks if the internal key store has been initialized. If not, it throws an exception indicating that the factory is not properly initialized.\n   - If the key store is initialized, the method creates and returns an array containing a single key manager initialized with the key store and the password (if provided).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/OpenSSLSignatureRawECDSA.java\n/*\n * Copyright (C) 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt;\n\nimport java.io.ByteArrayOutputStream;\nimport java.security.InvalidKeyException;\nimport java.security.InvalidParameterException;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.security.SignatureException;\nimport java.security.SignatureSpi;\n\n/**\n * Implements the JDK Signature interface needed for RAW ECDSA signature\n * generation and verification using BoringSSL.\n */\n@Internal\npublic class OpenSSLSignatureRawECDSA extends SignatureSpi {\n    /**\n     * The current OpenSSL key we're operating on.\n     */\n    private OpenSSLKey key;\n\n    /**\n     * Buffer to hold value to be signed or verified.\n     */\n    private ByteArrayOutputStream buffer = new ByteArrayOutputStream();\n\n    public OpenSSLSignatureRawECDSA() {}\n\n    @Override\n    protected void engineUpdate(byte input) {\n        buffer.write(input);\n    }\n\n    @Override\n    protected void engineUpdate(byte[] input, int offset, int len) {\n        buffer.write(input, offset, len);\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\")\n    protected Object engineGetParameter(String param) throws InvalidParameterException {\n        return null;\n    }\n\n    private static OpenSSLKey verifyKey(OpenSSLKey key) throws InvalidKeyException {\n        int pkeyType = NativeCrypto.EVP_PKEY_type(key.getNativeRef());\n...\n// Path: common/src/main/java/org/conscrypt/X509PublicKey.java\n/*\n * Copyright 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.PublicKey;\nimport java.util.Arrays;\n\n/**\n * A simple but useless key class that holds X.509 public key information when\n * the appropriate KeyFactory for the key algorithm is not available.\n */\nfinal class X509PublicKey implements PublicKey {\n    private static final long serialVersionUID = -8610156854731664298L;\n\n    private final String algorithm;\n\n    private final byte[] encoded;\n\n    X509PublicKey(String algorithm, byte[] encoded) {\n        this.algorithm = algorithm;\n        this.encoded = encoded;\n    }\n\n    @Override\n    public String getAlgorithm() {\n        return algorithm;\n    }\n\n    @Override\n    public String getFormat() {\n        return \"X.509\";\n    }\n\n    @Override\n    public byte[] getEncoded() {\n        return encoded;\n    }\n\n    @Override\n    public String toString() {\n        return \"X509PublicKey [algorithm=\" + algorithm + \", encoded=\" + Arrays.toString(encoded)\n                + \"]\";\n    }\n\n    @Override\n    public int hashCode() {\n        final int prime = 31;\n        int result = 1;\n        result = prime * result + ((algorithm == null) ? 0 : algorithm.hashCode());\n        result = prime * result + Arrays.hashCode(encoded);\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj)\n            return true;\n        if (obj == null)\n            return false;\n        if (getClass() != obj.getClass())\n            return false;\n        X509PublicKey other = (X509PublicKey) obj;\n        if (algorithm == null) {\n            if (other.algorithm != null)\n                return false;\n        } else if (!algorithm.equals(other.algorithm))\n            return false;\n        if (!Arrays.equals(encoded, other.encoded))\n            return false;\n        return true;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/KeyManagerFactoryImpl.java\n/*\n *  Licensed to the Apache Software Foundation (ASF) under one or more\n *  contributor license agreements.  See the NOTICE file distributed with\n *  this work for additional information regarding copyright ownership.\n *  The ASF licenses this file to You under the Apache License, Version 2.0\n *  (the \"License\"); you may not use this file except in compliance with\n *  the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\npackage org.conscrypt;\n\nimport java.io.File;\nimport java.io.FileInputStream;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.KeyStore;\nimport java.security.KeyStoreException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.UnrecoverableKeyException;\nimport java.security.cert.CertificateException;\nimport javax.net.ssl.KeyManager;\nimport javax.net.ssl.KeyManagerFactorySpi;\nimport javax.net.ssl.ManagerFactoryParameters;\nimport org.conscrypt.io.IoUtils;\n\n/**\n * KeyManagerFactory implementation.\n * @see KeyManagerFactorySpi\n */\n@Internal\npublic class KeyManagerFactoryImpl extends KeyManagerFactorySpi {\n\n    // source of key material\n    private KeyStore keyStore;\n\n    //password\n    private char[] pwd;\n\n    /**\n     * @see KeyManagerFactorySpi#engineInit(KeyStore ks, char[] password)\n     */\n    @Override\n    protected void engineInit(KeyStore ks, char[] password)\n            throws KeyStoreException, NoSuchAlgorithmException,\n            UnrecoverableKeyException {\n        if (ks != null) {\n            keyStore = ks;\n            if (password != null) {\n                pwd = password.clone();\n            } else {\n                pwd = EmptyArray.CHAR;\n            }\n        } else {\n            keyStore = KeyStore.getInstance(KeyStore.getDefaultType());\n            String keyStoreName = System.getProperty(\"javax.net.ssl.keyStore\");\n            String keyStorePwd = null;\n            if (keyStoreName == null || keyStoreName.equalsIgnoreCase(\"NONE\") || keyStoreName.isEmpty()) {\n                try {\n                    keyStore.load(null, null);\n                } catch (IOException e) {\n                    throw new KeyStoreException(e);\n                } catch (CertificateException e) {\n                    throw new KeyStoreException(e);\n                }\n            } else {\n                keyStorePwd = System.getProperty(\"javax.net.ssl.keyStorePassword\");\n                if (keyStorePwd == null) {\n                    pwd = EmptyArray.CHAR;\n                } else {\n                    pwd = keyStorePwd.toCharArray();\n                }\n                FileInputStream fis = null;\n                try {\n                    fis = new FileInputStream(new File(keyStoreName));\n                    keyStore.load(fis, pwd);\n                } catch (FileNotFoundException e) {\n                    throw new KeyStoreException(e);\n                } catch (IOException e) {\n                    throw new KeyStoreException(e);\n                } catch (CertificateException e) {\n                    throw new KeyStoreException(e);\n                } finally {\n                    IoUtils.closeQuietly(fis);\n                }\n            }\n\n        }\n\n    }\n\n    /**\n     * @see KeyManagerFactorySpi#engineInit(ManagerFactoryParameters spec)\n     */\n    @Override\n    protected void engineInit(ManagerFactoryParameters spec)\n            throws InvalidAlgorithmParameterException {\n        throw new InvalidAlgorithmParameterException(\n                \"ManagerFactoryParameters not supported\");\n\n    }\n\n    /**\n     * @see KeyManagerFactorySpi#engineGetKeyManagers()\n     */\n    \n@Override\n    protected KeyManager[] engineGetKeyManagers() {\n        if (keyStore == null) {\n            throw new IllegalStateException(\"KeyManagerFactory is not initialized\");\n        }\n        return new KeyManager[] { new KeyManagerImpl(keyStore, pwd) };\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLECKeyFactory.java\n/*\n * Copyright (C) 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.math.BigInteger;\nimport java.security.InvalidKeyException;\nimport java.security.Key;\nimport java.security.KeyFactorySpi;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.security.interfaces.ECPrivateKey;\nimport java.security.interfaces.ECPublicKey;\nimport java.security.spec.ECParameterSpec;\nimport java.security.spec.ECPoint;\nimport java.security.spec.ECPrivateKeySpec;\nimport java.security.spec.ECPublicKeySpec;\nimport java.security.spec.InvalidKeySpecException;\nimport java.security.spec.KeySpec;\nimport java.security.spec.PKCS8EncodedKeySpec;\nimport java.security.spec.X509EncodedKeySpec;\n\n/**\n * An implementation of a {@link KeyFactorySpi} for EC keys based on BoringSSL.\n */\n@Internal\npublic final class OpenSSLECKeyFactory extends KeyFactorySpi {\n\n    public OpenSSLECKeyFactory() {}\n\n    @Override\n    protected PublicKey engineGeneratePublic(KeySpec keySpec) throws InvalidKeySpecException {\n        if (keySpec == null) {\n            throw new InvalidKeySpecException(\"keySpec == null\");\n        }\n\n        if (keySpec instanceof ECPublicKeySpec) {\n            return new OpenSSLECPublicKey((ECPublicKeySpec) keySpec);\n        } else if (keySpec instanceof X509EncodedKeySpec) {\n            return OpenSSLKey.getPublicKey((X509EncodedKeySpec) keySpec, NativeConstants.EVP_PKEY_EC);\n        }\n        throw new InvalidKeySpecException(\"Must use ECPublicKeySpec or X509EncodedKeySpec; was \"\n                + keySpec.getClass().getName());\n    }\n\n    @Override\n    protected PrivateKey engineGeneratePrivate(KeySpec keySpec) throws InvalidKeySpecException {\n        if (keySpec == null) {\n            throw new InvalidKeySpecException(\"keySpec == null\");\n        }\n\n        if (keySpec instanceof ECPrivateKeySpec) {\n            return new OpenSSLECPrivateKey((ECPrivateKeySpec) keySpec);\n        } else if (keySpec instanceof PKCS8EncodedKeySpec) {\n            return OpenSSLKey.getPrivateKey((PKCS8EncodedKeySpec) keySpec,\n                    NativeConstants.EVP_PKEY_EC);\n        }\n        throw new InvalidKeySpecException(\"Must use ECPrivateKeySpec or PKCS8EncodedKeySpec; was \"\n                + keySpec.getClass().getName());\n    }\n\n    @Override\n    protected <T extends KeySpec> T engineGetKeySpec(Key key, Class<T> keySpec)\n            throws InvalidKeySpecException {\n        if (key == null) {\n            throw new InvalidKeySpecException(\"key == null\");\n        }\n\n        if (keySpec == null) {\n            throw new InvalidKeySpecException(\"keySpec == null\");\n        }\n\n        if (!\"EC\".equals(key.getAlgorithm())) {\n            throw new InvalidKeySpecException(\"Key must be an EC key\");\n        }\n\n        if (key instanceof ECPublicKey && ECPublicKeySpec.class.isAssignableFrom(keySpec)) {\n            ECPublicKey ecKey = (ECPublicKey) key;\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new ECPublicKeySpec(ecKey.getW(), ecKey.getParams());\n            return result;\n        } else if (key instanceof PublicKey && ECPublicKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"X.509\".equals(key.getFormat()) || encoded == null) {\n                throw new InvalidKeySpecException(\"Not a valid X.509 encoding\");\n            }\n            ECPublicKey ecKey = (ECPublicKey) engineGeneratePublic(new X509EncodedKeySpec(encoded));\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new ECPublicKeySpec(ecKey.getW(), ecKey.getParams());\n            return result;\n        } else if (key instanceof ECPrivateKey\n                && ECPrivateKeySpec.class.isAssignableFrom(keySpec)) {\n            ECPrivateKey ecKey = (ECPrivateKey) key;\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new ECPrivateKeySpec(ecKey.getS(), ecKey.getParams());\n            return result;\n        } else if (key instanceof PrivateKey && ECPrivateKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"PKCS#8\".equals(key.getFormat()) || encoded == null) {\n                throw new InvalidKeySpecException(\"Not a valid PKCS#8 encoding\");\n            }\n            ECPrivateKey ecKey =\n                    (ECPrivateKey) engineGeneratePrivate(new PKCS8EncodedKeySpec(encoded));\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new ECPrivateKeySpec(ecKey.getS(), ecKey.getParams());\n            return result;\n        } else if (key instanceof PrivateKey\n                && PKCS8EncodedKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"PKCS#8\".equals(key.getFormat())) {\n                throw new InvalidKeySpecException(\"Encoding type must be PKCS#8; was \"\n                        + key.getFormat());\n            } else if (encoded == null) {\n                throw new InvalidKeySpecException(\"Key is not encodable\");\n            }\n            @SuppressWarnings(\"unchecked\") T result = (T) new PKCS8EncodedKeySpec(encoded);\n            return result;\n        } else if (key instanceof PublicKey && X509EncodedKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"X.509\".equals(key.getFormat())) {\n                throw new InvalidKeySpecException(\"Encoding type must be X.509; was \"\n                        + key.getFormat());\n            } else if (encoded == null) {\n                throw new InvalidKeySpecException(\"Key is not encodable\");\n            }\n            @SuppressWarnings(\"unchecked\") T result = (T) new X509EncodedKeySpec(encoded);\n            return result;\n        } else {\n            throw new InvalidKeySpecException(\"Unsupported key type and key spec combination; key=\"\n                    + key.getClass().getName() + \", keySpec=\" + keySpec.getName());\n        }\n    }\n\n    @Override\n    protected Key engineTranslateKey(Key key) throws InvalidKeyException {\n        if (key == null) {\n            throw new InvalidKeyException(\"key == null\");\n        }\n        if ((key instanceof OpenSSLECPublicKey) || (key instanceof OpenSSLECPrivateKey)) {\n            return key;\n        } else if (key instanceof ECPublicKey) {\n            ECPublicKey ecKey = (ECPublicKey) key;\n\n            ECPoint w = ecKey.getW();\n\n            ECParameterSpec params = ecKey.getParams();\n\n            try {\n                return engineGeneratePublic(new ECPublicKeySpec(w, params));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else if (key instanceof ECPrivateKey) {\n            ECPrivateKey ecKey = (ECPrivateKey) key;\n\n            BigInteger s = ecKey.getS();\n\n            ECParameterSpec params = ecKey.getParams();\n\n            try {\n                return engineGeneratePrivate(new ECPrivateKeySpec(s, params));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else if ((key instanceof PrivateKey) && \"PKCS#8\".equals(key.getFormat())) {\n            byte[] encoded = key.getEncoded();\n            if (encoded == null) {\n                throw new InvalidKeyException(\"Key does not support encoding\");\n            }\n            try {\n                return engineGeneratePrivate(new PKCS8EncodedKeySpec(encoded));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else if ((key instanceof PublicKey) && \"X.509\".equals(key.getFormat())) {\n            byte[] encoded = key.getEncoded();\n            if (encoded == null) {\n                throw new InvalidKeyException(\"Key does not support encoding\");\n            }\n            try {\n                return engineGeneratePublic(new X509EncodedKeySpec(encoded));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else {\n            throw new InvalidKeyException(\"Key must be EC public or private key; was \"\n                    + key.getClass().getName());\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLSignature.java\n/*\n * Copyright (C) 2008 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.nio.ByteBuffer;\nimport java.security.AlgorithmParameters;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.InvalidParameterException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.PrivateKey;\nimport java.security.ProviderException;\nimport java.security.PublicKey;\nimport java.security.SignatureException;\nimport java.security.SignatureSpi;\nimport java.security.spec.AlgorithmParameterSpec;\nimport java.security.spec.InvalidParameterSpecException;\nimport java.security.spec.MGF1ParameterSpec;\nimport java.security.spec.PSSParameterSpec;\n\n/**\n * Implements the subset of the JDK Signature interface needed for\n * signature verification using OpenSSL.\n */\n@Internal\npublic class OpenSSLSignature extends SignatureSpi {\n    private enum EngineType {\n        RSA, EC,\n    }\n\n    private NativeRef.EVP_MD_CTX ctx;\n\n    /**\n     * The current OpenSSL key we're operating on.\n     */\n    private OpenSSLKey key;\n\n    /**\n     * Holds the type of the Java algorithm.\n     */\n    private final EngineType engineType;\n\n    /**\n     * Digest algorithm (reference to {@code EVP_MD}).\n     */\n    private final long evpMdRef;\n\n    /**\n     * Holds a dummy buffer for writing single bytes to the digest.\n     */\n    private final byte[] singleByte = new byte[1];\n\n    /**\n     * True when engine is initialized to signing.\n     */\n    private boolean signing;\n\n    /**\n     * Public key algorithm context (reference to {@code EVP_PKEY_CTX}).\n     */\n    private long evpPkeyCtx;\n\n    /**\n     * Creates a new OpenSSLSignature instance for the given algorithm name.\n     *\n     * @param evpMdRef digest algorithm ({@code EVP_MD} reference).\n     */\n    private OpenSSLSignature(long evpMdRef, EngineType engineType) {\n        this.engineType = engineType;\n        this.evpMdRef = evpMdRef;\n    }\n\n    private void resetContext() throws InvalidAlgorithmParameterException {\n        NativeRef.EVP_MD_CTX ctxLocal = new NativeRef.EVP_MD_CTX(NativeCrypto.EVP_MD_CTX_create());\n        if (signing) {\n            evpPkeyCtx = NativeCrypto.EVP_DigestSignInit(ctxLocal, evpMdRef, key.getNativeRef());\n        } else {\n            evpPkeyCtx = NativeCrypto.EVP_DigestVerifyInit(ctxLocal, evpMdRef, key.getNativeRef());\n        }\n        configureEVP_PKEY_CTX(evpPkeyCtx);\n        this.ctx = ctxLocal;\n    }\n\n    /**\n     * Configures the public key algorithm context ({@code EVP_PKEY_CTX}) associated with this\n     * operation.\n     *\n     * <p>The default implementation does nothing.\n     *\n     * @param ctx reference to the context ({@code EVP_PKEY_CTX}).\n     */\n    protected void configureEVP_PKEY_CTX(long ctx) throws InvalidAlgorithmParameterException {}\n\n    @Override\n    protected void engineUpdate(byte input) {\n        singleByte[0] = input;\n        engineUpdate(singleByte, 0, 1);\n    }\n\n    @Override\n    protected void engineUpdate(byte[] input, int offset, int len) {\n        final NativeRef.EVP_MD_CTX ctxLocal = ctx;\n        if (signing) {\n            NativeCrypto.EVP_DigestSignUpdate(ctxLocal, input, offset, len);\n        } else {\n            NativeCrypto.EVP_DigestVerifyUpdate(ctxLocal, input, offset, len);\n        }\n    }\n\n    @Override\n    protected void engineUpdate(ByteBuffer input) {\n        // Optimization: Avoid copying/allocation for direct buffers because their contents are\n        // stored as a contiguous region in memory and thus can be efficiently accessed from native\n        // code.\n\n        if (!input.hasRemaining()) {\n            return;\n        }\n\n        if (!input.isDirect()) {\n            super.engineUpdate(input);\n            return;\n        }\n\n        long baseAddress = NativeCrypto.getDirectBufferAddress(input);\n        if (baseAddress == 0) {\n            // Direct buffer's contents can't be accessed from JNI  -- superclass's implementation\n            // is good enough to handle this.\n            super.engineUpdate(input);\n            return;\n        }\n\n        // Process the contents between Buffer's position and limit (remaining() number of bytes)\n        int position = input.position();\n        if (position < 0) {\n            throw new RuntimeException(\"Negative position\");\n        }\n        long ptr = baseAddress + position;\n        int len = input.remaining();\n        if (len < 0) {\n            throw new RuntimeException(\"Negative remaining amount\");\n        }\n\n        final NativeRef.EVP_MD_CTX ctxLocal = ctx;\n        if (signing) {\n            NativeCrypto.EVP_DigestSignUpdateDirect(ctxLocal, ptr, len);\n        } else {\n            NativeCrypto.EVP_DigestVerifyUpdateDirect(ctxLocal, ptr, len);\n        }\n        input.position(position + len);\n    }\n\n    @Deprecated\n    @Override\n    protected Object engineGetParameter(String param) throws InvalidParameterException {\n        return null;\n    }\n\n    private void checkEngineType(OpenSSLKey pkey) throws InvalidKeyException {\n        final int pkeyType = NativeCrypto.EVP_PKEY_type(pkey.getNativeRef());\n\n        switch (engineType) {\n            case RSA:\n                if (pkeyType != NativeConstants.EVP_PKEY_RSA) {\n                    throw new InvalidKeyException(\"Signature initialized as \" + engineType\n                            + \" (not RSA)\");\n                }\n                break;\n            case EC:\n                if (pkeyType != NativeConstants.EVP_PKEY_EC) {\n                    throw new InvalidKeyException(\"Signature initialized as \" + engineType\n                            + \" (not EC)\");\n                }\n                break;\n            default:\n                throw new InvalidKeyException(\"Key must be of type \" + engineType);\n        }\n    }\n\n    private void initInternal(OpenSSLKey newKey, boolean signing) throws InvalidKeyException {\n        checkEngineType(newKey);\n        key = newKey;\n\n        this.signing = signing;\n        try {\n            resetContext();\n        } catch (InvalidAlgorithmParameterException e) {\n            throw new InvalidKeyException(e);\n        }\n    }\n\n    @Override\n    protected void engineInitSign(PrivateKey privateKey) throws InvalidKeyException {\n        initInternal(OpenSSLKey.fromPrivateKey(privateKey), true);\n    }\n\n    @Override\n    protected void engineInitVerify(PublicKey publicKey) throws InvalidKeyException {\n        initInternal(OpenSSLKey.fromPublicKey(publicKey), false);\n    }\n\n    @Deprecated\n    @Override\n    protected void engineSetParameter(String param, Object value) throws InvalidParameterException {\n    }\n\n    @Override\n    @SuppressWarnings(\"Finally\")\n    protected byte[] engineSign() throws SignatureException {\n        final NativeRef.EVP_MD_CTX ctxLocal = ctx;\n        try {\n            return NativeCrypto.EVP_DigestSignFinal(ctxLocal);\n        } catch (Exception ex) {\n            throw new SignatureException(ex);\n        } finally {\n            /*\n             * Java expects the digest context to be reset completely after sign\n             * calls.\n             */\n            try {\n                resetContext();\n            } catch (InvalidAlgorithmParameterException e) {\n                throw new AssertionError(\"Reset of context failed after it was successful once\");\n            }\n        }\n    }\n\n    @Override\n    @SuppressWarnings(\"Finally\")\n    protected boolean engineVerify(byte[] sigBytes) throws SignatureException {\n        final NativeRef.EVP_MD_CTX ctxLocal = ctx;\n        try {\n            return NativeCrypto.EVP_DigestVerifyFinal(ctxLocal, sigBytes, 0, sigBytes.length);\n        } catch (Exception ex) {\n            throw new SignatureException(ex);\n        } finally {\n            /*\n             * Java expects the digest context to be reset completely after\n             * verify calls.\n             */\n            try {\n                resetContext();\n            } catch (InvalidAlgorithmParameterException e) {\n                throw new AssertionError(\"Reset of context failed after it was successful once\");\n            }\n        }\n    }\n\n    /**\n     * Returns the public key algorithm context ({@code EVP_PKEY_CTX} reference) associated with\n     * this operation or {@code 0} if operation hasn't been initialized.\n     */\n    protected final long getEVP_PKEY_CTX() {\n        return evpPkeyCtx;\n    }\n\n    /**\n     * Base class for {@code RSASSA-PKCS1-v1_5} signatures.\n     */\n    abstract static class RSAPKCS1Padding extends OpenSSLSignature {\n        RSAPKCS1Padding(long evpMdRef) {\n            super(evpMdRef, EngineType.RSA);\n        }\n\n        @Override\n        protected final void configureEVP_PKEY_CTX(long ctx)\n                throws InvalidAlgorithmParameterException {\n            NativeCrypto.EVP_PKEY_CTX_set_rsa_padding(ctx, NativeConstants.RSA_PKCS1_PADDING);\n        }\n    }\n\n    public static final class MD5RSA extends RSAPKCS1Padding {\n        public MD5RSA() {\n            super(EvpMdRef.MD5.EVP_MD);\n        }\n    }\n    public static final class SHA1RSA extends RSAPKCS1Padding {\n        public SHA1RSA() {\n            super(EvpMdRef.SHA1.EVP_MD);\n        }\n    }\n    public static final class SHA224RSA extends RSAPKCS1Padding {\n        public SHA224RSA() {\n            super(EvpMdRef.SHA224.EVP_MD);\n        }\n    }\n    public static final class SHA256RSA extends RSAPKCS1Padding {\n        public SHA256RSA() {\n            super(EvpMdRef.SHA256.EVP_MD);\n        }\n    }\n    public static final class SHA384RSA extends RSAPKCS1Padding {\n        public SHA384RSA() {\n            super(EvpMdRef.SHA384.EVP_MD);\n        }\n    }\n    public static final class SHA512RSA extends RSAPKCS1Padding {\n        public SHA512RSA() {\n            super(EvpMdRef.SHA512.EVP_MD);\n        }\n    }\n\n    public static final class SHA1ECDSA extends OpenSSLSignature {\n        public SHA1ECDSA() {\n            super(EvpMdRef.SHA1.EVP_MD, EngineType.EC);\n        }\n    }\n    public static final class SHA224ECDSA extends OpenSSLSignature {\n        public SHA224ECDSA() {\n            super(EvpMdRef.SHA224.EVP_MD, EngineType.EC);\n        }\n    }\n    public static final class SHA256ECDSA extends OpenSSLSignature {\n        public SHA256ECDSA() {\n            super(EvpMdRef.SHA256.EVP_MD, EngineType.EC);\n        }\n    }\n    public static final class SHA384ECDSA extends OpenSSLSignature {\n        public SHA384ECDSA() {\n            super(EvpMdRef.SHA384.EVP_MD, EngineType.EC);\n        }\n    }\n    public static final class SHA512ECDSA extends OpenSSLSignature {\n        public SHA512ECDSA() {\n            super(EvpMdRef.SHA512.EVP_MD, EngineType.EC);\n        }\n    }\n\n    /**\n     * Base class for {@code RSASSA-PSS} signatures.\n     */\n    abstract static class RSAPSSPadding extends OpenSSLSignature {\n        private static final int TRAILER_FIELD_BC_ID = 1;\n\n        private final String contentDigestAlgorithm;\n\n        private String mgf1DigestAlgorithm;\n        private long mgf1EvpMdRef;\n        private int saltSizeBytes;\n\n        RSAPSSPadding(\n                long contentDigestEvpMdRef, String contentDigestAlgorithm, int saltSizeBytes) {\n            super(contentDigestEvpMdRef, EngineType.RSA);\n            this.contentDigestAlgorithm = contentDigestAlgorithm;\n            this.mgf1DigestAlgorithm = contentDigestAlgorithm;\n            this.mgf1EvpMdRef = contentDigestEvpMdRef;\n            this.saltSizeBytes = saltSizeBytes;\n        }\n\n        @Override\n        protected final void configureEVP_PKEY_CTX(long ctx)\n                throws InvalidAlgorithmParameterException {\n            NativeCrypto.EVP_PKEY_CTX_set_rsa_padding(ctx, NativeConstants.RSA_PKCS1_PSS_PADDING);\n            NativeCrypto.EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, mgf1EvpMdRef);\n            NativeCrypto.EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltSizeBytes);\n        }\n\n        @Override\n        protected final void engineSetParameter(AlgorithmParameterSpec params)\n                throws InvalidAlgorithmParameterException {\n            if (!(params instanceof PSSParameterSpec)) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Unsupported parameter: \" + params + \". Only \"\n                                + PSSParameterSpec.class.getName() + \" supported\");\n            }\n            PSSParameterSpec spec = (PSSParameterSpec) params;\n            String specContentDigest = EvpMdRef\n                    .getJcaDigestAlgorithmStandardName(spec.getDigestAlgorithm());\n            if (specContentDigest == null) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Unsupported content digest algorithm: \" + spec.getDigestAlgorithm());\n            } else if (!contentDigestAlgorithm.equalsIgnoreCase(specContentDigest)) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Changing content digest algorithm not supported\");\n            }\n\n            String specMgfAlgorithm = spec.getMGFAlgorithm();\n            if (!EvpMdRef.MGF1_ALGORITHM_NAME.equalsIgnoreCase(specMgfAlgorithm)\n                    && !EvpMdRef.MGF1_OID.equals(specMgfAlgorithm)) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Unsupported MGF algorithm: \" + specMgfAlgorithm + \". Only \"\n                                + EvpMdRef.MGF1_ALGORITHM_NAME + \" supported\");\n            }\n\n            AlgorithmParameterSpec mgfSpec = spec.getMGFParameters();\n            if (!(mgfSpec instanceof MGF1ParameterSpec)) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Unsupported MGF parameters: \" + mgfSpec + \". Only \"\n                                + MGF1ParameterSpec.class.getName() + \" supported\");\n            }\n            MGF1ParameterSpec specMgf1Spec = (MGF1ParameterSpec) spec.getMGFParameters();\n\n            String specMgf1Digest = EvpMdRef\n                    .getJcaDigestAlgorithmStandardName(specMgf1Spec.getDigestAlgorithm());\n            if (specMgf1Digest == null) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Unsupported MGF1 digest algorithm: \" + specMgf1Spec.getDigestAlgorithm());\n            }\n            long specMgf1EvpMdRef;\n            try {\n                specMgf1EvpMdRef = EvpMdRef\n                        .getEVP_MDByJcaDigestAlgorithmStandardName(specMgf1Digest);\n            } catch (NoSuchAlgorithmException e) {\n                throw new ProviderException(\"Failed to obtain EVP_MD for \" + specMgf1Digest, e);\n            }\n\n            int specSaltSizeBytes = spec.getSaltLength();\n            if (specSaltSizeBytes < 0) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Salt length must be non-negative: \" + specSaltSizeBytes);\n            }\n\n            int specTrailer = spec.getTrailerField();\n            if (specTrailer != TRAILER_FIELD_BC_ID) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Unsupported trailer field: \" + specTrailer + \". Only \"\n                                + TRAILER_FIELD_BC_ID + \" supported\");\n            }\n\n            this.mgf1DigestAlgorithm = specMgf1Digest;\n            this.mgf1EvpMdRef = specMgf1EvpMdRef;\n            this.saltSizeBytes = specSaltSizeBytes;\n\n            long ctx = getEVP_PKEY_CTX();\n            if (ctx != 0) {\n                configureEVP_PKEY_CTX(ctx);\n            }\n        }\n\n        @Override\n        protected final AlgorithmParameters engineGetParameters() {\n            try {\n                AlgorithmParameters result = AlgorithmParameters.getInstance(\"PSS\");\n                result.init(\n                        new PSSParameterSpec(\n                                contentDigestAlgorithm,\n                                EvpMdRef.MGF1_ALGORITHM_NAME,\n                                new MGF1ParameterSpec(mgf1DigestAlgorithm),\n                                saltSizeBytes,\n                                TRAILER_FIELD_BC_ID));\n                return result;\n            } catch (NoSuchAlgorithmException e) {\n                throw new ProviderException(\"Failed to create PSS AlgorithmParameters\", e);\n            } catch (InvalidParameterSpecException e) {\n                throw new ProviderException(\"Failed to create PSS AlgorithmParameters\", e);\n            }\n        }\n    }\n\n    public static final class SHA1RSAPSS extends RSAPSSPadding {\n        public SHA1RSAPSS() {\n            super(EvpMdRef.SHA1.EVP_MD, EvpMdRef.SHA1.JCA_NAME, EvpMdRef.SHA1.SIZE_BYTES);\n        }\n    }\n\n    public static final class SHA224RSAPSS extends RSAPSSPadding {\n        public SHA224RSAPSS() {\n            super(EvpMdRef.SHA224.EVP_MD, EvpMdRef.SHA224.JCA_NAME, EvpMdRef.SHA224.SIZE_BYTES);\n        }\n    }\n\n    public static final class SHA256RSAPSS extends RSAPSSPadding {\n        public SHA256RSAPSS() {\n            super(EvpMdRef.SHA256.EVP_MD, EvpMdRef.SHA256.JCA_NAME, EvpMdRef.SHA256.SIZE_BYTES);\n        }\n    }\n\n    public static final class SHA384RSAPSS extends RSAPSSPadding {\n        public SHA384RSAPSS() {\n            super(EvpMdRef.SHA384.EVP_MD, EvpMdRef.SHA384.JCA_NAME, EvpMdRef.SHA384.SIZE_BYTES);\n        }\n    }\n\n    public static final class SHA512RSAPSS extends RSAPSSPadding {\n        public SHA512RSAPSS() {\n            super(EvpMdRef.SHA512.EVP_MD, EvpMdRef.SHA512.JCA_NAME, EvpMdRef.SHA512.SIZE_BYTES);\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ct/CTLogInfo.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.security.InvalidKeyException;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.PublicKey;\nimport java.security.Signature;\nimport java.security.SignatureException;\nimport java.util.Arrays;\nimport org.conscrypt.Internal;\n\n/**\n * Properties about a Certificate Transparency Log.\n * This object stores information about a CT log, its public key, description and URL.\n * It allows verification of SCTs against the log's public key.\n */\n@Internal\npublic class CTLogInfo {\n    private final byte[] logId;\n    private final PublicKey publicKey;\n    private final String description;\n    private final String url;\n\n    public CTLogInfo(PublicKey publicKey, String description, String url) {\n        try {\n            this.logId = MessageDigest.getInstance(\"SHA-256\")\n                .digest(publicKey.getEncoded());\n        } catch (NoSuchAlgorithmException e) {\n            // SHA-256 is guaranteed to be available\n            throw new RuntimeException(e);\n        }\n\n        this.publicKey = publicKey;\n        this.description = description;\n        this.url = url;\n    }\n\n    /**\n     * Get the log's ID, that is the SHA-256 hash of it's public key\n     */\n    public byte[] getID() {\n        return logId;\n    }\n\n    public PublicKey getPublicKey() {\n        return publicKey;\n    }\n\n    public String getDescription() {\n        return description;\n    }\n\n    public String getUrl() {\n        return url;\n    }\n\n    @Override\n    public boolean equals(Object other) {\n        if (this == other) {\n            return true;\n        }\n        if (!(other instanceof CTLogInfo)) {\n            return false;\n        }\n\n        CTLogInfo that = (CTLogInfo)other;\n        return\n            this.publicKey.equals(that.publicKey) &&\n            this.description.equals(that.description) &&\n            this.url.equals(that.url);\n    }\n\n    @Override\n    public int hashCode() {\n        int hash = 1;\n        hash = hash * 31 + publicKey.hashCode();\n        hash = hash * 31 + description.hashCode();\n        hash = hash * 31 + url.hashCode();\n\n        return hash;\n    }\n\n    /**\n     * Verify the signature of a signed certificate timestamp for the given certificate entry\n     * against the log's public key.\n     *\n     * @return the result of the verification\n     */\n    public VerifiedSCT.Status verifySingleSCT(SignedCertificateTimestamp sct,\n                                              CertificateEntry entry) {\n        if (!Arrays.equals(sct.getLogID(), getID())) {\n            return VerifiedSCT.Status.UNKNOWN_LOG;\n        }\n\n        byte[] toVerify;\n        try {\n            toVerify = sct.encodeTBS(entry);\n        } catch (SerializationException e) {\n            return VerifiedSCT.Status.INVALID_SCT;\n        }\n\n        Signature signature;\n        try {\n            String algorithm = sct.getSignature().getAlgorithm();\n            signature = Signature.getInstance(algorithm);\n        } catch (NoSuchAlgorithmException e) {\n            return VerifiedSCT.Status.INVALID_SCT;\n        }\n\n        try {\n            signature.initVerify(publicKey);\n        } catch (InvalidKeyException e) {\n            return VerifiedSCT.Status.INVALID_SCT;\n        }\n\n        try {\n            signature.update(toVerify);\n            if (!signature.verify(sct.getSignature().getSignature())) {\n                return VerifiedSCT.Status.INVALID_SIGNATURE;\n            }\n            return VerifiedSCT.Status.VALID;\n        } catch (SignatureException e) {\n            // This only happens if the signature is not initialized,\n            // but we call initVerify just before, so it should never do\n            throw new RuntimeException(e);\n        }\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/SerializationException.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport org.conscrypt.Internal;\n\n@Internal\npublic class SerializationException extends Exception {\n    private static final long serialVersionUID = -5317873136664833411L;\n\n    public SerializationException() {\n    }\n\n    public SerializationException(String message) {\n        super(message);\n    }\n\n    public SerializationException(String message, Throwable cause) {\n        super(message, cause);\n    }\n\n    public SerializationException(Throwable cause) {\n        super(cause);\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/metrics/OptionalMethod.java\n/*\n * Copyright (C) 2020 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt.metrics;\n\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\n\nimport org.conscrypt.Internal;\n\n/**\n * Helper class to handle reflexive loading and invocation of methods which may be absent.\n *\n * @hide This class is not part of the Android public SDK API\n */\n@Internal\npublic final class OptionalMethod {\n    private final Method cachedMethod;\n\n    /**\n     * Instantiates a new OptionalMethod.\n     * <p>Does not throw any exceptions if the class or method can't be loaded, or if any parameter\n     * classes are {@code null} and instead behaves as a no-op, always returning {@code null}.\n     *\n     * @param clazz the Class to search for methods on\n     * @param methodName the name of the {@code Method} on {@code clazz}\n     * @param methodParams list of {@code Classes} of the {@code Method's} parameters\n     *\n     * @throws NullPointerException if the method name is {@code null}\n     */\n    public OptionalMethod(Class<?> clazz, String methodName, Class<?>... methodParams) {\n        this.cachedMethod = initializeMethod(clazz, methodName, methodParams);\n    }\n\n    private static Method initializeMethod(\n            Class<?> clazz, String methodName, Class<?>... methodParams) {\n        try {\n            for (Class<?> paramClass : methodParams) {\n                if (paramClass == null) {\n                    return null;\n                }\n            }\n            if (clazz != null) {\n                return clazz.getMethod(checkNotNull(methodName), methodParams);\n            }\n        } catch (NoSuchMethodException ignored) {\n            // Ignored\n        }\n        return null;\n    }\n\n    public Object invokeStatic(Object... args) {\n        // no-op if failed to load method in constructor\n        if (cachedMethod == null) {\n            return null;\n        }\n        try {\n            return cachedMethod.invoke(null, args);\n        } catch (IllegalAccessException ignored) {\n            // Ignored\n        } catch (InvocationTargetException ignored) {\n            // Ignored\n        }\n        return null;\n    }\n\n    public Object invoke(Object target, Object... args) {\n        // no-op if failed to load method in constructor or target is null\n        if (cachedMethod == null || target == null) {\n            return null;\n        }\n        try {\n            return cachedMethod.invoke(target, args);\n        } catch (IllegalAccessException ignored) {\n            // Ignored\n        } catch (InvocationTargetException ignored) {\n            // Ignored\n        }\n        return null;\n    }\n\n    private static <T> T checkNotNull(T reference) {\n        if (reference == null) {\n            throw new NullPointerException();\n        }\n        return reference;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/metrics/Source.java\n/*\n * Copyright (C) 2022 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt.metrics;\n\nimport org.conscrypt.Internal;\n\n/**\n * Data Sources to metric mapping for metrics instrumentation.\n *\n * Must be in sync with frameworks/base/cmds/statsd/src/atoms.proto\n */\n@Internal\npublic enum Source {\n    SOURCE_UNKNOWN,\n    SOURCE_MAINLINE,\n    SOURCE_GMS,\n    SOURCE_UNBUNDLED;\n}\n// Path: common/src/main/java/org/conscrypt/metrics/ReflexiveStatsEvent.java\n/*\n * Copyright (C) 2020 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt.metrics;\n\nimport org.conscrypt.Internal;\n\n/**\n * Reflection wrapper around android.util.StatsEvent.\n */\n@Internal\npublic class ReflexiveStatsEvent {\n    private static final OptionalMethod newBuilder;\n    private static final Class<?> c_statsEvent;\n\n    static {\n        c_statsEvent = initStatsEventClass();\n        newBuilder = new OptionalMethod(c_statsEvent, \"newBuilder\");\n    }\n\n    private static Class<?> initStatsEventClass() {\n        try {\n            return Class.forName(\"android.util.StatsEvent\");\n        } catch (ClassNotFoundException ignored) {\n            return null;\n        }\n    }\n\n    private final Object statsEvent;\n\n    private ReflexiveStatsEvent(Object statsEvent) {\n        this.statsEvent = statsEvent;\n    }\n\n    public Object getStatsEvent() {\n        return statsEvent;\n    }\n\n    public static ReflexiveStatsEvent.Builder newBuilder() {\n        return new ReflexiveStatsEvent.Builder();\n    }\n\n    public static ReflexiveStatsEvent buildEvent(\n            int atomId, boolean success, int protocol, int cipherSuite, int duration, int source) {\n        ReflexiveStatsEvent.Builder builder = ReflexiveStatsEvent.newBuilder();\n        builder.setAtomId(atomId);\n        builder.writeBoolean(success);\n        builder.writeInt(protocol);\n        builder.writeInt(cipherSuite);\n        builder.writeInt(duration);\n        builder.writeInt(source);\n        builder.usePooledBuffer();\n        return builder.build();\n    }\n\n    public static final class Builder {\n        private static final Class<?> c_statsEvent_Builder;\n        private static final OptionalMethod setAtomId;\n        private static final OptionalMethod writeBoolean;\n        private static final OptionalMethod writeInt;\n        private static final OptionalMethod build;\n        private static final OptionalMethod usePooledBuffer;\n\n        static {\n            c_statsEvent_Builder = initStatsEventBuilderClass();\n            setAtomId = new OptionalMethod(c_statsEvent_Builder, \"setAtomId\", int.class);\n            writeBoolean = new OptionalMethod(c_statsEvent_Builder, \"writeBoolean\", boolean.class);\n            writeInt = new OptionalMethod(c_statsEvent_Builder, \"writeInt\", int.class);\n            build = new OptionalMethod(c_statsEvent_Builder, \"build\");\n            usePooledBuffer = new OptionalMethod(c_statsEvent_Builder, \"usePooledBuffer\");\n        }\n\n        private static Class<?> initStatsEventBuilderClass() {\n            try {\n                return Class.forName(\"android.util.StatsEvent$Builder\");\n            } catch (ClassNotFoundException ignored) {\n                return null;\n            }\n        }\n\n        private final Object builder;\n\n        private Builder() {\n            this.builder = newBuilder.invokeStatic();\n        }\n\n        public Builder setAtomId(final int atomId) {\n            setAtomId.invoke(this.builder, atomId);\n            return this;\n        }\n\n        public Builder writeBoolean(final boolean value) {\n            writeBoolean.invoke(this.builder, value);\n            return this;\n        }\n\n        public Builder writeInt(final int value) {\n            writeInt.invoke(this.builder, value);\n            return this;\n        }\n\n        public void usePooledBuffer() {\n            usePooledBuffer.invoke(this.builder);\n        }\n\n        public ReflexiveStatsEvent build() {\n            Object statsEvent = build.invoke(this.builder);\n            return new ReflexiveStatsEvent(statsEvent);\n        }\n    }\n}\n// Path: common/src/main/java/org/conscrypt/metrics/ReflexiveStatsLog.java\n/*\n * Copyright (C) 2020 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt.metrics;\n\nimport org.conscrypt.Internal;\n\n/**\n * Reflection wrapper around android.util.StatsLog.\n */\n@Internal\npublic class ReflexiveStatsLog {\n    private static final Class<?> c_statsLog;\n    private static final Class<?> c_statsEvent;\n    private static final OptionalMethod write;\n\n    static {\n        c_statsLog = initStatsLogClass();\n        c_statsEvent = initStatsEventClass();\n        write = new OptionalMethod(c_statsLog, \"write\", c_statsEvent);\n    }\n\n    private static Class<?> initStatsLogClass() {\n        try {\n            return Class.forName(\"android.util.StatsLog\");\n        } catch (ClassNotFoundException ignored) {\n            return null;\n        }\n    }\n\n    private static Class<?> initStatsEventClass() {\n        try {\n            return Class.forName(\"android.util.StatsEvent\");\n        } catch (ClassNotFoundException ignored) {\n            return null;\n        }\n    }\n\n    private ReflexiveStatsLog() {}\n\n    public static void write(ReflexiveStatsEvent event) {\n        Object statsEvent = event.getStatsEvent();\n        if (statsEvent != null) {\n            write.invokeStatic(statsEvent);\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/metrics/ConscryptStatsLog.java\n/*\n * Copyright (C) 2020 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt.metrics;\n\nimport org.conscrypt.Internal;\n\n/**\n * Reimplement with reflection calls the logging class,\n * generated by frameworks/statsd.\n * <p>\n * In case atom is changed, generate new wrapper with stats-log-api-gen\n * tool as shown below and add corresponding methods to ReflexiveStatsEvent's\n * newEvent() method.\n * <p>\n * $ stats-log-api-gen \\\n *   --java \"common/src/main/java/org/conscrypt/metrics/ConscryptStatsLog.java\" \\\n *   --module conscrypt \\\n *   --javaPackage org.conscrypt.metrics \\\n *   --javaClass ConscryptStatsLog\n **/\n@Internal\npublic final class ConscryptStatsLog {\n    public static final int TLS_HANDSHAKE_REPORTED = 317;\n\n    private ConscryptStatsLog() {}\n\n    public static void write(\n            int atomId, boolean success, int protocol, int cipherSuite, int duration, Source source) {\n        ReflexiveStatsEvent event = ReflexiveStatsEvent.buildEvent(\n                atomId, success, protocol, cipherSuite, duration, source.ordinal());\n\n        ReflexiveStatsLog.write(event);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/metrics/Protocol.java\n/*\n * Copyright (C) 2020 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt.metrics;\n\nimport org.conscrypt.Internal;\n\n/**\n * Protocols to metric mapping for metrics instrumentation.\n *\n * Must be in sync with frameworks/base/cmds/statsd/src/atoms.proto\n */\n@Internal\npublic enum Protocol {\n    UNKNOWN_PROTO(0),\n    SSLv3(1),\n    TLSv1(2),\n    TLSv1_1(3),\n    TLSv1_2(4),\n    TLSv1_3(5),\n    TLS_PROTO_FAILED(0xFFFF),\n    ;\n\n    final byte id;\n\n    public int getId() {\n        return this.id;\n    }\n\n    public static Protocol forName(String name) {\n        switch (name) {\n            case \"SSLv3\":\n                return SSLv3;\n            case \"TLSv1\":\n                return TLSv1;\n            case \"TLSv1.1\":\n                return TLSv1_1;\n            case \"TLSv1.2\":\n                return TLSv1_2;\n            case \"TLSv1.3\":\n                return TLSv1_3;\n            case \"TLS_PROTO_FAILED\":\n                return TLS_PROTO_FAILED;\n            default:\n                return UNKNOWN_PROTO;\n        }\n    }\n\n    private Protocol(int id) {\n        this.id = (byte) id;\n    }\n}\n// Path: common/src/main/java/org/conscrypt/metrics/CipherSuite.java\n/*\n * Copyright (C) 2020 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt.metrics;\n\nimport org.conscrypt.Internal;\n\n/**\n * Cipher suites to metric mapping for metrics instrumentation.\n *\n * Must be in sync with frameworks/base/cmds/statsd/src/atoms.proto\n *\n...\n", "needle_token_start": 2343, "needle_token_end": 2410, "code_context_ntokens": 15944, "output": ["```java\n@Override\nprotected KeyManager[] engineGetKeyManagers() {\n    if (keyStore == null) {\n        throw new IllegalStateException(\"KeyManagerFactory is not initialized\");\n    }\n    return new KeyManager[] { new KeyManagerImpl(keyStore, pwd) };\n}\n```"]}
{"repo": "google/conscrypt", "name": "engineGenerateCertificate", "language": "java", "path": "common/src/main/java/org/conscrypt/OpenSSLX509CertificateFactory.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a certificate object from a provided input stream that contains the certificate data in a recognized format.\n2. **Input**: An input stream that should contain the certificate data, either in PEM or DER format.\n3. **Output**: Returns a certificate object if the parsing is successful. If the parsing fails due to data format issues, it throws an exception indicating the problem.\n4. **Procedure**: The function attempts to parse the certificate data from the input stream using a specific parser. If the parser encounters any issues (e.g., incorrect format, corrupted data), it catches a parsing exception and converts it into a certificate exception, which is then thrown to the caller.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/OpenSSLX509CertPath.java\n/*\n * Copyright (C) 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.PushbackInputStream;\nimport java.security.cert.CertPath;\nimport java.security.cert.Certificate;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateParsingException;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * An implementation of {@link CertPath} based on BoringSSL.\n */\nfinal class OpenSSLX509CertPath extends CertPath {\n    private static final long serialVersionUID = -3249106005255170761L;\n\n    private static final byte[] PKCS7_MARKER = new byte[] {\n            '-', '-', '-', '-', '-', 'B', 'E', 'G', 'I', 'N', ' ', 'P', 'K', 'C', 'S', '7'\n    };\n\n    private static final int PUSHBACK_SIZE = 64;\n\n    /**\n     * Supported encoding types for CerthPath. Used by the various APIs that\n     * encode this into bytes such as {@link #getEncoded()}.\n     */\n    private enum Encoding {\n        PKI_PATH(\"PkiPath\"),\n        PKCS7(\"PKCS7\");\n\n        private final String apiName;\n\n        Encoding(String apiName) {\n            this.apiName = apiName;\n        }\n\n        static Encoding findByApiName(String apiName) throws CertificateEncodingException {\n            for (Encoding element : values()) {\n                if (element.apiName.equals(apiName)) {\n                    return element;\n                }\n            }\n\n            return null;\n        }\n    }\n\n    /** Unmodifiable list of encodings for the API. */\n    private static final List<String> ALL_ENCODINGS = Collections.unmodifiableList(Arrays\n            .asList(new String[] {\n                    Encoding.PKI_PATH.apiName,\n                    Encoding.PKCS7.apiName,\n            }));\n\n    private static final Encoding DEFAULT_ENCODING = Encoding.PKI_PATH;\n\n    private final List<? extends X509Certificate> mCertificates;\n\n    static Iterator<String> getEncodingsIterator() {\n        return ALL_ENCODINGS.iterator();\n    }\n\n    OpenSSLX509CertPath(List<? extends X509Certificate> certificates) {\n        super(\"X.509\");\n\n        mCertificates = certificates;\n    }\n\n    @Override\n    public List<? extends Certificate> getCertificates() {\n        return Collections.unmodifiableList(mCertificates);\n    }\n\n    private byte[] getEncoded(Encoding encoding) throws CertificateEncodingException {\n        final OpenSSLX509Certificate[] certs = new OpenSSLX509Certificate[mCertificates.size()];\n        final long[] certRefs = new long[certs.length];\n...\n// Path: common/src/main/java/org/conscrypt/OpenSSLX509CertificateFactory.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.PushbackInputStream;\nimport java.security.cert.CRL;\nimport java.security.cert.CRLException;\nimport java.security.cert.CertPath;\nimport java.security.cert.Certificate;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateFactorySpi;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\n\n/**\n * An implementation of {@link java.security.cert.CertificateFactory} based on BoringSSL.\n */\n@Internal\npublic class OpenSSLX509CertificateFactory extends CertificateFactorySpi {\n    private static final byte[] PKCS7_MARKER = new byte[] {\n            '-', '-', '-', '-', '-', 'B', 'E', 'G', 'I', 'N', ' ', 'P', 'K', 'C', 'S', '7'\n    };\n\n    private static final int PUSHBACK_SIZE = 64;\n\n    static class ParsingException extends Exception {\n        private static final long serialVersionUID = 8390802697728301325L;\n\n        ParsingException(String message) {\n            super(message);\n        }\n\n        ParsingException(Exception cause) {\n            super(cause);\n        }\n\n        ParsingException(String message, Exception cause) {\n            super(message, cause);\n        }\n    }\n\n    private static boolean isMaybePkcs7(byte[] header) {\n        // The outer tag must be SEQUENCE.\n        if (header.length < 2 || header[0] != 0x30) {\n            return false;\n        }\n\n        // Bytes are signed in Java.\n        int lengthByte = header[1] & 0xff;\n\n        // Skip the length prefix to find the tag of the first child of SEQUENCE. This function is\n        // intentionally lax and does not attempt to parse the length itself. It is only necessary\n        // to return true on PKCS#7 inputs and false on X.509 inputs. Other structures can go either\n        // way.\n        int idx = 2;\n        if (lengthByte <= 0x80) {\n            // Short-form or indefinite length.\n        } else if (lengthByte == 0x81) {\n            idx += 1;\n        } else if (lengthByte == 0x82) {\n            idx += 2;\n        } else if (lengthByte == 0x83) {\n            idx += 3;\n        } else if (lengthByte == 0x84) {\n            idx += 4;\n        } else {\n            // BoringSSL stops at 4-byte lengths. A 5-byte length would require a 4GiB input.\n            return false;\n        }\n\n        // The first element of a PKCS#7 structure is OBJECT IDENTIFIER, which has tag 6. The first\n        // element of an X.509 structure is never OBJECT IDENTIFIER.\n        return idx < header.length && header[idx] == 0x06;\n    }\n\n    /**\n     * The code for X509 Certificates and CRL is pretty much the same. We use\n     * this abstract class to share the code between them. This makes it ugly,\n     * but it's already written in this language anyway.\n     */\n    private static abstract class Parser<T> {\n        T generateItem(InputStream inStream) throws ParsingException {\n            if (inStream == null) {\n                throw new ParsingException(\"inStream == null\");\n            }\n\n            final boolean markable = inStream.markSupported();\n            if (markable) {\n                inStream.mark(PKCS7_MARKER.length);\n            }\n\n            final PushbackInputStream pbis = new PushbackInputStream(inStream, PUSHBACK_SIZE);\n            try {\n                final byte[] buffer = new byte[PKCS7_MARKER.length];\n\n                final int len = pbis.read(buffer);\n                if (len < 0) {\n                    /* No need to reset here. The stream was empty or EOF. */\n                    throw new ParsingException(\"inStream is empty\");\n                }\n                pbis.unread(buffer, 0, len);\n\n                if (buffer[0] == '-') {\n                    return fromX509PemInputStream(pbis);\n                }\n\n                if (isMaybePkcs7(buffer)) {\n                    List<? extends T> certs = fromPkcs7DerInputStream(pbis);\n                    if (certs.size() == 0) {\n                        return null;\n                    }\n                    return certs.get(0);\n                } else {\n                    return fromX509DerInputStream(pbis);\n                }\n            } catch (Exception e) {\n                if (markable) {\n                    try {\n                        inStream.reset();\n                    } catch (IOException ignored) {\n                        // If resetting the stream fails, there's not much we can do\n                    }\n                }\n                throw new ParsingException(e);\n            }\n        }\n\n        Collection<? extends T> generateItems(InputStream inStream)\n                throws ParsingException {\n            if (inStream == null) {\n                throw new ParsingException(\"inStream == null\");\n            }\n\n            final boolean markable = inStream.markSupported();\n            if (markable) {\n                inStream.mark(PUSHBACK_SIZE);\n            }\n\n            /* Attempt to see if this is a PKCS#7 bag. */\n            final PushbackInputStream pbis = new PushbackInputStream(inStream, PUSHBACK_SIZE);\n            try {\n                final byte[] buffer = new byte[PKCS7_MARKER.length];\n\n                final int len = pbis.read(buffer);\n                if (len < 0) {\n                    // No need to reset here. The stream was empty or EOF so we return an empty\n                    // list, making it mutable for consistency with the other code paths.\n                    return new ArrayList<>();\n                }\n                pbis.unread(buffer, 0, len);\n\n                if (len == PKCS7_MARKER.length && Arrays.equals(PKCS7_MARKER, buffer)) {\n                    return fromPkcs7PemInputStream(pbis);\n                }\n\n                if (isMaybePkcs7(buffer)) {\n                    return fromPkcs7DerInputStream(pbis);\n                }\n            } catch (Exception e) {\n                if (markable) {\n                    try {\n                        inStream.reset();\n                    } catch (IOException ignored) {\n                        // If resetting the stream fails, there's not much we can do\n                    }\n                }\n                throw new ParsingException(e);\n            }\n\n            /*\n             * It wasn't, so just try to keep grabbing certificates until we\n             * can't anymore.\n             */\n            final List<T> coll = new ArrayList<T>();\n            T c;\n            do {\n                /*\n                 * If this stream supports marking, try to mark here in case\n                 * there is an error during certificate generation.\n                 */\n                if (markable) {\n                    inStream.mark(PUSHBACK_SIZE);\n                }\n\n                try {\n                    c = generateItem(pbis);\n                    coll.add(c);\n                } catch (ParsingException e) {\n                    /*\n                     * If this stream supports marking, attempt to reset it to\n                     * the mark before the failure.\n                     */\n                    if (markable) {\n                        try {\n                            inStream.reset();\n                        } catch (IOException ignored) {\n                            // If resetting the stream fails, there's not much we can do\n                        }\n                    }\n\n                    c = null;\n                }\n            } while (c != null);\n\n            return coll;\n        }\n\n        protected abstract T fromX509PemInputStream(InputStream pbis) throws ParsingException;\n\n        protected abstract T fromX509DerInputStream(InputStream pbis) throws ParsingException;\n\n        protected abstract List<? extends T> fromPkcs7PemInputStream(InputStream is)\n                throws ParsingException;\n\n        protected abstract List<? extends T> fromPkcs7DerInputStream(InputStream is)\n                throws ParsingException;\n    }\n\n    private Parser<OpenSSLX509Certificate> certificateParser =\n            new Parser<OpenSSLX509Certificate>() {\n                @Override\n                public OpenSSLX509Certificate fromX509PemInputStream(InputStream is)\n                        throws ParsingException {\n                    return OpenSSLX509Certificate.fromX509PemInputStream(is);\n                }\n\n                @Override\n                public OpenSSLX509Certificate fromX509DerInputStream(InputStream is)\n                        throws ParsingException {\n                    return OpenSSLX509Certificate.fromX509DerInputStream(is);\n                }\n\n                @Override\n                public List<? extends OpenSSLX509Certificate>\n                        fromPkcs7PemInputStream(InputStream is) throws ParsingException {\n                    return OpenSSLX509Certificate.fromPkcs7PemInputStream(is);\n                }\n\n                @Override\n                public List<? extends OpenSSLX509Certificate>\n                        fromPkcs7DerInputStream(InputStream is) throws ParsingException {\n                    return OpenSSLX509Certificate.fromPkcs7DerInputStream(is);\n                }\n            };\n\n    private Parser<OpenSSLX509CRL> crlParser =\n            new Parser<OpenSSLX509CRL>() {\n                @Override\n                public OpenSSLX509CRL fromX509PemInputStream(InputStream is)\n                        throws ParsingException {\n                    return OpenSSLX509CRL.fromX509PemInputStream(is);\n                }\n\n                @Override\n                public OpenSSLX509CRL fromX509DerInputStream(InputStream is)\n                        throws ParsingException {\n                    return OpenSSLX509CRL.fromX509DerInputStream(is);\n                }\n\n                @Override\n                public List<? extends OpenSSLX509CRL> fromPkcs7PemInputStream(InputStream is)\n                        throws ParsingException {\n                    return OpenSSLX509CRL.fromPkcs7PemInputStream(is);\n                }\n\n                @Override\n                public List<? extends OpenSSLX509CRL> fromPkcs7DerInputStream(InputStream is)\n                        throws ParsingException {\n                    return OpenSSLX509CRL.fromPkcs7DerInputStream(is);\n                }\n            };\n\n    public OpenSSLX509CertificateFactory() {}\n\n    \n@Override\n    public Certificate engineGenerateCertificate(InputStream inStream) throws CertificateException {\n        try {\n            return certificateParser.generateItem(inStream);\n        } catch (ParsingException e) {\n            throw new CertificateException(e);\n        }\n    }\n\n    @Override\n    public Collection<? extends Certificate> engineGenerateCertificates(\n            InputStream inStream) throws CertificateException {\n        try {\n            return certificateParser.generateItems(inStream);\n        } catch (ParsingException e) {\n            throw new CertificateException(e);\n        }\n    }\n\n    @Override\n    public CRL engineGenerateCRL(InputStream inStream) throws CRLException {\n        try {\n            return crlParser.generateItem(inStream);\n        } catch (ParsingException e) {\n            throw new CRLException(e);\n        }\n    }\n\n    @Override\n    public Collection<? extends CRL> engineGenerateCRLs(InputStream inStream) throws CRLException {\n        if (inStream == null) {\n            return Collections.emptyList();\n        }\n\n        try {\n            return crlParser.generateItems(inStream);\n        } catch (ParsingException e) {\n            throw new CRLException(e);\n        }\n    }\n\n    @Override\n    public Iterator<String> engineGetCertPathEncodings() {\n        return OpenSSLX509CertPath.getEncodingsIterator();\n    }\n\n    @Override\n    public CertPath engineGenerateCertPath(InputStream inStream) throws CertificateException {\n        return OpenSSLX509CertPath.fromEncoding(inStream);\n    }\n\n    @Override\n    public CertPath engineGenerateCertPath(InputStream inStream, String encoding)\n            throws CertificateException {\n        return OpenSSLX509CertPath.fromEncoding(inStream, encoding);\n    }\n\n    @Override\n    public CertPath engineGenerateCertPath(List<? extends Certificate> certificates)\n            throws CertificateException {\n        final List<X509Certificate> filtered = new ArrayList<X509Certificate>(certificates.size());\n        for (int i = 0; i < certificates.size(); i++) {\n            final Certificate c = certificates.get(i);\n\n            if (!(c instanceof X509Certificate)) {\n                throw new CertificateException(\"Certificate not X.509 type at index \" + i);\n            }\n\n            filtered.add((X509Certificate) c);\n        }\n\n        return new OpenSSLX509CertPath(filtered);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/EmptyArray.java\n/*\n * Copyright (C) 2010 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n// Copied from libcore.util.EmptyArray\n\npackage org.conscrypt;\n\nfinal class EmptyArray {\n    private EmptyArray() {}\n\n    static final boolean[] BOOLEAN = new boolean[0];\n    static final byte[] BYTE = new byte[0];\n    static final char[] CHAR = new char[0];\n    static final double[] DOUBLE = new double[0];\n    static final int[] INT = new int[0];\n\n    static final Class<?>[] CLASS = new Class<?>[ 0 ];\n    static final Object[] OBJECT = new Object[0];\n    static final String[] STRING = new String[0];\n    static final Throwable[] THROWABLE = new Throwable[0];\n    static final StackTraceElement[] STACK_TRACE_ELEMENT = new StackTraceElement[0];\n}\n\n// Path: common/src/main/java/org/conscrypt/SSLUtils.java\n/*\n * Copyright (C) 2016 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*\n * Copyright 2016 The Netty Project\n *\n * The Netty Project licenses this file to you under the Apache License,\n * version 2.0 (the \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at:\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n * License for the specific language governing permissions and limitations\n * under the License.\n */\n\npackage org.conscrypt;\n\nimport static java.lang.Math.min;\nimport static java.nio.charset.StandardCharsets.US_ASCII;\nimport static org.conscrypt.NativeConstants.SSL3_RT_ALERT;\nimport static org.conscrypt.NativeConstants.SSL3_RT_APPLICATION_DATA;\nimport static org.conscrypt.NativeConstants.SSL3_RT_CHANGE_CIPHER_SPEC;\nimport static org.conscrypt.NativeConstants.SSL3_RT_HANDSHAKE;\nimport static org.conscrypt.NativeConstants.SSL3_RT_HEADER_LENGTH;\nimport static org.conscrypt.NativeConstants.SSL3_RT_MAX_PACKET_SIZE;\n\nimport java.io.ByteArrayInputStream;\nimport java.nio.ByteBuffer;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateFactory;\nimport java.security.cert.X509Certificate;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.LinkedHashSet;\nimport java.util.Set;\nimport javax.net.ssl.SSLException;\nimport javax.net.ssl.SSLHandshakeException;\nimport javax.net.ssl.SSLPeerUnverifiedException;\n\n/**\n * Utility methods for SSL packet processing. Copied from the Netty project.\n * <p>\n * This is a public class to allow testing to occur on Android via CTS.\n */\nfinal class SSLUtils {\n    static final boolean USE_ENGINE_SOCKET_BY_DEFAULT = Boolean.parseBoolean(\n            System.getProperty(\"org.conscrypt.useEngineSocketByDefault\", \"true\"));\n    private static final int MAX_PROTOCOL_LENGTH = 255;\n\n    // TODO(nathanmittler): Should these be in NativeConstants?\n    enum SessionType {\n        /**\n         * Identifies OpenSSL sessions.\n         */\n        OPEN_SSL(1),\n\n        /**\n         * Identifies OpenSSL sessions with OCSP stapled data.\n         */\n        OPEN_SSL_WITH_OCSP(2),\n\n        /**\n         * Identifies OpenSSL sessions with TLS SCT data.\n         */\n        OPEN_SSL_WITH_TLS_SCT(3);\n\n        SessionType(int value) {\n            this.value = value;\n        }\n\n        static boolean isSupportedType(int type) {\n            return type == OPEN_SSL.value || type == OPEN_SSL_WITH_OCSP.value\n                    || type == OPEN_SSL_WITH_TLS_SCT.value;\n        }\n\n        final int value;\n    }\n\n    /**\n     * States for SSL engines.\n     */\n    static final class EngineStates {\n        private EngineStates() {}\n\n        /**\n         * The engine is constructed, but the initial handshake hasn't been started\n         */\n        static final int STATE_NEW = 0;\n\n        /**\n         * The client/server mode of the engine has been set.\n         */\n        static final int STATE_MODE_SET = 1;\n\n        /**\n         * The handshake has been started\n         */\n        static final int STATE_HANDSHAKE_STARTED = 2;\n\n        /**\n         * Listeners of the handshake have been notified of completion but the handshake call\n         * hasn't returned.\n         */\n        static final int STATE_HANDSHAKE_COMPLETED = 3;\n\n        /**\n         * The handshake call returned but the listeners have not yet been notified. This is expected\n         * behaviour in cut-through mode, where SSL_do_handshake returns before the handshake is\n         * complete. We can now start writing data to the socket.\n         */\n        static final int STATE_READY_HANDSHAKE_CUT_THROUGH = 4;\n\n        /**\n         * The handshake call has returned and the listeners have been notified. Ready to begin\n         * writing data.\n         */\n        static final int STATE_READY = 5;\n\n        /**\n         * The inbound direction of the engine has been closed.\n         */\n        static final int STATE_CLOSED_INBOUND = 6;\n\n        /**\n         * The outbound direction of the engine has been closed.\n         */\n        static final int STATE_CLOSED_OUTBOUND = 7;\n\n        /**\n         * The engine has been closed.\n         */\n        static final int STATE_CLOSED = 8;\n    }\n\n    /**\n     * This is the maximum overhead when encrypting plaintext as defined by\n     * <a href=\"https://www.ietf.org/rfc/rfc5246.txt\">rfc5264</a>,\n     * <a href=\"https://www.ietf.org/rfc/rfc5289.txt\">rfc5289</a>, and the BoringSSL\n     * implementation itself.\n     * <p>\n     * Please note that we use a padding of 16 here as BoringSSL uses PKCS#5 which uses 16 bytes\n     * while the spec itself allow up to 255 bytes. 16 bytes is the max for PKCS#5 (which handles it\n     * the same way as PKCS#7) as we use a block size of 16. See <a\n     * href=\"https://tools.ietf.org/html/rfc5652#section-6.3\">rfc5652#section-6.3</a>.\n     * <p>\n     * 16 (IV) + 48 (MAC) + 1 (Padding_length field) + 15 (Padding)\n     * + 1 (ContentType in TLSCiphertext) + 2 (ProtocolVersion) + 2 (Length)\n     * + 1 (ContentType in TLSInnerPlaintext)\n     */\n    private static final int MAX_ENCRYPTION_OVERHEAD_LENGTH = 15 + 48 + 1 + 16 + 1 + 2 + 2 + 1;\n\n    private static final int MAX_ENCRYPTION_OVERHEAD_DIFF =\n            Integer.MAX_VALUE - MAX_ENCRYPTION_OVERHEAD_LENGTH;\n\n    /** Key type: RSA certificate. */\n    private static final String KEY_TYPE_RSA = \"RSA\";\n\n    /** Key type: Elliptic Curve certificate. */\n    private static final String KEY_TYPE_EC = \"EC\";\n\n    static X509Certificate[] decodeX509CertificateChain(byte[][] certChain)\n            throws java.security.cert.CertificateException {\n        CertificateFactory certificateFactory = getCertificateFactory();\n        int numCerts = certChain.length;\n        X509Certificate[] decodedCerts = new X509Certificate[numCerts];\n        for (int i = 0; i < numCerts; i++) {\n            decodedCerts[i] = decodeX509Certificate(certificateFactory, certChain[i]);\n        }\n        return decodedCerts;\n    }\n\n    private static CertificateFactory getCertificateFactory() {\n        try {\n            return CertificateFactory.getInstance(\"X.509\");\n        } catch (java.security.cert.CertificateException e) {\n            return null;\n        }\n    }\n\n    private static X509Certificate decodeX509Certificate(CertificateFactory certificateFactory,\n            byte[] bytes) throws java.security.cert.CertificateException {\n        if (certificateFactory != null) {\n            return (X509Certificate) certificateFactory.generateCertificate(\n                    new ByteArrayInputStream(bytes));\n        }\n        return OpenSSLX509Certificate.fromX509Der(bytes);\n    }\n\n    /**\n     * Returns key type constant suitable for calling X509KeyManager.chooseServerAlias or\n     * X509ExtendedKeyManager.chooseEngineServerAlias. Returns {@code null} for key exchanges that\n     * do not use X.509 for server authentication.\n     */\n    static String getServerX509KeyType(long sslCipherNative) {\n        String kx_name = NativeCrypto.SSL_CIPHER_get_kx_name(sslCipherNative);\n        if (kx_name.equals(\"RSA\") || kx_name.equals(\"DHE_RSA\") || kx_name.equals(\"ECDHE_RSA\")) {\n            return KEY_TYPE_RSA;\n        } else if (kx_name.equals(\"ECDHE_ECDSA\")) {\n            return KEY_TYPE_EC;\n        } else {\n            return null;\n        }\n    }\n\n    /**\n     * Similar to getServerKeyType, but returns value given TLS\n     * ClientCertificateType byte values from a CertificateRequest\n     * message for use with X509KeyManager.chooseClientAlias or\n     * X509ExtendedKeyManager.chooseEngineClientAlias.\n     * <p>\n     * Visible for testing.\n     */\n    static String getClientKeyType(byte clientCertificateType) {\n        // See also\n        // https://www.ietf.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-2\n        switch (clientCertificateType) {\n            case NativeConstants.TLS_CT_RSA_SIGN:\n                return KEY_TYPE_RSA; // RFC rsa_sign\n            case NativeConstants.TLS_CT_ECDSA_SIGN:\n                return KEY_TYPE_EC; // RFC ecdsa_sign\n            default:\n                return null;\n        }\n    }\n\n    static String getClientKeyTypeFromSignatureAlg(int signatureAlg) {\n        // See also\n        // https://www.ietf.org/assignments/tls-parameters/tls-parameters.xml#tls-signaturescheme\n        switch (NativeCrypto.SSL_get_signature_algorithm_key_type(signatureAlg)) {\n            case NativeConstants.EVP_PKEY_RSA:\n                return KEY_TYPE_RSA;\n            case NativeConstants.EVP_PKEY_EC:\n                return KEY_TYPE_EC;\n            default:\n                return null;\n        }\n    }\n\n    /**\n     * Gets the supported key types for client certificates based on the\n     * {@code ClientCertificateType} values provided by the server.\n     *\n     * @param clientCertificateTypes\n     *         {@code ClientCertificateType} values provided by the server.\n     *         See https://www.ietf.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-2.\n     * @param signatureAlgs\n     *         {@code SignatureScheme} values provided by the server.\n     *         See https://www.ietf.org/assignments/tls-parameters/tls-parameters.xml#tls-signaturescheme\n     * @return supported key types that can be used in {@code X509KeyManager.chooseClientAlias} and\n     * {@code X509ExtendedKeyManager.chooseEngineClientAlias}.  If the inputs imply a preference\n     * order, the returned set will have an iteration order that respects that preference order,\n     * otherwise it will be in an arbitrary order.\n     *\n     * Visible for testing.\n     */\n    static Set<String> getSupportedClientKeyTypes(byte[] clientCertificateTypes,\n            int[] signatureAlgs) {\n        Set<String> fromClientCerts = new HashSet<>(clientCertificateTypes.length);\n        for (byte keyTypeCode : clientCertificateTypes) {\n            String keyType = SSLUtils.getClientKeyType(keyTypeCode);\n            if (keyType == null) {\n                // Unsupported client key type -- ignore\n                continue;\n            }\n            fromClientCerts.add(keyType);\n        }\n        // Signature algorithms are listed in preference order\n        Set<String> fromSigAlgs = new LinkedHashSet<>(signatureAlgs.length);\n        for (int signatureAlg : signatureAlgs) {\n            String keyType = SSLUtils.getClientKeyTypeFromSignatureAlg(signatureAlg);\n            if (keyType == null) {\n                // Unsupported client key type -- ignore\n                continue;\n            }\n            fromSigAlgs.add(keyType);\n        }\n        // If both are specified, the key needs to meet both sets of requirements.  Otherwise,\n        // just meet the set of requirements that were specified.  See RFC 5246, section 7.4.4.\n        // (In TLS 1.3, certificate_types is no longer used and is never present.)\n        if (clientCertificateTypes.length > 0 && signatureAlgs.length > 0) {\n            fromSigAlgs.retainAll(fromClientCerts);\n            return fromSigAlgs;\n        } else if (signatureAlgs.length > 0) {\n            return fromSigAlgs;\n        } else {\n            return fromClientCerts;\n        }\n    }\n\n    static byte[][] encodeSubjectX509Principals(X509Certificate[] certificates)\n            throws CertificateEncodingException {\n        byte[][] principalBytes = new byte[certificates.length][];\n        for (int i = 0; i < certificates.length; i++) {\n            principalBytes[i] = certificates[i].getSubjectX500Principal().getEncoded();\n        }\n        return principalBytes;\n    }\n\n    /**\n     * Converts the peer certificates into a cert chain.\n     */\n    @SuppressWarnings(\"deprecation\") // Used in public Conscrypt APIs\n    static javax.security.cert.X509Certificate[] toCertificateChain(X509Certificate[] certificates)\n            throws SSLPeerUnverifiedException {\n        try {\n            javax.security.cert.X509Certificate[] chain =\n                    new javax.security.cert.X509Certificate[certificates.length];\n\n            for (int i = 0; i < certificates.length; i++) {\n                byte[] encoded = certificates[i].getEncoded();\n                chain[i] = javax.security.cert.X509Certificate.getInstance(encoded);\n            }\n            return chain;\n        } catch (CertificateEncodingException | javax.security.cert.CertificateException e) {\n            SSLPeerUnverifiedException exception = new SSLPeerUnverifiedException(e.getMessage());\n            exception.initCause(e);\n            throw exception;\n        }\n    }\n\n    /**\n     * Calculates the minimum bytes required in the encrypted output buffer for the given number of\n     * plaintext source bytes.\n     */\n    static int calculateOutNetBufSize(int pendingBytes) {\n        return min(SSL3_RT_MAX_PACKET_SIZE,\n                MAX_ENCRYPTION_OVERHEAD_LENGTH + min(MAX_ENCRYPTION_OVERHEAD_DIFF, pendingBytes));\n    }\n\n    /**\n     * Wraps the given exception if it's not already a {@link SSLHandshakeException}.\n     */\n    static SSLHandshakeException toSSLHandshakeException(Throwable e) {\n        if (e instanceof SSLHandshakeException) {\n            return (SSLHandshakeException) e;\n        }\n\n        return (SSLHandshakeException) new SSLHandshakeException(e.getMessage()).initCause(e);\n    }\n\n    /**\n     * Wraps the given exception if it's not already a {@link SSLException}.\n     */\n    static SSLException toSSLException(Throwable e) {\n        if (e instanceof SSLException) {\n            return (SSLException) e;\n        }\n        return new SSLException(e);\n    }\n\n    static String toProtocolString(byte[] bytes) {\n        if (bytes == null) {\n            return null;\n        }\n        return new String(bytes, US_ASCII);\n    }\n\n    static byte[] toProtocolBytes(String protocol) {\n        if (protocol == null) {\n            return null;\n        }\n        return protocol.getBytes(US_ASCII);\n    }\n\n    /**\n     * Decodes the given list of protocols into {@link String}s.\n     * @param protocols the encoded protocol list\n     * @return the decoded protocols or {@link EmptyArray#BYTE} if {@code protocols} is\n     * empty.\n     * @throws NullPointerException if protocols is {@code null}.\n     */\n    static String[] decodeProtocols(byte[] protocols) {\n        if (protocols.length == 0) {\n            return EmptyArray.STRING;\n        }\n\n        int numProtocols = 0;\n        for (int i = 0; i < protocols.length;) {\n            int protocolLength = protocols[i];\n            if (protocolLength < 0 || protocolLength > protocols.length - i) {\n                throw new IllegalArgumentException(\n                    \"Protocol has invalid length (\" + protocolLength + \" at position \" + i\n                        + \"): \" + (protocols.length < 50\n                        ? Arrays.toString(protocols) : protocols.length + \" byte array\"));\n            }\n\n            numProtocols++;\n            i += 1 + protocolLength;\n        }\n\n        String[] decoded = new String[numProtocols];\n        for (int i = 0, d = 0; i < protocols.length;) {\n            int protocolLength = protocols[i];\n            decoded[d++] = protocolLength > 0\n                    ? new String(protocols, i + 1, protocolLength, US_ASCII)\n                    : \"\";\n            i += 1 + protocolLength;\n        }\n\n        return decoded;\n    }\n\n    /**\n     * Encodes a list of protocols into the wire-format (length-prefixed 8-bit strings).\n     * Requires that all strings be encoded with US-ASCII.\n     *\n     * @param protocols the list of protocols to be encoded\n     * @return the encoded form of the protocol list.\n     * @throws IllegalArgumentException if protocols is {@code null}, or if any element is\n     * {@code null} or an empty string.\n     */\n    static byte[] encodeProtocols(String[] protocols) {\n        if (protocols == null) {\n            throw new IllegalArgumentException(\"protocols array must be non-null\");\n        }\n\n        if (protocols.length == 0) {\n            return EmptyArray.BYTE;\n        }\n\n        // Calculate the encoded length.\n        int length = 0;\n        for (int i = 0; i < protocols.length; ++i) {\n            String protocol = protocols[i];\n            if (protocol == null) {\n                throw new IllegalArgumentException(\"protocol[\" + i + \"] is null\");\n            }\n            int protocolLength = protocols[i].length();\n\n            // Verify that the length is valid here, so that we don't attempt to allocate an array\n            // below if the threshold is violated.\n            if (protocolLength == 0 || protocolLength > MAX_PROTOCOL_LENGTH) {\n                throw new IllegalArgumentException(\n                    \"protocol[\" + i + \"] has invalid length: \" + protocolLength);\n            }\n\n            // Include a 1-byte prefix for each protocol.\n            length += 1 + protocolLength;\n        }\n\n        byte[] data = new byte[length];\n        for (int dataIndex = 0, i = 0; i < protocols.length; ++i) {\n            String protocol = protocols[i];\n            int protocolLength = protocol.length();\n\n            // Add the length prefix.\n            data[dataIndex++] = (byte) protocolLength;\n            for (int ci = 0; ci < protocolLength; ++ci) {\n                char c = protocol.charAt(ci);\n                if (c > Byte.MAX_VALUE) {\n                    // Enforce US-ASCII\n                    throw new IllegalArgumentException(\"Protocol contains invalid character: \"\n                        + c + \"(protocol=\" + protocol + \")\");\n                }\n                data[dataIndex++] = (byte) c;\n            }\n        }\n        return data;\n    }\n\n    /**\n     * Return how many bytes can be read out of the encrypted data. Be aware that this method will\n     * not increase the readerIndex of the given {@link ByteBuffer}.\n     *\n     * @param buffers The {@link ByteBuffer}s to read from. Be aware that they must have at least\n     * {@link org.conscrypt.NativeConstants#SSL3_RT_HEADER_LENGTH} bytes to read, otherwise it will\n     * throw an {@link IllegalArgumentException}.\n     * @return length The length of the encrypted packet that is included in the buffer. This will\n     * return {@code -1} if the given {@link ByteBuffer} is not encrypted at all.\n     * @throws IllegalArgumentException Is thrown if the given {@link ByteBuffer} has not at least\n     * {@link org.conscrypt.NativeConstants#SSL3_RT_HEADER_LENGTH} bytes to read.\n     */\n    static int getEncryptedPacketLength(ByteBuffer[] buffers, int offset) {\n        ByteBuffer buffer = buffers[offset];\n\n        // Check if everything we need is in one ByteBuffer. If so we can make use of the fast-path.\n        if (buffer.remaining() >= SSL3_RT_HEADER_LENGTH) {\n            return getEncryptedPacketLength(buffer);\n        }\n\n        // We need to copy 5 bytes into a temporary buffer so we can parse out the packet length\n        // easily.\n        ByteBuffer tmp = ByteBuffer.allocate(SSL3_RT_HEADER_LENGTH);\n        do {\n            buffer = buffers[offset++];\n            int pos = buffer.position();\n            int limit = buffer.limit();\n            if (buffer.remaining() > tmp.remaining()) {\n                buffer.limit(pos + tmp.remaining());\n            }\n            try {\n                tmp.put(buffer);\n            } finally {\n                // Restore the original indices.\n                buffer.limit(limit);\n                buffer.position(pos);\n            }\n        } while (tmp.hasRemaining());\n\n        // Done, flip the buffer so we can read from it.\n        tmp.flip();\n        return getEncryptedPacketLength(tmp);\n    }\n\n    private static int getEncryptedPacketLength(ByteBuffer buffer) {\n        int pos = buffer.position();\n        // SSLv3 or TLS - Check ContentType\n        switch (unsignedByte(buffer.get(pos))) {\n            case SSL3_RT_CHANGE_CIPHER_SPEC:\n            case SSL3_RT_ALERT:\n            case SSL3_RT_HANDSHAKE:\n            case SSL3_RT_APPLICATION_DATA:\n                break;\n            default:\n                // SSLv2 or bad data\n                return -1;\n        }\n\n        // SSLv3 or TLS - Check ProtocolVersion\n        int majorVersion = unsignedByte(buffer.get(pos + 1));\n        if (majorVersion != 3) {\n            // Neither SSLv3 or TLSv1 (i.e. SSLv2 or bad data)\n            return -1;\n        }\n\n        // SSLv3 or TLS\n        int packetLength = unsignedShort(buffer.getShort(pos + 3)) + SSL3_RT_HEADER_LENGTH;\n        if (packetLength <= SSL3_RT_HEADER_LENGTH) {\n            // Neither SSLv3 or TLSv1 (i.e. SSLv2 or bad data)\n            return -1;\n        }\n        return packetLength;\n    }\n\n    private static short unsignedByte(byte b) {\n        return (short) (b & 0xFF);\n    }\n\n    private static int unsignedShort(short s) {\n        return s & 0xFFFF;\n    }\n\n    static String[] concat(String[]... arrays) {\n        int resultLength = 0;\n        for (String[] array : arrays) {\n            resultLength += array.length;\n        }\n        String[] result = new String[resultLength];\n        int resultOffset = 0;\n        for (String[] array : arrays) {\n            System.arraycopy(array, 0, result, resultOffset, array.length);\n            resultOffset += array.length;\n        }\n        return result;\n    }\n\n    private SSLUtils() {}\n}\n\n// Path: common/src/main/java/org/conscrypt/NativeCrypto.java\n/*\n * Copyright (C) 2008 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.FileDescriptor;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.net.SocketTimeoutException;\nimport java.nio.Buffer;\nimport java.nio.ByteBuffer;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.PrivateKey;\nimport java.security.SignatureException;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateParsingException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.crypto.ShortBufferException;\nimport javax.net.ssl.SSLException;\nimport javax.security.auth.x500.X500Principal;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * Provides the Java side of our JNI glue for OpenSSL.\n * <p>\n * Note: Many methods in this class take a reference to a Java object that holds a\n * native pointer in the form of a long in addition to the long itself and don't use\n * the Java object in the native implementation.  This is to prevent the Java object\n * from becoming eligible for GC while the native method is executing.  See\n * <a href=\"https://github.com/google/error-prone/blob/master/docs/bugpattern/UnsafeFinalization.md\">this</a>\n * for more details.\n */\n@Internal\npublic final class NativeCrypto {\n    // --- OpenSSL library initialization --------------------------------------\n    private static final UnsatisfiedLinkError loadError;\n    static {\n        UnsatisfiedLinkError error = null;\n        try {\n            NativeCryptoJni.init();\n            clinit();\n        } catch (UnsatisfiedLinkError t) {\n            // Don't rethrow the error, so that we can later on interrogate the\n            // value of loadError.\n            error = t;\n        }\n        loadError = error;\n    }\n\n    private native static void clinit();\n\n    /**\n     * Checks to see whether or not the native library was successfully loaded. If not, throws\n     * the {@link UnsatisfiedLinkError} that was encountered while attempting to load the library.\n     */\n    static void checkAvailability() {\n        if (loadError != null) {\n            throw loadError;\n        }\n    }\n\n    // --- DSA/RSA public/private key handling functions -----------------------\n\n    static native long EVP_PKEY_new_RSA(byte[] n, byte[] e, byte[] d, byte[] p, byte[] q,\n            byte[] dmp1, byte[] dmq1, byte[] iqmp);\n\n    static native int EVP_PKEY_type(NativeRef.EVP_PKEY pkey);\n\n    static native String EVP_PKEY_print_public(NativeRef.EVP_PKEY pkeyRef);\n\n    static native String EVP_PKEY_print_params(NativeRef.EVP_PKEY pkeyRef);\n\n    static native void EVP_PKEY_free(long pkey);\n\n    static native int EVP_PKEY_cmp(NativeRef.EVP_PKEY pkey1, NativeRef.EVP_PKEY pkey2);\n\n    static native byte[] EVP_marshal_private_key(NativeRef.EVP_PKEY pkey);\n\n    static native long EVP_parse_private_key(byte[] data) throws ParsingException;\n\n    static native byte[] EVP_marshal_public_key(NativeRef.EVP_PKEY pkey);\n\n    static native byte[] EVP_raw_X25519_private_key(byte[] data)\n            throws ParsingException, InvalidKeyException;\n\n    static native long EVP_parse_public_key(byte[] data) throws ParsingException;\n\n    static native long PEM_read_bio_PUBKEY(long bioCtx);\n\n    static native long PEM_read_bio_PrivateKey(long bioCtx);\n\n    static native long getRSAPrivateKeyWrapper(PrivateKey key, byte[] modulus);\n\n    static native long getECPrivateKeyWrapper(PrivateKey key, NativeRef.EC_GROUP ecGroupRef);\n\n    static native long RSA_generate_key_ex(int modulusBits, byte[] publicExponent);\n\n    static native int RSA_size(NativeRef.EVP_PKEY pkey);\n\n    static native int RSA_private_encrypt(\n            int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey, int padding);\n\n    static native int RSA_public_decrypt(int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey,\n            int padding) throws BadPaddingException, SignatureException;\n\n    static native int RSA_public_encrypt(\n            int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey, int padding);\n\n    static native int RSA_private_decrypt(int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey,\n            int padding) throws BadPaddingException, SignatureException;\n\n    /**\n     * @return array of {n, e}\n     */\n    static native byte[][] get_RSA_public_params(NativeRef.EVP_PKEY rsa);\n\n    /**\n     * @return array of {n, e, d, p, q, dmp1, dmq1, iqmp}\n     */\n    static native byte[][] get_RSA_private_params(NativeRef.EVP_PKEY rsa);\n\n    // --- ChaCha20 -----------------------\n\n    /**\n     * Returns the encrypted or decrypted version of the data.\n     */\n    static native void chacha20_encrypt_decrypt(byte[] in, int inOffset, byte[] out, int outOffset,\n            int length, byte[] key, byte[] nonce, int blockCounter);\n\n    // --- EC functions --------------------------\n\n    static native long EVP_PKEY_new_EC_KEY(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pubkeyRef, byte[] privkey);\n\n    static native long EC_GROUP_new_by_curve_name(String curveName);\n\n    static native long EC_GROUP_new_arbitrary(\n            byte[] p, byte[] a, byte[] b, byte[] x, byte[] y, byte[] order, int cofactor);\n\n    static native String EC_GROUP_get_curve_name(NativeRef.EC_GROUP groupRef);\n\n    static native byte[][] EC_GROUP_get_curve(NativeRef.EC_GROUP groupRef);\n\n    static native void EC_GROUP_clear_free(long groupRef);\n\n    static native long EC_GROUP_get_generator(NativeRef.EC_GROUP groupRef);\n\n    static native byte[] EC_GROUP_get_order(NativeRef.EC_GROUP groupRef);\n\n    static native int EC_GROUP_get_degree(NativeRef.EC_GROUP groupRef);\n\n    static native byte[] EC_GROUP_get_cofactor(NativeRef.EC_GROUP groupRef);\n\n    static native long EC_POINT_new(NativeRef.EC_GROUP groupRef);\n\n    static native void EC_POINT_clear_free(long pointRef);\n\n    static native byte[][] EC_POINT_get_affine_coordinates(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pointRef);\n\n    static native void EC_POINT_set_affine_coordinates(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pointRef, byte[] x, byte[] y);\n\n    static native long EC_KEY_generate_key(NativeRef.EC_GROUP groupRef);\n\n    static native long EC_KEY_get1_group(NativeRef.EVP_PKEY pkeyRef);\n\n    static native byte[] EC_KEY_get_private_key(NativeRef.EVP_PKEY keyRef);\n\n    static native long EC_KEY_get_public_key(NativeRef.EVP_PKEY keyRef);\n\n    static native byte[] EC_KEY_marshal_curve_name(NativeRef.EC_GROUP groupRef) throws IOException;\n\n    static native long EC_KEY_parse_curve_name(byte[] encoded) throws IOException;\n\n    static native int ECDH_compute_key(byte[] out, int outOffset, NativeRef.EVP_PKEY publicKeyRef,\n            NativeRef.EVP_PKEY privateKeyRef) throws InvalidKeyException, IndexOutOfBoundsException;\n\n    static native int ECDSA_size(NativeRef.EVP_PKEY pkey);\n\n    static native int ECDSA_sign(byte[] data, byte[] sig, NativeRef.EVP_PKEY pkey);\n\n    static native int ECDSA_verify(byte[] data, byte[] sig, NativeRef.EVP_PKEY pkey);\n\n    // --- Curve25519 --------------\n\n    static native boolean X25519(byte[] out, byte[] privateKey, byte[] publicKey) throws InvalidKeyException;\n\n    static native void X25519_keypair(byte[] outPublicKey, byte[] outPrivateKey);\n\n    // --- Message digest functions --------------\n\n    // These return const references\n    static native long EVP_get_digestbyname(String name);\n\n    static native int EVP_MD_size(long evp_md_const);\n\n    // --- Message digest context functions --------------\n\n    static native long EVP_MD_CTX_create();\n\n    static native void EVP_MD_CTX_cleanup(NativeRef.EVP_MD_CTX ctx);\n\n    static native void EVP_MD_CTX_destroy(long ctx);\n\n    static native int EVP_MD_CTX_copy_ex(\n            NativeRef.EVP_MD_CTX dst_ctx, NativeRef.EVP_MD_CTX src_ctx);\n\n    // --- Digest handling functions -------------------------------------------\n\n    static native int EVP_DigestInit_ex(NativeRef.EVP_MD_CTX ctx, long evp_md);\n\n    static native void EVP_DigestUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native int EVP_DigestFinal_ex(NativeRef.EVP_MD_CTX ctx, byte[] hash, int offset);\n\n    // --- Signature handling functions ----------------------------------------\n\n    static native long EVP_DigestSignInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native long EVP_DigestVerifyInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native void EVP_DigestSignUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestSignUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native void EVP_DigestVerifyUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestVerifyUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native byte[] EVP_DigestSignFinal(NativeRef.EVP_MD_CTX ctx);\n\n    static native boolean EVP_DigestVerifyFinal(NativeRef.EVP_MD_CTX ctx, byte[] signature,\n            int offset, int length) throws IndexOutOfBoundsException;\n\n    static native long EVP_PKEY_encrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_encrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native long EVP_PKEY_decrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_decrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native void EVP_PKEY_CTX_free(long pkeyCtx);\n\n    static native void EVP_PKEY_CTX_set_rsa_padding(long ctx, int pad)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_pss_saltlen(long ctx, int len)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_mgf1_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_label(long ctx, byte[] label)\n            throws InvalidAlgorithmParameterException;\n\n    // --- Block ciphers -------------------------------------------------------\n\n    // These return const references\n    static native long EVP_get_cipherbyname(String string);\n\n    static native void EVP_CipherInit_ex(NativeRef.EVP_CIPHER_CTX ctx, long evpCipher, byte[] key,\n            byte[] iv, boolean encrypting);\n\n    static native int EVP_CipherUpdate(NativeRef.EVP_CIPHER_CTX ctx, byte[] out, int outOffset,\n            byte[] in, int inOffset, int inLength) throws IndexOutOfBoundsException;\n\n    static native int EVP_CipherFinal_ex(NativeRef.EVP_CIPHER_CTX ctx, byte[] out, int outOffset)\n            throws BadPaddingException, IllegalBlockSizeException;\n\n    static native int EVP_CIPHER_iv_length(long evpCipher);\n\n    static native long EVP_CIPHER_CTX_new();\n\n    static native int EVP_CIPHER_CTX_block_size(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native int get_EVP_CIPHER_CTX_buf_len(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native boolean get_EVP_CIPHER_CTX_final_used(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native void EVP_CIPHER_CTX_set_padding(\n            NativeRef.EVP_CIPHER_CTX ctx, boolean enablePadding);\n\n    static native void EVP_CIPHER_CTX_set_key_length(NativeRef.EVP_CIPHER_CTX ctx, int keyBitSize);\n\n    static native void EVP_CIPHER_CTX_free(long ctx);\n\n    // --- AEAD ----------------------------------------------------------------\n    static native long EVP_aead_aes_128_gcm();\n\n    static native long EVP_aead_aes_256_gcm();\n\n    static native long EVP_aead_chacha20_poly1305();\n\n    static native long EVP_aead_aes_128_gcm_siv();\n\n    static native long EVP_aead_aes_256_gcm_siv();\n\n    static native int EVP_AEAD_max_overhead(long evpAead);\n\n    static native int EVP_AEAD_nonce_length(long evpAead);\n\n    static native int EVP_AEAD_CTX_seal(long evpAead, byte[] key, int tagLengthInBytes, byte[] out,\n            int outOffset, byte[] nonce, byte[] in, int inOffset, int inLength, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_seal_buf(long evpAead, byte[] key, int tagLengthInBytes, ByteBuffer out,\n                                            byte[] nonce, ByteBuffer input, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_open(long evpAead, byte[] key, int tagLengthInBytes, byte[] out,\n            int outOffset, byte[] nonce, byte[] in, int inOffset, int inLength, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_open_buf(long evpAead, byte[] key, int tagLengthInBytes, ByteBuffer out,\n                                            byte[] nonce, ByteBuffer input, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    // --- CMAC functions ------------------------------------------------------\n\n    static native long CMAC_CTX_new();\n\n    static native void CMAC_CTX_free(long ctx);\n\n    static native void CMAC_Init(NativeRef.CMAC_CTX ctx, byte[] key);\n\n...\n", "needle_token_start": 3970, "needle_token_end": 4036, "code_context_ntokens": 16007, "output": ["```java\n@Override\npublic Certificate engineGenerateCertificate(InputStream inStream) throws CertificateException {\n    try {\n        return certificateParser.generateItem(inStream);\n    } catch (ParsingException e) {\n        throw new CertificateException(e);\n    }\n}\n```"]}
{"repo": "google/conscrypt", "name": "open", "language": "java", "path": "common/src/main/java/org/conscrypt/HpkeContextRecipient.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: Decrypts a message that was previously encrypted using a corresponding encryption method, ensuring data confidentiality and integrity in secure communications.\n2. **Input**: A byte array representing the encrypted data and an optional byte array of associated data that can be used to provide additional contextual information for the decryption process.\n3. **Output**: A byte array containing the decrypted version of the input data, which is the original plaintext.\n4. **Procedure**: Utilizes the internal key schedule maintained by a cryptographic context specifically initialized for decryption. It checks if the context is properly initialized for decryption and not for encryption. If the initialization is correct, it proceeds to decrypt the encrypted data using the provided associated data if any. If decryption fails due to any issue such as incorrect keys or tampering, it throws a security exception.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/ConscryptEngineSocket.java\n/*\n * Copyright 2016 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport static javax.net.ssl.SSLEngineResult.Status.CLOSED;\nimport static javax.net.ssl.SSLEngineResult.Status.OK;\nimport static org.conscrypt.SSLUtils.EngineStates.STATE_CLOSED;\nimport static org.conscrypt.SSLUtils.EngineStates.STATE_HANDSHAKE_COMPLETED;\nimport static org.conscrypt.SSLUtils.EngineStates.STATE_HANDSHAKE_STARTED;\nimport static org.conscrypt.SSLUtils.EngineStates.STATE_NEW;\nimport static org.conscrypt.SSLUtils.EngineStates.STATE_READY;\nimport static org.conscrypt.SSLUtils.EngineStates.STATE_READY_HANDSHAKE_CUT_THROUGH;\n\nimport java.io.EOFException;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.net.InetAddress;\nimport java.net.Socket;\nimport java.net.SocketException;\nimport java.nio.ByteBuffer;\nimport java.security.PrivateKey;\nimport java.security.cert.CertificateException;\nimport java.security.cert.X509Certificate;\nimport javax.net.ssl.SSLEngine;\nimport javax.net.ssl.SSLEngineResult;\nimport javax.net.ssl.SSLEngineResult.HandshakeStatus;\nimport javax.net.ssl.SSLException;\nimport javax.net.ssl.SSLParameters;\nimport javax.net.ssl.SSLSession;\nimport javax.net.ssl.X509ExtendedTrustManager;\nimport javax.net.ssl.X509KeyManager;\nimport javax.net.ssl.X509TrustManager;\nimport javax.security.auth.x500.X500Principal;\n\n/**\n * Implements crypto handling by delegating to {@link ConscryptEngine}.\n */\nclass ConscryptEngineSocket extends OpenSSLSocketImpl implements SSLParametersImpl.AliasChooser {\n    private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocate(0);\n\n    private final ConscryptEngine engine;\n    private final Object stateLock = new Object();\n    private final Object handshakeLock = new Object();\n\n    private SSLOutputStream out;\n    private SSLInputStream in;\n\n    private long handshakeStartedMillis = 0;\n\n    private BufferAllocator bufferAllocator = ConscryptEngine.getDefaultBufferAllocator();\n\n    // @GuardedBy(\"stateLock\");\n    private int state = STATE_NEW;\n\n    // The constructors should not be called except from the Platform class, because we may\n    // want to construct a subclass instead.\n    ConscryptEngineSocket(SSLParametersImpl sslParameters) throws IOException {\n        engine = newEngine(sslParameters, this);\n    }\n\n    ConscryptEngineSocket(String hostname, int port, SSLParametersImpl sslParameters)\n            throws IOException {\n        super(hostname, port);\n        engine = newEngine(sslParameters, this);\n    }\n\n    ConscryptEngineSocket(InetAddress address, int port, SSLParametersImpl sslParameters)\n            throws IOException {\n        super(address, port);\n        engine = newEngine(sslParameters, this);\n    }\n\n    ConscryptEngineSocket(String hostname, int port, InetAddress clientAddress, int clientPort,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(hostname, port, clientAddress, clientPort);\n        engine = newEngine(sslParameters, this);\n    }\n\n    ConscryptEngineSocket(InetAddress address, int port, InetAddress clientAddress, int clientPort,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(address, port, clientAddress, clientPort);\n        engine = newEngine(sslParameters, this);\n    }\n\n    ConscryptEngineSocket(Socket socket, String hostname, int port, boolean autoClose,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(socket, hostname, port, autoClose);\n        engine = newEngine(sslParameters, this);\n    }\n\n    private static ConscryptEngine newEngine(\n            SSLParametersImpl sslParameters, final ConscryptEngineSocket socket) {\n        SSLParametersImpl modifiedParams;\n        if (Platform.supportsX509ExtendedTrustManager()) {\n            modifiedParams = sslParameters.cloneWithTrustManager(\n                getDelegatingTrustManager(sslParameters.getX509TrustManager(), socket));\n        } else {\n            modifiedParams = sslParameters;\n        }\n        ConscryptEngine engine =\n            new ConscryptEngine(modifiedParams, socket.peerInfoProvider(), socket);\n\n        // When the handshake completes, notify any listeners.\n        engine.setHandshakeListener(new HandshakeListener() {\n            /**\n             * Protected by {@code stateLock}\n             */\n            @Override\n            public void onHandshakeFinished() {\n                // Just call the outer class method.\n                socket.onEngineHandshakeFinished();\n            }\n        });\n\n        // Transition the engine state to MODE_SET\n        engine.setUseClientMode(sslParameters.getUseClientMode());\n        return engine;\n    }\n\n    // Returns a trust manager that delegates to the given trust manager, but maps SSLEngine\n    // references to the given ConscryptEngineSocket.  Our internal engine will call\n    // the SSLEngine-receiving methods, but our callers expect the SSLSocket-receiving\n    // methods to get called.\n    private static X509TrustManager getDelegatingTrustManager(final X509TrustManager delegate,\n        final ConscryptEngineSocket socket) {\n        if (delegate instanceof X509ExtendedTrustManager) {\n            final X509ExtendedTrustManager extendedDelegate = (X509ExtendedTrustManager) delegate;\n            return new X509ExtendedTrustManager() {\n                @Override public void checkClientTrusted(X509Certificate[] x509Certificates,\n                    String s, Socket socket) throws CertificateException {\n                    throw new AssertionError(\"Should not be called\");\n                }\n                @Override public void checkServerTrusted(X509Certificate[] x509Certificates,\n                    String s, Socket socket) throws CertificateException {\n                    throw new AssertionError(\"Should not be called\");\n                }\n                @Override public void checkClientTrusted(X509Certificate[] x509Certificates,\n                    String s, SSLEngine sslEngine) throws CertificateException {\n                    extendedDelegate.checkClientTrusted(x509Certificates, s, socket);\n                }\n                @Override public void checkServerTrusted(X509Certificate[] x509Certificates,\n                    String s, SSLEngine sslEngine) throws CertificateException {\n                    extendedDelegate.checkServerTrusted(x509Certificates, s, socket);\n                }\n                @Override public void checkClientTrusted(X509Certificate[] x509Certificates,\n                    String s) throws CertificateException {\n                    extendedDelegate.checkClientTrusted(x509Certificates, s);\n                }\n                @Override public void checkServerTrusted(X509Certificate[] x509Certificates,\n                    String s) throws CertificateException {\n                    extendedDelegate.checkServerTrusted(x509Certificates, s);\n                }\n                @Override public X509Certificate[] getAcceptedIssuers() {\n                    return extendedDelegate.getAcceptedIssuers();\n                }\n            };\n        }\n        return delegate;\n    }\n\n    @Override\n    public final SSLParameters getSSLParameters() {\n        return engine.getSSLParameters();\n    }\n\n    @Override\n    public final void setSSLParameters(SSLParameters sslParameters) {\n        engine.setSSLParameters(sslParameters);\n    }\n\n    @Override\n    public final void startHandshake() throws IOException {\n        checkOpen();\n\n        try {\n            synchronized (handshakeLock) {\n                // Only lock stateLock when we begin the handshake. This is done so that we don't\n                // hold the stateLock when we invoke the handshake completion listeners.\n                synchronized (stateLock) {\n                    // Initialize the handshake if we haven't already.\n                    if (state == STATE_NEW) {\n                        transitionTo(STATE_HANDSHAKE_STARTED);\n                        engine.beginHandshake();\n                        createInputStream();\n                        createOutputStream();\n                    } else {\n                        // We've either started the handshake already or have been closed.\n                        // Do nothing in both cases.\n                        //\n                        // NOTE: BoringSSL does not support initiating renegotiation, so we always\n                        // ignore addition handshake calls.\n                        return;\n                    }\n                }\n                doHandshake();\n            }\n        } catch (IOException e) {\n            close();\n            throw e;\n        } catch (Exception e) {\n            close();\n            // Convert anything else to a handshake exception.\n            throw SSLUtils.toSSLHandshakeException(e);\n        }\n    }\n\n    private void doHandshake() throws IOException {\n        try {\n            boolean finished = false;\n            while (!finished) {\n                switch (engine.getHandshakeStatus()) {\n                    case NEED_UNWRAP:\n                        if (in.processDataFromSocket(EmptyArray.BYTE, 0, 0) < 0) {\n                            // Can't complete the handshake due to EOF.\n                            close();\n                            throw SSLUtils.toSSLHandshakeException(\n                                    new EOFException(\"connection closed\"));\n                        }\n                        break;\n                    case NEED_WRAP: {\n                        out.writeInternal(EMPTY_BUFFER);\n                        // Always flush handshake frames immediately.\n                        out.flushInternal();\n                        break;\n                    }\n                    case NEED_TASK: {\n                        // Should never get here, since our engine never provides tasks.\n                        close();\n                        throw new IllegalStateException(\"Engine tasks are unsupported\");\n                    }\n                    case NOT_HANDSHAKING:\n                    case FINISHED: {\n                        // Handshake is complete.\n                        finished = true;\n                        break;\n                    }\n                    default: {\n                        throw new IllegalStateException(\n                            \"Unknown handshake status: \" + engine.getHandshakeStatus());\n                    }\n                }\n            }\n            if (isState(STATE_HANDSHAKE_COMPLETED)) {\n                // STATE_READY_HANDSHAKE_CUT_THROUGH will wake up any waiting threads which can\n                // race with the listeners, but that's OK.\n                transitionTo(STATE_READY_HANDSHAKE_CUT_THROUGH);\n                notifyHandshakeCompletedListeners();\n                transitionTo(STATE_READY);\n            }\n        } catch (SSLException e) {\n            drainOutgoingQueue();\n            close();\n            throw e;\n        } catch (IOException e) {\n            close();\n            throw e;\n        } catch (Exception e) {\n            close();\n            // Convert anything else to a handshake exception.\n            throw SSLUtils.toSSLHandshakeException(e);\n        }\n    }\n\n    private boolean isState(int desiredState) {\n        synchronized (stateLock) {\n            return state == desiredState;\n        }\n    }\n\n    private int transitionTo(int newState) {\n        synchronized (stateLock) {\n            if (state == newState) {\n                return state;\n            }\n\n            int previousState = state;\n            boolean notify = false;\n            switch (newState) {\n                case STATE_HANDSHAKE_STARTED:\n                    handshakeStartedMillis = Platform.getMillisSinceBoot();\n                    break;\n\n                case STATE_READY_HANDSHAKE_CUT_THROUGH:\n                    if (handshakeStartedMillis > 0) {\n                        Platform.countTlsHandshake(true,\n                            engine.getSession().getProtocol(),\n                            engine.getSession().getCipherSuite(),\n                            Platform.getMillisSinceBoot() - handshakeStartedMillis);\n                        handshakeStartedMillis = 0;\n...\n// Path: common/src/main/java/org/conscrypt/ByteArray.java\n/*\n * Copyright (C) 2011 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.util.Arrays;\n\n/**\n * Byte array wrapper for hashtable use. Implements equals() and hashCode().\n */\nfinal class ByteArray {\n    private final byte[] bytes;\n    private final int hashCode;\n\n    ByteArray(byte[] bytes) {\n        this.bytes = bytes;\n        this.hashCode = Arrays.hashCode(bytes);\n    }\n\n    @Override\n    public int hashCode() {\n        return hashCode;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (!(o instanceof ByteArray)) {\n            return false;\n        }\n        ByteArray lhs = (ByteArray) o;\n        if (hashCode != lhs.hashCode) {\n            return false;\n        }\n        return Arrays.equals(bytes, lhs.bytes);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/HpkeSpi.java\npackage org.conscrypt;\n\nimport java.security.GeneralSecurityException;\nimport java.security.InvalidKeyException;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\n\n/**\n * SPI for HPKE clients to communicate with implementations.  The client API can use any\n * implementation which implements this interface, by duck-typing if necessary.\n */\npublic interface HpkeSpi {\n  byte[] DEFAULT_PSK = new byte[0];\n  byte[] DEFAULT_PSK_ID = DEFAULT_PSK;\n\n  /**\n   * Initialises an HPKE sender SPI.\n   *\n   * @param recipientKey public key of the recipient\n   * @param info application-supplied information, may be null or empty\n   * @param senderKey private key of the sender, for symmetric auth modes only, else null\n   * @param psk pre-shared key, for PSK auth modes only, else null\n   * @param psk_id pre-shared key ID, for PSK auth modes only, else null\n   * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n   * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n   * @throws IllegalStateException if this SPI has already been initialised\n   */\n  void engineInitSender(\n          PublicKey recipientKey,\n          byte[] info,\n          PrivateKey senderKey,\n          byte[] psk,\n          byte[] psk_id)\n          throws InvalidKeyException;\n\n  /**\n   * Initialises an HPKE sender SPI.\n   *\n   * @param recipientKey public key of the recipient\n   * @param info application-supplied information, may be null or empty\n   * @param senderKey private key of the sender, for symmetric auth modes only, else null\n   * @param psk pre-shared key, for PSK auth modes only, else null\n   * @param psk_id pre-shared key ID, for PSK auth modes only, else null\n   * @param sKe optional random seed, should be null for all uses except for validation against\n   *            known test vectors\n   * @throws InvalidKeyException if recipientKey is null or an unsupported key format or senderKey\n   *            is an unsupported key format\n   * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n   * @throws IllegalStateException if this SPI has already been initialised\n   */\n  void engineInitSenderForTesting(\n          PublicKey recipientKey,\n          byte[] info,\n          PrivateKey senderKey,\n          byte[] psk,\n          byte[] psk_id,\n          byte[] sKe)\n          throws InvalidKeyException;\n\n  /**\n   * Initialises an HPKE recipient SPI.\n   *\n   * @param encapsulated encapsulated ephemeral key from a sender\n   * @param recipientKey private key of the recipient\n   * @param info application-supplied information, may be null or empty\n   * @param senderKey public key of sender, for asymmetric auth modes only, else null\n   * @param psk pre-shared key, for PSK auth modes only, else null\n   * @param psk_id pre-shared key ID, for PSK auth modes only, else null\n   * @throws InvalidKeyException if recipientKey is null or an unsupported key format or senderKey\n   *         is an unsupported key format\n   * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n   * @throws IllegalStateException if this SPI has already been initialised\n   */\n  void engineInitRecipient(\n          byte[] encapsulated,\n          PrivateKey recipientKey,\n          byte[] info,\n          PublicKey senderKey,\n          byte[] psk,\n          byte[] psk_id)\n          throws InvalidKeyException;\n\n  /**\n   * Seals a message, using the internal key schedule maintained by an HPKE sender.\n   *\n   * @param plaintext the plaintext\n   * @param aad optional associated data, may be null or empty\n   * @return the ciphertext\n   * @throws NullPointerException if the plaintext is null\n   * @throws IllegalStateException if this SPI has not been initialised or if it was initialised\n   *         as a recipient\n   */\n  byte[] engineSeal(byte[] plaintext, byte[] aad);\n\n  /**\n   * Opens a message, using the internal key schedule maintained by an HPKE recipient.\n   *\n   * @param ciphertext the ciphertext\n   * @param aad optional associated data, may be null or empty\n   * @return the plaintext\n   * @throws IllegalStateException if this SPI has not been initialised or if it was initialised\n   *         as a sender\n   * @throws GeneralSecurityException on decryption failures\n   */\n  byte[] engineOpen(byte[] ciphertext, byte[] aad) throws GeneralSecurityException;\n\n  /**\n   * Exports secret key material from this SPI as described in RFC 9180.\n   *\n   * @param length  expected output length\n   * @param context optional context string, may be null or empty\n   * @return exported value\n   * @throws IllegalArgumentException if the length is not valid for the KDF in use\n   * @throws IllegalStateException if this SPI has not been initialised\n   *\n   */\n  byte[] engineExport(int length, byte[] context);\n\n  /**\n   * Returns the encapsulated key material for an HPKE sender.\n   *\n   * @return the key material\n   * @throws IllegalStateException if this SPI has not been initialised or if it was initialised\n   *         as a recipient\n   */\n  byte[] getEncapsulated();\n}\n\n// Path: common/src/main/java/org/conscrypt/HpkeContextRecipient.java\n/*\n * Copyright (C) 2023 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License\n */\n\npackage org.conscrypt;\n\nimport java.security.GeneralSecurityException;\nimport java.security.InvalidKeyException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.NoSuchProviderException;\nimport java.security.PrivateKey;\nimport java.security.Provider;\nimport java.security.PublicKey;\n\n/**\n * Hybrid Public Key Encryption (HPKE) recipient APIs.\n *\n * @see <a href=\"https://www.rfc-editor.org/rfc/rfc9180.html#hpke-export\">HPKE RFC 9180</a>\n *\n * Recipient subclass of HpkeContext.  See base class for details.\n */\npublic class HpkeContextRecipient extends HpkeContext {\n    private HpkeContextRecipient(HpkeSpi spi) {\n        super(spi);\n    }\n\n    /**\n     * Opens a message, using the internal key schedule maintained by this HpkeContextRecipient.\n     *\n     * @param ciphertext the ciphertext\n     * @param aad optional associated data, may be null or empty\n     * @return the plaintext\n     * @throws IllegalStateException if this HpkeContextRecipient has not been initialised\n     * @throws GeneralSecurityException on decryption failures\n     */\n    \npublic byte[] open(byte[] ciphertext, byte[] aad) throws GeneralSecurityException {\n        return spi.engineOpen(ciphertext, aad);\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextRecipient.\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @return an uninitialised HpkeContextRecipient for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     */\n    public static HpkeContextRecipient getInstance(String suite) throws NoSuchAlgorithmException {\n        return new HpkeContextRecipient(findSpi(suite));\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextRecipient from a specific {@link Provider}\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @param providerName the name of the Provider to use\n     * @return an uninitialised HpkeContextRecipient for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     * @throws NoSuchProviderException if providerName is null or no such Provider exists\n     */\n    public static HpkeContextRecipient getInstance(String suite, String providerName)\n        throws NoSuchAlgorithmException, NoSuchProviderException {\n        return new HpkeContextRecipient(findSpi(suite, providerName));\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextRecipient from a specific {@link Provider}\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @param provider the Provider to use\n     * @return an uninitialised HpkeContextRecipient for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     * @throws NoSuchProviderException if providerName is null or no such Provider exists\n     */\n    public static HpkeContextRecipient getInstance(String suite, Provider provider)\n        throws NoSuchAlgorithmException, NoSuchProviderException {\n        return new HpkeContextRecipient(findSpi(suite, provider));\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in BASE mode, i.e. no sender authentication.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info)\n            throws InvalidKeyException {\n        spi.engineInitRecipient(encapsulated, recipientKey, info, null,\n                HpkeSpi.DEFAULT_PSK, HpkeSpi.DEFAULT_PSK_ID);\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in AUTH mode, i.e. messages are authenticated using\n     * the sender's public key.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @param senderKey the public key of the sender\n     * @throws InvalidKeyException if either recipientKey or senderKey are null\n     *         or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info, PublicKey senderKey)\n        throws InvalidKeyException {\n        if (senderKey == null) {\n            throw new InvalidKeyException(\"null sender key\");\n        }\n        // Remaining argument checks are performed by the SPI\n        spi.engineInitRecipient(encapsulated, recipientKey, info, senderKey,\n                HpkeSpi.DEFAULT_PSK, HpkeSpi.DEFAULT_PSK_ID);\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in PSK_AUTH mode, i.e. messages are authenticated using\n     * a pre-shared secret key.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @param psk the a pre-shared secret key\n     * @param psk_id the id of the pre-shared secret key\n     * @throws NullPointerException if psk or psk_id are null\n     * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info,\n            byte[] psk, byte[] psk_id) throws InvalidKeyException {\n        spi.engineInitRecipient(encapsulated, recipientKey, info, null, psk, psk_id);\n    }\n\n    /**\n     * Initialises this HpkeContextRecipient in PSK_AUTH mode, i.e. messages are authenticated using\n     * both the sender's public key and a pre-shared secret key.\n     *\n     * @param encapsulated encapsulated ephemeral key from an {@link HpkeContextSender}\n     * @param recipientKey private key of the recipient\n     * @param info application-supplied information, may be null or empty\n     * @param senderKey the public key of the sender\n     * @param psk the a pre-shared secret key\n     * @param psk_id the id of the pre-shared secret key\n     * @throws NullPointerException if psk or psk_id are null\n     * @throws InvalidKeyException if either recipientKey or senderKey are null\n     *         or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextRecipient has already been initialised\n     */\n    public void init(byte[] encapsulated, PrivateKey recipientKey, byte[] info, PublicKey senderKey,\n            byte[] psk, byte[] psk_id) throws InvalidKeyException {\n        if (senderKey == null) {\n            throw new InvalidKeyException(\"null sender key\");\n        }\n        // Remaining argument checks are performed by the SPI\n        spi.engineInitRecipient(encapsulated, recipientKey, info, senderKey, psk, psk_id);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLRSAKeyPairGenerator.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.math.BigInteger;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.KeyPair;\nimport java.security.KeyPairGeneratorSpi;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport java.security.spec.RSAKeyGenParameterSpec;\n\n/**\n * An implementation of {@link java.security.KeyPairGenerator} which uses BoringSSL to perform all\n * the operations.\n */\n@Internal\npublic final class OpenSSLRSAKeyPairGenerator extends KeyPairGeneratorSpi {\n    /**\n     * Default modulus size is 0x10001 (65537)\n     */\n    private byte[] publicExponent = new byte[] {\n            0x01, 0x00, 0x01\n    };\n\n    /**\n     * Default RSA key size 2048 bits.\n     */\n    private int modulusBits = 2048;\n\n    @Override\n    public KeyPair generateKeyPair() {\n        final OpenSSLKey key = new OpenSSLKey(NativeCrypto.RSA_generate_key_ex(modulusBits,\n                publicExponent));\n\n        PrivateKey privKey = OpenSSLRSAPrivateKey.getInstance(key);\n        PublicKey pubKey = new OpenSSLRSAPublicKey(key);\n\n        return new KeyPair(pubKey, privKey);\n    }\n\n    @Override\n    public void initialize(int keysize, SecureRandom random) {\n        this.modulusBits = keysize;\n    }\n\n    @Override\n    public void initialize(AlgorithmParameterSpec params, SecureRandom random)\n            throws InvalidAlgorithmParameterException {\n        if (!(params instanceof RSAKeyGenParameterSpec)) {\n            throw new InvalidAlgorithmParameterException(\"Only RSAKeyGenParameterSpec supported\");\n        }\n\n        RSAKeyGenParameterSpec spec = (RSAKeyGenParameterSpec) params;\n\n        final BigInteger publicExponent = spec.getPublicExponent();\n        if (publicExponent != null) {\n            this.publicExponent = publicExponent.toByteArray();\n        }\n\n        this.modulusBits = spec.getKeysize();\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLBaseDHKeyAgreement.java\n/*\n * Copyright (C) 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.Key;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport javax.crypto.KeyAgreementSpi;\nimport javax.crypto.SecretKey;\nimport javax.crypto.ShortBufferException;\nimport javax.crypto.spec.SecretKeySpec;\n\n/**\n * Generic base classe for Diffie-Hellman style key agreement backed by the OpenSSL engine.\n */\n@Internal\npublic abstract class OpenSSLBaseDHKeyAgreement<T> extends KeyAgreementSpi {\n\n    /** OpenSSL handle of the private key. Only available after the engine has been initialized. */\n    private T mPrivateKey;\n\n    /**\n     * Expected length (in bytes) of the agreed key ({@link #mResult}). Only available after the\n     * engine has been initialized.\n     */\n    private int mExpectedResultLength;\n\n    /** Agreed key. Only available after {@link #engineDoPhase(Key, boolean)} completes. */\n    private byte[] mResult;\n\n    protected OpenSSLBaseDHKeyAgreement() {}\n\n    @Override\n    public Key engineDoPhase(Key key, boolean lastPhase) throws InvalidKeyException {\n        if (mPrivateKey == null) {\n            throw new IllegalStateException(\"Not initialized\");\n        }\n        if (!lastPhase) {\n            throw new IllegalStateException(\"DH only has one phase\");\n        }\n\n        if (key == null) {\n            throw new InvalidKeyException(\"key == null\");\n        }\n        if (!(key instanceof PublicKey)) {\n            throw new InvalidKeyException(\"Not a public key: \" + key.getClass());\n        }\n        T openSslPublicKey = convertPublicKey((PublicKey) key);\n\n        byte[] buffer = new byte[mExpectedResultLength];\n        int actualResultLength = computeKey(buffer, openSslPublicKey, mPrivateKey);\n        byte[] result;\n        if (actualResultLength == -1) {\n            throw new RuntimeException(\"Engine returned -1\");\n        } else if (actualResultLength == mExpectedResultLength) {\n            // The output is as long as expected -- use the whole buffer\n            result = buffer;\n        } else if (actualResultLength < mExpectedResultLength) {\n            // The output is shorter than expected -- use only what's produced by the engine\n            result = new byte[actualResultLength];\n            System.arraycopy(buffer, 0, mResult, 0, mResult.length);\n        } else {\n            // The output is longer than expected\n            throw new RuntimeException(\"Engine produced a longer than expected result. Expected: \"\n                + mExpectedResultLength + \", actual: \" + actualResultLength);\n        }\n        mResult = result;\n\n        return null; // No intermediate key\n    }\n\n    protected abstract T convertPublicKey(PublicKey key) throws InvalidKeyException;\n\n    protected abstract T convertPrivateKey(PrivateKey key) throws InvalidKeyException;\n\n    /**\n     * Given the public key {@code theirPublicKey} and the private key {@code ourPrivateKey} write the shared secret\n     * to {@code buffer} and return the actual output size.\n     */\n    protected abstract int computeKey(byte[] buffer, T theirPublicKey, T ourPrivateKey) throws InvalidKeyException;\n\n    @Override\n    protected int engineGenerateSecret(byte[] sharedSecret, int offset)\n            throws ShortBufferException {\n        checkCompleted();\n        int available = sharedSecret.length - offset;\n        if (mResult.length > available) {\n            throw new ShortBufferWithoutStackTraceException(\n                    \"Needed: \" + mResult.length + \", available: \" + available);\n        }\n\n        System.arraycopy(mResult, 0, sharedSecret, offset, mResult.length);\n        return mResult.length;\n    }\n\n    @Override\n    protected byte[] engineGenerateSecret() {\n        checkCompleted();\n        return mResult;\n    }\n\n    @Override\n    protected SecretKey engineGenerateSecret(String algorithm) {\n        checkCompleted();\n        return new SecretKeySpec(engineGenerateSecret(), algorithm);\n    }\n\n    @Override\n    protected void engineInit(Key key, SecureRandom random) throws InvalidKeyException {\n        if (key == null) {\n            throw new InvalidKeyException(\"key == null\");\n        }\n        if (!(key instanceof PrivateKey)) {\n            throw new InvalidKeyException(\"Not a private key: \" + key.getClass());\n        }\n\n        T privateKey = convertPrivateKey((PrivateKey) key);\n        mExpectedResultLength = getOutputSize(privateKey);\n        mPrivateKey = privateKey;\n    }\n\n    /** Returns the expected result length for the given {@code key}. */\n    protected abstract int getOutputSize(T key);\n\n    @Override\n    protected void engineInit(Key key, AlgorithmParameterSpec params,\n            SecureRandom random) throws InvalidKeyException, InvalidAlgorithmParameterException {\n        // ECDH doesn't need an AlgorithmParameterSpec\n        if (params != null) {\n          throw new InvalidAlgorithmParameterException(\"No algorithm parameters supported\");\n        }\n        engineInit(key, random);\n    }\n\n    private void checkCompleted() {\n        if (mResult == null) {\n            throw new IllegalStateException(\"Key agreement not completed\");\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ct/CTLogStore.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport org.conscrypt.Internal;\n\n@Internal\npublic interface CTLogStore {\n    CTLogInfo getKnownLog(byte[] logId);\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CTPolicy.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.security.cert.X509Certificate;\nimport org.conscrypt.Internal;\n\n@Internal\npublic interface CTPolicy {\n    boolean doesResultConformToPolicy(CTVerificationResult result, String hostname,\n            X509Certificate[] chain);\n}\n\n// Path: common/src/main/java/org/conscrypt/ct/VerifiedSCT.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport org.conscrypt.Internal;\n\n/**\n * Verification result for a single SCT.\n */\n@Internal\npublic final class VerifiedSCT {\n    public enum Status {\n        VALID,\n        INVALID_SIGNATURE,\n        UNKNOWN_LOG,\n        INVALID_SCT\n    }\n\n    public final SignedCertificateTimestamp sct;\n    public final Status status;\n\n    public VerifiedSCT(SignedCertificateTimestamp sct, Status status) {\n        this.sct = sct;\n        this.status = status;\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CTVerificationResult.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport org.conscrypt.Internal;\n\n@Internal\npublic class CTVerificationResult {\n    private final ArrayList<VerifiedSCT> validSCTs = new ArrayList<VerifiedSCT>();\n    private final ArrayList<VerifiedSCT> invalidSCTs = new ArrayList<VerifiedSCT>();\n\n    public void add(VerifiedSCT result) {\n        if (result.status == VerifiedSCT.Status.VALID) {\n            validSCTs.add(result);\n        } else {\n            invalidSCTs.add(result);\n        }\n    }\n\n    public List<VerifiedSCT> getValidSCTs() {\n        return Collections.unmodifiableList(validSCTs);\n    }\n\n    public List<VerifiedSCT> getInvalidSCTs() {\n        return Collections.unmodifiableList(invalidSCTs);\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CTConstants.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport org.conscrypt.Internal;\n\n@Internal\npublic class CTConstants {\n    public static final String X509_SCT_LIST_OID = \"1.3.6.1.4.1.11129.2.4.2\";\n    public static final String OCSP_SCT_LIST_OID = \"1.3.6.1.4.1.11129.2.4.5\";\n\n    public static final int VERSION_LENGTH = 1;\n    public static final int LOGID_LENGTH = 32;\n    public static final int TIMESTAMP_LENGTH = 8;\n    public static final int EXTENSIONS_LENGTH_BYTES = 2;\n\n    public static final int HASH_ALGORITHM_LENGTH = 1;\n    public static final int SIGNATURE_ALGORITHM_LENGTH = 1;\n    public static final int SIGNATURE_LENGTH_BYTES = 2;\n\n    public static final int SIGNATURE_TYPE_LENGTH = 1;\n    public static final int LOG_ENTRY_TYPE_LENGTH = 2;\n    public static final int CERTIFICATE_LENGTH_BYTES = 3;\n\n    public static final int SERIALIZED_SCT_LENGTH_BYTES = 2;\n    public static final int SCT_LIST_LENGTH_BYTES = 2;\n\n    public static final int ISSUER_KEY_HASH_LENGTH = 32;\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/Serialization.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.util.ArrayList;\nimport org.conscrypt.Internal;\n\n@Internal\npublic class Serialization {\n    private Serialization() {}\n\n    private static final int DER_TAG_MASK = 0x3f;\n    private static final int DER_TAG_OCTET_STRING = 0x4;\n    private static final int DER_LENGTH_LONG_FORM_FLAG = 0x80;\n\n    public static byte[] readDEROctetString(byte[] input)\n            throws SerializationException {\n        return readDEROctetString(new ByteArrayInputStream(input));\n    }\n\n    public static byte[] readDEROctetString(InputStream input)\n            throws SerializationException {\n        int tag = readByte(input) & DER_TAG_MASK;\n        if (tag != DER_TAG_OCTET_STRING) {\n            throw new SerializationException(\"Wrong DER tag, expected OCTET STRING, got \" + tag);\n        }\n        int length;\n        int width = readNumber(input, 1);\n        if ((width & DER_LENGTH_LONG_FORM_FLAG) != 0) {\n            length = readNumber(input, width & ~DER_LENGTH_LONG_FORM_FLAG);\n        } else {\n            length = width;\n        }\n\n        return readFixedBytes(input, length);\n    }\n\n    public static byte[][] readList(byte[] input, int listWidth, int elemWidth)\n            throws SerializationException {\n        return readList(new ByteArrayInputStream(input), listWidth, elemWidth);\n    }\n\n    /**\n     * Read a variable length vector of variable sized elements as described by RFC5246 section 4.3.\n     * The vector is prefixed by its total length, in bytes and in big endian format,\n     * so is each element contained in the vector.\n     * @param listWidth the width of the vector's length field, in bytes.\n     * @param elemWidth the width of each element's length field, in bytes.\n     * @throws SerializationException if EOF is encountered.\n     */\n    public static byte[][] readList(InputStream input, int listWidth, int elemWidth)\n            throws SerializationException {\n        ArrayList<byte[]> result = new ArrayList<byte[]>();\n        byte[] data = readVariableBytes(input, listWidth);\n        input = new ByteArrayInputStream(data);\n        try {\n            while (input.available() > 0) {\n                result.add(readVariableBytes(input, elemWidth));\n            }\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n        return result.toArray(new byte[result.size()][]);\n    }\n\n    /**\n     * Read a length-prefixed sequence of bytes.\n     * The length must be encoded in big endian format.\n     * @param width the width of the length prefix, in bytes.\n     * @throws SerializationException if EOF is encountered, or if {@code width} is negative or\n     * greater than 4\n     */\n    public static byte[] readVariableBytes(InputStream input, int width)\n            throws SerializationException {\n        int length = readNumber(input, width);\n        return readFixedBytes(input, length);\n    }\n\n    /**\n     * Read a fixed number of bytes from the input stream.\n     * @param length the number of bytes to read.\n     * @throws SerializationException if EOF is encountered.\n     */\n    public static byte[] readFixedBytes(InputStream input, int length)\n            throws SerializationException {\n        try {\n            if (length < 0) {\n                throw new SerializationException(\"Negative length: \" + length);\n            }\n\n            byte[] data = new byte[length];\n            int count = input.read(data);\n            if (count < length) {\n                throw new SerializationException(\"Premature end of input, expected \" + length +\n                                                 \" bytes, only read \" + count);\n            }\n            return data;\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n\n    /**\n     * Read a number in big endian format from the input stream.\n     * This methods only supports a width of up to 4 bytes.\n     * @param width the width of the number, in bytes.\n     * @throws SerializationException if EOF is encountered, or if {@code width} is negative or\n     * greater than 4\n     */\n    public static int readNumber(InputStream input, int width) throws SerializationException {\n        if (width > 4 || width < 0) {\n            throw new SerializationException(\"Invalid width: \" + width);\n        }\n\n        int result = 0;\n        for (int i = 0; i < width; i++) {\n            result = (result << 8) | (readByte(input) & 0xFF);\n        }\n\n        return result;\n    }\n\n    /**\n     * Read a number in big endian format from the input stream.\n     * This methods supports a width of up to 8 bytes.\n     * @param width the width of the number, in bytes.\n     * @throws SerializationException if EOF is encountered.\n     * @throws IllegalArgumentException if {@code width} is negative or greater than 8\n     */\n    public static long readLong(InputStream input, int width) throws SerializationException {\n        if (width > 8 || width < 0) {\n            throw new IllegalArgumentException(\"Invalid width: \" + width);\n        }\n\n        long result = 0;\n        for (int i = 0; i < width; i++) {\n            result = (result << 8) | (readByte(input) & 0xFF);\n        }\n\n        return result;\n    }\n\n    /**\n     * Read a single byte from the input stream.\n     * @throws SerializationException if EOF is encountered.\n     */\n    public static byte readByte(InputStream input) throws SerializationException {\n        try {\n            int b = input.read();\n            if (b == -1) {\n                throw new SerializationException(\"Premature end of input, could not read byte.\");\n            }\n            return (byte)b;\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n\n    /**\n     * Write length prefixed sequence of bytes to the ouput stream.\n     * The length prefix is encoded in big endian order.\n     * @param data the data to be written.\n     * @param width the width of the length prefix, in bytes.\n     * @throws SerializationException if the length of {@code data} is too large to fit in\n     * {@code width} bytes or {@code width} is negative.\n     */\n    public static void writeVariableBytes(OutputStream output, byte[] data, int width)\n            throws SerializationException {\n        writeNumber(output, data.length, width);\n        writeFixedBytes(output, data);\n    }\n\n    /**\n     * Write a fixed number sequence of bytes to the ouput stream.\n     * @param data the data to be written.\n     */\n    public static void writeFixedBytes(OutputStream output, byte[] data)\n            throws SerializationException {\n        try {\n            output.write(data);\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n\n    /**\n     * Write a number to the output stream.\n     * The number is encoded in big endian order.\n     * @param value the value to be written.\n     * @param width the width of the encoded number, in bytes\n     * @throws SerializationException if the number is too large to fit in {@code width} bytes or\n     * {@code width} is negative.\n     */\n    public static void writeNumber(OutputStream output, long value, int width)\n            throws SerializationException {\n        if (width < 0) {\n            throw new SerializationException(\"Negative width: \" + width);\n        }\n        if (width < 8 && value >= (1L << (8 * width))) {\n            throw new SerializationException(\n                    \"Number too large, \" + value + \" does not fit in \" + width + \" bytes\");\n        }\n\n        try {\n            while (width > 0) {\n                long shift = (width - 1) * 8L;\n                // Java behaves weirdly if shifting by more than the variable's size\n                if (shift < Long.SIZE) {\n                    output.write((byte) ((value >> shift) & 0xFF));\n                } else {\n                    output.write(0);\n                }\n\n                width--;\n            }\n        } catch (IOException e) {\n            throw new SerializationException(e);\n        }\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/CertificateEntry.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.io.OutputStream;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.X509Certificate;\nimport org.conscrypt.Internal;\nimport org.conscrypt.OpenSSLX509Certificate;\n\n/**\n * CertificateEntry structure.\n * This structure describes part of the data which is signed over in SCTs.\n * It is not defined by the RFC6962, but it is useful to have.\n *\n * It's definition would be :\n * struct {\n *     LogEntryType entry_type;\n *     select(entry_type) {\n *         case x509_entry: ASN.1Cert;\n *         case precert_entry: PreCert;\n *     } signed_entry;\n * } CertificateEntry;\n */\n@Internal\npublic class CertificateEntry {\n    public enum LogEntryType {\n        X509_ENTRY,\n        PRECERT_ENTRY\n    }\n\n    private final LogEntryType entryType;\n\n    // Only used when entryType is LOG_ENTRY_TYPE_PRECERT\n    private final byte[] issuerKeyHash;\n\n    /* If entryType == PRECERT_ENTRY, this is the encoded TBS of the precertificate.\n       If entryType == X509_ENTRY, this is the encoded leaf certificate. */\n    private final byte[] certificate;\n\n    private CertificateEntry(LogEntryType entryType, byte[] certificate, byte[] issuerKeyHash) {\n        if (entryType == LogEntryType.PRECERT_ENTRY && issuerKeyHash == null) {\n            throw new IllegalArgumentException(\"issuerKeyHash missing for precert entry.\");\n        } else if (entryType == LogEntryType.X509_ENTRY && issuerKeyHash != null) {\n            throw new IllegalArgumentException(\"unexpected issuerKeyHash for X509 entry.\");\n        }\n        \n        if (issuerKeyHash != null && issuerKeyHash.length != CTConstants.ISSUER_KEY_HASH_LENGTH) {\n            throw new IllegalArgumentException(\"issuerKeyHash must be 32 bytes long\");\n        }\n\n        this.entryType = entryType;\n        this.issuerKeyHash = issuerKeyHash;\n        this.certificate = certificate;\n    }\n\n    /**\n     * Creates a CertificateEntry with type PRECERT_ENTRY\n     *\n     * @throws IllegalArgumentException if issuerKeyHash isn't 32 bytes\n     */\n    public static CertificateEntry createForPrecertificate(byte[] tbsCertificate, byte[] issuerKeyHash) {\n        return new CertificateEntry(LogEntryType.PRECERT_ENTRY, tbsCertificate, issuerKeyHash);\n    }\n\n    public static CertificateEntry createForPrecertificate(OpenSSLX509Certificate leaf,\n            OpenSSLX509Certificate issuer) throws CertificateException {\n        try {\n            if (!leaf.getNonCriticalExtensionOIDs().contains(CTConstants.X509_SCT_LIST_OID)) {\n                throw new CertificateException(\"Certificate does not contain embedded signed timestamps\");\n            }\n\n            byte[] tbs = leaf.getTBSCertificateWithoutExtension(CTConstants.X509_SCT_LIST_OID);\n\n            byte[] issuerKey = issuer.getPublicKey().getEncoded();\n            MessageDigest md = MessageDigest.getInstance(\"SHA-256\");\n            md.update(issuerKey);\n            byte[] issuerKeyHash = md.digest();\n\n            return createForPrecertificate(tbs, issuerKeyHash);\n        } catch (NoSuchAlgorithmException e) {\n            // SHA-256 is guaranteed to be available\n            throw new RuntimeException(e);\n        }\n    }\n\n    public static CertificateEntry createForX509Certificate(byte[] x509Certificate) {\n        return new CertificateEntry(LogEntryType.X509_ENTRY, x509Certificate, null);\n    }\n\n    public static CertificateEntry createForX509Certificate(X509Certificate cert)\n            throws CertificateEncodingException {\n        return createForX509Certificate(cert.getEncoded());\n    }\n\n    public LogEntryType getEntryType() {\n        return entryType;\n    }\n    public byte[] getCertificate() {\n        return certificate;\n    }\n    public byte[] getIssuerKeyHash() {\n        return issuerKeyHash;\n    }\n\n    /**\n     * TLS encode the CertificateEntry structure.\n     */\n    public void encode(OutputStream output) throws SerializationException {\n        Serialization.writeNumber(output, entryType.ordinal(), CTConstants.LOG_ENTRY_TYPE_LENGTH);\n        if (entryType == LogEntryType.PRECERT_ENTRY) {\n            Serialization.writeFixedBytes(output, issuerKeyHash);\n        }\n        Serialization.writeVariableBytes(output, certificate, CTConstants.CERTIFICATE_LENGTH_BYTES);\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/ct/DigitallySigned.java\n/*\n * Copyright (C) 2015 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt.ct;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.InputStream;\nimport org.conscrypt.Internal;\n\n/**\n * DigitallySigned structure, as defined by RFC5246 Section 4.7.\n */\n@Internal\npublic class DigitallySigned {\n    public enum HashAlgorithm {\n        NONE,\n        MD5,\n        SHA1,\n        SHA224,\n        SHA256,\n        SHA384,\n        SHA512;\n\n        private static HashAlgorithm[] values = values();\n        public static HashAlgorithm valueOf(int ord) {\n            try {\n                return values[ord];\n            } catch (IndexOutOfBoundsException e) {\n                throw new IllegalArgumentException(\"Invalid hash algorithm \" + ord, e);\n            }\n        }\n    }\n\n    public enum SignatureAlgorithm {\n        ANONYMOUS,\n        RSA,\n        DSA,\n        ECDSA;\n\n        private static SignatureAlgorithm[] values = values();\n        public static SignatureAlgorithm valueOf(int ord) {\n            try {\n                return values[ord];\n            } catch (IndexOutOfBoundsException e) {\n                throw new IllegalArgumentException(\"Invalid signature algorithm \" + ord, e);\n...\n", "needle_token_start": 5611, "needle_token_end": 5651, "code_context_ntokens": 15989, "output": ["```java\n/**\n * Opens a message, using the internal key schedule maintained by this HpkeContextRecipient.\n *\n * @param ciphertext the ciphertext\n * @param aad optional associated data, may be null or empty\n * @return the plaintext\n * @throws IllegalStateException if this HpkeContextRecipient has not been initialised\n * @throws GeneralSecurityException on decryption failures\n */\npublic byte[] open(byte[] ciphertext, byte[] aad) throws GeneralSecurityException {\n    return spi.engineOpen(ciphertext, aad);\n}\n```"]}
{"repo": "google/conscrypt", "name": "asn1_read_free", "language": "java", "path": "common/src/main/java/org/conscrypt/NativeCrypto.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to release resources associated with a specific reference used in ASN.1 data reading operations, ensuring that system resources are properly managed and freed after use.\n2. **Input**: Accepts a long integer representing a reference to an ASN.1 data structure.\n3. **Output**: There is no return value as the primary function is to manage memory by freeing resources.\n4. **Procedure**: The function checks if the provided reference is non-zero and, if so, it releases any memory or resources tied to that reference. After execution, the reference should not be used in any subsequent operations to avoid errors or system crashes.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n    static native void X25519_keypair(byte[] outPublicKey, byte[] outPrivateKey);\n\n    // --- Message digest functions --------------\n\n    // These return const references\n    static native long EVP_get_digestbyname(String name);\n\n    static native int EVP_MD_size(long evp_md_const);\n\n    // --- Message digest context functions --------------\n\n    static native long EVP_MD_CTX_create();\n\n    static native void EVP_MD_CTX_cleanup(NativeRef.EVP_MD_CTX ctx);\n\n    static native void EVP_MD_CTX_destroy(long ctx);\n\n    static native int EVP_MD_CTX_copy_ex(\n            NativeRef.EVP_MD_CTX dst_ctx, NativeRef.EVP_MD_CTX src_ctx);\n\n    // --- Digest handling functions -------------------------------------------\n\n    static native int EVP_DigestInit_ex(NativeRef.EVP_MD_CTX ctx, long evp_md);\n\n    static native void EVP_DigestUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native int EVP_DigestFinal_ex(NativeRef.EVP_MD_CTX ctx, byte[] hash, int offset);\n\n    // --- Signature handling functions ----------------------------------------\n\n    static native long EVP_DigestSignInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native long EVP_DigestVerifyInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native void EVP_DigestSignUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestSignUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native void EVP_DigestVerifyUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestVerifyUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native byte[] EVP_DigestSignFinal(NativeRef.EVP_MD_CTX ctx);\n\n    static native boolean EVP_DigestVerifyFinal(NativeRef.EVP_MD_CTX ctx, byte[] signature,\n            int offset, int length) throws IndexOutOfBoundsException;\n\n    static native long EVP_PKEY_encrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_encrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native long EVP_PKEY_decrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_decrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native void EVP_PKEY_CTX_free(long pkeyCtx);\n\n    static native void EVP_PKEY_CTX_set_rsa_padding(long ctx, int pad)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_pss_saltlen(long ctx, int len)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_mgf1_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_label(long ctx, byte[] label)\n            throws InvalidAlgorithmParameterException;\n\n    // --- Block ciphers -------------------------------------------------------\n\n    // These return const references\n    static native long EVP_get_cipherbyname(String string);\n\n    static native void EVP_CipherInit_ex(NativeRef.EVP_CIPHER_CTX ctx, long evpCipher, byte[] key,\n            byte[] iv, boolean encrypting);\n\n    static native int EVP_CipherUpdate(NativeRef.EVP_CIPHER_CTX ctx, byte[] out, int outOffset,\n            byte[] in, int inOffset, int inLength) throws IndexOutOfBoundsException;\n\n    static native int EVP_CipherFinal_ex(NativeRef.EVP_CIPHER_CTX ctx, byte[] out, int outOffset)\n            throws BadPaddingException, IllegalBlockSizeException;\n\n    static native int EVP_CIPHER_iv_length(long evpCipher);\n\n    static native long EVP_CIPHER_CTX_new();\n\n    static native int EVP_CIPHER_CTX_block_size(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native int get_EVP_CIPHER_CTX_buf_len(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native boolean get_EVP_CIPHER_CTX_final_used(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native void EVP_CIPHER_CTX_set_padding(\n            NativeRef.EVP_CIPHER_CTX ctx, boolean enablePadding);\n\n    static native void EVP_CIPHER_CTX_set_key_length(NativeRef.EVP_CIPHER_CTX ctx, int keyBitSize);\n\n    static native void EVP_CIPHER_CTX_free(long ctx);\n\n    // --- AEAD ----------------------------------------------------------------\n    static native long EVP_aead_aes_128_gcm();\n\n    static native long EVP_aead_aes_256_gcm();\n\n    static native long EVP_aead_chacha20_poly1305();\n\n    static native long EVP_aead_aes_128_gcm_siv();\n\n    static native long EVP_aead_aes_256_gcm_siv();\n\n    static native int EVP_AEAD_max_overhead(long evpAead);\n\n    static native int EVP_AEAD_nonce_length(long evpAead);\n\n    static native int EVP_AEAD_CTX_seal(long evpAead, byte[] key, int tagLengthInBytes, byte[] out,\n            int outOffset, byte[] nonce, byte[] in, int inOffset, int inLength, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_seal_buf(long evpAead, byte[] key, int tagLengthInBytes, ByteBuffer out,\n                                            byte[] nonce, ByteBuffer input, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_open(long evpAead, byte[] key, int tagLengthInBytes, byte[] out,\n            int outOffset, byte[] nonce, byte[] in, int inOffset, int inLength, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_open_buf(long evpAead, byte[] key, int tagLengthInBytes, ByteBuffer out,\n                                            byte[] nonce, ByteBuffer input, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    // --- CMAC functions ------------------------------------------------------\n\n    static native long CMAC_CTX_new();\n\n    static native void CMAC_CTX_free(long ctx);\n\n    static native void CMAC_Init(NativeRef.CMAC_CTX ctx, byte[] key);\n\n    static native void CMAC_Update(NativeRef.CMAC_CTX ctx, byte[] in, int inOffset, int inLength);\n\n    static native void CMAC_UpdateDirect(NativeRef.CMAC_CTX ctx, long inPtr, int inLength);\n\n    static native byte[] CMAC_Final(NativeRef.CMAC_CTX ctx);\n\n    static native void CMAC_Reset(NativeRef.CMAC_CTX ctx);\n\n    // --- HMAC functions ------------------------------------------------------\n\n    static native long HMAC_CTX_new();\n\n    static native void HMAC_CTX_free(long ctx);\n\n    static native void HMAC_Init_ex(NativeRef.HMAC_CTX ctx, byte[] key, long evp_md);\n\n    static native void HMAC_Update(NativeRef.HMAC_CTX ctx, byte[] in, int inOffset, int inLength);\n\n    static native void HMAC_UpdateDirect(NativeRef.HMAC_CTX ctx, long inPtr, int inLength);\n\n    static native byte[] HMAC_Final(NativeRef.HMAC_CTX ctx);\n\n    static native void HMAC_Reset(NativeRef.HMAC_CTX ctx);\n\n    // --- HPKE functions ------------------------------------------------------\n    static native byte[] EVP_HPKE_CTX_export(\n            NativeRef.EVP_HPKE_CTX ctx, byte[] exporterCtx, int length);\n\n    static native void EVP_HPKE_CTX_free(long ctx);\n\n    static native byte[] EVP_HPKE_CTX_open(\n            NativeRef.EVP_HPKE_CTX ctx, byte[] ciphertext, byte[] aad) throws BadPaddingException;\n\n    static native byte[] EVP_HPKE_CTX_seal(\n            NativeRef.EVP_HPKE_CTX ctx, byte[] plaintext, byte[] aad);\n\n    static native Object EVP_HPKE_CTX_setup_base_mode_recipient(\n            int kem, int kdf, int aead, byte[] privateKey, byte[] enc, byte[] info);\n\n    static Object EVP_HPKE_CTX_setup_base_mode_recipient(\n            HpkeSuite suite, byte[] privateKey, byte[] enc, byte[] info) {\n        return EVP_HPKE_CTX_setup_base_mode_recipient(\n                suite.getKem().getId(), suite.getKdf().getId(), suite.getAead().getId(),\n                privateKey, enc, info);\n    }\n\n    static native Object[] EVP_HPKE_CTX_setup_base_mode_sender(\n            int kem, int kdf, int aead, byte[] publicKey, byte[] info);\n\n    static Object[] EVP_HPKE_CTX_setup_base_mode_sender(\n            HpkeSuite suite, byte[] publicKey, byte[] info) {\n        return EVP_HPKE_CTX_setup_base_mode_sender(\n                suite.getKem().getId(), suite.getKdf().getId(), suite.getAead().getId(),\n                publicKey, info);\n    }\n    static native Object[] EVP_HPKE_CTX_setup_base_mode_sender_with_seed_for_testing(\n            int kem, int kdf, int aead, byte[] publicKey, byte[] info, byte[] seed);\n\n    static Object[] EVP_HPKE_CTX_setup_base_mode_sender_with_seed_for_testing(\n            HpkeSuite suite, byte[] publicKey, byte[] info, byte[] seed) {\n        return EVP_HPKE_CTX_setup_base_mode_sender_with_seed_for_testing(\n                suite.getKem().getId(), suite.getKdf().getId(), suite.getAead().getId(),\n                publicKey, info, seed);\n    }\n\n    // --- RAND ----------------------------------------------------------------\n\n    static native void RAND_bytes(byte[] output);\n\n    // --- X509_NAME -----------------------------------------------------------\n\n    static int X509_NAME_hash(X500Principal principal) {\n        return X509_NAME_hash(principal, \"SHA1\");\n    }\n\n    public static int X509_NAME_hash_old(X500Principal principal) {\n        return X509_NAME_hash(principal, \"MD5\");\n    }\n    private static int X509_NAME_hash(X500Principal principal, String algorithm) {\n        try {\n            byte[] digest = MessageDigest.getInstance(algorithm).digest(principal.getEncoded());\n            int offset = 0;\n            return (((digest[offset++] & 0xff) << 0) | ((digest[offset++] & 0xff) << 8)\n                    | ((digest[offset++] & 0xff) << 16) | ((digest[offset] & 0xff) << 24));\n        } catch (NoSuchAlgorithmException e) {\n            throw new AssertionError(e);\n        }\n    }\n\n    // --- X509 ----------------------------------------------------------------\n\n    /** Used to request get_X509_GENERAL_NAME_stack get the \"altname\" field. */\n    static final int GN_STACK_SUBJECT_ALT_NAME = 1;\n\n    /**\n     * Used to request get_X509_GENERAL_NAME_stack get the issuerAlternativeName\n     * extension.\n     */\n    static final int GN_STACK_ISSUER_ALT_NAME = 2;\n\n    /**\n     * Used to request only non-critical types in get_X509*_ext_oids.\n     */\n    static final int EXTENSION_TYPE_NON_CRITICAL = 0;\n\n    /**\n     * Used to request only critical types in get_X509*_ext_oids.\n     */\n    static final int EXTENSION_TYPE_CRITICAL = 1;\n\n    static native long d2i_X509_bio(long bioCtx);\n\n    static native long d2i_X509(byte[] encoded) throws ParsingException;\n\n    static native long PEM_read_bio_X509(long bioCtx);\n\n    static native byte[] i2d_X509(long x509ctx, OpenSSLX509Certificate holder);\n\n    /** Takes an X509 context not an X509_PUBKEY context. */\n    static native byte[] i2d_X509_PUBKEY(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native byte[] ASN1_seq_pack_X509(long[] x509CertRefs);\n\n    static native long[] ASN1_seq_unpack_X509_bio(long bioRef) throws ParsingException;\n\n    static native void X509_free(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native int X509_cmp(long x509ctx1, OpenSSLX509Certificate holder, long x509ctx2, OpenSSLX509Certificate holder2);\n\n    static native void X509_print_ex(long bioCtx, long x509ctx, OpenSSLX509Certificate holder, long nmflag, long certflag);\n\n    static native byte[] X509_get_issuer_name(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native byte[] X509_get_subject_name(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native String get_X509_sig_alg_oid(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native byte[] get_X509_sig_alg_parameter(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native boolean[] get_X509_issuerUID(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native boolean[] get_X509_subjectUID(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native long X509_get_pubkey(long x509ctx, OpenSSLX509Certificate holder)\n            throws NoSuchAlgorithmException, InvalidKeyException;\n\n    static native String get_X509_pubkey_oid(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native byte[] X509_get_ext_oid(long x509ctx, OpenSSLX509Certificate holder, String oid);\n\n    static native String[] get_X509_ext_oids(long x509ctx, OpenSSLX509Certificate holder, int critical);\n\n    static native Object[][] get_X509_GENERAL_NAME_stack(long x509ctx, OpenSSLX509Certificate holder, int type)\n            throws CertificateParsingException;\n\n    static native boolean[] get_X509_ex_kusage(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native String[] get_X509_ex_xkusage(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native int get_X509_ex_pathlen(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native long X509_get_notBefore(long x509ctx, OpenSSLX509Certificate holder)\n            throws ParsingException;\n\n    static native long X509_get_notAfter(long x509ctx, OpenSSLX509Certificate holder)\n            throws ParsingException;\n\n    static native long X509_get_version(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native byte[] X509_get_serialNumber(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native void X509_verify(long x509ctx, OpenSSLX509Certificate holder, NativeRef.EVP_PKEY pkeyCtx)\n            throws BadPaddingException;\n\n    static native byte[] get_X509_tbs_cert(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native byte[] get_X509_tbs_cert_without_ext(long x509ctx, OpenSSLX509Certificate holder, String oid);\n\n    static native byte[] get_X509_signature(long x509ctx, OpenSSLX509Certificate holder);\n\n    static native int get_X509_ex_flags(long x509ctx, OpenSSLX509Certificate holder);\n\n    // Used by Android platform TrustedCertificateStore.\n    @SuppressWarnings(\"unused\")\n    static native int X509_check_issued(long ctx, OpenSSLX509Certificate holder, long ctx2, OpenSSLX509Certificate holder2);\n\n    // --- PKCS7 ---------------------------------------------------------------\n\n    /** Used as the \"which\" field in d2i_PKCS7_bio and PEM_read_bio_PKCS7. */\n    static final int PKCS7_CERTS = 1;\n\n    /** Used as the \"which\" field in d2i_PKCS7_bio and PEM_read_bio_PKCS7. */\n    static final int PKCS7_CRLS = 2;\n\n    /** Returns an array of X509 or X509_CRL pointers. */\n    static native long[] d2i_PKCS7_bio(long bioCtx, int which) throws ParsingException;\n\n    /** Returns an array of X509 or X509_CRL pointers. */\n    static native byte[] i2d_PKCS7(long[] certs);\n\n    /** Returns an array of X509 or X509_CRL pointers. */\n    static native long[] PEM_read_bio_PKCS7(long bioCtx, int which);\n\n    // --- X509_CRL ------------------------------------------------------------\n\n    static native long d2i_X509_CRL_bio(long bioCtx);\n\n    static native long PEM_read_bio_X509_CRL(long bioCtx);\n\n    static native byte[] i2d_X509_CRL(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native void X509_CRL_free(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native void X509_CRL_print(long bioCtx, long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native String get_X509_CRL_sig_alg_oid(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native byte[] get_X509_CRL_sig_alg_parameter(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native byte[] X509_CRL_get_issuer_name(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    /** Returns X509_REVOKED reference that is not duplicated! */\n    static native long X509_CRL_get0_by_cert(long x509CrlCtx, OpenSSLX509CRL holder, long x509Ctx, OpenSSLX509Certificate holder2);\n\n    /** Returns X509_REVOKED reference that is not duplicated! */\n    static native long X509_CRL_get0_by_serial(long x509CrlCtx, OpenSSLX509CRL holder, byte[] serial);\n\n    /** Returns an array of X509_REVOKED that are owned by the caller. */\n    static native long[] X509_CRL_get_REVOKED(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native String[] get_X509_CRL_ext_oids(long x509Crlctx, OpenSSLX509CRL holder, int critical);\n\n    static native byte[] X509_CRL_get_ext_oid(long x509CrlCtx, OpenSSLX509CRL holder, String oid);\n\n    static native long X509_CRL_get_version(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native long X509_CRL_get_ext(long x509CrlCtx, OpenSSLX509CRL holder, String oid);\n\n    static native byte[] get_X509_CRL_signature(long x509ctx, OpenSSLX509CRL holder);\n\n    static native void X509_CRL_verify(long x509CrlCtx, OpenSSLX509CRL holder,\n        NativeRef.EVP_PKEY pkeyCtx) throws BadPaddingException, SignatureException,\n        NoSuchAlgorithmException, InvalidKeyException, IllegalBlockSizeException;\n\n    static native byte[] get_X509_CRL_crl_enc(long x509CrlCtx, OpenSSLX509CRL holder);\n\n    static native long X509_CRL_get_lastUpdate(long x509CrlCtx, OpenSSLX509CRL holder)\n            throws ParsingException;\n\n    static native long X509_CRL_get_nextUpdate(long x509CrlCtx, OpenSSLX509CRL holder)\n            throws ParsingException;\n\n    // --- X509_REVOKED --------------------------------------------------------\n\n    static native long X509_REVOKED_dup(long x509RevokedCtx);\n\n    static native byte[] i2d_X509_REVOKED(long x509RevokedCtx);\n\n    static native String[] get_X509_REVOKED_ext_oids(long x509ctx, int critical);\n\n    static native byte[] X509_REVOKED_get_ext_oid(long x509RevokedCtx, String oid);\n\n    static native byte[] X509_REVOKED_get_serialNumber(long x509RevokedCtx);\n\n    static native long X509_REVOKED_get_ext(long x509RevokedCtx, String oid);\n\n    /** Returns ASN1_TIME reference. */\n    static native long get_X509_REVOKED_revocationDate(long x509RevokedCtx);\n\n    static native void X509_REVOKED_print(long bioRef, long x509RevokedCtx);\n\n    // --- X509_EXTENSION ------------------------------------------------------\n\n    static native int X509_supported_extension(long x509ExtensionRef);\n\n    // --- ASN1 Encoding -------------------------------------------------------\n\n    /**\n     * Allocates and returns an opaque reference to an object that can be used with other\n     * asn1_read_* functions to read the ASN.1-encoded data in val.  The returned object must\n     * be freed after use by calling asn1_read_free.\n     */\n    static native long asn1_read_init(byte[] val) throws IOException;\n\n    /**\n     * Allocates and returns an opaque reference to an object that can be used with other\n     * asn1_read_* functions to read the ASN.1 sequence pointed to by cbsRef.  The returned\n     * object must be freed after use by calling asn1_read_free.\n     */\n    static native long asn1_read_sequence(long cbsRef) throws IOException;\n\n    /**\n     * Returns whether the next object in the given reference is explicitly tagged with the\n     * given tag number.\n     */\n    static native boolean asn1_read_next_tag_is(long cbsRef, int tag) throws IOException;\n\n    /**\n     * Allocates and returns an opaque reference to an object that can be used with\n     * other asn1_read_* functions to read the ASN.1 data pointed to by cbsRef.  The returned\n     * object must be freed after use by calling asn1_read_free.\n     */\n    static native long asn1_read_tagged(long cbsRef) throws IOException;\n\n    /**\n     * Returns the contents of an ASN.1 octet string from the given reference.\n     */\n    static native byte[] asn1_read_octetstring(long cbsRef) throws IOException;\n\n    /**\n     * Returns an ASN.1 integer from the given reference.  If the integer doesn't fit\n     * in a uint64, this method will throw an IOException.\n     */\n    static native long asn1_read_uint64(long cbsRef) throws IOException;\n\n    /**\n     * Consumes an ASN.1 NULL from the given reference.\n     */\n    static native void asn1_read_null(long cbsRef) throws IOException;\n\n    /**\n     * Returns an ASN.1 OID in dotted-decimal notation (eg, \"1.3.14.3.2.26\" for SHA-1) from the\n     * given reference.\n     */\n    static native String asn1_read_oid(long cbsRef) throws IOException;\n\n    /**\n     * Returns whether or not the given reference has been read completely.\n     */\n    static native boolean asn1_read_is_empty(long cbsRef);\n\n    /**\n     * Frees any resources associated with the given reference.  After calling, the reference\n     * must not be used again.  This may be called with a zero reference, in which case nothing\n     * will be done.\n     */\n    \nstatic native void asn1_read_free(long cbsRef);\n\n    /**\n     * Allocates and returns an opaque reference to an object that can be used with other\n     * asn1_write_* functions to write ASN.1-encoded data.  The returned object must be finalized\n     * after use by calling either asn1_write_finish or asn1_write_cleanup, and its resources\n     * must be freed by calling asn1_write_free.\n     */\n    static native long asn1_write_init() throws IOException;\n\n    /**\n     * Allocates and returns an opaque reference to an object that can be used with other\n     * asn1_write_* functions to write an ASN.1 sequence into the given reference.  The returned\n     * reference may only be used until the next call on the parent reference.  The returned\n     * object must be freed after use by calling asn1_write_free.\n     */\n    static native long asn1_write_sequence(long cbbRef) throws IOException;\n\n    /**\n     * Allocates and returns an opaque reference to an object that can be used with other\n     * asn1_write_* functions to write a explicitly-tagged ASN.1 object with the given tag\n     * into the given reference. The returned reference may only be used until the next\n     * call on the parent reference.  The returned object must be freed after use by\n     * calling asn1_write_free.\n     */\n    static native long asn1_write_tag(long cbbRef, int tag) throws IOException;\n\n    /**\n     * Writes the given data into the given reference as an ASN.1-encoded octet string.\n     */\n    static native void asn1_write_octetstring(long cbbRef, byte[] data) throws IOException;\n\n    /**\n     * Writes the given value into the given reference as an ASN.1-encoded integer.\n     */\n    static native void asn1_write_uint64(long cbbRef, long value) throws IOException;\n\n    /**\n     * Writes a NULL value into the given reference.\n     */\n    static native void asn1_write_null(long cbbRef) throws IOException;\n\n    /**\n     * Writes the given OID (which must be in dotted-decimal notation) into the given reference.\n     */\n    static native void asn1_write_oid(long cbbRef, String oid) throws IOException;\n\n    /**\n     * Flushes the given reference, invalidating any child references and completing their\n     * operations.  This must be called if the child references are to be freed before\n     * asn1_write_finish is called on the ultimate parent.  The child references must still\n     * be freed.\n     */\n    static native void asn1_write_flush(long cbbRef) throws IOException;\n\n    /**\n     * Completes any in-progress operations and returns the ASN.1-encoded data.  Either this\n     * or asn1_write_cleanup must be called on any reference returned from asn1_write_init\n     * before it is freed.\n     */\n    static native byte[] asn1_write_finish(long cbbRef) throws IOException;\n\n    /**\n     * Cleans up intermediate state in the given reference.  Either this or asn1_write_finish\n     * must be called on any reference returned from asn1_write_init before it is freed.\n     */\n    static native void asn1_write_cleanup(long cbbRef);\n\n    /**\n     * Frees resources associated with the given reference.  After calling, the reference\n     * must not be used again.  This may be called with a zero reference, in which case nothing\n     * will be done.\n     */\n    static native void asn1_write_free(long cbbRef);\n\n    // --- BIO stream creation -------------------------------------------------\n\n    static native long create_BIO_InputStream(OpenSSLBIOInputStream is, boolean isFinite);\n\n    static native long create_BIO_OutputStream(OutputStream os);\n\n    static native void BIO_free_all(long bioRef);\n\n    // --- SSL handling --------------------------------------------------------\n\n    static final String OBSOLETE_PROTOCOL_SSLV3 = \"SSLv3\";\n    private static final String DEPRECATED_PROTOCOL_TLSV1 = \"TLSv1\";\n    private static final String DEPRECATED_PROTOCOL_TLSV1_1 = \"TLSv1.1\";\n    private static final String SUPPORTED_PROTOCOL_TLSV1_2 = \"TLSv1.2\";\n    static final String SUPPORTED_PROTOCOL_TLSV1_3 = \"TLSv1.3\";\n\n    static final String[] SUPPORTED_TLS_1_3_CIPHER_SUITES = new String[] {\n            \"TLS_AES_128_GCM_SHA256\",\n            \"TLS_AES_256_GCM_SHA384\",\n            \"TLS_CHACHA20_POLY1305_SHA256\",\n    };\n\n    // SUPPORTED_TLS_1_2_CIPHER_SUITES_SET contains all the supported cipher suites for TLS 1.2,\n    // using their Java names.\n    static final Set<String> SUPPORTED_TLS_1_2_CIPHER_SUITES_SET = new HashSet<String>();\n\n    // SUPPORTED_LEGACY_CIPHER_SUITES_SET contains all the supported cipher suites using the legacy\n    // OpenSSL-style names.\n    private static final Set<String> SUPPORTED_LEGACY_CIPHER_SUITES_SET = new HashSet<String>();\n\n    static final Set<String> SUPPORTED_TLS_1_3_CIPHER_SUITES_SET = new HashSet<String>(\n            Arrays.asList(SUPPORTED_TLS_1_3_CIPHER_SUITES));\n\n    /**\n     * TLS_EMPTY_RENEGOTIATION_INFO_SCSV is RFC 5746's renegotiation\n     * indication signaling cipher suite value. It is not a real\n     * cipher suite. It is just an indication in the default and\n     * supported cipher suite lists indicates that the implementation\n     * supports secure renegotiation.\n     * <p>\n     * In the RI, its presence means that the SCSV is sent in the\n     * cipher suite list to indicate secure renegotiation support and\n     * its absense means to send an empty TLS renegotiation info\n     * extension instead.\n     * <p>\n     * However, OpenSSL doesn't provide an API to give this level of\n     * control, instead always sending the SCSV and always including\n     * the empty renegotiation info if TLS is used (as opposed to\n     * SSL). So we simply allow TLS_EMPTY_RENEGOTIATION_INFO_SCSV to\n     * be passed for compatibility as to provide the hint that we\n     * support secure renegotiation.\n     */\n    static final String TLS_EMPTY_RENEGOTIATION_INFO_SCSV = \"TLS_EMPTY_RENEGOTIATION_INFO_SCSV\";\n\n    static String cipherSuiteToJava(String cipherSuite) {\n        // For historical reasons, Java uses a different name for TLS_RSA_WITH_3DES_EDE_CBC_SHA.\n        if (\"TLS_RSA_WITH_3DES_EDE_CBC_SHA\".equals(cipherSuite)) {\n            return \"SSL_RSA_WITH_3DES_EDE_CBC_SHA\";\n        }\n        return cipherSuite;\n    }\n\n    static String cipherSuiteFromJava(String javaCipherSuite) {\n        if (\"SSL_RSA_WITH_3DES_EDE_CBC_SHA\".equals(javaCipherSuite)) {\n            return \"TLS_RSA_WITH_3DES_EDE_CBC_SHA\";\n        }\n        return javaCipherSuite;\n    }\n\n    /**\n     * TLS_FALLBACK_SCSV is from\n     * https://tools.ietf.org/html/draft-ietf-tls-downgrade-scsv-00\n     * to indicate to the server that this is a fallback protocol\n     * request.\n     */\n    private static final String TLS_FALLBACK_SCSV = \"TLS_FALLBACK_SCSV\";\n\n    private static final boolean HAS_AES_HARDWARE;\n    private static final String[] SUPPORTED_TLS_1_2_CIPHER_SUITES;\n    static {\n        if (loadError == null) {\n            // If loadError is not null, it means the native code was not loaded, so\n            // get_cipher_names will throw UnsatisfiedLinkError. Populate the list of supported\n            // ciphers with BoringSSL's default, and also explicitly include 3DES.\n            // https://boringssl-review.googlesource.com/c/boringssl/+/59425 will remove 3DES\n            // from BoringSSL's default, but Conscrypt isn't quite ready to remove it yet.\n            String[] allCipherSuites = get_cipher_names(\"ALL:3DES\");\n\n            // get_cipher_names returns an array where even indices are the standard name and odd\n            // indices are the OpenSSL name.\n            int size = allCipherSuites.length;\n            if (size % 2 != 0) {\n                throw new IllegalArgumentException(\n                        \"Invalid cipher list returned by get_cipher_names\");\n            }\n            SUPPORTED_TLS_1_2_CIPHER_SUITES = new String[size / 2 + 2];\n            for (int i = 0; i < size; i += 2) {\n                String cipherSuite = cipherSuiteToJava(allCipherSuites[i]);\n                SUPPORTED_TLS_1_2_CIPHER_SUITES[i / 2] = cipherSuite;\n                SUPPORTED_TLS_1_2_CIPHER_SUITES_SET.add(cipherSuite);\n\n                SUPPORTED_LEGACY_CIPHER_SUITES_SET.add(allCipherSuites[i + 1]);\n            }\n            SUPPORTED_TLS_1_2_CIPHER_SUITES[size / 2] = TLS_EMPTY_RENEGOTIATION_INFO_SCSV;\n            SUPPORTED_TLS_1_2_CIPHER_SUITES[size / 2 + 1] = TLS_FALLBACK_SCSV;\n\n            HAS_AES_HARDWARE = EVP_has_aes_hardware() == 1;\n        } else {\n            HAS_AES_HARDWARE = false;\n            SUPPORTED_TLS_1_2_CIPHER_SUITES = new String[0];\n        }\n    }\n\n    /**\n     * Returns 1 if the BoringSSL believes the CPU has AES accelerated hardware\n     * instructions. Used to determine cipher suite ordering.\n     */\n    static native int EVP_has_aes_hardware();\n\n    static native long SSL_CTX_new();\n\n    // IMPLEMENTATION NOTE: The default list of cipher suites is a trade-off between what we'd like\n    // to use and what servers currently support. We strive to be secure enough by default. We thus\n    // avoid unacceptably weak suites (e.g., those with bulk cipher secret key shorter than 128\n    // bits), while maintaining the capability to connect to the majority of servers.\n    //\n    // Cipher suites are listed in preference order (favorite choice first) of the client. However,\n    // servers are not required to honor the order. The key rules governing the preference order\n    // are:\n    // * Prefer Forward Secrecy (i.e., cipher suites that use ECDHE and DHE for key agreement).\n    // * Prefer ChaCha20-Poly1305 to AES-GCM unless hardware support for AES is available.\n    // * Prefer AES-GCM to AES-CBC whose MAC-pad-then-encrypt approach leads to weaknesses (e.g.,\n    //   Lucky 13).\n    // * Prefer 128-bit bulk encryption to 256-bit one, because 128-bit is safe enough while\n    //   consuming less CPU/time/energy.\n    //\n    // NOTE: Removing cipher suites from this list needs to be done with caution, because this may\n    // prevent apps from connecting to servers they were previously able to connect to.\n\n    /** X.509 based cipher suites enabled by default (if requested), in preference order. */\n    static final String[] DEFAULT_X509_CIPHER_SUITES = HAS_AES_HARDWARE ?\n            new String[] {\n                    \"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\",\n                    \"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\",\n                    \"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\",\n                    \"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\",\n                    \"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\n                    \"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\",\n                    \"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\",\n                    \"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\",\n                    \"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\",\n                    \"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\",\n                    \"TLS_RSA_WITH_AES_128_GCM_SHA256\",\n                    \"TLS_RSA_WITH_AES_256_GCM_SHA384\",\n                    \"TLS_RSA_WITH_AES_128_CBC_SHA\",\n                    \"TLS_RSA_WITH_AES_256_CBC_SHA\",\n            } :\n            new String[] {\n                    \"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\",\n                    \"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\",\n                    \"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\",\n                    \"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\",\n                    \"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\",\n                    \"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\n                    \"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\",\n                    \"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\",\n                    \"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\",\n                    \"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\",\n                    \"TLS_RSA_WITH_AES_128_GCM_SHA256\",\n                    \"TLS_RSA_WITH_AES_256_GCM_SHA384\",\n                    \"TLS_RSA_WITH_AES_128_CBC_SHA\",\n                    \"TLS_RSA_WITH_AES_256_CBC_SHA\",\n            };\n\n    /** TLS-PSK cipher suites enabled by default (if requested), in preference order. */\n    static final String[] DEFAULT_PSK_CIPHER_SUITES = new String[] {\n            \"TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256\",\n            \"TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA\",\n            \"TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA\",\n            \"TLS_PSK_WITH_AES_128_CBC_SHA\",\n            \"TLS_PSK_WITH_AES_256_CBC_SHA\",\n    };\n\n    static String[] getSupportedCipherSuites() {\n        return SSLUtils.concat(SUPPORTED_TLS_1_3_CIPHER_SUITES, SUPPORTED_TLS_1_2_CIPHER_SUITES.clone());\n    }\n\n    static native void SSL_CTX_free(long ssl_ctx, AbstractSessionContext holder);\n\n    static native void SSL_CTX_set_session_id_context(long ssl_ctx, AbstractSessionContext holder, byte[] sid_ctx);\n\n    static native long SSL_CTX_set_timeout(long ssl_ctx, AbstractSessionContext holder, long seconds);\n\n    static native long SSL_new(long ssl_ctx, AbstractSessionContext holder) throws SSLException;\n\n    static native void SSL_enable_tls_channel_id(long ssl, NativeSsl ssl_holder) throws SSLException;\n\n    static native byte[] SSL_get_tls_channel_id(long ssl, NativeSsl ssl_holder) throws SSLException;\n\n    static native void SSL_set1_tls_channel_id(long ssl, NativeSsl ssl_holder, NativeRef.EVP_PKEY pkey);\n\n    /**\n     * Sets the local certificates and private key.\n     *\n     * @param ssl the SSL reference.\n     * @param encodedCertificates the encoded form of the local certificate chain.\n     * @param pkey a reference to the private key.\n     * @throws SSLException if a problem occurs setting the cert/key.\n     */\n    static native void setLocalCertsAndPrivateKey(long ssl, NativeSsl ssl_holder, byte[][] encodedCertificates,\n        NativeRef.EVP_PKEY pkey) throws SSLException;\n\n    static native void SSL_set_client_CA_list(long ssl, NativeSsl ssl_holder, byte[][] asn1DerEncodedX500Principals)\n            throws SSLException;\n\n    static native long SSL_set_mode(long ssl, NativeSsl ssl_holder, long mode);\n\n    static native long SSL_set_options(long ssl, NativeSsl ssl_holder, long options);\n\n    static native long SSL_clear_options(long ssl, NativeSsl ssl_holder, long options);\n\n    static native int SSL_set_protocol_versions(long ssl, NativeSsl ssl_holder, int min_version, int max_version);\n\n    static native void SSL_enable_signed_cert_timestamps(long ssl, NativeSsl ssl_holder);\n\n    static native byte[] SSL_get_signed_cert_timestamp_list(long ssl, NativeSsl ssl_holder);\n\n    static native void SSL_set_signed_cert_timestamp_list(long ssl, NativeSsl ssl_holder, byte[] list);\n\n    static native void SSL_enable_ocsp_stapling(long ssl, NativeSsl ssl_holder);\n\n    static native byte[] SSL_get_ocsp_response(long ssl, NativeSsl ssl_holder);\n\n    static native void SSL_set_ocsp_response(long ssl, NativeSsl ssl_holder, byte[] response);\n\n    static native byte[] SSL_get_tls_unique(long ssl, NativeSsl ssl_holder);\n\n    static native byte[] SSL_export_keying_material(long ssl, NativeSsl ssl_holder, byte[] label, byte[] context, int num_bytes) throws SSLException;\n\n    static native void SSL_use_psk_identity_hint(long ssl, NativeSsl ssl_holder, String identityHint) throws SSLException;\n\n    static native void set_SSL_psk_client_callback_enabled(long ssl, NativeSsl ssl_holder, boolean enabled);\n\n    static native void set_SSL_psk_server_callback_enabled(long ssl, NativeSsl ssl_holder, boolean enabled);\n\n    private static final String[] ENABLED_PROTOCOLS_TLSV1 = Platform.isTlsV1Deprecated()\n            ? new String[0]\n            : new String[] {\n                DEPRECATED_PROTOCOL_TLSV1,\n                DEPRECATED_PROTOCOL_TLSV1_1,\n            };\n\n\n    /** Protocols to enable by default when \"TLSv1.3\" is requested. */\n    static final String[] TLSV13_PROTOCOLS = ArrayUtils.concatValues(\n            ENABLED_PROTOCOLS_TLSV1,\n            SUPPORTED_PROTOCOL_TLSV1_2,\n            SUPPORTED_PROTOCOL_TLSV1_3);\n\n    /** Protocols to enable by default when \"TLSv1.2\" is requested. */\n    static final String[] TLSV12_PROTOCOLS = ArrayUtils.concatValues(\n            ENABLED_PROTOCOLS_TLSV1,\n            SUPPORTED_PROTOCOL_TLSV1_2);\n\n    /** Protocols to enable by default when \"TLSv1.1\" is requested. */\n    static final String[] TLSV11_PROTOCOLS = new String[] {\n            DEPRECATED_PROTOCOL_TLSV1,\n            DEPRECATED_PROTOCOL_TLSV1_1,\n            SUPPORTED_PROTOCOL_TLSV1_2,\n    };\n\n    /** Protocols to enable by default when \"TLSv1\" is requested. */\n    static final String[] TLSV1_PROTOCOLS = TLSV11_PROTOCOLS;\n\n    static final String[] DEFAULT_PROTOCOLS = TLSV13_PROTOCOLS;\n    private static final String[] SUPPORTED_PROTOCOLS = new String[] {\n            DEPRECATED_PROTOCOL_TLSV1,\n            DEPRECATED_PROTOCOL_TLSV1_1,\n            SUPPORTED_PROTOCOL_TLSV1_2,\n            SUPPORTED_PROTOCOL_TLSV1_3,\n    };\n\n    public static String[] getDefaultProtocols() {\n        if (Platform.isTlsV1Deprecated()) {\n          return DEFAULT_PROTOCOLS.clone();\n        }\n        return SUPPORTED_PROTOCOLS.clone();\n    }\n\n    static String[] getSupportedProtocols() {\n        return SUPPORTED_PROTOCOLS.clone();\n    }\n\n    private static class Range {\n        public final String min;\n        public final String max;\n        public Range(String min, String max) {\n            this.min = min;\n            this.max = max;\n        }\n    }\n\n    private static Range getProtocolRange(String[] protocols) {\n        // TLS protocol negotiation only allows a min and max version\n        // to be set, despite the Java API allowing a sparse set of\n        // protocols to be enabled.  Use the lowest contiguous range\n        // of protocols provided by the caller, which is what we've\n        // done historically.\n        List<String> protocolsList = Arrays.asList(protocols);\n        String min = null;\n        String max = null;\n        for (int i = 0; i < SUPPORTED_PROTOCOLS.length; i++) {\n            String protocol = SUPPORTED_PROTOCOLS[i];\n            if (protocolsList.contains(protocol)) {\n                if (min == null) {\n                    min = protocol;\n                }\n                max = protocol;\n            } else if (min != null) {\n                break;\n            }\n        }\n        if ((min == null) || (max == null)) {\n            throw new IllegalArgumentException(\"No protocols enabled.\");\n        }\n        return new Range(min, max);\n    }\n\n    static void setEnabledProtocols(long ssl, NativeSsl ssl_holder, String[] protocols) {\n        checkEnabledProtocols(protocols);\n        Range range = getProtocolRange(protocols);\n        SSL_set_protocol_versions(\n            ssl, ssl_holder, getProtocolConstant(range.min), getProtocolConstant(range.max));\n    }\n\n    private static int getProtocolConstant(String protocol) {\n        if (protocol.equals(DEPRECATED_PROTOCOL_TLSV1)) {\n            return NativeConstants.TLS1_VERSION;\n        } else if (protocol.equals(DEPRECATED_PROTOCOL_TLSV1_1)) {\n            return NativeConstants.TLS1_1_VERSION;\n        } else if (protocol.equals(SUPPORTED_PROTOCOL_TLSV1_2)) {\n            return NativeConstants.TLS1_2_VERSION;\n        } else if (protocol.equals(SUPPORTED_PROTOCOL_TLSV1_3)) {\n            return NativeConstants.TLS1_3_VERSION;\n        } else {\n            throw new AssertionError(\"Unknown protocol encountered: \" + protocol);\n        }\n    }\n\n    static String[] checkEnabledProtocols(String[] protocols) {\n        if (protocols == null) {\n            throw new IllegalArgumentException(\"protocols == null\");\n        }\n        for (String protocol : protocols) {\n            if (protocol == null) {\n                throw new IllegalArgumentException(\"protocols contains null\");\n            }\n            if (!protocol.equals(DEPRECATED_PROTOCOL_TLSV1)\n                    && !protocol.equals(DEPRECATED_PROTOCOL_TLSV1_1)\n                    && !protocol.equals(SUPPORTED_PROTOCOL_TLSV1_2)\n                    && !protocol.equals(SUPPORTED_PROTOCOL_TLSV1_3)\n                    && !protocol.equals(OBSOLETE_PROTOCOL_SSLV3)) {\n                throw new IllegalArgumentException(\"protocol \" + protocol + \" is not supported\");\n            }\n        }\n        return protocols;\n    }\n\n    static native void SSL_set_cipher_lists(long ssl, NativeSsl ssl_holder, String[] ciphers);\n\n    /**\n     * Gets the list of cipher suites enabled for the provided {@code SSL} instance.\n     *\n     * @return array of {@code SSL_CIPHER} references.\n     */\n    static native long[] SSL_get_ciphers(long ssl, NativeSsl ssl_holder);\n\n    static void setEnabledCipherSuites(long ssl, NativeSsl ssl_holder, String[] cipherSuites,\n            String[] protocols) {\n        checkEnabledCipherSuites(cipherSuites);\n        String maxProtocol = getProtocolRange(protocols).max;\n        List<String> opensslSuites = new ArrayList<String>();\n        for (int i = 0; i < cipherSuites.length; i++) {\n            String cipherSuite = cipherSuites[i];\n            if (cipherSuite.equals(TLS_EMPTY_RENEGOTIATION_INFO_SCSV)) {\n                continue;\n            }\n            // Only send TLS_FALLBACK_SCSV if max version >= 1.2 to prevent inadvertent connection\n            // problems when servers upgrade.  See https://github.com/google/conscrypt/issues/574\n            // for more discussion.\n            if (cipherSuite.equals(TLS_FALLBACK_SCSV)\n                    && (maxProtocol.equals(DEPRECATED_PROTOCOL_TLSV1)\n                        || maxProtocol.equals(DEPRECATED_PROTOCOL_TLSV1_1))) {\n                SSL_set_mode(ssl, ssl_holder, NativeConstants.SSL_MODE_SEND_FALLBACK_SCSV);\n                continue;\n            }\n            opensslSuites.add(cipherSuiteFromJava(cipherSuite));\n        }\n        SSL_set_cipher_lists(ssl, ssl_holder, opensslSuites.toArray(new String[opensslSuites.size()]));\n    }\n\n    static String[] checkEnabledCipherSuites(String[] cipherSuites) {\n        if (cipherSuites == null) {\n            throw new IllegalArgumentException(\"cipherSuites == null\");\n        }\n        // makes sure all suites are valid, throwing on error\n        for (int i = 0; i < cipherSuites.length; i++) {\n            if (cipherSuites[i] == null) {\n                throw new IllegalArgumentException(\"cipherSuites[\" + i + \"] == null\");\n            }\n            if (cipherSuites[i].equals(TLS_EMPTY_RENEGOTIATION_INFO_SCSV)\n                    || cipherSuites[i].equals(TLS_FALLBACK_SCSV)) {\n                continue;\n            }\n            if (SUPPORTED_TLS_1_2_CIPHER_SUITES_SET.contains(cipherSuites[i])) {\n                continue;\n            }\n\n            // For backwards compatibility, it's allowed for |cipherSuite| to\n            // be an OpenSSL-style cipher-suite name.\n            if (SUPPORTED_LEGACY_CIPHER_SUITES_SET.contains(cipherSuites[i])) {\n                // TODO log warning about using backward compatability\n                continue;\n            }\n            throw new IllegalArgumentException(\n                    \"cipherSuite \" + cipherSuites[i] + \" is not supported.\");\n        }\n        return cipherSuites;\n    }\n\n    static native void SSL_set_accept_state(long ssl, NativeSsl ssl_holder);\n\n    static native void SSL_set_connect_state(long ssl, NativeSsl ssl_holder);\n\n    static native void SSL_set_verify(long ssl, NativeSsl ssl_holder, int mode);\n\n    static native void SSL_set_session(long ssl, NativeSsl ssl_holder, long sslSessionNativePointer)\n            throws SSLException;\n\n    static native void SSL_set_session_creation_enabled(\n            long ssl, NativeSsl ssl_holder, boolean creationEnabled) throws SSLException;\n\n    static native boolean SSL_session_reused(long ssl, NativeSsl ssl_holder);\n\n    static native void SSL_accept_renegotiations(long ssl, NativeSsl ssl_holder) throws SSLException;\n\n    static native void SSL_set_tlsext_host_name(long ssl, NativeSsl ssl_holder, String hostname)\n            throws SSLException;\n    static native String SSL_get_servername(long ssl, NativeSsl ssl_holder);\n\n    static native void SSL_do_handshake(\n            long ssl, NativeSsl ssl_holder, FileDescriptor fd, SSLHandshakeCallbacks shc, int timeoutMillis)\n            throws SSLException, SocketTimeoutException, CertificateException;\n\n    public static native String SSL_get_current_cipher(long ssl, NativeSsl ssl_holder);\n\n    public static native String SSL_get_version(long ssl, NativeSsl ssl_holder);\n\n    /**\n     * Returns the peer certificate chain.\n     */\n    static native byte[][] SSL_get0_peer_certificates(long ssl, NativeSsl ssl_holder);\n\n    /**\n     * Reads with the native SSL_read function from the encrypted data stream\n     * @return -1 if error or the end of the stream is reached.\n     */\n    static native int SSL_read(long ssl, NativeSsl ssl_holder, FileDescriptor fd, SSLHandshakeCallbacks shc,\n            byte[] b, int off, int len, int readTimeoutMillis) throws IOException;\n\n    /**\n     * Writes with the native SSL_write function to the encrypted data stream.\n     */\n    static native void SSL_write(long ssl, NativeSsl ssl_holder, FileDescriptor fd,\n            SSLHandshakeCallbacks shc, byte[] b, int off, int len, int writeTimeoutMillis)\n            throws IOException;\n\n    static native void SSL_interrupt(long ssl, NativeSsl ssl_holder);\n    static native void SSL_shutdown(\n            long ssl, NativeSsl ssl_holder, FileDescriptor fd, SSLHandshakeCallbacks shc) throws IOException;\n\n    static native int SSL_get_shutdown(long ssl, NativeSsl ssl_holder);\n\n    static native void SSL_free(long ssl, NativeSsl ssl_holder);\n\n    static native long SSL_get_time(long ssl, NativeSsl ssl_holder);\n\n    static native long SSL_set_timeout(long ssl, NativeSsl ssl_holder, long millis);\n\n    static native long SSL_get_timeout(long ssl, NativeSsl ssl_holder);\n\n    static native int SSL_get_signature_algorithm_key_type(int signatureAlg);\n\n    static native byte[] SSL_session_id(long ssl, NativeSsl ssl_holder);\n\n    static native byte[] SSL_SESSION_session_id(long sslSessionNativePointer);\n\n    static native long SSL_SESSION_get_time(long sslSessionNativePointer);\n\n    static native long SSL_SESSION_get_timeout(long sslSessionNativePointer);\n\n    static native String SSL_SESSION_get_version(long sslSessionNativePointer);\n\n    static native String SSL_SESSION_cipher(long sslSessionNativePointer);\n\n    static native boolean SSL_SESSION_should_be_single_use(long sslSessionNativePointer);\n\n    static native void SSL_SESSION_up_ref(long sslSessionNativePointer);\n\n    static native void SSL_SESSION_free(long sslSessionNativePointer);\n\n    static native byte[] i2d_SSL_SESSION(long sslSessionNativePointer);\n\n    static native long d2i_SSL_SESSION(byte[] data) throws IOException;\n\n    /**\n     * A collection of callbacks from the native OpenSSL code that are\n     * related to the SSL handshake initiated by SSL_do_handshake.\n     */\n    interface SSLHandshakeCallbacks {\n        /**\n         * Verify that the certificate chain is trusted.\n         *\n         * @param certificateChain chain of X.509 certificates in their encoded form\n         * @param authMethod auth algorithm name\n         *\n         * @throws CertificateException if the certificate is untrusted\n         */\n        @SuppressWarnings(\"unused\")\n        void verifyCertificateChain(byte[][] certificateChain, String authMethod)\n                throws CertificateException;\n\n        /**\n         * Called on an SSL client when the server requests (or\n         * requires a certificate). The client can respond by using\n         * SSL_use_certificate and SSL_use_PrivateKey to set a\n         * certificate if has an appropriate one available, similar to\n         * how the server provides its certificate.\n         *\n         * @param keyTypes key types supported by the server,\n         * convertible to strings with #keyType\n         * @param asn1DerEncodedX500Principals CAs known to the server\n         */\n        @SuppressWarnings(\"unused\") void clientCertificateRequested(byte[] keyTypes, int[] signatureAlgs,\n...\n", "needle_token_start": 7170, "needle_token_end": 7186, "code_context_ntokens": 16079, "output": ["```java\nstatic native void asn1_read_free(long cbsRef);\n```"]}
{"repo": "google/conscrypt", "name": "engineGetServerSessionContext", "language": "java", "path": "common/src/main/java/org/conscrypt/OpenSSLContextImpl.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the session context used for managing server-side SSL sessions.\n2. **Input**: None.\n3. **Output**: Returns an instance of the server session context.\n4. **Procedure**: The method directly returns the instance of the server session context that is maintained within the SSL context implementation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/NativeCrypto.java\n/*\n * Copyright (C) 2008 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.FileDescriptor;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.net.SocketTimeoutException;\nimport java.nio.Buffer;\nimport java.nio.ByteBuffer;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.PrivateKey;\nimport java.security.SignatureException;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateParsingException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.crypto.ShortBufferException;\nimport javax.net.ssl.SSLException;\nimport javax.security.auth.x500.X500Principal;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * Provides the Java side of our JNI glue for OpenSSL.\n * <p>\n * Note: Many methods in this class take a reference to a Java object that holds a\n * native pointer in the form of a long in addition to the long itself and don't use\n * the Java object in the native implementation.  This is to prevent the Java object\n * from becoming eligible for GC while the native method is executing.  See\n * <a href=\"https://github.com/google/error-prone/blob/master/docs/bugpattern/UnsafeFinalization.md\">this</a>\n * for more details.\n */\n@Internal\npublic final class NativeCrypto {\n    // --- OpenSSL library initialization --------------------------------------\n    private static final UnsatisfiedLinkError loadError;\n    static {\n        UnsatisfiedLinkError error = null;\n        try {\n            NativeCryptoJni.init();\n            clinit();\n        } catch (UnsatisfiedLinkError t) {\n            // Don't rethrow the error, so that we can later on interrogate the\n            // value of loadError.\n            error = t;\n        }\n        loadError = error;\n    }\n\n    private native static void clinit();\n\n    /**\n     * Checks to see whether or not the native library was successfully loaded. If not, throws\n     * the {@link UnsatisfiedLinkError} that was encountered while attempting to load the library.\n     */\n    static void checkAvailability() {\n        if (loadError != null) {\n            throw loadError;\n        }\n    }\n\n    // --- DSA/RSA public/private key handling functions -----------------------\n\n    static native long EVP_PKEY_new_RSA(byte[] n, byte[] e, byte[] d, byte[] p, byte[] q,\n            byte[] dmp1, byte[] dmq1, byte[] iqmp);\n\n    static native int EVP_PKEY_type(NativeRef.EVP_PKEY pkey);\n\n    static native String EVP_PKEY_print_public(NativeRef.EVP_PKEY pkeyRef);\n\n    static native String EVP_PKEY_print_params(NativeRef.EVP_PKEY pkeyRef);\n\n    static native void EVP_PKEY_free(long pkey);\n\n    static native int EVP_PKEY_cmp(NativeRef.EVP_PKEY pkey1, NativeRef.EVP_PKEY pkey2);\n\n    static native byte[] EVP_marshal_private_key(NativeRef.EVP_PKEY pkey);\n\n    static native long EVP_parse_private_key(byte[] data) throws ParsingException;\n\n    static native byte[] EVP_marshal_public_key(NativeRef.EVP_PKEY pkey);\n\n    static native byte[] EVP_raw_X25519_private_key(byte[] data)\n            throws ParsingException, InvalidKeyException;\n\n    static native long EVP_parse_public_key(byte[] data) throws ParsingException;\n\n    static native long PEM_read_bio_PUBKEY(long bioCtx);\n\n    static native long PEM_read_bio_PrivateKey(long bioCtx);\n\n    static native long getRSAPrivateKeyWrapper(PrivateKey key, byte[] modulus);\n\n    static native long getECPrivateKeyWrapper(PrivateKey key, NativeRef.EC_GROUP ecGroupRef);\n\n    static native long RSA_generate_key_ex(int modulusBits, byte[] publicExponent);\n\n    static native int RSA_size(NativeRef.EVP_PKEY pkey);\n\n    static native int RSA_private_encrypt(\n            int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey, int padding);\n\n    static native int RSA_public_decrypt(int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey,\n            int padding) throws BadPaddingException, SignatureException;\n\n    static native int RSA_public_encrypt(\n            int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey, int padding);\n\n    static native int RSA_private_decrypt(int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey,\n            int padding) throws BadPaddingException, SignatureException;\n\n    /**\n     * @return array of {n, e}\n     */\n    static native byte[][] get_RSA_public_params(NativeRef.EVP_PKEY rsa);\n\n    /**\n     * @return array of {n, e, d, p, q, dmp1, dmq1, iqmp}\n     */\n    static native byte[][] get_RSA_private_params(NativeRef.EVP_PKEY rsa);\n\n    // --- ChaCha20 -----------------------\n\n    /**\n     * Returns the encrypted or decrypted version of the data.\n     */\n    static native void chacha20_encrypt_decrypt(byte[] in, int inOffset, byte[] out, int outOffset,\n            int length, byte[] key, byte[] nonce, int blockCounter);\n\n    // --- EC functions --------------------------\n\n    static native long EVP_PKEY_new_EC_KEY(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pubkeyRef, byte[] privkey);\n\n    static native long EC_GROUP_new_by_curve_name(String curveName);\n\n    static native long EC_GROUP_new_arbitrary(\n            byte[] p, byte[] a, byte[] b, byte[] x, byte[] y, byte[] order, int cofactor);\n\n    static native String EC_GROUP_get_curve_name(NativeRef.EC_GROUP groupRef);\n\n    static native byte[][] EC_GROUP_get_curve(NativeRef.EC_GROUP groupRef);\n\n    static native void EC_GROUP_clear_free(long groupRef);\n\n    static native long EC_GROUP_get_generator(NativeRef.EC_GROUP groupRef);\n\n    static native byte[] EC_GROUP_get_order(NativeRef.EC_GROUP groupRef);\n\n    static native int EC_GROUP_get_degree(NativeRef.EC_GROUP groupRef);\n\n    static native byte[] EC_GROUP_get_cofactor(NativeRef.EC_GROUP groupRef);\n\n    static native long EC_POINT_new(NativeRef.EC_GROUP groupRef);\n\n    static native void EC_POINT_clear_free(long pointRef);\n\n    static native byte[][] EC_POINT_get_affine_coordinates(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pointRef);\n\n    static native void EC_POINT_set_affine_coordinates(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pointRef, byte[] x, byte[] y);\n\n    static native long EC_KEY_generate_key(NativeRef.EC_GROUP groupRef);\n\n    static native long EC_KEY_get1_group(NativeRef.EVP_PKEY pkeyRef);\n\n    static native byte[] EC_KEY_get_private_key(NativeRef.EVP_PKEY keyRef);\n\n    static native long EC_KEY_get_public_key(NativeRef.EVP_PKEY keyRef);\n\n    static native byte[] EC_KEY_marshal_curve_name(NativeRef.EC_GROUP groupRef) throws IOException;\n\n    static native long EC_KEY_parse_curve_name(byte[] encoded) throws IOException;\n\n    static native int ECDH_compute_key(byte[] out, int outOffset, NativeRef.EVP_PKEY publicKeyRef,\n            NativeRef.EVP_PKEY privateKeyRef) throws InvalidKeyException, IndexOutOfBoundsException;\n\n    static native int ECDSA_size(NativeRef.EVP_PKEY pkey);\n\n    static native int ECDSA_sign(byte[] data, byte[] sig, NativeRef.EVP_PKEY pkey);\n\n    static native int ECDSA_verify(byte[] data, byte[] sig, NativeRef.EVP_PKEY pkey);\n\n    // --- Curve25519 --------------\n\n    static native boolean X25519(byte[] out, byte[] privateKey, byte[] publicKey) throws InvalidKeyException;\n\n    static native void X25519_keypair(byte[] outPublicKey, byte[] outPrivateKey);\n\n    // --- Message digest functions --------------\n\n    // These return const references\n    static native long EVP_get_digestbyname(String name);\n\n    static native int EVP_MD_size(long evp_md_const);\n\n    // --- Message digest context functions --------------\n\n    static native long EVP_MD_CTX_create();\n\n    static native void EVP_MD_CTX_cleanup(NativeRef.EVP_MD_CTX ctx);\n\n    static native void EVP_MD_CTX_destroy(long ctx);\n\n    static native int EVP_MD_CTX_copy_ex(\n            NativeRef.EVP_MD_CTX dst_ctx, NativeRef.EVP_MD_CTX src_ctx);\n\n    // --- Digest handling functions -------------------------------------------\n\n    static native int EVP_DigestInit_ex(NativeRef.EVP_MD_CTX ctx, long evp_md);\n\n    static native void EVP_DigestUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native int EVP_DigestFinal_ex(NativeRef.EVP_MD_CTX ctx, byte[] hash, int offset);\n\n    // --- Signature handling functions ----------------------------------------\n\n    static native long EVP_DigestSignInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native long EVP_DigestVerifyInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native void EVP_DigestSignUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestSignUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native void EVP_DigestVerifyUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestVerifyUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native byte[] EVP_DigestSignFinal(NativeRef.EVP_MD_CTX ctx);\n\n    static native boolean EVP_DigestVerifyFinal(NativeRef.EVP_MD_CTX ctx, byte[] signature,\n            int offset, int length) throws IndexOutOfBoundsException;\n\n    static native long EVP_PKEY_encrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_encrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native long EVP_PKEY_decrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_decrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native void EVP_PKEY_CTX_free(long pkeyCtx);\n\n    static native void EVP_PKEY_CTX_set_rsa_padding(long ctx, int pad)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_pss_saltlen(long ctx, int len)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_mgf1_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_label(long ctx, byte[] label)\n            throws InvalidAlgorithmParameterException;\n\n    // --- Block ciphers -------------------------------------------------------\n\n    // These return const references\n    static native long EVP_get_cipherbyname(String string);\n\n    static native void EVP_CipherInit_ex(NativeRef.EVP_CIPHER_CTX ctx, long evpCipher, byte[] key,\n            byte[] iv, boolean encrypting);\n\n    static native int EVP_CipherUpdate(NativeRef.EVP_CIPHER_CTX ctx, byte[] out, int outOffset,\n            byte[] in, int inOffset, int inLength) throws IndexOutOfBoundsException;\n\n...\n// Path: common/src/main/java/org/conscrypt/NativeRef.java\n/*\n * Copyright 2014 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\n/**\n * Used to hold onto native OpenSSL references and run finalization on those\n * objects. Individual types must subclass this and implement finalizer.\n */\nabstract class NativeRef {\n    final long address;\n\n    NativeRef(long address) {\n        if (address == 0) {\n            throw new NullPointerException(\"address == 0\");\n        }\n\n        this.address = address;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (!(o instanceof NativeRef)) {\n            return false;\n        }\n\n        return ((NativeRef) o).address == address;\n    }\n\n    @Override\n    public int hashCode() {\n        return (int) (address ^ (address >>> 32));\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\")\n    protected void finalize() throws Throwable {\n        try {\n            if (address != 0) {\n                doFree(address);\n            }\n        } finally {\n            super.finalize();\n        }\n    }\n\n    abstract void doFree(long context);\n\n    static final class CMAC_CTX extends NativeRef {\n        CMAC_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.CMAC_CTX_free(context);\n        }\n    }\n\n    static final class EC_GROUP extends NativeRef {\n        EC_GROUP(long ctx) {\n            super(ctx);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EC_GROUP_clear_free(context);\n        }\n    }\n\n    static final class EC_POINT extends NativeRef {\n        EC_POINT(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EC_POINT_clear_free(context);\n        }\n    }\n\n    static final class EVP_CIPHER_CTX extends NativeRef {\n        EVP_CIPHER_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_CIPHER_CTX_free(context);\n        }\n    }\n\n    static final class EVP_HPKE_CTX extends NativeRef {\n        EVP_HPKE_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_HPKE_CTX_free(context);\n        }\n    }\n\n    static final class EVP_MD_CTX extends NativeRef {\n        EVP_MD_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_MD_CTX_destroy(context);\n        }\n    }\n\n    static final class EVP_PKEY extends NativeRef {\n        EVP_PKEY(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_PKEY_free(context);\n        }\n    }\n\n    static final class EVP_PKEY_CTX extends NativeRef {\n        EVP_PKEY_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_PKEY_CTX_free(context);\n        }\n    }\n\n    static final class HMAC_CTX extends NativeRef {\n        HMAC_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.HMAC_CTX_free(context);\n        }\n    }\n\n    static final class SSL_SESSION extends NativeRef {\n        SSL_SESSION(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.SSL_SESSION_free(context);\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLEvpCipher.java\n/*\n * Copyright (C) 2019 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.crypto.ShortBufferException;\nimport javax.crypto.spec.IvParameterSpec;\nimport org.conscrypt.NativeRef.EVP_CIPHER_CTX;\n\n@Internal\npublic abstract class OpenSSLEvpCipher extends OpenSSLCipher {\n    /**\n     * Native pointer for the OpenSSL EVP_CIPHER context.\n     */\n    private final EVP_CIPHER_CTX cipherCtx = new EVP_CIPHER_CTX(\n            NativeCrypto.EVP_CIPHER_CTX_new());\n\n    /**\n     * Whether the cipher has processed any data yet. EVP_CIPHER doesn't\n     * like calling \"doFinal()\" in decryption mode without processing any\n     * updates.\n     */\n    private boolean calledUpdate;\n\n    /**\n     * The block size of the current mode.\n     */\n    private int modeBlockSize;\n\n    protected OpenSSLEvpCipher(Mode mode, Padding padding) {\n        super(mode, padding);\n    }\n\n    @Override\n    void engineInitInternal(byte[] encodedKey, AlgorithmParameterSpec params,\n            SecureRandom random) throws InvalidKeyException,\n        InvalidAlgorithmParameterException {\n        byte[] iv;\n        if (params instanceof IvParameterSpec) {\n            IvParameterSpec ivParams = (IvParameterSpec) params;\n            iv = ivParams.getIV();\n        } else {\n            iv = null;\n        }\n\n        final long cipherType = NativeCrypto.EVP_get_cipherbyname(getCipherName(\n                encodedKey.length, mode));\n        if (cipherType == 0) {\n            throw new InvalidAlgorithmParameterException(\"Cannot find name for key length = \"\n                    + (encodedKey.length * 8) + \" and mode = \" + mode);\n        }\n\n        final boolean encrypting = isEncrypting();\n\n        final int expectedIvLength = NativeCrypto.EVP_CIPHER_iv_length(cipherType);\n        if (iv == null && expectedIvLength != 0) {\n            if (!encrypting) {\n                throw new InvalidAlgorithmParameterException(\"IV must be specified in \" + mode\n                        + \" mode\");\n            }\n\n            iv = new byte[expectedIvLength];\n            if (random != null) {\n                random.nextBytes(iv);\n            } else {\n                NativeCrypto.RAND_bytes(iv);\n            }\n        } else if (expectedIvLength == 0 && iv != null) {\n            throw new InvalidAlgorithmParameterException(\"IV not used in \" + mode + \" mode\");\n        } else if (iv != null && iv.length != expectedIvLength) {\n            throw new InvalidAlgorithmParameterException(\"expected IV length of \"\n                    + expectedIvLength + \" but was \" + iv.length);\n        }\n\n        this.iv = iv;\n\n        if (supportsVariableSizeKey()) {\n            NativeCrypto.EVP_CipherInit_ex(cipherCtx, cipherType, null, null, encrypting);\n            NativeCrypto.EVP_CIPHER_CTX_set_key_length(cipherCtx, encodedKey.length);\n            NativeCrypto.EVP_CipherInit_ex(cipherCtx, 0, encodedKey, iv, isEncrypting());\n        } else {\n            NativeCrypto.EVP_CipherInit_ex(cipherCtx, cipherType, encodedKey, iv, encrypting);\n        }\n\n        // OpenSSL only supports PKCS5 Padding.\n        NativeCrypto\n                .EVP_CIPHER_CTX_set_padding(cipherCtx, getPadding() == Padding.PKCS5PADDING);\n        modeBlockSize = NativeCrypto.EVP_CIPHER_CTX_block_size(cipherCtx);\n        calledUpdate = false;\n    }\n\n    @Override\n    int updateInternal(byte[] input, int inputOffset, int inputLen, byte[] output,\n            int outputOffset, int maximumLen) throws ShortBufferException {\n        final int intialOutputOffset = outputOffset;\n\n        final int bytesLeft = output.length - outputOffset;\n        if (bytesLeft < maximumLen) {\n            throw new ShortBufferWithoutStackTraceException(\"output buffer too small during update: \"\n                    + bytesLeft + \" < \" + maximumLen);\n        }\n\n        outputOffset += NativeCrypto.EVP_CipherUpdate(cipherCtx, output, outputOffset, input,\n                inputOffset, inputLen);\n\n        calledUpdate = true;\n\n        return outputOffset - intialOutputOffset;\n    }\n\n    @Override\n    int doFinalInternal(byte[] output, int outputOffset, int maximumLen)\n            throws IllegalBlockSizeException, BadPaddingException, ShortBufferException {\n        /* Remember this so we can tell how many characters were written. */\n        final int initialOutputOffset = outputOffset;\n\n        /*\n         * If we're decrypting and haven't had any input, we should return\n         * null. Otherwise OpenSSL will complain if we call final.\n         */\n        if (!isEncrypting() && !calledUpdate) {\n            return 0;\n        }\n\n        /* Allow OpenSSL to pad if necessary and clean up state. */\n        final int bytesLeft = output.length - outputOffset;\n        final int writtenBytes;\n        if (bytesLeft >= maximumLen) {\n            writtenBytes = NativeCrypto.EVP_CipherFinal_ex(cipherCtx, output, outputOffset);\n        } else {\n            final byte[] lastBlock = new byte[maximumLen];\n            writtenBytes = NativeCrypto.EVP_CipherFinal_ex(cipherCtx, lastBlock, 0);\n            if (writtenBytes > bytesLeft) {\n                throw new ShortBufferWithoutStackTraceException(\"buffer is too short: \" + writtenBytes + \" > \"\n                        + bytesLeft);\n            } else if (writtenBytes > 0) {\n                System.arraycopy(lastBlock, 0, output, outputOffset, writtenBytes);\n            }\n        }\n        outputOffset += writtenBytes;\n\n        reset();\n\n        return outputOffset - initialOutputOffset;\n    }\n\n    @Override\n    int getOutputSizeForFinal(int inputLen) {\n        if (modeBlockSize == 1) {\n            return inputLen;\n        } else {\n            final int buffered = NativeCrypto.get_EVP_CIPHER_CTX_buf_len(cipherCtx);\n\n            if (getPadding() == Padding.NOPADDING) {\n                return buffered + inputLen;\n            } else {\n                final boolean finalUsed = NativeCrypto.get_EVP_CIPHER_CTX_final_used(cipherCtx);\n                // There is an additional buffer containing the possible final block.\n                int totalLen = inputLen + buffered + (finalUsed ? modeBlockSize : 0);\n                // Extra block for remainder bytes plus padding.\n                // In case it's encrypting and there are no remainder bytes, add an extra block\n                // consisting only of padding.\n                totalLen += ((totalLen % modeBlockSize != 0) || isEncrypting())\n                        ? modeBlockSize : 0;\n                // The minimum multiple of {@code modeBlockSize} that can hold all the bytes.\n                return totalLen - (totalLen % modeBlockSize);\n            }\n        }\n    }\n\n    @Override\n    int getOutputSizeForUpdate(int inputLen) {\n        return getOutputSizeForFinal(inputLen);\n    }\n\n    /**\n     * Returns the OpenSSL cipher name for the particular {@code keySize}\n     * and cipher {@code mode}.\n     */\n    abstract String getCipherName(int keySize, Mode mode);\n\n    /**\n     * Reset this Cipher instance state to process a new chunk of data.\n     */\n    private void reset() {\n        NativeCrypto.EVP_CipherInit_ex(cipherCtx, 0, encodedKey, iv, isEncrypting());\n        calledUpdate = false;\n    }\n\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLContextImpl.java\n/*\n * Copyright (C) 2010 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport static org.conscrypt.Platform.wrapEngine;\n\nimport java.io.IOException;\nimport java.security.GeneralSecurityException;\nimport java.security.KeyManagementException;\nimport java.security.SecureRandom;\nimport javax.net.ssl.KeyManager;\nimport javax.net.ssl.SSLContextSpi;\nimport javax.net.ssl.SSLEngine;\nimport javax.net.ssl.SSLServerSocketFactory;\nimport javax.net.ssl.SSLSessionContext;\nimport javax.net.ssl.SSLSocketFactory;\nimport javax.net.ssl.TrustManager;\n\n/**\n * OpenSSL-backed SSLContext service provider interface.\n *\n * <p>Public to allow contruction via the provider framework.\n */\n@Internal\npublic abstract class OpenSSLContextImpl extends SSLContextSpi {\n    /**\n     * The default SSLContextImpl for use with\n     * SSLContext.getInstance(\"Default\"). Protected by the\n     * DefaultSSLContextImpl.class monitor.\n     */\n    private static DefaultSSLContextImpl defaultSslContextImpl;\n\n    /** TLS protocols to enable by default. */\n    private final String[] protocols;\n\n    /** Client session cache. */\n    private final ClientSessionContext clientSessionContext;\n\n    /** Server session cache. */\n    private final ServerSessionContext serverSessionContext;\n\n    SSLParametersImpl sslParameters;\n\n    /** Allows outside callers to get the preferred SSLContext. */\n    static OpenSSLContextImpl getPreferred() {\n        return new TLSv13();\n    }\n\n    OpenSSLContextImpl(String[] protocols) {\n        this.protocols = protocols;\n        clientSessionContext = new ClientSessionContext();\n        serverSessionContext = new ServerSessionContext();\n    }\n\n    /**\n     * Constuctor for the DefaultSSLContextImpl.  The unused boolean parameter is solely to\n     * indicate that this constructor is desired.\n     */\n    @SuppressWarnings(\"StaticAssignmentInConstructor\")\n    OpenSSLContextImpl(String[] protocols, boolean unused)\n            throws GeneralSecurityException, IOException {\n        synchronized (DefaultSSLContextImpl.class) {\n            this.protocols = null;\n            // This is the only place defaultSslContextImpl is read or written so all\n            // access is actually synchronized.\n            if (defaultSslContextImpl == null) {\n                clientSessionContext = new ClientSessionContext();\n                serverSessionContext = new ServerSessionContext();\n                defaultSslContextImpl = (DefaultSSLContextImpl) this;\n            } else {\n                clientSessionContext =\n                    (ClientSessionContext) defaultSslContextImpl.engineGetClientSessionContext();\n                serverSessionContext =\n                    (ServerSessionContext) defaultSslContextImpl.engineGetServerSessionContext();\n            }\n            sslParameters = new SSLParametersImpl(defaultSslContextImpl.getKeyManagers(),\n                    defaultSslContextImpl.getTrustManagers(), null, clientSessionContext,\n                    serverSessionContext, protocols);\n        }\n    }\n\n    /**\n     * Initializes this {@code SSLContext} instance. All of the arguments are\n     * optional, and the security providers will be searched for the required\n     * implementations of the needed algorithms.\n     *\n     * @param kms the key sources or {@code null}\n     * @param tms the trust decision sources or {@code null}\n     * @param sr the randomness source or {@code null}\n     * @throws KeyManagementException if initializing this instance fails\n     */\n    @Override\n    public void engineInit(KeyManager[] kms, TrustManager[] tms, SecureRandom sr)\n            throws KeyManagementException {\n        sslParameters = new SSLParametersImpl(\n                kms, tms, sr, clientSessionContext, serverSessionContext, protocols);\n    }\n\n    @Override\n    public SSLSocketFactory engineGetSocketFactory() {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        return Platform.wrapSocketFactoryIfNeeded(new OpenSSLSocketFactoryImpl(sslParameters));\n    }\n\n    @Override\n    public SSLServerSocketFactory engineGetServerSocketFactory() {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        return new OpenSSLServerSocketFactoryImpl(sslParameters);\n    }\n\n    @Override\n    public SSLEngine engineCreateSSLEngine(String host, int port) {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        SSLParametersImpl p = (SSLParametersImpl) sslParameters.clone();\n        p.setUseClientMode(false);\n        return wrapEngine(new ConscryptEngine(host, port, p));\n    }\n\n    @Override\n    public SSLEngine engineCreateSSLEngine() {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        SSLParametersImpl p = (SSLParametersImpl) sslParameters.clone();\n        p.setUseClientMode(false);\n        return wrapEngine(new ConscryptEngine(p));\n    }\n\n    \n@Override\n    public SSLSessionContext engineGetServerSessionContext() {\n        return serverSessionContext;\n    }\n\n    @Override\n    public SSLSessionContext engineGetClientSessionContext() {\n        return clientSessionContext;\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv13 extends OpenSSLContextImpl {\n        public TLSv13() {\n            super(NativeCrypto.TLSV13_PROTOCOLS);\n        }\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv12 extends OpenSSLContextImpl {\n        public TLSv12() {\n            super(NativeCrypto.TLSV12_PROTOCOLS);\n        }\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv11 extends OpenSSLContextImpl {\n        public TLSv11() {\n            super(NativeCrypto.TLSV11_PROTOCOLS);\n        }\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv1 extends OpenSSLContextImpl {\n        public TLSv1() {\n            super(NativeCrypto.TLSV1_PROTOCOLS);\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/SessionSnapshot.java\n/*\n * Copyright (C) 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.Principal;\nimport java.security.cert.Certificate;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.List;\nimport javax.net.ssl.SSLPeerUnverifiedException;\nimport javax.net.ssl.SSLSessionContext;\n\n/**\n * A snapshot of the content of another {@link ConscryptSession}. This copies everything over\n * except for the certificates.\n */\nfinal class SessionSnapshot implements ConscryptSession {\n    private final SSLSessionContext sessionContext;\n    private final byte[] id;\n    private final String requestedServerName;\n    private final List<byte[]> statusResponses;\n    private final byte[] peerTlsSctData;\n    private final long creationTime;\n    private final long lastAccessedTime;\n    private final String cipherSuite;\n    private final String protocol;\n    private final String peerHost;\n    private final String applicationProtocol;\n    private final int peerPort;\n\n    SessionSnapshot(ConscryptSession session) {\n        sessionContext = session.getSessionContext();\n        id = session.getId();\n        requestedServerName = session.getRequestedServerName();\n        statusResponses = session.getStatusResponses();\n        peerTlsSctData = session.getPeerSignedCertificateTimestamp();\n        creationTime = session.getCreationTime();\n        lastAccessedTime = session.getLastAccessedTime();\n        cipherSuite = session.getCipherSuite();\n        protocol = session.getProtocol();\n        peerHost = session.getPeerHost();\n        peerPort = session.getPeerPort();\n        applicationProtocol = session.getApplicationProtocol();\n    }\n\n    @Override\n    public String getRequestedServerName() {\n        return requestedServerName;\n    }\n\n    @Override\n    public List<byte[]> getStatusResponses() {\n        List<byte[]> ret = new ArrayList<>(statusResponses.size());\n        for (byte[] resp : statusResponses) {\n            ret.add(resp.clone());\n        }\n        return ret;\n    }\n\n    @Override\n    public byte[] getPeerSignedCertificateTimestamp() {\n        return peerTlsSctData != null ? peerTlsSctData.clone() : null;\n    }\n\n    @Override\n    public byte[] getId() {\n        return id;\n    }\n\n    @Override\n    public SSLSessionContext getSessionContext() {\n        return sessionContext;\n    }\n\n    @Override\n    public long getCreationTime() {\n        return creationTime;\n    }\n\n    @Override\n    public long getLastAccessedTime() {\n        return lastAccessedTime;\n    }\n\n    @Override\n    public void invalidate() {\n        // Do nothing.\n    }\n\n    @Override\n    public boolean isValid() {\n        return false;\n    }\n\n    @Override\n    public void putValue(String s, Object o) {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public Object getValue(String s) {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public void removeValue(String s) {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public String[] getValueNames() {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public X509Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException {\n        throw new SSLPeerUnverifiedException(\"No peer certificates\");\n    }\n\n    @Override\n    public Certificate[] getLocalCertificates() {\n        return null;\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\") // Public API\n    public javax.security.cert.X509Certificate[] getPeerCertificateChain()\n        throws SSLPeerUnverifiedException {\n        if (!Platform.isJavaxCertificateSupported()) {\n            throw new UnsupportedOperationException(\"Use getPeerCertificates() instead\");\n        }\n\n        throw new SSLPeerUnverifiedException(\"No peer certificates\");\n    }\n\n    @Override\n    public Principal getPeerPrincipal() throws SSLPeerUnverifiedException {\n        throw new SSLPeerUnverifiedException(\"No peer certificates\");\n    }\n\n    @Override\n    public Principal getLocalPrincipal() {\n        return null;\n    }\n\n    @Override\n    public String getCipherSuite() {\n        return cipherSuite;\n    }\n\n    @Override\n    public String getProtocol() {\n        return protocol;\n    }\n\n    @Override\n    public String getPeerHost() {\n        return peerHost;\n    }\n\n    @Override\n    public int getPeerPort() {\n        return peerPort;\n    }\n\n    @Override\n    public int getPacketBufferSize() {\n        return NativeConstants.SSL3_RT_MAX_PACKET_SIZE;\n    }\n\n    @Override\n    public int getApplicationBufferSize() {\n        return NativeConstants.SSL3_RT_MAX_PLAIN_LENGTH;\n    }\n\n    @Override\n    public String getApplicationProtocol() {\n        return applicationProtocol;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX25519Key.java\npackage org.conscrypt;\n\npublic interface OpenSSLX25519Key {\n    int X25519_KEY_SIZE_BYTES = 32;\n\n    byte[] getU();\n}\n\n// Path: common/src/main/java/org/conscrypt/Java8EngineSocket.java\n/*\n * Copyright 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.net.Socket;\nimport java.util.List;\nimport java.util.function.BiFunction;\nimport javax.net.ssl.SSLEngine;\nimport javax.net.ssl.SSLSocket;\n\n/**\n * A version of ConscryptEngineSocket that includes the new Java 9 (and potentially later\n * patches of 8) {@code setHandshakeApplicationProtocolSelector} API (which requires Java 8 for\n * compilation, due to the use of {@link BiFunction}).\n */\nfinal class Java8EngineSocket extends ConscryptEngineSocket {\n    private BiFunction<SSLSocket, List<String>, String> selector;\n\n    Java8EngineSocket(SSLParametersImpl sslParameters) throws IOException {\n        super(sslParameters);\n    }\n\n    Java8EngineSocket(String hostname, int port, SSLParametersImpl sslParameters)\n            throws IOException {\n        super(hostname, port, sslParameters);\n    }\n\n    Java8EngineSocket(InetAddress address, int port, SSLParametersImpl sslParameters)\n            throws IOException {\n        super(address, port, sslParameters);\n    }\n\n    Java8EngineSocket(String hostname, int port, InetAddress clientAddress, int clientPort,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(hostname, port, clientAddress, clientPort, sslParameters);\n    }\n\n    Java8EngineSocket(InetAddress address, int port, InetAddress clientAddress, int clientPort,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(address, port, clientAddress, clientPort, sslParameters);\n    }\n\n    Java8EngineSocket(Socket socket, String hostname, int port, boolean autoClose,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(socket, hostname, port, autoClose, sslParameters);\n    }\n\n    /* @Override */\n    @SuppressWarnings(\"MissingOverride\") // For compilation with Java < 9.\n    public void setHandshakeApplicationProtocolSelector(\n            final BiFunction<SSLSocket, List<String>, String> selector) {\n        this.selector = selector;\n        setApplicationProtocolSelector(toApplicationProtocolSelector(selector));\n    }\n\n    /* @Override */\n    @SuppressWarnings(\"MissingOverride\") // For compilation with Java < 9.\n    public BiFunction<SSLSocket, List<String>, String> getHandshakeApplicationProtocolSelector() {\n        return selector;\n    }\n\n    private static ApplicationProtocolSelector toApplicationProtocolSelector(\n        final BiFunction<SSLSocket, List<String>, String> selector) {\n        return selector == null ? null : new ApplicationProtocolSelector() {\n            @Override\n            public String selectApplicationProtocol(SSLEngine socket, List<String> protocols) {\n                throw new UnsupportedOperationException();\n            }\n\n            @Override\n            public String selectApplicationProtocol(SSLSocket socket, List<String> protocols) {\n                return selector.apply(socket, protocols);\n            }\n        };\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/DuckTypedHpkeSpi.java\n/*\n * Copyright (C) 2023 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License\n */\n\npackage org.conscrypt;\n\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.security.GeneralSecurityException;\nimport java.security.InvalidKeyException;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * Duck typed implementation of {@link HpkeSpi}.\n * <p>\n * Will wrap any Object which implements all of the methods in HpkeSpi and delegate to them\n * by reflection.\n */\n@Internal\npublic class DuckTypedHpkeSpi implements HpkeSpi {\n  private final Object delegate;\n  private final Map<String, Method> methods = new HashMap<>();\n\n  private DuckTypedHpkeSpi(Object delegate) throws NoSuchMethodException {\n    this.delegate = delegate;\n\n    Class<?> sourceClass = delegate.getClass();\n    for (Method targetMethod : HpkeSpi.class.getMethods()) {\n      if (targetMethod.isSynthetic()) {\n        continue;\n      }\n\n      Method sourceMethod =\n          sourceClass.getMethod(targetMethod.getName(), targetMethod.getParameterTypes());\n      // Check that the return types match too.\n      Class<?> sourceReturnType = sourceMethod.getReturnType();\n      Class<?> targetReturnType = targetMethod.getReturnType();\n      if (!targetReturnType.isAssignableFrom(sourceReturnType)) {\n        throw new NoSuchMethodException(sourceMethod + \" return value (\" + sourceReturnType\n            + \") incompatible with target return value (\" + targetReturnType + \")\");\n      }\n      methods.put(sourceMethod.getName(), sourceMethod);\n    }\n  }\n\n  public static DuckTypedHpkeSpi newInstance(Object delegate) {\n    try {\n      return new DuckTypedHpkeSpi(delegate);\n    } catch (Exception ignored) {\n      return null;\n    }\n  }\n\n  private Object invoke(String methodName, Object... args) throws InvocationTargetException {\n    Method method = methods.get(methodName);\n    if (method == null) {\n      throw new IllegalStateException(\"DuckTypedHpkSpi internal error\");\n    }\n    try {\n      return method.invoke(delegate, args);\n    } catch (IllegalAccessException e) {\n      throw new IllegalStateException(\"DuckTypedHpkSpi internal error\", e);\n    } catch (InvocationTargetException e) {\n      if (e.getCause() instanceof RuntimeException){\n        throw (RuntimeException) e.getCause();\n      }\n      throw e;\n    }\n  }\n\n  private void invokeWithPossibleInvalidKey(String methodName, Object... args)\n      throws InvalidKeyException {\n    try {\n      invoke(methodName, args);\n    } catch (InvocationTargetException e) {\n      Throwable cause = e.getCause();\n      if (cause instanceof InvalidKeyException){\n        throw (InvalidKeyException) cause;\n      }\n      throw new IllegalStateException(cause);\n    }\n  }\n\n  private Object invokeWithPossibleGeneralSecurity(String methodName, Object... args)\n      throws GeneralSecurityException {\n    try {\n      return invoke(methodName, args);\n    } catch (InvocationTargetException e) {\n      Throwable cause = e.getCause();\n      if (cause instanceof GeneralSecurityException){\n        throw (GeneralSecurityException) cause;\n      }\n      throw new IllegalStateException(cause);\n    }\n  }\n\n  private Object invokeNoChecked(String methodName, Object... args) {\n    try {\n      return invoke(methodName, args);\n    } catch (InvocationTargetException e) {\n      throw new IllegalStateException(e.getCause());\n    }\n  }\n\n  // Visible for testing\n  public Object getDelegate() {\n    return delegate;\n  }\n\n  @Override\n  public void engineInitSender(\n          PublicKey recipientKey, byte[] info, PrivateKey senderKey, byte[] psk, byte[] pskId)\n          throws InvalidKeyException {\n    invokeWithPossibleInvalidKey(\"engineInitSender\", recipientKey, info, senderKey, psk, pskId);\n  }\n\n  @Override\n  public void engineInitSenderForTesting(PublicKey recipientKey, byte[] info, PrivateKey senderKey,\n          byte[] psk, byte[] pskId, byte[] sKe) throws InvalidKeyException {\n      invokeWithPossibleInvalidKey(\"engineInitSenderForTesting\",\n              recipientKey, info, senderKey, psk, pskId, sKe);\n  }\n\n  @Override\n  public void engineInitRecipient(byte[] encapsulated, PrivateKey key, byte[] info,\n          PublicKey senderKey, byte[] psk, byte[] psk_id) throws InvalidKeyException {\n    invokeWithPossibleInvalidKey(\n        \"engineInitRecipient\", encapsulated, key, info, senderKey, psk, psk_id);\n  }\n\n  @Override\n  public byte[] engineSeal(byte[] plaintext, byte[] aad) {\n      return (byte[]) invokeNoChecked(\"engineSeal\", plaintext, aad);\n  }\n\n  @Override\n  public byte[] engineExport(int length, byte[] exporterContext) {\n      return (byte[]) invokeNoChecked(\"engineExport\", length, exporterContext);\n  }\n\n  @Override\n  public byte[] engineOpen(byte[] ciphertext, byte[] aad) throws GeneralSecurityException {\n      return (byte[]) invokeWithPossibleGeneralSecurity(\"engineOpen\", ciphertext, aad);\n  }\n\n  @Override\n  public byte[] getEncapsulated() {\n      return (byte[]) invokeNoChecked(\"getEncapsulated\");\n  }\n}\n\n// Path: common/src/main/java/org/conscrypt/ChainStrengthAnalyzer.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.cert.CertificateException;\nimport java.security.cert.X509Certificate;\nimport java.security.interfaces.DSAPublicKey;\nimport java.security.interfaces.ECPublicKey;\nimport java.security.interfaces.RSAPublicKey;\nimport java.util.List;\n\n/**\n * Analyzes the cryptographic strength of a chain of X.509 certificates.\n */\n@Internal\npublic final class ChainStrengthAnalyzer {\n\n    private static final int MIN_RSA_MODULUS_LEN_BITS = 1024;\n\n    private static final int MIN_EC_FIELD_SIZE_BITS = 160;\n\n    private static final int MIN_DSA_P_LEN_BITS = 1024;\n    private static final int MIN_DSA_Q_LEN_BITS = 160;\n\n    private static final String[] SIGNATURE_ALGORITHM_OID_BLACKLIST = {\n        \"1.2.840.113549.1.1.2\", // md2WithRSAEncryption\n        \"1.2.840.113549.1.1.3\", // md4WithRSAEncryption\n        \"1.2.840.113549.1.1.4\", // md5WithRSAEncryption\n        \"1.2.840.113549.1.1.5\", // sha1WithRSAEncryption\n        \"1.2.840.10040.4.3\", //dsa-with-sha1\n        \"1.2.840.10045.4.1\", //ecdsa-with-sha1\n    };\n\n    public static final void check(X509Certificate[] chain) throws CertificateException {\n        for (X509Certificate cert : chain) {\n            try {\n                checkCert(cert);\n            } catch (CertificateException e) {\n                throw new CertificateException(\"Unacceptable certificate: \"\n                        + cert.getSubjectX500Principal(), e);\n            }\n        }\n    }\n\n    public static final void check(List<X509Certificate> chain) throws CertificateException {\n        for (X509Certificate cert : chain) {\n            try {\n                checkCert(cert);\n            } catch (CertificateException e) {\n                throw new CertificateException(\"Unacceptable certificate: \"\n                        + cert.getSubjectX500Principal(), e);\n            }\n        }\n    }\n\n    public static final void checkCert(X509Certificate cert) throws CertificateException {\n        checkKeyLength(cert);\n        checkSignatureAlgorithm(cert);\n    }\n\n    private static void checkKeyLength(X509Certificate cert) throws CertificateException {\n        Object pubkey = cert.getPublicKey();\n        if (pubkey instanceof RSAPublicKey) {\n            int modulusLength = ((RSAPublicKey) pubkey).getModulus().bitLength();\n            if (modulusLength < MIN_RSA_MODULUS_LEN_BITS) {\n                throw new CertificateException(\n                        \"RSA modulus is < \" + MIN_RSA_MODULUS_LEN_BITS + \" bits\");\n            }\n        } else if (pubkey instanceof ECPublicKey) {\n            int fieldSizeBits =\n                    ((ECPublicKey) pubkey).getParams().getCurve().getField().getFieldSize();\n            if (fieldSizeBits < MIN_EC_FIELD_SIZE_BITS) {\n                throw new CertificateException(\n                        \"EC key field size is < \" + MIN_EC_FIELD_SIZE_BITS + \" bits\");\n            }\n        } else if (pubkey instanceof DSAPublicKey) {\n            int pLength = ((DSAPublicKey) pubkey).getParams().getP().bitLength();\n            int qLength = ((DSAPublicKey) pubkey).getParams().getQ().bitLength();\n            if ((pLength < MIN_DSA_P_LEN_BITS) || (qLength < MIN_DSA_Q_LEN_BITS)) {\n                throw new CertificateException(\n                        \"DSA key length is < (\" + MIN_DSA_P_LEN_BITS + \", \" + MIN_DSA_Q_LEN_BITS\n                        + \") bits\");\n            }\n        } else {\n            // Unknown keys will be of type X509PublicKey.\n            throw new CertificateException(\"Rejecting unknown key class \" + pubkey.getClass().getName());\n        }\n    }\n\n    private static void checkSignatureAlgorithm(\n            X509Certificate cert) throws CertificateException {\n        String oid = cert.getSigAlgOID();\n        for (String blacklisted : SIGNATURE_ALGORITHM_OID_BLACKLIST) {\n            if (oid.equals(blacklisted)) {\n                throw new CertificateException(\"Signature uses an insecure hash function: \" + oid);\n            }\n        }\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/NativeSslSession.java\n/*\n * Copyright 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\", \"www.google.com\", 443);\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport static org.conscrypt.SSLUtils.SessionType.OPEN_SSL_WITH_OCSP;\nimport static org.conscrypt.SSLUtils.SessionType.OPEN_SSL_WITH_TLS_SCT;\nimport static org.conscrypt.SSLUtils.SessionType.isSupportedType;\n\nimport java.io.ByteArrayOutputStream;\nimport java.io.DataOutputStream;\nimport java.io.IOException;\nimport java.nio.BufferUnderflowException;\nimport java.nio.ByteBuffer;\nimport java.security.Principal;\nimport java.security.cert.Certificate;\nimport java.security.cert.CertificateEncodingException;\nimport java.util.List;\nimport java.util.logging.Level;\nimport java.util.logging.Logger;\nimport javax.net.ssl.SSLException;\nimport javax.net.ssl.SSLPeerUnverifiedException;\nimport javax.net.ssl.SSLSession;\nimport javax.net.ssl.SSLSessionContext;\n\n/**\n * A utility wrapper that abstracts operations on the underlying native SSL_SESSION instance.\n * <p>\n * This is abstract only to support mocking for tests.\n */\nabstract class NativeSslSession {\n    private static final Logger logger = Logger.getLogger(NativeSslSession.class.getName());\n\n    /**\n     * Creates a new instance. Since BoringSSL does not provide an API to get access to all\n     * session information via the SSL_SESSION, we get some values (e.g. peer certs) from\n     * the {@link ConscryptSession} instead (i.e. the SSL object).\n     */\n    static NativeSslSession newInstance(NativeRef.SSL_SESSION ref, ConscryptSession session)\n            throws SSLPeerUnverifiedException {\n        AbstractSessionContext context = (AbstractSessionContext) session.getSessionContext();\n        if (context instanceof ClientSessionContext) {\n            return new Impl(context, ref, session.getPeerHost(), session.getPeerPort(),\n                session.getPeerCertificates(), getOcspResponse(session),\n                session.getPeerSignedCertificateTimestamp());\n        }\n\n        // Server's will be cached by ID and won't have any of the extra fields.\n        return new Impl(context, ref, null, -1, null, null, null);\n    }\n\n    private static byte[] getOcspResponse(ConscryptSession session) {\n        List<byte[]> ocspResponseList = session.getStatusResponses();\n        if (ocspResponseList.size() >= 1) {\n            return ocspResponseList.get(0);\n        }\n        return null;\n    }\n\n    /**\n     * Creates a new {@link NativeSslSession} instance from the provided serialized bytes, which\n     * were generated by {@link #toBytes()}.\n     *\n     * @return The new instance if successful. If unable to parse the bytes for any reason, returns\n     * {@code null}.\n     */\n    static NativeSslSession newInstance(\n            AbstractSessionContext context, byte[] data, String host, int port) {\n        ByteBuffer buf = ByteBuffer.wrap(data);\n        try {\n            int type = buf.getInt();\n            if (!isSupportedType(type)) {\n                throw new IOException(\"Unexpected type ID: \" + type);\n            }\n\n            int length = buf.getInt();\n            checkRemaining(buf, length);\n\n            byte[] sessionData = new byte[length];\n            buf.get(sessionData);\n\n            int count = buf.getInt();\n            checkRemaining(buf, count);\n\n            java.security.cert.X509Certificate[] peerCerts =\n                    new java.security.cert.X509Certificate[count];\n            for (int i = 0; i < count; i++) {\n                length = buf.getInt();\n                checkRemaining(buf, length);\n\n                byte[] certData = new byte[length];\n                buf.get(certData);\n                try {\n                    peerCerts[i] = OpenSSLX509Certificate.fromX509Der(certData);\n...\n", "needle_token_start": 8729, "needle_token_end": 8754, "code_context_ntokens": 15926, "output": ["```java\n@Override\npublic SSLSessionContext engineGetServerSessionContext() {\n    return serverSessionContext;\n}\n```"]}
{"repo": "google/conscrypt", "name": "checkAndSetEncodedKey", "language": "java", "path": "common/src/main/java/org/conscrypt/OpenSSLCipher.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate and set the encoded form of a cryptographic key based on the specified operation mode. It ensures that the key is suitable for encryption or decryption tasks.\n2. **Input**: The function takes two parameters: an integer representing the operation mode (such as encrypt or decrypt) and a cryptographic key object.\n3. **Output**: Returns the byte array representation of the cryptographic key if it passes all checks.\n4. **Procedure**: \n   - First, the function determines whether the operation is for encryption or decryption based on the operation mode.\n   - It then checks if the provided key is of the correct type (specifically, a secret key).\n   - The function retrieves and checks if the byte array representation of the key is non-null.\n   - It verifies that the size of the key is supported.\n   - Finally, it sets an internal representation of the key to its encoded form and returns this byte array.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/Java7ExtendedSSLSession.java\n/*\n * Copyright 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt;\n\nimport java.security.Principal;\nimport java.security.cert.Certificate;\nimport java.util.List;\nimport javax.net.ssl.ExtendedSSLSession;\nimport javax.net.ssl.SSLPeerUnverifiedException;\nimport javax.net.ssl.SSLSessionContext;\n\n/**\n * This is an adapter that wraps the active session with {@link ExtendedSSLSession}, if running\n * on Java 7+.\n */\nclass Java7ExtendedSSLSession extends ExtendedSSLSession implements ConscryptSession {\n    // TODO: use BoringSSL API to actually fetch the real data\n    private static final String[] LOCAL_SUPPORTED_SIGNATURE_ALGORITHMS = new String[] {\n            \"SHA512withRSA\", \"SHA512withECDSA\", \"SHA384withRSA\", \"SHA384withECDSA\", \"SHA256withRSA\",\n            \"SHA256withECDSA\", \"SHA224withRSA\", \"SHA224withECDSA\", \"SHA1withRSA\", \"SHA1withECDSA\",\n    };\n    // TODO: use BoringSSL API to actually fetch the real data\n    private static final String[] PEER_SUPPORTED_SIGNATURE_ALGORITHMS =\n            new String[] {\"SHA1withRSA\", \"SHA1withECDSA\"};\n    protected final ExternalSession delegate;\n\n    Java7ExtendedSSLSession(ExternalSession delegate) {\n        this.delegate = delegate;\n    }\n\n    /* @Override */\n    @SuppressWarnings(\"MissingOverride\") // For Android backward-compatibility.\n    public final String[] getLocalSupportedSignatureAlgorithms() {\n...\n// Path: common/src/main/java/org/conscrypt/TrustedCertificateIndex.java\n/*\n * Copyright (C) 2009 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.PublicKey;\nimport java.security.cert.TrustAnchor;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\nimport javax.security.auth.x500.X500Principal;\n\n/**\n * Indexes {@code TrustAnchor} instances so they can be found in O(1)\n * time instead of O(N).\n */\n@Internal\npublic final class TrustedCertificateIndex {\n\n    private final Map<X500Principal, List<TrustAnchor>> subjectToTrustAnchors\n            = new HashMap<X500Principal, List<TrustAnchor>>();\n\n    public TrustedCertificateIndex() {}\n\n    public TrustedCertificateIndex(Set<TrustAnchor> anchors) {\n        index(anchors);\n    }\n\n    private void index(Set<TrustAnchor> anchors) {\n        for (TrustAnchor anchor : anchors) {\n            index(anchor);\n        }\n    }\n\n    public TrustAnchor index(X509Certificate cert) {\n        TrustAnchor anchor = new TrustAnchor(cert, null);\n        index(anchor);\n        return anchor;\n    }\n\n    public void index(TrustAnchor anchor) {\n        X500Principal subject;\n        X509Certificate cert = anchor.getTrustedCert();\n        if (cert != null) {\n            subject = cert.getSubjectX500Principal();\n        } else {\n            subject = anchor.getCA();\n        }\n\n        synchronized (subjectToTrustAnchors) {\n            List<TrustAnchor> anchors = subjectToTrustAnchors.get(subject);\n            if (anchors == null) {\n                anchors = new ArrayList<TrustAnchor>(1);\n                subjectToTrustAnchors.put(subject, anchors);\n            } else {\n                // Avoid indexing the same certificate multiple times\n                if (cert != null) {\n                    for (TrustAnchor entry : anchors) {\n                        if (cert.equals(entry.getTrustedCert())) {\n                            return;\n                        }\n                    }\n                }\n            }\n            anchors.add(anchor);\n        }\n    }\n\n    public void reset() {\n        synchronized (subjectToTrustAnchors) {\n            subjectToTrustAnchors.clear();\n        }\n    }\n\n    public void reset(Set<TrustAnchor> anchors) {\n        synchronized (subjectToTrustAnchors) {\n            reset();\n            index(anchors);\n        }\n    }\n\n    public TrustAnchor findByIssuerAndSignature(X509Certificate cert) {\n        X500Principal issuer = cert.getIssuerX500Principal();\n        synchronized (subjectToTrustAnchors) {\n            List<TrustAnchor> anchors = subjectToTrustAnchors.get(issuer);\n            if (anchors == null) {\n                return null;\n            }\n\n            for (TrustAnchor anchor : anchors) {\n                PublicKey publicKey;\n                try {\n                    X509Certificate caCert = anchor.getTrustedCert();\n                    if (caCert != null) {\n                        publicKey = caCert.getPublicKey();\n                    } else {\n                        publicKey = anchor.getCAPublicKey();\n                    }\n                    cert.verify(publicKey);\n                    return anchor;\n                } catch (Exception ignored) {\n                    // Ignored\n                }\n            }\n        }\n        return null;\n    }\n\n    public TrustAnchor findBySubjectAndPublicKey(X509Certificate cert) {\n        X500Principal subject = cert.getSubjectX500Principal();\n        synchronized (subjectToTrustAnchors) {\n            List<TrustAnchor> anchors = subjectToTrustAnchors.get(subject);\n            if (anchors == null) {\n                return null;\n            }\n            return findBySubjectAndPublicKey(cert, anchors);\n        }\n    }\n\n    private static TrustAnchor findBySubjectAndPublicKey(X509Certificate cert,\n                                                         Collection<TrustAnchor> anchors) {\n        PublicKey certPublicKey = cert.getPublicKey();\n        for (TrustAnchor anchor : anchors) {\n            PublicKey caPublicKey;\n            try {\n                X509Certificate caCert = anchor.getTrustedCert();\n                if (caCert != null) {\n                    caPublicKey = caCert.getPublicKey();\n                } else {\n                    caPublicKey = anchor.getCAPublicKey();\n                }\n                if (caPublicKey.equals(certPublicKey)) {\n                    return anchor;\n                } else {\n                    // PublicKey.equals is not required to compare keys across providers. Fall back\n                    // to checking using the encoded form.\n                    if (\"X.509\".equals(caPublicKey.getFormat())\n                            && \"X.509\".equals(certPublicKey.getFormat())) {\n                        byte[] caPublicKeyEncoded = caPublicKey.getEncoded();\n                        byte[] certPublicKeyEncoded = certPublicKey.getEncoded();\n                        if (certPublicKeyEncoded != null\n                                && caPublicKeyEncoded != null\n                                && Arrays.equals(caPublicKeyEncoded, certPublicKeyEncoded)) {\n                            return anchor;\n                        }\n                    }\n                }\n            } catch (Exception e) {\n                // can happen with unsupported public key types\n            }\n        }\n        return null;\n    }\n\n    public Set<TrustAnchor> findAllByIssuerAndSignature(X509Certificate cert) {\n        X500Principal issuer = cert.getIssuerX500Principal();\n        synchronized (subjectToTrustAnchors) {\n            List<TrustAnchor> anchors = subjectToTrustAnchors.get(issuer);\n            if (anchors == null) {\n                return Collections.<TrustAnchor>emptySet();\n            }\n\n            Set<TrustAnchor> result = new HashSet<TrustAnchor>();\n            for (TrustAnchor anchor : anchors) {\n                try {\n                    PublicKey publicKey;\n                    X509Certificate caCert = anchor.getTrustedCert();\n                    if (caCert != null) {\n                        publicKey = caCert.getPublicKey();\n                    } else {\n                        publicKey = anchor.getCAPublicKey();\n                    }\n                    if (publicKey == null) {\n                        continue;\n                    }\n                    cert.verify(publicKey);\n                    result.add(anchor);\n                } catch (Exception ignored) {\n                    // Ignored\n                }\n            }\n            return result;\n        }\n    }\n\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX509CRLEntry.java\n/*\n * Copyright (C) 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.ByteArrayOutputStream;\nimport java.math.BigInteger;\nimport java.security.cert.CRLException;\nimport java.security.cert.X509CRLEntry;\nimport java.util.Arrays;\nimport java.util.Date;\nimport java.util.HashSet;\nimport java.util.Set;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * An implementation of {@link X509CRLEntry} based on BoringSSL.\n */\nfinal class OpenSSLX509CRLEntry extends X509CRLEntry {\n    private final long mContext;\n    private final long revocationDate;\n\n    OpenSSLX509CRLEntry(long ctx) throws ParsingException {\n        mContext = ctx;\n        // The legacy X509 OpenSSL APIs don't validate ASN1_TIME structures until access, so\n        // parse them here because this is the only time we're allowed to throw ParsingException\n        revocationDate = NativeCrypto.get_X509_REVOKED_revocationDate(mContext);\n    }\n\n    @Override\n    public Set<String> getCriticalExtensionOIDs() {\n        String[] critOids =\n                NativeCrypto.get_X509_REVOKED_ext_oids(mContext,\n                        NativeCrypto.EXTENSION_TYPE_CRITICAL);\n\n        /*\n         * This API has a special case that if there are no extensions, we\n         * should return null. So if we have no critical extensions, we'll check\n         * non-critical extensions.\n         */\n        if ((critOids.length == 0)\n                && (NativeCrypto.get_X509_REVOKED_ext_oids(mContext,\n                        NativeCrypto.EXTENSION_TYPE_NON_CRITICAL).length == 0)) {\n            return null;\n        }\n\n        return new HashSet<String>(Arrays.asList(critOids));\n    }\n\n    @Override\n    public byte[] getExtensionValue(String oid) {\n        return NativeCrypto.X509_REVOKED_get_ext_oid(mContext, oid);\n    }\n\n    @Override\n    public Set<String> getNonCriticalExtensionOIDs() {\n        String[] critOids =\n                NativeCrypto.get_X509_REVOKED_ext_oids(mContext,\n                        NativeCrypto.EXTENSION_TYPE_NON_CRITICAL);\n\n        /*\n         * This API has a special case that if there are no extensions, we\n         * should return null. So if we have no non-critical extensions, we'll\n         * check critical extensions.\n         */\n        if ((critOids.length == 0)\n                && (NativeCrypto.get_X509_REVOKED_ext_oids(mContext,\n                        NativeCrypto.EXTENSION_TYPE_CRITICAL).length == 0)) {\n            return null;\n        }\n\n        return new HashSet<String>(Arrays.asList(critOids));\n    }\n\n    @Override\n    public boolean hasUnsupportedCriticalExtension() {\n        final String[] criticalOids =\n                NativeCrypto.get_X509_REVOKED_ext_oids(mContext,\n                        NativeCrypto.EXTENSION_TYPE_CRITICAL);\n        for (String oid : criticalOids) {\n            final long extensionRef = NativeCrypto.X509_REVOKED_get_ext(mContext, oid);\n            if (NativeCrypto.X509_supported_extension(extensionRef) != 1) {\n                return true;\n            }\n        }\n\n        return false;\n    }\n\n    @Override\n    public byte[] getEncoded() throws CRLException {\n        return NativeCrypto.i2d_X509_REVOKED(mContext);\n    }\n\n    @Override\n    public BigInteger getSerialNumber() {\n        return new BigInteger(NativeCrypto.X509_REVOKED_get_serialNumber(mContext));\n    }\n\n    @Override\n    public Date getRevocationDate() {\n        return new Date(revocationDate);\n    }\n\n    @Override\n    public boolean hasExtensions() {\n        return (NativeCrypto.get_X509_REVOKED_ext_oids(mContext,\n                NativeCrypto.EXTENSION_TYPE_NON_CRITICAL).length != 0)\n                || (NativeCrypto.get_X509_REVOKED_ext_oids(mContext,\n                        NativeCrypto.EXTENSION_TYPE_CRITICAL).length != 0);\n    }\n\n    @Override\n    public String toString() {\n        ByteArrayOutputStream os = new ByteArrayOutputStream();\n        long bioCtx = NativeCrypto.create_BIO_OutputStream(os);\n        try {\n            NativeCrypto.X509_REVOKED_print(bioCtx, mContext);\n            return os.toString();\n        } finally {\n            NativeCrypto.BIO_free_all(bioCtx);\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ScryptKeySpec.java\n/*\n * Copyright (C) 2022 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.spec.KeySpec;\n\n/**\n * Mirrors the <a\n * href=\"https://javadoc.io/static/org.bouncycastle/bcprov-jdk15on/1.68/org/bouncycastle/jcajce/spec/ScryptKeySpec.html\">class\n * of the same name</a> from BouncyCastle.\n */\npublic class ScryptKeySpec implements KeySpec {\n    private final char[] password;\n    private final byte[] salt;\n    private final int costParameter;\n    private final int blockSize;\n    private final int parallelizationParameter;\n    private final int keyOutputBits;\n\n    public ScryptKeySpec(\n            char[] password,\n            byte[] salt,\n            int costParameter,\n            int blockSize,\n            int parallelizationParameter,\n            int keyOutputBits) {\n        this.password = password;\n        this.salt = salt;\n        this.costParameter = costParameter;\n        this.blockSize = blockSize;\n        this.parallelizationParameter = parallelizationParameter;\n        this.keyOutputBits = keyOutputBits;\n    }\n\n    public char[] getPassword() {\n        return password;\n    }\n\n    public byte[] getSalt() {\n        return salt;\n    }\n\n    public int getCostParameter() {\n        return costParameter;\n    }\n\n    public int getBlockSize() {\n        return blockSize;\n    }\n\n    public int getParallelizationParameter() {\n        return parallelizationParameter;\n    }\n\n    public int getKeyLength() {\n        return keyOutputBits;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLECKeyPairGenerator.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidParameterException;\nimport java.security.KeyPair;\nimport java.security.KeyPairGenerator;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport java.security.spec.ECGenParameterSpec;\nimport java.security.spec.ECParameterSpec;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * An implementation of {@link KeyPairGenerator} for EC keys which uses BoringSSL to perform all the\n * operations.\n */\n@Internal\npublic final class OpenSSLECKeyPairGenerator extends KeyPairGenerator {\n    private static final String ALGORITHM = \"EC\";\n\n    private static final int DEFAULT_KEY_SIZE = 256;\n\n    private static final Map<Integer, String> SIZE_TO_CURVE_NAME = new HashMap<Integer, String>();\n\n    static {\n        /* NIST curves */\n        SIZE_TO_CURVE_NAME.put(224, \"secp224r1\");\n        SIZE_TO_CURVE_NAME.put(256, \"prime256v1\");\n        SIZE_TO_CURVE_NAME.put(384, \"secp384r1\");\n        SIZE_TO_CURVE_NAME.put(521, \"secp521r1\");\n    }\n\n    private OpenSSLECGroupContext group;\n\n    public OpenSSLECKeyPairGenerator() {\n        super(ALGORITHM);\n    }\n\n    @Override\n    public KeyPair generateKeyPair() {\n        if (group == null) {\n            final String curveName = SIZE_TO_CURVE_NAME.get(DEFAULT_KEY_SIZE);\n            group = OpenSSLECGroupContext.getCurveByName(curveName);\n            if (group == null) {\n                throw new RuntimeException(\"Curve not recognized: \" + curveName);\n            }\n        }\n\n        final OpenSSLKey key = new OpenSSLKey(\n                NativeCrypto.EC_KEY_generate_key(group.getNativeRef()));\n        return new KeyPair(new OpenSSLECPublicKey(group, key), new OpenSSLECPrivateKey(group, key));\n    }\n\n    @Override\n    public void initialize(int keysize, SecureRandom random) {\n        final String name = SIZE_TO_CURVE_NAME.get(keysize);\n        if (name == null) {\n            throw new InvalidParameterException(\"unknown key size \" + keysize);\n        }\n\n        /*\n         * Store the group in a temporary variable until we know this is a valid\n         * group.\n         */\n        final OpenSSLECGroupContext possibleGroup = OpenSSLECGroupContext.getCurveByName(name);\n        if (possibleGroup == null) {\n            throw new InvalidParameterException(\"unknown curve \" + name);\n        }\n\n        group = possibleGroup;\n    }\n\n    @Override\n    public void initialize(AlgorithmParameterSpec param, SecureRandom random)\n            throws InvalidAlgorithmParameterException {\n        if (param instanceof ECParameterSpec) {\n            ECParameterSpec ecParam = (ECParameterSpec) param;\n\n            group = OpenSSLECGroupContext.getInstance(ecParam);\n        } else if (param instanceof ECGenParameterSpec) {\n            ECGenParameterSpec ecParam = (ECGenParameterSpec) param;\n\n            final String curveName = ecParam.getName();\n\n            /*\n             * Store the group in a temporary variable until we know this is a\n             * valid group.\n             */\n            final OpenSSLECGroupContext possibleGroup = OpenSSLECGroupContext\n                    .getCurveByName(curveName);\n            if (possibleGroup == null) {\n                throw new InvalidAlgorithmParameterException(\"unknown curve name: \" + curveName);\n            }\n\n            group = possibleGroup;\n        } else {\n            throw new InvalidAlgorithmParameterException(\n                    \"parameter must be ECParameterSpec or ECGenParameterSpec\");\n        }\n    }\n\n    /** For testing. */\n    public static void assertCurvesAreValid() {\n        ArrayList<String> invalidCurves = new ArrayList<String>();\n        for (String curveName : SIZE_TO_CURVE_NAME.values()) {\n            if (OpenSSLECGroupContext.getCurveByName(curveName) == null) {\n                invalidCurves.add(curveName);\n            }\n        }\n        if (invalidCurves.size() > 0) {\n            throw new AssertionError(\"Invalid curve names: \"\n                    + Arrays.toString(invalidCurves.toArray()));\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLCipher.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.AlgorithmParameters;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.InvalidParameterException;\nimport java.security.Key;\nimport java.security.KeyFactory;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport java.security.spec.InvalidKeySpecException;\nimport java.security.spec.InvalidParameterSpecException;\nimport java.security.spec.PKCS8EncodedKeySpec;\nimport java.security.spec.X509EncodedKeySpec;\nimport java.util.Arrays;\nimport java.util.Locale;\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.Cipher;\nimport javax.crypto.CipherSpi;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.crypto.NoSuchPaddingException;\nimport javax.crypto.SecretKey;\nimport javax.crypto.ShortBufferException;\nimport javax.crypto.spec.IvParameterSpec;\nimport javax.crypto.spec.SecretKeySpec;\n\n/**\n * An implementation of {@link Cipher} using BoringSSL as the backing library.\n */\n@Internal\npublic abstract class OpenSSLCipher extends CipherSpi {\n\n    /**\n     * Modes that a block cipher may support.\n     */\n    enum Mode {\n        NONE,\n        CBC,\n        CTR,\n        ECB,\n        GCM,\n        GCM_SIV,\n        POLY1305,\n        ;\n\n        public static Mode getNormalized(String modeString) {\n            modeString = modeString.toUpperCase(Locale.US);\n            // We use GCM-SIV as the mode string, but - isn't a valid identifier character, so\n            // we need to ensure GCM-SIV is recognized and GCM_SIV isn't.\n            if (modeString.equals(\"GCM-SIV\")) {\n                return GCM_SIV;\n            } else if (modeString.equals(\"GCM_SIV\")) {\n                throw new IllegalArgumentException(\"Invalid mode\");\n            } else {\n                return Mode.valueOf(modeString);\n            }\n        }\n    }\n\n    /**\n     * Paddings that a block cipher may support.\n     */\n    enum Padding {\n        NOPADDING,\n        PKCS5PADDING,\n        PKCS7PADDING,\n        ;\n\n        public static Padding getNormalized(String value) {\n            Padding p = Padding.valueOf(value.toUpperCase(Locale.US));\n            if (p == PKCS7PADDING) {\n                return PKCS5PADDING;\n            }\n            return p;\n        }\n    }\n\n    /**\n     * The current cipher mode.\n     */\n    Mode mode = Mode.ECB;\n\n    /**\n     * The current cipher padding.\n     */\n    private Padding padding = Padding.PKCS5PADDING;\n\n    /**\n     * May be used when reseting the cipher instance after calling\n     * {@code doFinal}.\n     */\n    byte[] encodedKey;\n\n    /**\n     * The Initial Vector (IV) used for the current cipher.\n     */\n    byte[] iv;\n\n    /**\n     * Current cipher mode: encrypting or decrypting.\n     */\n    private boolean encrypting;\n\n    /**\n     * The block size of the current cipher.\n     */\n    private int blockSize;\n\n    OpenSSLCipher() {\n    }\n\n    OpenSSLCipher(Mode mode, Padding padding) {\n        this.mode = mode;\n        this.padding = padding;\n        blockSize = getCipherBlockSize();\n    }\n\n    /**\n     * API-specific implementation of initializing the cipher. The\n     * {@link #isEncrypting()} function will tell whether it should be\n     * initialized for encryption or decryption. The {@code encodedKey} will be\n     * the bytes of a supported key size.\n     */\n    abstract void engineInitInternal(byte[] encodedKey, AlgorithmParameterSpec params,\n            SecureRandom random) throws InvalidKeyException, InvalidAlgorithmParameterException;\n\n    /**\n     * API-specific implementation of updating the cipher. The\n     * {@code maximumLen} will be the maximum length of the output as returned\n     * by {@link #getOutputSizeForUpdate(int)}. The return value must be the\n     * number of bytes processed and placed into {@code output}. On error, an\n     * exception must be thrown.\n     */\n    abstract int updateInternal(byte[] input, int inputOffset, int inputLen,\n            byte[] output, int outputOffset, int maximumLen) throws ShortBufferException;\n\n    /**\n     * API-specific implementation of the final block. The {@code maximumLen}\n     * will be the maximum length of the possible output as returned by\n     * {@link #getOutputSizeForFinal(int)}. The return value must be the number\n     * of bytes processed and placed into {@code output}. On error, an exception\n     * must be thrown.\n     */\n    abstract int doFinalInternal(byte[] output, int outputOffset, int maximumLen)\n            throws IllegalBlockSizeException, BadPaddingException, ShortBufferException;\n\n    /**\n     * Returns the standard name for the particular algorithm.\n     */\n    abstract String getBaseCipherName();\n\n    /**\n     * Checks whether the cipher supports this particular {@code keySize} (in\n     * bytes) and throws {@code InvalidKeyException} if it doesn't.\n     */\n    abstract void checkSupportedKeySize(int keySize) throws InvalidKeyException;\n\n    /**\n     * Checks whether the cipher supports this particular cipher {@code mode}\n     * and throws {@code NoSuchAlgorithmException} if it doesn't.\n     */\n    abstract void checkSupportedMode(Mode mode) throws NoSuchAlgorithmException;\n\n    /**\n     * Checks whether the cipher supports this particular cipher {@code padding}\n     * and throws {@code NoSuchPaddingException} if it doesn't.\n     */\n    abstract void checkSupportedPadding(Padding padding) throws NoSuchPaddingException;\n\n    abstract int getCipherBlockSize();\n\n    boolean supportsVariableSizeKey() {\n        return false;\n    }\n\n    boolean supportsVariableSizeIv() {\n        return false;\n    }\n\n    @Override\n    protected void engineSetMode(String modeStr) throws NoSuchAlgorithmException {\n        final Mode mode;\n        try {\n            mode = Mode.getNormalized(modeStr);\n        } catch (IllegalArgumentException e) {\n            NoSuchAlgorithmException newE = new NoSuchAlgorithmException(\"No such mode: \" + modeStr);\n            newE.initCause(e);\n            throw newE;\n        }\n        checkSupportedMode(mode);\n        this.mode = mode;\n    }\n\n    @Override\n    protected void engineSetPadding(String paddingStr) throws NoSuchPaddingException {\n        final Padding padding;\n        try {\n            padding = Padding.getNormalized(paddingStr);\n        } catch (IllegalArgumentException e) {\n            NoSuchPaddingException newE = new NoSuchPaddingException(\"No such padding: \"\n                    + paddingStr);\n            newE.initCause(e);\n            throw newE;\n        }\n        checkSupportedPadding(padding);\n        this.padding = padding;\n    }\n\n    /**\n     * Returns the padding type for which this cipher is initialized.\n     */\n    Padding getPadding() {\n        return padding;\n    }\n\n    @Override\n    protected int engineGetBlockSize() {\n        return blockSize;\n    }\n\n    /**\n     * The size of output if {@code doFinal()} is called with this\n     * {@code inputLen}. If padding is enabled and the size of the input puts it\n     * right at the block size, it will add another block for the padding.\n     */\n    abstract int getOutputSizeForFinal(int inputLen);\n\n    /**\n     * The size of output if {@code update()} is called with this\n     * {@code inputLen}. If padding is enabled and the size of the input puts it\n     * right at the block size, it will add another block for the padding.\n     */\n    abstract int getOutputSizeForUpdate(int inputLen);\n\n    @Override\n    protected int engineGetOutputSize(int inputLen) {\n        return Math.max(getOutputSizeForUpdate(inputLen), getOutputSizeForFinal(inputLen));\n    }\n\n    @Override\n    protected byte[] engineGetIV() {\n        return iv;\n    }\n\n    @Override\n    protected AlgorithmParameters engineGetParameters() {\n        if (iv != null && iv.length > 0) {\n            try {\n                AlgorithmParameters params = AlgorithmParameters.getInstance(getBaseCipherName());\n                params.init(new IvParameterSpec(iv));\n                return params;\n            } catch (NoSuchAlgorithmException e) {\n                return null;\n            } catch (InvalidParameterSpecException e) {\n                return null;\n            }\n        }\n        return null;\n    }\n\n    protected AlgorithmParameterSpec getParameterSpec(AlgorithmParameters params)\n            throws InvalidAlgorithmParameterException {\n        if (params != null) {\n            try {\n                return params.getParameterSpec(IvParameterSpec.class);\n            } catch (InvalidParameterSpecException e) {\n                throw new InvalidAlgorithmParameterException(\n                        \"Params must be convertible to IvParameterSpec\", e);\n            }\n        }\n        return null;\n    }\n\n    @Override\n    protected void engineInit(int opmode, Key key, SecureRandom random) throws InvalidKeyException {\n        checkAndSetEncodedKey(opmode, key);\n        try {\n            engineInitInternal(this.encodedKey, null, random);\n        } catch (InvalidAlgorithmParameterException e) {\n            // This can't actually happen since we pass in null.\n            throw new RuntimeException(e);\n        }\n    }\n\n    @Override\n    protected void engineInit(int opmode, Key key, AlgorithmParameterSpec params,\n            SecureRandom random) throws InvalidKeyException, InvalidAlgorithmParameterException {\n        checkAndSetEncodedKey(opmode, key);\n        engineInitInternal(this.encodedKey, params, random);\n    }\n\n    @Override\n    protected void engineInit(int opmode, Key key, AlgorithmParameters params, SecureRandom random)\n            throws InvalidKeyException, InvalidAlgorithmParameterException {\n        AlgorithmParameterSpec spec = getParameterSpec(params);\n        engineInit(opmode, key, spec, random);\n    }\n\n    @Override\n    protected byte[] engineUpdate(byte[] input, int inputOffset, int inputLen) {\n        final int maximumLen = getOutputSizeForUpdate(inputLen);\n\n        /* See how large our output buffer would need to be. */\n        final byte[] output;\n        if (maximumLen > 0) {\n            output = new byte[maximumLen];\n        } else {\n            output = EmptyArray.BYTE;\n        }\n\n        final int bytesWritten;\n        try {\n            bytesWritten = updateInternal(input, inputOffset, inputLen, output, 0, maximumLen);\n        } catch (ShortBufferException e) {\n            /* This shouldn't happen. */\n            throw new RuntimeException(\"calculated buffer size was wrong: \" + maximumLen);\n        }\n\n        if (output.length == bytesWritten) {\n            return output;\n        } else if (bytesWritten == 0) {\n            return EmptyArray.BYTE;\n        } else {\n            return Arrays.copyOfRange(output, 0, bytesWritten);\n        }\n    }\n\n    @Override\n    protected int engineUpdate(byte[] input, int inputOffset, int inputLen, byte[] output,\n            int outputOffset) throws ShortBufferException {\n        final int maximumLen = getOutputSizeForUpdate(inputLen);\n        return updateInternal(input, inputOffset, inputLen, output, outputOffset, maximumLen);\n    }\n\n    @Override\n    protected byte[] engineDoFinal(byte[] input, int inputOffset, int inputLen)\n            throws IllegalBlockSizeException, BadPaddingException {\n        final int maximumLen = getOutputSizeForFinal(inputLen);\n        /* Assume that we'll output exactly on a byte boundary. */\n        final byte[] output = new byte[maximumLen];\n\n        int bytesWritten;\n        if (inputLen > 0) {\n            try {\n                bytesWritten = updateInternal(input, inputOffset, inputLen, output, 0, maximumLen);\n            } catch (ShortBufferException e) {\n                /* This should not happen since we sized our own buffer. */\n                throw new RuntimeException(\"our calculated buffer was too small\", e);\n            }\n        } else {\n            bytesWritten = 0;\n        }\n\n        try {\n            bytesWritten += doFinalInternal(output, bytesWritten, maximumLen - bytesWritten);\n        } catch (ShortBufferException e) {\n            /* This should not happen since we sized our own buffer. */\n            throw new RuntimeException(\"our calculated buffer was too small\", e);\n        }\n\n        if (bytesWritten == output.length) {\n            return output;\n        } else if (bytesWritten == 0) {\n            return EmptyArray.BYTE;\n        } else {\n            return Arrays.copyOfRange(output, 0, bytesWritten);\n        }\n    }\n\n    @Override\n    protected int engineDoFinal(byte[] input, int inputOffset, int inputLen, byte[] output,\n            int outputOffset) throws ShortBufferException, IllegalBlockSizeException,\n            BadPaddingException {\n        if (output == null) {\n            throw new NullPointerException(\"output == null\");\n        }\n\n        int maximumLen = getOutputSizeForFinal(inputLen);\n\n        final int bytesWritten;\n        if (inputLen > 0) {\n            bytesWritten = updateInternal(input, inputOffset, inputLen, output, outputOffset,\n                    maximumLen);\n            outputOffset += bytesWritten;\n            maximumLen -= bytesWritten;\n        } else {\n            bytesWritten = 0;\n        }\n\n        return bytesWritten + doFinalInternal(output, outputOffset, maximumLen);\n    }\n\n    @Override\n    protected byte[] engineWrap(Key key) throws IllegalBlockSizeException, InvalidKeyException {\n        try {\n            byte[] encoded = key.getEncoded();\n            return engineDoFinal(encoded, 0, encoded.length);\n        } catch (BadPaddingException e) {\n            IllegalBlockSizeException newE = new IllegalBlockSizeException();\n            newE.initCause(e);\n            throw newE;\n        }\n    }\n\n    @Override\n    protected Key engineUnwrap(byte[] wrappedKey, String wrappedKeyAlgorithm, int wrappedKeyType)\n            throws InvalidKeyException, NoSuchAlgorithmException {\n        try {\n            byte[] encoded = engineDoFinal(wrappedKey, 0, wrappedKey.length);\n            if (wrappedKeyType == Cipher.PUBLIC_KEY) {\n                KeyFactory keyFactory = KeyFactory.getInstance(wrappedKeyAlgorithm);\n                return keyFactory.generatePublic(new X509EncodedKeySpec(encoded));\n            } else if (wrappedKeyType == Cipher.PRIVATE_KEY) {\n                KeyFactory keyFactory = KeyFactory.getInstance(wrappedKeyAlgorithm);\n                return keyFactory.generatePrivate(new PKCS8EncodedKeySpec(encoded));\n            } else if (wrappedKeyType == Cipher.SECRET_KEY) {\n                return new SecretKeySpec(encoded, wrappedKeyAlgorithm);\n            } else {\n                throw new UnsupportedOperationException(\"wrappedKeyType == \" + wrappedKeyType);\n            }\n        } catch (IllegalBlockSizeException e) {\n            throw new InvalidKeyException(e);\n        } catch (BadPaddingException e) {\n            throw new InvalidKeyException(e);\n        } catch (InvalidKeySpecException e) {\n            throw new InvalidKeyException(e);\n        }\n    }\n\n    @Override\n    protected int engineGetKeySize(Key key) throws InvalidKeyException {\n        if (!(key instanceof SecretKey)) {\n            throw new InvalidKeyException(\"Only SecretKey is supported\");\n        }\n        byte[] encodedKey = key.getEncoded();\n        if (encodedKey == null) {\n            throw new InvalidKeyException(\"key.getEncoded() == null\");\n        }\n        checkSupportedKeySize(encodedKey.length);\n        // The return value is in bits\n        return encodedKey.length * 8;\n    }\n\n    \nprivate byte[] checkAndSetEncodedKey(int opmode, Key key) throws InvalidKeyException {\n        if (opmode == Cipher.ENCRYPT_MODE || opmode == Cipher.WRAP_MODE) {\n            encrypting = true;\n        } else if (opmode == Cipher.DECRYPT_MODE || opmode == Cipher.UNWRAP_MODE) {\n            encrypting = false;\n        } else {\n            throw new InvalidParameterException(\"Unsupported opmode \" + opmode);\n        }\n\n        if (!(key instanceof SecretKey)) {\n            throw new InvalidKeyException(\"Only SecretKey is supported\");\n        }\n\n        final byte[] encodedKey = key.getEncoded();\n        if (encodedKey == null) {\n            throw new InvalidKeyException(\"key.getEncoded() == null\");\n        }\n        checkSupportedKeySize(encodedKey.length);\n        this.encodedKey = encodedKey;\n        return encodedKey;\n    }\n\n    boolean isEncrypting() {\n        return encrypting;\n    }\n\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLRSAKeyFactory.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.math.BigInteger;\nimport java.security.InvalidKeyException;\nimport java.security.Key;\nimport java.security.KeyFactorySpi;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.security.interfaces.RSAPrivateCrtKey;\nimport java.security.interfaces.RSAPrivateKey;\nimport java.security.interfaces.RSAPublicKey;\nimport java.security.spec.InvalidKeySpecException;\nimport java.security.spec.KeySpec;\nimport java.security.spec.PKCS8EncodedKeySpec;\nimport java.security.spec.RSAPrivateCrtKeySpec;\nimport java.security.spec.RSAPrivateKeySpec;\nimport java.security.spec.RSAPublicKeySpec;\nimport java.security.spec.X509EncodedKeySpec;\n\n/**\n * An implementation of {@link java.security.KeyFactory} which uses BoringSSL to perform all the\n * operations.\n */\n@Internal\npublic final class OpenSSLRSAKeyFactory extends KeyFactorySpi {\n\n    @Override\n    protected PublicKey engineGeneratePublic(KeySpec keySpec) throws InvalidKeySpecException {\n        if (keySpec == null) {\n            throw new InvalidKeySpecException(\"keySpec == null\");\n        }\n\n        if (keySpec instanceof RSAPublicKeySpec) {\n            return new OpenSSLRSAPublicKey((RSAPublicKeySpec) keySpec);\n        } else if (keySpec instanceof X509EncodedKeySpec) {\n            return OpenSSLKey.getPublicKey((X509EncodedKeySpec) keySpec, NativeConstants.EVP_PKEY_RSA);\n        }\n        throw new InvalidKeySpecException(\"Must use RSAPublicKeySpec or X509EncodedKeySpec; was \"\n                + keySpec.getClass().getName());\n    }\n\n    @Override\n    protected PrivateKey engineGeneratePrivate(KeySpec keySpec) throws InvalidKeySpecException {\n        if (keySpec == null) {\n            throw new InvalidKeySpecException(\"keySpec == null\");\n        }\n\n        if (keySpec instanceof RSAPrivateCrtKeySpec) {\n            return new OpenSSLRSAPrivateCrtKey((RSAPrivateCrtKeySpec) keySpec);\n        } else if (keySpec instanceof RSAPrivateKeySpec) {\n            return new OpenSSLRSAPrivateKey((RSAPrivateKeySpec) keySpec);\n        } else if (keySpec instanceof PKCS8EncodedKeySpec) {\n            return OpenSSLKey.getPrivateKey((PKCS8EncodedKeySpec) keySpec,\n                    NativeConstants.EVP_PKEY_RSA);\n        }\n        throw new InvalidKeySpecException(\"Must use RSAPublicKeySpec or PKCS8EncodedKeySpec; was \"\n                + keySpec.getClass().getName());\n    }\n\n    @Override\n    protected <T extends KeySpec> T engineGetKeySpec(Key key, Class<T> keySpec)\n            throws InvalidKeySpecException {\n        if (key == null) {\n            throw new InvalidKeySpecException(\"key == null\");\n        }\n\n        if (keySpec == null) {\n            throw new InvalidKeySpecException(\"keySpec == null\");\n        }\n\n        if (!\"RSA\".equals(key.getAlgorithm())) {\n            throw new InvalidKeySpecException(\"Key must be a RSA key\");\n        }\n\n        if (key instanceof RSAPublicKey && RSAPublicKeySpec.class.isAssignableFrom(keySpec)) {\n            RSAPublicKey rsaKey = (RSAPublicKey) key;\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new RSAPublicKeySpec(rsaKey.getModulus(), rsaKey.getPublicExponent());\n            return result;\n        } else if (key instanceof PublicKey && RSAPublicKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"X.509\".equals(key.getFormat()) || encoded == null) {\n                throw new InvalidKeySpecException(\"Not a valid X.509 encoding\");\n            }\n            RSAPublicKey rsaKey =\n                    (RSAPublicKey) engineGeneratePublic(new X509EncodedKeySpec(encoded));\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new RSAPublicKeySpec(rsaKey.getModulus(), rsaKey.getPublicExponent());\n            return result;\n        } else if (key instanceof RSAPrivateCrtKey\n                && RSAPrivateCrtKeySpec.class.isAssignableFrom(keySpec)) {\n            RSAPrivateCrtKey rsaKey = (RSAPrivateCrtKey) key;\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new RSAPrivateCrtKeySpec(rsaKey.getModulus(), rsaKey.getPublicExponent(),\n                    rsaKey.getPrivateExponent(), rsaKey.getPrimeP(), rsaKey.getPrimeQ(),\n                    rsaKey.getPrimeExponentP(), rsaKey.getPrimeExponentQ(),\n                    rsaKey.getCrtCoefficient());\n            return result;\n        } else if (key instanceof RSAPrivateCrtKey\n                && RSAPrivateKeySpec.class.isAssignableFrom(keySpec)) {\n            RSAPrivateCrtKey rsaKey = (RSAPrivateCrtKey) key;\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new RSAPrivateKeySpec(rsaKey.getModulus(), rsaKey.getPrivateExponent());\n            return result;\n        } else if (key instanceof RSAPrivateKey\n                && RSAPrivateKeySpec.class.isAssignableFrom(keySpec)) {\n            RSAPrivateKey rsaKey = (RSAPrivateKey) key;\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new RSAPrivateKeySpec(rsaKey.getModulus(), rsaKey.getPrivateExponent());\n            return result;\n        } else if (key instanceof PrivateKey\n                && RSAPrivateCrtKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"PKCS#8\".equals(key.getFormat()) || encoded == null) {\n                throw new InvalidKeySpecException(\"Not a valid PKCS#8 encoding\");\n            }\n            RSAPrivateKey privKey =\n                    (RSAPrivateKey) engineGeneratePrivate(new PKCS8EncodedKeySpec(encoded));\n            if (privKey instanceof RSAPrivateCrtKey) {\n                RSAPrivateCrtKey rsaKey = (RSAPrivateCrtKey) privKey;\n                @SuppressWarnings(\"unchecked\")\n                T result = (T) new RSAPrivateCrtKeySpec(rsaKey.getModulus(),\n                        rsaKey.getPublicExponent(), rsaKey.getPrivateExponent(), rsaKey.getPrimeP(),\n                        rsaKey.getPrimeQ(), rsaKey.getPrimeExponentP(), rsaKey.getPrimeExponentQ(),\n                        rsaKey.getCrtCoefficient());\n                return result;\n            } else {\n                throw new InvalidKeySpecException(\"Encoded key is not an RSAPrivateCrtKey\");\n            }\n        } else if (key instanceof PrivateKey && RSAPrivateKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"PKCS#8\".equals(key.getFormat()) || encoded == null) {\n                throw new InvalidKeySpecException(\"Not a valid PKCS#8 encoding\");\n            }\n            RSAPrivateKey rsaKey =\n                    (RSAPrivateKey) engineGeneratePrivate(new PKCS8EncodedKeySpec(encoded));\n            @SuppressWarnings(\"unchecked\")\n            T result = (T) new RSAPrivateKeySpec(rsaKey.getModulus(), rsaKey.getPrivateExponent());\n            return result;\n        } else if (key instanceof PrivateKey\n                && PKCS8EncodedKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"PKCS#8\".equals(key.getFormat())) {\n                throw new InvalidKeySpecException(\"Encoding type must be PKCS#8; was \"\n                        + key.getFormat());\n            } else if (encoded == null) {\n                throw new InvalidKeySpecException(\"Key is not encodable\");\n            }\n            @SuppressWarnings(\"unchecked\") T result = (T) new PKCS8EncodedKeySpec(encoded);\n            return result;\n        } else if (key instanceof PublicKey && X509EncodedKeySpec.class.isAssignableFrom(keySpec)) {\n            final byte[] encoded = key.getEncoded();\n            if (!\"X.509\".equals(key.getFormat())) {\n                throw new InvalidKeySpecException(\"Encoding type must be X.509; was \"\n                        + key.getFormat());\n            } else if (encoded == null) {\n                throw new InvalidKeySpecException(\"Key is not encodable\");\n            }\n            @SuppressWarnings(\"unchecked\") T result = (T) new X509EncodedKeySpec(encoded);\n            return result;\n        } else {\n            throw new InvalidKeySpecException(\"Unsupported key type and key spec combination; key=\"\n                    + key.getClass().getName() + \", keySpec=\" + keySpec.getName());\n        }\n    }\n\n    @Override\n    protected Key engineTranslateKey(Key key) throws InvalidKeyException {\n        if (key == null) {\n            throw new InvalidKeyException(\"key == null\");\n        }\n\n        if ((key instanceof OpenSSLRSAPublicKey) || (key instanceof OpenSSLRSAPrivateKey)) {\n            return key;\n        } else if (key instanceof RSAPublicKey) {\n            RSAPublicKey rsaKey = (RSAPublicKey) key;\n\n            try {\n                return engineGeneratePublic(new RSAPublicKeySpec(rsaKey.getModulus(),\n                        rsaKey.getPublicExponent()));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else if (key instanceof RSAPrivateCrtKey) {\n            RSAPrivateCrtKey rsaKey = (RSAPrivateCrtKey) key;\n            BigInteger modulus = rsaKey.getModulus();\n            BigInteger publicExponent = rsaKey.getPublicExponent();\n            BigInteger privateExponent = rsaKey.getPrivateExponent();\n            BigInteger primeP = rsaKey.getPrimeP();\n            BigInteger primeQ = rsaKey.getPrimeQ();\n            BigInteger primeExponentP = rsaKey.getPrimeExponentP();\n            BigInteger primeExponentQ = rsaKey.getPrimeExponentQ();\n            BigInteger crtCoefficient = rsaKey.getCrtCoefficient();\n\n            try {\n                return engineGeneratePrivate(new RSAPrivateCrtKeySpec(modulus, publicExponent,\n                        privateExponent, primeP, primeQ, primeExponentP, primeExponentQ,\n                        crtCoefficient));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else if (key instanceof RSAPrivateKey) {\n            RSAPrivateKey rsaKey = (RSAPrivateKey) key;\n            BigInteger modulus = rsaKey.getModulus();\n            BigInteger privateExponent = rsaKey.getPrivateExponent();\n\n            try {\n                return engineGeneratePrivate(new RSAPrivateKeySpec(modulus, privateExponent));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else if ((key instanceof PrivateKey) && \"PKCS#8\".equals(key.getFormat())) {\n            byte[] encoded = key.getEncoded();\n            if (encoded == null) {\n                throw new InvalidKeyException(\"Key does not support encoding\");\n            }\n            try {\n                return engineGeneratePrivate(new PKCS8EncodedKeySpec(encoded));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else if ((key instanceof PublicKey) && \"X.509\".equals(key.getFormat())) {\n            byte[] encoded = key.getEncoded();\n            if (encoded == null) {\n                throw new InvalidKeyException(\"Key does not support encoding\");\n            }\n            try {\n                return engineGeneratePublic(new X509EncodedKeySpec(encoded));\n            } catch (InvalidKeySpecException e) {\n                throw new InvalidKeyException(e);\n            }\n        } else {\n            throw new InvalidKeyException(\"Key must be an RSA public or private key; was \"\n                    + key.getClass().getName());\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ConscryptCertStore.java\n/*\n * Copyright 2018 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.cert.X509Certificate;\nimport java.util.Set;\n\n/**\n * A certificate store that supports additional operations that are used in\n * TrustManagerImpl.  This is primarily implemented by the cert store on the\n * Android platform.\n */\n@Internal\npublic interface ConscryptCertStore {\n\n    /**\n     * Returns a stored CA certificate with the same name and public key as the\n     * provided {@link X509Certificate}.\n     */\n    X509Certificate getTrustAnchor(X509Certificate c);\n\n    /**\n     * Returns all CA certificates with the public key that was used to sign the\n     * provided {@link X509Certificate}.\n     */\n    Set<X509Certificate> findAllIssuers(X509Certificate c);\n}\n\n// Path: common/src/main/java/org/conscrypt/ConscryptHostnameVerifier.java\n/*\n * Copyright 2019 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.cert.X509Certificate;\nimport javax.net.ssl.SSLSession;\n\n/**\n * This interface is used to implement hostname verification in Conscrypt.  Unlike with\n * {@link javax.net.ssl.HostnameVerifier}, the hostname verifier is called whenever hostname\n * verification is needed, without any use of default rules.\n */\npublic interface ConscryptHostnameVerifier {\n\n  /**\n   * Returns whether the given hostname is allowable given the peer's authentication information\n   * from the given session.\n   */\n  boolean verify(X509Certificate[] certs, String hostname, SSLSession session);\n\n}\n\n// Path: common/src/main/java/org/conscrypt/DefaultSSLContextImpl.java\n/*\n *  Licensed to the Apache Software Foundation (ASF) under one or more\n *  contributor license agreements.  See the NOTICE file distributed with\n *  this work for additional information regarding copyright ownership.\n *  The ASF licenses this file to You under the Apache License, Version 2.0\n *  (the \"License\"); you may not use this file except in compliance with\n *  the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n *  Unless required by applicable law or agreed to in writing, software\n *  distributed under the License is distributed on an \"AS IS\" BASIS,\n *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n *  See the License for the specific language governing permissions and\n *  limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.BufferedInputStream;\nimport java.io.FileInputStream;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.security.GeneralSecurityException;\nimport java.security.KeyManagementException;\nimport java.security.KeyStore;\nimport java.security.SecureRandom;\nimport javax.net.ssl.KeyManager;\nimport javax.net.ssl.KeyManagerFactory;\nimport javax.net.ssl.TrustManager;\nimport javax.net.ssl.TrustManagerFactory;\n\n/**\n * Support class for this package.\n */\n@Internal\npublic class DefaultSSLContextImpl extends OpenSSLContextImpl {\n\n    /**\n     * Accessed by SSLContextImpl(DefaultSSLContextImpl) holding the\n     * DefaultSSLContextImpl.class monitor\n     */\n    private static KeyManager[] KEY_MANAGERS;\n\n    /**\n     * Accessed by SSLContextImpl(DefaultSSLContextImpl) holding the\n     * DefaultSSLContextImpl.class monitor\n     */\n    private static TrustManager[] TRUST_MANAGERS;\n\n    /**\n     * DefaultSSLContextImpl delegates the work to the super class since there\n     * is no way to put a synchronized around both the call to super and the\n     * rest of this constructor to guarantee that we don't have races in\n     * creating the state shared between all default SSLContexts.\n     */\n    private DefaultSSLContextImpl(String[] protocols) throws GeneralSecurityException, IOException {\n        super(protocols, true);\n    }\n\n    // TODO javax.net.ssl.keyStoreProvider system property\n    KeyManager[] getKeyManagers() throws GeneralSecurityException, IOException {\n        if (KEY_MANAGERS != null) {\n            return KEY_MANAGERS;\n        }\n        // find KeyStore, KeyManagers\n        String keystore = System.getProperty(\"javax.net.ssl.keyStore\");\n        if (keystore == null) {\n            return null;\n        }\n        String keystorepwd = System.getProperty(\"javax.net.ssl.keyStorePassword\");\n        char[] pwd = (keystorepwd == null) ? null : keystorepwd.toCharArray();\n\n        KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());\n        InputStream is = null;\n        try {\n            is = new BufferedInputStream(new FileInputStream(keystore));\n            ks.load(is, pwd);\n        } finally {\n            if (is != null) {\n                is.close();\n            }\n        }\n\n        String kmfAlg = KeyManagerFactory.getDefaultAlgorithm();\n        KeyManagerFactory kmf = KeyManagerFactory.getInstance(kmfAlg);\n        kmf.init(ks, pwd);\n        KEY_MANAGERS = kmf.getKeyManagers();\n        return KEY_MANAGERS;\n    }\n\n    // TODO javax.net.ssl.trustStoreProvider system property\n    TrustManager[] getTrustManagers() throws GeneralSecurityException, IOException {\n        if (TRUST_MANAGERS != null) {\n            return TRUST_MANAGERS;\n        }\n\n        // find TrustStore, TrustManagers\n        String keystore = System.getProperty(\"javax.net.ssl.trustStore\");\n        if (keystore == null) {\n            return null;\n        }\n        String keystorepwd = System.getProperty(\"javax.net.ssl.trustStorePassword\");\n        char[] pwd = (keystorepwd == null) ? null : keystorepwd.toCharArray();\n\n        // TODO Defaults: jssecacerts; cacerts\n        KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());\n        InputStream is = null;\n        try {\n            is = new BufferedInputStream(new FileInputStream(keystore));\n            ks.load(is, pwd);\n        } finally {\n            if (is != null) {\n                is.close();\n            }\n        }\n        String tmfAlg = TrustManagerFactory.getDefaultAlgorithm();\n        TrustManagerFactory tmf = TrustManagerFactory.getInstance(tmfAlg);\n        tmf.init(ks);\n        TRUST_MANAGERS = tmf.getTrustManagers();\n        return TRUST_MANAGERS;\n    }\n\n    @Override\n    public void engineInit(KeyManager[] kms, TrustManager[] tms,\n...\n", "needle_token_start": 10286, "needle_token_end": 10516, "code_context_ntokens": 16021, "output": ["```java\nprivate byte[] checkAndSetEncodedKey(int opmode, Key key) throws InvalidKeyException {\n    if (opmode == Cipher.ENCRYPT_MODE || opmode == Cipher.WRAP_MODE) {\n        encrypting = true;\n    } else if (opmode == Cipher.DECRYPT_MODE || opmode == Cipher.UNWRAP_MODE) {\n        encrypting = false;\n    } else {\n        throw new InvalidParameterException(\"Unsupported opmode \" + opmode);\n    }\n\n    if (!(key instanceof SecretKey)) {\n        throw new InvalidKeyException(\"Only SecretKey is supported\");\n    }\n\n    final byte[] encodedKey = key.getEncoded();\n    if (encodedKey == null) {\n        throw new InvalidKeyException(\"key.getEncoded() == null\");\n    }\n    checkSupportedKeySize(encodedKey.length);\n    this.encodedKey = encodedKey;\n    return encodedKey;\n}\n```"]}
{"repo": "google/conscrypt", "name": "isChannelIdEnabled", "language": "java", "path": "common/src/main/java/org/conscrypt/ConscryptServerSocket.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To determine if the TLS Channel ID extension is active for a server socket, which is used for additional security measures in client-server communications.\n2. **Input**: None.\n3. **Output**: A boolean value indicating whether the extension is enabled (true) or not (false).\n4. **Procedure**: Simply returns the state of a private boolean field that represents whether the TLS Channel ID extension is enabled.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/CertificatePriorityComparator.java\n/*\n * Copyright (C) 2016 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\npackage org.conscrypt;\n\nimport java.security.PublicKey;\nimport java.security.cert.X509Certificate;\nimport java.security.interfaces.ECPublicKey;\nimport java.security.interfaces.RSAPublicKey;\nimport java.util.Comparator;\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * {@link Comparator} for prioritizing certificates in path building.\n *\n * <p>\n * The sort order is as follows:\n * <ol>\n * <li>Self-issued certificates first.</li>\n * <li>Strength of certificates descending (EC before RSA, key size descending, signature\n * algorithm strength descending).</li>\n * <li>notAfter date descending.</li>\n * <li>notBefore date descending.</li>\n * </ol>\n * </p>\n */\n@Internal\npublic final class CertificatePriorityComparator implements Comparator<X509Certificate> {\n\n    /**\n     * Map of signature algorithm OIDs to priorities. OIDs with a lower priority will be sorted\n     * before those with higher.\n     */\n    private static final Map<String, Integer> ALGORITHM_OID_PRIORITY_MAP;\n\n    /*\n     * Priorities of digest algorithms. Lower is better.\n     */\n    private static final Integer PRIORITY_MD5 = 6;\n    private static final Integer PRIORITY_SHA1 = 5;\n    private static final Integer PRIORITY_SHA224 = 4;\n    private static final Integer PRIORITY_SHA256 = 3;\n    private static final Integer PRIORITY_SHA384 = 2;\n    private static final Integer PRIORITY_SHA512 = 1;\n    private static final Integer PRIORITY_UNKNOWN = -1;\n    static {\n        ALGORITHM_OID_PRIORITY_MAP = new HashMap<String, Integer>();\n        // RSA oids\n        ALGORITHM_OID_PRIORITY_MAP.put(\"1.2.840.113549.1.1.13\", PRIORITY_SHA512);\n        ALGORITHM_OID_PRIORITY_MAP.put(\"1.2.840.113549.1.1.12\", PRIORITY_SHA384);\n        ALGORITHM_OID_PRIORITY_MAP.put(\"1.2.840.113549.1.1.11\", PRIORITY_SHA256);\n        ALGORITHM_OID_PRIORITY_MAP.put(\"1.2.840.113549.1.1.14\", PRIORITY_SHA224);\n        ALGORITHM_OID_PRIORITY_MAP.put(\"1.2.840.113549.1.1.5\", PRIORITY_SHA1);\n        ALGORITHM_OID_PRIORITY_MAP.put(\"1.2.840.113549.1.1.4\", PRIORITY_MD5);\n        // ECDSA oids\n...\n// Path: common/src/main/java/org/conscrypt/OpenSSLX25519PrivateKey.java\n/*\n * Copyright 2022 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\nimport java.security.InvalidKeyException;\nimport java.security.PrivateKey;\nimport java.security.spec.EncodedKeySpec;\nimport java.security.spec.InvalidKeySpecException;\nimport java.util.Arrays;\n\npublic class OpenSSLX25519PrivateKey implements OpenSSLX25519Key, PrivateKey {\n    private static final long serialVersionUID = -3136201500221850916L;\n    private static final byte[] PKCS8_PREAMBLE = new byte[]{\n            0x30, 0x2e,                            // Sequence: 46 bytes\n                0x02, 0x01, 0x00,                  // Integer: 0 (version)\n                0x30, 0x05,                        // Sequence: 5 bytes\n                    0x06, 0x03, 0x2b, 0x65, 0x6e,  // OID: 1.3.101.110 (X25519)\n                0x04, 0x22, 0x04, 0x20,            // Octet string: 32 bytes\n            // Key bytes follow directly\n    };\n\n    private byte[] uCoordinate;\n\n    public OpenSSLX25519PrivateKey(EncodedKeySpec keySpec)\n            throws InvalidKeySpecException {\n        byte[] encoded = keySpec.getEncoded();\n        if (\"PKCS#8\".equals(keySpec.getFormat())) {\n            try {\n                uCoordinate = NativeCrypto.EVP_raw_X25519_private_key(encoded);\n            } catch (InvalidKeyException | ParsingException e) {\n                throw new InvalidKeySpecException(e);\n            }\n        } else if (\"raw\".equalsIgnoreCase(keySpec.getFormat())) {\n            uCoordinate = encoded;\n        } else {\n            throw new InvalidKeySpecException(\"Encoding must be in PKCS#8 or raw format\");\n        }\n        if (uCoordinate.length != X25519_KEY_SIZE_BYTES) {\n            throw new InvalidKeySpecException(\"Invalid key size\");\n        }\n    }\n\n    public OpenSSLX25519PrivateKey(byte[] coordinateBytes) {\n        if (coordinateBytes.length != X25519_KEY_SIZE_BYTES) {\n            throw new IllegalArgumentException(\"Invalid key size\");\n        }\n        uCoordinate = coordinateBytes.clone();\n    }\n\n    @Override\n    public String getAlgorithm() {\n        return \"XDH\";\n    }\n\n    @Override\n    public String getFormat() {\n        return \"PKCS#8\";\n    }\n\n    @Override\n    public byte[] getEncoded() {\n        if (uCoordinate == null) {\n            throw new IllegalStateException(\"key is destroyed\");\n        }\n        return ArrayUtils.concat(PKCS8_PREAMBLE, uCoordinate);\n    }\n\n    @Override\n    public byte[] getU() {\n        if (uCoordinate == null) {\n            throw new IllegalStateException(\"key is destroyed\");\n        }\n        return uCoordinate.clone();\n    }\n\n    @Override\n    public void destroy() {\n        if (uCoordinate != null) {\n            Arrays.fill(uCoordinate, (byte) 0);\n            uCoordinate = null;\n        }\n    }\n\n    @Override\n    public boolean isDestroyed() {\n        return uCoordinate == null;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) return true;\n        if (!(o instanceof OpenSSLX25519PrivateKey)) return false;\n        OpenSSLX25519PrivateKey that = (OpenSSLX25519PrivateKey) o;\n        return Arrays.equals(uCoordinate, that.uCoordinate);\n    }\n\n    @Override\n    public int hashCode() {\n        return Arrays.hashCode(uCoordinate);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/HpkeDecryptException.java\npackage org.conscrypt;\n\nimport java.security.GeneralSecurityException;\n\npublic class HpkeDecryptException extends GeneralSecurityException {\n    private static final long serialVersionUID = 5903211285098828754L;\n\n    public HpkeDecryptException(String msg) {\n        super(msg);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLEvpCipherDESEDE.java\n/*\n * Copyright (C) 2019 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidKeyException;\nimport java.security.NoSuchAlgorithmException;\nimport java.util.Locale;\nimport javax.crypto.NoSuchPaddingException;\n\n@Internal\npublic abstract class OpenSSLEvpCipherDESEDE extends OpenSSLEvpCipher {\n    private static final int DES_BLOCK_SIZE = 8;\n\n    OpenSSLEvpCipherDESEDE(Mode mode, Padding padding) {\n        super(mode, padding);\n    }\n\n    public static class CBC extends OpenSSLEvpCipherDESEDE {\n        CBC(Padding padding) {\n            super(Mode.CBC, padding);\n        }\n\n        public static class NoPadding extends CBC {\n            public NoPadding() {\n                super(Padding.NOPADDING);\n            }\n        }\n\n        public static class PKCS5Padding extends CBC {\n            public PKCS5Padding() {\n                super(Padding.PKCS5PADDING);\n            }\n        }\n    }\n\n    @Override\n    String getBaseCipherName() {\n        return \"DESede\";\n    }\n\n    @Override\n    String getCipherName(int keySize, Mode mode) {\n        final String baseCipherName;\n        if (keySize == 16) {\n            baseCipherName = \"des-ede\";\n        } else {\n            baseCipherName = \"des-ede3\";\n        }\n\n        return baseCipherName + \"-\" + mode.toString().toLowerCase(Locale.US);\n    }\n\n    @Override\n    void checkSupportedKeySize(int keySize) throws InvalidKeyException {\n        if (keySize != 16 && keySize != 24) {\n            throw new InvalidKeyException(\"key size must be 128 or 192 bits\");\n        }\n    }\n\n    @Override\n    void checkSupportedMode(Mode mode) throws NoSuchAlgorithmException {\n        if (mode != Mode.CBC) {\n            throw new NoSuchAlgorithmException(\"Unsupported mode \" + mode.toString());\n        }\n    }\n\n    @Override\n    void checkSupportedPadding(Padding padding) throws NoSuchPaddingException {\n        switch (padding) {\n            case NOPADDING:\n            case PKCS5PADDING:\n                return;\n            default:\n                throw new NoSuchPaddingException(\"Unsupported padding \"\n                        + padding.toString());\n        }\n    }\n\n    @Override\n    int getCipherBlockSize() {\n        return DES_BLOCK_SIZE;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/SSLClientSessionCache.java\n/*\n * Copyright (C) 2009 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport javax.net.ssl.SSLSession;\n\n/**\n * A persistent {@link javax.net.ssl.SSLSession} cache used by\n * {@link javax.net.ssl.SSLSessionContext} to share client-side SSL sessions\n * across processes. For example, this cache enables applications to\n * persist and reuse sessions across restarts.\n *\n * <p>The {@code SSLSessionContext} implementation converts\n * {@code SSLSession}s into raw bytes and vice versa. The exact makeup of the\n * session data is dependent upon the caller's implementation and is opaque to\n * the {@code SSLClientSessionCache} implementation.\n */\npublic interface SSLClientSessionCache {\n    /**\n     * Gets data from a pre-existing session for a given server host and port.\n     *\n     * @param host from {@link javax.net.ssl.SSLSession#getPeerHost()}\n     * @param port from {@link javax.net.ssl.SSLSession#getPeerPort()}\n     * @return the session data or null if none is cached\n     * @throws NullPointerException if host is null\n     */\n    byte[] getSessionData(String host, int port);\n\n    /**\n     * Stores session data for the given session.\n     *\n     * @param session to cache data for\n     * @param sessionData to cache\n     * @throws NullPointerException if session, result of\n     *  {@code session.getPeerHost()} or data is null\n     */\n    void putSessionData(SSLSession session, byte[] sessionData);\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLCipherChaCha20.java\n/*\n * Copyright (C) 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport javax.crypto.NoSuchPaddingException;\nimport javax.crypto.ShortBufferException;\nimport javax.crypto.spec.IvParameterSpec;\n\n/**\n * Implementation of the ChaCha20 stream cipher.\n */\n@Internal\npublic class OpenSSLCipherChaCha20 extends OpenSSLCipher {\n\n    private static final int BLOCK_SIZE_BYTES = 64;\n    private static final int NONCE_SIZE_BYTES = 12;\n\n    // BoringSSL's interface encrypts by the block, so we need to keep track of whether we\n    // had unused keystream bytes at the end of the previous encryption operation, so that\n    // we can use them before moving on to the next block.\n    private int currentBlockConsumedBytes = 0;\n    private int blockCounter = 0;\n\n    public OpenSSLCipherChaCha20() {}\n\n    @Override\n    void engineInitInternal(byte[] encodedKey, AlgorithmParameterSpec params, SecureRandom random)\n            throws InvalidAlgorithmParameterException {\n        if (params instanceof IvParameterSpec) {\n            IvParameterSpec ivParams = (IvParameterSpec) params;\n            if (ivParams.getIV().length != NONCE_SIZE_BYTES) {\n                throw new InvalidAlgorithmParameterException(\"IV must be 12 bytes long\");\n            }\n            iv = ivParams.getIV();\n        } else {\n            if (!isEncrypting()) {\n                throw new InvalidAlgorithmParameterException(\n                        \"IV must be specified when decrypting\");\n            }\n            iv = new byte[NONCE_SIZE_BYTES];\n            if (random != null) {\n                random.nextBytes(iv);\n            } else {\n                NativeCrypto.RAND_bytes(iv);\n            }\n        }\n    }\n\n    @Override\n    int updateInternal(byte[] input, int inputOffset, int inputLen, byte[] output, int outputOffset,\n            int maximumLen) throws ShortBufferException {\n        if (inputLen > output.length - outputOffset) {\n            throw new ShortBufferWithoutStackTraceException(\"Insufficient output space\");\n        }\n        int inputLenRemaining = inputLen;\n        if (currentBlockConsumedBytes > 0) {\n            // A previous operation ended with a partial block, so we need to encrypt using\n            // the remainder of that block before beginning to use the next block\n            int len = Math.min(BLOCK_SIZE_BYTES - currentBlockConsumedBytes, inputLenRemaining);\n            byte[] singleBlock = new byte[BLOCK_SIZE_BYTES];\n            byte[] singleBlockOut = new byte[BLOCK_SIZE_BYTES];\n            System.arraycopy(input, inputOffset, singleBlock, currentBlockConsumedBytes, len);\n            NativeCrypto.chacha20_encrypt_decrypt(singleBlock, 0, singleBlockOut, 0,\n                    BLOCK_SIZE_BYTES, encodedKey, iv, blockCounter);\n            System.arraycopy(singleBlockOut, currentBlockConsumedBytes, output, outputOffset, len);\n            currentBlockConsumedBytes += len;\n            if (currentBlockConsumedBytes < BLOCK_SIZE_BYTES) {\n                // We still didn't finish this block, so we're done.\n                return len;\n            }\n            currentBlockConsumedBytes = 0;\n            inputOffset += len;\n            outputOffset += len;\n            inputLenRemaining -= len;\n            blockCounter++;\n        }\n        NativeCrypto.chacha20_encrypt_decrypt(input, inputOffset, output,\n                outputOffset, inputLenRemaining, encodedKey, iv, blockCounter);\n        currentBlockConsumedBytes = inputLenRemaining % BLOCK_SIZE_BYTES;\n        blockCounter += inputLenRemaining / BLOCK_SIZE_BYTES;\n        return inputLen;\n    }\n\n    @Override\n    int doFinalInternal(byte[] output, int outputOffset, int maximumLen) {\n        reset();\n        return 0;\n    }\n\n    private void reset() {\n        blockCounter = 0;\n        currentBlockConsumedBytes = 0;\n    }\n\n    @Override\n    String getBaseCipherName() {\n        return \"ChaCha20\";\n    }\n\n    @Override\n    void checkSupportedKeySize(int keySize) throws InvalidKeyException {\n        if (keySize != 32) {\n            throw new InvalidKeyException(\"Unsupported key size: \" + keySize\n                    + \" bytes (must be 32)\");\n        }\n    }\n\n    @Override\n    void checkSupportedMode(Mode mode) throws NoSuchAlgorithmException {\n        if (mode != Mode.NONE) {\n            throw new NoSuchAlgorithmException(\"Mode must be NONE\");\n        }\n    }\n\n    @Override\n    void checkSupportedPadding(Padding padding) throws NoSuchPaddingException {\n        if (padding != Padding.NOPADDING) {\n            throw new NoSuchPaddingException(\"Must be NoPadding\");\n        }\n    }\n\n    @Override\n    int getCipherBlockSize() {\n        return 0;\n    }\n\n    @Override\n    int getOutputSizeForFinal(int inputLen) {\n        return inputLen;\n    }\n\n    @Override\n    int getOutputSizeForUpdate(int inputLen) {\n        return inputLen;\n    }\n\n}\n\n// Path: common/src/main/java/org/conscrypt/AbstractConscryptEngine.java\n/*\n * Copyright (C) 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.nio.ByteBuffer;\nimport java.security.PrivateKey;\nimport javax.net.ssl.SSLEngine;\nimport javax.net.ssl.SSLEngineResult;\nimport javax.net.ssl.SSLException;\nimport javax.net.ssl.SSLSession;\n\n/**\n * Abstract base class for all Conscrypt {@link SSLEngine} classes.\n */\nabstract class AbstractConscryptEngine extends SSLEngine {\n    abstract void setBufferAllocator(BufferAllocator bufferAllocator);\n\n    /**\n     * Returns the maximum overhead, in bytes, of sealing a record with SSL.\n     */\n    abstract int maxSealOverhead();\n\n    /**\n     * Enables/disables TLS Channel ID for this server engine.\n     *\n     * <p>This method needs to be invoked before the handshake starts.\n     *\n     * @throws IllegalStateException if this is a client engine or if the handshake has already\n     *         started.\n     */\n    abstract void setChannelIdEnabled(boolean enabled);\n\n    /**\n     * Gets the TLS Channel ID for this server engine. Channel ID is only available once the\n     * handshake completes.\n     *\n     * @return channel ID or {@code null} if not available.\n     *\n     * @throws IllegalStateException if this is a client engine or if the handshake has not yet\n     * completed.\n     * @throws SSLException if channel ID is available but could not be obtained.\n     */\n    abstract byte[] getChannelId() throws SSLException;\n\n    /**\n     * Sets the {@link PrivateKey} to be used for TLS Channel ID by this client engine.\n     *\n     * <p>This method needs to be invoked before the handshake starts.\n     *\n     * @param privateKey private key (enables TLS Channel ID) or {@code null} for no key (disables\n     *        TLS Channel ID). The private key must be an Elliptic Curve (EC) key based on the NIST\n     *        P-256 curve (aka SECG secp256r1 or ANSI X9.62 prime256v1).\n     *\n     * @throws IllegalStateException if this is a server engine or if the handshake has already\n     *         started.\n     */\n    abstract void setChannelIdPrivateKey(PrivateKey privateKey);\n\n    /**\n     * Sets the listener for the completion of the TLS handshake.\n     */\n    abstract void setHandshakeListener(HandshakeListener handshakeListener);\n\n    /**\n     * This method enables Server Name Indication (SNI) and overrides the {@link PeerInfoProvider}\n     * supplied during engine creation.\n     */\n    abstract void setHostname(String hostname);\n\n    /**\n     * Returns the hostname from {@link #setHostname(String)} or supplied by the\n     * {@link PeerInfoProvider} upon creation. No DNS resolution is attempted before\n     * returning the hostname.\n     */\n    abstract String getHostname();\n\n    @Override public abstract String getPeerHost();\n\n    @Override public abstract int getPeerPort();\n\n    /* @Override */\n    @SuppressWarnings(\"MissingOverride\") // For compilation with Java 6.\n    public final SSLSession getHandshakeSession() {\n        return handshakeSession();\n    }\n\n    /**\n     * Work-around to allow this method to be called on older versions of Android.\n     */\n    abstract SSLSession handshakeSession();\n\n    @Override\n    public abstract SSLEngineResult unwrap(ByteBuffer src, ByteBuffer dst) throws SSLException;\n\n    @Override\n    public abstract SSLEngineResult unwrap(ByteBuffer src, ByteBuffer[] dsts) throws SSLException;\n\n    @Override\n    public abstract SSLEngineResult unwrap(final ByteBuffer src, final ByteBuffer[] dsts,\n            final int offset, final int length) throws SSLException;\n\n    abstract SSLEngineResult unwrap(final ByteBuffer[] srcs, final ByteBuffer[] dsts)\n            throws SSLException;\n\n    abstract SSLEngineResult unwrap(final ByteBuffer[] srcs, int srcsOffset, final int srcsLength,\n            final ByteBuffer[] dsts, final int dstsOffset, final int dstsLength)\n            throws SSLException;\n\n    @Override\n    public abstract SSLEngineResult wrap(ByteBuffer src, ByteBuffer dst) throws SSLException;\n\n    @Override\n    public abstract SSLEngineResult wrap(\n            ByteBuffer[] srcs, int srcsOffset, int srcsLength, ByteBuffer dst) throws SSLException;\n\n    /**\n     * This method enables session ticket support.\n     *\n     * @param useSessionTickets True to enable session tickets\n     */\n    abstract void setUseSessionTickets(boolean useSessionTickets);\n\n    /**\n     * Sets the list of ALPN protocols.\n     *\n     * @param protocols the list of ALPN protocols\n     */\n    abstract void setApplicationProtocols(String[] protocols);\n\n    /**\n     * Returns the list of supported ALPN protocols.\n     */\n    abstract String[] getApplicationProtocols();\n\n    @SuppressWarnings(\"MissingOverride\") // For compiling pre Java 9.\n    public abstract String getApplicationProtocol();\n\n    @SuppressWarnings(\"MissingOverride\") // For compiling pre Java 9.\n    public abstract String getHandshakeApplicationProtocol();\n\n    /**\n     * Sets an application-provided ALPN protocol selector. If provided, this will override\n     * the list of protocols set by {@link #setApplicationProtocols(String[])}.\n     */\n    abstract void setApplicationProtocolSelector(ApplicationProtocolSelector selector);\n\n    /**\n     * Returns the tls-unique channel binding value for this connection, per RFC 5929.  This\n     * will return {@code null} if there is no such value available, such as if the handshake\n     * has not yet completed or this connection is closed.\n     */\n    abstract byte[] getTlsUnique();\n\n    /**\n     * Exports a value derived from the TLS master secret as described in RFC 5705.\n     *\n     * @param label the label to use in calculating the exported value.  This must be\n     * an ASCII-only string.\n     * @param context the application-specific context value to use in calculating the\n     * exported value.  This may be {@code null} to use no application context, which is\n     * treated differently than an empty byte array.\n     * @param length the number of bytes of keying material to return.\n     * @return a value of the specified length, or {@code null} if the handshake has not yet\n     * completed or the connection has been closed.\n     * @throws SSLException if the value could not be exported.\n     */\n    abstract byte[] exportKeyingMaterial(String label, byte[] context, int length)\n            throws SSLException;\n}\n\n// Path: common/src/main/java/org/conscrypt/ExternalSession.java\n/*\n * Copyright (C) 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.Principal;\nimport java.security.cert.Certificate;\nimport java.util.HashMap;\nimport java.util.List;\nimport javax.net.ssl.SSLPeerUnverifiedException;\nimport javax.net.ssl.SSLSession;\nimport javax.net.ssl.SSLSessionBindingEvent;\nimport javax.net.ssl.SSLSessionBindingListener;\nimport javax.net.ssl.SSLSessionContext;\n\n/**\n * An externalized view of the underlying {@link SSLSession} used within a\n * socket/engine. This class provides the caller with a consistent session\n * handle which will continue to be usable regardless of internal changes\n * to the connection.  It does this by delegating calls to the <b>current</b>\n * internal session, which is provided by the session {@code Provider}\n * (i.e. the socket or engine that owns the session).  This allows the provider\n * to switch implementations (for instance, using a JNI implementation to\n * access live values while the connection is open and a set of final values\n * when the connection is closed), even if the caller stores a reference to\n * the session object.\n *\n * <p>This class implements the {@link SSLSession} value API itself, rather\n * than delegating to the provided session, to ensure the caller has a consistent\n * value map, regardless of which internal session is currently being used by the\n * socket/engine.  This class will never call the value API methods on the\n * underlying sessions, so they need not be implemented.\n */\nfinal class ExternalSession implements ConscryptSession {\n\n  // Use an initial capacity of 2 to keep it small in the average case.\n  private final HashMap<String, Object> values = new HashMap<>(2);\n  private final Provider provider;\n\n  public ExternalSession(Provider provider) {\n    this.provider = provider;\n  }\n\n  @Override\n  public String getRequestedServerName() {\n    return provider.provideSession().getRequestedServerName();\n  }\n\n  @Override\n  public List<byte[]> getStatusResponses() {\n    return provider.provideSession().getStatusResponses();\n  }\n\n  @Override\n  public byte[] getPeerSignedCertificateTimestamp() {\n    return provider.provideSession().getPeerSignedCertificateTimestamp();\n  }\n\n  @Override\n  public byte[] getId() {\n    return provider.provideSession().getId();\n  }\n\n  @Override\n  public SSLSessionContext getSessionContext() {\n    return provider.provideSession().getSessionContext();\n  }\n\n  @Override\n  public long getCreationTime() {\n    return provider.provideSession().getCreationTime();\n  }\n\n  @Override\n  public long getLastAccessedTime() {\n    return provider.provideSession().getLastAccessedTime();\n  }\n\n  @Override\n  public void invalidate() {\n    provider.provideSession().invalidate();\n  }\n\n  @Override\n  public boolean isValid() {\n    return provider.provideSession().isValid();\n  }\n\n  @Override\n  public java.security.cert.X509Certificate[] getPeerCertificates()\n      throws SSLPeerUnverifiedException {\n    return provider.provideSession().getPeerCertificates();\n  }\n\n  @Override\n  public Certificate[] getLocalCertificates() {\n    return provider.provideSession().getLocalCertificates();\n  }\n\n  @Override\n  @SuppressWarnings(\"deprecation\") // Public API\n  public javax.security.cert.X509Certificate[] getPeerCertificateChain() throws SSLPeerUnverifiedException {\n    return provider.provideSession().getPeerCertificateChain();\n  }\n\n  @Override\n  public Principal getPeerPrincipal() throws SSLPeerUnverifiedException {\n    return provider.provideSession().getPeerPrincipal();\n  }\n\n  @Override\n  public Principal getLocalPrincipal() {\n    return provider.provideSession().getLocalPrincipal();\n  }\n\n  @Override\n  public String getCipherSuite() {\n    return provider.provideSession().getCipherSuite();\n  }\n\n  @Override\n  public String getProtocol() {\n    return provider.provideSession().getProtocol();\n  }\n\n  @Override\n  public String getPeerHost() {\n    return provider.provideSession().getPeerHost();\n  }\n\n  @Override\n  public int getPeerPort() {\n    return provider.provideSession().getPeerPort();\n  }\n\n  @Override\n  public int getPacketBufferSize() {\n    return provider.provideSession().getPacketBufferSize();\n  }\n\n  @Override\n  public int getApplicationBufferSize() {\n    return provider.provideSession().getApplicationBufferSize();\n  }\n\n  @Override\n  public String getApplicationProtocol() {\n    return provider.provideSession().getApplicationProtocol();\n  }\n\n  @Override\n  public Object getValue(String name) {\n    if (name == null) {\n      throw new IllegalArgumentException(\"name == null\");\n    }\n    return values.get(name);\n  }\n\n  @Override\n  public String[] getValueNames() {\n    return values.keySet().toArray(new String[0]);\n  }\n\n  @Override\n  public void putValue(String name, Object value) {\n    putValue(this, name, value);\n  }\n\n  void putValue(SSLSession session, String name, Object value) {\n    if (name == null || value == null) {\n      throw new IllegalArgumentException(\"name == null || value == null\");\n    }\n    Object old = values.put(name, value);\n    if (value instanceof SSLSessionBindingListener) {\n      ((SSLSessionBindingListener) value).valueBound(new SSLSessionBindingEvent(session, name));\n    }\n    if (old instanceof SSLSessionBindingListener) {\n      ((SSLSessionBindingListener) old).valueUnbound(new SSLSessionBindingEvent(session, name));\n    }\n  }\n\n  @Override\n  public void removeValue(String name) {\n    removeValue(this, name);\n  }\n\n  void removeValue(SSLSession session, String name) {\n    if (name == null) {\n      throw new IllegalArgumentException(\"name == null\");\n    }\n    Object old = values.remove(name);\n    if (old instanceof SSLSessionBindingListener) {\n      SSLSessionBindingListener listener = (SSLSessionBindingListener) old;\n      listener.valueUnbound(new SSLSessionBindingEvent(session, name));\n    }\n  }\n\n  /**\n   * The provider of the current delegate session.\n   */\n  interface Provider {\n    ConscryptSession provideSession();\n  }\n}\n\n// Path: common/src/main/java/org/conscrypt/SSLServerSessionCache.java\n/*\n * Copyright (C) 2009 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport javax.net.ssl.SSLSession;\n\n/**\n * A persistent {@link javax.net.ssl.SSLSession} cache used by\n * {@link javax.net.ssl.SSLSessionContext} to share server-side SSL sessions\n * across processes. For example, this cache enables one server to resume\n * a session started by a different server based on a session ID provided\n * by the client.\n *\n * <p>The {@code SSLSessionContext} implementation converts\n * {@code SSLSession}s into raw bytes and vice versa. The exact makeup of the\n * session data is dependent upon the caller's implementation and is opaque to\n * the {@code SSLServerSessionCache} implementation.\n */\npublic interface SSLServerSessionCache {\n    /**\n     * Gets the session data for given session ID.\n     *\n     * @param id from {@link javax.net.ssl.SSLSession#getId()}\n     * @return the session data or null if none is cached\n     * @throws NullPointerException if id is null\n     */\n    byte[] getSessionData(byte[] id);\n\n    /**\n     * Stores session data for the given session.\n     *\n     * @param session to cache data for\n     * @param sessionData to cache\n     * @throws NullPointerException if session or data is null\n     */\n    void putSessionData(SSLSession session, byte[] sessionData);\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX25519PublicKey.java\n/*\n * Copyright 2022 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.PublicKey;\nimport java.security.spec.EncodedKeySpec;\nimport java.security.spec.InvalidKeySpecException;\nimport java.util.Arrays;\n\npublic class OpenSSLX25519PublicKey implements OpenSSLX25519Key, PublicKey {\n    private static final long serialVersionUID = 453861992373478445L;\n\n    private static final byte[] X509_PREAMBLE = new byte[] {\n            0x30, 0x2a,                       // Sequence: 42 bytes\n                0x30, 0x05,                   // Sequence: 5 bytes\n                0x06, 0x03, 0x2b, 0x65, 0x6e, // OID: 1.3.101.110 (X25519)\n            0x03, 0x21, 0x00,                 // Bit string: 256 bits\n            // Key bytes follow directly\n    };\n\n    private final byte[] uCoordinate;\n\n    public OpenSSLX25519PublicKey(EncodedKeySpec keySpec) throws InvalidKeySpecException {\n        byte[] encoded = keySpec.getEncoded();\n        if (\"X.509\".equals(keySpec.getFormat())) {\n            if (!ArrayUtils.startsWith(encoded, X509_PREAMBLE)) {\n                throw new InvalidKeySpecException(\"Invalid format\");\n            }\n            int totalLength = X509_PREAMBLE.length + X25519_KEY_SIZE_BYTES;\n            if (encoded.length < totalLength) {\n                throw new InvalidKeySpecException(\"Invalid key size\");\n            }\n            uCoordinate = Arrays.copyOfRange(encoded, X509_PREAMBLE.length, totalLength);\n        } else if (\"raw\".equalsIgnoreCase(keySpec.getFormat())) {\n            if (encoded.length != X25519_KEY_SIZE_BYTES) {\n                throw new InvalidKeySpecException(\"Invalid key size\");\n            }\n            uCoordinate = encoded;\n        } else {\n            throw new InvalidKeySpecException(\"Encoding must be in X.509 or raw format\");\n        }\n    }\n\n    public OpenSSLX25519PublicKey(byte[] coordinateBytes) {\n        if (coordinateBytes.length != X25519_KEY_SIZE_BYTES) {\n            throw new IllegalArgumentException(\"Invalid key size\");\n        }\n        uCoordinate = coordinateBytes.clone();\n    }\n\n    @Override\n    public String getAlgorithm() {\n        return \"XDH\";\n    }\n\n    @Override\n    public String getFormat() {\n        return \"X.509\";\n    }\n\n    @Override\n    public byte[] getEncoded() {\n        if (uCoordinate == null) {\n            throw new IllegalStateException(\"key is destroyed\");\n        }\n\n        return ArrayUtils.concat(X509_PREAMBLE, uCoordinate);\n    }\n\n    @Override\n    public byte[] getU() {\n        if (uCoordinate == null) {\n            throw new IllegalStateException(\"key is destroyed\");\n        }\n        return uCoordinate.clone();\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (uCoordinate == null) {\n            throw new IllegalStateException(\"key is destroyed\");\n        }\n\n        if (this == o) return true;\n        if (!(o instanceof OpenSSLX25519PublicKey)) return false;\n        OpenSSLX25519PublicKey that = (OpenSSLX25519PublicKey) o;\n        return Arrays.equals(uCoordinate, that.uCoordinate);\n    }\n\n    @Override\n    public int hashCode() {\n        if (uCoordinate == null) {\n            throw new IllegalStateException(\"key is destroyed\");\n        }\n        return Arrays.hashCode(uCoordinate);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ConscryptServerSocket.java\n/*\n * Copyright (C) 2007 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.net.Socket;\nimport javax.net.ssl.SSLServerSocket;\n\n/**\n * BoringSSL-based implementation of server sockets.\n */\nfinal class ConscryptServerSocket extends SSLServerSocket {\n    private final SSLParametersImpl sslParameters;\n    private boolean channelIdEnabled;\n    private boolean useEngineSocket;\n\n    ConscryptServerSocket(SSLParametersImpl sslParameters) throws IOException {\n        this.sslParameters = sslParameters;\n    }\n\n    ConscryptServerSocket(int port, SSLParametersImpl sslParameters)\n        throws IOException {\n        super(port);\n        this.sslParameters = sslParameters;\n    }\n\n    ConscryptServerSocket(int port, int backlog, SSLParametersImpl sslParameters)\n        throws IOException {\n        super(port, backlog);\n        this.sslParameters = sslParameters;\n    }\n\n    ConscryptServerSocket(int port,\n                                      int backlog,\n                                      InetAddress iAddress,\n                                      SSLParametersImpl sslParameters)\n        throws IOException {\n        super(port, backlog, iAddress);\n        this.sslParameters = sslParameters;\n    }\n\n    /**\n     * Configures the socket to be created for this instance.\n     */\n    ConscryptServerSocket setUseEngineSocket(boolean useEngineSocket) {\n        this.useEngineSocket = useEngineSocket;\n        return this;\n    }\n\n    @Override\n    public boolean getEnableSessionCreation() {\n        return sslParameters.getEnableSessionCreation();\n    }\n\n    @Override\n    public void setEnableSessionCreation(boolean flag) {\n        sslParameters.setEnableSessionCreation(flag);\n    }\n\n    /**\n     * The names of the protocols' versions that may be used on this SSL\n     * connection.\n     * @return an array of protocols names\n     */\n    @Override\n    public String[] getSupportedProtocols() {\n        return NativeCrypto.getSupportedProtocols();\n    }\n\n    /**\n     * The names of the protocols' versions that in use on this SSL connection.\n     *\n     * @return an array of protocols names\n     */\n    @Override\n    public String[] getEnabledProtocols() {\n        return sslParameters.getEnabledProtocols();\n    }\n\n    /**\n     * This method enables the protocols' versions listed by\n     * getSupportedProtocols().\n     *\n     * @param protocols names of all the protocols to enable.\n     *\n     * @throws IllegalArgumentException when one or more of the names in the\n     *             array are not supported, or when the array is null.\n     */\n    @Override\n    public void setEnabledProtocols(String[] protocols) {\n        sslParameters.setEnabledProtocols(protocols);\n    }\n\n    @Override\n    public String[] getSupportedCipherSuites() {\n        return NativeCrypto.getSupportedCipherSuites();\n    }\n\n    @Override\n    public String[] getEnabledCipherSuites() {\n        return sslParameters.getEnabledCipherSuites();\n    }\n\n    /**\n     * Enables/disables the TLS Channel ID extension for this server socket.\n     */\n    void setChannelIdEnabled(boolean enabled) {\n      channelIdEnabled = enabled;\n    }\n\n    /**\n     * Checks whether the TLS Channel ID extension is enabled for this server socket.\n     */\n    \nboolean isChannelIdEnabled() {\n      return channelIdEnabled;\n    }\n\n    /**\n     * This method enables the cipher suites listed by\n     * getSupportedCipherSuites().\n     *\n     * @param suites the names of all the cipher suites to enable\n     * @throws IllegalArgumentException when one or more of the ciphers in array\n     *         suites are not supported, or when the array is null.\n     */\n    @Override\n    public void setEnabledCipherSuites(String[] suites) {\n        sslParameters.setEnabledCipherSuites(suites);\n    }\n\n    @Override\n    public boolean getWantClientAuth() {\n        return sslParameters.getWantClientAuth();\n    }\n\n    @Override\n    public void setWantClientAuth(boolean want) {\n        sslParameters.setWantClientAuth(want);\n    }\n\n    @Override\n    public boolean getNeedClientAuth() {\n        return sslParameters.getNeedClientAuth();\n    }\n\n    @Override\n    public void setNeedClientAuth(boolean need) {\n        sslParameters.setNeedClientAuth(need);\n    }\n\n    @Override\n    public void setUseClientMode(boolean mode) {\n        sslParameters.setUseClientMode(mode);\n    }\n\n    @Override\n    public boolean getUseClientMode() {\n        return sslParameters.getUseClientMode();\n    }\n\n    @Override\n    public Socket accept() throws IOException {\n        final AbstractConscryptSocket socket;\n        if (useEngineSocket) {\n            socket = Platform.createEngineSocket(sslParameters);\n        } else {\n            socket = Platform.createFileDescriptorSocket(sslParameters);\n        }\n\n        socket.setChannelIdEnabled(channelIdEnabled);\n        implAccept(socket);\n        return socket;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/ConscryptSession.java\n/*\n * Copyright (C) 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.cert.X509Certificate;\nimport java.util.List;\nimport javax.net.ssl.SSLPeerUnverifiedException;\nimport javax.net.ssl.SSLSession;\n\n/**\n * Extends the default interface for {@link SSLSession} to provide additional properties exposed\n * by Conscrypt.\n */\ninterface ConscryptSession extends SSLSession {\n\n  String getRequestedServerName();\n\n  /**\n   * Returns the OCSP stapled response. Returns a copy of the internal arrays.\n   *\n   * The method signature matches\n   * <a\n   * href=\"http://download.java.net/java/jdk9/docs/api/javax/net/ssl/ExtendedSSLSession.html#getStatusResponses--\">Java\n   * 9</a>.\n   *\n   * @see <a href=\"https://tools.ietf.org/html/rfc6066\">RFC 6066</a>\n   * @see <a href=\"https://tools.ietf.org/html/rfc6961\">RFC 6961</a>\n   */\n  List<byte[]> getStatusResponses();\n\n  /**\n   * Returns the signed certificate timestamp (SCT) received from the peer. Returns a\n   * copy of the internal array.\n   *\n   * @see <a href=\"https://tools.ietf.org/html/rfc6962\">RFC 6962</a>\n   */\n  byte[] getPeerSignedCertificateTimestamp();\n\n  @Override\n  X509Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException;\n\n  String getApplicationProtocol();\n}\n\n// Path: common/src/main/java/org/conscrypt/HpkeContextSender.java\n/*\n * Copyright (C) 2023 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidKeyException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.NoSuchProviderException;\nimport java.security.PrivateKey;\nimport java.security.Provider;\nimport java.security.PublicKey;\n\n/**\n * Hybrid Public Key Encryption (HPKE) sender APIs.\n *\n * @see <a href=\"https://www.rfc-editor.org/rfc/rfc9180.html#hpke-export\">HPKE RFC 9180</a>\n *\n * Sender subclass of HpkeContext.  See base class for details.\n */\npublic class HpkeContextSender extends HpkeContext {\n    private HpkeContextSender(HpkeSpi spi) {\n        super(spi);\n    }\n\n    /**\n     * Returns the encapsulated key created for this HpkeContextSender.\n     *\n     * @return the encapsulated key\n     * @throws IllegalStateException if this HpkeContextSender has not been initialised.\n     */\n    public byte[] getEncapsulated() {\n        return spi.getEncapsulated();\n    }\n\n    /**\n     * Seals a message, using the internal key schedule maintained by this HpkeContextSender.\n     *\n     * @param plaintext the plaintext\n     * @param aad optional associated data, may be null or empty\n     * @return the ciphertext\n     * @throws NullPointerException if the plaintext is null\n     * @throws IllegalStateException if this HpkeContextSender has not been initialised\n     */\n    public byte[] seal(byte[] plaintext, byte[] aad) {\n        return spi.engineSeal(plaintext, aad);\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextSender.\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @return an uninitialised HpkeContextSender for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     */\n    public static HpkeContextSender getInstance(String suite) throws NoSuchAlgorithmException {\n        return new HpkeContextSender(findSpi(suite));\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextSender from a specific {@link Provider}\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @param providerName the name of the Provider to use\n     * @return an uninitialised HpkeContextSender for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     * @throws NoSuchProviderException if providerName is null or no such Provider exists\n     */\n    public static HpkeContextSender getInstance(String suite, String providerName)\n        throws NoSuchAlgorithmException, NoSuchProviderException {\n        return new HpkeContextSender(findSpi(suite, providerName));\n    }\n\n    /**\n     * Returns an uninitialised HpkeContextSender from a specific {@link Provider}\n     *\n     * @param suite the HPKE suite to use.  @see {@link HpkeSuite} for details.\n     * @param provider the Provider to use\n     * @return an uninitialised HpkeContextSender for the requested suite\n     * @throws NoSuchAlgorithmException if no implementation could be found\n     * @throws NoSuchProviderException if provider is null\n     */\n    public static HpkeContextSender getInstance(String suite, Provider provider)\n        throws NoSuchAlgorithmException, NoSuchProviderException {\n        return new HpkeContextSender(findSpi(suite, provider));\n    }\n\n    /**\n     * Initialises this HpkeContextSender in BASE mode, i.e. with no sender authentication.\n     *\n     * @param recipientKey public key of the recipient\n     * @param info additional application-supplied information, may be null or empty\n     * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextSender has already been initialised\n     */\n    public void init(PublicKey recipientKey, byte[] info) throws InvalidKeyException {\n        spi.engineInitSender(\n                recipientKey, info, null, HpkeSpi.DEFAULT_PSK, HpkeSpi.DEFAULT_PSK_ID);\n    }\n\n    /**\n     * Initialises this HpkeContextSender in AUTH mode, i.e. messages are authenticated using\n     * the sender's public key.\n     *\n     * @param recipientKey public key of the recipient\n     * @param info additional application-supplied information, may be null or empty\n     * @param senderKey private key of the sender\n     * @throws InvalidKeyException if either recipientKey or senderKey are null\n     *         or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextSender has already been initialised\n     */\n    public void init(PublicKey recipientKey, byte[] info, PrivateKey senderKey)\n            throws InvalidKeyException {\n        if (senderKey == null) {\n            throw new InvalidKeyException(\"Sender private key is null\");\n        }\n        // Remaining argument checks are performed by the SPI\n        spi.engineInitSender(\n                recipientKey, info, senderKey, HpkeSpi.DEFAULT_PSK, HpkeSpi.DEFAULT_PSK_ID);\n    }\n\n    /**\n     * Initialises this HpkeContextSender in PSK mode, i.e. messages are authenticated using\n     * a pre-shared secret key.\n     *\n     * @param recipientKey public key of the recipient\n     * @param info additional application-supplied information, may be null or empty\n     * @param psk the a pre-shared secret key\n     * @param psk_id the id of the pre-shared secret key\n     * @throws NullPointerException if psk or psk_id are null\n     * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextSender has already been initialised\n     */\n    public void init(PublicKey recipientKey, byte[] info, byte[] psk, byte[] psk_id)\n            throws InvalidKeyException {\n        spi.engineInitSender(recipientKey, info, null, psk, psk_id);\n    }\n\n    /**\n     * Initialises this HpkeContextSender in PSK_AUTH mode, i.e. messages are authenticated using\n     * both the sender's public key and a pre-shared secret key.\n     *\n     * @param recipientKey public key of the recipient\n     * @param info additional application-supplied information, may be null or empty\n     * @param senderKey private key of the sender\n     * @param psk the a pre-shared secret key\n     * @param psk_id the id of the pre-shared secret key\n     * @throws NullPointerException if psk or psk_id are null\n     * @throws InvalidKeyException if either recipientKey or senderKey are null\n     *         or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextSender has already been initialised\n     */\n    public void init(PublicKey recipientKey, byte[] info, PrivateKey senderKey,\n            byte[] psk, byte[] psk_id) throws InvalidKeyException {\n        if (senderKey == null) {\n            throw new InvalidKeyException(\"Sender private key is null\");\n        }\n        // Remaining argument checks are performed by the SPI\n        spi.engineInitSender(recipientKey, info, senderKey, psk, psk_id);\n    }\n\n    /**\n     * Initialises this HpkeContextSender for testing in BASE mode ONLY.\n     *\n     * @param recipientKey public key of the recipient\n     * @param info additional application-supplied information, may be null or empty\n     * @param sKe random seed to use during testing\n     * @throws InvalidKeyException if recipientKey is null or an unsupported key format\n     * @throws UnsupportedOperationException if mode is not a supported HPKE mode\n     * @throws IllegalStateException if this HpkeContextSender has already been initialised\n     * @throws IllegalArgumentException if sKe is null\n     */\n    @Internal\n    public void initForTesting(PublicKey recipientKey, byte[] info, byte[] sKe)\n        throws InvalidKeyException {\n        if (sKe == null) {\n            throw new IllegalArgumentException(\"null seed\");\n        }\n        spi.engineInitSenderForTesting(\n                recipientKey, info, null, HpkeSpi.DEFAULT_PSK, HpkeSpi.DEFAULT_PSK, sKe);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLSocketImpl.java\n/*\n * Copyright (C) 2007 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.FileDescriptor;\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.net.Socket;\nimport java.net.SocketException;\nimport java.security.PrivateKey;\nimport javax.net.ssl.SSLException;\nimport javax.net.ssl.SSLSession;\n\n/**\n * Public shim allowing us to stay backward-compatible with legacy applications which were using\n * Conscrypt's extended socket API before the introduction of the {@link Conscrypt} class.\n */\n@Internal\npublic abstract class OpenSSLSocketImpl extends AbstractConscryptSocket {\n    OpenSSLSocketImpl() throws IOException {\n    }\n\n    OpenSSLSocketImpl(String hostname, int port) throws IOException {\n        super(hostname, port);\n    }\n\n    OpenSSLSocketImpl(InetAddress address, int port) throws IOException {\n        super(address, port);\n    }\n\n    OpenSSLSocketImpl(String hostname, int port, InetAddress clientAddress, int clientPort)\n        throws IOException {\n        super(hostname, port, clientAddress, clientPort);\n    }\n\n...\n", "needle_token_start": 11887, "needle_token_end": 11904, "code_context_ntokens": 15893, "output": ["```java\n/**\n * Checks whether the TLS Channel ID extension is enabled for this server socket.\n */\nboolean isChannelIdEnabled() {\n    return channelIdEnabled;\n}\n```"]}
{"repo": "google/conscrypt", "name": "findByApiName", "language": "java", "path": "common/src/main/java/org/conscrypt/OpenSSLX509CertPath.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To identify the enumeration type corresponding to a specified string that represents an encoding type for certificate paths.\n2. **Input**: A string representing the name of an encoding type.\n3. **Output**: The corresponding enumeration type if the input string matches one of the predefined encoding types; otherwise, `null`.\n4. **Procedure**: The function iterates through all values of the enumeration that defines encoding types. For each type, it checks if the input string matches the type's associated string identifier. If a match is found, that enumeration type is returned. If no match is found by the end of the iteration, the function returns `null`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/ArrayUtils.java\n/*\n * Copyright 2014 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.util.Arrays;\n\n/**\n * Compatibility utility for Arrays.\n */\npublic final class ArrayUtils {\n    private ArrayUtils() {}\n\n    /**\n     * Checks that the range described by {@code offset} and {@code count}\n     * doesn't exceed {@code arrayLength}.\n     */\n    static void checkOffsetAndCount(int arrayLength, int offset, int count) {\n        if ((offset | count) < 0 || offset > arrayLength || arrayLength - offset < count) {\n            throw new ArrayIndexOutOfBoundsException(\"length=\" + arrayLength + \"; regionStart=\"\n                    + offset + \"; regionLength=\" + count);\n        }\n    }\n\n    @SafeVarargs\n    public static <T> T[] concatValues(T[] a1, T... values) {\n        return concat (a1, values);\n    }\n\n    public static <T> T[] concat(T[] a1, T[] a2) {\n        T[] result = Arrays.copyOf(a1, a1.length + a2.length);\n        System.arraycopy(a2, 0, result, a1.length, a2.length);\n        return result;\n    }\n\n    public static byte[] concat(byte[] a1, byte[] a2) {\n        byte[] result = Arrays.copyOf(a1, a1.length + a2.length);\n        System.arraycopy(a2, 0, result, a1.length, a2.length);\n        return result;\n    }\n\n    public static boolean startsWith(byte[] array, byte[] startsWith) {\n        if (array.length < startsWith.length) {\n            return false;\n        }\n        for (int i = 0; i < startsWith.length; i++) {\n            if (array[i] != startsWith[i]) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    public static byte[] reverse(byte[] array) {\n        byte[] result = new byte[array.length];\n        int front = 0;\n        int back = array.length - 1;\n        while (back >= 0) {\n            result[front++] = array[back--];\n        }\n        return result;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OidData.java\n/*\n * Copyright (C) 2019 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * Data about OIDs.\n */\nfinal class OidData {\n\n  private OidData() {}\n\n  private static final Map<String, String> OID_TO_NAME_MAP = new HashMap<>();\n\n  static {\n    // NOTE: For the time being, we only have X509 signature algorithms here, since we only need\n    // them for determining the name of signature algorithms in certs and CRLs.  We can add more in\n    // the future if we need them.\n\n    // Signatures\n\n    // RFC 3279\n    OID_TO_NAME_MAP.put(\"1.2.840.113549.1.1.2\", \"MD2withRSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.113549.1.1.4\", \"MD5withRSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.113549.1.1.5\", \"SHA1withRSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.10040.4.3\", \"SHA1withDSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.10045.4.1\", \"SHA1withECDSA\");\n\n    // RFC 4055\n    OID_TO_NAME_MAP.put(\"1.2.840.113549.1.1.14\", \"SHA224withRSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.113549.1.1.11\", \"SHA256withRSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.113549.1.1.12\", \"SHA384withRSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.113549.1.1.13\", \"SHA512withRSA\");\n\n    // RFC 5758\n    OID_TO_NAME_MAP.put(\"2.16.840.1.101.3.4.3.1\", \"SHA224withDSA\");\n    OID_TO_NAME_MAP.put(\"2.16.840.1.101.3.4.3.2\", \"SHA256withDSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.10045.4.3.1\", \"SHA224withECDSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.10045.4.3.2\", \"SHA256withECDSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.10045.4.3.3\", \"SHA384withECDSA\");\n    OID_TO_NAME_MAP.put(\"1.2.840.10045.4.3.4\", \"SHA512withECDSA\");\n  }\n\n  public static String oidToAlgorithmName(String oid) {\n    return OID_TO_NAME_MAP.get(oid);\n  }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX509Certificate.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStream;\nimport java.math.BigInteger;\nimport java.security.InvalidKeyException;\nimport java.security.KeyFactory;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.NoSuchProviderException;\nimport java.security.Principal;\nimport java.security.Provider;\nimport java.security.PublicKey;\nimport java.security.Signature;\nimport java.security.SignatureException;\nimport java.security.cert.Certificate;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateExpiredException;\nimport java.security.cert.CertificateNotYetValidException;\nimport java.security.cert.CertificateParsingException;\nimport java.security.cert.X509Certificate;\nimport java.security.spec.InvalidKeySpecException;\nimport java.security.spec.X509EncodedKeySpec;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.Date;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport javax.crypto.BadPaddingException;\nimport javax.security.auth.x500.X500Principal;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * An implementation of {@link X509Certificate} based on BoringSSL.\n */\n@Internal\npublic final class OpenSSLX509Certificate extends X509Certificate {\n    private static final long serialVersionUID = 1992239142393372128L;\n\n    private transient volatile long mContext;\n    private transient Integer mHashCode;\n\n    private final long notBefore;\n    private final long notAfter;\n\n    OpenSSLX509Certificate(long ctx) throws ParsingException {\n        mContext = ctx;\n        // The legacy X509 OpenSSL APIs don't validate ASN1_TIME structures until access, so\n        // parse them here because this is the only time we're allowed to throw ParsingException\n        notBefore = NativeCrypto.X509_get_notBefore(mContext, this);\n        notAfter = NativeCrypto.X509_get_notAfter(mContext, this);\n    }\n\n    public static OpenSSLX509Certificate fromX509DerInputStream(InputStream is)\n            throws ParsingException {\n        @SuppressWarnings(\"resource\")\n        final OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        try {\n            final long certCtx = NativeCrypto.d2i_X509_bio(bis.getBioContext());\n            if (certCtx == 0) {\n                return null;\n            }\n            return new OpenSSLX509Certificate(certCtx);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n    }\n\n    public static OpenSSLX509Certificate fromX509Der(byte[] encoded)\n            throws CertificateEncodingException {\n        try {\n            return new OpenSSLX509Certificate(NativeCrypto.d2i_X509(encoded));\n        } catch (ParsingException e) {\n            throw new CertificateEncodingException(e);\n        }\n    }\n\n    public static List<OpenSSLX509Certificate> fromPkcs7DerInputStream(InputStream is)\n            throws ParsingException {\n        @SuppressWarnings(\"resource\")\n        OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        final long[] certRefs;\n        try {\n            certRefs = NativeCrypto.d2i_PKCS7_bio(bis.getBioContext(), NativeCrypto.PKCS7_CERTS);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n\n        if (certRefs == null) {\n            // To avoid returning a immutable list in only one path, we create an\n            // empty list here instead of using Collections.emptyList()\n            return new ArrayList<>();\n        }\n\n        final List<OpenSSLX509Certificate> certs = new ArrayList<>(\n                certRefs.length);\n        for (long certRef : certRefs) {\n            if (certRef == 0) {\n                continue;\n            }\n            certs.add(new OpenSSLX509Certificate(certRef));\n        }\n        return certs;\n    }\n\n    public static OpenSSLX509Certificate fromX509PemInputStream(InputStream is)\n            throws ParsingException {\n        @SuppressWarnings(\"resource\")\n        final OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        try {\n            final long certCtx = NativeCrypto.PEM_read_bio_X509(bis.getBioContext());\n            if (certCtx == 0L) {\n                return null;\n            }\n            return new OpenSSLX509Certificate(certCtx);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n    }\n\n    public static List<OpenSSLX509Certificate> fromPkcs7PemInputStream(InputStream is)\n            throws ParsingException {\n        @SuppressWarnings(\"resource\")\n        OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        final long[] certRefs;\n        try {\n            certRefs = NativeCrypto.PEM_read_bio_PKCS7(bis.getBioContext(),\n                    NativeCrypto.PKCS7_CERTS);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n\n        final List<OpenSSLX509Certificate> certs = new ArrayList<>(\n                certRefs.length);\n        for (long certRef : certRefs) {\n            if (certRef == 0) {\n                continue;\n            }\n            certs.add(new OpenSSLX509Certificate(certRef));\n        }\n        return certs;\n    }\n\n    public static OpenSSLX509Certificate fromCertificate(Certificate cert)\n            throws CertificateEncodingException {\n        if (cert instanceof OpenSSLX509Certificate) {\n            return (OpenSSLX509Certificate) cert;\n        } else if (cert instanceof X509Certificate) {\n            return fromX509Der(cert.getEncoded());\n        } else {\n            throw new CertificateEncodingException(\"Only X.509 certificates are supported\");\n        }\n    }\n\n    @Override\n    public Set<String> getCriticalExtensionOIDs() {\n        String[] critOids =\n                NativeCrypto.get_X509_ext_oids(mContext, this, NativeCrypto.EXTENSION_TYPE_CRITICAL);\n\n        /*\n         * This API has a special case that if there are no extensions, we\n         * should return null. So if we have no critical extensions, we'll check\n         * non-critical extensions.\n         */\n        if ((critOids.length == 0)\n                && (NativeCrypto.get_X509_ext_oids(mContext, this,\n                        NativeCrypto.EXTENSION_TYPE_NON_CRITICAL).length == 0)) {\n            return null;\n        }\n\n        return new HashSet<>(Arrays.asList(critOids));\n    }\n\n    @Override\n    public byte[] getExtensionValue(String oid) {\n        return NativeCrypto.X509_get_ext_oid(mContext, this, oid);\n    }\n\n    @Override\n    public Set<String> getNonCriticalExtensionOIDs() {\n        String[] nonCritOids =\n                NativeCrypto.get_X509_ext_oids(mContext, this, NativeCrypto.EXTENSION_TYPE_NON_CRITICAL);\n\n        /*\n         * This API has a special case that if there are no extensions, we\n         * should return null. So if we have no non-critical extensions, we'll\n         * check critical extensions.\n         */\n        if ((nonCritOids.length == 0)\n                && (NativeCrypto.get_X509_ext_oids(mContext, this,\n                        NativeCrypto.EXTENSION_TYPE_CRITICAL).length == 0)) {\n            return null;\n        }\n\n        return new HashSet<>(Arrays.asList(nonCritOids));\n    }\n\n    @Override\n    public boolean hasUnsupportedCriticalExtension() {\n        return (NativeCrypto.get_X509_ex_flags(mContext, this) & NativeConstants.EXFLAG_CRITICAL) != 0;\n    }\n\n    @Override\n    @SuppressWarnings(\"JdkObsolete\")  // Needed for API compatibility\n    public void checkValidity() throws CertificateExpiredException,\n            CertificateNotYetValidException {\n        checkValidity(new Date());\n    }\n\n    @Override\n    @SuppressWarnings(\"JdkObsolete\")  // Needed for API compatibility\n    public void checkValidity(Date date) throws CertificateExpiredException,\n            CertificateNotYetValidException {\n        if (getNotBefore().compareTo(date) > 0) {\n            throw new CertificateNotYetValidException(\"Certificate not valid until \"\n                    + getNotBefore() + \" (compared to \" + date + \")\");\n        }\n\n        if (getNotAfter().compareTo(date) < 0) {\n            throw new CertificateExpiredException(\"Certificate expired at \"\n                    + getNotAfter() + \" (compared to \" + date + \")\");\n        }\n    }\n\n    @Override\n    public int getVersion() {\n        return (int) NativeCrypto.X509_get_version(mContext, this) + 1;\n    }\n\n    @Override\n    public BigInteger getSerialNumber() {\n        return new BigInteger(NativeCrypto.X509_get_serialNumber(mContext, this));\n    }\n\n    @Override\n    public Principal getIssuerDN() {\n        return getIssuerX500Principal();\n    }\n\n    @Override\n    public Principal getSubjectDN() {\n        return getSubjectX500Principal();\n    }\n\n    @Override\n    public Date getNotBefore() {\n        return new Date(notBefore);\n    }\n\n    @Override\n    public Date getNotAfter() {\n        return new Date(notAfter);\n    }\n\n    @Override\n    public byte[] getTBSCertificate() throws CertificateEncodingException {\n        return NativeCrypto.get_X509_tbs_cert(mContext, this);\n    }\n\n    @Override\n    public byte[] getSignature() {\n        return NativeCrypto.get_X509_signature(mContext, this);\n    }\n\n    @Override\n    public String getSigAlgName() {\n        String oid = getSigAlgOID();\n        String algName = OidData.oidToAlgorithmName(oid);\n        if (algName != null) {\n            return algName;\n        }\n        algName = Platform.oidToAlgorithmName(oid);\n        if (algName != null) {\n            return algName;\n        }\n        return oid;\n    }\n\n    @Override\n    public String getSigAlgOID() {\n        return NativeCrypto.get_X509_sig_alg_oid(mContext, this);\n    }\n\n    @Override\n    public byte[] getSigAlgParams() {\n        return NativeCrypto.get_X509_sig_alg_parameter(mContext, this);\n    }\n\n    @Override\n    public boolean[] getIssuerUniqueID() {\n        return NativeCrypto.get_X509_issuerUID(mContext, this);\n    }\n\n    @Override\n    public boolean[] getSubjectUniqueID() {\n        return NativeCrypto.get_X509_subjectUID(mContext, this);\n    }\n\n    @Override\n    public boolean[] getKeyUsage() {\n        final boolean[] kusage = NativeCrypto.get_X509_ex_kusage(mContext, this);\n        if (kusage == null) {\n            return null;\n        }\n\n        if (kusage.length >= 9) {\n            return kusage;\n        }\n\n        final boolean[] resized = new boolean[9];\n        System.arraycopy(kusage, 0, resized, 0, kusage.length);\n        return resized;\n    }\n\n    @Override\n    public int getBasicConstraints() {\n        if ((NativeCrypto.get_X509_ex_flags(mContext, this) & NativeConstants.EXFLAG_CA) == 0) {\n            return -1;\n        }\n\n        final int pathLen = NativeCrypto.get_X509_ex_pathlen(mContext, this);\n        if (pathLen == -1) {\n            return Integer.MAX_VALUE;\n        }\n\n        return pathLen;\n    }\n\n    @Override\n    public byte[] getEncoded() throws CertificateEncodingException {\n        return NativeCrypto.i2d_X509(mContext, this);\n    }\n\n    private void verifyOpenSSL(OpenSSLKey pkey) throws CertificateException, SignatureException {\n        try {\n            NativeCrypto.X509_verify(mContext, this, pkey.getNativeRef());\n        } catch (RuntimeException e) {\n            throw new CertificateException(e);\n        } catch (BadPaddingException e) {\n            throw new SignatureException();\n        }\n    }\n\n    private void verifyInternal(PublicKey key, String sigProvider) throws\n            NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException,\n            SignatureException, CertificateEncodingException {\n        final Signature sig;\n        if (sigProvider == null) {\n            sig = Signature.getInstance(getSigAlgName());\n        } else {\n            sig = Signature.getInstance(getSigAlgName(), sigProvider);\n        }\n\n        sig.initVerify(key);\n        sig.update(getTBSCertificate());\n        if (!sig.verify(getSignature())) {\n            throw new SignatureException(\"signature did not verify\");\n        }\n    }\n\n    @Override\n    public void verify(PublicKey key) throws CertificateException, NoSuchAlgorithmException,\n            InvalidKeyException, NoSuchProviderException, SignatureException {\n        if (key instanceof OpenSSLKeyHolder) {\n            OpenSSLKey pkey = ((OpenSSLKeyHolder) key).getOpenSSLKey();\n            verifyOpenSSL(pkey);\n            return;\n        }\n\n        verifyInternal(key, null);\n    }\n\n    @Override\n    public void verify(PublicKey key, String sigProvider) throws CertificateException,\n            NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException,\n            SignatureException {\n        verifyInternal(key, sigProvider);\n    }\n\n    /* @Override */\n    @SuppressWarnings(\"MissingOverride\")  // For compilation with Java 7.\n    // noinspection Override\n    public void verify(PublicKey key, Provider sigProvider)\n            throws CertificateException, NoSuchAlgorithmException, InvalidKeyException,\n                   SignatureException {\n        if (key instanceof OpenSSLKeyHolder && sigProvider instanceof OpenSSLProvider) {\n            OpenSSLKey pkey = ((OpenSSLKeyHolder) key).getOpenSSLKey();\n            verifyOpenSSL(pkey);\n            return;\n        }\n\n        final Signature sig;\n        if (sigProvider == null) {\n            sig = Signature.getInstance(getSigAlgName());\n        } else {\n            sig = Signature.getInstance(getSigAlgName(), sigProvider);\n        }\n\n        sig.initVerify(key);\n        sig.update(getTBSCertificate());\n        if (!sig.verify(getSignature())) {\n            throw new SignatureException(\"signature did not verify\");\n        }\n    }\n\n    @Override\n    public String toString() {\n        ByteArrayOutputStream os = new ByteArrayOutputStream();\n        long bioCtx = NativeCrypto.create_BIO_OutputStream(os);\n        try {\n            NativeCrypto.X509_print_ex(bioCtx, mContext, this, 0, 0);\n            return os.toString();\n        } finally {\n            NativeCrypto.BIO_free_all(bioCtx);\n        }\n    }\n\n    @Override\n    public PublicKey getPublicKey() {\n        /* First try to generate the key from supported OpenSSL key types. */\n        try {\n            OpenSSLKey pkey = new OpenSSLKey(NativeCrypto.X509_get_pubkey(mContext, this));\n            return pkey.getPublicKey();\n        } catch (NoSuchAlgorithmException | InvalidKeyException ignored) {\n            // Ignored\n        }\n\n        /* Try generating the key using other Java providers. */\n        String oid = NativeCrypto.get_X509_pubkey_oid(mContext, this);\n        byte[] encoded = NativeCrypto.i2d_X509_PUBKEY(mContext, this);\n        try {\n            KeyFactory kf = KeyFactory.getInstance(oid);\n            return kf.generatePublic(new X509EncodedKeySpec(encoded));\n        } catch (NoSuchAlgorithmException | InvalidKeySpecException ignored) {\n            // Ignored\n        }\n\n        /*\n         * We couldn't find anything else, so just return a nearly-unusable\n         * X.509-encoded key.\n         */\n        return new X509PublicKey(oid, encoded);\n    }\n\n    @Override\n    public X500Principal getIssuerX500Principal() {\n        final byte[] issuer = NativeCrypto.X509_get_issuer_name(mContext, this);\n        return new X500Principal(issuer);\n    }\n\n    @Override\n    public X500Principal getSubjectX500Principal() {\n        final byte[] subject = NativeCrypto.X509_get_subject_name(mContext, this);\n        return new X500Principal(subject);\n    }\n\n    @Override\n    public List<String> getExtendedKeyUsage() {\n        String[] extUsage = NativeCrypto.get_X509_ex_xkusage(mContext, this);\n        if (extUsage == null) {\n            return null;\n        }\n\n        return Arrays.asList(extUsage);\n    }\n\n    private static Collection<List<?>> alternativeNameArrayToList(Object[][] altNameArray) {\n        if (altNameArray == null) {\n            return null;\n        }\n\n        Collection<List<?>> coll = new ArrayList<>(altNameArray.length);\n        for (Object[] objects : altNameArray) {\n            coll.add(Collections.unmodifiableList(Arrays.asList(objects)));\n        }\n\n        return Collections.unmodifiableCollection(coll);\n    }\n\n    @Override\n    public Collection<List<?>> getSubjectAlternativeNames() throws CertificateParsingException {\n        return alternativeNameArrayToList(NativeCrypto.get_X509_GENERAL_NAME_stack(mContext, this,\n                NativeCrypto.GN_STACK_SUBJECT_ALT_NAME));\n    }\n\n    @Override\n    public Collection<List<?>> getIssuerAlternativeNames() throws CertificateParsingException {\n        return alternativeNameArrayToList(NativeCrypto.get_X509_GENERAL_NAME_stack(mContext, this,\n                NativeCrypto.GN_STACK_ISSUER_ALT_NAME));\n    }\n\n    @Override\n    public boolean equals(Object other) {\n        if (other instanceof OpenSSLX509Certificate) {\n            OpenSSLX509Certificate o = (OpenSSLX509Certificate) other;\n\n            return NativeCrypto.X509_cmp(mContext, this, o.mContext, o) == 0;\n        }\n\n        return super.equals(other);\n    }\n\n    @Override\n    public int hashCode() {\n        if (mHashCode != null) {\n            return mHashCode;\n        }\n        mHashCode = super.hashCode();\n        return mHashCode;\n    }\n\n    /**\n     * Returns the raw pointer to the X509 context for use in JNI calls. The\n     * life cycle of this native pointer is managed by the\n     * {@code OpenSSLX509Certificate} instance and must not be destroyed or\n     * freed by users of this API.\n     */\n    public long getContext() {\n        return mContext;\n    }\n\n    /**\n     * Returns a re-encoded TBSCertificate with the extension identified by oid removed.\n     */\n    public byte[] getTBSCertificateWithoutExtension(String oid) {\n        return NativeCrypto.get_X509_tbs_cert_without_ext(mContext, this, oid);\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\")\n    protected void finalize() throws Throwable {\n        try {\n            long toFree = mContext;\n            if (toFree != 0) {\n                mContext = 0;\n                NativeCrypto.X509_free(toFree, this);\n            }\n        } finally {\n            super.finalize();\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX509CRL.java\n/*\n * Copyright (C) 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.InputStream;\nimport java.math.BigInteger;\nimport java.security.InvalidKeyException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.NoSuchProviderException;\nimport java.security.Principal;\nimport java.security.PublicKey;\nimport java.security.Signature;\nimport java.security.SignatureException;\nimport java.security.cert.CRLException;\nimport java.security.cert.Certificate;\nimport java.security.cert.X509CRL;\nimport java.security.cert.X509CRLEntry;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Date;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.security.auth.x500.X500Principal;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * An implementation of {@link X509CRL} based on BoringSSL.\n */\nfinal class OpenSSLX509CRL extends X509CRL {\n    private volatile long mContext;\n    private final long thisUpdate;\n    private final long nextUpdate;\n\n    private OpenSSLX509CRL(long ctx) throws ParsingException {\n        mContext = ctx;\n        // The legacy X509 OpenSSL APIs don't validate ASN1_TIME structures until access, so\n        // parse them here because this is the only time we're allowed to throw ParsingException\n        thisUpdate = NativeCrypto.X509_CRL_get_lastUpdate(mContext, this);\n        nextUpdate = NativeCrypto.X509_CRL_get_nextUpdate(mContext, this);\n    }\n\n    static OpenSSLX509CRL fromX509DerInputStream(InputStream is) throws ParsingException {\n        final OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        try {\n            final long crlCtx = NativeCrypto.d2i_X509_CRL_bio(bis.getBioContext());\n            if (crlCtx == 0) {\n                return null;\n            }\n            return new OpenSSLX509CRL(crlCtx);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n    }\n\n    static List<OpenSSLX509CRL> fromPkcs7DerInputStream(InputStream is)\n            throws ParsingException {\n        OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        final long[] certRefs;\n        try {\n            certRefs = NativeCrypto.d2i_PKCS7_bio(bis.getBioContext(), NativeCrypto.PKCS7_CRLS);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n\n        final List<OpenSSLX509CRL> certs = new ArrayList<>(certRefs.length);\n        for (long certRef : certRefs) {\n            if (certRef == 0) {\n                continue;\n            }\n            certs.add(new OpenSSLX509CRL(certRef));\n        }\n        return certs;\n    }\n\n    static OpenSSLX509CRL fromX509PemInputStream(InputStream is) throws ParsingException {\n        final OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        try {\n            final long crlCtx = NativeCrypto.PEM_read_bio_X509_CRL(bis.getBioContext());\n            if (crlCtx == 0) {\n                return null;\n            }\n            return new OpenSSLX509CRL(crlCtx);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n    }\n\n    static List<OpenSSLX509CRL> fromPkcs7PemInputStream(InputStream is)\n            throws ParsingException {\n        OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(is, true);\n\n        final long[] certRefs;\n        try {\n            certRefs = NativeCrypto.PEM_read_bio_PKCS7(bis.getBioContext(),\n                    NativeCrypto.PKCS7_CRLS);\n        } catch (Exception e) {\n            throw new ParsingException(e);\n        } finally {\n            bis.release();\n        }\n\n        final List<OpenSSLX509CRL> certs = new ArrayList<>(certRefs.length);\n        for (long certRef : certRefs) {\n            if (certRef == 0) {\n                continue;\n            }\n            certs.add(new OpenSSLX509CRL(certRef));\n        }\n        return certs;\n    }\n\n    @Override\n    public Set<String> getCriticalExtensionOIDs() {\n        String[] critOids =\n                NativeCrypto.get_X509_CRL_ext_oids(mContext, this, NativeCrypto.EXTENSION_TYPE_CRITICAL);\n\n        /*\n         * This API has a special case that if there are no extensions, we\n         * should return null. So if we have no critical extensions, we'll check\n         * non-critical extensions.\n         */\n        if ((critOids.length == 0)\n                && (NativeCrypto.get_X509_CRL_ext_oids(mContext, this,\n                        NativeCrypto.EXTENSION_TYPE_NON_CRITICAL).length == 0)) {\n            return null;\n        }\n\n        return new HashSet<>(Arrays.asList(critOids));\n    }\n\n    @Override\n    public byte[] getExtensionValue(String oid) {\n        return NativeCrypto.X509_CRL_get_ext_oid(mContext, this, oid);\n    }\n\n    @Override\n    public Set<String> getNonCriticalExtensionOIDs() {\n        String[] nonCritOids =\n                NativeCrypto.get_X509_CRL_ext_oids(mContext, this,\n                        NativeCrypto.EXTENSION_TYPE_NON_CRITICAL);\n\n        /*\n         * This API has a special case that if there are no extensions, we\n         * should return null. So if we have no non-critical extensions, we'll\n         * check critical extensions.\n         */\n        if ((nonCritOids.length == 0)\n                && (NativeCrypto.get_X509_CRL_ext_oids(mContext, this,\n                        NativeCrypto.EXTENSION_TYPE_CRITICAL).length == 0)) {\n            return null;\n        }\n\n        return new HashSet<>(Arrays.asList(nonCritOids));\n    }\n\n    @Override\n    public boolean hasUnsupportedCriticalExtension() {\n        final String[] criticalOids =\n                NativeCrypto.get_X509_CRL_ext_oids(mContext, this, NativeCrypto.EXTENSION_TYPE_CRITICAL);\n        for (String oid : criticalOids) {\n            final long extensionRef = NativeCrypto.X509_CRL_get_ext(mContext, this, oid);\n            if (NativeCrypto.X509_supported_extension(extensionRef) != 1) {\n                return true;\n            }\n        }\n\n        return false;\n    }\n\n    @Override\n    public byte[] getEncoded() throws CRLException {\n        return NativeCrypto.i2d_X509_CRL(mContext, this);\n    }\n\n    private void verifyOpenSSL(OpenSSLKey pkey) throws NoSuchAlgorithmException,\n            InvalidKeyException, SignatureException {\n        try {\n            NativeCrypto.X509_CRL_verify(mContext, this, pkey.getNativeRef());\n        } catch (BadPaddingException | IllegalBlockSizeException e) {\n            throw new SignatureException(e);\n        }\n    }\n\n    private void verifyInternal(PublicKey key, String sigProvider) throws NoSuchAlgorithmException,\n            InvalidKeyException, NoSuchProviderException, SignatureException {\n        String sigAlg = getSigAlgName();\n        if (sigAlg == null) {\n            sigAlg = getSigAlgOID();\n        }\n\n        final Signature sig;\n        if (sigProvider == null) {\n            sig = Signature.getInstance(sigAlg);\n        } else {\n            sig = Signature.getInstance(sigAlg, sigProvider);\n        }\n\n        sig.initVerify(key);\n        sig.update(getTBSCertList());\n        if (!sig.verify(getSignature())) {\n            throw new SignatureException(\"signature did not verify\");\n        }\n    }\n\n    @Override\n    public void verify(PublicKey key) throws CRLException, NoSuchAlgorithmException,\n            InvalidKeyException, NoSuchProviderException, SignatureException {\n        if (key instanceof OpenSSLKeyHolder) {\n            OpenSSLKey pkey = ((OpenSSLKeyHolder) key).getOpenSSLKey();\n            verifyOpenSSL(pkey);\n            return;\n        }\n\n        verifyInternal(key, null);\n    }\n\n    @Override\n    public void verify(PublicKey key, String sigProvider) throws CRLException,\n            NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException,\n            SignatureException {\n        verifyInternal(key, sigProvider);\n    }\n\n    @Override\n    public int getVersion() {\n        return (int) NativeCrypto.X509_CRL_get_version(mContext, this) + 1;\n    }\n\n    @Override\n    public Principal getIssuerDN() {\n        return getIssuerX500Principal();\n    }\n\n    @Override\n    public X500Principal getIssuerX500Principal() {\n        final byte[] issuer = NativeCrypto.X509_CRL_get_issuer_name(mContext, this);\n        return new X500Principal(issuer);\n    }\n\n    @Override\n    public Date getThisUpdate() {\n        return new Date(thisUpdate);\n    }\n\n    @Override\n    public Date getNextUpdate() {\n        return new Date(nextUpdate);\n    }\n\n    @Override\n    public X509CRLEntry getRevokedCertificate(BigInteger serialNumber) {\n        final long revokedRef = NativeCrypto.X509_CRL_get0_by_serial(mContext, this,\n                serialNumber.toByteArray());\n        if (revokedRef == 0) {\n            return null;\n        }\n        try {\n            return new OpenSSLX509CRLEntry(NativeCrypto.X509_REVOKED_dup(revokedRef));\n        } catch (ParsingException e) {\n            return null;\n        }\n    }\n\n    @Override\n    public X509CRLEntry getRevokedCertificate(X509Certificate certificate) {\n        if (certificate instanceof OpenSSLX509Certificate) {\n            OpenSSLX509Certificate osslCert = (OpenSSLX509Certificate) certificate;\n            final long x509RevokedRef = NativeCrypto.X509_CRL_get0_by_cert(mContext, this,\n                    osslCert.getContext(), osslCert);\n\n            if (x509RevokedRef == 0) {\n                return null;\n            }\n\n            try {\n                return new OpenSSLX509CRLEntry(NativeCrypto.X509_REVOKED_dup(x509RevokedRef));\n            } catch (ParsingException e) {\n                return null;\n            }\n        }\n\n        return getRevokedCertificate(certificate.getSerialNumber());\n    }\n\n    @Override\n    public Set<? extends X509CRLEntry> getRevokedCertificates() {\n        final long[] entryRefs = NativeCrypto.X509_CRL_get_REVOKED(mContext, this);\n        if (entryRefs == null || entryRefs.length == 0) {\n            return null;\n        }\n\n        final Set<OpenSSLX509CRLEntry> crlSet = new HashSet<>();\n        for (long entryRef : entryRefs) {\n            try {\n                crlSet.add(new OpenSSLX509CRLEntry(entryRef));\n            } catch (ParsingException e) {\n                // Skip this entry\n            }\n        }\n\n        return crlSet;\n    }\n\n    @Override\n    public byte[] getTBSCertList() {\n        return NativeCrypto.get_X509_CRL_crl_enc(mContext, this);\n    }\n\n    @Override\n    public byte[] getSignature() {\n        return NativeCrypto.get_X509_CRL_signature(mContext, this);\n    }\n\n    @Override\n    public String getSigAlgName() {\n        String oid = getSigAlgOID();\n        String algName = OidData.oidToAlgorithmName(oid);\n        if (algName != null) {\n            return algName;\n        }\n        algName = Platform.oidToAlgorithmName(oid);\n        if (algName != null) {\n            return algName;\n        }\n        return oid;\n    }\n\n    @Override\n    public String getSigAlgOID() {\n        return NativeCrypto.get_X509_CRL_sig_alg_oid(mContext, this);\n    }\n\n    @Override\n    public byte[] getSigAlgParams() {\n        return NativeCrypto.get_X509_CRL_sig_alg_parameter(mContext, this);\n    }\n\n    @Override\n    public boolean isRevoked(Certificate cert) {\n        if (!(cert instanceof X509Certificate)) {\n            return false;\n        }\n\n        final OpenSSLX509Certificate osslCert;\n        if (cert instanceof OpenSSLX509Certificate) {\n            osslCert = (OpenSSLX509Certificate) cert;\n        } else {\n            try {\n                osslCert = OpenSSLX509Certificate.fromX509DerInputStream(new ByteArrayInputStream(\n                        cert.getEncoded()));\n            } catch (Exception e) {\n                throw new RuntimeException(\"cannot convert certificate\", e);\n            }\n        }\n\n        final long x509RevokedRef = NativeCrypto.X509_CRL_get0_by_cert(mContext, this,\n                osslCert.getContext(), osslCert);\n\n        return x509RevokedRef != 0;\n    }\n\n    @Override\n    public String toString() {\n        ByteArrayOutputStream os = new ByteArrayOutputStream();\n        final long bioCtx = NativeCrypto.create_BIO_OutputStream(os);\n        try {\n            NativeCrypto.X509_CRL_print(bioCtx, mContext, this);\n            return os.toString();\n        } finally {\n            NativeCrypto.BIO_free_all(bioCtx);\n        }\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\")\n    protected void finalize() throws Throwable {\n        try {\n            long toFree = mContext;\n            if (toFree != 0) {\n                mContext = 0;\n                NativeCrypto.X509_CRL_free(toFree, this);\n            }\n        } finally {\n            super.finalize();\n        }\n    }\n\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX509CertPath.java\n/*\n * Copyright (C) 2013 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.PushbackInputStream;\nimport java.security.cert.CertPath;\nimport java.security.cert.Certificate;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateParsingException;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * An implementation of {@link CertPath} based on BoringSSL.\n */\nfinal class OpenSSLX509CertPath extends CertPath {\n    private static final long serialVersionUID = -3249106005255170761L;\n\n    private static final byte[] PKCS7_MARKER = new byte[] {\n            '-', '-', '-', '-', '-', 'B', 'E', 'G', 'I', 'N', ' ', 'P', 'K', 'C', 'S', '7'\n    };\n\n    private static final int PUSHBACK_SIZE = 64;\n\n    /**\n     * Supported encoding types for CerthPath. Used by the various APIs that\n     * encode this into bytes such as {@link #getEncoded()}.\n     */\n    private enum Encoding {\n        PKI_PATH(\"PkiPath\"),\n        PKCS7(\"PKCS7\");\n\n        private final String apiName;\n\n        Encoding(String apiName) {\n            this.apiName = apiName;\n        }\n\n        \nstatic Encoding findByApiName(String apiName) throws CertificateEncodingException {\n            for (Encoding element : values()) {\n                if (element.apiName.equals(apiName)) {\n                    return element;\n                }\n            }\n\n            return null;\n        }\n    }\n\n    /** Unmodifiable list of encodings for the API. */\n    private static final List<String> ALL_ENCODINGS = Collections.unmodifiableList(Arrays\n            .asList(new String[] {\n                    Encoding.PKI_PATH.apiName,\n                    Encoding.PKCS7.apiName,\n            }));\n\n    private static final Encoding DEFAULT_ENCODING = Encoding.PKI_PATH;\n\n    private final List<? extends X509Certificate> mCertificates;\n\n    static Iterator<String> getEncodingsIterator() {\n        return ALL_ENCODINGS.iterator();\n    }\n\n    OpenSSLX509CertPath(List<? extends X509Certificate> certificates) {\n        super(\"X.509\");\n\n        mCertificates = certificates;\n    }\n\n    @Override\n    public List<? extends Certificate> getCertificates() {\n        return Collections.unmodifiableList(mCertificates);\n    }\n\n    private byte[] getEncoded(Encoding encoding) throws CertificateEncodingException {\n        final OpenSSLX509Certificate[] certs = new OpenSSLX509Certificate[mCertificates.size()];\n        final long[] certRefs = new long[certs.length];\n\n        for (int i = 0, j = certs.length - 1; j >= 0; i++, j--) {\n            final X509Certificate cert = mCertificates.get(i);\n\n            if (cert instanceof OpenSSLX509Certificate) {\n                certs[j] = (OpenSSLX509Certificate) cert;\n            } else {\n                certs[j] = OpenSSLX509Certificate.fromX509Der(cert.getEncoded());\n            }\n\n            certRefs[j] = certs[j].getContext();\n        }\n\n        switch (encoding) {\n            case PKI_PATH:\n                return NativeCrypto.ASN1_seq_pack_X509(certRefs);\n            case PKCS7:\n                return NativeCrypto.i2d_PKCS7(certRefs);\n            default:\n                throw new CertificateEncodingException(\"Unknown encoding\");\n        }\n    }\n\n    @Override\n    public byte[] getEncoded() throws CertificateEncodingException {\n        return getEncoded(DEFAULT_ENCODING);\n    }\n\n    @Override\n    public byte[] getEncoded(String encoding) throws CertificateEncodingException {\n        Encoding enc = Encoding.findByApiName(encoding);\n        if (enc == null) {\n            throw new CertificateEncodingException(\"Invalid encoding: \" + encoding);\n        }\n\n        return getEncoded(enc);\n    }\n\n    @Override\n    public Iterator<String> getEncodings() {\n        return getEncodingsIterator();\n    }\n\n    private static CertPath fromPkiPathEncoding(InputStream inStream) throws CertificateException {\n        OpenSSLBIOInputStream bis = new OpenSSLBIOInputStream(inStream, true);\n\n        final boolean markable = inStream.markSupported();\n        if (markable) {\n            inStream.mark(PUSHBACK_SIZE);\n        }\n\n        final long[] certRefs;\n        try {\n            certRefs = NativeCrypto.ASN1_seq_unpack_X509_bio(bis.getBioContext());\n        } catch (Exception e) {\n            if (markable) {\n                try {\n                    inStream.reset();\n                } catch (IOException ignored) {\n                    // Ignored\n                }\n            }\n            throw new CertificateException(e);\n        } finally {\n            bis.release();\n        }\n\n        if (certRefs == null) {\n            return new OpenSSLX509CertPath(Collections.<X509Certificate> emptyList());\n        }\n\n        final List<OpenSSLX509Certificate> certs =\n                new ArrayList<OpenSSLX509Certificate>(certRefs.length);\n        for (int i = certRefs.length - 1; i >= 0; i--) {\n            if (certRefs[i] == 0) {\n                continue;\n            }\n            try {\n                certs.add(new OpenSSLX509Certificate(certRefs[i]));\n            } catch (ParsingException e) {\n                throw new CertificateParsingException(e);\n            }\n        }\n\n        return new OpenSSLX509CertPath(certs);\n    }\n\n    private static CertPath fromPkcs7Encoding(InputStream inStream) throws CertificateException {\n        try {\n            if (inStream == null || inStream.available() == 0) {\n                return new OpenSSLX509CertPath(Collections.<X509Certificate> emptyList());\n            }\n        } catch (IOException e) {\n            throw new CertificateException(\"Problem reading input stream\", e);\n        }\n\n        final boolean markable = inStream.markSupported();\n        if (markable) {\n            inStream.mark(PUSHBACK_SIZE);\n        }\n\n        /* Attempt to see if this is a PKCS#7 bag. */\n        final PushbackInputStream pbis = new PushbackInputStream(inStream, PUSHBACK_SIZE);\n        try {\n            final byte[] buffer = new byte[PKCS7_MARKER.length];\n\n            final int len = pbis.read(buffer);\n            if (len < 0) {\n                /* No need to reset here. The stream was empty or EOF. */\n                throw new ParsingException(\"inStream is empty\");\n            }\n            pbis.unread(buffer, 0, len);\n\n            if (len == PKCS7_MARKER.length && Arrays.equals(PKCS7_MARKER, buffer)) {\n                return new OpenSSLX509CertPath(OpenSSLX509Certificate.fromPkcs7PemInputStream(pbis));\n            }\n\n            return new OpenSSLX509CertPath(OpenSSLX509Certificate.fromPkcs7DerInputStream(pbis));\n        } catch (Exception e) {\n            if (markable) {\n                try {\n                    inStream.reset();\n                } catch (IOException ignored) {\n                    // Ignored\n                }\n            }\n            throw new CertificateException(e);\n        }\n    }\n\n    private static CertPath fromEncoding(InputStream inStream, Encoding encoding)\n            throws CertificateException {\n        switch (encoding) {\n            case PKI_PATH:\n                return fromPkiPathEncoding(inStream);\n            case PKCS7:\n                return fromPkcs7Encoding(inStream);\n            default:\n                throw new CertificateEncodingException(\"Unknown encoding\");\n        }\n    }\n\n    static CertPath fromEncoding(InputStream inStream, String encoding)\n            throws CertificateException {\n        if (inStream == null) {\n            throw new CertificateException(\"inStream == null\");\n        }\n\n        Encoding enc = Encoding.findByApiName(encoding);\n        if (enc == null) {\n            throw new CertificateException(\"Invalid encoding: \" + encoding);\n        }\n\n        return fromEncoding(inStream, enc);\n    }\n\n    static CertPath fromEncoding(InputStream inStream) throws CertificateException {\n        if (inStream == null) {\n            throw new CertificateException(\"inStream == null\");\n        }\n\n        return fromEncoding(inStream, DEFAULT_ENCODING);\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX509CertificateFactory.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.PushbackInputStream;\nimport java.security.cert.CRL;\nimport java.security.cert.CRLException;\nimport java.security.cert.CertPath;\nimport java.security.cert.Certificate;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateFactorySpi;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\n\n/**\n * An implementation of {@link java.security.cert.CertificateFactory} based on BoringSSL.\n */\n@Internal\npublic class OpenSSLX509CertificateFactory extends CertificateFactorySpi {\n    private static final byte[] PKCS7_MARKER = new byte[] {\n            '-', '-', '-', '-', '-', 'B', 'E', 'G', 'I', 'N', ' ', 'P', 'K', 'C', 'S', '7'\n    };\n\n    private static final int PUSHBACK_SIZE = 64;\n\n    static class ParsingException extends Exception {\n...\n", "needle_token_start": 12284, "needle_token_end": 12348, "code_context_ntokens": 14716, "output": ["```java\nstatic Encoding findByApiName(String apiName) throws CertificateEncodingException {\n    for (Encoding element : values()) {\n        if (element.apiName.equals(apiName)) {\n            return element;\n        }\n    }\n\n    return null;\n}\n```"]}
{"repo": "google/conscrypt", "name": "checkCert", "language": "java", "path": "common/src/main/java/org/conscrypt/ChainStrengthAnalyzer.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To evaluate the cryptographic strength and validity of a digital certificate by verifying its key length and signature algorithm against predefined security standards.\n2. **Input**: A single digital certificate of type X509Certificate.\n3. **Output**: Does not return any value; however, it throws a CertificateException if the certificate does not meet the required security criteria.\n4. **Procedure**: The function performs two main checks on the provided certificate: first, it verifies that the key length of the certificate's public key meets minimum security thresholds for RSA, DSA, and EC keys; second, it checks that the certificate's signature algorithm is not one of the deprecated or insecure algorithms listed in a predefined blacklist.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: common/src/main/java/org/conscrypt/NativeCrypto.java\n/*\n * Copyright (C) 2008 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.FileDescriptor;\nimport java.io.IOException;\nimport java.io.OutputStream;\nimport java.net.SocketTimeoutException;\nimport java.nio.Buffer;\nimport java.nio.ByteBuffer;\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.MessageDigest;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.PrivateKey;\nimport java.security.SignatureException;\nimport java.security.cert.CertificateEncodingException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.CertificateParsingException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Set;\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.crypto.ShortBufferException;\nimport javax.net.ssl.SSLException;\nimport javax.security.auth.x500.X500Principal;\nimport org.conscrypt.OpenSSLX509CertificateFactory.ParsingException;\n\n/**\n * Provides the Java side of our JNI glue for OpenSSL.\n * <p>\n * Note: Many methods in this class take a reference to a Java object that holds a\n * native pointer in the form of a long in addition to the long itself and don't use\n * the Java object in the native implementation.  This is to prevent the Java object\n * from becoming eligible for GC while the native method is executing.  See\n * <a href=\"https://github.com/google/error-prone/blob/master/docs/bugpattern/UnsafeFinalization.md\">this</a>\n * for more details.\n */\n@Internal\npublic final class NativeCrypto {\n    // --- OpenSSL library initialization --------------------------------------\n    private static final UnsatisfiedLinkError loadError;\n    static {\n        UnsatisfiedLinkError error = null;\n        try {\n            NativeCryptoJni.init();\n            clinit();\n        } catch (UnsatisfiedLinkError t) {\n            // Don't rethrow the error, so that we can later on interrogate the\n            // value of loadError.\n            error = t;\n        }\n        loadError = error;\n    }\n\n    private native static void clinit();\n\n    /**\n     * Checks to see whether or not the native library was successfully loaded. If not, throws\n     * the {@link UnsatisfiedLinkError} that was encountered while attempting to load the library.\n     */\n    static void checkAvailability() {\n        if (loadError != null) {\n            throw loadError;\n        }\n    }\n\n    // --- DSA/RSA public/private key handling functions -----------------------\n\n    static native long EVP_PKEY_new_RSA(byte[] n, byte[] e, byte[] d, byte[] p, byte[] q,\n            byte[] dmp1, byte[] dmq1, byte[] iqmp);\n\n    static native int EVP_PKEY_type(NativeRef.EVP_PKEY pkey);\n\n    static native String EVP_PKEY_print_public(NativeRef.EVP_PKEY pkeyRef);\n\n    static native String EVP_PKEY_print_params(NativeRef.EVP_PKEY pkeyRef);\n\n    static native void EVP_PKEY_free(long pkey);\n\n    static native int EVP_PKEY_cmp(NativeRef.EVP_PKEY pkey1, NativeRef.EVP_PKEY pkey2);\n\n    static native byte[] EVP_marshal_private_key(NativeRef.EVP_PKEY pkey);\n\n    static native long EVP_parse_private_key(byte[] data) throws ParsingException;\n\n    static native byte[] EVP_marshal_public_key(NativeRef.EVP_PKEY pkey);\n\n    static native byte[] EVP_raw_X25519_private_key(byte[] data)\n            throws ParsingException, InvalidKeyException;\n\n    static native long EVP_parse_public_key(byte[] data) throws ParsingException;\n\n    static native long PEM_read_bio_PUBKEY(long bioCtx);\n\n    static native long PEM_read_bio_PrivateKey(long bioCtx);\n\n    static native long getRSAPrivateKeyWrapper(PrivateKey key, byte[] modulus);\n\n    static native long getECPrivateKeyWrapper(PrivateKey key, NativeRef.EC_GROUP ecGroupRef);\n\n    static native long RSA_generate_key_ex(int modulusBits, byte[] publicExponent);\n\n    static native int RSA_size(NativeRef.EVP_PKEY pkey);\n\n    static native int RSA_private_encrypt(\n            int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey, int padding);\n\n    static native int RSA_public_decrypt(int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey,\n            int padding) throws BadPaddingException, SignatureException;\n\n    static native int RSA_public_encrypt(\n            int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey, int padding);\n\n    static native int RSA_private_decrypt(int flen, byte[] from, byte[] to, NativeRef.EVP_PKEY pkey,\n            int padding) throws BadPaddingException, SignatureException;\n\n    /**\n     * @return array of {n, e}\n     */\n    static native byte[][] get_RSA_public_params(NativeRef.EVP_PKEY rsa);\n\n    /**\n     * @return array of {n, e, d, p, q, dmp1, dmq1, iqmp}\n     */\n    static native byte[][] get_RSA_private_params(NativeRef.EVP_PKEY rsa);\n\n    // --- ChaCha20 -----------------------\n\n    /**\n     * Returns the encrypted or decrypted version of the data.\n     */\n    static native void chacha20_encrypt_decrypt(byte[] in, int inOffset, byte[] out, int outOffset,\n            int length, byte[] key, byte[] nonce, int blockCounter);\n\n    // --- EC functions --------------------------\n\n    static native long EVP_PKEY_new_EC_KEY(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pubkeyRef, byte[] privkey);\n\n    static native long EC_GROUP_new_by_curve_name(String curveName);\n\n    static native long EC_GROUP_new_arbitrary(\n            byte[] p, byte[] a, byte[] b, byte[] x, byte[] y, byte[] order, int cofactor);\n\n    static native String EC_GROUP_get_curve_name(NativeRef.EC_GROUP groupRef);\n\n    static native byte[][] EC_GROUP_get_curve(NativeRef.EC_GROUP groupRef);\n\n    static native void EC_GROUP_clear_free(long groupRef);\n\n    static native long EC_GROUP_get_generator(NativeRef.EC_GROUP groupRef);\n\n    static native byte[] EC_GROUP_get_order(NativeRef.EC_GROUP groupRef);\n\n    static native int EC_GROUP_get_degree(NativeRef.EC_GROUP groupRef);\n\n    static native byte[] EC_GROUP_get_cofactor(NativeRef.EC_GROUP groupRef);\n\n    static native long EC_POINT_new(NativeRef.EC_GROUP groupRef);\n\n    static native void EC_POINT_clear_free(long pointRef);\n\n    static native byte[][] EC_POINT_get_affine_coordinates(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pointRef);\n\n    static native void EC_POINT_set_affine_coordinates(\n            NativeRef.EC_GROUP groupRef, NativeRef.EC_POINT pointRef, byte[] x, byte[] y);\n\n    static native long EC_KEY_generate_key(NativeRef.EC_GROUP groupRef);\n\n    static native long EC_KEY_get1_group(NativeRef.EVP_PKEY pkeyRef);\n\n    static native byte[] EC_KEY_get_private_key(NativeRef.EVP_PKEY keyRef);\n\n    static native long EC_KEY_get_public_key(NativeRef.EVP_PKEY keyRef);\n\n    static native byte[] EC_KEY_marshal_curve_name(NativeRef.EC_GROUP groupRef) throws IOException;\n\n    static native long EC_KEY_parse_curve_name(byte[] encoded) throws IOException;\n\n    static native int ECDH_compute_key(byte[] out, int outOffset, NativeRef.EVP_PKEY publicKeyRef,\n            NativeRef.EVP_PKEY privateKeyRef) throws InvalidKeyException, IndexOutOfBoundsException;\n\n    static native int ECDSA_size(NativeRef.EVP_PKEY pkey);\n\n    static native int ECDSA_sign(byte[] data, byte[] sig, NativeRef.EVP_PKEY pkey);\n\n    static native int ECDSA_verify(byte[] data, byte[] sig, NativeRef.EVP_PKEY pkey);\n\n    // --- Curve25519 --------------\n\n    static native boolean X25519(byte[] out, byte[] privateKey, byte[] publicKey) throws InvalidKeyException;\n\n    static native void X25519_keypair(byte[] outPublicKey, byte[] outPrivateKey);\n\n    // --- Message digest functions --------------\n\n    // These return const references\n    static native long EVP_get_digestbyname(String name);\n\n    static native int EVP_MD_size(long evp_md_const);\n\n    // --- Message digest context functions --------------\n\n    static native long EVP_MD_CTX_create();\n\n    static native void EVP_MD_CTX_cleanup(NativeRef.EVP_MD_CTX ctx);\n\n    static native void EVP_MD_CTX_destroy(long ctx);\n\n    static native int EVP_MD_CTX_copy_ex(\n            NativeRef.EVP_MD_CTX dst_ctx, NativeRef.EVP_MD_CTX src_ctx);\n\n    // --- Digest handling functions -------------------------------------------\n\n    static native int EVP_DigestInit_ex(NativeRef.EVP_MD_CTX ctx, long evp_md);\n\n    static native void EVP_DigestUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native int EVP_DigestFinal_ex(NativeRef.EVP_MD_CTX ctx, byte[] hash, int offset);\n\n    // --- Signature handling functions ----------------------------------------\n\n    static native long EVP_DigestSignInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native long EVP_DigestVerifyInit(\n            NativeRef.EVP_MD_CTX ctx, long evpMdRef, NativeRef.EVP_PKEY key);\n\n    static native void EVP_DigestSignUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestSignUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native void EVP_DigestVerifyUpdate(\n            NativeRef.EVP_MD_CTX ctx, byte[] buffer, int offset, int length);\n\n    static native void EVP_DigestVerifyUpdateDirect(NativeRef.EVP_MD_CTX ctx, long ptr, int length);\n\n    static native byte[] EVP_DigestSignFinal(NativeRef.EVP_MD_CTX ctx);\n\n    static native boolean EVP_DigestVerifyFinal(NativeRef.EVP_MD_CTX ctx, byte[] signature,\n            int offset, int length) throws IndexOutOfBoundsException;\n\n    static native long EVP_PKEY_encrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_encrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native long EVP_PKEY_decrypt_init(NativeRef.EVP_PKEY pkey) throws InvalidKeyException;\n\n    static native int EVP_PKEY_decrypt(NativeRef.EVP_PKEY_CTX ctx, byte[] out, int outOffset,\n            byte[] input, int inOffset, int inLength)\n            throws IndexOutOfBoundsException, BadPaddingException;\n\n    static native void EVP_PKEY_CTX_free(long pkeyCtx);\n\n    static native void EVP_PKEY_CTX_set_rsa_padding(long ctx, int pad)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_pss_saltlen(long ctx, int len)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_mgf1_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_md(long ctx, long evpMdRef)\n            throws InvalidAlgorithmParameterException;\n\n    static native void EVP_PKEY_CTX_set_rsa_oaep_label(long ctx, byte[] label)\n            throws InvalidAlgorithmParameterException;\n\n    // --- Block ciphers -------------------------------------------------------\n\n    // These return const references\n    static native long EVP_get_cipherbyname(String string);\n\n    static native void EVP_CipherInit_ex(NativeRef.EVP_CIPHER_CTX ctx, long evpCipher, byte[] key,\n            byte[] iv, boolean encrypting);\n\n    static native int EVP_CipherUpdate(NativeRef.EVP_CIPHER_CTX ctx, byte[] out, int outOffset,\n            byte[] in, int inOffset, int inLength) throws IndexOutOfBoundsException;\n\n    static native int EVP_CipherFinal_ex(NativeRef.EVP_CIPHER_CTX ctx, byte[] out, int outOffset)\n            throws BadPaddingException, IllegalBlockSizeException;\n\n    static native int EVP_CIPHER_iv_length(long evpCipher);\n\n    static native long EVP_CIPHER_CTX_new();\n\n    static native int EVP_CIPHER_CTX_block_size(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native int get_EVP_CIPHER_CTX_buf_len(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native boolean get_EVP_CIPHER_CTX_final_used(NativeRef.EVP_CIPHER_CTX ctx);\n\n    static native void EVP_CIPHER_CTX_set_padding(\n            NativeRef.EVP_CIPHER_CTX ctx, boolean enablePadding);\n\n    static native void EVP_CIPHER_CTX_set_key_length(NativeRef.EVP_CIPHER_CTX ctx, int keyBitSize);\n\n    static native void EVP_CIPHER_CTX_free(long ctx);\n\n    // --- AEAD ----------------------------------------------------------------\n    static native long EVP_aead_aes_128_gcm();\n\n    static native long EVP_aead_aes_256_gcm();\n\n    static native long EVP_aead_chacha20_poly1305();\n\n    static native long EVP_aead_aes_128_gcm_siv();\n\n    static native long EVP_aead_aes_256_gcm_siv();\n\n    static native int EVP_AEAD_max_overhead(long evpAead);\n\n    static native int EVP_AEAD_nonce_length(long evpAead);\n\n    static native int EVP_AEAD_CTX_seal(long evpAead, byte[] key, int tagLengthInBytes, byte[] out,\n            int outOffset, byte[] nonce, byte[] in, int inOffset, int inLength, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_seal_buf(long evpAead, byte[] key, int tagLengthInBytes, ByteBuffer out,\n                                            byte[] nonce, ByteBuffer input, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_open(long evpAead, byte[] key, int tagLengthInBytes, byte[] out,\n            int outOffset, byte[] nonce, byte[] in, int inOffset, int inLength, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    static native int EVP_AEAD_CTX_open_buf(long evpAead, byte[] key, int tagLengthInBytes, ByteBuffer out,\n                                            byte[] nonce, ByteBuffer input, byte[] ad)\n            throws ShortBufferException, BadPaddingException;\n\n    // --- CMAC functions ------------------------------------------------------\n\n    static native long CMAC_CTX_new();\n\n    static native void CMAC_CTX_free(long ctx);\n\n    static native void CMAC_Init(NativeRef.CMAC_CTX ctx, byte[] key);\n\n    static native void CMAC_Update(NativeRef.CMAC_CTX ctx, byte[] in, int inOffset, int inLength);\n\n    static native void CMAC_UpdateDirect(NativeRef.CMAC_CTX ctx, long inPtr, int inLength);\n\n    static native byte[] CMAC_Final(NativeRef.CMAC_CTX ctx);\n\n    static native void CMAC_Reset(NativeRef.CMAC_CTX ctx);\n\n    // --- HMAC functions ------------------------------------------------------\n\n    static native long HMAC_CTX_new();\n\n    static native void HMAC_CTX_free(long ctx);\n\n...\n// Path: common/src/main/java/org/conscrypt/NativeRef.java\n/*\n * Copyright 2014 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\n/**\n * Used to hold onto native OpenSSL references and run finalization on those\n * objects. Individual types must subclass this and implement finalizer.\n */\nabstract class NativeRef {\n    final long address;\n\n    NativeRef(long address) {\n        if (address == 0) {\n            throw new NullPointerException(\"address == 0\");\n        }\n\n        this.address = address;\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (!(o instanceof NativeRef)) {\n            return false;\n        }\n\n        return ((NativeRef) o).address == address;\n    }\n\n    @Override\n    public int hashCode() {\n        return (int) (address ^ (address >>> 32));\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\")\n    protected void finalize() throws Throwable {\n        try {\n            if (address != 0) {\n                doFree(address);\n            }\n        } finally {\n            super.finalize();\n        }\n    }\n\n    abstract void doFree(long context);\n\n    static final class CMAC_CTX extends NativeRef {\n        CMAC_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.CMAC_CTX_free(context);\n        }\n    }\n\n    static final class EC_GROUP extends NativeRef {\n        EC_GROUP(long ctx) {\n            super(ctx);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EC_GROUP_clear_free(context);\n        }\n    }\n\n    static final class EC_POINT extends NativeRef {\n        EC_POINT(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EC_POINT_clear_free(context);\n        }\n    }\n\n    static final class EVP_CIPHER_CTX extends NativeRef {\n        EVP_CIPHER_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_CIPHER_CTX_free(context);\n        }\n    }\n\n    static final class EVP_HPKE_CTX extends NativeRef {\n        EVP_HPKE_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_HPKE_CTX_free(context);\n        }\n    }\n\n    static final class EVP_MD_CTX extends NativeRef {\n        EVP_MD_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_MD_CTX_destroy(context);\n        }\n    }\n\n    static final class EVP_PKEY extends NativeRef {\n        EVP_PKEY(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_PKEY_free(context);\n        }\n    }\n\n    static final class EVP_PKEY_CTX extends NativeRef {\n        EVP_PKEY_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.EVP_PKEY_CTX_free(context);\n        }\n    }\n\n    static final class HMAC_CTX extends NativeRef {\n        HMAC_CTX(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.HMAC_CTX_free(context);\n        }\n    }\n\n    static final class SSL_SESSION extends NativeRef {\n        SSL_SESSION(long nativePointer) {\n            super(nativePointer);\n        }\n\n        @Override\n        void doFree(long context) {\n            NativeCrypto.SSL_SESSION_free(context);\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLEvpCipher.java\n/*\n * Copyright (C) 2019 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.InvalidAlgorithmParameterException;\nimport java.security.InvalidKeyException;\nimport java.security.SecureRandom;\nimport java.security.spec.AlgorithmParameterSpec;\nimport javax.crypto.BadPaddingException;\nimport javax.crypto.IllegalBlockSizeException;\nimport javax.crypto.ShortBufferException;\nimport javax.crypto.spec.IvParameterSpec;\nimport org.conscrypt.NativeRef.EVP_CIPHER_CTX;\n\n@Internal\npublic abstract class OpenSSLEvpCipher extends OpenSSLCipher {\n    /**\n     * Native pointer for the OpenSSL EVP_CIPHER context.\n     */\n    private final EVP_CIPHER_CTX cipherCtx = new EVP_CIPHER_CTX(\n            NativeCrypto.EVP_CIPHER_CTX_new());\n\n    /**\n     * Whether the cipher has processed any data yet. EVP_CIPHER doesn't\n     * like calling \"doFinal()\" in decryption mode without processing any\n     * updates.\n     */\n    private boolean calledUpdate;\n\n    /**\n     * The block size of the current mode.\n     */\n    private int modeBlockSize;\n\n    protected OpenSSLEvpCipher(Mode mode, Padding padding) {\n        super(mode, padding);\n    }\n\n    @Override\n    void engineInitInternal(byte[] encodedKey, AlgorithmParameterSpec params,\n            SecureRandom random) throws InvalidKeyException,\n        InvalidAlgorithmParameterException {\n        byte[] iv;\n        if (params instanceof IvParameterSpec) {\n            IvParameterSpec ivParams = (IvParameterSpec) params;\n            iv = ivParams.getIV();\n        } else {\n            iv = null;\n        }\n\n        final long cipherType = NativeCrypto.EVP_get_cipherbyname(getCipherName(\n                encodedKey.length, mode));\n        if (cipherType == 0) {\n            throw new InvalidAlgorithmParameterException(\"Cannot find name for key length = \"\n                    + (encodedKey.length * 8) + \" and mode = \" + mode);\n        }\n\n        final boolean encrypting = isEncrypting();\n\n        final int expectedIvLength = NativeCrypto.EVP_CIPHER_iv_length(cipherType);\n        if (iv == null && expectedIvLength != 0) {\n            if (!encrypting) {\n                throw new InvalidAlgorithmParameterException(\"IV must be specified in \" + mode\n                        + \" mode\");\n            }\n\n            iv = new byte[expectedIvLength];\n            if (random != null) {\n                random.nextBytes(iv);\n            } else {\n                NativeCrypto.RAND_bytes(iv);\n            }\n        } else if (expectedIvLength == 0 && iv != null) {\n            throw new InvalidAlgorithmParameterException(\"IV not used in \" + mode + \" mode\");\n        } else if (iv != null && iv.length != expectedIvLength) {\n            throw new InvalidAlgorithmParameterException(\"expected IV length of \"\n                    + expectedIvLength + \" but was \" + iv.length);\n        }\n\n        this.iv = iv;\n\n        if (supportsVariableSizeKey()) {\n            NativeCrypto.EVP_CipherInit_ex(cipherCtx, cipherType, null, null, encrypting);\n            NativeCrypto.EVP_CIPHER_CTX_set_key_length(cipherCtx, encodedKey.length);\n            NativeCrypto.EVP_CipherInit_ex(cipherCtx, 0, encodedKey, iv, isEncrypting());\n        } else {\n            NativeCrypto.EVP_CipherInit_ex(cipherCtx, cipherType, encodedKey, iv, encrypting);\n        }\n\n        // OpenSSL only supports PKCS5 Padding.\n        NativeCrypto\n                .EVP_CIPHER_CTX_set_padding(cipherCtx, getPadding() == Padding.PKCS5PADDING);\n        modeBlockSize = NativeCrypto.EVP_CIPHER_CTX_block_size(cipherCtx);\n        calledUpdate = false;\n    }\n\n    @Override\n    int updateInternal(byte[] input, int inputOffset, int inputLen, byte[] output,\n            int outputOffset, int maximumLen) throws ShortBufferException {\n        final int intialOutputOffset = outputOffset;\n\n        final int bytesLeft = output.length - outputOffset;\n        if (bytesLeft < maximumLen) {\n            throw new ShortBufferWithoutStackTraceException(\"output buffer too small during update: \"\n                    + bytesLeft + \" < \" + maximumLen);\n        }\n\n        outputOffset += NativeCrypto.EVP_CipherUpdate(cipherCtx, output, outputOffset, input,\n                inputOffset, inputLen);\n\n        calledUpdate = true;\n\n        return outputOffset - intialOutputOffset;\n    }\n\n    @Override\n    int doFinalInternal(byte[] output, int outputOffset, int maximumLen)\n            throws IllegalBlockSizeException, BadPaddingException, ShortBufferException {\n        /* Remember this so we can tell how many characters were written. */\n        final int initialOutputOffset = outputOffset;\n\n        /*\n         * If we're decrypting and haven't had any input, we should return\n         * null. Otherwise OpenSSL will complain if we call final.\n         */\n        if (!isEncrypting() && !calledUpdate) {\n            return 0;\n        }\n\n        /* Allow OpenSSL to pad if necessary and clean up state. */\n        final int bytesLeft = output.length - outputOffset;\n        final int writtenBytes;\n        if (bytesLeft >= maximumLen) {\n            writtenBytes = NativeCrypto.EVP_CipherFinal_ex(cipherCtx, output, outputOffset);\n        } else {\n            final byte[] lastBlock = new byte[maximumLen];\n            writtenBytes = NativeCrypto.EVP_CipherFinal_ex(cipherCtx, lastBlock, 0);\n            if (writtenBytes > bytesLeft) {\n                throw new ShortBufferWithoutStackTraceException(\"buffer is too short: \" + writtenBytes + \" > \"\n                        + bytesLeft);\n            } else if (writtenBytes > 0) {\n                System.arraycopy(lastBlock, 0, output, outputOffset, writtenBytes);\n            }\n        }\n        outputOffset += writtenBytes;\n\n        reset();\n\n        return outputOffset - initialOutputOffset;\n    }\n\n    @Override\n    int getOutputSizeForFinal(int inputLen) {\n        if (modeBlockSize == 1) {\n            return inputLen;\n        } else {\n            final int buffered = NativeCrypto.get_EVP_CIPHER_CTX_buf_len(cipherCtx);\n\n            if (getPadding() == Padding.NOPADDING) {\n                return buffered + inputLen;\n            } else {\n                final boolean finalUsed = NativeCrypto.get_EVP_CIPHER_CTX_final_used(cipherCtx);\n                // There is an additional buffer containing the possible final block.\n                int totalLen = inputLen + buffered + (finalUsed ? modeBlockSize : 0);\n                // Extra block for remainder bytes plus padding.\n                // In case it's encrypting and there are no remainder bytes, add an extra block\n                // consisting only of padding.\n                totalLen += ((totalLen % modeBlockSize != 0) || isEncrypting())\n                        ? modeBlockSize : 0;\n                // The minimum multiple of {@code modeBlockSize} that can hold all the bytes.\n                return totalLen - (totalLen % modeBlockSize);\n            }\n        }\n    }\n\n    @Override\n    int getOutputSizeForUpdate(int inputLen) {\n        return getOutputSizeForFinal(inputLen);\n    }\n\n    /**\n     * Returns the OpenSSL cipher name for the particular {@code keySize}\n     * and cipher {@code mode}.\n     */\n    abstract String getCipherName(int keySize, Mode mode);\n\n    /**\n     * Reset this Cipher instance state to process a new chunk of data.\n     */\n    private void reset() {\n        NativeCrypto.EVP_CipherInit_ex(cipherCtx, 0, encodedKey, iv, isEncrypting());\n        calledUpdate = false;\n    }\n\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLContextImpl.java\n/*\n * Copyright (C) 2010 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport static org.conscrypt.Platform.wrapEngine;\n\nimport java.io.IOException;\nimport java.security.GeneralSecurityException;\nimport java.security.KeyManagementException;\nimport java.security.SecureRandom;\nimport javax.net.ssl.KeyManager;\nimport javax.net.ssl.SSLContextSpi;\nimport javax.net.ssl.SSLEngine;\nimport javax.net.ssl.SSLServerSocketFactory;\nimport javax.net.ssl.SSLSessionContext;\nimport javax.net.ssl.SSLSocketFactory;\nimport javax.net.ssl.TrustManager;\n\n/**\n * OpenSSL-backed SSLContext service provider interface.\n *\n * <p>Public to allow contruction via the provider framework.\n */\n@Internal\npublic abstract class OpenSSLContextImpl extends SSLContextSpi {\n    /**\n     * The default SSLContextImpl for use with\n     * SSLContext.getInstance(\"Default\"). Protected by the\n     * DefaultSSLContextImpl.class monitor.\n     */\n    private static DefaultSSLContextImpl defaultSslContextImpl;\n\n    /** TLS protocols to enable by default. */\n    private final String[] protocols;\n\n    /** Client session cache. */\n    private final ClientSessionContext clientSessionContext;\n\n    /** Server session cache. */\n    private final ServerSessionContext serverSessionContext;\n\n    SSLParametersImpl sslParameters;\n\n    /** Allows outside callers to get the preferred SSLContext. */\n    static OpenSSLContextImpl getPreferred() {\n        return new TLSv13();\n    }\n\n    OpenSSLContextImpl(String[] protocols) {\n        this.protocols = protocols;\n        clientSessionContext = new ClientSessionContext();\n        serverSessionContext = new ServerSessionContext();\n    }\n\n    /**\n     * Constuctor for the DefaultSSLContextImpl.  The unused boolean parameter is solely to\n     * indicate that this constructor is desired.\n     */\n    @SuppressWarnings(\"StaticAssignmentInConstructor\")\n    OpenSSLContextImpl(String[] protocols, boolean unused)\n            throws GeneralSecurityException, IOException {\n        synchronized (DefaultSSLContextImpl.class) {\n            this.protocols = null;\n            // This is the only place defaultSslContextImpl is read or written so all\n            // access is actually synchronized.\n            if (defaultSslContextImpl == null) {\n                clientSessionContext = new ClientSessionContext();\n                serverSessionContext = new ServerSessionContext();\n                defaultSslContextImpl = (DefaultSSLContextImpl) this;\n            } else {\n                clientSessionContext =\n                    (ClientSessionContext) defaultSslContextImpl.engineGetClientSessionContext();\n                serverSessionContext =\n                    (ServerSessionContext) defaultSslContextImpl.engineGetServerSessionContext();\n            }\n            sslParameters = new SSLParametersImpl(defaultSslContextImpl.getKeyManagers(),\n                    defaultSslContextImpl.getTrustManagers(), null, clientSessionContext,\n                    serverSessionContext, protocols);\n        }\n    }\n\n    /**\n     * Initializes this {@code SSLContext} instance. All of the arguments are\n     * optional, and the security providers will be searched for the required\n     * implementations of the needed algorithms.\n     *\n     * @param kms the key sources or {@code null}\n     * @param tms the trust decision sources or {@code null}\n     * @param sr the randomness source or {@code null}\n     * @throws KeyManagementException if initializing this instance fails\n     */\n    @Override\n    public void engineInit(KeyManager[] kms, TrustManager[] tms, SecureRandom sr)\n            throws KeyManagementException {\n        sslParameters = new SSLParametersImpl(\n                kms, tms, sr, clientSessionContext, serverSessionContext, protocols);\n    }\n\n    @Override\n    public SSLSocketFactory engineGetSocketFactory() {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        return Platform.wrapSocketFactoryIfNeeded(new OpenSSLSocketFactoryImpl(sslParameters));\n    }\n\n    @Override\n    public SSLServerSocketFactory engineGetServerSocketFactory() {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        return new OpenSSLServerSocketFactoryImpl(sslParameters);\n    }\n\n    @Override\n    public SSLEngine engineCreateSSLEngine(String host, int port) {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        SSLParametersImpl p = (SSLParametersImpl) sslParameters.clone();\n        p.setUseClientMode(false);\n        return wrapEngine(new ConscryptEngine(host, port, p));\n    }\n\n    @Override\n    public SSLEngine engineCreateSSLEngine() {\n        if (sslParameters == null) {\n            throw new IllegalStateException(\"SSLContext is not initialized.\");\n        }\n        SSLParametersImpl p = (SSLParametersImpl) sslParameters.clone();\n        p.setUseClientMode(false);\n        return wrapEngine(new ConscryptEngine(p));\n    }\n\n    @Override\n    public SSLSessionContext engineGetServerSessionContext() {\n        return serverSessionContext;\n    }\n\n    @Override\n    public SSLSessionContext engineGetClientSessionContext() {\n        return clientSessionContext;\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv13 extends OpenSSLContextImpl {\n        public TLSv13() {\n            super(NativeCrypto.TLSV13_PROTOCOLS);\n        }\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv12 extends OpenSSLContextImpl {\n        public TLSv12() {\n            super(NativeCrypto.TLSV12_PROTOCOLS);\n        }\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv11 extends OpenSSLContextImpl {\n        public TLSv11() {\n            super(NativeCrypto.TLSV11_PROTOCOLS);\n        }\n    }\n\n    /**\n     * Public to allow construction via the provider framework.\n     */\n    public static final class TLSv1 extends OpenSSLContextImpl {\n        public TLSv1() {\n            super(NativeCrypto.TLSV1_PROTOCOLS);\n        }\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/SessionSnapshot.java\n/*\n * Copyright (C) 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.Principal;\nimport java.security.cert.Certificate;\nimport java.security.cert.X509Certificate;\nimport java.util.ArrayList;\nimport java.util.List;\nimport javax.net.ssl.SSLPeerUnverifiedException;\nimport javax.net.ssl.SSLSessionContext;\n\n/**\n * A snapshot of the content of another {@link ConscryptSession}. This copies everything over\n * except for the certificates.\n */\nfinal class SessionSnapshot implements ConscryptSession {\n    private final SSLSessionContext sessionContext;\n    private final byte[] id;\n    private final String requestedServerName;\n    private final List<byte[]> statusResponses;\n    private final byte[] peerTlsSctData;\n    private final long creationTime;\n    private final long lastAccessedTime;\n    private final String cipherSuite;\n    private final String protocol;\n    private final String peerHost;\n    private final String applicationProtocol;\n    private final int peerPort;\n\n    SessionSnapshot(ConscryptSession session) {\n        sessionContext = session.getSessionContext();\n        id = session.getId();\n        requestedServerName = session.getRequestedServerName();\n        statusResponses = session.getStatusResponses();\n        peerTlsSctData = session.getPeerSignedCertificateTimestamp();\n        creationTime = session.getCreationTime();\n        lastAccessedTime = session.getLastAccessedTime();\n        cipherSuite = session.getCipherSuite();\n        protocol = session.getProtocol();\n        peerHost = session.getPeerHost();\n        peerPort = session.getPeerPort();\n        applicationProtocol = session.getApplicationProtocol();\n    }\n\n    @Override\n    public String getRequestedServerName() {\n        return requestedServerName;\n    }\n\n    @Override\n    public List<byte[]> getStatusResponses() {\n        List<byte[]> ret = new ArrayList<>(statusResponses.size());\n        for (byte[] resp : statusResponses) {\n            ret.add(resp.clone());\n        }\n        return ret;\n    }\n\n    @Override\n    public byte[] getPeerSignedCertificateTimestamp() {\n        return peerTlsSctData != null ? peerTlsSctData.clone() : null;\n    }\n\n    @Override\n    public byte[] getId() {\n        return id;\n    }\n\n    @Override\n    public SSLSessionContext getSessionContext() {\n        return sessionContext;\n    }\n\n    @Override\n    public long getCreationTime() {\n        return creationTime;\n    }\n\n    @Override\n    public long getLastAccessedTime() {\n        return lastAccessedTime;\n    }\n\n    @Override\n    public void invalidate() {\n        // Do nothing.\n    }\n\n    @Override\n    public boolean isValid() {\n        return false;\n    }\n\n    @Override\n    public void putValue(String s, Object o) {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public Object getValue(String s) {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public void removeValue(String s) {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public String[] getValueNames() {\n        throw new UnsupportedOperationException(\n                \"All calls to this method should be intercepted by ExternalSession.\");\n    }\n\n    @Override\n    public X509Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException {\n        throw new SSLPeerUnverifiedException(\"No peer certificates\");\n    }\n\n    @Override\n    public Certificate[] getLocalCertificates() {\n        return null;\n    }\n\n    @Override\n    @SuppressWarnings(\"deprecation\") // Public API\n    public javax.security.cert.X509Certificate[] getPeerCertificateChain()\n        throws SSLPeerUnverifiedException {\n        if (!Platform.isJavaxCertificateSupported()) {\n            throw new UnsupportedOperationException(\"Use getPeerCertificates() instead\");\n        }\n\n        throw new SSLPeerUnverifiedException(\"No peer certificates\");\n    }\n\n    @Override\n    public Principal getPeerPrincipal() throws SSLPeerUnverifiedException {\n        throw new SSLPeerUnverifiedException(\"No peer certificates\");\n    }\n\n    @Override\n    public Principal getLocalPrincipal() {\n        return null;\n    }\n\n    @Override\n    public String getCipherSuite() {\n        return cipherSuite;\n    }\n\n    @Override\n    public String getProtocol() {\n        return protocol;\n    }\n\n    @Override\n    public String getPeerHost() {\n        return peerHost;\n    }\n\n    @Override\n    public int getPeerPort() {\n        return peerPort;\n    }\n\n    @Override\n    public int getPacketBufferSize() {\n        return NativeConstants.SSL3_RT_MAX_PACKET_SIZE;\n    }\n\n    @Override\n    public int getApplicationBufferSize() {\n        return NativeConstants.SSL3_RT_MAX_PLAIN_LENGTH;\n    }\n\n    @Override\n    public String getApplicationProtocol() {\n        return applicationProtocol;\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/OpenSSLX25519Key.java\npackage org.conscrypt;\n\npublic interface OpenSSLX25519Key {\n    int X25519_KEY_SIZE_BYTES = 32;\n\n    byte[] getU();\n}\n\n// Path: common/src/main/java/org/conscrypt/Java8EngineSocket.java\n/*\n * Copyright 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.net.Socket;\nimport java.util.List;\nimport java.util.function.BiFunction;\nimport javax.net.ssl.SSLEngine;\nimport javax.net.ssl.SSLSocket;\n\n/**\n * A version of ConscryptEngineSocket that includes the new Java 9 (and potentially later\n * patches of 8) {@code setHandshakeApplicationProtocolSelector} API (which requires Java 8 for\n * compilation, due to the use of {@link BiFunction}).\n */\nfinal class Java8EngineSocket extends ConscryptEngineSocket {\n    private BiFunction<SSLSocket, List<String>, String> selector;\n\n    Java8EngineSocket(SSLParametersImpl sslParameters) throws IOException {\n        super(sslParameters);\n    }\n\n    Java8EngineSocket(String hostname, int port, SSLParametersImpl sslParameters)\n            throws IOException {\n        super(hostname, port, sslParameters);\n    }\n\n    Java8EngineSocket(InetAddress address, int port, SSLParametersImpl sslParameters)\n            throws IOException {\n        super(address, port, sslParameters);\n    }\n\n    Java8EngineSocket(String hostname, int port, InetAddress clientAddress, int clientPort,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(hostname, port, clientAddress, clientPort, sslParameters);\n    }\n\n    Java8EngineSocket(InetAddress address, int port, InetAddress clientAddress, int clientPort,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(address, port, clientAddress, clientPort, sslParameters);\n    }\n\n    Java8EngineSocket(Socket socket, String hostname, int port, boolean autoClose,\n            SSLParametersImpl sslParameters) throws IOException {\n        super(socket, hostname, port, autoClose, sslParameters);\n    }\n\n    /* @Override */\n    @SuppressWarnings(\"MissingOverride\") // For compilation with Java < 9.\n    public void setHandshakeApplicationProtocolSelector(\n            final BiFunction<SSLSocket, List<String>, String> selector) {\n        this.selector = selector;\n        setApplicationProtocolSelector(toApplicationProtocolSelector(selector));\n    }\n\n    /* @Override */\n    @SuppressWarnings(\"MissingOverride\") // For compilation with Java < 9.\n    public BiFunction<SSLSocket, List<String>, String> getHandshakeApplicationProtocolSelector() {\n        return selector;\n    }\n\n    private static ApplicationProtocolSelector toApplicationProtocolSelector(\n        final BiFunction<SSLSocket, List<String>, String> selector) {\n        return selector == null ? null : new ApplicationProtocolSelector() {\n            @Override\n            public String selectApplicationProtocol(SSLEngine socket, List<String> protocols) {\n                throw new UnsupportedOperationException();\n            }\n\n            @Override\n            public String selectApplicationProtocol(SSLSocket socket, List<String> protocols) {\n                return selector.apply(socket, protocols);\n            }\n        };\n    }\n}\n\n// Path: common/src/main/java/org/conscrypt/DuckTypedHpkeSpi.java\n/*\n * Copyright (C) 2023 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License\n */\n\npackage org.conscrypt;\n\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.security.GeneralSecurityException;\nimport java.security.InvalidKeyException;\nimport java.security.PrivateKey;\nimport java.security.PublicKey;\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * Duck typed implementation of {@link HpkeSpi}.\n * <p>\n * Will wrap any Object which implements all of the methods in HpkeSpi and delegate to them\n * by reflection.\n */\n@Internal\npublic class DuckTypedHpkeSpi implements HpkeSpi {\n  private final Object delegate;\n  private final Map<String, Method> methods = new HashMap<>();\n\n  private DuckTypedHpkeSpi(Object delegate) throws NoSuchMethodException {\n    this.delegate = delegate;\n\n    Class<?> sourceClass = delegate.getClass();\n    for (Method targetMethod : HpkeSpi.class.getMethods()) {\n      if (targetMethod.isSynthetic()) {\n        continue;\n      }\n\n      Method sourceMethod =\n          sourceClass.getMethod(targetMethod.getName(), targetMethod.getParameterTypes());\n      // Check that the return types match too.\n      Class<?> sourceReturnType = sourceMethod.getReturnType();\n      Class<?> targetReturnType = targetMethod.getReturnType();\n      if (!targetReturnType.isAssignableFrom(sourceReturnType)) {\n        throw new NoSuchMethodException(sourceMethod + \" return value (\" + sourceReturnType\n            + \") incompatible with target return value (\" + targetReturnType + \")\");\n      }\n      methods.put(sourceMethod.getName(), sourceMethod);\n    }\n  }\n\n  public static DuckTypedHpkeSpi newInstance(Object delegate) {\n    try {\n      return new DuckTypedHpkeSpi(delegate);\n    } catch (Exception ignored) {\n      return null;\n    }\n  }\n\n  private Object invoke(String methodName, Object... args) throws InvocationTargetException {\n    Method method = methods.get(methodName);\n    if (method == null) {\n      throw new IllegalStateException(\"DuckTypedHpkSpi internal error\");\n    }\n    try {\n      return method.invoke(delegate, args);\n    } catch (IllegalAccessException e) {\n      throw new IllegalStateException(\"DuckTypedHpkSpi internal error\", e);\n    } catch (InvocationTargetException e) {\n      if (e.getCause() instanceof RuntimeException){\n        throw (RuntimeException) e.getCause();\n      }\n      throw e;\n    }\n  }\n\n  private void invokeWithPossibleInvalidKey(String methodName, Object... args)\n      throws InvalidKeyException {\n    try {\n      invoke(methodName, args);\n    } catch (InvocationTargetException e) {\n      Throwable cause = e.getCause();\n      if (cause instanceof InvalidKeyException){\n        throw (InvalidKeyException) cause;\n      }\n      throw new IllegalStateException(cause);\n    }\n  }\n\n  private Object invokeWithPossibleGeneralSecurity(String methodName, Object... args)\n      throws GeneralSecurityException {\n    try {\n      return invoke(methodName, args);\n    } catch (InvocationTargetException e) {\n      Throwable cause = e.getCause();\n      if (cause instanceof GeneralSecurityException){\n        throw (GeneralSecurityException) cause;\n      }\n      throw new IllegalStateException(cause);\n    }\n  }\n\n  private Object invokeNoChecked(String methodName, Object... args) {\n    try {\n      return invoke(methodName, args);\n    } catch (InvocationTargetException e) {\n      throw new IllegalStateException(e.getCause());\n    }\n  }\n\n  // Visible for testing\n  public Object getDelegate() {\n    return delegate;\n  }\n\n  @Override\n  public void engineInitSender(\n          PublicKey recipientKey, byte[] info, PrivateKey senderKey, byte[] psk, byte[] pskId)\n          throws InvalidKeyException {\n    invokeWithPossibleInvalidKey(\"engineInitSender\", recipientKey, info, senderKey, psk, pskId);\n  }\n\n  @Override\n  public void engineInitSenderForTesting(PublicKey recipientKey, byte[] info, PrivateKey senderKey,\n          byte[] psk, byte[] pskId, byte[] sKe) throws InvalidKeyException {\n      invokeWithPossibleInvalidKey(\"engineInitSenderForTesting\",\n              recipientKey, info, senderKey, psk, pskId, sKe);\n  }\n\n  @Override\n  public void engineInitRecipient(byte[] encapsulated, PrivateKey key, byte[] info,\n          PublicKey senderKey, byte[] psk, byte[] psk_id) throws InvalidKeyException {\n    invokeWithPossibleInvalidKey(\n        \"engineInitRecipient\", encapsulated, key, info, senderKey, psk, psk_id);\n  }\n\n  @Override\n  public byte[] engineSeal(byte[] plaintext, byte[] aad) {\n      return (byte[]) invokeNoChecked(\"engineSeal\", plaintext, aad);\n  }\n\n  @Override\n  public byte[] engineExport(int length, byte[] exporterContext) {\n      return (byte[]) invokeNoChecked(\"engineExport\", length, exporterContext);\n  }\n\n  @Override\n  public byte[] engineOpen(byte[] ciphertext, byte[] aad) throws GeneralSecurityException {\n      return (byte[]) invokeWithPossibleGeneralSecurity(\"engineOpen\", ciphertext, aad);\n  }\n\n  @Override\n  public byte[] getEncapsulated() {\n      return (byte[]) invokeNoChecked(\"getEncapsulated\");\n  }\n}\n\n// Path: common/src/main/java/org/conscrypt/ChainStrengthAnalyzer.java\n/*\n * Copyright (C) 2012 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.conscrypt;\n\nimport java.security.cert.CertificateException;\nimport java.security.cert.X509Certificate;\nimport java.security.interfaces.DSAPublicKey;\nimport java.security.interfaces.ECPublicKey;\nimport java.security.interfaces.RSAPublicKey;\nimport java.util.List;\n\n/**\n * Analyzes the cryptographic strength of a chain of X.509 certificates.\n */\n@Internal\npublic final class ChainStrengthAnalyzer {\n\n    private static final int MIN_RSA_MODULUS_LEN_BITS = 1024;\n\n    private static final int MIN_EC_FIELD_SIZE_BITS = 160;\n\n    private static final int MIN_DSA_P_LEN_BITS = 1024;\n    private static final int MIN_DSA_Q_LEN_BITS = 160;\n\n    private static final String[] SIGNATURE_ALGORITHM_OID_BLACKLIST = {\n        \"1.2.840.113549.1.1.2\", // md2WithRSAEncryption\n        \"1.2.840.113549.1.1.3\", // md4WithRSAEncryption\n        \"1.2.840.113549.1.1.4\", // md5WithRSAEncryption\n        \"1.2.840.113549.1.1.5\", // sha1WithRSAEncryption\n        \"1.2.840.10040.4.3\", //dsa-with-sha1\n        \"1.2.840.10045.4.1\", //ecdsa-with-sha1\n    };\n\n    public static final void check(X509Certificate[] chain) throws CertificateException {\n        for (X509Certificate cert : chain) {\n            try {\n                checkCert(cert);\n            } catch (CertificateException e) {\n                throw new CertificateException(\"Unacceptable certificate: \"\n                        + cert.getSubjectX500Principal(), e);\n            }\n        }\n    }\n\n    public static final void check(List<X509Certificate> chain) throws CertificateException {\n        for (X509Certificate cert : chain) {\n            try {\n                checkCert(cert);\n            } catch (CertificateException e) {\n                throw new CertificateException(\"Unacceptable certificate: \"\n                        + cert.getSubjectX500Principal(), e);\n            }\n        }\n    }\n\n    \npublic static final void checkCert(X509Certificate cert) throws CertificateException {\n        checkKeyLength(cert);\n        checkSignatureAlgorithm(cert);\n    }\n\n    private static void checkKeyLength(X509Certificate cert) throws CertificateException {\n        Object pubkey = cert.getPublicKey();\n        if (pubkey instanceof RSAPublicKey) {\n            int modulusLength = ((RSAPublicKey) pubkey).getModulus().bitLength();\n            if (modulusLength < MIN_RSA_MODULUS_LEN_BITS) {\n                throw new CertificateException(\n                        \"RSA modulus is < \" + MIN_RSA_MODULUS_LEN_BITS + \" bits\");\n            }\n        } else if (pubkey instanceof ECPublicKey) {\n            int fieldSizeBits =\n                    ((ECPublicKey) pubkey).getParams().getCurve().getField().getFieldSize();\n            if (fieldSizeBits < MIN_EC_FIELD_SIZE_BITS) {\n                throw new CertificateException(\n                        \"EC key field size is < \" + MIN_EC_FIELD_SIZE_BITS + \" bits\");\n            }\n        } else if (pubkey instanceof DSAPublicKey) {\n            int pLength = ((DSAPublicKey) pubkey).getParams().getP().bitLength();\n            int qLength = ((DSAPublicKey) pubkey).getParams().getQ().bitLength();\n            if ((pLength < MIN_DSA_P_LEN_BITS) || (qLength < MIN_DSA_Q_LEN_BITS)) {\n                throw new CertificateException(\n                        \"DSA key length is < (\" + MIN_DSA_P_LEN_BITS + \", \" + MIN_DSA_Q_LEN_BITS\n                        + \") bits\");\n            }\n        } else {\n            // Unknown keys will be of type X509PublicKey.\n            throw new CertificateException(\"Rejecting unknown key class \" + pubkey.getClass().getName());\n        }\n    }\n\n    private static void checkSignatureAlgorithm(\n            X509Certificate cert) throws CertificateException {\n        String oid = cert.getSigAlgOID();\n        for (String blacklisted : SIGNATURE_ALGORITHM_OID_BLACKLIST) {\n            if (oid.equals(blacklisted)) {\n                throw new CertificateException(\"Signature uses an insecure hash function: \" + oid);\n            }\n        }\n    }\n}\n\n\n// Path: common/src/main/java/org/conscrypt/NativeSslSession.java\n/*\n * Copyright 2017 The Android Open Source Project\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\", \"www.google.com\", 443);\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n...\n", "needle_token_start": 15086, "needle_token_end": 15127, "code_context_ntokens": 15905, "output": ["```java\npublic static final void checkCert(X509Certificate cert) throws CertificateException {\n    checkKeyLength(cert);\n    checkSignatureAlgorithm(cert);\n}\n```"]}
{"repo": "apache/flink-ml", "name": "endInputX", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/AbstractBroadcastWrapperOperator.java", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: Handles the termination of input processing for a specific input channel in a data processing operator, ensuring that all pending data elements and watermarks are processed before concluding the input stream.\n2. **Input**: The function takes an integer representing the index of the input channel, and three consumer functions that handle different types of data: one for regular data elements, one for watermarks, and one for setting the key context.\n3. **Output**: There is no direct output; however, the function affects the state by processing any remaining data elements and watermarks.\n4. **Procedure**: The function first checks if the operator has additional functionalities (rich function). If not, it returns immediately. If there are additional functionalities, it waits until all broadcast variables are ready, yielding control if necessary. Once ready, it checks if there are any pending elements for the given input index. If there are, it processes these elements and watermarks using the provided consumer functions and then marks the pending elements as processed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        } else {\n            if (hasPendingElements[inputIndex]) {\n                processPendingElementsAndWatermarks(\n                        inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n                hasPendingElements[inputIndex] = false;\n            }\n            keyContextSetter.accept(streamRecord);\n            elementConsumer.accept(streamRecord);\n        }\n    }\n\n    /**\n     * Extracts common processing logic in subclasses' processing watermarks.\n     *\n     * @param watermark the input watermark.\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    protected void processWatermarkX(\n            Watermark watermark,\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        if (!hasRichFunction) {\n            watermarkConsumer.accept(watermark);\n        } else if (isBlocked[inputIndex]) {\n            while (!areBroadcastVariablesReady()) {\n                mailboxExecutor.yield();\n            }\n            watermarkConsumer.accept(watermark);\n        } else if (!areBroadcastVariablesReady()) {\n            dataCacheWriters[inputIndex].addRecord(\n                    CacheElement.newWatermark(watermark.getTimestamp()));\n        } else {\n            if (hasPendingElements[inputIndex]) {\n                processPendingElementsAndWatermarks(\n                        inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n                hasPendingElements[inputIndex] = false;\n            }\n            watermarkConsumer.accept(watermark);\n        }\n    }\n\n    /**\n     * Extracts common processing logic in subclasses' endInput(...).\n     *\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    \n@SuppressWarnings(\"rawtypes\")\n    protected void endInputX(\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        if (!hasRichFunction) {\n            return;\n        }\n        while (!areBroadcastVariablesReady()) {\n            mailboxExecutor.yield();\n        }\n        if (hasPendingElements[inputIndex]) {\n            processPendingElementsAndWatermarks(\n                    inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n            hasPendingElements[inputIndex] = false;\n        }\n    }\n\n    /**\n     * Processes the pending elements that are cached by {@link DataCacheWriter}.\n     *\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    private void processPendingElementsAndWatermarks(\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        List<Segment> pendingSegments = dataCacheWriters[inputIndex].getSegments();\n        if (pendingSegments.size() != 0) {\n            DataCacheReader dataCacheReader =\n                    new DataCacheReader<>(\n                            new CacheElementSerializer<>(inTypeSerializers[inputIndex]),\n                            pendingSegments);\n            while (dataCacheReader.hasNext()) {\n                CacheElement cacheElement = (CacheElement) dataCacheReader.next();\n                switch (cacheElement.getType()) {\n                    case RECORD:\n                        StreamRecord record = new StreamRecord(cacheElement.getRecord());\n                        keyContextSetter.accept(record);\n                        elementConsumer.accept(record);\n                        break;\n                    case WATERMARK:\n                        watermarkConsumer.accept(new Watermark(cacheElement.getWatermark()));\n                        break;\n                    default:\n                        throw new RuntimeException(\n                                \"Unsupported CacheElement type: \" + cacheElement.getType());\n                }\n            }\n            dataCacheWriters[inputIndex].clear();\n        }\n    }\n\n    @Override\n    public void open() throws Exception {\n        wrappedOperator.open();\n    }\n\n    @Override\n    public void close() throws Exception {\n        wrappedOperator.close();\n        if (!hasRichFunction) {\n            return;\n        }\n        for (String name : broadcastStreamNames) {\n            BroadcastContext.remove(name + \"-\" + indexOfSubtask);\n        }\n    }\n\n    @Override\n    public void finish() throws Exception {\n        wrappedOperator.finish();\n    }\n\n    @Override\n    public void prepareSnapshotPreBarrier(long checkpointId) throws Exception {\n        wrappedOperator.prepareSnapshotPreBarrier(checkpointId);\n    }\n\n    @Override\n    public void initializeState(StreamTaskStateInitializer streamTaskStateManager)\n            throws Exception {\n        final TypeSerializer<?> keySerializer =\n                streamConfig.getStateKeySerializer(containingTask.getUserCodeClassLoader());\n\n        StreamOperatorStateContext streamOperatorStateContext =\n                streamTaskStateManager.streamOperatorStateContext(\n                        getOperatorID(),\n                        getClass().getSimpleName(),\n                        parameters.getProcessingTimeService(),\n                        this,\n                        keySerializer,\n                        containingTask.getCancelables(),\n                        metrics,\n                        streamConfig.getManagedMemoryFractionOperatorUseCaseOfSlot(\n                                ManagedMemoryUseCase.STATE_BACKEND,\n                                containingTask\n                                        .getEnvironment()\n                                        .getTaskManagerInfo()\n                                        .getConfiguration(),\n                                containingTask.getUserCodeClassLoader()),\n                        false);\n        stateHandler =\n                new StreamOperatorStateHandler(\n                        streamOperatorStateContext,\n                        containingTask.getExecutionConfig(),\n                        containingTask.getCancelables());\n        stateHandler.initializeOperatorState(this);\n\n        timeServiceManager = streamOperatorStateContext.internalTimerServiceManager();\n\n        wrappedOperator.initializeState(\n                (operatorID,\n                        operatorClassName,\n                        processingTimeService,\n                        keyContext,\n                        keySerializerX,\n                        streamTaskCloseableRegistry,\n                        metricGroup,\n                        managedMemoryFraction,\n                        isUsingCustomRawKeyedState) ->\n                        new ProxyStreamOperatorStateContext(\n                                streamOperatorStateContext,\n                                \"wrapped-\",\n                                CloseableIterator.empty(),\n                                0));\n    }\n\n    @Override\n    public OperatorSnapshotFutures snapshotState(\n            long checkpointId,\n            long timestamp,\n            CheckpointOptions checkpointOptions,\n            CheckpointStreamFactory storageLocation)\n            throws Exception {\n        return stateHandler.snapshotState(\n                this,\n                Optional.ofNullable(timeServiceManager),\n                streamConfig.getOperatorName(),\n                checkpointId,\n                timestamp,\n                checkpointOptions,\n                storageLocation,\n                false);\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked, rawtypes\")\n    public void initializeState(StateInitializationContext stateInitializationContext)\n            throws Exception {\n        List<StatePartitionStreamProvider> inputs =\n                IteratorUtils.toList(\n                        stateInitializationContext.getRawOperatorStateInputs().iterator());\n        Preconditions.checkState(\n                inputs.size() < 2, \"The input from raw operator state should be one or zero.\");\n        if (!hasRichFunction) {\n            return;\n        }\n        if (inputs.size() == 0) {\n            for (int i = 0; i < numInputs; i++) {\n                dataCacheWriters[i] =\n                        new DataCacheWriter(\n                                new CacheElementSerializer(inTypeSerializers[i]),\n                                basePath.getFileSystem(),\n                                OperatorUtils.createDataCacheFileGenerator(\n                                        basePath, \"cache\", streamConfig.getOperatorID()));\n            }\n        } else {\n            InputStream inputStream = inputs.get(0).getStream();\n            DataInputStream dis =\n                    new DataInputStream(new NonClosingInputStreamDecorator(inputStream));\n            Preconditions.checkState(dis.readInt() == numInputs, \"Number of input is wrong.\");\n            for (int i = 0; i < numInputs; i++) {\n                DataCacheSnapshot dataCacheSnapshot =\n                        DataCacheSnapshot.recover(\n                                inputStream,\n                                basePath.getFileSystem(),\n                                OperatorUtils.createDataCacheFileGenerator(\n                                        basePath, \"cache\", streamConfig.getOperatorID()));\n                dataCacheWriters[i] =\n                        new DataCacheWriter(\n                                new CacheElementSerializer(inTypeSerializers[i]),\n                                basePath.getFileSystem(),\n                                OperatorUtils.createDataCacheFileGenerator(\n                                        basePath, \"cache\", streamConfig.getOperatorID()),\n                                dataCacheSnapshot.getSegments());\n            }\n        }\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    @Override\n    public void snapshotState(StateSnapshotContext stateSnapshotContext) throws Exception {\n        if (wrappedOperator instanceof StreamOperatorStateHandler.CheckpointedStreamOperator) {\n            ((CheckpointedStreamOperator) wrappedOperator).snapshotState(stateSnapshotContext);\n        }\n\n        if (!hasRichFunction) {\n            return;\n        }\n\n        OperatorStateCheckpointOutputStream checkpointOutputStream =\n                stateSnapshotContext.getRawOperatorStateOutput();\n        checkpointOutputStream.startNewPartition();\n        try (DataOutputStream dos =\n                new DataOutputStream(new NonClosingOutputStreamDecorator(checkpointOutputStream))) {\n            dos.writeInt(numInputs);\n        }\n        for (int i = 0; i < numInputs; i++) {\n            dataCacheWriters[i].writeSegmentsToFiles();\n            DataCacheSnapshot dataCacheSnapshot =\n                    new DataCacheSnapshot(\n                            basePath.getFileSystem(), null, dataCacheWriters[i].getSegments());\n            dataCacheSnapshot.writeTo(checkpointOutputStream);\n        }\n    }\n\n    @Override\n    public void setKeyContextElement1(StreamRecord<?> record) throws Exception {\n        wrappedOperator.setKeyContextElement1(record);\n    }\n\n    @Override\n    public void setKeyContextElement2(StreamRecord<?> record) throws Exception {\n        wrappedOperator.setKeyContextElement2(record);\n    }\n\n    @Override\n    public OperatorMetricGroup getMetricGroup() {\n        return wrappedOperator.getMetricGroup();\n    }\n\n    @Override\n    public OperatorID getOperatorID() {\n        return wrappedOperator.getOperatorID();\n    }\n\n    @Override\n    public void notifyCheckpointComplete(long checkpointId) throws Exception {\n        wrappedOperator.notifyCheckpointComplete(checkpointId);\n    }\n\n    @Override\n    public void notifyCheckpointAborted(long checkpointId) throws Exception {\n        wrappedOperator.notifyCheckpointAborted(checkpointId);\n    }\n\n    @Override\n    public void setCurrentKey(Object key) {\n        wrappedOperator.setCurrentKey(key);\n    }\n\n    @Override\n    public Object getCurrentKey() {\n        return wrappedOperator.getCurrentKey();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/util/RowUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.util;\n\nimport org.apache.flink.types.Row;\nimport org.apache.flink.types.RowKind;\n\n/** Utilities to deal with {@link Row} instances. */\npublic class RowUtils {\n\n    /**\n     * Creates a new row with fields that are copied from the existing one, and reserves some empty\n     * fields for further writing. The {@link RowKind} of the first row determines the {@link\n     * RowKind} of the result.\n     *\n     * @param existing The existing row to be cloned.\n     * @param reservedFields Num of fields to be reserved.\n     */\n    public static Row cloneWithReservedFields(Row existing, int reservedFields) {\n        Row result = new Row(existing.getKind(), existing.getArity() + reservedFields);\n        for (int i = 0; i < existing.getArity(); i++) {\n            result.setField(i, existing.getField(i));\n        }\n        return result;\n    }\n\n    /**\n     * Creates a new row with fields that are copied from the existing one, and appends a new filed\n     * with the specific value.\n     *\n     * @param existing The existing row to be cloned.\n     * @param value The value to be appended.\n     */\n    public static Row append(Row existing, Object value) {\n        Row result = cloneWithReservedFields(existing, 1);\n        result.setField(existing.getArity(), value);\n        return result;\n    }\n}\n\n", "needle_token_start": 719, "needle_token_end": 908, "code_context_ntokens": 3769, "output": ["```java\n@SuppressWarnings(\"rawtypes\")\nprotected void endInputX(\n        int inputIndex,\n        ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n        ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n        ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n        throws Exception {\n    if (!hasRichFunction) {\n        return;\n    }\n    while (!areBroadcastVariablesReady()) {\n        mailboxExecutor.yield();\n    }\n    if (hasPendingElements[inputIndex]) {\n        processPendingElementsAndWatermarks(\n                inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n        hasPendingElements[inputIndex] = false;\n    }\n}\n```"]}
{"repo": "apache/flink-ml", "name": "onEventTime", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/DataStreamUtils.java", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the event time-based triggering in a stream processing context, specifically to emit the accumulated result of a reduction operation when a specific time condition is met.\n2. **Input**: The function takes a timer object which contains the key associated with the data and a namespace indicating the scope of the timer.\n3. **Output**: Outputs a stream record containing the reduced value and a timestamp indicating the maximum long value, which typically represents an unbounded upper time limit.\n4. **Procedure**: The function first retrieves the current accumulated value from the state. If this value is not null, it then emits this value encapsulated in a stream record with the maximum possible timestamp, indicating the end of the event time window for this particular key.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "     *\n     * <p>The CoGroupFunction can iterate over the elements of both groups and return any number of\n     * elements including none.\n     *\n     * <p>NOTE: This method assumes both inputs are bounded.\n     *\n     * @param input1 The first data stream.\n     * @param input2 The second data stream.\n     * @param keySelector1 The KeySelector to be used for extracting the first input's key for\n     *     partitioning.\n     * @param keySelector2 The KeySelector to be used for extracting the second input's key for\n     *     partitioning.\n     * @param outTypeInformation The type information describing the output type.\n     * @param func The user-defined co-group function.\n     * @param <IN1> The class type of the first input.\n     * @param <IN2> The class type of the second input.\n     * @param <KEY> The class type of the key.\n     * @param <OUT> The class type of the output values.\n     * @return The result data stream.\n     */\n    public static <IN1, IN2, KEY extends Serializable, OUT> DataStream<OUT> coGroup(\n            DataStream<IN1> input1,\n            DataStream<IN2> input2,\n            KeySelector<IN1, KEY> keySelector1,\n            KeySelector<IN2, KEY> keySelector2,\n            TypeInformation<OUT> outTypeInformation,\n            CoGroupFunction<IN1, IN2, OUT> func) {\n        func = input1.getExecutionEnvironment().clean(func);\n        DataStream<OUT> result =\n                input1.connect(input2)\n                        .keyBy(keySelector1, keySelector2)\n                        .transform(\n                                \"CoGroupOperator\", outTypeInformation, new CoGroupOperator<>(func))\n                        .setParallelism(Math.max(input1.getParallelism(), input2.getParallelism()));\n        setManagedMemoryWeight(result, 100);\n        return result;\n    }\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    private static <IN, W extends Window> AllWindowedStream<IN, W> getAllWindowedStream(\n            DataStream<IN> input, Windows windows) {\n        if (windows instanceof CountTumblingWindows) {\n            long countWindowSize = ((CountTumblingWindows) windows).getSize();\n            return (AllWindowedStream<IN, W>) input.countWindowAll(countWindowSize);\n        } else {\n            return input.windowAll((WindowAssigner) getDataStreamTimeWindowAssigner(windows));\n        }\n    }\n\n    private static WindowAssigner<Object, TimeWindow> getDataStreamTimeWindowAssigner(\n            Windows windows) {\n        if (windows instanceof GlobalWindows) {\n            return EndOfStreamWindows.get();\n        } else if (windows instanceof EventTimeTumblingWindows) {\n            return TumblingEventTimeWindows.of(\n                    getStreamWindowTime(((EventTimeTumblingWindows) windows).getSize()));\n        } else if (windows instanceof ProcessingTimeTumblingWindows) {\n            return TumblingProcessingTimeWindows.of(\n                    getStreamWindowTime(((ProcessingTimeTumblingWindows) windows).getSize()));\n        } else if (windows instanceof EventTimeSessionWindows) {\n            return org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows\n                    .withGap(getStreamWindowTime(((EventTimeSessionWindows) windows).getGap()));\n        } else if (windows instanceof ProcessingTimeSessionWindows) {\n            return org.apache.flink.streaming.api.windowing.assigners.ProcessingTimeSessionWindows\n                    .withGap(\n                            getStreamWindowTime(((ProcessingTimeSessionWindows) windows).getGap()));\n        } else {\n            throw new UnsupportedOperationException(\n                    String.format(\n                            \"Unsupported Windows subclass: %s\", windows.getClass().getName()));\n        }\n    }\n\n    private static org.apache.flink.streaming.api.windowing.time.Time getStreamWindowTime(\n            Time time) {\n        return org.apache.flink.streaming.api.windowing.time.Time.of(\n                time.getSize(), time.getUnit());\n    }\n\n    /**\n     * A stream operator to apply {@link MapPartitionFunction} on each partition of the input\n     * bounded data stream.\n     */\n    private static class MapPartitionOperator<IN, OUT>\n            extends AbstractUdfStreamOperator<OUT, MapPartitionFunction<IN, OUT>>\n            implements OneInputStreamOperator<IN, OUT>, BoundedOneInput {\n\n        private ListStateWithCache<IN> valuesState;\n\n        public MapPartitionOperator(MapPartitionFunction<IN, OUT> mapPartitionFunc) {\n            super(mapPartitionFunc);\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n\n            final StreamTask<?, ?> containingTask = getContainingTask();\n            final OperatorID operatorID = config.getOperatorID();\n            final OperatorScopeManagedMemoryManager manager =\n                    OperatorScopeManagedMemoryManager.getOrCreate(containingTask, operatorID);\n            final String stateKey = \"values-state\";\n            manager.register(stateKey, 1.);\n            valuesState =\n                    new ListStateWithCache<>(\n                            getOperatorConfig().getTypeSerializerIn(0, getClass().getClassLoader()),\n                            stateKey,\n                            context,\n                            this);\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            valuesState.snapshotState(context);\n        }\n\n        @Override\n        public void processElement(StreamRecord<IN> input) throws Exception {\n            valuesState.add(input.getValue());\n        }\n\n        @Override\n        public void endInput() throws Exception {\n            userFunction.mapPartition(valuesState.get(), new TimestampedCollector<>(output));\n            valuesState.clear();\n        }\n    }\n\n    /** A stream operator to apply {@link ReduceFunction} on the input bounded data stream. */\n    private static class ReduceOperator<T> extends AbstractUdfStreamOperator<T, ReduceFunction<T>>\n            implements OneInputStreamOperator<T, T>, BoundedOneInput {\n        /** The temp result of the reduce function. */\n        private T result;\n\n        private ListState<T> state;\n\n        public ReduceOperator(ReduceFunction<T> userFunction) {\n            super(userFunction);\n        }\n\n        @Override\n        public void endInput() {\n            if (result != null) {\n                output.collect(new StreamRecord<>(result));\n            }\n        }\n\n        @Override\n        public void processElement(StreamRecord<T> streamRecord) throws Exception {\n            if (result == null) {\n                result = streamRecord.getValue();\n            } else {\n                result = userFunction.reduce(streamRecord.getValue(), result);\n            }\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n            state =\n                    context.getOperatorStateStore()\n                            .getListState(\n                                    new ListStateDescriptor<>(\n                                            \"state\",\n                                            getOperatorConfig()\n                                                    .getTypeSerializerIn(\n                                                            0, getClass().getClassLoader())));\n            result = OperatorStateUtils.getUniqueElement(state, \"state\").orElse(null);\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            state.clear();\n            if (result != null) {\n                state.add(result);\n            }\n        }\n    }\n\n    /**\n     * A stream operator to apply {@link ReduceFunction} on the input bounded keyed data stream.\n     *\n     * <p>Note: this class is a copy of {@link\n     * org.apache.flink.streaming.api.operators.BatchGroupedReduceOperator} in case of unexpected\n     * changes of its implementation.\n     */\n    private static class KeyedReduceOperator<IN, KEY>\n            extends AbstractUdfStreamOperator<IN, ReduceFunction<IN>>\n            implements OneInputStreamOperator<IN, IN>, Triggerable<KEY, VoidNamespace> {\n\n        private static final long serialVersionUID = 1L;\n\n        private static final String STATE_NAME = \"_op_state\";\n\n        private transient ValueState<IN> values;\n\n        private final TypeSerializer<IN> serializer;\n\n        private InternalTimerService<VoidNamespace> timerService;\n\n        public KeyedReduceOperator(ReduceFunction<IN> reducer, TypeSerializer<IN> serializer) {\n            super(reducer);\n            this.serializer = serializer;\n        }\n\n        @Override\n        public void open() throws Exception {\n            super.open();\n            ValueStateDescriptor<IN> stateId = new ValueStateDescriptor<>(STATE_NAME, serializer);\n            values = getPartitionedState(stateId);\n            timerService =\n                    getInternalTimerService(\"end-key-timers\", new VoidNamespaceSerializer(), this);\n        }\n\n        @Override\n        public void processElement(StreamRecord<IN> element) throws Exception {\n            IN value = element.getValue();\n            IN currentValue = values.value();\n\n            if (currentValue == null) {\n                // Registers a timer for emitting the result at the end when this is the\n                // first input for this key.\n                timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, Long.MAX_VALUE);\n            } else {\n                // Otherwise, reduces things.\n                value = userFunction.reduce(currentValue, value);\n            }\n            values.update(value);\n        }\n\n        \n@Override\n        public void onEventTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {\n            IN currentValue = values.value();\n            if (currentValue != null) {\n                output.collect(new StreamRecord<>(currentValue, Long.MAX_VALUE));\n            }\n        }\n\n        @Override\n        public void onProcessingTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {}\n    }\n\n    /**\n     * A stream operator to apply {@link AggregateFunction#add(IN, ACC)} on each partition of the\n     * input bounded data stream.\n     */\n    private static class PartialAggregateOperator<IN, ACC, OUT>\n            extends AbstractUdfStreamOperator<ACC, AggregateFunction<IN, ACC, OUT>>\n            implements OneInputStreamOperator<IN, ACC>, BoundedOneInput {\n        /** Type information of the accumulated result. */\n        private final TypeInformation<ACC> accType;\n        /** The accumulated result of the aggregate function in one partition. */\n        private ACC acc;\n        /** State of acc. */\n        private ListState<ACC> accState;\n\n        public PartialAggregateOperator(\n                AggregateFunction<IN, ACC, OUT> userFunction, TypeInformation<ACC> accType) {\n            super(userFunction);\n            this.accType = accType;\n        }\n\n        @Override\n        public void endInput() {\n            output.collect(new StreamRecord<>(acc));\n        }\n\n        @Override\n        public void processElement(StreamRecord<IN> streamRecord) throws Exception {\n            acc = userFunction.add(streamRecord.getValue(), acc);\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n            accState =\n                    context.getOperatorStateStore()\n                            .getListState(new ListStateDescriptor<>(\"accState\", accType));\n            acc =\n                    OperatorStateUtils.getUniqueElement(accState, \"accState\")\n                            .orElse(userFunction.createAccumulator());\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            accState.clear();\n            accState.add(acc);\n        }\n    }\n\n    /**\n     * A stream operator to apply {@link AggregateFunction#merge(ACC, ACC)} and {@link\n     * AggregateFunction#getResult(ACC)} on the input bounded data stream.\n     */\n    private static class AggregateOperator<IN, ACC, OUT>\n            extends AbstractUdfStreamOperator<OUT, AggregateFunction<IN, ACC, OUT>>\n            implements OneInputStreamOperator<ACC, OUT>, BoundedOneInput {\n        /** Type information of the accumulated result. */\n        private final TypeInformation<ACC> accType;\n        /** The accumulated result of the aggregate function in the final partition. */\n        private ACC acc;\n        /** State of acc. */\n        private ListState<ACC> accState;\n\n        public AggregateOperator(\n                AggregateFunction<IN, ACC, OUT> userFunction, TypeInformation<ACC> accType) {\n            super(userFunction);\n            this.accType = accType;\n        }\n\n        @Override\n        public void endInput() {\n            output.collect(new StreamRecord<>(userFunction.getResult(acc)));\n        }\n\n        @Override\n        public void processElement(StreamRecord<ACC> streamRecord) throws Exception {\n            if (acc == null) {\n                acc = streamRecord.getValue();\n            } else {\n                acc = userFunction.merge(streamRecord.getValue(), acc);\n            }\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n            accState =\n                    context.getOperatorStateStore()\n                            .getListState(new ListStateDescriptor<>(\"accState\", accType));\n            acc = OperatorStateUtils.getUniqueElement(accState, \"accState\").orElse(null);\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            accState.clear();\n            if (acc != null) {\n                accState.add(acc);\n            }\n        }\n    }\n\n    /**\n     * Splits the input data into global batches of batchSize. After splitting, each global batch is\n     * further split into local batches for downstream operators with each worker has one batch.\n     */\n    public static <T> DataStream<T[]> generateBatchData(\n            DataStream<T> inputData, final int downStreamParallelism, int batchSize) {\n        return inputData\n                .countWindowAll(batchSize)\n                .apply(new GlobalBatchCreator<>())\n                .flatMap(new GlobalBatchSplitter<>(downStreamParallelism))\n                .partitionCustom((chunkId, numPartitions) -> chunkId, x -> x.f0)\n                .map(\n                        new MapFunction<Tuple2<Integer, T[]>, T[]>() {\n                            @Override\n                            public T[] map(Tuple2<Integer, T[]> integerTuple2) throws Exception {\n                                return integerTuple2.f1;\n                            }\n                        });\n    }\n\n    /** Splits the input data into global batches. */\n    private static class GlobalBatchCreator<T> implements AllWindowFunction<T, T[], GlobalWindow> {\n        @Override\n        public void apply(GlobalWindow timeWindow, Iterable<T> iterable, Collector<T[]> collector) {\n            List<T> points = IteratorUtils.toList(iterable.iterator());\n            collector.collect(points.toArray((T[]) new Object[0]));\n        }\n    }\n\n    /**\n     * An operator that splits a global batch into evenly-sized local batches, and distributes them\n     * to downstream operator.\n     */\n    private static class GlobalBatchSplitter<T>\n            implements FlatMapFunction<T[], Tuple2<Integer, T[]>> {\n        private final int downStreamParallelism;\n\n        public GlobalBatchSplitter(int downStreamParallelism) {\n            this.downStreamParallelism = downStreamParallelism;\n        }\n\n        @Override\n        public void flatMap(T[] values, Collector<Tuple2<Integer, T[]>> collector) {\n            int div = values.length / downStreamParallelism;\n            int mod = values.length % downStreamParallelism;\n\n            int offset = 0;\n            int i = 0;\n\n            int size = div + 1;\n            for (; i < mod; i++) {\n                collector.collect(Tuple2.of(i, Arrays.copyOfRange(values, offset, offset + size)));\n                offset += size;\n            }\n\n            size = div;\n            for (; i < downStreamParallelism; i++) {\n                collector.collect(Tuple2.of(i, Arrays.copyOfRange(values, offset, offset + size)));\n                offset += size;\n            }\n        }\n    }\n\n    /*\n     * A stream operator that takes a randomly sampled subset of elements in a bounded data stream.\n     */\n    private static class SamplingOperator<T> extends AbstractStreamOperator<T>\n            implements OneInputStreamOperator<T, T>, BoundedOneInput {\n        private final int numSamples;\n\n        private final Random random;\n\n        private ListState<T> samplesState;\n\n        private List<T> samples;\n\n        private ListState<Integer> countState;\n\n        private int count;\n\n        SamplingOperator(int numSamples, long randomSeed) {\n            this.numSamples = numSamples;\n            this.random = new Random(randomSeed);\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n\n            ListStateDescriptor<T> samplesDescriptor =\n                    new ListStateDescriptor<>(\n                            \"samplesState\",\n                            getOperatorConfig()\n                                    .getTypeSerializerIn(0, getClass().getClassLoader()));\n            samplesState = context.getOperatorStateStore().getListState(samplesDescriptor);\n            samples = new ArrayList<>(numSamples);\n            samplesState.get().forEach(samples::add);\n\n            ListStateDescriptor<Integer> countDescriptor =\n                    new ListStateDescriptor<>(\"countState\", IntSerializer.INSTANCE);\n            countState = context.getOperatorStateStore().getListState(countDescriptor);\n            Iterator<Integer> countIterator = countState.get().iterator();\n            if (countIterator.hasNext()) {\n                count = countIterator.next();\n            } else {\n                count = 0;\n            }\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            samplesState.update(samples);\n            countState.update(Collections.singletonList(count));\n        }\n\n        @Override\n        public void processElement(StreamRecord<T> streamRecord) throws Exception {\n            T value = streamRecord.getValue();\n            count++;\n\n            if (samples.size() < numSamples) {\n                samples.add(value);\n            } else {\n                int index = random.nextInt(count);\n                if (index < numSamples) {\n                    samples.set(index, value);\n                }\n            }\n        }\n\n        @Override\n        public void endInput() throws Exception {\n            for (T sample : samples) {\n                output.collect(new StreamRecord<>(sample));\n            }\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/KeyAndValueSerializer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\nimport java.util.Objects;\n\n/**\n * A serializer used in {@link SortingDataInput} for serializing elements alongside their key and\n * timestamp. It serializes the record in a format known by the {@link FixedLengthByteKeyComparator}\n * and {@link VariableLengthByteKeyComparator}.\n *\n * <p>If the key is of known constant length, the length is not serialized with the data. Therefore\n * the serialized data is as follows:\n *\n * <pre>\n *      [key-length] | &lt;key&gt; | &lt;timestamp&gt; | &lt;record&gt;\n * </pre>\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class KeyAndValueSerializer<IN>\n        extends TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> {\n    private static final int TIMESTAMP_LENGTH = 8;\n    private final TypeSerializer<IN> valueSerializer;\n\n    // This represents either a variable length (-1) or a fixed one (>= 0).\n    private final int serializedKeyLength;\n\n    public KeyAndValueSerializer(TypeSerializer<IN> valueSerializer, int serializedKeyLength) {\n        this.valueSerializer = valueSerializer;\n        this.serializedKeyLength = serializedKeyLength;\n    }\n\n    @Override\n    public boolean isImmutableType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new KeyAndValueSerializer<>(valueSerializer.duplicate(), this.serializedKeyLength);\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> copy(Tuple2<byte[], StreamRecord<IN>> from) {\n        StreamRecord<IN> fromRecord = from.f1;\n        return Tuple2.of(\n                Arrays.copyOf(from.f0, from.f0.length),\n                fromRecord.copy(valueSerializer.copy(fromRecord.getValue())));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> createInstance() {\n        return Tuple2.of(new byte[0], new StreamRecord<>(valueSerializer.createInstance()));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> copy(\n            Tuple2<byte[], StreamRecord<IN>> from, Tuple2<byte[], StreamRecord<IN>> reuse) {\n        StreamRecord<IN> fromRecord = from.f1;\n        StreamRecord<IN> reuseRecord = reuse.f1;\n\n        IN valueCopy = valueSerializer.copy(fromRecord.getValue(), reuseRecord.getValue());\n        fromRecord.copyTo(valueCopy, reuseRecord);\n        reuse.f0 = Arrays.copyOf(from.f0, from.f0.length);\n        reuse.f1 = reuseRecord;\n        return reuse;\n    }\n\n    @Override\n    public int getLength() {\n        if (valueSerializer.getLength() < 0 || serializedKeyLength < 0) {\n            return -1;\n        }\n        return valueSerializer.getLength() + serializedKeyLength + TIMESTAMP_LENGTH;\n    }\n\n    @Override\n    public void serialize(Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target)\n            throws IOException {\n        if (serializedKeyLength < 0) {\n            target.writeInt(record.f0.length);\n        }\n        target.write(record.f0);\n        StreamRecord<IN> toSerialize = record.f1;\n        target.writeLong(toSerialize.getTimestamp());\n        valueSerializer.serialize(toSerialize.getValue(), target);\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> deserialize(DataInputView source) throws IOException {\n        final int length = getKeyLength(source);\n        byte[] bytes = new byte[length];\n        source.read(bytes);\n        long timestamp = source.readLong();\n        IN value = valueSerializer.deserialize(source);\n        return Tuple2.of(bytes, new StreamRecord<>(value, timestamp));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> deserialize(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        final int length = getKeyLength(source);\n        byte[] bytes = new byte[length];\n        source.read(bytes);\n        long timestamp = source.readLong();\n        IN value = valueSerializer.deserialize(source);\n        StreamRecord<IN> reuseRecord = reuse.f1;\n        reuseRecord.replace(value, timestamp);\n        reuse.f0 = bytes;\n        reuse.f1 = reuseRecord;\n        return reuse;\n    }\n\n    private int getKeyLength(DataInputView source) throws IOException {\n        final int length;\n        if (serializedKeyLength < 0) {\n            length = source.readInt();\n        } else {\n            length = serializedKeyLength;\n        }\n        return length;\n    }\n\n    @Override\n    public void copy(DataInputView source, DataOutputView target) throws IOException {\n        final int length;\n        if (serializedKeyLength < 0) {\n            length = source.readInt();\n            target.writeInt(length);\n        } else {\n            length = serializedKeyLength;\n        }\n        for (int i = 0; i < length; i++) {\n            target.writeByte(source.readByte());\n        }\n        target.writeLong(source.readLong());\n        valueSerializer.copy(source, target);\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        KeyAndValueSerializer<?> that = (KeyAndValueSerializer<?>) o;\n        return Objects.equals(valueSerializer, that.valueSerializer);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(valueSerializer);\n    }\n\n    @Override\n    public TypeSerializerSnapshot<Tuple2<byte[], StreamRecord<IN>>> snapshotConfiguration() {\n        throw new UnsupportedOperationException(\n                \"The KeyAndValueSerializer should not be used for persisting into State!\");\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/FixedLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps. It uses\n * binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are always of a fixed length and thus the length of the record is not\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class FixedLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    static final int TIMESTAMP_BYTE_SIZE = 8;\n    private final int keyLength;\n    private byte[] keyReference;\n    private long timestampReference;\n\n    public FixedLengthByteKeyComparator(int keyLength) {\n        this.keyLength = keyLength;\n    }\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = toCompare.f0;\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((FixedLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((FixedLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        for (int i = 0; i < keyLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return 0;\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int minCount = keyLength;\n        while (minCount-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return Long.compare(firstSource.readLong(), secondSource.readLong());\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return keyLength + TIMESTAMP_BYTE_SIZE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return keyBytes < getNormalizeKeyLen();\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(record, keyLength, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new FixedLengthByteKeyComparator<>(this.keyLength);\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/BytesKeyNormalizationUtil.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\n/**\n * Utility class for common key normalization used both in {@link VariableLengthByteKeyComparator}\n * and {@link FixedLengthByteKeyComparator}.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\nfinal class BytesKeyNormalizationUtil {\n    /**\n     * Writes the normalized key of given record. The normalized key consists of the key serialized\n     * as bytes and the timestamp of the record.\n     *\n     * <p>NOTE: The key does not represent a logical order. It can be used only for grouping keys!\n     */\n    static <IN> void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            int dataLength,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        byte[] data = record.f0;\n\n        if (dataLength >= numBytes) {\n            putBytesArray(target, offset, numBytes, data);\n        } else {\n            // whole key fits into the normalized key\n            putBytesArray(target, offset, dataLength, data);\n            int lastOffset = offset + numBytes;\n            offset += dataLength;\n            long valueOfTimestamp = record.f1.asRecord().getTimestamp() - Long.MIN_VALUE;\n            if (dataLength + FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE <= numBytes) {\n                // whole timestamp fits into the normalized key\n                target.putLong(offset, valueOfTimestamp);\n                offset += FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE;\n                // fill in the remaining space with zeros\n                while (offset < lastOffset) {\n                    target.put(offset++, (byte) 0);\n                }\n            } else {\n                // only part of the timestamp fits into normalized key\n                for (int i = 0; offset < lastOffset; offset++, i++) {\n                    target.put(offset, (byte) (valueOfTimestamp >>> ((7 - i) << 3)));\n                }\n            }\n        }\n    }\n\n    private static void putBytesArray(MemorySegment target, int offset, int numBytes, byte[] data) {\n        for (int i = 0; i < numBytes; i++) {\n            // We're converting the signed byte in data into an unsigned representation.\n            // A Java byte goes from -128 to 127, i.e. is signed. By subtracting -128 (MIN_VALUE)\n            // here we're shifting the number to be from 0 to 255. The normalized key sorter sorts\n            // bytes as \"unsigned\", so we need to convert here to maintain a correct ordering.\n            int highByte = data[i] & 0xff;\n            highByte -= Byte.MIN_VALUE;\n            target.put(offset + i, (byte) highByte);\n        }\n    }\n\n    private BytesKeyNormalizationUtil() {}\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/VariableLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps,. It\n * uses binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are of a variable length and thus expects the length of the record to be\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class VariableLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    private byte[] keyReference;\n    private long timestampReference;\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = Arrays.copyOf(toCompare.f0, toCompare.f0.length);\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((VariableLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((VariableLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        int firstLength = first.length;\n        int secondLength = second.length;\n        int minLength = Math.min(firstLength, secondLength);\n        for (int i = 0; i < minLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        return Integer.compare(firstLength, secondLength);\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int firstLength = firstSource.readInt();\n        int secondLength = secondSource.readInt();\n        int minLength = Math.min(firstLength, secondLength);\n        while (minLength-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        int lengthCompare = Integer.compare(firstLength, secondLength);\n        if (lengthCompare != 0) {\n            return lengthCompare;\n        } else {\n            return Long.compare(firstSource.readLong(), secondSource.readLong());\n        }\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return Integer.MAX_VALUE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return true;\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(\n                record, record.f0.length, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new VariableLengthByteKeyComparator<>();\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastContext.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.annotation.VisibleForTesting;\nimport org.apache.flink.api.common.operators.MailboxExecutor;\nimport org.apache.flink.api.java.tuple.Tuple2;\n\nimport javax.annotation.Nullable;\n\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\n\n/**\n * Context to hold the broadcast variables and provides some utility function for accessing\n * broadcast variables.\n */\npublic class BroadcastContext {\n\n    /**\n     * stores broadcast data streams in a map. The key is broadcastName-partitionId and the value is\n     * {@link BroadcastItem}.\n     */\n    private static final ConcurrentHashMap<String, BroadcastItem> BROADCAST_VARIABLES =\n            new ConcurrentHashMap<>();\n\n    @VisibleForTesting\n    public static void putBroadcastVariable(String key, Tuple2<Boolean, List<?>> variable) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(variable.f0, variable.f1, null)\n                                : new BroadcastItem(variable.f0, variable.f1, v.mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    public static void putMailBoxExecutor(String key, MailboxExecutor mailboxExecutor) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(false, null, mailboxExecutor)\n                                : new BroadcastItem(v.cacheReady, v.cacheList, mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    @SuppressWarnings({\"unchecked\"})\n    public static <T> List<T> getBroadcastVariable(String key) {\n        return (List<T>) BROADCAST_VARIABLES.get(key).cacheList;\n    }\n\n    @VisibleForTesting\n    public static void remove(String key) {\n        BROADCAST_VARIABLES.remove(key);\n    }\n\n    @VisibleForTesting\n    public static void notifyCacheFinished(String key) {\n        BROADCAST_VARIABLES.computeIfPresent(\n                key,\n                (k, v) -> {\n                    // sends a dummy email to avoid possible stuck.\n                    if (null != v.mailboxExecutor) {\n                        v.mailboxExecutor.execute(() -> {}, \"empty mail\");\n                    }\n                    return new BroadcastItem(true, v.cacheList, v.mailboxExecutor);\n                });\n    }\n\n    @VisibleForTesting\n    public static boolean isCacheFinished(String key) {\n        return BROADCAST_VARIABLES.get(key).cacheReady;\n    }\n\n    /** Utility class to organize broadcast variables. */\n    private static class BroadcastItem {\n\n        /** whether this broadcast variable is ready to be consumed. */\n        private boolean cacheReady;\n\n        /** the cached list. */\n        private List<?> cacheList;\n\n        /** the mailboxExecutor of the consumer, used to avoid the possible stuck of consumer. */\n        private MailboxExecutor mailboxExecutor;\n\n        BroadcastItem(\n                boolean cacheReady,\n                @Nullable List<?> cacheList,\n                @Nullable MailboxExecutor mailboxExecutor) {\n            this.cacheReady = cacheReady;\n            this.cacheList = cacheList;\n            this.mailboxExecutor = mailboxExecutor;\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastVariableReceiverOperatorFactory.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\n\nimport java.io.Serializable;\n\n/** Factory class for {@link BroadcastVariableReceiverOperator}. */\npublic class BroadcastVariableReceiverOperatorFactory<OUT>\n        extends AbstractStreamOperatorFactory<OUT> implements Serializable {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastNames;\n\n    /** Types of the broadcast data streams. */\n    private final TypeInformation<?>[] inTypes;\n\n    public BroadcastVariableReceiverOperatorFactory(\n            String[] broadcastNames, TypeInformation<?>[] inTypes) {\n        this.broadcastNames = broadcastNames;\n        this.inTypes = inTypes;\n    }\n\n    @Override\n    public <T extends StreamOperator<OUT>> T createStreamOperator(\n            StreamOperatorParameters<OUT> parameters) {\n        return (T) new BroadcastVariableReceiverOperator(parameters, broadcastNames, inTypes);\n    }\n\n    @Override\n    public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {\n        return BroadcastVariableReceiverOperator.class;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastWrapper.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.functions.KeySelector;\nimport org.apache.flink.iteration.operator.OperatorWrapper;\nimport org.apache.flink.streaming.api.operators.OneInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.TwoInputStreamOperator;\nimport org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;\nimport org.apache.flink.util.OutputTag;\n\n/** The operator wrapper for {@link AbstractBroadcastWrapperOperator}. */\npublic class BroadcastWrapper<T> implements OperatorWrapper<T, T> {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastStreamNames;\n\n    public BroadcastWrapper(String[] broadcastStreamNames) {\n        this.broadcastStreamNames = broadcastStreamNames;\n    }\n\n    @Override\n    public StreamOperator<T> wrap(\n            StreamOperatorParameters<T> operatorParameters,\n            StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new OneInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new TwoInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"Unsupported operator class for with-broadcast wrapper: \" + operatorClass);\n        }\n    }\n\n    @Override\n    public Class<? extends StreamOperator> getStreamOperatorClass(\n            ClassLoader classLoader, StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return OneInputBroadcastWrapperOperator.class;\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return TwoInputBroadcastWrapperOperator.class;\n        } else {\n            throw new UnsupportedOperationException(\n                    \"Unsupported operator class for with-broadcast wrapper: \" + operatorClass);\n        }\n    }\n\n    @Override\n    public <KEY> KeySelector<T, KEY> wrapKeySelector(KeySelector<T, KEY> keySelector) {\n        return keySelector;\n    }\n\n    @Override\n    public StreamPartitioner<T> wrapStreamPartitioner(StreamPartitioner<T> streamPartitioner) {\n        return streamPartitioner;\n    }\n\n    @Override\n    public OutputTag<T> wrapOutputTag(OutputTag<T> outputTag) {\n        return outputTag;\n    }\n\n    @Override\n    public TypeInformation<T> getWrappedTypeInfo(TypeInformation<T> typeInfo) {\n        return typeInfo;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.iteration.compile.DraftExecutionEnvironment;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastVariableReceiverOperatorFactory;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastWrapper;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.datastream.MultipleConnectedStreams;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.streaming.api.operators.ChainingStrategy;\nimport org.apache.flink.streaming.api.transformations.MultipleInputTransformation;\nimport org.apache.flink.streaming.api.transformations.PhysicalTransformation;\nimport org.apache.flink.util.AbstractID;\nimport org.apache.flink.util.Preconditions;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.function.Function;\n\n/** Utility class to support withBroadcast in DataStream. */\npublic class BroadcastUtils {\n    /**\n     * Supports withBroadcastStream in DataStream API. Broadcast data streams are available at all\n     * parallel instances of an operator that extends {@code\n     * org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator<OUT, ? extends\n     * org.apache.flink.api.common.functions.RichFunction>}. Users can access the broadcast\n     * variables by {@code RichFunction.getRuntimeContext().getBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().hasBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().getBroadcastVariableWithInitializer(...)}.\n     *\n     * <p>In detail, the broadcast input data streams will be consumed first and further consumed by\n     * non-broadcast inputs. For now the non-broadcast input are cached by default to avoid the\n     * possible deadlocks.\n     *\n     * @param inputList non-broadcast input list.\n     * @param bcStreams map of the broadcast data streams, where the key is the name and the value\n     *     is the corresponding data stream.\n     * @param userDefinedFunction the user defined logic in which users can access the broadcast\n     *     data streams and produce the output data stream. Note that though users can add more than\n     *     one operator in this logic, but only the operator that generates the result stream can\n     *     contain a rich function and access the broadcast variables. Other operators will\n     *     encounter NPE when accessing the broadcast variables.\n     * @return the output data stream.\n     */\n    public static <OUT> DataStream<OUT> withBroadcastStream(\n            List<DataStream<?>> inputList,\n            Map<String, DataStream<?>> bcStreams,\n            Function<List<DataStream<?>>, DataStream<OUT>> userDefinedFunction) {\n        Preconditions.checkArgument(inputList.size() > 0);\n\n        StreamExecutionEnvironment env = inputList.get(0).getExecutionEnvironment();\n        String[] broadcastNames = new String[bcStreams.size()];\n        DataStream<?>[] broadcastInputs = new DataStream[bcStreams.size()];\n        TypeInformation<?>[] broadcastInTypes = new TypeInformation[bcStreams.size()];\n        int idx = 0;\n        final String broadcastId = new AbstractID().toHexString();\n        for (String name : bcStreams.keySet()) {\n            broadcastNames[idx] = broadcastId + \"-\" + name;\n            broadcastInputs[idx] = bcStreams.get(name);\n            broadcastInTypes[idx] = broadcastInputs[idx].getType();\n            idx++;\n        }\n\n        DataStream<OUT> resultStream =\n                getResultStream(env, inputList, broadcastNames, userDefinedFunction);\n        TypeInformation<OUT> outType = resultStream.getType();\n        final String coLocationKey = \"broadcast-co-location-\" + UUID.randomUUID();\n        DataStream<OUT> cachedBroadcastInputs =\n                cacheBroadcastVariables(\n                        env,\n                        broadcastNames,\n                        broadcastInputs,\n                        broadcastInTypes,\n                        resultStream.getParallelism(),\n                        outType);\n\n        boolean canCoLocate =\n                cachedBroadcastInputs.getTransformation() instanceof PhysicalTransformation\n                        && resultStream.getTransformation() instanceof PhysicalTransformation;\n        if (canCoLocate) {\n            ((PhysicalTransformation<?>) cachedBroadcastInputs.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n            ((PhysicalTransformation<?>) resultStream.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"cannot set chaining strategy on \"\n                            + cachedBroadcastInputs.getTransformation()\n                            + \" and \"\n                            + resultStream.getTransformation()\n                            + \".\");\n        }\n        cachedBroadcastInputs.getTransformation().setCoLocationGroupKey(coLocationKey);\n        resultStream.getTransformation().setCoLocationGroupKey(coLocationKey);\n\n        return cachedBroadcastInputs.union(resultStream);\n    }\n\n    /**\n     * Caches all broadcast input data streams in static variables and returns the result\n     * multi-input stream operator. The result multi-input stream operator emits nothing and the\n     * only functionality of this operator is to cache all the input records in ${@link\n     * BroadcastContext}.\n     *\n     * @param env execution environment.\n     * @param broadcastInputNames names of the broadcast input data streams.\n     * @param broadcastInputs list of the broadcast data streams.\n     * @param broadcastInTypes output types of the broadcast input data streams.\n     * @param parallelism parallelism.\n     * @param outType output type.\n     * @param <OUT> output type.\n     * @return the result multi-input stream operator.\n     */\n...\n", "needle_token_start": 2393, "needle_token_end": 2462, "code_context_ntokens": 16017, "output": ["```java\n@Override\npublic void onEventTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {\n    IN currentValue = values.value();\n    if (currentValue != null) {\n        output.collect(new StreamRecord<>(currentValue, Long.MAX_VALUE));\n    }\n}\n```"]}
{"repo": "apache/flink-ml", "name": "coGroup", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/DataStreamUtils.java", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: This function is designed to merge two data streams based on matching keys extracted from each stream. It allows for the application of a user-defined function to pairs of grouped data from both streams, handling cases where one stream may have keys that do not exist in the other.\n2. **Input**: The function takes two data streams, each with a mechanism to extract keys used for grouping, a function to process the grouped data, and type information for the output.\n3. **Output**: The result is a new data stream consisting of elements generated by the user-defined function, which processes pairs of groups from the two input streams.\n4. **Procedure**: The function first connects the two input streams and applies a key-based grouping to each. It then processes these grouped pairs using the provided function, which can iterate over the elements of both groups and produce output elements. The output stream's parallelism is set to the maximum of the parallelisms of the input streams, and memory management settings are applied.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;\nimport org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;\nimport org.apache.flink.streaming.api.windowing.windows.GlobalWindow;\nimport org.apache.flink.streaming.api.windowing.windows.TimeWindow;\nimport org.apache.flink.streaming.api.windowing.windows.Window;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.tasks.StreamTask;\nimport org.apache.flink.util.Collector;\n\nimport org.apache.commons.collections.IteratorUtils;\n\nimport java.io.Serializable;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collections;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Random;\n\nimport static org.apache.flink.iteration.utils.DataStreamUtils.setManagedMemoryWeight;\n\n/** Provides utility functions for {@link DataStream}. */\n@Internal\npublic class DataStreamUtils {\n    /**\n     * Applies allReduceSum on the input data stream. The input data stream is supposed to contain\n     * up to one double array in each partition. The result data stream has the same parallelism as\n     * the input, where each partition contains one double array that sums all of the double arrays\n     * in the input data stream.\n     *\n     * <p>Note that we throw exception when one of the following two cases happen:\n     * <li>There exists one partition that contains more than one double array.\n     * <li>The length of the double array is not consistent among all partitions.\n     *\n     * @param input The input data stream.\n     * @return The result data stream.\n     */\n    public static DataStream<double[]> allReduceSum(DataStream<double[]> input) {\n        return AllReduceImpl.allReduceSum(input);\n    }\n\n    /**\n     * Applies a {@link MapPartitionFunction} on a bounded data stream.\n     *\n     * @param input The input data stream.\n     * @param func The user defined mapPartition function.\n     * @param <IN> The class type of the input.\n     * @param <OUT> The class type of output.\n     * @return The result data stream.\n     */\n    public static <IN, OUT> DataStream<OUT> mapPartition(\n            DataStream<IN> input, MapPartitionFunction<IN, OUT> func) {\n        TypeInformation<OUT> outType =\n                TypeExtractor.getMapPartitionReturnTypes(func, input.getType(), null, true);\n        return mapPartition(input, func, outType);\n    }\n\n    /**\n     * Applies a {@link MapPartitionFunction} on a bounded data stream.\n     *\n     * @param input The input data stream.\n     * @param func The user defined mapPartition function.\n     * @param outType The type information of the output.\n     * @param <IN> The class type of the input.\n     * @param <OUT> The class type of output.\n     * @return The result data stream.\n     */\n    public static <IN, OUT> DataStream<OUT> mapPartition(\n            DataStream<IN> input,\n            MapPartitionFunction<IN, OUT> func,\n            TypeInformation<OUT> outType) {\n        func = input.getExecutionEnvironment().clean(func);\n        return input.transform(\"mapPartition\", outType, new MapPartitionOperator<>(func))\n                .setParallelism(input.getParallelism());\n    }\n\n    /**\n     * Applies a {@link ReduceFunction} on a bounded data stream. The output stream contains at most\n     * one stream record and its parallelism is one.\n     *\n     * @param input The input data stream.\n     * @param func The user defined reduce function.\n     * @param <T> The class type of the input.\n     * @return The result data stream.\n     */\n    public static <T> DataStream<T> reduce(DataStream<T> input, ReduceFunction<T> func) {\n        return reduce(input, func, input.getType());\n    }\n\n    /**\n     * Applies a {@link ReduceFunction} on a bounded data stream. The output stream contains at most\n     * one stream record and its parallelism is one.\n     *\n     * @param input The input data stream.\n     * @param func The user defined reduce function.\n     * @param outType The type information of the output.\n     * @param <T> The class type of the input.\n     * @return The result data stream.\n     */\n    public static <T> DataStream<T> reduce(\n            DataStream<T> input, ReduceFunction<T> func, TypeInformation<T> outType) {\n        func = input.getExecutionEnvironment().clean(func);\n        DataStream<T> partialReducedStream =\n                input.transform(\"reduce\", outType, new ReduceOperator<>(func))\n                        .setParallelism(input.getParallelism());\n        if (partialReducedStream.getParallelism() == 1) {\n            return partialReducedStream;\n        } else {\n            return partialReducedStream\n                    .transform(\"reduce\", outType, new ReduceOperator<>(func))\n                    .setParallelism(1);\n        }\n    }\n\n    /**\n     * Applies a {@link ReduceFunction} on a bounded keyed data stream. The output stream contains\n     * one stream record for each key.\n     *\n     * @param input The input keyed data stream.\n     * @param func The user defined reduce function.\n     * @param <T> The class type of input.\n     * @param <K> The key type of input.\n     * @return The result data stream.\n     */\n    public static <T, K> DataStream<T> reduce(KeyedStream<T, K> input, ReduceFunction<T> func) {\n        return reduce(input, func, input.getType());\n    }\n\n    /**\n     * Applies a {@link ReduceFunction} on a bounded keyed data stream. The output stream contains\n     * one stream record for each key.\n     *\n     * @param input The input keyed data stream.\n     * @param func The user defined reduce function.\n     * @param outType The type information of the output.\n     * @param <T> The class type of input.\n     * @param <K> The key type of input.\n     * @return The result data stream.\n     */\n    public static <T, K> DataStream<T> reduce(\n            KeyedStream<T, K> input, ReduceFunction<T> func, TypeInformation<T> outType) {\n        func = input.getExecutionEnvironment().clean(func);\n        return input.transform(\n                        \"Keyed Reduce\",\n                        outType,\n                        new KeyedReduceOperator<>(\n                                func, outType.createSerializer(input.getExecutionConfig())))\n                .setParallelism(input.getParallelism());\n    }\n\n    /**\n     * Aggregates the elements in each partition of the input bounded stream, and then merges the\n     * partial results of all partitions. The output stream contains the aggregated result and its\n     * parallelism is one.\n     *\n     * <p>Note: If the parallelism of the input stream is N, this method would invoke {@link\n     * AggregateFunction#createAccumulator()} N times and {@link AggregateFunction#merge(Object,\n     * Object)} N - 1 times. Thus the initial accumulator should be neutral (e.g. empty list for\n     * list concatenation or `0` for summation), otherwise the aggregation result would be affected\n     * by the parallelism of the input stream.\n     *\n     * @param input The input data stream.\n     * @param func The user defined aggregate function.\n     * @param <IN> The class type of the input.\n     * @param <ACC> The class type of the accumulated values.\n     * @param <OUT> The class type of the output values.\n     * @return The result data stream.\n     */\n    public static <IN, ACC, OUT> DataStream<OUT> aggregate(\n            DataStream<IN> input, AggregateFunction<IN, ACC, OUT> func) {\n        TypeInformation<ACC> accType =\n                TypeExtractor.getAggregateFunctionAccumulatorType(\n                        func, input.getType(), null, true);\n        TypeInformation<OUT> outType =\n                TypeExtractor.getAggregateFunctionReturnType(func, input.getType(), null, true);\n\n        return aggregate(input, func, accType, outType);\n    }\n\n    /**\n     * Aggregates the elements in each partition of the input bounded stream, and then merges the\n     * partial results of all partitions. The output stream contains the aggregated result and its\n     * parallelism is one.\n     *\n     * <p>Note: If the parallelism of the input stream is N, this method would invoke {@link\n     * AggregateFunction#createAccumulator()} N times and {@link AggregateFunction#merge(Object,\n     * Object)} N - 1 times. Thus the initial accumulator should be neutral (e.g. empty list for\n     * list concatenation or `0` for summation), otherwise the aggregation result would be affected\n     * by the parallelism of the input stream.\n     *\n     * @param input The input data stream.\n     * @param func The user defined aggregate function.\n     * @param accType The type of the accumulated values.\n     * @param outType The types of the output.\n     * @param <IN> The class type of the input.\n     * @param <ACC> The class type of the accumulated values.\n     * @param <OUT> The class type of the output values.\n     * @return The result data stream.\n     */\n    public static <IN, ACC, OUT> DataStream<OUT> aggregate(\n            DataStream<IN> input,\n            AggregateFunction<IN, ACC, OUT> func,\n            TypeInformation<ACC> accType,\n            TypeInformation<OUT> outType) {\n        func = input.getExecutionEnvironment().clean(func);\n        DataStream<ACC> partialAggregatedStream =\n                input.transform(\n                        \"partialAggregate\", accType, new PartialAggregateOperator<>(func, accType));\n        DataStream<OUT> aggregatedStream =\n                partialAggregatedStream.transform(\n                        \"aggregate\", outType, new AggregateOperator<>(func, accType));\n        aggregatedStream.getTransformation().setParallelism(1);\n\n        return aggregatedStream;\n    }\n\n    /**\n     * Performs an approximate uniform sampling over the elements in a bounded data stream. The\n     * difference of probabilities of two data points been sampled is bounded by O(numSamples * p *\n     * p / (M * M)), where p is the parallelism of the input stream, M is the total number of data\n     * points that the input stream contains.\n     *\n     * <p>This method takes samples without replacement. If the number of elements in the stream is\n     * smaller than expected number of samples, all elements will be included in the sample.\n     *\n     * @param input The input data stream.\n     * @param numSamples The number of elements to be sampled.\n     * @param randomSeed The seed to randomly pick elements as sample.\n     * @return A data stream containing a list of the sampled elements.\n     */\n    public static <T> DataStream<T> sample(DataStream<T> input, int numSamples, long randomSeed) {\n        int inputParallelism = input.getParallelism();\n\n        // The maximum difference of number of data points in each partition after calling\n        // `rebalance` is `inputParallelism`. As a result, extra `inputParallelism` data points are\n        // sampled for each partition in the first round.\n        int firstRoundNumSamples =\n                Math.min((numSamples / inputParallelism) + inputParallelism, numSamples);\n        return input.rebalance()\n                .transform(\n                        \"firstRoundSampling\",\n                        input.getType(),\n                        new SamplingOperator<>(firstRoundNumSamples, randomSeed))\n                .setParallelism(inputParallelism)\n                .transform(\n                        \"secondRoundSampling\",\n                        input.getType(),\n                        new SamplingOperator<>(numSamples, randomSeed))\n                .setParallelism(1)\n                .map(x -> x, input.getType())\n                .setParallelism(inputParallelism);\n    }\n\n    /**\n     * Creates windows from data in the non key grouped input stream and applies the given window\n     * function to each window.\n     *\n     * @param input The input data stream to be windowed and processed.\n     * @param windows The windowing strategy that defines how input data would be sliced into\n     *     batches.\n     * @param function The user defined process function.\n     * @return The data stream that is the result of applying the window function to each window.\n     */\n    public static <IN, OUT, W extends Window> SingleOutputStreamOperator<OUT> windowAllAndProcess(\n            DataStream<IN> input, Windows windows, ProcessAllWindowFunction<IN, OUT, W> function) {\n        function = input.getExecutionEnvironment().clean(function);\n        AllWindowedStream<IN, W> allWindowedStream = getAllWindowedStream(input, windows);\n        return allWindowedStream.process(function);\n    }\n\n    /**\n     * Creates windows from data in the non key grouped input stream and applies the given window\n     * function to each window.\n     *\n     * @param input The input data stream to be windowed and processed.\n     * @param windows The windowing strategy that defines how input data would be sliced into\n     *     batches.\n     * @param function The user defined process function.\n     * @param outType The type information of the output.\n     * @return The data stream that is the result of applying the window function to each window.\n     */\n    public static <IN, OUT, W extends Window> SingleOutputStreamOperator<OUT> windowAllAndProcess(\n            DataStream<IN> input,\n            Windows windows,\n            ProcessAllWindowFunction<IN, OUT, W> function,\n            TypeInformation<OUT> outType) {\n        function = input.getExecutionEnvironment().clean(function);\n        AllWindowedStream<IN, W> allWindowedStream = getAllWindowedStream(input, windows);\n        return allWindowedStream.process(function, outType);\n    }\n\n    /**\n     * A CoGroup transformation combines the elements of two {@link DataStream DataStreams} into one\n     * DataStream. It groups each DataStream individually on a key and gives groups of both\n     * DataStreams with equal keys together into a {@link\n     * org.apache.flink.api.common.functions.CoGroupFunction}. If a DataStream has a group with no\n     * matching key in the other DataStream, the CoGroupFunction is called with an empty group for\n     * the non-existing group.\n     *\n     * <p>The CoGroupFunction can iterate over the elements of both groups and return any number of\n     * elements including none.\n     *\n     * <p>NOTE: This method assumes both inputs are bounded.\n     *\n     * @param input1 The first data stream.\n     * @param input2 The second data stream.\n     * @param keySelector1 The KeySelector to be used for extracting the first input's key for\n     *     partitioning.\n     * @param keySelector2 The KeySelector to be used for extracting the second input's key for\n     *     partitioning.\n     * @param outTypeInformation The type information describing the output type.\n     * @param func The user-defined co-group function.\n     * @param <IN1> The class type of the first input.\n     * @param <IN2> The class type of the second input.\n     * @param <KEY> The class type of the key.\n     * @param <OUT> The class type of the output values.\n     * @return The result data stream.\n     */\n    \npublic static <IN1, IN2, KEY extends Serializable, OUT> DataStream<OUT> coGroup(\n            DataStream<IN1> input1,\n            DataStream<IN2> input2,\n            KeySelector<IN1, KEY> keySelector1,\n            KeySelector<IN2, KEY> keySelector2,\n            TypeInformation<OUT> outTypeInformation,\n            CoGroupFunction<IN1, IN2, OUT> func) {\n        func = input1.getExecutionEnvironment().clean(func);\n        DataStream<OUT> result =\n                input1.connect(input2)\n                        .keyBy(keySelector1, keySelector2)\n                        .transform(\n                                \"CoGroupOperator\", outTypeInformation, new CoGroupOperator<>(func))\n                        .setParallelism(Math.max(input1.getParallelism(), input2.getParallelism()));\n        setManagedMemoryWeight(result, 100);\n        return result;\n    }\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    private static <IN, W extends Window> AllWindowedStream<IN, W> getAllWindowedStream(\n            DataStream<IN> input, Windows windows) {\n        if (windows instanceof CountTumblingWindows) {\n            long countWindowSize = ((CountTumblingWindows) windows).getSize();\n            return (AllWindowedStream<IN, W>) input.countWindowAll(countWindowSize);\n        } else {\n            return input.windowAll((WindowAssigner) getDataStreamTimeWindowAssigner(windows));\n        }\n    }\n\n    private static WindowAssigner<Object, TimeWindow> getDataStreamTimeWindowAssigner(\n            Windows windows) {\n        if (windows instanceof GlobalWindows) {\n            return EndOfStreamWindows.get();\n        } else if (windows instanceof EventTimeTumblingWindows) {\n            return TumblingEventTimeWindows.of(\n                    getStreamWindowTime(((EventTimeTumblingWindows) windows).getSize()));\n        } else if (windows instanceof ProcessingTimeTumblingWindows) {\n            return TumblingProcessingTimeWindows.of(\n                    getStreamWindowTime(((ProcessingTimeTumblingWindows) windows).getSize()));\n        } else if (windows instanceof EventTimeSessionWindows) {\n            return org.apache.flink.streaming.api.windowing.assigners.EventTimeSessionWindows\n                    .withGap(getStreamWindowTime(((EventTimeSessionWindows) windows).getGap()));\n        } else if (windows instanceof ProcessingTimeSessionWindows) {\n            return org.apache.flink.streaming.api.windowing.assigners.ProcessingTimeSessionWindows\n                    .withGap(\n                            getStreamWindowTime(((ProcessingTimeSessionWindows) windows).getGap()));\n        } else {\n            throw new UnsupportedOperationException(\n                    String.format(\n                            \"Unsupported Windows subclass: %s\", windows.getClass().getName()));\n        }\n    }\n\n    private static org.apache.flink.streaming.api.windowing.time.Time getStreamWindowTime(\n            Time time) {\n        return org.apache.flink.streaming.api.windowing.time.Time.of(\n                time.getSize(), time.getUnit());\n    }\n\n    /**\n     * A stream operator to apply {@link MapPartitionFunction} on each partition of the input\n     * bounded data stream.\n     */\n    private static class MapPartitionOperator<IN, OUT>\n            extends AbstractUdfStreamOperator<OUT, MapPartitionFunction<IN, OUT>>\n            implements OneInputStreamOperator<IN, OUT>, BoundedOneInput {\n\n        private ListStateWithCache<IN> valuesState;\n\n        public MapPartitionOperator(MapPartitionFunction<IN, OUT> mapPartitionFunc) {\n            super(mapPartitionFunc);\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n\n            final StreamTask<?, ?> containingTask = getContainingTask();\n            final OperatorID operatorID = config.getOperatorID();\n            final OperatorScopeManagedMemoryManager manager =\n                    OperatorScopeManagedMemoryManager.getOrCreate(containingTask, operatorID);\n            final String stateKey = \"values-state\";\n            manager.register(stateKey, 1.);\n            valuesState =\n                    new ListStateWithCache<>(\n                            getOperatorConfig().getTypeSerializerIn(0, getClass().getClassLoader()),\n                            stateKey,\n                            context,\n                            this);\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            valuesState.snapshotState(context);\n        }\n\n        @Override\n        public void processElement(StreamRecord<IN> input) throws Exception {\n            valuesState.add(input.getValue());\n        }\n\n        @Override\n        public void endInput() throws Exception {\n            userFunction.mapPartition(valuesState.get(), new TimestampedCollector<>(output));\n            valuesState.clear();\n        }\n    }\n\n    /** A stream operator to apply {@link ReduceFunction} on the input bounded data stream. */\n    private static class ReduceOperator<T> extends AbstractUdfStreamOperator<T, ReduceFunction<T>>\n            implements OneInputStreamOperator<T, T>, BoundedOneInput {\n        /** The temp result of the reduce function. */\n        private T result;\n\n        private ListState<T> state;\n\n        public ReduceOperator(ReduceFunction<T> userFunction) {\n            super(userFunction);\n        }\n\n        @Override\n        public void endInput() {\n            if (result != null) {\n                output.collect(new StreamRecord<>(result));\n            }\n        }\n\n        @Override\n        public void processElement(StreamRecord<T> streamRecord) throws Exception {\n            if (result == null) {\n                result = streamRecord.getValue();\n            } else {\n                result = userFunction.reduce(streamRecord.getValue(), result);\n            }\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n            state =\n                    context.getOperatorStateStore()\n                            .getListState(\n                                    new ListStateDescriptor<>(\n                                            \"state\",\n                                            getOperatorConfig()\n                                                    .getTypeSerializerIn(\n                                                            0, getClass().getClassLoader())));\n            result = OperatorStateUtils.getUniqueElement(state, \"state\").orElse(null);\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            state.clear();\n            if (result != null) {\n                state.add(result);\n            }\n        }\n    }\n\n    /**\n     * A stream operator to apply {@link ReduceFunction} on the input bounded keyed data stream.\n     *\n     * <p>Note: this class is a copy of {@link\n     * org.apache.flink.streaming.api.operators.BatchGroupedReduceOperator} in case of unexpected\n     * changes of its implementation.\n     */\n    private static class KeyedReduceOperator<IN, KEY>\n            extends AbstractUdfStreamOperator<IN, ReduceFunction<IN>>\n            implements OneInputStreamOperator<IN, IN>, Triggerable<KEY, VoidNamespace> {\n\n        private static final long serialVersionUID = 1L;\n\n        private static final String STATE_NAME = \"_op_state\";\n\n        private transient ValueState<IN> values;\n\n        private final TypeSerializer<IN> serializer;\n\n        private InternalTimerService<VoidNamespace> timerService;\n\n        public KeyedReduceOperator(ReduceFunction<IN> reducer, TypeSerializer<IN> serializer) {\n            super(reducer);\n            this.serializer = serializer;\n        }\n\n        @Override\n        public void open() throws Exception {\n            super.open();\n            ValueStateDescriptor<IN> stateId = new ValueStateDescriptor<>(STATE_NAME, serializer);\n            values = getPartitionedState(stateId);\n            timerService =\n                    getInternalTimerService(\"end-key-timers\", new VoidNamespaceSerializer(), this);\n        }\n\n        @Override\n        public void processElement(StreamRecord<IN> element) throws Exception {\n            IN value = element.getValue();\n            IN currentValue = values.value();\n\n            if (currentValue == null) {\n                // Registers a timer for emitting the result at the end when this is the\n                // first input for this key.\n                timerService.registerEventTimeTimer(VoidNamespace.INSTANCE, Long.MAX_VALUE);\n            } else {\n                // Otherwise, reduces things.\n                value = userFunction.reduce(currentValue, value);\n            }\n            values.update(value);\n        }\n\n        @Override\n        public void onEventTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {\n            IN currentValue = values.value();\n            if (currentValue != null) {\n                output.collect(new StreamRecord<>(currentValue, Long.MAX_VALUE));\n            }\n        }\n\n        @Override\n        public void onProcessingTime(InternalTimer<KEY, VoidNamespace> timer) throws Exception {}\n    }\n\n    /**\n     * A stream operator to apply {@link AggregateFunction#add(IN, ACC)} on each partition of the\n     * input bounded data stream.\n     */\n    private static class PartialAggregateOperator<IN, ACC, OUT>\n            extends AbstractUdfStreamOperator<ACC, AggregateFunction<IN, ACC, OUT>>\n            implements OneInputStreamOperator<IN, ACC>, BoundedOneInput {\n        /** Type information of the accumulated result. */\n        private final TypeInformation<ACC> accType;\n        /** The accumulated result of the aggregate function in one partition. */\n        private ACC acc;\n        /** State of acc. */\n        private ListState<ACC> accState;\n\n        public PartialAggregateOperator(\n                AggregateFunction<IN, ACC, OUT> userFunction, TypeInformation<ACC> accType) {\n            super(userFunction);\n            this.accType = accType;\n        }\n\n        @Override\n        public void endInput() {\n            output.collect(new StreamRecord<>(acc));\n        }\n\n        @Override\n        public void processElement(StreamRecord<IN> streamRecord) throws Exception {\n            acc = userFunction.add(streamRecord.getValue(), acc);\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n            accState =\n                    context.getOperatorStateStore()\n                            .getListState(new ListStateDescriptor<>(\"accState\", accType));\n            acc =\n                    OperatorStateUtils.getUniqueElement(accState, \"accState\")\n                            .orElse(userFunction.createAccumulator());\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            accState.clear();\n            accState.add(acc);\n        }\n    }\n\n    /**\n     * A stream operator to apply {@link AggregateFunction#merge(ACC, ACC)} and {@link\n     * AggregateFunction#getResult(ACC)} on the input bounded data stream.\n     */\n    private static class AggregateOperator<IN, ACC, OUT>\n            extends AbstractUdfStreamOperator<OUT, AggregateFunction<IN, ACC, OUT>>\n            implements OneInputStreamOperator<ACC, OUT>, BoundedOneInput {\n        /** Type information of the accumulated result. */\n        private final TypeInformation<ACC> accType;\n        /** The accumulated result of the aggregate function in the final partition. */\n        private ACC acc;\n        /** State of acc. */\n        private ListState<ACC> accState;\n\n        public AggregateOperator(\n                AggregateFunction<IN, ACC, OUT> userFunction, TypeInformation<ACC> accType) {\n            super(userFunction);\n            this.accType = accType;\n        }\n\n        @Override\n        public void endInput() {\n            output.collect(new StreamRecord<>(userFunction.getResult(acc)));\n        }\n\n        @Override\n        public void processElement(StreamRecord<ACC> streamRecord) throws Exception {\n            if (acc == null) {\n                acc = streamRecord.getValue();\n            } else {\n                acc = userFunction.merge(streamRecord.getValue(), acc);\n            }\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n            accState =\n                    context.getOperatorStateStore()\n                            .getListState(new ListStateDescriptor<>(\"accState\", accType));\n            acc = OperatorStateUtils.getUniqueElement(accState, \"accState\").orElse(null);\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            accState.clear();\n            if (acc != null) {\n                accState.add(acc);\n            }\n        }\n    }\n\n    /**\n     * Splits the input data into global batches of batchSize. After splitting, each global batch is\n     * further split into local batches for downstream operators with each worker has one batch.\n     */\n    public static <T> DataStream<T[]> generateBatchData(\n            DataStream<T> inputData, final int downStreamParallelism, int batchSize) {\n        return inputData\n                .countWindowAll(batchSize)\n                .apply(new GlobalBatchCreator<>())\n                .flatMap(new GlobalBatchSplitter<>(downStreamParallelism))\n                .partitionCustom((chunkId, numPartitions) -> chunkId, x -> x.f0)\n                .map(\n                        new MapFunction<Tuple2<Integer, T[]>, T[]>() {\n                            @Override\n                            public T[] map(Tuple2<Integer, T[]> integerTuple2) throws Exception {\n                                return integerTuple2.f1;\n                            }\n                        });\n    }\n\n    /** Splits the input data into global batches. */\n    private static class GlobalBatchCreator<T> implements AllWindowFunction<T, T[], GlobalWindow> {\n        @Override\n        public void apply(GlobalWindow timeWindow, Iterable<T> iterable, Collector<T[]> collector) {\n            List<T> points = IteratorUtils.toList(iterable.iterator());\n            collector.collect(points.toArray((T[]) new Object[0]));\n        }\n    }\n\n    /**\n     * An operator that splits a global batch into evenly-sized local batches, and distributes them\n     * to downstream operator.\n     */\n    private static class GlobalBatchSplitter<T>\n            implements FlatMapFunction<T[], Tuple2<Integer, T[]>> {\n        private final int downStreamParallelism;\n\n        public GlobalBatchSplitter(int downStreamParallelism) {\n            this.downStreamParallelism = downStreamParallelism;\n        }\n\n        @Override\n        public void flatMap(T[] values, Collector<Tuple2<Integer, T[]>> collector) {\n            int div = values.length / downStreamParallelism;\n            int mod = values.length % downStreamParallelism;\n\n            int offset = 0;\n            int i = 0;\n\n            int size = div + 1;\n            for (; i < mod; i++) {\n                collector.collect(Tuple2.of(i, Arrays.copyOfRange(values, offset, offset + size)));\n                offset += size;\n            }\n\n            size = div;\n            for (; i < downStreamParallelism; i++) {\n                collector.collect(Tuple2.of(i, Arrays.copyOfRange(values, offset, offset + size)));\n                offset += size;\n            }\n        }\n    }\n\n    /*\n     * A stream operator that takes a randomly sampled subset of elements in a bounded data stream.\n     */\n    private static class SamplingOperator<T> extends AbstractStreamOperator<T>\n            implements OneInputStreamOperator<T, T>, BoundedOneInput {\n        private final int numSamples;\n\n        private final Random random;\n\n        private ListState<T> samplesState;\n\n        private List<T> samples;\n\n        private ListState<Integer> countState;\n\n        private int count;\n\n        SamplingOperator(int numSamples, long randomSeed) {\n            this.numSamples = numSamples;\n            this.random = new Random(randomSeed);\n        }\n\n        @Override\n        public void initializeState(StateInitializationContext context) throws Exception {\n            super.initializeState(context);\n\n            ListStateDescriptor<T> samplesDescriptor =\n                    new ListStateDescriptor<>(\n                            \"samplesState\",\n                            getOperatorConfig()\n                                    .getTypeSerializerIn(0, getClass().getClassLoader()));\n            samplesState = context.getOperatorStateStore().getListState(samplesDescriptor);\n            samples = new ArrayList<>(numSamples);\n            samplesState.get().forEach(samples::add);\n\n            ListStateDescriptor<Integer> countDescriptor =\n                    new ListStateDescriptor<>(\"countState\", IntSerializer.INSTANCE);\n            countState = context.getOperatorStateStore().getListState(countDescriptor);\n            Iterator<Integer> countIterator = countState.get().iterator();\n            if (countIterator.hasNext()) {\n                count = countIterator.next();\n            } else {\n                count = 0;\n            }\n        }\n\n        @Override\n        public void snapshotState(StateSnapshotContext context) throws Exception {\n            super.snapshotState(context);\n            samplesState.update(samples);\n            countState.update(Collections.singletonList(count));\n        }\n\n        @Override\n        public void processElement(StreamRecord<T> streamRecord) throws Exception {\n            T value = streamRecord.getValue();\n            count++;\n\n            if (samples.size() < numSamples) {\n                samples.add(value);\n            } else {\n                int index = random.nextInt(count);\n                if (index < numSamples) {\n                    samples.set(index, value);\n                }\n            }\n        }\n\n        @Override\n        public void endInput() throws Exception {\n            for (T sample : samples) {\n                output.collect(new StreamRecord<>(sample));\n            }\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/KeyAndValueSerializer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\nimport java.util.Objects;\n\n/**\n * A serializer used in {@link SortingDataInput} for serializing elements alongside their key and\n * timestamp. It serializes the record in a format known by the {@link FixedLengthByteKeyComparator}\n * and {@link VariableLengthByteKeyComparator}.\n *\n * <p>If the key is of known constant length, the length is not serialized with the data. Therefore\n * the serialized data is as follows:\n *\n * <pre>\n *      [key-length] | &lt;key&gt; | &lt;timestamp&gt; | &lt;record&gt;\n * </pre>\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class KeyAndValueSerializer<IN>\n        extends TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> {\n    private static final int TIMESTAMP_LENGTH = 8;\n    private final TypeSerializer<IN> valueSerializer;\n\n    // This represents either a variable length (-1) or a fixed one (>= 0).\n    private final int serializedKeyLength;\n\n    public KeyAndValueSerializer(TypeSerializer<IN> valueSerializer, int serializedKeyLength) {\n        this.valueSerializer = valueSerializer;\n        this.serializedKeyLength = serializedKeyLength;\n    }\n\n    @Override\n    public boolean isImmutableType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new KeyAndValueSerializer<>(valueSerializer.duplicate(), this.serializedKeyLength);\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> copy(Tuple2<byte[], StreamRecord<IN>> from) {\n        StreamRecord<IN> fromRecord = from.f1;\n        return Tuple2.of(\n                Arrays.copyOf(from.f0, from.f0.length),\n                fromRecord.copy(valueSerializer.copy(fromRecord.getValue())));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> createInstance() {\n        return Tuple2.of(new byte[0], new StreamRecord<>(valueSerializer.createInstance()));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> copy(\n            Tuple2<byte[], StreamRecord<IN>> from, Tuple2<byte[], StreamRecord<IN>> reuse) {\n        StreamRecord<IN> fromRecord = from.f1;\n        StreamRecord<IN> reuseRecord = reuse.f1;\n\n        IN valueCopy = valueSerializer.copy(fromRecord.getValue(), reuseRecord.getValue());\n        fromRecord.copyTo(valueCopy, reuseRecord);\n        reuse.f0 = Arrays.copyOf(from.f0, from.f0.length);\n        reuse.f1 = reuseRecord;\n        return reuse;\n    }\n\n    @Override\n    public int getLength() {\n        if (valueSerializer.getLength() < 0 || serializedKeyLength < 0) {\n            return -1;\n        }\n        return valueSerializer.getLength() + serializedKeyLength + TIMESTAMP_LENGTH;\n    }\n\n    @Override\n    public void serialize(Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target)\n            throws IOException {\n        if (serializedKeyLength < 0) {\n            target.writeInt(record.f0.length);\n        }\n        target.write(record.f0);\n        StreamRecord<IN> toSerialize = record.f1;\n        target.writeLong(toSerialize.getTimestamp());\n        valueSerializer.serialize(toSerialize.getValue(), target);\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> deserialize(DataInputView source) throws IOException {\n        final int length = getKeyLength(source);\n        byte[] bytes = new byte[length];\n        source.read(bytes);\n        long timestamp = source.readLong();\n        IN value = valueSerializer.deserialize(source);\n        return Tuple2.of(bytes, new StreamRecord<>(value, timestamp));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> deserialize(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        final int length = getKeyLength(source);\n        byte[] bytes = new byte[length];\n        source.read(bytes);\n        long timestamp = source.readLong();\n        IN value = valueSerializer.deserialize(source);\n        StreamRecord<IN> reuseRecord = reuse.f1;\n        reuseRecord.replace(value, timestamp);\n        reuse.f0 = bytes;\n        reuse.f1 = reuseRecord;\n        return reuse;\n    }\n\n    private int getKeyLength(DataInputView source) throws IOException {\n        final int length;\n        if (serializedKeyLength < 0) {\n            length = source.readInt();\n        } else {\n            length = serializedKeyLength;\n        }\n        return length;\n    }\n\n    @Override\n    public void copy(DataInputView source, DataOutputView target) throws IOException {\n        final int length;\n        if (serializedKeyLength < 0) {\n            length = source.readInt();\n            target.writeInt(length);\n        } else {\n            length = serializedKeyLength;\n        }\n        for (int i = 0; i < length; i++) {\n            target.writeByte(source.readByte());\n        }\n        target.writeLong(source.readLong());\n        valueSerializer.copy(source, target);\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        KeyAndValueSerializer<?> that = (KeyAndValueSerializer<?>) o;\n        return Objects.equals(valueSerializer, that.valueSerializer);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(valueSerializer);\n    }\n\n    @Override\n    public TypeSerializerSnapshot<Tuple2<byte[], StreamRecord<IN>>> snapshotConfiguration() {\n        throw new UnsupportedOperationException(\n                \"The KeyAndValueSerializer should not be used for persisting into State!\");\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/FixedLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps. It uses\n * binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are always of a fixed length and thus the length of the record is not\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class FixedLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    static final int TIMESTAMP_BYTE_SIZE = 8;\n    private final int keyLength;\n    private byte[] keyReference;\n    private long timestampReference;\n\n    public FixedLengthByteKeyComparator(int keyLength) {\n        this.keyLength = keyLength;\n    }\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = toCompare.f0;\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((FixedLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((FixedLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        for (int i = 0; i < keyLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return 0;\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int minCount = keyLength;\n        while (minCount-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return Long.compare(firstSource.readLong(), secondSource.readLong());\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return keyLength + TIMESTAMP_BYTE_SIZE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return keyBytes < getNormalizeKeyLen();\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(record, keyLength, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new FixedLengthByteKeyComparator<>(this.keyLength);\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/BytesKeyNormalizationUtil.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\n/**\n * Utility class for common key normalization used both in {@link VariableLengthByteKeyComparator}\n * and {@link FixedLengthByteKeyComparator}.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\nfinal class BytesKeyNormalizationUtil {\n    /**\n     * Writes the normalized key of given record. The normalized key consists of the key serialized\n     * as bytes and the timestamp of the record.\n     *\n     * <p>NOTE: The key does not represent a logical order. It can be used only for grouping keys!\n     */\n    static <IN> void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            int dataLength,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        byte[] data = record.f0;\n\n        if (dataLength >= numBytes) {\n            putBytesArray(target, offset, numBytes, data);\n        } else {\n            // whole key fits into the normalized key\n            putBytesArray(target, offset, dataLength, data);\n            int lastOffset = offset + numBytes;\n            offset += dataLength;\n            long valueOfTimestamp = record.f1.asRecord().getTimestamp() - Long.MIN_VALUE;\n            if (dataLength + FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE <= numBytes) {\n                // whole timestamp fits into the normalized key\n                target.putLong(offset, valueOfTimestamp);\n                offset += FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE;\n                // fill in the remaining space with zeros\n                while (offset < lastOffset) {\n                    target.put(offset++, (byte) 0);\n                }\n            } else {\n                // only part of the timestamp fits into normalized key\n                for (int i = 0; offset < lastOffset; offset++, i++) {\n                    target.put(offset, (byte) (valueOfTimestamp >>> ((7 - i) << 3)));\n                }\n            }\n        }\n    }\n\n    private static void putBytesArray(MemorySegment target, int offset, int numBytes, byte[] data) {\n        for (int i = 0; i < numBytes; i++) {\n            // We're converting the signed byte in data into an unsigned representation.\n            // A Java byte goes from -128 to 127, i.e. is signed. By subtracting -128 (MIN_VALUE)\n            // here we're shifting the number to be from 0 to 255. The normalized key sorter sorts\n            // bytes as \"unsigned\", so we need to convert here to maintain a correct ordering.\n            int highByte = data[i] & 0xff;\n            highByte -= Byte.MIN_VALUE;\n            target.put(offset + i, (byte) highByte);\n        }\n    }\n\n    private BytesKeyNormalizationUtil() {}\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/VariableLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps,. It\n * uses binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are of a variable length and thus expects the length of the record to be\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class VariableLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    private byte[] keyReference;\n    private long timestampReference;\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = Arrays.copyOf(toCompare.f0, toCompare.f0.length);\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((VariableLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((VariableLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        int firstLength = first.length;\n        int secondLength = second.length;\n        int minLength = Math.min(firstLength, secondLength);\n        for (int i = 0; i < minLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        return Integer.compare(firstLength, secondLength);\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int firstLength = firstSource.readInt();\n        int secondLength = secondSource.readInt();\n        int minLength = Math.min(firstLength, secondLength);\n        while (minLength-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        int lengthCompare = Integer.compare(firstLength, secondLength);\n        if (lengthCompare != 0) {\n            return lengthCompare;\n        } else {\n            return Long.compare(firstSource.readLong(), secondSource.readLong());\n        }\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return Integer.MAX_VALUE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return true;\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(\n                record, record.f0.length, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new VariableLengthByteKeyComparator<>();\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastContext.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.annotation.VisibleForTesting;\nimport org.apache.flink.api.common.operators.MailboxExecutor;\nimport org.apache.flink.api.java.tuple.Tuple2;\n\nimport javax.annotation.Nullable;\n\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\n\n/**\n * Context to hold the broadcast variables and provides some utility function for accessing\n * broadcast variables.\n */\npublic class BroadcastContext {\n\n    /**\n     * stores broadcast data streams in a map. The key is broadcastName-partitionId and the value is\n     * {@link BroadcastItem}.\n     */\n    private static final ConcurrentHashMap<String, BroadcastItem> BROADCAST_VARIABLES =\n            new ConcurrentHashMap<>();\n\n    @VisibleForTesting\n    public static void putBroadcastVariable(String key, Tuple2<Boolean, List<?>> variable) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(variable.f0, variable.f1, null)\n                                : new BroadcastItem(variable.f0, variable.f1, v.mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    public static void putMailBoxExecutor(String key, MailboxExecutor mailboxExecutor) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(false, null, mailboxExecutor)\n                                : new BroadcastItem(v.cacheReady, v.cacheList, mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    @SuppressWarnings({\"unchecked\"})\n    public static <T> List<T> getBroadcastVariable(String key) {\n        return (List<T>) BROADCAST_VARIABLES.get(key).cacheList;\n    }\n\n    @VisibleForTesting\n    public static void remove(String key) {\n        BROADCAST_VARIABLES.remove(key);\n    }\n\n    @VisibleForTesting\n    public static void notifyCacheFinished(String key) {\n        BROADCAST_VARIABLES.computeIfPresent(\n                key,\n                (k, v) -> {\n                    // sends a dummy email to avoid possible stuck.\n                    if (null != v.mailboxExecutor) {\n                        v.mailboxExecutor.execute(() -> {}, \"empty mail\");\n                    }\n                    return new BroadcastItem(true, v.cacheList, v.mailboxExecutor);\n                });\n    }\n\n    @VisibleForTesting\n    public static boolean isCacheFinished(String key) {\n        return BROADCAST_VARIABLES.get(key).cacheReady;\n    }\n\n...\n", "needle_token_start": 3947, "needle_token_end": 4175, "code_context_ntokens": 16064, "output": ["```java\n/**\n * A CoGroup transformation combines the elements of two {@link DataStream DataStreams} into one\n * DataStream. It groups each DataStream individually on a key and gives groups of both\n * DataStreams with equal keys together into a {@link\n * org.apache.flink.api.common.functions.CoGroupFunction}. If a DataStream has a group with no\n * matching key in the other DataStream, the CoGroupFunction is called with an empty group for\n * the non-existing group.\n *\n * <p>The CoGroupFunction can iterate over the elements of both groups and return any number of\n * elements including none.\n *\n * <p>NOTE: This method assumes both inputs are bounded.\n *\n * @param input1 The first data stream.\n * @param input2 The second data stream.\n * @param keySelector1 The KeySelector to be used for extracting the first input's key for\n *     partitioning.\n * @param keySelector2 The KeySelector to be used for extracting the second input's key for\n *     partitioning.\n * @param outTypeInformation The type information describing the output type.\n * @param func The user-defined co-group function.\n * @param <IN1> The class type of the first input.\n * @param <IN2> The class type of the second input.\n * @param <KEY> The class type of the key.\n * @param <OUT> The class type of the output values.\n * @return The result data stream.\n */\npublic static <IN1, IN2, KEY extends Serializable, OUT> DataStream<OUT> coGroup(\n        DataStream<IN1> input1,\n        DataStream<IN2> input2,\n        KeySelector<IN1, KEY> keySelector1,\n        KeySelector<IN2, KEY> keySelector2,\n        TypeInformation<OUT> outTypeInformation,\n        CoGroupFunction<IN1, IN2, OUT> func) {\n    func = input1.getExecutionEnvironment().clean(func);\n    DataStream<OUT> result =\n            input1.connect(input2)\n                    .keyBy(keySelector1, keySelector2)\n                    .transform(\n                            \"CoGroupOperator\", outTypeInformation, new CoGroupOperator<>(func))\n                    .setParallelism(Math.max(input1.getParallelism(), input2.getParallelism()));\n    setManagedMemoryWeight(result, 100);\n    return result;\n}\n```"]}
{"repo": "apache/flink-ml", "name": "toList", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/builder/TableId.java", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert an array of objects that each contain an identifier into a list of these identifiers.\n2. **Input**: An array of objects, each with a unique identifier.\n3. **Output**: A list of integers, where each integer is the identifier extracted from the corresponding object in the input array.\n4. **Procedure**: The function initializes an empty list, iterates through each object in the input array, extracts the identifier from each object, and appends it to the list. The final list of identifiers is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: flink-ml-core/src/main/java/org/apache/flink/ml/api/AlgoOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.api;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.table.api.Table;\n\n/**\n * An AlgoOperator takes a list of tables as inputs and produces a list of tables as results. It can\n * be used to encode generic multi-input multi-output computation logic.\n *\n * @param <T> The class type of the AlgoOperator implementation itself.\n */\n@PublicEvolving\npublic interface AlgoOperator<T extends AlgoOperator<T>> extends Stage<T> {\n    /**\n     * Applies the AlgoOperator on the given input tables and returns the result tables.\n     *\n     * @param inputs a list of tables\n     * @return a list of tables\n     */\n    Table[] transform(Table... inputs);\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/api/Model.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.api;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.table.api.Table;\n\n/**\n * A Model is typically generated by invoking {@link Estimator#fit(Table...)}. A Model is a\n * Transformer with the extra APIs to set and get model data.\n *\n * @param <T> The class type of the Model implementation itself.\n */\n@PublicEvolving\npublic interface Model<T extends Model<T>> extends Transformer<T> {\n    /**\n     * Sets model data using the given list of tables. Each table could be an unbounded stream of\n     * model data changes.\n     *\n     * @param inputs a list of tables\n     */\n    default T setModelData(Table... inputs) {\n        throw new UnsupportedOperationException(\"this operation is not supported\");\n    }\n\n    /**\n     * Gets a list of tables representing the model data. Each table could be an unbounded stream of\n     * model data changes.\n     *\n     * @return a list of tables\n     */\n    default Table[] getModelData() {\n        throw new UnsupportedOperationException(\"this operation is not supported\");\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/api/Stage.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.api;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.ml.param.WithParams;\n\nimport java.io.IOException;\nimport java.io.Serializable;\n\n/**\n * Base class for a node in a Pipeline or Graph. The interface is only a concept, and does not have\n * any actual functionality. Its subclasses could be Estimator, Model, Transformer or AlgoOperator.\n * No other classes should inherit this interface directly.\n *\n * <p>Each stage is with parameters, and requires a public empty constructor for restoration.\n *\n * <p>NOTE: every Stage subclass should have a no-arg constructor.\n *\n * <p>NOTE: every Stage subclass should implement a static method with signature {@code static T\n * load(StreamTableEnvironment tEnv, String path)}, where {@code T} refers to the concrete subclass.\n * This static method should instantiate a new stage instance based on the data read from the given\n * path.\n *\n * @param <T> The class type of the Stage implementation itself.\n */\n@PublicEvolving\npublic interface Stage<T extends Stage<T>> extends WithParams<T>, Serializable {\n    /** Saves the metadata and bounded data of this stage to the given path. */\n    void save(String path) throws IOException;\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/api/Transformer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.api;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.ml.servable.api.TransformerServable;\n\n/**\n * A Transformer is an AlgoOperator with the semantic difference that it encodes the Transformation\n * logic, such that a record in the output typically corresponds to one record in the input. In\n * contrast, an AlgoOperator is a better fit to express aggregation logic where a record in the\n * output could be computed from an arbitrary number of records in the input.\n *\n * <p>NOTE: If a Transformer has a corresponding {@link TransformerServable}, it should implement a\n * static method with the signature {@code static T loadServable(String path)}, where {@code T}\n * refers to the concrete subclass of {@link TransformerServable}. This static method should\n * instantiate a new {@link TransformerServable} instance based on the data read from the given\n * path.\n *\n * @param <T> The class type of the Transformer implementation itself.\n */\n@PublicEvolving\npublic interface Transformer<T extends Transformer<T>> extends AlgoOperator<T> {}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/api/Estimator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.api;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.table.api.Table;\n\n/**\n * Estimators are responsible for training and generating Models.\n *\n * @param <E> class type of the Estimator implementation itself.\n * @param <M> class type of the Model this Estimator produces.\n */\n@PublicEvolving\npublic interface Estimator<E extends Estimator<E, M>, M extends Model<M>> extends Stage<E> {\n    /**\n     * Trains on the given inputs and produces a Model.\n     *\n     * @param inputs a list of tables\n     * @return a Model\n     */\n    M fit(Table... inputs);\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/TableId.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Objects;\n\n/**\n * The TableId is necessary to pass the inputs/outputs of various API calls across the nodes of\n * Graph and GraphModel.\n */\npublic class TableId {\n    public final int id;\n\n    public TableId(int id) {\n        this.id = id;\n    }\n\n    \npublic static List<Integer> toList(TableId[] tableIds) {\n        List<Integer> result = new ArrayList<>(tableIds.length);\n        for (int i = 0; i < tableIds.length; i++) {\n            result.add(tableIds[i].id);\n        }\n        return result;\n    }\n\n    public static TableId[] fromList(List<Integer> tableIds) {\n        TableId[] result = new TableId[tableIds.size()];\n        for (int i = 0; i < tableIds.size(); i++) {\n            result[i] = new TableId(tableIds.get(i));\n        }\n        return result;\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (obj == null) {\n            return false;\n        }\n        if (!(obj instanceof TableId)) {\n            return false;\n        }\n        TableId other = (TableId) obj;\n        return id == other.id;\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(id);\n    }\n\n    @Override\n    public String toString() {\n        return \"TableId(\" + id + \")\";\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/GraphNode.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.ml.api.Stage;\nimport org.apache.flink.util.Preconditions;\n\nimport javax.annotation.Nullable;\n\nimport java.util.Arrays;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Objects;\n\n/** The Graph node class. */\npublic class GraphNode {\n    /** This class specifies whether a node should be used as Estimator or AlgoOperator. */\n    public enum StageType {\n        ESTIMATOR,\n        ALGO_OPERATOR;\n    }\n\n    public final int nodeId;\n    public @Nullable Stage<?> stage;\n    public final StageType stageType;\n    public final @Nullable TableId[] estimatorInputIds;\n    public final TableId[] algoOpInputIds;\n    public final TableId[] outputIds;\n    public @Nullable TableId[] inputModelDataIds;\n    public @Nullable TableId[] outputModelDataIds;\n\n    public GraphNode(\n            int nodeId,\n            Stage<?> stage,\n            StageType stageType,\n            TableId[] estimatorInputIds,\n            TableId[] algoOpInputIds,\n            TableId[] outputIds,\n            TableId[] inputModelDataIds,\n            TableId[] outputModelDataIds) {\n        this.nodeId = Preconditions.checkNotNull(nodeId);\n        this.stage = stage;\n        this.stageType = Preconditions.checkNotNull(stageType);\n        this.estimatorInputIds = estimatorInputIds;\n        this.algoOpInputIds = Preconditions.checkNotNull(algoOpInputIds);\n        this.outputIds = Preconditions.checkNotNull(outputIds);\n        this.inputModelDataIds = inputModelDataIds;\n        this.outputModelDataIds = outputModelDataIds;\n    }\n\n    public Map<String, Object> toMap() {\n        Map<String, Object> result = new HashMap<>();\n        result.put(\"nodeId\", nodeId);\n        result.put(\"stageType\", stageType.name());\n        if (estimatorInputIds != null) {\n            result.put(\"estimatorInputIds\", TableId.toList(estimatorInputIds));\n        }\n        result.put(\"algoOpInputIds\", TableId.toList(algoOpInputIds));\n        result.put(\"outputIds\", TableId.toList(outputIds));\n        if (inputModelDataIds != null) {\n            result.put(\"inputModelDataIds\", TableId.toList(inputModelDataIds));\n        }\n        if (outputModelDataIds != null) {\n            result.put(\"outputModelDataIds\", TableId.toList(outputModelDataIds));\n        }\n        return result;\n    }\n\n    public static GraphNode fromMap(Map<String, Object> map) {\n        int nodeId = (Integer) map.get(\"nodeId\");\n        StageType stageType = StageType.valueOf((String) map.get(\"stageType\"));\n        TableId[] estimatorInputIds = null;\n        if (map.containsKey(\"estimatorInputIds\")) {\n            estimatorInputIds = TableId.fromList((List<Integer>) map.get(\"estimatorInputIds\"));\n        }\n        TableId[] algoOpInputIds = TableId.fromList((List<Integer>) map.get(\"algoOpInputIds\"));\n        TableId[] outputIds = TableId.fromList((List<Integer>) map.get(\"outputIds\"));\n        TableId[] inputModelDataIds = null;\n        if (map.containsKey(\"inputModelDataIds\")) {\n            inputModelDataIds = TableId.fromList((List<Integer>) map.get(\"inputModelDataIds\"));\n        }\n        TableId[] outputModelDataIds = null;\n        if (map.containsKey(\"outputModelDataIds\")) {\n            outputModelDataIds = TableId.fromList((List<Integer>) map.get(\"outputModelDataIds\"));\n        }\n        return new GraphNode(\n                nodeId,\n                null,\n                stageType,\n                estimatorInputIds,\n                algoOpInputIds,\n                outputIds,\n                inputModelDataIds,\n                outputModelDataIds);\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (obj == null) {\n            return false;\n        }\n        if (!(obj instanceof GraphNode)) {\n            return false;\n        }\n        GraphNode other = (GraphNode) obj;\n        return nodeId == other.nodeId;\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hashCode(nodeId);\n    }\n\n    @Override\n    public String toString() {\n        return String.format(\n                \"GraphNode(nodeId=%d, stageType=%s, estimatorInputIds=%s, algoOpInputIds=%s, outputIds=%s, inputModelDataIds=%s, outputModelDataIds=%s)\",\n                nodeId,\n                stageType.name(),\n                Arrays.toString(estimatorInputIds),\n                Arrays.toString(algoOpInputIds),\n                Arrays.toString(outputIds),\n                Arrays.toString(inputModelDataIds),\n                Arrays.toString(outputModelDataIds));\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/Graph.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.ml.api.AlgoOperator;\nimport org.apache.flink.ml.api.Estimator;\nimport org.apache.flink.ml.api.Model;\nimport org.apache.flink.ml.api.Stage;\nimport org.apache.flink.ml.param.Param;\nimport org.apache.flink.ml.util.ParamUtils;\nimport org.apache.flink.ml.util.ReadWriteUtils;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.util.Preconditions;\n\nimport javax.annotation.Nullable;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.apache.flink.ml.builder.GraphNode.StageType;\n\n/**\n * A Graph acts as an Estimator. A Graph consists of a DAG of stages, each of which could be an\n * Estimator, Model, Transformer or AlgoOperator. When `Graph::fit` is called, the stages are\n * executed in a topologically-sorted order. If a stage is an Estimator, its `Estimator::fit` method\n * will be called on the input tables (from the input edges) to fit a Model. Then the Model will be\n * used to transform the input tables and produce output tables to the output edges. If a stage is\n * an AlgoOperator, its `AlgoOperator::transform` method will be called on the input tables and\n * produce output tables to the output edges. The GraphModel fitted from a Graph consists of the\n * fitted Models and AlgoOperators, corresponding to the Graph's stages.\n */\n@PublicEvolving\npublic final class Graph implements Estimator<Graph, GraphModel> {\n    private static final long serialVersionUID = 6354253958813529308L;\n    private final Map<Param<?>, Object> paramMap = new HashMap<>();\n    private final List<GraphNode> nodes;\n    private final TableId[] estimatorInputIds;\n    private final TableId[] modelInputIds;\n    private final TableId[] outputIds;\n    private final @Nullable TableId[] inputModelDataIds;\n    private final @Nullable TableId[] outputModelDataIds;\n\n    public Graph(\n            List<GraphNode> nodes,\n            TableId[] estimatorInputIds,\n            TableId[] modelInputs,\n            TableId[] outputs,\n            TableId[] inputModelDataIds,\n            TableId[] outputModelDataIds) {\n        this.nodes = Preconditions.checkNotNull(nodes);\n        this.estimatorInputIds = Preconditions.checkNotNull(estimatorInputIds);\n        this.modelInputIds = Preconditions.checkNotNull(modelInputs);\n        this.outputIds = Preconditions.checkNotNull(outputs);\n        this.inputModelDataIds = inputModelDataIds;\n        this.outputModelDataIds = outputModelDataIds;\n        ParamUtils.initializeMapWithDefaultValues(paramMap, this);\n    }\n\n    @Override\n    public GraphModel fit(Table... inputTables) {\n        Preconditions.checkArgument(\n                estimatorInputIds.length == inputTables.length,\n                \"number of provided tables %s does not match the expected number of tables %s\",\n                inputTables.length,\n                estimatorInputIds.length);\n        List<GraphNode> modelNodes = new ArrayList<>();\n        GraphExecutionHelper executionHelper = new GraphExecutionHelper(nodes);\n        // Maps estimatorInputIds to inputTables.\n        executionHelper.setTables(estimatorInputIds, inputTables);\n        // Iterates until we have executed all ready nodes.\n        GraphNode node;\n        while ((node = executionHelper.pollNextReadyNode()) != null) {\n            Stage<?> stage = node.stage;\n            // Invokes fit(...) if stageType == ESTIMATOR.\n            if (node.stageType == StageType.ESTIMATOR) {\n                stage =\n                        ((Estimator<?, ?>) stage)\n                                .fit(executionHelper.getTables(node.estimatorInputIds));\n            }\n            // Invokes setModelData(...).\n            if (node.inputModelDataIds != null) {\n                Table[] nodeInputModelData = executionHelper.getTables(node.inputModelDataIds);\n                ((Model<?>) stage).setModelData(nodeInputModelData);\n            }\n            // Invokes transform(...).\n            Table[] nodeOutputs =\n                    ((AlgoOperator<?>) stage)\n                            .transform(executionHelper.getTables(node.algoOpInputIds));\n            executionHelper.setTables(node.outputIds, nodeOutputs);\n            // Invokes getModelData().\n            if (node.outputModelDataIds != null) {\n                Table[] nodeOutputModelData = ((Model<?>) stage).getModelData();\n                executionHelper.setTables(node.outputModelDataIds, nodeOutputModelData);\n            }\n\n            modelNodes.add(\n                    new GraphNode(\n                            node.nodeId,\n                            stage,\n                            StageType.ALGO_OPERATOR,\n                            null,\n                            node.algoOpInputIds,\n                            node.outputIds,\n                            node.inputModelDataIds,\n                            node.outputModelDataIds));\n        }\n        return new GraphModel(\n                modelNodes, modelInputIds, outputIds, inputModelDataIds, outputModelDataIds);\n    }\n\n    @Override\n    public Map<Param<?>, Object> getParamMap() {\n        return paramMap;\n    }\n\n    @Override\n    public void save(String path) throws IOException {\n        GraphData graphData =\n                new GraphData(\n                        nodes,\n                        estimatorInputIds,\n                        modelInputIds,\n                        outputIds,\n                        inputModelDataIds,\n                        outputModelDataIds);\n        ReadWriteUtils.saveGraph(this, graphData, path);\n    }\n\n    public static Graph load(StreamTableEnvironment tEnv, String path) throws IOException {\n        return (Graph) ReadWriteUtils.loadGraph(tEnv, path, Graph.class.getName());\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/GraphData.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.util.Preconditions;\n\nimport javax.annotation.Nullable;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/** This class contains fields that can be used to re-construct Graph and GraphModel. */\npublic class GraphData {\n    public final List<GraphNode> nodes;\n    public final @Nullable TableId[] estimatorInputIds;\n    public final TableId[] modelInputIds;\n    public final TableId[] outputIds;\n    public final @Nullable TableId[] inputModelDataIds;\n    public final @Nullable TableId[] outputModelDataIds;\n\n    public GraphData(\n            List<GraphNode> nodes,\n            TableId[] estimatorInputIds,\n            TableId[] modelInputIds,\n            TableId[] outputIds,\n            TableId[] inputModelDataIds,\n            TableId[] outputModelDataIds) {\n        this.nodes = Preconditions.checkNotNull(nodes);\n        this.estimatorInputIds = estimatorInputIds;\n        this.modelInputIds = Preconditions.checkNotNull(modelInputIds);\n        this.outputIds = Preconditions.checkNotNull(outputIds);\n        this.inputModelDataIds = inputModelDataIds;\n        this.outputModelDataIds = outputModelDataIds;\n    }\n\n    public Map<String, Object> toMap() {\n        Map<String, Object> result = new HashMap<>();\n\n        List<Map<String, Object>> nodeInfos = new ArrayList<>();\n        for (GraphNode node : nodes) {\n            nodeInfos.add(node.toMap());\n        }\n        result.put(\"nodes\", nodeInfos);\n        if (estimatorInputIds != null) {\n            result.put(\"estimatorInputIds\", TableId.toList(estimatorInputIds));\n        }\n        result.put(\"modelInputIds\", TableId.toList(modelInputIds));\n        result.put(\"outputIds\", TableId.toList(outputIds));\n        if (inputModelDataIds != null) {\n            result.put(\"inputModelDataIds\", TableId.toList(inputModelDataIds));\n        }\n        if (outputModelDataIds != null) {\n            result.put(\"outputModelDataIds\", TableId.toList(outputModelDataIds));\n        }\n        return result;\n    }\n\n    public static GraphData fromMap(Map<String, Object> map) {\n        List<GraphNode> nodes = new ArrayList<>();\n        List<Map<String, Object>> nodeInfos = (List<Map<String, Object>>) map.get(\"nodes\");\n        for (Map<String, Object> nodeInfo : nodeInfos) {\n            nodes.add(GraphNode.fromMap(nodeInfo));\n        }\n\n        TableId[] estimatorInputIds = null;\n        if (map.containsKey(\"estimatorInputIds\")) {\n            estimatorInputIds = TableId.fromList((List<Integer>) map.get(\"estimatorInputIds\"));\n        }\n        TableId[] modelInputIds = TableId.fromList((List<Integer>) map.get(\"modelInputIds\"));\n        TableId[] outputIds = TableId.fromList((List<Integer>) map.get(\"outputIds\"));\n        TableId[] inputModelDataIds = null;\n        if (map.containsKey(\"inputModelDataIds\")) {\n            inputModelDataIds = TableId.fromList((List<Integer>) map.get(\"inputModelDataIds\"));\n        }\n        TableId[] outputModelDataIds = null;\n        if (map.containsKey(\"outputModelDataIds\")) {\n            outputModelDataIds = TableId.fromList((List<Integer>) map.get(\"outputModelDataIds\"));\n        }\n        return new GraphData(\n                nodes,\n                estimatorInputIds,\n                modelInputIds,\n                outputIds,\n                inputModelDataIds,\n                outputModelDataIds);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/GraphModel.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.ml.api.AlgoOperator;\nimport org.apache.flink.ml.api.Estimator;\nimport org.apache.flink.ml.api.Model;\nimport org.apache.flink.ml.api.Stage;\nimport org.apache.flink.ml.param.Param;\nimport org.apache.flink.ml.util.ParamUtils;\nimport org.apache.flink.ml.util.ReadWriteUtils;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.util.Preconditions;\n\nimport javax.annotation.Nullable;\n\nimport java.io.IOException;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.apache.flink.ml.builder.GraphNode.StageType;\n\n/**\n * A GraphModel acts as a Model. A GraphModel consists of a DAG of stages, each of which could be an\n * Estimator, Model, Transformer or AlgoOperator. When `GraphModel::transform` is called, the stages\n * are executed in a topologically-sorted order. When a stage is executed, its\n * `AlgoOperator::transform` method will be called on the input tables (from the input edges) and\n * produce output tables to the output edges.\n */\n@PublicEvolving\npublic final class GraphModel implements Model<GraphModel> {\n    private static final long serialVersionUID = 6354856913812529398L;\n    private final Map<Param<?>, Object> paramMap = new HashMap<>();\n    private final List<GraphNode> nodes;\n    private final TableId[] inputIds;\n    private final TableId[] outputIds;\n    private final @Nullable TableId[] inputModelDataIds;\n    private final @Nullable TableId[] outputModelDataIds;\n    private final GraphExecutionHelper executionHelper;\n\n    public GraphModel(\n            List<GraphNode> nodes,\n            TableId[] inputIds,\n            TableId[] outputIds,\n            TableId[] inputModelDataIds,\n            TableId[] outputModelDataIds) {\n        this.nodes = Preconditions.checkNotNull(nodes);\n        this.inputIds = Preconditions.checkNotNull(inputIds);\n        this.outputIds = Preconditions.checkNotNull(outputIds);\n        this.inputModelDataIds = inputModelDataIds;\n        this.outputModelDataIds = outputModelDataIds;\n        ParamUtils.initializeMapWithDefaultValues(paramMap, this);\n        executionHelper = new GraphExecutionHelper(nodes);\n    }\n\n    @Override\n    public Table[] transform(Table... inputTables) {\n        Preconditions.checkArgument(\n                inputIds.length == inputTables.length,\n                \"number of provided tables %s does not match the expected number of tables %s\",\n                inputTables.length,\n                inputIds.length);\n        // Maps inputIds to inputTables.\n        executionHelper.setTables(inputIds, inputTables);\n        // Iterates until we have executed all ready nodes.\n        GraphNode node;\n        while ((node = executionHelper.pollNextReadyNode()) != null) {\n            Stage<?> stage = node.stage;\n            // Invokes fit(...) if stageType == ESTIMATOR.\n            if (node.stageType == StageType.ESTIMATOR) {\n                stage =\n                        ((Estimator<?, ?>) stage)\n                                .fit(executionHelper.getTables(node.estimatorInputIds));\n            }\n            // Invokes setModelData(...).\n            if (node.inputModelDataIds != null) {\n                Table[] nodeInputModelData = executionHelper.getTables(node.inputModelDataIds);\n                ((Model<?>) stage).setModelData(nodeInputModelData);\n            }\n            // Invokes transform(...).\n            Table[] nodeOutputs =\n                    ((AlgoOperator<?>) stage)\n                            .transform(executionHelper.getTables(node.algoOpInputIds));\n            executionHelper.setTables(node.outputIds, nodeOutputs);\n            // Invokes getModelData().\n            if (node.outputModelDataIds != null) {\n                Table[] nodeOutputModelData = ((Model<?>) stage).getModelData();\n                executionHelper.setTables(node.outputModelDataIds, nodeOutputModelData);\n            }\n        }\n        return executionHelper.getTables(outputIds);\n    }\n\n    @Override\n    public GraphModel setModelData(Table... inputTables) {\n        Preconditions.checkArgument(inputModelDataIds != null, \"setModelData() is not supported\");\n        Preconditions.checkArgument(\n                inputModelDataIds.length == inputTables.length,\n                \"number of provided tables %s does not match the expected number of tables %s\",\n                inputTables.length,\n                inputIds.length);\n        // Maps inputModelDataIds to inputTables.\n        executionHelper.setTables(inputModelDataIds, inputTables);\n        return this;\n    }\n\n    @Override\n    public Table[] getModelData() {\n        Preconditions.checkArgument(outputModelDataIds != null);\n        return executionHelper.getTables(outputModelDataIds);\n    }\n\n    @Override\n    public Map<Param<?>, Object> getParamMap() {\n        return paramMap;\n    }\n\n    @Override\n    public void save(String path) throws IOException {\n        GraphData graphData =\n                new GraphData(\n                        nodes, null, inputIds, outputIds, inputModelDataIds, outputModelDataIds);\n        ReadWriteUtils.saveGraph(this, graphData, path);\n    }\n\n    public static GraphModel load(StreamTableEnvironment tEnv, String path) throws IOException {\n        return (GraphModel) ReadWriteUtils.loadGraph(tEnv, path, GraphModel.class.getName());\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/TableUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.typeutils.RowTypeInfo;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.table.catalog.Column;\nimport org.apache.flink.table.catalog.ResolvedSchema;\nimport org.apache.flink.table.runtime.typeutils.ExternalTypeInfo;\nimport org.apache.flink.table.types.DataType;\nimport org.apache.flink.table.types.logical.LogicalTypeRoot;\nimport org.apache.flink.types.Row;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\n\n/** Utility class for operations related to Table API. */\npublic class TableUtils {\n\n    // Logical type roots that may cause wrong type conversion between Table and DataStream.\n    private static final Set<LogicalTypeRoot> LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO =\n            new HashSet<>();\n\n    static {\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.CHAR);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.VARCHAR);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.BINARY);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.VARBINARY);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.DECIMAL);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.DATE);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.TIME_WITHOUT_TIME_ZONE);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(\n                LogicalTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.INTERVAL_DAY_TIME);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(\n                LogicalTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.ARRAY);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.MAP);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.MULTISET);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.ROW);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.STRUCTURED_TYPE);\n    }\n\n    // Constructs a RowTypeInfo from the given schema. Currently, this function does not support\n    // the case when the input contains DataTypes.TIMESTAMP_WITH_TIME_ZONE().\n    public static RowTypeInfo getRowTypeInfo(ResolvedSchema schema) {\n        TypeInformation<?>[] types = new TypeInformation<?>[schema.getColumnCount()];\n        String[] names = new String[schema.getColumnCount()];\n\n        for (int i = 0; i < schema.getColumnCount(); i++) {\n            Column column = schema.getColumn(i).get();\n            types[i] = getTypeInformationFromDataType(column.getDataType());\n            names[i] = column.getName();\n        }\n        return new RowTypeInfo(types, names);\n    }\n\n    // Retrieves the TypeInformation of a column by name. Returns null if the name does not exist in\n    // the input schema.\n    public static TypeInformation<?> getTypeInfoByName(ResolvedSchema schema, String name) {\n        for (Column column : schema.getColumns()) {\n            if (column.getName().equals(name)) {\n                return getTypeInformationFromDataType(column.getDataType());\n            }\n        }\n        return null;\n    }\n\n    public static int[] getColumnIndexes(ResolvedSchema schema, String[] columnNames) {\n        Map<String, Integer> nameToIndex = new HashMap<>();\n        int[] result = new int[columnNames.length];\n\n        for (int i = 0; i < schema.getColumnCount(); i++) {\n            Column column = schema.getColumn(i).get();\n            nameToIndex.put(column.getName(), i);\n        }\n\n        for (int i = 0; i < columnNames.length; i++) {\n            result[i] = nameToIndex.get(columnNames[i]);\n        }\n        return result;\n    }\n\n    public static StreamExecutionEnvironment getExecutionEnvironment(StreamTableEnvironment tEnv) {\n        Table table = tEnv.fromValues();\n        DataStream<Row> dataStream = tEnv.toDataStream(table);\n        return dataStream.getExecutionEnvironment();\n    }\n\n    private static TypeInformation<?> getTypeInformationFromDataType(DataType dataType) {\n        if (LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.contains(\n                dataType.getLogicalType().getTypeRoot())) {\n            return ExternalTypeInfo.of(dataType);\n        }\n        return TypeInformation.of(dataType.getConversionClass());\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/util/ReadWriteUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.util;\n\nimport org.apache.flink.api.common.eventtime.WatermarkStrategy;\nimport org.apache.flink.api.common.serialization.Encoder;\nimport org.apache.flink.api.connector.source.Source;\nimport org.apache.flink.connector.file.sink.FileSink;\nimport org.apache.flink.connector.file.src.FileSource;\nimport org.apache.flink.connector.file.src.reader.SimpleStreamFormat;\nimport org.apache.flink.core.fs.Path;\nimport org.apache.flink.ml.api.Stage;\nimport org.apache.flink.ml.builder.Graph;\nimport org.apache.flink.ml.builder.GraphData;\nimport org.apache.flink.ml.builder.GraphModel;\nimport org.apache.flink.ml.builder.GraphNode;\nimport org.apache.flink.ml.common.datastream.TableUtils;\nimport org.apache.flink.ml.param.Param;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.BasePathBucketAssigner;\nimport org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.util.Preconditions;\n\nimport org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParser;\nimport org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;\n\nimport java.io.IOException;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/** Utility methods for reading and writing stages. */\npublic class ReadWriteUtils {\n    public static final ObjectMapper OBJECT_MAPPER =\n            new ObjectMapper().enable(JsonParser.Feature.ALLOW_COMMENTS);\n\n    // A helper method that calls encodes the given parameter value to a json string. We can not\n    // call param.jsonEncode(value) directly because Param::jsonEncode(...) needs the actual type\n    // of the value.\n    private static <T> Object jsonEncodeHelper(Param<T> param, Object value) throws IOException {\n        return param.jsonEncode((T) value);\n    }\n\n    // Converts Map<Param<?>, Object> to Map<String, Object> which maps the parameter name to the\n    // json-supported parameter value.\n    private static Map<String, Object> jsonEncode(Map<Param<?>, Object> paramMap)\n            throws IOException {\n        Map<String, Object> result = new HashMap<>(paramMap.size());\n        for (Map.Entry<Param<?>, Object> entry : paramMap.entrySet()) {\n            Object json = jsonEncodeHelper(entry.getKey(), entry.getValue());\n            result.put(entry.getKey().name, json);\n        }\n        return result;\n    }\n\n    /**\n     * Saves the metadata of the given stage and the extra metadata to a file named `metadata` under\n     * the given path. The metadata of a stage includes the stage class name, parameter values etc.\n     *\n     * <p>Required: the metadata file under the given path should not exist.\n     *\n     * @param stage The stage instance.\n     * @param path The parent directory to save the stage metadata.\n     * @param extraMetadata The extra metadata to be saved.\n     */\n    public static void saveMetadata(Stage<?> stage, String path, Map<String, ?> extraMetadata)\n            throws IOException {\n        Map<String, Object> metadata = new HashMap<>(extraMetadata);\n        metadata.put(\"className\", stage.getClass().getName());\n        metadata.put(\"timestamp\", System.currentTimeMillis());\n        metadata.put(\"paramMap\", jsonEncode(stage.getParamMap()));\n        // TODO: add version in the metadata.\n        String metadataStr = OBJECT_MAPPER.writeValueAsString(metadata);\n\n        FileUtils.saveToFile(new Path(path, \"metadata\").toUri().toString(), metadataStr, false);\n    }\n\n    /**\n     * Saves the metadata of the given stage to a file named `metadata` under the given path. The\n     * metadata of a stage includes the stage class name, parameter values etc.\n     *\n     * <p>Required: the metadata file under the given path should not exist.\n     *\n     * @param stage The stage instance.\n     * @param path The parent directory to save the stage metadata.\n     */\n    public static void saveMetadata(Stage<?> stage, String path) throws IOException {\n        saveMetadata(stage, path, new HashMap<>());\n    }\n\n    /**\n     * Saves a Pipeline or PipelineModel with the given list of stages to the given path.\n     *\n     * @param pipeline A Pipeline or PipelineModel instance.\n     * @param stages A list of stages of the given pipeline.\n     * @param path The parent directory to save the pipeline metadata and its stages.\n     */\n    public static void savePipeline(Stage<?> pipeline, List<Stage<?>> stages, String path)\n            throws IOException {\n        // Creates parent directories if not already created.\n        FileUtils.mkdirs(new Path(path));\n\n        Map<String, Object> extraMetadata = new HashMap<>();\n        extraMetadata.put(\"numStages\", stages.size());\n        saveMetadata(pipeline, path, extraMetadata);\n\n        int numStages = stages.size();\n        for (int i = 0; i < numStages; i++) {\n            String stagePath = FileUtils.getPathForPipelineStage(i, numStages, path);\n            stages.get(i).save(stagePath);\n        }\n    }\n\n    /**\n     * Loads the stages of a Pipeline or PipelineModel from the given path.\n     *\n     * <p>The method throws RuntimeException if the expectedClassName is not empty AND it does not\n     * match the className of the previously saved Pipeline or PipelineModel.\n     *\n     * @param tEnv A StreamTableEnvironment instance.\n     * @param path The parent directory to load the pipeline metadata and its stages.\n     * @param expectedClassName The expected class name of the pipeline.\n     * @return A list of stages.\n     */\n    public static List<Stage<?>> loadPipeline(\n            StreamTableEnvironment tEnv, String path, String expectedClassName) throws IOException {\n        Map<String, ?> metadata = FileUtils.loadMetadata(path, expectedClassName);\n        int numStages = (Integer) metadata.get(\"numStages\");\n        List<Stage<?>> stages = new ArrayList<>(numStages);\n\n        for (int i = 0; i < numStages; i++) {\n            String stagePath = FileUtils.getPathForPipelineStage(i, numStages, path);\n            stages.add(loadStage(tEnv, stagePath));\n        }\n        return stages;\n    }\n\n    /**\n     * Saves a Graph or GraphModel with the given GraphData to the given path.\n     *\n     * @param graph A Graph or GraphModel instance.\n     * @param graphData A GraphData instance.\n     * @param path The parent directory to save the graph metadata and its stages.\n     */\n    public static void saveGraph(Stage<?> graph, GraphData graphData, String path)\n            throws IOException {\n        // Creates parent directories if not already created.\n        FileUtils.mkdirs(new Path(path));\n\n        Map<String, Object> extraMetadata = new HashMap<>();\n        extraMetadata.put(\"graphData\", graphData.toMap());\n        saveMetadata(graph, path, extraMetadata);\n        int maxNodeId =\n...\n", "needle_token_start": 2860, "needle_token_end": 2937, "code_context_ntokens": 13344, "output": ["```java\npublic static List<Integer> toList(TableId[] tableIds) {\n    List<Integer> result = new ArrayList<>(tableIds.length);\n    for (int i = 0; i < tableIds.length; i++) {\n        result.add(tableIds[i].id);\n    }\n    return result;\n}\n```"]}
{"repo": "apache/flink-ml", "name": "putBroadcastVariable", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastContext.java", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To store or update broadcast data in a concurrent map, ensuring that the data is accessible across different components or tasks within a distributed computing environment.\n2. **Input**: A string identifier and a tuple containing a boolean flag and a list of data items.\n3. **Output**: There is no direct output as this method modifies the internal state of a concurrent map.\n4. **Procedure**: The method checks if an entry already exists for the given key in the map. If it exists, it updates the entry by creating a new broadcast item with the new data and retaining the existing mailbox executor. If it does not exist, it creates a new broadcast item with the provided data and a null mailbox executor.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/KeyAndValueSerializer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\nimport java.util.Objects;\n\n/**\n * A serializer used in {@link SortingDataInput} for serializing elements alongside their key and\n * timestamp. It serializes the record in a format known by the {@link FixedLengthByteKeyComparator}\n * and {@link VariableLengthByteKeyComparator}.\n *\n * <p>If the key is of known constant length, the length is not serialized with the data. Therefore\n * the serialized data is as follows:\n *\n * <pre>\n *      [key-length] | &lt;key&gt; | &lt;timestamp&gt; | &lt;record&gt;\n * </pre>\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class KeyAndValueSerializer<IN>\n        extends TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> {\n    private static final int TIMESTAMP_LENGTH = 8;\n    private final TypeSerializer<IN> valueSerializer;\n\n    // This represents either a variable length (-1) or a fixed one (>= 0).\n    private final int serializedKeyLength;\n\n    public KeyAndValueSerializer(TypeSerializer<IN> valueSerializer, int serializedKeyLength) {\n        this.valueSerializer = valueSerializer;\n        this.serializedKeyLength = serializedKeyLength;\n    }\n\n    @Override\n    public boolean isImmutableType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new KeyAndValueSerializer<>(valueSerializer.duplicate(), this.serializedKeyLength);\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> copy(Tuple2<byte[], StreamRecord<IN>> from) {\n        StreamRecord<IN> fromRecord = from.f1;\n        return Tuple2.of(\n                Arrays.copyOf(from.f0, from.f0.length),\n                fromRecord.copy(valueSerializer.copy(fromRecord.getValue())));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> createInstance() {\n        return Tuple2.of(new byte[0], new StreamRecord<>(valueSerializer.createInstance()));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> copy(\n            Tuple2<byte[], StreamRecord<IN>> from, Tuple2<byte[], StreamRecord<IN>> reuse) {\n        StreamRecord<IN> fromRecord = from.f1;\n        StreamRecord<IN> reuseRecord = reuse.f1;\n\n        IN valueCopy = valueSerializer.copy(fromRecord.getValue(), reuseRecord.getValue());\n        fromRecord.copyTo(valueCopy, reuseRecord);\n        reuse.f0 = Arrays.copyOf(from.f0, from.f0.length);\n        reuse.f1 = reuseRecord;\n        return reuse;\n    }\n\n    @Override\n    public int getLength() {\n        if (valueSerializer.getLength() < 0 || serializedKeyLength < 0) {\n            return -1;\n        }\n        return valueSerializer.getLength() + serializedKeyLength + TIMESTAMP_LENGTH;\n    }\n\n    @Override\n    public void serialize(Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target)\n            throws IOException {\n        if (serializedKeyLength < 0) {\n            target.writeInt(record.f0.length);\n        }\n        target.write(record.f0);\n        StreamRecord<IN> toSerialize = record.f1;\n        target.writeLong(toSerialize.getTimestamp());\n        valueSerializer.serialize(toSerialize.getValue(), target);\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> deserialize(DataInputView source) throws IOException {\n        final int length = getKeyLength(source);\n        byte[] bytes = new byte[length];\n        source.read(bytes);\n        long timestamp = source.readLong();\n        IN value = valueSerializer.deserialize(source);\n        return Tuple2.of(bytes, new StreamRecord<>(value, timestamp));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> deserialize(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        final int length = getKeyLength(source);\n        byte[] bytes = new byte[length];\n        source.read(bytes);\n        long timestamp = source.readLong();\n        IN value = valueSerializer.deserialize(source);\n        StreamRecord<IN> reuseRecord = reuse.f1;\n        reuseRecord.replace(value, timestamp);\n        reuse.f0 = bytes;\n        reuse.f1 = reuseRecord;\n        return reuse;\n    }\n\n    private int getKeyLength(DataInputView source) throws IOException {\n        final int length;\n        if (serializedKeyLength < 0) {\n            length = source.readInt();\n        } else {\n            length = serializedKeyLength;\n        }\n        return length;\n    }\n\n    @Override\n    public void copy(DataInputView source, DataOutputView target) throws IOException {\n        final int length;\n        if (serializedKeyLength < 0) {\n            length = source.readInt();\n            target.writeInt(length);\n        } else {\n            length = serializedKeyLength;\n        }\n        for (int i = 0; i < length; i++) {\n            target.writeByte(source.readByte());\n        }\n        target.writeLong(source.readLong());\n        valueSerializer.copy(source, target);\n    }\n\n    @Override\n    public boolean equals(Object o) {\n        if (this == o) {\n            return true;\n        }\n        if (o == null || getClass() != o.getClass()) {\n            return false;\n        }\n        KeyAndValueSerializer<?> that = (KeyAndValueSerializer<?>) o;\n        return Objects.equals(valueSerializer, that.valueSerializer);\n    }\n\n    @Override\n    public int hashCode() {\n        return Objects.hash(valueSerializer);\n    }\n\n    @Override\n    public TypeSerializerSnapshot<Tuple2<byte[], StreamRecord<IN>>> snapshotConfiguration() {\n        throw new UnsupportedOperationException(\n...\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/FixedLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps. It uses\n * binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are always of a fixed length and thus the length of the record is not\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class FixedLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    static final int TIMESTAMP_BYTE_SIZE = 8;\n    private final int keyLength;\n    private byte[] keyReference;\n    private long timestampReference;\n\n    public FixedLengthByteKeyComparator(int keyLength) {\n        this.keyLength = keyLength;\n    }\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = toCompare.f0;\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((FixedLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((FixedLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        for (int i = 0; i < keyLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return 0;\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int minCount = keyLength;\n        while (minCount-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return Long.compare(firstSource.readLong(), secondSource.readLong());\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return keyLength + TIMESTAMP_BYTE_SIZE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return keyBytes < getNormalizeKeyLen();\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(record, keyLength, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new FixedLengthByteKeyComparator<>(this.keyLength);\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/BytesKeyNormalizationUtil.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\n/**\n * Utility class for common key normalization used both in {@link VariableLengthByteKeyComparator}\n * and {@link FixedLengthByteKeyComparator}.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\nfinal class BytesKeyNormalizationUtil {\n    /**\n     * Writes the normalized key of given record. The normalized key consists of the key serialized\n     * as bytes and the timestamp of the record.\n     *\n     * <p>NOTE: The key does not represent a logical order. It can be used only for grouping keys!\n     */\n    static <IN> void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            int dataLength,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        byte[] data = record.f0;\n\n        if (dataLength >= numBytes) {\n            putBytesArray(target, offset, numBytes, data);\n        } else {\n            // whole key fits into the normalized key\n            putBytesArray(target, offset, dataLength, data);\n            int lastOffset = offset + numBytes;\n            offset += dataLength;\n            long valueOfTimestamp = record.f1.asRecord().getTimestamp() - Long.MIN_VALUE;\n            if (dataLength + FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE <= numBytes) {\n                // whole timestamp fits into the normalized key\n                target.putLong(offset, valueOfTimestamp);\n                offset += FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE;\n                // fill in the remaining space with zeros\n                while (offset < lastOffset) {\n                    target.put(offset++, (byte) 0);\n                }\n            } else {\n                // only part of the timestamp fits into normalized key\n                for (int i = 0; offset < lastOffset; offset++, i++) {\n                    target.put(offset, (byte) (valueOfTimestamp >>> ((7 - i) << 3)));\n                }\n            }\n        }\n    }\n\n    private static void putBytesArray(MemorySegment target, int offset, int numBytes, byte[] data) {\n        for (int i = 0; i < numBytes; i++) {\n            // We're converting the signed byte in data into an unsigned representation.\n            // A Java byte goes from -128 to 127, i.e. is signed. By subtracting -128 (MIN_VALUE)\n            // here we're shifting the number to be from 0 to 255. The normalized key sorter sorts\n            // bytes as \"unsigned\", so we need to convert here to maintain a correct ordering.\n            int highByte = data[i] & 0xff;\n            highByte -= Byte.MIN_VALUE;\n            target.put(offset + i, (byte) highByte);\n        }\n    }\n\n    private BytesKeyNormalizationUtil() {}\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/VariableLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps,. It\n * uses binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are of a variable length and thus expects the length of the record to be\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class VariableLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    private byte[] keyReference;\n    private long timestampReference;\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = Arrays.copyOf(toCompare.f0, toCompare.f0.length);\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((VariableLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((VariableLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        int firstLength = first.length;\n        int secondLength = second.length;\n        int minLength = Math.min(firstLength, secondLength);\n        for (int i = 0; i < minLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        return Integer.compare(firstLength, secondLength);\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int firstLength = firstSource.readInt();\n        int secondLength = secondSource.readInt();\n        int minLength = Math.min(firstLength, secondLength);\n        while (minLength-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        int lengthCompare = Integer.compare(firstLength, secondLength);\n        if (lengthCompare != 0) {\n            return lengthCompare;\n        } else {\n            return Long.compare(firstSource.readLong(), secondSource.readLong());\n        }\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return Integer.MAX_VALUE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return true;\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(\n                record, record.f0.length, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new VariableLengthByteKeyComparator<>();\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastContext.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.annotation.VisibleForTesting;\nimport org.apache.flink.api.common.operators.MailboxExecutor;\nimport org.apache.flink.api.java.tuple.Tuple2;\n\nimport javax.annotation.Nullable;\n\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\n\n/**\n * Context to hold the broadcast variables and provides some utility function for accessing\n * broadcast variables.\n */\npublic class BroadcastContext {\n\n    /**\n     * stores broadcast data streams in a map. The key is broadcastName-partitionId and the value is\n     * {@link BroadcastItem}.\n     */\n    private static final ConcurrentHashMap<String, BroadcastItem> BROADCAST_VARIABLES =\n            new ConcurrentHashMap<>();\n\n    \n@VisibleForTesting\n    public static void putBroadcastVariable(String key, Tuple2<Boolean, List<?>> variable) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(variable.f0, variable.f1, null)\n                                : new BroadcastItem(variable.f0, variable.f1, v.mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    public static void putMailBoxExecutor(String key, MailboxExecutor mailboxExecutor) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(false, null, mailboxExecutor)\n                                : new BroadcastItem(v.cacheReady, v.cacheList, mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    @SuppressWarnings({\"unchecked\"})\n    public static <T> List<T> getBroadcastVariable(String key) {\n        return (List<T>) BROADCAST_VARIABLES.get(key).cacheList;\n    }\n\n    @VisibleForTesting\n    public static void remove(String key) {\n        BROADCAST_VARIABLES.remove(key);\n    }\n\n    @VisibleForTesting\n    public static void notifyCacheFinished(String key) {\n        BROADCAST_VARIABLES.computeIfPresent(\n                key,\n                (k, v) -> {\n                    // sends a dummy email to avoid possible stuck.\n                    if (null != v.mailboxExecutor) {\n                        v.mailboxExecutor.execute(() -> {}, \"empty mail\");\n                    }\n                    return new BroadcastItem(true, v.cacheList, v.mailboxExecutor);\n                });\n    }\n\n    @VisibleForTesting\n    public static boolean isCacheFinished(String key) {\n        return BROADCAST_VARIABLES.get(key).cacheReady;\n    }\n\n    /** Utility class to organize broadcast variables. */\n    private static class BroadcastItem {\n\n        /** whether this broadcast variable is ready to be consumed. */\n        private boolean cacheReady;\n\n        /** the cached list. */\n        private List<?> cacheList;\n\n        /** the mailboxExecutor of the consumer, used to avoid the possible stuck of consumer. */\n        private MailboxExecutor mailboxExecutor;\n\n        BroadcastItem(\n                boolean cacheReady,\n                @Nullable List<?> cacheList,\n                @Nullable MailboxExecutor mailboxExecutor) {\n            this.cacheReady = cacheReady;\n            this.cacheList = cacheList;\n            this.mailboxExecutor = mailboxExecutor;\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastVariableReceiverOperatorFactory.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\n\nimport java.io.Serializable;\n\n/** Factory class for {@link BroadcastVariableReceiverOperator}. */\npublic class BroadcastVariableReceiverOperatorFactory<OUT>\n        extends AbstractStreamOperatorFactory<OUT> implements Serializable {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastNames;\n\n    /** Types of the broadcast data streams. */\n    private final TypeInformation<?>[] inTypes;\n\n    public BroadcastVariableReceiverOperatorFactory(\n            String[] broadcastNames, TypeInformation<?>[] inTypes) {\n        this.broadcastNames = broadcastNames;\n        this.inTypes = inTypes;\n    }\n\n    @Override\n    public <T extends StreamOperator<OUT>> T createStreamOperator(\n            StreamOperatorParameters<OUT> parameters) {\n        return (T) new BroadcastVariableReceiverOperator(parameters, broadcastNames, inTypes);\n    }\n\n    @Override\n    public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {\n        return BroadcastVariableReceiverOperator.class;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastWrapper.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.functions.KeySelector;\nimport org.apache.flink.iteration.operator.OperatorWrapper;\nimport org.apache.flink.streaming.api.operators.OneInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.TwoInputStreamOperator;\nimport org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;\nimport org.apache.flink.util.OutputTag;\n\n/** The operator wrapper for {@link AbstractBroadcastWrapperOperator}. */\npublic class BroadcastWrapper<T> implements OperatorWrapper<T, T> {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastStreamNames;\n\n    public BroadcastWrapper(String[] broadcastStreamNames) {\n        this.broadcastStreamNames = broadcastStreamNames;\n    }\n\n    @Override\n    public StreamOperator<T> wrap(\n            StreamOperatorParameters<T> operatorParameters,\n            StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new OneInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new TwoInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"Unsupported operator class for with-broadcast wrapper: \" + operatorClass);\n        }\n    }\n\n    @Override\n    public Class<? extends StreamOperator> getStreamOperatorClass(\n            ClassLoader classLoader, StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return OneInputBroadcastWrapperOperator.class;\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return TwoInputBroadcastWrapperOperator.class;\n        } else {\n            throw new UnsupportedOperationException(\n                    \"Unsupported operator class for with-broadcast wrapper: \" + operatorClass);\n        }\n    }\n\n    @Override\n    public <KEY> KeySelector<T, KEY> wrapKeySelector(KeySelector<T, KEY> keySelector) {\n        return keySelector;\n    }\n\n    @Override\n    public StreamPartitioner<T> wrapStreamPartitioner(StreamPartitioner<T> streamPartitioner) {\n        return streamPartitioner;\n    }\n\n    @Override\n    public OutputTag<T> wrapOutputTag(OutputTag<T> outputTag) {\n        return outputTag;\n    }\n\n    @Override\n    public TypeInformation<T> getWrappedTypeInfo(TypeInformation<T> typeInfo) {\n        return typeInfo;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.iteration.compile.DraftExecutionEnvironment;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastVariableReceiverOperatorFactory;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastWrapper;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.datastream.MultipleConnectedStreams;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.streaming.api.operators.ChainingStrategy;\nimport org.apache.flink.streaming.api.transformations.MultipleInputTransformation;\nimport org.apache.flink.streaming.api.transformations.PhysicalTransformation;\nimport org.apache.flink.util.AbstractID;\nimport org.apache.flink.util.Preconditions;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.function.Function;\n\n/** Utility class to support withBroadcast in DataStream. */\npublic class BroadcastUtils {\n    /**\n     * Supports withBroadcastStream in DataStream API. Broadcast data streams are available at all\n     * parallel instances of an operator that extends {@code\n     * org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator<OUT, ? extends\n     * org.apache.flink.api.common.functions.RichFunction>}. Users can access the broadcast\n     * variables by {@code RichFunction.getRuntimeContext().getBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().hasBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().getBroadcastVariableWithInitializer(...)}.\n     *\n     * <p>In detail, the broadcast input data streams will be consumed first and further consumed by\n     * non-broadcast inputs. For now the non-broadcast input are cached by default to avoid the\n     * possible deadlocks.\n     *\n     * @param inputList non-broadcast input list.\n     * @param bcStreams map of the broadcast data streams, where the key is the name and the value\n     *     is the corresponding data stream.\n     * @param userDefinedFunction the user defined logic in which users can access the broadcast\n     *     data streams and produce the output data stream. Note that though users can add more than\n     *     one operator in this logic, but only the operator that generates the result stream can\n     *     contain a rich function and access the broadcast variables. Other operators will\n     *     encounter NPE when accessing the broadcast variables.\n     * @return the output data stream.\n     */\n    public static <OUT> DataStream<OUT> withBroadcastStream(\n            List<DataStream<?>> inputList,\n            Map<String, DataStream<?>> bcStreams,\n            Function<List<DataStream<?>>, DataStream<OUT>> userDefinedFunction) {\n        Preconditions.checkArgument(inputList.size() > 0);\n\n        StreamExecutionEnvironment env = inputList.get(0).getExecutionEnvironment();\n        String[] broadcastNames = new String[bcStreams.size()];\n        DataStream<?>[] broadcastInputs = new DataStream[bcStreams.size()];\n        TypeInformation<?>[] broadcastInTypes = new TypeInformation[bcStreams.size()];\n        int idx = 0;\n        final String broadcastId = new AbstractID().toHexString();\n        for (String name : bcStreams.keySet()) {\n            broadcastNames[idx] = broadcastId + \"-\" + name;\n            broadcastInputs[idx] = bcStreams.get(name);\n            broadcastInTypes[idx] = broadcastInputs[idx].getType();\n            idx++;\n        }\n\n        DataStream<OUT> resultStream =\n                getResultStream(env, inputList, broadcastNames, userDefinedFunction);\n        TypeInformation<OUT> outType = resultStream.getType();\n        final String coLocationKey = \"broadcast-co-location-\" + UUID.randomUUID();\n        DataStream<OUT> cachedBroadcastInputs =\n                cacheBroadcastVariables(\n                        env,\n                        broadcastNames,\n                        broadcastInputs,\n                        broadcastInTypes,\n                        resultStream.getParallelism(),\n                        outType);\n\n        boolean canCoLocate =\n                cachedBroadcastInputs.getTransformation() instanceof PhysicalTransformation\n                        && resultStream.getTransformation() instanceof PhysicalTransformation;\n        if (canCoLocate) {\n            ((PhysicalTransformation<?>) cachedBroadcastInputs.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n            ((PhysicalTransformation<?>) resultStream.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"cannot set chaining strategy on \"\n                            + cachedBroadcastInputs.getTransformation()\n                            + \" and \"\n                            + resultStream.getTransformation()\n                            + \".\");\n        }\n        cachedBroadcastInputs.getTransformation().setCoLocationGroupKey(coLocationKey);\n        resultStream.getTransformation().setCoLocationGroupKey(coLocationKey);\n\n        return cachedBroadcastInputs.union(resultStream);\n    }\n\n    /**\n     * Caches all broadcast input data streams in static variables and returns the result\n     * multi-input stream operator. The result multi-input stream operator emits nothing and the\n     * only functionality of this operator is to cache all the input records in ${@link\n     * BroadcastContext}.\n     *\n     * @param env execution environment.\n     * @param broadcastInputNames names of the broadcast input data streams.\n     * @param broadcastInputs list of the broadcast data streams.\n     * @param broadcastInTypes output types of the broadcast input data streams.\n     * @param parallelism parallelism.\n     * @param outType output type.\n     * @param <OUT> output type.\n     * @return the result multi-input stream operator.\n     */\n    private static <OUT> DataStream<OUT> cacheBroadcastVariables(\n            StreamExecutionEnvironment env,\n            String[] broadcastInputNames,\n            DataStream<?>[] broadcastInputs,\n            TypeInformation<?>[] broadcastInTypes,\n            int parallelism,\n            TypeInformation<OUT> outType) {\n        MultipleInputTransformation<OUT> transformation =\n                new MultipleInputTransformation<>(\n                        \"broadcastInputs\",\n                        new BroadcastVariableReceiverOperatorFactory<>(\n                                broadcastInputNames, broadcastInTypes),\n                        outType,\n                        parallelism);\n        for (DataStream<?> dataStream : broadcastInputs) {\n            transformation.addInput(dataStream.broadcast().getTransformation());\n        }\n        env.addOperator(transformation);\n        return new MultipleConnectedStreams(env).transform(transformation);\n    }\n\n    /**\n     * Uses {@link DraftExecutionEnvironment} to execute the userDefinedFunction and returns the\n     * resultStream.\n     *\n     * @param env execution environment.\n     * @param inputList non-broadcast input list.\n     * @param broadcastStreamNames names of the broadcast data streams.\n     * @param graphBuilder user-defined logic.\n     * @param <OUT> output type of the result stream.\n     * @return the result stream by applying user-defined logic on the input list.\n     */\n    private static <OUT> DataStream<OUT> getResultStream(\n            StreamExecutionEnvironment env,\n            List<DataStream<?>> inputList,\n            String[] broadcastStreamNames,\n            Function<List<DataStream<?>>, DataStream<OUT>> graphBuilder) {\n        DraftExecutionEnvironment draftEnv =\n                new DraftExecutionEnvironment(env, new BroadcastWrapper<>(broadcastStreamNames));\n        List<DataStream<?>> draftSources = new ArrayList<>();\n        for (DataStream<?> dataStream : inputList) {\n            draftSources.add(draftEnv.addDraftSource(dataStream, dataStream.getType()));\n        }\n        DataStream<OUT> draftOutStream = graphBuilder.apply(draftSources);\n        draftEnv.copyToActualEnvironment();\n        return draftEnv.getActualStream(draftOutStream.getId());\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastStreamingRuntimeContext.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.annotation.Internal;\nimport org.apache.flink.api.common.accumulators.Accumulator;\nimport org.apache.flink.api.common.functions.BroadcastVariableInitializer;\nimport org.apache.flink.api.common.state.KeyedStateStore;\nimport org.apache.flink.metrics.groups.OperatorMetricGroup;\nimport org.apache.flink.runtime.execution.Environment;\nimport org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider;\nimport org.apache.flink.runtime.jobgraph.OperatorID;\nimport org.apache.flink.streaming.api.operators.StreamingRuntimeContext;\nimport org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;\n\nimport javax.annotation.Nullable;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * An subclass of {@link StreamingRuntimeContext} that provides accessibility of broadcast\n * variables.\n */\npublic class BroadcastStreamingRuntimeContext extends StreamingRuntimeContext {\n\n    Map<String, List<?>> broadcastVariables = new HashMap<>();\n\n    public BroadcastStreamingRuntimeContext(\n            Environment env,\n            Map<String, Accumulator<?, ?>> accumulators,\n            OperatorMetricGroup operatorMetricGroup,\n            OperatorID operatorID,\n            ProcessingTimeService processingTimeService,\n            @Nullable KeyedStateStore keyedStateStore,\n            ExternalResourceInfoProvider externalResourceInfoProvider) {\n        super(\n                env,\n                accumulators,\n                operatorMetricGroup,\n                operatorID,\n                processingTimeService,\n                keyedStateStore,\n                externalResourceInfoProvider);\n    }\n\n    @Override\n    public boolean hasBroadcastVariable(String name) {\n        return broadcastVariables.containsKey(name);\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked\")\n    public <RT> List<RT> getBroadcastVariable(String name) {\n        if (broadcastVariables.containsKey(name)) {\n            return (List<RT>) broadcastVariables.get(name);\n        } else {\n            throw new RuntimeException(\n                    \"Cannot get broadcast variables before processing elements.\");\n        }\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked\")\n    public <T, C> C getBroadcastVariableWithInitializer(\n            String name, BroadcastVariableInitializer<T, C> initializer) {\n        if (broadcastVariables.containsKey(name)) {\n            return initializer.initializeBroadcastVariable((List<T>) broadcastVariables.get(name));\n        } else {\n            throw new RuntimeException(\n                    \"Cannot get broadcast variables before processing elements.\");\n        }\n    }\n\n    @Internal\n    public void setBroadcastVariable(String name, List<?> broadcastVariable) {\n        broadcastVariables.put(name, broadcastVariable);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElement.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\n/**\n * The wrapper class for possible cached elements used in {@link\n * org.apache.flink.ml.common.broadcast.operator.AbstractBroadcastWrapperOperator}. It could be\n * either {@link org.apache.flink.streaming.api.watermark.Watermark}, {@link\n * org.apache.flink.streaming.runtime.streamrecord.StreamRecord}, etc.\n *\n * @param <T> the record type.\n */\npublic class CacheElement<T> {\n    private T record;\n    private long watermark;\n    private Type type;\n\n    public CacheElement(T record, long watermark, Type type) {\n        this.record = record;\n        this.watermark = watermark;\n        this.type = type;\n    }\n\n    public static <T> CacheElement<T> newRecord(T record) {\n        return new CacheElement<>(record, -1, Type.RECORD);\n    }\n\n    public static <T> CacheElement<T> newWatermark(long watermark) {\n        return new CacheElement<>(null, watermark, Type.WATERMARK);\n    }\n\n    public T getRecord() {\n        return record;\n    }\n\n    public void setRecord(T record) {\n        this.record = record;\n    }\n\n    public long getWatermark() {\n        return watermark;\n    }\n\n    public void setWatermark(long watermark) {\n        this.watermark = watermark;\n    }\n\n    public Type getType() {\n        return type;\n    }\n\n    public void setType(Type type) {\n        this.type = type;\n    }\n\n    /** The type of cached elements. */\n    public enum Type {\n        /** record type. */\n        RECORD,\n        /** watermark type. */\n        WATERMARK,\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElementTypeInfo.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\nimport org.apache.flink.api.common.ExecutionConfig;\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\n\nimport java.util.Objects;\n\n/**\n * TypeInformation for {@link CacheElement}.\n *\n * @param <T> the record type.\n */\npublic class CacheElementTypeInfo<T> extends TypeInformation<CacheElement<T>> {\n\n    private final TypeInformation<T> recordTypeInfo;\n\n    public CacheElementTypeInfo(TypeInformation<T> recordTypeInfo) {\n        this.recordTypeInfo = recordTypeInfo;\n    }\n\n    @Override\n    public boolean isBasicType() {\n        return false;\n    }\n\n    @Override\n    public boolean isTupleType() {\n        return false;\n    }\n\n    @Override\n    public int getArity() {\n        return 1;\n    }\n\n    @Override\n    public int getTotalFields() {\n        return 1;\n    }\n\n    @Override\n    public Class<CacheElement<T>> getTypeClass() {\n        return (Class) CacheElement.class;\n    }\n\n    @Override\n    public boolean isKeyType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<CacheElement<T>> createSerializer(ExecutionConfig config) {\n        return new CacheElementSerializer<>(recordTypeInfo.createSerializer(config));\n    }\n\n    @Override\n    public String toString() {\n        return \"RecordOrWatermark Type\";\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n\n        if (null == obj || getClass() != obj.getClass()) {\n            return false;\n        }\n\n        CacheElementTypeInfo<T> that = (CacheElementTypeInfo<T>) obj;\n        return Objects.equals(recordTypeInfo, that.recordTypeInfo);\n    }\n\n    @Override\n    public int hashCode() {\n        return recordTypeInfo != null ? recordTypeInfo.hashCode() : 0;\n    }\n\n    @Override\n    public boolean canEqual(Object obj) {\n        return obj instanceof CacheElementTypeInfo;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElementSerializer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\nimport org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;\nimport org.apache.flink.api.common.typeutils.base.LongSerializer;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.ml.common.broadcast.typeinfo.CacheElement.Type;\n\nimport java.io.IOException;\nimport java.util.Objects;\n\n/**\n * TypeSerializer for {@link CacheElement}.\n *\n * @param <T> the record type.\n */\npublic class CacheElementSerializer<T> extends TypeSerializer<CacheElement<T>> {\n\n    private final TypeSerializer<T> recordSerializer;\n\n    public CacheElementSerializer(TypeSerializer<T> recordSerializer) {\n        this.recordSerializer = recordSerializer;\n    }\n\n    @Override\n    public boolean isImmutableType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<CacheElement<T>> duplicate() {\n        return new CacheElementSerializer<>(recordSerializer.duplicate());\n    }\n\n    @Override\n    public CacheElement<T> createInstance() {\n        return null;\n    }\n\n    @Override\n    public CacheElement<T> copy(CacheElement<T> from) {\n        switch (from.getType()) {\n            case RECORD:\n                return CacheElement.newRecord(recordSerializer.copy(from.getRecord()));\n            case WATERMARK:\n                return CacheElement.newWatermark(from.getWatermark());\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + from.getType());\n        }\n    }\n\n    @Override\n    public CacheElement<T> copy(CacheElement<T> from, CacheElement<T> reuse) {\n        switch (from.getType()) {\n            case RECORD:\n                if (reuse.getRecord() != null) {\n                    recordSerializer.copy(from.getRecord(), reuse.getRecord());\n                } else {\n                    reuse.setRecord(recordSerializer.copy(from.getRecord()));\n                }\n                break;\n            case WATERMARK:\n                reuse.setWatermark(from.getWatermark());\n                break;\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + from.getType());\n        }\n\n        return reuse;\n    }\n\n    @Override\n    public int getLength() {\n        return -1;\n    }\n\n    @Override\n    public void serialize(CacheElement<T> record, DataOutputView target) throws IOException {\n        target.writeByte((byte) record.getType().ordinal());\n\n        switch (record.getType()) {\n            case RECORD:\n                recordSerializer.serialize(record.getRecord(), target);\n                break;\n            case WATERMARK:\n                LongSerializer.INSTANCE.serialize(record.getWatermark(), target);\n                break;\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + record.getType());\n        }\n    }\n\n    @Override\n    public CacheElement<T> deserialize(DataInputView source) throws IOException {\n        int type = source.readByte();\n        switch (CacheElement.Type.values()[type]) {\n            case RECORD:\n                T value = recordSerializer.deserialize(source);\n                return CacheElement.newRecord(value);\n            case WATERMARK:\n                long watermark = LongSerializer.INSTANCE.deserialize(source);\n                return CacheElement.newWatermark(watermark);\n            default:\n                throw new RuntimeException(\"Unsupported Record or Watermark type \" + type);\n        }\n    }\n\n    @Override\n    public CacheElement<T> deserialize(CacheElement<T> reuse, DataInputView source)\n            throws IOException {\n        int type = source.readByte();\n        switch (CacheElement.Type.values()[type]) {\n            case RECORD:\n                reuse.setType(Type.RECORD);\n                reuse.setRecord(recordSerializer.deserialize(source));\n                break;\n            case WATERMARK:\n                reuse.setType(Type.WATERMARK);\n                reuse.setWatermark(LongSerializer.INSTANCE.deserialize(source));\n                break;\n            default:\n                throw new RuntimeException(\"Unsupported Record or Watermark type \" + type);\n        }\n        return reuse;\n    }\n\n    @Override\n    public void copy(DataInputView source, DataOutputView target) throws IOException {\n        CacheElement<T> cacheElement = deserialize(source);\n        serialize(cacheElement, target);\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n\n        if (obj == null || getClass() != obj.getClass()) {\n            return false;\n        }\n        CacheElementSerializer<?> that = (CacheElementSerializer<?>) obj;\n...\n", "needle_token_start": 7095, "needle_token_end": 7206, "code_context_ntokens": 15912, "output": ["```java\n@VisibleForTesting\npublic static void putBroadcastVariable(String key, Tuple2<Boolean, List<?>> variable) {\n    BROADCAST_VARIABLES.compute(\n            key,\n            (k, v) ->\n                    null == v\n                            ? new BroadcastItem(variable.f0, variable.f1, null)\n                            : new BroadcastItem(variable.f0, variable.f1, v.mailboxExecutor));\n}\n```"]}
{"repo": "apache/flink-ml", "name": "getInputs", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastVariableReceiverOperator.java", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The method is designed to provide access to a list of input interfaces, each corresponding to a different data stream being processed by the operator. This facilitates handling multiple streams in parallel.\n2. **Input**: There are no parameters required for this method.\n3. **Output**: Returns a list of input interfaces, where each interface corresponds to a specific input stream that the operator processes.\n4. **Procedure**: The method simply returns a pre-initialized list that contains proxy input objects. Each proxy input is associated with a specific stream index, allowing the operator to manage data from multiple streams effectively.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastWrapper.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.functions.KeySelector;\nimport org.apache.flink.iteration.operator.OperatorWrapper;\nimport org.apache.flink.streaming.api.operators.OneInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.TwoInputStreamOperator;\nimport org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;\nimport org.apache.flink.util.OutputTag;\n\n/** The operator wrapper for {@link AbstractBroadcastWrapperOperator}. */\npublic class BroadcastWrapper<T> implements OperatorWrapper<T, T> {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastStreamNames;\n\n    public BroadcastWrapper(String[] broadcastStreamNames) {\n        this.broadcastStreamNames = broadcastStreamNames;\n    }\n\n    @Override\n    public StreamOperator<T> wrap(\n            StreamOperatorParameters<T> operatorParameters,\n            StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new OneInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new TwoInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"Unsupported operator class for with-broadcast wrapper: \" + operatorClass);\n        }\n    }\n\n    @Override\n    public Class<? extends StreamOperator> getStreamOperatorClass(\n            ClassLoader classLoader, StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return OneInputBroadcastWrapperOperator.class;\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return TwoInputBroadcastWrapperOperator.class;\n        } else {\n            throw new UnsupportedOperationException(\n...\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.iteration.compile.DraftExecutionEnvironment;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastVariableReceiverOperatorFactory;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastWrapper;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.datastream.MultipleConnectedStreams;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.streaming.api.operators.ChainingStrategy;\nimport org.apache.flink.streaming.api.transformations.MultipleInputTransformation;\nimport org.apache.flink.streaming.api.transformations.PhysicalTransformation;\nimport org.apache.flink.util.AbstractID;\nimport org.apache.flink.util.Preconditions;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.function.Function;\n\n/** Utility class to support withBroadcast in DataStream. */\npublic class BroadcastUtils {\n    /**\n     * Supports withBroadcastStream in DataStream API. Broadcast data streams are available at all\n     * parallel instances of an operator that extends {@code\n     * org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator<OUT, ? extends\n     * org.apache.flink.api.common.functions.RichFunction>}. Users can access the broadcast\n     * variables by {@code RichFunction.getRuntimeContext().getBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().hasBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().getBroadcastVariableWithInitializer(...)}.\n     *\n     * <p>In detail, the broadcast input data streams will be consumed first and further consumed by\n     * non-broadcast inputs. For now the non-broadcast input are cached by default to avoid the\n     * possible deadlocks.\n     *\n     * @param inputList non-broadcast input list.\n     * @param bcStreams map of the broadcast data streams, where the key is the name and the value\n     *     is the corresponding data stream.\n     * @param userDefinedFunction the user defined logic in which users can access the broadcast\n     *     data streams and produce the output data stream. Note that though users can add more than\n     *     one operator in this logic, but only the operator that generates the result stream can\n     *     contain a rich function and access the broadcast variables. Other operators will\n     *     encounter NPE when accessing the broadcast variables.\n     * @return the output data stream.\n     */\n    public static <OUT> DataStream<OUT> withBroadcastStream(\n            List<DataStream<?>> inputList,\n            Map<String, DataStream<?>> bcStreams,\n            Function<List<DataStream<?>>, DataStream<OUT>> userDefinedFunction) {\n        Preconditions.checkArgument(inputList.size() > 0);\n\n        StreamExecutionEnvironment env = inputList.get(0).getExecutionEnvironment();\n        String[] broadcastNames = new String[bcStreams.size()];\n        DataStream<?>[] broadcastInputs = new DataStream[bcStreams.size()];\n        TypeInformation<?>[] broadcastInTypes = new TypeInformation[bcStreams.size()];\n        int idx = 0;\n        final String broadcastId = new AbstractID().toHexString();\n        for (String name : bcStreams.keySet()) {\n            broadcastNames[idx] = broadcastId + \"-\" + name;\n            broadcastInputs[idx] = bcStreams.get(name);\n            broadcastInTypes[idx] = broadcastInputs[idx].getType();\n            idx++;\n        }\n\n        DataStream<OUT> resultStream =\n                getResultStream(env, inputList, broadcastNames, userDefinedFunction);\n        TypeInformation<OUT> outType = resultStream.getType();\n        final String coLocationKey = \"broadcast-co-location-\" + UUID.randomUUID();\n        DataStream<OUT> cachedBroadcastInputs =\n                cacheBroadcastVariables(\n                        env,\n                        broadcastNames,\n                        broadcastInputs,\n                        broadcastInTypes,\n                        resultStream.getParallelism(),\n                        outType);\n\n        boolean canCoLocate =\n                cachedBroadcastInputs.getTransformation() instanceof PhysicalTransformation\n                        && resultStream.getTransformation() instanceof PhysicalTransformation;\n        if (canCoLocate) {\n            ((PhysicalTransformation<?>) cachedBroadcastInputs.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n            ((PhysicalTransformation<?>) resultStream.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"cannot set chaining strategy on \"\n                            + cachedBroadcastInputs.getTransformation()\n                            + \" and \"\n                            + resultStream.getTransformation()\n                            + \".\");\n        }\n        cachedBroadcastInputs.getTransformation().setCoLocationGroupKey(coLocationKey);\n        resultStream.getTransformation().setCoLocationGroupKey(coLocationKey);\n\n        return cachedBroadcastInputs.union(resultStream);\n    }\n\n    /**\n     * Caches all broadcast input data streams in static variables and returns the result\n     * multi-input stream operator. The result multi-input stream operator emits nothing and the\n     * only functionality of this operator is to cache all the input records in ${@link\n     * BroadcastContext}.\n     *\n     * @param env execution environment.\n     * @param broadcastInputNames names of the broadcast input data streams.\n     * @param broadcastInputs list of the broadcast data streams.\n     * @param broadcastInTypes output types of the broadcast input data streams.\n     * @param parallelism parallelism.\n     * @param outType output type.\n     * @param <OUT> output type.\n     * @return the result multi-input stream operator.\n     */\n    private static <OUT> DataStream<OUT> cacheBroadcastVariables(\n            StreamExecutionEnvironment env,\n            String[] broadcastInputNames,\n            DataStream<?>[] broadcastInputs,\n            TypeInformation<?>[] broadcastInTypes,\n            int parallelism,\n            TypeInformation<OUT> outType) {\n        MultipleInputTransformation<OUT> transformation =\n                new MultipleInputTransformation<>(\n                        \"broadcastInputs\",\n                        new BroadcastVariableReceiverOperatorFactory<>(\n                                broadcastInputNames, broadcastInTypes),\n                        outType,\n                        parallelism);\n        for (DataStream<?> dataStream : broadcastInputs) {\n            transformation.addInput(dataStream.broadcast().getTransformation());\n        }\n        env.addOperator(transformation);\n        return new MultipleConnectedStreams(env).transform(transformation);\n    }\n\n    /**\n     * Uses {@link DraftExecutionEnvironment} to execute the userDefinedFunction and returns the\n     * resultStream.\n     *\n     * @param env execution environment.\n     * @param inputList non-broadcast input list.\n     * @param broadcastStreamNames names of the broadcast data streams.\n     * @param graphBuilder user-defined logic.\n     * @param <OUT> output type of the result stream.\n     * @return the result stream by applying user-defined logic on the input list.\n     */\n    private static <OUT> DataStream<OUT> getResultStream(\n            StreamExecutionEnvironment env,\n            List<DataStream<?>> inputList,\n            String[] broadcastStreamNames,\n            Function<List<DataStream<?>>, DataStream<OUT>> graphBuilder) {\n        DraftExecutionEnvironment draftEnv =\n                new DraftExecutionEnvironment(env, new BroadcastWrapper<>(broadcastStreamNames));\n        List<DataStream<?>> draftSources = new ArrayList<>();\n        for (DataStream<?> dataStream : inputList) {\n            draftSources.add(draftEnv.addDraftSource(dataStream, dataStream.getType()));\n        }\n        DataStream<OUT> draftOutStream = graphBuilder.apply(draftSources);\n        draftEnv.copyToActualEnvironment();\n        return draftEnv.getActualStream(draftOutStream.getId());\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastStreamingRuntimeContext.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.annotation.Internal;\nimport org.apache.flink.api.common.accumulators.Accumulator;\nimport org.apache.flink.api.common.functions.BroadcastVariableInitializer;\nimport org.apache.flink.api.common.state.KeyedStateStore;\nimport org.apache.flink.metrics.groups.OperatorMetricGroup;\nimport org.apache.flink.runtime.execution.Environment;\nimport org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider;\nimport org.apache.flink.runtime.jobgraph.OperatorID;\nimport org.apache.flink.streaming.api.operators.StreamingRuntimeContext;\nimport org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;\n\nimport javax.annotation.Nullable;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * An subclass of {@link StreamingRuntimeContext} that provides accessibility of broadcast\n * variables.\n */\npublic class BroadcastStreamingRuntimeContext extends StreamingRuntimeContext {\n\n    Map<String, List<?>> broadcastVariables = new HashMap<>();\n\n    public BroadcastStreamingRuntimeContext(\n            Environment env,\n            Map<String, Accumulator<?, ?>> accumulators,\n            OperatorMetricGroup operatorMetricGroup,\n            OperatorID operatorID,\n            ProcessingTimeService processingTimeService,\n            @Nullable KeyedStateStore keyedStateStore,\n            ExternalResourceInfoProvider externalResourceInfoProvider) {\n        super(\n                env,\n                accumulators,\n                operatorMetricGroup,\n                operatorID,\n                processingTimeService,\n                keyedStateStore,\n                externalResourceInfoProvider);\n    }\n\n    @Override\n    public boolean hasBroadcastVariable(String name) {\n        return broadcastVariables.containsKey(name);\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked\")\n    public <RT> List<RT> getBroadcastVariable(String name) {\n        if (broadcastVariables.containsKey(name)) {\n            return (List<RT>) broadcastVariables.get(name);\n        } else {\n            throw new RuntimeException(\n                    \"Cannot get broadcast variables before processing elements.\");\n        }\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked\")\n    public <T, C> C getBroadcastVariableWithInitializer(\n            String name, BroadcastVariableInitializer<T, C> initializer) {\n        if (broadcastVariables.containsKey(name)) {\n            return initializer.initializeBroadcastVariable((List<T>) broadcastVariables.get(name));\n        } else {\n            throw new RuntimeException(\n                    \"Cannot get broadcast variables before processing elements.\");\n        }\n    }\n\n    @Internal\n    public void setBroadcastVariable(String name, List<?> broadcastVariable) {\n        broadcastVariables.put(name, broadcastVariable);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElement.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\n/**\n * The wrapper class for possible cached elements used in {@link\n * org.apache.flink.ml.common.broadcast.operator.AbstractBroadcastWrapperOperator}. It could be\n * either {@link org.apache.flink.streaming.api.watermark.Watermark}, {@link\n * org.apache.flink.streaming.runtime.streamrecord.StreamRecord}, etc.\n *\n * @param <T> the record type.\n */\npublic class CacheElement<T> {\n    private T record;\n    private long watermark;\n    private Type type;\n\n    public CacheElement(T record, long watermark, Type type) {\n        this.record = record;\n        this.watermark = watermark;\n        this.type = type;\n    }\n\n    public static <T> CacheElement<T> newRecord(T record) {\n        return new CacheElement<>(record, -1, Type.RECORD);\n    }\n\n    public static <T> CacheElement<T> newWatermark(long watermark) {\n        return new CacheElement<>(null, watermark, Type.WATERMARK);\n    }\n\n    public T getRecord() {\n        return record;\n    }\n\n    public void setRecord(T record) {\n        this.record = record;\n    }\n\n    public long getWatermark() {\n        return watermark;\n    }\n\n    public void setWatermark(long watermark) {\n        this.watermark = watermark;\n    }\n\n    public Type getType() {\n        return type;\n    }\n\n    public void setType(Type type) {\n        this.type = type;\n    }\n\n    /** The type of cached elements. */\n    public enum Type {\n        /** record type. */\n        RECORD,\n        /** watermark type. */\n        WATERMARK,\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElementTypeInfo.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\nimport org.apache.flink.api.common.ExecutionConfig;\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\n\nimport java.util.Objects;\n\n/**\n * TypeInformation for {@link CacheElement}.\n *\n * @param <T> the record type.\n */\npublic class CacheElementTypeInfo<T> extends TypeInformation<CacheElement<T>> {\n\n    private final TypeInformation<T> recordTypeInfo;\n\n    public CacheElementTypeInfo(TypeInformation<T> recordTypeInfo) {\n        this.recordTypeInfo = recordTypeInfo;\n    }\n\n    @Override\n    public boolean isBasicType() {\n        return false;\n    }\n\n    @Override\n    public boolean isTupleType() {\n        return false;\n    }\n\n    @Override\n    public int getArity() {\n        return 1;\n    }\n\n    @Override\n    public int getTotalFields() {\n        return 1;\n    }\n\n    @Override\n    public Class<CacheElement<T>> getTypeClass() {\n        return (Class) CacheElement.class;\n    }\n\n    @Override\n    public boolean isKeyType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<CacheElement<T>> createSerializer(ExecutionConfig config) {\n        return new CacheElementSerializer<>(recordTypeInfo.createSerializer(config));\n    }\n\n    @Override\n    public String toString() {\n        return \"RecordOrWatermark Type\";\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n\n        if (null == obj || getClass() != obj.getClass()) {\n            return false;\n        }\n\n        CacheElementTypeInfo<T> that = (CacheElementTypeInfo<T>) obj;\n        return Objects.equals(recordTypeInfo, that.recordTypeInfo);\n    }\n\n    @Override\n    public int hashCode() {\n        return recordTypeInfo != null ? recordTypeInfo.hashCode() : 0;\n    }\n\n    @Override\n    public boolean canEqual(Object obj) {\n        return obj instanceof CacheElementTypeInfo;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElementSerializer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\nimport org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;\nimport org.apache.flink.api.common.typeutils.base.LongSerializer;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.ml.common.broadcast.typeinfo.CacheElement.Type;\n\nimport java.io.IOException;\nimport java.util.Objects;\n\n/**\n * TypeSerializer for {@link CacheElement}.\n *\n * @param <T> the record type.\n */\npublic class CacheElementSerializer<T> extends TypeSerializer<CacheElement<T>> {\n\n    private final TypeSerializer<T> recordSerializer;\n\n    public CacheElementSerializer(TypeSerializer<T> recordSerializer) {\n        this.recordSerializer = recordSerializer;\n    }\n\n    @Override\n    public boolean isImmutableType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<CacheElement<T>> duplicate() {\n        return new CacheElementSerializer<>(recordSerializer.duplicate());\n    }\n\n    @Override\n    public CacheElement<T> createInstance() {\n        return null;\n    }\n\n    @Override\n    public CacheElement<T> copy(CacheElement<T> from) {\n        switch (from.getType()) {\n            case RECORD:\n                return CacheElement.newRecord(recordSerializer.copy(from.getRecord()));\n            case WATERMARK:\n                return CacheElement.newWatermark(from.getWatermark());\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + from.getType());\n        }\n    }\n\n    @Override\n    public CacheElement<T> copy(CacheElement<T> from, CacheElement<T> reuse) {\n        switch (from.getType()) {\n            case RECORD:\n                if (reuse.getRecord() != null) {\n                    recordSerializer.copy(from.getRecord(), reuse.getRecord());\n                } else {\n                    reuse.setRecord(recordSerializer.copy(from.getRecord()));\n                }\n                break;\n            case WATERMARK:\n                reuse.setWatermark(from.getWatermark());\n                break;\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + from.getType());\n        }\n\n        return reuse;\n    }\n\n    @Override\n    public int getLength() {\n        return -1;\n    }\n\n    @Override\n    public void serialize(CacheElement<T> record, DataOutputView target) throws IOException {\n        target.writeByte((byte) record.getType().ordinal());\n\n        switch (record.getType()) {\n            case RECORD:\n                recordSerializer.serialize(record.getRecord(), target);\n                break;\n            case WATERMARK:\n                LongSerializer.INSTANCE.serialize(record.getWatermark(), target);\n                break;\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + record.getType());\n        }\n    }\n\n    @Override\n    public CacheElement<T> deserialize(DataInputView source) throws IOException {\n        int type = source.readByte();\n        switch (CacheElement.Type.values()[type]) {\n            case RECORD:\n                T value = recordSerializer.deserialize(source);\n                return CacheElement.newRecord(value);\n            case WATERMARK:\n                long watermark = LongSerializer.INSTANCE.deserialize(source);\n                return CacheElement.newWatermark(watermark);\n            default:\n                throw new RuntimeException(\"Unsupported Record or Watermark type \" + type);\n        }\n    }\n\n    @Override\n    public CacheElement<T> deserialize(CacheElement<T> reuse, DataInputView source)\n            throws IOException {\n        int type = source.readByte();\n        switch (CacheElement.Type.values()[type]) {\n            case RECORD:\n                reuse.setType(Type.RECORD);\n                reuse.setRecord(recordSerializer.deserialize(source));\n                break;\n            case WATERMARK:\n                reuse.setType(Type.WATERMARK);\n                reuse.setWatermark(LongSerializer.INSTANCE.deserialize(source));\n                break;\n            default:\n                throw new RuntimeException(\"Unsupported Record or Watermark type \" + type);\n        }\n        return reuse;\n    }\n\n    @Override\n    public void copy(DataInputView source, DataOutputView target) throws IOException {\n        CacheElement<T> cacheElement = deserialize(source);\n        serialize(cacheElement, target);\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n\n        if (obj == null || getClass() != obj.getClass()) {\n            return false;\n        }\n        CacheElementSerializer<?> that = (CacheElementSerializer<?>) obj;\n        return Objects.equals(recordSerializer, that.recordSerializer);\n    }\n\n    @Override\n    public int hashCode() {\n        return recordSerializer != null ? recordSerializer.hashCode() : 0;\n    }\n\n    @Override\n    public TypeSerializerSnapshot<CacheElement<T>> snapshotConfiguration() {\n        return new CacheElementSerializerSnapshot<>();\n    }\n\n    /** The serializer snapshot class for {@link CacheElementSerializer}. */\n    private static final class CacheElementSerializerSnapshot<T>\n            extends CompositeTypeSerializerSnapshot<CacheElement<T>, CacheElementSerializer<T>> {\n\n        private static final int CURRENT_VERSION = 1;\n\n        public CacheElementSerializerSnapshot() {\n            super(CacheElementSerializer.class);\n        }\n\n        @Override\n        protected int getCurrentOuterSnapshotVersion() {\n            return CURRENT_VERSION;\n        }\n\n        @Override\n        protected TypeSerializer<?>[] getNestedSerializers(\n                CacheElementSerializer<T> tIterationRecordSerializer) {\n            return new TypeSerializer[] {tIterationRecordSerializer.recordSerializer};\n        }\n\n        @Override\n        @SuppressWarnings(\"unchecked\")\n        protected CacheElementSerializer<T> createOuterSerializerWithNestedSerializers(\n                TypeSerializer<?>[] typeSerializers) {\n            TypeSerializer<T> elementSerializer = (TypeSerializer<T>) typeSerializers[0];\n            return new CacheElementSerializer<>(elementSerializer);\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastVariableReceiverOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.state.ListState;\nimport org.apache.flink.api.common.state.ListStateDescriptor;\nimport org.apache.flink.api.common.typeinfo.BasicTypeInfo;\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.iteration.operator.OperatorStateUtils;\nimport org.apache.flink.ml.common.broadcast.BroadcastContext;\nimport org.apache.flink.runtime.state.StateInitializationContext;\nimport org.apache.flink.runtime.state.StateSnapshotContext;\nimport org.apache.flink.streaming.api.operators.AbstractInput;\nimport org.apache.flink.streaming.api.operators.AbstractStreamOperatorV2;\nimport org.apache.flink.streaming.api.operators.BoundedMultiInput;\nimport org.apache.flink.streaming.api.operators.Input;\nimport org.apache.flink.streaming.api.operators.MultipleInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport org.apache.commons.collections.IteratorUtils;\n\nimport java.io.Serializable;\nimport java.util.ArrayList;\nimport java.util.List;\n\n/** The operator that process all broadcast inputs and stores them in {@link BroadcastContext}. */\npublic class BroadcastVariableReceiverOperator<OUT> extends AbstractStreamOperatorV2<OUT>\n        implements MultipleInputStreamOperator<OUT>, BoundedMultiInput, Serializable {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastStreamNames;\n\n    /** Output types of input data streams. */\n    private final TypeInformation<?>[] inTypes;\n\n    /** Input list of the multi-input operator. */\n    @SuppressWarnings(\"rawtypes\")\n    private final List<Input> inputList;\n\n    /** Whether each broadcast input has finished. */\n    private boolean[] cachesReady;\n\n    /** State storage of the broadcast inputs. */\n    @SuppressWarnings(\"rawtypes\")\n    private ListState[] cacheStates;\n\n    /** CacheReady state storage of the broadcast inputs. */\n    private ListState<Boolean>[] cacheReadyStates;\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    BroadcastVariableReceiverOperator(\n            StreamOperatorParameters<OUT> parameters,\n            String[] broadcastStreamNames,\n            TypeInformation<?>[] inTypes) {\n        super(parameters, broadcastStreamNames.length);\n        this.broadcastStreamNames = broadcastStreamNames;\n        this.inTypes = inTypes;\n        inputList = new ArrayList<>();\n        for (int i = 0; i < inTypes.length; i++) {\n            inputList.add(new ProxyInput(this, i + 1));\n        }\n        this.cachesReady = new boolean[inTypes.length];\n        this.cacheStates = new ListState[inTypes.length];\n        this.cacheReadyStates = new ListState[inTypes.length];\n    }\n\n    \n@Override\n    public List<Input> getInputs() {\n        return inputList;\n    }\n\n    @Override\n    @SuppressWarnings({\"unchecked\"})\n    public void endInput(int i) throws Exception {\n        cachesReady[i - 1] = true;\n        String key =\n                broadcastStreamNames[i - 1] + \"-\" + getRuntimeContext().getIndexOfThisSubtask();\n        BroadcastContext.putBroadcastVariable(\n                key,\n                Tuple2.of(\n                        true,\n                        IteratorUtils.toList(\n                                ((ListState<?>) cacheStates[i - 1]).get().iterator())));\n        BroadcastContext.notifyCacheFinished(key);\n    }\n\n    @Override\n    public void snapshotState(StateSnapshotContext context) throws Exception {\n        super.snapshotState(context);\n        for (int i = 0; i < inTypes.length; i++) {\n            cacheReadyStates[i].clear();\n            cacheReadyStates[i].add(cachesReady[i]);\n        }\n    }\n\n    @Override\n    @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n    public void initializeState(StateInitializationContext context) throws Exception {\n        super.initializeState(context);\n        for (int i = 0; i < inTypes.length; i++) {\n            cacheStates[i] =\n                    context.getOperatorStateStore()\n                            .getListState(new ListStateDescriptor(\"cache_data_\" + i, inTypes[i]));\n            cacheReadyStates[i] =\n                    context.getOperatorStateStore()\n                            .getListState(\n                                    new ListStateDescriptor<>(\n                                            \"cache_ready_state_\" + i,\n                                            BasicTypeInfo.BOOLEAN_TYPE_INFO));\n            boolean cacheReady =\n                    OperatorStateUtils.getUniqueElement(\n                                    cacheReadyStates[i], \"cache_ready_state_\" + i)\n                            .orElse(false);\n            // TODO: there may be a memory leak if the BroadcastWrapper finishes fast before this\n            // task finishes.\n            BroadcastContext.putBroadcastVariable(\n                    broadcastStreamNames[i] + \"-\" + getRuntimeContext().getIndexOfThisSubtask(),\n                    Tuple2.of(\n                            cacheReady,\n                            IteratorUtils.toList(\n                                    ((ListState<?>) cacheStates[i]).get().iterator())));\n        }\n    }\n\n    private class ProxyInput<IN, OT> extends AbstractInput<IN, OT> {\n\n        public ProxyInput(AbstractStreamOperatorV2<OT> owner, int inputId) {\n            super(owner, inputId);\n        }\n\n        @Override\n        @SuppressWarnings(\"unchecked\")\n        public void processElement(StreamRecord<IN> element) throws Exception {\n            (cacheStates[inputId - 1]).add(element.getValue());\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/OneInputBroadcastWrapperOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.iteration.operator.OperatorUtils;\nimport org.apache.flink.streaming.api.operators.BoundedOneInput;\nimport org.apache.flink.streaming.api.operators.OneInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.watermark.Watermark;\nimport org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;\n\n/** Wrapper for {@link OneInputStreamOperator}. */\npublic class OneInputBroadcastWrapperOperator<IN, OUT>\n        extends AbstractBroadcastWrapperOperator<OUT, OneInputStreamOperator<IN, OUT>>\n        implements OneInputStreamOperator<IN, OUT>, BoundedOneInput {\n\n    OneInputBroadcastWrapperOperator(\n            StreamOperatorParameters<OUT> parameters,\n            StreamOperatorFactory<OUT> operatorFactory,\n            String[] broadcastStreamNames) {\n        super(parameters, operatorFactory, broadcastStreamNames);\n    }\n\n    @Override\n    public void processElement(StreamRecord<IN> streamRecord) throws Exception {\n        processElementX(\n                streamRecord,\n                0,\n                wrappedOperator::processElement,\n                wrappedOperator::processWatermark,\n                wrappedOperator::setKeyContextElement);\n    }\n\n    @Override\n    public void endInput() throws Exception {\n        endInputX(\n                0,\n                wrappedOperator::processElement,\n                wrappedOperator::processWatermark,\n                wrappedOperator::setKeyContextElement);\n        OperatorUtils.processOperatorOrUdfIfSatisfy(\n                wrappedOperator, BoundedOneInput.class, BoundedOneInput::endInput);\n    }\n\n    @Override\n    public void processWatermark(Watermark watermark) throws Exception {\n        processWatermarkX(\n                watermark,\n                0,\n                wrappedOperator::processElement,\n                wrappedOperator::processWatermark,\n                wrappedOperator::setKeyContextElement);\n    }\n\n    @Override\n    public void processWatermarkStatus(WatermarkStatus watermarkStatus) throws Exception {\n        wrappedOperator.processWatermarkStatus(watermarkStatus);\n    }\n\n    @Override\n    public void processLatencyMarker(LatencyMarker latencyMarker) throws Exception {\n        wrappedOperator.processLatencyMarker(latencyMarker);\n    }\n\n    @Override\n    public void setKeyContextElement(StreamRecord<IN> streamRecord) throws Exception {\n        wrappedOperator.setKeyContextElement(streamRecord);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/TwoInputBroadcastWrapperOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.iteration.operator.OperatorUtils;\nimport org.apache.flink.streaming.api.operators.BoundedMultiInput;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.TwoInputStreamOperator;\nimport org.apache.flink.streaming.api.watermark.Watermark;\nimport org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;\n\n/** Wrapper for {@link TwoInputStreamOperator}. */\npublic class TwoInputBroadcastWrapperOperator<IN1, IN2, OUT>\n        extends AbstractBroadcastWrapperOperator<OUT, TwoInputStreamOperator<IN1, IN2, OUT>>\n        implements TwoInputStreamOperator<IN1, IN2, OUT>, BoundedMultiInput {\n\n    TwoInputBroadcastWrapperOperator(\n            StreamOperatorParameters<OUT> parameters,\n            StreamOperatorFactory<OUT> operatorFactory,\n            String[] broadcastStreamNames) {\n        super(parameters, operatorFactory, broadcastStreamNames);\n    }\n\n    @Override\n    public void processElement1(StreamRecord<IN1> streamRecord) throws Exception {\n        processElementX(\n                streamRecord,\n                0,\n                wrappedOperator::processElement1,\n                wrappedOperator::processWatermark1,\n                wrappedOperator::setKeyContextElement1);\n    }\n\n    @Override\n    public void processElement2(StreamRecord<IN2> streamRecord) throws Exception {\n        processElementX(\n                streamRecord,\n                1,\n                wrappedOperator::processElement2,\n                wrappedOperator::processWatermark2,\n                wrappedOperator::setKeyContextElement2);\n    }\n\n    @Override\n    public void endInput(int inputId) throws Exception {\n        if (inputId == 1) {\n            endInputX(\n                    inputId - 1,\n                    wrappedOperator::processElement1,\n                    wrappedOperator::processWatermark1,\n                    wrappedOperator::setKeyContextElement1);\n        } else {\n            endInputX(\n                    inputId - 1,\n                    wrappedOperator::processElement2,\n                    wrappedOperator::processWatermark2,\n                    wrappedOperator::setKeyContextElement2);\n        }\n        OperatorUtils.processOperatorOrUdfIfSatisfy(\n                wrappedOperator,\n                BoundedMultiInput.class,\n                boundedMultipleInput -> boundedMultipleInput.endInput(inputId));\n    }\n\n    @Override\n    public void processWatermark1(Watermark watermark) throws Exception {\n        processWatermarkX(\n                watermark,\n                0,\n                wrappedOperator::processElement1,\n                wrappedOperator::processWatermark1,\n                wrappedOperator::setKeyContextElement1);\n    }\n\n    @Override\n    public void processWatermark2(Watermark watermark) throws Exception {\n        processWatermarkX(\n                watermark,\n                1,\n                wrappedOperator::processElement2,\n                wrappedOperator::processWatermark2,\n                wrappedOperator::setKeyContextElement2);\n    }\n\n    @Override\n    public void processLatencyMarker1(LatencyMarker latencyMarker) throws Exception {\n        wrappedOperator.processLatencyMarker1(latencyMarker);\n    }\n\n    @Override\n    public void processLatencyMarker2(LatencyMarker latencyMarker) throws Exception {\n        wrappedOperator.processLatencyMarker2(latencyMarker);\n    }\n\n    @Override\n    public void processWatermarkStatus1(WatermarkStatus watermarkStatus) throws Exception {\n        wrappedOperator.processWatermarkStatus1(watermarkStatus);\n    }\n\n    @Override\n    public void processWatermarkStatus2(WatermarkStatus watermarkStatus) throws Exception {\n        wrappedOperator.processWatermarkStatus2(watermarkStatus);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/AbstractBroadcastWrapperOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.functions.RichFunction;\nimport org.apache.flink.api.common.operators.MailboxExecutor;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.core.fs.Path;\nimport org.apache.flink.core.memory.ManagedMemoryUseCase;\nimport org.apache.flink.iteration.datacache.nonkeyed.DataCacheReader;\nimport org.apache.flink.iteration.datacache.nonkeyed.DataCacheSnapshot;\nimport org.apache.flink.iteration.datacache.nonkeyed.DataCacheWriter;\nimport org.apache.flink.iteration.datacache.nonkeyed.Segment;\nimport org.apache.flink.iteration.operator.OperatorUtils;\nimport org.apache.flink.iteration.proxy.state.ProxyStreamOperatorStateContext;\nimport org.apache.flink.metrics.groups.OperatorMetricGroup;\nimport org.apache.flink.ml.common.broadcast.BroadcastContext;\nimport org.apache.flink.ml.common.broadcast.BroadcastStreamingRuntimeContext;\nimport org.apache.flink.ml.common.broadcast.typeinfo.CacheElement;\nimport org.apache.flink.ml.common.broadcast.typeinfo.CacheElementSerializer;\nimport org.apache.flink.runtime.checkpoint.CheckpointOptions;\nimport org.apache.flink.runtime.execution.Environment;\nimport org.apache.flink.runtime.jobgraph.OperatorID;\nimport org.apache.flink.runtime.metrics.groups.InternalOperatorIOMetricGroup;\nimport org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;\nimport org.apache.flink.runtime.state.CheckpointStreamFactory;\nimport org.apache.flink.runtime.state.OperatorStateCheckpointOutputStream;\nimport org.apache.flink.runtime.state.StateInitializationContext;\nimport org.apache.flink.runtime.state.StatePartitionStreamProvider;\nimport org.apache.flink.runtime.state.StateSnapshotContext;\nimport org.apache.flink.runtime.util.NonClosingInputStreamDecorator;\nimport org.apache.flink.runtime.util.NonClosingOutputStreamDecorator;\nimport org.apache.flink.streaming.api.graph.StreamConfig;\nimport org.apache.flink.streaming.api.graph.StreamConfig.InputConfig;\nimport org.apache.flink.streaming.api.graph.StreamConfig.NetworkInputConfig;\nimport org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;\nimport org.apache.flink.streaming.api.operators.InternalTimeServiceManager;\nimport org.apache.flink.streaming.api.operators.OperatorSnapshotFutures;\nimport org.apache.flink.streaming.api.operators.Output;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactoryUtil;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.StreamOperatorStateContext;\nimport org.apache.flink.streaming.api.operators.StreamOperatorStateHandler;\nimport org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.CheckpointedStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamTaskStateInitializer;\nimport org.apache.flink.streaming.api.watermark.Watermark;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.tasks.StreamTask;\nimport org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailbox;\nimport org.apache.flink.util.CloseableIterator;\nimport org.apache.flink.util.Preconditions;\nimport org.apache.flink.util.function.ThrowingConsumer;\n\nimport org.apache.commons.collections.IteratorUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.DataInputStream;\nimport java.io.DataOutputStream;\nimport java.io.InputStream;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.Optional;\n\n/**\n * Base class for the broadcast wrapper operators.\n *\n * <p>Note that not all instances of {@link AbstractBroadcastWrapperOperator} need to access the\n * broadcast variables. If one instance of {@link AbstractBroadcastWrapperOperator} does not contain\n * a rich function, then it can directly process the input without waiting for the broadcast\n * variables.\n */\npublic abstract class AbstractBroadcastWrapperOperator<T, S extends StreamOperator<T>>\n        implements StreamOperator<T>, StreamOperatorStateHandler.CheckpointedStreamOperator {\n\n    private static final Logger LOG =\n            LoggerFactory.getLogger(AbstractBroadcastWrapperOperator.class);\n\n    protected final StreamOperatorParameters<T> parameters;\n\n    protected final StreamConfig streamConfig;\n\n    protected final StreamTask<?, ?> containingTask;\n\n    protected final Output<StreamRecord<T>> output;\n\n    protected final StreamOperatorFactory<T> operatorFactory;\n\n    protected final OperatorMetricGroup metrics;\n\n    protected final S wrappedOperator;\n\n    protected transient StreamOperatorStateHandler stateHandler;\n\n    protected transient InternalTimeServiceManager<?> timeServiceManager;\n\n    // ---------------- context info for rich function ----------------\n    private MailboxExecutor mailboxExecutor;\n\n    private String[] broadcastStreamNames;\n\n    /**\n     * Whether each input is blocked. Inputs with broadcast variables can only process their input\n     * records after broadcast variables are ready. One input is non-blocked if it can consume its\n     * inputs (by caching) when broadcast variables are not ready. Otherwise it has to block the\n     * processing and wait until the broadcast variables are ready to be accessed.\n     */\n    private boolean[] isBlocked;\n\n    /** Type serializer of each input. */\n    private TypeSerializer<?>[] inTypeSerializers;\n\n    /** Whether all broadcast variables of this operator are ready. */\n    private boolean broadcastVariablesReady;\n    /** Index of this subtask. */\n    protected transient int indexOfSubtask;\n\n    /** Number of the inputs of this operator. */\n    protected int numInputs;\n\n    /** RuntimeContext of the rich function in wrapped operator. */\n    private BroadcastStreamingRuntimeContext wrappedOperatorRuntimeContext;\n\n    /**\n     * Path of the file used to store the cached records. It could be local file system or remote\n     * file system.\n     */\n    private Path basePath;\n\n    /** DataCacheWriter for each input. */\n    @SuppressWarnings(\"rawtypes\")\n    private DataCacheWriter[] dataCacheWriters;\n\n    /** Whether each input has pending elements. */\n    private boolean[] hasPendingElements;\n\n    /**\n     * Whether this operator has a rich function and needs to access the broadcast variable. If yes,\n     * it cannot process elements until the broadcast variables are ready.\n     */\n    private final boolean hasRichFunction;\n\n    @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n    AbstractBroadcastWrapperOperator(\n            StreamOperatorParameters<T> parameters,\n            StreamOperatorFactory<T> operatorFactory,\n            String[] broadcastStreamNames) {\n        this.parameters = Objects.requireNonNull(parameters);\n        this.streamConfig = Objects.requireNonNull(parameters.getStreamConfig());\n        this.containingTask = Objects.requireNonNull(parameters.getContainingTask());\n        this.output = Objects.requireNonNull(parameters.getOutput());\n        this.operatorFactory = Objects.requireNonNull(operatorFactory);\n        this.metrics = createOperatorMetricGroup(containingTask.getEnvironment(), streamConfig);\n        this.wrappedOperator =\n                (S)\n                        StreamOperatorFactoryUtil.<T, S>createOperator(\n                                        operatorFactory,\n                                        (StreamTask) containingTask,\n                                        streamConfig,\n                                        output,\n                                        parameters.getOperatorEventDispatcher())\n                                .f0;\n\n        this.hasRichFunction =\n                wrappedOperator instanceof AbstractUdfStreamOperator\n                        && ((AbstractUdfStreamOperator) wrappedOperator).getUserFunction()\n                                instanceof RichFunction;\n\n        if (hasRichFunction) {\n            this.wrappedOperatorRuntimeContext =\n                    new BroadcastStreamingRuntimeContext(\n                            containingTask.getEnvironment(),\n                            containingTask.getEnvironment().getAccumulatorRegistry().getUserMap(),\n                            wrappedOperator.getMetricGroup(),\n                            wrappedOperator.getOperatorID(),\n                            ((AbstractUdfStreamOperator) wrappedOperator)\n                                    .getProcessingTimeService(),\n                            null,\n                            containingTask.getEnvironment().getExternalResourceInfoProvider());\n\n            ((RichFunction) ((AbstractUdfStreamOperator) wrappedOperator).getUserFunction())\n                    .setRuntimeContext(wrappedOperatorRuntimeContext);\n\n            this.mailboxExecutor =\n                    containingTask\n                            .getMailboxExecutorFactory()\n                            .createExecutor(TaskMailbox.MIN_PRIORITY);\n\n            this.indexOfSubtask = containingTask.getIndexInSubtaskGroup();\n\n            // Puts in mailboxExecutor.\n            for (String name : broadcastStreamNames) {\n                BroadcastContext.putMailBoxExecutor(name + \"-\" + indexOfSubtask, mailboxExecutor);\n            }\n\n            this.broadcastStreamNames = broadcastStreamNames;\n\n            InputConfig[] inputConfigs =\n                    streamConfig.getInputs(containingTask.getUserCodeClassLoader());\n\n            int numNetworkInputs = 0;\n            while (numNetworkInputs < inputConfigs.length\n                    && inputConfigs[numNetworkInputs] instanceof NetworkInputConfig) {\n                numNetworkInputs++;\n            }\n            this.numInputs = numNetworkInputs;\n\n            this.isBlocked = new boolean[numInputs];\n            Arrays.fill(isBlocked, false);\n\n            this.inTypeSerializers = new TypeSerializer[numInputs];\n            for (int i = 0; i < numInputs; i++) {\n                inTypeSerializers[i] =\n                        streamConfig.getTypeSerializerIn(\n                                i, containingTask.getUserCodeClassLoader());\n            }\n\n            this.broadcastVariablesReady = false;\n\n            this.basePath =\n                    OperatorUtils.getDataCachePath(\n                            containingTask.getEnvironment().getTaskManagerInfo().getConfiguration(),\n                            containingTask\n                                    .getEnvironment()\n                                    .getIOManager()\n                                    .getSpillingDirectoriesPaths());\n            this.dataCacheWriters = new DataCacheWriter[numInputs];\n            this.hasPendingElements = new boolean[numInputs];\n            Arrays.fill(hasPendingElements, true);\n        }\n    }\n\n    /**\n     * Checks whether all broadcast variables are ready. Besides, it maintains a state\n     * {broadcastVariablesReady} to avoiding invoking {@code BroadcastContext.isCacheFinished(...)}\n     * repeatedly. Finally, it sets broadcast variables for {wrappedOperatorRuntimeContext} if the\n     * broadcast variables are ready.\n     *\n     * @return true if all broadcast variables are ready, false otherwise.\n     */\n    protected boolean areBroadcastVariablesReady() {\n        if (broadcastVariablesReady) {\n            return true;\n        }\n        for (String name : broadcastStreamNames) {\n            if (!BroadcastContext.isCacheFinished(name + \"-\" + indexOfSubtask)) {\n                return false;\n            } else {\n                String key = name + \"-\" + indexOfSubtask;\n                String userKey = name.substring(name.indexOf('-') + 1);\n                wrappedOperatorRuntimeContext.setBroadcastVariable(\n                        userKey, BroadcastContext.getBroadcastVariable(key));\n            }\n        }\n        broadcastVariablesReady = true;\n        return true;\n    }\n\n    private OperatorMetricGroup createOperatorMetricGroup(\n            Environment environment, StreamConfig streamConfig) {\n        try {\n            OperatorMetricGroup operatorMetricGroup =\n                    environment\n                            .getMetricGroup()\n                            .getOrAddOperator(\n                                    streamConfig.getOperatorID(), streamConfig.getOperatorName());\n            if (streamConfig.isChainEnd()) {\n                ((InternalOperatorIOMetricGroup) operatorMetricGroup.getIOMetricGroup())\n                        .reuseOutputMetricsForTask();\n            }\n            return operatorMetricGroup;\n        } catch (Exception e) {\n            LOG.warn(\"An error occurred while instantiating task metrics.\", e);\n            return UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();\n        }\n    }\n\n    /**\n     * Extracts common processing logic in subclasses' processing elements.\n     *\n     * @param streamRecord the input record.\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    protected void processElementX(\n            StreamRecord streamRecord,\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        if (!hasRichFunction) {\n            elementConsumer.accept(streamRecord);\n        } else if (isBlocked[inputIndex]) {\n            while (!areBroadcastVariablesReady()) {\n                mailboxExecutor.yield();\n            }\n            elementConsumer.accept(streamRecord);\n        } else if (!areBroadcastVariablesReady()) {\n            dataCacheWriters[inputIndex].addRecord(CacheElement.newRecord(streamRecord.getValue()));\n        } else {\n            if (hasPendingElements[inputIndex]) {\n                processPendingElementsAndWatermarks(\n                        inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n                hasPendingElements[inputIndex] = false;\n            }\n            keyContextSetter.accept(streamRecord);\n            elementConsumer.accept(streamRecord);\n        }\n    }\n\n    /**\n     * Extracts common processing logic in subclasses' processing watermarks.\n     *\n     * @param watermark the input watermark.\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    protected void processWatermarkX(\n            Watermark watermark,\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        if (!hasRichFunction) {\n            watermarkConsumer.accept(watermark);\n        } else if (isBlocked[inputIndex]) {\n            while (!areBroadcastVariablesReady()) {\n                mailboxExecutor.yield();\n            }\n...\n", "needle_token_start": 8756, "needle_token_end": 8779, "code_context_ntokens": 15991, "output": ["```java\n@Override\npublic List<Input> getInputs() {\n    return inputList;\n}\n```"]}
{"repo": "apache/flink-ml", "name": "setKeyContextElement1", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/AbstractBroadcastWrapperOperator.java", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to delegate the task of setting the key context for a record to another component, ensuring that the current processing element's key is correctly set in the context of the wrapped component.\n2. **Input**: It accepts a single parameter, which is a record encapsulating the data and metadata necessary for processing.\n3. **Output**: There is no direct output from this function; however, it affects the state of the wrapped component by setting the key context based on the provided record.\n4. **Procedure**: The function simply calls a similar function on a wrapped component, passing along the provided record. This action sets the key context in the wrapped component to match that of the current record, which is crucial for operations that depend on the key, such as state management and partitioning.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastVariableReceiverOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.state.ListState;\nimport org.apache.flink.api.common.state.ListStateDescriptor;\nimport org.apache.flink.api.common.typeinfo.BasicTypeInfo;\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.iteration.operator.OperatorStateUtils;\nimport org.apache.flink.ml.common.broadcast.BroadcastContext;\nimport org.apache.flink.runtime.state.StateInitializationContext;\nimport org.apache.flink.runtime.state.StateSnapshotContext;\nimport org.apache.flink.streaming.api.operators.AbstractInput;\nimport org.apache.flink.streaming.api.operators.AbstractStreamOperatorV2;\nimport org.apache.flink.streaming.api.operators.BoundedMultiInput;\nimport org.apache.flink.streaming.api.operators.Input;\nimport org.apache.flink.streaming.api.operators.MultipleInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport org.apache.commons.collections.IteratorUtils;\n\nimport java.io.Serializable;\nimport java.util.ArrayList;\nimport java.util.List;\n\n/** The operator that process all broadcast inputs and stores them in {@link BroadcastContext}. */\npublic class BroadcastVariableReceiverOperator<OUT> extends AbstractStreamOperatorV2<OUT>\n        implements MultipleInputStreamOperator<OUT>, BoundedMultiInput, Serializable {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastStreamNames;\n\n    /** Output types of input data streams. */\n    private final TypeInformation<?>[] inTypes;\n\n    /** Input list of the multi-input operator. */\n    @SuppressWarnings(\"rawtypes\")\n    private final List<Input> inputList;\n\n    /** Whether each broadcast input has finished. */\n    private boolean[] cachesReady;\n\n    /** State storage of the broadcast inputs. */\n    @SuppressWarnings(\"rawtypes\")\n    private ListState[] cacheStates;\n\n    /** CacheReady state storage of the broadcast inputs. */\n    private ListState<Boolean>[] cacheReadyStates;\n\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    BroadcastVariableReceiverOperator(\n            StreamOperatorParameters<OUT> parameters,\n            String[] broadcastStreamNames,\n            TypeInformation<?>[] inTypes) {\n        super(parameters, broadcastStreamNames.length);\n        this.broadcastStreamNames = broadcastStreamNames;\n        this.inTypes = inTypes;\n        inputList = new ArrayList<>();\n        for (int i = 0; i < inTypes.length; i++) {\n            inputList.add(new ProxyInput(this, i + 1));\n        }\n        this.cachesReady = new boolean[inTypes.length];\n        this.cacheStates = new ListState[inTypes.length];\n        this.cacheReadyStates = new ListState[inTypes.length];\n    }\n\n    @Override\n    public List<Input> getInputs() {\n        return inputList;\n    }\n\n    @Override\n    @SuppressWarnings({\"unchecked\"})\n    public void endInput(int i) throws Exception {\n        cachesReady[i - 1] = true;\n        String key =\n                broadcastStreamNames[i - 1] + \"-\" + getRuntimeContext().getIndexOfThisSubtask();\n        BroadcastContext.putBroadcastVariable(\n                key,\n                Tuple2.of(\n                        true,\n                        IteratorUtils.toList(\n                                ((ListState<?>) cacheStates[i - 1]).get().iterator())));\n        BroadcastContext.notifyCacheFinished(key);\n    }\n\n    @Override\n    public void snapshotState(StateSnapshotContext context) throws Exception {\n        super.snapshotState(context);\n        for (int i = 0; i < inTypes.length; i++) {\n            cacheReadyStates[i].clear();\n            cacheReadyStates[i].add(cachesReady[i]);\n        }\n    }\n\n    @Override\n    @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n    public void initializeState(StateInitializationContext context) throws Exception {\n        super.initializeState(context);\n        for (int i = 0; i < inTypes.length; i++) {\n            cacheStates[i] =\n                    context.getOperatorStateStore()\n                            .getListState(new ListStateDescriptor(\"cache_data_\" + i, inTypes[i]));\n            cacheReadyStates[i] =\n                    context.getOperatorStateStore()\n                            .getListState(\n                                    new ListStateDescriptor<>(\n                                            \"cache_ready_state_\" + i,\n                                            BasicTypeInfo.BOOLEAN_TYPE_INFO));\n...\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/OneInputBroadcastWrapperOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.iteration.operator.OperatorUtils;\nimport org.apache.flink.streaming.api.operators.BoundedOneInput;\nimport org.apache.flink.streaming.api.operators.OneInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.watermark.Watermark;\nimport org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;\n\n/** Wrapper for {@link OneInputStreamOperator}. */\npublic class OneInputBroadcastWrapperOperator<IN, OUT>\n        extends AbstractBroadcastWrapperOperator<OUT, OneInputStreamOperator<IN, OUT>>\n        implements OneInputStreamOperator<IN, OUT>, BoundedOneInput {\n\n    OneInputBroadcastWrapperOperator(\n            StreamOperatorParameters<OUT> parameters,\n            StreamOperatorFactory<OUT> operatorFactory,\n            String[] broadcastStreamNames) {\n        super(parameters, operatorFactory, broadcastStreamNames);\n    }\n\n    @Override\n    public void processElement(StreamRecord<IN> streamRecord) throws Exception {\n        processElementX(\n                streamRecord,\n                0,\n                wrappedOperator::processElement,\n                wrappedOperator::processWatermark,\n                wrappedOperator::setKeyContextElement);\n    }\n\n    @Override\n    public void endInput() throws Exception {\n        endInputX(\n                0,\n                wrappedOperator::processElement,\n                wrappedOperator::processWatermark,\n                wrappedOperator::setKeyContextElement);\n        OperatorUtils.processOperatorOrUdfIfSatisfy(\n                wrappedOperator, BoundedOneInput.class, BoundedOneInput::endInput);\n    }\n\n    @Override\n    public void processWatermark(Watermark watermark) throws Exception {\n        processWatermarkX(\n                watermark,\n                0,\n                wrappedOperator::processElement,\n                wrappedOperator::processWatermark,\n                wrappedOperator::setKeyContextElement);\n    }\n\n    @Override\n    public void processWatermarkStatus(WatermarkStatus watermarkStatus) throws Exception {\n        wrappedOperator.processWatermarkStatus(watermarkStatus);\n    }\n\n    @Override\n    public void processLatencyMarker(LatencyMarker latencyMarker) throws Exception {\n        wrappedOperator.processLatencyMarker(latencyMarker);\n    }\n\n    @Override\n    public void setKeyContextElement(StreamRecord<IN> streamRecord) throws Exception {\n        wrappedOperator.setKeyContextElement(streamRecord);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/TwoInputBroadcastWrapperOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.iteration.operator.OperatorUtils;\nimport org.apache.flink.streaming.api.operators.BoundedMultiInput;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.TwoInputStreamOperator;\nimport org.apache.flink.streaming.api.watermark.Watermark;\nimport org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;\n\n/** Wrapper for {@link TwoInputStreamOperator}. */\npublic class TwoInputBroadcastWrapperOperator<IN1, IN2, OUT>\n        extends AbstractBroadcastWrapperOperator<OUT, TwoInputStreamOperator<IN1, IN2, OUT>>\n        implements TwoInputStreamOperator<IN1, IN2, OUT>, BoundedMultiInput {\n\n    TwoInputBroadcastWrapperOperator(\n            StreamOperatorParameters<OUT> parameters,\n            StreamOperatorFactory<OUT> operatorFactory,\n            String[] broadcastStreamNames) {\n        super(parameters, operatorFactory, broadcastStreamNames);\n    }\n\n    @Override\n    public void processElement1(StreamRecord<IN1> streamRecord) throws Exception {\n        processElementX(\n                streamRecord,\n                0,\n                wrappedOperator::processElement1,\n                wrappedOperator::processWatermark1,\n                wrappedOperator::setKeyContextElement1);\n    }\n\n    @Override\n    public void processElement2(StreamRecord<IN2> streamRecord) throws Exception {\n        processElementX(\n                streamRecord,\n                1,\n                wrappedOperator::processElement2,\n                wrappedOperator::processWatermark2,\n                wrappedOperator::setKeyContextElement2);\n    }\n\n    @Override\n    public void endInput(int inputId) throws Exception {\n        if (inputId == 1) {\n            endInputX(\n                    inputId - 1,\n                    wrappedOperator::processElement1,\n                    wrappedOperator::processWatermark1,\n                    wrappedOperator::setKeyContextElement1);\n        } else {\n            endInputX(\n                    inputId - 1,\n                    wrappedOperator::processElement2,\n                    wrappedOperator::processWatermark2,\n                    wrappedOperator::setKeyContextElement2);\n        }\n        OperatorUtils.processOperatorOrUdfIfSatisfy(\n                wrappedOperator,\n                BoundedMultiInput.class,\n                boundedMultipleInput -> boundedMultipleInput.endInput(inputId));\n    }\n\n    @Override\n    public void processWatermark1(Watermark watermark) throws Exception {\n        processWatermarkX(\n                watermark,\n                0,\n                wrappedOperator::processElement1,\n                wrappedOperator::processWatermark1,\n                wrappedOperator::setKeyContextElement1);\n    }\n\n    @Override\n    public void processWatermark2(Watermark watermark) throws Exception {\n        processWatermarkX(\n                watermark,\n                1,\n                wrappedOperator::processElement2,\n                wrappedOperator::processWatermark2,\n                wrappedOperator::setKeyContextElement2);\n    }\n\n    @Override\n    public void processLatencyMarker1(LatencyMarker latencyMarker) throws Exception {\n        wrappedOperator.processLatencyMarker1(latencyMarker);\n    }\n\n    @Override\n    public void processLatencyMarker2(LatencyMarker latencyMarker) throws Exception {\n        wrappedOperator.processLatencyMarker2(latencyMarker);\n    }\n\n    @Override\n    public void processWatermarkStatus1(WatermarkStatus watermarkStatus) throws Exception {\n        wrappedOperator.processWatermarkStatus1(watermarkStatus);\n    }\n\n    @Override\n    public void processWatermarkStatus2(WatermarkStatus watermarkStatus) throws Exception {\n        wrappedOperator.processWatermarkStatus2(watermarkStatus);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/AbstractBroadcastWrapperOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.functions.RichFunction;\nimport org.apache.flink.api.common.operators.MailboxExecutor;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.core.fs.Path;\nimport org.apache.flink.core.memory.ManagedMemoryUseCase;\nimport org.apache.flink.iteration.datacache.nonkeyed.DataCacheReader;\nimport org.apache.flink.iteration.datacache.nonkeyed.DataCacheSnapshot;\nimport org.apache.flink.iteration.datacache.nonkeyed.DataCacheWriter;\nimport org.apache.flink.iteration.datacache.nonkeyed.Segment;\nimport org.apache.flink.iteration.operator.OperatorUtils;\nimport org.apache.flink.iteration.proxy.state.ProxyStreamOperatorStateContext;\nimport org.apache.flink.metrics.groups.OperatorMetricGroup;\nimport org.apache.flink.ml.common.broadcast.BroadcastContext;\nimport org.apache.flink.ml.common.broadcast.BroadcastStreamingRuntimeContext;\nimport org.apache.flink.ml.common.broadcast.typeinfo.CacheElement;\nimport org.apache.flink.ml.common.broadcast.typeinfo.CacheElementSerializer;\nimport org.apache.flink.runtime.checkpoint.CheckpointOptions;\nimport org.apache.flink.runtime.execution.Environment;\nimport org.apache.flink.runtime.jobgraph.OperatorID;\nimport org.apache.flink.runtime.metrics.groups.InternalOperatorIOMetricGroup;\nimport org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups;\nimport org.apache.flink.runtime.state.CheckpointStreamFactory;\nimport org.apache.flink.runtime.state.OperatorStateCheckpointOutputStream;\nimport org.apache.flink.runtime.state.StateInitializationContext;\nimport org.apache.flink.runtime.state.StatePartitionStreamProvider;\nimport org.apache.flink.runtime.state.StateSnapshotContext;\nimport org.apache.flink.runtime.util.NonClosingInputStreamDecorator;\nimport org.apache.flink.runtime.util.NonClosingOutputStreamDecorator;\nimport org.apache.flink.streaming.api.graph.StreamConfig;\nimport org.apache.flink.streaming.api.graph.StreamConfig.InputConfig;\nimport org.apache.flink.streaming.api.graph.StreamConfig.NetworkInputConfig;\nimport org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;\nimport org.apache.flink.streaming.api.operators.InternalTimeServiceManager;\nimport org.apache.flink.streaming.api.operators.OperatorSnapshotFutures;\nimport org.apache.flink.streaming.api.operators.Output;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactoryUtil;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.StreamOperatorStateContext;\nimport org.apache.flink.streaming.api.operators.StreamOperatorStateHandler;\nimport org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.CheckpointedStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamTaskStateInitializer;\nimport org.apache.flink.streaming.api.watermark.Watermark;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.tasks.StreamTask;\nimport org.apache.flink.streaming.runtime.tasks.mailbox.TaskMailbox;\nimport org.apache.flink.util.CloseableIterator;\nimport org.apache.flink.util.Preconditions;\nimport org.apache.flink.util.function.ThrowingConsumer;\n\nimport org.apache.commons.collections.IteratorUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.DataInputStream;\nimport java.io.DataOutputStream;\nimport java.io.InputStream;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Objects;\nimport java.util.Optional;\n\n/**\n * Base class for the broadcast wrapper operators.\n *\n * <p>Note that not all instances of {@link AbstractBroadcastWrapperOperator} need to access the\n * broadcast variables. If one instance of {@link AbstractBroadcastWrapperOperator} does not contain\n * a rich function, then it can directly process the input without waiting for the broadcast\n * variables.\n */\npublic abstract class AbstractBroadcastWrapperOperator<T, S extends StreamOperator<T>>\n        implements StreamOperator<T>, StreamOperatorStateHandler.CheckpointedStreamOperator {\n\n    private static final Logger LOG =\n            LoggerFactory.getLogger(AbstractBroadcastWrapperOperator.class);\n\n    protected final StreamOperatorParameters<T> parameters;\n\n    protected final StreamConfig streamConfig;\n\n    protected final StreamTask<?, ?> containingTask;\n\n    protected final Output<StreamRecord<T>> output;\n\n    protected final StreamOperatorFactory<T> operatorFactory;\n\n    protected final OperatorMetricGroup metrics;\n\n    protected final S wrappedOperator;\n\n    protected transient StreamOperatorStateHandler stateHandler;\n\n    protected transient InternalTimeServiceManager<?> timeServiceManager;\n\n    // ---------------- context info for rich function ----------------\n    private MailboxExecutor mailboxExecutor;\n\n    private String[] broadcastStreamNames;\n\n    /**\n     * Whether each input is blocked. Inputs with broadcast variables can only process their input\n     * records after broadcast variables are ready. One input is non-blocked if it can consume its\n     * inputs (by caching) when broadcast variables are not ready. Otherwise it has to block the\n     * processing and wait until the broadcast variables are ready to be accessed.\n     */\n    private boolean[] isBlocked;\n\n    /** Type serializer of each input. */\n    private TypeSerializer<?>[] inTypeSerializers;\n\n    /** Whether all broadcast variables of this operator are ready. */\n    private boolean broadcastVariablesReady;\n    /** Index of this subtask. */\n    protected transient int indexOfSubtask;\n\n    /** Number of the inputs of this operator. */\n    protected int numInputs;\n\n    /** RuntimeContext of the rich function in wrapped operator. */\n    private BroadcastStreamingRuntimeContext wrappedOperatorRuntimeContext;\n\n    /**\n     * Path of the file used to store the cached records. It could be local file system or remote\n     * file system.\n     */\n    private Path basePath;\n\n    /** DataCacheWriter for each input. */\n    @SuppressWarnings(\"rawtypes\")\n    private DataCacheWriter[] dataCacheWriters;\n\n    /** Whether each input has pending elements. */\n    private boolean[] hasPendingElements;\n\n    /**\n     * Whether this operator has a rich function and needs to access the broadcast variable. If yes,\n     * it cannot process elements until the broadcast variables are ready.\n     */\n    private final boolean hasRichFunction;\n\n    @SuppressWarnings({\"unchecked\", \"rawtypes\"})\n    AbstractBroadcastWrapperOperator(\n            StreamOperatorParameters<T> parameters,\n            StreamOperatorFactory<T> operatorFactory,\n            String[] broadcastStreamNames) {\n        this.parameters = Objects.requireNonNull(parameters);\n        this.streamConfig = Objects.requireNonNull(parameters.getStreamConfig());\n        this.containingTask = Objects.requireNonNull(parameters.getContainingTask());\n        this.output = Objects.requireNonNull(parameters.getOutput());\n        this.operatorFactory = Objects.requireNonNull(operatorFactory);\n        this.metrics = createOperatorMetricGroup(containingTask.getEnvironment(), streamConfig);\n        this.wrappedOperator =\n                (S)\n                        StreamOperatorFactoryUtil.<T, S>createOperator(\n                                        operatorFactory,\n                                        (StreamTask) containingTask,\n                                        streamConfig,\n                                        output,\n                                        parameters.getOperatorEventDispatcher())\n                                .f0;\n\n        this.hasRichFunction =\n                wrappedOperator instanceof AbstractUdfStreamOperator\n                        && ((AbstractUdfStreamOperator) wrappedOperator).getUserFunction()\n                                instanceof RichFunction;\n\n        if (hasRichFunction) {\n            this.wrappedOperatorRuntimeContext =\n                    new BroadcastStreamingRuntimeContext(\n                            containingTask.getEnvironment(),\n                            containingTask.getEnvironment().getAccumulatorRegistry().getUserMap(),\n                            wrappedOperator.getMetricGroup(),\n                            wrappedOperator.getOperatorID(),\n                            ((AbstractUdfStreamOperator) wrappedOperator)\n                                    .getProcessingTimeService(),\n                            null,\n                            containingTask.getEnvironment().getExternalResourceInfoProvider());\n\n            ((RichFunction) ((AbstractUdfStreamOperator) wrappedOperator).getUserFunction())\n                    .setRuntimeContext(wrappedOperatorRuntimeContext);\n\n            this.mailboxExecutor =\n                    containingTask\n                            .getMailboxExecutorFactory()\n                            .createExecutor(TaskMailbox.MIN_PRIORITY);\n\n            this.indexOfSubtask = containingTask.getIndexInSubtaskGroup();\n\n            // Puts in mailboxExecutor.\n            for (String name : broadcastStreamNames) {\n                BroadcastContext.putMailBoxExecutor(name + \"-\" + indexOfSubtask, mailboxExecutor);\n            }\n\n            this.broadcastStreamNames = broadcastStreamNames;\n\n            InputConfig[] inputConfigs =\n                    streamConfig.getInputs(containingTask.getUserCodeClassLoader());\n\n            int numNetworkInputs = 0;\n            while (numNetworkInputs < inputConfigs.length\n                    && inputConfigs[numNetworkInputs] instanceof NetworkInputConfig) {\n                numNetworkInputs++;\n            }\n            this.numInputs = numNetworkInputs;\n\n            this.isBlocked = new boolean[numInputs];\n            Arrays.fill(isBlocked, false);\n\n            this.inTypeSerializers = new TypeSerializer[numInputs];\n            for (int i = 0; i < numInputs; i++) {\n                inTypeSerializers[i] =\n                        streamConfig.getTypeSerializerIn(\n                                i, containingTask.getUserCodeClassLoader());\n            }\n\n            this.broadcastVariablesReady = false;\n\n            this.basePath =\n                    OperatorUtils.getDataCachePath(\n                            containingTask.getEnvironment().getTaskManagerInfo().getConfiguration(),\n                            containingTask\n                                    .getEnvironment()\n                                    .getIOManager()\n                                    .getSpillingDirectoriesPaths());\n            this.dataCacheWriters = new DataCacheWriter[numInputs];\n            this.hasPendingElements = new boolean[numInputs];\n            Arrays.fill(hasPendingElements, true);\n        }\n    }\n\n    /**\n     * Checks whether all broadcast variables are ready. Besides, it maintains a state\n     * {broadcastVariablesReady} to avoiding invoking {@code BroadcastContext.isCacheFinished(...)}\n     * repeatedly. Finally, it sets broadcast variables for {wrappedOperatorRuntimeContext} if the\n     * broadcast variables are ready.\n     *\n     * @return true if all broadcast variables are ready, false otherwise.\n     */\n    protected boolean areBroadcastVariablesReady() {\n        if (broadcastVariablesReady) {\n            return true;\n        }\n        for (String name : broadcastStreamNames) {\n            if (!BroadcastContext.isCacheFinished(name + \"-\" + indexOfSubtask)) {\n                return false;\n            } else {\n                String key = name + \"-\" + indexOfSubtask;\n                String userKey = name.substring(name.indexOf('-') + 1);\n                wrappedOperatorRuntimeContext.setBroadcastVariable(\n                        userKey, BroadcastContext.getBroadcastVariable(key));\n            }\n        }\n        broadcastVariablesReady = true;\n        return true;\n    }\n\n    private OperatorMetricGroup createOperatorMetricGroup(\n            Environment environment, StreamConfig streamConfig) {\n        try {\n            OperatorMetricGroup operatorMetricGroup =\n                    environment\n                            .getMetricGroup()\n                            .getOrAddOperator(\n                                    streamConfig.getOperatorID(), streamConfig.getOperatorName());\n            if (streamConfig.isChainEnd()) {\n                ((InternalOperatorIOMetricGroup) operatorMetricGroup.getIOMetricGroup())\n                        .reuseOutputMetricsForTask();\n            }\n            return operatorMetricGroup;\n        } catch (Exception e) {\n            LOG.warn(\"An error occurred while instantiating task metrics.\", e);\n            return UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();\n        }\n    }\n\n    /**\n     * Extracts common processing logic in subclasses' processing elements.\n     *\n     * @param streamRecord the input record.\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    protected void processElementX(\n            StreamRecord streamRecord,\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        if (!hasRichFunction) {\n            elementConsumer.accept(streamRecord);\n        } else if (isBlocked[inputIndex]) {\n            while (!areBroadcastVariablesReady()) {\n                mailboxExecutor.yield();\n            }\n            elementConsumer.accept(streamRecord);\n        } else if (!areBroadcastVariablesReady()) {\n            dataCacheWriters[inputIndex].addRecord(CacheElement.newRecord(streamRecord.getValue()));\n        } else {\n            if (hasPendingElements[inputIndex]) {\n                processPendingElementsAndWatermarks(\n                        inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n                hasPendingElements[inputIndex] = false;\n            }\n            keyContextSetter.accept(streamRecord);\n            elementConsumer.accept(streamRecord);\n        }\n    }\n\n    /**\n     * Extracts common processing logic in subclasses' processing watermarks.\n     *\n     * @param watermark the input watermark.\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    protected void processWatermarkX(\n            Watermark watermark,\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        if (!hasRichFunction) {\n            watermarkConsumer.accept(watermark);\n        } else if (isBlocked[inputIndex]) {\n            while (!areBroadcastVariablesReady()) {\n                mailboxExecutor.yield();\n            }\n            watermarkConsumer.accept(watermark);\n        } else if (!areBroadcastVariablesReady()) {\n            dataCacheWriters[inputIndex].addRecord(\n                    CacheElement.newWatermark(watermark.getTimestamp()));\n        } else {\n            if (hasPendingElements[inputIndex]) {\n                processPendingElementsAndWatermarks(\n                        inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n                hasPendingElements[inputIndex] = false;\n            }\n            watermarkConsumer.accept(watermark);\n        }\n    }\n\n    /**\n     * Extracts common processing logic in subclasses' endInput(...).\n     *\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings(\"rawtypes\")\n    protected void endInputX(\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        if (!hasRichFunction) {\n            return;\n        }\n        while (!areBroadcastVariablesReady()) {\n            mailboxExecutor.yield();\n        }\n        if (hasPendingElements[inputIndex]) {\n            processPendingElementsAndWatermarks(\n                    inputIndex, elementConsumer, watermarkConsumer, keyContextSetter);\n            hasPendingElements[inputIndex] = false;\n        }\n    }\n\n    /**\n     * Processes the pending elements that are cached by {@link DataCacheWriter}.\n     *\n     * @param inputIndex input id, starts from zero.\n     * @param elementConsumer the consumer function of StreamRecord, i.e.,\n     *     operator.processElement(...).\n     * @param watermarkConsumer the consumer function of WaterMark, i.e.,\n     *     operator.processWatermark(...).\n     * @param keyContextSetter the consumer function of setting key context, i.e.,\n     *     operator.setKeyContext(...).\n     * @throws Exception possible exception.\n     */\n    @SuppressWarnings({\"rawtypes\", \"unchecked\"})\n    private void processPendingElementsAndWatermarks(\n            int inputIndex,\n            ThrowingConsumer<StreamRecord, Exception> elementConsumer,\n            ThrowingConsumer<Watermark, Exception> watermarkConsumer,\n            ThrowingConsumer<StreamRecord, Exception> keyContextSetter)\n            throws Exception {\n        List<Segment> pendingSegments = dataCacheWriters[inputIndex].getSegments();\n        if (pendingSegments.size() != 0) {\n            DataCacheReader dataCacheReader =\n                    new DataCacheReader<>(\n                            new CacheElementSerializer<>(inTypeSerializers[inputIndex]),\n                            pendingSegments);\n            while (dataCacheReader.hasNext()) {\n                CacheElement cacheElement = (CacheElement) dataCacheReader.next();\n                switch (cacheElement.getType()) {\n                    case RECORD:\n                        StreamRecord record = new StreamRecord(cacheElement.getRecord());\n                        keyContextSetter.accept(record);\n                        elementConsumer.accept(record);\n                        break;\n                    case WATERMARK:\n                        watermarkConsumer.accept(new Watermark(cacheElement.getWatermark()));\n                        break;\n                    default:\n                        throw new RuntimeException(\n                                \"Unsupported CacheElement type: \" + cacheElement.getType());\n                }\n            }\n            dataCacheWriters[inputIndex].clear();\n        }\n    }\n\n    @Override\n    public void open() throws Exception {\n        wrappedOperator.open();\n    }\n\n    @Override\n    public void close() throws Exception {\n        wrappedOperator.close();\n        if (!hasRichFunction) {\n            return;\n        }\n        for (String name : broadcastStreamNames) {\n            BroadcastContext.remove(name + \"-\" + indexOfSubtask);\n        }\n    }\n\n    @Override\n    public void finish() throws Exception {\n        wrappedOperator.finish();\n    }\n\n    @Override\n    public void prepareSnapshotPreBarrier(long checkpointId) throws Exception {\n        wrappedOperator.prepareSnapshotPreBarrier(checkpointId);\n    }\n\n    @Override\n    public void initializeState(StreamTaskStateInitializer streamTaskStateManager)\n            throws Exception {\n        final TypeSerializer<?> keySerializer =\n                streamConfig.getStateKeySerializer(containingTask.getUserCodeClassLoader());\n\n        StreamOperatorStateContext streamOperatorStateContext =\n                streamTaskStateManager.streamOperatorStateContext(\n                        getOperatorID(),\n                        getClass().getSimpleName(),\n                        parameters.getProcessingTimeService(),\n                        this,\n                        keySerializer,\n                        containingTask.getCancelables(),\n                        metrics,\n                        streamConfig.getManagedMemoryFractionOperatorUseCaseOfSlot(\n                                ManagedMemoryUseCase.STATE_BACKEND,\n                                containingTask\n                                        .getEnvironment()\n                                        .getTaskManagerInfo()\n                                        .getConfiguration(),\n                                containingTask.getUserCodeClassLoader()),\n                        false);\n        stateHandler =\n                new StreamOperatorStateHandler(\n                        streamOperatorStateContext,\n                        containingTask.getExecutionConfig(),\n                        containingTask.getCancelables());\n        stateHandler.initializeOperatorState(this);\n\n        timeServiceManager = streamOperatorStateContext.internalTimerServiceManager();\n\n        wrappedOperator.initializeState(\n                (operatorID,\n                        operatorClassName,\n                        processingTimeService,\n                        keyContext,\n                        keySerializerX,\n                        streamTaskCloseableRegistry,\n                        metricGroup,\n                        managedMemoryFraction,\n                        isUsingCustomRawKeyedState) ->\n                        new ProxyStreamOperatorStateContext(\n                                streamOperatorStateContext,\n                                \"wrapped-\",\n                                CloseableIterator.empty(),\n                                0));\n    }\n\n    @Override\n    public OperatorSnapshotFutures snapshotState(\n            long checkpointId,\n            long timestamp,\n            CheckpointOptions checkpointOptions,\n            CheckpointStreamFactory storageLocation)\n            throws Exception {\n        return stateHandler.snapshotState(\n                this,\n                Optional.ofNullable(timeServiceManager),\n                streamConfig.getOperatorName(),\n                checkpointId,\n                timestamp,\n                checkpointOptions,\n                storageLocation,\n                false);\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked, rawtypes\")\n    public void initializeState(StateInitializationContext stateInitializationContext)\n            throws Exception {\n        List<StatePartitionStreamProvider> inputs =\n                IteratorUtils.toList(\n                        stateInitializationContext.getRawOperatorStateInputs().iterator());\n        Preconditions.checkState(\n                inputs.size() < 2, \"The input from raw operator state should be one or zero.\");\n        if (!hasRichFunction) {\n            return;\n        }\n        if (inputs.size() == 0) {\n            for (int i = 0; i < numInputs; i++) {\n                dataCacheWriters[i] =\n                        new DataCacheWriter(\n                                new CacheElementSerializer(inTypeSerializers[i]),\n                                basePath.getFileSystem(),\n                                OperatorUtils.createDataCacheFileGenerator(\n                                        basePath, \"cache\", streamConfig.getOperatorID()));\n            }\n        } else {\n            InputStream inputStream = inputs.get(0).getStream();\n            DataInputStream dis =\n                    new DataInputStream(new NonClosingInputStreamDecorator(inputStream));\n            Preconditions.checkState(dis.readInt() == numInputs, \"Number of input is wrong.\");\n            for (int i = 0; i < numInputs; i++) {\n                DataCacheSnapshot dataCacheSnapshot =\n                        DataCacheSnapshot.recover(\n                                inputStream,\n                                basePath.getFileSystem(),\n                                OperatorUtils.createDataCacheFileGenerator(\n                                        basePath, \"cache\", streamConfig.getOperatorID()));\n                dataCacheWriters[i] =\n                        new DataCacheWriter(\n                                new CacheElementSerializer(inTypeSerializers[i]),\n                                basePath.getFileSystem(),\n                                OperatorUtils.createDataCacheFileGenerator(\n                                        basePath, \"cache\", streamConfig.getOperatorID()),\n                                dataCacheSnapshot.getSegments());\n            }\n        }\n    }\n\n    @SuppressWarnings(\"unchecked\")\n    @Override\n    public void snapshotState(StateSnapshotContext stateSnapshotContext) throws Exception {\n        if (wrappedOperator instanceof StreamOperatorStateHandler.CheckpointedStreamOperator) {\n            ((CheckpointedStreamOperator) wrappedOperator).snapshotState(stateSnapshotContext);\n        }\n\n        if (!hasRichFunction) {\n            return;\n        }\n\n        OperatorStateCheckpointOutputStream checkpointOutputStream =\n                stateSnapshotContext.getRawOperatorStateOutput();\n        checkpointOutputStream.startNewPartition();\n        try (DataOutputStream dos =\n                new DataOutputStream(new NonClosingOutputStreamDecorator(checkpointOutputStream))) {\n            dos.writeInt(numInputs);\n        }\n        for (int i = 0; i < numInputs; i++) {\n            dataCacheWriters[i].writeSegmentsToFiles();\n            DataCacheSnapshot dataCacheSnapshot =\n                    new DataCacheSnapshot(\n                            basePath.getFileSystem(), null, dataCacheWriters[i].getSegments());\n            dataCacheSnapshot.writeTo(checkpointOutputStream);\n        }\n    }\n\n    \n@Override\n    public void setKeyContextElement1(StreamRecord<?> record) throws Exception {\n        wrappedOperator.setKeyContextElement1(record);\n    }\n\n    @Override\n    public void setKeyContextElement2(StreamRecord<?> record) throws Exception {\n        wrappedOperator.setKeyContextElement2(record);\n    }\n\n    @Override\n    public OperatorMetricGroup getMetricGroup() {\n        return wrappedOperator.getMetricGroup();\n    }\n\n    @Override\n    public OperatorID getOperatorID() {\n        return wrappedOperator.getOperatorID();\n    }\n\n    @Override\n    public void notifyCheckpointComplete(long checkpointId) throws Exception {\n        wrappedOperator.notifyCheckpointComplete(checkpointId);\n    }\n\n    @Override\n    public void notifyCheckpointAborted(long checkpointId) throws Exception {\n        wrappedOperator.notifyCheckpointAborted(checkpointId);\n    }\n\n    @Override\n    public void setCurrentKey(Object key) {\n        wrappedOperator.setCurrentKey(key);\n    }\n\n    @Override\n    public Object getCurrentKey() {\n        return wrappedOperator.getCurrentKey();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/util/RowUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.util;\n\nimport org.apache.flink.types.Row;\nimport org.apache.flink.types.RowKind;\n\n/** Utilities to deal with {@link Row} instances. */\npublic class RowUtils {\n\n    /**\n     * Creates a new row with fields that are copied from the existing one, and reserves some empty\n     * fields for further writing. The {@link RowKind} of the first row determines the {@link\n     * RowKind} of the result.\n     *\n     * @param existing The existing row to be cloned.\n     * @param reservedFields Num of fields to be reserved.\n     */\n    public static Row cloneWithReservedFields(Row existing, int reservedFields) {\n        Row result = new Row(existing.getKind(), existing.getArity() + reservedFields);\n        for (int i = 0; i < existing.getArity(); i++) {\n            result.setField(i, existing.getField(i));\n        }\n        return result;\n    }\n\n    /**\n     * Creates a new row with fields that are copied from the existing one, and appends a new filed\n     * with the specific value.\n     *\n     * @param existing The existing row to be cloned.\n     * @param value The value to be appended.\n     */\n    public static Row append(Row existing, Object value) {\n        Row result = cloneWithReservedFields(existing, 1);\n        result.setField(existing.getArity(), value);\n        return result;\n    }\n}\n\n", "needle_token_start": 10448, "needle_token_end": 10485, "code_context_ntokens": 11351, "output": ["```java\n@Override\npublic void setKeyContextElement1(StreamRecord<?> record) throws Exception {\n    wrappedOperator.setKeyContextElement1(record);\n}\n\n@Override\npublic void setKeyContextElement2(StreamRecord<?> record) throws Exception {\n    wrappedOperator.setKeyContextElement2(record);\n}\n```"]}
{"repo": "apache/flink-ml", "name": "getModelDataFromModel", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/builder/GraphBuilder.java", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve the output data identifiers from a specific model within a computational graph, ensuring that these identifiers are ready for subsequent operations when the graph executes.\n2. **Input**: The function takes a single instance of a model as its input.\n3. **Output**: It returns a list of identifiers that represent the outputs of a data retrieval operation from the model.\n4. **Procedure**: \n   - The function first checks if the model is registered in the graph. If not, it throws an error.\n   - It then verifies that the model is classified correctly as an algorithm operator within the graph. If misclassified, an error is thrown.\n   - The function checks if the output data identifiers for the model have already been fetched. If they have, an error is thrown to prevent redundancy.\n   - If all checks are passed, the function generates and assigns new data identifiers based on a predefined maximum number, which are then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/TableUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.typeutils.RowTypeInfo;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.table.catalog.Column;\nimport org.apache.flink.table.catalog.ResolvedSchema;\nimport org.apache.flink.table.runtime.typeutils.ExternalTypeInfo;\nimport org.apache.flink.table.types.DataType;\nimport org.apache.flink.table.types.logical.LogicalTypeRoot;\nimport org.apache.flink.types.Row;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.Map;\nimport java.util.Set;\n\n/** Utility class for operations related to Table API. */\npublic class TableUtils {\n\n    // Logical type roots that may cause wrong type conversion between Table and DataStream.\n    private static final Set<LogicalTypeRoot> LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO =\n            new HashSet<>();\n\n    static {\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.CHAR);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.VARCHAR);\n        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.BINARY);\n...\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/util/ReadWriteUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.util;\n\nimport org.apache.flink.api.common.eventtime.WatermarkStrategy;\nimport org.apache.flink.api.common.serialization.Encoder;\nimport org.apache.flink.api.connector.source.Source;\nimport org.apache.flink.connector.file.sink.FileSink;\nimport org.apache.flink.connector.file.src.FileSource;\nimport org.apache.flink.connector.file.src.reader.SimpleStreamFormat;\nimport org.apache.flink.core.fs.Path;\nimport org.apache.flink.ml.api.Stage;\nimport org.apache.flink.ml.builder.Graph;\nimport org.apache.flink.ml.builder.GraphData;\nimport org.apache.flink.ml.builder.GraphModel;\nimport org.apache.flink.ml.builder.GraphNode;\nimport org.apache.flink.ml.common.datastream.TableUtils;\nimport org.apache.flink.ml.param.Param;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.BasePathBucketAssigner;\nimport org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.util.Preconditions;\n\nimport org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonParser;\nimport org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;\n\nimport java.io.IOException;\nimport java.lang.reflect.InvocationTargetException;\nimport java.lang.reflect.Method;\nimport java.util.ArrayList;\nimport java.util.Comparator;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/** Utility methods for reading and writing stages. */\npublic class ReadWriteUtils {\n    public static final ObjectMapper OBJECT_MAPPER =\n            new ObjectMapper().enable(JsonParser.Feature.ALLOW_COMMENTS);\n\n    // A helper method that calls encodes the given parameter value to a json string. We can not\n    // call param.jsonEncode(value) directly because Param::jsonEncode(...) needs the actual type\n    // of the value.\n    private static <T> Object jsonEncodeHelper(Param<T> param, Object value) throws IOException {\n        return param.jsonEncode((T) value);\n    }\n\n    // Converts Map<Param<?>, Object> to Map<String, Object> which maps the parameter name to the\n    // json-supported parameter value.\n    private static Map<String, Object> jsonEncode(Map<Param<?>, Object> paramMap)\n            throws IOException {\n        Map<String, Object> result = new HashMap<>(paramMap.size());\n        for (Map.Entry<Param<?>, Object> entry : paramMap.entrySet()) {\n            Object json = jsonEncodeHelper(entry.getKey(), entry.getValue());\n            result.put(entry.getKey().name, json);\n        }\n        return result;\n    }\n\n    /**\n     * Saves the metadata of the given stage and the extra metadata to a file named `metadata` under\n     * the given path. The metadata of a stage includes the stage class name, parameter values etc.\n     *\n     * <p>Required: the metadata file under the given path should not exist.\n     *\n     * @param stage The stage instance.\n     * @param path The parent directory to save the stage metadata.\n     * @param extraMetadata The extra metadata to be saved.\n     */\n    public static void saveMetadata(Stage<?> stage, String path, Map<String, ?> extraMetadata)\n            throws IOException {\n        Map<String, Object> metadata = new HashMap<>(extraMetadata);\n        metadata.put(\"className\", stage.getClass().getName());\n        metadata.put(\"timestamp\", System.currentTimeMillis());\n        metadata.put(\"paramMap\", jsonEncode(stage.getParamMap()));\n        // TODO: add version in the metadata.\n        String metadataStr = OBJECT_MAPPER.writeValueAsString(metadata);\n\n        FileUtils.saveToFile(new Path(path, \"metadata\").toUri().toString(), metadataStr, false);\n    }\n\n    /**\n     * Saves the metadata of the given stage to a file named `metadata` under the given path. The\n     * metadata of a stage includes the stage class name, parameter values etc.\n     *\n     * <p>Required: the metadata file under the given path should not exist.\n     *\n     * @param stage The stage instance.\n     * @param path The parent directory to save the stage metadata.\n     */\n    public static void saveMetadata(Stage<?> stage, String path) throws IOException {\n        saveMetadata(stage, path, new HashMap<>());\n    }\n\n    /**\n     * Saves a Pipeline or PipelineModel with the given list of stages to the given path.\n     *\n     * @param pipeline A Pipeline or PipelineModel instance.\n     * @param stages A list of stages of the given pipeline.\n     * @param path The parent directory to save the pipeline metadata and its stages.\n     */\n    public static void savePipeline(Stage<?> pipeline, List<Stage<?>> stages, String path)\n            throws IOException {\n        // Creates parent directories if not already created.\n        FileUtils.mkdirs(new Path(path));\n\n        Map<String, Object> extraMetadata = new HashMap<>();\n        extraMetadata.put(\"numStages\", stages.size());\n        saveMetadata(pipeline, path, extraMetadata);\n\n        int numStages = stages.size();\n        for (int i = 0; i < numStages; i++) {\n            String stagePath = FileUtils.getPathForPipelineStage(i, numStages, path);\n            stages.get(i).save(stagePath);\n        }\n    }\n\n    /**\n     * Loads the stages of a Pipeline or PipelineModel from the given path.\n     *\n     * <p>The method throws RuntimeException if the expectedClassName is not empty AND it does not\n     * match the className of the previously saved Pipeline or PipelineModel.\n     *\n     * @param tEnv A StreamTableEnvironment instance.\n     * @param path The parent directory to load the pipeline metadata and its stages.\n     * @param expectedClassName The expected class name of the pipeline.\n     * @return A list of stages.\n     */\n    public static List<Stage<?>> loadPipeline(\n            StreamTableEnvironment tEnv, String path, String expectedClassName) throws IOException {\n        Map<String, ?> metadata = FileUtils.loadMetadata(path, expectedClassName);\n        int numStages = (Integer) metadata.get(\"numStages\");\n        List<Stage<?>> stages = new ArrayList<>(numStages);\n\n        for (int i = 0; i < numStages; i++) {\n            String stagePath = FileUtils.getPathForPipelineStage(i, numStages, path);\n            stages.add(loadStage(tEnv, stagePath));\n        }\n        return stages;\n    }\n\n    /**\n     * Saves a Graph or GraphModel with the given GraphData to the given path.\n     *\n     * @param graph A Graph or GraphModel instance.\n     * @param graphData A GraphData instance.\n     * @param path The parent directory to save the graph metadata and its stages.\n     */\n    public static void saveGraph(Stage<?> graph, GraphData graphData, String path)\n            throws IOException {\n        // Creates parent directories if not already created.\n        FileUtils.mkdirs(new Path(path));\n\n        Map<String, Object> extraMetadata = new HashMap<>();\n        extraMetadata.put(\"graphData\", graphData.toMap());\n        saveMetadata(graph, path, extraMetadata);\n        int maxNodeId =\n                graphData.nodes.stream()\n                        .map(node -> node.nodeId)\n                        .max(Comparator.naturalOrder())\n                        .orElse(-1);\n\n        for (GraphNode node : graphData.nodes) {\n            String stagePath = FileUtils.getPathForPipelineStage(node.nodeId, maxNodeId + 1, path);\n            node.stage.save(stagePath);\n        }\n    }\n\n    /**\n     * Loads a Graph or GraphModel from the given path.\n     *\n     * <p>The method throws RuntimeException if the expectedClassName is not empty AND it does not\n     * match the className of the previously saved Pipeline or PipelineModel.\n     *\n     * @param tEnv A StreamTableEnvironment instance.\n     * @param path The parent directory to load the pipeline metadata and its stages.\n     * @param expectedClassName The expected class name of the pipeline.\n     * @return A Graph or GraphModel instance.\n     */\n    @SuppressWarnings(\"unchecked\")\n    public static Stage<?> loadGraph(\n            StreamTableEnvironment tEnv, String path, String expectedClassName) throws IOException {\n        Map<String, ?> metadata = FileUtils.loadMetadata(path, expectedClassName);\n        GraphData graphData = GraphData.fromMap((Map<String, Object>) metadata.get(\"graphData\"));\n\n        int maxNodeId =\n                graphData.nodes.stream()\n                        .map(node -> node.nodeId)\n                        .max(Comparator.naturalOrder())\n                        .orElse(-1);\n\n        for (GraphNode node : graphData.nodes) {\n            String stagePath = FileUtils.getPathForPipelineStage(node.nodeId, maxNodeId + 1, path);\n            node.stage = loadStage(tEnv, stagePath);\n        }\n\n        if (expectedClassName.equals(GraphModel.class.getName())) {\n            return new GraphModel(\n                    graphData.nodes,\n                    graphData.modelInputIds,\n                    graphData.outputIds,\n                    graphData.inputModelDataIds,\n                    graphData.outputModelDataIds);\n        }\n        Preconditions.checkState(expectedClassName.equals(Graph.class.getName()));\n        return new Graph(\n                graphData.nodes,\n                graphData.estimatorInputIds,\n                graphData.modelInputIds,\n                graphData.outputIds,\n                graphData.inputModelDataIds,\n                graphData.outputModelDataIds);\n    }\n\n    /**\n     * Loads the stage with the saved parameters from the given path. This method reads the metadata\n     * file under the given path, instantiates the stage using its no-argument constructor, and\n     * loads the stage with the paramMap from the metadata file.\n     *\n     * <p>Note: This method does not attempt to read model data from the given path. Caller needs to\n     * read model data from the given path if the stage has model data.\n     *\n     * <p>Required: the class with type T must have a no-argument constructor.\n     *\n     * @param path The parent directory of the stage metadata file.\n     * @param <T> The class type of the Stage subclass.\n     * @return An instance of class type T.\n     */\n    public static <T extends Stage<T>> T loadStageParam(String path) throws IOException {\n        try {\n            return ParamUtils.instantiateWithParams(FileUtils.loadMetadata(path, \"\"));\n        } catch (ClassNotFoundException e) {\n            throw new RuntimeException(\"Failed to load stage.\", e);\n        }\n    }\n\n    /**\n     * Loads the stage from the given path by invoking the static load() method of the stage. The\n     * stage class name is read from the metadata file under the given path. The load() method is\n     * expected to construct the stage instance with the saved parameters, model data and other\n     * metadata if exists.\n     *\n     * <p>Required: the stage class must have a static load() method.\n     *\n     * @param tEnv A StreamTableEnvironment instance.\n     * @param path The parent directory of the stage metadata file.\n     * @return An instance of Stage.\n     */\n    public static Stage<?> loadStage(StreamTableEnvironment tEnv, String path) throws IOException {\n        Map<String, ?> metadata = FileUtils.loadMetadata(path, \"\");\n        String className = (String) metadata.get(\"className\");\n\n        try {\n            Class<?> clazz = Class.forName(className);\n            Method method = clazz.getMethod(\"load\", StreamTableEnvironment.class, String.class);\n            method.setAccessible(true);\n            return (Stage<?>) method.invoke(null, tEnv, path);\n        } catch (NoSuchMethodException e) {\n            String methodName =\n                    String.format(\"%s::load(StreamTableEnvironment, String)\", className);\n            throw new RuntimeException(\n                    \"Failed to load stage because the static method \"\n                            + methodName\n                            + \" is not implemented.\",\n                    e);\n        } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException e) {\n            throw new RuntimeException(\"Failed to load stage.\", e);\n        }\n    }\n\n    /**\n     * Saves the model data stream to the given path using the model encoder.\n     *\n     * @param model The model data stream.\n     * @param path The parent directory of the model data file.\n     * @param modelEncoder The encoder to encode the model data.\n     * @param <T> The class type of the model data.\n     */\n    public static <T> void saveModelData(\n            DataStream<T> model, String path, Encoder<T> modelEncoder) {\n        FileSink<T> sink =\n                FileSink.forRowFormat(FileUtils.getDataPath(path), modelEncoder)\n                        .withRollingPolicy(OnCheckpointRollingPolicy.build())\n                        .withBucketAssigner(new BasePathBucketAssigner<>())\n                        .build();\n        model.sinkTo(sink);\n    }\n\n    /**\n     * Loads the model data from the given path using the model decoder.\n     *\n     * @param tEnv A StreamTableEnvironment instance.\n     * @param path The parent directory of the model data file.\n     * @param modelDecoder The decoder used to decode the model data.\n     * @param <T> The class type of the model data.\n     * @return The loaded model data.\n     */\n    public static <T> Table loadModelData(\n            StreamTableEnvironment tEnv, String path, SimpleStreamFormat<T> modelDecoder) {\n        StreamExecutionEnvironment env = TableUtils.getExecutionEnvironment(tEnv);\n        Source<T, ?, ?> source =\n                FileSource.forRecordStreamFormat(modelDecoder, FileUtils.getDataPath(path)).build();\n        DataStream<T> modelDataStream =\n                env.fromSource(source, WatermarkStrategy.noWatermarks(), \"modelData\");\n        return tEnv.fromDataStream(modelDataStream);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/PipelineModel.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.annotation.VisibleForTesting;\nimport org.apache.flink.ml.api.AlgoOperator;\nimport org.apache.flink.ml.api.Model;\nimport org.apache.flink.ml.api.Stage;\nimport org.apache.flink.ml.api.Transformer;\nimport org.apache.flink.ml.param.Param;\nimport org.apache.flink.ml.servable.api.TransformerServable;\nimport org.apache.flink.ml.servable.builder.PipelineModelServable;\nimport org.apache.flink.ml.util.ParamUtils;\nimport org.apache.flink.ml.util.ReadWriteUtils;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.util.Preconditions;\n\nimport java.io.IOException;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * A PipelineModel acts as a Model. It consists of an ordered list of stages, each of which could be\n * a Model, Transformer or AlgoOperator.\n */\n@PublicEvolving\npublic final class PipelineModel implements Model<PipelineModel> {\n    private static final long serialVersionUID = 6184950154217411318L;\n    private final List<Stage<?>> stages;\n    private final Map<Param<?>, Object> paramMap = new HashMap<>();\n\n    public PipelineModel(List<Stage<?>> stages) {\n        this.stages = Preconditions.checkNotNull(stages);\n        ParamUtils.initializeMapWithDefaultValues(paramMap, this);\n    }\n\n    /**\n     * Applies all stages in this PipelineModel on the input tables in order. The output of one\n     * stage is used as the input of the next stage (if any). The output of the last stage is\n     * returned as the result of this method.\n     *\n     * @param inputs a list of tables\n     * @return a list of tables\n     */\n    @Override\n    public Table[] transform(Table... inputs) {\n        for (Stage<?> stage : stages) {\n            inputs = ((AlgoOperator<?>) stage).transform(inputs);\n        }\n        return inputs;\n    }\n\n    @Override\n    public Map<Param<?>, Object> getParamMap() {\n        return paramMap;\n    }\n\n    @Override\n    public void save(String path) throws IOException {\n        ReadWriteUtils.savePipeline(this, stages, path);\n    }\n\n    public static PipelineModel load(StreamTableEnvironment tEnv, String path) throws IOException {\n        return new PipelineModel(\n                ReadWriteUtils.loadPipeline(tEnv, path, PipelineModel.class.getName()));\n    }\n\n    public static PipelineModelServable loadServable(String path) throws IOException {\n        return PipelineModelServable.load(path);\n    }\n\n    /**\n     * Whether all stages in the pipeline have corresponding {@link TransformerServable} so that the\n     * PipelineModel can be turned into a TransformerServable and used in an online inference\n     * program.\n     *\n     * @return true if all stages have corresponding TransformerServable, false if not.\n     */\n    public boolean supportServable() {\n        for (Stage<?> stage : stages) {\n            if (!(stage instanceof Transformer)) {\n                return false;\n            }\n            Transformer<?> transformer = (Transformer<?>) stage;\n            Class<?> clazz = transformer.getClass();\n            try {\n                clazz.getMethod(\"loadServable\", String.class);\n            } catch (NoSuchMethodException e) {\n                return false;\n            }\n        }\n        return true;\n    }\n\n    /**\n     * Returns a list of all stages in this PipelineModel in order. The list is immutable.\n     *\n     * @return an immutable list of transformers.\n     */\n    @VisibleForTesting\n    List<Stage<?>> getStages() {\n        return Collections.unmodifiableList(stages);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/GraphExecutionHelper.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.util.Preconditions;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Deque;\nimport java.util.HashMap;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * A container class that maintains the execution state of the graph (e.g. which nodes are ready to\n * run).\n */\nclass GraphExecutionHelper {\n    /** A map from tableId to the list of nodes which take this table as input. */\n    private final Map<TableId, List<GraphNode>> consumerNodes = new HashMap<>();\n    /**\n     * A map from tableId to the corresponding table. A TableId would be mapped iff its\n     * corresponding Table has been constructed.\n     */\n    private final Map<TableId, Table> constructedTables = new HashMap<>();\n    /**\n     * A map that maintains the number of input tables that have not been constructed for each node\n     * in the graph.\n     */\n    private final Map<GraphNode, Integer> numUnConstructedInputTables = new HashMap<>();\n    /**\n     * An ordered list of nodes whose input tables have all been constructed AND who has not been\n     * fetch via pollNextReadyNode.\n     */\n    private final Deque<GraphNode> unFetchedReadyNodes = new LinkedList<>();\n\n    public GraphExecutionHelper(List<GraphNode> nodes) {\n        // Initializes dependentNodes and numUnConstructedInputs.\n        for (GraphNode node : nodes) {\n            List<TableId> inputs = new ArrayList<>();\n            inputs.addAll(Arrays.asList(node.algoOpInputIds));\n            if (node.estimatorInputIds != null) {\n                inputs.addAll(Arrays.asList(node.estimatorInputIds));\n            }\n            if (node.inputModelDataIds != null) {\n                inputs.addAll(Arrays.asList(node.inputModelDataIds));\n            }\n            numUnConstructedInputTables.put(node, inputs.size());\n            for (TableId tableId : inputs) {\n                consumerNodes.putIfAbsent(tableId, new ArrayList<>());\n                consumerNodes.get(tableId).add(node);\n            }\n        }\n    }\n\n    public void setTables(TableId[] tableIds, Table[] tables) {\n        // The length of tableIds could be larger than the length of tables because we over-allocate\n        // the number of tableIds (which is 20 by default) as placeholder of the stage's output\n        // tables when adding a stage in GraphBuilder.\n        Preconditions.checkArgument(\n                tableIds.length >= tables.length,\n                \"the length of tablesIds %s is less than the length of tables %s\",\n                tableIds.length,\n                tables.length);\n        for (int i = 0; i < tables.length; i++) {\n            setTable(tableIds[i], tables[i]);\n        }\n    }\n\n    private void setTable(TableId tableId, Table table) {\n        Preconditions.checkArgument(\n                !constructedTables.containsKey(tableId),\n                \"the table with id=%s has already been constructed\",\n                tableId.toString());\n        constructedTables.put(tableId, table);\n\n        for (GraphNode node : consumerNodes.getOrDefault(tableId, new ArrayList<>())) {\n            int prevNum = numUnConstructedInputTables.get(node);\n            if (prevNum == 1) {\n                unFetchedReadyNodes.addLast(node);\n                numUnConstructedInputTables.remove(node);\n            } else {\n                numUnConstructedInputTables.put(node, prevNum - 1);\n            }\n        }\n    }\n\n    public Table[] getTables(TableId[] tableIds) {\n        Table[] tables = new Table[tableIds.length];\n        for (int i = 0; i < tableIds.length; i++) {\n            tables[i] = getTable(tableIds[i]);\n        }\n        return tables;\n    }\n\n    private Table getTable(TableId tableId) {\n        Preconditions.checkArgument(\n                constructedTables.containsKey(tableId),\n                \"the table with id=%s has not been constructed yet\",\n                tableId.toString());\n        return constructedTables.get(tableId);\n    }\n\n    public GraphNode pollNextReadyNode() {\n        if (unFetchedReadyNodes.isEmpty() && !numUnConstructedInputTables.isEmpty()) {\n            throw new RuntimeException(\"there exists node whose input can not be constructed\");\n        }\n        return unFetchedReadyNodes.pollFirst();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/Pipeline.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.annotation.VisibleForTesting;\nimport org.apache.flink.ml.api.AlgoOperator;\nimport org.apache.flink.ml.api.Estimator;\nimport org.apache.flink.ml.api.Stage;\nimport org.apache.flink.ml.param.Param;\nimport org.apache.flink.ml.util.ParamUtils;\nimport org.apache.flink.ml.util.ReadWriteUtils;\nimport org.apache.flink.table.api.Table;\nimport org.apache.flink.table.api.bridge.java.StreamTableEnvironment;\nimport org.apache.flink.util.Preconditions;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * A Pipeline acts as an Estimator. It consists of an ordered list of stages, each of which could be\n * an Estimator, Model, Transformer or AlgoOperator.\n */\n@PublicEvolving\npublic final class Pipeline implements Estimator<Pipeline, PipelineModel> {\n    private static final long serialVersionUID = 6384850154817512318L;\n    private final List<Stage<?>> stages;\n    private final Map<Param<?>, Object> paramMap = new HashMap<>();\n\n    public Pipeline(List<Stage<?>> stages) {\n        this.stages = Preconditions.checkNotNull(stages);\n        ParamUtils.initializeMapWithDefaultValues(paramMap, this);\n    }\n\n    /**\n     * Trains the pipeline to fit on the given tables.\n     *\n     * <p>This method goes through all stages of this pipeline in order and does the following on\n     * each stage until the last Estimator (inclusive).\n     *\n     * <ul>\n     *   <li>If a stage is an Estimator, invoke {@link Estimator#fit(Table...)} with the input\n     *       tables to generate a Model. And if there is Estimator after this stage, transform the\n     *       input tables using the generated Model to get result tables, then pass the result\n     *       tables to the next stage as inputs.\n     *   <li>If a stage is an AlgoOperator AND there is Estimator after this stage, transform the\n     *       input tables using this stage to get result tables, then pass the result tables to the\n     *       next stage as inputs.\n     * </ul>\n     *\n     * <p>After all the Estimators are trained to fit their input tables, a new PipelineModel will\n     * be created with the same stages in this pipeline, except that all the Estimators in the\n     * PipelineModel are replaced with the models generated in the above process.\n     *\n     * @param inputs a list of tables\n     * @return a PipelineModel\n     */\n    @Override\n    public PipelineModel fit(Table... inputs) {\n        int lastEstimatorIdx = -1;\n        for (int i = 0; i < stages.size(); i++) {\n            if (stages.get(i) instanceof Estimator) {\n                lastEstimatorIdx = i;\n            }\n        }\n\n        List<Stage<?>> modelStages = new ArrayList<>(stages.size());\n        Table[] lastInputs = inputs;\n\n        for (int i = 0; i < stages.size(); i++) {\n            Stage<?> stage = stages.get(i);\n            AlgoOperator<?> modelStage;\n            if (stage instanceof AlgoOperator) {\n                modelStage = (AlgoOperator<?>) stage;\n            } else {\n                modelStage = ((Estimator<?, ?>) stage).fit(lastInputs);\n            }\n            modelStages.add(modelStage);\n\n            // Transforms inputs only if there exists Estimator stage after this stage.\n            if (i < lastEstimatorIdx) {\n                lastInputs = modelStage.transform(lastInputs);\n            }\n        }\n\n        return new PipelineModel(modelStages);\n    }\n\n    @Override\n    public Map<Param<?>, Object> getParamMap() {\n        return paramMap;\n    }\n\n    @Override\n    public void save(String path) throws IOException {\n        ReadWriteUtils.savePipeline(this, stages, path);\n    }\n\n    public static Pipeline load(StreamTableEnvironment tEnv, String path) throws IOException {\n        return new Pipeline(ReadWriteUtils.loadPipeline(tEnv, path, Pipeline.class.getName()));\n    }\n\n    /**\n     * Returns a list of all stages in this Pipeline in order. The list is immutable.\n     *\n     * @return an immutable list of stages.\n     */\n    @VisibleForTesting\n    List<Stage<?>> getStages() {\n        return Collections.unmodifiableList(stages);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/GraphBuilder.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.ml.api.AlgoOperator;\nimport org.apache.flink.ml.api.Estimator;\nimport org.apache.flink.ml.api.Model;\nimport org.apache.flink.ml.api.Stage;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.apache.flink.ml.builder.GraphNode.StageType;\n\n/**\n * A GraphBuilder provides APIs to build Estimator/Model/AlgoOperator from a DAG of stages, each of\n * which could be an Estimator, Model, Transformer or AlgoOperator.\n */\n@PublicEvolving\npublic final class GraphBuilder {\n\n    private int maxOutputLength = 20;\n\n    private int nextTableId = 0;\n\n    private int nextNodeId = 0;\n\n    /** An ordered list of nodes in the graph. */\n    private final List<GraphNode> nodes = new ArrayList<>();\n    /** A map from stage instance to the corresponding node in the graph. */\n    private final Map<Stage<?>, GraphNode> existingNodes = new HashMap<>();\n\n    public GraphBuilder() {}\n\n    /**\n     * Specifies the loose upper bound of the number of output tables that can be returned by the\n     * Model::getModelData() and AlgoOperator::transform() methods, for any stage involved in this\n     * Graph.\n     *\n     * <p>The default upper bound is 20.\n     */\n    public GraphBuilder setMaxOutputTableNum(int maxOutputLength) {\n        this.maxOutputLength = maxOutputLength;\n        return this;\n    }\n\n    /**\n     * Creates a TableId associated with this GraphBuilder. It can be used to specify the passing of\n     * tables between stages, as well as the input/output tables of the Graph/GraphModel generated\n     * by this builder.\n     *\n     * @return A TableId.\n     */\n    public TableId createTableId() {\n        return new TableId(nextTableId++);\n    }\n\n    /**\n     * Adds an AlgoOperator in the graph.\n     *\n     * <p>When the graph runs as Estimator, the transform() of the given AlgoOperator would be\n     * invoked with the given inputs. Then when the GraphModel fitted by this graph runs, the\n     * transform() of the given AlgoOperator would be invoked with the given inputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the transform() of the given AlgoOperator\n     * would be invoked with the given inputs.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by transform(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by transform().\n     *\n     * @param algoOp An AlgoOperator instance.\n     * @param inputs A list of TableIds which represents inputs to transform() of the given\n     *     AlgoOperator.\n     * @return A list of TableIds which represents the outputs of transform() of the given\n     *     AlgoOperator.\n     */\n    public TableId[] addAlgoOperator(AlgoOperator<?> algoOp, TableId... inputs) {\n        return addStage(algoOp, StageType.ALGO_OPERATOR, null, inputs);\n    }\n\n    /**\n     * Adds an Estimator in the graph.\n     *\n     * <p>When the graph runs as Estimator, the fit() of the given Estimator would be invoked with\n     * the given inputs. Then when the GraphModel fitted by this graph runs, the transform() of the\n     * Model fitted by the given Estimator would be invoked with the given inputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the fit() of the given Estimator would be\n     * invoked with the given inputs, then the transform() of the Model fitted by the given\n     * Estimator would be invoked with the given inputs.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by transform(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by transform().\n     *\n     * @param estimator An Estimator instance.\n     * @param inputs A list of TableIds which represents inputs to fit() of the given Estimator as\n     *     well as inputs to transform() of the Model fitted by the given Estimator.\n     * @return A list of TableIds which represents the outputs of transform() of the Model fitted by\n     *     the given Estimator.\n     */\n    public TableId[] addEstimator(Estimator<?, ?> estimator, TableId... inputs) {\n        return addEstimator(estimator, inputs, inputs);\n    }\n\n    /**\n     * Adds an Estimator in the graph.\n     *\n     * <p>When the graph runs as Estimator, the fit() of the given Estimator would be invoked with\n     * estimatorInputs. Then when the GraphModel fitted by this graph runs, the transform() of the\n     * Model fitted by the given Estimator would be invoked with modelInputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the fit() of the given Estimator would be\n     * invoked with estimatorInputs, then the transform() of the Model fitted by the given Estimator\n     * would be invoked with modelInputs.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by transform(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by transform().\n     *\n     * @param estimator An Estimator instance.\n     * @param estimatorInputs A list of TableIds which represents inputs to fit() of the given\n     *     Estimator.\n     * @param modelInputs A list of TableIds which represents inputs to transform() of the Model\n     *     fitted by the given Estimator.\n     * @return A list of TableIds which represents the outputs of transform() of the Model fitted by\n     *     the given Estimator.\n     */\n    public TableId[] addEstimator(\n            Estimator<?, ?> estimator, TableId[] estimatorInputs, TableId[] modelInputs) {\n        return addStage(estimator, StageType.ESTIMATOR, estimatorInputs, modelInputs);\n    }\n\n    /**\n     * When the graph runs as Estimator, it first generates a GraphModel that contains the Model\n     * fitted by the given Estimator. Then when this GraphModel runs, the setModelData() of the\n     * fitted Model would be invoked with the given inputs before its transform() is invoked.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the setModelData() of the Model fitted by\n     * the given Estimator would be invoked with the given inputs before its transform() is invoked.\n     *\n     * @param estimator An Estimator instance.\n     * @param inputs A list of TableIds which represents inputs to setModelData() of the Model\n     *     fitted by the given Estimator.\n     */\n    public void setModelDataOnEstimator(Estimator<?, ?> estimator, TableId... inputs) {\n        GraphNode node = existingNodes.get(estimator);\n        if (node == null) {\n            throw new RuntimeException(\"the Estimator has not been added to the graph\");\n        }\n        if (node.stageType != StageType.ESTIMATOR) {\n            throw new RuntimeException(\"the Estimator was previously added as an AlgoOperator\");\n        }\n        if (node.inputModelDataIds != null) {\n            throw new RuntimeException(\"the model data of this Estimator has already been set\");\n        }\n        node.inputModelDataIds = inputs;\n    }\n\n    /**\n     * When the graph runs as Estimator, the setModelData() of the given Model would be invoked with\n     * the given inputs before its transform() is invoked. Then when the GraphModel fitted by this\n     * graph runs, the setModelData() of the given Model would be invoked with the given inputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the setModelData() of the given Model would\n     * be invoked with the given inputs before its transform() is invoked.\n     *\n     * @param model A Model instance.\n     * @param inputs A list of TableIds which represents inputs to setModelData() of the given\n     *     Model.\n     */\n    public void setModelDataOnModel(Model<?> model, TableId... inputs) {\n        GraphNode node = existingNodes.get(model);\n        if (node == null) {\n            throw new RuntimeException(\"the Model has not been added to the graph\");\n        }\n        if (node.stageType != StageType.ALGO_OPERATOR) {\n            throw new RuntimeException(\"the Model was previously added as an Estimator\");\n        }\n        if (node.inputModelDataIds != null) {\n            throw new RuntimeException(\"the model data of this Model has already been set\");\n        }\n        node.inputModelDataIds = inputs;\n    }\n\n    /**\n     * When the graph runs as Estimator, it first generates a GraphModel that contains the Model\n     * fitted by the given Estimator. Then when this GraphModel runs, the getModelData() of the\n     * fitted Model would be invoked.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the getModelData() of the Model fitted by\n     * the given Estimator would be invoked.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by getModelData(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by getModelData().\n     *\n     * @param estimator An Estimator instance.\n     * @return A list of TableIds which represents the outputs of getModelData() of the Model fitted\n     *     by the given Estimator.\n     */\n    public TableId[] getModelDataFromEstimator(Estimator<?, ?> estimator) {\n        GraphNode node = existingNodes.get(estimator);\n        if (node == null) {\n            throw new RuntimeException(\"the Estimator has not been added to the graph\");\n        }\n        if (node.stageType != StageType.ESTIMATOR) {\n            throw new RuntimeException(\"the Estimator was previously added as an AlgoOperator\");\n        }\n        if (node.outputModelDataIds != null) {\n            throw new RuntimeException(\"the model data of this Estimator has already been fetched\");\n        }\n        node.outputModelDataIds = createTableIds(maxOutputLength);\n        return node.outputModelDataIds;\n    }\n\n    /**\n     * When the graph runs as Estimator, the getModelData() of the given Model would be invoked.\n     * Then when the GraphModel fitted by this graph runs, the getModelData() of the given Model\n     * would be invoked.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the getModelData() of the given Model would\n     * be invoked.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by getModelData(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by getModelData().\n     *\n     * @param model A Model instance.\n     * @return A list of TableIds which represents the outputs of getModelData() of the given Model.\n     */\n    \npublic TableId[] getModelDataFromModel(Model<?> model) {\n        GraphNode node = existingNodes.get(model);\n        if (node == null) {\n            throw new RuntimeException(\"the Model has not been added to the graph\");\n        }\n        if (node.stageType != StageType.ALGO_OPERATOR) {\n            throw new RuntimeException(\"the Model was previously added as an Estimator\");\n        }\n        if (node.outputModelDataIds != null) {\n            throw new RuntimeException(\"the model data of this Model has already been fetched\");\n        }\n        node.outputModelDataIds = createTableIds(maxOutputLength);\n        return node.outputModelDataIds;\n    }\n\n    /**\n     * Wraps nodes of the graph into an Estimator.\n     *\n     * <p>When the returned Estimator runs, and when the Model fitted by the returned Estimator\n     * runs, the sequence of operations recorded by the {@code addAlgoOperator(...)}, {@code\n     * addEstimator(...)}, {@code setModelData(...)} and {@code getModelData(...)} would be executed\n     * as specified in the Java doc of the corresponding methods.\n     *\n     * @param inputs A list of TableIds which represents inputs to fit() of the returned Estimator\n     *     as well as inputs to transform() of the Model fitted by the returned Estimator.\n     * @param outputs A list of TableIds which represents outputs of transform() of the Model fitted\n     *     by the returned Estimator.\n     * @return An Estimator which wraps the nodes of this graph.\n     */\n    public Estimator<?, ?> buildEstimator(TableId[] inputs, TableId[] outputs) {\n        return buildEstimator(inputs, inputs, outputs, null, null);\n    }\n\n    /**\n     * Wraps nodes of the graph into an Estimator.\n     *\n     * <p>When the returned Estimator runs, and when the Model fitted by the returned Estimator\n     * runs, the sequence of operations recorded by the {@code addAlgoOperator(...)}, {@code\n     * addEstimator(...)}, {@code setModelData(...)} and {@code getModelData(...)} would be executed\n     * as specified in the Java doc of the corresponding methods.\n     *\n     * @param inputs A list of TableIds which represents inputs to fit() of the returned Estimator\n     *     as well as inputs to transform() of the Model fitted by the returned Estimator.\n     * @param outputs A list of TableIds which represents outputs of transform() of the Model fitted\n     *     by the returned Estimator.\n     * @param inputModelData A list of TableIds which represents inputs to setModelData() of the\n     *     Model fitted by the returned Estimator.\n     * @param outputModelData A list of TableIds which represents outputs of getModelData() of the\n     *     Model fitted by the returned Estimator.\n     * @return An Estimator which wraps the nodes of this graph.\n     */\n    public Estimator<?, ?> buildEstimator(\n            TableId[] inputs,\n            TableId[] outputs,\n            TableId[] inputModelData,\n            TableId[] outputModelData) {\n        return buildEstimator(inputs, inputs, outputs, inputModelData, outputModelData);\n    }\n\n    /**\n     * Wraps nodes of the graph into an Estimator.\n     *\n     * <p>When the returned Estimator runs, and when the Model fitted by the returned Estimator\n     * runs, the sequence of operations recorded by the {@code addAlgoOperator(...)}, {@code\n     * addEstimator(...)}, {@code setModelData(...)} and {@code getModelData(...)} would be executed\n     * as specified in the Java doc of the corresponding methods.\n     *\n     * @param estimatorInputs A list of TableIds which represents inputs to fit() of the returned\n     *     Estimator.\n     * @param modelInputs A list of TableIds which represents inputs to transform() of the Model\n     *     fitted by the returned Estimator.\n     * @param outputs A list of TableIds which represents outputs of transform() of the Model fitted\n     *     by the returned Estimator.\n     * @param inputModelData A list of TableIds which represents inputs to setModelData() of the\n     *     Model fitted by the returned Estimator.\n     * @param outputModelData A list of TableIds which represents outputs of getModelData() of the\n     *     Model fitted by the returned Estimator.\n     * @return An Estimator which wraps the nodes of this graph.\n     */\n    public Estimator<?, ?> buildEstimator(\n            TableId[] estimatorInputs,\n            TableId[] modelInputs,\n            TableId[] outputs,\n            TableId[] inputModelData,\n            TableId[] outputModelData) {\n        return new Graph(\n                nodes, estimatorInputs, modelInputs, outputs, inputModelData, outputModelData);\n    }\n\n    /**\n     * Wraps nodes of the graph into an AlgoOperator.\n     *\n     * <p>When the returned AlgoOperator runs, the sequence of operations recorded by the {@code\n     * addAlgoOperator(...)} and {@code addEstimator(...)} would be executed as specified in the\n     * Java doc of the corresponding methods.\n     *\n     * @param inputs A list of TableIds which represents inputs to transform() of the returned\n     *     AlgoOperator.\n     * @param outputs A list of TableIds which represents outputs of transform() of the returned\n     *     AlgoOperator.\n     * @return An AlgoOperator which wraps the nodes of this graph.\n     */\n    public AlgoOperator<?> buildAlgoOperator(TableId[] inputs, TableId[] outputs) {\n        return buildModel(inputs, outputs, null, null);\n    }\n\n    /**\n     * Wraps nodes of the graph into a Model.\n     *\n     * <p>When the returned Model runs, the sequence of operations recorded by the {@code\n     * addAlgoOperator(...)} and {@code addEstimator(...)} would be executed as specified in the\n     * Java doc of the corresponding methods.\n     *\n     * @param inputs A list of TableIds which represents inputs to transform() of the returned\n     *     Model.\n     * @param outputs A list of TableIds which represents outputs of transform() of the returned\n     *     Model.\n     * @return A Model which wraps the nodes of this graph.\n     */\n    public Model<?> buildModel(TableId[] inputs, TableId[] outputs) {\n        return buildModel(inputs, outputs, null, null);\n    }\n\n    /**\n     * Wraps nodes of the graph into a Model.\n     *\n     * <p>When the returned Model runs, the sequence of operations recorded by the {@code\n     * addAlgoOperator(...)}, {@code addEstimator(...)}, {@code setModelData(...)} and {@code\n     * getModelData(...)} would be executed as specified in the Java doc of the corresponding\n     * methods.\n     *\n     * @param inputs A list of TableIds which represents inputs to transform() of the returned\n     *     Model.\n     * @param outputs A list of TableIds which represents outputs of transform() of the returned\n     *     Model.\n     * @param inputModelData A list of TableIds which represents inputs to setModelData() of the\n     *     returned Model.\n     * @param outputModelData A list of TableIds which represents outputs of getModelData() of the\n     *     returned Model.\n     * @return A Model which wraps the nodes of this graph.\n     */\n    public Model<?> buildModel(\n            TableId[] inputs,\n            TableId[] outputs,\n            TableId[] inputModelData,\n            TableId[] outputModelData) {\n        return new GraphModel(nodes, inputs, outputs, inputModelData, outputModelData);\n    }\n\n    private TableId[] createTableIds(int count) {\n        TableId[] result = new TableId[count];\n        for (int i = 0; i < count; i++) {\n            result[i] = createTableId();\n        }\n        return result;\n    }\n\n    private TableId[] addStage(\n            Stage<?> stage, StageType stageType, TableId[] estimatorInputs, TableId[] modelInputs) {\n        TableId[] outputs = createTableIds(maxOutputLength);\n        if (existingNodes.containsKey(stage)) {\n            throw new RuntimeException(\"The stage \" + stage + \" has already been added.\");\n        }\n        GraphNode node =\n                new GraphNode(\n                        nextNodeId++,\n                        stage,\n                        stageType,\n                        estimatorInputs,\n                        modelInputs,\n                        outputs,\n                        null,\n                        null);\n        nodes.add(node);\n        existingNodes.put(stage, node);\n        return outputs;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/CountTumblingWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into fixed-size windows based on the count number of\n * the elements. Windows do not overlap.\n */\npublic class CountTumblingWindows implements Windows {\n    /** Size of this window as row-count interval. */\n    private final long size;\n\n    private CountTumblingWindows(long size) {\n        Preconditions.checkArgument(\n                size > 0, \"The size of a count window must be a positive value\");\n        this.size = size;\n    }\n\n    /**\n     * Creates a new {@link CountTumblingWindows}.\n     *\n     * @param size the size of the window as row-count interval.\n     */\n    public static CountTumblingWindows of(long size) {\n        return new CountTumblingWindows(size);\n    }\n\n    public long getSize() {\n        return size;\n    }\n\n    @Override\n    public int hashCode() {\n        return Long.hashCode(size);\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof CountTumblingWindows)) {\n            return false;\n        }\n\n        CountTumblingWindows windows = (CountTumblingWindows) obj;\n        return this.size == windows.size;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/EventTimeSessionWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.api.common.time.Time;\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into sessions based on the timestamp of the elements.\n * Windows do not overlap.\n */\npublic class EventTimeSessionWindows implements Windows {\n    /** The session timeout, i.e. the time gap between sessions. */\n    private final Time gap;\n\n    private EventTimeSessionWindows(Time gap) {\n        this.gap = Preconditions.checkNotNull(gap);\n    }\n\n    /**\n     * Creates a new {@link EventTimeSessionWindows}.\n     *\n     * @param gap The session timeout, i.e. the time gap between sessions\n     */\n    public static EventTimeSessionWindows withGap(Time gap) {\n        return new EventTimeSessionWindows(gap);\n    }\n\n    public Time getGap() {\n        return gap;\n    }\n\n    @Override\n    public int hashCode() {\n        return gap.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof EventTimeSessionWindows)) {\n            return false;\n        }\n\n        EventTimeSessionWindows window = (EventTimeSessionWindows) obj;\n\n        return this.gap.equals(window.gap);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/EventTimeTumblingWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.api.common.time.Time;\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into fixed-size windows based on the timestamp of the\n * elements. Windows do not overlap.\n */\npublic class EventTimeTumblingWindows implements Windows {\n    /** Size of this window as time interval. */\n    private final Time size;\n\n    private EventTimeTumblingWindows(Time size) {\n        this.size = Preconditions.checkNotNull(size);\n    }\n\n    /**\n     * Creates a new {@link EventTimeTumblingWindows}.\n     *\n     * @param size the size of the window as time interval.\n     */\n    public static EventTimeTumblingWindows of(Time size) {\n        return new EventTimeTumblingWindows(size);\n    }\n\n    public Time getSize() {\n        return size;\n    }\n\n    @Override\n    public int hashCode() {\n        return size.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof EventTimeTumblingWindows)) {\n            return false;\n        }\n\n        EventTimeTumblingWindows window = (EventTimeTumblingWindows) obj;\n\n        return this.size.equals(window.size);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/GlobalWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n...\n", "needle_token_start": 11986, "needle_token_end": 12152, "code_context_ntokens": 16050, "output": ["```java\n/**\n * Retrieves the output data identifiers from a specific model within a computational graph.\n *\n * @param model The model from which to retrieve the output data identifiers.\n * @return A list of identifiers representing the outputs of a data retrieval operation from the model.\n * @throws RuntimeException If the model is not registered in the graph, is not classified as an algorithm operator, or if the output data identifiers have already been fetched.\n */\npublic TableId[] getModelDataFromModel(Model<?> model) {\n    GraphNode node = existingNodes.get(model);\n    if (node == null) {\n        throw new RuntimeException(\"the Model has not been added to the graph\");\n    }\n    if (node.stageType != StageType.ALGO_OPERATOR) {\n        throw new RuntimeException(\"the Model was previously added as an Estimator\");\n    }\n    if (node.outputModelDataIds != null) {\n        throw new RuntimeException(\"the model data of this Model has already been fetched\");\n    }\n    node.outputModelDataIds = createTableIds(maxOutputLength);\n    return node.outputModelDataIds;\n}\n```"]}
{"repo": "apache/flink-ml", "name": "getNumChunks", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/AllReduceImpl.java", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To determine the total number of segments an array can be divided into, given a predefined segment size.\n2. **Input**: A single integer representing the total length of the array.\n3. **Output**: A single integer indicating the total number of segments the array is divided into.\n4. **Procedure**: The function calculates how many times the predefined segment size can fully fit into the total array length and checks if there is any remainder. If there is a remainder, it adds an additional segment to account for this leftover part.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: flink-ml-core/src/main/java/org/apache/flink/ml/builder/GraphBuilder.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.builder;\n\nimport org.apache.flink.annotation.PublicEvolving;\nimport org.apache.flink.ml.api.AlgoOperator;\nimport org.apache.flink.ml.api.Estimator;\nimport org.apache.flink.ml.api.Model;\nimport org.apache.flink.ml.api.Stage;\n\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\nimport static org.apache.flink.ml.builder.GraphNode.StageType;\n\n/**\n * A GraphBuilder provides APIs to build Estimator/Model/AlgoOperator from a DAG of stages, each of\n * which could be an Estimator, Model, Transformer or AlgoOperator.\n */\n@PublicEvolving\npublic final class GraphBuilder {\n\n    private int maxOutputLength = 20;\n\n    private int nextTableId = 0;\n\n    private int nextNodeId = 0;\n\n    /** An ordered list of nodes in the graph. */\n    private final List<GraphNode> nodes = new ArrayList<>();\n    /** A map from stage instance to the corresponding node in the graph. */\n    private final Map<Stage<?>, GraphNode> existingNodes = new HashMap<>();\n\n    public GraphBuilder() {}\n\n    /**\n     * Specifies the loose upper bound of the number of output tables that can be returned by the\n     * Model::getModelData() and AlgoOperator::transform() methods, for any stage involved in this\n     * Graph.\n     *\n     * <p>The default upper bound is 20.\n     */\n    public GraphBuilder setMaxOutputTableNum(int maxOutputLength) {\n        this.maxOutputLength = maxOutputLength;\n        return this;\n    }\n\n    /**\n     * Creates a TableId associated with this GraphBuilder. It can be used to specify the passing of\n     * tables between stages, as well as the input/output tables of the Graph/GraphModel generated\n     * by this builder.\n     *\n     * @return A TableId.\n     */\n    public TableId createTableId() {\n        return new TableId(nextTableId++);\n    }\n\n    /**\n     * Adds an AlgoOperator in the graph.\n     *\n     * <p>When the graph runs as Estimator, the transform() of the given AlgoOperator would be\n     * invoked with the given inputs. Then when the GraphModel fitted by this graph runs, the\n     * transform() of the given AlgoOperator would be invoked with the given inputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the transform() of the given AlgoOperator\n     * would be invoked with the given inputs.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by transform(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by transform().\n     *\n     * @param algoOp An AlgoOperator instance.\n     * @param inputs A list of TableIds which represents inputs to transform() of the given\n     *     AlgoOperator.\n     * @return A list of TableIds which represents the outputs of transform() of the given\n     *     AlgoOperator.\n     */\n    public TableId[] addAlgoOperator(AlgoOperator<?> algoOp, TableId... inputs) {\n        return addStage(algoOp, StageType.ALGO_OPERATOR, null, inputs);\n    }\n\n    /**\n     * Adds an Estimator in the graph.\n     *\n     * <p>When the graph runs as Estimator, the fit() of the given Estimator would be invoked with\n     * the given inputs. Then when the GraphModel fitted by this graph runs, the transform() of the\n     * Model fitted by the given Estimator would be invoked with the given inputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the fit() of the given Estimator would be\n     * invoked with the given inputs, then the transform() of the Model fitted by the given\n     * Estimator would be invoked with the given inputs.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by transform(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by transform().\n     *\n     * @param estimator An Estimator instance.\n     * @param inputs A list of TableIds which represents inputs to fit() of the given Estimator as\n     *     well as inputs to transform() of the Model fitted by the given Estimator.\n     * @return A list of TableIds which represents the outputs of transform() of the Model fitted by\n     *     the given Estimator.\n     */\n    public TableId[] addEstimator(Estimator<?, ?> estimator, TableId... inputs) {\n        return addEstimator(estimator, inputs, inputs);\n    }\n\n    /**\n     * Adds an Estimator in the graph.\n     *\n     * <p>When the graph runs as Estimator, the fit() of the given Estimator would be invoked with\n     * estimatorInputs. Then when the GraphModel fitted by this graph runs, the transform() of the\n     * Model fitted by the given Estimator would be invoked with modelInputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the fit() of the given Estimator would be\n     * invoked with estimatorInputs, then the transform() of the Model fitted by the given Estimator\n     * would be invoked with modelInputs.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by transform(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by transform().\n     *\n     * @param estimator An Estimator instance.\n     * @param estimatorInputs A list of TableIds which represents inputs to fit() of the given\n     *     Estimator.\n     * @param modelInputs A list of TableIds which represents inputs to transform() of the Model\n     *     fitted by the given Estimator.\n     * @return A list of TableIds which represents the outputs of transform() of the Model fitted by\n     *     the given Estimator.\n     */\n    public TableId[] addEstimator(\n            Estimator<?, ?> estimator, TableId[] estimatorInputs, TableId[] modelInputs) {\n        return addStage(estimator, StageType.ESTIMATOR, estimatorInputs, modelInputs);\n    }\n\n    /**\n     * When the graph runs as Estimator, it first generates a GraphModel that contains the Model\n     * fitted by the given Estimator. Then when this GraphModel runs, the setModelData() of the\n     * fitted Model would be invoked with the given inputs before its transform() is invoked.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the setModelData() of the Model fitted by\n     * the given Estimator would be invoked with the given inputs before its transform() is invoked.\n     *\n     * @param estimator An Estimator instance.\n     * @param inputs A list of TableIds which represents inputs to setModelData() of the Model\n     *     fitted by the given Estimator.\n     */\n    public void setModelDataOnEstimator(Estimator<?, ?> estimator, TableId... inputs) {\n        GraphNode node = existingNodes.get(estimator);\n        if (node == null) {\n            throw new RuntimeException(\"the Estimator has not been added to the graph\");\n        }\n        if (node.stageType != StageType.ESTIMATOR) {\n            throw new RuntimeException(\"the Estimator was previously added as an AlgoOperator\");\n        }\n        if (node.inputModelDataIds != null) {\n            throw new RuntimeException(\"the model data of this Estimator has already been set\");\n        }\n        node.inputModelDataIds = inputs;\n    }\n\n    /**\n     * When the graph runs as Estimator, the setModelData() of the given Model would be invoked with\n     * the given inputs before its transform() is invoked. Then when the GraphModel fitted by this\n     * graph runs, the setModelData() of the given Model would be invoked with the given inputs.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the setModelData() of the given Model would\n     * be invoked with the given inputs before its transform() is invoked.\n     *\n     * @param model A Model instance.\n     * @param inputs A list of TableIds which represents inputs to setModelData() of the given\n     *     Model.\n     */\n    public void setModelDataOnModel(Model<?> model, TableId... inputs) {\n        GraphNode node = existingNodes.get(model);\n        if (node == null) {\n            throw new RuntimeException(\"the Model has not been added to the graph\");\n        }\n        if (node.stageType != StageType.ALGO_OPERATOR) {\n            throw new RuntimeException(\"the Model was previously added as an Estimator\");\n        }\n        if (node.inputModelDataIds != null) {\n            throw new RuntimeException(\"the model data of this Model has already been set\");\n        }\n        node.inputModelDataIds = inputs;\n    }\n\n    /**\n     * When the graph runs as Estimator, it first generates a GraphModel that contains the Model\n     * fitted by the given Estimator. Then when this GraphModel runs, the getModelData() of the\n     * fitted Model would be invoked.\n     *\n     * <p>When the graph runs as AlgoOperator or Model, the getModelData() of the Model fitted by\n     * the given Estimator would be invoked.\n     *\n     * <p>NOTE: the number of the returned TableIds does not represent the actual number of Tables\n     * outputted by getModelData(). This number could be configured using {@link\n     * #setMaxOutputTableNum(int)}. Users should make sure that this number >= the actual number of\n     * Tables outputted by getModelData().\n     *\n     * @param estimator An Estimator instance.\n     * @return A list of TableIds which represents the outputs of getModelData() of the Model fitted\n     *     by the given Estimator.\n     */\n...\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/CountTumblingWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into fixed-size windows based on the count number of\n * the elements. Windows do not overlap.\n */\npublic class CountTumblingWindows implements Windows {\n    /** Size of this window as row-count interval. */\n    private final long size;\n\n    private CountTumblingWindows(long size) {\n        Preconditions.checkArgument(\n                size > 0, \"The size of a count window must be a positive value\");\n        this.size = size;\n    }\n\n    /**\n     * Creates a new {@link CountTumblingWindows}.\n     *\n     * @param size the size of the window as row-count interval.\n     */\n    public static CountTumblingWindows of(long size) {\n        return new CountTumblingWindows(size);\n    }\n\n    public long getSize() {\n        return size;\n    }\n\n    @Override\n    public int hashCode() {\n        return Long.hashCode(size);\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof CountTumblingWindows)) {\n            return false;\n        }\n\n        CountTumblingWindows windows = (CountTumblingWindows) obj;\n        return this.size == windows.size;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/EventTimeSessionWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.api.common.time.Time;\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into sessions based on the timestamp of the elements.\n * Windows do not overlap.\n */\npublic class EventTimeSessionWindows implements Windows {\n    /** The session timeout, i.e. the time gap between sessions. */\n    private final Time gap;\n\n    private EventTimeSessionWindows(Time gap) {\n        this.gap = Preconditions.checkNotNull(gap);\n    }\n\n    /**\n     * Creates a new {@link EventTimeSessionWindows}.\n     *\n     * @param gap The session timeout, i.e. the time gap between sessions\n     */\n    public static EventTimeSessionWindows withGap(Time gap) {\n        return new EventTimeSessionWindows(gap);\n    }\n\n    public Time getGap() {\n        return gap;\n    }\n\n    @Override\n    public int hashCode() {\n        return gap.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof EventTimeSessionWindows)) {\n            return false;\n        }\n\n        EventTimeSessionWindows window = (EventTimeSessionWindows) obj;\n\n        return this.gap.equals(window.gap);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/EventTimeTumblingWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.api.common.time.Time;\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into fixed-size windows based on the timestamp of the\n * elements. Windows do not overlap.\n */\npublic class EventTimeTumblingWindows implements Windows {\n    /** Size of this window as time interval. */\n    private final Time size;\n\n    private EventTimeTumblingWindows(Time size) {\n        this.size = Preconditions.checkNotNull(size);\n    }\n\n    /**\n     * Creates a new {@link EventTimeTumblingWindows}.\n     *\n     * @param size the size of the window as time interval.\n     */\n    public static EventTimeTumblingWindows of(Time size) {\n        return new EventTimeTumblingWindows(size);\n    }\n\n    public Time getSize() {\n        return size;\n    }\n\n    @Override\n    public int hashCode() {\n        return size.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof EventTimeTumblingWindows)) {\n            return false;\n        }\n\n        EventTimeTumblingWindows window = (EventTimeTumblingWindows) obj;\n\n        return this.size.equals(window.size);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/GlobalWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\n/**\n * A windowing strategy that groups all elements into a single global window. This strategy assumes\n * that the input strategy is bounded.\n */\npublic class GlobalWindows implements Windows {\n    private static final GlobalWindows INSTANCE = new GlobalWindows();\n\n    private GlobalWindows() {}\n\n    /** Gets a {@link GlobalWindows} instance. */\n    public static GlobalWindows getInstance() {\n        return INSTANCE;\n    }\n\n    @Override\n    public int hashCode() {\n        return GlobalWindows.class.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        return obj instanceof GlobalWindows;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/ProcessingTimeSessionWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.api.common.time.Time;\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into sessions based on the current system time of the\n * machine the operation is running on. Windows do not overlap.\n */\npublic class ProcessingTimeSessionWindows implements Windows {\n    /** The session timeout, i.e. the time gap between sessions. */\n    private final Time gap;\n\n    private ProcessingTimeSessionWindows(Time gap) {\n        this.gap = Preconditions.checkNotNull(gap);\n    }\n\n    /**\n     * Creates a new {@link ProcessingTimeSessionWindows}.\n     *\n     * @param gap The session timeout, i.e. the time gap between sessions\n     */\n    public static ProcessingTimeSessionWindows withGap(Time gap) {\n        return new ProcessingTimeSessionWindows(gap);\n    }\n\n    public Time getGap() {\n        return gap;\n    }\n\n    @Override\n    public int hashCode() {\n        return gap.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof ProcessingTimeSessionWindows)) {\n            return false;\n        }\n\n        ProcessingTimeSessionWindows window = (ProcessingTimeSessionWindows) obj;\n\n        return this.gap.equals(window.gap);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/ProcessingTimeTumblingWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\nimport org.apache.flink.api.common.time.Time;\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A windowing strategy that groups elements into fixed-size windows based on the current system\n * time of the machine the operation is running on. Windows do not overlap.\n */\npublic class ProcessingTimeTumblingWindows implements Windows {\n    /** Size of this window as time interval. */\n    private final Time size;\n\n    private ProcessingTimeTumblingWindows(Time size) {\n        this.size = Preconditions.checkNotNull(size);\n    }\n\n    /**\n     * Creates a new {@link ProcessingTimeTumblingWindows}.\n     *\n     * @param size the size of the window as time interval.\n     */\n    public static ProcessingTimeTumblingWindows of(Time size) {\n        return new ProcessingTimeTumblingWindows(size);\n    }\n\n    public Time getSize() {\n        return size;\n    }\n\n    @Override\n    public int hashCode() {\n        return size.hashCode();\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (!(obj instanceof ProcessingTimeTumblingWindows)) {\n            return false;\n        }\n\n        ProcessingTimeTumblingWindows window = (ProcessingTimeTumblingWindows) obj;\n\n        return this.size.equals(window.size);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/window/Windows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.window;\n\n/** Windowing strategy that determines how to create mini-batches from input data. */\npublic interface Windows {}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/param/WindowsParam.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.param;\n\nimport org.apache.flink.api.common.time.Time;\nimport org.apache.flink.ml.common.window.CountTumblingWindows;\nimport org.apache.flink.ml.common.window.EventTimeSessionWindows;\nimport org.apache.flink.ml.common.window.EventTimeTumblingWindows;\nimport org.apache.flink.ml.common.window.GlobalWindows;\nimport org.apache.flink.ml.common.window.ProcessingTimeSessionWindows;\nimport org.apache.flink.ml.common.window.ProcessingTimeTumblingWindows;\nimport org.apache.flink.ml.common.window.Windows;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\n/** Class for the {@link Windows} parameter. */\npublic class WindowsParam extends Param<Windows> {\n    public WindowsParam(\n            String name,\n            String description,\n            Windows defaultValue,\n            ParamValidator<Windows> validator) {\n        super(name, Windows.class, description, defaultValue, validator);\n    }\n\n    @Override\n    public Object jsonEncode(Windows value) {\n        Map<String, Object> map = new HashMap<>();\n\n        map.put(\"class\", value.getClass().getName());\n        if (value instanceof GlobalWindows) {\n            return map;\n        } else if (value instanceof CountTumblingWindows) {\n            map.put(\"size\", ((CountTumblingWindows) value).getSize());\n        } else if (value instanceof ProcessingTimeTumblingWindows) {\n            map.put(\"size\", ((ProcessingTimeTumblingWindows) value).getSize().toMilliseconds());\n        } else if (value instanceof EventTimeTumblingWindows) {\n            map.put(\"size\", ((EventTimeTumblingWindows) value).getSize().toMilliseconds());\n        } else if (value instanceof ProcessingTimeSessionWindows) {\n            map.put(\"gap\", ((ProcessingTimeSessionWindows) value).getGap().toMilliseconds());\n        } else if (value instanceof EventTimeSessionWindows) {\n            map.put(\"gap\", ((EventTimeSessionWindows) value).getGap().toMilliseconds());\n        } else {\n            throw new UnsupportedOperationException(\n                    String.format(\"Unsupported %s subclass: %s\", Windows.class, value.getClass()));\n        }\n        return map;\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked\")\n    public Windows jsonDecode(Object json) {\n        Map<String, Object> map = (Map<String, Object>) json;\n\n        String classString = (String) map.get(\"class\");\n        if (classString.equals(GlobalWindows.class.getName())) {\n            return GlobalWindows.getInstance();\n        } else if (classString.equals(CountTumblingWindows.class.getName())) {\n            long size = ((Number) map.get(\"size\")).longValue();\n            return CountTumblingWindows.of(size);\n        } else if (classString.equals(ProcessingTimeTumblingWindows.class.getName())) {\n            Time size = Time.milliseconds(((Number) map.get(\"size\")).longValue());\n            return ProcessingTimeTumblingWindows.of(size);\n        } else if (classString.equals(EventTimeTumblingWindows.class.getName())) {\n            Time size = Time.milliseconds(((Number) map.get(\"size\")).longValue());\n            return EventTimeTumblingWindows.of(size);\n        } else if (classString.equals(ProcessingTimeSessionWindows.class.getName())) {\n            Time gap = Time.milliseconds(((Number) map.get(\"gap\")).longValue());\n            return ProcessingTimeSessionWindows.withGap(gap);\n        } else if (classString.equals(EventTimeSessionWindows.class.getName())) {\n            Time gap = Time.milliseconds(((Number) map.get(\"gap\")).longValue());\n            return EventTimeSessionWindows.withGap(gap);\n        } else {\n            throw new UnsupportedOperationException(\n                    String.format(\"Unsupported %s subclass: %s\", Windows.class, classString));\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/iteration/ForwardInputsOfLastRound.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.iteration;\n\nimport org.apache.flink.api.common.functions.FlatMapFunction;\nimport org.apache.flink.iteration.IterationListener;\nimport org.apache.flink.util.Collector;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * A FlatMapFunction which forwards input values in the last round of the iteration to the output.\n * Other input values will be dropped.\n *\n * @param <T> The class type of the input element.\n */\npublic class ForwardInputsOfLastRound<T> implements FlatMapFunction<T, T>, IterationListener<T> {\n    private List<T> valuesInLastEpoch = new ArrayList<>();\n    private List<T> valuesInCurrentEpoch = new ArrayList<>();\n\n    @Override\n    public void flatMap(T value, Collector<T> out) {\n        valuesInCurrentEpoch.add(value);\n    }\n\n    @Override\n    public void onEpochWatermarkIncremented(int epochWatermark, Context context, Collector<T> out) {\n        valuesInLastEpoch = valuesInCurrentEpoch;\n        valuesInCurrentEpoch = new ArrayList<>();\n    }\n\n    @Override\n    public void onIterationTerminated(Context context, Collector<T> out) {\n        for (T value : valuesInLastEpoch) {\n            out.collect(value);\n        }\n        if (!valuesInCurrentEpoch.isEmpty()) {\n            throw new IllegalStateException(\n                    \"flatMap() is invoked since the last onEpochWatermarkIncremented callback\");\n        }\n        valuesInLastEpoch.clear();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/iteration/TerminateOnMaxIterOrTol.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.iteration;\n\nimport org.apache.flink.api.common.functions.FlatMapFunction;\nimport org.apache.flink.iteration.IterationListener;\nimport org.apache.flink.iteration.IterationRecord;\nimport org.apache.flink.util.Collector;\nimport org.apache.flink.util.Preconditions;\n\n/**\n * A FlatMapFunction that emits values iff the iteration's epochWatermark does not exceed a certain\n * threshold and the loss exceeds a certain tolerance.\n *\n * <p>When the output of this FlatMapFunction is used as the termination criteria of an iteration\n * body, the iteration will be executed for at most the given `maxIter` iterations. And the\n * iteration will terminate once any input value is smaller than or equal to the given `tol`.\n */\npublic class TerminateOnMaxIterOrTol\n        implements IterationListener<Integer>, FlatMapFunction<Double, Integer> {\n\n    private final int maxIter;\n\n    private final double tol;\n\n    private double loss = Double.MAX_VALUE;\n\n    public TerminateOnMaxIterOrTol(Integer maxIter, Double tol) {\n        this.maxIter = maxIter;\n        this.tol = tol;\n    }\n\n    public TerminateOnMaxIterOrTol(Double tol) {\n        this.maxIter = IterationRecord.END_EPOCH_WATERMARK;\n        this.tol = tol;\n    }\n\n    @Override\n    public void flatMap(Double value, Collector<Integer> out) {\n        Preconditions.checkArgument(\n                Double.compare(loss, Double.MAX_VALUE) == 0,\n                \"Each epoch should contain only one loss value.\");\n        loss = value;\n    }\n\n    @Override\n    public void onEpochWatermarkIncremented(\n            int epochWatermark, Context context, Collector<Integer> collector) {\n        if ((epochWatermark + 1) < maxIter && loss > tol) {\n            collector.collect(0);\n        }\n        loss = Double.MAX_VALUE;\n    }\n\n    @Override\n    public void onIterationTerminated(Context context, Collector<Integer> collector) {}\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/iteration/TerminateOnMaxIter.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.iteration;\n\nimport org.apache.flink.api.common.functions.FlatMapFunction;\nimport org.apache.flink.iteration.IterationListener;\nimport org.apache.flink.util.Collector;\n\n/**\n * A FlatMapFunction that emits values iff the iteration's epochWatermark does not exceed a certain\n * threshold.\n *\n * <p>When the output of this FlatMapFunction is used as the termination criteria of an iteration\n * body, the iteration will be executed for at most the given `maxIter` iterations.\n *\n * @param <T> The class type of the input element.\n */\npublic class TerminateOnMaxIter<T>\n        implements FlatMapFunction<T, Integer>, IterationListener<Integer> {\n\n    private final int maxIter;\n\n    public TerminateOnMaxIter(int maxIter) {\n        this.maxIter = maxIter;\n    }\n\n    @Override\n    public void flatMap(T integer, Collector<Integer> collector) {}\n\n    @Override\n    public void onEpochWatermarkIncremented(\n            int epochWatermark, Context context, Collector<Integer> out) {\n        if (epochWatermark + 1 < maxIter) {\n            out.collect(0);\n        }\n    }\n\n    @Override\n    public void onIterationTerminated(Context context, Collector<Integer> collector) {}\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/EndOfStreamWindows.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream;\n\nimport org.apache.flink.api.common.ExecutionConfig;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;\nimport org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;\nimport org.apache.flink.streaming.api.windowing.triggers.Trigger;\nimport org.apache.flink.streaming.api.windowing.windows.TimeWindow;\n\nimport java.util.Collection;\nimport java.util.Collections;\n\n/**\n * A {@link WindowAssigner} that assigns all elements of a bounded input stream into one window\n * pane. The results are emitted once the input stream has ended.\n */\npublic class EndOfStreamWindows extends WindowAssigner<Object, TimeWindow> {\n\n    private static final EndOfStreamWindows INSTANCE = new EndOfStreamWindows();\n\n    private static final TimeWindow TIME_WINDOW_INSTANCE =\n            new TimeWindow(Long.MIN_VALUE, Long.MAX_VALUE);\n\n    private EndOfStreamWindows() {}\n\n    public static EndOfStreamWindows get() {\n        return INSTANCE;\n    }\n\n    @Override\n    public Collection<TimeWindow> assignWindows(\n            Object element, long timestamp, WindowAssignerContext context) {\n        return Collections.singletonList(TIME_WINDOW_INSTANCE);\n    }\n\n    @Override\n    public Trigger<Object, TimeWindow> getDefaultTrigger(StreamExecutionEnvironment env) {\n        return EventTimeTrigger.create();\n    }\n\n    @Override\n    public String toString() {\n        return \"EndOfStreamWindows()\";\n    }\n\n    @Override\n    public TypeSerializer<TimeWindow> getWindowSerializer(ExecutionConfig executionConfig) {\n        return new TimeWindow.Serializer();\n    }\n\n    @Override\n    public boolean isEventTime() {\n        return true;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/AllReduceImpl.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream;\n\nimport org.apache.flink.annotation.Internal;\nimport org.apache.flink.annotation.VisibleForTesting;\nimport org.apache.flink.api.common.functions.RichFlatMapFunction;\nimport org.apache.flink.api.common.typeinfo.BasicTypeInfo;\nimport org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.api.java.tuple.Tuple3;\nimport org.apache.flink.api.java.tuple.Tuple4;\nimport org.apache.flink.api.java.typeutils.TupleTypeInfo;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.operators.AbstractStreamOperator;\nimport org.apache.flink.streaming.api.operators.BoundedOneInput;\nimport org.apache.flink.streaming.api.operators.OneInputStreamOperator;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.util.Collector;\n\nimport java.util.HashMap;\nimport java.util.Map;\n\n/**\n * Applies all-reduce on a data stream where each partition contains up to one double array.\n *\n * <p>AllReduce is a communication primitive widely used in MPI. In this implementation, all workers\n * do reduce on a partition of the whole data and they all get the final reduce result. In detail,\n * we split each double array into chunks of fixed size buffer (32KB by default) and let each\n * subtask handle several chunks.\n *\n * <p>There're mainly three stages:\n * <li>All workers send their partial data to other workers for reduce.\n * <li>All workers do reduce on all data it received and then broadcast partial results to others.\n * <li>All workers merge partial results into final result.\n */\n@Internal\nclass AllReduceImpl {\n\n    @VisibleForTesting static final int CHUNK_SIZE = 1024 * 4;\n\n    /**\n     * Applies allReduceSum on the input data stream. The input data stream is supposed to contain\n     * up to one double array in each worker. The result data stream has the same parallelism as the\n     * input, where each worker contains one double array that sums all of the double arrays in the\n     * input data stream.\n     *\n     * <p>We throw exception when one of the following two cases happen:\n     * <li>There exists one worker that contains more than one double array.\n     * <li>The length of double array is not consistent among all workers.\n     *\n     * @param input The input data stream.\n     * @return The result data stream.\n     */\n    static DataStream<double[]> allReduceSum(DataStream<double[]> input) {\n        // chunkId, originalArrayLength, arrayChunk\n        DataStream<Tuple3<Integer, Integer, double[]>> allReduceSend =\n                input.flatMap(new AllReduceSend())\n                        .setParallelism(input.getParallelism())\n                        .name(\"all-reduce-send\");\n\n        // taskId, chunkId, originalArrayLength, arrayChunk\n        DataStream<Tuple4<Integer, Integer, Integer, double[]>> allReduceSum =\n                allReduceSend\n                        .partitionCustom(\n                                (chunkId, numPartitions) -> chunkId % numPartitions, x -> x.f0)\n                        .transform(\n                                \"all-reduce-sum\",\n                                new TupleTypeInfo<>(\n                                        BasicTypeInfo.INT_TYPE_INFO,\n                                        BasicTypeInfo.INT_TYPE_INFO,\n                                        BasicTypeInfo.INT_TYPE_INFO,\n                                        PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO),\n                                new AllReduceSum())\n                        .setParallelism(input.getParallelism())\n                        .name(\"all-reduce-sum\");\n\n        return allReduceSum\n                .partitionCustom((taskIdx, numPartitions) -> taskIdx % numPartitions, x -> x.f0)\n                .transform(\n                        \"all-reduce-recv\",\n                        PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO,\n                        new AllReduceRecv())\n                .setParallelism(input.getParallelism())\n                .name(\"all-reduce-recv\");\n    }\n\n    /**\n     * Splits each double array into multiple chunks and sends each chunk to the corresponding\n     * worker.\n     */\n    private static class AllReduceSend\n            extends RichFlatMapFunction<double[], Tuple3<Integer, Integer, double[]>> {\n\n        private boolean hasReceivedOneRecord = false;\n\n        private double[] transferBuffer = new double[CHUNK_SIZE];\n\n        @Override\n        public void flatMap(\n                double[] inputArray, Collector<Tuple3<Integer, Integer, double[]>> out) {\n            if (hasReceivedOneRecord) {\n                throw new RuntimeException(\"The input cannot contain more than one double array.\");\n            }\n            hasReceivedOneRecord = true;\n            int numTasks = getRuntimeContext().getNumberOfParallelSubtasks();\n\n            for (int taskId = 0; taskId < numTasks; taskId++) {\n                int startChunkId = getStartChunkId(taskId, numTasks, inputArray.length);\n                int numChunksToHandle = getNumChunksByTaskId(taskId, numTasks, inputArray.length);\n                for (int chunkId = startChunkId;\n                        chunkId < numChunksToHandle + startChunkId;\n                        chunkId++) {\n                    System.arraycopy(\n                            inputArray,\n                            chunkId * CHUNK_SIZE,\n                            transferBuffer,\n                            0,\n                            getLengthOfChunk(chunkId, inputArray.length));\n                    out.collect(Tuple3.of(chunkId, inputArray.length, transferBuffer));\n                }\n            }\n        }\n    }\n\n    /**\n     * Aggregates partitioned array chunks from other workers and broadcast the aggregated array\n     * chunk to each worker.\n     */\n    private static class AllReduceSum\n            extends AbstractStreamOperator<Tuple4<Integer, Integer, Integer, double[]>>\n            implements OneInputStreamOperator<\n                            Tuple3<Integer, Integer, double[]>,\n                            Tuple4<Integer, Integer, Integer, double[]>>,\n                    BoundedOneInput {\n\n        /**\n         * A map that aggregates the received array chunks. The key is chunkId, the value is\n         * (originalArrayLength, aggregatedArrayChunk).\n         */\n        private Map<Integer, Tuple2<Integer, double[]>> aggregatedArrayChunkByChunkId =\n                new HashMap<>();\n\n        @Override\n        public void endInput() {\n            int numTasks = getRuntimeContext().getNumberOfParallelSubtasks();\n            for (Map.Entry<Integer, Tuple2<Integer, double[]>> entry :\n                    aggregatedArrayChunkByChunkId.entrySet()) {\n                for (int taskId = 0; taskId < numTasks; taskId++) {\n                    int chunkId = entry.getKey();\n                    int originalArrayLength = entry.getValue().f0;\n                    double[] aggregatedArrayChunk = entry.getValue().f1;\n                    output.collect(\n                            new StreamRecord<>(\n                                    Tuple4.of(\n                                            taskId,\n                                            chunkId,\n                                            originalArrayLength,\n                                            aggregatedArrayChunk)));\n                }\n            }\n        }\n\n        @Override\n        public void processElement(StreamRecord<Tuple3<Integer, Integer, double[]>> streamRecord) {\n            Tuple3<Integer, Integer, double[]> record = streamRecord.getValue();\n            int chunkId = record.f0;\n            int originalArrayLength = record.f1;\n            double[] arrayChunk = record.f2;\n            if (aggregatedArrayChunkByChunkId.containsKey(chunkId)) {\n                if (aggregatedArrayChunkByChunkId.get(chunkId).f0 != originalArrayLength) {\n                    throw new RuntimeException(\"The input double array must have same length.\");\n                }\n                double[] curAggregatedArrayChunk = aggregatedArrayChunkByChunkId.get(chunkId).f1;\n                for (int i = 0; i < curAggregatedArrayChunk.length; i++) {\n                    curAggregatedArrayChunk[i] += arrayChunk[i];\n                }\n            } else {\n                aggregatedArrayChunkByChunkId.put(\n                        chunkId, Tuple2.of(originalArrayLength, arrayChunk));\n            }\n        }\n    }\n\n    /** Organizes the received chunks into the result array. */\n    private static class AllReduceRecv extends AbstractStreamOperator<double[]>\n            implements OneInputStreamOperator<\n                            Tuple4<Integer, Integer, Integer, double[]>, double[]>,\n                    BoundedOneInput {\n\n        /** Stores the reduced results. */\n        double[] resultArray;\n\n        @Override\n        public void endInput() {\n            if (null != resultArray) {\n                output.collect(new StreamRecord<>(resultArray));\n            }\n        }\n\n        @Override\n        public void processElement(\n                StreamRecord<Tuple4<Integer, Integer, Integer, double[]>> streamRecord) {\n            Tuple4<Integer, Integer, Integer, double[]> ele = streamRecord.getValue();\n            int chunkId = ele.f1;\n            int originalArrayLength = ele.f2;\n            double[] aggregatedArrayChunk = ele.f3;\n            if (null == resultArray) {\n                resultArray = new double[originalArrayLength];\n            }\n            System.arraycopy(\n                    aggregatedArrayChunk,\n                    0,\n                    resultArray,\n                    chunkId * CHUNK_SIZE,\n                    getLengthOfChunk(chunkId, resultArray.length));\n        }\n    }\n\n    /**\n     * Computes how many chunks is an array with length ${len} going to be split into.\n     *\n     * @param len Length of the array.\n     * @return Number of chunks the array is split into.\n     */\n    \nprivate static int getNumChunks(int len) {\n        int div = len / CHUNK_SIZE;\n        int mod = len % CHUNK_SIZE;\n        return mod == 0 ? div : div + 1;\n    }\n\n    /**\n     * Computes the length of the last chunk of an array with length ${len}.\n     *\n     * @param len Length of the array.\n     * @return Length of the last chunk.\n     */\n    private static int getLengthOfChunk(int chunkId, int len) {\n        if (chunkId == getNumChunks(len) - 1) {\n            int mod = len % CHUNK_SIZE;\n            return mod == 0 ? CHUNK_SIZE : mod;\n        } else {\n            return CHUNK_SIZE;\n        }\n    }\n\n    /**\n     * Computes the index of the first chunk that one task needs to handle.\n     *\n     * @param taskId Index of the current task.\n     * @param numTasks Number of parallel tasks.\n     * @param len Length of the array to be reduced.\n     * @return Start position of this task.\n     */\n    private static int getStartChunkId(int taskId, int numTasks, int len) {\n        int numChunks = getNumChunks(len);\n        int div = numChunks / numTasks;\n        int mod = numChunks % numTasks;\n\n        if (taskId >= mod) {\n            return div * taskId + mod;\n        } else {\n            return div * taskId + taskId;\n        }\n    }\n\n    /**\n     * Computes the number of chunks that one task needs to handle.\n     *\n     * @param taskId Index of the current task.\n     * @param parallelism Number of parallel tasks.\n     * @param len Length of the array to be reduced.\n     * @return Number of chunks this task needs to handle.\n     */\n    private static int getNumChunksByTaskId(int taskId, int parallelism, int len) {\n        int numChunks = getNumChunks(len);\n        int div = numChunks / parallelism;\n        int mod = numChunks % parallelism;\n\n        if (taskId >= mod) {\n            return div;\n        } else {\n            return div + 1;\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/CoGroupOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.ExecutionConfig;\nimport org.apache.flink.api.common.functions.CoGroupFunction;\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.common.typeutils.TypePairComparator;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.java.functions.KeySelector;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.api.java.typeutils.runtime.RuntimePairComparatorFactory;\nimport org.apache.flink.configuration.AlgorithmOptions;\nimport org.apache.flink.configuration.Configuration;\nimport org.apache.flink.core.memory.DataOutputSerializer;\nimport org.apache.flink.core.memory.ManagedMemoryUseCase;\nimport org.apache.flink.runtime.io.disk.iomanager.IOManager;\nimport org.apache.flink.runtime.memory.MemoryAllocationException;\nimport org.apache.flink.runtime.memory.MemoryManager;\nimport org.apache.flink.runtime.operators.sort.ExternalSorter;\nimport org.apache.flink.runtime.operators.sort.NonReusingSortMergeCoGroupIterator;\nimport org.apache.flink.runtime.operators.sort.PushSorter;\nimport org.apache.flink.runtime.operators.sort.ReusingSortMergeCoGroupIterator;\nimport org.apache.flink.runtime.operators.util.CoGroupTaskIterator;\nimport org.apache.flink.streaming.api.graph.StreamConfig;\nimport org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;\nimport org.apache.flink.streaming.api.operators.BoundedMultiInput;\nimport org.apache.flink.streaming.api.operators.Output;\nimport org.apache.flink.streaming.api.operators.TimestampedCollector;\nimport org.apache.flink.streaming.api.operators.TwoInputStreamOperator;\nimport org.apache.flink.streaming.api.watermark.Watermark;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\nimport org.apache.flink.streaming.runtime.tasks.StreamTask;\nimport org.apache.flink.util.MutableObjectIterator;\nimport org.apache.flink.util.TraversableOnceException;\n\nimport java.io.Serializable;\nimport java.util.Iterator;\n\n/**\n * An operator that implements the co-group logic.\n *\n * @param <IN1> The class type of the first input.\n * @param <IN2> The class type of the second input.\n * @param <KEY> The class type of the key.\n * @param <OUT> The class type of the output values.\n */\npublic class CoGroupOperator<IN1, IN2, KEY extends Serializable, OUT>\n        extends AbstractUdfStreamOperator<OUT, CoGroupFunction<IN1, IN2, OUT>>\n        implements TwoInputStreamOperator<IN1, IN2, OUT>, BoundedMultiInput {\n\n    private PushSorter<Tuple2<byte[], StreamRecord<IN1>>> sorterA;\n    private PushSorter<Tuple2<byte[], StreamRecord<IN2>>> sorterB;\n    private TypeComparator<Tuple2<byte[], StreamRecord<IN1>>> comparatorA;\n    private TypeComparator<Tuple2<byte[], StreamRecord<IN2>>> comparatorB;\n    private KeySelector<IN1, KEY> keySelectorA;\n    private KeySelector<IN2, KEY> keySelectorB;\n    private TypeSerializer<Tuple2<byte[], StreamRecord<IN1>>> keyAndValueSerializerA;\n    private TypeSerializer<Tuple2<byte[], StreamRecord<IN2>>> keyAndValueSerializerB;\n    private TypeSerializer<KEY> keySerializer;\n    private DataOutputSerializer dataOutputSerializer;\n    private long lastWatermarkTimestamp = Long.MIN_VALUE;\n    private int remainingInputNum = 2;\n\n    public CoGroupOperator(CoGroupFunction<IN1, IN2, OUT> function) {\n        super(function);\n    }\n\n    @Override\n    public void setup(\n            StreamTask<?, ?> containingTask,\n            StreamConfig config,\n            Output<StreamRecord<OUT>> output) {\n        super.setup(containingTask, config, output);\n        ClassLoader userCodeClassLoader = containingTask.getUserCodeClassLoader();\n        MemoryManager memoryManager = containingTask.getEnvironment().getMemoryManager();\n        IOManager ioManager = containingTask.getEnvironment().getIOManager();\n\n        keySelectorA = config.getStatePartitioner(0, userCodeClassLoader);\n        keySelectorB = config.getStatePartitioner(1, userCodeClassLoader);\n        keySerializer = config.getStateKeySerializer(userCodeClassLoader);\n        int keyLength = keySerializer.getLength();\n\n        TypeSerializer<IN1> typeSerializerA = config.getTypeSerializerIn(0, userCodeClassLoader);\n        TypeSerializer<IN2> typeSerializerB = config.getTypeSerializerIn(1, userCodeClassLoader);\n        keyAndValueSerializerA = new KeyAndValueSerializer<>(typeSerializerA, keyLength);\n        keyAndValueSerializerB = new KeyAndValueSerializer<>(typeSerializerB, keyLength);\n\n        if (keyLength > 0) {\n            dataOutputSerializer = new DataOutputSerializer(keyLength);\n            comparatorA = new FixedLengthByteKeyComparator<>(keyLength);\n            comparatorB = new FixedLengthByteKeyComparator<>(keyLength);\n        } else {\n            dataOutputSerializer = new DataOutputSerializer(64);\n            comparatorA = new VariableLengthByteKeyComparator<>();\n            comparatorB = new VariableLengthByteKeyComparator<>();\n        }\n\n        ExecutionConfig executionConfig = containingTask.getEnvironment().getExecutionConfig();\n        double managedMemoryFraction =\n                config.getManagedMemoryFractionOperatorUseCaseOfSlot(\n                                ManagedMemoryUseCase.OPERATOR,\n                                containingTask.getEnvironment().getTaskConfiguration(),\n                                userCodeClassLoader)\n                        / 2;\n        Configuration jobConfiguration = containingTask.getEnvironment().getJobConfiguration();\n\n        try {\n            sorterA =\n                    ExternalSorter.newBuilder(\n                                    memoryManager,\n                                    containingTask,\n                                    keyAndValueSerializerA,\n                                    comparatorA,\n                                    executionConfig)\n                            .memoryFraction(managedMemoryFraction)\n                            .enableSpilling(\n                                    ioManager,\n...\n", "needle_token_start": 13455, "needle_token_end": 13511, "code_context_ntokens": 15898, "output": ["```java\nprivate static int getNumChunks(int len) {\n    int div = len / CHUNK_SIZE;\n    int mod = len % CHUNK_SIZE;\n    return mod == 0 ? div : div + 1;\n}\n```"]}
{"repo": "apache/flink-ml", "name": "getCurrentOuterSnapshotVersion", "language": "java", "path": "flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElementSerializer.java", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide the current version number of the serializer snapshot, which is used for versioning the serialization format.\n2. **Input**: There are no inputs required for this function.\n3. **Output**: Returns an integer representing the version number of the serialization snapshot.\n4. **Procedure**: The function simply returns a predefined constant that represents the current version of the serializer's configuration snapshot.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/KeyAndValueSerializer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\nimport java.util.Objects;\n\n/**\n * A serializer used in {@link SortingDataInput} for serializing elements alongside their key and\n * timestamp. It serializes the record in a format known by the {@link FixedLengthByteKeyComparator}\n * and {@link VariableLengthByteKeyComparator}.\n *\n * <p>If the key is of known constant length, the length is not serialized with the data. Therefore\n * the serialized data is as follows:\n *\n * <pre>\n *      [key-length] | &lt;key&gt; | &lt;timestamp&gt; | &lt;record&gt;\n * </pre>\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class KeyAndValueSerializer<IN>\n        extends TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> {\n    private static final int TIMESTAMP_LENGTH = 8;\n    private final TypeSerializer<IN> valueSerializer;\n\n    // This represents either a variable length (-1) or a fixed one (>= 0).\n    private final int serializedKeyLength;\n\n    public KeyAndValueSerializer(TypeSerializer<IN> valueSerializer, int serializedKeyLength) {\n        this.valueSerializer = valueSerializer;\n        this.serializedKeyLength = serializedKeyLength;\n    }\n\n    @Override\n    public boolean isImmutableType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new KeyAndValueSerializer<>(valueSerializer.duplicate(), this.serializedKeyLength);\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> copy(Tuple2<byte[], StreamRecord<IN>> from) {\n        StreamRecord<IN> fromRecord = from.f1;\n        return Tuple2.of(\n                Arrays.copyOf(from.f0, from.f0.length),\n                fromRecord.copy(valueSerializer.copy(fromRecord.getValue())));\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> createInstance() {\n        return Tuple2.of(new byte[0], new StreamRecord<>(valueSerializer.createInstance()));\n    }\n\n    @Override\n...\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/FixedLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps. It uses\n * binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are always of a fixed length and thus the length of the record is not\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class FixedLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    static final int TIMESTAMP_BYTE_SIZE = 8;\n    private final int keyLength;\n    private byte[] keyReference;\n    private long timestampReference;\n\n    public FixedLengthByteKeyComparator(int keyLength) {\n        this.keyLength = keyLength;\n    }\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = toCompare.f0;\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((FixedLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((FixedLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        for (int i = 0; i < keyLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return 0;\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int minCount = keyLength;\n        while (minCount-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp < 0 ? -1 : 1;\n            }\n        }\n\n        return Long.compare(firstSource.readLong(), secondSource.readLong());\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return keyLength + TIMESTAMP_BYTE_SIZE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return keyBytes < getNormalizeKeyLen();\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(record, keyLength, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new FixedLengthByteKeyComparator<>(this.keyLength);\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/BytesKeyNormalizationUtil.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\n/**\n * Utility class for common key normalization used both in {@link VariableLengthByteKeyComparator}\n * and {@link FixedLengthByteKeyComparator}.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\nfinal class BytesKeyNormalizationUtil {\n    /**\n     * Writes the normalized key of given record. The normalized key consists of the key serialized\n     * as bytes and the timestamp of the record.\n     *\n     * <p>NOTE: The key does not represent a logical order. It can be used only for grouping keys!\n     */\n    static <IN> void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            int dataLength,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        byte[] data = record.f0;\n\n        if (dataLength >= numBytes) {\n            putBytesArray(target, offset, numBytes, data);\n        } else {\n            // whole key fits into the normalized key\n            putBytesArray(target, offset, dataLength, data);\n            int lastOffset = offset + numBytes;\n            offset += dataLength;\n            long valueOfTimestamp = record.f1.asRecord().getTimestamp() - Long.MIN_VALUE;\n            if (dataLength + FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE <= numBytes) {\n                // whole timestamp fits into the normalized key\n                target.putLong(offset, valueOfTimestamp);\n                offset += FixedLengthByteKeyComparator.TIMESTAMP_BYTE_SIZE;\n                // fill in the remaining space with zeros\n                while (offset < lastOffset) {\n                    target.put(offset++, (byte) 0);\n                }\n            } else {\n                // only part of the timestamp fits into normalized key\n                for (int i = 0; offset < lastOffset; offset++, i++) {\n                    target.put(offset, (byte) (valueOfTimestamp >>> ((7 - i) << 3)));\n                }\n            }\n        }\n    }\n\n    private static void putBytesArray(MemorySegment target, int offset, int numBytes, byte[] data) {\n        for (int i = 0; i < numBytes; i++) {\n            // We're converting the signed byte in data into an unsigned representation.\n            // A Java byte goes from -128 to 127, i.e. is signed. By subtracting -128 (MIN_VALUE)\n            // here we're shifting the number to be from 0 to 255. The normalized key sorter sorts\n            // bytes as \"unsigned\", so we need to convert here to maintain a correct ordering.\n            int highByte = data[i] & 0xff;\n            highByte -= Byte.MIN_VALUE;\n            target.put(offset + i, (byte) highByte);\n        }\n    }\n\n    private BytesKeyNormalizationUtil() {}\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/sort/VariableLengthByteKeyComparator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.datastream.sort;\n\nimport org.apache.flink.api.common.typeutils.TypeComparator;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.core.memory.MemorySegment;\nimport org.apache.flink.streaming.api.operators.sort.SortingDataInput;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport java.io.IOException;\nimport java.util.Arrays;\n\n/**\n * A comparator used in {@link SortingDataInput} which compares records keys and timestamps,. It\n * uses binary format produced by the {@link KeyAndValueSerializer}.\n *\n * <p>It assumes keys are of a variable length and thus expects the length of the record to be\n * serialized.\n *\n * <p>TODO: remove this class after making the corresponding class in Flink public.\n */\npublic final class VariableLengthByteKeyComparator<IN>\n        extends TypeComparator<Tuple2<byte[], StreamRecord<IN>>> {\n    private byte[] keyReference;\n    private long timestampReference;\n\n    @Override\n    public int hash(Tuple2<byte[], StreamRecord<IN>> record) {\n        return record.hashCode();\n    }\n\n    @Override\n    public void setReference(Tuple2<byte[], StreamRecord<IN>> toCompare) {\n        this.keyReference = Arrays.copyOf(toCompare.f0, toCompare.f0.length);\n        this.timestampReference = toCompare.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public boolean equalToReference(Tuple2<byte[], StreamRecord<IN>> candidate) {\n        return Arrays.equals(keyReference, candidate.f0)\n                && timestampReference == candidate.f1.asRecord().getTimestamp();\n    }\n\n    @Override\n    public int compareToReference(\n            TypeComparator<Tuple2<byte[], StreamRecord<IN>>> referencedComparator) {\n        byte[] otherKey = ((VariableLengthByteKeyComparator<IN>) referencedComparator).keyReference;\n        long otherTimestamp =\n                ((VariableLengthByteKeyComparator<IN>) referencedComparator).timestampReference;\n\n        int keyCmp = compare(otherKey, this.keyReference);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(otherTimestamp, this.timestampReference);\n    }\n\n    @Override\n    public int compare(\n            Tuple2<byte[], StreamRecord<IN>> first, Tuple2<byte[], StreamRecord<IN>> second) {\n        int keyCmp = compare(first.f0, second.f0);\n        if (keyCmp != 0) {\n            return keyCmp;\n        }\n        return Long.compare(\n                first.f1.asRecord().getTimestamp(), second.f1.asRecord().getTimestamp());\n    }\n\n    private int compare(byte[] first, byte[] second) {\n        int firstLength = first.length;\n        int secondLength = second.length;\n        int minLength = Math.min(firstLength, secondLength);\n        for (int i = 0; i < minLength; i++) {\n            int cmp = Byte.compare(first[i], second[i]);\n\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        return Integer.compare(firstLength, secondLength);\n    }\n\n    @Override\n    public int compareSerialized(DataInputView firstSource, DataInputView secondSource)\n            throws IOException {\n        int firstLength = firstSource.readInt();\n        int secondLength = secondSource.readInt();\n        int minLength = Math.min(firstLength, secondLength);\n        while (minLength-- > 0) {\n            byte firstValue = firstSource.readByte();\n            byte secondValue = secondSource.readByte();\n\n            int cmp = Byte.compare(firstValue, secondValue);\n            if (cmp != 0) {\n                return cmp;\n            }\n        }\n\n        int lengthCompare = Integer.compare(firstLength, secondLength);\n        if (lengthCompare != 0) {\n            return lengthCompare;\n        } else {\n            return Long.compare(firstSource.readLong(), secondSource.readLong());\n        }\n    }\n\n    @Override\n    public boolean supportsNormalizedKey() {\n        return true;\n    }\n\n    @Override\n    public int getNormalizeKeyLen() {\n        return Integer.MAX_VALUE;\n    }\n\n    @Override\n    public boolean isNormalizedKeyPrefixOnly(int keyBytes) {\n        return true;\n    }\n\n    @Override\n    public void putNormalizedKey(\n            Tuple2<byte[], StreamRecord<IN>> record,\n            MemorySegment target,\n            int offset,\n            int numBytes) {\n        BytesKeyNormalizationUtil.putNormalizedKey(\n                record, record.f0.length, target, offset, numBytes);\n    }\n\n    @Override\n    public boolean invertNormalizedKey() {\n        return false;\n    }\n\n    @Override\n    public TypeComparator<Tuple2<byte[], StreamRecord<IN>>> duplicate() {\n        return new VariableLengthByteKeyComparator<>();\n    }\n\n    @Override\n    public int extractKeys(Object record, Object[] target, int index) {\n        target[index] = record;\n        return 1;\n    }\n\n    @Override\n    public TypeComparator<?>[] getFlatComparators() {\n        return new TypeComparator[] {this};\n    }\n\n    // --------------------------------------------------------------------------------------------\n    // unsupported normalization\n    // --------------------------------------------------------------------------------------------\n\n    @Override\n    public boolean supportsSerializationWithKeyNormalization() {\n        return false;\n    }\n\n    @Override\n    public void writeWithKeyNormalization(\n            Tuple2<byte[], StreamRecord<IN>> record, DataOutputView target) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n\n    @Override\n    public Tuple2<byte[], StreamRecord<IN>> readWithKeyDenormalization(\n            Tuple2<byte[], StreamRecord<IN>> reuse, DataInputView source) throws IOException {\n        throw new UnsupportedOperationException();\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastContext.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.annotation.VisibleForTesting;\nimport org.apache.flink.api.common.operators.MailboxExecutor;\nimport org.apache.flink.api.java.tuple.Tuple2;\n\nimport javax.annotation.Nullable;\n\nimport java.util.List;\nimport java.util.concurrent.ConcurrentHashMap;\n\n/**\n * Context to hold the broadcast variables and provides some utility function for accessing\n * broadcast variables.\n */\npublic class BroadcastContext {\n\n    /**\n     * stores broadcast data streams in a map. The key is broadcastName-partitionId and the value is\n     * {@link BroadcastItem}.\n     */\n    private static final ConcurrentHashMap<String, BroadcastItem> BROADCAST_VARIABLES =\n            new ConcurrentHashMap<>();\n\n    @VisibleForTesting\n    public static void putBroadcastVariable(String key, Tuple2<Boolean, List<?>> variable) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(variable.f0, variable.f1, null)\n                                : new BroadcastItem(variable.f0, variable.f1, v.mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    public static void putMailBoxExecutor(String key, MailboxExecutor mailboxExecutor) {\n        BROADCAST_VARIABLES.compute(\n                key,\n                (k, v) ->\n                        null == v\n                                ? new BroadcastItem(false, null, mailboxExecutor)\n                                : new BroadcastItem(v.cacheReady, v.cacheList, mailboxExecutor));\n    }\n\n    @VisibleForTesting\n    @SuppressWarnings({\"unchecked\"})\n    public static <T> List<T> getBroadcastVariable(String key) {\n        return (List<T>) BROADCAST_VARIABLES.get(key).cacheList;\n    }\n\n    @VisibleForTesting\n    public static void remove(String key) {\n        BROADCAST_VARIABLES.remove(key);\n    }\n\n    @VisibleForTesting\n    public static void notifyCacheFinished(String key) {\n        BROADCAST_VARIABLES.computeIfPresent(\n                key,\n                (k, v) -> {\n                    // sends a dummy email to avoid possible stuck.\n                    if (null != v.mailboxExecutor) {\n                        v.mailboxExecutor.execute(() -> {}, \"empty mail\");\n                    }\n                    return new BroadcastItem(true, v.cacheList, v.mailboxExecutor);\n                });\n    }\n\n    @VisibleForTesting\n    public static boolean isCacheFinished(String key) {\n        return BROADCAST_VARIABLES.get(key).cacheReady;\n    }\n\n    /** Utility class to organize broadcast variables. */\n    private static class BroadcastItem {\n\n        /** whether this broadcast variable is ready to be consumed. */\n        private boolean cacheReady;\n\n        /** the cached list. */\n        private List<?> cacheList;\n\n        /** the mailboxExecutor of the consumer, used to avoid the possible stuck of consumer. */\n        private MailboxExecutor mailboxExecutor;\n\n        BroadcastItem(\n                boolean cacheReady,\n                @Nullable List<?> cacheList,\n                @Nullable MailboxExecutor mailboxExecutor) {\n            this.cacheReady = cacheReady;\n            this.cacheList = cacheList;\n            this.mailboxExecutor = mailboxExecutor;\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastVariableReceiverOperatorFactory.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\n\nimport java.io.Serializable;\n\n/** Factory class for {@link BroadcastVariableReceiverOperator}. */\npublic class BroadcastVariableReceiverOperatorFactory<OUT>\n        extends AbstractStreamOperatorFactory<OUT> implements Serializable {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastNames;\n\n    /** Types of the broadcast data streams. */\n    private final TypeInformation<?>[] inTypes;\n\n    public BroadcastVariableReceiverOperatorFactory(\n            String[] broadcastNames, TypeInformation<?>[] inTypes) {\n        this.broadcastNames = broadcastNames;\n        this.inTypes = inTypes;\n    }\n\n    @Override\n    public <T extends StreamOperator<OUT>> T createStreamOperator(\n            StreamOperatorParameters<OUT> parameters) {\n        return (T) new BroadcastVariableReceiverOperator(parameters, broadcastNames, inTypes);\n    }\n\n    @Override\n    public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {\n        return BroadcastVariableReceiverOperator.class;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastWrapper.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.functions.KeySelector;\nimport org.apache.flink.iteration.operator.OperatorWrapper;\nimport org.apache.flink.streaming.api.operators.OneInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorFactory;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.api.operators.TwoInputStreamOperator;\nimport org.apache.flink.streaming.runtime.partitioner.StreamPartitioner;\nimport org.apache.flink.util.OutputTag;\n\n/** The operator wrapper for {@link AbstractBroadcastWrapperOperator}. */\npublic class BroadcastWrapper<T> implements OperatorWrapper<T, T> {\n\n    /** Names of the broadcast data streams. */\n    private final String[] broadcastStreamNames;\n\n    public BroadcastWrapper(String[] broadcastStreamNames) {\n        this.broadcastStreamNames = broadcastStreamNames;\n    }\n\n    @Override\n    public StreamOperator<T> wrap(\n            StreamOperatorParameters<T> operatorParameters,\n            StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new OneInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return new TwoInputBroadcastWrapperOperator<>(\n                    operatorParameters, operatorFactory, broadcastStreamNames);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"Unsupported operator class for with-broadcast wrapper: \" + operatorClass);\n        }\n    }\n\n    @Override\n    public Class<? extends StreamOperator> getStreamOperatorClass(\n            ClassLoader classLoader, StreamOperatorFactory<T> operatorFactory) {\n        Class<? extends StreamOperator> operatorClass =\n                operatorFactory.getStreamOperatorClass(getClass().getClassLoader());\n        if (OneInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return OneInputBroadcastWrapperOperator.class;\n        } else if (TwoInputStreamOperator.class.isAssignableFrom(operatorClass)) {\n            return TwoInputBroadcastWrapperOperator.class;\n        } else {\n            throw new UnsupportedOperationException(\n                    \"Unsupported operator class for with-broadcast wrapper: \" + operatorClass);\n        }\n    }\n\n    @Override\n    public <KEY> KeySelector<T, KEY> wrapKeySelector(KeySelector<T, KEY> keySelector) {\n        return keySelector;\n    }\n\n    @Override\n    public StreamPartitioner<T> wrapStreamPartitioner(StreamPartitioner<T> streamPartitioner) {\n        return streamPartitioner;\n    }\n\n    @Override\n    public OutputTag<T> wrapOutputTag(OutputTag<T> outputTag) {\n        return outputTag;\n    }\n\n    @Override\n    public TypeInformation<T> getWrappedTypeInfo(TypeInformation<T> typeInfo) {\n        return typeInfo;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastUtils.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.iteration.compile.DraftExecutionEnvironment;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastVariableReceiverOperatorFactory;\nimport org.apache.flink.ml.common.broadcast.operator.BroadcastWrapper;\nimport org.apache.flink.streaming.api.datastream.DataStream;\nimport org.apache.flink.streaming.api.datastream.MultipleConnectedStreams;\nimport org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;\nimport org.apache.flink.streaming.api.operators.ChainingStrategy;\nimport org.apache.flink.streaming.api.transformations.MultipleInputTransformation;\nimport org.apache.flink.streaming.api.transformations.PhysicalTransformation;\nimport org.apache.flink.util.AbstractID;\nimport org.apache.flink.util.Preconditions;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\nimport java.util.UUID;\nimport java.util.function.Function;\n\n/** Utility class to support withBroadcast in DataStream. */\npublic class BroadcastUtils {\n    /**\n     * Supports withBroadcastStream in DataStream API. Broadcast data streams are available at all\n     * parallel instances of an operator that extends {@code\n     * org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator<OUT, ? extends\n     * org.apache.flink.api.common.functions.RichFunction>}. Users can access the broadcast\n     * variables by {@code RichFunction.getRuntimeContext().getBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().hasBroadcastVariable(...)} or {@code\n     * RichFunction.getRuntimeContext().getBroadcastVariableWithInitializer(...)}.\n     *\n     * <p>In detail, the broadcast input data streams will be consumed first and further consumed by\n     * non-broadcast inputs. For now the non-broadcast input are cached by default to avoid the\n     * possible deadlocks.\n     *\n     * @param inputList non-broadcast input list.\n     * @param bcStreams map of the broadcast data streams, where the key is the name and the value\n     *     is the corresponding data stream.\n     * @param userDefinedFunction the user defined logic in which users can access the broadcast\n     *     data streams and produce the output data stream. Note that though users can add more than\n     *     one operator in this logic, but only the operator that generates the result stream can\n     *     contain a rich function and access the broadcast variables. Other operators will\n     *     encounter NPE when accessing the broadcast variables.\n     * @return the output data stream.\n     */\n    public static <OUT> DataStream<OUT> withBroadcastStream(\n            List<DataStream<?>> inputList,\n            Map<String, DataStream<?>> bcStreams,\n            Function<List<DataStream<?>>, DataStream<OUT>> userDefinedFunction) {\n        Preconditions.checkArgument(inputList.size() > 0);\n\n        StreamExecutionEnvironment env = inputList.get(0).getExecutionEnvironment();\n        String[] broadcastNames = new String[bcStreams.size()];\n        DataStream<?>[] broadcastInputs = new DataStream[bcStreams.size()];\n        TypeInformation<?>[] broadcastInTypes = new TypeInformation[bcStreams.size()];\n        int idx = 0;\n        final String broadcastId = new AbstractID().toHexString();\n        for (String name : bcStreams.keySet()) {\n            broadcastNames[idx] = broadcastId + \"-\" + name;\n            broadcastInputs[idx] = bcStreams.get(name);\n            broadcastInTypes[idx] = broadcastInputs[idx].getType();\n            idx++;\n        }\n\n        DataStream<OUT> resultStream =\n                getResultStream(env, inputList, broadcastNames, userDefinedFunction);\n        TypeInformation<OUT> outType = resultStream.getType();\n        final String coLocationKey = \"broadcast-co-location-\" + UUID.randomUUID();\n        DataStream<OUT> cachedBroadcastInputs =\n                cacheBroadcastVariables(\n                        env,\n                        broadcastNames,\n                        broadcastInputs,\n                        broadcastInTypes,\n                        resultStream.getParallelism(),\n                        outType);\n\n        boolean canCoLocate =\n                cachedBroadcastInputs.getTransformation() instanceof PhysicalTransformation\n                        && resultStream.getTransformation() instanceof PhysicalTransformation;\n        if (canCoLocate) {\n            ((PhysicalTransformation<?>) cachedBroadcastInputs.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n            ((PhysicalTransformation<?>) resultStream.getTransformation())\n                    .setChainingStrategy(ChainingStrategy.HEAD);\n        } else {\n            throw new UnsupportedOperationException(\n                    \"cannot set chaining strategy on \"\n                            + cachedBroadcastInputs.getTransformation()\n                            + \" and \"\n                            + resultStream.getTransformation()\n                            + \".\");\n        }\n        cachedBroadcastInputs.getTransformation().setCoLocationGroupKey(coLocationKey);\n        resultStream.getTransformation().setCoLocationGroupKey(coLocationKey);\n\n        return cachedBroadcastInputs.union(resultStream);\n    }\n\n    /**\n     * Caches all broadcast input data streams in static variables and returns the result\n     * multi-input stream operator. The result multi-input stream operator emits nothing and the\n     * only functionality of this operator is to cache all the input records in ${@link\n     * BroadcastContext}.\n     *\n     * @param env execution environment.\n     * @param broadcastInputNames names of the broadcast input data streams.\n     * @param broadcastInputs list of the broadcast data streams.\n     * @param broadcastInTypes output types of the broadcast input data streams.\n     * @param parallelism parallelism.\n     * @param outType output type.\n     * @param <OUT> output type.\n     * @return the result multi-input stream operator.\n     */\n    private static <OUT> DataStream<OUT> cacheBroadcastVariables(\n            StreamExecutionEnvironment env,\n            String[] broadcastInputNames,\n            DataStream<?>[] broadcastInputs,\n            TypeInformation<?>[] broadcastInTypes,\n            int parallelism,\n            TypeInformation<OUT> outType) {\n        MultipleInputTransformation<OUT> transformation =\n                new MultipleInputTransformation<>(\n                        \"broadcastInputs\",\n                        new BroadcastVariableReceiverOperatorFactory<>(\n                                broadcastInputNames, broadcastInTypes),\n                        outType,\n                        parallelism);\n        for (DataStream<?> dataStream : broadcastInputs) {\n            transformation.addInput(dataStream.broadcast().getTransformation());\n        }\n        env.addOperator(transformation);\n        return new MultipleConnectedStreams(env).transform(transformation);\n    }\n\n    /**\n     * Uses {@link DraftExecutionEnvironment} to execute the userDefinedFunction and returns the\n     * resultStream.\n     *\n     * @param env execution environment.\n     * @param inputList non-broadcast input list.\n     * @param broadcastStreamNames names of the broadcast data streams.\n     * @param graphBuilder user-defined logic.\n     * @param <OUT> output type of the result stream.\n     * @return the result stream by applying user-defined logic on the input list.\n     */\n    private static <OUT> DataStream<OUT> getResultStream(\n            StreamExecutionEnvironment env,\n            List<DataStream<?>> inputList,\n            String[] broadcastStreamNames,\n            Function<List<DataStream<?>>, DataStream<OUT>> graphBuilder) {\n        DraftExecutionEnvironment draftEnv =\n                new DraftExecutionEnvironment(env, new BroadcastWrapper<>(broadcastStreamNames));\n        List<DataStream<?>> draftSources = new ArrayList<>();\n        for (DataStream<?> dataStream : inputList) {\n            draftSources.add(draftEnv.addDraftSource(dataStream, dataStream.getType()));\n        }\n        DataStream<OUT> draftOutStream = graphBuilder.apply(draftSources);\n        draftEnv.copyToActualEnvironment();\n        return draftEnv.getActualStream(draftOutStream.getId());\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/BroadcastStreamingRuntimeContext.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast;\n\nimport org.apache.flink.annotation.Internal;\nimport org.apache.flink.api.common.accumulators.Accumulator;\nimport org.apache.flink.api.common.functions.BroadcastVariableInitializer;\nimport org.apache.flink.api.common.state.KeyedStateStore;\nimport org.apache.flink.metrics.groups.OperatorMetricGroup;\nimport org.apache.flink.runtime.execution.Environment;\nimport org.apache.flink.runtime.externalresource.ExternalResourceInfoProvider;\nimport org.apache.flink.runtime.jobgraph.OperatorID;\nimport org.apache.flink.streaming.api.operators.StreamingRuntimeContext;\nimport org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;\n\nimport javax.annotation.Nullable;\n\nimport java.util.HashMap;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * An subclass of {@link StreamingRuntimeContext} that provides accessibility of broadcast\n * variables.\n */\npublic class BroadcastStreamingRuntimeContext extends StreamingRuntimeContext {\n\n    Map<String, List<?>> broadcastVariables = new HashMap<>();\n\n    public BroadcastStreamingRuntimeContext(\n            Environment env,\n            Map<String, Accumulator<?, ?>> accumulators,\n            OperatorMetricGroup operatorMetricGroup,\n            OperatorID operatorID,\n            ProcessingTimeService processingTimeService,\n            @Nullable KeyedStateStore keyedStateStore,\n            ExternalResourceInfoProvider externalResourceInfoProvider) {\n        super(\n                env,\n                accumulators,\n                operatorMetricGroup,\n                operatorID,\n                processingTimeService,\n                keyedStateStore,\n                externalResourceInfoProvider);\n    }\n\n    @Override\n    public boolean hasBroadcastVariable(String name) {\n        return broadcastVariables.containsKey(name);\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked\")\n    public <RT> List<RT> getBroadcastVariable(String name) {\n        if (broadcastVariables.containsKey(name)) {\n            return (List<RT>) broadcastVariables.get(name);\n        } else {\n            throw new RuntimeException(\n                    \"Cannot get broadcast variables before processing elements.\");\n        }\n    }\n\n    @Override\n    @SuppressWarnings(\"unchecked\")\n    public <T, C> C getBroadcastVariableWithInitializer(\n            String name, BroadcastVariableInitializer<T, C> initializer) {\n        if (broadcastVariables.containsKey(name)) {\n            return initializer.initializeBroadcastVariable((List<T>) broadcastVariables.get(name));\n        } else {\n            throw new RuntimeException(\n                    \"Cannot get broadcast variables before processing elements.\");\n        }\n    }\n\n    @Internal\n    public void setBroadcastVariable(String name, List<?> broadcastVariable) {\n        broadcastVariables.put(name, broadcastVariable);\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElement.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\n/**\n * The wrapper class for possible cached elements used in {@link\n * org.apache.flink.ml.common.broadcast.operator.AbstractBroadcastWrapperOperator}. It could be\n * either {@link org.apache.flink.streaming.api.watermark.Watermark}, {@link\n * org.apache.flink.streaming.runtime.streamrecord.StreamRecord}, etc.\n *\n * @param <T> the record type.\n */\npublic class CacheElement<T> {\n    private T record;\n    private long watermark;\n    private Type type;\n\n    public CacheElement(T record, long watermark, Type type) {\n        this.record = record;\n        this.watermark = watermark;\n        this.type = type;\n    }\n\n    public static <T> CacheElement<T> newRecord(T record) {\n        return new CacheElement<>(record, -1, Type.RECORD);\n    }\n\n    public static <T> CacheElement<T> newWatermark(long watermark) {\n        return new CacheElement<>(null, watermark, Type.WATERMARK);\n    }\n\n    public T getRecord() {\n        return record;\n    }\n\n    public void setRecord(T record) {\n        this.record = record;\n    }\n\n    public long getWatermark() {\n        return watermark;\n    }\n\n    public void setWatermark(long watermark) {\n        this.watermark = watermark;\n    }\n\n    public Type getType() {\n        return type;\n    }\n\n    public void setType(Type type) {\n        this.type = type;\n    }\n\n    /** The type of cached elements. */\n    public enum Type {\n        /** record type. */\n        RECORD,\n        /** watermark type. */\n        WATERMARK,\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElementTypeInfo.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\nimport org.apache.flink.api.common.ExecutionConfig;\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\n\nimport java.util.Objects;\n\n/**\n * TypeInformation for {@link CacheElement}.\n *\n * @param <T> the record type.\n */\npublic class CacheElementTypeInfo<T> extends TypeInformation<CacheElement<T>> {\n\n    private final TypeInformation<T> recordTypeInfo;\n\n    public CacheElementTypeInfo(TypeInformation<T> recordTypeInfo) {\n        this.recordTypeInfo = recordTypeInfo;\n    }\n\n    @Override\n    public boolean isBasicType() {\n        return false;\n    }\n\n    @Override\n    public boolean isTupleType() {\n        return false;\n    }\n\n    @Override\n    public int getArity() {\n        return 1;\n    }\n\n    @Override\n    public int getTotalFields() {\n        return 1;\n    }\n\n    @Override\n    public Class<CacheElement<T>> getTypeClass() {\n        return (Class) CacheElement.class;\n    }\n\n    @Override\n    public boolean isKeyType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<CacheElement<T>> createSerializer(ExecutionConfig config) {\n        return new CacheElementSerializer<>(recordTypeInfo.createSerializer(config));\n    }\n\n    @Override\n    public String toString() {\n        return \"RecordOrWatermark Type\";\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n\n        if (null == obj || getClass() != obj.getClass()) {\n            return false;\n        }\n\n        CacheElementTypeInfo<T> that = (CacheElementTypeInfo<T>) obj;\n        return Objects.equals(recordTypeInfo, that.recordTypeInfo);\n    }\n\n    @Override\n    public int hashCode() {\n        return recordTypeInfo != null ? recordTypeInfo.hashCode() : 0;\n    }\n\n    @Override\n    public boolean canEqual(Object obj) {\n        return obj instanceof CacheElementTypeInfo;\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/typeinfo/CacheElementSerializer.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.typeinfo;\n\nimport org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot;\nimport org.apache.flink.api.common.typeutils.TypeSerializer;\nimport org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;\nimport org.apache.flink.api.common.typeutils.base.LongSerializer;\nimport org.apache.flink.core.memory.DataInputView;\nimport org.apache.flink.core.memory.DataOutputView;\nimport org.apache.flink.ml.common.broadcast.typeinfo.CacheElement.Type;\n\nimport java.io.IOException;\nimport java.util.Objects;\n\n/**\n * TypeSerializer for {@link CacheElement}.\n *\n * @param <T> the record type.\n */\npublic class CacheElementSerializer<T> extends TypeSerializer<CacheElement<T>> {\n\n    private final TypeSerializer<T> recordSerializer;\n\n    public CacheElementSerializer(TypeSerializer<T> recordSerializer) {\n        this.recordSerializer = recordSerializer;\n    }\n\n    @Override\n    public boolean isImmutableType() {\n        return false;\n    }\n\n    @Override\n    public TypeSerializer<CacheElement<T>> duplicate() {\n        return new CacheElementSerializer<>(recordSerializer.duplicate());\n    }\n\n    @Override\n    public CacheElement<T> createInstance() {\n        return null;\n    }\n\n    @Override\n    public CacheElement<T> copy(CacheElement<T> from) {\n        switch (from.getType()) {\n            case RECORD:\n                return CacheElement.newRecord(recordSerializer.copy(from.getRecord()));\n            case WATERMARK:\n                return CacheElement.newWatermark(from.getWatermark());\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + from.getType());\n        }\n    }\n\n    @Override\n    public CacheElement<T> copy(CacheElement<T> from, CacheElement<T> reuse) {\n        switch (from.getType()) {\n            case RECORD:\n                if (reuse.getRecord() != null) {\n                    recordSerializer.copy(from.getRecord(), reuse.getRecord());\n                } else {\n                    reuse.setRecord(recordSerializer.copy(from.getRecord()));\n                }\n                break;\n            case WATERMARK:\n                reuse.setWatermark(from.getWatermark());\n                break;\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + from.getType());\n        }\n\n        return reuse;\n    }\n\n    @Override\n    public int getLength() {\n        return -1;\n    }\n\n    @Override\n    public void serialize(CacheElement<T> record, DataOutputView target) throws IOException {\n        target.writeByte((byte) record.getType().ordinal());\n\n        switch (record.getType()) {\n            case RECORD:\n                recordSerializer.serialize(record.getRecord(), target);\n                break;\n            case WATERMARK:\n                LongSerializer.INSTANCE.serialize(record.getWatermark(), target);\n                break;\n            default:\n                throw new RuntimeException(\n                        \"Unsupported Record or Watermark type \" + record.getType());\n        }\n    }\n\n    @Override\n    public CacheElement<T> deserialize(DataInputView source) throws IOException {\n        int type = source.readByte();\n        switch (CacheElement.Type.values()[type]) {\n            case RECORD:\n                T value = recordSerializer.deserialize(source);\n                return CacheElement.newRecord(value);\n            case WATERMARK:\n                long watermark = LongSerializer.INSTANCE.deserialize(source);\n                return CacheElement.newWatermark(watermark);\n            default:\n                throw new RuntimeException(\"Unsupported Record or Watermark type \" + type);\n        }\n    }\n\n    @Override\n    public CacheElement<T> deserialize(CacheElement<T> reuse, DataInputView source)\n            throws IOException {\n        int type = source.readByte();\n        switch (CacheElement.Type.values()[type]) {\n            case RECORD:\n                reuse.setType(Type.RECORD);\n                reuse.setRecord(recordSerializer.deserialize(source));\n                break;\n            case WATERMARK:\n                reuse.setType(Type.WATERMARK);\n                reuse.setWatermark(LongSerializer.INSTANCE.deserialize(source));\n                break;\n            default:\n                throw new RuntimeException(\"Unsupported Record or Watermark type \" + type);\n        }\n        return reuse;\n    }\n\n    @Override\n    public void copy(DataInputView source, DataOutputView target) throws IOException {\n        CacheElement<T> cacheElement = deserialize(source);\n        serialize(cacheElement, target);\n    }\n\n    @Override\n    public boolean equals(Object obj) {\n        if (this == obj) {\n            return true;\n        }\n\n        if (obj == null || getClass() != obj.getClass()) {\n            return false;\n        }\n        CacheElementSerializer<?> that = (CacheElementSerializer<?>) obj;\n        return Objects.equals(recordSerializer, that.recordSerializer);\n    }\n\n    @Override\n    public int hashCode() {\n        return recordSerializer != null ? recordSerializer.hashCode() : 0;\n    }\n\n    @Override\n    public TypeSerializerSnapshot<CacheElement<T>> snapshotConfiguration() {\n        return new CacheElementSerializerSnapshot<>();\n    }\n\n    /** The serializer snapshot class for {@link CacheElementSerializer}. */\n    private static final class CacheElementSerializerSnapshot<T>\n            extends CompositeTypeSerializerSnapshot<CacheElement<T>, CacheElementSerializer<T>> {\n\n        private static final int CURRENT_VERSION = 1;\n\n        public CacheElementSerializerSnapshot() {\n            super(CacheElementSerializer.class);\n        }\n\n        \n@Override\n        protected int getCurrentOuterSnapshotVersion() {\n            return CURRENT_VERSION;\n        }\n\n        @Override\n        protected TypeSerializer<?>[] getNestedSerializers(\n                CacheElementSerializer<T> tIterationRecordSerializer) {\n            return new TypeSerializer[] {tIterationRecordSerializer.recordSerializer};\n        }\n\n        @Override\n        @SuppressWarnings(\"unchecked\")\n        protected CacheElementSerializer<T> createOuterSerializerWithNestedSerializers(\n                TypeSerializer<?>[] typeSerializers) {\n            TypeSerializer<T> elementSerializer = (TypeSerializer<T>) typeSerializers[0];\n            return new CacheElementSerializer<>(elementSerializer);\n        }\n    }\n}\n\n// Path: flink-ml-core/src/main/java/org/apache/flink/ml/common/broadcast/operator/BroadcastVariableReceiverOperator.java\n/*\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements.  See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership.  The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License.  You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage org.apache.flink.ml.common.broadcast.operator;\n\nimport org.apache.flink.api.common.state.ListState;\nimport org.apache.flink.api.common.state.ListStateDescriptor;\nimport org.apache.flink.api.common.typeinfo.BasicTypeInfo;\nimport org.apache.flink.api.common.typeinfo.TypeInformation;\nimport org.apache.flink.api.java.tuple.Tuple2;\nimport org.apache.flink.iteration.operator.OperatorStateUtils;\nimport org.apache.flink.ml.common.broadcast.BroadcastContext;\nimport org.apache.flink.runtime.state.StateInitializationContext;\nimport org.apache.flink.runtime.state.StateSnapshotContext;\nimport org.apache.flink.streaming.api.operators.AbstractInput;\nimport org.apache.flink.streaming.api.operators.AbstractStreamOperatorV2;\nimport org.apache.flink.streaming.api.operators.BoundedMultiInput;\nimport org.apache.flink.streaming.api.operators.Input;\nimport org.apache.flink.streaming.api.operators.MultipleInputStreamOperator;\nimport org.apache.flink.streaming.api.operators.StreamOperatorParameters;\nimport org.apache.flink.streaming.runtime.streamrecord.StreamRecord;\n\nimport org.apache.commons.collections.IteratorUtils;\n\nimport java.io.Serializable;\nimport java.util.ArrayList;\nimport java.util.List;\n\n...\n", "needle_token_start": 15104, "needle_token_end": 15130, "code_context_ntokens": 15913, "output": ["```java\n@Override\nprotected int getCurrentOuterSnapshotVersion() {\n    return CURRENT_VERSION;\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "reshape", "language": "typescript", "path": "src/utils/tensor.js", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform a one-dimensional array into a multi-dimensional array based on specified dimensions.\n2. **Input**: The function accepts two parameters: a one-dimensional array containing data elements, and an array of integers representing the desired dimensions of the output array.\n3. **Output**: It returns a nested array structured according to the provided dimensions. The depth of the nested array corresponds to the number of dimensions specified.\n4. **Procedure**: The function first calculates the total number of elements in the input array and the product of the specified dimensions to ensure they match. If they do not match, an error is thrown. It then iteratively groups elements of the input array into nested arrays based on the dimensions specified, starting from the innermost dimension and working outwards.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        return this.clone().round_();\n    }\n\n    /**\n     * Performs Tensor dtype conversion.\n     * @param {DataType} type The desired data type.\n     * @returns {Tensor} The converted tensor.\n     */\n    to(type) {\n        // If the self Tensor already has the correct dtype, then self is returned.\n        if (this.type === type) return this;\n\n        // Otherwise, the returned tensor is a copy of self with the desired dtype.\n        if (!DataTypeMap.hasOwnProperty(type)) {\n            throw new Error(`Unsupported type: ${type}`);\n        }\n        // @ts-ignore\n        return new Tensor(type, DataTypeMap[type].from(this.data), this.dims);\n    }\n}\n\n/**\n * This creates a nested array of a given type and depth (see examples).\n * \n * @example\n *   NestArray<string, 1>; // string[]\n * @example\n *   NestArray<number, 2>; // number[][]\n * @example\n *   NestArray<string, 3>; // string[][][] etc.\n * @template T\n * @template {number} Depth\n * @template {never[]} [Acc=[]]\n * @typedef {Acc['length'] extends Depth ? T : NestArray<T[], Depth, [...Acc, never]>} NestArray\n */\n\n/**\n * Reshapes a 1-dimensional array into an n-dimensional array, according to the provided dimensions.\n *\n * @example\n *   reshape([10                    ], [1      ]); // Type: number[]      Value: [10]\n *   reshape([1, 2, 3, 4            ], [2, 2   ]); // Type: number[][]    Value: [[1, 2], [3, 4]]\n *   reshape([1, 2, 3, 4, 5, 6, 7, 8], [2, 2, 2]); // Type: number[][][]  Value: [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]\n *   reshape([1, 2, 3, 4, 5, 6, 7, 8], [4, 2   ]); // Type: number[][]    Value: [[1, 2], [3, 4], [5, 6], [7, 8]]\n * @param {T[]|DataArray} data The input array to reshape.\n * @param {DIM} dimensions The target shape/dimensions.\n * @template T\n * @template {[number]|number[]} DIM\n * @returns {NestArray<T, DIM[\"length\"]>} The reshaped array.\n */\n\nfunction reshape(data, dimensions) {\n\n    const totalElements = data.length;\n    const dimensionSize = dimensions.reduce((a, b) => a * b);\n\n    if (totalElements !== dimensionSize) {\n        throw Error(`cannot reshape array of size ${totalElements} into shape (${dimensions})`);\n    }\n\n    /** @type {any} */\n    let reshapedArray = data;\n\n    for (let i = dimensions.length - 1; i >= 0; i--) {\n        reshapedArray = reshapedArray.reduce((acc, val) => {\n            let lastArray = acc[acc.length - 1];\n\n            if (lastArray.length < dimensions[i]) {\n                lastArray.push(val);\n            } else {\n                acc.push([val]);\n            }\n\n            return acc;\n        }, [[]]);\n    }\n\n    return reshapedArray[0];\n}\n\n/**\n * Permutes a tensor according to the provided axes.\n * @param {any} tensor The input tensor to permute.\n * @param {Array} axes The axes to permute the tensor along.\n * @returns {Tensor} The permuted tensor.\n */\nexport function permute(tensor, axes) {\n    const [permutedData, shape] = permute_data(tensor.data, tensor.dims, axes);\n    return new Tensor(tensor.type, permutedData, shape);\n}\n\n\n/**\n * Interpolates an Tensor to the given size.\n * @param {Tensor} input The input tensor to interpolate. Data must be channel-first (i.e., [c, h, w])\n * @param {number[]} size The output size of the image\n * @param {string} mode The interpolation mode\n * @param {boolean} align_corners Whether to align corners.\n * @returns {Tensor} The interpolated tensor.\n */\nexport function interpolate(input, [out_height, out_width], mode = 'bilinear', align_corners = false) {\n\n    // Input image dimensions\n    const in_channels = input.dims.at(-3) ?? 1;\n    const in_height = input.dims.at(-2);\n    const in_width = input.dims.at(-1);\n\n    let output = interpolate_data(\n        /** @type {import('./maths.js').TypedArray}*/(input.data),\n        [in_channels, in_height, in_width],\n        [out_height, out_width],\n        mode,\n        align_corners\n    );\n    return new Tensor(input.type, output, [in_channels, out_height, out_width]);\n}\n\n/**\n * Perform mean pooling of the last hidden state followed by a normalization step.\n * @param {Tensor} last_hidden_state Tensor of shape [batchSize, seqLength, embedDim]\n * @param {Tensor} attention_mask Tensor of shape [batchSize, seqLength]\n * @returns {Tensor} Returns a new Tensor of shape [batchSize, embedDim].\n */\nexport function mean_pooling(last_hidden_state, attention_mask) {\n    // last_hidden_state: [batchSize, seqLength, embedDim]\n    // attention_mask:    [batchSize, seqLength]\n\n    let shape = [last_hidden_state.dims[0], last_hidden_state.dims[2]];\n    // @ts-ignore\n    let returnedData = new last_hidden_state.data.constructor(shape[0] * shape[1]);\n    let [batchSize, seqLength, embedDim] = last_hidden_state.dims;\n\n    let outIndex = 0;\n    for (let i = 0; i < batchSize; ++i) {\n        let offset = i * embedDim * seqLength;\n\n        for (let k = 0; k < embedDim; ++k) {\n            let sum = 0;\n            let count = 0;\n\n            let attnMaskOffset = i * seqLength;\n            let offset2 = offset + k;\n            // Pool over all words in sequence\n            for (let j = 0; j < seqLength; ++j) {\n                // index into attention mask\n                let attn = Number(attention_mask.data[attnMaskOffset + j]);\n\n                count += attn;\n                sum += last_hidden_state.data[offset2 + j * embedDim] * attn;\n            }\n\n            let avg = sum / count;\n            returnedData[outIndex++] = avg;\n        }\n    }\n\n    return new Tensor(\n        last_hidden_state.type,\n        returnedData,\n        shape\n    )\n}\n\n/**\n * Apply Layer Normalization for last certain number of dimensions.\n * @param {Tensor} input The input tensor\n * @param {number[]} normalized_shape input shape from an expected input of size\n * @param {Object} options The options for the layer normalization\n * @param {number} [options.eps=1e-5] A value added to the denominator for numerical stability.\n * @returns {Tensor} The normalized tensor.\n */\nexport function layer_norm(input, normalized_shape, {\n    eps = 1e-5,\n} = {}) {\n    if (input.dims.length !== 2) {\n        throw new Error('`layer_norm` currently only supports 2D input.');\n    }\n\n    const [batchSize, featureDim] = input.dims;\n\n    if (normalized_shape.length !== 1 && normalized_shape[0] !== featureDim) {\n        throw new Error('`normalized_shape` must be a 1D array with shape `[input.dims[1]]`.');\n    }\n\n    const [std, mean] = std_mean(input, 1, 0, true);\n\n    // @ts-ignore\n    const returnedData = new input.data.constructor(input.data.length);\n\n    for (let i = 0; i < batchSize; ++i) {\n        const offset = i * featureDim;\n        for (let j = 0; j < featureDim; ++j) {\n            const offset2 = offset + j;\n            returnedData[offset2] = (input.data[offset2] - mean.data[i]) / (std.data[i] + eps);\n        }\n    }\n    return new Tensor(input.type, returnedData, input.dims);\n}\n\n/**\n * Helper function to calculate new dimensions when performing a squeeze operation.\n * @param {number[]} dims The dimensions of the tensor.\n * @param {number|number[]|null} dim The dimension(s) to squeeze.\n * @returns The new dimensions.\n * @private\n */\nfunction calc_squeeze_dims(dims, dim) {\n    dims = dims.slice();\n    if (dim === null) {\n        dims = dims.filter((d) => d !== 1);\n    } else if (typeof dim === 'number') {\n        if (dims[dim] === 1) {\n            dims.splice(dim, 1);\n        }\n    } else if (Array.isArray(dim)) {\n        dims = dims.filter((x, i) => {\n            return x !== 1 || !dim.includes(i);\n        });\n    }\n    return dims;\n}\n\n/**\n * Helper function to calculate new dimensions when performing an unsqueeze operation.\n * @param {number[]} dims The dimensions of the tensor.\n * @param {number} dim The dimension to unsqueeze.\n * @returns The new dimensions.\n * @private\n */\nfunction calc_unsqueeze_dims(dims, dim) {\n    // Dimension out of range (e.g., \"expected to be in range of [-4, 3], but got 4\")\n    // + 1 since we allow inserting at the end (i.e. dim = -1)\n    dim = safeIndex(dim, dims.length + 1);\n    dims = dims.slice();\n    // Insert 1 into specified dimension\n    dims.splice(dim, 0, 1);\n    return dims;\n}\n\n/**\n * Safely calculate the index for an array of a given size, allowing negative indexing.\n * @param {number} index The index that will be used.\n * @param {number} size The size of the array.\n * @param {number} [dimension=null] The dimension that the index is for (optional).\n * @returns {number} The index, guaranteed to be non-negative and less than `arrayLength`.\n * \n * @throws {Error} If the index is out of range.\n * @private\n */\nfunction safeIndex(index, size, dimension = null) {\n    if (index < -size || index >= size) {\n        throw new Error(`IndexError: index ${index} is out of bounds for dimension${dimension === null ? '' : ' ' + dimension} with size ${size}`);\n    }\n\n    if (index < 0) {\n        // Negative indexing, ensuring positive index\n        index = ((index % size) + size) % size;\n    }\n    return index;\n}\n\n/**\n * Concatenates an array of tensors along a specified dimension.\n * @param {Tensor[]} tensors The array of tensors to concatenate.\n * @param {number} dim The dimension to concatenate along.\n * @returns {Tensor} The concatenated tensor.\n */\nexport function cat(tensors, dim = 0) {\n    dim = safeIndex(dim, tensors[0].dims.length);\n\n    // TODO do validation of shapes\n\n    const resultDims = tensors[0].dims.slice();\n    resultDims[dim] = tensors.reduce((a, b) => a + b.dims[dim], 0);\n\n    // Create a new array to store the accumulated values\n    const resultSize = resultDims.reduce((a, b) => a * b, 1);\n    // @ts-ignore\n    const result = new tensors[0].data.constructor(resultSize);\n\n    // Create output tensor of same type as first\n    const resultType = tensors[0].type;\n\n    if (dim === 0) {\n        // Handle special case for performance reasons\n\n        let offset = 0;\n        for (let t of tensors) {\n            result.set(t.data, offset);\n            offset += t.data.length;\n        }\n\n    } else {\n\n        let currentDim = 0;\n\n        for (let t = 0; t < tensors.length; ++t) {\n            let tensor = tensors[t];\n\n            // Iterate over the data array\n            for (let i = 0; i < tensor.data.length; ++i) {\n                // Calculate the index in the resulting array\n                let resultIndex = 0;\n\n                for (let j = tensor.dims.length - 1, num = i, resultMultiplier = 1; j >= 0; --j) {\n                    const size = tensor.dims[j];\n                    let index = num % size;\n                    if (j === dim) {\n                        index += currentDim;\n                    }\n                    resultIndex += index * resultMultiplier;\n                    resultMultiplier *= resultDims[j];\n                    num = Math.floor(num / size);\n                }\n                // Accumulate the value at the current index\n                result[resultIndex] = tensor.data[i];\n            }\n\n            currentDim += tensor.dims[dim];\n        }\n    }\n    return new Tensor(resultType, result, resultDims);\n}\n\n/**\n * Stack an array of tensors along a specified dimension.\n * @param {Tensor[]} tensors The array of tensors to stack.\n * @param {number} dim The dimension to stack along.\n * @returns {Tensor} The stacked tensor.\n */\nexport function stack(tensors, dim = 0) {\n    // TODO do validation of shapes\n    // NOTE: stack expects each tensor to be equal size\n    return cat(tensors.map(t => t.unsqueeze(dim)), dim);\n}\n\n\n/**\n * Calculates the standard deviation and mean over the dimensions specified by dim. dim can be a single dimension or `null` to reduce over all dimensions.\n * @param {Tensor} input the input tenso\n * @param {number|null} dim the dimension to reduce. If None, all dimensions are reduced.\n * @param {number} correction difference between the sample size and sample degrees of freedom. Defaults to Bessel's correction, correction=1.\n * @param {boolean} keepdim whether the output tensor has dim retained or not.\n * @returns {Tensor[]} A tuple of (std, mean) tensors.\n */\nexport function std_mean(input, dim = null, correction = 1, keepdim = false) {\n\n    if (dim === null) {\n        // None to reduce over all dimensions.\n        // @ts-ignore\n        const sum = input.data.reduce((a, b) => a + b, 0);\n        const mean = sum / input.data.length;\n        // @ts-ignore\n        const std = Math.sqrt(input.data.reduce((a, b) => a + (b - mean) ** 2, 0) / (input.data.length - correction));\n\n        const meanTensor = new Tensor(input.type, [mean], [/* scalar */]);\n        const stdTensor = new Tensor(input.type, [std], [/* scalar */]);\n\n        return [stdTensor, meanTensor];\n    }\n\n    // Negative indexing\n    dim = safeIndex(dim, input.dims.length);\n\n    const meanTensor = mean(input, dim, keepdim);\n\n    // Calculate the shape of the resulting array after summation\n    const resultDims = input.dims.slice(); // Copy the original dimensions\n    resultDims[dim] = 1; // Remove the specified axis\n\n    // Create a new array to store the accumulated values\n    // @ts-ignore\n    const result = new input.data.constructor(input.data.length / input.dims[dim]);\n\n    // Iterate over the data array\n    for (let i = 0; i < input.data.length; ++i) {\n\n        // Calculate the index in the resulting array\n        let resultIndex = 0;\n\n        for (let j = input.dims.length - 1, num = i, resultMultiplier = 1; j >= 0; --j) {\n            const size = input.dims[j];\n            if (j !== dim) {\n                const index = num % size;\n                resultIndex += index * resultMultiplier;\n                resultMultiplier *= resultDims[j];\n            }\n            num = Math.floor(num / size);\n        }\n\n        // Accumulate the value at the current index\n        result[resultIndex] += (input.data[i] - meanTensor.data[resultIndex]) ** 2;\n    }\n\n    for (let i = 0; i < result.length; ++i) {\n        result[i] = Math.sqrt(result[i] / (input.dims[dim] - correction));\n    }\n\n    if (!keepdim) {\n        resultDims.splice(dim, 1);\n    }\n\n    const stdTensor = new Tensor(input.type, result, resultDims);\n\n    return [stdTensor, meanTensor];\n}\n\n\n/**\n * Returns the mean value of each row of the input tensor in the given dimension dim.\n * @param {Tensor} input the input tensor.\n * @param {number|null} dim the dimension to reduce.\n * @param {boolean} keepdim whether the output tensor has dim retained or not.\n * @returns A new tensor with means taken along the specified dimension.\n */\nexport function mean(input, dim = null, keepdim = false) {\n\n    if (dim === null) {\n        // None to reduce over all dimensions.\n        // @ts-ignore\n        let val = input.data.reduce((a, b) => a + b, 0);\n        return new Tensor(input.type, [val / input.data.length], [/* scalar */]);\n    }\n\n    // Negative indexing\n    dim = safeIndex(dim, input.dims.length);\n\n    // Calculate the shape of the resulting array after summation\n    const resultDims = input.dims.slice(); // Copy the original dimensions\n    resultDims[dim] = 1; // Remove the specified axis\n\n    // Create a new array to store the accumulated values\n    // @ts-ignore\n    const result = new input.data.constructor(input.data.length / input.dims[dim]);\n\n    // Iterate over the data array\n    for (let i = 0; i < input.data.length; ++i) {\n\n        // Calculate the index in the resulting array\n        let resultIndex = 0;\n\n        for (let j = input.dims.length - 1, num = i, resultMultiplier = 1; j >= 0; --j) {\n            const size = input.dims[j];\n            if (j !== dim) {\n                const index = num % size;\n                resultIndex += index * resultMultiplier;\n                resultMultiplier *= resultDims[j];\n            }\n            num = Math.floor(num / size);\n        }\n\n        // Accumulate the value at the current index\n        result[resultIndex] += input.data[i];\n    }\n\n    if (input.dims[dim] !== 1) {\n        for (let i = 0; i < result.length; ++i) {\n            result[i] = result[i] / input.dims[dim];\n        }\n    }\n\n    if (!keepdim) {\n        resultDims.splice(dim, 1);\n    }\n\n    return new Tensor(input.type, result, resultDims);\n}\n\n\n/**\n *\n * Measures similarity between two temporal sequences (e.g., input audio and output tokens\n * to generate token-level timestamps).\n * @param {Tensor} matrix \n * @returns {number[][]}\n */\nexport function dynamicTimeWarping(matrix) {\n    const [output_length, input_length] = matrix.dims;\n\n    const outputShape = [output_length + 1, input_length + 1];\n\n    const cost = new Tensor(\n        'float32',\n        new Float32Array(outputShape[0] * outputShape[1]).fill(Infinity),\n        outputShape\n    );\n\n    const trace = new Tensor(\n        'float32',\n        new Float32Array(outputShape[0] * outputShape[1]).fill(-1),\n        outputShape\n    )\n\n    // same as `cost[0][0] = 0`;\n    cost[0].data[0] = 0;\n\n    for (let j = 1; j < input_length + 1; ++j) {\n        for (let i = 1; i < output_length + 1; ++i) {\n\n            const c0 = cost[i - 1][j - 1].item();\n            const c1 = cost[i - 1][j].item();\n            const c2 = cost[i][j - 1].item();\n\n            let c, t;\n            if (c0 < c1 && c0 < c2) {\n                c = c0;\n                t = 0;\n            } else if (c1 < c0 && c1 < c2) {\n                c = c1;\n                t = 1;\n            } else {\n                c = c2;\n                t = 2;\n            }\n\n            cost[i].data[j] = matrix[i - 1][j - 1].item() + c;\n            trace[i].data[j] = t;\n        }\n    }\n\n    // backtrace\n    let i = output_length;\n    let j = input_length;\n\n    // @ts-ignore\n    trace.data.fill(2, 0, outputShape[1]) // trace[0, :] = 2\n    for (let i = 0; i < outputShape[0]; ++i) { // trace[:, 0] = 1\n        trace[i].data[0] = 1;\n    }\n\n    let text_indices = [];\n    let time_indices = [];\n\n    while (i > 0 || j > 0) {\n        text_indices.push(i - 1);\n        time_indices.push(j - 1);\n\n        const t = trace[i][j].item();\n        switch (t) {\n            case 0:\n                --i; --j;\n                break;\n            case 1:\n                --i;\n                break;\n            case 2:\n                --j;\n                break;\n            default:\n                throw new Error(\n                    `Internal error in dynamic time warping. Unexpected trace[${i}, ${j}]. Please file a bug report.`\n                )\n        }\n    }\n\n    text_indices.reverse();\n    time_indices.reverse();\n\n    return [text_indices, time_indices];\n\n}\n\nfunction dimsToStride(dims) {\n    const stride = new Array(dims.length);\n    for (let i = dims.length - 1, s2 = 1; i >= 0; --i) {\n        stride[i] = s2;\n        s2 *= dims[i];\n    }\n    return stride;\n}\n\n/**\n * Returns a tensor filled with the scalar value 1, with the shape defined by the variable argument size.\n * @param {number[]} size A sequence of integers defining the shape of the output tensor.\n */\nexport function ones(size) {\n    const numElements = size.reduce((a, b) => a * b, 1);\n    return new Tensor(\n        'int64',\n        new BigInt64Array(numElements).fill(1n),\n        size\n    )\n}\n\n/**\n * Returns a tensor filled with the scalar value 1, with the same size as input.\n * @param {Tensor} tensor The size of input will determine size of the output tensor.\n * @returns The ones tensor.\n */\nexport function ones_like(tensor) {\n    return ones(tensor.dims);\n}\n\n/**\n * Quantizes the embeddings tensor to binary or unsigned binary precision.\n * @param {Tensor} tensor The tensor to quantize.\n * @param {'binary'|'ubinary'} precision The precision to use for quantization.\n * @returns {Tensor} The quantized tensor.\n */\nexport function quantize_embeddings(tensor, precision) {\n    if (tensor.dims.length !== 2) {\n        throw new Error(\"The tensor must have 2 dimensions\");\n    }\n    if (tensor.dims.at(-1) % 8 !== 0) {\n        throw new Error(\"The last dimension of the tensor must be a multiple of 8\");\n    }\n    if (!['binary', 'ubinary'].includes(precision)) {\n        throw new Error(\"The precision must be either 'binary' or 'ubinary'\");\n    }\n\n    const signed = precision === 'binary';\n    const dtype = signed ? 'int8' : 'uint8';\n\n    // Create a typed array to store the packed bits\n    const cls = signed ? Int8Array : Uint8Array;\n    const inputData = tensor.data;\n    const outputData = new cls(inputData.length / 8);\n\n    // Iterate over each number in the array\n    for (let i = 0; i < inputData.length; ++i) {\n        // Determine if the number is greater than 0\n        const bit = inputData[i] > 0 ? 1 : 0;\n\n        // Calculate the index in the typed array and the position within the byte\n        const arrayIndex = Math.floor(i / 8);\n        const bitPosition = i % 8;\n\n        // Pack the bit into the typed array\n        outputData[arrayIndex] |= bit << (7 - bitPosition);\n        if (signed && bitPosition === 0) {\n            outputData[arrayIndex] -= 128;\n        }\n    };\n\n    return new Tensor(dtype, outputData, [tensor.dims[0], tensor.dims[1] / 8]);\n}\n\n// Path: src/utils/image.js\n\n/**\n * @file Helper module for image processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/image\n */\n\nimport { getFile } from './hub.js';\nimport { env } from '../env.js';\nimport { Tensor } from './tensor.js';\n\n// Will be empty (or not used) if running in browser or web-worker\nimport sharp from 'sharp';\n\nconst BROWSER_ENV = typeof self !== 'undefined';\nconst WEBWORKER_ENV = BROWSER_ENV && self.constructor.name === 'DedicatedWorkerGlobalScope';\n\nlet createCanvasFunction;\nlet ImageDataClass;\nlet loadImageFunction;\nif (BROWSER_ENV) {\n    // Running in browser or web-worker\n    createCanvasFunction = (/** @type {number} */ width, /** @type {number} */ height) => {\n        if (!self.OffscreenCanvas) {\n            throw new Error('OffscreenCanvas not supported by this browser.');\n        }\n        return new self.OffscreenCanvas(width, height)\n    };\n    loadImageFunction = self.createImageBitmap;\n    ImageDataClass = self.ImageData;\n\n} else if (sharp) {\n    // Running in Node.js, electron, or other non-browser environment\n\n    loadImageFunction = async (/**@type {sharp.Sharp}*/img) => {\n        const metadata = await img.metadata();\n        const rawChannels = metadata.channels;\n\n        let { data, info } = await img.raw().toBuffer({ resolveWithObject: true });\n\n        const newImage = new RawImage(new Uint8ClampedArray(data), info.width, info.height, info.channels);\n        if (rawChannels !== undefined && rawChannels !== info.channels) {\n            // Make sure the new image has the same number of channels as the input image.\n            // This is necessary for grayscale images.\n            newImage.convert(rawChannels);\n        }\n        return newImage;\n    }\n\n} else {\n    throw new Error('Unable to load image processing library.');\n}\n\n\n// Defined here: https://github.com/python-pillow/Pillow/blob/a405e8406b83f8bfb8916e93971edc7407b8b1ff/src/libImaging/Imaging.h#L262-L268\nconst RESAMPLING_MAPPING = {\n    0: 'nearest',\n    1: 'lanczos',\n    2: 'bilinear',\n    3: 'bicubic',\n    4: 'box',\n    5: 'hamming',\n}\n\n/**\n * Mapping from file extensions to MIME types.\n */\nconst CONTENT_TYPE_MAP = new Map([\n    ['png', 'image/png'],\n    ['jpg', 'image/jpeg'],\n    ['jpeg', 'image/jpeg'],\n    ['gif', 'image/gif'],\n]);\n\nexport class RawImage {\n\n    /**\n     * Create a new `RawImage` object.\n     * @param {Uint8ClampedArray|Uint8Array} data The pixel data.\n     * @param {number} width The width of the image.\n     * @param {number} height The height of the image.\n     * @param {1|2|3|4} channels The number of channels.\n     */\n    constructor(data, width, height, channels) {\n        this.data = data;\n        this.width = width;\n        this.height = height;\n        this.channels = channels;\n    }\n\n    /** \n     * Returns the size of the image (width, height).\n     * @returns {[number, number]} The size of the image (width, height).\n     */\n    get size() {\n        return [this.width, this.height];\n    }\n\n    /**\n     * Helper method for reading an image from a variety of input types.\n     * @param {RawImage|string|URL} input \n     * @returns The image object.\n     * \n     * **Example:** Read image from a URL.\n     * ```javascript\n     * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg');\n     * // RawImage {\n     * //   \"data\": Uint8ClampedArray [ 25, 25, 25, 19, 19, 19, ... ],\n     * //   \"width\": 800,\n     * //   \"height\": 533,\n     * //   \"channels\": 3\n     * // }\n     * ```\n     */\n    static async read(input) {\n        if (input instanceof RawImage) {\n            return input;\n        } else if (typeof input === 'string' || input instanceof URL) {\n            return await this.fromURL(input);\n        } else {\n            throw new Error(`Unsupported input type: ${typeof input}`);\n        }\n    }\n\n\n    /**\n     * Read an image from a URL or file path.\n     * @param {string|URL} url The URL or file path to read the image from.\n     * @returns {Promise<RawImage>} The image object.\n     */\n    static async fromURL(url) {\n        let response = await getFile(url);\n        if (response.status !== 200) {\n            throw new Error(`Unable to read image from \"${url}\" (${response.status} ${response.statusText})`);\n        }\n        let blob = await response.blob();\n        return this.fromBlob(blob);\n    }\n\n    /**\n     * Helper method to create a new Image from a blob.\n     * @param {Blob} blob The blob to read the image from.\n     * @returns {Promise<RawImage>} The image object.\n     */\n    static async fromBlob(blob) {\n        if (BROWSER_ENV) {\n            // Running in environment with canvas\n            let img = await loadImageFunction(blob);\n\n            const ctx = createCanvasFunction(img.width, img.height).getContext('2d');\n\n            // Draw image to context\n            ctx.drawImage(img, 0, 0);\n\n            return new this(ctx.getImageData(0, 0, img.width, img.height).data, img.width, img.height, 4);\n\n        } else {\n            // Use sharp.js to read (and possible resize) the image.\n            let img = sharp(await blob.arrayBuffer());\n\n            return await loadImageFunction(img);\n        }\n    }\n\n    /**\n     * Helper method to create a new Image from a tensor\n     * @param {Tensor} tensor \n     */\n    static fromTensor(tensor, channel_format = 'CHW') {\n        if (tensor.dims.length !== 3) {\n            throw new Error(`Tensor should have 3 dimensions, but has ${tensor.dims.length} dimensions.`);\n        }\n\n        if (channel_format === 'CHW') {\n            tensor = tensor.transpose(1, 2, 0);\n        } else if (channel_format === 'HWC') {\n            // Do nothing\n        } else {\n            throw new Error(`Unsupported channel format: ${channel_format}`);\n        }\n        if (!(tensor.data instanceof Uint8ClampedArray || tensor.data instanceof Uint8Array)) {\n            throw new Error(`Unsupported tensor type: ${tensor.type}`);\n        }\n        switch (tensor.dims[2]) {\n            case 1:\n            case 2:\n            case 3:\n            case 4:\n                return new RawImage(tensor.data, tensor.dims[1], tensor.dims[0], tensor.dims[2]);\n            default:\n                throw new Error(`Unsupported number of channels: ${tensor.dims[2]}`);\n        }\n    }\n\n    /**\n     * Convert the image to grayscale format.\n     * @returns {RawImage} `this` to support chaining.\n     */\n    grayscale() {\n        if (this.channels === 1) {\n            return this;\n        }\n\n        let newData = new Uint8ClampedArray(this.width * this.height * 1);\n        switch (this.channels) {\n            case 3: // rgb to grayscale\n            case 4: // rgba to grayscale\n                for (let i = 0, offset = 0; i < this.data.length; i += this.channels) {\n                    const red = this.data[i];\n                    const green = this.data[i + 1];\n                    const blue = this.data[i + 2];\n\n                    newData[offset++] = Math.round(0.2989 * red + 0.5870 * green + 0.1140 * blue);\n                }\n                break;\n            default:\n                throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`);\n        }\n        return this._update(newData, this.width, this.height, 1);\n    }\n\n    /**\n     * Convert the image to RGB format.\n     * @returns {RawImage} `this` to support chaining.\n     */\n    rgb() {\n        if (this.channels === 3) {\n            return this;\n        }\n\n        let newData = new Uint8ClampedArray(this.width * this.height * 3);\n\n        switch (this.channels) {\n            case 1: // grayscale to rgb\n                for (let i = 0, offset = 0; i < this.data.length; ++i) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                }\n                break;\n            case 4: // rgba to rgb\n                for (let i = 0, offset = 0; i < this.data.length; i += 4) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i + 1];\n                    newData[offset++] = this.data[i + 2];\n                }\n                break;\n            default:\n                throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`);\n        }\n        return this._update(newData, this.width, this.height, 3);\n\n    }\n\n    /**\n     * Convert the image to RGBA format.\n     * @returns {RawImage} `this` to support chaining.\n     */\n    rgba() {\n        if (this.channels === 4) {\n            return this;\n        }\n\n        let newData = new Uint8ClampedArray(this.width * this.height * 4);\n\n        switch (this.channels) {\n            case 1: // grayscale to rgba\n                for (let i = 0, offset = 0; i < this.data.length; ++i) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = 255;\n                }\n                break;\n            case 3: // rgb to rgba\n                for (let i = 0, offset = 0; i < this.data.length; i += 3) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i + 1];\n                    newData[offset++] = this.data[i + 2];\n                    newData[offset++] = 255;\n                }\n                break;\n            default:\n                throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`);\n        }\n\n        return this._update(newData, this.width, this.height, 4);\n    }\n\n    /**\n     * Resize the image to the given dimensions. This method uses the canvas API to perform the resizing.\n     * @param {number} width The width of the new image.\n     * @param {number} height The height of the new image.\n     * @param {Object} options Additional options for resizing.\n     * @param {0|1|2|3|4|5|string} [options.resample] The resampling method to use.\n     * @returns {Promise<RawImage>} `this` to support chaining.\n     */\n    async resize(width, height, {\n        resample = 2,\n    } = {}) {\n\n        // Ensure resample method is a string\n        let resampleMethod = RESAMPLING_MAPPING[resample] ?? resample;\n\n        if (BROWSER_ENV) {\n            // TODO use `resample` in browser environment\n\n            // Store number of channels before resizing\n            let numChannels = this.channels;\n\n            // Create canvas object for this image\n            let canvas = this.toCanvas();\n\n            // Actually perform resizing using the canvas API\n            const ctx = createCanvasFunction(width, height).getContext('2d');\n\n            // Draw image to context, resizing in the process\n            ctx.drawImage(canvas, 0, 0, width, height);\n\n            // Create image from the resized data\n            let resizedImage = new RawImage(ctx.getImageData(0, 0, width, height).data, width, height, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return resizedImage.convert(numChannels);\n\n        } else {\n            // Create sharp image from raw data, and resize\n            let img = this.toSharp();\n\n            switch (resampleMethod) {\n                case 'box':\n                case 'hamming':\n                    if (resampleMethod === 'box' || resampleMethod === 'hamming') {\n                        console.warn(`Resampling method ${resampleMethod} is not yet supported. Using bilinear instead.`);\n                        resampleMethod = 'bilinear';\n                    }\n\n                case 'nearest':\n                case 'bilinear':\n                case 'bicubic':\n                    // Perform resizing using affine transform. \n                    // This matches how the python Pillow library does it.\n                    img = img.affine([width / this.width, 0, 0, height / this.height], {\n                        interpolator: resampleMethod\n                    });\n                    break;\n\n                case 'lanczos':\n                    // https://github.com/python-pillow/Pillow/discussions/5519\n                    // https://github.com/lovell/sharp/blob/main/docs/api-resize.md\n                    img = img.resize({\n                        width, height,\n                        fit: 'fill',\n                        kernel: 'lanczos3', // PIL Lanczos uses a kernel size of 3 \n                    });\n                    break;\n\n                default:\n                    throw new Error(`Resampling method ${resampleMethod} is not supported.`);\n            }\n\n            return await loadImageFunction(img);\n        }\n\n    }\n\n    async pad([left, right, top, bottom]) {\n        left = Math.max(left, 0);\n        right = Math.max(right, 0);\n        top = Math.max(top, 0);\n        bottom = Math.max(bottom, 0);\n\n        if (left === 0 && right === 0 && top === 0 && bottom === 0) {\n            // No padding needed\n            return this;\n        }\n\n        if (BROWSER_ENV) {\n            // Store number of channels before padding\n            let numChannels = this.channels;\n\n            // Create canvas object for this image\n            let canvas = this.toCanvas();\n\n            let newWidth = this.width + left + right;\n            let newHeight = this.height + top + bottom;\n\n            // Create a new canvas of the desired size.\n            const ctx = createCanvasFunction(newWidth, newHeight).getContext('2d');\n\n            // Draw image to context, padding in the process\n            ctx.drawImage(canvas,\n                0, 0, this.width, this.height,\n                left, top, newWidth, newHeight\n            );\n\n            // Create image from the padded data\n            let paddedImage = new RawImage(\n                ctx.getImageData(0, 0, newWidth, newHeight).data,\n                newWidth, newHeight, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return paddedImage.convert(numChannels);\n\n        } else {\n            let img = this.toSharp().extend({ left, right, top, bottom });\n            return await loadImageFunction(img);\n        }\n    }\n\n    async crop([x_min, y_min, x_max, y_max]) {\n        // Ensure crop bounds are within the image\n        x_min = Math.max(x_min, 0);\n        y_min = Math.max(y_min, 0);\n        x_max = Math.min(x_max, this.width - 1);\n        y_max = Math.min(y_max, this.height - 1);\n\n        // Do nothing if the crop is the entire image\n        if (x_min === 0 && y_min === 0 && x_max === this.width - 1 && y_max === this.height - 1) {\n            return this;\n        }\n\n        const crop_width = x_max - x_min + 1;\n        const crop_height = y_max - y_min + 1;\n\n        if (BROWSER_ENV) {\n            // Store number of channels before resizing\n            const numChannels = this.channels;\n\n            // Create canvas object for this image\n            const canvas = this.toCanvas();\n\n            // Create a new canvas of the desired size. This is needed since if the \n            // image is too small, we need to pad it with black pixels.\n            const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d');\n\n            // Draw image to context, cropping in the process\n            ctx.drawImage(canvas,\n                x_min, y_min, crop_width, crop_height,\n                0, 0, crop_width, crop_height\n            );\n\n            // Create image from the resized data\n            const resizedImage = new RawImage(ctx.getImageData(0, 0, crop_width, crop_height).data, crop_width, crop_height, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return resizedImage.convert(numChannels);\n\n        } else {\n            // Create sharp image from raw data\n            const img = this.toSharp().extract({\n                left: x_min,\n                top: y_min,\n                width: crop_width,\n                height: crop_height,\n            });\n\n            return await loadImageFunction(img);\n        }\n\n    }\n\n    async center_crop(crop_width, crop_height) {\n        // If the image is already the desired size, return it\n        if (this.width === crop_width && this.height === crop_height) {\n            return this;\n        }\n\n        // Determine bounds of the image in the new canvas\n        let width_offset = (this.width - crop_width) / 2;\n        let height_offset = (this.height - crop_height) / 2;\n\n\n        if (BROWSER_ENV) {\n            // Store number of channels before resizing\n            let numChannels = this.channels;\n\n            // Create canvas object for this image\n            let canvas = this.toCanvas();\n\n            // Create a new canvas of the desired size. This is needed since if the \n            // image is too small, we need to pad it with black pixels.\n            const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d');\n\n            let sourceX = 0;\n            let sourceY = 0;\n            let destX = 0;\n            let destY = 0;\n\n            if (width_offset >= 0) {\n                sourceX = width_offset;\n            } else {\n                destX = -width_offset;\n            }\n\n            if (height_offset >= 0) {\n                sourceY = height_offset;\n            } else {\n                destY = -height_offset;\n            }\n\n            // Draw image to context, cropping in the process\n            ctx.drawImage(canvas,\n                sourceX, sourceY, crop_width, crop_height,\n                destX, destY, crop_width, crop_height\n            );\n\n            // Create image from the resized data\n            let resizedImage = new RawImage(ctx.getImageData(0, 0, crop_width, crop_height).data, crop_width, crop_height, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return resizedImage.convert(numChannels);\n\n        } else {\n            // Create sharp image from raw data\n            let img = this.toSharp();\n\n            if (width_offset >= 0 && height_offset >= 0) {\n                // Cropped image lies entirely within the original image\n                img = img.extract({\n                    left: Math.floor(width_offset),\n                    top: Math.floor(height_offset),\n                    width: crop_width,\n                    height: crop_height,\n                })\n            } else if (width_offset <= 0 && height_offset <= 0) {\n                // Cropped image lies entirely outside the original image,\n                // so we add padding\n                let top = Math.floor(-height_offset);\n                let left = Math.floor(-width_offset);\n                img = img.extend({\n                    top: top,\n                    left: left,\n\n                    // Ensures the resulting image has the desired dimensions\n                    right: crop_width - this.width - left,\n                    bottom: crop_height - this.height - top,\n                });\n            } else {\n                // Cropped image lies partially outside the original image.\n                // We first pad, then crop.\n\n                let y_padding = [0, 0];\n                let y_extract = 0;\n                if (height_offset < 0) {\n                    y_padding[0] = Math.floor(-height_offset);\n                    y_padding[1] = crop_height - this.height - y_padding[0];\n                } else {\n                    y_extract = Math.floor(height_offset);\n                }\n\n                let x_padding = [0, 0];\n                let x_extract = 0;\n                if (width_offset < 0) {\n                    x_padding[0] = Math.floor(-width_offset);\n                    x_padding[1] = crop_width - this.width - x_padding[0];\n                } else {\n                    x_extract = Math.floor(width_offset);\n                }\n\n                img = img.extend({\n                    top: y_padding[0],\n                    bottom: y_padding[1],\n                    left: x_padding[0],\n                    right: x_padding[1],\n                }).extract({\n                    left: x_extract,\n                    top: y_extract,\n                    width: crop_width,\n                    height: crop_height,\n                })\n            }\n\n            return await loadImageFunction(img);\n        }\n    }\n\n    async toBlob(type = 'image/png', quality = 1) {\n        if (!BROWSER_ENV) {\n            throw new Error('toBlob() is only supported in browser environments.')\n        }\n\n        const canvas = this.toCanvas();\n        return await canvas.convertToBlob({ type, quality });\n    }\n\n    toTensor(channel_format = 'CHW') {\n        let tensor = new Tensor(\n            'uint8',\n            new Uint8Array(this.data),\n            [this.height, this.width, this.channels]\n        );\n\n        if (channel_format === 'HWC') {\n            // Do nothing\n        } else if (channel_format === 'CHW') { // hwc -> chw\n            tensor = tensor.permute(2, 0, 1);\n        } else {\n            throw new Error(`Unsupported channel format: ${channel_format}`);\n        }\n        return tensor;\n    }\n\n    toCanvas() {\n        if (!BROWSER_ENV) {\n            throw new Error('toCanvas() is only supported in browser environments.')\n        }\n\n        // Clone, and convert data to RGBA before drawing to canvas.\n        // This is because the canvas API only supports RGBA\n        let cloned = this.clone().rgba();\n\n        // Create canvas object for the cloned image\n        let clonedCanvas = createCanvasFunction(cloned.width, cloned.height);\n\n        // Draw image to context\n        let data = new ImageDataClass(cloned.data, cloned.width, cloned.height);\n        clonedCanvas.getContext('2d').putImageData(data, 0, 0);\n\n        return clonedCanvas;\n    }\n\n    /**\n     * Helper method to update the image data.\n     * @param {Uint8ClampedArray} data The new image data.\n     * @param {number} width The new width of the image.\n     * @param {number} height The new height of the image.\n     * @param {1|2|3|4|null} [channels] The new number of channels of the image.\n     * @private\n     */\n    _update(data, width, height, channels = null) {\n        this.data = data;\n        this.width = width;\n        this.height = height;\n        if (channels !== null) {\n            this.channels = channels;\n        }\n        return this;\n    }\n\n    /**\n     * Clone the image\n     * @returns {RawImage} The cloned image\n     */\n    clone() {\n        return new RawImage(this.data.slice(), this.width, this.height, this.channels);\n    }\n\n    /**\n     * Helper method for converting image to have a certain number of channels\n     * @param {number} numChannels The number of channels. Must be 1, 3, or 4.\n     * @returns {RawImage} `this` to support chaining.\n     */\n    convert(numChannels) {\n        if (this.channels === numChannels) return this; // Already correct number of channels\n\n        switch (numChannels) {\n            case 1:\n                this.grayscale();\n                break;\n            case 3:\n                this.rgb();\n                break;\n            case 4:\n                this.rgba();\n                break;\n            default:\n                throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`);\n        }\n        return this;\n    }\n\n    /**\n     * Save the image to the given path.\n     * @param {string} path The path to save the image to.\n     */\n    async save(path) {\n\n        if (BROWSER_ENV) {\n            if (WEBWORKER_ENV) {\n                throw new Error('Unable to save an image from a Web Worker.')\n            }\n\n            const extension = path.split('.').pop().toLowerCase();\n            const mime = CONTENT_TYPE_MAP.get(extension) ?? 'image/png';\n\n            // Convert image to Blob\n            const blob = await this.toBlob(mime);\n\n            // Convert the canvas content to a data URL\n            const dataURL = URL.createObjectURL(blob);\n\n            // Create an anchor element with the data URL as the href attribute\n            const downloadLink = document.createElement('a');\n            downloadLink.href = dataURL;\n\n            // Set the download attribute to specify the desired filename for the downloaded image\n            downloadLink.download = path;\n\n            // Trigger the download\n            downloadLink.click();\n\n            // Clean up: remove the anchor element from the DOM\n            downloadLink.remove();\n\n        } else if (!env.useFS) {\n            throw new Error('Unable to save the image because filesystem is disabled in this environment.')\n\n        } else {\n            const img = this.toSharp();\n            return await img.toFile(path);\n        }\n    }\n\n    toSharp() {\n        if (BROWSER_ENV) {\n            throw new Error('toSharp() is only supported in server-side environments.')\n        }\n\n        return sharp(this.data, {\n            raw: {\n                width: this.width,\n                height: this.height,\n                channels: this.channels\n            }\n        });\n    }\n}\n\n// Path: src/utils/audio.js\n/**\n * @file Helper module for audio processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/audio\n */\n\nimport {\n    getFile,\n} from './hub.js';\nimport { FFT, max } from './maths.js';\nimport {\n    calculateReflectOffset,\n} from './core.js';\n\n\n/**\n * Helper function to read audio from a path/URL.\n * @param {string|URL} url The path/URL to load the audio from.\n * @param {number} sampling_rate The sampling rate to use when decoding the audio.\n * @returns {Promise<Float32Array>} The decoded audio as a `Float32Array`.\n */\nexport async function read_audio(url, sampling_rate) {\n    if (typeof AudioContext === 'undefined') {\n        // Running in node or an environment without AudioContext\n        throw Error(\n            \"Unable to load audio from path/URL since `AudioContext` is not available in your environment. \" +\n            \"Instead, audio data should be passed directly to the pipeline/processor. \" +\n            \"For more information and some example code, see https://huggingface.co/docs/transformers.js/guides/node-audio-processing.\"\n        )\n    }\n\n    const response = await (await getFile(url)).arrayBuffer();\n    const audioCTX = new AudioContext({ sampleRate: sampling_rate });\n    if (typeof sampling_rate === 'undefined') {\n        console.warn(`No sampling rate provided, using default of ${audioCTX.sampleRate}Hz.`)\n    }\n    const decoded = await audioCTX.decodeAudioData(response);\n\n    /** @type {Float32Array} */\n    let audio;\n\n    // We now replicate HuggingFace's `ffmpeg_read` method:\n    if (decoded.numberOfChannels === 2) {\n        // When downmixing a stereo audio file to mono using the -ac 1 option in FFmpeg,\n        // the audio signal is summed across both channels to create a single mono channel.\n        // However, if the audio is at full scale (i.e. the highest possible volume level),\n        // the summing of the two channels can cause the audio signal to clip or distort.\n\n        // To prevent this clipping, FFmpeg applies a scaling factor of 1/sqrt(2) (~ 0.707)\n        // to the audio signal before summing the two channels. This scaling factor ensures\n        // that the combined audio signal will not exceed the maximum possible level, even\n        // if both channels are at full scale.\n\n        // After applying this scaling factor, the audio signal from both channels is summed\n        // to create a single mono channel. It's worth noting that this scaling factor is\n        // only applied when downmixing stereo audio to mono using the -ac 1 option in FFmpeg.\n        // If you're using a different downmixing method, or if you're not downmixing the\n        // audio at all, this scaling factor may not be needed.\n        const SCALING_FACTOR = Math.sqrt(2);\n\n        const left = decoded.getChannelData(0);\n        const right = decoded.getChannelData(1);\n\n        audio = new Float32Array(left.length);\n        for (let i = 0; i < decoded.length; ++i) {\n            audio[i] = SCALING_FACTOR * (left[i] + right[i]) / 2;\n        }\n\n    } else {\n        // If the audio is not stereo, we can just use the first channel:\n        audio = decoded.getChannelData(0);\n    }\n\n    return audio;\n}\n\n/**\n * Generates a Hanning window of length M.\n *\n * @param {number} M The length of the Hanning window to generate.\n * @returns {Float64Array} The generated Hanning window.\n */\nexport function hanning(M) {\n    if (M < 1) {\n        return new Float64Array();\n    }\n    if (M === 1) {\n        return new Float64Array([1]);\n    }\n    const denom = M - 1;\n    const factor = Math.PI / denom;\n    const cos_vals = new Float64Array(M);\n    for (let i = 0; i < M; ++i) {\n        const n = 2 * i - denom;\n        cos_vals[i] = 0.5 + 0.5 * Math.cos(factor * n);\n    }\n    return cos_vals;\n}\n\nconst HERTZ_TO_MEL_MAPPING = {\n    \"htk\": (/** @type {number} */ freq) => 2595.0 * Math.log10(1.0 + (freq / 700.0)),\n    \"kaldi\": (/** @type {number} */ freq) => 1127.0 * Math.log(1.0 + (freq / 700.0)),\n    \"slaney\": (/** @type {number} */ freq, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = 27.0 / Math.log(6.4)) =>\n        freq >= min_log_hertz\n            ? min_log_mel + Math.log(freq / min_log_hertz) * logstep\n            : 3.0 * freq / 200.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} freq \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction hertz_to_mel(freq, mel_scale = \"htk\") {\n    const fn = HERTZ_TO_MEL_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof freq === 'number' ? fn(freq) : freq.map(x => fn(x));\n}\n\nconst MEL_TO_HERTZ_MAPPING = {\n    \"htk\": (/** @type {number} */ mels) => 700.0 * (10.0 ** (mels / 2595.0) - 1.0),\n    \"kaldi\": (/** @type {number} */ mels) => 700.0 * (Math.exp(mels / 1127.0) - 1.0),\n    \"slaney\": (/** @type {number} */ mels, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = Math.log(6.4) / 27.0) => mels >= min_log_mel\n        ? min_log_hertz * Math.exp(logstep * (mels - min_log_mel))\n        : 200.0 * mels / 3.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} mels \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction mel_to_hertz(mels, mel_scale = \"htk\") {\n    const fn = MEL_TO_HERTZ_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof mels === 'number' ? fn(mels) : mels.map(x => fn(x));\n}\n\n/**\n* Creates a triangular filter bank.\n*\n* Adapted from torchaudio and librosa.\n*\n* @param {Float64Array} fft_freqs Discrete frequencies of the FFT bins in Hz, of shape `(num_frequency_bins,)`.\n* @param {Float64Array} filter_freqs Center frequencies of the triangular filters to create, in Hz, of shape `(num_mel_filters,)`.\n* @returns {number[][]} of shape `(num_frequency_bins, num_mel_filters)`.\n*/\nfunction _create_triangular_filter_bank(fft_freqs, filter_freqs) {\n    const filter_diff = Float64Array.from(\n        { length: filter_freqs.length - 1 },\n...\n", "needle_token_start": 675, "needle_token_end": 901, "code_context_ntokens": 15976, "output": ["```javascript\n/**\n * Reshapes a 1-dimensional array into an n-dimensional array, according to the provided dimensions.\n *\n * @example\n *   reshape([10                    ], [1      ]); // Type: number[]      Value: [10]\n *   reshape([1, 2, 3, 4            ], [2, 2   ]); // Type: number[][]    Value: [[1, 2], [3, 4]]\n *   reshape([1, 2, 3, 4, 5, 6, 7, 8], [2, 2, 2]); // Type: number[][][]  Value: [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]\n *   reshape([1, 2, 3, 4, 5, 6, 7, 8], [4, 2   ]); // Type: number[][]    Value: [[1, 2], [3, 4], [5, 6], [7, 8]]\n * @param {T[]|DataArray} data The input array to reshape.\n * @param {DIM} dimensions The target shape/dimensions.\n * @template T\n * @template {[number]|number[]} DIM\n * @returns {NestArray<T, DIM[\"length\"]>} The reshaped array.\n */\nfunction reshape(data, dimensions) {\n\n    const totalElements = data.length;\n    const dimensionSize = dimensions.reduce((a, b) => a * b);\n\n    if (totalElements !== dimensionSize) {\n        throw Error(`cannot reshape array of size ${totalElements} into shape (${dimensions})`);\n    }\n\n    /** @type {any} */\n    let reshapedArray = data;\n\n    for (let i = dimensions.length - 1; i >= 0; i--) {\n        reshapedArray = reshapedArray.reduce((acc, val) => {\n            let lastArray = acc[acc.length - 1];\n\n            if (lastArray.length < dimensions[i]) {\n                lastArray.push(val);\n            } else {\n                acc.push([val]);\n            }\n\n            return acc;\n        }, [[]]);\n    }\n\n    return reshapedArray[0];\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "padHelper", "language": "typescript", "path": "src/tokenizers.js", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to extend the length of arrays within an object to a specified length by adding additional elements either to the beginning or the end of each array.\n2. **Input**: The function takes four parameters: an object where each property is an array, a target length for the arrays, a function that determines the value to be added based on the array's key, and a direction indicating whether to add the new elements to the 'right' (end) or 'left' (beginning) of the array.\n3. **Output**: There is no direct output as the function modifies the arrays in the input object in place.\n4. **Procedure**: The function iterates over each key in the object. For each key, it calculates the difference between the current array length and the target length. It then creates a new array filled with the value returned by the provided function for that key, with a length equal to the calculated difference. Depending on the specified direction, this new array is concatenated to either the beginning or the end of the original array, updating the array in the object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "     * @param {string} config.replacement The string to replace spaces with.\n     */\n    constructor(config) {\n        super(config);\n\n        this.addPrefixSpace = config.add_prefix_space;\n        this.replacement = config.replacement;\n    }\n\n    /** @type {Decoder['decode_chain']} */\n    decode_chain(tokens) {\n        const result = [];\n        for (let i = 0; i < tokens.length; ++i) {\n            let normalized = tokens[i].replaceAll(this.replacement, ' ');\n            if (this.addPrefixSpace && i == 0 && normalized.startsWith(' ')) {\n                normalized = normalized.substring(1);\n            }\n            result.push(normalized);\n        }\n        return result;\n    }\n}\n\n/**\n * A normalizer that applies a precompiled charsmap.\n * This is useful for applying complex normalizations in C++ and exposing them to JavaScript.\n * @extends Normalizer\n * @param {Object} config The configuration object for the Precompiled normalizer.\n * @param {Object} config.precompiled_charsmap The precompiled charsmap object.\n */\nclass Precompiled extends Normalizer {\n    /**\n     * Create a new instance of Precompiled normalizer.\n     * @param {Object} config The configuration object.\n     * @param {any} config.precompiled_charsmap Precompiled chars mapping.\n     */\n    constructor(config) {\n        super(config);\n        this.charsmap = config.precompiled_charsmap;\n    }\n\n    /**\n     * Normalizes the given text by applying the precompiled charsmap.\n     * @param {string} text The text to normalize.\n     * @returns {string} The normalized text.\n     */\n    normalize(text) {\n        // As stated in the sentencepiece normalization docs (https://github.com/google/sentencepiece/blob/master/doc/normalization.md#use-pre-defined-normalization-rule),\n        // there are 5 pre-defined normalization rules:\n        //  1. nmt_nfkc: NFKC normalization with some additional normalization around spaces. (default)\n        //  2. nfkc: original NFKC normalization.\n        //  3. nmt_nfkc_cf: nmt_nfkc + Unicode case folding (mostly lower casing)\n        //  4. nfkc_cf: nfkc + Unicode case folding.\n        //  5. identity: no normalization\n        // \n        // For now, we only implement the default (nmt_nfkc).\n        // See https://raw.githubusercontent.com/google/sentencepiece/master/data/nmt_nfkc.tsv for the full list of rules.\n        // TODO: detect when a different `this.charsmap` is used.\n\n        text = text.replace(/[\\u0001-\\u0008\\u000B\\u000E-\\u001F\\u007F\\u008F\\u009F]/gm, ''); // Remove control characters\n        text = text.replace(/[\\u0009\\u000A\\u000C\\u000D\\u1680\\u200B\\u200C\\u200E\\u200F\\u2028\\u2029\\u2581\\uFEFF\\uFFFD]/gm, '\\u0020'); // Replace certain characters with a space\n\n        if (text.includes('\\uFF5E')) {\n            // To match the sentencepiece implementation 100%, we must handle a very strange edge-case.\n            // For some reason, the \"Fullwidth Tilde\" character (\\uFF5E) should not be converted to the standard Tilde character (\\u007E).\n            // However, NFKC normalization does do this conversion. As a result, we split the string on the Fullwidth Tilde character,\n            // perform NFKC normalization on each substring, and then join them back together with the Fullwidth Tilde character.\n            const parts = text.split('\\uFF5E');\n            text = parts.map(part => part.normalize('NFKC')).join('\\uFF5E');\n        } else {\n            text = text.normalize('NFKC');\n        }\n\n        return text;\n    }\n}\n\n/**\n * A pre-tokenizer that applies a sequence of pre-tokenizers to the input text.\n * @extends PreTokenizer\n */\nclass PreTokenizerSequence extends PreTokenizer {\n    /**\n     * Creates an instance of PreTokenizerSequence.\n     * @param {Object} config The configuration object for the pre-tokenizer sequence.\n     * @param {Object[]} config.pretokenizers An array of pre-tokenizer configurations.\n     */\n    constructor(config) {\n        super();\n        this.tokenizers = config.pretokenizers.map(x => PreTokenizer.fromConfig(x));\n    }\n\n    /**\n     * Applies each pre-tokenizer in the sequence to the input text in turn.\n     * @param {string} text The text to pre-tokenize.\n     * @param {Object} [options] Additional options for the pre-tokenization logic.\n     * @returns {string[]} The pre-tokenized text.\n     */\n    pre_tokenize_text(text, options) {\n        // Use reduce to apply each tokenizer to the text\n        return this.tokenizers.reduce((preTokenizedText, tokenizer) => {\n            return tokenizer.pre_tokenize(preTokenizedText, options);\n        }, [text]);\n    }\n}\n\n/**\n * Splits on word boundaries (using the following regular expression: `\\w+|[^\\w\\s]+`).\n */\nclass WhitespacePreTokenizer extends PreTokenizer {\n    /**\n     * Creates an instance of WhitespacePreTokenizer.\n     * @param {Object} config The configuration object for the pre-tokenizer.\n     */\n    constructor(config) {\n        super();\n    }\n    /**\n     * Pre-tokenizes the input text by splitting it on word boundaries.\n     * @param {string} text The text to be pre-tokenized.\n     * @param {Object} [options] Additional options for the pre-tokenization logic.\n     * @returns {string[]} An array of tokens produced by splitting the input text on whitespace.\n     */\n    pre_tokenize_text(text, options) {\n        return text.match(/\\w+|[^\\w\\s]+/g) || [];\n    }\n}\n\n/**\n * Splits a string of text by whitespace characters into individual tokens.\n * @extends PreTokenizer\n */\nclass WhitespaceSplit extends PreTokenizer {\n    /**\n     * Creates an instance of WhitespaceSplit.\n     * @param {Object} config The configuration object for the pre-tokenizer.\n     */\n    constructor(config) {\n        super();\n    }\n    /**\n     * Pre-tokenizes the input text by splitting it on whitespace characters.\n     * @param {string} text The text to be pre-tokenized.\n     * @param {Object} [options] Additional options for the pre-tokenization logic.\n     * @returns {string[]} An array of tokens produced by splitting the input text on whitespace.\n     */\n    pre_tokenize_text(text, options) {\n        return whitespace_split(text);\n    }\n}\n\n// NOTE: `ReplacePreTokenizer` is custom (to support `BlenderbotSmallTokenizer`)\nclass ReplacePreTokenizer extends PreTokenizer {\n    /**\n     * @param {Object} config The configuration options for the pre-tokenizer.\n     * @param {Object} config.pattern The pattern used to split the text. Can be a string or a regex object.\n     * @param {string} config.content What to replace the pattern with.\n     */\n    constructor(config) {\n        super();\n        this.config = config;\n        this.pattern = createPattern(this.config.pattern);\n        this.content = this.config.content;\n    }\n\n    /**\n     * Pre-tokenizes the input text by replacing certain characters.\n     * @param {string} text The text to be pre-tokenized.\n     * @param {Object} [options] Additional options for the pre-tokenization logic.\n     * @returns {string[]} An array of tokens produced by replacing certain characters.\n     */\n    pre_tokenize_text(text, options) {\n        if (this.pattern === null) {\n            return [text];\n        }\n        return [text.replaceAll(this.pattern, this.config.content)];\n    }\n}\n\nconst SPECIAL_TOKEN_ATTRIBUTES = [\n    'bos_token',\n    'eos_token',\n    'unk_token',\n    'sep_token',\n    'pad_token',\n    'cls_token',\n    'mask_token',\n    // additional_special_tokens (TODO)\n]\n\n/**\n * \n * Helper function for padding values of an object, which are each arrays.\n * NOTE: No additional checks are made here for validity of arguments.\n * @param {Record<string, any[]>} item The input object.\n * @param {number} length The length to pad to.\n * @param {(key: string) => any} value_fn Determine the value to fill the array, based on its key.\n * @param {'right'|'left'} side Which side to pad the array.\n * @private\n */\nfuncti\non padHelper(item, length, value_fn, side) {\n    for (const key of Object.keys(item)) {\n        const diff = length - item[key].length;\n        const value = value_fn(key);\n\n        const padData = new Array(diff).fill(value);\n        item[key] = side === 'right'\n            ? mergeArrays(item[key], padData)\n            : mergeArrays(padData, item[key]);\n    }\n}\n\n/**\n * Helper function for truncating values of an object, which are each arrays.\n * NOTE: No additional checks are made here for validity of arguments.\n * @param {Record<string, any[]>} item The input object.\n * @param {number} length The length to truncate to.\n * @private\n */\nfunction truncateHelper(item, length) {\n    // Setting .length to a lower value truncates the array in-place:\n    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/length\n    for (const key of Object.keys(item)) {\n        item[key].length = length;\n    }\n}\n\n\n/**\n * @typedef {Object} Message\n * @property {string} role The role of the message (e.g., \"user\" or \"assistant\" or \"system\").\n * @property {string} content The content of the message.\n */\n\nexport class PreTrainedTokenizer extends Callable {\n    return_token_type_ids = false;\n\n    _default_chat_template = `{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}`;\n\n    /**\n     * Create a new PreTrainedTokenizer instance.\n     * @param {Object} tokenizerJSON The JSON of the tokenizer.\n     * @param {Object} tokenizerConfig The config of the tokenizer.\n     */\n    constructor(tokenizerJSON, tokenizerConfig) {\n        super();\n\n        this._tokenizer_config = tokenizerConfig;\n\n        // Construct parts of the tokenizer from the JSON\n        this.normalizer = Normalizer.fromConfig(tokenizerJSON.normalizer);\n        this.pre_tokenizer = PreTokenizer.fromConfig(tokenizerJSON.pre_tokenizer);\n        this.model = TokenizerModel.fromConfig(tokenizerJSON.model, tokenizerConfig);\n        this.post_processor = PostProcessor.fromConfig(tokenizerJSON.post_processor);\n        this.decoder = Decoder.fromConfig(tokenizerJSON.decoder);\n\n        // Add added_tokens to model\n        this.special_tokens = [];\n        this.all_special_ids = [];\n\n        /** @type {AddedToken[]} */\n        this.added_tokens = [];\n        for (const addedToken of tokenizerJSON.added_tokens) {\n            const token = new AddedToken(addedToken);\n            this.added_tokens.push(token);\n\n            this.model.tokens_to_ids.set(token.content, token.id);\n            this.model.vocab[token.id] = token.content;\n\n            if (token.special) {\n                this.special_tokens.push(token.content);\n                this.all_special_ids.push(token.id);\n            }\n        }\n\n        // Update additional_special_tokens\n        this.additional_special_tokens = tokenizerConfig.additional_special_tokens ?? [];\n        this.special_tokens.push(...this.additional_special_tokens);\n        this.special_tokens = [...new Set(this.special_tokens)]; // Remove duplicates\n\n        if (this.decoder) {\n            // Slight hack, but it prevents code duplication:\n            this.decoder.added_tokens = this.added_tokens;\n\n            // Another slight hack to add `end_of_word_suffix` (if present) to the decoder\n            // This is needed for cases where BPE model and ByteLevel decoder are used\n            // For more information, see https://github.com/xenova/transformers.js/issues/74\n            // TODO: save this to the decoder when exporting?\n            this.decoder.end_of_word_suffix = this.model.end_of_word_suffix;\n        }\n\n\n        this.added_tokens_regex = this.added_tokens.length > 0 ? new RegExp(\n            this.added_tokens.map(x => `${x.lstrip ? '\\\\s*' : ''}(${escapeRegExp(x.content)})${x.rstrip ? '\\\\s*' : ''}`).join('|')\n        ) : null;\n\n        // Set mask token if present (otherwise will be undefined, which is fine)\n        this.mask_token = this.getToken('mask_token');\n        this.mask_token_id = this.model.tokens_to_ids.get(this.mask_token);\n\n        this.pad_token = this.getToken('pad_token', 'eos_token');\n        this.pad_token_id = this.model.tokens_to_ids.get(this.pad_token);\n\n        this.sep_token = this.getToken('sep_token');\n        this.sep_token_id = this.model.tokens_to_ids.get(this.sep_token);\n\n        this.unk_token = this.getToken('unk_token');\n        this.unk_token_id = this.model.tokens_to_ids.get(this.unk_token);\n\n        this.model_max_length = tokenizerConfig.model_max_length;\n\n        /** @type {boolean} Whether or not to strip the text when tokenizing (removing excess spaces before and after the string). */\n        this.remove_space = tokenizerConfig.remove_space;\n\n        this.clean_up_tokenization_spaces = tokenizerConfig.clean_up_tokenization_spaces ?? true;\n        this.do_lowercase_and_remove_accent = tokenizerConfig.do_lowercase_and_remove_accent ?? false;\n\n        // TODO allow user to change this\n        /** @type {'right'|'left'} */\n        this.padding_side = 'right';\n\n        this.legacy = false;\n\n        this.chat_template = tokenizerConfig.chat_template ?? null;\n        if (Array.isArray(this.chat_template)) {\n            // Chat templates are stored as lists of dicts with fixed key names,\n            // we reconstruct that into a single dict while loading them.\n            const chat_template = Object.create(null);\n            for (const { name, template } of this.chat_template) {\n                if (typeof name !== 'string' || typeof template !== 'string') {\n                    throw new Error('Chat template must be a list of objects with \"name\" and \"template\" properties');\n                }\n                chat_template[name] = template;\n            }\n            this.chat_template = chat_template;\n        }\n        this._compiled_template_cache = new Map();\n    }\n\n    /**\n     * Returns the value of the first matching key in the tokenizer config object.\n     * @param {...string} keys One or more keys to search for in the tokenizer config object.\n     * @returns {string|null} The value associated with the first matching key, or null if no match is found.\n     * @throws {Error} If an object is found for a matching key and its __type property is not \"AddedToken\".\n     */\n    getToken(...keys) {\n        for (const key of keys) {\n            const item = this._tokenizer_config[key];\n\n            if (!item) continue;\n\n            if (typeof item === 'object') {\n                if (item.__type === 'AddedToken') {\n                    return item.content;\n                } else {\n                    throw Error(`Unknown token: ${item}`);\n                }\n            } else {\n                return item;\n            }\n        }\n        return null;\n    }\n\n    /**\n     * Loads a pre-trained tokenizer from the given `pretrained_model_name_or_path`. \n     * \n     * @param {string} pretrained_model_name_or_path The path to the pre-trained tokenizer.\n     * @param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer.\n     * \n     * @throws {Error} Throws an error if the tokenizer.json or tokenizer_config.json files are not found in the `pretrained_model_name_or_path`.\n     * @returns {Promise<PreTrainedTokenizer>} A new instance of the `PreTrainedTokenizer` class.\n     */\n    static async from_pretrained(pretrained_model_name_or_path, {\n        progress_callback = null,\n        config = null,\n        cache_dir = null,\n        local_files_only = false,\n        revision = 'main',\n        legacy = null,\n    } = {}) {\n\n        const info = await loadTokenizer(pretrained_model_name_or_path, {\n            progress_callback,\n            config,\n            cache_dir,\n            local_files_only,\n            revision,\n            legacy,\n        })\n\n        // @ts-ignore\n        return new this(...info);\n    }\n\n    /**\n     * @typedef {number[]|number[][]|Tensor} BatchEncodingItem\n     * \n     * @typedef {Object} BatchEncoding Holds the output of the tokenizer's call function.\n     * @property {BatchEncodingItem} input_ids List of token ids to be fed to a model.\n     * @property {BatchEncodingItem} attention_mask List of indices specifying which tokens should be attended to by the model.\n     * @property {BatchEncodingItem} [token_type_ids] List of token type ids to be fed to a model.\n     */\n\n    /**\n     * Encode/tokenize the given text(s).\n     * @param {string|string[]} text The text to tokenize.\n     * @param {Object} options An optional object containing the following properties:\n     * @param {string|string[]} [options.text_pair=null] Optional second sequence to be encoded. If set, must be the same type as text.\n     * @param {boolean|'max_length'} [options.padding=false] Whether to pad the input sequences.\n     * @param {boolean} [options.add_special_tokens=true] Whether or not to add the special tokens associated with the corresponding model.\n     * @param {boolean} [options.truncation=null] Whether to truncate the input sequences.\n     * @param {number} [options.max_length=null] Maximum length of the returned list and optionally padding length.\n     * @param {boolean} [options.return_tensor=true] Whether to return the results as Tensors or arrays.\n     * @returns {BatchEncoding} Object to be passed to the model.\n     */\n    _call(\n        // Required positional arguments\n        text,\n\n        // Optional keyword arguments\n        {\n            text_pair = null,\n            add_special_tokens = true,\n            padding = false,\n            truncation = null,\n            max_length = null,\n            return_tensor = true, // Different to HF\n        } = {},\n    ) {\n\n        const isBatched = Array.isArray(text);\n\n        /** @type {EncodingSingle[]} */\n        let encodedTokens;\n\n        if (isBatched) {\n            if (text.length === 0) {\n                throw Error('text array must be non-empty')\n            }\n\n            if (text_pair !== null) {\n                if (!Array.isArray(text_pair)) {\n                    throw Error('text_pair must also be an array')\n\n                } else if (text.length !== text_pair.length) {\n                    throw Error('text and text_pair must have the same length')\n                }\n\n                encodedTokens = text.map(\n                    (t, i) => this._encode_plus(t, text_pair[i], { add_special_tokens })\n                )\n\n            } else {\n                encodedTokens = text.map(x => this._encode_plus(x, null, { add_special_tokens }));\n            }\n\n        } else {\n            if (text === null || text === undefined) {\n                throw Error('text may not be null or undefined')\n            }\n\n            if (Array.isArray(text_pair)) {\n                throw Error('When specifying `text_pair`, since `text` is a string, `text_pair` must also be a string (i.e., not an array).')\n            }\n\n            // For single input, we just wrap in an array, and then unwrap later.\n            encodedTokens = [this._encode_plus(text, text_pair, { add_special_tokens })];\n        }\n        // At this point, tokens is batched: [batch_size, tokens]\n        // However, array may be jagged. So, we pad to max_length\n\n        if (max_length === null) {\n            if (padding === 'max_length') {\n                max_length = this.model_max_length;\n            } else {\n                // Calculate max length from sequences\n                max_length = max(encodedTokens.map(x => x.input_ids.length))[0];\n            }\n        } else {\n            if (!truncation) {\n                console.warn(`Truncation was not explicitly activated but \\`max_length\\` is provided a specific value, please use \\`truncation=true\\` to explicitly truncate examples to max length.`)\n            }\n        }\n\n        // Ensure it is less than model max length\n        max_length = Math.min(max_length, this.model_max_length)\n\n        if (padding || truncation) {\n\n            // Perform padding and/or truncation\n            for (let i = 0; i < encodedTokens.length; ++i) {\n                if (encodedTokens[i].input_ids.length === max_length) {\n                    continue;\n\n                } else if (encodedTokens[i].input_ids.length > max_length) {\n                    // possibly truncate\n                    if (truncation) {\n                        truncateHelper(encodedTokens[i], max_length);\n                    }\n\n                } else { // t.length < max_length\n                    // possibly pad\n                    if (padding) {\n                        padHelper(\n                            encodedTokens[i],\n                            max_length,\n                            key => key === 'input_ids' ? this.pad_token_id : 0,\n                            this.padding_side\n                        );\n                    }\n                }\n            }\n        }\n\n        const result = {};\n\n        if (return_tensor) {\n            if (!(padding && truncation)) {\n                // Not, guaranteed that all items have same length, so\n                // we perform additional check\n\n                if (\n                    encodedTokens.some(x => {\n                        for (const key of Object.keys(x)) {\n                            if (x[key].length !== encodedTokens[0][key]?.length) {\n                                return true;\n                            }\n                        }\n                        return false;\n                    })\n                ) {\n                    throw Error(\n                        \"Unable to create tensor, you should probably activate truncation and/or padding \" +\n                        \"with 'padding=true' and 'truncation=true' to have batched tensors with the same length.\"\n                    )\n                }\n            }\n\n            // Now we actually convert to tensor\n            // NOTE: In the same way as the python library, we return a batched tensor, regardless of\n            // whether we have a single input or multiple inputs.\n            const dims = [encodedTokens.length, encodedTokens[0].input_ids.length];\n\n            for (const key of Object.keys(encodedTokens[0])) {\n                result[key] = new Tensor('int64',\n                    BigInt64Array.from(encodedTokens.flatMap(x => x[key]).map(BigInt)),\n                    dims\n                );\n            }\n\n        } else {\n            for (const key of Object.keys(encodedTokens[0])) {\n                result[key] = encodedTokens.map(x => x[key]);\n            }\n\n            // If not returning a tensor, we match the input type\n            if (!isBatched) {\n                // Input was not batched, so we unwrap\n                for (const key of Object.keys(result)) {\n                    result[key] = result[key][0];\n                }\n            }\n        }\n\n        return /** @type {BatchEncoding} */(result);\n    }\n\n    /**\n     * Encodes a single text using the preprocessor pipeline of the tokenizer.\n     *\n     * @param {string|null} text The text to encode.\n     * @returns {string[]|null} The encoded tokens.\n     */\n    _encode_text(text) {\n        if (text === null) return null;\n\n        // Actual function which does encoding, for a single text\n        // First, we take care of special tokens. Needed to avoid issues arising from\n        // normalization and/or pretokenization (which may not preserve special tokens)\n        const sections = this.added_tokens_regex ? text.split(this.added_tokens_regex).filter(x => x) : [text];\n\n        const tokens = sections.map((x, section_index) => {\n            const addedToken = this.added_tokens.find(t => t.content === x);\n            if (addedToken !== undefined) {\n                // Ignore added tokens\n                return x\n            } else {\n                if (this.remove_space === true) {\n                    x = x.trim().split(/\\s+/).join(' ');\n                }\n                if (this.do_lowercase_and_remove_accent) {\n                    x = lowercase_and_remove_accent(x);\n                }\n\n                if (this.normalizer !== null) {\n                    x = this.normalizer(x);\n                }\n\n                // If, after normalization, this section is empty (e.g., trimming whitespace),\n                // we return an empty array\n                if (x.length === 0) {\n                    return [];\n                }\n\n                const sectionTokens = (this.pre_tokenizer !== null) ? this.pre_tokenizer(x, {\n                    section_index,\n                }) : [x];\n\n                const tokens = this.model(sectionTokens);\n\n                return tokens;\n            }\n        }).flat();\n\n        return tokens;\n    }\n\n    /**\n     * Encodes a single text or a pair of texts using the model's tokenizer.\n     *\n     * @param {string} text The text to encode.\n     * @param {string|null} text_pair The optional second text to encode.\n     * @param {Object} options An optional object containing the following properties:\n     * @param {boolean} [options.add_special_tokens=true] Whether or not to add the special tokens associated with the corresponding model.\n     * @returns {EncodingSingle} An object containing the encoded text.\n     * @private\n     */\n    _encode_plus(text, text_pair = null, {\n        add_special_tokens = true,\n    } = {}) {\n        // Function called by users to encode possibly multiple texts\n        const tokens = this._encode_text(text);\n        const tokens2 = this._encode_text(text_pair);\n\n        const combinedTokens = this.post_processor\n            ? this.post_processor(tokens, tokens2, { add_special_tokens })\n            : { tokens: mergeArrays(tokens ?? [], tokens2 ?? []) };\n\n        const input_ids = this.model.convert_tokens_to_ids(combinedTokens.tokens);\n\n        const result = {\n            input_ids,\n            attention_mask: new Array(input_ids.length).fill(1),\n        }\n        if (this.return_token_type_ids && combinedTokens.token_type_ids) {\n            result.token_type_ids = combinedTokens.token_type_ids;\n        }\n        return result;\n    }\n\n    /**\n     * Encodes a single text or a pair of texts using the model's tokenizer.\n     *\n     * @param {string} text The text to encode.\n     * @param {string|null} text_pair The optional second text to encode.\n     * @param {Object} options An optional object containing the following properties:\n     * @param {boolean} [options.add_special_tokens=true] Whether or not to add the special tokens associated with the corresponding model.\n     * @returns {number[]} An array of token IDs representing the encoded text(s).\n     */\n    encode(text, text_pair = null, {\n        add_special_tokens = true,\n    } = {}) {\n        const { input_ids } = this._encode_plus(text, text_pair, {\n            add_special_tokens,\n        });\n        return input_ids;\n    }\n\n    /**\n     * Decode a batch of tokenized sequences.\n     * @param {number[][]|Tensor} batch List/Tensor of tokenized input sequences.\n     * @param {Object} decode_args (Optional) Object with decoding arguments.\n     * @returns {string[]} List of decoded sequences.\n     */\n    batch_decode(batch, decode_args = {}) {\n        if (batch instanceof Tensor) {\n            batch = batch.tolist();\n        }\n        return batch.map(x => this.decode(x, decode_args));\n    }\n\n    /**\n     * Decodes a sequence of token IDs back to a string.\n     *\n     * @param {number[]|Tensor} token_ids List/Tensor of token IDs to decode.\n     * @param {Object} [decode_args={}]\n     * @param {boolean} [decode_args.skip_special_tokens=false] If true, special tokens are removed from the output string.\n     * @param {boolean} [decode_args.clean_up_tokenization_spaces=true] If true, spaces before punctuations and abbreviated forms are removed.\n     *\n     * @returns {string} The decoded string.\n     * @throws {Error} If `token_ids` is not a non-empty array of integers.\n     */\n    decode(\n        token_ids,\n        decode_args = {},\n    ) {\n        if (token_ids instanceof Tensor) {\n            token_ids = prepareTensorForDecode(token_ids);\n        }\n\n        if (!Array.isArray(token_ids) || token_ids.length === 0 || !isIntegralNumber(token_ids[0])) {\n            throw Error(\"token_ids must be a non-empty array of integers.\");\n        }\n\n        return this.decode_single(token_ids, decode_args)\n    }\n\n    /**\n     * Decode a single list of token ids to a string.\n     * @param {number[]} token_ids List of token ids to decode\n     * @param {Object} decode_args Optional arguments for decoding\n     * @param {boolean} [decode_args.skip_special_tokens=false] Whether to skip special tokens during decoding\n     * @param {boolean} [decode_args.clean_up_tokenization_spaces=null] Whether to clean up tokenization spaces during decoding.\n     * If null, the value is set to `this.decoder.cleanup` if it exists, falling back to `this.clean_up_tokenization_spaces` if it exists, falling back to `true`.\n     * @returns {string} The decoded string\n     */\n    decode_single(\n        token_ids,\n        {\n            skip_special_tokens = false,\n            clean_up_tokenization_spaces = null,\n        }\n    ) {\n        let tokens = this.model.convert_ids_to_tokens(token_ids);\n        if (skip_special_tokens) {\n            tokens = tokens.filter(x => !this.special_tokens.includes(x));\n        }\n\n        // If `this.decoder` is null, we just join tokens with a space:\n        // https://github.com/huggingface/tokenizers/blob/8edec536a737cb04494b454805be16c020abb14f/tokenizers/src/tokenizer/mod.rs#L835\n        /** @type {string} */\n        let decoded = this.decoder ? this.decoder(tokens) : tokens.join(' ');\n\n        // Slight hack, but prevents having to pass `skip_special_tokens` to\n        // each call to `decode`, which would lead to code duplication.\n        if (this.decoder && this.decoder.end_of_word_suffix) {\n            decoded = decoded.replaceAll(this.decoder.end_of_word_suffix, ' ');\n            if (skip_special_tokens) {\n                decoded = decoded.trim();\n            }\n        }\n\n        if (clean_up_tokenization_spaces ?? this.clean_up_tokenization_spaces) {\n            decoded = clean_up_tokenization(decoded);\n        }\n\n        return decoded;\n    }\n\n    get default_chat_template() {\n        if (!this._warned_about_chat_template) {\n            console.warn(\n                \"No chat template is defined for this tokenizer - using a default chat template \" +\n                \"that implements the ChatML format. If the default is not appropriate for \" +\n                \"your model, please set `tokenizer.chat_template` to an appropriate template. \" +\n                \"See https://huggingface.co/docs/transformers/main/chat_templating for more information.\"\n            )\n            this._warned_about_chat_template = true; // TODO move to logger.warning_once()\n        }\n\n        return this._default_chat_template;\n    }\n\n    /**\n     * Converts a list of message objects with `\"role\"` and `\"content\"` keys to a list of token\n     * ids. This method is intended for use with chat models, and will read the tokenizer's chat_template attribute to\n     * determine the format and control tokens to use when converting. When chat_template is None, it will fall back\n     * to the default_chat_template specified at the class level.\n     * \n     * See [here](https://huggingface.co/docs/transformers/chat_templating) for more information.\n     * \n     * **Example:** Applying a chat template to a conversation.\n     * \n     * ```javascript\n     * import { AutoTokenizer } from \"@xenova/transformers\";\n     * \n     * const tokenizer = await AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.1\");\n     * \n     * const chat = [\n     *   { \"role\": \"user\", \"content\": \"Hello, how are you?\" },\n     *   { \"role\": \"assistant\", \"content\": \"I'm doing great. How can I help you today?\" },\n     *   { \"role\": \"user\", \"content\": \"I'd like to show off how chat templating works!\" },\n     * ]\n     * \n     * const text = tokenizer.apply_chat_template(chat, { tokenize: false });\n     * // \"<s>[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?</s> [INST] I'd like to show off how chat templating works! [/INST]\"\n     * \n     * const input_ids = tokenizer.apply_chat_template(chat, { tokenize: true, return_tensor: false });\n     * // [1, 733, 16289, 28793, 22557, 28725, 910, 460, 368, 28804, 733, 28748, 16289, 28793, 28737, 28742, 28719, 2548, 1598, 28723, 1602, 541, 315, 1316, 368, 3154, 28804, 2, 28705, 733, 16289, 28793, 315, 28742, 28715, 737, 298, 1347, 805, 910, 10706, 5752, 1077, 3791, 28808, 733, 28748, 16289, 28793]\n     * ```\n     * \n     * @param {Message[]} conversation A list of message objects with `\"role\"` and `\"content\"` keys.\n     * @param {Object} options An optional object containing the following properties:\n     * @param {string} [options.chat_template=null] A Jinja template to use for this conversion. If\n     * this is not passed, the model's default chat template will be used instead.\n     * @param {boolean} [options.add_generation_prompt=false] Whether to end the prompt with the token(s) that indicate\n     * the start of an assistant message. This is useful when you want to generate a response from the model.\n     * Note that this argument will be passed to the chat template, and so it must be supported in the\n     * template for this argument to have any effect.\n     * @param {boolean} [options.tokenize=true] Whether to tokenize the output. If false, the output will be a string.\n     * @param {boolean} [options.padding=false] Whether to pad sequences to the maximum length. Has no effect if tokenize is false.\n     * @param {boolean} [options.truncation=false] Whether to truncate sequences to the maximum length. Has no effect if tokenize is false.\n     * @param {number} [options.max_length=null] Maximum length (in tokens) to use for padding or truncation. Has no effect if tokenize is false.\n     * If not specified, the tokenizer's `max_length` attribute will be used as a default.\n     * @param {boolean} [options.return_tensor=true] Whether to return the output as a Tensor or an Array. Has no effect if tokenize is false.\n     * @param {Object} [options.tokenizer_kwargs={}] Additional options to pass to the tokenizer.\n     * @returns {string | Tensor | number[]| number[][]} The tokenized output.\n     */\n    apply_chat_template(conversation, {\n        chat_template = null,\n        add_generation_prompt = false,\n        tokenize = true,\n        padding = false,\n        truncation = false,\n        max_length = null,\n        return_tensor = true,\n        tokenizer_kwargs = {},\n        ...kwargs\n    } = {}) {\n\n        // First, handle the cases when the model has a dict of multiple templates\n        if (\n            (this.chat_template && typeof this.chat_template === 'object') ||\n            (this.chat_template === null && this.default_chat_template && typeof this.default_chat_template === 'object')\n        ) {\n            const template_dict = this.chat_template ?? this.default_chat_template; // Guaranteed to be a non-null object\n\n            if (chat_template !== null && Object.hasOwn(template_dict, chat_template)) {\n                // The user can pass the name of a template to the chat template argument instead of an entire template\n                chat_template = template_dict[chat_template];\n            } else if (chat_template === null && 'default' in template_dict) {\n                chat_template = template_dict['default'];\n            } else if (chat_template === null) {\n                throw Error(\n                    `This model has multiple chat templates with no default specified! Please either pass a chat ` +\n                    `template or the name of the template you wish to use to the 'chat_template' argument. Available ` +\n                    `template names are ${Object.keys(template_dict).sort()}.`\n                )\n            }\n        } else {\n            // These are the cases when the model has a single template\n            // priority: `chat_template` argument > `tokenizer.chat_template` > `tokenizer.default_chat_template\n            chat_template ??= this.chat_template ?? this.default_chat_template;\n        }\n        if (typeof chat_template !== 'string') {\n            throw Error(`chat_template must be a string, but got ${typeof chat_template}`);\n        }\n\n        // Compilation function uses a cache to avoid recompiling the same template\n        let compiledTemplate = this._compiled_template_cache.get(chat_template);\n        if (compiledTemplate === undefined) {\n            compiledTemplate = new Template(chat_template);\n            this._compiled_template_cache.set(chat_template, compiledTemplate);\n        }\n\n        const special_tokens_map = Object.create(null);\n        for (const key of SPECIAL_TOKEN_ATTRIBUTES) {\n            const value = this.getToken(key);\n            if (value) {\n                special_tokens_map[key] = value;\n            }\n        }\n\n        const rendered = compiledTemplate.render({\n            messages: conversation,\n            add_generation_prompt: add_generation_prompt,\n\n            ...special_tokens_map,\n            ...kwargs,\n        });\n\n        if (tokenize) {\n            return this._call(rendered, {\n                add_special_tokens: false,\n                padding,\n                truncation,\n                max_length,\n                return_tensor,\n                ...tokenizer_kwargs,\n            }).input_ids;\n        }\n\n        return rendered;\n    }\n}\n\n/**\n * BertTokenizer is a class used to tokenize text for BERT models.\n * @extends PreTrainedTokenizer\n */\nexport class BertTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\n/**\n * Albert tokenizer\n * @extends PreTrainedTokenizer\n */\nexport class AlbertTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class MobileBertTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class SqueezeBertTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class DebertaTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class DebertaV2Tokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class HerbertTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class ConvBertTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class RoFormerTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\nexport class DistilBertTokenizer extends PreTrainedTokenizer { }\nexport class CamembertTokenizer extends PreTrainedTokenizer { }\nexport class XLMTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n\n    constructor(tokenizerJSON, tokenizerConfig) {\n        super(tokenizerJSON, tokenizerConfig);\n        console.warn('WARNING: `XLMTokenizer` is not yet supported by Hugging Face\\'s \"fast\" tokenizers library. Therefore, you may experience slightly inaccurate results.')\n    }\n}\nexport class ElectraTokenizer extends PreTrainedTokenizer {\n    return_token_type_ids = true;\n}\n\nexport class T5Tokenizer extends PreTrainedTokenizer { }\nexport class GPT2Tokenizer extends PreTrainedTokenizer {\n    _default_chat_template = `{% for message in messages %}\" \"{{ message.content }}{{ eos_token }}\" \"{% endfor %}`\n}\nexport class BartTokenizer extends PreTrainedTokenizer { }\nexport class MBartTokenizer extends PreTrainedTokenizer {\n    constructor(tokenizerJSON, tokenizerConfig) {\n        super(tokenizerJSON, tokenizerConfig);\n\n        this.languageRegex = /^[a-z]{2}_[A-Z]{2}$/;\n        this.language_codes = this.special_tokens.filter(x => this.languageRegex.test(x));\n        this.lang_to_token = x => x; // Identity function\n    }\n\n    /**\n     * Helper function to build translation inputs for an `MBartTokenizer`.\n     * @param {string|string[]} raw_inputs The text to tokenize.\n     * @param {Object} tokenizer_options Options to be sent to the tokenizer\n     * @param {Object} generate_kwargs Generation options.\n     * @returns {Object} Object to be passed to the model.\n     */\n    _build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) {\n        return _build_translation_inputs(this, raw_inputs, tokenizer_options, generate_kwargs);\n    }\n}\nexport class MBart50Tokenizer extends MBartTokenizer { } // NOTE: extends MBartTokenizer\n\nexport class RobertaTokenizer extends PreTrainedTokenizer { }\n\nexport class BloomTokenizer extends GPT2Tokenizer { // NOTE: `GPT2Tokenizer` to get the correct chat template\n\n    constructor(tokenizerJSON, tokenizerConfig) {\n        // Override the default (invalid) regex of the pretokenizer.\n        // For more information, see https://github.com/xenova/transformers.js/issues/94\n        const splitChars = '.,!?\\u2026\\u3002\\uff0c\\u3001\\u0964\\u06d4\\u060c';\n        const patternObject = tokenizerJSON.pre_tokenizer?.pretokenizers[0]?.pattern;\n        if (patternObject && patternObject.Regex === ` ?[^(\\\\s|[${splitChars}])]+`) {\n            patternObject.Regex = ` ?[^\\\\s${splitChars}]+`;\n        }\n        super(tokenizerJSON, tokenizerConfig);\n    }\n}\n\nconst SPIECE_UNDERLINE = \"\u2581\";\n\nexport class LlamaTokenizer extends PreTrainedTokenizer {\n    _default_chat_template = `{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif USE_DEFAULT_PROMPT == true and not '<<SYS>>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<<SYS>>\\n' + content.strip() + '\\n<</SYS>>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}`\n\n    DEFAULT_SYSTEM_PROMPT =\n        \"You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your \" +\n        \"answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure \" +\n        \"that your responses are socially unbiased and positive in nature.\\n\\n\" +\n        \"If a question does not make any sense, or is not factually coherent, explain why instead of answering something not \" +\n        \"correct. If you don't know the answer to a question, please don't share false information.\"\n\n    constructor(tokenizerJSON, tokenizerConfig) {\n        super(tokenizerJSON, tokenizerConfig);\n        this.use_default_system_prompt = tokenizerConfig.use_default_system_prompt ?? false;\n\n        this.legacy = tokenizerConfig.legacy ?? true;\n        if (!this.legacy) {\n            // See https://github.com/huggingface/transformers/pull/24565 for more information\n            this.normalizer = null;\n            this.pre_tokenizer = new MetaspacePreTokenizer({\n                replacement: SPIECE_UNDERLINE,\n                add_prefix_space: true,\n                prepend_scheme: \"first\",\n            });\n        }\n    }\n\n    /**\n     * Helper function to handle legacy encoding of SPM tokenizers.\n     * Adapted from https://github.com/huggingface/transformers/blob/e6dcf8abd6f65bb4b6dfc1831b20d9ba49ce00e2/src/transformers/models/t5/tokenization_t5.py#L374-L387\n     * @param {string} text The text to encode.\n     * @returns {string[]} The encoded tokens.\n     */\n    _encode_text(text) {\n        if (text === null) return null;\n\n        if (this.legacy || text.length === 0) {\n            return super._encode_text(text);\n        }\n\n        let tokens = super._encode_text(SPIECE_UNDERLINE + text.replaceAll(SPIECE_UNDERLINE, \" \"));\n        if (tokens.length > 1 && tokens[0] === SPIECE_UNDERLINE && this.special_tokens.includes(tokens[1])) {\n            tokens = tokens.slice(1);\n        }\n        return tokens;\n    }\n\n    get default_chat_template() {\n        return super.default_chat_template\n            .replaceAll('USE_DEFAULT_PROMPT', this.use_default_system_prompt ? 'true' : 'false')\n            .replaceAll('DEFAULT_SYSTEM_MESSAGE', this.DEFAULT_SYSTEM_PROMPT.replaceAll(\"\\n\", \"\\\\n\").replaceAll(\"'\", \"\\\\'\"));\n    }\n}\nexport class CodeLlamaTokenizer extends LlamaTokenizer { } // NOTE: `LlamaTokenizer` to get the correct chat template\n\nexport class XLMRobertaTokenizer extends PreTrainedTokenizer { }\nexport class MPNetTokenizer extends PreTrainedTokenizer { }\n\nexport class FalconTokenizer extends PreTrainedTokenizer { }\n\nexport class GPTNeoXTokenizer extends PreTrainedTokenizer { }\n\nexport class EsmTokenizer extends PreTrainedTokenizer { }\n\nexport class Qwen2Tokenizer extends PreTrainedTokenizer { }\n\nexport class GemmaTokenizer extends PreTrainedTokenizer {\n    _default_chat_template = \"{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\\n' + message['content'] | trim + '<end_of_turn>\\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\\n'}}{% endif %}\"\n}\n\nexport class Grok1Tokenizer extends PreTrainedTokenizer { }\n\n/**\n * Helper function to build translation inputs for an `NllbTokenizer` or `M2M100Tokenizer`.\n * @param {PreTrainedTokenizer} self The tokenizer instance.\n * @param {string|string[]} raw_inputs The text to tokenize.\n * @param {Object} tokenizer_options Options to be sent to the tokenizer\n * @param {Object} generate_kwargs Generation options.\n * @returns {Object} Object to be passed to the model.\n * @private\n */\nfunction _build_translation_inputs(self, raw_inputs, tokenizer_options, generate_kwargs) {\n    if (!('language_codes' in self) || !Array.isArray(self.language_codes)) {\n        throw new Error('Tokenizer must have `language_codes` attribute set and it should be an array of language ids.')\n    }\n    if (!('languageRegex' in self) || !(self.languageRegex instanceof RegExp)) {\n        throw new Error('Tokenizer must have `languageRegex` attribute set and it should be a regular expression.')\n    }\n    if (!('lang_to_token' in self) || typeof self.lang_to_token !== 'function') {\n        throw new Error('Tokenizer must have `lang_to_token` attribute set and it should be a function.')\n    }\n    const src_lang_token = generate_kwargs.src_lang;\n    const tgt_lang_token = generate_kwargs.tgt_lang;\n\n    // Check that the target language is valid:\n    if (!self.language_codes.includes(tgt_lang_token)) {\n        throw new Error(`Target language code \"${tgt_lang_token}\" is not valid. Must be one of: {${self.language_codes.join(', ')}}`);\n    }\n\n    // Allow `src_lang` to be optional. If not set, we'll use the tokenizer's default.\n    if (src_lang_token !== undefined) {\n        // Check that the source language is valid:\n        if (!self.language_codes.includes(src_lang_token)) {\n            throw new Error(`Source language code \"${src_lang_token}\" is not valid. Must be one of: {${self.language_codes.join(', ')}}`);\n        }\n\n        // In the same way as the Python library, we override the post-processor\n        // to force the source language to be first:\n        for (const item of self.post_processor.config.single) {\n            if ('SpecialToken' in item && self.languageRegex.test(item.SpecialToken.id)) {\n                item.SpecialToken.id = self.lang_to_token(src_lang_token);\n                break;\n            }\n        }\n        // TODO: Do the same for pair?\n    }\n\n    // Override the `forced_bos_token_id` to force the correct language\n    generate_kwargs.forced_bos_token_id = self.model.convert_tokens_to_ids([self.lang_to_token(tgt_lang_token)])[0];\n\n    return self._call(raw_inputs, tokenizer_options);\n}\n\n/**\n * The NllbTokenizer class is used to tokenize text for NLLB (\"No Language Left Behind\") models.\n * \n * No Language Left Behind (NLLB) is a first-of-its-kind, AI breakthrough project\n * that open-sources models capable of delivering high-quality translations directly\n * between any pair of 200+ languages \u2014 including low-resource languages like Asturian,\n * Luganda, Urdu and more. It aims to help people communicate with anyone, anywhere,\n * regardless of their language preferences. For more information, check out their\n * [paper](https://arxiv.org/abs/2207.04672).\n * \n * For a list of supported languages (along with their language codes),\n * @see {@link https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200}\n */\nexport class NllbTokenizer extends PreTrainedTokenizer {\n\n    constructor(tokenizerJSON, tokenizerConfig) {\n        super(tokenizerJSON, tokenizerConfig);\n\n        this.languageRegex = /^[a-z]{3}_[A-Z][a-z]{3}$/;\n        this.language_codes = this.special_tokens.filter(x => this.languageRegex.test(x));\n        this.lang_to_token = x => x; // Identity function\n    }\n\n    /**\n     * Helper function to build translation inputs for an `NllbTokenizer`.\n     * @param {string|string[]} raw_inputs The text to tokenize.\n     * @param {Object} tokenizer_options Options to be sent to the tokenizer\n     * @param {Object} generate_kwargs Generation options.\n     * @returns {Object} Object to be passed to the model.\n     */\n    _build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) {\n        return _build_translation_inputs(this, raw_inputs, tokenizer_options, generate_kwargs);\n    }\n}\n\n/**\n * The M2M100Tokenizer class is used to tokenize text for M2M100 (\"Many-to-Many\") models.\n * \n * M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many\n * multilingual translation. It was introduced in this [paper](https://arxiv.org/abs/2010.11125)\n * and first released in [this](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100) repository.\n * \n * For a list of supported languages (along with their language codes),\n * @see {@link https://huggingface.co/facebook/m2m100_418M#languages-covered}\n */\nexport class M2M100Tokenizer extends PreTrainedTokenizer {\n    constructor(tokenizerJSON, tokenizerConfig) {\n        super(tokenizerJSON, tokenizerConfig);\n\n        this.languageRegex = /^__[a-z]{2,3}__$/;\n        this.language_codes = this.special_tokens\n            .filter(x => this.languageRegex.test(x))\n            .map(x => x.slice(2, -2));\n        this.lang_to_token = x => `__${x}__`;\n    }\n\n    /**\n     * Helper function to build translation inputs for an `M2M100Tokenizer`.\n     * @param {string|string[]} raw_inputs The text to tokenize.\n     * @param {Object} tokenizer_options Options to be sent to the tokenizer\n     * @param {Object} generate_kwargs Generation options.\n     * @returns {Object} Object to be passed to the model.\n     */\n    _build_translation_inputs(raw_inputs, tokenizer_options, generate_kwargs) {\n        return _build_translation_inputs(this, raw_inputs, tokenizer_options, generate_kwargs);\n    }\n}\n\n\nconst WHISPER_LANGUAGES = [\n    [\"en\", \"english\"],\n    [\"zh\", \"chinese\"],\n    [\"de\", \"german\"],\n    [\"es\", \"spanish\"],\n    [\"ru\", \"russian\"],\n    [\"ko\", \"korean\"],\n    [\"fr\", \"french\"],\n    [\"ja\", \"japanese\"],\n    [\"pt\", \"portuguese\"],\n    [\"tr\", \"turkish\"],\n    [\"pl\", \"polish\"],\n    [\"ca\", \"catalan\"],\n    [\"nl\", \"dutch\"],\n    [\"ar\", \"arabic\"],\n    [\"sv\", \"swedish\"],\n    [\"it\", \"italian\"],\n    [\"id\", \"indonesian\"],\n    [\"hi\", \"hindi\"],\n    [\"fi\", \"finnish\"],\n    [\"vi\", \"vietnamese\"],\n    [\"he\", \"hebrew\"],\n    [\"uk\", \"ukrainian\"],\n    [\"el\", \"greek\"],\n    [\"ms\", \"malay\"],\n    [\"cs\", \"czech\"],\n    [\"ro\", \"romanian\"],\n    [\"da\", \"danish\"],\n    [\"hu\", \"hungarian\"],\n    [\"ta\", \"tamil\"],\n    [\"no\", \"norwegian\"],\n    [\"th\", \"thai\"],\n    [\"ur\", \"urdu\"],\n    [\"hr\", \"croatian\"],\n    [\"bg\", \"bulgarian\"],\n    [\"lt\", \"lithuanian\"],\n    [\"la\", \"latin\"],\n    [\"mi\", \"maori\"],\n    [\"ml\", \"malayalam\"],\n    [\"cy\", \"welsh\"],\n    [\"sk\", \"slovak\"],\n    [\"te\", \"telugu\"],\n    [\"fa\", \"persian\"],\n    [\"lv\", \"latvian\"],\n    [\"bn\", \"bengali\"],\n    [\"sr\", \"serbian\"],\n    [\"az\", \"azerbaijani\"],\n    [\"sl\", \"slovenian\"],\n    [\"kn\", \"kannada\"],\n    [\"et\", \"estonian\"],\n    [\"mk\", \"macedonian\"],\n    [\"br\", \"breton\"],\n    [\"eu\", \"basque\"],\n    [\"is\", \"icelandic\"],\n    [\"hy\", \"armenian\"],\n    [\"ne\", \"nepali\"],\n    [\"mn\", \"mongolian\"],\n    [\"bs\", \"bosnian\"],\n    [\"kk\", \"kazakh\"],\n    [\"sq\", \"albanian\"],\n    [\"sw\", \"swahili\"],\n    [\"gl\", \"galician\"],\n    [\"mr\", \"marathi\"],\n    [\"pa\", \"punjabi\"],\n    [\"si\", \"sinhala\"],\n    [\"km\", \"khmer\"],\n    [\"sn\", \"shona\"],\n    [\"yo\", \"yoruba\"],\n    [\"so\", \"somali\"],\n    [\"af\", \"afrikaans\"],\n    [\"oc\", \"occitan\"],\n    [\"ka\", \"georgian\"],\n    [\"be\", \"belarusian\"],\n    [\"tg\", \"tajik\"],\n    [\"sd\", \"sindhi\"],\n    [\"gu\", \"gujarati\"],\n    [\"am\", \"amharic\"],\n    [\"yi\", \"yiddish\"],\n    [\"lo\", \"lao\"],\n    [\"uz\", \"uzbek\"],\n    [\"fo\", \"faroese\"],\n    [\"ht\", \"haitian creole\"],\n    [\"ps\", \"pashto\"],\n    [\"tk\", \"turkmen\"],\n    [\"nn\", \"nynorsk\"],\n    [\"mt\", \"maltese\"],\n    [\"sa\", \"sanskrit\"],\n    [\"lb\", \"luxembourgish\"],\n    [\"my\", \"myanmar\"],\n    [\"bo\", \"tibetan\"],\n    [\"tl\", \"tagalog\"],\n    [\"mg\", \"malagasy\"],\n    [\"as\", \"assamese\"],\n    [\"tt\", \"tatar\"],\n    [\"haw\", \"hawaiian\"],\n    [\"ln\", \"lingala\"],\n    [\"ha\", \"hausa\"],\n    [\"ba\", \"bashkir\"],\n    [\"jw\", \"javanese\"],\n    [\"su\", \"sundanese\"],\n]\n\n...\n", "needle_token_start": 2339, "needle_token_end": 2457, "code_context_ntokens": 16146, "output": ["```javascript\n/**\n * Helper function for padding values of an object, which are each arrays.\n * NOTE: No additional checks are made here for validity of arguments.\n * @param {Record<string, any[]>} item The input object.\n * @param {number} length The length to pad to.\n * @param {(key: string) => any} value_fn Determine the value to fill the array, based on its key.\n * @param {'right'|'left'} side Which side to pad the array.\n * @private\n */\nfunction padHelper(item, length, value_fn, side) {\n    for (const key of Object.keys(item)) {\n        const diff = length - item[key].length;\n        const value = value_fn(key);\n\n        const padData = new Array(diff).fill(value);\n        item[key] = side === 'right'\n            ? mergeArrays(item[key], padData)\n            : mergeArrays(padData, item[key]);\n    }\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "prepareImages", "language": "typescript", "path": "src/pipelines.js", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To convert a variety of image formats into a standardized format suitable for further processing in machine learning tasks.\n2. **Input**: A single image or an array of images, where each image can be a string URL, a path, or a raw image object.\n3. **Output**: A promise that resolves to an array of processed raw image objects.\n4. **Procedure**: The function first ensures the input is in array form. It then processes each item in the array by converting it into a raw image object, handling different input formats like URLs or paths. This conversion is done concurrently for all items in the array.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/tokenizers.js\n\n/**\n * @file Tokenizers are used to prepare textual inputs for a model.\n * \n * **Example:** Create an `AutoTokenizer` and use it to tokenize a sentence.\n * This will automatically detect the tokenizer type based on the tokenizer class defined in `tokenizer.json`.\n * ```javascript\n * import { AutoTokenizer } from '@xenova/transformers';\n * \n * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');\n * const { input_ids } = await tokenizer('I love transformers!');\n * // Tensor {\n * //   data: BigInt64Array(6) [101n, 1045n, 2293n, 19081n, 999n, 102n],\n * //   dims: [1, 6],\n * //   type: 'int64',\n * //   size: 6,\n * // }\n * ```\n * \n * @module tokenizers\n */\n\nimport {\n    Callable,\n    reverseDictionary,\n    escapeRegExp,\n    isIntegralNumber,\n    mergeArrays,\n} from './utils/core.js';\n\nimport {\n    getModelJSON,\n} from './utils/hub.js';\n\nimport { max, min, round } from './utils/maths.js';\nimport { Tensor } from './utils/tensor.js';\n\nimport {\n    PriorityQueue,\n    TokenLattice,\n    CharTrie,\n} from './utils/data-structures.js';\n\nimport { Template } from '@huggingface/jinja';\n\n\n/**\n * @typedef {Object} TokenizerProperties Additional tokenizer-specific properties.\n * @property {boolean} [legacy=false] Whether or not the `legacy` behavior of the tokenizer should be used.\n * @typedef {import('./utils/hub.js').PretrainedOptions & TokenizerProperties} PretrainedTokenizerOptions\n */\n\n/**\n * Loads a tokenizer from the specified path.\n * @param {string} pretrained_model_name_or_path The path to the tokenizer directory.\n * @param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer.\n * @returns {Promise<any[]>} A promise that resolves with information about the loaded tokenizer.\n */\nasync function loadTokenizer(pretrained_model_name_or_path, options) {\n\n    const info = await Promise.all([\n        getModelJSON(pretrained_model_name_or_path, 'tokenizer.json', true, options),\n        getModelJSON(pretrained_model_name_or_path, 'tokenizer_config.json', true, options),\n    ])\n\n    // Override legacy option if `options.legacy` is not null\n    if (options.legacy !== null) {\n        info[1].legacy = options.legacy;\n    }\n    return info;\n}\n\n\n/**\n * Helper function to split a string on a regex, but keep the delimiters.\n * This is required, because the JavaScript `.split()` method does not keep the delimiters,\n * and wrapping in a capturing group causes issues with existing capturing groups (due to nesting).\n * @param {string} text The text to split.\n * @param {RegExp} regex The regex to split on.\n * @returns {string[]} The split string.\n */\nfunction regexSplit(text, regex) {\n    const result = [];\n    let prev = 0;\n    for (const match of text.matchAll(regex)) {\n        const fullMatch = match[0];\n        if (prev < match.index) {\n            result.push(text.slice(prev, match.index));\n        }\n        if (fullMatch.length > 0) {\n            result.push(fullMatch);\n        }\n        prev = match.index + fullMatch.length;\n    }\n    if (prev < text.length) {\n        result.push(text.slice(prev));\n    }\n    return result;\n}\n\n\n/**\n * Helper method to construct a pattern from a config object.\n * @param {Object} pattern The pattern object.\n * @param {boolean} invert Whether to invert the pattern.\n * @returns {RegExp|null} The compiled pattern.\n */\nfunction createPattern(pattern, invert = true) {\n\n    if (pattern.Regex !== undefined) {\n        // In certain cases, the pattern may contain unnecessary escape sequences (e.g., \\# or \\& or \\~).\n        // i.e., valid in Python (where the patterns are exported from) but invalid in JavaScript (where the patterns are parsed).\n        // This isn't an issue when creating the regex w/o the 'u' flag, but it is when the 'u' flag is used.\n        // For this reason, it is necessary to remove these backslashes before creating the regex.\n        // See https://stackoverflow.com/a/63007777/13989043 for more information\n        let regex = pattern.Regex.replace(/\\\\([#&~])/g, '$1'); // TODO: add more characters to this list if necessary\n\n        // We also handle special cases where the regex contains invalid (non-JS compatible) syntax.\n        for (const [key, value] of PROBLEMATIC_REGEX_MAP) {\n            regex = regex.replaceAll(key, value);\n        }\n\n        return new RegExp(regex, 'gu');\n\n    } else if (pattern.String !== undefined) {\n        const escaped = escapeRegExp(pattern.String);\n        // NOTE: if invert is true, we wrap the pattern in a group so that it is kept when performing .split()\n        return new RegExp(invert ? escaped : `(${escaped})`, 'gu');\n\n    } else {\n        console.warn('Unknown pattern type:', pattern)\n        return null;\n    }\n}\n\n/**\n * Helper function to convert an Object to a Map\n * @param {Object} obj The object to convert.\n * @returns {Map<string, any>} The map.\n */\nfunction objectToMap(obj) {\n    return new Map(Object.entries(obj));\n}\n\n/**\n * Helper function to convert a tensor to a list before decoding.\n * @param {Tensor} tensor The tensor to convert.\n * @returns {number[]} The tensor as a list.\n */\nfunction prepareTensorForDecode(tensor) {\n    const dims = tensor.dims;\n    switch (dims.length) {\n        case 1:\n            return tensor.tolist();\n        case 2:\n            if (dims[0] !== 1) {\n                throw new Error('Unable to decode tensor with `batch size !== 1`. Use `tokenizer.batch_decode(...)` for batched inputs.');\n            }\n            return tensor.tolist()[0];\n        default:\n            throw new Error(`Expected tensor to have 1-2 dimensions, got ${dims.length}.`)\n    }\n}\n\n/**\n * Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms\n * @param {string} text The text to clean up.\n * @returns {string} The cleaned up text.\n */\nfunction clean_up_tokenization(text) {\n    // Clean up a list of simple English tokenization artifacts\n    // like spaces before punctuations and abbreviated forms\n    return text.replace(/ \\./g, '.')\n        .replace(/ \\?/g, '?')\n        .replace(/ \\!/g, '!')\n        .replace(/ ,/g, ',')\n        .replace(/ \\' /g, \"'\")\n        .replace(/ n\\'t/g, \"n't\")\n        .replace(/ \\'m/g, \"'m\")\n        .replace(/ \\'s/g, \"'s\")\n        .replace(/ \\'ve/g, \"'ve\")\n        .replace(/ \\'re/g, \"'re\");\n}\n\n/**\n * Helper function to remove accents from a string.\n * @param {string} text The text to remove accents from.\n * @returns {string} The text with accents removed.\n */\nfunction remove_accents(text) {\n    return text.replace(/[\\u0300-\\u036f]/g, '');\n}\n\n/**\n * Helper function to lowercase a string and remove accents.\n * @param {string} text The text to lowercase and remove accents from.\n * @returns {string} The lowercased text with accents removed.\n */\nfunction lowercase_and_remove_accent(text) {\n    return remove_accents(text.toLowerCase());\n}\n\n/**\n * Helper function to fuse consecutive values in an array equal to the specified value.\n * @param {string[]} arr The input array\n * @param {any} value The value to fuse on.\n * @param {Map<string, any>} mapping The mapping from input domain to value.\n */\nfunction fuse(arr, value, mapping) {\n    const fused = [];\n    let i = 0;\n    while (i < arr.length) {\n        fused.push(arr[i])\n        if ((mapping.get(arr[i]) ?? value) !== value) {\n            ++i;\n            continue;\n        }\n\n        while (i < arr.length && (mapping.get(arr[i]) ?? value) === value) {\n            ++i;\n        }\n    }\n\n    return fused;\n}\n\n/**\n * Split a string on whitespace.\n * @param {string} text The text to split.\n * @returns {string[]} The split string.\n */\nfunction whitespace_split(text) {\n    return text.match(/\\S+/g) || [];\n}\n\nconst PUNCTUATION_REGEX = '\\\\p{P}\\\\u0021-\\\\u002F\\\\u003A-\\\\u0040\\\\u005B-\\\\u0060\\\\u007B-\\\\u007E';\n\n// A mapping of regex patterns to their equivalent (but longer) JS-compatible versions.\nconst PROBLEMATIC_REGEX_MAP = new Map([\n    // This uses the case insensitive group modifier, which is not supported in JavaScript.\n    // When parsing the regex, an \"Invalid group\" error is thrown.\n    [\"(?i:'s|'t|'re|'ve|'m|'ll|'d)\", \"(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))\"],\n])\n\n\n/**\n * Represent a token added by the user on top of the existing Model vocabulary.\n * AddedToken can be configured to specify the behavior they should have in various situations like:\n *   - Whether they should only match single words\n *   - Whether to include any whitespace on its left or right\n */\nclass AddedToken {\n    /**\n     * Creates a new instance of AddedToken.\n     * @param {Object} config Added token configuration object.\n     * @param {string} config.content The content of the added token.\n     * @param {number} config.id The id of the added token.\n     * @param {boolean} [config.single_word=false] Whether this token must be a single word or can break words.\n     * @param {boolean} [config.lstrip=false] Whether this token should strip whitespaces on its left.\n     * @param {boolean} [config.rstrip=false] Whether this token should strip whitespaces on its right.\n     * @param {boolean} [config.normalized=false] Whether this token should be normalized.\n     * @param {boolean} [config.special=false] Whether this token is special.\n     */\n    constructor(config) {\n        this.content = config.content;\n        this.id = config.id;\n        this.single_word = config.single_word ?? false;\n        this.lstrip = config.lstrip ?? false;\n        this.rstrip = config.rstrip ?? false;\n        this.special = config.special ?? false;\n        this.normalized = config.normalized ?? null;\n    }\n}\n\n/**\n * Abstract base class for tokenizer models.\n *\n * @extends Callable\n */\nexport class TokenizerModel extends Callable {\n    /**\n     * Creates a new instance of TokenizerModel.\n     * @param {Object} config The configuration object for the TokenizerModel.\n     */\n    constructor(config) {\n        super();\n        this.config = config;\n\n        /** @type {string[]} */\n        this.vocab = [];\n\n        /**\n         * A mapping of tokens to ids.\n         * @type {Map<string, number>}\n         */\n        this.tokens_to_ids = new Map();\n\n        this.unk_token_id = undefined;\n        this.unk_token = undefined;\n        this.end_of_word_suffix = undefined;\n\n        /** @type {boolean} Whether to fuse unknown tokens when encoding. Defaults to false. */\n        this.fuse_unk = this.config.fuse_unk ?? false;\n    }\n\n    /**\n     * Instantiates a new TokenizerModel instance based on the configuration object provided.\n     * @param {Object} config The configuration object for the TokenizerModel.\n...\n// Path: src/pipelines.js\n/**\n * @file Pipelines provide a high-level, easy to use, API for running machine learning models.\n * \n * **Example:** Instantiate pipeline using the `pipeline` function.\n * ```javascript\n * import { pipeline } from '@xenova/transformers';\n * \n * const classifier = await pipeline('sentiment-analysis');\n * const output = await classifier('I love transformers!');\n * // [{'label': 'POSITIVE', 'score': 0.999817686}]\n * ```\n * \n * @module pipelines\n */\n\nimport {\n    AutoTokenizer,\n    PreTrainedTokenizer,\n} from './tokenizers.js';\nimport {\n    AutoModel,\n    AutoModelForSequenceClassification,\n    AutoModelForAudioClassification,\n    AutoModelForTokenClassification,\n    AutoModelForQuestionAnswering,\n    AutoModelForMaskedLM,\n    AutoModelForSeq2SeqLM,\n    AutoModelForSpeechSeq2Seq,\n    AutoModelForTextToWaveform,\n    AutoModelForTextToSpectrogram,\n    AutoModelForCTC,\n    AutoModelForCausalLM,\n    AutoModelForVision2Seq,\n    AutoModelForImageClassification,\n    AutoModelForImageSegmentation,\n    AutoModelForSemanticSegmentation,\n    AutoModelForObjectDetection,\n    AutoModelForZeroShotObjectDetection,\n    AutoModelForDocumentQuestionAnswering,\n    AutoModelForImageToImage,\n    AutoModelForDepthEstimation,\n    AutoModelForImageFeatureExtraction,\n    PreTrainedModel,\n} from './models.js';\nimport {\n    AutoProcessor,\n    Processor\n} from './processors.js';\n\n\nimport {\n    Callable,\n    dispatchCallback,\n    pop,\n    product,\n} from './utils/core.js';\nimport {\n    softmax,\n    max,\n    getTopItems,\n    round,\n} from './utils/maths.js';\nimport {\n    read_audio\n} from './utils/audio.js';\nimport {\n    Tensor,\n    mean_pooling,\n    interpolate,\n    quantize_embeddings,\n} from './utils/tensor.js';\nimport { RawImage } from './utils/image.js';\n\n\n/**\n * @typedef {string | RawImage | URL} ImageInput\n * @typedef {ImageInput|ImageInput[]} ImagePipelineInputs\n */\n\n/**\n * Prepare images for further tasks.\n * @param {ImagePipelineInputs} images images to prepare.\n * @returns {Promise<RawImage[]>} returns processed images.\n * @private\n */\n\nasync function prepareImages(images) {\n    if (!Array.isArray(images)) {\n        images = [images];\n    }\n\n    // Possibly convert any non-images to images\n    return await Promise.all(images.map(x => RawImage.read(x)));\n}\n\n/**\n * @typedef {string | URL | Float32Array | Float64Array} AudioInput\n * @typedef {AudioInput|AudioInput[]} AudioPipelineInputs\n */\n\n/**\n * Prepare audios for further tasks.\n * @param {AudioPipelineInputs} audios audios to prepare.\n * @param {number} sampling_rate sampling rate of the audios.\n * @returns {Promise<Float32Array[]>} The preprocessed audio data.\n * @private\n */\nasync function prepareAudios(audios, sampling_rate) {\n    if (!Array.isArray(audios)) {\n        audios = [audios];\n    }\n\n    return await Promise.all(audios.map(x => {\n        if (typeof x === 'string' || x instanceof URL) {\n            return read_audio(x, sampling_rate);\n        } else if (x instanceof Float64Array) {\n            return new Float32Array(x);\n        }\n        return x;\n    }));\n}\n\n/**\n * @typedef {Object} BoundingBox\n * @property {number} xmin The minimum x coordinate of the bounding box.\n * @property {number} ymin The minimum y coordinate of the bounding box.\n * @property {number} xmax The maximum x coordinate of the bounding box.\n * @property {number} ymax The maximum y coordinate of the bounding box.\n */\n\n/**\n * Helper function to convert list [xmin, xmax, ymin, ymax] into object { \"xmin\": xmin, ... }\n * @param {number[]} box The bounding box as a list.\n * @param {boolean} asInteger Whether to cast to integers.\n * @returns {BoundingBox} The bounding box as an object.\n * @private\n */\nfunction get_bounding_box(box, asInteger) {\n    if (asInteger) {\n        box = box.map(x => x | 0);\n    }\n    const [xmin, ymin, xmax, ymax] = box;\n\n    return { xmin, ymin, xmax, ymax };\n}\n\n\n/**\n * @callback DisposeType Disposes the item.\n * @returns {Promise<void>} A promise that resolves when the item has been disposed.\n * \n * @typedef {Object} Disposable\n * @property {DisposeType} dispose A promise that resolves when the pipeline has been disposed.\n */\n\n/**\n * The Pipeline class is the class from which all pipelines inherit.\n * Refer to this class for methods shared across different pipelines.\n * @extends Callable\n */\nexport class Pipeline extends Callable {\n    /**\n     * Create a new Pipeline.\n     * @param {Object} options An object containing the following properties:\n     * @param {string} [options.task] The task of the pipeline. Useful for specifying subtasks.\n     * @param {PreTrainedModel} [options.model] The model used by the pipeline.\n     * @param {PreTrainedTokenizer} [options.tokenizer=null] The tokenizer used by the pipeline (if any).\n     * @param {Processor} [options.processor=null] The processor used by the pipeline (if any).\n     */\n    constructor({ task, model, tokenizer = null, processor = null }) {\n        super();\n        this.task = task;\n        this.model = model;\n        this.tokenizer = tokenizer;\n        this.processor = processor;\n    }\n\n    /** @type {DisposeType} */\n    async dispose() {\n        await this.model.dispose();\n    }\n}\n\n/**\n * @typedef {Object} ModelTokenizerConstructorArgs\n * @property {string} task The task of the pipeline. Useful for specifying subtasks.\n * @property {PreTrainedModel} model The model used by the pipeline.\n * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline.\n * \n * @typedef {ModelTokenizerConstructorArgs} TextPipelineConstructorArgs An object used to instantiate a text-based pipeline.\n */\n\n/**\n * @typedef {Object} ModelProcessorConstructorArgs\n * @property {string} task The task of the pipeline. Useful for specifying subtasks.\n * @property {PreTrainedModel} model The model used by the pipeline.\n * @property {Processor} processor The processor used by the pipeline.\n * \n * @typedef {ModelProcessorConstructorArgs} AudioPipelineConstructorArgs An object used to instantiate an audio-based pipeline.\n * @typedef {ModelProcessorConstructorArgs} ImagePipelineConstructorArgs An object used to instantiate an image-based pipeline.\n */\n\n\n/**\n * @typedef {Object} ModelTokenizerProcessorConstructorArgs\n * @property {string} task The task of the pipeline. Useful for specifying subtasks.\n * @property {PreTrainedModel} model The model used by the pipeline.\n * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline.\n * @property {Processor} processor The processor used by the pipeline.\n * \n * @typedef {ModelTokenizerProcessorConstructorArgs} TextAudioPipelineConstructorArgs An object used to instantiate a text- and audio-based pipeline.\n * @typedef {ModelTokenizerProcessorConstructorArgs} TextImagePipelineConstructorArgs An object used to instantiate a text- and image-based pipeline.\n */\n\n/**\n * @typedef {Object} TextClassificationSingle\n * @property {string} label The label predicted.\n * @property {number} score The corresponding probability.\n * @typedef {TextClassificationSingle[]} TextClassificationOutput\n * \n * @typedef {Object} TextClassificationPipelineOptions Parameters specific to text classification pipelines.\n * @property {number} [topk=1] The number of top predictions to be returned.\n * \n * @callback TextClassificationPipelineCallback Classify the text(s) given as inputs.\n * @param {string|string[]} texts The input text(s) to be classified.\n * @param {TextClassificationPipelineOptions} [options] The options to use for text classification.\n * @returns {Promise<TextClassificationOutput|TextClassificationOutput[]>} An array or object containing the predicted labels and scores.\n * \n * @typedef {TextPipelineConstructorArgs & TextClassificationPipelineCallback & Disposable} TextClassificationPipelineType\n */\n\n/**\n * Text classification pipeline using any `ModelForSequenceClassification`.\n *\n * **Example:** Sentiment-analysis w/ `Xenova/distilbert-base-uncased-finetuned-sst-2-english`.\n * ```javascript\n * const classifier = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english');\n * const output = await classifier('I love transformers!');\n * // [{ label: 'POSITIVE', score: 0.999788761138916 }]\n * ```\n * \n * **Example:** Multilingual sentiment-analysis w/ `Xenova/bert-base-multilingual-uncased-sentiment` (and return top 5 classes).\n * ```javascript\n * const classifier = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment');\n * const output = await classifier('Le meilleur film de tous les temps.', { topk: 5 });\n * // [\n * //   { label: '5 stars', score: 0.9610759615898132 },\n * //   { label: '4 stars', score: 0.03323351591825485 },\n * //   { label: '3 stars', score: 0.0036155181005597115 },\n * //   { label: '1 star', score: 0.0011325967498123646 },\n * //   { label: '2 stars', score: 0.0009423971059732139 }\n * // ]\n * ```\n * \n * **Example:** Toxic comment classification w/ `Xenova/toxic-bert` (and return all classes).\n * ```javascript\n * const classifier = await pipeline('text-classification', 'Xenova/toxic-bert');\n * const output = await classifier('I hate you!', { topk: null });\n * // [\n * //   { label: 'toxic', score: 0.9593140482902527 },\n * //   { label: 'insult', score: 0.16187334060668945 },\n * //   { label: 'obscene', score: 0.03452680632472038 },\n * //   { label: 'identity_hate', score: 0.0223250575363636 },\n * //   { label: 'threat', score: 0.019197041168808937 },\n * //   { label: 'severe_toxic', score: 0.005651099607348442 }\n * // ]\n * ```\n */\nexport class TextClassificationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TextClassificationPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new TextClassificationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {TextClassificationPipelineCallback} */\n    async _call(texts, {\n        topk = 1\n    } = {}) {\n\n        // Run tokenization\n        const model_inputs = this.tokenizer(texts, {\n            padding: true,\n            truncation: true,\n        });\n\n        // Run model\n        const outputs = await this.model(model_inputs)\n\n        // TODO: Use softmax tensor function\n        const function_to_apply =\n            this.model.config.problem_type === 'multi_label_classification'\n                ? batch => batch.sigmoid().data\n                : batch => softmax(batch.data); // single_label_classification (default)\n\n        const id2label = this.model.config.id2label;\n\n        const toReturn = [];\n        for (const batch of outputs.logits) {\n            const output = function_to_apply(batch);\n            const scores = getTopItems(output, topk);\n\n            const vals = scores.map(x => ({\n                label: id2label[x[0]],\n                score: x[1],\n            }));\n            if (topk === 1) {\n                toReturn.push(...vals);\n            } else {\n                toReturn.push(vals);\n            }\n        }\n\n        return Array.isArray(texts) || topk === 1 ? /** @type {TextClassificationOutput} */ (toReturn) : /** @type {TextClassificationOutput[]} */ (toReturn)[0];\n    }\n}\n\n/**\n * @typedef {Object} TokenClassificationSingle\n * @property {string} word The token/word classified. This is obtained by decoding the selected tokens.\n * @property {number} score The corresponding probability for `entity`.\n * @property {string} entity The entity predicted for that token/word.\n * @property {number} index The index of the corresponding token in the sentence.\n * @property {number} [start] The index of the start of the corresponding entity in the sentence.\n * @property {number} [end] The index of the end of the corresponding entity in the sentence.\n * @typedef {TokenClassificationSingle[]} TokenClassificationOutput\n * \n * @typedef {Object} TokenClassificationPipelineOptions Parameters specific to token classification pipelines.\n * @property {string[]} [ignore_labels] A list of labels to ignore.\n * \n * @callback TokenClassificationPipelineCallback Classify each token of the text(s) given as inputs.\n * @param {string|string[]} texts One or several texts (or one list of texts) for token classification.\n * @param {TokenClassificationPipelineOptions} [options] The options to use for token classification.\n * @returns {Promise<TokenClassificationOutput|TokenClassificationOutput[]>} The result.\n * \n * @typedef {TextPipelineConstructorArgs & TokenClassificationPipelineCallback & Disposable} TokenClassificationPipelineType\n */\n\n/**\n * Named Entity Recognition pipeline using any `ModelForTokenClassification`.\n * \n * **Example:** Perform named entity recognition with `Xenova/bert-base-NER`.\n * ```javascript\n * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER');\n * const output = await classifier('My name is Sarah and I live in London');\n * // [\n * //   { entity: 'B-PER', score: 0.9980202913284302, index: 4, word: 'Sarah' },\n * //   { entity: 'B-LOC', score: 0.9994474053382874, index: 9, word: 'London' }\n * // ]\n * ```\n * \n * **Example:** Perform named entity recognition with `Xenova/bert-base-NER` (and return all labels).\n * ```javascript\n * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER');\n * const output = await classifier('Sarah lives in the United States of America', { ignore_labels: [] });\n * // [\n * //   { entity: 'B-PER', score: 0.9966587424278259, index: 1, word: 'Sarah' },\n * //   { entity: 'O', score: 0.9987385869026184, index: 2, word: 'lives' },\n * //   { entity: 'O', score: 0.9990072846412659, index: 3, word: 'in' },\n * //   { entity: 'O', score: 0.9988298416137695, index: 4, word: 'the' },\n * //   { entity: 'B-LOC', score: 0.9995510578155518, index: 5, word: 'United' },\n * //   { entity: 'I-LOC', score: 0.9990395307540894, index: 6, word: 'States' },\n * //   { entity: 'I-LOC', score: 0.9986724853515625, index: 7, word: 'of' },\n * //   { entity: 'I-LOC', score: 0.9975294470787048, index: 8, word: 'America' }\n * // ]\n * ```\n */\nexport class TokenClassificationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TokenClassificationPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new TokenClassificationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {TokenClassificationPipelineCallback} */\n    async _call(texts, {\n        ignore_labels = ['O'],\n    } = {}) {\n\n        const isBatched = Array.isArray(texts);\n\n        // Run tokenization\n        const model_inputs = this.tokenizer(isBatched ? texts : [texts], {\n            padding: true,\n            truncation: true,\n        });\n\n        // Run model\n        const outputs = await this.model(model_inputs)\n\n        const logits = outputs.logits;\n        const id2label = this.model.config.id2label;\n\n        const toReturn = [];\n        for (let i = 0; i < logits.dims[0]; ++i) {\n            const ids = model_inputs.input_ids[i];\n            const batch = logits[i];\n\n            // List of tokens that aren't ignored\n            const tokens = [];\n            for (let j = 0; j < batch.dims[0]; ++j) {\n                const tokenData = batch[j];\n                const topScoreIndex = max(tokenData.data)[1];\n\n                const entity = id2label ? id2label[topScoreIndex] : `LABEL_${topScoreIndex}`;\n                if (ignore_labels.includes(entity)) {\n                    // We predicted a token that should be ignored. So, we skip it.\n                    continue;\n                }\n\n                // TODO add option to keep special tokens?\n                const word = this.tokenizer.decode([ids[j].item()], { skip_special_tokens: true });\n                if (word === '') {\n                    // Was a special token. So, we skip it.\n                    continue;\n                }\n\n                const scores = softmax(tokenData.data);\n\n                tokens.push({\n                    entity: entity,\n                    score: scores[topScoreIndex],\n                    index: j,\n                    word: word,\n\n                    // TODO: null for now, but will add\n                    start: null,\n                    end: null,\n                });\n            }\n            toReturn.push(tokens);\n        }\n        return isBatched ? toReturn : toReturn[0];\n    }\n}\n\n/**\n * @typedef {Object} QuestionAnsweringOutput\n * @property {number} score The probability associated to the answer.\n * @property {number} [start] The character start index of the answer (in the tokenized version of the input).\n * @property {number} [end] The character end index of the answer (in the tokenized version of the input).\n * @property {string} answer The answer to the question.\n * \n * @typedef {Object} QuestionAnsweringPipelineOptions Parameters specific to question answering pipelines.\n * @property {number} [topk=1] The number of top answer predictions to be returned.\n * \n * @callback QuestionAnsweringPipelineCallback Answer the question(s) given as inputs by using the context(s).\n * @param {string|string[]} question One or several question(s) (must be used in conjunction with the `context` argument).\n * @param {string|string[]} context One or several context(s) associated with the question(s) (must be used in conjunction with the `question` argument).\n * @param {QuestionAnsweringPipelineOptions} [options] The options to use for question answering.\n * @returns {Promise<QuestionAnsweringOutput|QuestionAnsweringOutput[]>} An array or object containing the predicted answers and scores.\n * \n * @typedef {TextPipelineConstructorArgs & QuestionAnsweringPipelineCallback & Disposable} QuestionAnsweringPipelineType\n */\n\n/**\n * Question Answering pipeline using any `ModelForQuestionAnswering`.\n * \n * **Example:** Run question answering with `Xenova/distilbert-base-uncased-distilled-squad`.\n * ```javascript\n * const answerer = await pipeline('question-answering', 'Xenova/distilbert-base-uncased-distilled-squad');\n * const question = 'Who was Jim Henson?';\n * const context = 'Jim Henson was a nice puppet.';\n * const output = await answerer(question, context);\n * // {\n * //   answer: \"a nice puppet\",\n * //   score: 0.5768911502526741\n * // }\n * ```\n */\nexport class QuestionAnsweringPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => QuestionAnsweringPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new QuestionAnsweringPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {QuestionAnsweringPipelineCallback} */\n    async _call(question, context, {\n        topk = 1\n    } = {}) {\n\n        // Run tokenization\n        const inputs = this.tokenizer(question, {\n            text_pair: context,\n            padding: true,\n            truncation: true,\n        });\n\n        const output = await this.model(inputs);\n\n        /** @type {QuestionAnsweringOutput[]} */\n        const toReturn = [];\n        for (let j = 0; j < output.start_logits.dims[0]; ++j) {\n            const ids = inputs.input_ids[j];\n            const sepIndex = ids.indexOf(this.tokenizer.sep_token_id);\n\n            const s1 = Array.from(softmax(output.start_logits[j].data))\n                .map((x, i) => [x, i])\n                .filter(x => x[1] > sepIndex);\n            const e1 = Array.from(softmax(output.end_logits[j].data))\n                .map((x, i) => [x, i])\n                .filter(x => x[1] > sepIndex);\n\n            const options = product(s1, e1)\n                .filter(x => x[0][1] <= x[1][1])\n                .map(x => [x[0][1], x[1][1], x[0][0] * x[1][0]])\n                .sort((a, b) => b[2] - a[2]);\n\n            for (let k = 0; k < Math.min(options.length, topk); ++k) {\n                const [start, end, score] = options[k];\n\n                const answer_tokens = [...ids].slice(start, end + 1)\n\n                const answer = this.tokenizer.decode(answer_tokens, {\n                    skip_special_tokens: true,\n                });\n\n                // TODO add start and end?\n                // NOTE: HF returns character index\n                toReturn.push({\n                    answer, score\n                });\n            }\n        }\n\n        // Mimic HF's return type based on topk\n        return (topk === 1) ? toReturn[0] : toReturn;\n    }\n}\n\n\n/**\n * @typedef {Object} FillMaskSingle\n * @property {string} sequence The corresponding input with the mask token prediction.\n * @property {number} score The corresponding probability.\n * @property {number} token The predicted token id (to replace the masked one).\n * @property {string} token_str The predicted token (to replace the masked one).\n * @typedef {FillMaskSingle[]} FillMaskOutput\n * \n * @typedef {Object} FillMaskPipelineOptions Parameters specific to fill mask pipelines.\n * @property {number} [topk=5] When passed, overrides the number of predictions to return.\n * \n * @callback FillMaskPipelineCallback Fill the masked token in the text(s) given as inputs.\n * @param {string|string[]} texts One or several texts (or one list of prompts) with masked tokens.\n * @param {FillMaskPipelineOptions} [options] The options to use for masked language modelling.\n * @returns {Promise<FillMaskOutput|FillMaskOutput[]>} An array of objects containing the score, predicted token, predicted token string,\n * and the sequence with the predicted token filled in, or an array of such arrays (one for each input text).\n * If only one input text is given, the output will be an array of objects.\n * @throws {Error} When the mask token is not found in the input text.\n * \n * @typedef {TextPipelineConstructorArgs & FillMaskPipelineCallback & Disposable} FillMaskPipelineType\n */\n\n/**\n * Masked language modeling prediction pipeline using any `ModelWithLMHead`.\n * \n * **Example:** Perform masked language modelling (a.k.a. \"fill-mask\") with `Xenova/bert-base-uncased`.\n * ```javascript\n * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased');\n * const output = await unmasker('The goal of life is [MASK].');\n * // [\n * //   { token_str: 'survival', score: 0.06137419492006302, token: 8115, sequence: 'The goal of life is survival.' },\n * //   { token_str: 'love', score: 0.03902450203895569, token: 1567, sequence: 'The goal of life is love.' },\n * //   { token_str: 'happiness', score: 0.03253183513879776, token: 9266, sequence: 'The goal of life is happiness.' },\n * //   { token_str: 'freedom', score: 0.018736306577920914, token: 4438, sequence: 'The goal of life is freedom.' },\n * //   { token_str: 'life', score: 0.01859794743359089, token: 1297, sequence: 'The goal of life is life.' }\n * // ]\n * ```\n * \n * **Example:** Perform masked language modelling (a.k.a. \"fill-mask\") with `Xenova/bert-base-cased` (and return top result).\n * ```javascript\n * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased');\n * const output = await unmasker('The Milky Way is a [MASK] galaxy.', { topk: 1 });\n * // [{ token_str: 'spiral', score: 0.6299987435340881, token: 14061, sequence: 'The Milky Way is a spiral galaxy.' }]\n * ```\n */\nexport class FillMaskPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => FillMaskPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new FillMaskPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {FillMaskPipelineCallback} */\n    async _call(texts, {\n        topk = 5\n    } = {}) {\n\n        // Run tokenization\n        const model_inputs = this.tokenizer(texts, {\n            padding: true,\n            truncation: true,\n        });\n\n        // Run model\n        const outputs = await this.model(model_inputs)\n\n        const toReturn = [];\n\n        for (let i = 0; i < model_inputs.input_ids.dims[0]; ++i) {\n            const ids = model_inputs.input_ids[i];\n            const mask_token_index = ids.indexOf(this.tokenizer.mask_token_id)\n\n            if (mask_token_index === -1) {\n                throw Error(`Mask token (${this.tokenizer.mask_token}) not found in text.`)\n            }\n            const logits = outputs.logits[i];\n            const itemLogits = logits[mask_token_index];\n\n            const scores = getTopItems(softmax(itemLogits.data), topk);\n\n            toReturn.push(scores.map(x => {\n                const sequence = [...ids];\n                sequence[mask_token_index] = x[0];\n\n                return {\n                    score: x[1],\n                    token: x[0],\n                    token_str: this.tokenizer.model.vocab[x[0]],\n                    sequence: this.tokenizer.decode(sequence, { skip_special_tokens: true }),\n                }\n            }));\n        }\n        return Array.isArray(texts) ? toReturn : toReturn[0];\n    }\n}\n\n\n/**\n * @typedef {Object} Text2TextGenerationSingle\n * @property {string} generated_text The generated text.\n * @typedef {Text2TextGenerationSingle[]} Text2TextGenerationOutput\n * \n * @callback Text2TextGenerationPipelineCallback Generate the output text(s) using text(s) given as inputs.\n * @param {string|string[]} texts Input text for the encoder.\n * @param {import('./utils/generation.js').GenerationConfigType} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<Text2TextGenerationOutput|Text2TextGenerationOutput[]>}\n * \n * @typedef {TextPipelineConstructorArgs & Text2TextGenerationPipelineCallback & Disposable} Text2TextGenerationPipelineType\n */\n\n/**\n * Text2TextGenerationPipeline class for generating text using a model that performs text-to-text generation tasks.\n * \n * **Example:** Text-to-text generation w/ `Xenova/LaMini-Flan-T5-783M`.\n * ```javascript\n * const generator = await pipeline('text2text-generation', 'Xenova/LaMini-Flan-T5-783M');\n * const output = await generator('how can I become more healthy?', {\n *   max_new_tokens: 100,\n * });\n * // [{ generated_text: \"To become more healthy, you can: 1. Eat a balanced diet with plenty of fruits, vegetables, whole grains, lean proteins, and healthy fats. 2. Stay hydrated by drinking plenty of water. 3. Get enough sleep and manage stress levels. 4. Avoid smoking and excessive alcohol consumption. 5. Regularly exercise and maintain a healthy weight. 6. Practice good hygiene and sanitation. 7. Seek medical attention if you experience any health issues.\" }]\n * ```\n */\nexport class Text2TextGenerationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => Text2TextGenerationPipelineType} */ (Pipeline)) {\n    /** @type {'generated_text'} */\n    _key = 'generated_text';\n\n    /**\n     * Create a new Text2TextGenerationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {Text2TextGenerationPipelineCallback} */\n    async _call(texts, generate_kwargs = {}) {\n        if (!Array.isArray(texts)) {\n            texts = [texts];\n        }\n\n\n        // Add global prefix, if present\n        if (this.model.config.prefix) {\n            texts = texts.map(x => this.model.config.prefix + x)\n        }\n\n        // Handle task specific params:\n        const task_specific_params = this.model.config.task_specific_params\n        if (task_specific_params && task_specific_params[this.task]) {\n            // Add prefixes, if present\n            if (task_specific_params[this.task].prefix) {\n                texts = texts.map(x => task_specific_params[this.task].prefix + x)\n            }\n\n            // TODO update generation config\n        }\n\n        const tokenizer = this.tokenizer;\n        const tokenizer_options = {\n            padding: true,\n            truncation: true,\n        }\n        let input_ids;\n        if (this instanceof TranslationPipeline && '_build_translation_inputs' in tokenizer) {\n            // TODO: move to Translation pipeline?\n            // Currently put here to avoid code duplication\n            // @ts-ignore\n            input_ids = tokenizer._build_translation_inputs(texts, tokenizer_options, generate_kwargs).input_ids;\n\n        } else {\n            input_ids = tokenizer(texts, tokenizer_options).input_ids;\n        }\n\n        const outputTokenIds = await this.model.generate(input_ids, generate_kwargs);\n\n        return tokenizer.batch_decode(outputTokenIds, {\n            skip_special_tokens: true,\n        }).map(text => ({ [this._key]: text }));\n    }\n}\n\n\n/**\n * @typedef {Object} SummarizationSingle\n * @property {string} summary_text The summary text.\n * @typedef {SummarizationSingle[]} SummarizationOutput\n * \n * @callback SummarizationPipelineCallback Summarize the text(s) given as inputs.\n * @param {string|string[]} texts One or several articles (or one list of articles) to summarize.\n * @param {import('./utils/generation.js').GenerationConfigType} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<SummarizationOutput|SummarizationOutput[]>}\n * \n * @typedef {TextPipelineConstructorArgs & SummarizationPipelineCallback & Disposable} SummarizationPipelineType\n */\n\n/**\n * A pipeline for summarization tasks, inheriting from Text2TextGenerationPipeline.\n * \n * **Example:** Summarization w/ `Xenova/distilbart-cnn-6-6`.\n * ```javascript\n * const generator = await pipeline('summarization', 'Xenova/distilbart-cnn-6-6');\n * const text = 'The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, ' +\n *   'and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. ' +\n *   'During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest ' +\n *   'man-made structure in the world, a title it held for 41 years until the Chrysler Building in New ' +\n *   'York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to ' +\n *   'the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the ' +\n *   'Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second ' +\n *   'tallest free-standing structure in France after the Millau Viaduct.';\n * const output = await generator(text, {\n *   max_new_tokens: 100,\n * });\n * // [{ summary_text: ' The Eiffel Tower is about the same height as an 81-storey building and the tallest structure in Paris. It is the second tallest free-standing structure in France after the Millau Viaduct.' }]\n * ```\n */\nexport class SummarizationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => SummarizationPipelineType} */ (/** @type {any} */ (Text2TextGenerationPipeline))) {\n    /** @type {'summary_text'} */\n    _key = 'summary_text';\n\n    /**\n     * Create a new SummarizationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n}\n\n\n/**\n * @typedef {Object} TranslationSingle\n * @property {string} translation_text The translated text.\n * @typedef {TranslationSingle[]} TranslationOutput\n * \n * @callback TranslationPipelineCallback Translate the text(s) given as inputs.\n * @param {string|string[]} texts Texts to be translated.\n * @param {import('./utils/generation.js').GenerationConfigType} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<TranslationOutput|TranslationOutput[]>}\n * \n * @typedef {TextPipelineConstructorArgs & TranslationPipelineCallback & Disposable} TranslationPipelineType\n */\n\n/**\n * Translates text from one language to another.\n * \n * **Example:** Multilingual translation w/ `Xenova/nllb-200-distilled-600M`.\n * \n * See [here](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200)\n * for the full list of languages and their corresponding codes.\n * \n * ```javascript\n * const translator = await pipeline('translation', 'Xenova/nllb-200-distilled-600M');\n * const output = await translator('\u091c\u0940\u0935\u0928 \u090f\u0915 \u091a\u0949\u0915\u0932\u0947\u091f \u092c\u0949\u0915\u094d\u0938 \u0915\u0940 \u0924\u0930\u0939 \u0939\u0948\u0964', {\n *   src_lang: 'hin_Deva', // Hindi\n *   tgt_lang: 'fra_Latn', // French\n * });\n * // [{ translation_text: 'La vie est comme une bo\u00eete \u00e0 chocolat.' }]\n * ```\n * \n * **Example:** Multilingual translation w/ `Xenova/m2m100_418M`.\n * \n * See [here](https://huggingface.co/facebook/m2m100_418M#languages-covered)\n * for the full list of languages and their corresponding codes.\n * \n * ```javascript\n * const translator = await pipeline('translation', 'Xenova/m2m100_418M');\n * const output = await translator('\u751f\u6d3b\u5c31\u50cf\u4e00\u76d2\u5de7\u514b\u529b\u3002', {\n *   src_lang: 'zh', // Chinese\n *   tgt_lang: 'en', // English\n * });\n * // [{ translation_text: 'Life is like a box of chocolate.' }]\n * ```\n * \n * **Example:** Multilingual translation w/ `Xenova/mbart-large-50-many-to-many-mmt`.\n * \n * See [here](https://huggingface.co/facebook/mbart-large-50-many-to-many-mmt#languages-covered)\n * for the full list of languages and their corresponding codes.\n * \n * ```javascript\n * const translator = await pipeline('translation', 'Xenova/mbart-large-50-many-to-many-mmt');\n * const output = await translator('\u0938\u0902\u092f\u0941\u0915\u094d\u0924 \u0930\u093e\u0937\u094d\u091f\u094d\u0930 \u0915\u0947 \u092a\u094d\u0930\u092e\u0941\u0916 \u0915\u093e \u0915\u0939\u0928\u093e \u0939\u0948 \u0915\u093f \u0938\u0940\u0930\u093f\u092f\u093e \u092e\u0947\u0902 \u0915\u094b\u0908 \u0938\u0948\u0928\u094d\u092f \u0938\u092e\u093e\u0927\u093e\u0928 \u0928\u0939\u0940\u0902 \u0939\u0948', {\n *   src_lang: 'hi_IN', // Hindi\n *   tgt_lang: 'fr_XX', // French\n * });\n * // [{ translation_text: 'Le chef des Nations affirme qu 'il n 'y a military solution in Syria.' }]\n * ```\n */\nexport class TranslationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TranslationPipelineType} */ (/** @type {any} */ (Text2TextGenerationPipeline))) {\n    /** @type {'translation_text'} */\n    _key = 'translation_text';\n\n    /**\n     * Create a new TranslationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n}\n\nfunction isChat(x) {\n    return Array.isArray(x) && x.every(x => 'role' in x && 'content' in x);\n}\n\n/**\n * @typedef {import('./tokenizers.js').Message[]} Chat\n * \n * @typedef {Object} TextGenerationSingle\n * @property {string|Chat} generated_text The generated text.\n * @typedef {TextGenerationSingle[]} TextGenerationOutput\n * \n * @typedef {Object} TextGenerationSpecificParams Parameters specific to text-generation pipelines.\n * @property {boolean} [add_special_tokens] Whether or not to add special tokens when tokenizing the sequences.\n * @property {boolean} [return_full_text=true] If set to `false` only added text is returned, otherwise the full text is returned.\n * @typedef {import('./utils/generation.js').GenerationConfigType & TextGenerationSpecificParams} TextGenerationConfig\n * \n * @callback TextGenerationPipelineCallback Complete the prompt(s) given as inputs.\n * @param {string|string[]|Chat|Chat[]} texts One or several prompts (or one list of prompts) to complete.\n * @param {TextGenerationConfig} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<TextGenerationOutput|TextGenerationOutput[]>} An array or object containing the generated texts.\n * \n * @typedef {TextPipelineConstructorArgs & TextGenerationPipelineCallback & Disposable} TextGenerationPipelineType\n */\n\n/**\n * Language generation pipeline using any `ModelWithLMHead` or `ModelForCausalLM`.\n * This pipeline predicts the words that will follow a specified text prompt.\n * NOTE: For the full list of generation parameters, see [`GenerationConfig`](./utils/generation#module_utils/generation.GenerationConfig).\n * \n * **Example:** Text generation with `Xenova/distilgpt2` (default settings).\n * ```javascript\n * const generator = await pipeline('text-generation', 'Xenova/distilgpt2');\n * const text = 'I enjoy walking with my cute dog,';\n * const output = await generator(text);\n * // [{ generated_text: \"I enjoy walking with my cute dog, and I love to play with the other dogs.\" }]\n * ```\n * \n * **Example:** Text generation with `Xenova/distilgpt2` (custom settings).\n * ```javascript\n * const generator = await pipeline('text-generation', 'Xenova/distilgpt2');\n * const text = 'Once upon a time, there was';\n * const output = await generator(text, {\n *   temperature: 2,\n *   max_new_tokens: 10,\n *   repetition_penalty: 1.5,\n *   no_repeat_ngram_size: 2,\n *   num_beams: 2,\n *   num_return_sequences: 2,\n * });\n * // [{\n * //   \"generated_text\": \"Once upon a time, there was an abundance of information about the history and activities that\"\n * // }, {\n * //   \"generated_text\": \"Once upon a time, there was an abundance of information about the most important and influential\"\n * // }]\n * ```\n * \n * **Example:** Run code generation with `Xenova/codegen-350M-mono`.\n * ```javascript\n * const generator = await pipeline('text-generation', 'Xenova/codegen-350M-mono');\n * const text = 'def fib(n):';\n * const output = await generator(text, {\n *   max_new_tokens: 44,\n * });\n * // [{\n * //   generated_text: 'def fib(n):\\n' +\n * //     '    if n == 0:\\n' +\n * //     '        return 0\\n' +\n * //     '    elif n == 1:\\n' +\n * //     '        return 1\\n' +\n * //     '    else:\\n' +\n * //     '        return fib(n-1) + fib(n-2)\\n'\n * // }]\n * ```\n */\nexport class TextGenerationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TextGenerationPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new TextGenerationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {TextGenerationPipelineCallback} */\n    async _call(texts, generate_kwargs = {}) {\n        let isBatched = false;\n        let isChatInput = false;\n\n        // Normalize inputs\n        /** @type {string[]} */\n        let inputs;\n        if (typeof texts === 'string') {\n            inputs = texts = [texts];\n        } else if (Array.isArray(texts) && texts.every(x => typeof x === 'string')) {\n            isBatched = true;\n            inputs = /** @type {string[]} */(texts);\n        } else {\n            if (isChat(texts)) {\n                texts = [/** @type {Chat} */(texts)];\n            } else if (Array.isArray(texts) && texts.every(isChat)) {\n                isBatched = true;\n            } else {\n                throw new Error('Input must be a string, an array of strings, a Chat, or an array of Chats');\n            }\n            isChatInput = true;\n\n            // If the input is a chat, we need to apply the chat template\n            inputs = /** @type {string[]} */(/** @type {Chat[]} */ (texts).map(\n                x => this.tokenizer.apply_chat_template(x, {\n                    tokenize: false,\n                    add_generation_prompt: true,\n                })\n            ));\n        }\n\n        // By default, do not add special tokens\n        const add_special_tokens = generate_kwargs.add_special_tokens ?? false;\n\n        // By default, return full text\n        const return_full_text = isChatInput\n            ? false\n            : generate_kwargs.return_full_text ?? true;\n\n        this.tokenizer.padding_side = 'left';\n        const { input_ids, attention_mask } = this.tokenizer(inputs, {\n            add_special_tokens,\n            padding: true,\n            truncation: true,\n        });\n\n        const outputTokenIds = await this.model.generate(input_ids, generate_kwargs, null, {\n            inputs_attention_mask: attention_mask\n        });\n\n        let decoded = this.tokenizer.batch_decode(outputTokenIds, {\n            skip_special_tokens: true,\n        });\n\n\n        let promptLengths;\n        if (!return_full_text && input_ids.dims.at(-1) > 0) {\n            promptLengths = this.tokenizer.batch_decode(input_ids, {\n                skip_special_tokens: true,\n            }).map(x => x.length);\n        }\n\n        /** @type {TextGenerationOutput[]} */\n        const toReturn = Array.from({ length: texts.length }, _ => []);\n        for (let i = 0; i < decoded.length; ++i) {\n            const textIndex = Math.floor(i / outputTokenIds.length * texts.length);\n\n            if (promptLengths) {\n                // Trim the decoded text to only include the generated part\n...\n", "needle_token_start": 3919, "needle_token_end": 3985, "code_context_ntokens": 15731, "output": ["```javascript\n/**\n * Prepare images for further tasks.\n * @param {ImagePipelineInputs} images images to prepare.\n * @returns {Promise<RawImage[]>} returns processed images.\n * @private\n */\n\nasync function prepareImages(images) {\n    if (!Array.isArray(images)) {\n        images = [images];\n    }\n\n    // Possibly convert any non-images to images\n    return await Promise.all(images.map(x => RawImage.read(x)));\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "getModelJSON", "language": "typescript", "path": "src/utils/hub.js", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To fetch a JSON file from a specified directory and file name, decode it, and parse it into a JavaScript object.\n2. **Input**: \n   - A path to the directory containing the JSON file.\n   - The name of the JSON file.\n   - A boolean indicating whether to throw an error if the file is not found (defaults to true).\n   - An optional object containing additional parameters.\n3. **Output**: A promise that resolves to the parsed JavaScript object from the JSON file, or an empty object if the file is not found and not throwing an error.\n4. **Procedure**: \n   - The function first retrieves the file as a byte array using another function.\n   - If the file is not found and the error is non-fatal, it returns an empty object.\n   - The byte array is then decoded into a string using UTF-8 encoding.\n   - Finally, the string is parsed into a JavaScript object using JSON parsing.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            this.headers.set('content-length', stats.size.toString());\n\n            this.updateContentType();\n\n            let self = this;\n            this.body = new ReadableStream({\n                start(controller) {\n                    self.arrayBuffer().then(buffer => {\n                        controller.enqueue(new Uint8Array(buffer));\n                        controller.close();\n                    })\n                }\n            });\n        } else {\n            this.status = 404;\n            this.statusText = 'Not Found';\n            this.body = null;\n        }\n    }\n\n    /**\n     * Updates the 'content-type' header property of the response based on the extension of\n     * the file specified by the filePath property of the current object.\n     * @returns {void}\n     */\n    updateContentType() {\n        // Set content-type header based on file extension\n        const extension = this.filePath.toString().split('.').pop().toLowerCase();\n        this.headers.set('content-type', this._CONTENT_TYPE_MAP[extension] ?? 'application/octet-stream');\n    }\n\n    /**\n     * Clone the current FileResponse object.\n     * @returns {FileResponse} A new FileResponse object with the same properties as the current object.\n     */\n    clone() {\n        let response = new FileResponse(this.filePath);\n        response.exists = this.exists;\n        response.status = this.status;\n        response.statusText = this.statusText;\n        response.headers = new Headers(this.headers);\n        return response;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with an ArrayBuffer containing the file's contents.\n     * @returns {Promise<ArrayBuffer>} A Promise that resolves with an ArrayBuffer containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async arrayBuffer() {\n        const data = await fs.promises.readFile(this.filePath);\n        return data.buffer;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a Blob containing the file's contents.\n     * @returns {Promise<Blob>} A Promise that resolves with a Blob containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async blob() {\n        const data = await fs.promises.readFile(this.filePath);\n        return new Blob([data], { type: this.headers.get('content-type') });\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a string containing the file's contents.\n     * @returns {Promise<string>} A Promise that resolves with a string containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async text() {\n        const data = await fs.promises.readFile(this.filePath, 'utf8');\n        return data;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a parsed JavaScript object containing the file's contents.\n     * \n     * @returns {Promise<Object>} A Promise that resolves with a parsed JavaScript object containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async json() {\n        return JSON.parse(await this.text());\n    }\n}\n\n/**\n * Determines whether the given string is a valid HTTP or HTTPS URL.\n * @param {string|URL} string The string to test for validity as an HTTP or HTTPS URL.\n * @param {string[]} [validHosts=null] A list of valid hostnames. If specified, the URL's hostname must be in this list.\n * @returns {boolean} True if the string is a valid HTTP or HTTPS URL, false otherwise.\n */\nfunction isValidHttpUrl(string, validHosts = null) {\n    // https://stackoverflow.com/a/43467144\n    let url;\n    try {\n        url = new URL(string);\n    } catch (_) {\n        return false;\n    }\n    if (validHosts && !validHosts.includes(url.hostname)) {\n        return false;\n    }\n    return url.protocol === \"http:\" || url.protocol === \"https:\";\n}\n\n/**\n * Helper function to get a file, using either the Fetch API or FileSystem API.\n *\n * @param {URL|string} urlOrPath The URL/path of the file to get.\n * @returns {Promise<FileResponse|Response>} A promise that resolves to a FileResponse object (if the file is retrieved using the FileSystem API), or a Response object (if the file is retrieved using the Fetch API).\n */\nexport async function getFile(urlOrPath) {\n\n    if (env.useFS && !isValidHttpUrl(urlOrPath)) {\n        return new FileResponse(urlOrPath);\n\n    } else if (typeof process !== 'undefined' && process?.release?.name === 'node') {\n        const IS_CI = !!process.env?.TESTING_REMOTELY;\n        const version = env.version;\n\n        const headers = new Headers();\n        headers.set('User-Agent', `transformers.js/${version}; is_ci/${IS_CI};`);\n\n        // Check whether we are making a request to the Hugging Face Hub.\n        const isHFURL = isValidHttpUrl(urlOrPath, ['huggingface.co', 'hf.co']);\n        if (isHFURL) {\n            // If an access token is present in the environment variables,\n            // we add it to the request headers.\n            // NOTE: We keep `HF_ACCESS_TOKEN` for backwards compatibility (as a fallback).\n            const token = process.env?.HF_TOKEN ?? process.env?.HF_ACCESS_TOKEN;\n            if (token) {\n                headers.set('Authorization', `Bearer ${token}`);\n            }\n        }\n        return fetch(urlOrPath, { headers });\n    } else {\n        // Running in a browser-environment, so we use default headers\n        // NOTE: We do not allow passing authorization headers in the browser,\n        // since this would require exposing the token to the client.\n        return fetch(urlOrPath);\n    }\n}\n\nconst ERROR_MAPPING = {\n    // 4xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#client_error_responses)\n    400: 'Bad request error occurred while trying to load file',\n    401: 'Unauthorized access to file',\n    403: 'Forbidden access to file',\n    404: 'Could not locate file',\n    408: 'Request timeout error occurred while trying to load file',\n\n    // 5xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#server_error_responses)\n    500: 'Internal server error error occurred while trying to load file',\n    502: 'Bad gateway error occurred while trying to load file',\n    503: 'Service unavailable error occurred while trying to load file',\n    504: 'Gateway timeout error occurred while trying to load file',\n}\n/**\n * Helper method to handle fatal errors that occur while trying to load a file from the Hugging Face Hub.\n * @param {number} status The HTTP status code of the error.\n * @param {string} remoteURL The URL of the file that could not be loaded.\n * @param {boolean} fatal Whether to raise an error if the file could not be loaded.\n * @returns {null} Returns `null` if `fatal = true`.\n * @throws {Error} If `fatal = false`.\n */\nfunction handleError(status, remoteURL, fatal) {\n    if (!fatal) {\n        // File was not loaded correctly, but it is optional.\n        // TODO in future, cache the response?\n        return null;\n    }\n\n    const message = ERROR_MAPPING[status] ?? `Error (${status}) occurred while trying to load file`;\n    throw Error(`${message}: \"${remoteURL}\".`);\n}\n\nclass FileCache {\n    /**\n     * Instantiate a `FileCache` object.\n     * @param {string} path \n     */\n    constructor(path) {\n        this.path = path;\n    }\n\n    /**\n     * Checks whether the given request is in the cache.\n     * @param {string} request \n     * @returns {Promise<FileResponse | undefined>}\n     */\n    async match(request) {\n\n        let filePath = path.join(this.path, request);\n        let file = new FileResponse(filePath);\n\n        if (file.exists) {\n            return file;\n        } else {\n            return undefined;\n        }\n    }\n\n    /**\n     * Adds the given response to the cache.\n     * @param {string} request \n     * @param {Response|FileResponse} response \n     * @returns {Promise<void>}\n     */\n    async put(request, response) {\n        const buffer = Buffer.from(await response.arrayBuffer());\n\n        let outputPath = path.join(this.path, request);\n\n        try {\n            await fs.promises.mkdir(path.dirname(outputPath), { recursive: true });\n            await fs.promises.writeFile(outputPath, buffer);\n\n        } catch (err) {\n            console.warn('An error occurred while writing the file to cache:', err)\n        }\n    }\n\n    // TODO add the rest?\n    // addAll(requests: RequestInfo[]): Promise<void>;\n    // delete(request: RequestInfo | URL, options?: CacheQueryOptions): Promise<boolean>;\n    // keys(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise<ReadonlyArray<Request>>;\n    // match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise<Response | undefined>;\n    // matchAll(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise<ReadonlyArray<Response>>;\n}\n\n/**\n * \n * @param {FileCache|Cache} cache The cache to search\n * @param {string[]} names The names of the item to search for\n * @returns {Promise<FileResponse|Response|undefined>} The item from the cache, or undefined if not found.\n */\nasync function tryCache(cache, ...names) {\n    for (let name of names) {\n        try {\n            let result = await cache.match(name);\n            if (result) return result;\n        } catch (e) {\n            continue;\n        }\n    }\n    return undefined;\n}\n\n/**\n * \n * Retrieves a file from either a remote URL using the Fetch API or from the local file system using the FileSystem API.\n * If the filesystem is available and `env.useCache = true`, the file will be downloaded and cached.\n * \n * @param {string} path_or_repo_id This can be either:\n * - a string, the *model id* of a model repo on huggingface.co.\n * - a path to a *directory* potentially containing the file.\n * @param {string} filename The name of the file to locate in `path_or_repo`.\n * @param {boolean} [fatal=true] Whether to throw an error if the file is not found.\n * @param {PretrainedOptions} [options] An object containing optional parameters.\n * \n * @throws Will throw an error if the file is not found and `fatal` is true.\n * @returns {Promise} A Promise that resolves with the file content as a buffer.\n */\nexport async function getModelFile(path_or_repo_id, filename, fatal = true, options = {}) {\n\n    if (!env.allowLocalModels) {\n        // User has disabled local models, so we just make sure other settings are correct.\n\n        if (options.local_files_only) {\n            throw Error(\"Invalid configuration detected: local models are disabled (`env.allowLocalModels=false`) but you have requested to only use local models (`local_files_only=true`).\")\n        } else if (!env.allowRemoteModels) {\n            throw Error(\"Invalid configuration detected: both local and remote models are disabled. Fix by setting `env.allowLocalModels` or `env.allowRemoteModels` to `true`.\")\n        }\n    }\n\n    // Initiate file retrieval\n    dispatchCallback(options.progress_callback, {\n        status: 'initiate',\n        name: path_or_repo_id,\n        file: filename\n    })\n\n    // First, check if the a caching backend is available\n    // If no caching mechanism available, will download the file every time\n    let cache;\n    if (!cache && env.useBrowserCache) {\n        if (typeof caches === 'undefined') {\n            throw Error('Browser cache is not available in this environment.')\n        }\n        try {\n            // In some cases, the browser cache may be visible, but not accessible due to security restrictions.\n            // For example, when running an application in an iframe, if a user attempts to load the page in\n            // incognito mode, the following error is thrown: `DOMException: Failed to execute 'open' on 'CacheStorage':\n            // An attempt was made to break through the security policy of the user agent.`\n            // So, instead of crashing, we just ignore the error and continue without using the cache.\n            cache = await caches.open('transformers-cache');\n        } catch (e) {\n            console.warn('An error occurred while opening the browser cache:', e);\n        }\n    }\n\n    if (!cache && env.useFSCache) {\n        // TODO throw error if not available\n\n        // If `cache_dir` is not specified, use the default cache directory\n        cache = new FileCache(options.cache_dir ?? env.cacheDir);\n    }\n\n    if (!cache && env.useCustomCache) {\n        // Allow the user to specify a custom cache system.\n        if (!env.customCache) {\n            throw Error('`env.useCustomCache=true`, but `env.customCache` is not defined.')\n        }\n\n        // Check that the required methods are defined:\n        if (!env.customCache.match || !env.customCache.put) {\n            throw new Error(\n                \"`env.customCache` must be an object which implements the `match` and `put` functions of the Web Cache API. \" +\n                \"For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache\"\n            )\n        }\n        cache = env.customCache;\n    }\n\n    const revision = options.revision ?? 'main';\n\n    let requestURL = pathJoin(path_or_repo_id, filename);\n    let localPath = pathJoin(env.localModelPath, requestURL);\n\n    let remoteURL = pathJoin(\n        env.remoteHost,\n        env.remotePathTemplate\n            .replaceAll('{model}', path_or_repo_id)\n            .replaceAll('{revision}', encodeURIComponent(revision)),\n        filename\n    );\n\n    // Choose cache key for filesystem cache\n    // When using the main revision (default), we use the request URL as the cache key.\n    // If a specific revision is requested, we account for this in the cache key.\n    let fsCacheKey = revision === 'main' ? requestURL : pathJoin(path_or_repo_id, revision, filename);\n\n    /** @type {string} */\n    let cacheKey;\n    let proposedCacheKey = cache instanceof FileCache ? fsCacheKey : remoteURL;\n\n    // Whether to cache the final response in the end.\n    let toCacheResponse = false;\n\n    /** @type {Response|FileResponse|undefined} */\n    let response;\n\n    if (cache) {\n        // A caching system is available, so we try to get the file from it.\n        //  1. We first try to get from cache using the local path. In some environments (like deno),\n        //     non-URL cache keys are not allowed. In these cases, `response` will be undefined.\n        //  2. If no response is found, we try to get from cache using the remote URL or file system cache.\n        response = await tryCache(cache, localPath, proposedCacheKey);\n    }\n\n    const cacheHit = response !== undefined;\n\n    if (response === undefined) {\n        // Caching not available, or file is not cached, so we perform the request\n\n        if (env.allowLocalModels) {\n            // Accessing local models is enabled, so we try to get the file locally.\n            // If request is a valid HTTP URL, we skip the local file check. Otherwise, we try to get the file locally.\n            const isURL = isValidHttpUrl(requestURL);\n            if (!isURL) {\n                try {\n                    response = await getFile(localPath);\n                    cacheKey = localPath; // Update the cache key to be the local path\n                } catch (e) {\n                    // Something went wrong while trying to get the file locally.\n                    // NOTE: error handling is done in the next step (since `response` will be undefined)\n                    console.warn(`Unable to load from local path \"${localPath}\": \"${e}\"`);\n                }\n            } else if (options.local_files_only) {\n                throw new Error(`\\`local_files_only=true\\`, but attempted to load a remote file from: ${requestURL}.`);\n            } else if (!env.allowRemoteModels) {\n                throw new Error(`\\`env.allowRemoteModels=false\\`, but attempted to load a remote file from: ${requestURL}.`);\n            }\n        }\n\n        if (response === undefined || response.status === 404) {\n            // File not found locally. This means either:\n            // - The user has disabled local file access (`env.allowLocalModels=false`)\n            // - the path is a valid HTTP url (`response === undefined`)\n            // - the path is not a valid HTTP url and the file is not present on the file system or local server (`response.status === 404`)\n\n            if (options.local_files_only || !env.allowRemoteModels) {\n                // User requested local files only, but the file is not found locally.\n                if (fatal) {\n                    throw Error(`\\`local_files_only=true\\` or \\`env.allowRemoteModels=false\\` and file was not found locally at \"${localPath}\".`);\n                } else {\n                    // File not found, but this file is optional.\n                    // TODO in future, cache the response?\n                    return null;\n                }\n            }\n\n            // File not found locally, so we try to download it from the remote server\n            response = await getFile(remoteURL);\n\n            if (response.status !== 200) {\n                return handleError(response.status, remoteURL, fatal);\n            }\n\n            // Success! We use the proposed cache key from earlier\n            cacheKey = proposedCacheKey;\n        }\n\n        // Only cache the response if:\n        toCacheResponse =\n            cache                              // 1. A caching system is available\n            && typeof Response !== 'undefined' // 2. `Response` is defined (i.e., we are in a browser-like environment)\n            && response instanceof Response    // 3. result is a `Response` object (i.e., not a `FileResponse`)\n            && response.status === 200         // 4. request was successful (status code 200)\n    }\n\n    // Start downloading\n    dispatchCallback(options.progress_callback, {\n        status: 'download',\n        name: path_or_repo_id,\n        file: filename\n    })\n\n    const progressInfo = {\n        status: 'progress',\n        name: path_or_repo_id,\n        file: filename\n    }\n\n    /** @type {Uint8Array} */\n    let buffer;\n\n    if (!options.progress_callback) {\n        // If no progress callback is specified, we can use the `.arrayBuffer()`\n        // method to read the response.\n        buffer = new Uint8Array(await response.arrayBuffer());\n\n    } else if (\n        cacheHit // The item is being read from the cache\n        &&\n        typeof navigator !== 'undefined' && /firefox/i.test(navigator.userAgent) // We are in Firefox\n    ) {\n        // Due to bug in Firefox, we cannot display progress when loading from cache.\n        // Fortunately, since this should be instantaneous, this should not impact users too much.\n        buffer = new Uint8Array(await response.arrayBuffer());\n\n        // For completeness, we still fire the final progress callback\n        dispatchCallback(options.progress_callback, {\n            ...progressInfo,\n            progress: 100,\n            loaded: buffer.length,\n            total: buffer.length,\n        })\n    } else {\n        buffer = await readResponse(response, data => {\n            dispatchCallback(options.progress_callback, {\n                ...progressInfo,\n                ...data,\n            })\n        })\n    }\n\n    if (\n        // Only cache web responses\n        // i.e., do not cache FileResponses (prevents duplication)\n        toCacheResponse && cacheKey\n        &&\n        // Check again whether request is in cache. If not, we add the response to the cache\n        (await cache.match(cacheKey) === undefined)\n    ) {\n        // NOTE: We use `new Response(buffer, ...)` instead of `response.clone()` to handle LFS files\n        await cache.put(cacheKey, new Response(buffer, {\n            headers: response.headers\n        }))\n            .catch(err => {\n                // Do not crash if unable to add to cache (e.g., QuotaExceededError).\n                // Rather, log a warning and proceed with execution.\n                console.warn(`Unable to add response to browser cache: ${err}.`);\n            });\n\n    }\n\n    dispatchCallback(options.progress_callback, {\n        status: 'done',\n        name: path_or_repo_id,\n        file: filename\n    });\n\n    return buffer;\n}\n\n/**\n * Fetches a JSON file from a given path and file name.\n *\n * @param {string} modelPath The path to the directory containing the file.\n * @param {string} fileName The name of the file to fetch.\n * @param {boolean} [fatal=true] Whether to throw an error if the file is not found.\n * @param {PretrainedOptions} [options] An object containing optional parameters.\n * @returns {Promise<Object>} The JSON data parsed into a JavaScript object.\n * @throws Will throw an error if the file is not found and `fatal` is true.\n */\nexport \nasync function getModelJSON(modelPath, fileName, fatal = true, options = {}) {\n    let buffer = await getModelFile(modelPath, fileName, fatal, options);\n    if (buffer === null) {\n        // Return empty object\n        return {}\n    }\n\n    let decoder = new TextDecoder('utf-8');\n    let jsonData = decoder.decode(buffer);\n\n    return JSON.parse(jsonData);\n}\n\n/**\n * Read and track progress when reading a Response object\n *\n * @param {any} response The Response object to read\n * @param {function} progress_callback The function to call with progress updates\n * @returns {Promise<Uint8Array>} A Promise that resolves with the Uint8Array buffer\n */\nasync function readResponse(response, progress_callback) {\n\n    const contentLength = response.headers.get('Content-Length');\n    if (contentLength === null) {\n        console.warn('Unable to determine content-length from response headers. Will expand buffer when needed.')\n    }\n    let total = parseInt(contentLength ?? '0');\n    let buffer = new Uint8Array(total);\n    let loaded = 0;\n\n    const reader = response.body.getReader();\n    async function read() {\n        const { done, value } = await reader.read();\n        if (done) return;\n\n        let newLoaded = loaded + value.length;\n        if (newLoaded > total) {\n            total = newLoaded;\n\n            // Adding the new data will overflow buffer.\n            // In this case, we extend the buffer\n            let newBuffer = new Uint8Array(total);\n\n            // copy contents\n            newBuffer.set(buffer);\n\n            buffer = newBuffer;\n        }\n        buffer.set(value, loaded)\n        loaded = newLoaded;\n\n        const progress = (loaded / total) * 100;\n\n        // Call your function here\n        progress_callback({\n            progress: progress,\n            loaded: loaded,\n            total: total,\n        })\n\n        return read();\n    }\n\n    // Actually read\n    await read();\n\n    return buffer;\n}\n\n/**\n * Joins multiple parts of a path into a single path, while handling leading and trailing slashes.\n *\n * @param {...string} parts Multiple parts of a path.\n * @returns {string} A string representing the joined path.\n */\nfunction pathJoin(...parts) {\n    // https://stackoverflow.com/a/55142565\n    parts = parts.map((part, index) => {\n        if (index) {\n            part = part.replace(new RegExp('^/'), '');\n        }\n        if (index !== parts.length - 1) {\n            part = part.replace(new RegExp('/$'), '');\n        }\n        return part;\n    })\n    return parts.join('/');\n}\n\n// Path: src/configs.js\n\n/**\n * @file Helper module for using model configs. For more information, see the corresponding\n * [Python documentation](https://huggingface.co/docs/transformers/main/en/model_doc/auto#transformers.AutoConfig).\n * \n * **Example:** Load an `AutoConfig`.\n * \n * ```javascript\n * import { AutoConfig } from '@xenova/transformers';\n * let config = await AutoConfig.from_pretrained('bert-base-uncased');\n * console.log(config);\n * // PretrainedConfig {\n * //   \"model_type\": \"bert\",\n * //   \"is_encoder_decoder\": false,\n * //   \"architectures\": [\n * //       \"BertForMaskedLM\"\n * //   ],\n * //   \"vocab_size\": 30522\n * //   \"num_attention_heads\": 12,\n * //   \"num_hidden_layers\": 12,\n * //   \"hidden_size\": 768,\n * //   \"max_position_embeddings\": 512,\n * //   ...\n * // }\n * ```\n * \n * @module configs\n */\n\nimport {\n    getModelJSON,\n} from './utils/hub.js';\n\n/**\n * @typedef {import('./utils/hub.js').PretrainedOptions} PretrainedOptions\n */\n\n\n/**\n * Loads a config from the specified path.\n * @param {string} pretrained_model_name_or_path The path to the config directory.\n * @param {PretrainedOptions} options Additional options for loading the config.\n * @returns {Promise<Array>} A promise that resolves with information about the loaded config.\n */\nasync function loadConfig(pretrained_model_name_or_path, options) {\n    let info = await getModelJSON(pretrained_model_name_or_path, 'config.json', true, options);\n    return info;\n}\n\n/**\n * Base class for all configuration classes. For more information, see the corresponding\n * [Python documentation](https://huggingface.co/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig).\n */\nexport class PretrainedConfig {\n    // NOTE: Typo in original\n\n    /**\n     * Create a new PreTrainedTokenizer instance.\n     * @param {Object} configJSON The JSON of the config.\n     */\n    constructor(configJSON) {\n        this.model_type = null;\n        this.is_encoder_decoder = false;\n\n        Object.assign(this, configJSON);\n    }\n\n    /**\n     * Loads a pre-trained config from the given `pretrained_model_name_or_path`. \n     * \n     * @param {string} pretrained_model_name_or_path The path to the pre-trained config.\n     * @param {PretrainedOptions} options Additional options for loading the config.\n     * @throws {Error} Throws an error if the config.json is not found in the `pretrained_model_name_or_path`.\n     * \n     * @returns {Promise<PretrainedConfig>} A new instance of the `PretrainedConfig` class.\n     */\n    static async from_pretrained(pretrained_model_name_or_path, {\n        progress_callback = null,\n        config = null,\n        cache_dir = null,\n        local_files_only = false,\n        revision = 'main',\n    } = {}) {\n\n        let data = config ?? await loadConfig(pretrained_model_name_or_path, {\n            progress_callback,\n            config,\n            cache_dir,\n            local_files_only,\n            revision,\n        })\n        return new this(data);\n    }\n}\n\n/**\n * Helper class which is used to instantiate pretrained configs with the `from_pretrained` function.\n * \n * @example\n * let config = await AutoConfig.from_pretrained('bert-base-uncased'); \n */\nexport class AutoConfig {\n    /** @type {PretrainedConfig.from_pretrained} */\n    static async from_pretrained(...args) {\n        return PretrainedConfig.from_pretrained(...args);\n    }\n}\n\n// Path: src/utils/maths.js\n\n/**\n * @file Helper module for mathematical processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/maths\n */\n\n/**\n * @typedef {Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array} TypedArray\n * @typedef {BigInt64Array | BigUint64Array} BigTypedArray\n * @typedef {TypedArray | BigTypedArray} AnyTypedArray\n */\n\n/**\n * @param {TypedArray} input\n */\nexport function interpolate_data(input, [in_channels, in_height, in_width], [out_height, out_width], mode = 'bilinear', align_corners = false) {\n    // TODO use mode and align_corners\n\n    // Output image dimensions\n    const x_scale = out_width / in_width;\n    const y_scale = out_height / in_height;\n\n    // Output image\n    // @ts-ignore\n    const out_img = new input.constructor(out_height * out_width * in_channels);\n\n    // Pre-calculate strides\n    const inStride = in_height * in_width;\n    const outStride = out_height * out_width;\n\n    for (let i = 0; i < out_height; ++i) {\n        for (let j = 0; j < out_width; ++j) {\n            // Calculate output offset\n            const outOffset = i * out_width + j;\n\n            // Calculate input pixel coordinates\n            const x = (j + 0.5) / x_scale - 0.5;\n            const y = (i + 0.5) / y_scale - 0.5;\n\n            // Calculate the four nearest input pixels\n            // We also check if the input pixel coordinates are within the image bounds\n            let x1 = Math.floor(x);\n            let y1 = Math.floor(y);\n            const x2 = Math.min(x1 + 1, in_width - 1);\n            const y2 = Math.min(y1 + 1, in_height - 1);\n\n            x1 = Math.max(x1, 0);\n            y1 = Math.max(y1, 0);\n\n\n            // Calculate the fractional distances between the input pixel and the four nearest pixels\n            const s = x - x1;\n            const t = y - y1;\n\n            // Perform bilinear interpolation\n            const w1 = (1 - s) * (1 - t);\n            const w2 = s * (1 - t);\n            const w3 = (1 - s) * t;\n            const w4 = s * t;\n\n            // Calculate the four nearest input pixel indices\n            const yStride = y1 * in_width;\n            const xStride = y2 * in_width;\n            const idx1 = yStride + x1;\n            const idx2 = yStride + x2;\n            const idx3 = xStride + x1;\n            const idx4 = xStride + x2;\n\n            for (let k = 0; k < in_channels; ++k) {\n                // Calculate channel offset\n                const cOffset = k * inStride;\n\n                out_img[k * outStride + outOffset] =\n                    w1 * input[cOffset + idx1] +\n                    w2 * input[cOffset + idx2] +\n                    w3 * input[cOffset + idx3] +\n                    w4 * input[cOffset + idx4];\n            }\n        }\n    }\n\n    return out_img;\n}\n\n\n/**\n * Helper method to permute a `AnyTypedArray` directly\n * @template {AnyTypedArray} T \n * @param {T} array \n * @param {number[]} dims \n * @param {number[]} axes \n * @returns {[T, number[]]} The permuted array and the new shape.\n */\nexport function permute_data(array, dims, axes) {\n    // Calculate the new shape of the permuted array\n    // and the stride of the original array\n    const shape = new Array(axes.length);\n    const stride = new Array(axes.length);\n\n    for (let i = axes.length - 1, s = 1; i >= 0; --i) {\n        stride[i] = s;\n        shape[i] = dims[axes[i]];\n        s *= shape[i];\n    }\n\n    // Precompute inverse mapping of stride\n    const invStride = axes.map((_, i) => stride[axes.indexOf(i)]);\n\n    // Create the permuted array with the new shape\n    // @ts-ignore\n    const permutedData = new array.constructor(array.length);\n\n    // Permute the original array to the new array\n    for (let i = 0; i < array.length; ++i) {\n        let newIndex = 0;\n        for (let j = dims.length - 1, k = i; j >= 0; --j) {\n            newIndex += (k % dims[j]) * invStride[j];\n            k = Math.floor(k / dims[j]);\n        }\n        permutedData[newIndex] = array[i];\n    }\n\n    return [permutedData, shape];\n}\n\n\n/**\n * Compute the softmax of an array of numbers.\n * @template {TypedArray|number[]} T\n * @param {T} arr The array of numbers to compute the softmax of.\n * @returns {T} The softmax array.\n */\nexport function softmax(arr) {\n    // Compute the maximum value in the array\n    const maxVal = max(arr)[0];\n\n    // Compute the exponentials of the array values\n    const exps = arr.map(x => Math.exp(x - maxVal));\n\n    // Compute the sum of the exponentials\n    // @ts-ignore\n    const sumExps = exps.reduce((acc, val) => acc + val, 0);\n\n    // Compute the softmax values\n    const softmaxArr = exps.map(x => x / sumExps);\n\n    return /** @type {T} */(softmaxArr);\n}\n\n/**\n * Calculates the logarithm of the softmax function for the input array.\n * @template {TypedArray|number[]} T\n * @param {T} arr The input array to calculate the log_softmax function for.\n * @returns {T} The resulting log_softmax array.\n */\nexport function log_softmax(arr) {\n    // Compute the softmax values\n    const softmaxArr = softmax(arr);\n\n    // Apply log formula to each element\n    const logSoftmaxArr = softmaxArr.map(x => Math.log(x));\n\n    return /** @type {T} */(logSoftmaxArr);\n}\n\n/**\n * Calculates the dot product of two arrays.\n * @param {number[]} arr1 The first array.\n * @param {number[]} arr2 The second array.\n * @returns {number} The dot product of arr1 and arr2.\n */\nexport function dot(arr1, arr2) {\n    let result = 0;\n    for (let i = 0; i < arr1.length; ++i) {\n        result += arr1[i] * arr2[i];\n    }\n    return result;\n}\n\n\n/**\n * Get the top k items from an iterable, sorted by descending order\n * @param {any[]|TypedArray} items The items to be sorted\n * @param {number|null} [top_k=0] The number of top items to return (default: 0 = return all)\n * @returns {[number, any][]} The top k items, sorted by descending order\n */\nexport function getTopItems(items, top_k = 0) {\n    // if top == 0, return all\n\n    items = Array.from(items)\n        .map((x, i) => [i, x])            // Get indices ([index, score])\n        .sort((a, b) => b[1] - a[1])      // Sort by log probabilities\n\n    if (top_k !== null && top_k > 0) {\n        items = items.slice(0, top_k);    // Get top k items\n    }\n\n    return items\n}\n\n/**\n * Computes the cosine similarity between two arrays.\n *\n * @param {number[]} arr1 The first array.\n * @param {number[]} arr2 The second array.\n * @returns {number} The cosine similarity between the two arrays.\n */\nexport function cos_sim(arr1, arr2) {\n    // Calculate dot product of the two arrays\n    const dotProduct = dot(arr1, arr2);\n\n    // Calculate the magnitude of the first array\n    const magnitudeA = magnitude(arr1);\n\n    // Calculate the magnitude of the second array\n    const magnitudeB = magnitude(arr2);\n\n    // Calculate the cosine similarity\n    const cosineSimilarity = dotProduct / (magnitudeA * magnitudeB);\n\n    return cosineSimilarity;\n}\n\n/**\n * Calculates the magnitude of a given array.\n * @param {number[]} arr The array to calculate the magnitude of.\n * @returns {number} The magnitude of the array.\n */\nexport function magnitude(arr) {\n    return Math.sqrt(arr.reduce((acc, val) => acc + val * val, 0));\n}\n\n\n/**\n * Returns the value and index of the minimum element in an array.\n * @param {number[]|TypedArray} arr array of numbers.\n * @returns {number[]} the value and index of the minimum element, of the form: [valueOfMin, indexOfMin]\n * @throws {Error} If array is empty.\n */\nexport function min(arr) {\n    if (arr.length === 0) throw Error('Array must not be empty');\n    let min = arr[0];\n    let indexOfMin = 0;\n    for (let i = 1; i < arr.length; ++i) {\n        if (arr[i] < min) {\n            min = arr[i];\n            indexOfMin = i;\n        }\n    }\n    return [min, indexOfMin];\n}\n\n\n/**\n * Returns the value and index of the maximum element in an array.\n * @param {number[]|AnyTypedArray} arr array of numbers.\n * @returns {[number, number]} the value and index of the maximum element, of the form: [valueOfMax, indexOfMax]\n * @throws {Error} If array is empty.\n */\nexport function max(arr) {\n    if (arr.length === 0) throw Error('Array must not be empty');\n    let max = arr[0];\n    let indexOfMax = 0;\n    for (let i = 1; i < arr.length; ++i) {\n        if (arr[i] > max) {\n            max = arr[i];\n            indexOfMax = i;\n        }\n    }\n    return [Number(max), indexOfMax];\n}\n\nfunction isPowerOfTwo(number) {\n    // Check if the number is greater than 0 and has only one bit set to 1\n    return (number > 0) && ((number & (number - 1)) === 0);\n}\n\n/**\n * Implementation of Radix-4 FFT.\n * \n * P2FFT class provides functionality for performing Fast Fourier Transform on arrays\n * which are a power of two in length.\n * Code adapted from https://www.npmjs.com/package/fft.js\n */\nclass P2FFT {\n    /**\n     * @param {number} size The size of the input array. Must be a power of two larger than 1.\n     * @throws {Error} FFT size must be a power of two larger than 1.\n     */\n    constructor(size) {\n        this.size = size | 0; // convert to a 32-bit signed integer\n        if (this.size <= 1 || !isPowerOfTwo(this.size))\n            throw new Error('FFT size must be a power of two larger than 1');\n\n        this._csize = size << 1;\n\n        this.table = new Float64Array(this.size * 2);\n        for (let i = 0; i < this.table.length; i += 2) {\n            const angle = Math.PI * i / this.size;\n            this.table[i] = Math.cos(angle);\n            this.table[i + 1] = -Math.sin(angle);\n        }\n\n        // Find size's power of two\n        let power = 0;\n        for (let t = 1; this.size > t; t <<= 1)\n            ++power;\n\n        // Calculate initial step's width:\n        //   * If we are full radix-4, it is 2x smaller to give inital len=8\n        //   * Otherwise it is the same as `power` to give len=4\n        this._width = power % 2 === 0 ? power - 1 : power;\n\n        // Pre-compute bit-reversal patterns\n        this._bitrev = new Int32Array(1 << this._width);\n        for (let j = 0; j < this._bitrev.length; ++j) {\n            this._bitrev[j] = 0;\n            for (let shift = 0; shift < this._width; shift += 2) {\n                const revShift = this._width - shift - 2;\n                this._bitrev[j] |= ((j >>> shift) & 3) << revShift;\n            }\n        }\n    }\n\n    /**\n     * Create a complex number array with size `2 * size`\n     *\n     * @returns {Float64Array} A complex number array with size `2 * size`\n     */\n    createComplexArray() {\n        return new Float64Array(this._csize);\n    }\n\n    /**\n     * Converts a complex number representation stored in a Float64Array to an array of real numbers.\n     * \n     * @param {Float64Array} complex The complex number representation to be converted.\n     * @param {number[]} [storage] An optional array to store the result in.\n     * @returns {number[]} An array of real numbers representing the input complex number representation.\n     */\n    fromComplexArray(complex, storage) {\n        const res = storage || new Array(complex.length >>> 1);\n        for (let i = 0; i < complex.length; i += 2)\n            res[i >>> 1] = complex[i];\n        return res;\n    }\n\n    /**\n     * Convert a real-valued input array to a complex-valued output array.\n     * @param {Float64Array} input The real-valued input array.\n     * @param {Float64Array} [storage] Optional buffer to store the output array.\n     * @returns {Float64Array} The complex-valued output array.\n     */\n    toComplexArray(input, storage) {\n        const res = storage || this.createComplexArray();\n        for (let i = 0; i < res.length; i += 2) {\n            res[i] = input[i >>> 1];\n            res[i + 1] = 0;\n        }\n        return res;\n    }\n\n    /**\n     * Completes the spectrum by adding its mirrored negative frequency components.\n     * @param {Float64Array} spectrum The input spectrum.\n     * @returns {void}\n     */\n    completeSpectrum(spectrum) {\n        const size = this._csize;\n        const half = size >>> 1;\n        for (let i = 2; i < half; i += 2) {\n            spectrum[size - i] = spectrum[i];\n            spectrum[size - i + 1] = -spectrum[i + 1];\n        }\n    }\n\n    /**\n     * Performs a Fast Fourier Transform (FFT) on the given input data and stores the result in the output buffer.\n     * \n     * @param {Float64Array} out The output buffer to store the result.\n     * @param {Float64Array} data The input data to transform.\n     * \n     * @throws {Error} Input and output buffers must be different.\n     * \n     * @returns {void}\n     */\n    transform(out, data) {\n        if (out === data)\n            throw new Error('Input and output buffers must be different');\n\n        this._transform4(out, data, 1 /* DONE */);\n    }\n\n    /**\n     * Performs a real-valued forward FFT on the given input buffer and stores the result in the given output buffer.\n     * The input buffer must contain real values only, while the output buffer will contain complex values. The input and\n     * output buffers must be different.\n     *\n     * @param {Float64Array} out The output buffer.\n     * @param {Float64Array} data The input buffer containing real values.\n     *\n     * @throws {Error} If the input and output buffers are the same.\n     */\n    realTransform(out, data) {\n        if (out === data)\n            throw new Error('Input and output buffers must be different');\n\n        this._realTransform4(out, data, 1 /* DONE */);\n    }\n\n    /**\n     * Performs an inverse FFT transformation on the given `data` array, and stores the result in `out`.\n     * The `out` array must be a different buffer than the `data` array. The `out` array will contain the\n     * result of the transformation. The `data` array will not be modified.\n     * \n     * @param {Float64Array} out The output buffer for the transformed data.\n     * @param {Float64Array} data The input data to transform.\n     * @throws {Error} If `out` and `data` refer to the same buffer.\n     * @returns {void}\n     */\n    inverseTransform(out, data) {\n        if (out === data)\n            throw new Error('Input and output buffers must be different');\n\n        this._transform4(out, data, -1 /* DONE */);\n        for (let i = 0; i < out.length; ++i)\n            out[i] /= this.size;\n    }\n\n    /**\n     * Performs a radix-4 implementation of a discrete Fourier transform on a given set of data.\n     *\n     * @param {Float64Array} out The output buffer for the transformed data.\n     * @param {Float64Array} data The input buffer of data to be transformed.\n     * @param {number} inv A scaling factor to apply to the transform.\n     * @returns {void}\n     */\n    _transform4(out, data, inv) {\n        // radix-4 implementation\n\n        const size = this._csize;\n\n        // Initial step (permute and transform)\n        const width = this._width;\n        let step = 1 << width;\n        let len = (size / step) << 1;\n\n        let outOff;\n        let t;\n        const bitrev = this._bitrev;\n        if (len === 4) {\n            for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) {\n                const off = bitrev[t];\n                this._singleTransform2(data, out, outOff, off, step);\n            }\n        } else {\n            // len === 8\n            for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) {\n                const off = bitrev[t];\n                this._singleTransform4(data, out, outOff, off, step, inv);\n            }\n        }\n\n        // Loop through steps in decreasing order\n        for (step >>= 2; step >= 2; step >>= 2) {\n            len = (size / step) << 1;\n            const quarterLen = len >>> 2;\n\n            // Loop through offsets in the data\n            for (outOff = 0; outOff < size; outOff += len) {\n                // Full case\n                const limit = outOff + quarterLen - 1;\n                for (let i = outOff, k = 0; i < limit; i += 2, k += step) {\n                    const A = i;\n                    const B = A + quarterLen;\n                    const C = B + quarterLen;\n                    const D = C + quarterLen;\n\n                    // Original values\n                    const Ar = out[A];\n                    const Ai = out[A + 1];\n                    const Br = out[B];\n                    const Bi = out[B + 1];\n                    const Cr = out[C];\n                    const Ci = out[C + 1];\n                    const Dr = out[D];\n                    const Di = out[D + 1];\n\n                    const tableBr = this.table[k];\n                    const tableBi = inv * this.table[k + 1];\n                    const MBr = Br * tableBr - Bi * tableBi;\n                    const MBi = Br * tableBi + Bi * tableBr;\n\n                    const tableCr = this.table[2 * k];\n                    const tableCi = inv * this.table[2 * k + 1];\n                    const MCr = Cr * tableCr - Ci * tableCi;\n                    const MCi = Cr * tableCi + Ci * tableCr;\n\n                    const tableDr = this.table[3 * k];\n                    const tableDi = inv * this.table[3 * k + 1];\n                    const MDr = Dr * tableDr - Di * tableDi;\n                    const MDi = Dr * tableDi + Di * tableDr;\n\n                    // Pre-Final values\n                    const T0r = Ar + MCr;\n                    const T0i = Ai + MCi;\n                    const T1r = Ar - MCr;\n                    const T1i = Ai - MCi;\n                    const T2r = MBr + MDr;\n                    const T2i = MBi + MDi;\n                    const T3r = inv * (MBr - MDr);\n                    const T3i = inv * (MBi - MDi);\n\n                    // Final values\n                    out[A] = T0r + T2r;\n                    out[A + 1] = T0i + T2i;\n                    out[B] = T1r + T3i;\n                    out[B + 1] = T1i - T3r;\n                    out[C] = T0r - T2r;\n                    out[C + 1] = T0i - T2i;\n                    out[D] = T1r - T3i;\n                    out[D + 1] = T1i + T3r;\n                }\n            }\n        }\n    }\n\n    /**\n     * Performs a radix-2 implementation of a discrete Fourier transform on a given set of data.\n     *\n     * @param {Float64Array} data The input buffer of data to be transformed.\n     * @param {Float64Array} out The output buffer for the transformed data.\n     * @param {number} outOff The offset at which to write the output data.\n     * @param {number} off The offset at which to begin reading the input data.\n     * @param {number} step The step size for indexing the input data.\n     * @returns {void}\n     */\n    _singleTransform2(data, out, outOff, off, step) {\n        // radix-2 implementation\n        // NOTE: Only called for len=4\n\n        const evenR = data[off];\n        const evenI = data[off + 1];\n        const oddR = data[off + step];\n        const oddI = data[off + step + 1];\n\n        out[outOff] = evenR + oddR;\n        out[outOff + 1] = evenI + oddI;\n        out[outOff + 2] = evenR - oddR;\n        out[outOff + 3] = evenI - oddI;\n    }\n\n    /**\n     * Performs radix-4 transformation on input data of length 8\n     *\n     * @param {Float64Array} data Input data array of length 8\n     * @param {Float64Array} out Output data array of length 8\n     * @param {number} outOff Index of output array to start writing from\n     * @param {number} off Index of input array to start reading from\n     * @param {number} step Step size between elements in input array\n     * @param {number} inv Scaling factor for inverse transform\n     * \n     * @returns {void}\n     */\n    _singleTransform4(data, out, outOff, off, step, inv) {\n        // radix-4\n        // NOTE: Only called for len=8\n        const step2 = step * 2;\n        const step3 = step * 3;\n\n        // Original values\n        const Ar = data[off];\n        const Ai = data[off + 1];\n        const Br = data[off + step];\n        const Bi = data[off + step + 1];\n        const Cr = data[off + step2];\n        const Ci = data[off + step2 + 1];\n        const Dr = data[off + step3];\n        const Di = data[off + step3 + 1];\n\n        // Pre-Final values\n        const T0r = Ar + Cr;\n        const T0i = Ai + Ci;\n        const T1r = Ar - Cr;\n        const T1i = Ai - Ci;\n        const T2r = Br + Dr;\n        const T2i = Bi + Di;\n        const T3r = inv * (Br - Dr);\n        const T3i = inv * (Bi - Di);\n\n        // Final values\n        out[outOff] = T0r + T2r;\n        out[outOff + 1] = T0i + T2i;\n        out[outOff + 2] = T1r + T3i;\n        out[outOff + 3] = T1i - T3r;\n        out[outOff + 4] = T0r - T2r;\n        out[outOff + 5] = T0i - T2i;\n        out[outOff + 6] = T1r - T3i;\n        out[outOff + 7] = T1i + T3r;\n    }\n\n    /**\n     * Real input radix-4 implementation\n     * @param {Float64Array} out Output array for the transformed data\n     * @param {Float64Array} data Input array of real data to be transformed\n     * @param {number} inv The scale factor used to normalize the inverse transform\n     */\n    _realTransform4(out, data, inv) {\n        // Real input radix-4 implementation\n        const size = this._csize;\n\n        // Initial step (permute and transform)\n        const width = this._width;\n        let step = 1 << width;\n        let len = (size / step) << 1;\n\n        let outOff;\n        let t;\n        const bitrev = this._bitrev;\n        if (len === 4) {\n            for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) {\n                const off = bitrev[t];\n                this._singleRealTransform2(data, out, outOff, off >>> 1, step >>> 1);\n            }\n        } else {\n            // len === 8\n            for (outOff = 0, t = 0; outOff < size; outOff += len, ++t) {\n                const off = bitrev[t];\n                this._singleRealTransform4(data, out, outOff, off >>> 1, step >>> 1, inv);\n            }\n        }\n\n        // TODO: Optimize once https://github.com/indutny/fft.js/issues/25 is fixed\n        // Loop through steps in decreasing order\n        for (step >>= 2; step >= 2; step >>= 2) {\n            len = (size / step) << 1;\n            const quarterLen = len >>> 2;\n\n            // Loop through offsets in the data\n            for (outOff = 0; outOff < size; outOff += len) {\n                // Full case\n                const limit = outOff + quarterLen - 1;\n                for (let i = outOff, k = 0; i < limit; i += 2, k += step) {\n                    const A = i;\n                    const B = A + quarterLen;\n                    const C = B + quarterLen;\n                    const D = C + quarterLen;\n\n                    // Original values\n                    const Ar = out[A];\n                    const Ai = out[A + 1];\n                    const Br = out[B];\n                    const Bi = out[B + 1];\n                    const Cr = out[C];\n                    const Ci = out[C + 1];\n                    const Dr = out[D];\n                    const Di = out[D + 1];\n\n                    const tableBr = this.table[k];\n                    const tableBi = inv * this.table[k + 1];\n                    const MBr = Br * tableBr - Bi * tableBi;\n                    const MBi = Br * tableBi + Bi * tableBr;\n\n                    const tableCr = this.table[2 * k];\n                    const tableCi = inv * this.table[2 * k + 1];\n                    const MCr = Cr * tableCr - Ci * tableCi;\n                    const MCi = Cr * tableCi + Ci * tableCr;\n\n                    const tableDr = this.table[3 * k];\n                    const tableDi = inv * this.table[3 * k + 1];\n                    const MDr = Dr * tableDr - Di * tableDi;\n                    const MDi = Dr * tableDi + Di * tableDr;\n\n                    // Pre-Final values\n                    const T0r = Ar + MCr;\n                    const T0i = Ai + MCi;\n                    const T1r = Ar - MCr;\n                    const T1i = Ai - MCi;\n                    const T2r = MBr + MDr;\n                    const T2i = MBi + MDi;\n                    const T3r = inv * (MBr - MDr);\n                    const T3i = inv * (MBi - MDi);\n\n                    // Final values\n                    out[A] = T0r + T2r;\n                    out[A + 1] = T0i + T2i;\n                    out[B] = T1r + T3i;\n                    out[B + 1] = T1i - T3r;\n                    out[C] = T0r - T2r;\n                    out[C + 1] = T0i - T2i;\n                    out[D] = T1r - T3i;\n                    out[D + 1] = T1i + T3r;\n                }\n            }\n        }\n    }\n\n    /**\n     * Performs a single real input radix-2 transformation on the provided data\n     * \n     * @param {Float64Array} data The input data array\n     * @param {Float64Array} out The output data array\n     * @param {number} outOff The output offset\n     * @param {number} off The input offset\n     * @param {number} step The step\n     * \n     * @returns {void}\n     */\n    _singleRealTransform2(data, out, outOff, off, step) {\n        // radix-2 implementation\n        // NOTE: Only called for len=4\n\n        const evenR = data[off];\n        const oddR = data[off + step];\n\n        out[outOff] = evenR + oddR;\n        out[outOff + 1] = 0;\n        out[outOff + 2] = evenR - oddR;\n        out[outOff + 3] = 0;\n    }\n\n    /**\n     * Computes a single real-valued transform using radix-4 algorithm.\n     * This method is only called for len=8.\n     *\n     * @param {Float64Array} data The input data array.\n     * @param {Float64Array} out The output data array.\n     * @param {number} outOff The offset into the output array.\n     * @param {number} off The offset into the input array.\n     * @param {number} step The step size for the input array.\n     * @param {number} inv The value of inverse.\n     */\n    _singleRealTransform4(data, out, outOff, off, step, inv) {\n        // radix-4\n        // NOTE: Only called for len=8\n        const step2 = step * 2;\n        const step3 = step * 3;\n\n        // Original values\n        const Ar = data[off];\n        const Br = data[off + step];\n        const Cr = data[off + step2];\n        const Dr = data[off + step3];\n\n        // Pre-Final values\n        const T0r = Ar + Cr;\n        const T1r = Ar - Cr;\n        const T2r = Br + Dr;\n        const T3r = inv * (Br - Dr);\n\n        // Final values\n        out[outOff] = T0r + T2r;\n        out[outOff + 1] = 0;\n        out[outOff + 2] = T1r;\n        out[outOff + 3] = -T3r;\n        out[outOff + 4] = T0r - T2r;\n        out[outOff + 5] = 0;\n        out[outOff + 6] = T1r;\n        out[outOff + 7] = T3r;\n    }\n}\n\n/**\n * NP2FFT class provides functionality for performing Fast Fourier Transform on arrays\n * which are not a power of two in length. In such cases, the chirp-z transform is used.\n * \n * For more information, see: https://math.stackexchange.com/questions/77118/non-power-of-2-ffts/77156#77156\n */\nclass NP2FFT {\n\n    /**\n     * Constructs a new NP2FFT object.\n     * @param {number} fft_length The length of the FFT\n     */\n    constructor(fft_length) {\n        // Helper variables\n        const a = 2 * (fft_length - 1);\n        const b = 2 * (2 * fft_length - 1);\n        const nextP2 = 2 ** (Math.ceil(Math.log2(b)))\n        this.bufferSize = nextP2;\n        this._a = a;\n\n        // Define buffers\n        // Compute chirp for transform\n        const chirp = new Float64Array(b);\n        const ichirp = new Float64Array(nextP2);\n        this._chirpBuffer = new Float64Array(nextP2);\n        this._buffer1 = new Float64Array(nextP2);\n...\n", "needle_token_start": 5562, "needle_token_end": 5665, "code_context_ntokens": 16011, "output": ["```javascript\n/**\n * Fetches a JSON file from a given path and file name.\n *\n * @param {string} modelPath The path to the directory containing the file.\n * @param {string} fileName The name of the file to fetch.\n * @param {boolean} [fatal=true] Whether to throw an error if the file is not found.\n * @param {PretrainedOptions} [options] An object containing optional parameters.\n * @returns {Promise<Object>} The JSON data parsed into a JavaScript object.\n * @throws Will throw an error if the file is not found and `fatal` is true.\n */\nexport async function getModelJSON(modelPath, fileName, fatal = true, options = {}) {\n    let buffer = await getModelFile(modelPath, fileName, fatal, options);\n    if (buffer === null) {\n        // Return empty object\n        return {}\n    }\n\n    let decoder = new TextDecoder('utf-8');\n    let jsonData = decoder.decode(buffer);\n\n    return JSON.parse(jsonData);\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "hanning", "language": "typescript", "path": "src/utils/audio.js", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a specific type of window function commonly used in signal processing to taper the edges of a signal. This helps to minimize the side effects in the Fourier transform that occur due to the discontinuities at the boundaries of a finite signal.\n2. **Input**: It accepts a single integer that specifies the length of the window.\n3. **Output**: It returns an array of floating-point numbers representing the window values.\n4. **Procedure**: The function calculates the window values using a mathematical formula that involves the cosine function. Each element of the window is computed based on its index, using a specific scaling and shifting of the cosine function to ensure that the first and last values of the window taper smoothly to zero.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/image.js\n\n/**\n * @file Helper module for image processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/image\n */\n\nimport { getFile } from './hub.js';\nimport { env } from '../env.js';\nimport { Tensor } from './tensor.js';\n\n// Will be empty (or not used) if running in browser or web-worker\nimport sharp from 'sharp';\n\nconst BROWSER_ENV = typeof self !== 'undefined';\nconst WEBWORKER_ENV = BROWSER_ENV && self.constructor.name === 'DedicatedWorkerGlobalScope';\n\nlet createCanvasFunction;\nlet ImageDataClass;\nlet loadImageFunction;\nif (BROWSER_ENV) {\n    // Running in browser or web-worker\n    createCanvasFunction = (/** @type {number} */ width, /** @type {number} */ height) => {\n        if (!self.OffscreenCanvas) {\n            throw new Error('OffscreenCanvas not supported by this browser.');\n        }\n        return new self.OffscreenCanvas(width, height)\n    };\n    loadImageFunction = self.createImageBitmap;\n    ImageDataClass = self.ImageData;\n\n} else if (sharp) {\n    // Running in Node.js, electron, or other non-browser environment\n\n    loadImageFunction = async (/**@type {sharp.Sharp}*/img) => {\n        const metadata = await img.metadata();\n        const rawChannels = metadata.channels;\n\n        let { data, info } = await img.raw().toBuffer({ resolveWithObject: true });\n\n        const newImage = new RawImage(new Uint8ClampedArray(data), info.width, info.height, info.channels);\n        if (rawChannels !== undefined && rawChannels !== info.channels) {\n            // Make sure the new image has the same number of channels as the input image.\n            // This is necessary for grayscale images.\n            newImage.convert(rawChannels);\n        }\n        return newImage;\n    }\n\n} else {\n    throw new Error('Unable to load image processing library.');\n}\n\n\n// Defined here: https://github.com/python-pillow/Pillow/blob/a405e8406b83f8bfb8916e93971edc7407b8b1ff/src/libImaging/Imaging.h#L262-L268\nconst RESAMPLING_MAPPING = {\n    0: 'nearest',\n    1: 'lanczos',\n    2: 'bilinear',\n    3: 'bicubic',\n    4: 'box',\n    5: 'hamming',\n}\n\n/**\n * Mapping from file extensions to MIME types.\n */\nconst CONTENT_TYPE_MAP = new Map([\n    ['png', 'image/png'],\n    ['jpg', 'image/jpeg'],\n    ['jpeg', 'image/jpeg'],\n    ['gif', 'image/gif'],\n]);\n\nexport class RawImage {\n\n    /**\n     * Create a new `RawImage` object.\n     * @param {Uint8ClampedArray|Uint8Array} data The pixel data.\n     * @param {number} width The width of the image.\n     * @param {number} height The height of the image.\n     * @param {1|2|3|4} channels The number of channels.\n     */\n    constructor(data, width, height, channels) {\n        this.data = data;\n        this.width = width;\n        this.height = height;\n        this.channels = channels;\n    }\n\n    /** \n     * Returns the size of the image (width, height).\n     * @returns {[number, number]} The size of the image (width, height).\n     */\n    get size() {\n        return [this.width, this.height];\n    }\n\n    /**\n     * Helper method for reading an image from a variety of input types.\n     * @param {RawImage|string|URL} input \n     * @returns The image object.\n     * \n     * **Example:** Read image from a URL.\n     * ```javascript\n     * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg');\n     * // RawImage {\n     * //   \"data\": Uint8ClampedArray [ 25, 25, 25, 19, 19, 19, ... ],\n     * //   \"width\": 800,\n     * //   \"height\": 533,\n     * //   \"channels\": 3\n     * // }\n     * ```\n     */\n    static async read(input) {\n        if (input instanceof RawImage) {\n            return input;\n        } else if (typeof input === 'string' || input instanceof URL) {\n            return await this.fromURL(input);\n        } else {\n            throw new Error(`Unsupported input type: ${typeof input}`);\n        }\n    }\n\n\n    /**\n     * Read an image from a URL or file path.\n     * @param {string|URL} url The URL or file path to read the image from.\n     * @returns {Promise<RawImage>} The image object.\n     */\n    static async fromURL(url) {\n        let response = await getFile(url);\n        if (response.status !== 200) {\n            throw new Error(`Unable to read image from \"${url}\" (${response.status} ${response.statusText})`);\n        }\n        let blob = await response.blob();\n        return this.fromBlob(blob);\n    }\n\n    /**\n     * Helper method to create a new Image from a blob.\n     * @param {Blob} blob The blob to read the image from.\n     * @returns {Promise<RawImage>} The image object.\n     */\n    static async fromBlob(blob) {\n        if (BROWSER_ENV) {\n            // Running in environment with canvas\n            let img = await loadImageFunction(blob);\n\n            const ctx = createCanvasFunction(img.width, img.height).getContext('2d');\n\n            // Draw image to context\n            ctx.drawImage(img, 0, 0);\n\n            return new this(ctx.getImageData(0, 0, img.width, img.height).data, img.width, img.height, 4);\n\n        } else {\n            // Use sharp.js to read (and possible resize) the image.\n            let img = sharp(await blob.arrayBuffer());\n\n            return await loadImageFunction(img);\n        }\n    }\n\n    /**\n     * Helper method to create a new Image from a tensor\n     * @param {Tensor} tensor \n     */\n    static fromTensor(tensor, channel_format = 'CHW') {\n        if (tensor.dims.length !== 3) {\n            throw new Error(`Tensor should have 3 dimensions, but has ${tensor.dims.length} dimensions.`);\n        }\n\n        if (channel_format === 'CHW') {\n            tensor = tensor.transpose(1, 2, 0);\n        } else if (channel_format === 'HWC') {\n            // Do nothing\n        } else {\n            throw new Error(`Unsupported channel format: ${channel_format}`);\n        }\n        if (!(tensor.data instanceof Uint8ClampedArray || tensor.data instanceof Uint8Array)) {\n            throw new Error(`Unsupported tensor type: ${tensor.type}`);\n        }\n        switch (tensor.dims[2]) {\n            case 1:\n            case 2:\n            case 3:\n            case 4:\n                return new RawImage(tensor.data, tensor.dims[1], tensor.dims[0], tensor.dims[2]);\n            default:\n                throw new Error(`Unsupported number of channels: ${tensor.dims[2]}`);\n        }\n    }\n\n    /**\n     * Convert the image to grayscale format.\n     * @returns {RawImage} `this` to support chaining.\n     */\n    grayscale() {\n        if (this.channels === 1) {\n            return this;\n        }\n\n        let newData = new Uint8ClampedArray(this.width * this.height * 1);\n        switch (this.channels) {\n            case 3: // rgb to grayscale\n            case 4: // rgba to grayscale\n                for (let i = 0, offset = 0; i < this.data.length; i += this.channels) {\n                    const red = this.data[i];\n                    const green = this.data[i + 1];\n                    const blue = this.data[i + 2];\n\n                    newData[offset++] = Math.round(0.2989 * red + 0.5870 * green + 0.1140 * blue);\n                }\n                break;\n            default:\n                throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`);\n        }\n        return this._update(newData, this.width, this.height, 1);\n    }\n\n    /**\n     * Convert the image to RGB format.\n     * @returns {RawImage} `this` to support chaining.\n     */\n    rgb() {\n        if (this.channels === 3) {\n            return this;\n        }\n\n        let newData = new Uint8ClampedArray(this.width * this.height * 3);\n\n        switch (this.channels) {\n            case 1: // grayscale to rgb\n                for (let i = 0, offset = 0; i < this.data.length; ++i) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                }\n                break;\n            case 4: // rgba to rgb\n                for (let i = 0, offset = 0; i < this.data.length; i += 4) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i + 1];\n                    newData[offset++] = this.data[i + 2];\n                }\n                break;\n            default:\n                throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`);\n        }\n        return this._update(newData, this.width, this.height, 3);\n\n    }\n\n    /**\n     * Convert the image to RGBA format.\n     * @returns {RawImage} `this` to support chaining.\n     */\n    rgba() {\n        if (this.channels === 4) {\n            return this;\n        }\n\n        let newData = new Uint8ClampedArray(this.width * this.height * 4);\n\n        switch (this.channels) {\n            case 1: // grayscale to rgba\n                for (let i = 0, offset = 0; i < this.data.length; ++i) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = 255;\n                }\n                break;\n            case 3: // rgb to rgba\n                for (let i = 0, offset = 0; i < this.data.length; i += 3) {\n                    newData[offset++] = this.data[i];\n                    newData[offset++] = this.data[i + 1];\n                    newData[offset++] = this.data[i + 2];\n                    newData[offset++] = 255;\n                }\n                break;\n            default:\n                throw new Error(`Conversion failed due to unsupported number of channels: ${this.channels}`);\n        }\n\n        return this._update(newData, this.width, this.height, 4);\n    }\n\n    /**\n     * Resize the image to the given dimensions. This method uses the canvas API to perform the resizing.\n     * @param {number} width The width of the new image.\n     * @param {number} height The height of the new image.\n     * @param {Object} options Additional options for resizing.\n     * @param {0|1|2|3|4|5|string} [options.resample] The resampling method to use.\n     * @returns {Promise<RawImage>} `this` to support chaining.\n     */\n    async resize(width, height, {\n        resample = 2,\n    } = {}) {\n\n        // Ensure resample method is a string\n        let resampleMethod = RESAMPLING_MAPPING[resample] ?? resample;\n\n        if (BROWSER_ENV) {\n            // TODO use `resample` in browser environment\n\n            // Store number of channels before resizing\n            let numChannels = this.channels;\n\n            // Create canvas object for this image\n            let canvas = this.toCanvas();\n\n            // Actually perform resizing using the canvas API\n            const ctx = createCanvasFunction(width, height).getContext('2d');\n\n            // Draw image to context, resizing in the process\n            ctx.drawImage(canvas, 0, 0, width, height);\n\n            // Create image from the resized data\n            let resizedImage = new RawImage(ctx.getImageData(0, 0, width, height).data, width, height, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return resizedImage.convert(numChannels);\n\n        } else {\n            // Create sharp image from raw data, and resize\n            let img = this.toSharp();\n\n            switch (resampleMethod) {\n                case 'box':\n                case 'hamming':\n                    if (resampleMethod === 'box' || resampleMethod === 'hamming') {\n                        console.warn(`Resampling method ${resampleMethod} is not yet supported. Using bilinear instead.`);\n                        resampleMethod = 'bilinear';\n                    }\n\n                case 'nearest':\n                case 'bilinear':\n                case 'bicubic':\n                    // Perform resizing using affine transform. \n                    // This matches how the python Pillow library does it.\n                    img = img.affine([width / this.width, 0, 0, height / this.height], {\n                        interpolator: resampleMethod\n                    });\n                    break;\n\n                case 'lanczos':\n                    // https://github.com/python-pillow/Pillow/discussions/5519\n                    // https://github.com/lovell/sharp/blob/main/docs/api-resize.md\n                    img = img.resize({\n                        width, height,\n                        fit: 'fill',\n                        kernel: 'lanczos3', // PIL Lanczos uses a kernel size of 3 \n                    });\n                    break;\n\n                default:\n                    throw new Error(`Resampling method ${resampleMethod} is not supported.`);\n            }\n\n            return await loadImageFunction(img);\n        }\n\n    }\n\n    async pad([left, right, top, bottom]) {\n        left = Math.max(left, 0);\n        right = Math.max(right, 0);\n        top = Math.max(top, 0);\n        bottom = Math.max(bottom, 0);\n\n        if (left === 0 && right === 0 && top === 0 && bottom === 0) {\n            // No padding needed\n            return this;\n        }\n\n        if (BROWSER_ENV) {\n            // Store number of channels before padding\n            let numChannels = this.channels;\n\n            // Create canvas object for this image\n            let canvas = this.toCanvas();\n\n            let newWidth = this.width + left + right;\n            let newHeight = this.height + top + bottom;\n\n            // Create a new canvas of the desired size.\n            const ctx = createCanvasFunction(newWidth, newHeight).getContext('2d');\n\n            // Draw image to context, padding in the process\n            ctx.drawImage(canvas,\n                0, 0, this.width, this.height,\n                left, top, newWidth, newHeight\n            );\n\n            // Create image from the padded data\n            let paddedImage = new RawImage(\n                ctx.getImageData(0, 0, newWidth, newHeight).data,\n                newWidth, newHeight, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return paddedImage.convert(numChannels);\n\n        } else {\n            let img = this.toSharp().extend({ left, right, top, bottom });\n            return await loadImageFunction(img);\n        }\n    }\n\n    async crop([x_min, y_min, x_max, y_max]) {\n        // Ensure crop bounds are within the image\n        x_min = Math.max(x_min, 0);\n        y_min = Math.max(y_min, 0);\n        x_max = Math.min(x_max, this.width - 1);\n        y_max = Math.min(y_max, this.height - 1);\n\n        // Do nothing if the crop is the entire image\n        if (x_min === 0 && y_min === 0 && x_max === this.width - 1 && y_max === this.height - 1) {\n            return this;\n        }\n\n        const crop_width = x_max - x_min + 1;\n        const crop_height = y_max - y_min + 1;\n\n        if (BROWSER_ENV) {\n            // Store number of channels before resizing\n            const numChannels = this.channels;\n\n            // Create canvas object for this image\n            const canvas = this.toCanvas();\n\n            // Create a new canvas of the desired size. This is needed since if the \n            // image is too small, we need to pad it with black pixels.\n            const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d');\n\n            // Draw image to context, cropping in the process\n            ctx.drawImage(canvas,\n                x_min, y_min, crop_width, crop_height,\n                0, 0, crop_width, crop_height\n            );\n\n            // Create image from the resized data\n            const resizedImage = new RawImage(ctx.getImageData(0, 0, crop_width, crop_height).data, crop_width, crop_height, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return resizedImage.convert(numChannels);\n\n        } else {\n            // Create sharp image from raw data\n            const img = this.toSharp().extract({\n                left: x_min,\n                top: y_min,\n                width: crop_width,\n                height: crop_height,\n            });\n\n            return await loadImageFunction(img);\n        }\n\n    }\n\n    async center_crop(crop_width, crop_height) {\n        // If the image is already the desired size, return it\n        if (this.width === crop_width && this.height === crop_height) {\n            return this;\n        }\n\n        // Determine bounds of the image in the new canvas\n        let width_offset = (this.width - crop_width) / 2;\n        let height_offset = (this.height - crop_height) / 2;\n\n\n        if (BROWSER_ENV) {\n            // Store number of channels before resizing\n            let numChannels = this.channels;\n\n            // Create canvas object for this image\n            let canvas = this.toCanvas();\n\n            // Create a new canvas of the desired size. This is needed since if the \n            // image is too small, we need to pad it with black pixels.\n            const ctx = createCanvasFunction(crop_width, crop_height).getContext('2d');\n\n            let sourceX = 0;\n            let sourceY = 0;\n            let destX = 0;\n            let destY = 0;\n\n            if (width_offset >= 0) {\n                sourceX = width_offset;\n            } else {\n                destX = -width_offset;\n            }\n\n            if (height_offset >= 0) {\n                sourceY = height_offset;\n            } else {\n                destY = -height_offset;\n            }\n\n            // Draw image to context, cropping in the process\n            ctx.drawImage(canvas,\n                sourceX, sourceY, crop_width, crop_height,\n                destX, destY, crop_width, crop_height\n            );\n\n            // Create image from the resized data\n            let resizedImage = new RawImage(ctx.getImageData(0, 0, crop_width, crop_height).data, crop_width, crop_height, 4);\n\n            // Convert back so that image has the same number of channels as before\n            return resizedImage.convert(numChannels);\n\n        } else {\n            // Create sharp image from raw data\n            let img = this.toSharp();\n\n            if (width_offset >= 0 && height_offset >= 0) {\n                // Cropped image lies entirely within the original image\n                img = img.extract({\n                    left: Math.floor(width_offset),\n                    top: Math.floor(height_offset),\n                    width: crop_width,\n                    height: crop_height,\n                })\n            } else if (width_offset <= 0 && height_offset <= 0) {\n                // Cropped image lies entirely outside the original image,\n                // so we add padding\n                let top = Math.floor(-height_offset);\n                let left = Math.floor(-width_offset);\n                img = img.extend({\n                    top: top,\n                    left: left,\n\n                    // Ensures the resulting image has the desired dimensions\n                    right: crop_width - this.width - left,\n                    bottom: crop_height - this.height - top,\n                });\n            } else {\n                // Cropped image lies partially outside the original image.\n                // We first pad, then crop.\n\n                let y_padding = [0, 0];\n                let y_extract = 0;\n                if (height_offset < 0) {\n                    y_padding[0] = Math.floor(-height_offset);\n                    y_padding[1] = crop_height - this.height - y_padding[0];\n                } else {\n                    y_extract = Math.floor(height_offset);\n                }\n\n                let x_padding = [0, 0];\n                let x_extract = 0;\n                if (width_offset < 0) {\n                    x_padding[0] = Math.floor(-width_offset);\n                    x_padding[1] = crop_width - this.width - x_padding[0];\n                } else {\n                    x_extract = Math.floor(width_offset);\n                }\n\n                img = img.extend({\n                    top: y_padding[0],\n                    bottom: y_padding[1],\n                    left: x_padding[0],\n                    right: x_padding[1],\n                }).extract({\n                    left: x_extract,\n                    top: y_extract,\n                    width: crop_width,\n                    height: crop_height,\n                })\n            }\n\n            return await loadImageFunction(img);\n        }\n    }\n\n    async toBlob(type = 'image/png', quality = 1) {\n        if (!BROWSER_ENV) {\n            throw new Error('toBlob() is only supported in browser environments.')\n        }\n\n        const canvas = this.toCanvas();\n        return await canvas.convertToBlob({ type, quality });\n    }\n\n    toTensor(channel_format = 'CHW') {\n        let tensor = new Tensor(\n            'uint8',\n            new Uint8Array(this.data),\n            [this.height, this.width, this.channels]\n        );\n\n        if (channel_format === 'HWC') {\n            // Do nothing\n        } else if (channel_format === 'CHW') { // hwc -> chw\n            tensor = tensor.permute(2, 0, 1);\n        } else {\n            throw new Error(`Unsupported channel format: ${channel_format}`);\n        }\n        return tensor;\n    }\n\n    toCanvas() {\n        if (!BROWSER_ENV) {\n            throw new Error('toCanvas() is only supported in browser environments.')\n        }\n\n...\n// Path: src/utils/audio.js\n/**\n * @file Helper module for audio processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/audio\n */\n\nimport {\n    getFile,\n} from './hub.js';\nimport { FFT, max } from './maths.js';\nimport {\n    calculateReflectOffset,\n} from './core.js';\n\n\n/**\n * Helper function to read audio from a path/URL.\n * @param {string|URL} url The path/URL to load the audio from.\n * @param {number} sampling_rate The sampling rate to use when decoding the audio.\n * @returns {Promise<Float32Array>} The decoded audio as a `Float32Array`.\n */\nexport async function read_audio(url, sampling_rate) {\n    if (typeof AudioContext === 'undefined') {\n        // Running in node or an environment without AudioContext\n        throw Error(\n            \"Unable to load audio from path/URL since `AudioContext` is not available in your environment. \" +\n            \"Instead, audio data should be passed directly to the pipeline/processor. \" +\n            \"For more information and some example code, see https://huggingface.co/docs/transformers.js/guides/node-audio-processing.\"\n        )\n    }\n\n    const response = await (await getFile(url)).arrayBuffer();\n    const audioCTX = new AudioContext({ sampleRate: sampling_rate });\n    if (typeof sampling_rate === 'undefined') {\n        console.warn(`No sampling rate provided, using default of ${audioCTX.sampleRate}Hz.`)\n    }\n    const decoded = await audioCTX.decodeAudioData(response);\n\n    /** @type {Float32Array} */\n    let audio;\n\n    // We now replicate HuggingFace's `ffmpeg_read` method:\n    if (decoded.numberOfChannels === 2) {\n        // When downmixing a stereo audio file to mono using the -ac 1 option in FFmpeg,\n        // the audio signal is summed across both channels to create a single mono channel.\n        // However, if the audio is at full scale (i.e. the highest possible volume level),\n        // the summing of the two channels can cause the audio signal to clip or distort.\n\n        // To prevent this clipping, FFmpeg applies a scaling factor of 1/sqrt(2) (~ 0.707)\n        // to the audio signal before summing the two channels. This scaling factor ensures\n        // that the combined audio signal will not exceed the maximum possible level, even\n        // if both channels are at full scale.\n\n        // After applying this scaling factor, the audio signal from both channels is summed\n        // to create a single mono channel. It's worth noting that this scaling factor is\n        // only applied when downmixing stereo audio to mono using the -ac 1 option in FFmpeg.\n        // If you're using a different downmixing method, or if you're not downmixing the\n        // audio at all, this scaling factor may not be needed.\n        const SCALING_FACTOR = Math.sqrt(2);\n\n        const left = decoded.getChannelData(0);\n        const right = decoded.getChannelData(1);\n\n        audio = new Float32Array(left.length);\n        for (let i = 0; i < decoded.length; ++i) {\n            audio[i] = SCALING_FACTOR * (left[i] + right[i]) / 2;\n        }\n\n    } else {\n        // If the audio is not stereo, we can just use the first channel:\n        audio = decoded.getChannelData(0);\n    }\n\n    return audio;\n}\n\n/**\n * Generates a Hanning window of length M.\n *\n * @param {number} M The length of the Hanning window to generate.\n * @returns {Float64Array} The generated Hanning window.\n */\nexport \nfunction hanning(M) {\n    if (M < 1) {\n        return new Float64Array();\n    }\n    if (M === 1) {\n        return new Float64Array([1]);\n    }\n    const denom = M - 1;\n    const factor = Math.PI / denom;\n    const cos_vals = new Float64Array(M);\n    for (let i = 0; i < M; ++i) {\n        const n = 2 * i - denom;\n        cos_vals[i] = 0.5 + 0.5 * Math.cos(factor * n);\n    }\n    return cos_vals;\n}\n\nconst HERTZ_TO_MEL_MAPPING = {\n    \"htk\": (/** @type {number} */ freq) => 2595.0 * Math.log10(1.0 + (freq / 700.0)),\n    \"kaldi\": (/** @type {number} */ freq) => 1127.0 * Math.log(1.0 + (freq / 700.0)),\n    \"slaney\": (/** @type {number} */ freq, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = 27.0 / Math.log(6.4)) =>\n        freq >= min_log_hertz\n            ? min_log_mel + Math.log(freq / min_log_hertz) * logstep\n            : 3.0 * freq / 200.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} freq \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction hertz_to_mel(freq, mel_scale = \"htk\") {\n    const fn = HERTZ_TO_MEL_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof freq === 'number' ? fn(freq) : freq.map(x => fn(x));\n}\n\nconst MEL_TO_HERTZ_MAPPING = {\n    \"htk\": (/** @type {number} */ mels) => 700.0 * (10.0 ** (mels / 2595.0) - 1.0),\n    \"kaldi\": (/** @type {number} */ mels) => 700.0 * (Math.exp(mels / 1127.0) - 1.0),\n    \"slaney\": (/** @type {number} */ mels, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = Math.log(6.4) / 27.0) => mels >= min_log_mel\n        ? min_log_hertz * Math.exp(logstep * (mels - min_log_mel))\n        : 200.0 * mels / 3.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} mels \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction mel_to_hertz(mels, mel_scale = \"htk\") {\n    const fn = MEL_TO_HERTZ_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof mels === 'number' ? fn(mels) : mels.map(x => fn(x));\n}\n\n/**\n* Creates a triangular filter bank.\n*\n* Adapted from torchaudio and librosa.\n*\n* @param {Float64Array} fft_freqs Discrete frequencies of the FFT bins in Hz, of shape `(num_frequency_bins,)`.\n* @param {Float64Array} filter_freqs Center frequencies of the triangular filters to create, in Hz, of shape `(num_mel_filters,)`.\n* @returns {number[][]} of shape `(num_frequency_bins, num_mel_filters)`.\n*/\nfunction _create_triangular_filter_bank(fft_freqs, filter_freqs) {\n    const filter_diff = Float64Array.from(\n        { length: filter_freqs.length - 1 },\n        (_, i) => filter_freqs[i + 1] - filter_freqs[i]\n    );\n\n    const slopes = Array.from({\n        length: fft_freqs.length\n    }, () => new Array(filter_freqs.length));\n\n    for (let j = 0; j < fft_freqs.length; ++j) {\n        const slope = slopes[j];\n        for (let i = 0; i < filter_freqs.length; ++i) {\n            slope[i] = filter_freqs[i] - fft_freqs[j];\n        }\n    }\n\n    const numFreqs = filter_freqs.length - 2;\n    const ret = Array.from({ length: numFreqs }, () => new Array(fft_freqs.length));\n\n    for (let j = 0; j < fft_freqs.length; ++j) { // 201\n        const slope = slopes[j];\n        for (let i = 0; i < numFreqs; ++i) { // 80\n            const down = -slope[i] / filter_diff[i];\n            const up = slope[i + 2] / filter_diff[i + 1];\n            ret[i][j] = Math.max(0, Math.min(down, up));\n        }\n    }\n    return ret;\n}\n\n/**\n * Return evenly spaced numbers over a specified interval.\n * @param {number} start The starting value of the sequence.\n * @param {number} end The end value of the sequence.\n * @param {number} num Number of samples to generate.\n * @returns `num` evenly spaced samples, calculated over the interval `[start, stop]`.\n */\nfunction linspace(start, end, num) {\n    const step = (end - start) / (num - 1);\n    return Float64Array.from({ length: num }, (_, i) => start + step * i);\n}\n\n/**\n * Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and\n * various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters\n * are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these\n * features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency.\n * @param {number} num_frequency_bins Number of frequencies used to compute the spectrogram (should be the same as in `stft`).\n * @param {number} num_mel_filters Number of mel filters to generate.\n * @param {number} min_frequency Lowest frequency of interest in Hz.\n * @param {number} max_frequency Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`.\n * @param {number} sampling_rate Sample rate of the audio waveform.\n * @param {string} [norm] If `\"slaney\"`, divide the triangular mel weights by the width of the mel band (area normalization).\n * @param {string} [mel_scale] The mel frequency scale to use, `\"htk\"` or `\"slaney\"`.\n * @param {boolean} [triangularize_in_mel_space] If this option is enabled, the triangular filter is applied in mel space rather than frequency space.\n * This should be set to `true` in order to get the same results as `torchaudio` when computing mel filters.\n * @returns {number[][]} Triangular filter bank matrix, which is a 2D array of shape (`num_frequency_bins`, `num_mel_filters`).\n * This is a projection matrix to go from a spectrogram to a mel spectrogram.\n */\nexport function mel_filter_bank(\n    num_frequency_bins,\n    num_mel_filters,\n    min_frequency,\n    max_frequency,\n    sampling_rate,\n    norm = null,\n    mel_scale = \"htk\",\n    triangularize_in_mel_space = false,\n) {\n    if (norm !== null && norm !== \"slaney\") {\n        throw new Error('norm must be one of null or \"slaney\"');\n    }\n\n    const mel_min = hertz_to_mel(min_frequency, mel_scale);\n    const mel_max = hertz_to_mel(max_frequency, mel_scale);\n    const mel_freqs = linspace(mel_min, mel_max, num_mel_filters + 2);\n\n    let filter_freqs = mel_to_hertz(mel_freqs, mel_scale);\n    let fft_freqs; // frequencies of FFT bins in Hz\n\n    if (triangularize_in_mel_space) {\n        const fft_bin_width = sampling_rate / (num_frequency_bins * 2);\n        fft_freqs = hertz_to_mel(Float64Array.from({ length: num_frequency_bins }, (_, i) => i * fft_bin_width), mel_scale);\n        filter_freqs = mel_freqs;\n    } else {\n        fft_freqs = linspace(0, Math.floor(sampling_rate / 2), num_frequency_bins);\n    }\n\n    const mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs);\n\n    if (norm !== null && norm === \"slaney\") {\n        // Slaney-style mel is scaled to be approx constant energy per channel\n        for (let i = 0; i < num_mel_filters; ++i) {\n            const filter = mel_filters[i];\n            const enorm = 2.0 / (filter_freqs[i + 2] - filter_freqs[i]);\n            for (let j = 0; j < num_frequency_bins; ++j) {\n                // Apply this enorm to all frequency bins\n                filter[j] *= enorm;\n            }\n        }\n    }\n\n    // TODO warn if there is a zero row\n\n    return mel_filters;\n\n}\n\n/**\n * @template {Float32Array|Float64Array} T\n * Pads an array with a reflected version of itself on both ends.\n * @param {T} array The array to pad.\n * @param {number} left The amount of padding to add to the left.\n * @param {number} right The amount of padding to add to the right.\n * @returns {T} The padded array.\n */\nfunction padReflect(array, left, right) {\n    // @ts-ignore\n    const padded = new array.constructor(array.length + left + right);\n    const w = array.length - 1;\n\n    for (let i = 0; i < array.length; ++i) {\n        padded[left + i] = array[i];\n    }\n\n    for (let i = 1; i <= left; ++i) {\n        padded[left - i] = array[calculateReflectOffset(i, w)];\n    }\n\n    for (let i = 1; i <= right; ++i) {\n        padded[w + left + i] = array[calculateReflectOffset(w - i, w)];\n    }\n\n    return padded;\n}\n\n/**\n * Helper function to compute `amplitude_to_db` and `power_to_db`.\n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram \n * @param {number} factor \n * @param {number} reference \n * @param {number} min_value \n * @param {number} db_range \n * @returns {T}\n */\nfunction _db_conversion_helper(spectrogram, factor, reference, min_value, db_range) {\n    if (reference <= 0) {\n        throw new Error('reference must be greater than zero');\n    }\n\n    if (min_value <= 0) {\n        throw new Error('min_value must be greater than zero');\n    }\n\n    reference = Math.max(min_value, reference);\n\n    const logReference = Math.log10(reference);\n    for (let i = 0; i < spectrogram.length; ++i) {\n        spectrogram[i] = factor * Math.log10(Math.max(min_value, spectrogram[i]) - logReference)\n    }\n\n    if (db_range !== null) {\n        if (db_range <= 0) {\n            throw new Error('db_range must be greater than zero');\n        }\n        const maxValue = max(spectrogram)[0] - db_range;\n        for (let i = 0; i < spectrogram.length; ++i) {\n            spectrogram[i] = Math.max(spectrogram[i], maxValue);\n        }\n    }\n\n    return spectrogram;\n}\n\n/**\n * Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`,\n * using basic logarithm properties for numerical stability. NOTE: Operates in-place.\n * \n * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a\n * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.\n * This means that large variations in energy may not sound all that different if the sound is loud to begin with.\n * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.\n * \n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram The input amplitude (mel) spectrogram.\n * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB.\n * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [min_value=1e-5] The spectrogram will be clipped to this minimum value before conversion to decibels,\n * to avoid taking `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero.\n * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the\n * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @returns {T} The modified spectrogram in decibels.\n */\nfunction amplitude_to_db(spectrogram, reference = 1.0, min_value = 1e-5, db_range = null) {\n    return _db_conversion_helper(spectrogram, 20.0, reference, min_value, db_range);\n}\n\n/**\n * Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`,\n * using basic logarithm properties for numerical stability. NOTE: Operates in-place.\n * \n * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a\n * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.\n * This means that large variations in energy may not sound all that different if the sound is loud to begin with.\n * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.\n * \n * Based on the implementation of `librosa.power_to_db`.\n * \n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared!\n * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB.\n * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels,\n * to avoid taking `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero.\n * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the\n * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @returns {T} The modified spectrogram in decibels.\n */\nfunction power_to_db(spectrogram, reference = 1.0, min_value = 1e-10, db_range = null) {\n    return _db_conversion_helper(spectrogram, 10.0, reference, min_value, db_range);\n}\n\n/**\n * Calculates a spectrogram over one waveform using the Short-Time Fourier Transform.\n * \n * This function can create the following kinds of spectrograms:\n *   - amplitude spectrogram (`power = 1.0`)\n *   - power spectrogram (`power = 2.0`)\n *   - complex-valued spectrogram (`power = None`)\n *   - log spectrogram (use `log_mel` argument)\n *   - mel spectrogram (provide `mel_filters`)\n *   - log-mel spectrogram (provide `mel_filters` and `log_mel`)\n *\n * In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame.\n * A padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, \n * typically the next power of two.\n * \n * @param {Float32Array|Float64Array} waveform The input waveform of shape `(length,)`. This must be a single real-valued, mono waveform.\n * @param {Float32Array|Float64Array} window The windowing function to apply of shape `(frame_length,)`, including zero-padding if necessary. The actual window length may be\n * shorter than `frame_length`, but we're assuming the array has already been zero-padded.\n * @param {number} frame_length The length of the analysis frames in samples (a.k.a., `fft_length`).\n * @param {number} hop_length The stride between successive analysis frames in samples.\n * @param {Object} options\n * @param {number} [options.fft_length=null] The size of the FFT buffer in samples. This determines how many frequency bins the spectrogram will have.\n * For optimal speed, this should be a power of two. If `null`, uses `frame_length`.\n * @param {number} [options.power=1.0] If 1.0, returns the amplitude spectrogram. If 2.0, returns the power spectrogram. If `null`, returns complex numbers.\n * @param {boolean} [options.center=true] Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `false`, frame\n * `t` will start at time `t * hop_length`.\n * @param {string} [options.pad_mode=\"reflect\"] Padding mode used when `center` is `true`. Possible values are: `\"constant\"` (pad with zeros),\n * `\"edge\"` (pad with edge values), `\"reflect\"` (pads with mirrored values).\n * @param {boolean} [options.onesided=true] If `true`, only computes the positive frequencies and returns a spectrogram containing `fft_length // 2 + 1`\n * frequency bins. If `false`, also computes the negative frequencies and returns `fft_length` frequency bins.\n * @param {number} [options.preemphasis=null] Coefficient for a low-pass filter that applies pre-emphasis before the DFT.\n * @param {number[][]} [options.mel_filters=null] The mel filter bank of shape `(num_freq_bins, num_mel_filters)`.\n * If supplied, applies this filter bank to create a mel spectrogram.\n * @param {number} [options.mel_floor=1e-10] Minimum value of mel frequency banks.\n * @param {string} [options.log_mel=null] How to convert the spectrogram to log scale. Possible options are:\n * `null` (don't convert), `\"log\"` (take the natural logarithm) `\"log10\"` (take the base-10 logarithm), `\"dB\"` (convert to decibels).\n * Can only be used when `power` is not `null`.\n * @param {number} [options.reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. For example, use `max(spectrogram)[0]` to set\n * the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [options.min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking `log(0)`.\n * For a power spectrogram, the default of `1e-10` corresponds to a minimum of -100 dB. For an amplitude spectrogram, the value `1e-5` corresponds to -100 dB.\n * Must be greater than zero.\n * @param {number} [options.db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the\n * peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @param {boolean} [options.remove_dc_offset=null] Subtract mean from waveform on each frame, applied before pre-emphasis. This should be set to `true` in\n * order to get the same results as `torchaudio.compliance.kaldi.fbank` when computing mel filters.\n * @param {number} [options.max_num_frames=null] If provided, limits the number of frames to compute to this value.\n * @param {boolean} [options.do_pad=true] If `true`, pads the output spectrogram to have `max_num_frames` frames.\n * @param {boolean} [options.transpose=false] If `true`, the returned spectrogram will have shape `(num_frames, num_frequency_bins/num_mel_filters)`. If `false`, the returned spectrogram will have shape `(num_frequency_bins/num_mel_filters, num_frames)`.\n * @returns {{data: Float32Array, dims: number[]}} Spectrogram of shape `(num_frequency_bins, length)` (regular spectrogram) or shape `(num_mel_filters, length)` (mel spectrogram).\n */\nexport function spectrogram(\n    waveform,\n    window,\n    frame_length,\n    hop_length,\n    {\n        fft_length = null,\n        power = 1.0,\n        center = true,\n        pad_mode = \"reflect\",\n        onesided = true,\n        preemphasis = null,\n        mel_filters = null,\n        mel_floor = 1e-10,\n        log_mel = null,\n        reference = 1.0,\n        min_value = 1e-10,\n        db_range = null,\n        remove_dc_offset = null,\n\n        // Custom parameters for efficiency reasons\n        max_num_frames = null,\n        do_pad = true,\n        transpose = false,\n    } = {}\n) {\n    const window_length = window.length;\n    if (fft_length === null) {\n        fft_length = frame_length;\n    }\n    if (frame_length > fft_length) {\n        throw Error(`frame_length (${frame_length}) may not be larger than fft_length (${fft_length})`)\n    }\n\n    if (window_length !== frame_length) {\n        throw new Error(`Length of the window (${window_length}) must equal frame_length (${frame_length})`);\n    }\n\n    if (hop_length <= 0) {\n        throw new Error(\"hop_length must be greater than zero\");\n    }\n\n    if (center) {\n        if (pad_mode !== 'reflect') {\n            throw new Error(`pad_mode=\"${pad_mode}\" not implemented yet.`)\n        }\n        const half_window = Math.floor((fft_length - 1) / 2) + 1;\n        waveform = padReflect(waveform, half_window, half_window);\n    }\n\n    // split waveform into frames of frame_length size\n    const num_frames = Math.floor(1 + Math.floor((waveform.length - frame_length) / hop_length))\n\n    const num_frequency_bins = onesided ? Math.floor(fft_length / 2) + 1 : fft_length\n\n    let d1 = num_frames;\n    let d1Max = num_frames;\n\n    // If maximum number of frames is provided, we must either pad or truncate\n    if (max_num_frames !== null) {\n        if (max_num_frames > num_frames) { // input is too short, so we pad\n            if (do_pad) {\n                d1Max = max_num_frames;\n            }\n        } else { // input is too long, so we truncate\n            d1Max = d1 = max_num_frames;\n        }\n    }\n\n    // Preallocate arrays to store output.\n    const fft = new FFT(fft_length);\n    const inputBuffer = new Float64Array(fft_length);\n    const outputBuffer = new Float64Array(fft.outputBufferSize);\n    const magnitudes = new Array(d1);\n\n    for (let i = 0; i < d1; ++i) {\n        // Populate buffer with waveform data\n        const offset = i * hop_length;\n        for (let j = 0; j < frame_length; ++j) {\n            inputBuffer[j] = waveform[offset + j];\n        }\n\n        if (remove_dc_offset) {\n            let sum = 0;\n            for (let j = 0; j < frame_length; ++j) {\n                sum += inputBuffer[j];\n            }\n            const mean = sum / frame_length;\n            for (let j = 0; j < frame_length; ++j) {\n                inputBuffer[j] -= mean;\n            }\n        }\n\n        if (preemphasis !== null) {\n            // Done in reverse to avoid copies and distructive modification\n            for (let j = frame_length - 1; j >= 1; --j) {\n                inputBuffer[j] -= preemphasis * inputBuffer[j - 1];\n            }\n            inputBuffer[0] *= 1 - preemphasis;\n        }\n\n        for (let j = 0; j < window.length; ++j) {\n            inputBuffer[j] *= window[j];\n        }\n\n        fft.realTransform(outputBuffer, inputBuffer);\n\n        // compute magnitudes\n        const row = new Array(num_frequency_bins);\n        for (let j = 0; j < row.length; ++j) {\n            const j2 = j << 1;\n            row[j] = outputBuffer[j2] ** 2 + outputBuffer[j2 + 1] ** 2;\n        }\n        magnitudes[i] = row;\n    }\n\n    // TODO what should happen if power is None?\n    // https://github.com/huggingface/transformers/issues/27772\n    if (power !== null && power !== 2) {\n        // slight optimization to not sqrt\n        const pow = 2 / power; // we use 2 since we already squared\n        for (let i = 0; i < magnitudes.length; ++i) {\n            const magnitude = magnitudes[i];\n            for (let j = 0; j < magnitude.length; ++j) {\n                magnitude[j] **= pow;\n            }\n        }\n    }\n\n    // TODO: What if `mel_filters` is null?\n    const num_mel_filters = mel_filters.length;\n\n    // Only here do we create Float32Array\n    const mel_spec = new Float32Array(num_mel_filters * d1Max);\n\n    // Perform matrix muliplication:\n    // mel_spec = mel_filters @ magnitudes.T\n    //  - mel_filters.shape=(80, 201)\n    //  - magnitudes.shape=(3000, 201) => - magnitudes.T.shape=(201, 3000)\n    //  - mel_spec.shape=(80, 3000)\n    const dims = transpose ? [d1Max, num_mel_filters] : [num_mel_filters, d1Max];\n    for (let i = 0; i < num_mel_filters; ++i) { // num melfilters (e.g., 80)\n        const filter = mel_filters[i];\n        for (let j = 0; j < d1; ++j) { // num frames (e.g., 3000)\n            const magnitude = magnitudes[j];\n\n            let sum = 0;\n            for (let k = 0; k < num_frequency_bins; ++k) { // num frequency bins (e.g., 201)\n                sum += filter[k] * magnitude[k];\n            }\n\n            mel_spec[\n                transpose\n                    ? j * num_mel_filters + i\n                    : i * d1 + j\n            ] = Math.max(mel_floor, sum);\n        }\n    }\n\n    if (power !== null && log_mel !== null) {\n        const o = Math.min(mel_spec.length, d1 * num_mel_filters);\n        switch (log_mel) {\n            case 'log':\n                for (let i = 0; i < o; ++i) {\n                    mel_spec[i] = Math.log(mel_spec[i]);\n                }\n                break;\n            case 'log10':\n                for (let i = 0; i < o; ++i) {\n                    mel_spec[i] = Math.log10(mel_spec[i]);\n                }\n                break;\n            case 'dB':\n                if (power === 1.0) {\n                    // NOTE: operates in-place\n                    amplitude_to_db(mel_spec, reference, min_value, db_range);\n                } else if (power === 2.0) {\n                    power_to_db(mel_spec, reference, min_value, db_range);\n                } else {\n                    throw new Error(`Cannot use log_mel option '${log_mel}' with power ${power}`)\n                }\n                break;\n            default:\n                throw new Error(`log_mel must be one of null, 'log', 'log10' or 'dB'. Got '${log_mel}'`);\n        }\n    }\n\n    return { data: mel_spec, dims };\n}\n\n/**\n * Returns an array containing the specified window.\n * @param {number} window_length The length of the window in samples.\n * @param {string} name The name of the window function.\n * @param {Object} options Additional options.\n * @param {boolean} [options.periodic=true] Whether the window is periodic or symmetric.\n * @param {number} [options.frame_length=null] The length of the analysis frames in samples.\n * Provide a value for `frame_length` if the window is smaller than the frame length, so that it will be zero-padded.\n * @param {boolean} [options.center=true] Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided.\n * @returns {Float64Array} The window of shape `(window_length,)` or `(frame_length,)`.\n */\nexport function window_function(window_length, name, {\n    periodic = true,\n    frame_length = null,\n    center = true,\n} = {}) {\n    const length = periodic ? window_length + 1 : window_length;\n    let window;\n    switch (name) {\n        case 'boxcar':\n            window = new Float64Array(length).fill(1.0);\n            break;\n        case 'hann':\n        case 'hann_window':\n            window = hanning(length);\n            break;\n        case 'povey':\n            window = hanning(length).map(x => Math.pow(x, 0.85));\n            break;\n        default:\n            throw new Error(`Unknown window type ${name}.`);\n    }\n    if (periodic) {\n        window = window.subarray(0, window_length);\n    }\n    if (frame_length === null) {\n        return window;\n    }\n    if (window_length > frame_length) {\n        throw new Error(`Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`);\n    }\n\n    return window;\n}\n\n// Path: src/processors.js\n\n/**\n * @file Processors are used to prepare non-textual inputs (e.g., image or audio) for a model.\n * \n * **Example:** Using a `WhisperProcessor` to prepare an audio input for a model.\n * ```javascript\n * import { AutoProcessor, read_audio } from '@xenova/transformers';\n *\n * let processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en');\n * let audio = await read_audio('https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac', 16000);\n * let { input_features } = await processor(audio);\n * // Tensor {\n * //   data: Float32Array(240000) [0.4752984642982483, 0.5597258806228638, 0.56434166431427, ...],\n * //   dims: [1, 80, 3000],\n * //   type: 'float32',\n * //   size: 240000,\n * // }\n * ```\n * \n * @module processors\n */\nimport {\n    Callable,\n    calculateDimensions,\n    calculateReflectOffset,\n} from './utils/core.js';\n\nimport {\n    getModelJSON,\n} from './utils/hub.js';\n\nimport {\n    min,\n    max,\n    softmax,\n    bankers_round,\n} from './utils/maths.js';\n\n\nimport { Tensor, permute, cat, interpolate, stack } from './utils/tensor.js';\n\nimport { RawImage } from './utils/image.js';\nimport {\n...\n", "needle_token_start": 7143, "needle_token_end": 7307, "code_context_ntokens": 16004, "output": ["```javascript\n/**\n * Returns an array containing the specified window.\n * @param {number} window_length The length of the window in samples.\n * @param {string} name The name of the window function.\n * @param {Object} options Additional options.\n * @param {boolean} [options.periodic=true] Whether the window is periodic or symmetric.\n * @param {number} [options.frame_length=null] The length of the analysis frames in samples.\n * Provide a value for `frame_length` if the window is smaller than the frame length, so that it will be zero-padded.\n * @param {boolean} [options.center=true] Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided.\n * @returns {Float64Array} The window of shape `(window_length,)` or `(frame_length,)`.\n */\nexport function window_function(window_length, name, {\n    periodic = true,\n    frame_length = null,\n    center = true,\n} = {}) {\n    const length = periodic ? window_length + 1 : window_length;\n    let window;\n    switch (name) {\n        case 'boxcar':\n            window = new Float64Array(length).fill(1.0);\n            break;\n        case 'hann':\n        case 'hann_window':\n            window = hanning(length);\n            break;\n        case 'povey':\n            window = hanning(length).map(x => Math.pow(x, 0.85));\n            break;\n        default:\n            throw new Error(`Unknown window type ${name}.`);\n    }\n    if (periodic) {\n        window = window.subarray(0, window_length);\n    }\n    if (frame_length === null) {\n        return window;\n    }\n    if (window_length > frame_length) {\n        throw new Error(`Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`);\n    }\n\n    return window;\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "dispatchCallback", "language": "typescript", "path": "src/utils/core.js", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To execute a given callback function with specified data, primarily used for updating or notifying progress within internal operations.\n2. **Input**: A callback function and the data to be passed to this function.\n3. **Output**: There is no return value; the function operates solely to invoke the callback.\n4. **Procedure**: Checks if the callback function is provided; if it is, the function is called with the specified data as its argument.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/core.js\n\n/**\n * @file Core utility functions/classes for Transformers.js.\n * \n * These are only used internally, meaning an end-user shouldn't\n * need to access anything here.\n * \n * @module utils/core\n */\n\n/**\n * Helper function to dispatch progress callbacks.\n *\n * @param {Function} progress_callback The progress callback function to dispatch.\n * @param {any} data The data to pass to the progress callback function.\n * @returns {void}\n * @private\n */\nexport \nfunction dispatchCallback(progress_callback, data) {\n    if (progress_callback) progress_callback(data);\n}\n\n/**\n * Reverses the keys and values of an object.\n *\n * @param {Object} data The object to reverse.\n * @returns {Object} The reversed object.\n * @see https://ultimatecourses.com/blog/reverse-object-keys-and-values-in-javascript\n */\nexport function reverseDictionary(data) {\n    // https://ultimatecourses.com/blog/reverse-object-keys-and-values-in-javascript\n    return Object.fromEntries(Object.entries(data).map(([key, value]) => [value, key]));\n}\n\n/**\n * Escapes regular expression special characters from a string by replacing them with their escaped counterparts.\n *\n * @param {string} string The string to escape.\n * @returns {string} The escaped string.\n */\nexport function escapeRegExp(string) {\n    return string.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'); // $& means the whole matched string\n}\n\n/**\n * A base class for creating callable objects.\n * \n * @type {new () => {(...args: any[]): any, _call(...args: any[]): any}}\n */\nexport const Callable = /** @type {any} */ (class {\n    /**\n    * Creates a new instance of the Callable class.\n    */\n    constructor() {\n        /**\n         * Creates a closure that delegates to a private method '_call' with the given arguments.\n         * @type {any}\n         * @param {...any} args Zero or more arguments to pass to the '_call' method.\n         * @returns {*} The result of calling the '_call' method.\n         */\n        let closure = function (...args) {\n            return closure._call(...args)\n        }\n        return Object.setPrototypeOf(closure, new.target.prototype)\n    }\n\n    /**\n     * This method should be implemented in subclasses to provide the\n     * functionality of the callable object.\n     *\n     * @param {any[]} args\n     * @throws {Error} If the subclass does not implement the `_call` method.\n     */\n    _call(...args) {\n        throw Error('Must implement _call method in subclass')\n    }\n});\n\n/**\n * Check if a value is a typed array.\n * @param {*} val The value to check.\n * @returns {boolean} True if the value is a `TypedArray`, false otherwise.\n * \n * Adapted from https://stackoverflow.com/a/71091338/13989043\n */\nexport function isTypedArray(val) {\n    return val?.prototype?.__proto__?.constructor?.name === 'TypedArray';\n}\n\n\n/**\n * Check if a value is an integer.\n * @param {*} x The value to check.\n * @returns {boolean} True if the value is a string, false otherwise.\n */\nexport function isIntegralNumber(x) {\n    return Number.isInteger(x) || typeof x === 'bigint'\n}\n\n/**\n * Check if a value is exists.\n * @param {*} x The value to check.\n * @returns {boolean} True if the value exists, false otherwise.\n */\nexport function exists(x) {\n    return x !== undefined && x !== null;\n}\n\n/**\n * Calculates the dimensions of a nested array.\n *\n * @param {any[]} arr The nested array to calculate dimensions for.\n * @returns {number[]} An array containing the dimensions of the input array.\n */\nexport function calculateDimensions(arr) {\n    const dimensions = [];\n    let current = arr;\n    while (Array.isArray(current)) {\n        dimensions.push(current.length);\n        current = current[0];\n    }\n    return dimensions;\n}\n\n/**\n * Replicate python's .pop() method for objects.\n * @param {Object} obj The object to pop from.\n * @param {string} key The key to pop.\n * @param {*} defaultValue The default value to return if the key does not exist.\n * @returns {*} The value of the popped key.\n * @throws {Error} If the key does not exist and no default value is provided.\n */\nexport function pop(obj, key, defaultValue = undefined) {\n    const value = obj[key];\n    if (value !== undefined) {\n        delete obj[key];\n        return value;\n    }\n    if (defaultValue === undefined) {\n        throw Error(`Key ${key} does not exist in object.`)\n    }\n    return defaultValue;\n}\n\n/**\n * Efficiently merge arrays, creating a new copy.\n * Adapted from https://stackoverflow.com/a/6768642/13989043\n * @param  {Array[]} arrs Arrays to merge.\n * @returns {Array} The merged array.\n */\nexport function mergeArrays(...arrs) {\n    return Array.prototype.concat.apply([], arrs);\n}\n\n/**\n * Compute the Cartesian product of given arrays\n * @param {...Array} a Arrays to compute the product\n * @returns {Array} Returns the computed Cartesian product as an array\n * @private\n */\nexport function product(...a) {\n    // Cartesian product of items\n    // Adapted from https://stackoverflow.com/a/43053803\n    return a.reduce((a, b) => a.flatMap(d => b.map(e => [d, e])));\n}\n\n/**\n * Calculates the index offset for a given index and window size.\n * @param {number} i The index.\n * @param {number} w The window size.\n * @returns {number} The index offset.\n */\nexport function calculateReflectOffset(i, w) {\n    return Math.abs((i + w) % (2 * w) - w);\n}\n\n// Path: src/backends/onnx.js\n/**\n * @file Handler file for choosing the correct version of ONNX Runtime, based on the environment.\n * Ideally, we could import the `onnxruntime-web` and `onnxruntime-node` packages only when needed,\n * but dynamic imports don't seem to work with the current webpack version and/or configuration.\n * This is possibly due to the experimental nature of top-level await statements.\n * So, we just import both packages, and use the appropriate one based on the environment:\n *   - When running in node, we use `onnxruntime-node`.\n *   - When running in the browser, we use `onnxruntime-web` (`onnxruntime-node` is not bundled).\n * \n * This module is not directly exported, but can be accessed through the environment variables:\n * ```javascript\n * import { env } from '@xenova/transformers';\n * console.log(env.backends.onnx);\n * ```\n * \n * @module backends/onnx\n */\n\n// NOTE: Import order matters here. We need to import `onnxruntime-node` before `onnxruntime-web`.\n// In either case, we select the default export if it exists, otherwise we use the named export.\nimport * as ONNX_NODE from 'onnxruntime-node';\nimport * as ONNX_WEB from 'onnxruntime-web';\n\n/** @type {import('onnxruntime-web')} The ONNX runtime module. */\nexport let ONNX;\n\nexport const executionProviders = [\n    // 'webgpu',\n    'wasm'\n];\n\nif (typeof process !== 'undefined' && process?.release?.name === 'node') {\n    // Running in a node-like environment.\n    ONNX = ONNX_NODE.default ?? ONNX_NODE;\n\n    // Add `cpu` execution provider, with higher precedence that `wasm`.\n    executionProviders.unshift('cpu');\n\n} else {\n    // Running in a browser-environment\n    ONNX = ONNX_WEB.default ?? ONNX_WEB;\n\n    // SIMD for WebAssembly does not operate correctly in some recent versions of iOS (16.4.x).\n    // As a temporary fix, we disable it for now.\n    // For more information, see: https://github.com/microsoft/onnxruntime/issues/15644\n    const isIOS = typeof navigator !== 'undefined' && /iP(hone|od|ad).+16_4.+AppleWebKit/.test(navigator.userAgent);\n    if (isIOS) {\n        ONNX.env.wasm.simd = false;\n    }\n}\n\n// Path: src/env.js\n/**\n * @file Module used to configure Transformers.js.\n * \n * **Example:** Disable remote models.\n * ```javascript\n * import { env } from '@xenova/transformers';\n * env.allowRemoteModels = false;\n * ```\n * \n * **Example:** Set local model path.\n * ```javascript\n * import { env } from '@xenova/transformers';\n * env.localModelPath = '/path/to/local/models/';\n * ```\n * \n * **Example:** Set cache directory.\n * ```javascript\n * import { env } from '@xenova/transformers';\n * env.cacheDir = '/path/to/cache/directory/';\n * ```\n * \n * @module env\n */\n\nimport fs from 'fs';\nimport path from 'path';\nimport url from 'url';\n\nimport { ONNX } from './backends/onnx.js';\nconst { env: onnx_env } = ONNX;\n\nconst VERSION = '2.17.0';\n\n// Check if various APIs are available (depends on environment)\nconst WEB_CACHE_AVAILABLE = typeof self !== 'undefined' && 'caches' in self;\nconst FS_AVAILABLE = !isEmpty(fs); // check if file system is available\nconst PATH_AVAILABLE = !isEmpty(path); // check if path is available\n\nconst RUNNING_LOCALLY = FS_AVAILABLE && PATH_AVAILABLE;\n\nconst __dirname = RUNNING_LOCALLY\n    ? path.dirname(path.dirname(url.fileURLToPath(import.meta.url)))\n    : './';\n\n// Only used for environments with access to file system\nconst DEFAULT_CACHE_DIR = RUNNING_LOCALLY\n    ? path.join(__dirname, '/.cache/')\n    : null;\n\n// Set local model path, based on available APIs\nconst DEFAULT_LOCAL_MODEL_PATH = '/models/';\nconst localModelPath = RUNNING_LOCALLY\n    ? path.join(__dirname, DEFAULT_LOCAL_MODEL_PATH)\n    : DEFAULT_LOCAL_MODEL_PATH;\n\nif (onnx_env?.wasm) {\n    // Set path to wasm files. This is needed when running in a web worker.\n    // https://onnxruntime.ai/docs/api/js/interfaces/Env.WebAssemblyFlags.html#wasmPaths\n    // We use remote wasm files by default to make it easier for newer users.\n    // In practice, users should probably self-host the necessary .wasm files.\n    onnx_env.wasm.wasmPaths = RUNNING_LOCALLY\n        ? path.join(__dirname, '/dist/')\n        : `https://cdn.jsdelivr.net/npm/@xenova/transformers@${VERSION}/dist/`;\n}\n\n/**\n * Global variable used to control execution. This provides users a simple way to configure Transformers.js.\n * @property {Object} backends Expose environment variables of different backends,\n * allowing users to set these variables if they want to.\n * @property {string} __dirname Directory name of module. Useful for resolving local paths.\n * @property {string} version This version of Transformers.js.\n * @property {boolean} allowRemoteModels Whether to allow loading of remote files, defaults to `true`.\n * If set to `false`, it will have the same effect as setting `local_files_only=true` when loading pipelines, models, tokenizers, processors, etc.\n * @property {string} remoteHost Host URL to load models from. Defaults to the Hugging Face Hub.\n * @property {string} remotePathTemplate Path template to fill in and append to `remoteHost` when loading models.\n * @property {boolean} allowLocalModels Whether to allow loading of local files, defaults to `true`.\n * If set to `false`, it will skip the local file check and try to load the model from the remote host.\n * @property {string} localModelPath Path to load local models from. Defaults to `/models/`.\n * @property {boolean} useFS Whether to use the file system to load files. By default, it is `true` if available.\n * @property {boolean} useBrowserCache Whether to use Cache API to cache models. By default, it is `true` if available.\n * @property {boolean} useFSCache Whether to use the file system to cache files. By default, it is `true` if available.\n * @property {string} cacheDir The directory to use for caching files with the file system. By default, it is `./.cache`.\n * @property {boolean} useCustomCache Whether to use a custom cache system (defined by `customCache`), defaults to `false`.\n * @property {Object} customCache The custom cache to use. Defaults to `null`. Note: this must be an object which\n * implements the `match` and `put` functions of the Web Cache API. For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache\n */\nexport const env = {\n    /////////////////// Backends settings ///////////////////\n    backends: {\n        // onnxruntime-web/onnxruntime-node\n        onnx: onnx_env,\n\n        // TensorFlow.js\n        tfjs: {},\n    },\n\n    __dirname,\n    version: VERSION,\n\n    /////////////////// Model settings ///////////////////\n    allowRemoteModels: true,\n    remoteHost: 'https://huggingface.co/',\n    remotePathTemplate: '{model}/resolve/{revision}/',\n\n    allowLocalModels: true,\n    localModelPath: localModelPath,\n    useFS: FS_AVAILABLE,\n\n    /////////////////// Cache settings ///////////////////\n    useBrowserCache: WEB_CACHE_AVAILABLE,\n\n    useFSCache: FS_AVAILABLE,\n    cacheDir: DEFAULT_CACHE_DIR,\n\n    useCustomCache: false,\n    customCache: null,\n    //////////////////////////////////////////////////////\n}\n\n\n/**\n * @param {Object} obj\n * @private\n */\nfunction isEmpty(obj) {\n    return Object.keys(obj).length === 0;\n}\n\n\n// Path: src/utils/hub.js\n\n/**\n * @file Utility functions to interact with the Hugging Face Hub (https://huggingface.co/models)\n * \n * @module utils/hub\n */\n\nimport fs from 'fs';\nimport path from 'path';\nimport stream from 'stream/web';\n\nimport { env } from '../env.js';\nimport { dispatchCallback } from './core.js';\n\nif (!globalThis.ReadableStream) {\n    // @ts-ignore\n    globalThis.ReadableStream = stream.ReadableStream; // ReadableStream is not a global with Node 16\n}\n\n/**\n * @typedef {Object} PretrainedOptions Options for loading a pretrained model.     \n * @property {boolean?} [quantized=true] Whether to load the 8-bit quantized version of the model (only applicable when loading model files).\n * @property {function} [progress_callback=null] If specified, this function will be called during model construction, to provide the user with progress updates.\n * @property {Object} [config=null] Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when:\n * - The model is a model provided by the library (loaded with the *model id* string of a pretrained model).\n * - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a configuration JSON file named *config.json* is found in the directory.\n * @property {string} [cache_dir=null] Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used.\n * @property {boolean} [local_files_only=false] Whether or not to only look at local files (e.g., not try downloading the model).\n * @property {string} [revision='main'] The specific model version to use. It can be a branch name, a tag name, or a commit id,\n * since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git.\n * NOTE: This setting is ignored for local requests.\n * @property {string} [model_file_name=null] If specified, load the model with this name (excluding the .onnx suffix). Currently only valid for encoder- or decoder-only models.\n */\n\nclass FileResponse {\n    /**\n     * Mapping from file extensions to MIME types.\n     */\n    _CONTENT_TYPE_MAP = {\n        'txt': 'text/plain',\n        'html': 'text/html',\n        'css': 'text/css',\n        'js': 'text/javascript',\n        'json': 'application/json',\n        'png': 'image/png',\n        'jpg': 'image/jpeg',\n        'jpeg': 'image/jpeg',\n        'gif': 'image/gif',\n    }\n    /**\n     * Creates a new `FileResponse` object.\n     * @param {string|URL} filePath\n     */\n    constructor(filePath) {\n        this.filePath = filePath;\n        this.headers = new Headers();\n\n        this.exists = fs.existsSync(filePath);\n        if (this.exists) {\n            this.status = 200;\n            this.statusText = 'OK';\n\n            let stats = fs.statSync(filePath);\n            this.headers.set('content-length', stats.size.toString());\n\n            this.updateContentType();\n\n            let self = this;\n            this.body = new ReadableStream({\n                start(controller) {\n                    self.arrayBuffer().then(buffer => {\n                        controller.enqueue(new Uint8Array(buffer));\n                        controller.close();\n                    })\n                }\n            });\n        } else {\n            this.status = 404;\n            this.statusText = 'Not Found';\n            this.body = null;\n        }\n    }\n\n    /**\n     * Updates the 'content-type' header property of the response based on the extension of\n     * the file specified by the filePath property of the current object.\n     * @returns {void}\n     */\n    updateContentType() {\n        // Set content-type header based on file extension\n        const extension = this.filePath.toString().split('.').pop().toLowerCase();\n        this.headers.set('content-type', this._CONTENT_TYPE_MAP[extension] ?? 'application/octet-stream');\n    }\n\n    /**\n     * Clone the current FileResponse object.\n     * @returns {FileResponse} A new FileResponse object with the same properties as the current object.\n     */\n    clone() {\n        let response = new FileResponse(this.filePath);\n        response.exists = this.exists;\n        response.status = this.status;\n        response.statusText = this.statusText;\n        response.headers = new Headers(this.headers);\n        return response;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with an ArrayBuffer containing the file's contents.\n     * @returns {Promise<ArrayBuffer>} A Promise that resolves with an ArrayBuffer containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async arrayBuffer() {\n        const data = await fs.promises.readFile(this.filePath);\n        return data.buffer;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a Blob containing the file's contents.\n     * @returns {Promise<Blob>} A Promise that resolves with a Blob containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async blob() {\n        const data = await fs.promises.readFile(this.filePath);\n        return new Blob([data], { type: this.headers.get('content-type') });\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a string containing the file's contents.\n     * @returns {Promise<string>} A Promise that resolves with a string containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async text() {\n        const data = await fs.promises.readFile(this.filePath, 'utf8');\n        return data;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a parsed JavaScript object containing the file's contents.\n     * \n     * @returns {Promise<Object>} A Promise that resolves with a parsed JavaScript object containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async json() {\n        return JSON.parse(await this.text());\n    }\n}\n\n/**\n * Determines whether the given string is a valid HTTP or HTTPS URL.\n * @param {string|URL} string The string to test for validity as an HTTP or HTTPS URL.\n * @param {string[]} [validHosts=null] A list of valid hostnames. If specified, the URL's hostname must be in this list.\n * @returns {boolean} True if the string is a valid HTTP or HTTPS URL, false otherwise.\n */\nfunction isValidHttpUrl(string, validHosts = null) {\n    // https://stackoverflow.com/a/43467144\n    let url;\n    try {\n        url = new URL(string);\n    } catch (_) {\n        return false;\n    }\n    if (validHosts && !validHosts.includes(url.hostname)) {\n        return false;\n    }\n    return url.protocol === \"http:\" || url.protocol === \"https:\";\n}\n\n/**\n * Helper function to get a file, using either the Fetch API or FileSystem API.\n *\n * @param {URL|string} urlOrPath The URL/path of the file to get.\n * @returns {Promise<FileResponse|Response>} A promise that resolves to a FileResponse object (if the file is retrieved using the FileSystem API), or a Response object (if the file is retrieved using the Fetch API).\n */\nexport async function getFile(urlOrPath) {\n\n    if (env.useFS && !isValidHttpUrl(urlOrPath)) {\n        return new FileResponse(urlOrPath);\n\n    } else if (typeof process !== 'undefined' && process?.release?.name === 'node') {\n        const IS_CI = !!process.env?.TESTING_REMOTELY;\n        const version = env.version;\n\n        const headers = new Headers();\n        headers.set('User-Agent', `transformers.js/${version}; is_ci/${IS_CI};`);\n\n        // Check whether we are making a request to the Hugging Face Hub.\n        const isHFURL = isValidHttpUrl(urlOrPath, ['huggingface.co', 'hf.co']);\n        if (isHFURL) {\n            // If an access token is present in the environment variables,\n            // we add it to the request headers.\n            // NOTE: We keep `HF_ACCESS_TOKEN` for backwards compatibility (as a fallback).\n            const token = process.env?.HF_TOKEN ?? process.env?.HF_ACCESS_TOKEN;\n            if (token) {\n                headers.set('Authorization', `Bearer ${token}`);\n            }\n        }\n        return fetch(urlOrPath, { headers });\n    } else {\n        // Running in a browser-environment, so we use default headers\n        // NOTE: We do not allow passing authorization headers in the browser,\n        // since this would require exposing the token to the client.\n        return fetch(urlOrPath);\n    }\n}\n\nconst ERROR_MAPPING = {\n    // 4xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#client_error_responses)\n    400: 'Bad request error occurred while trying to load file',\n    401: 'Unauthorized access to file',\n    403: 'Forbidden access to file',\n    404: 'Could not locate file',\n    408: 'Request timeout error occurred while trying to load file',\n\n    // 5xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#server_error_responses)\n    500: 'Internal server error error occurred while trying to load file',\n    502: 'Bad gateway error occurred while trying to load file',\n    503: 'Service unavailable error occurred while trying to load file',\n    504: 'Gateway timeout error occurred while trying to load file',\n}\n/**\n * Helper method to handle fatal errors that occur while trying to load a file from the Hugging Face Hub.\n * @param {number} status The HTTP status code of the error.\n * @param {string} remoteURL The URL of the file that could not be loaded.\n * @param {boolean} fatal Whether to raise an error if the file could not be loaded.\n * @returns {null} Returns `null` if `fatal = true`.\n * @throws {Error} If `fatal = false`.\n */\nfunction handleError(status, remoteURL, fatal) {\n    if (!fatal) {\n        // File was not loaded correctly, but it is optional.\n        // TODO in future, cache the response?\n        return null;\n    }\n\n    const message = ERROR_MAPPING[status] ?? `Error (${status}) occurred while trying to load file`;\n    throw Error(`${message}: \"${remoteURL}\".`);\n}\n\nclass FileCache {\n    /**\n     * Instantiate a `FileCache` object.\n     * @param {string} path \n     */\n    constructor(path) {\n        this.path = path;\n    }\n\n    /**\n     * Checks whether the given request is in the cache.\n     * @param {string} request \n     * @returns {Promise<FileResponse | undefined>}\n     */\n    async match(request) {\n\n        let filePath = path.join(this.path, request);\n        let file = new FileResponse(filePath);\n\n        if (file.exists) {\n            return file;\n        } else {\n            return undefined;\n        }\n    }\n\n    /**\n     * Adds the given response to the cache.\n     * @param {string} request \n     * @param {Response|FileResponse} response \n     * @returns {Promise<void>}\n     */\n    async put(request, response) {\n        const buffer = Buffer.from(await response.arrayBuffer());\n\n        let outputPath = path.join(this.path, request);\n\n        try {\n            await fs.promises.mkdir(path.dirname(outputPath), { recursive: true });\n            await fs.promises.writeFile(outputPath, buffer);\n\n        } catch (err) {\n            console.warn('An error occurred while writing the file to cache:', err)\n        }\n    }\n\n    // TODO add the rest?\n    // addAll(requests: RequestInfo[]): Promise<void>;\n    // delete(request: RequestInfo | URL, options?: CacheQueryOptions): Promise<boolean>;\n    // keys(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise<ReadonlyArray<Request>>;\n    // match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise<Response | undefined>;\n    // matchAll(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise<ReadonlyArray<Response>>;\n}\n\n/**\n * \n * @param {FileCache|Cache} cache The cache to search\n * @param {string[]} names The names of the item to search for\n * @returns {Promise<FileResponse|Response|undefined>} The item from the cache, or undefined if not found.\n */\nasync function tryCache(cache, ...names) {\n    for (let name of names) {\n        try {\n            let result = await cache.match(name);\n            if (result) return result;\n        } catch (e) {\n            continue;\n        }\n    }\n    return undefined;\n}\n\n/**\n * \n * Retrieves a file from either a remote URL using the Fetch API or from the local file system using the FileSystem API.\n * If the filesystem is available and `env.useCache = true`, the file will be downloaded and cached.\n * \n * @param {string} path_or_repo_id This can be either:\n...\n", "needle_token_start": 137, "needle_token_end": 164, "code_context_ntokens": 7274, "output": ["```javascript\n/**\n * Helper function to dispatch progress callbacks.\n *\n * @param {Function} progress_callback The progress callback function to dispatch.\n * @param {any} data The data to pass to the progress callback function.\n * @returns {void}\n * @private\n */\nexport function dispatchCallback(progress_callback, data) {\n    if (progress_callback) progress_callback(data);\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "window_function", "language": "typescript", "path": "src/utils/audio.js", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function generates a windowing function used in signal processing to taper a signal before applying a Fourier transform. This helps to minimize spectral leakage.\n2. **Input**: The function takes three primary inputs: the length of the window in samples, the type of window function desired (e.g., 'hann', 'boxcar'), and an options object that can specify whether the window is periodic, the length of the analysis frames, and whether to center the window in the FFT buffer.\n3. **Output**: It returns an array representing the window. The array's length matches either the specified window length or the frame length if provided and zero-padding is applied.\n4. **Procedure**: The function first determines the effective length of the window based on whether it should be periodic. It then creates the window based on the specified type. If the window is periodic, the last sample is excluded to ensure periodicity. If a frame length is provided and it is larger than the window length, the window is optionally centered and zero-padded to match the frame length.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/image.js\n\n/**\n * @file Helper module for image processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/image\n */\n\nimport { getFile } from './hub.js';\nimport { env } from '../env.js';\nimport { Tensor } from './tensor.js';\n\n// Will be empty (or not used) if running in browser or web-worker\nimport sharp from 'sharp';\n\nconst BROWSER_ENV = typeof self !== 'undefined';\nconst WEBWORKER_ENV = BROWSER_ENV && self.constructor.name === 'DedicatedWorkerGlobalScope';\n\nlet createCanvasFunction;\nlet ImageDataClass;\nlet loadImageFunction;\nif (BROWSER_ENV) {\n    // Running in browser or web-worker\n    createCanvasFunction = (/** @type {number} */ width, /** @type {number} */ height) => {\n        if (!self.OffscreenCanvas) {\n            throw new Error('OffscreenCanvas not supported by this browser.');\n        }\n        return new self.OffscreenCanvas(width, height)\n    };\n    loadImageFunction = self.createImageBitmap;\n    ImageDataClass = self.ImageData;\n\n} else if (sharp) {\n    // Running in Node.js, electron, or other non-browser environment\n\n    loadImageFunction = async (/**@type {sharp.Sharp}*/img) => {\n        const metadata = await img.metadata();\n        const rawChannels = metadata.channels;\n\n        let { data, info } = await img.raw().toBuffer({ resolveWithObject: true });\n\n        const newImage = new RawImage(new Uint8ClampedArray(data), info.width, info.height, info.channels);\n        if (rawChannels !== undefined && rawChannels !== info.channels) {\n            // Make sure the new image has the same number of channels as the input image.\n            // This is necessary for grayscale images.\n            newImage.convert(rawChannels);\n        }\n        return newImage;\n    }\n\n} else {\n    throw new Error('Unable to load image processing library.');\n}\n\n\n// Defined here: https://github.com/python-pillow/Pillow/blob/a405e8406b83f8bfb8916e93971edc7407b8b1ff/src/libImaging/Imaging.h#L262-L268\nconst RESAMPLING_MAPPING = {\n    0: 'nearest',\n    1: 'lanczos',\n    2: 'bilinear',\n    3: 'bicubic',\n    4: 'box',\n    5: 'hamming',\n}\n\n/**\n * Mapping from file extensions to MIME types.\n */\nconst CONTENT_TYPE_MAP = new Map([\n    ['png', 'image/png'],\n    ['jpg', 'image/jpeg'],\n    ['jpeg', 'image/jpeg'],\n    ['gif', 'image/gif'],\n]);\n\nexport class RawImage {\n\n    /**\n     * Create a new `RawImage` object.\n     * @param {Uint8ClampedArray|Uint8Array} data The pixel data.\n     * @param {number} width The width of the image.\n     * @param {number} height The height of the image.\n     * @param {1|2|3|4} channels The number of channels.\n     */\n    constructor(data, width, height, channels) {\n        this.data = data;\n        this.width = width;\n        this.height = height;\n        this.channels = channels;\n    }\n\n    /** \n     * Returns the size of the image (width, height).\n     * @returns {[number, number]} The size of the image (width, height).\n     */\n    get size() {\n        return [this.width, this.height];\n    }\n\n    /**\n     * Helper method for reading an image from a variety of input types.\n     * @param {RawImage|string|URL} input \n     * @returns The image object.\n     * \n     * **Example:** Read image from a URL.\n     * ```javascript\n     * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg');\n     * // RawImage {\n     * //   \"data\": Uint8ClampedArray [ 25, 25, 25, 19, 19, 19, ... ],\n...\n// Path: src/utils/audio.js\n/**\n * @file Helper module for audio processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/audio\n */\n\nimport {\n    getFile,\n} from './hub.js';\nimport { FFT, max } from './maths.js';\nimport {\n    calculateReflectOffset,\n} from './core.js';\n\n\n/**\n * Helper function to read audio from a path/URL.\n * @param {string|URL} url The path/URL to load the audio from.\n * @param {number} sampling_rate The sampling rate to use when decoding the audio.\n * @returns {Promise<Float32Array>} The decoded audio as a `Float32Array`.\n */\nexport async function read_audio(url, sampling_rate) {\n    if (typeof AudioContext === 'undefined') {\n        // Running in node or an environment without AudioContext\n        throw Error(\n            \"Unable to load audio from path/URL since `AudioContext` is not available in your environment. \" +\n            \"Instead, audio data should be passed directly to the pipeline/processor. \" +\n            \"For more information and some example code, see https://huggingface.co/docs/transformers.js/guides/node-audio-processing.\"\n        )\n    }\n\n    const response = await (await getFile(url)).arrayBuffer();\n    const audioCTX = new AudioContext({ sampleRate: sampling_rate });\n    if (typeof sampling_rate === 'undefined') {\n        console.warn(`No sampling rate provided, using default of ${audioCTX.sampleRate}Hz.`)\n    }\n    const decoded = await audioCTX.decodeAudioData(response);\n\n    /** @type {Float32Array} */\n    let audio;\n\n    // We now replicate HuggingFace's `ffmpeg_read` method:\n    if (decoded.numberOfChannels === 2) {\n        // When downmixing a stereo audio file to mono using the -ac 1 option in FFmpeg,\n        // the audio signal is summed across both channels to create a single mono channel.\n        // However, if the audio is at full scale (i.e. the highest possible volume level),\n        // the summing of the two channels can cause the audio signal to clip or distort.\n\n        // To prevent this clipping, FFmpeg applies a scaling factor of 1/sqrt(2) (~ 0.707)\n        // to the audio signal before summing the two channels. This scaling factor ensures\n        // that the combined audio signal will not exceed the maximum possible level, even\n        // if both channels are at full scale.\n\n        // After applying this scaling factor, the audio signal from both channels is summed\n        // to create a single mono channel. It's worth noting that this scaling factor is\n        // only applied when downmixing stereo audio to mono using the -ac 1 option in FFmpeg.\n        // If you're using a different downmixing method, or if you're not downmixing the\n        // audio at all, this scaling factor may not be needed.\n        const SCALING_FACTOR = Math.sqrt(2);\n\n        const left = decoded.getChannelData(0);\n        const right = decoded.getChannelData(1);\n\n        audio = new Float32Array(left.length);\n        for (let i = 0; i < decoded.length; ++i) {\n            audio[i] = SCALING_FACTOR * (left[i] + right[i]) / 2;\n        }\n\n    } else {\n        // If the audio is not stereo, we can just use the first channel:\n        audio = decoded.getChannelData(0);\n    }\n\n    return audio;\n}\n\n/**\n * Generates a Hanning window of length M.\n *\n * @param {number} M The length of the Hanning window to generate.\n * @returns {Float64Array} The generated Hanning window.\n */\nexport function hanning(M) {\n    if (M < 1) {\n        return new Float64Array();\n    }\n    if (M === 1) {\n        return new Float64Array([1]);\n    }\n    const denom = M - 1;\n    const factor = Math.PI / denom;\n    const cos_vals = new Float64Array(M);\n    for (let i = 0; i < M; ++i) {\n        const n = 2 * i - denom;\n        cos_vals[i] = 0.5 + 0.5 * Math.cos(factor * n);\n    }\n    return cos_vals;\n}\n\nconst HERTZ_TO_MEL_MAPPING = {\n    \"htk\": (/** @type {number} */ freq) => 2595.0 * Math.log10(1.0 + (freq / 700.0)),\n    \"kaldi\": (/** @type {number} */ freq) => 1127.0 * Math.log(1.0 + (freq / 700.0)),\n    \"slaney\": (/** @type {number} */ freq, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = 27.0 / Math.log(6.4)) =>\n        freq >= min_log_hertz\n            ? min_log_mel + Math.log(freq / min_log_hertz) * logstep\n            : 3.0 * freq / 200.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} freq \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction hertz_to_mel(freq, mel_scale = \"htk\") {\n    const fn = HERTZ_TO_MEL_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof freq === 'number' ? fn(freq) : freq.map(x => fn(x));\n}\n\nconst MEL_TO_HERTZ_MAPPING = {\n    \"htk\": (/** @type {number} */ mels) => 700.0 * (10.0 ** (mels / 2595.0) - 1.0),\n    \"kaldi\": (/** @type {number} */ mels) => 700.0 * (Math.exp(mels / 1127.0) - 1.0),\n    \"slaney\": (/** @type {number} */ mels, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = Math.log(6.4) / 27.0) => mels >= min_log_mel\n        ? min_log_hertz * Math.exp(logstep * (mels - min_log_mel))\n        : 200.0 * mels / 3.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} mels \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction mel_to_hertz(mels, mel_scale = \"htk\") {\n    const fn = MEL_TO_HERTZ_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof mels === 'number' ? fn(mels) : mels.map(x => fn(x));\n}\n\n/**\n* Creates a triangular filter bank.\n*\n* Adapted from torchaudio and librosa.\n*\n* @param {Float64Array} fft_freqs Discrete frequencies of the FFT bins in Hz, of shape `(num_frequency_bins,)`.\n* @param {Float64Array} filter_freqs Center frequencies of the triangular filters to create, in Hz, of shape `(num_mel_filters,)`.\n* @returns {number[][]} of shape `(num_frequency_bins, num_mel_filters)`.\n*/\nfunction _create_triangular_filter_bank(fft_freqs, filter_freqs) {\n    const filter_diff = Float64Array.from(\n        { length: filter_freqs.length - 1 },\n        (_, i) => filter_freqs[i + 1] - filter_freqs[i]\n    );\n\n    const slopes = Array.from({\n        length: fft_freqs.length\n    }, () => new Array(filter_freqs.length));\n\n    for (let j = 0; j < fft_freqs.length; ++j) {\n        const slope = slopes[j];\n        for (let i = 0; i < filter_freqs.length; ++i) {\n            slope[i] = filter_freqs[i] - fft_freqs[j];\n        }\n    }\n\n    const numFreqs = filter_freqs.length - 2;\n    const ret = Array.from({ length: numFreqs }, () => new Array(fft_freqs.length));\n\n    for (let j = 0; j < fft_freqs.length; ++j) { // 201\n        const slope = slopes[j];\n        for (let i = 0; i < numFreqs; ++i) { // 80\n            const down = -slope[i] / filter_diff[i];\n            const up = slope[i + 2] / filter_diff[i + 1];\n            ret[i][j] = Math.max(0, Math.min(down, up));\n        }\n    }\n    return ret;\n}\n\n/**\n * Return evenly spaced numbers over a specified interval.\n * @param {number} start The starting value of the sequence.\n * @param {number} end The end value of the sequence.\n * @param {number} num Number of samples to generate.\n * @returns `num` evenly spaced samples, calculated over the interval `[start, stop]`.\n */\nfunction linspace(start, end, num) {\n    const step = (end - start) / (num - 1);\n    return Float64Array.from({ length: num }, (_, i) => start + step * i);\n}\n\n/**\n * Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and\n * various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters\n * are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these\n * features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency.\n * @param {number} num_frequency_bins Number of frequencies used to compute the spectrogram (should be the same as in `stft`).\n * @param {number} num_mel_filters Number of mel filters to generate.\n * @param {number} min_frequency Lowest frequency of interest in Hz.\n * @param {number} max_frequency Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`.\n * @param {number} sampling_rate Sample rate of the audio waveform.\n * @param {string} [norm] If `\"slaney\"`, divide the triangular mel weights by the width of the mel band (area normalization).\n * @param {string} [mel_scale] The mel frequency scale to use, `\"htk\"` or `\"slaney\"`.\n * @param {boolean} [triangularize_in_mel_space] If this option is enabled, the triangular filter is applied in mel space rather than frequency space.\n * This should be set to `true` in order to get the same results as `torchaudio` when computing mel filters.\n * @returns {number[][]} Triangular filter bank matrix, which is a 2D array of shape (`num_frequency_bins`, `num_mel_filters`).\n * This is a projection matrix to go from a spectrogram to a mel spectrogram.\n */\nexport function mel_filter_bank(\n    num_frequency_bins,\n    num_mel_filters,\n    min_frequency,\n    max_frequency,\n    sampling_rate,\n    norm = null,\n    mel_scale = \"htk\",\n    triangularize_in_mel_space = false,\n) {\n    if (norm !== null && norm !== \"slaney\") {\n        throw new Error('norm must be one of null or \"slaney\"');\n    }\n\n    const mel_min = hertz_to_mel(min_frequency, mel_scale);\n    const mel_max = hertz_to_mel(max_frequency, mel_scale);\n    const mel_freqs = linspace(mel_min, mel_max, num_mel_filters + 2);\n\n    let filter_freqs = mel_to_hertz(mel_freqs, mel_scale);\n    let fft_freqs; // frequencies of FFT bins in Hz\n\n    if (triangularize_in_mel_space) {\n        const fft_bin_width = sampling_rate / (num_frequency_bins * 2);\n        fft_freqs = hertz_to_mel(Float64Array.from({ length: num_frequency_bins }, (_, i) => i * fft_bin_width), mel_scale);\n        filter_freqs = mel_freqs;\n    } else {\n        fft_freqs = linspace(0, Math.floor(sampling_rate / 2), num_frequency_bins);\n    }\n\n    const mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs);\n\n    if (norm !== null && norm === \"slaney\") {\n        // Slaney-style mel is scaled to be approx constant energy per channel\n        for (let i = 0; i < num_mel_filters; ++i) {\n            const filter = mel_filters[i];\n            const enorm = 2.0 / (filter_freqs[i + 2] - filter_freqs[i]);\n            for (let j = 0; j < num_frequency_bins; ++j) {\n                // Apply this enorm to all frequency bins\n                filter[j] *= enorm;\n            }\n        }\n    }\n\n    // TODO warn if there is a zero row\n\n    return mel_filters;\n\n}\n\n/**\n * @template {Float32Array|Float64Array} T\n * Pads an array with a reflected version of itself on both ends.\n * @param {T} array The array to pad.\n * @param {number} left The amount of padding to add to the left.\n * @param {number} right The amount of padding to add to the right.\n * @returns {T} The padded array.\n */\nfunction padReflect(array, left, right) {\n    // @ts-ignore\n    const padded = new array.constructor(array.length + left + right);\n    const w = array.length - 1;\n\n    for (let i = 0; i < array.length; ++i) {\n        padded[left + i] = array[i];\n    }\n\n    for (let i = 1; i <= left; ++i) {\n        padded[left - i] = array[calculateReflectOffset(i, w)];\n    }\n\n    for (let i = 1; i <= right; ++i) {\n        padded[w + left + i] = array[calculateReflectOffset(w - i, w)];\n    }\n\n    return padded;\n}\n\n/**\n * Helper function to compute `amplitude_to_db` and `power_to_db`.\n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram \n * @param {number} factor \n * @param {number} reference \n * @param {number} min_value \n * @param {number} db_range \n * @returns {T}\n */\nfunction _db_conversion_helper(spectrogram, factor, reference, min_value, db_range) {\n    if (reference <= 0) {\n        throw new Error('reference must be greater than zero');\n    }\n\n    if (min_value <= 0) {\n        throw new Error('min_value must be greater than zero');\n    }\n\n    reference = Math.max(min_value, reference);\n\n    const logReference = Math.log10(reference);\n    for (let i = 0; i < spectrogram.length; ++i) {\n        spectrogram[i] = factor * Math.log10(Math.max(min_value, spectrogram[i]) - logReference)\n    }\n\n    if (db_range !== null) {\n        if (db_range <= 0) {\n            throw new Error('db_range must be greater than zero');\n        }\n        const maxValue = max(spectrogram)[0] - db_range;\n        for (let i = 0; i < spectrogram.length; ++i) {\n            spectrogram[i] = Math.max(spectrogram[i], maxValue);\n        }\n    }\n\n    return spectrogram;\n}\n\n/**\n * Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`,\n * using basic logarithm properties for numerical stability. NOTE: Operates in-place.\n * \n * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a\n * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.\n * This means that large variations in energy may not sound all that different if the sound is loud to begin with.\n * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.\n * \n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram The input amplitude (mel) spectrogram.\n * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB.\n * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [min_value=1e-5] The spectrogram will be clipped to this minimum value before conversion to decibels,\n * to avoid taking `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero.\n * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the\n * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @returns {T} The modified spectrogram in decibels.\n */\nfunction amplitude_to_db(spectrogram, reference = 1.0, min_value = 1e-5, db_range = null) {\n    return _db_conversion_helper(spectrogram, 20.0, reference, min_value, db_range);\n}\n\n/**\n * Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`,\n * using basic logarithm properties for numerical stability. NOTE: Operates in-place.\n * \n * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a\n * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.\n * This means that large variations in energy may not sound all that different if the sound is loud to begin with.\n * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.\n * \n * Based on the implementation of `librosa.power_to_db`.\n * \n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared!\n * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB.\n * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels,\n * to avoid taking `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero.\n * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the\n * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @returns {T} The modified spectrogram in decibels.\n */\nfunction power_to_db(spectrogram, reference = 1.0, min_value = 1e-10, db_range = null) {\n    return _db_conversion_helper(spectrogram, 10.0, reference, min_value, db_range);\n}\n\n/**\n * Calculates a spectrogram over one waveform using the Short-Time Fourier Transform.\n * \n * This function can create the following kinds of spectrograms:\n *   - amplitude spectrogram (`power = 1.0`)\n *   - power spectrogram (`power = 2.0`)\n *   - complex-valued spectrogram (`power = None`)\n *   - log spectrogram (use `log_mel` argument)\n *   - mel spectrogram (provide `mel_filters`)\n *   - log-mel spectrogram (provide `mel_filters` and `log_mel`)\n *\n * In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame.\n * A padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, \n * typically the next power of two.\n * \n * @param {Float32Array|Float64Array} waveform The input waveform of shape `(length,)`. This must be a single real-valued, mono waveform.\n * @param {Float32Array|Float64Array} window The windowing function to apply of shape `(frame_length,)`, including zero-padding if necessary. The actual window length may be\n * shorter than `frame_length`, but we're assuming the array has already been zero-padded.\n * @param {number} frame_length The length of the analysis frames in samples (a.k.a., `fft_length`).\n * @param {number} hop_length The stride between successive analysis frames in samples.\n * @param {Object} options\n * @param {number} [options.fft_length=null] The size of the FFT buffer in samples. This determines how many frequency bins the spectrogram will have.\n * For optimal speed, this should be a power of two. If `null`, uses `frame_length`.\n * @param {number} [options.power=1.0] If 1.0, returns the amplitude spectrogram. If 2.0, returns the power spectrogram. If `null`, returns complex numbers.\n * @param {boolean} [options.center=true] Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `false`, frame\n * `t` will start at time `t * hop_length`.\n * @param {string} [options.pad_mode=\"reflect\"] Padding mode used when `center` is `true`. Possible values are: `\"constant\"` (pad with zeros),\n * `\"edge\"` (pad with edge values), `\"reflect\"` (pads with mirrored values).\n * @param {boolean} [options.onesided=true] If `true`, only computes the positive frequencies and returns a spectrogram containing `fft_length // 2 + 1`\n * frequency bins. If `false`, also computes the negative frequencies and returns `fft_length` frequency bins.\n * @param {number} [options.preemphasis=null] Coefficient for a low-pass filter that applies pre-emphasis before the DFT.\n * @param {number[][]} [options.mel_filters=null] The mel filter bank of shape `(num_freq_bins, num_mel_filters)`.\n * If supplied, applies this filter bank to create a mel spectrogram.\n * @param {number} [options.mel_floor=1e-10] Minimum value of mel frequency banks.\n * @param {string} [options.log_mel=null] How to convert the spectrogram to log scale. Possible options are:\n * `null` (don't convert), `\"log\"` (take the natural logarithm) `\"log10\"` (take the base-10 logarithm), `\"dB\"` (convert to decibels).\n * Can only be used when `power` is not `null`.\n * @param {number} [options.reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. For example, use `max(spectrogram)[0]` to set\n * the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [options.min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking `log(0)`.\n * For a power spectrogram, the default of `1e-10` corresponds to a minimum of -100 dB. For an amplitude spectrogram, the value `1e-5` corresponds to -100 dB.\n * Must be greater than zero.\n * @param {number} [options.db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the\n * peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @param {boolean} [options.remove_dc_offset=null] Subtract mean from waveform on each frame, applied before pre-emphasis. This should be set to `true` in\n * order to get the same results as `torchaudio.compliance.kaldi.fbank` when computing mel filters.\n * @param {number} [options.max_num_frames=null] If provided, limits the number of frames to compute to this value.\n * @param {boolean} [options.do_pad=true] If `true`, pads the output spectrogram to have `max_num_frames` frames.\n * @param {boolean} [options.transpose=false] If `true`, the returned spectrogram will have shape `(num_frames, num_frequency_bins/num_mel_filters)`. If `false`, the returned spectrogram will have shape `(num_frequency_bins/num_mel_filters, num_frames)`.\n * @returns {{data: Float32Array, dims: number[]}} Spectrogram of shape `(num_frequency_bins, length)` (regular spectrogram) or shape `(num_mel_filters, length)` (mel spectrogram).\n */\nexport function spectrogram(\n    waveform,\n    window,\n    frame_length,\n    hop_length,\n    {\n        fft_length = null,\n        power = 1.0,\n        center = true,\n        pad_mode = \"reflect\",\n        onesided = true,\n        preemphasis = null,\n        mel_filters = null,\n        mel_floor = 1e-10,\n        log_mel = null,\n        reference = 1.0,\n        min_value = 1e-10,\n        db_range = null,\n        remove_dc_offset = null,\n\n        // Custom parameters for efficiency reasons\n        max_num_frames = null,\n        do_pad = true,\n        transpose = false,\n    } = {}\n) {\n    const window_length = window.length;\n    if (fft_length === null) {\n        fft_length = frame_length;\n    }\n    if (frame_length > fft_length) {\n        throw Error(`frame_length (${frame_length}) may not be larger than fft_length (${fft_length})`)\n    }\n\n    if (window_length !== frame_length) {\n        throw new Error(`Length of the window (${window_length}) must equal frame_length (${frame_length})`);\n    }\n\n    if (hop_length <= 0) {\n        throw new Error(\"hop_length must be greater than zero\");\n    }\n\n    if (center) {\n        if (pad_mode !== 'reflect') {\n            throw new Error(`pad_mode=\"${pad_mode}\" not implemented yet.`)\n        }\n        const half_window = Math.floor((fft_length - 1) / 2) + 1;\n        waveform = padReflect(waveform, half_window, half_window);\n    }\n\n    // split waveform into frames of frame_length size\n    const num_frames = Math.floor(1 + Math.floor((waveform.length - frame_length) / hop_length))\n\n    const num_frequency_bins = onesided ? Math.floor(fft_length / 2) + 1 : fft_length\n\n    let d1 = num_frames;\n    let d1Max = num_frames;\n\n    // If maximum number of frames is provided, we must either pad or truncate\n    if (max_num_frames !== null) {\n        if (max_num_frames > num_frames) { // input is too short, so we pad\n            if (do_pad) {\n                d1Max = max_num_frames;\n            }\n        } else { // input is too long, so we truncate\n            d1Max = d1 = max_num_frames;\n        }\n    }\n\n    // Preallocate arrays to store output.\n    const fft = new FFT(fft_length);\n    const inputBuffer = new Float64Array(fft_length);\n    const outputBuffer = new Float64Array(fft.outputBufferSize);\n    const magnitudes = new Array(d1);\n\n    for (let i = 0; i < d1; ++i) {\n        // Populate buffer with waveform data\n        const offset = i * hop_length;\n        for (let j = 0; j < frame_length; ++j) {\n            inputBuffer[j] = waveform[offset + j];\n        }\n\n        if (remove_dc_offset) {\n            let sum = 0;\n            for (let j = 0; j < frame_length; ++j) {\n                sum += inputBuffer[j];\n            }\n            const mean = sum / frame_length;\n            for (let j = 0; j < frame_length; ++j) {\n                inputBuffer[j] -= mean;\n            }\n        }\n\n        if (preemphasis !== null) {\n            // Done in reverse to avoid copies and distructive modification\n            for (let j = frame_length - 1; j >= 1; --j) {\n                inputBuffer[j] -= preemphasis * inputBuffer[j - 1];\n            }\n            inputBuffer[0] *= 1 - preemphasis;\n        }\n\n        for (let j = 0; j < window.length; ++j) {\n            inputBuffer[j] *= window[j];\n        }\n\n        fft.realTransform(outputBuffer, inputBuffer);\n\n        // compute magnitudes\n        const row = new Array(num_frequency_bins);\n        for (let j = 0; j < row.length; ++j) {\n            const j2 = j << 1;\n            row[j] = outputBuffer[j2] ** 2 + outputBuffer[j2 + 1] ** 2;\n        }\n        magnitudes[i] = row;\n    }\n\n    // TODO what should happen if power is None?\n    // https://github.com/huggingface/transformers/issues/27772\n    if (power !== null && power !== 2) {\n        // slight optimization to not sqrt\n        const pow = 2 / power; // we use 2 since we already squared\n        for (let i = 0; i < magnitudes.length; ++i) {\n            const magnitude = magnitudes[i];\n            for (let j = 0; j < magnitude.length; ++j) {\n                magnitude[j] **= pow;\n            }\n        }\n    }\n\n    // TODO: What if `mel_filters` is null?\n    const num_mel_filters = mel_filters.length;\n\n    // Only here do we create Float32Array\n    const mel_spec = new Float32Array(num_mel_filters * d1Max);\n\n    // Perform matrix muliplication:\n    // mel_spec = mel_filters @ magnitudes.T\n    //  - mel_filters.shape=(80, 201)\n    //  - magnitudes.shape=(3000, 201) => - magnitudes.T.shape=(201, 3000)\n    //  - mel_spec.shape=(80, 3000)\n    const dims = transpose ? [d1Max, num_mel_filters] : [num_mel_filters, d1Max];\n    for (let i = 0; i < num_mel_filters; ++i) { // num melfilters (e.g., 80)\n        const filter = mel_filters[i];\n        for (let j = 0; j < d1; ++j) { // num frames (e.g., 3000)\n            const magnitude = magnitudes[j];\n\n            let sum = 0;\n            for (let k = 0; k < num_frequency_bins; ++k) { // num frequency bins (e.g., 201)\n                sum += filter[k] * magnitude[k];\n            }\n\n            mel_spec[\n                transpose\n                    ? j * num_mel_filters + i\n                    : i * d1 + j\n            ] = Math.max(mel_floor, sum);\n        }\n    }\n\n    if (power !== null && log_mel !== null) {\n        const o = Math.min(mel_spec.length, d1 * num_mel_filters);\n        switch (log_mel) {\n            case 'log':\n                for (let i = 0; i < o; ++i) {\n                    mel_spec[i] = Math.log(mel_spec[i]);\n                }\n                break;\n            case 'log10':\n                for (let i = 0; i < o; ++i) {\n                    mel_spec[i] = Math.log10(mel_spec[i]);\n                }\n                break;\n            case 'dB':\n                if (power === 1.0) {\n                    // NOTE: operates in-place\n                    amplitude_to_db(mel_spec, reference, min_value, db_range);\n                } else if (power === 2.0) {\n                    power_to_db(mel_spec, reference, min_value, db_range);\n                } else {\n                    throw new Error(`Cannot use log_mel option '${log_mel}' with power ${power}`)\n                }\n                break;\n            default:\n                throw new Error(`log_mel must be one of null, 'log', 'log10' or 'dB'. Got '${log_mel}'`);\n        }\n    }\n\n    return { data: mel_spec, dims };\n}\n\n/**\n * Returns an array containing the specified window.\n * @param {number} window_length The length of the window in samples.\n * @param {string} name The name of the window function.\n * @param {Object} options Additional options.\n * @param {boolean} [options.periodic=true] Whether the window is periodic or symmetric.\n * @param {number} [options.frame_length=null] The length of the analysis frames in samples.\n * Provide a value for `frame_length` if the window is smaller than the frame length, so that it will be zero-padded.\n * @param {boolean} [options.center=true] Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided.\n * @returns {Float64Array} The window of shape `(window_length,)` or `(frame_length,)`.\n */\nexport \nfunction window_function(window_length, name, {\n    periodic = true,\n    frame_length = null,\n    center = true,\n} = {}) {\n    const length = periodic ? window_length + 1 : window_length;\n    let window;\n    switch (name) {\n        case 'boxcar':\n            window = new Float64Array(length).fill(1.0);\n            break;\n        case 'hann':\n        case 'hann_window':\n            window = hanning(length);\n            break;\n        case 'povey':\n            window = hanning(length).map(x => Math.pow(x, 0.85));\n            break;\n        default:\n            throw new Error(`Unknown window type ${name}.`);\n    }\n    if (periodic) {\n        window = window.subarray(0, window_length);\n    }\n    if (frame_length === null) {\n        return window;\n    }\n    if (window_length > frame_length) {\n        throw new Error(`Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`);\n    }\n\n    return window;\n}\n\n// Path: src/processors.js\n\n/**\n * @file Processors are used to prepare non-textual inputs (e.g., image or audio) for a model.\n * \n * **Example:** Using a `WhisperProcessor` to prepare an audio input for a model.\n * ```javascript\n * import { AutoProcessor, read_audio } from '@xenova/transformers';\n *\n * let processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en');\n * let audio = await read_audio('https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac', 16000);\n * let { input_features } = await processor(audio);\n * // Tensor {\n * //   data: Float32Array(240000) [0.4752984642982483, 0.5597258806228638, 0.56434166431427, ...],\n * //   dims: [1, 80, 3000],\n * //   type: 'float32',\n * //   size: 240000,\n * // }\n * ```\n * \n * @module processors\n */\nimport {\n    Callable,\n    calculateDimensions,\n    calculateReflectOffset,\n} from './utils/core.js';\n\nimport {\n    getModelJSON,\n} from './utils/hub.js';\n\nimport {\n    min,\n    max,\n    softmax,\n    bankers_round,\n} from './utils/maths.js';\n\n\nimport { Tensor, permute, cat, interpolate, stack } from './utils/tensor.js';\n\nimport { RawImage } from './utils/image.js';\nimport {\n    window_function,\n    spectrogram,\n    mel_filter_bank,\n} from './utils/audio.js';\n\n\n// Helper functions\n\n/**\n * Converts bounding boxes from center format to corners format.\n * \n * @param {number[]} arr The coordinate for the center of the box and its width, height dimensions (center_x, center_y, width, height)\n * @returns {number[]} The coodinates for the top-left and bottom-right corners of the box (top_left_x, top_left_y, bottom_right_x, bottom_right_y)\n */\nfunction center_to_corners_format([centerX, centerY, width, height]) {\n    return [\n        centerX - width / 2,\n        centerY - height / 2,\n        centerX + width / 2,\n        centerY + height / 2\n    ];\n}\n\n/**\n * Post-processes the outputs of the model (for object detection).\n * @param {Object} outputs The outputs of the model that must be post-processed\n * @param {Tensor} outputs.logits The logits\n * @param {Tensor} outputs.pred_boxes The predicted boxes.\n * @param {number} [threshold=0.5] The threshold to use for the scores.\n * @param {number[][]} [target_sizes=null] The sizes of the original images.\n * @param {boolean} [is_zero_shot=false] Whether zero-shot object detection was performed.\n * @return {Object[]} An array of objects containing the post-processed outputs.\n * @private\n */\nfunction post_process_object_detection(outputs, threshold = 0.5, target_sizes = null, is_zero_shot = false) {\n    const out_logits = outputs.logits;\n    const out_bbox = outputs.pred_boxes;\n    const [batch_size, num_boxes, num_classes] = out_logits.dims;\n\n    if (target_sizes !== null && target_sizes.length !== batch_size) {\n        throw Error(\"Make sure that you pass in as many target sizes as the batch dimension of the logits\")\n    }\n    let toReturn = [];\n    for (let i = 0; i < batch_size; ++i) {\n        let target_size = target_sizes !== null ? target_sizes[i] : null;\n        let info = {\n            boxes: [],\n            classes: [],\n            scores: []\n        }\n        let logits = out_logits[i];\n        let bbox = out_bbox[i];\n\n        for (let j = 0; j < num_boxes; ++j) {\n            let logit = logits[j];\n\n            let indices = [];\n            let probs;\n            if (is_zero_shot) {\n                // Get indices of classes with high enough probability\n                probs = logit.sigmoid().data;\n                for (let k = 0; k < probs.length; ++k) {\n                    if (probs[k] > threshold) {\n                        indices.push(k);\n                    }\n                }\n\n            } else {\n                // Get most probable class\n                let maxIndex = max(logit.data)[1];\n\n                if (maxIndex === num_classes - 1) {\n                    // This is the background class, skip it\n                    continue;\n                }\n                indices.push(maxIndex);\n\n                // Compute softmax over classes\n                probs = softmax(logit.data);\n            }\n\n            for (const index of indices) {\n\n                // Some class has a high enough probability\n                /** @type {number[]} */\n                let box = bbox[j].data;\n\n                // convert to [x0, y0, x1, y1] format\n                box = center_to_corners_format(box)\n                if (target_size !== null) {\n                    box = box.map((x, i) => x * target_size[(i + 1) % 2])\n                }\n\n                info.boxes.push(box);\n                info.classes.push(index);\n                info.scores.push(probs[index]);\n            }\n        }\n        toReturn.push(info);\n    }\n    return toReturn;\n}\n\n/**\n * Named tuple to indicate the order we are using is (height x width), even though\n * the Graphics\u2019 industry standard is (width x height).\n * @typedef {[height: number, width: number]} HeightWidth\n */\n\n/**\n * Helper function to validate audio inputs.\n * @param {any} audio The audio data.\n * @param {string} feature_extractor The name of the feature extractor.\n * @private\n */\nfunction validate_audio_inputs(audio, feature_extractor) {\n    if (!(audio instanceof Float32Array || audio instanceof Float64Array)) {\n        throw new Error(\n            `${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. ` +\n            `If using the feature extractor directly, remember to use \\`read_audio(url, sampling_rate)\\` to obtain the raw audio data of the file/url.`\n        )\n    }\n}\n\n/**\n * Helper function to constrain a value to be a multiple of a number.\n * @param {number} val The value to constrain.\n * @param {number} multiple The number to constrain to.\n * @param {number} [minVal=0] The minimum value to constrain to.\n * @param {number} [maxVal=null] The maximum value to constrain to.\n * @returns {number} The constrained value.\n * @private\n */\nfunction constraint_to_multiple_of(val, multiple, minVal = 0, maxVal = null) {\n    const a = val / multiple;\n    let x = bankers_round(a) * multiple;\n\n    if (maxVal !== null && x > maxVal) {\n        x = Math.floor(a) * multiple;\n    }\n\n    if (x < minVal) {\n        x = Math.ceil(a) * multiple;\n    }\n\n    return x;\n}\n\n/**\n * Rounds the height and width down to the closest multiple of size_divisibility\n * @param {[number, number]} size The size of the image\n * @param {number} divisor The divisor to use.\n * @returns {[number, number]} The rounded size.\n */\nfunction enforce_size_divisibility([width, height], divisor) {\n    return [\n        Math.max(Math.floor(width / divisor), 1) * divisor,\n        Math.max(Math.floor(height / divisor), 1) * divisor\n    ];\n}\n\n\n/**\n * Base class for feature extractors.\n *\n * @extends Callable\n */\nexport class FeatureExtractor extends Callable {\n    /**\n     * Constructs a new FeatureExtractor instance.\n     *\n     * @param {Object} config The configuration for the feature extractor.\n     */\n    constructor(config) {\n        super();\n        this.config = config\n    }\n}\n\n/**\n * @typedef {object} ImageFeatureExtractorResult\n * @property {Tensor} pixel_values The pixel values of the batched preprocessed images.\n * @property {HeightWidth[]} original_sizes Array of two-dimensional tuples like [[480, 640]].\n * @property {HeightWidth[]} reshaped_input_sizes Array of two-dimensional tuples like [[1000, 1330]].\n */\n\n/**\n * Feature extractor for image models.\n *\n * @extends FeatureExtractor\n */\nexport class ImageFeatureExtractor extends FeatureExtractor {\n\n    /**\n     * Constructs a new ImageFeatureExtractor instance.\n     *\n     * @param {Object} config The configuration for the feature extractor.\n     * @param {number[]} config.image_mean The mean values for image normalization.\n     * @param {number[]} config.image_std The standard deviation values for image normalization.\n     * @param {boolean} config.do_rescale Whether to rescale the image pixel values to the [0,1] range.\n     * @param {number} config.rescale_factor The factor to use for rescaling the image pixel values.\n     * @param {boolean} config.do_normalize Whether to normalize the image pixel values.\n     * @param {boolean} config.do_resize Whether to resize the image.\n     * @param {number} config.resample What method to use for resampling.\n     * @param {number|Object} config.size The size to resize the image to.\n     */\n    constructor(config) {\n        super(config);\n\n        this.image_mean = this.config.image_mean ?? this.config.mean;\n        this.image_std = this.config.image_std ?? this.config.std;\n\n        this.resample = this.config.resample ?? 2; // 2 => bilinear\n        this.do_rescale = this.config.do_rescale ?? true;\n        this.rescale_factor = this.config.rescale_factor ?? (1 / 255);\n        this.do_normalize = this.config.do_normalize;\n\n        this.do_resize = this.config.do_resize;\n        this.do_thumbnail = this.config.do_thumbnail;\n        this.size = this.config.size;\n        this.size_divisibility = this.config.size_divisibility ?? this.config.size_divisor;\n\n        this.do_center_crop = this.config.do_center_crop;\n        this.crop_size = this.config.crop_size;\n        this.do_convert_rgb = this.config.do_convert_rgb ?? true;\n        this.do_crop_margin = this.config.do_crop_margin;\n\n        this.pad_size = this.config.pad_size;\n        this.do_pad = this.config.do_pad;\n\n        if (this.do_pad && !this.pad_size && this.size && this.size.width !== undefined && this.size.height !== undefined) {\n            // Should pad, but no pad size specified\n            // We infer the pad size from the resize size\n            this.pad_size = this.size\n        }\n    }\n\n    /**\n     * Resize the image to make a thumbnail. The image is resized so that no dimension is larger than any\n     * corresponding dimension of the specified size.\n     * @param {RawImage} image The image to be resized.\n     * @param {{height:number, width:number}} size The size `{\"height\": h, \"width\": w}` to resize the image to.\n     * @param {string | 0 | 1 | 2 | 3 | 4 | 5} [resample=2] The resampling filter to use.\n     * @returns {Promise<RawImage>} The resized image.\n     */\n    async thumbnail(image, size, resample = 2) {\n        const input_height = image.height;\n        const input_width = image.width;\n\n        const output_height = size.height;\n        const output_width = size.width;\n\n        // We always resize to the smallest of either the input or output size.\n        let height = Math.min(input_height, output_height)\n        let width = Math.min(input_width, output_width)\n\n        if (height === input_height && width === input_width) {\n            return image;\n        }\n        if (input_height > input_width) {\n            width = Math.floor(input_width * height / input_height);\n        } else if (input_width > input_height) {\n            height = Math.floor(input_height * width / input_width);\n        }\n        return await image.resize(width, height, { resample });\n    }\n\n\n    /**\n     * Crops the margin of the image. Gray pixels are considered margin (i.e., pixels with a value below the threshold).\n     * @param {RawImage} image The image to be cropped.\n     * @param {number} gray_threshold Value below which pixels are considered to be gray.\n     * @returns {Promise<RawImage>} The cropped image.\n     */\n    async crop_margin(image, gray_threshold = 200) {\n\n        const gray_image = image.clone().grayscale();\n\n        const minValue = min(gray_image.data)[0];\n        const maxValue = max(gray_image.data)[0];\n        const diff = maxValue - minValue;\n\n        if (diff === 0) {\n            return image;\n        }\n\n        const threshold = gray_threshold / 255;\n\n        let x_min = gray_image.width, y_min = gray_image.height, x_max = 0, y_max = 0;\n        for (let j = 0; j < gray_image.height; ++j) {\n            const row = j * gray_image.width;\n            for (let i = 0; i < gray_image.width; ++i) {\n                if ((gray_image.data[row + i] - minValue) / diff < threshold) {\n                    // We have a non-zero pixel, so we update the min/max values accordingly\n                    x_min = Math.min(x_min, i);\n                    y_min = Math.min(y_min, j);\n                    x_max = Math.max(x_max, i);\n                    y_max = Math.max(y_max, j);\n                }\n            }\n        }\n\n        image = await image.crop([x_min, y_min, x_max, y_max]);\n        return image;\n    }\n\n    /**\n     * Pad the image by a certain amount.\n     * @param {Float32Array} pixelData The pixel data to pad.\n     * @param {number[]} imgDims The dimensions of the image (height, width, channels).\n     * @param {{width:number; height:number}|number} padSize The dimensions of the padded image.\n     * @param {Object} options The options for padding.\n     * @param {'constant'|'symmetric'} [options.mode='constant'] The type of padding to add.\n     * @param {boolean} [options.center=false] Whether to center the image.\n     * @param {number} [options.constant_values=0] The constant value to use for padding.\n     * @returns {[Float32Array, number[]]} The padded pixel data and image dimensions.\n     */\n    pad_image(pixelData, imgDims, padSize, {\n        mode = 'constant',\n        center = false,\n        constant_values = 0,\n    } = {}) {\n        const [imageHeight, imageWidth, imageChannels] = imgDims;\n\n        let paddedImageWidth, paddedImageHeight;\n        if (typeof padSize === 'number') {\n            paddedImageWidth = padSize;\n            paddedImageHeight = padSize;\n        } else {\n            paddedImageWidth = padSize.width;\n            paddedImageHeight = padSize.height;\n        }\n\n        // Only add padding if there is a difference in size\n        if (paddedImageWidth !== imageWidth || paddedImageHeight !== imageHeight) {\n            const paddedPixelData = new Float32Array(paddedImageWidth * paddedImageHeight * imageChannels);\n            if (Array.isArray(constant_values)) {\n                // Fill with constant values, cycling through the array\n                for (let i = 0; i < paddedPixelData.length; ++i) {\n                    paddedPixelData[i] = constant_values[i % imageChannels];\n                }\n            } else if (constant_values !== 0) {\n                paddedPixelData.fill(constant_values);\n            }\n\n            const [left, top] = center\n                ? [Math.floor((paddedImageWidth - imageWidth) / 2), Math.floor((paddedImageHeight - imageHeight) / 2)]\n                : [0, 0];\n\n            // Copy the original image into the padded image\n            for (let i = 0; i < imageHeight; ++i) {\n                const a = (i + top) * paddedImageWidth;\n                const b = i * imageWidth;\n                for (let j = 0; j < imageWidth; ++j) {\n                    const c = (a + j + left) * imageChannels;\n                    const d = (b + j) * imageChannels;\n                    for (let k = 0; k < imageChannels; ++k) {\n                        paddedPixelData[c + k] = pixelData[d + k];\n                    }\n                }\n            }\n\n            if (mode === 'symmetric') {\n                if (center) {\n                    throw new Error('`center` padding is not supported when `mode` is set to `symmetric`.');\n                    // TODO: Implement this\n                }\n                const h1 = imageHeight - 1;\n                const w1 = imageWidth - 1;\n                for (let i = 0; i < paddedImageHeight; ++i) {\n                    const a = i * paddedImageWidth;\n                    const b = calculateReflectOffset(i, h1) * imageWidth;\n\n                    for (let j = 0; j < paddedImageWidth; ++j) {\n                        if (i < imageHeight && j < imageWidth) continue; // Do not overwrite original image\n                        const c = (a + j) * imageChannels;\n                        const d = (b + calculateReflectOffset(j, w1)) * imageChannels;\n\n                        // Copy channel-wise\n                        for (let k = 0; k < imageChannels; ++k) {\n                            paddedPixelData[c + k] = pixelData[d + k];\n                        }\n                    }\n                }\n            }\n\n\n            // Update pixel data and image dimensions\n            pixelData = paddedPixelData;\n            imgDims = [paddedImageHeight, paddedImageWidth, imageChannels]\n        }\n        return [pixelData, imgDims];\n    }\n\n    /**\n     * Rescale the image' pixel values by `this.rescale_factor`.\n     * @param {Float32Array} pixelData The pixel data to rescale.\n     * @returns {void}\n     */\n    rescale(pixelData) {\n        for (let i = 0; i < pixelData.length; ++i) {\n            pixelData[i] = this.rescale_factor * pixelData[i];\n        }\n    }\n\n    /**\n     * Find the target (width, height) dimension of the output image after\n     * resizing given the input image and the desired size.\n     * @param {RawImage} image The image to resize.\n     * @param {any} size The size to use for resizing the image. \n     * @returns {[number, number]} The target (width, height) dimension of the output image after resizing.\n     */\n    get_resize_output_image_size(image, size) {\n        // `size` comes in many forms, so we need to handle them all here:\n        // 1. `size` is an integer, in which case we resize the image to be a square \n\n        const [srcWidth, srcHeight] = image.size;\n\n        let shortest_edge;\n        let longest_edge;\n\n        if (this.do_thumbnail) {\n            // NOTE: custom logic for `Donut` models\n            const { height, width } = size;\n            shortest_edge = Math.min(height, width)\n        }\n        // Support both formats for backwards compatibility\n        else if (Number.isInteger(size)) {\n            shortest_edge = size;\n            longest_edge = this.config.max_size ?? shortest_edge;\n\n        } else if (size !== undefined) {\n            // Extract known properties from `size`\n            shortest_edge = size.shortest_edge;\n            longest_edge = size.longest_edge;\n        }\n\n        // If `longest_edge` and `shortest_edge` are set, maintain aspect ratio and resize to `shortest_edge`\n        // while keeping the largest dimension <= `longest_edge`\n...\n", "needle_token_start": 10237, "needle_token_end": 10522, "code_context_ntokens": 15974, "output": ["```javascript\n/**\n * Returns an array containing the specified window.\n * @param {number} window_length The length of the window in samples.\n * @param {string} name The name of the window function.\n * @param {Object} options Additional options.\n * @param {boolean} [options.periodic=true] Whether the window is periodic or symmetric.\n * @param {number} [options.frame_length=null] The length of the analysis frames in samples.\n * Provide a value for `frame_length` if the window is smaller than the frame length, so that it will be zero-padded.\n * @param {boolean} [options.center=true] Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided.\n * @returns {Float64Array} The window of shape `(window_length,)` or `(frame_length,)`.\n */\nexport function window_function(window_length, name, {\n    periodic = true,\n    frame_length = null,\n    center = true,\n} = {}) {\n    const length = periodic ? window_length + 1 : window_length;\n    let window;\n    switch (name) {\n        case 'boxcar':\n            window = new Float64Array(length).fill(1.0);\n            break;\n        case 'hann':\n        case 'hann_window':\n            window = hanning(length);\n            break;\n        case 'povey':\n            window = hanning(length).map(x => Math.pow(x, 0.85));\n            break;\n        default:\n            throw new Error(`Unknown window type ${name}.`);\n    }\n    if (periodic) {\n        window = window.subarray(0, window_length);\n    }\n    if (frame_length === null) {\n        return window;\n    }\n    if (window_length > frame_length) {\n        throw new Error(`Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`);\n    }\n\n    return window;\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "validate_audio_inputs", "language": "typescript", "path": "src/processors.js", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: Ensures that the audio data provided is in the correct format for processing by a specific audio feature extraction tool.\n2. **Input**: Takes two parameters: the audio data and the name of the feature extractor.\n3. **Output**: Does not return any value; however, it throws an error if the validation fails.\n4. **Procedure**: Checks if the audio data is either a `Float32Array` or a `Float64Array`. If the audio data does not match these types, an error is thrown, indicating the expected data types and providing a hint to use a specific function to obtain the correct audio data format.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/image.js\n\n/**\n * @file Helper module for image processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/image\n */\n\nimport { getFile } from './hub.js';\nimport { env } from '../env.js';\nimport { Tensor } from './tensor.js';\n\n// Will be empty (or not used) if running in browser or web-worker\nimport sharp from 'sharp';\n\nconst BROWSER_ENV = typeof self !== 'undefined';\nconst WEBWORKER_ENV = BROWSER_ENV && self.constructor.name === 'DedicatedWorkerGlobalScope';\n\nlet createCanvasFunction;\nlet ImageDataClass;\nlet loadImageFunction;\nif (BROWSER_ENV) {\n    // Running in browser or web-worker\n    createCanvasFunction = (/** @type {number} */ width, /** @type {number} */ height) => {\n        if (!self.OffscreenCanvas) {\n            throw new Error('OffscreenCanvas not supported by this browser.');\n        }\n        return new self.OffscreenCanvas(width, height)\n    };\n    loadImageFunction = self.createImageBitmap;\n    ImageDataClass = self.ImageData;\n\n} else if (sharp) {\n    // Running in Node.js, electron, or other non-browser environment\n\n    loadImageFunction = async (/**@type {sharp.Sharp}*/img) => {\n        const metadata = await img.metadata();\n        const rawChannels = metadata.channels;\n\n        let { data, info } = await img.raw().toBuffer({ resolveWithObject: true });\n\n        const newImage = new RawImage(new Uint8ClampedArray(data), info.width, info.height, info.channels);\n        if (rawChannels !== undefined && rawChannels !== info.channels) {\n            // Make sure the new image has the same number of channels as the input image.\n            // This is necessary for grayscale images.\n            newImage.convert(rawChannels);\n        }\n        return newImage;\n    }\n\n} else {\n    throw new Error('Unable to load image processing library.');\n}\n\n\n// Defined here: https://github.com/python-pillow/Pillow/blob/a405e8406b83f8bfb8916e93971edc7407b8b1ff/src/libImaging/Imaging.h#L262-L268\nconst RESAMPLING_MAPPING = {\n    0: 'nearest',\n    1: 'lanczos',\n    2: 'bilinear',\n    3: 'bicubic',\n    4: 'box',\n    5: 'hamming',\n}\n\n/**\n * Mapping from file extensions to MIME types.\n */\nconst CONTENT_TYPE_MAP = new Map([\n    ['png', 'image/png'],\n    ['jpg', 'image/jpeg'],\n    ['jpeg', 'image/jpeg'],\n    ['gif', 'image/gif'],\n]);\n\nexport class RawImage {\n\n    /**\n     * Create a new `RawImage` object.\n     * @param {Uint8ClampedArray|Uint8Array} data The pixel data.\n     * @param {number} width The width of the image.\n     * @param {number} height The height of the image.\n     * @param {1|2|3|4} channels The number of channels.\n     */\n    constructor(data, width, height, channels) {\n        this.data = data;\n        this.width = width;\n...\n// Path: src/utils/audio.js\n/**\n * @file Helper module for audio processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/audio\n */\n\nimport {\n    getFile,\n} from './hub.js';\nimport { FFT, max } from './maths.js';\nimport {\n    calculateReflectOffset,\n} from './core.js';\n\n\n/**\n * Helper function to read audio from a path/URL.\n * @param {string|URL} url The path/URL to load the audio from.\n * @param {number} sampling_rate The sampling rate to use when decoding the audio.\n * @returns {Promise<Float32Array>} The decoded audio as a `Float32Array`.\n */\nexport async function read_audio(url, sampling_rate) {\n    if (typeof AudioContext === 'undefined') {\n        // Running in node or an environment without AudioContext\n        throw Error(\n            \"Unable to load audio from path/URL since `AudioContext` is not available in your environment. \" +\n            \"Instead, audio data should be passed directly to the pipeline/processor. \" +\n            \"For more information and some example code, see https://huggingface.co/docs/transformers.js/guides/node-audio-processing.\"\n        )\n    }\n\n    const response = await (await getFile(url)).arrayBuffer();\n    const audioCTX = new AudioContext({ sampleRate: sampling_rate });\n    if (typeof sampling_rate === 'undefined') {\n        console.warn(`No sampling rate provided, using default of ${audioCTX.sampleRate}Hz.`)\n    }\n    const decoded = await audioCTX.decodeAudioData(response);\n\n    /** @type {Float32Array} */\n    let audio;\n\n    // We now replicate HuggingFace's `ffmpeg_read` method:\n    if (decoded.numberOfChannels === 2) {\n        // When downmixing a stereo audio file to mono using the -ac 1 option in FFmpeg,\n        // the audio signal is summed across both channels to create a single mono channel.\n        // However, if the audio is at full scale (i.e. the highest possible volume level),\n        // the summing of the two channels can cause the audio signal to clip or distort.\n\n        // To prevent this clipping, FFmpeg applies a scaling factor of 1/sqrt(2) (~ 0.707)\n        // to the audio signal before summing the two channels. This scaling factor ensures\n        // that the combined audio signal will not exceed the maximum possible level, even\n        // if both channels are at full scale.\n\n        // After applying this scaling factor, the audio signal from both channels is summed\n        // to create a single mono channel. It's worth noting that this scaling factor is\n        // only applied when downmixing stereo audio to mono using the -ac 1 option in FFmpeg.\n        // If you're using a different downmixing method, or if you're not downmixing the\n        // audio at all, this scaling factor may not be needed.\n        const SCALING_FACTOR = Math.sqrt(2);\n\n        const left = decoded.getChannelData(0);\n        const right = decoded.getChannelData(1);\n\n        audio = new Float32Array(left.length);\n        for (let i = 0; i < decoded.length; ++i) {\n            audio[i] = SCALING_FACTOR * (left[i] + right[i]) / 2;\n        }\n\n    } else {\n        // If the audio is not stereo, we can just use the first channel:\n        audio = decoded.getChannelData(0);\n    }\n\n    return audio;\n}\n\n/**\n * Generates a Hanning window of length M.\n *\n * @param {number} M The length of the Hanning window to generate.\n * @returns {Float64Array} The generated Hanning window.\n */\nexport function hanning(M) {\n    if (M < 1) {\n        return new Float64Array();\n    }\n    if (M === 1) {\n        return new Float64Array([1]);\n    }\n    const denom = M - 1;\n    const factor = Math.PI / denom;\n    const cos_vals = new Float64Array(M);\n    for (let i = 0; i < M; ++i) {\n        const n = 2 * i - denom;\n        cos_vals[i] = 0.5 + 0.5 * Math.cos(factor * n);\n    }\n    return cos_vals;\n}\n\nconst HERTZ_TO_MEL_MAPPING = {\n    \"htk\": (/** @type {number} */ freq) => 2595.0 * Math.log10(1.0 + (freq / 700.0)),\n    \"kaldi\": (/** @type {number} */ freq) => 1127.0 * Math.log(1.0 + (freq / 700.0)),\n    \"slaney\": (/** @type {number} */ freq, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = 27.0 / Math.log(6.4)) =>\n        freq >= min_log_hertz\n            ? min_log_mel + Math.log(freq / min_log_hertz) * logstep\n            : 3.0 * freq / 200.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} freq \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction hertz_to_mel(freq, mel_scale = \"htk\") {\n    const fn = HERTZ_TO_MEL_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof freq === 'number' ? fn(freq) : freq.map(x => fn(x));\n}\n\nconst MEL_TO_HERTZ_MAPPING = {\n    \"htk\": (/** @type {number} */ mels) => 700.0 * (10.0 ** (mels / 2595.0) - 1.0),\n    \"kaldi\": (/** @type {number} */ mels) => 700.0 * (Math.exp(mels / 1127.0) - 1.0),\n    \"slaney\": (/** @type {number} */ mels, min_log_hertz = 1000.0, min_log_mel = 15.0, logstep = Math.log(6.4) / 27.0) => mels >= min_log_mel\n        ? min_log_hertz * Math.exp(logstep * (mels - min_log_mel))\n        : 200.0 * mels / 3.0,\n}\n\n/**\n * @template {Float32Array|Float64Array|number} T \n * @param {T} mels \n * @param {string} [mel_scale]\n * @returns {T}\n */\nfunction mel_to_hertz(mels, mel_scale = \"htk\") {\n    const fn = MEL_TO_HERTZ_MAPPING[mel_scale];\n    if (!fn) {\n        throw new Error('mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\".');\n    }\n\n    return typeof mels === 'number' ? fn(mels) : mels.map(x => fn(x));\n}\n\n/**\n* Creates a triangular filter bank.\n*\n* Adapted from torchaudio and librosa.\n*\n* @param {Float64Array} fft_freqs Discrete frequencies of the FFT bins in Hz, of shape `(num_frequency_bins,)`.\n* @param {Float64Array} filter_freqs Center frequencies of the triangular filters to create, in Hz, of shape `(num_mel_filters,)`.\n* @returns {number[][]} of shape `(num_frequency_bins, num_mel_filters)`.\n*/\nfunction _create_triangular_filter_bank(fft_freqs, filter_freqs) {\n    const filter_diff = Float64Array.from(\n        { length: filter_freqs.length - 1 },\n        (_, i) => filter_freqs[i + 1] - filter_freqs[i]\n    );\n\n    const slopes = Array.from({\n        length: fft_freqs.length\n    }, () => new Array(filter_freqs.length));\n\n    for (let j = 0; j < fft_freqs.length; ++j) {\n        const slope = slopes[j];\n        for (let i = 0; i < filter_freqs.length; ++i) {\n            slope[i] = filter_freqs[i] - fft_freqs[j];\n        }\n    }\n\n    const numFreqs = filter_freqs.length - 2;\n    const ret = Array.from({ length: numFreqs }, () => new Array(fft_freqs.length));\n\n    for (let j = 0; j < fft_freqs.length; ++j) { // 201\n        const slope = slopes[j];\n        for (let i = 0; i < numFreqs; ++i) { // 80\n            const down = -slope[i] / filter_diff[i];\n            const up = slope[i + 2] / filter_diff[i + 1];\n            ret[i][j] = Math.max(0, Math.min(down, up));\n        }\n    }\n    return ret;\n}\n\n/**\n * Return evenly spaced numbers over a specified interval.\n * @param {number} start The starting value of the sequence.\n * @param {number} end The end value of the sequence.\n * @param {number} num Number of samples to generate.\n * @returns `num` evenly spaced samples, calculated over the interval `[start, stop]`.\n */\nfunction linspace(start, end, num) {\n    const step = (end - start) / (num - 1);\n    return Float64Array.from({ length: num }, (_, i) => start + step * i);\n}\n\n/**\n * Creates a frequency bin conversion matrix used to obtain a mel spectrogram. This is called a *mel filter bank*, and\n * various implementation exist, which differ in the number of filters, the shape of the filters, the way the filters\n * are spaced, the bandwidth of the filters, and the manner in which the spectrum is warped. The goal of these\n * features is to approximate the non-linear human perception of the variation in pitch with respect to the frequency.\n * @param {number} num_frequency_bins Number of frequencies used to compute the spectrogram (should be the same as in `stft`).\n * @param {number} num_mel_filters Number of mel filters to generate.\n * @param {number} min_frequency Lowest frequency of interest in Hz.\n * @param {number} max_frequency Highest frequency of interest in Hz. This should not exceed `sampling_rate / 2`.\n * @param {number} sampling_rate Sample rate of the audio waveform.\n * @param {string} [norm] If `\"slaney\"`, divide the triangular mel weights by the width of the mel band (area normalization).\n * @param {string} [mel_scale] The mel frequency scale to use, `\"htk\"` or `\"slaney\"`.\n * @param {boolean} [triangularize_in_mel_space] If this option is enabled, the triangular filter is applied in mel space rather than frequency space.\n * This should be set to `true` in order to get the same results as `torchaudio` when computing mel filters.\n * @returns {number[][]} Triangular filter bank matrix, which is a 2D array of shape (`num_frequency_bins`, `num_mel_filters`).\n * This is a projection matrix to go from a spectrogram to a mel spectrogram.\n */\nexport function mel_filter_bank(\n    num_frequency_bins,\n    num_mel_filters,\n    min_frequency,\n    max_frequency,\n    sampling_rate,\n    norm = null,\n    mel_scale = \"htk\",\n    triangularize_in_mel_space = false,\n) {\n    if (norm !== null && norm !== \"slaney\") {\n        throw new Error('norm must be one of null or \"slaney\"');\n    }\n\n    const mel_min = hertz_to_mel(min_frequency, mel_scale);\n    const mel_max = hertz_to_mel(max_frequency, mel_scale);\n    const mel_freqs = linspace(mel_min, mel_max, num_mel_filters + 2);\n\n    let filter_freqs = mel_to_hertz(mel_freqs, mel_scale);\n    let fft_freqs; // frequencies of FFT bins in Hz\n\n    if (triangularize_in_mel_space) {\n        const fft_bin_width = sampling_rate / (num_frequency_bins * 2);\n        fft_freqs = hertz_to_mel(Float64Array.from({ length: num_frequency_bins }, (_, i) => i * fft_bin_width), mel_scale);\n        filter_freqs = mel_freqs;\n    } else {\n        fft_freqs = linspace(0, Math.floor(sampling_rate / 2), num_frequency_bins);\n    }\n\n    const mel_filters = _create_triangular_filter_bank(fft_freqs, filter_freqs);\n\n    if (norm !== null && norm === \"slaney\") {\n        // Slaney-style mel is scaled to be approx constant energy per channel\n        for (let i = 0; i < num_mel_filters; ++i) {\n            const filter = mel_filters[i];\n            const enorm = 2.0 / (filter_freqs[i + 2] - filter_freqs[i]);\n            for (let j = 0; j < num_frequency_bins; ++j) {\n                // Apply this enorm to all frequency bins\n                filter[j] *= enorm;\n            }\n        }\n    }\n\n    // TODO warn if there is a zero row\n\n    return mel_filters;\n\n}\n\n/**\n * @template {Float32Array|Float64Array} T\n * Pads an array with a reflected version of itself on both ends.\n * @param {T} array The array to pad.\n * @param {number} left The amount of padding to add to the left.\n * @param {number} right The amount of padding to add to the right.\n * @returns {T} The padded array.\n */\nfunction padReflect(array, left, right) {\n    // @ts-ignore\n    const padded = new array.constructor(array.length + left + right);\n    const w = array.length - 1;\n\n    for (let i = 0; i < array.length; ++i) {\n        padded[left + i] = array[i];\n    }\n\n    for (let i = 1; i <= left; ++i) {\n        padded[left - i] = array[calculateReflectOffset(i, w)];\n    }\n\n    for (let i = 1; i <= right; ++i) {\n        padded[w + left + i] = array[calculateReflectOffset(w - i, w)];\n    }\n\n    return padded;\n}\n\n/**\n * Helper function to compute `amplitude_to_db` and `power_to_db`.\n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram \n * @param {number} factor \n * @param {number} reference \n * @param {number} min_value \n * @param {number} db_range \n * @returns {T}\n */\nfunction _db_conversion_helper(spectrogram, factor, reference, min_value, db_range) {\n    if (reference <= 0) {\n        throw new Error('reference must be greater than zero');\n    }\n\n    if (min_value <= 0) {\n        throw new Error('min_value must be greater than zero');\n    }\n\n    reference = Math.max(min_value, reference);\n\n    const logReference = Math.log10(reference);\n    for (let i = 0; i < spectrogram.length; ++i) {\n        spectrogram[i] = factor * Math.log10(Math.max(min_value, spectrogram[i]) - logReference)\n    }\n\n    if (db_range !== null) {\n        if (db_range <= 0) {\n            throw new Error('db_range must be greater than zero');\n        }\n        const maxValue = max(spectrogram)[0] - db_range;\n        for (let i = 0; i < spectrogram.length; ++i) {\n            spectrogram[i] = Math.max(spectrogram[i], maxValue);\n        }\n    }\n\n    return spectrogram;\n}\n\n/**\n * Converts an amplitude spectrogram to the decibel scale. This computes `20 * log10(spectrogram / reference)`,\n * using basic logarithm properties for numerical stability. NOTE: Operates in-place.\n * \n * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a\n * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.\n * This means that large variations in energy may not sound all that different if the sound is loud to begin with.\n * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.\n * \n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram The input amplitude (mel) spectrogram.\n * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB.\n * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [min_value=1e-5] The spectrogram will be clipped to this minimum value before conversion to decibels,\n * to avoid taking `log(0)`. The default of `1e-5` corresponds to a minimum of -100 dB. Must be greater than zero.\n * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the\n * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @returns {T} The modified spectrogram in decibels.\n */\nfunction amplitude_to_db(spectrogram, reference = 1.0, min_value = 1e-5, db_range = null) {\n    return _db_conversion_helper(spectrogram, 20.0, reference, min_value, db_range);\n}\n\n/**\n * Converts a power spectrogram to the decibel scale. This computes `10 * log10(spectrogram / reference)`,\n * using basic logarithm properties for numerical stability. NOTE: Operates in-place.\n * \n * The motivation behind applying the log function on the (mel) spectrogram is that humans do not hear loudness on a\n * linear scale. Generally to double the perceived volume of a sound we need to put 8 times as much energy into it.\n * This means that large variations in energy may not sound all that different if the sound is loud to begin with.\n * This compression operation makes the (mel) spectrogram features match more closely what humans actually hear.\n * \n * Based on the implementation of `librosa.power_to_db`.\n * \n * @template {Float32Array|Float64Array} T\n * @param {T} spectrogram The input power (mel) spectrogram. Note that a power spectrogram has the amplitudes squared!\n * @param {number} [reference=1.0] Sets the input spectrogram value that corresponds to 0 dB.\n * For example, use `np.max(spectrogram)` to set the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels,\n * to avoid taking `log(0)`. The default of `1e-10` corresponds to a minimum of -100 dB. Must be greater than zero.\n * @param {number} [db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the\n * difference between the peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @returns {T} The modified spectrogram in decibels.\n */\nfunction power_to_db(spectrogram, reference = 1.0, min_value = 1e-10, db_range = null) {\n    return _db_conversion_helper(spectrogram, 10.0, reference, min_value, db_range);\n}\n\n/**\n * Calculates a spectrogram over one waveform using the Short-Time Fourier Transform.\n * \n * This function can create the following kinds of spectrograms:\n *   - amplitude spectrogram (`power = 1.0`)\n *   - power spectrogram (`power = 2.0`)\n *   - complex-valued spectrogram (`power = None`)\n *   - log spectrogram (use `log_mel` argument)\n *   - mel spectrogram (provide `mel_filters`)\n *   - log-mel spectrogram (provide `mel_filters` and `log_mel`)\n *\n * In this implementation, the window is assumed to be zero-padded to have the same size as the analysis frame.\n * A padded window can be obtained from `window_function()`. The FFT input buffer may be larger than the analysis frame, \n * typically the next power of two.\n * \n * @param {Float32Array|Float64Array} waveform The input waveform of shape `(length,)`. This must be a single real-valued, mono waveform.\n * @param {Float32Array|Float64Array} window The windowing function to apply of shape `(frame_length,)`, including zero-padding if necessary. The actual window length may be\n * shorter than `frame_length`, but we're assuming the array has already been zero-padded.\n * @param {number} frame_length The length of the analysis frames in samples (a.k.a., `fft_length`).\n * @param {number} hop_length The stride between successive analysis frames in samples.\n * @param {Object} options\n * @param {number} [options.fft_length=null] The size of the FFT buffer in samples. This determines how many frequency bins the spectrogram will have.\n * For optimal speed, this should be a power of two. If `null`, uses `frame_length`.\n * @param {number} [options.power=1.0] If 1.0, returns the amplitude spectrogram. If 2.0, returns the power spectrogram. If `null`, returns complex numbers.\n * @param {boolean} [options.center=true] Whether to pad the waveform so that frame `t` is centered around time `t * hop_length`. If `false`, frame\n * `t` will start at time `t * hop_length`.\n * @param {string} [options.pad_mode=\"reflect\"] Padding mode used when `center` is `true`. Possible values are: `\"constant\"` (pad with zeros),\n * `\"edge\"` (pad with edge values), `\"reflect\"` (pads with mirrored values).\n * @param {boolean} [options.onesided=true] If `true`, only computes the positive frequencies and returns a spectrogram containing `fft_length // 2 + 1`\n * frequency bins. If `false`, also computes the negative frequencies and returns `fft_length` frequency bins.\n * @param {number} [options.preemphasis=null] Coefficient for a low-pass filter that applies pre-emphasis before the DFT.\n * @param {number[][]} [options.mel_filters=null] The mel filter bank of shape `(num_freq_bins, num_mel_filters)`.\n * If supplied, applies this filter bank to create a mel spectrogram.\n * @param {number} [options.mel_floor=1e-10] Minimum value of mel frequency banks.\n * @param {string} [options.log_mel=null] How to convert the spectrogram to log scale. Possible options are:\n * `null` (don't convert), `\"log\"` (take the natural logarithm) `\"log10\"` (take the base-10 logarithm), `\"dB\"` (convert to decibels).\n * Can only be used when `power` is not `null`.\n * @param {number} [options.reference=1.0] Sets the input spectrogram value that corresponds to 0 dB. For example, use `max(spectrogram)[0]` to set\n * the loudest part to 0 dB. Must be greater than zero.\n * @param {number} [options.min_value=1e-10] The spectrogram will be clipped to this minimum value before conversion to decibels, to avoid taking `log(0)`.\n * For a power spectrogram, the default of `1e-10` corresponds to a minimum of -100 dB. For an amplitude spectrogram, the value `1e-5` corresponds to -100 dB.\n * Must be greater than zero.\n * @param {number} [options.db_range=null] Sets the maximum dynamic range in decibels. For example, if `db_range = 80`, the difference between the\n * peak value and the smallest value will never be more than 80 dB. Must be greater than zero.\n * @param {boolean} [options.remove_dc_offset=null] Subtract mean from waveform on each frame, applied before pre-emphasis. This should be set to `true` in\n * order to get the same results as `torchaudio.compliance.kaldi.fbank` when computing mel filters.\n * @param {number} [options.max_num_frames=null] If provided, limits the number of frames to compute to this value.\n * @param {boolean} [options.do_pad=true] If `true`, pads the output spectrogram to have `max_num_frames` frames.\n * @param {boolean} [options.transpose=false] If `true`, the returned spectrogram will have shape `(num_frames, num_frequency_bins/num_mel_filters)`. If `false`, the returned spectrogram will have shape `(num_frequency_bins/num_mel_filters, num_frames)`.\n * @returns {{data: Float32Array, dims: number[]}} Spectrogram of shape `(num_frequency_bins, length)` (regular spectrogram) or shape `(num_mel_filters, length)` (mel spectrogram).\n */\nexport function spectrogram(\n    waveform,\n    window,\n    frame_length,\n    hop_length,\n    {\n        fft_length = null,\n        power = 1.0,\n        center = true,\n        pad_mode = \"reflect\",\n        onesided = true,\n        preemphasis = null,\n        mel_filters = null,\n        mel_floor = 1e-10,\n        log_mel = null,\n        reference = 1.0,\n        min_value = 1e-10,\n        db_range = null,\n        remove_dc_offset = null,\n\n        // Custom parameters for efficiency reasons\n        max_num_frames = null,\n        do_pad = true,\n        transpose = false,\n    } = {}\n) {\n    const window_length = window.length;\n    if (fft_length === null) {\n        fft_length = frame_length;\n    }\n    if (frame_length > fft_length) {\n        throw Error(`frame_length (${frame_length}) may not be larger than fft_length (${fft_length})`)\n    }\n\n    if (window_length !== frame_length) {\n        throw new Error(`Length of the window (${window_length}) must equal frame_length (${frame_length})`);\n    }\n\n    if (hop_length <= 0) {\n        throw new Error(\"hop_length must be greater than zero\");\n    }\n\n    if (center) {\n        if (pad_mode !== 'reflect') {\n            throw new Error(`pad_mode=\"${pad_mode}\" not implemented yet.`)\n        }\n        const half_window = Math.floor((fft_length - 1) / 2) + 1;\n        waveform = padReflect(waveform, half_window, half_window);\n    }\n\n    // split waveform into frames of frame_length size\n    const num_frames = Math.floor(1 + Math.floor((waveform.length - frame_length) / hop_length))\n\n    const num_frequency_bins = onesided ? Math.floor(fft_length / 2) + 1 : fft_length\n\n    let d1 = num_frames;\n    let d1Max = num_frames;\n\n    // If maximum number of frames is provided, we must either pad or truncate\n    if (max_num_frames !== null) {\n        if (max_num_frames > num_frames) { // input is too short, so we pad\n            if (do_pad) {\n                d1Max = max_num_frames;\n            }\n        } else { // input is too long, so we truncate\n            d1Max = d1 = max_num_frames;\n        }\n    }\n\n    // Preallocate arrays to store output.\n    const fft = new FFT(fft_length);\n    const inputBuffer = new Float64Array(fft_length);\n    const outputBuffer = new Float64Array(fft.outputBufferSize);\n    const magnitudes = new Array(d1);\n\n    for (let i = 0; i < d1; ++i) {\n        // Populate buffer with waveform data\n        const offset = i * hop_length;\n        for (let j = 0; j < frame_length; ++j) {\n            inputBuffer[j] = waveform[offset + j];\n        }\n\n        if (remove_dc_offset) {\n            let sum = 0;\n            for (let j = 0; j < frame_length; ++j) {\n                sum += inputBuffer[j];\n            }\n            const mean = sum / frame_length;\n            for (let j = 0; j < frame_length; ++j) {\n                inputBuffer[j] -= mean;\n            }\n        }\n\n        if (preemphasis !== null) {\n            // Done in reverse to avoid copies and distructive modification\n            for (let j = frame_length - 1; j >= 1; --j) {\n                inputBuffer[j] -= preemphasis * inputBuffer[j - 1];\n            }\n            inputBuffer[0] *= 1 - preemphasis;\n        }\n\n        for (let j = 0; j < window.length; ++j) {\n            inputBuffer[j] *= window[j];\n        }\n\n        fft.realTransform(outputBuffer, inputBuffer);\n\n        // compute magnitudes\n        const row = new Array(num_frequency_bins);\n        for (let j = 0; j < row.length; ++j) {\n            const j2 = j << 1;\n            row[j] = outputBuffer[j2] ** 2 + outputBuffer[j2 + 1] ** 2;\n        }\n        magnitudes[i] = row;\n    }\n\n    // TODO what should happen if power is None?\n    // https://github.com/huggingface/transformers/issues/27772\n    if (power !== null && power !== 2) {\n        // slight optimization to not sqrt\n        const pow = 2 / power; // we use 2 since we already squared\n        for (let i = 0; i < magnitudes.length; ++i) {\n            const magnitude = magnitudes[i];\n            for (let j = 0; j < magnitude.length; ++j) {\n                magnitude[j] **= pow;\n            }\n        }\n    }\n\n    // TODO: What if `mel_filters` is null?\n    const num_mel_filters = mel_filters.length;\n\n    // Only here do we create Float32Array\n    const mel_spec = new Float32Array(num_mel_filters * d1Max);\n\n    // Perform matrix muliplication:\n    // mel_spec = mel_filters @ magnitudes.T\n    //  - mel_filters.shape=(80, 201)\n    //  - magnitudes.shape=(3000, 201) => - magnitudes.T.shape=(201, 3000)\n    //  - mel_spec.shape=(80, 3000)\n    const dims = transpose ? [d1Max, num_mel_filters] : [num_mel_filters, d1Max];\n    for (let i = 0; i < num_mel_filters; ++i) { // num melfilters (e.g., 80)\n        const filter = mel_filters[i];\n        for (let j = 0; j < d1; ++j) { // num frames (e.g., 3000)\n            const magnitude = magnitudes[j];\n\n            let sum = 0;\n            for (let k = 0; k < num_frequency_bins; ++k) { // num frequency bins (e.g., 201)\n                sum += filter[k] * magnitude[k];\n            }\n\n            mel_spec[\n                transpose\n                    ? j * num_mel_filters + i\n                    : i * d1 + j\n            ] = Math.max(mel_floor, sum);\n        }\n    }\n\n    if (power !== null && log_mel !== null) {\n        const o = Math.min(mel_spec.length, d1 * num_mel_filters);\n        switch (log_mel) {\n            case 'log':\n                for (let i = 0; i < o; ++i) {\n                    mel_spec[i] = Math.log(mel_spec[i]);\n                }\n                break;\n            case 'log10':\n                for (let i = 0; i < o; ++i) {\n                    mel_spec[i] = Math.log10(mel_spec[i]);\n                }\n                break;\n            case 'dB':\n                if (power === 1.0) {\n                    // NOTE: operates in-place\n                    amplitude_to_db(mel_spec, reference, min_value, db_range);\n                } else if (power === 2.0) {\n                    power_to_db(mel_spec, reference, min_value, db_range);\n                } else {\n                    throw new Error(`Cannot use log_mel option '${log_mel}' with power ${power}`)\n                }\n                break;\n            default:\n                throw new Error(`log_mel must be one of null, 'log', 'log10' or 'dB'. Got '${log_mel}'`);\n        }\n    }\n\n    return { data: mel_spec, dims };\n}\n\n/**\n * Returns an array containing the specified window.\n * @param {number} window_length The length of the window in samples.\n * @param {string} name The name of the window function.\n * @param {Object} options Additional options.\n * @param {boolean} [options.periodic=true] Whether the window is periodic or symmetric.\n * @param {number} [options.frame_length=null] The length of the analysis frames in samples.\n * Provide a value for `frame_length` if the window is smaller than the frame length, so that it will be zero-padded.\n * @param {boolean} [options.center=true] Whether to center the window inside the FFT buffer. Only used when `frame_length` is provided.\n * @returns {Float64Array} The window of shape `(window_length,)` or `(frame_length,)`.\n */\nexport function window_function(window_length, name, {\n    periodic = true,\n    frame_length = null,\n    center = true,\n} = {}) {\n    const length = periodic ? window_length + 1 : window_length;\n    let window;\n    switch (name) {\n        case 'boxcar':\n            window = new Float64Array(length).fill(1.0);\n            break;\n        case 'hann':\n        case 'hann_window':\n            window = hanning(length);\n            break;\n        case 'povey':\n            window = hanning(length).map(x => Math.pow(x, 0.85));\n            break;\n        default:\n            throw new Error(`Unknown window type ${name}.`);\n    }\n    if (periodic) {\n        window = window.subarray(0, window_length);\n    }\n    if (frame_length === null) {\n        return window;\n    }\n    if (window_length > frame_length) {\n        throw new Error(`Length of the window (${window_length}) may not be larger than frame_length (${frame_length})`);\n    }\n\n    return window;\n}\n\n// Path: src/processors.js\n\n/**\n * @file Processors are used to prepare non-textual inputs (e.g., image or audio) for a model.\n * \n * **Example:** Using a `WhisperProcessor` to prepare an audio input for a model.\n * ```javascript\n * import { AutoProcessor, read_audio } from '@xenova/transformers';\n *\n * let processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en');\n * let audio = await read_audio('https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac', 16000);\n * let { input_features } = await processor(audio);\n * // Tensor {\n * //   data: Float32Array(240000) [0.4752984642982483, 0.5597258806228638, 0.56434166431427, ...],\n * //   dims: [1, 80, 3000],\n * //   type: 'float32',\n * //   size: 240000,\n * // }\n * ```\n * \n * @module processors\n */\nimport {\n    Callable,\n    calculateDimensions,\n    calculateReflectOffset,\n} from './utils/core.js';\n\nimport {\n    getModelJSON,\n} from './utils/hub.js';\n\nimport {\n    min,\n    max,\n    softmax,\n    bankers_round,\n} from './utils/maths.js';\n\n\nimport { Tensor, permute, cat, interpolate, stack } from './utils/tensor.js';\n\nimport { RawImage } from './utils/image.js';\nimport {\n    window_function,\n    spectrogram,\n    mel_filter_bank,\n} from './utils/audio.js';\n\n\n// Helper functions\n\n/**\n * Converts bounding boxes from center format to corners format.\n * \n * @param {number[]} arr The coordinate for the center of the box and its width, height dimensions (center_x, center_y, width, height)\n * @returns {number[]} The coodinates for the top-left and bottom-right corners of the box (top_left_x, top_left_y, bottom_right_x, bottom_right_y)\n */\nfunction center_to_corners_format([centerX, centerY, width, height]) {\n    return [\n        centerX - width / 2,\n        centerY - height / 2,\n        centerX + width / 2,\n        centerY + height / 2\n    ];\n}\n\n/**\n * Post-processes the outputs of the model (for object detection).\n * @param {Object} outputs The outputs of the model that must be post-processed\n * @param {Tensor} outputs.logits The logits\n * @param {Tensor} outputs.pred_boxes The predicted boxes.\n * @param {number} [threshold=0.5] The threshold to use for the scores.\n * @param {number[][]} [target_sizes=null] The sizes of the original images.\n * @param {boolean} [is_zero_shot=false] Whether zero-shot object detection was performed.\n * @return {Object[]} An array of objects containing the post-processed outputs.\n * @private\n */\nfunction post_process_object_detection(outputs, threshold = 0.5, target_sizes = null, is_zero_shot = false) {\n    const out_logits = outputs.logits;\n    const out_bbox = outputs.pred_boxes;\n    const [batch_size, num_boxes, num_classes] = out_logits.dims;\n\n    if (target_sizes !== null && target_sizes.length !== batch_size) {\n        throw Error(\"Make sure that you pass in as many target sizes as the batch dimension of the logits\")\n    }\n    let toReturn = [];\n    for (let i = 0; i < batch_size; ++i) {\n        let target_size = target_sizes !== null ? target_sizes[i] : null;\n        let info = {\n            boxes: [],\n            classes: [],\n            scores: []\n        }\n        let logits = out_logits[i];\n        let bbox = out_bbox[i];\n\n        for (let j = 0; j < num_boxes; ++j) {\n            let logit = logits[j];\n\n            let indices = [];\n            let probs;\n            if (is_zero_shot) {\n                // Get indices of classes with high enough probability\n                probs = logit.sigmoid().data;\n                for (let k = 0; k < probs.length; ++k) {\n                    if (probs[k] > threshold) {\n                        indices.push(k);\n                    }\n                }\n\n            } else {\n                // Get most probable class\n                let maxIndex = max(logit.data)[1];\n\n                if (maxIndex === num_classes - 1) {\n                    // This is the background class, skip it\n                    continue;\n                }\n                indices.push(maxIndex);\n\n                // Compute softmax over classes\n                probs = softmax(logit.data);\n            }\n\n            for (const index of indices) {\n\n                // Some class has a high enough probability\n                /** @type {number[]} */\n                let box = bbox[j].data;\n\n                // convert to [x0, y0, x1, y1] format\n                box = center_to_corners_format(box)\n                if (target_size !== null) {\n                    box = box.map((x, i) => x * target_size[(i + 1) % 2])\n                }\n\n                info.boxes.push(box);\n                info.classes.push(index);\n                info.scores.push(probs[index]);\n            }\n        }\n        toReturn.push(info);\n    }\n    return toReturn;\n}\n\n/**\n * Named tuple to indicate the order we are using is (height x width), even though\n * the Graphics\u2019 industry standard is (width x height).\n * @typedef {[height: number, width: number]} HeightWidth\n */\n\n/**\n * Helper function to validate audio inputs.\n * @param {any} audio The audio data.\n * @param {string} feature_extractor The name of the feature extractor.\n * @private\n */\nfu\nnction validate_audio_inputs(audio, feature_extractor) {\n    if (!(audio instanceof Float32Array || audio instanceof Float64Array)) {\n        throw new Error(\n            `${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. ` +\n            `If using the feature extractor directly, remember to use \\`read_audio(url, sampling_rate)\\` to obtain the raw audio data of the file/url.`\n        )\n    }\n}\n\n/**\n * Helper function to constrain a value to be a multiple of a number.\n * @param {number} val The value to constrain.\n * @param {number} multiple The number to constrain to.\n * @param {number} [minVal=0] The minimum value to constrain to.\n * @param {number} [maxVal=null] The maximum value to constrain to.\n * @returns {number} The constrained value.\n * @private\n */\nfunction constraint_to_multiple_of(val, multiple, minVal = 0, maxVal = null) {\n    const a = val / multiple;\n    let x = bankers_round(a) * multiple;\n\n    if (maxVal !== null && x > maxVal) {\n        x = Math.floor(a) * multiple;\n    }\n\n    if (x < minVal) {\n        x = Math.ceil(a) * multiple;\n    }\n\n    return x;\n}\n\n/**\n * Rounds the height and width down to the closest multiple of size_divisibility\n * @param {[number, number]} size The size of the image\n * @param {number} divisor The divisor to use.\n * @returns {[number, number]} The rounded size.\n */\nfunction enforce_size_divisibility([width, height], divisor) {\n    return [\n        Math.max(Math.floor(width / divisor), 1) * divisor,\n        Math.max(Math.floor(height / divisor), 1) * divisor\n    ];\n}\n\n\n/**\n * Base class for feature extractors.\n *\n * @extends Callable\n */\nexport class FeatureExtractor extends Callable {\n    /**\n     * Constructs a new FeatureExtractor instance.\n     *\n     * @param {Object} config The configuration for the feature extractor.\n     */\n    constructor(config) {\n        super();\n        this.config = config\n    }\n}\n\n/**\n * @typedef {object} ImageFeatureExtractorResult\n * @property {Tensor} pixel_values The pixel values of the batched preprocessed images.\n * @property {HeightWidth[]} original_sizes Array of two-dimensional tuples like [[480, 640]].\n * @property {HeightWidth[]} reshaped_input_sizes Array of two-dimensional tuples like [[1000, 1330]].\n */\n\n/**\n * Feature extractor for image models.\n *\n * @extends FeatureExtractor\n */\nexport class ImageFeatureExtractor extends FeatureExtractor {\n\n    /**\n     * Constructs a new ImageFeatureExtractor instance.\n     *\n     * @param {Object} config The configuration for the feature extractor.\n     * @param {number[]} config.image_mean The mean values for image normalization.\n     * @param {number[]} config.image_std The standard deviation values for image normalization.\n     * @param {boolean} config.do_rescale Whether to rescale the image pixel values to the [0,1] range.\n     * @param {number} config.rescale_factor The factor to use for rescaling the image pixel values.\n     * @param {boolean} config.do_normalize Whether to normalize the image pixel values.\n     * @param {boolean} config.do_resize Whether to resize the image.\n     * @param {number} config.resample What method to use for resampling.\n     * @param {number|Object} config.size The size to resize the image to.\n     */\n    constructor(config) {\n        super(config);\n\n        this.image_mean = this.config.image_mean ?? this.config.mean;\n        this.image_std = this.config.image_std ?? this.config.std;\n\n        this.resample = this.config.resample ?? 2; // 2 => bilinear\n        this.do_rescale = this.config.do_rescale ?? true;\n        this.rescale_factor = this.config.rescale_factor ?? (1 / 255);\n        this.do_normalize = this.config.do_normalize;\n\n        this.do_resize = this.config.do_resize;\n        this.do_thumbnail = this.config.do_thumbnail;\n        this.size = this.config.size;\n        this.size_divisibility = this.config.size_divisibility ?? this.config.size_divisor;\n\n        this.do_center_crop = this.config.do_center_crop;\n        this.crop_size = this.config.crop_size;\n        this.do_convert_rgb = this.config.do_convert_rgb ?? true;\n        this.do_crop_margin = this.config.do_crop_margin;\n\n        this.pad_size = this.config.pad_size;\n        this.do_pad = this.config.do_pad;\n\n        if (this.do_pad && !this.pad_size && this.size && this.size.width !== undefined && this.size.height !== undefined) {\n            // Should pad, but no pad size specified\n            // We infer the pad size from the resize size\n            this.pad_size = this.size\n        }\n    }\n\n    /**\n     * Resize the image to make a thumbnail. The image is resized so that no dimension is larger than any\n     * corresponding dimension of the specified size.\n     * @param {RawImage} image The image to be resized.\n     * @param {{height:number, width:number}} size The size `{\"height\": h, \"width\": w}` to resize the image to.\n     * @param {string | 0 | 1 | 2 | 3 | 4 | 5} [resample=2] The resampling filter to use.\n     * @returns {Promise<RawImage>} The resized image.\n     */\n    async thumbnail(image, size, resample = 2) {\n        const input_height = image.height;\n        const input_width = image.width;\n\n        const output_height = size.height;\n        const output_width = size.width;\n\n        // We always resize to the smallest of either the input or output size.\n        let height = Math.min(input_height, output_height)\n        let width = Math.min(input_width, output_width)\n\n        if (height === input_height && width === input_width) {\n            return image;\n        }\n        if (input_height > input_width) {\n            width = Math.floor(input_width * height / input_height);\n        } else if (input_width > input_height) {\n            height = Math.floor(input_height * width / input_width);\n        }\n        return await image.resize(width, height, { resample });\n    }\n\n\n    /**\n     * Crops the margin of the image. Gray pixels are considered margin (i.e., pixels with a value below the threshold).\n     * @param {RawImage} image The image to be cropped.\n     * @param {number} gray_threshold Value below which pixels are considered to be gray.\n     * @returns {Promise<RawImage>} The cropped image.\n     */\n    async crop_margin(image, gray_threshold = 200) {\n\n        const gray_image = image.clone().grayscale();\n\n        const minValue = min(gray_image.data)[0];\n        const maxValue = max(gray_image.data)[0];\n        const diff = maxValue - minValue;\n\n        if (diff === 0) {\n            return image;\n        }\n\n        const threshold = gray_threshold / 255;\n\n        let x_min = gray_image.width, y_min = gray_image.height, x_max = 0, y_max = 0;\n        for (let j = 0; j < gray_image.height; ++j) {\n            const row = j * gray_image.width;\n            for (let i = 0; i < gray_image.width; ++i) {\n                if ((gray_image.data[row + i] - minValue) / diff < threshold) {\n                    // We have a non-zero pixel, so we update the min/max values accordingly\n                    x_min = Math.min(x_min, i);\n                    y_min = Math.min(y_min, j);\n                    x_max = Math.max(x_max, i);\n                    y_max = Math.max(y_max, j);\n                }\n            }\n        }\n\n        image = await image.crop([x_min, y_min, x_max, y_max]);\n        return image;\n    }\n\n    /**\n     * Pad the image by a certain amount.\n     * @param {Float32Array} pixelData The pixel data to pad.\n     * @param {number[]} imgDims The dimensions of the image (height, width, channels).\n     * @param {{width:number; height:number}|number} padSize The dimensions of the padded image.\n     * @param {Object} options The options for padding.\n     * @param {'constant'|'symmetric'} [options.mode='constant'] The type of padding to add.\n     * @param {boolean} [options.center=false] Whether to center the image.\n     * @param {number} [options.constant_values=0] The constant value to use for padding.\n     * @returns {[Float32Array, number[]]} The padded pixel data and image dimensions.\n     */\n    pad_image(pixelData, imgDims, padSize, {\n        mode = 'constant',\n        center = false,\n        constant_values = 0,\n    } = {}) {\n        const [imageHeight, imageWidth, imageChannels] = imgDims;\n\n        let paddedImageWidth, paddedImageHeight;\n        if (typeof padSize === 'number') {\n            paddedImageWidth = padSize;\n            paddedImageHeight = padSize;\n        } else {\n            paddedImageWidth = padSize.width;\n            paddedImageHeight = padSize.height;\n        }\n\n        // Only add padding if there is a difference in size\n        if (paddedImageWidth !== imageWidth || paddedImageHeight !== imageHeight) {\n            const paddedPixelData = new Float32Array(paddedImageWidth * paddedImageHeight * imageChannels);\n            if (Array.isArray(constant_values)) {\n                // Fill with constant values, cycling through the array\n                for (let i = 0; i < paddedPixelData.length; ++i) {\n                    paddedPixelData[i] = constant_values[i % imageChannels];\n                }\n            } else if (constant_values !== 0) {\n                paddedPixelData.fill(constant_values);\n            }\n\n            const [left, top] = center\n                ? [Math.floor((paddedImageWidth - imageWidth) / 2), Math.floor((paddedImageHeight - imageHeight) / 2)]\n                : [0, 0];\n\n            // Copy the original image into the padded image\n            for (let i = 0; i < imageHeight; ++i) {\n                const a = (i + top) * paddedImageWidth;\n                const b = i * imageWidth;\n                for (let j = 0; j < imageWidth; ++j) {\n                    const c = (a + j + left) * imageChannels;\n                    const d = (b + j) * imageChannels;\n                    for (let k = 0; k < imageChannels; ++k) {\n                        paddedPixelData[c + k] = pixelData[d + k];\n                    }\n                }\n            }\n\n            if (mode === 'symmetric') {\n                if (center) {\n                    throw new Error('`center` padding is not supported when `mode` is set to `symmetric`.');\n                    // TODO: Implement this\n                }\n                const h1 = imageHeight - 1;\n                const w1 = imageWidth - 1;\n                for (let i = 0; i < paddedImageHeight; ++i) {\n                    const a = i * paddedImageWidth;\n                    const b = calculateReflectOffset(i, h1) * imageWidth;\n\n                    for (let j = 0; j < paddedImageWidth; ++j) {\n                        if (i < imageHeight && j < imageWidth) continue; // Do not overwrite original image\n                        const c = (a + j) * imageChannels;\n                        const d = (b + calculateReflectOffset(j, w1)) * imageChannels;\n\n                        // Copy channel-wise\n                        for (let k = 0; k < imageChannels; ++k) {\n                            paddedPixelData[c + k] = pixelData[d + k];\n                        }\n                    }\n                }\n            }\n\n\n            // Update pixel data and image dimensions\n            pixelData = paddedPixelData;\n            imgDims = [paddedImageHeight, paddedImageWidth, imageChannels]\n        }\n        return [pixelData, imgDims];\n    }\n\n    /**\n     * Rescale the image' pixel values by `this.rescale_factor`.\n     * @param {Float32Array} pixelData The pixel data to rescale.\n     * @returns {void}\n     */\n    rescale(pixelData) {\n        for (let i = 0; i < pixelData.length; ++i) {\n            pixelData[i] = this.rescale_factor * pixelData[i];\n        }\n    }\n\n    /**\n     * Find the target (width, height) dimension of the output image after\n     * resizing given the input image and the desired size.\n     * @param {RawImage} image The image to resize.\n     * @param {any} size The size to use for resizing the image. \n     * @returns {[number, number]} The target (width, height) dimension of the output image after resizing.\n     */\n    get_resize_output_image_size(image, size) {\n        // `size` comes in many forms, so we need to handle them all here:\n        // 1. `size` is an integer, in which case we resize the image to be a square \n\n        const [srcWidth, srcHeight] = image.size;\n\n        let shortest_edge;\n        let longest_edge;\n\n        if (this.do_thumbnail) {\n            // NOTE: custom logic for `Donut` models\n            const { height, width } = size;\n            shortest_edge = Math.min(height, width)\n        }\n        // Support both formats for backwards compatibility\n        else if (Number.isInteger(size)) {\n            shortest_edge = size;\n            longest_edge = this.config.max_size ?? shortest_edge;\n\n        } else if (size !== undefined) {\n            // Extract known properties from `size`\n            shortest_edge = size.shortest_edge;\n            longest_edge = size.longest_edge;\n        }\n\n        // If `longest_edge` and `shortest_edge` are set, maintain aspect ratio and resize to `shortest_edge`\n        // while keeping the largest dimension <= `longest_edge`\n        if (shortest_edge !== undefined || longest_edge !== undefined) {\n            // http://opensourcehacker.com/2011/12/01/calculate-aspect-ratio-conserving-resize-for-images-in-javascript/\n            // Try resize so that shortest edge is `shortest_edge` (target)\n            const shortResizeFactor = shortest_edge === undefined\n                ? 1 // If `shortest_edge` is not set, don't upscale\n                : Math.max(shortest_edge / srcWidth, shortest_edge / srcHeight);\n\n            const newWidth = srcWidth * shortResizeFactor;\n            const newHeight = srcHeight * shortResizeFactor;\n\n            // The new width and height might be greater than `longest_edge`, so\n            // we downscale again to ensure the largest dimension is `longest_edge` \n            const longResizeFactor = longest_edge === undefined\n                ? 1 // If `longest_edge` is not set, don't downscale\n...\n", "needle_token_start": 11894, "needle_token_end": 12027, "code_context_ntokens": 15981, "output": ["```javascript\n/**\n * Helper function to validate audio inputs.\n * @param {any} audio The audio data.\n * @param {string} feature_extractor The name of the feature extractor.\n * @private\n */\nfunction validate_audio_inputs(audio, feature_extractor) {\n    if (!(audio instanceof Float32Array || audio instanceof Float64Array)) {\n        throw new Error(\n            `${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. ` +\n            `If using the feature extractor directly, remember to use \\`read_audio(url, sampling_rate)\\` to obtain the raw audio data of the file/url.`\n        )\n    }\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "permute_data", "language": "typescript", "path": "src/utils/maths.js", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function rearranges the elements of a multi-dimensional array based on specified new axis orders, effectively reorienting the data structure.\n2. **Input**: The function takes three parameters: a multi-dimensional array, its current dimensions, and the new order of axes.\n3. **Output**: It returns a tuple containing the newly arranged array and its new shape.\n4. **Procedure**: The function first calculates the new shape and stride for the array based on the new axis order. It then computes an inverse mapping of the stride to facilitate re-indexing. Using this mapping, it iterates over the original array, calculates the new index for each element in the rearranged array, and assigns the values accordingly.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/core.js\n\n/**\n * @file Core utility functions/classes for Transformers.js.\n * \n * These are only used internally, meaning an end-user shouldn't\n * need to access anything here.\n * \n * @module utils/core\n */\n\n/**\n * Helper function to dispatch progress callbacks.\n *\n * @param {Function} progress_callback The progress callback function to dispatch.\n * @param {any} data The data to pass to the progress callback function.\n * @returns {void}\n * @private\n */\nexport function dispatchCallback(progress_callback, data) {\n    if (progress_callback) progress_callback(data);\n}\n\n/**\n * Reverses the keys and values of an object.\n *\n * @param {Object} data The object to reverse.\n * @returns {Object} The reversed object.\n * @see https://ultimatecourses.com/blog/reverse-object-keys-and-values-in-javascript\n */\nexport function reverseDictionary(data) {\n    // https://ultimatecourses.com/blog/reverse-object-keys-and-values-in-javascript\n    return Object.fromEntries(Object.entries(data).map(([key, value]) => [value, key]));\n}\n\n/**\n * Escapes regular expression special characters from a string by replacing them with their escaped counterparts.\n *\n * @param {string} string The string to escape.\n * @returns {string} The escaped string.\n */\nexport function escapeRegExp(string) {\n    return string.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'); // $& means the whole matched string\n}\n\n/**\n * A base class for creating callable objects.\n * \n * @type {new () => {(...args: any[]): any, _call(...args: any[]): any}}\n */\nexport const Callable = /** @type {any} */ (class {\n    /**\n    * Creates a new instance of the Callable class.\n    */\n    constructor() {\n        /**\n         * Creates a closure that delegates to a private method '_call' with the given arguments.\n         * @type {any}\n         * @param {...any} args Zero or more arguments to pass to the '_call' method.\n         * @returns {*} The result of calling the '_call' method.\n         */\n        let closure = function (...args) {\n            return closure._call(...args)\n        }\n        return Object.setPrototypeOf(closure, new.target.prototype)\n    }\n\n    /**\n     * This method should be implemented in subclasses to provide the\n     * functionality of the callable object.\n     *\n     * @param {any[]} args\n     * @throws {Error} If the subclass does not implement the `_call` method.\n     */\n    _call(...args) {\n        throw Error('Must implement _call method in subclass')\n    }\n});\n\n/**\n * Check if a value is a typed array.\n * @param {*} val The value to check.\n * @returns {boolean} True if the value is a `TypedArray`, false otherwise.\n * \n * Adapted from https://stackoverflow.com/a/71091338/13989043\n */\nexport function isTypedArray(val) {\n    return val?.prototype?.__proto__?.constructor?.name === 'TypedArray';\n}\n\n\n/**\n * Check if a value is an integer.\n * @param {*} x The value to check.\n * @returns {boolean} True if the value is a string, false otherwise.\n */\nexport function isIntegralNumber(x) {\n    return Number.isInteger(x) || typeof x === 'bigint'\n}\n\n/**\n * Check if a value is exists.\n * @param {*} x The value to check.\n * @returns {boolean} True if the value exists, false otherwise.\n */\nexport function exists(x) {\n    return x !== undefined && x !== null;\n}\n\n/**\n * Calculates the dimensions of a nested array.\n *\n * @param {any[]} arr The nested array to calculate dimensions for.\n * @returns {number[]} An array containing the dimensions of the input array.\n */\nexport function calculateDimensions(arr) {\n    const dimensions = [];\n    let current = arr;\n    while (Array.isArray(current)) {\n        dimensions.push(current.length);\n        current = current[0];\n    }\n    return dimensions;\n}\n\n/**\n * Replicate python's .pop() method for objects.\n * @param {Object} obj The object to pop from.\n * @param {string} key The key to pop.\n * @param {*} defaultValue The default value to return if the key does not exist.\n * @returns {*} The value of the popped key.\n * @throws {Error} If the key does not exist and no default value is provided.\n */\nexport function pop(obj, key, defaultValue = undefined) {\n    const value = obj[key];\n    if (value !== undefined) {\n        delete obj[key];\n        return value;\n    }\n    if (defaultValue === undefined) {\n        throw Error(`Key ${key} does not exist in object.`)\n    }\n    return defaultValue;\n}\n\n/**\n * Efficiently merge arrays, creating a new copy.\n * Adapted from https://stackoverflow.com/a/6768642/13989043\n * @param  {Array[]} arrs Arrays to merge.\n * @returns {Array} The merged array.\n */\nexport function mergeArrays(...arrs) {\n    return Array.prototype.concat.apply([], arrs);\n}\n\n/**\n * Compute the Cartesian product of given arrays\n * @param {...Array} a Arrays to compute the product\n * @returns {Array} Returns the computed Cartesian product as an array\n * @private\n */\nexport function product(...a) {\n    // Cartesian product of items\n    // Adapted from https://stackoverflow.com/a/43053803\n    return a.reduce((a, b) => a.flatMap(d => b.map(e => [d, e])));\n}\n\n/**\n * Calculates the index offset for a given index and window size.\n * @param {number} i The index.\n * @param {number} w The window size.\n * @returns {number} The index offset.\n */\nexport function calculateReflectOffset(i, w) {\n    return Math.abs((i + w) % (2 * w) - w);\n}\n\n// Path: src/backends/onnx.js\n/**\n * @file Handler file for choosing the correct version of ONNX Runtime, based on the environment.\n * Ideally, we could import the `onnxruntime-web` and `onnxruntime-node` packages only when needed,\n * but dynamic imports don't seem to work with the current webpack version and/or configuration.\n * This is possibly due to the experimental nature of top-level await statements.\n * So, we just import both packages, and use the appropriate one based on the environment:\n *   - When running in node, we use `onnxruntime-node`.\n *   - When running in the browser, we use `onnxruntime-web` (`onnxruntime-node` is not bundled).\n * \n * This module is not directly exported, but can be accessed through the environment variables:\n * ```javascript\n * import { env } from '@xenova/transformers';\n * console.log(env.backends.onnx);\n * ```\n * \n * @module backends/onnx\n */\n\n// NOTE: Import order matters here. We need to import `onnxruntime-node` before `onnxruntime-web`.\n// In either case, we select the default export if it exists, otherwise we use the named export.\nimport * as ONNX_NODE from 'onnxruntime-node';\nimport * as ONNX_WEB from 'onnxruntime-web';\n\n/** @type {import('onnxruntime-web')} The ONNX runtime module. */\nexport let ONNX;\n\nexport const executionProviders = [\n    // 'webgpu',\n    'wasm'\n];\n\nif (typeof process !== 'undefined' && process?.release?.name === 'node') {\n    // Running in a node-like environment.\n    ONNX = ONNX_NODE.default ?? ONNX_NODE;\n\n    // Add `cpu` execution provider, with higher precedence that `wasm`.\n    executionProviders.unshift('cpu');\n\n} else {\n    // Running in a browser-environment\n    ONNX = ONNX_WEB.default ?? ONNX_WEB;\n\n    // SIMD for WebAssembly does not operate correctly in some recent versions of iOS (16.4.x).\n    // As a temporary fix, we disable it for now.\n    // For more information, see: https://github.com/microsoft/onnxruntime/issues/15644\n    const isIOS = typeof navigator !== 'undefined' && /iP(hone|od|ad).+16_4.+AppleWebKit/.test(navigator.userAgent);\n    if (isIOS) {\n        ONNX.env.wasm.simd = false;\n    }\n}\n\n// Path: src/env.js\n/**\n * @file Module used to configure Transformers.js.\n * \n * **Example:** Disable remote models.\n * ```javascript\n * import { env } from '@xenova/transformers';\n * env.allowRemoteModels = false;\n * ```\n * \n * **Example:** Set local model path.\n * ```javascript\n * import { env } from '@xenova/transformers';\n * env.localModelPath = '/path/to/local/models/';\n * ```\n * \n * **Example:** Set cache directory.\n * ```javascript\n * import { env } from '@xenova/transformers';\n * env.cacheDir = '/path/to/cache/directory/';\n * ```\n * \n * @module env\n */\n\nimport fs from 'fs';\nimport path from 'path';\nimport url from 'url';\n\nimport { ONNX } from './backends/onnx.js';\nconst { env: onnx_env } = ONNX;\n\nconst VERSION = '2.17.0';\n\n// Check if various APIs are available (depends on environment)\nconst WEB_CACHE_AVAILABLE = typeof self !== 'undefined' && 'caches' in self;\nconst FS_AVAILABLE = !isEmpty(fs); // check if file system is available\nconst PATH_AVAILABLE = !isEmpty(path); // check if path is available\n\nconst RUNNING_LOCALLY = FS_AVAILABLE && PATH_AVAILABLE;\n\nconst __dirname = RUNNING_LOCALLY\n    ? path.dirname(path.dirname(url.fileURLToPath(import.meta.url)))\n    : './';\n\n// Only used for environments with access to file system\nconst DEFAULT_CACHE_DIR = RUNNING_LOCALLY\n    ? path.join(__dirname, '/.cache/')\n    : null;\n\n// Set local model path, based on available APIs\nconst DEFAULT_LOCAL_MODEL_PATH = '/models/';\nconst localModelPath = RUNNING_LOCALLY\n    ? path.join(__dirname, DEFAULT_LOCAL_MODEL_PATH)\n    : DEFAULT_LOCAL_MODEL_PATH;\n\nif (onnx_env?.wasm) {\n    // Set path to wasm files. This is needed when running in a web worker.\n    // https://onnxruntime.ai/docs/api/js/interfaces/Env.WebAssemblyFlags.html#wasmPaths\n    // We use remote wasm files by default to make it easier for newer users.\n    // In practice, users should probably self-host the necessary .wasm files.\n    onnx_env.wasm.wasmPaths = RUNNING_LOCALLY\n        ? path.join(__dirname, '/dist/')\n        : `https://cdn.jsdelivr.net/npm/@xenova/transformers@${VERSION}/dist/`;\n}\n\n/**\n * Global variable used to control execution. This provides users a simple way to configure Transformers.js.\n * @property {Object} backends Expose environment variables of different backends,\n * allowing users to set these variables if they want to.\n * @property {string} __dirname Directory name of module. Useful for resolving local paths.\n * @property {string} version This version of Transformers.js.\n * @property {boolean} allowRemoteModels Whether to allow loading of remote files, defaults to `true`.\n * If set to `false`, it will have the same effect as setting `local_files_only=true` when loading pipelines, models, tokenizers, processors, etc.\n * @property {string} remoteHost Host URL to load models from. Defaults to the Hugging Face Hub.\n * @property {string} remotePathTemplate Path template to fill in and append to `remoteHost` when loading models.\n * @property {boolean} allowLocalModels Whether to allow loading of local files, defaults to `true`.\n * If set to `false`, it will skip the local file check and try to load the model from the remote host.\n * @property {string} localModelPath Path to load local models from. Defaults to `/models/`.\n * @property {boolean} useFS Whether to use the file system to load files. By default, it is `true` if available.\n * @property {boolean} useBrowserCache Whether to use Cache API to cache models. By default, it is `true` if available.\n * @property {boolean} useFSCache Whether to use the file system to cache files. By default, it is `true` if available.\n * @property {string} cacheDir The directory to use for caching files with the file system. By default, it is `./.cache`.\n * @property {boolean} useCustomCache Whether to use a custom cache system (defined by `customCache`), defaults to `false`.\n * @property {Object} customCache The custom cache to use. Defaults to `null`. Note: this must be an object which\n * implements the `match` and `put` functions of the Web Cache API. For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache\n */\nexport const env = {\n    /////////////////// Backends settings ///////////////////\n    backends: {\n        // onnxruntime-web/onnxruntime-node\n        onnx: onnx_env,\n\n        // TensorFlow.js\n        tfjs: {},\n    },\n\n    __dirname,\n    version: VERSION,\n\n    /////////////////// Model settings ///////////////////\n    allowRemoteModels: true,\n    remoteHost: 'https://huggingface.co/',\n    remotePathTemplate: '{model}/resolve/{revision}/',\n\n    allowLocalModels: true,\n    localModelPath: localModelPath,\n    useFS: FS_AVAILABLE,\n\n    /////////////////// Cache settings ///////////////////\n    useBrowserCache: WEB_CACHE_AVAILABLE,\n\n    useFSCache: FS_AVAILABLE,\n    cacheDir: DEFAULT_CACHE_DIR,\n\n    useCustomCache: false,\n    customCache: null,\n    //////////////////////////////////////////////////////\n}\n\n\n/**\n * @param {Object} obj\n * @private\n */\nfunction isEmpty(obj) {\n    return Object.keys(obj).length === 0;\n}\n\n\n// Path: src/utils/hub.js\n\n/**\n * @file Utility functions to interact with the Hugging Face Hub (https://huggingface.co/models)\n * \n * @module utils/hub\n */\n\nimport fs from 'fs';\nimport path from 'path';\nimport stream from 'stream/web';\n\nimport { env } from '../env.js';\nimport { dispatchCallback } from './core.js';\n\nif (!globalThis.ReadableStream) {\n    // @ts-ignore\n    globalThis.ReadableStream = stream.ReadableStream; // ReadableStream is not a global with Node 16\n}\n\n/**\n * @typedef {Object} PretrainedOptions Options for loading a pretrained model.     \n * @property {boolean?} [quantized=true] Whether to load the 8-bit quantized version of the model (only applicable when loading model files).\n * @property {function} [progress_callback=null] If specified, this function will be called during model construction, to provide the user with progress updates.\n * @property {Object} [config=null] Configuration for the model to use instead of an automatically loaded configuration. Configuration can be automatically loaded when:\n * - The model is a model provided by the library (loaded with the *model id* string of a pretrained model).\n * - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a configuration JSON file named *config.json* is found in the directory.\n * @property {string} [cache_dir=null] Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used.\n * @property {boolean} [local_files_only=false] Whether or not to only look at local files (e.g., not try downloading the model).\n * @property {string} [revision='main'] The specific model version to use. It can be a branch name, a tag name, or a commit id,\n * since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git.\n * NOTE: This setting is ignored for local requests.\n * @property {string} [model_file_name=null] If specified, load the model with this name (excluding the .onnx suffix). Currently only valid for encoder- or decoder-only models.\n */\n\nclass FileResponse {\n    /**\n     * Mapping from file extensions to MIME types.\n     */\n    _CONTENT_TYPE_MAP = {\n        'txt': 'text/plain',\n        'html': 'text/html',\n        'css': 'text/css',\n        'js': 'text/javascript',\n        'json': 'application/json',\n        'png': 'image/png',\n        'jpg': 'image/jpeg',\n        'jpeg': 'image/jpeg',\n        'gif': 'image/gif',\n    }\n    /**\n     * Creates a new `FileResponse` object.\n     * @param {string|URL} filePath\n     */\n    constructor(filePath) {\n        this.filePath = filePath;\n        this.headers = new Headers();\n\n        this.exists = fs.existsSync(filePath);\n        if (this.exists) {\n            this.status = 200;\n            this.statusText = 'OK';\n\n            let stats = fs.statSync(filePath);\n            this.headers.set('content-length', stats.size.toString());\n\n            this.updateContentType();\n\n            let self = this;\n            this.body = new ReadableStream({\n                start(controller) {\n                    self.arrayBuffer().then(buffer => {\n                        controller.enqueue(new Uint8Array(buffer));\n                        controller.close();\n                    })\n                }\n            });\n        } else {\n            this.status = 404;\n            this.statusText = 'Not Found';\n            this.body = null;\n        }\n    }\n\n    /**\n     * Updates the 'content-type' header property of the response based on the extension of\n     * the file specified by the filePath property of the current object.\n     * @returns {void}\n     */\n    updateContentType() {\n        // Set content-type header based on file extension\n        const extension = this.filePath.toString().split('.').pop().toLowerCase();\n        this.headers.set('content-type', this._CONTENT_TYPE_MAP[extension] ?? 'application/octet-stream');\n    }\n\n    /**\n     * Clone the current FileResponse object.\n     * @returns {FileResponse} A new FileResponse object with the same properties as the current object.\n     */\n    clone() {\n        let response = new FileResponse(this.filePath);\n        response.exists = this.exists;\n        response.status = this.status;\n        response.statusText = this.statusText;\n        response.headers = new Headers(this.headers);\n        return response;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with an ArrayBuffer containing the file's contents.\n     * @returns {Promise<ArrayBuffer>} A Promise that resolves with an ArrayBuffer containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async arrayBuffer() {\n        const data = await fs.promises.readFile(this.filePath);\n        return data.buffer;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a Blob containing the file's contents.\n     * @returns {Promise<Blob>} A Promise that resolves with a Blob containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async blob() {\n        const data = await fs.promises.readFile(this.filePath);\n        return new Blob([data], { type: this.headers.get('content-type') });\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a string containing the file's contents.\n     * @returns {Promise<string>} A Promise that resolves with a string containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async text() {\n        const data = await fs.promises.readFile(this.filePath, 'utf8');\n        return data;\n    }\n\n    /**\n     * Reads the contents of the file specified by the filePath property and returns a Promise that\n     * resolves with a parsed JavaScript object containing the file's contents.\n     * \n     * @returns {Promise<Object>} A Promise that resolves with a parsed JavaScript object containing the file's contents.\n     * @throws {Error} If the file cannot be read.\n     */\n    async json() {\n        return JSON.parse(await this.text());\n    }\n}\n\n/**\n * Determines whether the given string is a valid HTTP or HTTPS URL.\n * @param {string|URL} string The string to test for validity as an HTTP or HTTPS URL.\n * @param {string[]} [validHosts=null] A list of valid hostnames. If specified, the URL's hostname must be in this list.\n * @returns {boolean} True if the string is a valid HTTP or HTTPS URL, false otherwise.\n */\nfunction isValidHttpUrl(string, validHosts = null) {\n    // https://stackoverflow.com/a/43467144\n    let url;\n    try {\n        url = new URL(string);\n    } catch (_) {\n        return false;\n    }\n    if (validHosts && !validHosts.includes(url.hostname)) {\n        return false;\n    }\n    return url.protocol === \"http:\" || url.protocol === \"https:\";\n}\n\n/**\n * Helper function to get a file, using either the Fetch API or FileSystem API.\n *\n * @param {URL|string} urlOrPath The URL/path of the file to get.\n * @returns {Promise<FileResponse|Response>} A promise that resolves to a FileResponse object (if the file is retrieved using the FileSystem API), or a Response object (if the file is retrieved using the Fetch API).\n */\nexport async function getFile(urlOrPath) {\n\n    if (env.useFS && !isValidHttpUrl(urlOrPath)) {\n        return new FileResponse(urlOrPath);\n\n    } else if (typeof process !== 'undefined' && process?.release?.name === 'node') {\n        const IS_CI = !!process.env?.TESTING_REMOTELY;\n        const version = env.version;\n\n        const headers = new Headers();\n        headers.set('User-Agent', `transformers.js/${version}; is_ci/${IS_CI};`);\n\n        // Check whether we are making a request to the Hugging Face Hub.\n        const isHFURL = isValidHttpUrl(urlOrPath, ['huggingface.co', 'hf.co']);\n        if (isHFURL) {\n            // If an access token is present in the environment variables,\n            // we add it to the request headers.\n            // NOTE: We keep `HF_ACCESS_TOKEN` for backwards compatibility (as a fallback).\n            const token = process.env?.HF_TOKEN ?? process.env?.HF_ACCESS_TOKEN;\n            if (token) {\n                headers.set('Authorization', `Bearer ${token}`);\n            }\n        }\n        return fetch(urlOrPath, { headers });\n    } else {\n        // Running in a browser-environment, so we use default headers\n        // NOTE: We do not allow passing authorization headers in the browser,\n        // since this would require exposing the token to the client.\n        return fetch(urlOrPath);\n    }\n}\n\nconst ERROR_MAPPING = {\n    // 4xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#client_error_responses)\n    400: 'Bad request error occurred while trying to load file',\n    401: 'Unauthorized access to file',\n    403: 'Forbidden access to file',\n    404: 'Could not locate file',\n    408: 'Request timeout error occurred while trying to load file',\n\n    // 5xx errors (https://developer.mozilla.org/en-US/docs/Web/HTTP/Status#server_error_responses)\n    500: 'Internal server error error occurred while trying to load file',\n    502: 'Bad gateway error occurred while trying to load file',\n    503: 'Service unavailable error occurred while trying to load file',\n    504: 'Gateway timeout error occurred while trying to load file',\n}\n/**\n * Helper method to handle fatal errors that occur while trying to load a file from the Hugging Face Hub.\n * @param {number} status The HTTP status code of the error.\n * @param {string} remoteURL The URL of the file that could not be loaded.\n * @param {boolean} fatal Whether to raise an error if the file could not be loaded.\n * @returns {null} Returns `null` if `fatal = true`.\n * @throws {Error} If `fatal = false`.\n */\nfunction handleError(status, remoteURL, fatal) {\n    if (!fatal) {\n        // File was not loaded correctly, but it is optional.\n        // TODO in future, cache the response?\n        return null;\n    }\n\n    const message = ERROR_MAPPING[status] ?? `Error (${status}) occurred while trying to load file`;\n    throw Error(`${message}: \"${remoteURL}\".`);\n}\n\nclass FileCache {\n    /**\n     * Instantiate a `FileCache` object.\n     * @param {string} path \n     */\n    constructor(path) {\n        this.path = path;\n    }\n\n    /**\n     * Checks whether the given request is in the cache.\n     * @param {string} request \n     * @returns {Promise<FileResponse | undefined>}\n     */\n    async match(request) {\n\n        let filePath = path.join(this.path, request);\n        let file = new FileResponse(filePath);\n\n        if (file.exists) {\n            return file;\n        } else {\n            return undefined;\n        }\n    }\n\n    /**\n     * Adds the given response to the cache.\n     * @param {string} request \n     * @param {Response|FileResponse} response \n     * @returns {Promise<void>}\n     */\n    async put(request, response) {\n        const buffer = Buffer.from(await response.arrayBuffer());\n\n        let outputPath = path.join(this.path, request);\n\n        try {\n            await fs.promises.mkdir(path.dirname(outputPath), { recursive: true });\n            await fs.promises.writeFile(outputPath, buffer);\n\n        } catch (err) {\n            console.warn('An error occurred while writing the file to cache:', err)\n        }\n    }\n\n    // TODO add the rest?\n    // addAll(requests: RequestInfo[]): Promise<void>;\n    // delete(request: RequestInfo | URL, options?: CacheQueryOptions): Promise<boolean>;\n    // keys(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise<ReadonlyArray<Request>>;\n    // match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise<Response | undefined>;\n    // matchAll(request?: RequestInfo | URL, options?: CacheQueryOptions): Promise<ReadonlyArray<Response>>;\n}\n\n/**\n * \n * @param {FileCache|Cache} cache The cache to search\n * @param {string[]} names The names of the item to search for\n * @returns {Promise<FileResponse|Response|undefined>} The item from the cache, or undefined if not found.\n */\nasync function tryCache(cache, ...names) {\n    for (let name of names) {\n        try {\n            let result = await cache.match(name);\n            if (result) return result;\n        } catch (e) {\n            continue;\n        }\n    }\n    return undefined;\n}\n\n/**\n * \n * Retrieves a file from either a remote URL using the Fetch API or from the local file system using the FileSystem API.\n * If the filesystem is available and `env.useCache = true`, the file will be downloaded and cached.\n * \n * @param {string} path_or_repo_id This can be either:\n * - a string, the *model id* of a model repo on huggingface.co.\n * - a path to a *directory* potentially containing the file.\n * @param {string} filename The name of the file to locate in `path_or_repo`.\n * @param {boolean} [fatal=true] Whether to throw an error if the file is not found.\n * @param {PretrainedOptions} [options] An object containing optional parameters.\n * \n * @throws Will throw an error if the file is not found and `fatal` is true.\n * @returns {Promise} A Promise that resolves with the file content as a buffer.\n */\nexport async function getModelFile(path_or_repo_id, filename, fatal = true, options = {}) {\n\n    if (!env.allowLocalModels) {\n        // User has disabled local models, so we just make sure other settings are correct.\n\n        if (options.local_files_only) {\n            throw Error(\"Invalid configuration detected: local models are disabled (`env.allowLocalModels=false`) but you have requested to only use local models (`local_files_only=true`).\")\n        } else if (!env.allowRemoteModels) {\n            throw Error(\"Invalid configuration detected: both local and remote models are disabled. Fix by setting `env.allowLocalModels` or `env.allowRemoteModels` to `true`.\")\n        }\n    }\n\n    // Initiate file retrieval\n    dispatchCallback(options.progress_callback, {\n        status: 'initiate',\n        name: path_or_repo_id,\n        file: filename\n    })\n\n    // First, check if the a caching backend is available\n    // If no caching mechanism available, will download the file every time\n    let cache;\n    if (!cache && env.useBrowserCache) {\n        if (typeof caches === 'undefined') {\n            throw Error('Browser cache is not available in this environment.')\n        }\n        try {\n            // In some cases, the browser cache may be visible, but not accessible due to security restrictions.\n            // For example, when running an application in an iframe, if a user attempts to load the page in\n            // incognito mode, the following error is thrown: `DOMException: Failed to execute 'open' on 'CacheStorage':\n            // An attempt was made to break through the security policy of the user agent.`\n            // So, instead of crashing, we just ignore the error and continue without using the cache.\n            cache = await caches.open('transformers-cache');\n        } catch (e) {\n            console.warn('An error occurred while opening the browser cache:', e);\n        }\n    }\n\n    if (!cache && env.useFSCache) {\n        // TODO throw error if not available\n\n        // If `cache_dir` is not specified, use the default cache directory\n        cache = new FileCache(options.cache_dir ?? env.cacheDir);\n    }\n\n    if (!cache && env.useCustomCache) {\n        // Allow the user to specify a custom cache system.\n        if (!env.customCache) {\n            throw Error('`env.useCustomCache=true`, but `env.customCache` is not defined.')\n        }\n\n        // Check that the required methods are defined:\n        if (!env.customCache.match || !env.customCache.put) {\n            throw new Error(\n                \"`env.customCache` must be an object which implements the `match` and `put` functions of the Web Cache API. \" +\n                \"For more information, see https://developer.mozilla.org/en-US/docs/Web/API/Cache\"\n            )\n        }\n        cache = env.customCache;\n    }\n\n    const revision = options.revision ?? 'main';\n\n    let requestURL = pathJoin(path_or_repo_id, filename);\n    let localPath = pathJoin(env.localModelPath, requestURL);\n\n    let remoteURL = pathJoin(\n        env.remoteHost,\n        env.remotePathTemplate\n            .replaceAll('{model}', path_or_repo_id)\n            .replaceAll('{revision}', encodeURIComponent(revision)),\n        filename\n    );\n\n    // Choose cache key for filesystem cache\n    // When using the main revision (default), we use the request URL as the cache key.\n    // If a specific revision is requested, we account for this in the cache key.\n    let fsCacheKey = revision === 'main' ? requestURL : pathJoin(path_or_repo_id, revision, filename);\n\n    /** @type {string} */\n    let cacheKey;\n    let proposedCacheKey = cache instanceof FileCache ? fsCacheKey : remoteURL;\n\n    // Whether to cache the final response in the end.\n    let toCacheResponse = false;\n\n    /** @type {Response|FileResponse|undefined} */\n    let response;\n\n    if (cache) {\n        // A caching system is available, so we try to get the file from it.\n        //  1. We first try to get from cache using the local path. In some environments (like deno),\n        //     non-URL cache keys are not allowed. In these cases, `response` will be undefined.\n        //  2. If no response is found, we try to get from cache using the remote URL or file system cache.\n        response = await tryCache(cache, localPath, proposedCacheKey);\n    }\n\n    const cacheHit = response !== undefined;\n\n    if (response === undefined) {\n        // Caching not available, or file is not cached, so we perform the request\n\n        if (env.allowLocalModels) {\n            // Accessing local models is enabled, so we try to get the file locally.\n            // If request is a valid HTTP URL, we skip the local file check. Otherwise, we try to get the file locally.\n            const isURL = isValidHttpUrl(requestURL);\n            if (!isURL) {\n                try {\n                    response = await getFile(localPath);\n                    cacheKey = localPath; // Update the cache key to be the local path\n                } catch (e) {\n                    // Something went wrong while trying to get the file locally.\n                    // NOTE: error handling is done in the next step (since `response` will be undefined)\n                    console.warn(`Unable to load from local path \"${localPath}\": \"${e}\"`);\n                }\n            } else if (options.local_files_only) {\n                throw new Error(`\\`local_files_only=true\\`, but attempted to load a remote file from: ${requestURL}.`);\n            } else if (!env.allowRemoteModels) {\n                throw new Error(`\\`env.allowRemoteModels=false\\`, but attempted to load a remote file from: ${requestURL}.`);\n            }\n        }\n\n        if (response === undefined || response.status === 404) {\n            // File not found locally. This means either:\n            // - The user has disabled local file access (`env.allowLocalModels=false`)\n            // - the path is a valid HTTP url (`response === undefined`)\n            // - the path is not a valid HTTP url and the file is not present on the file system or local server (`response.status === 404`)\n\n            if (options.local_files_only || !env.allowRemoteModels) {\n                // User requested local files only, but the file is not found locally.\n                if (fatal) {\n                    throw Error(`\\`local_files_only=true\\` or \\`env.allowRemoteModels=false\\` and file was not found locally at \"${localPath}\".`);\n                } else {\n                    // File not found, but this file is optional.\n                    // TODO in future, cache the response?\n                    return null;\n                }\n            }\n\n            // File not found locally, so we try to download it from the remote server\n            response = await getFile(remoteURL);\n\n            if (response.status !== 200) {\n                return handleError(response.status, remoteURL, fatal);\n            }\n\n            // Success! We use the proposed cache key from earlier\n            cacheKey = proposedCacheKey;\n        }\n\n        // Only cache the response if:\n        toCacheResponse =\n            cache                              // 1. A caching system is available\n            && typeof Response !== 'undefined' // 2. `Response` is defined (i.e., we are in a browser-like environment)\n            && response instanceof Response    // 3. result is a `Response` object (i.e., not a `FileResponse`)\n            && response.status === 200         // 4. request was successful (status code 200)\n    }\n\n    // Start downloading\n    dispatchCallback(options.progress_callback, {\n        status: 'download',\n        name: path_or_repo_id,\n        file: filename\n    })\n\n    const progressInfo = {\n        status: 'progress',\n        name: path_or_repo_id,\n        file: filename\n    }\n\n    /** @type {Uint8Array} */\n    let buffer;\n\n    if (!options.progress_callback) {\n        // If no progress callback is specified, we can use the `.arrayBuffer()`\n        // method to read the response.\n        buffer = new Uint8Array(await response.arrayBuffer());\n\n    } else if (\n        cacheHit // The item is being read from the cache\n        &&\n        typeof navigator !== 'undefined' && /firefox/i.test(navigator.userAgent) // We are in Firefox\n    ) {\n        // Due to bug in Firefox, we cannot display progress when loading from cache.\n        // Fortunately, since this should be instantaneous, this should not impact users too much.\n        buffer = new Uint8Array(await response.arrayBuffer());\n\n        // For completeness, we still fire the final progress callback\n        dispatchCallback(options.progress_callback, {\n            ...progressInfo,\n            progress: 100,\n            loaded: buffer.length,\n            total: buffer.length,\n        })\n    } else {\n        buffer = await readResponse(response, data => {\n            dispatchCallback(options.progress_callback, {\n                ...progressInfo,\n                ...data,\n            })\n        })\n    }\n\n    if (\n        // Only cache web responses\n        // i.e., do not cache FileResponses (prevents duplication)\n        toCacheResponse && cacheKey\n        &&\n        // Check again whether request is in cache. If not, we add the response to the cache\n        (await cache.match(cacheKey) === undefined)\n    ) {\n        // NOTE: We use `new Response(buffer, ...)` instead of `response.clone()` to handle LFS files\n        await cache.put(cacheKey, new Response(buffer, {\n            headers: response.headers\n        }))\n            .catch(err => {\n                // Do not crash if unable to add to cache (e.g., QuotaExceededError).\n                // Rather, log a warning and proceed with execution.\n                console.warn(`Unable to add response to browser cache: ${err}.`);\n            });\n\n    }\n\n    dispatchCallback(options.progress_callback, {\n        status: 'done',\n        name: path_or_repo_id,\n        file: filename\n    });\n\n    return buffer;\n}\n\n/**\n * Fetches a JSON file from a given path and file name.\n *\n * @param {string} modelPath The path to the directory containing the file.\n * @param {string} fileName The name of the file to fetch.\n * @param {boolean} [fatal=true] Whether to throw an error if the file is not found.\n * @param {PretrainedOptions} [options] An object containing optional parameters.\n * @returns {Promise<Object>} The JSON data parsed into a JavaScript object.\n * @throws Will throw an error if the file is not found and `fatal` is true.\n */\nexport async function getModelJSON(modelPath, fileName, fatal = true, options = {}) {\n    let buffer = await getModelFile(modelPath, fileName, fatal, options);\n    if (buffer === null) {\n        // Return empty object\n        return {}\n    }\n\n    let decoder = new TextDecoder('utf-8');\n    let jsonData = decoder.decode(buffer);\n\n    return JSON.parse(jsonData);\n}\n\n/**\n * Read and track progress when reading a Response object\n *\n * @param {any} response The Response object to read\n * @param {function} progress_callback The function to call with progress updates\n * @returns {Promise<Uint8Array>} A Promise that resolves with the Uint8Array buffer\n */\nasync function readResponse(response, progress_callback) {\n\n    const contentLength = response.headers.get('Content-Length');\n    if (contentLength === null) {\n        console.warn('Unable to determine content-length from response headers. Will expand buffer when needed.')\n    }\n    let total = parseInt(contentLength ?? '0');\n    let buffer = new Uint8Array(total);\n    let loaded = 0;\n\n    const reader = response.body.getReader();\n    async function read() {\n        const { done, value } = await reader.read();\n        if (done) return;\n\n        let newLoaded = loaded + value.length;\n        if (newLoaded > total) {\n            total = newLoaded;\n\n            // Adding the new data will overflow buffer.\n            // In this case, we extend the buffer\n            let newBuffer = new Uint8Array(total);\n\n            // copy contents\n            newBuffer.set(buffer);\n\n            buffer = newBuffer;\n        }\n        buffer.set(value, loaded)\n        loaded = newLoaded;\n\n        const progress = (loaded / total) * 100;\n\n        // Call your function here\n        progress_callback({\n            progress: progress,\n            loaded: loaded,\n            total: total,\n        })\n\n        return read();\n    }\n\n    // Actually read\n    await read();\n\n    return buffer;\n}\n\n/**\n * Joins multiple parts of a path into a single path, while handling leading and trailing slashes.\n *\n * @param {...string} parts Multiple parts of a path.\n * @returns {string} A string representing the joined path.\n */\nfunction pathJoin(...parts) {\n    // https://stackoverflow.com/a/55142565\n    parts = parts.map((part, index) => {\n        if (index) {\n            part = part.replace(new RegExp('^/'), '');\n        }\n        if (index !== parts.length - 1) {\n            part = part.replace(new RegExp('/$'), '');\n        }\n        return part;\n    })\n    return parts.join('/');\n}\n\n// Path: src/configs.js\n\n/**\n * @file Helper module for using model configs. For more information, see the corresponding\n * [Python documentation](https://huggingface.co/docs/transformers/main/en/model_doc/auto#transformers.AutoConfig).\n * \n * **Example:** Load an `AutoConfig`.\n * \n * ```javascript\n * import { AutoConfig } from '@xenova/transformers';\n * let config = await AutoConfig.from_pretrained('bert-base-uncased');\n * console.log(config);\n * // PretrainedConfig {\n * //   \"model_type\": \"bert\",\n * //   \"is_encoder_decoder\": false,\n * //   \"architectures\": [\n * //       \"BertForMaskedLM\"\n * //   ],\n * //   \"vocab_size\": 30522\n * //   \"num_attention_heads\": 12,\n * //   \"num_hidden_layers\": 12,\n * //   \"hidden_size\": 768,\n * //   \"max_position_embeddings\": 512,\n * //   ...\n * // }\n * ```\n * \n * @module configs\n */\n\nimport {\n    getModelJSON,\n} from './utils/hub.js';\n\n/**\n * @typedef {import('./utils/hub.js').PretrainedOptions} PretrainedOptions\n */\n\n\n/**\n * Loads a config from the specified path.\n * @param {string} pretrained_model_name_or_path The path to the config directory.\n * @param {PretrainedOptions} options Additional options for loading the config.\n * @returns {Promise<Array>} A promise that resolves with information about the loaded config.\n */\nasync function loadConfig(pretrained_model_name_or_path, options) {\n    let info = await getModelJSON(pretrained_model_name_or_path, 'config.json', true, options);\n    return info;\n}\n\n/**\n * Base class for all configuration classes. For more information, see the corresponding\n * [Python documentation](https://huggingface.co/docs/transformers/main/en/main_classes/configuration#transformers.PretrainedConfig).\n */\nexport class PretrainedConfig {\n    // NOTE: Typo in original\n\n    /**\n     * Create a new PreTrainedTokenizer instance.\n     * @param {Object} configJSON The JSON of the config.\n     */\n    constructor(configJSON) {\n        this.model_type = null;\n        this.is_encoder_decoder = false;\n\n        Object.assign(this, configJSON);\n    }\n\n    /**\n     * Loads a pre-trained config from the given `pretrained_model_name_or_path`. \n     * \n     * @param {string} pretrained_model_name_or_path The path to the pre-trained config.\n     * @param {PretrainedOptions} options Additional options for loading the config.\n     * @throws {Error} Throws an error if the config.json is not found in the `pretrained_model_name_or_path`.\n     * \n     * @returns {Promise<PretrainedConfig>} A new instance of the `PretrainedConfig` class.\n     */\n    static async from_pretrained(pretrained_model_name_or_path, {\n        progress_callback = null,\n        config = null,\n        cache_dir = null,\n        local_files_only = false,\n        revision = 'main',\n    } = {}) {\n\n        let data = config ?? await loadConfig(pretrained_model_name_or_path, {\n            progress_callback,\n            config,\n            cache_dir,\n            local_files_only,\n            revision,\n        })\n        return new this(data);\n    }\n}\n\n/**\n * Helper class which is used to instantiate pretrained configs with the `from_pretrained` function.\n * \n * @example\n * let config = await AutoConfig.from_pretrained('bert-base-uncased'); \n */\nexport class AutoConfig {\n    /** @type {PretrainedConfig.from_pretrained} */\n    static async from_pretrained(...args) {\n        return PretrainedConfig.from_pretrained(...args);\n    }\n}\n\n// Path: src/utils/maths.js\n\n/**\n * @file Helper module for mathematical processing. \n * \n * These functions and classes are only used internally, \n * meaning an end-user shouldn't need to access anything here.\n * \n * @module utils/maths\n */\n\n/**\n * @typedef {Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array} TypedArray\n * @typedef {BigInt64Array | BigUint64Array} BigTypedArray\n * @typedef {TypedArray | BigTypedArray} AnyTypedArray\n */\n\n/**\n * @param {TypedArray} input\n */\nexport function interpolate_data(input, [in_channels, in_height, in_width], [out_height, out_width], mode = 'bilinear', align_corners = false) {\n    // TODO use mode and align_corners\n\n    // Output image dimensions\n    const x_scale = out_width / in_width;\n    const y_scale = out_height / in_height;\n\n    // Output image\n    // @ts-ignore\n    const out_img = new input.constructor(out_height * out_width * in_channels);\n\n    // Pre-calculate strides\n    const inStride = in_height * in_width;\n    const outStride = out_height * out_width;\n\n    for (let i = 0; i < out_height; ++i) {\n        for (let j = 0; j < out_width; ++j) {\n            // Calculate output offset\n            const outOffset = i * out_width + j;\n\n            // Calculate input pixel coordinates\n            const x = (j + 0.5) / x_scale - 0.5;\n            const y = (i + 0.5) / y_scale - 0.5;\n\n            // Calculate the four nearest input pixels\n            // We also check if the input pixel coordinates are within the image bounds\n            let x1 = Math.floor(x);\n            let y1 = Math.floor(y);\n            const x2 = Math.min(x1 + 1, in_width - 1);\n            const y2 = Math.min(y1 + 1, in_height - 1);\n\n            x1 = Math.max(x1, 0);\n            y1 = Math.max(y1, 0);\n\n\n            // Calculate the fractional distances between the input pixel and the four nearest pixels\n            const s = x - x1;\n            const t = y - y1;\n\n            // Perform bilinear interpolation\n            const w1 = (1 - s) * (1 - t);\n            const w2 = s * (1 - t);\n            const w3 = (1 - s) * t;\n            const w4 = s * t;\n\n            // Calculate the four nearest input pixel indices\n            const yStride = y1 * in_width;\n            const xStride = y2 * in_width;\n            const idx1 = yStride + x1;\n            const idx2 = yStride + x2;\n            const idx3 = xStride + x1;\n            const idx4 = xStride + x2;\n\n            for (let k = 0; k < in_channels; ++k) {\n                // Calculate channel offset\n                const cOffset = k * inStride;\n\n                out_img[k * outStride + outOffset] =\n                    w1 * input[cOffset + idx1] +\n                    w2 * input[cOffset + idx2] +\n                    w3 * input[cOffset + idx3] +\n                    w4 * input[cOffset + idx4];\n            }\n        }\n    }\n\n    return out_img;\n}\n\n\n/**\n * Helper method to permute a `AnyTypedArray` directly\n * @template {AnyTypedArray} T \n * @param {T} array \n * @param {number[]} dims \n * @param {number[]} axes \n * @returns {[T, number[]]} The permuted array and the new shape.\n */\nexport \nfunction permute_data(array, dims, axes) {\n    // Calculate the new shape of the permuted array\n    // and the stride of the original array\n    const shape = new Array(axes.length);\n    const stride = new Array(axes.length);\n\n    for (let i = axes.length - 1, s = 1; i >= 0; --i) {\n        stride[i] = s;\n        shape[i] = dims[axes[i]];\n        s *= shape[i];\n    }\n\n    // Precompute inverse mapping of stride\n    const invStride = axes.map((_, i) => stride[axes.indexOf(i)]);\n\n    // Create the permuted array with the new shape\n    // @ts-ignore\n    const permutedData = new array.constructor(array.length);\n\n    // Permute the original array to the new array\n    for (let i = 0; i < array.length; ++i) {\n        let newIndex = 0;\n        for (let j = dims.length - 1, k = i; j >= 0; --j) {\n            newIndex += (k % dims[j]) * invStride[j];\n            k = Math.floor(k / dims[j]);\n        }\n        permutedData[newIndex] = array[i];\n    }\n\n    return [permutedData, shape];\n}\n\n\n/**\n * Compute the softmax of an array of numbers.\n * @template {TypedArray|number[]} T\n * @param {T} arr The array of numbers to compute the softmax of.\n * @returns {T} The softmax array.\n */\nexport function softmax(arr) {\n    // Compute the maximum value in the array\n    const maxVal = max(arr)[0];\n\n    // Compute the exponentials of the array values\n    const exps = arr.map(x => Math.exp(x - maxVal));\n\n    // Compute the sum of the exponentials\n    // @ts-ignore\n    const sumExps = exps.reduce((acc, val) => acc + val, 0);\n\n    // Compute the softmax values\n    const softmaxArr = exps.map(x => x / sumExps);\n\n    return /** @type {T} */(softmaxArr);\n}\n\n/**\n * Calculates the logarithm of the softmax function for the input array.\n * @template {TypedArray|number[]} T\n * @param {T} arr The input array to calculate the log_softmax function for.\n * @returns {T} The resulting log_softmax array.\n */\nexport function log_softmax(arr) {\n    // Compute the softmax values\n    const softmaxArr = softmax(arr);\n\n    // Apply log formula to each element\n    const logSoftmaxArr = softmaxArr.map(x => Math.log(x));\n\n    return /** @type {T} */(logSoftmaxArr);\n}\n\n/**\n * Calculates the dot product of two arrays.\n * @param {number[]} arr1 The first array.\n * @param {number[]} arr2 The second array.\n * @returns {number} The dot product of arr1 and arr2.\n */\nexport function dot(arr1, arr2) {\n    let result = 0;\n    for (let i = 0; i < arr1.length; ++i) {\n        result += arr1[i] * arr2[i];\n    }\n    return result;\n}\n\n\n/**\n * Get the top k items from an iterable, sorted by descending order\n * @param {any[]|TypedArray} items The items to be sorted\n * @param {number|null} [top_k=0] The number of top items to return (default: 0 = return all)\n * @returns {[number, any][]} The top k items, sorted by descending order\n */\nexport function getTopItems(items, top_k = 0) {\n    // if top == 0, return all\n\n    items = Array.from(items)\n        .map((x, i) => [i, x])            // Get indices ([index, score])\n        .sort((a, b) => b[1] - a[1])      // Sort by log probabilities\n\n    if (top_k !== null && top_k > 0) {\n        items = items.slice(0, top_k);    // Get top k items\n    }\n\n    return items\n}\n\n/**\n * Computes the cosine similarity between two arrays.\n *\n * @param {number[]} arr1 The first array.\n * @param {number[]} arr2 The second array.\n * @returns {number} The cosine similarity between the two arrays.\n */\nexport function cos_sim(arr1, arr2) {\n    // Calculate dot product of the two arrays\n    const dotProduct = dot(arr1, arr2);\n\n    // Calculate the magnitude of the first array\n    const magnitudeA = magnitude(arr1);\n\n    // Calculate the magnitude of the second array\n    const magnitudeB = magnitude(arr2);\n\n    // Calculate the cosine similarity\n    const cosineSimilarity = dotProduct / (magnitudeA * magnitudeB);\n\n    return cosineSimilarity;\n}\n\n/**\n * Calculates the magnitude of a given array.\n * @param {number[]} arr The array to calculate the magnitude of.\n * @returns {number} The magnitude of the array.\n */\nexport function magnitude(arr) {\n    return Math.sqrt(arr.reduce((acc, val) => acc + val * val, 0));\n}\n\n\n/**\n * Returns the value and index of the minimum element in an array.\n * @param {number[]|TypedArray} arr array of numbers.\n * @returns {number[]} the value and index of the minimum element, of the form: [valueOfMin, indexOfMin]\n * @throws {Error} If array is empty.\n */\nexport function min(arr) {\n    if (arr.length === 0) throw Error('Array must not be empty');\n    let min = arr[0];\n    let indexOfMin = 0;\n    for (let i = 1; i < arr.length; ++i) {\n        if (arr[i] < min) {\n            min = arr[i];\n            indexOfMin = i;\n        }\n    }\n    return [min, indexOfMin];\n}\n\n\n/**\n * Returns the value and index of the maximum element in an array.\n * @param {number[]|AnyTypedArray} arr array of numbers.\n * @returns {[number, number]} the value and index of the maximum element, of the form: [valueOfMax, indexOfMax]\n * @throws {Error} If array is empty.\n */\nexport function max(arr) {\n    if (arr.length === 0) throw Error('Array must not be empty');\n    let max = arr[0];\n    let indexOfMax = 0;\n    for (let i = 1; i < arr.length; ++i) {\n        if (arr[i] > max) {\n            max = arr[i];\n            indexOfMax = i;\n        }\n    }\n    return [Number(max), indexOfMax];\n}\n\nfunction isPowerOfTwo(number) {\n    // Check if the number is greater than 0 and has only one bit set to 1\n    return (number > 0) && ((number & (number - 1)) === 0);\n}\n\n/**\n * Implementation of Radix-4 FFT.\n * \n * P2FFT class provides functionality for performing Fast Fourier Transform on arrays\n * which are a power of two in length.\n * Code adapted from https://www.npmjs.com/package/fft.js\n */\nclass P2FFT {\n    /**\n     * @param {number} size The size of the input array. Must be a power of two larger than 1.\n     * @throws {Error} FFT size must be a power of two larger than 1.\n     */\n    constructor(size) {\n        this.size = size | 0; // convert to a 32-bit signed integer\n        if (this.size <= 1 || !isPowerOfTwo(this.size))\n            throw new Error('FFT size must be a power of two larger than 1');\n\n        this._csize = size << 1;\n\n        this.table = new Float64Array(this.size * 2);\n        for (let i = 0; i < this.table.length; i += 2) {\n            const angle = Math.PI * i / this.size;\n            this.table[i] = Math.cos(angle);\n            this.table[i + 1] = -Math.sin(angle);\n        }\n\n        // Find size's power of two\n        let power = 0;\n        for (let t = 1; this.size > t; t <<= 1)\n            ++power;\n\n        // Calculate initial step's width:\n        //   * If we are full radix-4, it is 2x smaller to give inital len=8\n        //   * Otherwise it is the same as `power` to give len=4\n        this._width = power % 2 === 0 ? power - 1 : power;\n\n        // Pre-compute bit-reversal patterns\n        this._bitrev = new Int32Array(1 << this._width);\n        for (let j = 0; j < this._bitrev.length; ++j) {\n            this._bitrev[j] = 0;\n            for (let shift = 0; shift < this._width; shift += 2) {\n                const revShift = this._width - shift - 2;\n                this._bitrev[j] |= ((j >>> shift) & 3) << revShift;\n            }\n        }\n    }\n\n    /**\n     * Create a complex number array with size `2 * size`\n     *\n     * @returns {Float64Array} A complex number array with size `2 * size`\n     */\n    createComplexArray() {\n        return new Float64Array(this._csize);\n    }\n\n    /**\n     * Converts a complex number representation stored in a Float64Array to an array of real numbers.\n     * \n     * @param {Float64Array} complex The complex number representation to be converted.\n     * @param {number[]} [storage] An optional array to store the result in.\n     * @returns {number[]} An array of real numbers representing the input complex number representation.\n     */\n...\n", "needle_token_start": 12886, "needle_token_end": 13228, "code_context_ntokens": 15432, "output": ["```javascript\n/**\n * Helper method to permute a `AnyTypedArray` directly\n * @template {AnyTypedArray} T \n * @param {T} array \n * @param {number[]} dims \n * @param {number[]} axes \n * @returns {[T, number[]]} The permuted array and the new shape.\n */\nexport function permute_data(array, dims, axes) {\n    // Calculate the new shape of the permuted array\n    // and the stride of the original array\n    const shape = new Array(axes.length);\n    const stride = new Array(axes.length);\n\n    for (let i = axes.length - 1, s = 1; i >= 0; --i) {\n        stride[i] = s;\n        shape[i] = dims[axes[i]];\n        s *= shape[i];\n    }\n\n    // Precompute inverse mapping of stride\n    const invStride = axes.map((_, i) => stride[axes.indexOf(i)]);\n\n    // Create the permuted array with the new shape\n    // @ts-ignore\n    const permutedData = new array.constructor(array.length);\n\n    // Permute the original array to the new array\n    for (let i = 0; i < array.length; ++i) {\n        let newIndex = 0;\n        for (let j = dims.length - 1, k = i; j >= 0; --j) {\n            newIndex += (k % dims[j]) * invStride[j];\n            k = Math.floor(k / dims[j]);\n        }\n        permutedData[newIndex] = array[i];\n    }\n\n    return [permutedData, shape];\n}\n```"]}
{"repo": "xenova/transformers.js", "name": "isChat", "language": "typescript", "path": "src/pipelines.js", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine if a given input is a structured collection representing a conversation, where each item in the collection contains specific attributes related to dialogue roles and content.\n2. **Input**: The function expects an input that could potentially be an array of objects.\n3. **Output**: The function returns a boolean value. It outputs `true` if the input is an array where every element is an object containing specific properties related to conversational roles and content; otherwise, it returns `false`.\n4. **Procedure**: The function first checks if the input is an array. If it is, it then verifies that every element in the array is an object with the required properties indicating the role and content of a dialogue participant. If all elements meet these criteria, the function returns `true`; if any element does not, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/tokenizers.js\n\n/**\n * @file Tokenizers are used to prepare textual inputs for a model.\n * \n * **Example:** Create an `AutoTokenizer` and use it to tokenize a sentence.\n * This will automatically detect the tokenizer type based on the tokenizer class defined in `tokenizer.json`.\n * ```javascript\n * import { AutoTokenizer } from '@xenova/transformers';\n * \n * const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');\n * const { input_ids } = await tokenizer('I love transformers!');\n * // Tensor {\n * //   data: BigInt64Array(6) [101n, 1045n, 2293n, 19081n, 999n, 102n],\n * //   dims: [1, 6],\n * //   type: 'int64',\n * //   size: 6,\n * // }\n * ```\n * \n * @module tokenizers\n */\n\nimport {\n    Callable,\n    reverseDictionary,\n    escapeRegExp,\n    isIntegralNumber,\n    mergeArrays,\n} from './utils/core.js';\n\nimport {\n    getModelJSON,\n} from './utils/hub.js';\n\nimport { max, min, round } from './utils/maths.js';\nimport { Tensor } from './utils/tensor.js';\n\nimport {\n    PriorityQueue,\n    TokenLattice,\n    CharTrie,\n} from './utils/data-structures.js';\n\nimport { Template } from '@huggingface/jinja';\n\n\n/**\n * @typedef {Object} TokenizerProperties Additional tokenizer-specific properties.\n * @property {boolean} [legacy=false] Whether or not the `legacy` behavior of the tokenizer should be used.\n * @typedef {import('./utils/hub.js').PretrainedOptions & TokenizerProperties} PretrainedTokenizerOptions\n */\n\n/**\n * Loads a tokenizer from the specified path.\n * @param {string} pretrained_model_name_or_path The path to the tokenizer directory.\n * @param {PretrainedTokenizerOptions} options Additional options for loading the tokenizer.\n * @returns {Promise<any[]>} A promise that resolves with information about the loaded tokenizer.\n */\nasync function loadTokenizer(pretrained_model_name_or_path, options) {\n\n    const info = await Promise.all([\n        getModelJSON(pretrained_model_name_or_path, 'tokenizer.json', true, options),\n        getModelJSON(pretrained_model_name_or_path, 'tokenizer_config.json', true, options),\n    ])\n\n    // Override legacy option if `options.legacy` is not null\n    if (options.legacy !== null) {\n        info[1].legacy = options.legacy;\n    }\n    return info;\n}\n\n\n/**\n * Helper function to split a string on a regex, but keep the delimiters.\n * This is required, because the JavaScript `.split()` method does not keep the delimiters,\n * and wrapping in a capturing group causes issues with existing capturing groups (due to nesting).\n * @param {string} text The text to split.\n * @param {RegExp} regex The regex to split on.\n * @returns {string[]} The split string.\n */\nfunction regexSplit(text, regex) {\n    const result = [];\n    let prev = 0;\n    for (const match of text.matchAll(regex)) {\n        const fullMatch = match[0];\n        if (prev < match.index) {\n            result.push(text.slice(prev, match.index));\n        }\n        if (fullMatch.length > 0) {\n            result.push(fullMatch);\n        }\n        prev = match.index + fullMatch.length;\n    }\n    if (prev < text.length) {\n        result.push(text.slice(prev));\n    }\n    return result;\n}\n\n\n/**\n * Helper method to construct a pattern from a config object.\n * @param {Object} pattern The pattern object.\n * @param {boolean} invert Whether to invert the pattern.\n * @returns {RegExp|null} The compiled pattern.\n */\nfunction createPattern(pattern, invert = true) {\n\n    if (pattern.Regex !== undefined) {\n        // In certain cases, the pattern may contain unnecessary escape sequences (e.g., \\# or \\& or \\~).\n        // i.e., valid in Python (where the patterns are exported from) but invalid in JavaScript (where the patterns are parsed).\n        // This isn't an issue when creating the regex w/o the 'u' flag, but it is when the 'u' flag is used.\n        // For this reason, it is necessary to remove these backslashes before creating the regex.\n        // See https://stackoverflow.com/a/63007777/13989043 for more information\n        let regex = pattern.Regex.replace(/\\\\([#&~])/g, '$1'); // TODO: add more characters to this list if necessary\n\n        // We also handle special cases where the regex contains invalid (non-JS compatible) syntax.\n        for (const [key, value] of PROBLEMATIC_REGEX_MAP) {\n            regex = regex.replaceAll(key, value);\n        }\n\n        return new RegExp(regex, 'gu');\n\n    } else if (pattern.String !== undefined) {\n        const escaped = escapeRegExp(pattern.String);\n        // NOTE: if invert is true, we wrap the pattern in a group so that it is kept when performing .split()\n        return new RegExp(invert ? escaped : `(${escaped})`, 'gu');\n\n    } else {\n        console.warn('Unknown pattern type:', pattern)\n        return null;\n    }\n}\n\n/**\n * Helper function to convert an Object to a Map\n * @param {Object} obj The object to convert.\n * @returns {Map<string, any>} The map.\n */\nfunction objectToMap(obj) {\n    return new Map(Object.entries(obj));\n}\n\n/**\n * Helper function to convert a tensor to a list before decoding.\n * @param {Tensor} tensor The tensor to convert.\n * @returns {number[]} The tensor as a list.\n */\nfunction prepareTensorForDecode(tensor) {\n    const dims = tensor.dims;\n    switch (dims.length) {\n        case 1:\n            return tensor.tolist();\n        case 2:\n            if (dims[0] !== 1) {\n                throw new Error('Unable to decode tensor with `batch size !== 1`. Use `tokenizer.batch_decode(...)` for batched inputs.');\n            }\n            return tensor.tolist()[0];\n        default:\n            throw new Error(`Expected tensor to have 1-2 dimensions, got ${dims.length}.`)\n    }\n}\n\n/**\n * Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms\n * @param {string} text The text to clean up.\n * @returns {string} The cleaned up text.\n */\nfunction clean_up_tokenization(text) {\n    // Clean up a list of simple English tokenization artifacts\n    // like spaces before punctuations and abbreviated forms\n    return text.replace(/ \\./g, '.')\n        .replace(/ \\?/g, '?')\n        .replace(/ \\!/g, '!')\n        .replace(/ ,/g, ',')\n        .replace(/ \\' /g, \"'\")\n        .replace(/ n\\'t/g, \"n't\")\n        .replace(/ \\'m/g, \"'m\")\n        .replace(/ \\'s/g, \"'s\")\n        .replace(/ \\'ve/g, \"'ve\")\n        .replace(/ \\'re/g, \"'re\");\n}\n\n/**\n * Helper function to remove accents from a string.\n * @param {string} text The text to remove accents from.\n * @returns {string} The text with accents removed.\n */\nfunction remove_accents(text) {\n    return text.replace(/[\\u0300-\\u036f]/g, '');\n}\n\n/**\n * Helper function to lowercase a string and remove accents.\n * @param {string} text The text to lowercase and remove accents from.\n * @returns {string} The lowercased text with accents removed.\n */\nfunction lowercase_and_remove_accent(text) {\n    return remove_accents(text.toLowerCase());\n}\n\n/**\n * Helper function to fuse consecutive values in an array equal to the specified value.\n * @param {string[]} arr The input array\n * @param {any} value The value to fuse on.\n * @param {Map<string, any>} mapping The mapping from input domain to value.\n */\nfunction fuse(arr, value, mapping) {\n    const fused = [];\n    let i = 0;\n    while (i < arr.length) {\n        fused.push(arr[i])\n        if ((mapping.get(arr[i]) ?? value) !== value) {\n            ++i;\n            continue;\n        }\n\n        while (i < arr.length && (mapping.get(arr[i]) ?? value) === value) {\n            ++i;\n        }\n    }\n\n    return fused;\n}\n\n/**\n * Split a string on whitespace.\n * @param {string} text The text to split.\n * @returns {string[]} The split string.\n */\nfunction whitespace_split(text) {\n    return text.match(/\\S+/g) || [];\n}\n\nconst PUNCTUATION_REGEX = '\\\\p{P}\\\\u0021-\\\\u002F\\\\u003A-\\\\u0040\\\\u005B-\\\\u0060\\\\u007B-\\\\u007E';\n\n// A mapping of regex patterns to their equivalent (but longer) JS-compatible versions.\nconst PROBLEMATIC_REGEX_MAP = new Map([\n    // This uses the case insensitive group modifier, which is not supported in JavaScript.\n    // When parsing the regex, an \"Invalid group\" error is thrown.\n    [\"(?i:'s|'t|'re|'ve|'m|'ll|'d)\", \"(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))\"],\n])\n\n\n/**\n * Represent a token added by the user on top of the existing Model vocabulary.\n * AddedToken can be configured to specify the behavior they should have in various situations like:\n *   - Whether they should only match single words\n *   - Whether to include any whitespace on its left or right\n */\nclass AddedToken {\n    /**\n     * Creates a new instance of AddedToken.\n     * @param {Object} config Added token configuration object.\n     * @param {string} config.content The content of the added token.\n     * @param {number} config.id The id of the added token.\n     * @param {boolean} [config.single_word=false] Whether this token must be a single word or can break words.\n     * @param {boolean} [config.lstrip=false] Whether this token should strip whitespaces on its left.\n     * @param {boolean} [config.rstrip=false] Whether this token should strip whitespaces on its right.\n     * @param {boolean} [config.normalized=false] Whether this token should be normalized.\n     * @param {boolean} [config.special=false] Whether this token is special.\n     */\n    constructor(config) {\n        this.content = config.content;\n        this.id = config.id;\n        this.single_word = config.single_word ?? false;\n        this.lstrip = config.lstrip ?? false;\n        this.rstrip = config.rstrip ?? false;\n        this.special = config.special ?? false;\n        this.normalized = config.normalized ?? null;\n    }\n}\n\n/**\n * Abstract base class for tokenizer models.\n *\n * @extends Callable\n */\nexport class TokenizerModel extends Callable {\n    /**\n     * Creates a new instance of TokenizerModel.\n     * @param {Object} config The configuration object for the TokenizerModel.\n     */\n    constructor(config) {\n        super();\n        this.config = config;\n\n        /** @type {string[]} */\n        this.vocab = [];\n\n        /**\n         * A mapping of tokens to ids.\n         * @type {Map<string, number>}\n         */\n        this.tokens_to_ids = new Map();\n\n        this.unk_token_id = undefined;\n        this.unk_token = undefined;\n        this.end_of_word_suffix = undefined;\n\n        /** @type {boolean} Whether to fuse unknown tokens when encoding. Defaults to false. */\n        this.fuse_unk = this.config.fuse_unk ?? false;\n    }\n\n    /**\n     * Instantiates a new TokenizerModel instance based on the configuration object provided.\n     * @param {Object} config The configuration object for the TokenizerModel.\n     * @param {...*} args Optional arguments to pass to the specific TokenizerModel constructor.\n     * @returns {TokenizerModel} A new instance of a TokenizerModel.\n     * @throws Will throw an error if the TokenizerModel type in the config is not recognized.\n     */\n    static fromConfig(config, ...args) {\n        switch (config.type) {\n            case 'WordPiece':\n                return new WordPieceTokenizer(config);\n            case 'Unigram':\n                // @ts-ignore\n                return new Unigram(config, ...args);\n\n            case 'BPE':\n                return new BPE(config);\n\n            default:\n                if (config.vocab) {\n                    // @ts-ignore\n                    return new LegacyTokenizerModel(config, ...args);\n                }\n                throw new Error(`Unknown TokenizerModel type: ${config.type}`);\n        }\n    }\n\n    /**\n     * Internal function to call the TokenizerModel instance.\n     * @param {string[]} tokens The tokens to encode.\n     * @returns {string[]} The encoded token IDs.\n     */\n    _call(tokens) {\n        let ids = this.encode(tokens);\n        if (this.fuse_unk) {\n            // Fuse unknown tokens\n            ids = fuse(ids, this.unk_token_id, this.tokens_to_ids);\n        }\n        return ids;\n    }\n\n    /**\n     * Encodes a list of tokens into a list of token IDs.\n     * @param {string[]} tokens The tokens to encode.\n     * @returns {string[]} The encoded tokens.\n     * @throws Will throw an error if not implemented in a subclass.\n     */\n    encode(tokens) {\n        throw Error(\"encode should be implemented in subclass.\")\n    }\n\n    /**\n     * Converts a list of tokens into a list of token IDs.\n     * @param {string[]} tokens The tokens to convert.\n     * @returns {number[]} The converted token IDs.\n     */\n    convert_tokens_to_ids(tokens) {\n        return tokens.map(t => this.tokens_to_ids.get(t) ?? this.unk_token_id);\n    }\n\n    /**\n     * Converts a list of token IDs into a list of tokens.\n     * @param {number[]} ids The token IDs to convert.\n     * @returns {string[]} The converted tokens.\n     */\n    convert_ids_to_tokens(ids) {\n        return ids.map(i => this.vocab[i] ?? this.unk_token);\n    }\n}\n\n/**\n * A subclass of TokenizerModel that uses WordPiece encoding to encode tokens.\n * @extends TokenizerModel\n */\nclass WordPieceTokenizer extends TokenizerModel {\n    /**\n     * @param {Object} config The configuration object.\n     * @param {Object} config.vocab A mapping of tokens to ids.\n     * @param {string} config.unk_token The unknown token string.\n     * @param {string} config.continuing_subword_prefix The prefix to use for continuing subwords.\n     * @param {number} [config.max_input_chars_per_word=100] The maximum number of characters per word.\n     */\n    constructor(config) {\n        super(config);\n        /**\n         * A mapping of tokens to ids.\n         * @type {Map<string, number>}\n         */\n        this.tokens_to_ids = objectToMap(config.vocab);\n\n        /**\n         * The id of the unknown token.\n         * @type {number}\n         */\n        this.unk_token_id = this.tokens_to_ids.get(config.unk_token);\n\n        /**\n         * The unknown token string.\n         * @type {string}\n         */\n        this.unk_token = config.unk_token;\n\n        /**\n         * The maximum number of characters allowed per word.\n         * @type {number}\n         */\n        this.max_input_chars_per_word = config.max_input_chars_per_word ?? 100;\n\n        /**\n         * An array of tokens.\n         * @type {string[]}\n         */\n...\n// Path: src/pipelines.js\n/**\n * @file Pipelines provide a high-level, easy to use, API for running machine learning models.\n * \n * **Example:** Instantiate pipeline using the `pipeline` function.\n * ```javascript\n * import { pipeline } from '@xenova/transformers';\n * \n * const classifier = await pipeline('sentiment-analysis');\n * const output = await classifier('I love transformers!');\n * // [{'label': 'POSITIVE', 'score': 0.999817686}]\n * ```\n * \n * @module pipelines\n */\n\nimport {\n    AutoTokenizer,\n    PreTrainedTokenizer,\n} from './tokenizers.js';\nimport {\n    AutoModel,\n    AutoModelForSequenceClassification,\n    AutoModelForAudioClassification,\n    AutoModelForTokenClassification,\n    AutoModelForQuestionAnswering,\n    AutoModelForMaskedLM,\n    AutoModelForSeq2SeqLM,\n    AutoModelForSpeechSeq2Seq,\n    AutoModelForTextToWaveform,\n    AutoModelForTextToSpectrogram,\n    AutoModelForCTC,\n    AutoModelForCausalLM,\n    AutoModelForVision2Seq,\n    AutoModelForImageClassification,\n    AutoModelForImageSegmentation,\n    AutoModelForSemanticSegmentation,\n    AutoModelForObjectDetection,\n    AutoModelForZeroShotObjectDetection,\n    AutoModelForDocumentQuestionAnswering,\n    AutoModelForImageToImage,\n    AutoModelForDepthEstimation,\n    AutoModelForImageFeatureExtraction,\n    PreTrainedModel,\n} from './models.js';\nimport {\n    AutoProcessor,\n    Processor\n} from './processors.js';\n\n\nimport {\n    Callable,\n    dispatchCallback,\n    pop,\n    product,\n} from './utils/core.js';\nimport {\n    softmax,\n    max,\n    getTopItems,\n    round,\n} from './utils/maths.js';\nimport {\n    read_audio\n} from './utils/audio.js';\nimport {\n    Tensor,\n    mean_pooling,\n    interpolate,\n    quantize_embeddings,\n} from './utils/tensor.js';\nimport { RawImage } from './utils/image.js';\n\n\n/**\n * @typedef {string | RawImage | URL} ImageInput\n * @typedef {ImageInput|ImageInput[]} ImagePipelineInputs\n */\n\n/**\n * Prepare images for further tasks.\n * @param {ImagePipelineInputs} images images to prepare.\n * @returns {Promise<RawImage[]>} returns processed images.\n * @private\n */\nasync function prepareImages(images) {\n    if (!Array.isArray(images)) {\n        images = [images];\n    }\n\n    // Possibly convert any non-images to images\n    return await Promise.all(images.map(x => RawImage.read(x)));\n}\n\n/**\n * @typedef {string | URL | Float32Array | Float64Array} AudioInput\n * @typedef {AudioInput|AudioInput[]} AudioPipelineInputs\n */\n\n/**\n * Prepare audios for further tasks.\n * @param {AudioPipelineInputs} audios audios to prepare.\n * @param {number} sampling_rate sampling rate of the audios.\n * @returns {Promise<Float32Array[]>} The preprocessed audio data.\n * @private\n */\nasync function prepareAudios(audios, sampling_rate) {\n    if (!Array.isArray(audios)) {\n        audios = [audios];\n    }\n\n    return await Promise.all(audios.map(x => {\n        if (typeof x === 'string' || x instanceof URL) {\n            return read_audio(x, sampling_rate);\n        } else if (x instanceof Float64Array) {\n            return new Float32Array(x);\n        }\n        return x;\n    }));\n}\n\n/**\n * @typedef {Object} BoundingBox\n * @property {number} xmin The minimum x coordinate of the bounding box.\n * @property {number} ymin The minimum y coordinate of the bounding box.\n * @property {number} xmax The maximum x coordinate of the bounding box.\n * @property {number} ymax The maximum y coordinate of the bounding box.\n */\n\n/**\n * Helper function to convert list [xmin, xmax, ymin, ymax] into object { \"xmin\": xmin, ... }\n * @param {number[]} box The bounding box as a list.\n * @param {boolean} asInteger Whether to cast to integers.\n * @returns {BoundingBox} The bounding box as an object.\n * @private\n */\nfunction get_bounding_box(box, asInteger) {\n    if (asInteger) {\n        box = box.map(x => x | 0);\n    }\n    const [xmin, ymin, xmax, ymax] = box;\n\n    return { xmin, ymin, xmax, ymax };\n}\n\n\n/**\n * @callback DisposeType Disposes the item.\n * @returns {Promise<void>} A promise that resolves when the item has been disposed.\n * \n * @typedef {Object} Disposable\n * @property {DisposeType} dispose A promise that resolves when the pipeline has been disposed.\n */\n\n/**\n * The Pipeline class is the class from which all pipelines inherit.\n * Refer to this class for methods shared across different pipelines.\n * @extends Callable\n */\nexport class Pipeline extends Callable {\n    /**\n     * Create a new Pipeline.\n     * @param {Object} options An object containing the following properties:\n     * @param {string} [options.task] The task of the pipeline. Useful for specifying subtasks.\n     * @param {PreTrainedModel} [options.model] The model used by the pipeline.\n     * @param {PreTrainedTokenizer} [options.tokenizer=null] The tokenizer used by the pipeline (if any).\n     * @param {Processor} [options.processor=null] The processor used by the pipeline (if any).\n     */\n    constructor({ task, model, tokenizer = null, processor = null }) {\n        super();\n        this.task = task;\n        this.model = model;\n        this.tokenizer = tokenizer;\n        this.processor = processor;\n    }\n\n    /** @type {DisposeType} */\n    async dispose() {\n        await this.model.dispose();\n    }\n}\n\n/**\n * @typedef {Object} ModelTokenizerConstructorArgs\n * @property {string} task The task of the pipeline. Useful for specifying subtasks.\n * @property {PreTrainedModel} model The model used by the pipeline.\n * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline.\n * \n * @typedef {ModelTokenizerConstructorArgs} TextPipelineConstructorArgs An object used to instantiate a text-based pipeline.\n */\n\n/**\n * @typedef {Object} ModelProcessorConstructorArgs\n * @property {string} task The task of the pipeline. Useful for specifying subtasks.\n * @property {PreTrainedModel} model The model used by the pipeline.\n * @property {Processor} processor The processor used by the pipeline.\n * \n * @typedef {ModelProcessorConstructorArgs} AudioPipelineConstructorArgs An object used to instantiate an audio-based pipeline.\n * @typedef {ModelProcessorConstructorArgs} ImagePipelineConstructorArgs An object used to instantiate an image-based pipeline.\n */\n\n\n/**\n * @typedef {Object} ModelTokenizerProcessorConstructorArgs\n * @property {string} task The task of the pipeline. Useful for specifying subtasks.\n * @property {PreTrainedModel} model The model used by the pipeline.\n * @property {PreTrainedTokenizer} tokenizer The tokenizer used by the pipeline.\n * @property {Processor} processor The processor used by the pipeline.\n * \n * @typedef {ModelTokenizerProcessorConstructorArgs} TextAudioPipelineConstructorArgs An object used to instantiate a text- and audio-based pipeline.\n * @typedef {ModelTokenizerProcessorConstructorArgs} TextImagePipelineConstructorArgs An object used to instantiate a text- and image-based pipeline.\n */\n\n/**\n * @typedef {Object} TextClassificationSingle\n * @property {string} label The label predicted.\n * @property {number} score The corresponding probability.\n * @typedef {TextClassificationSingle[]} TextClassificationOutput\n * \n * @typedef {Object} TextClassificationPipelineOptions Parameters specific to text classification pipelines.\n * @property {number} [topk=1] The number of top predictions to be returned.\n * \n * @callback TextClassificationPipelineCallback Classify the text(s) given as inputs.\n * @param {string|string[]} texts The input text(s) to be classified.\n * @param {TextClassificationPipelineOptions} [options] The options to use for text classification.\n * @returns {Promise<TextClassificationOutput|TextClassificationOutput[]>} An array or object containing the predicted labels and scores.\n * \n * @typedef {TextPipelineConstructorArgs & TextClassificationPipelineCallback & Disposable} TextClassificationPipelineType\n */\n\n/**\n * Text classification pipeline using any `ModelForSequenceClassification`.\n *\n * **Example:** Sentiment-analysis w/ `Xenova/distilbert-base-uncased-finetuned-sst-2-english`.\n * ```javascript\n * const classifier = await pipeline('sentiment-analysis', 'Xenova/distilbert-base-uncased-finetuned-sst-2-english');\n * const output = await classifier('I love transformers!');\n * // [{ label: 'POSITIVE', score: 0.999788761138916 }]\n * ```\n * \n * **Example:** Multilingual sentiment-analysis w/ `Xenova/bert-base-multilingual-uncased-sentiment` (and return top 5 classes).\n * ```javascript\n * const classifier = await pipeline('sentiment-analysis', 'Xenova/bert-base-multilingual-uncased-sentiment');\n * const output = await classifier('Le meilleur film de tous les temps.', { topk: 5 });\n * // [\n * //   { label: '5 stars', score: 0.9610759615898132 },\n * //   { label: '4 stars', score: 0.03323351591825485 },\n * //   { label: '3 stars', score: 0.0036155181005597115 },\n * //   { label: '1 star', score: 0.0011325967498123646 },\n * //   { label: '2 stars', score: 0.0009423971059732139 }\n * // ]\n * ```\n * \n * **Example:** Toxic comment classification w/ `Xenova/toxic-bert` (and return all classes).\n * ```javascript\n * const classifier = await pipeline('text-classification', 'Xenova/toxic-bert');\n * const output = await classifier('I hate you!', { topk: null });\n * // [\n * //   { label: 'toxic', score: 0.9593140482902527 },\n * //   { label: 'insult', score: 0.16187334060668945 },\n * //   { label: 'obscene', score: 0.03452680632472038 },\n * //   { label: 'identity_hate', score: 0.0223250575363636 },\n * //   { label: 'threat', score: 0.019197041168808937 },\n * //   { label: 'severe_toxic', score: 0.005651099607348442 }\n * // ]\n * ```\n */\nexport class TextClassificationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TextClassificationPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new TextClassificationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {TextClassificationPipelineCallback} */\n    async _call(texts, {\n        topk = 1\n    } = {}) {\n\n        // Run tokenization\n        const model_inputs = this.tokenizer(texts, {\n            padding: true,\n            truncation: true,\n        });\n\n        // Run model\n        const outputs = await this.model(model_inputs)\n\n        // TODO: Use softmax tensor function\n        const function_to_apply =\n            this.model.config.problem_type === 'multi_label_classification'\n                ? batch => batch.sigmoid().data\n                : batch => softmax(batch.data); // single_label_classification (default)\n\n        const id2label = this.model.config.id2label;\n\n        const toReturn = [];\n        for (const batch of outputs.logits) {\n            const output = function_to_apply(batch);\n            const scores = getTopItems(output, topk);\n\n            const vals = scores.map(x => ({\n                label: id2label[x[0]],\n                score: x[1],\n            }));\n            if (topk === 1) {\n                toReturn.push(...vals);\n            } else {\n                toReturn.push(vals);\n            }\n        }\n\n        return Array.isArray(texts) || topk === 1 ? /** @type {TextClassificationOutput} */ (toReturn) : /** @type {TextClassificationOutput[]} */ (toReturn)[0];\n    }\n}\n\n/**\n * @typedef {Object} TokenClassificationSingle\n * @property {string} word The token/word classified. This is obtained by decoding the selected tokens.\n * @property {number} score The corresponding probability for `entity`.\n * @property {string} entity The entity predicted for that token/word.\n * @property {number} index The index of the corresponding token in the sentence.\n * @property {number} [start] The index of the start of the corresponding entity in the sentence.\n * @property {number} [end] The index of the end of the corresponding entity in the sentence.\n * @typedef {TokenClassificationSingle[]} TokenClassificationOutput\n * \n * @typedef {Object} TokenClassificationPipelineOptions Parameters specific to token classification pipelines.\n * @property {string[]} [ignore_labels] A list of labels to ignore.\n * \n * @callback TokenClassificationPipelineCallback Classify each token of the text(s) given as inputs.\n * @param {string|string[]} texts One or several texts (or one list of texts) for token classification.\n * @param {TokenClassificationPipelineOptions} [options] The options to use for token classification.\n * @returns {Promise<TokenClassificationOutput|TokenClassificationOutput[]>} The result.\n * \n * @typedef {TextPipelineConstructorArgs & TokenClassificationPipelineCallback & Disposable} TokenClassificationPipelineType\n */\n\n/**\n * Named Entity Recognition pipeline using any `ModelForTokenClassification`.\n * \n * **Example:** Perform named entity recognition with `Xenova/bert-base-NER`.\n * ```javascript\n * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER');\n * const output = await classifier('My name is Sarah and I live in London');\n * // [\n * //   { entity: 'B-PER', score: 0.9980202913284302, index: 4, word: 'Sarah' },\n * //   { entity: 'B-LOC', score: 0.9994474053382874, index: 9, word: 'London' }\n * // ]\n * ```\n * \n * **Example:** Perform named entity recognition with `Xenova/bert-base-NER` (and return all labels).\n * ```javascript\n * const classifier = await pipeline('token-classification', 'Xenova/bert-base-NER');\n * const output = await classifier('Sarah lives in the United States of America', { ignore_labels: [] });\n * // [\n * //   { entity: 'B-PER', score: 0.9966587424278259, index: 1, word: 'Sarah' },\n * //   { entity: 'O', score: 0.9987385869026184, index: 2, word: 'lives' },\n * //   { entity: 'O', score: 0.9990072846412659, index: 3, word: 'in' },\n * //   { entity: 'O', score: 0.9988298416137695, index: 4, word: 'the' },\n * //   { entity: 'B-LOC', score: 0.9995510578155518, index: 5, word: 'United' },\n * //   { entity: 'I-LOC', score: 0.9990395307540894, index: 6, word: 'States' },\n * //   { entity: 'I-LOC', score: 0.9986724853515625, index: 7, word: 'of' },\n * //   { entity: 'I-LOC', score: 0.9975294470787048, index: 8, word: 'America' }\n * // ]\n * ```\n */\nexport class TokenClassificationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TokenClassificationPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new TokenClassificationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {TokenClassificationPipelineCallback} */\n    async _call(texts, {\n        ignore_labels = ['O'],\n    } = {}) {\n\n        const isBatched = Array.isArray(texts);\n\n        // Run tokenization\n        const model_inputs = this.tokenizer(isBatched ? texts : [texts], {\n            padding: true,\n            truncation: true,\n        });\n\n        // Run model\n        const outputs = await this.model(model_inputs)\n\n        const logits = outputs.logits;\n        const id2label = this.model.config.id2label;\n\n        const toReturn = [];\n        for (let i = 0; i < logits.dims[0]; ++i) {\n            const ids = model_inputs.input_ids[i];\n            const batch = logits[i];\n\n            // List of tokens that aren't ignored\n            const tokens = [];\n            for (let j = 0; j < batch.dims[0]; ++j) {\n                const tokenData = batch[j];\n                const topScoreIndex = max(tokenData.data)[1];\n\n                const entity = id2label ? id2label[topScoreIndex] : `LABEL_${topScoreIndex}`;\n                if (ignore_labels.includes(entity)) {\n                    // We predicted a token that should be ignored. So, we skip it.\n                    continue;\n                }\n\n                // TODO add option to keep special tokens?\n                const word = this.tokenizer.decode([ids[j].item()], { skip_special_tokens: true });\n                if (word === '') {\n                    // Was a special token. So, we skip it.\n                    continue;\n                }\n\n                const scores = softmax(tokenData.data);\n\n                tokens.push({\n                    entity: entity,\n                    score: scores[topScoreIndex],\n                    index: j,\n                    word: word,\n\n                    // TODO: null for now, but will add\n                    start: null,\n                    end: null,\n                });\n            }\n            toReturn.push(tokens);\n        }\n        return isBatched ? toReturn : toReturn[0];\n    }\n}\n\n/**\n * @typedef {Object} QuestionAnsweringOutput\n * @property {number} score The probability associated to the answer.\n * @property {number} [start] The character start index of the answer (in the tokenized version of the input).\n * @property {number} [end] The character end index of the answer (in the tokenized version of the input).\n * @property {string} answer The answer to the question.\n * \n * @typedef {Object} QuestionAnsweringPipelineOptions Parameters specific to question answering pipelines.\n * @property {number} [topk=1] The number of top answer predictions to be returned.\n * \n * @callback QuestionAnsweringPipelineCallback Answer the question(s) given as inputs by using the context(s).\n * @param {string|string[]} question One or several question(s) (must be used in conjunction with the `context` argument).\n * @param {string|string[]} context One or several context(s) associated with the question(s) (must be used in conjunction with the `question` argument).\n * @param {QuestionAnsweringPipelineOptions} [options] The options to use for question answering.\n * @returns {Promise<QuestionAnsweringOutput|QuestionAnsweringOutput[]>} An array or object containing the predicted answers and scores.\n * \n * @typedef {TextPipelineConstructorArgs & QuestionAnsweringPipelineCallback & Disposable} QuestionAnsweringPipelineType\n */\n\n/**\n * Question Answering pipeline using any `ModelForQuestionAnswering`.\n * \n * **Example:** Run question answering with `Xenova/distilbert-base-uncased-distilled-squad`.\n * ```javascript\n * const answerer = await pipeline('question-answering', 'Xenova/distilbert-base-uncased-distilled-squad');\n * const question = 'Who was Jim Henson?';\n * const context = 'Jim Henson was a nice puppet.';\n * const output = await answerer(question, context);\n * // {\n * //   answer: \"a nice puppet\",\n * //   score: 0.5768911502526741\n * // }\n * ```\n */\nexport class QuestionAnsweringPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => QuestionAnsweringPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new QuestionAnsweringPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {QuestionAnsweringPipelineCallback} */\n    async _call(question, context, {\n        topk = 1\n    } = {}) {\n\n        // Run tokenization\n        const inputs = this.tokenizer(question, {\n            text_pair: context,\n            padding: true,\n            truncation: true,\n        });\n\n        const output = await this.model(inputs);\n\n        /** @type {QuestionAnsweringOutput[]} */\n        const toReturn = [];\n        for (let j = 0; j < output.start_logits.dims[0]; ++j) {\n            const ids = inputs.input_ids[j];\n            const sepIndex = ids.indexOf(this.tokenizer.sep_token_id);\n\n            const s1 = Array.from(softmax(output.start_logits[j].data))\n                .map((x, i) => [x, i])\n                .filter(x => x[1] > sepIndex);\n            const e1 = Array.from(softmax(output.end_logits[j].data))\n                .map((x, i) => [x, i])\n                .filter(x => x[1] > sepIndex);\n\n            const options = product(s1, e1)\n                .filter(x => x[0][1] <= x[1][1])\n                .map(x => [x[0][1], x[1][1], x[0][0] * x[1][0]])\n                .sort((a, b) => b[2] - a[2]);\n\n            for (let k = 0; k < Math.min(options.length, topk); ++k) {\n                const [start, end, score] = options[k];\n\n                const answer_tokens = [...ids].slice(start, end + 1)\n\n                const answer = this.tokenizer.decode(answer_tokens, {\n                    skip_special_tokens: true,\n                });\n\n                // TODO add start and end?\n                // NOTE: HF returns character index\n                toReturn.push({\n                    answer, score\n                });\n            }\n        }\n\n        // Mimic HF's return type based on topk\n        return (topk === 1) ? toReturn[0] : toReturn;\n    }\n}\n\n\n/**\n * @typedef {Object} FillMaskSingle\n * @property {string} sequence The corresponding input with the mask token prediction.\n * @property {number} score The corresponding probability.\n * @property {number} token The predicted token id (to replace the masked one).\n * @property {string} token_str The predicted token (to replace the masked one).\n * @typedef {FillMaskSingle[]} FillMaskOutput\n * \n * @typedef {Object} FillMaskPipelineOptions Parameters specific to fill mask pipelines.\n * @property {number} [topk=5] When passed, overrides the number of predictions to return.\n * \n * @callback FillMaskPipelineCallback Fill the masked token in the text(s) given as inputs.\n * @param {string|string[]} texts One or several texts (or one list of prompts) with masked tokens.\n * @param {FillMaskPipelineOptions} [options] The options to use for masked language modelling.\n * @returns {Promise<FillMaskOutput|FillMaskOutput[]>} An array of objects containing the score, predicted token, predicted token string,\n * and the sequence with the predicted token filled in, or an array of such arrays (one for each input text).\n * If only one input text is given, the output will be an array of objects.\n * @throws {Error} When the mask token is not found in the input text.\n * \n * @typedef {TextPipelineConstructorArgs & FillMaskPipelineCallback & Disposable} FillMaskPipelineType\n */\n\n/**\n * Masked language modeling prediction pipeline using any `ModelWithLMHead`.\n * \n * **Example:** Perform masked language modelling (a.k.a. \"fill-mask\") with `Xenova/bert-base-uncased`.\n * ```javascript\n * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased');\n * const output = await unmasker('The goal of life is [MASK].');\n * // [\n * //   { token_str: 'survival', score: 0.06137419492006302, token: 8115, sequence: 'The goal of life is survival.' },\n * //   { token_str: 'love', score: 0.03902450203895569, token: 1567, sequence: 'The goal of life is love.' },\n * //   { token_str: 'happiness', score: 0.03253183513879776, token: 9266, sequence: 'The goal of life is happiness.' },\n * //   { token_str: 'freedom', score: 0.018736306577920914, token: 4438, sequence: 'The goal of life is freedom.' },\n * //   { token_str: 'life', score: 0.01859794743359089, token: 1297, sequence: 'The goal of life is life.' }\n * // ]\n * ```\n * \n * **Example:** Perform masked language modelling (a.k.a. \"fill-mask\") with `Xenova/bert-base-cased` (and return top result).\n * ```javascript\n * const unmasker = await pipeline('fill-mask', 'Xenova/bert-base-cased');\n * const output = await unmasker('The Milky Way is a [MASK] galaxy.', { topk: 1 });\n * // [{ token_str: 'spiral', score: 0.6299987435340881, token: 14061, sequence: 'The Milky Way is a spiral galaxy.' }]\n * ```\n */\nexport class FillMaskPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => FillMaskPipelineType} */ (Pipeline)) {\n\n    /**\n     * Create a new FillMaskPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {FillMaskPipelineCallback} */\n    async _call(texts, {\n        topk = 5\n    } = {}) {\n\n        // Run tokenization\n        const model_inputs = this.tokenizer(texts, {\n            padding: true,\n            truncation: true,\n        });\n\n        // Run model\n        const outputs = await this.model(model_inputs)\n\n        const toReturn = [];\n\n        for (let i = 0; i < model_inputs.input_ids.dims[0]; ++i) {\n            const ids = model_inputs.input_ids[i];\n            const mask_token_index = ids.indexOf(this.tokenizer.mask_token_id)\n\n            if (mask_token_index === -1) {\n                throw Error(`Mask token (${this.tokenizer.mask_token}) not found in text.`)\n            }\n            const logits = outputs.logits[i];\n            const itemLogits = logits[mask_token_index];\n\n            const scores = getTopItems(softmax(itemLogits.data), topk);\n\n            toReturn.push(scores.map(x => {\n                const sequence = [...ids];\n                sequence[mask_token_index] = x[0];\n\n                return {\n                    score: x[1],\n                    token: x[0],\n                    token_str: this.tokenizer.model.vocab[x[0]],\n                    sequence: this.tokenizer.decode(sequence, { skip_special_tokens: true }),\n                }\n            }));\n        }\n        return Array.isArray(texts) ? toReturn : toReturn[0];\n    }\n}\n\n\n/**\n * @typedef {Object} Text2TextGenerationSingle\n * @property {string} generated_text The generated text.\n * @typedef {Text2TextGenerationSingle[]} Text2TextGenerationOutput\n * \n * @callback Text2TextGenerationPipelineCallback Generate the output text(s) using text(s) given as inputs.\n * @param {string|string[]} texts Input text for the encoder.\n * @param {import('./utils/generation.js').GenerationConfigType} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<Text2TextGenerationOutput|Text2TextGenerationOutput[]>}\n * \n * @typedef {TextPipelineConstructorArgs & Text2TextGenerationPipelineCallback & Disposable} Text2TextGenerationPipelineType\n */\n\n/**\n * Text2TextGenerationPipeline class for generating text using a model that performs text-to-text generation tasks.\n * \n * **Example:** Text-to-text generation w/ `Xenova/LaMini-Flan-T5-783M`.\n * ```javascript\n * const generator = await pipeline('text2text-generation', 'Xenova/LaMini-Flan-T5-783M');\n * const output = await generator('how can I become more healthy?', {\n *   max_new_tokens: 100,\n * });\n * // [{ generated_text: \"To become more healthy, you can: 1. Eat a balanced diet with plenty of fruits, vegetables, whole grains, lean proteins, and healthy fats. 2. Stay hydrated by drinking plenty of water. 3. Get enough sleep and manage stress levels. 4. Avoid smoking and excessive alcohol consumption. 5. Regularly exercise and maintain a healthy weight. 6. Practice good hygiene and sanitation. 7. Seek medical attention if you experience any health issues.\" }]\n * ```\n */\nexport class Text2TextGenerationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => Text2TextGenerationPipelineType} */ (Pipeline)) {\n    /** @type {'generated_text'} */\n    _key = 'generated_text';\n\n    /**\n     * Create a new Text2TextGenerationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n\n    /** @type {Text2TextGenerationPipelineCallback} */\n    async _call(texts, generate_kwargs = {}) {\n        if (!Array.isArray(texts)) {\n            texts = [texts];\n        }\n\n\n        // Add global prefix, if present\n        if (this.model.config.prefix) {\n            texts = texts.map(x => this.model.config.prefix + x)\n        }\n\n        // Handle task specific params:\n        const task_specific_params = this.model.config.task_specific_params\n        if (task_specific_params && task_specific_params[this.task]) {\n            // Add prefixes, if present\n            if (task_specific_params[this.task].prefix) {\n                texts = texts.map(x => task_specific_params[this.task].prefix + x)\n            }\n\n            // TODO update generation config\n        }\n\n        const tokenizer = this.tokenizer;\n        const tokenizer_options = {\n            padding: true,\n            truncation: true,\n        }\n        let input_ids;\n        if (this instanceof TranslationPipeline && '_build_translation_inputs' in tokenizer) {\n            // TODO: move to Translation pipeline?\n            // Currently put here to avoid code duplication\n            // @ts-ignore\n            input_ids = tokenizer._build_translation_inputs(texts, tokenizer_options, generate_kwargs).input_ids;\n\n        } else {\n            input_ids = tokenizer(texts, tokenizer_options).input_ids;\n        }\n\n        const outputTokenIds = await this.model.generate(input_ids, generate_kwargs);\n\n        return tokenizer.batch_decode(outputTokenIds, {\n            skip_special_tokens: true,\n        }).map(text => ({ [this._key]: text }));\n    }\n}\n\n\n/**\n * @typedef {Object} SummarizationSingle\n * @property {string} summary_text The summary text.\n * @typedef {SummarizationSingle[]} SummarizationOutput\n * \n * @callback SummarizationPipelineCallback Summarize the text(s) given as inputs.\n * @param {string|string[]} texts One or several articles (or one list of articles) to summarize.\n * @param {import('./utils/generation.js').GenerationConfigType} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<SummarizationOutput|SummarizationOutput[]>}\n * \n * @typedef {TextPipelineConstructorArgs & SummarizationPipelineCallback & Disposable} SummarizationPipelineType\n */\n\n/**\n * A pipeline for summarization tasks, inheriting from Text2TextGenerationPipeline.\n * \n * **Example:** Summarization w/ `Xenova/distilbart-cnn-6-6`.\n * ```javascript\n * const generator = await pipeline('summarization', 'Xenova/distilbart-cnn-6-6');\n * const text = 'The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, ' +\n *   'and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. ' +\n *   'During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest ' +\n *   'man-made structure in the world, a title it held for 41 years until the Chrysler Building in New ' +\n *   'York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to ' +\n *   'the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the ' +\n *   'Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second ' +\n *   'tallest free-standing structure in France after the Millau Viaduct.';\n * const output = await generator(text, {\n *   max_new_tokens: 100,\n * });\n * // [{ summary_text: ' The Eiffel Tower is about the same height as an 81-storey building and the tallest structure in Paris. It is the second tallest free-standing structure in France after the Millau Viaduct.' }]\n * ```\n */\nexport class SummarizationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => SummarizationPipelineType} */ (/** @type {any} */ (Text2TextGenerationPipeline))) {\n    /** @type {'summary_text'} */\n    _key = 'summary_text';\n\n    /**\n     * Create a new SummarizationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n}\n\n\n/**\n * @typedef {Object} TranslationSingle\n * @property {string} translation_text The translated text.\n * @typedef {TranslationSingle[]} TranslationOutput\n * \n * @callback TranslationPipelineCallback Translate the text(s) given as inputs.\n * @param {string|string[]} texts Texts to be translated.\n * @param {import('./utils/generation.js').GenerationConfigType} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<TranslationOutput|TranslationOutput[]>}\n * \n * @typedef {TextPipelineConstructorArgs & TranslationPipelineCallback & Disposable} TranslationPipelineType\n */\n\n/**\n * Translates text from one language to another.\n * \n * **Example:** Multilingual translation w/ `Xenova/nllb-200-distilled-600M`.\n * \n * See [here](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200)\n * for the full list of languages and their corresponding codes.\n * \n * ```javascript\n * const translator = await pipeline('translation', 'Xenova/nllb-200-distilled-600M');\n * const output = await translator('\u091c\u0940\u0935\u0928 \u090f\u0915 \u091a\u0949\u0915\u0932\u0947\u091f \u092c\u0949\u0915\u094d\u0938 \u0915\u0940 \u0924\u0930\u0939 \u0939\u0948\u0964', {\n *   src_lang: 'hin_Deva', // Hindi\n *   tgt_lang: 'fra_Latn', // French\n * });\n * // [{ translation_text: 'La vie est comme une bo\u00eete \u00e0 chocolat.' }]\n * ```\n * \n * **Example:** Multilingual translation w/ `Xenova/m2m100_418M`.\n * \n * See [here](https://huggingface.co/facebook/m2m100_418M#languages-covered)\n * for the full list of languages and their corresponding codes.\n * \n * ```javascript\n * const translator = await pipeline('translation', 'Xenova/m2m100_418M');\n * const output = await translator('\u751f\u6d3b\u5c31\u50cf\u4e00\u76d2\u5de7\u514b\u529b\u3002', {\n *   src_lang: 'zh', // Chinese\n *   tgt_lang: 'en', // English\n * });\n * // [{ translation_text: 'Life is like a box of chocolate.' }]\n * ```\n * \n * **Example:** Multilingual translation w/ `Xenova/mbart-large-50-many-to-many-mmt`.\n * \n * See [here](https://huggingface.co/facebook/mbart-large-50-many-to-many-mmt#languages-covered)\n * for the full list of languages and their corresponding codes.\n * \n * ```javascript\n * const translator = await pipeline('translation', 'Xenova/mbart-large-50-many-to-many-mmt');\n * const output = await translator('\u0938\u0902\u092f\u0941\u0915\u094d\u0924 \u0930\u093e\u0937\u094d\u091f\u094d\u0930 \u0915\u0947 \u092a\u094d\u0930\u092e\u0941\u0916 \u0915\u093e \u0915\u0939\u0928\u093e \u0939\u0948 \u0915\u093f \u0938\u0940\u0930\u093f\u092f\u093e \u092e\u0947\u0902 \u0915\u094b\u0908 \u0938\u0948\u0928\u094d\u092f \u0938\u092e\u093e\u0927\u093e\u0928 \u0928\u0939\u0940\u0902 \u0939\u0948', {\n *   src_lang: 'hi_IN', // Hindi\n *   tgt_lang: 'fr_XX', // French\n * });\n * // [{ translation_text: 'Le chef des Nations affirme qu 'il n 'y a military solution in Syria.' }]\n * ```\n */\nexport class TranslationPipeline extends (/** @type {new (options: TextPipelineConstructorArgs) => TranslationPipelineType} */ (/** @type {any} */ (Text2TextGenerationPipeline))) {\n    /** @type {'translation_text'} */\n    _key = 'translation_text';\n\n    /**\n     * Create a new TranslationPipeline.\n     * @param {TextPipelineConstructorArgs} options An object used to instantiate the pipeline.\n     */\n    constructor(options) {\n        super(options);\n    }\n}\n\nfunction isChat(x) {\n    return Array.isArray(x) && x.every(x => 'role' in x && 'content' in x);\n}\n\n/**\n * @typedef {import('./tokenizers.js').Message[]} Chat\n * \n * @typedef {Object} TextGenera\ntionSingle\n * @property {string|Chat} generated_text The generated text.\n * @typedef {TextGenerationSingle[]} TextGenerationOutput\n * \n * @typedef {Object} TextGenerationSpecificParams Parameters specific to text-generation pipelines.\n * @property {boolean} [add_special_tokens] Whether or not to add special tokens when tokenizing the sequences.\n * @property {boolean} [return_full_text=true] If set to `false` only added text is returned, otherwise the full text is returned.\n * @typedef {import('./utils/generation.js').GenerationConfigType & TextGenerationSpecificParams} TextGenerationConfig\n * \n * @callback TextGenerationPipelineCallback Complete the prompt(s) given as inputs.\n * @param {string|string[]|Chat|Chat[]} texts One or several prompts (or one list of prompts) to complete.\n * @param {TextGenerationConfig} [options] Additional keyword arguments to pass along to the generate method of the model.\n * @returns {Promise<TextGenerationOutput|TextGenerationOutput[]>} An array or object containing the generated texts.\n * \n * @typedef {TextPipelineConstructorArgs & TextGenerationPipelineCallback & Disposable} TextGenerationPipelineType\n */\n\n/**\n * Language generation pipeline using any `ModelWithLMHead` or `ModelForCausalLM`.\n * This pipeline predicts the words that will follow a specified text prompt.\n * NOTE: For the full list of generation parameters, see [`GenerationConfig`](./utils/generation#module_utils/generation.GenerationConfig).\n * \n * **Example:** Text generation with `Xenova/distilgpt2` (default settings).\n * ```javascript\n * const generator = await pipeline('text-generation', 'Xenova/distilgpt2');\n * const text = 'I enjoy walking with my cute dog,';\n * const output = await generator(text);\n * // [{ generated_text: \"I enjoy walking with my cute dog, and I love to play with the other dogs.\" }]\n * ```\n * \n * **Example:** Text generation with `Xenova/distilgpt2` (custom settings).\n * ```javascript\n * const generator = await pipeline('text-generation', 'Xenova/distilgpt2');\n * const text = 'Once upon a time, there was';\n * const output = await generator(text, {\n *   temperature: 2,\n *   max_new_tokens: 10,\n *   repetition_penalty: 1.5,\n *   no_repeat_ngram_size: 2,\n *   num_beams: 2,\n *   num_return_sequences: 2,\n * });\n * // [{\n * //   \"generated_text\": \"Once upon a time, there was an abundance of information about the history and activities that\"\n * // }, {\n * //   \"generated_text\": \"Once upon a time, there was an abundance of information about the most important and influential\"\n * // }]\n * ```\n * \n * **Example:** Run code generation with `Xenova/codegen-350M-mono`.\n * ```javascript\n...\n", "needle_token_start": 14963, "needle_token_end": 14993, "code_context_ntokens": 15736, "output": ["```javascript\nfunction isChat(x) {\n    return Array.isArray(x) && x.every(x => 'role' in x && 'content' in x);\n}\n```"]}
{"repo": "expressjs/express", "name": "sendfile", "language": "typescript", "path": "lib/response.js", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the streaming of a file to the client, managing various events such as errors, directory access, and request abortion, ensuring a robust file transfer process.\n2. **Input**: The function takes three parameters: a response object, a file stream, and an options object, along with a callback function to handle completion or errors.\n3. **Output**: There is no direct return value; instead, the function uses the callback to signal completion or errors during the file transfer process.\n4. **Procedure**: \n   - The function first sets up handlers for various file stream events including directory access, file access, streaming, and errors.\n   - It listens for the end of the file stream and the completion of the response to handle normal and error conditions appropriately.\n   - If specified in the options, headers are set on the response based on successful file transfer.\n   - The file stream is piped directly to the response object, allowing for efficient data transfer.\n   - The function ensures that the transfer is properly finalized or aborted based on the state of the request and any occurring errors.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "  // Respond\n  this.statusCode = status;\n  this.set('Content-Length', Buffer.byteLength(body));\n\n  if (this.req.method === 'HEAD') {\n    this.end();\n  } else {\n    this.end(body);\n  }\n};\n\n/**\n * Add `field` to Vary. If already present in the Vary set, then\n * this call is simply ignored.\n *\n * @param {Array|String} field\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.vary = function(field){\n  // checks for back-compat\n  if (!field || (Array.isArray(field) && !field.length)) {\n    deprecate('res.vary(): Provide a field name');\n    return this;\n  }\n\n  vary(this, field);\n\n  return this;\n};\n\n/**\n * Render `view` with the given `options` and optional callback `fn`.\n * When a callback function is given a response will _not_ be made\n * automatically, otherwise a response of _200_ and _text/html_ is given.\n *\n * Options:\n *\n *  - `cache`     boolean hinting to the engine it should cache\n *  - `filename`  filename of the view being rendered\n *\n * @public\n */\n\nres.render = function render(view, options, callback) {\n  var app = this.req.app;\n  var done = callback;\n  var opts = options || {};\n  var req = this.req;\n  var self = this;\n\n  // support callback function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // merge res.locals\n  opts._locals = self.locals;\n\n  // default callback to respond\n  done = done || function (err, str) {\n    if (err) return req.next(err);\n    self.send(str);\n  };\n\n  // render\n  app.render(view, opts, done);\n};\n\n// pipe the send file stream\n\nfunction sendfile(res, file, options, callback) {\n  var done = false;\n  var streaming;\n\n  // request aborted\n  function onaborted() {\n    if (done) return;\n    done = true;\n\n    var err = new Error('Request aborted');\n    err.code = 'ECONNABORTED';\n    callback(err);\n  }\n\n  // directory\n  function ondirectory() {\n    if (done) return;\n    done = true;\n\n    var err = new Error('EISDIR, read');\n    err.code = 'EISDIR';\n    callback(err);\n  }\n\n  // errors\n  function onerror(err) {\n    if (done) return;\n    done = true;\n    callback(err);\n  }\n\n  // ended\n  function onend() {\n    if (done) return;\n    done = true;\n    callback();\n  }\n\n  // file\n  function onfile() {\n    streaming = false;\n  }\n\n  // finished\n  function onfinish(err) {\n    if (err && err.code === 'ECONNRESET') return onaborted();\n    if (err) return onerror(err);\n    if (done) return;\n\n    setImmediate(function () {\n      if (streaming !== false && !done) {\n        onaborted();\n        return;\n      }\n\n      if (done) return;\n      done = true;\n      callback();\n    });\n  }\n\n  // streaming\n  function onstream() {\n    streaming = true;\n  }\n\n  file.on('directory', ondirectory);\n  file.on('end', onend);\n  file.on('error', onerror);\n  file.on('file', onfile);\n  file.on('stream', onstream);\n  onFinished(res, onfinish);\n\n  if (options.headers) {\n    // set headers on successful transfer\n    file.on('headers', function headers(res) {\n      var obj = options.headers;\n      var keys = Object.keys(obj);\n\n      for (var i = 0; i < keys.length; i++) {\n        var k = keys[i];\n        res.setHeader(k, obj[k]);\n      }\n    });\n  }\n\n  // pipe\n  file.pipe(res);\n}\n\n/**\n * Stringify JSON, like JSON.stringify, but v8 optimized, with the\n * ability to escape characters that can trigger HTML sniffing.\n *\n * @param {*} value\n * @param {function} replacer\n * @param {number} spaces\n * @param {boolean} escape\n * @returns {string}\n * @private\n */\n\nfunction stringify (value, replacer, spaces, escape) {\n  // v8 checks arguments.length for optimizing simple call\n  // https://bugs.chromium.org/p/v8/issues/detail?id=4730\n  var json = replacer || spaces\n    ? JSON.stringify(value, replacer, spaces)\n    : JSON.stringify(value);\n\n  if (escape && typeof json === 'string') {\n    json = json.replace(/[<>&]/g, function (c) {\n      switch (c.charCodeAt(0)) {\n        case 0x3c:\n          return '\\\\u003c'\n        case 0x3e:\n          return '\\\\u003e'\n        case 0x26:\n          return '\\\\u0026'\n        /* istanbul ignore next: unreachable default */\n        default:\n          return c\n      }\n    })\n  }\n\n  return json\n}\n\n// Path: lib/request.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar accepts = require('accepts');\nvar deprecate = require('depd')('express');\nvar isIP = require('net').isIP;\nvar typeis = require('type-is');\nvar http = require('http');\nvar fresh = require('fresh');\nvar parseRange = require('range-parser');\nvar parse = require('parseurl');\nvar proxyaddr = require('proxy-addr');\n\n/**\n * Request prototype.\n * @public\n */\n\nvar req = Object.create(http.IncomingMessage.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = req\n\n/**\n * Return request header.\n *\n * The `Referrer` header field is special-cased,\n * both `Referrer` and `Referer` are interchangeable.\n *\n * Examples:\n *\n *     req.get('Content-Type');\n *     // => \"text/plain\"\n *\n *     req.get('content-type');\n *     // => \"text/plain\"\n *\n *     req.get('Something');\n *     // => undefined\n *\n * Aliased as `req.header()`.\n *\n * @param {String} name\n * @return {String}\n * @public\n */\n\nreq.get =\nreq.header = function header(name) {\n  if (!name) {\n    throw new TypeError('name argument is required to req.get');\n  }\n\n  if (typeof name !== 'string') {\n    throw new TypeError('name must be a string to req.get');\n  }\n\n  var lc = name.toLowerCase();\n\n  switch (lc) {\n    case 'referer':\n    case 'referrer':\n      return this.headers.referrer\n        || this.headers.referer;\n    default:\n      return this.headers[lc];\n  }\n};\n\n/**\n * To do: update docs.\n *\n * Check if the given `type(s)` is acceptable, returning\n * the best match when true, otherwise `undefined`, in which\n * case you should respond with 406 \"Not Acceptable\".\n *\n * The `type` value may be a single MIME type string\n * such as \"application/json\", an extension name\n * such as \"json\", a comma-delimited list such as \"json, html, text/plain\",\n * an argument list such as `\"json\", \"html\", \"text/plain\"`,\n * or an array `[\"json\", \"html\", \"text/plain\"]`. When a list\n * or array is given, the _best_ match, if any is returned.\n *\n * Examples:\n *\n *     // Accept: text/html\n *     req.accepts('html');\n *     // => \"html\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('html');\n *     // => \"html\"\n *     req.accepts('text/html');\n *     // => \"text/html\"\n *     req.accepts('json, text');\n *     // => \"json\"\n *     req.accepts('application/json');\n *     // => \"application/json\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('image/png');\n *     req.accepts('png');\n *     // => undefined\n *\n *     // Accept: text/*;q=.5, application/json\n *     req.accepts(['html', 'json']);\n *     req.accepts('html', 'json');\n *     req.accepts('html, json');\n *     // => \"json\"\n *\n * @param {String|Array} type(s)\n * @return {String|Array|Boolean}\n * @public\n */\n\nreq.accepts = function(){\n  var accept = accepts(this);\n  return accept.types.apply(accept, arguments);\n};\n\n/**\n * Check if the given `encoding`s are accepted.\n *\n * @param {String} ...encoding\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsEncodings = function(){\n  var accept = accepts(this);\n  return accept.encodings.apply(accept, arguments);\n};\n\nreq.acceptsEncoding = deprecate.function(req.acceptsEncodings,\n  'req.acceptsEncoding: Use acceptsEncodings instead');\n\n/**\n * Check if the given `charset`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...charset\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsCharsets = function(){\n  var accept = accepts(this);\n  return accept.charsets.apply(accept, arguments);\n};\n\nreq.acceptsCharset = deprecate.function(req.acceptsCharsets,\n  'req.acceptsCharset: Use acceptsCharsets instead');\n\n/**\n * Check if the given `lang`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...lang\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsLanguages = function(){\n  var accept = accepts(this);\n  return accept.languages.apply(accept, arguments);\n};\n\nreq.acceptsLanguage = deprecate.function(req.acceptsLanguages,\n  'req.acceptsLanguage: Use acceptsLanguages instead');\n\n/**\n * Parse Range header field, capping to the given `size`.\n *\n * Unspecified ranges such as \"0-\" require knowledge of your resource length. In\n * the case of a byte range this is of course the total number of bytes. If the\n * Range header field is not given `undefined` is returned, `-1` when unsatisfiable,\n * and `-2` when syntactically invalid.\n *\n * When ranges are returned, the array has a \"type\" property which is the type of\n * range that is required (most commonly, \"bytes\"). Each array element is an object\n * with a \"start\" and \"end\" property for the portion of the range.\n *\n * The \"combine\" option can be set to `true` and overlapping & adjacent ranges\n * will be combined into a single range.\n *\n * NOTE: remember that ranges are inclusive, so for example \"Range: users=0-3\"\n * should respond with 4 users when available, not 3.\n *\n * @param {number} size\n * @param {object} [options]\n * @param {boolean} [options.combine=false]\n * @return {number|array}\n * @public\n */\n\nreq.range = function range(size, options) {\n  var range = this.get('Range');\n  if (!range) return;\n  return parseRange(size, range, options);\n};\n\n/**\n * Return the value of param `name` when present or `defaultValue`.\n *\n *  - Checks route placeholders, ex: _/user/:id_\n *  - Checks body params, ex: id=12, {\"id\":12}\n *  - Checks query string params, ex: ?id=12\n *\n * To utilize request bodies, `req.body`\n * should be an object. This can be done by using\n * the `bodyParser()` middleware.\n *\n * @param {String} name\n * @param {Mixed} [defaultValue]\n * @return {String}\n * @public\n */\n\nreq.param = function param(name, defaultValue) {\n  var params = this.params || {};\n  var body = this.body || {};\n  var query = this.query || {};\n\n  var args = arguments.length === 1\n    ? 'name'\n    : 'name, default';\n  deprecate('req.param(' + args + '): Use req.params, req.body, or req.query instead');\n\n  if (null != params[name] && params.hasOwnProperty(name)) return params[name];\n  if (null != body[name]) return body[name];\n  if (null != query[name]) return query[name];\n\n  return defaultValue;\n};\n\n/**\n * Check if the incoming request contains the \"Content-Type\"\n * header field, and it contains the given mime `type`.\n *\n * Examples:\n *\n *      // With Content-Type: text/html; charset=utf-8\n *      req.is('html');\n *      req.is('text/html');\n *      req.is('text/*');\n *      // => true\n *\n *      // When Content-Type is application/json\n *      req.is('json');\n *      req.is('application/json');\n *      req.is('application/*');\n *      // => true\n *\n *      req.is('html');\n *      // => false\n *\n * @param {String|Array} types...\n * @return {String|false|null}\n * @public\n */\n\nreq.is = function is(types) {\n  var arr = types;\n\n  // support flattened arguments\n  if (!Array.isArray(types)) {\n    arr = new Array(arguments.length);\n    for (var i = 0; i < arr.length; i++) {\n      arr[i] = arguments[i];\n    }\n  }\n\n  return typeis(this, arr);\n};\n\n/**\n * Return the protocol string \"http\" or \"https\"\n * when requested with TLS. When the \"trust proxy\"\n * setting trusts the socket address, the\n * \"X-Forwarded-Proto\" header field will be trusted\n * and used if present.\n *\n * If you're running behind a reverse proxy that\n * supplies https for you this may be enabled.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'protocol', function protocol(){\n  var proto = this.connection.encrypted\n    ? 'https'\n    : 'http';\n  var trust = this.app.get('trust proxy fn');\n\n  if (!trust(this.connection.remoteAddress, 0)) {\n    return proto;\n  }\n\n  // Note: X-Forwarded-Proto is normally only ever a\n  //       single value, but this is to be safe.\n  var header = this.get('X-Forwarded-Proto') || proto\n  var index = header.indexOf(',')\n\n  return index !== -1\n    ? header.substring(0, index).trim()\n    : header.trim()\n});\n\n/**\n * Short-hand for:\n *\n *    req.protocol === 'https'\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'secure', function secure(){\n  return this.protocol === 'https';\n});\n\n/**\n * Return the remote address from the trusted proxy.\n *\n * The is the remote address on the socket unless\n * \"trust proxy\" is set.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'ip', function ip(){\n  var trust = this.app.get('trust proxy fn');\n  return proxyaddr(this, trust);\n});\n\n/**\n * When \"trust proxy\" is set, trusted proxy addresses + client.\n *\n * For example if the value were \"client, proxy1, proxy2\"\n * you would receive the array `[\"client\", \"proxy1\", \"proxy2\"]`\n * where \"proxy2\" is the furthest down-stream and \"proxy1\" and\n * \"proxy2\" were trusted.\n *\n * @return {Array}\n * @public\n */\n\ndefineGetter(req, 'ips', function ips() {\n  var trust = this.app.get('trust proxy fn');\n  var addrs = proxyaddr.all(this, trust);\n\n  // reverse the order (to farthest -> closest)\n  // and remove socket address\n  addrs.reverse().pop()\n\n  return addrs\n});\n\n/**\n * Return subdomains as an array.\n *\n * Subdomains are the dot-separated parts of the host before the main domain of\n * the app. By default, the domain of the app is assumed to be the last two\n * parts of the host. This can be changed by setting \"subdomain offset\".\n *\n * For example, if the domain is \"tobi.ferrets.example.com\":\n * If \"subdomain offset\" is not set, req.subdomains is `[\"ferrets\", \"tobi\"]`.\n * If \"subdomain offset\" is 3, req.subdomains is `[\"tobi\"]`.\n *\n * @return {Array}\n * @public\n */\n\ndefineGetter(req, 'subdomains', function subdomains() {\n  var hostname = this.hostname;\n\n  if (!hostname) return [];\n\n  var offset = this.app.get('subdomain offset');\n  var subdomains = !isIP(hostname)\n    ? hostname.split('.').reverse()\n    : [hostname];\n\n  return subdomains.slice(offset);\n});\n\n/**\n * Short-hand for `url.parse(req.url).pathname`.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'path', function path() {\n  return parse(this).pathname;\n});\n\n/**\n * Parse the \"Host\" header field to a hostname.\n *\n * When the \"trust proxy\" setting trusts the socket\n * address, the \"X-Forwarded-Host\" header field will\n * be trusted.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'hostname', function hostname(){\n  var trust = this.app.get('trust proxy fn');\n  var host = this.get('X-Forwarded-Host');\n\n  if (!host || !trust(this.connection.remoteAddress, 0)) {\n    host = this.get('Host');\n  } else if (host.indexOf(',') !== -1) {\n    // Note: X-Forwarded-Host is normally only ever a\n    //       single value, but this is to be safe.\n    host = host.substring(0, host.indexOf(',')).trimRight()\n  }\n\n  if (!host) return;\n\n  // IPv6 literal support\n  var offset = host[0] === '['\n    ? host.indexOf(']') + 1\n    : 0;\n  var index = host.indexOf(':', offset);\n\n  return index !== -1\n    ? host.substring(0, index)\n    : host;\n});\n\n// TODO: change req.host to return host in next major\n\ndefineGetter(req, 'host', deprecate.function(function host(){\n  return this.hostname;\n}, 'req.host: Use req.hostname instead'));\n\n/**\n * Check if the request is fresh, aka\n * Last-Modified and/or the ETag\n * still match.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'fresh', function(){\n  var method = this.method;\n  var res = this.res\n  var status = res.statusCode\n\n  // GET or HEAD for weak freshness validation only\n  if ('GET' !== method && 'HEAD' !== method) return false;\n\n  // 2xx or 304 as per rfc2616 14.26\n  if ((status >= 200 && status < 300) || 304 === status) {\n    return fresh(this.headers, {\n      'etag': res.get('ETag'),\n      'last-modified': res.get('Last-Modified')\n    })\n  }\n\n  return false;\n});\n\n/**\n * Check if the request is stale, aka\n * \"Last-Modified\" and / or the \"ETag\" for the\n * resource has changed.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'stale', function stale(){\n  return !this.fresh;\n});\n\n/**\n * Check if the request was an _XMLHttpRequest_.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'xhr', function xhr(){\n  var val = this.get('X-Requested-With') || '';\n  return val.toLowerCase() === 'xmlhttprequest';\n});\n\n/**\n * Helper function for creating a getter on an object.\n *\n * @param {Object} obj\n * @param {String} name\n * @param {Function} getter\n * @private\n */\nfunction defineGetter(obj, name, getter) {\n  Object.defineProperty(obj, name, {\n    configurable: true,\n    enumerable: true,\n    get: getter\n  });\n}\n\n// Path: lib/router/layer.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar pathRegexp = require('path-to-regexp');\nvar debug = require('debug')('express:router:layer');\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Layer;\n\nfunction Layer(path, options, fn) {\n  if (!(this instanceof Layer)) {\n    return new Layer(path, options, fn);\n  }\n\n  debug('new %o', path)\n  var opts = options || {};\n\n  this.handle = fn;\n  this.name = fn.name || '<anonymous>';\n  this.params = undefined;\n  this.path = undefined;\n  this.regexp = pathRegexp(path, this.keys = [], opts);\n\n  // set fast path flags\n  this.regexp.fast_star = path === '*'\n  this.regexp.fast_slash = path === '/' && opts.end === false\n}\n\n/**\n * Handle the error for the layer.\n *\n * @param {Error} error\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_error = function handle_error(error, req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length !== 4) {\n    // not a standard error handler\n    return next(error);\n  }\n\n  try {\n    fn(error, req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Handle the request for the layer.\n *\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_request = function handle(req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length > 3) {\n    // not a standard request handler\n    return next();\n  }\n\n  try {\n    fn(req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Check if this route matches `path`, if so\n * populate `.params`.\n *\n * @param {String} path\n * @return {Boolean}\n * @api private\n */\n\nLayer.prototype.match = function match(path) {\n  var match\n\n  if (path != null) {\n    // fast path non-ending match for / (any path matches)\n    if (this.regexp.fast_slash) {\n      this.params = {}\n      this.path = ''\n      return true\n    }\n\n    // fast path for * (everything matched in a param)\n    if (this.regexp.fast_star) {\n      this.params = {'0': decode_param(path)}\n      this.path = path\n      return true\n    }\n\n    // match the path\n    match = this.regexp.exec(path)\n  }\n\n  if (!match) {\n    this.params = undefined;\n    this.path = undefined;\n    return false;\n  }\n\n  // store values\n  this.params = {};\n  this.path = match[0]\n\n  var keys = this.keys;\n  var params = this.params;\n\n  for (var i = 1; i < match.length; i++) {\n    var key = keys[i - 1];\n    var prop = key.name;\n    var val = decode_param(match[i])\n\n    if (val !== undefined || !(hasOwnProperty.call(params, prop))) {\n      params[prop] = val;\n    }\n  }\n\n  return true;\n};\n\n/**\n * Decode param value.\n *\n * @param {string} val\n * @return {string}\n * @private\n */\n\nfunction decode_param(val) {\n  if (typeof val !== 'string' || val.length === 0) {\n    return val;\n  }\n\n  try {\n    return decodeURIComponent(val);\n  } catch (err) {\n    if (err instanceof URIError) {\n      err.message = 'Failed to decode param \\'' + val + '\\'';\n      err.status = err.statusCode = 400;\n    }\n\n    throw err;\n  }\n}\n\n// Path: lib/router/route.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:router:route');\nvar flatten = require('array-flatten');\nvar Layer = require('./layer');\nvar methods = require('methods');\n\n/**\n * Module variables.\n * @private\n */\n\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Route;\n\n/**\n * Initialize `Route` with the given `path`,\n *\n * @param {String} path\n * @public\n */\n\nfunction Route(path) {\n  this.path = path;\n  this.stack = [];\n\n  debug('new %o', path)\n\n  // route handlers for various http methods\n  this.methods = {};\n}\n\n/**\n * Determine if the route handles a given method.\n * @private\n */\n\nRoute.prototype._handles_method = function _handles_method(method) {\n  if (this.methods._all) {\n    return true;\n  }\n\n  // normalize name\n  var name = typeof method === 'string'\n    ? method.toLowerCase()\n    : method\n\n  if (name === 'head' && !this.methods['head']) {\n    name = 'get';\n  }\n\n  return Boolean(this.methods[name]);\n};\n\n/**\n * @return {Array} supported HTTP methods\n * @private\n */\n\nRoute.prototype._options = function _options() {\n  var methods = Object.keys(this.methods);\n\n  // append automatic head\n  if (this.methods.get && !this.methods.head) {\n    methods.push('head');\n  }\n\n  for (var i = 0; i < methods.length; i++) {\n    // make upper case\n    methods[i] = methods[i].toUpperCase();\n  }\n\n  return methods;\n};\n\n/**\n * dispatch req, res into this route\n * @private\n */\n\nRoute.prototype.dispatch = function dispatch(req, res, done) {\n  var idx = 0;\n  var stack = this.stack;\n  var sync = 0\n\n  if (stack.length === 0) {\n    return done();\n  }\n  var method = typeof req.method === 'string'\n    ? req.method.toLowerCase()\n    : req.method\n\n  if (method === 'head' && !this.methods['head']) {\n    method = 'get';\n  }\n\n  req.route = this;\n\n  next();\n\n  function next(err) {\n    // signal to exit route\n    if (err && err === 'route') {\n      return done();\n    }\n\n    // signal to exit router\n    if (err && err === 'router') {\n      return done(err)\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    var layer = stack[idx++]\n\n    // end of layers\n    if (!layer) {\n      return done(err)\n    }\n\n    if (layer.method && layer.method !== method) {\n      next(err)\n    } else if (err) {\n      layer.handle_error(err, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n\n    sync = 0\n  }\n};\n\n/**\n * Add a handler for all HTTP verbs to this route.\n *\n * Behaves just like middleware and can respond or call `next`\n * to continue processing.\n *\n * You can use multiple `.all` call to add multiple handlers.\n *\n *   function check_something(req, res, next){\n *     next();\n *   };\n *\n *   function validate_user(req, res, next){\n *     next();\n *   };\n *\n *   route\n *   .all(validate_user)\n *   .all(check_something)\n *   .get(function(req, res, next){\n *     res.send('hello world');\n *   });\n *\n * @param {function} handler\n * @return {Route} for chaining\n * @api public\n */\n\nRoute.prototype.all = function all() {\n  var handles = flatten(slice.call(arguments));\n\n  for (var i = 0; i < handles.length; i++) {\n    var handle = handles[i];\n\n    if (typeof handle !== 'function') {\n      var type = toString.call(handle);\n      var msg = 'Route.all() requires a callback function but got a ' + type\n      throw new TypeError(msg);\n    }\n\n    var layer = Layer('/', {}, handle);\n    layer.method = undefined;\n\n    this.methods._all = true;\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\nmethods.forEach(function(method){\n  Route.prototype[method] = function(){\n    var handles = flatten(slice.call(arguments));\n\n    for (var i = 0; i < handles.length; i++) {\n      var handle = handles[i];\n\n      if (typeof handle !== 'function') {\n        var type = toString.call(handle);\n        var msg = 'Route.' + method + '() requires a callback function but got a ' + type\n        throw new Error(msg);\n      }\n\n      debug('%s %o', method, this.path)\n\n      var layer = Layer('/', {}, handle);\n      layer.method = method;\n\n      this.methods[method] = true;\n      this.stack.push(layer);\n    }\n\n    return this;\n  };\n});\n\n// Path: lib/router/index.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Route = require('./route');\nvar Layer = require('./layer');\nvar methods = require('methods');\nvar mixin = require('utils-merge');\nvar debug = require('debug')('express:router');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar parseUrl = require('parseurl');\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar objectRegExp = /^\\[object (\\S+)\\]$/;\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n\n/**\n * Map the given param placeholder `name`(s) to the given callback.\n *\n * Parameter mapping is used to provide pre-conditions to routes\n * which use normalized placeholders. For example a _:user_id_ parameter\n * could automatically load a user's information from the database without\n * any additional code,\n *\n * The callback uses the same signature as middleware, the only difference\n * being that the value of the placeholder is passed, in this case the _id_\n * of the user. Once the `next()` function is invoked, just like middleware\n * it will continue on to execute the route, or subsequent parameter functions.\n *\n * Just like in middleware, you must either respond to the request or call next\n * to avoid stalling the request.\n *\n *  app.param('user_id', function(req, res, next, id){\n *    User.find(id, function(err, user){\n *      if (err) {\n *        return next(err);\n *      } else if (!user) {\n *        return next(new Error('failed to load user'));\n *      }\n *      req.user = user;\n *      next();\n *    });\n *  });\n *\n * @param {String} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\nproto.param = function param(name, fn) {\n  // param logic\n  if (typeof name === 'function') {\n    deprecate('router.param(fn): Refactor to use path params');\n    this._params.push(name);\n    return;\n  }\n\n  // apply param functions\n  var params = this._params;\n  var len = params.length;\n  var ret;\n\n  if (name[0] === ':') {\n    deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.slice(1)) + ', fn) instead')\n    name = name.slice(1)\n  }\n\n  for (var i = 0; i < len; ++i) {\n    if (ret = params[i](name, fn)) {\n      fn = ret;\n    }\n  }\n\n  // ensure we end up with a\n  // middleware function\n  if ('function' !== typeof fn) {\n    throw new Error('invalid param() call for ' + name + ', got ' + fn);\n  }\n\n  (this.params[name] = this.params[name] || []).push(fn);\n  return this;\n};\n\n/**\n * Dispatch a req, res into the router.\n * @private\n */\n\nproto.handle = function handle(req, res, out) {\n  var self = this;\n\n  debug('dispatching %s %s', req.method, req.url);\n\n  var idx = 0;\n  var protohost = getProtohost(req.url) || ''\n  var removed = '';\n  var slashAdded = false;\n  var sync = 0\n  var paramcalled = {};\n\n  // store options for OPTIONS request\n  // only used if OPTIONS request\n  var options = [];\n\n  // middleware and routes\n  var stack = self.stack;\n\n  // manage inter-router variables\n  var parentParams = req.params;\n  var parentUrl = req.baseUrl || '';\n  var done = restore(out, req, 'baseUrl', 'next', 'params');\n\n  // setup next layer\n  req.next = next;\n\n  // for options requests, respond with a default if nothing else responds\n  if (req.method === 'OPTIONS') {\n    done = wrap(done, function(old, err) {\n      if (err || options.length === 0) return old(err);\n      sendOptionsResponse(res, options, old);\n    });\n  }\n\n  // setup basic req values\n  req.baseUrl = parentUrl;\n  req.originalUrl = req.originalUrl || req.url;\n\n  next();\n\n  function next(err) {\n    var layerError = err === 'route'\n      ? null\n      : err;\n\n    // remove added slash\n    if (slashAdded) {\n      req.url = req.url.slice(1)\n      slashAdded = false;\n    }\n\n    // restore altered req.url\n    if (removed.length !== 0) {\n      req.baseUrl = parentUrl;\n      req.url = protohost + removed + req.url.slice(protohost.length)\n      removed = '';\n    }\n\n    // signal to exit router\n    if (layerError === 'router') {\n      setImmediate(done, null)\n      return\n    }\n\n    // no more matching layers\n    if (idx >= stack.length) {\n      setImmediate(done, layerError);\n      return;\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    // get pathname of request\n    var path = getPathname(req);\n\n    if (path == null) {\n      return done(layerError);\n    }\n\n    // find next matching layer\n    var layer;\n    var match;\n    var route;\n\n    while (match !== true && idx < stack.length) {\n      layer = stack[idx++];\n      match = matchLayer(layer, path);\n      route = layer.route;\n\n      if (typeof match !== 'boolean') {\n        // hold on to layerError\n        layerError = layerError || match;\n      }\n\n      if (match !== true) {\n        continue;\n      }\n\n      if (!route) {\n        // process non-route handlers normally\n        continue;\n      }\n\n      if (layerError) {\n        // routes do not match with a pending error\n        match = false;\n        continue;\n      }\n\n      var method = req.method;\n      var has_method = route._handles_method(method);\n\n      // build up automatic options response\n      if (!has_method && method === 'OPTIONS') {\n        appendMethods(options, route._options());\n      }\n\n      // don't even bother matching route\n      if (!has_method && method !== 'HEAD') {\n        match = false;\n      }\n    }\n\n    // no match\n    if (match !== true) {\n      return done(layerError);\n    }\n\n    // store route for dispatch on change\n    if (route) {\n      req.route = route;\n    }\n\n    // Capture one-time layer values\n    req.params = self.mergeParams\n      ? mergeParams(layer.params, parentParams)\n      : layer.params;\n    var layerPath = layer.path;\n\n    // this should be done for the layer\n    self.process_params(layer, paramcalled, req, res, function (err) {\n      if (err) {\n        next(layerError || err)\n      } else if (route) {\n        layer.handle_request(req, res, next)\n      } else {\n        trim_prefix(layer, layerError, layerPath, path)\n      }\n\n      sync = 0\n    });\n  }\n\n  function trim_prefix(layer, layerError, layerPath, path) {\n    if (layerPath.length !== 0) {\n      // Validate path is a prefix match\n      if (layerPath !== path.slice(0, layerPath.length)) {\n        next(layerError)\n        return\n      }\n\n      // Validate path breaks on a path separator\n      var c = path[layerPath.length]\n      if (c && c !== '/' && c !== '.') return next(layerError)\n\n      // Trim off the part of the url that matches the route\n      // middleware (.use stuff) needs to have the path stripped\n      debug('trim prefix (%s) from url %s', layerPath, req.url);\n      removed = layerPath;\n      req.url = protohost + req.url.slice(protohost.length + removed.length)\n\n      // Ensure leading slash\n      if (!protohost && req.url[0] !== '/') {\n        req.url = '/' + req.url;\n        slashAdded = true;\n      }\n\n      // Setup base URL (no trailing slash)\n      req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'\n        ? removed.substring(0, removed.length - 1)\n        : removed);\n    }\n\n    debug('%s %s : %s', layer.name, layerPath, req.originalUrl);\n\n    if (layerError) {\n      layer.handle_error(layerError, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n  }\n};\n\n/**\n * Process any parameters for the layer.\n * @private\n */\n\nproto.process_params = function process_params(layer, called, req, res, done) {\n  var params = this.params;\n\n  // captured parameters from the layer, keys and values\n  var keys = layer.keys;\n\n  // fast track\n  if (!keys || keys.length === 0) {\n    return done();\n  }\n\n  var i = 0;\n  var name;\n  var paramIndex = 0;\n  var key;\n  var paramVal;\n  var paramCallbacks;\n  var paramCalled;\n\n  // process params in order\n  // param callbacks can be async\n  function param(err) {\n    if (err) {\n      return done(err);\n    }\n\n    if (i >= keys.length ) {\n      return done();\n    }\n\n    paramIndex = 0;\n    key = keys[i++];\n    name = key.name;\n    paramVal = req.params[name];\n    paramCallbacks = params[name];\n    paramCalled = called[name];\n\n    if (paramVal === undefined || !paramCallbacks) {\n      return param();\n    }\n\n    // param previously called with same value or error occurred\n    if (paramCalled && (paramCalled.match === paramVal\n      || (paramCalled.error && paramCalled.error !== 'route'))) {\n      // restore value\n      req.params[name] = paramCalled.value;\n\n      // next param\n      return param(paramCalled.error);\n    }\n\n    called[name] = paramCalled = {\n      error: null,\n      match: paramVal,\n      value: paramVal\n    };\n\n    paramCallback();\n  }\n\n  // single param callbacks\n  function paramCallback(err) {\n    var fn = paramCallbacks[paramIndex++];\n\n    // store updated value\n    paramCalled.value = req.params[key.name];\n\n    if (err) {\n      // store error\n      paramCalled.error = err;\n      param(err);\n      return;\n    }\n\n    if (!fn) return param();\n\n    try {\n      fn(req, res, paramCallback, paramVal, key.name);\n    } catch (e) {\n      paramCallback(e);\n    }\n  }\n\n  param();\n};\n\n/**\n * Use the given middleware function, with optional path, defaulting to \"/\".\n *\n * Use (like `.all`) will run for any http METHOD, but it will not add\n * handlers for those methods so OPTIONS requests will not consider `.use`\n * functions even if they could respond.\n *\n * The other difference is that _route_ path is stripped and not visible\n * to the handler function. The main effect of this feature is that mounted\n * handlers can operate without any code changes regardless of the \"prefix\"\n * pathname.\n *\n * @public\n */\n\nproto.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate router.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var callbacks = flatten(slice.call(arguments, offset));\n\n  if (callbacks.length === 0) {\n    throw new TypeError('Router.use() requires a middleware function')\n  }\n\n  for (var i = 0; i < callbacks.length; i++) {\n    var fn = callbacks[i];\n\n    if (typeof fn !== 'function') {\n      throw new TypeError('Router.use() requires a middleware function but got a ' + gettype(fn))\n    }\n\n    // add the middleware\n    debug('use %o %s', path, fn.name || '<anonymous>')\n\n    var layer = new Layer(path, {\n      sensitive: this.caseSensitive,\n      strict: false,\n      end: false\n    }, fn);\n\n    layer.route = undefined;\n\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\n/**\n * Create a new Route for the given path.\n *\n * Each route contains a separate middleware stack and VERB handlers.\n *\n * See the Route api documentation for details on adding handlers\n * and middleware to routes.\n *\n * @param {String} path\n * @return {Route}\n * @public\n */\n\nproto.route = function route(path) {\n  var route = new Route(path);\n\n  var layer = new Layer(path, {\n    sensitive: this.caseSensitive,\n    strict: this.strict,\n    end: true\n  }, route.dispatch.bind(route));\n\n  layer.route = route;\n\n  this.stack.push(layer);\n  return route;\n};\n\n// create Router#VERB functions\nmethods.concat('all').forEach(function(method){\n  proto[method] = function(path){\n    var route = this.route(path)\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n// append methods to a list of methods\nfunction appendMethods(list, addition) {\n  for (var i = 0; i < addition.length; i++) {\n    var method = addition[i];\n    if (list.indexOf(method) === -1) {\n      list.push(method);\n    }\n  }\n}\n\n// get pathname of request\nfunction getPathname(req) {\n  try {\n    return parseUrl(req).pathname;\n  } catch (err) {\n    return undefined;\n  }\n}\n\n// Get get protocol + host for a URL\nfunction getProtohost(url) {\n  if (typeof url !== 'string' || url.length === 0 || url[0] === '/') {\n    return undefined\n  }\n\n  var searchIndex = url.indexOf('?')\n  var pathLength = searchIndex !== -1\n    ? searchIndex\n    : url.length\n  var fqdnIndex = url.slice(0, pathLength).indexOf('://')\n\n  return fqdnIndex !== -1\n    ? url.substring(0, url.indexOf('/', 3 + fqdnIndex))\n    : undefined\n}\n\n// get type for error message\nfunction gettype(obj) {\n  var type = typeof obj;\n\n  if (type !== 'object') {\n    return type;\n  }\n\n  // inspect [[Class]] for objects\n  return toString.call(obj)\n    .replace(objectRegExp, '$1');\n}\n\n/**\n * Match path to a layer.\n *\n * @param {Layer} layer\n * @param {string} path\n * @private\n */\n\nfunction matchLayer(layer, path) {\n  try {\n    return layer.match(path);\n  } catch (err) {\n    return err;\n  }\n}\n\n// merge params with parent params\nfunction mergeParams(params, parent) {\n  if (typeof parent !== 'object' || !parent) {\n    return params;\n  }\n\n  // make copy of parent for base\n  var obj = mixin({}, parent);\n\n  // simple non-numeric merging\n  if (!(0 in params) || !(0 in parent)) {\n    return mixin(obj, params);\n  }\n\n  var i = 0;\n  var o = 0;\n\n  // determine numeric gaps\n  while (i in params) {\n    i++;\n  }\n\n  while (o in parent) {\n    o++;\n  }\n\n  // offset numeric indices in params before merge\n  for (i--; i >= 0; i--) {\n    params[i + o] = params[i];\n\n    // create holes for the merge when necessary\n    if (i < o) {\n      delete params[i];\n    }\n  }\n\n  return mixin(obj, params);\n}\n\n// restore obj props after function\nfunction restore(fn, obj) {\n  var props = new Array(arguments.length - 2);\n  var vals = new Array(arguments.length - 2);\n\n  for (var i = 0; i < props.length; i++) {\n    props[i] = arguments[i + 2];\n    vals[i] = obj[props[i]];\n  }\n\n  return function () {\n    // restore vals\n    for (var i = 0; i < props.length; i++) {\n      obj[props[i]] = vals[i];\n    }\n\n    return fn.apply(this, arguments);\n  };\n}\n\n// send an OPTIONS response\nfunction sendOptionsResponse(res, options, next) {\n  try {\n    var body = options.join(',');\n    res.set('Allow', body);\n    res.send(body);\n  } catch (err) {\n    next(err);\n  }\n}\n\n// wrap a function\nfunction wrap(old, fn) {\n  return function proxy() {\n    var args = new Array(arguments.length + 1);\n\n    args[0] = old;\n    for (var i = 0, len = arguments.length; i < len; i++) {\n      args[i + 1] = arguments[i];\n    }\n\n    fn.apply(this, args);\n  };\n}\n\n// Path: lib/middleware/init.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Initialization middleware, exposing the\n * request and response to each other, as well\n * as defaulting the X-Powered-By header field.\n *\n * @param {Function} app\n * @return {Function}\n * @api private\n */\n\nexports.init = function(app){\n  return function expressInit(req, res, next){\n    if (app.enabled('x-powered-by')) res.setHeader('X-Powered-By', 'Express');\n    req.res = res;\n    res.req = req;\n    req.next = next;\n\n    setPrototypeOf(req, app.request)\n    setPrototypeOf(res, app.response)\n\n    res.locals = res.locals || Object.create(null);\n\n    next();\n  };\n};\n\n\n// Path: lib/view.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:view');\nvar path = require('path');\nvar fs = require('fs');\n\n/**\n * Module variables.\n * @private\n */\n\nvar dirname = path.dirname;\nvar basename = path.basename;\nvar extname = path.extname;\nvar join = path.join;\nvar resolve = path.resolve;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = View;\n\n/**\n * Initialize a new `View` with the given `name`.\n *\n * Options:\n *\n *   - `defaultEngine` the default template engine name\n *   - `engines` template engine require() cache\n *   - `root` root path for view lookup\n *\n * @param {string} name\n * @param {object} options\n * @public\n */\n\nfunction View(name, options) {\n  var opts = options || {};\n\n  this.defaultEngine = opts.defaultEngine;\n  this.ext = extname(name);\n  this.name = name;\n  this.root = opts.root;\n\n  if (!this.ext && !this.defaultEngine) {\n    throw new Error('No default engine was specified and no extension was provided.');\n  }\n\n  var fileName = name;\n\n  if (!this.ext) {\n    // get extension from default engine name\n    this.ext = this.defaultEngine[0] !== '.'\n      ? '.' + this.defaultEngine\n      : this.defaultEngine;\n\n    fileName += this.ext;\n  }\n\n  if (!opts.engines[this.ext]) {\n    // load engine\n    var mod = this.ext.slice(1)\n    debug('require \"%s\"', mod)\n\n    // default engine export\n    var fn = require(mod).__express\n\n    if (typeof fn !== 'function') {\n      throw new Error('Module \"' + mod + '\" does not provide a view engine.')\n    }\n\n    opts.engines[this.ext] = fn\n  }\n\n  // store loaded engine\n  this.engine = opts.engines[this.ext];\n\n  // lookup path\n  this.path = this.lookup(fileName);\n}\n\n/**\n * Lookup view by the given `name`\n *\n * @param {string} name\n * @private\n */\n\nView.prototype.lookup = function lookup(name) {\n  var path;\n  var roots = [].concat(this.root);\n\n  debug('lookup \"%s\"', name);\n\n  for (var i = 0; i < roots.length && !path; i++) {\n    var root = roots[i];\n\n    // resolve the path\n    var loc = resolve(root, name);\n    var dir = dirname(loc);\n    var file = basename(loc);\n\n    // resolve the file\n    path = this.resolve(dir, file);\n  }\n\n  return path;\n};\n\n/**\n * Render with the given options.\n *\n * @param {object} options\n * @param {function} callback\n * @private\n */\n\nView.prototype.render = function render(options, callback) {\n  debug('render \"%s\"', this.path);\n  this.engine(this.path, options, callback);\n};\n\n/**\n * Resolve the file within the given directory.\n *\n * @param {string} dir\n * @param {string} file\n * @private\n */\n\nView.prototype.resolve = function resolve(dir, file) {\n  var ext = this.ext;\n\n  // <path>.<ext>\n  var path = join(dir, file);\n  var stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n\n  // <path>/index.<ext>\n  path = join(dir, basename(file, ext), 'index' + ext);\n  stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n};\n\n/**\n * Return a stat, maybe.\n *\n * @param {string} path\n * @return {fs.Stats}\n * @private\n */\n\nfunction tryStat(path) {\n  debug('stat \"%s\"', path);\n\n  try {\n    return fs.statSync(path);\n  } catch (e) {\n    return undefined;\n  }\n}\n\n// Path: lib/middleware/query.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar merge = require('utils-merge')\nvar parseUrl = require('parseurl');\nvar qs = require('qs');\n\n/**\n * @param {Object} options\n * @return {Function}\n * @api public\n */\n\nmodule.exports = function query(options) {\n  var opts = merge({}, options)\n  var queryparse = qs.parse;\n\n  if (typeof options === 'function') {\n    queryparse = options;\n    opts = undefined;\n  }\n\n  if (opts !== undefined && opts.allowPrototypes === undefined) {\n    // back-compat for qs module\n    opts.allowPrototypes = true;\n  }\n\n  return function query(req, res, next){\n    if (!req.query) {\n      var val = parseUrl(req).query;\n      req.query = queryparse(val, opts);\n    }\n\n    next();\n  };\n};\n\n// Path: lib/application.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar finalhandler = require('finalhandler');\nvar Router = require('./router');\nvar methods = require('methods');\nvar middleware = require('./middleware/init');\nvar query = require('./middleware/query');\nvar debug = require('debug')('express:application');\nvar View = require('./view');\nvar http = require('http');\nvar compileETag = require('./utils').compileETag;\nvar compileQueryParser = require('./utils').compileQueryParser;\nvar compileTrust = require('./utils').compileTrust;\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar merge = require('utils-merge');\nvar resolve = require('path').resolve;\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty\nvar slice = Array.prototype.slice;\n\n/**\n * Application prototype.\n */\n\nvar app = exports = module.exports = {};\n\n/**\n * Variable for trust proxy inheritance back-compat\n * @private\n */\n\nvar trustProxyDefaultSymbol = '@@symbol:trust_proxy_default';\n\n/**\n * Initialize the server.\n *\n *   - setup default configuration\n *   - setup default middleware\n *   - setup route reflection methods\n *\n * @private\n */\n\napp.init = function init() {\n  this.cache = {};\n  this.engines = {};\n  this.settings = {};\n\n  this.defaultConfiguration();\n};\n\n/**\n * Initialize application configuration.\n * @private\n */\n\napp.defaultConfiguration = function defaultConfiguration() {\n  var env = process.env.NODE_ENV || 'development';\n\n  // default settings\n  this.enable('x-powered-by');\n...\n", "needle_token_start": 504, "needle_token_end": 1071, "code_context_ntokens": 15428, "output": ["```javascript\nfunction sendfile(res, file, options, callback) {\n  var done = false;\n  var streaming;\n\n  // request aborted\n  function onaborted() {\n    if (done) return;\n    done = true;\n\n    var err = new Error('Request aborted');\n    err.code = 'ECONNABORTED';\n    callback(err);\n  }\n\n  // directory\n  function ondirectory() {\n    if (done) return;\n    done = true;\n\n    var err = new Error('EISDIR, read');\n    err.code = 'EISDIR';\n    callback(err);\n  }\n\n  // errors\n  function onerror(err) {\n    if (done) return;\n    done = true;\n    callback(err);\n  }\n\n  // ended\n  function onend() {\n    if (done) return;\n    done = true;\n    callback();\n  }\n\n  // file\n  function onfile() {\n    streaming = false;\n  }\n\n  // finished\n  function onfinish(err) {\n    if (err && err.code === 'ECONNRESET') return onaborted();\n    if (err) return onerror(err);\n    if (done) return;\n\n    setImmediate(function () {\n      if (streaming !== false && !done) {\n        onaborted();\n        return;\n      }\n\n      if (done) return;\n      done = true;\n      callback();\n    });\n  }\n\n  // streaming\n  function onstream() {\n    streaming = true;\n  }\n\n  file.on('directory', ondirectory);\n  file.on('end', onend);\n  file.on('error', onerror);\n  file.on('file', onfile);\n  file.on('stream', onstream);\n  onFinished(res, onfinish);\n\n  if (options.headers) {\n    // set headers on successful transfer\n    file.on('headers', function headers(res) {\n      var obj = options.headers;\n      var keys = Object.keys(obj);\n\n      for (var i = 0; i < keys.length; i++) {\n        var k = keys[i];\n        res.setHeader(k, obj[k]);\n      }\n    });\n  }\n\n  // pipe\n  file.pipe(res);\n}\n```"]}
{"repo": "expressjs/express", "name": "trim_prefix", "language": "typescript", "path": "lib/router/index.js", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to adjust the URL of a request by removing a specified prefix that matches the beginning of the request's path. This is typically used in middleware operations where the base part of the URL needs to be stripped before further processing.\n2. **Input**: The function takes four parameters: a middleware layer object, an error object, a string representing the path prefix to be removed, and the full path of the request.\n3. **Output**: There is no direct output returned; however, the function modifies the request URL and potentially handles errors or continues request handling based on the presence of a prefix match and error conditions.\n4. **Procedure**: \n   - First, the function checks if the prefix is non-empty and matches the start of the request's path.\n   - If the prefix does not match or does not properly break on a path separator, an error handler is invoked.\n   - If a match is confirmed, the prefix is removed from the request's URL, and adjustments are made to ensure the URL starts with a slash and to set up the base URL correctly.\n   - Depending on whether an error was initially present, the function either calls an error handler or proceeds with the normal request handling.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Route = require('./route');\nvar Layer = require('./layer');\nvar methods = require('methods');\nvar mixin = require('utils-merge');\nvar debug = require('debug')('express:router');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar parseUrl = require('parseurl');\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar objectRegExp = /^\\[object (\\S+)\\]$/;\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n\n/**\n * Map the given param placeholder `name`(s) to the given callback.\n *\n * Parameter mapping is used to provide pre-conditions to routes\n * which use normalized placeholders. For example a _:user_id_ parameter\n * could automatically load a user's information from the database without\n * any additional code,\n *\n * The callback uses the same signature as middleware, the only difference\n * being that the value of the placeholder is passed, in this case the _id_\n * of the user. Once the `next()` function is invoked, just like middleware\n * it will continue on to execute the route, or subsequent parameter functions.\n *\n * Just like in middleware, you must either respond to the request or call next\n * to avoid stalling the request.\n *\n *  app.param('user_id', function(req, res, next, id){\n *    User.find(id, function(err, user){\n *      if (err) {\n *        return next(err);\n *      } else if (!user) {\n *        return next(new Error('failed to load user'));\n *      }\n *      req.user = user;\n *      next();\n *    });\n *  });\n *\n * @param {String} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\nproto.param = function param(name, fn) {\n  // param logic\n  if (typeof name === 'function') {\n    deprecate('router.param(fn): Refactor to use path params');\n    this._params.push(name);\n    return;\n  }\n\n  // apply param functions\n  var params = this._params;\n  var len = params.length;\n  var ret;\n\n  if (name[0] === ':') {\n    deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.slice(1)) + ', fn) instead')\n    name = name.slice(1)\n  }\n\n  for (var i = 0; i < len; ++i) {\n    if (ret = params[i](name, fn)) {\n      fn = ret;\n    }\n  }\n\n  // ensure we end up with a\n  // middleware function\n  if ('function' !== typeof fn) {\n    throw new Error('invalid param() call for ' + name + ', got ' + fn);\n  }\n\n  (this.params[name] = this.params[name] || []).push(fn);\n  return this;\n};\n\n/**\n * Dispatch a req, res into the router.\n * @private\n */\n\nproto.handle = function handle(req, res, out) {\n  var self = this;\n\n  debug('dispatching %s %s', req.method, req.url);\n\n  var idx = 0;\n  var protohost = getProtohost(req.url) || ''\n  var removed = '';\n  var slashAdded = false;\n  var sync = 0\n  var paramcalled = {};\n\n  // store options for OPTIONS request\n  // only used if OPTIONS request\n  var options = [];\n\n  // middleware and routes\n  var stack = self.stack;\n\n  // manage inter-router variables\n  var parentParams = req.params;\n  var parentUrl = req.baseUrl || '';\n  var done = restore(out, req, 'baseUrl', 'next', 'params');\n\n  // setup next layer\n  req.next = next;\n\n  // for options requests, respond with a default if nothing else responds\n  if (req.method === 'OPTIONS') {\n    done = wrap(done, function(old, err) {\n      if (err || options.length === 0) return old(err);\n      sendOptionsResponse(res, options, old);\n    });\n  }\n\n  // setup basic req values\n  req.baseUrl = parentUrl;\n  req.originalUrl = req.originalUrl || req.url;\n\n  next();\n\n  function next(err) {\n    var layerError = err === 'route'\n      ? null\n      : err;\n\n    // remove added slash\n    if (slashAdded) {\n      req.url = req.url.slice(1)\n      slashAdded = false;\n    }\n\n    // restore altered req.url\n    if (removed.length !== 0) {\n      req.baseUrl = parentUrl;\n      req.url = protohost + removed + req.url.slice(protohost.length)\n      removed = '';\n    }\n\n    // signal to exit router\n    if (layerError === 'router') {\n      setImmediate(done, null)\n      return\n    }\n\n    // no more matching layers\n    if (idx >= stack.length) {\n      setImmediate(done, layerError);\n      return;\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    // get pathname of request\n    var path = getPathname(req);\n\n    if (path == null) {\n      return done(layerError);\n    }\n\n    // find next matching layer\n    var layer;\n    var match;\n    var route;\n\n    while (match !== true && idx < stack.length) {\n      layer = stack[idx++];\n      match = matchLayer(layer, path);\n      route = layer.route;\n\n      if (typeof match !== 'boolean') {\n        // hold on to layerError\n        layerError = layerError || match;\n      }\n\n      if (match !== true) {\n        continue;\n      }\n\n      if (!route) {\n        // process non-route handlers normally\n        continue;\n      }\n\n      if (layerError) {\n        // routes do not match with a pending error\n        match = false;\n        continue;\n      }\n\n      var method = req.method;\n      var has_method = route._handles_method(method);\n\n      // build up automatic options response\n      if (!has_method && method === 'OPTIONS') {\n        appendMethods(options, route._options());\n      }\n\n      // don't even bother matching route\n      if (!has_method && method !== 'HEAD') {\n        match = false;\n      }\n    }\n\n    // no match\n    if (match !== true) {\n      return done(layerError);\n    }\n\n    // store route for dispatch on change\n    if (route) {\n      req.route = route;\n    }\n\n    // Capture one-time layer values\n    req.params = self.mergeParams\n      ? mergeParams(layer.params, parentParams)\n      : layer.params;\n    var layerPath = layer.path;\n\n    // this should be done for the layer\n    self.process_params(layer, paramcalled, req, res, function (err) {\n      if (err) {\n        next(layerError || err)\n      } else if (route) {\n        layer.handle_request(req, res, next)\n      } else {\n        trim_prefix(layer, layerError, layerPath, path)\n      }\n\n      sync = 0\n    });\n  }\n\n  \nfunction trim_prefix(layer, layerError, layerPath, path) {\n    if (layerPath.length !== 0) {\n      // Validate path is a prefix match\n      if (layerPath !== path.slice(0, layerPath.length)) {\n        next(layerError)\n        return\n      }\n\n      // Validate path breaks on a path separator\n      var c = path[layerPath.length]\n      if (c && c !== '/' && c !== '.') return next(layerError)\n\n      // Trim off the part of the url that matches the route\n      // middleware (.use stuff) needs to have the path stripped\n      debug('trim prefix (%s) from url %s', layerPath, req.url);\n      removed = layerPath;\n      req.url = protohost + req.url.slice(protohost.length + removed.length)\n\n      // Ensure leading slash\n      if (!protohost && req.url[0] !== '/') {\n        req.url = '/' + req.url;\n        slashAdded = true;\n      }\n\n      // Setup base URL (no trailing slash)\n      req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'\n        ? removed.substring(0, removed.length - 1)\n        : removed);\n    }\n\n    debug('%s %s : %s', layer.name, layerPath, req.originalUrl);\n\n    if (layerError) {\n      layer.handle_error(layerError, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n  }\n};\n\n/**\n * Process any parameters for the layer.\n * @private\n */\n\nproto.process_params = function process_params(layer, called, req, res, done) {\n  var params = this.params;\n\n  // captured parameters from the layer, keys and values\n  var keys = layer.keys;\n\n  // fast track\n  if (!keys || keys.length === 0) {\n    return done();\n  }\n\n  var i = 0;\n  var name;\n  var paramIndex = 0;\n  var key;\n  var paramVal;\n  var paramCallbacks;\n  var paramCalled;\n\n  // process params in order\n  // param callbacks can be async\n  function param(err) {\n    if (err) {\n      return done(err);\n    }\n\n    if (i >= keys.length ) {\n      return done();\n    }\n\n    paramIndex = 0;\n    key = keys[i++];\n    name = key.name;\n    paramVal = req.params[name];\n    paramCallbacks = params[name];\n    paramCalled = called[name];\n\n    if (paramVal === undefined || !paramCallbacks) {\n      return param();\n    }\n\n    // param previously called with same value or error occurred\n    if (paramCalled && (paramCalled.match === paramVal\n      || (paramCalled.error && paramCalled.error !== 'route'))) {\n      // restore value\n      req.params[name] = paramCalled.value;\n\n      // next param\n      return param(paramCalled.error);\n    }\n\n    called[name] = paramCalled = {\n      error: null,\n      match: paramVal,\n      value: paramVal\n    };\n\n    paramCallback();\n  }\n\n  // single param callbacks\n  function paramCallback(err) {\n    var fn = paramCallbacks[paramIndex++];\n\n    // store updated value\n    paramCalled.value = req.params[key.name];\n\n    if (err) {\n      // store error\n      paramCalled.error = err;\n      param(err);\n      return;\n    }\n\n    if (!fn) return param();\n\n    try {\n      fn(req, res, paramCallback, paramVal, key.name);\n    } catch (e) {\n      paramCallback(e);\n    }\n  }\n\n  param();\n};\n\n/**\n * Use the given middleware function, with optional path, defaulting to \"/\".\n *\n * Use (like `.all`) will run for any http METHOD, but it will not add\n * handlers for those methods so OPTIONS requests will not consider `.use`\n * functions even if they could respond.\n *\n * The other difference is that _route_ path is stripped and not visible\n * to the handler function. The main effect of this feature is that mounted\n * handlers can operate without any code changes regardless of the \"prefix\"\n * pathname.\n *\n * @public\n */\n\nproto.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate router.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var callbacks = flatten(slice.call(arguments, offset));\n\n  if (callbacks.length === 0) {\n    throw new TypeError('Router.use() requires a middleware function')\n  }\n\n  for (var i = 0; i < callbacks.length; i++) {\n    var fn = callbacks[i];\n\n    if (typeof fn !== 'function') {\n      throw new TypeError('Router.use() requires a middleware function but got a ' + gettype(fn))\n    }\n\n    // add the middleware\n    debug('use %o %s', path, fn.name || '<anonymous>')\n\n    var layer = new Layer(path, {\n      sensitive: this.caseSensitive,\n      strict: false,\n      end: false\n    }, fn);\n\n    layer.route = undefined;\n\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\n/**\n * Create a new Route for the given path.\n *\n * Each route contains a separate middleware stack and VERB handlers.\n *\n * See the Route api documentation for details on adding handlers\n * and middleware to routes.\n *\n * @param {String} path\n * @return {Route}\n * @public\n */\n\nproto.route = function route(path) {\n  var route = new Route(path);\n\n  var layer = new Layer(path, {\n    sensitive: this.caseSensitive,\n    strict: this.strict,\n    end: true\n  }, route.dispatch.bind(route));\n\n  layer.route = route;\n\n  this.stack.push(layer);\n  return route;\n};\n\n// create Router#VERB functions\nmethods.concat('all').forEach(function(method){\n  proto[method] = function(path){\n    var route = this.route(path)\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n// append methods to a list of methods\nfunction appendMethods(list, addition) {\n  for (var i = 0; i < addition.length; i++) {\n    var method = addition[i];\n    if (list.indexOf(method) === -1) {\n      list.push(method);\n    }\n  }\n}\n\n// get pathname of request\nfunction getPathname(req) {\n  try {\n    return parseUrl(req).pathname;\n  } catch (err) {\n    return undefined;\n  }\n}\n\n// Get get protocol + host for a URL\nfunction getProtohost(url) {\n  if (typeof url !== 'string' || url.length === 0 || url[0] === '/') {\n    return undefined\n  }\n\n  var searchIndex = url.indexOf('?')\n  var pathLength = searchIndex !== -1\n    ? searchIndex\n    : url.length\n  var fqdnIndex = url.slice(0, pathLength).indexOf('://')\n\n  return fqdnIndex !== -1\n    ? url.substring(0, url.indexOf('/', 3 + fqdnIndex))\n    : undefined\n}\n\n// get type for error message\nfunction gettype(obj) {\n  var type = typeof obj;\n\n  if (type !== 'object') {\n    return type;\n  }\n\n  // inspect [[Class]] for objects\n  return toString.call(obj)\n    .replace(objectRegExp, '$1');\n}\n\n/**\n * Match path to a layer.\n *\n * @param {Layer} layer\n * @param {string} path\n * @private\n */\n\nfunction matchLayer(layer, path) {\n  try {\n    return layer.match(path);\n  } catch (err) {\n    return err;\n  }\n}\n\n// merge params with parent params\nfunction mergeParams(params, parent) {\n  if (typeof parent !== 'object' || !parent) {\n    return params;\n  }\n\n  // make copy of parent for base\n  var obj = mixin({}, parent);\n\n  // simple non-numeric merging\n  if (!(0 in params) || !(0 in parent)) {\n    return mixin(obj, params);\n  }\n\n  var i = 0;\n  var o = 0;\n\n  // determine numeric gaps\n  while (i in params) {\n    i++;\n  }\n\n  while (o in parent) {\n    o++;\n  }\n\n  // offset numeric indices in params before merge\n  for (i--; i >= 0; i--) {\n    params[i + o] = params[i];\n\n    // create holes for the merge when necessary\n    if (i < o) {\n      delete params[i];\n    }\n  }\n\n  return mixin(obj, params);\n}\n\n// restore obj props after function\nfunction restore(fn, obj) {\n  var props = new Array(arguments.length - 2);\n  var vals = new Array(arguments.length - 2);\n\n  for (var i = 0; i < props.length; i++) {\n    props[i] = arguments[i + 2];\n    vals[i] = obj[props[i]];\n  }\n\n  return function () {\n    // restore vals\n    for (var i = 0; i < props.length; i++) {\n      obj[props[i]] = vals[i];\n    }\n\n    return fn.apply(this, arguments);\n  };\n}\n\n// send an OPTIONS response\nfunction sendOptionsResponse(res, options, next) {\n  try {\n    var body = options.join(',');\n    res.set('Allow', body);\n    res.send(body);\n  } catch (err) {\n    next(err);\n  }\n}\n\n// wrap a function\nfunction wrap(old, fn) {\n  return function proxy() {\n    var args = new Array(arguments.length + 1);\n\n    args[0] = old;\n    for (var i = 0, len = arguments.length; i < len; i++) {\n      args[i + 1] = arguments[i];\n    }\n\n    fn.apply(this, args);\n  };\n}\n\n// Path: lib/middleware/init.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Initialization middleware, exposing the\n * request and response to each other, as well\n * as defaulting the X-Powered-By header field.\n *\n * @param {Function} app\n * @return {Function}\n * @api private\n */\n\nexports.init = function(app){\n  return function expressInit(req, res, next){\n    if (app.enabled('x-powered-by')) res.setHeader('X-Powered-By', 'Express');\n    req.res = res;\n    res.req = req;\n    req.next = next;\n\n    setPrototypeOf(req, app.request)\n    setPrototypeOf(res, app.response)\n\n    res.locals = res.locals || Object.create(null);\n\n    next();\n  };\n};\n\n\n// Path: lib/view.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:view');\nvar path = require('path');\nvar fs = require('fs');\n\n/**\n * Module variables.\n * @private\n */\n\nvar dirname = path.dirname;\nvar basename = path.basename;\nvar extname = path.extname;\nvar join = path.join;\nvar resolve = path.resolve;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = View;\n\n/**\n * Initialize a new `View` with the given `name`.\n *\n * Options:\n *\n *   - `defaultEngine` the default template engine name\n *   - `engines` template engine require() cache\n *   - `root` root path for view lookup\n *\n * @param {string} name\n * @param {object} options\n * @public\n */\n\nfunction View(name, options) {\n  var opts = options || {};\n\n  this.defaultEngine = opts.defaultEngine;\n  this.ext = extname(name);\n  this.name = name;\n  this.root = opts.root;\n\n  if (!this.ext && !this.defaultEngine) {\n    throw new Error('No default engine was specified and no extension was provided.');\n  }\n\n  var fileName = name;\n\n  if (!this.ext) {\n    // get extension from default engine name\n    this.ext = this.defaultEngine[0] !== '.'\n      ? '.' + this.defaultEngine\n      : this.defaultEngine;\n\n    fileName += this.ext;\n  }\n\n  if (!opts.engines[this.ext]) {\n    // load engine\n    var mod = this.ext.slice(1)\n    debug('require \"%s\"', mod)\n\n    // default engine export\n    var fn = require(mod).__express\n\n    if (typeof fn !== 'function') {\n      throw new Error('Module \"' + mod + '\" does not provide a view engine.')\n    }\n\n    opts.engines[this.ext] = fn\n  }\n\n  // store loaded engine\n  this.engine = opts.engines[this.ext];\n\n  // lookup path\n  this.path = this.lookup(fileName);\n}\n\n/**\n * Lookup view by the given `name`\n *\n * @param {string} name\n * @private\n */\n\nView.prototype.lookup = function lookup(name) {\n  var path;\n  var roots = [].concat(this.root);\n\n  debug('lookup \"%s\"', name);\n\n  for (var i = 0; i < roots.length && !path; i++) {\n    var root = roots[i];\n\n    // resolve the path\n    var loc = resolve(root, name);\n    var dir = dirname(loc);\n    var file = basename(loc);\n\n    // resolve the file\n    path = this.resolve(dir, file);\n  }\n\n  return path;\n};\n\n/**\n * Render with the given options.\n *\n * @param {object} options\n * @param {function} callback\n * @private\n */\n\nView.prototype.render = function render(options, callback) {\n  debug('render \"%s\"', this.path);\n  this.engine(this.path, options, callback);\n};\n\n/**\n * Resolve the file within the given directory.\n *\n * @param {string} dir\n * @param {string} file\n * @private\n */\n\nView.prototype.resolve = function resolve(dir, file) {\n  var ext = this.ext;\n\n  // <path>.<ext>\n  var path = join(dir, file);\n  var stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n\n  // <path>/index.<ext>\n  path = join(dir, basename(file, ext), 'index' + ext);\n  stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n};\n\n/**\n * Return a stat, maybe.\n *\n * @param {string} path\n * @return {fs.Stats}\n * @private\n */\n\nfunction tryStat(path) {\n  debug('stat \"%s\"', path);\n\n  try {\n    return fs.statSync(path);\n  } catch (e) {\n    return undefined;\n  }\n}\n\n// Path: lib/middleware/query.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar merge = require('utils-merge')\nvar parseUrl = require('parseurl');\nvar qs = require('qs');\n\n/**\n * @param {Object} options\n * @return {Function}\n * @api public\n */\n\nmodule.exports = function query(options) {\n  var opts = merge({}, options)\n  var queryparse = qs.parse;\n\n  if (typeof options === 'function') {\n    queryparse = options;\n    opts = undefined;\n  }\n\n  if (opts !== undefined && opts.allowPrototypes === undefined) {\n    // back-compat for qs module\n    opts.allowPrototypes = true;\n  }\n\n  return function query(req, res, next){\n    if (!req.query) {\n      var val = parseUrl(req).query;\n      req.query = queryparse(val, opts);\n    }\n\n    next();\n  };\n};\n\n// Path: lib/application.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar finalhandler = require('finalhandler');\nvar Router = require('./router');\nvar methods = require('methods');\nvar middleware = require('./middleware/init');\nvar query = require('./middleware/query');\nvar debug = require('debug')('express:application');\nvar View = require('./view');\nvar http = require('http');\nvar compileETag = require('./utils').compileETag;\nvar compileQueryParser = require('./utils').compileQueryParser;\nvar compileTrust = require('./utils').compileTrust;\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar merge = require('utils-merge');\nvar resolve = require('path').resolve;\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty\nvar slice = Array.prototype.slice;\n\n/**\n * Application prototype.\n */\n\nvar app = exports = module.exports = {};\n\n/**\n * Variable for trust proxy inheritance back-compat\n * @private\n */\n\nvar trustProxyDefaultSymbol = '@@symbol:trust_proxy_default';\n\n/**\n * Initialize the server.\n *\n *   - setup default configuration\n *   - setup default middleware\n *   - setup route reflection methods\n *\n * @private\n */\n\napp.init = function init() {\n  this.cache = {};\n  this.engines = {};\n  this.settings = {};\n\n  this.defaultConfiguration();\n};\n\n/**\n * Initialize application configuration.\n * @private\n */\n\napp.defaultConfiguration = function defaultConfiguration() {\n  var env = process.env.NODE_ENV || 'development';\n\n  // default settings\n  this.enable('x-powered-by');\n  this.set('etag', 'weak');\n  this.set('env', env);\n  this.set('query parser', 'extended');\n  this.set('subdomain offset', 2);\n  this.set('trust proxy', false);\n\n  // trust proxy inherit back-compat\n  Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n    configurable: true,\n    value: true\n  });\n\n  debug('booting in %s mode', env);\n\n  this.on('mount', function onmount(parent) {\n    // inherit trust proxy\n    if (this.settings[trustProxyDefaultSymbol] === true\n      && typeof parent.settings['trust proxy fn'] === 'function') {\n      delete this.settings['trust proxy'];\n      delete this.settings['trust proxy fn'];\n    }\n\n    // inherit protos\n    setPrototypeOf(this.request, parent.request)\n    setPrototypeOf(this.response, parent.response)\n    setPrototypeOf(this.engines, parent.engines)\n    setPrototypeOf(this.settings, parent.settings)\n  });\n\n  // setup locals\n  this.locals = Object.create(null);\n\n  // top-most app is mounted at /\n  this.mountpath = '/';\n\n  // default locals\n  this.locals.settings = this.settings;\n\n  // default configuration\n  this.set('view', View);\n  this.set('views', resolve('views'));\n  this.set('jsonp callback name', 'callback');\n\n  if (env === 'production') {\n    this.enable('view cache');\n  }\n\n  Object.defineProperty(this, 'router', {\n    get: function() {\n      throw new Error('\\'app.router\\' is deprecated!\\nPlease see the 3.x to 4.x migration guide for details on how to update your app.');\n    }\n  });\n};\n\n/**\n * lazily adds the base router if it has not yet been added.\n *\n * We cannot add the base router in the defaultConfiguration because\n * it reads app settings which might be set after that has run.\n *\n * @private\n */\napp.lazyrouter = function lazyrouter() {\n  if (!this._router) {\n    this._router = new Router({\n      caseSensitive: this.enabled('case sensitive routing'),\n      strict: this.enabled('strict routing')\n    });\n\n    this._router.use(query(this.get('query parser fn')));\n    this._router.use(middleware.init(this));\n  }\n};\n\n/**\n * Dispatch a req, res pair into the application. Starts pipeline processing.\n *\n * If no callback is provided, then default error handlers will respond\n * in the event of an error bubbling through the stack.\n *\n * @private\n */\n\napp.handle = function handle(req, res, callback) {\n  var router = this._router;\n\n  // final handler\n  var done = callback || finalhandler(req, res, {\n    env: this.get('env'),\n    onerror: logerror.bind(this)\n  });\n\n  // no routes\n  if (!router) {\n    debug('no routes defined on app');\n    done();\n    return;\n  }\n\n  router.handle(req, res, done);\n};\n\n/**\n * Proxy `Router#use()` to add middleware to the app router.\n * See Router#use() documentation for details.\n *\n * If the _fn_ parameter is an express app, then it will be\n * mounted at the _route_ specified.\n *\n * @public\n */\n\napp.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate app.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var fns = flatten(slice.call(arguments, offset));\n\n  if (fns.length === 0) {\n    throw new TypeError('app.use() requires a middleware function')\n  }\n\n  // setup router\n  this.lazyrouter();\n  var router = this._router;\n\n  fns.forEach(function (fn) {\n    // non-express app\n    if (!fn || !fn.handle || !fn.set) {\n      return router.use(path, fn);\n    }\n\n    debug('.use app under %s', path);\n    fn.mountpath = path;\n    fn.parent = this;\n\n    // restore .app property on req and res\n    router.use(path, function mounted_app(req, res, next) {\n      var orig = req.app;\n      fn.handle(req, res, function (err) {\n        setPrototypeOf(req, orig.request)\n        setPrototypeOf(res, orig.response)\n        next(err);\n      });\n    });\n\n    // mounted an app\n    fn.emit('mount', this);\n  }, this);\n\n  return this;\n};\n\n/**\n * Proxy to the app `Router#route()`\n * Returns a new `Route` instance for the _path_.\n *\n * Routes are isolated middleware stacks for specific paths.\n * See the Route api docs for details.\n *\n * @public\n */\n\napp.route = function route(path) {\n  this.lazyrouter();\n  return this._router.route(path);\n};\n\n/**\n * Register the given template engine callback `fn`\n * as `ext`.\n *\n * By default will `require()` the engine based on the\n * file extension. For example if you try to render\n * a \"foo.ejs\" file Express will invoke the following internally:\n *\n *     app.engine('ejs', require('ejs').__express);\n *\n * For engines that do not provide `.__express` out of the box,\n * or if you wish to \"map\" a different extension to the template engine\n * you may use this method. For example mapping the EJS template engine to\n * \".html\" files:\n *\n *     app.engine('html', require('ejs').renderFile);\n *\n * In this case EJS provides a `.renderFile()` method with\n * the same signature that Express expects: `(path, options, callback)`,\n * though note that it aliases this method as `ejs.__express` internally\n * so if you're using \".ejs\" extensions you don't need to do anything.\n *\n * Some template engines do not follow this convention, the\n * [Consolidate.js](https://github.com/tj/consolidate.js)\n * library was created to map all of node's popular template\n * engines to follow this convention, thus allowing them to\n * work seamlessly within Express.\n *\n * @param {String} ext\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.engine = function engine(ext, fn) {\n  if (typeof fn !== 'function') {\n    throw new Error('callback function required');\n  }\n\n  // get file extension\n  var extension = ext[0] !== '.'\n    ? '.' + ext\n    : ext;\n\n  // store engine\n  this.engines[extension] = fn;\n\n  return this;\n};\n\n/**\n * Proxy to `Router#param()` with one added api feature. The _name_ parameter\n * can be an array of names.\n *\n * See the Router#param() docs for more details.\n *\n * @param {String|Array} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.param = function param(name, fn) {\n  this.lazyrouter();\n\n  if (Array.isArray(name)) {\n    for (var i = 0; i < name.length; i++) {\n      this.param(name[i], fn);\n    }\n\n    return this;\n  }\n\n  this._router.param(name, fn);\n\n  return this;\n};\n\n/**\n * Assign `setting` to `val`, or return `setting`'s value.\n *\n *    app.set('foo', 'bar');\n *    app.set('foo');\n *    // => \"bar\"\n *\n * Mounted servers inherit their parent server's settings.\n *\n * @param {String} setting\n * @param {*} [val]\n * @return {Server} for chaining\n * @public\n */\n\napp.set = function set(setting, val) {\n  if (arguments.length === 1) {\n    // app.get(setting)\n    var settings = this.settings\n\n    while (settings && settings !== Object.prototype) {\n      if (hasOwnProperty.call(settings, setting)) {\n        return settings[setting]\n      }\n\n      settings = Object.getPrototypeOf(settings)\n    }\n\n    return undefined\n  }\n\n  debug('set \"%s\" to %o', setting, val);\n\n  // set value\n  this.settings[setting] = val;\n\n  // trigger matched settings\n  switch (setting) {\n    case 'etag':\n      this.set('etag fn', compileETag(val));\n      break;\n    case 'query parser':\n      this.set('query parser fn', compileQueryParser(val));\n      break;\n    case 'trust proxy':\n      this.set('trust proxy fn', compileTrust(val));\n\n      // trust proxy inherit back-compat\n      Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n        configurable: true,\n        value: false\n      });\n\n      break;\n  }\n\n  return this;\n};\n\n/**\n * Return the app's absolute pathname\n * based on the parent(s) that have\n * mounted it.\n *\n * For example if the application was\n * mounted as \"/admin\", which itself\n * was mounted as \"/blog\" then the\n * return value would be \"/blog/admin\".\n *\n * @return {String}\n * @private\n */\n\napp.path = function path() {\n  return this.parent\n    ? this.parent.path() + this.mountpath\n    : '';\n};\n\n/**\n * Check if `setting` is enabled (truthy).\n *\n *    app.enabled('foo')\n *    // => false\n *\n *    app.enable('foo')\n *    app.enabled('foo')\n *    // => true\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.enabled = function enabled(setting) {\n  return Boolean(this.set(setting));\n};\n\n/**\n * Check if `setting` is disabled.\n *\n *    app.disabled('foo')\n *    // => true\n *\n *    app.enable('foo')\n *    app.disabled('foo')\n *    // => false\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.disabled = function disabled(setting) {\n  return !this.set(setting);\n};\n\n/**\n * Enable `setting`.\n *\n * @param {String} setting\n * @return {app} for chaining\n * @public\n */\n\napp.enable = function enable(setting) {\n  return this.set(setting, true);\n};\n\n/**\n * Disable `setting`.\n *\n * @param {String} setting\n * @return {app} for chaining\n * @public\n */\n\napp.disable = function disable(setting) {\n  return this.set(setting, false);\n};\n\n/**\n * Delegate `.VERB(...)` calls to `router.VERB(...)`.\n */\n\nmethods.forEach(function(method){\n  app[method] = function(path){\n    if (method === 'get' && arguments.length === 1) {\n      // app.get(setting)\n      return this.set(path);\n    }\n\n    this.lazyrouter();\n\n    var route = this._router.route(path);\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n/**\n * Special-cased \"all\" method, applying the given route `path`,\n * middleware, and callback to _every_ HTTP method.\n *\n * @param {String} path\n * @param {Function} ...\n * @return {app} for chaining\n * @public\n */\n\napp.all = function all(path) {\n  this.lazyrouter();\n\n  var route = this._router.route(path);\n  var args = slice.call(arguments, 1);\n\n  for (var i = 0; i < methods.length; i++) {\n    route[methods[i]].apply(route, args);\n  }\n\n  return this;\n};\n\n// del -> delete alias\n\napp.del = deprecate.function(app.delete, 'app.del: Use app.delete instead');\n\n/**\n * Render the given view `name` name with `options`\n * and a callback accepting an error and the\n * rendered template string.\n *\n * Example:\n *\n *    app.render('email', { name: 'Tobi' }, function(err, html){\n *      // ...\n *    })\n *\n * @param {String} name\n * @param {Object|Function} options or fn\n * @param {Function} callback\n * @public\n */\n\napp.render = function render(name, options, callback) {\n  var cache = this.cache;\n  var done = callback;\n  var engines = this.engines;\n  var opts = options;\n  var renderOptions = {};\n  var view;\n\n  // support callback function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // merge app.locals\n  merge(renderOptions, this.locals);\n\n  // merge options._locals\n  if (opts._locals) {\n    merge(renderOptions, opts._locals);\n  }\n\n  // merge options\n  merge(renderOptions, opts);\n\n  // set .cache unless explicitly provided\n  if (renderOptions.cache == null) {\n    renderOptions.cache = this.enabled('view cache');\n  }\n\n  // primed cache\n  if (renderOptions.cache) {\n    view = cache[name];\n  }\n\n  // view\n  if (!view) {\n    var View = this.get('view');\n\n    view = new View(name, {\n      defaultEngine: this.get('view engine'),\n      root: this.get('views'),\n      engines: engines\n    });\n\n    if (!view.path) {\n      var dirs = Array.isArray(view.root) && view.root.length > 1\n        ? 'directories \"' + view.root.slice(0, -1).join('\", \"') + '\" or \"' + view.root[view.root.length - 1] + '\"'\n        : 'directory \"' + view.root + '\"'\n      var err = new Error('Failed to lookup view \"' + name + '\" in views ' + dirs);\n      err.view = view;\n      return done(err);\n    }\n\n    // prime the cache\n    if (renderOptions.cache) {\n      cache[name] = view;\n    }\n  }\n\n  // render\n  tryRender(view, renderOptions, done);\n};\n\n/**\n * Listen for connections.\n *\n * A node `http.Server` is returned, with this\n * application (which is a `Function`) as its\n * callback. If you wish to create both an HTTP\n * and HTTPS server you may do so with the \"http\"\n * and \"https\" modules as shown here:\n *\n *    var http = require('http')\n *      , https = require('https')\n *      , express = require('express')\n *      , app = express();\n *\n *    http.createServer(app).listen(80);\n *    https.createServer({ ... }, app).listen(443);\n *\n * @return {http.Server}\n * @public\n */\n\napp.listen = function listen() {\n  var server = http.createServer(this);\n  return server.listen.apply(server, arguments);\n};\n\n/**\n * Log error using console.error.\n *\n * @param {Error} err\n * @private\n */\n\nfunction logerror(err) {\n  /* istanbul ignore next */\n  if (this.get('env') !== 'test') console.error(err.stack || err.toString());\n}\n\n/**\n * Try rendering a view.\n * @private\n */\n\nfunction tryRender(view, options, callback) {\n  try {\n    view.render(options, callback);\n  } catch (err) {\n    callback(err);\n  }\n}\n\n// Path: lib/express.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar bodyParser = require('body-parser')\nvar EventEmitter = require('events').EventEmitter;\nvar mixin = require('merge-descriptors');\nvar proto = require('./application');\nvar Route = require('./router/route');\nvar Router = require('./router');\nvar req = require('./request');\nvar res = require('./response');\n\n/**\n * Expose `createApplication()`.\n */\n\nexports = module.exports = createApplication;\n\n/**\n * Create an express application.\n *\n * @return {Function}\n * @api public\n */\n\nfunction createApplication() {\n  var app = function(req, res, next) {\n    app.handle(req, res, next);\n  };\n\n  mixin(app, EventEmitter.prototype, false);\n  mixin(app, proto, false);\n\n  // expose the prototype that will get set on requests\n  app.request = Object.create(req, {\n    app: { configurable: true, enumerable: true, writable: true, value: app }\n  })\n\n  // expose the prototype that will get set on responses\n  app.response = Object.create(res, {\n    app: { configurable: true, enumerable: true, writable: true, value: app }\n  })\n\n  app.init();\n  return app;\n}\n\n/**\n * Expose the prototypes.\n */\n\nexports.application = proto;\nexports.request = req;\nexports.response = res;\n\n/**\n * Expose constructors.\n */\n\nexports.Route = Route;\nexports.Router = Router;\n\n/**\n * Expose middleware\n */\n\nexports.json = bodyParser.json\nexports.query = require('./middleware/query');\nexports.raw = bodyParser.raw\nexports.static = require('serve-static');\nexports.text = bodyParser.text\nexports.urlencoded = bodyParser.urlencoded\n\n/**\n * Replace removed middleware with an appropriate error message.\n */\n\nvar removedMiddlewares = [\n  'bodyParser',\n  'compress',\n  'cookieSession',\n  'session',\n  'logger',\n  'cookieParser',\n  'favicon',\n  'responseTime',\n  'errorHandler',\n  'timeout',\n  'methodOverride',\n  'vhost',\n  'csrf',\n  'directory',\n  'limit',\n  'multipart',\n  'staticCache'\n]\n\nremovedMiddlewares.forEach(function (name) {\n  Object.defineProperty(exports, name, {\n    get: function () {\n      throw new Error('Most middleware (like ' + name + ') is no longer bundled with Express and must be installed separately. Please see https://github.com/senchalabs/connect#middleware.');\n    },\n    configurable: true\n  });\n});\n\n", "needle_token_start": 2137, "needle_token_end": 2525, "code_context_ntokens": 12107, "output": ["```javascript\nfunction trim_prefix(layer, layerError, layerPath, path) {\n  if (layerPath.length !== 0) {\n    // Validate path is a prefix match\n    if (layerPath !== path.slice(0, layerPath.length)) {\n      next(layerError)\n      return\n    }\n\n    // Validate path breaks on a path separator\n    var c = path[layerPath.length]\n    if (c && c !== '/' && c !== '.') return next(layerError)\n\n    // Trim off the part of the url that matches the route\n    // middleware (.use stuff) needs to have the path stripped\n    debug('trim prefix (%s) from url %s', layerPath, req.url);\n    removed = layerPath;\n    req.url = protohost + req.url.slice(protohost.length + removed.length)\n\n    // Ensure leading slash\n    if (!protohost && req.url[0] !== '/') {\n      req.url = '/' + req.url;\n      slashAdded = true;\n    }\n\n    // Setup base URL (no trailing slash)\n    req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'\n      ? removed.substring(0, removed.length - 1)\n      : removed);\n  }\n\n  debug('%s %s : %s', layer.name, layerPath, req.originalUrl);\n\n  if (layerError) {\n    layer.handle_error(layerError, req, res, next);\n  } else {\n    layer.handle_request(req, res, next);\n  }\n}\n```"]}
{"repo": "expressjs/express", "name": "decode_param", "language": "typescript", "path": "lib/router/layer.js", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to safely convert a URL-encoded string back to its original representation.\n2. **Input**: A single string which is expected to be URL-encoded.\n3. **Output**: Returns the original string after decoding, or throws an error if the decoding fails.\n4. **Procedure**: \n   - Checks if the input is a string and is not empty.\n   - Attempts to decode the string using `decodeURIComponent`.\n   - If decoding is successful, the decoded string is returned.\n   - If a decoding error occurs (specifically a `URIError`), the error is modified to include a custom message and a status code, then rethrown.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/request.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar accepts = require('accepts');\nvar deprecate = require('depd')('express');\nvar isIP = require('net').isIP;\nvar typeis = require('type-is');\nvar http = require('http');\nvar fresh = require('fresh');\nvar parseRange = require('range-parser');\nvar parse = require('parseurl');\nvar proxyaddr = require('proxy-addr');\n\n/**\n * Request prototype.\n * @public\n */\n\nvar req = Object.create(http.IncomingMessage.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = req\n\n/**\n * Return request header.\n *\n * The `Referrer` header field is special-cased,\n * both `Referrer` and `Referer` are interchangeable.\n *\n * Examples:\n *\n *     req.get('Content-Type');\n *     // => \"text/plain\"\n *\n *     req.get('content-type');\n *     // => \"text/plain\"\n *\n *     req.get('Something');\n *     // => undefined\n *\n * Aliased as `req.header()`.\n *\n * @param {String} name\n * @return {String}\n * @public\n */\n\nreq.get =\nreq.header = function header(name) {\n  if (!name) {\n    throw new TypeError('name argument is required to req.get');\n  }\n\n  if (typeof name !== 'string') {\n    throw new TypeError('name must be a string to req.get');\n  }\n\n  var lc = name.toLowerCase();\n\n  switch (lc) {\n    case 'referer':\n    case 'referrer':\n      return this.headers.referrer\n        || this.headers.referer;\n    default:\n      return this.headers[lc];\n  }\n};\n\n/**\n * To do: update docs.\n *\n * Check if the given `type(s)` is acceptable, returning\n * the best match when true, otherwise `undefined`, in which\n * case you should respond with 406 \"Not Acceptable\".\n *\n * The `type` value may be a single MIME type string\n * such as \"application/json\", an extension name\n * such as \"json\", a comma-delimited list such as \"json, html, text/plain\",\n * an argument list such as `\"json\", \"html\", \"text/plain\"`,\n * or an array `[\"json\", \"html\", \"text/plain\"]`. When a list\n * or array is given, the _best_ match, if any is returned.\n *\n * Examples:\n *\n *     // Accept: text/html\n *     req.accepts('html');\n *     // => \"html\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('html');\n *     // => \"html\"\n *     req.accepts('text/html');\n *     // => \"text/html\"\n *     req.accepts('json, text');\n *     // => \"json\"\n *     req.accepts('application/json');\n *     // => \"application/json\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('image/png');\n *     req.accepts('png');\n *     // => undefined\n *\n *     // Accept: text/*;q=.5, application/json\n *     req.accepts(['html', 'json']);\n *     req.accepts('html', 'json');\n *     req.accepts('html, json');\n *     // => \"json\"\n *\n * @param {String|Array} type(s)\n * @return {String|Array|Boolean}\n * @public\n */\n\nreq.accepts = function(){\n  var accept = accepts(this);\n  return accept.types.apply(accept, arguments);\n};\n\n/**\n * Check if the given `encoding`s are accepted.\n *\n * @param {String} ...encoding\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsEncodings = function(){\n  var accept = accepts(this);\n  return accept.encodings.apply(accept, arguments);\n};\n\nreq.acceptsEncoding = deprecate.function(req.acceptsEncodings,\n  'req.acceptsEncoding: Use acceptsEncodings instead');\n\n/**\n * Check if the given `charset`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...charset\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsCharsets = function(){\n  var accept = accepts(this);\n  return accept.charsets.apply(accept, arguments);\n};\n\nreq.acceptsCharset = deprecate.function(req.acceptsCharsets,\n  'req.acceptsCharset: Use acceptsCharsets instead');\n\n/**\n * Check if the given `lang`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...lang\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsLanguages = function(){\n  var accept = accepts(this);\n  return accept.languages.apply(accept, arguments);\n};\n\nreq.acceptsLanguage = deprecate.function(req.acceptsLanguages,\n  'req.acceptsLanguage: Use acceptsLanguages instead');\n\n/**\n * Parse Range header field, capping to the given `size`.\n *\n * Unspecified ranges such as \"0-\" require knowledge of your resource length. In\n * the case of a byte range this is of course the total number of bytes. If the\n * Range header field is not given `undefined` is returned, `-1` when unsatisfiable,\n * and `-2` when syntactically invalid.\n *\n * When ranges are returned, the array has a \"type\" property which is the type of\n * range that is required (most commonly, \"bytes\"). Each array element is an object\n * with a \"start\" and \"end\" property for the portion of the range.\n *\n * The \"combine\" option can be set to `true` and overlapping & adjacent ranges\n * will be combined into a single range.\n *\n * NOTE: remember that ranges are inclusive, so for example \"Range: users=0-3\"\n * should respond with 4 users when available, not 3.\n *\n * @param {number} size\n * @param {object} [options]\n * @param {boolean} [options.combine=false]\n * @return {number|array}\n * @public\n */\n\nreq.range = function range(size, options) {\n  var range = this.get('Range');\n  if (!range) return;\n  return parseRange(size, range, options);\n};\n\n/**\n * Return the value of param `name` when present or `defaultValue`.\n *\n *  - Checks route placeholders, ex: _/user/:id_\n *  - Checks body params, ex: id=12, {\"id\":12}\n *  - Checks query string params, ex: ?id=12\n *\n * To utilize request bodies, `req.body`\n * should be an object. This can be done by using\n * the `bodyParser()` middleware.\n *\n * @param {String} name\n * @param {Mixed} [defaultValue]\n * @return {String}\n * @public\n */\n\nreq.param = function param(name, defaultValue) {\n  var params = this.params || {};\n  var body = this.body || {};\n  var query = this.query || {};\n\n  var args = arguments.length === 1\n    ? 'name'\n    : 'name, default';\n  deprecate('req.param(' + args + '): Use req.params, req.body, or req.query instead');\n\n  if (null != params[name] && params.hasOwnProperty(name)) return params[name];\n  if (null != body[name]) return body[name];\n  if (null != query[name]) return query[name];\n\n  return defaultValue;\n};\n\n/**\n * Check if the incoming request contains the \"Content-Type\"\n * header field, and it contains the given mime `type`.\n *\n * Examples:\n *\n *      // With Content-Type: text/html; charset=utf-8\n *      req.is('html');\n *      req.is('text/html');\n *      req.is('text/*');\n *      // => true\n *\n *      // When Content-Type is application/json\n *      req.is('json');\n *      req.is('application/json');\n *      req.is('application/*');\n *      // => true\n *\n *      req.is('html');\n *      // => false\n *\n * @param {String|Array} types...\n * @return {String|false|null}\n * @public\n */\n\nreq.is = function is(types) {\n  var arr = types;\n\n  // support flattened arguments\n  if (!Array.isArray(types)) {\n    arr = new Array(arguments.length);\n    for (var i = 0; i < arr.length; i++) {\n      arr[i] = arguments[i];\n    }\n  }\n\n  return typeis(this, arr);\n};\n\n/**\n * Return the protocol string \"http\" or \"https\"\n * when requested with TLS. When the \"trust proxy\"\n * setting trusts the socket address, the\n * \"X-Forwarded-Proto\" header field will be trusted\n * and used if present.\n *\n * If you're running behind a reverse proxy that\n * supplies https for you this may be enabled.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'protocol', function protocol(){\n  var proto = this.connection.encrypted\n    ? 'https'\n    : 'http';\n  var trust = this.app.get('trust proxy fn');\n\n  if (!trust(this.connection.remoteAddress, 0)) {\n    return proto;\n  }\n\n  // Note: X-Forwarded-Proto is normally only ever a\n  //       single value, but this is to be safe.\n  var header = this.get('X-Forwarded-Proto') || proto\n  var index = header.indexOf(',')\n\n  return index !== -1\n    ? header.substring(0, index).trim()\n    : header.trim()\n});\n\n/**\n * Short-hand for:\n *\n *    req.protocol === 'https'\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'secure', function secure(){\n  return this.protocol === 'https';\n...\n// Path: lib/router/layer.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar pathRegexp = require('path-to-regexp');\nvar debug = require('debug')('express:router:layer');\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Layer;\n\nfunction Layer(path, options, fn) {\n  if (!(this instanceof Layer)) {\n    return new Layer(path, options, fn);\n  }\n\n  debug('new %o', path)\n  var opts = options || {};\n\n  this.handle = fn;\n  this.name = fn.name || '<anonymous>';\n  this.params = undefined;\n  this.path = undefined;\n  this.regexp = pathRegexp(path, this.keys = [], opts);\n\n  // set fast path flags\n  this.regexp.fast_star = path === '*'\n  this.regexp.fast_slash = path === '/' && opts.end === false\n}\n\n/**\n * Handle the error for the layer.\n *\n * @param {Error} error\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_error = function handle_error(error, req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length !== 4) {\n    // not a standard error handler\n    return next(error);\n  }\n\n  try {\n    fn(error, req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Handle the request for the layer.\n *\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_request = function handle(req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length > 3) {\n    // not a standard request handler\n    return next();\n  }\n\n  try {\n    fn(req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Check if this route matches `path`, if so\n * populate `.params`.\n *\n * @param {String} path\n * @return {Boolean}\n * @api private\n */\n\nLayer.prototype.match = function match(path) {\n  var match\n\n  if (path != null) {\n    // fast path non-ending match for / (any path matches)\n    if (this.regexp.fast_slash) {\n      this.params = {}\n      this.path = ''\n      return true\n    }\n\n    // fast path for * (everything matched in a param)\n    if (this.regexp.fast_star) {\n      this.params = {'0': decode_param(path)}\n      this.path = path\n      return true\n    }\n\n    // match the path\n    match = this.regexp.exec(path)\n  }\n\n  if (!match) {\n    this.params = undefined;\n    this.path = undefined;\n    return false;\n  }\n\n  // store values\n  this.params = {};\n  this.path = match[0]\n\n  var keys = this.keys;\n  var params = this.params;\n\n  for (var i = 1; i < match.length; i++) {\n    var key = keys[i - 1];\n    var prop = key.name;\n    var val = decode_param(match[i])\n\n    if (val !== undefined || !(hasOwnProperty.call(params, prop))) {\n      params[prop] = val;\n    }\n  }\n\n  return true;\n};\n\n/**\n * Decode param value.\n *\n * @param {string} val\n * @return {string}\n * @private\n */\n\n\nfunction decode_param(val) {\n  if (typeof val !== 'string' || val.length === 0) {\n    return val;\n  }\n\n  try {\n    return decodeURIComponent(val);\n  } catch (err) {\n    if (err instanceof URIError) {\n      err.message = 'Failed to decode param \\'' + val + '\\'';\n      err.status = err.statusCode = 400;\n    }\n\n    throw err;\n  }\n}\n\n// Path: lib/router/route.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:router:route');\nvar flatten = require('array-flatten');\nvar Layer = require('./layer');\nvar methods = require('methods');\n\n/**\n * Module variables.\n * @private\n */\n\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Route;\n\n/**\n * Initialize `Route` with the given `path`,\n *\n * @param {String} path\n * @public\n */\n\nfunction Route(path) {\n  this.path = path;\n  this.stack = [];\n\n  debug('new %o', path)\n\n  // route handlers for various http methods\n  this.methods = {};\n}\n\n/**\n * Determine if the route handles a given method.\n * @private\n */\n\nRoute.prototype._handles_method = function _handles_method(method) {\n  if (this.methods._all) {\n    return true;\n  }\n\n  // normalize name\n  var name = typeof method === 'string'\n    ? method.toLowerCase()\n    : method\n\n  if (name === 'head' && !this.methods['head']) {\n    name = 'get';\n  }\n\n  return Boolean(this.methods[name]);\n};\n\n/**\n * @return {Array} supported HTTP methods\n * @private\n */\n\nRoute.prototype._options = function _options() {\n  var methods = Object.keys(this.methods);\n\n  // append automatic head\n  if (this.methods.get && !this.methods.head) {\n    methods.push('head');\n  }\n\n  for (var i = 0; i < methods.length; i++) {\n    // make upper case\n    methods[i] = methods[i].toUpperCase();\n  }\n\n  return methods;\n};\n\n/**\n * dispatch req, res into this route\n * @private\n */\n\nRoute.prototype.dispatch = function dispatch(req, res, done) {\n  var idx = 0;\n  var stack = this.stack;\n  var sync = 0\n\n  if (stack.length === 0) {\n    return done();\n  }\n  var method = typeof req.method === 'string'\n    ? req.method.toLowerCase()\n    : req.method\n\n  if (method === 'head' && !this.methods['head']) {\n    method = 'get';\n  }\n\n  req.route = this;\n\n  next();\n\n  function next(err) {\n    // signal to exit route\n    if (err && err === 'route') {\n      return done();\n    }\n\n    // signal to exit router\n    if (err && err === 'router') {\n      return done(err)\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    var layer = stack[idx++]\n\n    // end of layers\n    if (!layer) {\n      return done(err)\n    }\n\n    if (layer.method && layer.method !== method) {\n      next(err)\n    } else if (err) {\n      layer.handle_error(err, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n\n    sync = 0\n  }\n};\n\n/**\n * Add a handler for all HTTP verbs to this route.\n *\n * Behaves just like middleware and can respond or call `next`\n * to continue processing.\n *\n * You can use multiple `.all` call to add multiple handlers.\n *\n *   function check_something(req, res, next){\n *     next();\n *   };\n *\n *   function validate_user(req, res, next){\n *     next();\n *   };\n *\n *   route\n *   .all(validate_user)\n *   .all(check_something)\n *   .get(function(req, res, next){\n *     res.send('hello world');\n *   });\n *\n * @param {function} handler\n * @return {Route} for chaining\n * @api public\n */\n\nRoute.prototype.all = function all() {\n  var handles = flatten(slice.call(arguments));\n\n  for (var i = 0; i < handles.length; i++) {\n    var handle = handles[i];\n\n    if (typeof handle !== 'function') {\n      var type = toString.call(handle);\n      var msg = 'Route.all() requires a callback function but got a ' + type\n      throw new TypeError(msg);\n    }\n\n    var layer = Layer('/', {}, handle);\n    layer.method = undefined;\n\n    this.methods._all = true;\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\nmethods.forEach(function(method){\n  Route.prototype[method] = function(){\n    var handles = flatten(slice.call(arguments));\n\n    for (var i = 0; i < handles.length; i++) {\n      var handle = handles[i];\n\n      if (typeof handle !== 'function') {\n        var type = toString.call(handle);\n        var msg = 'Route.' + method + '() requires a callback function but got a ' + type\n        throw new Error(msg);\n      }\n\n      debug('%s %o', method, this.path)\n\n      var layer = Layer('/', {}, handle);\n      layer.method = method;\n\n      this.methods[method] = true;\n      this.stack.push(layer);\n    }\n\n    return this;\n  };\n});\n\n// Path: lib/router/index.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Route = require('./route');\nvar Layer = require('./layer');\nvar methods = require('methods');\nvar mixin = require('utils-merge');\nvar debug = require('debug')('express:router');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar parseUrl = require('parseurl');\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar objectRegExp = /^\\[object (\\S+)\\]$/;\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n\n/**\n * Map the given param placeholder `name`(s) to the given callback.\n *\n * Parameter mapping is used to provide pre-conditions to routes\n * which use normalized placeholders. For example a _:user_id_ parameter\n * could automatically load a user's information from the database without\n * any additional code,\n *\n * The callback uses the same signature as middleware, the only difference\n * being that the value of the placeholder is passed, in this case the _id_\n * of the user. Once the `next()` function is invoked, just like middleware\n * it will continue on to execute the route, or subsequent parameter functions.\n *\n * Just like in middleware, you must either respond to the request or call next\n * to avoid stalling the request.\n *\n *  app.param('user_id', function(req, res, next, id){\n *    User.find(id, function(err, user){\n *      if (err) {\n *        return next(err);\n *      } else if (!user) {\n *        return next(new Error('failed to load user'));\n *      }\n *      req.user = user;\n *      next();\n *    });\n *  });\n *\n * @param {String} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\nproto.param = function param(name, fn) {\n  // param logic\n  if (typeof name === 'function') {\n    deprecate('router.param(fn): Refactor to use path params');\n    this._params.push(name);\n    return;\n  }\n\n  // apply param functions\n  var params = this._params;\n  var len = params.length;\n  var ret;\n\n  if (name[0] === ':') {\n    deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.slice(1)) + ', fn) instead')\n    name = name.slice(1)\n  }\n\n  for (var i = 0; i < len; ++i) {\n    if (ret = params[i](name, fn)) {\n      fn = ret;\n    }\n  }\n\n  // ensure we end up with a\n  // middleware function\n  if ('function' !== typeof fn) {\n    throw new Error('invalid param() call for ' + name + ', got ' + fn);\n  }\n\n  (this.params[name] = this.params[name] || []).push(fn);\n  return this;\n};\n\n/**\n * Dispatch a req, res into the router.\n * @private\n */\n\nproto.handle = function handle(req, res, out) {\n  var self = this;\n\n  debug('dispatching %s %s', req.method, req.url);\n\n  var idx = 0;\n  var protohost = getProtohost(req.url) || ''\n  var removed = '';\n  var slashAdded = false;\n  var sync = 0\n  var paramcalled = {};\n\n  // store options for OPTIONS request\n  // only used if OPTIONS request\n  var options = [];\n\n  // middleware and routes\n  var stack = self.stack;\n\n  // manage inter-router variables\n  var parentParams = req.params;\n  var parentUrl = req.baseUrl || '';\n  var done = restore(out, req, 'baseUrl', 'next', 'params');\n\n  // setup next layer\n  req.next = next;\n\n  // for options requests, respond with a default if nothing else responds\n  if (req.method === 'OPTIONS') {\n    done = wrap(done, function(old, err) {\n      if (err || options.length === 0) return old(err);\n      sendOptionsResponse(res, options, old);\n    });\n  }\n\n  // setup basic req values\n  req.baseUrl = parentUrl;\n  req.originalUrl = req.originalUrl || req.url;\n\n  next();\n\n  function next(err) {\n    var layerError = err === 'route'\n      ? null\n      : err;\n\n    // remove added slash\n    if (slashAdded) {\n      req.url = req.url.slice(1)\n      slashAdded = false;\n    }\n\n    // restore altered req.url\n    if (removed.length !== 0) {\n      req.baseUrl = parentUrl;\n      req.url = protohost + removed + req.url.slice(protohost.length)\n      removed = '';\n    }\n\n    // signal to exit router\n    if (layerError === 'router') {\n      setImmediate(done, null)\n      return\n    }\n\n    // no more matching layers\n    if (idx >= stack.length) {\n      setImmediate(done, layerError);\n      return;\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    // get pathname of request\n    var path = getPathname(req);\n\n    if (path == null) {\n      return done(layerError);\n    }\n\n    // find next matching layer\n    var layer;\n    var match;\n    var route;\n\n    while (match !== true && idx < stack.length) {\n      layer = stack[idx++];\n      match = matchLayer(layer, path);\n      route = layer.route;\n\n      if (typeof match !== 'boolean') {\n        // hold on to layerError\n        layerError = layerError || match;\n      }\n\n      if (match !== true) {\n        continue;\n      }\n\n      if (!route) {\n        // process non-route handlers normally\n        continue;\n      }\n\n      if (layerError) {\n        // routes do not match with a pending error\n        match = false;\n        continue;\n      }\n\n      var method = req.method;\n      var has_method = route._handles_method(method);\n\n      // build up automatic options response\n      if (!has_method && method === 'OPTIONS') {\n        appendMethods(options, route._options());\n      }\n\n      // don't even bother matching route\n      if (!has_method && method !== 'HEAD') {\n        match = false;\n      }\n    }\n\n    // no match\n    if (match !== true) {\n      return done(layerError);\n    }\n\n    // store route for dispatch on change\n    if (route) {\n      req.route = route;\n    }\n\n    // Capture one-time layer values\n    req.params = self.mergeParams\n      ? mergeParams(layer.params, parentParams)\n      : layer.params;\n    var layerPath = layer.path;\n\n    // this should be done for the layer\n    self.process_params(layer, paramcalled, req, res, function (err) {\n      if (err) {\n        next(layerError || err)\n      } else if (route) {\n        layer.handle_request(req, res, next)\n      } else {\n        trim_prefix(layer, layerError, layerPath, path)\n      }\n\n      sync = 0\n    });\n  }\n\n  function trim_prefix(layer, layerError, layerPath, path) {\n    if (layerPath.length !== 0) {\n      // Validate path is a prefix match\n      if (layerPath !== path.slice(0, layerPath.length)) {\n        next(layerError)\n        return\n      }\n\n      // Validate path breaks on a path separator\n      var c = path[layerPath.length]\n      if (c && c !== '/' && c !== '.') return next(layerError)\n\n      // Trim off the part of the url that matches the route\n      // middleware (.use stuff) needs to have the path stripped\n      debug('trim prefix (%s) from url %s', layerPath, req.url);\n      removed = layerPath;\n      req.url = protohost + req.url.slice(protohost.length + removed.length)\n\n      // Ensure leading slash\n      if (!protohost && req.url[0] !== '/') {\n        req.url = '/' + req.url;\n        slashAdded = true;\n      }\n\n      // Setup base URL (no trailing slash)\n      req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'\n        ? removed.substring(0, removed.length - 1)\n        : removed);\n    }\n\n    debug('%s %s : %s', layer.name, layerPath, req.originalUrl);\n\n    if (layerError) {\n      layer.handle_error(layerError, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n  }\n};\n\n/**\n * Process any parameters for the layer.\n * @private\n */\n\nproto.process_params = function process_params(layer, called, req, res, done) {\n  var params = this.params;\n\n  // captured parameters from the layer, keys and values\n  var keys = layer.keys;\n\n  // fast track\n  if (!keys || keys.length === 0) {\n    return done();\n  }\n\n  var i = 0;\n  var name;\n  var paramIndex = 0;\n  var key;\n  var paramVal;\n  var paramCallbacks;\n  var paramCalled;\n\n  // process params in order\n  // param callbacks can be async\n  function param(err) {\n    if (err) {\n      return done(err);\n    }\n\n    if (i >= keys.length ) {\n      return done();\n    }\n\n    paramIndex = 0;\n    key = keys[i++];\n    name = key.name;\n    paramVal = req.params[name];\n    paramCallbacks = params[name];\n    paramCalled = called[name];\n\n    if (paramVal === undefined || !paramCallbacks) {\n      return param();\n    }\n\n    // param previously called with same value or error occurred\n    if (paramCalled && (paramCalled.match === paramVal\n      || (paramCalled.error && paramCalled.error !== 'route'))) {\n      // restore value\n      req.params[name] = paramCalled.value;\n\n      // next param\n      return param(paramCalled.error);\n    }\n\n    called[name] = paramCalled = {\n      error: null,\n      match: paramVal,\n      value: paramVal\n    };\n\n    paramCallback();\n  }\n\n  // single param callbacks\n  function paramCallback(err) {\n    var fn = paramCallbacks[paramIndex++];\n\n    // store updated value\n    paramCalled.value = req.params[key.name];\n\n    if (err) {\n      // store error\n      paramCalled.error = err;\n      param(err);\n      return;\n    }\n\n    if (!fn) return param();\n\n    try {\n      fn(req, res, paramCallback, paramVal, key.name);\n    } catch (e) {\n      paramCallback(e);\n    }\n  }\n\n  param();\n};\n\n/**\n * Use the given middleware function, with optional path, defaulting to \"/\".\n *\n * Use (like `.all`) will run for any http METHOD, but it will not add\n * handlers for those methods so OPTIONS requests will not consider `.use`\n * functions even if they could respond.\n *\n * The other difference is that _route_ path is stripped and not visible\n * to the handler function. The main effect of this feature is that mounted\n * handlers can operate without any code changes regardless of the \"prefix\"\n * pathname.\n *\n * @public\n */\n\nproto.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate router.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var callbacks = flatten(slice.call(arguments, offset));\n\n  if (callbacks.length === 0) {\n    throw new TypeError('Router.use() requires a middleware function')\n  }\n\n  for (var i = 0; i < callbacks.length; i++) {\n    var fn = callbacks[i];\n\n    if (typeof fn !== 'function') {\n      throw new TypeError('Router.use() requires a middleware function but got a ' + gettype(fn))\n    }\n\n    // add the middleware\n    debug('use %o %s', path, fn.name || '<anonymous>')\n\n    var layer = new Layer(path, {\n      sensitive: this.caseSensitive,\n      strict: false,\n      end: false\n    }, fn);\n\n    layer.route = undefined;\n\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\n/**\n * Create a new Route for the given path.\n *\n * Each route contains a separate middleware stack and VERB handlers.\n *\n * See the Route api documentation for details on adding handlers\n * and middleware to routes.\n *\n * @param {String} path\n * @return {Route}\n * @public\n */\n\nproto.route = function route(path) {\n  var route = new Route(path);\n\n  var layer = new Layer(path, {\n    sensitive: this.caseSensitive,\n    strict: this.strict,\n    end: true\n  }, route.dispatch.bind(route));\n\n  layer.route = route;\n\n  this.stack.push(layer);\n  return route;\n};\n\n// create Router#VERB functions\nmethods.concat('all').forEach(function(method){\n  proto[method] = function(path){\n    var route = this.route(path)\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n// append methods to a list of methods\nfunction appendMethods(list, addition) {\n  for (var i = 0; i < addition.length; i++) {\n    var method = addition[i];\n    if (list.indexOf(method) === -1) {\n      list.push(method);\n    }\n  }\n}\n\n// get pathname of request\nfunction getPathname(req) {\n  try {\n    return parseUrl(req).pathname;\n  } catch (err) {\n    return undefined;\n  }\n}\n\n// Get get protocol + host for a URL\nfunction getProtohost(url) {\n  if (typeof url !== 'string' || url.length === 0 || url[0] === '/') {\n    return undefined\n  }\n\n  var searchIndex = url.indexOf('?')\n  var pathLength = searchIndex !== -1\n    ? searchIndex\n    : url.length\n  var fqdnIndex = url.slice(0, pathLength).indexOf('://')\n\n  return fqdnIndex !== -1\n    ? url.substring(0, url.indexOf('/', 3 + fqdnIndex))\n    : undefined\n}\n\n// get type for error message\nfunction gettype(obj) {\n  var type = typeof obj;\n\n  if (type !== 'object') {\n    return type;\n  }\n\n  // inspect [[Class]] for objects\n  return toString.call(obj)\n    .replace(objectRegExp, '$1');\n}\n\n/**\n * Match path to a layer.\n *\n * @param {Layer} layer\n * @param {string} path\n * @private\n */\n\nfunction matchLayer(layer, path) {\n  try {\n    return layer.match(path);\n  } catch (err) {\n    return err;\n  }\n}\n\n// merge params with parent params\nfunction mergeParams(params, parent) {\n  if (typeof parent !== 'object' || !parent) {\n    return params;\n  }\n\n  // make copy of parent for base\n  var obj = mixin({}, parent);\n\n  // simple non-numeric merging\n  if (!(0 in params) || !(0 in parent)) {\n    return mixin(obj, params);\n  }\n\n  var i = 0;\n  var o = 0;\n\n  // determine numeric gaps\n  while (i in params) {\n    i++;\n  }\n\n  while (o in parent) {\n    o++;\n  }\n\n  // offset numeric indices in params before merge\n  for (i--; i >= 0; i--) {\n    params[i + o] = params[i];\n\n    // create holes for the merge when necessary\n    if (i < o) {\n      delete params[i];\n    }\n  }\n\n  return mixin(obj, params);\n}\n\n// restore obj props after function\nfunction restore(fn, obj) {\n  var props = new Array(arguments.length - 2);\n  var vals = new Array(arguments.length - 2);\n\n  for (var i = 0; i < props.length; i++) {\n    props[i] = arguments[i + 2];\n    vals[i] = obj[props[i]];\n  }\n\n  return function () {\n    // restore vals\n    for (var i = 0; i < props.length; i++) {\n      obj[props[i]] = vals[i];\n    }\n\n    return fn.apply(this, arguments);\n  };\n}\n\n// send an OPTIONS response\nfunction sendOptionsResponse(res, options, next) {\n  try {\n    var body = options.join(',');\n    res.set('Allow', body);\n    res.send(body);\n  } catch (err) {\n    next(err);\n  }\n}\n\n// wrap a function\nfunction wrap(old, fn) {\n  return function proxy() {\n    var args = new Array(arguments.length + 1);\n\n    args[0] = old;\n    for (var i = 0, len = arguments.length; i < len; i++) {\n      args[i + 1] = arguments[i];\n    }\n\n    fn.apply(this, args);\n  };\n}\n\n// Path: lib/middleware/init.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Initialization middleware, exposing the\n * request and response to each other, as well\n * as defaulting the X-Powered-By header field.\n *\n * @param {Function} app\n * @return {Function}\n * @api private\n */\n\nexports.init = function(app){\n  return function expressInit(req, res, next){\n    if (app.enabled('x-powered-by')) res.setHeader('X-Powered-By', 'Express');\n    req.res = res;\n    res.req = req;\n    req.next = next;\n\n    setPrototypeOf(req, app.request)\n    setPrototypeOf(res, app.response)\n\n    res.locals = res.locals || Object.create(null);\n\n    next();\n  };\n};\n\n\n// Path: lib/view.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:view');\nvar path = require('path');\nvar fs = require('fs');\n\n/**\n * Module variables.\n * @private\n */\n\nvar dirname = path.dirname;\nvar basename = path.basename;\nvar extname = path.extname;\nvar join = path.join;\nvar resolve = path.resolve;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = View;\n\n/**\n * Initialize a new `View` with the given `name`.\n *\n * Options:\n *\n *   - `defaultEngine` the default template engine name\n *   - `engines` template engine require() cache\n *   - `root` root path for view lookup\n *\n * @param {string} name\n * @param {object} options\n * @public\n */\n\nfunction View(name, options) {\n  var opts = options || {};\n\n  this.defaultEngine = opts.defaultEngine;\n  this.ext = extname(name);\n  this.name = name;\n  this.root = opts.root;\n\n  if (!this.ext && !this.defaultEngine) {\n    throw new Error('No default engine was specified and no extension was provided.');\n  }\n\n  var fileName = name;\n\n  if (!this.ext) {\n    // get extension from default engine name\n    this.ext = this.defaultEngine[0] !== '.'\n      ? '.' + this.defaultEngine\n      : this.defaultEngine;\n\n    fileName += this.ext;\n  }\n\n  if (!opts.engines[this.ext]) {\n    // load engine\n    var mod = this.ext.slice(1)\n    debug('require \"%s\"', mod)\n\n    // default engine export\n    var fn = require(mod).__express\n\n    if (typeof fn !== 'function') {\n      throw new Error('Module \"' + mod + '\" does not provide a view engine.')\n    }\n\n    opts.engines[this.ext] = fn\n  }\n\n  // store loaded engine\n  this.engine = opts.engines[this.ext];\n\n  // lookup path\n  this.path = this.lookup(fileName);\n}\n\n/**\n * Lookup view by the given `name`\n *\n * @param {string} name\n * @private\n */\n\nView.prototype.lookup = function lookup(name) {\n  var path;\n  var roots = [].concat(this.root);\n\n  debug('lookup \"%s\"', name);\n\n  for (var i = 0; i < roots.length && !path; i++) {\n    var root = roots[i];\n\n    // resolve the path\n    var loc = resolve(root, name);\n    var dir = dirname(loc);\n    var file = basename(loc);\n\n    // resolve the file\n    path = this.resolve(dir, file);\n  }\n\n  return path;\n};\n\n/**\n * Render with the given options.\n *\n * @param {object} options\n * @param {function} callback\n * @private\n */\n\nView.prototype.render = function render(options, callback) {\n  debug('render \"%s\"', this.path);\n  this.engine(this.path, options, callback);\n};\n\n/**\n * Resolve the file within the given directory.\n *\n * @param {string} dir\n * @param {string} file\n * @private\n */\n\nView.prototype.resolve = function resolve(dir, file) {\n  var ext = this.ext;\n\n  // <path>.<ext>\n  var path = join(dir, file);\n  var stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n\n  // <path>/index.<ext>\n  path = join(dir, basename(file, ext), 'index' + ext);\n  stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n};\n\n/**\n * Return a stat, maybe.\n *\n * @param {string} path\n * @return {fs.Stats}\n * @private\n */\n\nfunction tryStat(path) {\n  debug('stat \"%s\"', path);\n\n  try {\n    return fs.statSync(path);\n  } catch (e) {\n    return undefined;\n  }\n}\n\n// Path: lib/middleware/query.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar merge = require('utils-merge')\nvar parseUrl = require('parseurl');\nvar qs = require('qs');\n\n/**\n * @param {Object} options\n * @return {Function}\n * @api public\n */\n\nmodule.exports = function query(options) {\n  var opts = merge({}, options)\n  var queryparse = qs.parse;\n\n  if (typeof options === 'function') {\n    queryparse = options;\n    opts = undefined;\n  }\n\n  if (opts !== undefined && opts.allowPrototypes === undefined) {\n    // back-compat for qs module\n    opts.allowPrototypes = true;\n  }\n\n  return function query(req, res, next){\n    if (!req.query) {\n      var val = parseUrl(req).query;\n      req.query = queryparse(val, opts);\n    }\n\n    next();\n  };\n};\n\n// Path: lib/application.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar finalhandler = require('finalhandler');\nvar Router = require('./router');\nvar methods = require('methods');\nvar middleware = require('./middleware/init');\nvar query = require('./middleware/query');\nvar debug = require('debug')('express:application');\nvar View = require('./view');\nvar http = require('http');\nvar compileETag = require('./utils').compileETag;\nvar compileQueryParser = require('./utils').compileQueryParser;\nvar compileTrust = require('./utils').compileTrust;\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar merge = require('utils-merge');\nvar resolve = require('path').resolve;\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty\nvar slice = Array.prototype.slice;\n\n/**\n * Application prototype.\n */\n\nvar app = exports = module.exports = {};\n\n/**\n * Variable for trust proxy inheritance back-compat\n * @private\n */\n\nvar trustProxyDefaultSymbol = '@@symbol:trust_proxy_default';\n\n/**\n * Initialize the server.\n *\n *   - setup default configuration\n *   - setup default middleware\n *   - setup route reflection methods\n *\n * @private\n */\n\napp.init = function init() {\n  this.cache = {};\n  this.engines = {};\n  this.settings = {};\n\n  this.defaultConfiguration();\n};\n\n/**\n * Initialize application configuration.\n * @private\n */\n\napp.defaultConfiguration = function defaultConfiguration() {\n  var env = process.env.NODE_ENV || 'development';\n\n  // default settings\n  this.enable('x-powered-by');\n  this.set('etag', 'weak');\n  this.set('env', env);\n  this.set('query parser', 'extended');\n  this.set('subdomain offset', 2);\n  this.set('trust proxy', false);\n\n  // trust proxy inherit back-compat\n  Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n    configurable: true,\n    value: true\n  });\n\n  debug('booting in %s mode', env);\n\n  this.on('mount', function onmount(parent) {\n    // inherit trust proxy\n    if (this.settings[trustProxyDefaultSymbol] === true\n      && typeof parent.settings['trust proxy fn'] === 'function') {\n      delete this.settings['trust proxy'];\n      delete this.settings['trust proxy fn'];\n    }\n\n    // inherit protos\n    setPrototypeOf(this.request, parent.request)\n    setPrototypeOf(this.response, parent.response)\n    setPrototypeOf(this.engines, parent.engines)\n    setPrototypeOf(this.settings, parent.settings)\n  });\n\n  // setup locals\n  this.locals = Object.create(null);\n\n  // top-most app is mounted at /\n  this.mountpath = '/';\n\n  // default locals\n  this.locals.settings = this.settings;\n\n  // default configuration\n  this.set('view', View);\n  this.set('views', resolve('views'));\n  this.set('jsonp callback name', 'callback');\n\n  if (env === 'production') {\n    this.enable('view cache');\n  }\n\n  Object.defineProperty(this, 'router', {\n    get: function() {\n      throw new Error('\\'app.router\\' is deprecated!\\nPlease see the 3.x to 4.x migration guide for details on how to update your app.');\n    }\n  });\n};\n\n/**\n * lazily adds the base router if it has not yet been added.\n *\n * We cannot add the base router in the defaultConfiguration because\n * it reads app settings which might be set after that has run.\n *\n * @private\n */\napp.lazyrouter = function lazyrouter() {\n  if (!this._router) {\n    this._router = new Router({\n      caseSensitive: this.enabled('case sensitive routing'),\n      strict: this.enabled('strict routing')\n    });\n\n    this._router.use(query(this.get('query parser fn')));\n    this._router.use(middleware.init(this));\n  }\n};\n\n/**\n * Dispatch a req, res pair into the application. Starts pipeline processing.\n *\n * If no callback is provided, then default error handlers will respond\n * in the event of an error bubbling through the stack.\n *\n * @private\n */\n\napp.handle = function handle(req, res, callback) {\n  var router = this._router;\n\n  // final handler\n  var done = callback || finalhandler(req, res, {\n    env: this.get('env'),\n    onerror: logerror.bind(this)\n  });\n\n  // no routes\n  if (!router) {\n    debug('no routes defined on app');\n    done();\n    return;\n  }\n\n  router.handle(req, res, done);\n};\n\n/**\n * Proxy `Router#use()` to add middleware to the app router.\n * See Router#use() documentation for details.\n *\n * If the _fn_ parameter is an express app, then it will be\n * mounted at the _route_ specified.\n *\n * @public\n */\n\napp.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate app.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var fns = flatten(slice.call(arguments, offset));\n\n  if (fns.length === 0) {\n    throw new TypeError('app.use() requires a middleware function')\n  }\n\n  // setup router\n  this.lazyrouter();\n  var router = this._router;\n\n  fns.forEach(function (fn) {\n    // non-express app\n    if (!fn || !fn.handle || !fn.set) {\n      return router.use(path, fn);\n    }\n\n    debug('.use app under %s', path);\n    fn.mountpath = path;\n    fn.parent = this;\n\n    // restore .app property on req and res\n    router.use(path, function mounted_app(req, res, next) {\n      var orig = req.app;\n      fn.handle(req, res, function (err) {\n        setPrototypeOf(req, orig.request)\n        setPrototypeOf(res, orig.response)\n        next(err);\n      });\n    });\n\n    // mounted an app\n    fn.emit('mount', this);\n  }, this);\n\n  return this;\n};\n\n/**\n * Proxy to the app `Router#route()`\n * Returns a new `Route` instance for the _path_.\n *\n * Routes are isolated middleware stacks for specific paths.\n * See the Route api docs for details.\n *\n * @public\n */\n\napp.route = function route(path) {\n  this.lazyrouter();\n  return this._router.route(path);\n};\n\n/**\n * Register the given template engine callback `fn`\n * as `ext`.\n *\n * By default will `require()` the engine based on the\n * file extension. For example if you try to render\n * a \"foo.ejs\" file Express will invoke the following internally:\n *\n *     app.engine('ejs', require('ejs').__express);\n *\n * For engines that do not provide `.__express` out of the box,\n * or if you wish to \"map\" a different extension to the template engine\n * you may use this method. For example mapping the EJS template engine to\n * \".html\" files:\n *\n *     app.engine('html', require('ejs').renderFile);\n *\n * In this case EJS provides a `.renderFile()` method with\n * the same signature that Express expects: `(path, options, callback)`,\n * though note that it aliases this method as `ejs.__express` internally\n * so if you're using \".ejs\" extensions you don't need to do anything.\n *\n * Some template engines do not follow this convention, the\n * [Consolidate.js](https://github.com/tj/consolidate.js)\n * library was created to map all of node's popular template\n * engines to follow this convention, thus allowing them to\n * work seamlessly within Express.\n *\n * @param {String} ext\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.engine = function engine(ext, fn) {\n  if (typeof fn !== 'function') {\n    throw new Error('callback function required');\n  }\n\n  // get file extension\n  var extension = ext[0] !== '.'\n    ? '.' + ext\n    : ext;\n\n  // store engine\n  this.engines[extension] = fn;\n\n  return this;\n};\n\n/**\n * Proxy to `Router#param()` with one added api feature. The _name_ parameter\n * can be an array of names.\n *\n * See the Router#param() docs for more details.\n *\n * @param {String|Array} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.param = function param(name, fn) {\n  this.lazyrouter();\n\n  if (Array.isArray(name)) {\n    for (var i = 0; i < name.length; i++) {\n      this.param(name[i], fn);\n    }\n\n    return this;\n  }\n\n  this._router.param(name, fn);\n\n  return this;\n};\n\n/**\n * Assign `setting` to `val`, or return `setting`'s value.\n *\n *    app.set('foo', 'bar');\n *    app.set('foo');\n *    // => \"bar\"\n *\n * Mounted servers inherit their parent server's settings.\n *\n * @param {String} setting\n * @param {*} [val]\n * @return {Server} for chaining\n * @public\n */\n\napp.set = function set(setting, val) {\n  if (arguments.length === 1) {\n    // app.get(setting)\n    var settings = this.settings\n\n    while (settings && settings !== Object.prototype) {\n      if (hasOwnProperty.call(settings, setting)) {\n        return settings[setting]\n      }\n\n      settings = Object.getPrototypeOf(settings)\n    }\n\n    return undefined\n  }\n\n  debug('set \"%s\" to %o', setting, val);\n\n  // set value\n  this.settings[setting] = val;\n\n  // trigger matched settings\n  switch (setting) {\n    case 'etag':\n      this.set('etag fn', compileETag(val));\n      break;\n    case 'query parser':\n      this.set('query parser fn', compileQueryParser(val));\n      break;\n    case 'trust proxy':\n      this.set('trust proxy fn', compileTrust(val));\n\n      // trust proxy inherit back-compat\n      Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n        configurable: true,\n        value: false\n      });\n\n      break;\n  }\n\n  return this;\n};\n\n/**\n * Return the app's absolute pathname\n * based on the parent(s) that have\n * mounted it.\n *\n * For example if the application was\n * mounted as \"/admin\", which itself\n * was mounted as \"/blog\" then the\n * return value would be \"/blog/admin\".\n *\n * @return {String}\n * @private\n */\n\napp.path = function path() {\n  return this.parent\n    ? this.parent.path() + this.mountpath\n    : '';\n};\n\n/**\n * Check if `setting` is enabled (truthy).\n *\n *    app.enabled('foo')\n *    // => false\n *\n *    app.enable('foo')\n *    app.enabled('foo')\n *    // => true\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.enabled = function enabled(setting) {\n  return Boolean(this.set(setting));\n};\n\n/**\n * Check if `setting` is disabled.\n *\n *    app.disabled('foo')\n *    // => true\n *\n *    app.enable('foo')\n *    app.disabled('foo')\n *    // => false\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.disabled = function disabled(setting) {\n  return !this.set(setting);\n};\n\n/**\n * Enable `setting`.\n *\n * @param {String} setting\n * @return {app} for chaining\n * @public\n */\n\napp.enable = function enable(setting) {\n  return this.set(setting, true);\n};\n\n/**\n * Disable `setting`.\n *\n...\n", "needle_token_start": 3737, "needle_token_end": 3853, "code_context_ntokens": 15387, "output": ["```javascript\nfunction decode_param(val) {\n  if (typeof val !== 'string' || val.length === 0) {\n    return val;\n  }\n\n  try {\n    return decodeURIComponent(val);\n  } catch (err) {\n    if (err instanceof URIError) {\n      err.message = 'Failed to decode param \\'' + val + '\\'';\n      err.status = err.statusCode = 400;\n    }\n\n    throw err;\n  }\n}\n```"]}
{"repo": "expressjs/express", "name": "gettype", "language": "typescript", "path": "lib/router/index.js", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the type of a given variable, providing more specific type information especially for objects where it identifies the internal class.\n2. **Input**: A single variable of any type.\n3. **Output**: A string representing the type of the input variable. For non-object types, it returns the basic type (e.g., 'number', 'string'). For objects, it returns a more specific class type (e.g., 'Array', 'Date').\n4. **Procedure**: The function first checks the basic type of the input. If it is not an object, it directly returns this type. If the input is an object, it uses a method to extract the internal class name from the object's constructor and returns this as the type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/router/route.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:router:route');\nvar flatten = require('array-flatten');\nvar Layer = require('./layer');\nvar methods = require('methods');\n\n/**\n * Module variables.\n * @private\n */\n\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Route;\n\n/**\n * Initialize `Route` with the given `path`,\n *\n * @param {String} path\n * @public\n */\n\nfunction Route(path) {\n  this.path = path;\n  this.stack = [];\n\n  debug('new %o', path)\n\n  // route handlers for various http methods\n  this.methods = {};\n}\n\n/**\n * Determine if the route handles a given method.\n * @private\n */\n\nRoute.prototype._handles_method = function _handles_method(method) {\n  if (this.methods._all) {\n    return true;\n  }\n\n  // normalize name\n  var name = typeof method === 'string'\n    ? method.toLowerCase()\n    : method\n\n  if (name === 'head' && !this.methods['head']) {\n    name = 'get';\n  }\n\n  return Boolean(this.methods[name]);\n};\n\n/**\n * @return {Array} supported HTTP methods\n * @private\n */\n\nRoute.prototype._options = function _options() {\n  var methods = Object.keys(this.methods);\n\n  // append automatic head\n  if (this.methods.get && !this.methods.head) {\n    methods.push('head');\n  }\n\n  for (var i = 0; i < methods.length; i++) {\n    // make upper case\n    methods[i] = methods[i].toUpperCase();\n  }\n\n  return methods;\n};\n\n/**\n * dispatch req, res into this route\n * @private\n */\n\nRoute.prototype.dispatch = function dispatch(req, res, done) {\n  var idx = 0;\n  var stack = this.stack;\n  var sync = 0\n\n  if (stack.length === 0) {\n    return done();\n  }\n  var method = typeof req.method === 'string'\n    ? req.method.toLowerCase()\n    : req.method\n\n  if (method === 'head' && !this.methods['head']) {\n    method = 'get';\n  }\n\n  req.route = this;\n\n  next();\n\n  function next(err) {\n    // signal to exit route\n    if (err && err === 'route') {\n      return done();\n    }\n\n    // signal to exit router\n    if (err && err === 'router') {\n      return done(err)\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    var layer = stack[idx++]\n\n    // end of layers\n    if (!layer) {\n      return done(err)\n    }\n\n    if (layer.method && layer.method !== method) {\n      next(err)\n    } else if (err) {\n      layer.handle_error(err, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n\n    sync = 0\n  }\n};\n\n/**\n * Add a handler for all HTTP verbs to this route.\n *\n * Behaves just like middleware and can respond or call `next`\n * to continue processing.\n *\n * You can use multiple `.all` call to add multiple handlers.\n *\n *   function check_something(req, res, next){\n *     next();\n *   };\n *\n *   function validate_user(req, res, next){\n *     next();\n *   };\n *\n *   route\n *   .all(validate_user)\n *   .all(check_something)\n *   .get(function(req, res, next){\n *     res.send('hello world');\n *   });\n *\n * @param {function} handler\n * @return {Route} for chaining\n * @api public\n */\n\nRoute.prototype.all = function all() {\n  var handles = flatten(slice.call(arguments));\n\n  for (var i = 0; i < handles.length; i++) {\n    var handle = handles[i];\n\n    if (typeof handle !== 'function') {\n      var type = toString.call(handle);\n      var msg = 'Route.all() requires a callback function but got a ' + type\n      throw new TypeError(msg);\n    }\n\n    var layer = Layer('/', {}, handle);\n...\n// Path: lib/router/index.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Route = require('./route');\nvar Layer = require('./layer');\nvar methods = require('methods');\nvar mixin = require('utils-merge');\nvar debug = require('debug')('express:router');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar parseUrl = require('parseurl');\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar objectRegExp = /^\\[object (\\S+)\\]$/;\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n\n/**\n * Map the given param placeholder `name`(s) to the given callback.\n *\n * Parameter mapping is used to provide pre-conditions to routes\n * which use normalized placeholders. For example a _:user_id_ parameter\n * could automatically load a user's information from the database without\n * any additional code,\n *\n * The callback uses the same signature as middleware, the only difference\n * being that the value of the placeholder is passed, in this case the _id_\n * of the user. Once the `next()` function is invoked, just like middleware\n * it will continue on to execute the route, or subsequent parameter functions.\n *\n * Just like in middleware, you must either respond to the request or call next\n * to avoid stalling the request.\n *\n *  app.param('user_id', function(req, res, next, id){\n *    User.find(id, function(err, user){\n *      if (err) {\n *        return next(err);\n *      } else if (!user) {\n *        return next(new Error('failed to load user'));\n *      }\n *      req.user = user;\n *      next();\n *    });\n *  });\n *\n * @param {String} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\nproto.param = function param(name, fn) {\n  // param logic\n  if (typeof name === 'function') {\n    deprecate('router.param(fn): Refactor to use path params');\n    this._params.push(name);\n    return;\n  }\n\n  // apply param functions\n  var params = this._params;\n  var len = params.length;\n  var ret;\n\n  if (name[0] === ':') {\n    deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.slice(1)) + ', fn) instead')\n    name = name.slice(1)\n  }\n\n  for (var i = 0; i < len; ++i) {\n    if (ret = params[i](name, fn)) {\n      fn = ret;\n    }\n  }\n\n  // ensure we end up with a\n  // middleware function\n  if ('function' !== typeof fn) {\n    throw new Error('invalid param() call for ' + name + ', got ' + fn);\n  }\n\n  (this.params[name] = this.params[name] || []).push(fn);\n  return this;\n};\n\n/**\n * Dispatch a req, res into the router.\n * @private\n */\n\nproto.handle = function handle(req, res, out) {\n  var self = this;\n\n  debug('dispatching %s %s', req.method, req.url);\n\n  var idx = 0;\n  var protohost = getProtohost(req.url) || ''\n  var removed = '';\n  var slashAdded = false;\n  var sync = 0\n  var paramcalled = {};\n\n  // store options for OPTIONS request\n  // only used if OPTIONS request\n  var options = [];\n\n  // middleware and routes\n  var stack = self.stack;\n\n  // manage inter-router variables\n  var parentParams = req.params;\n  var parentUrl = req.baseUrl || '';\n  var done = restore(out, req, 'baseUrl', 'next', 'params');\n\n  // setup next layer\n  req.next = next;\n\n  // for options requests, respond with a default if nothing else responds\n  if (req.method === 'OPTIONS') {\n    done = wrap(done, function(old, err) {\n      if (err || options.length === 0) return old(err);\n      sendOptionsResponse(res, options, old);\n    });\n  }\n\n  // setup basic req values\n  req.baseUrl = parentUrl;\n  req.originalUrl = req.originalUrl || req.url;\n\n  next();\n\n  function next(err) {\n    var layerError = err === 'route'\n      ? null\n      : err;\n\n    // remove added slash\n    if (slashAdded) {\n      req.url = req.url.slice(1)\n      slashAdded = false;\n    }\n\n    // restore altered req.url\n    if (removed.length !== 0) {\n      req.baseUrl = parentUrl;\n      req.url = protohost + removed + req.url.slice(protohost.length)\n      removed = '';\n    }\n\n    // signal to exit router\n    if (layerError === 'router') {\n      setImmediate(done, null)\n      return\n    }\n\n    // no more matching layers\n    if (idx >= stack.length) {\n      setImmediate(done, layerError);\n      return;\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    // get pathname of request\n    var path = getPathname(req);\n\n    if (path == null) {\n      return done(layerError);\n    }\n\n    // find next matching layer\n    var layer;\n    var match;\n    var route;\n\n    while (match !== true && idx < stack.length) {\n      layer = stack[idx++];\n      match = matchLayer(layer, path);\n      route = layer.route;\n\n      if (typeof match !== 'boolean') {\n        // hold on to layerError\n        layerError = layerError || match;\n      }\n\n      if (match !== true) {\n        continue;\n      }\n\n      if (!route) {\n        // process non-route handlers normally\n        continue;\n      }\n\n      if (layerError) {\n        // routes do not match with a pending error\n        match = false;\n        continue;\n      }\n\n      var method = req.method;\n      var has_method = route._handles_method(method);\n\n      // build up automatic options response\n      if (!has_method && method === 'OPTIONS') {\n        appendMethods(options, route._options());\n      }\n\n      // don't even bother matching route\n      if (!has_method && method !== 'HEAD') {\n        match = false;\n      }\n    }\n\n    // no match\n    if (match !== true) {\n      return done(layerError);\n    }\n\n    // store route for dispatch on change\n    if (route) {\n      req.route = route;\n    }\n\n    // Capture one-time layer values\n    req.params = self.mergeParams\n      ? mergeParams(layer.params, parentParams)\n      : layer.params;\n    var layerPath = layer.path;\n\n    // this should be done for the layer\n    self.process_params(layer, paramcalled, req, res, function (err) {\n      if (err) {\n        next(layerError || err)\n      } else if (route) {\n        layer.handle_request(req, res, next)\n      } else {\n        trim_prefix(layer, layerError, layerPath, path)\n      }\n\n      sync = 0\n    });\n  }\n\n  function trim_prefix(layer, layerError, layerPath, path) {\n    if (layerPath.length !== 0) {\n      // Validate path is a prefix match\n      if (layerPath !== path.slice(0, layerPath.length)) {\n        next(layerError)\n        return\n      }\n\n      // Validate path breaks on a path separator\n      var c = path[layerPath.length]\n      if (c && c !== '/' && c !== '.') return next(layerError)\n\n      // Trim off the part of the url that matches the route\n      // middleware (.use stuff) needs to have the path stripped\n      debug('trim prefix (%s) from url %s', layerPath, req.url);\n      removed = layerPath;\n      req.url = protohost + req.url.slice(protohost.length + removed.length)\n\n      // Ensure leading slash\n      if (!protohost && req.url[0] !== '/') {\n        req.url = '/' + req.url;\n        slashAdded = true;\n      }\n\n      // Setup base URL (no trailing slash)\n      req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'\n        ? removed.substring(0, removed.length - 1)\n        : removed);\n    }\n\n    debug('%s %s : %s', layer.name, layerPath, req.originalUrl);\n\n    if (layerError) {\n      layer.handle_error(layerError, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n  }\n};\n\n/**\n * Process any parameters for the layer.\n * @private\n */\n\nproto.process_params = function process_params(layer, called, req, res, done) {\n  var params = this.params;\n\n  // captured parameters from the layer, keys and values\n  var keys = layer.keys;\n\n  // fast track\n  if (!keys || keys.length === 0) {\n    return done();\n  }\n\n  var i = 0;\n  var name;\n  var paramIndex = 0;\n  var key;\n  var paramVal;\n  var paramCallbacks;\n  var paramCalled;\n\n  // process params in order\n  // param callbacks can be async\n  function param(err) {\n    if (err) {\n      return done(err);\n    }\n\n    if (i >= keys.length ) {\n      return done();\n    }\n\n    paramIndex = 0;\n    key = keys[i++];\n    name = key.name;\n    paramVal = req.params[name];\n    paramCallbacks = params[name];\n    paramCalled = called[name];\n\n    if (paramVal === undefined || !paramCallbacks) {\n      return param();\n    }\n\n    // param previously called with same value or error occurred\n    if (paramCalled && (paramCalled.match === paramVal\n      || (paramCalled.error && paramCalled.error !== 'route'))) {\n      // restore value\n      req.params[name] = paramCalled.value;\n\n      // next param\n      return param(paramCalled.error);\n    }\n\n    called[name] = paramCalled = {\n      error: null,\n      match: paramVal,\n      value: paramVal\n    };\n\n    paramCallback();\n  }\n\n  // single param callbacks\n  function paramCallback(err) {\n    var fn = paramCallbacks[paramIndex++];\n\n    // store updated value\n    paramCalled.value = req.params[key.name];\n\n    if (err) {\n      // store error\n      paramCalled.error = err;\n      param(err);\n      return;\n    }\n\n    if (!fn) return param();\n\n    try {\n      fn(req, res, paramCallback, paramVal, key.name);\n    } catch (e) {\n      paramCallback(e);\n    }\n  }\n\n  param();\n};\n\n/**\n * Use the given middleware function, with optional path, defaulting to \"/\".\n *\n * Use (like `.all`) will run for any http METHOD, but it will not add\n * handlers for those methods so OPTIONS requests will not consider `.use`\n * functions even if they could respond.\n *\n * The other difference is that _route_ path is stripped and not visible\n * to the handler function. The main effect of this feature is that mounted\n * handlers can operate without any code changes regardless of the \"prefix\"\n * pathname.\n *\n * @public\n */\n\nproto.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate router.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var callbacks = flatten(slice.call(arguments, offset));\n\n  if (callbacks.length === 0) {\n    throw new TypeError('Router.use() requires a middleware function')\n  }\n\n  for (var i = 0; i < callbacks.length; i++) {\n    var fn = callbacks[i];\n\n    if (typeof fn !== 'function') {\n      throw new TypeError('Router.use() requires a middleware function but got a ' + gettype(fn))\n    }\n\n    // add the middleware\n    debug('use %o %s', path, fn.name || '<anonymous>')\n\n    var layer = new Layer(path, {\n      sensitive: this.caseSensitive,\n      strict: false,\n      end: false\n    }, fn);\n\n    layer.route = undefined;\n\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\n/**\n * Create a new Route for the given path.\n *\n * Each route contains a separate middleware stack and VERB handlers.\n *\n * See the Route api documentation for details on adding handlers\n * and middleware to routes.\n *\n * @param {String} path\n * @return {Route}\n * @public\n */\n\nproto.route = function route(path) {\n  var route = new Route(path);\n\n  var layer = new Layer(path, {\n    sensitive: this.caseSensitive,\n    strict: this.strict,\n    end: true\n  }, route.dispatch.bind(route));\n\n  layer.route = route;\n\n  this.stack.push(layer);\n  return route;\n};\n\n// create Router#VERB functions\nmethods.concat('all').forEach(function(method){\n  proto[method] = function(path){\n    var route = this.route(path)\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n// append methods to a list of methods\nfunction appendMethods(list, addition) {\n  for (var i = 0; i < addition.length; i++) {\n    var method = addition[i];\n    if (list.indexOf(method) === -1) {\n      list.push(method);\n    }\n  }\n}\n\n// get pathname of request\nfunction getPathname(req) {\n  try {\n    return parseUrl(req).pathname;\n  } catch (err) {\n    return undefined;\n  }\n}\n\n// Get get protocol + host for a URL\nfunction getProtohost(url) {\n  if (typeof url !== 'string' || url.length === 0 || url[0] === '/') {\n    return undefined\n  }\n\n  var searchIndex = url.indexOf('?')\n  var pathLength = searchIndex !== -1\n    ? searchIndex\n    : url.length\n  var fqdnIndex = url.slice(0, pathLength).indexOf('://')\n\n  return fqdnIndex !== -1\n    ? url.substring(0, url.indexOf('/', 3 + fqdnIndex))\n    : undefined\n}\n\n// get type for error message\n\nfunction gettype(obj) {\n  var type = typeof obj;\n\n  if (type !== 'object') {\n    return type;\n  }\n\n  // inspect [[Class]] for objects\n  return toString.call(obj)\n    .replace(objectRegExp, '$1');\n}\n\n/**\n * Match path to a layer.\n *\n * @param {Layer} layer\n * @param {string} path\n * @private\n */\n\nfunction matchLayer(layer, path) {\n  try {\n    return layer.match(path);\n  } catch (err) {\n    return err;\n  }\n}\n\n// merge params with parent params\nfunction mergeParams(params, parent) {\n  if (typeof parent !== 'object' || !parent) {\n    return params;\n  }\n\n  // make copy of parent for base\n  var obj = mixin({}, parent);\n\n  // simple non-numeric merging\n  if (!(0 in params) || !(0 in parent)) {\n    return mixin(obj, params);\n  }\n\n  var i = 0;\n  var o = 0;\n\n  // determine numeric gaps\n  while (i in params) {\n    i++;\n  }\n\n  while (o in parent) {\n    o++;\n  }\n\n  // offset numeric indices in params before merge\n  for (i--; i >= 0; i--) {\n    params[i + o] = params[i];\n\n    // create holes for the merge when necessary\n    if (i < o) {\n      delete params[i];\n    }\n  }\n\n  return mixin(obj, params);\n}\n\n// restore obj props after function\nfunction restore(fn, obj) {\n  var props = new Array(arguments.length - 2);\n  var vals = new Array(arguments.length - 2);\n\n  for (var i = 0; i < props.length; i++) {\n    props[i] = arguments[i + 2];\n    vals[i] = obj[props[i]];\n  }\n\n  return function () {\n    // restore vals\n    for (var i = 0; i < props.length; i++) {\n      obj[props[i]] = vals[i];\n    }\n\n    return fn.apply(this, arguments);\n  };\n}\n\n// send an OPTIONS response\nfunction sendOptionsResponse(res, options, next) {\n  try {\n    var body = options.join(',');\n    res.set('Allow', body);\n    res.send(body);\n  } catch (err) {\n    next(err);\n  }\n}\n\n// wrap a function\nfunction wrap(old, fn) {\n  return function proxy() {\n    var args = new Array(arguments.length + 1);\n\n    args[0] = old;\n    for (var i = 0, len = arguments.length; i < len; i++) {\n      args[i + 1] = arguments[i];\n    }\n\n    fn.apply(this, args);\n  };\n}\n\n// Path: lib/middleware/init.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Initialization middleware, exposing the\n * request and response to each other, as well\n * as defaulting the X-Powered-By header field.\n *\n * @param {Function} app\n * @return {Function}\n * @api private\n */\n\nexports.init = function(app){\n  return function expressInit(req, res, next){\n    if (app.enabled('x-powered-by')) res.setHeader('X-Powered-By', 'Express');\n    req.res = res;\n    res.req = req;\n    req.next = next;\n\n    setPrototypeOf(req, app.request)\n    setPrototypeOf(res, app.response)\n\n    res.locals = res.locals || Object.create(null);\n\n    next();\n  };\n};\n\n\n// Path: lib/view.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:view');\nvar path = require('path');\nvar fs = require('fs');\n\n/**\n * Module variables.\n * @private\n */\n\nvar dirname = path.dirname;\nvar basename = path.basename;\nvar extname = path.extname;\nvar join = path.join;\nvar resolve = path.resolve;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = View;\n\n/**\n * Initialize a new `View` with the given `name`.\n *\n * Options:\n *\n *   - `defaultEngine` the default template engine name\n *   - `engines` template engine require() cache\n *   - `root` root path for view lookup\n *\n * @param {string} name\n * @param {object} options\n * @public\n */\n\nfunction View(name, options) {\n  var opts = options || {};\n\n  this.defaultEngine = opts.defaultEngine;\n  this.ext = extname(name);\n  this.name = name;\n  this.root = opts.root;\n\n  if (!this.ext && !this.defaultEngine) {\n    throw new Error('No default engine was specified and no extension was provided.');\n  }\n\n  var fileName = name;\n\n  if (!this.ext) {\n    // get extension from default engine name\n    this.ext = this.defaultEngine[0] !== '.'\n      ? '.' + this.defaultEngine\n      : this.defaultEngine;\n\n    fileName += this.ext;\n  }\n\n  if (!opts.engines[this.ext]) {\n    // load engine\n    var mod = this.ext.slice(1)\n    debug('require \"%s\"', mod)\n\n    // default engine export\n    var fn = require(mod).__express\n\n    if (typeof fn !== 'function') {\n      throw new Error('Module \"' + mod + '\" does not provide a view engine.')\n    }\n\n    opts.engines[this.ext] = fn\n  }\n\n  // store loaded engine\n  this.engine = opts.engines[this.ext];\n\n  // lookup path\n  this.path = this.lookup(fileName);\n}\n\n/**\n * Lookup view by the given `name`\n *\n * @param {string} name\n * @private\n */\n\nView.prototype.lookup = function lookup(name) {\n  var path;\n  var roots = [].concat(this.root);\n\n  debug('lookup \"%s\"', name);\n\n  for (var i = 0; i < roots.length && !path; i++) {\n    var root = roots[i];\n\n    // resolve the path\n    var loc = resolve(root, name);\n    var dir = dirname(loc);\n    var file = basename(loc);\n\n    // resolve the file\n    path = this.resolve(dir, file);\n  }\n\n  return path;\n};\n\n/**\n * Render with the given options.\n *\n * @param {object} options\n * @param {function} callback\n * @private\n */\n\nView.prototype.render = function render(options, callback) {\n  debug('render \"%s\"', this.path);\n  this.engine(this.path, options, callback);\n};\n\n/**\n * Resolve the file within the given directory.\n *\n * @param {string} dir\n * @param {string} file\n * @private\n */\n\nView.prototype.resolve = function resolve(dir, file) {\n  var ext = this.ext;\n\n  // <path>.<ext>\n  var path = join(dir, file);\n  var stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n\n  // <path>/index.<ext>\n  path = join(dir, basename(file, ext), 'index' + ext);\n  stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n};\n\n/**\n * Return a stat, maybe.\n *\n * @param {string} path\n * @return {fs.Stats}\n * @private\n */\n\nfunction tryStat(path) {\n  debug('stat \"%s\"', path);\n\n  try {\n    return fs.statSync(path);\n  } catch (e) {\n    return undefined;\n  }\n}\n\n// Path: lib/middleware/query.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar merge = require('utils-merge')\nvar parseUrl = require('parseurl');\nvar qs = require('qs');\n\n/**\n * @param {Object} options\n * @return {Function}\n * @api public\n */\n\nmodule.exports = function query(options) {\n  var opts = merge({}, options)\n  var queryparse = qs.parse;\n\n  if (typeof options === 'function') {\n    queryparse = options;\n    opts = undefined;\n  }\n\n  if (opts !== undefined && opts.allowPrototypes === undefined) {\n    // back-compat for qs module\n    opts.allowPrototypes = true;\n  }\n\n  return function query(req, res, next){\n    if (!req.query) {\n      var val = parseUrl(req).query;\n      req.query = queryparse(val, opts);\n    }\n\n    next();\n  };\n};\n\n// Path: lib/application.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar finalhandler = require('finalhandler');\nvar Router = require('./router');\nvar methods = require('methods');\nvar middleware = require('./middleware/init');\nvar query = require('./middleware/query');\nvar debug = require('debug')('express:application');\nvar View = require('./view');\nvar http = require('http');\nvar compileETag = require('./utils').compileETag;\nvar compileQueryParser = require('./utils').compileQueryParser;\nvar compileTrust = require('./utils').compileTrust;\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar merge = require('utils-merge');\nvar resolve = require('path').resolve;\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty\nvar slice = Array.prototype.slice;\n\n/**\n * Application prototype.\n */\n\nvar app = exports = module.exports = {};\n\n/**\n * Variable for trust proxy inheritance back-compat\n * @private\n */\n\nvar trustProxyDefaultSymbol = '@@symbol:trust_proxy_default';\n\n/**\n * Initialize the server.\n *\n *   - setup default configuration\n *   - setup default middleware\n *   - setup route reflection methods\n *\n * @private\n */\n\napp.init = function init() {\n  this.cache = {};\n  this.engines = {};\n  this.settings = {};\n\n  this.defaultConfiguration();\n};\n\n/**\n * Initialize application configuration.\n * @private\n */\n\napp.defaultConfiguration = function defaultConfiguration() {\n  var env = process.env.NODE_ENV || 'development';\n\n  // default settings\n  this.enable('x-powered-by');\n  this.set('etag', 'weak');\n  this.set('env', env);\n  this.set('query parser', 'extended');\n  this.set('subdomain offset', 2);\n  this.set('trust proxy', false);\n\n  // trust proxy inherit back-compat\n  Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n    configurable: true,\n    value: true\n  });\n\n  debug('booting in %s mode', env);\n\n  this.on('mount', function onmount(parent) {\n    // inherit trust proxy\n    if (this.settings[trustProxyDefaultSymbol] === true\n      && typeof parent.settings['trust proxy fn'] === 'function') {\n      delete this.settings['trust proxy'];\n      delete this.settings['trust proxy fn'];\n    }\n\n    // inherit protos\n    setPrototypeOf(this.request, parent.request)\n    setPrototypeOf(this.response, parent.response)\n    setPrototypeOf(this.engines, parent.engines)\n    setPrototypeOf(this.settings, parent.settings)\n  });\n\n  // setup locals\n  this.locals = Object.create(null);\n\n  // top-most app is mounted at /\n  this.mountpath = '/';\n\n  // default locals\n  this.locals.settings = this.settings;\n\n  // default configuration\n  this.set('view', View);\n  this.set('views', resolve('views'));\n  this.set('jsonp callback name', 'callback');\n\n  if (env === 'production') {\n    this.enable('view cache');\n  }\n\n  Object.defineProperty(this, 'router', {\n    get: function() {\n      throw new Error('\\'app.router\\' is deprecated!\\nPlease see the 3.x to 4.x migration guide for details on how to update your app.');\n    }\n  });\n};\n\n/**\n * lazily adds the base router if it has not yet been added.\n *\n * We cannot add the base router in the defaultConfiguration because\n * it reads app settings which might be set after that has run.\n *\n * @private\n */\napp.lazyrouter = function lazyrouter() {\n  if (!this._router) {\n    this._router = new Router({\n      caseSensitive: this.enabled('case sensitive routing'),\n      strict: this.enabled('strict routing')\n    });\n\n    this._router.use(query(this.get('query parser fn')));\n    this._router.use(middleware.init(this));\n  }\n};\n\n/**\n * Dispatch a req, res pair into the application. Starts pipeline processing.\n *\n * If no callback is provided, then default error handlers will respond\n * in the event of an error bubbling through the stack.\n *\n * @private\n */\n\napp.handle = function handle(req, res, callback) {\n  var router = this._router;\n\n  // final handler\n  var done = callback || finalhandler(req, res, {\n    env: this.get('env'),\n    onerror: logerror.bind(this)\n  });\n\n  // no routes\n  if (!router) {\n    debug('no routes defined on app');\n    done();\n    return;\n  }\n\n  router.handle(req, res, done);\n};\n\n/**\n * Proxy `Router#use()` to add middleware to the app router.\n * See Router#use() documentation for details.\n *\n * If the _fn_ parameter is an express app, then it will be\n * mounted at the _route_ specified.\n *\n * @public\n */\n\napp.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate app.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var fns = flatten(slice.call(arguments, offset));\n\n  if (fns.length === 0) {\n    throw new TypeError('app.use() requires a middleware function')\n  }\n\n  // setup router\n  this.lazyrouter();\n  var router = this._router;\n\n  fns.forEach(function (fn) {\n    // non-express app\n    if (!fn || !fn.handle || !fn.set) {\n      return router.use(path, fn);\n    }\n\n    debug('.use app under %s', path);\n    fn.mountpath = path;\n    fn.parent = this;\n\n    // restore .app property on req and res\n    router.use(path, function mounted_app(req, res, next) {\n      var orig = req.app;\n      fn.handle(req, res, function (err) {\n        setPrototypeOf(req, orig.request)\n        setPrototypeOf(res, orig.response)\n        next(err);\n      });\n    });\n\n    // mounted an app\n    fn.emit('mount', this);\n  }, this);\n\n  return this;\n};\n\n/**\n * Proxy to the app `Router#route()`\n * Returns a new `Route` instance for the _path_.\n *\n * Routes are isolated middleware stacks for specific paths.\n * See the Route api docs for details.\n *\n * @public\n */\n\napp.route = function route(path) {\n  this.lazyrouter();\n  return this._router.route(path);\n};\n\n/**\n * Register the given template engine callback `fn`\n * as `ext`.\n *\n * By default will `require()` the engine based on the\n * file extension. For example if you try to render\n * a \"foo.ejs\" file Express will invoke the following internally:\n *\n *     app.engine('ejs', require('ejs').__express);\n *\n * For engines that do not provide `.__express` out of the box,\n * or if you wish to \"map\" a different extension to the template engine\n * you may use this method. For example mapping the EJS template engine to\n * \".html\" files:\n *\n *     app.engine('html', require('ejs').renderFile);\n *\n * In this case EJS provides a `.renderFile()` method with\n * the same signature that Express expects: `(path, options, callback)`,\n * though note that it aliases this method as `ejs.__express` internally\n * so if you're using \".ejs\" extensions you don't need to do anything.\n *\n * Some template engines do not follow this convention, the\n * [Consolidate.js](https://github.com/tj/consolidate.js)\n * library was created to map all of node's popular template\n * engines to follow this convention, thus allowing them to\n * work seamlessly within Express.\n *\n * @param {String} ext\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.engine = function engine(ext, fn) {\n  if (typeof fn !== 'function') {\n    throw new Error('callback function required');\n  }\n\n  // get file extension\n  var extension = ext[0] !== '.'\n    ? '.' + ext\n    : ext;\n\n  // store engine\n  this.engines[extension] = fn;\n\n  return this;\n};\n\n/**\n * Proxy to `Router#param()` with one added api feature. The _name_ parameter\n * can be an array of names.\n *\n * See the Router#param() docs for more details.\n *\n * @param {String|Array} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.param = function param(name, fn) {\n  this.lazyrouter();\n\n  if (Array.isArray(name)) {\n    for (var i = 0; i < name.length; i++) {\n      this.param(name[i], fn);\n    }\n\n    return this;\n  }\n\n  this._router.param(name, fn);\n\n  return this;\n};\n\n/**\n * Assign `setting` to `val`, or return `setting`'s value.\n *\n *    app.set('foo', 'bar');\n *    app.set('foo');\n *    // => \"bar\"\n *\n * Mounted servers inherit their parent server's settings.\n *\n * @param {String} setting\n * @param {*} [val]\n * @return {Server} for chaining\n * @public\n */\n\napp.set = function set(setting, val) {\n  if (arguments.length === 1) {\n    // app.get(setting)\n    var settings = this.settings\n\n    while (settings && settings !== Object.prototype) {\n      if (hasOwnProperty.call(settings, setting)) {\n        return settings[setting]\n      }\n\n      settings = Object.getPrototypeOf(settings)\n    }\n\n    return undefined\n  }\n\n  debug('set \"%s\" to %o', setting, val);\n\n  // set value\n  this.settings[setting] = val;\n\n  // trigger matched settings\n  switch (setting) {\n    case 'etag':\n      this.set('etag fn', compileETag(val));\n      break;\n    case 'query parser':\n      this.set('query parser fn', compileQueryParser(val));\n      break;\n    case 'trust proxy':\n      this.set('trust proxy fn', compileTrust(val));\n\n      // trust proxy inherit back-compat\n      Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n        configurable: true,\n        value: false\n      });\n\n      break;\n  }\n\n  return this;\n};\n\n/**\n * Return the app's absolute pathname\n * based on the parent(s) that have\n * mounted it.\n *\n * For example if the application was\n * mounted as \"/admin\", which itself\n * was mounted as \"/blog\" then the\n * return value would be \"/blog/admin\".\n *\n * @return {String}\n * @private\n */\n\napp.path = function path() {\n  return this.parent\n    ? this.parent.path() + this.mountpath\n    : '';\n};\n\n/**\n * Check if `setting` is enabled (truthy).\n *\n *    app.enabled('foo')\n *    // => false\n *\n *    app.enable('foo')\n *    app.enabled('foo')\n *    // => true\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.enabled = function enabled(setting) {\n  return Boolean(this.set(setting));\n};\n\n/**\n * Check if `setting` is disabled.\n *\n *    app.disabled('foo')\n *    // => true\n *\n *    app.enable('foo')\n *    app.disabled('foo')\n *    // => false\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.disabled = function disabled(setting) {\n  return !this.set(setting);\n};\n\n/**\n * Enable `setting`.\n *\n * @param {String} setting\n * @return {app} for chaining\n * @public\n */\n\napp.enable = function enable(setting) {\n  return this.set(setting, true);\n};\n\n/**\n * Disable `setting`.\n *\n * @param {String} setting\n * @return {app} for chaining\n * @public\n */\n\napp.disable = function disable(setting) {\n  return this.set(setting, false);\n};\n\n/**\n * Delegate `.VERB(...)` calls to `router.VERB(...)`.\n */\n\nmethods.forEach(function(method){\n  app[method] = function(path){\n    if (method === 'get' && arguments.length === 1) {\n      // app.get(setting)\n      return this.set(path);\n    }\n\n    this.lazyrouter();\n\n    var route = this._router.route(path);\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n/**\n * Special-cased \"all\" method, applying the given route `path`,\n * middleware, and callback to _every_ HTTP method.\n *\n * @param {String} path\n * @param {Function} ...\n * @return {app} for chaining\n * @public\n */\n\napp.all = function all(path) {\n  this.lazyrouter();\n\n  var route = this._router.route(path);\n  var args = slice.call(arguments, 1);\n\n  for (var i = 0; i < methods.length; i++) {\n    route[methods[i]].apply(route, args);\n  }\n\n  return this;\n};\n\n// del -> delete alias\n\napp.del = deprecate.function(app.delete, 'app.del: Use app.delete instead');\n\n/**\n * Render the given view `name` name with `options`\n * and a callback accepting an error and the\n * rendered template string.\n *\n * Example:\n *\n *    app.render('email', { name: 'Tobi' }, function(err, html){\n *      // ...\n *    })\n *\n * @param {String} name\n * @param {Object|Function} options or fn\n * @param {Function} callback\n * @public\n */\n\napp.render = function render(name, options, callback) {\n  var cache = this.cache;\n  var done = callback;\n  var engines = this.engines;\n  var opts = options;\n  var renderOptions = {};\n  var view;\n\n  // support callback function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // merge app.locals\n  merge(renderOptions, this.locals);\n\n  // merge options._locals\n  if (opts._locals) {\n    merge(renderOptions, opts._locals);\n  }\n\n  // merge options\n  merge(renderOptions, opts);\n\n  // set .cache unless explicitly provided\n  if (renderOptions.cache == null) {\n    renderOptions.cache = this.enabled('view cache');\n  }\n\n  // primed cache\n  if (renderOptions.cache) {\n    view = cache[name];\n  }\n\n  // view\n  if (!view) {\n    var View = this.get('view');\n\n    view = new View(name, {\n      defaultEngine: this.get('view engine'),\n      root: this.get('views'),\n      engines: engines\n    });\n\n    if (!view.path) {\n      var dirs = Array.isArray(view.root) && view.root.length > 1\n        ? 'directories \"' + view.root.slice(0, -1).join('\", \"') + '\" or \"' + view.root[view.root.length - 1] + '\"'\n        : 'directory \"' + view.root + '\"'\n      var err = new Error('Failed to lookup view \"' + name + '\" in views ' + dirs);\n      err.view = view;\n      return done(err);\n    }\n\n    // prime the cache\n    if (renderOptions.cache) {\n      cache[name] = view;\n    }\n  }\n\n  // render\n  tryRender(view, renderOptions, done);\n};\n\n/**\n * Listen for connections.\n *\n * A node `http.Server` is returned, with this\n * application (which is a `Function`) as its\n * callback. If you wish to create both an HTTP\n * and HTTPS server you may do so with the \"http\"\n * and \"https\" modules as shown here:\n *\n *    var http = require('http')\n *      , https = require('https')\n *      , express = require('express')\n *      , app = express();\n *\n *    http.createServer(app).listen(80);\n *    https.createServer({ ... }, app).listen(443);\n *\n * @return {http.Server}\n * @public\n */\n\napp.listen = function listen() {\n  var server = http.createServer(this);\n  return server.listen.apply(server, arguments);\n};\n\n/**\n * Log error using console.error.\n *\n * @param {Error} err\n * @private\n */\n\nfunction logerror(err) {\n  /* istanbul ignore next */\n  if (this.get('env') !== 'test') console.error(err.stack || err.toString());\n}\n\n/**\n * Try rendering a view.\n * @private\n */\n\nfunction tryRender(view, options, callback) {\n  try {\n    view.render(options, callback);\n  } catch (err) {\n    callback(err);\n  }\n}\n\n// Path: lib/express.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar bodyParser = require('body-parser')\nvar EventEmitter = require('events').EventEmitter;\nvar mixin = require('merge-descriptors');\nvar proto = require('./application');\nvar Route = require('./router/route');\nvar Router = require('./router');\nvar req = require('./request');\nvar res = require('./response');\n\n/**\n * Expose `createApplication()`.\n */\n\nexports = module.exports = createApplication;\n\n/**\n * Create an express application.\n *\n * @return {Function}\n * @api public\n */\n\nfunction createApplication() {\n  var app = function(req, res, next) {\n    app.handle(req, res, next);\n  };\n\n  mixin(app, EventEmitter.prototype, false);\n  mixin(app, proto, false);\n\n  // expose the prototype that will get set on requests\n  app.request = Object.create(req, {\n    app: { configurable: true, enumerable: true, writable: true, value: app }\n  })\n\n  // expose the prototype that will get set on responses\n  app.response = Object.create(res, {\n    app: { configurable: true, enumerable: true, writable: true, value: app }\n  })\n\n  app.init();\n  return app;\n}\n\n/**\n * Expose the prototypes.\n */\n\nexports.application = proto;\nexports.request = req;\nexports.response = res;\n\n/**\n * Expose constructors.\n */\n\nexports.Route = Route;\nexports.Router = Router;\n\n/**\n * Expose middleware\n */\n\nexports.json = bodyParser.json\nexports.query = require('./middleware/query');\nexports.raw = bodyParser.raw\nexports.static = require('serve-static');\nexports.text = bodyParser.text\nexports.urlencoded = bodyParser.urlencoded\n\n/**\n * Replace removed middleware with an appropriate error message.\n */\n\nvar removedMiddlewares = [\n  'bodyParser',\n  'compress',\n  'cookieSession',\n  'session',\n  'logger',\n  'cookieParser',\n  'favicon',\n  'responseTime',\n  'errorHandler',\n  'timeout',\n  'methodOverride',\n  'vhost',\n  'csrf',\n  'directory',\n  'limit',\n  'multipart',\n  'staticCache'\n]\n\nremovedMiddlewares.forEach(function (name) {\n  Object.defineProperty(exports, name, {\n    get: function () {\n      throw new Error('Most middleware (like ' + name + ') is no longer bundled with Express and must be installed separately. Please see https://github.com/senchalabs/connect#middleware.');\n    },\n    configurable: true\n  });\n});\n\n", "needle_token_start": 5421, "needle_token_end": 5489, "code_context_ntokens": 13395, "output": ["```javascript\n/**\n * Get type for error message\n *\n * @param {Object} obj\n * @return {String}\n * @private\n */\n\nfunction gettype(obj) {\n  var type = typeof obj;\n\n  if (type !== 'object') {\n    return type;\n  }\n\n  // inspect [[Class]] for objects\n  return toString.call(obj)\n    .replace(objectRegExp, '$1');\n}\n```"]}
{"repo": "expressjs/express", "name": "newObject", "language": "typescript", "path": "lib/utils.js", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To generate a fresh, empty object without any predefined properties or methods.\n2. **Input**: This function does not take any input parameters.\n3. **Output**: Returns a new, empty object.\n4. **Procedure**: The function simply creates and returns an empty object using object literal notation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/utils.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @api private\n */\n\nvar Buffer = require('safe-buffer').Buffer\nvar contentDisposition = require('content-disposition');\nvar contentType = require('content-type');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar mime = require('send').mime;\nvar etag = require('etag');\nvar proxyaddr = require('proxy-addr');\nvar qs = require('qs');\nvar querystring = require('querystring');\n\n/**\n * Return strong ETag for `body`.\n *\n * @param {String|Buffer} body\n * @param {String} [encoding]\n * @return {String}\n * @api private\n */\n\nexports.etag = createETagGenerator({ weak: false })\n\n/**\n * Return weak ETag for `body`.\n *\n * @param {String|Buffer} body\n * @param {String} [encoding]\n * @return {String}\n * @api private\n */\n\nexports.wetag = createETagGenerator({ weak: true })\n\n/**\n * Check if `path` looks absolute.\n *\n * @param {String} path\n * @return {Boolean}\n * @api private\n */\n\nexports.isAbsolute = function(path){\n  if ('/' === path[0]) return true;\n  if (':' === path[1] && ('\\\\' === path[2] || '/' === path[2])) return true; // Windows device path\n  if ('\\\\\\\\' === path.substring(0, 2)) return true; // Microsoft Azure absolute path\n};\n\n/**\n * Flatten the given `arr`.\n *\n * @param {Array} arr\n * @return {Array}\n * @api private\n */\n\nexports.flatten = deprecate.function(flatten,\n  'utils.flatten: use array-flatten npm module instead');\n\n/**\n * Normalize the given `type`, for example \"html\" becomes \"text/html\".\n *\n * @param {String} type\n * @return {Object}\n * @api private\n */\n\nexports.normalizeType = function(type){\n  return ~type.indexOf('/')\n    ? acceptParams(type)\n    : { value: mime.lookup(type), params: {} };\n};\n\n/**\n * Normalize `types`, for example \"html\" becomes \"text/html\".\n *\n * @param {Array} types\n * @return {Array}\n * @api private\n */\n\nexports.normalizeTypes = function(types){\n  var ret = [];\n\n  for (var i = 0; i < types.length; ++i) {\n    ret.push(exports.normalizeType(types[i]));\n  }\n\n  return ret;\n};\n\n/**\n * Generate Content-Disposition header appropriate for the filename.\n * non-ascii filenames are urlencoded and a filename* parameter is added\n *\n * @param {String} filename\n * @return {String}\n * @api private\n */\n\nexports.contentDisposition = deprecate.function(contentDisposition,\n  'utils.contentDisposition: use content-disposition npm module instead');\n\n/**\n * Parse accept params `str` returning an\n * object with `.value`, `.quality` and `.params`.\n *\n * @param {String} str\n * @return {Object}\n * @api private\n */\n\nfunction acceptParams (str) {\n  var parts = str.split(/ *; */);\n  var ret = { value: parts[0], quality: 1, params: {} }\n\n  for (var i = 1; i < parts.length; ++i) {\n    var pms = parts[i].split(/ *= */);\n    if ('q' === pms[0]) {\n      ret.quality = parseFloat(pms[1]);\n    } else {\n      ret.params[pms[0]] = pms[1];\n    }\n  }\n\n  return ret;\n}\n\n/**\n * Compile \"etag\" value to function.\n *\n * @param  {Boolean|String|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileETag = function(val) {\n  var fn;\n\n  if (typeof val === 'function') {\n    return val;\n  }\n\n  switch (val) {\n    case true:\n    case 'weak':\n      fn = exports.wetag;\n      break;\n    case false:\n      break;\n    case 'strong':\n      fn = exports.etag;\n      break;\n    default:\n      throw new TypeError('unknown value for etag function: ' + val);\n  }\n\n  return fn;\n}\n\n/**\n * Compile \"query parser\" value to function.\n *\n * @param  {String|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileQueryParser = function compileQueryParser(val) {\n  var fn;\n\n  if (typeof val === 'function') {\n    return val;\n  }\n\n  switch (val) {\n    case true:\n    case 'simple':\n      fn = querystring.parse;\n      break;\n    case false:\n      fn = newObject;\n      break;\n    case 'extended':\n      fn = parseExtendedQueryString;\n      break;\n    default:\n      throw new TypeError('unknown value for query parser function: ' + val);\n  }\n\n  return fn;\n}\n\n/**\n * Compile \"proxy trust\" value to function.\n *\n * @param  {Boolean|String|Number|Array|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileTrust = function(val) {\n  if (typeof val === 'function') return val;\n\n  if (val === true) {\n    // Support plain true/false\n    return function(){ return true };\n  }\n\n  if (typeof val === 'number') {\n    // Support trusting hop count\n    return function(a, i){ return i < val };\n  }\n\n  if (typeof val === 'string') {\n    // Support comma-separated values\n    val = val.split(',')\n      .map(function (v) { return v.trim() })\n  }\n\n  return proxyaddr.compile(val || []);\n}\n\n/**\n * Set the charset in a given Content-Type string.\n *\n * @param {String} type\n * @param {String} charset\n * @return {String}\n * @api private\n */\n\nexports.setCharset = function setCharset(type, charset) {\n  if (!type || !charset) {\n    return type;\n  }\n\n  // parse type\n  var parsed = contentType.parse(type);\n\n  // set charset\n  parsed.parameters.charset = charset;\n\n  // format type\n  return contentType.format(parsed);\n};\n\n/**\n * Create an ETag generator function, generating ETags with\n * the given options.\n *\n * @param {object} options\n * @return {function}\n * @private\n */\n\nfunction createETagGenerator (options) {\n  return function generateETag (body, encoding) {\n    var buf = !Buffer.isBuffer(body)\n      ? Buffer.from(body, encoding)\n      : body\n\n    return etag(buf, options)\n  }\n}\n\n/**\n * Parse an extended query string with qs.\n *\n * @param {String} str\n * @return {Object}\n * @private\n */\n\nfunction parseExtendedQueryString(str) {\n  return qs.parse(str, {\n    allowPrototypes: true\n  });\n}\n\n/**\n * Return new empty object.\n *\n * @return {Object}\n * @api private\n */\n\n\nfunction newObject() {\n  return {};\n}\n\n// Path: lib/response.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Buffer = require('safe-buffer').Buffer\nvar contentDisposition = require('content-disposition');\nvar createError = require('http-errors')\nvar deprecate = require('depd')('express');\nvar encodeUrl = require('encodeurl');\nvar escapeHtml = require('escape-html');\nvar http = require('http');\nvar isAbsolute = require('./utils').isAbsolute;\nvar onFinished = require('on-finished');\nvar path = require('path');\nvar statuses = require('statuses')\nvar merge = require('utils-merge');\nvar sign = require('cookie-signature').sign;\nvar normalizeType = require('./utils').normalizeType;\nvar normalizeTypes = require('./utils').normalizeTypes;\nvar setCharset = require('./utils').setCharset;\nvar cookie = require('cookie');\nvar send = require('send');\nvar extname = path.extname;\nvar mime = send.mime;\nvar resolve = path.resolve;\nvar vary = require('vary');\n\n/**\n * Response prototype.\n * @public\n */\n\nvar res = Object.create(http.ServerResponse.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = res\n\n/**\n * Module variables.\n * @private\n */\n\nvar charsetRegExp = /;\\s*charset\\s*=/;\nvar schemaAndHostRegExp = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?\\/\\/[^\\\\\\/\\?]+/;\n\n/**\n * Set status `code`.\n *\n * @param {Number} code\n * @return {ServerResponse}\n * @public\n */\n\nres.status = function status(code) {\n  if ((typeof code === 'string' || Math.floor(code) !== code) && code > 99 && code < 1000) {\n    deprecate('res.status(' + JSON.stringify(code) + '): use res.status(' + Math.floor(code) + ') instead')\n  }\n  this.statusCode = code;\n  return this;\n};\n\n/**\n * Set Link header field with the given `links`.\n *\n * Examples:\n *\n *    res.links({\n *      next: 'http://api.example.com/users?page=2',\n *      last: 'http://api.example.com/users?page=5'\n *    });\n *\n * @param {Object} links\n * @return {ServerResponse}\n * @public\n */\n\nres.links = function(links){\n  var link = this.get('Link') || '';\n  if (link) link += ', ';\n  return this.set('Link', link + Object.keys(links).map(function(rel){\n    return '<' + links[rel] + '>; rel=\"' + rel + '\"';\n  }).join(', '));\n};\n\n/**\n * Send a response.\n *\n * Examples:\n *\n *     res.send(Buffer.from('wahoo'));\n *     res.send({ some: 'json' });\n *     res.send('<p>some html</p>');\n *\n * @param {string|number|boolean|object|Buffer} body\n * @public\n */\n\nres.send = function send(body) {\n  var chunk = body;\n  var encoding;\n  var req = this.req;\n  var type;\n\n  // settings\n  var app = this.app;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.send(body, status) backwards compat\n    if (typeof arguments[0] !== 'number' && typeof arguments[1] === 'number') {\n      deprecate('res.send(body, status): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.send(status, body): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[0];\n      chunk = arguments[1];\n    }\n  }\n\n  // disambiguate res.send(status) and res.send(status, num)\n  if (typeof chunk === 'number' && arguments.length === 1) {\n    // res.send(status) will set status message as text string\n    if (!this.get('Content-Type')) {\n      this.type('txt');\n    }\n\n    deprecate('res.send(status): Use res.sendStatus(status) instead');\n    this.statusCode = chunk;\n    chunk = statuses.message[chunk]\n  }\n\n  switch (typeof chunk) {\n    // string defaulting to html\n    case 'string':\n      if (!this.get('Content-Type')) {\n        this.type('html');\n      }\n      break;\n    case 'boolean':\n    case 'number':\n    case 'object':\n      if (chunk === null) {\n        chunk = '';\n      } else if (Buffer.isBuffer(chunk)) {\n        if (!this.get('Content-Type')) {\n          this.type('bin');\n        }\n      } else {\n        return this.json(chunk);\n      }\n      break;\n  }\n\n  // write strings in utf-8\n  if (typeof chunk === 'string') {\n    encoding = 'utf8';\n    type = this.get('Content-Type');\n\n    // reflect this in content-type\n    if (typeof type === 'string') {\n      this.set('Content-Type', setCharset(type, 'utf-8'));\n    }\n  }\n\n  // determine if ETag should be generated\n  var etagFn = app.get('etag fn')\n  var generateETag = !this.get('ETag') && typeof etagFn === 'function'\n\n  // populate Content-Length\n  var len\n  if (chunk !== undefined) {\n    if (Buffer.isBuffer(chunk)) {\n      // get length of Buffer\n      len = chunk.length\n    } else if (!generateETag && chunk.length < 1000) {\n      // just calculate length when no ETag + small chunk\n      len = Buffer.byteLength(chunk, encoding)\n    } else {\n      // convert chunk to Buffer and calculate\n      chunk = Buffer.from(chunk, encoding)\n      encoding = undefined;\n      len = chunk.length\n    }\n\n    this.set('Content-Length', len);\n  }\n\n  // populate ETag\n  var etag;\n  if (generateETag && len !== undefined) {\n    if ((etag = etagFn(chunk, encoding))) {\n      this.set('ETag', etag);\n    }\n  }\n\n  // freshness\n  if (req.fresh) this.statusCode = 304;\n\n  // strip irrelevant headers\n  if (204 === this.statusCode || 304 === this.statusCode) {\n    this.removeHeader('Content-Type');\n    this.removeHeader('Content-Length');\n    this.removeHeader('Transfer-Encoding');\n    chunk = '';\n  }\n\n  // alter headers for 205\n  if (this.statusCode === 205) {\n    this.set('Content-Length', '0')\n    this.removeHeader('Transfer-Encoding')\n    chunk = ''\n  }\n\n  if (req.method === 'HEAD') {\n    // skip body for HEAD\n    this.end();\n  } else {\n    // respond\n    this.end(chunk, encoding);\n  }\n\n  return this;\n};\n\n/**\n * Send JSON response.\n *\n * Examples:\n *\n *     res.json(null);\n *     res.json({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.json = function json(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.json(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.json(obj, status): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.json(status, obj): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('Content-Type', 'application/json');\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send JSON response with JSONP callback support.\n *\n * Examples:\n *\n *     res.jsonp(null);\n *     res.jsonp({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.jsonp = function jsonp(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.jsonp(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.jsonp(obj, status): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.jsonp(status, obj): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n  var callback = this.req.query[app.get('jsonp callback name')];\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'application/json');\n  }\n\n  // fixup callback\n  if (Array.isArray(callback)) {\n    callback = callback[0];\n  }\n\n  // jsonp\n  if (typeof callback === 'string' && callback.length !== 0) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'text/javascript');\n\n    // restrict callback charset\n    callback = callback.replace(/[^\\[\\]\\w$.]/g, '');\n\n    if (body === undefined) {\n      // empty argument\n      body = ''\n    } else if (typeof body === 'string') {\n      // replace chars not allowed in JavaScript that are in JSON\n      body = body\n        .replace(/\\u2028/g, '\\\\u2028')\n        .replace(/\\u2029/g, '\\\\u2029')\n    }\n\n    // the /**/ is a specific security mitigation for \"Rosetta Flash JSONP abuse\"\n    // the typeof check is just to reduce client error noise\n    body = '/**/ typeof ' + callback + ' === \\'function\\' && ' + callback + '(' + body + ');';\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send given HTTP status code.\n *\n * Sets the response status to `statusCode` and the body of the\n * response to the standard description from node's http.STATUS_CODES\n * or the statusCode number if no description.\n *\n * Examples:\n *\n *     res.sendStatus(200);\n *\n * @param {number} statusCode\n * @public\n */\n\nres.sendStatus = function sendStatus(statusCode) {\n  var body = statuses.message[statusCode] || String(statusCode)\n\n  this.statusCode = statusCode;\n  this.type('txt');\n\n  return this.send(body);\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendFile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendFile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendFile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendFile = function sendFile(path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  if (!path) {\n    throw new TypeError('path argument is required to res.sendFile');\n  }\n\n  if (typeof path !== 'string') {\n    throw new TypeError('path must be a string to res.sendFile')\n  }\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  if (!opts.root && !isAbsolute(path)) {\n    throw new TypeError('path must be absolute or specify root to res.sendFile');\n  }\n\n  // create file stream\n  var pathname = encodeURI(path);\n  var file = send(req, pathname, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendfile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendfile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendfile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendfile = function (path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // create file stream\n  var file = send(req, path, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\nres.sendfile = deprecate.function(res.sendfile,\n  'res.sendfile: Use res.sendFile instead');\n\n/**\n * Transfer the file at the given `path` as an attachment.\n *\n * Optionally providing an alternate attachment `filename`,\n * and optional callback `callback(err)`. The callback is invoked\n * when the data transfer is complete, or when an error has\n * occurred. Be sure to check `res.headersSent` if you plan to respond.\n *\n * Optionally providing an `options` object to use with `res.sendFile()`.\n * This function will set the `Content-Disposition` header, overriding\n * any `Content-Disposition` header passed as header options in order\n * to set the attachment and filename.\n *\n * This method uses `res.sendFile()`.\n *\n * @public\n */\n\nres.download = function download (path, filename, options, callback) {\n  var done = callback;\n  var name = filename;\n  var opts = options || null\n\n  // support function as second or third arg\n  if (typeof filename === 'function') {\n    done = filename;\n    name = null;\n    opts = null\n  } else if (typeof options === 'function') {\n    done = options\n    opts = null\n  }\n\n  // support optional filename, where options may be in it's place\n  if (typeof filename === 'object' &&\n    (typeof options === 'function' || options === undefined)) {\n    name = null\n    opts = filename\n  }\n\n  // set Content-Disposition when file is sent\n  var headers = {\n    'Content-Disposition': contentDisposition(name || path)\n  };\n\n  // merge user-provided headers\n  if (opts && opts.headers) {\n    var keys = Object.keys(opts.headers)\n    for (var i = 0; i < keys.length; i++) {\n      var key = keys[i]\n      if (key.toLowerCase() !== 'content-disposition') {\n        headers[key] = opts.headers[key]\n      }\n    }\n  }\n\n  // merge user-provided options\n  opts = Object.create(opts)\n  opts.headers = headers\n\n  // Resolve the full path for sendFile\n  var fullPath = !opts.root\n    ? resolve(path)\n    : path\n\n  // send file\n  return this.sendFile(fullPath, opts, done)\n};\n\n/**\n * Set _Content-Type_ response header with `type` through `mime.lookup()`\n * when it does not contain \"/\", or set the Content-Type to `type` otherwise.\n *\n * Examples:\n *\n *     res.type('.html');\n *     res.type('html');\n *     res.type('json');\n *     res.type('application/json');\n *     res.type('png');\n *\n * @param {String} type\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.contentType =\nres.type = function contentType(type) {\n  var ct = type.indexOf('/') === -1\n    ? mime.lookup(type)\n    : type;\n\n  return this.set('Content-Type', ct);\n};\n\n/**\n * Respond to the Acceptable formats using an `obj`\n * of mime-type callbacks.\n *\n * This method uses `req.accepted`, an array of\n * acceptable types ordered by their quality values.\n * When \"Accept\" is not present the _first_ callback\n * is invoked, otherwise the first match is used. When\n * no match is performed the server responds with\n * 406 \"Not Acceptable\".\n *\n * Content-Type is set for you, however if you choose\n * you may alter this within the callback using `res.type()`\n * or `res.set('Content-Type', ...)`.\n *\n *    res.format({\n *      'text/plain': function(){\n *        res.send('hey');\n *      },\n *\n *      'text/html': function(){\n *        res.send('<p>hey</p>');\n *      },\n *\n *      'application/json': function () {\n *        res.send({ message: 'hey' });\n *      }\n *    });\n *\n * In addition to canonicalized MIME types you may\n * also use extnames mapped to these types:\n *\n *    res.format({\n *      text: function(){\n *        res.send('hey');\n *      },\n *\n *      html: function(){\n *        res.send('<p>hey</p>');\n *      },\n *\n *      json: function(){\n *        res.send({ message: 'hey' });\n *      }\n *    });\n *\n * By default Express passes an `Error`\n * with a `.status` of 406 to `next(err)`\n * if a match is not made. If you provide\n * a `.default` callback it will be invoked\n * instead.\n *\n * @param {Object} obj\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.format = function(obj){\n  var req = this.req;\n  var next = req.next;\n\n  var keys = Object.keys(obj)\n    .filter(function (v) { return v !== 'default' })\n\n  var key = keys.length > 0\n    ? req.accepts(keys)\n    : false;\n\n  this.vary(\"Accept\");\n\n  if (key) {\n    this.set('Content-Type', normalizeType(key).value);\n    obj[key](req, this, next);\n  } else if (obj.default) {\n    obj.default(req, this, next)\n  } else {\n    next(createError(406, {\n      types: normalizeTypes(keys).map(function (o) { return o.value })\n    }))\n  }\n\n  return this;\n};\n\n/**\n * Set _Content-Disposition_ header to _attachment_ with optional `filename`.\n *\n * @param {String} filename\n * @return {ServerResponse}\n * @public\n */\n\nres.attachment = function attachment(filename) {\n  if (filename) {\n    this.type(extname(filename));\n  }\n\n  this.set('Content-Disposition', contentDisposition(filename));\n\n  return this;\n};\n\n/**\n * Append additional header `field` with value `val`.\n *\n * Example:\n *\n *    res.append('Link', ['<http://localhost/>', '<http://localhost:3000/>']);\n *    res.append('Set-Cookie', 'foo=bar; Path=/; HttpOnly');\n *    res.append('Warning', '199 Miscellaneous warning');\n *\n * @param {String} field\n * @param {String|Array} val\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.append = function append(field, val) {\n  var prev = this.get(field);\n  var value = val;\n\n  if (prev) {\n    // concat the new and prev vals\n    value = Array.isArray(prev) ? prev.concat(val)\n      : Array.isArray(val) ? [prev].concat(val)\n        : [prev, val]\n  }\n\n  return this.set(field, value);\n};\n\n/**\n * Set header `field` to `val`, or pass\n * an object of header fields.\n *\n * Examples:\n *\n *    res.set('Foo', ['bar', 'baz']);\n *    res.set('Accept', 'application/json');\n *    res.set({ Accept: 'text/plain', 'X-API-Key': 'tobi' });\n *\n * Aliased as `res.header()`.\n *\n * @param {String|Object} field\n * @param {String|Array} val\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.set =\nres.header = function header(field, val) {\n  if (arguments.length === 2) {\n    var value = Array.isArray(val)\n      ? val.map(String)\n      : String(val);\n\n    // add charset to content-type\n    if (field.toLowerCase() === 'content-type') {\n      if (Array.isArray(value)) {\n        throw new TypeError('Content-Type cannot be set to an Array');\n      }\n      if (!charsetRegExp.test(value)) {\n        var charset = mime.charsets.lookup(value.split(';')[0]);\n        if (charset) value += '; charset=' + charset.toLowerCase();\n      }\n    }\n\n    this.setHeader(field, value);\n  } else {\n    for (var key in field) {\n      this.set(key, field[key]);\n    }\n  }\n  return this;\n};\n\n/**\n * Get value for header `field`.\n *\n * @param {String} field\n * @return {String}\n * @public\n */\n\nres.get = function(field){\n  return this.getHeader(field);\n};\n\n/**\n * Clear cookie `name`.\n *\n * @param {String} name\n * @param {Object} [options]\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.clearCookie = function clearCookie(name, options) {\n  var opts = merge({ expires: new Date(1), path: '/' }, options);\n\n  return this.cookie(name, '', opts);\n};\n\n/**\n * Set cookie `name` to `value`, with the given `options`.\n *\n * Options:\n *\n *    - `maxAge`   max-age in milliseconds, converted to `expires`\n *    - `signed`   sign the cookie\n *    - `path`     defaults to \"/\"\n *\n * Examples:\n *\n *    // \"Remember Me\" for 15 minutes\n *    res.cookie('rememberme', '1', { expires: new Date(Date.now() + 900000), httpOnly: true });\n *\n *    // same as above\n *    res.cookie('rememberme', '1', { maxAge: 900000, httpOnly: true })\n *\n * @param {String} name\n * @param {String|Object} value\n * @param {Object} [options]\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.cookie = function (name, value, options) {\n  var opts = merge({}, options);\n  var secret = this.req.secret;\n  var signed = opts.signed;\n\n  if (signed && !secret) {\n    throw new Error('cookieParser(\"secret\") required for signed cookies');\n  }\n\n  var val = typeof value === 'object'\n    ? 'j:' + JSON.stringify(value)\n    : String(value);\n\n  if (signed) {\n    val = 's:' + sign(val, secret);\n  }\n\n  if (opts.maxAge != null) {\n    var maxAge = opts.maxAge - 0\n\n    if (!isNaN(maxAge)) {\n      opts.expires = new Date(Date.now() + maxAge)\n      opts.maxAge = Math.floor(maxAge / 1000)\n    }\n  }\n\n  if (opts.path == null) {\n    opts.path = '/';\n  }\n\n  this.append('Set-Cookie', cookie.serialize(name, String(val), opts));\n\n  return this;\n};\n\n/**\n * Set the location header to `url`.\n *\n * The given `url` can also be \"back\", which redirects\n * to the _Referrer_ or _Referer_ headers or \"/\".\n *\n * Examples:\n *\n *    res.location('/foo/bar').;\n *    res.location('http://example.com');\n *    res.location('../login');\n *\n * @param {String} url\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.location = function location(url) {\n  var loc;\n\n  // \"back\" is an alias for the referrer\n  if (url === 'back') {\n    loc = this.req.get('Referrer') || '/';\n  } else {\n    loc = String(url);\n  }\n\n  var m = schemaAndHostRegExp.exec(loc);\n  var pos = m ? m[0].length + 1 : 0;\n\n  // Only encode after host to avoid invalid encoding which can introduce\n  // vulnerabilities (e.g. `\\\\` to `%5C`).\n  loc = loc.slice(0, pos) + encodeUrl(loc.slice(pos));\n\n  return this.set('Location', loc);\n};\n\n/**\n * Redirect to the given `url` with optional response `status`\n * defaulting to 302.\n *\n * The resulting `url` is determined by `res.location()`, so\n * it will play nicely with mounted apps, relative paths,\n * `\"back\"` etc.\n *\n * Examples:\n *\n *    res.redirect('/foo/bar');\n *    res.redirect('http://example.com');\n *    res.redirect(301, 'http://example.com');\n *    res.redirect('../login'); // /blog/post/1 -> /blog/login\n *\n * @public\n */\n\nres.redirect = function redirect(url) {\n  var address = url;\n  var body;\n  var status = 302;\n\n  // allow status / url\n  if (arguments.length === 2) {\n    if (typeof arguments[0] === 'number') {\n      status = arguments[0];\n      address = arguments[1];\n    } else {\n      deprecate('res.redirect(url, status): Use res.redirect(status, url) instead');\n      status = arguments[1];\n    }\n  }\n\n  // Set location header\n  address = this.location(address).get('Location');\n\n  // Support text/{plain,html} by default\n  this.format({\n    text: function(){\n      body = statuses.message[status] + '. Redirecting to ' + address\n    },\n\n    html: function(){\n      var u = escapeHtml(address);\n      body = '<p>' + statuses.message[status] + '. Redirecting to <a href=\"' + u + '\">' + u + '</a></p>'\n    },\n\n    default: function(){\n      body = '';\n    }\n  });\n\n  // Respond\n  this.statusCode = status;\n  this.set('Content-Length', Buffer.byteLength(body));\n\n  if (this.req.method === 'HEAD') {\n    this.end();\n  } else {\n    this.end(body);\n  }\n};\n\n/**\n * Add `field` to Vary. If already present in the Vary set, then\n * this call is simply ignored.\n *\n * @param {Array|String} field\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.vary = function(field){\n  // checks for back-compat\n  if (!field || (Array.isArray(field) && !field.length)) {\n    deprecate('res.vary(): Provide a field name');\n    return this;\n  }\n\n  vary(this, field);\n\n  return this;\n};\n\n/**\n * Render `view` with the given `options` and optional callback `fn`.\n * When a callback function is given a response will _not_ be made\n * automatically, otherwise a response of _200_ and _text/html_ is given.\n *\n * Options:\n *\n *  - `cache`     boolean hinting to the engine it should cache\n *  - `filename`  filename of the view being rendered\n *\n * @public\n */\n\nres.render = function render(view, options, callback) {\n  var app = this.req.app;\n  var done = callback;\n  var opts = options || {};\n  var req = this.req;\n  var self = this;\n\n  // support callback function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // merge res.locals\n  opts._locals = self.locals;\n\n  // default callback to respond\n  done = done || function (err, str) {\n    if (err) return req.next(err);\n    self.send(str);\n  };\n\n  // render\n  app.render(view, opts, done);\n};\n\n// pipe the send file stream\nfunction sendfile(res, file, options, callback) {\n  var done = false;\n  var streaming;\n\n  // request aborted\n  function onaborted() {\n    if (done) return;\n    done = true;\n...\n", "needle_token_start": 1936, "needle_token_end": 1947, "code_context_ntokens": 10450, "output": ["```javascript\n/**\n * Return new empty object.\n *\n * @return {Object}\n * @api private\n */\nfunction newObject() {\n  return {};\n}\n```"]}
{"repo": "expressjs/express", "name": "logerror", "language": "typescript", "path": "lib/application.js", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to log errors to the console, specifically avoiding logging when the application is in a test environment.\n2. **Input**: It accepts an error object as its sole parameter.\n3. **Output**: There is no return value; the output is the error information printed to the console.\n4. **Procedure**: The function first checks if the application's environment is not set to 'test'. If this condition is true, it logs the error to the console. It prints either the error stack if available, or the error's string representation if the stack is not available.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/router/index.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Route = require('./route');\nvar Layer = require('./layer');\nvar methods = require('methods');\nvar mixin = require('utils-merge');\nvar debug = require('debug')('express:router');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar parseUrl = require('parseurl');\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar objectRegExp = /^\\[object (\\S+)\\]$/;\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n\n/**\n * Map the given param placeholder `name`(s) to the given callback.\n *\n * Parameter mapping is used to provide pre-conditions to routes\n * which use normalized placeholders. For example a _:user_id_ parameter\n * could automatically load a user's information from the database without\n * any additional code,\n *\n * The callback uses the same signature as middleware, the only difference\n * being that the value of the placeholder is passed, in this case the _id_\n * of the user. Once the `next()` function is invoked, just like middleware\n * it will continue on to execute the route, or subsequent parameter functions.\n *\n * Just like in middleware, you must either respond to the request or call next\n * to avoid stalling the request.\n *\n *  app.param('user_id', function(req, res, next, id){\n *    User.find(id, function(err, user){\n *      if (err) {\n *        return next(err);\n *      } else if (!user) {\n *        return next(new Error('failed to load user'));\n *      }\n *      req.user = user;\n *      next();\n *    });\n *  });\n *\n * @param {String} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\nproto.param = function param(name, fn) {\n  // param logic\n  if (typeof name === 'function') {\n    deprecate('router.param(fn): Refactor to use path params');\n    this._params.push(name);\n    return;\n  }\n\n  // apply param functions\n  var params = this._params;\n  var len = params.length;\n  var ret;\n\n  if (name[0] === ':') {\n    deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.slice(1)) + ', fn) instead')\n    name = name.slice(1)\n  }\n\n  for (var i = 0; i < len; ++i) {\n    if (ret = params[i](name, fn)) {\n      fn = ret;\n    }\n  }\n\n  // ensure we end up with a\n  // middleware function\n  if ('function' !== typeof fn) {\n    throw new Error('invalid param() call for ' + name + ', got ' + fn);\n  }\n\n  (this.params[name] = this.params[name] || []).push(fn);\n  return this;\n};\n\n/**\n * Dispatch a req, res into the router.\n * @private\n */\n\nproto.handle = function handle(req, res, out) {\n  var self = this;\n\n  debug('dispatching %s %s', req.method, req.url);\n\n  var idx = 0;\n  var protohost = getProtohost(req.url) || ''\n  var removed = '';\n  var slashAdded = false;\n  var sync = 0\n  var paramcalled = {};\n\n  // store options for OPTIONS request\n  // only used if OPTIONS request\n  var options = [];\n\n  // middleware and routes\n  var stack = self.stack;\n\n  // manage inter-router variables\n  var parentParams = req.params;\n  var parentUrl = req.baseUrl || '';\n  var done = restore(out, req, 'baseUrl', 'next', 'params');\n\n  // setup next layer\n  req.next = next;\n\n  // for options requests, respond with a default if nothing else responds\n  if (req.method === 'OPTIONS') {\n    done = wrap(done, function(old, err) {\n      if (err || options.length === 0) return old(err);\n      sendOptionsResponse(res, options, old);\n    });\n  }\n\n  // setup basic req values\n  req.baseUrl = parentUrl;\n  req.originalUrl = req.originalUrl || req.url;\n\n  next();\n\n  function next(err) {\n    var layerError = err === 'route'\n      ? null\n      : err;\n\n    // remove added slash\n    if (slashAdded) {\n      req.url = req.url.slice(1)\n      slashAdded = false;\n    }\n\n    // restore altered req.url\n    if (removed.length !== 0) {\n      req.baseUrl = parentUrl;\n      req.url = protohost + removed + req.url.slice(protohost.length)\n      removed = '';\n    }\n\n    // signal to exit router\n    if (layerError === 'router') {\n      setImmediate(done, null)\n      return\n    }\n\n    // no more matching layers\n    if (idx >= stack.length) {\n      setImmediate(done, layerError);\n      return;\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    // get pathname of request\n    var path = getPathname(req);\n\n    if (path == null) {\n      return done(layerError);\n    }\n\n    // find next matching layer\n    var layer;\n    var match;\n    var route;\n\n    while (match !== true && idx < stack.length) {\n      layer = stack[idx++];\n      match = matchLayer(layer, path);\n      route = layer.route;\n\n      if (typeof match !== 'boolean') {\n        // hold on to layerError\n        layerError = layerError || match;\n      }\n\n      if (match !== true) {\n        continue;\n      }\n\n      if (!route) {\n        // process non-route handlers normally\n        continue;\n      }\n\n      if (layerError) {\n        // routes do not match with a pending error\n        match = false;\n        continue;\n      }\n\n      var method = req.method;\n      var has_method = route._handles_method(method);\n\n      // build up automatic options response\n      if (!has_method && method === 'OPTIONS') {\n        appendMethods(options, route._options());\n      }\n\n      // don't even bother matching route\n      if (!has_method && method !== 'HEAD') {\n        match = false;\n      }\n    }\n\n    // no match\n    if (match !== true) {\n      return done(layerError);\n    }\n\n    // store route for dispatch on change\n    if (route) {\n      req.route = route;\n    }\n\n    // Capture one-time layer values\n    req.params = self.mergeParams\n      ? mergeParams(layer.params, parentParams)\n      : layer.params;\n    var layerPath = layer.path;\n\n    // this should be done for the layer\n...\n// Path: lib/middleware/init.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Initialization middleware, exposing the\n * request and response to each other, as well\n * as defaulting the X-Powered-By header field.\n *\n * @param {Function} app\n * @return {Function}\n * @api private\n */\n\nexports.init = function(app){\n  return function expressInit(req, res, next){\n    if (app.enabled('x-powered-by')) res.setHeader('X-Powered-By', 'Express');\n    req.res = res;\n    res.req = req;\n    req.next = next;\n\n    setPrototypeOf(req, app.request)\n    setPrototypeOf(res, app.response)\n\n    res.locals = res.locals || Object.create(null);\n\n    next();\n  };\n};\n\n\n// Path: lib/view.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:view');\nvar path = require('path');\nvar fs = require('fs');\n\n/**\n * Module variables.\n * @private\n */\n\nvar dirname = path.dirname;\nvar basename = path.basename;\nvar extname = path.extname;\nvar join = path.join;\nvar resolve = path.resolve;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = View;\n\n/**\n * Initialize a new `View` with the given `name`.\n *\n * Options:\n *\n *   - `defaultEngine` the default template engine name\n *   - `engines` template engine require() cache\n *   - `root` root path for view lookup\n *\n * @param {string} name\n * @param {object} options\n * @public\n */\n\nfunction View(name, options) {\n  var opts = options || {};\n\n  this.defaultEngine = opts.defaultEngine;\n  this.ext = extname(name);\n  this.name = name;\n  this.root = opts.root;\n\n  if (!this.ext && !this.defaultEngine) {\n    throw new Error('No default engine was specified and no extension was provided.');\n  }\n\n  var fileName = name;\n\n  if (!this.ext) {\n    // get extension from default engine name\n    this.ext = this.defaultEngine[0] !== '.'\n      ? '.' + this.defaultEngine\n      : this.defaultEngine;\n\n    fileName += this.ext;\n  }\n\n  if (!opts.engines[this.ext]) {\n    // load engine\n    var mod = this.ext.slice(1)\n    debug('require \"%s\"', mod)\n\n    // default engine export\n    var fn = require(mod).__express\n\n    if (typeof fn !== 'function') {\n      throw new Error('Module \"' + mod + '\" does not provide a view engine.')\n    }\n\n    opts.engines[this.ext] = fn\n  }\n\n  // store loaded engine\n  this.engine = opts.engines[this.ext];\n\n  // lookup path\n  this.path = this.lookup(fileName);\n}\n\n/**\n * Lookup view by the given `name`\n *\n * @param {string} name\n * @private\n */\n\nView.prototype.lookup = function lookup(name) {\n  var path;\n  var roots = [].concat(this.root);\n\n  debug('lookup \"%s\"', name);\n\n  for (var i = 0; i < roots.length && !path; i++) {\n    var root = roots[i];\n\n    // resolve the path\n    var loc = resolve(root, name);\n    var dir = dirname(loc);\n    var file = basename(loc);\n\n    // resolve the file\n    path = this.resolve(dir, file);\n  }\n\n  return path;\n};\n\n/**\n * Render with the given options.\n *\n * @param {object} options\n * @param {function} callback\n * @private\n */\n\nView.prototype.render = function render(options, callback) {\n  debug('render \"%s\"', this.path);\n  this.engine(this.path, options, callback);\n};\n\n/**\n * Resolve the file within the given directory.\n *\n * @param {string} dir\n * @param {string} file\n * @private\n */\n\nView.prototype.resolve = function resolve(dir, file) {\n  var ext = this.ext;\n\n  // <path>.<ext>\n  var path = join(dir, file);\n  var stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n\n  // <path>/index.<ext>\n  path = join(dir, basename(file, ext), 'index' + ext);\n  stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n};\n\n/**\n * Return a stat, maybe.\n *\n * @param {string} path\n * @return {fs.Stats}\n * @private\n */\n\nfunction tryStat(path) {\n  debug('stat \"%s\"', path);\n\n  try {\n    return fs.statSync(path);\n  } catch (e) {\n    return undefined;\n  }\n}\n\n// Path: lib/middleware/query.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar merge = require('utils-merge')\nvar parseUrl = require('parseurl');\nvar qs = require('qs');\n\n/**\n * @param {Object} options\n * @return {Function}\n * @api public\n */\n\nmodule.exports = function query(options) {\n  var opts = merge({}, options)\n  var queryparse = qs.parse;\n\n  if (typeof options === 'function') {\n    queryparse = options;\n    opts = undefined;\n  }\n\n  if (opts !== undefined && opts.allowPrototypes === undefined) {\n    // back-compat for qs module\n    opts.allowPrototypes = true;\n  }\n\n  return function query(req, res, next){\n    if (!req.query) {\n      var val = parseUrl(req).query;\n      req.query = queryparse(val, opts);\n    }\n\n    next();\n  };\n};\n\n// Path: lib/application.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar finalhandler = require('finalhandler');\nvar Router = require('./router');\nvar methods = require('methods');\nvar middleware = require('./middleware/init');\nvar query = require('./middleware/query');\nvar debug = require('debug')('express:application');\nvar View = require('./view');\nvar http = require('http');\nvar compileETag = require('./utils').compileETag;\nvar compileQueryParser = require('./utils').compileQueryParser;\nvar compileTrust = require('./utils').compileTrust;\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar merge = require('utils-merge');\nvar resolve = require('path').resolve;\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty\nvar slice = Array.prototype.slice;\n\n/**\n * Application prototype.\n */\n\nvar app = exports = module.exports = {};\n\n/**\n * Variable for trust proxy inheritance back-compat\n * @private\n */\n\nvar trustProxyDefaultSymbol = '@@symbol:trust_proxy_default';\n\n/**\n * Initialize the server.\n *\n *   - setup default configuration\n *   - setup default middleware\n *   - setup route reflection methods\n *\n * @private\n */\n\napp.init = function init() {\n  this.cache = {};\n  this.engines = {};\n  this.settings = {};\n\n  this.defaultConfiguration();\n};\n\n/**\n * Initialize application configuration.\n * @private\n */\n\napp.defaultConfiguration = function defaultConfiguration() {\n  var env = process.env.NODE_ENV || 'development';\n\n  // default settings\n  this.enable('x-powered-by');\n  this.set('etag', 'weak');\n  this.set('env', env);\n  this.set('query parser', 'extended');\n  this.set('subdomain offset', 2);\n  this.set('trust proxy', false);\n\n  // trust proxy inherit back-compat\n  Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n    configurable: true,\n    value: true\n  });\n\n  debug('booting in %s mode', env);\n\n  this.on('mount', function onmount(parent) {\n    // inherit trust proxy\n    if (this.settings[trustProxyDefaultSymbol] === true\n      && typeof parent.settings['trust proxy fn'] === 'function') {\n      delete this.settings['trust proxy'];\n      delete this.settings['trust proxy fn'];\n    }\n\n    // inherit protos\n    setPrototypeOf(this.request, parent.request)\n    setPrototypeOf(this.response, parent.response)\n    setPrototypeOf(this.engines, parent.engines)\n    setPrototypeOf(this.settings, parent.settings)\n  });\n\n  // setup locals\n  this.locals = Object.create(null);\n\n  // top-most app is mounted at /\n  this.mountpath = '/';\n\n  // default locals\n  this.locals.settings = this.settings;\n\n  // default configuration\n  this.set('view', View);\n  this.set('views', resolve('views'));\n  this.set('jsonp callback name', 'callback');\n\n  if (env === 'production') {\n    this.enable('view cache');\n  }\n\n  Object.defineProperty(this, 'router', {\n    get: function() {\n      throw new Error('\\'app.router\\' is deprecated!\\nPlease see the 3.x to 4.x migration guide for details on how to update your app.');\n    }\n  });\n};\n\n/**\n * lazily adds the base router if it has not yet been added.\n *\n * We cannot add the base router in the defaultConfiguration because\n * it reads app settings which might be set after that has run.\n *\n * @private\n */\napp.lazyrouter = function lazyrouter() {\n  if (!this._router) {\n    this._router = new Router({\n      caseSensitive: this.enabled('case sensitive routing'),\n      strict: this.enabled('strict routing')\n    });\n\n    this._router.use(query(this.get('query parser fn')));\n    this._router.use(middleware.init(this));\n  }\n};\n\n/**\n * Dispatch a req, res pair into the application. Starts pipeline processing.\n *\n * If no callback is provided, then default error handlers will respond\n * in the event of an error bubbling through the stack.\n *\n * @private\n */\n\napp.handle = function handle(req, res, callback) {\n  var router = this._router;\n\n  // final handler\n  var done = callback || finalhandler(req, res, {\n    env: this.get('env'),\n    onerror: logerror.bind(this)\n  });\n\n  // no routes\n  if (!router) {\n    debug('no routes defined on app');\n    done();\n    return;\n  }\n\n  router.handle(req, res, done);\n};\n\n/**\n * Proxy `Router#use()` to add middleware to the app router.\n * See Router#use() documentation for details.\n *\n * If the _fn_ parameter is an express app, then it will be\n * mounted at the _route_ specified.\n *\n * @public\n */\n\napp.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate app.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var fns = flatten(slice.call(arguments, offset));\n\n  if (fns.length === 0) {\n    throw new TypeError('app.use() requires a middleware function')\n  }\n\n  // setup router\n  this.lazyrouter();\n  var router = this._router;\n\n  fns.forEach(function (fn) {\n    // non-express app\n    if (!fn || !fn.handle || !fn.set) {\n      return router.use(path, fn);\n    }\n\n    debug('.use app under %s', path);\n    fn.mountpath = path;\n    fn.parent = this;\n\n    // restore .app property on req and res\n    router.use(path, function mounted_app(req, res, next) {\n      var orig = req.app;\n      fn.handle(req, res, function (err) {\n        setPrototypeOf(req, orig.request)\n        setPrototypeOf(res, orig.response)\n        next(err);\n      });\n    });\n\n    // mounted an app\n    fn.emit('mount', this);\n  }, this);\n\n  return this;\n};\n\n/**\n * Proxy to the app `Router#route()`\n * Returns a new `Route` instance for the _path_.\n *\n * Routes are isolated middleware stacks for specific paths.\n * See the Route api docs for details.\n *\n * @public\n */\n\napp.route = function route(path) {\n  this.lazyrouter();\n  return this._router.route(path);\n};\n\n/**\n * Register the given template engine callback `fn`\n * as `ext`.\n *\n * By default will `require()` the engine based on the\n * file extension. For example if you try to render\n * a \"foo.ejs\" file Express will invoke the following internally:\n *\n *     app.engine('ejs', require('ejs').__express);\n *\n * For engines that do not provide `.__express` out of the box,\n * or if you wish to \"map\" a different extension to the template engine\n * you may use this method. For example mapping the EJS template engine to\n * \".html\" files:\n *\n *     app.engine('html', require('ejs').renderFile);\n *\n * In this case EJS provides a `.renderFile()` method with\n * the same signature that Express expects: `(path, options, callback)`,\n * though note that it aliases this method as `ejs.__express` internally\n * so if you're using \".ejs\" extensions you don't need to do anything.\n *\n * Some template engines do not follow this convention, the\n * [Consolidate.js](https://github.com/tj/consolidate.js)\n * library was created to map all of node's popular template\n * engines to follow this convention, thus allowing them to\n * work seamlessly within Express.\n *\n * @param {String} ext\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.engine = function engine(ext, fn) {\n  if (typeof fn !== 'function') {\n    throw new Error('callback function required');\n  }\n\n  // get file extension\n  var extension = ext[0] !== '.'\n    ? '.' + ext\n    : ext;\n\n  // store engine\n  this.engines[extension] = fn;\n\n  return this;\n};\n\n/**\n * Proxy to `Router#param()` with one added api feature. The _name_ parameter\n * can be an array of names.\n *\n * See the Router#param() docs for more details.\n *\n * @param {String|Array} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\napp.param = function param(name, fn) {\n  this.lazyrouter();\n\n  if (Array.isArray(name)) {\n    for (var i = 0; i < name.length; i++) {\n      this.param(name[i], fn);\n    }\n\n    return this;\n  }\n\n  this._router.param(name, fn);\n\n  return this;\n};\n\n/**\n * Assign `setting` to `val`, or return `setting`'s value.\n *\n *    app.set('foo', 'bar');\n *    app.set('foo');\n *    // => \"bar\"\n *\n * Mounted servers inherit their parent server's settings.\n *\n * @param {String} setting\n * @param {*} [val]\n * @return {Server} for chaining\n * @public\n */\n\napp.set = function set(setting, val) {\n  if (arguments.length === 1) {\n    // app.get(setting)\n    var settings = this.settings\n\n    while (settings && settings !== Object.prototype) {\n      if (hasOwnProperty.call(settings, setting)) {\n        return settings[setting]\n      }\n\n      settings = Object.getPrototypeOf(settings)\n    }\n\n    return undefined\n  }\n\n  debug('set \"%s\" to %o', setting, val);\n\n  // set value\n  this.settings[setting] = val;\n\n  // trigger matched settings\n  switch (setting) {\n    case 'etag':\n      this.set('etag fn', compileETag(val));\n      break;\n    case 'query parser':\n      this.set('query parser fn', compileQueryParser(val));\n      break;\n    case 'trust proxy':\n      this.set('trust proxy fn', compileTrust(val));\n\n      // trust proxy inherit back-compat\n      Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n        configurable: true,\n        value: false\n      });\n\n      break;\n  }\n\n  return this;\n};\n\n/**\n * Return the app's absolute pathname\n * based on the parent(s) that have\n * mounted it.\n *\n * For example if the application was\n * mounted as \"/admin\", which itself\n * was mounted as \"/blog\" then the\n * return value would be \"/blog/admin\".\n *\n * @return {String}\n * @private\n */\n\napp.path = function path() {\n  return this.parent\n    ? this.parent.path() + this.mountpath\n    : '';\n};\n\n/**\n * Check if `setting` is enabled (truthy).\n *\n *    app.enabled('foo')\n *    // => false\n *\n *    app.enable('foo')\n *    app.enabled('foo')\n *    // => true\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.enabled = function enabled(setting) {\n  return Boolean(this.set(setting));\n};\n\n/**\n * Check if `setting` is disabled.\n *\n *    app.disabled('foo')\n *    // => true\n *\n *    app.enable('foo')\n *    app.disabled('foo')\n *    // => false\n *\n * @param {String} setting\n * @return {Boolean}\n * @public\n */\n\napp.disabled = function disabled(setting) {\n  return !this.set(setting);\n};\n\n/**\n * Enable `setting`.\n *\n * @param {String} setting\n * @return {app} for chaining\n * @public\n */\n\napp.enable = function enable(setting) {\n  return this.set(setting, true);\n};\n\n/**\n * Disable `setting`.\n *\n * @param {String} setting\n * @return {app} for chaining\n * @public\n */\n\napp.disable = function disable(setting) {\n  return this.set(setting, false);\n};\n\n/**\n * Delegate `.VERB(...)` calls to `router.VERB(...)`.\n */\n\nmethods.forEach(function(method){\n  app[method] = function(path){\n    if (method === 'get' && arguments.length === 1) {\n      // app.get(setting)\n      return this.set(path);\n    }\n\n    this.lazyrouter();\n\n    var route = this._router.route(path);\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n/**\n * Special-cased \"all\" method, applying the given route `path`,\n * middleware, and callback to _every_ HTTP method.\n *\n * @param {String} path\n * @param {Function} ...\n * @return {app} for chaining\n * @public\n */\n\napp.all = function all(path) {\n  this.lazyrouter();\n\n  var route = this._router.route(path);\n  var args = slice.call(arguments, 1);\n\n  for (var i = 0; i < methods.length; i++) {\n    route[methods[i]].apply(route, args);\n  }\n\n  return this;\n};\n\n// del -> delete alias\n\napp.del = deprecate.function(app.delete, 'app.del: Use app.delete instead');\n\n/**\n * Render the given view `name` name with `options`\n * and a callback accepting an error and the\n * rendered template string.\n *\n * Example:\n *\n *    app.render('email', { name: 'Tobi' }, function(err, html){\n *      // ...\n *    })\n *\n * @param {String} name\n * @param {Object|Function} options or fn\n * @param {Function} callback\n * @public\n */\n\napp.render = function render(name, options, callback) {\n  var cache = this.cache;\n  var done = callback;\n  var engines = this.engines;\n  var opts = options;\n  var renderOptions = {};\n  var view;\n\n  // support callback function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // merge app.locals\n  merge(renderOptions, this.locals);\n\n  // merge options._locals\n  if (opts._locals) {\n    merge(renderOptions, opts._locals);\n  }\n\n  // merge options\n  merge(renderOptions, opts);\n\n  // set .cache unless explicitly provided\n  if (renderOptions.cache == null) {\n    renderOptions.cache = this.enabled('view cache');\n  }\n\n  // primed cache\n  if (renderOptions.cache) {\n    view = cache[name];\n  }\n\n  // view\n  if (!view) {\n    var View = this.get('view');\n\n    view = new View(name, {\n      defaultEngine: this.get('view engine'),\n      root: this.get('views'),\n      engines: engines\n    });\n\n    if (!view.path) {\n      var dirs = Array.isArray(view.root) && view.root.length > 1\n        ? 'directories \"' + view.root.slice(0, -1).join('\", \"') + '\" or \"' + view.root[view.root.length - 1] + '\"'\n        : 'directory \"' + view.root + '\"'\n      var err = new Error('Failed to lookup view \"' + name + '\" in views ' + dirs);\n      err.view = view;\n      return done(err);\n    }\n\n    // prime the cache\n    if (renderOptions.cache) {\n      cache[name] = view;\n    }\n  }\n\n  // render\n  tryRender(view, renderOptions, done);\n};\n\n/**\n * Listen for connections.\n *\n * A node `http.Server` is returned, with this\n * application (which is a `Function`) as its\n * callback. If you wish to create both an HTTP\n * and HTTPS server you may do so with the \"http\"\n * and \"https\" modules as shown here:\n *\n *    var http = require('http')\n *      , https = require('https')\n *      , express = require('express')\n *      , app = express();\n *\n *    http.createServer(app).listen(80);\n *    https.createServer({ ... }, app).listen(443);\n *\n * @return {http.Server}\n * @public\n */\n\napp.listen = function listen() {\n  var server = http.createServer(this);\n  return server.listen.apply(server, arguments);\n};\n\n/**\n * Log error using console.error.\n *\n * @param {Error} err\n * @private\n */\n\n\nfunction logerror(err) {\n  /* istanbul ignore next */\n  if (this.get('env') !== 'test') console.error(err.stack || err.toString());\n}\n\n/**\n * Try rendering a view.\n * @private\n */\n\nfunction tryRender(view, options, callback) {\n  try {\n    view.render(options, callback);\n  } catch (err) {\n    callback(err);\n  }\n}\n\n// Path: lib/express.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar bodyParser = require('body-parser')\nvar EventEmitter = require('events').EventEmitter;\nvar mixin = require('merge-descriptors');\nvar proto = require('./application');\nvar Route = require('./router/route');\nvar Router = require('./router');\nvar req = require('./request');\nvar res = require('./response');\n\n/**\n * Expose `createApplication()`.\n */\n\nexports = module.exports = createApplication;\n\n/**\n * Create an express application.\n *\n * @return {Function}\n * @api public\n */\n\nfunction createApplication() {\n  var app = function(req, res, next) {\n    app.handle(req, res, next);\n  };\n\n  mixin(app, EventEmitter.prototype, false);\n  mixin(app, proto, false);\n\n  // expose the prototype that will get set on requests\n  app.request = Object.create(req, {\n    app: { configurable: true, enumerable: true, writable: true, value: app }\n  })\n\n  // expose the prototype that will get set on responses\n  app.response = Object.create(res, {\n    app: { configurable: true, enumerable: true, writable: true, value: app }\n  })\n\n  app.init();\n  return app;\n}\n\n/**\n * Expose the prototypes.\n */\n\nexports.application = proto;\nexports.request = req;\nexports.response = res;\n\n/**\n * Expose constructors.\n */\n\nexports.Route = Route;\nexports.Router = Router;\n\n/**\n * Expose middleware\n */\n\nexports.json = bodyParser.json\nexports.query = require('./middleware/query');\nexports.raw = bodyParser.raw\nexports.static = require('serve-static');\nexports.text = bodyParser.text\nexports.urlencoded = bodyParser.urlencoded\n\n/**\n * Replace removed middleware with an appropriate error message.\n */\n\nvar removedMiddlewares = [\n  'bodyParser',\n  'compress',\n  'cookieSession',\n  'session',\n  'logger',\n  'cookieParser',\n  'favicon',\n  'responseTime',\n  'errorHandler',\n  'timeout',\n  'methodOverride',\n  'vhost',\n  'csrf',\n  'directory',\n  'limit',\n  'multipart',\n  'staticCache'\n]\n\nremovedMiddlewares.forEach(function (name) {\n  Object.defineProperty(exports, name, {\n    get: function () {\n      throw new Error('Most middleware (like ' + name + ') is no longer bundled with Express and must be installed separately. Please see https://github.com/senchalabs/connect#middleware.');\n    },\n    configurable: true\n  });\n});\n\n", "needle_token_start": 8425, "needle_token_end": 8469, "code_context_ntokens": 9319, "output": ["```javascript\n/**\n * Log error using console.error.\n *\n * @param {Error} err\n * @private\n */\n\nfunction logerror(err) {\n  /* istanbul ignore next */\n  if (this.get('env') !== 'test') console.error(err.stack || err.toString());\n}\n```"]}
{"repo": "expressjs/express", "name": "acceptParams", "language": "typescript", "path": "lib/utils.js", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to parse a string representing media type parameters into a structured format, extracting and interpreting each component, including quality factors and any additional parameters.\n2. **Input**: A single string that typically includes a media type followed by semicolon-separated parameters.\n3. **Output**: An object that includes the main media type value, a quality factor (defaulting to 1 if not specified), and an object containing any additional parameters specified in the input string.\n4. **Procedure**: The function splits the input string by semicolons to separate the main value from any parameters. The first part is treated as the media type value. Subsequent parts are further split by equals signs to separate parameter names from their values. If a parameter is recognized as a quality factor ('q'), it is converted to a floating-point number; otherwise, it is stored as a string in the parameters object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/utils.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @api private\n */\n\nvar Buffer = require('safe-buffer').Buffer\nvar contentDisposition = require('content-disposition');\nvar contentType = require('content-type');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar mime = require('send').mime;\nvar etag = require('etag');\nvar proxyaddr = require('proxy-addr');\nvar qs = require('qs');\nvar querystring = require('querystring');\n\n/**\n * Return strong ETag for `body`.\n *\n * @param {String|Buffer} body\n * @param {String} [encoding]\n * @return {String}\n * @api private\n */\n\nexports.etag = createETagGenerator({ weak: false })\n\n/**\n * Return weak ETag for `body`.\n *\n * @param {String|Buffer} body\n * @param {String} [encoding]\n * @return {String}\n * @api private\n */\n\nexports.wetag = createETagGenerator({ weak: true })\n\n/**\n * Check if `path` looks absolute.\n *\n * @param {String} path\n * @return {Boolean}\n * @api private\n */\n\nexports.isAbsolute = function(path){\n  if ('/' === path[0]) return true;\n  if (':' === path[1] && ('\\\\' === path[2] || '/' === path[2])) return true; // Windows device path\n  if ('\\\\\\\\' === path.substring(0, 2)) return true; // Microsoft Azure absolute path\n};\n\n/**\n * Flatten the given `arr`.\n *\n * @param {Array} arr\n * @return {Array}\n * @api private\n */\n\nexports.flatten = deprecate.function(flatten,\n  'utils.flatten: use array-flatten npm module instead');\n\n/**\n * Normalize the given `type`, for example \"html\" becomes \"text/html\".\n *\n * @param {String} type\n * @return {Object}\n * @api private\n */\n\nexports.normalizeType = function(type){\n  return ~type.indexOf('/')\n    ? acceptParams(type)\n    : { value: mime.lookup(type), params: {} };\n};\n\n/**\n * Normalize `types`, for example \"html\" becomes \"text/html\".\n *\n * @param {Array} types\n * @return {Array}\n * @api private\n */\n\nexports.normalizeTypes = function(types){\n  var ret = [];\n\n  for (var i = 0; i < types.length; ++i) {\n    ret.push(exports.normalizeType(types[i]));\n  }\n\n  return ret;\n};\n\n/**\n * Generate Content-Disposition header appropriate for the filename.\n * non-ascii filenames are urlencoded and a filename* parameter is added\n *\n * @param {String} filename\n * @return {String}\n * @api private\n */\n\nexports.contentDisposition = deprecate.function(contentDisposition,\n  'utils.contentDisposition: use content-disposition npm module instead');\n\n/**\n * Parse accept params `str` returning an\n * object with `.value`, `.quality` and `.params`.\n *\n * @param {String} str\n * @return {Object}\n * @api private\n */\n\n\nfunction acceptParams (str) {\n  var parts = str.split(/ *; */);\n  var ret = { value: parts[0], quality: 1, params: {} }\n\n  for (var i = 1; i < parts.length; ++i) {\n    var pms = parts[i].split(/ *= */);\n    if ('q' === pms[0]) {\n      ret.quality = parseFloat(pms[1]);\n    } else {\n      ret.params[pms[0]] = pms[1];\n    }\n  }\n\n  return ret;\n}\n\n/**\n * Compile \"etag\" value to function.\n *\n * @param  {Boolean|String|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileETag = function(val) {\n  var fn;\n\n  if (typeof val === 'function') {\n    return val;\n  }\n\n  switch (val) {\n    case true:\n    case 'weak':\n      fn = exports.wetag;\n      break;\n    case false:\n      break;\n    case 'strong':\n      fn = exports.etag;\n      break;\n    default:\n      throw new TypeError('unknown value for etag function: ' + val);\n  }\n\n  return fn;\n}\n\n/**\n * Compile \"query parser\" value to function.\n *\n * @param  {String|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileQueryParser = function compileQueryParser(val) {\n  var fn;\n\n  if (typeof val === 'function') {\n    return val;\n  }\n\n  switch (val) {\n    case true:\n    case 'simple':\n      fn = querystring.parse;\n      break;\n    case false:\n      fn = newObject;\n      break;\n    case 'extended':\n      fn = parseExtendedQueryString;\n      break;\n    default:\n      throw new TypeError('unknown value for query parser function: ' + val);\n  }\n\n  return fn;\n}\n\n/**\n * Compile \"proxy trust\" value to function.\n *\n * @param  {Boolean|String|Number|Array|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileTrust = function(val) {\n  if (typeof val === 'function') return val;\n\n  if (val === true) {\n    // Support plain true/false\n    return function(){ return true };\n  }\n\n  if (typeof val === 'number') {\n    // Support trusting hop count\n    return function(a, i){ return i < val };\n  }\n\n  if (typeof val === 'string') {\n    // Support comma-separated values\n    val = val.split(',')\n      .map(function (v) { return v.trim() })\n  }\n\n  return proxyaddr.compile(val || []);\n}\n\n/**\n * Set the charset in a given Content-Type string.\n *\n * @param {String} type\n * @param {String} charset\n * @return {String}\n * @api private\n */\n\nexports.setCharset = function setCharset(type, charset) {\n  if (!type || !charset) {\n    return type;\n  }\n\n  // parse type\n  var parsed = contentType.parse(type);\n\n  // set charset\n  parsed.parameters.charset = charset;\n\n  // format type\n  return contentType.format(parsed);\n};\n\n/**\n * Create an ETag generator function, generating ETags with\n * the given options.\n *\n * @param {object} options\n * @return {function}\n * @private\n */\n\nfunction createETagGenerator (options) {\n  return function generateETag (body, encoding) {\n    var buf = !Buffer.isBuffer(body)\n      ? Buffer.from(body, encoding)\n      : body\n\n    return etag(buf, options)\n  }\n}\n\n/**\n * Parse an extended query string with qs.\n *\n * @param {String} str\n * @return {Object}\n * @private\n */\n\nfunction parseExtendedQueryString(str) {\n  return qs.parse(str, {\n    allowPrototypes: true\n  });\n}\n\n/**\n * Return new empty object.\n *\n * @return {Object}\n * @api private\n */\n\nfunction newObject() {\n  return {};\n}\n\n// Path: lib/response.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Buffer = require('safe-buffer').Buffer\nvar contentDisposition = require('content-disposition');\nvar createError = require('http-errors')\nvar deprecate = require('depd')('express');\nvar encodeUrl = require('encodeurl');\nvar escapeHtml = require('escape-html');\nvar http = require('http');\nvar isAbsolute = require('./utils').isAbsolute;\nvar onFinished = require('on-finished');\nvar path = require('path');\nvar statuses = require('statuses')\nvar merge = require('utils-merge');\nvar sign = require('cookie-signature').sign;\nvar normalizeType = require('./utils').normalizeType;\nvar normalizeTypes = require('./utils').normalizeTypes;\nvar setCharset = require('./utils').setCharset;\nvar cookie = require('cookie');\nvar send = require('send');\nvar extname = path.extname;\nvar mime = send.mime;\nvar resolve = path.resolve;\nvar vary = require('vary');\n\n/**\n * Response prototype.\n * @public\n */\n\nvar res = Object.create(http.ServerResponse.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = res\n\n/**\n * Module variables.\n * @private\n */\n\nvar charsetRegExp = /;\\s*charset\\s*=/;\nvar schemaAndHostRegExp = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?\\/\\/[^\\\\\\/\\?]+/;\n\n/**\n * Set status `code`.\n *\n * @param {Number} code\n * @return {ServerResponse}\n * @public\n */\n\nres.status = function status(code) {\n  if ((typeof code === 'string' || Math.floor(code) !== code) && code > 99 && code < 1000) {\n    deprecate('res.status(' + JSON.stringify(code) + '): use res.status(' + Math.floor(code) + ') instead')\n  }\n  this.statusCode = code;\n  return this;\n};\n\n/**\n * Set Link header field with the given `links`.\n *\n * Examples:\n *\n *    res.links({\n *      next: 'http://api.example.com/users?page=2',\n *      last: 'http://api.example.com/users?page=5'\n *    });\n *\n * @param {Object} links\n * @return {ServerResponse}\n * @public\n */\n\nres.links = function(links){\n  var link = this.get('Link') || '';\n  if (link) link += ', ';\n  return this.set('Link', link + Object.keys(links).map(function(rel){\n    return '<' + links[rel] + '>; rel=\"' + rel + '\"';\n  }).join(', '));\n};\n\n/**\n * Send a response.\n *\n * Examples:\n *\n *     res.send(Buffer.from('wahoo'));\n *     res.send({ some: 'json' });\n *     res.send('<p>some html</p>');\n *\n * @param {string|number|boolean|object|Buffer} body\n * @public\n */\n\nres.send = function send(body) {\n  var chunk = body;\n  var encoding;\n  var req = this.req;\n  var type;\n\n  // settings\n  var app = this.app;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.send(body, status) backwards compat\n    if (typeof arguments[0] !== 'number' && typeof arguments[1] === 'number') {\n      deprecate('res.send(body, status): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.send(status, body): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[0];\n      chunk = arguments[1];\n    }\n  }\n\n  // disambiguate res.send(status) and res.send(status, num)\n  if (typeof chunk === 'number' && arguments.length === 1) {\n    // res.send(status) will set status message as text string\n    if (!this.get('Content-Type')) {\n      this.type('txt');\n    }\n\n    deprecate('res.send(status): Use res.sendStatus(status) instead');\n    this.statusCode = chunk;\n    chunk = statuses.message[chunk]\n  }\n\n  switch (typeof chunk) {\n    // string defaulting to html\n    case 'string':\n      if (!this.get('Content-Type')) {\n        this.type('html');\n      }\n      break;\n    case 'boolean':\n    case 'number':\n    case 'object':\n      if (chunk === null) {\n        chunk = '';\n      } else if (Buffer.isBuffer(chunk)) {\n        if (!this.get('Content-Type')) {\n          this.type('bin');\n        }\n      } else {\n        return this.json(chunk);\n      }\n      break;\n  }\n\n  // write strings in utf-8\n  if (typeof chunk === 'string') {\n    encoding = 'utf8';\n    type = this.get('Content-Type');\n\n    // reflect this in content-type\n    if (typeof type === 'string') {\n      this.set('Content-Type', setCharset(type, 'utf-8'));\n    }\n  }\n\n  // determine if ETag should be generated\n  var etagFn = app.get('etag fn')\n  var generateETag = !this.get('ETag') && typeof etagFn === 'function'\n\n  // populate Content-Length\n  var len\n  if (chunk !== undefined) {\n    if (Buffer.isBuffer(chunk)) {\n      // get length of Buffer\n      len = chunk.length\n    } else if (!generateETag && chunk.length < 1000) {\n      // just calculate length when no ETag + small chunk\n      len = Buffer.byteLength(chunk, encoding)\n    } else {\n      // convert chunk to Buffer and calculate\n      chunk = Buffer.from(chunk, encoding)\n      encoding = undefined;\n      len = chunk.length\n    }\n\n    this.set('Content-Length', len);\n  }\n\n  // populate ETag\n  var etag;\n  if (generateETag && len !== undefined) {\n    if ((etag = etagFn(chunk, encoding))) {\n      this.set('ETag', etag);\n    }\n  }\n\n  // freshness\n  if (req.fresh) this.statusCode = 304;\n\n  // strip irrelevant headers\n  if (204 === this.statusCode || 304 === this.statusCode) {\n    this.removeHeader('Content-Type');\n    this.removeHeader('Content-Length');\n    this.removeHeader('Transfer-Encoding');\n    chunk = '';\n  }\n\n  // alter headers for 205\n  if (this.statusCode === 205) {\n    this.set('Content-Length', '0')\n    this.removeHeader('Transfer-Encoding')\n    chunk = ''\n  }\n\n  if (req.method === 'HEAD') {\n    // skip body for HEAD\n    this.end();\n  } else {\n    // respond\n    this.end(chunk, encoding);\n  }\n\n  return this;\n};\n\n/**\n * Send JSON response.\n *\n * Examples:\n *\n *     res.json(null);\n *     res.json({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.json = function json(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.json(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.json(obj, status): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.json(status, obj): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('Content-Type', 'application/json');\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send JSON response with JSONP callback support.\n *\n * Examples:\n *\n *     res.jsonp(null);\n *     res.jsonp({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.jsonp = function jsonp(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.jsonp(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.jsonp(obj, status): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.jsonp(status, obj): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n  var callback = this.req.query[app.get('jsonp callback name')];\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'application/json');\n  }\n\n  // fixup callback\n  if (Array.isArray(callback)) {\n    callback = callback[0];\n  }\n\n  // jsonp\n  if (typeof callback === 'string' && callback.length !== 0) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'text/javascript');\n\n    // restrict callback charset\n    callback = callback.replace(/[^\\[\\]\\w$.]/g, '');\n\n    if (body === undefined) {\n      // empty argument\n      body = ''\n    } else if (typeof body === 'string') {\n      // replace chars not allowed in JavaScript that are in JSON\n      body = body\n        .replace(/\\u2028/g, '\\\\u2028')\n        .replace(/\\u2029/g, '\\\\u2029')\n    }\n\n    // the /**/ is a specific security mitigation for \"Rosetta Flash JSONP abuse\"\n    // the typeof check is just to reduce client error noise\n    body = '/**/ typeof ' + callback + ' === \\'function\\' && ' + callback + '(' + body + ');';\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send given HTTP status code.\n *\n * Sets the response status to `statusCode` and the body of the\n * response to the standard description from node's http.STATUS_CODES\n * or the statusCode number if no description.\n *\n * Examples:\n *\n *     res.sendStatus(200);\n *\n * @param {number} statusCode\n * @public\n */\n\nres.sendStatus = function sendStatus(statusCode) {\n  var body = statuses.message[statusCode] || String(statusCode)\n\n  this.statusCode = statusCode;\n  this.type('txt');\n\n  return this.send(body);\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendFile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendFile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendFile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendFile = function sendFile(path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  if (!path) {\n    throw new TypeError('path argument is required to res.sendFile');\n  }\n\n  if (typeof path !== 'string') {\n    throw new TypeError('path must be a string to res.sendFile')\n  }\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  if (!opts.root && !isAbsolute(path)) {\n    throw new TypeError('path must be absolute or specify root to res.sendFile');\n  }\n\n  // create file stream\n  var pathname = encodeURI(path);\n  var file = send(req, pathname, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendfile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendfile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendfile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendfile = function (path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // create file stream\n  var file = send(req, path, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\n...\n", "needle_token_start": 879, "needle_token_end": 1020, "code_context_ntokens": 6382, "output": ["```javascript\n/**\n * Parse accept params `str` returning an\n * object with `.value`, `.quality` and `.params`.\n *\n * @param {String} str\n * @return {Object}\n * @api private\n */\n\nfunction acceptParams (str) {\n  var parts = str.split(/ *; */);\n  var ret = { value: parts[0], quality: 1, params: {} }\n\n  for (var i = 1; i < parts.length; ++i) {\n    var pms = parts[i].split(/ *= */);\n    if ('q' === pms[0]) {\n      ret.quality = parseFloat(pms[1]);\n    } else {\n      ret.params[pms[0]] = pms[1];\n    }\n  }\n\n  return ret;\n}\n```"]}
{"repo": "expressjs/express", "name": "Route", "language": "typescript", "path": "lib/router/route.js", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this function is to initialize and manage a specific path within a web application, handling different HTTP methods associated with that path and matching incoming requests to the path.\n2. **Input**: The primary input is a string representing the path which the function will handle.\n3. **Output**: There is no direct output from the constructor function itself, but it sets up an environment to manage HTTP requests for the specified path, including storing handlers and parameters.\n4. **Procedure**: The procedure involves initializing the path and an empty list for middleware handlers. It sets up debugging for the path and prepares to handle various HTTP methods. When a request is received, it matches the request's path against the initialized path, extracts parameters, and processes the request using appropriate handlers.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/response.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Buffer = require('safe-buffer').Buffer\nvar contentDisposition = require('content-disposition');\nvar createError = require('http-errors')\nvar deprecate = require('depd')('express');\nvar encodeUrl = require('encodeurl');\nvar escapeHtml = require('escape-html');\nvar http = require('http');\nvar isAbsolute = require('./utils').isAbsolute;\nvar onFinished = require('on-finished');\nvar path = require('path');\nvar statuses = require('statuses')\nvar merge = require('utils-merge');\nvar sign = require('cookie-signature').sign;\nvar normalizeType = require('./utils').normalizeType;\nvar normalizeTypes = require('./utils').normalizeTypes;\nvar setCharset = require('./utils').setCharset;\nvar cookie = require('cookie');\nvar send = require('send');\nvar extname = path.extname;\nvar mime = send.mime;\nvar resolve = path.resolve;\nvar vary = require('vary');\n\n/**\n * Response prototype.\n * @public\n */\n\nvar res = Object.create(http.ServerResponse.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = res\n\n/**\n * Module variables.\n * @private\n */\n\nvar charsetRegExp = /;\\s*charset\\s*=/;\nvar schemaAndHostRegExp = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?\\/\\/[^\\\\\\/\\?]+/;\n\n/**\n * Set status `code`.\n *\n * @param {Number} code\n * @return {ServerResponse}\n * @public\n */\n\nres.status = function status(code) {\n  if ((typeof code === 'string' || Math.floor(code) !== code) && code > 99 && code < 1000) {\n    deprecate('res.status(' + JSON.stringify(code) + '): use res.status(' + Math.floor(code) + ') instead')\n  }\n  this.statusCode = code;\n  return this;\n};\n\n/**\n * Set Link header field with the given `links`.\n *\n * Examples:\n *\n *    res.links({\n *      next: 'http://api.example.com/users?page=2',\n *      last: 'http://api.example.com/users?page=5'\n *    });\n *\n * @param {Object} links\n * @return {ServerResponse}\n * @public\n */\n\nres.links = function(links){\n  var link = this.get('Link') || '';\n  if (link) link += ', ';\n  return this.set('Link', link + Object.keys(links).map(function(rel){\n    return '<' + links[rel] + '>; rel=\"' + rel + '\"';\n  }).join(', '));\n};\n\n/**\n * Send a response.\n *\n * Examples:\n *\n *     res.send(Buffer.from('wahoo'));\n *     res.send({ some: 'json' });\n *     res.send('<p>some html</p>');\n *\n * @param {string|number|boolean|object|Buffer} body\n * @public\n */\n\nres.send = function send(body) {\n  var chunk = body;\n  var encoding;\n  var req = this.req;\n  var type;\n\n  // settings\n  var app = this.app;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.send(body, status) backwards compat\n    if (typeof arguments[0] !== 'number' && typeof arguments[1] === 'number') {\n      deprecate('res.send(body, status): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.send(status, body): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[0];\n      chunk = arguments[1];\n    }\n  }\n\n  // disambiguate res.send(status) and res.send(status, num)\n  if (typeof chunk === 'number' && arguments.length === 1) {\n    // res.send(status) will set status message as text string\n    if (!this.get('Content-Type')) {\n      this.type('txt');\n    }\n\n    deprecate('res.send(status): Use res.sendStatus(status) instead');\n    this.statusCode = chunk;\n    chunk = statuses.message[chunk]\n  }\n\n  switch (typeof chunk) {\n    // string defaulting to html\n    case 'string':\n      if (!this.get('Content-Type')) {\n        this.type('html');\n      }\n      break;\n    case 'boolean':\n    case 'number':\n    case 'object':\n      if (chunk === null) {\n        chunk = '';\n      } else if (Buffer.isBuffer(chunk)) {\n        if (!this.get('Content-Type')) {\n          this.type('bin');\n        }\n      } else {\n        return this.json(chunk);\n      }\n      break;\n  }\n\n  // write strings in utf-8\n  if (typeof chunk === 'string') {\n    encoding = 'utf8';\n    type = this.get('Content-Type');\n\n    // reflect this in content-type\n    if (typeof type === 'string') {\n      this.set('Content-Type', setCharset(type, 'utf-8'));\n    }\n  }\n\n  // determine if ETag should be generated\n  var etagFn = app.get('etag fn')\n  var generateETag = !this.get('ETag') && typeof etagFn === 'function'\n\n  // populate Content-Length\n  var len\n  if (chunk !== undefined) {\n    if (Buffer.isBuffer(chunk)) {\n      // get length of Buffer\n      len = chunk.length\n    } else if (!generateETag && chunk.length < 1000) {\n      // just calculate length when no ETag + small chunk\n      len = Buffer.byteLength(chunk, encoding)\n    } else {\n      // convert chunk to Buffer and calculate\n      chunk = Buffer.from(chunk, encoding)\n      encoding = undefined;\n      len = chunk.length\n    }\n\n    this.set('Content-Length', len);\n  }\n\n  // populate ETag\n  var etag;\n  if (generateETag && len !== undefined) {\n    if ((etag = etagFn(chunk, encoding))) {\n      this.set('ETag', etag);\n    }\n  }\n\n  // freshness\n  if (req.fresh) this.statusCode = 304;\n\n  // strip irrelevant headers\n  if (204 === this.statusCode || 304 === this.statusCode) {\n    this.removeHeader('Content-Type');\n    this.removeHeader('Content-Length');\n    this.removeHeader('Transfer-Encoding');\n    chunk = '';\n  }\n\n  // alter headers for 205\n  if (this.statusCode === 205) {\n    this.set('Content-Length', '0')\n    this.removeHeader('Transfer-Encoding')\n    chunk = ''\n  }\n\n  if (req.method === 'HEAD') {\n    // skip body for HEAD\n    this.end();\n  } else {\n    // respond\n    this.end(chunk, encoding);\n  }\n\n  return this;\n};\n\n/**\n * Send JSON response.\n *\n * Examples:\n *\n *     res.json(null);\n *     res.json({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.json = function json(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.json(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.json(obj, status): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.json(status, obj): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('Content-Type', 'application/json');\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send JSON response with JSONP callback support.\n *\n * Examples:\n *\n *     res.jsonp(null);\n *     res.jsonp({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.jsonp = function jsonp(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.jsonp(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.jsonp(obj, status): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.jsonp(status, obj): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n  var callback = this.req.query[app.get('jsonp callback name')];\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'application/json');\n  }\n\n  // fixup callback\n  if (Array.isArray(callback)) {\n    callback = callback[0];\n  }\n\n  // jsonp\n  if (typeof callback === 'string' && callback.length !== 0) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'text/javascript');\n\n    // restrict callback charset\n    callback = callback.replace(/[^\\[\\]\\w$.]/g, '');\n\n    if (body === undefined) {\n      // empty argument\n      body = ''\n    } else if (typeof body === 'string') {\n      // replace chars not allowed in JavaScript that are in JSON\n      body = body\n        .replace(/\\u2028/g, '\\\\u2028')\n        .replace(/\\u2029/g, '\\\\u2029')\n    }\n\n    // the /**/ is a specific security mitigation for \"Rosetta Flash JSONP abuse\"\n    // the typeof check is just to reduce client error noise\n    body = '/**/ typeof ' + callback + ' === \\'function\\' && ' + callback + '(' + body + ');';\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send given HTTP status code.\n *\n * Sets the response status to `statusCode` and the body of the\n * response to the standard description from node's http.STATUS_CODES\n * or the statusCode number if no description.\n *\n * Examples:\n *\n *     res.sendStatus(200);\n *\n * @param {number} statusCode\n * @public\n */\n\nres.sendStatus = function sendStatus(statusCode) {\n  var body = statuses.message[statusCode] || String(statusCode)\n\n  this.statusCode = statusCode;\n  this.type('txt');\n\n  return this.send(body);\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendFile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendFile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendFile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendFile = function sendFile(path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  if (!path) {\n    throw new TypeError('path argument is required to res.sendFile');\n  }\n\n  if (typeof path !== 'string') {\n    throw new TypeError('path must be a string to res.sendFile')\n  }\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  if (!opts.root && !isAbsolute(path)) {\n    throw new TypeError('path must be absolute or specify root to res.sendFile');\n  }\n\n  // create file stream\n  var pathname = encodeURI(path);\n  var file = send(req, pathname, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendfile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendfile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendfile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendfile = function (path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // create file stream\n  var file = send(req, path, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\nres.sendfile = deprecate.function(res.sendfile,\n  'res.sendfile: Use res.sendFile instead');\n\n/**\n * Transfer the file at the given `path` as an attachment.\n *\n * Optionally providing an alternate attachment `filename`,\n * and optional callback `callback(err)`. The callback is invoked\n * when the data transfer is complete, or when an error has\n * occurred. Be sure to check `res.headersSent` if you plan to respond.\n *\n * Optionally providing an `options` object to use with `res.sendFile()`.\n * This function will set the `Content-Disposition` header, overriding\n * any `Content-Disposition` header passed as header options in order\n * to set the attachment and filename.\n *\n * This method uses `res.sendFile()`.\n *\n * @public\n */\n\nres.download = function download (path, filename, options, callback) {\n  var done = callback;\n  var name = filename;\n  var opts = options || null\n\n  // support function as second or third arg\n  if (typeof filename === 'function') {\n    done = filename;\n    name = null;\n    opts = null\n  } else if (typeof options === 'function') {\n    done = options\n    opts = null\n  }\n\n  // support optional filename, where options may be in it's place\n  if (typeof filename === 'object' &&\n    (typeof options === 'function' || options === undefined)) {\n    name = null\n    opts = filename\n  }\n\n  // set Content-Disposition when file is sent\n  var headers = {\n    'Content-Disposition': contentDisposition(name || path)\n  };\n\n  // merge user-provided headers\n  if (opts && opts.headers) {\n    var keys = Object.keys(opts.headers)\n    for (var i = 0; i < keys.length; i++) {\n      var key = keys[i]\n      if (key.toLowerCase() !== 'content-disposition') {\n        headers[key] = opts.headers[key]\n      }\n    }\n  }\n\n  // merge user-provided options\n  opts = Object.create(opts)\n  opts.headers = headers\n\n  // Resolve the full path for sendFile\n  var fullPath = !opts.root\n    ? resolve(path)\n    : path\n\n  // send file\n  return this.sendFile(fullPath, opts, done)\n};\n\n/**\n * Set _Content-Type_ response header with `type` through `mime.lookup()`\n * when it does not contain \"/\", or set the Content-Type to `type` otherwise.\n *\n * Examples:\n *\n *     res.type('.html');\n *     res.type('html');\n *     res.type('json');\n *     res.type('application/json');\n *     res.type('png');\n *\n * @param {String} type\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.contentType =\nres.type = function contentType(type) {\n  var ct = type.indexOf('/') === -1\n    ? mime.lookup(type)\n    : type;\n\n  return this.set('Content-Type', ct);\n};\n\n/**\n * Respond to the Acceptable formats using an `obj`\n * of mime-type callbacks.\n *\n * This method uses `req.accepted`, an array of\n * acceptable types ordered by their quality values.\n * When \"Accept\" is not present the _first_ callback\n * is invoked, otherwise the first match is used. When\n * no match is performed the server responds with\n * 406 \"Not Acceptable\".\n *\n * Content-Type is set for you, however if you choose\n * you may alter this within the callback using `res.type()`\n * or `res.set('Content-Type', ...)`.\n *\n *    res.format({\n *      'text/plain': function(){\n *        res.send('hey');\n *      },\n *\n *      'text/html': function(){\n *        res.send('<p>hey</p>');\n *      },\n *\n *      'application/json': function () {\n *        res.send({ message: 'hey' });\n *      }\n *    });\n *\n * In addition to canonicalized MIME types you may\n * also use extnames mapped to these types:\n *\n *    res.format({\n *      text: function(){\n *        res.send('hey');\n *      },\n *\n *      html: function(){\n *        res.send('<p>hey</p>');\n *      },\n *\n *      json: function(){\n *        res.send({ message: 'hey' });\n *      }\n *    });\n *\n * By default Express passes an `Error`\n * with a `.status` of 406 to `next(err)`\n * if a match is not made. If you provide\n * a `.default` callback it will be invoked\n * instead.\n *\n * @param {Object} obj\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.format = function(obj){\n  var req = this.req;\n  var next = req.next;\n\n  var keys = Object.keys(obj)\n    .filter(function (v) { return v !== 'default' })\n\n  var key = keys.length > 0\n    ? req.accepts(keys)\n    : false;\n\n  this.vary(\"Accept\");\n\n  if (key) {\n    this.set('Content-Type', normalizeType(key).value);\n    obj[key](req, this, next);\n  } else if (obj.default) {\n    obj.default(req, this, next)\n  } else {\n    next(createError(406, {\n      types: normalizeTypes(keys).map(function (o) { return o.value })\n    }))\n  }\n\n  return this;\n};\n\n/**\n * Set _Content-Disposition_ header to _attachment_ with optional `filename`.\n *\n * @param {String} filename\n * @return {ServerResponse}\n * @public\n */\n\nres.attachment = function attachment(filename) {\n  if (filename) {\n...\n// Path: lib/request.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar accepts = require('accepts');\nvar deprecate = require('depd')('express');\nvar isIP = require('net').isIP;\nvar typeis = require('type-is');\nvar http = require('http');\nvar fresh = require('fresh');\nvar parseRange = require('range-parser');\nvar parse = require('parseurl');\nvar proxyaddr = require('proxy-addr');\n\n/**\n * Request prototype.\n * @public\n */\n\nvar req = Object.create(http.IncomingMessage.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = req\n\n/**\n * Return request header.\n *\n * The `Referrer` header field is special-cased,\n * both `Referrer` and `Referer` are interchangeable.\n *\n * Examples:\n *\n *     req.get('Content-Type');\n *     // => \"text/plain\"\n *\n *     req.get('content-type');\n *     // => \"text/plain\"\n *\n *     req.get('Something');\n *     // => undefined\n *\n * Aliased as `req.header()`.\n *\n * @param {String} name\n * @return {String}\n * @public\n */\n\nreq.get =\nreq.header = function header(name) {\n  if (!name) {\n    throw new TypeError('name argument is required to req.get');\n  }\n\n  if (typeof name !== 'string') {\n    throw new TypeError('name must be a string to req.get');\n  }\n\n  var lc = name.toLowerCase();\n\n  switch (lc) {\n    case 'referer':\n    case 'referrer':\n      return this.headers.referrer\n        || this.headers.referer;\n    default:\n      return this.headers[lc];\n  }\n};\n\n/**\n * To do: update docs.\n *\n * Check if the given `type(s)` is acceptable, returning\n * the best match when true, otherwise `undefined`, in which\n * case you should respond with 406 \"Not Acceptable\".\n *\n * The `type` value may be a single MIME type string\n * such as \"application/json\", an extension name\n * such as \"json\", a comma-delimited list such as \"json, html, text/plain\",\n * an argument list such as `\"json\", \"html\", \"text/plain\"`,\n * or an array `[\"json\", \"html\", \"text/plain\"]`. When a list\n * or array is given, the _best_ match, if any is returned.\n *\n * Examples:\n *\n *     // Accept: text/html\n *     req.accepts('html');\n *     // => \"html\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('html');\n *     // => \"html\"\n *     req.accepts('text/html');\n *     // => \"text/html\"\n *     req.accepts('json, text');\n *     // => \"json\"\n *     req.accepts('application/json');\n *     // => \"application/json\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('image/png');\n *     req.accepts('png');\n *     // => undefined\n *\n *     // Accept: text/*;q=.5, application/json\n *     req.accepts(['html', 'json']);\n *     req.accepts('html', 'json');\n *     req.accepts('html, json');\n *     // => \"json\"\n *\n * @param {String|Array} type(s)\n * @return {String|Array|Boolean}\n * @public\n */\n\nreq.accepts = function(){\n  var accept = accepts(this);\n  return accept.types.apply(accept, arguments);\n};\n\n/**\n * Check if the given `encoding`s are accepted.\n *\n * @param {String} ...encoding\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsEncodings = function(){\n  var accept = accepts(this);\n  return accept.encodings.apply(accept, arguments);\n};\n\nreq.acceptsEncoding = deprecate.function(req.acceptsEncodings,\n  'req.acceptsEncoding: Use acceptsEncodings instead');\n\n/**\n * Check if the given `charset`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...charset\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsCharsets = function(){\n  var accept = accepts(this);\n  return accept.charsets.apply(accept, arguments);\n};\n\nreq.acceptsCharset = deprecate.function(req.acceptsCharsets,\n  'req.acceptsCharset: Use acceptsCharsets instead');\n\n/**\n * Check if the given `lang`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...lang\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsLanguages = function(){\n  var accept = accepts(this);\n  return accept.languages.apply(accept, arguments);\n};\n\nreq.acceptsLanguage = deprecate.function(req.acceptsLanguages,\n  'req.acceptsLanguage: Use acceptsLanguages instead');\n\n/**\n * Parse Range header field, capping to the given `size`.\n *\n * Unspecified ranges such as \"0-\" require knowledge of your resource length. In\n * the case of a byte range this is of course the total number of bytes. If the\n * Range header field is not given `undefined` is returned, `-1` when unsatisfiable,\n * and `-2` when syntactically invalid.\n *\n * When ranges are returned, the array has a \"type\" property which is the type of\n * range that is required (most commonly, \"bytes\"). Each array element is an object\n * with a \"start\" and \"end\" property for the portion of the range.\n *\n * The \"combine\" option can be set to `true` and overlapping & adjacent ranges\n * will be combined into a single range.\n *\n * NOTE: remember that ranges are inclusive, so for example \"Range: users=0-3\"\n * should respond with 4 users when available, not 3.\n *\n * @param {number} size\n * @param {object} [options]\n * @param {boolean} [options.combine=false]\n * @return {number|array}\n * @public\n */\n\nreq.range = function range(size, options) {\n  var range = this.get('Range');\n  if (!range) return;\n  return parseRange(size, range, options);\n};\n\n/**\n * Return the value of param `name` when present or `defaultValue`.\n *\n *  - Checks route placeholders, ex: _/user/:id_\n *  - Checks body params, ex: id=12, {\"id\":12}\n *  - Checks query string params, ex: ?id=12\n *\n * To utilize request bodies, `req.body`\n * should be an object. This can be done by using\n * the `bodyParser()` middleware.\n *\n * @param {String} name\n * @param {Mixed} [defaultValue]\n * @return {String}\n * @public\n */\n\nreq.param = function param(name, defaultValue) {\n  var params = this.params || {};\n  var body = this.body || {};\n  var query = this.query || {};\n\n  var args = arguments.length === 1\n    ? 'name'\n    : 'name, default';\n  deprecate('req.param(' + args + '): Use req.params, req.body, or req.query instead');\n\n  if (null != params[name] && params.hasOwnProperty(name)) return params[name];\n  if (null != body[name]) return body[name];\n  if (null != query[name]) return query[name];\n\n  return defaultValue;\n};\n\n/**\n * Check if the incoming request contains the \"Content-Type\"\n * header field, and it contains the given mime `type`.\n *\n * Examples:\n *\n *      // With Content-Type: text/html; charset=utf-8\n *      req.is('html');\n *      req.is('text/html');\n *      req.is('text/*');\n *      // => true\n *\n *      // When Content-Type is application/json\n *      req.is('json');\n *      req.is('application/json');\n *      req.is('application/*');\n *      // => true\n *\n *      req.is('html');\n *      // => false\n *\n * @param {String|Array} types...\n * @return {String|false|null}\n * @public\n */\n\nreq.is = function is(types) {\n  var arr = types;\n\n  // support flattened arguments\n  if (!Array.isArray(types)) {\n    arr = new Array(arguments.length);\n    for (var i = 0; i < arr.length; i++) {\n      arr[i] = arguments[i];\n    }\n  }\n\n  return typeis(this, arr);\n};\n\n/**\n * Return the protocol string \"http\" or \"https\"\n * when requested with TLS. When the \"trust proxy\"\n * setting trusts the socket address, the\n * \"X-Forwarded-Proto\" header field will be trusted\n * and used if present.\n *\n * If you're running behind a reverse proxy that\n * supplies https for you this may be enabled.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'protocol', function protocol(){\n  var proto = this.connection.encrypted\n    ? 'https'\n    : 'http';\n  var trust = this.app.get('trust proxy fn');\n\n  if (!trust(this.connection.remoteAddress, 0)) {\n    return proto;\n  }\n\n  // Note: X-Forwarded-Proto is normally only ever a\n  //       single value, but this is to be safe.\n  var header = this.get('X-Forwarded-Proto') || proto\n  var index = header.indexOf(',')\n\n  return index !== -1\n    ? header.substring(0, index).trim()\n    : header.trim()\n});\n\n/**\n * Short-hand for:\n *\n *    req.protocol === 'https'\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'secure', function secure(){\n  return this.protocol === 'https';\n});\n\n/**\n * Return the remote address from the trusted proxy.\n *\n * The is the remote address on the socket unless\n * \"trust proxy\" is set.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'ip', function ip(){\n  var trust = this.app.get('trust proxy fn');\n  return proxyaddr(this, trust);\n});\n\n/**\n * When \"trust proxy\" is set, trusted proxy addresses + client.\n *\n * For example if the value were \"client, proxy1, proxy2\"\n * you would receive the array `[\"client\", \"proxy1\", \"proxy2\"]`\n * where \"proxy2\" is the furthest down-stream and \"proxy1\" and\n * \"proxy2\" were trusted.\n *\n * @return {Array}\n * @public\n */\n\ndefineGetter(req, 'ips', function ips() {\n  var trust = this.app.get('trust proxy fn');\n  var addrs = proxyaddr.all(this, trust);\n\n  // reverse the order (to farthest -> closest)\n  // and remove socket address\n  addrs.reverse().pop()\n\n  return addrs\n});\n\n/**\n * Return subdomains as an array.\n *\n * Subdomains are the dot-separated parts of the host before the main domain of\n * the app. By default, the domain of the app is assumed to be the last two\n * parts of the host. This can be changed by setting \"subdomain offset\".\n *\n * For example, if the domain is \"tobi.ferrets.example.com\":\n * If \"subdomain offset\" is not set, req.subdomains is `[\"ferrets\", \"tobi\"]`.\n * If \"subdomain offset\" is 3, req.subdomains is `[\"tobi\"]`.\n *\n * @return {Array}\n * @public\n */\n\ndefineGetter(req, 'subdomains', function subdomains() {\n  var hostname = this.hostname;\n\n  if (!hostname) return [];\n\n  var offset = this.app.get('subdomain offset');\n  var subdomains = !isIP(hostname)\n    ? hostname.split('.').reverse()\n    : [hostname];\n\n  return subdomains.slice(offset);\n});\n\n/**\n * Short-hand for `url.parse(req.url).pathname`.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'path', function path() {\n  return parse(this).pathname;\n});\n\n/**\n * Parse the \"Host\" header field to a hostname.\n *\n * When the \"trust proxy\" setting trusts the socket\n * address, the \"X-Forwarded-Host\" header field will\n * be trusted.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'hostname', function hostname(){\n  var trust = this.app.get('trust proxy fn');\n  var host = this.get('X-Forwarded-Host');\n\n  if (!host || !trust(this.connection.remoteAddress, 0)) {\n    host = this.get('Host');\n  } else if (host.indexOf(',') !== -1) {\n    // Note: X-Forwarded-Host is normally only ever a\n    //       single value, but this is to be safe.\n    host = host.substring(0, host.indexOf(',')).trimRight()\n  }\n\n  if (!host) return;\n\n  // IPv6 literal support\n  var offset = host[0] === '['\n    ? host.indexOf(']') + 1\n    : 0;\n  var index = host.indexOf(':', offset);\n\n  return index !== -1\n    ? host.substring(0, index)\n    : host;\n});\n\n// TODO: change req.host to return host in next major\n\ndefineGetter(req, 'host', deprecate.function(function host(){\n  return this.hostname;\n}, 'req.host: Use req.hostname instead'));\n\n/**\n * Check if the request is fresh, aka\n * Last-Modified and/or the ETag\n * still match.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'fresh', function(){\n  var method = this.method;\n  var res = this.res\n  var status = res.statusCode\n\n  // GET or HEAD for weak freshness validation only\n  if ('GET' !== method && 'HEAD' !== method) return false;\n\n  // 2xx or 304 as per rfc2616 14.26\n  if ((status >= 200 && status < 300) || 304 === status) {\n    return fresh(this.headers, {\n      'etag': res.get('ETag'),\n      'last-modified': res.get('Last-Modified')\n    })\n  }\n\n  return false;\n});\n\n/**\n * Check if the request is stale, aka\n * \"Last-Modified\" and / or the \"ETag\" for the\n * resource has changed.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'stale', function stale(){\n  return !this.fresh;\n});\n\n/**\n * Check if the request was an _XMLHttpRequest_.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'xhr', function xhr(){\n  var val = this.get('X-Requested-With') || '';\n  return val.toLowerCase() === 'xmlhttprequest';\n});\n\n/**\n * Helper function for creating a getter on an object.\n *\n * @param {Object} obj\n * @param {String} name\n * @param {Function} getter\n * @private\n */\nfunction defineGetter(obj, name, getter) {\n  Object.defineProperty(obj, name, {\n    configurable: true,\n    enumerable: true,\n    get: getter\n  });\n}\n\n// Path: lib/router/layer.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar pathRegexp = require('path-to-regexp');\nvar debug = require('debug')('express:router:layer');\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Layer;\n\nfunction Layer(path, options, fn) {\n  if (!(this instanceof Layer)) {\n    return new Layer(path, options, fn);\n  }\n\n  debug('new %o', path)\n  var opts = options || {};\n\n  this.handle = fn;\n  this.name = fn.name || '<anonymous>';\n  this.params = undefined;\n  this.path = undefined;\n  this.regexp = pathRegexp(path, this.keys = [], opts);\n\n  // set fast path flags\n  this.regexp.fast_star = path === '*'\n  this.regexp.fast_slash = path === '/' && opts.end === false\n}\n\n/**\n * Handle the error for the layer.\n *\n * @param {Error} error\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_error = function handle_error(error, req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length !== 4) {\n    // not a standard error handler\n    return next(error);\n  }\n\n  try {\n    fn(error, req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Handle the request for the layer.\n *\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_request = function handle(req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length > 3) {\n    // not a standard request handler\n    return next();\n  }\n\n  try {\n    fn(req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Check if this route matches `path`, if so\n * populate `.params`.\n *\n * @param {String} path\n * @return {Boolean}\n * @api private\n */\n\nLayer.prototype.match = function match(path) {\n  var match\n\n  if (path != null) {\n    // fast path non-ending match for / (any path matches)\n    if (this.regexp.fast_slash) {\n      this.params = {}\n      this.path = ''\n      return true\n    }\n\n    // fast path for * (everything matched in a param)\n    if (this.regexp.fast_star) {\n      this.params = {'0': decode_param(path)}\n      this.path = path\n      return true\n    }\n\n    // match the path\n    match = this.regexp.exec(path)\n  }\n\n  if (!match) {\n    this.params = undefined;\n    this.path = undefined;\n    return false;\n  }\n\n  // store values\n  this.params = {};\n  this.path = match[0]\n\n  var keys = this.keys;\n  var params = this.params;\n\n  for (var i = 1; i < match.length; i++) {\n    var key = keys[i - 1];\n    var prop = key.name;\n    var val = decode_param(match[i])\n\n    if (val !== undefined || !(hasOwnProperty.call(params, prop))) {\n      params[prop] = val;\n    }\n  }\n\n  return true;\n};\n\n/**\n * Decode param value.\n *\n * @param {string} val\n * @return {string}\n * @private\n */\n\nfunction decode_param(val) {\n  if (typeof val !== 'string' || val.length === 0) {\n    return val;\n  }\n\n  try {\n    return decodeURIComponent(val);\n  } catch (err) {\n    if (err instanceof URIError) {\n      err.message = 'Failed to decode param \\'' + val + '\\'';\n      err.status = err.statusCode = 400;\n    }\n\n    throw err;\n  }\n}\n\n// Path: lib/router/route.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:router:route');\nvar flatten = require('array-flatten');\nvar Layer = require('./layer');\nvar methods = require('methods');\n\n/**\n * Module variables.\n * @private\n */\n\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Route;\n\n/**\n * Initialize `Route` with the given `path`,\n *\n * @param {String} path\n * @public\n */\n\n\nfunction Route(path) {\n  this.path = path;\n  this.stack = [];\n\n  debug('new %o', path)\n\n  // route handlers for various http methods\n  this.methods = {};\n}\n\n/**\n * Determine if the route handles a given method.\n * @private\n */\n\nRoute.prototype._handles_method = function _handles_method(method) {\n  if (this.methods._all) {\n    return true;\n  }\n\n  // normalize name\n  var name = typeof method === 'string'\n    ? method.toLowerCase()\n    : method\n\n  if (name === 'head' && !this.methods['head']) {\n    name = 'get';\n  }\n\n  return Boolean(this.methods[name]);\n};\n\n/**\n * @return {Array} supported HTTP methods\n * @private\n */\n\nRoute.prototype._options = function _options() {\n  var methods = Object.keys(this.methods);\n\n  // append automatic head\n  if (this.methods.get && !this.methods.head) {\n    methods.push('head');\n  }\n\n  for (var i = 0; i < methods.length; i++) {\n    // make upper case\n    methods[i] = methods[i].toUpperCase();\n  }\n\n  return methods;\n};\n\n/**\n * dispatch req, res into this route\n * @private\n */\n\nRoute.prototype.dispatch = function dispatch(req, res, done) {\n  var idx = 0;\n  var stack = this.stack;\n  var sync = 0\n\n  if (stack.length === 0) {\n    return done();\n  }\n  var method = typeof req.method === 'string'\n    ? req.method.toLowerCase()\n    : req.method\n\n  if (method === 'head' && !this.methods['head']) {\n    method = 'get';\n  }\n\n  req.route = this;\n\n  next();\n\n  function next(err) {\n    // signal to exit route\n    if (err && err === 'route') {\n      return done();\n    }\n\n    // signal to exit router\n    if (err && err === 'router') {\n      return done(err)\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    var layer = stack[idx++]\n\n    // end of layers\n    if (!layer) {\n      return done(err)\n    }\n\n    if (layer.method && layer.method !== method) {\n      next(err)\n    } else if (err) {\n      layer.handle_error(err, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n\n    sync = 0\n  }\n};\n\n/**\n * Add a handler for all HTTP verbs to this route.\n *\n * Behaves just like middleware and can respond or call `next`\n * to continue processing.\n *\n * You can use multiple `.all` call to add multiple handlers.\n *\n *   function check_something(req, res, next){\n *     next();\n *   };\n *\n *   function validate_user(req, res, next){\n *     next();\n *   };\n *\n *   route\n *   .all(validate_user)\n *   .all(check_something)\n *   .get(function(req, res, next){\n *     res.send('hello world');\n *   });\n *\n * @param {function} handler\n * @return {Route} for chaining\n * @api public\n */\n\nRoute.prototype.all = function all() {\n  var handles = flatten(slice.call(arguments));\n\n  for (var i = 0; i < handles.length; i++) {\n    var handle = handles[i];\n\n    if (typeof handle !== 'function') {\n      var type = toString.call(handle);\n      var msg = 'Route.all() requires a callback function but got a ' + type\n      throw new TypeError(msg);\n    }\n\n    var layer = Layer('/', {}, handle);\n    layer.method = undefined;\n\n    this.methods._all = true;\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\nmethods.forEach(function(method){\n  Route.prototype[method] = function(){\n    var handles = flatten(slice.call(arguments));\n\n    for (var i = 0; i < handles.length; i++) {\n      var handle = handles[i];\n\n      if (typeof handle !== 'function') {\n        var type = toString.call(handle);\n        var msg = 'Route.' + method + '() requires a callback function but got a ' + type\n        throw new Error(msg);\n      }\n\n      debug('%s %o', method, this.path)\n\n      var layer = Layer('/', {}, handle);\n      layer.method = method;\n\n      this.methods[method] = true;\n      this.stack.push(layer);\n    }\n\n    return this;\n  };\n});\n\n// Path: lib/router/index.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Route = require('./route');\nvar Layer = require('./layer');\nvar methods = require('methods');\nvar mixin = require('utils-merge');\nvar debug = require('debug')('express:router');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar parseUrl = require('parseurl');\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar objectRegExp = /^\\[object (\\S+)\\]$/;\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n\n/**\n * Map the given param placeholder `name`(s) to the given callback.\n *\n * Parameter mapping is used to provide pre-conditions to routes\n * which use normalized placeholders. For example a _:user_id_ parameter\n * could automatically load a user's information from the database without\n * any additional code,\n *\n * The callback uses the same signature as middleware, the only difference\n * being that the value of the placeholder is passed, in this case the _id_\n * of the user. Once the `next()` function is invoked, just like middleware\n * it will continue on to execute the route, or subsequent parameter functions.\n *\n * Just like in middleware, you must either respond to the request or call next\n * to avoid stalling the request.\n *\n *  app.param('user_id', function(req, res, next, id){\n *    User.find(id, function(err, user){\n *      if (err) {\n *        return next(err);\n *      } else if (!user) {\n *        return next(new Error('failed to load user'));\n *      }\n *      req.user = user;\n *      next();\n *    });\n *  });\n *\n * @param {String} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\nproto.param = function param(name, fn) {\n  // param logic\n  if (typeof name === 'function') {\n    deprecate('router.param(fn): Refactor to use path params');\n    this._params.push(name);\n    return;\n  }\n\n  // apply param functions\n  var params = this._params;\n  var len = params.length;\n  var ret;\n\n  if (name[0] === ':') {\n    deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.slice(1)) + ', fn) instead')\n    name = name.slice(1)\n  }\n\n  for (var i = 0; i < len; ++i) {\n    if (ret = params[i](name, fn)) {\n      fn = ret;\n    }\n  }\n\n  // ensure we end up with a\n  // middleware function\n  if ('function' !== typeof fn) {\n    throw new Error('invalid param() call for ' + name + ', got ' + fn);\n  }\n\n  (this.params[name] = this.params[name] || []).push(fn);\n  return this;\n};\n\n/**\n * Dispatch a req, res into the router.\n * @private\n */\n\nproto.handle = function handle(req, res, out) {\n  var self = this;\n\n  debug('dispatching %s %s', req.method, req.url);\n\n  var idx = 0;\n  var protohost = getProtohost(req.url) || ''\n  var removed = '';\n  var slashAdded = false;\n  var sync = 0\n  var paramcalled = {};\n\n  // store options for OPTIONS request\n  // only used if OPTIONS request\n  var options = [];\n\n  // middleware and routes\n  var stack = self.stack;\n\n  // manage inter-router variables\n  var parentParams = req.params;\n  var parentUrl = req.baseUrl || '';\n  var done = restore(out, req, 'baseUrl', 'next', 'params');\n\n  // setup next layer\n  req.next = next;\n\n  // for options requests, respond with a default if nothing else responds\n  if (req.method === 'OPTIONS') {\n    done = wrap(done, function(old, err) {\n      if (err || options.length === 0) return old(err);\n      sendOptionsResponse(res, options, old);\n    });\n  }\n\n  // setup basic req values\n  req.baseUrl = parentUrl;\n  req.originalUrl = req.originalUrl || req.url;\n\n  next();\n\n  function next(err) {\n    var layerError = err === 'route'\n      ? null\n      : err;\n\n    // remove added slash\n    if (slashAdded) {\n      req.url = req.url.slice(1)\n      slashAdded = false;\n    }\n\n    // restore altered req.url\n    if (removed.length !== 0) {\n      req.baseUrl = parentUrl;\n      req.url = protohost + removed + req.url.slice(protohost.length)\n      removed = '';\n    }\n\n    // signal to exit router\n    if (layerError === 'router') {\n      setImmediate(done, null)\n      return\n    }\n\n    // no more matching layers\n    if (idx >= stack.length) {\n      setImmediate(done, layerError);\n      return;\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    // get pathname of request\n    var path = getPathname(req);\n\n    if (path == null) {\n      return done(layerError);\n    }\n\n    // find next matching layer\n    var layer;\n    var match;\n    var route;\n\n    while (match !== true && idx < stack.length) {\n      layer = stack[idx++];\n      match = matchLayer(layer, path);\n      route = layer.route;\n\n      if (typeof match !== 'boolean') {\n        // hold on to layerError\n        layerError = layerError || match;\n      }\n\n      if (match !== true) {\n        continue;\n      }\n\n      if (!route) {\n        // process non-route handlers normally\n        continue;\n      }\n\n      if (layerError) {\n        // routes do not match with a pending error\n        match = false;\n        continue;\n      }\n\n      var method = req.method;\n      var has_method = route._handles_method(method);\n\n      // build up automatic options response\n      if (!has_method && method === 'OPTIONS') {\n        appendMethods(options, route._options());\n      }\n\n      // don't even bother matching route\n      if (!has_method && method !== 'HEAD') {\n        match = false;\n      }\n    }\n\n    // no match\n    if (match !== true) {\n      return done(layerError);\n    }\n\n    // store route for dispatch on change\n    if (route) {\n      req.route = route;\n    }\n\n    // Capture one-time layer values\n    req.params = self.mergeParams\n      ? mergeParams(layer.params, parentParams)\n      : layer.params;\n    var layerPath = layer.path;\n\n    // this should be done for the layer\n    self.process_params(layer, paramcalled, req, res, function (err) {\n      if (err) {\n        next(layerError || err)\n      } else if (route) {\n        layer.handle_request(req, res, next)\n      } else {\n        trim_prefix(layer, layerError, layerPath, path)\n      }\n\n      sync = 0\n    });\n  }\n\n  function trim_prefix(layer, layerError, layerPath, path) {\n    if (layerPath.length !== 0) {\n      // Validate path is a prefix match\n      if (layerPath !== path.slice(0, layerPath.length)) {\n        next(layerError)\n        return\n      }\n\n      // Validate path breaks on a path separator\n      var c = path[layerPath.length]\n      if (c && c !== '/' && c !== '.') return next(layerError)\n\n      // Trim off the part of the url that matches the route\n      // middleware (.use stuff) needs to have the path stripped\n      debug('trim prefix (%s) from url %s', layerPath, req.url);\n      removed = layerPath;\n      req.url = protohost + req.url.slice(protohost.length + removed.length)\n\n      // Ensure leading slash\n      if (!protohost && req.url[0] !== '/') {\n        req.url = '/' + req.url;\n        slashAdded = true;\n      }\n\n      // Setup base URL (no trailing slash)\n      req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'\n        ? removed.substring(0, removed.length - 1)\n        : removed);\n    }\n\n    debug('%s %s : %s', layer.name, layerPath, req.originalUrl);\n\n    if (layerError) {\n      layer.handle_error(layerError, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n  }\n};\n\n/**\n * Process any parameters for the layer.\n * @private\n */\n\nproto.process_params = function process_params(layer, called, req, res, done) {\n  var params = this.params;\n\n  // captured parameters from the layer, keys and values\n  var keys = layer.keys;\n\n  // fast track\n  if (!keys || keys.length === 0) {\n    return done();\n  }\n\n  var i = 0;\n  var name;\n...\n", "needle_token_start": 11483, "needle_token_end": 11534, "code_context_ntokens": 15408, "output": ["```javascript\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n```"]}
{"repo": "expressjs/express", "name": "tryStat", "language": "typescript", "path": "lib/view.js", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To safely attempt to retrieve file metadata from the filesystem without throwing an error if the file does not exist.\n2. **Input**: A string representing the file path.\n3. **Output**: Returns the file metadata as an object if the file exists, or `undefined` if the file does not exist or an error occurs.\n4. **Procedure**: The function attempts to access the file's metadata using a synchronous method. If the file is accessible, its metadata is returned. If the file is not found or another error occurs, the function catches the error and returns `undefined`, preventing the error from propagating.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/request.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar accepts = require('accepts');\nvar deprecate = require('depd')('express');\nvar isIP = require('net').isIP;\nvar typeis = require('type-is');\nvar http = require('http');\nvar fresh = require('fresh');\nvar parseRange = require('range-parser');\nvar parse = require('parseurl');\nvar proxyaddr = require('proxy-addr');\n\n/**\n * Request prototype.\n * @public\n */\n\nvar req = Object.create(http.IncomingMessage.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = req\n\n/**\n * Return request header.\n *\n * The `Referrer` header field is special-cased,\n * both `Referrer` and `Referer` are interchangeable.\n *\n * Examples:\n *\n *     req.get('Content-Type');\n *     // => \"text/plain\"\n *\n *     req.get('content-type');\n *     // => \"text/plain\"\n *\n *     req.get('Something');\n *     // => undefined\n *\n * Aliased as `req.header()`.\n *\n * @param {String} name\n * @return {String}\n * @public\n */\n\nreq.get =\nreq.header = function header(name) {\n  if (!name) {\n    throw new TypeError('name argument is required to req.get');\n  }\n\n  if (typeof name !== 'string') {\n    throw new TypeError('name must be a string to req.get');\n  }\n\n  var lc = name.toLowerCase();\n\n  switch (lc) {\n    case 'referer':\n    case 'referrer':\n      return this.headers.referrer\n        || this.headers.referer;\n    default:\n      return this.headers[lc];\n  }\n};\n\n/**\n * To do: update docs.\n *\n * Check if the given `type(s)` is acceptable, returning\n * the best match when true, otherwise `undefined`, in which\n * case you should respond with 406 \"Not Acceptable\".\n *\n * The `type` value may be a single MIME type string\n * such as \"application/json\", an extension name\n * such as \"json\", a comma-delimited list such as \"json, html, text/plain\",\n * an argument list such as `\"json\", \"html\", \"text/plain\"`,\n * or an array `[\"json\", \"html\", \"text/plain\"]`. When a list\n * or array is given, the _best_ match, if any is returned.\n *\n * Examples:\n *\n *     // Accept: text/html\n *     req.accepts('html');\n *     // => \"html\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('html');\n *     // => \"html\"\n *     req.accepts('text/html');\n *     // => \"text/html\"\n *     req.accepts('json, text');\n *     // => \"json\"\n *     req.accepts('application/json');\n *     // => \"application/json\"\n *\n *     // Accept: text/*, application/json\n *     req.accepts('image/png');\n *     req.accepts('png');\n *     // => undefined\n *\n *     // Accept: text/*;q=.5, application/json\n *     req.accepts(['html', 'json']);\n *     req.accepts('html', 'json');\n *     req.accepts('html, json');\n *     // => \"json\"\n *\n * @param {String|Array} type(s)\n * @return {String|Array|Boolean}\n * @public\n */\n\nreq.accepts = function(){\n  var accept = accepts(this);\n  return accept.types.apply(accept, arguments);\n};\n\n/**\n * Check if the given `encoding`s are accepted.\n *\n * @param {String} ...encoding\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsEncodings = function(){\n  var accept = accepts(this);\n  return accept.encodings.apply(accept, arguments);\n};\n\nreq.acceptsEncoding = deprecate.function(req.acceptsEncodings,\n  'req.acceptsEncoding: Use acceptsEncodings instead');\n\n/**\n * Check if the given `charset`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...charset\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsCharsets = function(){\n  var accept = accepts(this);\n  return accept.charsets.apply(accept, arguments);\n};\n\nreq.acceptsCharset = deprecate.function(req.acceptsCharsets,\n  'req.acceptsCharset: Use acceptsCharsets instead');\n\n/**\n * Check if the given `lang`s are acceptable,\n * otherwise you should respond with 406 \"Not Acceptable\".\n *\n * @param {String} ...lang\n * @return {String|Array}\n * @public\n */\n\nreq.acceptsLanguages = function(){\n  var accept = accepts(this);\n  return accept.languages.apply(accept, arguments);\n};\n\nreq.acceptsLanguage = deprecate.function(req.acceptsLanguages,\n  'req.acceptsLanguage: Use acceptsLanguages instead');\n\n/**\n * Parse Range header field, capping to the given `size`.\n *\n * Unspecified ranges such as \"0-\" require knowledge of your resource length. In\n * the case of a byte range this is of course the total number of bytes. If the\n * Range header field is not given `undefined` is returned, `-1` when unsatisfiable,\n * and `-2` when syntactically invalid.\n *\n * When ranges are returned, the array has a \"type\" property which is the type of\n * range that is required (most commonly, \"bytes\"). Each array element is an object\n * with a \"start\" and \"end\" property for the portion of the range.\n *\n * The \"combine\" option can be set to `true` and overlapping & adjacent ranges\n * will be combined into a single range.\n *\n * NOTE: remember that ranges are inclusive, so for example \"Range: users=0-3\"\n * should respond with 4 users when available, not 3.\n *\n * @param {number} size\n * @param {object} [options]\n * @param {boolean} [options.combine=false]\n * @return {number|array}\n * @public\n */\n\nreq.range = function range(size, options) {\n  var range = this.get('Range');\n  if (!range) return;\n  return parseRange(size, range, options);\n};\n\n/**\n * Return the value of param `name` when present or `defaultValue`.\n *\n *  - Checks route placeholders, ex: _/user/:id_\n *  - Checks body params, ex: id=12, {\"id\":12}\n *  - Checks query string params, ex: ?id=12\n *\n * To utilize request bodies, `req.body`\n * should be an object. This can be done by using\n * the `bodyParser()` middleware.\n *\n * @param {String} name\n * @param {Mixed} [defaultValue]\n * @return {String}\n * @public\n */\n\nreq.param = function param(name, defaultValue) {\n  var params = this.params || {};\n  var body = this.body || {};\n  var query = this.query || {};\n\n  var args = arguments.length === 1\n    ? 'name'\n    : 'name, default';\n  deprecate('req.param(' + args + '): Use req.params, req.body, or req.query instead');\n\n  if (null != params[name] && params.hasOwnProperty(name)) return params[name];\n  if (null != body[name]) return body[name];\n  if (null != query[name]) return query[name];\n\n  return defaultValue;\n};\n\n/**\n * Check if the incoming request contains the \"Content-Type\"\n * header field, and it contains the given mime `type`.\n *\n * Examples:\n *\n *      // With Content-Type: text/html; charset=utf-8\n *      req.is('html');\n *      req.is('text/html');\n *      req.is('text/*');\n *      // => true\n *\n *      // When Content-Type is application/json\n *      req.is('json');\n *      req.is('application/json');\n *      req.is('application/*');\n *      // => true\n *\n *      req.is('html');\n *      // => false\n *\n * @param {String|Array} types...\n * @return {String|false|null}\n * @public\n */\n\nreq.is = function is(types) {\n  var arr = types;\n\n  // support flattened arguments\n  if (!Array.isArray(types)) {\n    arr = new Array(arguments.length);\n    for (var i = 0; i < arr.length; i++) {\n      arr[i] = arguments[i];\n    }\n  }\n\n  return typeis(this, arr);\n};\n\n/**\n * Return the protocol string \"http\" or \"https\"\n * when requested with TLS. When the \"trust proxy\"\n * setting trusts the socket address, the\n * \"X-Forwarded-Proto\" header field will be trusted\n * and used if present.\n *\n * If you're running behind a reverse proxy that\n * supplies https for you this may be enabled.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'protocol', function protocol(){\n  var proto = this.connection.encrypted\n    ? 'https'\n    : 'http';\n  var trust = this.app.get('trust proxy fn');\n\n  if (!trust(this.connection.remoteAddress, 0)) {\n    return proto;\n  }\n\n  // Note: X-Forwarded-Proto is normally only ever a\n  //       single value, but this is to be safe.\n  var header = this.get('X-Forwarded-Proto') || proto\n  var index = header.indexOf(',')\n\n  return index !== -1\n    ? header.substring(0, index).trim()\n    : header.trim()\n});\n\n/**\n * Short-hand for:\n *\n *    req.protocol === 'https'\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'secure', function secure(){\n  return this.protocol === 'https';\n});\n\n/**\n * Return the remote address from the trusted proxy.\n *\n * The is the remote address on the socket unless\n * \"trust proxy\" is set.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'ip', function ip(){\n  var trust = this.app.get('trust proxy fn');\n  return proxyaddr(this, trust);\n});\n\n/**\n * When \"trust proxy\" is set, trusted proxy addresses + client.\n *\n * For example if the value were \"client, proxy1, proxy2\"\n * you would receive the array `[\"client\", \"proxy1\", \"proxy2\"]`\n * where \"proxy2\" is the furthest down-stream and \"proxy1\" and\n * \"proxy2\" were trusted.\n *\n * @return {Array}\n * @public\n */\n\ndefineGetter(req, 'ips', function ips() {\n  var trust = this.app.get('trust proxy fn');\n  var addrs = proxyaddr.all(this, trust);\n\n  // reverse the order (to farthest -> closest)\n  // and remove socket address\n  addrs.reverse().pop()\n\n  return addrs\n});\n\n/**\n * Return subdomains as an array.\n *\n * Subdomains are the dot-separated parts of the host before the main domain of\n * the app. By default, the domain of the app is assumed to be the last two\n * parts of the host. This can be changed by setting \"subdomain offset\".\n *\n * For example, if the domain is \"tobi.ferrets.example.com\":\n * If \"subdomain offset\" is not set, req.subdomains is `[\"ferrets\", \"tobi\"]`.\n * If \"subdomain offset\" is 3, req.subdomains is `[\"tobi\"]`.\n *\n * @return {Array}\n * @public\n */\n\ndefineGetter(req, 'subdomains', function subdomains() {\n  var hostname = this.hostname;\n\n  if (!hostname) return [];\n\n  var offset = this.app.get('subdomain offset');\n  var subdomains = !isIP(hostname)\n    ? hostname.split('.').reverse()\n    : [hostname];\n\n  return subdomains.slice(offset);\n});\n\n/**\n * Short-hand for `url.parse(req.url).pathname`.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'path', function path() {\n  return parse(this).pathname;\n});\n\n/**\n * Parse the \"Host\" header field to a hostname.\n *\n * When the \"trust proxy\" setting trusts the socket\n * address, the \"X-Forwarded-Host\" header field will\n * be trusted.\n *\n * @return {String}\n * @public\n */\n\ndefineGetter(req, 'hostname', function hostname(){\n  var trust = this.app.get('trust proxy fn');\n  var host = this.get('X-Forwarded-Host');\n\n  if (!host || !trust(this.connection.remoteAddress, 0)) {\n    host = this.get('Host');\n  } else if (host.indexOf(',') !== -1) {\n    // Note: X-Forwarded-Host is normally only ever a\n    //       single value, but this is to be safe.\n    host = host.substring(0, host.indexOf(',')).trimRight()\n  }\n\n  if (!host) return;\n\n  // IPv6 literal support\n  var offset = host[0] === '['\n    ? host.indexOf(']') + 1\n    : 0;\n  var index = host.indexOf(':', offset);\n\n  return index !== -1\n    ? host.substring(0, index)\n    : host;\n});\n\n// TODO: change req.host to return host in next major\n\ndefineGetter(req, 'host', deprecate.function(function host(){\n  return this.hostname;\n}, 'req.host: Use req.hostname instead'));\n\n/**\n * Check if the request is fresh, aka\n * Last-Modified and/or the ETag\n * still match.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'fresh', function(){\n  var method = this.method;\n  var res = this.res\n  var status = res.statusCode\n\n  // GET or HEAD for weak freshness validation only\n  if ('GET' !== method && 'HEAD' !== method) return false;\n\n  // 2xx or 304 as per rfc2616 14.26\n  if ((status >= 200 && status < 300) || 304 === status) {\n    return fresh(this.headers, {\n      'etag': res.get('ETag'),\n      'last-modified': res.get('Last-Modified')\n    })\n  }\n\n  return false;\n});\n\n/**\n * Check if the request is stale, aka\n * \"Last-Modified\" and / or the \"ETag\" for the\n * resource has changed.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'stale', function stale(){\n  return !this.fresh;\n});\n\n/**\n * Check if the request was an _XMLHttpRequest_.\n *\n * @return {Boolean}\n * @public\n */\n\ndefineGetter(req, 'xhr', function xhr(){\n  var val = this.get('X-Requested-With') || '';\n  return val.toLowerCase() === 'xmlhttprequest';\n});\n\n/**\n * Helper function for creating a getter on an object.\n *\n * @param {Object} obj\n * @param {String} name\n * @param {Function} getter\n...\n// Path: lib/router/layer.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar pathRegexp = require('path-to-regexp');\nvar debug = require('debug')('express:router:layer');\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Layer;\n\nfunction Layer(path, options, fn) {\n  if (!(this instanceof Layer)) {\n    return new Layer(path, options, fn);\n  }\n\n  debug('new %o', path)\n  var opts = options || {};\n\n  this.handle = fn;\n  this.name = fn.name || '<anonymous>';\n  this.params = undefined;\n  this.path = undefined;\n  this.regexp = pathRegexp(path, this.keys = [], opts);\n\n  // set fast path flags\n  this.regexp.fast_star = path === '*'\n  this.regexp.fast_slash = path === '/' && opts.end === false\n}\n\n/**\n * Handle the error for the layer.\n *\n * @param {Error} error\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_error = function handle_error(error, req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length !== 4) {\n    // not a standard error handler\n    return next(error);\n  }\n\n  try {\n    fn(error, req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Handle the request for the layer.\n *\n * @param {Request} req\n * @param {Response} res\n * @param {function} next\n * @api private\n */\n\nLayer.prototype.handle_request = function handle(req, res, next) {\n  var fn = this.handle;\n\n  if (fn.length > 3) {\n    // not a standard request handler\n    return next();\n  }\n\n  try {\n    fn(req, res, next);\n  } catch (err) {\n    next(err);\n  }\n};\n\n/**\n * Check if this route matches `path`, if so\n * populate `.params`.\n *\n * @param {String} path\n * @return {Boolean}\n * @api private\n */\n\nLayer.prototype.match = function match(path) {\n  var match\n\n  if (path != null) {\n    // fast path non-ending match for / (any path matches)\n    if (this.regexp.fast_slash) {\n      this.params = {}\n      this.path = ''\n      return true\n    }\n\n    // fast path for * (everything matched in a param)\n    if (this.regexp.fast_star) {\n      this.params = {'0': decode_param(path)}\n      this.path = path\n      return true\n    }\n\n    // match the path\n    match = this.regexp.exec(path)\n  }\n\n  if (!match) {\n    this.params = undefined;\n    this.path = undefined;\n    return false;\n  }\n\n  // store values\n  this.params = {};\n  this.path = match[0]\n\n  var keys = this.keys;\n  var params = this.params;\n\n  for (var i = 1; i < match.length; i++) {\n    var key = keys[i - 1];\n    var prop = key.name;\n    var val = decode_param(match[i])\n\n    if (val !== undefined || !(hasOwnProperty.call(params, prop))) {\n      params[prop] = val;\n    }\n  }\n\n  return true;\n};\n\n/**\n * Decode param value.\n *\n * @param {string} val\n * @return {string}\n * @private\n */\n\nfunction decode_param(val) {\n  if (typeof val !== 'string' || val.length === 0) {\n    return val;\n  }\n\n  try {\n    return decodeURIComponent(val);\n  } catch (err) {\n    if (err instanceof URIError) {\n      err.message = 'Failed to decode param \\'' + val + '\\'';\n      err.status = err.statusCode = 400;\n    }\n\n    throw err;\n  }\n}\n\n// Path: lib/router/route.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:router:route');\nvar flatten = require('array-flatten');\nvar Layer = require('./layer');\nvar methods = require('methods');\n\n/**\n * Module variables.\n * @private\n */\n\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = Route;\n\n/**\n * Initialize `Route` with the given `path`,\n *\n * @param {String} path\n * @public\n */\n\nfunction Route(path) {\n  this.path = path;\n  this.stack = [];\n\n  debug('new %o', path)\n\n  // route handlers for various http methods\n  this.methods = {};\n}\n\n/**\n * Determine if the route handles a given method.\n * @private\n */\n\nRoute.prototype._handles_method = function _handles_method(method) {\n  if (this.methods._all) {\n    return true;\n  }\n\n  // normalize name\n  var name = typeof method === 'string'\n    ? method.toLowerCase()\n    : method\n\n  if (name === 'head' && !this.methods['head']) {\n    name = 'get';\n  }\n\n  return Boolean(this.methods[name]);\n};\n\n/**\n * @return {Array} supported HTTP methods\n * @private\n */\n\nRoute.prototype._options = function _options() {\n  var methods = Object.keys(this.methods);\n\n  // append automatic head\n  if (this.methods.get && !this.methods.head) {\n    methods.push('head');\n  }\n\n  for (var i = 0; i < methods.length; i++) {\n    // make upper case\n    methods[i] = methods[i].toUpperCase();\n  }\n\n  return methods;\n};\n\n/**\n * dispatch req, res into this route\n * @private\n */\n\nRoute.prototype.dispatch = function dispatch(req, res, done) {\n  var idx = 0;\n  var stack = this.stack;\n  var sync = 0\n\n  if (stack.length === 0) {\n    return done();\n  }\n  var method = typeof req.method === 'string'\n    ? req.method.toLowerCase()\n    : req.method\n\n  if (method === 'head' && !this.methods['head']) {\n    method = 'get';\n  }\n\n  req.route = this;\n\n  next();\n\n  function next(err) {\n    // signal to exit route\n    if (err && err === 'route') {\n      return done();\n    }\n\n    // signal to exit router\n    if (err && err === 'router') {\n      return done(err)\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    var layer = stack[idx++]\n\n    // end of layers\n    if (!layer) {\n      return done(err)\n    }\n\n    if (layer.method && layer.method !== method) {\n      next(err)\n    } else if (err) {\n      layer.handle_error(err, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n\n    sync = 0\n  }\n};\n\n/**\n * Add a handler for all HTTP verbs to this route.\n *\n * Behaves just like middleware and can respond or call `next`\n * to continue processing.\n *\n * You can use multiple `.all` call to add multiple handlers.\n *\n *   function check_something(req, res, next){\n *     next();\n *   };\n *\n *   function validate_user(req, res, next){\n *     next();\n *   };\n *\n *   route\n *   .all(validate_user)\n *   .all(check_something)\n *   .get(function(req, res, next){\n *     res.send('hello world');\n *   });\n *\n * @param {function} handler\n * @return {Route} for chaining\n * @api public\n */\n\nRoute.prototype.all = function all() {\n  var handles = flatten(slice.call(arguments));\n\n  for (var i = 0; i < handles.length; i++) {\n    var handle = handles[i];\n\n    if (typeof handle !== 'function') {\n      var type = toString.call(handle);\n      var msg = 'Route.all() requires a callback function but got a ' + type\n      throw new TypeError(msg);\n    }\n\n    var layer = Layer('/', {}, handle);\n    layer.method = undefined;\n\n    this.methods._all = true;\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\nmethods.forEach(function(method){\n  Route.prototype[method] = function(){\n    var handles = flatten(slice.call(arguments));\n\n    for (var i = 0; i < handles.length; i++) {\n      var handle = handles[i];\n\n      if (typeof handle !== 'function') {\n        var type = toString.call(handle);\n        var msg = 'Route.' + method + '() requires a callback function but got a ' + type\n        throw new Error(msg);\n      }\n\n      debug('%s %o', method, this.path)\n\n      var layer = Layer('/', {}, handle);\n      layer.method = method;\n\n      this.methods[method] = true;\n      this.stack.push(layer);\n    }\n\n    return this;\n  };\n});\n\n// Path: lib/router/index.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Route = require('./route');\nvar Layer = require('./layer');\nvar methods = require('methods');\nvar mixin = require('utils-merge');\nvar debug = require('debug')('express:router');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar parseUrl = require('parseurl');\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar objectRegExp = /^\\[object (\\S+)\\]$/;\nvar slice = Array.prototype.slice;\nvar toString = Object.prototype.toString;\n\n/**\n * Initialize a new `Router` with the given `options`.\n *\n * @param {Object} [options]\n * @return {Router} which is a callable function\n * @public\n */\n\nvar proto = module.exports = function(options) {\n  var opts = options || {};\n\n  function router(req, res, next) {\n    router.handle(req, res, next);\n  }\n\n  // mixin Router class functions\n  setPrototypeOf(router, proto)\n\n  router.params = {};\n  router._params = [];\n  router.caseSensitive = opts.caseSensitive;\n  router.mergeParams = opts.mergeParams;\n  router.strict = opts.strict;\n  router.stack = [];\n\n  return router;\n};\n\n/**\n * Map the given param placeholder `name`(s) to the given callback.\n *\n * Parameter mapping is used to provide pre-conditions to routes\n * which use normalized placeholders. For example a _:user_id_ parameter\n * could automatically load a user's information from the database without\n * any additional code,\n *\n * The callback uses the same signature as middleware, the only difference\n * being that the value of the placeholder is passed, in this case the _id_\n * of the user. Once the `next()` function is invoked, just like middleware\n * it will continue on to execute the route, or subsequent parameter functions.\n *\n * Just like in middleware, you must either respond to the request or call next\n * to avoid stalling the request.\n *\n *  app.param('user_id', function(req, res, next, id){\n *    User.find(id, function(err, user){\n *      if (err) {\n *        return next(err);\n *      } else if (!user) {\n *        return next(new Error('failed to load user'));\n *      }\n *      req.user = user;\n *      next();\n *    });\n *  });\n *\n * @param {String} name\n * @param {Function} fn\n * @return {app} for chaining\n * @public\n */\n\nproto.param = function param(name, fn) {\n  // param logic\n  if (typeof name === 'function') {\n    deprecate('router.param(fn): Refactor to use path params');\n    this._params.push(name);\n    return;\n  }\n\n  // apply param functions\n  var params = this._params;\n  var len = params.length;\n  var ret;\n\n  if (name[0] === ':') {\n    deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.slice(1)) + ', fn) instead')\n    name = name.slice(1)\n  }\n\n  for (var i = 0; i < len; ++i) {\n    if (ret = params[i](name, fn)) {\n      fn = ret;\n    }\n  }\n\n  // ensure we end up with a\n  // middleware function\n  if ('function' !== typeof fn) {\n    throw new Error('invalid param() call for ' + name + ', got ' + fn);\n  }\n\n  (this.params[name] = this.params[name] || []).push(fn);\n  return this;\n};\n\n/**\n * Dispatch a req, res into the router.\n * @private\n */\n\nproto.handle = function handle(req, res, out) {\n  var self = this;\n\n  debug('dispatching %s %s', req.method, req.url);\n\n  var idx = 0;\n  var protohost = getProtohost(req.url) || ''\n  var removed = '';\n  var slashAdded = false;\n  var sync = 0\n  var paramcalled = {};\n\n  // store options for OPTIONS request\n  // only used if OPTIONS request\n  var options = [];\n\n  // middleware and routes\n  var stack = self.stack;\n\n  // manage inter-router variables\n  var parentParams = req.params;\n  var parentUrl = req.baseUrl || '';\n  var done = restore(out, req, 'baseUrl', 'next', 'params');\n\n  // setup next layer\n  req.next = next;\n\n  // for options requests, respond with a default if nothing else responds\n  if (req.method === 'OPTIONS') {\n    done = wrap(done, function(old, err) {\n      if (err || options.length === 0) return old(err);\n      sendOptionsResponse(res, options, old);\n    });\n  }\n\n  // setup basic req values\n  req.baseUrl = parentUrl;\n  req.originalUrl = req.originalUrl || req.url;\n\n  next();\n\n  function next(err) {\n    var layerError = err === 'route'\n      ? null\n      : err;\n\n    // remove added slash\n    if (slashAdded) {\n      req.url = req.url.slice(1)\n      slashAdded = false;\n    }\n\n    // restore altered req.url\n    if (removed.length !== 0) {\n      req.baseUrl = parentUrl;\n      req.url = protohost + removed + req.url.slice(protohost.length)\n      removed = '';\n    }\n\n    // signal to exit router\n    if (layerError === 'router') {\n      setImmediate(done, null)\n      return\n    }\n\n    // no more matching layers\n    if (idx >= stack.length) {\n      setImmediate(done, layerError);\n      return;\n    }\n\n    // max sync stack\n    if (++sync > 100) {\n      return setImmediate(next, err)\n    }\n\n    // get pathname of request\n    var path = getPathname(req);\n\n    if (path == null) {\n      return done(layerError);\n    }\n\n    // find next matching layer\n    var layer;\n    var match;\n    var route;\n\n    while (match !== true && idx < stack.length) {\n      layer = stack[idx++];\n      match = matchLayer(layer, path);\n      route = layer.route;\n\n      if (typeof match !== 'boolean') {\n        // hold on to layerError\n        layerError = layerError || match;\n      }\n\n      if (match !== true) {\n        continue;\n      }\n\n      if (!route) {\n        // process non-route handlers normally\n        continue;\n      }\n\n      if (layerError) {\n        // routes do not match with a pending error\n        match = false;\n        continue;\n      }\n\n      var method = req.method;\n      var has_method = route._handles_method(method);\n\n      // build up automatic options response\n      if (!has_method && method === 'OPTIONS') {\n        appendMethods(options, route._options());\n      }\n\n      // don't even bother matching route\n      if (!has_method && method !== 'HEAD') {\n        match = false;\n      }\n    }\n\n    // no match\n    if (match !== true) {\n      return done(layerError);\n    }\n\n    // store route for dispatch on change\n    if (route) {\n      req.route = route;\n    }\n\n    // Capture one-time layer values\n    req.params = self.mergeParams\n      ? mergeParams(layer.params, parentParams)\n      : layer.params;\n    var layerPath = layer.path;\n\n    // this should be done for the layer\n    self.process_params(layer, paramcalled, req, res, function (err) {\n      if (err) {\n        next(layerError || err)\n      } else if (route) {\n        layer.handle_request(req, res, next)\n      } else {\n        trim_prefix(layer, layerError, layerPath, path)\n      }\n\n      sync = 0\n    });\n  }\n\n  function trim_prefix(layer, layerError, layerPath, path) {\n    if (layerPath.length !== 0) {\n      // Validate path is a prefix match\n      if (layerPath !== path.slice(0, layerPath.length)) {\n        next(layerError)\n        return\n      }\n\n      // Validate path breaks on a path separator\n      var c = path[layerPath.length]\n      if (c && c !== '/' && c !== '.') return next(layerError)\n\n      // Trim off the part of the url that matches the route\n      // middleware (.use stuff) needs to have the path stripped\n      debug('trim prefix (%s) from url %s', layerPath, req.url);\n      removed = layerPath;\n      req.url = protohost + req.url.slice(protohost.length + removed.length)\n\n      // Ensure leading slash\n      if (!protohost && req.url[0] !== '/') {\n        req.url = '/' + req.url;\n        slashAdded = true;\n      }\n\n      // Setup base URL (no trailing slash)\n      req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'\n        ? removed.substring(0, removed.length - 1)\n        : removed);\n    }\n\n    debug('%s %s : %s', layer.name, layerPath, req.originalUrl);\n\n    if (layerError) {\n      layer.handle_error(layerError, req, res, next);\n    } else {\n      layer.handle_request(req, res, next);\n    }\n  }\n};\n\n/**\n * Process any parameters for the layer.\n * @private\n */\n\nproto.process_params = function process_params(layer, called, req, res, done) {\n  var params = this.params;\n\n  // captured parameters from the layer, keys and values\n  var keys = layer.keys;\n\n  // fast track\n  if (!keys || keys.length === 0) {\n    return done();\n  }\n\n  var i = 0;\n  var name;\n  var paramIndex = 0;\n  var key;\n  var paramVal;\n  var paramCallbacks;\n  var paramCalled;\n\n  // process params in order\n  // param callbacks can be async\n  function param(err) {\n    if (err) {\n      return done(err);\n    }\n\n    if (i >= keys.length ) {\n      return done();\n    }\n\n    paramIndex = 0;\n    key = keys[i++];\n    name = key.name;\n    paramVal = req.params[name];\n    paramCallbacks = params[name];\n    paramCalled = called[name];\n\n    if (paramVal === undefined || !paramCallbacks) {\n      return param();\n    }\n\n    // param previously called with same value or error occurred\n    if (paramCalled && (paramCalled.match === paramVal\n      || (paramCalled.error && paramCalled.error !== 'route'))) {\n      // restore value\n      req.params[name] = paramCalled.value;\n\n      // next param\n      return param(paramCalled.error);\n    }\n\n    called[name] = paramCalled = {\n      error: null,\n      match: paramVal,\n      value: paramVal\n    };\n\n    paramCallback();\n  }\n\n  // single param callbacks\n  function paramCallback(err) {\n    var fn = paramCallbacks[paramIndex++];\n\n    // store updated value\n    paramCalled.value = req.params[key.name];\n\n    if (err) {\n      // store error\n      paramCalled.error = err;\n      param(err);\n      return;\n    }\n\n    if (!fn) return param();\n\n    try {\n      fn(req, res, paramCallback, paramVal, key.name);\n    } catch (e) {\n      paramCallback(e);\n    }\n  }\n\n  param();\n};\n\n/**\n * Use the given middleware function, with optional path, defaulting to \"/\".\n *\n * Use (like `.all`) will run for any http METHOD, but it will not add\n * handlers for those methods so OPTIONS requests will not consider `.use`\n * functions even if they could respond.\n *\n * The other difference is that _route_ path is stripped and not visible\n * to the handler function. The main effect of this feature is that mounted\n * handlers can operate without any code changes regardless of the \"prefix\"\n * pathname.\n *\n * @public\n */\n\nproto.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate router.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var callbacks = flatten(slice.call(arguments, offset));\n\n  if (callbacks.length === 0) {\n    throw new TypeError('Router.use() requires a middleware function')\n  }\n\n  for (var i = 0; i < callbacks.length; i++) {\n    var fn = callbacks[i];\n\n    if (typeof fn !== 'function') {\n      throw new TypeError('Router.use() requires a middleware function but got a ' + gettype(fn))\n    }\n\n    // add the middleware\n    debug('use %o %s', path, fn.name || '<anonymous>')\n\n    var layer = new Layer(path, {\n      sensitive: this.caseSensitive,\n      strict: false,\n      end: false\n    }, fn);\n\n    layer.route = undefined;\n\n    this.stack.push(layer);\n  }\n\n  return this;\n};\n\n/**\n * Create a new Route for the given path.\n *\n * Each route contains a separate middleware stack and VERB handlers.\n *\n * See the Route api documentation for details on adding handlers\n * and middleware to routes.\n *\n * @param {String} path\n * @return {Route}\n * @public\n */\n\nproto.route = function route(path) {\n  var route = new Route(path);\n\n  var layer = new Layer(path, {\n    sensitive: this.caseSensitive,\n    strict: this.strict,\n    end: true\n  }, route.dispatch.bind(route));\n\n  layer.route = route;\n\n  this.stack.push(layer);\n  return route;\n};\n\n// create Router#VERB functions\nmethods.concat('all').forEach(function(method){\n  proto[method] = function(path){\n    var route = this.route(path)\n    route[method].apply(route, slice.call(arguments, 1));\n    return this;\n  };\n});\n\n// append methods to a list of methods\nfunction appendMethods(list, addition) {\n  for (var i = 0; i < addition.length; i++) {\n    var method = addition[i];\n    if (list.indexOf(method) === -1) {\n      list.push(method);\n    }\n  }\n}\n\n// get pathname of request\nfunction getPathname(req) {\n  try {\n    return parseUrl(req).pathname;\n  } catch (err) {\n    return undefined;\n  }\n}\n\n// Get get protocol + host for a URL\nfunction getProtohost(url) {\n  if (typeof url !== 'string' || url.length === 0 || url[0] === '/') {\n    return undefined\n  }\n\n  var searchIndex = url.indexOf('?')\n  var pathLength = searchIndex !== -1\n    ? searchIndex\n    : url.length\n  var fqdnIndex = url.slice(0, pathLength).indexOf('://')\n\n  return fqdnIndex !== -1\n    ? url.substring(0, url.indexOf('/', 3 + fqdnIndex))\n    : undefined\n}\n\n// get type for error message\nfunction gettype(obj) {\n  var type = typeof obj;\n\n  if (type !== 'object') {\n    return type;\n  }\n\n  // inspect [[Class]] for objects\n  return toString.call(obj)\n    .replace(objectRegExp, '$1');\n}\n\n/**\n * Match path to a layer.\n *\n * @param {Layer} layer\n * @param {string} path\n * @private\n */\n\nfunction matchLayer(layer, path) {\n  try {\n    return layer.match(path);\n  } catch (err) {\n    return err;\n  }\n}\n\n// merge params with parent params\nfunction mergeParams(params, parent) {\n  if (typeof parent !== 'object' || !parent) {\n    return params;\n  }\n\n  // make copy of parent for base\n  var obj = mixin({}, parent);\n\n  // simple non-numeric merging\n  if (!(0 in params) || !(0 in parent)) {\n    return mixin(obj, params);\n  }\n\n  var i = 0;\n  var o = 0;\n\n  // determine numeric gaps\n  while (i in params) {\n    i++;\n  }\n\n  while (o in parent) {\n    o++;\n  }\n\n  // offset numeric indices in params before merge\n  for (i--; i >= 0; i--) {\n    params[i + o] = params[i];\n\n    // create holes for the merge when necessary\n    if (i < o) {\n      delete params[i];\n    }\n  }\n\n  return mixin(obj, params);\n}\n\n// restore obj props after function\nfunction restore(fn, obj) {\n  var props = new Array(arguments.length - 2);\n  var vals = new Array(arguments.length - 2);\n\n  for (var i = 0; i < props.length; i++) {\n    props[i] = arguments[i + 2];\n    vals[i] = obj[props[i]];\n  }\n\n  return function () {\n    // restore vals\n    for (var i = 0; i < props.length; i++) {\n      obj[props[i]] = vals[i];\n    }\n\n    return fn.apply(this, arguments);\n  };\n}\n\n// send an OPTIONS response\nfunction sendOptionsResponse(res, options, next) {\n  try {\n    var body = options.join(',');\n    res.set('Allow', body);\n    res.send(body);\n  } catch (err) {\n    next(err);\n  }\n}\n\n// wrap a function\nfunction wrap(old, fn) {\n  return function proxy() {\n    var args = new Array(arguments.length + 1);\n\n    args[0] = old;\n    for (var i = 0, len = arguments.length; i < len; i++) {\n      args[i + 1] = arguments[i];\n    }\n\n    fn.apply(this, args);\n  };\n}\n\n// Path: lib/middleware/init.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Initialization middleware, exposing the\n * request and response to each other, as well\n * as defaulting the X-Powered-By header field.\n *\n * @param {Function} app\n * @return {Function}\n * @api private\n */\n\nexports.init = function(app){\n  return function expressInit(req, res, next){\n    if (app.enabled('x-powered-by')) res.setHeader('X-Powered-By', 'Express');\n    req.res = res;\n    res.req = req;\n    req.next = next;\n\n    setPrototypeOf(req, app.request)\n    setPrototypeOf(res, app.response)\n\n    res.locals = res.locals || Object.create(null);\n\n    next();\n  };\n};\n\n\n// Path: lib/view.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar debug = require('debug')('express:view');\nvar path = require('path');\nvar fs = require('fs');\n\n/**\n * Module variables.\n * @private\n */\n\nvar dirname = path.dirname;\nvar basename = path.basename;\nvar extname = path.extname;\nvar join = path.join;\nvar resolve = path.resolve;\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = View;\n\n/**\n * Initialize a new `View` with the given `name`.\n *\n * Options:\n *\n *   - `defaultEngine` the default template engine name\n *   - `engines` template engine require() cache\n *   - `root` root path for view lookup\n *\n * @param {string} name\n * @param {object} options\n * @public\n */\n\nfunction View(name, options) {\n  var opts = options || {};\n\n  this.defaultEngine = opts.defaultEngine;\n  this.ext = extname(name);\n  this.name = name;\n  this.root = opts.root;\n\n  if (!this.ext && !this.defaultEngine) {\n    throw new Error('No default engine was specified and no extension was provided.');\n  }\n\n  var fileName = name;\n\n  if (!this.ext) {\n    // get extension from default engine name\n    this.ext = this.defaultEngine[0] !== '.'\n      ? '.' + this.defaultEngine\n      : this.defaultEngine;\n\n    fileName += this.ext;\n  }\n\n  if (!opts.engines[this.ext]) {\n    // load engine\n    var mod = this.ext.slice(1)\n    debug('require \"%s\"', mod)\n\n    // default engine export\n    var fn = require(mod).__express\n\n    if (typeof fn !== 'function') {\n      throw new Error('Module \"' + mod + '\" does not provide a view engine.')\n    }\n\n    opts.engines[this.ext] = fn\n  }\n\n  // store loaded engine\n  this.engine = opts.engines[this.ext];\n\n  // lookup path\n  this.path = this.lookup(fileName);\n}\n\n/**\n * Lookup view by the given `name`\n *\n * @param {string} name\n * @private\n */\n\nView.prototype.lookup = function lookup(name) {\n  var path;\n  var roots = [].concat(this.root);\n\n  debug('lookup \"%s\"', name);\n\n  for (var i = 0; i < roots.length && !path; i++) {\n    var root = roots[i];\n\n    // resolve the path\n    var loc = resolve(root, name);\n    var dir = dirname(loc);\n    var file = basename(loc);\n\n    // resolve the file\n    path = this.resolve(dir, file);\n  }\n\n  return path;\n};\n\n/**\n * Render with the given options.\n *\n * @param {object} options\n * @param {function} callback\n * @private\n */\n\nView.prototype.render = function render(options, callback) {\n  debug('render \"%s\"', this.path);\n  this.engine(this.path, options, callback);\n};\n\n/**\n * Resolve the file within the given directory.\n *\n * @param {string} dir\n * @param {string} file\n * @private\n */\n\nView.prototype.resolve = function resolve(dir, file) {\n  var ext = this.ext;\n\n  // <path>.<ext>\n  var path = join(dir, file);\n  var stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n\n  // <path>/index.<ext>\n  path = join(dir, basename(file, ext), 'index' + ext);\n  stat = tryStat(path);\n\n  if (stat && stat.isFile()) {\n    return path;\n  }\n};\n\n/**\n * Return a stat, maybe.\n *\n * @param {string} path\n * @return {fs.Stats}\n * @private\n */\n\n\nfunction tryStat(path) {\n  debug('stat \"%s\"', path);\n\n  try {\n    return fs.statSync(path);\n  } catch (e) {\n    return undefined;\n  }\n}\n\n// Path: lib/middleware/query.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n */\n\nvar merge = require('utils-merge')\nvar parseUrl = require('parseurl');\nvar qs = require('qs');\n\n/**\n * @param {Object} options\n * @return {Function}\n * @api public\n */\n\nmodule.exports = function query(options) {\n  var opts = merge({}, options)\n  var queryparse = qs.parse;\n\n  if (typeof options === 'function') {\n    queryparse = options;\n    opts = undefined;\n  }\n\n  if (opts !== undefined && opts.allowPrototypes === undefined) {\n    // back-compat for qs module\n    opts.allowPrototypes = true;\n  }\n\n  return function query(req, res, next){\n    if (!req.query) {\n      var val = parseUrl(req).query;\n      req.query = queryparse(val, opts);\n    }\n\n    next();\n  };\n};\n\n// Path: lib/application.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar finalhandler = require('finalhandler');\nvar Router = require('./router');\nvar methods = require('methods');\nvar middleware = require('./middleware/init');\nvar query = require('./middleware/query');\nvar debug = require('debug')('express:application');\nvar View = require('./view');\nvar http = require('http');\nvar compileETag = require('./utils').compileETag;\nvar compileQueryParser = require('./utils').compileQueryParser;\nvar compileTrust = require('./utils').compileTrust;\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar merge = require('utils-merge');\nvar resolve = require('path').resolve;\nvar setPrototypeOf = require('setprototypeof')\n\n/**\n * Module variables.\n * @private\n */\n\nvar hasOwnProperty = Object.prototype.hasOwnProperty\nvar slice = Array.prototype.slice;\n\n/**\n * Application prototype.\n */\n\nvar app = exports = module.exports = {};\n\n/**\n * Variable for trust proxy inheritance back-compat\n * @private\n */\n\nvar trustProxyDefaultSymbol = '@@symbol:trust_proxy_default';\n\n/**\n * Initialize the server.\n *\n *   - setup default configuration\n *   - setup default middleware\n *   - setup route reflection methods\n *\n * @private\n */\n\napp.init = function init() {\n  this.cache = {};\n  this.engines = {};\n  this.settings = {};\n\n  this.defaultConfiguration();\n};\n\n/**\n * Initialize application configuration.\n * @private\n */\n\napp.defaultConfiguration = function defaultConfiguration() {\n  var env = process.env.NODE_ENV || 'development';\n\n  // default settings\n  this.enable('x-powered-by');\n  this.set('etag', 'weak');\n  this.set('env', env);\n  this.set('query parser', 'extended');\n  this.set('subdomain offset', 2);\n  this.set('trust proxy', false);\n\n  // trust proxy inherit back-compat\n  Object.defineProperty(this.settings, trustProxyDefaultSymbol, {\n    configurable: true,\n    value: true\n  });\n\n  debug('booting in %s mode', env);\n\n  this.on('mount', function onmount(parent) {\n    // inherit trust proxy\n    if (this.settings[trustProxyDefaultSymbol] === true\n      && typeof parent.settings['trust proxy fn'] === 'function') {\n      delete this.settings['trust proxy'];\n      delete this.settings['trust proxy fn'];\n    }\n\n    // inherit protos\n    setPrototypeOf(this.request, parent.request)\n    setPrototypeOf(this.response, parent.response)\n    setPrototypeOf(this.engines, parent.engines)\n    setPrototypeOf(this.settings, parent.settings)\n  });\n\n  // setup locals\n  this.locals = Object.create(null);\n\n  // top-most app is mounted at /\n  this.mountpath = '/';\n\n  // default locals\n  this.locals.settings = this.settings;\n\n  // default configuration\n  this.set('view', View);\n  this.set('views', resolve('views'));\n  this.set('jsonp callback name', 'callback');\n\n  if (env === 'production') {\n    this.enable('view cache');\n  }\n\n  Object.defineProperty(this, 'router', {\n    get: function() {\n      throw new Error('\\'app.router\\' is deprecated!\\nPlease see the 3.x to 4.x migration guide for details on how to update your app.');\n    }\n  });\n};\n\n/**\n * lazily adds the base router if it has not yet been added.\n *\n * We cannot add the base router in the defaultConfiguration because\n * it reads app settings which might be set after that has run.\n *\n * @private\n */\napp.lazyrouter = function lazyrouter() {\n  if (!this._router) {\n    this._router = new Router({\n      caseSensitive: this.enabled('case sensitive routing'),\n      strict: this.enabled('strict routing')\n    });\n\n    this._router.use(query(this.get('query parser fn')));\n    this._router.use(middleware.init(this));\n  }\n};\n\n/**\n * Dispatch a req, res pair into the application. Starts pipeline processing.\n *\n * If no callback is provided, then default error handlers will respond\n * in the event of an error bubbling through the stack.\n *\n * @private\n */\n\napp.handle = function handle(req, res, callback) {\n  var router = this._router;\n\n  // final handler\n  var done = callback || finalhandler(req, res, {\n    env: this.get('env'),\n    onerror: logerror.bind(this)\n  });\n\n  // no routes\n  if (!router) {\n    debug('no routes defined on app');\n    done();\n    return;\n  }\n\n  router.handle(req, res, done);\n};\n\n/**\n * Proxy `Router#use()` to add middleware to the app router.\n * See Router#use() documentation for details.\n *\n * If the _fn_ parameter is an express app, then it will be\n * mounted at the _route_ specified.\n *\n * @public\n */\n\napp.use = function use(fn) {\n  var offset = 0;\n  var path = '/';\n\n  // default path to '/'\n  // disambiguate app.use([fn])\n  if (typeof fn !== 'function') {\n    var arg = fn;\n\n    while (Array.isArray(arg) && arg.length !== 0) {\n      arg = arg[0];\n    }\n\n    // first arg is the path\n    if (typeof arg !== 'function') {\n      offset = 1;\n      path = fn;\n    }\n  }\n\n  var fns = flatten(slice.call(arguments, offset));\n\n  if (fns.length === 0) {\n    throw new TypeError('app.use() requires a middleware function')\n  }\n\n  // setup router\n  this.lazyrouter();\n  var router = this._router;\n\n  fns.forEach(function (fn) {\n    // non-express app\n    if (!fn || !fn.handle || !fn.set) {\n      return router.use(path, fn);\n    }\n\n    debug('.use app under %s', path);\n    fn.mountpath = path;\n    fn.parent = this;\n\n    // restore .app property on req and res\n    router.use(path, function mounted_app(req, res, next) {\n      var orig = req.app;\n      fn.handle(req, res, function (err) {\n        setPrototypeOf(req, orig.request)\n        setPrototypeOf(res, orig.response)\n        next(err);\n      });\n    });\n\n    // mounted an app\n    fn.emit('mount', this);\n  }, this);\n\n  return this;\n};\n\n/**\n * Proxy to the app `Router#route()`\n * Returns a new `Route` instance for the _path_.\n *\n * Routes are isolated middleware stacks for specific paths.\n * See the Route api docs for details.\n *\n * @public\n */\n\napp.route = function route(path) {\n  this.lazyrouter();\n  return this._router.route(path);\n};\n\n/**\n * Register the given template engine callback `fn`\n * as `ext`.\n *\n * By default will `require()` the engine based on the\n * file extension. For example if you try to render\n...\n", "needle_token_start": 13065, "needle_token_end": 13116, "code_context_ntokens": 15415, "output": ["```javascript\nfunction tryStat(path) {\n  debug('stat \"%s\"', path);\n\n  try {\n    return fs.statSync(path);\n  } catch (e) {\n    return undefined;\n  }\n}\n```"]}
{"repo": "expressjs/express", "name": "stringify", "language": "typescript", "path": "lib/response.js", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a JavaScript value into a JSON string. It enhances the standard JSON conversion by allowing optional character escaping to prevent HTML sniffing attacks.\n2. **Input**: This function accepts four parameters: a value to be converted, an optional replacer function for altering how object values are stringified, an optional space value for adding indentation to the output, and a boolean to indicate whether certain characters should be escaped.\n3. **Output**: It returns a JSON string representation of the input value. If the escape parameter is true, specific characters within the JSON string are replaced with their Unicode escape sequences to enhance security.\n4. **Procedure**: Initially, the function checks if the replacer or space parameters are provided to use an enhanced JSON.stringify method. After converting the value to a JSON string, it conditionally checks if escaping is needed. If so, it replaces less-than, greater-than, and ampersand characters with their respective Unicode escape sequences to prevent misuse in HTML contexts.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/utils.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @api private\n */\n\nvar Buffer = require('safe-buffer').Buffer\nvar contentDisposition = require('content-disposition');\nvar contentType = require('content-type');\nvar deprecate = require('depd')('express');\nvar flatten = require('array-flatten');\nvar mime = require('send').mime;\nvar etag = require('etag');\nvar proxyaddr = require('proxy-addr');\nvar qs = require('qs');\nvar querystring = require('querystring');\n\n/**\n * Return strong ETag for `body`.\n *\n * @param {String|Buffer} body\n * @param {String} [encoding]\n * @return {String}\n * @api private\n */\n\nexports.etag = createETagGenerator({ weak: false })\n\n/**\n * Return weak ETag for `body`.\n *\n * @param {String|Buffer} body\n * @param {String} [encoding]\n * @return {String}\n * @api private\n */\n\nexports.wetag = createETagGenerator({ weak: true })\n\n/**\n * Check if `path` looks absolute.\n *\n * @param {String} path\n * @return {Boolean}\n * @api private\n */\n\nexports.isAbsolute = function(path){\n  if ('/' === path[0]) return true;\n  if (':' === path[1] && ('\\\\' === path[2] || '/' === path[2])) return true; // Windows device path\n  if ('\\\\\\\\' === path.substring(0, 2)) return true; // Microsoft Azure absolute path\n};\n\n/**\n * Flatten the given `arr`.\n *\n * @param {Array} arr\n * @return {Array}\n * @api private\n */\n\nexports.flatten = deprecate.function(flatten,\n  'utils.flatten: use array-flatten npm module instead');\n\n/**\n * Normalize the given `type`, for example \"html\" becomes \"text/html\".\n *\n * @param {String} type\n * @return {Object}\n * @api private\n */\n\nexports.normalizeType = function(type){\n  return ~type.indexOf('/')\n    ? acceptParams(type)\n    : { value: mime.lookup(type), params: {} };\n};\n\n/**\n * Normalize `types`, for example \"html\" becomes \"text/html\".\n *\n * @param {Array} types\n * @return {Array}\n * @api private\n */\n\nexports.normalizeTypes = function(types){\n  var ret = [];\n\n  for (var i = 0; i < types.length; ++i) {\n    ret.push(exports.normalizeType(types[i]));\n  }\n\n  return ret;\n};\n\n/**\n * Generate Content-Disposition header appropriate for the filename.\n * non-ascii filenames are urlencoded and a filename* parameter is added\n *\n * @param {String} filename\n * @return {String}\n * @api private\n */\n\nexports.contentDisposition = deprecate.function(contentDisposition,\n  'utils.contentDisposition: use content-disposition npm module instead');\n\n/**\n * Parse accept params `str` returning an\n * object with `.value`, `.quality` and `.params`.\n *\n * @param {String} str\n * @return {Object}\n * @api private\n */\n\nfunction acceptParams (str) {\n  var parts = str.split(/ *; */);\n  var ret = { value: parts[0], quality: 1, params: {} }\n\n  for (var i = 1; i < parts.length; ++i) {\n    var pms = parts[i].split(/ *= */);\n    if ('q' === pms[0]) {\n      ret.quality = parseFloat(pms[1]);\n    } else {\n      ret.params[pms[0]] = pms[1];\n    }\n  }\n\n  return ret;\n}\n\n/**\n * Compile \"etag\" value to function.\n *\n * @param  {Boolean|String|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileETag = function(val) {\n  var fn;\n\n  if (typeof val === 'function') {\n    return val;\n  }\n\n  switch (val) {\n    case true:\n    case 'weak':\n      fn = exports.wetag;\n      break;\n    case false:\n      break;\n    case 'strong':\n      fn = exports.etag;\n      break;\n    default:\n      throw new TypeError('unknown value for etag function: ' + val);\n  }\n\n  return fn;\n}\n\n/**\n * Compile \"query parser\" value to function.\n *\n * @param  {String|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileQueryParser = function compileQueryParser(val) {\n  var fn;\n\n  if (typeof val === 'function') {\n    return val;\n  }\n\n  switch (val) {\n    case true:\n    case 'simple':\n      fn = querystring.parse;\n      break;\n    case false:\n      fn = newObject;\n      break;\n    case 'extended':\n      fn = parseExtendedQueryString;\n      break;\n    default:\n      throw new TypeError('unknown value for query parser function: ' + val);\n  }\n\n  return fn;\n}\n\n/**\n * Compile \"proxy trust\" value to function.\n *\n * @param  {Boolean|String|Number|Array|Function} val\n * @return {Function}\n * @api private\n */\n\nexports.compileTrust = function(val) {\n  if (typeof val === 'function') return val;\n\n  if (val === true) {\n    // Support plain true/false\n    return function(){ return true };\n  }\n\n  if (typeof val === 'number') {\n    // Support trusting hop count\n    return function(a, i){ return i < val };\n  }\n\n  if (typeof val === 'string') {\n    // Support comma-separated values\n    val = val.split(',')\n      .map(function (v) { return v.trim() })\n  }\n\n  return proxyaddr.compile(val || []);\n}\n\n/**\n * Set the charset in a given Content-Type string.\n *\n * @param {String} type\n * @param {String} charset\n * @return {String}\n * @api private\n */\n\nexports.setCharset = function setCharset(type, charset) {\n  if (!type || !charset) {\n    return type;\n  }\n\n  // parse type\n  var parsed = contentType.parse(type);\n\n  // set charset\n  parsed.parameters.charset = charset;\n\n  // format type\n  return contentType.format(parsed);\n};\n\n/**\n * Create an ETag generator function, generating ETags with\n * the given options.\n *\n * @param {object} options\n * @return {function}\n * @private\n */\n\nfunction createETagGenerator (options) {\n  return function generateETag (body, encoding) {\n    var buf = !Buffer.isBuffer(body)\n      ? Buffer.from(body, encoding)\n      : body\n\n    return etag(buf, options)\n  }\n}\n\n/**\n * Parse an extended query string with qs.\n *\n * @param {String} str\n * @return {Object}\n * @private\n */\n\nfunction parseExtendedQueryString(str) {\n  return qs.parse(str, {\n    allowPrototypes: true\n  });\n}\n\n/**\n * Return new empty object.\n *\n * @return {Object}\n * @api private\n */\n\nfunction newObject() {\n  return {};\n}\n\n// Path: lib/response.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar Buffer = require('safe-buffer').Buffer\nvar contentDisposition = require('content-disposition');\nvar createError = require('http-errors')\nvar deprecate = require('depd')('express');\nvar encodeUrl = require('encodeurl');\nvar escapeHtml = require('escape-html');\nvar http = require('http');\nvar isAbsolute = require('./utils').isAbsolute;\nvar onFinished = require('on-finished');\nvar path = require('path');\nvar statuses = require('statuses')\nvar merge = require('utils-merge');\nvar sign = require('cookie-signature').sign;\nvar normalizeType = require('./utils').normalizeType;\nvar normalizeTypes = require('./utils').normalizeTypes;\nvar setCharset = require('./utils').setCharset;\nvar cookie = require('cookie');\nvar send = require('send');\nvar extname = path.extname;\nvar mime = send.mime;\nvar resolve = path.resolve;\nvar vary = require('vary');\n\n/**\n * Response prototype.\n * @public\n */\n\nvar res = Object.create(http.ServerResponse.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = res\n\n/**\n * Module variables.\n * @private\n */\n\nvar charsetRegExp = /;\\s*charset\\s*=/;\nvar schemaAndHostRegExp = /^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?\\/\\/[^\\\\\\/\\?]+/;\n\n/**\n * Set status `code`.\n *\n * @param {Number} code\n * @return {ServerResponse}\n * @public\n */\n\nres.status = function status(code) {\n  if ((typeof code === 'string' || Math.floor(code) !== code) && code > 99 && code < 1000) {\n    deprecate('res.status(' + JSON.stringify(code) + '): use res.status(' + Math.floor(code) + ') instead')\n  }\n  this.statusCode = code;\n  return this;\n};\n\n/**\n * Set Link header field with the given `links`.\n *\n * Examples:\n *\n *    res.links({\n *      next: 'http://api.example.com/users?page=2',\n *      last: 'http://api.example.com/users?page=5'\n *    });\n *\n * @param {Object} links\n * @return {ServerResponse}\n * @public\n */\n\nres.links = function(links){\n  var link = this.get('Link') || '';\n  if (link) link += ', ';\n  return this.set('Link', link + Object.keys(links).map(function(rel){\n    return '<' + links[rel] + '>; rel=\"' + rel + '\"';\n  }).join(', '));\n};\n\n/**\n * Send a response.\n *\n * Examples:\n *\n *     res.send(Buffer.from('wahoo'));\n *     res.send({ some: 'json' });\n *     res.send('<p>some html</p>');\n *\n * @param {string|number|boolean|object|Buffer} body\n * @public\n */\n\nres.send = function send(body) {\n  var chunk = body;\n  var encoding;\n  var req = this.req;\n  var type;\n\n  // settings\n  var app = this.app;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.send(body, status) backwards compat\n    if (typeof arguments[0] !== 'number' && typeof arguments[1] === 'number') {\n      deprecate('res.send(body, status): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.send(status, body): Use res.status(status).send(body) instead');\n      this.statusCode = arguments[0];\n      chunk = arguments[1];\n    }\n  }\n\n  // disambiguate res.send(status) and res.send(status, num)\n  if (typeof chunk === 'number' && arguments.length === 1) {\n    // res.send(status) will set status message as text string\n    if (!this.get('Content-Type')) {\n      this.type('txt');\n    }\n\n    deprecate('res.send(status): Use res.sendStatus(status) instead');\n    this.statusCode = chunk;\n    chunk = statuses.message[chunk]\n  }\n\n  switch (typeof chunk) {\n    // string defaulting to html\n    case 'string':\n      if (!this.get('Content-Type')) {\n        this.type('html');\n      }\n      break;\n    case 'boolean':\n    case 'number':\n    case 'object':\n      if (chunk === null) {\n        chunk = '';\n      } else if (Buffer.isBuffer(chunk)) {\n        if (!this.get('Content-Type')) {\n          this.type('bin');\n        }\n      } else {\n        return this.json(chunk);\n      }\n      break;\n  }\n\n  // write strings in utf-8\n  if (typeof chunk === 'string') {\n    encoding = 'utf8';\n    type = this.get('Content-Type');\n\n    // reflect this in content-type\n    if (typeof type === 'string') {\n      this.set('Content-Type', setCharset(type, 'utf-8'));\n    }\n  }\n\n  // determine if ETag should be generated\n  var etagFn = app.get('etag fn')\n  var generateETag = !this.get('ETag') && typeof etagFn === 'function'\n\n  // populate Content-Length\n  var len\n  if (chunk !== undefined) {\n    if (Buffer.isBuffer(chunk)) {\n      // get length of Buffer\n      len = chunk.length\n    } else if (!generateETag && chunk.length < 1000) {\n      // just calculate length when no ETag + small chunk\n      len = Buffer.byteLength(chunk, encoding)\n    } else {\n      // convert chunk to Buffer and calculate\n      chunk = Buffer.from(chunk, encoding)\n      encoding = undefined;\n      len = chunk.length\n    }\n\n    this.set('Content-Length', len);\n  }\n\n  // populate ETag\n  var etag;\n  if (generateETag && len !== undefined) {\n    if ((etag = etagFn(chunk, encoding))) {\n      this.set('ETag', etag);\n    }\n  }\n\n  // freshness\n  if (req.fresh) this.statusCode = 304;\n\n  // strip irrelevant headers\n  if (204 === this.statusCode || 304 === this.statusCode) {\n    this.removeHeader('Content-Type');\n    this.removeHeader('Content-Length');\n    this.removeHeader('Transfer-Encoding');\n    chunk = '';\n  }\n\n  // alter headers for 205\n  if (this.statusCode === 205) {\n    this.set('Content-Length', '0')\n    this.removeHeader('Transfer-Encoding')\n    chunk = ''\n  }\n\n  if (req.method === 'HEAD') {\n    // skip body for HEAD\n    this.end();\n  } else {\n    // respond\n    this.end(chunk, encoding);\n  }\n\n  return this;\n};\n\n/**\n * Send JSON response.\n *\n * Examples:\n *\n *     res.json(null);\n *     res.json({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.json = function json(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.json(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.json(obj, status): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.json(status, obj): Use res.status(status).json(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('Content-Type', 'application/json');\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send JSON response with JSONP callback support.\n *\n * Examples:\n *\n *     res.jsonp(null);\n *     res.jsonp({ user: 'tj' });\n *\n * @param {string|number|boolean|object} obj\n * @public\n */\n\nres.jsonp = function jsonp(obj) {\n  var val = obj;\n\n  // allow status / body\n  if (arguments.length === 2) {\n    // res.jsonp(body, status) backwards compat\n    if (typeof arguments[1] === 'number') {\n      deprecate('res.jsonp(obj, status): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[1];\n    } else {\n      deprecate('res.jsonp(status, obj): Use res.status(status).jsonp(obj) instead');\n      this.statusCode = arguments[0];\n      val = arguments[1];\n    }\n  }\n\n  // settings\n  var app = this.app;\n  var escape = app.get('json escape')\n  var replacer = app.get('json replacer');\n  var spaces = app.get('json spaces');\n  var body = stringify(val, replacer, spaces, escape)\n  var callback = this.req.query[app.get('jsonp callback name')];\n\n  // content-type\n  if (!this.get('Content-Type')) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'application/json');\n  }\n\n  // fixup callback\n  if (Array.isArray(callback)) {\n    callback = callback[0];\n  }\n\n  // jsonp\n  if (typeof callback === 'string' && callback.length !== 0) {\n    this.set('X-Content-Type-Options', 'nosniff');\n    this.set('Content-Type', 'text/javascript');\n\n    // restrict callback charset\n    callback = callback.replace(/[^\\[\\]\\w$.]/g, '');\n\n    if (body === undefined) {\n      // empty argument\n      body = ''\n    } else if (typeof body === 'string') {\n      // replace chars not allowed in JavaScript that are in JSON\n      body = body\n        .replace(/\\u2028/g, '\\\\u2028')\n        .replace(/\\u2029/g, '\\\\u2029')\n    }\n\n    // the /**/ is a specific security mitigation for \"Rosetta Flash JSONP abuse\"\n    // the typeof check is just to reduce client error noise\n    body = '/**/ typeof ' + callback + ' === \\'function\\' && ' + callback + '(' + body + ');';\n  }\n\n  return this.send(body);\n};\n\n/**\n * Send given HTTP status code.\n *\n * Sets the response status to `statusCode` and the body of the\n * response to the standard description from node's http.STATUS_CODES\n * or the statusCode number if no description.\n *\n * Examples:\n *\n *     res.sendStatus(200);\n *\n * @param {number} statusCode\n * @public\n */\n\nres.sendStatus = function sendStatus(statusCode) {\n  var body = statuses.message[statusCode] || String(statusCode)\n\n  this.statusCode = statusCode;\n  this.type('txt');\n\n  return this.send(body);\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendFile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendFile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendFile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendFile = function sendFile(path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  if (!path) {\n    throw new TypeError('path argument is required to res.sendFile');\n  }\n\n  if (typeof path !== 'string') {\n    throw new TypeError('path must be a string to res.sendFile')\n  }\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  if (!opts.root && !isAbsolute(path)) {\n    throw new TypeError('path must be absolute or specify root to res.sendFile');\n  }\n\n  // create file stream\n  var pathname = encodeURI(path);\n  var file = send(req, pathname, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\n/**\n * Transfer the file at the given `path`.\n *\n * Automatically sets the _Content-Type_ response header field.\n * The callback `callback(err)` is invoked when the transfer is complete\n * or when an error occurs. Be sure to check `res.headersSent`\n * if you wish to attempt responding, as the header and some data\n * may have already been transferred.\n *\n * Options:\n *\n *   - `maxAge`   defaulting to 0 (can be string converted by `ms`)\n *   - `root`     root directory for relative filenames\n *   - `headers`  object of headers to serve with file\n *   - `dotfiles` serve dotfiles, defaulting to false; can be `\"allow\"` to send them\n *\n * Other options are passed along to `send`.\n *\n * Examples:\n *\n *  The following example illustrates how `res.sendfile()` may\n *  be used as an alternative for the `static()` middleware for\n *  dynamic situations. The code backing `res.sendfile()` is actually\n *  the same code, so HTTP cache support etc is identical.\n *\n *     app.get('/user/:uid/photos/:file', function(req, res){\n *       var uid = req.params.uid\n *         , file = req.params.file;\n *\n *       req.user.mayViewFilesFrom(uid, function(yes){\n *         if (yes) {\n *           res.sendfile('/uploads/' + uid + '/' + file);\n *         } else {\n *           res.send(403, 'Sorry! you cant see that.');\n *         }\n *       });\n *     });\n *\n * @public\n */\n\nres.sendfile = function (path, options, callback) {\n  var done = callback;\n  var req = this.req;\n  var res = this;\n  var next = req.next;\n  var opts = options || {};\n\n  // support function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // create file stream\n  var file = send(req, path, opts);\n\n  // transfer\n  sendfile(res, file, opts, function (err) {\n    if (done) return done(err);\n    if (err && err.code === 'EISDIR') return next();\n\n    // next() all but write errors\n    if (err && err.code !== 'ECONNABORTED' && err.syscall !== 'write') {\n      next(err);\n    }\n  });\n};\n\nres.sendfile = deprecate.function(res.sendfile,\n  'res.sendfile: Use res.sendFile instead');\n\n/**\n * Transfer the file at the given `path` as an attachment.\n *\n * Optionally providing an alternate attachment `filename`,\n * and optional callback `callback(err)`. The callback is invoked\n * when the data transfer is complete, or when an error has\n * occurred. Be sure to check `res.headersSent` if you plan to respond.\n *\n * Optionally providing an `options` object to use with `res.sendFile()`.\n * This function will set the `Content-Disposition` header, overriding\n * any `Content-Disposition` header passed as header options in order\n * to set the attachment and filename.\n *\n * This method uses `res.sendFile()`.\n *\n * @public\n */\n\nres.download = function download (path, filename, options, callback) {\n  var done = callback;\n  var name = filename;\n  var opts = options || null\n\n  // support function as second or third arg\n  if (typeof filename === 'function') {\n    done = filename;\n    name = null;\n    opts = null\n  } else if (typeof options === 'function') {\n    done = options\n    opts = null\n  }\n\n  // support optional filename, where options may be in it's place\n  if (typeof filename === 'object' &&\n    (typeof options === 'function' || options === undefined)) {\n    name = null\n    opts = filename\n  }\n\n  // set Content-Disposition when file is sent\n  var headers = {\n    'Content-Disposition': contentDisposition(name || path)\n  };\n\n  // merge user-provided headers\n  if (opts && opts.headers) {\n    var keys = Object.keys(opts.headers)\n    for (var i = 0; i < keys.length; i++) {\n      var key = keys[i]\n      if (key.toLowerCase() !== 'content-disposition') {\n        headers[key] = opts.headers[key]\n      }\n    }\n  }\n\n  // merge user-provided options\n  opts = Object.create(opts)\n  opts.headers = headers\n\n  // Resolve the full path for sendFile\n  var fullPath = !opts.root\n    ? resolve(path)\n    : path\n\n  // send file\n  return this.sendFile(fullPath, opts, done)\n};\n\n/**\n * Set _Content-Type_ response header with `type` through `mime.lookup()`\n * when it does not contain \"/\", or set the Content-Type to `type` otherwise.\n *\n * Examples:\n *\n *     res.type('.html');\n *     res.type('html');\n *     res.type('json');\n *     res.type('application/json');\n *     res.type('png');\n *\n * @param {String} type\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.contentType =\nres.type = function contentType(type) {\n  var ct = type.indexOf('/') === -1\n    ? mime.lookup(type)\n    : type;\n\n  return this.set('Content-Type', ct);\n};\n\n/**\n * Respond to the Acceptable formats using an `obj`\n * of mime-type callbacks.\n *\n * This method uses `req.accepted`, an array of\n * acceptable types ordered by their quality values.\n * When \"Accept\" is not present the _first_ callback\n * is invoked, otherwise the first match is used. When\n * no match is performed the server responds with\n * 406 \"Not Acceptable\".\n *\n * Content-Type is set for you, however if you choose\n * you may alter this within the callback using `res.type()`\n * or `res.set('Content-Type', ...)`.\n *\n *    res.format({\n *      'text/plain': function(){\n *        res.send('hey');\n *      },\n *\n *      'text/html': function(){\n *        res.send('<p>hey</p>');\n *      },\n *\n *      'application/json': function () {\n *        res.send({ message: 'hey' });\n *      }\n *    });\n *\n * In addition to canonicalized MIME types you may\n * also use extnames mapped to these types:\n *\n *    res.format({\n *      text: function(){\n *        res.send('hey');\n *      },\n *\n *      html: function(){\n *        res.send('<p>hey</p>');\n *      },\n *\n *      json: function(){\n *        res.send({ message: 'hey' });\n *      }\n *    });\n *\n * By default Express passes an `Error`\n * with a `.status` of 406 to `next(err)`\n * if a match is not made. If you provide\n * a `.default` callback it will be invoked\n * instead.\n *\n * @param {Object} obj\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.format = function(obj){\n  var req = this.req;\n  var next = req.next;\n\n  var keys = Object.keys(obj)\n    .filter(function (v) { return v !== 'default' })\n\n  var key = keys.length > 0\n    ? req.accepts(keys)\n    : false;\n\n  this.vary(\"Accept\");\n\n  if (key) {\n    this.set('Content-Type', normalizeType(key).value);\n    obj[key](req, this, next);\n  } else if (obj.default) {\n    obj.default(req, this, next)\n  } else {\n    next(createError(406, {\n      types: normalizeTypes(keys).map(function (o) { return o.value })\n    }))\n  }\n\n  return this;\n};\n\n/**\n * Set _Content-Disposition_ header to _attachment_ with optional `filename`.\n *\n * @param {String} filename\n * @return {ServerResponse}\n * @public\n */\n\nres.attachment = function attachment(filename) {\n  if (filename) {\n    this.type(extname(filename));\n  }\n\n  this.set('Content-Disposition', contentDisposition(filename));\n\n  return this;\n};\n\n/**\n * Append additional header `field` with value `val`.\n *\n * Example:\n *\n *    res.append('Link', ['<http://localhost/>', '<http://localhost:3000/>']);\n *    res.append('Set-Cookie', 'foo=bar; Path=/; HttpOnly');\n *    res.append('Warning', '199 Miscellaneous warning');\n *\n * @param {String} field\n * @param {String|Array} val\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.append = function append(field, val) {\n  var prev = this.get(field);\n  var value = val;\n\n  if (prev) {\n    // concat the new and prev vals\n    value = Array.isArray(prev) ? prev.concat(val)\n      : Array.isArray(val) ? [prev].concat(val)\n        : [prev, val]\n  }\n\n  return this.set(field, value);\n};\n\n/**\n * Set header `field` to `val`, or pass\n * an object of header fields.\n *\n * Examples:\n *\n *    res.set('Foo', ['bar', 'baz']);\n *    res.set('Accept', 'application/json');\n *    res.set({ Accept: 'text/plain', 'X-API-Key': 'tobi' });\n *\n * Aliased as `res.header()`.\n *\n * @param {String|Object} field\n * @param {String|Array} val\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.set =\nres.header = function header(field, val) {\n  if (arguments.length === 2) {\n    var value = Array.isArray(val)\n      ? val.map(String)\n      : String(val);\n\n    // add charset to content-type\n    if (field.toLowerCase() === 'content-type') {\n      if (Array.isArray(value)) {\n        throw new TypeError('Content-Type cannot be set to an Array');\n      }\n      if (!charsetRegExp.test(value)) {\n        var charset = mime.charsets.lookup(value.split(';')[0]);\n        if (charset) value += '; charset=' + charset.toLowerCase();\n      }\n    }\n\n    this.setHeader(field, value);\n  } else {\n    for (var key in field) {\n      this.set(key, field[key]);\n    }\n  }\n  return this;\n};\n\n/**\n * Get value for header `field`.\n *\n * @param {String} field\n * @return {String}\n * @public\n */\n\nres.get = function(field){\n  return this.getHeader(field);\n};\n\n/**\n * Clear cookie `name`.\n *\n * @param {String} name\n * @param {Object} [options]\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.clearCookie = function clearCookie(name, options) {\n  var opts = merge({ expires: new Date(1), path: '/' }, options);\n\n  return this.cookie(name, '', opts);\n};\n\n/**\n * Set cookie `name` to `value`, with the given `options`.\n *\n * Options:\n *\n *    - `maxAge`   max-age in milliseconds, converted to `expires`\n *    - `signed`   sign the cookie\n *    - `path`     defaults to \"/\"\n *\n * Examples:\n *\n *    // \"Remember Me\" for 15 minutes\n *    res.cookie('rememberme', '1', { expires: new Date(Date.now() + 900000), httpOnly: true });\n *\n *    // same as above\n *    res.cookie('rememberme', '1', { maxAge: 900000, httpOnly: true })\n *\n * @param {String} name\n * @param {String|Object} value\n * @param {Object} [options]\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.cookie = function (name, value, options) {\n  var opts = merge({}, options);\n  var secret = this.req.secret;\n  var signed = opts.signed;\n\n  if (signed && !secret) {\n    throw new Error('cookieParser(\"secret\") required for signed cookies');\n  }\n\n  var val = typeof value === 'object'\n    ? 'j:' + JSON.stringify(value)\n    : String(value);\n\n  if (signed) {\n    val = 's:' + sign(val, secret);\n  }\n\n  if (opts.maxAge != null) {\n    var maxAge = opts.maxAge - 0\n\n    if (!isNaN(maxAge)) {\n      opts.expires = new Date(Date.now() + maxAge)\n      opts.maxAge = Math.floor(maxAge / 1000)\n    }\n  }\n\n  if (opts.path == null) {\n    opts.path = '/';\n  }\n\n  this.append('Set-Cookie', cookie.serialize(name, String(val), opts));\n\n  return this;\n};\n\n/**\n * Set the location header to `url`.\n *\n * The given `url` can also be \"back\", which redirects\n * to the _Referrer_ or _Referer_ headers or \"/\".\n *\n * Examples:\n *\n *    res.location('/foo/bar').;\n *    res.location('http://example.com');\n *    res.location('../login');\n *\n * @param {String} url\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.location = function location(url) {\n  var loc;\n\n  // \"back\" is an alias for the referrer\n  if (url === 'back') {\n    loc = this.req.get('Referrer') || '/';\n  } else {\n    loc = String(url);\n  }\n\n  var m = schemaAndHostRegExp.exec(loc);\n  var pos = m ? m[0].length + 1 : 0;\n\n  // Only encode after host to avoid invalid encoding which can introduce\n  // vulnerabilities (e.g. `\\\\` to `%5C`).\n  loc = loc.slice(0, pos) + encodeUrl(loc.slice(pos));\n\n  return this.set('Location', loc);\n};\n\n/**\n * Redirect to the given `url` with optional response `status`\n * defaulting to 302.\n *\n * The resulting `url` is determined by `res.location()`, so\n * it will play nicely with mounted apps, relative paths,\n * `\"back\"` etc.\n *\n * Examples:\n *\n *    res.redirect('/foo/bar');\n *    res.redirect('http://example.com');\n *    res.redirect(301, 'http://example.com');\n *    res.redirect('../login'); // /blog/post/1 -> /blog/login\n *\n * @public\n */\n\nres.redirect = function redirect(url) {\n  var address = url;\n  var body;\n  var status = 302;\n\n  // allow status / url\n  if (arguments.length === 2) {\n    if (typeof arguments[0] === 'number') {\n      status = arguments[0];\n      address = arguments[1];\n    } else {\n      deprecate('res.redirect(url, status): Use res.redirect(status, url) instead');\n      status = arguments[1];\n    }\n  }\n\n  // Set location header\n  address = this.location(address).get('Location');\n\n  // Support text/{plain,html} by default\n  this.format({\n    text: function(){\n      body = statuses.message[status] + '. Redirecting to ' + address\n    },\n\n    html: function(){\n      var u = escapeHtml(address);\n      body = '<p>' + statuses.message[status] + '. Redirecting to <a href=\"' + u + '\">' + u + '</a></p>'\n    },\n\n    default: function(){\n      body = '';\n    }\n  });\n\n  // Respond\n  this.statusCode = status;\n  this.set('Content-Length', Buffer.byteLength(body));\n\n  if (this.req.method === 'HEAD') {\n    this.end();\n  } else {\n    this.end(body);\n  }\n};\n\n/**\n * Add `field` to Vary. If already present in the Vary set, then\n * this call is simply ignored.\n *\n * @param {Array|String} field\n * @return {ServerResponse} for chaining\n * @public\n */\n\nres.vary = function(field){\n  // checks for back-compat\n  if (!field || (Array.isArray(field) && !field.length)) {\n    deprecate('res.vary(): Provide a field name');\n    return this;\n  }\n\n  vary(this, field);\n\n  return this;\n};\n\n/**\n * Render `view` with the given `options` and optional callback `fn`.\n * When a callback function is given a response will _not_ be made\n * automatically, otherwise a response of _200_ and _text/html_ is given.\n *\n * Options:\n *\n *  - `cache`     boolean hinting to the engine it should cache\n *  - `filename`  filename of the view being rendered\n *\n * @public\n */\n\nres.render = function render(view, options, callback) {\n  var app = this.req.app;\n  var done = callback;\n  var opts = options || {};\n  var req = this.req;\n  var self = this;\n\n  // support callback function as second arg\n  if (typeof options === 'function') {\n    done = options;\n    opts = {};\n  }\n\n  // merge res.locals\n  opts._locals = self.locals;\n\n  // default callback to respond\n  done = done || function (err, str) {\n    if (err) return req.next(err);\n    self.send(str);\n  };\n\n  // render\n  app.render(view, opts, done);\n};\n\n// pipe the send file stream\nfunction sendfile(res, file, options, callback) {\n  var done = false;\n  var streaming;\n\n  // request aborted\n  function onaborted() {\n    if (done) return;\n    done = true;\n\n    var err = new Error('Request aborted');\n    err.code = 'ECONNABORTED';\n    callback(err);\n  }\n\n  // directory\n  function ondirectory() {\n    if (done) return;\n    done = true;\n\n    var err = new Error('EISDIR, read');\n    err.code = 'EISDIR';\n    callback(err);\n  }\n\n  // errors\n  function onerror(err) {\n    if (done) return;\n    done = true;\n    callback(err);\n  }\n\n  // ended\n  function onend() {\n    if (done) return;\n    done = true;\n    callback();\n  }\n\n  // file\n  function onfile() {\n    streaming = false;\n  }\n\n  // finished\n  function onfinish(err) {\n    if (err && err.code === 'ECONNRESET') return onaborted();\n    if (err) return onerror(err);\n    if (done) return;\n\n    setImmediate(function () {\n      if (streaming !== false && !done) {\n        onaborted();\n        return;\n      }\n\n      if (done) return;\n      done = true;\n      callback();\n    });\n  }\n\n  // streaming\n  function onstream() {\n    streaming = true;\n  }\n\n  file.on('directory', ondirectory);\n  file.on('end', onend);\n  file.on('error', onerror);\n  file.on('file', onfile);\n  file.on('stream', onstream);\n  onFinished(res, onfinish);\n\n  if (options.headers) {\n    // set headers on successful transfer\n    file.on('headers', function headers(res) {\n      var obj = options.headers;\n      var keys = Object.keys(obj);\n\n      for (var i = 0; i < keys.length; i++) {\n        var k = keys[i];\n        res.setHeader(k, obj[k]);\n      }\n    });\n  }\n\n  // pipe\n  file.pipe(res);\n}\n\n/**\n * Stringify JSON, like JSON.stringify, but v8 optimized, with the\n * ability to escape characters that can trigger HTML sniffing.\n *\n * @param {*} value\n * @param {function} replacer\n * @param {number} spaces\n * @param {boolean} escape\n * @returns {string}\n * @private\n */\n\n\nfunction stringify (value, replacer, spaces, escape) {\n  // v8 checks arguments.length for optimizing simple call\n  // https://bugs.chromium.org/p/v8/issues/detail?id=4730\n  var json = replacer || spaces\n    ? JSON.stringify(value, replacer, spaces)\n    : JSON.stringify(value);\n\n  if (escape && typeof json === 'string') {\n    json = json.replace(/[<>&]/g, function (c) {\n      switch (c.charCodeAt(0)) {\n        case 0x3c:\n          return '\\\\u003c'\n        case 0x3e:\n          return '\\\\u003e'\n        case 0x26:\n          return '\\\\u0026'\n        /* istanbul ignore next: unreachable default */\n        default:\n          return c\n      }\n    })\n  }\n\n  return json\n}\n\n// Path: lib/request.js\n/*!\n * express\n * Copyright(c) 2009-2013 TJ Holowaychuk\n * Copyright(c) 2013 Roman Shtylman\n * Copyright(c) 2014-2015 Douglas Christopher Wilson\n * MIT Licensed\n */\n\n'use strict';\n\n/**\n * Module dependencies.\n * @private\n */\n\nvar accepts = require('accepts');\nvar deprecate = require('depd')('express');\nvar isIP = require('net').isIP;\nvar typeis = require('type-is');\nvar http = require('http');\nvar fresh = require('fresh');\nvar parseRange = require('range-parser');\nvar parse = require('parseurl');\nvar proxyaddr = require('proxy-addr');\n\n/**\n * Request prototype.\n * @public\n */\n\nvar req = Object.create(http.IncomingMessage.prototype)\n\n/**\n * Module exports.\n * @public\n */\n\nmodule.exports = req\n\n/**\n * Return request header.\n *\n * The `Referrer` header field is special-cased,\n * both `Referrer` and `Referer` are interchangeable.\n *\n * Examples:\n *\n *     req.get('Content-Type');\n *     // => \"text/plain\"\n *\n *     req.get('content-type');\n *     // => \"text/plain\"\n *\n *     req.get('Something');\n *     // => undefined\n *\n * Aliased as `req.header()`.\n *\n * @param {String} name\n * @return {String}\n * @public\n */\n\nreq.get =\nreq.header = function header(name) {\n  if (!name) {\n    throw new TypeError('name argument is required to req.get');\n  }\n\n  if (typeof name !== 'string') {\n    throw new TypeError('name must be a string to req.get');\n  }\n\n  var lc = name.toLowerCase();\n\n  switch (lc) {\n    case 'referer':\n    case 'referrer':\n      return this.headers.referrer\n        || this.headers.referer;\n    default:\n      return this.headers[lc];\n  }\n};\n\n/**\n * To do: update docs.\n *\n * Check if the given `type(s)` is acceptable, returning\n * the best match when true, otherwise `undefined`, in which\n * case you should respond with 406 \"Not Acceptable\".\n *\n * The `type` value may be a single MIME type string\n * such as \"application/json\", an extension name\n...\n", "needle_token_start": 11043, "needle_token_end": 11276, "code_context_ntokens": 11914, "output": ["```javascript\n/**\n * Stringify JSON, like JSON.stringify, but v8 optimized, with the\n * ability to escape characters that can trigger HTML sniffing.\n *\n * @param {*} value\n * @param {function} replacer\n * @param {number} spaces\n * @param {boolean} escape\n * @returns {string}\n * @private\n */\nfunction stringify (value, replacer, spaces, escape) {\n  // v8 checks arguments.length for optimizing simple call\n  // https://bugs.chromium.org/p/v8/issues/detail?id=4730\n  var json = replacer || spaces\n    ? JSON.stringify(value, replacer, spaces)\n    : JSON.stringify(value);\n\n  if (escape && typeof json === 'string') {\n    json = json.replace(/[<>&]/g, function (c) {\n      switch (c.charCodeAt(0)) {\n        case 0x3c:\n          return '\\\\u003c'\n        case 0x3e:\n          return '\\\\u003e'\n        case 0x26:\n          return '\\\\u0026'\n        /* istanbul ignore next: unreachable default */\n        default:\n          return c\n      }\n    })\n  }\n\n  return json\n}\n```"]}
{"repo": "axios/axios", "name": "done", "language": "typescript", "path": "lib/adapters/xhr.js", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to clean up and release resources or subscriptions that are no longer needed once a network request has been completed or cancelled.\n2. **Input**: The function does not take any direct inputs as it operates within the scope of its enclosing function, utilizing the configuration object of the network request.\n3. **Output**: There is no return value; the function's role is solely to perform cleanup operations.\n4. **Procedure**: The function checks if there are any active subscriptions or event listeners related to cancellation tokens or abort signals. If present, it unsubscribes from the cancellation token and removes the event listener from the abort signal, effectively detaching the request operation from these mechanisms.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/helpers/isURLSameOrigin.js\n'use strict';\n\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n// Standard browser envs have full support of the APIs needed to test\n// whether the request URL is of the same origin as current location.\n  (function standardBrowserEnv() {\n    const msie = /(msie|trident)/i.test(navigator.userAgent);\n    const urlParsingNode = document.createElement('a');\n    let originURL;\n\n    /**\n    * Parse a URL to discover its components\n    *\n    * @param {String} url The URL to be parsed\n    * @returns {Object}\n    */\n    function resolveURL(url) {\n      let href = url;\n\n      if (msie) {\n        // IE needs attribute set twice to normalize properties\n...\n// Path: lib/adapters/xhr.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport cookies from './../helpers/cookies.js';\nimport buildURL from './../helpers/buildURL.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport isURLSameOrigin from './../helpers/isURLSameOrigin.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport parseProtocol from '../helpers/parseProtocol.js';\nimport platform from '../platform/index.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport speedometer from '../helpers/speedometer.js';\n\nfunction progressEventReducer(listener, isDownloadStream) {\n  let bytesNotified = 0;\n  const _speedometer = speedometer(50, 250);\n\n  return e => {\n    const loaded = e.loaded;\n    const total = e.lengthComputable ? e.total : undefined;\n    const progressBytes = loaded - bytesNotified;\n    const rate = _speedometer(progressBytes);\n    const inRange = loaded <= total;\n\n    bytesNotified = loaded;\n\n    const data = {\n      loaded,\n      total,\n      progress: total ? (loaded / total) : undefined,\n      bytes: progressBytes,\n      rate: rate ? rate : undefined,\n      estimated: rate && total && inRange ? (total - loaded) / rate : undefined,\n      event: e\n    };\n\n    data[isDownloadStream ? 'download' : 'upload'] = true;\n\n    listener(data);\n  };\n}\n\nconst isXHRAdapterSupported = typeof XMLHttpRequest !== 'undefined';\n\nexport default isXHRAdapterSupported && function (config) {\n  return new Promise(function dispatchXhrRequest(resolve, reject) {\n    let requestData = config.data;\n    const requestHeaders = AxiosHeaders.from(config.headers).normalize();\n    let {responseType, withXSRFToken} = config;\n    let onCanceled;\n    \nfunction done() {\n      if (config.cancelToken) {\n        config.cancelToken.unsubscribe(onCanceled);\n      }\n\n      if (config.signal) {\n        config.signal.removeEventListener('abort', onCanceled);\n      }\n    }\n\n    let contentType;\n\n    if (utils.isFormData(requestData)) {\n      if (platform.hasStandardBrowserEnv || platform.hasStandardBrowserWebWorkerEnv) {\n        requestHeaders.setContentType(false); // Let the browser set it\n      } else if ((contentType = requestHeaders.getContentType()) !== false) {\n        // fix semicolon duplication issue for ReactNative FormData implementation\n        const [type, ...tokens] = contentType ? contentType.split(';').map(token => token.trim()).filter(Boolean) : [];\n        requestHeaders.setContentType([type || 'multipart/form-data', ...tokens].join('; '));\n      }\n    }\n\n    let request = new XMLHttpRequest();\n\n    // HTTP basic authentication\n    if (config.auth) {\n      const username = config.auth.username || '';\n      const password = config.auth.password ? unescape(encodeURIComponent(config.auth.password)) : '';\n      requestHeaders.set('Authorization', 'Basic ' + btoa(username + ':' + password));\n    }\n\n    const fullPath = buildFullPath(config.baseURL, config.url);\n\n    request.open(config.method.toUpperCase(), buildURL(fullPath, config.params, config.paramsSerializer), true);\n\n    // Set the request timeout in MS\n    request.timeout = config.timeout;\n\n    function onloadend() {\n      if (!request) {\n        return;\n      }\n      // Prepare the response\n      const responseHeaders = AxiosHeaders.from(\n        'getAllResponseHeaders' in request && request.getAllResponseHeaders()\n      );\n      const responseData = !responseType || responseType === 'text' || responseType === 'json' ?\n        request.responseText : request.response;\n      const response = {\n        data: responseData,\n        status: request.status,\n        statusText: request.statusText,\n        headers: responseHeaders,\n        config,\n        request\n      };\n\n      settle(function _resolve(value) {\n        resolve(value);\n        done();\n      }, function _reject(err) {\n        reject(err);\n        done();\n      }, response);\n\n      // Clean up request\n      request = null;\n    }\n\n    if ('onloadend' in request) {\n      // Use onloadend if available\n      request.onloadend = onloadend;\n    } else {\n      // Listen for ready state to emulate onloadend\n      request.onreadystatechange = function handleLoad() {\n        if (!request || request.readyState !== 4) {\n          return;\n        }\n\n        // The request errored out and we didn't get a response, this will be\n        // handled by onerror instead\n        // With one exception: request that using file: protocol, most browsers\n        // will return status as 0 even though it's a successful request\n        if (request.status === 0 && !(request.responseURL && request.responseURL.indexOf('file:') === 0)) {\n          return;\n        }\n        // readystate handler is calling before onerror or ontimeout handlers,\n        // so we should call onloadend on the next 'tick'\n        setTimeout(onloadend);\n      };\n    }\n\n    // Handle browser request cancellation (as opposed to a manual cancellation)\n    request.onabort = function handleAbort() {\n      if (!request) {\n        return;\n      }\n\n      reject(new AxiosError('Request aborted', AxiosError.ECONNABORTED, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle low level network errors\n    request.onerror = function handleError() {\n      // Real errors are hidden from us by the browser\n      // onerror should only fire if it's a network error\n      reject(new AxiosError('Network Error', AxiosError.ERR_NETWORK, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle timeout\n    request.ontimeout = function handleTimeout() {\n      let timeoutErrorMessage = config.timeout ? 'timeout of ' + config.timeout + 'ms exceeded' : 'timeout exceeded';\n      const transitional = config.transitional || transitionalDefaults;\n      if (config.timeoutErrorMessage) {\n        timeoutErrorMessage = config.timeoutErrorMessage;\n      }\n      reject(new AxiosError(\n        timeoutErrorMessage,\n        transitional.clarifyTimeoutError ? AxiosError.ETIMEDOUT : AxiosError.ECONNABORTED,\n        config,\n        request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Add xsrf header\n    // This is only done if running in a standard browser environment.\n    // Specifically not if we're in a web worker, or react-native.\n    if(platform.hasStandardBrowserEnv) {\n      withXSRFToken && utils.isFunction(withXSRFToken) && (withXSRFToken = withXSRFToken(config));\n\n      if (withXSRFToken || (withXSRFToken !== false && isURLSameOrigin(fullPath))) {\n        // Add xsrf header\n        const xsrfValue = config.xsrfHeaderName && config.xsrfCookieName && cookies.read(config.xsrfCookieName);\n\n        if (xsrfValue) {\n          requestHeaders.set(config.xsrfHeaderName, xsrfValue);\n        }\n      }\n    }\n\n    // Remove Content-Type if data is undefined\n    requestData === undefined && requestHeaders.setContentType(null);\n\n    // Add headers to the request\n    if ('setRequestHeader' in request) {\n      utils.forEach(requestHeaders.toJSON(), function setRequestHeader(val, key) {\n        request.setRequestHeader(key, val);\n      });\n    }\n\n    // Add withCredentials to request if needed\n    if (!utils.isUndefined(config.withCredentials)) {\n      request.withCredentials = !!config.withCredentials;\n    }\n\n    // Add responseType to request if needed\n    if (responseType && responseType !== 'json') {\n      request.responseType = config.responseType;\n    }\n\n    // Handle progress if needed\n    if (typeof config.onDownloadProgress === 'function') {\n      request.addEventListener('progress', progressEventReducer(config.onDownloadProgress, true));\n    }\n\n    // Not all browsers support upload events\n    if (typeof config.onUploadProgress === 'function' && request.upload) {\n      request.upload.addEventListener('progress', progressEventReducer(config.onUploadProgress));\n    }\n\n    if (config.cancelToken || config.signal) {\n      // Handle cancellation\n      // eslint-disable-next-line func-names\n      onCanceled = cancel => {\n        if (!request) {\n          return;\n        }\n        reject(!cancel || cancel.type ? new CanceledError(null, config, request) : cancel);\n        request.abort();\n        request = null;\n      };\n\n      config.cancelToken && config.cancelToken.subscribe(onCanceled);\n      if (config.signal) {\n        config.signal.aborted ? onCanceled() : config.signal.addEventListener('abort', onCanceled);\n      }\n    }\n\n    const protocol = parseProtocol(fullPath);\n\n    if (protocol && platform.protocols.indexOf(protocol) === -1) {\n      reject(new AxiosError('Unsupported protocol ' + protocol + ':', AxiosError.ERR_BAD_REQUEST, config));\n      return;\n    }\n\n\n    // Send the request\n    request.send(requestData || null);\n  });\n}\n\n// Path: lib/helpers/fromDataURI.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport parseProtocol from './parseProtocol.js';\nimport platform from '../platform/index.js';\n\nconst DATA_URL_PATTERN = /^(?:([^;]+);)?(?:[^;]+;)?(base64|),([\\s\\S]*)$/;\n\n/**\n * Parse data uri to a Buffer or Blob\n *\n * @param {String} uri\n * @param {?Boolean} asBlob\n * @param {?Object} options\n * @param {?Function} options.Blob\n *\n * @returns {Buffer|Blob}\n */\nexport default function fromDataURI(uri, asBlob, options) {\n  const _Blob = options && options.Blob || platform.classes.Blob;\n  const protocol = parseProtocol(uri);\n\n  if (asBlob === undefined && _Blob) {\n    asBlob = true;\n  }\n\n  if (protocol === 'data') {\n    uri = protocol.length ? uri.slice(protocol.length + 1) : uri;\n\n    const match = DATA_URL_PATTERN.exec(uri);\n\n    if (!match) {\n      throw new AxiosError('Invalid URL', AxiosError.ERR_INVALID_URL);\n    }\n\n    const mime = match[1];\n    const isBase64 = match[2];\n    const body = match[3];\n    const buffer = Buffer.from(decodeURIComponent(body), isBase64 ? 'base64' : 'utf8');\n\n    if (asBlob) {\n      if (!_Blob) {\n        throw new AxiosError('Blob is not supported', AxiosError.ERR_NOT_SUPPORT);\n      }\n\n      return new _Blob([buffer], {type: mime});\n    }\n\n    return buffer;\n  }\n\n  throw new AxiosError('Unsupported protocol ' + protocol, AxiosError.ERR_NOT_SUPPORT);\n}\n\n// Path: lib/helpers/readBlob.js\nconst {asyncIterator} = Symbol;\n\nconst readBlob = async function* (blob) {\n  if (blob.stream) {\n    yield* blob.stream()\n  } else if (blob.arrayBuffer) {\n    yield await blob.arrayBuffer()\n  } else if (blob[asyncIterator]) {\n    yield* blob[asyncIterator]();\n  } else {\n    yield blob;\n  }\n}\n\nexport default readBlob;\n\n// Path: lib/helpers/formDataToStream.js\nimport {TextEncoder} from 'util';\nimport {Readable} from 'stream';\nimport utils from \"../utils.js\";\nimport readBlob from \"./readBlob.js\";\n\nconst BOUNDARY_ALPHABET = utils.ALPHABET.ALPHA_DIGIT + '-_';\n\nconst textEncoder = new TextEncoder();\n\nconst CRLF = '\\r\\n';\nconst CRLF_BYTES = textEncoder.encode(CRLF);\nconst CRLF_BYTES_COUNT = 2;\n\nclass FormDataPart {\n  constructor(name, value) {\n    const {escapeName} = this.constructor;\n    const isStringValue = utils.isString(value);\n\n    let headers = `Content-Disposition: form-data; name=\"${escapeName(name)}\"${\n      !isStringValue && value.name ? `; filename=\"${escapeName(value.name)}\"` : ''\n    }${CRLF}`;\n\n    if (isStringValue) {\n      value = textEncoder.encode(String(value).replace(/\\r?\\n|\\r\\n?/g, CRLF));\n    } else {\n      headers += `Content-Type: ${value.type || \"application/octet-stream\"}${CRLF}`\n    }\n\n    this.headers = textEncoder.encode(headers + CRLF);\n\n    this.contentLength = isStringValue ? value.byteLength : value.size;\n\n    this.size = this.headers.byteLength + this.contentLength + CRLF_BYTES_COUNT;\n\n    this.name = name;\n    this.value = value;\n  }\n\n  async *encode(){\n    yield this.headers;\n\n    const {value} = this;\n\n    if(utils.isTypedArray(value)) {\n      yield value;\n    } else {\n      yield* readBlob(value);\n    }\n\n    yield CRLF_BYTES;\n  }\n\n  static escapeName(name) {\n      return String(name).replace(/[\\r\\n\"]/g, (match) => ({\n        '\\r' : '%0D',\n        '\\n' : '%0A',\n        '\"' : '%22',\n      }[match]));\n  }\n}\n\nconst formDataToStream = (form, headersHandler, options) => {\n  const {\n    tag = 'form-data-boundary',\n    size = 25,\n    boundary = tag + '-' + utils.generateString(size, BOUNDARY_ALPHABET)\n  } = options || {};\n\n  if(!utils.isFormData(form)) {\n    throw TypeError('FormData instance required');\n  }\n\n  if (boundary.length < 1 || boundary.length > 70) {\n    throw Error('boundary must be 10-70 characters long')\n  }\n\n  const boundaryBytes = textEncoder.encode('--' + boundary + CRLF);\n  const footerBytes = textEncoder.encode('--' + boundary + '--' + CRLF + CRLF);\n  let contentLength = footerBytes.byteLength;\n\n  const parts = Array.from(form.entries()).map(([name, value]) => {\n    const part = new FormDataPart(name, value);\n    contentLength += part.size;\n    return part;\n  });\n\n  contentLength += boundaryBytes.byteLength * parts.length;\n\n  contentLength = utils.toFiniteNumber(contentLength);\n\n  const computedHeaders = {\n    'Content-Type': `multipart/form-data; boundary=${boundary}`\n  }\n\n  if (Number.isFinite(contentLength)) {\n    computedHeaders['Content-Length'] = contentLength;\n  }\n\n  headersHandler && headersHandler(computedHeaders);\n\n  return Readable.from((async function *() {\n    for(const part of parts) {\n      yield boundaryBytes;\n      yield* part.encode();\n    }\n\n    yield footerBytes;\n  })());\n};\n\nexport default formDataToStream;\n\n// Path: lib/helpers/callbackify.js\nimport utils from \"../utils.js\";\n\nconst callbackify = (fn, reducer) => {\n  return utils.isAsyncFn(fn) ? function (...args) {\n    const cb = args.pop();\n    fn.apply(this, args).then((value) => {\n      try {\n        reducer ? cb(null, ...reducer(value)) : cb(null, value);\n      } catch (err) {\n        cb(err);\n      }\n    }, cb);\n  } : fn;\n}\n\nexport default callbackify;\n\n// Path: lib/helpers/ZlibHeaderTransformStream.js\n\"use strict\";\n\nimport stream from \"stream\";\n\nclass ZlibHeaderTransformStream extends stream.Transform {\n  __transform(chunk, encoding, callback) {\n    this.push(chunk);\n    callback();\n  }\n\n  _transform(chunk, encoding, callback) {\n    if (chunk.length !== 0) {\n      this._transform = this.__transform;\n\n      // Add Default Compression headers if no zlib headers are present\n      if (chunk[0] !== 120) { // Hex: 78\n        const header = Buffer.alloc(2);\n        header[0] = 120; // Hex: 78\n        header[1] = 156; // Hex: 9C \n        this.push(header, encoding);\n      }\n    }\n\n    this.__transform(chunk, encoding, callback);\n  }\n}\n\nexport default ZlibHeaderTransformStream;\n\n// Path: lib/helpers/throttle.js\n'use strict';\n\n/**\n * Throttle decorator\n * @param {Function} fn\n * @param {Number} freq\n * @return {Function}\n */\nfunction throttle(fn, freq) {\n  let timestamp = 0;\n  const threshold = 1000 / freq;\n  let timer = null;\n  return function throttled(force, args) {\n    const now = Date.now();\n    if (force || now - timestamp > threshold) {\n      if (timer) {\n        clearTimeout(timer);\n        timer = null;\n      }\n      timestamp = now;\n      return fn.apply(null, args);\n    }\n    if (!timer) {\n      timer = setTimeout(() => {\n        timer = null;\n        timestamp = Date.now();\n        return fn.apply(null, args);\n      }, threshold - (now - timestamp));\n    }\n  };\n}\n\nexport default throttle;\n\n// Path: lib/helpers/AxiosTransformStream.js\n'use strict';\n\nimport stream from 'stream';\nimport utils from '../utils.js';\nimport throttle from './throttle.js';\nimport speedometer from './speedometer.js';\n\nconst kInternals = Symbol('internals');\n\nclass AxiosTransformStream extends stream.Transform{\n  constructor(options) {\n    options = utils.toFlatObject(options, {\n      maxRate: 0,\n      chunkSize: 64 * 1024,\n      minChunkSize: 100,\n      timeWindow: 500,\n      ticksRate: 2,\n      samplesCount: 15\n    }, null, (prop, source) => {\n      return !utils.isUndefined(source[prop]);\n    });\n\n    super({\n      readableHighWaterMark: options.chunkSize\n    });\n\n    const self = this;\n\n    const internals = this[kInternals] = {\n      length: options.length,\n      timeWindow: options.timeWindow,\n      ticksRate: options.ticksRate,\n      chunkSize: options.chunkSize,\n      maxRate: options.maxRate,\n      minChunkSize: options.minChunkSize,\n      bytesSeen: 0,\n      isCaptured: false,\n      notifiedBytesLoaded: 0,\n      ts: Date.now(),\n      bytes: 0,\n      onReadCallback: null\n    };\n\n    const _speedometer = speedometer(internals.ticksRate * options.samplesCount, internals.timeWindow);\n\n    this.on('newListener', event => {\n      if (event === 'progress') {\n        if (!internals.isCaptured) {\n          internals.isCaptured = true;\n        }\n      }\n    });\n\n    let bytesNotified = 0;\n\n    internals.updateProgress = throttle(function throttledHandler() {\n      const totalBytes = internals.length;\n      const bytesTransferred = internals.bytesSeen;\n      const progressBytes = bytesTransferred - bytesNotified;\n      if (!progressBytes || self.destroyed) return;\n\n      const rate = _speedometer(progressBytes);\n\n      bytesNotified = bytesTransferred;\n\n      process.nextTick(() => {\n        self.emit('progress', {\n          'loaded': bytesTransferred,\n          'total': totalBytes,\n          'progress': totalBytes ? (bytesTransferred / totalBytes) : undefined,\n          'bytes': progressBytes,\n          'rate': rate ? rate : undefined,\n          'estimated': rate && totalBytes && bytesTransferred <= totalBytes ?\n            (totalBytes - bytesTransferred) / rate : undefined\n        });\n      });\n    }, internals.ticksRate);\n\n    const onFinish = () => {\n      internals.updateProgress(true);\n    };\n\n    this.once('end', onFinish);\n    this.once('error', onFinish);\n  }\n\n  _read(size) {\n    const internals = this[kInternals];\n\n    if (internals.onReadCallback) {\n      internals.onReadCallback();\n    }\n\n    return super._read(size);\n  }\n\n  _transform(chunk, encoding, callback) {\n    const self = this;\n    const internals = this[kInternals];\n    const maxRate = internals.maxRate;\n\n    const readableHighWaterMark = this.readableHighWaterMark;\n\n    const timeWindow = internals.timeWindow;\n\n    const divider = 1000 / timeWindow;\n    const bytesThreshold = (maxRate / divider);\n    const minChunkSize = internals.minChunkSize !== false ? Math.max(internals.minChunkSize, bytesThreshold * 0.01) : 0;\n\n    function pushChunk(_chunk, _callback) {\n      const bytes = Buffer.byteLength(_chunk);\n      internals.bytesSeen += bytes;\n      internals.bytes += bytes;\n\n      if (internals.isCaptured) {\n        internals.updateProgress();\n      }\n\n      if (self.push(_chunk)) {\n        process.nextTick(_callback);\n      } else {\n        internals.onReadCallback = () => {\n          internals.onReadCallback = null;\n          process.nextTick(_callback);\n        };\n      }\n    }\n\n    const transformChunk = (_chunk, _callback) => {\n      const chunkSize = Buffer.byteLength(_chunk);\n      let chunkRemainder = null;\n      let maxChunkSize = readableHighWaterMark;\n      let bytesLeft;\n      let passed = 0;\n\n      if (maxRate) {\n        const now = Date.now();\n\n        if (!internals.ts || (passed = (now - internals.ts)) >= timeWindow) {\n          internals.ts = now;\n          bytesLeft = bytesThreshold - internals.bytes;\n          internals.bytes = bytesLeft < 0 ? -bytesLeft : 0;\n          passed = 0;\n        }\n\n        bytesLeft = bytesThreshold - internals.bytes;\n      }\n\n      if (maxRate) {\n        if (bytesLeft <= 0) {\n          // next time window\n          return setTimeout(() => {\n            _callback(null, _chunk);\n          }, timeWindow - passed);\n        }\n\n        if (bytesLeft < maxChunkSize) {\n          maxChunkSize = bytesLeft;\n        }\n      }\n\n      if (maxChunkSize && chunkSize > maxChunkSize && (chunkSize - maxChunkSize) > minChunkSize) {\n        chunkRemainder = _chunk.subarray(maxChunkSize);\n        _chunk = _chunk.subarray(0, maxChunkSize);\n      }\n\n      pushChunk(_chunk, chunkRemainder ? () => {\n        process.nextTick(_callback, null, chunkRemainder);\n      } : _callback);\n    };\n\n    transformChunk(chunk, function transformNextChunk(err, _chunk) {\n      if (err) {\n        return callback(err);\n      }\n\n      if (_chunk) {\n        transformChunk(_chunk, transformNextChunk);\n      } else {\n        callback(null);\n      }\n    });\n  }\n\n  setLength(length) {\n    this[kInternals].length = +length;\n    return this;\n  }\n}\n\nexport default AxiosTransformStream;\n\n// Path: lib/env/data.js\nexport const VERSION = \"1.6.8\";\n// Path: lib/adapters/http.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport buildURL from './../helpers/buildURL.js';\nimport {getProxyForUrl} from 'proxy-from-env';\nimport http from 'http';\nimport https from 'https';\nimport util from 'util';\nimport followRedirects from 'follow-redirects';\nimport zlib from 'zlib';\nimport {VERSION} from '../env/data.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport platform from '../platform/index.js';\nimport fromDataURI from '../helpers/fromDataURI.js';\nimport stream from 'stream';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport AxiosTransformStream from '../helpers/AxiosTransformStream.js';\nimport {EventEmitter} from 'events';\nimport formDataToStream from \"../helpers/formDataToStream.js\";\nimport readBlob from \"../helpers/readBlob.js\";\nimport ZlibHeaderTransformStream from '../helpers/ZlibHeaderTransformStream.js';\nimport callbackify from \"../helpers/callbackify.js\";\n\nconst zlibOptions = {\n  flush: zlib.constants.Z_SYNC_FLUSH,\n  finishFlush: zlib.constants.Z_SYNC_FLUSH\n};\n\nconst brotliOptions = {\n  flush: zlib.constants.BROTLI_OPERATION_FLUSH,\n  finishFlush: zlib.constants.BROTLI_OPERATION_FLUSH\n}\n\nconst isBrotliSupported = utils.isFunction(zlib.createBrotliDecompress);\n\nconst {http: httpFollow, https: httpsFollow} = followRedirects;\n\nconst isHttps = /https:?/;\n\nconst supportedProtocols = platform.protocols.map(protocol => {\n  return protocol + ':';\n});\n\n/**\n * If the proxy or config beforeRedirects functions are defined, call them with the options\n * object.\n *\n * @param {Object<string, any>} options - The options object that was passed to the request.\n *\n * @returns {Object<string, any>}\n */\nfunction dispatchBeforeRedirect(options, responseDetails) {\n  if (options.beforeRedirects.proxy) {\n    options.beforeRedirects.proxy(options);\n  }\n  if (options.beforeRedirects.config) {\n    options.beforeRedirects.config(options, responseDetails);\n  }\n}\n\n/**\n * If the proxy or config afterRedirects functions are defined, call them with the options\n *\n * @param {http.ClientRequestArgs} options\n * @param {AxiosProxyConfig} configProxy configuration from Axios options object\n * @param {string} location\n *\n * @returns {http.ClientRequestArgs}\n */\nfunction setProxy(options, configProxy, location) {\n  let proxy = configProxy;\n  if (!proxy && proxy !== false) {\n    const proxyUrl = getProxyForUrl(location);\n    if (proxyUrl) {\n      proxy = new URL(proxyUrl);\n    }\n  }\n  if (proxy) {\n    // Basic proxy authorization\n    if (proxy.username) {\n      proxy.auth = (proxy.username || '') + ':' + (proxy.password || '');\n    }\n\n    if (proxy.auth) {\n      // Support proxy auth object form\n      if (proxy.auth.username || proxy.auth.password) {\n        proxy.auth = (proxy.auth.username || '') + ':' + (proxy.auth.password || '');\n      }\n      const base64 = Buffer\n        .from(proxy.auth, 'utf8')\n        .toString('base64');\n      options.headers['Proxy-Authorization'] = 'Basic ' + base64;\n    }\n\n    options.headers.host = options.hostname + (options.port ? ':' + options.port : '');\n    const proxyHost = proxy.hostname || proxy.host;\n    options.hostname = proxyHost;\n    // Replace 'host' since options is not a URL object\n    options.host = proxyHost;\n    options.port = proxy.port;\n    options.path = location;\n    if (proxy.protocol) {\n      options.protocol = proxy.protocol.includes(':') ? proxy.protocol : `${proxy.protocol}:`;\n    }\n  }\n\n  options.beforeRedirects.proxy = function beforeRedirect(redirectOptions) {\n    // Configure proxy for redirected request, passing the original config proxy to apply\n    // the exact same logic as if the redirected request was performed by axios directly.\n    setProxy(redirectOptions, configProxy, redirectOptions.href);\n  };\n}\n\nconst isHttpAdapterSupported = typeof process !== 'undefined' && utils.kindOf(process) === 'process';\n\n// temporary hotfix\n\nconst wrapAsync = (asyncExecutor) => {\n  return new Promise((resolve, reject) => {\n    let onDone;\n    let isDone;\n\n    const done = (value, isRejected) => {\n      if (isDone) return;\n      isDone = true;\n      onDone && onDone(value, isRejected);\n    }\n\n    const _resolve = (value) => {\n      done(value);\n      resolve(value);\n    };\n\n    const _reject = (reason) => {\n      done(reason, true);\n      reject(reason);\n    }\n\n    asyncExecutor(_resolve, _reject, (onDoneHandler) => (onDone = onDoneHandler)).catch(_reject);\n  })\n};\n\nconst resolveFamily = ({address, family}) => {\n  if (!utils.isString(address)) {\n    throw TypeError('address must be a string');\n  }\n  return ({\n    address,\n    family: family || (address.indexOf('.') < 0 ? 6 : 4)\n  });\n}\n\nconst buildAddressEntry = (address, family) => resolveFamily(utils.isObject(address) ? address : {address, family});\n\n/*eslint consistent-return:0*/\nexport default isHttpAdapterSupported && function httpAdapter(config) {\n  return wrapAsync(async function dispatchHttpRequest(resolve, reject, onDone) {\n    let {data, lookup, family} = config;\n    const {responseType, responseEncoding} = config;\n    const method = config.method.toUpperCase();\n    let isDone;\n    let rejected = false;\n    let req;\n\n    if (lookup) {\n      const _lookup = callbackify(lookup, (value) => utils.isArray(value) ? value : [value]);\n      // hotfix to support opt.all option which is required for node 20.x\n      lookup = (hostname, opt, cb) => {\n        _lookup(hostname, opt, (err, arg0, arg1) => {\n          if (err) {\n            return cb(err);\n          }\n\n          const addresses = utils.isArray(arg0) ? arg0.map(addr => buildAddressEntry(addr)) : [buildAddressEntry(arg0, arg1)];\n\n          opt.all ? cb(err, addresses) : cb(err, addresses[0].address, addresses[0].family);\n        });\n      }\n    }\n\n    // temporary internal emitter until the AxiosRequest class will be implemented\n    const emitter = new EventEmitter();\n\n    const onFinished = () => {\n      if (config.cancelToken) {\n        config.cancelToken.unsubscribe(abort);\n      }\n\n      if (config.signal) {\n        config.signal.removeEventListener('abort', abort);\n      }\n\n      emitter.removeAllListeners();\n    }\n\n    onDone((value, isRejected) => {\n      isDone = true;\n      if (isRejected) {\n        rejected = true;\n        onFinished();\n      }\n    });\n\n    function abort(reason) {\n      emitter.emit('abort', !reason || reason.type ? new CanceledError(null, config, req) : reason);\n    }\n\n    emitter.once('abort', reject);\n\n    if (config.cancelToken || config.signal) {\n      config.cancelToken && config.cancelToken.subscribe(abort);\n      if (config.signal) {\n        config.signal.aborted ? abort() : config.signal.addEventListener('abort', abort);\n      }\n    }\n\n    // Parse url\n    const fullPath = buildFullPath(config.baseURL, config.url);\n    const parsed = new URL(fullPath, 'http://localhost');\n    const protocol = parsed.protocol || supportedProtocols[0];\n\n    if (protocol === 'data:') {\n      let convertedData;\n\n      if (method !== 'GET') {\n        return settle(resolve, reject, {\n          status: 405,\n          statusText: 'method not allowed',\n          headers: {},\n          config\n        });\n      }\n\n      try {\n        convertedData = fromDataURI(config.url, responseType === 'blob', {\n          Blob: config.env && config.env.Blob\n        });\n      } catch (err) {\n        throw AxiosError.from(err, AxiosError.ERR_BAD_REQUEST, config);\n      }\n\n      if (responseType === 'text') {\n        convertedData = convertedData.toString(responseEncoding);\n\n        if (!responseEncoding || responseEncoding === 'utf8') {\n          convertedData = utils.stripBOM(convertedData);\n        }\n      } else if (responseType === 'stream') {\n        convertedData = stream.Readable.from(convertedData);\n      }\n\n      return settle(resolve, reject, {\n        data: convertedData,\n        status: 200,\n        statusText: 'OK',\n        headers: new AxiosHeaders(),\n        config\n      });\n    }\n\n    if (supportedProtocols.indexOf(protocol) === -1) {\n      return reject(new AxiosError(\n        'Unsupported protocol ' + protocol,\n        AxiosError.ERR_BAD_REQUEST,\n        config\n      ));\n    }\n\n    const headers = AxiosHeaders.from(config.headers).normalize();\n\n    // Set User-Agent (required by some servers)\n    // See https://github.com/axios/axios/issues/69\n    // User-Agent is specified; handle case where no UA header is desired\n    // Only set header if it hasn't been set in config\n    headers.set('User-Agent', 'axios/' + VERSION, false);\n\n    const onDownloadProgress = config.onDownloadProgress;\n    const onUploadProgress = config.onUploadProgress;\n    const maxRate = config.maxRate;\n    let maxUploadRate = undefined;\n    let maxDownloadRate = undefined;\n\n    // support for spec compliant FormData objects\n    if (utils.isSpecCompliantForm(data)) {\n      const userBoundary = headers.getContentType(/boundary=([-_\\w\\d]{10,70})/i);\n\n      data = formDataToStream(data, (formHeaders) => {\n        headers.set(formHeaders);\n      }, {\n        tag: `axios-${VERSION}-boundary`,\n        boundary: userBoundary && userBoundary[1] || undefined\n      });\n      // support for https://www.npmjs.com/package/form-data api\n    } else if (utils.isFormData(data) && utils.isFunction(data.getHeaders)) {\n      headers.set(data.getHeaders());\n\n      if (!headers.hasContentLength()) {\n        try {\n          const knownLength = await util.promisify(data.getLength).call(data);\n          Number.isFinite(knownLength) && knownLength >= 0 && headers.setContentLength(knownLength);\n          /*eslint no-empty:0*/\n        } catch (e) {\n        }\n      }\n    } else if (utils.isBlob(data)) {\n      data.size && headers.setContentType(data.type || 'application/octet-stream');\n      headers.setContentLength(data.size || 0);\n      data = stream.Readable.from(readBlob(data));\n    } else if (data && !utils.isStream(data)) {\n      if (Buffer.isBuffer(data)) {\n        // Nothing to do...\n      } else if (utils.isArrayBuffer(data)) {\n        data = Buffer.from(new Uint8Array(data));\n      } else if (utils.isString(data)) {\n        data = Buffer.from(data, 'utf-8');\n      } else {\n        return reject(new AxiosError(\n          'Data after transformation must be a string, an ArrayBuffer, a Buffer, or a Stream',\n          AxiosError.ERR_BAD_REQUEST,\n          config\n        ));\n      }\n\n      // Add Content-Length header if data exists\n      headers.setContentLength(data.length, false);\n\n      if (config.maxBodyLength > -1 && data.length > config.maxBodyLength) {\n        return reject(new AxiosError(\n          'Request body larger than maxBodyLength limit',\n          AxiosError.ERR_BAD_REQUEST,\n          config\n        ));\n      }\n    }\n\n    const contentLength = utils.toFiniteNumber(headers.getContentLength());\n\n    if (utils.isArray(maxRate)) {\n      maxUploadRate = maxRate[0];\n      maxDownloadRate = maxRate[1];\n    } else {\n      maxUploadRate = maxDownloadRate = maxRate;\n    }\n\n    if (data && (onUploadProgress || maxUploadRate)) {\n      if (!utils.isStream(data)) {\n        data = stream.Readable.from(data, {objectMode: false});\n      }\n\n      data = stream.pipeline([data, new AxiosTransformStream({\n        length: contentLength,\n        maxRate: utils.toFiniteNumber(maxUploadRate)\n      })], utils.noop);\n\n      onUploadProgress && data.on('progress', progress => {\n        onUploadProgress(Object.assign(progress, {\n          upload: true\n        }));\n      });\n    }\n\n    // HTTP basic authentication\n    let auth = undefined;\n    if (config.auth) {\n      const username = config.auth.username || '';\n      const password = config.auth.password || '';\n      auth = username + ':' + password;\n    }\n\n    if (!auth && parsed.username) {\n      const urlUsername = parsed.username;\n      const urlPassword = parsed.password;\n      auth = urlUsername + ':' + urlPassword;\n    }\n\n    auth && headers.delete('authorization');\n\n    let path;\n\n    try {\n      path = buildURL(\n        parsed.pathname + parsed.search,\n        config.params,\n        config.paramsSerializer\n      ).replace(/^\\?/, '');\n    } catch (err) {\n      const customErr = new Error(err.message);\n      customErr.config = config;\n      customErr.url = config.url;\n      customErr.exists = true;\n      return reject(customErr);\n    }\n\n    headers.set(\n      'Accept-Encoding',\n      'gzip, compress, deflate' + (isBrotliSupported ? ', br' : ''), false\n      );\n\n    const options = {\n      path,\n      method: method,\n      headers: headers.toJSON(),\n      agents: { http: config.httpAgent, https: config.httpsAgent },\n      auth,\n      protocol,\n      family,\n      beforeRedirect: dispatchBeforeRedirect,\n      beforeRedirects: {}\n    };\n\n    // cacheable-lookup integration hotfix\n    !utils.isUndefined(lookup) && (options.lookup = lookup);\n\n    if (config.socketPath) {\n      options.socketPath = config.socketPath;\n    } else {\n      options.hostname = parsed.hostname;\n      options.port = parsed.port;\n      setProxy(options, config.proxy, protocol + '//' + parsed.hostname + (parsed.port ? ':' + parsed.port : '') + options.path);\n    }\n\n    let transport;\n    const isHttpsRequest = isHttps.test(options.protocol);\n    options.agent = isHttpsRequest ? config.httpsAgent : config.httpAgent;\n    if (config.transport) {\n      transport = config.transport;\n    } else if (config.maxRedirects === 0) {\n      transport = isHttpsRequest ? https : http;\n    } else {\n      if (config.maxRedirects) {\n        options.maxRedirects = config.maxRedirects;\n      }\n      if (config.beforeRedirect) {\n        options.beforeRedirects.config = config.beforeRedirect;\n      }\n      transport = isHttpsRequest ? httpsFollow : httpFollow;\n    }\n\n    if (config.maxBodyLength > -1) {\n      options.maxBodyLength = config.maxBodyLength;\n    } else {\n      // follow-redirects does not skip comparison, so it should always succeed for axios -1 unlimited\n      options.maxBodyLength = Infinity;\n    }\n\n    if (config.insecureHTTPParser) {\n      options.insecureHTTPParser = config.insecureHTTPParser;\n    }\n\n    // Create the request\n    req = transport.request(options, function handleResponse(res) {\n      if (req.destroyed) return;\n\n      const streams = [res];\n\n      const responseLength = +res.headers['content-length'];\n\n      if (onDownloadProgress) {\n        const transformStream = new AxiosTransformStream({\n          length: utils.toFiniteNumber(responseLength),\n          maxRate: utils.toFiniteNumber(maxDownloadRate)\n        });\n\n        onDownloadProgress && transformStream.on('progress', progress => {\n          onDownloadProgress(Object.assign(progress, {\n            download: true\n          }));\n        });\n\n        streams.push(transformStream);\n      }\n\n      // decompress the response body transparently if required\n      let responseStream = res;\n\n      // return the last request in case of redirects\n      const lastRequest = res.req || req;\n\n      // if decompress disabled we should not decompress\n      if (config.decompress !== false && res.headers['content-encoding']) {\n        // if no content, but headers still say that it is encoded,\n        // remove the header not confuse downstream operations\n        if (method === 'HEAD' || res.statusCode === 204) {\n          delete res.headers['content-encoding'];\n        }\n\n        switch ((res.headers['content-encoding'] || '').toLowerCase()) {\n        /*eslint default-case:0*/\n        case 'gzip':\n        case 'x-gzip':\n        case 'compress':\n        case 'x-compress':\n          // add the unzipper to the body stream processing pipeline\n          streams.push(zlib.createUnzip(zlibOptions));\n\n          // remove the content-encoding in order to not confuse downstream operations\n          delete res.headers['content-encoding'];\n          break;\n        case 'deflate':\n          streams.push(new ZlibHeaderTransformStream());\n\n          // add the unzipper to the body stream processing pipeline\n          streams.push(zlib.createUnzip(zlibOptions));\n\n          // remove the content-encoding in order to not confuse downstream operations\n          delete res.headers['content-encoding'];\n          break;\n        case 'br':\n          if (isBrotliSupported) {\n            streams.push(zlib.createBrotliDecompress(brotliOptions));\n            delete res.headers['content-encoding'];\n          }\n        }\n      }\n\n      responseStream = streams.length > 1 ? stream.pipeline(streams, utils.noop) : streams[0];\n\n      const offListeners = stream.finished(responseStream, () => {\n        offListeners();\n        onFinished();\n      });\n\n      const response = {\n        status: res.statusCode,\n        statusText: res.statusMessage,\n        headers: new AxiosHeaders(res.headers),\n        config,\n        request: lastRequest\n      };\n\n      if (responseType === 'stream') {\n        response.data = responseStream;\n        settle(resolve, reject, response);\n      } else {\n        const responseBuffer = [];\n        let totalResponseBytes = 0;\n\n        responseStream.on('data', function handleStreamData(chunk) {\n          responseBuffer.push(chunk);\n          totalResponseBytes += chunk.length;\n\n          // make sure the content length is not over the maxContentLength if specified\n          if (config.maxContentLength > -1 && totalResponseBytes > config.maxContentLength) {\n            // stream.destroy() emit aborted event before calling reject() on Node.js v16\n            rejected = true;\n            responseStream.destroy();\n            reject(new AxiosError('maxContentLength size of ' + config.maxContentLength + ' exceeded',\n              AxiosError.ERR_BAD_RESPONSE, config, lastRequest));\n          }\n        });\n\n        responseStream.on('aborted', function handlerStreamAborted() {\n          if (rejected) {\n            return;\n          }\n\n          const err = new AxiosError(\n            'maxContentLength size of ' + config.maxContentLength + ' exceeded',\n            AxiosError.ERR_BAD_RESPONSE,\n            config,\n            lastRequest\n          );\n          responseStream.destroy(err);\n          reject(err);\n        });\n\n        responseStream.on('error', function handleStreamError(err) {\n          if (req.destroyed) return;\n          reject(AxiosError.from(err, null, config, lastRequest));\n        });\n\n        responseStream.on('end', function handleStreamEnd() {\n          try {\n            let responseData = responseBuffer.length === 1 ? responseBuffer[0] : Buffer.concat(responseBuffer);\n            if (responseType !== 'arraybuffer') {\n              responseData = responseData.toString(responseEncoding);\n              if (!responseEncoding || responseEncoding === 'utf8') {\n                responseData = utils.stripBOM(responseData);\n              }\n            }\n            response.data = responseData;\n          } catch (err) {\n            return reject(AxiosError.from(err, null, config, response.request, response));\n          }\n          settle(resolve, reject, response);\n        });\n      }\n\n      emitter.once('abort', err => {\n        if (!responseStream.destroyed) {\n          responseStream.emit('error', err);\n          responseStream.destroy();\n        }\n      });\n    });\n\n    emitter.once('abort', err => {\n      reject(err);\n      req.destroy(err);\n    });\n\n    // Handle errors\n    req.on('error', function handleRequestError(err) {\n      // @todo remove\n      // if (req.aborted && err.code !== AxiosError.ERR_FR_TOO_MANY_REDIRECTS) return;\n      reject(AxiosError.from(err, null, config, req));\n    });\n\n    // set tcp keep alive to prevent drop connection by peer\n    req.on('socket', function handleRequestSocket(socket) {\n      // default interval of sending ack packet is 1 minute\n      socket.setKeepAlive(true, 1000 * 60);\n    });\n\n    // Handle request timeout\n    if (config.timeout) {\n      // This is forcing a int timeout to avoid problems if the `req` interface doesn't handle other types.\n      const timeout = parseInt(config.timeout, 10);\n\n      if (Number.isNaN(timeout)) {\n        reject(new AxiosError(\n          'error trying to parse `config.timeout` to int',\n          AxiosError.ERR_BAD_OPTION_VALUE,\n          config,\n          req\n        ));\n\n        return;\n      }\n\n      // Sometime, the response will be very slow, and does not respond, the connect event will be block by event loop system.\n      // And timer callback will be fired, and abort() will be invoked before connection, then get \"socket hang up\" and code ECONNRESET.\n      // At this time, if we have a large number of request, nodejs will hang up some socket on background. and the number will up and up.\n      // And then these socket which be hang up will devouring CPU little by little.\n      // ClientRequest.setTimeout will be fired on the specify milliseconds, and can make sure that abort() will be fired after connect.\n      req.setTimeout(timeout, function handleRequestTimeout() {\n        if (isDone) return;\n        let timeoutErrorMessage = config.timeout ? 'timeout of ' + config.timeout + 'ms exceeded' : 'timeout exceeded';\n        const transitional = config.transitional || transitionalDefaults;\n        if (config.timeoutErrorMessage) {\n          timeoutErrorMessage = config.timeoutErrorMessage;\n        }\n        reject(new AxiosError(\n          timeoutErrorMessage,\n          transitional.clarifyTimeoutError ? AxiosError.ETIMEDOUT : AxiosError.ECONNABORTED,\n          config,\n          req\n        ));\n        abort();\n      });\n    }\n\n\n    // Send the request\n    if (utils.isStream(data)) {\n      let ended = false;\n      let errored = false;\n\n      data.on('end', () => {\n        ended = true;\n      });\n\n      data.once('error', err => {\n        errored = true;\n        req.destroy(err);\n      });\n\n      data.on('close', () => {\n        if (!ended && !errored) {\n          abort(new CanceledError('Request stream has been aborted', config, req));\n        }\n      });\n\n      data.pipe(req);\n    } else {\n      req.end(data);\n    }\n  });\n}\n\nexport const __setProxy = setProxy;\n\n// Path: lib/adapters/adapters.js\nimport utils from '../utils.js';\nimport httpAdapter from './http.js';\nimport xhrAdapter from './xhr.js';\nimport AxiosError from \"../core/AxiosError.js\";\n\nconst knownAdapters = {\n  http: httpAdapter,\n  xhr: xhrAdapter\n}\n\nutils.forEach(knownAdapters, (fn, value) => {\n  if (fn) {\n    try {\n      Object.defineProperty(fn, 'name', {value});\n    } catch (e) {\n      // eslint-disable-next-line no-empty\n    }\n    Object.defineProperty(fn, 'adapterName', {value});\n  }\n});\n\nconst renderReason = (reason) => `- ${reason}`;\n\nconst isResolvedHandle = (adapter) => utils.isFunction(adapter) || adapter === null || adapter === false;\n\nexport default {\n  getAdapter: (adapters) => {\n    adapters = utils.isArray(adapters) ? adapters : [adapters];\n\n    const {length} = adapters;\n    let nameOrAdapter;\n    let adapter;\n\n    const rejectedReasons = {};\n\n    for (let i = 0; i < length; i++) {\n      nameOrAdapter = adapters[i];\n      let id;\n\n      adapter = nameOrAdapter;\n\n      if (!isResolvedHandle(nameOrAdapter)) {\n        adapter = knownAdapters[(id = String(nameOrAdapter)).toLowerCase()];\n\n        if (adapter === undefined) {\n          throw new AxiosError(`Unknown adapter '${id}'`);\n        }\n      }\n\n      if (adapter) {\n        break;\n      }\n\n      rejectedReasons[id || '#' + i] = adapter;\n    }\n\n    if (!adapter) {\n\n      const reasons = Object.entries(rejectedReasons)\n        .map(([id, state]) => `adapter ${id} ` +\n          (state === false ? 'is not supported by the environment' : 'is not available in the build')\n        );\n\n      let s = length ?\n        (reasons.length > 1 ? 'since :\\n' + reasons.map(renderReason).join('\\n') : ' ' + renderReason(reasons[0])) :\n        'as no adapter specified';\n\n      throw new AxiosError(\n        `There is no suitable adapter to dispatch the request ` + s,\n        'ERR_NOT_SUPPORT'\n      );\n    }\n\n    return adapter;\n  },\n  adapters: knownAdapters\n}\n\n// Path: lib/helpers/toURLEncodedForm.js\n'use strict';\n\nimport utils from '../utils.js';\nimport toFormData from './toFormData.js';\nimport platform from '../platform/index.js';\n\nexport default function toURLEncodedForm(data, options) {\n  return toFormData(data, new platform.classes.URLSearchParams(), Object.assign({\n    visitor: function(value, key, path, helpers) {\n      if (platform.isNode && utils.isBuffer(value)) {\n        this.append(key, value.toString('base64'));\n        return false;\n      }\n\n      return helpers.defaultVisitor.apply(this, arguments);\n    }\n  }, options));\n}\n\n// Path: lib/helpers/formDataToJSON.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * It takes a string like `foo[x][y][z]` and returns an array like `['foo', 'x', 'y', 'z']\n *\n * @param {string} name - The name of the property to get.\n *\n * @returns An array of strings.\n */\nfunction parsePropPath(name) {\n  // foo[x][y][z]\n  // foo.x.y.z\n  // foo-x-y-z\n  // foo x y z\n  return utils.matchAll(/\\w+|\\[(\\w*)]/g, name).map(match => {\n    return match[0] === '[]' ? '' : match[1] || match[0];\n  });\n}\n\n/**\n * Convert an array to an object.\n *\n * @param {Array<any>} arr - The array to convert to an object.\n *\n * @returns An object with the same keys and values as the array.\n */\nfunction arrayToObject(arr) {\n  const obj = {};\n  const keys = Object.keys(arr);\n  let i;\n  const len = keys.length;\n  let key;\n  for (i = 0; i < len; i++) {\n    key = keys[i];\n    obj[key] = arr[key];\n  }\n  return obj;\n}\n\n/**\n * It takes a FormData object and returns a JavaScript object\n *\n * @param {string} formData The FormData object to convert to JSON.\n *\n * @returns {Object<string, any> | null} The converted object.\n */\nfunction formDataToJSON(formData) {\n  function buildPath(path, value, target, index) {\n    let name = path[index++];\n\n    if (name === '__proto__') return true;\n\n    const isNumericKey = Number.isFinite(+name);\n    const isLast = index >= path.length;\n    name = !name && utils.isArray(target) ? target.length : name;\n\n    if (isLast) {\n      if (utils.hasOwnProp(target, name)) {\n        target[name] = [target[name], value];\n      } else {\n        target[name] = value;\n      }\n\n      return !isNumericKey;\n    }\n\n    if (!target[name] || !utils.isObject(target[name])) {\n      target[name] = [];\n    }\n\n    const result = buildPath(path, value, target[name], index);\n\n    if (result && utils.isArray(target[name])) {\n      target[name] = arrayToObject(target[name]);\n    }\n\n    return !isNumericKey;\n  }\n\n  if (utils.isFormData(formData) && utils.isFunction(formData.entries)) {\n    const obj = {};\n\n    utils.forEachEntry(formData, (name, value) => {\n      buildPath(parsePropPath(name), value, obj, 0);\n    });\n\n    return obj;\n  }\n\n  return null;\n}\n\nexport default formDataToJSON;\n\n// Path: lib/defaults/index.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\nimport transitionalDefaults from './transitional.js';\nimport toFormData from '../helpers/toFormData.js';\nimport toURLEncodedForm from '../helpers/toURLEncodedForm.js';\nimport platform from '../platform/index.js';\nimport formDataToJSON from '../helpers/formDataToJSON.js';\n\n/**\n * It takes a string, tries to parse it, and if it fails, it returns the stringified version\n * of the input\n *\n * @param {any} rawValue - The value to be stringified.\n * @param {Function} parser - A function that parses a string into a JavaScript object.\n * @param {Function} encoder - A function that takes a value and returns a string.\n *\n * @returns {string} A stringified version of the rawValue.\n */\nfunction stringifySafely(rawValue, parser, encoder) {\n  if (utils.isString(rawValue)) {\n    try {\n      (parser || JSON.parse)(rawValue);\n      return utils.trim(rawValue);\n    } catch (e) {\n      if (e.name !== 'SyntaxError') {\n        throw e;\n      }\n    }\n  }\n\n  return (encoder || JSON.stringify)(rawValue);\n}\n\nconst defaults = {\n\n  transitional: transitionalDefaults,\n\n  adapter: ['xhr', 'http'],\n\n  transformRequest: [function transformRequest(data, headers) {\n    const contentType = headers.getContentType() || '';\n    const hasJSONContentType = contentType.indexOf('application/json') > -1;\n    const isObjectPayload = utils.isObject(data);\n\n    if (isObjectPayload && utils.isHTMLForm(data)) {\n      data = new FormData(data);\n    }\n\n    const isFormData = utils.isFormData(data);\n\n    if (isFormData) {\n      return hasJSONContentType ? JSON.stringify(formDataToJSON(data)) : data;\n    }\n\n    if (utils.isArrayBuffer(data) ||\n      utils.isBuffer(data) ||\n      utils.isStream(data) ||\n      utils.isFile(data) ||\n      utils.isBlob(data)\n    ) {\n      return data;\n    }\n    if (utils.isArrayBufferView(data)) {\n      return data.buffer;\n    }\n    if (utils.isURLSearchParams(data)) {\n      headers.setContentType('application/x-www-form-urlencoded;charset=utf-8', false);\n      return data.toString();\n    }\n\n    let isFileList;\n\n    if (isObjectPayload) {\n      if (contentType.indexOf('application/x-www-form-urlencoded') > -1) {\n        return toURLEncodedForm(data, this.formSerializer).toString();\n      }\n\n      if ((isFileList = utils.isFileList(data)) || contentType.indexOf('multipart/form-data') > -1) {\n        const _FormData = this.env && this.env.FormData;\n\n        return toFormData(\n          isFileList ? {'files[]': data} : data,\n          _FormData && new _FormData(),\n          this.formSerializer\n        );\n      }\n    }\n\n    if (isObjectPayload || hasJSONContentType ) {\n      headers.setContentType('application/json', false);\n      return stringifySafely(data);\n    }\n\n    return data;\n  }],\n\n  transformResponse: [function transformResponse(data) {\n    const transitional = this.transitional || defaults.transitional;\n    const forcedJSONParsing = transitional && transitional.forcedJSONParsing;\n    const JSONRequested = this.responseType === 'json';\n\n    if (data && utils.isString(data) && ((forcedJSONParsing && !this.responseType) || JSONRequested)) {\n      const silentJSONParsing = transitional && transitional.silentJSONParsing;\n      const strictJSONParsing = !silentJSONParsing && JSONRequested;\n\n      try {\n        return JSON.parse(data);\n      } catch (e) {\n        if (strictJSONParsing) {\n          if (e.name === 'SyntaxError') {\n            throw AxiosError.from(e, AxiosError.ERR_BAD_RESPONSE, this, null, this.response);\n          }\n          throw e;\n        }\n      }\n    }\n\n    return data;\n  }],\n\n  /**\n   * A timeout in milliseconds to abort a request. If set to 0 (default) a\n   * timeout is not created.\n   */\n  timeout: 0,\n\n  xsrfCookieName: 'XSRF-TOKEN',\n  xsrfHeaderName: 'X-XSRF-TOKEN',\n\n  maxContentLength: -1,\n  maxBodyLength: -1,\n\n  env: {\n    FormData: platform.classes.FormData,\n    Blob: platform.classes.Blob\n  },\n\n  validateStatus: function validateStatus(status) {\n    return status >= 200 && status < 300;\n  },\n\n  headers: {\n    common: {\n      'Accept': 'application/json, text/plain, */*',\n      'Content-Type': undefined\n    }\n  }\n};\n\nutils.forEach(['delete', 'get', 'head', 'post', 'put', 'patch'], (method) => {\n  defaults.headers[method] = {};\n});\n\nexport default defaults;\n\n// Path: lib/core/transformData.js\n'use strict';\n\nimport utils from './../utils.js';\nimport defaults from '../defaults/index.js';\n...\n", "needle_token_start": 774, "needle_token_end": 839, "code_context_ntokens": 16004, "output": ["```javascript\nfunction done() {\n  if (config.cancelToken) {\n    config.cancelToken.unsubscribe(onCanceled);\n  }\n\n  if (config.signal) {\n    config.signal.removeEventListener('abort', onCanceled);\n  }\n}\n```"]}
{"repo": "axios/axios", "name": "formatMessage", "language": "typescript", "path": "lib/helpers/validator.js", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to construct a formatted warning message regarding the use of transitional options in a library, incorporating the library version and additional details about the option's deprecation.\n2. **Input**: It accepts three parameters: the name of the transitional option, a description of the option's status (whether it's deprecated or removed), and an optional message providing further details.\n3. **Output**: Returns a string that combines the library version, the transitional option's name, its status, and any additional information provided.\n4. **Procedure**: The function constructs a message string by concatenating the library version, the name of the transitional option, its status description, and an optional additional message. This results in a comprehensive warning message that informs the user about the transitional option's current status.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/defaults/index.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\nimport transitionalDefaults from './transitional.js';\nimport toFormData from '../helpers/toFormData.js';\nimport toURLEncodedForm from '../helpers/toURLEncodedForm.js';\nimport platform from '../platform/index.js';\nimport formDataToJSON from '../helpers/formDataToJSON.js';\n\n/**\n * It takes a string, tries to parse it, and if it fails, it returns the stringified version\n * of the input\n *\n * @param {any} rawValue - The value to be stringified.\n * @param {Function} parser - A function that parses a string into a JavaScript object.\n * @param {Function} encoder - A function that takes a value and returns a string.\n *\n * @returns {string} A stringified version of the rawValue.\n */\nfunction stringifySafely(rawValue, parser, encoder) {\n  if (utils.isString(rawValue)) {\n    try {\n      (parser || JSON.parse)(rawValue);\n      return utils.trim(rawValue);\n    } catch (e) {\n      if (e.name !== 'SyntaxError') {\n        throw e;\n      }\n    }\n  }\n\n  return (encoder || JSON.stringify)(rawValue);\n}\n\nconst defaults = {\n\n  transitional: transitionalDefaults,\n\n  adapter: ['xhr', 'http'],\n\n  transformRequest: [function transformRequest(data, headers) {\n    const contentType = headers.getContentType() || '';\n    const hasJSONContentType = contentType.indexOf('application/json') > -1;\n    const isObjectPayload = utils.isObject(data);\n\n    if (isObjectPayload && utils.isHTMLForm(data)) {\n      data = new FormData(data);\n    }\n\n    const isFormData = utils.isFormData(data);\n\n    if (isFormData) {\n      return hasJSONContentType ? JSON.stringify(formDataToJSON(data)) : data;\n    }\n\n    if (utils.isArrayBuffer(data) ||\n      utils.isBuffer(data) ||\n      utils.isStream(data) ||\n      utils.isFile(data) ||\n      utils.isBlob(data)\n    ) {\n      return data;\n    }\n    if (utils.isArrayBufferView(data)) {\n      return data.buffer;\n    }\n    if (utils.isURLSearchParams(data)) {\n      headers.setContentType('application/x-www-form-urlencoded;charset=utf-8', false);\n      return data.toString();\n    }\n\n    let isFileList;\n\n    if (isObjectPayload) {\n      if (contentType.indexOf('application/x-www-form-urlencoded') > -1) {\n        return toURLEncodedForm(data, this.formSerializer).toString();\n      }\n\n      if ((isFileList = utils.isFileList(data)) || contentType.indexOf('multipart/form-data') > -1) {\n        const _FormData = this.env && this.env.FormData;\n\n        return toFormData(\n          isFileList ? {'files[]': data} : data,\n          _FormData && new _FormData(),\n          this.formSerializer\n        );\n      }\n    }\n\n    if (isObjectPayload || hasJSONContentType ) {\n      headers.setContentType('application/json', false);\n      return stringifySafely(data);\n    }\n\n    return data;\n  }],\n\n  transformResponse: [function transformResponse(data) {\n    const transitional = this.transitional || defaults.transitional;\n    const forcedJSONParsing = transitional && transitional.forcedJSONParsing;\n    const JSONRequested = this.responseType === 'json';\n\n    if (data && utils.isString(data) && ((forcedJSONParsing && !this.responseType) || JSONRequested)) {\n      const silentJSONParsing = transitional && transitional.silentJSONParsing;\n      const strictJSONParsing = !silentJSONParsing && JSONRequested;\n\n      try {\n        return JSON.parse(data);\n      } catch (e) {\n        if (strictJSONParsing) {\n          if (e.name === 'SyntaxError') {\n            throw AxiosError.from(e, AxiosError.ERR_BAD_RESPONSE, this, null, this.response);\n          }\n          throw e;\n        }\n      }\n    }\n\n    return data;\n  }],\n\n  /**\n...\n// Path: lib/core/transformData.js\n'use strict';\n\nimport utils from './../utils.js';\nimport defaults from '../defaults/index.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\n\n/**\n * Transform the data for a request or a response\n *\n * @param {Array|Function} fns A single function or Array of functions\n * @param {?Object} response The response object\n *\n * @returns {*} The resulting transformed data\n */\nexport default function transformData(fns, response) {\n  const config = this || defaults;\n  const context = response || config;\n  const headers = AxiosHeaders.from(context.headers);\n  let data = context.data;\n\n  utils.forEach(fns, function transform(fn) {\n    data = fn.call(config, data, headers.normalize(), response ? response.status : undefined);\n  });\n\n  headers.normalize();\n\n  return data;\n}\n\n// Path: lib/cancel/isCancel.js\n'use strict';\n\nexport default function isCancel(value) {\n  return !!(value && value.__CANCEL__);\n}\n\n// Path: lib/core/dispatchRequest.js\n'use strict';\n\nimport transformData from './transformData.js';\nimport isCancel from '../cancel/isCancel.js';\nimport defaults from '../defaults/index.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport adapters from \"../adapters/adapters.js\";\n\n/**\n * Throws a `CanceledError` if cancellation has been requested.\n *\n * @param {Object} config The config that is to be used for the request\n *\n * @returns {void}\n */\nfunction throwIfCancellationRequested(config) {\n  if (config.cancelToken) {\n    config.cancelToken.throwIfRequested();\n  }\n\n  if (config.signal && config.signal.aborted) {\n    throw new CanceledError(null, config);\n  }\n}\n\n/**\n * Dispatch a request to the server using the configured adapter.\n *\n * @param {object} config The config that is to be used for the request\n *\n * @returns {Promise} The Promise to be fulfilled\n */\nexport default function dispatchRequest(config) {\n  throwIfCancellationRequested(config);\n\n  config.headers = AxiosHeaders.from(config.headers);\n\n  // Transform request data\n  config.data = transformData.call(\n    config,\n    config.transformRequest\n  );\n\n  if (['post', 'put', 'patch'].indexOf(config.method) !== -1) {\n    config.headers.setContentType('application/x-www-form-urlencoded', false);\n  }\n\n  const adapter = adapters.getAdapter(config.adapter || defaults.adapter);\n\n  return adapter(config).then(function onAdapterResolution(response) {\n    throwIfCancellationRequested(config);\n\n    // Transform response data\n    response.data = transformData.call(\n      config,\n      config.transformResponse,\n      response\n    );\n\n    response.headers = AxiosHeaders.from(response.headers);\n\n    return response;\n  }, function onAdapterRejection(reason) {\n    if (!isCancel(reason)) {\n      throwIfCancellationRequested(config);\n\n      // Transform response data\n      if (reason && reason.response) {\n        reason.response.data = transformData.call(\n          config,\n          config.transformResponse,\n          reason.response\n        );\n        reason.response.headers = AxiosHeaders.from(reason.response.headers);\n      }\n    }\n\n    return Promise.reject(reason);\n  });\n}\n\n// Path: lib/helpers/validator.js\n'use strict';\n\nimport {VERSION} from '../env/data.js';\nimport AxiosError from '../core/AxiosError.js';\n\nconst validators = {};\n\n// eslint-disable-next-line func-names\n['object', 'boolean', 'number', 'function', 'string', 'symbol'].forEach((type, i) => {\n  validators[type] = function validator(thing) {\n    return typeof thing === type || 'a' + (i < 1 ? 'n ' : ' ') + type;\n  };\n});\n\nconst deprecatedWarnings = {};\n\n/**\n * Transitional option validator\n *\n * @param {function|boolean?} validator - set to false if the transitional option has been removed\n * @param {string?} version - deprecated version / removed since version\n * @param {string?} message - some message with additional info\n *\n * @returns {function}\n */\nvalidators.transitional = function transitional(validator, version, message) {\n  \nfunction formatMessage(opt, desc) {\n    return '[Axios v' + VERSION + '] Transitional option \\'' + opt + '\\'' + desc + (message ? '. ' + message : '');\n  }\n\n  // eslint-disable-next-line func-names\n  return (value, opt, opts) => {\n    if (validator === false) {\n      throw new AxiosError(\n        formatMessage(opt, ' has been removed' + (version ? ' in ' + version : '')),\n        AxiosError.ERR_DEPRECATED\n      );\n    }\n\n    if (version && !deprecatedWarnings[opt]) {\n      deprecatedWarnings[opt] = true;\n      // eslint-disable-next-line no-console\n      console.warn(\n        formatMessage(\n          opt,\n          ' has been deprecated since v' + version + ' and will be removed in the near future'\n        )\n      );\n    }\n\n    return validator ? validator(value, opt, opts) : true;\n  };\n};\n\n/**\n * Assert object's properties type\n *\n * @param {object} options\n * @param {object} schema\n * @param {boolean?} allowUnknown\n *\n * @returns {object}\n */\n\nfunction assertOptions(options, schema, allowUnknown) {\n  if (typeof options !== 'object') {\n    throw new AxiosError('options must be an object', AxiosError.ERR_BAD_OPTION_VALUE);\n  }\n  const keys = Object.keys(options);\n  let i = keys.length;\n  while (i-- > 0) {\n    const opt = keys[i];\n    const validator = schema[opt];\n    if (validator) {\n      const value = options[opt];\n      const result = value === undefined || validator(value, opt, options);\n      if (result !== true) {\n        throw new AxiosError('option ' + opt + ' must be ' + result, AxiosError.ERR_BAD_OPTION_VALUE);\n      }\n      continue;\n    }\n    if (allowUnknown !== true) {\n      throw new AxiosError('Unknown option ' + opt, AxiosError.ERR_BAD_OPTION);\n    }\n  }\n}\n\nexport default {\n  assertOptions,\n  validators\n};\n\n// Path: lib/core/mergeConfig.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosHeaders from \"./AxiosHeaders.js\";\n\nconst headersToObject = (thing) => thing instanceof AxiosHeaders ? { ...thing } : thing;\n\n/**\n * Config-specific merge-function which creates a new config-object\n * by merging two configuration objects together.\n *\n * @param {Object} config1\n * @param {Object} config2\n *\n * @returns {Object} New object resulting from merging config2 to config1\n */\nexport default function mergeConfig(config1, config2) {\n  // eslint-disable-next-line no-param-reassign\n  config2 = config2 || {};\n  const config = {};\n\n  function getMergedValue(target, source, caseless) {\n    if (utils.isPlainObject(target) && utils.isPlainObject(source)) {\n      return utils.merge.call({caseless}, target, source);\n    } else if (utils.isPlainObject(source)) {\n      return utils.merge({}, source);\n    } else if (utils.isArray(source)) {\n      return source.slice();\n    }\n    return source;\n  }\n\n  // eslint-disable-next-line consistent-return\n  function mergeDeepProperties(a, b, caseless) {\n    if (!utils.isUndefined(b)) {\n      return getMergedValue(a, b, caseless);\n    } else if (!utils.isUndefined(a)) {\n      return getMergedValue(undefined, a, caseless);\n    }\n  }\n\n  // eslint-disable-next-line consistent-return\n  function valueFromConfig2(a, b) {\n    if (!utils.isUndefined(b)) {\n      return getMergedValue(undefined, b);\n    }\n  }\n\n  // eslint-disable-next-line consistent-return\n  function defaultToConfig2(a, b) {\n    if (!utils.isUndefined(b)) {\n      return getMergedValue(undefined, b);\n    } else if (!utils.isUndefined(a)) {\n      return getMergedValue(undefined, a);\n    }\n  }\n\n  // eslint-disable-next-line consistent-return\n  function mergeDirectKeys(a, b, prop) {\n    if (prop in config2) {\n      return getMergedValue(a, b);\n    } else if (prop in config1) {\n      return getMergedValue(undefined, a);\n    }\n  }\n\n  const mergeMap = {\n    url: valueFromConfig2,\n    method: valueFromConfig2,\n    data: valueFromConfig2,\n    baseURL: defaultToConfig2,\n    transformRequest: defaultToConfig2,\n    transformResponse: defaultToConfig2,\n    paramsSerializer: defaultToConfig2,\n    timeout: defaultToConfig2,\n    timeoutMessage: defaultToConfig2,\n    withCredentials: defaultToConfig2,\n    withXSRFToken: defaultToConfig2,\n    adapter: defaultToConfig2,\n    responseType: defaultToConfig2,\n    xsrfCookieName: defaultToConfig2,\n    xsrfHeaderName: defaultToConfig2,\n    onUploadProgress: defaultToConfig2,\n    onDownloadProgress: defaultToConfig2,\n    decompress: defaultToConfig2,\n    maxContentLength: defaultToConfig2,\n    maxBodyLength: defaultToConfig2,\n    beforeRedirect: defaultToConfig2,\n    transport: defaultToConfig2,\n    httpAgent: defaultToConfig2,\n    httpsAgent: defaultToConfig2,\n    cancelToken: defaultToConfig2,\n    socketPath: defaultToConfig2,\n    responseEncoding: defaultToConfig2,\n    validateStatus: mergeDirectKeys,\n    headers: (a, b) => mergeDeepProperties(headersToObject(a), headersToObject(b), true)\n  };\n\n  utils.forEach(Object.keys(Object.assign({}, config1, config2)), function computeConfigValue(prop) {\n    const merge = mergeMap[prop] || mergeDeepProperties;\n    const configValue = merge(config1[prop], config2[prop], prop);\n    (utils.isUndefined(configValue) && merge !== mergeDirectKeys) || (config[prop] = configValue);\n  });\n\n  return config;\n}\n\n// Path: lib/core/InterceptorManager.js\n'use strict';\n\nimport utils from './../utils.js';\n\nclass InterceptorManager {\n  constructor() {\n    this.handlers = [];\n  }\n\n  /**\n   * Add a new interceptor to the stack\n   *\n   * @param {Function} fulfilled The function to handle `then` for a `Promise`\n   * @param {Function} rejected The function to handle `reject` for a `Promise`\n   *\n   * @return {Number} An ID used to remove interceptor later\n   */\n  use(fulfilled, rejected, options) {\n    this.handlers.push({\n      fulfilled,\n      rejected,\n      synchronous: options ? options.synchronous : false,\n      runWhen: options ? options.runWhen : null\n    });\n    return this.handlers.length - 1;\n  }\n\n  /**\n   * Remove an interceptor from the stack\n   *\n   * @param {Number} id The ID that was returned by `use`\n   *\n   * @returns {Boolean} `true` if the interceptor was removed, `false` otherwise\n   */\n  eject(id) {\n    if (this.handlers[id]) {\n      this.handlers[id] = null;\n    }\n  }\n\n  /**\n   * Clear all interceptors from the stack\n   *\n   * @returns {void}\n   */\n  clear() {\n    if (this.handlers) {\n      this.handlers = [];\n    }\n  }\n\n  /**\n   * Iterate over all the registered interceptors\n   *\n   * This method is particularly useful for skipping over any\n   * interceptors that may have become `null` calling `eject`.\n   *\n   * @param {Function} fn The function to call for each interceptor\n   *\n   * @returns {void}\n   */\n  forEach(fn) {\n    utils.forEach(this.handlers, function forEachHandler(h) {\n      if (h !== null) {\n        fn(h);\n      }\n    });\n  }\n}\n\nexport default InterceptorManager;\n\n// Path: lib/core/Axios.js\n'use strict';\n\nimport utils from './../utils.js';\nimport buildURL from '../helpers/buildURL.js';\nimport InterceptorManager from './InterceptorManager.js';\nimport dispatchRequest from './dispatchRequest.js';\nimport mergeConfig from './mergeConfig.js';\nimport buildFullPath from './buildFullPath.js';\nimport validator from '../helpers/validator.js';\nimport AxiosHeaders from './AxiosHeaders.js';\n\nconst validators = validator.validators;\n\n/**\n * Create a new instance of Axios\n *\n * @param {Object} instanceConfig The default config for the instance\n *\n * @return {Axios} A new instance of Axios\n */\nclass Axios {\n  constructor(instanceConfig) {\n    this.defaults = instanceConfig;\n    this.interceptors = {\n      request: new InterceptorManager(),\n      response: new InterceptorManager()\n    };\n  }\n\n  /**\n   * Dispatch a request\n   *\n   * @param {String|Object} configOrUrl The config specific for this request (merged with this.defaults)\n   * @param {?Object} config\n   *\n   * @returns {Promise} The Promise to be fulfilled\n   */\n  async request(configOrUrl, config) {\n    try {\n      return await this._request(configOrUrl, config);\n    } catch (err) {\n      if (err instanceof Error) {\n        let dummy;\n\n        Error.captureStackTrace ? Error.captureStackTrace(dummy = {}) : (dummy = new Error());\n\n        // slice off the Error: ... line\n        const stack = dummy.stack ? dummy.stack.replace(/^.+\\n/, '') : '';\n\n        if (!err.stack) {\n          err.stack = stack;\n          // match without the 2 top stack lines\n        } else if (stack && !String(err.stack).endsWith(stack.replace(/^.+\\n.+\\n/, ''))) {\n          err.stack += '\\n' + stack\n        }\n      }\n\n      throw err;\n    }\n  }\n\n  _request(configOrUrl, config) {\n    /*eslint no-param-reassign:0*/\n    // Allow for axios('example/url'[, config]) a la fetch API\n    if (typeof configOrUrl === 'string') {\n      config = config || {};\n      config.url = configOrUrl;\n    } else {\n      config = configOrUrl || {};\n    }\n\n    config = mergeConfig(this.defaults, config);\n\n    const {transitional, paramsSerializer, headers} = config;\n\n    if (transitional !== undefined) {\n      validator.assertOptions(transitional, {\n        silentJSONParsing: validators.transitional(validators.boolean),\n        forcedJSONParsing: validators.transitional(validators.boolean),\n        clarifyTimeoutError: validators.transitional(validators.boolean)\n      }, false);\n    }\n\n    if (paramsSerializer != null) {\n      if (utils.isFunction(paramsSerializer)) {\n        config.paramsSerializer = {\n          serialize: paramsSerializer\n        }\n      } else {\n        validator.assertOptions(paramsSerializer, {\n          encode: validators.function,\n          serialize: validators.function\n        }, true);\n      }\n    }\n\n    // Set config.method\n    config.method = (config.method || this.defaults.method || 'get').toLowerCase();\n\n    // Flatten headers\n    let contextHeaders = headers && utils.merge(\n      headers.common,\n      headers[config.method]\n    );\n\n    headers && utils.forEach(\n      ['delete', 'get', 'head', 'post', 'put', 'patch', 'common'],\n      (method) => {\n        delete headers[method];\n      }\n    );\n\n    config.headers = AxiosHeaders.concat(contextHeaders, headers);\n\n    // filter out skipped interceptors\n    const requestInterceptorChain = [];\n    let synchronousRequestInterceptors = true;\n    this.interceptors.request.forEach(function unshiftRequestInterceptors(interceptor) {\n      if (typeof interceptor.runWhen === 'function' && interceptor.runWhen(config) === false) {\n        return;\n      }\n\n      synchronousRequestInterceptors = synchronousRequestInterceptors && interceptor.synchronous;\n\n      requestInterceptorChain.unshift(interceptor.fulfilled, interceptor.rejected);\n    });\n\n    const responseInterceptorChain = [];\n    this.interceptors.response.forEach(function pushResponseInterceptors(interceptor) {\n      responseInterceptorChain.push(interceptor.fulfilled, interceptor.rejected);\n    });\n\n    let promise;\n    let i = 0;\n    let len;\n\n    if (!synchronousRequestInterceptors) {\n      const chain = [dispatchRequest.bind(this), undefined];\n      chain.unshift.apply(chain, requestInterceptorChain);\n      chain.push.apply(chain, responseInterceptorChain);\n      len = chain.length;\n\n      promise = Promise.resolve(config);\n\n      while (i < len) {\n        promise = promise.then(chain[i++], chain[i++]);\n      }\n\n      return promise;\n    }\n\n    len = requestInterceptorChain.length;\n\n    let newConfig = config;\n\n    i = 0;\n\n    while (i < len) {\n      const onFulfilled = requestInterceptorChain[i++];\n      const onRejected = requestInterceptorChain[i++];\n      try {\n        newConfig = onFulfilled(newConfig);\n      } catch (error) {\n        onRejected.call(this, error);\n        break;\n      }\n    }\n\n    try {\n      promise = dispatchRequest.call(this, newConfig);\n    } catch (error) {\n      return Promise.reject(error);\n    }\n\n    i = 0;\n    len = responseInterceptorChain.length;\n\n    while (i < len) {\n      promise = promise.then(responseInterceptorChain[i++], responseInterceptorChain[i++]);\n    }\n\n    return promise;\n  }\n\n  getUri(config) {\n    config = mergeConfig(this.defaults, config);\n    const fullPath = buildFullPath(config.baseURL, config.url);\n    return buildURL(fullPath, config.params, config.paramsSerializer);\n  }\n}\n\n// Provide aliases for supported request methods\nutils.forEach(['delete', 'get', 'head', 'options'], function forEachMethodNoData(method) {\n  /*eslint func-names:0*/\n  Axios.prototype[method] = function(url, config) {\n    return this.request(mergeConfig(config || {}, {\n      method,\n      url,\n      data: (config || {}).data\n    }));\n  };\n});\n\nutils.forEach(['post', 'put', 'patch'], function forEachMethodWithData(method) {\n  /*eslint func-names:0*/\n\n  function generateHTTPMethod(isForm) {\n    return function httpMethod(url, data, config) {\n      return this.request(mergeConfig(config || {}, {\n        method,\n        headers: isForm ? {\n          'Content-Type': 'multipart/form-data'\n        } : {},\n        url,\n        data\n      }));\n    };\n  }\n\n  Axios.prototype[method] = generateHTTPMethod();\n\n  Axios.prototype[method + 'Form'] = generateHTTPMethod(true);\n});\n\nexport default Axios;\n\n// Path: lib/helpers/HttpStatusCode.js\nconst HttpStatusCode = {\n  Continue: 100,\n  SwitchingProtocols: 101,\n  Processing: 102,\n  EarlyHints: 103,\n  Ok: 200,\n  Created: 201,\n  Accepted: 202,\n  NonAuthoritativeInformation: 203,\n  NoContent: 204,\n  ResetContent: 205,\n  PartialContent: 206,\n  MultiStatus: 207,\n  AlreadyReported: 208,\n  ImUsed: 226,\n  MultipleChoices: 300,\n  MovedPermanently: 301,\n  Found: 302,\n  SeeOther: 303,\n  NotModified: 304,\n  UseProxy: 305,\n  Unused: 306,\n  TemporaryRedirect: 307,\n  PermanentRedirect: 308,\n  BadRequest: 400,\n  Unauthorized: 401,\n  PaymentRequired: 402,\n  Forbidden: 403,\n  NotFound: 404,\n  MethodNotAllowed: 405,\n  NotAcceptable: 406,\n  ProxyAuthenticationRequired: 407,\n  RequestTimeout: 408,\n  Conflict: 409,\n  Gone: 410,\n  LengthRequired: 411,\n  PreconditionFailed: 412,\n  PayloadTooLarge: 413,\n  UriTooLong: 414,\n  UnsupportedMediaType: 415,\n  RangeNotSatisfiable: 416,\n  ExpectationFailed: 417,\n  ImATeapot: 418,\n  MisdirectedRequest: 421,\n  UnprocessableEntity: 422,\n  Locked: 423,\n  FailedDependency: 424,\n  TooEarly: 425,\n  UpgradeRequired: 426,\n  PreconditionRequired: 428,\n  TooManyRequests: 429,\n  RequestHeaderFieldsTooLarge: 431,\n  UnavailableForLegalReasons: 451,\n  InternalServerError: 500,\n  NotImplemented: 501,\n  BadGateway: 502,\n  ServiceUnavailable: 503,\n  GatewayTimeout: 504,\n  HttpVersionNotSupported: 505,\n  VariantAlsoNegotiates: 506,\n  InsufficientStorage: 507,\n  LoopDetected: 508,\n  NotExtended: 510,\n  NetworkAuthenticationRequired: 511,\n};\n\nObject.entries(HttpStatusCode).forEach(([key, value]) => {\n  HttpStatusCode[value] = key;\n});\n\nexport default HttpStatusCode;\n\n// Path: lib/helpers/spread.js\n'use strict';\n\n/**\n * Syntactic sugar for invoking a function and expanding an array for arguments.\n *\n * Common use case would be to use `Function.prototype.apply`.\n *\n *  ```js\n *  function f(x, y, z) {}\n *  var args = [1, 2, 3];\n *  f.apply(null, args);\n *  ```\n *\n * With `spread` this example can be re-written.\n *\n *  ```js\n *  spread(function(x, y, z) {})([1, 2, 3]);\n *  ```\n *\n * @param {Function} callback\n *\n * @returns {Function}\n */\nexport default function spread(callback) {\n  return function wrap(arr) {\n    return callback.apply(null, arr);\n  };\n}\n\n// Path: lib/cancel/CancelToken.js\n'use strict';\n\nimport CanceledError from './CanceledError.js';\n\n/**\n * A `CancelToken` is an object that can be used to request cancellation of an operation.\n *\n * @param {Function} executor The executor function.\n *\n * @returns {CancelToken}\n */\nclass CancelToken {\n  constructor(executor) {\n    if (typeof executor !== 'function') {\n      throw new TypeError('executor must be a function.');\n    }\n\n    let resolvePromise;\n\n    this.promise = new Promise(function promiseExecutor(resolve) {\n      resolvePromise = resolve;\n    });\n\n    const token = this;\n\n    // eslint-disable-next-line func-names\n    this.promise.then(cancel => {\n      if (!token._listeners) return;\n\n      let i = token._listeners.length;\n\n      while (i-- > 0) {\n        token._listeners[i](cancel);\n      }\n      token._listeners = null;\n    });\n\n    // eslint-disable-next-line func-names\n    this.promise.then = onfulfilled => {\n      let _resolve;\n      // eslint-disable-next-line func-names\n      const promise = new Promise(resolve => {\n        token.subscribe(resolve);\n        _resolve = resolve;\n      }).then(onfulfilled);\n\n      promise.cancel = function reject() {\n        token.unsubscribe(_resolve);\n      };\n\n      return promise;\n    };\n\n    executor(function cancel(message, config, request) {\n      if (token.reason) {\n        // Cancellation has already been requested\n        return;\n      }\n\n      token.reason = new CanceledError(message, config, request);\n      resolvePromise(token.reason);\n    });\n  }\n\n  /**\n   * Throws a `CanceledError` if cancellation has been requested.\n   */\n  throwIfRequested() {\n    if (this.reason) {\n      throw this.reason;\n    }\n  }\n\n  /**\n   * Subscribe to the cancel signal\n   */\n\n  subscribe(listener) {\n    if (this.reason) {\n      listener(this.reason);\n      return;\n    }\n\n    if (this._listeners) {\n      this._listeners.push(listener);\n    } else {\n      this._listeners = [listener];\n    }\n  }\n\n  /**\n   * Unsubscribe from the cancel signal\n   */\n\n  unsubscribe(listener) {\n    if (!this._listeners) {\n      return;\n    }\n    const index = this._listeners.indexOf(listener);\n    if (index !== -1) {\n      this._listeners.splice(index, 1);\n    }\n  }\n\n  /**\n   * Returns an object that contains a new `CancelToken` and a function that, when called,\n   * cancels the `CancelToken`.\n   */\n  static source() {\n    let cancel;\n    const token = new CancelToken(function executor(c) {\n      cancel = c;\n    });\n    return {\n      token,\n      cancel\n    };\n  }\n}\n\nexport default CancelToken;\n\n// Path: lib/helpers/isAxiosError.js\n'use strict';\n\nimport utils from './../utils.js';\n\n/**\n * Determines whether the payload is an error thrown by Axios\n *\n * @param {*} payload The value to test\n *\n * @returns {boolean} True if the payload is an error thrown by Axios, otherwise false\n */\nexport default function isAxiosError(payload) {\n  return utils.isObject(payload) && (payload.isAxiosError === true);\n}\n\n// Path: lib/axios.js\n'use strict';\n\nimport utils from './utils.js';\nimport bind from './helpers/bind.js';\nimport Axios from './core/Axios.js';\nimport mergeConfig from './core/mergeConfig.js';\nimport defaults from './defaults/index.js';\nimport formDataToJSON from './helpers/formDataToJSON.js';\nimport CanceledError from './cancel/CanceledError.js';\nimport CancelToken from './cancel/CancelToken.js';\nimport isCancel from './cancel/isCancel.js';\nimport {VERSION} from './env/data.js';\nimport toFormData from './helpers/toFormData.js';\nimport AxiosError from './core/AxiosError.js';\nimport spread from './helpers/spread.js';\nimport isAxiosError from './helpers/isAxiosError.js';\nimport AxiosHeaders from \"./core/AxiosHeaders.js\";\nimport adapters from './adapters/adapters.js';\nimport HttpStatusCode from './helpers/HttpStatusCode.js';\n\n/**\n * Create an instance of Axios\n *\n * @param {Object} defaultConfig The default config for the instance\n *\n * @returns {Axios} A new instance of Axios\n */\nfunction createInstance(defaultConfig) {\n  const context = new Axios(defaultConfig);\n  const instance = bind(Axios.prototype.request, context);\n\n  // Copy axios.prototype to instance\n  utils.extend(instance, Axios.prototype, context, {allOwnKeys: true});\n\n  // Copy context to instance\n  utils.extend(instance, context, null, {allOwnKeys: true});\n\n  // Factory for creating new instances\n  instance.create = function create(instanceConfig) {\n    return createInstance(mergeConfig(defaultConfig, instanceConfig));\n  };\n\n  return instance;\n}\n\n// Create the default instance to be exported\nconst axios = createInstance(defaults);\n\n// Expose Axios class to allow class inheritance\naxios.Axios = Axios;\n\n// Expose Cancel & CancelToken\naxios.CanceledError = CanceledError;\naxios.CancelToken = CancelToken;\naxios.isCancel = isCancel;\naxios.VERSION = VERSION;\naxios.toFormData = toFormData;\n\n// Expose AxiosError class\naxios.AxiosError = AxiosError;\n\n// alias for CanceledError for backward compatibility\naxios.Cancel = axios.CanceledError;\n\n// Expose all/spread\naxios.all = function all(promises) {\n  return Promise.all(promises);\n};\n\naxios.spread = spread;\n\n// Expose isAxiosError\naxios.isAxiosError = isAxiosError;\n\n// Expose mergeConfig\naxios.mergeConfig = mergeConfig;\n\naxios.AxiosHeaders = AxiosHeaders;\n\naxios.formToJSON = thing => formDataToJSON(utils.isHTMLForm(thing) ? new FormData(thing) : thing);\n\naxios.getAdapter = adapters.getAdapter;\n\naxios.HttpStatusCode = HttpStatusCode;\n\naxios.default = axios;\n\n// this module should only have a default export\nexport default axios\n\n// Path: lib/env/classes/FormData.js\nimport _FormData from 'form-data';\nexport default typeof FormData !== 'undefined' ? FormData : _FormData;\n\n// Path: lib/helpers/deprecatedMethod.js\n'use strict';\n\n/*eslint no-console:0*/\n\n/**\n * Supply a warning to the developer that a method they are using\n * has been deprecated.\n *\n * @param {string} method The name of the deprecated method\n * @param {string} [instead] The alternate method to use if applicable\n * @param {string} [docs] The documentation URL to get further details\n *\n * @returns {void}\n */\nexport default function deprecatedMethod(method, instead, docs) {\n  try {\n    console.warn(\n      'DEPRECATED method `' + method + '`.' +\n      (instead ? ' Use `' + instead + '` instead.' : '') +\n      ' This method will be removed in a future release.');\n\n    if (docs) {\n      console.warn('For more information about usage see ' + docs);\n    }\n  } catch (e) { /* Ignore */ }\n}\n\n// Path: lib/helpers/null.js\n// eslint-disable-next-line strict\nexport default null;\n\n// Path: lib/platform/browser/classes/Blob.js\n'use strict'\n\nexport default typeof Blob !== 'undefined' ? Blob : null\n\n// Path: lib/platform/browser/classes/URLSearchParams.js\n'use strict';\n\nimport AxiosURLSearchParams from '../../../helpers/AxiosURLSearchParams.js';\nexport default typeof URLSearchParams !== 'undefined' ? URLSearchParams : AxiosURLSearchParams;\n\n// Path: lib/platform/browser/classes/FormData.js\n'use strict';\n\nexport default typeof FormData !== 'undefined' ? FormData : null;\n\n// Path: lib/platform/browser/index.js\nimport URLSearchParams from './classes/URLSearchParams.js'\nimport FormData from './classes/FormData.js'\nimport Blob from './classes/Blob.js'\n\nexport default {\n  isBrowser: true,\n  classes: {\n    URLSearchParams,\n    FormData,\n    Blob\n  },\n  protocols: ['http', 'https', 'file', 'blob', 'url', 'data']\n};\n\n", "needle_token_start": 2337, "needle_token_end": 2389, "code_context_ntokens": 9612, "output": ["```javascript\nfunction formatMessage(opt, desc) {\n  return '[Axios v' + VERSION + '] Transitional option \\'' + opt + '\\'' + desc + (message ? '. ' + message : '');\n}\n```"]}
{"repo": "axios/axios", "name": "buildURL", "language": "typescript", "path": "lib/helpers/buildURL.js", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To construct a complete URL by appending query parameters to a base URL, optionally using custom serialization and encoding methods.\n2. **Input**: A base URL string, an optional object containing query parameters, and an optional options object that may specify custom serialization and encoding functions.\n3. **Output**: A string representing the fully constructed URL with query parameters appended.\n4. **Procedure**: \n   - If no parameters are provided, the base URL is returned as is.\n   - If options are provided, they are used to determine the encoding function for the parameters.\n   - Parameters are serialized into a query string using either a provided custom serialization function or a default method.\n   - The serialized query string is appended to the base URL. If the URL already contains a query string, the new parameters are appended with an ampersand; otherwise, they are appended with a question mark.\n   - If the URL contains a hash fragment, it is preserved and the parameters are inserted before the fragment.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/utils.js\n'use strict';\n\nimport bind from './helpers/bind.js';\n\n// utils is a library of generic helper functions non-specific to axios\n\nconst {toString} = Object.prototype;\nconst {getPrototypeOf} = Object;\n\nconst kindOf = (cache => thing => {\n    const str = toString.call(thing);\n    return cache[str] || (cache[str] = str.slice(8, -1).toLowerCase());\n})(Object.create(null));\n\nconst kindOfTest = (type) => {\n  type = type.toLowerCase();\n  return (thing) => kindOf(thing) === type\n...\n// Path: lib/core/AxiosError.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: utils.toJSONObject(this.config),\n      code: this.code,\n      status: this.response && this.response.status ? this.response.status : null\n    };\n  }\n});\n\nconst prototype = AxiosError.prototype;\nconst descriptors = {};\n\n[\n  'ERR_BAD_OPTION_VALUE',\n  'ERR_BAD_OPTION',\n  'ECONNABORTED',\n  'ETIMEDOUT',\n  'ERR_NETWORK',\n  'ERR_FR_TOO_MANY_REDIRECTS',\n  'ERR_DEPRECATED',\n  'ERR_BAD_RESPONSE',\n  'ERR_BAD_REQUEST',\n  'ERR_CANCELED',\n  'ERR_NOT_SUPPORT',\n  'ERR_INVALID_URL'\n// eslint-disable-next-line func-names\n].forEach(code => {\n  descriptors[code] = {value: code};\n});\n\nObject.defineProperties(AxiosError, descriptors);\nObject.defineProperty(prototype, 'isAxiosError', {value: true});\n\n// eslint-disable-next-line func-names\nAxiosError.from = (error, code, config, request, response, customProps) => {\n  const axiosError = Object.create(prototype);\n\n  utils.toFlatObject(error, axiosError, function filter(obj) {\n    return obj !== Error.prototype;\n  }, prop => {\n    return prop !== 'isAxiosError';\n  });\n\n  AxiosError.call(axiosError, error.message, code, config, request, response);\n\n  axiosError.cause = error;\n\n  axiosError.name = error.name;\n\n  customProps && Object.assign(axiosError, customProps);\n\n  return axiosError;\n};\n\nexport default AxiosError;\n\n// Path: lib/helpers/toFormData.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\n// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored\nimport PlatformFormData from '../platform/node/classes/FormData.js';\n\n/**\n * Determines if the given thing is a array or js object.\n *\n * @param {string} thing - The object or array to be visited.\n *\n * @returns {boolean}\n */\nfunction isVisitable(thing) {\n  return utils.isPlainObject(thing) || utils.isArray(thing);\n}\n\n/**\n * It removes the brackets from the end of a string\n *\n * @param {string} key - The key of the parameter.\n *\n * @returns {string} the key without the brackets.\n */\nfunction removeBrackets(key) {\n  return utils.endsWith(key, '[]') ? key.slice(0, -2) : key;\n}\n\n/**\n * It takes a path, a key, and a boolean, and returns a string\n *\n * @param {string} path - The path to the current key.\n * @param {string} key - The key of the current object being iterated over.\n * @param {string} dots - If true, the key will be rendered with dots instead of brackets.\n *\n * @returns {string} The path to the current key.\n */\nfunction renderKey(path, key, dots) {\n  if (!path) return key;\n  return path.concat(key).map(function each(token, i) {\n    // eslint-disable-next-line no-param-reassign\n    token = removeBrackets(token);\n    return !dots && i ? '[' + token + ']' : token;\n  }).join(dots ? '.' : '');\n}\n\n/**\n * If the array is an array and none of its elements are visitable, then it's a flat array.\n *\n * @param {Array<any>} arr - The array to check\n *\n * @returns {boolean}\n */\nfunction isFlatArray(arr) {\n  return utils.isArray(arr) && !arr.some(isVisitable);\n}\n\nconst predicates = utils.toFlatObject(utils, {}, null, function filter(prop) {\n  return /^is[A-Z]/.test(prop);\n});\n\n/**\n * Convert a data object to FormData\n *\n * @param {Object} obj\n * @param {?Object} [formData]\n * @param {?Object} [options]\n * @param {Function} [options.visitor]\n * @param {Boolean} [options.metaTokens = true]\n * @param {Boolean} [options.dots = false]\n * @param {?Boolean} [options.indexes = false]\n *\n * @returns {Object}\n **/\n\n/**\n * It converts an object into a FormData object\n *\n * @param {Object<any, any>} obj - The object to convert to form data.\n * @param {string} formData - The FormData object to append to.\n * @param {Object<string, any>} options\n *\n * @returns\n */\nfunction toFormData(obj, formData, options) {\n  if (!utils.isObject(obj)) {\n    throw new TypeError('target must be an object');\n  }\n\n  // eslint-disable-next-line no-param-reassign\n  formData = formData || new (PlatformFormData || FormData)();\n\n  // eslint-disable-next-line no-param-reassign\n  options = utils.toFlatObject(options, {\n    metaTokens: true,\n    dots: false,\n    indexes: false\n  }, false, function defined(option, source) {\n    // eslint-disable-next-line no-eq-null,eqeqeq\n    return !utils.isUndefined(source[option]);\n  });\n\n  const metaTokens = options.metaTokens;\n  // eslint-disable-next-line no-use-before-define\n  const visitor = options.visitor || defaultVisitor;\n  const dots = options.dots;\n  const indexes = options.indexes;\n  const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob;\n  const useBlob = _Blob && utils.isSpecCompliantForm(formData);\n\n  if (!utils.isFunction(visitor)) {\n    throw new TypeError('visitor must be a function');\n  }\n\n  function convertValue(value) {\n    if (value === null) return '';\n\n    if (utils.isDate(value)) {\n      return value.toISOString();\n    }\n\n    if (!useBlob && utils.isBlob(value)) {\n      throw new AxiosError('Blob is not supported. Use a Buffer instead.');\n    }\n\n    if (utils.isArrayBuffer(value) || utils.isTypedArray(value)) {\n      return useBlob && typeof Blob === 'function' ? new Blob([value]) : Buffer.from(value);\n    }\n\n    return value;\n  }\n\n  /**\n   * Default visitor.\n   *\n   * @param {*} value\n   * @param {String|Number} key\n   * @param {Array<String|Number>} path\n   * @this {FormData}\n   *\n   * @returns {boolean} return true to visit the each prop of the value recursively\n   */\n  function defaultVisitor(value, key, path) {\n    let arr = value;\n\n    if (value && !path && typeof value === 'object') {\n      if (utils.endsWith(key, '{}')) {\n        // eslint-disable-next-line no-param-reassign\n        key = metaTokens ? key : key.slice(0, -2);\n        // eslint-disable-next-line no-param-reassign\n        value = JSON.stringify(value);\n      } else if (\n        (utils.isArray(value) && isFlatArray(value)) ||\n        ((utils.isFileList(value) || utils.endsWith(key, '[]')) && (arr = utils.toArray(value))\n        )) {\n        // eslint-disable-next-line no-param-reassign\n        key = removeBrackets(key);\n\n        arr.forEach(function each(el, index) {\n          !(utils.isUndefined(el) || el === null) && formData.append(\n            // eslint-disable-next-line no-nested-ternary\n            indexes === true ? renderKey([key], index, dots) : (indexes === null ? key : key + '[]'),\n            convertValue(el)\n          );\n        });\n        return false;\n      }\n    }\n\n    if (isVisitable(value)) {\n      return true;\n    }\n\n    formData.append(renderKey(path, key, dots), convertValue(value));\n\n    return false;\n  }\n\n  const stack = [];\n\n  const exposedHelpers = Object.assign(predicates, {\n    defaultVisitor,\n    convertValue,\n    isVisitable\n  });\n\n  function build(value, path) {\n    if (utils.isUndefined(value)) return;\n\n    if (stack.indexOf(value) !== -1) {\n      throw Error('Circular reference detected in ' + path.join('.'));\n    }\n\n    stack.push(value);\n\n    utils.forEach(value, function each(el, key) {\n      const result = !(utils.isUndefined(el) || el === null) && visitor.call(\n        formData, el, utils.isString(key) ? key.trim() : key, path, exposedHelpers\n      );\n\n      if (result === true) {\n        build(el, path ? path.concat(key) : [key]);\n      }\n    });\n\n    stack.pop();\n  }\n\n  if (!utils.isObject(obj)) {\n    throw new TypeError('data must be an object');\n  }\n\n  build(obj);\n\n  return formData;\n}\n\nexport default toFormData;\n\n// Path: lib/helpers/AxiosURLSearchParams.js\n'use strict';\n\nimport toFormData from './toFormData.js';\n\n/**\n * It encodes a string by replacing all characters that are not in the unreserved set with\n * their percent-encoded equivalents\n *\n * @param {string} str - The string to encode.\n *\n * @returns {string} The encoded string.\n */\nfunction encode(str) {\n  const charMap = {\n    '!': '%21',\n    \"'\": '%27',\n    '(': '%28',\n    ')': '%29',\n    '~': '%7E',\n    '%20': '+',\n    '%00': '\\x00'\n  };\n  return encodeURIComponent(str).replace(/[!'()~]|%20|%00/g, function replacer(match) {\n    return charMap[match];\n  });\n}\n\n/**\n * It takes a params object and converts it to a FormData object\n *\n * @param {Object<string, any>} params - The parameters to be converted to a FormData object.\n * @param {Object<string, any>} options - The options object passed to the Axios constructor.\n *\n * @returns {void}\n */\nfunction AxiosURLSearchParams(params, options) {\n  this._pairs = [];\n\n  params && toFormData(params, this, options);\n}\n\nconst prototype = AxiosURLSearchParams.prototype;\n\nprototype.append = function append(name, value) {\n  this._pairs.push([name, value]);\n};\n\nprototype.toString = function toString(encoder) {\n  const _encode = encoder ? function(value) {\n    return encoder.call(this, value, encode);\n  } : encode;\n\n  return this._pairs.map(function each(pair) {\n    return _encode(pair[0]) + '=' + _encode(pair[1]);\n  }, '').join('&');\n};\n\nexport default AxiosURLSearchParams;\n\n// Path: lib/helpers/buildURL.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosURLSearchParams from '../helpers/AxiosURLSearchParams.js';\n\n/**\n * It replaces all instances of the characters `:`, `$`, `,`, `+`, `[`, and `]` with their\n * URI encoded counterparts\n *\n * @param {string} val The value to be encoded.\n *\n * @returns {string} The encoded value.\n */\nfunction encode(val) {\n  return encodeURIComponent(val).\n    replace(/%3A/gi, ':').\n    replace(/%24/g, '$').\n    replace(/%2C/gi, ',').\n    replace(/%20/g, '+').\n    replace(/%5B/gi, '[').\n    replace(/%5D/gi, ']');\n}\n\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default \nfunction buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n\n// Path: lib/cancel/CanceledError.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n\n/**\n * A `CanceledError` is an object that is thrown when an operation is canceled.\n *\n * @param {string=} message The message.\n * @param {Object=} config The config.\n * @param {Object=} request The request.\n *\n * @returns {CanceledError} The created error.\n */\nfunction CanceledError(message, config, request) {\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  AxiosError.call(this, message == null ? 'canceled' : message, AxiosError.ERR_CANCELED, config, request);\n  this.name = 'CanceledError';\n}\n\nutils.inherits(CanceledError, AxiosError, {\n  __CANCEL__: true\n});\n\nexport default CanceledError;\n\n// Path: lib/defaults/transitional.js\n'use strict';\n\nexport default {\n  silentJSONParsing: true,\n  forcedJSONParsing: true,\n  clarifyTimeoutError: false\n};\n\n// Path: lib/helpers/speedometer.js\n'use strict';\n\n/**\n * Calculate data maxRate\n * @param {Number} [samplesCount= 10]\n * @param {Number} [min= 1000]\n * @returns {Function}\n */\nfunction speedometer(samplesCount, min) {\n  samplesCount = samplesCount || 10;\n  const bytes = new Array(samplesCount);\n  const timestamps = new Array(samplesCount);\n  let head = 0;\n  let tail = 0;\n  let firstSampleTS;\n\n  min = min !== undefined ? min : 1000;\n\n  return function push(chunkLength) {\n    const now = Date.now();\n\n    const startedAt = timestamps[tail];\n\n    if (!firstSampleTS) {\n      firstSampleTS = now;\n    }\n\n    bytes[head] = chunkLength;\n    timestamps[head] = now;\n\n    let i = tail;\n    let bytesCount = 0;\n\n    while (i !== head) {\n      bytesCount += bytes[i++];\n      i = i % samplesCount;\n    }\n\n    head = (head + 1) % samplesCount;\n\n    if (head === tail) {\n      tail = (tail + 1) % samplesCount;\n    }\n\n    if (now - firstSampleTS < min) {\n      return;\n    }\n\n    const passed = startedAt && now - startedAt;\n\n    return passed ? Math.round(bytesCount * 1000 / passed) : undefined;\n  };\n}\n\nexport default speedometer;\n\n// Path: lib/helpers/parseHeaders.js\n'use strict';\n\nimport utils from './../utils.js';\n\n// RawAxiosHeaders whose duplicates are ignored by node\n// c.f. https://nodejs.org/api/http.html#http_message_headers\nconst ignoreDuplicateOf = utils.toObjectSet([\n  'age', 'authorization', 'content-length', 'content-type', 'etag',\n  'expires', 'from', 'host', 'if-modified-since', 'if-unmodified-since',\n  'last-modified', 'location', 'max-forwards', 'proxy-authorization',\n  'referer', 'retry-after', 'user-agent'\n]);\n\n/**\n * Parse headers into an object\n *\n * ```\n * Date: Wed, 27 Aug 2014 08:58:49 GMT\n * Content-Type: application/json\n * Connection: keep-alive\n * Transfer-Encoding: chunked\n * ```\n *\n * @param {String} rawHeaders Headers needing to be parsed\n *\n * @returns {Object} Headers parsed into an object\n */\nexport default rawHeaders => {\n  const parsed = {};\n  let key;\n  let val;\n  let i;\n\n  rawHeaders && rawHeaders.split('\\n').forEach(function parser(line) {\n    i = line.indexOf(':');\n    key = line.substring(0, i).trim().toLowerCase();\n    val = line.substring(i + 1).trim();\n\n    if (!key || (parsed[key] && ignoreDuplicateOf[key])) {\n      return;\n    }\n\n    if (key === 'set-cookie') {\n      if (parsed[key]) {\n        parsed[key].push(val);\n      } else {\n        parsed[key] = [val];\n      }\n    } else {\n      parsed[key] = parsed[key] ? parsed[key] + ', ' + val : val;\n    }\n  });\n\n  return parsed;\n};\n\n// Path: lib/core/AxiosHeaders.js\n'use strict';\n\nimport utils from '../utils.js';\nimport parseHeaders from '../helpers/parseHeaders.js';\n\nconst $internals = Symbol('internals');\n\nfunction normalizeHeader(header) {\n  return header && String(header).trim().toLowerCase();\n}\n\nfunction normalizeValue(value) {\n  if (value === false || value == null) {\n    return value;\n  }\n\n  return utils.isArray(value) ? value.map(normalizeValue) : String(value);\n}\n\nfunction parseTokens(str) {\n  const tokens = Object.create(null);\n  const tokensRE = /([^\\s,;=]+)\\s*(?:=\\s*([^,;]+))?/g;\n  let match;\n\n  while ((match = tokensRE.exec(str))) {\n    tokens[match[1]] = match[2];\n  }\n\n  return tokens;\n}\n\nconst isValidHeaderName = (str) => /^[-_a-zA-Z0-9^`|~,!#$%&'*+.]+$/.test(str.trim());\n\nfunction matchHeaderValue(context, value, header, filter, isHeaderNameFilter) {\n  if (utils.isFunction(filter)) {\n    return filter.call(this, value, header);\n  }\n\n  if (isHeaderNameFilter) {\n    value = header;\n  }\n\n  if (!utils.isString(value)) return;\n\n  if (utils.isString(filter)) {\n    return value.indexOf(filter) !== -1;\n  }\n\n  if (utils.isRegExp(filter)) {\n    return filter.test(value);\n  }\n}\n\nfunction formatHeader(header) {\n  return header.trim()\n    .toLowerCase().replace(/([a-z\\d])(\\w*)/g, (w, char, str) => {\n      return char.toUpperCase() + str;\n    });\n}\n\nfunction buildAccessors(obj, header) {\n  const accessorName = utils.toCamelCase(' ' + header);\n\n  ['get', 'set', 'has'].forEach(methodName => {\n    Object.defineProperty(obj, methodName + accessorName, {\n      value: function(arg1, arg2, arg3) {\n        return this[methodName].call(this, header, arg1, arg2, arg3);\n      },\n      configurable: true\n    });\n  });\n}\n\nclass AxiosHeaders {\n  constructor(headers) {\n    headers && this.set(headers);\n  }\n\n  set(header, valueOrRewrite, rewrite) {\n    const self = this;\n\n    function setHeader(_value, _header, _rewrite) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!lHeader) {\n        throw new Error('header name must be a non-empty string');\n      }\n\n      const key = utils.findKey(self, lHeader);\n\n      if(!key || self[key] === undefined || _rewrite === true || (_rewrite === undefined && self[key] !== false)) {\n        self[key || _header] = normalizeValue(_value);\n      }\n    }\n\n    const setHeaders = (headers, _rewrite) =>\n      utils.forEach(headers, (_value, _header) => setHeader(_value, _header, _rewrite));\n\n    if (utils.isPlainObject(header) || header instanceof this.constructor) {\n      setHeaders(header, valueOrRewrite)\n    } else if(utils.isString(header) && (header = header.trim()) && !isValidHeaderName(header)) {\n      setHeaders(parseHeaders(header), valueOrRewrite);\n    } else {\n      header != null && setHeader(valueOrRewrite, header, rewrite);\n    }\n\n    return this;\n  }\n\n  get(header, parser) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      if (key) {\n        const value = this[key];\n\n        if (!parser) {\n          return value;\n        }\n\n        if (parser === true) {\n          return parseTokens(value);\n        }\n\n        if (utils.isFunction(parser)) {\n          return parser.call(this, value, key);\n        }\n\n        if (utils.isRegExp(parser)) {\n          return parser.exec(value);\n        }\n\n        throw new TypeError('parser must be boolean|regexp|function');\n      }\n    }\n  }\n\n  has(header, matcher) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      return !!(key && this[key] !== undefined && (!matcher || matchHeaderValue(this, this[key], key, matcher)));\n    }\n\n    return false;\n  }\n\n  delete(header, matcher) {\n    const self = this;\n    let deleted = false;\n\n    function deleteHeader(_header) {\n      _header = normalizeHeader(_header);\n\n      if (_header) {\n        const key = utils.findKey(self, _header);\n\n        if (key && (!matcher || matchHeaderValue(self, self[key], key, matcher))) {\n          delete self[key];\n\n          deleted = true;\n        }\n      }\n    }\n\n    if (utils.isArray(header)) {\n      header.forEach(deleteHeader);\n    } else {\n      deleteHeader(header);\n    }\n\n    return deleted;\n  }\n\n  clear(matcher) {\n    const keys = Object.keys(this);\n    let i = keys.length;\n    let deleted = false;\n\n    while (i--) {\n      const key = keys[i];\n      if(!matcher || matchHeaderValue(this, this[key], key, matcher, true)) {\n        delete this[key];\n        deleted = true;\n      }\n    }\n\n    return deleted;\n  }\n\n  normalize(format) {\n    const self = this;\n    const headers = {};\n\n    utils.forEach(this, (value, header) => {\n      const key = utils.findKey(headers, header);\n\n      if (key) {\n        self[key] = normalizeValue(value);\n        delete self[header];\n        return;\n      }\n\n      const normalized = format ? formatHeader(header) : String(header).trim();\n\n      if (normalized !== header) {\n        delete self[header];\n      }\n\n      self[normalized] = normalizeValue(value);\n\n      headers[normalized] = true;\n    });\n\n    return this;\n  }\n\n  concat(...targets) {\n    return this.constructor.concat(this, ...targets);\n  }\n\n  toJSON(asStrings) {\n    const obj = Object.create(null);\n\n    utils.forEach(this, (value, header) => {\n      value != null && value !== false && (obj[header] = asStrings && utils.isArray(value) ? value.join(', ') : value);\n    });\n\n    return obj;\n  }\n\n  [Symbol.iterator]() {\n    return Object.entries(this.toJSON())[Symbol.iterator]();\n  }\n\n  toString() {\n    return Object.entries(this.toJSON()).map(([header, value]) => header + ': ' + value).join('\\n');\n  }\n\n  get [Symbol.toStringTag]() {\n    return 'AxiosHeaders';\n  }\n\n  static from(thing) {\n    return thing instanceof this ? thing : new this(thing);\n  }\n\n  static concat(first, ...targets) {\n    const computed = new this(first);\n\n    targets.forEach((target) => computed.set(target));\n\n    return computed;\n  }\n\n  static accessor(header) {\n    const internals = this[$internals] = (this[$internals] = {\n      accessors: {}\n    });\n\n    const accessors = internals.accessors;\n    const prototype = this.prototype;\n\n    function defineAccessor(_header) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!accessors[lHeader]) {\n        buildAccessors(prototype, _header);\n        accessors[lHeader] = true;\n      }\n    }\n\n    utils.isArray(header) ? header.forEach(defineAccessor) : defineAccessor(header);\n\n    return this;\n  }\n}\n\nAxiosHeaders.accessor(['Content-Type', 'Content-Length', 'Accept', 'Accept-Encoding', 'User-Agent', 'Authorization']);\n\n// reserved names hotfix\nutils.reduceDescriptors(AxiosHeaders.prototype, ({value}, key) => {\n  let mapped = key[0].toUpperCase() + key.slice(1); // map `set` => `Set`\n  return {\n    get: () => value,\n    set(headerValue) {\n      this[mapped] = headerValue;\n    }\n  }\n});\n\nutils.freezeMethods(AxiosHeaders);\n\nexport default AxiosHeaders;\n\n// Path: lib/core/settle.js\n'use strict';\n\nimport AxiosError from './AxiosError.js';\n\n/**\n * Resolve or reject a Promise based on response status.\n *\n * @param {Function} resolve A function that resolves the promise.\n * @param {Function} reject A function that rejects the promise.\n * @param {object} response The response.\n *\n * @returns {object} The response.\n */\nexport default function settle(resolve, reject, response) {\n  const validateStatus = response.config.validateStatus;\n  if (!response.status || !validateStatus || validateStatus(response.status)) {\n    resolve(response);\n  } else {\n    reject(new AxiosError(\n      'Request failed with status code ' + response.status,\n      [AxiosError.ERR_BAD_REQUEST, AxiosError.ERR_BAD_RESPONSE][Math.floor(response.status / 100) - 4],\n      response.config,\n      response.request,\n      response\n    ));\n  }\n}\n\n// Path: lib/helpers/parseProtocol.js\n'use strict';\n\nexport default function parseProtocol(url) {\n  const match = /^([-+\\w]{1,25})(:?\\/\\/|:)/.exec(url);\n  return match && match[1] || '';\n}\n\n// Path: lib/helpers/cookies.js\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n  // Standard browser envs support document.cookie\n  {\n    write(name, value, expires, path, domain, secure) {\n      const cookie = [name + '=' + encodeURIComponent(value)];\n\n      utils.isNumber(expires) && cookie.push('expires=' + new Date(expires).toGMTString());\n\n      utils.isString(path) && cookie.push('path=' + path);\n\n      utils.isString(domain) && cookie.push('domain=' + domain);\n\n      secure === true && cookie.push('secure');\n\n      document.cookie = cookie.join('; ');\n    },\n\n    read(name) {\n      const match = document.cookie.match(new RegExp('(^|;\\\\s*)(' + name + ')=([^;]*)'));\n      return (match ? decodeURIComponent(match[3]) : null);\n    },\n\n    remove(name) {\n      this.write(name, '', Date.now() - 86400000);\n    }\n  }\n\n  :\n\n  // Non-standard browser env (web workers, react-native) lack needed support.\n  {\n    write() {},\n    read() {\n      return null;\n    },\n    remove() {}\n  };\n\n\n// Path: lib/helpers/isURLSameOrigin.js\n'use strict';\n\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n// Standard browser envs have full support of the APIs needed to test\n// whether the request URL is of the same origin as current location.\n  (function standardBrowserEnv() {\n    const msie = /(msie|trident)/i.test(navigator.userAgent);\n    const urlParsingNode = document.createElement('a');\n    let originURL;\n\n    /**\n    * Parse a URL to discover its components\n    *\n    * @param {String} url The URL to be parsed\n    * @returns {Object}\n    */\n    function resolveURL(url) {\n      let href = url;\n\n      if (msie) {\n        // IE needs attribute set twice to normalize properties\n        urlParsingNode.setAttribute('href', href);\n        href = urlParsingNode.href;\n      }\n\n      urlParsingNode.setAttribute('href', href);\n\n      // urlParsingNode provides the UrlUtils interface - http://url.spec.whatwg.org/#urlutils\n      return {\n        href: urlParsingNode.href,\n        protocol: urlParsingNode.protocol ? urlParsingNode.protocol.replace(/:$/, '') : '',\n        host: urlParsingNode.host,\n        search: urlParsingNode.search ? urlParsingNode.search.replace(/^\\?/, '') : '',\n        hash: urlParsingNode.hash ? urlParsingNode.hash.replace(/^#/, '') : '',\n        hostname: urlParsingNode.hostname,\n        port: urlParsingNode.port,\n        pathname: (urlParsingNode.pathname.charAt(0) === '/') ?\n          urlParsingNode.pathname :\n          '/' + urlParsingNode.pathname\n      };\n    }\n\n    originURL = resolveURL(window.location.href);\n\n    /**\n    * Determine if a URL shares the same origin as the current location\n    *\n    * @param {String} requestURL The URL to test\n    * @returns {boolean} True if URL shares the same origin, otherwise false\n    */\n    return function isURLSameOrigin(requestURL) {\n      const parsed = (utils.isString(requestURL)) ? resolveURL(requestURL) : requestURL;\n      return (parsed.protocol === originURL.protocol &&\n          parsed.host === originURL.host);\n    };\n  })() :\n\n  // Non standard browser envs (web workers, react-native) lack needed support.\n  (function nonStandardBrowserEnv() {\n    return function isURLSameOrigin() {\n      return true;\n    };\n  })();\n\n// Path: lib/adapters/xhr.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport cookies from './../helpers/cookies.js';\nimport buildURL from './../helpers/buildURL.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport isURLSameOrigin from './../helpers/isURLSameOrigin.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport parseProtocol from '../helpers/parseProtocol.js';\nimport platform from '../platform/index.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport speedometer from '../helpers/speedometer.js';\n\nfunction progressEventReducer(listener, isDownloadStream) {\n  let bytesNotified = 0;\n  const _speedometer = speedometer(50, 250);\n\n  return e => {\n    const loaded = e.loaded;\n    const total = e.lengthComputable ? e.total : undefined;\n    const progressBytes = loaded - bytesNotified;\n    const rate = _speedometer(progressBytes);\n    const inRange = loaded <= total;\n\n    bytesNotified = loaded;\n\n    const data = {\n      loaded,\n      total,\n      progress: total ? (loaded / total) : undefined,\n      bytes: progressBytes,\n      rate: rate ? rate : undefined,\n      estimated: rate && total && inRange ? (total - loaded) / rate : undefined,\n      event: e\n    };\n\n    data[isDownloadStream ? 'download' : 'upload'] = true;\n\n    listener(data);\n  };\n}\n\nconst isXHRAdapterSupported = typeof XMLHttpRequest !== 'undefined';\n\nexport default isXHRAdapterSupported && function (config) {\n  return new Promise(function dispatchXhrRequest(resolve, reject) {\n    let requestData = config.data;\n    const requestHeaders = AxiosHeaders.from(config.headers).normalize();\n    let {responseType, withXSRFToken} = config;\n    let onCanceled;\n    function done() {\n      if (config.cancelToken) {\n        config.cancelToken.unsubscribe(onCanceled);\n      }\n\n      if (config.signal) {\n        config.signal.removeEventListener('abort', onCanceled);\n      }\n    }\n\n    let contentType;\n\n    if (utils.isFormData(requestData)) {\n      if (platform.hasStandardBrowserEnv || platform.hasStandardBrowserWebWorkerEnv) {\n        requestHeaders.setContentType(false); // Let the browser set it\n      } else if ((contentType = requestHeaders.getContentType()) !== false) {\n        // fix semicolon duplication issue for ReactNative FormData implementation\n        const [type, ...tokens] = contentType ? contentType.split(';').map(token => token.trim()).filter(Boolean) : [];\n        requestHeaders.setContentType([type || 'multipart/form-data', ...tokens].join('; '));\n      }\n    }\n\n    let request = new XMLHttpRequest();\n\n    // HTTP basic authentication\n    if (config.auth) {\n      const username = config.auth.username || '';\n      const password = config.auth.password ? unescape(encodeURIComponent(config.auth.password)) : '';\n      requestHeaders.set('Authorization', 'Basic ' + btoa(username + ':' + password));\n    }\n\n    const fullPath = buildFullPath(config.baseURL, config.url);\n\n    request.open(config.method.toUpperCase(), buildURL(fullPath, config.params, config.paramsSerializer), true);\n\n    // Set the request timeout in MS\n    request.timeout = config.timeout;\n\n    function onloadend() {\n      if (!request) {\n        return;\n      }\n      // Prepare the response\n      const responseHeaders = AxiosHeaders.from(\n        'getAllResponseHeaders' in request && request.getAllResponseHeaders()\n      );\n      const responseData = !responseType || responseType === 'text' || responseType === 'json' ?\n        request.responseText : request.response;\n      const response = {\n        data: responseData,\n        status: request.status,\n        statusText: request.statusText,\n        headers: responseHeaders,\n        config,\n        request\n      };\n\n      settle(function _resolve(value) {\n        resolve(value);\n        done();\n      }, function _reject(err) {\n        reject(err);\n        done();\n      }, response);\n\n      // Clean up request\n      request = null;\n    }\n\n    if ('onloadend' in request) {\n      // Use onloadend if available\n      request.onloadend = onloadend;\n    } else {\n      // Listen for ready state to emulate onloadend\n      request.onreadystatechange = function handleLoad() {\n        if (!request || request.readyState !== 4) {\n          return;\n        }\n\n        // The request errored out and we didn't get a response, this will be\n        // handled by onerror instead\n        // With one exception: request that using file: protocol, most browsers\n        // will return status as 0 even though it's a successful request\n        if (request.status === 0 && !(request.responseURL && request.responseURL.indexOf('file:') === 0)) {\n          return;\n        }\n        // readystate handler is calling before onerror or ontimeout handlers,\n        // so we should call onloadend on the next 'tick'\n        setTimeout(onloadend);\n      };\n    }\n\n    // Handle browser request cancellation (as opposed to a manual cancellation)\n    request.onabort = function handleAbort() {\n      if (!request) {\n        return;\n      }\n\n      reject(new AxiosError('Request aborted', AxiosError.ECONNABORTED, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle low level network errors\n    request.onerror = function handleError() {\n      // Real errors are hidden from us by the browser\n      // onerror should only fire if it's a network error\n      reject(new AxiosError('Network Error', AxiosError.ERR_NETWORK, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle timeout\n    request.ontimeout = function handleTimeout() {\n      let timeoutErrorMessage = config.timeout ? 'timeout of ' + config.timeout + 'ms exceeded' : 'timeout exceeded';\n      const transitional = config.transitional || transitionalDefaults;\n      if (config.timeoutErrorMessage) {\n        timeoutErrorMessage = config.timeoutErrorMessage;\n      }\n      reject(new AxiosError(\n        timeoutErrorMessage,\n        transitional.clarifyTimeoutError ? AxiosError.ETIMEDOUT : AxiosError.ECONNABORTED,\n        config,\n        request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Add xsrf header\n    // This is only done if running in a standard browser environment.\n    // Specifically not if we're in a web worker, or react-native.\n    if(platform.hasStandardBrowserEnv) {\n      withXSRFToken && utils.isFunction(withXSRFToken) && (withXSRFToken = withXSRFToken(config));\n\n      if (withXSRFToken || (withXSRFToken !== false && isURLSameOrigin(fullPath))) {\n        // Add xsrf header\n        const xsrfValue = config.xsrfHeaderName && config.xsrfCookieName && cookies.read(config.xsrfCookieName);\n\n        if (xsrfValue) {\n          requestHeaders.set(config.xsrfHeaderName, xsrfValue);\n        }\n      }\n    }\n\n    // Remove Content-Type if data is undefined\n    requestData === undefined && requestHeaders.setContentType(null);\n\n    // Add headers to the request\n    if ('setRequestHeader' in request) {\n      utils.forEach(requestHeaders.toJSON(), function setRequestHeader(val, key) {\n        request.setRequestHeader(key, val);\n      });\n    }\n\n    // Add withCredentials to request if needed\n    if (!utils.isUndefined(config.withCredentials)) {\n      request.withCredentials = !!config.withCredentials;\n    }\n\n    // Add responseType to request if needed\n    if (responseType && responseType !== 'json') {\n      request.responseType = config.responseType;\n    }\n\n    // Handle progress if needed\n    if (typeof config.onDownloadProgress === 'function') {\n      request.addEventListener('progress', progressEventReducer(config.onDownloadProgress, true));\n    }\n\n    // Not all browsers support upload events\n    if (typeof config.onUploadProgress === 'function' && request.upload) {\n      request.upload.addEventListener('progress', progressEventReducer(config.onUploadProgress));\n    }\n\n    if (config.cancelToken || config.signal) {\n      // Handle cancellation\n      // eslint-disable-next-line func-names\n      onCanceled = cancel => {\n        if (!request) {\n          return;\n        }\n        reject(!cancel || cancel.type ? new CanceledError(null, config, request) : cancel);\n        request.abort();\n        request = null;\n      };\n\n      config.cancelToken && config.cancelToken.subscribe(onCanceled);\n      if (config.signal) {\n        config.signal.aborted ? onCanceled() : config.signal.addEventListener('abort', onCanceled);\n      }\n    }\n\n    const protocol = parseProtocol(fullPath);\n\n    if (protocol && platform.protocols.indexOf(protocol) === -1) {\n      reject(new AxiosError('Unsupported protocol ' + protocol + ':', AxiosError.ERR_BAD_REQUEST, config));\n      return;\n    }\n\n\n    // Send the request\n    request.send(requestData || null);\n  });\n}\n\n// Path: lib/helpers/fromDataURI.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport parseProtocol from './parseProtocol.js';\nimport platform from '../platform/index.js';\n\nconst DATA_URL_PATTERN = /^(?:([^;]+);)?(?:[^;]+;)?(base64|),([\\s\\S]*)$/;\n\n/**\n * Parse data uri to a Buffer or Blob\n *\n * @param {String} uri\n * @param {?Boolean} asBlob\n * @param {?Object} options\n * @param {?Function} options.Blob\n *\n * @returns {Buffer|Blob}\n */\nexport default function fromDataURI(uri, asBlob, options) {\n  const _Blob = options && options.Blob || platform.classes.Blob;\n  const protocol = parseProtocol(uri);\n\n  if (asBlob === undefined && _Blob) {\n    asBlob = true;\n  }\n\n  if (protocol === 'data') {\n    uri = protocol.length ? uri.slice(protocol.length + 1) : uri;\n\n    const match = DATA_URL_PATTERN.exec(uri);\n\n    if (!match) {\n      throw new AxiosError('Invalid URL', AxiosError.ERR_INVALID_URL);\n    }\n\n    const mime = match[1];\n    const isBase64 = match[2];\n    const body = match[3];\n    const buffer = Buffer.from(decodeURIComponent(body), isBase64 ? 'base64' : 'utf8');\n\n    if (asBlob) {\n      if (!_Blob) {\n        throw new AxiosError('Blob is not supported', AxiosError.ERR_NOT_SUPPORT);\n      }\n\n      return new _Blob([buffer], {type: mime});\n    }\n\n    return buffer;\n  }\n\n  throw new AxiosError('Unsupported protocol ' + protocol, AxiosError.ERR_NOT_SUPPORT);\n}\n\n// Path: lib/helpers/readBlob.js\nconst {asyncIterator} = Symbol;\n\nconst readBlob = async function* (blob) {\n  if (blob.stream) {\n    yield* blob.stream()\n  } else if (blob.arrayBuffer) {\n    yield await blob.arrayBuffer()\n  } else if (blob[asyncIterator]) {\n    yield* blob[asyncIterator]();\n  } else {\n    yield blob;\n  }\n}\n\nexport default readBlob;\n\n// Path: lib/helpers/formDataToStream.js\nimport {TextEncoder} from 'util';\nimport {Readable} from 'stream';\nimport utils from \"../utils.js\";\nimport readBlob from \"./readBlob.js\";\n\nconst BOUNDARY_ALPHABET = utils.ALPHABET.ALPHA_DIGIT + '-_';\n\nconst textEncoder = new TextEncoder();\n\nconst CRLF = '\\r\\n';\nconst CRLF_BYTES = textEncoder.encode(CRLF);\nconst CRLF_BYTES_COUNT = 2;\n\nclass FormDataPart {\n  constructor(name, value) {\n    const {escapeName} = this.constructor;\n    const isStringValue = utils.isString(value);\n\n    let headers = `Content-Disposition: form-data; name=\"${escapeName(name)}\"${\n      !isStringValue && value.name ? `; filename=\"${escapeName(value.name)}\"` : ''\n    }${CRLF}`;\n\n    if (isStringValue) {\n      value = textEncoder.encode(String(value).replace(/\\r?\\n|\\r\\n?/g, CRLF));\n    } else {\n      headers += `Content-Type: ${value.type || \"application/octet-stream\"}${CRLF}`\n    }\n\n    this.headers = textEncoder.encode(headers + CRLF);\n\n    this.contentLength = isStringValue ? value.byteLength : value.size;\n\n    this.size = this.headers.byteLength + this.contentLength + CRLF_BYTES_COUNT;\n\n    this.name = name;\n    this.value = value;\n  }\n\n  async *encode(){\n    yield this.headers;\n\n    const {value} = this;\n\n    if(utils.isTypedArray(value)) {\n      yield value;\n    } else {\n      yield* readBlob(value);\n    }\n\n    yield CRLF_BYTES;\n  }\n\n  static escapeName(name) {\n      return String(name).replace(/[\\r\\n\"]/g, (match) => ({\n        '\\r' : '%0D',\n        '\\n' : '%0A',\n        '\"' : '%22',\n      }[match]));\n  }\n}\n\nconst formDataToStream = (form, headersHandler, options) => {\n  const {\n    tag = 'form-data-boundary',\n    size = 25,\n    boundary = tag + '-' + utils.generateString(size, BOUNDARY_ALPHABET)\n  } = options || {};\n\n  if(!utils.isFormData(form)) {\n    throw TypeError('FormData instance required');\n  }\n\n  if (boundary.length < 1 || boundary.length > 70) {\n    throw Error('boundary must be 10-70 characters long')\n  }\n\n  const boundaryBytes = textEncoder.encode('--' + boundary + CRLF);\n  const footerBytes = textEncoder.encode('--' + boundary + '--' + CRLF + CRLF);\n  let contentLength = footerBytes.byteLength;\n\n  const parts = Array.from(form.entries()).map(([name, value]) => {\n    const part = new FormDataPart(name, value);\n    contentLength += part.size;\n    return part;\n  });\n\n  contentLength += boundaryBytes.byteLength * parts.length;\n\n  contentLength = utils.toFiniteNumber(contentLength);\n\n  const computedHeaders = {\n    'Content-Type': `multipart/form-data; boundary=${boundary}`\n  }\n\n  if (Number.isFinite(contentLength)) {\n    computedHeaders['Content-Length'] = contentLength;\n  }\n\n  headersHandler && headersHandler(computedHeaders);\n\n  return Readable.from((async function *() {\n    for(const part of parts) {\n      yield boundaryBytes;\n      yield* part.encode();\n    }\n\n    yield footerBytes;\n  })());\n};\n\nexport default formDataToStream;\n\n// Path: lib/helpers/callbackify.js\nimport utils from \"../utils.js\";\n\nconst callbackify = (fn, reducer) => {\n  return utils.isAsyncFn(fn) ? function (...args) {\n    const cb = args.pop();\n    fn.apply(this, args).then((value) => {\n      try {\n        reducer ? cb(null, ...reducer(value)) : cb(null, value);\n      } catch (err) {\n        cb(err);\n      }\n    }, cb);\n  } : fn;\n}\n\nexport default callbackify;\n\n// Path: lib/helpers/ZlibHeaderTransformStream.js\n\"use strict\";\n\nimport stream from \"stream\";\n\nclass ZlibHeaderTransformStream extends stream.Transform {\n  __transform(chunk, encoding, callback) {\n    this.push(chunk);\n    callback();\n  }\n\n  _transform(chunk, encoding, callback) {\n    if (chunk.length !== 0) {\n      this._transform = this.__transform;\n\n      // Add Default Compression headers if no zlib headers are present\n      if (chunk[0] !== 120) { // Hex: 78\n        const header = Buffer.alloc(2);\n        header[0] = 120; // Hex: 78\n        header[1] = 156; // Hex: 9C \n        this.push(header, encoding);\n      }\n    }\n\n    this.__transform(chunk, encoding, callback);\n  }\n}\n\nexport default ZlibHeaderTransformStream;\n\n// Path: lib/helpers/throttle.js\n'use strict';\n\n/**\n * Throttle decorator\n * @param {Function} fn\n * @param {Number} freq\n * @return {Function}\n */\nfunction throttle(fn, freq) {\n  let timestamp = 0;\n  const threshold = 1000 / freq;\n  let timer = null;\n  return function throttled(force, args) {\n    const now = Date.now();\n    if (force || now - timestamp > threshold) {\n      if (timer) {\n        clearTimeout(timer);\n        timer = null;\n      }\n      timestamp = now;\n      return fn.apply(null, args);\n    }\n    if (!timer) {\n      timer = setTimeout(() => {\n        timer = null;\n        timestamp = Date.now();\n        return fn.apply(null, args);\n      }, threshold - (now - timestamp));\n    }\n  };\n}\n\nexport default throttle;\n\n// Path: lib/helpers/AxiosTransformStream.js\n'use strict';\n\nimport stream from 'stream';\nimport utils from '../utils.js';\nimport throttle from './throttle.js';\nimport speedometer from './speedometer.js';\n\nconst kInternals = Symbol('internals');\n\nclass AxiosTransformStream extends stream.Transform{\n  constructor(options) {\n    options = utils.toFlatObject(options, {\n      maxRate: 0,\n      chunkSize: 64 * 1024,\n      minChunkSize: 100,\n      timeWindow: 500,\n      ticksRate: 2,\n      samplesCount: 15\n    }, null, (prop, source) => {\n      return !utils.isUndefined(source[prop]);\n    });\n\n    super({\n      readableHighWaterMark: options.chunkSize\n    });\n\n    const self = this;\n\n    const internals = this[kInternals] = {\n      length: options.length,\n      timeWindow: options.timeWindow,\n      ticksRate: options.ticksRate,\n      chunkSize: options.chunkSize,\n      maxRate: options.maxRate,\n      minChunkSize: options.minChunkSize,\n      bytesSeen: 0,\n      isCaptured: false,\n      notifiedBytesLoaded: 0,\n      ts: Date.now(),\n      bytes: 0,\n      onReadCallback: null\n    };\n\n    const _speedometer = speedometer(internals.ticksRate * options.samplesCount, internals.timeWindow);\n\n    this.on('newListener', event => {\n      if (event === 'progress') {\n        if (!internals.isCaptured) {\n          internals.isCaptured = true;\n        }\n      }\n    });\n\n    let bytesNotified = 0;\n\n    internals.updateProgress = throttle(function throttledHandler() {\n      const totalBytes = internals.length;\n      const bytesTransferred = internals.bytesSeen;\n      const progressBytes = bytesTransferred - bytesNotified;\n      if (!progressBytes || self.destroyed) return;\n\n      const rate = _speedometer(progressBytes);\n\n      bytesNotified = bytesTransferred;\n\n      process.nextTick(() => {\n        self.emit('progress', {\n          'loaded': bytesTransferred,\n          'total': totalBytes,\n          'progress': totalBytes ? (bytesTransferred / totalBytes) : undefined,\n          'bytes': progressBytes,\n          'rate': rate ? rate : undefined,\n          'estimated': rate && totalBytes && bytesTransferred <= totalBytes ?\n            (totalBytes - bytesTransferred) / rate : undefined\n        });\n      });\n    }, internals.ticksRate);\n\n    const onFinish = () => {\n      internals.updateProgress(true);\n    };\n\n    this.once('end', onFinish);\n    this.once('error', onFinish);\n  }\n\n  _read(size) {\n    const internals = this[kInternals];\n\n    if (internals.onReadCallback) {\n      internals.onReadCallback();\n    }\n\n    return super._read(size);\n  }\n\n  _transform(chunk, encoding, callback) {\n    const self = this;\n    const internals = this[kInternals];\n    const maxRate = internals.maxRate;\n\n    const readableHighWaterMark = this.readableHighWaterMark;\n\n    const timeWindow = internals.timeWindow;\n\n    const divider = 1000 / timeWindow;\n    const bytesThreshold = (maxRate / divider);\n    const minChunkSize = internals.minChunkSize !== false ? Math.max(internals.minChunkSize, bytesThreshold * 0.01) : 0;\n\n    function pushChunk(_chunk, _callback) {\n      const bytes = Buffer.byteLength(_chunk);\n      internals.bytesSeen += bytes;\n      internals.bytes += bytes;\n\n      if (internals.isCaptured) {\n        internals.updateProgress();\n      }\n\n      if (self.push(_chunk)) {\n        process.nextTick(_callback);\n      } else {\n        internals.onReadCallback = () => {\n          internals.onReadCallback = null;\n          process.nextTick(_callback);\n        };\n      }\n    }\n\n    const transformChunk = (_chunk, _callback) => {\n      const chunkSize = Buffer.byteLength(_chunk);\n      let chunkRemainder = null;\n      let maxChunkSize = readableHighWaterMark;\n      let bytesLeft;\n      let passed = 0;\n\n      if (maxRate) {\n        const now = Date.now();\n\n        if (!internals.ts || (passed = (now - internals.ts)) >= timeWindow) {\n          internals.ts = now;\n          bytesLeft = bytesThreshold - internals.bytes;\n          internals.bytes = bytesLeft < 0 ? -bytesLeft : 0;\n          passed = 0;\n        }\n\n        bytesLeft = bytesThreshold - internals.bytes;\n      }\n\n      if (maxRate) {\n        if (bytesLeft <= 0) {\n          // next time window\n          return setTimeout(() => {\n            _callback(null, _chunk);\n          }, timeWindow - passed);\n        }\n\n        if (bytesLeft < maxChunkSize) {\n          maxChunkSize = bytesLeft;\n        }\n      }\n\n      if (maxChunkSize && chunkSize > maxChunkSize && (chunkSize - maxChunkSize) > minChunkSize) {\n        chunkRemainder = _chunk.subarray(maxChunkSize);\n        _chunk = _chunk.subarray(0, maxChunkSize);\n      }\n\n      pushChunk(_chunk, chunkRemainder ? () => {\n        process.nextTick(_callback, null, chunkRemainder);\n      } : _callback);\n    };\n\n    transformChunk(chunk, function transformNextChunk(err, _chunk) {\n      if (err) {\n        return callback(err);\n      }\n\n      if (_chunk) {\n        transformChunk(_chunk, transformNextChunk);\n      } else {\n        callback(null);\n      }\n    });\n  }\n\n  setLength(length) {\n    this[kInternals].length = +length;\n    return this;\n  }\n}\n\nexport default AxiosTransformStream;\n\n// Path: lib/env/data.js\nexport const VERSION = \"1.6.8\";\n// Path: lib/adapters/http.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport buildURL from './../helpers/buildURL.js';\nimport {getProxyForUrl} from 'proxy-from-env';\nimport http from 'http';\nimport https from 'https';\nimport util from 'util';\nimport followRedirects from 'follow-redirects';\nimport zlib from 'zlib';\nimport {VERSION} from '../env/data.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport platform from '../platform/index.js';\nimport fromDataURI from '../helpers/fromDataURI.js';\nimport stream from 'stream';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport AxiosTransformStream from '../helpers/AxiosTransformStream.js';\nimport {EventEmitter} from 'events';\nimport formDataToStream from \"../helpers/formDataToStream.js\";\nimport readBlob from \"../helpers/readBlob.js\";\nimport ZlibHeaderTransformStream from '../helpers/ZlibHeaderTransformStream.js';\nimport callbackify from \"../helpers/callbackify.js\";\n\nconst zlibOptions = {\n  flush: zlib.constants.Z_SYNC_FLUSH,\n  finishFlush: zlib.constants.Z_SYNC_FLUSH\n};\n\nconst brotliOptions = {\n  flush: zlib.constants.BROTLI_OPERATION_FLUSH,\n  finishFlush: zlib.constants.BROTLI_OPERATION_FLUSH\n}\n\nconst isBrotliSupported = utils.isFunction(zlib.createBrotliDecompress);\n\nconst {http: httpFollow, https: httpsFollow} = followRedirects;\n\nconst isHttps = /https:?/;\n\nconst supportedProtocols = platform.protocols.map(protocol => {\n  return protocol + ':';\n});\n\n/**\n * If the proxy or config beforeRedirects functions are defined, call them with the options\n * object.\n *\n * @param {Object<string, any>} options - The options object that was passed to the request.\n *\n * @returns {Object<string, any>}\n */\nfunction dispatchBeforeRedirect(options, responseDetails) {\n  if (options.beforeRedirects.proxy) {\n...\n", "needle_token_start": 3821, "needle_token_end": 4060, "code_context_ntokens": 15877, "output": ["```javascript\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default function buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n```"]}
{"repo": "axios/axios", "name": "combineURLs", "language": "typescript", "path": "lib/helpers/combineURLs.js", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to merge a base URL with a relative path to form a complete URL.\n2. **Input**: Two strings are required as inputs; the first string represents the base URL, and the second string represents the relative URL.\n3. **Output**: Outputs a single string that represents the fully combined URL.\n4. **Procedure**: The function first checks if the relative URL is provided. If it is, it trims any trailing slashes from the base URL and any leading slashes from the relative URL before concatenating them with a single slash in between. If the relative URL is not provided, it simply returns the base URL as is.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/platform/node/classes/FormData.js\nimport FormData from 'form-data';\n\nexport default FormData;\n\n// Path: lib/platform/node/classes/URLSearchParams.js\n'use strict';\n\nimport url from 'url';\nexport default url.URLSearchParams;\n\n// Path: lib/platform/node/index.js\nimport URLSearchParams from './classes/URLSearchParams.js'\nimport FormData from './classes/FormData.js'\n\nexport default {\n  isNode: true,\n  classes: {\n    URLSearchParams,\n    FormData,\n    Blob: typeof Blob !== 'undefined' && Blob || null\n  },\n  protocols: [ 'http', 'https', 'file', 'data' ]\n};\n\n// Path: lib/platform/common/utils.js\nconst hasBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';\n\n/**\n * Determine if we're running in a standard browser environment\n *\n * This allows axios to run in a web worker, and react-native.\n * Both environments support XMLHttpRequest, but not fully standard globals.\n *\n * web workers:\n *  typeof window -> undefined\n *  typeof document -> undefined\n *\n * react-native:\n *  navigator.product -> 'ReactNative'\n * nativescript\n *  navigator.product -> 'NativeScript' or 'NS'\n *\n * @returns {boolean}\n */\nconst hasStandardBrowserEnv = (\n  (product) => {\n    return hasBrowserEnv && ['ReactNative', 'NativeScript', 'NS'].indexOf(product) < 0\n  })(typeof navigator !== 'undefined' && navigator.product);\n\n/**\n * Determine if we're running in a standard browser webWorker environment\n *\n * Although the `isStandardBrowserEnv` method indicates that\n * `allows axios to run in a web worker`, the WebWorker will still be\n * filtered out due to its judgment standard\n * `typeof window !== 'undefined' && typeof document !== 'undefined'`.\n * This leads to a problem when axios post `FormData` in webWorker\n */\nconst hasStandardBrowserWebWorkerEnv = (() => {\n  return (\n    typeof WorkerGlobalScope !== 'undefined' &&\n    // eslint-disable-next-line no-undef\n    self instanceof WorkerGlobalScope &&\n    typeof self.importScripts === 'function'\n  );\n})();\n\nexport {\n  hasBrowserEnv,\n  hasStandardBrowserWebWorkerEnv,\n  hasStandardBrowserEnv\n}\n\n// Path: lib/platform/index.js\nimport platform from './node/index.js';\nimport * as utils from './common/utils.js';\n\nexport default {\n  ...utils,\n  ...platform\n}\n\n// Path: lib/helpers/combineURLs.js\n'use strict';\n\n/**\n * Creates a new URL by combining the specified URLs\n *\n * @param {string} baseURL The base URL\n * @param {string} relativeURL The relative URL\n *\n * @returns {string} The combined URL\n */\nexport default \nfunction combineURLs(baseURL, relativeURL) {\n  return relativeURL\n    ? baseURL.replace(/\\/?\\/$/, '') + '/' + relativeURL.replace(/^\\/+/, '')\n    : baseURL;\n}\n\n// Path: lib/helpers/isAbsoluteURL.js\n'use strict';\n\n/**\n * Determines whether the specified URL is absolute\n *\n * @param {string} url The URL to test\n *\n * @returns {boolean} True if the specified URL is absolute, otherwise false\n */\nexport default function isAbsoluteURL(url) {\n  // A URL is considered absolute if it begins with \"<scheme>://\" or \"//\" (protocol-relative URL).\n  // RFC 3986 defines scheme name as a sequence of characters beginning with a letter and followed\n  // by any combination of letters, digits, plus, period, or hyphen.\n  return /^([a-z][a-z\\d+\\-.]*:)?\\/\\//i.test(url);\n}\n\n// Path: lib/core/buildFullPath.js\n'use strict';\n\nimport isAbsoluteURL from '../helpers/isAbsoluteURL.js';\nimport combineURLs from '../helpers/combineURLs.js';\n\n/**\n * Creates a new URL by combining the baseURL with the requestedURL,\n * only when the requestedURL is not already an absolute URL.\n * If the requestURL is absolute, this function returns the requestedURL untouched.\n *\n * @param {string} baseURL The base URL\n * @param {string} requestedURL Absolute or relative URL to combine\n *\n * @returns {string} The combined full path\n */\nexport default function buildFullPath(baseURL, requestedURL) {\n  if (baseURL && !isAbsoluteURL(requestedURL)) {\n    return combineURLs(baseURL, requestedURL);\n  }\n  return requestedURL;\n}\n\n// Path: lib/helpers/bind.js\n'use strict';\n\nexport default function bind(fn, thisArg) {\n  return function wrap() {\n    return fn.apply(thisArg, arguments);\n  };\n}\n\n// Path: lib/utils.js\n'use strict';\n\nimport bind from './helpers/bind.js';\n\n// utils is a library of generic helper functions non-specific to axios\n\nconst {toString} = Object.prototype;\nconst {getPrototypeOf} = Object;\n\nconst kindOf = (cache => thing => {\n    const str = toString.call(thing);\n    return cache[str] || (cache[str] = str.slice(8, -1).toLowerCase());\n})(Object.create(null));\n\nconst kindOfTest = (type) => {\n  type = type.toLowerCase();\n  return (thing) => kindOf(thing) === type\n}\n\nconst typeOfTest = type => thing => typeof thing === type;\n\n/**\n * Determine if a value is an Array\n *\n * @param {Object} val The value to test\n *\n * @returns {boolean} True if value is an Array, otherwise false\n */\nconst {isArray} = Array;\n\n/**\n * Determine if a value is undefined\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if the value is undefined, otherwise false\n */\nconst isUndefined = typeOfTest('undefined');\n\n/**\n * Determine if a value is a Buffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Buffer, otherwise false\n */\nfunction isBuffer(val) {\n  return val !== null && !isUndefined(val) && val.constructor !== null && !isUndefined(val.constructor)\n    && isFunction(val.constructor.isBuffer) && val.constructor.isBuffer(val);\n}\n\n/**\n * Determine if a value is an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is an ArrayBuffer, otherwise false\n */\nconst isArrayBuffer = kindOfTest('ArrayBuffer');\n\n\n/**\n * Determine if a value is a view on an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a view on an ArrayBuffer, otherwise false\n */\nfunction isArrayBufferView(val) {\n  let result;\n  if ((typeof ArrayBuffer !== 'undefined') && (ArrayBuffer.isView)) {\n    result = ArrayBuffer.isView(val);\n  } else {\n    result = (val) && (val.buffer) && (isArrayBuffer(val.buffer));\n  }\n  return result;\n}\n\n/**\n * Determine if a value is a String\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a String, otherwise false\n */\nconst isString = typeOfTest('string');\n\n/**\n * Determine if a value is a Function\n *\n * @param {*} val The value to test\n * @returns {boolean} True if value is a Function, otherwise false\n */\nconst isFunction = typeOfTest('function');\n\n/**\n * Determine if a value is a Number\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Number, otherwise false\n */\nconst isNumber = typeOfTest('number');\n\n/**\n * Determine if a value is an Object\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an Object, otherwise false\n */\nconst isObject = (thing) => thing !== null && typeof thing === 'object';\n\n/**\n * Determine if a value is a Boolean\n *\n * @param {*} thing The value to test\n * @returns {boolean} True if value is a Boolean, otherwise false\n */\nconst isBoolean = thing => thing === true || thing === false;\n\n/**\n * Determine if a value is a plain Object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a plain Object, otherwise false\n */\nconst isPlainObject = (val) => {\n  if (kindOf(val) !== 'object') {\n    return false;\n  }\n\n  const prototype = getPrototypeOf(val);\n  return (prototype === null || prototype === Object.prototype || Object.getPrototypeOf(prototype) === null) && !(Symbol.toStringTag in val) && !(Symbol.iterator in val);\n}\n\n/**\n * Determine if a value is a Date\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Date, otherwise false\n */\nconst isDate = kindOfTest('Date');\n\n/**\n * Determine if a value is a File\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFile = kindOfTest('File');\n\n/**\n * Determine if a value is a Blob\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Blob, otherwise false\n */\nconst isBlob = kindOfTest('Blob');\n\n/**\n * Determine if a value is a FileList\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFileList = kindOfTest('FileList');\n\n/**\n * Determine if a value is a Stream\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Stream, otherwise false\n */\nconst isStream = (val) => isObject(val) && isFunction(val.pipe);\n\n/**\n * Determine if a value is a FormData\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an FormData, otherwise false\n */\nconst isFormData = (thing) => {\n  let kind;\n  return thing && (\n    (typeof FormData === 'function' && thing instanceof FormData) || (\n      isFunction(thing.append) && (\n        (kind = kindOf(thing)) === 'formdata' ||\n        // detect form-data instance\n        (kind === 'object' && isFunction(thing.toString) && thing.toString() === '[object FormData]')\n      )\n    )\n  )\n}\n\n/**\n * Determine if a value is a URLSearchParams object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a URLSearchParams object, otherwise false\n */\nconst isURLSearchParams = kindOfTest('URLSearchParams');\n\n/**\n * Trim excess whitespace off the beginning and end of a string\n *\n * @param {String} str The String to trim\n *\n * @returns {String} The String freed of excess whitespace\n */\nconst trim = (str) => str.trim ?\n  str.trim() : str.replace(/^[\\s\\uFEFF\\xA0]+|[\\s\\uFEFF\\xA0]+$/g, '');\n\n/**\n * Iterate over an Array or an Object invoking a function for each item.\n *\n * If `obj` is an Array callback will be called passing\n * the value, index, and complete array for each item.\n *\n * If 'obj' is an Object callback will be called passing\n * the value, key, and complete object for each property.\n *\n * @param {Object|Array} obj The object to iterate\n * @param {Function} fn The callback to invoke for each item\n *\n * @param {Boolean} [allOwnKeys = false]\n * @returns {any}\n */\nfunction forEach(obj, fn, {allOwnKeys = false} = {}) {\n  // Don't bother if no value provided\n  if (obj === null || typeof obj === 'undefined') {\n    return;\n  }\n\n  let i;\n  let l;\n\n  // Force an array if not already something iterable\n  if (typeof obj !== 'object') {\n    /*eslint no-param-reassign:0*/\n    obj = [obj];\n  }\n\n  if (isArray(obj)) {\n    // Iterate over array values\n    for (i = 0, l = obj.length; i < l; i++) {\n      fn.call(null, obj[i], i, obj);\n    }\n  } else {\n    // Iterate over object keys\n    const keys = allOwnKeys ? Object.getOwnPropertyNames(obj) : Object.keys(obj);\n    const len = keys.length;\n    let key;\n\n    for (i = 0; i < len; i++) {\n      key = keys[i];\n      fn.call(null, obj[key], key, obj);\n    }\n  }\n}\n\nfunction findKey(obj, key) {\n  key = key.toLowerCase();\n  const keys = Object.keys(obj);\n  let i = keys.length;\n  let _key;\n  while (i-- > 0) {\n    _key = keys[i];\n    if (key === _key.toLowerCase()) {\n      return _key;\n    }\n  }\n  return null;\n}\n\nconst _global = (() => {\n  /*eslint no-undef:0*/\n  if (typeof globalThis !== \"undefined\") return globalThis;\n  return typeof self !== \"undefined\" ? self : (typeof window !== 'undefined' ? window : global)\n})();\n\nconst isContextDefined = (context) => !isUndefined(context) && context !== _global;\n\n/**\n * Accepts varargs expecting each argument to be an object, then\n * immutably merges the properties of each object and returns result.\n *\n * When multiple objects contain the same key the later object in\n * the arguments list will take precedence.\n *\n * Example:\n *\n * ```js\n * var result = merge({foo: 123}, {foo: 456});\n * console.log(result.foo); // outputs 456\n * ```\n *\n * @param {Object} obj1 Object to merge\n *\n * @returns {Object} Result of all merge properties\n */\nfunction merge(/* obj1, obj2, obj3, ... */) {\n  const {caseless} = isContextDefined(this) && this || {};\n  const result = {};\n  const assignValue = (val, key) => {\n    const targetKey = caseless && findKey(result, key) || key;\n    if (isPlainObject(result[targetKey]) && isPlainObject(val)) {\n      result[targetKey] = merge(result[targetKey], val);\n    } else if (isPlainObject(val)) {\n      result[targetKey] = merge({}, val);\n    } else if (isArray(val)) {\n      result[targetKey] = val.slice();\n    } else {\n      result[targetKey] = val;\n    }\n  }\n\n  for (let i = 0, l = arguments.length; i < l; i++) {\n    arguments[i] && forEach(arguments[i], assignValue);\n  }\n  return result;\n}\n\n/**\n * Extends object a by mutably adding to it the properties of object b.\n *\n * @param {Object} a The object to be extended\n * @param {Object} b The object to copy properties from\n * @param {Object} thisArg The object to bind function to\n *\n * @param {Boolean} [allOwnKeys]\n * @returns {Object} The resulting value of object a\n */\nconst extend = (a, b, thisArg, {allOwnKeys}= {}) => {\n  forEach(b, (val, key) => {\n    if (thisArg && isFunction(val)) {\n      a[key] = bind(val, thisArg);\n    } else {\n      a[key] = val;\n    }\n  }, {allOwnKeys});\n  return a;\n}\n\n/**\n * Remove byte order marker. This catches EF BB BF (the UTF-8 BOM)\n *\n * @param {string} content with BOM\n *\n * @returns {string} content value without BOM\n */\nconst stripBOM = (content) => {\n  if (content.charCodeAt(0) === 0xFEFF) {\n    content = content.slice(1);\n  }\n  return content;\n}\n\n/**\n * Inherit the prototype methods from one constructor into another\n * @param {function} constructor\n * @param {function} superConstructor\n * @param {object} [props]\n * @param {object} [descriptors]\n *\n * @returns {void}\n */\nconst inherits = (constructor, superConstructor, props, descriptors) => {\n  constructor.prototype = Object.create(superConstructor.prototype, descriptors);\n  constructor.prototype.constructor = constructor;\n  Object.defineProperty(constructor, 'super', {\n    value: superConstructor.prototype\n  });\n  props && Object.assign(constructor.prototype, props);\n}\n\n/**\n * Resolve object with deep prototype chain to a flat object\n * @param {Object} sourceObj source object\n * @param {Object} [destObj]\n * @param {Function|Boolean} [filter]\n * @param {Function} [propFilter]\n *\n * @returns {Object}\n */\nconst toFlatObject = (sourceObj, destObj, filter, propFilter) => {\n  let props;\n  let i;\n  let prop;\n  const merged = {};\n\n  destObj = destObj || {};\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  if (sourceObj == null) return destObj;\n\n  do {\n    props = Object.getOwnPropertyNames(sourceObj);\n    i = props.length;\n    while (i-- > 0) {\n      prop = props[i];\n      if ((!propFilter || propFilter(prop, sourceObj, destObj)) && !merged[prop]) {\n        destObj[prop] = sourceObj[prop];\n        merged[prop] = true;\n      }\n    }\n    sourceObj = filter !== false && getPrototypeOf(sourceObj);\n  } while (sourceObj && (!filter || filter(sourceObj, destObj)) && sourceObj !== Object.prototype);\n\n  return destObj;\n}\n\n/**\n * Determines whether a string ends with the characters of a specified string\n *\n * @param {String} str\n * @param {String} searchString\n * @param {Number} [position= 0]\n *\n * @returns {boolean}\n */\nconst endsWith = (str, searchString, position) => {\n  str = String(str);\n  if (position === undefined || position > str.length) {\n    position = str.length;\n  }\n  position -= searchString.length;\n  const lastIndex = str.indexOf(searchString, position);\n  return lastIndex !== -1 && lastIndex === position;\n}\n\n\n/**\n * Returns new array from array like object or null if failed\n *\n * @param {*} [thing]\n *\n * @returns {?Array}\n */\nconst toArray = (thing) => {\n  if (!thing) return null;\n  if (isArray(thing)) return thing;\n  let i = thing.length;\n  if (!isNumber(i)) return null;\n  const arr = new Array(i);\n  while (i-- > 0) {\n    arr[i] = thing[i];\n  }\n  return arr;\n}\n\n/**\n * Checking if the Uint8Array exists and if it does, it returns a function that checks if the\n * thing passed in is an instance of Uint8Array\n *\n * @param {TypedArray}\n *\n * @returns {Array}\n */\n// eslint-disable-next-line func-names\nconst isTypedArray = (TypedArray => {\n  // eslint-disable-next-line func-names\n  return thing => {\n    return TypedArray && thing instanceof TypedArray;\n  };\n})(typeof Uint8Array !== 'undefined' && getPrototypeOf(Uint8Array));\n\n/**\n * For each entry in the object, call the function with the key and value.\n *\n * @param {Object<any, any>} obj - The object to iterate over.\n * @param {Function} fn - The function to call for each entry.\n *\n * @returns {void}\n */\nconst forEachEntry = (obj, fn) => {\n  const generator = obj && obj[Symbol.iterator];\n\n  const iterator = generator.call(obj);\n\n  let result;\n\n  while ((result = iterator.next()) && !result.done) {\n    const pair = result.value;\n    fn.call(obj, pair[0], pair[1]);\n  }\n}\n\n/**\n * It takes a regular expression and a string, and returns an array of all the matches\n *\n * @param {string} regExp - The regular expression to match against.\n * @param {string} str - The string to search.\n *\n * @returns {Array<boolean>}\n */\nconst matchAll = (regExp, str) => {\n  let matches;\n  const arr = [];\n\n  while ((matches = regExp.exec(str)) !== null) {\n    arr.push(matches);\n  }\n\n  return arr;\n}\n\n/* Checking if the kindOfTest function returns true when passed an HTMLFormElement. */\nconst isHTMLForm = kindOfTest('HTMLFormElement');\n\nconst toCamelCase = str => {\n  return str.toLowerCase().replace(/[-_\\s]([a-z\\d])(\\w*)/g,\n    function replacer(m, p1, p2) {\n      return p1.toUpperCase() + p2;\n    }\n  );\n};\n\n/* Creating a function that will check if an object has a property. */\nconst hasOwnProperty = (({hasOwnProperty}) => (obj, prop) => hasOwnProperty.call(obj, prop))(Object.prototype);\n\n/**\n * Determine if a value is a RegExp object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a RegExp object, otherwise false\n */\nconst isRegExp = kindOfTest('RegExp');\n\nconst reduceDescriptors = (obj, reducer) => {\n  const descriptors = Object.getOwnPropertyDescriptors(obj);\n  const reducedDescriptors = {};\n\n  forEach(descriptors, (descriptor, name) => {\n    let ret;\n    if ((ret = reducer(descriptor, name, obj)) !== false) {\n      reducedDescriptors[name] = ret || descriptor;\n    }\n  });\n\n  Object.defineProperties(obj, reducedDescriptors);\n}\n\n/**\n * Makes all methods read-only\n * @param {Object} obj\n */\n\nconst freezeMethods = (obj) => {\n  reduceDescriptors(obj, (descriptor, name) => {\n    // skip restricted props in strict mode\n    if (isFunction(obj) && ['arguments', 'caller', 'callee'].indexOf(name) !== -1) {\n      return false;\n    }\n\n    const value = obj[name];\n\n    if (!isFunction(value)) return;\n\n    descriptor.enumerable = false;\n\n    if ('writable' in descriptor) {\n      descriptor.writable = false;\n      return;\n    }\n\n    if (!descriptor.set) {\n      descriptor.set = () => {\n        throw Error('Can not rewrite read-only method \\'' + name + '\\'');\n      };\n    }\n  });\n}\n\nconst toObjectSet = (arrayOrString, delimiter) => {\n  const obj = {};\n\n  const define = (arr) => {\n    arr.forEach(value => {\n      obj[value] = true;\n    });\n  }\n\n  isArray(arrayOrString) ? define(arrayOrString) : define(String(arrayOrString).split(delimiter));\n\n  return obj;\n}\n\nconst noop = () => {}\n\nconst toFiniteNumber = (value, defaultValue) => {\n  value = +value;\n  return Number.isFinite(value) ? value : defaultValue;\n}\n\nconst ALPHA = 'abcdefghijklmnopqrstuvwxyz'\n\nconst DIGIT = '0123456789';\n\nconst ALPHABET = {\n  DIGIT,\n  ALPHA,\n  ALPHA_DIGIT: ALPHA + ALPHA.toUpperCase() + DIGIT\n}\n\nconst generateString = (size = 16, alphabet = ALPHABET.ALPHA_DIGIT) => {\n  let str = '';\n  const {length} = alphabet;\n  while (size--) {\n    str += alphabet[Math.random() * length|0]\n  }\n\n  return str;\n}\n\n/**\n * If the thing is a FormData object, return true, otherwise return false.\n *\n * @param {unknown} thing - The thing to check.\n *\n * @returns {boolean}\n */\nfunction isSpecCompliantForm(thing) {\n  return !!(thing && isFunction(thing.append) && thing[Symbol.toStringTag] === 'FormData' && thing[Symbol.iterator]);\n}\n\nconst toJSONObject = (obj) => {\n  const stack = new Array(10);\n\n  const visit = (source, i) => {\n\n    if (isObject(source)) {\n      if (stack.indexOf(source) >= 0) {\n        return;\n      }\n\n      if(!('toJSON' in source)) {\n        stack[i] = source;\n        const target = isArray(source) ? [] : {};\n\n        forEach(source, (value, key) => {\n          const reducedValue = visit(value, i + 1);\n          !isUndefined(reducedValue) && (target[key] = reducedValue);\n        });\n\n        stack[i] = undefined;\n\n        return target;\n      }\n    }\n\n    return source;\n  }\n\n  return visit(obj, 0);\n}\n\nconst isAsyncFn = kindOfTest('AsyncFunction');\n\nconst isThenable = (thing) =>\n  thing && (isObject(thing) || isFunction(thing)) && isFunction(thing.then) && isFunction(thing.catch);\n\nexport default {\n  isArray,\n  isArrayBuffer,\n  isBuffer,\n  isFormData,\n  isArrayBufferView,\n  isString,\n  isNumber,\n  isBoolean,\n  isObject,\n  isPlainObject,\n  isUndefined,\n  isDate,\n  isFile,\n  isBlob,\n  isRegExp,\n  isFunction,\n  isStream,\n  isURLSearchParams,\n  isTypedArray,\n  isFileList,\n  forEach,\n  merge,\n  extend,\n  trim,\n  stripBOM,\n  inherits,\n  toFlatObject,\n  kindOf,\n  kindOfTest,\n  endsWith,\n  toArray,\n  forEachEntry,\n  matchAll,\n  isHTMLForm,\n  hasOwnProperty,\n  hasOwnProp: hasOwnProperty, // an alias to avoid ESLint no-prototype-builtins detection\n  reduceDescriptors,\n  freezeMethods,\n  toObjectSet,\n  toCamelCase,\n  noop,\n  toFiniteNumber,\n  findKey,\n  global: _global,\n  isContextDefined,\n  ALPHABET,\n  generateString,\n  isSpecCompliantForm,\n  toJSONObject,\n  isAsyncFn,\n  isThenable\n};\n\n// Path: lib/core/AxiosError.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: utils.toJSONObject(this.config),\n      code: this.code,\n      status: this.response && this.response.status ? this.response.status : null\n    };\n  }\n});\n\nconst prototype = AxiosError.prototype;\nconst descriptors = {};\n\n[\n  'ERR_BAD_OPTION_VALUE',\n  'ERR_BAD_OPTION',\n  'ECONNABORTED',\n  'ETIMEDOUT',\n  'ERR_NETWORK',\n  'ERR_FR_TOO_MANY_REDIRECTS',\n  'ERR_DEPRECATED',\n  'ERR_BAD_RESPONSE',\n  'ERR_BAD_REQUEST',\n  'ERR_CANCELED',\n  'ERR_NOT_SUPPORT',\n  'ERR_INVALID_URL'\n// eslint-disable-next-line func-names\n].forEach(code => {\n  descriptors[code] = {value: code};\n});\n\nObject.defineProperties(AxiosError, descriptors);\nObject.defineProperty(prototype, 'isAxiosError', {value: true});\n\n// eslint-disable-next-line func-names\nAxiosError.from = (error, code, config, request, response, customProps) => {\n  const axiosError = Object.create(prototype);\n\n  utils.toFlatObject(error, axiosError, function filter(obj) {\n    return obj !== Error.prototype;\n  }, prop => {\n    return prop !== 'isAxiosError';\n  });\n\n  AxiosError.call(axiosError, error.message, code, config, request, response);\n\n  axiosError.cause = error;\n\n  axiosError.name = error.name;\n\n  customProps && Object.assign(axiosError, customProps);\n\n  return axiosError;\n};\n\nexport default AxiosError;\n\n// Path: lib/helpers/toFormData.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\n// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored\nimport PlatformFormData from '../platform/node/classes/FormData.js';\n\n/**\n * Determines if the given thing is a array or js object.\n *\n * @param {string} thing - The object or array to be visited.\n *\n * @returns {boolean}\n */\nfunction isVisitable(thing) {\n  return utils.isPlainObject(thing) || utils.isArray(thing);\n}\n\n/**\n * It removes the brackets from the end of a string\n *\n * @param {string} key - The key of the parameter.\n *\n * @returns {string} the key without the brackets.\n */\nfunction removeBrackets(key) {\n  return utils.endsWith(key, '[]') ? key.slice(0, -2) : key;\n}\n\n/**\n * It takes a path, a key, and a boolean, and returns a string\n *\n * @param {string} path - The path to the current key.\n * @param {string} key - The key of the current object being iterated over.\n * @param {string} dots - If true, the key will be rendered with dots instead of brackets.\n *\n * @returns {string} The path to the current key.\n */\nfunction renderKey(path, key, dots) {\n  if (!path) return key;\n  return path.concat(key).map(function each(token, i) {\n    // eslint-disable-next-line no-param-reassign\n    token = removeBrackets(token);\n    return !dots && i ? '[' + token + ']' : token;\n  }).join(dots ? '.' : '');\n}\n\n/**\n * If the array is an array and none of its elements are visitable, then it's a flat array.\n *\n * @param {Array<any>} arr - The array to check\n *\n * @returns {boolean}\n */\nfunction isFlatArray(arr) {\n  return utils.isArray(arr) && !arr.some(isVisitable);\n}\n\nconst predicates = utils.toFlatObject(utils, {}, null, function filter(prop) {\n  return /^is[A-Z]/.test(prop);\n});\n\n/**\n * Convert a data object to FormData\n *\n * @param {Object} obj\n * @param {?Object} [formData]\n * @param {?Object} [options]\n * @param {Function} [options.visitor]\n * @param {Boolean} [options.metaTokens = true]\n * @param {Boolean} [options.dots = false]\n * @param {?Boolean} [options.indexes = false]\n *\n * @returns {Object}\n **/\n\n/**\n * It converts an object into a FormData object\n *\n * @param {Object<any, any>} obj - The object to convert to form data.\n * @param {string} formData - The FormData object to append to.\n * @param {Object<string, any>} options\n *\n * @returns\n */\nfunction toFormData(obj, formData, options) {\n  if (!utils.isObject(obj)) {\n    throw new TypeError('target must be an object');\n  }\n\n  // eslint-disable-next-line no-param-reassign\n  formData = formData || new (PlatformFormData || FormData)();\n\n  // eslint-disable-next-line no-param-reassign\n  options = utils.toFlatObject(options, {\n    metaTokens: true,\n    dots: false,\n    indexes: false\n  }, false, function defined(option, source) {\n    // eslint-disable-next-line no-eq-null,eqeqeq\n    return !utils.isUndefined(source[option]);\n  });\n\n  const metaTokens = options.metaTokens;\n  // eslint-disable-next-line no-use-before-define\n  const visitor = options.visitor || defaultVisitor;\n  const dots = options.dots;\n  const indexes = options.indexes;\n  const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob;\n  const useBlob = _Blob && utils.isSpecCompliantForm(formData);\n\n  if (!utils.isFunction(visitor)) {\n    throw new TypeError('visitor must be a function');\n  }\n\n  function convertValue(value) {\n    if (value === null) return '';\n\n    if (utils.isDate(value)) {\n      return value.toISOString();\n    }\n\n    if (!useBlob && utils.isBlob(value)) {\n      throw new AxiosError('Blob is not supported. Use a Buffer instead.');\n    }\n\n    if (utils.isArrayBuffer(value) || utils.isTypedArray(value)) {\n      return useBlob && typeof Blob === 'function' ? new Blob([value]) : Buffer.from(value);\n    }\n\n    return value;\n  }\n\n  /**\n   * Default visitor.\n   *\n   * @param {*} value\n   * @param {String|Number} key\n   * @param {Array<String|Number>} path\n   * @this {FormData}\n   *\n   * @returns {boolean} return true to visit the each prop of the value recursively\n   */\n  function defaultVisitor(value, key, path) {\n    let arr = value;\n\n    if (value && !path && typeof value === 'object') {\n      if (utils.endsWith(key, '{}')) {\n        // eslint-disable-next-line no-param-reassign\n        key = metaTokens ? key : key.slice(0, -2);\n        // eslint-disable-next-line no-param-reassign\n        value = JSON.stringify(value);\n      } else if (\n        (utils.isArray(value) && isFlatArray(value)) ||\n        ((utils.isFileList(value) || utils.endsWith(key, '[]')) && (arr = utils.toArray(value))\n        )) {\n        // eslint-disable-next-line no-param-reassign\n        key = removeBrackets(key);\n\n        arr.forEach(function each(el, index) {\n          !(utils.isUndefined(el) || el === null) && formData.append(\n            // eslint-disable-next-line no-nested-ternary\n            indexes === true ? renderKey([key], index, dots) : (indexes === null ? key : key + '[]'),\n            convertValue(el)\n          );\n        });\n        return false;\n      }\n    }\n\n    if (isVisitable(value)) {\n      return true;\n    }\n\n    formData.append(renderKey(path, key, dots), convertValue(value));\n\n    return false;\n  }\n\n  const stack = [];\n\n  const exposedHelpers = Object.assign(predicates, {\n    defaultVisitor,\n    convertValue,\n    isVisitable\n  });\n\n  function build(value, path) {\n    if (utils.isUndefined(value)) return;\n\n    if (stack.indexOf(value) !== -1) {\n      throw Error('Circular reference detected in ' + path.join('.'));\n    }\n\n    stack.push(value);\n\n    utils.forEach(value, function each(el, key) {\n      const result = !(utils.isUndefined(el) || el === null) && visitor.call(\n        formData, el, utils.isString(key) ? key.trim() : key, path, exposedHelpers\n      );\n\n      if (result === true) {\n        build(el, path ? path.concat(key) : [key]);\n      }\n    });\n\n    stack.pop();\n  }\n\n  if (!utils.isObject(obj)) {\n    throw new TypeError('data must be an object');\n  }\n\n  build(obj);\n\n  return formData;\n}\n\nexport default toFormData;\n\n// Path: lib/helpers/AxiosURLSearchParams.js\n'use strict';\n\nimport toFormData from './toFormData.js';\n\n/**\n * It encodes a string by replacing all characters that are not in the unreserved set with\n * their percent-encoded equivalents\n *\n * @param {string} str - The string to encode.\n *\n * @returns {string} The encoded string.\n */\nfunction encode(str) {\n  const charMap = {\n    '!': '%21',\n    \"'\": '%27',\n    '(': '%28',\n    ')': '%29',\n    '~': '%7E',\n    '%20': '+',\n    '%00': '\\x00'\n  };\n  return encodeURIComponent(str).replace(/[!'()~]|%20|%00/g, function replacer(match) {\n    return charMap[match];\n  });\n}\n\n/**\n * It takes a params object and converts it to a FormData object\n *\n * @param {Object<string, any>} params - The parameters to be converted to a FormData object.\n * @param {Object<string, any>} options - The options object passed to the Axios constructor.\n *\n * @returns {void}\n */\nfunction AxiosURLSearchParams(params, options) {\n  this._pairs = [];\n\n  params && toFormData(params, this, options);\n}\n\nconst prototype = AxiosURLSearchParams.prototype;\n\nprototype.append = function append(name, value) {\n  this._pairs.push([name, value]);\n};\n\nprototype.toString = function toString(encoder) {\n  const _encode = encoder ? function(value) {\n    return encoder.call(this, value, encode);\n  } : encode;\n\n  return this._pairs.map(function each(pair) {\n    return _encode(pair[0]) + '=' + _encode(pair[1]);\n  }, '').join('&');\n};\n\nexport default AxiosURLSearchParams;\n\n// Path: lib/helpers/buildURL.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosURLSearchParams from '../helpers/AxiosURLSearchParams.js';\n\n/**\n * It replaces all instances of the characters `:`, `$`, `,`, `+`, `[`, and `]` with their\n * URI encoded counterparts\n *\n * @param {string} val The value to be encoded.\n *\n * @returns {string} The encoded value.\n */\nfunction encode(val) {\n  return encodeURIComponent(val).\n    replace(/%3A/gi, ':').\n    replace(/%24/g, '$').\n    replace(/%2C/gi, ',').\n    replace(/%20/g, '+').\n    replace(/%5B/gi, '[').\n    replace(/%5D/gi, ']');\n}\n\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default function buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n\n// Path: lib/cancel/CanceledError.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n...\n", "needle_token_start": 734, "needle_token_end": 787, "code_context_ntokens": 10912, "output": ["```javascript\n'use strict';\n\n/**\n * Creates a new URL by combining the specified URLs\n *\n * @param {string} baseURL The base URL\n * @param {string} relativeURL The relative URL\n *\n * @returns {string} The combined URL\n */\nexport default \nfunction combineURLs(baseURL, relativeURL) {\n  return relativeURL\n    ? baseURL.replace(/\\/?\\/$/, '') + '/' + relativeURL.replace(/^\\/+/, '')\n    : baseURL;\n}\n```"]}
{"repo": "axios/axios", "name": "settle", "language": "typescript", "path": "lib/core/settle.js", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the outcome of a network request by either resolving or rejecting a promise based on the status of the response.\n2. **Input**: It takes three parameters: two callback functions (one for resolving and one for rejecting a promise) and an object representing the network response.\n3. **Output**: The function does not return a value directly; instead, it controls the flow of promise resolution or rejection based on the response status.\n4. **Procedure**: The function first checks if the response status exists and if a custom status validation function is provided. If the response status passes the custom validation, the promise is resolved with the response object. If the status fails the validation, the promise is rejected with a custom error that includes the response status and additional details.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/core/AxiosError.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n...\n// Path: lib/helpers/toFormData.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\n// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored\nimport PlatformFormData from '../platform/node/classes/FormData.js';\n\n/**\n * Determines if the given thing is a array or js object.\n *\n * @param {string} thing - The object or array to be visited.\n *\n * @returns {boolean}\n */\nfunction isVisitable(thing) {\n  return utils.isPlainObject(thing) || utils.isArray(thing);\n}\n\n/**\n * It removes the brackets from the end of a string\n *\n * @param {string} key - The key of the parameter.\n *\n * @returns {string} the key without the brackets.\n */\nfunction removeBrackets(key) {\n  return utils.endsWith(key, '[]') ? key.slice(0, -2) : key;\n}\n\n/**\n * It takes a path, a key, and a boolean, and returns a string\n *\n * @param {string} path - The path to the current key.\n * @param {string} key - The key of the current object being iterated over.\n * @param {string} dots - If true, the key will be rendered with dots instead of brackets.\n *\n * @returns {string} The path to the current key.\n */\nfunction renderKey(path, key, dots) {\n  if (!path) return key;\n  return path.concat(key).map(function each(token, i) {\n    // eslint-disable-next-line no-param-reassign\n    token = removeBrackets(token);\n    return !dots && i ? '[' + token + ']' : token;\n  }).join(dots ? '.' : '');\n}\n\n/**\n * If the array is an array and none of its elements are visitable, then it's a flat array.\n *\n * @param {Array<any>} arr - The array to check\n *\n * @returns {boolean}\n */\nfunction isFlatArray(arr) {\n  return utils.isArray(arr) && !arr.some(isVisitable);\n}\n\nconst predicates = utils.toFlatObject(utils, {}, null, function filter(prop) {\n  return /^is[A-Z]/.test(prop);\n});\n\n/**\n * Convert a data object to FormData\n *\n * @param {Object} obj\n * @param {?Object} [formData]\n * @param {?Object} [options]\n * @param {Function} [options.visitor]\n * @param {Boolean} [options.metaTokens = true]\n * @param {Boolean} [options.dots = false]\n * @param {?Boolean} [options.indexes = false]\n *\n * @returns {Object}\n **/\n\n/**\n * It converts an object into a FormData object\n *\n * @param {Object<any, any>} obj - The object to convert to form data.\n * @param {string} formData - The FormData object to append to.\n * @param {Object<string, any>} options\n *\n * @returns\n */\nfunction toFormData(obj, formData, options) {\n  if (!utils.isObject(obj)) {\n    throw new TypeError('target must be an object');\n  }\n\n  // eslint-disable-next-line no-param-reassign\n  formData = formData || new (PlatformFormData || FormData)();\n\n  // eslint-disable-next-line no-param-reassign\n  options = utils.toFlatObject(options, {\n    metaTokens: true,\n    dots: false,\n    indexes: false\n  }, false, function defined(option, source) {\n    // eslint-disable-next-line no-eq-null,eqeqeq\n    return !utils.isUndefined(source[option]);\n  });\n\n  const metaTokens = options.metaTokens;\n  // eslint-disable-next-line no-use-before-define\n  const visitor = options.visitor || defaultVisitor;\n  const dots = options.dots;\n  const indexes = options.indexes;\n  const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob;\n  const useBlob = _Blob && utils.isSpecCompliantForm(formData);\n\n  if (!utils.isFunction(visitor)) {\n    throw new TypeError('visitor must be a function');\n  }\n\n  function convertValue(value) {\n    if (value === null) return '';\n\n    if (utils.isDate(value)) {\n      return value.toISOString();\n    }\n\n    if (!useBlob && utils.isBlob(value)) {\n      throw new AxiosError('Blob is not supported. Use a Buffer instead.');\n    }\n\n    if (utils.isArrayBuffer(value) || utils.isTypedArray(value)) {\n      return useBlob && typeof Blob === 'function' ? new Blob([value]) : Buffer.from(value);\n    }\n\n    return value;\n  }\n\n  /**\n   * Default visitor.\n   *\n   * @param {*} value\n   * @param {String|Number} key\n   * @param {Array<String|Number>} path\n   * @this {FormData}\n   *\n   * @returns {boolean} return true to visit the each prop of the value recursively\n   */\n  function defaultVisitor(value, key, path) {\n    let arr = value;\n\n    if (value && !path && typeof value === 'object') {\n      if (utils.endsWith(key, '{}')) {\n        // eslint-disable-next-line no-param-reassign\n        key = metaTokens ? key : key.slice(0, -2);\n        // eslint-disable-next-line no-param-reassign\n        value = JSON.stringify(value);\n      } else if (\n        (utils.isArray(value) && isFlatArray(value)) ||\n        ((utils.isFileList(value) || utils.endsWith(key, '[]')) && (arr = utils.toArray(value))\n        )) {\n        // eslint-disable-next-line no-param-reassign\n        key = removeBrackets(key);\n\n        arr.forEach(function each(el, index) {\n          !(utils.isUndefined(el) || el === null) && formData.append(\n            // eslint-disable-next-line no-nested-ternary\n            indexes === true ? renderKey([key], index, dots) : (indexes === null ? key : key + '[]'),\n            convertValue(el)\n          );\n        });\n        return false;\n      }\n    }\n\n    if (isVisitable(value)) {\n      return true;\n    }\n\n    formData.append(renderKey(path, key, dots), convertValue(value));\n\n    return false;\n  }\n\n  const stack = [];\n\n  const exposedHelpers = Object.assign(predicates, {\n    defaultVisitor,\n    convertValue,\n    isVisitable\n  });\n\n  function build(value, path) {\n    if (utils.isUndefined(value)) return;\n\n    if (stack.indexOf(value) !== -1) {\n      throw Error('Circular reference detected in ' + path.join('.'));\n    }\n\n    stack.push(value);\n\n    utils.forEach(value, function each(el, key) {\n      const result = !(utils.isUndefined(el) || el === null) && visitor.call(\n        formData, el, utils.isString(key) ? key.trim() : key, path, exposedHelpers\n      );\n\n      if (result === true) {\n        build(el, path ? path.concat(key) : [key]);\n      }\n    });\n\n    stack.pop();\n  }\n\n  if (!utils.isObject(obj)) {\n    throw new TypeError('data must be an object');\n  }\n\n  build(obj);\n\n  return formData;\n}\n\nexport default toFormData;\n\n// Path: lib/helpers/AxiosURLSearchParams.js\n'use strict';\n\nimport toFormData from './toFormData.js';\n\n/**\n * It encodes a string by replacing all characters that are not in the unreserved set with\n * their percent-encoded equivalents\n *\n * @param {string} str - The string to encode.\n *\n * @returns {string} The encoded string.\n */\nfunction encode(str) {\n  const charMap = {\n    '!': '%21',\n    \"'\": '%27',\n    '(': '%28',\n    ')': '%29',\n    '~': '%7E',\n    '%20': '+',\n    '%00': '\\x00'\n  };\n  return encodeURIComponent(str).replace(/[!'()~]|%20|%00/g, function replacer(match) {\n    return charMap[match];\n  });\n}\n\n/**\n * It takes a params object and converts it to a FormData object\n *\n * @param {Object<string, any>} params - The parameters to be converted to a FormData object.\n * @param {Object<string, any>} options - The options object passed to the Axios constructor.\n *\n * @returns {void}\n */\nfunction AxiosURLSearchParams(params, options) {\n  this._pairs = [];\n\n  params && toFormData(params, this, options);\n}\n\nconst prototype = AxiosURLSearchParams.prototype;\n\nprototype.append = function append(name, value) {\n  this._pairs.push([name, value]);\n};\n\nprototype.toString = function toString(encoder) {\n  const _encode = encoder ? function(value) {\n    return encoder.call(this, value, encode);\n  } : encode;\n\n  return this._pairs.map(function each(pair) {\n    return _encode(pair[0]) + '=' + _encode(pair[1]);\n  }, '').join('&');\n};\n\nexport default AxiosURLSearchParams;\n\n// Path: lib/helpers/buildURL.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosURLSearchParams from '../helpers/AxiosURLSearchParams.js';\n\n/**\n * It replaces all instances of the characters `:`, `$`, `,`, `+`, `[`, and `]` with their\n * URI encoded counterparts\n *\n * @param {string} val The value to be encoded.\n *\n * @returns {string} The encoded value.\n */\nfunction encode(val) {\n  return encodeURIComponent(val).\n    replace(/%3A/gi, ':').\n    replace(/%24/g, '$').\n    replace(/%2C/gi, ',').\n    replace(/%20/g, '+').\n    replace(/%5B/gi, '[').\n    replace(/%5D/gi, ']');\n}\n\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default function buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n\n// Path: lib/cancel/CanceledError.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n\n/**\n * A `CanceledError` is an object that is thrown when an operation is canceled.\n *\n * @param {string=} message The message.\n * @param {Object=} config The config.\n * @param {Object=} request The request.\n *\n * @returns {CanceledError} The created error.\n */\nfunction CanceledError(message, config, request) {\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  AxiosError.call(this, message == null ? 'canceled' : message, AxiosError.ERR_CANCELED, config, request);\n  this.name = 'CanceledError';\n}\n\nutils.inherits(CanceledError, AxiosError, {\n  __CANCEL__: true\n});\n\nexport default CanceledError;\n\n// Path: lib/defaults/transitional.js\n'use strict';\n\nexport default {\n  silentJSONParsing: true,\n  forcedJSONParsing: true,\n  clarifyTimeoutError: false\n};\n\n// Path: lib/helpers/speedometer.js\n'use strict';\n\n/**\n * Calculate data maxRate\n * @param {Number} [samplesCount= 10]\n * @param {Number} [min= 1000]\n * @returns {Function}\n */\nfunction speedometer(samplesCount, min) {\n  samplesCount = samplesCount || 10;\n  const bytes = new Array(samplesCount);\n  const timestamps = new Array(samplesCount);\n  let head = 0;\n  let tail = 0;\n  let firstSampleTS;\n\n  min = min !== undefined ? min : 1000;\n\n  return function push(chunkLength) {\n    const now = Date.now();\n\n    const startedAt = timestamps[tail];\n\n    if (!firstSampleTS) {\n      firstSampleTS = now;\n    }\n\n    bytes[head] = chunkLength;\n    timestamps[head] = now;\n\n    let i = tail;\n    let bytesCount = 0;\n\n    while (i !== head) {\n      bytesCount += bytes[i++];\n      i = i % samplesCount;\n    }\n\n    head = (head + 1) % samplesCount;\n\n    if (head === tail) {\n      tail = (tail + 1) % samplesCount;\n    }\n\n    if (now - firstSampleTS < min) {\n      return;\n    }\n\n    const passed = startedAt && now - startedAt;\n\n    return passed ? Math.round(bytesCount * 1000 / passed) : undefined;\n  };\n}\n\nexport default speedometer;\n\n// Path: lib/helpers/parseHeaders.js\n'use strict';\n\nimport utils from './../utils.js';\n\n// RawAxiosHeaders whose duplicates are ignored by node\n// c.f. https://nodejs.org/api/http.html#http_message_headers\nconst ignoreDuplicateOf = utils.toObjectSet([\n  'age', 'authorization', 'content-length', 'content-type', 'etag',\n  'expires', 'from', 'host', 'if-modified-since', 'if-unmodified-since',\n  'last-modified', 'location', 'max-forwards', 'proxy-authorization',\n  'referer', 'retry-after', 'user-agent'\n]);\n\n/**\n * Parse headers into an object\n *\n * ```\n * Date: Wed, 27 Aug 2014 08:58:49 GMT\n * Content-Type: application/json\n * Connection: keep-alive\n * Transfer-Encoding: chunked\n * ```\n *\n * @param {String} rawHeaders Headers needing to be parsed\n *\n * @returns {Object} Headers parsed into an object\n */\nexport default rawHeaders => {\n  const parsed = {};\n  let key;\n  let val;\n  let i;\n\n  rawHeaders && rawHeaders.split('\\n').forEach(function parser(line) {\n    i = line.indexOf(':');\n    key = line.substring(0, i).trim().toLowerCase();\n    val = line.substring(i + 1).trim();\n\n    if (!key || (parsed[key] && ignoreDuplicateOf[key])) {\n      return;\n    }\n\n    if (key === 'set-cookie') {\n      if (parsed[key]) {\n        parsed[key].push(val);\n      } else {\n        parsed[key] = [val];\n      }\n    } else {\n      parsed[key] = parsed[key] ? parsed[key] + ', ' + val : val;\n    }\n  });\n\n  return parsed;\n};\n\n// Path: lib/core/AxiosHeaders.js\n'use strict';\n\nimport utils from '../utils.js';\nimport parseHeaders from '../helpers/parseHeaders.js';\n\nconst $internals = Symbol('internals');\n\nfunction normalizeHeader(header) {\n  return header && String(header).trim().toLowerCase();\n}\n\nfunction normalizeValue(value) {\n  if (value === false || value == null) {\n    return value;\n  }\n\n  return utils.isArray(value) ? value.map(normalizeValue) : String(value);\n}\n\nfunction parseTokens(str) {\n  const tokens = Object.create(null);\n  const tokensRE = /([^\\s,;=]+)\\s*(?:=\\s*([^,;]+))?/g;\n  let match;\n\n  while ((match = tokensRE.exec(str))) {\n    tokens[match[1]] = match[2];\n  }\n\n  return tokens;\n}\n\nconst isValidHeaderName = (str) => /^[-_a-zA-Z0-9^`|~,!#$%&'*+.]+$/.test(str.trim());\n\nfunction matchHeaderValue(context, value, header, filter, isHeaderNameFilter) {\n  if (utils.isFunction(filter)) {\n    return filter.call(this, value, header);\n  }\n\n  if (isHeaderNameFilter) {\n    value = header;\n  }\n\n  if (!utils.isString(value)) return;\n\n  if (utils.isString(filter)) {\n    return value.indexOf(filter) !== -1;\n  }\n\n  if (utils.isRegExp(filter)) {\n    return filter.test(value);\n  }\n}\n\nfunction formatHeader(header) {\n  return header.trim()\n    .toLowerCase().replace(/([a-z\\d])(\\w*)/g, (w, char, str) => {\n      return char.toUpperCase() + str;\n    });\n}\n\nfunction buildAccessors(obj, header) {\n  const accessorName = utils.toCamelCase(' ' + header);\n\n  ['get', 'set', 'has'].forEach(methodName => {\n    Object.defineProperty(obj, methodName + accessorName, {\n      value: function(arg1, arg2, arg3) {\n        return this[methodName].call(this, header, arg1, arg2, arg3);\n      },\n      configurable: true\n    });\n  });\n}\n\nclass AxiosHeaders {\n  constructor(headers) {\n    headers && this.set(headers);\n  }\n\n  set(header, valueOrRewrite, rewrite) {\n    const self = this;\n\n    function setHeader(_value, _header, _rewrite) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!lHeader) {\n        throw new Error('header name must be a non-empty string');\n      }\n\n      const key = utils.findKey(self, lHeader);\n\n      if(!key || self[key] === undefined || _rewrite === true || (_rewrite === undefined && self[key] !== false)) {\n        self[key || _header] = normalizeValue(_value);\n      }\n    }\n\n    const setHeaders = (headers, _rewrite) =>\n      utils.forEach(headers, (_value, _header) => setHeader(_value, _header, _rewrite));\n\n    if (utils.isPlainObject(header) || header instanceof this.constructor) {\n      setHeaders(header, valueOrRewrite)\n    } else if(utils.isString(header) && (header = header.trim()) && !isValidHeaderName(header)) {\n      setHeaders(parseHeaders(header), valueOrRewrite);\n    } else {\n      header != null && setHeader(valueOrRewrite, header, rewrite);\n    }\n\n    return this;\n  }\n\n  get(header, parser) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      if (key) {\n        const value = this[key];\n\n        if (!parser) {\n          return value;\n        }\n\n        if (parser === true) {\n          return parseTokens(value);\n        }\n\n        if (utils.isFunction(parser)) {\n          return parser.call(this, value, key);\n        }\n\n        if (utils.isRegExp(parser)) {\n          return parser.exec(value);\n        }\n\n        throw new TypeError('parser must be boolean|regexp|function');\n      }\n    }\n  }\n\n  has(header, matcher) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      return !!(key && this[key] !== undefined && (!matcher || matchHeaderValue(this, this[key], key, matcher)));\n    }\n\n    return false;\n  }\n\n  delete(header, matcher) {\n    const self = this;\n    let deleted = false;\n\n    function deleteHeader(_header) {\n      _header = normalizeHeader(_header);\n\n      if (_header) {\n        const key = utils.findKey(self, _header);\n\n        if (key && (!matcher || matchHeaderValue(self, self[key], key, matcher))) {\n          delete self[key];\n\n          deleted = true;\n        }\n      }\n    }\n\n    if (utils.isArray(header)) {\n      header.forEach(deleteHeader);\n    } else {\n      deleteHeader(header);\n    }\n\n    return deleted;\n  }\n\n  clear(matcher) {\n    const keys = Object.keys(this);\n    let i = keys.length;\n    let deleted = false;\n\n    while (i--) {\n      const key = keys[i];\n      if(!matcher || matchHeaderValue(this, this[key], key, matcher, true)) {\n        delete this[key];\n        deleted = true;\n      }\n    }\n\n    return deleted;\n  }\n\n  normalize(format) {\n    const self = this;\n    const headers = {};\n\n    utils.forEach(this, (value, header) => {\n      const key = utils.findKey(headers, header);\n\n      if (key) {\n        self[key] = normalizeValue(value);\n        delete self[header];\n        return;\n      }\n\n      const normalized = format ? formatHeader(header) : String(header).trim();\n\n      if (normalized !== header) {\n        delete self[header];\n      }\n\n      self[normalized] = normalizeValue(value);\n\n      headers[normalized] = true;\n    });\n\n    return this;\n  }\n\n  concat(...targets) {\n    return this.constructor.concat(this, ...targets);\n  }\n\n  toJSON(asStrings) {\n    const obj = Object.create(null);\n\n    utils.forEach(this, (value, header) => {\n      value != null && value !== false && (obj[header] = asStrings && utils.isArray(value) ? value.join(', ') : value);\n    });\n\n    return obj;\n  }\n\n  [Symbol.iterator]() {\n    return Object.entries(this.toJSON())[Symbol.iterator]();\n  }\n\n  toString() {\n    return Object.entries(this.toJSON()).map(([header, value]) => header + ': ' + value).join('\\n');\n  }\n\n  get [Symbol.toStringTag]() {\n    return 'AxiosHeaders';\n  }\n\n  static from(thing) {\n    return thing instanceof this ? thing : new this(thing);\n  }\n\n  static concat(first, ...targets) {\n    const computed = new this(first);\n\n    targets.forEach((target) => computed.set(target));\n\n    return computed;\n  }\n\n  static accessor(header) {\n    const internals = this[$internals] = (this[$internals] = {\n      accessors: {}\n    });\n\n    const accessors = internals.accessors;\n    const prototype = this.prototype;\n\n    function defineAccessor(_header) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!accessors[lHeader]) {\n        buildAccessors(prototype, _header);\n        accessors[lHeader] = true;\n      }\n    }\n\n    utils.isArray(header) ? header.forEach(defineAccessor) : defineAccessor(header);\n\n    return this;\n  }\n}\n\nAxiosHeaders.accessor(['Content-Type', 'Content-Length', 'Accept', 'Accept-Encoding', 'User-Agent', 'Authorization']);\n\n// reserved names hotfix\nutils.reduceDescriptors(AxiosHeaders.prototype, ({value}, key) => {\n  let mapped = key[0].toUpperCase() + key.slice(1); // map `set` => `Set`\n  return {\n    get: () => value,\n    set(headerValue) {\n      this[mapped] = headerValue;\n    }\n  }\n});\n\nutils.freezeMethods(AxiosHeaders);\n\nexport default AxiosHeaders;\n\n// Path: lib/core/settle.js\n'use strict';\n\nimport AxiosError from './AxiosError.js';\n\n/**\n * Resolve or reject a Promise based on response status.\n *\n * @param {Function} resolve A function that resolves the promise.\n * @param {Function} reject A function that rejects the promise.\n * @param {object} response The response.\n *\n * @returns {object} The response.\n */\nexport default \nfunction settle(resolve, reject, response) {\n  const validateStatus = response.config.validateStatus;\n  if (!response.status || !validateStatus || validateStatus(response.status)) {\n    resolve(response);\n  } else {\n    reject(new AxiosError(\n      'Request failed with status code ' + response.status,\n      [AxiosError.ERR_BAD_REQUEST, AxiosError.ERR_BAD_RESPONSE][Math.floor(response.status / 100) - 4],\n      response.config,\n      response.request,\n      response\n    ));\n  }\n}\n\n// Path: lib/helpers/parseProtocol.js\n'use strict';\n\nexport default function parseProtocol(url) {\n  const match = /^([-+\\w]{1,25})(:?\\/\\/|:)/.exec(url);\n  return match && match[1] || '';\n}\n\n// Path: lib/helpers/cookies.js\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n  // Standard browser envs support document.cookie\n  {\n    write(name, value, expires, path, domain, secure) {\n      const cookie = [name + '=' + encodeURIComponent(value)];\n\n      utils.isNumber(expires) && cookie.push('expires=' + new Date(expires).toGMTString());\n\n      utils.isString(path) && cookie.push('path=' + path);\n\n      utils.isString(domain) && cookie.push('domain=' + domain);\n\n      secure === true && cookie.push('secure');\n\n      document.cookie = cookie.join('; ');\n    },\n\n    read(name) {\n      const match = document.cookie.match(new RegExp('(^|;\\\\s*)(' + name + ')=([^;]*)'));\n      return (match ? decodeURIComponent(match[3]) : null);\n    },\n\n    remove(name) {\n      this.write(name, '', Date.now() - 86400000);\n    }\n  }\n\n  :\n\n  // Non-standard browser env (web workers, react-native) lack needed support.\n  {\n    write() {},\n    read() {\n      return null;\n    },\n    remove() {}\n  };\n\n\n// Path: lib/helpers/isURLSameOrigin.js\n'use strict';\n\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n// Standard browser envs have full support of the APIs needed to test\n// whether the request URL is of the same origin as current location.\n  (function standardBrowserEnv() {\n    const msie = /(msie|trident)/i.test(navigator.userAgent);\n    const urlParsingNode = document.createElement('a');\n    let originURL;\n\n    /**\n    * Parse a URL to discover its components\n    *\n    * @param {String} url The URL to be parsed\n    * @returns {Object}\n    */\n    function resolveURL(url) {\n      let href = url;\n\n      if (msie) {\n        // IE needs attribute set twice to normalize properties\n        urlParsingNode.setAttribute('href', href);\n        href = urlParsingNode.href;\n      }\n\n      urlParsingNode.setAttribute('href', href);\n\n      // urlParsingNode provides the UrlUtils interface - http://url.spec.whatwg.org/#urlutils\n      return {\n        href: urlParsingNode.href,\n        protocol: urlParsingNode.protocol ? urlParsingNode.protocol.replace(/:$/, '') : '',\n        host: urlParsingNode.host,\n        search: urlParsingNode.search ? urlParsingNode.search.replace(/^\\?/, '') : '',\n        hash: urlParsingNode.hash ? urlParsingNode.hash.replace(/^#/, '') : '',\n        hostname: urlParsingNode.hostname,\n        port: urlParsingNode.port,\n        pathname: (urlParsingNode.pathname.charAt(0) === '/') ?\n          urlParsingNode.pathname :\n          '/' + urlParsingNode.pathname\n      };\n    }\n\n    originURL = resolveURL(window.location.href);\n\n    /**\n    * Determine if a URL shares the same origin as the current location\n    *\n    * @param {String} requestURL The URL to test\n    * @returns {boolean} True if URL shares the same origin, otherwise false\n    */\n    return function isURLSameOrigin(requestURL) {\n      const parsed = (utils.isString(requestURL)) ? resolveURL(requestURL) : requestURL;\n      return (parsed.protocol === originURL.protocol &&\n          parsed.host === originURL.host);\n    };\n  })() :\n\n  // Non standard browser envs (web workers, react-native) lack needed support.\n  (function nonStandardBrowserEnv() {\n    return function isURLSameOrigin() {\n      return true;\n    };\n  })();\n\n// Path: lib/adapters/xhr.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport cookies from './../helpers/cookies.js';\nimport buildURL from './../helpers/buildURL.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport isURLSameOrigin from './../helpers/isURLSameOrigin.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport parseProtocol from '../helpers/parseProtocol.js';\nimport platform from '../platform/index.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport speedometer from '../helpers/speedometer.js';\n\nfunction progressEventReducer(listener, isDownloadStream) {\n  let bytesNotified = 0;\n  const _speedometer = speedometer(50, 250);\n\n  return e => {\n    const loaded = e.loaded;\n    const total = e.lengthComputable ? e.total : undefined;\n    const progressBytes = loaded - bytesNotified;\n    const rate = _speedometer(progressBytes);\n    const inRange = loaded <= total;\n\n    bytesNotified = loaded;\n\n    const data = {\n      loaded,\n      total,\n      progress: total ? (loaded / total) : undefined,\n      bytes: progressBytes,\n      rate: rate ? rate : undefined,\n      estimated: rate && total && inRange ? (total - loaded) / rate : undefined,\n      event: e\n    };\n\n    data[isDownloadStream ? 'download' : 'upload'] = true;\n\n    listener(data);\n  };\n}\n\nconst isXHRAdapterSupported = typeof XMLHttpRequest !== 'undefined';\n\nexport default isXHRAdapterSupported && function (config) {\n  return new Promise(function dispatchXhrRequest(resolve, reject) {\n    let requestData = config.data;\n    const requestHeaders = AxiosHeaders.from(config.headers).normalize();\n    let {responseType, withXSRFToken} = config;\n    let onCanceled;\n    function done() {\n      if (config.cancelToken) {\n        config.cancelToken.unsubscribe(onCanceled);\n      }\n\n      if (config.signal) {\n        config.signal.removeEventListener('abort', onCanceled);\n      }\n    }\n\n    let contentType;\n\n    if (utils.isFormData(requestData)) {\n      if (platform.hasStandardBrowserEnv || platform.hasStandardBrowserWebWorkerEnv) {\n        requestHeaders.setContentType(false); // Let the browser set it\n      } else if ((contentType = requestHeaders.getContentType()) !== false) {\n        // fix semicolon duplication issue for ReactNative FormData implementation\n        const [type, ...tokens] = contentType ? contentType.split(';').map(token => token.trim()).filter(Boolean) : [];\n        requestHeaders.setContentType([type || 'multipart/form-data', ...tokens].join('; '));\n      }\n    }\n\n    let request = new XMLHttpRequest();\n\n    // HTTP basic authentication\n    if (config.auth) {\n      const username = config.auth.username || '';\n      const password = config.auth.password ? unescape(encodeURIComponent(config.auth.password)) : '';\n      requestHeaders.set('Authorization', 'Basic ' + btoa(username + ':' + password));\n    }\n\n    const fullPath = buildFullPath(config.baseURL, config.url);\n\n    request.open(config.method.toUpperCase(), buildURL(fullPath, config.params, config.paramsSerializer), true);\n\n    // Set the request timeout in MS\n    request.timeout = config.timeout;\n\n    function onloadend() {\n      if (!request) {\n        return;\n      }\n      // Prepare the response\n      const responseHeaders = AxiosHeaders.from(\n        'getAllResponseHeaders' in request && request.getAllResponseHeaders()\n      );\n      const responseData = !responseType || responseType === 'text' || responseType === 'json' ?\n        request.responseText : request.response;\n      const response = {\n        data: responseData,\n        status: request.status,\n        statusText: request.statusText,\n        headers: responseHeaders,\n        config,\n        request\n      };\n\n      settle(function _resolve(value) {\n        resolve(value);\n        done();\n      }, function _reject(err) {\n        reject(err);\n        done();\n      }, response);\n\n      // Clean up request\n      request = null;\n    }\n\n    if ('onloadend' in request) {\n      // Use onloadend if available\n      request.onloadend = onloadend;\n    } else {\n      // Listen for ready state to emulate onloadend\n      request.onreadystatechange = function handleLoad() {\n        if (!request || request.readyState !== 4) {\n          return;\n        }\n\n        // The request errored out and we didn't get a response, this will be\n        // handled by onerror instead\n        // With one exception: request that using file: protocol, most browsers\n        // will return status as 0 even though it's a successful request\n        if (request.status === 0 && !(request.responseURL && request.responseURL.indexOf('file:') === 0)) {\n          return;\n        }\n        // readystate handler is calling before onerror or ontimeout handlers,\n        // so we should call onloadend on the next 'tick'\n        setTimeout(onloadend);\n      };\n    }\n\n    // Handle browser request cancellation (as opposed to a manual cancellation)\n    request.onabort = function handleAbort() {\n      if (!request) {\n        return;\n      }\n\n      reject(new AxiosError('Request aborted', AxiosError.ECONNABORTED, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle low level network errors\n    request.onerror = function handleError() {\n      // Real errors are hidden from us by the browser\n      // onerror should only fire if it's a network error\n      reject(new AxiosError('Network Error', AxiosError.ERR_NETWORK, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle timeout\n    request.ontimeout = function handleTimeout() {\n      let timeoutErrorMessage = config.timeout ? 'timeout of ' + config.timeout + 'ms exceeded' : 'timeout exceeded';\n      const transitional = config.transitional || transitionalDefaults;\n      if (config.timeoutErrorMessage) {\n        timeoutErrorMessage = config.timeoutErrorMessage;\n      }\n      reject(new AxiosError(\n        timeoutErrorMessage,\n        transitional.clarifyTimeoutError ? AxiosError.ETIMEDOUT : AxiosError.ECONNABORTED,\n        config,\n        request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Add xsrf header\n    // This is only done if running in a standard browser environment.\n    // Specifically not if we're in a web worker, or react-native.\n    if(platform.hasStandardBrowserEnv) {\n      withXSRFToken && utils.isFunction(withXSRFToken) && (withXSRFToken = withXSRFToken(config));\n\n      if (withXSRFToken || (withXSRFToken !== false && isURLSameOrigin(fullPath))) {\n        // Add xsrf header\n        const xsrfValue = config.xsrfHeaderName && config.xsrfCookieName && cookies.read(config.xsrfCookieName);\n\n        if (xsrfValue) {\n          requestHeaders.set(config.xsrfHeaderName, xsrfValue);\n        }\n      }\n    }\n\n    // Remove Content-Type if data is undefined\n    requestData === undefined && requestHeaders.setContentType(null);\n\n    // Add headers to the request\n    if ('setRequestHeader' in request) {\n      utils.forEach(requestHeaders.toJSON(), function setRequestHeader(val, key) {\n        request.setRequestHeader(key, val);\n      });\n    }\n\n    // Add withCredentials to request if needed\n    if (!utils.isUndefined(config.withCredentials)) {\n      request.withCredentials = !!config.withCredentials;\n    }\n\n    // Add responseType to request if needed\n    if (responseType && responseType !== 'json') {\n      request.responseType = config.responseType;\n    }\n\n    // Handle progress if needed\n    if (typeof config.onDownloadProgress === 'function') {\n      request.addEventListener('progress', progressEventReducer(config.onDownloadProgress, true));\n    }\n\n    // Not all browsers support upload events\n    if (typeof config.onUploadProgress === 'function' && request.upload) {\n      request.upload.addEventListener('progress', progressEventReducer(config.onUploadProgress));\n    }\n\n    if (config.cancelToken || config.signal) {\n      // Handle cancellation\n      // eslint-disable-next-line func-names\n      onCanceled = cancel => {\n        if (!request) {\n          return;\n        }\n        reject(!cancel || cancel.type ? new CanceledError(null, config, request) : cancel);\n        request.abort();\n        request = null;\n      };\n\n      config.cancelToken && config.cancelToken.subscribe(onCanceled);\n      if (config.signal) {\n        config.signal.aborted ? onCanceled() : config.signal.addEventListener('abort', onCanceled);\n      }\n    }\n\n    const protocol = parseProtocol(fullPath);\n\n    if (protocol && platform.protocols.indexOf(protocol) === -1) {\n      reject(new AxiosError('Unsupported protocol ' + protocol + ':', AxiosError.ERR_BAD_REQUEST, config));\n      return;\n    }\n\n\n    // Send the request\n    request.send(requestData || null);\n  });\n}\n\n// Path: lib/helpers/fromDataURI.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport parseProtocol from './parseProtocol.js';\nimport platform from '../platform/index.js';\n\nconst DATA_URL_PATTERN = /^(?:([^;]+);)?(?:[^;]+;)?(base64|),([\\s\\S]*)$/;\n\n/**\n * Parse data uri to a Buffer or Blob\n *\n * @param {String} uri\n * @param {?Boolean} asBlob\n * @param {?Object} options\n * @param {?Function} options.Blob\n *\n * @returns {Buffer|Blob}\n */\nexport default function fromDataURI(uri, asBlob, options) {\n  const _Blob = options && options.Blob || platform.classes.Blob;\n  const protocol = parseProtocol(uri);\n\n  if (asBlob === undefined && _Blob) {\n    asBlob = true;\n  }\n\n  if (protocol === 'data') {\n    uri = protocol.length ? uri.slice(protocol.length + 1) : uri;\n\n    const match = DATA_URL_PATTERN.exec(uri);\n\n    if (!match) {\n      throw new AxiosError('Invalid URL', AxiosError.ERR_INVALID_URL);\n    }\n\n    const mime = match[1];\n    const isBase64 = match[2];\n    const body = match[3];\n    const buffer = Buffer.from(decodeURIComponent(body), isBase64 ? 'base64' : 'utf8');\n\n    if (asBlob) {\n      if (!_Blob) {\n        throw new AxiosError('Blob is not supported', AxiosError.ERR_NOT_SUPPORT);\n      }\n\n      return new _Blob([buffer], {type: mime});\n    }\n\n    return buffer;\n  }\n\n  throw new AxiosError('Unsupported protocol ' + protocol, AxiosError.ERR_NOT_SUPPORT);\n}\n\n// Path: lib/helpers/readBlob.js\nconst {asyncIterator} = Symbol;\n\nconst readBlob = async function* (blob) {\n  if (blob.stream) {\n    yield* blob.stream()\n  } else if (blob.arrayBuffer) {\n    yield await blob.arrayBuffer()\n  } else if (blob[asyncIterator]) {\n    yield* blob[asyncIterator]();\n  } else {\n    yield blob;\n  }\n}\n\nexport default readBlob;\n\n// Path: lib/helpers/formDataToStream.js\nimport {TextEncoder} from 'util';\nimport {Readable} from 'stream';\nimport utils from \"../utils.js\";\nimport readBlob from \"./readBlob.js\";\n\nconst BOUNDARY_ALPHABET = utils.ALPHABET.ALPHA_DIGIT + '-_';\n\nconst textEncoder = new TextEncoder();\n\nconst CRLF = '\\r\\n';\nconst CRLF_BYTES = textEncoder.encode(CRLF);\nconst CRLF_BYTES_COUNT = 2;\n\nclass FormDataPart {\n  constructor(name, value) {\n    const {escapeName} = this.constructor;\n    const isStringValue = utils.isString(value);\n\n    let headers = `Content-Disposition: form-data; name=\"${escapeName(name)}\"${\n      !isStringValue && value.name ? `; filename=\"${escapeName(value.name)}\"` : ''\n    }${CRLF}`;\n\n    if (isStringValue) {\n      value = textEncoder.encode(String(value).replace(/\\r?\\n|\\r\\n?/g, CRLF));\n    } else {\n      headers += `Content-Type: ${value.type || \"application/octet-stream\"}${CRLF}`\n    }\n\n    this.headers = textEncoder.encode(headers + CRLF);\n\n    this.contentLength = isStringValue ? value.byteLength : value.size;\n\n    this.size = this.headers.byteLength + this.contentLength + CRLF_BYTES_COUNT;\n\n    this.name = name;\n    this.value = value;\n  }\n\n  async *encode(){\n    yield this.headers;\n\n    const {value} = this;\n\n    if(utils.isTypedArray(value)) {\n      yield value;\n    } else {\n      yield* readBlob(value);\n    }\n\n    yield CRLF_BYTES;\n  }\n\n  static escapeName(name) {\n      return String(name).replace(/[\\r\\n\"]/g, (match) => ({\n        '\\r' : '%0D',\n        '\\n' : '%0A',\n        '\"' : '%22',\n      }[match]));\n  }\n}\n\nconst formDataToStream = (form, headersHandler, options) => {\n  const {\n    tag = 'form-data-boundary',\n    size = 25,\n    boundary = tag + '-' + utils.generateString(size, BOUNDARY_ALPHABET)\n  } = options || {};\n\n  if(!utils.isFormData(form)) {\n    throw TypeError('FormData instance required');\n  }\n\n  if (boundary.length < 1 || boundary.length > 70) {\n    throw Error('boundary must be 10-70 characters long')\n  }\n\n  const boundaryBytes = textEncoder.encode('--' + boundary + CRLF);\n  const footerBytes = textEncoder.encode('--' + boundary + '--' + CRLF + CRLF);\n  let contentLength = footerBytes.byteLength;\n\n  const parts = Array.from(form.entries()).map(([name, value]) => {\n    const part = new FormDataPart(name, value);\n    contentLength += part.size;\n    return part;\n  });\n\n  contentLength += boundaryBytes.byteLength * parts.length;\n\n  contentLength = utils.toFiniteNumber(contentLength);\n\n  const computedHeaders = {\n    'Content-Type': `multipart/form-data; boundary=${boundary}`\n  }\n\n  if (Number.isFinite(contentLength)) {\n    computedHeaders['Content-Length'] = contentLength;\n  }\n\n  headersHandler && headersHandler(computedHeaders);\n\n  return Readable.from((async function *() {\n    for(const part of parts) {\n      yield boundaryBytes;\n      yield* part.encode();\n    }\n\n    yield footerBytes;\n  })());\n};\n\nexport default formDataToStream;\n\n// Path: lib/helpers/callbackify.js\nimport utils from \"../utils.js\";\n\nconst callbackify = (fn, reducer) => {\n  return utils.isAsyncFn(fn) ? function (...args) {\n    const cb = args.pop();\n    fn.apply(this, args).then((value) => {\n      try {\n        reducer ? cb(null, ...reducer(value)) : cb(null, value);\n      } catch (err) {\n        cb(err);\n      }\n    }, cb);\n  } : fn;\n}\n\nexport default callbackify;\n\n// Path: lib/helpers/ZlibHeaderTransformStream.js\n\"use strict\";\n\nimport stream from \"stream\";\n\nclass ZlibHeaderTransformStream extends stream.Transform {\n  __transform(chunk, encoding, callback) {\n    this.push(chunk);\n    callback();\n  }\n\n  _transform(chunk, encoding, callback) {\n    if (chunk.length !== 0) {\n      this._transform = this.__transform;\n\n      // Add Default Compression headers if no zlib headers are present\n      if (chunk[0] !== 120) { // Hex: 78\n        const header = Buffer.alloc(2);\n        header[0] = 120; // Hex: 78\n        header[1] = 156; // Hex: 9C \n        this.push(header, encoding);\n      }\n    }\n\n    this.__transform(chunk, encoding, callback);\n  }\n}\n\nexport default ZlibHeaderTransformStream;\n\n// Path: lib/helpers/throttle.js\n'use strict';\n\n/**\n * Throttle decorator\n * @param {Function} fn\n * @param {Number} freq\n * @return {Function}\n */\nfunction throttle(fn, freq) {\n  let timestamp = 0;\n  const threshold = 1000 / freq;\n  let timer = null;\n  return function throttled(force, args) {\n    const now = Date.now();\n    if (force || now - timestamp > threshold) {\n      if (timer) {\n        clearTimeout(timer);\n        timer = null;\n      }\n      timestamp = now;\n      return fn.apply(null, args);\n    }\n    if (!timer) {\n      timer = setTimeout(() => {\n        timer = null;\n        timestamp = Date.now();\n        return fn.apply(null, args);\n      }, threshold - (now - timestamp));\n    }\n  };\n}\n\nexport default throttle;\n\n// Path: lib/helpers/AxiosTransformStream.js\n'use strict';\n\nimport stream from 'stream';\nimport utils from '../utils.js';\nimport throttle from './throttle.js';\nimport speedometer from './speedometer.js';\n\nconst kInternals = Symbol('internals');\n\nclass AxiosTransformStream extends stream.Transform{\n  constructor(options) {\n    options = utils.toFlatObject(options, {\n      maxRate: 0,\n      chunkSize: 64 * 1024,\n      minChunkSize: 100,\n      timeWindow: 500,\n      ticksRate: 2,\n      samplesCount: 15\n    }, null, (prop, source) => {\n      return !utils.isUndefined(source[prop]);\n    });\n\n    super({\n      readableHighWaterMark: options.chunkSize\n    });\n\n    const self = this;\n\n    const internals = this[kInternals] = {\n      length: options.length,\n      timeWindow: options.timeWindow,\n      ticksRate: options.ticksRate,\n      chunkSize: options.chunkSize,\n      maxRate: options.maxRate,\n      minChunkSize: options.minChunkSize,\n      bytesSeen: 0,\n      isCaptured: false,\n      notifiedBytesLoaded: 0,\n      ts: Date.now(),\n      bytes: 0,\n      onReadCallback: null\n    };\n\n    const _speedometer = speedometer(internals.ticksRate * options.samplesCount, internals.timeWindow);\n\n    this.on('newListener', event => {\n      if (event === 'progress') {\n        if (!internals.isCaptured) {\n          internals.isCaptured = true;\n        }\n      }\n    });\n\n    let bytesNotified = 0;\n\n    internals.updateProgress = throttle(function throttledHandler() {\n      const totalBytes = internals.length;\n      const bytesTransferred = internals.bytesSeen;\n      const progressBytes = bytesTransferred - bytesNotified;\n      if (!progressBytes || self.destroyed) return;\n\n      const rate = _speedometer(progressBytes);\n\n      bytesNotified = bytesTransferred;\n\n      process.nextTick(() => {\n        self.emit('progress', {\n          'loaded': bytesTransferred,\n          'total': totalBytes,\n          'progress': totalBytes ? (bytesTransferred / totalBytes) : undefined,\n          'bytes': progressBytes,\n          'rate': rate ? rate : undefined,\n          'estimated': rate && totalBytes && bytesTransferred <= totalBytes ?\n            (totalBytes - bytesTransferred) / rate : undefined\n        });\n      });\n    }, internals.ticksRate);\n\n    const onFinish = () => {\n      internals.updateProgress(true);\n    };\n\n    this.once('end', onFinish);\n    this.once('error', onFinish);\n  }\n\n  _read(size) {\n    const internals = this[kInternals];\n\n    if (internals.onReadCallback) {\n      internals.onReadCallback();\n    }\n\n    return super._read(size);\n  }\n\n  _transform(chunk, encoding, callback) {\n    const self = this;\n    const internals = this[kInternals];\n    const maxRate = internals.maxRate;\n\n    const readableHighWaterMark = this.readableHighWaterMark;\n\n    const timeWindow = internals.timeWindow;\n\n    const divider = 1000 / timeWindow;\n    const bytesThreshold = (maxRate / divider);\n    const minChunkSize = internals.minChunkSize !== false ? Math.max(internals.minChunkSize, bytesThreshold * 0.01) : 0;\n\n    function pushChunk(_chunk, _callback) {\n      const bytes = Buffer.byteLength(_chunk);\n      internals.bytesSeen += bytes;\n      internals.bytes += bytes;\n\n      if (internals.isCaptured) {\n        internals.updateProgress();\n      }\n\n      if (self.push(_chunk)) {\n        process.nextTick(_callback);\n      } else {\n        internals.onReadCallback = () => {\n          internals.onReadCallback = null;\n          process.nextTick(_callback);\n        };\n      }\n    }\n\n    const transformChunk = (_chunk, _callback) => {\n      const chunkSize = Buffer.byteLength(_chunk);\n      let chunkRemainder = null;\n      let maxChunkSize = readableHighWaterMark;\n      let bytesLeft;\n      let passed = 0;\n\n      if (maxRate) {\n        const now = Date.now();\n\n        if (!internals.ts || (passed = (now - internals.ts)) >= timeWindow) {\n          internals.ts = now;\n          bytesLeft = bytesThreshold - internals.bytes;\n          internals.bytes = bytesLeft < 0 ? -bytesLeft : 0;\n          passed = 0;\n        }\n\n        bytesLeft = bytesThreshold - internals.bytes;\n      }\n\n      if (maxRate) {\n        if (bytesLeft <= 0) {\n          // next time window\n          return setTimeout(() => {\n            _callback(null, _chunk);\n          }, timeWindow - passed);\n        }\n\n        if (bytesLeft < maxChunkSize) {\n          maxChunkSize = bytesLeft;\n        }\n      }\n\n      if (maxChunkSize && chunkSize > maxChunkSize && (chunkSize - maxChunkSize) > minChunkSize) {\n        chunkRemainder = _chunk.subarray(maxChunkSize);\n        _chunk = _chunk.subarray(0, maxChunkSize);\n      }\n\n      pushChunk(_chunk, chunkRemainder ? () => {\n        process.nextTick(_callback, null, chunkRemainder);\n      } : _callback);\n    };\n\n    transformChunk(chunk, function transformNextChunk(err, _chunk) {\n      if (err) {\n        return callback(err);\n      }\n\n      if (_chunk) {\n        transformChunk(_chunk, transformNextChunk);\n      } else {\n        callback(null);\n      }\n    });\n  }\n\n  setLength(length) {\n    this[kInternals].length = +length;\n    return this;\n  }\n}\n\nexport default AxiosTransformStream;\n\n// Path: lib/env/data.js\nexport const VERSION = \"1.6.8\";\n// Path: lib/adapters/http.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport buildURL from './../helpers/buildURL.js';\nimport {getProxyForUrl} from 'proxy-from-env';\nimport http from 'http';\nimport https from 'https';\nimport util from 'util';\nimport followRedirects from 'follow-redirects';\nimport zlib from 'zlib';\nimport {VERSION} from '../env/data.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport platform from '../platform/index.js';\nimport fromDataURI from '../helpers/fromDataURI.js';\nimport stream from 'stream';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport AxiosTransformStream from '../helpers/AxiosTransformStream.js';\nimport {EventEmitter} from 'events';\nimport formDataToStream from \"../helpers/formDataToStream.js\";\nimport readBlob from \"../helpers/readBlob.js\";\nimport ZlibHeaderTransformStream from '../helpers/ZlibHeaderTransformStream.js';\nimport callbackify from \"../helpers/callbackify.js\";\n\nconst zlibOptions = {\n  flush: zlib.constants.Z_SYNC_FLUSH,\n  finishFlush: zlib.constants.Z_SYNC_FLUSH\n};\n\nconst brotliOptions = {\n  flush: zlib.constants.BROTLI_OPERATION_FLUSH,\n  finishFlush: zlib.constants.BROTLI_OPERATION_FLUSH\n}\n\nconst isBrotliSupported = utils.isFunction(zlib.createBrotliDecompress);\n\nconst {http: httpFollow, https: httpsFollow} = followRedirects;\n\nconst isHttps = /https:?/;\n\nconst supportedProtocols = platform.protocols.map(protocol => {\n  return protocol + ':';\n});\n\n/**\n * If the proxy or config beforeRedirects functions are defined, call them with the options\n * object.\n *\n * @param {Object<string, any>} options - The options object that was passed to the request.\n *\n * @returns {Object<string, any>}\n */\nfunction dispatchBeforeRedirect(options, responseDetails) {\n  if (options.beforeRedirects.proxy) {\n    options.beforeRedirects.proxy(options);\n  }\n  if (options.beforeRedirects.config) {\n    options.beforeRedirects.config(options, responseDetails);\n  }\n}\n\n/**\n * If the proxy or config afterRedirects functions are defined, call them with the options\n *\n * @param {http.ClientRequestArgs} options\n * @param {AxiosProxyConfig} configProxy configuration from Axios options object\n * @param {string} location\n *\n * @returns {http.ClientRequestArgs}\n */\nfunction setProxy(options, configProxy, location) {\n  let proxy = configProxy;\n  if (!proxy && proxy !== false) {\n    const proxyUrl = getProxyForUrl(location);\n    if (proxyUrl) {\n      proxy = new URL(proxyUrl);\n    }\n  }\n  if (proxy) {\n    // Basic proxy authorization\n    if (proxy.username) {\n      proxy.auth = (proxy.username || '') + ':' + (proxy.password || '');\n    }\n\n    if (proxy.auth) {\n      // Support proxy auth object form\n      if (proxy.auth.username || proxy.auth.password) {\n        proxy.auth = (proxy.auth.username || '') + ':' + (proxy.auth.password || '');\n      }\n      const base64 = Buffer\n        .from(proxy.auth, 'utf8')\n        .toString('base64');\n      options.headers['Proxy-Authorization'] = 'Basic ' + base64;\n    }\n\n    options.headers.host = options.hostname + (options.port ? ':' + options.port : '');\n    const proxyHost = proxy.hostname || proxy.host;\n    options.hostname = proxyHost;\n    // Replace 'host' since options is not a URL object\n    options.host = proxyHost;\n    options.port = proxy.port;\n    options.path = location;\n    if (proxy.protocol) {\n      options.protocol = proxy.protocol.includes(':') ? proxy.protocol : `${proxy.protocol}:`;\n    }\n  }\n\n  options.beforeRedirects.proxy = function beforeRedirect(redirectOptions) {\n    // Configure proxy for redirected request, passing the original config proxy to apply\n    // the exact same logic as if the redirected request was performed by axios directly.\n    setProxy(redirectOptions, configProxy, redirectOptions.href);\n  };\n}\n\nconst isHttpAdapterSupported = typeof process !== 'undefined' && utils.kindOf(process) === 'process';\n\n// temporary hotfix\n\nconst wrapAsync = (asyncExecutor) => {\n...\n", "needle_token_start": 6998, "needle_token_end": 7146, "code_context_ntokens": 15865, "output": ["```javascript\n/**\n * Resolve or reject a Promise based on response status.\n *\n * @param {Function} resolve A function that resolves the promise.\n * @param {Function} reject A function that rejects the promise.\n * @param {object} response The response.\n *\n * @returns {object} The response.\n */\nexport default function settle(resolve, reject, response) {\n  const validateStatus = response.config.validateStatus;\n  if (!response.status || !validateStatus || validateStatus(response.status)) {\n    resolve(response);\n  } else {\n    reject(new AxiosError(\n      'Request failed with status code ' + response.status,\n      [AxiosError.ERR_BAD_REQUEST, AxiosError.ERR_BAD_RESPONSE][Math.floor(response.status / 100) - 4],\n      response.config,\n      response.request,\n      response\n    ));\n  }\n}\n```"]}
{"repo": "axios/axios", "name": "AxiosError", "language": "typescript", "path": "lib/core/AxiosError.js", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a specialized error object that includes additional information useful for debugging HTTP requests. It extends the standard error by including details such as the HTTP request configuration, the request itself, and the server's response.\n2. **Input**: The function accepts several parameters: a message string describing the error, an optional error code string, an optional configuration object of the HTTP request, an optional request object, and an optional response object from the server.\n3. **Output**: The output is an error object that contains the standard error properties along with additional properties that provide more context about the error related to HTTP requests.\n4. **Procedure**: The function first calls the standard Error constructor to initialize the base properties. It then optionally captures the stack trace to help with debugging. The function assigns the provided message to the error object and sets its name to a specific string indicating it is a specialized error type. If provided, additional properties such as the error code, configuration, request, and response are attached to the error object to provide further details.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/platform/node/classes/FormData.js\nimport FormData from 'form-data';\n\nexport default FormData;\n\n// Path: lib/platform/node/classes/URLSearchParams.js\n'use strict';\n\nimport url from 'url';\nexport default url.URLSearchParams;\n\n// Path: lib/platform/node/index.js\nimport URLSearchParams from './classes/URLSearchParams.js'\nimport FormData from './classes/FormData.js'\n\nexport default {\n  isNode: true,\n  classes: {\n    URLSearchParams,\n    FormData,\n    Blob: typeof Blob !== 'undefined' && Blob || null\n  },\n  protocols: [ 'http', 'https', 'file', 'data' ]\n};\n\n// Path: lib/platform/common/utils.js\nconst hasBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';\n\n/**\n * Determine if we're running in a standard browser environment\n *\n * This allows axios to run in a web worker, and react-native.\n * Both environments support XMLHttpRequest, but not fully standard globals.\n *\n * web workers:\n *  typeof window -> undefined\n *  typeof document -> undefined\n *\n * react-native:\n *  navigator.product -> 'ReactNative'\n * nativescript\n *  navigator.product -> 'NativeScript' or 'NS'\n *\n * @returns {boolean}\n */\nconst hasStandardBrowserEnv = (\n  (product) => {\n    return hasBrowserEnv && ['ReactNative', 'NativeScript', 'NS'].indexOf(product) < 0\n  })(typeof navigator !== 'undefined' && navigator.product);\n\n/**\n * Determine if we're running in a standard browser webWorker environment\n *\n * Although the `isStandardBrowserEnv` method indicates that\n * `allows axios to run in a web worker`, the WebWorker will still be\n * filtered out due to its judgment standard\n * `typeof window !== 'undefined' && typeof document !== 'undefined'`.\n * This leads to a problem when axios post `FormData` in webWorker\n */\nconst hasStandardBrowserWebWorkerEnv = (() => {\n  return (\n    typeof WorkerGlobalScope !== 'undefined' &&\n    // eslint-disable-next-line no-undef\n    self instanceof WorkerGlobalScope &&\n    typeof self.importScripts === 'function'\n  );\n})();\n\nexport {\n  hasBrowserEnv,\n  hasStandardBrowserWebWorkerEnv,\n  hasStandardBrowserEnv\n}\n\n// Path: lib/platform/index.js\nimport platform from './node/index.js';\nimport * as utils from './common/utils.js';\n\nexport default {\n  ...utils,\n  ...platform\n}\n\n// Path: lib/helpers/combineURLs.js\n'use strict';\n\n/**\n * Creates a new URL by combining the specified URLs\n *\n * @param {string} baseURL The base URL\n * @param {string} relativeURL The relative URL\n *\n * @returns {string} The combined URL\n */\nexport default function combineURLs(baseURL, relativeURL) {\n  return relativeURL\n    ? baseURL.replace(/\\/?\\/$/, '') + '/' + relativeURL.replace(/^\\/+/, '')\n    : baseURL;\n}\n\n// Path: lib/helpers/isAbsoluteURL.js\n'use strict';\n\n/**\n * Determines whether the specified URL is absolute\n *\n * @param {string} url The URL to test\n *\n * @returns {boolean} True if the specified URL is absolute, otherwise false\n */\nexport default function isAbsoluteURL(url) {\n  // A URL is considered absolute if it begins with \"<scheme>://\" or \"//\" (protocol-relative URL).\n  // RFC 3986 defines scheme name as a sequence of characters beginning with a letter and followed\n  // by any combination of letters, digits, plus, period, or hyphen.\n  return /^([a-z][a-z\\d+\\-.]*:)?\\/\\//i.test(url);\n}\n\n// Path: lib/core/buildFullPath.js\n'use strict';\n\nimport isAbsoluteURL from '../helpers/isAbsoluteURL.js';\nimport combineURLs from '../helpers/combineURLs.js';\n\n/**\n * Creates a new URL by combining the baseURL with the requestedURL,\n * only when the requestedURL is not already an absolute URL.\n * If the requestURL is absolute, this function returns the requestedURL untouched.\n *\n * @param {string} baseURL The base URL\n * @param {string} requestedURL Absolute or relative URL to combine\n *\n * @returns {string} The combined full path\n */\nexport default function buildFullPath(baseURL, requestedURL) {\n  if (baseURL && !isAbsoluteURL(requestedURL)) {\n    return combineURLs(baseURL, requestedURL);\n  }\n  return requestedURL;\n}\n\n// Path: lib/helpers/bind.js\n'use strict';\n\nexport default function bind(fn, thisArg) {\n  return function wrap() {\n    return fn.apply(thisArg, arguments);\n  };\n}\n\n// Path: lib/utils.js\n'use strict';\n\nimport bind from './helpers/bind.js';\n\n// utils is a library of generic helper functions non-specific to axios\n\nconst {toString} = Object.prototype;\nconst {getPrototypeOf} = Object;\n\nconst kindOf = (cache => thing => {\n    const str = toString.call(thing);\n    return cache[str] || (cache[str] = str.slice(8, -1).toLowerCase());\n})(Object.create(null));\n\nconst kindOfTest = (type) => {\n  type = type.toLowerCase();\n  return (thing) => kindOf(thing) === type\n}\n\nconst typeOfTest = type => thing => typeof thing === type;\n\n/**\n * Determine if a value is an Array\n *\n * @param {Object} val The value to test\n *\n * @returns {boolean} True if value is an Array, otherwise false\n */\nconst {isArray} = Array;\n\n/**\n * Determine if a value is undefined\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if the value is undefined, otherwise false\n */\nconst isUndefined = typeOfTest('undefined');\n\n/**\n * Determine if a value is a Buffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Buffer, otherwise false\n */\nfunction isBuffer(val) {\n  return val !== null && !isUndefined(val) && val.constructor !== null && !isUndefined(val.constructor)\n    && isFunction(val.constructor.isBuffer) && val.constructor.isBuffer(val);\n}\n\n/**\n * Determine if a value is an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is an ArrayBuffer, otherwise false\n */\nconst isArrayBuffer = kindOfTest('ArrayBuffer');\n\n\n/**\n * Determine if a value is a view on an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a view on an ArrayBuffer, otherwise false\n */\nfunction isArrayBufferView(val) {\n  let result;\n  if ((typeof ArrayBuffer !== 'undefined') && (ArrayBuffer.isView)) {\n    result = ArrayBuffer.isView(val);\n  } else {\n    result = (val) && (val.buffer) && (isArrayBuffer(val.buffer));\n  }\n  return result;\n}\n\n/**\n * Determine if a value is a String\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a String, otherwise false\n */\nconst isString = typeOfTest('string');\n\n/**\n * Determine if a value is a Function\n *\n * @param {*} val The value to test\n * @returns {boolean} True if value is a Function, otherwise false\n */\nconst isFunction = typeOfTest('function');\n\n/**\n * Determine if a value is a Number\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Number, otherwise false\n */\nconst isNumber = typeOfTest('number');\n\n/**\n * Determine if a value is an Object\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an Object, otherwise false\n */\nconst isObject = (thing) => thing !== null && typeof thing === 'object';\n\n/**\n * Determine if a value is a Boolean\n *\n * @param {*} thing The value to test\n * @returns {boolean} True if value is a Boolean, otherwise false\n */\nconst isBoolean = thing => thing === true || thing === false;\n\n/**\n * Determine if a value is a plain Object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a plain Object, otherwise false\n */\nconst isPlainObject = (val) => {\n  if (kindOf(val) !== 'object') {\n    return false;\n  }\n\n  const prototype = getPrototypeOf(val);\n  return (prototype === null || prototype === Object.prototype || Object.getPrototypeOf(prototype) === null) && !(Symbol.toStringTag in val) && !(Symbol.iterator in val);\n}\n\n/**\n * Determine if a value is a Date\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Date, otherwise false\n */\nconst isDate = kindOfTest('Date');\n\n/**\n * Determine if a value is a File\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFile = kindOfTest('File');\n\n/**\n * Determine if a value is a Blob\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Blob, otherwise false\n */\nconst isBlob = kindOfTest('Blob');\n\n/**\n * Determine if a value is a FileList\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFileList = kindOfTest('FileList');\n\n/**\n * Determine if a value is a Stream\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Stream, otherwise false\n */\nconst isStream = (val) => isObject(val) && isFunction(val.pipe);\n\n/**\n * Determine if a value is a FormData\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an FormData, otherwise false\n */\nconst isFormData = (thing) => {\n  let kind;\n  return thing && (\n    (typeof FormData === 'function' && thing instanceof FormData) || (\n      isFunction(thing.append) && (\n        (kind = kindOf(thing)) === 'formdata' ||\n        // detect form-data instance\n        (kind === 'object' && isFunction(thing.toString) && thing.toString() === '[object FormData]')\n      )\n    )\n  )\n}\n\n/**\n * Determine if a value is a URLSearchParams object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a URLSearchParams object, otherwise false\n */\nconst isURLSearchParams = kindOfTest('URLSearchParams');\n\n/**\n * Trim excess whitespace off the beginning and end of a string\n *\n * @param {String} str The String to trim\n *\n * @returns {String} The String freed of excess whitespace\n */\nconst trim = (str) => str.trim ?\n  str.trim() : str.replace(/^[\\s\\uFEFF\\xA0]+|[\\s\\uFEFF\\xA0]+$/g, '');\n\n/**\n * Iterate over an Array or an Object invoking a function for each item.\n *\n * If `obj` is an Array callback will be called passing\n * the value, index, and complete array for each item.\n *\n * If 'obj' is an Object callback will be called passing\n * the value, key, and complete object for each property.\n *\n * @param {Object|Array} obj The object to iterate\n * @param {Function} fn The callback to invoke for each item\n *\n * @param {Boolean} [allOwnKeys = false]\n * @returns {any}\n */\nfunction forEach(obj, fn, {allOwnKeys = false} = {}) {\n  // Don't bother if no value provided\n  if (obj === null || typeof obj === 'undefined') {\n    return;\n  }\n\n  let i;\n  let l;\n\n  // Force an array if not already something iterable\n  if (typeof obj !== 'object') {\n    /*eslint no-param-reassign:0*/\n    obj = [obj];\n  }\n\n  if (isArray(obj)) {\n    // Iterate over array values\n    for (i = 0, l = obj.length; i < l; i++) {\n      fn.call(null, obj[i], i, obj);\n    }\n  } else {\n    // Iterate over object keys\n    const keys = allOwnKeys ? Object.getOwnPropertyNames(obj) : Object.keys(obj);\n    const len = keys.length;\n    let key;\n\n    for (i = 0; i < len; i++) {\n      key = keys[i];\n      fn.call(null, obj[key], key, obj);\n    }\n  }\n}\n\nfunction findKey(obj, key) {\n  key = key.toLowerCase();\n  const keys = Object.keys(obj);\n  let i = keys.length;\n  let _key;\n  while (i-- > 0) {\n    _key = keys[i];\n    if (key === _key.toLowerCase()) {\n      return _key;\n    }\n  }\n  return null;\n}\n\nconst _global = (() => {\n  /*eslint no-undef:0*/\n  if (typeof globalThis !== \"undefined\") return globalThis;\n  return typeof self !== \"undefined\" ? self : (typeof window !== 'undefined' ? window : global)\n})();\n\nconst isContextDefined = (context) => !isUndefined(context) && context !== _global;\n\n/**\n * Accepts varargs expecting each argument to be an object, then\n * immutably merges the properties of each object and returns result.\n *\n * When multiple objects contain the same key the later object in\n * the arguments list will take precedence.\n *\n * Example:\n *\n * ```js\n * var result = merge({foo: 123}, {foo: 456});\n * console.log(result.foo); // outputs 456\n * ```\n *\n * @param {Object} obj1 Object to merge\n *\n * @returns {Object} Result of all merge properties\n */\nfunction merge(/* obj1, obj2, obj3, ... */) {\n  const {caseless} = isContextDefined(this) && this || {};\n  const result = {};\n  const assignValue = (val, key) => {\n    const targetKey = caseless && findKey(result, key) || key;\n    if (isPlainObject(result[targetKey]) && isPlainObject(val)) {\n      result[targetKey] = merge(result[targetKey], val);\n    } else if (isPlainObject(val)) {\n      result[targetKey] = merge({}, val);\n    } else if (isArray(val)) {\n      result[targetKey] = val.slice();\n    } else {\n      result[targetKey] = val;\n    }\n  }\n\n  for (let i = 0, l = arguments.length; i < l; i++) {\n    arguments[i] && forEach(arguments[i], assignValue);\n  }\n  return result;\n}\n\n/**\n * Extends object a by mutably adding to it the properties of object b.\n *\n * @param {Object} a The object to be extended\n * @param {Object} b The object to copy properties from\n * @param {Object} thisArg The object to bind function to\n *\n * @param {Boolean} [allOwnKeys]\n * @returns {Object} The resulting value of object a\n */\nconst extend = (a, b, thisArg, {allOwnKeys}= {}) => {\n  forEach(b, (val, key) => {\n    if (thisArg && isFunction(val)) {\n      a[key] = bind(val, thisArg);\n    } else {\n      a[key] = val;\n    }\n  }, {allOwnKeys});\n  return a;\n}\n\n/**\n * Remove byte order marker. This catches EF BB BF (the UTF-8 BOM)\n *\n * @param {string} content with BOM\n *\n * @returns {string} content value without BOM\n */\nconst stripBOM = (content) => {\n  if (content.charCodeAt(0) === 0xFEFF) {\n    content = content.slice(1);\n  }\n  return content;\n}\n\n/**\n * Inherit the prototype methods from one constructor into another\n * @param {function} constructor\n * @param {function} superConstructor\n * @param {object} [props]\n * @param {object} [descriptors]\n *\n * @returns {void}\n */\nconst inherits = (constructor, superConstructor, props, descriptors) => {\n  constructor.prototype = Object.create(superConstructor.prototype, descriptors);\n  constructor.prototype.constructor = constructor;\n  Object.defineProperty(constructor, 'super', {\n    value: superConstructor.prototype\n  });\n  props && Object.assign(constructor.prototype, props);\n}\n\n/**\n * Resolve object with deep prototype chain to a flat object\n * @param {Object} sourceObj source object\n * @param {Object} [destObj]\n * @param {Function|Boolean} [filter]\n * @param {Function} [propFilter]\n *\n * @returns {Object}\n */\nconst toFlatObject = (sourceObj, destObj, filter, propFilter) => {\n  let props;\n  let i;\n  let prop;\n  const merged = {};\n\n  destObj = destObj || {};\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  if (sourceObj == null) return destObj;\n\n  do {\n    props = Object.getOwnPropertyNames(sourceObj);\n    i = props.length;\n    while (i-- > 0) {\n      prop = props[i];\n      if ((!propFilter || propFilter(prop, sourceObj, destObj)) && !merged[prop]) {\n        destObj[prop] = sourceObj[prop];\n        merged[prop] = true;\n      }\n    }\n    sourceObj = filter !== false && getPrototypeOf(sourceObj);\n  } while (sourceObj && (!filter || filter(sourceObj, destObj)) && sourceObj !== Object.prototype);\n\n  return destObj;\n}\n\n/**\n * Determines whether a string ends with the characters of a specified string\n *\n * @param {String} str\n * @param {String} searchString\n * @param {Number} [position= 0]\n *\n * @returns {boolean}\n */\nconst endsWith = (str, searchString, position) => {\n  str = String(str);\n  if (position === undefined || position > str.length) {\n    position = str.length;\n  }\n  position -= searchString.length;\n  const lastIndex = str.indexOf(searchString, position);\n  return lastIndex !== -1 && lastIndex === position;\n}\n\n\n/**\n * Returns new array from array like object or null if failed\n *\n * @param {*} [thing]\n *\n * @returns {?Array}\n */\nconst toArray = (thing) => {\n  if (!thing) return null;\n  if (isArray(thing)) return thing;\n  let i = thing.length;\n  if (!isNumber(i)) return null;\n  const arr = new Array(i);\n  while (i-- > 0) {\n    arr[i] = thing[i];\n  }\n  return arr;\n}\n\n/**\n * Checking if the Uint8Array exists and if it does, it returns a function that checks if the\n * thing passed in is an instance of Uint8Array\n *\n * @param {TypedArray}\n *\n * @returns {Array}\n */\n// eslint-disable-next-line func-names\nconst isTypedArray = (TypedArray => {\n  // eslint-disable-next-line func-names\n  return thing => {\n    return TypedArray && thing instanceof TypedArray;\n  };\n})(typeof Uint8Array !== 'undefined' && getPrototypeOf(Uint8Array));\n\n/**\n * For each entry in the object, call the function with the key and value.\n *\n * @param {Object<any, any>} obj - The object to iterate over.\n * @param {Function} fn - The function to call for each entry.\n *\n * @returns {void}\n */\nconst forEachEntry = (obj, fn) => {\n  const generator = obj && obj[Symbol.iterator];\n\n  const iterator = generator.call(obj);\n\n  let result;\n\n  while ((result = iterator.next()) && !result.done) {\n    const pair = result.value;\n    fn.call(obj, pair[0], pair[1]);\n  }\n}\n\n/**\n * It takes a regular expression and a string, and returns an array of all the matches\n *\n * @param {string} regExp - The regular expression to match against.\n * @param {string} str - The string to search.\n *\n * @returns {Array<boolean>}\n */\nconst matchAll = (regExp, str) => {\n  let matches;\n  const arr = [];\n\n  while ((matches = regExp.exec(str)) !== null) {\n    arr.push(matches);\n  }\n\n  return arr;\n}\n\n/* Checking if the kindOfTest function returns true when passed an HTMLFormElement. */\nconst isHTMLForm = kindOfTest('HTMLFormElement');\n\nconst toCamelCase = str => {\n  return str.toLowerCase().replace(/[-_\\s]([a-z\\d])(\\w*)/g,\n    function replacer(m, p1, p2) {\n      return p1.toUpperCase() + p2;\n    }\n  );\n};\n\n/* Creating a function that will check if an object has a property. */\nconst hasOwnProperty = (({hasOwnProperty}) => (obj, prop) => hasOwnProperty.call(obj, prop))(Object.prototype);\n\n/**\n * Determine if a value is a RegExp object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a RegExp object, otherwise false\n */\nconst isRegExp = kindOfTest('RegExp');\n\nconst reduceDescriptors = (obj, reducer) => {\n  const descriptors = Object.getOwnPropertyDescriptors(obj);\n  const reducedDescriptors = {};\n\n  forEach(descriptors, (descriptor, name) => {\n    let ret;\n    if ((ret = reducer(descriptor, name, obj)) !== false) {\n      reducedDescriptors[name] = ret || descriptor;\n    }\n  });\n\n  Object.defineProperties(obj, reducedDescriptors);\n}\n\n/**\n * Makes all methods read-only\n * @param {Object} obj\n */\n\nconst freezeMethods = (obj) => {\n  reduceDescriptors(obj, (descriptor, name) => {\n    // skip restricted props in strict mode\n    if (isFunction(obj) && ['arguments', 'caller', 'callee'].indexOf(name) !== -1) {\n      return false;\n    }\n\n    const value = obj[name];\n\n    if (!isFunction(value)) return;\n\n    descriptor.enumerable = false;\n\n    if ('writable' in descriptor) {\n      descriptor.writable = false;\n      return;\n    }\n\n    if (!descriptor.set) {\n      descriptor.set = () => {\n        throw Error('Can not rewrite read-only method \\'' + name + '\\'');\n      };\n    }\n  });\n}\n\nconst toObjectSet = (arrayOrString, delimiter) => {\n  const obj = {};\n\n  const define = (arr) => {\n    arr.forEach(value => {\n      obj[value] = true;\n    });\n  }\n\n  isArray(arrayOrString) ? define(arrayOrString) : define(String(arrayOrString).split(delimiter));\n\n  return obj;\n}\n\nconst noop = () => {}\n\nconst toFiniteNumber = (value, defaultValue) => {\n  value = +value;\n  return Number.isFinite(value) ? value : defaultValue;\n}\n\nconst ALPHA = 'abcdefghijklmnopqrstuvwxyz'\n\nconst DIGIT = '0123456789';\n\nconst ALPHABET = {\n  DIGIT,\n  ALPHA,\n  ALPHA_DIGIT: ALPHA + ALPHA.toUpperCase() + DIGIT\n}\n\nconst generateString = (size = 16, alphabet = ALPHABET.ALPHA_DIGIT) => {\n  let str = '';\n  const {length} = alphabet;\n  while (size--) {\n    str += alphabet[Math.random() * length|0]\n  }\n\n  return str;\n}\n\n/**\n * If the thing is a FormData object, return true, otherwise return false.\n *\n * @param {unknown} thing - The thing to check.\n *\n * @returns {boolean}\n */\nfunction isSpecCompliantForm(thing) {\n  return !!(thing && isFunction(thing.append) && thing[Symbol.toStringTag] === 'FormData' && thing[Symbol.iterator]);\n}\n\nconst toJSONObject = (obj) => {\n  const stack = new Array(10);\n\n  const visit = (source, i) => {\n\n    if (isObject(source)) {\n      if (stack.indexOf(source) >= 0) {\n        return;\n      }\n\n      if(!('toJSON' in source)) {\n        stack[i] = source;\n        const target = isArray(source) ? [] : {};\n\n        forEach(source, (value, key) => {\n          const reducedValue = visit(value, i + 1);\n          !isUndefined(reducedValue) && (target[key] = reducedValue);\n        });\n\n        stack[i] = undefined;\n\n        return target;\n      }\n    }\n\n    return source;\n  }\n\n  return visit(obj, 0);\n}\n\nconst isAsyncFn = kindOfTest('AsyncFunction');\n\nconst isThenable = (thing) =>\n  thing && (isObject(thing) || isFunction(thing)) && isFunction(thing.then) && isFunction(thing.catch);\n\nexport default {\n  isArray,\n  isArrayBuffer,\n  isBuffer,\n  isFormData,\n  isArrayBufferView,\n  isString,\n  isNumber,\n  isBoolean,\n  isObject,\n  isPlainObject,\n  isUndefined,\n  isDate,\n  isFile,\n  isBlob,\n  isRegExp,\n  isFunction,\n  isStream,\n  isURLSearchParams,\n  isTypedArray,\n  isFileList,\n  forEach,\n  merge,\n  extend,\n  trim,\n  stripBOM,\n  inherits,\n  toFlatObject,\n  kindOf,\n  kindOfTest,\n  endsWith,\n  toArray,\n  forEachEntry,\n  matchAll,\n  isHTMLForm,\n  hasOwnProperty,\n  hasOwnProp: hasOwnProperty, // an alias to avoid ESLint no-prototype-builtins detection\n  reduceDescriptors,\n  freezeMethods,\n  toObjectSet,\n  toCamelCase,\n  noop,\n  toFiniteNumber,\n  findKey,\n  global: _global,\n  isContextDefined,\n  ALPHABET,\n  generateString,\n  isSpecCompliantForm,\n  toJSONObject,\n  isAsyncFn,\n  isThenable\n};\n\n// Path: lib/core/AxiosError.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\n\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: utils.toJSONObject(this.config),\n      code: this.code,\n      status: this.response && this.response.status ? this.response.status : null\n    };\n  }\n});\n\nconst prototype = AxiosError.prototype;\nconst descriptors = {};\n\n[\n  'ERR_BAD_OPTION_VALUE',\n  'ERR_BAD_OPTION',\n  'ECONNABORTED',\n  'ETIMEDOUT',\n  'ERR_NETWORK',\n  'ERR_FR_TOO_MANY_REDIRECTS',\n  'ERR_DEPRECATED',\n  'ERR_BAD_RESPONSE',\n  'ERR_BAD_REQUEST',\n  'ERR_CANCELED',\n  'ERR_NOT_SUPPORT',\n  'ERR_INVALID_URL'\n// eslint-disable-next-line func-names\n].forEach(code => {\n  descriptors[code] = {value: code};\n});\n\nObject.defineProperties(AxiosError, descriptors);\nObject.defineProperty(prototype, 'isAxiosError', {value: true});\n\n// eslint-disable-next-line func-names\nAxiosError.from = (error, code, config, request, response, customProps) => {\n  const axiosError = Object.create(prototype);\n\n  utils.toFlatObject(error, axiosError, function filter(obj) {\n    return obj !== Error.prototype;\n  }, prop => {\n    return prop !== 'isAxiosError';\n  });\n\n  AxiosError.call(axiosError, error.message, code, config, request, response);\n\n  axiosError.cause = error;\n\n  axiosError.name = error.name;\n\n  customProps && Object.assign(axiosError, customProps);\n\n  return axiosError;\n};\n\nexport default AxiosError;\n\n// Path: lib/helpers/toFormData.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\n// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored\nimport PlatformFormData from '../platform/node/classes/FormData.js';\n\n/**\n * Determines if the given thing is a array or js object.\n *\n * @param {string} thing - The object or array to be visited.\n *\n * @returns {boolean}\n */\nfunction isVisitable(thing) {\n  return utils.isPlainObject(thing) || utils.isArray(thing);\n}\n\n/**\n * It removes the brackets from the end of a string\n *\n * @param {string} key - The key of the parameter.\n *\n * @returns {string} the key without the brackets.\n */\nfunction removeBrackets(key) {\n  return utils.endsWith(key, '[]') ? key.slice(0, -2) : key;\n}\n\n/**\n * It takes a path, a key, and a boolean, and returns a string\n *\n * @param {string} path - The path to the current key.\n * @param {string} key - The key of the current object being iterated over.\n * @param {string} dots - If true, the key will be rendered with dots instead of brackets.\n *\n * @returns {string} The path to the current key.\n */\nfunction renderKey(path, key, dots) {\n  if (!path) return key;\n  return path.concat(key).map(function each(token, i) {\n    // eslint-disable-next-line no-param-reassign\n    token = removeBrackets(token);\n    return !dots && i ? '[' + token + ']' : token;\n  }).join(dots ? '.' : '');\n}\n\n/**\n * If the array is an array and none of its elements are visitable, then it's a flat array.\n *\n * @param {Array<any>} arr - The array to check\n *\n * @returns {boolean}\n */\nfunction isFlatArray(arr) {\n  return utils.isArray(arr) && !arr.some(isVisitable);\n}\n\nconst predicates = utils.toFlatObject(utils, {}, null, function filter(prop) {\n  return /^is[A-Z]/.test(prop);\n});\n\n/**\n * Convert a data object to FormData\n *\n * @param {Object} obj\n * @param {?Object} [formData]\n * @param {?Object} [options]\n * @param {Function} [options.visitor]\n * @param {Boolean} [options.metaTokens = true]\n * @param {Boolean} [options.dots = false]\n * @param {?Boolean} [options.indexes = false]\n *\n * @returns {Object}\n **/\n\n/**\n * It converts an object into a FormData object\n *\n * @param {Object<any, any>} obj - The object to convert to form data.\n * @param {string} formData - The FormData object to append to.\n * @param {Object<string, any>} options\n *\n * @returns\n */\nfunction toFormData(obj, formData, options) {\n  if (!utils.isObject(obj)) {\n    throw new TypeError('target must be an object');\n  }\n\n  // eslint-disable-next-line no-param-reassign\n  formData = formData || new (PlatformFormData || FormData)();\n\n  // eslint-disable-next-line no-param-reassign\n  options = utils.toFlatObject(options, {\n    metaTokens: true,\n    dots: false,\n    indexes: false\n  }, false, function defined(option, source) {\n    // eslint-disable-next-line no-eq-null,eqeqeq\n    return !utils.isUndefined(source[option]);\n  });\n\n  const metaTokens = options.metaTokens;\n  // eslint-disable-next-line no-use-before-define\n  const visitor = options.visitor || defaultVisitor;\n  const dots = options.dots;\n  const indexes = options.indexes;\n  const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob;\n  const useBlob = _Blob && utils.isSpecCompliantForm(formData);\n\n  if (!utils.isFunction(visitor)) {\n    throw new TypeError('visitor must be a function');\n  }\n\n  function convertValue(value) {\n    if (value === null) return '';\n\n    if (utils.isDate(value)) {\n      return value.toISOString();\n    }\n\n    if (!useBlob && utils.isBlob(value)) {\n      throw new AxiosError('Blob is not supported. Use a Buffer instead.');\n    }\n\n    if (utils.isArrayBuffer(value) || utils.isTypedArray(value)) {\n      return useBlob && typeof Blob === 'function' ? new Blob([value]) : Buffer.from(value);\n    }\n\n    return value;\n  }\n\n  /**\n   * Default visitor.\n   *\n   * @param {*} value\n   * @param {String|Number} key\n   * @param {Array<String|Number>} path\n   * @this {FormData}\n   *\n   * @returns {boolean} return true to visit the each prop of the value recursively\n   */\n  function defaultVisitor(value, key, path) {\n    let arr = value;\n\n    if (value && !path && typeof value === 'object') {\n      if (utils.endsWith(key, '{}')) {\n        // eslint-disable-next-line no-param-reassign\n        key = metaTokens ? key : key.slice(0, -2);\n        // eslint-disable-next-line no-param-reassign\n        value = JSON.stringify(value);\n      } else if (\n        (utils.isArray(value) && isFlatArray(value)) ||\n        ((utils.isFileList(value) || utils.endsWith(key, '[]')) && (arr = utils.toArray(value))\n        )) {\n        // eslint-disable-next-line no-param-reassign\n        key = removeBrackets(key);\n\n        arr.forEach(function each(el, index) {\n          !(utils.isUndefined(el) || el === null) && formData.append(\n            // eslint-disable-next-line no-nested-ternary\n            indexes === true ? renderKey([key], index, dots) : (indexes === null ? key : key + '[]'),\n            convertValue(el)\n          );\n        });\n        return false;\n      }\n    }\n\n    if (isVisitable(value)) {\n      return true;\n    }\n\n    formData.append(renderKey(path, key, dots), convertValue(value));\n\n    return false;\n  }\n\n  const stack = [];\n\n  const exposedHelpers = Object.assign(predicates, {\n    defaultVisitor,\n    convertValue,\n    isVisitable\n  });\n\n  function build(value, path) {\n    if (utils.isUndefined(value)) return;\n\n    if (stack.indexOf(value) !== -1) {\n      throw Error('Circular reference detected in ' + path.join('.'));\n    }\n\n    stack.push(value);\n\n    utils.forEach(value, function each(el, key) {\n      const result = !(utils.isUndefined(el) || el === null) && visitor.call(\n        formData, el, utils.isString(key) ? key.trim() : key, path, exposedHelpers\n      );\n\n      if (result === true) {\n        build(el, path ? path.concat(key) : [key]);\n      }\n    });\n\n    stack.pop();\n  }\n\n  if (!utils.isObject(obj)) {\n    throw new TypeError('data must be an object');\n  }\n\n  build(obj);\n\n  return formData;\n}\n\nexport default toFormData;\n\n// Path: lib/helpers/AxiosURLSearchParams.js\n'use strict';\n\nimport toFormData from './toFormData.js';\n\n/**\n * It encodes a string by replacing all characters that are not in the unreserved set with\n * their percent-encoded equivalents\n *\n * @param {string} str - The string to encode.\n *\n * @returns {string} The encoded string.\n */\nfunction encode(str) {\n  const charMap = {\n    '!': '%21',\n    \"'\": '%27',\n    '(': '%28',\n    ')': '%29',\n    '~': '%7E',\n    '%20': '+',\n    '%00': '\\x00'\n  };\n  return encodeURIComponent(str).replace(/[!'()~]|%20|%00/g, function replacer(match) {\n    return charMap[match];\n  });\n}\n\n/**\n * It takes a params object and converts it to a FormData object\n *\n * @param {Object<string, any>} params - The parameters to be converted to a FormData object.\n * @param {Object<string, any>} options - The options object passed to the Axios constructor.\n *\n * @returns {void}\n */\nfunction AxiosURLSearchParams(params, options) {\n  this._pairs = [];\n\n  params && toFormData(params, this, options);\n}\n\nconst prototype = AxiosURLSearchParams.prototype;\n\nprototype.append = function append(name, value) {\n  this._pairs.push([name, value]);\n};\n\nprototype.toString = function toString(encoder) {\n  const _encode = encoder ? function(value) {\n    return encoder.call(this, value, encode);\n  } : encode;\n\n  return this._pairs.map(function each(pair) {\n    return _encode(pair[0]) + '=' + _encode(pair[1]);\n  }, '').join('&');\n};\n\nexport default AxiosURLSearchParams;\n\n// Path: lib/helpers/buildURL.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosURLSearchParams from '../helpers/AxiosURLSearchParams.js';\n\n/**\n * It replaces all instances of the characters `:`, `$`, `,`, `+`, `[`, and `]` with their\n * URI encoded counterparts\n *\n * @param {string} val The value to be encoded.\n *\n * @returns {string} The encoded value.\n */\nfunction encode(val) {\n  return encodeURIComponent(val).\n    replace(/%3A/gi, ':').\n    replace(/%24/g, '$').\n    replace(/%2C/gi, ',').\n    replace(/%20/g, '+').\n    replace(/%5B/gi, '[').\n    replace(/%5D/gi, ']');\n}\n\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default function buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n\n// Path: lib/cancel/CanceledError.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n\n/**\n * A `CanceledError` is an object that is thrown when an operation is canceled.\n *\n * @param {string=} message The message.\n * @param {Object=} config The config.\n * @param {Object=} request The request.\n *\n * @returns {CanceledError} The created error.\n */\nfunction CanceledError(message, config, request) {\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  AxiosError.call(this, message == null ? 'canceled' : message, AxiosError.ERR_CANCELED, config, request);\n  this.name = 'CanceledError';\n}\n\nutils.inherits(CanceledError, AxiosError, {\n  __CANCEL__: true\n});\n\nexport default CanceledError;\n\n// Path: lib/defaults/transitional.js\n'use strict';\n\nexport default {\n  silentJSONParsing: true,\n  forcedJSONParsing: true,\n  clarifyTimeoutError: false\n};\n\n// Path: lib/helpers/speedometer.js\n'use strict';\n\n/**\n * Calculate data maxRate\n * @param {Number} [samplesCount= 10]\n * @param {Number} [min= 1000]\n * @returns {Function}\n */\nfunction speedometer(samplesCount, min) {\n  samplesCount = samplesCount || 10;\n  const bytes = new Array(samplesCount);\n  const timestamps = new Array(samplesCount);\n  let head = 0;\n  let tail = 0;\n  let firstSampleTS;\n\n  min = min !== undefined ? min : 1000;\n\n  return function push(chunkLength) {\n    const now = Date.now();\n\n    const startedAt = timestamps[tail];\n\n    if (!firstSampleTS) {\n      firstSampleTS = now;\n    }\n\n    bytes[head] = chunkLength;\n    timestamps[head] = now;\n\n    let i = tail;\n    let bytesCount = 0;\n\n    while (i !== head) {\n      bytesCount += bytes[i++];\n      i = i % samplesCount;\n    }\n\n    head = (head + 1) % samplesCount;\n\n    if (head === tail) {\n      tail = (tail + 1) % samplesCount;\n    }\n\n    if (now - firstSampleTS < min) {\n      return;\n    }\n\n    const passed = startedAt && now - startedAt;\n\n    return passed ? Math.round(bytesCount * 1000 / passed) : undefined;\n  };\n}\n\nexport default speedometer;\n\n// Path: lib/helpers/parseHeaders.js\n'use strict';\n\nimport utils from './../utils.js';\n\n// RawAxiosHeaders whose duplicates are ignored by node\n// c.f. https://nodejs.org/api/http.html#http_message_headers\nconst ignoreDuplicateOf = utils.toObjectSet([\n  'age', 'authorization', 'content-length', 'content-type', 'etag',\n  'expires', 'from', 'host', 'if-modified-since', 'if-unmodified-since',\n  'last-modified', 'location', 'max-forwards', 'proxy-authorization',\n  'referer', 'retry-after', 'user-agent'\n]);\n\n/**\n * Parse headers into an object\n *\n * ```\n * Date: Wed, 27 Aug 2014 08:58:49 GMT\n * Content-Type: application/json\n * Connection: keep-alive\n * Transfer-Encoding: chunked\n * ```\n *\n * @param {String} rawHeaders Headers needing to be parsed\n *\n * @returns {Object} Headers parsed into an object\n */\nexport default rawHeaders => {\n  const parsed = {};\n  let key;\n  let val;\n  let i;\n\n  rawHeaders && rawHeaders.split('\\n').forEach(function parser(line) {\n    i = line.indexOf(':');\n    key = line.substring(0, i).trim().toLowerCase();\n    val = line.substring(i + 1).trim();\n\n    if (!key || (parsed[key] && ignoreDuplicateOf[key])) {\n      return;\n    }\n\n    if (key === 'set-cookie') {\n      if (parsed[key]) {\n        parsed[key].push(val);\n      } else {\n        parsed[key] = [val];\n      }\n    } else {\n      parsed[key] = parsed[key] ? parsed[key] + ', ' + val : val;\n    }\n  });\n\n  return parsed;\n};\n\n// Path: lib/core/AxiosHeaders.js\n'use strict';\n\nimport utils from '../utils.js';\nimport parseHeaders from '../helpers/parseHeaders.js';\n\nconst $internals = Symbol('internals');\n\nfunction normalizeHeader(header) {\n  return header && String(header).trim().toLowerCase();\n}\n\nfunction normalizeValue(value) {\n  if (value === false || value == null) {\n    return value;\n  }\n\n  return utils.isArray(value) ? value.map(normalizeValue) : String(value);\n}\n\nfunction parseTokens(str) {\n  const tokens = Object.create(null);\n  const tokensRE = /([^\\s,;=]+)\\s*(?:=\\s*([^,;]+))?/g;\n  let match;\n\n  while ((match = tokensRE.exec(str))) {\n    tokens[match[1]] = match[2];\n  }\n\n  return tokens;\n}\n\nconst isValidHeaderName = (str) => /^[-_a-zA-Z0-9^`|~,!#$%&'*+.]+$/.test(str.trim());\n\nfunction matchHeaderValue(context, value, header, filter, isHeaderNameFilter) {\n  if (utils.isFunction(filter)) {\n    return filter.call(this, value, header);\n  }\n\n  if (isHeaderNameFilter) {\n    value = header;\n  }\n\n  if (!utils.isString(value)) return;\n\n  if (utils.isString(filter)) {\n    return value.indexOf(filter) !== -1;\n  }\n\n  if (utils.isRegExp(filter)) {\n    return filter.test(value);\n  }\n}\n\nfunction formatHeader(header) {\n  return header.trim()\n    .toLowerCase().replace(/([a-z\\d])(\\w*)/g, (w, char, str) => {\n      return char.toUpperCase() + str;\n    });\n}\n\nfunction buildAccessors(obj, header) {\n  const accessorName = utils.toCamelCase(' ' + header);\n\n  ['get', 'set', 'has'].forEach(methodName => {\n    Object.defineProperty(obj, methodName + accessorName, {\n      value: function(arg1, arg2, arg3) {\n        return this[methodName].call(this, header, arg1, arg2, arg3);\n      },\n      configurable: true\n    });\n  });\n}\n\nclass AxiosHeaders {\n  constructor(headers) {\n    headers && this.set(headers);\n  }\n\n  set(header, valueOrRewrite, rewrite) {\n    const self = this;\n\n    function setHeader(_value, _header, _rewrite) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!lHeader) {\n        throw new Error('header name must be a non-empty string');\n      }\n\n      const key = utils.findKey(self, lHeader);\n\n      if(!key || self[key] === undefined || _rewrite === true || (_rewrite === undefined && self[key] !== false)) {\n        self[key || _header] = normalizeValue(_value);\n      }\n    }\n\n    const setHeaders = (headers, _rewrite) =>\n      utils.forEach(headers, (_value, _header) => setHeader(_value, _header, _rewrite));\n\n    if (utils.isPlainObject(header) || header instanceof this.constructor) {\n      setHeaders(header, valueOrRewrite)\n    } else if(utils.isString(header) && (header = header.trim()) && !isValidHeaderName(header)) {\n      setHeaders(parseHeaders(header), valueOrRewrite);\n    } else {\n      header != null && setHeader(valueOrRewrite, header, rewrite);\n    }\n\n    return this;\n  }\n\n  get(header, parser) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      if (key) {\n        const value = this[key];\n\n        if (!parser) {\n          return value;\n        }\n\n        if (parser === true) {\n          return parseTokens(value);\n        }\n\n        if (utils.isFunction(parser)) {\n          return parser.call(this, value, key);\n        }\n\n        if (utils.isRegExp(parser)) {\n          return parser.exec(value);\n        }\n\n        throw new TypeError('parser must be boolean|regexp|function');\n      }\n    }\n  }\n\n  has(header, matcher) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      return !!(key && this[key] !== undefined && (!matcher || matchHeaderValue(this, this[key], key, matcher)));\n    }\n\n    return false;\n  }\n\n  delete(header, matcher) {\n    const self = this;\n    let deleted = false;\n\n    function deleteHeader(_header) {\n      _header = normalizeHeader(_header);\n\n      if (_header) {\n        const key = utils.findKey(self, _header);\n\n        if (key && (!matcher || matchHeaderValue(self, self[key], key, matcher))) {\n          delete self[key];\n\n          deleted = true;\n        }\n      }\n    }\n\n    if (utils.isArray(header)) {\n      header.forEach(deleteHeader);\n    } else {\n      deleteHeader(header);\n    }\n\n    return deleted;\n  }\n\n  clear(matcher) {\n    const keys = Object.keys(this);\n    let i = keys.length;\n    let deleted = false;\n\n    while (i--) {\n      const key = keys[i];\n      if(!matcher || matchHeaderValue(this, this[key], key, matcher, true)) {\n        delete this[key];\n        deleted = true;\n      }\n    }\n\n    return deleted;\n  }\n\n  normalize(format) {\n    const self = this;\n    const headers = {};\n\n    utils.forEach(this, (value, header) => {\n      const key = utils.findKey(headers, header);\n\n      if (key) {\n        self[key] = normalizeValue(value);\n        delete self[header];\n        return;\n      }\n\n      const normalized = format ? formatHeader(header) : String(header).trim();\n\n      if (normalized !== header) {\n        delete self[header];\n      }\n\n      self[normalized] = normalizeValue(value);\n\n      headers[normalized] = true;\n    });\n\n    return this;\n  }\n\n  concat(...targets) {\n    return this.constructor.concat(this, ...targets);\n  }\n\n  toJSON(asStrings) {\n    const obj = Object.create(null);\n\n    utils.forEach(this, (value, header) => {\n      value != null && value !== false && (obj[header] = asStrings && utils.isArray(value) ? value.join(', ') : value);\n    });\n\n    return obj;\n  }\n\n  [Symbol.iterator]() {\n    return Object.entries(this.toJSON())[Symbol.iterator]();\n  }\n\n  toString() {\n    return Object.entries(this.toJSON()).map(([header, value]) => header + ': ' + value).join('\\n');\n  }\n\n  get [Symbol.toStringTag]() {\n    return 'AxiosHeaders';\n  }\n\n  static from(thing) {\n    return thing instanceof this ? thing : new this(thing);\n  }\n\n  static concat(first, ...targets) {\n    const computed = new this(first);\n\n    targets.forEach((target) => computed.set(target));\n\n    return computed;\n  }\n\n  static accessor(header) {\n    const internals = this[$internals] = (this[$internals] = {\n      accessors: {}\n    });\n\n    const accessors = internals.accessors;\n    const prototype = this.prototype;\n\n    function defineAccessor(_header) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!accessors[lHeader]) {\n        buildAccessors(prototype, _header);\n        accessors[lHeader] = true;\n      }\n    }\n\n    utils.isArray(header) ? header.forEach(defineAccessor) : defineAccessor(header);\n\n    return this;\n  }\n}\n\nAxiosHeaders.accessor(['Content-Type', 'Content-Length', 'Accept', 'Accept-Encoding', 'User-Agent', 'Authorization']);\n\n// reserved names hotfix\nutils.reduceDescriptors(AxiosHeaders.prototype, ({value}, key) => {\n  let mapped = key[0].toUpperCase() + key.slice(1); // map `set` => `Set`\n  return {\n    get: () => value,\n    set(headerValue) {\n...\n", "needle_token_start": 7111, "needle_token_end": 7248, "code_context_ntokens": 14277, "output": ["```javascript\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\n\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: utils.toJSONObject(this.config),\n      code: this.code,\n      status: this.response && this.response.status ? this.response.status : null\n    };\n  }\n});\n\nconst prototype = AxiosError.prototype;\nconst descriptors = {};\n\n[\n  'ERR_BAD_OPTION_VALUE',\n  'ERR_BAD_OPTION',\n  'ECONNABORTED',\n  'ETIMEDOUT',\n  'ERR_NETWORK',\n  'ERR_FR_TOO_MANY_REDIRECTS',\n  'ERR_DEPRECATED',\n  'ERR_BAD_RESPONSE',\n  'ERR_BAD_REQUEST',\n  'ERR_CANCELED',\n  'ERR_NOT_SUPPORT',\n  'ERR_INVALID_URL'\n// eslint-disable-next-line func-names\n].forEach(code => {\n  descriptors[code] = {value: code};\n});\n\nObject.defineProperties(AxiosError, descriptors);\nObject.defineProperty(prototype, 'isAxiosError', {value: true});\n\n// eslint-disable-next-line func-names\nAxiosError.from = (error, code, config, request, response, customProps) => {\n  const axiosError = Object.create(prototype);\n\n  utils.toFlatObject(error, axiosError, function filter(obj) {\n    return obj !== Error.prototype;\n  }, prop => {\n    return prop !== 'isAxiosError';\n  });\n\n  AxiosError.call(axiosError, error.message, code, config, request, response);\n\n  axiosError.cause = error;\n\n  axiosError.name = error.name;\n\n  customProps && Object.assign(axiosError, customProps);\n\n  return axiosError;\n};\n\nexport default AxiosError;\n```"]}
{"repo": "axios/axios", "name": "formatHeader", "language": "typescript", "path": "lib/core/AxiosHeaders.js", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to reformat a header string by capitalizing the first letter of each word while ensuring the rest of the characters in each word remain in lowercase.\n2. **Input**: A single string representing a header, which may contain multiple words separated by hyphens.\n3. **Output**: A string where the first letter of each word is capitalized, and the remaining letters are in lowercase.\n4. **Procedure**: The function trims whitespace from the input string, converts the entire string to lowercase, and then capitalizes the first letter of each word. This transformation is applied using a regular expression that matches each word in the string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/utils.js\n'use strict';\n\nimport bind from './helpers/bind.js';\n\n// utils is a library of generic helper functions non-specific to axios\n\nconst {toString} = Object.prototype;\nconst {getPrototypeOf} = Object;\n\nconst kindOf = (cache => thing => {\n    const str = toString.call(thing);\n    return cache[str] || (cache[str] = str.slice(8, -1).toLowerCase());\n})(Object.create(null));\n\nconst kindOfTest = (type) => {\n  type = type.toLowerCase();\n  return (thing) => kindOf(thing) === type\n}\n\nconst typeOfTest = type => thing => typeof thing === type;\n\n/**\n * Determine if a value is an Array\n *\n * @param {Object} val The value to test\n *\n * @returns {boolean} True if value is an Array, otherwise false\n */\nconst {isArray} = Array;\n\n/**\n * Determine if a value is undefined\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if the value is undefined, otherwise false\n */\nconst isUndefined = typeOfTest('undefined');\n\n/**\n * Determine if a value is a Buffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Buffer, otherwise false\n */\nfunction isBuffer(val) {\n  return val !== null && !isUndefined(val) && val.constructor !== null && !isUndefined(val.constructor)\n    && isFunction(val.constructor.isBuffer) && val.constructor.isBuffer(val);\n}\n\n/**\n * Determine if a value is an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is an ArrayBuffer, otherwise false\n */\nconst isArrayBuffer = kindOfTest('ArrayBuffer');\n\n\n/**\n * Determine if a value is a view on an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a view on an ArrayBuffer, otherwise false\n */\nfunction isArrayBufferView(val) {\n  let result;\n  if ((typeof ArrayBuffer !== 'undefined') && (ArrayBuffer.isView)) {\n    result = ArrayBuffer.isView(val);\n  } else {\n    result = (val) && (val.buffer) && (isArrayBuffer(val.buffer));\n  }\n  return result;\n}\n\n/**\n * Determine if a value is a String\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a String, otherwise false\n */\nconst isString = typeOfTest('string');\n\n/**\n * Determine if a value is a Function\n *\n * @param {*} val The value to test\n * @returns {boolean} True if value is a Function, otherwise false\n */\nconst isFunction = typeOfTest('function');\n\n/**\n * Determine if a value is a Number\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Number, otherwise false\n */\nconst isNumber = typeOfTest('number');\n\n/**\n * Determine if a value is an Object\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an Object, otherwise false\n */\nconst isObject = (thing) => thing !== null && typeof thing === 'object';\n\n/**\n * Determine if a value is a Boolean\n *\n * @param {*} thing The value to test\n * @returns {boolean} True if value is a Boolean, otherwise false\n */\nconst isBoolean = thing => thing === true || thing === false;\n\n/**\n * Determine if a value is a plain Object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a plain Object, otherwise false\n */\nconst isPlainObject = (val) => {\n  if (kindOf(val) !== 'object') {\n    return false;\n  }\n\n  const prototype = getPrototypeOf(val);\n  return (prototype === null || prototype === Object.prototype || Object.getPrototypeOf(prototype) === null) && !(Symbol.toStringTag in val) && !(Symbol.iterator in val);\n}\n\n/**\n * Determine if a value is a Date\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Date, otherwise false\n */\nconst isDate = kindOfTest('Date');\n\n/**\n * Determine if a value is a File\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFile = kindOfTest('File');\n\n/**\n * Determine if a value is a Blob\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Blob, otherwise false\n */\nconst isBlob = kindOfTest('Blob');\n\n/**\n * Determine if a value is a FileList\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFileList = kindOfTest('FileList');\n\n/**\n * Determine if a value is a Stream\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Stream, otherwise false\n */\nconst isStream = (val) => isObject(val) && isFunction(val.pipe);\n\n/**\n * Determine if a value is a FormData\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an FormData, otherwise false\n */\nconst isFormData = (thing) => {\n  let kind;\n  return thing && (\n    (typeof FormData === 'function' && thing instanceof FormData) || (\n      isFunction(thing.append) && (\n        (kind = kindOf(thing)) === 'formdata' ||\n        // detect form-data instance\n        (kind === 'object' && isFunction(thing.toString) && thing.toString() === '[object FormData]')\n      )\n    )\n  )\n}\n\n/**\n * Determine if a value is a URLSearchParams object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a URLSearchParams object, otherwise false\n */\nconst isURLSearchParams = kindOfTest('URLSearchParams');\n\n/**\n * Trim excess whitespace off the beginning and end of a string\n *\n * @param {String} str The String to trim\n *\n * @returns {String} The String freed of excess whitespace\n */\nconst trim = (str) => str.trim ?\n  str.trim() : str.replace(/^[\\s\\uFEFF\\xA0]+|[\\s\\uFEFF\\xA0]+$/g, '');\n\n/**\n * Iterate over an Array or an Object invoking a function for each item.\n *\n * If `obj` is an Array callback will be called passing\n * the value, index, and complete array for each item.\n *\n * If 'obj' is an Object callback will be called passing\n * the value, key, and complete object for each property.\n *\n * @param {Object|Array} obj The object to iterate\n * @param {Function} fn The callback to invoke for each item\n *\n * @param {Boolean} [allOwnKeys = false]\n * @returns {any}\n */\nfunction forEach(obj, fn, {allOwnKeys = false} = {}) {\n  // Don't bother if no value provided\n  if (obj === null || typeof obj === 'undefined') {\n    return;\n  }\n\n  let i;\n  let l;\n\n  // Force an array if not already something iterable\n  if (typeof obj !== 'object') {\n    /*eslint no-param-reassign:0*/\n    obj = [obj];\n  }\n\n  if (isArray(obj)) {\n    // Iterate over array values\n    for (i = 0, l = obj.length; i < l; i++) {\n      fn.call(null, obj[i], i, obj);\n    }\n  } else {\n    // Iterate over object keys\n    const keys = allOwnKeys ? Object.getOwnPropertyNames(obj) : Object.keys(obj);\n    const len = keys.length;\n    let key;\n\n    for (i = 0; i < len; i++) {\n      key = keys[i];\n      fn.call(null, obj[key], key, obj);\n    }\n  }\n}\n\nfunction findKey(obj, key) {\n  key = key.toLowerCase();\n  const keys = Object.keys(obj);\n  let i = keys.length;\n  let _key;\n  while (i-- > 0) {\n    _key = keys[i];\n    if (key === _key.toLowerCase()) {\n      return _key;\n    }\n  }\n  return null;\n}\n\nconst _global = (() => {\n  /*eslint no-undef:0*/\n  if (typeof globalThis !== \"undefined\") return globalThis;\n  return typeof self !== \"undefined\" ? self : (typeof window !== 'undefined' ? window : global)\n})();\n\nconst isContextDefined = (context) => !isUndefined(context) && context !== _global;\n\n/**\n * Accepts varargs expecting each argument to be an object, then\n * immutably merges the properties of each object and returns result.\n *\n * When multiple objects contain the same key the later object in\n * the arguments list will take precedence.\n *\n * Example:\n *\n * ```js\n * var result = merge({foo: 123}, {foo: 456});\n * console.log(result.foo); // outputs 456\n * ```\n *\n * @param {Object} obj1 Object to merge\n *\n * @returns {Object} Result of all merge properties\n */\nfunction merge(/* obj1, obj2, obj3, ... */) {\n  const {caseless} = isContextDefined(this) && this || {};\n  const result = {};\n  const assignValue = (val, key) => {\n    const targetKey = caseless && findKey(result, key) || key;\n    if (isPlainObject(result[targetKey]) && isPlainObject(val)) {\n      result[targetKey] = merge(result[targetKey], val);\n    } else if (isPlainObject(val)) {\n      result[targetKey] = merge({}, val);\n    } else if (isArray(val)) {\n      result[targetKey] = val.slice();\n    } else {\n      result[targetKey] = val;\n    }\n  }\n\n  for (let i = 0, l = arguments.length; i < l; i++) {\n    arguments[i] && forEach(arguments[i], assignValue);\n  }\n  return result;\n}\n\n/**\n * Extends object a by mutably adding to it the properties of object b.\n *\n * @param {Object} a The object to be extended\n * @param {Object} b The object to copy properties from\n * @param {Object} thisArg The object to bind function to\n *\n * @param {Boolean} [allOwnKeys]\n * @returns {Object} The resulting value of object a\n */\nconst extend = (a, b, thisArg, {allOwnKeys}= {}) => {\n  forEach(b, (val, key) => {\n    if (thisArg && isFunction(val)) {\n      a[key] = bind(val, thisArg);\n    } else {\n      a[key] = val;\n    }\n  }, {allOwnKeys});\n  return a;\n}\n\n/**\n * Remove byte order marker. This catches EF BB BF (the UTF-8 BOM)\n *\n * @param {string} content with BOM\n *\n * @returns {string} content value without BOM\n */\nconst stripBOM = (content) => {\n  if (content.charCodeAt(0) === 0xFEFF) {\n    content = content.slice(1);\n  }\n  return content;\n}\n\n/**\n * Inherit the prototype methods from one constructor into another\n * @param {function} constructor\n * @param {function} superConstructor\n * @param {object} [props]\n * @param {object} [descriptors]\n *\n * @returns {void}\n */\nconst inherits = (constructor, superConstructor, props, descriptors) => {\n  constructor.prototype = Object.create(superConstructor.prototype, descriptors);\n  constructor.prototype.constructor = constructor;\n  Object.defineProperty(constructor, 'super', {\n    value: superConstructor.prototype\n  });\n  props && Object.assign(constructor.prototype, props);\n}\n\n/**\n * Resolve object with deep prototype chain to a flat object\n * @param {Object} sourceObj source object\n * @param {Object} [destObj]\n * @param {Function|Boolean} [filter]\n * @param {Function} [propFilter]\n *\n * @returns {Object}\n */\nconst toFlatObject = (sourceObj, destObj, filter, propFilter) => {\n  let props;\n  let i;\n  let prop;\n  const merged = {};\n\n  destObj = destObj || {};\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  if (sourceObj == null) return destObj;\n\n  do {\n    props = Object.getOwnPropertyNames(sourceObj);\n    i = props.length;\n    while (i-- > 0) {\n      prop = props[i];\n      if ((!propFilter || propFilter(prop, sourceObj, destObj)) && !merged[prop]) {\n        destObj[prop] = sourceObj[prop];\n        merged[prop] = true;\n      }\n    }\n    sourceObj = filter !== false && getPrototypeOf(sourceObj);\n  } while (sourceObj && (!filter || filter(sourceObj, destObj)) && sourceObj !== Object.prototype);\n\n  return destObj;\n}\n\n/**\n * Determines whether a string ends with the characters of a specified string\n *\n * @param {String} str\n * @param {String} searchString\n * @param {Number} [position= 0]\n *\n * @returns {boolean}\n */\nconst endsWith = (str, searchString, position) => {\n  str = String(str);\n  if (position === undefined || position > str.length) {\n    position = str.length;\n  }\n  position -= searchString.length;\n  const lastIndex = str.indexOf(searchString, position);\n  return lastIndex !== -1 && lastIndex === position;\n}\n\n\n/**\n * Returns new array from array like object or null if failed\n *\n * @param {*} [thing]\n *\n * @returns {?Array}\n */\nconst toArray = (thing) => {\n  if (!thing) return null;\n  if (isArray(thing)) return thing;\n  let i = thing.length;\n  if (!isNumber(i)) return null;\n  const arr = new Array(i);\n  while (i-- > 0) {\n    arr[i] = thing[i];\n  }\n  return arr;\n}\n\n/**\n * Checking if the Uint8Array exists and if it does, it returns a function that checks if the\n * thing passed in is an instance of Uint8Array\n *\n * @param {TypedArray}\n *\n * @returns {Array}\n */\n// eslint-disable-next-line func-names\nconst isTypedArray = (TypedArray => {\n  // eslint-disable-next-line func-names\n  return thing => {\n    return TypedArray && thing instanceof TypedArray;\n  };\n})(typeof Uint8Array !== 'undefined' && getPrototypeOf(Uint8Array));\n\n/**\n * For each entry in the object, call the function with the key and value.\n *\n * @param {Object<any, any>} obj - The object to iterate over.\n * @param {Function} fn - The function to call for each entry.\n *\n * @returns {void}\n */\nconst forEachEntry = (obj, fn) => {\n  const generator = obj && obj[Symbol.iterator];\n\n  const iterator = generator.call(obj);\n\n  let result;\n\n  while ((result = iterator.next()) && !result.done) {\n    const pair = result.value;\n    fn.call(obj, pair[0], pair[1]);\n  }\n}\n\n/**\n * It takes a regular expression and a string, and returns an array of all the matches\n *\n * @param {string} regExp - The regular expression to match against.\n * @param {string} str - The string to search.\n *\n * @returns {Array<boolean>}\n */\nconst matchAll = (regExp, str) => {\n  let matches;\n  const arr = [];\n\n  while ((matches = regExp.exec(str)) !== null) {\n    arr.push(matches);\n  }\n\n  return arr;\n}\n\n/* Checking if the kindOfTest function returns true when passed an HTMLFormElement. */\nconst isHTMLForm = kindOfTest('HTMLFormElement');\n\nconst toCamelCase = str => {\n  return str.toLowerCase().replace(/[-_\\s]([a-z\\d])(\\w*)/g,\n    function replacer(m, p1, p2) {\n      return p1.toUpperCase() + p2;\n    }\n  );\n};\n\n/* Creating a function that will check if an object has a property. */\nconst hasOwnProperty = (({hasOwnProperty}) => (obj, prop) => hasOwnProperty.call(obj, prop))(Object.prototype);\n\n/**\n * Determine if a value is a RegExp object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a RegExp object, otherwise false\n */\nconst isRegExp = kindOfTest('RegExp');\n\nconst reduceDescriptors = (obj, reducer) => {\n  const descriptors = Object.getOwnPropertyDescriptors(obj);\n  const reducedDescriptors = {};\n\n  forEach(descriptors, (descriptor, name) => {\n    let ret;\n    if ((ret = reducer(descriptor, name, obj)) !== false) {\n      reducedDescriptors[name] = ret || descriptor;\n    }\n  });\n\n  Object.defineProperties(obj, reducedDescriptors);\n}\n\n/**\n * Makes all methods read-only\n * @param {Object} obj\n */\n\nconst freezeMethods = (obj) => {\n  reduceDescriptors(obj, (descriptor, name) => {\n    // skip restricted props in strict mode\n...\n// Path: lib/core/AxiosError.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: utils.toJSONObject(this.config),\n      code: this.code,\n      status: this.response && this.response.status ? this.response.status : null\n    };\n  }\n});\n\nconst prototype = AxiosError.prototype;\nconst descriptors = {};\n\n[\n  'ERR_BAD_OPTION_VALUE',\n  'ERR_BAD_OPTION',\n  'ECONNABORTED',\n  'ETIMEDOUT',\n  'ERR_NETWORK',\n  'ERR_FR_TOO_MANY_REDIRECTS',\n  'ERR_DEPRECATED',\n  'ERR_BAD_RESPONSE',\n  'ERR_BAD_REQUEST',\n  'ERR_CANCELED',\n  'ERR_NOT_SUPPORT',\n  'ERR_INVALID_URL'\n// eslint-disable-next-line func-names\n].forEach(code => {\n  descriptors[code] = {value: code};\n});\n\nObject.defineProperties(AxiosError, descriptors);\nObject.defineProperty(prototype, 'isAxiosError', {value: true});\n\n// eslint-disable-next-line func-names\nAxiosError.from = (error, code, config, request, response, customProps) => {\n  const axiosError = Object.create(prototype);\n\n  utils.toFlatObject(error, axiosError, function filter(obj) {\n    return obj !== Error.prototype;\n  }, prop => {\n    return prop !== 'isAxiosError';\n  });\n\n  AxiosError.call(axiosError, error.message, code, config, request, response);\n\n  axiosError.cause = error;\n\n  axiosError.name = error.name;\n\n  customProps && Object.assign(axiosError, customProps);\n\n  return axiosError;\n};\n\nexport default AxiosError;\n\n// Path: lib/helpers/toFormData.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\n// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored\nimport PlatformFormData from '../platform/node/classes/FormData.js';\n\n/**\n * Determines if the given thing is a array or js object.\n *\n * @param {string} thing - The object or array to be visited.\n *\n * @returns {boolean}\n */\nfunction isVisitable(thing) {\n  return utils.isPlainObject(thing) || utils.isArray(thing);\n}\n\n/**\n * It removes the brackets from the end of a string\n *\n * @param {string} key - The key of the parameter.\n *\n * @returns {string} the key without the brackets.\n */\nfunction removeBrackets(key) {\n  return utils.endsWith(key, '[]') ? key.slice(0, -2) : key;\n}\n\n/**\n * It takes a path, a key, and a boolean, and returns a string\n *\n * @param {string} path - The path to the current key.\n * @param {string} key - The key of the current object being iterated over.\n * @param {string} dots - If true, the key will be rendered with dots instead of brackets.\n *\n * @returns {string} The path to the current key.\n */\nfunction renderKey(path, key, dots) {\n  if (!path) return key;\n  return path.concat(key).map(function each(token, i) {\n    // eslint-disable-next-line no-param-reassign\n    token = removeBrackets(token);\n    return !dots && i ? '[' + token + ']' : token;\n  }).join(dots ? '.' : '');\n}\n\n/**\n * If the array is an array and none of its elements are visitable, then it's a flat array.\n *\n * @param {Array<any>} arr - The array to check\n *\n * @returns {boolean}\n */\nfunction isFlatArray(arr) {\n  return utils.isArray(arr) && !arr.some(isVisitable);\n}\n\nconst predicates = utils.toFlatObject(utils, {}, null, function filter(prop) {\n  return /^is[A-Z]/.test(prop);\n});\n\n/**\n * Convert a data object to FormData\n *\n * @param {Object} obj\n * @param {?Object} [formData]\n * @param {?Object} [options]\n * @param {Function} [options.visitor]\n * @param {Boolean} [options.metaTokens = true]\n * @param {Boolean} [options.dots = false]\n * @param {?Boolean} [options.indexes = false]\n *\n * @returns {Object}\n **/\n\n/**\n * It converts an object into a FormData object\n *\n * @param {Object<any, any>} obj - The object to convert to form data.\n * @param {string} formData - The FormData object to append to.\n * @param {Object<string, any>} options\n *\n * @returns\n */\nfunction toFormData(obj, formData, options) {\n  if (!utils.isObject(obj)) {\n    throw new TypeError('target must be an object');\n  }\n\n  // eslint-disable-next-line no-param-reassign\n  formData = formData || new (PlatformFormData || FormData)();\n\n  // eslint-disable-next-line no-param-reassign\n  options = utils.toFlatObject(options, {\n    metaTokens: true,\n    dots: false,\n    indexes: false\n  }, false, function defined(option, source) {\n    // eslint-disable-next-line no-eq-null,eqeqeq\n    return !utils.isUndefined(source[option]);\n  });\n\n  const metaTokens = options.metaTokens;\n  // eslint-disable-next-line no-use-before-define\n  const visitor = options.visitor || defaultVisitor;\n  const dots = options.dots;\n  const indexes = options.indexes;\n  const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob;\n  const useBlob = _Blob && utils.isSpecCompliantForm(formData);\n\n  if (!utils.isFunction(visitor)) {\n    throw new TypeError('visitor must be a function');\n  }\n\n  function convertValue(value) {\n    if (value === null) return '';\n\n    if (utils.isDate(value)) {\n      return value.toISOString();\n    }\n\n    if (!useBlob && utils.isBlob(value)) {\n      throw new AxiosError('Blob is not supported. Use a Buffer instead.');\n    }\n\n    if (utils.isArrayBuffer(value) || utils.isTypedArray(value)) {\n      return useBlob && typeof Blob === 'function' ? new Blob([value]) : Buffer.from(value);\n    }\n\n    return value;\n  }\n\n  /**\n   * Default visitor.\n   *\n   * @param {*} value\n   * @param {String|Number} key\n   * @param {Array<String|Number>} path\n   * @this {FormData}\n   *\n   * @returns {boolean} return true to visit the each prop of the value recursively\n   */\n  function defaultVisitor(value, key, path) {\n    let arr = value;\n\n    if (value && !path && typeof value === 'object') {\n      if (utils.endsWith(key, '{}')) {\n        // eslint-disable-next-line no-param-reassign\n        key = metaTokens ? key : key.slice(0, -2);\n        // eslint-disable-next-line no-param-reassign\n        value = JSON.stringify(value);\n      } else if (\n        (utils.isArray(value) && isFlatArray(value)) ||\n        ((utils.isFileList(value) || utils.endsWith(key, '[]')) && (arr = utils.toArray(value))\n        )) {\n        // eslint-disable-next-line no-param-reassign\n        key = removeBrackets(key);\n\n        arr.forEach(function each(el, index) {\n          !(utils.isUndefined(el) || el === null) && formData.append(\n            // eslint-disable-next-line no-nested-ternary\n            indexes === true ? renderKey([key], index, dots) : (indexes === null ? key : key + '[]'),\n            convertValue(el)\n          );\n        });\n        return false;\n      }\n    }\n\n    if (isVisitable(value)) {\n      return true;\n    }\n\n    formData.append(renderKey(path, key, dots), convertValue(value));\n\n    return false;\n  }\n\n  const stack = [];\n\n  const exposedHelpers = Object.assign(predicates, {\n    defaultVisitor,\n    convertValue,\n    isVisitable\n  });\n\n  function build(value, path) {\n    if (utils.isUndefined(value)) return;\n\n    if (stack.indexOf(value) !== -1) {\n      throw Error('Circular reference detected in ' + path.join('.'));\n    }\n\n    stack.push(value);\n\n    utils.forEach(value, function each(el, key) {\n      const result = !(utils.isUndefined(el) || el === null) && visitor.call(\n        formData, el, utils.isString(key) ? key.trim() : key, path, exposedHelpers\n      );\n\n      if (result === true) {\n        build(el, path ? path.concat(key) : [key]);\n      }\n    });\n\n    stack.pop();\n  }\n\n  if (!utils.isObject(obj)) {\n    throw new TypeError('data must be an object');\n  }\n\n  build(obj);\n\n  return formData;\n}\n\nexport default toFormData;\n\n// Path: lib/helpers/AxiosURLSearchParams.js\n'use strict';\n\nimport toFormData from './toFormData.js';\n\n/**\n * It encodes a string by replacing all characters that are not in the unreserved set with\n * their percent-encoded equivalents\n *\n * @param {string} str - The string to encode.\n *\n * @returns {string} The encoded string.\n */\nfunction encode(str) {\n  const charMap = {\n    '!': '%21',\n    \"'\": '%27',\n    '(': '%28',\n    ')': '%29',\n    '~': '%7E',\n    '%20': '+',\n    '%00': '\\x00'\n  };\n  return encodeURIComponent(str).replace(/[!'()~]|%20|%00/g, function replacer(match) {\n    return charMap[match];\n  });\n}\n\n/**\n * It takes a params object and converts it to a FormData object\n *\n * @param {Object<string, any>} params - The parameters to be converted to a FormData object.\n * @param {Object<string, any>} options - The options object passed to the Axios constructor.\n *\n * @returns {void}\n */\nfunction AxiosURLSearchParams(params, options) {\n  this._pairs = [];\n\n  params && toFormData(params, this, options);\n}\n\nconst prototype = AxiosURLSearchParams.prototype;\n\nprototype.append = function append(name, value) {\n  this._pairs.push([name, value]);\n};\n\nprototype.toString = function toString(encoder) {\n  const _encode = encoder ? function(value) {\n    return encoder.call(this, value, encode);\n  } : encode;\n\n  return this._pairs.map(function each(pair) {\n    return _encode(pair[0]) + '=' + _encode(pair[1]);\n  }, '').join('&');\n};\n\nexport default AxiosURLSearchParams;\n\n// Path: lib/helpers/buildURL.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosURLSearchParams from '../helpers/AxiosURLSearchParams.js';\n\n/**\n * It replaces all instances of the characters `:`, `$`, `,`, `+`, `[`, and `]` with their\n * URI encoded counterparts\n *\n * @param {string} val The value to be encoded.\n *\n * @returns {string} The encoded value.\n */\nfunction encode(val) {\n  return encodeURIComponent(val).\n    replace(/%3A/gi, ':').\n    replace(/%24/g, '$').\n    replace(/%2C/gi, ',').\n    replace(/%20/g, '+').\n    replace(/%5B/gi, '[').\n    replace(/%5D/gi, ']');\n}\n\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default function buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n\n// Path: lib/cancel/CanceledError.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n\n/**\n * A `CanceledError` is an object that is thrown when an operation is canceled.\n *\n * @param {string=} message The message.\n * @param {Object=} config The config.\n * @param {Object=} request The request.\n *\n * @returns {CanceledError} The created error.\n */\nfunction CanceledError(message, config, request) {\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  AxiosError.call(this, message == null ? 'canceled' : message, AxiosError.ERR_CANCELED, config, request);\n  this.name = 'CanceledError';\n}\n\nutils.inherits(CanceledError, AxiosError, {\n  __CANCEL__: true\n});\n\nexport default CanceledError;\n\n// Path: lib/defaults/transitional.js\n'use strict';\n\nexport default {\n  silentJSONParsing: true,\n  forcedJSONParsing: true,\n  clarifyTimeoutError: false\n};\n\n// Path: lib/helpers/speedometer.js\n'use strict';\n\n/**\n * Calculate data maxRate\n * @param {Number} [samplesCount= 10]\n * @param {Number} [min= 1000]\n * @returns {Function}\n */\nfunction speedometer(samplesCount, min) {\n  samplesCount = samplesCount || 10;\n  const bytes = new Array(samplesCount);\n  const timestamps = new Array(samplesCount);\n  let head = 0;\n  let tail = 0;\n  let firstSampleTS;\n\n  min = min !== undefined ? min : 1000;\n\n  return function push(chunkLength) {\n    const now = Date.now();\n\n    const startedAt = timestamps[tail];\n\n    if (!firstSampleTS) {\n      firstSampleTS = now;\n    }\n\n    bytes[head] = chunkLength;\n    timestamps[head] = now;\n\n    let i = tail;\n    let bytesCount = 0;\n\n    while (i !== head) {\n      bytesCount += bytes[i++];\n      i = i % samplesCount;\n    }\n\n    head = (head + 1) % samplesCount;\n\n    if (head === tail) {\n      tail = (tail + 1) % samplesCount;\n    }\n\n    if (now - firstSampleTS < min) {\n      return;\n    }\n\n    const passed = startedAt && now - startedAt;\n\n    return passed ? Math.round(bytesCount * 1000 / passed) : undefined;\n  };\n}\n\nexport default speedometer;\n\n// Path: lib/helpers/parseHeaders.js\n'use strict';\n\nimport utils from './../utils.js';\n\n// RawAxiosHeaders whose duplicates are ignored by node\n// c.f. https://nodejs.org/api/http.html#http_message_headers\nconst ignoreDuplicateOf = utils.toObjectSet([\n  'age', 'authorization', 'content-length', 'content-type', 'etag',\n  'expires', 'from', 'host', 'if-modified-since', 'if-unmodified-since',\n  'last-modified', 'location', 'max-forwards', 'proxy-authorization',\n  'referer', 'retry-after', 'user-agent'\n]);\n\n/**\n * Parse headers into an object\n *\n * ```\n * Date: Wed, 27 Aug 2014 08:58:49 GMT\n * Content-Type: application/json\n * Connection: keep-alive\n * Transfer-Encoding: chunked\n * ```\n *\n * @param {String} rawHeaders Headers needing to be parsed\n *\n * @returns {Object} Headers parsed into an object\n */\nexport default rawHeaders => {\n  const parsed = {};\n  let key;\n  let val;\n  let i;\n\n  rawHeaders && rawHeaders.split('\\n').forEach(function parser(line) {\n    i = line.indexOf(':');\n    key = line.substring(0, i).trim().toLowerCase();\n    val = line.substring(i + 1).trim();\n\n    if (!key || (parsed[key] && ignoreDuplicateOf[key])) {\n      return;\n    }\n\n    if (key === 'set-cookie') {\n      if (parsed[key]) {\n        parsed[key].push(val);\n      } else {\n        parsed[key] = [val];\n      }\n    } else {\n      parsed[key] = parsed[key] ? parsed[key] + ', ' + val : val;\n    }\n  });\n\n  return parsed;\n};\n\n// Path: lib/core/AxiosHeaders.js\n'use strict';\n\nimport utils from '../utils.js';\nimport parseHeaders from '../helpers/parseHeaders.js';\n\nconst $internals = Symbol('internals');\n\nfunction normalizeHeader(header) {\n  return header && String(header).trim().toLowerCase();\n}\n\nfunction normalizeValue(value) {\n  if (value === false || value == null) {\n    return value;\n  }\n\n  return utils.isArray(value) ? value.map(normalizeValue) : String(value);\n}\n\nfunction parseTokens(str) {\n  const tokens = Object.create(null);\n  const tokensRE = /([^\\s,;=]+)\\s*(?:=\\s*([^,;]+))?/g;\n  let match;\n\n  while ((match = tokensRE.exec(str))) {\n    tokens[match[1]] = match[2];\n  }\n\n  return tokens;\n}\n\nconst isValidHeaderName = (str) => /^[-_a-zA-Z0-9^`|~,!#$%&'*+.]+$/.test(str.trim());\n\nfunction matchHeaderValue(context, value, header, filter, isHeaderNameFilter) {\n  if (utils.isFunction(filter)) {\n    return filter.call(this, value, header);\n  }\n\n  if (isHeaderNameFilter) {\n    value = header;\n  }\n\n  if (!utils.isString(value)) return;\n\n  if (utils.isString(filter)) {\n    return value.indexOf(filter) !== -1;\n  }\n\n  if (utils.isRegExp(filter)) {\n    return filter.test(value);\n  }\n}\n\n\nfunction formatHeader(header) {\n  return header.trim()\n    .toLowerCase().replace(/([a-z\\d])(\\w*)/g, (w, char, str) => {\n      return char.toUpperCase() + str;\n    });\n}\n\nfunction buildAccessors(obj, header) {\n  const accessorName = utils.toCamelCase(' ' + header);\n\n  ['get', 'set', 'has'].forEach(methodName => {\n    Object.defineProperty(obj, methodName + accessorName, {\n      value: function(arg1, arg2, arg3) {\n        return this[methodName].call(this, header, arg1, arg2, arg3);\n      },\n      configurable: true\n    });\n  });\n}\n\nclass AxiosHeaders {\n  constructor(headers) {\n    headers && this.set(headers);\n  }\n\n  set(header, valueOrRewrite, rewrite) {\n    const self = this;\n\n    function setHeader(_value, _header, _rewrite) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!lHeader) {\n        throw new Error('header name must be a non-empty string');\n      }\n\n      const key = utils.findKey(self, lHeader);\n\n      if(!key || self[key] === undefined || _rewrite === true || (_rewrite === undefined && self[key] !== false)) {\n        self[key || _header] = normalizeValue(_value);\n      }\n    }\n\n    const setHeaders = (headers, _rewrite) =>\n      utils.forEach(headers, (_value, _header) => setHeader(_value, _header, _rewrite));\n\n    if (utils.isPlainObject(header) || header instanceof this.constructor) {\n      setHeaders(header, valueOrRewrite)\n    } else if(utils.isString(header) && (header = header.trim()) && !isValidHeaderName(header)) {\n      setHeaders(parseHeaders(header), valueOrRewrite);\n    } else {\n      header != null && setHeader(valueOrRewrite, header, rewrite);\n    }\n\n    return this;\n  }\n\n  get(header, parser) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      if (key) {\n        const value = this[key];\n\n        if (!parser) {\n          return value;\n        }\n\n        if (parser === true) {\n          return parseTokens(value);\n        }\n\n        if (utils.isFunction(parser)) {\n          return parser.call(this, value, key);\n        }\n\n        if (utils.isRegExp(parser)) {\n          return parser.exec(value);\n        }\n\n        throw new TypeError('parser must be boolean|regexp|function');\n      }\n    }\n  }\n\n  has(header, matcher) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      return !!(key && this[key] !== undefined && (!matcher || matchHeaderValue(this, this[key], key, matcher)));\n    }\n\n    return false;\n  }\n\n  delete(header, matcher) {\n    const self = this;\n    let deleted = false;\n\n    function deleteHeader(_header) {\n      _header = normalizeHeader(_header);\n\n      if (_header) {\n        const key = utils.findKey(self, _header);\n\n        if (key && (!matcher || matchHeaderValue(self, self[key], key, matcher))) {\n          delete self[key];\n\n          deleted = true;\n        }\n      }\n    }\n\n    if (utils.isArray(header)) {\n      header.forEach(deleteHeader);\n    } else {\n      deleteHeader(header);\n    }\n\n    return deleted;\n  }\n\n  clear(matcher) {\n    const keys = Object.keys(this);\n    let i = keys.length;\n    let deleted = false;\n\n    while (i--) {\n      const key = keys[i];\n      if(!matcher || matchHeaderValue(this, this[key], key, matcher, true)) {\n        delete this[key];\n        deleted = true;\n      }\n    }\n\n    return deleted;\n  }\n\n  normalize(format) {\n    const self = this;\n    const headers = {};\n\n    utils.forEach(this, (value, header) => {\n      const key = utils.findKey(headers, header);\n\n      if (key) {\n        self[key] = normalizeValue(value);\n        delete self[header];\n        return;\n      }\n\n      const normalized = format ? formatHeader(header) : String(header).trim();\n\n      if (normalized !== header) {\n        delete self[header];\n      }\n\n      self[normalized] = normalizeValue(value);\n\n      headers[normalized] = true;\n    });\n\n    return this;\n  }\n\n  concat(...targets) {\n    return this.constructor.concat(this, ...targets);\n  }\n\n  toJSON(asStrings) {\n    const obj = Object.create(null);\n\n    utils.forEach(this, (value, header) => {\n      value != null && value !== false && (obj[header] = asStrings && utils.isArray(value) ? value.join(', ') : value);\n    });\n\n    return obj;\n  }\n\n  [Symbol.iterator]() {\n    return Object.entries(this.toJSON())[Symbol.iterator]();\n  }\n\n  toString() {\n    return Object.entries(this.toJSON()).map(([header, value]) => header + ': ' + value).join('\\n');\n  }\n\n  get [Symbol.toStringTag]() {\n    return 'AxiosHeaders';\n  }\n\n  static from(thing) {\n    return thing instanceof this ? thing : new this(thing);\n  }\n\n  static concat(first, ...targets) {\n    const computed = new this(first);\n\n    targets.forEach((target) => computed.set(target));\n\n    return computed;\n  }\n\n  static accessor(header) {\n    const internals = this[$internals] = (this[$internals] = {\n      accessors: {}\n    });\n\n    const accessors = internals.accessors;\n    const prototype = this.prototype;\n\n    function defineAccessor(_header) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!accessors[lHeader]) {\n        buildAccessors(prototype, _header);\n        accessors[lHeader] = true;\n      }\n    }\n\n    utils.isArray(header) ? header.forEach(defineAccessor) : defineAccessor(header);\n\n    return this;\n  }\n}\n\nAxiosHeaders.accessor(['Content-Type', 'Content-Length', 'Accept', 'Accept-Encoding', 'User-Agent', 'Authorization']);\n\n// reserved names hotfix\nutils.reduceDescriptors(AxiosHeaders.prototype, ({value}, key) => {\n  let mapped = key[0].toUpperCase() + key.slice(1); // map `set` => `Set`\n  return {\n    get: () => value,\n    set(headerValue) {\n      this[mapped] = headerValue;\n    }\n  }\n});\n\nutils.freezeMethods(AxiosHeaders);\n\nexport default AxiosHeaders;\n\n// Path: lib/core/settle.js\n'use strict';\n\nimport AxiosError from './AxiosError.js';\n\n/**\n * Resolve or reject a Promise based on response status.\n *\n * @param {Function} resolve A function that resolves the promise.\n * @param {Function} reject A function that rejects the promise.\n * @param {object} response The response.\n *\n * @returns {object} The response.\n */\nexport default function settle(resolve, reject, response) {\n  const validateStatus = response.config.validateStatus;\n  if (!response.status || !validateStatus || validateStatus(response.status)) {\n    resolve(response);\n  } else {\n    reject(new AxiosError(\n      'Request failed with status code ' + response.status,\n      [AxiosError.ERR_BAD_REQUEST, AxiosError.ERR_BAD_RESPONSE][Math.floor(response.status / 100) - 4],\n      response.config,\n      response.request,\n      response\n    ));\n  }\n}\n\n// Path: lib/helpers/parseProtocol.js\n'use strict';\n\nexport default function parseProtocol(url) {\n  const match = /^([-+\\w]{1,25})(:?\\/\\/|:)/.exec(url);\n  return match && match[1] || '';\n}\n\n// Path: lib/helpers/cookies.js\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n  // Standard browser envs support document.cookie\n  {\n    write(name, value, expires, path, domain, secure) {\n      const cookie = [name + '=' + encodeURIComponent(value)];\n\n      utils.isNumber(expires) && cookie.push('expires=' + new Date(expires).toGMTString());\n\n      utils.isString(path) && cookie.push('path=' + path);\n\n      utils.isString(domain) && cookie.push('domain=' + domain);\n\n      secure === true && cookie.push('secure');\n\n      document.cookie = cookie.join('; ');\n    },\n\n    read(name) {\n      const match = document.cookie.match(new RegExp('(^|;\\\\s*)(' + name + ')=([^;]*)'));\n      return (match ? decodeURIComponent(match[3]) : null);\n    },\n\n    remove(name) {\n      this.write(name, '', Date.now() - 86400000);\n    }\n  }\n\n  :\n\n  // Non-standard browser env (web workers, react-native) lack needed support.\n  {\n    write() {},\n    read() {\n      return null;\n    },\n    remove() {}\n  };\n\n\n// Path: lib/helpers/isURLSameOrigin.js\n'use strict';\n\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n// Standard browser envs have full support of the APIs needed to test\n// whether the request URL is of the same origin as current location.\n  (function standardBrowserEnv() {\n    const msie = /(msie|trident)/i.test(navigator.userAgent);\n    const urlParsingNode = document.createElement('a');\n    let originURL;\n\n    /**\n    * Parse a URL to discover its components\n    *\n    * @param {String} url The URL to be parsed\n    * @returns {Object}\n    */\n    function resolveURL(url) {\n      let href = url;\n\n      if (msie) {\n        // IE needs attribute set twice to normalize properties\n        urlParsingNode.setAttribute('href', href);\n        href = urlParsingNode.href;\n      }\n\n      urlParsingNode.setAttribute('href', href);\n\n      // urlParsingNode provides the UrlUtils interface - http://url.spec.whatwg.org/#urlutils\n      return {\n        href: urlParsingNode.href,\n        protocol: urlParsingNode.protocol ? urlParsingNode.protocol.replace(/:$/, '') : '',\n        host: urlParsingNode.host,\n        search: urlParsingNode.search ? urlParsingNode.search.replace(/^\\?/, '') : '',\n        hash: urlParsingNode.hash ? urlParsingNode.hash.replace(/^#/, '') : '',\n        hostname: urlParsingNode.hostname,\n        port: urlParsingNode.port,\n        pathname: (urlParsingNode.pathname.charAt(0) === '/') ?\n          urlParsingNode.pathname :\n          '/' + urlParsingNode.pathname\n      };\n    }\n\n    originURL = resolveURL(window.location.href);\n\n    /**\n    * Determine if a URL shares the same origin as the current location\n    *\n    * @param {String} requestURL The URL to test\n    * @returns {boolean} True if URL shares the same origin, otherwise false\n    */\n    return function isURLSameOrigin(requestURL) {\n      const parsed = (utils.isString(requestURL)) ? resolveURL(requestURL) : requestURL;\n      return (parsed.protocol === originURL.protocol &&\n          parsed.host === originURL.host);\n    };\n  })() :\n\n  // Non standard browser envs (web workers, react-native) lack needed support.\n  (function nonStandardBrowserEnv() {\n    return function isURLSameOrigin() {\n      return true;\n    };\n  })();\n\n// Path: lib/adapters/xhr.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport cookies from './../helpers/cookies.js';\nimport buildURL from './../helpers/buildURL.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport isURLSameOrigin from './../helpers/isURLSameOrigin.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport parseProtocol from '../helpers/parseProtocol.js';\nimport platform from '../platform/index.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport speedometer from '../helpers/speedometer.js';\n\nfunction progressEventReducer(listener, isDownloadStream) {\n  let bytesNotified = 0;\n  const _speedometer = speedometer(50, 250);\n\n  return e => {\n    const loaded = e.loaded;\n    const total = e.lengthComputable ? e.total : undefined;\n    const progressBytes = loaded - bytesNotified;\n    const rate = _speedometer(progressBytes);\n    const inRange = loaded <= total;\n\n    bytesNotified = loaded;\n\n    const data = {\n      loaded,\n      total,\n      progress: total ? (loaded / total) : undefined,\n      bytes: progressBytes,\n      rate: rate ? rate : undefined,\n      estimated: rate && total && inRange ? (total - loaded) / rate : undefined,\n      event: e\n    };\n\n    data[isDownloadStream ? 'download' : 'upload'] = true;\n\n    listener(data);\n  };\n}\n\nconst isXHRAdapterSupported = typeof XMLHttpRequest !== 'undefined';\n\nexport default isXHRAdapterSupported && function (config) {\n  return new Promise(function dispatchXhrRequest(resolve, reject) {\n    let requestData = config.data;\n    const requestHeaders = AxiosHeaders.from(config.headers).normalize();\n    let {responseType, withXSRFToken} = config;\n    let onCanceled;\n    function done() {\n      if (config.cancelToken) {\n        config.cancelToken.unsubscribe(onCanceled);\n      }\n\n      if (config.signal) {\n        config.signal.removeEventListener('abort', onCanceled);\n      }\n    }\n\n    let contentType;\n\n    if (utils.isFormData(requestData)) {\n      if (platform.hasStandardBrowserEnv || platform.hasStandardBrowserWebWorkerEnv) {\n        requestHeaders.setContentType(false); // Let the browser set it\n      } else if ((contentType = requestHeaders.getContentType()) !== false) {\n        // fix semicolon duplication issue for ReactNative FormData implementation\n        const [type, ...tokens] = contentType ? contentType.split(';').map(token => token.trim()).filter(Boolean) : [];\n        requestHeaders.setContentType([type || 'multipart/form-data', ...tokens].join('; '));\n      }\n    }\n\n    let request = new XMLHttpRequest();\n\n    // HTTP basic authentication\n    if (config.auth) {\n      const username = config.auth.username || '';\n      const password = config.auth.password ? unescape(encodeURIComponent(config.auth.password)) : '';\n      requestHeaders.set('Authorization', 'Basic ' + btoa(username + ':' + password));\n    }\n\n    const fullPath = buildFullPath(config.baseURL, config.url);\n\n    request.open(config.method.toUpperCase(), buildURL(fullPath, config.params, config.paramsSerializer), true);\n\n    // Set the request timeout in MS\n    request.timeout = config.timeout;\n\n    function onloadend() {\n      if (!request) {\n        return;\n      }\n      // Prepare the response\n      const responseHeaders = AxiosHeaders.from(\n        'getAllResponseHeaders' in request && request.getAllResponseHeaders()\n      );\n      const responseData = !responseType || responseType === 'text' || responseType === 'json' ?\n        request.responseText : request.response;\n      const response = {\n        data: responseData,\n        status: request.status,\n        statusText: request.statusText,\n        headers: responseHeaders,\n        config,\n        request\n      };\n\n      settle(function _resolve(value) {\n        resolve(value);\n        done();\n      }, function _reject(err) {\n        reject(err);\n        done();\n      }, response);\n\n      // Clean up request\n      request = null;\n    }\n\n    if ('onloadend' in request) {\n      // Use onloadend if available\n      request.onloadend = onloadend;\n    } else {\n      // Listen for ready state to emulate onloadend\n      request.onreadystatechange = function handleLoad() {\n        if (!request || request.readyState !== 4) {\n          return;\n        }\n\n        // The request errored out and we didn't get a response, this will be\n        // handled by onerror instead\n        // With one exception: request that using file: protocol, most browsers\n        // will return status as 0 even though it's a successful request\n        if (request.status === 0 && !(request.responseURL && request.responseURL.indexOf('file:') === 0)) {\n          return;\n        }\n        // readystate handler is calling before onerror or ontimeout handlers,\n        // so we should call onloadend on the next 'tick'\n        setTimeout(onloadend);\n      };\n    }\n\n    // Handle browser request cancellation (as opposed to a manual cancellation)\n    request.onabort = function handleAbort() {\n      if (!request) {\n        return;\n      }\n\n      reject(new AxiosError('Request aborted', AxiosError.ECONNABORTED, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle low level network errors\n    request.onerror = function handleError() {\n      // Real errors are hidden from us by the browser\n      // onerror should only fire if it's a network error\n      reject(new AxiosError('Network Error', AxiosError.ERR_NETWORK, config, request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Handle timeout\n    request.ontimeout = function handleTimeout() {\n      let timeoutErrorMessage = config.timeout ? 'timeout of ' + config.timeout + 'ms exceeded' : 'timeout exceeded';\n      const transitional = config.transitional || transitionalDefaults;\n      if (config.timeoutErrorMessage) {\n        timeoutErrorMessage = config.timeoutErrorMessage;\n      }\n      reject(new AxiosError(\n        timeoutErrorMessage,\n        transitional.clarifyTimeoutError ? AxiosError.ETIMEDOUT : AxiosError.ECONNABORTED,\n        config,\n        request));\n\n      // Clean up request\n      request = null;\n    };\n\n    // Add xsrf header\n    // This is only done if running in a standard browser environment.\n    // Specifically not if we're in a web worker, or react-native.\n    if(platform.hasStandardBrowserEnv) {\n      withXSRFToken && utils.isFunction(withXSRFToken) && (withXSRFToken = withXSRFToken(config));\n\n      if (withXSRFToken || (withXSRFToken !== false && isURLSameOrigin(fullPath))) {\n        // Add xsrf header\n        const xsrfValue = config.xsrfHeaderName && config.xsrfCookieName && cookies.read(config.xsrfCookieName);\n\n        if (xsrfValue) {\n          requestHeaders.set(config.xsrfHeaderName, xsrfValue);\n        }\n      }\n    }\n\n    // Remove Content-Type if data is undefined\n    requestData === undefined && requestHeaders.setContentType(null);\n\n    // Add headers to the request\n    if ('setRequestHeader' in request) {\n      utils.forEach(requestHeaders.toJSON(), function setRequestHeader(val, key) {\n        request.setRequestHeader(key, val);\n      });\n    }\n\n    // Add withCredentials to request if needed\n    if (!utils.isUndefined(config.withCredentials)) {\n      request.withCredentials = !!config.withCredentials;\n    }\n\n    // Add responseType to request if needed\n    if (responseType && responseType !== 'json') {\n      request.responseType = config.responseType;\n    }\n\n    // Handle progress if needed\n    if (typeof config.onDownloadProgress === 'function') {\n      request.addEventListener('progress', progressEventReducer(config.onDownloadProgress, true));\n    }\n\n    // Not all browsers support upload events\n    if (typeof config.onUploadProgress === 'function' && request.upload) {\n      request.upload.addEventListener('progress', progressEventReducer(config.onUploadProgress));\n    }\n\n    if (config.cancelToken || config.signal) {\n      // Handle cancellation\n      // eslint-disable-next-line func-names\n      onCanceled = cancel => {\n        if (!request) {\n          return;\n        }\n        reject(!cancel || cancel.type ? new CanceledError(null, config, request) : cancel);\n        request.abort();\n        request = null;\n      };\n\n      config.cancelToken && config.cancelToken.subscribe(onCanceled);\n      if (config.signal) {\n        config.signal.aborted ? onCanceled() : config.signal.addEventListener('abort', onCanceled);\n      }\n    }\n\n    const protocol = parseProtocol(fullPath);\n\n    if (protocol && platform.protocols.indexOf(protocol) === -1) {\n...\n", "needle_token_start": 10097, "needle_token_end": 10159, "code_context_ntokens": 15711, "output": ["```javascript\nconst toCamelCase = str => {\n  return str.toLowerCase().replace(/[-_\\s]([a-z\\d])(\\w*)/g,\n    function replacer(m, p1, p2) {\n      return p1.toUpperCase() + p2;\n    }\n  );\n};\n```"]}
{"repo": "axios/axios", "name": "CanceledError", "language": "typescript", "path": "lib/cancel/CanceledError.js", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The purpose of this function is to create an error object specifically for operations that have been canceled, providing a standardized error handling mechanism.\n2. **Input**: The function accepts three optional parameters: a message string to describe the error, a configuration object related to the context of the error, and a request object that was involved when the error occurred.\n3. **Output**: The function returns an error object that includes properties such as a custom message, error type identifier, and references to the configuration and request involved.\n4. **Procedure**: The function first checks if a message is provided; if not, it defaults to 'canceled'. It then invokes a parent error constructor with the message and specific error type identifier, along with the configuration and request objects. This integrates the new error object into a broader error handling system.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/platform/node/classes/FormData.js\nimport FormData from 'form-data';\n\nexport default FormData;\n\n// Path: lib/platform/node/classes/URLSearchParams.js\n'use strict';\n\nimport url from 'url';\nexport default url.URLSearchParams;\n\n// Path: lib/platform/node/index.js\nimport URLSearchParams from './classes/URLSearchParams.js'\nimport FormData from './classes/FormData.js'\n\nexport default {\n  isNode: true,\n  classes: {\n    URLSearchParams,\n    FormData,\n    Blob: typeof Blob !== 'undefined' && Blob || null\n  },\n  protocols: [ 'http', 'https', 'file', 'data' ]\n};\n\n// Path: lib/platform/common/utils.js\nconst hasBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';\n\n/**\n * Determine if we're running in a standard browser environment\n *\n * This allows axios to run in a web worker, and react-native.\n * Both environments support XMLHttpRequest, but not fully standard globals.\n *\n * web workers:\n *  typeof window -> undefined\n *  typeof document -> undefined\n *\n * react-native:\n *  navigator.product -> 'ReactNative'\n * nativescript\n *  navigator.product -> 'NativeScript' or 'NS'\n *\n * @returns {boolean}\n */\nconst hasStandardBrowserEnv = (\n  (product) => {\n    return hasBrowserEnv && ['ReactNative', 'NativeScript', 'NS'].indexOf(product) < 0\n  })(typeof navigator !== 'undefined' && navigator.product);\n\n/**\n * Determine if we're running in a standard browser webWorker environment\n *\n * Although the `isStandardBrowserEnv` method indicates that\n * `allows axios to run in a web worker`, the WebWorker will still be\n * filtered out due to its judgment standard\n * `typeof window !== 'undefined' && typeof document !== 'undefined'`.\n * This leads to a problem when axios post `FormData` in webWorker\n */\nconst hasStandardBrowserWebWorkerEnv = (() => {\n  return (\n    typeof WorkerGlobalScope !== 'undefined' &&\n    // eslint-disable-next-line no-undef\n    self instanceof WorkerGlobalScope &&\n    typeof self.importScripts === 'function'\n  );\n})();\n\nexport {\n  hasBrowserEnv,\n  hasStandardBrowserWebWorkerEnv,\n  hasStandardBrowserEnv\n}\n\n// Path: lib/platform/index.js\nimport platform from './node/index.js';\nimport * as utils from './common/utils.js';\n\nexport default {\n  ...utils,\n  ...platform\n}\n\n// Path: lib/helpers/combineURLs.js\n'use strict';\n\n/**\n * Creates a new URL by combining the specified URLs\n *\n * @param {string} baseURL The base URL\n * @param {string} relativeURL The relative URL\n *\n * @returns {string} The combined URL\n */\nexport default function combineURLs(baseURL, relativeURL) {\n  return relativeURL\n    ? baseURL.replace(/\\/?\\/$/, '') + '/' + relativeURL.replace(/^\\/+/, '')\n    : baseURL;\n}\n\n// Path: lib/helpers/isAbsoluteURL.js\n'use strict';\n\n/**\n * Determines whether the specified URL is absolute\n *\n * @param {string} url The URL to test\n *\n * @returns {boolean} True if the specified URL is absolute, otherwise false\n */\nexport default function isAbsoluteURL(url) {\n  // A URL is considered absolute if it begins with \"<scheme>://\" or \"//\" (protocol-relative URL).\n  // RFC 3986 defines scheme name as a sequence of characters beginning with a letter and followed\n  // by any combination of letters, digits, plus, period, or hyphen.\n  return /^([a-z][a-z\\d+\\-.]*:)?\\/\\//i.test(url);\n}\n\n// Path: lib/core/buildFullPath.js\n'use strict';\n\nimport isAbsoluteURL from '../helpers/isAbsoluteURL.js';\nimport combineURLs from '../helpers/combineURLs.js';\n\n/**\n * Creates a new URL by combining the baseURL with the requestedURL,\n * only when the requestedURL is not already an absolute URL.\n * If the requestURL is absolute, this function returns the requestedURL untouched.\n *\n * @param {string} baseURL The base URL\n * @param {string} requestedURL Absolute or relative URL to combine\n *\n * @returns {string} The combined full path\n */\nexport default function buildFullPath(baseURL, requestedURL) {\n  if (baseURL && !isAbsoluteURL(requestedURL)) {\n    return combineURLs(baseURL, requestedURL);\n  }\n  return requestedURL;\n}\n\n// Path: lib/helpers/bind.js\n'use strict';\n\nexport default function bind(fn, thisArg) {\n  return function wrap() {\n    return fn.apply(thisArg, arguments);\n  };\n}\n\n// Path: lib/utils.js\n'use strict';\n\nimport bind from './helpers/bind.js';\n\n// utils is a library of generic helper functions non-specific to axios\n\nconst {toString} = Object.prototype;\nconst {getPrototypeOf} = Object;\n\nconst kindOf = (cache => thing => {\n    const str = toString.call(thing);\n    return cache[str] || (cache[str] = str.slice(8, -1).toLowerCase());\n})(Object.create(null));\n\nconst kindOfTest = (type) => {\n  type = type.toLowerCase();\n  return (thing) => kindOf(thing) === type\n}\n\nconst typeOfTest = type => thing => typeof thing === type;\n\n/**\n * Determine if a value is an Array\n *\n * @param {Object} val The value to test\n *\n * @returns {boolean} True if value is an Array, otherwise false\n */\nconst {isArray} = Array;\n\n/**\n * Determine if a value is undefined\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if the value is undefined, otherwise false\n */\nconst isUndefined = typeOfTest('undefined');\n\n/**\n * Determine if a value is a Buffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Buffer, otherwise false\n */\nfunction isBuffer(val) {\n  return val !== null && !isUndefined(val) && val.constructor !== null && !isUndefined(val.constructor)\n    && isFunction(val.constructor.isBuffer) && val.constructor.isBuffer(val);\n}\n\n/**\n * Determine if a value is an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is an ArrayBuffer, otherwise false\n */\nconst isArrayBuffer = kindOfTest('ArrayBuffer');\n\n\n/**\n * Determine if a value is a view on an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a view on an ArrayBuffer, otherwise false\n */\nfunction isArrayBufferView(val) {\n  let result;\n  if ((typeof ArrayBuffer !== 'undefined') && (ArrayBuffer.isView)) {\n    result = ArrayBuffer.isView(val);\n  } else {\n    result = (val) && (val.buffer) && (isArrayBuffer(val.buffer));\n  }\n  return result;\n}\n\n/**\n * Determine if a value is a String\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a String, otherwise false\n */\nconst isString = typeOfTest('string');\n\n/**\n * Determine if a value is a Function\n *\n * @param {*} val The value to test\n * @returns {boolean} True if value is a Function, otherwise false\n */\nconst isFunction = typeOfTest('function');\n\n/**\n * Determine if a value is a Number\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Number, otherwise false\n */\nconst isNumber = typeOfTest('number');\n\n/**\n * Determine if a value is an Object\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an Object, otherwise false\n */\nconst isObject = (thing) => thing !== null && typeof thing === 'object';\n\n/**\n * Determine if a value is a Boolean\n *\n * @param {*} thing The value to test\n * @returns {boolean} True if value is a Boolean, otherwise false\n */\nconst isBoolean = thing => thing === true || thing === false;\n\n/**\n * Determine if a value is a plain Object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a plain Object, otherwise false\n */\nconst isPlainObject = (val) => {\n  if (kindOf(val) !== 'object') {\n    return false;\n  }\n\n  const prototype = getPrototypeOf(val);\n  return (prototype === null || prototype === Object.prototype || Object.getPrototypeOf(prototype) === null) && !(Symbol.toStringTag in val) && !(Symbol.iterator in val);\n}\n\n/**\n * Determine if a value is a Date\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Date, otherwise false\n */\nconst isDate = kindOfTest('Date');\n\n/**\n * Determine if a value is a File\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFile = kindOfTest('File');\n\n/**\n * Determine if a value is a Blob\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Blob, otherwise false\n */\nconst isBlob = kindOfTest('Blob');\n\n/**\n * Determine if a value is a FileList\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFileList = kindOfTest('FileList');\n\n/**\n * Determine if a value is a Stream\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Stream, otherwise false\n */\nconst isStream = (val) => isObject(val) && isFunction(val.pipe);\n\n/**\n * Determine if a value is a FormData\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an FormData, otherwise false\n */\nconst isFormData = (thing) => {\n  let kind;\n  return thing && (\n    (typeof FormData === 'function' && thing instanceof FormData) || (\n      isFunction(thing.append) && (\n        (kind = kindOf(thing)) === 'formdata' ||\n        // detect form-data instance\n        (kind === 'object' && isFunction(thing.toString) && thing.toString() === '[object FormData]')\n      )\n    )\n  )\n}\n\n/**\n * Determine if a value is a URLSearchParams object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a URLSearchParams object, otherwise false\n */\nconst isURLSearchParams = kindOfTest('URLSearchParams');\n\n/**\n * Trim excess whitespace off the beginning and end of a string\n *\n * @param {String} str The String to trim\n *\n * @returns {String} The String freed of excess whitespace\n */\nconst trim = (str) => str.trim ?\n  str.trim() : str.replace(/^[\\s\\uFEFF\\xA0]+|[\\s\\uFEFF\\xA0]+$/g, '');\n\n/**\n * Iterate over an Array or an Object invoking a function for each item.\n *\n * If `obj` is an Array callback will be called passing\n * the value, index, and complete array for each item.\n *\n * If 'obj' is an Object callback will be called passing\n * the value, key, and complete object for each property.\n *\n * @param {Object|Array} obj The object to iterate\n * @param {Function} fn The callback to invoke for each item\n *\n * @param {Boolean} [allOwnKeys = false]\n * @returns {any}\n */\nfunction forEach(obj, fn, {allOwnKeys = false} = {}) {\n  // Don't bother if no value provided\n  if (obj === null || typeof obj === 'undefined') {\n    return;\n  }\n\n  let i;\n  let l;\n\n  // Force an array if not already something iterable\n  if (typeof obj !== 'object') {\n    /*eslint no-param-reassign:0*/\n    obj = [obj];\n  }\n\n  if (isArray(obj)) {\n    // Iterate over array values\n    for (i = 0, l = obj.length; i < l; i++) {\n      fn.call(null, obj[i], i, obj);\n    }\n  } else {\n    // Iterate over object keys\n    const keys = allOwnKeys ? Object.getOwnPropertyNames(obj) : Object.keys(obj);\n    const len = keys.length;\n    let key;\n\n    for (i = 0; i < len; i++) {\n      key = keys[i];\n      fn.call(null, obj[key], key, obj);\n    }\n  }\n}\n\nfunction findKey(obj, key) {\n  key = key.toLowerCase();\n  const keys = Object.keys(obj);\n  let i = keys.length;\n  let _key;\n  while (i-- > 0) {\n    _key = keys[i];\n    if (key === _key.toLowerCase()) {\n      return _key;\n    }\n  }\n  return null;\n}\n\nconst _global = (() => {\n  /*eslint no-undef:0*/\n  if (typeof globalThis !== \"undefined\") return globalThis;\n  return typeof self !== \"undefined\" ? self : (typeof window !== 'undefined' ? window : global)\n})();\n\nconst isContextDefined = (context) => !isUndefined(context) && context !== _global;\n\n/**\n * Accepts varargs expecting each argument to be an object, then\n * immutably merges the properties of each object and returns result.\n *\n * When multiple objects contain the same key the later object in\n * the arguments list will take precedence.\n *\n * Example:\n *\n * ```js\n * var result = merge({foo: 123}, {foo: 456});\n * console.log(result.foo); // outputs 456\n * ```\n *\n * @param {Object} obj1 Object to merge\n *\n * @returns {Object} Result of all merge properties\n */\nfunction merge(/* obj1, obj2, obj3, ... */) {\n  const {caseless} = isContextDefined(this) && this || {};\n  const result = {};\n  const assignValue = (val, key) => {\n    const targetKey = caseless && findKey(result, key) || key;\n    if (isPlainObject(result[targetKey]) && isPlainObject(val)) {\n      result[targetKey] = merge(result[targetKey], val);\n    } else if (isPlainObject(val)) {\n      result[targetKey] = merge({}, val);\n    } else if (isArray(val)) {\n      result[targetKey] = val.slice();\n    } else {\n      result[targetKey] = val;\n    }\n  }\n\n  for (let i = 0, l = arguments.length; i < l; i++) {\n    arguments[i] && forEach(arguments[i], assignValue);\n  }\n  return result;\n}\n\n/**\n * Extends object a by mutably adding to it the properties of object b.\n *\n * @param {Object} a The object to be extended\n * @param {Object} b The object to copy properties from\n * @param {Object} thisArg The object to bind function to\n *\n * @param {Boolean} [allOwnKeys]\n * @returns {Object} The resulting value of object a\n */\nconst extend = (a, b, thisArg, {allOwnKeys}= {}) => {\n  forEach(b, (val, key) => {\n    if (thisArg && isFunction(val)) {\n      a[key] = bind(val, thisArg);\n    } else {\n      a[key] = val;\n    }\n  }, {allOwnKeys});\n  return a;\n}\n\n/**\n * Remove byte order marker. This catches EF BB BF (the UTF-8 BOM)\n *\n * @param {string} content with BOM\n *\n * @returns {string} content value without BOM\n */\nconst stripBOM = (content) => {\n  if (content.charCodeAt(0) === 0xFEFF) {\n    content = content.slice(1);\n  }\n  return content;\n}\n\n/**\n * Inherit the prototype methods from one constructor into another\n * @param {function} constructor\n * @param {function} superConstructor\n * @param {object} [props]\n * @param {object} [descriptors]\n *\n * @returns {void}\n */\nconst inherits = (constructor, superConstructor, props, descriptors) => {\n  constructor.prototype = Object.create(superConstructor.prototype, descriptors);\n  constructor.prototype.constructor = constructor;\n  Object.defineProperty(constructor, 'super', {\n    value: superConstructor.prototype\n  });\n  props && Object.assign(constructor.prototype, props);\n}\n\n/**\n * Resolve object with deep prototype chain to a flat object\n * @param {Object} sourceObj source object\n * @param {Object} [destObj]\n * @param {Function|Boolean} [filter]\n * @param {Function} [propFilter]\n *\n * @returns {Object}\n */\nconst toFlatObject = (sourceObj, destObj, filter, propFilter) => {\n  let props;\n  let i;\n  let prop;\n  const merged = {};\n\n  destObj = destObj || {};\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  if (sourceObj == null) return destObj;\n\n  do {\n    props = Object.getOwnPropertyNames(sourceObj);\n    i = props.length;\n    while (i-- > 0) {\n      prop = props[i];\n      if ((!propFilter || propFilter(prop, sourceObj, destObj)) && !merged[prop]) {\n        destObj[prop] = sourceObj[prop];\n        merged[prop] = true;\n      }\n    }\n    sourceObj = filter !== false && getPrototypeOf(sourceObj);\n  } while (sourceObj && (!filter || filter(sourceObj, destObj)) && sourceObj !== Object.prototype);\n\n  return destObj;\n}\n\n/**\n * Determines whether a string ends with the characters of a specified string\n *\n * @param {String} str\n * @param {String} searchString\n * @param {Number} [position= 0]\n *\n * @returns {boolean}\n */\nconst endsWith = (str, searchString, position) => {\n  str = String(str);\n  if (position === undefined || position > str.length) {\n    position = str.length;\n  }\n  position -= searchString.length;\n  const lastIndex = str.indexOf(searchString, position);\n  return lastIndex !== -1 && lastIndex === position;\n}\n\n\n/**\n * Returns new array from array like object or null if failed\n *\n * @param {*} [thing]\n *\n * @returns {?Array}\n */\nconst toArray = (thing) => {\n  if (!thing) return null;\n  if (isArray(thing)) return thing;\n  let i = thing.length;\n  if (!isNumber(i)) return null;\n  const arr = new Array(i);\n  while (i-- > 0) {\n    arr[i] = thing[i];\n  }\n  return arr;\n}\n\n/**\n * Checking if the Uint8Array exists and if it does, it returns a function that checks if the\n * thing passed in is an instance of Uint8Array\n *\n * @param {TypedArray}\n *\n * @returns {Array}\n */\n// eslint-disable-next-line func-names\nconst isTypedArray = (TypedArray => {\n  // eslint-disable-next-line func-names\n  return thing => {\n    return TypedArray && thing instanceof TypedArray;\n  };\n})(typeof Uint8Array !== 'undefined' && getPrototypeOf(Uint8Array));\n\n/**\n * For each entry in the object, call the function with the key and value.\n *\n * @param {Object<any, any>} obj - The object to iterate over.\n * @param {Function} fn - The function to call for each entry.\n *\n * @returns {void}\n */\nconst forEachEntry = (obj, fn) => {\n  const generator = obj && obj[Symbol.iterator];\n\n  const iterator = generator.call(obj);\n\n  let result;\n\n  while ((result = iterator.next()) && !result.done) {\n    const pair = result.value;\n    fn.call(obj, pair[0], pair[1]);\n  }\n}\n\n/**\n * It takes a regular expression and a string, and returns an array of all the matches\n *\n * @param {string} regExp - The regular expression to match against.\n * @param {string} str - The string to search.\n *\n * @returns {Array<boolean>}\n */\nconst matchAll = (regExp, str) => {\n  let matches;\n  const arr = [];\n\n  while ((matches = regExp.exec(str)) !== null) {\n    arr.push(matches);\n  }\n\n  return arr;\n}\n\n/* Checking if the kindOfTest function returns true when passed an HTMLFormElement. */\nconst isHTMLForm = kindOfTest('HTMLFormElement');\n\nconst toCamelCase = str => {\n  return str.toLowerCase().replace(/[-_\\s]([a-z\\d])(\\w*)/g,\n    function replacer(m, p1, p2) {\n      return p1.toUpperCase() + p2;\n    }\n  );\n};\n\n/* Creating a function that will check if an object has a property. */\nconst hasOwnProperty = (({hasOwnProperty}) => (obj, prop) => hasOwnProperty.call(obj, prop))(Object.prototype);\n\n/**\n * Determine if a value is a RegExp object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a RegExp object, otherwise false\n */\nconst isRegExp = kindOfTest('RegExp');\n\nconst reduceDescriptors = (obj, reducer) => {\n  const descriptors = Object.getOwnPropertyDescriptors(obj);\n  const reducedDescriptors = {};\n\n  forEach(descriptors, (descriptor, name) => {\n    let ret;\n    if ((ret = reducer(descriptor, name, obj)) !== false) {\n      reducedDescriptors[name] = ret || descriptor;\n    }\n  });\n\n  Object.defineProperties(obj, reducedDescriptors);\n}\n\n/**\n * Makes all methods read-only\n * @param {Object} obj\n */\n\nconst freezeMethods = (obj) => {\n  reduceDescriptors(obj, (descriptor, name) => {\n    // skip restricted props in strict mode\n    if (isFunction(obj) && ['arguments', 'caller', 'callee'].indexOf(name) !== -1) {\n      return false;\n    }\n\n    const value = obj[name];\n\n    if (!isFunction(value)) return;\n\n    descriptor.enumerable = false;\n\n    if ('writable' in descriptor) {\n      descriptor.writable = false;\n      return;\n    }\n\n    if (!descriptor.set) {\n      descriptor.set = () => {\n        throw Error('Can not rewrite read-only method \\'' + name + '\\'');\n      };\n    }\n  });\n}\n\nconst toObjectSet = (arrayOrString, delimiter) => {\n  const obj = {};\n\n  const define = (arr) => {\n    arr.forEach(value => {\n      obj[value] = true;\n    });\n  }\n\n  isArray(arrayOrString) ? define(arrayOrString) : define(String(arrayOrString).split(delimiter));\n\n  return obj;\n}\n\nconst noop = () => {}\n\nconst toFiniteNumber = (value, defaultValue) => {\n  value = +value;\n  return Number.isFinite(value) ? value : defaultValue;\n}\n\nconst ALPHA = 'abcdefghijklmnopqrstuvwxyz'\n\nconst DIGIT = '0123456789';\n\nconst ALPHABET = {\n  DIGIT,\n  ALPHA,\n  ALPHA_DIGIT: ALPHA + ALPHA.toUpperCase() + DIGIT\n}\n\nconst generateString = (size = 16, alphabet = ALPHABET.ALPHA_DIGIT) => {\n  let str = '';\n  const {length} = alphabet;\n  while (size--) {\n    str += alphabet[Math.random() * length|0]\n  }\n\n  return str;\n}\n\n/**\n * If the thing is a FormData object, return true, otherwise return false.\n *\n * @param {unknown} thing - The thing to check.\n *\n * @returns {boolean}\n */\nfunction isSpecCompliantForm(thing) {\n  return !!(thing && isFunction(thing.append) && thing[Symbol.toStringTag] === 'FormData' && thing[Symbol.iterator]);\n}\n\nconst toJSONObject = (obj) => {\n  const stack = new Array(10);\n\n  const visit = (source, i) => {\n\n    if (isObject(source)) {\n      if (stack.indexOf(source) >= 0) {\n        return;\n      }\n\n      if(!('toJSON' in source)) {\n        stack[i] = source;\n        const target = isArray(source) ? [] : {};\n\n        forEach(source, (value, key) => {\n          const reducedValue = visit(value, i + 1);\n          !isUndefined(reducedValue) && (target[key] = reducedValue);\n        });\n\n        stack[i] = undefined;\n\n        return target;\n      }\n    }\n\n    return source;\n  }\n\n  return visit(obj, 0);\n}\n\nconst isAsyncFn = kindOfTest('AsyncFunction');\n\nconst isThenable = (thing) =>\n  thing && (isObject(thing) || isFunction(thing)) && isFunction(thing.then) && isFunction(thing.catch);\n\nexport default {\n  isArray,\n  isArrayBuffer,\n  isBuffer,\n  isFormData,\n  isArrayBufferView,\n  isString,\n  isNumber,\n  isBoolean,\n  isObject,\n  isPlainObject,\n  isUndefined,\n  isDate,\n  isFile,\n  isBlob,\n  isRegExp,\n  isFunction,\n  isStream,\n  isURLSearchParams,\n  isTypedArray,\n  isFileList,\n  forEach,\n  merge,\n  extend,\n  trim,\n  stripBOM,\n  inherits,\n  toFlatObject,\n  kindOf,\n  kindOfTest,\n  endsWith,\n  toArray,\n  forEachEntry,\n  matchAll,\n  isHTMLForm,\n  hasOwnProperty,\n  hasOwnProp: hasOwnProperty, // an alias to avoid ESLint no-prototype-builtins detection\n  reduceDescriptors,\n  freezeMethods,\n  toObjectSet,\n  toCamelCase,\n  noop,\n  toFiniteNumber,\n  findKey,\n  global: _global,\n  isContextDefined,\n  ALPHABET,\n  generateString,\n  isSpecCompliantForm,\n  toJSONObject,\n  isAsyncFn,\n  isThenable\n};\n\n// Path: lib/core/AxiosError.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: utils.toJSONObject(this.config),\n      code: this.code,\n      status: this.response && this.response.status ? this.response.status : null\n    };\n  }\n});\n\nconst prototype = AxiosError.prototype;\nconst descriptors = {};\n\n[\n  'ERR_BAD_OPTION_VALUE',\n  'ERR_BAD_OPTION',\n  'ECONNABORTED',\n  'ETIMEDOUT',\n  'ERR_NETWORK',\n  'ERR_FR_TOO_MANY_REDIRECTS',\n  'ERR_DEPRECATED',\n  'ERR_BAD_RESPONSE',\n  'ERR_BAD_REQUEST',\n  'ERR_CANCELED',\n  'ERR_NOT_SUPPORT',\n  'ERR_INVALID_URL'\n// eslint-disable-next-line func-names\n].forEach(code => {\n  descriptors[code] = {value: code};\n});\n\nObject.defineProperties(AxiosError, descriptors);\nObject.defineProperty(prototype, 'isAxiosError', {value: true});\n\n// eslint-disable-next-line func-names\nAxiosError.from = (error, code, config, request, response, customProps) => {\n  const axiosError = Object.create(prototype);\n\n  utils.toFlatObject(error, axiosError, function filter(obj) {\n    return obj !== Error.prototype;\n  }, prop => {\n    return prop !== 'isAxiosError';\n  });\n\n  AxiosError.call(axiosError, error.message, code, config, request, response);\n\n  axiosError.cause = error;\n\n  axiosError.name = error.name;\n\n  customProps && Object.assign(axiosError, customProps);\n\n  return axiosError;\n};\n\nexport default AxiosError;\n\n// Path: lib/helpers/toFormData.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\n// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored\nimport PlatformFormData from '../platform/node/classes/FormData.js';\n\n/**\n * Determines if the given thing is a array or js object.\n *\n * @param {string} thing - The object or array to be visited.\n *\n * @returns {boolean}\n */\nfunction isVisitable(thing) {\n  return utils.isPlainObject(thing) || utils.isArray(thing);\n}\n\n/**\n * It removes the brackets from the end of a string\n *\n * @param {string} key - The key of the parameter.\n *\n * @returns {string} the key without the brackets.\n */\nfunction removeBrackets(key) {\n  return utils.endsWith(key, '[]') ? key.slice(0, -2) : key;\n}\n\n/**\n * It takes a path, a key, and a boolean, and returns a string\n *\n * @param {string} path - The path to the current key.\n * @param {string} key - The key of the current object being iterated over.\n * @param {string} dots - If true, the key will be rendered with dots instead of brackets.\n *\n * @returns {string} The path to the current key.\n */\nfunction renderKey(path, key, dots) {\n  if (!path) return key;\n  return path.concat(key).map(function each(token, i) {\n    // eslint-disable-next-line no-param-reassign\n    token = removeBrackets(token);\n    return !dots && i ? '[' + token + ']' : token;\n  }).join(dots ? '.' : '');\n}\n\n/**\n * If the array is an array and none of its elements are visitable, then it's a flat array.\n *\n * @param {Array<any>} arr - The array to check\n *\n * @returns {boolean}\n */\nfunction isFlatArray(arr) {\n  return utils.isArray(arr) && !arr.some(isVisitable);\n}\n\nconst predicates = utils.toFlatObject(utils, {}, null, function filter(prop) {\n  return /^is[A-Z]/.test(prop);\n});\n\n/**\n * Convert a data object to FormData\n *\n * @param {Object} obj\n * @param {?Object} [formData]\n * @param {?Object} [options]\n * @param {Function} [options.visitor]\n * @param {Boolean} [options.metaTokens = true]\n * @param {Boolean} [options.dots = false]\n * @param {?Boolean} [options.indexes = false]\n *\n * @returns {Object}\n **/\n\n/**\n * It converts an object into a FormData object\n *\n * @param {Object<any, any>} obj - The object to convert to form data.\n * @param {string} formData - The FormData object to append to.\n * @param {Object<string, any>} options\n *\n * @returns\n */\nfunction toFormData(obj, formData, options) {\n  if (!utils.isObject(obj)) {\n    throw new TypeError('target must be an object');\n  }\n\n  // eslint-disable-next-line no-param-reassign\n  formData = formData || new (PlatformFormData || FormData)();\n\n  // eslint-disable-next-line no-param-reassign\n  options = utils.toFlatObject(options, {\n    metaTokens: true,\n    dots: false,\n    indexes: false\n  }, false, function defined(option, source) {\n    // eslint-disable-next-line no-eq-null,eqeqeq\n    return !utils.isUndefined(source[option]);\n  });\n\n  const metaTokens = options.metaTokens;\n  // eslint-disable-next-line no-use-before-define\n  const visitor = options.visitor || defaultVisitor;\n  const dots = options.dots;\n  const indexes = options.indexes;\n  const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob;\n  const useBlob = _Blob && utils.isSpecCompliantForm(formData);\n\n  if (!utils.isFunction(visitor)) {\n    throw new TypeError('visitor must be a function');\n  }\n\n  function convertValue(value) {\n    if (value === null) return '';\n\n    if (utils.isDate(value)) {\n      return value.toISOString();\n    }\n\n    if (!useBlob && utils.isBlob(value)) {\n      throw new AxiosError('Blob is not supported. Use a Buffer instead.');\n    }\n\n    if (utils.isArrayBuffer(value) || utils.isTypedArray(value)) {\n      return useBlob && typeof Blob === 'function' ? new Blob([value]) : Buffer.from(value);\n    }\n\n    return value;\n  }\n\n  /**\n   * Default visitor.\n   *\n   * @param {*} value\n   * @param {String|Number} key\n   * @param {Array<String|Number>} path\n   * @this {FormData}\n   *\n   * @returns {boolean} return true to visit the each prop of the value recursively\n   */\n  function defaultVisitor(value, key, path) {\n    let arr = value;\n\n    if (value && !path && typeof value === 'object') {\n      if (utils.endsWith(key, '{}')) {\n        // eslint-disable-next-line no-param-reassign\n        key = metaTokens ? key : key.slice(0, -2);\n        // eslint-disable-next-line no-param-reassign\n        value = JSON.stringify(value);\n      } else if (\n        (utils.isArray(value) && isFlatArray(value)) ||\n        ((utils.isFileList(value) || utils.endsWith(key, '[]')) && (arr = utils.toArray(value))\n        )) {\n        // eslint-disable-next-line no-param-reassign\n        key = removeBrackets(key);\n\n        arr.forEach(function each(el, index) {\n          !(utils.isUndefined(el) || el === null) && formData.append(\n            // eslint-disable-next-line no-nested-ternary\n            indexes === true ? renderKey([key], index, dots) : (indexes === null ? key : key + '[]'),\n            convertValue(el)\n          );\n        });\n        return false;\n      }\n    }\n\n    if (isVisitable(value)) {\n      return true;\n    }\n\n    formData.append(renderKey(path, key, dots), convertValue(value));\n\n    return false;\n  }\n\n  const stack = [];\n\n  const exposedHelpers = Object.assign(predicates, {\n    defaultVisitor,\n    convertValue,\n    isVisitable\n  });\n\n  function build(value, path) {\n    if (utils.isUndefined(value)) return;\n\n    if (stack.indexOf(value) !== -1) {\n      throw Error('Circular reference detected in ' + path.join('.'));\n    }\n\n    stack.push(value);\n\n    utils.forEach(value, function each(el, key) {\n      const result = !(utils.isUndefined(el) || el === null) && visitor.call(\n        formData, el, utils.isString(key) ? key.trim() : key, path, exposedHelpers\n      );\n\n      if (result === true) {\n        build(el, path ? path.concat(key) : [key]);\n      }\n    });\n\n    stack.pop();\n  }\n\n  if (!utils.isObject(obj)) {\n    throw new TypeError('data must be an object');\n  }\n\n  build(obj);\n\n  return formData;\n}\n\nexport default toFormData;\n\n// Path: lib/helpers/AxiosURLSearchParams.js\n'use strict';\n\nimport toFormData from './toFormData.js';\n\n/**\n * It encodes a string by replacing all characters that are not in the unreserved set with\n * their percent-encoded equivalents\n *\n * @param {string} str - The string to encode.\n *\n * @returns {string} The encoded string.\n */\nfunction encode(str) {\n  const charMap = {\n    '!': '%21',\n    \"'\": '%27',\n    '(': '%28',\n    ')': '%29',\n    '~': '%7E',\n    '%20': '+',\n    '%00': '\\x00'\n  };\n  return encodeURIComponent(str).replace(/[!'()~]|%20|%00/g, function replacer(match) {\n    return charMap[match];\n  });\n}\n\n/**\n * It takes a params object and converts it to a FormData object\n *\n * @param {Object<string, any>} params - The parameters to be converted to a FormData object.\n * @param {Object<string, any>} options - The options object passed to the Axios constructor.\n *\n * @returns {void}\n */\nfunction AxiosURLSearchParams(params, options) {\n  this._pairs = [];\n\n  params && toFormData(params, this, options);\n}\n\nconst prototype = AxiosURLSearchParams.prototype;\n\nprototype.append = function append(name, value) {\n  this._pairs.push([name, value]);\n};\n\nprototype.toString = function toString(encoder) {\n  const _encode = encoder ? function(value) {\n    return encoder.call(this, value, encode);\n  } : encode;\n\n  return this._pairs.map(function each(pair) {\n    return _encode(pair[0]) + '=' + _encode(pair[1]);\n  }, '').join('&');\n};\n\nexport default AxiosURLSearchParams;\n\n// Path: lib/helpers/buildURL.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosURLSearchParams from '../helpers/AxiosURLSearchParams.js';\n\n/**\n * It replaces all instances of the characters `:`, `$`, `,`, `+`, `[`, and `]` with their\n * URI encoded counterparts\n *\n * @param {string} val The value to be encoded.\n *\n * @returns {string} The encoded value.\n */\nfunction encode(val) {\n  return encodeURIComponent(val).\n    replace(/%3A/gi, ':').\n    replace(/%24/g, '$').\n    replace(/%2C/gi, ',').\n    replace(/%20/g, '+').\n    replace(/%5B/gi, '[').\n    replace(/%5D/gi, ']');\n}\n\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default function buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n\n// Path: lib/cancel/CanceledError.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n\n/**\n * A `CanceledError` is an object that is thrown when an operation is canceled.\n *\n * @param {string=} message The message.\n * @param {Object=} config The config.\n * @param {Object=} request The request.\n *\n * @returns {CanceledError} The created error.\n */\n\nfunction CanceledError(message, config, request) {\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  AxiosError.call(this, message == null ? 'canceled' : message, AxiosError.ERR_CANCELED, config, request);\n  this.name = 'CanceledError';\n}\n\nutils.inherits(CanceledError, AxiosError, {\n  __CANCEL__: true\n});\n\nexport default CanceledError;\n\n// Path: lib/defaults/transitional.js\n'use strict';\n\nexport default {\n  silentJSONParsing: true,\n  forcedJSONParsing: true,\n  clarifyTimeoutError: false\n};\n\n// Path: lib/helpers/speedometer.js\n'use strict';\n\n/**\n * Calculate data maxRate\n * @param {Number} [samplesCount= 10]\n * @param {Number} [min= 1000]\n * @returns {Function}\n */\nfunction speedometer(samplesCount, min) {\n  samplesCount = samplesCount || 10;\n  const bytes = new Array(samplesCount);\n  const timestamps = new Array(samplesCount);\n  let head = 0;\n  let tail = 0;\n  let firstSampleTS;\n\n  min = min !== undefined ? min : 1000;\n\n  return function push(chunkLength) {\n    const now = Date.now();\n\n    const startedAt = timestamps[tail];\n\n    if (!firstSampleTS) {\n      firstSampleTS = now;\n    }\n\n    bytes[head] = chunkLength;\n    timestamps[head] = now;\n\n    let i = tail;\n    let bytesCount = 0;\n\n    while (i !== head) {\n      bytesCount += bytes[i++];\n      i = i % samplesCount;\n    }\n\n    head = (head + 1) % samplesCount;\n\n    if (head === tail) {\n      tail = (tail + 1) % samplesCount;\n    }\n\n    if (now - firstSampleTS < min) {\n      return;\n    }\n\n    const passed = startedAt && now - startedAt;\n\n    return passed ? Math.round(bytesCount * 1000 / passed) : undefined;\n  };\n}\n\nexport default speedometer;\n\n// Path: lib/helpers/parseHeaders.js\n'use strict';\n\nimport utils from './../utils.js';\n\n// RawAxiosHeaders whose duplicates are ignored by node\n// c.f. https://nodejs.org/api/http.html#http_message_headers\nconst ignoreDuplicateOf = utils.toObjectSet([\n  'age', 'authorization', 'content-length', 'content-type', 'etag',\n  'expires', 'from', 'host', 'if-modified-since', 'if-unmodified-since',\n  'last-modified', 'location', 'max-forwards', 'proxy-authorization',\n  'referer', 'retry-after', 'user-agent'\n]);\n\n/**\n * Parse headers into an object\n *\n * ```\n * Date: Wed, 27 Aug 2014 08:58:49 GMT\n * Content-Type: application/json\n * Connection: keep-alive\n * Transfer-Encoding: chunked\n * ```\n *\n * @param {String} rawHeaders Headers needing to be parsed\n *\n * @returns {Object} Headers parsed into an object\n */\nexport default rawHeaders => {\n  const parsed = {};\n  let key;\n  let val;\n  let i;\n\n  rawHeaders && rawHeaders.split('\\n').forEach(function parser(line) {\n    i = line.indexOf(':');\n    key = line.substring(0, i).trim().toLowerCase();\n    val = line.substring(i + 1).trim();\n\n    if (!key || (parsed[key] && ignoreDuplicateOf[key])) {\n      return;\n    }\n\n    if (key === 'set-cookie') {\n      if (parsed[key]) {\n        parsed[key].push(val);\n      } else {\n        parsed[key] = [val];\n      }\n    } else {\n      parsed[key] = parsed[key] ? parsed[key] + ', ' + val : val;\n    }\n  });\n\n  return parsed;\n};\n\n// Path: lib/core/AxiosHeaders.js\n'use strict';\n\nimport utils from '../utils.js';\nimport parseHeaders from '../helpers/parseHeaders.js';\n\nconst $internals = Symbol('internals');\n\nfunction normalizeHeader(header) {\n  return header && String(header).trim().toLowerCase();\n}\n\nfunction normalizeValue(value) {\n  if (value === false || value == null) {\n    return value;\n  }\n\n  return utils.isArray(value) ? value.map(normalizeValue) : String(value);\n}\n\nfunction parseTokens(str) {\n  const tokens = Object.create(null);\n  const tokensRE = /([^\\s,;=]+)\\s*(?:=\\s*([^,;]+))?/g;\n  let match;\n\n  while ((match = tokensRE.exec(str))) {\n    tokens[match[1]] = match[2];\n  }\n\n  return tokens;\n}\n\nconst isValidHeaderName = (str) => /^[-_a-zA-Z0-9^`|~,!#$%&'*+.]+$/.test(str.trim());\n\nfunction matchHeaderValue(context, value, header, filter, isHeaderNameFilter) {\n  if (utils.isFunction(filter)) {\n    return filter.call(this, value, header);\n  }\n\n  if (isHeaderNameFilter) {\n    value = header;\n  }\n\n  if (!utils.isString(value)) return;\n\n  if (utils.isString(filter)) {\n    return value.indexOf(filter) !== -1;\n  }\n\n  if (utils.isRegExp(filter)) {\n    return filter.test(value);\n  }\n}\n\nfunction formatHeader(header) {\n  return header.trim()\n    .toLowerCase().replace(/([a-z\\d])(\\w*)/g, (w, char, str) => {\n      return char.toUpperCase() + str;\n    });\n}\n\nfunction buildAccessors(obj, header) {\n  const accessorName = utils.toCamelCase(' ' + header);\n\n  ['get', 'set', 'has'].forEach(methodName => {\n    Object.defineProperty(obj, methodName + accessorName, {\n      value: function(arg1, arg2, arg3) {\n        return this[methodName].call(this, header, arg1, arg2, arg3);\n      },\n      configurable: true\n    });\n  });\n}\n\nclass AxiosHeaders {\n  constructor(headers) {\n    headers && this.set(headers);\n  }\n\n  set(header, valueOrRewrite, rewrite) {\n    const self = this;\n\n    function setHeader(_value, _header, _rewrite) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!lHeader) {\n        throw new Error('header name must be a non-empty string');\n      }\n\n      const key = utils.findKey(self, lHeader);\n\n      if(!key || self[key] === undefined || _rewrite === true || (_rewrite === undefined && self[key] !== false)) {\n        self[key || _header] = normalizeValue(_value);\n      }\n    }\n\n    const setHeaders = (headers, _rewrite) =>\n      utils.forEach(headers, (_value, _header) => setHeader(_value, _header, _rewrite));\n\n    if (utils.isPlainObject(header) || header instanceof this.constructor) {\n      setHeaders(header, valueOrRewrite)\n    } else if(utils.isString(header) && (header = header.trim()) && !isValidHeaderName(header)) {\n      setHeaders(parseHeaders(header), valueOrRewrite);\n    } else {\n      header != null && setHeader(valueOrRewrite, header, rewrite);\n    }\n\n    return this;\n  }\n\n  get(header, parser) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      if (key) {\n        const value = this[key];\n\n        if (!parser) {\n          return value;\n        }\n\n        if (parser === true) {\n          return parseTokens(value);\n        }\n\n        if (utils.isFunction(parser)) {\n          return parser.call(this, value, key);\n        }\n\n        if (utils.isRegExp(parser)) {\n          return parser.exec(value);\n        }\n\n        throw new TypeError('parser must be boolean|regexp|function');\n      }\n    }\n  }\n\n  has(header, matcher) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      return !!(key && this[key] !== undefined && (!matcher || matchHeaderValue(this, this[key], key, matcher)));\n    }\n\n    return false;\n  }\n\n  delete(header, matcher) {\n    const self = this;\n    let deleted = false;\n\n    function deleteHeader(_header) {\n      _header = normalizeHeader(_header);\n\n      if (_header) {\n        const key = utils.findKey(self, _header);\n\n        if (key && (!matcher || matchHeaderValue(self, self[key], key, matcher))) {\n          delete self[key];\n\n          deleted = true;\n        }\n      }\n    }\n\n    if (utils.isArray(header)) {\n      header.forEach(deleteHeader);\n    } else {\n      deleteHeader(header);\n    }\n\n    return deleted;\n  }\n\n  clear(matcher) {\n    const keys = Object.keys(this);\n    let i = keys.length;\n    let deleted = false;\n\n    while (i--) {\n      const key = keys[i];\n      if(!matcher || matchHeaderValue(this, this[key], key, matcher, true)) {\n        delete this[key];\n        deleted = true;\n      }\n    }\n\n    return deleted;\n  }\n\n  normalize(format) {\n    const self = this;\n    const headers = {};\n\n    utils.forEach(this, (value, header) => {\n      const key = utils.findKey(headers, header);\n\n      if (key) {\n        self[key] = normalizeValue(value);\n        delete self[header];\n        return;\n      }\n\n      const normalized = format ? formatHeader(header) : String(header).trim();\n\n      if (normalized !== header) {\n        delete self[header];\n      }\n\n      self[normalized] = normalizeValue(value);\n\n      headers[normalized] = true;\n    });\n\n    return this;\n  }\n\n  concat(...targets) {\n    return this.constructor.concat(this, ...targets);\n  }\n\n  toJSON(asStrings) {\n    const obj = Object.create(null);\n\n    utils.forEach(this, (value, header) => {\n      value != null && value !== false && (obj[header] = asStrings && utils.isArray(value) ? value.join(', ') : value);\n    });\n\n    return obj;\n  }\n\n  [Symbol.iterator]() {\n    return Object.entries(this.toJSON())[Symbol.iterator]();\n  }\n\n  toString() {\n    return Object.entries(this.toJSON()).map(([header, value]) => header + ': ' + value).join('\\n');\n  }\n\n  get [Symbol.toStringTag]() {\n    return 'AxiosHeaders';\n  }\n\n  static from(thing) {\n    return thing instanceof this ? thing : new this(thing);\n  }\n\n  static concat(first, ...targets) {\n    const computed = new this(first);\n\n    targets.forEach((target) => computed.set(target));\n\n    return computed;\n  }\n\n  static accessor(header) {\n    const internals = this[$internals] = (this[$internals] = {\n      accessors: {}\n    });\n\n    const accessors = internals.accessors;\n    const prototype = this.prototype;\n\n    function defineAccessor(_header) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!accessors[lHeader]) {\n        buildAccessors(prototype, _header);\n        accessors[lHeader] = true;\n      }\n    }\n\n    utils.isArray(header) ? header.forEach(defineAccessor) : defineAccessor(header);\n\n    return this;\n  }\n}\n\nAxiosHeaders.accessor(['Content-Type', 'Content-Length', 'Accept', 'Accept-Encoding', 'User-Agent', 'Authorization']);\n\n// reserved names hotfix\nutils.reduceDescriptors(AxiosHeaders.prototype, ({value}, key) => {\n  let mapped = key[0].toUpperCase() + key.slice(1); // map `set` => `Set`\n  return {\n    get: () => value,\n    set(headerValue) {\n      this[mapped] = headerValue;\n    }\n  }\n});\n\nutils.freezeMethods(AxiosHeaders);\n\nexport default AxiosHeaders;\n\n// Path: lib/core/settle.js\n'use strict';\n\nimport AxiosError from './AxiosError.js';\n\n/**\n * Resolve or reject a Promise based on response status.\n *\n * @param {Function} resolve A function that resolves the promise.\n * @param {Function} reject A function that rejects the promise.\n * @param {object} response The response.\n *\n * @returns {object} The response.\n */\nexport default function settle(resolve, reject, response) {\n  const validateStatus = response.config.validateStatus;\n  if (!response.status || !validateStatus || validateStatus(response.status)) {\n    resolve(response);\n  } else {\n    reject(new AxiosError(\n      'Request failed with status code ' + response.status,\n      [AxiosError.ERR_BAD_REQUEST, AxiosError.ERR_BAD_RESPONSE][Math.floor(response.status / 100) - 4],\n      response.config,\n      response.request,\n      response\n    ));\n  }\n}\n\n// Path: lib/helpers/parseProtocol.js\n'use strict';\n\nexport default function parseProtocol(url) {\n  const match = /^([-+\\w]{1,25})(:?\\/\\/|:)/.exec(url);\n  return match && match[1] || '';\n}\n\n// Path: lib/helpers/cookies.js\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n  // Standard browser envs support document.cookie\n  {\n    write(name, value, expires, path, domain, secure) {\n      const cookie = [name + '=' + encodeURIComponent(value)];\n\n      utils.isNumber(expires) && cookie.push('expires=' + new Date(expires).toGMTString());\n\n      utils.isString(path) && cookie.push('path=' + path);\n\n      utils.isString(domain) && cookie.push('domain=' + domain);\n\n      secure === true && cookie.push('secure');\n\n      document.cookie = cookie.join('; ');\n    },\n\n    read(name) {\n      const match = document.cookie.match(new RegExp('(^|;\\\\s*)(' + name + ')=([^;]*)'));\n      return (match ? decodeURIComponent(match[3]) : null);\n    },\n\n    remove(name) {\n      this.write(name, '', Date.now() - 86400000);\n    }\n  }\n\n  :\n\n  // Non-standard browser env (web workers, react-native) lack needed support.\n  {\n...\n", "needle_token_start": 10990, "needle_token_end": 11073, "code_context_ntokens": 14978, "output": ["```javascript\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n\n/**\n * A `CanceledError` is an object that is thrown when an operation is canceled.\n *\n * @param {string=} message The message.\n * @param {Object=} config The config.\n * @param {Object=} request The request.\n *\n * @returns {CanceledError} The created error.\n */\n\nfunction CanceledError(message, config, request) {\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  AxiosError.call(this, message == null ? 'canceled' : message, AxiosError.ERR_CANCELED, config, request);\n  this.name = 'CanceledError';\n}\n\nutils.inherits(CanceledError, AxiosError, {\n  __CANCEL__: true\n});\n\nexport default CanceledError;\n```"]}
{"repo": "axios/axios", "name": "deleteHeader", "language": "typescript", "path": "lib/core/AxiosHeaders.js", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to remove specific entries from an object based on header names, optionally using a matching condition.\n2. **Input**: The function takes two parameters: a header name and an optional matcher. The header name is a string that specifies which entry to consider for deletion. The matcher is an optional parameter that can be used to apply additional matching conditions to the header value before deletion.\n3. **Output**: The function does not return a value but modifies the object by potentially removing one or more of its properties.\n4. **Procedure**: The function first normalizes the header name. It then searches for this normalized header within the object's properties. If found, and if no matcher is provided or if the matcher condition is satisfied, the property corresponding to the header is deleted from the object. A flag is set to true if any property is deleted.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "...\n// Path: lib/platform/node/index.js\nimport URLSearchParams from './classes/URLSearchParams.js'\nimport FormData from './classes/FormData.js'\n\nexport default {\n  isNode: true,\n  classes: {\n    URLSearchParams,\n    FormData,\n    Blob: typeof Blob !== 'undefined' && Blob || null\n  },\n  protocols: [ 'http', 'https', 'file', 'data' ]\n};\n\n// Path: lib/platform/common/utils.js\nconst hasBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';\n\n/**\n * Determine if we're running in a standard browser environment\n *\n * This allows axios to run in a web worker, and react-native.\n * Both environments support XMLHttpRequest, but not fully standard globals.\n *\n * web workers:\n *  typeof window -> undefined\n *  typeof document -> undefined\n *\n * react-native:\n *  navigator.product -> 'ReactNative'\n * nativescript\n *  navigator.product -> 'NativeScript' or 'NS'\n *\n * @returns {boolean}\n */\nconst hasStandardBrowserEnv = (\n  (product) => {\n    return hasBrowserEnv && ['ReactNative', 'NativeScript', 'NS'].indexOf(product) < 0\n  })(typeof navigator !== 'undefined' && navigator.product);\n\n/**\n * Determine if we're running in a standard browser webWorker environment\n *\n * Although the `isStandardBrowserEnv` method indicates that\n * `allows axios to run in a web worker`, the WebWorker will still be\n * filtered out due to its judgment standard\n * `typeof window !== 'undefined' && typeof document !== 'undefined'`.\n * This leads to a problem when axios post `FormData` in webWorker\n */\nconst hasStandardBrowserWebWorkerEnv = (() => {\n  return (\n    typeof WorkerGlobalScope !== 'undefined' &&\n    // eslint-disable-next-line no-undef\n    self instanceof WorkerGlobalScope &&\n    typeof self.importScripts === 'function'\n  );\n})();\n\nexport {\n  hasBrowserEnv,\n  hasStandardBrowserWebWorkerEnv,\n  hasStandardBrowserEnv\n}\n\n// Path: lib/platform/index.js\nimport platform from './node/index.js';\nimport * as utils from './common/utils.js';\n\nexport default {\n  ...utils,\n  ...platform\n}\n\n// Path: lib/helpers/combineURLs.js\n'use strict';\n\n/**\n * Creates a new URL by combining the specified URLs\n *\n * @param {string} baseURL The base URL\n * @param {string} relativeURL The relative URL\n *\n * @returns {string} The combined URL\n */\nexport default function combineURLs(baseURL, relativeURL) {\n  return relativeURL\n    ? baseURL.replace(/\\/?\\/$/, '') + '/' + relativeURL.replace(/^\\/+/, '')\n    : baseURL;\n}\n\n// Path: lib/helpers/isAbsoluteURL.js\n'use strict';\n\n/**\n * Determines whether the specified URL is absolute\n *\n * @param {string} url The URL to test\n *\n * @returns {boolean} True if the specified URL is absolute, otherwise false\n */\nexport default function isAbsoluteURL(url) {\n  // A URL is considered absolute if it begins with \"<scheme>://\" or \"//\" (protocol-relative URL).\n  // RFC 3986 defines scheme name as a sequence of characters beginning with a letter and followed\n  // by any combination of letters, digits, plus, period, or hyphen.\n  return /^([a-z][a-z\\d+\\-.]*:)?\\/\\//i.test(url);\n}\n\n// Path: lib/core/buildFullPath.js\n'use strict';\n\nimport isAbsoluteURL from '../helpers/isAbsoluteURL.js';\nimport combineURLs from '../helpers/combineURLs.js';\n\n/**\n * Creates a new URL by combining the baseURL with the requestedURL,\n * only when the requestedURL is not already an absolute URL.\n * If the requestURL is absolute, this function returns the requestedURL untouched.\n *\n * @param {string} baseURL The base URL\n * @param {string} requestedURL Absolute or relative URL to combine\n *\n * @returns {string} The combined full path\n */\nexport default function buildFullPath(baseURL, requestedURL) {\n  if (baseURL && !isAbsoluteURL(requestedURL)) {\n    return combineURLs(baseURL, requestedURL);\n  }\n  return requestedURL;\n}\n\n// Path: lib/helpers/bind.js\n'use strict';\n\nexport default function bind(fn, thisArg) {\n  return function wrap() {\n    return fn.apply(thisArg, arguments);\n  };\n}\n\n// Path: lib/utils.js\n'use strict';\n\nimport bind from './helpers/bind.js';\n\n// utils is a library of generic helper functions non-specific to axios\n\nconst {toString} = Object.prototype;\nconst {getPrototypeOf} = Object;\n\nconst kindOf = (cache => thing => {\n    const str = toString.call(thing);\n    return cache[str] || (cache[str] = str.slice(8, -1).toLowerCase());\n})(Object.create(null));\n\nconst kindOfTest = (type) => {\n  type = type.toLowerCase();\n  return (thing) => kindOf(thing) === type\n}\n\nconst typeOfTest = type => thing => typeof thing === type;\n\n/**\n * Determine if a value is an Array\n *\n * @param {Object} val The value to test\n *\n * @returns {boolean} True if value is an Array, otherwise false\n */\nconst {isArray} = Array;\n\n/**\n * Determine if a value is undefined\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if the value is undefined, otherwise false\n */\nconst isUndefined = typeOfTest('undefined');\n\n/**\n * Determine if a value is a Buffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Buffer, otherwise false\n */\nfunction isBuffer(val) {\n  return val !== null && !isUndefined(val) && val.constructor !== null && !isUndefined(val.constructor)\n    && isFunction(val.constructor.isBuffer) && val.constructor.isBuffer(val);\n}\n\n/**\n * Determine if a value is an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is an ArrayBuffer, otherwise false\n */\nconst isArrayBuffer = kindOfTest('ArrayBuffer');\n\n\n/**\n * Determine if a value is a view on an ArrayBuffer\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a view on an ArrayBuffer, otherwise false\n */\nfunction isArrayBufferView(val) {\n  let result;\n  if ((typeof ArrayBuffer !== 'undefined') && (ArrayBuffer.isView)) {\n    result = ArrayBuffer.isView(val);\n  } else {\n    result = (val) && (val.buffer) && (isArrayBuffer(val.buffer));\n  }\n  return result;\n}\n\n/**\n * Determine if a value is a String\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a String, otherwise false\n */\nconst isString = typeOfTest('string');\n\n/**\n * Determine if a value is a Function\n *\n * @param {*} val The value to test\n * @returns {boolean} True if value is a Function, otherwise false\n */\nconst isFunction = typeOfTest('function');\n\n/**\n * Determine if a value is a Number\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Number, otherwise false\n */\nconst isNumber = typeOfTest('number');\n\n/**\n * Determine if a value is an Object\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an Object, otherwise false\n */\nconst isObject = (thing) => thing !== null && typeof thing === 'object';\n\n/**\n * Determine if a value is a Boolean\n *\n * @param {*} thing The value to test\n * @returns {boolean} True if value is a Boolean, otherwise false\n */\nconst isBoolean = thing => thing === true || thing === false;\n\n/**\n * Determine if a value is a plain Object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a plain Object, otherwise false\n */\nconst isPlainObject = (val) => {\n  if (kindOf(val) !== 'object') {\n    return false;\n  }\n\n  const prototype = getPrototypeOf(val);\n  return (prototype === null || prototype === Object.prototype || Object.getPrototypeOf(prototype) === null) && !(Symbol.toStringTag in val) && !(Symbol.iterator in val);\n}\n\n/**\n * Determine if a value is a Date\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Date, otherwise false\n */\nconst isDate = kindOfTest('Date');\n\n/**\n * Determine if a value is a File\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFile = kindOfTest('File');\n\n/**\n * Determine if a value is a Blob\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Blob, otherwise false\n */\nconst isBlob = kindOfTest('Blob');\n\n/**\n * Determine if a value is a FileList\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a File, otherwise false\n */\nconst isFileList = kindOfTest('FileList');\n\n/**\n * Determine if a value is a Stream\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a Stream, otherwise false\n */\nconst isStream = (val) => isObject(val) && isFunction(val.pipe);\n\n/**\n * Determine if a value is a FormData\n *\n * @param {*} thing The value to test\n *\n * @returns {boolean} True if value is an FormData, otherwise false\n */\nconst isFormData = (thing) => {\n  let kind;\n  return thing && (\n    (typeof FormData === 'function' && thing instanceof FormData) || (\n      isFunction(thing.append) && (\n        (kind = kindOf(thing)) === 'formdata' ||\n        // detect form-data instance\n        (kind === 'object' && isFunction(thing.toString) && thing.toString() === '[object FormData]')\n      )\n    )\n  )\n}\n\n/**\n * Determine if a value is a URLSearchParams object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a URLSearchParams object, otherwise false\n */\nconst isURLSearchParams = kindOfTest('URLSearchParams');\n\n/**\n * Trim excess whitespace off the beginning and end of a string\n *\n * @param {String} str The String to trim\n *\n * @returns {String} The String freed of excess whitespace\n */\nconst trim = (str) => str.trim ?\n  str.trim() : str.replace(/^[\\s\\uFEFF\\xA0]+|[\\s\\uFEFF\\xA0]+$/g, '');\n\n/**\n * Iterate over an Array or an Object invoking a function for each item.\n *\n * If `obj` is an Array callback will be called passing\n * the value, index, and complete array for each item.\n *\n * If 'obj' is an Object callback will be called passing\n * the value, key, and complete object for each property.\n *\n * @param {Object|Array} obj The object to iterate\n * @param {Function} fn The callback to invoke for each item\n *\n * @param {Boolean} [allOwnKeys = false]\n * @returns {any}\n */\nfunction forEach(obj, fn, {allOwnKeys = false} = {}) {\n  // Don't bother if no value provided\n  if (obj === null || typeof obj === 'undefined') {\n    return;\n  }\n\n  let i;\n  let l;\n\n  // Force an array if not already something iterable\n  if (typeof obj !== 'object') {\n    /*eslint no-param-reassign:0*/\n    obj = [obj];\n  }\n\n  if (isArray(obj)) {\n    // Iterate over array values\n    for (i = 0, l = obj.length; i < l; i++) {\n      fn.call(null, obj[i], i, obj);\n    }\n  } else {\n    // Iterate over object keys\n    const keys = allOwnKeys ? Object.getOwnPropertyNames(obj) : Object.keys(obj);\n    const len = keys.length;\n    let key;\n\n    for (i = 0; i < len; i++) {\n      key = keys[i];\n      fn.call(null, obj[key], key, obj);\n    }\n  }\n}\n\nfunction findKey(obj, key) {\n  key = key.toLowerCase();\n  const keys = Object.keys(obj);\n  let i = keys.length;\n  let _key;\n  while (i-- > 0) {\n    _key = keys[i];\n    if (key === _key.toLowerCase()) {\n      return _key;\n    }\n  }\n  return null;\n}\n\nconst _global = (() => {\n  /*eslint no-undef:0*/\n  if (typeof globalThis !== \"undefined\") return globalThis;\n  return typeof self !== \"undefined\" ? self : (typeof window !== 'undefined' ? window : global)\n})();\n\nconst isContextDefined = (context) => !isUndefined(context) && context !== _global;\n\n/**\n * Accepts varargs expecting each argument to be an object, then\n * immutably merges the properties of each object and returns result.\n *\n * When multiple objects contain the same key the later object in\n * the arguments list will take precedence.\n *\n * Example:\n *\n * ```js\n * var result = merge({foo: 123}, {foo: 456});\n * console.log(result.foo); // outputs 456\n * ```\n *\n * @param {Object} obj1 Object to merge\n *\n * @returns {Object} Result of all merge properties\n */\nfunction merge(/* obj1, obj2, obj3, ... */) {\n  const {caseless} = isContextDefined(this) && this || {};\n  const result = {};\n  const assignValue = (val, key) => {\n    const targetKey = caseless && findKey(result, key) || key;\n    if (isPlainObject(result[targetKey]) && isPlainObject(val)) {\n      result[targetKey] = merge(result[targetKey], val);\n    } else if (isPlainObject(val)) {\n      result[targetKey] = merge({}, val);\n    } else if (isArray(val)) {\n      result[targetKey] = val.slice();\n    } else {\n      result[targetKey] = val;\n    }\n  }\n\n  for (let i = 0, l = arguments.length; i < l; i++) {\n    arguments[i] && forEach(arguments[i], assignValue);\n  }\n  return result;\n}\n\n/**\n * Extends object a by mutably adding to it the properties of object b.\n *\n * @param {Object} a The object to be extended\n * @param {Object} b The object to copy properties from\n * @param {Object} thisArg The object to bind function to\n *\n * @param {Boolean} [allOwnKeys]\n * @returns {Object} The resulting value of object a\n */\nconst extend = (a, b, thisArg, {allOwnKeys}= {}) => {\n  forEach(b, (val, key) => {\n    if (thisArg && isFunction(val)) {\n      a[key] = bind(val, thisArg);\n    } else {\n      a[key] = val;\n    }\n  }, {allOwnKeys});\n  return a;\n}\n\n/**\n * Remove byte order marker. This catches EF BB BF (the UTF-8 BOM)\n *\n * @param {string} content with BOM\n *\n * @returns {string} content value without BOM\n */\nconst stripBOM = (content) => {\n  if (content.charCodeAt(0) === 0xFEFF) {\n    content = content.slice(1);\n  }\n  return content;\n}\n\n/**\n * Inherit the prototype methods from one constructor into another\n * @param {function} constructor\n * @param {function} superConstructor\n * @param {object} [props]\n * @param {object} [descriptors]\n *\n * @returns {void}\n */\nconst inherits = (constructor, superConstructor, props, descriptors) => {\n  constructor.prototype = Object.create(superConstructor.prototype, descriptors);\n  constructor.prototype.constructor = constructor;\n  Object.defineProperty(constructor, 'super', {\n    value: superConstructor.prototype\n  });\n  props && Object.assign(constructor.prototype, props);\n}\n\n/**\n * Resolve object with deep prototype chain to a flat object\n * @param {Object} sourceObj source object\n * @param {Object} [destObj]\n * @param {Function|Boolean} [filter]\n * @param {Function} [propFilter]\n *\n * @returns {Object}\n */\nconst toFlatObject = (sourceObj, destObj, filter, propFilter) => {\n  let props;\n  let i;\n  let prop;\n  const merged = {};\n\n  destObj = destObj || {};\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  if (sourceObj == null) return destObj;\n\n  do {\n    props = Object.getOwnPropertyNames(sourceObj);\n    i = props.length;\n    while (i-- > 0) {\n      prop = props[i];\n      if ((!propFilter || propFilter(prop, sourceObj, destObj)) && !merged[prop]) {\n        destObj[prop] = sourceObj[prop];\n        merged[prop] = true;\n      }\n    }\n    sourceObj = filter !== false && getPrototypeOf(sourceObj);\n  } while (sourceObj && (!filter || filter(sourceObj, destObj)) && sourceObj !== Object.prototype);\n\n  return destObj;\n}\n\n/**\n * Determines whether a string ends with the characters of a specified string\n *\n * @param {String} str\n * @param {String} searchString\n * @param {Number} [position= 0]\n *\n * @returns {boolean}\n */\nconst endsWith = (str, searchString, position) => {\n  str = String(str);\n  if (position === undefined || position > str.length) {\n    position = str.length;\n  }\n  position -= searchString.length;\n  const lastIndex = str.indexOf(searchString, position);\n  return lastIndex !== -1 && lastIndex === position;\n}\n\n\n/**\n * Returns new array from array like object or null if failed\n *\n * @param {*} [thing]\n *\n * @returns {?Array}\n */\nconst toArray = (thing) => {\n  if (!thing) return null;\n  if (isArray(thing)) return thing;\n  let i = thing.length;\n  if (!isNumber(i)) return null;\n  const arr = new Array(i);\n  while (i-- > 0) {\n    arr[i] = thing[i];\n  }\n  return arr;\n}\n\n/**\n * Checking if the Uint8Array exists and if it does, it returns a function that checks if the\n * thing passed in is an instance of Uint8Array\n *\n * @param {TypedArray}\n *\n * @returns {Array}\n */\n// eslint-disable-next-line func-names\nconst isTypedArray = (TypedArray => {\n  // eslint-disable-next-line func-names\n  return thing => {\n    return TypedArray && thing instanceof TypedArray;\n  };\n})(typeof Uint8Array !== 'undefined' && getPrototypeOf(Uint8Array));\n\n/**\n * For each entry in the object, call the function with the key and value.\n *\n * @param {Object<any, any>} obj - The object to iterate over.\n * @param {Function} fn - The function to call for each entry.\n *\n * @returns {void}\n */\nconst forEachEntry = (obj, fn) => {\n  const generator = obj && obj[Symbol.iterator];\n\n  const iterator = generator.call(obj);\n\n  let result;\n\n  while ((result = iterator.next()) && !result.done) {\n    const pair = result.value;\n    fn.call(obj, pair[0], pair[1]);\n  }\n}\n\n/**\n * It takes a regular expression and a string, and returns an array of all the matches\n *\n * @param {string} regExp - The regular expression to match against.\n * @param {string} str - The string to search.\n *\n * @returns {Array<boolean>}\n */\nconst matchAll = (regExp, str) => {\n  let matches;\n  const arr = [];\n\n  while ((matches = regExp.exec(str)) !== null) {\n    arr.push(matches);\n  }\n\n  return arr;\n}\n\n/* Checking if the kindOfTest function returns true when passed an HTMLFormElement. */\nconst isHTMLForm = kindOfTest('HTMLFormElement');\n\nconst toCamelCase = str => {\n  return str.toLowerCase().replace(/[-_\\s]([a-z\\d])(\\w*)/g,\n    function replacer(m, p1, p2) {\n      return p1.toUpperCase() + p2;\n    }\n  );\n};\n\n/* Creating a function that will check if an object has a property. */\nconst hasOwnProperty = (({hasOwnProperty}) => (obj, prop) => hasOwnProperty.call(obj, prop))(Object.prototype);\n\n/**\n * Determine if a value is a RegExp object\n *\n * @param {*} val The value to test\n *\n * @returns {boolean} True if value is a RegExp object, otherwise false\n */\nconst isRegExp = kindOfTest('RegExp');\n\nconst reduceDescriptors = (obj, reducer) => {\n  const descriptors = Object.getOwnPropertyDescriptors(obj);\n  const reducedDescriptors = {};\n\n  forEach(descriptors, (descriptor, name) => {\n    let ret;\n    if ((ret = reducer(descriptor, name, obj)) !== false) {\n      reducedDescriptors[name] = ret || descriptor;\n    }\n  });\n\n  Object.defineProperties(obj, reducedDescriptors);\n}\n\n/**\n * Makes all methods read-only\n * @param {Object} obj\n */\n\nconst freezeMethods = (obj) => {\n  reduceDescriptors(obj, (descriptor, name) => {\n    // skip restricted props in strict mode\n    if (isFunction(obj) && ['arguments', 'caller', 'callee'].indexOf(name) !== -1) {\n      return false;\n    }\n\n    const value = obj[name];\n\n    if (!isFunction(value)) return;\n\n    descriptor.enumerable = false;\n\n    if ('writable' in descriptor) {\n      descriptor.writable = false;\n      return;\n    }\n\n    if (!descriptor.set) {\n      descriptor.set = () => {\n        throw Error('Can not rewrite read-only method \\'' + name + '\\'');\n      };\n    }\n  });\n}\n\nconst toObjectSet = (arrayOrString, delimiter) => {\n  const obj = {};\n\n  const define = (arr) => {\n    arr.forEach(value => {\n      obj[value] = true;\n    });\n  }\n\n  isArray(arrayOrString) ? define(arrayOrString) : define(String(arrayOrString).split(delimiter));\n\n  return obj;\n}\n\nconst noop = () => {}\n\nconst toFiniteNumber = (value, defaultValue) => {\n  value = +value;\n  return Number.isFinite(value) ? value : defaultValue;\n}\n\nconst ALPHA = 'abcdefghijklmnopqrstuvwxyz'\n\nconst DIGIT = '0123456789';\n\nconst ALPHABET = {\n  DIGIT,\n  ALPHA,\n  ALPHA_DIGIT: ALPHA + ALPHA.toUpperCase() + DIGIT\n}\n\nconst generateString = (size = 16, alphabet = ALPHABET.ALPHA_DIGIT) => {\n  let str = '';\n  const {length} = alphabet;\n  while (size--) {\n    str += alphabet[Math.random() * length|0]\n  }\n\n  return str;\n}\n\n/**\n * If the thing is a FormData object, return true, otherwise return false.\n *\n * @param {unknown} thing - The thing to check.\n *\n * @returns {boolean}\n */\nfunction isSpecCompliantForm(thing) {\n  return !!(thing && isFunction(thing.append) && thing[Symbol.toStringTag] === 'FormData' && thing[Symbol.iterator]);\n}\n\nconst toJSONObject = (obj) => {\n  const stack = new Array(10);\n\n  const visit = (source, i) => {\n\n    if (isObject(source)) {\n      if (stack.indexOf(source) >= 0) {\n        return;\n      }\n\n      if(!('toJSON' in source)) {\n        stack[i] = source;\n        const target = isArray(source) ? [] : {};\n\n        forEach(source, (value, key) => {\n          const reducedValue = visit(value, i + 1);\n          !isUndefined(reducedValue) && (target[key] = reducedValue);\n        });\n\n        stack[i] = undefined;\n\n        return target;\n      }\n    }\n\n    return source;\n  }\n\n  return visit(obj, 0);\n}\n\nconst isAsyncFn = kindOfTest('AsyncFunction');\n\nconst isThenable = (thing) =>\n  thing && (isObject(thing) || isFunction(thing)) && isFunction(thing.then) && isFunction(thing.catch);\n\nexport default {\n  isArray,\n  isArrayBuffer,\n  isBuffer,\n  isFormData,\n  isArrayBufferView,\n  isString,\n  isNumber,\n  isBoolean,\n  isObject,\n  isPlainObject,\n  isUndefined,\n  isDate,\n  isFile,\n  isBlob,\n  isRegExp,\n  isFunction,\n  isStream,\n  isURLSearchParams,\n  isTypedArray,\n  isFileList,\n  forEach,\n  merge,\n  extend,\n  trim,\n  stripBOM,\n  inherits,\n  toFlatObject,\n  kindOf,\n  kindOfTest,\n  endsWith,\n  toArray,\n  forEachEntry,\n  matchAll,\n  isHTMLForm,\n  hasOwnProperty,\n  hasOwnProp: hasOwnProperty, // an alias to avoid ESLint no-prototype-builtins detection\n  reduceDescriptors,\n  freezeMethods,\n  toObjectSet,\n  toCamelCase,\n  noop,\n  toFiniteNumber,\n  findKey,\n  global: _global,\n  isContextDefined,\n  ALPHABET,\n  generateString,\n  isSpecCompliantForm,\n  toJSONObject,\n  isAsyncFn,\n  isThenable\n};\n\n// Path: lib/core/AxiosError.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [config] The config.\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n *\n * @returns {Error} The created error.\n */\nfunction AxiosError(message, code, config, request, response) {\n  Error.call(this);\n\n  if (Error.captureStackTrace) {\n    Error.captureStackTrace(this, this.constructor);\n  } else {\n    this.stack = (new Error()).stack;\n  }\n\n  this.message = message;\n  this.name = 'AxiosError';\n  code && (this.code = code);\n  config && (this.config = config);\n  request && (this.request = request);\n  response && (this.response = response);\n}\n\nutils.inherits(AxiosError, Error, {\n  toJSON: function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: utils.toJSONObject(this.config),\n      code: this.code,\n      status: this.response && this.response.status ? this.response.status : null\n    };\n  }\n});\n\nconst prototype = AxiosError.prototype;\nconst descriptors = {};\n\n[\n  'ERR_BAD_OPTION_VALUE',\n  'ERR_BAD_OPTION',\n  'ECONNABORTED',\n  'ETIMEDOUT',\n  'ERR_NETWORK',\n  'ERR_FR_TOO_MANY_REDIRECTS',\n  'ERR_DEPRECATED',\n  'ERR_BAD_RESPONSE',\n  'ERR_BAD_REQUEST',\n  'ERR_CANCELED',\n  'ERR_NOT_SUPPORT',\n  'ERR_INVALID_URL'\n// eslint-disable-next-line func-names\n].forEach(code => {\n  descriptors[code] = {value: code};\n});\n\nObject.defineProperties(AxiosError, descriptors);\nObject.defineProperty(prototype, 'isAxiosError', {value: true});\n\n// eslint-disable-next-line func-names\nAxiosError.from = (error, code, config, request, response, customProps) => {\n  const axiosError = Object.create(prototype);\n\n  utils.toFlatObject(error, axiosError, function filter(obj) {\n    return obj !== Error.prototype;\n  }, prop => {\n    return prop !== 'isAxiosError';\n  });\n\n  AxiosError.call(axiosError, error.message, code, config, request, response);\n\n  axiosError.cause = error;\n\n  axiosError.name = error.name;\n\n  customProps && Object.assign(axiosError, customProps);\n\n  return axiosError;\n};\n\nexport default AxiosError;\n\n// Path: lib/helpers/toFormData.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\n// temporary hotfix to avoid circular references until AxiosURLSearchParams is refactored\nimport PlatformFormData from '../platform/node/classes/FormData.js';\n\n/**\n * Determines if the given thing is a array or js object.\n *\n * @param {string} thing - The object or array to be visited.\n *\n * @returns {boolean}\n */\nfunction isVisitable(thing) {\n  return utils.isPlainObject(thing) || utils.isArray(thing);\n}\n\n/**\n * It removes the brackets from the end of a string\n *\n * @param {string} key - The key of the parameter.\n *\n * @returns {string} the key without the brackets.\n */\nfunction removeBrackets(key) {\n  return utils.endsWith(key, '[]') ? key.slice(0, -2) : key;\n}\n\n/**\n * It takes a path, a key, and a boolean, and returns a string\n *\n * @param {string} path - The path to the current key.\n * @param {string} key - The key of the current object being iterated over.\n * @param {string} dots - If true, the key will be rendered with dots instead of brackets.\n *\n * @returns {string} The path to the current key.\n */\nfunction renderKey(path, key, dots) {\n  if (!path) return key;\n  return path.concat(key).map(function each(token, i) {\n    // eslint-disable-next-line no-param-reassign\n    token = removeBrackets(token);\n    return !dots && i ? '[' + token + ']' : token;\n  }).join(dots ? '.' : '');\n}\n\n/**\n * If the array is an array and none of its elements are visitable, then it's a flat array.\n *\n * @param {Array<any>} arr - The array to check\n *\n * @returns {boolean}\n */\nfunction isFlatArray(arr) {\n  return utils.isArray(arr) && !arr.some(isVisitable);\n}\n\nconst predicates = utils.toFlatObject(utils, {}, null, function filter(prop) {\n  return /^is[A-Z]/.test(prop);\n});\n\n/**\n * Convert a data object to FormData\n *\n * @param {Object} obj\n * @param {?Object} [formData]\n * @param {?Object} [options]\n * @param {Function} [options.visitor]\n * @param {Boolean} [options.metaTokens = true]\n * @param {Boolean} [options.dots = false]\n * @param {?Boolean} [options.indexes = false]\n *\n * @returns {Object}\n **/\n\n/**\n * It converts an object into a FormData object\n *\n * @param {Object<any, any>} obj - The object to convert to form data.\n * @param {string} formData - The FormData object to append to.\n * @param {Object<string, any>} options\n *\n * @returns\n */\nfunction toFormData(obj, formData, options) {\n  if (!utils.isObject(obj)) {\n    throw new TypeError('target must be an object');\n  }\n\n  // eslint-disable-next-line no-param-reassign\n  formData = formData || new (PlatformFormData || FormData)();\n\n  // eslint-disable-next-line no-param-reassign\n  options = utils.toFlatObject(options, {\n    metaTokens: true,\n    dots: false,\n    indexes: false\n  }, false, function defined(option, source) {\n    // eslint-disable-next-line no-eq-null,eqeqeq\n    return !utils.isUndefined(source[option]);\n  });\n\n  const metaTokens = options.metaTokens;\n  // eslint-disable-next-line no-use-before-define\n  const visitor = options.visitor || defaultVisitor;\n  const dots = options.dots;\n  const indexes = options.indexes;\n  const _Blob = options.Blob || typeof Blob !== 'undefined' && Blob;\n  const useBlob = _Blob && utils.isSpecCompliantForm(formData);\n\n  if (!utils.isFunction(visitor)) {\n    throw new TypeError('visitor must be a function');\n  }\n\n  function convertValue(value) {\n    if (value === null) return '';\n\n    if (utils.isDate(value)) {\n      return value.toISOString();\n    }\n\n    if (!useBlob && utils.isBlob(value)) {\n      throw new AxiosError('Blob is not supported. Use a Buffer instead.');\n    }\n\n    if (utils.isArrayBuffer(value) || utils.isTypedArray(value)) {\n      return useBlob && typeof Blob === 'function' ? new Blob([value]) : Buffer.from(value);\n    }\n\n    return value;\n  }\n\n  /**\n   * Default visitor.\n   *\n   * @param {*} value\n   * @param {String|Number} key\n   * @param {Array<String|Number>} path\n   * @this {FormData}\n   *\n   * @returns {boolean} return true to visit the each prop of the value recursively\n   */\n  function defaultVisitor(value, key, path) {\n    let arr = value;\n\n    if (value && !path && typeof value === 'object') {\n      if (utils.endsWith(key, '{}')) {\n        // eslint-disable-next-line no-param-reassign\n        key = metaTokens ? key : key.slice(0, -2);\n        // eslint-disable-next-line no-param-reassign\n        value = JSON.stringify(value);\n      } else if (\n        (utils.isArray(value) && isFlatArray(value)) ||\n        ((utils.isFileList(value) || utils.endsWith(key, '[]')) && (arr = utils.toArray(value))\n        )) {\n        // eslint-disable-next-line no-param-reassign\n        key = removeBrackets(key);\n\n        arr.forEach(function each(el, index) {\n          !(utils.isUndefined(el) || el === null) && formData.append(\n            // eslint-disable-next-line no-nested-ternary\n            indexes === true ? renderKey([key], index, dots) : (indexes === null ? key : key + '[]'),\n            convertValue(el)\n          );\n        });\n        return false;\n      }\n    }\n\n    if (isVisitable(value)) {\n      return true;\n    }\n\n    formData.append(renderKey(path, key, dots), convertValue(value));\n\n    return false;\n  }\n\n  const stack = [];\n\n  const exposedHelpers = Object.assign(predicates, {\n    defaultVisitor,\n    convertValue,\n    isVisitable\n  });\n\n  function build(value, path) {\n    if (utils.isUndefined(value)) return;\n\n    if (stack.indexOf(value) !== -1) {\n      throw Error('Circular reference detected in ' + path.join('.'));\n    }\n\n    stack.push(value);\n\n    utils.forEach(value, function each(el, key) {\n      const result = !(utils.isUndefined(el) || el === null) && visitor.call(\n        formData, el, utils.isString(key) ? key.trim() : key, path, exposedHelpers\n      );\n\n      if (result === true) {\n        build(el, path ? path.concat(key) : [key]);\n      }\n    });\n\n    stack.pop();\n  }\n\n  if (!utils.isObject(obj)) {\n    throw new TypeError('data must be an object');\n  }\n\n  build(obj);\n\n  return formData;\n}\n\nexport default toFormData;\n\n// Path: lib/helpers/AxiosURLSearchParams.js\n'use strict';\n\nimport toFormData from './toFormData.js';\n\n/**\n * It encodes a string by replacing all characters that are not in the unreserved set with\n * their percent-encoded equivalents\n *\n * @param {string} str - The string to encode.\n *\n * @returns {string} The encoded string.\n */\nfunction encode(str) {\n  const charMap = {\n    '!': '%21',\n    \"'\": '%27',\n    '(': '%28',\n    ')': '%29',\n    '~': '%7E',\n    '%20': '+',\n    '%00': '\\x00'\n  };\n  return encodeURIComponent(str).replace(/[!'()~]|%20|%00/g, function replacer(match) {\n    return charMap[match];\n  });\n}\n\n/**\n * It takes a params object and converts it to a FormData object\n *\n * @param {Object<string, any>} params - The parameters to be converted to a FormData object.\n * @param {Object<string, any>} options - The options object passed to the Axios constructor.\n *\n * @returns {void}\n */\nfunction AxiosURLSearchParams(params, options) {\n  this._pairs = [];\n\n  params && toFormData(params, this, options);\n}\n\nconst prototype = AxiosURLSearchParams.prototype;\n\nprototype.append = function append(name, value) {\n  this._pairs.push([name, value]);\n};\n\nprototype.toString = function toString(encoder) {\n  const _encode = encoder ? function(value) {\n    return encoder.call(this, value, encode);\n  } : encode;\n\n  return this._pairs.map(function each(pair) {\n    return _encode(pair[0]) + '=' + _encode(pair[1]);\n  }, '').join('&');\n};\n\nexport default AxiosURLSearchParams;\n\n// Path: lib/helpers/buildURL.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosURLSearchParams from '../helpers/AxiosURLSearchParams.js';\n\n/**\n * It replaces all instances of the characters `:`, `$`, `,`, `+`, `[`, and `]` with their\n * URI encoded counterparts\n *\n * @param {string} val The value to be encoded.\n *\n * @returns {string} The encoded value.\n */\nfunction encode(val) {\n  return encodeURIComponent(val).\n    replace(/%3A/gi, ':').\n    replace(/%24/g, '$').\n    replace(/%2C/gi, ',').\n    replace(/%20/g, '+').\n    replace(/%5B/gi, '[').\n    replace(/%5D/gi, ']');\n}\n\n/**\n * Build a URL by appending params to the end\n *\n * @param {string} url The base of the url (e.g., http://www.google.com)\n * @param {object} [params] The params to be appended\n * @param {?object} options\n *\n * @returns {string} The formatted url\n */\nexport default function buildURL(url, params, options) {\n  /*eslint no-param-reassign:0*/\n  if (!params) {\n    return url;\n  }\n  \n  const _encode = options && options.encode || encode;\n\n  const serializeFn = options && options.serialize;\n\n  let serializedParams;\n\n  if (serializeFn) {\n    serializedParams = serializeFn(params, options);\n  } else {\n    serializedParams = utils.isURLSearchParams(params) ?\n      params.toString() :\n      new AxiosURLSearchParams(params, options).toString(_encode);\n  }\n\n  if (serializedParams) {\n    const hashmarkIndex = url.indexOf(\"#\");\n\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n    url += (url.indexOf('?') === -1 ? '?' : '&') + serializedParams;\n  }\n\n  return url;\n}\n\n// Path: lib/cancel/CanceledError.js\n'use strict';\n\nimport AxiosError from '../core/AxiosError.js';\nimport utils from '../utils.js';\n\n/**\n * A `CanceledError` is an object that is thrown when an operation is canceled.\n *\n * @param {string=} message The message.\n * @param {Object=} config The config.\n * @param {Object=} request The request.\n *\n * @returns {CanceledError} The created error.\n */\nfunction CanceledError(message, config, request) {\n  // eslint-disable-next-line no-eq-null,eqeqeq\n  AxiosError.call(this, message == null ? 'canceled' : message, AxiosError.ERR_CANCELED, config, request);\n  this.name = 'CanceledError';\n}\n\nutils.inherits(CanceledError, AxiosError, {\n  __CANCEL__: true\n});\n\nexport default CanceledError;\n\n// Path: lib/defaults/transitional.js\n'use strict';\n\nexport default {\n  silentJSONParsing: true,\n  forcedJSONParsing: true,\n  clarifyTimeoutError: false\n};\n\n// Path: lib/helpers/speedometer.js\n'use strict';\n\n/**\n * Calculate data maxRate\n * @param {Number} [samplesCount= 10]\n * @param {Number} [min= 1000]\n * @returns {Function}\n */\nfunction speedometer(samplesCount, min) {\n  samplesCount = samplesCount || 10;\n  const bytes = new Array(samplesCount);\n  const timestamps = new Array(samplesCount);\n  let head = 0;\n  let tail = 0;\n  let firstSampleTS;\n\n  min = min !== undefined ? min : 1000;\n\n  return function push(chunkLength) {\n    const now = Date.now();\n\n    const startedAt = timestamps[tail];\n\n    if (!firstSampleTS) {\n      firstSampleTS = now;\n    }\n\n    bytes[head] = chunkLength;\n    timestamps[head] = now;\n\n    let i = tail;\n    let bytesCount = 0;\n\n    while (i !== head) {\n      bytesCount += bytes[i++];\n      i = i % samplesCount;\n    }\n\n    head = (head + 1) % samplesCount;\n\n    if (head === tail) {\n      tail = (tail + 1) % samplesCount;\n    }\n\n    if (now - firstSampleTS < min) {\n      return;\n    }\n\n    const passed = startedAt && now - startedAt;\n\n    return passed ? Math.round(bytesCount * 1000 / passed) : undefined;\n  };\n}\n\nexport default speedometer;\n\n// Path: lib/helpers/parseHeaders.js\n'use strict';\n\nimport utils from './../utils.js';\n\n// RawAxiosHeaders whose duplicates are ignored by node\n// c.f. https://nodejs.org/api/http.html#http_message_headers\nconst ignoreDuplicateOf = utils.toObjectSet([\n  'age', 'authorization', 'content-length', 'content-type', 'etag',\n  'expires', 'from', 'host', 'if-modified-since', 'if-unmodified-since',\n  'last-modified', 'location', 'max-forwards', 'proxy-authorization',\n  'referer', 'retry-after', 'user-agent'\n]);\n\n/**\n * Parse headers into an object\n *\n * ```\n * Date: Wed, 27 Aug 2014 08:58:49 GMT\n * Content-Type: application/json\n * Connection: keep-alive\n * Transfer-Encoding: chunked\n * ```\n *\n * @param {String} rawHeaders Headers needing to be parsed\n *\n * @returns {Object} Headers parsed into an object\n */\nexport default rawHeaders => {\n  const parsed = {};\n  let key;\n  let val;\n  let i;\n\n  rawHeaders && rawHeaders.split('\\n').forEach(function parser(line) {\n    i = line.indexOf(':');\n    key = line.substring(0, i).trim().toLowerCase();\n    val = line.substring(i + 1).trim();\n\n    if (!key || (parsed[key] && ignoreDuplicateOf[key])) {\n      return;\n    }\n\n    if (key === 'set-cookie') {\n      if (parsed[key]) {\n        parsed[key].push(val);\n      } else {\n        parsed[key] = [val];\n      }\n    } else {\n      parsed[key] = parsed[key] ? parsed[key] + ', ' + val : val;\n    }\n  });\n\n  return parsed;\n};\n\n// Path: lib/core/AxiosHeaders.js\n'use strict';\n\nimport utils from '../utils.js';\nimport parseHeaders from '../helpers/parseHeaders.js';\n\nconst $internals = Symbol('internals');\n\nfunction normalizeHeader(header) {\n  return header && String(header).trim().toLowerCase();\n}\n\nfunction normalizeValue(value) {\n  if (value === false || value == null) {\n    return value;\n  }\n\n  return utils.isArray(value) ? value.map(normalizeValue) : String(value);\n}\n\nfunction parseTokens(str) {\n  const tokens = Object.create(null);\n  const tokensRE = /([^\\s,;=]+)\\s*(?:=\\s*([^,;]+))?/g;\n  let match;\n\n  while ((match = tokensRE.exec(str))) {\n    tokens[match[1]] = match[2];\n  }\n\n  return tokens;\n}\n\nconst isValidHeaderName = (str) => /^[-_a-zA-Z0-9^`|~,!#$%&'*+.]+$/.test(str.trim());\n\nfunction matchHeaderValue(context, value, header, filter, isHeaderNameFilter) {\n  if (utils.isFunction(filter)) {\n    return filter.call(this, value, header);\n  }\n\n  if (isHeaderNameFilter) {\n    value = header;\n  }\n\n  if (!utils.isString(value)) return;\n\n  if (utils.isString(filter)) {\n    return value.indexOf(filter) !== -1;\n  }\n\n  if (utils.isRegExp(filter)) {\n    return filter.test(value);\n  }\n}\n\nfunction formatHeader(header) {\n  return header.trim()\n    .toLowerCase().replace(/([a-z\\d])(\\w*)/g, (w, char, str) => {\n      return char.toUpperCase() + str;\n    });\n}\n\nfunction buildAccessors(obj, header) {\n  const accessorName = utils.toCamelCase(' ' + header);\n\n  ['get', 'set', 'has'].forEach(methodName => {\n    Object.defineProperty(obj, methodName + accessorName, {\n      value: function(arg1, arg2, arg3) {\n        return this[methodName].call(this, header, arg1, arg2, arg3);\n      },\n      configurable: true\n    });\n  });\n}\n\nclass AxiosHeaders {\n  constructor(headers) {\n    headers && this.set(headers);\n  }\n\n  set(header, valueOrRewrite, rewrite) {\n    const self = this;\n\n    function setHeader(_value, _header, _rewrite) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!lHeader) {\n        throw new Error('header name must be a non-empty string');\n      }\n\n      const key = utils.findKey(self, lHeader);\n\n      if(!key || self[key] === undefined || _rewrite === true || (_rewrite === undefined && self[key] !== false)) {\n        self[key || _header] = normalizeValue(_value);\n      }\n    }\n\n    const setHeaders = (headers, _rewrite) =>\n      utils.forEach(headers, (_value, _header) => setHeader(_value, _header, _rewrite));\n\n    if (utils.isPlainObject(header) || header instanceof this.constructor) {\n      setHeaders(header, valueOrRewrite)\n    } else if(utils.isString(header) && (header = header.trim()) && !isValidHeaderName(header)) {\n      setHeaders(parseHeaders(header), valueOrRewrite);\n    } else {\n      header != null && setHeader(valueOrRewrite, header, rewrite);\n    }\n\n    return this;\n  }\n\n  get(header, parser) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      if (key) {\n        const value = this[key];\n\n        if (!parser) {\n          return value;\n        }\n\n        if (parser === true) {\n          return parseTokens(value);\n        }\n\n        if (utils.isFunction(parser)) {\n          return parser.call(this, value, key);\n        }\n\n        if (utils.isRegExp(parser)) {\n          return parser.exec(value);\n        }\n\n        throw new TypeError('parser must be boolean|regexp|function');\n      }\n    }\n  }\n\n  has(header, matcher) {\n    header = normalizeHeader(header);\n\n    if (header) {\n      const key = utils.findKey(this, header);\n\n      return !!(key && this[key] !== undefined && (!matcher || matchHeaderValue(this, this[key], key, matcher)));\n    }\n\n    return false;\n  }\n\n  delete(header, matcher) {\n    const self = this;\n    let deleted = false;\n\n    \nfunction deleteHeader(_header) {\n      _header = normalizeHeader(_header);\n\n      if (_header) {\n        const key = utils.findKey(self, _header);\n\n        if (key && (!matcher || matchHeaderValue(self, self[key], key, matcher))) {\n          delete self[key];\n\n          deleted = true;\n        }\n      }\n    }\n\n    if (utils.isArray(header)) {\n      header.forEach(deleteHeader);\n    } else {\n      deleteHeader(header);\n    }\n\n    return deleted;\n  }\n\n  clear(matcher) {\n    const keys = Object.keys(this);\n    let i = keys.length;\n    let deleted = false;\n\n    while (i--) {\n      const key = keys[i];\n      if(!matcher || matchHeaderValue(this, this[key], key, matcher, true)) {\n        delete this[key];\n        deleted = true;\n      }\n    }\n\n    return deleted;\n  }\n\n  normalize(format) {\n    const self = this;\n    const headers = {};\n\n    utils.forEach(this, (value, header) => {\n      const key = utils.findKey(headers, header);\n\n      if (key) {\n        self[key] = normalizeValue(value);\n        delete self[header];\n        return;\n      }\n\n      const normalized = format ? formatHeader(header) : String(header).trim();\n\n      if (normalized !== header) {\n        delete self[header];\n      }\n\n      self[normalized] = normalizeValue(value);\n\n      headers[normalized] = true;\n    });\n\n    return this;\n  }\n\n  concat(...targets) {\n    return this.constructor.concat(this, ...targets);\n  }\n\n  toJSON(asStrings) {\n    const obj = Object.create(null);\n\n    utils.forEach(this, (value, header) => {\n      value != null && value !== false && (obj[header] = asStrings && utils.isArray(value) ? value.join(', ') : value);\n    });\n\n    return obj;\n  }\n\n  [Symbol.iterator]() {\n    return Object.entries(this.toJSON())[Symbol.iterator]();\n  }\n\n  toString() {\n    return Object.entries(this.toJSON()).map(([header, value]) => header + ': ' + value).join('\\n');\n  }\n\n  get [Symbol.toStringTag]() {\n    return 'AxiosHeaders';\n  }\n\n  static from(thing) {\n    return thing instanceof this ? thing : new this(thing);\n  }\n\n  static concat(first, ...targets) {\n    const computed = new this(first);\n\n    targets.forEach((target) => computed.set(target));\n\n    return computed;\n  }\n\n  static accessor(header) {\n    const internals = this[$internals] = (this[$internals] = {\n      accessors: {}\n    });\n\n    const accessors = internals.accessors;\n    const prototype = this.prototype;\n\n    function defineAccessor(_header) {\n      const lHeader = normalizeHeader(_header);\n\n      if (!accessors[lHeader]) {\n        buildAccessors(prototype, _header);\n        accessors[lHeader] = true;\n      }\n    }\n\n    utils.isArray(header) ? header.forEach(defineAccessor) : defineAccessor(header);\n\n    return this;\n  }\n}\n\nAxiosHeaders.accessor(['Content-Type', 'Content-Length', 'Accept', 'Accept-Encoding', 'User-Agent', 'Authorization']);\n\n// reserved names hotfix\nutils.reduceDescriptors(AxiosHeaders.prototype, ({value}, key) => {\n  let mapped = key[0].toUpperCase() + key.slice(1); // map `set` => `Set`\n  return {\n    get: () => value,\n    set(headerValue) {\n      this[mapped] = headerValue;\n    }\n  }\n});\n\nutils.freezeMethods(AxiosHeaders);\n\nexport default AxiosHeaders;\n\n// Path: lib/core/settle.js\n'use strict';\n\nimport AxiosError from './AxiosError.js';\n\n/**\n * Resolve or reject a Promise based on response status.\n *\n * @param {Function} resolve A function that resolves the promise.\n * @param {Function} reject A function that rejects the promise.\n * @param {object} response The response.\n *\n * @returns {object} The response.\n */\nexport default function settle(resolve, reject, response) {\n  const validateStatus = response.config.validateStatus;\n  if (!response.status || !validateStatus || validateStatus(response.status)) {\n    resolve(response);\n  } else {\n    reject(new AxiosError(\n      'Request failed with status code ' + response.status,\n      [AxiosError.ERR_BAD_REQUEST, AxiosError.ERR_BAD_RESPONSE][Math.floor(response.status / 100) - 4],\n      response.config,\n      response.request,\n      response\n    ));\n  }\n}\n\n// Path: lib/helpers/parseProtocol.js\n'use strict';\n\nexport default function parseProtocol(url) {\n  const match = /^([-+\\w]{1,25})(:?\\/\\/|:)/.exec(url);\n  return match && match[1] || '';\n}\n\n// Path: lib/helpers/cookies.js\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n  // Standard browser envs support document.cookie\n  {\n    write(name, value, expires, path, domain, secure) {\n      const cookie = [name + '=' + encodeURIComponent(value)];\n\n      utils.isNumber(expires) && cookie.push('expires=' + new Date(expires).toGMTString());\n\n      utils.isString(path) && cookie.push('path=' + path);\n\n      utils.isString(domain) && cookie.push('domain=' + domain);\n\n      secure === true && cookie.push('secure');\n\n      document.cookie = cookie.join('; ');\n    },\n\n    read(name) {\n      const match = document.cookie.match(new RegExp('(^|;\\\\s*)(' + name + ')=([^;]*)'));\n      return (match ? decodeURIComponent(match[3]) : null);\n    },\n\n    remove(name) {\n      this.write(name, '', Date.now() - 86400000);\n    }\n  }\n\n  :\n\n  // Non-standard browser env (web workers, react-native) lack needed support.\n  {\n    write() {},\n    read() {\n      return null;\n    },\n    remove() {}\n  };\n\n\n// Path: lib/helpers/isURLSameOrigin.js\n'use strict';\n\nimport utils from './../utils.js';\nimport platform from '../platform/index.js';\n\nexport default platform.hasStandardBrowserEnv ?\n\n// Standard browser envs have full support of the APIs needed to test\n// whether the request URL is of the same origin as current location.\n  (function standardBrowserEnv() {\n    const msie = /(msie|trident)/i.test(navigator.userAgent);\n    const urlParsingNode = document.createElement('a');\n    let originURL;\n\n    /**\n    * Parse a URL to discover its components\n    *\n    * @param {String} url The URL to be parsed\n    * @returns {Object}\n    */\n    function resolveURL(url) {\n      let href = url;\n\n      if (msie) {\n        // IE needs attribute set twice to normalize properties\n        urlParsingNode.setAttribute('href', href);\n        href = urlParsingNode.href;\n      }\n\n      urlParsingNode.setAttribute('href', href);\n\n      // urlParsingNode provides the UrlUtils interface - http://url.spec.whatwg.org/#urlutils\n      return {\n        href: urlParsingNode.href,\n        protocol: urlParsingNode.protocol ? urlParsingNode.protocol.replace(/:$/, '') : '',\n        host: urlParsingNode.host,\n        search: urlParsingNode.search ? urlParsingNode.search.replace(/^\\?/, '') : '',\n        hash: urlParsingNode.hash ? urlParsingNode.hash.replace(/^#/, '') : '',\n        hostname: urlParsingNode.hostname,\n        port: urlParsingNode.port,\n        pathname: (urlParsingNode.pathname.charAt(0) === '/') ?\n          urlParsingNode.pathname :\n          '/' + urlParsingNode.pathname\n      };\n    }\n\n    originURL = resolveURL(window.location.href);\n\n    /**\n    * Determine if a URL shares the same origin as the current location\n    *\n    * @param {String} requestURL The URL to test\n    * @returns {boolean} True if URL shares the same origin, otherwise false\n    */\n    return function isURLSameOrigin(requestURL) {\n      const parsed = (utils.isString(requestURL)) ? resolveURL(requestURL) : requestURL;\n      return (parsed.protocol === originURL.protocol &&\n          parsed.host === originURL.host);\n    };\n  })() :\n\n  // Non standard browser envs (web workers, react-native) lack needed support.\n  (function nonStandardBrowserEnv() {\n    return function isURLSameOrigin() {\n      return true;\n    };\n  })();\n\n// Path: lib/adapters/xhr.js\n'use strict';\n\nimport utils from './../utils.js';\n...\n", "needle_token_start": 13229, "needle_token_end": 13321, "code_context_ntokens": 15650, "output": ["```javascript\nfunction deleteHeader(_header) {\n  _header = normalizeHeader(_header);\n\n  if (_header) {\n    const key = utils.findKey(self, _header);\n\n    if (key && (!matcher || matchHeaderValue(self, self[key], key, matcher))) {\n      delete self[key];\n\n      deleted = true;\n    }\n  }\n}\n```"]}
{"repo": "axios/axios", "name": "isAxiosError", "language": "typescript", "path": "lib/helpers/isAxiosError.js", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to check if a given value is an error specifically generated by a popular HTTP client library.\n2. **Input**: The function takes a single parameter, which can be any type of value.\n3. **Output**: It returns a boolean value; `true` if the input is an error created by the HTTP client library, otherwise `false`.\n4. **Procedure**: The function first checks if the input is an object. If it is, it then checks if this object has a specific property set to `true` that identifies it as an error from the HTTP client library. If both conditions are met, it returns `true`; otherwise, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: lib/adapters/http.js\n'use strict';\n\nimport utils from './../utils.js';\nimport settle from './../core/settle.js';\nimport buildFullPath from '../core/buildFullPath.js';\nimport buildURL from './../helpers/buildURL.js';\nimport {getProxyForUrl} from 'proxy-from-env';\nimport http from 'http';\nimport https from 'https';\nimport util from 'util';\nimport followRedirects from 'follow-redirects';\nimport zlib from 'zlib';\nimport {VERSION} from '../env/data.js';\nimport transitionalDefaults from '../defaults/transitional.js';\nimport AxiosError from '../core/AxiosError.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport platform from '../platform/index.js';\nimport fromDataURI from '../helpers/fromDataURI.js';\nimport stream from 'stream';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport AxiosTransformStream from '../helpers/AxiosTransformStream.js';\nimport {EventEmitter} from 'events';\nimport formDataToStream from \"../helpers/formDataToStream.js\";\nimport readBlob from \"../helpers/readBlob.js\";\nimport ZlibHeaderTransformStream from '../helpers/ZlibHeaderTransformStream.js';\nimport callbackify from \"../helpers/callbackify.js\";\n\nconst zlibOptions = {\n  flush: zlib.constants.Z_SYNC_FLUSH,\n  finishFlush: zlib.constants.Z_SYNC_FLUSH\n};\n\nconst brotliOptions = {\n  flush: zlib.constants.BROTLI_OPERATION_FLUSH,\n  finishFlush: zlib.constants.BROTLI_OPERATION_FLUSH\n}\n\nconst isBrotliSupported = utils.isFunction(zlib.createBrotliDecompress);\n\nconst {http: httpFollow, https: httpsFollow} = followRedirects;\n\nconst isHttps = /https:?/;\n\nconst supportedProtocols = platform.protocols.map(protocol => {\n  return protocol + ':';\n});\n\n/**\n * If the proxy or config beforeRedirects functions are defined, call them with the options\n * object.\n *\n * @param {Object<string, any>} options - The options object that was passed to the request.\n *\n * @returns {Object<string, any>}\n */\nfunction dispatchBeforeRedirect(options, responseDetails) {\n  if (options.beforeRedirects.proxy) {\n    options.beforeRedirects.proxy(options);\n  }\n  if (options.beforeRedirects.config) {\n    options.beforeRedirects.config(options, responseDetails);\n  }\n}\n\n/**\n * If the proxy or config afterRedirects functions are defined, call them with the options\n *\n * @param {http.ClientRequestArgs} options\n * @param {AxiosProxyConfig} configProxy configuration from Axios options object\n * @param {string} location\n *\n * @returns {http.ClientRequestArgs}\n */\nfunction setProxy(options, configProxy, location) {\n  let proxy = configProxy;\n  if (!proxy && proxy !== false) {\n    const proxyUrl = getProxyForUrl(location);\n    if (proxyUrl) {\n      proxy = new URL(proxyUrl);\n    }\n  }\n  if (proxy) {\n    // Basic proxy authorization\n    if (proxy.username) {\n      proxy.auth = (proxy.username || '') + ':' + (proxy.password || '');\n    }\n\n    if (proxy.auth) {\n      // Support proxy auth object form\n      if (proxy.auth.username || proxy.auth.password) {\n        proxy.auth = (proxy.auth.username || '') + ':' + (proxy.auth.password || '');\n      }\n      const base64 = Buffer\n        .from(proxy.auth, 'utf8')\n        .toString('base64');\n      options.headers['Proxy-Authorization'] = 'Basic ' + base64;\n    }\n\n    options.headers.host = options.hostname + (options.port ? ':' + options.port : '');\n    const proxyHost = proxy.hostname || proxy.host;\n    options.hostname = proxyHost;\n    // Replace 'host' since options is not a URL object\n    options.host = proxyHost;\n    options.port = proxy.port;\n    options.path = location;\n    if (proxy.protocol) {\n      options.protocol = proxy.protocol.includes(':') ? proxy.protocol : `${proxy.protocol}:`;\n    }\n  }\n\n  options.beforeRedirects.proxy = function beforeRedirect(redirectOptions) {\n    // Configure proxy for redirected request, passing the original config proxy to apply\n    // the exact same logic as if the redirected request was performed by axios directly.\n    setProxy(redirectOptions, configProxy, redirectOptions.href);\n  };\n}\n\nconst isHttpAdapterSupported = typeof process !== 'undefined' && utils.kindOf(process) === 'process';\n\n// temporary hotfix\n\nconst wrapAsync = (asyncExecutor) => {\n  return new Promise((resolve, reject) => {\n    let onDone;\n    let isDone;\n\n    const done = (value, isRejected) => {\n      if (isDone) return;\n      isDone = true;\n      onDone && onDone(value, isRejected);\n    }\n\n    const _resolve = (value) => {\n      done(value);\n      resolve(value);\n    };\n\n    const _reject = (reason) => {\n      done(reason, true);\n      reject(reason);\n    }\n\n    asyncExecutor(_resolve, _reject, (onDoneHandler) => (onDone = onDoneHandler)).catch(_reject);\n  })\n};\n\nconst resolveFamily = ({address, family}) => {\n  if (!utils.isString(address)) {\n    throw TypeError('address must be a string');\n  }\n  return ({\n    address,\n    family: family || (address.indexOf('.') < 0 ? 6 : 4)\n  });\n}\n\nconst buildAddressEntry = (address, family) => resolveFamily(utils.isObject(address) ? address : {address, family});\n\n/*eslint consistent-return:0*/\nexport default isHttpAdapterSupported && function httpAdapter(config) {\n  return wrapAsync(async function dispatchHttpRequest(resolve, reject, onDone) {\n    let {data, lookup, family} = config;\n    const {responseType, responseEncoding} = config;\n    const method = config.method.toUpperCase();\n    let isDone;\n    let rejected = false;\n    let req;\n\n    if (lookup) {\n      const _lookup = callbackify(lookup, (value) => utils.isArray(value) ? value : [value]);\n      // hotfix to support opt.all option which is required for node 20.x\n      lookup = (hostname, opt, cb) => {\n        _lookup(hostname, opt, (err, arg0, arg1) => {\n          if (err) {\n            return cb(err);\n          }\n\n          const addresses = utils.isArray(arg0) ? arg0.map(addr => buildAddressEntry(addr)) : [buildAddressEntry(arg0, arg1)];\n\n          opt.all ? cb(err, addresses) : cb(err, addresses[0].address, addresses[0].family);\n        });\n      }\n    }\n\n    // temporary internal emitter until the AxiosRequest class will be implemented\n    const emitter = new EventEmitter();\n\n    const onFinished = () => {\n      if (config.cancelToken) {\n        config.cancelToken.unsubscribe(abort);\n      }\n\n      if (config.signal) {\n        config.signal.removeEventListener('abort', abort);\n      }\n\n      emitter.removeAllListeners();\n    }\n\n    onDone((value, isRejected) => {\n      isDone = true;\n      if (isRejected) {\n        rejected = true;\n        onFinished();\n      }\n    });\n\n    function abort(reason) {\n      emitter.emit('abort', !reason || reason.type ? new CanceledError(null, config, req) : reason);\n    }\n\n    emitter.once('abort', reject);\n\n    if (config.cancelToken || config.signal) {\n      config.cancelToken && config.cancelToken.subscribe(abort);\n      if (config.signal) {\n        config.signal.aborted ? abort() : config.signal.addEventListener('abort', abort);\n      }\n    }\n\n    // Parse url\n    const fullPath = buildFullPath(config.baseURL, config.url);\n    const parsed = new URL(fullPath, 'http://localhost');\n    const protocol = parsed.protocol || supportedProtocols[0];\n\n    if (protocol === 'data:') {\n      let convertedData;\n\n      if (method !== 'GET') {\n        return settle(resolve, reject, {\n          status: 405,\n          statusText: 'method not allowed',\n          headers: {},\n          config\n        });\n      }\n\n      try {\n        convertedData = fromDataURI(config.url, responseType === 'blob', {\n          Blob: config.env && config.env.Blob\n        });\n      } catch (err) {\n        throw AxiosError.from(err, AxiosError.ERR_BAD_REQUEST, config);\n      }\n\n      if (responseType === 'text') {\n        convertedData = convertedData.toString(responseEncoding);\n\n        if (!responseEncoding || responseEncoding === 'utf8') {\n          convertedData = utils.stripBOM(convertedData);\n        }\n      } else if (responseType === 'stream') {\n        convertedData = stream.Readable.from(convertedData);\n      }\n\n      return settle(resolve, reject, {\n        data: convertedData,\n        status: 200,\n        statusText: 'OK',\n        headers: new AxiosHeaders(),\n        config\n      });\n    }\n\n    if (supportedProtocols.indexOf(protocol) === -1) {\n      return reject(new AxiosError(\n        'Unsupported protocol ' + protocol,\n        AxiosError.ERR_BAD_REQUEST,\n        config\n      ));\n    }\n\n    const headers = AxiosHeaders.from(config.headers).normalize();\n\n    // Set User-Agent (required by some servers)\n    // See https://github.com/axios/axios/issues/69\n    // User-Agent is specified; handle case where no UA header is desired\n    // Only set header if it hasn't been set in config\n    headers.set('User-Agent', 'axios/' + VERSION, false);\n\n    const onDownloadProgress = config.onDownloadProgress;\n    const onUploadProgress = config.onUploadProgress;\n    const maxRate = config.maxRate;\n    let maxUploadRate = undefined;\n    let maxDownloadRate = undefined;\n\n    // support for spec compliant FormData objects\n    if (utils.isSpecCompliantForm(data)) {\n      const userBoundary = headers.getContentType(/boundary=([-_\\w\\d]{10,70})/i);\n\n      data = formDataToStream(data, (formHeaders) => {\n        headers.set(formHeaders);\n      }, {\n        tag: `axios-${VERSION}-boundary`,\n        boundary: userBoundary && userBoundary[1] || undefined\n      });\n      // support for https://www.npmjs.com/package/form-data api\n    } else if (utils.isFormData(data) && utils.isFunction(data.getHeaders)) {\n      headers.set(data.getHeaders());\n\n      if (!headers.hasContentLength()) {\n        try {\n          const knownLength = await util.promisify(data.getLength).call(data);\n          Number.isFinite(knownLength) && knownLength >= 0 && headers.setContentLength(knownLength);\n          /*eslint no-empty:0*/\n        } catch (e) {\n        }\n      }\n    } else if (utils.isBlob(data)) {\n      data.size && headers.setContentType(data.type || 'application/octet-stream');\n      headers.setContentLength(data.size || 0);\n      data = stream.Readable.from(readBlob(data));\n    } else if (data && !utils.isStream(data)) {\n      if (Buffer.isBuffer(data)) {\n        // Nothing to do...\n      } else if (utils.isArrayBuffer(data)) {\n        data = Buffer.from(new Uint8Array(data));\n      } else if (utils.isString(data)) {\n        data = Buffer.from(data, 'utf-8');\n      } else {\n        return reject(new AxiosError(\n          'Data after transformation must be a string, an ArrayBuffer, a Buffer, or a Stream',\n          AxiosError.ERR_BAD_REQUEST,\n          config\n        ));\n      }\n\n      // Add Content-Length header if data exists\n      headers.setContentLength(data.length, false);\n\n      if (config.maxBodyLength > -1 && data.length > config.maxBodyLength) {\n        return reject(new AxiosError(\n          'Request body larger than maxBodyLength limit',\n          AxiosError.ERR_BAD_REQUEST,\n          config\n        ));\n      }\n    }\n\n    const contentLength = utils.toFiniteNumber(headers.getContentLength());\n\n    if (utils.isArray(maxRate)) {\n      maxUploadRate = maxRate[0];\n      maxDownloadRate = maxRate[1];\n    } else {\n      maxUploadRate = maxDownloadRate = maxRate;\n    }\n\n    if (data && (onUploadProgress || maxUploadRate)) {\n      if (!utils.isStream(data)) {\n        data = stream.Readable.from(data, {objectMode: false});\n      }\n\n      data = stream.pipeline([data, new AxiosTransformStream({\n        length: contentLength,\n        maxRate: utils.toFiniteNumber(maxUploadRate)\n      })], utils.noop);\n\n      onUploadProgress && data.on('progress', progress => {\n        onUploadProgress(Object.assign(progress, {\n          upload: true\n        }));\n      });\n    }\n\n    // HTTP basic authentication\n    let auth = undefined;\n    if (config.auth) {\n      const username = config.auth.username || '';\n      const password = config.auth.password || '';\n      auth = username + ':' + password;\n    }\n\n    if (!auth && parsed.username) {\n      const urlUsername = parsed.username;\n      const urlPassword = parsed.password;\n      auth = urlUsername + ':' + urlPassword;\n    }\n\n    auth && headers.delete('authorization');\n\n    let path;\n\n    try {\n      path = buildURL(\n        parsed.pathname + parsed.search,\n        config.params,\n        config.paramsSerializer\n      ).replace(/^\\?/, '');\n    } catch (err) {\n      const customErr = new Error(err.message);\n      customErr.config = config;\n      customErr.url = config.url;\n      customErr.exists = true;\n      return reject(customErr);\n    }\n\n    headers.set(\n      'Accept-Encoding',\n      'gzip, compress, deflate' + (isBrotliSupported ? ', br' : ''), false\n      );\n\n    const options = {\n      path,\n      method: method,\n      headers: headers.toJSON(),\n      agents: { http: config.httpAgent, https: config.httpsAgent },\n      auth,\n      protocol,\n      family,\n      beforeRedirect: dispatchBeforeRedirect,\n      beforeRedirects: {}\n    };\n\n    // cacheable-lookup integration hotfix\n    !utils.isUndefined(lookup) && (options.lookup = lookup);\n\n    if (config.socketPath) {\n      options.socketPath = config.socketPath;\n    } else {\n      options.hostname = parsed.hostname;\n      options.port = parsed.port;\n      setProxy(options, config.proxy, protocol + '//' + parsed.hostname + (parsed.port ? ':' + parsed.port : '') + options.path);\n    }\n\n    let transport;\n    const isHttpsRequest = isHttps.test(options.protocol);\n    options.agent = isHttpsRequest ? config.httpsAgent : config.httpAgent;\n    if (config.transport) {\n      transport = config.transport;\n    } else if (config.maxRedirects === 0) {\n      transport = isHttpsRequest ? https : http;\n    } else {\n      if (config.maxRedirects) {\n        options.maxRedirects = config.maxRedirects;\n      }\n      if (config.beforeRedirect) {\n        options.beforeRedirects.config = config.beforeRedirect;\n      }\n      transport = isHttpsRequest ? httpsFollow : httpFollow;\n    }\n\n    if (config.maxBodyLength > -1) {\n      options.maxBodyLength = config.maxBodyLength;\n    } else {\n      // follow-redirects does not skip comparison, so it should always succeed for axios -1 unlimited\n      options.maxBodyLength = Infinity;\n    }\n\n    if (config.insecureHTTPParser) {\n      options.insecureHTTPParser = config.insecureHTTPParser;\n    }\n\n    // Create the request\n    req = transport.request(options, function handleResponse(res) {\n      if (req.destroyed) return;\n\n      const streams = [res];\n\n      const responseLength = +res.headers['content-length'];\n\n      if (onDownloadProgress) {\n        const transformStream = new AxiosTransformStream({\n          length: utils.toFiniteNumber(responseLength),\n          maxRate: utils.toFiniteNumber(maxDownloadRate)\n        });\n\n        onDownloadProgress && transformStream.on('progress', progress => {\n          onDownloadProgress(Object.assign(progress, {\n            download: true\n          }));\n        });\n\n        streams.push(transformStream);\n      }\n\n      // decompress the response body transparently if required\n      let responseStream = res;\n\n      // return the last request in case of redirects\n      const lastRequest = res.req || req;\n\n      // if decompress disabled we should not decompress\n      if (config.decompress !== false && res.headers['content-encoding']) {\n        // if no content, but headers still say that it is encoded,\n        // remove the header not confuse downstream operations\n        if (method === 'HEAD' || res.statusCode === 204) {\n          delete res.headers['content-encoding'];\n        }\n\n        switch ((res.headers['content-encoding'] || '').toLowerCase()) {\n        /*eslint default-case:0*/\n        case 'gzip':\n        case 'x-gzip':\n        case 'compress':\n        case 'x-compress':\n          // add the unzipper to the body stream processing pipeline\n          streams.push(zlib.createUnzip(zlibOptions));\n\n          // remove the content-encoding in order to not confuse downstream operations\n          delete res.headers['content-encoding'];\n          break;\n        case 'deflate':\n          streams.push(new ZlibHeaderTransformStream());\n\n          // add the unzipper to the body stream processing pipeline\n          streams.push(zlib.createUnzip(zlibOptions));\n\n          // remove the content-encoding in order to not confuse downstream operations\n          delete res.headers['content-encoding'];\n          break;\n        case 'br':\n          if (isBrotliSupported) {\n            streams.push(zlib.createBrotliDecompress(brotliOptions));\n            delete res.headers['content-encoding'];\n          }\n        }\n      }\n\n      responseStream = streams.length > 1 ? stream.pipeline(streams, utils.noop) : streams[0];\n\n      const offListeners = stream.finished(responseStream, () => {\n        offListeners();\n        onFinished();\n      });\n\n      const response = {\n        status: res.statusCode,\n        statusText: res.statusMessage,\n        headers: new AxiosHeaders(res.headers),\n        config,\n        request: lastRequest\n      };\n\n      if (responseType === 'stream') {\n        response.data = responseStream;\n        settle(resolve, reject, response);\n      } else {\n        const responseBuffer = [];\n        let totalResponseBytes = 0;\n\n        responseStream.on('data', function handleStreamData(chunk) {\n          responseBuffer.push(chunk);\n          totalResponseBytes += chunk.length;\n\n          // make sure the content length is not over the maxContentLength if specified\n...\n// Path: lib/adapters/adapters.js\nimport utils from '../utils.js';\nimport httpAdapter from './http.js';\nimport xhrAdapter from './xhr.js';\nimport AxiosError from \"../core/AxiosError.js\";\n\nconst knownAdapters = {\n  http: httpAdapter,\n  xhr: xhrAdapter\n}\n\nutils.forEach(knownAdapters, (fn, value) => {\n  if (fn) {\n    try {\n      Object.defineProperty(fn, 'name', {value});\n    } catch (e) {\n      // eslint-disable-next-line no-empty\n    }\n    Object.defineProperty(fn, 'adapterName', {value});\n  }\n});\n\nconst renderReason = (reason) => `- ${reason}`;\n\nconst isResolvedHandle = (adapter) => utils.isFunction(adapter) || adapter === null || adapter === false;\n\nexport default {\n  getAdapter: (adapters) => {\n    adapters = utils.isArray(adapters) ? adapters : [adapters];\n\n    const {length} = adapters;\n    let nameOrAdapter;\n    let adapter;\n\n    const rejectedReasons = {};\n\n    for (let i = 0; i < length; i++) {\n      nameOrAdapter = adapters[i];\n      let id;\n\n      adapter = nameOrAdapter;\n\n      if (!isResolvedHandle(nameOrAdapter)) {\n        adapter = knownAdapters[(id = String(nameOrAdapter)).toLowerCase()];\n\n        if (adapter === undefined) {\n          throw new AxiosError(`Unknown adapter '${id}'`);\n        }\n      }\n\n      if (adapter) {\n        break;\n      }\n\n      rejectedReasons[id || '#' + i] = adapter;\n    }\n\n    if (!adapter) {\n\n      const reasons = Object.entries(rejectedReasons)\n        .map(([id, state]) => `adapter ${id} ` +\n          (state === false ? 'is not supported by the environment' : 'is not available in the build')\n        );\n\n      let s = length ?\n        (reasons.length > 1 ? 'since :\\n' + reasons.map(renderReason).join('\\n') : ' ' + renderReason(reasons[0])) :\n        'as no adapter specified';\n\n      throw new AxiosError(\n        `There is no suitable adapter to dispatch the request ` + s,\n        'ERR_NOT_SUPPORT'\n      );\n    }\n\n    return adapter;\n  },\n  adapters: knownAdapters\n}\n\n// Path: lib/helpers/toURLEncodedForm.js\n'use strict';\n\nimport utils from '../utils.js';\nimport toFormData from './toFormData.js';\nimport platform from '../platform/index.js';\n\nexport default function toURLEncodedForm(data, options) {\n  return toFormData(data, new platform.classes.URLSearchParams(), Object.assign({\n    visitor: function(value, key, path, helpers) {\n      if (platform.isNode && utils.isBuffer(value)) {\n        this.append(key, value.toString('base64'));\n        return false;\n      }\n\n      return helpers.defaultVisitor.apply(this, arguments);\n    }\n  }, options));\n}\n\n// Path: lib/helpers/formDataToJSON.js\n'use strict';\n\nimport utils from '../utils.js';\n\n/**\n * It takes a string like `foo[x][y][z]` and returns an array like `['foo', 'x', 'y', 'z']\n *\n * @param {string} name - The name of the property to get.\n *\n * @returns An array of strings.\n */\nfunction parsePropPath(name) {\n  // foo[x][y][z]\n  // foo.x.y.z\n  // foo-x-y-z\n  // foo x y z\n  return utils.matchAll(/\\w+|\\[(\\w*)]/g, name).map(match => {\n    return match[0] === '[]' ? '' : match[1] || match[0];\n  });\n}\n\n/**\n * Convert an array to an object.\n *\n * @param {Array<any>} arr - The array to convert to an object.\n *\n * @returns An object with the same keys and values as the array.\n */\nfunction arrayToObject(arr) {\n  const obj = {};\n  const keys = Object.keys(arr);\n  let i;\n  const len = keys.length;\n  let key;\n  for (i = 0; i < len; i++) {\n    key = keys[i];\n    obj[key] = arr[key];\n  }\n  return obj;\n}\n\n/**\n * It takes a FormData object and returns a JavaScript object\n *\n * @param {string} formData The FormData object to convert to JSON.\n *\n * @returns {Object<string, any> | null} The converted object.\n */\nfunction formDataToJSON(formData) {\n  function buildPath(path, value, target, index) {\n    let name = path[index++];\n\n    if (name === '__proto__') return true;\n\n    const isNumericKey = Number.isFinite(+name);\n    const isLast = index >= path.length;\n    name = !name && utils.isArray(target) ? target.length : name;\n\n    if (isLast) {\n      if (utils.hasOwnProp(target, name)) {\n        target[name] = [target[name], value];\n      } else {\n        target[name] = value;\n      }\n\n      return !isNumericKey;\n    }\n\n    if (!target[name] || !utils.isObject(target[name])) {\n      target[name] = [];\n    }\n\n    const result = buildPath(path, value, target[name], index);\n\n    if (result && utils.isArray(target[name])) {\n      target[name] = arrayToObject(target[name]);\n    }\n\n    return !isNumericKey;\n  }\n\n  if (utils.isFormData(formData) && utils.isFunction(formData.entries)) {\n    const obj = {};\n\n    utils.forEachEntry(formData, (name, value) => {\n      buildPath(parsePropPath(name), value, obj, 0);\n    });\n\n    return obj;\n  }\n\n  return null;\n}\n\nexport default formDataToJSON;\n\n// Path: lib/defaults/index.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosError from '../core/AxiosError.js';\nimport transitionalDefaults from './transitional.js';\nimport toFormData from '../helpers/toFormData.js';\nimport toURLEncodedForm from '../helpers/toURLEncodedForm.js';\nimport platform from '../platform/index.js';\nimport formDataToJSON from '../helpers/formDataToJSON.js';\n\n/**\n * It takes a string, tries to parse it, and if it fails, it returns the stringified version\n * of the input\n *\n * @param {any} rawValue - The value to be stringified.\n * @param {Function} parser - A function that parses a string into a JavaScript object.\n * @param {Function} encoder - A function that takes a value and returns a string.\n *\n * @returns {string} A stringified version of the rawValue.\n */\nfunction stringifySafely(rawValue, parser, encoder) {\n  if (utils.isString(rawValue)) {\n    try {\n      (parser || JSON.parse)(rawValue);\n      return utils.trim(rawValue);\n    } catch (e) {\n      if (e.name !== 'SyntaxError') {\n        throw e;\n      }\n    }\n  }\n\n  return (encoder || JSON.stringify)(rawValue);\n}\n\nconst defaults = {\n\n  transitional: transitionalDefaults,\n\n  adapter: ['xhr', 'http'],\n\n  transformRequest: [function transformRequest(data, headers) {\n    const contentType = headers.getContentType() || '';\n    const hasJSONContentType = contentType.indexOf('application/json') > -1;\n    const isObjectPayload = utils.isObject(data);\n\n    if (isObjectPayload && utils.isHTMLForm(data)) {\n      data = new FormData(data);\n    }\n\n    const isFormData = utils.isFormData(data);\n\n    if (isFormData) {\n      return hasJSONContentType ? JSON.stringify(formDataToJSON(data)) : data;\n    }\n\n    if (utils.isArrayBuffer(data) ||\n      utils.isBuffer(data) ||\n      utils.isStream(data) ||\n      utils.isFile(data) ||\n      utils.isBlob(data)\n    ) {\n      return data;\n    }\n    if (utils.isArrayBufferView(data)) {\n      return data.buffer;\n    }\n    if (utils.isURLSearchParams(data)) {\n      headers.setContentType('application/x-www-form-urlencoded;charset=utf-8', false);\n      return data.toString();\n    }\n\n    let isFileList;\n\n    if (isObjectPayload) {\n      if (contentType.indexOf('application/x-www-form-urlencoded') > -1) {\n        return toURLEncodedForm(data, this.formSerializer).toString();\n      }\n\n      if ((isFileList = utils.isFileList(data)) || contentType.indexOf('multipart/form-data') > -1) {\n        const _FormData = this.env && this.env.FormData;\n\n        return toFormData(\n          isFileList ? {'files[]': data} : data,\n          _FormData && new _FormData(),\n          this.formSerializer\n        );\n      }\n    }\n\n    if (isObjectPayload || hasJSONContentType ) {\n      headers.setContentType('application/json', false);\n      return stringifySafely(data);\n    }\n\n    return data;\n  }],\n\n  transformResponse: [function transformResponse(data) {\n    const transitional = this.transitional || defaults.transitional;\n    const forcedJSONParsing = transitional && transitional.forcedJSONParsing;\n    const JSONRequested = this.responseType === 'json';\n\n    if (data && utils.isString(data) && ((forcedJSONParsing && !this.responseType) || JSONRequested)) {\n      const silentJSONParsing = transitional && transitional.silentJSONParsing;\n      const strictJSONParsing = !silentJSONParsing && JSONRequested;\n\n      try {\n        return JSON.parse(data);\n      } catch (e) {\n        if (strictJSONParsing) {\n          if (e.name === 'SyntaxError') {\n            throw AxiosError.from(e, AxiosError.ERR_BAD_RESPONSE, this, null, this.response);\n          }\n          throw e;\n        }\n      }\n    }\n\n    return data;\n  }],\n\n  /**\n   * A timeout in milliseconds to abort a request. If set to 0 (default) a\n   * timeout is not created.\n   */\n  timeout: 0,\n\n  xsrfCookieName: 'XSRF-TOKEN',\n  xsrfHeaderName: 'X-XSRF-TOKEN',\n\n  maxContentLength: -1,\n  maxBodyLength: -1,\n\n  env: {\n    FormData: platform.classes.FormData,\n    Blob: platform.classes.Blob\n  },\n\n  validateStatus: function validateStatus(status) {\n    return status >= 200 && status < 300;\n  },\n\n  headers: {\n    common: {\n      'Accept': 'application/json, text/plain, */*',\n      'Content-Type': undefined\n    }\n  }\n};\n\nutils.forEach(['delete', 'get', 'head', 'post', 'put', 'patch'], (method) => {\n  defaults.headers[method] = {};\n});\n\nexport default defaults;\n\n// Path: lib/core/transformData.js\n'use strict';\n\nimport utils from './../utils.js';\nimport defaults from '../defaults/index.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\n\n/**\n * Transform the data for a request or a response\n *\n * @param {Array|Function} fns A single function or Array of functions\n * @param {?Object} response The response object\n *\n * @returns {*} The resulting transformed data\n */\nexport default function transformData(fns, response) {\n  const config = this || defaults;\n  const context = response || config;\n  const headers = AxiosHeaders.from(context.headers);\n  let data = context.data;\n\n  utils.forEach(fns, function transform(fn) {\n    data = fn.call(config, data, headers.normalize(), response ? response.status : undefined);\n  });\n\n  headers.normalize();\n\n  return data;\n}\n\n// Path: lib/cancel/isCancel.js\n'use strict';\n\nexport default function isCancel(value) {\n  return !!(value && value.__CANCEL__);\n}\n\n// Path: lib/core/dispatchRequest.js\n'use strict';\n\nimport transformData from './transformData.js';\nimport isCancel from '../cancel/isCancel.js';\nimport defaults from '../defaults/index.js';\nimport CanceledError from '../cancel/CanceledError.js';\nimport AxiosHeaders from '../core/AxiosHeaders.js';\nimport adapters from \"../adapters/adapters.js\";\n\n/**\n * Throws a `CanceledError` if cancellation has been requested.\n *\n * @param {Object} config The config that is to be used for the request\n *\n * @returns {void}\n */\nfunction throwIfCancellationRequested(config) {\n  if (config.cancelToken) {\n    config.cancelToken.throwIfRequested();\n  }\n\n  if (config.signal && config.signal.aborted) {\n    throw new CanceledError(null, config);\n  }\n}\n\n/**\n * Dispatch a request to the server using the configured adapter.\n *\n * @param {object} config The config that is to be used for the request\n *\n * @returns {Promise} The Promise to be fulfilled\n */\nexport default function dispatchRequest(config) {\n  throwIfCancellationRequested(config);\n\n  config.headers = AxiosHeaders.from(config.headers);\n\n  // Transform request data\n  config.data = transformData.call(\n    config,\n    config.transformRequest\n  );\n\n  if (['post', 'put', 'patch'].indexOf(config.method) !== -1) {\n    config.headers.setContentType('application/x-www-form-urlencoded', false);\n  }\n\n  const adapter = adapters.getAdapter(config.adapter || defaults.adapter);\n\n  return adapter(config).then(function onAdapterResolution(response) {\n    throwIfCancellationRequested(config);\n\n    // Transform response data\n    response.data = transformData.call(\n      config,\n      config.transformResponse,\n      response\n    );\n\n    response.headers = AxiosHeaders.from(response.headers);\n\n    return response;\n  }, function onAdapterRejection(reason) {\n    if (!isCancel(reason)) {\n      throwIfCancellationRequested(config);\n\n      // Transform response data\n      if (reason && reason.response) {\n        reason.response.data = transformData.call(\n          config,\n          config.transformResponse,\n          reason.response\n        );\n        reason.response.headers = AxiosHeaders.from(reason.response.headers);\n      }\n    }\n\n    return Promise.reject(reason);\n  });\n}\n\n// Path: lib/helpers/validator.js\n'use strict';\n\nimport {VERSION} from '../env/data.js';\nimport AxiosError from '../core/AxiosError.js';\n\nconst validators = {};\n\n// eslint-disable-next-line func-names\n['object', 'boolean', 'number', 'function', 'string', 'symbol'].forEach((type, i) => {\n  validators[type] = function validator(thing) {\n    return typeof thing === type || 'a' + (i < 1 ? 'n ' : ' ') + type;\n  };\n});\n\nconst deprecatedWarnings = {};\n\n/**\n * Transitional option validator\n *\n * @param {function|boolean?} validator - set to false if the transitional option has been removed\n * @param {string?} version - deprecated version / removed since version\n * @param {string?} message - some message with additional info\n *\n * @returns {function}\n */\nvalidators.transitional = function transitional(validator, version, message) {\n  function formatMessage(opt, desc) {\n    return '[Axios v' + VERSION + '] Transitional option \\'' + opt + '\\'' + desc + (message ? '. ' + message : '');\n  }\n\n  // eslint-disable-next-line func-names\n  return (value, opt, opts) => {\n    if (validator === false) {\n      throw new AxiosError(\n        formatMessage(opt, ' has been removed' + (version ? ' in ' + version : '')),\n        AxiosError.ERR_DEPRECATED\n      );\n    }\n\n    if (version && !deprecatedWarnings[opt]) {\n      deprecatedWarnings[opt] = true;\n      // eslint-disable-next-line no-console\n      console.warn(\n        formatMessage(\n          opt,\n          ' has been deprecated since v' + version + ' and will be removed in the near future'\n        )\n      );\n    }\n\n    return validator ? validator(value, opt, opts) : true;\n  };\n};\n\n/**\n * Assert object's properties type\n *\n * @param {object} options\n * @param {object} schema\n * @param {boolean?} allowUnknown\n *\n * @returns {object}\n */\n\nfunction assertOptions(options, schema, allowUnknown) {\n  if (typeof options !== 'object') {\n    throw new AxiosError('options must be an object', AxiosError.ERR_BAD_OPTION_VALUE);\n  }\n  const keys = Object.keys(options);\n  let i = keys.length;\n  while (i-- > 0) {\n    const opt = keys[i];\n    const validator = schema[opt];\n    if (validator) {\n      const value = options[opt];\n      const result = value === undefined || validator(value, opt, options);\n      if (result !== true) {\n        throw new AxiosError('option ' + opt + ' must be ' + result, AxiosError.ERR_BAD_OPTION_VALUE);\n      }\n      continue;\n    }\n    if (allowUnknown !== true) {\n      throw new AxiosError('Unknown option ' + opt, AxiosError.ERR_BAD_OPTION);\n    }\n  }\n}\n\nexport default {\n  assertOptions,\n  validators\n};\n\n// Path: lib/core/mergeConfig.js\n'use strict';\n\nimport utils from '../utils.js';\nimport AxiosHeaders from \"./AxiosHeaders.js\";\n\nconst headersToObject = (thing) => thing instanceof AxiosHeaders ? { ...thing } : thing;\n\n/**\n * Config-specific merge-function which creates a new config-object\n * by merging two configuration objects together.\n *\n * @param {Object} config1\n * @param {Object} config2\n *\n * @returns {Object} New object resulting from merging config2 to config1\n */\nexport default function mergeConfig(config1, config2) {\n  // eslint-disable-next-line no-param-reassign\n  config2 = config2 || {};\n  const config = {};\n\n  function getMergedValue(target, source, caseless) {\n    if (utils.isPlainObject(target) && utils.isPlainObject(source)) {\n      return utils.merge.call({caseless}, target, source);\n    } else if (utils.isPlainObject(source)) {\n      return utils.merge({}, source);\n    } else if (utils.isArray(source)) {\n      return source.slice();\n    }\n    return source;\n  }\n\n  // eslint-disable-next-line consistent-return\n  function mergeDeepProperties(a, b, caseless) {\n    if (!utils.isUndefined(b)) {\n      return getMergedValue(a, b, caseless);\n    } else if (!utils.isUndefined(a)) {\n      return getMergedValue(undefined, a, caseless);\n    }\n  }\n\n  // eslint-disable-next-line consistent-return\n  function valueFromConfig2(a, b) {\n    if (!utils.isUndefined(b)) {\n      return getMergedValue(undefined, b);\n    }\n  }\n\n  // eslint-disable-next-line consistent-return\n  function defaultToConfig2(a, b) {\n    if (!utils.isUndefined(b)) {\n      return getMergedValue(undefined, b);\n    } else if (!utils.isUndefined(a)) {\n      return getMergedValue(undefined, a);\n    }\n  }\n\n  // eslint-disable-next-line consistent-return\n  function mergeDirectKeys(a, b, prop) {\n    if (prop in config2) {\n      return getMergedValue(a, b);\n    } else if (prop in config1) {\n      return getMergedValue(undefined, a);\n    }\n  }\n\n  const mergeMap = {\n    url: valueFromConfig2,\n    method: valueFromConfig2,\n    data: valueFromConfig2,\n    baseURL: defaultToConfig2,\n    transformRequest: defaultToConfig2,\n    transformResponse: defaultToConfig2,\n    paramsSerializer: defaultToConfig2,\n    timeout: defaultToConfig2,\n    timeoutMessage: defaultToConfig2,\n    withCredentials: defaultToConfig2,\n    withXSRFToken: defaultToConfig2,\n    adapter: defaultToConfig2,\n    responseType: defaultToConfig2,\n    xsrfCookieName: defaultToConfig2,\n    xsrfHeaderName: defaultToConfig2,\n    onUploadProgress: defaultToConfig2,\n    onDownloadProgress: defaultToConfig2,\n    decompress: defaultToConfig2,\n    maxContentLength: defaultToConfig2,\n    maxBodyLength: defaultToConfig2,\n    beforeRedirect: defaultToConfig2,\n    transport: defaultToConfig2,\n    httpAgent: defaultToConfig2,\n    httpsAgent: defaultToConfig2,\n    cancelToken: defaultToConfig2,\n    socketPath: defaultToConfig2,\n    responseEncoding: defaultToConfig2,\n    validateStatus: mergeDirectKeys,\n    headers: (a, b) => mergeDeepProperties(headersToObject(a), headersToObject(b), true)\n  };\n\n  utils.forEach(Object.keys(Object.assign({}, config1, config2)), function computeConfigValue(prop) {\n    const merge = mergeMap[prop] || mergeDeepProperties;\n    const configValue = merge(config1[prop], config2[prop], prop);\n    (utils.isUndefined(configValue) && merge !== mergeDirectKeys) || (config[prop] = configValue);\n  });\n\n  return config;\n}\n\n// Path: lib/core/InterceptorManager.js\n'use strict';\n\nimport utils from './../utils.js';\n\nclass InterceptorManager {\n  constructor() {\n    this.handlers = [];\n  }\n\n  /**\n   * Add a new interceptor to the stack\n   *\n   * @param {Function} fulfilled The function to handle `then` for a `Promise`\n   * @param {Function} rejected The function to handle `reject` for a `Promise`\n   *\n   * @return {Number} An ID used to remove interceptor later\n   */\n  use(fulfilled, rejected, options) {\n    this.handlers.push({\n      fulfilled,\n      rejected,\n      synchronous: options ? options.synchronous : false,\n      runWhen: options ? options.runWhen : null\n    });\n    return this.handlers.length - 1;\n  }\n\n  /**\n   * Remove an interceptor from the stack\n   *\n   * @param {Number} id The ID that was returned by `use`\n   *\n   * @returns {Boolean} `true` if the interceptor was removed, `false` otherwise\n   */\n  eject(id) {\n    if (this.handlers[id]) {\n      this.handlers[id] = null;\n    }\n  }\n\n  /**\n   * Clear all interceptors from the stack\n   *\n   * @returns {void}\n   */\n  clear() {\n    if (this.handlers) {\n      this.handlers = [];\n    }\n  }\n\n  /**\n   * Iterate over all the registered interceptors\n   *\n   * This method is particularly useful for skipping over any\n   * interceptors that may have become `null` calling `eject`.\n   *\n   * @param {Function} fn The function to call for each interceptor\n   *\n   * @returns {void}\n   */\n  forEach(fn) {\n    utils.forEach(this.handlers, function forEachHandler(h) {\n      if (h !== null) {\n        fn(h);\n      }\n    });\n  }\n}\n\nexport default InterceptorManager;\n\n// Path: lib/core/Axios.js\n'use strict';\n\nimport utils from './../utils.js';\nimport buildURL from '../helpers/buildURL.js';\nimport InterceptorManager from './InterceptorManager.js';\nimport dispatchRequest from './dispatchRequest.js';\nimport mergeConfig from './mergeConfig.js';\nimport buildFullPath from './buildFullPath.js';\nimport validator from '../helpers/validator.js';\nimport AxiosHeaders from './AxiosHeaders.js';\n\nconst validators = validator.validators;\n\n/**\n * Create a new instance of Axios\n *\n * @param {Object} instanceConfig The default config for the instance\n *\n * @return {Axios} A new instance of Axios\n */\nclass Axios {\n  constructor(instanceConfig) {\n    this.defaults = instanceConfig;\n    this.interceptors = {\n      request: new InterceptorManager(),\n      response: new InterceptorManager()\n    };\n  }\n\n  /**\n   * Dispatch a request\n   *\n   * @param {String|Object} configOrUrl The config specific for this request (merged with this.defaults)\n   * @param {?Object} config\n   *\n   * @returns {Promise} The Promise to be fulfilled\n   */\n  async request(configOrUrl, config) {\n    try {\n      return await this._request(configOrUrl, config);\n    } catch (err) {\n      if (err instanceof Error) {\n        let dummy;\n\n        Error.captureStackTrace ? Error.captureStackTrace(dummy = {}) : (dummy = new Error());\n\n        // slice off the Error: ... line\n        const stack = dummy.stack ? dummy.stack.replace(/^.+\\n/, '') : '';\n\n        if (!err.stack) {\n          err.stack = stack;\n          // match without the 2 top stack lines\n        } else if (stack && !String(err.stack).endsWith(stack.replace(/^.+\\n.+\\n/, ''))) {\n          err.stack += '\\n' + stack\n        }\n      }\n\n      throw err;\n    }\n  }\n\n  _request(configOrUrl, config) {\n    /*eslint no-param-reassign:0*/\n    // Allow for axios('example/url'[, config]) a la fetch API\n    if (typeof configOrUrl === 'string') {\n      config = config || {};\n      config.url = configOrUrl;\n    } else {\n      config = configOrUrl || {};\n    }\n\n    config = mergeConfig(this.defaults, config);\n\n    const {transitional, paramsSerializer, headers} = config;\n\n    if (transitional !== undefined) {\n      validator.assertOptions(transitional, {\n        silentJSONParsing: validators.transitional(validators.boolean),\n        forcedJSONParsing: validators.transitional(validators.boolean),\n        clarifyTimeoutError: validators.transitional(validators.boolean)\n      }, false);\n    }\n\n    if (paramsSerializer != null) {\n      if (utils.isFunction(paramsSerializer)) {\n        config.paramsSerializer = {\n          serialize: paramsSerializer\n        }\n      } else {\n        validator.assertOptions(paramsSerializer, {\n          encode: validators.function,\n          serialize: validators.function\n        }, true);\n      }\n    }\n\n    // Set config.method\n    config.method = (config.method || this.defaults.method || 'get').toLowerCase();\n\n    // Flatten headers\n    let contextHeaders = headers && utils.merge(\n      headers.common,\n      headers[config.method]\n    );\n\n    headers && utils.forEach(\n      ['delete', 'get', 'head', 'post', 'put', 'patch', 'common'],\n      (method) => {\n        delete headers[method];\n      }\n    );\n\n    config.headers = AxiosHeaders.concat(contextHeaders, headers);\n\n    // filter out skipped interceptors\n    const requestInterceptorChain = [];\n    let synchronousRequestInterceptors = true;\n    this.interceptors.request.forEach(function unshiftRequestInterceptors(interceptor) {\n      if (typeof interceptor.runWhen === 'function' && interceptor.runWhen(config) === false) {\n        return;\n      }\n\n      synchronousRequestInterceptors = synchronousRequestInterceptors && interceptor.synchronous;\n\n      requestInterceptorChain.unshift(interceptor.fulfilled, interceptor.rejected);\n    });\n\n    const responseInterceptorChain = [];\n    this.interceptors.response.forEach(function pushResponseInterceptors(interceptor) {\n      responseInterceptorChain.push(interceptor.fulfilled, interceptor.rejected);\n    });\n\n    let promise;\n    let i = 0;\n    let len;\n\n    if (!synchronousRequestInterceptors) {\n      const chain = [dispatchRequest.bind(this), undefined];\n      chain.unshift.apply(chain, requestInterceptorChain);\n      chain.push.apply(chain, responseInterceptorChain);\n      len = chain.length;\n\n      promise = Promise.resolve(config);\n\n      while (i < len) {\n        promise = promise.then(chain[i++], chain[i++]);\n      }\n\n      return promise;\n    }\n\n    len = requestInterceptorChain.length;\n\n    let newConfig = config;\n\n    i = 0;\n\n    while (i < len) {\n      const onFulfilled = requestInterceptorChain[i++];\n      const onRejected = requestInterceptorChain[i++];\n      try {\n        newConfig = onFulfilled(newConfig);\n      } catch (error) {\n        onRejected.call(this, error);\n        break;\n      }\n    }\n\n    try {\n      promise = dispatchRequest.call(this, newConfig);\n    } catch (error) {\n      return Promise.reject(error);\n    }\n\n    i = 0;\n    len = responseInterceptorChain.length;\n\n    while (i < len) {\n      promise = promise.then(responseInterceptorChain[i++], responseInterceptorChain[i++]);\n    }\n\n    return promise;\n  }\n\n  getUri(config) {\n    config = mergeConfig(this.defaults, config);\n    const fullPath = buildFullPath(config.baseURL, config.url);\n    return buildURL(fullPath, config.params, config.paramsSerializer);\n  }\n}\n\n// Provide aliases for supported request methods\nutils.forEach(['delete', 'get', 'head', 'options'], function forEachMethodNoData(method) {\n  /*eslint func-names:0*/\n  Axios.prototype[method] = function(url, config) {\n    return this.request(mergeConfig(config || {}, {\n      method,\n      url,\n      data: (config || {}).data\n    }));\n  };\n});\n\nutils.forEach(['post', 'put', 'patch'], function forEachMethodWithData(method) {\n  /*eslint func-names:0*/\n\n  function generateHTTPMethod(isForm) {\n    return function httpMethod(url, data, config) {\n      return this.request(mergeConfig(config || {}, {\n        method,\n        headers: isForm ? {\n          'Content-Type': 'multipart/form-data'\n        } : {},\n        url,\n        data\n      }));\n    };\n  }\n\n  Axios.prototype[method] = generateHTTPMethod();\n\n  Axios.prototype[method + 'Form'] = generateHTTPMethod(true);\n});\n\nexport default Axios;\n\n// Path: lib/helpers/HttpStatusCode.js\nconst HttpStatusCode = {\n  Continue: 100,\n  SwitchingProtocols: 101,\n  Processing: 102,\n  EarlyHints: 103,\n  Ok: 200,\n  Created: 201,\n  Accepted: 202,\n  NonAuthoritativeInformation: 203,\n  NoContent: 204,\n  ResetContent: 205,\n  PartialContent: 206,\n  MultiStatus: 207,\n  AlreadyReported: 208,\n  ImUsed: 226,\n  MultipleChoices: 300,\n  MovedPermanently: 301,\n  Found: 302,\n  SeeOther: 303,\n  NotModified: 304,\n  UseProxy: 305,\n  Unused: 306,\n  TemporaryRedirect: 307,\n  PermanentRedirect: 308,\n  BadRequest: 400,\n  Unauthorized: 401,\n  PaymentRequired: 402,\n  Forbidden: 403,\n  NotFound: 404,\n  MethodNotAllowed: 405,\n  NotAcceptable: 406,\n  ProxyAuthenticationRequired: 407,\n  RequestTimeout: 408,\n  Conflict: 409,\n  Gone: 410,\n  LengthRequired: 411,\n  PreconditionFailed: 412,\n  PayloadTooLarge: 413,\n  UriTooLong: 414,\n  UnsupportedMediaType: 415,\n  RangeNotSatisfiable: 416,\n  ExpectationFailed: 417,\n  ImATeapot: 418,\n  MisdirectedRequest: 421,\n  UnprocessableEntity: 422,\n  Locked: 423,\n  FailedDependency: 424,\n  TooEarly: 425,\n  UpgradeRequired: 426,\n  PreconditionRequired: 428,\n  TooManyRequests: 429,\n  RequestHeaderFieldsTooLarge: 431,\n  UnavailableForLegalReasons: 451,\n  InternalServerError: 500,\n  NotImplemented: 501,\n  BadGateway: 502,\n  ServiceUnavailable: 503,\n  GatewayTimeout: 504,\n  HttpVersionNotSupported: 505,\n  VariantAlsoNegotiates: 506,\n  InsufficientStorage: 507,\n  LoopDetected: 508,\n  NotExtended: 510,\n  NetworkAuthenticationRequired: 511,\n};\n\nObject.entries(HttpStatusCode).forEach(([key, value]) => {\n  HttpStatusCode[value] = key;\n});\n\nexport default HttpStatusCode;\n\n// Path: lib/helpers/spread.js\n'use strict';\n\n/**\n * Syntactic sugar for invoking a function and expanding an array for arguments.\n *\n * Common use case would be to use `Function.prototype.apply`.\n *\n *  ```js\n *  function f(x, y, z) {}\n *  var args = [1, 2, 3];\n *  f.apply(null, args);\n *  ```\n *\n * With `spread` this example can be re-written.\n *\n *  ```js\n *  spread(function(x, y, z) {})([1, 2, 3]);\n *  ```\n *\n * @param {Function} callback\n *\n * @returns {Function}\n */\nexport default function spread(callback) {\n  return function wrap(arr) {\n    return callback.apply(null, arr);\n  };\n}\n\n// Path: lib/cancel/CancelToken.js\n'use strict';\n\nimport CanceledError from './CanceledError.js';\n\n/**\n * A `CancelToken` is an object that can be used to request cancellation of an operation.\n *\n * @param {Function} executor The executor function.\n *\n * @returns {CancelToken}\n */\nclass CancelToken {\n  constructor(executor) {\n    if (typeof executor !== 'function') {\n      throw new TypeError('executor must be a function.');\n    }\n\n    let resolvePromise;\n\n    this.promise = new Promise(function promiseExecutor(resolve) {\n      resolvePromise = resolve;\n    });\n\n    const token = this;\n\n    // eslint-disable-next-line func-names\n    this.promise.then(cancel => {\n      if (!token._listeners) return;\n\n      let i = token._listeners.length;\n\n      while (i-- > 0) {\n        token._listeners[i](cancel);\n      }\n      token._listeners = null;\n    });\n\n    // eslint-disable-next-line func-names\n    this.promise.then = onfulfilled => {\n      let _resolve;\n      // eslint-disable-next-line func-names\n      const promise = new Promise(resolve => {\n        token.subscribe(resolve);\n        _resolve = resolve;\n      }).then(onfulfilled);\n\n      promise.cancel = function reject() {\n        token.unsubscribe(_resolve);\n      };\n\n      return promise;\n    };\n\n    executor(function cancel(message, config, request) {\n      if (token.reason) {\n        // Cancellation has already been requested\n        return;\n      }\n\n      token.reason = new CanceledError(message, config, request);\n      resolvePromise(token.reason);\n    });\n  }\n\n  /**\n   * Throws a `CanceledError` if cancellation has been requested.\n   */\n  throwIfRequested() {\n    if (this.reason) {\n      throw this.reason;\n    }\n  }\n\n  /**\n   * Subscribe to the cancel signal\n   */\n\n  subscribe(listener) {\n    if (this.reason) {\n      listener(this.reason);\n      return;\n    }\n\n    if (this._listeners) {\n      this._listeners.push(listener);\n    } else {\n      this._listeners = [listener];\n    }\n  }\n\n  /**\n   * Unsubscribe from the cancel signal\n   */\n\n  unsubscribe(listener) {\n    if (!this._listeners) {\n      return;\n    }\n    const index = this._listeners.indexOf(listener);\n    if (index !== -1) {\n      this._listeners.splice(index, 1);\n    }\n  }\n\n  /**\n   * Returns an object that contains a new `CancelToken` and a function that, when called,\n   * cancels the `CancelToken`.\n   */\n  static source() {\n    let cancel;\n    const token = new CancelToken(function executor(c) {\n      cancel = c;\n    });\n    return {\n      token,\n      cancel\n    };\n  }\n}\n\nexport default CancelToken;\n\n// Path: lib/helpers/isAxiosError.js\n'use strict';\n\nimport utils from './../utils.js';\n\n/**\n * Determines whether the payload is an error thrown by Axios\n *\n * @param {*} payload The value to test\n *\n * @returns {boolean} True if the payload is an error thrown by Axios, otherwise false\n */\nexport default \nfunction isAxiosError(payload) {\n  return utils.isObject(payload) && (payload.isAxiosError === true);\n}\n\n// Path: lib/axios.js\n'use strict';\n\nimport utils from './utils.js';\nimport bind from './helpers/bind.js';\nimport Axios from './core/Axios.js';\nimport mergeConfig from './core/mergeConfig.js';\nimport defaults from './defaults/index.js';\nimport formDataToJSON from './helpers/formDataToJSON.js';\nimport CanceledError from './cancel/CanceledError.js';\nimport CancelToken from './cancel/CancelToken.js';\nimport isCancel from './cancel/isCancel.js';\nimport {VERSION} from './env/data.js';\nimport toFormData from './helpers/toFormData.js';\nimport AxiosError from './core/AxiosError.js';\nimport spread from './helpers/spread.js';\nimport isAxiosError from './helpers/isAxiosError.js';\nimport AxiosHeaders from \"./core/AxiosHeaders.js\";\nimport adapters from './adapters/adapters.js';\nimport HttpStatusCode from './helpers/HttpStatusCode.js';\n\n/**\n * Create an instance of Axios\n *\n * @param {Object} defaultConfig The default config for the instance\n *\n * @returns {Axios} A new instance of Axios\n */\nfunction createInstance(defaultConfig) {\n  const context = new Axios(defaultConfig);\n  const instance = bind(Axios.prototype.request, context);\n\n  // Copy axios.prototype to instance\n  utils.extend(instance, Axios.prototype, context, {allOwnKeys: true});\n\n  // Copy context to instance\n  utils.extend(instance, context, null, {allOwnKeys: true});\n\n  // Factory for creating new instances\n  instance.create = function create(instanceConfig) {\n    return createInstance(mergeConfig(defaultConfig, instanceConfig));\n  };\n\n  return instance;\n}\n\n// Create the default instance to be exported\nconst axios = createInstance(defaults);\n\n// Expose Axios class to allow class inheritance\naxios.Axios = Axios;\n\n// Expose Cancel & CancelToken\naxios.CanceledError = CanceledError;\naxios.CancelToken = CancelToken;\naxios.isCancel = isCancel;\naxios.VERSION = VERSION;\naxios.toFormData = toFormData;\n\n// Expose AxiosError class\naxios.AxiosError = AxiosError;\n\n// alias for CanceledError for backward compatibility\naxios.Cancel = axios.CanceledError;\n\n// Expose all/spread\naxios.all = function all(promises) {\n  return Promise.all(promises);\n};\n\naxios.spread = spread;\n\n// Expose isAxiosError\naxios.isAxiosError = isAxiosError;\n\n// Expose mergeConfig\naxios.mergeConfig = mergeConfig;\n\naxios.AxiosHeaders = AxiosHeaders;\n\naxios.formToJSON = thing => formDataToJSON(utils.isHTMLForm(thing) ? new FormData(thing) : thing);\n\n...\n", "needle_token_start": 15131, "needle_token_end": 15166, "code_context_ntokens": 15933, "output": ["```javascript\n/**\n * Determines whether the payload is an error thrown by Axios\n *\n * @param {*} payload The value to test\n *\n * @returns {boolean} True if the payload is an error thrown by Axios, otherwise false\n */\nexport default function isAxiosError(payload) {\n  return utils.isObject(payload) && (payload.isAxiosError === true);\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "execScript", "language": "typescript", "path": "src/isDate/test.ts", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to dynamically execute JavaScript code within the context of an iframe's document, primarily for testing or manipulating the iframe's environment.\n2. **Input**: A string containing the JavaScript code to be executed.\n3. **Output**: There is no direct output from the function itself; however, the effects of the executed script can modify the state or content of the iframe's document.\n4. **Procedure**: The function first locates an iframe element by its ID. It then checks if the iframe and its document body are properly accessible. If they are, it creates a new script element, sets its text content to the provided JavaScript code, and appends this script to the body of the iframe's document, thereby executing the code within the iframe's context.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/eachWeekendOfYear/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { eachWeekendOfYear } from \"./index.js\";\nimport { isWeekend } from \"../isWeekend/index.js\";\n\ndescribe(\"eachWeekendOfYear\", () => {\n  it(\"returns all weekends of the given year\", () => {\n    const result = eachWeekendOfYear(new Date(2020, 0, 1));\n    expect(result.length).toBe(104);\n    expect(result.every(isWeekend)).toBe(true);\n    expect(result[0]).toEqual(new Date(2020, 0, 4));\n    expect(result[103]).toEqual(new Date(2020, 11, 27));\n  });\n\n  it(\"returns an empty asrray when the expected year is an Invalid Date\", () => {\n...\n// Path: src/isDate/test.ts\nimport { afterEach, describe, expect, it } from \"vitest\";\nimport { assertType } from \"../_lib/test/index.js\";\nimport { isDate } from \"./index.js\";\n\ndescribe(\"isDate\", () => {\n  it(\"returns true if the given value is a date object\", () => {\n    expect(isDate(new Date())).toBe(true);\n  });\n\n  it(\"returns true if the given value is an Invalid Date\", () => {\n    expect(isDate(new Date(NaN))).toBe(true);\n  });\n\n  it(\"ensures that the passed argument is an instance of Date\", () => {\n    const date: unknown = new Date();\n\n    if (isDate(date)) {\n      assertType<Date>(date);\n    } else {\n      assertType<unknown>(date);\n    }\n  });\n\n  describe(\"with date passed from another iframe\", () => {\n    // If in the browser, run the test in an iframe\n    if (typeof window !== \"undefined\") {\n      afterEach(() => {\n        const iframe = document.getElementById(\"iframe\");\n        iframe && iframe.remove();\n      });\n\n      it(\"returns true for a date passed from another iframe\", () =>\n        new Promise((resolve) => {\n          const iframe = document.createElement(\"iframe\");\n          iframe.id = \"iframe\";\n          iframe.addEventListener(\"load\", () => {\n            execScript(\"window.date = new Date()\");\n            // eslint-disable-next-line @typescript-eslint/no-explicit-any -- It's ok we're messing with iframes\n            expect(isDate((iframe.contentWindow as any).date)).toBe(true);\n            resolve(void 0);\n          });\n          if (!document.body) throw new Error(\"document.body is not defined\");\n          document.body.appendChild(iframe);\n        }));\n    } else {\n      it.skip(\"returns true for a date passed from another iframe\");\n    }\n\n    \nfunction execScript(scriptText: string) {\n      const iframe = document.querySelector(\"iframe#iframe\");\n      if (!iframe || !(iframe instanceof HTMLIFrameElement)) {\n        throw new Error(\"Can't execute the script because iframe isn't found\");\n      }\n      const doc = iframe.contentDocument!;\n      const script = doc.createElement(\"script\");\n      script.innerText = scriptText;\n      // @ts-expect-error - We're messing with iframes\n      if (!(doc.body instanceof iframe.contentWindow.HTMLBodyElement)) {\n        throw new Error(\n          \"Can't execute the script because iframe does not have body\",\n        );\n      }\n      doc.body.append(script);\n    }\n  });\n\n  it(\"returns false if the given value is not a date object\", () => {\n    expect(!isDate(new Date().getTime())).toBe(true);\n    expect(!isDate(new Date().toISOString())).toBe(true);\n    expect(!isDate({})).toBe(true);\n    expect(!isDate(null)).toBe(true);\n    expect(!isDate(0)).toBe(true);\n  });\n});\n\n// Path: src/getHours/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { getHours } from \"./index.js\";\n\ndescribe(\"getHours\", () => {\n  it(\"returns the hours of the given date\", () => {\n    const result = getHours(new Date(2012, 1 /* Feb */, 29, 11, 45));\n    expect(result).toBe(11);\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const result = getHours(new Date(2014, 3 /* Apr */, 2, 23, 30).getTime());\n    expect(result).toBe(23);\n  });\n\n  it(\"returns NaN if the given date is invalid\", () => {\n    const result = getHours(new Date(NaN));\n    expect(isNaN(result)).toBe(true);\n  });\n});\n\n// Path: src/isSunday/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { isSunday } from \"./index.js\";\n\ndescribe(\"isSunday\", () => {\n  it(\"returns true if the given date is Sunday\", () => {\n    const result = isSunday(new Date(2014, 8 /* Sep */, 21));\n    expect(result).toBe(true);\n  });\n\n  it(\"returns false if the given date is not Sunday\", () => {\n    const result = isSunday(new Date(2014, 8 /* Sep */, 25));\n    expect(result).toBe(false);\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const result = isSunday(new Date(2014, 1 /* Feb */, 9).getTime());\n    expect(result).toBe(true);\n  });\n\n  it(\"returns false if the given date is `Invalid Date`\", () => {\n    const result = isSunday(new Date(NaN));\n    expect(result).toBe(false);\n  });\n});\n\n// Path: src/differenceInMonths/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { differenceInMonths } from \"./index.js\";\n\ndescribe(\"differenceInMonths\", () => {\n  it(\"returns the number of full months between the given dates\", () => {\n    const result = differenceInMonths(\n      new Date(2012, 6 /* Jul */, 2, 18, 0),\n      new Date(2011, 6 /* Jul */, 2, 6, 0),\n    );\n    expect(result).toBe(12);\n  });\n\n  it(\"returns a negative number if the time value of the first date is smaller\", () => {\n    const result = differenceInMonths(\n      new Date(2011, 6 /* Jul */, 2, 6, 0),\n      new Date(2012, 6 /* Jul */, 2, 18, 0),\n    );\n    expect(result).toBe(-12);\n  });\n\n  it(\"accepts timestamps\", () => {\n    const result = differenceInMonths(\n      new Date(2014, 7 /* Aug */, 2).getTime(),\n      new Date(2010, 6 /* Jul */, 2).getTime(),\n    );\n    expect(result).toBe(49);\n  });\n\n  describe(\"edge cases\", () => {\n    it(\"it returns diff of 1 month between Feb 28 2021 and Jan 30 2021\", () => {\n      const result = differenceInMonths(\n        new Date(2021, 1 /* Feb */, 28),\n        new Date(2021, 0 /* Jan */, 30),\n      );\n      expect(result).toBe(1);\n    });\n\n    it(\"it returns diff of 1 month between Feb 28 2021 and Jan 31 2021\", () => {\n      const result = differenceInMonths(\n        new Date(2021, 1 /* Feb */, 28),\n        new Date(2021, 0 /* Jan */, 31),\n      );\n      expect(result).toBe(1);\n    });\n\n    it(\"it returns diff of 1 month between Nov, 30 2021 and Oct, 31 2021\", () => {\n      const result = differenceInMonths(\n        new Date(2021, 10 /* Nov */, 30),\n        new Date(2021, 9 /* Oct */, 31),\n      );\n      expect(result).toBe(1);\n    });\n\n    it(\"it returns diff of 1 month between Oct, 31 2021 and Sep, 30 2021\", () => {\n      const result = differenceInMonths(\n        new Date(2021, 9 /* Oct */, 31),\n        new Date(2021, 8 /* Sep */, 30),\n      );\n      expect(result).toBe(1);\n    });\n\n    it(\"it returns diff of 6 month between Oct, 31 2021 and Apr, 30 2021\", () => {\n      const result = differenceInMonths(\n        new Date(2021, 9 /* Oct */, 31),\n        new Date(2021, 3 /* Apr */, 30),\n      );\n      expect(result).toBe(6);\n    });\n\n    it(\"it returns diff of -1 month between Sep, 30 2021 and Oct, 31 2021\", () => {\n      const result = differenceInMonths(\n        new Date(2021, 8 /* Sep */, 30),\n        new Date(2021, 9 /* Oct */, 31),\n      );\n      expect(result).toBe(-1);\n    });\n\n    it(\"the difference is less than a month, but the given dates are in different calendar months\", () => {\n      const result = differenceInMonths(\n        new Date(2014, 7 /* Aug */, 1),\n        new Date(2014, 6 /* Jul */, 31),\n      );\n      expect(result).toBe(0);\n    });\n\n    it(\"the same for the swapped dates\", () => {\n      const result = differenceInMonths(\n        new Date(2014, 6 /* Jul */, 31),\n        new Date(2014, 7 /* Aug */, 1),\n      );\n      expect(result).toBe(0);\n    });\n\n    it(\"the days of months of the given dates are the same\", () => {\n      const result = differenceInMonths(\n        new Date(2014, 8 /* Sep */, 6),\n        new Date(2014, 7 /* Aug */, 6),\n      );\n      expect(result).toBe(1);\n    });\n\n    it(\"the given dates are the same\", () => {\n      const result = differenceInMonths(\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n      );\n      expect(result).toBe(0);\n    });\n\n    it(\"does not return -0 when the given dates are the same\", () => {\n      function isNegativeZero(x: number): boolean {\n        return x === 0 && 1 / x < 0;\n      }\n\n      const result = differenceInMonths(\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n      );\n\n      const resultIsNegative = isNegativeZero(result);\n      expect(resultIsNegative).toBe(false);\n    });\n  });\n\n  it(\"returns NaN if the first date is `Invalid Date`\", () => {\n    const result = differenceInMonths(\n      new Date(NaN),\n      new Date(2017, 0 /* Jan */, 1),\n    );\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the second date is `Invalid Date`\", () => {\n    const result = differenceInMonths(\n      new Date(2017, 0 /* Jan */, 1),\n      new Date(NaN),\n    );\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the both dates are `Invalid Date`\", () => {\n    const result = differenceInMonths(new Date(NaN), new Date(NaN));\n    expect(isNaN(result)).toBe(true);\n  });\n\n  describe(\"edge cases\", () => {\n    it(\"returns the number of full months between the given dates - end of Feb\", () => {\n      expect(differenceInMonths(\n        new Date(2012, 1 /* Feb */, 29, 9, 0, 0),\n        new Date(2012, 1 /* Feb */, 29, 10, 0, 0),\n      )).toBe(0);\n      expect(differenceInMonths(\n        new Date(2012, 1 /* Feb */, 28, 9, 0, 0),\n        new Date(2012, 1 /* Feb */, 29, 10, 0, 0),\n      )).toBe(0);\n      expect(differenceInMonths(\n        new Date(2012, 1 /* Feb */, 27, 9, 0, 0),\n        new Date(2012, 1 /* Feb */, 27, 10, 0, 0),\n      )).toBe(0);\n      expect(differenceInMonths(\n        new Date(2012, 1 /* Feb */, 28, 9, 0, 0),\n        new Date(2012, 1 /* Feb */, 28, 10, 0, 0),\n      )).toBe(0);\n    });\n\n    expect(differenceInMonths(\n      new Date(2021, 1 /* Feb */, 28, 7, 23, 7),\n      new Date(2021, 1 /* Feb */, 28, 7, 38, 18),\n    )).toBe(0);\n  });\n});\n\n// Path: src/lastDayOfDecade/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { lastDayOfDecade } from \"./index.js\";\n\ndescribe(\"lastDayOfDecade\", () => {\n  it(\"returns the date with the time set to 00:00:00 and the date set to the last day of a decade\", () => {\n    const date = new Date(1985, 9 /* Oct */, 20);\n    const result = lastDayOfDecade(date);\n    expect(result).toEqual(new Date(1989, 11 /* Dec */, 31));\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const date = new Date(1975, 0 /* Jan */, 19).getTime();\n    const result = lastDayOfDecade(date);\n    expect(result).toEqual(new Date(1979, 11 /* Dec */, 31));\n  });\n\n  it(\"does not mutate the original date\", () => {\n    const date = new Date(2013, 3 /* Apr */, 23);\n    lastDayOfDecade(date);\n    expect(date).toEqual(new Date(2013, 3 /* Apr */, 23));\n  });\n\n  it(\"returns `Invalid Date` if the given date is invalid\", () => {\n    const result = lastDayOfDecade(new Date(NaN));\n    expect(result instanceof Date && isNaN(result.getTime())).toBe(true);\n  });\n\n  it(\"properly works with negative numbers\", () => {\n    expect(lastDayOfDecade(new Date(2001, 0, 1))).toEqual(\n      new Date(2009, 11, 31),\n    );\n    expect(lastDayOfDecade(new Date(-2009, 0, 1))).toEqual(\n      new Date(-2001, 11, 31),\n    );\n  });\n});\n\n// Path: src/eachMinuteOfInterval/test.ts\nimport { describe, expect, it } from \"vitest\";\n\nimport { eachMinuteOfInterval } from \"./index.js\";\n\ndescribe(\"eachMinuteOfInterval\", () => {\n  it(\"should return an array of Date objects containing a Date for each minute between the interval\", () => {\n    const result = eachMinuteOfInterval({\n      start: new Date(2020, 10, 14, 13, 0),\n      end: new Date(2020, 10, 14, 13, 5),\n    });\n\n    expect(result).toEqual([\n      new Date(2020, 10, 14, 13, 0),\n      new Date(2020, 10, 14, 13, 1),\n      new Date(2020, 10, 14, 13, 2),\n      new Date(2020, 10, 14, 13, 3),\n      new Date(2020, 10, 14, 13, 4),\n      new Date(2020, 10, 14, 13, 5),\n    ]);\n  });\n\n  it(\"should handle all the minutes that are not in the begining\", () => {\n    const result = eachMinuteOfInterval({\n      start: new Date(2020, 10, 14, 13, 0, 33),\n      end: new Date(2020, 10, 14, 13, 2),\n    });\n\n    expect(result[0]).toEqual(new Date(2020, 10, 14, 13));\n    expect(result[2]).toEqual(new Date(2020, 10, 14, 13, 2));\n  });\n\n  it(\"should accept timestamps\", () => {\n    const start = new Date(2020, 10, 14, 13, 0).getTime();\n    const end = new Date(2020, 10, 14, 13, 2).getTime();\n\n    const result = eachMinuteOfInterval({\n      start,\n      end,\n    });\n\n    expect(result).toEqual([\n      new Date(2020, 10, 14, 13, 0),\n      new Date(2020, 10, 14, 13, 1),\n      new Date(2020, 10, 14, 13, 2),\n    ]);\n  });\n\n  it(\"treats intervals shorter than a minute as valid\", () => {\n    const block = eachMinuteOfInterval.bind(null, {\n      start: new Date(2014, 10, 14, 10, 1, 0),\n      end: new Date(2014, 10, 14, 10, 1, 1),\n    });\n    expect(block).not.toThrow();\n  });\n\n  it(\"returns reversed array if the start date is after the end date\", () => {\n    const result = eachMinuteOfInterval({\n      start: new Date(2020, 10, 14, 13, 5),\n      end: new Date(2020, 10, 14, 13, 0),\n    });\n\n    expect(result).toEqual([\n      new Date(2020, 10, 14, 13, 5),\n      new Date(2020, 10, 14, 13, 4),\n      new Date(2020, 10, 14, 13, 3),\n      new Date(2020, 10, 14, 13, 2),\n      new Date(2020, 10, 14, 13, 1),\n      new Date(2020, 10, 14, 13, 0),\n    ]);\n  });\n\n  it(\"returns an empty array if the start date is `Invalid Date`\", () => {\n    const result = eachMinuteOfInterval({\n      start: new Date(NaN),\n      end: new Date(2014, 9 /* Oct */, 6),\n    });\n    expect(result).toEqual([]);\n  });\n\n  it(\"returns an empty array if the end date is `Invalid Date`\", () => {\n    const result = eachMinuteOfInterval({\n      start: new Date(2014, 9 /* Oct */, 12),\n      end: new Date(NaN),\n    });\n    expect(result).toEqual([]);\n  });\n\n  it(\"returns an empty array if both of the properties are `Invalid Date`\", () => {\n    const result = eachMinuteOfInterval({\n      start: new Date(NaN),\n      end: new Date(NaN),\n    });\n    expect(result).toEqual([]);\n  });\n\n  describe(\"options.step\", () => {\n    const interval = {\n      start: new Date(2020, 9, 14, 13, 1),\n      end: new Date(2020, 9, 14, 13, 7),\n    };\n\n    it(\"returns an array with starts of hours from the hour of the start date to the hour of the end date with the given step\", () => {\n      const result = eachMinuteOfInterval(interval, { step: 3 });\n      expect(result).toEqual([\n        new Date(2020, 9, 14, 13, 1),\n        new Date(2020, 9, 14, 13, 4),\n        new Date(2020, 9, 14, 13, 7),\n      ]);\n    });\n\n    it(\"returns reversed array if `options.step` is negative\", () => {\n      const result = eachMinuteOfInterval(\n        {\n          start: new Date(2020, 10, 14, 13, 0),\n          end: new Date(2020, 10, 14, 13, 5),\n        },\n        { step: -1 },\n      );\n\n      expect(result).toEqual([\n        new Date(2020, 10, 14, 13, 5),\n        new Date(2020, 10, 14, 13, 4),\n        new Date(2020, 10, 14, 13, 3),\n        new Date(2020, 10, 14, 13, 2),\n        new Date(2020, 10, 14, 13, 1),\n        new Date(2020, 10, 14, 13, 0),\n      ]);\n    });\n\n    it(\"reverses array twice if `options.step` is negative and the interval is negative too\", () => {\n      const result = eachMinuteOfInterval(\n        {\n          start: new Date(2020, 10, 14, 13, 5),\n          end: new Date(2020, 10, 14, 13, 0),\n        },\n        { step: -1 },\n      );\n\n      expect(result).toEqual([\n        new Date(2020, 10, 14, 13, 0),\n        new Date(2020, 10, 14, 13, 1),\n        new Date(2020, 10, 14, 13, 2),\n        new Date(2020, 10, 14, 13, 3),\n        new Date(2020, 10, 14, 13, 4),\n        new Date(2020, 10, 14, 13, 5),\n      ]);\n    });\n\n    it(\"returns empty array if `options.step` is less than 1\", () => {\n      const result = eachMinuteOfInterval(interval, { step: 0 });\n      expect(result).toEqual([]);\n    });\n\n    it(\"returns empty array if `options.step` is NaN\", () => {\n      const result = eachMinuteOfInterval(interval, { step: NaN });\n      expect(result).toEqual([]);\n    });\n  });\n});\n\n// Path: src/startOfDay/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { startOfDay } from \"./index.js\";\n\ndescribe(\"startOfDay\", () => {\n  it(\"returns the date with the time set to 00:00:00\", () => {\n    const date = new Date(2014, 8 /* Sep */, 2, 11, 55, 0);\n    const result = startOfDay(date);\n    expect(result).toEqual(new Date(2014, 8 /* Sep */, 2, 0, 0, 0, 0));\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const date = new Date(2014, 8 /* Sep */, 2, 11, 55, 0).getTime();\n    const result = startOfDay(date);\n    expect(result).toEqual(new Date(2014, 8 /* Sep */, 2, 0, 0, 0, 0));\n  });\n\n  it(\"does not mutate the original date\", () => {\n    const date = new Date(2014, 8 /* Sep */, 2, 11, 55, 0);\n    startOfDay(date);\n    expect(date).toEqual(new Date(2014, 8 /* Sep */, 2, 11, 55, 0));\n  });\n\n  it(\"returns `Invalid Date` if the given date is invalid\", () => {\n    const result = startOfDay(new Date(NaN));\n    expect(result instanceof Date && isNaN(result.getTime())).toBe(true);\n  });\n});\n\n// Path: src/compareDesc/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { compareDesc } from \"./index.js\";\n\ndescribe(\"compareDesc\", () => {\n  it(\"returns 0 if the given dates are equal\", () => {\n    const result = compareDesc(\n      new Date(1989, 6 /* Jul */, 10),\n      new Date(1989, 6 /* Jul */, 10),\n    );\n    expect(result).toBe(0);\n  });\n\n  it(\"returns 1 if the first date is before the second one\", () => {\n    const result = compareDesc(\n      new Date(1987, 1 /* Feb */, 11),\n      new Date(1989, 6 /* Jul */, 10),\n    );\n    expect(result).toBe(1);\n  });\n\n  it(\"returns -1 if the first date is after the second one\", () => {\n    const result = compareDesc(\n      new Date(1989, 6 /* Jul */, 10),\n      new Date(1987, 1 /* Feb */, 11),\n    );\n    expect(result).toBe(-1);\n  });\n\n  it(\"sorts the dates array in the reverse chronological order when function is passed as the argument to Array.prototype.sort()\", () => {\n    const unsortedArray = [\n      new Date(1995, 6 /* Jul */, 2),\n      new Date(1987, 1 /* Feb */, 11),\n      new Date(1989, 6 /* Jul */, 10),\n    ];\n\n    const sortedArray = [\n      new Date(1995, 6 /* Jul */, 2),\n      new Date(1989, 6 /* Jul */, 10),\n      new Date(1987, 1 /* Feb */, 11),\n    ];\n\n    unsortedArray.sort(compareDesc);\n    const result = unsortedArray;\n\n    expect(result).toEqual(sortedArray);\n  });\n\n  it(\"accepts timestamps\", () => {\n    const result = compareDesc(\n      new Date(1987, 1 /* Feb */, 11).getTime(),\n      new Date(1989, 6 /* Jul */, 10).getTime(),\n    );\n    expect(result).toBe(1);\n  });\n\n  it(\"returns NaN if the first date is `Invalid Date`\", () => {\n    const result = compareDesc(new Date(NaN), new Date(1989, 6 /* Jul */, 10));\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the second date is `Invalid Date`\", () => {\n    const result = compareDesc(new Date(1989, 6 /* Jul */, 10), new Date(NaN));\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the both dates are `Invalid Date`\", () => {\n    const result = compareDesc(new Date(1989, 6 /* Jul */, 10), new Date(NaN));\n    expect(isNaN(result)).toBe(true);\n  });\n});\n\n// Path: src/isSaturday/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { isSaturday } from \"./index.js\";\n\ndescribe(\"isSaturday\", () => {\n  it(\"returns true if the given date is Saturday\", () => {\n    const result = isSaturday(new Date(2014, 8 /* Sep */, 27));\n    expect(result).toBe(true);\n  });\n\n  it(\"returns false if the given date is not Saturday\", () => {\n    const result = isSaturday(new Date(2014, 8 /* Sep */, 25));\n    expect(result).toBe(false);\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const result = isSaturday(new Date(2014, 1 /* Feb */, 15).getTime());\n    expect(result).toBe(true);\n  });\n\n  it(\"returns false if the given date is `Invalid Date`\", () => {\n    const result = isSaturday(new Date(NaN));\n    expect(result).toBe(false);\n  });\n});\n\n// Path: src/endOfISOWeekYear/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { endOfISOWeekYear } from \"./index.js\";\n\ndescribe(\"endOfISOWeekYear\", () => {\n  it(\"returns the date with the time set to 23:59:59.999 and the date set to the last day of an ISO year\", () => {\n    const result = endOfISOWeekYear(new Date(2009, 0 /* Jan */, 1, 16, 0));\n    expect(result).toEqual(new Date(2010, 0 /* Jan */, 3, 23, 59, 59, 999));\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const result = endOfISOWeekYear(\n      new Date(2005, 0 /* Jan */, 1, 6, 0).getTime(),\n    );\n    expect(result).toEqual(new Date(2005, 0 /* Jan */, 2, 23, 59, 59, 999));\n  });\n\n  it(\"does not mutate the original date\", () => {\n    const date = new Date(2014, 6 /* Jul */, 2);\n    endOfISOWeekYear(date);\n    expect(date).toEqual(new Date(2014, 6 /* Jul */, 2));\n  });\n\n  it(\"handles dates before 100 AD\", () => {\n    const initialDate = new Date(0);\n    initialDate.setFullYear(5, 0 /* Jan */, 4);\n    initialDate.setHours(16, 0, 0, 0);\n    const expectedResult = new Date(0);\n    expectedResult.setFullYear(6, 0 /* Jan */, 1);\n    expectedResult.setHours(23, 59, 59, 999);\n    const result = endOfISOWeekYear(initialDate);\n    expect(result).toEqual(expectedResult);\n  });\n\n  it(\"returns `Invalid Date` if the given date is invalid\", () => {\n    const result = endOfISOWeekYear(new Date(NaN));\n    expect(result instanceof Date && isNaN(result.getTime())).toBe(true);\n  });\n});\n\n// Path: src/intlFormatDistance/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { intlFormatDistance } from \"./index.js\";\n\ndescribe(\"intlFormatDistance\", () => {\n  describe(\"with default values\", () => {\n    describe(\"works with same dates\", () => {\n      it(\"outputs `now`\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 0),\n          new Date(1986, 3, 5, 10, 30, 0),\n        );\n        expect(result).toBe(\"now\");\n      });\n    });\n\n    describe(\"works with single second\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 1),\n          new Date(1986, 3, 5, 10, 30, 0),\n        );\n        expect(result).toBe(\"in 1 second\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 0),\n          new Date(1986, 3, 5, 10, 30, 1),\n        );\n        expect(result).toBe(\"1 second ago\");\n      });\n    });\n\n    describe(\"works with multiple seconds\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 59),\n          new Date(1986, 3, 5, 10, 30, 0),\n        );\n        expect(result).toBe(\"in 59 seconds\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 0),\n          new Date(1986, 3, 5, 10, 30, 59),\n        );\n        expect(result).toBe(\"59 seconds ago\");\n      });\n    });\n\n    describe(\"works with single minute\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 59),\n          new Date(1986, 3, 5, 10, 29, 59),\n        );\n        expect(result).toBe(\"in 1 minute\");\n      });\n\n      it(\"works with future with over a minute\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30),\n          new Date(1986, 3, 5, 10, 28, 10),\n        );\n        expect(result).toBe(\"in 1 minute\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 29, 59),\n          new Date(1986, 3, 5, 10, 30, 59),\n        );\n        expect(result).toBe(\"1 minute ago\");\n      });\n\n      it(\"works with past with over a minute\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 28, 10),\n          new Date(1986, 3, 5, 10, 30),\n        );\n        expect(result).toBe(\"1 minute ago\");\n      });\n    });\n\n    describe(\"works with multiple minutes\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30),\n          new Date(1986, 3, 5, 10, 28),\n        );\n        expect(result).toBe(\"in 2 minutes\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 28),\n          new Date(1986, 3, 5, 10, 30),\n        );\n        expect(result).toBe(\"2 minutes ago\");\n      });\n    });\n\n    describe(\"works with 60 seconds\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 4, 11, 30, 0),\n          new Date(1986, 3, 4, 10, 30, 0),\n        );\n        expect(result).toBe(\"in 1 hour\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 4, 10, 30, 0),\n          new Date(1986, 3, 4, 11, 30, 0),\n        );\n        expect(result).toBe(\"1 hour ago\");\n      });\n    });\n\n    describe(\"works with single hour\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10),\n          new Date(1986, 3, 5, 9),\n        );\n        expect(result).toBe(\"in 1 hour\");\n      });\n\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30),\n          new Date(1986, 3, 5, 9),\n        );\n        expect(result).toBe(\"in 1 hour\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 9),\n          new Date(1986, 3, 5, 10),\n        );\n        expect(result).toBe(\"1 hour ago\");\n      });\n\n      it(\"works with past with over an hour\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 9),\n          new Date(1986, 3, 5, 10, 30),\n        );\n        expect(result).toBe(\"1 hour ago\");\n      });\n    });\n\n    describe(\"works with multiple hours\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10),\n          new Date(1986, 3, 5, 8),\n        );\n        expect(result).toBe(\"in 2 hours\");\n      });\n\n      it(\"works with future with extra minutes\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30),\n          new Date(1986, 3, 5, 8),\n        );\n        expect(result).toBe(\"in 2 hours\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 8),\n          new Date(1986, 3, 5, 10),\n        );\n        expect(result).toBe(\"2 hours ago\");\n      });\n\n      it(\"works with past with extra minutes\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 8),\n          new Date(1986, 3, 5, 10),\n        );\n        expect(result).toBe(\"2 hours ago\");\n      });\n    });\n\n    describe(\"works with single day\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 6, 22),\n          new Date(1986, 3, 5, 22),\n        );\n        expect(result).toBe(\"tomorrow\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 22),\n          new Date(1986, 3, 6, 22),\n        );\n        expect(result).toBe(\"yesterday\");\n      });\n\n      it(\"works with past with an extra hour\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 22),\n          new Date(1986, 3, 6, 22, 5),\n        );\n        expect(result).toBe(\"yesterday\");\n      });\n    });\n\n    describe(\"works with multiple days\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 6, 22),\n          new Date(1986, 3, 4, 22),\n        );\n        expect(result).toBe(\"in 2 days\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 4, 22),\n          new Date(1986, 3, 6, 22),\n        );\n        expect(result).toBe(\"2 days ago\");\n      });\n    });\n\n    describe(\"works with single week\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 10, 22),\n          new Date(1986, 3, 3, 22),\n        );\n        expect(result).toBe(\"next week\");\n      });\n\n      it(\"works with future with more than a week\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 11, 22),\n          new Date(1986, 3, 3, 22),\n        );\n        expect(result).toBe(\"next week\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 3, 22),\n          new Date(1986, 3, 10, 22),\n        );\n        expect(result).toBe(\"last week\");\n      });\n\n      it(\"works with past with more than a week\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 3, 22),\n          new Date(1986, 3, 11, 22),\n        );\n        expect(result).toBe(\"last week\");\n      });\n    });\n\n    describe(\"works with multiple weeks\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 15),\n          new Date(1986, 3, 1),\n        );\n        expect(result).toBe(\"in 2 weeks\");\n      });\n\n      it(\"works with future with more than 2 weeks\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 17),\n          new Date(1986, 3, 1),\n        );\n        expect(result).toBe(\"in 2 weeks\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 1),\n          new Date(1986, 3, 15),\n        );\n        expect(result).toBe(\"2 weeks ago\");\n      });\n\n      it(\"works with past with more than 2 weeks\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 1),\n          new Date(1986, 3, 17),\n        );\n        expect(result).toBe(\"2 weeks ago\");\n      });\n    });\n\n    describe(\"works with single month\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 4, 2),\n          new Date(1986, 3, 1),\n        );\n        expect(result).toBe(\"next month\");\n      });\n\n      it(\"works with future with more than a month\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 4, 22),\n          new Date(1986, 3, 1),\n        );\n        expect(result).toBe(\"next month\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 1),\n          new Date(1986, 4, 2),\n        );\n        expect(result).toBe(\"last month\");\n      });\n\n      it(\"works with past with more than a month\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 1),\n          new Date(1986, 4, 22),\n        );\n        expect(result).toBe(\"last month\");\n      });\n    });\n\n    describe(\"works with single quarter\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 5, 2),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"next quarter\");\n      });\n\n      it(\"works with future with more than a quarter\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 5, 22),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"next quarter\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 1, 1),\n          new Date(1986, 5, 2),\n        );\n        expect(result).toBe(\"last quarter\");\n      });\n\n      it(\"works with past with more than a quarter\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 1, 1),\n          new Date(1986, 5, 22),\n        );\n        expect(result).toBe(\"last quarter\");\n      });\n    });\n\n    describe(\"works with multiple quarters\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 6, 2),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"in 2 quarters\");\n      });\n\n      it(\"works with future with more that X quarters\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 6, 22),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"in 2 quarters\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 1, 1),\n          new Date(1986, 6, 2),\n        );\n        expect(result).toBe(\"2 quarters ago\");\n      });\n\n      it(\"works with past with more that X quarters\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 1, 1),\n          new Date(1986, 6, 22),\n        );\n        expect(result).toBe(\"2 quarters ago\");\n      });\n    });\n\n    describe(\"works with single year\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1987, 1, 2),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"next year\");\n      });\n\n      it(\"works with future with more that a year\", () => {\n        const result = intlFormatDistance(\n          new Date(1987, 10, 2),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"next year\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 1, 1),\n          new Date(1987, 1, 2),\n        );\n        expect(result).toBe(\"last year\");\n      });\n\n      it(\"works with past with more than a year\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 1, 1),\n          new Date(1987, 10, 2),\n        );\n        expect(result).toBe(\"last year\");\n      });\n    });\n\n    describe(\"works with multiple years\", () => {\n      it(\"works with future\", () => {\n        const result = intlFormatDistance(\n          new Date(1988, 1, 1),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"in 2 years\");\n      });\n\n      it(\"works with future with x years\", () => {\n        const result = intlFormatDistance(\n          new Date(2088, 1, 1),\n          new Date(1986, 1, 1),\n        );\n        expect(result).toBe(\"in 102 years\");\n      });\n\n      it(\"works with past\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 1, 1),\n          new Date(1988, 1, 1),\n        );\n        expect(result).toBe(\"2 years ago\");\n      });\n\n      it(\"works with past with x years\", () => {\n        const result = intlFormatDistance(\n          new Date(1988, 1, 1),\n          new Date(2086, 1, 1),\n        );\n        expect(result).toBe(\"98 years ago\");\n      });\n    });\n  });\n\n  describe(\"with options\", () => {\n    describe(\"unit option\", () => {\n      describe(\"seconds\", () => {\n        it(\"works with future with seconds\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 3, 4, 10, 31, 30),\n            new Date(1987, 3, 4, 10, 30, 0),\n            { unit: \"second\" },\n          );\n          expect(result).toBe(\"in 90 seconds\");\n        });\n\n        it(\"works with past with seconds\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 3, 4, 10, 30, 0),\n            new Date(1987, 3, 4, 10, 31, 30),\n            { unit: \"second\" },\n          );\n          expect(result).toBe(\"90 seconds ago\");\n        });\n\n        it(\"works with future with quarters\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 6, 4, 10, 30, 0),\n            new Date(1986, 3, 4, 10, 30, 0),\n            { unit: \"quarter\" },\n          );\n          expect(result).toBe(\"in 5 quarters\");\n        });\n      });\n\n      describe(\"minutes\", () => {\n        it(\"works with future\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 4, 11, 30, 0),\n            new Date(1986, 3, 4, 10, 30, 0),\n            { unit: \"minute\" },\n          );\n          expect(result).toBe(\"in 60 minutes\");\n        });\n\n        it(\"works with the past\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 4, 10, 30, 0),\n            new Date(1986, 3, 4, 11, 30, 0),\n            { unit: \"minute\" },\n          );\n          expect(result).toBe(\"60 minutes ago\");\n        });\n      });\n\n      describe(\"hours\", () => {\n        it(\"works the future\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 4, 11, 30, 0),\n            new Date(1986, 3, 4, 10, 30, 0),\n\n            { unit: \"hour\" },\n          );\n          expect(result).toBe(\"in 1 hour\");\n        });\n\n        it(\"works with the past\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 4, 10, 30, 0),\n            new Date(1986, 3, 4, 11, 30, 0),\n\n            { unit: \"hour\" },\n          );\n          expect(result).toBe(\"1 hour ago\");\n        });\n      });\n\n      describe(\"single day\", () => {\n        it(\"works with the future\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 3, 4, 11, 30, 0),\n            new Date(1987, 3, 4, 10, 30, 0),\n            { unit: \"day\" },\n          );\n          expect(result).toBe(\"today\");\n        });\n\n        it(\"works with the past\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 5, 10, 30, 0),\n            new Date(1986, 3, 4, 10, 30, 0),\n          );\n          expect(result).toBe(\"tomorrow\");\n        });\n      });\n\n      describe(\"multiple days\", () => {\n        it(\"works with the future\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 3, 5, 10, 30, 0),\n            new Date(1986, 3, 4, 10, 30, 0),\n            { unit: \"day\" },\n          );\n          expect(result).toBe(\"in 366 days\");\n        });\n\n        it(\"works with the past\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 4, 10, 30, 0),\n            new Date(1987, 3, 5, 10, 30, 0),\n\n            { unit: \"day\" },\n          );\n          expect(result).toBe(\"366 days ago\");\n        });\n      });\n\n      describe(\"single weeks\", () => {\n        it(\"works with the future\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 3, 11, 10, 30, 0),\n            new Date(1987, 3, 4, 10, 30, 0),\n            { unit: \"week\" },\n          );\n          expect(result).toBe(\"next week\");\n        });\n\n        it(\"works with the past\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 3, 4, 10, 30, 0),\n            new Date(1987, 3, 11, 10, 30, 0),\n            { unit: \"week\" },\n          );\n          expect(result).toBe(\"last week\");\n        });\n      });\n\n      describe(\"multiple weeks\", () => {\n        it(\"works with the future\", () => {\n          const result = intlFormatDistance(\n            new Date(1987, 3, 6, 10, 30, 0),\n            new Date(1986, 3, 4, 10, 30, 0),\n            { unit: \"week\" },\n          );\n          expect(result).toBe(\"in 53 weeks\");\n        });\n\n        it(\"works with the past\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 4, 10, 30, 0),\n            new Date(1987, 3, 6, 10, 30, 0),\n\n            { unit: \"week\" },\n          );\n          expect(result).toBe(\"53 weeks ago\");\n        });\n      });\n    });\n\n    describe(\"numeric option\", () => {\n      describe(\"works with weeks\", () => {\n        it(\"works with past\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 3, 22),\n            new Date(1986, 3, 10, 22),\n            { numeric: \"always\" },\n          );\n          expect(result).toBe(\"1 week ago\");\n        });\n\n        it(\"works with future\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 3, 10, 22),\n            new Date(1986, 3, 3, 22),\n            { numeric: \"always\" },\n          );\n          expect(result).toBe(\"in 1 week\");\n        });\n      });\n\n      it(\"works with days\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 0),\n          new Date(1986, 3, 4, 10, 30, 0),\n          { numeric: \"always\" },\n        );\n        expect(result).toBe(\"in 1 day\");\n      });\n\n      it(\"works with the same dates\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 10, 30, 0),\n          new Date(1986, 3, 5, 10, 30, 0),\n          { numeric: \"auto\" },\n        );\n        expect(result).toBe(\"now\");\n      });\n    });\n\n    describe(\"locale option\", () => {\n      describe(\"locale\", () => {\n        it(\"allows to set Spanish locale\", () => {\n          const result = intlFormatDistance(\n            new Date(1986, 4, 4, 10, 30, 0),\n            new Date(1985, 4, 4, 10, 30, 0),\n            { locale: \"es\" },\n          );\n          expect(result).toBe(\"el pr\u00f3ximo a\u00f1o\");\n        });\n      });\n    });\n\n    describe(\"style option\", () => {\n      it(\"works with years\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 4, 4, 10, 30, 0),\n          new Date(1985, 4, 4, 10, 30, 0),\n          { style: \"long\" },\n        );\n        expect(result).toBe(\"next year\");\n      });\n    });\n\n    describe(\"unit and locale options\", () => {\n      it(\"works with multiple options\", () => {\n        const result = intlFormatDistance(\n          new Date(1986, 3, 5, 11, 30, 0),\n          new Date(1986, 3, 5, 10, 30, 0),\n          { unit: \"minute\", locale: \"de\" },\n        );\n        expect(result).toBe(\"in 60 Minuten\");\n      });\n    });\n\n    describe(\"edge cases\", () => {\n      it(\"falls back to { numeric: always }\", () => {\n        const result = intlFormatDistance(\n          new Date(1985, 4, 5, 10, 30, 0),\n          new Date(1985, 4, 4, 10, 30, 0),\n          { style: \"long\", numeric: \"auto\" },\n        );\n        expect(result).toBe(\"tomorrow\");\n      });\n\n      it(\"handles dates before 100 AD\", () => {\n        const result = intlFormatDistance(\n          new Date(1, 3, 4, 11, 30, 0),\n          new Date(1, 3, 4, 10, 30, 0),\n          { unit: \"minute\" },\n        );\n        expect(result).toBe(\"in 60 minutes\");\n      });\n    });\n\n    describe(\"errors\", () => {\n      it(\"checks the first date\", () => {\n        expect(intlFormatDistance.bind(\n          null,\n          new Date(NaN),\n          new Date(1986, 3, 4, 10, 30, 0),\n        )).toThrow(RangeError);\n      });\n\n      it(\"checks the second date\", () => {\n        expect(intlFormatDistance.bind(\n          null,\n          new Date(1986, 3, 4, 10, 30, 0),\n          new Date(NaN),\n        )).toThrow(RangeError);\n      });\n\n      it(\"checks both dates\", () => {\n        expect(intlFormatDistance.bind(null, new Date(NaN), new Date(NaN))).toThrow(RangeError);\n      });\n\n      it(\"checks unit\", () => {\n        expect(intlFormatDistance.bind(\n          null,\n          new Date(1986, 3, 4, 10, 30, 0),\n...\n", "needle_token_start": 721, "needle_token_end": 893, "code_context_ntokens": 16206, "output": ["```javascript\nfunction execScript(scriptText) {\n  const iframe = document.querySelector(\"iframe#iframe\");\n  if (!iframe || !(iframe instanceof HTMLIFrameElement)) {\n    throw new Error(\"Can't execute the script because iframe isn't found\");\n  }\n  const doc = iframe.contentDocument!;\n  const script = doc.createElement(\"script\");\n  script.innerText = scriptText;\n  // @ts-expect-error - We're messing with iframes\n  if (!(doc.body instanceof iframe.contentWindow.HTMLBodyElement)) {\n    throw new Error(\n      \"Can't execute the script because iframe does not have body\",\n    );\n  }\n  doc.body.append(script);\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "addHours", "language": "typescript", "path": "src/addHours/index.ts", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To increase a given date by a specified number of hours.\n2. **Input**: A date (which can be a `Date` object, a number representing a timestamp, or a string that can be converted to a date) and a numerical value representing the number of hours to add.\n3. **Output**: A new date object that represents the original date plus the additional hours.\n4. **Procedure**: The function first converts the input into a date object if it is not already one. It then calculates the total number of milliseconds to add by multiplying the number of hours by the number of milliseconds in one hour. This total is added to the original date's timestamp to produce a new date.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/getISOWeeksInYear/index.ts\nimport { addWeeks } from \"../addWeeks/index.js\";\nimport { millisecondsInWeek } from \"../constants/index.js\";\nimport { startOfISOWeekYear } from \"../startOfISOWeekYear/index.js\";\n\n/**\n * @name getISOWeeksInYear\n * @category ISO Week-Numbering Year Helpers\n * @summary Get the number of weeks in an ISO week-numbering year of the given date.\n *\n * @description\n * Get the number of weeks in an ISO week-numbering year of the given date.\n *\n * ISO week-numbering year: http://en.wikipedia.org/wiki/ISO_week_date\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The number of ISO weeks in a year\n *\n * @example\n...\n// Path: src/formatRFC3339/index.ts\nimport { isValid } from \"../isValid/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport { addLeadingZeros } from \"../_lib/addLeadingZeros/index.js\";\n\n/**\n * The {@link formatRFC3339} function options.\n */\nexport interface FormatRFC3339Options {\n  /** The number of digits after the decimal point after seconds, defaults to 0 */\n  fractionDigits?: 0 | 1 | 2 | 3;\n}\n\n/**\n * @name formatRFC3339\n * @category Common Helpers\n * @summary Format the date according to the RFC 3339 standard (https://tools.ietf.org/html/rfc3339#section-5.6).\n *\n * @description\n * Return the formatted date string in RFC 3339 format. Options may be passed to control the parts and notations of the date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n * @param options - An object with options.\n *\n * @returns The formatted date string\n *\n * @throws `date` must not be Invalid Date\n *\n * @example\n * // Represent 18 September 2019 in RFC 3339 format:\n * formatRFC3339(new Date(2019, 8, 18, 19, 0, 52))\n * //=> '2019-09-18T19:00:52Z'\n *\n * @example\n * // Represent 18 September 2019 in RFC 3339 format, 3 digits of second fraction\n * formatRFC3339(new Date(2019, 8, 18, 19, 0, 52, 234), {\n *   fractionDigits: 3\n * })\n * //=> '2019-09-18T19:00:52.234Z'\n */\nexport function formatRFC3339<DateType extends Date>(\n  date: DateType | number | string,\n  options?: FormatRFC3339Options,\n): string {\n  const _date = toDate(date);\n\n  if (!isValid(_date)) {\n    throw new RangeError(\"Invalid time value\");\n  }\n\n  const fractionDigits = options?.fractionDigits ?? 0;\n\n  const day = addLeadingZeros(_date.getDate(), 2);\n  const month = addLeadingZeros(_date.getMonth() + 1, 2);\n  const year = _date.getFullYear();\n\n  const hour = addLeadingZeros(_date.getHours(), 2);\n  const minute = addLeadingZeros(_date.getMinutes(), 2);\n  const second = addLeadingZeros(_date.getSeconds(), 2);\n\n  let fractionalSecond = \"\";\n  if (fractionDigits > 0) {\n    const milliseconds = _date.getMilliseconds();\n    const fractionalSeconds = Math.trunc(\n      milliseconds * Math.pow(10, fractionDigits - 3),\n    );\n    fractionalSecond = \".\" + addLeadingZeros(fractionalSeconds, fractionDigits);\n  }\n\n  let offset = \"\";\n  const tzOffset = _date.getTimezoneOffset();\n\n  if (tzOffset !== 0) {\n    const absoluteOffset = Math.abs(tzOffset);\n    const hourOffset = addLeadingZeros(Math.trunc(absoluteOffset / 60), 2);\n    const minuteOffset = addLeadingZeros(absoluteOffset % 60, 2);\n    // If less than 0, the sign is +, because it is ahead of time.\n    const sign = tzOffset < 0 ? \"+\" : \"-\";\n\n    offset = `${sign}${hourOffset}:${minuteOffset}`;\n  } else {\n    offset = \"Z\";\n  }\n\n  return `${year}-${month}-${day}T${hour}:${minute}:${second}${fractionalSecond}${offset}`;\n}\n\n// Path: src/getMilliseconds/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getMilliseconds\n * @category Millisecond Helpers\n * @summary Get the milliseconds of the given date.\n *\n * @description\n * Get the milliseconds of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The milliseconds\n *\n * @example\n * // Get the milliseconds of 29 February 2012 11:45:05.123:\n * const result = getMilliseconds(new Date(2012, 1, 29, 11, 45, 5, 123))\n * //=> 123\n */\nexport function getMilliseconds<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const milliseconds = _date.getMilliseconds();\n  return milliseconds;\n}\n\n// Path: src/addMilliseconds/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\n\n/**\n * @name addMilliseconds\n * @category Millisecond Helpers\n * @summary Add the specified number of milliseconds to the given date.\n *\n * @description\n * Add the specified number of milliseconds to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of milliseconds to be added.\n *\n * @returns The new date with the milliseconds added\n *\n * @example\n * // Add 750 milliseconds to 10 July 2014 12:45:30.000:\n * const result = addMilliseconds(new Date(2014, 6, 10, 12, 45, 30, 0), 750)\n * //=> Thu Jul 10 2014 12:45:30.750\n */\nexport function addMilliseconds<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  const timestamp = +toDate(date);\n  return constructFrom(date, timestamp + amount);\n}\n\n// Path: src/addHours/index.ts\nimport { addMilliseconds } from \"../addMilliseconds/index.js\";\nimport { millisecondsInHour } from \"../constants/index.js\";\n\n/**\n * @name addHours\n * @category Hour Helpers\n * @summary Add the specified number of hours to the given date.\n *\n * @description\n * Add the specified number of hours to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of hours to be added.\n *\n * @returns The new date with the hours added\n *\n * @example\n * // Add 2 hours to 10 July 2014 23:00:00:\n * const result = addHours(new Date(2014, 6, 10, 23, 0), 2)\n * //=> Fri Jul 11 2014 01:00:00\n */\nexport \nfunction addHours<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addMilliseconds(date, amount * millisecondsInHour);\n}\n\n// Path: src/subHours/index.ts\nimport { addHours } from \"../addHours/index.js\";\n\n/**\n * @name subHours\n * @category Hour Helpers\n * @summary Subtract the specified number of hours from the given date.\n *\n * @description\n * Subtract the specified number of hours from the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of hours to be subtracted.\n *\n * @returns The new date with the hours subtracted\n *\n * @example\n * // Subtract 2 hours from 11 July 2014 01:00:00:\n * const result = subHours(new Date(2014, 6, 11, 1, 0), 2)\n * //=> Thu Jul 10 2014 23:00:00\n */\nexport function subHours<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addHours(date, -amount);\n}\n\n// Path: src/getTime/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getTime\n * @category Timestamp Helpers\n * @summary Get the milliseconds timestamp of the given date.\n *\n * @description\n * Get the milliseconds timestamp of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The timestamp\n *\n * @example\n * // Get the timestamp of 29 February 2012 11:45:05.123:\n * const result = getTime(new Date(2012, 1, 29, 11, 45, 5, 123))\n * //=> 1330515905123\n */\nexport function getTime<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const timestamp = _date.getTime();\n  return timestamp;\n}\n\n// Path: src/lastDayOfWeek/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport type { LocalizedOptions, WeekOptions } from \"../types.js\";\nimport { getDefaultOptions } from \"../_lib/defaultOptions/index.js\";\n\n/**\n * The {@link lastDayOfWeek} function options.\n */\nexport interface LastDayOfWeekOptions\n  extends LocalizedOptions<\"options\">,\n    WeekOptions {}\n\n/**\n * @name lastDayOfWeek\n * @category Week Helpers\n * @summary Return the last day of a week for the given date.\n *\n * @description\n * Return the last day of a week for the given date.\n * The result will be in the local timezone.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n * @param options - An object with options\n *\n * @returns The last day of a week\n *\n * @example\n * // The last day of a week for 2 September 2014 11:55:00:\n * const result = lastDayOfWeek(new Date(2014, 8, 2, 11, 55, 0))\n * //=> Sat Sep 06 2014 00:00:00\n *\n * @example\n * // If the week starts on Monday, the last day of the week for 2 September 2014 11:55:00:\n * const result = lastDayOfWeek(new Date(2014, 8, 2, 11, 55, 0), { weekStartsOn: 1 })\n * //=> Sun Sep 07 2014 00:00:00\n */\nexport function lastDayOfWeek<DateType extends Date>(\n  date: DateType | number | string,\n  options?: LastDayOfWeekOptions,\n): DateType {\n  const defaultOptions = getDefaultOptions();\n  const weekStartsOn =\n    options?.weekStartsOn ??\n    options?.locale?.options?.weekStartsOn ??\n    defaultOptions.weekStartsOn ??\n    defaultOptions.locale?.options?.weekStartsOn ??\n    0;\n\n  const _date = toDate(date);\n  const day = _date.getDay();\n  const diff = (day < weekStartsOn ? -7 : 0) + 6 - (day - weekStartsOn);\n\n  _date.setHours(0, 0, 0, 0);\n  _date.setDate(_date.getDate() + diff);\n  return _date;\n}\n\n// Path: src/lastDayOfISOWeek/index.ts\nimport { lastDayOfWeek } from \"../lastDayOfWeek/index.js\";\n\n/**\n * @name lastDayOfISOWeek\n * @category ISO Week Helpers\n * @summary Return the last day of an ISO week for the given date.\n *\n * @description\n * Return the last day of an ISO week for the given date.\n * The result will be in the local timezone.\n *\n * ISO week-numbering year: http://en.wikipedia.org/wiki/ISO_week_date\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The last day of an ISO week\n *\n * @example\n * // The last day of an ISO week for 2 September 2014 11:55:00:\n * const result = lastDayOfISOWeek(new Date(2014, 8, 2, 11, 55, 0))\n * //=> Sun Sep 07 2014 00:00:00\n */\nexport function lastDayOfISOWeek<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  return lastDayOfWeek(date, { weekStartsOn: 1 });\n}\n\n// Path: src/setISODay/index.ts\nimport { addDays } from \"../addDays/index.js\";\nimport { getISODay } from \"../getISODay/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setISODay\n * @category Weekday Helpers\n * @summary Set the day of the ISO week to the given date.\n *\n * @description\n * Set the day of the ISO week to the given date.\n * ISO week starts with Monday.\n * 7 is the index of Sunday, 1 is the index of Monday etc.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param day - The day of the ISO week of the new date\n *\n * @returns The new date with the day of the ISO week set\n *\n * @example\n * // Set Sunday to 1 September 2014:\n * const result = setISODay(new Date(2014, 8, 1), 7)\n * //=> Sun Sep 07 2014 00:00:00\n */\nexport function setISODay<DateType extends Date>(\n  date: DateType | number | string,\n  day: number,\n): DateType {\n  const _date = toDate(date);\n  const currentDay = getISODay(_date);\n  const diff = day - currentDay;\n  return addDays(_date, diff);\n}\n\n// Path: src/eachYearOfInterval/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport type { Interval, StepOptions } from \"../types.js\";\n\n/**\n * The {@link eachYearOfInterval} function options.\n */\nexport interface EachYearOfIntervalOptions extends StepOptions {}\n\n/**\n * @name eachYearOfInterval\n * @category Interval Helpers\n * @summary Return the array of yearly timestamps within the specified time interval.\n *\n * @description\n * Return the array of yearly timestamps within the specified time interval.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param interval - The interval.\n *\n * @returns The array with starts of yearly timestamps from the month of the interval start to the month of the interval end\n *\n * @example\n * // Each year between 6 February 2014 and 10 August 2017:\n * const result = eachYearOfInterval({\n *   start: new Date(2014, 1, 6),\n *   end: new Date(2017, 7, 10)\n * })\n * //=> [\n * //   Wed Jan 01 2014 00:00:00,\n * //   Thu Jan 01 2015 00:00:00,\n * //   Fri Jan 01 2016 00:00:00,\n * //   Sun Jan 01 2017 00:00:00\n * // ]\n */\nexport function eachYearOfInterval<DateType extends Date>(\n  interval: Interval<DateType>,\n  options?: EachYearOfIntervalOptions,\n): DateType[] {\n  const startDate = toDate(interval.start);\n  const endDate = toDate(interval.end);\n\n  let reversed = +startDate > +endDate;\n  const endTime = reversed ? +startDate : +endDate;\n  const currentDate = reversed ? endDate : startDate;\n  currentDate.setHours(0, 0, 0, 0);\n  currentDate.setMonth(0, 1);\n\n  let step = options?.step ?? 1;\n  if (!step) return [];\n  if (step < 0) {\n    step = -step;\n    reversed = !reversed;\n  }\n\n  const dates = [];\n\n  while (+currentDate <= endTime) {\n    dates.push(toDate(currentDate));\n    currentDate.setFullYear(currentDate.getFullYear() + step);\n  }\n\n  return reversed ? dates.reverse() : dates;\n}\n\n// Path: src/endOfISOWeekYear/index.ts\nimport { getISOWeekYear } from \"../getISOWeekYear/index.js\";\nimport { startOfISOWeek } from \"../startOfISOWeek/index.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\n\n/**\n * @name endOfISOWeekYear\n * @category ISO Week-Numbering Year Helpers\n * @summary Return the end of an ISO week-numbering year for the given date.\n *\n * @description\n * Return the end of an ISO week-numbering year,\n * which always starts 3 days before the year's first Thursday.\n * The result will be in the local timezone.\n *\n * ISO week-numbering year: http://en.wikipedia.org/wiki/ISO_week_date\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The end of an ISO week-numbering year\n *\n * @example\n * // The end of an ISO week-numbering year for 2 July 2005:\n * const result = endOfISOWeekYear(new Date(2005, 6, 2))\n * //=> Sun Jan 01 2006 23:59:59.999\n */\nexport function endOfISOWeekYear<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  const year = getISOWeekYear(date);\n  const fourthOfJanuaryOfNextYear = constructFrom(date, 0);\n  fourthOfJanuaryOfNextYear.setFullYear(year + 1, 0, 4);\n  fourthOfJanuaryOfNextYear.setHours(0, 0, 0, 0);\n  const _date = startOfISOWeek(fourthOfJanuaryOfNextYear);\n  _date.setMilliseconds(_date.getMilliseconds() - 1);\n  return _date;\n}\n\n// Path: src/addQuarters/index.ts\nimport { addMonths } from \"../addMonths/index.js\";\n\n/**\n * @name addQuarters\n * @category Quarter Helpers\n * @summary Add the specified number of year quarters to the given date.\n *\n * @description\n * Add the specified number of year quarters to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of quarters to be added.\n *\n * @returns The new date with the quarters added\n *\n * @example\n * // Add 1 quarter to 1 September 2014:\n * const result = addQuarters(new Date(2014, 8, 1), 1)\n * //=> Mon Dec 01 2014 00:00:00\n */\nexport function addQuarters<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  const months = amount * 3;\n  return addMonths(date, months);\n}\n\n// Path: src/subQuarters/index.ts\nimport { addQuarters } from \"../addQuarters/index.js\";\n\n/**\n * @name subQuarters\n * @category Quarter Helpers\n * @summary Subtract the specified number of year quarters from the given date.\n *\n * @description\n * Subtract the specified number of year quarters from the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of quarters to be subtracted.\n *\n * @returns The new date with the quarters subtracted\n *\n * @example\n * // Subtract 3 quarters from 1 September 2014:\n * const result = subQuarters(new Date(2014, 8, 1), 3)\n * //=> Sun Dec 01 2013 00:00:00\n */\nexport function subQuarters<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addQuarters(date, -amount);\n}\n\n// Path: src/isBefore/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isBefore\n * @category Common Helpers\n * @summary Is the first date before the second one?\n *\n * @description\n * Is the first date before the second one?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date that should be before the other one to return true\n * @param dateToCompare - The date to compare with\n *\n * @returns The first date is before the second date\n *\n * @example\n * // Is 10 July 1989 before 11 February 1987?\n * const result = isBefore(new Date(1989, 6, 10), new Date(1987, 1, 11))\n * //=> false\n */\nexport function isBefore<DateType extends Date>(\n  date: DateType | number | string,\n  dateToCompare: DateType | number | string,\n): boolean {\n  const _date = toDate(date);\n  const _dateToCompare = toDate(dateToCompare);\n  return +_date < +_dateToCompare;\n}\n\n// Path: src/getQuarter/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getQuarter\n * @category Quarter Helpers\n * @summary Get the year quarter of the given date.\n *\n * @description\n * Get the year quarter of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The quarter\n *\n * @example\n * // Which quarter is 2 July 2014?\n * const result = getQuarter(new Date(2014, 6, 2))\n * //=> 3\n */\nexport function getQuarter<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const quarter = Math.trunc(_date.getMonth() / 3) + 1;\n  return quarter;\n}\n\n// Path: src/differenceInCalendarQuarters/index.ts\nimport { getQuarter } from \"../getQuarter/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name differenceInCalendarQuarters\n * @category Quarter Helpers\n * @summary Get the number of calendar quarters between the given dates.\n *\n * @description\n * Get the number of calendar quarters between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n\n * @returns The number of calendar quarters\n *\n * @example\n * // How many calendar quarters are between 31 December 2013 and 2 July 2014?\n * const result = differenceInCalendarQuarters(\n *   new Date(2014, 6, 2),\n *   new Date(2013, 11, 31)\n * )\n * //=> 3\n */\nexport function differenceInCalendarQuarters<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n): number {\n  const _dateLeft = toDate(dateLeft);\n  const _dateRight = toDate(dateRight);\n\n  const yearDiff = _dateLeft.getFullYear() - _dateRight.getFullYear();\n  const quarterDiff = getQuarter(_dateLeft) - getQuarter(_dateRight);\n\n  return yearDiff * 4 + quarterDiff;\n}\n\n// Path: src/formatDuration/index.ts\nimport type { FormatDistanceToken } from \"../locale/types.js\";\nimport type { Duration, DurationUnit, LocalizedOptions } from \"../types.js\";\nimport { defaultLocale } from \"../_lib/defaultLocale/index.js\";\nimport { getDefaultOptions } from \"../_lib/defaultOptions/index.js\";\n\n/**\n * The {@link formatDuration} function options.\n */\nexport interface FormatDurationOptions\n  extends LocalizedOptions<\"formatDistance\"> {\n  /** The array of units to format */\n  format?: DurationUnit[];\n  /** Should be zeros be included in the output? */\n  zero?: boolean;\n  /** The delimiter string to use */\n  delimiter?: string;\n}\n\nconst defaultFormat: DurationUnit[] = [\n  \"years\",\n  \"months\",\n  \"weeks\",\n  \"days\",\n  \"hours\",\n  \"minutes\",\n  \"seconds\",\n];\n\n/**\n * @name formatDuration\n * @category Common Helpers\n * @summary Formats a duration in human-readable format\n *\n * @description\n * Return human-readable duration string i.e. \"9 months 2 days\"\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param duration - The duration to format\n * @param options - An object with options.\n *\n * @returns The formatted date string\n *\n * @example\n * // Format full duration\n * formatDuration({\n *   years: 2,\n *   months: 9,\n *   weeks: 1,\n *   days: 7,\n *   hours: 5,\n *   minutes: 9,\n *   seconds: 30\n * })\n * //=> '2 years 9 months 1 week 7 days 5 hours 9 minutes 30 seconds'\n *\n * @example\n * // Format partial duration\n * formatDuration({ months: 9, days: 2 })\n * //=> '9 months 2 days'\n *\n * @example\n * // Customize the format\n * formatDuration(\n *   {\n *     years: 2,\n *     months: 9,\n *     weeks: 1,\n *     days: 7,\n *     hours: 5,\n *     minutes: 9,\n *     seconds: 30\n *   },\n *   { format: ['months', 'weeks'] }\n * ) === '9 months 1 week'\n *\n * @example\n * // Customize the zeros presence\n * formatDuration({ years: 0, months: 9 })\n * //=> '9 months'\n * formatDuration({ years: 0, months: 9 }, { zero: true })\n * //=> '0 years 9 months'\n *\n * @example\n * // Customize the delimiter\n * formatDuration({ years: 2, months: 9, weeks: 3 }, { delimiter: ', ' })\n * //=> '2 years, 9 months, 3 weeks'\n */\nexport function formatDuration(\n  duration: Duration,\n  options?: FormatDurationOptions,\n): string {\n  const defaultOptions = getDefaultOptions();\n  const locale = options?.locale ?? defaultOptions.locale ?? defaultLocale;\n  const format = options?.format ?? defaultFormat;\n  const zero = options?.zero ?? false;\n  const delimiter = options?.delimiter ?? \" \";\n\n  if (!locale.formatDistance) {\n    return \"\";\n  }\n\n  const result = format\n    .reduce((acc, unit) => {\n      const token = `x${unit.replace(/(^.)/, (m) =>\n        m.toUpperCase(),\n      )}` as FormatDistanceToken;\n      const value = duration[unit];\n      if (value !== undefined && (zero || duration[unit])) {\n        return acc.concat(locale.formatDistance(token, value));\n      }\n      return acc;\n    }, [] as string[])\n    .join(delimiter);\n\n  return result;\n}\n\n// Path: src/eachHourOfInterval/index.ts\nimport { addHours } from \"../addHours/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { Interval, StepOptions } from \"../types.js\";\n\n/**\n * The {@link eachHourOfInterval} function options.\n */\nexport interface EachHourOfIntervalOptions extends StepOptions {}\n\n/**\n * @name eachHourOfInterval\n * @category Interval Helpers\n * @summary Return the array of hours within the specified time interval.\n *\n * @description\n * Return the array of hours within the specified time interval.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param interval - The interval.\n * @param options - An object with options.\n *\n * @returns The array with starts of hours from the hour of the interval start to the hour of the interval end\n *\n * @example\n * // Each hour between 6 October 2014, 12:00 and 6 October 2014, 15:00\n * const result = eachHourOfInterval({\n *   start: new Date(2014, 9, 6, 12),\n *   end: new Date(2014, 9, 6, 15)\n * })\n * //=> [\n * //   Mon Oct 06 2014 12:00:00,\n * //   Mon Oct 06 2014 13:00:00,\n * //   Mon Oct 06 2014 14:00:00,\n * //   Mon Oct 06 2014 15:00:00\n * // ]\n */\nexport function eachHourOfInterval<DateType extends Date>(\n  interval: Interval<DateType>,\n  options?: EachHourOfIntervalOptions,\n): DateType[] {\n  const startDate = toDate(interval.start);\n  const endDate = toDate(interval.end);\n\n  let reversed = +startDate > +endDate;\n  const endTime = reversed ? +startDate : +endDate;\n  let currentDate = reversed ? endDate : startDate;\n  currentDate.setMinutes(0, 0, 0);\n\n  let step = options?.step ?? 1;\n  if (!step) return [];\n  if (step < 0) {\n    step = -step;\n    reversed = !reversed;\n  }\n\n  const dates = [];\n\n  while (+currentDate <= endTime) {\n    dates.push(toDate(currentDate));\n    currentDate = addHours(currentDate, step);\n  }\n\n  return reversed ? dates.reverse() : dates;\n}\n\n// Path: src/startOfHour/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name startOfHour\n * @category Hour Helpers\n * @summary Return the start of an hour for the given date.\n *\n * @description\n * Return the start of an hour for the given date.\n * The result will be in the local timezone.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The start of an hour\n *\n * @example\n * // The start of an hour for 2 September 2014 11:55:00:\n * const result = startOfHour(new Date(2014, 8, 2, 11, 55))\n * //=> Tue Sep 02 2014 11:00:00\n */\nexport function startOfHour<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  const _date = toDate(date);\n  _date.setMinutes(0, 0, 0);\n  return _date;\n}\n\n// Path: src/isSameHour/index.ts\nimport { startOfHour } from \"../startOfHour/index.js\";\n\n/**\n * @name isSameHour\n * @category Hour Helpers\n * @summary Are the given dates in the same hour (and same day)?\n *\n * @description\n * Are the given dates in the same hour (and same day)?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The first date to check\n * @param dateRight - The second date to check\n *\n * @returns The dates are in the same hour (and same day)\n *\n * @example\n * // Are 4 September 2014 06:00:00 and 4 September 06:30:00 in the same hour?\n * const result = isSameHour(new Date(2014, 8, 4, 6, 0), new Date(2014, 8, 4, 6, 30))\n * //=> true\n *\n * @example\n * // Are 4 September 2014 06:00:00 and 5 September 06:00:00 in the same hour?\n * const result = isSameHour(new Date(2014, 8, 4, 6, 0), new Date(2014, 8, 5, 6, 0))\n * //=> false\n */\nexport function isSameHour<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n): boolean {\n  const dateLeftStartOfHour = startOfHour(dateLeft);\n  const dateRightStartOfHour = startOfHour(dateRight);\n\n  return +dateLeftStartOfHour === +dateRightStartOfHour;\n}\n\n// Path: src/setYear/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setYear\n * @category Year Helpers\n * @summary Set the year to the given date.\n *\n * @description\n * Set the year to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param year - The year of the new date\n *\n * @returns The new date with the year set\n *\n * @example\n * // Set year 2013 to 1 September 2014:\n * const result = setYear(new Date(2014, 8, 1), 2013)\n * //=> Sun Sep 01 2013 00:00:00\n */\nexport function setYear<DateType extends Date>(\n  date: DateType | number | string,\n  year: number,\n): DateType {\n  const _date = toDate(date);\n\n  // Check if date is Invalid Date because Date.prototype.setFullYear ignores the value of Invalid Date\n  if (isNaN(+_date)) {\n    return constructFrom(date, NaN);\n  }\n\n  _date.setFullYear(year);\n  return _date;\n}\n\n// Path: src/closestTo/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name closestTo\n * @category Common Helpers\n * @summary Return a date from the array closest to the given date.\n *\n * @description\n * Return a date from the array closest to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateToCompare - The date to compare with\n * @param dates - The array to search\n *\n * @returns The date from the array closest to the given date or undefined if no valid value is given\n *\n * @example\n * // Which date is closer to 6 September 2015: 1 January 2000 or 1 January 2030?\n * const dateToCompare = new Date(2015, 8, 6)\n * const result = closestTo(dateToCompare, [\n *   new Date(2000, 0, 1),\n *   new Date(2030, 0, 1)\n * ])\n * //=> Tue Jan 01 2030 00:00:00\n */\nexport function closestTo<DateType extends Date>(\n  dateToCompare: DateType | number | string,\n  dates: Array<DateType | number | string>,\n): DateType | undefined {\n  const date = toDate(dateToCompare);\n\n  if (isNaN(Number(date))) return constructFrom(dateToCompare, NaN);\n\n  const timeToCompare = date.getTime();\n\n  let result: DateType | undefined;\n  let minDistance: number;\n  dates.forEach((dirtyDate) => {\n    const currentDate = toDate(dirtyDate);\n\n    if (isNaN(Number(currentDate))) {\n      result = constructFrom(dateToCompare, NaN);\n      minDistance = NaN;\n      return;\n    }\n\n    const distance = Math.abs(timeToCompare - currentDate.getTime());\n    if (result == null || distance < minDistance) {\n      result = currentDate;\n      minDistance = distance;\n    }\n  });\n\n  return result;\n}\n\n// Path: src/setDayOfYear/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setDayOfYear\n * @category Day Helpers\n * @summary Set the day of the year to the given date.\n *\n * @description\n * Set the day of the year to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param dayOfYear - The day of the year of the new date\n *\n * @returns The new date with the day of the year set\n *\n * @example\n * // Set the 2nd day of the year to 2 July 2014:\n * const result = setDayOfYear(new Date(2014, 6, 2), 2)\n * //=> Thu Jan 02 2014 00:00:00\n */\nexport function setDayOfYear<DateType extends Date>(\n  date: DateType | number | string,\n  dayOfYear: number,\n): DateType {\n  const _date = toDate(date);\n  _date.setMonth(0);\n  _date.setDate(dayOfYear);\n  return _date;\n}\n\n// Path: src/min/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name min\n * @category Common Helpers\n * @summary Returns the earliest of the given dates.\n *\n * @description\n * Returns the earliest of the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dates - The dates to compare\n *\n * @returns The earliest of the dates\n *\n * @example\n * // Which of these dates is the earliest?\n * const result = min([\n *   new Date(1989, 6, 10),\n *   new Date(1987, 1, 11),\n *   new Date(1995, 6, 2),\n *   new Date(1990, 0, 1)\n * ])\n * //=> Wed Feb 11 1987 00:00:00\n */\nexport function min<DateType extends Date>(\n  dates: Array<DateType | number | string>,\n): DateType | Date {\n  let result: Date | undefined;\n\n  dates.forEach((dirtyDate) => {\n    const date = toDate(dirtyDate);\n    if (!result || result > date || isNaN(+date)) {\n      result = date;\n    }\n  });\n\n  return result || new Date(NaN);\n}\n\n// Path: src/isWednesday/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isWednesday\n * @category Weekday Helpers\n * @summary Is the given date Wednesday?\n *\n * @description\n * Is the given date Wednesday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is Wednesday\n *\n * @example\n * // Is 24 September 2014 Wednesday?\n * const result = isWednesday(new Date(2014, 8, 24))\n * //=> true\n */\nexport function isWednesday<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return toDate(date).getDay() === 3;\n}\n\n// Path: src/milliseconds/index.ts\nimport { daysInYear } from \"../constants/index.js\";\nimport type { Duration } from \"../types.js\";\n\n/**\n * @name milliseconds\n * @category Millisecond Helpers\n * @summary\n * Returns the number of milliseconds in the specified, years, months, weeks, days, hours, minutes and seconds.\n *\n * @description\n * Returns the number of milliseconds in the specified, years, months, weeks, days, hours, minutes and seconds.\n *\n * One years equals 365.2425 days according to the formula:\n *\n * > Leap year occures every 4 years, except for years that are divisable by 100 and not divisable by 400.\n * > 1 mean year = (365+1/4-1/100+1/400) days = 365.2425 days\n *\n * One month is a year divided by 12.\n *\n * @param duration - The object with years, months, weeks, days, hours, minutes and seconds to be added.\n *\n * @returns The milliseconds\n *\n * @example\n * // 1 year in milliseconds\n * milliseconds({ years: 1 })\n * //=> 31556952000\n *\n * // 3 months in milliseconds\n * milliseconds({ months: 3 })\n * //=> 7889238000\n */\nexport function milliseconds({\n  years,\n  months,\n  weeks,\n  days,\n  hours,\n  minutes,\n  seconds,\n}: Duration): number {\n  let totalDays = 0;\n\n  if (years) totalDays += years * daysInYear;\n  if (months) totalDays += months * (daysInYear / 12);\n  if (weeks) totalDays += weeks * 7;\n  if (days) totalDays += days;\n\n  let totalSeconds = totalDays * 24 * 60 * 60;\n\n  if (hours) totalSeconds += hours * 60 * 60;\n  if (minutes) totalSeconds += minutes * 60;\n  if (seconds) totalSeconds += seconds;\n\n  return Math.trunc(totalSeconds * 1000);\n}\n\n// Path: src/getDaysInMonth/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\n\n/**\n * @name getDaysInMonth\n * @category Month Helpers\n * @summary Get the number of days in a month of the given date.\n *\n * @description\n * Get the number of days in a month of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The number of days in a month\n *\n * @example\n * // How many days are in February 2000?\n * const result = getDaysInMonth(new Date(2000, 1))\n * //=> 29\n */\nexport function getDaysInMonth<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const year = _date.getFullYear();\n  const monthIndex = _date.getMonth();\n  const lastDayOfMonth = constructFrom(date, 0);\n  lastDayOfMonth.setFullYear(year, monthIndex + 1, 0);\n  lastDayOfMonth.setHours(0, 0, 0, 0);\n  return lastDayOfMonth.getDate();\n}\n\n// Path: src/setMonth/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { getDaysInMonth } from \"../getDaysInMonth/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setMonth\n * @category Month Helpers\n * @summary Set the month to the given date.\n *\n * @description\n * Set the month to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param month - The month index to set (0-11)\n *\n * @returns The new date with the month set\n *\n * @example\n * // Set February to 1 September 2014:\n * const result = setMonth(new Date(2014, 8, 1), 1)\n * //=> Sat Feb 01 2014 00:00:00\n */\nexport function setMonth<DateType extends Date>(\n  date: DateType | number | string,\n  month: number,\n): DateType {\n  const _date = toDate(date);\n  const year = _date.getFullYear();\n  const day = _date.getDate();\n\n  const dateWithDesiredMonth = constructFrom(date, 0);\n  dateWithDesiredMonth.setFullYear(year, month, 15);\n  dateWithDesiredMonth.setHours(0, 0, 0, 0);\n  const daysInMonth = getDaysInMonth(dateWithDesiredMonth);\n  // Set the last day of the new month\n  // if the original date was the last day of the longer month\n  _date.setMonth(month, Math.min(day, daysInMonth));\n  return _date;\n}\n\n// Path: src/set/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { setMonth } from \"../setMonth/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { DateValues } from \"../types.js\";\n\n/**\n * @name set\n * @category Common Helpers\n * @summary Set date values to a given date.\n *\n * @description\n * Set date values to a given date.\n *\n * Sets time values to date from object `values`.\n * A value is not set if it is undefined or null or doesn't exist in `values`.\n *\n * Note about bundle size: `set` does not internally use `setX` functions from date-fns but instead opts\n * to use native `Date#setX` methods. If you use this function, you may not want to include the\n * other `setX` functions that date-fns provides if you are concerned about the bundle size.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param values - The date values to be set\n *\n * @returns The new date with options set\n *\n * @example\n * // Transform 1 September 2014 into 20 October 2015 in a single line:\n * const result = set(new Date(2014, 8, 20), { year: 2015, month: 9, date: 20 })\n * //=> Tue Oct 20 2015 00:00:00\n *\n * @example\n * // Set 12 PM to 1 September 2014 01:23:45 to 1 September 2014 12:00:00:\n * const result = set(new Date(2014, 8, 1, 1, 23, 45), { hours: 12 })\n * //=> Mon Sep 01 2014 12:23:45\n */\n\nexport function set<DateType extends Date>(\n  date: DateType | number | string,\n  values: DateValues,\n): DateType {\n  let _date = toDate(date);\n\n  // Check if date is Invalid Date because Date.prototype.setFullYear ignores the value of Invalid Date\n  if (isNaN(+_date)) {\n    return constructFrom(date, NaN);\n  }\n\n  if (values.year != null) {\n    _date.setFullYear(values.year);\n  }\n\n  if (values.month != null) {\n    _date = setMonth(_date, values.month);\n  }\n\n  if (values.date != null) {\n    _date.setDate(values.date);\n  }\n\n  if (values.hours != null) {\n    _date.setHours(values.hours);\n  }\n\n  if (values.minutes != null) {\n    _date.setMinutes(values.minutes);\n  }\n\n  if (values.seconds != null) {\n    _date.setSeconds(values.seconds);\n  }\n\n  if (values.milliseconds != null) {\n    _date.setMilliseconds(values.milliseconds);\n  }\n\n  return _date;\n}\n\n// Path: src/hoursToSeconds/index.ts\nimport { secondsInHour } from \"../constants/index.js\";\n\n/**\n * @name hoursToSeconds\n * @category Conversion Helpers\n * @summary Convert hours to seconds.\n *\n * @description\n * Convert a number of hours to a full number of seconds.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param hours - The number of hours to be converted\n *\n * @returns The number of hours converted in seconds\n *\n * @example\n * // Convert 2 hours to seconds:\n * const result = hoursToSeconds(2)\n * //=> 7200\n */\nexport function hoursToSeconds(hours: number): number {\n  return Math.trunc(hours * secondsInHour);\n}\n\n// Path: src/differenceInCalendarYears/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name differenceInCalendarYears\n * @category Year Helpers\n * @summary Get the number of calendar years between the given dates.\n *\n * @description\n * Get the number of calendar years between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n\n * @returns The number of calendar years\n *\n * @example\n * // How many calendar years are between 31 December 2013 and 11 February 2015?\n * const result = differenceInCalendarYears(\n *   new Date(2015, 1, 11),\n *   new Date(2013, 11, 31)\n * )\n * //=> 2\n */\nexport function differenceInCalendarYears<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n): number {\n  const _dateLeft = toDate(dateLeft);\n  const _dateRight = toDate(dateRight);\n\n  return _dateLeft.getFullYear() - _dateRight.getFullYear();\n}\n\n// Path: src/differenceInSeconds/index.ts\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { differenceInMilliseconds } from \"../differenceInMilliseconds/index.js\";\nimport type { RoundingOptions } from \"../types.js\";\n\n/**\n * The {@link differenceInSeconds} function options.\n */\nexport interface DifferenceInSecondsOptions extends RoundingOptions {}\n\n/**\n * @name differenceInSeconds\n * @category Second Helpers\n * @summary Get the number of seconds between the given dates.\n *\n * @description\n * Get the number of seconds between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options.\n *\n * @returns The number of seconds\n *\n * @example\n * // How many seconds are between\n * // 2 July 2014 12:30:07.999 and 2 July 2014 12:30:20.000?\n * const result = differenceInSeconds(\n *   new Date(2014, 6, 2, 12, 30, 20, 0),\n *   new Date(2014, 6, 2, 12, 30, 7, 999)\n * )\n * //=> 12\n */\nexport function differenceInSeconds<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n  options?: DifferenceInSecondsOptions,\n): number {\n  const diff = differenceInMilliseconds(dateLeft, dateRight) / 1000;\n  return getRoundingMethod(options?.roundingMethod)(diff);\n}\n\n// Path: src/differenceInHours/index.ts\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { millisecondsInHour } from \"../constants/index.js\";\nimport { differenceInMilliseconds } from \"../differenceInMilliseconds/index.js\";\nimport type { RoundingOptions } from \"../types.js\";\n\n/**\n * The {@link differenceInHours} function options.\n */\nexport interface DifferenceInHoursOptions extends RoundingOptions {}\n\n/**\n * @name differenceInHours\n * @category Hour Helpers\n * @summary Get the number of hours between the given dates.\n *\n * @description\n * Get the number of hours between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options.\n *\n * @returns The number of hours\n *\n * @example\n * // How many hours are between 2 July 2014 06:50:00 and 2 July 2014 19:00:00?\n * const result = differenceInHours(\n *   new Date(2014, 6, 2, 19, 0),\n *   new Date(2014, 6, 2, 6, 50)\n * )\n * //=> 12\n */\n...\n", "needle_token_start": 2304, "needle_token_end": 2355, "code_context_ntokens": 15444, "output": ["```typescript\nimport { addMilliseconds } from \"../addMilliseconds/index.js\";\nimport { millisecondsInHour } from \"../constants/index.js\";\n\n/**\n * @name addHours\n * @category Hour Helpers\n * @summary Add the specified number of hours to the given date.\n *\n * @description\n * Add the specified number of hours to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of hours to be added.\n *\n * @returns The new date with the hours added\n *\n * @example\n * // Add 2 hours to 10 July 2014 23:00:00:\n * const result = addHours(new Date(2014, 6, 10, 23, 0), 2)\n * //=> Fri Jul 11 2014 01:00:00\n */\nexport function addHours<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addMilliseconds(date, amount * millisecondsInHour);\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "convertToFP", "language": "typescript", "path": "src/fp/_lib/convertToFP/index.ts", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform a standard function into a curried function that accepts its arguments in reverse order. This allows for partial application of arguments from the last to the first.\n2. **Input**: It takes three parameters: a function to be transformed, the number of arguments (arity) the function expects, and an optional array of initially provided arguments for currying.\n3. **Output**: It returns a new function that can be called with fewer arguments than the original function requires. If enough arguments are provided, it executes the original function with the arguments in reverse order.\n4. **Procedure**: The function checks if the number of provided arguments matches the arity. If it does, it calls the original function with these arguments reversed. If not enough arguments are provided, it returns a new function that waits for the remaining arguments. This new function, when called with additional arguments, will recursively call itself until the required number of arguments is reached.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/intervalToDuration/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { intervalToDuration } from \"./index.js\";\nimport { addMonths } from \"../addMonths/index.js\";\n\ndescribe(\"intervalToDuration\", () => {\n  it(\"returns correct duration for arbitrary dates\", () => {\n    const start = new Date(1929, 0, 15, 12, 0, 0);\n    const end = new Date(1968, 3, 4, 19, 5, 0);\n    const result = intervalToDuration({ start, end });\n\n    expect(result).toEqual({\n      years: 39,\n      months: 2,\n      days: 20,\n      hours: 7,\n      minutes: 5,\n    });\n  });\n\n  it(\"returns correct duration (1 of everything)\", () => {\n    const start = new Date(2020, 2, 1, 12, 0, 0);\n    const end = new Date(2021, 3, 2, 13, 1, 1);\n    const result = intervalToDuration({ start, end });\n\n    expect(result).toEqual({\n      years: 1,\n      months: 1,\n      days: 1,\n      hours: 1,\n      minutes: 1,\n      seconds: 1,\n    });\n  });\n\n  it(\"returns duration of 0 when the dates are the same\", () => {\n    const start = new Date(2020, 2, 1, 12, 0, 0);\n    const end = new Date(2020, 2, 1, 12, 0, 0);\n    const result = intervalToDuration({ start, end });\n\n    expect(result).toEqual({});\n  });\n\n  it(\"returns a negative duration if interval's start date is greater than its end date\", () => {\n    const interval = {\n      start: new Date(2020, 3, 1),\n      end: new Date(2020, 2, 1),\n    };\n    const result = intervalToDuration(interval);\n\n    expect(result).toEqual({ months: -1 });\n  });\n\n  it(\"returns an empty object if interval's start date invalid\", () => {\n    const interval = {\n      start: new Date(NaN),\n      end: new Date(2020, 0, 1),\n    };\n    const result = intervalToDuration(interval);\n\n    expect(result).toEqual({});\n  });\n\n  it(\"returns an empty object  if interval's end date invalid\", () => {\n    const interval = {\n      start: new Date(2020, 0, 1),\n      end: new Date(NaN),\n    };\n    const result = intervalToDuration(interval);\n\n    expect(result).toEqual({});\n  });\n\n  describe(\"edge cases\", () => {\n    it(\"returns correct duration for dates in the end of Feb - issue 2255\", () => {\n      expect(intervalToDuration({\n        start: new Date(2012, 1 /* Feb */, 28, 9, 0, 0),\n        end: new Date(2012, 1 /* Feb */, 29, 10, 0, 0),\n      })).toEqual({\n        days: 1,\n        hours: 1,\n      });\n\n      expect(intervalToDuration({\n        start: new Date(2012, 1 /* Feb */, 29, 9, 0, 0),\n        end: new Date(2012, 1 /* Feb */, 29, 10, 0, 0),\n      })).toEqual({\n        hours: 1,\n      });\n\n      expect(intervalToDuration({\n        start: new Date(2012, 1 /* Feb */, 28, 9, 0, 0),\n        end: new Date(2012, 1 /* Feb */, 28, 10, 0, 0),\n      })).toEqual({\n        hours: 1,\n      });\n\n      // Issue 2261\n      expect(intervalToDuration({\n        start: new Date(2021, 1 /* Feb */, 28, 7, 23, 7),\n        end: new Date(2021, 1 /* Feb */, 28, 7, 38, 18),\n      })).toEqual({\n        minutes: 15,\n        seconds: 11,\n      });\n    });\n\n    it(\"returns correct duration for end of month start dates - issue 2611\", () => {\n      const start = new Date(2021, 7, 31);\n      const end = addMonths(start, 1);\n\n      expect(end).toEqual(new Date(2021, 8, 30));\n\n      const duration = intervalToDuration({ start, end });\n      const expectedDuration = {\n        months: 1,\n      };\n\n      expect(duration).toEqual(expectedDuration);\n    });\n\n    it(\"returns correct duration for Feb 29 on leap year + 1 month - issue 1778\", () => {\n      const duration = intervalToDuration({\n        start: new Date(2020, 1, 29),\n        end: new Date(2020, 2, 29),\n      });\n      const expectedDuration = {\n        months: 1,\n      };\n\n      expect(duration).toEqual(expectedDuration);\n    });\n\n    it(\"returns correct duration for Feb 28 to Apr 30 interval - issue 2910\", () => {\n      const duration = intervalToDuration({\n        start: new Date(2022, 1, 28),\n        end: new Date(2022, 3, 30),\n      });\n      const expectedDuration = {\n        months: 2,\n        days: 2,\n      };\n\n      expect(duration).toEqual(expectedDuration);\n    });\n\n    describe(\"issue 2470\", () => {\n      it(\"returns correct duration for Feb 28 to Aug 31 interval\", () => {\n        const duration = intervalToDuration({\n          start: new Date(2021, 1, 28),\n          end: new Date(2021, 7, 31),\n        });\n        const expectedDuration = {\n          months: 6,\n          days: 3,\n        };\n\n        expect(duration).toEqual(expectedDuration);\n      });\n\n      it(\"returns correct duration for Feb 28 to Aug 30 interval\", () => {\n        const duration = intervalToDuration({\n          start: new Date(2021, 1, 28),\n          end: new Date(2021, 7, 30),\n        });\n        const expectedDuration = {\n          months: 6,\n          days: 2,\n        };\n\n        expect(duration).toEqual(expectedDuration);\n      });\n\n      it(\"returns correct duration for Feb 28 to Aug 29 interval\", () => {\n        const duration = intervalToDuration({\n          start: new Date(2021, 1, 28),\n          end: new Date(2021, 7, 29),\n        });\n        const expectedDuration = {\n          months: 6,\n          days: 1,\n        };\n\n        expect(duration).toEqual(expectedDuration);\n      });\n\n      it(\"returns correct duration for Feb 28 to Aug 28 interval\", () => {\n        const duration = intervalToDuration({\n          start: new Date(2021, 1, 28),\n          end: new Date(2021, 7, 28),\n        });\n        const expectedDuration = {\n          months: 6,\n        };\n\n        expect(duration).toEqual(expectedDuration);\n      });\n\n      it(\"returns correct duration for Feb 28 to Aug 27 interval\", () => {\n        // Feb 28 to July 28 is 5 months, July 28 to Aug 27 is 30 days\n\n        const duration = intervalToDuration({\n          start: new Date(2021, 1, 28),\n          end: new Date(2021, 7, 27),\n        });\n        const expectedDuration = {\n          months: 5,\n          days: 30,\n        };\n\n        expect(duration).toEqual(expectedDuration);\n      });\n\n      it(\"returns correct duration for Apr 30 to May 31 interval\", () => {\n        const duration = intervalToDuration({\n          start: new Date(2021, 3, 30),\n          end: new Date(2021, 4, 31),\n        });\n        const expectedDuration = {\n          months: 1,\n          days: 1,\n...\n// Path: src/nextWednesday/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { nextWednesday } from \"./index.js\";\n\ndescribe(\"nextWednesday\", () => {\n  it(\"returns the following Wednesday given various dates before the same\", () => {\n    expect(nextWednesday(new Date(2020, 4 /* May */, 23))).toEqual(new Date(2020, 4 /* May */, 27));\n\n    expect(nextWednesday(new Date(2020, 4 /* May */, 22))).toEqual(new Date(2020, 4 /* May */, 27));\n\n    expect(nextWednesday(new Date(2020, 4 /* May */, 21))).toEqual(new Date(2020, 4 /* May */, 27));\n\n    expect(nextWednesday(new Date(2020, 4 /* May */, 20))).toEqual(new Date(2020, 4 /* May */, 27));\n\n    expect(nextWednesday(new Date(2020, 4 /* May */, 19))).toEqual(new Date(2020, 4 /* May */, 20));\n\n    expect(nextWednesday(new Date(2020, 4 /* May */, 18))).toEqual(new Date(2020, 4 /* May */, 20));\n\n    expect(nextWednesday(new Date(2020, 4 /* May */, 17))).toEqual(new Date(2020, 4 /* May */, 20));\n  });\n\n  it(\"returns `Invalid Date` if the given date is invalid\", () => {\n    expect(nextWednesday(new Date(NaN)) instanceof Date).toBe(true);\n  });\n});\n\n// Path: src/fp/types.ts\n/**\n * The type of a function that can be converted to FP.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any -- We want any here.\nexport type FPFnInput = (...args: any[]) => any;\n\n/**\n * The supported arity type.\n */\nexport type FPArity = 1 | 2 | 3 | 4;\n\n/**\n * FP function interface. It infers the arity of the function and returns the\n * corresponding FP function interface.\n */\nexport type FPFn<Fn extends FPFnInput, Arity extends FPArity> = Arity extends 4\n  ? FPFn4<\n      ReturnType<Fn>,\n      Parameters<Fn>[3],\n      Parameters<Fn>[2],\n      Parameters<Fn>[1],\n      Parameters<Fn>[0]\n    >\n  : Arity extends 3\n    ? FPFn3<\n        ReturnType<Fn>,\n        Parameters<Fn>[2],\n        Parameters<Fn>[1],\n        Parameters<Fn>[0]\n      >\n    : Arity extends 2\n      ? FPFn2<ReturnType<Fn>, Parameters<Fn>[1], Parameters<Fn>[0]>\n      : Arity extends 1\n        ? FPFn1<ReturnType<Fn>, Parameters<Fn>[0]>\n        : never;\n\n/**\n * FP function interface with 1 arguments.\n */\nexport interface FPFn1<Result, Arg> {\n  /**\n   * Curried version of the function. Returns itself.\n   */\n  (): FPFn1<Result, Arg>;\n\n  /**\n   * Returns the result of the function call.\n   */\n  (arg: Arg): Result;\n}\n\n/**\n * FP function interface with 2 arguments.\n */\nexport interface FPFn2<Result, Arg2, Arg1> {\n  /**\n   * Curried version of the function. Returns itself.\n   */\n  (): FPFn2<Result, Arg2, Arg1>;\n\n  /**\n   * Curried version of the function. Returns a function that accepts the rest\n   * arguments.\n   */\n  (arg2: Arg2): FPFn1<Result, Arg1>;\n\n  /**\n   * Returns the result of the function call.\n   */\n  (arg2: Arg2, arg1: Arg1): Result;\n}\n\n/**\n * FP function interface with 3 arguments.\n */\nexport interface FPFn3<Result, Arg3, Arg2, Arg1> {\n  /**\n   * Curried version of the function. Returns itself.\n   */\n  (): FPFn3<Result, Arg3, Arg2, Arg1>;\n\n  /**\n   * Curried version of the function. Returns a function that accepts the rest\n   * arguments.\n   */\n  (arg3: Arg3): FPFn2<Result, Arg2, Arg1>;\n\n  /**\n   * Curried version of the function. Returns a function that accepts the rest\n   * arguments.\n   */\n  (arg3: Arg3, arg2: Arg2): FPFn1<Result, Arg1>;\n\n  /**\n   * Returns the result of the function call.\n   */\n  (arg3: Arg3, arg2: Arg2, arg1: Arg1): Result;\n}\n\n/**\n * FP function interface with 4 arguments.\n */\nexport interface FPFn4<Result, Arg4, Arg3, Arg2, Arg1> {\n  /**\n   * Curried version of the function. Returns itself.\n   */\n  (): FPFn4<Result, Arg4, Arg3, Arg2, Arg1>;\n\n  /**\n   * Curried version of the function. Returns a function that accepts the rest\n   * arguments.\n   */\n  (arg4: Arg4): FPFn3<Result, Arg3, Arg2, Arg1>;\n\n  /**\n   * Curried version of the function. Returns a function that accepts the rest\n   * arguments.\n   */\n  (arg4: Arg4, arg3: Arg3): FPFn2<Result, Arg2, Arg1>;\n\n  /**\n   * Curried version of the function. Returns a function that accepts the rest\n   * arguments.\n   */\n  (arg4: Arg4, arg3: Arg3, arg2: Arg2): FPFn1<Result, Arg1>;\n\n  /**\n   * Returns the result of the function call.\n   */\n  (arg4: Arg4, arg3: Arg3, arg2: Arg2, arg1: Arg1): Result;\n}\n\n// Path: src/fp/_lib/convertToFP/index.ts\nimport type { FPArity, FPFn, FPFnInput } from \"../../types\";\n\n/**\n * Converts a function to a curried function that accepts arguments in reverse\n * order.\n *\n * @param fn - The function to convert to FP\n * @param arity - The arity of the function\n * @param curriedArgs - The curried arguments\n *\n * @returns FP version of the function\n *\n * @private\n */\nexport \nfunction convertToFP<Fn extends FPFnInput, Arity extends FPArity>(\n  fn: Fn,\n  arity: Arity,\n  curriedArgs: unknown[] = [],\n): FPFn<Fn, Arity> {\n  return (\n    curriedArgs.length >= arity\n      ? fn(...curriedArgs.slice(0, arity).reverse())\n      : (...args: unknown[]) => convertToFP(fn, arity, curriedArgs.concat(args))\n  ) as FPFn<Fn, Arity>;\n}\n\n// Path: src/fp/formatRFC3339/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatRFC3339 as fn } from \"../../formatRFC3339/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatRFC3339 = convertToFP(fn, 1);\n\n// Path: src/fp/startOfMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfMonth as fn } from \"../../startOfMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfMonth = convertToFP(fn, 1);\n\n// Path: src/fp/formatISO9075WithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatISO9075 as fn } from \"../../formatISO9075/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatISO9075WithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/differenceInCalendarYears/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInCalendarYears as fn } from \"../../differenceInCalendarYears/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInCalendarYears = convertToFP(fn, 2);\n\n// Path: src/fp/roundToNearestMinutes/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { roundToNearestMinutes as fn } from \"../../roundToNearestMinutes/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const roundToNearestMinutes = convertToFP(fn, 1);\n\n// Path: src/fp/secondsToHours/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { secondsToHours as fn } from \"../../secondsToHours/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const secondsToHours = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInCalendarWeeksWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInCalendarWeeks as fn } from \"../../differenceInCalendarWeeks/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInCalendarWeeksWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/isSameYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameYear as fn } from \"../../isSameYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameYear = convertToFP(fn, 2);\n\n// Path: src/fp/millisecondsToHours/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { millisecondsToHours as fn } from \"../../millisecondsToHours/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const millisecondsToHours = convertToFP(fn, 1);\n\n// Path: src/fp/nextWednesday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { nextWednesday as fn } from \"../../nextWednesday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const nextWednesday = convertToFP(fn, 1);\n\n// Path: src/fp/setDay/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setDay as fn } from \"../../setDay/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setDay = convertToFP(fn, 2);\n\n// Path: src/fp/differenceInWeeks/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInWeeks as fn } from \"../../differenceInWeeks/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInWeeks = convertToFP(fn, 2);\n\n// Path: src/fp/yearsToDays/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { yearsToDays as fn } from \"../../yearsToDays/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const yearsToDays = convertToFP(fn, 1);\n\n// Path: src/fp/lightFormat/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { lightFormat as fn } from \"../../lightFormat/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const lightFormat = convertToFP(fn, 2);\n\n// Path: src/fp/transpose/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { transpose as fn } from \"../../transpose/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const transpose = convertToFP(fn, 2);\n\n// Path: src/fp/differenceInBusinessDays/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInBusinessDays as fn } from \"../../differenceInBusinessDays/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInBusinessDays = convertToFP(fn, 2);\n\n// Path: src/fp/getISOWeekYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getISOWeekYear as fn } from \"../../getISOWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getISOWeekYear = convertToFP(fn, 1);\n\n// Path: src/fp/getTime/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getTime as fn } from \"../../getTime/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getTime = convertToFP(fn, 1);\n\n// Path: src/fp/isAfter/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isAfter as fn } from \"../../isAfter/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isAfter = convertToFP(fn, 2);\n\n// Path: src/fp/formatISO/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatISO as fn } from \"../../formatISO/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatISO = convertToFP(fn, 1);\n\n// Path: src/fp/eachDayOfInterval/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachDayOfInterval as fn } from \"../../eachDayOfInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachDayOfInterval = convertToFP(fn, 1);\n\n// Path: src/fp/isSameWeekWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameWeek as fn } from \"../../isSameWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameWeekWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/add/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { add as fn } from \"../../add/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const add = convertToFP(fn, 2);\n\n// Path: src/fp/subBusinessDays/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { subBusinessDays as fn } from \"../../subBusinessDays/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const subBusinessDays = convertToFP(fn, 2);\n\n// Path: src/fp/formatISOWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatISO as fn } from \"../../formatISO/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatISOWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/getDay/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getDay as fn } from \"../../getDay/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getDay = convertToFP(fn, 1);\n\n// Path: src/fp/isBefore/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isBefore as fn } from \"../../isBefore/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isBefore = convertToFP(fn, 2);\n\n// Path: src/fp/differenceInQuartersWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInQuarters as fn } from \"../../differenceInQuarters/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInQuartersWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/addHours/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { addHours as fn } from \"../../addHours/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const addHours = convertToFP(fn, 2);\n\n// Path: src/fp/closestTo/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { closestTo as fn } from \"../../closestTo/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const closestTo = convertToFP(fn, 2);\n\n// Path: src/fp/isSaturday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSaturday as fn } from \"../../isSaturday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSaturday = convertToFP(fn, 1);\n\n// Path: src/fp/endOfDecade/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { endOfDecade as fn } from \"../../endOfDecade/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const endOfDecade = convertToFP(fn, 1);\n\n// Path: src/fp/endOfQuarter/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { endOfQuarter as fn } from \"../../endOfQuarter/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const endOfQuarter = convertToFP(fn, 1);\n\n// Path: src/fp/formatDuration/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatDuration as fn } from \"../../formatDuration/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatDuration = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInCalendarDays/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInCalendarDays as fn } from \"../../differenceInCalendarDays/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInCalendarDays = convertToFP(fn, 2);\n\n// Path: src/fp/getWeekYearWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getWeekYear as fn } from \"../../getWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getWeekYearWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/addQuarters/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { addQuarters as fn } from \"../../addQuarters/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const addQuarters = convertToFP(fn, 2);\n\n// Path: src/fp/parseWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { parse as fn } from \"../../parse/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const parseWithOptions = convertToFP(fn, 4);\n\n// Path: src/fp/setQuarter/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setQuarter as fn } from \"../../setQuarter/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setQuarter = convertToFP(fn, 2);\n\n// Path: src/fp/isDate/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isDate as fn } from \"../../isDate/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isDate = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInMinutes/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInMinutes as fn } from \"../../differenceInMinutes/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInMinutes = convertToFP(fn, 2);\n\n// Path: src/fp/setDayOfYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setDayOfYear as fn } from \"../../setDayOfYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setDayOfYear = convertToFP(fn, 2);\n\n// Path: src/fp/setWeekWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setWeek as fn } from \"../../setWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setWeekWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/eachMinuteOfIntervalWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachMinuteOfInterval as fn } from \"../../eachMinuteOfInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachMinuteOfIntervalWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/nextSunday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { nextSunday as fn } from \"../../nextSunday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const nextSunday = convertToFP(fn, 1);\n\n// Path: src/fp/isWeekend/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isWeekend as fn } from \"../../isWeekend/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isWeekend = convertToFP(fn, 1);\n\n// Path: src/fp/endOfHour/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { endOfHour as fn } from \"../../endOfHour/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const endOfHour = convertToFP(fn, 1);\n\n// Path: src/fp/min/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { min as fn } from \"../../min/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const min = convertToFP(fn, 1);\n\n// Path: src/fp/nextTuesday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { nextTuesday as fn } from \"../../nextTuesday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const nextTuesday = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInCalendarMonths/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInCalendarMonths as fn } from \"../../differenceInCalendarMonths/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInCalendarMonths = convertToFP(fn, 2);\n\n// Path: src/fp/previousThursday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { previousThursday as fn } from \"../../previousThursday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const previousThursday = convertToFP(fn, 1);\n\n// Path: src/fp/parse/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { parse as fn } from \"../../parse/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const parse = convertToFP(fn, 3);\n\n// Path: src/fp/isSameDay/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameDay as fn } from \"../../isSameDay/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameDay = convertToFP(fn, 2);\n\n// Path: src/fp/compareDesc/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { compareDesc as fn } from \"../../compareDesc/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const compareDesc = convertToFP(fn, 2);\n\n// Path: src/fp/eachMonthOfInterval/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachMonthOfInterval as fn } from \"../../eachMonthOfInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachMonthOfInterval = convertToFP(fn, 1);\n\n// Path: src/fp/getWeekYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getWeekYear as fn } from \"../../getWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getWeekYear = convertToFP(fn, 1);\n\n// Path: src/fp/closestIndexTo/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { closestIndexTo as fn } from \"../../closestIndexTo/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const closestIndexTo = convertToFP(fn, 2);\n\n// Path: src/fp/previousWednesday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { previousWednesday as fn } from \"../../previousWednesday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const previousWednesday = convertToFP(fn, 1);\n\n// Path: src/fp/areIntervalsOverlappingWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { areIntervalsOverlapping as fn } from \"../../areIntervalsOverlapping/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const areIntervalsOverlappingWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/lastDayOfWeekWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { lastDayOfWeek as fn } from \"../../lastDayOfWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const lastDayOfWeekWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/isSunday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSunday as fn } from \"../../isSunday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSunday = convertToFP(fn, 1);\n\n// Path: src/fp/eachWeekendOfInterval/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachWeekendOfInterval as fn } from \"../../eachWeekendOfInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachWeekendOfInterval = convertToFP(fn, 1);\n\n// Path: src/fp/getYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getYear as fn } from \"../../getYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getYear = convertToFP(fn, 1);\n\n// Path: src/fp/isMatch/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isMatch as fn } from \"../../isMatch/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isMatch = convertToFP(fn, 2);\n\n// Path: src/fp/endOfMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { endOfMonth as fn } from \"../../endOfMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const endOfMonth = convertToFP(fn, 1);\n\n// Path: src/fp/quartersToYears/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { quartersToYears as fn } from \"../../quartersToYears/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const quartersToYears = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInMilliseconds/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInMilliseconds as fn } from \"../../differenceInMilliseconds/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInMilliseconds = convertToFP(fn, 2);\n\n// Path: src/fp/intlFormatDistance/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { intlFormatDistance as fn } from \"../../intlFormatDistance/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const intlFormatDistance = convertToFP(fn, 2);\n\n// Path: src/fp/eachMonthOfIntervalWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachMonthOfInterval as fn } from \"../../eachMonthOfInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachMonthOfIntervalWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/isFriday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isFriday as fn } from \"../../isFriday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isFriday = convertToFP(fn, 1);\n\n// Path: src/fp/hoursToSeconds/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { hoursToSeconds as fn } from \"../../hoursToSeconds/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const hoursToSeconds = convertToFP(fn, 1);\n\n// Path: src/fp/formatISODuration/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatISODuration as fn } from \"../../formatISODuration/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatISODuration = convertToFP(fn, 1);\n\n// Path: src/fp/startOfYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfYear as fn } from \"../../startOfYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfYear = convertToFP(fn, 1);\n\n// Path: src/fp/monthsToQuarters/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { monthsToQuarters as fn } from \"../../monthsToQuarters/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const monthsToQuarters = convertToFP(fn, 1);\n\n// Path: src/fp/startOfSecond/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfSecond as fn } from \"../../startOfSecond/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfSecond = convertToFP(fn, 1);\n\n// Path: src/fp/getDecade/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getDecade as fn } from \"../../getDecade/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getDecade = convertToFP(fn, 1);\n\n// Path: src/fp/parseISOWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { parseISO as fn } from \"../../parseISO/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const parseISOWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/isLastDayOfMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isLastDayOfMonth as fn } from \"../../isLastDayOfMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isLastDayOfMonth = convertToFP(fn, 1);\n\n// Path: src/fp/getMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getMonth as fn } from \"../../getMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getMonth = convertToFP(fn, 1);\n\n// Path: src/fp/hoursToMilliseconds/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { hoursToMilliseconds as fn } from \"../../hoursToMilliseconds/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const hoursToMilliseconds = convertToFP(fn, 1);\n\n// Path: src/fp/nextDay/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { nextDay as fn } from \"../../nextDay/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const nextDay = convertToFP(fn, 2);\n\n// Path: src/fp/endOfISOWeekYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { endOfISOWeekYear as fn } from \"../../endOfISOWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const endOfISOWeekYear = convertToFP(fn, 1);\n\n// Path: src/fp/formatDistanceWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatDistance as fn } from \"../../formatDistance/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatDistanceWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/isSameMinute/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameMinute as fn } from \"../../isSameMinute/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameMinute = convertToFP(fn, 2);\n\n// Path: src/fp/previousSunday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { previousSunday as fn } from \"../../previousSunday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const previousSunday = convertToFP(fn, 1);\n\n// Path: src/fp/isExists/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isExists as fn } from \"../../isExists/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isExists = convertToFP(fn, 3);\n\n// Path: src/fp/getISOWeeksInYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getISOWeeksInYear as fn } from \"../../getISOWeeksInYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getISOWeeksInYear = convertToFP(fn, 1);\n\n// Path: src/fp/setISODay/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setISODay as fn } from \"../../setISODay/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setISODay = convertToFP(fn, 2);\n\n// Path: src/fp/differenceInMonths/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInMonths as fn } from \"../../differenceInMonths/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInMonths = convertToFP(fn, 2);\n\n// Path: src/fp/clamp/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { clamp as fn } from \"../../clamp/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const clamp = convertToFP(fn, 2);\n\n// Path: src/fp/isSameISOWeekYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameISOWeekYear as fn } from \"../../isSameISOWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameISOWeekYear = convertToFP(fn, 2);\n\n// Path: src/fp/toDate/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { toDate as fn } from \"../../toDate/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const toDate = convertToFP(fn, 1);\n\n// Path: src/fp/startOfWeekYearWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfWeekYear as fn } from \"../../startOfWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfWeekYearWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/differenceInYears/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInYears as fn } from \"../../differenceInYears/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInYears = convertToFP(fn, 2);\n\n// Path: src/fp/nextSaturday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { nextSaturday as fn } from \"../../nextSaturday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const nextSaturday = convertToFP(fn, 1);\n\n// Path: src/fp/lastDayOfISOWeek/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { lastDayOfISOWeek as fn } from \"../../lastDayOfISOWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const lastDayOfISOWeek = convertToFP(fn, 1);\n\n// Path: src/fp/millisecondsToMinutes/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { millisecondsToMinutes as fn } from \"../../millisecondsToMinutes/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const millisecondsToMinutes = convertToFP(fn, 1);\n\n// Path: src/fp/monthsToYears/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { monthsToYears as fn } from \"../../monthsToYears/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const monthsToYears = convertToFP(fn, 1);\n\n// Path: src/fp/yearsToMonths/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { yearsToMonths as fn } from \"../../yearsToMonths/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const yearsToMonths = convertToFP(fn, 1);\n\n// Path: src/fp/eachQuarterOfIntervalWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachQuarterOfInterval as fn } from \"../../eachQuarterOfInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachQuarterOfIntervalWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/isSameISOWeek/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameISOWeek as fn } from \"../../isSameISOWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameISOWeek = convertToFP(fn, 2);\n\n// Path: src/fp/addSeconds/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { addSeconds as fn } from \"../../addSeconds/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const addSeconds = convertToFP(fn, 2);\n\n// Path: src/fp/getWeekOfMonthWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getWeekOfMonth as fn } from \"../../getWeekOfMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getWeekOfMonthWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/eachDayOfIntervalWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachDayOfInterval as fn } from \"../../eachDayOfInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachDayOfIntervalWithOptions = convertToFP(fn, 2);\n\n// Path: src/fp/roundToNearestHours/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { roundToNearestHours as fn } from \"../../roundToNearestHours/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const roundToNearestHours = convertToFP(fn, 1);\n\n// Path: src/fp/setDayWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setDay as fn } from \"../../setDay/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setDayWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/startOfWeekYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfWeekYear as fn } from \"../../startOfWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfWeekYear = convertToFP(fn, 1);\n\n// Path: src/fp/getDaysInYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getDaysInYear as fn } from \"../../getDaysInYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getDaysInYear = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInWeeksWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInWeeks as fn } from \"../../differenceInWeeks/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInWeeksWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/differenceInHoursWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInHours as fn } from \"../../differenceInHours/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInHoursWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/differenceInMinutesWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInMinutes as fn } from \"../../differenceInMinutes/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInMinutesWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/formatDistanceStrict/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatDistanceStrict as fn } from \"../../formatDistanceStrict/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatDistanceStrict = convertToFP(fn, 2);\n\n// Path: src/fp/formatISO9075/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatISO9075 as fn } from \"../../formatISO9075/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatISO9075 = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInISOWeekYears/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInISOWeekYears as fn } from \"../../differenceInISOWeekYears/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInISOWeekYears = convertToFP(fn, 2);\n\n// Path: src/fp/isSameSecond/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\n...\n", "needle_token_start": 3943, "needle_token_end": 4074, "code_context_ntokens": 15945, "output": ["```typescript\n/**\n * Converts a function to a curried function that accepts arguments in reverse\n * order.\n *\n * @param fn - The function to convert to FP\n * @param arity - The arity of the function\n * @param curriedArgs - The curried arguments\n *\n * @returns FP version of the function\n *\n * @private\n */\nexport function convertToFP<Fn extends FPFnInput, Arity extends FPArity>(\n  fn: Fn,\n  arity: Arity,\n  curriedArgs: unknown[] = [],\n): FPFn<Fn, Arity> {\n  return (\n    curriedArgs.length >= arity\n      ? fn(...curriedArgs.slice(0, arity).reverse())\n      : (...args: unknown[]) => convertToFP(fn, arity, curriedArgs.concat(args))\n  ) as FPFn<Fn, Arity>;\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "intervalToDuration", "language": "typescript", "path": "src/intervalToDuration/index.ts", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: Converts a specified time interval between two dates into a detailed duration object, breaking down the period into years, months, days, hours, minutes, and seconds.\n2. **Input**: An object containing two `Date` objects, representing the start and end of the interval.\n3. **Output**: A duration object that details how many years, months, days, hours, minutes, and seconds the interval spans.\n4. **Procedure**: The function first converts the start and end dates to valid date objects. It then calculates the difference in years, months, days, hours, minutes, and seconds sequentially, adjusting the start date progressively to account for each component calculated. This ensures that each component (like months, days, etc.) is correctly computed without overlap, providing an accurate breakdown of the interval into a comprehensive duration.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "...\n// Path: src/eachWeekendOfYear/index.ts\nimport { eachWeekendOfInterval } from \"../eachWeekendOfInterval/index.js\";\nimport { endOfYear } from \"../endOfYear/index.js\";\nimport { startOfYear } from \"../startOfYear/index.js\";\n\n/**\n * @name eachWeekendOfYear\n * @category Year Helpers\n * @summary List all the Saturdays and Sundays in the year.\n *\n * @description\n * Get all the Saturdays and Sundays in the year.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given year\n *\n * @returns An array containing all the Saturdays and Sundays\n *\n * @example\n * // Lists all Saturdays and Sundays in the year\n * const result = eachWeekendOfYear(new Date(2020, 1, 1))\n * //=> [\n * //   Sat Jan 03 2020 00:00:00,\n * //   Sun Jan 04 2020 00:00:00,\n * //   ...\n * //   Sun Dec 27 2020 00:00:00\n * // ]\n * ]\n */\nexport function eachWeekendOfYear<DateType extends Date>(\n  date: DateType | number | string,\n): DateType[] {\n  const start = startOfYear(date);\n  const end = endOfYear(date);\n  return eachWeekendOfInterval({ start, end });\n}\n\n// Path: src/nextFriday/index.ts\nimport { nextDay } from \"../nextDay/index.js\";\n\n/**\n * @name nextFriday\n * @category Weekday Helpers\n * @summary When is the next Friday?\n *\n * @description\n * When is the next Friday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to start counting from\n *\n * @returns The next Friday\n *\n * @example\n * // When is the next Friday after Mar, 22, 2020?\n * const result = nextFriday(new Date(2020, 2, 22))\n * //=> Fri Mar 27 2020 00:00:00\n */\nexport function nextFriday<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  return nextDay(date, 5);\n}\n\n// Path: src/daysToWeeks/index.ts\nimport { daysInWeek } from \"../constants/index.js\";\n\n/**\n * @name daysToWeeks\n * @category Conversion Helpers\n * @summary Convert days to weeks.\n *\n * @description\n * Convert a number of days to a full number of weeks.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param days - The number of days to be converted\n *\n * @returns The number of days converted in weeks\n *\n * @example\n * // Convert 14 days to weeks:\n * const result = daysToWeeks(14)\n * //=> 2\n *\n * @example\n * // It uses trunc rounding:\n * const result = daysToWeeks(13)\n * //=> 1\n */\nexport function daysToWeeks(days: number): number {\n  const weeks = days / daysInWeek;\n  const result = Math.trunc(weeks);\n  // Prevent negative zero\n  return result === 0 ? 0 : result;\n}\n\n// Path: src/lastDayOfQuarter/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name lastDayOfQuarter\n * @category Quarter Helpers\n * @summary Return the last day of a year quarter for the given date.\n *\n * @description\n * Return the last day of a year quarter for the given date.\n * The result will be in the local timezone.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The last day of a quarter\n *\n * @example\n * // The last day of a quarter for 2 September 2014 11:55:00:\n * const result = lastDayOfQuarter(new Date(2014, 8, 2, 11, 55, 0))\n * //=> Tue Sep 30 2014 00:00:00\n */\nexport function lastDayOfQuarter<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  const _date = toDate(date);\n  const currentMonth = _date.getMonth();\n  const month = currentMonth - (currentMonth % 3) + 3;\n  _date.setMonth(month, 0);\n  _date.setHours(0, 0, 0, 0);\n  return _date;\n}\n\n// Path: src/isThursday/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isThursday\n * @category Weekday Helpers\n * @summary Is the given date Thursday?\n *\n * @description\n * Is the given date Thursday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is Thursday\n *\n * @example\n * // Is 25 September 2014 Thursday?\n * const result = isThursday(new Date(2014, 8, 25))\n * //=> true\n */\nexport function isThursday<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return toDate(date).getDay() === 4;\n}\n\n// Path: src/isEqual/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isEqual\n * @category Common Helpers\n * @summary Are the given dates equal?\n *\n * @description\n * Are the given dates equal?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The first date to compare\n * @param dateRight - The second date to compare\n *\n * @returns The dates are equal\n *\n * @example\n * // Are 2 July 2014 06:30:45.000 and 2 July 2014 06:30:45.500 equal?\n * const result = isEqual(\n *   new Date(2014, 6, 2, 6, 30, 45, 0),\n *   new Date(2014, 6, 2, 6, 30, 45, 500)\n * )\n * //=> false\n */\nexport function isEqual<DateType extends Date>(\n  leftDate: DateType | number | string,\n  rightDate: DateType | number | string,\n): boolean {\n  const _dateLeft = toDate(leftDate);\n  const _dateRight = toDate(rightDate);\n  return +_dateLeft === +_dateRight;\n}\n\n// Path: src/getMonth/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getMonth\n * @category Month Helpers\n * @summary Get the month of the given date.\n *\n * @description\n * Get the month of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The month index (0-11)\n *\n * @example\n * // Which month is 29 February 2012?\n * const result = getMonth(new Date(2012, 1, 29))\n * //=> 1\n */\nexport function getMonth<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const month = _date.getMonth();\n  return month;\n}\n\n// Path: src/hoursToMinutes/index.ts\nimport { minutesInHour } from \"../constants/index.js\";\n\n/**\n * @name hoursToMinutes\n * @category Conversion Helpers\n * @summary Convert hours to minutes.\n *\n * @description\n * Convert a number of hours to a full number of minutes.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param hours - number of hours to be converted\n *\n * @returns The number of hours converted in minutes\n *\n * @example\n * // Convert 2 hours to minutes:\n * const result = hoursToMinutes(2)\n * //=> 120\n */\nexport function hoursToMinutes(hours: number): number {\n  return Math.trunc(hours * minutesInHour);\n}\n\n// Path: src/startOfTomorrow/index.ts\n/**\n * @name startOfTomorrow\n * @category Day Helpers\n * @summary Return the start of tomorrow.\n * @pure false\n *\n * @description\n * Return the start of tomorrow.\n *\n * @returns The start of tomorrow\n *\n * @example\n * // If today is 6 October 2014:\n * const result = startOfTomorrow()\n * //=> Tue Oct 7 2014 00:00:00\n */\nexport function startOfTomorrow(): Date {\n  const now = new Date();\n  const year = now.getFullYear();\n  const month = now.getMonth();\n  const day = now.getDate();\n\n  const date = new Date(0);\n  date.setFullYear(year, month, day + 1);\n  date.setHours(0, 0, 0, 0);\n  return date;\n}\n\n// Path: src/nextSaturday/index.ts\nimport { nextDay } from \"../nextDay/index.js\";\n\n/**\n * @name nextSaturday\n * @category Weekday Helpers\n * @summary When is the next Saturday?\n *\n * @description\n * When is the next Saturday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to start counting from\n *\n * @returns The next Saturday\n *\n * @example\n * // When is the next Saturday after Mar, 22, 2020?\n * const result = nextSaturday(new Date(2020, 2, 22))\n * //=> Sat Mar 28 2020 00:00:00\n */\nexport function nextSaturday<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  return nextDay(date, 6);\n}\n\n// Path: src/lightFormat/index.ts\nimport { isValid } from \"../isValid/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport { lightFormatters } from \"../_lib/format/lightFormatters/index.js\";\n\n// Rexports of internal for libraries to use.\n// See: https://github.com/date-fns/date-fns/issues/3638#issuecomment-1877082874\nexport { lightFormatters };\n\n// This RegExp consists of three parts separated by `|`:\n// - (\\w)\\1* matches any sequences of the same letter\n// - '' matches two quote characters in a row\n// - '(''|[^'])+('|$) matches anything surrounded by two quote characters ('),\n//   except a single quote symbol, which ends the sequence.\n//   Two quote characters do not end the sequence.\n//   If there is no matching single quote\n//   then the sequence will continue until the end of the string.\n// - . matches any single character unmatched by previous parts of the RegExps\nconst formattingTokensRegExp = /(\\w)\\1*|''|'(''|[^'])+('|$)|./g;\n\nconst escapedStringRegExp = /^'([^]*?)'?$/;\nconst doubleQuoteRegExp = /''/g;\nconst unescapedLatinCharacterRegExp = /[a-zA-Z]/;\n\n/**\n * @private\n */\ntype Token = keyof typeof lightFormatters;\n\n/**\n * @name lightFormat\n * @category Common Helpers\n * @summary Format the date.\n *\n * @description\n * Return the formatted date string in the given format. Unlike `format`,\n * `lightFormat` doesn't use locales and outputs date using the most popular tokens.\n *\n * > \u26a0\ufe0f Please note that the `lightFormat` tokens differ from Moment.js and other libraries.\n * > See: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n *\n * The characters wrapped between two single quotes characters (') are escaped.\n * Two single quotes in a row, whether inside or outside a quoted sequence, represent a 'real' single quote.\n *\n * Format of the string is based on Unicode Technical Standard #35:\n * https://www.unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table\n *\n * Accepted patterns:\n * | Unit                            | Pattern | Result examples                   |\n * |---------------------------------|---------|-----------------------------------|\n * | AM, PM                          | a..aaa  | AM, PM                            |\n * |                                 | aaaa    | a.m., p.m.                        |\n * |                                 | aaaaa   | a, p                              |\n * | Calendar year                   | y       | 44, 1, 1900, 2017                 |\n * |                                 | yy      | 44, 01, 00, 17                    |\n * |                                 | yyy     | 044, 001, 000, 017                |\n * |                                 | yyyy    | 0044, 0001, 1900, 2017            |\n * | Month (formatting)              | M       | 1, 2, ..., 12                     |\n * |                                 | MM      | 01, 02, ..., 12                   |\n * | Day of month                    | d       | 1, 2, ..., 31                     |\n * |                                 | dd      | 01, 02, ..., 31                   |\n * | Hour [1-12]                     | h       | 1, 2, ..., 11, 12                 |\n * |                                 | hh      | 01, 02, ..., 11, 12               |\n * | Hour [0-23]                     | H       | 0, 1, 2, ..., 23                  |\n * |                                 | HH      | 00, 01, 02, ..., 23               |\n * | Minute                          | m       | 0, 1, ..., 59                     |\n * |                                 | mm      | 00, 01, ..., 59                   |\n * | Second                          | s       | 0, 1, ..., 59                     |\n * |                                 | ss      | 00, 01, ..., 59                   |\n * | Fraction of second              | S       | 0, 1, ..., 9                      |\n * |                                 | SS      | 00, 01, ..., 99                   |\n * |                                 | SSS     | 000, 001, ..., 999                |\n * |                                 | SSSS    | ...                               |\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n * @param format - The string of tokens\n *\n * @returns The formatted date string\n *\n * @throws `Invalid time value` if the date is invalid\n * @throws format string contains an unescaped latin alphabet character\n *\n * @example\n * const result = lightFormat(new Date(2014, 1, 11), 'yyyy-MM-dd')\n * //=> '2014-02-11'\n */\nexport function lightFormat<DateType extends Date>(\n  date: DateType | number | string,\n  formatStr: string,\n): string {\n  const _date = toDate(date);\n\n  if (!isValid(_date)) {\n    throw new RangeError(\"Invalid time value\");\n  }\n\n  const tokens = formatStr.match(formattingTokensRegExp);\n\n  // The only case when formattingTokensRegExp doesn't match the string is when it's empty\n  if (!tokens) return \"\";\n\n  const result = tokens\n    .map((substring) => {\n      // Replace two single quote characters with one single quote character\n      if (substring === \"''\") {\n        return \"'\";\n      }\n\n      const firstCharacter = substring[0];\n      if (firstCharacter === \"'\") {\n        return cleanEscapedString(substring);\n      }\n\n      const formatter = lightFormatters[firstCharacter as Token];\n      if (formatter) {\n        return formatter(_date, substring);\n      }\n\n      if (firstCharacter.match(unescapedLatinCharacterRegExp)) {\n        throw new RangeError(\n          \"Format string contains an unescaped latin alphabet character `\" +\n            firstCharacter +\n            \"`\",\n        );\n      }\n\n      return substring;\n    })\n    .join(\"\");\n\n  return result;\n}\n\nfunction cleanEscapedString(input: string) {\n  const matches = input.match(escapedStringRegExp);\n\n  if (!matches) {\n    return input;\n  }\n\n  return matches[1].replace(doubleQuoteRegExp, \"'\");\n}\n\n// Path: src/intervalToDuration/index.ts\nimport { add } from \"../add/index.js\";\nimport { differenceInDays } from \"../differenceInDays/index.js\";\nimport { differenceInHours } from \"../differenceInHours/index.js\";\nimport { differenceInMinutes } from \"../differenceInMinutes/index.js\";\nimport { differenceInMonths } from \"../differenceInMonths/index.js\";\nimport { differenceInSeconds } from \"../differenceInSeconds/index.js\";\nimport { differenceInYears } from \"../differenceInYears/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { Duration, Interval } from \"../types.js\";\n\n/**\n * @name intervalToDuration\n * @category Common Helpers\n * @summary Convert interval to duration\n *\n * @description\n * Convert a interval object to a duration object.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param interval - The interval to convert to duration\n *\n * @returns The duration object\n *\n * @example\n * // Get the duration between January 15, 1929 and April 4, 1968.\n * intervalToDuration({\n *   start: new Date(1929, 0, 15, 12, 0, 0),\n *   end: new Date(1968, 3, 4, 19, 5, 0)\n * })\n * // => { years: 39, months: 2, days: 20, hours: 7, minutes: 5, seconds: 0 }\n */\nexport \nfunction intervalToDuration<DateType extends Date>(\n  interval: Interval<DateType>,\n): Duration {\n  const start = toDate(interval.start);\n  const end = toDate(interval.end);\n\n  const duration: Duration = {};\n\n  const years = differenceInYears(end, start);\n  if (years) duration.years = years;\n\n  const remainingMonths = add(start, { years: duration.years });\n\n  const months = differenceInMonths(end, remainingMonths);\n  if (months) duration.months = months;\n\n  const remainingDays = add(remainingMonths, { months: duration.months });\n\n  const days = differenceInDays(end, remainingDays);\n  if (days) duration.days = days;\n\n  const remainingHours = add(remainingDays, { days: duration.days });\n\n  const hours = differenceInHours(end, remainingHours);\n  if (hours) duration.hours = hours;\n\n  const remainingMinutes = add(remainingHours, { hours: duration.hours });\n\n  const minutes = differenceInMinutes(end, remainingMinutes);\n  if (minutes) duration.minutes = minutes;\n\n  const remainingSeconds = add(remainingMinutes, { minutes: duration.minutes });\n\n  const seconds = differenceInSeconds(end, remainingSeconds);\n  if (seconds) duration.seconds = seconds;\n\n  return duration;\n}\n\n// Path: src/isTomorrow/index.ts\nimport { addDays } from \"../addDays/index.js\";\nimport { constructNow } from \"../constructNow/index.js\";\nimport { isSameDay } from \"../isSameDay/index.js\";\n\n/**\n * @name isTomorrow\n * @category Day Helpers\n * @summary Is the given date tomorrow?\n * @pure false\n *\n * @description\n * Is the given date tomorrow?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is tomorrow\n *\n * @example\n * // If today is 6 October 2014, is 7 October 14:00:00 tomorrow?\n * const result = isTomorrow(new Date(2014, 9, 7, 14, 0))\n * //=> true\n */\nexport function isTomorrow<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return isSameDay(date, addDays(constructNow(date), 1));\n}\n\n// Path: src/sub/index.ts\nimport { subDays } from \"../subDays/index.js\";\nimport { subMonths } from \"../subMonths/index.js\";\nimport type { Duration } from \"../types.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\n\n/**\n * @name sub\n * @category Common Helpers\n * @summary Subtract the specified years, months, weeks, days, hours, minutes and seconds from the given date.\n *\n * @description\n * Subtract the specified years, months, weeks, days, hours, minutes and seconds from the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param duration - The object with years, months, weeks, days, hours, minutes and seconds to be subtracted\n *\n * | Key     | Description                        |\n * |---------|------------------------------------|\n * | years   | Amount of years to be subtracted   |\n * | months  | Amount of months to be subtracted  |\n * | weeks   | Amount of weeks to be subtracted   |\n * | days    | Amount of days to be subtracted    |\n * | hours   | Amount of hours to be subtracted   |\n * | minutes | Amount of minutes to be subtracted |\n * | seconds | Amount of seconds to be subtracted |\n *\n * All values default to 0\n *\n * @returns The new date with the seconds subtracted\n *\n * @example\n * // Subtract the following duration from 15 June 2017 15:29:20\n * const result = sub(new Date(2017, 5, 15, 15, 29, 20), {\n *   years: 2,\n *   months: 9,\n *   weeks: 1,\n *   days: 7,\n *   hours: 5,\n *   minutes: 9,\n *   seconds: 30\n * })\n * //=> Mon Sep 1 2014 10:19:50\n */\nexport function sub<DateType extends Date>(\n  date: DateType | number | string,\n  duration: Duration,\n): DateType {\n  const {\n    years = 0,\n    months = 0,\n    weeks = 0,\n    days = 0,\n    hours = 0,\n    minutes = 0,\n    seconds = 0,\n  } = duration;\n\n  // Subtract years and months\n  const dateWithoutMonths = subMonths(date, months + years * 12);\n\n  // Subtract weeks and days\n  const dateWithoutDays = subDays(dateWithoutMonths, days + weeks * 7);\n\n  // Subtract hours, minutes and seconds\n  const minutestoSub = minutes + hours * 60;\n  const secondstoSub = seconds + minutestoSub * 60;\n  const mstoSub = secondstoSub * 1000;\n  const finalDate = constructFrom(date, dateWithoutDays.getTime() - mstoSub);\n\n  return finalDate;\n}\n\n// Path: src/subYears/index.ts\nimport { addYears } from \"../addYears/index.js\";\n\n/**\n * @name subYears\n * @category Year Helpers\n * @summary Subtract the specified number of years from the given date.\n *\n * @description\n * Subtract the specified number of years from the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of years to be subtracted.\n *\n * @returns The new date with the years subtracted\n *\n * @example\n * // Subtract 5 years from 1 September 2014:\n * const result = subYears(new Date(2014, 8, 1), 5)\n * //=> Tue Sep 01 2009 00:00:00\n */\nexport function subYears<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addYears(date, -amount);\n}\n\n// Path: src/endOfMinute/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name endOfMinute\n * @category Minute Helpers\n * @summary Return the end of a minute for the given date.\n *\n * @description\n * Return the end of a minute for the given date.\n * The result will be in the local timezone.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The end of a minute\n *\n * @example\n * // The end of a minute for 1 December 2014 22:15:45.400:\n * const result = endOfMinute(new Date(2014, 11, 1, 22, 15, 45, 400))\n * //=> Mon Dec 01 2014 22:15:59.999\n */\nexport function endOfMinute<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  const _date = toDate(date);\n  _date.setSeconds(59, 999);\n  return _date;\n}\n\n// Path: src/endOfHour/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name endOfHour\n * @category Hour Helpers\n * @summary Return the end of an hour for the given date.\n *\n * @description\n * Return the end of an hour for the given date.\n * The result will be in the local timezone.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The end of an hour\n *\n * @example\n * // The end of an hour for 2 September 2014 11:55:00:\n * const result = endOfHour(new Date(2014, 8, 2, 11, 55))\n * //=> Tue Sep 02 2014 11:59:59.999\n */\nexport function endOfHour<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  const _date = toDate(date);\n  _date.setMinutes(59, 59, 999);\n  return _date;\n}\n\n// Path: src/subSeconds/index.ts\nimport { addSeconds } from \"../addSeconds/index.js\";\n\n/**\n * @name subSeconds\n * @category Second Helpers\n * @summary Subtract the specified number of seconds from the given date.\n *\n * @description\n * Subtract the specified number of seconds from the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of seconds to be subtracted.\n *\n * @returns The new date with the seconds subtracted\n *\n * @example\n * // Subtract 30 seconds from 10 July 2014 12:45:00:\n * const result = subSeconds(new Date(2014, 6, 10, 12, 45, 0), 30)\n * //=> Thu Jul 10 2014 12:44:30\n */\nexport function subSeconds<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addSeconds(date, -amount);\n}\n\n// Path: src/index.ts\n// This file is generated automatically by `scripts/build/indices.ts`. Please, don't change it.\n\nexport * from \"./add/index.js\";\nexport * from \"./addBusinessDays/index.js\";\nexport * from \"./addDays/index.js\";\nexport * from \"./addHours/index.js\";\nexport * from \"./addISOWeekYears/index.js\";\nexport * from \"./addMilliseconds/index.js\";\nexport * from \"./addMinutes/index.js\";\nexport * from \"./addMonths/index.js\";\nexport * from \"./addQuarters/index.js\";\nexport * from \"./addSeconds/index.js\";\nexport * from \"./addWeeks/index.js\";\nexport * from \"./addYears/index.js\";\nexport * from \"./areIntervalsOverlapping/index.js\";\nexport * from \"./clamp/index.js\";\nexport * from \"./closestIndexTo/index.js\";\nexport * from \"./closestTo/index.js\";\nexport * from \"./compareAsc/index.js\";\nexport * from \"./compareDesc/index.js\";\nexport * from \"./constructFrom/index.js\";\nexport * from \"./constructNow/index.js\";\nexport * from \"./daysToWeeks/index.js\";\nexport * from \"./differenceInBusinessDays/index.js\";\nexport * from \"./differenceInCalendarDays/index.js\";\nexport * from \"./differenceInCalendarISOWeekYears/index.js\";\nexport * from \"./differenceInCalendarISOWeeks/index.js\";\nexport * from \"./differenceInCalendarMonths/index.js\";\nexport * from \"./differenceInCalendarQuarters/index.js\";\nexport * from \"./differenceInCalendarWeeks/index.js\";\nexport * from \"./differenceInCalendarYears/index.js\";\nexport * from \"./differenceInDays/index.js\";\nexport * from \"./differenceInHours/index.js\";\nexport * from \"./differenceInISOWeekYears/index.js\";\nexport * from \"./differenceInMilliseconds/index.js\";\nexport * from \"./differenceInMinutes/index.js\";\nexport * from \"./differenceInMonths/index.js\";\nexport * from \"./differenceInQuarters/index.js\";\nexport * from \"./differenceInSeconds/index.js\";\nexport * from \"./differenceInWeeks/index.js\";\nexport * from \"./differenceInYears/index.js\";\nexport * from \"./eachDayOfInterval/index.js\";\nexport * from \"./eachHourOfInterval/index.js\";\nexport * from \"./eachMinuteOfInterval/index.js\";\nexport * from \"./eachMonthOfInterval/index.js\";\nexport * from \"./eachQuarterOfInterval/index.js\";\nexport * from \"./eachWeekOfInterval/index.js\";\nexport * from \"./eachWeekendOfInterval/index.js\";\nexport * from \"./eachWeekendOfMonth/index.js\";\nexport * from \"./eachWeekendOfYear/index.js\";\nexport * from \"./eachYearOfInterval/index.js\";\nexport * from \"./endOfDay/index.js\";\nexport * from \"./endOfDecade/index.js\";\nexport * from \"./endOfHour/index.js\";\nexport * from \"./endOfISOWeek/index.js\";\nexport * from \"./endOfISOWeekYear/index.js\";\nexport * from \"./endOfMinute/index.js\";\nexport * from \"./endOfMonth/index.js\";\nexport * from \"./endOfQuarter/index.js\";\nexport * from \"./endOfSecond/index.js\";\nexport * from \"./endOfToday/index.js\";\nexport * from \"./endOfTomorrow/index.js\";\nexport * from \"./endOfWeek/index.js\";\nexport * from \"./endOfYear/index.js\";\nexport * from \"./endOfYesterday/index.js\";\nexport * from \"./format/index.js\";\nexport * from \"./formatDistance/index.js\";\nexport * from \"./formatDistanceStrict/index.js\";\nexport * from \"./formatDistanceToNow/index.js\";\nexport * from \"./formatDistanceToNowStrict/index.js\";\nexport * from \"./formatDuration/index.js\";\nexport * from \"./formatISO/index.js\";\nexport * from \"./formatISO9075/index.js\";\nexport * from \"./formatISODuration/index.js\";\nexport * from \"./formatRFC3339/index.js\";\nexport * from \"./formatRFC7231/index.js\";\nexport * from \"./formatRelative/index.js\";\nexport * from \"./fromUnixTime/index.js\";\nexport * from \"./getDate/index.js\";\nexport * from \"./getDay/index.js\";\nexport * from \"./getDayOfYear/index.js\";\nexport * from \"./getDaysInMonth/index.js\";\nexport * from \"./getDaysInYear/index.js\";\nexport * from \"./getDecade/index.js\";\nexport * from \"./getDefaultOptions/index.js\";\nexport * from \"./getHours/index.js\";\nexport * from \"./getISODay/index.js\";\nexport * from \"./getISOWeek/index.js\";\nexport * from \"./getISOWeekYear/index.js\";\nexport * from \"./getISOWeeksInYear/index.js\";\nexport * from \"./getMilliseconds/index.js\";\nexport * from \"./getMinutes/index.js\";\nexport * from \"./getMonth/index.js\";\nexport * from \"./getOverlappingDaysInIntervals/index.js\";\nexport * from \"./getQuarter/index.js\";\nexport * from \"./getSeconds/index.js\";\nexport * from \"./getTime/index.js\";\nexport * from \"./getUnixTime/index.js\";\nexport * from \"./getWeek/index.js\";\nexport * from \"./getWeekOfMonth/index.js\";\nexport * from \"./getWeekYear/index.js\";\nexport * from \"./getWeeksInMonth/index.js\";\nexport * from \"./getYear/index.js\";\nexport * from \"./hoursToMilliseconds/index.js\";\nexport * from \"./hoursToMinutes/index.js\";\nexport * from \"./hoursToSeconds/index.js\";\nexport * from \"./interval/index.js\";\nexport * from \"./intervalToDuration/index.js\";\nexport * from \"./intlFormat/index.js\";\nexport * from \"./intlFormatDistance/index.js\";\nexport * from \"./isAfter/index.js\";\nexport * from \"./isBefore/index.js\";\nexport * from \"./isDate/index.js\";\nexport * from \"./isEqual/index.js\";\nexport * from \"./isExists/index.js\";\nexport * from \"./isFirstDayOfMonth/index.js\";\nexport * from \"./isFriday/index.js\";\nexport * from \"./isFuture/index.js\";\nexport * from \"./isLastDayOfMonth/index.js\";\nexport * from \"./isLeapYear/index.js\";\nexport * from \"./isMatch/index.js\";\nexport * from \"./isMonday/index.js\";\nexport * from \"./isPast/index.js\";\nexport * from \"./isSameDay/index.js\";\nexport * from \"./isSameHour/index.js\";\nexport * from \"./isSameISOWeek/index.js\";\nexport * from \"./isSameISOWeekYear/index.js\";\nexport * from \"./isSameMinute/index.js\";\nexport * from \"./isSameMonth/index.js\";\nexport * from \"./isSameQuarter/index.js\";\nexport * from \"./isSameSecond/index.js\";\nexport * from \"./isSameWeek/index.js\";\nexport * from \"./isSameYear/index.js\";\nexport * from \"./isSaturday/index.js\";\nexport * from \"./isSunday/index.js\";\nexport * from \"./isThisHour/index.js\";\nexport * from \"./isThisISOWeek/index.js\";\nexport * from \"./isThisMinute/index.js\";\nexport * from \"./isThisMonth/index.js\";\nexport * from \"./isThisQuarter/index.js\";\nexport * from \"./isThisSecond/index.js\";\nexport * from \"./isThisWeek/index.js\";\nexport * from \"./isThisYear/index.js\";\nexport * from \"./isThursday/index.js\";\nexport * from \"./isToday/index.js\";\nexport * from \"./isTomorrow/index.js\";\nexport * from \"./isTuesday/index.js\";\nexport * from \"./isValid/index.js\";\nexport * from \"./isWednesday/index.js\";\nexport * from \"./isWeekend/index.js\";\nexport * from \"./isWithinInterval/index.js\";\nexport * from \"./isYesterday/index.js\";\nexport * from \"./lastDayOfDecade/index.js\";\nexport * from \"./lastDayOfISOWeek/index.js\";\nexport * from \"./lastDayOfISOWeekYear/index.js\";\nexport * from \"./lastDayOfMonth/index.js\";\nexport * from \"./lastDayOfQuarter/index.js\";\nexport * from \"./lastDayOfWeek/index.js\";\nexport * from \"./lastDayOfYear/index.js\";\nexport * from \"./lightFormat/index.js\";\nexport * from \"./max/index.js\";\nexport * from \"./milliseconds/index.js\";\nexport * from \"./millisecondsToHours/index.js\";\nexport * from \"./millisecondsToMinutes/index.js\";\nexport * from \"./millisecondsToSeconds/index.js\";\nexport * from \"./min/index.js\";\nexport * from \"./minutesToHours/index.js\";\nexport * from \"./minutesToMilliseconds/index.js\";\nexport * from \"./minutesToSeconds/index.js\";\nexport * from \"./monthsToQuarters/index.js\";\nexport * from \"./monthsToYears/index.js\";\nexport * from \"./nextDay/index.js\";\nexport * from \"./nextFriday/index.js\";\nexport * from \"./nextMonday/index.js\";\nexport * from \"./nextSaturday/index.js\";\nexport * from \"./nextSunday/index.js\";\nexport * from \"./nextThursday/index.js\";\nexport * from \"./nextTuesday/index.js\";\nexport * from \"./nextWednesday/index.js\";\nexport * from \"./parse/index.js\";\nexport * from \"./parseISO/index.js\";\nexport * from \"./parseJSON/index.js\";\nexport * from \"./previousDay/index.js\";\nexport * from \"./previousFriday/index.js\";\nexport * from \"./previousMonday/index.js\";\nexport * from \"./previousSaturday/index.js\";\nexport * from \"./previousSunday/index.js\";\nexport * from \"./previousThursday/index.js\";\nexport * from \"./previousTuesday/index.js\";\nexport * from \"./previousWednesday/index.js\";\nexport * from \"./quartersToMonths/index.js\";\nexport * from \"./quartersToYears/index.js\";\nexport * from \"./roundToNearestHours/index.js\";\nexport * from \"./roundToNearestMinutes/index.js\";\nexport * from \"./secondsToHours/index.js\";\nexport * from \"./secondsToMilliseconds/index.js\";\nexport * from \"./secondsToMinutes/index.js\";\nexport * from \"./set/index.js\";\nexport * from \"./setDate/index.js\";\nexport * from \"./setDay/index.js\";\nexport * from \"./setDayOfYear/index.js\";\nexport * from \"./setDefaultOptions/index.js\";\nexport * from \"./setHours/index.js\";\nexport * from \"./setISODay/index.js\";\nexport * from \"./setISOWeek/index.js\";\nexport * from \"./setISOWeekYear/index.js\";\nexport * from \"./setMilliseconds/index.js\";\nexport * from \"./setMinutes/index.js\";\nexport * from \"./setMonth/index.js\";\nexport * from \"./setQuarter/index.js\";\nexport * from \"./setSeconds/index.js\";\nexport * from \"./setWeek/index.js\";\nexport * from \"./setWeekYear/index.js\";\nexport * from \"./setYear/index.js\";\nexport * from \"./startOfDay/index.js\";\nexport * from \"./startOfDecade/index.js\";\nexport * from \"./startOfHour/index.js\";\nexport * from \"./startOfISOWeek/index.js\";\nexport * from \"./startOfISOWeekYear/index.js\";\nexport * from \"./startOfMinute/index.js\";\nexport * from \"./startOfMonth/index.js\";\nexport * from \"./startOfQuarter/index.js\";\nexport * from \"./startOfSecond/index.js\";\nexport * from \"./startOfToday/index.js\";\nexport * from \"./startOfTomorrow/index.js\";\nexport * from \"./startOfWeek/index.js\";\nexport * from \"./startOfWeekYear/index.js\";\nexport * from \"./startOfYear/index.js\";\nexport * from \"./startOfYesterday/index.js\";\nexport * from \"./sub/index.js\";\nexport * from \"./subBusinessDays/index.js\";\nexport * from \"./subDays/index.js\";\nexport * from \"./subHours/index.js\";\nexport * from \"./subISOWeekYears/index.js\";\nexport * from \"./subMilliseconds/index.js\";\nexport * from \"./subMinutes/index.js\";\nexport * from \"./subMonths/index.js\";\nexport * from \"./subQuarters/index.js\";\nexport * from \"./subSeconds/index.js\";\nexport * from \"./subWeeks/index.js\";\nexport * from \"./subYears/index.js\";\nexport * from \"./toDate/index.js\";\nexport * from \"./transpose/index.js\";\nexport * from \"./weeksToDays/index.js\";\nexport * from \"./yearsToDays/index.js\";\nexport * from \"./yearsToMonths/index.js\";\nexport * from \"./yearsToQuarters/index.js\";\nexport type * from \"./types.js\";\n\n// Path: src/getTime/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { getTime } from \"./index.js\";\n\ndescribe(\"getTime\", () => {\n  it(\"returns the timestamp of the given date\", () => {\n    const timestamp = 1483228800000;\n    const result = getTime(new Date(timestamp));\n    expect(result).toBe(timestamp);\n  });\n\n  it(\"accepts a timestamp (and returns it unchanged)\", () => {\n    const timestamp = 804643200000;\n    const result = getTime(timestamp);\n    expect(result).toBe(timestamp);\n  });\n\n  it(\"returns NaN if the given date is invalid\", () => {\n    const result = getTime(new Date(NaN));\n    expect(isNaN(result)).toBe(true);\n  });\n});\n\n// Path: src/closestIndexTo/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { closestIndexTo } from \"./index.js\";\n\ndescribe(\"closestIndexTo\", () => {\n  it(\"returns the date index from the given array closest to the given date\", () => {\n    const date = new Date(2014, 6 /* Jul */, 2);\n    const result = closestIndexTo(date, [\n      new Date(2015, 7 /* Aug */, 31),\n      new Date(2012, 6 /* Jul */, 2),\n    ]);\n    expect(result).toBe(0);\n  });\n\n  it(\"works if the closest date from the given array is before the given date\", () => {\n    const date = new Date(2014, 6 /* Jul */, 2, 6, 30, 4, 500);\n    const result = closestIndexTo(date, [\n      new Date(2014, 6 /* Jul */, 2, 6, 30, 5, 900),\n      new Date(2014, 6 /* Jul */, 2, 6, 30, 3, 900),\n      new Date(2014, 6 /* Jul */, 2, 6, 30, 10),\n    ]);\n    expect(result).toBe(1);\n  });\n\n  it(\"accepts timestamps\", () => {\n    const date = new Date(2014, 6 /* Jul */, 2).getTime();\n    const result = closestIndexTo(date, [\n      new Date(2015, 7 /* Aug */, 31).getTime(),\n      new Date(2012, 6 /* Jul */, 2).getTime(),\n    ]);\n    expect(result).toBe(0);\n  });\n\n  it(\"returns undefined if the given array is empty\", () => {\n    const date = new Date(2014, 6 /* Jul */, 2).getTime();\n    const result = closestIndexTo(date, []);\n    expect(result).toBe(undefined);\n  });\n\n  it(\"returns NaN if the given date is `Invalid Date`\", () => {\n    const date = new Date(NaN);\n    const result = closestIndexTo(date, [\n      new Date(2015, 7 /* Aug */, 31),\n      new Date(2012, 6 /* Jul */, 2),\n    ]);\n    expect(result != null && isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if any date in the given array is `Invalid Date`\", () => {\n    const date = new Date(2014, 6 /* Jul */, 2);\n    const result = closestIndexTo(date, [\n      new Date(2015, 7 /* Aug */, 31),\n      new Date(NaN),\n      new Date(2012, 6 /* Jul */, 2),\n    ]);\n    expect(result != null && isNaN(result)).toBe(true);\n  });\n});\n\n// Path: src/millisecondsToHours/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { millisecondsToHours } from \"./index.js\";\n\ndescribe(\"millisecondsToHours\", () => {\n  it(\"converts milliseconds to hours\", () => {\n    expect(millisecondsToHours(3600000)).toBe(1);\n    expect(millisecondsToHours(7200000)).toBe(2);\n  });\n\n  it(\"uses floor rounding\", () => {\n    expect(millisecondsToHours(3600001)).toBe(1);\n    expect(millisecondsToHours(3599999)).toBe(0);\n  });\n\n  it(\"handles border values\", () => {\n    expect(millisecondsToHours(3600000.5)).toBe(1);\n    expect(millisecondsToHours(0)).toBe(0);\n  });\n\n  it(\"properly works with negative numbers\", () => {\n    expect(millisecondsToHours(123456789)).toBe(34);\n    expect(millisecondsToHours(-123456789)).toBe(-34);\n  });\n});\n\n// Path: src/isThisMonth/test.ts\nimport { UTCDate } from \"@date-fns/utc\";\nimport sinon from \"sinon\";\nimport { afterEach, beforeEach, describe, expect, it } from \"vitest\";\nimport { isThisMonth } from \"./index.js\";\n\ndescribe(\"isThisMonth\", () => {\n  let clock: sinon.SinonFakeTimers;\n  beforeEach(() => {\n    clock = sinon.useFakeTimers(new Date(2014, 8 /* Sep */, 1).getTime());\n  });\n\n  afterEach(() => {\n    clock.restore();\n  });\n\n  it(\"returns true if the given date and the current date have the same month (and year)\", () => {\n    const date = new Date(2014, 8 /* Sep */, 15);\n    expect(isThisMonth(date)).toBe(true);\n  });\n\n  it(\"returns false if the given date and the current date have different months\", () => {\n    const date = new Date(2013, 7 /* Aug */, 31);\n    expect(isThisMonth(date)).toBe(false);\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const date = new Date(2014, 8 /* Sep */, 30).getTime();\n    expect(isThisMonth(date)).toBe(true);\n  });\n\n  it(\"respects date extensions\", () => {\n    expect(isThisMonth(new UTCDate(+new Date(2014, 8 /* Sep */, 1)))).toBe(\n      true,\n    );\n  });\n});\n\n// Path: src/subMonths/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { subMonths } from \"./index.js\";\n\ndescribe(\"subMonths\", () => {\n  it(\"subtracts the given number of months\", () => {\n    const result = subMonths(new Date(2015, 1 /* Feb */, 1), 5);\n    expect(result).toEqual(new Date(2014, 8 /* Sep */, 1));\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const result = subMonths(new Date(2015, 8 /* Sep */, 1).getTime(), 12);\n    expect(result).toEqual(new Date(2014, 8 /* Sep */, 1));\n  });\n\n  it(\"does not mutate the original date\", () => {\n    const date = new Date(2014, 8 /* Sep */, 1);\n    subMonths(date, 12);\n    expect(date).toEqual(new Date(2014, 8 /* Sep */, 1));\n  });\n\n  it(\"works well if the desired month has fewer days and the provided date is in the last day of a month\", () => {\n    const date = new Date(2014, 11 /* Dec */, 31);\n    const result = subMonths(date, 3);\n    expect(result).toEqual(new Date(2014, 8 /* Sep */, 30));\n  });\n\n  it(\"handles dates before 100 AD\", () => {\n    const initialDate = new Date(0);\n    initialDate.setFullYear(1, 2 /* Mar */, 31);\n    initialDate.setHours(0, 0, 0, 0);\n    const expectedResult = new Date(0);\n    expectedResult.setFullYear(1, 1 /* Feb */, 28);\n    expectedResult.setHours(0, 0, 0, 0);\n    const result = subMonths(initialDate, 1);\n    expect(result).toEqual(expectedResult);\n  });\n\n  it(\"returns `Invalid Date` if the given date is invalid\", () => {\n    const result = subMonths(new Date(NaN), 5);\n    expect(result instanceof Date && isNaN(result.getTime())).toBe(true);\n  });\n\n  it(\"returns `Invalid Date` if the given amount is NaN\", () => {\n    const result = subMonths(new Date(2015, 1 /* Feb */, 1), NaN);\n    expect(result instanceof Date && isNaN(result.getTime())).toBe(true);\n  });\n});\n\n// Path: src/max/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { max } from \"./index.js\";\n\ndescribe(\"max\", () => {\n  it(\"returns the latest date\", () => {\n    const result = max([\n      new Date(1989, 6 /* Jul */, 10),\n      new Date(1987, 1 /* Feb */, 11),\n    ]);\n    expect(result).toEqual(new Date(1989, 6 /* Jul */, 10));\n  });\n\n  it(\"accepts array with more than 2 entries\", () => {\n    const result = max([\n      new Date(1987, 1 /* Feb */, 11),\n      new Date(1989, 6 /* Jul */, 10),\n      new Date(1995, 6 /* Jul */, 2),\n      new Date(1990, 0 /* Jan */, 1),\n    ]);\n    expect(result).toEqual(new Date(1995, 6 /* Jul */, 2));\n  });\n\n  it(\"accepts timestamps\", () => {\n    const result = max([\n      new Date(1989, 6 /* Jul */, 10).getTime(),\n      new Date(1987, 1 /* Feb */, 11).getTime(),\n    ]);\n    expect(result).toEqual(new Date(1989, 6 /* Jul */, 10));\n  });\n\n  it(\"returns `Invalid Date` if any given date is invalid\", () => {\n    const result = max([\n      new Date(1989, 6 /* Jul */, 10),\n      new Date(NaN),\n      new Date(1987, 1 /* Feb */, 11),\n    ]);\n    expect(isNaN(+result)).toBe(true);\n  });\n\n  it(\"returns `Invalid Date` for empty array\", () => {\n    const result = max([]);\n    expect(isNaN(+result)).toBe(true);\n  });\n});\n\n// Path: src/_lib/test/index.ts\nimport { addLeadingZeros } from \"../addLeadingZeros/index.js\";\nimport { setDefaultOptions } from \"../defaultOptions/index.js\";\n\nexport function assertType<T>(_: T): void {}\n\nexport function resetDefaultOptions(): void {\n  setDefaultOptions({});\n}\n\n// This makes sure we create the consistent offsets across timezones, no matter where these tests are ran.\nexport function generateOffset(originalDate: Date) {\n  // Add the timezone.\n  let offset = \"\";\n  const tzOffset = originalDate.getTimezoneOffset();\n\n  if (tzOffset !== 0) {\n    const absoluteOffset = Math.abs(tzOffset);\n    const hourOffset = addLeadingZeros(Math.trunc(absoluteOffset / 60), 2);\n    const minuteOffset = addLeadingZeros(absoluteOffset % 60, 2);\n    // If less than 0, the sign is +, because it is ahead of time.\n    const sign = tzOffset < 0 ? \"+\" : \"-\";\n\n    offset = `${sign}${hourOffset}:${minuteOffset}`;\n  } else {\n    offset = \"Z\";\n  }\n\n  return offset;\n}\n\n// Path: src/formatISO/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport sinon from \"sinon\";\nimport { formatISO } from \"./index.js\";\nimport { generateOffset } from \"../_lib/test/index.js\";\n\ndescribe(\"formatISO\", () => {\n  it(\"formats ISO-8601 extended format\", () => {\n    const date = new Date(2019, 2 /* Mar */, 3, 19, 0, 52, 123);\n    const tzOffsetExtended = generateOffset(date);\n    expect(formatISO(date)).toBe(`2019-03-03T19:00:52${tzOffsetExtended}`);\n\n    const getTimezoneOffsetStub = sinon.stub(\n      Date.prototype,\n      \"getTimezoneOffset\",\n    );\n\n    getTimezoneOffsetStub.returns(480);\n    const tzNegativeOffsetExtended = generateOffset(date);\n    expect(formatISO(date)).toBe(`2019-03-03T19:00:52${tzNegativeOffsetExtended}`);\n\n    getTimezoneOffsetStub.returns(0);\n    const tzZOffsetExtended = generateOffset(date);\n    expect(formatISO(date)).toBe(`2019-03-03T19:00:52${tzZOffsetExtended}`);\n\n    getTimezoneOffsetStub.restore();\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const date = new Date(2019, 2 /* Mar */, 3, 19, 0, 52, 123).getTime();\n    const tzOffsetExtended = generateOffset(new Date(date));\n    expect(formatISO(date)).toBe(`2019-03-03T19:00:52${tzOffsetExtended}`);\n  });\n\n  it(\"formats ISO-8601 basic format\", () => {\n    const date = new Date(2019, 9 /* Oct */, 4, 12, 30, 13, 456);\n    const tzOffsetBasic = generateOffset(date);\n    expect(formatISO(date, { format: \"basic\" })).toBe(`20191004T123013${tzOffsetBasic}`);\n  });\n\n  it(\"formats only date\", () => {\n    const date = new Date(2019, 11 /* Dec */, 11, 1, 0, 0, 789);\n\n    expect(formatISO(date, { representation: \"date\", format: \"extended\" })).toBe(\"2019-12-11\");\n    expect(formatISO(date, { representation: \"date\", format: \"basic\" })).toBe(\"20191211\");\n  });\n\n  it(\"formats only time\", () => {\n    const date = new Date(2019, 2 /* Mar */, 3, 19, 0, 52, 123);\n    const tzOffset = generateOffset(date);\n\n    expect(formatISO(date, { representation: \"time\", format: \"extended\" })).toBe(`19:00:52${tzOffset}`);\n    expect(formatISO(date, { representation: \"time\", format: \"basic\" })).toBe(`190052${tzOffset}`);\n  });\n\n  it(\"throws RangeError if the time value is invalid\", () => {\n    expect(formatISO.bind(null, new Date(NaN))).toThrow(RangeError);\n  });\n});\n\n// Path: src/startOfSecond/test.ts\nimport { describe, expect, it } from \"vitest\";\n...\n", "needle_token_start": 5234, "needle_token_end": 5585, "code_context_ntokens": 15852, "output": ["```typescript\nimport { add } from \"../add/index.js\";\nimport { differenceInDays } from \"../differenceInDays/index.js\";\nimport { differenceInHours } from \"../differenceInHours/index.js\";\nimport { differenceInMinutes } from \"../differenceInMinutes/index.js\";\nimport { differenceInMonths } from \"../differenceInMonths/index.js\";\nimport { differenceInSeconds } from \"../differenceInSeconds/index.js\";\nimport { differenceInYears } from \"../differenceInYears/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { Duration, Interval } from \"../types.js\";\n\n/**\n * @name intervalToDuration\n * @category Common Helpers\n * @summary Convert interval to duration\n *\n * @description\n * Convert a interval object to a duration object.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param interval - The interval to convert to duration\n *\n * @returns The duration object\n *\n * @example\n * // Get the duration between January 15, 1929 and April 4, 1968.\n * intervalToDuration({\n *   start: new Date(1929, 0, 15, 12, 0, 0),\n *   end: new Date(1968, 3, 4, 19, 5, 0)\n * })\n * // => { years: 39, months: 2, days: 20, hours: 7, minutes: 5, seconds: 0 }\n */\nexport function intervalToDuration<DateType extends Date>(\n  interval: Interval<DateType>,\n): Duration {\n  const start = toDate(interval.start);\n  const end = toDate(interval.end);\n\n  const duration: Duration = {};\n\n  const years = differenceInYears(end, start);\n  if (years) duration.years = years;\n\n  const remainingMonths = add(start, { years: duration.years });\n\n  const months = differenceInMonths(end, remainingMonths);\n  if (months) duration.months = months;\n\n  const remainingDays = add(remainingMonths, { months: duration.months });\n\n  const days = differenceInDays(end, remainingDays);\n  if (days) duration.days = days;\n\n  const remainingHours = add(remainingDays, { days: duration.days });\n\n  const hours = differenceInHours(end, remainingHours);\n  if (hours) duration.hours = hours;\n\n  const remainingMinutes = add(remainingHours, { hours: duration.hours });\n\n  const minutes = differenceInMinutes(end, remainingMinutes);\n  if (minutes) duration.minutes = minutes;\n\n  const remainingSeconds = add(remainingMinutes, { minutes: duration.minutes });\n\n  const seconds = differenceInSeconds(end, remainingSeconds);\n  if (seconds) duration.seconds = seconds;\n\n  return duration;\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "formatTimezoneShort", "language": "typescript", "path": "src/_lib/format/formatters/index.ts", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To generate a string representation of a timezone offset in a concise format, optionally including a delimiter between hours and minutes.\n2. **Input**: Takes two parameters: an integer representing the timezone offset in minutes, and an optional string delimiter (default is empty).\n3. **Output**: Returns a string that represents the timezone offset. The output includes a sign ('+' or '-'), the absolute value of the hours, and, if applicable, the minutes with leading zeros. If the delimiter is provided, it is used between hours and minutes.\n4. **Procedure**: \n   - First, determine the sign based on whether the offset is positive or negative.\n   - Calculate the absolute value of the offset.\n   - Compute the hours by dividing the absolute offset by 60 and truncating the result.\n   - Calculate the remaining minutes using the modulus operation.\n   - If minutes are zero, return the sign concatenated with the hours. If minutes are not zero, return the sign, hours, delimiter (if provided), and minutes (with leading zeros if necessary).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "  midnight: \"midnight\",\n  noon: \"noon\",\n  morning: \"morning\",\n  afternoon: \"afternoon\",\n  evening: \"evening\",\n  night: \"night\",\n} as const;\n\ntype Formatter = (\n  date: Date,\n  token: string,\n  localize: Localize,\n  options: Required<\n    LocalizedOptions<\"options\"> & WeekOptions & FirstWeekContainsDateOptions\n  >,\n) => string;\n\n/*\n * |     | Unit                           |     | Unit                           |\n * |-----|--------------------------------|-----|--------------------------------|\n * |  a  | AM, PM                         |  A* | Milliseconds in day            |\n * |  b  | AM, PM, noon, midnight         |  B  | Flexible day period            |\n * |  c  | Stand-alone local day of week  |  C* | Localized hour w/ day period   |\n * |  d  | Day of month                   |  D  | Day of year                    |\n * |  e  | Local day of week              |  E  | Day of week                    |\n * |  f  |                                |  F* | Day of week in month           |\n * |  g* | Modified Julian day            |  G  | Era                            |\n * |  h  | Hour [1-12]                    |  H  | Hour [0-23]                    |\n * |  i! | ISO day of week                |  I! | ISO week of year               |\n * |  j* | Localized hour w/ day period   |  J* | Localized hour w/o day period  |\n * |  k  | Hour [1-24]                    |  K  | Hour [0-11]                    |\n * |  l* | (deprecated)                   |  L  | Stand-alone month              |\n * |  m  | Minute                         |  M  | Month                          |\n * |  n  |                                |  N  |                                |\n * |  o! | Ordinal number modifier        |  O  | Timezone (GMT)                 |\n * |  p! | Long localized time            |  P! | Long localized date            |\n * |  q  | Stand-alone quarter            |  Q  | Quarter                        |\n * |  r* | Related Gregorian year         |  R! | ISO week-numbering year        |\n * |  s  | Second                         |  S  | Fraction of second             |\n * |  t! | Seconds timestamp              |  T! | Milliseconds timestamp         |\n * |  u  | Extended year                  |  U* | Cyclic year                    |\n * |  v* | Timezone (generic non-locat.)  |  V* | Timezone (location)            |\n * |  w  | Local week of year             |  W* | Week of month                  |\n * |  x  | Timezone (ISO-8601 w/o Z)      |  X  | Timezone (ISO-8601)            |\n * |  y  | Year (abs)                     |  Y  | Local week-numbering year      |\n * |  z  | Timezone (specific non-locat.) |  Z* | Timezone (aliases)             |\n *\n * Letters marked by * are not implemented but reserved by Unicode standard.\n *\n * Letters marked by ! are non-standard, but implemented by date-fns:\n * - `o` modifies the previous token to turn it into an ordinal (see `format` docs)\n * - `i` is ISO day of week. For `i` and `ii` is returns numeric ISO week days,\n *   i.e. 7 for Sunday, 1 for Monday, etc.\n * - `I` is ISO week of year, as opposed to `w` which is local week of year.\n * - `R` is ISO week-numbering year, as opposed to `Y` which is local week-numbering year.\n *   `R` is supposed to be used in conjunction with `I` and `i`\n *   for universal ISO week-numbering date, whereas\n *   `Y` is supposed to be used in conjunction with `w` and `e`\n *   for week-numbering date specific to the locale.\n * - `P` is long localized date format\n * - `p` is long localized time format\n */\n\nexport const formatters: { [token: string]: Formatter } = {\n  // Era\n  G: function (date, token, localize) {\n    const era: Era = date.getFullYear() > 0 ? 1 : 0;\n    switch (token) {\n      // AD, BC\n      case \"G\":\n      case \"GG\":\n      case \"GGG\":\n        return localize.era(era, { width: \"abbreviated\" });\n      // A, B\n      case \"GGGGG\":\n        return localize.era(era, { width: \"narrow\" });\n      // Anno Domini, Before Christ\n      case \"GGGG\":\n      default:\n        return localize.era(era, { width: \"wide\" });\n    }\n  },\n\n  // Year\n  y: function (date, token, localize) {\n    // Ordinal number\n    if (token === \"yo\") {\n      const signedYear = date.getFullYear();\n      // Returns 1 for 1 BC (which is year 0 in JavaScript)\n      const year = signedYear > 0 ? signedYear : 1 - signedYear;\n      return localize.ordinalNumber(year, { unit: \"year\" });\n    }\n\n    return lightFormatters.y(date, token);\n  },\n\n  // Local week-numbering year\n  Y: function (date, token, localize, options) {\n    const signedWeekYear = getWeekYear(date, options);\n    // Returns 1 for 1 BC (which is year 0 in JavaScript)\n    const weekYear = signedWeekYear > 0 ? signedWeekYear : 1 - signedWeekYear;\n\n    // Two digit year\n    if (token === \"YY\") {\n      const twoDigitYear = weekYear % 100;\n      return addLeadingZeros(twoDigitYear, 2);\n    }\n\n    // Ordinal number\n    if (token === \"Yo\") {\n      return localize.ordinalNumber(weekYear, { unit: \"year\" });\n    }\n\n    // Padding\n    return addLeadingZeros(weekYear, token.length);\n  },\n\n  // ISO week-numbering year\n  R: function (date, token) {\n    const isoWeekYear = getISOWeekYear(date);\n\n    // Padding\n    return addLeadingZeros(isoWeekYear, token.length);\n  },\n\n  // Extended year. This is a single number designating the year of this calendar system.\n  // The main difference between `y` and `u` localizers are B.C. years:\n  // | Year | `y` | `u` |\n  // |------|-----|-----|\n  // | AC 1 |   1 |   1 |\n  // | BC 1 |   1 |   0 |\n  // | BC 2 |   2 |  -1 |\n  // Also `yy` always returns the last two digits of a year,\n  // while `uu` pads single digit years to 2 characters and returns other years unchanged.\n  u: function (date, token) {\n    const year = date.getFullYear();\n    return addLeadingZeros(year, token.length);\n  },\n\n  // Quarter\n  Q: function (date, token, localize) {\n    const quarter = Math.ceil((date.getMonth() + 1) / 3) as Quarter;\n    switch (token) {\n      // 1, 2, 3, 4\n      case \"Q\":\n        return String(quarter);\n      // 01, 02, 03, 04\n      case \"QQ\":\n        return addLeadingZeros(quarter, 2);\n      // 1st, 2nd, 3rd, 4th\n      case \"Qo\":\n        return localize.ordinalNumber(quarter, { unit: \"quarter\" });\n      // Q1, Q2, Q3, Q4\n      case \"QQQ\":\n        return localize.quarter(quarter, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      // 1, 2, 3, 4 (narrow quarter; could be not numerical)\n      case \"QQQQQ\":\n        return localize.quarter(quarter, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      // 1st quarter, 2nd quarter, ...\n      case \"QQQQ\":\n      default:\n        return localize.quarter(quarter, {\n          width: \"wide\",\n          context: \"formatting\",\n        });\n    }\n  },\n\n  // Stand-alone quarter\n  q: function (date, token, localize) {\n    const quarter = Math.ceil((date.getMonth() + 1) / 3) as Quarter;\n    switch (token) {\n      // 1, 2, 3, 4\n      case \"q\":\n        return String(quarter);\n      // 01, 02, 03, 04\n      case \"qq\":\n        return addLeadingZeros(quarter, 2);\n      // 1st, 2nd, 3rd, 4th\n      case \"qo\":\n        return localize.ordinalNumber(quarter, { unit: \"quarter\" });\n      // Q1, Q2, Q3, Q4\n      case \"qqq\":\n        return localize.quarter(quarter, {\n          width: \"abbreviated\",\n          context: \"standalone\",\n        });\n      // 1, 2, 3, 4 (narrow quarter; could be not numerical)\n      case \"qqqqq\":\n        return localize.quarter(quarter, {\n          width: \"narrow\",\n          context: \"standalone\",\n        });\n      // 1st quarter, 2nd quarter, ...\n      case \"qqqq\":\n      default:\n        return localize.quarter(quarter, {\n          width: \"wide\",\n          context: \"standalone\",\n        });\n    }\n  },\n\n  // Month\n  M: function (date, token, localize) {\n    const month = date.getMonth() as Month;\n    switch (token) {\n      case \"M\":\n      case \"MM\":\n        return lightFormatters.M(date, token);\n      // 1st, 2nd, ..., 12th\n      case \"Mo\":\n        return localize.ordinalNumber(month + 1, { unit: \"month\" });\n      // Jan, Feb, ..., Dec\n      case \"MMM\":\n        return localize.month(month, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      // J, F, ..., D\n      case \"MMMMM\":\n        return localize.month(month, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      // January, February, ..., December\n      case \"MMMM\":\n      default:\n        return localize.month(month, { width: \"wide\", context: \"formatting\" });\n    }\n  },\n\n  // Stand-alone month\n  L: function (date, token, localize) {\n    const month = date.getMonth() as Month;\n    switch (token) {\n      // 1, 2, ..., 12\n      case \"L\":\n        return String(month + 1);\n      // 01, 02, ..., 12\n      case \"LL\":\n        return addLeadingZeros(month + 1, 2);\n      // 1st, 2nd, ..., 12th\n      case \"Lo\":\n        return localize.ordinalNumber(month + 1, { unit: \"month\" });\n      // Jan, Feb, ..., Dec\n      case \"LLL\":\n        return localize.month(month, {\n          width: \"abbreviated\",\n          context: \"standalone\",\n        });\n      // J, F, ..., D\n      case \"LLLLL\":\n        return localize.month(month, {\n          width: \"narrow\",\n          context: \"standalone\",\n        });\n      // January, February, ..., December\n      case \"LLLL\":\n      default:\n        return localize.month(month, { width: \"wide\", context: \"standalone\" });\n    }\n  },\n\n  // Local week of year\n  w: function (date, token, localize, options) {\n    const week = getWeek(date, options);\n\n    if (token === \"wo\") {\n      return localize.ordinalNumber(week, { unit: \"week\" });\n    }\n\n    return addLeadingZeros(week, token.length);\n  },\n\n  // ISO week of year\n  I: function (date, token, localize) {\n    const isoWeek = getISOWeek(date);\n\n    if (token === \"Io\") {\n      return localize.ordinalNumber(isoWeek, { unit: \"week\" });\n    }\n\n    return addLeadingZeros(isoWeek, token.length);\n  },\n\n  // Day of the month\n  d: function (date, token, localize) {\n    if (token === \"do\") {\n      return localize.ordinalNumber(date.getDate(), { unit: \"date\" });\n    }\n\n    return lightFormatters.d(date, token);\n  },\n\n  // Day of year\n  D: function (date, token, localize) {\n    const dayOfYear = getDayOfYear(date);\n\n    if (token === \"Do\") {\n      return localize.ordinalNumber(dayOfYear, { unit: \"dayOfYear\" });\n    }\n\n    return addLeadingZeros(dayOfYear, token.length);\n  },\n\n  // Day of week\n  E: function (date, token, localize) {\n    const dayOfWeek = date.getDay() as Day;\n    switch (token) {\n      // Tue\n      case \"E\":\n      case \"EE\":\n      case \"EEE\":\n        return localize.day(dayOfWeek, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      // T\n      case \"EEEEE\":\n        return localize.day(dayOfWeek, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      // Tu\n      case \"EEEEEE\":\n        return localize.day(dayOfWeek, {\n          width: \"short\",\n          context: \"formatting\",\n        });\n      // Tuesday\n      case \"EEEE\":\n      default:\n        return localize.day(dayOfWeek, {\n          width: \"wide\",\n          context: \"formatting\",\n        });\n    }\n  },\n\n  // Local day of week\n  e: function (date, token, localize, options) {\n    const dayOfWeek = date.getDay() as Day;\n    const localDayOfWeek = (dayOfWeek - options.weekStartsOn + 8) % 7 || 7;\n    switch (token) {\n      // Numerical value (Nth day of week with current locale or weekStartsOn)\n      case \"e\":\n        return String(localDayOfWeek);\n      // Padded numerical value\n      case \"ee\":\n        return addLeadingZeros(localDayOfWeek, 2);\n      // 1st, 2nd, ..., 7th\n      case \"eo\":\n        return localize.ordinalNumber(localDayOfWeek, { unit: \"day\" });\n      case \"eee\":\n        return localize.day(dayOfWeek, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      // T\n      case \"eeeee\":\n        return localize.day(dayOfWeek, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      // Tu\n      case \"eeeeee\":\n        return localize.day(dayOfWeek, {\n          width: \"short\",\n          context: \"formatting\",\n        });\n      // Tuesday\n      case \"eeee\":\n      default:\n        return localize.day(dayOfWeek, {\n          width: \"wide\",\n          context: \"formatting\",\n        });\n    }\n  },\n\n  // Stand-alone local day of week\n  c: function (date, token, localize, options) {\n    const dayOfWeek = date.getDay() as Day;\n    const localDayOfWeek = (dayOfWeek - options.weekStartsOn + 8) % 7 || 7;\n    switch (token) {\n      // Numerical value (same as in `e`)\n      case \"c\":\n        return String(localDayOfWeek);\n      // Padded numerical value\n      case \"cc\":\n        return addLeadingZeros(localDayOfWeek, token.length);\n      // 1st, 2nd, ..., 7th\n      case \"co\":\n        return localize.ordinalNumber(localDayOfWeek, { unit: \"day\" });\n      case \"ccc\":\n        return localize.day(dayOfWeek, {\n          width: \"abbreviated\",\n          context: \"standalone\",\n        });\n      // T\n      case \"ccccc\":\n        return localize.day(dayOfWeek, {\n          width: \"narrow\",\n          context: \"standalone\",\n        });\n      // Tu\n      case \"cccccc\":\n        return localize.day(dayOfWeek, {\n          width: \"short\",\n          context: \"standalone\",\n        });\n      // Tuesday\n      case \"cccc\":\n      default:\n        return localize.day(dayOfWeek, {\n          width: \"wide\",\n          context: \"standalone\",\n        });\n    }\n  },\n\n  // ISO day of week\n  i: function (date, token, localize) {\n    const dayOfWeek = date.getDay() as Day;\n    const isoDayOfWeek = dayOfWeek === 0 ? 7 : dayOfWeek;\n    switch (token) {\n      // 2\n      case \"i\":\n        return String(isoDayOfWeek);\n      // 02\n      case \"ii\":\n        return addLeadingZeros(isoDayOfWeek, token.length);\n      // 2nd\n      case \"io\":\n        return localize.ordinalNumber(isoDayOfWeek, { unit: \"day\" });\n      // Tue\n      case \"iii\":\n        return localize.day(dayOfWeek, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      // T\n      case \"iiiii\":\n        return localize.day(dayOfWeek, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      // Tu\n      case \"iiiiii\":\n        return localize.day(dayOfWeek, {\n          width: \"short\",\n          context: \"formatting\",\n        });\n      // Tuesday\n      case \"iiii\":\n      default:\n        return localize.day(dayOfWeek, {\n          width: \"wide\",\n          context: \"formatting\",\n        });\n    }\n  },\n\n  // AM or PM\n  a: function (date, token, localize) {\n    const hours = date.getHours();\n    const dayPeriodEnumValue = hours / 12 >= 1 ? \"pm\" : \"am\";\n\n    switch (token) {\n      case \"a\":\n      case \"aa\":\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      case \"aaa\":\n        return localize\n          .dayPeriod(dayPeriodEnumValue, {\n            width: \"abbreviated\",\n            context: \"formatting\",\n          })\n          .toLowerCase();\n      case \"aaaaa\":\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      case \"aaaa\":\n      default:\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"wide\",\n          context: \"formatting\",\n        });\n    }\n  },\n\n  // AM, PM, midnight, noon\n  b: function (date, token, localize) {\n    const hours = date.getHours();\n    let dayPeriodEnumValue: LocaleDayPeriod;\n    if (hours === 12) {\n      dayPeriodEnumValue = dayPeriodEnum.noon;\n    } else if (hours === 0) {\n      dayPeriodEnumValue = dayPeriodEnum.midnight;\n    } else {\n      dayPeriodEnumValue = hours / 12 >= 1 ? \"pm\" : \"am\";\n    }\n\n    switch (token) {\n      case \"b\":\n      case \"bb\":\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      case \"bbb\":\n        return localize\n          .dayPeriod(dayPeriodEnumValue, {\n            width: \"abbreviated\",\n            context: \"formatting\",\n          })\n          .toLowerCase();\n      case \"bbbbb\":\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      case \"bbbb\":\n      default:\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"wide\",\n          context: \"formatting\",\n        });\n    }\n  },\n\n  // in the morning, in the afternoon, in the evening, at night\n  B: function (date, token, localize) {\n    const hours = date.getHours();\n    let dayPeriodEnumValue: LocaleDayPeriod;\n    if (hours >= 17) {\n      dayPeriodEnumValue = dayPeriodEnum.evening;\n    } else if (hours >= 12) {\n      dayPeriodEnumValue = dayPeriodEnum.afternoon;\n    } else if (hours >= 4) {\n      dayPeriodEnumValue = dayPeriodEnum.morning;\n    } else {\n      dayPeriodEnumValue = dayPeriodEnum.night;\n    }\n\n    switch (token) {\n      case \"B\":\n      case \"BB\":\n      case \"BBB\":\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"abbreviated\",\n          context: \"formatting\",\n        });\n      case \"BBBBB\":\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"narrow\",\n          context: \"formatting\",\n        });\n      case \"BBBB\":\n      default:\n        return localize.dayPeriod(dayPeriodEnumValue, {\n          width: \"wide\",\n          context: \"formatting\",\n        });\n    }\n  },\n\n  // Hour [1-12]\n  h: function (date, token, localize) {\n    if (token === \"ho\") {\n      let hours = date.getHours() % 12;\n      if (hours === 0) hours = 12;\n      return localize.ordinalNumber(hours, { unit: \"hour\" });\n    }\n\n    return lightFormatters.h(date, token);\n  },\n\n  // Hour [0-23]\n  H: function (date, token, localize) {\n    if (token === \"Ho\") {\n      return localize.ordinalNumber(date.getHours(), { unit: \"hour\" });\n    }\n\n    return lightFormatters.H(date, token);\n  },\n\n  // Hour [0-11]\n  K: function (date, token, localize) {\n    const hours = date.getHours() % 12;\n\n    if (token === \"Ko\") {\n      return localize.ordinalNumber(hours, { unit: \"hour\" });\n    }\n\n    return addLeadingZeros(hours, token.length);\n  },\n\n  // Hour [1-24]\n  k: function (date, token, localize) {\n    let hours = date.getHours();\n    if (hours === 0) hours = 24;\n\n    if (token === \"ko\") {\n      return localize.ordinalNumber(hours, { unit: \"hour\" });\n    }\n\n    return addLeadingZeros(hours, token.length);\n  },\n\n  // Minute\n  m: function (date, token, localize) {\n    if (token === \"mo\") {\n      return localize.ordinalNumber(date.getMinutes(), { unit: \"minute\" });\n    }\n\n    return lightFormatters.m(date, token);\n  },\n\n  // Second\n  s: function (date, token, localize) {\n    if (token === \"so\") {\n      return localize.ordinalNumber(date.getSeconds(), { unit: \"second\" });\n    }\n\n    return lightFormatters.s(date, token);\n  },\n\n  // Fraction of second\n  S: function (date, token) {\n    return lightFormatters.S(date, token);\n  },\n\n  // Timezone (ISO-8601. If offset is 0, output is always `'Z'`)\n  X: function (date, token, _localize) {\n    const timezoneOffset = date.getTimezoneOffset();\n\n    if (timezoneOffset === 0) {\n      return \"Z\";\n    }\n\n    switch (token) {\n      // Hours and optional minutes\n      case \"X\":\n        return formatTimezoneWithOptionalMinutes(timezoneOffset);\n\n      // Hours, minutes and optional seconds without `:` delimiter\n      // Note: neither ISO-8601 nor JavaScript supports seconds in timezone offsets\n      // so this token always has the same output as `XX`\n      case \"XXXX\":\n      case \"XX\": // Hours and minutes without `:` delimiter\n        return formatTimezone(timezoneOffset);\n\n      // Hours, minutes and optional seconds with `:` delimiter\n      // Note: neither ISO-8601 nor JavaScript supports seconds in timezone offsets\n      // so this token always has the same output as `XXX`\n      case \"XXXXX\":\n      case \"XXX\": // Hours and minutes with `:` delimiter\n      default:\n        return formatTimezone(timezoneOffset, \":\");\n    }\n  },\n\n  // Timezone (ISO-8601. If offset is 0, output is `'+00:00'` or equivalent)\n  x: function (date, token, _localize) {\n    const timezoneOffset = date.getTimezoneOffset();\n\n    switch (token) {\n      // Hours and optional minutes\n      case \"x\":\n        return formatTimezoneWithOptionalMinutes(timezoneOffset);\n\n      // Hours, minutes and optional seconds without `:` delimiter\n      // Note: neither ISO-8601 nor JavaScript supports seconds in timezone offsets\n      // so this token always has the same output as `xx`\n      case \"xxxx\":\n      case \"xx\": // Hours and minutes without `:` delimiter\n        return formatTimezone(timezoneOffset);\n\n      // Hours, minutes and optional seconds with `:` delimiter\n      // Note: neither ISO-8601 nor JavaScript supports seconds in timezone offsets\n      // so this token always has the same output as `xxx`\n      case \"xxxxx\":\n      case \"xxx\": // Hours and minutes with `:` delimiter\n      default:\n        return formatTimezone(timezoneOffset, \":\");\n    }\n  },\n\n  // Timezone (GMT)\n  O: function (date, token, _localize) {\n    const timezoneOffset = date.getTimezoneOffset();\n\n    switch (token) {\n      // Short\n      case \"O\":\n      case \"OO\":\n      case \"OOO\":\n        return \"GMT\" + formatTimezoneShort(timezoneOffset, \":\");\n      // Long\n      case \"OOOO\":\n      default:\n        return \"GMT\" + formatTimezone(timezoneOffset, \":\");\n    }\n  },\n\n  // Timezone (specific non-location)\n  z: function (date, token, _localize) {\n    const timezoneOffset = date.getTimezoneOffset();\n\n    switch (token) {\n      // Short\n      case \"z\":\n      case \"zz\":\n      case \"zzz\":\n        return \"GMT\" + formatTimezoneShort(timezoneOffset, \":\");\n      // Long\n      case \"zzzz\":\n      default:\n        return \"GMT\" + formatTimezone(timezoneOffset, \":\");\n    }\n  },\n\n  // Seconds timestamp\n  t: function (date, token, _localize) {\n    const timestamp = Math.trunc(date.getTime() / 1000);\n    return addLeadingZeros(timestamp, token.length);\n  },\n\n  // Milliseconds timestamp\n  T: function (date, token, _localize) {\n    const timestamp = date.getTime();\n    return addLeadingZeros(timestamp, token.length);\n  },\n};\n\n\nfunction formatTimezoneShort(offset: number, delimiter: string = \"\"): string {\n  const sign = offset > 0 ? \"-\" : \"+\";\n  const absOffset = Math.abs(offset);\n  const hours = Math.trunc(absOffset / 60);\n  const minutes = absOffset % 60;\n  if (minutes === 0) {\n    return sign + String(hours);\n  }\n  return sign + String(hours) + delimiter + addLeadingZeros(minutes, 2);\n}\n\nfunction formatTimezoneWithOptionalMinutes(\n  offset: number,\n  delimiter?: string,\n): string {\n  if (offset % 60 === 0) {\n    const sign = offset > 0 ? \"-\" : \"+\";\n    return sign + addLeadingZeros(Math.abs(offset) / 60, 2);\n  }\n  return formatTimezone(offset, delimiter);\n}\n\nfunction formatTimezone(offset: number, delimiter: string = \"\"): string {\n  const sign = offset > 0 ? \"-\" : \"+\";\n  const absOffset = Math.abs(offset);\n  const hours = addLeadingZeros(Math.trunc(absOffset / 60), 2);\n  const minutes = addLeadingZeros(absOffset % 60, 2);\n  return sign + hours + delimiter + minutes;\n}\n\n// Path: src/format/index.ts\nimport { defaultLocale } from \"../_lib/defaultLocale/index.js\";\nimport { getDefaultOptions } from \"../_lib/defaultOptions/index.js\";\nimport { formatters } from \"../_lib/format/formatters/index.js\";\nimport { longFormatters } from \"../_lib/format/longFormatters/index.js\";\nimport {\n  isProtectedDayOfYearToken,\n  isProtectedWeekYearToken,\n  warnOrThrowProtectedError,\n} from \"../_lib/protectedTokens/index.js\";\nimport { isValid } from \"../isValid/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type {\n  AdditionalTokensOptions,\n  FirstWeekContainsDateOptions,\n  FormatPart,\n  LocalizedOptions,\n  WeekOptions,\n} from \"../types.js\";\n\n// Rexports of internal for libraries to use.\n// See: https://github.com/date-fns/date-fns/issues/3638#issuecomment-1877082874\nexport { formatters, longFormatters };\n\n// This RegExp consists of three parts separated by `|`:\n// - [yYQqMLwIdDecihHKkms]o matches any available ordinal number token\n//   (one of the certain letters followed by `o`)\n// - (\\w)\\1* matches any sequences of the same letter\n// - '' matches two quote characters in a row\n// - '(''|[^'])+('|$) matches anything surrounded by two quote characters ('),\n//   except a single quote symbol, which ends the sequence.\n//   Two quote characters do not end the sequence.\n//   If there is no matching single quote\n//   then the sequence will continue until the end of the string.\n// - . matches any single character unmatched by previous parts of the RegExps\nconst formattingTokensRegExp =\n  /[yYQqMLwIdDecihHKkms]o|(\\w)\\1*|''|'(''|[^'])+('|$)|./g;\n\n// This RegExp catches symbols escaped by quotes, and also\n// sequences of symbols P, p, and the combinations like `PPPPPPPppppp`\nconst longFormattingTokensRegExp = /P+p+|P+|p+|''|'(''|[^'])+('|$)|./g;\n\nconst escapedStringRegExp = /^'([^]*?)'?$/;\nconst doubleQuoteRegExp = /''/g;\nconst unescapedLatinCharacterRegExp = /[a-zA-Z]/;\n\nexport { format as formatDate };\nexport type { FormatOptions as FormatDateOptions };\n\n/**\n * The {@link format} function options.\n */\nexport interface FormatOptions\n  extends LocalizedOptions<\"options\" | \"localize\" | \"formatLong\">,\n    WeekOptions,\n    FirstWeekContainsDateOptions,\n    AdditionalTokensOptions {}\n\n/**\n * @name format\n * @alias formatDate\n * @category Common Helpers\n * @summary Format the date.\n *\n * @description\n * Return the formatted date string in the given format. The result may vary by locale.\n *\n * > \u26a0\ufe0f Please note that the `format` tokens differ from Moment.js and other libraries.\n * > See: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n *\n * The characters wrapped between two single quotes characters (') are escaped.\n * Two single quotes in a row, whether inside or outside a quoted sequence, represent a 'real' single quote.\n * (see the last example)\n *\n * Format of the string is based on Unicode Technical Standard #35:\n * https://www.unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table\n * with a few additions (see note 7 below the table).\n *\n * Accepted patterns:\n * | Unit                            | Pattern | Result examples                   | Notes |\n * |---------------------------------|---------|-----------------------------------|-------|\n * | Era                             | G..GGG  | AD, BC                            |       |\n * |                                 | GGGG    | Anno Domini, Before Christ        | 2     |\n * |                                 | GGGGG   | A, B                              |       |\n * | Calendar year                   | y       | 44, 1, 1900, 2017                 | 5     |\n * |                                 | yo      | 44th, 1st, 0th, 17th              | 5,7   |\n * |                                 | yy      | 44, 01, 00, 17                    | 5     |\n * |                                 | yyy     | 044, 001, 1900, 2017              | 5     |\n * |                                 | yyyy    | 0044, 0001, 1900, 2017            | 5     |\n * |                                 | yyyyy   | ...                               | 3,5   |\n * | Local week-numbering year       | Y       | 44, 1, 1900, 2017                 | 5     |\n * |                                 | Yo      | 44th, 1st, 1900th, 2017th         | 5,7   |\n * |                                 | YY      | 44, 01, 00, 17                    | 5,8   |\n * |                                 | YYY     | 044, 001, 1900, 2017              | 5     |\n * |                                 | YYYY    | 0044, 0001, 1900, 2017            | 5,8   |\n * |                                 | YYYYY   | ...                               | 3,5   |\n * | ISO week-numbering year         | R       | -43, 0, 1, 1900, 2017             | 5,7   |\n * |                                 | RR      | -43, 00, 01, 1900, 2017           | 5,7   |\n * |                                 | RRR     | -043, 000, 001, 1900, 2017        | 5,7   |\n * |                                 | RRRR    | -0043, 0000, 0001, 1900, 2017     | 5,7   |\n * |                                 | RRRRR   | ...                               | 3,5,7 |\n * | Extended year                   | u       | -43, 0, 1, 1900, 2017             | 5     |\n * |                                 | uu      | -43, 01, 1900, 2017               | 5     |\n * |                                 | uuu     | -043, 001, 1900, 2017             | 5     |\n * |                                 | uuuu    | -0043, 0001, 1900, 2017           | 5     |\n * |                                 | uuuuu   | ...                               | 3,5   |\n * | Quarter (formatting)            | Q       | 1, 2, 3, 4                        |       |\n * |                                 | Qo      | 1st, 2nd, 3rd, 4th                | 7     |\n * |                                 | QQ      | 01, 02, 03, 04                    |       |\n * |                                 | QQQ     | Q1, Q2, Q3, Q4                    |       |\n * |                                 | QQQQ    | 1st quarter, 2nd quarter, ...     | 2     |\n * |                                 | QQQQQ   | 1, 2, 3, 4                        | 4     |\n * | Quarter (stand-alone)           | q       | 1, 2, 3, 4                        |       |\n * |                                 | qo      | 1st, 2nd, 3rd, 4th                | 7     |\n * |                                 | qq      | 01, 02, 03, 04                    |       |\n * |                                 | qqq     | Q1, Q2, Q3, Q4                    |       |\n * |                                 | qqqq    | 1st quarter, 2nd quarter, ...     | 2     |\n * |                                 | qqqqq   | 1, 2, 3, 4                        | 4     |\n * | Month (formatting)              | M       | 1, 2, ..., 12                     |       |\n * |                                 | Mo      | 1st, 2nd, ..., 12th               | 7     |\n * |                                 | MM      | 01, 02, ..., 12                   |       |\n * |                                 | MMM     | Jan, Feb, ..., Dec                |       |\n * |                                 | MMMM    | January, February, ..., December  | 2     |\n * |                                 | MMMMM   | J, F, ..., D                      |       |\n * | Month (stand-alone)             | L       | 1, 2, ..., 12                     |       |\n * |                                 | Lo      | 1st, 2nd, ..., 12th               | 7     |\n * |                                 | LL      | 01, 02, ..., 12                   |       |\n * |                                 | LLL     | Jan, Feb, ..., Dec                |       |\n * |                                 | LLLL    | January, February, ..., December  | 2     |\n * |                                 | LLLLL   | J, F, ..., D                      |       |\n * | Local week of year              | w       | 1, 2, ..., 53                     |       |\n * |                                 | wo      | 1st, 2nd, ..., 53th               | 7     |\n * |                                 | ww      | 01, 02, ..., 53                   |       |\n * | ISO week of year                | I       | 1, 2, ..., 53                     | 7     |\n * |                                 | Io      | 1st, 2nd, ..., 53th               | 7     |\n * |                                 | II      | 01, 02, ..., 53                   | 7     |\n * | Day of month                    | d       | 1, 2, ..., 31                     |       |\n * |                                 | do      | 1st, 2nd, ..., 31st               | 7     |\n * |                                 | dd      | 01, 02, ..., 31                   |       |\n * | Day of year                     | D       | 1, 2, ..., 365, 366               | 9     |\n * |                                 | Do      | 1st, 2nd, ..., 365th, 366th       | 7     |\n * |                                 | DD      | 01, 02, ..., 365, 366             | 9     |\n * |                                 | DDD     | 001, 002, ..., 365, 366           |       |\n * |                                 | DDDD    | ...                               | 3     |\n * | Day of week (formatting)        | E..EEE  | Mon, Tue, Wed, ..., Sun           |       |\n * |                                 | EEEE    | Monday, Tuesday, ..., Sunday      | 2     |\n * |                                 | EEEEE   | M, T, W, T, F, S, S               |       |\n * |                                 | EEEEEE  | Mo, Tu, We, Th, Fr, Sa, Su        |       |\n * | ISO day of week (formatting)    | i       | 1, 2, 3, ..., 7                   | 7     |\n * |                                 | io      | 1st, 2nd, ..., 7th                | 7     |\n * |                                 | ii      | 01, 02, ..., 07                   | 7     |\n * |                                 | iii     | Mon, Tue, Wed, ..., Sun           | 7     |\n * |                                 | iiii    | Monday, Tuesday, ..., Sunday      | 2,7   |\n * |                                 | iiiii   | M, T, W, T, F, S, S               | 7     |\n * |                                 | iiiiii  | Mo, Tu, We, Th, Fr, Sa, Su        | 7     |\n * | Local day of week (formatting)  | e       | 2, 3, 4, ..., 1                   |       |\n * |                                 | eo      | 2nd, 3rd, ..., 1st                | 7     |\n * |                                 | ee      | 02, 03, ..., 01                   |       |\n * |                                 | eee     | Mon, Tue, Wed, ..., Sun           |       |\n * |                                 | eeee    | Monday, Tuesday, ..., Sunday      | 2     |\n * |                                 | eeeee   | M, T, W, T, F, S, S               |       |\n * |                                 | eeeeee  | Mo, Tu, We, Th, Fr, Sa, Su        |       |\n * | Local day of week (stand-alone) | c       | 2, 3, 4, ..., 1                   |       |\n * |                                 | co      | 2nd, 3rd, ..., 1st                | 7     |\n * |                                 | cc      | 02, 03, ..., 01                   |       |\n * |                                 | ccc     | Mon, Tue, Wed, ..., Sun           |       |\n * |                                 | cccc    | Monday, Tuesday, ..., Sunday      | 2     |\n * |                                 | ccccc   | M, T, W, T, F, S, S               |       |\n * |                                 | cccccc  | Mo, Tu, We, Th, Fr, Sa, Su        |       |\n * | AM, PM                          | a..aa   | AM, PM                            |       |\n * |                                 | aaa     | am, pm                            |       |\n * |                                 | aaaa    | a.m., p.m.                        | 2     |\n * |                                 | aaaaa   | a, p                              |       |\n * | AM, PM, noon, midnight          | b..bb   | AM, PM, noon, midnight            |       |\n * |                                 | bbb     | am, pm, noon, midnight            |       |\n * |                                 | bbbb    | a.m., p.m., noon, midnight        | 2     |\n * |                                 | bbbbb   | a, p, n, mi                       |       |\n * | Flexible day period             | B..BBB  | at night, in the morning, ...     |       |\n * |                                 | BBBB    | at night, in the morning, ...     | 2     |\n * |                                 | BBBBB   | at night, in the morning, ...     |       |\n * | Hour [1-12]                     | h       | 1, 2, ..., 11, 12                 |       |\n * |                                 | ho      | 1st, 2nd, ..., 11th, 12th         | 7     |\n * |                                 | hh      | 01, 02, ..., 11, 12               |       |\n * | Hour [0-23]                     | H       | 0, 1, 2, ..., 23                  |       |\n * |                                 | Ho      | 0th, 1st, 2nd, ..., 23rd          | 7     |\n * |                                 | HH      | 00, 01, 02, ..., 23               |       |\n * | Hour [0-11]                     | K       | 1, 2, ..., 11, 0                  |       |\n * |                                 | Ko      | 1st, 2nd, ..., 11th, 0th          | 7     |\n * |                                 | KK      | 01, 02, ..., 11, 00               |       |\n * | Hour [1-24]                     | k       | 24, 1, 2, ..., 23                 |       |\n * |                                 | ko      | 24th, 1st, 2nd, ..., 23rd         | 7     |\n * |                                 | kk      | 24, 01, 02, ..., 23               |       |\n * | Minute                          | m       | 0, 1, ..., 59                     |       |\n * |                                 | mo      | 0th, 1st, ..., 59th               | 7     |\n * |                                 | mm      | 00, 01, ..., 59                   |       |\n * | Second                          | s       | 0, 1, ..., 59                     |       |\n * |                                 | so      | 0th, 1st, ..., 59th               | 7     |\n * |                                 | ss      | 00, 01, ..., 59                   |       |\n * | Fraction of second              | S       | 0, 1, ..., 9                      |       |\n * |                                 | SS      | 00, 01, ..., 99                   |       |\n * |                                 | SSS     | 000, 001, ..., 999                |       |\n * |                                 | SSSS    | ...                               | 3     |\n * | Timezone (ISO-8601 w/ Z)        | X       | -08, +0530, Z                     |       |\n * |                                 | XX      | -0800, +0530, Z                   |       |\n * |                                 | XXX     | -08:00, +05:30, Z                 |       |\n * |                                 | XXXX    | -0800, +0530, Z, +123456          | 2     |\n * |                                 | XXXXX   | -08:00, +05:30, Z, +12:34:56      |       |\n * | Timezone (ISO-8601 w/o Z)       | x       | -08, +0530, +00                   |       |\n * |                                 | xx      | -0800, +0530, +0000               |       |\n * |                                 | xxx     | -08:00, +05:30, +00:00            | 2     |\n * |                                 | xxxx    | -0800, +0530, +0000, +123456      |       |\n * |                                 | xxxxx   | -08:00, +05:30, +00:00, +12:34:56 |       |\n * | Timezone (GMT)                  | O...OOO | GMT-8, GMT+5:30, GMT+0            |       |\n * |                                 | OOOO    | GMT-08:00, GMT+05:30, GMT+00:00   | 2     |\n * | Timezone (specific non-locat.)  | z...zzz | GMT-8, GMT+5:30, GMT+0            | 6     |\n * |                                 | zzzz    | GMT-08:00, GMT+05:30, GMT+00:00   | 2,6   |\n * | Seconds timestamp               | t       | 512969520                         | 7     |\n * |                                 | tt      | ...                               | 3,7   |\n * | Milliseconds timestamp          | T       | 512969520900                      | 7     |\n * |                                 | TT      | ...                               | 3,7   |\n * | Long localized date             | P       | 04/29/1453                        | 7     |\n * |                                 | PP      | Apr 29, 1453                      | 7     |\n * |                                 | PPP     | April 29th, 1453                  | 7     |\n * |                                 | PPPP    | Friday, April 29th, 1453          | 2,7   |\n * | Long localized time             | p       | 12:00 AM                          | 7     |\n * |                                 | pp      | 12:00:00 AM                       | 7     |\n * |                                 | ppp     | 12:00:00 AM GMT+2                 | 7     |\n * |                                 | pppp    | 12:00:00 AM GMT+02:00             | 2,7   |\n * | Combination of date and time    | Pp      | 04/29/1453, 12:00 AM              | 7     |\n * |                                 | PPpp    | Apr 29, 1453, 12:00:00 AM         | 7     |\n * |                                 | PPPppp  | April 29th, 1453 at ...           | 7     |\n * |                                 | PPPPpppp| Friday, April 29th, 1453 at ...   | 2,7   |\n * Notes:\n * 1. \"Formatting\" units (e.g. formatting quarter) in the default en-US locale\n *    are the same as \"stand-alone\" units, but are different in some languages.\n *    \"Formatting\" units are declined according to the rules of the language\n *    in the context of a date. \"Stand-alone\" units are always nominative singular:\n *\n *    `format(new Date(2017, 10, 6), 'do LLLL', {locale: cs}) //=> '6. listopad'`\n *\n *    `format(new Date(2017, 10, 6), 'do MMMM', {locale: cs}) //=> '6. listopadu'`\n *\n * 2. Any sequence of the identical letters is a pattern, unless it is escaped by\n *    the single quote characters (see below).\n *    If the sequence is longer than listed in table (e.g. `EEEEEEEEEEE`)\n *    the output will be the same as default pattern for this unit, usually\n *    the longest one (in case of ISO weekdays, `EEEE`). Default patterns for units\n *    are marked with \"2\" in the last column of the table.\n *\n *    `format(new Date(2017, 10, 6), 'MMM') //=> 'Nov'`\n *\n *    `format(new Date(2017, 10, 6), 'MMMM') //=> 'November'`\n *\n *    `format(new Date(2017, 10, 6), 'MMMMM') //=> 'N'`\n *\n *    `format(new Date(2017, 10, 6), 'MMMMMM') //=> 'November'`\n *\n *    `format(new Date(2017, 10, 6), 'MMMMMMM') //=> 'November'`\n *\n * 3. Some patterns could be unlimited length (such as `yyyyyyyy`).\n *    The output will be padded with zeros to match the length of the pattern.\n *\n *    `format(new Date(2017, 10, 6), 'yyyyyyyy') //=> '00002017'`\n *\n * 4. `QQQQQ` and `qqqqq` could be not strictly numerical in some locales.\n *    These tokens represent the shortest form of the quarter.\n *\n * 5. The main difference between `y` and `u` patterns are B.C. years:\n *\n *    | Year | `y` | `u` |\n *    |------|-----|-----|\n *    | AC 1 |   1 |   1 |\n *    | BC 1 |   1 |   0 |\n *    | BC 2 |   2 |  -1 |\n *\n *    Also `yy` always returns the last two digits of a year,\n *    while `uu` pads single digit years to 2 characters and returns other years unchanged:\n *\n *    | Year | `yy` | `uu` |\n *    |------|------|------|\n *    | 1    |   01 |   01 |\n *    | 14   |   14 |   14 |\n *    | 376  |   76 |  376 |\n *    | 1453 |   53 | 1453 |\n *\n *    The same difference is true for local and ISO week-numbering years (`Y` and `R`),\n *    except local week-numbering years are dependent on `options.weekStartsOn`\n *    and `options.firstWeekContainsDate` (compare [getISOWeekYear](https://date-fns.org/docs/getISOWeekYear)\n *    and [getWeekYear](https://date-fns.org/docs/getWeekYear)).\n *\n * 6. Specific non-location timezones are currently unavailable in `date-fns`,\n *    so right now these tokens fall back to GMT timezones.\n *\n * 7. These patterns are not in the Unicode Technical Standard #35:\n *    - `i`: ISO day of week\n *    - `I`: ISO week of year\n *    - `R`: ISO week-numbering year\n *    - `t`: seconds timestamp\n *    - `T`: milliseconds timestamp\n *    - `o`: ordinal number modifier\n *    - `P`: long localized date\n *    - `p`: long localized time\n *\n * 8. `YY` and `YYYY` tokens represent week-numbering years but they are often confused with years.\n *    You should enable `options.useAdditionalWeekYearTokens` to use them. See: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n *\n * 9. `D` and `DD` tokens represent days of the year but they are often confused with days of the month.\n *    You should enable `options.useAdditionalDayOfYearTokens` to use them. See: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n * @param format - The string of tokens\n * @param options - An object with options\n *\n * @returns The formatted date string\n *\n * @throws `date` must not be Invalid Date\n * @throws `options.locale` must contain `localize` property\n * @throws `options.locale` must contain `formatLong` property\n * @throws use `yyyy` instead of `YYYY` for formatting years using [format provided] to the input [input provided]; see: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n * @throws use `yy` instead of `YY` for formatting years using [format provided] to the input [input provided]; see: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n * @throws use `d` instead of `D` for formatting days of the month using [format provided] to the input [input provided]; see: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n * @throws use `dd` instead of `DD` for formatting days of the month using [format provided] to the input [input provided]; see: https://github.com/date-fns/date-fns/blob/master/docs/unicodeTokens.md\n * @throws format string contains an unescaped latin alphabet character\n *\n * @example\n * // Represent 11 February 2014 in middle-endian format:\n * const result = format(new Date(2014, 1, 11), 'MM/dd/yyyy')\n * //=> '02/11/2014'\n *\n * @example\n * // Represent 2 July 2014 in Esperanto:\n * import { eoLocale } from 'date-fns/locale/eo'\n * const result = format(new Date(2014, 6, 2), \"do 'de' MMMM yyyy\", {\n *   locale: eoLocale\n * })\n * //=> '2-a de julio 2014'\n *\n * @example\n * // Escape string by single quote characters:\n * const result = format(new Date(2014, 6, 2, 15), \"h 'o''clock'\")\n * //=> \"3 o'clock\"\n */\nexport function format<DateType extends Date>(\n  date: DateType | number | string,\n  formatStr: string,\n  options?: FormatOptions,\n): string {\n  const defaultOptions = getDefaultOptions();\n  const locale = options?.locale ?? defaultOptions.locale ?? defaultLocale;\n\n  const firstWeekContainsDate =\n    options?.firstWeekContainsDate ??\n    options?.locale?.options?.firstWeekContainsDate ??\n    defaultOptions.firstWeekContainsDate ??\n    defaultOptions.locale?.options?.firstWeekContainsDate ??\n    1;\n\n  const weekStartsOn =\n    options?.weekStartsOn ??\n    options?.locale?.options?.weekStartsOn ??\n    defaultOptions.weekStartsOn ??\n    defaultOptions.locale?.options?.weekStartsOn ??\n    0;\n\n  const originalDate = toDate(date);\n\n  if (!isValid(originalDate)) {\n    throw new RangeError(\"Invalid time value\");\n  }\n\n  let parts: FormatPart[] = formatStr\n    .match(longFormattingTokensRegExp)!\n    .map((substring) => {\n      const firstCharacter = substring[0];\n      if (firstCharacter === \"p\" || firstCharacter === \"P\") {\n        const longFormatter = longFormatters[firstCharacter];\n        return longFormatter(substring, locale.formatLong);\n      }\n      return substring;\n    })\n    .join(\"\")\n    .match(formattingTokensRegExp)!\n    .map((substring) => {\n      // Replace two single quote characters with one single quote character\n      if (substring === \"''\") {\n        return { isToken: false, value: \"'\" };\n      }\n\n      const firstCharacter = substring[0];\n      if (firstCharacter === \"'\") {\n        return { isToken: false, value: cleanEscapedString(substring) };\n      }\n\n      if (formatters[firstCharacter]) {\n        return { isToken: true, value: substring };\n      }\n\n      if (firstCharacter.match(unescapedLatinCharacterRegExp)) {\n        throw new RangeError(\n          \"Format string contains an unescaped latin alphabet character `\" +\n            firstCharacter +\n            \"`\",\n        );\n      }\n\n      return { isToken: false, value: substring };\n    });\n\n  // invoke localize preprocessor (only for french locales at the moment)\n  if (locale.localize.preprocessor) {\n    parts = locale.localize.preprocessor(originalDate, parts);\n  }\n\n  const formatterOptions = {\n    firstWeekContainsDate,\n    weekStartsOn,\n    locale,\n  };\n\n  return parts\n    .map((part) => {\n      if (!part.isToken) return part.value;\n\n      const token = part.value;\n\n      if (\n        (!options?.useAdditionalWeekYearTokens &&\n          isProtectedWeekYearToken(token)) ||\n        (!options?.useAdditionalDayOfYearTokens &&\n          isProtectedDayOfYearToken(token))\n      ) {\n        warnOrThrowProtectedError(token, formatStr, String(date));\n      }\n\n      const formatter = formatters[token[0]];\n      return formatter(originalDate, token, locale.localize, formatterOptions);\n    })\n    .join(\"\");\n}\n\nfunction cleanEscapedString(input: string): string {\n  const matched = input.match(escapedStringRegExp);\n\n  if (!matched) {\n    return input;\n  }\n\n  return matched[1].replace(doubleQuoteRegExp, \"'\");\n}\n\n// Path: src/formatRelative/index.ts\nimport { differenceInCalendarDays } from \"../differenceInCalendarDays/index.js\";\nimport { format } from \"../format/index.js\";\nimport type { FormatRelativeToken } from \"../locale/types.js\";\nimport { toDate } from \"../toDate/index.js\";\n...\n", "needle_token_start": 7190, "needle_token_end": 7316, "code_context_ntokens": 15937, "output": ["```typescript\nfunction formatTimezoneShort(offset: number, delimiter: string = \"\"): string {\n  const sign = offset > 0 ? \"-\" : \"+\";\n  const absOffset = Math.abs(offset);\n  const hours = Math.trunc(absOffset / 60);\n  const minutes = absOffset % 60;\n  if (minutes === 0) {\n    return sign + String(hours);\n  }\n  return sign + String(hours) + delimiter + addLeadingZeros(minutes, 2);\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "fn", "language": "typescript", "path": "src/fp/_lib/convertToFP/test.ts", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function constructs a string by replacing placeholders with the string representations of three provided arguments.\n2. **Input**: Three arguments of any type.\n3. **Output**: A string that concatenates the string representations of the three inputs, separated by spaces.\n4. **Procedure**: The function takes three inputs, converts each to its string representation, and then constructs a single string where each input value is separated by a space.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/fp/formatRFC3339WithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatRFC3339 as fn } from \"../../formatRFC3339/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\n...\n// Path: src/fp/isTuesday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isTuesday as fn } from \"../../isTuesday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isTuesday = convertToFP(fn, 1);\n\n// Path: src/fp/getMinutes/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getMinutes as fn } from \"../../getMinutes/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getMinutes = convertToFP(fn, 1);\n\n// Path: src/fp/setISOWeek/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setISOWeek as fn } from \"../../setISOWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setISOWeek = convertToFP(fn, 2);\n\n// Path: src/fp/yearsToQuarters/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { yearsToQuarters as fn } from \"../../yearsToQuarters/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const yearsToQuarters = convertToFP(fn, 1);\n\n// Path: src/fp/nextThursday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { nextThursday as fn } from \"../../nextThursday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const nextThursday = convertToFP(fn, 1);\n\n// Path: src/fp/subQuarters/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { subQuarters as fn } from \"../../subQuarters/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const subQuarters = convertToFP(fn, 2);\n\n// Path: src/fp/getISODay/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getISODay as fn } from \"../../getISODay/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getISODay = convertToFP(fn, 1);\n\n// Path: src/fp/endOfMinute/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { endOfMinute as fn } from \"../../endOfMinute/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const endOfMinute = convertToFP(fn, 1);\n\n// Path: src/fp/lastDayOfMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { lastDayOfMonth as fn } from \"../../lastDayOfMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const lastDayOfMonth = convertToFP(fn, 1);\n\n// Path: src/fp/startOfMinute/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfMinute as fn } from \"../../startOfMinute/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfMinute = convertToFP(fn, 1);\n\n// Path: src/fp/startOfDecade/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfDecade as fn } from \"../../startOfDecade/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfDecade = convertToFP(fn, 1);\n\n// Path: src/fp/secondsToMilliseconds/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { secondsToMilliseconds as fn } from \"../../secondsToMilliseconds/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const secondsToMilliseconds = convertToFP(fn, 1);\n\n// Path: src/fp/startOfHour/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { startOfHour as fn } from \"../../startOfHour/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const startOfHour = convertToFP(fn, 1);\n\n// Path: src/fp/isWithinInterval/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isWithinInterval as fn } from \"../../isWithinInterval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isWithinInterval = convertToFP(fn, 2);\n\n// Path: src/fp/endOfWeek/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { endOfWeek as fn } from \"../../endOfWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const endOfWeek = convertToFP(fn, 1);\n\n// Path: src/fp/secondsToMinutes/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { secondsToMinutes as fn } from \"../../secondsToMinutes/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const secondsToMinutes = convertToFP(fn, 1);\n\n// Path: src/fp/isWednesday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isWednesday as fn } from \"../../isWednesday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isWednesday = convertToFP(fn, 1);\n\n// Path: src/fp/differenceInCalendarISOWeeks/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInCalendarISOWeeks as fn } from \"../../differenceInCalendarISOWeeks/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInCalendarISOWeeks = convertToFP(fn, 2);\n\n// Path: src/fp/set/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { set as fn } from \"../../set/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const set = convertToFP(fn, 2);\n\n// Path: src/fp/lastDayOfWeek/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { lastDayOfWeek as fn } from \"../../lastDayOfWeek/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const lastDayOfWeek = convertToFP(fn, 1);\n\n// Path: src/fp/getWeekOfMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getWeekOfMonth as fn } from \"../../getWeekOfMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getWeekOfMonth = convertToFP(fn, 1);\n\n// Path: src/fp/eachWeekendOfMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { eachWeekendOfMonth as fn } from \"../../eachWeekendOfMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const eachWeekendOfMonth = convertToFP(fn, 1);\n\n// Path: src/fp/intlFormat/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { intlFormat as fn } from \"../../intlFormat/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const intlFormat = convertToFP(fn, 3);\n\n// Path: src/fp/isMatchWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isMatch as fn } from \"../../isMatch/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isMatchWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/isMonday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isMonday as fn } from \"../../isMonday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isMonday = convertToFP(fn, 1);\n\n// Path: src/fp/getDayOfYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getDayOfYear as fn } from \"../../getDayOfYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getDayOfYear = convertToFP(fn, 1);\n\n// Path: src/fp/previousMonday/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { previousMonday as fn } from \"../../previousMonday/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const previousMonday = convertToFP(fn, 1);\n\n// Path: src/fp/isSameMonth/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameMonth as fn } from \"../../isSameMonth/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameMonth = convertToFP(fn, 2);\n\n// Path: src/fp/sub/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { sub as fn } from \"../../sub/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const sub = convertToFP(fn, 2);\n\n// Path: src/fp/differenceInCalendarQuarters/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { differenceInCalendarQuarters as fn } from \"../../differenceInCalendarQuarters/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const differenceInCalendarQuarters = convertToFP(fn, 2);\n\n// Path: src/fp/setWeekYearWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setWeekYear as fn } from \"../../setWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setWeekYearWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/isSameHour/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isSameHour as fn } from \"../../isSameHour/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isSameHour = convertToFP(fn, 2);\n\n// Path: src/fp/isValid/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isValid as fn } from \"../../isValid/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isValid = convertToFP(fn, 1);\n\n// Path: src/fp/millisecondsToSeconds/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { millisecondsToSeconds as fn } from \"../../millisecondsToSeconds/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const millisecondsToSeconds = convertToFP(fn, 1);\n\n// Path: src/fp/weeksToDays/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { weeksToDays as fn } from \"../../weeksToDays/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const weeksToDays = convertToFP(fn, 1);\n\n// Path: src/fp/setWeekYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { setWeekYear as fn } from \"../../setWeekYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const setWeekYear = convertToFP(fn, 2);\n\n// Path: src/fp/getMilliseconds/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { getMilliseconds as fn } from \"../../getMilliseconds/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const getMilliseconds = convertToFP(fn, 1);\n\n// Path: src/fp/subMinutes/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { subMinutes as fn } from \"../../subMinutes/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const subMinutes = convertToFP(fn, 2);\n\n// Path: src/fp/formatRFC7231/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatRFC7231 as fn } from \"../../formatRFC7231/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatRFC7231 = convertToFP(fn, 1);\n\n// Path: src/fp/interval/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { interval as fn } from \"../../interval/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const interval = convertToFP(fn, 2);\n\n// Path: src/fp/formatRelativeWithOptions/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { formatRelative as fn } from \"../../formatRelative/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const formatRelativeWithOptions = convertToFP(fn, 3);\n\n// Path: src/fp/subMonths/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { subMonths as fn } from \"../../subMonths/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const subMonths = convertToFP(fn, 2);\n\n// Path: src/fp/addISOWeekYears/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { addISOWeekYears as fn } from \"../../addISOWeekYears/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const addISOWeekYears = convertToFP(fn, 2);\n\n// Path: src/fp/isLeapYear/index.ts\n// This file is generated automatically by `scripts/build/fp.ts`. Please, don't change it.\n\nimport { isLeapYear as fn } from \"../../isLeapYear/index.js\";\nimport { convertToFP } from \"../_lib/convertToFP/index.js\";\n\nexport const isLeapYear = convertToFP(fn, 1);\n\n// Path: src/fp/index.ts\n// This file is generated automatically by `scripts/build/indices.ts`. Please, don't change it.\n\nexport * from \"./add/index.js\";\nexport * from \"./addBusinessDays/index.js\";\nexport * from \"./addDays/index.js\";\nexport * from \"./addHours/index.js\";\nexport * from \"./addISOWeekYears/index.js\";\nexport * from \"./addMilliseconds/index.js\";\nexport * from \"./addMinutes/index.js\";\nexport * from \"./addMonths/index.js\";\nexport * from \"./addQuarters/index.js\";\nexport * from \"./addSeconds/index.js\";\nexport * from \"./addWeeks/index.js\";\nexport * from \"./addYears/index.js\";\nexport * from \"./areIntervalsOverlapping/index.js\";\nexport * from \"./areIntervalsOverlappingWithOptions/index.js\";\nexport * from \"./clamp/index.js\";\nexport * from \"./closestIndexTo/index.js\";\nexport * from \"./closestTo/index.js\";\nexport * from \"./compareAsc/index.js\";\nexport * from \"./compareDesc/index.js\";\nexport * from \"./constructFrom/index.js\";\nexport * from \"./daysToWeeks/index.js\";\nexport * from \"./differenceInBusinessDays/index.js\";\nexport * from \"./differenceInCalendarDays/index.js\";\nexport * from \"./differenceInCalendarISOWeekYears/index.js\";\nexport * from \"./differenceInCalendarISOWeeks/index.js\";\nexport * from \"./differenceInCalendarMonths/index.js\";\nexport * from \"./differenceInCalendarQuarters/index.js\";\nexport * from \"./differenceInCalendarWeeks/index.js\";\nexport * from \"./differenceInCalendarWeeksWithOptions/index.js\";\nexport * from \"./differenceInCalendarYears/index.js\";\nexport * from \"./differenceInDays/index.js\";\nexport * from \"./differenceInHours/index.js\";\nexport * from \"./differenceInHoursWithOptions/index.js\";\nexport * from \"./differenceInISOWeekYears/index.js\";\nexport * from \"./differenceInMilliseconds/index.js\";\nexport * from \"./differenceInMinutes/index.js\";\nexport * from \"./differenceInMinutesWithOptions/index.js\";\nexport * from \"./differenceInMonths/index.js\";\nexport * from \"./differenceInQuarters/index.js\";\nexport * from \"./differenceInQuartersWithOptions/index.js\";\nexport * from \"./differenceInSeconds/index.js\";\nexport * from \"./differenceInSecondsWithOptions/index.js\";\nexport * from \"./differenceInWeeks/index.js\";\nexport * from \"./differenceInWeeksWithOptions/index.js\";\nexport * from \"./differenceInYears/index.js\";\nexport * from \"./eachDayOfInterval/index.js\";\nexport * from \"./eachDayOfIntervalWithOptions/index.js\";\nexport * from \"./eachHourOfInterval/index.js\";\nexport * from \"./eachHourOfIntervalWithOptions/index.js\";\nexport * from \"./eachMinuteOfInterval/index.js\";\nexport * from \"./eachMinuteOfIntervalWithOptions/index.js\";\nexport * from \"./eachMonthOfInterval/index.js\";\nexport * from \"./eachMonthOfIntervalWithOptions/index.js\";\nexport * from \"./eachQuarterOfInterval/index.js\";\nexport * from \"./eachQuarterOfIntervalWithOptions/index.js\";\nexport * from \"./eachWeekOfInterval/index.js\";\nexport * from \"./eachWeekOfIntervalWithOptions/index.js\";\nexport * from \"./eachWeekendOfInterval/index.js\";\nexport * from \"./eachWeekendOfMonth/index.js\";\nexport * from \"./eachWeekendOfYear/index.js\";\nexport * from \"./eachYearOfInterval/index.js\";\nexport * from \"./eachYearOfIntervalWithOptions/index.js\";\nexport * from \"./endOfDay/index.js\";\nexport * from \"./endOfDecade/index.js\";\nexport * from \"./endOfHour/index.js\";\nexport * from \"./endOfISOWeek/index.js\";\nexport * from \"./endOfISOWeekYear/index.js\";\nexport * from \"./endOfMinute/index.js\";\nexport * from \"./endOfMonth/index.js\";\nexport * from \"./endOfQuarter/index.js\";\nexport * from \"./endOfSecond/index.js\";\nexport * from \"./endOfWeek/index.js\";\nexport * from \"./endOfWeekWithOptions/index.js\";\nexport * from \"./endOfYear/index.js\";\nexport * from \"./format/index.js\";\nexport * from \"./formatDistance/index.js\";\nexport * from \"./formatDistanceStrict/index.js\";\nexport * from \"./formatDistanceStrictWithOptions/index.js\";\nexport * from \"./formatDistanceWithOptions/index.js\";\nexport * from \"./formatDuration/index.js\";\nexport * from \"./formatDurationWithOptions/index.js\";\nexport * from \"./formatISO/index.js\";\nexport * from \"./formatISO9075/index.js\";\nexport * from \"./formatISO9075WithOptions/index.js\";\nexport * from \"./formatISODuration/index.js\";\nexport * from \"./formatISOWithOptions/index.js\";\nexport * from \"./formatRFC3339/index.js\";\nexport * from \"./formatRFC3339WithOptions/index.js\";\nexport * from \"./formatRFC7231/index.js\";\nexport * from \"./formatRelative/index.js\";\nexport * from \"./formatRelativeWithOptions/index.js\";\nexport * from \"./formatWithOptions/index.js\";\nexport * from \"./fromUnixTime/index.js\";\nexport * from \"./getDate/index.js\";\nexport * from \"./getDay/index.js\";\nexport * from \"./getDayOfYear/index.js\";\nexport * from \"./getDaysInMonth/index.js\";\nexport * from \"./getDaysInYear/index.js\";\nexport * from \"./getDecade/index.js\";\nexport * from \"./getHours/index.js\";\nexport * from \"./getISODay/index.js\";\nexport * from \"./getISOWeek/index.js\";\nexport * from \"./getISOWeekYear/index.js\";\nexport * from \"./getISOWeeksInYear/index.js\";\nexport * from \"./getMilliseconds/index.js\";\nexport * from \"./getMinutes/index.js\";\nexport * from \"./getMonth/index.js\";\nexport * from \"./getOverlappingDaysInIntervals/index.js\";\nexport * from \"./getQuarter/index.js\";\nexport * from \"./getSeconds/index.js\";\nexport * from \"./getTime/index.js\";\nexport * from \"./getUnixTime/index.js\";\nexport * from \"./getWeek/index.js\";\nexport * from \"./getWeekOfMonth/index.js\";\nexport * from \"./getWeekOfMonthWithOptions/index.js\";\nexport * from \"./getWeekWithOptions/index.js\";\nexport * from \"./getWeekYear/index.js\";\nexport * from \"./getWeekYearWithOptions/index.js\";\nexport * from \"./getWeeksInMonth/index.js\";\nexport * from \"./getWeeksInMonthWithOptions/index.js\";\nexport * from \"./getYear/index.js\";\nexport * from \"./hoursToMilliseconds/index.js\";\nexport * from \"./hoursToMinutes/index.js\";\nexport * from \"./hoursToSeconds/index.js\";\nexport * from \"./interval/index.js\";\nexport * from \"./intervalToDuration/index.js\";\nexport * from \"./intervalWithOptions/index.js\";\nexport * from \"./intlFormat/index.js\";\nexport * from \"./intlFormatDistance/index.js\";\nexport * from \"./intlFormatDistanceWithOptions/index.js\";\nexport * from \"./isAfter/index.js\";\nexport * from \"./isBefore/index.js\";\nexport * from \"./isDate/index.js\";\nexport * from \"./isEqual/index.js\";\nexport * from \"./isExists/index.js\";\nexport * from \"./isFirstDayOfMonth/index.js\";\nexport * from \"./isFriday/index.js\";\nexport * from \"./isLastDayOfMonth/index.js\";\nexport * from \"./isLeapYear/index.js\";\nexport * from \"./isMatch/index.js\";\nexport * from \"./isMatchWithOptions/index.js\";\nexport * from \"./isMonday/index.js\";\nexport * from \"./isSameDay/index.js\";\nexport * from \"./isSameHour/index.js\";\nexport * from \"./isSameISOWeek/index.js\";\nexport * from \"./isSameISOWeekYear/index.js\";\nexport * from \"./isSameMinute/index.js\";\nexport * from \"./isSameMonth/index.js\";\nexport * from \"./isSameQuarter/index.js\";\nexport * from \"./isSameSecond/index.js\";\nexport * from \"./isSameWeek/index.js\";\nexport * from \"./isSameWeekWithOptions/index.js\";\nexport * from \"./isSameYear/index.js\";\nexport * from \"./isSaturday/index.js\";\nexport * from \"./isSunday/index.js\";\nexport * from \"./isThursday/index.js\";\nexport * from \"./isTuesday/index.js\";\nexport * from \"./isValid/index.js\";\nexport * from \"./isWednesday/index.js\";\nexport * from \"./isWeekend/index.js\";\nexport * from \"./isWithinInterval/index.js\";\nexport * from \"./lastDayOfDecade/index.js\";\nexport * from \"./lastDayOfISOWeek/index.js\";\nexport * from \"./lastDayOfISOWeekYear/index.js\";\nexport * from \"./lastDayOfMonth/index.js\";\nexport * from \"./lastDayOfQuarter/index.js\";\nexport * from \"./lastDayOfWeek/index.js\";\nexport * from \"./lastDayOfWeekWithOptions/index.js\";\nexport * from \"./lastDayOfYear/index.js\";\nexport * from \"./lightFormat/index.js\";\nexport * from \"./max/index.js\";\nexport * from \"./milliseconds/index.js\";\nexport * from \"./millisecondsToHours/index.js\";\nexport * from \"./millisecondsToMinutes/index.js\";\nexport * from \"./millisecondsToSeconds/index.js\";\nexport * from \"./min/index.js\";\nexport * from \"./minutesToHours/index.js\";\nexport * from \"./minutesToMilliseconds/index.js\";\nexport * from \"./minutesToSeconds/index.js\";\nexport * from \"./monthsToQuarters/index.js\";\nexport * from \"./monthsToYears/index.js\";\nexport * from \"./nextDay/index.js\";\nexport * from \"./nextFriday/index.js\";\nexport * from \"./nextMonday/index.js\";\nexport * from \"./nextSaturday/index.js\";\nexport * from \"./nextSunday/index.js\";\nexport * from \"./nextThursday/index.js\";\nexport * from \"./nextTuesday/index.js\";\nexport * from \"./nextWednesday/index.js\";\nexport * from \"./parse/index.js\";\nexport * from \"./parseISO/index.js\";\nexport * from \"./parseISOWithOptions/index.js\";\nexport * from \"./parseJSON/index.js\";\nexport * from \"./parseWithOptions/index.js\";\nexport * from \"./previousDay/index.js\";\nexport * from \"./previousFriday/index.js\";\nexport * from \"./previousMonday/index.js\";\nexport * from \"./previousSaturday/index.js\";\nexport * from \"./previousSunday/index.js\";\nexport * from \"./previousThursday/index.js\";\nexport * from \"./previousTuesday/index.js\";\nexport * from \"./previousWednesday/index.js\";\nexport * from \"./quartersToMonths/index.js\";\nexport * from \"./quartersToYears/index.js\";\nexport * from \"./roundToNearestHours/index.js\";\nexport * from \"./roundToNearestHoursWithOptions/index.js\";\nexport * from \"./roundToNearestMinutes/index.js\";\nexport * from \"./roundToNearestMinutesWithOptions/index.js\";\nexport * from \"./secondsToHours/index.js\";\nexport * from \"./secondsToMilliseconds/index.js\";\nexport * from \"./secondsToMinutes/index.js\";\nexport * from \"./set/index.js\";\nexport * from \"./setDate/index.js\";\nexport * from \"./setDay/index.js\";\nexport * from \"./setDayOfYear/index.js\";\nexport * from \"./setDayWithOptions/index.js\";\nexport * from \"./setHours/index.js\";\nexport * from \"./setISODay/index.js\";\nexport * from \"./setISOWeek/index.js\";\nexport * from \"./setISOWeekYear/index.js\";\nexport * from \"./setMilliseconds/index.js\";\nexport * from \"./setMinutes/index.js\";\nexport * from \"./setMonth/index.js\";\nexport * from \"./setQuarter/index.js\";\nexport * from \"./setSeconds/index.js\";\nexport * from \"./setWeek/index.js\";\nexport * from \"./setWeekWithOptions/index.js\";\nexport * from \"./setWeekYear/index.js\";\nexport * from \"./setWeekYearWithOptions/index.js\";\nexport * from \"./setYear/index.js\";\nexport * from \"./startOfDay/index.js\";\nexport * from \"./startOfDecade/index.js\";\nexport * from \"./startOfHour/index.js\";\nexport * from \"./startOfISOWeek/index.js\";\nexport * from \"./startOfISOWeekYear/index.js\";\nexport * from \"./startOfMinute/index.js\";\nexport * from \"./startOfMonth/index.js\";\nexport * from \"./startOfQuarter/index.js\";\nexport * from \"./startOfSecond/index.js\";\nexport * from \"./startOfWeek/index.js\";\nexport * from \"./startOfWeekWithOptions/index.js\";\nexport * from \"./startOfWeekYear/index.js\";\nexport * from \"./startOfWeekYearWithOptions/index.js\";\nexport * from \"./startOfYear/index.js\";\nexport * from \"./sub/index.js\";\nexport * from \"./subBusinessDays/index.js\";\nexport * from \"./subDays/index.js\";\nexport * from \"./subHours/index.js\";\nexport * from \"./subISOWeekYears/index.js\";\nexport * from \"./subMilliseconds/index.js\";\nexport * from \"./subMinutes/index.js\";\nexport * from \"./subMonths/index.js\";\nexport * from \"./subQuarters/index.js\";\nexport * from \"./subSeconds/index.js\";\nexport * from \"./subWeeks/index.js\";\nexport * from \"./subYears/index.js\";\nexport * from \"./toDate/index.js\";\nexport * from \"./transpose/index.js\";\nexport * from \"./weeksToDays/index.js\";\nexport * from \"./yearsToDays/index.js\";\nexport * from \"./yearsToMonths/index.js\";\nexport * from \"./yearsToQuarters/index.js\";\nexport type * from \"../types.js\";\n\n// Path: src/fp/_lib/convertToFP/test.ts\nimport { pipe } from \"fp-ts/function\";\nimport { flow as jsFnsFlow } from \"js-fns\";\n// eslint-disable-next-line @typescript-eslint/ban-ts-comment -- Lodash types trip in different environemnts, so we can't put ts-expect-error\n// @ts-ignore - Lodash types are tripping (\"Module '\"lodash\"' has no exported member 'flow'.ts(2305)\")\nimport { flow as lodashFlow } from \"lodash\";\nimport { describe, expect, it } from \"vitest\";\nimport { addDays, addHours, isEqual } from \"../../index.js\";\nimport { convertToFP } from \"./index.js\";\n\ndescribe(\"convertToFP\", () => {\n  \nfunction fn(a: unknown, b: unknown, c: unknown) {\n    return \"a b c\"\n      .replace(\"a\", String(a))\n      .replace(\"b\", String(b))\n      .replace(\"c\", String(c));\n  }\n\n  describe(\"arity of converted function === arity of initial function\", () => {\n    it(\"allows arguments to be curried (and reverses their order)\", () => {\n      const fpFn = convertToFP(fn, 3);\n      expect(fpFn(3)(2)(1)).toBe(\"1 2 3\");\n    });\n\n    it(\"allows to group arguments\", () => {\n      const fpFn = convertToFP(fn, 3);\n      expect(fpFn(3, 2)(1)).toBe(\"1 2 3\");\n      expect(fpFn(3)(2, 1)).toBe(\"1 2 3\");\n    });\n\n    it(\"allows the function to be called with all arguments in the reversed order\", () => {\n      const fpFn = convertToFP(fn, 3);\n      expect(fpFn(3, 2, 1)).toBe(\"1 2 3\");\n    });\n\n    it(\"ignores calls without curried arguments\", () => {\n      const fpFn = convertToFP(fn, 3);\n      expect(fpFn()()(3, 2)()()(1)).toBe(\"1 2 3\");\n    });\n\n    it(\"ignores extra curried arguments in the last group\", () => {\n      const fpFn = convertToFP(fn, 3);\n      // @ts-expect-error - It's ok, we're testing the function\n      expect(fpFn(3, 2, 1, 0, -1, -2)).toBe(\"1 2 3\");\n      // @ts-expect-error - It's ok, we're testing the function\n      expect(fpFn(3)(2)(1, 0, -1, -2)).toBe(\"1 2 3\");\n    });\n  });\n\n  describe(\"arity of converted function < arity of initial function\", () => {\n    it(\"calls the initial function with a short list of arguments\", () => {\n      const fpFn = convertToFP(fn, 2);\n      expect(fpFn(2)(1)).toBe(\"1 2 undefined\");\n      expect(fpFn(2, 1)).toBe(\"1 2 undefined\");\n    });\n\n    it(\"ignores extra curried arguments in the last group\", () => {\n      const fpFn = convertToFP(fn, 2);\n      // @ts-expect-error - It's ok, we're testing the function\n      expect(fpFn(3)(2, 1)).toBe(\"2 3 undefined\");\n      // @ts-expect-error - It's ok, we're testing the function\n      expect(fpFn(3, 2, 1)).toBe(\"2 3 undefined\");\n    });\n  });\n\n  describe(\"arity of converted function > arity of initial function\", () => {\n    it(\"works, but ignores the extra arguments\", () => {\n      const fpFn = convertToFP(fn, 4);\n      // @ts-expect-error - It's ok, we're testing the function\n      expect(fpFn(4)(3)(2)(1)).toBe(\"1 2 3\");\n      // @ts-expect-error - It's ok, we're testing the function\n      expect(fpFn(4, 3, 2, 1)).toBe(\"1 2 3\");\n    });\n  });\n\n  describe(\"arity of converted function === 0\", () => {\n    it(\"returns the constant instead of function\", () => {\n      // @ts-expect-error - It's ok, we're testing the function\n      const result = convertToFP(fn, 0);\n      expect(result).toBe(\"undefined undefined undefined\");\n    });\n  });\n\n  describe(\"types\", () => {\n    it(\"resolves proper types\", () => {\n      const fn1 = addDays();\n      const fn2 = fn1(1);\n      const result = fn2(new Date(1987, 1));\n      expect(result.getFullYear()).toBe(1987);\n    });\n  });\n\n  describe(\"Lodash\", () => {\n    it(\"works with flow\", () => {\n      const fn = lodashFlow(addDays(1), addHours(1));\n      const result = fn(new Date(1987, 1, 11));\n      expect(result.getFullYear()).toBe(1987);\n      expect(result).toEqual(new Date(1987, 1, 12, 1));\n    });\n  });\n\n  describe(\"fp-ts\", () => {\n    it(\"works with pipe\", () => {\n      const result = pipe(\n        new Date(1987, 1, 11),\n        isEqual(new Date(1987, 1, 11)),\n      );\n      const _assign: boolean = result;\n      expect(result).toBe(true);\n    });\n  });\n\n  describe(\"js-fns\", () => {\n    it(\"works with flow\", () => {\n      const fn = jsFnsFlow(addDays(1), addHours(1));\n      const result = fn(new Date(1987, 1, 11));\n      expect(result.getFullYear()).toBe(1987);\n      expect(result).toEqual(new Date(1987, 1, 12, 1));\n    });\n  });\n});\n\n// Path: src/minutesToSeconds/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { minutesToSeconds } from \"./index.js\";\n\ndescribe(\"minutesToSeconds\", () => {\n  it(\"converts minutes to seconds\", () => {\n    expect(minutesToSeconds(1)).toBe(60);\n    expect(minutesToSeconds(2)).toBe(120);\n  });\n\n  it(\"uses floor rounding\", () => {\n    expect(minutesToSeconds(0.123456)).toBe(7);\n  });\n\n  it(\"handles border values\", () => {\n    expect(minutesToSeconds(1.5)).toBe(90);\n    expect(minutesToSeconds(0)).toBe(0);\n  });\n\n  it(\"properly works with negative numbers\", () => {\n    expect(minutesToSeconds(1.23)).toBe(73);\n    expect(minutesToSeconds(-1.23)).toBe(-73);\n  });\n});\n\n// Path: src/isFriday/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { isFriday } from \"./index.js\";\n\ndescribe(\"isFriday\", () => {\n  it(\"returns true if the given date is Friday\", () => {\n    const result = isFriday(new Date(2014, 8 /* Sep */, 26));\n    expect(result).toBe(true);\n  });\n\n  it(\"returns false if the given date is not Friday\", () => {\n    const result = isFriday(new Date(2014, 8 /* Sep */, 25));\n    expect(result).toBe(false);\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const result = isFriday(new Date(2014, 1 /* Feb */, 14).getTime());\n    expect(result).toBe(true);\n  });\n\n  it(\"returns false if the given date is `Invalid Date`\", () => {\n    const result = isFriday(new Date(NaN));\n    expect(result).toBe(false);\n  });\n});\n\n// Path: src/formatRFC3339/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport sinon from \"sinon\";\nimport { formatRFC3339 } from \"./index.js\";\nimport { generateOffset } from \"../_lib/test/index.js\";\n\ndescribe(\"formatRFC3339\", () => {\n  it(\"formats RFC-3339 date string\", () => {\n    const date = new Date(2019, 2 /* Mar */, 3, 19, 0, 52, 123);\n    expect(formatRFC3339(date)).toBe(`2019-03-03T19:00:52${generateOffset(date)}`);\n\n    const getTimezoneOffsetStub = sinon.stub(\n      Date.prototype,\n      \"getTimezoneOffset\",\n    );\n\n    getTimezoneOffsetStub.returns(0);\n    expect(formatRFC3339(date)).toBe(\"2019-03-03T19:00:52Z\");\n\n    getTimezoneOffsetStub.returns(480);\n    expect(formatRFC3339(date)).toBe(\"2019-03-03T19:00:52-08:00\");\n\n    getTimezoneOffsetStub.restore();\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const date = new Date(2019, 9 /* Oct */, 4, 12, 30, 13, 456);\n    const time = date.getTime();\n    expect(formatRFC3339(time)).toBe(`2019-10-04T12:30:13${generateOffset(date)}`);\n  });\n\n  it(\"allows to specify digits of second fractions\", () => {\n    const date = new Date(2019, 11 /* Dec */, 11, 1, 0, 0, 789);\n    expect(formatRFC3339(date, { fractionDigits: 3 })).toBe(`2019-12-11T01:00:00.789${generateOffset(date)}`);\n  });\n\n  it(\"works when ms < 100\", () => {\n    const date = new Date(2019, 11 /* Dec */, 11, 1, 0, 0, 12);\n    expect(formatRFC3339(date, { fractionDigits: 2 })).toBe(`2019-12-11T01:00:00.01${generateOffset(date)}`);\n  });\n\n  it(\"throws RangeError if the time value is invalid\", () => {\n    expect(formatRFC3339.bind(null, new Date(NaN))).toThrow(RangeError);\n  });\n});\n\n// Path: src/differenceInCalendarWeeks/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { differenceInCalendarWeeks } from \"./index.js\";\n\ndescribe(\"differenceInCalendarWeeks\", () => {\n  it(\"returns the number of calendar weeks between the given dates\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2014, 6 /* Jul */, 8, 18, 0),\n      new Date(2014, 5 /* Jun */, 29, 6, 0),\n    );\n    expect(result).toBe(1);\n  });\n\n  it(\"allows to specify which day is the first day of the week\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2014, 6 /* Jul */, 8, 18, 0),\n      new Date(2014, 5 /* Jun */, 29, 6, 0),\n      { weekStartsOn: 1 },\n    );\n    expect(result).toBe(2);\n  });\n\n  it(\"allows to specify which day is the first day of the week in locale\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2014, 6 /* Jul */, 8, 18, 0),\n      new Date(2014, 5 /* Jun */, 29, 6, 0),\n      {\n        locale: {\n          options: { weekStartsOn: 1 },\n        },\n      },\n    );\n    expect(result).toBe(2);\n  });\n\n  it(\"`options.weekStartsOn` overwrites the first day of the week specified in locale\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2014, 6 /* Jul */, 8, 18, 0),\n      new Date(2014, 5 /* Jun */, 29, 6, 0),\n      {\n        weekStartsOn: 1,\n        locale: {\n          options: { weekStartsOn: 0 },\n        },\n      },\n    );\n    expect(result).toBe(2);\n  });\n\n  it(\"returns a positive number if the time value of the second date is smaller\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2014, 6 /* Jul */, 8, 18, 0),\n      new Date(2014, 5 /* Jun */, 29, 6, 0),\n      {\n        weekStartsOn: 1,\n      },\n    );\n    expect(result).toBe(2);\n  });\n\n  it(\"returns a negative number if the time value of the first date is smaller\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2014, 5 /* Jun */, 29, 6, 0),\n      new Date(2014, 6 /* Jul */, 8, 18, 0),\n    );\n    expect(result).toBe(-1);\n  });\n\n  it(\"accepts timestamps\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2014, 6 /* Jul */, 12).getTime(),\n      new Date(2014, 6 /* Jul */, 2).getTime(),\n    );\n    expect(result).toBe(1);\n  });\n\n  describe(\"edge cases\", () => {\n    it(\"the difference is less than a week, but the given dates are in different calendar weeks\", () => {\n      const result = differenceInCalendarWeeks(\n        new Date(2014, 6 /* Jul */, 6),\n        new Date(2014, 6 /* Jul */, 5),\n      );\n      expect(result).toBe(1);\n    });\n\n    it(\"the same for the swapped dates\", () => {\n      const result = differenceInCalendarWeeks(\n        new Date(2014, 6 /* Jul */, 5),\n        new Date(2014, 6 /* Jul */, 6),\n      );\n      expect(result).toBe(-1);\n    });\n\n    it(\"the days of weeks of the given dates are the same\", () => {\n      const result = differenceInCalendarWeeks(\n        new Date(2014, 6 /* Jul */, 9),\n        new Date(2014, 6 /* Jul */, 2),\n      );\n      expect(result).toBe(1);\n    });\n\n    it(\"the given dates are the same\", () => {\n      const result = differenceInCalendarWeeks(\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n      );\n      expect(result).toBe(0);\n    });\n\n    it(\"does not return -0 when the given dates are the same\", () => {\n      function isNegativeZero(x: number): boolean {\n        return x === 0 && 1 / x < 0;\n      }\n\n      const result = differenceInCalendarWeeks(\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n        new Date(2014, 8 /* Sep */, 5, 0, 0),\n      );\n\n      const resultIsNegative = isNegativeZero(result);\n      expect(resultIsNegative).toBe(false);\n    });\n\n    it(\"properly works with negative numbers\", () => {\n      const a = new Date(2014, 6 /* Jul */, 9);\n      const b = new Date(2014, 6 /* Jul */, 19);\n      expect(differenceInCalendarWeeks(b, a)).toBe(1);\n      expect(differenceInCalendarWeeks(a, b)).toBe(-1);\n    });\n  });\n\n  it(\"returns NaN if the first date is `Invalid Date`\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(NaN),\n      new Date(2017, 0 /* Jan */, 1),\n    );\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the second date is `Invalid Date`\", () => {\n    const result = differenceInCalendarWeeks(\n      new Date(2017, 0 /* Jan */, 1),\n      new Date(NaN),\n    );\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the both dates are `Invalid Date`\", () => {\n    const result = differenceInCalendarWeeks(new Date(NaN), new Date(NaN));\n    expect(isNaN(result)).toBe(true);\n  });\n});\n\n// Path: src/secondsToMilliseconds/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { secondsToMilliseconds } from \"./index.js\";\n\ndescribe(\"secondsToMilliseconds\", () => {\n  it(\"converts seconds to milliseconds\", () => {\n    expect(secondsToMilliseconds(1)).toBe(1000);\n    expect(secondsToMilliseconds(2)).toBe(2000);\n  });\n\n  it(\"handles border values\", () => {\n    expect(secondsToMilliseconds(1.5)).toBe(1500);\n    expect(secondsToMilliseconds(0)).toBe(0);\n  });\n});\n\n// Path: src/formatISO9075/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { formatISO9075 } from \"./index.js\";\n\ndescribe(\"formatISO9075\", () => {\n  it(\"formats ISO-9075 extended date format\", () => {\n    const date = new Date(2019, 2 /* Mar */, 3, 19, 0, 52, 123);\n    expect(formatISO9075(date)).toBe(\"2019-03-03 19:00:52\");\n  });\n\n  it(\"accepts a timestamp\", () => {\n    const date = new Date(2019, 2 /* Mar */, 3, 19, 0, 52, 123).getTime();\n    expect(formatISO9075(date)).toBe(\"2019-03-03 19:00:52\");\n  });\n\n  it(\"formats ISO-8601 basic date format\", () => {\n    const date = new Date(2019, 9 /* Oct */, 4, 12, 30, 13, 456);\n    expect(formatISO9075(date, { format: \"basic\" })).toBe(\"20191004 123013\");\n  });\n\n  it(\"formats only date\", () => {\n    const date = new Date(2019, 11 /* Dec */, 11, 1, 0, 0, 789);\n    expect(formatISO9075(date, { representation: \"date\", format: \"extended\" })).toBe(\"2019-12-11\");\n    expect(formatISO9075(date, { representation: \"date\", format: \"basic\" })).toBe(\"20191211\");\n  });\n\n  it(\"formats only time\", () => {\n    const date = new Date(2019, 2 /* Mar */, 3, 19, 0, 52, 123);\n    expect(formatISO9075(date, { representation: \"time\", format: \"extended\" })).toBe(\"19:00:52\");\n    expect(formatISO9075(date, { representation: \"time\", format: \"basic\" })).toBe(\"190052\");\n  });\n\n  it(\"throws RangeError if the time value is invalid\", () => {\n    expect(formatISO9075.bind(null, new Date(NaN))).toThrow(RangeError);\n  });\n});\n\n// Path: src/differenceInMilliseconds/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { differenceInMilliseconds } from \"./index.js\";\n\ndescribe(\"differenceInMilliseconds\", () => {\n  it(\"returns the number of milliseconds between the given dates\", () => {\n    const result = differenceInMilliseconds(\n      new Date(2014, 6 /* Jul */, 2, 12, 30, 20, 700),\n      new Date(2014, 6 /* Jul */, 2, 12, 30, 20, 600),\n    );\n    expect(result).toBe(100);\n  });\n\n  it(\"returns a negative number if the time value of the first date is smaller\", () => {\n    const result = differenceInMilliseconds(\n      new Date(2014, 6 /* Jul */, 2, 12, 30, 20, 600),\n      new Date(2014, 6 /* Jul */, 2, 12, 30, 20, 700),\n    );\n    expect(result).toBe(-100);\n  });\n\n  it(\"accepts timestamps\", () => {\n    const result = differenceInMilliseconds(\n      new Date(2014, 8 /* Sep */, 5, 18, 30, 45, 500).getTime(),\n      new Date(2014, 8 /* Sep */, 5, 18, 30, 45, 500).getTime(),\n    );\n    expect(result).toBe(0);\n  });\n\n  it(\"does not return -0 when the given dates are the same\", () => {\n    function isNegativeZero(x: number): boolean {\n      return x === 0 && 1 / x < 0;\n    }\n    const result = differenceInMilliseconds(\n      new Date(2014, 8 /* Sep */, 5, 0, 0),\n      new Date(2014, 8 /* Sep */, 5, 0, 0),\n    );\n\n    const resultIsNegative = isNegativeZero(result);\n    expect(resultIsNegative).toBe(false);\n  });\n\n  it(\"returns NaN if the first date is `Invalid Date`\", () => {\n    const result = differenceInMilliseconds(\n      new Date(NaN),\n      new Date(2017, 0 /* Jan */, 1),\n    );\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the second date is `Invalid Date`\", () => {\n    const result = differenceInMilliseconds(\n      new Date(2017, 0 /* Jan */, 1),\n      new Date(NaN),\n    );\n    expect(isNaN(result)).toBe(true);\n  });\n\n  it(\"returns NaN if the both dates are `Invalid Date`\", () => {\n    const result = differenceInMilliseconds(new Date(NaN), new Date(NaN));\n    expect(isNaN(result)).toBe(true);\n  });\n});\n\n// Path: src/eachQuarterOfInterval/test.ts\nimport { describe, expect, it } from \"vitest\";\nimport { eachQuarterOfInterval } from \"./index.js\";\n\ndescribe(\"eachQuarterOfInterval\", () => {\n  it(\"returns an array with starts of quarters from the quarter of the start date to the quarter of the end date\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 2 /* Mar */, 6),\n      end: new Date(2014, 7 /* Aug */, 12),\n    });\n    expect(result).toEqual([\n      new Date(2014, 0 /* Jan */, 1),\n      new Date(2014, 3 /* Apr */, 1),\n      new Date(2014, 6 /* Jul */, 1),\n    ]);\n  });\n\n  it(\"accepts timestamps\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 2 /* Mar */, 6).getTime(),\n      end: new Date(2014, 7 /* Aug */, 12).getTime(),\n    });\n    expect(result).toEqual([\n      new Date(2014, 0 /* Jan */, 1),\n      new Date(2014, 3 /* Apr */, 1),\n      new Date(2014, 6 /* Jul */, 1),\n    ]);\n  });\n\n  it(\"handles the dates that are not starts of days\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 2 /* Mar */, 6, 6, 35),\n      end: new Date(2014, 7 /* Aug */, 12, 22, 15),\n    });\n    expect(result).toEqual([\n      new Date(2014, 0 /* Jan */, 1),\n      new Date(2014, 3 /* Apr */, 1),\n      new Date(2014, 6 /* Jul */, 1),\n    ]);\n  });\n\n  it(\"handles the dates that are not containing days\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 2 /* Mar */),\n      end: new Date(2014, 7 /* Oct */),\n    });\n    expect(result).toEqual([\n      new Date(2014, 0 /* Jan */, 1),\n      new Date(2014, 3 /* Apr */, 1),\n      new Date(2014, 6 /* Jul */, 1),\n    ]);\n  });\n\n  it(\"returns one quarter if the both arguments are on the same quarter\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 0 /* Jan */, 6, 14),\n      end: new Date(2014, 2 /* Feb */, 9, 15),\n    });\n    expect(result).toEqual([new Date(2014, 0 /* Jan */, 1)]);\n  });\n\n  it(\"returns one quarter if the both arguments are the same\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 9 /* Oct */, 6, 14),\n      end: new Date(2014, 9 /* Oct */, 6, 14),\n    });\n    expect(result).toEqual([new Date(2014, 9 /* Oct */, 1)]);\n  });\n\n  it(\"returns reversed array if the start date is after the end date\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 7 /* Aug */, 12),\n      end: new Date(2014, 2 /* Mar */, 6),\n    });\n    expect(result).toEqual([\n      new Date(2014, 6 /* Jul */, 1),\n      new Date(2014, 3 /* Apr */, 1),\n      new Date(2014, 0 /* Jan */, 1),\n    ]);\n  });\n\n  it(\"returns an empty array if the start date is `Invalid Date`\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(NaN),\n      end: new Date(2014, 9 /* Oct */, 6),\n    });\n    expect(result).toEqual([]);\n  });\n\n  it(\"returns an empty array if the end date is `Invalid Date`\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(2014, 9 /* Oct */, 12),\n      end: new Date(NaN),\n    });\n    expect(result).toEqual([]);\n  });\n\n  it(\"returns an empty array if both of the properties are `Invalid Date`\", () => {\n    const result = eachQuarterOfInterval({\n      start: new Date(NaN),\n      end: new Date(NaN),\n    });\n    expect(result).toEqual([]);\n  });\n\n  describe(\"options.step\", () => {\n    const interval = {\n      start: new Date(2014, 2 /* Mar */, 6),\n...\n", "needle_token_start": 8827, "needle_token_end": 8887, "code_context_ntokens": 16153, "output": ["```typescript\nfunction fn(a: unknown, b: unknown, c: unknown) {\n  return \"a b c\"\n    .replace(\"a\", String(a))\n    .replace(\"b\", String(b))\n    .replace(\"c\", String(c));\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "differenceInHours", "language": "typescript", "path": "src/differenceInHours/index.ts", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: Calculate the total number of hours between two specified dates, potentially rounding the result based on provided options.\n2. **Input**: Two dates (as Date objects, numbers, or strings representing dates) indicating the start and end points of the time interval, and an optional object containing rounding options.\n3. **Output**: A number representing the total hours between the two dates, possibly adjusted according to the specified rounding method.\n4. **Procedure**: The function first calculates the difference in milliseconds between the two dates. This difference is then divided by the number of milliseconds in an hour to convert it to hours. If rounding options are provided, the result is adjusted using the specified rounding method before being returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/endOfISOWeekYear/index.ts\nimport { getISOWeekYear } from \"../getISOWeekYear/index.js\";\nimport { startOfISOWeek } from \"../startOfISOWeek/index.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\n\n/**\n * @name endOfISOWeekYear\n * @category ISO Week-Numbering Year Helpers\n * @summary Return the end of an ISO week-numbering year for the given date.\n *\n * @description\n * Return the end of an ISO week-numbering year,\n...\n// Path: src/addQuarters/index.ts\nimport { addMonths } from \"../addMonths/index.js\";\n\n/**\n * @name addQuarters\n * @category Quarter Helpers\n * @summary Add the specified number of year quarters to the given date.\n *\n * @description\n * Add the specified number of year quarters to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of quarters to be added.\n *\n * @returns The new date with the quarters added\n *\n * @example\n * // Add 1 quarter to 1 September 2014:\n * const result = addQuarters(new Date(2014, 8, 1), 1)\n * //=> Mon Dec 01 2014 00:00:00\n */\nexport function addQuarters<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  const months = amount * 3;\n  return addMonths(date, months);\n}\n\n// Path: src/subQuarters/index.ts\nimport { addQuarters } from \"../addQuarters/index.js\";\n\n/**\n * @name subQuarters\n * @category Quarter Helpers\n * @summary Subtract the specified number of year quarters from the given date.\n *\n * @description\n * Subtract the specified number of year quarters from the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of quarters to be subtracted.\n *\n * @returns The new date with the quarters subtracted\n *\n * @example\n * // Subtract 3 quarters from 1 September 2014:\n * const result = subQuarters(new Date(2014, 8, 1), 3)\n * //=> Sun Dec 01 2013 00:00:00\n */\nexport function subQuarters<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addQuarters(date, -amount);\n}\n\n// Path: src/isBefore/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isBefore\n * @category Common Helpers\n * @summary Is the first date before the second one?\n *\n * @description\n * Is the first date before the second one?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date that should be before the other one to return true\n * @param dateToCompare - The date to compare with\n *\n * @returns The first date is before the second date\n *\n * @example\n * // Is 10 July 1989 before 11 February 1987?\n * const result = isBefore(new Date(1989, 6, 10), new Date(1987, 1, 11))\n * //=> false\n */\nexport function isBefore<DateType extends Date>(\n  date: DateType | number | string,\n  dateToCompare: DateType | number | string,\n): boolean {\n  const _date = toDate(date);\n  const _dateToCompare = toDate(dateToCompare);\n  return +_date < +_dateToCompare;\n}\n\n// Path: src/getQuarter/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getQuarter\n * @category Quarter Helpers\n * @summary Get the year quarter of the given date.\n *\n * @description\n * Get the year quarter of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The quarter\n *\n * @example\n * // Which quarter is 2 July 2014?\n * const result = getQuarter(new Date(2014, 6, 2))\n * //=> 3\n */\nexport function getQuarter<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const quarter = Math.trunc(_date.getMonth() / 3) + 1;\n  return quarter;\n}\n\n// Path: src/differenceInCalendarQuarters/index.ts\nimport { getQuarter } from \"../getQuarter/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name differenceInCalendarQuarters\n * @category Quarter Helpers\n * @summary Get the number of calendar quarters between the given dates.\n *\n * @description\n * Get the number of calendar quarters between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n\n * @returns The number of calendar quarters\n *\n * @example\n * // How many calendar quarters are between 31 December 2013 and 2 July 2014?\n * const result = differenceInCalendarQuarters(\n *   new Date(2014, 6, 2),\n *   new Date(2013, 11, 31)\n * )\n * //=> 3\n */\nexport function differenceInCalendarQuarters<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n): number {\n  const _dateLeft = toDate(dateLeft);\n  const _dateRight = toDate(dateRight);\n\n  const yearDiff = _dateLeft.getFullYear() - _dateRight.getFullYear();\n  const quarterDiff = getQuarter(_dateLeft) - getQuarter(_dateRight);\n\n  return yearDiff * 4 + quarterDiff;\n}\n\n// Path: src/formatDuration/index.ts\nimport type { FormatDistanceToken } from \"../locale/types.js\";\nimport type { Duration, DurationUnit, LocalizedOptions } from \"../types.js\";\nimport { defaultLocale } from \"../_lib/defaultLocale/index.js\";\nimport { getDefaultOptions } from \"../_lib/defaultOptions/index.js\";\n\n/**\n * The {@link formatDuration} function options.\n */\nexport interface FormatDurationOptions\n  extends LocalizedOptions<\"formatDistance\"> {\n  /** The array of units to format */\n  format?: DurationUnit[];\n  /** Should be zeros be included in the output? */\n  zero?: boolean;\n  /** The delimiter string to use */\n  delimiter?: string;\n}\n\nconst defaultFormat: DurationUnit[] = [\n  \"years\",\n  \"months\",\n  \"weeks\",\n  \"days\",\n  \"hours\",\n  \"minutes\",\n  \"seconds\",\n];\n\n/**\n * @name formatDuration\n * @category Common Helpers\n * @summary Formats a duration in human-readable format\n *\n * @description\n * Return human-readable duration string i.e. \"9 months 2 days\"\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param duration - The duration to format\n * @param options - An object with options.\n *\n * @returns The formatted date string\n *\n * @example\n * // Format full duration\n * formatDuration({\n *   years: 2,\n *   months: 9,\n *   weeks: 1,\n *   days: 7,\n *   hours: 5,\n *   minutes: 9,\n *   seconds: 30\n * })\n * //=> '2 years 9 months 1 week 7 days 5 hours 9 minutes 30 seconds'\n *\n * @example\n * // Format partial duration\n * formatDuration({ months: 9, days: 2 })\n * //=> '9 months 2 days'\n *\n * @example\n * // Customize the format\n * formatDuration(\n *   {\n *     years: 2,\n *     months: 9,\n *     weeks: 1,\n *     days: 7,\n *     hours: 5,\n *     minutes: 9,\n *     seconds: 30\n *   },\n *   { format: ['months', 'weeks'] }\n * ) === '9 months 1 week'\n *\n * @example\n * // Customize the zeros presence\n * formatDuration({ years: 0, months: 9 })\n * //=> '9 months'\n * formatDuration({ years: 0, months: 9 }, { zero: true })\n * //=> '0 years 9 months'\n *\n * @example\n * // Customize the delimiter\n * formatDuration({ years: 2, months: 9, weeks: 3 }, { delimiter: ', ' })\n * //=> '2 years, 9 months, 3 weeks'\n */\nexport function formatDuration(\n  duration: Duration,\n  options?: FormatDurationOptions,\n): string {\n  const defaultOptions = getDefaultOptions();\n  const locale = options?.locale ?? defaultOptions.locale ?? defaultLocale;\n  const format = options?.format ?? defaultFormat;\n  const zero = options?.zero ?? false;\n  const delimiter = options?.delimiter ?? \" \";\n\n  if (!locale.formatDistance) {\n    return \"\";\n  }\n\n  const result = format\n    .reduce((acc, unit) => {\n      const token = `x${unit.replace(/(^.)/, (m) =>\n        m.toUpperCase(),\n      )}` as FormatDistanceToken;\n      const value = duration[unit];\n      if (value !== undefined && (zero || duration[unit])) {\n        return acc.concat(locale.formatDistance(token, value));\n      }\n      return acc;\n    }, [] as string[])\n    .join(delimiter);\n\n  return result;\n}\n\n// Path: src/eachHourOfInterval/index.ts\nimport { addHours } from \"../addHours/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { Interval, StepOptions } from \"../types.js\";\n\n/**\n * The {@link eachHourOfInterval} function options.\n */\nexport interface EachHourOfIntervalOptions extends StepOptions {}\n\n/**\n * @name eachHourOfInterval\n * @category Interval Helpers\n * @summary Return the array of hours within the specified time interval.\n *\n * @description\n * Return the array of hours within the specified time interval.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param interval - The interval.\n * @param options - An object with options.\n *\n * @returns The array with starts of hours from the hour of the interval start to the hour of the interval end\n *\n * @example\n * // Each hour between 6 October 2014, 12:00 and 6 October 2014, 15:00\n * const result = eachHourOfInterval({\n *   start: new Date(2014, 9, 6, 12),\n *   end: new Date(2014, 9, 6, 15)\n * })\n * //=> [\n * //   Mon Oct 06 2014 12:00:00,\n * //   Mon Oct 06 2014 13:00:00,\n * //   Mon Oct 06 2014 14:00:00,\n * //   Mon Oct 06 2014 15:00:00\n * // ]\n */\nexport function eachHourOfInterval<DateType extends Date>(\n  interval: Interval<DateType>,\n  options?: EachHourOfIntervalOptions,\n): DateType[] {\n  const startDate = toDate(interval.start);\n  const endDate = toDate(interval.end);\n\n  let reversed = +startDate > +endDate;\n  const endTime = reversed ? +startDate : +endDate;\n  let currentDate = reversed ? endDate : startDate;\n  currentDate.setMinutes(0, 0, 0);\n\n  let step = options?.step ?? 1;\n  if (!step) return [];\n  if (step < 0) {\n    step = -step;\n    reversed = !reversed;\n  }\n\n  const dates = [];\n\n  while (+currentDate <= endTime) {\n    dates.push(toDate(currentDate));\n    currentDate = addHours(currentDate, step);\n  }\n\n  return reversed ? dates.reverse() : dates;\n}\n\n// Path: src/startOfHour/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name startOfHour\n * @category Hour Helpers\n * @summary Return the start of an hour for the given date.\n *\n * @description\n * Return the start of an hour for the given date.\n * The result will be in the local timezone.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The start of an hour\n *\n * @example\n * // The start of an hour for 2 September 2014 11:55:00:\n * const result = startOfHour(new Date(2014, 8, 2, 11, 55))\n * //=> Tue Sep 02 2014 11:00:00\n */\nexport function startOfHour<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  const _date = toDate(date);\n  _date.setMinutes(0, 0, 0);\n  return _date;\n}\n\n// Path: src/isSameHour/index.ts\nimport { startOfHour } from \"../startOfHour/index.js\";\n\n/**\n * @name isSameHour\n * @category Hour Helpers\n * @summary Are the given dates in the same hour (and same day)?\n *\n * @description\n * Are the given dates in the same hour (and same day)?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The first date to check\n * @param dateRight - The second date to check\n *\n * @returns The dates are in the same hour (and same day)\n *\n * @example\n * // Are 4 September 2014 06:00:00 and 4 September 06:30:00 in the same hour?\n * const result = isSameHour(new Date(2014, 8, 4, 6, 0), new Date(2014, 8, 4, 6, 30))\n * //=> true\n *\n * @example\n * // Are 4 September 2014 06:00:00 and 5 September 06:00:00 in the same hour?\n * const result = isSameHour(new Date(2014, 8, 4, 6, 0), new Date(2014, 8, 5, 6, 0))\n * //=> false\n */\nexport function isSameHour<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n): boolean {\n  const dateLeftStartOfHour = startOfHour(dateLeft);\n  const dateRightStartOfHour = startOfHour(dateRight);\n\n  return +dateLeftStartOfHour === +dateRightStartOfHour;\n}\n\n// Path: src/setYear/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setYear\n * @category Year Helpers\n * @summary Set the year to the given date.\n *\n * @description\n * Set the year to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param year - The year of the new date\n *\n * @returns The new date with the year set\n *\n * @example\n * // Set year 2013 to 1 September 2014:\n * const result = setYear(new Date(2014, 8, 1), 2013)\n * //=> Sun Sep 01 2013 00:00:00\n */\nexport function setYear<DateType extends Date>(\n  date: DateType | number | string,\n  year: number,\n): DateType {\n  const _date = toDate(date);\n\n  // Check if date is Invalid Date because Date.prototype.setFullYear ignores the value of Invalid Date\n  if (isNaN(+_date)) {\n    return constructFrom(date, NaN);\n  }\n\n  _date.setFullYear(year);\n  return _date;\n}\n\n// Path: src/closestTo/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name closestTo\n * @category Common Helpers\n * @summary Return a date from the array closest to the given date.\n *\n * @description\n * Return a date from the array closest to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateToCompare - The date to compare with\n * @param dates - The array to search\n *\n * @returns The date from the array closest to the given date or undefined if no valid value is given\n *\n * @example\n * // Which date is closer to 6 September 2015: 1 January 2000 or 1 January 2030?\n * const dateToCompare = new Date(2015, 8, 6)\n * const result = closestTo(dateToCompare, [\n *   new Date(2000, 0, 1),\n *   new Date(2030, 0, 1)\n * ])\n * //=> Tue Jan 01 2030 00:00:00\n */\nexport function closestTo<DateType extends Date>(\n  dateToCompare: DateType | number | string,\n  dates: Array<DateType | number | string>,\n): DateType | undefined {\n  const date = toDate(dateToCompare);\n\n  if (isNaN(Number(date))) return constructFrom(dateToCompare, NaN);\n\n  const timeToCompare = date.getTime();\n\n  let result: DateType | undefined;\n  let minDistance: number;\n  dates.forEach((dirtyDate) => {\n    const currentDate = toDate(dirtyDate);\n\n    if (isNaN(Number(currentDate))) {\n      result = constructFrom(dateToCompare, NaN);\n      minDistance = NaN;\n      return;\n    }\n\n    const distance = Math.abs(timeToCompare - currentDate.getTime());\n    if (result == null || distance < minDistance) {\n      result = currentDate;\n      minDistance = distance;\n    }\n  });\n\n  return result;\n}\n\n// Path: src/setDayOfYear/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setDayOfYear\n * @category Day Helpers\n * @summary Set the day of the year to the given date.\n *\n * @description\n * Set the day of the year to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param dayOfYear - The day of the year of the new date\n *\n * @returns The new date with the day of the year set\n *\n * @example\n * // Set the 2nd day of the year to 2 July 2014:\n * const result = setDayOfYear(new Date(2014, 6, 2), 2)\n * //=> Thu Jan 02 2014 00:00:00\n */\nexport function setDayOfYear<DateType extends Date>(\n  date: DateType | number | string,\n  dayOfYear: number,\n): DateType {\n  const _date = toDate(date);\n  _date.setMonth(0);\n  _date.setDate(dayOfYear);\n  return _date;\n}\n\n// Path: src/min/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name min\n * @category Common Helpers\n * @summary Returns the earliest of the given dates.\n *\n * @description\n * Returns the earliest of the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dates - The dates to compare\n *\n * @returns The earliest of the dates\n *\n * @example\n * // Which of these dates is the earliest?\n * const result = min([\n *   new Date(1989, 6, 10),\n *   new Date(1987, 1, 11),\n *   new Date(1995, 6, 2),\n *   new Date(1990, 0, 1)\n * ])\n * //=> Wed Feb 11 1987 00:00:00\n */\nexport function min<DateType extends Date>(\n  dates: Array<DateType | number | string>,\n): DateType | Date {\n  let result: Date | undefined;\n\n  dates.forEach((dirtyDate) => {\n    const date = toDate(dirtyDate);\n    if (!result || result > date || isNaN(+date)) {\n      result = date;\n    }\n  });\n\n  return result || new Date(NaN);\n}\n\n// Path: src/isWednesday/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isWednesday\n * @category Weekday Helpers\n * @summary Is the given date Wednesday?\n *\n * @description\n * Is the given date Wednesday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is Wednesday\n *\n * @example\n * // Is 24 September 2014 Wednesday?\n * const result = isWednesday(new Date(2014, 8, 24))\n * //=> true\n */\nexport function isWednesday<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return toDate(date).getDay() === 3;\n}\n\n// Path: src/milliseconds/index.ts\nimport { daysInYear } from \"../constants/index.js\";\nimport type { Duration } from \"../types.js\";\n\n/**\n * @name milliseconds\n * @category Millisecond Helpers\n * @summary\n * Returns the number of milliseconds in the specified, years, months, weeks, days, hours, minutes and seconds.\n *\n * @description\n * Returns the number of milliseconds in the specified, years, months, weeks, days, hours, minutes and seconds.\n *\n * One years equals 365.2425 days according to the formula:\n *\n * > Leap year occures every 4 years, except for years that are divisable by 100 and not divisable by 400.\n * > 1 mean year = (365+1/4-1/100+1/400) days = 365.2425 days\n *\n * One month is a year divided by 12.\n *\n * @param duration - The object with years, months, weeks, days, hours, minutes and seconds to be added.\n *\n * @returns The milliseconds\n *\n * @example\n * // 1 year in milliseconds\n * milliseconds({ years: 1 })\n * //=> 31556952000\n *\n * // 3 months in milliseconds\n * milliseconds({ months: 3 })\n * //=> 7889238000\n */\nexport function milliseconds({\n  years,\n  months,\n  weeks,\n  days,\n  hours,\n  minutes,\n  seconds,\n}: Duration): number {\n  let totalDays = 0;\n\n  if (years) totalDays += years * daysInYear;\n  if (months) totalDays += months * (daysInYear / 12);\n  if (weeks) totalDays += weeks * 7;\n  if (days) totalDays += days;\n\n  let totalSeconds = totalDays * 24 * 60 * 60;\n\n  if (hours) totalSeconds += hours * 60 * 60;\n  if (minutes) totalSeconds += minutes * 60;\n  if (seconds) totalSeconds += seconds;\n\n  return Math.trunc(totalSeconds * 1000);\n}\n\n// Path: src/getDaysInMonth/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\n\n/**\n * @name getDaysInMonth\n * @category Month Helpers\n * @summary Get the number of days in a month of the given date.\n *\n * @description\n * Get the number of days in a month of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The number of days in a month\n *\n * @example\n * // How many days are in February 2000?\n * const result = getDaysInMonth(new Date(2000, 1))\n * //=> 29\n */\nexport function getDaysInMonth<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const year = _date.getFullYear();\n  const monthIndex = _date.getMonth();\n  const lastDayOfMonth = constructFrom(date, 0);\n  lastDayOfMonth.setFullYear(year, monthIndex + 1, 0);\n  lastDayOfMonth.setHours(0, 0, 0, 0);\n  return lastDayOfMonth.getDate();\n}\n\n// Path: src/setMonth/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { getDaysInMonth } from \"../getDaysInMonth/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setMonth\n * @category Month Helpers\n * @summary Set the month to the given date.\n *\n * @description\n * Set the month to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param month - The month index to set (0-11)\n *\n * @returns The new date with the month set\n *\n * @example\n * // Set February to 1 September 2014:\n * const result = setMonth(new Date(2014, 8, 1), 1)\n * //=> Sat Feb 01 2014 00:00:00\n */\nexport function setMonth<DateType extends Date>(\n  date: DateType | number | string,\n  month: number,\n): DateType {\n  const _date = toDate(date);\n  const year = _date.getFullYear();\n  const day = _date.getDate();\n\n  const dateWithDesiredMonth = constructFrom(date, 0);\n  dateWithDesiredMonth.setFullYear(year, month, 15);\n  dateWithDesiredMonth.setHours(0, 0, 0, 0);\n  const daysInMonth = getDaysInMonth(dateWithDesiredMonth);\n  // Set the last day of the new month\n  // if the original date was the last day of the longer month\n  _date.setMonth(month, Math.min(day, daysInMonth));\n  return _date;\n}\n\n// Path: src/set/index.ts\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { setMonth } from \"../setMonth/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { DateValues } from \"../types.js\";\n\n/**\n * @name set\n * @category Common Helpers\n * @summary Set date values to a given date.\n *\n * @description\n * Set date values to a given date.\n *\n * Sets time values to date from object `values`.\n * A value is not set if it is undefined or null or doesn't exist in `values`.\n *\n * Note about bundle size: `set` does not internally use `setX` functions from date-fns but instead opts\n * to use native `Date#setX` methods. If you use this function, you may not want to include the\n * other `setX` functions that date-fns provides if you are concerned about the bundle size.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param values - The date values to be set\n *\n * @returns The new date with options set\n *\n * @example\n * // Transform 1 September 2014 into 20 October 2015 in a single line:\n * const result = set(new Date(2014, 8, 20), { year: 2015, month: 9, date: 20 })\n * //=> Tue Oct 20 2015 00:00:00\n *\n * @example\n * // Set 12 PM to 1 September 2014 01:23:45 to 1 September 2014 12:00:00:\n * const result = set(new Date(2014, 8, 1, 1, 23, 45), { hours: 12 })\n * //=> Mon Sep 01 2014 12:23:45\n */\n\nexport function set<DateType extends Date>(\n  date: DateType | number | string,\n  values: DateValues,\n): DateType {\n  let _date = toDate(date);\n\n  // Check if date is Invalid Date because Date.prototype.setFullYear ignores the value of Invalid Date\n  if (isNaN(+_date)) {\n    return constructFrom(date, NaN);\n  }\n\n  if (values.year != null) {\n    _date.setFullYear(values.year);\n  }\n\n  if (values.month != null) {\n    _date = setMonth(_date, values.month);\n  }\n\n  if (values.date != null) {\n    _date.setDate(values.date);\n  }\n\n  if (values.hours != null) {\n    _date.setHours(values.hours);\n  }\n\n  if (values.minutes != null) {\n    _date.setMinutes(values.minutes);\n  }\n\n  if (values.seconds != null) {\n    _date.setSeconds(values.seconds);\n  }\n\n  if (values.milliseconds != null) {\n    _date.setMilliseconds(values.milliseconds);\n  }\n\n  return _date;\n}\n\n// Path: src/hoursToSeconds/index.ts\nimport { secondsInHour } from \"../constants/index.js\";\n\n/**\n * @name hoursToSeconds\n * @category Conversion Helpers\n * @summary Convert hours to seconds.\n *\n * @description\n * Convert a number of hours to a full number of seconds.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param hours - The number of hours to be converted\n *\n * @returns The number of hours converted in seconds\n *\n * @example\n * // Convert 2 hours to seconds:\n * const result = hoursToSeconds(2)\n * //=> 7200\n */\nexport function hoursToSeconds(hours: number): number {\n  return Math.trunc(hours * secondsInHour);\n}\n\n// Path: src/differenceInCalendarYears/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name differenceInCalendarYears\n * @category Year Helpers\n * @summary Get the number of calendar years between the given dates.\n *\n * @description\n * Get the number of calendar years between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n\n * @returns The number of calendar years\n *\n * @example\n * // How many calendar years are between 31 December 2013 and 11 February 2015?\n * const result = differenceInCalendarYears(\n *   new Date(2015, 1, 11),\n *   new Date(2013, 11, 31)\n * )\n * //=> 2\n */\nexport function differenceInCalendarYears<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n): number {\n  const _dateLeft = toDate(dateLeft);\n  const _dateRight = toDate(dateRight);\n\n  return _dateLeft.getFullYear() - _dateRight.getFullYear();\n}\n\n// Path: src/differenceInSeconds/index.ts\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { differenceInMilliseconds } from \"../differenceInMilliseconds/index.js\";\nimport type { RoundingOptions } from \"../types.js\";\n\n/**\n * The {@link differenceInSeconds} function options.\n */\nexport interface DifferenceInSecondsOptions extends RoundingOptions {}\n\n/**\n * @name differenceInSeconds\n * @category Second Helpers\n * @summary Get the number of seconds between the given dates.\n *\n * @description\n * Get the number of seconds between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options.\n *\n * @returns The number of seconds\n *\n * @example\n * // How many seconds are between\n * // 2 July 2014 12:30:07.999 and 2 July 2014 12:30:20.000?\n * const result = differenceInSeconds(\n *   new Date(2014, 6, 2, 12, 30, 20, 0),\n *   new Date(2014, 6, 2, 12, 30, 7, 999)\n * )\n * //=> 12\n */\nexport function differenceInSeconds<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n  options?: DifferenceInSecondsOptions,\n): number {\n  const diff = differenceInMilliseconds(dateLeft, dateRight) / 1000;\n  return getRoundingMethod(options?.roundingMethod)(diff);\n}\n\n// Path: src/differenceInHours/index.ts\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { millisecondsInHour } from \"../constants/index.js\";\nimport { differenceInMilliseconds } from \"../differenceInMilliseconds/index.js\";\nimport type { RoundingOptions } from \"../types.js\";\n\n/**\n * The {@link differenceInHours} function options.\n */\nexport interface DifferenceInHoursOptions extends RoundingOptions {}\n\n/**\n * @name differenceInHours\n * @category Hour Helpers\n * @summary Get the number of hours between the given dates.\n *\n * @description\n * Get the number of hours between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options.\n *\n * @returns The number of hours\n *\n * @example\n * // How many hours are between 2 July 2014 06:50:00 and 2 July 2014 19:00:00?\n * const result = differenceInHours(\n *   new Date(2014, 6, 2, 19, 0),\n *   new Date(2014, 6, 2, 6, 50)\n * )\n * //=> 12\n */\nexport \nfunction differenceInHours<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n  options?: DifferenceInHoursOptions,\n): number {\n  const diff =\n    differenceInMilliseconds(dateLeft, dateRight) / millisecondsInHour;\n  return getRoundingMethod(options?.roundingMethod)(diff);\n}\n\n// Path: src/differenceInCalendarWeeks/index.ts\nimport { millisecondsInWeek } from \"../constants/index.js\";\nimport { startOfWeek } from \"../startOfWeek/index.js\";\nimport type { LocalizedOptions, WeekOptions } from \"../types.js\";\nimport { getTimezoneOffsetInMilliseconds } from \"../_lib/getTimezoneOffsetInMilliseconds/index.js\";\n\n/**\n * The {@link differenceInCalendarWeeks} function options.\n */\nexport interface DifferenceInCalendarWeeksOptions\n  extends LocalizedOptions<\"options\">,\n    WeekOptions {}\n\n/**\n * @name differenceInCalendarWeeks\n * @category Week Helpers\n * @summary Get the number of calendar weeks between the given dates.\n *\n * @description\n * Get the number of calendar weeks between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options.\n *\n * @returns The number of calendar weeks\n *\n * @example\n * // How many calendar weeks are between 5 July 2014 and 20 July 2014?\n * const result = differenceInCalendarWeeks(\n *   new Date(2014, 6, 20),\n *   new Date(2014, 6, 5)\n * )\n * //=> 3\n *\n * @example\n * // If the week starts on Monday,\n * // how many calendar weeks are between 5 July 2014 and 20 July 2014?\n * const result = differenceInCalendarWeeks(\n *   new Date(2014, 6, 20),\n *   new Date(2014, 6, 5),\n *   { weekStartsOn: 1 }\n * )\n * //=> 2\n */\nexport function differenceInCalendarWeeks<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n  options?: DifferenceInCalendarWeeksOptions,\n): number {\n  const startOfWeekLeft = startOfWeek(dateLeft, options);\n  const startOfWeekRight = startOfWeek(dateRight, options);\n\n  const timestampLeft =\n    +startOfWeekLeft - getTimezoneOffsetInMilliseconds(startOfWeekLeft);\n  const timestampRight =\n    +startOfWeekRight - getTimezoneOffsetInMilliseconds(startOfWeekRight);\n\n  // Round the number of days to the nearest integer because the number of\n  // milliseconds in a days is not constant (e.g. it's different in the week of\n  // the daylight saving time clock shift).\n  return Math.round((timestampLeft - timestampRight) / millisecondsInWeek);\n}\n\n// Path: src/differenceInMinutes/index.ts\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { millisecondsInMinute } from \"../constants/index.js\";\nimport { differenceInMilliseconds } from \"../differenceInMilliseconds/index.js\";\nimport type { RoundingOptions } from \"../types.js\";\n\n/**\n * The {@link differenceInMinutes} function options.\n */\nexport interface DifferenceInMinutesOptions extends RoundingOptions {}\n\n/**\n * @name differenceInMinutes\n * @category Minute Helpers\n * @summary Get the number of minutes between the given dates.\n *\n * @description\n * Get the signed number of full (rounded towards 0) minutes between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options.\n *\n * @returns The number of minutes\n *\n * @example\n * // How many minutes are between 2 July 2014 12:07:59 and 2 July 2014 12:20:00?\n * const result = differenceInMinutes(\n *   new Date(2014, 6, 2, 12, 20, 0),\n *   new Date(2014, 6, 2, 12, 7, 59)\n * )\n * //=> 12\n *\n * @example\n * // How many minutes are between 10:01:59 and 10:00:00\n * const result = differenceInMinutes(\n *   new Date(2000, 0, 1, 10, 0, 0),\n *   new Date(2000, 0, 1, 10, 1, 59)\n * )\n * //=> -1\n */\nexport function differenceInMinutes<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n  options?: DifferenceInMinutesOptions,\n): number {\n  const diff =\n    differenceInMilliseconds(dateLeft, dateRight) / millisecondsInMinute;\n  return getRoundingMethod(options?.roundingMethod)(diff);\n}\n\n// Path: src/intlFormatDistance/index.ts\nimport {\n  secondsInDay,\n  secondsInHour,\n  secondsInMinute,\n  secondsInMonth,\n  secondsInQuarter,\n  secondsInWeek,\n  secondsInYear,\n} from \"../constants/index.js\";\nimport { differenceInCalendarDays } from \"../differenceInCalendarDays/index.js\";\nimport { differenceInCalendarMonths } from \"../differenceInCalendarMonths/index.js\";\nimport { differenceInCalendarQuarters } from \"../differenceInCalendarQuarters/index.js\";\nimport { differenceInCalendarWeeks } from \"../differenceInCalendarWeeks/index.js\";\nimport { differenceInCalendarYears } from \"../differenceInCalendarYears/index.js\";\nimport { differenceInHours } from \"../differenceInHours/index.js\";\nimport { differenceInMinutes } from \"../differenceInMinutes/index.js\";\nimport { differenceInSeconds } from \"../differenceInSeconds/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * The {@link intlFormatDistance} function options.\n */\nexport interface IntlFormatDistanceOptions {\n  /** Force the distance unit */\n  unit?: IntlFormatDistanceUnit;\n  /** The locale(s) to use (see: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl#locales_argument) */\n  locale?: Intl.UnicodeBCP47LocaleIdentifier | Intl.UnicodeBCP47LocaleIdentifier[];\n  /** The locale matching algorithm to use. Other value: 'lookup'. See MDN for details [Locale identification and negotiation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl#locale_identification_and_negotiation) */\n  localeMatcher?: Intl.RelativeTimeFormatLocaleMatcher;\n  /** The output message format. The values are 'auto' (e.g. `yesterday`), 'always'(e.g. `1 day ago`) */\n  numeric?: Intl.RelativeTimeFormatNumeric;\n  /** The length of the result. The values are: 'long' (e.g. `1 month`), 'short' (e.g. 'in 1 mo.'), 'narrow' (e.g. 'in 1 mo.'). The narrow one could be similar to the short one for some locales. */\n  style?: Intl.RelativeTimeFormatStyle;\n}\n\n/**\n * The unit used to format the distance in {@link intlFormatDistance}.\n */\nexport type IntlFormatDistanceUnit =\n  | \"year\"\n  | \"quarter\"\n  | \"month\"\n  | \"week\"\n  | \"day\"\n  | \"hour\"\n  | \"minute\"\n  | \"second\";\n\n/**\n * @name intlFormatDistance\n * @category Common Helpers\n * @summary Formats distance between two dates in a human-readable format\n * @description\n * The function calculates the difference between two dates and formats it as a human-readable string.\n *\n * The function will pick the most appropriate unit depending on the distance between dates. For example, if the distance is a few hours, it might return `x hours`. If the distance is a few months, it might return `x months`.\n *\n * You can also specify a unit to force using it regardless of the distance to get a result like `123456 hours`.\n *\n * See the table below for the unit picking logic:\n *\n * | Distance between dates | Result (past)  | Result (future) |\n * | ---------------------- | -------------- | --------------- |\n * | 0 seconds              | now            | now             |\n * | 1-59 seconds           | X seconds ago  | in X seconds    |\n * | 1-59 minutes           | X minutes ago  | in X minutes    |\n * | 1-23 hours             | X hours ago    | in X hours      |\n * | 1 day                  | yesterday      | tomorrow        |\n * | 2-6 days               | X days ago     | in X days       |\n * | 7 days                 | last week      | next week       |\n * | 8 days-1 month         | X weeks ago    | in X weeks      |\n * | 1 month                | last month     | next month      |\n * | 2-3 months             | X months ago   | in X months     |\n * | 1 quarter              | last quarter   | next quarter    |\n * | 2-3 quarters           | X quarters ago | in X quarters   |\n * | 1 year                 | last year      | next year       |\n * | 2+ years               | X years ago    | in X years      |\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date\n * @param baseDate - The date to compare with.\n * @param options - An object with options.\n * See MDN for details [Locale identification and negotiation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl#locale_identification_and_negotiation)\n * The narrow one could be similar to the short one for some locales.\n *\n * @returns The distance in words according to language-sensitive relative time formatting.\n *\n * @throws `date` must not be Invalid Date\n * @throws `baseDate` must not be Invalid Date\n * @throws `options.unit` must not be invalid Unit\n * @throws `options.locale` must not be invalid locale\n * @throws `options.localeMatcher` must not be invalid localeMatcher\n * @throws `options.numeric` must not be invalid numeric\n * @throws `options.style` must not be invalid style\n *\n * @example\n * // What is the distance between the dates when the fist date is after the second?\n * intlFormatDistance(\n *   new Date(1986, 3, 4, 11, 30, 0),\n *   new Date(1986, 3, 4, 10, 30, 0)\n * )\n * //=> 'in 1 hour'\n *\n * // What is the distance between the dates when the fist date is before the second?\n * intlFormatDistance(\n *   new Date(1986, 3, 4, 10, 30, 0),\n *   new Date(1986, 3, 4, 11, 30, 0)\n * )\n * //=> '1 hour ago'\n *\n * @example\n * // Use the unit option to force the function to output the result in quarters. Without setting it, the example would return \"next year\"\n * intlFormatDistance(\n *   new Date(1987, 6, 4, 10, 30, 0),\n *   new Date(1986, 3, 4, 10, 30, 0),\n *   { unit: 'quarter' }\n * )\n * //=> 'in 5 quarters'\n *\n * @example\n * // Use the locale option to get the result in Spanish. Without setting it, the example would return \"in 1 hour\".\n * intlFormatDistance(\n *   new Date(1986, 3, 4, 11, 30, 0),\n *   new Date(1986, 3, 4, 10, 30, 0),\n *   { locale: 'es' }\n * )\n * //=> 'dentro de 1 hora'\n *\n * @example\n * // Use the numeric option to force the function to use numeric values. Without setting it, the example would return \"tomorrow\".\n * intlFormatDistance(\n *   new Date(1986, 3, 5, 11, 30, 0),\n *   new Date(1986, 3, 4, 11, 30, 0),\n *   { numeric: 'always' }\n * )\n * //=> 'in 1 day'\n *\n * @example\n * // Use the style option to force the function to use short values. Without setting it, the example would return \"in 2 years\".\n * intlFormatDistance(\n *   new Date(1988, 3, 4, 11, 30, 0),\n *   new Date(1986, 3, 4, 11, 30, 0),\n *   { style: 'short' }\n * )\n * //=> 'in 2 yr'\n */\nexport function intlFormatDistance<DateType extends Date>(\n  date: DateType | number | string,\n  baseDate: DateType | number | string,\n  options?: IntlFormatDistanceOptions,\n): string {\n  let value: number = 0;\n  let unit: Intl.RelativeTimeFormatUnit;\n  const dateLeft = toDate(date);\n  const dateRight = toDate(baseDate);\n\n  if (!options?.unit) {\n    // Get the unit based on diffInSeconds calculations if no unit is specified\n    const diffInSeconds = differenceInSeconds(dateLeft, dateRight); // The smallest unit\n\n    if (Math.abs(diffInSeconds) < secondsInMinute) {\n      value = differenceInSeconds(dateLeft, dateRight);\n      unit = \"second\";\n    } else if (Math.abs(diffInSeconds) < secondsInHour) {\n      value = differenceInMinutes(dateLeft, dateRight);\n      unit = \"minute\";\n    } else if (\n      Math.abs(diffInSeconds) < secondsInDay &&\n      Math.abs(differenceInCalendarDays(dateLeft, dateRight)) < 1\n    ) {\n      value = differenceInHours(dateLeft, dateRight);\n      unit = \"hour\";\n    } else if (\n      Math.abs(diffInSeconds) < secondsInWeek &&\n      (value = differenceInCalendarDays(dateLeft, dateRight)) &&\n      Math.abs(value) < 7\n    ) {\n      unit = \"day\";\n    } else if (Math.abs(diffInSeconds) < secondsInMonth) {\n      value = differenceInCalendarWeeks(dateLeft, dateRight);\n      unit = \"week\";\n    } else if (Math.abs(diffInSeconds) < secondsInQuarter) {\n      value = differenceInCalendarMonths(dateLeft, dateRight);\n      unit = \"month\";\n    } else if (Math.abs(diffInSeconds) < secondsInYear) {\n      if (differenceInCalendarQuarters(dateLeft, dateRight) < 4) {\n        // To filter out cases that are less than a year but match 4 quarters\n        value = differenceInCalendarQuarters(dateLeft, dateRight);\n        unit = \"quarter\";\n      } else {\n        value = differenceInCalendarYears(dateLeft, dateRight);\n        unit = \"year\";\n      }\n    } else {\n      value = differenceInCalendarYears(dateLeft, dateRight);\n      unit = \"year\";\n    }\n  } else {\n    // Get the value if unit is specified\n    unit = options?.unit;\n    if (unit === \"second\") {\n      value = differenceInSeconds(dateLeft, dateRight);\n    } else if (unit === \"minute\") {\n      value = differenceInMinutes(dateLeft, dateRight);\n    } else if (unit === \"hour\") {\n      value = differenceInHours(dateLeft, dateRight);\n    } else if (unit === \"day\") {\n      value = differenceInCalendarDays(dateLeft, dateRight);\n    } else if (unit === \"week\") {\n      value = differenceInCalendarWeeks(dateLeft, dateRight);\n    } else if (unit === \"month\") {\n      value = differenceInCalendarMonths(dateLeft, dateRight);\n    } else if (unit === \"quarter\") {\n      value = differenceInCalendarQuarters(dateLeft, dateRight);\n    } else if (unit === \"year\") {\n      value = differenceInCalendarYears(dateLeft, dateRight);\n    }\n  }\n\n  const rtf = new Intl.RelativeTimeFormat(options?.locale, {\n    localeMatcher: options?.localeMatcher,\n    numeric: options?.numeric || \"auto\",\n    style: options?.style,\n  });\n\n  return rtf.format(value, unit);\n}\n\n// Path: src/isThisHour/index.ts\nimport { constructNow } from \"../constructNow/index.js\";\nimport { isSameHour } from \"../isSameHour/index.js\";\n\n/**\n * @name isThisHour\n * @category Hour Helpers\n * @summary Is the given date in the same hour as the current date?\n * @pure false\n *\n * @description\n * Is the given date in the same hour as the current date?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is in this hour\n *\n * @example\n * // If now is 25 September 2014 18:30:15.500,\n * // is 25 September 2014 18:00:00 in this hour?\n * const result = isThisHour(new Date(2014, 8, 25, 18))\n * //=> true\n */\nexport function isThisHour<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return isSameHour(date, constructNow(date));\n}\n\n// Path: src/eachDayOfInterval/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport type { Interval, StepOptions } from \"../types.js\";\n\n/**\n * The {@link eachDayOfInterval} function options.\n */\nexport interface EachDayOfIntervalOptions extends StepOptions {}\n\n/**\n * @name eachDayOfInterval\n * @category Interval Helpers\n * @summary Return the array of dates within the specified time interval.\n *\n * @description\n * Return the array of dates within the specified time interval.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param interval - The interval.\n * @param options - An object with options.\n *\n * @returns The array with starts of days from the day of the interval start to the day of the interval end\n *\n * @example\n * // Each day between 6 October 2014 and 10 October 2014:\n * const result = eachDayOfInterval({\n *   start: new Date(2014, 9, 6),\n *   end: new Date(2014, 9, 10)\n * })\n * //=> [\n * //   Mon Oct 06 2014 00:00:00,\n * //   Tue Oct 07 2014 00:00:00,\n * //   Wed Oct 08 2014 00:00:00,\n * //   Thu Oct 09 2014 00:00:00,\n * //   Fri Oct 10 2014 00:00:00\n * // ]\n */\nexport function eachDayOfInterval<DateType extends Date>(\n  interval: Interval<DateType>,\n  options?: EachDayOfIntervalOptions,\n): DateType[] {\n  const startDate = toDate(interval.start);\n  const endDate = toDate(interval.end);\n\n  let reversed = +startDate > +endDate;\n  const endTime = reversed ? +startDate : +endDate;\n  const currentDate = reversed ? endDate : startDate;\n  currentDate.setHours(0, 0, 0, 0);\n\n  let step = options?.step ?? 1;\n  if (!step) return [];\n  if (step < 0) {\n    step = -step;\n    reversed = !reversed;\n  }\n\n  const dates = [];\n\n  while (+currentDate <= endTime) {\n    dates.push(toDate(currentDate));\n    currentDate.setDate(currentDate.getDate() + step);\n    currentDate.setHours(0, 0, 0, 0);\n  }\n\n  return reversed ? dates.reverse() : dates;\n}\n\n// Path: src/eachWeekendOfInterval/index.ts\n...\n", "needle_token_start": 9968, "needle_token_end": 10061, "code_context_ntokens": 15496, "output": ["```typescript\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { millisecondsInHour } from \"../constants/index.js\";\nimport { differenceInMilliseconds } from \"../differenceInMilliseconds/index.js\";\nimport type { RoundingOptions } from \"../types.js\";\n\n/**\n * The {@link differenceInHours} function options.\n */\nexport interface DifferenceInHoursOptions extends RoundingOptions {}\n\n/**\n * @name differenceInHours\n * @category Hour Helpers\n * @summary Get the number of hours between the given dates.\n *\n * @description\n * Get the number of hours between the given dates.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options.\n *\n * @returns The number of hours\n *\n * @example\n * // How many hours are between 2 July 2014 06:50:00 and 2 July 2014 19:00:00?\n * const result = differenceInHours(\n *   new Date(2014, 6, 2, 19, 0),\n *   new Date(2014, 6, 2, 6, 50)\n * )\n * //=> 12\n */\nexport function differenceInHours<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n  options?: DifferenceInHoursOptions,\n): number {\n  const diff =\n    differenceInMilliseconds(dateLeft, dateRight) / millisecondsInHour;\n  return getRoundingMethod(options?.roundingMethod)(diff);\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "getResultByTense", "language": "typescript", "path": "src/locale/kn/_lib/formatDistance/index.ts", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to select the appropriate string representation of time intervals based on tense and optional formatting conditions.\n2. **Input**: It accepts a structure containing tense-specific string templates and an optional object that specifies formatting preferences, such as whether to add a suffix indicating past or future tense, and the nature of the temporal comparison.\n3. **Output**: It returns a string that represents a time interval in either its default form, or modified to indicate past or future context based on the provided options.\n4. **Procedure**: The function first checks if a suffix should be added based on the provided options. If so, it determines whether the time interval should be represented as occurring in the future or the past by examining the comparison value. It then returns the appropriate string based on this evaluation. If no suffix is to be added, it simply returns the default string representation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/locale/nl/_lib/formatDistance/index.ts\nimport type { FormatDistanceFn, FormatDistanceLocale } from \"../../../types.js\";\n\ntype FormatDistanceTokenValue =\n  | string\n  | {\n      one: string;\n      other: string;\n    };\n\nconst formatDistanceLocale: FormatDistanceLocale<FormatDistanceTokenValue> = {\n  lessThanXSeconds: {\n    one: \"minder dan een seconde\",\n    other: \"minder dan {{count}} seconden\",\n  },\n\n  xSeconds: {\n    one: \"1 seconde\",\n    other: \"{{count}} seconden\",\n  },\n\n  halfAMinute: \"een halve minuut\",\n\n  lessThanXMinutes: {\n    one: \"minder dan een minuut\",\n    other: \"minder dan {{count}} minuten\",\n  },\n\n  xMinutes: {\n    one: \"een minuut\",\n    other: \"{{count}} minuten\",\n  },\n\n  aboutXHours: {\n    one: \"ongeveer 1 uur\",\n    other: \"ongeveer {{count}} uur\",\n  },\n\n  xHours: {\n    one: \"1 uur\",\n    other: \"{{count}} uur\",\n  },\n\n  xDays: {\n    one: \"1 dag\",\n    other: \"{{count}} dagen\",\n  },\n\n  aboutXWeeks: {\n    one: \"ongeveer 1 week\",\n    other: \"ongeveer {{count}} weken\",\n  },\n\n  xWeeks: {\n    one: \"1 week\",\n    other: \"{{count}} weken\",\n  },\n\n  aboutXMonths: {\n    one: \"ongeveer 1 maand\",\n    other: \"ongeveer {{count}} maanden\",\n  },\n\n  xMonths: {\n    one: \"1 maand\",\n    other: \"{{count}} maanden\",\n  },\n\n  aboutXYears: {\n    one: \"ongeveer 1 jaar\",\n    other: \"ongeveer {{count}} jaar\",\n  },\n\n  xYears: {\n    one: \"1 jaar\",\n    other: \"{{count}} jaar\",\n  },\n\n  overXYears: {\n    one: \"meer dan 1 jaar\",\n    other: \"meer dan {{count}} jaar\",\n  },\n\n  almostXYears: {\n    one: \"bijna 1 jaar\",\n    other: \"bijna {{count}} jaar\",\n  },\n};\n\nexport const formatDistance: FormatDistanceFn = (token, count, options) => {\n  let result;\n\n  const tokenValue = formatDistanceLocale[token];\n  if (typeof tokenValue === \"string\") {\n    result = tokenValue;\n  } else if (count === 1) {\n...\n// Path: src/locale/nl/_lib/formatLong/index.ts\nimport type { FormatLong } from \"../../../types.js\";\nimport { buildFormatLongFn } from \"../../../_lib/buildFormatLongFn/index.js\";\n\nconst dateFormats = {\n  full: \"EEEE d MMMM y\",\n  long: \"d MMMM y\",\n  medium: \"d MMM y\",\n  short: \"dd-MM-y\",\n};\n\nconst timeFormats = {\n  full: \"HH:mm:ss zzzz\",\n  long: \"HH:mm:ss z\",\n  medium: \"HH:mm:ss\",\n  short: \"HH:mm\",\n};\n\nconst dateTimeFormats = {\n  full: \"{{date}} 'om' {{time}}\",\n  long: \"{{date}} 'om' {{time}}\",\n  medium: \"{{date}}, {{time}}\",\n  short: \"{{date}}, {{time}}\",\n};\n\nexport const formatLong: FormatLong = {\n  date: buildFormatLongFn({\n    formats: dateFormats,\n    defaultWidth: \"full\",\n  }),\n\n  time: buildFormatLongFn({\n    formats: timeFormats,\n    defaultWidth: \"full\",\n  }),\n\n  dateTime: buildFormatLongFn({\n    formats: dateTimeFormats,\n    defaultWidth: \"full\",\n  }),\n};\n\n// Path: src/locale/nl/index.ts\nimport type { Locale } from \"../types.js\";\nimport { formatDistance } from \"./_lib/formatDistance/index.js\";\nimport { formatLong } from \"./_lib/formatLong/index.js\";\nimport { formatRelative } from \"./_lib/formatRelative/index.js\";\nimport { localize } from \"./_lib/localize/index.js\";\nimport { match } from \"./_lib/match/index.js\";\n\n/**\n * @category Locales\n * @summary Dutch locale.\n * @language Dutch\n * @iso-639-2 nld\n * @author Jorik Tangelder [@jtangelder](https://github.com/jtangelder)\n * @author Ruben Stolk [@rubenstolk](https://github.com/rubenstolk)\n * @author Lode Vanhove [@bitcrumb](https://github.com/bitcrumb)\n * @author Edo Rivai [@edorivai](https://github.com/edorivai)\n * @author Niels Keurentjes [@curry684](https://github.com/curry684)\n * @author Stefan Vermaas [@stefanvermaas](https://github.com/stefanvermaas)\n */\nexport const nl: Locale = {\n  code: \"nl\",\n  formatDistance: formatDistance,\n  formatLong: formatLong,\n  formatRelative: formatRelative,\n  localize: localize,\n  match: match,\n  options: {\n    weekStartsOn: 1 /* Monday */,\n    firstWeekContainsDate: 4,\n  },\n};\n\n// Path: src/locale/fr/_lib/formatRelative/index.ts\nimport type { FormatRelativeFn } from \"../../../types.js\";\n\nconst formatRelativeLocale = {\n  lastWeek: \"eeee 'dernier \u00e0' p\",\n  yesterday: \"'hier \u00e0' p\",\n  today: \"'aujourd\u2019hui \u00e0' p\",\n  tomorrow: \"'demain \u00e0' p'\",\n  nextWeek: \"eeee 'prochain \u00e0' p\",\n  other: \"P\",\n};\n\nexport const formatRelative: FormatRelativeFn = (\n  token,\n  _date,\n  _baseDate,\n  _options,\n) => formatRelativeLocale[token];\n\n// Path: src/locale/fr/_lib/formatLong/index.ts\nimport type { FormatLong } from \"../../../types.js\";\nimport { buildFormatLongFn } from \"../../../_lib/buildFormatLongFn/index.js\";\n\nconst dateFormats = {\n  full: \"EEEE d MMMM y\",\n  long: \"d MMMM y\",\n  medium: \"d MMM y\",\n  short: \"dd/MM/y\",\n};\n\nconst timeFormats = {\n  full: \"HH:mm:ss zzzz\",\n  long: \"HH:mm:ss z\",\n  medium: \"HH:mm:ss\",\n  short: \"HH:mm\",\n};\n\nconst dateTimeFormats = {\n  full: \"{{date}} '\u00e0' {{time}}\",\n  long: \"{{date}} '\u00e0' {{time}}\",\n  medium: \"{{date}}, {{time}}\",\n  short: \"{{date}}, {{time}}\",\n};\n\nexport const formatLong: FormatLong = {\n  date: buildFormatLongFn({\n    formats: dateFormats,\n    defaultWidth: \"full\",\n  }),\n\n  time: buildFormatLongFn({\n    formats: timeFormats,\n    defaultWidth: \"full\",\n  }),\n\n  dateTime: buildFormatLongFn({\n    formats: dateTimeFormats,\n    defaultWidth: \"full\",\n  }),\n};\n\n// Path: src/locale/fr/index.ts\nimport type { Locale } from \"../types.js\";\nimport { formatDistance } from \"./_lib/formatDistance/index.js\";\nimport { formatLong } from \"./_lib/formatLong/index.js\";\nimport { formatRelative } from \"./_lib/formatRelative/index.js\";\nimport { localize } from \"./_lib/localize/index.js\";\nimport { match } from \"./_lib/match/index.js\";\n\n/**\n * @category Locales\n * @summary French locale.\n * @language French\n * @iso-639-2 fra\n * @author Jean Dupouy [@izeau](https://github.com/izeau)\n * @author Fran\u00e7ois B [@fbonzon](https://github.com/fbonzon)\n */\nexport const fr: Locale = {\n  code: \"fr\",\n  formatDistance: formatDistance,\n  formatLong: formatLong,\n  formatRelative: formatRelative,\n  localize: localize,\n  match: match,\n  options: {\n    weekStartsOn: 1 /* Monday */,\n    firstWeekContainsDate: 4,\n  },\n};\n\n// Path: src/locale/kn/_lib/localize/index.ts\nimport type { Localize, LocalizeFn } from \"../../../types.js\";\nimport { buildLocalizeFn } from \"../../../_lib/buildLocalizeFn/index.js\";\n\n// Reference: https://www.unicode.org/cldr/charts/32/summary/kn.html\n\nconst eraValues = {\n  narrow: [\"\u0c95\u0ccd\u0cb0\u0cbf.\u0caa\u0cc2\", \"\u0c95\u0ccd\u0cb0\u0cbf.\u0cb6\"] as const,\n  abbreviated: [\"\u0c95\u0ccd\u0cb0\u0cbf.\u0caa\u0cc2\", \"\u0c95\u0ccd\u0cb0\u0cbf.\u0cb6\"] as const, // CLDR #1618, #1620\n  wide: [\"\u0c95\u0ccd\u0cb0\u0cbf\u0cb8\u0ccd\u0ca4 \u0caa\u0cc2\u0cb0\u0ccd\u0cb5\", \"\u0c95\u0ccd\u0cb0\u0cbf\u0cb8\u0ccd\u0ca4 \u0cb6\u0c95\"] as const, // CLDR #1614, #1616\n};\n\nconst quarterValues = {\n  narrow: [\"1\", \"2\", \"3\", \"4\"] as const,\n  abbreviated: [\"\u0ca4\u0ccd\u0cb0\u0cc8 1\", \"\u0ca4\u0ccd\u0cb0\u0cc8 2\", \"\u0ca4\u0ccd\u0cb0\u0cc8 3\", \"\u0ca4\u0ccd\u0cb0\u0cc8 4\"] as const, // CLDR #1630 - #1638\n  wide: [\n    \"1\u0ca8\u0cc7 \u0ca4\u0ccd\u0cb0\u0cc8\u0cae\u0cbe\u0cb8\u0cbf\u0c95\",\n    \"2\u0ca8\u0cc7 \u0ca4\u0ccd\u0cb0\u0cc8\u0cae\u0cbe\u0cb8\u0cbf\u0c95\",\n    \"3\u0ca8\u0cc7 \u0ca4\u0ccd\u0cb0\u0cc8\u0cae\u0cbe\u0cb8\u0cbf\u0c95\",\n    \"4\u0ca8\u0cc7 \u0ca4\u0ccd\u0cb0\u0cc8\u0cae\u0cbe\u0cb8\u0cbf\u0c95\",\n  ] as const, // CLDR #1622 - #1629\n};\n\n// CLDR #1646 - #1717\nconst monthValues = {\n  narrow: [\n    \"\u0c9c\",\n    \"\u0cab\u0cc6\",\n    \"\u0cae\u0cbe\",\n    \"\u0c8f\",\n    \"\u0cae\u0cc7\",\n    \"\u0c9c\u0cc2\",\n    \"\u0c9c\u0cc1\",\n    \"\u0c86\",\n    \"\u0cb8\u0cc6\",\n    \"\u0c85\",\n    \"\u0ca8\",\n    \"\u0ca1\u0cbf\",\n  ] as const,\n  abbreviated: [\n    \"\u0c9c\u0ca8\",\n    \"\u0cab\u0cc6\u0cac\u0ccd\u0cb0\",\n    \"\u0cae\u0cbe\u0cb0\u0ccd\u0c9a\u0ccd\",\n    \"\u0c8f\u0caa\u0ccd\u0cb0\u0cbf\",\n    \"\u0cae\u0cc7\",\n    \"\u0c9c\u0cc2\u0ca8\u0ccd\",\n    \"\u0c9c\u0cc1\u0cb2\u0cc8\",\n    \"\u0c86\u0c97\",\n    \"\u0cb8\u0cc6\u0caa\u0ccd\u0c9f\u0cc6\u0c82\",\n    \"\u0c85\u0c95\u0ccd\u0c9f\u0ccb\",\n    \"\u0ca8\u0cb5\u0cc6\u0c82\",\n    \"\u0ca1\u0cbf\u0cb8\u0cc6\u0c82\",\n  ] as const,\n  wide: [\n    \"\u0c9c\u0ca8\u0cb5\u0cb0\u0cbf\",\n    \"\u0cab\u0cc6\u0cac\u0ccd\u0cb0\u0cb5\u0cb0\u0cbf\",\n    \"\u0cae\u0cbe\u0cb0\u0ccd\u0c9a\u0ccd\",\n    \"\u0c8f\u0caa\u0ccd\u0cb0\u0cbf\u0cb2\u0ccd\",\n    \"\u0cae\u0cc7\",\n    \"\u0c9c\u0cc2\u0ca8\u0ccd\",\n    \"\u0c9c\u0cc1\u0cb2\u0cc8\",\n    \"\u0c86\u0c97\u0cb8\u0ccd\u0c9f\u0ccd\",\n    \"\u0cb8\u0cc6\u0caa\u0ccd\u0c9f\u0cc6\u0c82\u0cac\u0cb0\u0ccd\",\n    \"\u0c85\u0c95\u0ccd\u0c9f\u0ccb\u0cac\u0cb0\u0ccd\",\n    \"\u0ca8\u0cb5\u0cc6\u0c82\u0cac\u0cb0\u0ccd\",\n    \"\u0ca1\u0cbf\u0cb8\u0cc6\u0c82\u0cac\u0cb0\u0ccd\",\n  ] as const,\n};\n\n// CLDR #1718 - #1773\nconst dayValues = {\n  narrow: [\"\u0cad\u0cbe\", \"\u0cb8\u0ccb\", \"\u0cae\u0c82\", \"\u0cac\u0cc1\", \"\u0c97\u0cc1\", \"\u0cb6\u0cc1\", \"\u0cb6\"] as const,\n  short: [\"\u0cad\u0cbe\u0ca8\u0cc1\", \"\u0cb8\u0ccb\u0cae\", \"\u0cae\u0c82\u0c97\u0cb3\", \"\u0cac\u0cc1\u0ca7\", \"\u0c97\u0cc1\u0cb0\u0cc1\", \"\u0cb6\u0cc1\u0c95\u0ccd\u0cb0\", \"\u0cb6\u0ca8\u0cbf\"] as const,\n  abbreviated: [\"\u0cad\u0cbe\u0ca8\u0cc1\", \"\u0cb8\u0ccb\u0cae\", \"\u0cae\u0c82\u0c97\u0cb3\", \"\u0cac\u0cc1\u0ca7\", \"\u0c97\u0cc1\u0cb0\u0cc1\", \"\u0cb6\u0cc1\u0c95\u0ccd\u0cb0\", \"\u0cb6\u0ca8\u0cbf\"] as const,\n  wide: [\n    \"\u0cad\u0cbe\u0ca8\u0cc1\u0cb5\u0cbe\u0cb0\",\n    \"\u0cb8\u0ccb\u0cae\u0cb5\u0cbe\u0cb0\",\n    \"\u0cae\u0c82\u0c97\u0cb3\u0cb5\u0cbe\u0cb0\",\n    \"\u0cac\u0cc1\u0ca7\u0cb5\u0cbe\u0cb0\",\n    \"\u0c97\u0cc1\u0cb0\u0cc1\u0cb5\u0cbe\u0cb0\",\n    \"\u0cb6\u0cc1\u0c95\u0ccd\u0cb0\u0cb5\u0cbe\u0cb0\",\n    \"\u0cb6\u0ca8\u0cbf\u0cb5\u0cbe\u0cb0\",\n  ] as const,\n};\n\n// CLDR #1774 - #1815\nconst dayPeriodValues = {\n  narrow: {\n    am: \"\u0caa\u0cc2\u0cb0\u0ccd\u0cb5\u0cbe\u0cb9\u0ccd\u0ca8\",\n    pm: \"\u0c85\u0caa\u0cb0\u0cbe\u0cb9\u0ccd\u0ca8\",\n    midnight: \"\u0cae\u0ca7\u0ccd\u0caf\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n    noon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0cb9\u0ccd\u0ca8\",\n    morning: \"\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6\",\n    afternoon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0cb9\u0ccd\u0ca8\",\n    evening: \"\u0cb8\u0c82\u0c9c\u0cc6\",\n    night: \"\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n  },\n  abbreviated: {\n    am: \"\u0caa\u0cc2\u0cb0\u0ccd\u0cb5\u0cbe\u0cb9\u0ccd\u0ca8\",\n    pm: \"\u0c85\u0caa\u0cb0\u0cbe\u0cb9\u0ccd\u0ca8\",\n    midnight: \"\u0cae\u0ca7\u0ccd\u0caf\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n    noon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    morning: \"\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6\",\n    afternoon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    evening: \"\u0cb8\u0c82\u0c9c\u0cc6\",\n    night: \"\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n  },\n  wide: {\n    am: \"\u0caa\u0cc2\u0cb0\u0ccd\u0cb5\u0cbe\u0cb9\u0ccd\u0ca8\",\n    pm: \"\u0c85\u0caa\u0cb0\u0cbe\u0cb9\u0ccd\u0ca8\",\n    midnight: \"\u0cae\u0ca7\u0ccd\u0caf\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n    noon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    morning: \"\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6\",\n    afternoon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    evening: \"\u0cb8\u0c82\u0c9c\u0cc6\",\n    night: \"\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n  },\n};\n\nconst formattingDayPeriodValues = {\n  narrow: {\n    am: \"\u0caa\u0cc2\",\n    pm: \"\u0c85\",\n    midnight: \"\u0cae\u0ca7\u0ccd\u0caf\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n    noon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    morning: \"\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6\",\n    afternoon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    evening: \"\u0cb8\u0c82\u0c9c\u0cc6\",\n    night: \"\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n  },\n  abbreviated: {\n    am: \"\u0caa\u0cc2\u0cb0\u0ccd\u0cb5\u0cbe\u0cb9\u0ccd\u0ca8\",\n    pm: \"\u0c85\u0caa\u0cb0\u0cbe\u0cb9\u0ccd\u0ca8\",\n    midnight: \"\u0cae\u0ca7\u0ccd\u0caf \u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n    noon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    morning: \"\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6\",\n    afternoon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    evening: \"\u0cb8\u0c82\u0c9c\u0cc6\",\n    night: \"\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n  },\n  wide: {\n    am: \"\u0caa\u0cc2\u0cb0\u0ccd\u0cb5\u0cbe\u0cb9\u0ccd\u0ca8\",\n    pm: \"\u0c85\u0caa\u0cb0\u0cbe\u0cb9\u0ccd\u0ca8\",\n    midnight: \"\u0cae\u0ca7\u0ccd\u0caf \u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n    noon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    morning: \"\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6\",\n    afternoon: \"\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9\",\n    evening: \"\u0cb8\u0c82\u0c9c\u0cc6\",\n    night: \"\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf\",\n  },\n};\n\nconst ordinalNumber: LocalizeFn<number> = (dirtyNumber, _options) => {\n  const number = Number(dirtyNumber);\n  return number + \"\u0ca8\u0cc7\";\n};\n\nexport const localize: Localize = {\n  ordinalNumber,\n\n  era: buildLocalizeFn({\n    values: eraValues,\n    defaultWidth: \"wide\",\n  }),\n\n  quarter: buildLocalizeFn({\n    values: quarterValues,\n    defaultWidth: \"wide\",\n    argumentCallback: (quarter) => quarter - 1,\n  }),\n\n  month: buildLocalizeFn({\n    values: monthValues,\n    defaultWidth: \"wide\",\n  }),\n\n  day: buildLocalizeFn({\n    values: dayValues,\n    defaultWidth: \"wide\",\n  }),\n\n  dayPeriod: buildLocalizeFn({\n    values: dayPeriodValues,\n    defaultWidth: \"wide\",\n    formattingValues: formattingDayPeriodValues,\n    defaultFormattingWidth: \"wide\",\n  }),\n};\n\n// Path: src/locale/kn/_lib/match/index.ts\nimport type { Quarter } from \"../../../../types.js\";\nimport type { Match } from \"../../../types.js\";\nimport { buildMatchFn } from \"../../../_lib/buildMatchFn/index.js\";\nimport { buildMatchPatternFn } from \"../../../_lib/buildMatchPatternFn/index.js\";\n\nconst matchOrdinalNumberPattern = /^(\\d+)(\u0ca8\u0cc7|\u0ca8\u0cc6)?/i;\nconst parseOrdinalNumberPattern = /\\d+/i;\n\nconst matchEraPatterns = {\n  narrow: /^(\u0c95\u0ccd\u0cb0\u0cbf.\u0caa\u0cc2|\u0c95\u0ccd\u0cb0\u0cbf.\u0cb6)/i,\n  abbreviated: /^(\u0c95\u0ccd\u0cb0\u0cbf\\.?\\s?\u0caa\u0cc2\\.?|\u0c95\u0ccd\u0cb0\u0cbf\\.?\\s?\u0cb6\\.?|\u0caa\u0ccd\u0cb0\\.?\\s?\u0cb6\\.?)/i,\n  wide: /^(\u0c95\u0ccd\u0cb0\u0cbf\u0cb8\u0ccd\u0ca4 \u0caa\u0cc2\u0cb0\u0ccd\u0cb5|\u0c95\u0ccd\u0cb0\u0cbf\u0cb8\u0ccd\u0ca4 \u0cb6\u0c95|\u0caa\u0ccd\u0cb0\u0cb8\u0c95\u0ccd\u0ca4 \u0cb6\u0c95)/i,\n};\nconst parseEraPatterns = {\n  any: [/^\u0caa\u0cc2/i, /^(\u0cb6|\u0caa\u0ccd\u0cb0)/i] as const,\n};\n\nconst matchQuarterPatterns = {\n  narrow: /^[1234]/i,\n  abbreviated: /^\u0ca4\u0ccd\u0cb0\u0cc8[1234]|\u0ca4\u0ccd\u0cb0\u0cc8 [1234]| [1234]\u0ca4\u0ccd\u0cb0\u0cc8/i,\n  wide: /^[1234](\u0ca8\u0cc7)? \u0ca4\u0ccd\u0cb0\u0cc8\u0cae\u0cbe\u0cb8\u0cbf\u0c95/i,\n};\nconst parseQuarterPatterns = {\n  any: [/1/i, /2/i, /3/i, /4/i] as const,\n};\n\nconst matchMonthPatterns = {\n  narrow: /^(\u0c9c\u0cc2|\u0c9c\u0cc1|\u0c9c|\u0cab\u0cc6|\u0cae\u0cbe|\u0c8f|\u0cae\u0cc7|\u0c86|\u0cb8\u0cc6|\u0c85|\u0ca8|\u0ca1\u0cbf)/i,\n  abbreviated:\n    /^(\u0c9c\u0ca8|\u0cab\u0cc6\u0cac\u0ccd\u0cb0|\u0cae\u0cbe\u0cb0\u0ccd\u0c9a\u0ccd|\u0c8f\u0caa\u0ccd\u0cb0\u0cbf|\u0cae\u0cc7|\u0c9c\u0cc2\u0ca8\u0ccd|\u0c9c\u0cc1\u0cb2\u0cc8|\u0c86\u0c97|\u0cb8\u0cc6\u0caa\u0ccd\u0c9f\u0cc6\u0c82|\u0c85\u0c95\u0ccd\u0c9f\u0ccb|\u0ca8\u0cb5\u0cc6\u0c82|\u0ca1\u0cbf\u0cb8\u0cc6\u0c82)/i,\n  wide: /^(\u0c9c\u0ca8\u0cb5\u0cb0\u0cbf|\u0cab\u0cc6\u0cac\u0ccd\u0cb0\u0cb5\u0cb0\u0cbf|\u0cae\u0cbe\u0cb0\u0ccd\u0c9a\u0ccd|\u0c8f\u0caa\u0ccd\u0cb0\u0cbf\u0cb2\u0ccd|\u0cae\u0cc7|\u0c9c\u0cc2\u0ca8\u0ccd|\u0c9c\u0cc1\u0cb2\u0cc8|\u0c86\u0c97\u0cb8\u0ccd\u0c9f\u0ccd|\u0cb8\u0cc6\u0caa\u0ccd\u0c9f\u0cc6\u0c82\u0cac\u0cb0\u0ccd|\u0c85\u0c95\u0ccd\u0c9f\u0ccb\u0cac\u0cb0\u0ccd|\u0ca8\u0cb5\u0cc6\u0c82\u0cac\u0cb0\u0ccd|\u0ca1\u0cbf\u0cb8\u0cc6\u0c82\u0cac\u0cb0\u0ccd)/i,\n};\nconst parseMonthPatterns = {\n  narrow: [\n    /^\u0c9c$/i,\n    /^\u0cab\u0cc6/i,\n    /^\u0cae\u0cbe/i,\n    /^\u0c8f/i,\n    /^\u0cae\u0cc7/i,\n    /^\u0c9c\u0cc2/i,\n    /^\u0c9c\u0cc1$/i,\n    /^\u0c86/i,\n    /^\u0cb8\u0cc6/i,\n    /^\u0c85/i,\n    /^\u0ca8/i,\n    /^\u0ca1\u0cbf/i,\n  ] as const,\n  any: [\n    /^\u0c9c\u0ca8/i,\n    /^\u0cab\u0cc6/i,\n    /^\u0cae\u0cbe/i,\n    /^\u0c8f/i,\n    /^\u0cae\u0cc7/i,\n    /^\u0c9c\u0cc2\u0ca8\u0ccd/i,\n    /^\u0c9c\u0cc1\u0cb2\u0cc8/i,\n    /^\u0c86/i,\n    /^\u0cb8\u0cc6/i,\n    /^\u0c85/i,\n    /^\u0ca8/i,\n    /^\u0ca1\u0cbf/i,\n  ] as const,\n};\n\nconst matchDayPatterns = {\n  narrow: /^(\u0cad\u0cbe|\u0cb8\u0ccb|\u0cae|\u0cac\u0cc1|\u0c97\u0cc1|\u0cb6\u0cc1|\u0cb6)/i,\n  short: /^(\u0cad\u0cbe\u0ca8\u0cc1|\u0cb8\u0ccb\u0cae|\u0cae\u0c82\u0c97\u0cb3|\u0cac\u0cc1\u0ca7|\u0c97\u0cc1\u0cb0\u0cc1|\u0cb6\u0cc1\u0c95\u0ccd\u0cb0|\u0cb6\u0ca8\u0cbf)/i,\n  abbreviated: /^(\u0cad\u0cbe\u0ca8\u0cc1|\u0cb8\u0ccb\u0cae|\u0cae\u0c82\u0c97\u0cb3|\u0cac\u0cc1\u0ca7|\u0c97\u0cc1\u0cb0\u0cc1|\u0cb6\u0cc1\u0c95\u0ccd\u0cb0|\u0cb6\u0ca8\u0cbf)/i,\n  wide: /^(\u0cad\u0cbe\u0ca8\u0cc1\u0cb5\u0cbe\u0cb0|\u0cb8\u0ccb\u0cae\u0cb5\u0cbe\u0cb0|\u0cae\u0c82\u0c97\u0cb3\u0cb5\u0cbe\u0cb0|\u0cac\u0cc1\u0ca7\u0cb5\u0cbe\u0cb0|\u0c97\u0cc1\u0cb0\u0cc1\u0cb5\u0cbe\u0cb0|\u0cb6\u0cc1\u0c95\u0ccd\u0cb0\u0cb5\u0cbe\u0cb0|\u0cb6\u0ca8\u0cbf\u0cb5\u0cbe\u0cb0)/i,\n};\nconst parseDayPatterns = {\n  narrow: [/^\u0cad\u0cbe/i, /^\u0cb8\u0ccb/i, /^\u0cae/i, /^\u0cac\u0cc1/i, /^\u0c97\u0cc1/i, /^\u0cb6\u0cc1/i, /^\u0cb6/i] as const,\n  any: [/^\u0cad\u0cbe/i, /^\u0cb8\u0ccb/i, /^\u0cae/i, /^\u0cac\u0cc1/i, /^\u0c97\u0cc1/i, /^\u0cb6\u0cc1/i, /^\u0cb6/i] as const,\n};\n\nconst matchDayPeriodPatterns = {\n  narrow: /^(\u0caa\u0cc2|\u0c85|\u0cae\u0ca7\u0ccd\u0caf\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf|\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9|\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6|\u0cb8\u0c82\u0c9c\u0cc6|\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf)/i,\n  any: /^(\u0caa\u0cc2\u0cb0\u0ccd\u0cb5\u0cbe\u0cb9\u0ccd\u0ca8|\u0c85\u0caa\u0cb0\u0cbe\u0cb9\u0ccd\u0ca8|\u0cae\u0ca7\u0ccd\u0caf\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf|\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9|\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6|\u0cb8\u0c82\u0c9c\u0cc6|\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf)/i,\n};\nconst parseDayPeriodPatterns = {\n  any: {\n    am: /^\u0caa\u0cc2/i,\n    pm: /^\u0c85/i,\n    midnight: /\u0cae\u0ca7\u0ccd\u0caf\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf/i,\n    noon: /\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9/i,\n    morning: /\u0cac\u0cc6\u0cb3\u0c97\u0ccd\u0c97\u0cc6/i,\n    afternoon: /\u0cae\u0ca7\u0ccd\u0caf\u0cbe\u0ca8\u0ccd\u0cb9/i,\n    evening: /\u0cb8\u0c82\u0c9c\u0cc6/i,\n    night: /\u0cb0\u0cbe\u0ca4\u0ccd\u0cb0\u0cbf/i,\n  },\n};\n\nexport const match: Match = {\n  ordinalNumber: buildMatchPatternFn({\n    matchPattern: matchOrdinalNumberPattern,\n    parsePattern: parseOrdinalNumberPattern,\n    valueCallback: (value) => parseInt(value, 10),\n  }),\n\n  era: buildMatchFn({\n    matchPatterns: matchEraPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseEraPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  quarter: buildMatchFn({\n    matchPatterns: matchQuarterPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseQuarterPatterns,\n    defaultParseWidth: \"any\",\n    valueCallback: (index) => (index + 1) as Quarter,\n  }),\n\n  month: buildMatchFn({\n    matchPatterns: matchMonthPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseMonthPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  day: buildMatchFn({\n    matchPatterns: matchDayPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseDayPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  dayPeriod: buildMatchFn({\n    matchPatterns: matchDayPeriodPatterns,\n    defaultMatchWidth: \"any\",\n    parsePatterns: parseDayPeriodPatterns,\n    defaultParseWidth: \"any\",\n  }),\n};\n\n// Path: src/locale/kn/_lib/formatDistance/index.ts\nimport type {\n  FormatDistanceFn,\n  FormatDistanceLocale,\n  FormatDistanceFnOptions,\n} from \"../../../types.js\";\n\ntype Tense = {\n  default: string;\n  future: string;\n  past: string;\n};\n\ntype FormatDistanceTokenValue = {\n  one?: Tense;\n  other: Tense;\n};\n\n// note: no implementation for weeks\ntype FormatDistanceLocaleNoWeeks = Omit<\n  FormatDistanceLocale<FormatDistanceTokenValue>,\n  \"aboutXWeeks\" | \"xWeeks\"\n>;\n\nconst formatDistanceLocale: FormatDistanceLocaleNoWeeks = {\n  lessThanXSeconds: {\n    one: {\n      default: \"1 \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0c97\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      future: \"1 \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0c97\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      past: \"1 \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0c97\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0c97\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      future: \"{{count}} \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0c97\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      past: \"{{count}} \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0c97\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n    },\n  },\n\n  xSeconds: {\n    one: {\n      default: \"1 \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\",\n      future: \"1 \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0ca8\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"1 \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0cc1\u0c97\u0cb3\u0cc1\",\n      future: \"{{count}} \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd\u200c\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"{{count}} \u0cb8\u0cc6\u0c95\u0cc6\u0c82\u0ca1\u0ccd \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  halfAMinute: {\n    other: {\n      default: \"\u0c85\u0cb0\u0ccd\u0ca7 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\",\n      future: \"\u0c85\u0cb0\u0ccd\u0ca7 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0c85\u0cb0\u0ccd\u0ca7 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0ca6 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  lessThanXMinutes: {\n    one: {\n      default: \"1 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c95\u0ccd\u0c95\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      future: \"1 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c95\u0ccd\u0c95\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      past: \"1 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c95\u0ccd\u0c95\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c95\u0ccd\u0c95\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      future: \"{{count}} \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c95\u0ccd\u0c95\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n      past: \"{{count}} \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c95\u0ccd\u0c95\u0cbf\u0c82\u0ca4 \u0c95\u0ca1\u0cbf\u0cae\u0cc6\",\n    },\n  },\n\n  xMinutes: {\n    one: {\n      default: \"1 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\",\n      future: \"1 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"1 \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0ca6 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c97\u0cb3\u0cc1\",\n      future: \"{{count}} \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"{{count}} \u0ca8\u0cbf\u0cae\u0cbf\u0cb7\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  aboutXHours: {\n    one: {\n      default: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0c97\u0c82\u0c9f\u0cc6\",\n      future: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0c97\u0c82\u0c9f\u0cc6\u0caf\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0c97\u0c82\u0c9f\u0cc6 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0c97\u0c82\u0c9f\u0cc6\u0c97\u0cb3\u0cc1\",\n      future: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0c97\u0c82\u0c9f\u0cc6\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0c97\u0c82\u0c9f\u0cc6\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  xHours: {\n    one: {\n      default: \"1 \u0c97\u0c82\u0c9f\u0cc6\",\n      future: \"1 \u0c97\u0c82\u0c9f\u0cc6\u0caf\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"1 \u0c97\u0c82\u0c9f\u0cc6 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0c97\u0c82\u0c9f\u0cc6\u0c97\u0cb3\u0cc1\",\n      future: \"{{count}} \u0c97\u0c82\u0c9f\u0cc6\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"{{count}} \u0c97\u0c82\u0c9f\u0cc6\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  xDays: {\n    one: {\n      default: \"1 \u0ca6\u0cbf\u0ca8\",\n      future: \"1 \u0ca6\u0cbf\u0ca8\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"1 \u0ca6\u0cbf\u0ca8\u0ca6 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0ca6\u0cbf\u0ca8\u0c97\u0cb3\u0cc1\",\n      future: \"{{count}} \u0ca6\u0cbf\u0ca8\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"{{count}} \u0ca6\u0cbf\u0ca8\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  // TODO\n  // aboutXWeeks: {},\n\n  // TODO\n  // xWeeks: {},\n\n  aboutXMonths: {\n    one: {\n      default: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\",\n      future: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0ca4\u0cbf\u0c82\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\",\n      future: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  xMonths: {\n    one: {\n      default: \"1 \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\",\n      future: \"1 \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"1 \u0ca4\u0cbf\u0c82\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\",\n      future: \"{{count}} \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"{{count}} \u0ca4\u0cbf\u0c82\u0c97\u0cb3\u0cc1\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  aboutXYears: {\n    one: {\n      default: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0cb5\u0cb0\u0ccd\u0cb7\",\n      future: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3\u0cc1\",\n      future: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cb8\u0cc1\u0cae\u0cbe\u0cb0\u0cc1 {{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  xYears: {\n    one: {\n      default: \"1 \u0cb5\u0cb0\u0ccd\u0cb7\",\n      future: \"1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3\u0cc1\",\n      future: \"{{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"{{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3 \u0cb9\u0cbf\u0c82\u0ca6\u0cc6\",\n    },\n  },\n\n  overXYears: {\n    one: {\n      default: \"1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6 \u0cae\u0cc7\u0cb2\u0cc6\",\n      future: \"1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6 \u0cae\u0cc7\u0cb2\u0cc6\",\n      past: \"1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6 \u0cae\u0cc7\u0cb2\u0cc6\",\n    },\n    other: {\n      default: \"{{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3 \u0cae\u0cc7\u0cb2\u0cc6\",\n      future: \"{{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3 \u0cae\u0cc7\u0cb2\u0cc6\",\n      past: \"{{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3 \u0cae\u0cc7\u0cb2\u0cc6\",\n    },\n  },\n\n  almostXYears: {\n    one: {\n      default: \"\u0cac\u0cb9\u0cc1\u0ca4\u0cc7\u0c95 1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n      future: \"\u0cac\u0cb9\u0cc1\u0ca4\u0cc7\u0c95 1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cac\u0cb9\u0cc1\u0ca4\u0cc7\u0c95 1 \u0cb5\u0cb0\u0ccd\u0cb7\u0ca6\u0cb2\u0ccd\u0cb2\u0cbf\",\n    },\n    other: {\n      default: \"\u0cac\u0cb9\u0cc1\u0ca4\u0cc7\u0c95 {{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      future: \"\u0cac\u0cb9\u0cc1\u0ca4\u0cc7\u0c95 {{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n      past: \"\u0cac\u0cb9\u0cc1\u0ca4\u0cc7\u0c95 {{count}} \u0cb5\u0cb0\u0ccd\u0cb7\u0c97\u0cb3\u0cb2\u0ccd\u0cb2\u0cbf\",\n    },\n  },\n};\n\nfunction getResultByTense(\n  parentToken: Tense,\n  options?: FormatDistanceFnOptions,\n): string {\n  if (options?.addSuffix) {\n    if (options.comparison && options.comparison > 0) {\n      return parentToken.future;\n    } else {\n      return parentToken.past;\n    }\n  }\n  return parentToken.default;\n}\n\nexport const formatDistance: FormatDistanceFn = (token, count, options) => {\n  let result;\n\n  const tokenValue =\n    formatDistanceLocale[token as keyof typeof formatDistanceLocale];\n\n  if (tokenValue.one && count === 1) {\n    result = getResultByTense(tokenValue.one, options);\n  } else {\n    result = getResultByTense(tokenValue.other, options);\n  }\n\n  return result.replace(\"{{count}}\", String(count));\n};\n\n\n// Path: src/locale/kn/_lib/formatRelative/index.ts\nimport type { FormatRelativeFn } from \"../../../types.js\";\n\nconst formatRelativeLocale = {\n  lastWeek: \"'\u0c95\u0cb3\u0cc6\u0ca6' eeee p '\u0c95\u0ccd\u0c95\u0cc6'\",\n  yesterday: \"'\u0ca8\u0cbf\u0ca8\u0ccd\u0ca8\u0cc6' p '\u0c95\u0ccd\u0c95\u0cc6'\",\n  today: \"'\u0c87\u0c82\u0ca6\u0cc1' p '\u0c95\u0ccd\u0c95\u0cc6'\",\n  tomorrow: \"'\u0ca8\u0cbe\u0cb3\u0cc6' p '\u0c95\u0ccd\u0c95\u0cc6'\",\n  nextWeek: \"eeee p '\u0c95\u0ccd\u0c95\u0cc6'\",\n  other: \"P\",\n};\n\nexport const formatRelative: FormatRelativeFn = (\n  token,\n  _date,\n  _baseDate,\n  _options,\n) => formatRelativeLocale[token];\n\n// Path: src/locale/kn/_lib/formatLong/index.ts\nimport type { FormatLong } from \"../../../types.js\";\nimport { buildFormatLongFn } from \"../../../_lib/buildFormatLongFn/index.js\";\n\n// Reference: https://www.unicode.org/cldr/charts/32/summary/kn.html\n\nconst dateFormats = {\n  full: \"EEEE, MMMM d, y\", // CLDR 1816\n  long: \"MMMM d, y\", // CLDR 1817\n  medium: \"MMM d, y\", // CLDR 1818\n  short: \"d/M/yy\", // CLDR 1819\n};\n\nconst timeFormats = {\n  full: \"hh:mm:ss a zzzz\", // CLDR 1820\n  long: \"hh:mm:ss a z\", // CLDR 1821\n  medium: \"hh:mm:ss a\", // CLDR 1822\n  short: \"hh:mm a\", // CLDR 1823\n};\n\nconst dateTimeFormats = {\n  full: \"{{date}} {{time}}\", // CLDR 1824\n  long: \"{{date}} {{time}}\", // CLDR 1825\n  medium: \"{{date}} {{time}}\", // CLDR 1826\n  short: \"{{date}} {{time}}\", // CLDR 1827\n};\n\nexport const formatLong: FormatLong = {\n  date: buildFormatLongFn({\n    formats: dateFormats,\n    defaultWidth: \"full\",\n  }),\n\n  time: buildFormatLongFn({\n    formats: timeFormats,\n    defaultWidth: \"full\",\n  }),\n\n  dateTime: buildFormatLongFn({\n    formats: dateTimeFormats,\n    defaultWidth: \"full\",\n  }),\n};\n\n// Path: src/locale/kn/index.ts\nimport type { Locale } from \"../types.js\";\nimport { formatDistance } from \"./_lib/formatDistance/index.js\";\nimport { formatLong } from \"./_lib/formatLong/index.js\";\nimport { formatRelative } from \"./_lib/formatRelative/index.js\";\nimport { localize } from \"./_lib/localize/index.js\";\nimport { match } from \"./_lib/match/index.js\";\n\n/**\n * @category Locales\n * @summary Kannada locale (India).\n * @language Kannada\n * @iso-639-2 kan\n * @author Manjunatha Gouli [@developergouli](https://github.com/developergouli)\n */\nexport const kn: Locale = {\n  code: \"kn\",\n  formatDistance: formatDistance,\n  formatLong: formatLong,\n  formatRelative: formatRelative,\n  localize: localize,\n  match: match,\n  options: {\n    weekStartsOn: 1 /* Monday */,\n    firstWeekContainsDate: 1,\n  },\n};\n\n// Path: src/locale/da/_lib/formatRelative/index.ts\nimport type { FormatRelativeFn } from \"../../../types.js\";\n\nconst formatRelativeLocale = {\n  lastWeek: \"'sidste' eeee 'kl.' p\",\n  yesterday: \"'i g\u00e5r kl.' p\",\n  today: \"'i dag kl.' p\",\n  tomorrow: \"'i morgen kl.' p\",\n  nextWeek: \"'p\u00e5' eeee 'kl.' p\",\n  other: \"P\",\n};\n\nexport const formatRelative: FormatRelativeFn = (\n  token,\n  _date,\n  _baseDate,\n  _options,\n) => formatRelativeLocale[token];\n\n// Path: src/locale/da/_lib/localize/index.ts\nimport type { Localize, LocalizeFn } from \"../../../types.js\";\nimport { buildLocalizeFn } from \"../../../_lib/buildLocalizeFn/index.js\";\n\nconst eraValues = {\n  narrow: [\"fvt\", \"vt\"] as const,\n  abbreviated: [\"f.v.t.\", \"v.t.\"] as const,\n  wide: [\"f\u00f8r vesterlandsk tidsregning\", \"vesterlandsk tidsregning\"] as const,\n};\n\nconst quarterValues = {\n  narrow: [\"1\", \"2\", \"3\", \"4\"] as const,\n  abbreviated: [\"1. kvt.\", \"2. kvt.\", \"3. kvt.\", \"4. kvt.\"] as const,\n  wide: [\"1. kvartal\", \"2. kvartal\", \"3. kvartal\", \"4. kvartal\"] as const,\n};\n\nconst monthValues = {\n  narrow: [\"J\", \"F\", \"M\", \"A\", \"M\", \"J\", \"J\", \"A\", \"S\", \"O\", \"N\", \"D\"] as const,\n  abbreviated: [\n    \"jan.\",\n    \"feb.\",\n    \"mar.\",\n    \"apr.\",\n    \"maj\",\n    \"jun.\",\n    \"jul.\",\n    \"aug.\",\n    \"sep.\",\n    \"okt.\",\n    \"nov.\",\n    \"dec.\",\n  ] as const,\n  wide: [\n    \"januar\",\n    \"februar\",\n    \"marts\",\n    \"april\",\n    \"maj\",\n    \"juni\",\n    \"juli\",\n    \"august\",\n    \"september\",\n    \"oktober\",\n    \"november\",\n    \"december\",\n  ] as const,\n};\n\n// Note that 'Days - abbreviated - Formatting' has periods at the end.\n// https://www.unicode.org/cldr/charts/32/summary/da.html#1760\n// This makes grammatical sense in danish, as most abbreviations have periods.\nconst dayValues = {\n  narrow: [\"S\", \"M\", \"T\", \"O\", \"T\", \"F\", \"L\"] as const,\n  short: [\"s\u00f8\", \"ma\", \"ti\", \"on\", \"to\", \"fr\", \"l\u00f8\"] as const,\n  abbreviated: [\n    \"s\u00f8n.\",\n    \"man.\",\n    \"tir.\",\n    \"ons.\",\n    \"tor.\",\n    \"fre.\",\n    \"l\u00f8r.\",\n  ] as const,\n  wide: [\n    \"s\u00f8ndag\",\n    \"mandag\",\n    \"tirsdag\",\n    \"onsdag\",\n    \"torsdag\",\n    \"fredag\",\n    \"l\u00f8rdag\",\n  ] as const,\n};\n\nconst dayPeriodValues = {\n  narrow: {\n    am: \"a\",\n    pm: \"p\",\n    midnight: \"midnat\",\n    noon: \"middag\",\n    morning: \"morgen\",\n    afternoon: \"eftermiddag\",\n    evening: \"aften\",\n    night: \"nat\",\n  },\n  abbreviated: {\n    am: \"AM\",\n    pm: \"PM\",\n    midnight: \"midnat\",\n    noon: \"middag\",\n    morning: \"morgen\",\n    afternoon: \"eftermiddag\",\n    evening: \"aften\",\n    night: \"nat\",\n  },\n  wide: {\n    am: \"a.m.\",\n    pm: \"p.m.\",\n    midnight: \"midnat\",\n    noon: \"middag\",\n    morning: \"morgen\",\n    afternoon: \"eftermiddag\",\n    evening: \"aften\",\n    night: \"nat\",\n  },\n};\n\nconst formattingDayPeriodValues = {\n  narrow: {\n    am: \"a\",\n    pm: \"p\",\n    midnight: \"midnat\",\n    noon: \"middag\",\n    morning: \"om morgenen\",\n    afternoon: \"om eftermiddagen\",\n    evening: \"om aftenen\",\n    night: \"om natten\",\n  },\n  abbreviated: {\n    am: \"AM\",\n    pm: \"PM\",\n    midnight: \"midnat\",\n    noon: \"middag\",\n    morning: \"om morgenen\",\n    afternoon: \"om eftermiddagen\",\n    evening: \"om aftenen\",\n    night: \"om natten\",\n  },\n  wide: {\n    am: \"a.m.\",\n    pm: \"p.m.\",\n    midnight: \"midnat\",\n    noon: \"middag\",\n    morning: \"om morgenen\",\n    afternoon: \"om eftermiddagen\",\n    evening: \"om aftenen\",\n    night: \"om natten\",\n  },\n};\n\nconst ordinalNumber: LocalizeFn<number> = (dirtyNumber, _options) => {\n  const number = Number(dirtyNumber);\n  return number + \".\";\n};\n\nexport const localize: Localize = {\n  ordinalNumber,\n\n  era: buildLocalizeFn({\n    values: eraValues,\n    defaultWidth: \"wide\",\n  }),\n\n  quarter: buildLocalizeFn({\n    values: quarterValues,\n    defaultWidth: \"wide\",\n    argumentCallback: (quarter) => quarter - 1,\n  }),\n\n  month: buildLocalizeFn({\n    values: monthValues,\n    defaultWidth: \"wide\",\n  }),\n\n  day: buildLocalizeFn({\n    values: dayValues,\n    defaultWidth: \"wide\",\n  }),\n\n  dayPeriod: buildLocalizeFn({\n    values: dayPeriodValues,\n    defaultWidth: \"wide\",\n    formattingValues: formattingDayPeriodValues,\n    defaultFormattingWidth: \"wide\",\n  }),\n};\n\n// Path: src/locale/da/_lib/match/index.ts\nimport type { Quarter } from \"../../../../types.js\";\nimport type { Match } from \"../../../types.js\";\nimport { buildMatchFn } from \"../../../_lib/buildMatchFn/index.js\";\nimport { buildMatchPatternFn } from \"../../../_lib/buildMatchPatternFn/index.js\";\n\nconst matchOrdinalNumberPattern = /^(\\d+)(\\.)?/i;\nconst parseOrdinalNumberPattern = /\\d+/i;\n\nconst matchEraPatterns = {\n  narrow: /^(fKr|fvt|eKr|vt)/i,\n  abbreviated: /^(f\\.Kr\\.?|f\\.v\\.t\\.?|e\\.Kr\\.?|v\\.t\\.)/i,\n  wide: /^(f.Kr.|f\u00f8r vesterlandsk tidsregning|e.Kr.|vesterlandsk tidsregning)/i,\n};\nconst parseEraPatterns = {\n  any: [/^f/i, /^(v|e)/i] as const,\n};\n\nconst matchQuarterPatterns = {\n  narrow: /^[1234]/i,\n  abbreviated: /^[1234]. kvt\\./i,\n  wide: /^[1234]\\.? kvartal/i,\n};\nconst parseQuarterPatterns = {\n  any: [/1/i, /2/i, /3/i, /4/i] as const,\n};\n\nconst matchMonthPatterns = {\n  narrow: /^[jfmasond]/i,\n  abbreviated: /^(jan.|feb.|mar.|apr.|maj|jun.|jul.|aug.|sep.|okt.|nov.|dec.)/i,\n  wide: /^(januar|februar|marts|april|maj|juni|juli|august|september|oktober|november|december)/i,\n};\nconst parseMonthPatterns = {\n  narrow: [\n    /^j/i,\n    /^f/i,\n    /^m/i,\n    /^a/i,\n    /^m/i,\n    /^j/i,\n    /^j/i,\n    /^a/i,\n    /^s/i,\n    /^o/i,\n    /^n/i,\n    /^d/i,\n  ] as const,\n  any: [\n    /^ja/i,\n    /^f/i,\n    /^mar/i,\n    /^ap/i,\n    /^maj/i,\n    /^jun/i,\n    /^jul/i,\n    /^au/i,\n    /^s/i,\n    /^o/i,\n    /^n/i,\n    /^d/i,\n  ] as const,\n};\n\nconst matchDayPatterns = {\n  narrow: /^[smtofl]/i,\n  short: /^(s\u00f8n.|man.|tir.|ons.|tor.|fre.|l\u00f8r.)/i,\n  abbreviated: /^(s\u00f8n|man|tir|ons|tor|fre|l\u00f8r)/i,\n  wide: /^(s\u00f8ndag|mandag|tirsdag|onsdag|torsdag|fredag|l\u00f8rdag)/i,\n};\nconst parseDayPatterns = {\n  narrow: [/^s/i, /^m/i, /^t/i, /^o/i, /^t/i, /^f/i, /^l/i] as const,\n  any: [/^s/i, /^m/i, /^ti/i, /^o/i, /^to/i, /^f/i, /^l/i] as const,\n};\n\nconst matchDayPeriodPatterns = {\n  narrow: /^(a|p|midnat|middag|(om) (morgenen|eftermiddagen|aftenen|natten))/i,\n  any: /^([ap]\\.?\\s?m\\.?|midnat|middag|(om) (morgenen|eftermiddagen|aftenen|natten))/i,\n};\nconst parseDayPeriodPatterns = {\n  any: {\n    am: /^a/i,\n    pm: /^p/i,\n    midnight: /midnat/i,\n    noon: /middag/i,\n    morning: /morgen/i,\n    afternoon: /eftermiddag/i,\n    evening: /aften/i,\n    night: /nat/i,\n  },\n};\n\nexport const match: Match = {\n  ordinalNumber: buildMatchPatternFn({\n    matchPattern: matchOrdinalNumberPattern,\n    parsePattern: parseOrdinalNumberPattern,\n    valueCallback: (value) => parseInt(value, 10),\n  }),\n\n  era: buildMatchFn({\n    matchPatterns: matchEraPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseEraPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  quarter: buildMatchFn({\n    matchPatterns: matchQuarterPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseQuarterPatterns,\n    defaultParseWidth: \"any\",\n    valueCallback: (index) => (index + 1) as Quarter,\n  }),\n\n  month: buildMatchFn({\n    matchPatterns: matchMonthPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseMonthPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  day: buildMatchFn({\n    matchPatterns: matchDayPatterns,\n    defaultMatchWidth: \"wide\",\n...\n", "needle_token_start": 12177, "needle_token_end": 12177, "code_context_ntokens": 16224, "output": ["```typescript\nfunction getResultByTense(\n  parentToken: Tense,\n  options?: FormatDistanceFnOptions,\n): string {\n  if (options?.addSuffix) {\n    if (options.comparison && options.comparison > 0) {\n      return parentToken.future;\n    } else {\n      return parentToken.past;\n    }\n  }\n  return parentToken.default;\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "millisecondsToSeconds", "language": "typescript", "path": "src/millisecondsToSeconds/index.ts", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To convert a duration specified in milliseconds into an equivalent duration in whole seconds, using truncation to handle any fractional part.\n2. **Input**: A single integer representing the duration in milliseconds.\n3. **Output**: An integer representing the converted duration in whole seconds.\n4. **Procedure**: The function divides the input number of milliseconds by the constant value representing the number of milliseconds in one second (1000), and then truncates the result to remove any decimal, effectively performing floor rounding.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/format/index.ts\nimport { defaultLocale } from \"../_lib/defaultLocale/index.js\";\nimport { getDefaultOptions } from \"../_lib/defaultOptions/index.js\";\nimport { formatters } from \"../_lib/format/formatters/index.js\";\nimport { longFormatters } from \"../_lib/format/longFormatters/index.js\";\nimport {\n  isProtectedDayOfYearToken,\n  isProtectedWeekYearToken,\n  warnOrThrowProtectedError,\n} from \"../_lib/protectedTokens/index.js\";\nimport { isValid } from \"../isValid/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type {\n  AdditionalTokensOptions,\n  FirstWeekContainsDateOptions,\n  FormatPart,\n  LocalizedOptions,\n  WeekOptions,\n} from \"../types.js\";\n\n// Rexports of internal for libraries to use.\n// See: https://github.com/date-fns/date-fns/issues/3638#issuecomment-1877082874\nexport { formatters, longFormatters };\n\n// This RegExp consists of three parts separated by `|`:\n// - [yYQqMLwIdDecihHKkms]o matches any available ordinal number token\n//   (one of the certain letters followed by `o`)\n...\n// Path: src/formatRelative/index.ts\nimport { differenceInCalendarDays } from \"../differenceInCalendarDays/index.js\";\nimport { format } from \"../format/index.js\";\nimport type { FormatRelativeToken } from \"../locale/types.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { LocalizedOptions, WeekOptions } from \"../types.js\";\nimport { defaultLocale } from \"../_lib/defaultLocale/index.js\";\nimport { getDefaultOptions } from \"../_lib/defaultOptions/index.js\";\n\n/**\n * The {@link formatRelative} function options.\n */\nexport interface FormatRelativeOptions\n  extends LocalizedOptions<\n      \"options\" | \"localize\" | \"formatLong\" | \"formatRelative\"\n    >,\n    WeekOptions {}\n\n/**\n * @name formatRelative\n * @category Common Helpers\n * @summary Represent the date in words relative to the given base date.\n *\n * @description\n * Represent the date in words relative to the given base date.\n *\n * | Distance to the base date | Result                    |\n * |---------------------------|---------------------------|\n * | Previous 6 days           | last Sunday at 04:30 AM   |\n * | Last day                  | yesterday at 04:30 AM     |\n * | Same day                  | today at 04:30 AM         |\n * | Next day                  | tomorrow at 04:30 AM      |\n * | Next 6 days               | Sunday at 04:30 AM        |\n * | Other                     | 12/31/2017                |\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to format\n * @param baseDate - The date to compare with\n * @param options - An object with options\n *\n * @returns The date in words\n *\n * @throws `date` must not be Invalid Date\n * @throws `baseDate` must not be Invalid Date\n * @throws `options.locale` must contain `localize` property\n * @throws `options.locale` must contain `formatLong` property\n * @throws `options.locale` must contain `formatRelative` property\n *\n * @example\n * // Represent the date of 6 days ago in words relative to the given base date. In this example, today is Wednesday\n * const result = formatRelative(subDays(new Date(), 6), new Date())\n * //=> \"last Thursday at 12:45 AM\"\n */\nexport function formatRelative<DateType extends Date>(\n  date: DateType | number | string,\n  baseDate: DateType | number | string,\n  options?: FormatRelativeOptions,\n): string {\n  const _date = toDate(date);\n  const _baseDate = toDate(baseDate);\n\n  const defaultOptions = getDefaultOptions();\n  const locale = options?.locale ?? defaultOptions.locale ?? defaultLocale;\n  const weekStartsOn =\n    options?.weekStartsOn ??\n    options?.locale?.options?.weekStartsOn ??\n    defaultOptions.weekStartsOn ??\n    defaultOptions.locale?.options?.weekStartsOn ??\n    0;\n\n  const diff = differenceInCalendarDays(_date, _baseDate);\n\n  if (isNaN(diff)) {\n    throw new RangeError(\"Invalid time value\");\n  }\n\n  let token: FormatRelativeToken;\n  if (diff < -6) {\n    token = \"other\";\n  } else if (diff < -1) {\n    token = \"lastWeek\";\n  } else if (diff < 0) {\n    token = \"yesterday\";\n  } else if (diff < 1) {\n    token = \"today\";\n  } else if (diff < 2) {\n    token = \"tomorrow\";\n  } else if (diff < 7) {\n    token = \"nextWeek\";\n  } else {\n    token = \"other\";\n  }\n\n  const formatStr = locale.formatRelative(token, _date, _baseDate, {\n    locale,\n    weekStartsOn,\n  });\n  return format(_date, formatStr, { locale, weekStartsOn });\n}\n\n// Path: src/setSeconds/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setSeconds\n * @category Second Helpers\n * @summary Set the seconds to the given date.\n *\n * @description\n * Set the seconds to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param seconds - The seconds of the new date\n *\n * @returns The new date with the seconds set\n *\n * @example\n * // Set 45 seconds to 1 September 2014 11:30:40:\n * const result = setSeconds(new Date(2014, 8, 1, 11, 30, 40), 45)\n * //=> Mon Sep 01 2014 11:30:45\n */\nexport function setSeconds<DateType extends Date>(\n  date: DateType | number | string,\n  seconds: number,\n): DateType {\n  const _date = toDate(date);\n  _date.setSeconds(seconds);\n  return _date;\n}\n\n// Path: src/addSeconds/index.ts\nimport { addMilliseconds } from \"../addMilliseconds/index.js\";\n\n/**\n * @name addSeconds\n * @category Second Helpers\n * @summary Add the specified number of seconds to the given date.\n *\n * @description\n * Add the specified number of seconds to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of seconds to be added.\n *\n * @returns The new date with the seconds added\n *\n * @example\n * // Add 30 seconds to 10 July 2014 12:45:00:\n * const result = addSeconds(new Date(2014, 6, 10, 12, 45, 0), 30)\n * //=> Thu Jul 10 2014 12:45:30\n */\nexport function addSeconds<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addMilliseconds(date, amount * 1000);\n}\n\n// Path: src/interval/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport type { NormalizedInterval } from \"../types.js\";\n\n/**\n * The {@link interval} function options.\n */\nexport interface IntervalOptions {\n  /** Asserts that the interval is positive (start is after the end). */\n  assertPositive?: boolean;\n}\n\n/**\n * @name interval\n * @category Interval Helpers\n * @summary Creates an interval object and validates its values.\n *\n * @description\n * Creates a normalized interval object and validates its values. If the interval is invalid, an exception is thrown.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param start - The start of the interval.\n * @param end - The end of the interval.\n * @param options - The options object.\n *\n * @throws `Start date is invalid` when `start` is invalid.\n * @throws `End date is invalid` when `end` is invalid.\n * @throws `End date must be after start date` when end is before `start` and `options.assertPositive` is true.\n *\n * @returns The normalized and validated interval object.\n */\nexport function interval<DateType extends Date>(\n  start: DateType | number | string,\n  end: DateType | number | string,\n  options?: IntervalOptions,\n): NormalizedInterval<DateType> {\n  const _start = toDate(start);\n  if (isNaN(+_start)) throw new TypeError(\"Start date is invalid\");\n\n  const _end = toDate(end);\n  if (isNaN(+_end)) throw new TypeError(\"End date is invalid\");\n\n  if (options?.assertPositive && +_start > +_end)\n    throw new TypeError(\"End date must be after start date\");\n\n  return { start: _start, end: _end };\n}\n\n// Path: src/parseJSON/index.ts\n/**\n * @name parseJSON\n * @category Common Helpers\n * @summary Parse a JSON date string\n *\n * @description\n * Converts a complete ISO date string in UTC time, the typical format for transmitting\n * a date in JSON, to a JavaScript `Date` instance.\n *\n * This is a minimal implementation for converting dates retrieved from a JSON API to\n * a `Date` instance which can be used with other functions in the `date-fns` library.\n * The following formats are supported:\n *\n * - `2000-03-15T05:20:10.123Z`: The output of `.toISOString()` and `JSON.stringify(new Date())`\n * - `2000-03-15T05:20:10Z`: Without milliseconds\n * - `2000-03-15T05:20:10+00:00`: With a zero offset, the default JSON encoded format in some other languages\n * - `2000-03-15T05:20:10+05:45`: With a positive or negative offset, the default JSON encoded format in some other languages\n * - `2000-03-15T05:20:10+0000`: With a zero offset without a colon\n * - `2000-03-15T05:20:10`: Without a trailing 'Z' symbol\n * - `2000-03-15T05:20:10.1234567`: Up to 7 digits in milliseconds field. Only first 3 are taken into account since JS does not allow fractional milliseconds\n * - `2000-03-15 05:20:10`: With a space instead of a 'T' separator for APIs returning a SQL date without reformatting\n *\n * For convenience and ease of use these other input types are also supported\n * via [toDate](https://date-fns.org/docs/toDate):\n *\n * - A `Date` instance will be cloned\n * - A `number` will be treated as a timestamp\n *\n * Any other input type or invalid date strings will return an `Invalid Date`.\n *\n * @param dateStr - A fully formed ISO8601 date string to convert\n *\n * @returns The parsed date in the local time zone\n */\nexport function parseJSON(dateStr: string): Date {\n  const parts = dateStr.match(\n    /(\\d{4})-(\\d{2})-(\\d{2})[T ](\\d{2}):(\\d{2}):(\\d{2})(?:\\.(\\d{0,7}))?(?:Z|(.)(\\d{2}):?(\\d{2})?)?/,\n  );\n  if (parts) {\n    // Group 8 matches the sign\n    return new Date(\n      Date.UTC(\n        +parts[1],\n        +parts[2] - 1,\n        +parts[3],\n        +parts[4] - (+parts[9] || 0) * (parts[8] == \"-\" ? -1 : 1),\n        +parts[5] - (+parts[10] || 0) * (parts[8] == \"-\" ? -1 : 1),\n        +parts[6],\n        +((parts[7] || \"0\") + \"00\").substring(0, 3),\n      ),\n    );\n  }\n  return new Date(NaN);\n}\n\n// Path: src/previousFriday/index.ts\nimport { previousDay } from \"../previousDay/index.js\";\n\n/**\n * @name previousFriday\n * @category Weekday Helpers\n * @summary When is the previous Friday?\n *\n * @description\n * When is the previous Friday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to start counting from\n *\n * @returns The previous Friday\n *\n * @example\n * // When is the previous Friday before Jun, 19, 2021?\n * const result = previousFriday(new Date(2021, 5, 19))\n * //=> Fri June 18 2021 00:00:00\n */\nexport function previousFriday<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  return previousDay(date, 5);\n}\n\n// Path: src/isThisQuarter/index.ts\nimport { constructNow } from \"../constructNow/index.js\";\nimport { isSameQuarter } from \"../isSameQuarter/index.js\";\n\n/**\n * @name isThisQuarter\n * @category Quarter Helpers\n * @summary Is the given date in the same quarter as the current date?\n * @pure false\n *\n * @description\n * Is the given date in the same quarter as the current date?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is in this quarter\n *\n * @example\n * // If today is 25 September 2014, is 2 July 2014 in this quarter?\n * const result = isThisQuarter(new Date(2014, 6, 2))\n * //=> true\n */\nexport function isThisQuarter<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return isSameQuarter(date, constructNow(date));\n}\n\n// Path: src/getDate/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getDate\n * @category Day Helpers\n * @summary Get the day of the month of the given date.\n *\n * @description\n * Get the day of the month of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The day of month\n *\n * @example\n * // Which day of the month is 29 February 2012?\n * const result = getDate(new Date(2012, 1, 29))\n * //=> 29\n */\nexport function getDate<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const dayOfMonth = _date.getDate();\n  return dayOfMonth;\n}\n\n// Path: src/differenceInWeeks/index.ts\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { differenceInDays } from \"../differenceInDays/index.js\";\nimport type { RoundingOptions } from \"../types.js\";\n\n/**\n * The {@link differenceInWeeks} function options.\n */\nexport interface DifferenceInWeeksOptions extends RoundingOptions {}\n\n/**\n * @name differenceInWeeks\n * @category Week Helpers\n * @summary Get the number of full weeks between the given dates.\n *\n * @description\n * Get the number of full weeks between two dates. Fractional weeks are\n * truncated towards zero by default.\n *\n * One \"full week\" is the distance between a local time in one day to the same\n * local time 7 days earlier or later. A full week can sometimes be less than\n * or more than 7*24 hours if a daylight savings change happens between two dates.\n *\n * To ignore DST and only measure exact 7*24-hour periods, use this instead:\n * `Math.trunc(differenceInHours(dateLeft, dateRight)/(7*24))|0`.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n * @param options - An object with options\n *\n * @returns The number of full weeks\n *\n * @example\n * // How many full weeks are between 5 July 2014 and 20 July 2014?\n * const result = differenceInWeeks(new Date(2014, 6, 20), new Date(2014, 6, 5))\n * //=> 2\n *\n * @example\n * // How many full weeks are between\n * // 1 March 2020 0:00 and 6 June 2020 0:00 ?\n * // Note: because local time is used, the\n * // result will always be 8 weeks (54 days),\n * // even if DST starts and the period has\n * // only 54*24-1 hours.\n * const result = differenceInWeeks(\n *   new Date(2020, 5, 1),\n *   new Date(2020, 2, 6)\n * )\n * //=> 8\n */\nexport function differenceInWeeks<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n  options?: DifferenceInWeeksOptions,\n): number {\n  const diff = differenceInDays(dateLeft, dateRight) / 7;\n  return getRoundingMethod(options?.roundingMethod)(diff);\n}\n\n// Path: src/isPast/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isPast\n * @category Common Helpers\n * @summary Is the given date in the past?\n * @pure false\n *\n * @description\n * Is the given date in the past?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is in the past\n *\n * @example\n * // If today is 6 October 2014, is 2 July 2014 in the past?\n * const result = isPast(new Date(2014, 6, 2))\n * //=> true\n */\nexport function isPast<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return +toDate(date) < Date.now();\n}\n\n// Path: src/isTuesday/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isTuesday\n * @category Weekday Helpers\n * @summary Is the given date Tuesday?\n *\n * @description\n * Is the given date Tuesday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is Tuesday\n *\n * @example\n * // Is 23 September 2014 Tuesday?\n * const result = isTuesday(new Date(2014, 8, 23))\n * //=> true\n */\nexport function isTuesday<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return toDate(date).getDay() === 2;\n}\n\n// Path: src/lastDayOfISOWeekYear/index.ts\nimport { getISOWeekYear } from \"../getISOWeekYear/index.js\";\nimport { startOfISOWeek } from \"../startOfISOWeek/index.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\n\n/**\n * @name lastDayOfISOWeekYear\n * @category ISO Week-Numbering Year Helpers\n * @summary Return the last day of an ISO week-numbering year for the given date.\n *\n * @description\n * Return the last day of an ISO week-numbering year,\n * which always starts 3 days before the year's first Thursday.\n * The result will be in the local timezone.\n *\n * ISO week-numbering year: http://en.wikipedia.org/wiki/ISO_week_date\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The end of an ISO week-numbering year\n *\n * @example\n * // The last day of an ISO week-numbering year for 2 July 2005:\n * const result = lastDayOfISOWeekYear(new Date(2005, 6, 2))\n * //=> Sun Jan 01 2006 00:00:00\n */\nexport function lastDayOfISOWeekYear<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  const year = getISOWeekYear(date);\n  const fourthOfJanuary = constructFrom(date, 0);\n  fourthOfJanuary.setFullYear(year + 1, 0, 4);\n  fourthOfJanuary.setHours(0, 0, 0, 0);\n  const _date = startOfISOWeek(fourthOfJanuary);\n  _date.setDate(_date.getDate() - 1);\n  return _date;\n}\n\n// Path: src/getDaysInYear/index.ts\nimport { isLeapYear } from \"../isLeapYear/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getDaysInYear\n * @category Year Helpers\n * @summary Get the number of days in a year of the given date.\n *\n * @description\n * Get the number of days in a year of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The number of days in a year\n *\n * @example\n * // How many days are in 2012?\n * const result = getDaysInYear(new Date(2012, 0, 1))\n * //=> 366\n */\nexport function getDaysInYear<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n\n  if (String(new Date(_date)) === \"Invalid Date\") {\n    return NaN;\n  }\n\n  return isLeapYear(_date) ? 366 : 365;\n}\n\n// Path: src/subMinutes/index.ts\nimport { addMinutes } from \"../addMinutes/index.js\";\n\n/**\n * @name subMinutes\n * @category Minute Helpers\n * @summary Subtract the specified number of minutes from the given date.\n *\n * @description\n * Subtract the specified number of minutes from the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param amount - The amount of minutes to be subtracted.\n *\n * @returns The new date with the minutes subtracted\n *\n * @example\n * // Subtract 30 minutes from 10 July 2014 12:00:00:\n * const result = subMinutes(new Date(2014, 6, 10, 12, 0), 30)\n * //=> Thu Jul 10 2014 11:30:00\n */\nexport function subMinutes<DateType extends Date>(\n  date: DateType | number | string,\n  amount: number,\n): DateType {\n  return addMinutes(date, -amount);\n}\n\n// Path: src/isExists/index.ts\n/**\n * @name isExists\n * @category Common Helpers\n * @summary Is the given date exists?\n *\n * @description\n * Checks if the given arguments convert to an existing date.\n *\n * @param year - The year of the date to check\n * @param month - The month of the date to check\n * @param day - The day of the date to check\n *\n * @returns `true` if the date exists\n *\n * @example\n * // For the valid date:\n * const result = isExists(2018, 0, 31)\n * //=> true\n *\n * @example\n * // For the invalid date:\n * const result = isExists(2018, 1, 31)\n * //=> false\n */\nexport function isExists(year: number, month: number, day: number): boolean {\n  const date = new Date(year, month, day);\n  return (\n    date.getFullYear() === year &&\n    date.getMonth() === month &&\n    date.getDate() === day\n  );\n}\n\n// Path: src/endOfDecade/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name endOfDecade\n * @category Decade Helpers\n * @summary Return the end of a decade for the given date.\n *\n * @description\n * Return the end of a decade for the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The end of a decade\n *\n * @example\n * // The end of a decade for 12 May 1984 00:00:00:\n * const result = endOfDecade(new Date(1984, 4, 12, 00, 00, 00))\n * //=> Dec 31 1989 23:59:59.999\n */\nexport function endOfDecade<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  // TODO: Switch to more technical definition in of decades that start with 1\n  // end with 0. I.e. 2001-2010 instead of current 2000-2009. It's a breaking\n  // change, so it can only be done in 4.0.\n  const _date = toDate(date);\n  const year = _date.getFullYear();\n  const decade = 9 + Math.floor(year / 10) * 10;\n  _date.setFullYear(decade, 11, 31);\n  _date.setHours(23, 59, 59, 999);\n  return _date;\n}\n\n// Path: src/differenceInCalendarISOWeekYears/index.ts\nimport { getISOWeekYear } from \"../getISOWeekYear/index.js\";\n\n/**\n * @name differenceInCalendarISOWeekYears\n * @category ISO Week-Numbering Year Helpers\n * @summary Get the number of calendar ISO week-numbering years between the given dates.\n *\n * @description\n * Get the number of calendar ISO week-numbering years between the given dates.\n *\n * ISO week-numbering year: http://en.wikipedia.org/wiki/ISO_week_date\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param dateLeft - The later date\n * @param dateRight - The earlier date\n *\n * @returns The number of calendar ISO week-numbering years\n *\n * @example\n * // How many calendar ISO week-numbering years are 1 January 2010 and 1 January 2012?\n * const result = differenceInCalendarISOWeekYears(\n *   new Date(2012, 0, 1),\n *   new Date(2010, 0, 1)\n * )\n * //=> 2\n */\nexport function differenceInCalendarISOWeekYears<DateType extends Date>(\n  dateLeft: DateType | number | string,\n  dateRight: DateType | number | string,\n): number {\n  return getISOWeekYear(dateLeft) - getISOWeekYear(dateRight);\n}\n\n// Path: src/isAfter/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name isAfter\n * @category Common Helpers\n * @summary Is the first date after the second one?\n *\n * @description\n * Is the first date after the second one?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date that should be after the other one to return true\n * @param dateToCompare - The date to compare with\n *\n * @returns The first date is after the second date\n *\n * @example\n * // Is 10 July 1989 after 11 February 1987?\n * const result = isAfter(new Date(1989, 6, 10), new Date(1987, 1, 11))\n * //=> true\n */\nexport function isAfter<DateType extends Date>(\n  date: DateType | number | string,\n  dateToCompare: DateType | number | string,\n): boolean {\n  const _date = toDate(date);\n  const _dateToCompare = toDate(dateToCompare);\n  return _date.getTime() > _dateToCompare.getTime();\n}\n\n// Path: src/roundToNearestHours/index.ts\nimport { getRoundingMethod } from \"../_lib/getRoundingMethod/index.js\";\nimport { constructFrom } from \"../constructFrom/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type {\n  NearestHours,\n  NearestToUnitOptions,\n  RoundingOptions,\n} from \"../types.js\";\n\n/**\n * The {@link roundToNearestHours} function options.\n */\nexport interface RoundToNearestHoursOptions\n  extends NearestToUnitOptions<NearestHours>,\n    RoundingOptions {}\n\n/**\n * @name roundToNearestHours\n * @category Hour Helpers\n * @summary Rounds the given date to the nearest hour\n *\n * @description\n * Rounds the given date to the nearest hour (or number of hours).\n * Rounds up when the given date is exactly between the nearest round hours.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to round\n * @param options - An object with options.\n *\n * @returns The new date rounded to the closest hour\n *\n * @example\n * // Round 10 July 2014 12:34:56 to nearest hour:\n * const result = roundToNearestHours(new Date(2014, 6, 10, 12, 34, 56))\n * //=> Thu Jul 10 2014 13:00:00\n *\n * @example\n * // Round 10 July 2014 12:34:56 to nearest half hour:\n * const result = roundToNearestHours(new Date(2014, 6, 10, 12, 34, 56), { nearestTo: 6 })\n * //=> Thu Jul 10 2014 12:00:00\n\n * @example\n * // Round 10 July 2014 12:34:56 to nearest half hour:\n * const result = roundToNearestHours(new Date(2014, 6, 10, 12, 34, 56), { nearestTo: 8 })\n * //=> Thu Jul 10 2014 16:00:00\n\n* @example\n * // Floor (rounds down) 10 July 2014 12:34:56 to nearest hour:\n * const result = roundToNearestHours(new Date(2014, 6, 10, 1, 23, 45), { roundingMethod: 'ceil' })\n * //=> Thu Jul 10 2014 02:00:00\n *\n * @example\n * // Ceil (rounds up) 10 July 2014 12:34:56 to nearest quarter hour:\n * const result = roundToNearestHours(new Date(2014, 6, 10, 12, 34, 56), { roundingMethod: 'floor', nearestTo: 8 })\n * //=> Thu Jul 10 2014 08:00:00\n */\nexport function roundToNearestHours<DateType extends Date>(\n  date: DateType | number | string,\n  options?: RoundToNearestHoursOptions,\n): Date {\n  const nearestTo = options?.nearestTo ?? 1;\n\n  if (nearestTo < 1 || nearestTo > 12) return constructFrom(date, NaN);\n\n  const _date = toDate(date);\n  const fractionalMinutes = _date.getMinutes() / 60;\n  const fractionalSeconds = _date.getSeconds() / 60 / 60;\n  const fractionalMilliseconds = _date.getMilliseconds() / 1000 / 60 / 60;\n  const hours =\n    _date.getHours() +\n    fractionalMinutes +\n    fractionalSeconds +\n    fractionalMilliseconds;\n\n  // Unlike the `differenceIn*` functions, the default rounding behavior is `round` and not 'trunc'\n  const method = options?.roundingMethod ?? \"round\";\n  const roundingMethod = getRoundingMethod(method);\n\n  // nearestTo option does not care daylight savings time\n  const roundedHours = roundingMethod(hours / nearestTo) * nearestTo;\n\n  const result = constructFrom(date, _date);\n  result.setHours(roundedHours, 0, 0, 0);\n  return result;\n}\n\n// Path: src/lastDayOfDecade/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name lastDayOfDecade\n * @category Decade Helpers\n * @summary Return the last day of a decade for the given date.\n *\n * @description\n * Return the last day of a decade for the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n *\n * @returns The last day of a decade\n *\n * @example\n * // The last day of a decade for 21 December 2012 21:12:00:\n * const result = lastDayOfDecade(new Date(2012, 11, 21, 21, 12, 00))\n * //=> Wed Dec 31 2019 00:00:00\n */\nexport function lastDayOfDecade<DateType extends Date>(\n  date: DateType | number | string,\n): DateType {\n  // TODO: Switch to more technical definition in of decades that start with 1\n  // end with 0. I.e. 2001-2010 instead of current 2000-2009. It's a breaking\n  // change, so it can only be done in 4.0.\n  const _date = toDate(date);\n  const year = _date.getFullYear();\n  const decade = 9 + Math.floor(year / 10) * 10;\n  _date.setFullYear(decade + 1, 0, 0);\n  _date.setHours(0, 0, 0, 0);\n  return _date;\n}\n\n// Path: src/isYesterday/index.ts\nimport { constructNow } from \"../constructNow/index.js\";\nimport { isSameDay } from \"../isSameDay/index.js\";\nimport { subDays } from \"../subDays/index.js\";\n\n/**\n * @name isYesterday\n * @category Day Helpers\n * @summary Is the given date yesterday?\n * @pure false\n *\n * @description\n * Is the given date yesterday?\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to check\n *\n * @returns The date is yesterday\n *\n * @example\n * // If today is 6 October 2014, is 5 October 14:00:00 yesterday?\n * const result = isYesterday(new Date(2014, 9, 5, 14, 0))\n * //=> true\n */\nexport function isYesterday<DateType extends Date>(\n  date: DateType | number | string,\n): boolean {\n  return isSameDay(date, subDays(constructNow(date), 1));\n}\n\n// Path: src/eachMinuteOfInterval/index.ts\nimport { addMinutes } from \"../addMinutes/index.js\";\nimport { startOfMinute } from \"../startOfMinute/index.js\";\nimport { toDate } from \"../toDate/index.js\";\nimport type { Interval, StepOptions } from \"../types.js\";\n\n/**\n * The {@link eachMinuteOfInterval} function options.\n */\nexport interface EachMinuteOfIntervalOptions extends StepOptions {}\n\n/**\n * @name eachMinuteOfInterval\n * @category Interval Helpers\n * @summary Return the array of minutes within the specified time interval.\n *\n * @description\n * Returns the array of minutes within the specified time interval.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param interval - The interval.\n * @param options - An object with options.\n *\n * @returns The array with starts of minutes from the minute of the interval start to the minute of the interval end\n *\n * @example\n * // Each minute between 14 October 2020, 13:00 and 14 October 2020, 13:03\n * const result = eachMinuteOfInterval({\n *   start: new Date(2014, 9, 14, 13),\n *   end: new Date(2014, 9, 14, 13, 3)\n * })\n * //=> [\n * //   Wed Oct 14 2014 13:00:00,\n * //   Wed Oct 14 2014 13:01:00,\n * //   Wed Oct 14 2014 13:02:00,\n * //   Wed Oct 14 2014 13:03:00\n * // ]\n */\nexport function eachMinuteOfInterval<DateType extends Date>(\n  interval: Interval<DateType>,\n  options?: EachMinuteOfIntervalOptions,\n): DateType[] {\n  const startDate = startOfMinute(toDate(interval.start));\n  const endDate = toDate(interval.end);\n\n  let reversed = +startDate > +endDate;\n  const endTime = reversed ? +startDate : +endDate;\n  let currentDate = reversed ? endDate : startDate;\n\n  let step = options?.step ?? 1;\n  if (!step) return [];\n  if (step < 0) {\n    step = -step;\n    reversed = !reversed;\n  }\n\n  const dates = [];\n\n  while (+currentDate <= endTime) {\n    dates.push(toDate(currentDate));\n    currentDate = addMinutes(currentDate, step);\n  }\n\n  return reversed ? dates.reverse() : dates;\n}\n\n// Path: src/clamp/index.ts\nimport { max } from \"../max/index.js\";\nimport { min } from \"../min/index.js\";\nimport type { Interval } from \"../types.js\";\n\n/**\n * @name clamp\n * @category Interval Helpers\n * @summary Return a date bounded by the start and the end of the given interval\n *\n * @description\n * Clamps a date to the lower bound with the start of the interval and the upper\n * bound with the end of the interval.\n *\n * - When the date is less than the start of the interval, the start is returned.\n * - When the date is greater than the end of the interval, the end is returned.\n * - Otherwise the date is returned.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be bounded\n * @param interval - The interval to bound to\n *\n * @returns The date bounded by the start and the end of the interval\n *\n * @example\n * // What is Mar, 21, 2021 bounded to an interval starting at Mar, 22, 2021 and ending at Apr, 01, 2021\n * const result = clamp(new Date(2021, 2, 21), {\n *   start: new Date(2021, 2, 22),\n *   end: new Date(2021, 3, 1),\n * })\n * //=> Mon Mar 22 2021 00:00:00\n */\nexport function clamp<DateType extends Date>(\n  date: DateType | number | string,\n  interval: Interval,\n): DateType | Date {\n  return min([max([date, interval.start]), interval.end]);\n}\n\n// Path: src/minutesToMilliseconds/index.ts\nimport { millisecondsInMinute } from \"../constants/index.js\";\n\n/**\n * @name minutesToMilliseconds\n * @category Conversion Helpers\n * @summary Convert minutes to milliseconds.\n *\n * @description\n * Convert a number of minutes to a full number of milliseconds.\n *\n * @param minutes - The number of minutes to be converted\n *\n * @returns The number of minutes converted in milliseconds\n *\n * @example\n * // Convert 2 minutes to milliseconds\n * const result = minutesToMilliseconds(2)\n * //=> 120000\n */\nexport function minutesToMilliseconds(minutes: number): number {\n  return Math.trunc(minutes * millisecondsInMinute);\n}\n\n// Path: src/setMilliseconds/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setMilliseconds\n * @category Millisecond Helpers\n * @summary Set the milliseconds to the given date.\n *\n * @description\n * Set the milliseconds to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param milliseconds - The milliseconds of the new date\n *\n * @returns The new date with the milliseconds set\n *\n * @example\n * // Set 300 milliseconds to 1 September 2014 11:30:40.500:\n * const result = setMilliseconds(new Date(2014, 8, 1, 11, 30, 40, 500), 300)\n * //=> Mon Sep 01 2014 11:30:40.300\n */\nexport function setMilliseconds<DateType extends Date>(\n  date: DateType | number | string,\n  milliseconds: number,\n): DateType {\n  const _date = toDate(date);\n  _date.setMilliseconds(milliseconds);\n  return _date;\n}\n\n// Path: src/monthsToYears/index.ts\nimport { monthsInYear } from \"../constants/index.js\";\n\n/**\n * @name monthsToYears\n * @category Conversion Helpers\n * @summary Convert number of months to years.\n *\n * @description\n * Convert a number of months to a full number of years.\n *\n * @param months - The number of months to be converted\n *\n * @returns The number of months converted in years\n *\n * @example\n * // Convert 36 months to years:\n * const result = monthsToYears(36)\n * //=> 3\n *\n * // It uses floor rounding:\n * const result = monthsToYears(40)\n * //=> 3\n */\nexport function monthsToYears(months: number): number {\n  const years = months / monthsInYear;\n  return Math.trunc(years);\n}\n\n// Path: src/getUnixTime/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getUnixTime\n * @category Timestamp Helpers\n * @summary Get the seconds timestamp of the given date.\n *\n * @description\n * Get the seconds timestamp of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The timestamp\n *\n * @example\n * // Get the timestamp of 29 February 2012 11:45:05 CET:\n * const result = getUnixTime(new Date(2012, 1, 29, 11, 45, 5))\n * //=> 1330512305\n */\nexport function getUnixTime<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  return Math.trunc(+toDate(date) / 1000);\n}\n\n// Path: src/setQuarter/index.ts\nimport { setMonth } from \"../setMonth/index.js\";\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setQuarter\n * @category Quarter Helpers\n * @summary Set the year quarter to the given date.\n *\n * @description\n * Set the year quarter to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param quarter - The quarter of the new date\n *\n * @returns The new date with the quarter set\n *\n * @example\n * // Set the 2nd quarter to 2 July 2014:\n * const result = setQuarter(new Date(2014, 6, 2), 2)\n * //=> Wed Apr 02 2014 00:00:00\n */\nexport function setQuarter<DateType extends Date>(\n  date: DateType | number | string,\n  quarter: number,\n): DateType {\n  const _date = toDate(date);\n  const oldQuarter = Math.trunc(_date.getMonth() / 3) + 1;\n  const diff = quarter - oldQuarter;\n  return setMonth(_date, _date.getMonth() + diff * 3);\n}\n\n// Path: src/quartersToMonths/index.ts\nimport { monthsInQuarter } from \"../constants/index.js\";\n\n/**\n * @name quartersToMonths\n * @category Conversion Helpers\n * @summary Convert number of quarters to months.\n *\n * @description\n * Convert a number of quarters to a full number of months.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param quarters - The number of quarters to be converted\n *\n * @returns The number of quarters converted in months\n *\n * @example\n * // Convert 2 quarters to months\n * const result = quartersToMonths(2)\n * //=> 6\n */\nexport function quartersToMonths(quarters: number): number {\n  return Math.trunc(quarters * monthsInQuarter);\n}\n\n// Path: src/millisecondsToSeconds/index.ts\nimport { millisecondsInSecond } from \"../constants/index.js\";\n\n/**\n * @name millisecondsToSeconds\n * @category Conversion Helpers\n * @summary Convert milliseconds to seconds.\n *\n * @description\n * Convert a number of milliseconds to a full number of seconds.\n *\n * @param milliseconds - The number of milliseconds to be converted\n *\n * @returns The number of milliseconds converted in seconds\n *\n * @example\n * // Convert 1000 miliseconds to seconds:\n * const result = millisecondsToSeconds(1000)\n * //=> 1\n *\n * @example\n * // It uses floor rounding:\n * const result = millisecondsToSeconds(1999)\n * //=> 1\n */\nexport \nfunction millisecondsToSeconds(milliseconds: number): number {\n  const seconds = milliseconds / millisecondsInSecond;\n  return Math.trunc(seconds);\n}\n\n// Path: src/startOfToday/index.ts\nimport { startOfDay } from \"../startOfDay/index.js\";\n\n/**\n * @name startOfToday\n * @category Day Helpers\n * @summary Return the start of today.\n * @pure false\n *\n * @description\n * Return the start of today.\n *\n * @returns The start of today\n *\n * @example\n * // If today is 6 October 2014:\n * const result = startOfToday()\n * //=> Mon Oct 6 2014 00:00:00\n */\nexport function startOfToday(): Date {\n  return startOfDay(Date.now());\n}\n\n// Path: src/getWeekOfMonth/index.ts\nimport { getDate } from \"../getDate/index.js\";\nimport { getDay } from \"../getDay/index.js\";\nimport { startOfMonth } from \"../startOfMonth/index.js\";\nimport type { LocalizedOptions, WeekOptions } from \"../types.js\";\nimport { getDefaultOptions } from \"../_lib/defaultOptions/index.js\";\n\n/**\n * The {@link getWeekOfMonth} function options.\n */\nexport interface GetWeekOfMonthOptions\n  extends LocalizedOptions<\"options\">,\n    WeekOptions {}\n\n/**\n * @name getWeekOfMonth\n * @category Week Helpers\n * @summary Get the week of the month of the given date.\n *\n * @description\n * Get the week of the month of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n * @param options - An object with options.\n *\n * @returns The week of month\n *\n * @example\n * // Which week of the month is 9 November 2017?\n * const result = getWeekOfMonth(new Date(2017, 10, 9))\n * //=> 2\n */\nexport function getWeekOfMonth<DateType extends Date>(\n  date: DateType | number | string,\n  options?: GetWeekOfMonthOptions,\n): number {\n  const defaultOptions = getDefaultOptions();\n  const weekStartsOn =\n    options?.weekStartsOn ??\n    options?.locale?.options?.weekStartsOn ??\n    defaultOptions.weekStartsOn ??\n    defaultOptions.locale?.options?.weekStartsOn ??\n    0;\n\n  const currentDayOfMonth = getDate(date);\n  if (isNaN(currentDayOfMonth)) return NaN;\n\n  const startWeekDay = getDay(startOfMonth(date));\n\n  let lastDayOfFirstWeek = weekStartsOn - startWeekDay;\n  if (lastDayOfFirstWeek <= 0) lastDayOfFirstWeek += 7;\n\n  const remainingDaysAfterFirstWeek = currentDayOfMonth - lastDayOfFirstWeek;\n  return Math.ceil(remainingDaysAfterFirstWeek / 7) + 1;\n}\n\n// Path: src/getSeconds/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name getSeconds\n * @category Second Helpers\n * @summary Get the seconds of the given date.\n *\n * @description\n * Get the seconds of the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The given date\n *\n * @returns The seconds\n *\n * @example\n * // Get the seconds of 29 February 2012 11:45:05.123:\n * const result = getSeconds(new Date(2012, 1, 29, 11, 45, 5, 123))\n * //=> 5\n */\nexport function getSeconds<DateType extends Date>(\n  date: DateType | number | string,\n): number {\n  const _date = toDate(date);\n  const seconds = _date.getSeconds();\n  return seconds;\n}\n\n// Path: src/setMinutes/index.ts\nimport { toDate } from \"../toDate/index.js\";\n\n/**\n * @name setMinutes\n * @category Minute Helpers\n * @summary Set the minutes to the given date.\n *\n * @description\n * Set the minutes to the given date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The date to be changed\n * @param minutes - The minutes of the new date\n *\n * @returns The new date with the minutes set\n *\n * @example\n * // Set 45 minutes to 1 September 2014 11:30:40:\n * const result = setMinutes(new Date(2014, 8, 1, 11, 30, 40), 45)\n * //=> Mon Sep 01 2014 11:45:40\n */\nexport function setMinutes<DateType extends Date>(\n  date: DateType | number | string,\n  minutes: number,\n): DateType {\n  const _date = toDate(date);\n  _date.setMinutes(minutes);\n  return _date;\n}\n\n// Path: src/formatISO/index.ts\nimport { toDate } from \"../toDate/index.js\";\nimport type { ISOFormatOptions } from \"../types.js\";\nimport { addLeadingZeros } from \"../_lib/addLeadingZeros/index.js\";\n\n/**\n * The {@link formatISO} function options.\n */\nexport interface FormatISOOptions extends ISOFormatOptions {}\n\n/**\n * @name formatISO\n * @category Common Helpers\n * @summary Format the date according to the ISO 8601 standard (https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a003169814.htm).\n *\n * @description\n * Return the formatted date string in ISO 8601 format. Options may be passed to control the parts and notations of the date.\n *\n * @typeParam DateType - The `Date` type, the function operates on. Gets inferred from passed arguments. Allows to use extensions like [`UTCDate`](https://github.com/date-fns/utc).\n *\n * @param date - The original date\n * @param options - An object with options.\n *\n * @returns The formatted date string (in loca.l time zone)\n *\n * @throws `date` must not be Invalid Date\n *\n * @example\n * // Represent 18 September 2019 in ISO 8601 format (local time zone is UTC):\n * const result = formatISO(new Date(2019, 8, 18, 19, 0, 52))\n * //=> '2019-09-18T19:00:52Z'\n *\n * @example\n * // Represent 18 September 2019 in ISO 8601, short format (local time zone is UTC):\n * const result = formatISO(new Date(2019, 8, 18, 19, 0, 52), { format: 'basic' })\n * //=> '20190918T190052'\n *\n * @example\n * // Represent 18 September 2019 in ISO 8601 format, date only:\n * const result = formatISO(new Date(2019, 8, 18, 19, 0, 52), { representation: 'date' })\n * //=> '2019-09-18'\n *\n * @example\n * // Represent 18 September 2019 in ISO 8601 format, time only (local time zone is UTC):\n * const result = formatISO(new Date(2019, 8, 18, 19, 0, 52), { representation: 'time' })\n * //=> '19:00:52Z'\n */\nexport function formatISO<DateType extends Date>(\n  date: DateType | number | string,\n  options?: FormatISOOptions,\n): string {\n  const _date = toDate(date);\n\n  if (isNaN(_date.getTime())) {\n    throw new RangeError(\"Invalid time value\");\n  }\n\n  const format = options?.format ?? \"extended\";\n  const representation = options?.representation ?? \"complete\";\n\n  let result = \"\";\n  let tzOffset = \"\";\n\n  const dateDelimiter = format === \"extended\" ? \"-\" : \"\";\n  const timeDelimiter = format === \"extended\" ? \":\" : \"\";\n\n  // Representation is either 'date' or 'complete'\n  if (representation !== \"time\") {\n    const day = addLeadingZeros(_date.getDate(), 2);\n    const month = addLeadingZeros(_date.getMonth() + 1, 2);\n...\n", "needle_token_start": 13119, "needle_token_end": 13157, "code_context_ntokens": 15449, "output": ["```typescript\nimport { millisecondsInSecond } from \"../constants/index.js\";\n\n/**\n * @name millisecondsToSeconds\n * @category Conversion Helpers\n * @summary Convert milliseconds to seconds.\n *\n * @description\n * Convert a number of milliseconds to a full number of seconds.\n *\n * @param milliseconds - The number of milliseconds to be converted\n *\n * @returns The number of milliseconds converted in seconds\n *\n * @example\n * // Convert 1000 miliseconds to seconds:\n * const result = millisecondsToSeconds(1000)\n * //=> 1\n *\n * @example\n * // It uses floor rounding:\n * const result = millisecondsToSeconds(1999)\n * //=> 1\n */\nexport function millisecondsToSeconds(milliseconds: number): number {\n  const seconds = milliseconds / millisecondsInSecond;\n  return Math.trunc(seconds);\n}\n```"]}
{"repo": "date-fns/date-fns", "name": "dateOrdinalNumber", "language": "typescript", "path": "src/locale/bn/_lib/localize/index.ts", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a numerical day of the month into its corresponding ordinal representation in Bengali.\n2. **Input**: It takes two parameters: an integer representing the day of the month, and a string representing the Bengali numeral for that day.\n3. **Output**: It returns a string which is the Bengali ordinal form of the day of the month.\n4. **Procedure**: The function first checks if the day number is between 19 and 31; if so, it appends a specific suffix to the Bengali numeral. For specific numbers like 1, 2, 3, and 4, it appends unique suffixes. For all other numbers (5 to 18), it appends a common suffix.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/locale/nn/_lib/localize/index.ts\nimport type { Localize, LocalizeFn } from \"../../../types.js\";\nimport { buildLocalizeFn } from \"../../../_lib/buildLocalizeFn/index.js\";\n\nconst eraValues = {\n  narrow: [\"f.Kr.\", \"e.Kr.\"] as const,\n  abbreviated: [\"f.Kr.\", \"e.Kr.\"] as const,\n  wide: [\"f\u00f8r Kristus\", \"etter Kristus\"] as const,\n};\n\nconst quarterValues = {\n  narrow: [\"1\", \"2\", \"3\", \"4\"] as const,\n  abbreviated: [\"Q1\", \"Q2\", \"Q3\", \"Q4\"] as const,\n  wide: [\"1. kvartal\", \"2. kvartal\", \"3. kvartal\", \"4. kvartal\"] as const,\n};\n\nconst monthValues = {\n  narrow: [\"J\", \"F\", \"M\", \"A\", \"M\", \"J\", \"J\", \"A\", \"S\", \"O\", \"N\", \"D\"] as const,\n  abbreviated: [\n    \"jan.\",\n    \"feb.\",\n    \"mars\",\n    \"apr.\",\n    \"mai\",\n    \"juni\",\n    \"juli\",\n    \"aug.\",\n    \"sep.\",\n    \"okt.\",\n    \"nov.\",\n    \"des.\",\n  ] as const,\n  wide: [\n    \"januar\",\n    \"februar\",\n    \"mars\",\n    \"april\",\n    \"mai\",\n    \"juni\",\n    \"juli\",\n    \"august\",\n    \"september\",\n    \"oktober\",\n    \"november\",\n    \"desember\",\n  ] as const,\n};\n\nconst dayValues = {\n  narrow: [\"S\", \"M\", \"T\", \"O\", \"T\", \"F\", \"L\"] as const,\n  short: [\"su\", \"m\u00e5\", \"ty\", \"on\", \"to\", \"fr\", \"lau\"] as const,\n  abbreviated: [\"sun\", \"m\u00e5n\", \"tys\", \"ons\", \"tor\", \"fre\", \"laur\"] as const,\n  wide: [\n    \"sundag\",\n    \"m\u00e5ndag\",\n    \"tysdag\",\n...\n// Path: src/locale/nn/_lib/match/index.ts\nimport type { Quarter } from \"../../../../types.js\";\nimport type { Match } from \"../../../types.js\";\nimport { buildMatchFn } from \"../../../_lib/buildMatchFn/index.js\";\nimport { buildMatchPatternFn } from \"../../../_lib/buildMatchPatternFn/index.js\";\n\nconst matchOrdinalNumberPattern = /^(\\d+)\\.?/i;\nconst parseOrdinalNumberPattern = /\\d+/i;\n\nconst matchEraPatterns = {\n  narrow: /^(f\\.? ?Kr\\.?|fvt\\.?|e\\.? ?Kr\\.?|evt\\.?)/i,\n  abbreviated: /^(f\\.? ?Kr\\.?|fvt\\.?|e\\.? ?Kr\\.?|evt\\.?)/i,\n  wide: /^(f\u00f8r Kristus|f\u00f8r v\u00e5r tid|etter Kristus|v\u00e5r tid)/i,\n};\nconst parseEraPatterns = {\n  any: [/^f/i, /^e/i] as const,\n};\n\nconst matchQuarterPatterns = {\n  narrow: /^[1234]/i,\n  abbreviated: /^q[1234]/i,\n  wide: /^[1234](\\.)? kvartal/i,\n};\nconst parseQuarterPatterns = {\n  any: [/1/i, /2/i, /3/i, /4/i] as const,\n};\n\nconst matchMonthPatterns = {\n  narrow: /^[jfmasond]/i,\n  abbreviated: /^(jan|feb|mars?|apr|mai|juni?|juli?|aug|sep|okt|nov|des)\\.?/i,\n  wide: /^(januar|februar|mars|april|mai|juni|juli|august|september|oktober|november|desember)/i,\n};\nconst parseMonthPatterns = {\n  narrow: [\n    /^j/i,\n    /^f/i,\n    /^m/i,\n    /^a/i,\n    /^m/i,\n    /^j/i,\n    /^j/i,\n    /^a/i,\n    /^s/i,\n    /^o/i,\n    /^n/i,\n    /^d/i,\n  ] as const,\n  any: [\n    /^ja/i,\n    /^f/i,\n    /^mar/i,\n    /^ap/i,\n    /^mai/i,\n    /^jun/i,\n    /^jul/i,\n    /^aug/i,\n    /^s/i,\n    /^o/i,\n    /^n/i,\n    /^d/i,\n  ] as const,\n};\n\nconst matchDayPatterns = {\n  narrow: /^[smtofl]/i,\n  short: /^(su|m\u00e5|ty|on|to|fr|la)/i,\n  abbreviated: /^(sun|m\u00e5n|tys|ons|tor|fre|laur)/i,\n  wide: /^(sundag|m\u00e5ndag|tysdag|onsdag|torsdag|fredag|laurdag)/i,\n};\nconst parseDayPatterns = {\n  any: [/^s/i, /^m/i, /^ty/i, /^o/i, /^to/i, /^f/i, /^l/i] as const,\n};\n\nconst matchDayPeriodPatterns = {\n  narrow: /^(midnatt|middag|(p\u00e5) (morgonen|ettermiddagen|kvelden|natta)|[ap])/i,\n  any: /^([ap]\\.?\\s?m\\.?|midnatt|middag|(p\u00e5) (morgonen|ettermiddagen|kvelden|natta))/i,\n};\nconst parseDayPeriodPatterns = {\n  any: {\n    am: /^a(\\.?\\s?m\\.?)?$/i,\n    pm: /^p(\\.?\\s?m\\.?)?$/i,\n    midnight: /^midn/i,\n    noon: /^midd/i,\n    morning: /morgon/i,\n    afternoon: /ettermiddag/i,\n    evening: /kveld/i,\n    night: /natt/i,\n  },\n};\n\nexport const match: Match = {\n  ordinalNumber: buildMatchPatternFn({\n    matchPattern: matchOrdinalNumberPattern,\n    parsePattern: parseOrdinalNumberPattern,\n    valueCallback: (value) => parseInt(value, 10),\n  }),\n\n  era: buildMatchFn({\n    matchPatterns: matchEraPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseEraPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  quarter: buildMatchFn({\n    matchPatterns: matchQuarterPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseQuarterPatterns,\n    defaultParseWidth: \"any\",\n    valueCallback: (index) => (index + 1) as Quarter,\n  }),\n\n  month: buildMatchFn({\n    matchPatterns: matchMonthPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseMonthPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  day: buildMatchFn({\n    matchPatterns: matchDayPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseDayPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  dayPeriod: buildMatchFn({\n    matchPatterns: matchDayPeriodPatterns,\n    defaultMatchWidth: \"any\",\n    parsePatterns: parseDayPeriodPatterns,\n    defaultParseWidth: \"any\",\n  }),\n};\n\n// Path: src/locale/nn/_lib/formatDistance/index.ts\nimport type { FormatDistanceFn, FormatDistanceLocale } from \"../../../types.js\";\n\ntype FormatDistanceTokenValue =\n  | string\n  | {\n      one: string;\n      other: string;\n    };\n\nconst formatDistanceLocale: FormatDistanceLocale<FormatDistanceTokenValue> = {\n  lessThanXSeconds: {\n    one: \"mindre enn eitt sekund\",\n    other: \"mindre enn {{count}} sekund\",\n  },\n\n  xSeconds: {\n    one: \"eitt sekund\",\n    other: \"{{count}} sekund\",\n  },\n\n  halfAMinute: \"eit halvt minutt\",\n\n  lessThanXMinutes: {\n    one: \"mindre enn eitt minutt\",\n    other: \"mindre enn {{count}} minutt\",\n  },\n\n  xMinutes: {\n    one: \"eitt minutt\",\n    other: \"{{count}} minutt\",\n  },\n\n  aboutXHours: {\n    one: \"omtrent ein time\",\n    other: \"omtrent {{count}} timar\",\n  },\n\n  xHours: {\n    one: \"ein time\",\n    other: \"{{count}} timar\",\n  },\n\n  xDays: {\n    one: \"ein dag\",\n    other: \"{{count}} dagar\",\n  },\n\n  aboutXWeeks: {\n    one: \"omtrent ei veke\",\n    other: \"omtrent {{count}} veker\",\n  },\n\n  xWeeks: {\n    one: \"ei veke\",\n    other: \"{{count}} veker\",\n  },\n\n  aboutXMonths: {\n    one: \"omtrent ein m\u00e5nad\",\n    other: \"omtrent {{count}} m\u00e5nader\",\n  },\n\n  xMonths: {\n    one: \"ein m\u00e5nad\",\n    other: \"{{count}} m\u00e5nader\",\n  },\n\n  aboutXYears: {\n    one: \"omtrent eitt \u00e5r\",\n    other: \"omtrent {{count}} \u00e5r\",\n  },\n\n  xYears: {\n    one: \"eitt \u00e5r\",\n    other: \"{{count}} \u00e5r\",\n  },\n\n  overXYears: {\n    one: \"over eitt \u00e5r\",\n    other: \"over {{count}} \u00e5r\",\n  },\n\n  almostXYears: {\n    one: \"nesten eitt \u00e5r\",\n    other: \"nesten {{count}} \u00e5r\",\n  },\n};\n\nconst wordMapping = [\n  \"null\",\n  \"ein\",\n  \"to\",\n  \"tre\",\n  \"fire\",\n  \"fem\",\n  \"seks\",\n  \"sju\",\n  \"\u00e5tte\",\n  \"ni\",\n  \"ti\",\n  \"elleve\",\n  \"tolv\",\n];\n\nexport const formatDistance: FormatDistanceFn = (token, count, options) => {\n  let result;\n\n  const tokenValue = formatDistanceLocale[token];\n  if (typeof tokenValue === \"string\") {\n    result = tokenValue;\n  } else if (count === 1) {\n    result = tokenValue.one;\n  } else {\n    result = tokenValue.other.replace(\n      \"{{count}}\",\n      count < 13 ? wordMapping[count] : String(count),\n    );\n  }\n\n  if (options?.addSuffix) {\n    if (options.comparison && options.comparison > 0) {\n      return \"om \" + result;\n    } else {\n      return result + \" sidan\";\n    }\n  }\n\n  return result;\n};\n\n// Path: src/locale/nn/_lib/formatRelative/index.ts\nimport type { FormatRelativeFn } from \"../../../types.js\";\n\nconst formatRelativeLocale = {\n  lastWeek: \"'f\u00f8rre' eeee 'kl.' p\",\n  yesterday: \"'i g\u00e5r kl.' p\",\n  today: \"'i dag kl.' p\",\n  tomorrow: \"'i morgon kl.' p\",\n  nextWeek: \"EEEE 'kl.' p\",\n  other: \"P\",\n};\n\nexport const formatRelative: FormatRelativeFn = (\n  token,\n  _date,\n  _baseDate,\n  _options,\n) => formatRelativeLocale[token];\n\n// Path: src/locale/nn/_lib/formatLong/index.ts\nimport type { FormatLong } from \"../../../types.js\";\nimport { buildFormatLongFn } from \"../../../_lib/buildFormatLongFn/index.js\";\n\nconst dateFormats = {\n  full: \"EEEE d. MMMM y\",\n  long: \"d. MMMM y\",\n  medium: \"d. MMM y\",\n  short: \"dd.MM.y\",\n};\n\nconst timeFormats = {\n  full: \"'kl'. HH:mm:ss zzzz\",\n  long: \"HH:mm:ss z\",\n  medium: \"HH:mm:ss\",\n  short: \"HH:mm\",\n};\n\nconst dateTimeFormats = {\n  full: \"{{date}} 'kl.' {{time}}\",\n  long: \"{{date}} 'kl.' {{time}}\",\n  medium: \"{{date}} {{time}}\",\n  short: \"{{date}} {{time}}\",\n};\n\nexport const formatLong: FormatLong = {\n  date: buildFormatLongFn({\n    formats: dateFormats,\n    defaultWidth: \"full\",\n  }),\n\n  time: buildFormatLongFn({\n    formats: timeFormats,\n    defaultWidth: \"full\",\n  }),\n\n  dateTime: buildFormatLongFn({\n    formats: dateTimeFormats,\n    defaultWidth: \"full\",\n  }),\n};\n\n// Path: src/locale/nn/index.ts\nimport type { Locale } from \"../types.js\";\nimport { formatDistance } from \"./_lib/formatDistance/index.js\";\nimport { formatLong } from \"./_lib/formatLong/index.js\";\nimport { formatRelative } from \"./_lib/formatRelative/index.js\";\nimport { localize } from \"./_lib/localize/index.js\";\nimport { match } from \"./_lib/match/index.js\";\n\n/**\n * @category Locales\n * @summary Norwegian Nynorsk locale.\n * @language Norwegian Nynorsk\n * @iso-639-2 nno\n * @author Mats Byrkjeland [@draperunner](https://github.com/draperunner)\n */\nexport const nn: Locale = {\n  code: \"nn\",\n  formatDistance: formatDistance,\n  formatLong: formatLong,\n  formatRelative: formatRelative,\n  localize: localize,\n  match: match,\n  options: {\n    weekStartsOn: 1 /* Monday */,\n    firstWeekContainsDate: 4,\n  },\n};\n\n// Path: src/locale/mk/_lib/formatRelative/index.ts\nimport { isSameWeek } from \"../../../../isSameWeek/index.js\";\nimport type { Day } from \"../../../../types.js\";\nimport type {\n  FormatRelativeFn,\n  FormatRelativeFnOptions,\n} from \"../../../types.js\";\n\nconst weekdays = [\n  \"\u043d\u0435\u0434\u0435\u043b\u0430\",\n  \"\u043f\u043e\u043d\u0435\u0434\u0435\u043b\u043d\u0438\u043a\",\n  \"\u0432\u0442\u043e\u0440\u043d\u0438\u043a\",\n  \"\u0441\u0440\u0435\u0434\u0430\",\n  \"\u0447\u0435\u0442\u0432\u0440\u0442\u043e\u043a\",\n  \"\u043f\u0435\u0442\u043e\u043a\",\n  \"\u0441\u0430\u0431\u043e\u0442\u0430\",\n];\n\nfunction lastWeek(day: Day): string {\n  const weekday = weekdays[day];\n\n  switch (day) {\n    case 0:\n    case 3:\n    case 6:\n      return \"'\u043c\u0438\u043d\u0430\u0442\u0430\u0442\u0430 \" + weekday + \" \u0432\u043e' p\";\n    case 1:\n    case 2:\n    case 4:\n    case 5:\n      return \"'\u043c\u0438\u043d\u0430\u0442\u0438\u043e\u0442 \" + weekday + \" \u0432\u043e' p\";\n  }\n}\n\nfunction thisWeek(day: Day): string {\n  const weekday = weekdays[day];\n\n  switch (day) {\n    case 0:\n    case 3:\n    case 6:\n      return \"'\u043e\u0432\u0430 \" + weekday + \" \u0432o' p\";\n    case 1:\n    case 2:\n    case 4:\n    case 5:\n      return \"'\u043e\u0432\u043e\u0458 \" + weekday + \" \u0432o' p\";\n  }\n}\n\nfunction nextWeek(day: Day): string {\n  const weekday = weekdays[day];\n\n  switch (day) {\n    case 0:\n    case 3:\n    case 6:\n      return \"'\u0441\u043b\u0435\u0434\u043d\u0430\u0442\u0430 \" + weekday + \" \u0432o' p\";\n    case 1:\n    case 2:\n    case 4:\n    case 5:\n      return \"'\u0441\u043b\u0435\u0434\u043d\u0438\u043e\u0442 \" + weekday + \" \u0432o' p\";\n  }\n}\n\nconst formatRelativeLocale = {\n  lastWeek: <DateType extends Date>(\n    date: DateType,\n    baseDate: DateType,\n    options?: FormatRelativeFnOptions,\n  ): string => {\n    const day = date.getDay() as Day;\n    if (isSameWeek(date, baseDate, options)) {\n      return thisWeek(day);\n    } else {\n      return lastWeek(day);\n    }\n  },\n  yesterday: \"'\u0432\u0447\u0435\u0440\u0430 \u0432\u043e' p\",\n  today: \"'\u0434\u0435\u043d\u0435\u0441 \u0432\u043e' p\",\n  tomorrow: \"'\u0443\u0442\u0440\u0435 \u0432\u043e' p\",\n  nextWeek: <DateType extends Date>(\n    date: DateType,\n    baseDate: DateType,\n    options?: FormatRelativeFnOptions,\n  ): string => {\n    const day = date.getDay() as Day;\n    if (isSameWeek(date, baseDate, options)) {\n      return thisWeek(day);\n    } else {\n      return nextWeek(day);\n    }\n  },\n  other: \"P\",\n};\n\nexport const formatRelative: FormatRelativeFn = (\n  token,\n  date,\n  baseDate,\n  options,\n) => {\n  const format = formatRelativeLocale[token];\n\n  if (typeof format === \"function\") {\n    return format(date, baseDate, options);\n  }\n\n  return format;\n};\n\n// Path: src/locale/mk/_lib/formatDistance/index.ts\nimport type { FormatDistanceFn, FormatDistanceLocale } from \"../../../types.js\";\n\ntype FormatDistanceTokenValue =\n  | string\n  | {\n      one: string;\n      other: string;\n    };\n\nconst formatDistanceLocale: FormatDistanceLocale<FormatDistanceTokenValue> = {\n  lessThanXSeconds: {\n    one: \"\u043f\u043e\u043c\u0430\u043b\u043a\u0443 \u043e\u0434 \u0441\u0435\u043a\u0443\u043d\u0434\u0430\",\n    other: \"\u043f\u043e\u043c\u0430\u043b\u043a\u0443 \u043e\u0434 {{count}} \u0441\u0435\u043a\u0443\u043d\u0434\u0438\",\n  },\n\n  xSeconds: {\n    one: \"1 \u0441\u0435\u043a\u0443\u043d\u0434\u0430\",\n    other: \"{{count}} \u0441\u0435\u043a\u0443\u043d\u0434\u0438\",\n  },\n\n  halfAMinute: \"\u043f\u043e\u043b\u043e\u0432\u0438\u043d\u0430 \u043c\u0438\u043d\u0443\u0442\u0430\",\n\n  lessThanXMinutes: {\n    one: \"\u043f\u043e\u043c\u0430\u043b\u043a\u0443 \u043e\u0434 \u043c\u0438\u043d\u0443\u0442\u0430\",\n    other: \"\u043f\u043e\u043c\u0430\u043b\u043a\u0443 \u043e\u0434 {{count}} \u043c\u0438\u043d\u0443\u0442\u0438\",\n  },\n\n  xMinutes: {\n    one: \"1 \u043c\u0438\u043d\u0443\u0442\u0430\",\n    other: \"{{count}} \u043c\u0438\u043d\u0443\u0442\u0438\",\n  },\n\n  aboutXHours: {\n    one: \"\u043e\u043a\u043e\u043b\u0443 1 \u0447\u0430\u0441\",\n    other: \"\u043e\u043a\u043e\u043b\u0443 {{count}} \u0447\u0430\u0441\u0430\",\n  },\n\n  xHours: {\n    one: \"1 \u0447\u0430\u0441\",\n    other: \"{{count}} \u0447\u0430\u0441\u0430\",\n  },\n\n  xDays: {\n    one: \"1 \u0434\u0435\u043d\",\n    other: \"{{count}} \u0434\u0435\u043d\u0430\",\n  },\n\n  aboutXWeeks: {\n    one: \"\u043e\u043a\u043e\u043b\u0443 1 \u043d\u0435\u0434\u0435\u043b\u0430\",\n    other: \"\u043e\u043a\u043e\u043b\u0443 {{count}} \u043c\u0435\u0441\u0435\u0446\u0438\",\n  },\n\n  xWeeks: {\n    one: \"1 \u043d\u0435\u0434\u0435\u043b\u0430\",\n    other: \"{{count}} \u043d\u0435\u0434\u0435\u043b\u0438\",\n  },\n\n  aboutXMonths: {\n    one: \"\u043e\u043a\u043e\u043b\u0443 1 \u043c\u0435\u0441\u0435\u0446\",\n    other: \"\u043e\u043a\u043e\u043b\u0443 {{count}} \u043d\u0435\u0434\u0435\u043b\u0438\",\n  },\n\n  xMonths: {\n    one: \"1 \u043c\u0435\u0441\u0435\u0446\",\n    other: \"{{count}} \u043c\u0435\u0441\u0435\u0446\u0438\",\n  },\n\n  aboutXYears: {\n    one: \"\u043e\u043a\u043e\u043b\u0443 1 \u0433\u043e\u0434\u0438\u043d\u0430\",\n    other: \"\u043e\u043a\u043e\u043b\u0443 {{count}} \u0433\u043e\u0434\u0438\u043d\u0438\",\n  },\n\n  xYears: {\n    one: \"1 \u0433\u043e\u0434\u0438\u043d\u0430\",\n    other: \"{{count}} \u0433\u043e\u0434\u0438\u043d\u0438\",\n  },\n\n  overXYears: {\n    one: \"\u043f\u043e\u0432\u0435\u045c\u0435 \u043e\u0434 1 \u0433\u043e\u0434\u0438\u043d\u0430\",\n    other: \"\u043f\u043e\u0432\u0435\u045c\u0435 \u043e\u0434 {{count}} \u0433\u043e\u0434\u0438\u043d\u0438\",\n  },\n\n  almostXYears: {\n    one: \"\u0431\u0435\u0437\u043c\u0430\u043b\u043a\u0443 1 \u0433\u043e\u0434\u0438\u043d\u0430\",\n    other: \"\u0431\u0435\u0437\u043c\u0430\u043b\u043a\u0443 {{count}} \u0433\u043e\u0434\u0438\u043d\u0438\",\n  },\n};\n\nexport const formatDistance: FormatDistanceFn = (token, count, options) => {\n  let result;\n\n  const tokenValue = formatDistanceLocale[token];\n  if (typeof tokenValue === \"string\") {\n    result = tokenValue;\n  } else if (count === 1) {\n    result = tokenValue.one;\n  } else {\n    result = tokenValue.other.replace(\"{{count}}\", String(count));\n  }\n\n  if (options?.addSuffix) {\n    if (options.comparison && options.comparison > 0) {\n      return \"\u0437\u0430 \" + result;\n    } else {\n      return \"\u043f\u0440\u0435\u0434 \" + result;\n    }\n  }\n\n  return result;\n};\n\n// Path: src/locale/mk/_lib/formatLong/index.ts\nimport type { FormatLong } from \"../../../types.js\";\nimport { buildFormatLongFn } from \"../../../_lib/buildFormatLongFn/index.js\";\n\nconst dateFormats = {\n  full: \"EEEE, dd MMMM yyyy\",\n  long: \"dd MMMM yyyy\",\n  medium: \"dd MMM yyyy\",\n  short: \"dd/MM/yyyy\",\n};\n\nconst timeFormats = {\n  full: \"HH:mm:ss zzzz\",\n  long: \"HH:mm:ss z\",\n  medium: \"HH:mm:ss\",\n  short: \"H:mm\",\n};\n\nconst dateTimeFormats = {\n  any: \"{{date}} {{time}}\",\n};\n\nexport const formatLong: FormatLong = {\n  date: buildFormatLongFn({\n    formats: dateFormats,\n    defaultWidth: \"full\",\n  }),\n\n  time: buildFormatLongFn({\n    formats: timeFormats,\n    defaultWidth: \"full\",\n  }),\n\n  dateTime: buildFormatLongFn({\n    formats: dateTimeFormats,\n    defaultWidth: \"any\",\n  }),\n};\n\n// Path: src/locale/mk/_lib/match/index.ts\nimport type { Quarter } from \"../../../../types.js\";\nimport type { Match } from \"../../../types.js\";\nimport { buildMatchFn } from \"../../../_lib/buildMatchFn/index.js\";\nimport { buildMatchPatternFn } from \"../../../_lib/buildMatchPatternFn/index.js\";\n\nconst matchOrdinalNumberPattern = /^(\\d+)(-?[\u0432\u0440\u043c\u0442][\u0438])?/i;\nconst parseOrdinalNumberPattern = /\\d+/i;\n\nconst matchEraPatterns = {\n  narrow: /^((\u043f\u0440)?\u043d\\.?\\s?\u0435\\.?)/i,\n  abbreviated: /^((\u043f\u0440)?\u043d\\.?\\s?\u0435\\.?)/i,\n  wide: /^(\u043f\u0440\u0435\u0434 \u043d\u0430\u0448\u0430\u0442\u0430 \u0435\u0440\u0430|\u043d\u0430\u0448\u0430\u0442\u0430 \u0435\u0440\u0430)/i,\n};\nconst parseEraPatterns = {\n  any: [/^\u043f/i, /^\u043d/i] as const,\n};\n\nconst matchQuarterPatterns = {\n  narrow: /^[1234]/i,\n  abbreviated: /^[1234](-?[\u0432\u0440\u0442]?\u0438?)? \u043a\u0432.?/i,\n  wide: /^[1234](-?[\u0432\u0440\u0442]?\u0438?)? \u043a\u0432\u0430\u0440\u0442\u0430\u043b/i,\n};\nconst parseQuarterPatterns = {\n  any: [/1/i, /2/i, /3/i, /4/i] as const,\n};\n\nconst matchDayPatterns = {\n  narrow: /^[\u043d\u043f\u0432\u0441\u0447]/i,\n  short: /^(\u043d\u0435|\u043f\u043e|\u0432\u0442|\u0441\u0440|\u0447\u0435|\u043f\u0435|\u0441\u0430)/i,\n  abbreviated: /^(\u043d\u0435\u0434|\u043f\u043e\u043d|\u0432\u0442\u043e|\u0441\u0440\u0435|\u0447\u0435\u0442|\u043f\u0435\u0442|\u0441\u0430\u0431)/i,\n  wide: /^(\u043d\u0435\u0434\u0435\u043b\u0430|\u043f\u043e\u043d\u0435\u0434\u0435\u043b\u043d\u0438\u043a|\u0432\u0442\u043e\u0440\u043d\u0438\u043a|\u0441\u0440\u0435\u0434\u0430|\u0447\u0435\u0442\u0432\u0440\u0442\u043e\u043a|\u043f\u0435\u0442\u043e\u043a|\u0441\u0430\u0431\u043e\u0442\u0430)/i,\n};\n\nconst parseDayPatterns = {\n  narrow: [/^\u043d/i, /^\u043f/i, /^\u0432/i, /^\u0441/i, /^\u0447/i, /^\u043f/i, /^\u0441/i] as const,\n  any: [\n    /^\u043d[\u0435\u0434]/i,\n    /^\u043f[\u043e\u043d]/i,\n    /^\u0432\u0442/i,\n    /^\u0441\u0440/i,\n    /^\u0447[\u0435\u0442]/i,\n    /^\u043f[\u0435\u0442]/i,\n    /^\u0441[\u0430\u0431]/i,\n  ] as const,\n};\n\nconst matchMonthPatterns = {\n  abbreviated: /^(\u0458\u0430\u043d|\u0444\u0435\u0432|\u043c\u0430\u0440|\u0430\u043f\u0440|\u043c\u0430\u0458|\u0458\u0443\u043d|\u0458\u0443\u043b|\u0430\u0432\u0433|\u0441\u0435\u043f|\u043e\u043a\u0442|\u043d\u043e\u0435\u043c|\u0434\u0435\u043a)/i,\n  wide: /^(\u0458\u0430\u043d\u0443\u0430\u0440\u0438|\u0444\u0435\u0432\u0440\u0443\u0430\u0440\u0438|\u043c\u0430\u0440\u0442|\u0430\u043f\u0440\u0438\u043b|\u043c\u0430\u0458|\u0458\u0443\u043d\u0438|\u0458\u0443\u043b\u0438|\u0430\u0432\u0433\u0443\u0441\u0442|\u0441\u0435\u043f\u0442\u0435\u043c\u0432\u0440\u0438|\u043e\u043a\u0442\u043e\u043c\u0432\u0440\u0438|\u043d\u043e\u0435\u043c\u0432\u0440\u0438|\u0434\u0435\u043a\u0435\u043c\u0432\u0440\u0438)/i,\n};\nconst parseMonthPatterns = {\n  any: [\n    /^\u0458\u0430/i,\n    /^\u0424/i,\n    /^\u043c\u0430\u0440/i,\n    /^\u0430\u043f/i,\n    /^\u043c\u0430\u0458/i,\n    /^\u0458\u0443\u043d/i,\n    /^\u0458\u0443\u043b/i,\n    /^\u0430\u0432/i,\n    /^\u0441\u0435/i,\n    /^\u043e\u043a\u0442/i,\n    /^\u043d\u043e/i,\n    /^\u0434\u0435/i,\n  ] as const,\n};\n\nconst matchDayPeriodPatterns = {\n  any: /^(\u043f\u0440\u0435\u0442\u043f|\u043f\u043e\u043f\u043b|\u043f\u043e\u043b\u043d\u043e\u045c|\u0443\u0442\u0440\u043e|\u043f\u043b\u0430\u0434\u043d\u0435|\u0432\u0435\u0447\u0435\u0440|\u043d\u043e\u045c)/i,\n};\nconst parseDayPeriodPatterns = {\n  any: {\n    am: /\u043f\u0440\u0435\u0442\u043f\u043b\u0430\u0434\u043d\u0435/i,\n    pm: /\u043f\u043e\u043f\u043b\u0430\u0434\u043d\u0435/i,\n    midnight: /\u043f\u043e\u043b\u043d\u043e\u045c/i,\n    noon: /\u043d\u0430\u043f\u043b\u0430\u0434\u043d\u0435/i,\n    morning: /\u043d\u0430\u0443\u0442\u0440\u043e/i,\n    afternoon: /\u043f\u043e\u043f\u043b\u0430\u0434\u043d\u0435/i,\n    evening: /\u043d\u0430\u0432\u0435\u0447\u0435\u0440/i,\n    night: /\u043d\u043e\u045c\u0435/i,\n  },\n};\n\nexport const match: Match = {\n  ordinalNumber: buildMatchPatternFn({\n    matchPattern: matchOrdinalNumberPattern,\n    parsePattern: parseOrdinalNumberPattern,\n    valueCallback: (value) => parseInt(value, 10),\n  }),\n\n  era: buildMatchFn({\n    matchPatterns: matchEraPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseEraPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  quarter: buildMatchFn({\n    matchPatterns: matchQuarterPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseQuarterPatterns,\n    defaultParseWidth: \"any\",\n    valueCallback: (index) => (index + 1) as Quarter,\n  }),\n\n  month: buildMatchFn({\n    matchPatterns: matchMonthPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseMonthPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  day: buildMatchFn({\n    matchPatterns: matchDayPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseDayPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  dayPeriod: buildMatchFn({\n    matchPatterns: matchDayPeriodPatterns,\n    defaultMatchWidth: \"any\",\n    parsePatterns: parseDayPeriodPatterns,\n    defaultParseWidth: \"any\",\n  }),\n};\n\n// Path: src/locale/mk/_lib/localize/index.ts\nimport type { Localize, LocalizeFn } from \"../../../types.js\";\nimport { buildLocalizeFn } from \"../../../_lib/buildLocalizeFn/index.js\";\n\nconst eraValues = {\n  narrow: [\"\u043f\u0440.\u043d.\u0435.\", \"\u043d.\u0435.\"] as const,\n  abbreviated: [\"\u043f\u0440\u0435\u0434 \u043d. \u0435.\", \"\u043d. \u0435.\"] as const,\n  wide: [\"\u043f\u0440\u0435\u0434 \u043d\u0430\u0448\u0430\u0442\u0430 \u0435\u0440\u0430\", \"\u043d\u0430\u0448\u0430\u0442\u0430 \u0435\u0440\u0430\"] as const,\n};\n\nconst quarterValues = {\n  narrow: [\"1\", \"2\", \"3\", \"4\"] as const,\n  abbreviated: [\"1-\u0432\u0438 \u043a\u0432.\", \"2-\u0440\u0438 \u043a\u0432.\", \"3-\u0442\u0438 \u043a\u0432.\", \"4-\u0442\u0438 \u043a\u0432.\"] as const,\n  wide: [\n    \"1-\u0432\u0438 \u043a\u0432\u0430\u0440\u0442\u0430\u043b\",\n    \"2-\u0440\u0438 \u043a\u0432\u0430\u0440\u0442\u0430\u043b\",\n    \"3-\u0442\u0438 \u043a\u0432\u0430\u0440\u0442\u0430\u043b\",\n    \"4-\u0442\u0438 \u043a\u0432\u0430\u0440\u0442\u0430\u043b\",\n  ] as const,\n};\n\nconst monthValues = {\n  abbreviated: [\n    \"\u0458\u0430\u043d\",\n    \"\u0444\u0435\u0432\",\n    \"\u043c\u0430\u0440\",\n    \"\u0430\u043f\u0440\",\n    \"\u043c\u0430\u0458\",\n    \"\u0458\u0443\u043d\",\n    \"\u0458\u0443\u043b\",\n    \"\u0430\u0432\u0433\",\n    \"\u0441\u0435\u043f\u0442\",\n    \"\u043e\u043a\u0442\",\n    \"\u043d\u043e\u0435\u043c\",\n    \"\u0434\u0435\u043a\",\n  ] as const,\n  wide: [\n    \"\u0458\u0430\u043d\u0443\u0430\u0440\u0438\",\n    \"\u0444\u0435\u0432\u0440\u0443\u0430\u0440\u0438\",\n    \"\u043c\u0430\u0440\u0442\",\n    \"\u0430\u043f\u0440\u0438\u043b\",\n    \"\u043c\u0430\u0458\",\n    \"\u0458\u0443\u043d\u0438\",\n    \"\u0458\u0443\u043b\u0438\",\n    \"\u0430\u0432\u0433\u0443\u0441\u0442\",\n    \"\u0441\u0435\u043f\u0442\u0435\u043c\u0432\u0440\u0438\",\n    \"\u043e\u043a\u0442\u043e\u043c\u0432\u0440\u0438\",\n    \"\u043d\u043e\u0435\u043c\u0432\u0440\u0438\",\n    \"\u0434\u0435\u043a\u0435\u043c\u0432\u0440\u0438\",\n  ] as const,\n};\n\nconst dayValues = {\n  narrow: [\"\u041d\", \"\u041f\", \"\u0412\", \"\u0421\", \"\u0427\", \"\u041f\", \"\u0421\"] as const,\n  short: [\"\u043d\u0435\", \"\u043f\u043e\", \"\u0432\u0442\", \"\u0441\u0440\", \"\u0447\u0435\", \"\u043f\u0435\", \"\u0441\u0430\"] as const,\n  abbreviated: [\"\u043d\u0435\u0434\", \"\u043f\u043e\u043d\", \"\u0432\u0442\u043e\", \"\u0441\u0440\u0435\", \"\u0447\u0435\u0442\", \"\u043f\u0435\u0442\", \"\u0441\u0430\u0431\"] as const,\n  wide: [\n    \"\u043d\u0435\u0434\u0435\u043b\u0430\",\n    \"\u043f\u043e\u043d\u0435\u0434\u0435\u043b\u043d\u0438\u043a\",\n    \"\u0432\u0442\u043e\u0440\u043d\u0438\u043a\",\n    \"\u0441\u0440\u0435\u0434\u0430\",\n    \"\u0447\u0435\u0442\u0432\u0440\u0442\u043e\u043a\",\n    \"\u043f\u0435\u0442\u043e\u043a\",\n    \"\u0441\u0430\u0431\u043e\u0442\u0430\",\n  ] as const,\n};\n\nconst dayPeriodValues = {\n  wide: {\n    am: \"\u043f\u0440\u0435\u0442\u043f\u043b\u0430\u0434\u043d\u0435\",\n    pm: \"\u043f\u043e\u043f\u043b\u0430\u0434\u043d\u0435\",\n    midnight: \"\u043f\u043e\u043b\u043d\u043e\u045c\",\n    noon: \"\u043d\u0430\u043f\u043b\u0430\u0434\u043d\u0435\",\n    morning: \"\u043d\u0430\u0443\u0442\u0440\u043e\",\n    afternoon: \"\u043f\u043e\u043f\u043b\u0430\u0434\u043d\u0435\",\n    evening: \"\u043d\u0430\u0432\u0435\u0447\u0435\u0440\",\n    night: \"\u043d\u043e\u045c\u0435\",\n  },\n};\n\nconst ordinalNumber: LocalizeFn<number> = (dirtyNumber, _options) => {\n  const number = Number(dirtyNumber);\n\n  const rem100 = number % 100;\n  if (rem100 > 20 || rem100 < 10) {\n    switch (rem100 % 10) {\n      case 1:\n        return number + \"-\u0432\u0438\";\n      case 2:\n        return number + \"-\u0440\u0438\";\n      case 7:\n      case 8:\n        return number + \"-\u043c\u0438\";\n    }\n  }\n  return number + \"-\u0442\u0438\";\n};\n\nexport const localize: Localize = {\n  ordinalNumber,\n\n  era: buildLocalizeFn({\n    values: eraValues,\n    defaultWidth: \"wide\",\n  }),\n\n  quarter: buildLocalizeFn({\n    values: quarterValues,\n    defaultWidth: \"wide\",\n    argumentCallback: (quarter) => quarter - 1,\n  }),\n\n  month: buildLocalizeFn({\n    values: monthValues,\n    defaultWidth: \"wide\",\n  }),\n\n  day: buildLocalizeFn({\n    values: dayValues,\n    defaultWidth: \"wide\",\n  }),\n\n  dayPeriod: buildLocalizeFn({\n    values: dayPeriodValues,\n    defaultWidth: \"wide\",\n  }),\n};\n\n// Path: src/locale/mk/index.ts\nimport type { Locale } from \"../types.js\";\nimport { formatDistance } from \"./_lib/formatDistance/index.js\";\nimport { formatLong } from \"./_lib/formatLong/index.js\";\nimport { formatRelative } from \"./_lib/formatRelative/index.js\";\nimport { localize } from \"./_lib/localize/index.js\";\nimport { match } from \"./_lib/match/index.js\";\n\n/**\n * @category Locales\n * @summary Macedonian locale.\n * @language Macedonian\n * @iso-639-2 mkd\n * @author Petar Vlahu [@vlahupetar](https://github.com/vlahupetar)\n * @author Altrim Beqiri [@altrim](https://github.com/altrim)\n */\nexport const mk: Locale = {\n  code: \"mk\",\n  formatDistance: formatDistance,\n  formatLong: formatLong,\n  formatRelative: formatRelative,\n  localize: localize,\n  match: match,\n  options: {\n    weekStartsOn: 1 /* Monday */,\n    firstWeekContainsDate: 4,\n  },\n};\n\n// Path: src/locale/se/_lib/match/index.ts\nimport type { Quarter } from \"../../../../types.js\";\nimport type { Match } from \"../../../types.js\";\nimport { buildMatchFn } from \"../../../_lib/buildMatchFn/index.js\";\nimport { buildMatchPatternFn } from \"../../../_lib/buildMatchPatternFn/index.js\";\n\nconst matchOrdinalNumberPattern = /^(\\d+)\\.?/i;\nconst parseOrdinalNumberPattern = /\\d+/i;\n\nconst matchEraPatterns = {\n  narrow: /^(o\\.? ?Kr\\.?|m\\.? ?Kr\\.?)/i,\n  abbreviated: /^(o\\.? ?Kr\\.?|m\\.? ?Kr\\.?)/i,\n  wide: /^(ovdal Kristusa|ovdal min \u00e1iggi|ma\u014b\u014bel Kristusa|min \u00e1igi)/i,\n};\nconst parseEraPatterns = {\n  any: [/^o/i, /^m/i] as const,\n};\n\nconst matchQuarterPatterns = {\n  narrow: /^[1234]/i,\n  abbreviated: /^q[1234]/i,\n  wide: /^[1234](\\.)? kvart\u00e1la/i,\n};\nconst parseQuarterPatterns = {\n  any: [/1/i, /2/i, /3/i, /4/i] as const,\n};\n\nconst matchMonthPatterns = {\n  narrow: /^[ogncmsb\u010dj]/i,\n  abbreviated:\n    /^(o\u0111\u0111a|guov|njuk|cuo|mies|geas|suoi|borg|\u010dak\u010d|golg|sk\u00e1b|juov)\\.?/i,\n  wide: /^(o\u0111\u0111ajagem\u00e1nnu|guovvam\u00e1nnu|njuk\u010dam\u00e1nnu|cuo\u014bom\u00e1nnu|miessem\u00e1nnu|geassem\u00e1nnu|suoidnem\u00e1nnu|borgem\u00e1nnu|\u010dak\u010dam\u00e1nnu|golggotm\u00e1nnu|sk\u00e1bmam\u00e1nnu|juovlam\u00e1nnu)/i,\n};\nconst parseMonthPatterns = {\n  narrow: [\n    /^o/i,\n    /^g/i,\n    /^n/i,\n    /^c/i,\n    /^m/i,\n    /^g/i,\n    /^s/i,\n    /^b/i,\n    /^\u010d/i,\n    /^g/i,\n    /^s/i,\n    /^j/i,\n  ] as const,\n  any: [\n    /^o/i,\n    /^gu/i,\n    /^n/i,\n    /^c/i,\n    /^m/i,\n    /^ge/i,\n    /^su/i,\n    /^b/i,\n    /^\u010d/i,\n    /^go/i,\n    /^sk/i,\n    /^j/i,\n  ] as const,\n};\n\nconst matchDayPatterns = {\n  narrow: /^[svmgdbl]/i,\n  short: /^(sotn|vuos|ma\u014b|gask|duor|bear|l\u00e1v)/i,\n  abbreviated: /^(sotn|vuos|ma\u014b|gask|duor|bear|l\u00e1v)/i,\n  wide: /^(sotnabeaivi|vuoss\u00e1rga|ma\u014b\u014beb\u00e1rga|gaskavahkku|duorastat|bearjadat|l\u00e1vvardat)/i,\n};\nconst parseDayPatterns = {\n  any: [/^s/i, /^v/i, /^m/i, /^g/i, /^d/i, /^b/i, /^l/i] as const,\n};\n\nconst matchDayPeriodPatterns = {\n  narrow:\n    /^(gaskaidja|gaskabeaivvi|(p\u00e5) (i\u0111\u0111es|ma\u014b\u014bel gaskabeaivvi|eahkes|ihkku)|[ap])/i,\n  any: /^([ap]\\.?\\s?m\\.?|gaskaidja|gaskabeaivvi|(p\u00e5) (i\u0111\u0111es|ma\u014b\u014bel gaskabeaivvi|eahkes|ihkku))/i,\n};\nconst parseDayPeriodPatterns = {\n  any: {\n    am: /^a(\\.?\\s?m\\.?)?$/i,\n    pm: /^p(\\.?\\s?m\\.?)?$/i,\n    midnight: /^gaskai/i,\n    noon: /^gaskab/i,\n    morning: /i\u0111\u0111es/i,\n    afternoon: /ma\u014b\u014bel gaskabeaivvi/i,\n    evening: /eahkes/i,\n    night: /ihkku/i,\n  },\n};\n\nexport const match: Match = {\n  ordinalNumber: buildMatchPatternFn({\n    matchPattern: matchOrdinalNumberPattern,\n    parsePattern: parseOrdinalNumberPattern,\n    valueCallback: (value) => parseInt(value, 10),\n  }),\n\n  era: buildMatchFn({\n    matchPatterns: matchEraPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseEraPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  quarter: buildMatchFn({\n    matchPatterns: matchQuarterPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseQuarterPatterns,\n    defaultParseWidth: \"any\",\n    valueCallback: (index) => (index + 1) as Quarter,\n  }),\n\n  month: buildMatchFn({\n    matchPatterns: matchMonthPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseMonthPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  day: buildMatchFn({\n    matchPatterns: matchDayPatterns,\n    defaultMatchWidth: \"wide\",\n    parsePatterns: parseDayPatterns,\n    defaultParseWidth: \"any\",\n  }),\n\n  dayPeriod: buildMatchFn({\n    matchPatterns: matchDayPeriodPatterns,\n    defaultMatchWidth: \"any\",\n    parsePatterns: parseDayPeriodPatterns,\n    defaultParseWidth: \"any\",\n  }),\n};\n\n// Path: src/locale/se/_lib/formatDistance/index.ts\nimport type { FormatDistanceFn, FormatDistanceLocale } from \"../../../types.js\";\n\ntype FormatDistanceTokenValue =\n  | string\n  | {\n      one: string;\n      other: string;\n    };\n\nconst formatDistanceLocale: FormatDistanceLocale<FormatDistanceTokenValue> = {\n  lessThanXSeconds: {\n    one: \"unnit go ovtta sekundda\",\n    other: \"unnit go {{count}} sekundda\",\n  },\n\n  xSeconds: {\n    one: \"sekundda\",\n    other: \"{{count}} sekundda\",\n  },\n\n  halfAMinute: \"bealle minuhta\",\n\n  lessThanXMinutes: {\n    one: \"unnit go bealle minuhta\",\n    other: \"unnit go {{count}} minuhta\",\n  },\n\n  xMinutes: {\n    one: \"minuhta\",\n    other: \"{{count}} minuhta\",\n  },\n\n  aboutXHours: {\n    one: \"sullii ovtta diimmu\",\n    other: \"sullii {{count}} diimmu\",\n  },\n\n  xHours: {\n    one: \"diimmu\",\n    other: \"{{count}} diimmu\",\n  },\n\n  xDays: {\n    one: \"beaivvi\",\n    other: \"{{count}} beaivvi\",\n  },\n\n  aboutXWeeks: {\n    one: \"sullii ovtta vahku\",\n    other: \"sullii {{count}} vahku\",\n  },\n\n  xWeeks: {\n    one: \"vahku\",\n    other: \"{{count}} vahku\",\n  },\n\n  aboutXMonths: {\n    one: \"sullii ovtta m\u00e1nu\",\n    other: \"sullii {{count}} m\u00e1nu\",\n  },\n\n  xMonths: {\n    one: \"m\u00e1nu\",\n    other: \"{{count}} m\u00e1nu\",\n  },\n\n  aboutXYears: {\n    one: \"sullii ovtta jagi\",\n    other: \"sullii {{count}} jagi\",\n  },\n\n  xYears: {\n    one: \"jagi\",\n    other: \"{{count}} jagi\",\n  },\n\n  overXYears: {\n    one: \"guhkit go jagi\",\n    other: \"guhkit go {{count}} jagi\",\n  },\n\n  almostXYears: {\n    one: \"measta jagi\",\n    other: \"measta {{count}} jagi\",\n  },\n};\n\nexport const formatDistance: FormatDistanceFn = (token, count, options) => {\n  let result;\n\n  const tokenValue = formatDistanceLocale[token];\n  if (typeof tokenValue === \"string\") {\n    result = tokenValue;\n  } else if (count === 1) {\n    result = tokenValue.one;\n  } else {\n    result = tokenValue.other.replace(\"{{count}}\", String(count));\n  }\n\n  if (options?.addSuffix) {\n    if (options.comparison && options.comparison > 0) {\n      return \"geah\u010den \" + result;\n    } else {\n      return result + \" \u00e1igi\";\n    }\n  }\n\n  return result;\n};\n\n// Path: src/locale/se/_lib/formatLong/index.ts\nimport type { FormatLong } from \"../../../types.js\";\nimport { buildFormatLongFn } from \"../../../_lib/buildFormatLongFn/index.js\";\n\nconst dateFormats = {\n  full: \"EEEE MMMM d. 'b.' y\",\n  long: \"MMMM d. 'b.' y\",\n  medium: \"MMM d. 'b.' y\",\n  short: \"dd.MM.y\",\n};\n\nconst timeFormats = {\n  full: \"'dii.' HH:mm:ss zzzz\",\n  long: \"HH:mm:ss z\",\n  medium: \"HH:mm:ss\",\n  short: \"HH:mm\",\n};\n\nconst dateTimeFormats = {\n  full: \"{{date}} 'dii.' {{time}}\",\n  long: \"{{date}} 'dii.' {{time}}\",\n  medium: \"{{date}} {{time}}\",\n  short: \"{{date}} {{time}}\",\n};\n\nexport const formatLong: FormatLong = {\n  date: buildFormatLongFn({\n    formats: dateFormats,\n    defaultWidth: \"full\",\n  }),\n\n  time: buildFormatLongFn({\n    formats: timeFormats,\n    defaultWidth: \"full\",\n  }),\n\n  dateTime: buildFormatLongFn({\n    formats: dateTimeFormats,\n    defaultWidth: \"full\",\n  }),\n};\n\n// Path: src/locale/se/_lib/localize/index.ts\nimport type { Localize, LocalizeFn } from \"../../../types.js\";\nimport { buildLocalizeFn } from \"../../../_lib/buildLocalizeFn/index.js\";\n\nconst eraValues = {\n  narrow: [\"o.Kr.\", \"m.Kr.\"] as const,\n  abbreviated: [\"o.Kr.\", \"m.Kr.\"] as const,\n  wide: [\"ovdal Kristusa\", \"ma\u014b\u014bel Kristusa\"] as const,\n};\n\nconst quarterValues = {\n  narrow: [\"1\", \"2\", \"3\", \"4\"] as const,\n  abbreviated: [\"Q1\", \"Q2\", \"Q3\", \"Q4\"] as const,\n  wide: [\"1. kvart\u00e1la\", \"2. kvart\u00e1la\", \"3. kvart\u00e1la\", \"4. kvart\u00e1la\"] as const,\n};\n\nconst monthValues = {\n  narrow: [\"O\", \"G\", \"N\", \"C\", \"M\", \"G\", \"S\", \"B\", \"\u010c\", \"G\", \"S\", \"J\"] as const,\n  abbreviated: [\n    \"o\u0111\u0111a\",\n    \"guov\",\n    \"njuk\",\n    \"cuo\",\n    \"mies\",\n    \"geas\",\n    \"suoi\",\n    \"borg\",\n    \"\u010dak\u010d\",\n    \"golg\",\n    \"sk\u00e1b\",\n    \"juov\",\n  ] as const,\n  wide: [\n    \"o\u0111\u0111ajagem\u00e1nnu\",\n    \"guovvam\u00e1nnu\",\n    \"njuk\u010dam\u00e1nnu\",\n    \"cuo\u014bom\u00e1nnu\",\n    \"miessem\u00e1nnu\",\n    \"geassem\u00e1nnu\",\n    \"suoidnem\u00e1nnu\",\n    \"borgem\u00e1nnu\",\n    \"\u010dak\u010dam\u00e1nnu\",\n    \"golggotm\u00e1nnu\",\n    \"sk\u00e1bmam\u00e1nnu\",\n    \"juovlam\u00e1nnu\",\n  ] as const,\n};\n\nconst dayValues = {\n  narrow: [\"S\", \"V\", \"M\", \"G\", \"D\", \"B\", \"L\"] as const,\n  short: [\"sotn\", \"vuos\", \"ma\u014b\", \"gask\", \"duor\", \"bear\", \"l\u00e1v\"] as const,\n  abbreviated: [\"sotn\", \"vuos\", \"ma\u014b\", \"gask\", \"duor\", \"bear\", \"l\u00e1v\"] as const,\n  wide: [\n    \"sotnabeaivi\",\n    \"vuoss\u00e1rga\",\n    \"ma\u014b\u014beb\u00e1rga\",\n    \"gaskavahkku\",\n    \"duorastat\",\n    \"bearjadat\",\n    \"l\u00e1vvardat\",\n  ] as const,\n};\n\nconst dayPeriodValues = {\n  narrow: {\n    am: \"a\",\n    pm: \"p\",\n    midnight: \"gaskaidja\",\n    noon: \"gaskabeaivi\",\n    morning: \"i\u0111\u0111es\",\n    afternoon: \"ma\u014b\u014bel gaska.\",\n    evening: \"eahkes\",\n    night: \"ihkku\",\n  },\n  abbreviated: {\n    am: \"a.m.\",\n    pm: \"p.m.\",\n    midnight: \"gaskaidja\",\n    noon: \"gaskabeaivvi\",\n    morning: \"i\u0111\u0111es\",\n    afternoon: \"ma\u014b\u014bel gaskabea.\",\n    evening: \"eahkes\",\n    night: \"ihkku\",\n  },\n  wide: {\n    am: \"a.m.\",\n    pm: \"p.m.\",\n    midnight: \"gaskaidja\",\n    noon: \"gaskabeavvi\",\n    morning: \"i\u0111\u0111es\",\n    afternoon: \"ma\u014b\u014bel gaskabeaivvi\",\n    evening: \"eahkes\",\n    night: \"ihkku\",\n  },\n};\n\nconst ordinalNumber: LocalizeFn<number> = (dirtyNumber, _options) => {\n  const number = Number(dirtyNumber);\n  return number + \".\";\n};\n\nexport const localize: Localize = {\n  ordinalNumber,\n\n  era: buildLocalizeFn({\n    values: eraValues,\n    defaultWidth: \"wide\",\n  }),\n\n  quarter: buildLocalizeFn({\n    values: quarterValues,\n    defaultWidth: \"wide\",\n    argumentCallback: (quarter) => quarter - 1,\n  }),\n\n  month: buildLocalizeFn({\n    values: monthValues,\n    defaultWidth: \"wide\",\n  }),\n\n  day: buildLocalizeFn({\n    values: dayValues,\n    defaultWidth: \"wide\",\n  }),\n\n  dayPeriod: buildLocalizeFn({\n    values: dayPeriodValues,\n    defaultWidth: \"wide\",\n  }),\n};\n\n// Path: src/locale/se/_lib/formatRelative/index.ts\nimport type { FormatRelativeFn } from \"../../../types.js\";\n\nconst formatRelativeLocale = {\n  lastWeek: \"'ovddit' eeee 'dii.' p\",\n  yesterday: \"'ikte dii.' p\",\n  today: \"'odne dii.' p\",\n  tomorrow: \"'ihtin dii.' p\",\n  nextWeek: \"EEEE 'dii.' p\",\n  other: \"P\",\n};\n\nexport const formatRelative: FormatRelativeFn = (\n  token,\n  _date,\n  _baseDate,\n  _options,\n) => formatRelativeLocale[token];\n\n// Path: src/locale/se/index.ts\nimport type { Locale } from \"../types.js\";\nimport { formatDistance } from \"./_lib/formatDistance/index.js\";\nimport { formatLong } from \"./_lib/formatLong/index.js\";\nimport { formatRelative } from \"./_lib/formatRelative/index.js\";\nimport { localize } from \"./_lib/localize/index.js\";\nimport { match } from \"./_lib/match/index.js\";\n\n/**\n * @category Locales\n * @summary Northern S\u00e1mi locale.\n * @language Northern S\u00e1mi\n * @iso-639-2 sme\n * @author Audun Rundberg [@audunru](https://github.com/audunru)\n */\nexport const se: Locale = {\n  code: \"se\",\n  formatDistance: formatDistance,\n  formatLong: formatLong,\n  formatRelative: formatRelative,\n  localize: localize,\n  match: match,\n  options: {\n    weekStartsOn: 1 /* Monday */,\n    firstWeekContainsDate: 4,\n  },\n};\n\n// Path: src/locale/bn/_lib/localize/index.ts\nimport type { Localize, LocalizeFn } from \"../../../types.js\";\nimport { buildLocalizeFn } from \"../../../_lib/buildLocalizeFn/index.js\";\n\nconst numberValues = {\n  locale: {\n    \"1\": \"\u09e7\",\n    \"2\": \"\u09e8\",\n    \"3\": \"\u09e9\",\n    \"4\": \"\u09ea\",\n    \"5\": \"\u09eb\",\n    \"6\": \"\u09ec\",\n    \"7\": \"\u09ed\",\n    \"8\": \"\u09ee\",\n    \"9\": \"\u09ef\",\n    \"0\": \"\u09e6\",\n  },\n  number: {\n    \"\u09e7\": \"1\",\n    \"\u09e8\": \"2\",\n    \"\u09e9\": \"3\",\n    \"\u09ea\": \"4\",\n    \"\u09eb\": \"5\",\n    \"\u09ec\": \"6\",\n    \"\u09ed\": \"7\",\n    \"\u09ee\": \"8\",\n    \"\u09ef\": \"9\",\n    \"\u09e6\": \"0\",\n  },\n};\n\nconst eraValues = {\n  narrow: [\"\u0996\u09cd\u09b0\u09bf\u0983\u09aa\u09c2\u0983\", \"\u0996\u09cd\u09b0\u09bf\u0983\"] as const,\n  abbreviated: [\"\u0996\u09cd\u09b0\u09bf\u0983\u09aa\u09c2\u09b0\u09cd\u09ac\", \"\u0996\u09cd\u09b0\u09bf\u0983\"] as const,\n  wide: [\"\u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09aa\u09c2\u09b0\u09cd\u09ac\", \"\u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\"] as const,\n};\n\nconst quarterValues = {\n  narrow: [\"\u09e7\", \"\u09e8\", \"\u09e9\", \"\u09ea\"] as const,\n  abbreviated: [\"\u09e7\u09a4\u09cd\u09b0\u09c8\", \"\u09e8\u09a4\u09cd\u09b0\u09c8\", \"\u09e9\u09a4\u09cd\u09b0\u09c8\", \"\u09ea\u09a4\u09cd\u09b0\u09c8\"] as const,\n  wide: [\n    \"\u09e7\u09ae \u09a4\u09cd\u09b0\u09c8\u09ae\u09be\u09b8\u09bf\u0995\",\n    \"\u09e8\u09df \u09a4\u09cd\u09b0\u09c8\u09ae\u09be\u09b8\u09bf\u0995\",\n    \"\u09e9\u09df \u09a4\u09cd\u09b0\u09c8\u09ae\u09be\u09b8\u09bf\u0995\",\n    \"\u09ea\u09b0\u09cd\u09a5 \u09a4\u09cd\u09b0\u09c8\u09ae\u09be\u09b8\u09bf\u0995\",\n  ] as const,\n};\n\nconst monthValues = {\n  narrow: [\n    \"\u099c\u09be\u09a8\u09c1\",\n    \"\u09ab\u09c7\u09ac\u09cd\u09b0\u09c1\",\n    \"\u09ae\u09be\u09b0\u09cd\u099a\",\n    \"\u098f\u09aa\u09cd\u09b0\u09bf\u09b2\",\n    \"\u09ae\u09c7\",\n    \"\u099c\u09c1\u09a8\",\n    \"\u099c\u09c1\u09b2\u09be\u0987\",\n    \"\u0986\u0997\u09b8\u09cd\u099f\",\n    \"\u09b8\u09c7\u09aa\u09cd\u099f\",\n    \"\u0985\u0995\u09cd\u099f\u09cb\",\n    \"\u09a8\u09ad\u09c7\",\n    \"\u09a1\u09bf\u09b8\u09c7\",\n  ] as const,\n  abbreviated: [\n    \"\u099c\u09be\u09a8\u09c1\",\n    \"\u09ab\u09c7\u09ac\u09cd\u09b0\u09c1\",\n    \"\u09ae\u09be\u09b0\u09cd\u099a\",\n    \"\u098f\u09aa\u09cd\u09b0\u09bf\u09b2\",\n    \"\u09ae\u09c7\",\n    \"\u099c\u09c1\u09a8\",\n    \"\u099c\u09c1\u09b2\u09be\u0987\",\n    \"\u0986\u0997\u09b8\u09cd\u099f\",\n    \"\u09b8\u09c7\u09aa\u09cd\u099f\",\n    \"\u0985\u0995\u09cd\u099f\u09cb\",\n    \"\u09a8\u09ad\u09c7\",\n    \"\u09a1\u09bf\u09b8\u09c7\",\n  ] as const,\n  wide: [\n    \"\u099c\u09be\u09a8\u09c1\u09df\u09be\u09b0\u09bf\",\n    \"\u09ab\u09c7\u09ac\u09cd\u09b0\u09c1\u09df\u09be\u09b0\u09bf\",\n    \"\u09ae\u09be\u09b0\u09cd\u099a\",\n    \"\u098f\u09aa\u09cd\u09b0\u09bf\u09b2\",\n    \"\u09ae\u09c7\",\n    \"\u099c\u09c1\u09a8\",\n    \"\u099c\u09c1\u09b2\u09be\u0987\",\n    \"\u0986\u0997\u09b8\u09cd\u099f\",\n    \"\u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0\",\n    \"\u0985\u0995\u09cd\u099f\u09cb\u09ac\u09b0\",\n    \"\u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\",\n    \"\u09a1\u09bf\u09b8\u09c7\u09ae\u09cd\u09ac\u09b0\",\n  ] as const,\n};\n\nconst dayValues = {\n  narrow: [\"\u09b0\", \"\u09b8\u09cb\", \"\u09ae\", \"\u09ac\u09c1\", \"\u09ac\u09c3\", \"\u09b6\u09c1\", \"\u09b6\"] as const,\n  short: [\"\u09b0\u09ac\u09bf\", \"\u09b8\u09cb\u09ae\", \"\u09ae\u0999\u09cd\u0997\u09b2\", \"\u09ac\u09c1\u09a7\", \"\u09ac\u09c3\u09b9\", \"\u09b6\u09c1\u0995\u09cd\u09b0\", \"\u09b6\u09a8\u09bf\"] as const,\n  abbreviated: [\"\u09b0\u09ac\u09bf\", \"\u09b8\u09cb\u09ae\", \"\u09ae\u0999\u09cd\u0997\u09b2\", \"\u09ac\u09c1\u09a7\", \"\u09ac\u09c3\u09b9\", \"\u09b6\u09c1\u0995\u09cd\u09b0\", \"\u09b6\u09a8\u09bf\"] as const,\n  wide: [\n    \"\u09b0\u09ac\u09bf\u09ac\u09be\u09b0\",\n    \"\u09b8\u09cb\u09ae\u09ac\u09be\u09b0\",\n    \"\u09ae\u0999\u09cd\u0997\u09b2\u09ac\u09be\u09b0\",\n    \"\u09ac\u09c1\u09a7\u09ac\u09be\u09b0\",\n    \"\u09ac\u09c3\u09b9\u09b8\u09cd\u09aa\u09a4\u09bf\u09ac\u09be\u09b0 \",\n    \"\u09b6\u09c1\u0995\u09cd\u09b0\u09ac\u09be\u09b0\",\n    \"\u09b6\u09a8\u09bf\u09ac\u09be\u09b0\",\n  ] as const,\n};\n\nconst dayPeriodValues = {\n  narrow: {\n    am: \"\u09aa\u09c2\",\n    pm: \"\u0985\u09aa\",\n    midnight: \"\u09ae\u09a7\u09cd\u09af\u09b0\u09be\u09a4\",\n    noon: \"\u09ae\u09a7\u09cd\u09af\u09be\u09b9\u09cd\u09a8\",\n    morning: \"\u09b8\u0995\u09be\u09b2\",\n    afternoon: \"\u09ac\u09bf\u0995\u09be\u09b2\",\n    evening: \"\u09b8\u09a8\u09cd\u09a7\u09cd\u09af\u09be\",\n    night: \"\u09b0\u09be\u09a4\",\n  },\n  abbreviated: {\n    am: \"\u09aa\u09c2\u09b0\u09cd\u09ac\u09be\u09b9\u09cd\u09a8\",\n    pm: \"\u0985\u09aa\u09b0\u09be\u09b9\u09cd\u09a8\",\n    midnight: \"\u09ae\u09a7\u09cd\u09af\u09b0\u09be\u09a4\",\n    noon: \"\u09ae\u09a7\u09cd\u09af\u09be\u09b9\u09cd\u09a8\",\n    morning: \"\u09b8\u0995\u09be\u09b2\",\n    afternoon: \"\u09ac\u09bf\u0995\u09be\u09b2\",\n    evening: \"\u09b8\u09a8\u09cd\u09a7\u09cd\u09af\u09be\",\n    night: \"\u09b0\u09be\u09a4\",\n  },\n  wide: {\n    am: \"\u09aa\u09c2\u09b0\u09cd\u09ac\u09be\u09b9\u09cd\u09a8\",\n    pm: \"\u0985\u09aa\u09b0\u09be\u09b9\u09cd\u09a8\",\n    midnight: \"\u09ae\u09a7\u09cd\u09af\u09b0\u09be\u09a4\",\n    noon: \"\u09ae\u09a7\u09cd\u09af\u09be\u09b9\u09cd\u09a8\",\n    morning: \"\u09b8\u0995\u09be\u09b2\",\n    afternoon: \"\u09ac\u09bf\u0995\u09be\u09b2\",\n    evening: \"\u09b8\u09a8\u09cd\u09a7\u09cd\u09af\u09be\",\n    night: \"\u09b0\u09be\u09a4\",\n  },\n};\n\nconst formattingDayPeriodValues = {\n  narrow: {\n    am: \"\u09aa\u09c2\",\n    pm: \"\u0985\u09aa\",\n    midnight: \"\u09ae\u09a7\u09cd\u09af\u09b0\u09be\u09a4\",\n    noon: \"\u09ae\u09a7\u09cd\u09af\u09be\u09b9\u09cd\u09a8\",\n    morning: \"\u09b8\u0995\u09be\u09b2\",\n    afternoon: \"\u09ac\u09bf\u0995\u09be\u09b2\",\n    evening: \"\u09b8\u09a8\u09cd\u09a7\u09cd\u09af\u09be\",\n    night: \"\u09b0\u09be\u09a4\",\n  },\n  abbreviated: {\n    am: \"\u09aa\u09c2\u09b0\u09cd\u09ac\u09be\u09b9\u09cd\u09a8\",\n    pm: \"\u0985\u09aa\u09b0\u09be\u09b9\u09cd\u09a8\",\n    midnight: \"\u09ae\u09a7\u09cd\u09af\u09b0\u09be\u09a4\",\n    noon: \"\u09ae\u09a7\u09cd\u09af\u09be\u09b9\u09cd\u09a8\",\n    morning: \"\u09b8\u0995\u09be\u09b2\",\n    afternoon: \"\u09ac\u09bf\u0995\u09be\u09b2\",\n    evening: \"\u09b8\u09a8\u09cd\u09a7\u09cd\u09af\u09be\",\n    night: \"\u09b0\u09be\u09a4\",\n  },\n  wide: {\n    am: \"\u09aa\u09c2\u09b0\u09cd\u09ac\u09be\u09b9\u09cd\u09a8\",\n    pm: \"\u0985\u09aa\u09b0\u09be\u09b9\u09cd\u09a8\",\n    midnight: \"\u09ae\u09a7\u09cd\u09af\u09b0\u09be\u09a4\",\n    noon: \"\u09ae\u09a7\u09cd\u09af\u09be\u09b9\u09cd\u09a8\",\n    morning: \"\u09b8\u0995\u09be\u09b2\",\n    afternoon: \"\u09ac\u09bf\u0995\u09be\u09b2\",\n    evening: \"\u09b8\u09a8\u09cd\u09a7\u09cd\u09af\u09be\",\n    night: \"\u09b0\u09be\u09a4\",\n  },\n};\n\nfunction dateOrdinalNumber(number: number, localeNumber: string): string {\n  if (number > 18 && number <= 31) {\n    return localeNumber + \"\u09b6\u09c7\";\n  } else {\n    switch (number) {\n      case 1:\n        return localeNumber + \"\u09b2\u09be\";\n      case 2:\n      case 3:\n        return localeNumber + \"\u09b0\u09be\";\n      case 4:\n        return localeNumber + \"\u09a0\u09be\";\n      default:\n        return localeNumber + \"\u0987\";\n    }\n  }\n}\n\nconst ordinalNumber: LocalizeFn<number> = (dirtyNumber, options) => {\n  const number = Number(dirtyNumber);\n  const localeNumber = numberToLocale(number);\n  const unit = options?.unit;\n\n  if (unit === \"date\") {\n    return dateOrdinalNumber(number, localeNumber);\n  }\n  if (number > 10 || number === 0) return localeNumber + \"\u09a4\u09ae\";\n\n  const rem10 = number % 10;\n  switch (rem10) {\n    case 2:\n    case 3:\n      return localeNumber + \"\u09df\";\n    case 4:\n      return localeNumber + \"\u09b0\u09cd\u09a5\";\n    case 6:\n      return localeNumber + \"\u09b7\u09cd\u09a0\";\n    default:\n      return localeNumber + \"\u09ae\";\n  }\n};\n\n// function localeToNumber(locale: string): number {\n//   const enNumber = locale.toString().replace(/[\u09e7\u09e8\u09e9\u09ea\u09eb\u09ec\u09ed\u09ee\u09ef\u09e6]/g, function (match) {\n//     return numberValues.number[match as keyof typeof numberValues.number]\n//   })\n//   return Number(enNumber)\n// }\n\nexport function numberToLocale(enNumber: number) {\n  return enNumber.toString().replace(/\\d/g, function (match) {\n    return numberValues.locale[match as keyof typeof numberValu\nes.locale];\n  });\n}\n\nexport const localize: Localize = {\n  ordinalNumber,\n\n  era: buildLocalizeFn({\n    values: eraValues,\n    defaultWidth: \"wide\",\n  }),\n\n  quarter: buildLocalizeFn({\n    values: quarterValues,\n    defaultWidth: \"wide\",\n    argumentCallback: (quarter) => quarter - 1,\n  }),\n\n  month: buildLocalizeFn({\n    values: monthValues,\n    defaultWidth: \"wide\",\n  }),\n\n  day: buildLocalizeFn({\n    values: dayValues,\n    defaultWidth: \"wide\",\n  }),\n\n  dayPeriod: buildLocalizeFn({\n    values: dayPeriodValues,\n    defaultWidth: \"wide\",\n    formattingValues: formattingDayPeriodValues,\n    defaultFormattingWidth: \"wide\",\n  }),\n};\n\n// Path: src/locale/bn/_lib/formatDistance/index.ts\nimport type { FormatDistanceFn, FormatDistanceLocale } from \"../../../types.js\";\nimport { numberToLocale } from \"../localize/index.js\";\n\ntype FormatDistanceTokenValue =\n  | string\n  | {\n      one: string;\n      other: string;\n    };\n\nconst formatDistanceLocale: FormatDistanceLocale<FormatDistanceTokenValue> = {\n  lessThanXSeconds: {\n    one: \"\u09aa\u09cd\u09b0\u09be\u09df \u09e7 \u09b8\u09c7\u0995\u09c7\u09a8\u09cd\u09a1\",\n    other: \"\u09aa\u09cd\u09b0\u09be\u09df {{count}} \u09b8\u09c7\u0995\u09c7\u09a8\u09cd\u09a1\",\n  },\n\n  xSeconds: {\n    one: \"\u09e7 \u09b8\u09c7\u0995\u09c7\u09a8\u09cd\u09a1\",\n    other: \"{{count}} \u09b8\u09c7\u0995\u09c7\u09a8\u09cd\u09a1\",\n  },\n\n  halfAMinute: \"\u0986\u09a7 \u09ae\u09bf\u09a8\u09bf\u099f\",\n\n  lessThanXMinutes: {\n    one: \"\u09aa\u09cd\u09b0\u09be\u09df \u09e7 \u09ae\u09bf\u09a8\u09bf\u099f\",\n    other: \"\u09aa\u09cd\u09b0\u09be\u09df {{count}} \u09ae\u09bf\u09a8\u09bf\u099f\",\n  },\n\n  xMinutes: {\n    one: \"\u09e7 \u09ae\u09bf\u09a8\u09bf\u099f\",\n    other: \"{{count}} \u09ae\u09bf\u09a8\u09bf\u099f\",\n  },\n\n  aboutXHours: {\n    one: \"\u09aa\u09cd\u09b0\u09be\u09df \u09e7 \u0998\u09a8\u09cd\u099f\u09be\",\n    other: \"\u09aa\u09cd\u09b0\u09be\u09df {{count}} \u0998\u09a8\u09cd\u099f\u09be\",\n  },\n\n  xHours: {\n    one: \"\u09e7 \u0998\u09a8\u09cd\u099f\u09be\",\n    other: \"{{count}} \u0998\u09a8\u09cd\u099f\u09be\",\n  },\n\n  xDays: {\n    one: \"\u09e7 \u09a6\u09bf\u09a8\",\n    other: \"{{count}} \u09a6\u09bf\u09a8\",\n  },\n\n  aboutXWeeks: {\n    one: \"\u09aa\u09cd\u09b0\u09be\u09df \u09e7 \u09b8\u09aa\u09cd\u09a4\u09be\u09b9\",\n    other: \"\u09aa\u09cd\u09b0\u09be\u09df {{count}} \u09b8\u09aa\u09cd\u09a4\u09be\u09b9\",\n  },\n\n  xWeeks: {\n    one: \"\u09e7 \u09b8\u09aa\u09cd\u09a4\u09be\u09b9\",\n    other: \"{{count}} \u09b8\u09aa\u09cd\u09a4\u09be\u09b9\",\n  },\n\n  aboutXMonths: {\n    one: \"\u09aa\u09cd\u09b0\u09be\u09df \u09e7 \u09ae\u09be\u09b8\",\n    other: \"\u09aa\u09cd\u09b0\u09be\u09df {{count}} \u09ae\u09be\u09b8\",\n  },\n\n...\n", "needle_token_start": 15278, "needle_token_end": 15423, "code_context_ntokens": 16154, "output": ["```typescript\nfunction dateOrdinalNumber(number: number, localeNumber: string): string {\n  if (number > 18 && number <= 31) {\n    return localeNumber + \"\u09b6\u09c7\";\n  } else {\n    switch (number) {\n      case 1:\n        return localeNumber + \"\u09b2\u09be\";\n      case 2:\n      case 3:\n        return localeNumber + \"\u09b0\u09be\";\n      case 4:\n        return localeNumber + \"\u09a0\u09be\";\n      default:\n        return localeNumber + \"\u0987\";\n    }\n  }\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "isBinaryPath", "language": "typescript", "path": "langchain/src/document_loaders/web/github.ts", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function determines if a given file path corresponds to a binary file by examining the file extension.\n2. **Input**: A string representing the file path.\n3. **Output**: A boolean value; `true` if the file is binary, otherwise `false`.\n4. **Procedure**: The function extracts the extension from the file path, converts it to lowercase, and checks if this extension is included in a predefined set of binary file extensions.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/document_loaders/web/s3.ts\nimport * as fsDefault from \"node:fs\";\nimport * as path from \"node:path\";\nimport * as os from \"node:os\";\nimport { Readable } from \"node:stream\";\nimport { S3Client, GetObjectCommand, S3ClientConfig } from \"@aws-sdk/client-s3\";\nimport { BaseDocumentLoader } from \"../base.js\";\nimport { UnstructuredLoader as UnstructuredLoaderDefault } from \"../fs/unstructured.js\";\n\n/**\n * Represents the configuration options for the S3 client. It extends the\n * S3ClientConfig interface from the \"@aws-sdk/client-s3\" package and\n * includes additional deprecated properties for access key ID and secret\n * access key.\n */\nexport type S3Config = S3ClientConfig & {\n  /** @deprecated Use the credentials object instead */\n  accessKeyId?: string;\n  /** @deprecated Use the credentials object instead */\n  secretAccessKey?: string;\n};\n\n/**\n * Represents the parameters for the S3Loader class. It includes\n * properties such as the S3 bucket, key, unstructured API URL,\n * unstructured API key, S3 configuration, file system module, and\n * UnstructuredLoader module.\n */\nexport interface S3LoaderParams {\n  bucket: string;\n  key: string;\n  unstructuredAPIURL: string;\n  unstructuredAPIKey: string;\n  s3Config?: S3Config & {\n    /** @deprecated Use the credentials object instead */\n    accessKeyId?: string;\n    /** @deprecated Use the credentials object instead */\n    secretAccessKey?: string;\n  };\n  fs?: typeof fsDefault;\n  UnstructuredLoader?: typeof UnstructuredLoaderDefault;\n}\n\n/**\n * A class that extends the BaseDocumentLoader class. It represents a\n * document loader for loading files from an S3 bucket.\n * @example\n * ```typescript\n * const loader = new S3Loader({\n *   bucket: \"my-document-bucket-123\",\n *   key: \"AccountingOverview.pdf\",\n *   s3Config: {\n *     region: \"us-east-1\",\n *     credentials: {\n...\n// Path: langchain/src/document_loaders/web/github.ts\nimport ignore, { Ignore } from \"ignore\";\nimport binaryExtensions from \"binary-extensions\";\n\nimport { Document } from \"@langchain/core/documents\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport {\n  AsyncCaller,\n  AsyncCallerParams,\n} from \"@langchain/core/utils/async_caller\";\nimport { BaseDocumentLoader } from \"../base.js\";\nimport { UnknownHandling } from \"../fs/directory.js\";\nimport { extname } from \"../../util/extname.js\";\n\nconst extensions = /* #__PURE__ */ new Set(binaryExtensions);\n\n/**\n * A function that checks if a file path is a binary file based on its\n * extension.\n * @param name The file path to check.\n * @returns A boolean indicating whether the file path is a binary file.\n */\n\nfunction isBinaryPath(name: string) {\n  return extensions.has(extname(name).slice(1).toLowerCase());\n}\n\n/**\n * An interface that represents a file in a GitHub repository. It has\n * properties for the file name, path, SHA, size, URLs, type, and links.\n */\nexport interface GithubFile {\n  name: string;\n  path: string;\n  sha: string;\n  size: number;\n  url: string;\n  html_url: string;\n  git_url: string;\n  download_url: string;\n  type: string;\n  _links: {\n    self: string;\n    git: string;\n    html: string;\n  };\n}\n\n/**\n * An interface that represents the response from fetching the content of\n * a file. It has properties for the file contents and metadata.\n */\ninterface GetContentResponse {\n  contents: string;\n  metadata: { source: string; repository: string; branch: string };\n}\n\n/**\n * An interface describing the submodules of a Git repository.\n */\ninterface SubmoduleInfo {\n  name: string;\n  path: string;\n  url: string;\n  ref: string;\n}\n\n/**\n * An interface that represents the parameters for the GithubRepoLoader\n * class. It extends the AsyncCallerParams interface and adds additional\n * properties specific to the GitHub repository loader.\n */\nexport interface GithubRepoLoaderParams extends AsyncCallerParams {\n  /**\n   * The base URL of the GitHub instance.\n   * To be used when you are not targeting github.com, e.g. a GitHub Enterprise instance.\n   */\n  baseUrl?: string;\n  /**\n   * The API endpoint URL of the GitHub instance.\n   * To be used when you are not targeting github.com, e.g. a GitHub Enterprise instance.\n   */\n  apiUrl?: string;\n  branch?: string;\n  recursive?: boolean;\n  /**\n   * Set to true to recursively process submodules. Is only effective, when recursive=true.\n   */\n  processSubmodules?: boolean;\n  unknown?: UnknownHandling;\n  accessToken?: string;\n  ignoreFiles?: (string | RegExp)[];\n  ignorePaths?: string[];\n  verbose?: boolean;\n  /**\n   * The maximum number of concurrent calls that can be made. Defaults to 2.\n   */\n  maxConcurrency?: number;\n  /**\n   * The maximum number of retries that can be made for a single call,\n   * with an exponential backoff between each attempt. Defaults to 2.\n   */\n  maxRetries?: number;\n}\n\n/**\n * A class that extends the BaseDocumentLoader and implements the\n * GithubRepoLoaderParams interface. It represents a document loader for\n * loading files from a GitHub repository.\n */\nexport class GithubRepoLoader\n  extends BaseDocumentLoader\n  implements GithubRepoLoaderParams\n{\n  public baseUrl: string;\n\n  public apiUrl: string;\n\n  private readonly owner: string;\n\n  private readonly repo: string;\n\n  private readonly initialPath: string;\n\n  private headers: Record<string, string> = {};\n\n  public branch: string;\n\n  public recursive: boolean;\n\n  public processSubmodules: boolean;\n\n  public unknown: UnknownHandling;\n\n  public accessToken?: string;\n\n  public ignoreFiles: (string | RegExp)[];\n\n  public ignore?: Ignore;\n\n  public verbose?: boolean;\n\n  public maxConcurrency?: number;\n\n  public maxRetries?: number;\n\n  protected caller: AsyncCaller;\n\n  public ignorePaths?: string[];\n\n  private submoduleInfos: SubmoduleInfo[];\n\n  constructor(\n    githubUrl: string,\n    {\n      accessToken = getEnvironmentVariable(\"GITHUB_ACCESS_TOKEN\"),\n      baseUrl = \"https://github.com\",\n      apiUrl = \"https://api.github.com\",\n      branch = \"main\",\n      recursive = true,\n      processSubmodules = false,\n      unknown = UnknownHandling.Warn,\n      ignoreFiles = [],\n      ignorePaths,\n      verbose = false,\n      maxConcurrency = 2,\n      maxRetries = 2,\n      ...rest\n    }: GithubRepoLoaderParams = {}\n  ) {\n    super();\n    this.baseUrl = baseUrl;\n    this.apiUrl = apiUrl;\n    const { owner, repo, path } = this.extractOwnerAndRepoAndPath(githubUrl);\n    this.owner = owner;\n    this.repo = repo;\n    this.initialPath = path;\n    this.branch = branch;\n    this.recursive = recursive;\n    // processing submodules without processing contents of other directories makes no sense\n    if (processSubmodules && !recursive) {\n      throw new Error(\n        `Input property \"recursive\" must be true if \"processSubmodules\" is true.`\n      );\n    }\n    this.processSubmodules = processSubmodules;\n    this.unknown = unknown;\n    this.accessToken = accessToken;\n    this.ignoreFiles = ignoreFiles;\n    this.verbose = verbose;\n    this.maxConcurrency = maxConcurrency;\n    this.maxRetries = maxRetries;\n    this.headers = {\n      \"User-Agent\": \"langchain\",\n    };\n    this.caller = new AsyncCaller({\n      maxConcurrency,\n      maxRetries,\n      ...rest,\n    });\n    this.ignorePaths = ignorePaths;\n    if (ignorePaths) {\n      this.ignore = ignore.default().add(ignorePaths);\n    }\n    if (this.accessToken) {\n      this.headers = {\n        ...this.headers,\n        Authorization: `Bearer ${this.accessToken}`,\n      };\n    }\n  }\n\n  /**\n   * Extracts the owner, repository, and path from a GitHub URL.\n   * @param url The GitHub URL to extract information from.\n   * @returns An object containing the owner, repository, and path extracted from the GitHub URL.\n   */\n  private extractOwnerAndRepoAndPath(url: string): {\n    owner: string;\n    repo: string;\n    path: string;\n  } {\n    const match = url.match(\n      new RegExp(`${this.baseUrl}/([^/]+)/([^/]+)(/tree/[^/]+/(.+))?`, \"i\")\n    );\n\n    if (!match) {\n      throw new Error(\"Invalid GitHub URL format.\");\n    }\n\n    return { owner: match[1], repo: match[2], path: match[4] || \"\" };\n  }\n\n  /**\n   * Fetches the files from the GitHub repository and creates Document\n   * instances for each file. It also handles error handling based on the\n   * unknown handling option.\n   * @returns A promise that resolves to an array of Document instances.\n   */\n  public async load(): Promise<Document[]> {\n    this.log(\n      `Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`\n    );\n    // process repository without submodules\n    const documents: Document[] = (await this.processRepo()).map(\n      (fileResponse) =>\n        new Document({\n          pageContent: fileResponse.contents,\n          metadata: fileResponse.metadata,\n        })\n    );\n    if (this.processSubmodules) {\n      // process submodules\n      await this.getSubmoduleInfo();\n      for (const submoduleInfo of this.submoduleInfos) {\n        documents.push(...(await this.loadSubmodule(submoduleInfo)));\n      }\n    }\n    return documents;\n  }\n\n  /**\n   * Asynchronously streams documents from the entire GitHub repository.\n   * It is suitable for situations where processing large repositories in a memory-efficient manner is required.\n   * @yields Yields a Promise that resolves to a Document object for each file or submodule content found in the repository.\n   */\n  public async *loadAsStream(): AsyncGenerator<Document, void, undefined> {\n    this.log(\n      `Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`\n    );\n    yield* await this.processRepoAsStream(this.initialPath);\n\n    if (!this.processSubmodules) {\n      return;\n    }\n\n    await this.getSubmoduleInfo();\n    for (const submoduleInfo of this.submoduleInfos) {\n      yield* await this.loadSubmoduleAsStream(submoduleInfo);\n    }\n  }\n\n  /**\n   * Loads the information about Git submodules from the repository, if available.\n   */\n  private async getSubmoduleInfo(): Promise<void> {\n    this.log(\"Loading info about submodules...\");\n    // we have to fetch the files of the root directory to get the download url of the .gitmodules file\n    // however, we cannot reuse the files retrieved in processRepo() as initialPath may be != \"\"\n    // so it may be that we end up fetching this file list twice\n    const repoFiles = await this.fetchRepoFiles(\"\");\n    const gitmodulesFile = repoFiles.filter(\n      ({ name }) => name === \".gitmodules\"\n    )?.[0];\n    if (gitmodulesFile) {\n      const gitmodulesContent = await this.fetchFileContent({\n        download_url: gitmodulesFile.download_url,\n      } as GithubFile);\n      this.submoduleInfos = await this.parseGitmodules(gitmodulesContent);\n    } else {\n      this.submoduleInfos = [];\n    }\n    this.log(`Found ${this.submoduleInfos.length} submodules:`);\n    for (const submoduleInfo of this.submoduleInfos) {\n      this.log(JSON.stringify(submoduleInfo));\n    }\n  }\n\n  /**\n   * Parses the given content of a .gitmodules file. Furthermore, queries the current SHA ref of all submodules.\n   * Returns the submodule information as array.\n   * @param gitmodulesContent the content of a .gitmodules file\n   */\n  private async parseGitmodules(\n    gitmodulesContent: string\n  ): Promise<SubmoduleInfo[]> {\n    // catches the initial line of submodule entries\n    const submodulePattern = /\\[submodule \"(.*?)\"]\\n((\\s+.*?\\s*=\\s*.*?\\n)*)/g;\n    // catches the properties of a submodule\n    const keyValuePattern = /\\s+(.*?)\\s*=\\s*(.*?)\\s/g;\n\n    const submoduleInfos = [];\n    for (const [, name, propertyLines] of gitmodulesContent.matchAll(\n      submodulePattern\n    )) {\n      if (!name || !propertyLines) {\n        throw new Error(\"Could not parse submodule entry\");\n      }\n      const submodulePropertyLines = propertyLines.matchAll(keyValuePattern);\n      let path;\n      let url;\n      for (const [, key, value] of submodulePropertyLines) {\n        if (!key || !value) {\n          throw new Error(\n            `Could not parse key/value pairs for submodule ${name}`\n          );\n        }\n        switch (key) {\n          case \"path\":\n            path = value;\n            break;\n          case \"url\":\n            url = value;\n            if (url.endsWith(\".git\")) {\n              url = url.substring(0, url.length - 4);\n            }\n            break;\n          default:\n          // ignoring unused keys\n        }\n      }\n      if (!path || !url) {\n        throw new Error(`Missing properties for submodule ${name}`);\n      }\n      // fetch the current ref of the submodule\n      const files = await this.fetchRepoFiles(path);\n      const submoduleInfo: SubmoduleInfo = {\n        name,\n        path,\n        url,\n        ref: files[0].sha,\n      };\n      submoduleInfos.push(submoduleInfo);\n    }\n    return submoduleInfos;\n  }\n\n  /**\n   * Loads the documents of the given submodule. Uses the same parameters as for the current repository.\n   * External submodules, i.e. submodules pointing to another GitHub instance, are ignored.\n   * @param submoduleInfo the info about the submodule to be loaded\n   */\n  private async loadSubmodule(\n    submoduleInfo: SubmoduleInfo\n  ): Promise<Document[]> {\n    if (!submoduleInfo.url.startsWith(this.baseUrl)) {\n      this.log(`Ignoring external submodule ${submoduleInfo.url}.`);\n      return [];\n    } else if (!submoduleInfo.path.startsWith(this.initialPath)) {\n      this.log(\n        `Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`\n      );\n      return [];\n    } else {\n      this.log(\n        `Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`\n      );\n      return new GithubRepoLoader(submoduleInfo.url, {\n        accessToken: this.accessToken,\n        apiUrl: this.apiUrl,\n        baseUrl: this.baseUrl,\n        branch: submoduleInfo.ref,\n        recursive: this.recursive,\n        processSubmodules: this.processSubmodules,\n        unknown: this.unknown,\n        ignoreFiles: this.ignoreFiles,\n        ignorePaths: this.ignorePaths,\n        verbose: this.verbose,\n        maxConcurrency: this.maxConcurrency,\n        maxRetries: this.maxRetries,\n      }).load();\n    }\n  }\n\n  /**\n   * Asynchronously processes and streams the contents of a specified submodule in the GitHub repository.\n   * @param submoduleInfo the info about the submodule to be loaded\n   * @yields Yields a Promise that resolves to a Document object for each file found in the submodule.\n   */\n  private async *loadSubmoduleAsStream(\n    submoduleInfo: SubmoduleInfo\n  ): AsyncGenerator<Document, void, undefined> {\n    if (!submoduleInfo.url.startsWith(this.baseUrl)) {\n      this.log(`Ignoring external submodule ${submoduleInfo.url}.`);\n      yield* [];\n    }\n\n    if (!submoduleInfo.path.startsWith(this.initialPath)) {\n      this.log(\n        `Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`\n      );\n      yield* [];\n    }\n\n    this.log(\n      `Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`\n    );\n    const submoduleLoader = new GithubRepoLoader(submoduleInfo.url, {\n      accessToken: this.accessToken,\n      baseUrl: this.baseUrl,\n      apiUrl: this.apiUrl,\n      branch: submoduleInfo.ref,\n      recursive: this.recursive,\n      processSubmodules: this.processSubmodules,\n      unknown: this.unknown,\n      ignoreFiles: this.ignoreFiles,\n      ignorePaths: this.ignorePaths,\n      verbose: this.verbose,\n      maxConcurrency: this.maxConcurrency,\n      maxRetries: this.maxRetries,\n    });\n\n    yield* await submoduleLoader.processRepoAsStream(submoduleInfo.path);\n  }\n\n  /**\n   * Determines whether a file or directory should be ignored based on its\n   * path and type.\n   * @param path The path of the file or directory.\n   * @param fileType The type of the file or directory.\n   * @returns A boolean indicating whether the file or directory should be ignored.\n   */\n  protected shouldIgnore(path: string, fileType: string): boolean {\n    if (fileType !== \"dir\" && isBinaryPath(path)) {\n      return true;\n    }\n    if (this.ignore !== undefined) {\n      return this.ignore.ignores(path);\n    }\n    return (\n      fileType !== \"dir\" &&\n      this.ignoreFiles.some((pattern) => {\n        if (typeof pattern === \"string\") {\n          return path === pattern;\n        }\n\n        try {\n          return pattern.test(path);\n        } catch {\n          throw new Error(`Unknown ignore file pattern: ${pattern}`);\n        }\n      })\n    );\n  }\n\n  /**\n   * Takes the file info and wrap it in a promise that will resolve to the file content and metadata\n   * @param file\n   * @returns\n   */\n  private async fetchFileContentWrapper(\n    file: GithubFile\n  ): Promise<GetContentResponse> {\n    const fileContent = await this.fetchFileContent(file).catch((error) => {\n      this.handleError(`Failed wrap file content: ${file}, ${error}`);\n    });\n    return {\n      contents: fileContent || \"\",\n      metadata: {\n        source: file.path,\n        repository: `${this.baseUrl}/${this.owner}/${this.repo}`,\n        branch: this.branch,\n      },\n    };\n  }\n\n  /**\n   * Maps a list of files / directories to a list of promises that will fetch the file / directory contents\n   */\n  private async getCurrentDirectoryFilePromises(\n    files: GithubFile[]\n  ): Promise<Promise<GetContentResponse>[]> {\n    const currentDirectoryFilePromises: Promise<GetContentResponse>[] = [];\n    // Directories have nested files / directories, which is why this is a list of promises of promises\n    const currentDirectoryDirectoryPromises: Promise<\n      Promise<GetContentResponse>[]\n    >[] = [];\n\n    for (const file of files) {\n      if (this.shouldIgnore(file.path, file.type)) {\n        continue;\n      }\n      if (file.type === \"file\" && file.size === 0) {\n        // this is a submodule. ignoring for the moment. submodule processing is done separately\n        continue;\n      }\n      if (file.type !== \"dir\") {\n        try {\n          currentDirectoryFilePromises.push(this.fetchFileContentWrapper(file));\n        } catch (e) {\n          this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);\n        }\n      } else if (this.recursive) {\n        currentDirectoryDirectoryPromises.push(\n          this.processDirectory(file.path)\n        );\n      }\n    }\n\n    const curDirDirectories: Promise<GetContentResponse>[][] =\n      await Promise.all(currentDirectoryDirectoryPromises);\n\n    return [...currentDirectoryFilePromises, ...curDirDirectories.flat()];\n  }\n\n  /**\n   * Begins the process of fetching the contents of the repository\n   */\n  private async processRepo(): Promise<GetContentResponse[]> {\n    try {\n      // Get the list of file / directory names in the root directory\n      const files = await this.fetchRepoFiles(this.initialPath);\n      // Map the file / directory paths to promises that will fetch the file / directory contents\n      const currentDirectoryFilePromises =\n        await this.getCurrentDirectoryFilePromises(files);\n      return Promise.all(currentDirectoryFilePromises);\n    } catch (error) {\n      this.handleError(\n        `Failed to process directory: ${this.initialPath}, ${error}`\n      );\n      return Promise.reject(error);\n    }\n  }\n\n  /**\n   * Asynchronously processes the contents of the entire GitHub repository,\n   * streaming each file as a Document object.\n   * @param path The path of the directory to process.\n   * @yields Yields a Promise that resolves to a Document object for each file found in the repository.\n   */\n  private async *processRepoAsStream(\n    path: string\n  ): AsyncGenerator<Document, void, undefined> {\n    const files = await this.fetchRepoFiles(path);\n    for (const file of files) {\n      if (this.shouldIgnore(file.path, file.type)) {\n        continue;\n      }\n\n      if (file.type === \"file\") {\n        try {\n          const fileResponse = await this.fetchFileContentWrapper(file);\n\n          yield new Document({\n            pageContent: fileResponse.contents,\n            metadata: fileResponse.metadata,\n          });\n        } catch (error) {\n          this.handleError(\n            `Failed to fetch file content: ${file.path}, ${error}`\n          );\n        }\n      } else if (this.recursive) {\n        yield* await this.processDirectoryAsStream(file.path);\n      }\n    }\n  }\n\n  /**\n   * Fetches the contents of a directory and maps the file / directory paths\n   * to promises that will fetch the file / directory contents.\n   * @param path The path of the directory to process.\n   * @returns A promise that resolves to an array of promises that will fetch the file / directory contents.\n   */\n  private async processDirectory(\n    path: string\n  ): Promise<Promise<GetContentResponse>[]> {\n    try {\n      const files = await this.fetchRepoFiles(path);\n      return this.getCurrentDirectoryFilePromises(files);\n    } catch (error) {\n      this.handleError(`Failed to process directory: ${path}, ${error}`);\n      return Promise.reject(error);\n    }\n  }\n\n  /**\n   * Asynchronously processes the contents of a given directory in the GitHub repository,\n   * streaming each file as a Document object.\n   * @param path The path of the directory to process.\n   * @yields Yields a Promise that resolves to a Document object for each file in the directory.\n   */\n  private async *processDirectoryAsStream(\n    path: string\n  ): AsyncGenerator<Document, void, undefined> {\n    const files = await this.fetchRepoFiles(path);\n\n    for (const file of files) {\n      if (this.shouldIgnore(file.path, file.type)) {\n        continue;\n      }\n\n      if (file.type === \"file\") {\n        try {\n          const fileResponse = await this.fetchFileContentWrapper(file);\n\n          yield new Document({\n            pageContent: fileResponse.contents,\n            metadata: fileResponse.metadata,\n          });\n        } catch {\n          this.handleError(`Failed to fetch file content: ${file.path}`);\n        }\n      } else if (this.recursive) {\n        yield* await this.processDirectoryAsStream(file.path);\n      }\n    }\n  }\n\n  /**\n   * Fetches the files from a GitHub repository.\n   * If the path denotes a single file, the resulting array contains only one element.\n   * @param path The path of the repository to fetch the files from.\n   * @returns A promise that resolves to an array of GithubFile instances.\n   */\n  private async fetchRepoFiles(path: string): Promise<GithubFile[]> {\n    const url = `${this.apiUrl}/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;\n    return this.caller.call(async () => {\n      this.log(`Fetching ${url}`);\n      const response = await fetch(url, { headers: this.headers });\n      const data = await response.json();\n      if (!response.ok) {\n        throw new Error(\n          `Unable to fetch repository files: ${\n            response.status\n          } ${JSON.stringify(data)}`\n        );\n      }\n\n      if (Array.isArray(data)) {\n        return data as GithubFile[];\n      } else {\n        return [data as GithubFile];\n      }\n    });\n  }\n\n  /**\n   * Fetches the content of a file from a GitHub repository.\n   * @param file The file to fetch the content from.\n   * @returns A promise that resolves to the content of the file.\n   */\n  private async fetchFileContent(file: GithubFile): Promise<string> {\n    return this.caller.call(async () => {\n      this.log(`Fetching ${file.download_url}`);\n      const response = await fetch(file.download_url, {\n        headers: this.headers,\n      });\n      return response.text();\n    });\n  }\n\n  /**\n   * Handles errors based on the unknown handling option.\n   * @param message The error message.\n   * @returns void\n   */\n  private handleError(message: string): void {\n    switch (this.unknown) {\n      case UnknownHandling.Ignore:\n        break;\n      case UnknownHandling.Warn:\n        console.warn(message);\n        break;\n      case UnknownHandling.Error:\n        throw new Error(message);\n      default:\n        throw new Error(`Unknown unknown handling: ${this.unknown}`);\n    }\n  }\n\n  /**\n   * Logs the given message to the console, if parameter 'verbose' is set to true.\n   * @param message the message to be logged.\n   */\n  private log(message: string): void {\n    if (this.verbose) {\n      console.log(message);\n    }\n  }\n}\n\n// Path: langchain/src/document_loaders/web/college_confidential.ts\nimport { Document } from \"@langchain/core/documents\";\nimport { CheerioWebBaseLoader } from \"./cheerio.js\";\n\n/**\n * A document loader specifically designed for loading documents from the\n * College Confidential website. It extends the CheerioWebBaseLoader.\n * @example\n * ```typescript\n * const loader = new CollegeConfidentialLoader(\"https:exampleurl.com\");\n * const docs = await loader.load();\n * console.log({ docs });\n * ```\n */\nexport class CollegeConfidentialLoader extends CheerioWebBaseLoader {\n  constructor(webPath: string) {\n    super(webPath);\n  }\n\n  /**\n   * Overrides the base load() method to extract the text content from the\n   * loaded document using a specific selector for the College Confidential\n   * website. It creates a Document instance with the extracted text and\n   * metadata, and returns an array containing the Document instance.\n   * @returns An array containing a Document instance with the extracted text and metadata from the loaded College Confidential web document.\n   */\n  public async load(): Promise<Document[]> {\n    const $ = await this.scrape();\n    const text = $(\"main[class='skin-handler']\").text();\n    const metadata = { source: this.webPath };\n    return [new Document({ pageContent: text, metadata })];\n  }\n}\n\n// Path: langchain/src/document_loaders/web/pdf.ts\nimport { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"../base.js\";\nimport { formatDocumentsAsString } from \"../../util/document.js\";\n\n/**\n * A document loader for loading data from PDFs.\n * @example\n * ```typescript\n * const loader = new WebPDFLoader(new Blob());\n * const docs = await loader.load();\n * console.log({ docs });\n * ```\n */\nexport class WebPDFLoader extends BaseDocumentLoader {\n  protected blob: Blob;\n\n  protected splitPages = true;\n\n  private pdfjs: typeof PDFLoaderImports;\n\n  protected parsedItemSeparator: string;\n\n  constructor(\n    blob: Blob,\n    {\n      splitPages = true,\n      pdfjs = PDFLoaderImports,\n      parsedItemSeparator = \"\",\n    } = {}\n  ) {\n    super();\n    this.blob = blob;\n    this.splitPages = splitPages ?? this.splitPages;\n    this.pdfjs = pdfjs;\n    this.parsedItemSeparator = parsedItemSeparator;\n  }\n\n  /**\n   * Loads the contents of the PDF as documents.\n   * @returns An array of Documents representing the retrieved data.\n   */\n  async load(): Promise<Document[]> {\n    const { getDocument, version } = await this.pdfjs();\n    const parsedPdf = await getDocument({\n      data: new Uint8Array(await this.blob.arrayBuffer()),\n      useWorkerFetch: false,\n      isEvalSupported: false,\n      useSystemFonts: true,\n    }).promise;\n    const meta = await parsedPdf.getMetadata().catch(() => null);\n\n    const documents: Document[] = [];\n\n    for (let i = 1; i <= parsedPdf.numPages; i += 1) {\n      const page = await parsedPdf.getPage(i);\n      const content = await page.getTextContent();\n\n      if (content.items.length === 0) {\n        continue;\n      }\n\n      // Eliminate excessive newlines\n      // Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16\n      let lastY;\n      const textItems = [];\n      for (const item of content.items) {\n        if (\"str\" in item) {\n          if (lastY === item.transform[5] || !lastY) {\n            textItems.push(item.str);\n          } else {\n            textItems.push(`\\n${item.str}`);\n          }\n          // eslint-disable-next-line prefer-destructuring\n          lastY = item.transform[5];\n        }\n      }\n      const text = textItems.join(this.parsedItemSeparator);\n\n      documents.push(\n        new Document({\n          pageContent: text,\n          metadata: {\n            pdf: {\n              version,\n              info: meta?.info,\n              metadata: meta?.metadata,\n              totalPages: parsedPdf.numPages,\n            },\n            loc: {\n              pageNumber: i,\n            },\n          },\n        })\n      );\n    }\n\n    if (this.splitPages) {\n      return documents;\n    }\n\n    if (documents.length === 0) {\n      return [];\n    }\n\n    return [\n      new Document({\n        pageContent: formatDocumentsAsString(documents),\n        metadata: {\n          pdf: {\n            version,\n            info: meta?.info,\n            metadata: meta?.metadata,\n            totalPages: parsedPdf.numPages,\n          },\n        },\n      }),\n    ];\n\n    return documents;\n  }\n}\n\nasync function PDFLoaderImports() {\n  try {\n    const { default: mod } = await import(\n      \"pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js\"\n    );\n    const { getDocument, version } = mod;\n    return { getDocument, version };\n  } catch (e) {\n    console.error(e);\n    throw new Error(\n      \"Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`.\"\n    );\n  }\n}\n\n// Path: langchain/src/document_loaders/web/confluence.ts\nimport { htmlToText } from \"html-to-text\";\nimport { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"../base.js\";\n\n/**\n * Interface representing the parameters for configuring the\n * ConfluencePagesLoader.\n */\nexport interface ConfluencePagesLoaderParams {\n  baseUrl: string;\n  spaceKey: string;\n  username?: string;\n  accessToken?: string;\n  personalAccessToken?: string;\n  limit?: number;\n  expand?: string;\n}\n\n/**\n * Interface representing a Confluence page.\n */\nexport interface ConfluencePage {\n  id: string;\n  title: string;\n  type: string;\n  body: {\n    storage: {\n      value: string;\n    };\n  };\n  status: string;\n  version?: {\n    number: number;\n    when: string;\n    by: {\n      displayName: string;\n    };\n  };\n}\n\n/**\n * Interface representing the response from the Confluence API.\n */\nexport interface ConfluenceAPIResponse {\n  size: number;\n  results: ConfluencePage[];\n}\n\n/**\n * Class representing a document loader for loading pages from Confluence.\n * @example\n * ```typescript\n * const loader = new ConfluencePagesLoader({\n *   baseUrl: \"https:\n *   spaceKey: \"~EXAMPLE362906de5d343d49dcdbae5dEXAMPLE\",\n *   username: \"your-username\",\n *   accessToken: \"your-access-token\",\n * });\n * const documents = await loader.load();\n * console.log(documents);\n * ```\n */\nexport class ConfluencePagesLoader extends BaseDocumentLoader {\n  public readonly baseUrl: string;\n\n  public readonly spaceKey: string;\n\n  public readonly username?: string;\n\n  public readonly accessToken?: string;\n\n  public readonly limit: number;\n\n  /**\n   * expand parameter for confluence rest api\n   * description can be found at https://developer.atlassian.com/server/confluence/expansions-in-the-rest-api/\n   */\n  public readonly expand?: string;\n\n  public readonly personalAccessToken?: string;\n\n  constructor({\n    baseUrl,\n    spaceKey,\n    username,\n    accessToken,\n    limit = 25,\n    expand = \"body.storage,version\",\n    personalAccessToken,\n  }: ConfluencePagesLoaderParams) {\n    super();\n    this.baseUrl = baseUrl;\n    this.spaceKey = spaceKey;\n    this.username = username;\n    this.accessToken = accessToken;\n    this.limit = limit;\n    this.expand = expand;\n    this.personalAccessToken = personalAccessToken;\n  }\n\n  /**\n   * Returns the authorization header for the request.\n   * @returns The authorization header as a string, or undefined if no credentials were provided.\n   */\n  private get authorizationHeader(): string | undefined {\n    if (this.personalAccessToken) {\n      return `Bearer ${this.personalAccessToken}`;\n    } else if (this.username && this.accessToken) {\n      const authToken = Buffer.from(\n        `${this.username}:${this.accessToken}`\n      ).toString(\"base64\");\n      return `Basic ${authToken}`;\n    }\n\n    return undefined;\n  }\n\n  /**\n   * Fetches all the pages in the specified space and converts each page to\n   * a Document instance.\n   * @param options the extra options of the load function\n   * @param options.limit The limit parameter to overwrite the size to fetch pages.\n   * @param options.start The start parameter to set inital offset to fetch pages.\n   * @returns Promise resolving to an array of Document instances.\n   */\n  public async load(options?: {\n    start?: number;\n    limit?: number;\n  }): Promise<Document[]> {\n    try {\n      const pages = await this.fetchAllPagesInSpace(\n        options?.start,\n        options?.limit\n      );\n      return pages.map((page) => this.createDocumentFromPage(page));\n    } catch (error) {\n      console.error(\"Error:\", error);\n      return [];\n    }\n  }\n\n  /**\n   * Fetches data from the Confluence API using the provided URL.\n   * @param url The URL to fetch data from.\n   * @returns Promise resolving to the JSON response from the API.\n   */\n  protected async fetchConfluenceData(\n    url: string\n  ): Promise<ConfluenceAPIResponse> {\n    try {\n      const initialHeaders: HeadersInit = {\n        \"Content-Type\": \"application/json\",\n        Accept: \"application/json\",\n      };\n\n      const authHeader = this.authorizationHeader;\n      if (authHeader) {\n        initialHeaders.Authorization = authHeader;\n      }\n\n      const response = await fetch(url, {\n        headers: initialHeaders,\n      });\n\n      if (!response.ok) {\n        throw new Error(\n          `Failed to fetch ${url} from Confluence: ${response.status}`\n        );\n      }\n\n      return await response.json();\n    } catch (error) {\n      throw new Error(`Failed to fetch ${url} from Confluence: ${error}`);\n    }\n  }\n\n  /**\n   * Recursively fetches all the pages in the specified space.\n   * @param start The start parameter to paginate through the results.\n   * @returns Promise resolving to an array of ConfluencePage objects.\n   */\n  private async fetchAllPagesInSpace(\n    start = 0,\n    limit = this.limit\n  ): Promise<ConfluencePage[]> {\n    const url = `${this.baseUrl}/rest/api/content?spaceKey=${this.spaceKey}&limit=${limit}&start=${start}&expand=${this.expand}`;\n    const data = await this.fetchConfluenceData(url);\n\n    if (data.size === 0) {\n      return [];\n    }\n\n    const nextPageStart = start + data.size;\n    const nextPageResults = await this.fetchAllPagesInSpace(\n      nextPageStart,\n      limit\n    );\n\n    return data.results.concat(nextPageResults);\n  }\n\n  /**\n   * Creates a Document instance from a ConfluencePage object.\n   * @param page The ConfluencePage object to convert.\n   * @returns A Document instance.\n   */\n  private createDocumentFromPage(page: ConfluencePage): Document {\n    // Convert the HTML content to plain text\n    const plainTextContent = htmlToText(page.body.storage.value, {\n      wordwrap: false,\n      preserveNewlines: false,\n    });\n\n    // Remove empty lines\n    const textWithoutEmptyLines = plainTextContent.replace(/^\\s*[\\r\\n]/gm, \"\");\n\n    // Generate the URL\n    const pageUrl = `${this.baseUrl}/spaces/${this.spaceKey}/pages/${page.id}`;\n\n    // Return a langchain document\n    return new Document({\n      pageContent: textWithoutEmptyLines,\n      metadata: {\n        id: page.id,\n        status: page.status,\n        title: page.title,\n        type: page.type,\n        url: pageUrl,\n        version: page.version?.number,\n        updated_by: page.version?.by?.displayName,\n        updated_at: page.version?.when,\n      },\n    });\n  }\n}\n\n// Path: langchain/src/types/assemblyai-types.ts\nimport { BaseServiceParams } from \"assemblyai\";\nimport { Optional } from \"./type-utils.js\";\n\nexport type * from \"assemblyai\";\n\nexport type AssemblyAIOptions = Optional<BaseServiceParams, \"apiKey\">;\n\n// Path: langchain/src/document_loaders/web/assemblyai.ts\nimport {\n  AssemblyAI,\n  BaseServiceParams,\n  TranscribeParams,\n  SubtitleFormat,\n  Transcript,\n  TranscriptParagraph,\n  TranscriptSentence,\n  CreateTranscriptParameters,\n} from \"assemblyai\";\nimport { Document } from \"@langchain/core/documents\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { BaseDocumentLoader } from \"../base.js\";\nimport { AssemblyAIOptions } from \"../../types/assemblyai-types.js\";\n\nexport type * from \"../../types/assemblyai-types.js\";\n\n/**\n * Base class for AssemblyAI loaders.\n */\nabstract class AssemblyAILoader extends BaseDocumentLoader {\n  protected client: AssemblyAI;\n\n  /**\n   * Create a new AssemblyAI loader.\n   * @param assemblyAIOptions The options to configure the AssemblyAI loader.\n   * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable.\n   */\n  constructor(assemblyAIOptions?: AssemblyAIOptions) {\n    super();\n    let options = assemblyAIOptions;\n    if (!options) {\n      options = {};\n    }\n    if (!options.apiKey) {\n      options.apiKey = getEnvironmentVariable(\"ASSEMBLYAI_API_KEY\");\n    }\n    if (!options.apiKey) {\n      throw new Error(\"No AssemblyAI API key provided\");\n    }\n\n    this.client = new AssemblyAI(options as BaseServiceParams);\n  }\n}\n\nabstract class CreateTranscriptLoader extends AssemblyAILoader {\n  protected transcribeParams?: TranscribeParams | CreateTranscriptParameters;\n\n  protected transcriptId?: string;\n\n  /**\n   * Transcribe audio or retrieve an existing transcript by its ID.\n   * @param params The parameters to transcribe audio, or the ID of the transcript to retrieve.\n   * @param assemblyAIOptions The options to configure the AssemblyAI loader.\n   * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable.\n   */\n  constructor(\n    params: TranscribeParams | CreateTranscriptParameters | string,\n    assemblyAIOptions?: AssemblyAIOptions\n  ) {\n    super(assemblyAIOptions);\n    if (typeof params === \"string\") {\n      this.transcriptId = params;\n    } else {\n      this.transcribeParams = params;\n    }\n  }\n\n  protected async transcribeOrGetTranscript() {\n    if (this.transcriptId) {\n      return await this.client.transcripts.get(this.transcriptId);\n    }\n    if (this.transcribeParams) {\n      let transcribeParams: TranscribeParams;\n      if (\"audio_url\" in this.transcribeParams) {\n        transcribeParams = {\n          ...this.transcribeParams,\n          audio: this.transcribeParams.audio_url,\n        };\n      } else {\n        transcribeParams = this.transcribeParams;\n      }\n\n      return await this.client.transcripts.transcribe(transcribeParams);\n    } else {\n      throw new Error(\"No transcript ID or transcribe parameters provided\");\n    }\n  }\n}\n\n/**\n * Transcribe audio and load the transcript as a document using AssemblyAI.\n */\nexport class AudioTranscriptLoader extends CreateTranscriptLoader {\n  /**\n   * Transcribe audio and load the transcript as a document using AssemblyAI.\n   * @returns A promise that resolves to a single document containing the transcript text\n   * as the page content, and the transcript object as the metadata.\n   */\n  override async load(): Promise<Document<Transcript>[]> {\n    const transcript = await this.transcribeOrGetTranscript();\n\n    return [\n      new Document({\n        pageContent: transcript.text as string,\n        metadata: transcript,\n      }),\n    ];\n  }\n}\n\n/**\n * Transcribe audio and load the paragraphs of the transcript, creating a document for each paragraph.\n */\nexport class AudioTranscriptParagraphsLoader extends CreateTranscriptLoader {\n  /**\n   * Transcribe audio and load the paragraphs of the transcript, creating a document for each paragraph.\n   * @returns A promise that resolves to an array of documents, each containing a paragraph of the transcript.\n   */\n  override async load(): Promise<Document<TranscriptParagraph>[]> {\n    const transcript = await this.transcribeOrGetTranscript();\n    const paragraphsResponse = await this.client.transcripts.paragraphs(\n      transcript.id\n    );\n    return paragraphsResponse.paragraphs.map(\n      (p: TranscriptParagraph) =>\n        new Document({\n          pageContent: p.text,\n          metadata: p,\n        })\n    );\n  }\n}\n\n/**\n * Transcribe audio and load the sentences of the transcript, creating a document for each sentence.\n */\nexport class AudioTranscriptSentencesLoader extends CreateTranscriptLoader {\n  /**\n   * Transcribe audio and load the sentences of the transcript, creating a document for each sentence.\n   * @returns A promise that resolves to an array of documents, each containing a sentence of the transcript.\n   */\n  override async load(): Promise<Document<TranscriptSentence>[]> {\n    const transcript = await this.transcribeOrGetTranscript();\n    const sentencesResponse = await this.client.transcripts.sentences(\n      transcript.id\n    );\n    return sentencesResponse.sentences.map(\n      (p: TranscriptSentence) =>\n        new Document({\n          pageContent: p.text,\n          metadata: p,\n        })\n    );\n  }\n}\n\n/**\n * Transcribe audio and load subtitles for the transcript as `srt` or `vtt` format.\n */\nexport class AudioSubtitleLoader extends CreateTranscriptLoader {\n  /**\n   * Create a new AudioSubtitleLoader.\n   * @param transcribeParams The parameters to transcribe audio.\n   * @param subtitleFormat The format of the subtitles, either `srt` or `vtt`.\n   * @param assemblyAIOptions The options to configure the AssemblyAI loader.\n   * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable.\n   */\n  constructor(\n    transcribeParams: TranscribeParams | CreateTranscriptParameters,\n    subtitleFormat: SubtitleFormat,\n    assemblyAIOptions?: AssemblyAIOptions\n  );\n\n  /**\n   * Create a new AudioSubtitleLoader.\n   * @param transcriptId The ID of the transcript to retrieve.\n   * @param subtitleFormat The format of the subtitles, either `srt` or `vtt`.\n   * @param assemblyAIOptions The options to configure the AssemblyAI loader.\n   * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable.\n   */\n  constructor(\n    transcriptId: string,\n    subtitleFormat: SubtitleFormat,\n    assemblyAIOptions?: AssemblyAIOptions\n  );\n\n  /**\n   * Create a new AudioSubtitleLoader.\n   * @param params The parameters to transcribe audio, or the ID of the transcript to retrieve.\n   * @param subtitleFormat The format of the subtitles, either `srt` or `vtt`.\n   * @param assemblyAIOptions The options to configure the AssemblyAI loader.\n   * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable.\n   */\n  constructor(\n    params: TranscribeParams | CreateTranscriptParameters | string,\n    private subtitleFormat: SubtitleFormat = \"srt\",\n    assemblyAIOptions?: AssemblyAIOptions\n  ) {\n    super(params, assemblyAIOptions);\n    this.subtitleFormat = subtitleFormat;\n  }\n\n  /**\n   * Transcribe audio and load subtitles for the transcript as `srt` or `vtt` format.\n   * @returns A promise that resolves a document containing the subtitles as the page content.\n   */\n  override async load(): Promise<Document[]> {\n    const transcript = await this.transcribeOrGetTranscript();\n    const subtitles = await this.client.transcripts.subtitles(\n      transcript.id,\n      this.subtitleFormat\n    );\n\n    return [\n      new Document({\n        pageContent: subtitles,\n      }),\n    ];\n  }\n}\n\n// Path: langchain/src/document_loaders/web/notionapi.ts\nimport {\n  APIResponseError,\n  Client,\n  isFullBlock,\n  isFullPage,\n  iteratePaginatedAPI,\n  APIErrorCode,\n  isNotionClientError,\n  isFullDatabase,\n} from \"@notionhq/client\";\nimport { NotionToMarkdown } from \"notion-to-md\";\nimport { getBlockChildren } from \"notion-to-md/build/utils/notion.js\";\nimport type {\n  ListBlockChildrenResponseResults,\n  MdBlock,\n} from \"notion-to-md/build/types\";\nimport yaml from \"js-yaml\";\n\nimport { Document } from \"@langchain/core/documents\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport { BaseDocumentLoader } from \"../base.js\";\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype GuardType<T> = T extends (x: any, ...rest: any) => x is infer U\n  ? U\n  : never;\n\nexport type GetBlockResponse = Parameters<typeof isFullBlock>[0];\nexport type GetPageResponse = Parameters<typeof isFullPage>[0];\nexport type GetDatabaseResponse = Parameters<typeof isFullDatabase>[0];\n\nexport type BlockObjectResponse = GuardType<typeof isFullBlock>;\nexport type PageObjectResponse = GuardType<typeof isFullPage>;\nexport type DatabaseObjectResponse = GuardType<typeof isFullDatabase>;\n\nexport type GetResponse =\n  | GetBlockResponse\n  | GetPageResponse\n  | GetDatabaseResponse\n  | APIResponseError;\n\nexport type PagePropertiesType = PageObjectResponse[\"properties\"];\nexport type PagePropertiesValue = PagePropertiesType[keyof PagePropertiesType];\n\nexport const isPageResponse = (res: GetResponse): res is GetPageResponse =>\n  !isNotionClientError(res) && res.object === \"page\";\nexport const isDatabaseResponse = (\n  res: GetResponse\n): res is GetDatabaseResponse =>\n  !isNotionClientError(res) && res.object === \"database\";\nexport const isErrorResponse = (res: GetResponse): res is APIResponseError =>\n  isNotionClientError(res);\n\nexport const isPage = (res: GetResponse): res is PageObjectResponse =>\n  isPageResponse(res) && isFullPage(res);\nexport const isDatabase = (res: GetResponse): res is DatabaseObjectResponse =>\n  isDatabaseResponse(res) && isFullDatabase(res);\n\n/**\n * Represents the type of Notion API to load documents from. The options\n * are \"database\" or \"page\".\n */\n// @deprecated `type` property is now automatically determined.\nexport type NotionAPIType = \"database\" | \"page\";\n\nexport type OnDocumentLoadedCallback = (\n  current: number,\n  total: number,\n  currentTitle?: string,\n  rootTitle?: string\n) => void;\n\nexport type NotionAPILoaderOptions = {\n  clientOptions: ConstructorParameters<typeof Client>[0];\n  id: string;\n  type?: NotionAPIType; // @deprecated `type` property is now automatically determined.\n  callerOptions?: ConstructorParameters<typeof AsyncCaller>[0];\n  onDocumentLoaded?: OnDocumentLoadedCallback;\n  propertiesAsHeader?: boolean;\n};\n\n/**\n * A class that extends the BaseDocumentLoader class. It represents a\n * document loader for loading documents from Notion using the Notion API.\n * @example\n * ```typescript\n * const pageLoader = new NotionAPILoader({\n *   clientOptions: { auth: \"<NOTION_INTEGRATION_TOKEN>\" },\n *   id: \"<PAGE_ID>\",\n *   type: \"page\",\n * });\n * const pageDocs = await pageLoader.loadAndSplit();\n * const dbLoader = new NotionAPILoader({\n *   clientOptions: { auth: \"<NOTION_INTEGRATION_TOKEN>\" },\n *   id: \"<DATABASE_ID>\",\n *   type: \"database\",\n *   propertiesAsHeader: true,\n * });\n * const dbDocs = await dbLoader.load();\n * ```\n */\nexport class NotionAPILoader extends BaseDocumentLoader {\n  private caller: AsyncCaller;\n\n  private notionClient: Client;\n\n  private n2mClient: NotionToMarkdown;\n\n  private id: string;\n\n  private pageQueue: string[];\n\n  private pageCompleted: string[];\n\n  public pageQueueTotal: number;\n\n  private documents: Document[];\n\n  private rootTitle: string;\n\n  private onDocumentLoaded: OnDocumentLoadedCallback;\n\n  private propertiesAsHeader: boolean;\n\n  constructor(options: NotionAPILoaderOptions) {\n    super();\n\n    this.caller = new AsyncCaller({\n      maxConcurrency: 64,\n      ...options.callerOptions,\n    });\n    this.notionClient = new Client({\n      logger: () => {}, // Suppress Notion SDK logger\n      ...options.clientOptions,\n    });\n    this.n2mClient = new NotionToMarkdown({\n      notionClient: this.notionClient,\n      config: { parseChildPages: false, convertImagesToBase64: false },\n    });\n    this.id = options.id;\n    this.pageQueue = [];\n    this.pageCompleted = [];\n    this.pageQueueTotal = 0;\n    this.documents = [];\n    this.rootTitle = \"\";\n    this.onDocumentLoaded = options.onDocumentLoaded ?? ((_ti, _cu) => {});\n    this.propertiesAsHeader = options.propertiesAsHeader || false;\n  }\n\n  /**\n   * Adds a selection of page ids to the pageQueue and removes duplicates.\n   * @param items An array of string ids\n   */\n  private addToQueue(...items: string[]) {\n    const deDuped = items.filter(\n      (item) => !this.pageCompleted.concat(this.pageQueue).includes(item)\n    );\n    this.pageQueue.push(...deDuped);\n    this.pageQueueTotal += deDuped.length;\n  }\n\n  /**\n   * Parses a Notion GetResponse object (page or database) and returns a string of the title.\n   * @param obj The Notion GetResponse object to parse.\n   * @returns The string of the title.\n   */\n  private getTitle(obj: GetResponse) {\n    if (isPage(obj)) {\n      const titleProp = Object.values(obj.properties).find(\n        (prop) => prop.type === \"title\"\n      );\n      if (titleProp) return this.getPropValue(titleProp);\n    }\n    if (isDatabase(obj))\n      return obj.title\n        .map((v) =>\n          this.n2mClient.annotatePlainText(v.plain_text, v.annotations)\n        )\n        .join(\"\");\n    return null;\n  }\n\n  /**\n   * Parses the property type and returns a string\n   * @param page The Notion page property to parse.\n   * @returns A string of parsed property.\n   */\n  private getPropValue(prop: PagePropertiesValue) {\n    switch (prop.type) {\n      case \"number\": {\n        const propNumber = prop[prop.type];\n        return propNumber !== null ? propNumber.toString() : \"\";\n      }\n      case \"url\":\n        return prop[prop.type] || \"\";\n      case \"select\":\n        return prop[prop.type]?.name ?? \"\";\n      case \"multi_select\":\n        return `[${prop[prop.type].map((v) => `\"${v.name}\"`).join(\", \")}]`;\n      case \"status\":\n        return prop[prop.type]?.name ?? \"\";\n      case \"date\":\n        return `${prop[prop.type]?.start ?? \"\"}${\n          prop[prop.type]?.end ? ` - ${prop[prop.type]?.end}` : \"\"\n        }`;\n      case \"email\":\n        return prop[prop.type] || \"\";\n      case \"phone_number\":\n        return prop[prop.type] || \"\";\n      case \"checkbox\":\n        return prop[prop.type].toString();\n      case \"files\":\n        return `[${prop[prop.type].map((v) => `\"${v.name}\"`).join(\", \")}]`;\n      case \"created_by\":\n        return `[\"${prop[prop.type].object}\", \"${prop[prop.type].id}\"]`;\n      case \"created_time\":\n        return prop[prop.type];\n      case \"last_edited_by\":\n        return `[\"${prop[prop.type].object}\", \"${prop[prop.type].id}\"]`;\n      case \"last_edited_time\":\n        return prop[prop.type];\n      case \"title\":\n        return prop[prop.type]\n          .map((v) =>\n            this.n2mClient.annotatePlainText(v.plain_text, v.annotations)\n          )\n          .join(\"\");\n      case \"rich_text\":\n        return prop[prop.type]\n          .map((v) =>\n            this.n2mClient.annotatePlainText(v.plain_text, v.annotations)\n          )\n          .join(\"\");\n      case \"people\":\n        return `[${prop[prop.type]\n          .map((v) => `[\"${v.object}\", \"${v.id}\"]`)\n          .join(\", \")}]`;\n      case \"unique_id\":\n        return `${prop[prop.type].prefix || \"\"}${prop[prop.type].number}`;\n      case \"relation\":\n        return `[${prop[prop.type].map((v) => `\"${v.id}\"`).join(\", \")}]`;\n      default:\n        return `Unsupported type: ${prop.type}`;\n    }\n  }\n\n  /**\n   * Parses the properties of a Notion page and returns them as key-value\n   * pairs.\n   * @param page The Notion page to parse.\n   * @returns An object containing the parsed properties as key-value pairs.\n   */\n  private parsePageProperties(page: PageObjectResponse) {\n    return Object.entries(page.properties).reduce((accum, [propName, prop]) => {\n      const value = this.getPropValue(prop);\n      const props = { ...accum, [propName]: value };\n      return prop.type === \"title\" ? { ...props, _title: value } : props;\n    }, {} as { [key: string]: string });\n  }\n\n  /**\n   * Parses the details of a Notion page and returns them as an object.\n   * @param page The Notion page to parse.\n   * @returns An object containing the parsed details of the page.\n   */\n  private parsePageDetails(page: PageObjectResponse) {\n    const { id, ...rest } = page;\n    return {\n      ...rest,\n      notionId: id,\n      properties: this.parsePageProperties(page),\n    };\n  }\n\n  /**\n   * Loads a Notion block and returns it as an MdBlock object.\n   * @param block The Notion block to load.\n   * @returns A Promise that resolves to an MdBlock object.\n   */\n  private async loadBlock(block: BlockObjectResponse): Promise<MdBlock> {\n    const mdBlock: MdBlock = {\n      type: block.type,\n      blockId: block.id,\n      parent: await this.caller.call(() =>\n        this.n2mClient.blockToMarkdown(block)\n      ),\n      children: [],\n    };\n\n    if (block.has_children) {\n      const block_id =\n        block.type === \"synced_block\" &&\n        block.synced_block?.synced_from?.block_id\n          ? block.synced_block.synced_from.block_id\n          : block.id;\n\n      const childBlocks = await this.loadBlocks(\n        await this.caller.call(() =>\n          getBlockChildren(this.notionClient, block_id, null)\n        )\n      );\n\n      mdBlock.children = childBlocks;\n    }\n\n    return mdBlock;\n  }\n\n  /**\n   * Loads Notion blocks and their children recursively.\n   * @param blocksResponse The response from the Notion API containing the blocks to load.\n   * @returns A Promise that resolves to an array containing the loaded MdBlocks.\n   */\n  private async loadBlocks(\n    blocksResponse: ListBlockChildrenResponseResults\n  ): Promise<MdBlock[]> {\n    const blocks = blocksResponse.filter(isFullBlock);\n\n    // Add child pages to queue\n    const childPages = blocks\n      .filter((block) => block.type.includes(\"child_page\"))\n      .map((block) => block.id);\n    if (childPages.length > 0) this.addToQueue(...childPages);\n\n    // Add child database pages to queue\n    const childDatabases = blocks\n      .filter((block) => block.type.includes(\"child_database\"))\n      .map((block) => this.caller.call(() => this.loadDatabase(block.id)));\n\n    // Load this block and child blocks\n    const loadingMdBlocks = blocks\n      .filter((block) => ![\"child_page\", \"child_database\"].includes(block.type))\n      .map((block) => this.loadBlock(block));\n\n    const [mdBlocks] = await Promise.all([\n      Promise.all(loadingMdBlocks),\n      Promise.all(childDatabases),\n    ]);\n\n    return mdBlocks;\n  }\n\n  /**\n   * Loads a Notion page and its child documents, then adds it to the completed documents array.\n   * @param page The Notion page or page ID to load.\n   */\n  private async loadPage(page: string | PageObjectResponse) {\n    // Check page is a page ID or a PageObjectResponse\n    const [pageData, pageId] =\n      typeof page === \"string\"\n        ? [\n            this.caller.call(() =>\n              this.notionClient.pages.retrieve({ page_id: page })\n            ),\n            page,\n          ]\n        : [page, page.id];\n\n    const [pageDetails, pageBlocks] = await Promise.all([\n      pageData,\n      this.caller.call(() => getBlockChildren(this.notionClient, pageId, null)),\n    ]);\n\n    if (!isFullPage(pageDetails)) {\n      this.pageCompleted.push(pageId);\n      return;\n    }\n\n    const mdBlocks = await this.loadBlocks(pageBlocks);\n    const mdStringObject = this.n2mClient.toMarkdownString(mdBlocks);\n\n    let pageContent = mdStringObject.parent;\n    const metadata = this.parsePageDetails(pageDetails);\n\n    if (this.propertiesAsHeader) {\n      pageContent =\n        `---\\n` +\n        `${yaml.dump(metadata.properties)}` +\n        `---\\n\\n` +\n        `${pageContent ?? \"\"}`;\n    }\n\n    if (!pageContent) {\n      this.pageCompleted.push(pageId);\n      return;\n    }\n\n    const pageDocument = new Document({ pageContent, metadata });\n\n    this.documents.push(pageDocument);\n    this.pageCompleted.push(pageId);\n    this.onDocumentLoaded(\n      this.documents.length,\n      this.pageQueueTotal,\n      this.getTitle(pageDetails) || undefined,\n      this.rootTitle\n    );\n...\n", "needle_token_start": 766, "needle_token_end": 798, "code_context_ntokens": 16083, "output": ["```typescript\nfunction isBinaryPath(name: string) {\n  return extensions.has(extname(name).slice(1).toLowerCase());\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "isFilterEmpty", "language": "typescript", "path": "langchain/src/retrievers/self_query/utils.ts", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To determine if a given argument, which can be a function, object, string, or undefined, lacks meaningful content or definition.\n2. **Input**: A single argument that can be of various types including a function, an object, a string, or undefined.\n3. **Output**: A boolean value; `true` if the argument is undefined, an empty string, an empty object, or simply not provided, and `false` otherwise.\n4. **Procedure**: The function first checks if the argument is falsy (e.g., undefined or an empty string). If it is a string and not empty, it returns `false`. If it is a function, it also returns `false`. For objects, it checks if the object is not an array and if it has no properties, returning `true` if both conditions are met. Otherwise, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/retrievers/tests/score_threshold.int.test.ts\n/* eslint-disable no-process-env */\nimport { expect, test } from \"@jest/globals\";\nimport { OpenAIEmbeddings, ChatOpenAI } from \"@langchain/openai\";\nimport { ConversationalRetrievalQAChain } from \"../../chains/conversational_retrieval_chain.js\";\nimport { BufferMemory } from \"../../memory/buffer_memory.js\";\nimport { MemoryVectorStore } from \"../../vectorstores/memory.js\";\nimport { ScoreThresholdRetriever } from \"../score_threshold.js\";\n\ntest(\"ConversationalRetrievalQAChain.fromLLM should use its vector store recursively until it gets all the similar results with the minimum similarity score provided\", async () => {\n  const vectorStore = await MemoryVectorStore.fromTexts(\n    [\n...\n// Path: langchain/src/retrievers/tests/matryoshka_retriever.int.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { OpenAIEmbeddings } from \"@langchain/openai\";\nimport { faker } from \"@faker-js/faker\";\nimport { Document } from \"@langchain/core/documents\";\nimport { v4 as uuidV4 } from \"uuid\";\nimport { SyntheticEmbeddings } from \"@langchain/core/utils/testing\";\nimport { MatryoshkaRetriever } from \"../matryoshka_retriever.js\";\nimport { MemoryVectorStore } from \"../../vectorstores/memory.js\";\n\ntest(\"MatryoshkaRetriever can retrieve\", async () => {\n  const smallEmbeddings = new OpenAIEmbeddings({\n    modelName: \"text-embedding-3-small\",\n    dimensions: 512, // Min num for small\n  });\n  const largeEmbeddings = new OpenAIEmbeddings({\n    modelName: \"text-embedding-3-large\",\n    dimensions: 3072, // Max num for large\n  });\n\n  const vectorStore = new MemoryVectorStore(smallEmbeddings);\n\n  const retriever = new MatryoshkaRetriever({\n    largeEmbeddingModel: largeEmbeddings,\n    vectorStore,\n    largeK: 5,\n  });\n\n  const irrelevantDocs = Array.from({ length: 250 }).map(\n    () =>\n      new Document({\n        pageContent: faker.lorem.words(7),\n        metadata: { id: uuidV4() },\n      })\n  );\n  const relevantDocContents = [\n    \"LangChain is an open source github repo\",\n    \"There are JS and PY versions of the LangChain github repos\",\n    \"LangGraph is a new open source library by the LangChain team\",\n    \"LangChain announced GA of LangSmith last week!\",\n    \"I heart LangChain\",\n  ];\n  const relevantDocs = [\n    new Document({\n      pageContent: relevantDocContents[0],\n      metadata: { id: uuidV4() },\n    }),\n    new Document({\n      pageContent: relevantDocContents[1],\n      metadata: { id: uuidV4() },\n    }),\n    new Document({\n      pageContent: relevantDocContents[2],\n      metadata: { id: uuidV4() },\n    }),\n    new Document({\n      pageContent: relevantDocContents[3],\n      metadata: { id: uuidV4() },\n    }),\n    new Document({\n      pageContent: relevantDocContents[4],\n      metadata: { id: uuidV4() },\n    }),\n  ];\n  const allDocs = [...irrelevantDocs, ...relevantDocs];\n\n  await retriever.addDocuments(allDocs);\n\n  const query = \"What is LangChain?\";\n  console.log(\"Querying documents\");\n  const results = await retriever.getRelevantDocuments(query);\n\n  const retrieverResultContents = new Set(\n    results.map((doc) => doc.pageContent)\n  );\n\n  console.log([...retrieverResultContents]);\n  expect(results.length).toBe(5);\n  expect(retrieverResultContents).toEqual(new Set(relevantDocContents));\n});\n\ntest(\"Can change number of docs returned (largeK)\", async () => {\n  const smallEmbeddings = new SyntheticEmbeddings({\n    vectorSize: 512,\n  });\n  const largeEmbeddings = new SyntheticEmbeddings({\n    vectorSize: 3072,\n  });\n\n  const vectorStore = new MemoryVectorStore(smallEmbeddings);\n\n  const docsWithId = Array.from({ length: 10 }).map(\n    () =>\n      new Document({\n        pageContent: \"hello world\",\n        metadata: { id: uuidV4() },\n      })\n  );\n  const retriever = new MatryoshkaRetriever({\n    largeEmbeddingModel: largeEmbeddings,\n    vectorStore,\n    largeK: 10,\n  });\n\n  await retriever.addDocuments(docsWithId);\n\n  const query = \"hello world\";\n  const results = await retriever.getRelevantDocuments(query);\n  expect(results.length).toBe(10);\n});\n\ntest(\"AddDocunents adds large embeddings metadata field\", async () => {\n  const testId = uuidV4();\n  const doc = new Document({\n    pageContent: \"hello world\",\n    metadata: { id: testId },\n  });\n\n  const smallEmbeddings = new OpenAIEmbeddings({\n    modelName: \"text-embedding-3-small\",\n    dimensions: 512, // Min num for small\n  });\n  const largeEmbeddings = new OpenAIEmbeddings({\n    modelName: \"text-embedding-3-large\",\n    dimensions: 3072, // Max num for large\n  });\n\n  const vectorStore = new MemoryVectorStore(smallEmbeddings);\n\n  const retriever = new MatryoshkaRetriever({\n    largeEmbeddingModel: largeEmbeddings,\n    vectorStore,\n  });\n\n  await retriever.addDocuments([doc]);\n\n  const relevantDocs = await retriever.getRelevantDocuments(\"hello world\");\n  expect(relevantDocs[0].metadata.id).toBe(testId);\n  expect(relevantDocs[0].metadata[retriever.largeEmbeddingKey]).toBeDefined();\n  expect(\n    JSON.parse(relevantDocs[0].metadata[retriever.largeEmbeddingKey]).length\n  ).toBe(3072);\n});\n\n// Path: langchain/src/retrievers/tests/hyde.int.test.ts\nimport { expect, test } from \"@jest/globals\";\nimport { OpenAIEmbeddings, OpenAI } from \"@langchain/openai\";\nimport { Document } from \"@langchain/core/documents\";\nimport { MemoryVectorStore } from \"../../vectorstores/memory.js\";\nimport { HydeRetriever } from \"../hyde.js\";\n\ntest(\"Hyde retriever\", async () => {\n  const embeddings = new OpenAIEmbeddings();\n  const vectorStore = new MemoryVectorStore(embeddings);\n  const llm = new OpenAI();\n  const retriever = new HydeRetriever({\n    vectorStore,\n    llm,\n    k: 1,\n  });\n\n  await vectorStore.addDocuments(\n    [\n      \"My name is John.\",\n      \"My name is Bob.\",\n      \"My favourite food is pizza.\",\n      \"My favourite food is pasta.\",\n    ].map((pageContent) => new Document({ pageContent }))\n  );\n\n  const results = await retriever.getRelevantDocuments(\n    \"What is my favourite food?\"\n  );\n\n  expect(results.length).toBe(1);\n  console.log(results);\n});\n\ntest(\"Hyde retriever with default prompt template\", async () => {\n  const embeddings = new OpenAIEmbeddings();\n  const vectorStore = new MemoryVectorStore(embeddings);\n  const llm = new OpenAI();\n  const retriever = new HydeRetriever({\n    vectorStore,\n    llm,\n    k: 1,\n    promptTemplate: \"websearch\",\n  });\n\n  await vectorStore.addDocuments(\n    [\n      \"My name is John.\",\n      \"My name is Bob.\",\n      \"My favourite food is pizza.\",\n      \"My favourite food is pasta.\",\n    ].map((pageContent) => new Document({ pageContent }))\n  );\n\n  const results = await retriever.getRelevantDocuments(\n    \"What is my favourite food?\"\n  );\n\n  expect(results.length).toBe(1);\n  console.log(results);\n});\n\n// Path: langchain/src/retrievers/self_query/utils.ts\n/**\n * Checks if the provided argument is an object and not an array.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport function isObject(obj: any): obj is object {\n  return obj && typeof obj === \"object\" && !Array.isArray(obj);\n}\n\n/**\n * Checks if a provided filter is empty. The filter can be a function, an\n * object, a string, or undefined.\n */\nexport \nfunction isFilterEmpty(\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  filter: ((q: any) => any) | object | string | undefined\n): filter is undefined {\n  if (!filter) return true;\n  // for Milvus\n  if (typeof filter === \"string\" && filter.length > 0) {\n    return false;\n  }\n  if (typeof filter === \"function\") {\n    return false;\n  }\n  return isObject(filter) && Object.keys(filter).length === 0;\n}\n\n/**\n * Checks if the provided value is an integer.\n */\nexport function isInt(value: unknown): boolean {\n  if (typeof value === \"number\") {\n    return value % 1 === 0;\n  } else if (typeof value === \"string\") {\n    const numberValue = parseInt(value, 10);\n    return (\n      !Number.isNaN(numberValue) &&\n      numberValue % 1 === 0 &&\n      numberValue.toString() === value\n    );\n  }\n\n  return false;\n}\n\n/**\n * Checks if the provided value is a floating-point number.\n */\nexport function isFloat(value: unknown): boolean {\n  if (typeof value === \"number\") {\n    return value % 1 !== 0;\n  } else if (typeof value === \"string\") {\n    const numberValue = parseFloat(value);\n    return (\n      !Number.isNaN(numberValue) &&\n      numberValue % 1 !== 0 &&\n      numberValue.toString() === value\n    );\n  }\n\n  return false;\n}\n\n/**\n * Checks if the provided value is a string that cannot be parsed into a\n * number.\n */\nexport function isString(value: unknown): boolean {\n  return (\n    typeof value === \"string\" &&\n    (Number.isNaN(parseFloat(value)) || parseFloat(value).toString() !== value)\n  );\n}\n\n/**\n * Casts a value that might be string or number to actual string or number.\n * Since LLM might return back an integer/float as a string, we need to cast\n * it back to a number, as many vector databases can't handle number as string\n * values as a comparator.\n */\nexport function castValue(input: unknown): string | number {\n  let value;\n  if (isString(input)) {\n    value = input as string;\n  } else if (isInt(input)) {\n    value = parseInt(input as string, 10);\n  } else if (isFloat(input)) {\n    value = parseFloat(input as string);\n  } else {\n    throw new Error(\"Unsupported value type\");\n  }\n\n  return value;\n}\n\n// Path: langchain/src/chains/query_constructor/ir.ts\nimport { VectorStore } from \"@langchain/core/vectorstores\";\n\n/**\n * Represents logical AND operator.\n */\nexport type AND = \"and\";\n/**\n * Represents logical OR operator.\n */\nexport type OR = \"or\";\n/**\n * Represents logical NOT operator.\n */\nexport type NOT = \"not\";\n\n/**\n * Represents a logical operator which can be AND, OR, or NOT.\n */\nexport type Operator = AND | OR | NOT;\n\n/**\n * Represents equality comparison operator.\n */\nexport type EQ = \"eq\";\n/**\n * Represents inequality comparison operator.\n */\nexport type NE = \"ne\";\n/**\n * Represents less than comparison operator.\n */\nexport type LT = \"lt\";\n/**\n * Represents greater than comparison operator.\n */\nexport type GT = \"gt\";\n/**\n * Represents less than or equal to comparison operator.\n */\nexport type LTE = \"lte\";\n/**\n * Represents greater than or equal to comparison operator.\n */\nexport type GTE = \"gte\";\n\n/**\n * Represents a comparison operator which can be EQ, NE, LT, GT, LTE, or\n * GTE.\n */\nexport type Comparator = EQ | NE | LT | GT | LTE | GTE;\n\nexport const Operators: { [key: string]: Operator } = {\n  and: \"and\",\n  or: \"or\",\n  not: \"not\",\n};\n\nexport const Comparators: { [key: string]: Comparator } = {\n  eq: \"eq\",\n  ne: \"ne\",\n  lt: \"lt\",\n  gt: \"gt\",\n  lte: \"lte\",\n  gte: \"gte\",\n};\n\n/**\n * Represents the result of visiting an operation or comparison\n * expression.\n */\nexport type VisitorResult = VisitorOperationResult | VisitorComparisonResult;\n\n/**\n * Represents the result of visiting an operation expression.\n */\nexport type VisitorOperationResult = {\n  [operator: string]: VisitorResult[];\n};\n\n/**\n * Represents the result of visiting a comparison expression.\n */\nexport type VisitorComparisonResult = {\n  [attr: string]: {\n    [comparator: string]: string | number;\n  };\n};\n\n/**\n * Represents the result of visiting a structured query expression.\n */\nexport type VisitorStructuredQueryResult = {\n  filter?: VisitorComparisonResult | VisitorOperationResult;\n};\n\n/**\n * Abstract class for visiting expressions. Subclasses must implement\n * visitOperation, visitComparison, and visitStructuredQuery methods.\n */\nexport abstract class Visitor<T extends VectorStore = VectorStore> {\n  declare VisitOperationOutput: object;\n\n  declare VisitComparisonOutput: object;\n\n  declare VisitStructuredQueryOutput: { filter?: T[\"FilterType\"] };\n\n  abstract allowedOperators: Operator[];\n\n  abstract allowedComparators: Comparator[];\n\n  abstract visitOperation(operation: Operation): this[\"VisitOperationOutput\"];\n\n  abstract visitComparison(\n    comparison: Comparison\n  ): this[\"VisitComparisonOutput\"];\n\n  abstract visitStructuredQuery(\n    structuredQuery: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"];\n}\n\n/**\n * Abstract class representing an expression. Subclasses must implement\n * the exprName property and the accept method.\n */\nexport abstract class Expression {\n  abstract exprName: \"Operation\" | \"Comparison\" | \"StructuredQuery\";\n\n  accept(visitor: Visitor) {\n    if (this.exprName === \"Operation\") {\n      return visitor.visitOperation(this as unknown as Operation);\n    } else if (this.exprName === \"Comparison\") {\n      return visitor.visitComparison(this as unknown as Comparison);\n    } else if (this.exprName === \"StructuredQuery\") {\n      return visitor.visitStructuredQuery(this as unknown as StructuredQuery);\n    } else {\n      throw new Error(\"Unknown Expression type\");\n    }\n  }\n}\n\n/**\n * Abstract class representing a filter directive. It extends the\n * Expression class.\n */\nexport abstract class FilterDirective extends Expression {}\n\n/**\n * Class representing a comparison filter directive. It extends the\n * FilterDirective class.\n */\nexport class Comparison extends FilterDirective {\n  exprName = \"Comparison\" as const;\n\n  constructor(\n    public comparator: Comparator,\n    public attribute: string,\n    public value: string | number\n  ) {\n    super();\n  }\n}\n\n/**\n * Class representing an operation filter directive. It extends the\n * FilterDirective class.\n */\nexport class Operation extends FilterDirective {\n  exprName = \"Operation\" as const;\n\n  constructor(public operator: Operator, public args?: FilterDirective[]) {\n    super();\n  }\n}\n\n/**\n * Class representing a structured query expression. It extends the\n * Expression class.\n */\nexport class StructuredQuery extends Expression {\n  exprName = \"StructuredQuery\" as const;\n\n  constructor(public query: string, public filter?: FilterDirective) {\n    super();\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/base.ts\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n  Visitor,\n  VisitorComparisonResult,\n  VisitorOperationResult,\n  VisitorResult,\n  VisitorStructuredQueryResult,\n} from \"../../chains/query_constructor/ir.js\";\nimport { isFilterEmpty, castValue } from \"./utils.js\";\n\n/**\n * Options object for the BasicTranslator class. Specifies the allowed\n * operators and comparators.\n */\nexport type TranslatorOpts = {\n  allowedOperators: Operator[];\n  allowedComparators: Comparator[];\n};\n\n/**\n * Abstract class that provides a blueprint for creating specific\n * translator classes. Defines two abstract methods: formatFunction and\n * mergeFilters.\n */\nexport abstract class BaseTranslator<\n  T extends VectorStore = VectorStore\n> extends Visitor<T> {\n  /**\n   * Formats a given function (either an operator or a comparator) into a\n   * string.\n   * @param func The function to format.\n   * @returns Formatted string representation of the function.\n   */\n  abstract formatFunction(func: Operator | Comparator): string;\n\n  /**\n   * Merges two filters into one, using a specified merge type.\n   * @param defaultFilter The default filter.\n   * @param generatedFilter The generated filter.\n   * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'.\n   * @param forceDefaultFilter If true, the default filter will be used even if the generated filter is not empty.\n   * @returns The merged filter, or undefined if both filters are empty.\n   */\n  abstract mergeFilters(\n    defaultFilter: this[\"VisitStructuredQueryOutput\"][\"filter\"] | undefined,\n    generatedFilter: this[\"VisitStructuredQueryOutput\"][\"filter\"] | undefined,\n    mergeType?: \"and\" | \"or\" | \"replace\",\n    forceDefaultFilter?: boolean\n  ): this[\"VisitStructuredQueryOutput\"][\"filter\"] | undefined;\n}\n\n/**\n * Class that extends the BaseTranslator class and provides concrete\n * implementations for the abstract methods. Also declares three types:\n * VisitOperationOutput, VisitComparisonOutput, and\n * VisitStructuredQueryOutput, which are used as the return types for the\n * visitOperation, visitComparison, and visitStructuredQuery methods\n * respectively.\n */\nexport class BasicTranslator<\n  T extends VectorStore = VectorStore\n> extends BaseTranslator<T> {\n  declare VisitOperationOutput: VisitorOperationResult;\n\n  declare VisitComparisonOutput: VisitorComparisonResult;\n\n  declare VisitStructuredQueryOutput: VisitorStructuredQueryResult;\n\n  allowedOperators: Operator[];\n\n  allowedComparators: Comparator[];\n\n  constructor(opts?: TranslatorOpts) {\n    super();\n    this.allowedOperators = opts?.allowedOperators ?? [\n      Operators.and,\n      Operators.or,\n    ];\n    this.allowedComparators = opts?.allowedComparators ?? [\n      Comparators.eq,\n      Comparators.ne,\n      Comparators.gt,\n      Comparators.gte,\n      Comparators.lt,\n      Comparators.lte,\n    ];\n  }\n\n  formatFunction(func: Operator | Comparator): string {\n    if (func in Comparators) {\n      if (\n        this.allowedComparators.length > 0 &&\n        this.allowedComparators.indexOf(func as Comparator) === -1\n      ) {\n        throw new Error(\n          `Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else if (func in Operators) {\n      if (\n        this.allowedOperators.length > 0 &&\n        this.allowedOperators.indexOf(func as Operator) === -1\n      ) {\n        throw new Error(\n          `Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else {\n      throw new Error(\"Unknown comparator or operator\");\n    }\n    return `$${func}`;\n  }\n\n  /**\n   * Visits an operation and returns a result.\n   * @param operation The operation to visit.\n   * @returns The result of visiting the operation.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const args = operation.args?.map((arg) =>\n      arg.accept(this)\n    ) as VisitorResult[];\n    return {\n      [this.formatFunction(operation.operator)]: args,\n    };\n  }\n\n  /**\n   * Visits a comparison and returns a result.\n   * @param comparison The comparison to visit.\n   * @returns The result of visiting the comparison.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    return {\n      [comparison.attribute]: {\n        [this.formatFunction(comparison.comparator)]: castValue(\n          comparison.value\n        ),\n      },\n    };\n  }\n\n  /**\n   * Visits a structured query and returns a result.\n   * @param query The structured query to visit.\n   * @returns The result of visiting the structured query.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    let nextArg = {};\n    if (query.filter) {\n      nextArg = {\n        filter: query.filter.accept(this),\n      };\n    }\n    return nextArg;\n  }\n\n  mergeFilters(\n    defaultFilter: VisitorStructuredQueryResult[\"filter\"] | undefined,\n    generatedFilter: VisitorStructuredQueryResult[\"filter\"] | undefined,\n    mergeType = \"and\",\n    forceDefaultFilter = false\n  ): VisitorStructuredQueryResult[\"filter\"] | undefined {\n    if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter)) {\n      if (forceDefaultFilter) {\n        return defaultFilter;\n      }\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n    if (mergeType === \"and\") {\n      return {\n        $and: [defaultFilter, generatedFilter],\n      };\n    } else if (mergeType === \"or\") {\n      return {\n        $or: [defaultFilter, generatedFilter],\n      };\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/weaviate.ts\nimport {\n  WeaviateFilter,\n  WeaviateStore,\n} from \"@langchain/community/vectorstores/weaviate\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  NOT,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n  Visitor,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { isFilterEmpty, isString, isInt, isFloat } from \"./utils.js\";\n\ntype AllowedOperator = Exclude<Operator, NOT>;\n\ntype WeaviateOperatorValues = {\n  valueText: string;\n  valueInt: number;\n  valueNumber: number;\n  valueBoolean: boolean;\n};\n\ntype WeaviateOperatorKeys = keyof WeaviateOperatorValues;\n\ntype ExclusiveOperatorValue = {\n  [L in WeaviateOperatorKeys]: {\n    [key in L]: WeaviateOperatorValues[key];\n  } & Omit<{ [key in WeaviateOperatorKeys]?: never }, L>;\n}[WeaviateOperatorKeys];\n\nexport type WeaviateVisitorResult =\n  | WeaviateOperationResult\n  | WeaviateComparisonResult\n  | WeaviateStructuredQueryResult;\n\nexport type WeaviateOperationResult = {\n  operator: string;\n  operands: WeaviateVisitorResult[];\n};\nexport type WeaviateComparisonResult = {\n  path: [string];\n  operator: string;\n} & ExclusiveOperatorValue;\n\nexport type WeaviateStructuredQueryResult = {\n  filter?: {\n    where?: WeaviateComparisonResult | WeaviateOperationResult;\n  };\n};\n\n/**\n * A class that translates or converts data into a format that can be used\n * with Weaviate, a vector search engine. It extends the `BaseTranslator`\n * class and provides specific implementation for Weaviate.\n * @example\n * ```typescript\n * const selfQueryRetriever = new SelfQueryRetriever({\n *   llm: new ChatOpenAI(),\n *   vectorStore: new WeaviateStore(),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: [],\n *   structuredQueryTranslator: new WeaviateTranslator(),\n * });\n *\n * const relevantDocuments = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are rated higher than 8.5?\",\n * );\n * ```\n */\nexport class WeaviateTranslator<\n  T extends WeaviateStore\n> extends BaseTranslator<T> {\n  declare VisitOperationOutput: WeaviateOperationResult;\n\n  declare VisitComparisonOutput: WeaviateComparisonResult;\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.lt,\n    Comparators.lte,\n    Comparators.gt,\n    Comparators.gte,\n  ];\n\n  /**\n   * Formats the given function into a string representation. Throws an\n   * error if the function is not a known comparator or operator, or if it\n   * is not allowed.\n   * @param func The function to format, which can be an Operator or Comparator.\n   * @returns A string representation of the function.\n   */\n  formatFunction(func: Operator | Comparator): string {\n    if (func in Comparators) {\n      if (\n        this.allowedComparators.length > 0 &&\n        this.allowedComparators.indexOf(func as Comparator) === -1\n      ) {\n        throw new Error(\n          `Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else if (func in Operators) {\n      if (\n        this.allowedOperators.length > 0 &&\n        this.allowedOperators.indexOf(func as Operator) === -1\n      ) {\n        throw new Error(\n          `Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else {\n      throw new Error(\"Unknown comparator or operator\");\n    }\n    const dict = {\n      and: \"And\",\n      or: \"Or\",\n      eq: \"Equal\",\n      ne: \"NotEqual\",\n      lt: \"LessThan\",\n      lte: \"LessThanEqual\",\n      gt: \"GreaterThan\",\n      gte: \"GreaterThanEqual\",\n    };\n    return dict[func as Comparator | AllowedOperator];\n  }\n\n  /**\n   * Visits an operation and returns a WeaviateOperationResult. The\n   * operation's arguments are visited and the operator is formatted.\n   * @param operation The operation to visit.\n   * @returns A WeaviateOperationResult.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const args = operation.args?.map((arg) =>\n      arg.accept(this as Visitor)\n    ) as WeaviateVisitorResult[];\n    return {\n      operator: this.formatFunction(operation.operator),\n      operands: args,\n    };\n  }\n\n  /**\n   * Visits a comparison and returns a WeaviateComparisonResult. The\n   * comparison's value is checked for type and the comparator is formatted.\n   * Throws an error if the value type is not supported.\n   * @param comparison The comparison to visit.\n   * @returns A WeaviateComparisonResult.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    if (isString(comparison.value)) {\n      return {\n        path: [comparison.attribute],\n        operator: this.formatFunction(comparison.comparator),\n        valueText: comparison.value as string,\n      };\n    }\n    if (isInt(comparison.value)) {\n      return {\n        path: [comparison.attribute],\n        operator: this.formatFunction(comparison.comparator),\n        valueInt: parseInt(comparison.value as string, 10),\n      };\n    }\n    if (isFloat(comparison.value)) {\n      return {\n        path: [comparison.attribute],\n        operator: this.formatFunction(comparison.comparator),\n        valueNumber: parseFloat(comparison.value as string),\n      };\n    }\n\n    throw new Error(\"Value type is not supported\");\n  }\n\n  /**\n   * Visits a structured query and returns a WeaviateStructuredQueryResult.\n   * If the query has a filter, it is visited.\n   * @param query The structured query to visit.\n   * @returns A WeaviateStructuredQueryResult.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    let nextArg = {};\n    if (query.filter) {\n      nextArg = {\n        filter: { where: query.filter.accept(this as Visitor) },\n      };\n    }\n    return nextArg;\n  }\n\n  /**\n   * Merges two filters into one. If both filters are empty, returns\n   * undefined. If one filter is empty or the merge type is 'replace',\n   * returns the other filter. If the merge type is 'and' or 'or', returns a\n   * new filter with the merged results. Throws an error for unknown merge\n   * types.\n   * @param defaultFilter The default filter to merge.\n   * @param generatedFilter The generated filter to merge.\n   * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'. Defaults to 'and'.\n   * @returns A merged WeaviateFilter, or undefined if both filters are empty.\n   */\n  mergeFilters(\n    defaultFilter: WeaviateFilter | undefined,\n    generatedFilter: WeaviateFilter | undefined,\n    mergeType = \"and\"\n  ): WeaviateFilter | undefined {\n    if (\n      isFilterEmpty(defaultFilter?.where) &&\n      isFilterEmpty(generatedFilter?.where)\n    ) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter?.where) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter?.where)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter?.where)) {\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n    const merged: WeaviateOperationResult = {\n      operator: \"And\",\n      operands: [\n        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n        defaultFilter!.where as WeaviateVisitorResult,\n        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n        generatedFilter!.where as WeaviateVisitorResult,\n      ],\n    };\n    if (mergeType === \"and\") {\n      return {\n        where: merged,\n      } as WeaviateFilter;\n    } else if (mergeType === \"or\") {\n      merged.operator = \"Or\";\n      return {\n        where: merged,\n      } as WeaviateFilter;\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/chroma.ts\nimport { Chroma } from \"@langchain/community/vectorstores/chroma\";\nimport { Comparators, Operators } from \"../../chains/query_constructor/ir.js\";\nimport { BasicTranslator } from \"./base.js\";\n\n/**\n * Specialized translator for the Chroma vector database. It extends the\n * BasicTranslator class and translates internal query language elements\n * to valid filters. The class defines a subset of allowed logical\n * operators and comparators that can be used in the translation process.\n * @example\n * ```typescript\n * const chromaTranslator = new ChromaTranslator();\n * const selfQueryRetriever = new SelfQueryRetriever({\n *   llm: new ChatOpenAI(),\n *   vectorStore: new Chroma(),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: [],\n *   structuredQueryTranslator: chromaTranslator,\n * });\n *\n * const relevantDocuments = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are directed by Greta Gerwig?\",\n * );\n * ```\n */\nexport class ChromaTranslator<T extends Chroma> extends BasicTranslator<T> {\n  constructor() {\n    super({\n      allowedOperators: [Operators.and, Operators.or],\n      allowedComparators: [\n        Comparators.eq,\n        Comparators.ne,\n        Comparators.gt,\n        Comparators.gte,\n        Comparators.lt,\n        Comparators.lte,\n      ],\n    });\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/supabase_utils.ts\nimport type {\n  SupabaseFilter,\n  SupabaseFilterRPCCall,\n  SupabaseMetadata,\n} from \"@langchain/community/vectorstores/supabase\";\nimport {\n  Comparators,\n  Comparison,\n  Operation,\n  Operators,\n  StructuredQuery,\n} from \"../../chains/query_constructor/ir.js\";\n\ntype SupabaseFilterProps = keyof SupabaseFilter;\n\n/**\n * Utility class used to duplicate parameters for a proxy object,\n * specifically designed to work with `SupabaseFilter` objects. It\n * contains methods to handle different types of operations such as \"or\",\n * \"filter\", \"in\", \"contains\", \"textSearch\", \"match\", \"not\", and default\n * operations.\n */\nexport class ProxyParamsDuplicator {\n  duplicationAllowedOps: string[] = [\n    \"eq\",\n    \"neq\",\n    \"lt\",\n    \"lte\",\n    \"gt\",\n    \"gte\",\n    \"like\",\n    \"ilike\",\n    \"or\",\n    \"in\",\n    \"contains\",\n    \"match\",\n    \"not\",\n    \"textSearch\",\n    \"filter\",\n  ];\n\n  values: [string, string][] = [];\n\n  /**\n   * Creates a proxy handler for a `SupabaseFilter` object. The handler\n   * intercepts get operations and applies specific logic based on the\n   * property being accessed.\n   * @returns A proxy handler for a `SupabaseFilter` object.\n   */\n  buildProxyHandler() {\n    const proxyHandler: ProxyHandler<SupabaseFilter> = {\n      get: (target, prop, receiver) => {\n        if (typeof target[prop as SupabaseFilterProps] === \"function\") {\n          // eslint-disable-next-line @typescript-eslint/no-explicit-any\n          return (...args: any[]) => {\n            if (this.duplicationAllowedOps.includes(String(prop))) {\n              switch (String(prop)) {\n                case \"or\":\n                  // args[0]: filters, args[1]: { foreignTable }\n                  this.addOrClause(args[0], args[1]);\n                  break;\n                case \"filter\":\n                  // args[0]: column, args[1]: operator, args[2]: value\n                  this.addFilterClause(args[0], args[1], args[2]);\n                  break;\n                case \"in\":\n                  // args[0]: column, args[1]: values\n                  this.addInClause(args[0], args[1]);\n                  break;\n                case \"contains\":\n                  // args[0]: column, args[1]: value\n                  this.addContainsClause(args[0], args[1]);\n                  break;\n                case \"textSearch\":\n                  // args[0]: column, args[1]: query, args[2]: { config, type }\n                  this.addTextSearchClause(args[0], args[1], args[2]);\n                  break;\n                case \"match\":\n                  // args[0]: query\n                  this.addMatchClause(args[0]);\n                  break;\n                case \"not\":\n                  // args[0]: column, args[1]: operator, args[2]: value\n                  this.addNotClause(args[0], args[1], args[2]);\n                  break;\n                default:\n                  // args[0]: column, args[1]: value\n                  this.addDefaultOpClause(prop as string, args[0], args[1]);\n              }\n              return new Proxy(target, proxyHandler);\n            } else {\n              throw new Error(\n                \"Filter operation not supported for 'or' mergeFiltersOperator\"\n              );\n            }\n          };\n        } else {\n          return Reflect.get(target, prop, receiver);\n        }\n      },\n    };\n\n    return proxyHandler;\n  }\n\n  /**\n   * Removes type annotations from a value string.\n   * @param value The value string to clean.\n   * @returns The cleaned value string.\n   */\n  removeType(value: string) {\n    let cleanedValue = value;\n    if (cleanedValue.includes(\"::float\")) {\n      cleanedValue = cleanedValue.replace(\"::float\", \"\");\n    }\n    if (cleanedValue.includes(\"::int\")) {\n      cleanedValue = cleanedValue.replace(\"::int\", \"\");\n    }\n    return cleanedValue;\n  }\n\n  /**\n   * Adds a default operation clause to the values array.\n   * @param prop The operation property.\n   * @param column The column to apply the operation to.\n   * @param value The value for the operation.\n   */\n  addDefaultOpClause(prop: string, column: string, value: unknown) {\n    this.values.push([this.removeType(column), `${String(prop)}.${value}`]);\n  }\n\n  /**\n   * Adds an 'or' clause to the values array.\n   * @param filters The filters for the 'or' clause.\n   * @param foreignTable Optional foreign table for the 'or' clause.\n   */\n  addOrClause(\n    filters: string,\n    { foreignTable }: { foreignTable?: string } = {}\n  ) {\n    const key = foreignTable ? `${foreignTable}.or` : \"or\";\n    this.values.push([this.removeType(key), `(${filters})`]);\n  }\n\n  /**\n   * Adds a 'filter' clause to the values array.\n   * @param column The column to apply the filter to.\n   * @param operator The operator for the filter.\n   * @param value The value for the filter.\n   */\n  addFilterClause(column: string, operator: string, value: unknown) {\n    this.values.push([this.removeType(column), `${operator}.${value}`]);\n  }\n\n  /**\n   * Adds an 'in' clause to the values array.\n   * @param column The column to apply the 'in' clause to.\n   * @param values The values for the 'in' clause.\n   */\n  addInClause(column: string, values: unknown[]) {\n    const cleanedValues = values\n      .map((s) => {\n        if (typeof s === \"string\" && /[,()]/.test(s)) return `\"${s}\"`;\n        else return `${s}`;\n      })\n      .join(\",\");\n    this.values.push([this.removeType(column), `in.(${cleanedValues})`]);\n  }\n\n  /**\n   * Adds a 'contains' clause to the values array.\n   * @param column The column to apply the 'contains' clause to.\n   * @param value The value for the 'contains' clause.\n   */\n  addContainsClause(column: string, value: unknown) {\n    if (typeof value === \"string\") {\n      this.values.push([this.removeType(column), `cs.${value}`]);\n    } else if (Array.isArray(value)) {\n      this.values.push([this.removeType(column), `cs.{${value.join(\",\")}}`]);\n    } else {\n      this.values.push([\n        this.removeType(column),\n        `cs.${JSON.stringify(value)}`,\n      ]);\n    }\n  }\n\n  /**\n   * Adds a 'textSearch' clause to the values array.\n   * @param column The column to apply the 'textSearch' clause to.\n   * @param query The query for the 'textSearch' clause.\n   * @param config Optional configuration for the 'textSearch' clause.\n   * @param type Optional type for the 'textSearch' clause.\n   */\n  addTextSearchClause(\n    column: string,\n    query: string[],\n    {\n      config,\n      type,\n    }: { config?: string; type?: \"plain\" | \"phrase\" | \"websearch\" } = {}\n  ) {\n    let typePart = \"\";\n    if (type === \"plain\") {\n      typePart = \"pl\";\n    } else if (type === \"phrase\") {\n      typePart = \"ph\";\n    } else if (type === \"websearch\") {\n      typePart = \"w\";\n    }\n    const configPart = config === undefined ? \"\" : `(${config})`;\n    this.values.push([\n      this.removeType(column),\n      `${typePart}fts${configPart}.${query}`,\n    ]);\n  }\n\n  /**\n   * Adds a 'not' clause to the values array.\n   * @param column The column to apply the 'not' clause to.\n   * @param operator The operator for the 'not' clause.\n   * @param value The value for the 'not' clause.\n   */\n  addNotClause(column: string, operator: string, value: unknown) {\n    this.values.push([column, `not.${operator}.${value}`]);\n  }\n\n  /**\n   * Adds a 'match' clause to the values array.\n   * @param query The query for the 'match' clause.\n   */\n  addMatchClause(query: Record<string, unknown>) {\n    Object.entries(query).forEach(([column, value]) => {\n      this.values.push([column, `eq.${value}`]);\n    });\n  }\n\n  /**\n   * Returns the flattened parameters as a string.\n   * @returns The flattened parameters as a string.\n   */\n  flattenedParams() {\n    const mapped = this.values.map(([k, v]) => `${k}.${v}`);\n    if (mapped.length === 1) return mapped[0];\n    return `and(${mapped.join(\",\")})`;\n  }\n\n  /**\n   * Gets flattened parameters from a `SupabaseFilter` and a\n   * `SupabaseFilterRPCCall`.\n   * @param rpc The `SupabaseFilter` object.\n   * @param filter The `SupabaseFilterRPCCall` object.\n   * @returns The flattened parameters as a string.\n   */\n  static getFlattenedParams(\n    rpc: SupabaseFilter,\n    filter: SupabaseFilterRPCCall\n  ) {\n    const proxiedDuplicator = new ProxyParamsDuplicator();\n    const proxiedRpc = new Proxy(rpc, proxiedDuplicator.buildProxyHandler());\n    void filter(proxiedRpc);\n    return proxiedDuplicator.flattenedParams();\n  }\n}\n\n/**\n * Converts a `SupabaseMetadata` object into a `StructuredQuery` object.\n * The function creates a new `StructuredQuery` object and uses the\n * `Operation` and `Comparison` classes to build the query.\n */\nexport function convertObjectFilterToStructuredQuery(\n  objFilter: SupabaseMetadata\n): StructuredQuery {\n  return new StructuredQuery(\n    \"\",\n    new Operation(\n      Operators.and,\n      Object.entries(objFilter).map(\n        ([column, value]) => new Comparison(Comparators.eq, column, value)\n      )\n    )\n  );\n}\n\n// Path: langchain/src/chains/query_constructor/parser.ts\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  FilterDirective,\n  Operation,\n  Operator,\n  Operators,\n} from \"./ir.js\";\nimport {\n  CallExpressionType,\n  ExpressionParser,\n  ParsedType,\n} from \"../../output_parsers/expression.js\";\n\n/**\n * A type representing the possible types that can be traversed in an\n * expression.\n */\nexport type TraverseType =\n  | boolean\n  | Operation\n  | Comparison\n  | string\n  | number\n  | { [key: string]: TraverseType }\n  | TraverseType[];\n\n/**\n * A class for transforming and parsing query expressions.\n */\nexport class QueryTransformer {\n  constructor(\n    public allowedComparators: Comparator[] = [],\n    public allowedOperators: Operator[] = []\n  ) {}\n\n  /**\n   * Matches a function name to a comparator or operator. Throws an error if\n   * the function name is unknown or not allowed.\n   * @param funcName The function name to match.\n   * @returns The matched function name.\n   */\n  private matchFunctionName(funcName: string) {\n    if (funcName in Comparators) {\n      if (this.allowedComparators.length > 0) {\n        if (this.allowedComparators.includes(funcName as Comparator)) {\n          return funcName;\n        } else {\n          throw new Error(\"Received comparator not allowed\");\n        }\n      } else {\n        return funcName;\n      }\n    }\n    if (funcName in Operators) {\n      if (this.allowedOperators.length > 0) {\n        if (this.allowedOperators.includes(funcName as Operator)) {\n          return funcName;\n        } else {\n          throw new Error(\"Received operator not allowed\");\n        }\n      } else {\n        return funcName;\n      }\n    }\n    throw new Error(\"Unknown function name\");\n  }\n\n  /**\n   * Transforms a parsed expression into an operation or comparison. Throws\n   * an error if the parsed expression is not supported.\n   * @param parsed The parsed expression to transform.\n   * @returns The transformed operation or comparison.\n   */\n  private transform(parsed: CallExpressionType): Operation | Comparison {\n    const traverse = (node: ParsedType): TraverseType => {\n      switch (node.type) {\n        case \"call_expression\": {\n          if (typeof node.funcCall !== \"string\") {\n            throw new Error(\n              \"Property access expression and element access expression not supported\"\n            );\n          }\n          const funcName = this.matchFunctionName(node.funcCall);\n          if (funcName in Operators) {\n            return new Operation(\n              funcName as Operator,\n              node.args?.map((arg) => traverse(arg)) as FilterDirective[]\n            );\n          }\n          if (funcName in Comparators) {\n            if (node.args && node.args.length === 2) {\n              return new Comparison(\n                funcName as Comparator,\n                traverse(node.args[0]) as string,\n                traverse(node.args[1]) as string | number\n              );\n            }\n            throw new Error(\"Comparator must have exactly 2 arguments\");\n          }\n          throw new Error(\"Function name neither operator nor comparator\");\n        }\n        case \"string_literal\": {\n          return node.value;\n        }\n        case \"numeric_literal\": {\n          return node.value;\n        }\n        case \"array_literal\": {\n          return node.values.map((value) => traverse(value));\n        }\n        case \"object_literal\": {\n          return node.values.reduce((acc, value) => {\n            acc[value.identifier] = traverse(value.value);\n            return acc;\n          }, {} as { [key: string]: TraverseType });\n        }\n        case \"boolean_literal\": {\n          return node.value;\n        }\n        default: {\n          throw new Error(\"Unknown node type\");\n        }\n      }\n    };\n    return traverse(parsed) as Operation | Comparison;\n  }\n\n  /**\n   * Parses an expression and returns the transformed operation or\n   * comparison. Throws an error if the expression cannot be parsed.\n   * @param expression The expression to parse.\n   * @returns A Promise that resolves to the transformed operation or comparison.\n   */\n  async parse(expression: string): Promise<Operation | Comparison> {\n    const expressionParser = new ExpressionParser();\n    const parsed = (await expressionParser.parse(\n      expression\n    )) as CallExpressionType;\n    if (!parsed) {\n      throw new Error(\"Could not parse expression\");\n    }\n    return this.transform(parsed);\n  }\n}\n\n// Path: langchain/src/chains/query_constructor/prompt.ts\nimport { PromptTemplate } from \"@langchain/core/prompts\";\n\nexport const SONG_DATA_SOURCE = `\\\n\\`\\`\\`json\n{\n    \"content\": \"Lyrics of a song\",\n    \"attributes\": {\n        \"artist\": {\n            \"type\": \"string\",\n            \"description\": \"Name of the song artist\"\n        },\n        \"length\": {\n            \"type\": \"integer\",\n            \"description\": \"Length of the song in seconds\"\n        },\n        \"genre\": {\n            \"type\": \"string\",\n            \"description\": \"The song genre, one of 'pop', 'rock' or 'rap'\"\n        }\n    }\n}\n\\`\\`\\`\\\n`\n  .replaceAll(\"{\", \"{{\")\n  .replaceAll(\"}\", \"}}\");\n\nexport const FULL_ANSWER = `\\\n\\`\\`\\`json\n{{\n    \"query\": \"teenager love\",\n    \"filter\": \"and(or(eq(\\\\\"artist\\\\\", \\\\\"Taylor Swift\\\\\"), eq(\\\\\"artist\\\\\", \\\\\"Katy Perry\\\\\")), \\\nlt(\\\\\"length\\\\\", 180), eq(\\\\\"genre\\\\\", \\\\\"pop\\\\\"))\"\n}}`;\n\nexport const NO_FILTER_ANSWER = `\\\n\\`\\`\\`json\n{{\n    \"query\": \"\",\n    \"filter\": \"NO_FILTER\"\n}}\n\\`\\`\\`\\\n`;\n\nexport const DEFAULT_EXAMPLES = [\n  {\n    i: \"1\",\n    data_source: SONG_DATA_SOURCE,\n    user_query:\n      \"What are songs by Taylor Swift or Katy Perry about teenage romance under 3 minutes long in the dance pop genre\",\n    structured_request: FULL_ANSWER,\n  },\n  {\n    i: \"2\",\n    data_source: SONG_DATA_SOURCE,\n    user_query: \"What are songs that were not published on Spotify\",\n    structured_request: NO_FILTER_ANSWER,\n  },\n];\n\nexport const EXAMPLE_PROMPT_TEMPLATE = `\\\n<< Example {i}. >>\nData Source:\n{data_source}\n\nUser Query:\n{user_query}\n\nStructured Request:\n{structured_request}\n`;\n\nexport const EXAMPLE_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"i\", \"data_source\", \"user_query\", \"structured_request\"],\n  template: EXAMPLE_PROMPT_TEMPLATE,\n});\n\nexport const DEFAULT_SCHEMA = `\\\n<< Structured Request Schema >>\nWhen responding use a markdown code snippet with a JSON object formatted in the \\\nfollowing schema:\n\n\\`\\`\\`json\n{{{{\n    \"query\": string \\\\ text string to compare to document contents\n    \"filter\": string \\\\ logical condition statement for filtering documents\n}}}}\n\\`\\`\\`\n\nThe query string should contain only text that is expected to match the contents of \\\ndocuments. Any conditions in the filter should not be mentioned in the query as well.\n\nA logical condition statement is composed of one or more comparison and logical \\\noperation statements.\n\nA comparison statement takes the form: \\`comp(attr, val)\\`:\n- \\`comp\\` ({allowed_comparators}): comparator\n- \\`attr\\` (string):  name of attribute to apply the comparison to\n- \\`val\\` (string): is the comparison value\n\nA logical operation statement takes the form \\`op(statement1, statement2, ...)\\`:\n- \\`op\\` ({allowed_operators}): logical operator\n- \\`statement1\\`, \\`statement2\\`, ... (comparison statements or logical operation \\\nstatements): one or more statements to apply the operation to\n\nMake sure that you only use the comparators and logical operators listed above and \\\nno others.\nMake sure that filters only refer to attributes that exist in the data source.\nMake sure that filters only use the attributed names with its function names if there are functions applied on them.\nMake sure that filters only use format \\`YYYY-MM-DD\\` when handling timestamp data typed values.\nMake sure that filters take into account the descriptions of attributes and only make \\\ncomparisons that are feasible given the type of data being stored.\nMake sure that filters are only used as needed. If there are no filters that should be \\\napplied return \"NO_FILTER\" for the filter value.\\\n`;\n\nexport const DEFAULT_PREFIX = `\\\nYour goal is to structure the user's query to match the request schema provided below.\n\n{schema}\\\n`;\n\nexport const DEFAULT_SUFFIX = `\\\n<< Example {i}. >>\nData Source:\n\\`\\`\\`json\n{{{{\n    \"content\": \"{content}\",\n    \"attributes\": {attributes}\n}}}}\n\\`\\`\\`\n\nUser Query:\n{{query}}\n\nStructured Request:\n`;\n\n// Path: langchain/src/chains/query_constructor/index.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { z } from \"zod\";\nimport {\n  Example,\n  interpolateFString,\n  FewShotPromptTemplate,\n} from \"@langchain/core/prompts\";\nimport { InputValues } from \"@langchain/core/utils/types\";\nimport { QueryTransformer, TraverseType } from \"./parser.js\";\nimport {\n  Comparator,\n  Comparators,\n  Operator,\n  Operators,\n  StructuredQuery,\n} from \"./ir.js\";\nimport {\n  DEFAULT_EXAMPLES,\n  DEFAULT_PREFIX,\n  DEFAULT_SCHEMA,\n  DEFAULT_SUFFIX,\n  EXAMPLE_PROMPT,\n} from \"./prompt.js\";\nimport { LLMChain } from \"../llm_chain.js\";\nimport { AsymmetricStructuredOutputParser } from \"../../output_parsers/structured.js\";\nimport { AttributeInfo } from \"../../schema/query_constructor.js\";\n\nexport { QueryTransformer, type TraverseType };\nexport {\n  DEFAULT_EXAMPLES,\n  DEFAULT_PREFIX,\n  DEFAULT_SCHEMA,\n  DEFAULT_SUFFIX,\n  EXAMPLE_PROMPT,\n};\n\nconst queryInputSchema = /* #__PURE__ */ z.object({\n  query: /* #__PURE__ */ z\n    .string()\n    .describe(\"text string to compare to document contents\"),\n  filter: /* #__PURE__ */ z\n    .string()\n    .optional()\n    .describe(\"logical condition statement for filtering documents\"),\n});\n\n/**\n * A class that extends AsymmetricStructuredOutputParser to parse\n * structured query output.\n */\nexport class StructuredQueryOutputParser extends AsymmetricStructuredOutputParser<\n  typeof queryInputSchema,\n  StructuredQuery\n> {\n  lc_namespace = [\"langchain\", \"chains\", \"query_constructor\"];\n\n  queryTransformer: QueryTransformer;\n\n  constructor(fields: {\n    allowedComparators: Comparator[];\n    allowedOperators: Operator[];\n  }) {\n    super({ ...fields, inputSchema: queryInputSchema });\n\n    const { allowedComparators, allowedOperators } = fields;\n    this.queryTransformer = new QueryTransformer(\n      allowedComparators,\n      allowedOperators\n    );\n  }\n\n  /**\n   * Processes the output of a structured query.\n   * @param query The query string.\n   * @param filter The filter condition.\n   * @returns A Promise that resolves to a StructuredQuery instance.\n   */\n  async outputProcessor({\n    query,\n    filter,\n  }: z.infer<typeof queryInputSchema>): Promise<StructuredQuery> {\n    let myQuery = query;\n    if (myQuery.length === 0) {\n      myQuery = \" \";\n    }\n    if (filter === \"NO_FILTER\" || filter === undefined) {\n      return new StructuredQuery(query);\n    } else {\n      const parsedFilter = await this.queryTransformer.parse(filter);\n      return new StructuredQuery(query, parsedFilter);\n    }\n  }\n\n  /**\n   * Creates a new StructuredQueryOutputParser instance from the provided\n   * components.\n   * @param allowedComparators An array of allowed Comparator instances.\n   * @param allowedOperators An array of allowed Operator instances.\n   * @returns A new StructuredQueryOutputParser instance.\n   */\n  static fromComponents(\n    allowedComparators: Comparator[] = [],\n    allowedOperators: Operator[] = []\n  ) {\n    return new StructuredQueryOutputParser({\n      allowedComparators,\n      allowedOperators,\n    });\n  }\n}\n\nexport function formatAttributeInfo(info: AttributeInfo[]) {\n  const infoObj = info.reduce((acc, attr) => {\n    acc[attr.name] = {\n      type: attr.type,\n      description: attr.description,\n    };\n    return acc;\n  }, {} as { [name: string]: { type: string; description: string } });\n\n  return JSON.stringify(infoObj, null, 2)\n    .replaceAll(\"{\", \"{{\")\n    .replaceAll(\"}\", \"}}\");\n}\n\nconst defaultExample = DEFAULT_EXAMPLES.map((EXAMPLE) => EXAMPLE as Example);\n\nfunction _getPrompt(\n  documentContents: string,\n  attributeInfo: AttributeInfo[],\n  allowedComparators?: Comparator[],\n  allowedOperators?: Operator[],\n  examples: InputValues[] = defaultExample\n) {\n  const myAllowedComparators: Comparator[] =\n    allowedComparators ?? Object.values(Comparators);\n  const myAllowedOperators: Operator[] =\n    allowedOperators ?? Object.values(Operators);\n  const attributeJSON = formatAttributeInfo(attributeInfo);\n  const schema = interpolateFString(DEFAULT_SCHEMA, {\n    allowed_comparators: myAllowedComparators.join(\" | \"),\n    allowed_operators: myAllowedOperators.join(\" | \"),\n  });\n  const prefix = interpolateFString(DEFAULT_PREFIX, {\n    schema,\n  });\n  const suffix = interpolateFString(DEFAULT_SUFFIX, {\n    i: examples.length + 1,\n    content: documentContents,\n    attributes: attributeJSON,\n  });\n\n  const outputParser = StructuredQueryOutputParser.fromComponents(\n    allowedComparators,\n    allowedOperators\n  );\n\n  return new FewShotPromptTemplate({\n    examples,\n    examplePrompt: EXAMPLE_PROMPT,\n    inputVariables: [\"query\"],\n    suffix,\n    prefix,\n    outputParser,\n  });\n}\n\n/**\n * A type that represents options for the query constructor chain.\n */\nexport type QueryConstructorChainOptions = {\n  llm: BaseLanguageModelInterface;\n  documentContents: string;\n  attributeInfo: AttributeInfo[];\n  examples?: InputValues[];\n  allowedComparators?: Comparator[];\n  allowedOperators?: Operator[];\n};\n\nexport function loadQueryConstructorChain(opts: QueryConstructorChainOptions) {\n  const prompt = _getPrompt(\n    opts.documentContents,\n    opts.attributeInfo,\n    opts.allowedComparators,\n    opts.allowedOperators,\n    opts.examples\n  );\n  return new LLMChain({\n    llm: opts.llm,\n    prompt,\n  });\n}\n\n// Path: langchain/src/retrievers/self_query/functional.ts\nimport { Document } from \"@langchain/core/documents\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { castValue, isFilterEmpty } from \"./utils.js\";\n\n/**\n * A type alias for an object that maps comparison operators to string or\n * number values. This is used in the comparison functions to determine\n * the result of a comparison operation.\n */\ntype ValueType = {\n  eq: string | number;\n  ne: string | number;\n  lt: string | number;\n  lte: string | number;\n  gt: string | number;\n  gte: string | number;\n};\n\n/**\n * A type alias for a function that takes a `Document` as an argument and\n * returns a boolean. This function is used as a filter for documents.\n */\nexport type FunctionFilter = (document: Document) => boolean;\n\n/**\n * A class that extends `BaseTranslator` to translate structured queries\n * into functional filters.\n * @example\n * ```typescript\n * const functionalTranslator = new FunctionalTranslator();\n * const relevantDocuments = await functionalTranslator.getRelevantDocuments(\n *   \"Which movies are rated higher than 8.5?\",\n * );\n * ```\n */\nexport class FunctionalTranslator extends BaseTranslator {\n  declare VisitOperationOutput: FunctionFilter;\n\n  declare VisitComparisonOutput: FunctionFilter;\n\n  declare VisitStructuredQueryOutput:\n    | { filter: FunctionFilter }\n    | { [k: string]: never };\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.gt,\n    Comparators.gte,\n    Comparators.lt,\n    Comparators.lte,\n  ];\n\n  formatFunction(): string {\n    throw new Error(\"Not implemented\");\n  }\n\n  /**\n...\n", "needle_token_start": 2352, "needle_token_end": 2489, "code_context_ntokens": 16049, "output": ["```typescript\n/**\n * Checks if a provided filter is empty. The filter can be a function, an\n * object, a string, or undefined.\n */\nexport function isFilterEmpty(\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  filter: ((q: any) => any) | object | string | undefined\n): filter is undefined {\n  if (!filter) return true;\n  // for Milvus\n  if (typeof filter === \"string\" && filter.length > 0) {\n    return false;\n  }\n  if (typeof filter === \"function\") {\n    return false;\n  }\n  return isObject(filter) && Object.keys(filter).length === 0;\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "createOpenAIToolsAgent", "language": "typescript", "path": "langchain/src/agents/openai_tools/index.ts", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To construct an agent that utilizes a language model equipped with specific tools for processing and responding to inputs, following a defined prompt structure. This agent is capable of handling complex interactions by leveraging both the language model and the tools.\n2. **Input**: The function requires a language model that supports tool integration, a set of tools, a structured prompt template, and an optional boolean to enable streaming mode.\n3. **Output**: The function returns a sequence of operations encapsulated as an agent, which can process inputs and generate actions or conclude interactions based on the structured prompt and tools.\n4. **Procedure**: The function first validates the prompt to ensure it contains a specific input variable. It then binds the tools to the language model, formats the input steps for tool usage, and sequences these operations along with the prompt and a parser to form the agent. This agent can then be used to interactively process inputs and generate outputs.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/agents/toolkits/sql/sql.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { ToolInterface } from \"@langchain/core/tools\";\nimport { Toolkit } from \"@langchain/community/agents/toolkits/base\";\nimport { renderTemplate } from \"@langchain/core/prompts\";\nimport {\n  InfoSqlTool,\n  ListTablesSqlTool,\n  QueryCheckerTool,\n  QuerySqlTool,\n} from \"../../../tools/sql.js\";\nimport { SQL_PREFIX, SQL_SUFFIX } from \"./prompt.js\";\nimport { LLMChain } from \"../../../chains/llm_chain.js\";\nimport { ZeroShotAgent, ZeroShotCreatePromptArgs } from \"../../mrkl/index.js\";\nimport { AgentExecutor } from \"../../executor.js\";\nimport { SqlDatabase } from \"../../../sql_db.js\";\n\n/**\n * Interface that extends ZeroShotCreatePromptArgs and adds an optional\n * topK parameter for specifying the number of results to return.\n */\nexport interface SqlCreatePromptArgs extends ZeroShotCreatePromptArgs {\n  /** Number of results to return. */\n  topK?: number;\n}\n\n/**\n * Class that represents a toolkit for working with SQL databases. It\n * initializes SQL tools based on the provided SQL database.\n * @example\n * ```typescript\n * const model = new ChatOpenAI({});\n * const toolkit = new SqlToolkit(sqlDb, model);\n * const executor = createSqlAgent(model, toolkit);\n * const result = await executor.invoke({ input: 'List the total sales per country. Which country's customers spent the most?' });\n * console.log(`Got output ${result.output}`);\n * ```\n */\nexport class SqlToolkit extends Toolkit {\n  tools: ToolInterface[];\n\n  db: SqlDatabase;\n\n  dialect = \"sqlite\";\n\n  constructor(db: SqlDatabase, llm?: BaseLanguageModelInterface) {\n    super();\n    this.db = db;\n    this.tools = [\n      new QuerySqlTool(db),\n      new InfoSqlTool(db),\n      new ListTablesSqlTool(db),\n      new QueryCheckerTool({ llm }),\n    ];\n  }\n}\n\nexport function createSqlAgent(\n  llm: BaseLanguageModelInterface,\n...\n// Path: langchain/src/agents/toolkits/sql/index.ts\nexport { SqlCreatePromptArgs, SqlToolkit, createSqlAgent } from \"./sql.js\";\nexport { SQL_PREFIX, SQL_SUFFIX } from \"./prompt.js\";\n\n// Path: langchain/src/tools/vectorstore.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport { Tool } from \"@langchain/core/tools\";\nimport { VectorDBQAChain } from \"../chains/vector_db_qa.js\";\n\n/**\n * Interface for tools that interact with a Vector Store.\n */\ninterface VectorStoreTool {\n  vectorStore: VectorStoreInterface;\n  llm: BaseLanguageModelInterface;\n}\n\n/**\n * A tool for the VectorDBQA chain to interact with a Vector Store. It is\n * used to answer questions about a specific topic. The input to this tool\n * should be a fully formed question.\n */\nexport class VectorStoreQATool extends Tool implements VectorStoreTool {\n  static lc_name() {\n    return \"VectorStoreQATool\";\n  }\n\n  vectorStore: VectorStoreInterface;\n\n  llm: BaseLanguageModelInterface;\n\n  name: string;\n\n  description: string;\n\n  chain: VectorDBQAChain;\n\n  constructor(name: string, description: string, fields: VectorStoreTool) {\n    super(...arguments);\n    this.name = name;\n    this.description = description;\n    this.vectorStore = fields.vectorStore;\n    this.llm = fields.llm;\n    this.chain = VectorDBQAChain.fromLLM(this.llm, this.vectorStore);\n  }\n\n  /**\n   * Returns a string that describes what the tool does.\n   * @param name The name of the tool.\n   * @param description A description of what the tool does.\n   * @returns A string that describes what the tool does.\n   */\n  static getDescription(name: string, description: string): string {\n    return `Useful for when you need to answer questions about ${name}. Whenever you need information about ${description} you should ALWAYS use this. Input should be a fully formed question.`;\n  }\n\n  /** @ignore */\n  async _call(input: string) {\n    return this.chain.run(input);\n  }\n}\n\n// Path: langchain/src/agents/toolkits/vectorstore/prompt.ts\nexport const VECTOR_PREFIX = `You are an agent designed to answer questions about sets of documents.\nYou have access to tools for interacting with the documents, and the inputs to the tools are questions.\nSometimes, you will be asked to provide sources for your questions, in which case you should use the appropriate tool to do so.\nIf the question does not seem relevant to any of the tools provided, just return \"I don't know\" as the answer.`;\n\nexport const VECTOR_ROUTER_PREFIX = `You are an agent designed to answer questions.\nYou have access to tools for interacting with different sources, and the inputs to the tools are questions.\nYour main task is to decide which of the tools is relevant for answering question at hand.\nFor complex questions, you can break the question down into sub questions and use tools to answers the sub questions.`;\n\n// Path: langchain/src/agents/toolkits/vectorstore/vectorstore.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport { ToolInterface } from \"@langchain/core/tools\";\nimport { Toolkit } from \"@langchain/community/agents/toolkits/base\";\nimport { VectorStoreQATool } from \"../../../tools/vectorstore.js\";\nimport { ZeroShotCreatePromptArgs, ZeroShotAgent } from \"../../mrkl/index.js\";\nimport { VECTOR_PREFIX, VECTOR_ROUTER_PREFIX } from \"./prompt.js\";\nimport { SUFFIX } from \"../../mrkl/prompt.js\";\nimport { LLMChain } from \"../../../chains/llm_chain.js\";\nimport { AgentExecutor } from \"../../executor.js\";\n\n/**\n * Interface that defines the information about a vector store, including\n * the vector store itself, its name, and description.\n */\nexport interface VectorStoreInfo {\n  vectorStore: VectorStoreInterface;\n  name: string;\n  description: string;\n}\n\n/**\n * Class representing a toolkit for working with a single vector store. It\n * initializes the vector store QA tool based on the provided vector store\n * information and language model.\n * @example\n * ```typescript\n * const toolkit = new VectorStoreToolkit(\n *   {\n *     name: \"state_of_union_address\",\n *     description: \"the most recent state of the Union address\",\n *     vectorStore: new HNSWLib(),\n *   },\n *   new ChatOpenAI({ temperature: 0 }),\n * );\n * const result = await toolkit.invoke({\n *   input:\n *     \"What did biden say about Ketanji Brown Jackson in the state of the union address?\",\n * });\n * console.log(`Got output ${result.output}`);\n * ```\n */\nexport class VectorStoreToolkit extends Toolkit {\n  tools: ToolInterface[];\n\n  llm: BaseLanguageModelInterface;\n\n  constructor(\n    vectorStoreInfo: VectorStoreInfo,\n    llm: BaseLanguageModelInterface\n  ) {\n    super();\n    const description = VectorStoreQATool.getDescription(\n      vectorStoreInfo.name,\n      vectorStoreInfo.description\n    );\n    this.llm = llm;\n    this.tools = [\n      new VectorStoreQATool(vectorStoreInfo.name, description, {\n        vectorStore: vectorStoreInfo.vectorStore,\n        llm: this.llm,\n      }),\n    ];\n  }\n}\n\n/**\n * Class representing a toolkit for working with multiple vector stores.\n * It initializes multiple vector store QA tools based on the provided\n * vector store information and language model.\n */\nexport class VectorStoreRouterToolkit extends Toolkit {\n  tools: ToolInterface[];\n\n  vectorStoreInfos: VectorStoreInfo[];\n\n  llm: BaseLanguageModelInterface;\n\n  constructor(\n    vectorStoreInfos: VectorStoreInfo[],\n    llm: BaseLanguageModelInterface\n  ) {\n    super();\n    this.llm = llm;\n    this.vectorStoreInfos = vectorStoreInfos;\n    this.tools = vectorStoreInfos.map((vectorStoreInfo) => {\n      const description = VectorStoreQATool.getDescription(\n        vectorStoreInfo.name,\n        vectorStoreInfo.description\n      );\n      return new VectorStoreQATool(vectorStoreInfo.name, description, {\n        vectorStore: vectorStoreInfo.vectorStore,\n        llm: this.llm,\n      });\n    });\n  }\n}\n\n/** @deprecated Create a specific agent with a custom tool instead. */\nexport function createVectorStoreAgent(\n  llm: BaseLanguageModelInterface,\n  toolkit: VectorStoreToolkit,\n  args?: ZeroShotCreatePromptArgs\n) {\n  const {\n    prefix = VECTOR_PREFIX,\n    suffix = SUFFIX,\n    inputVariables = [\"input\", \"agent_scratchpad\"],\n  } = args ?? {};\n  const { tools } = toolkit;\n  const prompt = ZeroShotAgent.createPrompt(tools, {\n    prefix,\n    suffix,\n    inputVariables,\n  });\n  const chain = new LLMChain({ prompt, llm });\n  const agent = new ZeroShotAgent({\n    llmChain: chain,\n    allowedTools: tools.map((t) => t.name),\n  });\n  return AgentExecutor.fromAgentAndTools({\n    agent,\n    tools,\n    returnIntermediateSteps: true,\n  });\n}\n\n/** @deprecated Create a specific agent with a custom tool instead. */\nexport function createVectorStoreRouterAgent(\n  llm: BaseLanguageModelInterface,\n  toolkit: VectorStoreRouterToolkit,\n  args?: ZeroShotCreatePromptArgs\n) {\n  const {\n    prefix = VECTOR_ROUTER_PREFIX,\n    suffix = SUFFIX,\n    inputVariables = [\"input\", \"agent_scratchpad\"],\n  } = args ?? {};\n  const { tools } = toolkit;\n  const prompt = ZeroShotAgent.createPrompt(tools, {\n    prefix,\n    suffix,\n    inputVariables,\n  });\n  const chain = new LLMChain({ prompt, llm });\n  const agent = new ZeroShotAgent({\n    llmChain: chain,\n    allowedTools: tools.map((t) => t.name),\n  });\n  return AgentExecutor.fromAgentAndTools({\n    agent,\n    tools,\n    returnIntermediateSteps: true,\n  });\n}\n\n// Path: langchain/src/agents/openai_tools/index.ts\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport type {\n  BaseChatModel,\n  BaseChatModelCallOptions,\n} from \"@langchain/core/language_models/chat_models\";\nimport { ChatPromptTemplate } from \"@langchain/core/prompts\";\nimport { RunnablePassthrough } from \"@langchain/core/runnables\";\nimport { OpenAIClient } from \"@langchain/openai\";\nimport { convertToOpenAITool } from \"@langchain/core/utils/function_calling\";\nimport { formatToOpenAIToolMessages } from \"../format_scratchpad/openai_tools.js\";\nimport {\n  OpenAIToolsAgentOutputParser,\n  type ToolsAgentStep,\n} from \"./output_parser.js\";\nimport { AgentRunnableSequence } from \"../agent.js\";\n\nexport { OpenAIToolsAgentOutputParser, type ToolsAgentStep };\n\n/**\n * Params used by the createOpenAIToolsAgent function.\n */\nexport type CreateOpenAIToolsAgentParams = {\n  /**\n   * LLM to use as the agent. Should work with OpenAI tool calling,\n   * so must either be an OpenAI model that supports that or a wrapper of\n   * a different model that adds in equivalent support.\n   */\n  llm: BaseChatModel<\n    BaseChatModelCallOptions & {\n      tools?:\n        | StructuredToolInterface[]\n        | OpenAIClient.ChatCompletionTool[]\n        // eslint-disable-next-line @typescript-eslint/no-explicit-any\n        | any[];\n    }\n  >;\n  /** Tools this agent has access to. */\n  tools: StructuredToolInterface[];\n  /** The prompt to use, must have an input key of `agent_scratchpad`. */\n  prompt: ChatPromptTemplate;\n  /**\n   * Whether to invoke the underlying model in streaming mode,\n   * allowing streaming of intermediate steps. Defaults to true.\n   */\n  streamRunnable?: boolean;\n};\n\n/**\n * Create an agent that uses OpenAI-style tool calling.\n * @param params Params required to create the agent. Includes an LLM, tools, and prompt.\n * @returns A runnable sequence representing an agent. It takes as input all the same input\n *     variables as the prompt passed in does. It returns as output either an\n *     AgentAction or AgentFinish.\n *\n * @example\n * ```typescript\n * import { AgentExecutor, createOpenAIToolsAgent } from \"langchain/agents\";\n * import { pull } from \"langchain/hub\";\n * import type { ChatPromptTemplate } from \"@langchain/core/prompts\";\n * import { AIMessage, HumanMessage } from \"@langchain/core/messages\";\n *\n * import { ChatOpenAI } from \"@langchain/openai\";\n *\n * // Define the tools the agent will have access to.\n * const tools = [...];\n *\n * // Get the prompt to use - you can modify this!\n * // If you want to see the prompt in full, you can at:\n * // https://smith.langchain.com/hub/hwchase17/openai-tools-agent\n * const prompt = await pull<ChatPromptTemplate>(\n *   \"hwchase17/openai-tools-agent\"\n * );\n *\n * const llm = new ChatOpenAI({\n *   temperature: 0,\n *   modelName: \"gpt-3.5-turbo-1106\",\n * });\n *\n * const agent = await createOpenAIToolsAgent({\n *   llm,\n *   tools,\n *   prompt,\n * });\n *\n * const agentExecutor = new AgentExecutor({\n *   agent,\n *   tools,\n * });\n *\n * const result = await agentExecutor.invoke({\n *   input: \"what is LangChain?\",\n * });\n *\n * // With chat history\n * const result2 = await agentExecutor.invoke({\n *   input: \"what's my name?\",\n *   chat_history: [\n *     new HumanMessage(\"hi! my name is cob\"),\n *     new AIMessage(\"Hello Cob! How can I assist you today?\"),\n *   ],\n * });\n * ```\n */\nexport \nasync function createOpenAIToolsAgent({\n  llm,\n  tools,\n  prompt,\n  streamRunnable,\n}: CreateOpenAIToolsAgentParams) {\n  if (!prompt.inputVariables.includes(\"agent_scratchpad\")) {\n    throw new Error(\n      [\n        `Prompt must have an input variable named \"agent_scratchpad\".`,\n        `Found ${JSON.stringify(prompt.inputVariables)} instead.`,\n      ].join(\"\\n\")\n    );\n  }\n  const modelWithTools = llm.bind({ tools: tools.map(convertToOpenAITool) });\n  const agent = AgentRunnableSequence.fromRunnables(\n    [\n      RunnablePassthrough.assign({\n        agent_scratchpad: (input: { steps: ToolsAgentStep[] }) =>\n          formatToOpenAIToolMessages(input.steps),\n      }),\n      prompt,\n      modelWithTools,\n      new OpenAIToolsAgentOutputParser(),\n    ],\n    {\n      name: \"OpenAIToolsAgent\",\n      streamRunnable,\n      singleAction: false,\n    }\n  );\n  return agent;\n}\n\n// Path: langchain/src/agents/tests/create_structured_chat_agent.int.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { ChatOpenAI } from \"@langchain/openai\";\nimport type { ChatPromptTemplate } from \"@langchain/core/prompts\";\nimport { TavilySearchResults } from \"@langchain/community/tools/tavily_search\";\nimport { pull } from \"../../hub.js\";\nimport { AgentExecutor, createStructuredChatAgent } from \"../index.js\";\n\nconst tools = [new TavilySearchResults({ maxResults: 1 })];\n\ntest(\"createStructuredChatAgent works\", async () => {\n  const prompt = await pull<ChatPromptTemplate>(\n    \"hwchase17/structured-chat-agent\"\n  );\n  const llm = new ChatOpenAI({\n    modelName: \"gpt-3.5-turbo-1106\",\n    temperature: 0,\n  });\n  const agent = await createStructuredChatAgent({\n    llm,\n    tools,\n    prompt,\n  });\n  const agentExecutor = new AgentExecutor({\n    agent,\n    tools,\n  });\n  const input = \"what is LangChain?\";\n  const result = await agentExecutor.invoke({\n    input,\n  });\n\n  console.log(result);\n\n  expect(result.input).toBe(input);\n  expect(typeof result.output).toBe(\"string\");\n  // Length greater than 10 because any less than that would warrant\n  // an investigation into why such a short generation was returned.\n  expect(result.output.length).toBeGreaterThan(10);\n});\n\n// Path: langchain/src/agents/tests/chat_convo_output_parser.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { AgentAction, AgentFinish } from \"@langchain/core/agents\";\nimport { ChatConversationalAgentOutputParser } from \"../chat_convo/outputParser.js\";\n\ntest(\"Can parse JSON with text in front of it\", async () => {\n  const testCases = [\n    {\n      input: `Based on the information from the search, I can provide you with a query to get all the orders for the email \\`example@gmail.com\\`. Here's the query:\\n\\n\\`\\`\\`sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = 'example@gmail.com'\\n\\`\\`\\`\\n\\nPlease make any necessary modifications depending on your database schema and table structures. Run this query on your database to retrieve the orders made by the specified user.\\n\\n\\`\\`\\`json\\n{\\n  \"action\": \"Final Answer\",\\n  \"action_input\": \"To get all the orders for a user with the email \\`example@gmail.com\\`, you can use the following query:\\\\n\\\\n\\`\\`\\`\\\\nSELECT * FROM orders\\\\nJOIN users ON users.id = orders.user_id\\\\nWHERE users.email = 'example@gmail.com'\\\\n\\`\\`\\`\\\\n\\\\nPlease make any necessary modifications depending on your database schema and table structures. Run this query on your database to retrieve the orders made by the specified user.\"\\n}\\n\\`\\`\\``,\n      output: `{\\n  \"action\": \"Final Answer\",\\n  \"action_input\": \"To get all the orders for a user with the email \\`example@gmail.com\\`, you can use the following query:\\\\n\\\\n\\`\\`\\`\\\\nSELECT * FROM orders\\\\nJOIN users ON users.id = orders.user_id\\\\nWHERE users.email = 'example@gmail.com'\\\\n\\`\\`\\`\\\\n\\\\nPlease make any necessary modifications depending on your database schema and table structures. Run this query on your database to retrieve the  made by the specifsredroied user.\"\\n}`,\n      tool: \"Final Answer\",\n      toolInput: \"To get all the orders for a user with the email \",\n    },\n\n    {\n      input:\n        'Here is an example of a valid JSON object matching the provided spec:\\n\\n```json\\n{\\n  \"action\": \"metabase\",\\n  \"action_input\": [\"GET\", \"/api/table/1\"]\\n}\\n```\\n\\nIn this example, the \"action\" key has a string value of \"metabase\", and the \"action_input\" key has an array value containing two elements: a string value of \"GET\" and a string value of \"/api/table/1\". This JSON object could be used to make a request to a Metabase API endpoint with the specified method and arguments.',\n      output: `{ \"action\": \"metabase\", \"action_input\": [\"GET\", \"/api/table/1\"] } `,\n      tool: \"metabase\",\n      toolInput: [\"GET\", \"/api/table/1\"],\n    },\n    {\n      input:\n        '```\\n{\\n  \"action\": \"metabase\",\\n  \"action_input\": [\"GET\", \"/api/table/1\"]\\n}\\n```',\n      output: `{ \"action\": \"metabase\", \"action_input\": [\"GET\", \"/api/table/1\"] } `,\n      tool: \"metabase\",\n      toolInput: [\"GET\", \"/api/table/1\"],\n    },\n    {\n      input:\n        'Here we have some boilerplate nonsense```\\n{\\n \"action\": \"blogpost\",\\n  \"action_input\": \"```sql\\\\nSELECT * FROM orders\\\\nJOIN users ON users.id = orders.user_id\\\\nWHERE users.email = \\'bud\\'```\"\\n}\\n``` and at the end there is more nonsense',\n      output:\n        '{\"action\":\"blogpost\",\"action_input\":\"```sql\\\\nSELECT * FROM orders\\\\nJOIN users ON users.id = orders.user_id\\\\nWHERE users.email = \\'bud\\'```\"}',\n      tool: \"blogpost\",\n      toolInput:\n        \"```sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = 'bud'```\",\n    },\n    {\n      input:\n        'Here we have some boilerplate nonsense```json\\n{\\n \\t\\r\\n\"action\": \"blogpost\",\\n\\t\\r  \"action_input\": \"```sql\\\\nSELECT * FROM orders\\\\nJOIN users ON users.id = orders.user_id\\\\nWHERE users.email = \\'bud\\'```\"\\n\\t\\r}\\n\\n\\n\\t\\r``` and at the end there is more nonsense',\n      output:\n        '{\"action\":\"blogpost\",\"action_input\":\"```sql\\\\nSELECT * FROM orders\\\\nJOIN users ON users.id = orders.user_id\\\\nWHERE users.email = \\'bud\\'```\"}',\n      tool: \"blogpost\",\n      toolInput:\n        \"```sql\\nSELECT * FROM orders\\nJOIN users ON users.id = orders.user_id\\nWHERE users.email = 'bud'```\",\n    },\n    {\n      input:\n        '{\\n \\t\\r\\n\"action\":\"Final Answer\",\\n\\t\\r  \"action_input\":\"The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"\\n\\t\\r}',\n      output:\n        '{\"action\":\"Final Answer\",\"action_input\":\"The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"}',\n      tool: \"Final Answer\",\n      toolInput: 'The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```',\n    },\n    {\n      input:\n        '```json\\n{\\n \\t\\r\\n\"action\":\"Final Answer\",\\n\\t\\r  \"action_input\":\"The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"\\n\\t\\r}\\n\\n\\n\\t\\r```',\n      output:\n        '{\"action\":\"Final Answer\",\"action_input\":\"The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"}',\n      tool: \"Final Answer\",\n      toolInput: 'The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```',\n    },\n    {\n      input:\n        'Here we have some boilerplate nonsense```json\\n{\\n \\t\\r\\n\"action\":\"Final Answer\",\\n\\t\\r  \"action_input\":\"The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"\\n\\t\\r}\\n\\n\\n\\t\\r``` and at the end there is more nonsense',\n      output:\n        '{\"action\":\"Final Answer\",\"action_input\":\"The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"}',\n      tool: \"Final Answer\",\n      toolInput: 'The tool input ```json\\\\n{\\\\\"yes\\\\\":true}\\\\n```',\n    },\n    {\n      input:\n        'Here we have some boilerplate nonsense```\\n{\\n \\t\\r\\n\"action\":\"Final Answer\",\\n\\t\\r  \"action_input\":\"The tool input ```javascript\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"\\n\\t\\r}\\n\\n\\n\\t\\r``` and at the end there is more nonsense',\n      output:\n        '{\"action\":\"Final Answer\",\"action_input\":\"The tool input ```javascript\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"}',\n      tool: \"Final Answer\",\n      toolInput: 'The tool input ```javascript\\\\n{\\\\\"yes\\\\\":true}\\\\n```',\n    },\n    {\n      input:\n        '{\\n \\t\\r\\n\"action\":\"Final Answer\",\\n\\t\\r  \"action_input\":\"The tool input ```javascript\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"\\n\\t\\r}',\n      output:\n        '{\"action\":\"Final Answer\",\"action_input\":\"The tool input ```javascript\\\\n{\\\\\"yes\\\\\":true}\\\\n```\"}',\n      tool: \"Final Answer\",\n      toolInput: 'The tool input ```javascript\\\\n{\\\\\"yes\\\\\":true}\\\\n```',\n    },\n    {\n      input:\n        '{\\n \\t\\r\\n\"action\":\"Final Answer\",\\n\\t\\r  \"action_input\":\"this is a regular text response\"\\n\\t\\r}',\n      output:\n        '{\"action\":\"Final Answer\",\"action_input\":\"this is a regular text response\"}',\n      tool: \"Final Answer\",\n      toolInput: \"this is a regular text response\",\n    },\n  ];\n\n  const p = new ChatConversationalAgentOutputParser({\n    toolNames: [\"blogpost\", \"metabase\", \"ToolWithJson\"],\n  });\n  for (const message of testCases) {\n    const parsed = await p.parse(message.input);\n    expect(parsed).toBeDefined();\n    if (message.tool === \"Final Answer\") {\n      expect((parsed as AgentFinish).returnValues).toBeDefined();\n    } else {\n      expect((parsed as AgentAction).tool).toEqual(message.tool);\n\n      if (typeof message.toolInput === \"object\") {\n        expect(message.toolInput).toEqual((parsed as AgentAction).toolInput);\n      }\n      if (typeof message.toolInput === \"string\") {\n        expect(message.toolInput).toContain((parsed as AgentAction).toolInput);\n      }\n    }\n  }\n});\n\n// Path: langchain/src/agents/tests/sql.test.ts\n/* eslint-disable no-process-env */\nimport { test, expect, beforeEach, afterEach } from \"@jest/globals\";\nimport { DataSource } from \"typeorm\";\nimport {\n  InfoSqlTool,\n  QuerySqlTool,\n  ListTablesSqlTool,\n  QueryCheckerTool,\n} from \"../../tools/sql.js\";\nimport { SqlDatabase } from \"../../sql_db.js\";\n\nconst previousEnv = process.env;\n\nlet db: SqlDatabase;\n\nbeforeEach(async () => {\n  const datasource = new DataSource({\n    type: \"sqlite\",\n    database: \":memory:\",\n    synchronize: true,\n  });\n\n  await datasource.initialize();\n\n  await datasource.query(`\n        CREATE TABLE products (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, price INTEGER);\n    `);\n  await datasource.query(`\n        INSERT INTO products (name, price) VALUES ('Apple', 100);\n    `);\n  await datasource.query(`\n        INSERT INTO products (name, price) VALUES ('Banana', 200);\n    `);\n  await datasource.query(`\n        INSERT INTO products (name, price) VALUES ('Orange', 300);\n    `);\n  await datasource.query(`\n        CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, age INTEGER);\n    `);\n  await datasource.query(`\n        INSERT INTO users (name, age) VALUES ('Alice', 20);\n    `);\n  await datasource.query(`\n        INSERT INTO users (name, age) VALUES ('Bob', 21);\n    `);\n  await datasource.query(`\n        INSERT INTO users (name, age) VALUES ('Charlie', 22);\n    `);\n\n  db = await SqlDatabase.fromDataSourceParams({\n    appDataSource: datasource,\n  });\n\n  process.env = { ...previousEnv, OPENAI_API_KEY: \"test\" };\n});\n\nafterEach(async () => {\n  process.env = previousEnv;\n  await db.appDataSource.destroy();\n});\n\ntest.skip(\"QuerySqlTool\", async () => {\n  const querySqlTool = new QuerySqlTool(db);\n  const result = await querySqlTool.call(\"SELECT * FROM users\");\n  expect(result).toBe(\n    `[{\"id\":1,\"name\":\"Alice\",\"age\":20},{\"id\":2,\"name\":\"Bob\",\"age\":21},{\"id\":3,\"name\":\"Charlie\",\"age\":22}]`\n  );\n});\n\ntest.skip(\"QuerySqlTool with error\", async () => {\n  const querySqlTool = new QuerySqlTool(db);\n  const result = await querySqlTool.call(\"SELECT * FROM userss\");\n  expect(result).toBe(`QueryFailedError: SQLITE_ERROR: no such table: userss`);\n});\n\ntest.skip(\"InfoSqlTool\", async () => {\n  const infoSqlTool = new InfoSqlTool(db);\n  const result = await infoSqlTool.call(\"users, products\");\n  const expectStr = `\nCREATE TABLE products (\nid INTEGER , name TEXT , price INTEGER ) \nSELECT * FROM \"products\" LIMIT 3;\n id name price\n 1 Apple 100\n 2 Banana 200\n 3 Orange 300\nCREATE TABLE users (\nid INTEGER , name TEXT , age INTEGER ) \nSELECT * FROM \"users\" LIMIT 3;\n id name age\n 1 Alice 20\n 2 Bob 21\n 3 Charlie 22`;\n  expect(result.trim()).toBe(expectStr.trim());\n});\n\ntest.skip(\"InfoSqlTool with customDescription\", async () => {\n  db.customDescription = {\n    products: \"Custom Description for Products Table\",\n    users: \"Custom Description for Users Table\",\n    userss: \"Should not appear\",\n  };\n  const infoSqlTool = new InfoSqlTool(db);\n  const result = await infoSqlTool.call(\"users, products\");\n  const expectStr = `\nCustom Description for Products Table\nCREATE TABLE products (\nid INTEGER , name TEXT , price INTEGER ) \nSELECT * FROM \"products\" LIMIT 3;\n id name price\n 1 Apple 100\n 2 Banana 200\n 3 Orange 300\nCustom Description for Users Table\nCREATE TABLE users (\nid INTEGER , name TEXT , age INTEGER ) \nSELECT * FROM \"users\" LIMIT 3;\n id name age\n 1 Alice 20\n 2 Bob 21\n 3 Charlie 22`;\n  expect(result.trim()).toBe(expectStr.trim());\n});\n\ntest.skip(\"InfoSqlTool with error\", async () => {\n  const infoSqlTool = new InfoSqlTool(db);\n  const result = await infoSqlTool.call(\"userss, products\");\n  expect(result).toBe(\n    `Error: Wrong target table name: the table userss was not found in the database`\n  );\n});\n\ntest.skip(\"ListTablesSqlTool\", async () => {\n  const listSqlTool = new ListTablesSqlTool(db);\n  const result = await listSqlTool.call(\"\");\n  expect(result).toBe(`products, users`);\n});\n\ntest.skip(\"QueryCheckerTool\", async () => {\n  const queryCheckerTool = new QueryCheckerTool();\n  expect(queryCheckerTool.llmChain).not.toBeNull();\n  expect(queryCheckerTool.llmChain.inputKeys).toEqual([\"query\"]);\n});\n\ntest.skip(\"ListTablesSqlTool with include tables\", async () => {\n  const includesTables = [\"users\"];\n  db.includesTables = includesTables;\n  const listSqlTool = new ListTablesSqlTool(db);\n  const result = await listSqlTool.call(\"\");\n  expect(result).toBe(\"users\");\n});\n\ntest.skip(\"ListTablesSqlTool with ignore tables\", async () => {\n  const ignoreTables = [\"products\"];\n  db.ignoreTables = ignoreTables;\n  const listSqlTool = new ListTablesSqlTool(db);\n  const result = await listSqlTool.call(\"\");\n  expect(result).toBe(\"users\");\n});\n\n// Path: langchain/src/util/event-source-parse.ts\nexport * from \"@langchain/community/utils/event_source_parse\";\n\n// Path: langchain/src/util/axios-fetch-adapter.js\n/* eslint-disable no-plusplus */\n/* eslint-disable prefer-template */\n/* eslint-disable prefer-arrow-callback */\n/* eslint-disable no-var */\n/* eslint-disable vars-on-top */\n/* eslint-disable no-param-reassign */\n/* eslint-disable import/no-extraneous-dependencies */\n\n/**\n * This is copied from @vespaiach/axios-fetch-adapter, which exposes an ESM\n * module without setting the \"type\" field in package.json.\n */\n\nimport axios from \"axios\";\nimport {\n  EventStreamContentType,\n  getLines,\n  getBytes,\n  getMessages,\n} from \"./event-source-parse.js\";\n\nfunction tryJsonStringify(data) {\n  try {\n    return JSON.stringify(data);\n  } catch (e) {\n    return data;\n  }\n}\n\n/**\n * In order to avoid import issues with axios 1.x, copying here the internal\n * utility functions that we used to import directly from axios.\n */\n\n// Copied from axios/lib/core/settle.js\nfunction settle(resolve, reject, response) {\n  const { validateStatus } = response.config;\n  if (!response.status || !validateStatus || validateStatus(response.status)) {\n    resolve(response);\n  } else {\n    reject(\n      createError(\n        `Request failed with status code ${response.status} and body ${\n          typeof response.data === \"string\"\n            ? response.data\n            : tryJsonStringify(response.data)\n        }`,\n        response.config,\n        null,\n        response.request,\n        response\n      )\n    );\n  }\n}\n\n// Copied from axios/lib/helpers/isAbsoluteURL.js\nfunction isAbsoluteURL(url) {\n  // A URL is considered absolute if it begins with \"<scheme>://\" or \"//\" (protocol-relative URL).\n  // RFC 3986 defines scheme name as a sequence of characters beginning with a letter and followed\n  // by any combination of letters, digits, plus, period, or hyphen.\n  return /^([a-z][a-z\\d+\\-.]*:)?\\/\\//i.test(url);\n}\n\n// Copied from axios/lib/helpers/combineURLs.js\nfunction combineURLs(baseURL, relativeURL) {\n  return relativeURL\n    ? baseURL.replace(/\\/+$/, \"\") + \"/\" + relativeURL.replace(/^\\/+/, \"\")\n    : baseURL;\n}\n\n// Copied from axios/lib/helpers/buildURL.js\nfunction encode(val) {\n  return encodeURIComponent(val)\n    .replace(/%3A/gi, \":\")\n    .replace(/%24/g, \"$\")\n    .replace(/%2C/gi, \",\")\n    .replace(/%20/g, \"+\")\n    .replace(/%5B/gi, \"[\")\n    .replace(/%5D/gi, \"]\");\n}\n\nfunction buildURL(url, params, paramsSerializer) {\n  if (!params) {\n    return url;\n  }\n\n  var serializedParams;\n  if (paramsSerializer) {\n    serializedParams = paramsSerializer(params);\n  } else if (isURLSearchParams(params)) {\n    serializedParams = params.toString();\n  } else {\n    var parts = [];\n\n    forEach(params, function serialize(val, key) {\n      if (val === null || typeof val === \"undefined\") {\n        return;\n      }\n\n      if (isArray(val)) {\n        key = `${key}[]`;\n      } else {\n        val = [val];\n      }\n\n      forEach(val, function parseValue(v) {\n        if (isDate(v)) {\n          v = v.toISOString();\n        } else if (isObject(v)) {\n          v = JSON.stringify(v);\n        }\n        parts.push(`${encode(key)}=${encode(v)}`);\n      });\n    });\n\n    serializedParams = parts.join(\"&\");\n  }\n\n  if (serializedParams) {\n    var hashmarkIndex = url.indexOf(\"#\");\n    if (hashmarkIndex !== -1) {\n      url = url.slice(0, hashmarkIndex);\n    }\n\n    url += (url.indexOf(\"?\") === -1 ? \"?\" : \"&\") + serializedParams;\n  }\n\n  return url;\n}\n\n// Copied from axios/lib/core/buildFullPath.js\nfunction buildFullPath(baseURL, requestedURL) {\n  if (baseURL && !isAbsoluteURL(requestedURL)) {\n    return combineURLs(baseURL, requestedURL);\n  }\n  return requestedURL;\n}\n\n// Copied from axios/lib/utils.js\nfunction isUndefined(val) {\n  return typeof val === \"undefined\";\n}\n\nfunction isObject(val) {\n  return val !== null && typeof val === \"object\";\n}\n\nfunction isDate(val) {\n  return toString.call(val) === \"[object Date]\";\n}\n\nfunction isURLSearchParams(val) {\n  return toString.call(val) === \"[object URLSearchParams]\";\n}\n\nfunction isArray(val) {\n  return Array.isArray(val);\n}\n\nfunction forEach(obj, fn) {\n  // Don't bother if no value provided\n  if (obj === null || typeof obj === \"undefined\") {\n    return;\n  }\n\n  // Force an array if not already something iterable\n  if (typeof obj !== \"object\") {\n    obj = [obj];\n  }\n\n  if (isArray(obj)) {\n    // Iterate over array values\n    for (var i = 0, l = obj.length; i < l; i++) {\n      fn.call(null, obj[i], i, obj);\n    }\n  } else {\n    // Iterate over object keys\n    for (var key in obj) {\n      if (Object.prototype.hasOwnProperty.call(obj, key)) {\n        fn.call(null, obj[key], key, obj);\n      }\n    }\n  }\n}\n\nfunction isFormData(val) {\n  return toString.call(val) === \"[object FormData]\";\n}\n\n// TODO this needs to be fixed to run in newer browser-like environments\n// https://github.com/vespaiach/axios-fetch-adapter/issues/20#issue-1396365322\nfunction isStandardBrowserEnv() {\n  if (\n    typeof navigator !== \"undefined\" &&\n    // eslint-disable-next-line no-undef\n    (navigator.product === \"ReactNative\" ||\n      // eslint-disable-next-line no-undef\n      navigator.product === \"NativeScript\" ||\n      // eslint-disable-next-line no-undef\n      navigator.product === \"NS\")\n  ) {\n    return false;\n  }\n  return typeof window !== \"undefined\" && typeof document !== \"undefined\";\n}\n\n/**\n * - Create a request object\n * - Get response body\n * - Check if timeout\n */\nexport default async function fetchAdapter(config) {\n  const request = createRequest(config);\n  const data = await getResponse(request, config);\n\n  return new Promise((resolve, reject) => {\n    if (data instanceof Error) {\n      reject(data);\n    } else {\n      // eslint-disable-next-line no-unused-expressions\n      Object.prototype.toString.call(config.settle) === \"[object Function]\"\n        ? config.settle(resolve, reject, data)\n        : settle(resolve, reject, data);\n    }\n  });\n}\n\n/**\n * Fetch API stage two is to get response body. This funtion tries to retrieve\n * response body based on response's type\n */\nasync function getResponse(request, config) {\n  let stageOne;\n  try {\n    stageOne = await fetch(request);\n  } catch (e) {\n    if (e && e.name === \"AbortError\") {\n      return createError(\"Request aborted\", config, \"ECONNABORTED\", request);\n    }\n    if (e && e.name === \"TimeoutError\") {\n      return createError(\"Request timeout\", config, \"ECONNABORTED\", request);\n    }\n    return createError(\"Network Error\", config, \"ERR_NETWORK\", request);\n  }\n\n  const headers = {};\n  stageOne.headers.forEach((value, key) => {\n    headers[key] = value;\n  });\n\n  const response = {\n    ok: stageOne.ok,\n    status: stageOne.status,\n    statusText: stageOne.statusText,\n    headers,\n    config,\n    request,\n  };\n\n  if (stageOne.status >= 200 && stageOne.status !== 204) {\n    if (config.responseType === \"stream\") {\n      const contentType = stageOne.headers.get(\"content-type\");\n      if (!contentType?.startsWith(EventStreamContentType)) {\n        // If the content-type is not stream, response is most likely an error\n        if (stageOne.status >= 400) {\n          // If the error is a JSON, parse it. Otherwise, return as text\n          if (contentType?.startsWith(\"application/json\")) {\n            response.data = await stageOne.json();\n            return response;\n          } else {\n            response.data = await stageOne.text();\n            return response;\n          }\n        }\n        // If the non-stream response is also not an error, throw\n        throw new Error(\n          `Expected content-type to be ${EventStreamContentType}, Actual: ${contentType}`\n        );\n      }\n      await getBytes(stageOne.body, getLines(getMessages(config.onmessage)));\n    } else {\n      switch (config.responseType) {\n        case \"arraybuffer\":\n          response.data = await stageOne.arrayBuffer();\n          break;\n        case \"blob\":\n          response.data = await stageOne.blob();\n          break;\n        case \"json\":\n          response.data = await stageOne.json();\n          break;\n        case \"formData\":\n          response.data = await stageOne.formData();\n          break;\n        default:\n          response.data = await stageOne.text();\n          break;\n      }\n    }\n  }\n\n  return response;\n}\n\n/**\n * This function will create a Request object based on configuration's axios\n */\nfunction createRequest(config) {\n  const headers = new Headers(config.headers);\n\n  // HTTP basic authentication\n  if (config.auth) {\n    const username = config.auth.username || \"\";\n    const password = config.auth.password\n      ? decodeURI(encodeURIComponent(config.auth.password))\n      : \"\";\n    headers.set(\"Authorization\", `Basic ${btoa(`${username}:${password}`)}`);\n  }\n\n  const method = config.method.toUpperCase();\n  const options = {\n    headers,\n    method,\n  };\n  if (method !== \"GET\" && method !== \"HEAD\") {\n    options.body = config.data;\n\n    // In these cases the browser will automatically set the correct Content-Type,\n    // but only if that header hasn't been set yet. So that's why we're deleting it.\n    if (isFormData(options.body) && isStandardBrowserEnv()) {\n      headers.delete(\"Content-Type\");\n    }\n  }\n  // Some `fetch` implementations will override the Content-Type to text/plain\n  // when body is a string.\n  // See https://github.com/langchain-ai/langchainjs/issues/1010\n  if (typeof options.body === \"string\") {\n    options.body = new TextEncoder().encode(options.body);\n  }\n  if (config.mode) {\n    options.mode = config.mode;\n  }\n  if (config.cache) {\n    options.cache = config.cache;\n  }\n  if (config.integrity) {\n    options.integrity = config.integrity;\n  }\n  if (config.redirect) {\n    options.redirect = config.redirect;\n  }\n  if (config.referrer) {\n    options.referrer = config.referrer;\n  }\n  if (config.timeout && config.timeout > 0) {\n    options.signal = AbortSignal.timeout(config.timeout);\n  }\n  if (config.signal) {\n    // this overrides the timeout signal if both are set\n    options.signal = config.signal;\n  }\n  // This config is similar to XHR\u2019s withCredentials flag, but with three available values instead of two.\n  // So if withCredentials is not set, default value 'same-origin' will be used\n  if (!isUndefined(config.withCredentials)) {\n    options.credentials = config.withCredentials ? \"include\" : \"omit\";\n  }\n  // for streaming\n  if (config.responseType === \"stream\") {\n    options.headers.set(\"Accept\", EventStreamContentType);\n  }\n\n  const fullPath = buildFullPath(config.baseURL, config.url);\n  const url = buildURL(fullPath, config.params, config.paramsSerializer);\n\n  // Expected browser to throw error if there is any wrong configuration value\n  return new Request(url, options);\n}\n\n/**\n * Note:\n *\n *   From version >= 0.27.0, createError function is replaced by AxiosError class.\n *   So I copy the old createError function here for backward compatible.\n *\n *\n *\n * Create an Error with the specified message, config, error code, request and response.\n *\n * @param {string} message The error message.\n * @param {Object} config The config.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n * @returns {Error} The created error.\n */\nfunction createError(message, config, code, request, response) {\n  if (axios.AxiosError && typeof axios.AxiosError === \"function\") {\n    return new axios.AxiosError(\n      message,\n      axios.AxiosError[code],\n      config,\n      request,\n      response\n    );\n  }\n\n  const error = new Error(message);\n  return enhanceError(error, config, code, request, response);\n}\n\n/**\n *\n * Note:\n *\n *   This function is for backward compatible.\n *\n *\n * Update an Error with the specified config, error code, and response.\n *\n * @param {Error} error The error to update.\n * @param {Object} config The config.\n * @param {string} [code] The error code (for example, 'ECONNABORTED').\n * @param {Object} [request] The request.\n * @param {Object} [response] The response.\n * @returns {Error} The error.\n */\nfunction enhanceError(error, config, code, request, response) {\n  error.config = config;\n  if (code) {\n    error.code = code;\n  }\n\n  error.request = request;\n  error.response = response;\n  error.isAxiosError = true;\n\n  error.toJSON = function toJSON() {\n    return {\n      // Standard\n      message: this.message,\n      name: this.name,\n      // Microsoft\n      description: this.description,\n      number: this.number,\n      // Mozilla\n      fileName: this.fileName,\n      lineNumber: this.lineNumber,\n      columnNumber: this.columnNumber,\n      stack: this.stack,\n      // Axios\n      config: this.config,\n      code: this.code,\n      status:\n        this.response && this.response.status ? this.response.status : null,\n    };\n  };\n  return error;\n}\n\n// Path: langchain/src/tools/webbrowser.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { Document } from \"@langchain/core/documents\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport axiosMod, { AxiosRequestConfig, AxiosStatic } from \"axios\";\nimport * as cheerio from \"cheerio\";\nimport {\n  CallbackManager,\n  CallbackManagerForToolRun,\n} from \"@langchain/core/callbacks/manager\";\nimport { isNode } from \"@langchain/core/utils/env\";\nimport { Tool, ToolParams } from \"@langchain/core/tools\";\nimport { RunnableSequence } from \"@langchain/core/runnables\";\nimport { StringOutputParser } from \"@langchain/core/output_parsers\";\nimport {\n  RecursiveCharacterTextSplitter,\n  TextSplitter,\n} from \"../text_splitter.js\";\nimport { MemoryVectorStore } from \"../vectorstores/memory.js\";\nimport fetchAdapter from \"../util/axios-fetch-adapter.js\";\nimport { formatDocumentsAsString } from \"../util/document.js\";\n\nexport const parseInputs = (inputs: string): [string, string] => {\n  const [baseUrl, task] = inputs.split(\",\").map((input) => {\n    let t = input.trim();\n    t = t.startsWith('\"') ? t.slice(1) : t;\n    t = t.endsWith('\"') ? t.slice(0, -1) : t;\n    // it likes to put / at the end of urls, wont matter for task\n    t = t.endsWith(\"/\") ? t.slice(0, -1) : t;\n    return t.trim();\n  });\n\n  return [baseUrl, task];\n};\n\nexport const getText = (\n  html: string,\n  baseUrl: string,\n  summary: boolean\n): string => {\n  // scriptingEnabled so noscript elements are parsed\n  const $ = cheerio.load(html, { scriptingEnabled: true });\n\n  let text = \"\";\n\n  // lets only get the body if its a summary, dont need to summarize header or footer etc\n  const rootElement = summary ? \"body \" : \"*\";\n\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  $(`${rootElement}:not(style):not(script):not(svg)`).each((_i, elem: any) => {\n    // we dont want duplicated content as we drill down so remove children\n    let content = $(elem).clone().children().remove().end().text().trim();\n    const $el = $(elem);\n\n    // if its an ahref, print the content and url\n    let href = $el.attr(\"href\");\n    if ($el.prop(\"tagName\")?.toLowerCase() === \"a\" && href) {\n      if (!href.startsWith(\"http\")) {\n        try {\n          href = new URL(href, baseUrl).toString();\n        } catch {\n          // if this fails thats fine, just no url for this\n          href = \"\";\n        }\n      }\n\n      const imgAlt = $el.find(\"img[alt]\").attr(\"alt\")?.trim();\n      if (imgAlt) {\n        content += ` ${imgAlt}`;\n      }\n\n      text += ` [${content}](${href})`;\n    }\n    // otherwise just print the content\n    else if (content !== \"\") {\n      text += ` ${content}`;\n    }\n  });\n\n  return text.trim().replace(/\\n+/g, \" \");\n};\n\nconst getHtml = async (\n  baseUrl: string,\n  h: Headers,\n  config: AxiosRequestConfig\n) => {\n  const axios = (\n    \"default\" in axiosMod ? axiosMod.default : axiosMod\n  ) as AxiosStatic;\n\n  const domain = new URL(baseUrl).hostname;\n\n  const headers = { ...h };\n  // these appear to be positional, which means they have to exist in the headers passed in\n  headers.Host = domain;\n  headers[\"Alt-Used\"] = domain;\n\n  let htmlResponse;\n  try {\n    htmlResponse = await axios.get(baseUrl, {\n      ...config,\n      headers,\n    });\n  } catch (e) {\n    if (axios.isAxiosError(e) && e.response && e.response.status) {\n      throw new Error(`http response ${e.response.status}`);\n    }\n    throw e;\n  }\n\n  const allowedContentTypes = [\n    \"text/html\",\n    \"application/json\",\n    \"application/xml\",\n    \"application/javascript\",\n    \"text/plain\",\n  ];\n\n  const contentType = htmlResponse.headers[\"content-type\"];\n  const contentTypeArray = contentType.split(\";\");\n  if (\n    contentTypeArray[0] &&\n    !allowedContentTypes.includes(contentTypeArray[0])\n  ) {\n    throw new Error(\"returned page was not utf8\");\n  }\n  return htmlResponse.data;\n};\n\nconst DEFAULT_HEADERS = {\n  Accept:\n    \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8\",\n  \"Accept-Encoding\": \"gzip, deflate\",\n  \"Accept-Language\": \"en-US,en;q=0.5\",\n  \"Alt-Used\": \"LEAVE-THIS-KEY-SET-BY-TOOL\",\n  Connection: \"keep-alive\",\n  Host: \"LEAVE-THIS-KEY-SET-BY-TOOL\",\n  Referer: \"https://www.google.com/\",\n  \"Sec-Fetch-Dest\": \"document\",\n  \"Sec-Fetch-Mode\": \"navigate\",\n  \"Sec-Fetch-Site\": \"cross-site\",\n  \"Upgrade-Insecure-Requests\": \"1\",\n  \"User-Agent\":\n    \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/111.0\",\n};\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype Headers = Record<string, any>;\n\n/**\n * Defines the arguments that can be passed to the WebBrowser constructor.\n * It extends the ToolParams interface and includes properties for a\n * language model, embeddings, HTTP headers, an Axios configuration, a\n * callback manager, and a text splitter.\n */\nexport interface WebBrowserArgs extends ToolParams {\n  model: BaseLanguageModelInterface;\n\n  embeddings: EmbeddingsInterface;\n\n  headers?: Headers;\n\n  axiosConfig?: Omit<AxiosRequestConfig, \"url\">;\n\n  /** @deprecated */\n  callbackManager?: CallbackManager;\n\n  textSplitter?: TextSplitter;\n}\n\n/**\n * A class designed to interact with web pages, either to extract\n * information from them or to summarize their content. It uses the axios\n * library to send HTTP requests and the cheerio library to parse the\n * returned HTML.\n * @example\n * ```typescript\n * const browser = new WebBrowser({\n *   model: new ChatOpenAI({ temperature: 0 }),\n *   embeddings: new OpenAIEmbeddings({}),\n * });\n * const result = await browser.invoke(\"https:exampleurl.com\");\n * ```\n */\nexport class WebBrowser extends Tool {\n  static lc_name() {\n    return \"WebBrowser\";\n  }\n\n  get lc_namespace() {\n    return [...super.lc_namespace, \"webbrowser\"];\n  }\n\n  private model: BaseLanguageModelInterface;\n\n  private embeddings: EmbeddingsInterface;\n\n  private headers: Headers;\n\n  private axiosConfig: Omit<AxiosRequestConfig, \"url\">;\n\n  private textSplitter: TextSplitter;\n\n  constructor({\n    model,\n    headers,\n    embeddings,\n    axiosConfig,\n    textSplitter,\n  }: WebBrowserArgs) {\n    super(...arguments);\n\n    this.model = model;\n    this.embeddings = embeddings;\n    this.headers = headers ?? DEFAULT_HEADERS;\n    this.axiosConfig = {\n      withCredentials: true,\n      adapter: isNode() ? undefined : fetchAdapter,\n      ...axiosConfig,\n    };\n    this.textSplitter =\n      textSplitter ??\n      new RecursiveCharacterTextSplitter({\n        chunkSize: 2000,\n        chunkOverlap: 200,\n      });\n  }\n\n  /** @ignore */\n  async _call(inputs: string, runManager?: CallbackManagerForToolRun) {\n    const [baseUrl, task] = parseInputs(inputs);\n    const doSummary = !task;\n\n    let text;\n    try {\n      const html = await getHtml(baseUrl, this.headers, this.axiosConfig);\n      text = getText(html, baseUrl, doSummary);\n    } catch (e) {\n      if (e) {\n        return e.toString();\n      }\n      return \"There was a problem connecting to the site\";\n    }\n\n    const texts = await this.textSplitter.splitText(text);\n\n    let context;\n    // if we want a summary grab first 4\n    if (doSummary) {\n      context = texts.slice(0, 4).join(\"\\n\");\n    }\n    // search term well embed and grab top 4\n    else {\n      const docs = texts.map(\n        (pageContent) =>\n          new Document({\n            pageContent,\n            metadata: [],\n          })\n      );\n\n      const vectorStore = await MemoryVectorStore.fromDocuments(\n        docs,\n        this.embeddings\n      );\n      const results = await vectorStore.similaritySearch(\n        task,\n        4,\n        undefined,\n        runManager?.getChild(\"vectorstore\")\n      );\n      context = formatDocumentsAsString(results);\n    }\n\n    const input = `Text:${context}\\n\\nI need ${\n      doSummary ? \"a summary\" : task\n    } from the above text, also provide up to 5 markdown links from within that would be of interest (always including URL and text). Links should be provided, if present, in markdown syntax as a list under the heading \"Relevant Links:\".`;\n\n    const chain = RunnableSequence.from([this.model, new StringOutputParser()]);\n    return chain.invoke(input, runManager?.getChild());\n  }\n\n  name = \"web-browser\";\n\n  description = `useful for when you need to find something on or summarize a webpage. input should be a comma separated list of \"ONE valid http URL including protocol\",\"what you want to find on the page or empty string for a summary\".`;\n}\n\n// Path: langchain/src/agents/tests/agent.int.test.ts\n/* eslint-disable no-process-env */\nimport { expect, test } from \"@jest/globals\";\nimport { OpenAI, OpenAIEmbeddings, ChatOpenAI } from \"@langchain/openai\";\nimport { Tool } from \"@langchain/core/tools\";\nimport { RunnableSequence } from \"@langchain/core/runnables\";\nimport { OutputParserException } from \"@langchain/core/output_parsers\";\nimport { AIMessage } from \"@langchain/core/messages\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { SerpAPI } from \"@langchain/community/tools/serpapi\";\nimport { ChatMessageHistory } from \"@langchain/community/stores/message/in_memory\";\nimport { AgentExecutor, ZeroShotAgent } from \"../index.js\";\nimport { Calculator } from \"../../tools/calculator.js\";\nimport { initializeAgentExecutorWithOptions } from \"../initialize.js\";\nimport { WebBrowser } from \"../../tools/webbrowser.js\";\nimport { BufferMemory } from \"../../memory/buffer_memory.js\";\n\ntest(\"Pass runnable to agent executor\", async () => {\n  const model = new ChatOpenAI({ temperature: 0, modelName: \"gpt-3.5-turbo\" });\n  const tools: Tool[] = [\n    new SerpAPI(undefined, {\n      location: \"Austin,Texas,United States\",\n      hl: \"en\",\n      gl: \"us\",\n    }),\n    new Calculator(),\n  ];\n\n  const prompt = ZeroShotAgent.createPrompt(tools);\n  const outputParser = ZeroShotAgent.getDefaultOutputParser();\n\n  const runnable = RunnableSequence.from([\n    {\n      input: (i: { input: string }) => i.input,\n      agent_scratchpad: (i: { input: string }) => i.input,\n    },\n    prompt,\n    model,\n    outputParser,\n  ]);\n\n  const executor = AgentExecutor.fromAgentAndTools({\n    agent: runnable,\n    tools,\n  });\n  const res = await executor.invoke({\n    input:\n      \"Who is Olivia Wilde's boyfriend? What is his current age raised to the 0.23 power?\",\n  });\n  console.log(\n    {\n      res,\n    },\n    \"Pass runnable to agent executor\"\n  );\n  expect(res.output).not.toEqual(\"\");\n  expect(res.output).not.toEqual(\"Agent stopped due to max iterations.\");\n});\n\ntest(\"Custom output parser\", async () => {\n  const model = new ChatOpenAI({ temperature: 0, modelName: \"gpt-3.5-turbo\" });\n  const tools: Tool[] = [\n    new SerpAPI(undefined, {\n      location: \"Austin,Texas,United States\",\n      hl: \"en\",\n      gl: \"us\",\n    }),\n    new Calculator(),\n  ];\n\n  const parser = (output: AIMessage) => {\n    const text = output.content;\n    if (typeof text !== \"string\") {\n      throw new Error(\"Cannot parse non-string output.\");\n    }\n    if (text.includes(\"Final Answer:\")) {\n      return {\n        returnValues: {\n          output: \"We did it!\",\n        },\n        log: text,\n      };\n    }\n\n    const match = /Action:([\\s\\S]*?)(?:\\nAction Input:([\\s\\S]*?))?$/.exec(text);\n    if (!match) {\n      throw new OutputParserException(`Could not parse LLM output: ${text}`);\n    }\n\n    return {\n      tool: match[1].trim(),\n      toolInput: match[2]\n        ? match[2].trim().replace(/^(\"+)(.*?)(\\1)$/, \"$2\")\n        : \"\",\n      log: text,\n    };\n  };\n\n...\n", "needle_token_start": 3818, "needle_token_end": 4095, "code_context_ntokens": 16038, "output": ["```typescript\n/**\n * Create an agent that uses OpenAI-style tool calling.\n * @param params Params required to create the agent. Includes an LLM, tools, and prompt.\n * @returns A runnable sequence representing an agent. It takes as input all the same input\n *     variables as the prompt passed in does. It returns as output either an\n *     AgentAction or AgentFinish.\n *\n * @example\n * ```typescript\n * import { AgentExecutor, createOpenAIToolsAgent } from \"langchain/agents\";\n * import { pull } from \"../../hub.js\";\n * import type { ChatPromptTemplate } from \"@langchain/core/prompts\";\n * import { AIMessage, HumanMessage } from \"@langchain/core/messages\";\n *\n * import { ChatOpenAI } from \"@langchain/openai\";\n *\n * // Define the tools the agent will have access to.\n * const tools = [...];\n *\n * // Get the prompt to use - you can modify this!\n * // If you want to see the prompt in full, you can at:\n * // https://smith.langchain.com/hub/hwchase17/openai-tools-agent\n * const prompt = await pull<ChatPromptTemplate>(\n *   \"hwchase17/openai-tools-agent\"\n * );\n *\n * const llm = new ChatOpenAI({\n *   temperature: 0,\n *   modelName: \"gpt-3.5-turbo-1106\",\n * });\n *\n * const agent = await createOpenAIToolsAgent({\n *   llm,\n *   tools,\n *   prompt,\n * });\n *\n * const agentExecutor = new AgentExecutor({\n *   agent,\n *   tools,\n * });\n *\n * const result = await agentExecutor.invoke({\n *   input: \"what is LangChain?\",\n * });\n *\n * // With chat history\n * const result2 = await agentExecutor.invoke({\n *   input: \"what's my name?\",\n *   chat_history: [\n *     new HumanMessage(\"hi! my name is cob\"),\n *     new AIMessage(\"Hello Cob! How can I assist you today?\"),\n *   ],\n * });\n * ```\n */\nexport async function createOpenAIToolsAgent({\n  llm,\n  tools,\n  prompt,\n  streamRunnable,\n}: CreateOpenAIToolsAgentParams) {\n  if (!prompt.inputVariables.includes(\"agent_scratchpad\")) {\n    throw new Error(\n      [\n        `Prompt must have an input variable named \"agent_scratchpad\".`,\n        `Found ${JSON.stringify(prompt.inputVariables)} instead.`,\n      ].join(\"\\n\")\n    );\n  }\n  const modelWithTools = llm.bind({ tools: tools.map(convertToOpenAITool) });\n  const agent = AgentRunnableSequence.fromRunnables(\n    [\n      RunnablePassthrough.assign({\n        agent_scratchpad: (input: { steps: ToolsAgentStep[] }) =>\n          formatToOpenAIToolMessages(input.steps),\n      }),\n      prompt,\n      modelWithTools,\n      new OpenAIToolsAgentOutputParser(),\n    ],\n    {\n      name: \"OpenAIToolsAgent\",\n      streamRunnable,\n      singleAction: false,\n    }\n  );\n  return agent;\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "processValue", "language": "typescript", "path": "langchain/src/retrievers/self_query/vectara.ts", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To convert a numerical or textual value into a string format suitable for use in queries, ensuring textual values are appropriately quoted.\n2. **Input**: A value which can be either a number or a string.\n3. **Output**: A string representation of the input value, with strings enclosed in single quotes.\n4. **Procedure**: Checks the type of the input value. If it is a string, it encloses the value in single quotes and returns it. If it is a number, it converts the number to a string without adding any quotes and returns it.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/chains/query_constructor/index.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { z } from \"zod\";\nimport {\n  Example,\n  interpolateFString,\n  FewShotPromptTemplate,\n} from \"@langchain/core/prompts\";\nimport { InputValues } from \"@langchain/core/utils/types\";\nimport { QueryTransformer, TraverseType } from \"./parser.js\";\nimport {\n  Comparator,\n  Comparators,\n  Operator,\n  Operators,\n  StructuredQuery,\n} from \"./ir.js\";\nimport {\n  DEFAULT_EXAMPLES,\n  DEFAULT_PREFIX,\n  DEFAULT_SCHEMA,\n  DEFAULT_SUFFIX,\n  EXAMPLE_PROMPT,\n} from \"./prompt.js\";\nimport { LLMChain } from \"../llm_chain.js\";\nimport { AsymmetricStructuredOutputParser } from \"../../output_parsers/structured.js\";\nimport { AttributeInfo } from \"../../schema/query_constructor.js\";\n\nexport { QueryTransformer, type TraverseType };\nexport {\n  DEFAULT_EXAMPLES,\n  DEFAULT_PREFIX,\n  DEFAULT_SCHEMA,\n  DEFAULT_SUFFIX,\n  EXAMPLE_PROMPT,\n};\n\nconst queryInputSchema = /* #__PURE__ */ z.object({\n  query: /* #__PURE__ */ z\n    .string()\n    .describe(\"text string to compare to document contents\"),\n  filter: /* #__PURE__ */ z\n    .string()\n    .optional()\n    .describe(\"logical condition statement for filtering documents\"),\n});\n\n/**\n * A class that extends AsymmetricStructuredOutputParser to parse\n * structured query output.\n */\nexport class StructuredQueryOutputParser extends AsymmetricStructuredOutputParser<\n  typeof queryInputSchema,\n  StructuredQuery\n> {\n  lc_namespace = [\"langchain\", \"chains\", \"query_constructor\"];\n\n  queryTransformer: QueryTransformer;\n\n  constructor(fields: {\n    allowedComparators: Comparator[];\n    allowedOperators: Operator[];\n  }) {\n    super({ ...fields, inputSchema: queryInputSchema });\n\n    const { allowedComparators, allowedOperators } = fields;\n    this.queryTransformer = new QueryTransformer(\n      allowedComparators,\n      allowedOperators\n    );\n  }\n\n  /**\n   * Processes the output of a structured query.\n   * @param query The query string.\n   * @param filter The filter condition.\n   * @returns A Promise that resolves to a StructuredQuery instance.\n   */\n  async outputProcessor({\n    query,\n    filter,\n  }: z.infer<typeof queryInputSchema>): Promise<StructuredQuery> {\n    let myQuery = query;\n    if (myQuery.length === 0) {\n      myQuery = \" \";\n    }\n    if (filter === \"NO_FILTER\" || filter === undefined) {\n      return new StructuredQuery(query);\n    } else {\n      const parsedFilter = await this.queryTransformer.parse(filter);\n      return new StructuredQuery(query, parsedFilter);\n    }\n  }\n\n  /**\n   * Creates a new StructuredQueryOutputParser instance from the provided\n   * components.\n   * @param allowedComparators An array of allowed Comparator instances.\n   * @param allowedOperators An array of allowed Operator instances.\n   * @returns A new StructuredQueryOutputParser instance.\n   */\n  static fromComponents(\n    allowedComparators: Comparator[] = [],\n    allowedOperators: Operator[] = []\n  ) {\n    return new StructuredQueryOutputParser({\n      allowedComparators,\n      allowedOperators,\n    });\n  }\n}\n\nexport function formatAttributeInfo(info: AttributeInfo[]) {\n  const infoObj = info.reduce((acc, attr) => {\n    acc[attr.name] = {\n      type: attr.type,\n      description: attr.description,\n    };\n    return acc;\n  }, {} as { [name: string]: { type: string; description: string } });\n\n  return JSON.stringify(infoObj, null, 2)\n    .replaceAll(\"{\", \"{{\")\n    .replaceAll(\"}\", \"}}\");\n}\n\nconst defaultExample = DEFAULT_EXAMPLES.map((EXAMPLE) => EXAMPLE as Example);\n\nfunction _getPrompt(\n  documentContents: string,\n  attributeInfo: AttributeInfo[],\n  allowedComparators?: Comparator[],\n  allowedOperators?: Operator[],\n  examples: InputValues[] = defaultExample\n) {\n  const myAllowedComparators: Comparator[] =\n    allowedComparators ?? Object.values(Comparators);\n  const myAllowedOperators: Operator[] =\n    allowedOperators ?? Object.values(Operators);\n  const attributeJSON = formatAttributeInfo(attributeInfo);\n  const schema = interpolateFString(DEFAULT_SCHEMA, {\n    allowed_comparators: myAllowedComparators.join(\" | \"),\n    allowed_operators: myAllowedOperators.join(\" | \"),\n  });\n  const prefix = interpolateFString(DEFAULT_PREFIX, {\n    schema,\n  });\n  const suffix = interpolateFString(DEFAULT_SUFFIX, {\n    i: examples.length + 1,\n    content: documentContents,\n    attributes: attributeJSON,\n  });\n\n  const outputParser = StructuredQueryOutputParser.fromComponents(\n    allowedComparators,\n    allowedOperators\n  );\n\n  return new FewShotPromptTemplate({\n    examples,\n    examplePrompt: EXAMPLE_PROMPT,\n    inputVariables: [\"query\"],\n    suffix,\n    prefix,\n    outputParser,\n  });\n}\n\n/**\n * A type that represents options for the query constructor chain.\n */\nexport type QueryConstructorChainOptions = {\n  llm: BaseLanguageModelInterface;\n  documentContents: string;\n  attributeInfo: AttributeInfo[];\n  examples?: InputValues[];\n  allowedComparators?: Comparator[];\n  allowedOperators?: Operator[];\n};\n\nexport function loadQueryConstructorChain(opts: QueryConstructorChainOptions) {\n  const prompt = _getPrompt(\n    opts.documentContents,\n    opts.attributeInfo,\n    opts.allowedComparators,\n    opts.allowedOperators,\n...\n// Path: langchain/src/retrievers/self_query/functional.ts\nimport { Document } from \"@langchain/core/documents\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { castValue, isFilterEmpty } from \"./utils.js\";\n\n/**\n * A type alias for an object that maps comparison operators to string or\n * number values. This is used in the comparison functions to determine\n * the result of a comparison operation.\n */\ntype ValueType = {\n  eq: string | number;\n  ne: string | number;\n  lt: string | number;\n  lte: string | number;\n  gt: string | number;\n  gte: string | number;\n};\n\n/**\n * A type alias for a function that takes a `Document` as an argument and\n * returns a boolean. This function is used as a filter for documents.\n */\nexport type FunctionFilter = (document: Document) => boolean;\n\n/**\n * A class that extends `BaseTranslator` to translate structured queries\n * into functional filters.\n * @example\n * ```typescript\n * const functionalTranslator = new FunctionalTranslator();\n * const relevantDocuments = await functionalTranslator.getRelevantDocuments(\n *   \"Which movies are rated higher than 8.5?\",\n * );\n * ```\n */\nexport class FunctionalTranslator extends BaseTranslator {\n  declare VisitOperationOutput: FunctionFilter;\n\n  declare VisitComparisonOutput: FunctionFilter;\n\n  declare VisitStructuredQueryOutput:\n    | { filter: FunctionFilter }\n    | { [k: string]: never };\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.gt,\n    Comparators.gte,\n    Comparators.lt,\n    Comparators.lte,\n  ];\n\n  formatFunction(): string {\n    throw new Error(\"Not implemented\");\n  }\n\n  /**\n   * Returns a function that performs a comparison based on the provided\n   * comparator.\n   * @param comparator The comparator to base the comparison function on.\n   * @returns A function that takes two arguments and returns a boolean based on the comparison.\n   */\n  getComparatorFunction<C extends Comparator>(\n    comparator: Comparator\n  ): (a: string | number, b: ValueType[C]) => boolean {\n    switch (comparator) {\n      case Comparators.eq: {\n        return (a: string | number, b: ValueType[C]) => a === b;\n      }\n      case Comparators.ne: {\n        return (a: string | number, b: ValueType[C]) => a !== b;\n      }\n      case Comparators.gt: {\n        return (a: string | number, b: ValueType[C]) => a > b;\n      }\n      case Comparators.gte: {\n        return (a: string | number, b: ValueType[C]) => a >= b;\n      }\n      case Comparators.lt: {\n        return (a: string | number, b: ValueType[C]) => a < b;\n      }\n      case Comparators.lte: {\n        return (a: string | number, b: ValueType[C]) => a <= b;\n      }\n      default: {\n        throw new Error(\"Unknown comparator\");\n      }\n    }\n  }\n\n  /**\n   * Returns a function that performs an operation based on the provided\n   * operator.\n   * @param operator The operator to base the operation function on.\n   * @returns A function that takes two boolean arguments and returns a boolean based on the operation.\n   */\n  getOperatorFunction(operator: Operator): (a: boolean, b: boolean) => boolean {\n    switch (operator) {\n      case Operators.and: {\n        return (a, b) => a && b;\n      }\n      case Operators.or: {\n        return (a, b) => a || b;\n      }\n      default: {\n        throw new Error(\"Unknown operator\");\n      }\n    }\n  }\n\n  /**\n   * Visits the operation part of a structured query and translates it into\n   * a functional filter.\n   * @param operation The operation part of a structured query.\n   * @returns A function that takes a `Document` as an argument and returns a boolean based on the operation.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const { operator, args } = operation;\n    if (this.allowedOperators.includes(operator)) {\n      const operatorFunction = this.getOperatorFunction(operator);\n      return (document: Document) => {\n        if (!args) {\n          return true;\n        }\n\n        return args.reduce((acc, arg) => {\n          const result = arg.accept(this);\n          if (typeof result === \"function\") {\n            return operatorFunction(acc, result(document));\n          } else {\n            throw new Error(\"Filter is not a function\");\n          }\n        }, true);\n      };\n    } else {\n      throw new Error(\"Operator not allowed\");\n    }\n  }\n\n  /**\n   * Visits the comparison part of a structured query and translates it into\n   * a functional filter.\n   * @param comparison The comparison part of a structured query.\n   * @returns A function that takes a `Document` as an argument and returns a boolean based on the comparison.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    const { comparator, attribute, value } = comparison;\n    const undefinedTrue = [Comparators.ne];\n    if (this.allowedComparators.includes(comparator)) {\n      const comparatorFunction = this.getComparatorFunction(comparator);\n      return (document: Document) => {\n        const documentValue = document.metadata[attribute];\n        if (documentValue === undefined) {\n          if (undefinedTrue.includes(comparator)) {\n            return true;\n          }\n          return false;\n        }\n        return comparatorFunction(documentValue, castValue(value));\n      };\n    } else {\n      throw new Error(\"Comparator not allowed\");\n    }\n  }\n\n  /**\n   * Visits a structured query and translates it into a functional filter.\n   * @param query The structured query to translate.\n   * @returns An object containing a `filter` property, which is a function that takes a `Document` as an argument and returns a boolean based on the structured query.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    if (!query.filter) {\n      return {};\n    }\n    const filterFunction = query.filter?.accept(this);\n    if (typeof filterFunction !== \"function\") {\n      throw new Error(\"Structured query filter is not a function\");\n    }\n    return { filter: filterFunction as FunctionFilter };\n  }\n\n  /**\n   * Merges two filters into one, based on the specified merge type.\n   * @param defaultFilter The default filter function.\n   * @param generatedFilter The generated filter function.\n   * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'. Default is 'and'.\n   * @returns A function that takes a `Document` as an argument and returns a boolean based on the merged filters, or `undefined` if both filters are empty.\n   */\n  mergeFilters(\n    defaultFilter: FunctionFilter,\n    generatedFilter: FunctionFilter,\n    mergeType = \"and\"\n  ): FunctionFilter | undefined {\n    if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter)) {\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n\n    if (mergeType === \"and\") {\n      return (document: Document) =>\n        defaultFilter(document) && generatedFilter(document);\n    } else if (mergeType === \"or\") {\n      return (document: Document) =>\n        defaultFilter(document) || generatedFilter(document);\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/index.ts\nimport {\n  BaseRetriever,\n  type BaseRetrieverInput,\n} from \"@langchain/core/retrievers\";\nimport { Document } from \"@langchain/core/documents\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\nimport {\n  QueryConstructorChainOptions,\n  loadQueryConstructorChain,\n} from \"../../chains/query_constructor/index.js\";\nimport { StructuredQuery } from \"../../chains/query_constructor/ir.js\";\nimport { FunctionalTranslator } from \"./functional.js\";\nimport { BaseTranslator, BasicTranslator } from \"./base.js\";\n\nexport { BaseTranslator, BasicTranslator, FunctionalTranslator };\n\n/**\n * Interface for the arguments required to create a SelfQueryRetriever\n * instance. It extends the BaseRetrieverInput interface.\n */\nexport interface SelfQueryRetrieverArgs<T extends VectorStore>\n  extends BaseRetrieverInput {\n  vectorStore: T;\n  structuredQueryTranslator: BaseTranslator<T>;\n  llmChain: LLMChain;\n  verbose?: boolean;\n  useOriginalQuery?: boolean;\n  searchParams?: {\n    k?: number;\n    filter?: T[\"FilterType\"];\n    mergeFiltersOperator?: \"or\" | \"and\" | \"replace\";\n    forceDefaultFilter?: boolean;\n  };\n}\n\n/**\n * Class for question answering over an index. It retrieves relevant\n * documents based on a query. It extends the BaseRetriever class and\n * implements the SelfQueryRetrieverArgs interface.\n * @example\n * ```typescript\n * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({\n *   llm: new ChatOpenAI(),\n *   vectorStore: await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings()),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: attributeInfo,\n *   structuredQueryTranslator: new FunctionalTranslator(),\n * });\n * const relevantDocuments = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are directed by Greta Gerwig?\",\n * );\n * ```\n */\nexport class SelfQueryRetriever<T extends VectorStore>\n  extends BaseRetriever\n  implements SelfQueryRetrieverArgs<T>\n{\n  static lc_name() {\n    return \"SelfQueryRetriever\";\n  }\n\n  get lc_namespace() {\n    return [\"langchain\", \"retrievers\", \"self_query\"];\n  }\n\n  vectorStore: T;\n\n  llmChain: LLMChain;\n\n  verbose?: boolean;\n\n  structuredQueryTranslator: BaseTranslator<T>;\n\n  useOriginalQuery = false;\n\n  searchParams?: {\n    k?: number;\n    filter?: T[\"FilterType\"];\n    mergeFiltersOperator?: \"or\" | \"and\" | \"replace\";\n    forceDefaultFilter?: boolean;\n  } = { k: 4, forceDefaultFilter: false };\n\n  constructor(options: SelfQueryRetrieverArgs<T>) {\n    super(options);\n    this.vectorStore = options.vectorStore;\n    this.llmChain = options.llmChain;\n    this.verbose = options.verbose ?? false;\n    this.searchParams = options.searchParams ?? this.searchParams;\n    this.useOriginalQuery = options.useOriginalQuery ?? this.useOriginalQuery;\n    this.structuredQueryTranslator = options.structuredQueryTranslator;\n  }\n\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Document<Record<string, unknown>>[]> {\n    const { [this.llmChain.outputKey]: output } = await this.llmChain.call(\n      {\n        [this.llmChain.inputKeys[0]]: query,\n      },\n      runManager?.getChild(\"llm_chain\")\n    );\n\n    const generatedStructuredQuery = output as StructuredQuery;\n\n    const nextArg = this.structuredQueryTranslator.visitStructuredQuery(\n      generatedStructuredQuery\n    );\n\n    const filter = this.structuredQueryTranslator.mergeFilters(\n      this.searchParams?.filter,\n      nextArg.filter,\n      this.searchParams?.mergeFiltersOperator,\n      this.searchParams?.forceDefaultFilter\n    );\n\n    const generatedQuery = generatedStructuredQuery.query;\n    let myQuery = query;\n\n    if (!this.useOriginalQuery && generatedQuery && generatedQuery.length > 0) {\n      myQuery = generatedQuery;\n    }\n\n    return this.vectorStore.similaritySearch(\n      myQuery,\n      this.searchParams?.k,\n      filter,\n      runManager?.getChild(\"vectorstore\")\n    );\n  }\n\n  /**\n   * Static method to create a new SelfQueryRetriever instance from a\n   * BaseLanguageModel and a VectorStore. It first loads a query constructor\n   * chain using the loadQueryConstructorChain function, then creates a new\n   * SelfQueryRetriever instance with the loaded chain and the provided\n   * options.\n   * @param options The options used to create the SelfQueryRetriever instance. It includes the QueryConstructorChainOptions and all the SelfQueryRetrieverArgs except 'llmChain'.\n   * @returns A new instance of SelfQueryRetriever.\n   */\n  static fromLLM<T extends VectorStore>(\n    options: QueryConstructorChainOptions &\n      Omit<SelfQueryRetrieverArgs<T>, \"llmChain\">\n  ): SelfQueryRetriever<T> {\n    const {\n      structuredQueryTranslator,\n      allowedComparators,\n      allowedOperators,\n      llm,\n      documentContents,\n      attributeInfo,\n      examples,\n      vectorStore,\n      ...rest\n    } = options;\n    const llmChain = loadQueryConstructorChain({\n      llm,\n      documentContents,\n      attributeInfo,\n      examples,\n      allowedComparators:\n        allowedComparators ?? structuredQueryTranslator.allowedComparators,\n      allowedOperators:\n        allowedOperators ?? structuredQueryTranslator.allowedOperators,\n    });\n    return new SelfQueryRetriever<T>({\n      ...rest,\n      llmChain,\n      vectorStore,\n      structuredQueryTranslator,\n    });\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/vectara.ts\nimport {\n  VectaraFilter,\n  VectaraStore,\n} from \"@langchain/community/vectorstores/vectara\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  NOT,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n  Visitor,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { isFilterEmpty } from \"./utils.js\";\n\ntype AllowedOperator = Exclude<Operator, NOT>;\n\nexport type VectaraVisitorResult =\n  | VectaraOperationResult\n  | VectaraComparisonResult\n  | VectaraVisitorStructuredQueryResult;\n// eslint-disable-next-line @typescript-eslint/ban-types\nexport type VectaraOperationResult = String;\n// eslint-disable-next-line @typescript-eslint/ban-types\nexport type VectaraComparisonResult = String;\nexport type VectaraVisitorStructuredQueryResult = {\n  filter?: { filter?: VectaraOperationResult | VectaraComparisonResult };\n};\n\ntype Value = number | string;\n\nfunction processValue(value: Value): string {\n  /** Convert a value to a string and add single quotes if it is a string. */\n  if (typeof value === \"string\") {\n    return `'${value}'`;\n  } else {\n    return String(value);\n  }\n}\n\nexport class VectaraTranslator<\n  T extends VectaraStore\n> extends BaseTranslator<T> {\n  declare VisitOperationOutput: VectaraOperationResult;\n\n  declare VisitComparisonOutput: VectaraComparisonResult;\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.lt,\n    Comparators.lte,\n    Comparators.gt,\n    Comparators.gte,\n  ];\n\n  formatFunction(func: Operator | Comparator): string {\n    if (func in Comparators) {\n      if (\n        this.allowedComparators.length > 0 &&\n        this.allowedComparators.indexOf(func as Comparator) === -1\n      ) {\n        throw new Error(\n          `Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else if (func in Operators) {\n      if (\n        this.allowedOperators.length > 0 &&\n        this.allowedOperators.indexOf(func as Operator) === -1\n      ) {\n        throw new Error(\n          `Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else {\n      throw new Error(\"Unknown comparator or operator\");\n    }\n\n    const mapDict = {\n      and: \" and \",\n      or: \" or \",\n      eq: \"=\",\n      ne: \"!=\",\n      lt: \"<\",\n      lte: \"<=\",\n      gt: \">\",\n      gte: \">=\",\n    };\n    return mapDict[func as Comparator | AllowedOperator];\n  }\n\n  /**\n   * Visits an operation and returns a VectaraOperationResult. The\n   * operation's arguments are visited and the operator is formatted.\n   * @param operation The operation to visit.\n   * @returns A VectaraOperationResult.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const args = operation.args?.map((arg) =>\n      arg.accept(this as Visitor)\n    ) as VectaraVisitorResult[];\n    const operator = this.formatFunction(operation.operator);\n    return `( ${args.join(operator)} )`;\n  }\n\n  /**\n   * Visits a comparison and returns a VectaraComparisonResult. The\n   * comparison's value is checked for type and the comparator is formatted.\n   * Throws an error if the value type is not supported.\n   * @param comparison The comparison to visit.\n   * @returns A VectaraComparisonResult.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    const comparator = this.formatFunction(comparison.comparator);\n    return `( doc.${comparison.attribute} ${comparator} ${processValue(\n      comparison.value\n    )} )`;\n  }\n\n  /**\n   * Visits a structured query and returns a VectaraStructuredQueryResult.\n   * If the query has a filter, it is visited.\n   * @param query The structured query to visit.\n   * @returns A VectaraStructuredQueryResult.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    let nextArg = {};\n    if (query.filter) {\n      nextArg = {\n        filter: { filter: query.filter.accept(this as Visitor) },\n      };\n    }\n    return nextArg;\n  }\n\n  mergeFilters(\n    defaultFilter: VectaraFilter | undefined,\n    generatedFilter: VectaraFilter | undefined,\n    mergeType = \"and\",\n    forceDefaultFilter = false\n  ): VectaraFilter | undefined {\n    if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter)) {\n      if (forceDefaultFilter) {\n        return defaultFilter;\n      }\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n\n    if (mergeType === \"and\") {\n      return {\n        filter: `${defaultFilter} and ${generatedFilter}`,\n      } as VectaraFilter;\n    } else if (mergeType === \"or\") {\n      return {\n        filter: `${defaultFilter} or ${generatedFilter}`,\n      };\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/supabase.ts\nimport type {\n  SupabaseFilterRPCCall,\n  SupabaseMetadata,\n  SupabaseVectorStore,\n} from \"@langchain/community/vectorstores/supabase\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { isFilterEmpty, isFloat, isInt, isObject, isString } from \"./utils.js\";\nimport {\n  ProxyParamsDuplicator,\n  convertObjectFilterToStructuredQuery,\n} from \"./supabase_utils.js\";\n\n/**\n * Represents the possible values that can be used in a comparison in a\n * structured query. It can be a string or a number.\n */\ntype ValueType = {\n  eq: string | number;\n  ne: string | number;\n  lt: string | number;\n  lte: string | number;\n  gt: string | number;\n  gte: string | number;\n};\n\n/**\n * A specialized translator designed to work with Supabase, extending the\n * BaseTranslator class. It translates structured queries into a format\n * that can be understood by the Supabase database.\n * @example\n * ```typescript\n * const selfQueryRetriever = new SelfQueryRetriever({\n *   llm: new ChatOpenAI(),\n *   vectorStore: new SupabaseVectorStore(),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: [],\n *   structuredQueryTranslator: new SupabaseTranslator(),\n * });\n *\n * const queryResult = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are directed by Greta Gerwig?\",\n * );\n * ```\n */\nexport class SupabaseTranslator<\n  T extends SupabaseVectorStore\n> extends BaseTranslator<T> {\n  declare VisitOperationOutput: SupabaseFilterRPCCall;\n\n  declare VisitComparisonOutput: SupabaseFilterRPCCall;\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.gt,\n    Comparators.gte,\n    Comparators.lt,\n    Comparators.lte,\n  ];\n\n  formatFunction(): string {\n    throw new Error(\"Not implemented\");\n  }\n\n  /**\n   * Returns a function that applies the appropriate comparator operation on\n   * the attribute and value provided. The function returned is used to\n   * filter data in a Supabase database.\n   * @param comparator The comparator to be used in the operation.\n   * @returns A function that applies the comparator operation on the attribute and value provided.\n   */\n  getComparatorFunction<C extends Comparator>(\n    comparator: Comparator\n  ): (attr: string, value: ValueType[C]) => SupabaseFilterRPCCall {\n    switch (comparator) {\n      case Comparators.eq: {\n        return (attr: string, value: ValueType[C]) => (rpc) =>\n          rpc.eq(this.buildColumnName(attr, value), value);\n      }\n      case Comparators.ne: {\n        return (attr: string, value: ValueType[C]) => (rpc) =>\n          rpc.neq(this.buildColumnName(attr, value), value);\n      }\n      case Comparators.gt: {\n        return (attr: string, value: ValueType[C]) => (rpc) =>\n          rpc.gt(this.buildColumnName(attr, value), value);\n      }\n      case Comparators.gte: {\n        return (attr: string, value: ValueType[C]) => (rpc) =>\n          rpc.gte(this.buildColumnName(attr, value), value);\n      }\n      case Comparators.lt: {\n        return (attr: string, value: ValueType[C]) => (rpc) =>\n          rpc.lt(this.buildColumnName(attr, value), value);\n      }\n      case Comparators.lte: {\n        return (attr: string, value: ValueType[C]) => (rpc) =>\n          rpc.lte(this.buildColumnName(attr, value), value);\n      }\n      default: {\n        throw new Error(\"Unknown comparator\");\n      }\n    }\n  }\n\n  /**\n   * Builds a column name based on the attribute and value provided. The\n   * column name is used in filtering data in a Supabase database.\n   * @param attr The attribute to be used in the column name.\n   * @param value The value to be used in the column name.\n   * @param includeType Whether to include the data type in the column name.\n   * @returns The built column name.\n   */\n  buildColumnName(attr: string, value: string | number, includeType = true) {\n    let column = \"\";\n    if (isString(value)) {\n      column = `metadata->>${attr}`;\n    } else if (isInt(value)) {\n      column = `metadata->${attr}${includeType ? \"::int\" : \"\"}`;\n    } else if (isFloat(value)) {\n      column = `metadata->${attr}${includeType ? \"::float\" : \"\"}`;\n    } else {\n      throw new Error(\"Data type not supported\");\n    }\n\n    return column;\n  }\n\n  /**\n   * Visits an operation and returns a string representation of it. This is\n   * used in translating a structured query into a format that can be\n   * understood by Supabase.\n   * @param operation The operation to be visited.\n   * @returns A string representation of the operation.\n   */\n  visitOperationAsString(operation: Operation): string {\n    const { args } = operation;\n    if (!args) {\n      return \"\";\n    }\n    return args\n      ?.reduce((acc, arg) => {\n        if (arg.exprName === \"Comparison\") {\n          acc.push(this.visitComparisonAsString(arg as Comparison));\n        } else if (arg.exprName === \"Operation\") {\n          const { operator: innerOperator } = arg as Operation;\n          acc.push(\n            `${innerOperator}(${this.visitOperationAsString(arg as Operation)})`\n          );\n        }\n        return acc;\n      }, [] as string[])\n      .join(\",\");\n  }\n\n  /**\n   * Visits an operation and returns a function that applies the operation\n   * on a Supabase database. This is used in translating a structured query\n   * into a format that can be understood by Supabase.\n   * @param operation The operation to be visited.\n   * @returns A function that applies the operation on a Supabase database.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const { operator, args } = operation;\n    if (this.allowedOperators.includes(operator)) {\n      if (operator === Operators.and) {\n        if (!args) {\n          return (rpc) => rpc;\n        }\n        const filter: SupabaseFilterRPCCall = (rpc) =>\n          args.reduce((acc, arg) => {\n            const filter = arg.accept(this) as SupabaseFilterRPCCall;\n            return filter(acc);\n          }, rpc);\n        return filter;\n      } else if (operator === Operators.or) {\n        return (rpc) => rpc.or(this.visitOperationAsString(operation));\n      } else {\n        throw new Error(\"Unknown operator\");\n      }\n    } else {\n      throw new Error(\"Operator not allowed\");\n    }\n  }\n\n  /**\n   * Visits a comparison and returns a string representation of it. This is\n   * used in translating a structured query into a format that can be\n   * understood by Supabase.\n   * @param comparison The comparison to be visited.\n   * @returns A string representation of the comparison.\n   */\n  visitComparisonAsString(comparison: Comparison): string {\n    let { value } = comparison;\n    const { comparator: _comparator, attribute } = comparison;\n    let comparator = _comparator as string;\n    if (comparator === Comparators.ne) {\n      comparator = \"neq\";\n    }\n    if (Array.isArray(value)) {\n      value = `(${value\n        .map((v) => {\n          if (typeof v === \"string\" && /[,()]/.test(v)) return `\"${v}\"`;\n          return v;\n        })\n        .join(\",\")})`;\n    }\n    return `${this.buildColumnName(\n      attribute,\n      value,\n      false\n    )}.${comparator}.${value}}`;\n  }\n\n  /**\n   * Visits a comparison and returns a function that applies the comparison\n   * on a Supabase database. This is used in translating a structured query\n   * into a format that can be understood by Supabase.\n   * @param comparison The comparison to be visited.\n   * @returns A function that applies the comparison on a Supabase database.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    const { comparator, attribute, value } = comparison;\n    if (this.allowedComparators.includes(comparator)) {\n      const comparatorFunction = this.getComparatorFunction(\n        comparator as Comparator\n      );\n      return comparatorFunction(attribute, value);\n    } else {\n      throw new Error(\"Comparator not allowed\");\n    }\n  }\n\n  /**\n   * Visits a structured query and returns a function that applies the query\n   * on a Supabase database. This is used in translating a structured query\n   * into a format that can be understood by Supabase.\n   * @param query The structured query to be visited.\n   * @returns A function that applies the query on a Supabase database.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    if (!query.filter) {\n      return {};\n    }\n    const filterFunction = query.filter?.accept(this);\n    return { filter: (filterFunction as SupabaseFilterRPCCall) ?? {} };\n  }\n\n  /**\n   * Merges two filters into one. The merged filter can be used to filter\n   * data in a Supabase database.\n   * @param defaultFilter The default filter to be merged.\n   * @param generatedFilter The generated filter to be merged.\n   * @param mergeType The type of merge to be performed. It can be 'and', 'or', or 'replace'.\n   * @returns The merged filter.\n   */\n  mergeFilters(\n    defaultFilter: SupabaseFilterRPCCall | SupabaseMetadata | undefined,\n    generatedFilter: SupabaseFilterRPCCall | undefined,\n    mergeType = \"and\"\n  ): SupabaseFilterRPCCall | SupabaseMetadata | undefined {\n    if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter)) {\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n\n    let myDefaultFilter = defaultFilter;\n    if (isObject(defaultFilter)) {\n      const { filter } = this.visitStructuredQuery(\n        convertObjectFilterToStructuredQuery(defaultFilter)\n      );\n\n      // just in case the built filter is empty somehow\n      if (isFilterEmpty(filter)) {\n        if (isFilterEmpty(generatedFilter)) {\n          return undefined;\n        }\n        return generatedFilter;\n      }\n      myDefaultFilter = filter;\n    }\n    // After this point, myDefaultFilter will always be SupabaseFilterRPCCall\n    if (mergeType === \"or\") {\n      return (rpc) => {\n        const defaultFlattenedParams = ProxyParamsDuplicator.getFlattenedParams(\n          rpc,\n          myDefaultFilter as SupabaseFilterRPCCall\n        );\n        const generatedFlattenedParams =\n          ProxyParamsDuplicator.getFlattenedParams(rpc, generatedFilter);\n        return rpc.or(`${defaultFlattenedParams},${generatedFlattenedParams}`);\n      };\n    } else if (mergeType === \"and\") {\n      return (rpc) =>\n        generatedFilter((myDefaultFilter as SupabaseFilterRPCCall)(rpc));\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/pinecone.ts\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport { Comparators, Operators } from \"../../chains/query_constructor/ir.js\";\nimport { BasicTranslator } from \"./base.js\";\n\n/**\n * Specialized translator class that extends the BasicTranslator. It is\n * designed to work with PineconeStore, a type of vector store in\n * LangChain. The class is initialized with a set of allowed operators and\n * comparators, which are used in the translation process to construct\n * queries and compare results.\n * @example\n * ```typescript\n * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({\n *   llm: new ChatOpenAI(),\n *   vectorStore: new PineconeStore(),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: [],\n *   structuredQueryTranslator: new PineconeTranslator(),\n * });\n *\n * const queryResult = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are directed by Greta Gerwig?\",\n * );\n * ```\n */\nexport class PineconeTranslator<\n  T extends VectorStoreInterface\n> extends BasicTranslator<T> {\n  constructor() {\n    super({\n      allowedOperators: [Operators.and, Operators.or],\n      allowedComparators: [\n        Comparators.eq,\n        Comparators.ne,\n        Comparators.gt,\n        Comparators.gte,\n        Comparators.lt,\n        Comparators.lte,\n      ],\n    });\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/tests/vectara_self_query.int.test.ts\n/* eslint-disable no-process-env */\nimport { test } from \"@jest/globals\";\nimport { Document } from \"@langchain/core/documents\";\nimport { OpenAI } from \"@langchain/openai\";\nimport { VectaraStore } from \"@langchain/community/vectorstores/vectara\";\nimport { FakeEmbeddings } from \"@langchain/core/utils/testing\";\nimport { AttributeInfo } from \"../../../schema/query_constructor.js\";\nimport { SelfQueryRetriever } from \"../index.js\";\nimport { VectaraTranslator } from \"../vectara.js\";\n\ntest.skip(\"Vectara Self Query Retriever Test\", async () => {\n  const docs = [\n    new Document({\n      pageContent:\n        \"A bunch of scientists bring back dinosaurs and mayhem breaks loose\",\n      metadata: { year: 1993, rating: 7.7, genre: \"science fiction\" },\n    }),\n    new Document({\n      pageContent:\n        \"Leo DiCaprio gets lost in a dream within a dream within a dream within a ...\",\n      metadata: { year: 2010, director: \"Christopher Nolan\", rating: 8.2 },\n    }),\n    new Document({\n      pageContent:\n        \"A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea\",\n      metadata: { year: 2006, director: \"Satoshi Kon\", rating: 8.6 },\n    }),\n    new Document({\n      pageContent:\n        \"A bunch of normal-sized women are supremely wholesome and some men pine after them\",\n      metadata: { year: 2019, director: \"Greta Gerwig\", rating: 8.3 },\n    }),\n    new Document({\n      pageContent: \"Toys come alive and have a blast doing so\",\n      metadata: { year: 1995, genre: \"animated\" },\n    }),\n    new Document({\n      pageContent:\n        \"Three men walk into the Zone, three men walk out of the Zone\",\n      metadata: {\n        year: 1979,\n        rating: 9.9,\n        director: \"Andrei Tarkovsky\",\n        genre: \"science fiction\",\n      },\n    }),\n  ];\n\n  const attributeInfo: AttributeInfo[] = [\n    {\n      name: \"genre\",\n      description: \"The genre of the movie\",\n      type: \"string or array of strings\",\n    },\n    {\n      name: \"year\",\n      description: \"The year the movie was released\",\n      type: \"number\",\n    },\n    {\n      name: \"director\",\n      description: \"The director of the movie\",\n      type: \"string\",\n    },\n    {\n      name: \"rating\",\n      description: \"The rating of the movie (1-10)\",\n      type: \"number\",\n    },\n  ];\n  const config = {\n    customerId: Number(process.env.VECTARA_CUSTOMER_ID),\n    corpusId: Number(process.env.VECTARA_CORPUS_ID),\n    apiKey: String(process.env.VECTARA_API_KEY),\n    verbose: true,\n  };\n\n  const vectorStore = await VectaraStore.fromDocuments(\n    docs,\n    new FakeEmbeddings(),\n    config\n  );\n\n  const llm = new OpenAI();\n  const documentContents = \"Brief summary of a movie\";\n\n  const selfQueryRetriever = await SelfQueryRetriever.fromLLM({\n    llm,\n    vectorStore,\n    documentContents,\n    attributeInfo,\n\n    structuredQueryTranslator: new VectaraTranslator(),\n  });\n\n  const query1 = await selfQueryRetriever.getRelevantDocuments(\n    \"I want to watch a movie rated higher than 8.5\"\n  );\n  const query2 = await selfQueryRetriever.getRelevantDocuments(\n    \"Which movies are directed by Greta Gerwig?\"\n  );\n  const query3 = await selfQueryRetriever.getRelevantDocuments(\n    \"Which movies are either comedy or science fiction and are rated higher than 8.5?\"\n  );\n  const query4 = await selfQueryRetriever.getRelevantDocuments(\n    \"Wau wau wau wau hello gello hello?\"\n  );\n  console.log(query1, query2, query3, query4);\n  expect(query1.length).toBe(2);\n  expect(query2.length).toBe(1);\n  expect(query3.length).toBe(1);\n  expect(query4.length).toBe(0);\n});\n\n// Path: langchain/src/retrievers/self_query/tests/weaviate_self_query.int.test.ts\n/* eslint-disable no-process-env */\nimport { test } from \"@jest/globals\";\nimport weaviate from \"weaviate-ts-client\";\nimport { Document } from \"@langchain/core/documents\";\nimport { OpenAIEmbeddings, OpenAI } from \"@langchain/openai\";\nimport { WeaviateStore } from \"@langchain/community/vectorstores/weaviate\";\nimport { AttributeInfo } from \"../../../schema/query_constructor.js\";\nimport { SelfQueryRetriever } from \"../index.js\";\nimport { WeaviateTranslator } from \"../weaviate.js\";\n\ntest.skip(\"Weaviate Self Query Retriever Test\", async () => {\n  const docs = [\n    new Document({\n      pageContent:\n        \"A bunch of scientists bring back dinosaurs and mayhem breaks loose\",\n      metadata: { year: 1993, rating: 7.7, genre: \"science fiction\" },\n    }),\n    new Document({\n      pageContent:\n        \"Leo DiCaprio gets lost in a dream within a dream within a dream within a ...\",\n      metadata: { year: 2010, director: \"Christopher Nolan\", rating: 8.2 },\n    }),\n    new Document({\n      pageContent:\n        \"A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea\",\n      metadata: { year: 2006, director: \"Satoshi Kon\", rating: 8.6 },\n    }),\n    new Document({\n      pageContent:\n        \"A bunch of normal-sized women are supremely wholesome and some men pine after them\",\n      metadata: { year: 2019, director: \"Greta Gerwig\", rating: 8.3 },\n    }),\n    new Document({\n      pageContent: \"Toys come alive and have a blast doing so\",\n      metadata: { year: 1995, genre: \"animated\" },\n    }),\n    new Document({\n      pageContent:\n        \"Three men walk into the Zone, three men walk out of the Zone\",\n      metadata: {\n        year: 1979,\n        director: \"Andrei Tarkovsky\",\n        genre: \"science fiction\",\n        rating: 9.9,\n      },\n    }),\n  ];\n\n  const attributeInfo: AttributeInfo[] = [\n    {\n      name: \"genre\",\n      description: \"The genre of the movie\",\n      type: \"string or array of strings\",\n    },\n    {\n      name: \"year\",\n      description: \"The year the movie was released\",\n      type: \"number\",\n    },\n    {\n      name: \"director\",\n      description: \"The director of the movie\",\n      type: \"string\",\n    },\n    {\n      name: \"rating\",\n      description: \"The rating of the movie (1-10)\",\n      type: \"number\",\n    },\n    {\n      name: \"length\",\n      description: \"The length of the movie in minutes\",\n      type: \"number\",\n    },\n  ];\n\n  const embeddings = new OpenAIEmbeddings();\n  const llm = new OpenAI({\n    modelName: \"gpt-3.5-turbo\",\n  });\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  const client = (weaviate as any).client({\n    scheme:\n      process.env.WEAVIATE_SCHEME ||\n      (process.env.WEAVIATE_HOST ? \"https\" : \"http\"),\n    host: process.env.WEAVIATE_HOST || \"localhost:8080\",\n    apiKey: process.env.WEAVIATE_API_KEY\n      ? // eslint-disable-next-line @typescript-eslint/no-explicit-any\n        new (weaviate as any).ApiKey(process.env.WEAVIATE_API_KEY)\n      : undefined,\n  });\n\n  const documentContents = \"Brief summary of a movie\";\n  const vectorStore = await WeaviateStore.fromDocuments(docs, embeddings, {\n    client,\n    indexName: \"Test\",\n    textKey: \"text\",\n    metadataKeys: [\"year\", \"director\", \"rating\", \"genre\"],\n  });\n  const selfQueryRetriever = await SelfQueryRetriever.fromLLM({\n    llm,\n    vectorStore,\n    documentContents,\n    attributeInfo,\n    structuredQueryTranslator: new WeaviateTranslator(),\n  });\n\n  const query2 = await selfQueryRetriever.getRelevantDocuments(\n    \"Which movies are rated higher than 8.5?\"\n  );\n  const query3 = await selfQueryRetriever.getRelevantDocuments(\n    \"Which movies are directed by Greta Gerwig?\"\n  );\n  const query4 = await selfQueryRetriever.getRelevantDocuments(\n    \"Wau wau wau wau hello gello hello?\"\n  );\n  console.log(query2, query3, query4); // query4 has to return empty array\n  expect(query4.length).toBe(0);\n});\n\ntest.skip(\"Weaviate Vector Store Self Query Retriever Test With Default Filter Or Merge Operator\", async () => {\n  const docs = [\n    new Document({\n      pageContent:\n        \"A bunch of scientists bring back dinosaurs and mayhem breaks loose\",\n      metadata: {\n        year: 1993,\n        rating: 7.7,\n        genre: \"science fiction\",\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent:\n        \"Leo DiCaprio gets lost in a dream within a dream within a dream within a ...\",\n      metadata: {\n        year: 2010,\n        director: \"Christopher Nolan\",\n        rating: 8.2,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent:\n        \"A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea\",\n      metadata: {\n        year: 2006,\n        director: \"Satoshi Kon\",\n        rating: 8.6,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent:\n        \"A bunch of normal-sized women are supremely wholesome and some men pine after them\",\n      metadata: {\n        year: 2019,\n        director: \"Greta Gerwig\",\n        rating: 8.3,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent: \"Toys come alive and have a blast doing so\",\n      metadata: { year: 1995, genre: \"animated\", type: \"movie\" },\n    }),\n    new Document({\n      pageContent:\n        \"Three men walk into the Zone, three men walk out of the Zone\",\n      metadata: {\n        year: 1979,\n        director: \"Andrei Tarkovsky\",\n        genre: \"science fiction\",\n        rating: 9.9,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent: \"10x the previous gecs\",\n      metadata: {\n        year: 2023,\n        title: \"10000 gecs\",\n        artist: \"100 gecs\",\n        rating: 9.9,\n        type: \"album\",\n      },\n    }),\n  ];\n\n  const attributeInfo: AttributeInfo[] = [\n    {\n      name: \"genre\",\n      description: \"The genre of the movie\",\n      type: \"string or array of strings\",\n    },\n    {\n      name: \"year\",\n      description: \"The year the movie was released\",\n      type: \"number\",\n    },\n    {\n      name: \"director\",\n      description: \"The director of the movie\",\n      type: \"string\",\n    },\n    {\n      name: \"rating\",\n      description: \"The rating of the movie (1-10)\",\n      type: \"number\",\n    },\n    {\n      name: \"length\",\n      description: \"The length of the movie in minutes\",\n      type: \"number\",\n    },\n  ];\n\n  const embeddings = new OpenAIEmbeddings();\n  const llm = new OpenAI({\n    modelName: \"gpt-3.5-turbo\",\n  });\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  const client = (weaviate as any).client({\n    scheme:\n      process.env.WEAVIATE_SCHEME ||\n      (process.env.WEAVIATE_HOST ? \"https\" : \"http\"),\n    host: process.env.WEAVIATE_HOST || \"localhost:8080\",\n    apiKey: process.env.WEAVIATE_API_KEY\n      ? // eslint-disable-next-line @typescript-eslint/no-explicit-any\n        new (weaviate as any).ApiKey(process.env.WEAVIATE_API_KEY)\n      : undefined,\n  });\n\n  const documentContents = \"Brief summary of a movie\";\n  const vectorStore = await WeaviateStore.fromDocuments(docs, embeddings, {\n    client,\n    indexName: \"Test\",\n    textKey: \"text\",\n    metadataKeys: [\"year\", \"director\", \"rating\", \"genre\", \"type\"],\n  });\n\n  const selfQueryRetriever = await SelfQueryRetriever.fromLLM({\n    llm,\n    vectorStore,\n    documentContents,\n    attributeInfo,\n    structuredQueryTranslator: new WeaviateTranslator(),\n    searchParams: {\n      filter: {\n        where: {\n          operator: \"Equal\",\n          path: [\"type\"],\n          valueText: \"movie\",\n        },\n      },\n      mergeFiltersOperator: \"or\",\n      k: docs.length,\n    },\n  });\n\n  const query4 = await selfQueryRetriever.getRelevantDocuments(\n    \"Wau wau wau wau hello gello hello?\"\n  );\n  console.log(query4); // query4 has to return documents, since the default filter takes over with\n  expect(query4.length).toEqual(7);\n});\n\ntest.skip(\"Weaviate Vector Store Self Query Retriever Test With Default Filter And Merge Operator\", async () => {\n  const docs = [\n    new Document({\n      pageContent:\n        \"A bunch of scientists bring back dinosaurs and mayhem breaks loose\",\n      metadata: {\n        year: 1993,\n        rating: 7.7,\n        genre: \"science fiction\",\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent:\n        \"Leo DiCaprio gets lost in a dream within a dream within a dream within a ...\",\n      metadata: {\n        year: 2010,\n        director: \"Christopher Nolan\",\n        rating: 8.2,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent:\n        \"A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea\",\n      metadata: {\n        year: 2006,\n        director: \"Satoshi Kon\",\n        rating: 8.6,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent:\n        \"A bunch of normal-sized women are supremely wholesome and some men pine after them\",\n      metadata: {\n        year: 2019,\n        director: \"Greta Gerwig\",\n        rating: 8.3,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent: \"Toys come alive and have a blast doing so\",\n      metadata: { year: 1995, genre: \"animated\", type: \"movie\" },\n    }),\n    new Document({\n      pageContent:\n        \"Three men walk into the Zone, three men walk out of the Zone\",\n      metadata: {\n        year: 1979,\n        director: \"Andrei Tarkovsky\",\n        genre: \"science fiction\",\n        rating: 9.9,\n        type: \"movie\",\n      },\n    }),\n    new Document({\n      pageContent: \"10x the previous gecs\",\n      metadata: {\n        year: 2023,\n        title: \"10000 gecs\",\n        artist: \"100 gecs\",\n        rating: 9.9,\n        type: \"album\",\n      },\n    }),\n  ];\n\n  const attributeInfo: AttributeInfo[] = [\n    {\n      name: \"genre\",\n      description: \"The genre of the movie\",\n      type: \"string or array of strings\",\n    },\n    {\n      name: \"year\",\n      description: \"The year the movie was released\",\n      type: \"number\",\n    },\n    {\n      name: \"director\",\n      description: \"The director of the movie\",\n      type: \"string\",\n    },\n    {\n      name: \"rating\",\n      description: \"The rating of the movie (1-10)\",\n      type: \"number\",\n    },\n    {\n      name: \"length\",\n      description: \"The length of the movie in minutes\",\n      type: \"number\",\n    },\n  ];\n\n  const embeddings = new OpenAIEmbeddings();\n  const llm = new OpenAI({\n    modelName: \"gpt-3.5-turbo\",\n  });\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  const client = (weaviate as any).client({\n    scheme:\n      process.env.WEAVIATE_SCHEME ||\n      (process.env.WEAVIATE_HOST ? \"https\" : \"http\"),\n    host: process.env.WEAVIATE_HOST || \"localhost:8080\",\n    apiKey: process.env.WEAVIATE_API_KEY\n      ? // eslint-disable-next-line @typescript-eslint/no-explicit-any\n        new (weaviate as any).ApiKey(process.env.WEAVIATE_API_KEY)\n      : undefined,\n  });\n\n  const documentContents = \"Brief summary of a movie\";\n  const vectorStore = await WeaviateStore.fromDocuments(docs, embeddings, {\n    client,\n    indexName: \"Test\",\n    textKey: \"text\",\n    metadataKeys: [\"year\", \"director\", \"rating\", \"genre\", \"type\"],\n  });\n\n  const selfQueryRetriever = await SelfQueryRetriever.fromLLM({\n    llm,\n    vectorStore,\n    documentContents,\n    attributeInfo,\n    structuredQueryTranslator: new WeaviateTranslator(),\n    searchParams: {\n      filter: {\n        where: {\n          operator: \"Equal\",\n          path: [\"type\"],\n          valueText: \"movie\",\n        },\n      },\n      mergeFiltersOperator: \"and\",\n      k: docs.length,\n    },\n  });\n\n  const query4 = await selfQueryRetriever.getRelevantDocuments(\n    \"Wau wau wau wau hello gello hello?\"\n  );\n  console.log(query4); // query4 has to return empty array, since the default filter takes over with and filter\n  expect(query4.length).toEqual(0);\n});\n\n// Path: langchain/src/retrievers/self_query/tests/chroma_self_query.int.test.ts\nimport { test } from \"@jest/globals\";\nimport { Document } from \"@langchain/core/documents\";\nimport { OpenAIEmbeddings, OpenAI } from \"@langchain/openai\";\nimport { Chroma } from \"@langchain/community/vectorstores/chroma\";\nimport { AttributeInfo } from \"../../../schema/query_constructor.js\";\nimport { SelfQueryRetriever } from \"../index.js\";\nimport { ChromaTranslator } from \"../chroma.js\";\n\ntest.skip(\"Chroma Store Self Query Retriever Test\", async () => {\n  const docs = [\n    new Document({\n      pageContent:\n        \"A bunch of scientists bring back dinosaurs and mayhem breaks loose\",\n      metadata: { year: 1993, rating: 7.7, genre: \"science fiction\" },\n    }),\n    new Document({\n      pageContent:\n        \"Leo DiCaprio gets lost in a dream within a dream within a dream within a ...\",\n      metadata: { year: 2010, director: \"Christopher Nolan\", rating: 8.2 },\n    }),\n    new Document({\n      pageContent:\n        \"A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea\",\n      metadata: { year: 2006, director: \"Satoshi Kon\", rating: 8.6 },\n    }),\n    new Document({\n      pageContent:\n        \"A bunch of normal-sized women are supremely wholesome and some men pine after them\",\n      metadata: { year: 2019, director: \"Greta Gerwig\", rating: 8.3 },\n    }),\n    new Document({\n      pageContent: \"Toys come alive and have a blast doing so\",\n      metadata: { year: 1995, genre: \"animated\" },\n    }),\n    new Document({\n      pageContent:\n        \"Three men walk into the Zone, three men walk out of the Zone\",\n      metadata: {\n        year: 1979,\n        director: \"Andrei Tarkovsky\",\n        genre: \"science fiction\",\n        rating: 9.9,\n      },\n    }),\n  ];\n\n  const attributeInfo: AttributeInfo[] = [\n    {\n      name: \"genre\",\n      description: \"The genre of the movie\",\n      type: \"string or array of strings\",\n    },\n    {\n      name: \"year\",\n      description: \"The year the movie was released\",\n      type: \"number\",\n    },\n    {\n      name: \"director\",\n      description: \"The director of the movie\",\n...\n", "needle_token_start": 5600, "needle_token_end": 5666, "code_context_ntokens": 16181, "output": ["```typescript\nfunction processValue(value: Value): string {\n  /** Convert a value to a string and add single quotes if it is a string. */\n  if (typeof value === \"string\") {\n    return `'${value}'`;\n  } else {\n    return String(value);\n  }\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "convertObjectFilterToStructuredQuery", "language": "typescript", "path": "langchain/src/retrievers/self_query/supabase_utils.ts", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform a metadata object into a structured query format suitable for database operations, specifically for filtering data based on multiple conditions combined using a logical 'AND'.\n2. **Input**: Accepts a metadata object where keys represent database column names and values represent the corresponding filter criteria for these columns.\n3. **Output**: Returns a structured query object that encapsulates the filtering logic, which is constructed using a series of comparison operations that are all combined under a logical 'AND' operation.\n4. **Procedure**: The function initializes a new structured query object. It then iterates over each entry in the input metadata object, creating a comparison operation for each (equating the column to the specified value). These comparisons are then collectively used to create a compound operation that logically 'ANDs' all the individual comparisons, which is set as the main operation of the structured query.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/chains/query_constructor/ir.ts\nimport { VectorStore } from \"@langchain/core/vectorstores\";\n\n/**\n * Represents logical AND operator.\n */\nexport type AND = \"and\";\n/**\n * Represents logical OR operator.\n */\n...\n// Path: langchain/src/retrievers/self_query/base.ts\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n  Visitor,\n  VisitorComparisonResult,\n  VisitorOperationResult,\n  VisitorResult,\n  VisitorStructuredQueryResult,\n} from \"../../chains/query_constructor/ir.js\";\nimport { isFilterEmpty, castValue } from \"./utils.js\";\n\n/**\n * Options object for the BasicTranslator class. Specifies the allowed\n * operators and comparators.\n */\nexport type TranslatorOpts = {\n  allowedOperators: Operator[];\n  allowedComparators: Comparator[];\n};\n\n/**\n * Abstract class that provides a blueprint for creating specific\n * translator classes. Defines two abstract methods: formatFunction and\n * mergeFilters.\n */\nexport abstract class BaseTranslator<\n  T extends VectorStore = VectorStore\n> extends Visitor<T> {\n  /**\n   * Formats a given function (either an operator or a comparator) into a\n   * string.\n   * @param func The function to format.\n   * @returns Formatted string representation of the function.\n   */\n  abstract formatFunction(func: Operator | Comparator): string;\n\n  /**\n   * Merges two filters into one, using a specified merge type.\n   * @param defaultFilter The default filter.\n   * @param generatedFilter The generated filter.\n   * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'.\n   * @param forceDefaultFilter If true, the default filter will be used even if the generated filter is not empty.\n   * @returns The merged filter, or undefined if both filters are empty.\n   */\n  abstract mergeFilters(\n    defaultFilter: this[\"VisitStructuredQueryOutput\"][\"filter\"] | undefined,\n    generatedFilter: this[\"VisitStructuredQueryOutput\"][\"filter\"] | undefined,\n    mergeType?: \"and\" | \"or\" | \"replace\",\n    forceDefaultFilter?: boolean\n  ): this[\"VisitStructuredQueryOutput\"][\"filter\"] | undefined;\n}\n\n/**\n * Class that extends the BaseTranslator class and provides concrete\n * implementations for the abstract methods. Also declares three types:\n * VisitOperationOutput, VisitComparisonOutput, and\n * VisitStructuredQueryOutput, which are used as the return types for the\n * visitOperation, visitComparison, and visitStructuredQuery methods\n * respectively.\n */\nexport class BasicTranslator<\n  T extends VectorStore = VectorStore\n> extends BaseTranslator<T> {\n  declare VisitOperationOutput: VisitorOperationResult;\n\n  declare VisitComparisonOutput: VisitorComparisonResult;\n\n  declare VisitStructuredQueryOutput: VisitorStructuredQueryResult;\n\n  allowedOperators: Operator[];\n\n  allowedComparators: Comparator[];\n\n  constructor(opts?: TranslatorOpts) {\n    super();\n    this.allowedOperators = opts?.allowedOperators ?? [\n      Operators.and,\n      Operators.or,\n    ];\n    this.allowedComparators = opts?.allowedComparators ?? [\n      Comparators.eq,\n      Comparators.ne,\n      Comparators.gt,\n      Comparators.gte,\n      Comparators.lt,\n      Comparators.lte,\n    ];\n  }\n\n  formatFunction(func: Operator | Comparator): string {\n    if (func in Comparators) {\n      if (\n        this.allowedComparators.length > 0 &&\n        this.allowedComparators.indexOf(func as Comparator) === -1\n      ) {\n        throw new Error(\n          `Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else if (func in Operators) {\n      if (\n        this.allowedOperators.length > 0 &&\n        this.allowedOperators.indexOf(func as Operator) === -1\n      ) {\n        throw new Error(\n          `Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else {\n      throw new Error(\"Unknown comparator or operator\");\n    }\n    return `$${func}`;\n  }\n\n  /**\n   * Visits an operation and returns a result.\n   * @param operation The operation to visit.\n   * @returns The result of visiting the operation.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const args = operation.args?.map((arg) =>\n      arg.accept(this)\n    ) as VisitorResult[];\n    return {\n      [this.formatFunction(operation.operator)]: args,\n    };\n  }\n\n  /**\n   * Visits a comparison and returns a result.\n   * @param comparison The comparison to visit.\n   * @returns The result of visiting the comparison.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    return {\n      [comparison.attribute]: {\n        [this.formatFunction(comparison.comparator)]: castValue(\n          comparison.value\n        ),\n      },\n    };\n  }\n\n  /**\n   * Visits a structured query and returns a result.\n   * @param query The structured query to visit.\n   * @returns The result of visiting the structured query.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    let nextArg = {};\n    if (query.filter) {\n      nextArg = {\n        filter: query.filter.accept(this),\n      };\n    }\n    return nextArg;\n  }\n\n  mergeFilters(\n    defaultFilter: VisitorStructuredQueryResult[\"filter\"] | undefined,\n    generatedFilter: VisitorStructuredQueryResult[\"filter\"] | undefined,\n    mergeType = \"and\",\n    forceDefaultFilter = false\n  ): VisitorStructuredQueryResult[\"filter\"] | undefined {\n    if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter)) {\n      if (forceDefaultFilter) {\n        return defaultFilter;\n      }\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n    if (mergeType === \"and\") {\n      return {\n        $and: [defaultFilter, generatedFilter],\n      };\n    } else if (mergeType === \"or\") {\n      return {\n        $or: [defaultFilter, generatedFilter],\n      };\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/weaviate.ts\nimport {\n  WeaviateFilter,\n  WeaviateStore,\n} from \"@langchain/community/vectorstores/weaviate\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  NOT,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n  Visitor,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { isFilterEmpty, isString, isInt, isFloat } from \"./utils.js\";\n\ntype AllowedOperator = Exclude<Operator, NOT>;\n\ntype WeaviateOperatorValues = {\n  valueText: string;\n  valueInt: number;\n  valueNumber: number;\n  valueBoolean: boolean;\n};\n\ntype WeaviateOperatorKeys = keyof WeaviateOperatorValues;\n\ntype ExclusiveOperatorValue = {\n  [L in WeaviateOperatorKeys]: {\n    [key in L]: WeaviateOperatorValues[key];\n  } & Omit<{ [key in WeaviateOperatorKeys]?: never }, L>;\n}[WeaviateOperatorKeys];\n\nexport type WeaviateVisitorResult =\n  | WeaviateOperationResult\n  | WeaviateComparisonResult\n  | WeaviateStructuredQueryResult;\n\nexport type WeaviateOperationResult = {\n  operator: string;\n  operands: WeaviateVisitorResult[];\n};\nexport type WeaviateComparisonResult = {\n  path: [string];\n  operator: string;\n} & ExclusiveOperatorValue;\n\nexport type WeaviateStructuredQueryResult = {\n  filter?: {\n    where?: WeaviateComparisonResult | WeaviateOperationResult;\n  };\n};\n\n/**\n * A class that translates or converts data into a format that can be used\n * with Weaviate, a vector search engine. It extends the `BaseTranslator`\n * class and provides specific implementation for Weaviate.\n * @example\n * ```typescript\n * const selfQueryRetriever = new SelfQueryRetriever({\n *   llm: new ChatOpenAI(),\n *   vectorStore: new WeaviateStore(),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: [],\n *   structuredQueryTranslator: new WeaviateTranslator(),\n * });\n *\n * const relevantDocuments = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are rated higher than 8.5?\",\n * );\n * ```\n */\nexport class WeaviateTranslator<\n  T extends WeaviateStore\n> extends BaseTranslator<T> {\n  declare VisitOperationOutput: WeaviateOperationResult;\n\n  declare VisitComparisonOutput: WeaviateComparisonResult;\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.lt,\n    Comparators.lte,\n    Comparators.gt,\n    Comparators.gte,\n  ];\n\n  /**\n   * Formats the given function into a string representation. Throws an\n   * error if the function is not a known comparator or operator, or if it\n   * is not allowed.\n   * @param func The function to format, which can be an Operator or Comparator.\n   * @returns A string representation of the function.\n   */\n  formatFunction(func: Operator | Comparator): string {\n    if (func in Comparators) {\n      if (\n        this.allowedComparators.length > 0 &&\n        this.allowedComparators.indexOf(func as Comparator) === -1\n      ) {\n        throw new Error(\n          `Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else if (func in Operators) {\n      if (\n        this.allowedOperators.length > 0 &&\n        this.allowedOperators.indexOf(func as Operator) === -1\n      ) {\n        throw new Error(\n          `Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else {\n      throw new Error(\"Unknown comparator or operator\");\n    }\n    const dict = {\n      and: \"And\",\n      or: \"Or\",\n      eq: \"Equal\",\n      ne: \"NotEqual\",\n      lt: \"LessThan\",\n      lte: \"LessThanEqual\",\n      gt: \"GreaterThan\",\n      gte: \"GreaterThanEqual\",\n    };\n    return dict[func as Comparator | AllowedOperator];\n  }\n\n  /**\n   * Visits an operation and returns a WeaviateOperationResult. The\n   * operation's arguments are visited and the operator is formatted.\n   * @param operation The operation to visit.\n   * @returns A WeaviateOperationResult.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const args = operation.args?.map((arg) =>\n      arg.accept(this as Visitor)\n    ) as WeaviateVisitorResult[];\n    return {\n      operator: this.formatFunction(operation.operator),\n      operands: args,\n    };\n  }\n\n  /**\n   * Visits a comparison and returns a WeaviateComparisonResult. The\n   * comparison's value is checked for type and the comparator is formatted.\n   * Throws an error if the value type is not supported.\n   * @param comparison The comparison to visit.\n   * @returns A WeaviateComparisonResult.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    if (isString(comparison.value)) {\n      return {\n        path: [comparison.attribute],\n        operator: this.formatFunction(comparison.comparator),\n        valueText: comparison.value as string,\n      };\n    }\n    if (isInt(comparison.value)) {\n      return {\n        path: [comparison.attribute],\n        operator: this.formatFunction(comparison.comparator),\n        valueInt: parseInt(comparison.value as string, 10),\n      };\n    }\n    if (isFloat(comparison.value)) {\n      return {\n        path: [comparison.attribute],\n        operator: this.formatFunction(comparison.comparator),\n        valueNumber: parseFloat(comparison.value as string),\n      };\n    }\n\n    throw new Error(\"Value type is not supported\");\n  }\n\n  /**\n   * Visits a structured query and returns a WeaviateStructuredQueryResult.\n   * If the query has a filter, it is visited.\n   * @param query The structured query to visit.\n   * @returns A WeaviateStructuredQueryResult.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    let nextArg = {};\n    if (query.filter) {\n      nextArg = {\n        filter: { where: query.filter.accept(this as Visitor) },\n      };\n    }\n    return nextArg;\n  }\n\n  /**\n   * Merges two filters into one. If both filters are empty, returns\n   * undefined. If one filter is empty or the merge type is 'replace',\n   * returns the other filter. If the merge type is 'and' or 'or', returns a\n   * new filter with the merged results. Throws an error for unknown merge\n   * types.\n   * @param defaultFilter The default filter to merge.\n   * @param generatedFilter The generated filter to merge.\n   * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'. Defaults to 'and'.\n   * @returns A merged WeaviateFilter, or undefined if both filters are empty.\n   */\n  mergeFilters(\n    defaultFilter: WeaviateFilter | undefined,\n    generatedFilter: WeaviateFilter | undefined,\n    mergeType = \"and\"\n  ): WeaviateFilter | undefined {\n    if (\n      isFilterEmpty(defaultFilter?.where) &&\n      isFilterEmpty(generatedFilter?.where)\n    ) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter?.where) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter?.where)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter?.where)) {\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n    const merged: WeaviateOperationResult = {\n      operator: \"And\",\n      operands: [\n        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n        defaultFilter!.where as WeaviateVisitorResult,\n        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n        generatedFilter!.where as WeaviateVisitorResult,\n      ],\n    };\n    if (mergeType === \"and\") {\n      return {\n        where: merged,\n      } as WeaviateFilter;\n    } else if (mergeType === \"or\") {\n      merged.operator = \"Or\";\n      return {\n        where: merged,\n      } as WeaviateFilter;\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/chroma.ts\nimport { Chroma } from \"@langchain/community/vectorstores/chroma\";\nimport { Comparators, Operators } from \"../../chains/query_constructor/ir.js\";\nimport { BasicTranslator } from \"./base.js\";\n\n/**\n * Specialized translator for the Chroma vector database. It extends the\n * BasicTranslator class and translates internal query language elements\n * to valid filters. The class defines a subset of allowed logical\n * operators and comparators that can be used in the translation process.\n * @example\n * ```typescript\n * const chromaTranslator = new ChromaTranslator();\n * const selfQueryRetriever = new SelfQueryRetriever({\n *   llm: new ChatOpenAI(),\n *   vectorStore: new Chroma(),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: [],\n *   structuredQueryTranslator: chromaTranslator,\n * });\n *\n * const relevantDocuments = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are directed by Greta Gerwig?\",\n * );\n * ```\n */\nexport class ChromaTranslator<T extends Chroma> extends BasicTranslator<T> {\n  constructor() {\n    super({\n      allowedOperators: [Operators.and, Operators.or],\n      allowedComparators: [\n        Comparators.eq,\n        Comparators.ne,\n        Comparators.gt,\n        Comparators.gte,\n        Comparators.lt,\n        Comparators.lte,\n      ],\n    });\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/supabase_utils.ts\nimport type {\n  SupabaseFilter,\n  SupabaseFilterRPCCall,\n  SupabaseMetadata,\n} from \"@langchain/community/vectorstores/supabase\";\nimport {\n  Comparators,\n  Comparison,\n  Operation,\n  Operators,\n  StructuredQuery,\n} from \"../../chains/query_constructor/ir.js\";\n\ntype SupabaseFilterProps = keyof SupabaseFilter;\n\n/**\n * Utility class used to duplicate parameters for a proxy object,\n * specifically designed to work with `SupabaseFilter` objects. It\n * contains methods to handle different types of operations such as \"or\",\n * \"filter\", \"in\", \"contains\", \"textSearch\", \"match\", \"not\", and default\n * operations.\n */\nexport class ProxyParamsDuplicator {\n  duplicationAllowedOps: string[] = [\n    \"eq\",\n    \"neq\",\n    \"lt\",\n    \"lte\",\n    \"gt\",\n    \"gte\",\n    \"like\",\n    \"ilike\",\n    \"or\",\n    \"in\",\n    \"contains\",\n    \"match\",\n    \"not\",\n    \"textSearch\",\n    \"filter\",\n  ];\n\n  values: [string, string][] = [];\n\n  /**\n   * Creates a proxy handler for a `SupabaseFilter` object. The handler\n   * intercepts get operations and applies specific logic based on the\n   * property being accessed.\n   * @returns A proxy handler for a `SupabaseFilter` object.\n   */\n  buildProxyHandler() {\n    const proxyHandler: ProxyHandler<SupabaseFilter> = {\n      get: (target, prop, receiver) => {\n        if (typeof target[prop as SupabaseFilterProps] === \"function\") {\n          // eslint-disable-next-line @typescript-eslint/no-explicit-any\n          return (...args: any[]) => {\n            if (this.duplicationAllowedOps.includes(String(prop))) {\n              switch (String(prop)) {\n                case \"or\":\n                  // args[0]: filters, args[1]: { foreignTable }\n                  this.addOrClause(args[0], args[1]);\n                  break;\n                case \"filter\":\n                  // args[0]: column, args[1]: operator, args[2]: value\n                  this.addFilterClause(args[0], args[1], args[2]);\n                  break;\n                case \"in\":\n                  // args[0]: column, args[1]: values\n                  this.addInClause(args[0], args[1]);\n                  break;\n                case \"contains\":\n                  // args[0]: column, args[1]: value\n                  this.addContainsClause(args[0], args[1]);\n                  break;\n                case \"textSearch\":\n                  // args[0]: column, args[1]: query, args[2]: { config, type }\n                  this.addTextSearchClause(args[0], args[1], args[2]);\n                  break;\n                case \"match\":\n                  // args[0]: query\n                  this.addMatchClause(args[0]);\n                  break;\n                case \"not\":\n                  // args[0]: column, args[1]: operator, args[2]: value\n                  this.addNotClause(args[0], args[1], args[2]);\n                  break;\n                default:\n                  // args[0]: column, args[1]: value\n                  this.addDefaultOpClause(prop as string, args[0], args[1]);\n              }\n              return new Proxy(target, proxyHandler);\n            } else {\n              throw new Error(\n                \"Filter operation not supported for 'or' mergeFiltersOperator\"\n              );\n            }\n          };\n        } else {\n          return Reflect.get(target, prop, receiver);\n        }\n      },\n    };\n\n    return proxyHandler;\n  }\n\n  /**\n   * Removes type annotations from a value string.\n   * @param value The value string to clean.\n   * @returns The cleaned value string.\n   */\n  removeType(value: string) {\n    let cleanedValue = value;\n    if (cleanedValue.includes(\"::float\")) {\n      cleanedValue = cleanedValue.replace(\"::float\", \"\");\n    }\n    if (cleanedValue.includes(\"::int\")) {\n      cleanedValue = cleanedValue.replace(\"::int\", \"\");\n    }\n    return cleanedValue;\n  }\n\n  /**\n   * Adds a default operation clause to the values array.\n   * @param prop The operation property.\n   * @param column The column to apply the operation to.\n   * @param value The value for the operation.\n   */\n  addDefaultOpClause(prop: string, column: string, value: unknown) {\n    this.values.push([this.removeType(column), `${String(prop)}.${value}`]);\n  }\n\n  /**\n   * Adds an 'or' clause to the values array.\n   * @param filters The filters for the 'or' clause.\n   * @param foreignTable Optional foreign table for the 'or' clause.\n   */\n  addOrClause(\n    filters: string,\n    { foreignTable }: { foreignTable?: string } = {}\n  ) {\n    const key = foreignTable ? `${foreignTable}.or` : \"or\";\n    this.values.push([this.removeType(key), `(${filters})`]);\n  }\n\n  /**\n   * Adds a 'filter' clause to the values array.\n   * @param column The column to apply the filter to.\n   * @param operator The operator for the filter.\n   * @param value The value for the filter.\n   */\n  addFilterClause(column: string, operator: string, value: unknown) {\n    this.values.push([this.removeType(column), `${operator}.${value}`]);\n  }\n\n  /**\n   * Adds an 'in' clause to the values array.\n   * @param column The column to apply the 'in' clause to.\n   * @param values The values for the 'in' clause.\n   */\n  addInClause(column: string, values: unknown[]) {\n    const cleanedValues = values\n      .map((s) => {\n        if (typeof s === \"string\" && /[,()]/.test(s)) return `\"${s}\"`;\n        else return `${s}`;\n      })\n      .join(\",\");\n    this.values.push([this.removeType(column), `in.(${cleanedValues})`]);\n  }\n\n  /**\n   * Adds a 'contains' clause to the values array.\n   * @param column The column to apply the 'contains' clause to.\n   * @param value The value for the 'contains' clause.\n   */\n  addContainsClause(column: string, value: unknown) {\n    if (typeof value === \"string\") {\n      this.values.push([this.removeType(column), `cs.${value}`]);\n    } else if (Array.isArray(value)) {\n      this.values.push([this.removeType(column), `cs.{${value.join(\",\")}}`]);\n    } else {\n      this.values.push([\n        this.removeType(column),\n        `cs.${JSON.stringify(value)}`,\n      ]);\n    }\n  }\n\n  /**\n   * Adds a 'textSearch' clause to the values array.\n   * @param column The column to apply the 'textSearch' clause to.\n   * @param query The query for the 'textSearch' clause.\n   * @param config Optional configuration for the 'textSearch' clause.\n   * @param type Optional type for the 'textSearch' clause.\n   */\n  addTextSearchClause(\n    column: string,\n    query: string[],\n    {\n      config,\n      type,\n    }: { config?: string; type?: \"plain\" | \"phrase\" | \"websearch\" } = {}\n  ) {\n    let typePart = \"\";\n    if (type === \"plain\") {\n      typePart = \"pl\";\n    } else if (type === \"phrase\") {\n      typePart = \"ph\";\n    } else if (type === \"websearch\") {\n      typePart = \"w\";\n    }\n    const configPart = config === undefined ? \"\" : `(${config})`;\n    this.values.push([\n      this.removeType(column),\n      `${typePart}fts${configPart}.${query}`,\n    ]);\n  }\n\n  /**\n   * Adds a 'not' clause to the values array.\n   * @param column The column to apply the 'not' clause to.\n   * @param operator The operator for the 'not' clause.\n   * @param value The value for the 'not' clause.\n   */\n  addNotClause(column: string, operator: string, value: unknown) {\n    this.values.push([column, `not.${operator}.${value}`]);\n  }\n\n  /**\n   * Adds a 'match' clause to the values array.\n   * @param query The query for the 'match' clause.\n   */\n  addMatchClause(query: Record<string, unknown>) {\n    Object.entries(query).forEach(([column, value]) => {\n      this.values.push([column, `eq.${value}`]);\n    });\n  }\n\n  /**\n   * Returns the flattened parameters as a string.\n   * @returns The flattened parameters as a string.\n   */\n  flattenedParams() {\n    const mapped = this.values.map(([k, v]) => `${k}.${v}`);\n    if (mapped.length === 1) return mapped[0];\n    return `and(${mapped.join(\",\")})`;\n  }\n\n  /**\n   * Gets flattened parameters from a `SupabaseFilter` and a\n   * `SupabaseFilterRPCCall`.\n   * @param rpc The `SupabaseFilter` object.\n   * @param filter The `SupabaseFilterRPCCall` object.\n   * @returns The flattened parameters as a string.\n   */\n  static getFlattenedParams(\n    rpc: SupabaseFilter,\n    filter: SupabaseFilterRPCCall\n  ) {\n    const proxiedDuplicator = new ProxyParamsDuplicator();\n    const proxiedRpc = new Proxy(rpc, proxiedDuplicator.buildProxyHandler());\n    void filter(proxiedRpc);\n    return proxiedDuplicator.flattenedParams();\n  }\n}\n\n/**\n * Converts a `SupabaseMetadata` object into a `StructuredQuery` object.\n * The function creates a new `StructuredQuery` object and uses the\n * `Operation` and `Comparison` classes to build the query.\n */\nexport \nfunction convertObjectFilterToStructuredQuery(\n  objFilter: SupabaseMetadata\n): StructuredQuery {\n  return new StructuredQuery(\n    \"\",\n    new Operation(\n      Operators.and,\n      Object.entries(objFilter).map(\n        ([column, value]) => new Comparison(Comparators.eq, column, value)\n      )\n    )\n  );\n}\n\n// Path: langchain/src/chains/query_constructor/parser.ts\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  FilterDirective,\n  Operation,\n  Operator,\n  Operators,\n} from \"./ir.js\";\nimport {\n  CallExpressionType,\n  ExpressionParser,\n  ParsedType,\n} from \"../../output_parsers/expression.js\";\n\n/**\n * A type representing the possible types that can be traversed in an\n * expression.\n */\nexport type TraverseType =\n  | boolean\n  | Operation\n  | Comparison\n  | string\n  | number\n  | { [key: string]: TraverseType }\n  | TraverseType[];\n\n/**\n * A class for transforming and parsing query expressions.\n */\nexport class QueryTransformer {\n  constructor(\n    public allowedComparators: Comparator[] = [],\n    public allowedOperators: Operator[] = []\n  ) {}\n\n  /**\n   * Matches a function name to a comparator or operator. Throws an error if\n   * the function name is unknown or not allowed.\n   * @param funcName The function name to match.\n   * @returns The matched function name.\n   */\n  private matchFunctionName(funcName: string) {\n    if (funcName in Comparators) {\n      if (this.allowedComparators.length > 0) {\n        if (this.allowedComparators.includes(funcName as Comparator)) {\n          return funcName;\n        } else {\n          throw new Error(\"Received comparator not allowed\");\n        }\n      } else {\n        return funcName;\n      }\n    }\n    if (funcName in Operators) {\n      if (this.allowedOperators.length > 0) {\n        if (this.allowedOperators.includes(funcName as Operator)) {\n          return funcName;\n        } else {\n          throw new Error(\"Received operator not allowed\");\n        }\n      } else {\n        return funcName;\n      }\n    }\n    throw new Error(\"Unknown function name\");\n  }\n\n  /**\n   * Transforms a parsed expression into an operation or comparison. Throws\n   * an error if the parsed expression is not supported.\n   * @param parsed The parsed expression to transform.\n   * @returns The transformed operation or comparison.\n   */\n  private transform(parsed: CallExpressionType): Operation | Comparison {\n    const traverse = (node: ParsedType): TraverseType => {\n      switch (node.type) {\n        case \"call_expression\": {\n          if (typeof node.funcCall !== \"string\") {\n            throw new Error(\n              \"Property access expression and element access expression not supported\"\n            );\n          }\n          const funcName = this.matchFunctionName(node.funcCall);\n          if (funcName in Operators) {\n            return new Operation(\n              funcName as Operator,\n              node.args?.map((arg) => traverse(arg)) as FilterDirective[]\n            );\n          }\n          if (funcName in Comparators) {\n            if (node.args && node.args.length === 2) {\n              return new Comparison(\n                funcName as Comparator,\n                traverse(node.args[0]) as string,\n                traverse(node.args[1]) as string | number\n              );\n            }\n            throw new Error(\"Comparator must have exactly 2 arguments\");\n          }\n          throw new Error(\"Function name neither operator nor comparator\");\n        }\n        case \"string_literal\": {\n          return node.value;\n        }\n        case \"numeric_literal\": {\n          return node.value;\n        }\n        case \"array_literal\": {\n          return node.values.map((value) => traverse(value));\n        }\n        case \"object_literal\": {\n          return node.values.reduce((acc, value) => {\n            acc[value.identifier] = traverse(value.value);\n            return acc;\n          }, {} as { [key: string]: TraverseType });\n        }\n        case \"boolean_literal\": {\n          return node.value;\n        }\n        default: {\n          throw new Error(\"Unknown node type\");\n        }\n      }\n    };\n    return traverse(parsed) as Operation | Comparison;\n  }\n\n  /**\n   * Parses an expression and returns the transformed operation or\n   * comparison. Throws an error if the expression cannot be parsed.\n   * @param expression The expression to parse.\n   * @returns A Promise that resolves to the transformed operation or comparison.\n   */\n  async parse(expression: string): Promise<Operation | Comparison> {\n    const expressionParser = new ExpressionParser();\n    const parsed = (await expressionParser.parse(\n      expression\n    )) as CallExpressionType;\n    if (!parsed) {\n      throw new Error(\"Could not parse expression\");\n    }\n    return this.transform(parsed);\n  }\n}\n\n// Path: langchain/src/chains/query_constructor/prompt.ts\nimport { PromptTemplate } from \"@langchain/core/prompts\";\n\nexport const SONG_DATA_SOURCE = `\\\n\\`\\`\\`json\n{\n    \"content\": \"Lyrics of a song\",\n    \"attributes\": {\n        \"artist\": {\n            \"type\": \"string\",\n            \"description\": \"Name of the song artist\"\n        },\n        \"length\": {\n            \"type\": \"integer\",\n            \"description\": \"Length of the song in seconds\"\n        },\n        \"genre\": {\n            \"type\": \"string\",\n            \"description\": \"The song genre, one of 'pop', 'rock' or 'rap'\"\n        }\n    }\n}\n\\`\\`\\`\\\n`\n  .replaceAll(\"{\", \"{{\")\n  .replaceAll(\"}\", \"}}\");\n\nexport const FULL_ANSWER = `\\\n\\`\\`\\`json\n{{\n    \"query\": \"teenager love\",\n    \"filter\": \"and(or(eq(\\\\\"artist\\\\\", \\\\\"Taylor Swift\\\\\"), eq(\\\\\"artist\\\\\", \\\\\"Katy Perry\\\\\")), \\\nlt(\\\\\"length\\\\\", 180), eq(\\\\\"genre\\\\\", \\\\\"pop\\\\\"))\"\n}}`;\n\nexport const NO_FILTER_ANSWER = `\\\n\\`\\`\\`json\n{{\n    \"query\": \"\",\n    \"filter\": \"NO_FILTER\"\n}}\n\\`\\`\\`\\\n`;\n\nexport const DEFAULT_EXAMPLES = [\n  {\n    i: \"1\",\n    data_source: SONG_DATA_SOURCE,\n    user_query:\n      \"What are songs by Taylor Swift or Katy Perry about teenage romance under 3 minutes long in the dance pop genre\",\n    structured_request: FULL_ANSWER,\n  },\n  {\n    i: \"2\",\n    data_source: SONG_DATA_SOURCE,\n    user_query: \"What are songs that were not published on Spotify\",\n    structured_request: NO_FILTER_ANSWER,\n  },\n];\n\nexport const EXAMPLE_PROMPT_TEMPLATE = `\\\n<< Example {i}. >>\nData Source:\n{data_source}\n\nUser Query:\n{user_query}\n\nStructured Request:\n{structured_request}\n`;\n\nexport const EXAMPLE_PROMPT = /* #__PURE__ */ new PromptTemplate({\n  inputVariables: [\"i\", \"data_source\", \"user_query\", \"structured_request\"],\n  template: EXAMPLE_PROMPT_TEMPLATE,\n});\n\nexport const DEFAULT_SCHEMA = `\\\n<< Structured Request Schema >>\nWhen responding use a markdown code snippet with a JSON object formatted in the \\\nfollowing schema:\n\n\\`\\`\\`json\n{{{{\n    \"query\": string \\\\ text string to compare to document contents\n    \"filter\": string \\\\ logical condition statement for filtering documents\n}}}}\n\\`\\`\\`\n\nThe query string should contain only text that is expected to match the contents of \\\ndocuments. Any conditions in the filter should not be mentioned in the query as well.\n\nA logical condition statement is composed of one or more comparison and logical \\\noperation statements.\n\nA comparison statement takes the form: \\`comp(attr, val)\\`:\n- \\`comp\\` ({allowed_comparators}): comparator\n- \\`attr\\` (string):  name of attribute to apply the comparison to\n- \\`val\\` (string): is the comparison value\n\nA logical operation statement takes the form \\`op(statement1, statement2, ...)\\`:\n- \\`op\\` ({allowed_operators}): logical operator\n- \\`statement1\\`, \\`statement2\\`, ... (comparison statements or logical operation \\\nstatements): one or more statements to apply the operation to\n\nMake sure that you only use the comparators and logical operators listed above and \\\nno others.\nMake sure that filters only refer to attributes that exist in the data source.\nMake sure that filters only use the attributed names with its function names if there are functions applied on them.\nMake sure that filters only use format \\`YYYY-MM-DD\\` when handling timestamp data typed values.\nMake sure that filters take into account the descriptions of attributes and only make \\\ncomparisons that are feasible given the type of data being stored.\nMake sure that filters are only used as needed. If there are no filters that should be \\\napplied return \"NO_FILTER\" for the filter value.\\\n`;\n\nexport const DEFAULT_PREFIX = `\\\nYour goal is to structure the user's query to match the request schema provided below.\n\n{schema}\\\n`;\n\nexport const DEFAULT_SUFFIX = `\\\n<< Example {i}. >>\nData Source:\n\\`\\`\\`json\n{{{{\n    \"content\": \"{content}\",\n    \"attributes\": {attributes}\n}}}}\n\\`\\`\\`\n\nUser Query:\n{{query}}\n\nStructured Request:\n`;\n\n// Path: langchain/src/chains/query_constructor/index.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { z } from \"zod\";\nimport {\n  Example,\n  interpolateFString,\n  FewShotPromptTemplate,\n} from \"@langchain/core/prompts\";\nimport { InputValues } from \"@langchain/core/utils/types\";\nimport { QueryTransformer, TraverseType } from \"./parser.js\";\nimport {\n  Comparator,\n  Comparators,\n  Operator,\n  Operators,\n  StructuredQuery,\n} from \"./ir.js\";\nimport {\n  DEFAULT_EXAMPLES,\n  DEFAULT_PREFIX,\n  DEFAULT_SCHEMA,\n  DEFAULT_SUFFIX,\n  EXAMPLE_PROMPT,\n} from \"./prompt.js\";\nimport { LLMChain } from \"../llm_chain.js\";\nimport { AsymmetricStructuredOutputParser } from \"../../output_parsers/structured.js\";\nimport { AttributeInfo } from \"../../schema/query_constructor.js\";\n\nexport { QueryTransformer, type TraverseType };\nexport {\n  DEFAULT_EXAMPLES,\n  DEFAULT_PREFIX,\n  DEFAULT_SCHEMA,\n  DEFAULT_SUFFIX,\n  EXAMPLE_PROMPT,\n};\n\nconst queryInputSchema = /* #__PURE__ */ z.object({\n  query: /* #__PURE__ */ z\n    .string()\n    .describe(\"text string to compare to document contents\"),\n  filter: /* #__PURE__ */ z\n    .string()\n    .optional()\n    .describe(\"logical condition statement for filtering documents\"),\n});\n\n/**\n * A class that extends AsymmetricStructuredOutputParser to parse\n * structured query output.\n */\nexport class StructuredQueryOutputParser extends AsymmetricStructuredOutputParser<\n  typeof queryInputSchema,\n  StructuredQuery\n> {\n  lc_namespace = [\"langchain\", \"chains\", \"query_constructor\"];\n\n  queryTransformer: QueryTransformer;\n\n  constructor(fields: {\n    allowedComparators: Comparator[];\n    allowedOperators: Operator[];\n  }) {\n    super({ ...fields, inputSchema: queryInputSchema });\n\n    const { allowedComparators, allowedOperators } = fields;\n    this.queryTransformer = new QueryTransformer(\n      allowedComparators,\n      allowedOperators\n    );\n  }\n\n  /**\n   * Processes the output of a structured query.\n   * @param query The query string.\n   * @param filter The filter condition.\n   * @returns A Promise that resolves to a StructuredQuery instance.\n   */\n  async outputProcessor({\n    query,\n    filter,\n  }: z.infer<typeof queryInputSchema>): Promise<StructuredQuery> {\n    let myQuery = query;\n    if (myQuery.length === 0) {\n      myQuery = \" \";\n    }\n    if (filter === \"NO_FILTER\" || filter === undefined) {\n      return new StructuredQuery(query);\n    } else {\n      const parsedFilter = await this.queryTransformer.parse(filter);\n      return new StructuredQuery(query, parsedFilter);\n    }\n  }\n\n  /**\n   * Creates a new StructuredQueryOutputParser instance from the provided\n   * components.\n   * @param allowedComparators An array of allowed Comparator instances.\n   * @param allowedOperators An array of allowed Operator instances.\n   * @returns A new StructuredQueryOutputParser instance.\n   */\n  static fromComponents(\n    allowedComparators: Comparator[] = [],\n    allowedOperators: Operator[] = []\n  ) {\n    return new StructuredQueryOutputParser({\n      allowedComparators,\n      allowedOperators,\n    });\n  }\n}\n\nexport function formatAttributeInfo(info: AttributeInfo[]) {\n  const infoObj = info.reduce((acc, attr) => {\n    acc[attr.name] = {\n      type: attr.type,\n      description: attr.description,\n    };\n    return acc;\n  }, {} as { [name: string]: { type: string; description: string } });\n\n  return JSON.stringify(infoObj, null, 2)\n    .replaceAll(\"{\", \"{{\")\n    .replaceAll(\"}\", \"}}\");\n}\n\nconst defaultExample = DEFAULT_EXAMPLES.map((EXAMPLE) => EXAMPLE as Example);\n\nfunction _getPrompt(\n  documentContents: string,\n  attributeInfo: AttributeInfo[],\n  allowedComparators?: Comparator[],\n  allowedOperators?: Operator[],\n  examples: InputValues[] = defaultExample\n) {\n  const myAllowedComparators: Comparator[] =\n    allowedComparators ?? Object.values(Comparators);\n  const myAllowedOperators: Operator[] =\n    allowedOperators ?? Object.values(Operators);\n  const attributeJSON = formatAttributeInfo(attributeInfo);\n  const schema = interpolateFString(DEFAULT_SCHEMA, {\n    allowed_comparators: myAllowedComparators.join(\" | \"),\n    allowed_operators: myAllowedOperators.join(\" | \"),\n  });\n  const prefix = interpolateFString(DEFAULT_PREFIX, {\n    schema,\n  });\n  const suffix = interpolateFString(DEFAULT_SUFFIX, {\n    i: examples.length + 1,\n    content: documentContents,\n    attributes: attributeJSON,\n  });\n\n  const outputParser = StructuredQueryOutputParser.fromComponents(\n    allowedComparators,\n    allowedOperators\n  );\n\n  return new FewShotPromptTemplate({\n    examples,\n    examplePrompt: EXAMPLE_PROMPT,\n    inputVariables: [\"query\"],\n    suffix,\n    prefix,\n    outputParser,\n  });\n}\n\n/**\n * A type that represents options for the query constructor chain.\n */\nexport type QueryConstructorChainOptions = {\n  llm: BaseLanguageModelInterface;\n  documentContents: string;\n  attributeInfo: AttributeInfo[];\n  examples?: InputValues[];\n  allowedComparators?: Comparator[];\n  allowedOperators?: Operator[];\n};\n\nexport function loadQueryConstructorChain(opts: QueryConstructorChainOptions) {\n  const prompt = _getPrompt(\n    opts.documentContents,\n    opts.attributeInfo,\n    opts.allowedComparators,\n    opts.allowedOperators,\n    opts.examples\n  );\n  return new LLMChain({\n    llm: opts.llm,\n    prompt,\n  });\n}\n\n// Path: langchain/src/retrievers/self_query/functional.ts\nimport { Document } from \"@langchain/core/documents\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { castValue, isFilterEmpty } from \"./utils.js\";\n\n/**\n * A type alias for an object that maps comparison operators to string or\n * number values. This is used in the comparison functions to determine\n * the result of a comparison operation.\n */\ntype ValueType = {\n  eq: string | number;\n  ne: string | number;\n  lt: string | number;\n  lte: string | number;\n  gt: string | number;\n  gte: string | number;\n};\n\n/**\n * A type alias for a function that takes a `Document` as an argument and\n * returns a boolean. This function is used as a filter for documents.\n */\nexport type FunctionFilter = (document: Document) => boolean;\n\n/**\n * A class that extends `BaseTranslator` to translate structured queries\n * into functional filters.\n * @example\n * ```typescript\n * const functionalTranslator = new FunctionalTranslator();\n * const relevantDocuments = await functionalTranslator.getRelevantDocuments(\n *   \"Which movies are rated higher than 8.5?\",\n * );\n * ```\n */\nexport class FunctionalTranslator extends BaseTranslator {\n  declare VisitOperationOutput: FunctionFilter;\n\n  declare VisitComparisonOutput: FunctionFilter;\n\n  declare VisitStructuredQueryOutput:\n    | { filter: FunctionFilter }\n    | { [k: string]: never };\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.gt,\n    Comparators.gte,\n    Comparators.lt,\n    Comparators.lte,\n  ];\n\n  formatFunction(): string {\n    throw new Error(\"Not implemented\");\n  }\n\n  /**\n   * Returns a function that performs a comparison based on the provided\n   * comparator.\n   * @param comparator The comparator to base the comparison function on.\n   * @returns A function that takes two arguments and returns a boolean based on the comparison.\n   */\n  getComparatorFunction<C extends Comparator>(\n    comparator: Comparator\n  ): (a: string | number, b: ValueType[C]) => boolean {\n    switch (comparator) {\n      case Comparators.eq: {\n        return (a: string | number, b: ValueType[C]) => a === b;\n      }\n      case Comparators.ne: {\n        return (a: string | number, b: ValueType[C]) => a !== b;\n      }\n      case Comparators.gt: {\n        return (a: string | number, b: ValueType[C]) => a > b;\n      }\n      case Comparators.gte: {\n        return (a: string | number, b: ValueType[C]) => a >= b;\n      }\n      case Comparators.lt: {\n        return (a: string | number, b: ValueType[C]) => a < b;\n      }\n      case Comparators.lte: {\n        return (a: string | number, b: ValueType[C]) => a <= b;\n      }\n      default: {\n        throw new Error(\"Unknown comparator\");\n      }\n    }\n  }\n\n  /**\n   * Returns a function that performs an operation based on the provided\n   * operator.\n   * @param operator The operator to base the operation function on.\n   * @returns A function that takes two boolean arguments and returns a boolean based on the operation.\n   */\n  getOperatorFunction(operator: Operator): (a: boolean, b: boolean) => boolean {\n    switch (operator) {\n      case Operators.and: {\n        return (a, b) => a && b;\n      }\n      case Operators.or: {\n        return (a, b) => a || b;\n      }\n      default: {\n        throw new Error(\"Unknown operator\");\n      }\n    }\n  }\n\n  /**\n   * Visits the operation part of a structured query and translates it into\n   * a functional filter.\n   * @param operation The operation part of a structured query.\n   * @returns A function that takes a `Document` as an argument and returns a boolean based on the operation.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const { operator, args } = operation;\n    if (this.allowedOperators.includes(operator)) {\n      const operatorFunction = this.getOperatorFunction(operator);\n      return (document: Document) => {\n        if (!args) {\n          return true;\n        }\n\n        return args.reduce((acc, arg) => {\n          const result = arg.accept(this);\n          if (typeof result === \"function\") {\n            return operatorFunction(acc, result(document));\n          } else {\n            throw new Error(\"Filter is not a function\");\n          }\n        }, true);\n      };\n    } else {\n      throw new Error(\"Operator not allowed\");\n    }\n  }\n\n  /**\n   * Visits the comparison part of a structured query and translates it into\n   * a functional filter.\n   * @param comparison The comparison part of a structured query.\n   * @returns A function that takes a `Document` as an argument and returns a boolean based on the comparison.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    const { comparator, attribute, value } = comparison;\n    const undefinedTrue = [Comparators.ne];\n    if (this.allowedComparators.includes(comparator)) {\n      const comparatorFunction = this.getComparatorFunction(comparator);\n      return (document: Document) => {\n        const documentValue = document.metadata[attribute];\n        if (documentValue === undefined) {\n          if (undefinedTrue.includes(comparator)) {\n            return true;\n          }\n          return false;\n        }\n        return comparatorFunction(documentValue, castValue(value));\n      };\n    } else {\n      throw new Error(\"Comparator not allowed\");\n    }\n  }\n\n  /**\n   * Visits a structured query and translates it into a functional filter.\n   * @param query The structured query to translate.\n   * @returns An object containing a `filter` property, which is a function that takes a `Document` as an argument and returns a boolean based on the structured query.\n   */\n  visitStructuredQuery(\n    query: StructuredQuery\n  ): this[\"VisitStructuredQueryOutput\"] {\n    if (!query.filter) {\n      return {};\n    }\n    const filterFunction = query.filter?.accept(this);\n    if (typeof filterFunction !== \"function\") {\n      throw new Error(\"Structured query filter is not a function\");\n    }\n    return { filter: filterFunction as FunctionFilter };\n  }\n\n  /**\n   * Merges two filters into one, based on the specified merge type.\n   * @param defaultFilter The default filter function.\n   * @param generatedFilter The generated filter function.\n   * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'. Default is 'and'.\n   * @returns A function that takes a `Document` as an argument and returns a boolean based on the merged filters, or `undefined` if both filters are empty.\n   */\n  mergeFilters(\n    defaultFilter: FunctionFilter,\n    generatedFilter: FunctionFilter,\n    mergeType = \"and\"\n  ): FunctionFilter | undefined {\n    if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {\n      return undefined;\n    }\n    if (isFilterEmpty(defaultFilter) || mergeType === \"replace\") {\n      if (isFilterEmpty(generatedFilter)) {\n        return undefined;\n      }\n      return generatedFilter;\n    }\n    if (isFilterEmpty(generatedFilter)) {\n      if (mergeType === \"and\") {\n        return undefined;\n      }\n      return defaultFilter;\n    }\n\n    if (mergeType === \"and\") {\n      return (document: Document) =>\n        defaultFilter(document) && generatedFilter(document);\n    } else if (mergeType === \"or\") {\n      return (document: Document) =>\n        defaultFilter(document) || generatedFilter(document);\n    } else {\n      throw new Error(\"Unknown merge type\");\n    }\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/index.ts\nimport {\n  BaseRetriever,\n  type BaseRetrieverInput,\n} from \"@langchain/core/retrievers\";\nimport { Document } from \"@langchain/core/documents\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\nimport {\n  QueryConstructorChainOptions,\n  loadQueryConstructorChain,\n} from \"../../chains/query_constructor/index.js\";\nimport { StructuredQuery } from \"../../chains/query_constructor/ir.js\";\nimport { FunctionalTranslator } from \"./functional.js\";\nimport { BaseTranslator, BasicTranslator } from \"./base.js\";\n\nexport { BaseTranslator, BasicTranslator, FunctionalTranslator };\n\n/**\n * Interface for the arguments required to create a SelfQueryRetriever\n * instance. It extends the BaseRetrieverInput interface.\n */\nexport interface SelfQueryRetrieverArgs<T extends VectorStore>\n  extends BaseRetrieverInput {\n  vectorStore: T;\n  structuredQueryTranslator: BaseTranslator<T>;\n  llmChain: LLMChain;\n  verbose?: boolean;\n  useOriginalQuery?: boolean;\n  searchParams?: {\n    k?: number;\n    filter?: T[\"FilterType\"];\n    mergeFiltersOperator?: \"or\" | \"and\" | \"replace\";\n    forceDefaultFilter?: boolean;\n  };\n}\n\n/**\n * Class for question answering over an index. It retrieves relevant\n * documents based on a query. It extends the BaseRetriever class and\n * implements the SelfQueryRetrieverArgs interface.\n * @example\n * ```typescript\n * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({\n *   llm: new ChatOpenAI(),\n *   vectorStore: await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings()),\n *   documentContents: \"Brief summary of a movie\",\n *   attributeInfo: attributeInfo,\n *   structuredQueryTranslator: new FunctionalTranslator(),\n * });\n * const relevantDocuments = await selfQueryRetriever.getRelevantDocuments(\n *   \"Which movies are directed by Greta Gerwig?\",\n * );\n * ```\n */\nexport class SelfQueryRetriever<T extends VectorStore>\n  extends BaseRetriever\n  implements SelfQueryRetrieverArgs<T>\n{\n  static lc_name() {\n    return \"SelfQueryRetriever\";\n  }\n\n  get lc_namespace() {\n    return [\"langchain\", \"retrievers\", \"self_query\"];\n  }\n\n  vectorStore: T;\n\n  llmChain: LLMChain;\n\n  verbose?: boolean;\n\n  structuredQueryTranslator: BaseTranslator<T>;\n\n  useOriginalQuery = false;\n\n  searchParams?: {\n    k?: number;\n    filter?: T[\"FilterType\"];\n    mergeFiltersOperator?: \"or\" | \"and\" | \"replace\";\n    forceDefaultFilter?: boolean;\n  } = { k: 4, forceDefaultFilter: false };\n\n  constructor(options: SelfQueryRetrieverArgs<T>) {\n    super(options);\n    this.vectorStore = options.vectorStore;\n    this.llmChain = options.llmChain;\n    this.verbose = options.verbose ?? false;\n    this.searchParams = options.searchParams ?? this.searchParams;\n    this.useOriginalQuery = options.useOriginalQuery ?? this.useOriginalQuery;\n    this.structuredQueryTranslator = options.structuredQueryTranslator;\n  }\n\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Document<Record<string, unknown>>[]> {\n    const { [this.llmChain.outputKey]: output } = await this.llmChain.call(\n      {\n        [this.llmChain.inputKeys[0]]: query,\n      },\n      runManager?.getChild(\"llm_chain\")\n    );\n\n    const generatedStructuredQuery = output as StructuredQuery;\n\n    const nextArg = this.structuredQueryTranslator.visitStructuredQuery(\n      generatedStructuredQuery\n    );\n\n    const filter = this.structuredQueryTranslator.mergeFilters(\n      this.searchParams?.filter,\n      nextArg.filter,\n      this.searchParams?.mergeFiltersOperator,\n      this.searchParams?.forceDefaultFilter\n    );\n\n    const generatedQuery = generatedStructuredQuery.query;\n    let myQuery = query;\n\n    if (!this.useOriginalQuery && generatedQuery && generatedQuery.length > 0) {\n      myQuery = generatedQuery;\n    }\n\n    return this.vectorStore.similaritySearch(\n      myQuery,\n      this.searchParams?.k,\n      filter,\n      runManager?.getChild(\"vectorstore\")\n    );\n  }\n\n  /**\n   * Static method to create a new SelfQueryRetriever instance from a\n   * BaseLanguageModel and a VectorStore. It first loads a query constructor\n   * chain using the loadQueryConstructorChain function, then creates a new\n   * SelfQueryRetriever instance with the loaded chain and the provided\n   * options.\n   * @param options The options used to create the SelfQueryRetriever instance. It includes the QueryConstructorChainOptions and all the SelfQueryRetrieverArgs except 'llmChain'.\n   * @returns A new instance of SelfQueryRetriever.\n   */\n  static fromLLM<T extends VectorStore>(\n    options: QueryConstructorChainOptions &\n      Omit<SelfQueryRetrieverArgs<T>, \"llmChain\">\n  ): SelfQueryRetriever<T> {\n    const {\n      structuredQueryTranslator,\n      allowedComparators,\n      allowedOperators,\n      llm,\n      documentContents,\n      attributeInfo,\n      examples,\n      vectorStore,\n      ...rest\n    } = options;\n    const llmChain = loadQueryConstructorChain({\n      llm,\n      documentContents,\n      attributeInfo,\n      examples,\n      allowedComparators:\n        allowedComparators ?? structuredQueryTranslator.allowedComparators,\n      allowedOperators:\n        allowedOperators ?? structuredQueryTranslator.allowedOperators,\n    });\n    return new SelfQueryRetriever<T>({\n      ...rest,\n      llmChain,\n      vectorStore,\n      structuredQueryTranslator,\n    });\n  }\n}\n\n// Path: langchain/src/retrievers/self_query/vectara.ts\nimport {\n  VectaraFilter,\n  VectaraStore,\n} from \"@langchain/community/vectorstores/vectara\";\nimport {\n  Comparator,\n  Comparators,\n  Comparison,\n  NOT,\n  Operation,\n  Operator,\n  Operators,\n  StructuredQuery,\n  Visitor,\n} from \"../../chains/query_constructor/ir.js\";\nimport { BaseTranslator } from \"./base.js\";\nimport { isFilterEmpty } from \"./utils.js\";\n\ntype AllowedOperator = Exclude<Operator, NOT>;\n\nexport type VectaraVisitorResult =\n  | VectaraOperationResult\n  | VectaraComparisonResult\n  | VectaraVisitorStructuredQueryResult;\n// eslint-disable-next-line @typescript-eslint/ban-types\nexport type VectaraOperationResult = String;\n// eslint-disable-next-line @typescript-eslint/ban-types\nexport type VectaraComparisonResult = String;\nexport type VectaraVisitorStructuredQueryResult = {\n  filter?: { filter?: VectaraOperationResult | VectaraComparisonResult };\n};\n\ntype Value = number | string;\nfunction processValue(value: Value): string {\n  /** Convert a value to a string and add single quotes if it is a string. */\n  if (typeof value === \"string\") {\n    return `'${value}'`;\n  } else {\n    return String(value);\n  }\n}\n\nexport class VectaraTranslator<\n  T extends VectaraStore\n> extends BaseTranslator<T> {\n  declare VisitOperationOutput: VectaraOperationResult;\n\n  declare VisitComparisonOutput: VectaraComparisonResult;\n\n  allowedOperators: Operator[] = [Operators.and, Operators.or];\n\n  allowedComparators: Comparator[] = [\n    Comparators.eq,\n    Comparators.ne,\n    Comparators.lt,\n    Comparators.lte,\n    Comparators.gt,\n    Comparators.gte,\n  ];\n\n  formatFunction(func: Operator | Comparator): string {\n    if (func in Comparators) {\n      if (\n        this.allowedComparators.length > 0 &&\n        this.allowedComparators.indexOf(func as Comparator) === -1\n      ) {\n        throw new Error(\n          `Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else if (func in Operators) {\n      if (\n        this.allowedOperators.length > 0 &&\n        this.allowedOperators.indexOf(func as Operator) === -1\n      ) {\n        throw new Error(\n          `Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(\n            \", \"\n          )}`\n        );\n      }\n    } else {\n      throw new Error(\"Unknown comparator or operator\");\n    }\n\n    const mapDict = {\n      and: \" and \",\n      or: \" or \",\n      eq: \"=\",\n      ne: \"!=\",\n      lt: \"<\",\n      lte: \"<=\",\n      gt: \">\",\n      gte: \">=\",\n    };\n    return mapDict[func as Comparator | AllowedOperator];\n  }\n\n  /**\n   * Visits an operation and returns a VectaraOperationResult. The\n   * operation's arguments are visited and the operator is formatted.\n   * @param operation The operation to visit.\n   * @returns A VectaraOperationResult.\n   */\n  visitOperation(operation: Operation): this[\"VisitOperationOutput\"] {\n    const args = operation.args?.map((arg) =>\n      arg.accept(this as Visitor)\n    ) as VectaraVisitorResult[];\n    const operator = this.formatFunction(operation.operator);\n    return `( ${args.join(operator)} )`;\n  }\n\n  /**\n   * Visits a comparison and returns a VectaraComparisonResult. The\n   * comparison's value is checked for type and the comparator is formatted.\n   * Throws an error if the value type is not supported.\n   * @param comparison The comparison to visit.\n   * @returns A VectaraComparisonResult.\n   */\n  visitComparison(comparison: Comparison): this[\"VisitComparisonOutput\"] {\n    const comparator = this.formatFunction(comparison.comparator);\n...\n", "needle_token_start": 7206, "needle_token_end": 7296, "code_context_ntokens": 16109, "output": ["```typescript\n/**\n * Converts a metadata object into a structured query format suitable for database operations.\n * @param objFilter - The metadata object where keys are column names and values are filter criteria.\n * @returns A structured query object encapsulating the filtering logic.\n */\nexport function convertObjectFilterToStructuredQuery(\n  objFilter: SupabaseMetadata\n): StructuredQuery {\n  return new StructuredQuery(\n    \"\",\n    new Operation(\n      Operators.and,\n      Object.entries(objFilter).map(\n        ([column, value]) => new Comparison(Comparators.eq, column, value)\n      )\n    )\n  );\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "createOpenAIFunctionsAgent", "language": "typescript", "path": "langchain/src/agents/openai_functions/index.ts", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To construct an agent that utilizes a language model capable of OpenAI-style function calling, integrating various tools to enhance its capabilities.\n2. **Input**: Parameters include a language model, a set of tools, a structured prompt template, and an optional flag to enable streaming of intermediate steps.\n3. **Output**: A runnable sequence representing an agent that processes input variables as defined in the prompt and returns actions or final results based on the interaction.\n4. **Procedure**: The function first validates the prompt's input requirements. It then binds the provided tools to the language model, creating a sequence of operations that includes formatting input, processing it through the model, and parsing the output to generate actionable responses or conclusions.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/agents/executor.ts\nimport {\n  type StructuredToolInterface,\n  type ToolInterface,\n  ToolInputParsingException,\n  Tool,\n} from \"@langchain/core/tools\";\nimport {\n  Runnable,\n  type RunnableConfig,\n  patchConfig,\n} from \"@langchain/core/runnables\";\nimport { AgentAction, AgentFinish, AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  CallbackManager,\n  CallbackManagerForChainRun,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { OutputParserException } from \"@langchain/core/output_parsers\";\nimport { Serializable } from \"@langchain/core/load/serializable\";\nimport { SerializedLLMChain } from \"../chains/serde.js\";\nimport { StoppingMethod } from \"./types.js\";\nimport {\n  AgentRunnableSequence,\n  BaseMultiActionAgent,\n  BaseSingleActionAgent,\n  RunnableMultiActionAgent,\n  RunnableSingleActionAgent,\n  isRunnableAgent,\n} from \"./agent.js\";\nimport { BaseChain, ChainInputs } from \"../chains/base.js\";\n\ninterface AgentExecutorIteratorInput {\n  agentExecutor: AgentExecutor;\n  inputs: Record<string, string>;\n  config?: RunnableConfig;\n  /** @deprecated Use \"config\" */\n  callbacks?: Callbacks;\n  /** @deprecated Use \"config\" */\n  tags?: string[];\n  /** @deprecated Use \"config\" */\n  metadata?: Record<string, unknown>;\n  runName?: string;\n  runManager?: CallbackManagerForChainRun;\n}\n\nexport class AgentExecutorIterator\n  extends Serializable\n  implements AgentExecutorIteratorInput\n{\n  lc_namespace = [\"langchain\", \"agents\", \"executor_iterator\"];\n\n  agentExecutor: AgentExecutor;\n\n  inputs: Record<string, string>;\n\n  config?: RunnableConfig;\n\n  /** @deprecated Use \"config\" */\n  callbacks?: Callbacks;\n\n  /** @deprecated Use \"config\" */\n  tags: string[] | undefined;\n\n  /** @deprecated Use \"config\" */\n  metadata: Record<string, unknown> | undefined;\n\n  /** @deprecated Use \"config\" */\n  runName: string | undefined;\n\n  private _finalOutputs: Record<string, unknown> | undefined;\n\n  get finalOutputs(): Record<string, unknown> | undefined {\n    return this._finalOutputs;\n  }\n\n  /** Intended to be used as a setter method, needs to be async. */\n  async setFinalOutputs(value: Record<string, unknown> | undefined) {\n    this._finalOutputs = undefined;\n    if (value) {\n      const preparedOutputs: Record<string, unknown> =\n        await this.agentExecutor.prepOutputs(this.inputs, value, true);\n      this._finalOutputs = preparedOutputs;\n    }\n  }\n\n  runManager: CallbackManagerForChainRun | undefined;\n\n  intermediateSteps: AgentStep[] = [];\n\n  iterations = 0;\n\n  get nameToToolMap(): Record<string, ToolInterface> {\n    const toolMap = this.agentExecutor.tools.map((tool) => ({\n      [tool.name]: tool,\n    }));\n    return Object.assign({}, ...toolMap);\n  }\n\n  constructor(fields: AgentExecutorIteratorInput) {\n    super(fields);\n    this.agentExecutor = fields.agentExecutor;\n    this.inputs = fields.inputs;\n    this.callbacks = fields.callbacks;\n    this.tags = fields.tags;\n    this.metadata = fields.metadata;\n    this.runName = fields.runName;\n    this.runManager = fields.runManager;\n    this.config = fields.config;\n  }\n\n  /**\n   * Reset the iterator to its initial state, clearing intermediate steps,\n   * iterations, and the final output.\n   */\n  reset(): void {\n    this.intermediateSteps = [];\n    this.iterations = 0;\n    this._finalOutputs = undefined;\n  }\n\n  updateIterations(): void {\n    this.iterations += 1;\n  }\n\n  async *streamIterator() {\n    this.reset();\n\n    // Loop to handle iteration\n    while (true) {\n      try {\n        if (this.iterations === 0) {\n          await this.onFirstStep();\n        }\n\n        const result = await this._callNext();\n        yield result;\n        // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      } catch (e: any) {\n        if (\n          \"message\" in e &&\n          e.message.startsWith(\"Final outputs already reached: \")\n        ) {\n          if (!this.finalOutputs) {\n            throw e;\n          }\n          return this.finalOutputs;\n        }\n        if (this.runManager) {\n          await this.runManager.handleChainError(e);\n        }\n        throw e;\n      }\n    }\n  }\n\n  /**\n   * Perform any necessary setup for the first step\n   * of the asynchronous iterator.\n   */\n  async onFirstStep(): Promise<void> {\n    if (this.iterations === 0) {\n      const callbackManager = await CallbackManager.configure(\n        this.callbacks,\n        this.agentExecutor.callbacks,\n        this.tags,\n        this.agentExecutor.tags,\n        this.metadata,\n        this.agentExecutor.metadata,\n        {\n          verbose: this.agentExecutor.verbose,\n        }\n      );\n      this.runManager = await callbackManager?.handleChainStart(\n        this.agentExecutor.toJSON(),\n        this.inputs,\n        undefined,\n        undefined,\n        this.tags,\n        this.metadata,\n        this.runName\n      );\n    }\n  }\n\n  /**\n   * Execute the next step in the chain using the\n   * AgentExecutor's _takeNextStep method.\n   */\n  async _executeNextStep(\n    runManager?: CallbackManagerForChainRun\n  ): Promise<AgentFinish | AgentStep[]> {\n    return this.agentExecutor._takeNextStep(\n      this.nameToToolMap,\n      this.inputs,\n      this.intermediateSteps,\n      runManager,\n      this.config\n    );\n  }\n\n  /**\n   * Process the output of the next step,\n   * handling AgentFinish and tool return cases.\n   */\n  async _processNextStepOutput(\n    nextStepOutput: AgentFinish | AgentStep[],\n    runManager?: CallbackManagerForChainRun\n  ): Promise<Record<string, string | AgentStep[]>> {\n    if (\"returnValues\" in nextStepOutput) {\n      const output = await this.agentExecutor._return(\n        nextStepOutput as AgentFinish,\n        this.intermediateSteps,\n        runManager\n      );\n      if (this.runManager) {\n        await this.runManager.handleChainEnd(output);\n      }\n      await this.setFinalOutputs(output);\n      return output;\n    }\n\n    this.intermediateSteps = this.intermediateSteps.concat(\n      nextStepOutput as AgentStep[]\n    );\n\n    let output: Record<string, string | AgentStep[]> = {};\n    if (Array.isArray(nextStepOutput) && nextStepOutput.length === 1) {\n      const nextStep = nextStepOutput[0];\n      const toolReturn = await this.agentExecutor._getToolReturn(nextStep);\n      if (toolReturn) {\n        output = await this.agentExecutor._return(\n          toolReturn,\n          this.intermediateSteps,\n          runManager\n        );\n        if (this.runManager) {\n          await this.runManager.handleChainEnd(output);\n        }\n        await this.setFinalOutputs(output);\n      }\n    }\n    output = { intermediateSteps: nextStepOutput as AgentStep[] };\n    return output;\n  }\n\n  async _stop(): Promise<Record<string, unknown>> {\n    const output = await this.agentExecutor.agent.returnStoppedResponse(\n      this.agentExecutor.earlyStoppingMethod,\n      this.intermediateSteps,\n      this.inputs\n    );\n    const returnedOutput = await this.agentExecutor._return(\n      output,\n      this.intermediateSteps,\n      this.runManager\n    );\n    await this.setFinalOutputs(returnedOutput);\n    return returnedOutput;\n  }\n\n  async _callNext(): Promise<Record<string, unknown>> {\n    // final output already reached: stopiteration (final output)\n    if (this.finalOutputs) {\n      throw new Error(\n        `Final outputs already reached: ${JSON.stringify(\n          this.finalOutputs,\n          null,\n          2\n        )}`\n      );\n    }\n    // timeout/max iterations: stopiteration (stopped response)\n    if (!this.agentExecutor.shouldContinueGetter(this.iterations)) {\n      return this._stop();\n    }\n    const nextStepOutput = await this._executeNextStep(this.runManager);\n    const output = await this._processNextStepOutput(\n      nextStepOutput,\n      this.runManager\n    );\n    this.updateIterations();\n    return output;\n  }\n}\n\ntype ExtractToolType<T> = T extends { ToolType: infer ToolInterface }\n  ? ToolInterface\n  : StructuredToolInterface;\n\n/**\n * Interface defining the structure of input data for creating an\n * AgentExecutor. It extends ChainInputs and includes additional\n * properties specific to agent execution.\n */\nexport interface AgentExecutorInput extends ChainInputs {\n  agent:\n    | BaseSingleActionAgent\n    | BaseMultiActionAgent\n    | Runnable<\n        ChainValues & { steps?: AgentStep[] },\n        AgentAction[] | AgentAction | AgentFinish\n      >;\n  tools: ExtractToolType<this[\"agent\"]>[];\n  returnIntermediateSteps?: boolean;\n  maxIterations?: number;\n...\n// Path: langchain/src/agents/toolkits/json/prompt.ts\nexport const JSON_PREFIX = `You are an agent designed to interact with JSON.\nYour goal is to return a final answer by interacting with the JSON.\nYou have access to the following tools which help you learn more about the JSON you are interacting with.\nOnly use the below tools. Only use the information returned by the below tools to construct your final answer.\nDo not make up any information that is not contained in the JSON.\nYour input to the tools should be in the form of in json pointer syntax (e.g. /key1/0/key2).\nYou must escape a slash in a key with a ~1, and escape a tilde with a ~0.\nFor example, to access the key /foo, you would use /~1foo\nYou should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling 'json_list_keys'.\nIf you have not seen a key in one of those responses, you cannot use it.\nYou should only add one key at a time to the path. You cannot add multiple keys at once.\nIf you encounter a null or undefined value, go back to the previous key, look at the available keys, and try again.\n\nIf the question does not seem to be related to the JSON, just return \"I don't know\" as the answer.\nAlways begin your interaction with the 'json_list_keys' with an empty string as the input to see what keys exist in the JSON.\n\nNote that sometimes the value at a given path is large. In this case, you will get an error \"Value is a large dictionary, should explore its keys directly\".\nIn this case, you should ALWAYS follow up by using the 'json_list_keys' tool to see what keys exist at that path.\nDo not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it.`;\n\nexport const JSON_SUFFIX = `Begin!\"\n\nQuestion: {input}\nThought: I should look at the keys that exist to see what I can query. I should use the 'json_list_keys' tool with an empty string as the input.\n{agent_scratchpad}`;\n\n// Path: langchain/src/agents/toolkits/json/json.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { ToolInterface } from \"@langchain/core/tools\";\nimport { Toolkit } from \"@langchain/community/agents/toolkits/base\";\nimport {\n  JsonGetValueTool,\n  JsonListKeysTool,\n  JsonSpec,\n} from \"../../../tools/json.js\";\nimport { JSON_PREFIX, JSON_SUFFIX } from \"./prompt.js\";\nimport { LLMChain } from \"../../../chains/llm_chain.js\";\nimport { ZeroShotCreatePromptArgs, ZeroShotAgent } from \"../../mrkl/index.js\";\nimport { AgentExecutor } from \"../../executor.js\";\n\n/**\n * Represents a toolkit for working with JSON data. It initializes the\n * JSON tools based on the provided JSON specification.\n * @example\n * ```typescript\n * const toolkit = new JsonToolkit(new JsonSpec());\n * const executor = createJsonAgent(model, toolkit);\n * const result = await executor.invoke({\n *   input: 'What are the required parameters in the request body to the /completions endpoint?'\n * });\n * ```\n */\nexport class JsonToolkit extends Toolkit {\n  tools: ToolInterface[];\n\n  constructor(public jsonSpec: JsonSpec) {\n    super();\n    this.tools = [\n      new JsonListKeysTool(jsonSpec),\n      new JsonGetValueTool(jsonSpec),\n    ];\n  }\n}\n\n/**\n * @deprecated Create a specific agent with a custom tool instead.\n *\n * Creates a JSON agent using a language model, a JSON toolkit, and\n * optional prompt arguments. It creates a prompt for the agent using the\n * JSON tools and the provided prefix and suffix. It then creates a\n * ZeroShotAgent with the prompt and the JSON tools, and returns an\n * AgentExecutor for executing the agent with the tools.\n * @param llm The language model used to create the JSON agent.\n * @param toolkit The JSON toolkit used to create the JSON agent.\n * @param args Optional prompt arguments used to create the JSON agent.\n * @returns An AgentExecutor for executing the created JSON agent with the tools.\n */\nexport function createJsonAgent(\n  llm: BaseLanguageModelInterface,\n  toolkit: JsonToolkit,\n  args?: ZeroShotCreatePromptArgs\n) {\n  const {\n    prefix = JSON_PREFIX,\n    suffix = JSON_SUFFIX,\n    inputVariables = [\"input\", \"agent_scratchpad\"],\n  } = args ?? {};\n  const { tools } = toolkit;\n  const prompt = ZeroShotAgent.createPrompt(tools, {\n    prefix,\n    suffix,\n    inputVariables,\n  });\n  const chain = new LLMChain({ prompt, llm });\n  const agent = new ZeroShotAgent({\n    llmChain: chain,\n    allowedTools: tools.map((t) => t.name),\n  });\n  return AgentExecutor.fromAgentAndTools({\n    agent,\n    tools,\n    returnIntermediateSteps: true,\n  });\n}\n\n// Path: langchain/src/agents/toolkits/openapi/prompt.ts\nexport const OPENAPI_PREFIX = `You are an agent designed to answer questions by making web requests to an API given the OpenAPI spec.\n\nIf the question does not seem related to the API, return I don't know. Do not make up an answer.\nOnly use information provided by the tools to construct your response.\n\nTo find information in the OpenAPI spec, use the 'json_explorer' tool. The input to this tool is a question about the API.\n\nTake the following steps:\nFirst, find the base URL needed to make the request.\n\nSecond, find the relevant paths needed to answer the question. Take note that, sometimes, you might need to make more than one request to more than one path to answer the question.\n\nThird, find the required parameters needed to make the request. For GET requests, these are usually URL parameters and for POST requests, these are request body parameters.\n\nFourth, make the requests needed to answer the question. Ensure that you are sending the correct parameters to the request by checking which parameters are required. For parameters with a fixed set of values, please use the spec to look at which values are allowed.\n\nUse the exact parameter names as listed in the spec, do not make up any names or abbreviate the names of parameters.\nIf you get a not found error, ensure that you are using a path that actually exists in the spec.`;\n\nexport const OPENAPI_SUFFIX = `Begin!\"\n\nQuestion: {input}\nThought: I should explore the spec to find the base url for the API.\n{agent_scratchpad}`;\nexport const JSON_EXPLORER_DESCRIPTION = `\nCan be used to answer questions about the openapi spec for the API. Always use this tool before trying to make a request. \nExample inputs to this tool: \n    'What are the required query parameters for a GET request to the /bar endpoint?'\n    'What are the required parameters in the request body for a POST request to the /foo endpoint?'\nAlways give this tool a specific question.`;\n\n// Path: langchain/src/agents/toolkits/openapi/openapi.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { ToolInterface } from \"@langchain/core/tools\";\nimport { DynamicTool } from \"@langchain/core/tools\";\nimport { Toolkit } from \"@langchain/community/agents/toolkits/base\";\nimport { JsonSpec } from \"../../../tools/json.js\";\nimport { AgentExecutor } from \"../../executor.js\";\nimport {\n  OPENAPI_PREFIX,\n  OPENAPI_SUFFIX,\n  JSON_EXPLORER_DESCRIPTION,\n} from \"./prompt.js\";\nimport { LLMChain } from \"../../../chains/llm_chain.js\";\nimport { ZeroShotCreatePromptArgs, ZeroShotAgent } from \"../../mrkl/index.js\";\nimport {\n  Headers,\n  RequestsGetTool,\n  RequestsPostTool,\n} from \"../../../tools/requests.js\";\nimport { createJsonAgent, JsonToolkit } from \"../json/json.js\";\n\n/**\n * Represents a toolkit for making HTTP requests. It initializes the\n * request tools based on the provided headers.\n */\nexport class RequestsToolkit extends Toolkit {\n  tools: ToolInterface[];\n\n  constructor(headers?: Headers) {\n    super();\n    this.tools = [new RequestsGetTool(headers), new RequestsPostTool(headers)];\n  }\n}\n\n/**\n * Extends the `RequestsToolkit` class and adds a dynamic tool for\n * exploring JSON data. It creates a JSON agent using the `JsonToolkit`\n * and the provided language model, and adds the JSON explorer tool to the\n * toolkit.\n * @example\n * ```typescript\n * const toolkit = new OpenApiToolkit(\n *   new JsonSpec({\n *   }),\n *   new ChatOpenAI({ temperature: 0 }),\n *   {\n *     \"Content-Type\": \"application/json\",\n *     Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,\n *   },\n * );\n *\n * const result = await toolkit.invoke({\n *   input:\n *     \"Make a POST request to openai /completions. The prompt should be 'tell me a joke.'\",\n * });\n * console.log(`Got output ${result.output}`);\n * ```\n */\nexport class OpenApiToolkit extends RequestsToolkit {\n  constructor(\n    jsonSpec: JsonSpec,\n    llm: BaseLanguageModelInterface,\n    headers?: Headers\n  ) {\n    super(headers);\n    const jsonAgent = createJsonAgent(llm, new JsonToolkit(jsonSpec));\n    this.tools = [\n      ...this.tools,\n      new DynamicTool({\n        name: \"json_explorer\",\n        func: async (input: string) => {\n          const result = await jsonAgent.call({ input });\n          return result.output as string;\n        },\n        description: JSON_EXPLORER_DESCRIPTION,\n      }),\n    ];\n  }\n}\n\n/**\n * @deprecated Create a specific agent with a custom tool instead.\n *\n * Creates an OpenAPI agent using a language model, an OpenAPI toolkit,\n * and optional prompt arguments. It creates a prompt for the agent using\n * the OpenAPI tools and the provided prefix and suffix. It then creates a\n * ZeroShotAgent with the prompt and the OpenAPI tools, and returns an\n * AgentExecutor for executing the agent with the tools.\n * @param llm The language model to use.\n * @param openApiToolkit The OpenAPI toolkit to use.\n * @param args Optional arguments for creating the prompt.\n * @returns An AgentExecutor for executing the agent with the tools.\n *\n * @security **Security Notice** This agent provides access to external APIs.\n * Use with caution as this agent can make API calls with arbitrary headers.\n * Exposing this agent to users could lead to security vulnerabilities. Consider\n * limiting access to what endpoints it can hit, what actions can be taken, and\n * more.\n *\n * @link See https://js.langchain.com/docs/security for more information.\n */\nexport function createOpenApiAgent(\n  llm: BaseLanguageModelInterface,\n  openApiToolkit: OpenApiToolkit,\n  args?: ZeroShotCreatePromptArgs\n) {\n  const {\n    prefix = OPENAPI_PREFIX,\n    suffix = OPENAPI_SUFFIX,\n    inputVariables = [\"input\", \"agent_scratchpad\"],\n  } = args ?? {};\n\n  const { tools } = openApiToolkit;\n  const prompt = ZeroShotAgent.createPrompt(tools, {\n    prefix,\n    suffix,\n    inputVariables,\n  });\n  const chain = new LLMChain({\n    prompt,\n    llm,\n  });\n  const toolNames = tools.map((tool) => tool.name);\n  const agent = new ZeroShotAgent({ llmChain: chain, allowedTools: toolNames });\n  return AgentExecutor.fromAgentAndTools({\n    agent,\n    tools,\n    returnIntermediateSteps: true,\n  });\n}\n\n// Path: langchain/src/agents/openai_functions/prompt.ts\nexport const PREFIX = `You are a helpful AI assistant.`;\nexport const SUFFIX = ``;\n\n// Path: langchain/src/agents/openai_functions/index.ts\nimport type {\n  BaseLanguageModelInterface,\n  BaseLanguageModelInput,\n  BaseFunctionCallOptions,\n} from \"@langchain/core/language_models/base\";\nimport type { StructuredToolInterface } from \"@langchain/core/tools\";\nimport type { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport { Runnable, RunnablePassthrough } from \"@langchain/core/runnables\";\nimport { ChatOpenAI, ChatOpenAICallOptions } from \"@langchain/openai\";\nimport type {\n  AgentAction,\n  AgentFinish,\n  AgentStep,\n} from \"@langchain/core/agents\";\nimport { convertToOpenAIFunction } from \"@langchain/core/utils/function_calling\";\nimport {\n  AIMessage,\n  BaseMessage,\n  FunctionMessage,\n  SystemMessage,\n  BaseMessageChunk,\n} from \"@langchain/core/messages\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  ChatPromptTemplate,\n  HumanMessagePromptTemplate,\n  MessagesPlaceholder,\n  SystemMessagePromptTemplate,\n  BasePromptTemplate,\n} from \"@langchain/core/prompts\";\nimport { CallbackManager } from \"@langchain/core/callbacks/manager\";\nimport { Agent, AgentArgs, AgentRunnableSequence } from \"../agent.js\";\nimport { AgentInput } from \"../types.js\";\nimport { PREFIX } from \"./prompt.js\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\nimport {\n  FunctionsAgentAction,\n  OpenAIFunctionsAgentOutputParser,\n} from \"../openai/output_parser.js\";\nimport { formatToOpenAIFunctionMessages } from \"../format_scratchpad/openai_functions.js\";\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype CallOptionsIfAvailable<T> = T extends { CallOptions: infer CO } ? CO : any;\n\n/**\n * Checks if the given action is a FunctionsAgentAction.\n * @param action The action to check.\n * @returns True if the action is a FunctionsAgentAction, false otherwise.\n */\nfunction isFunctionsAgentAction(\n  action: AgentAction | FunctionsAgentAction\n): action is FunctionsAgentAction {\n  return (action as FunctionsAgentAction).messageLog !== undefined;\n}\n\nfunction _convertAgentStepToMessages(\n  action: AgentAction | FunctionsAgentAction,\n  observation: string\n) {\n  if (isFunctionsAgentAction(action) && action.messageLog !== undefined) {\n    return action.messageLog?.concat(\n      new FunctionMessage(observation, action.tool)\n    );\n  } else {\n    return [new AIMessage(action.log)];\n  }\n}\n\nexport function _formatIntermediateSteps(\n  intermediateSteps: AgentStep[]\n): BaseMessage[] {\n  return intermediateSteps.flatMap(({ action, observation }) =>\n    _convertAgentStepToMessages(action, observation)\n  );\n}\n\n/**\n * Interface for the input data required to create an OpenAIAgent.\n */\nexport interface OpenAIAgentInput extends AgentInput {\n  tools: StructuredToolInterface[];\n}\n\n/**\n * Interface for the arguments required to create a prompt for an\n * OpenAIAgent.\n */\nexport interface OpenAIAgentCreatePromptArgs {\n  prefix?: string;\n  systemMessage?: SystemMessage;\n}\n\n/**\n * Class representing an agent for the OpenAI chat model in LangChain. It\n * extends the Agent class and provides additional functionality specific\n * to the OpenAIAgent type.\n *\n * @deprecated Use the {@link https://api.js.langchain.com/functions/langchain_agents.createOpenAIFunctionsAgent.html | createOpenAIFunctionsAgent method instead}.\n */\nexport class OpenAIAgent extends Agent {\n  static lc_name() {\n    return \"OpenAIAgent\";\n  }\n\n  lc_namespace = [\"langchain\", \"agents\", \"openai\"];\n\n  _agentType() {\n    return \"openai-functions\" as const;\n  }\n\n  observationPrefix() {\n    return \"Observation: \";\n  }\n\n  llmPrefix() {\n    return \"Thought:\";\n  }\n\n  _stop(): string[] {\n    return [\"Observation:\"];\n  }\n\n  tools: StructuredToolInterface[];\n\n  outputParser: OpenAIFunctionsAgentOutputParser =\n    new OpenAIFunctionsAgentOutputParser();\n\n  constructor(input: Omit<OpenAIAgentInput, \"outputParser\">) {\n    super({ ...input, outputParser: undefined });\n    this.tools = input.tools;\n  }\n\n  /**\n   * Creates a prompt for the OpenAIAgent using the provided tools and\n   * fields.\n   * @param _tools The tools to be used in the prompt.\n   * @param fields Optional fields for creating the prompt.\n   * @returns A BasePromptTemplate object representing the created prompt.\n   */\n  static createPrompt(\n    _tools: StructuredToolInterface[],\n    fields?: OpenAIAgentCreatePromptArgs\n  ): BasePromptTemplate {\n    const { prefix = PREFIX } = fields || {};\n    return ChatPromptTemplate.fromMessages([\n      SystemMessagePromptTemplate.fromTemplate(prefix),\n      new MessagesPlaceholder(\"chat_history\"),\n      HumanMessagePromptTemplate.fromTemplate(\"{input}\"),\n      new MessagesPlaceholder(\"agent_scratchpad\"),\n    ]);\n  }\n\n  /**\n   * Creates an OpenAIAgent from a BaseLanguageModel and a list of tools.\n   * @param llm The BaseLanguageModel to use.\n   * @param tools The tools to be used by the agent.\n   * @param args Optional arguments for creating the agent.\n   * @returns An instance of OpenAIAgent.\n   */\n  static fromLLMAndTools(\n    llm: BaseLanguageModelInterface,\n    tools: StructuredToolInterface[],\n    args?: OpenAIAgentCreatePromptArgs & Pick<AgentArgs, \"callbacks\">\n  ) {\n    OpenAIAgent.validateTools(tools);\n    if (llm._modelType() !== \"base_chat_model\" || llm._llmType() !== \"openai\") {\n      throw new Error(\"OpenAIAgent requires an OpenAI chat model\");\n    }\n    const prompt = OpenAIAgent.createPrompt(tools, args);\n    const chain = new LLMChain({\n      prompt,\n      llm,\n      callbacks: args?.callbacks,\n    });\n    return new OpenAIAgent({\n      llmChain: chain,\n      allowedTools: tools.map((t) => t.name),\n      tools,\n    });\n  }\n\n  /**\n   * Constructs a scratch pad from a list of agent steps.\n   * @param steps The steps to include in the scratch pad.\n   * @returns A string or a list of BaseMessages representing the constructed scratch pad.\n   */\n  async constructScratchPad(\n    steps: AgentStep[]\n  ): Promise<string | BaseMessage[]> {\n    return _formatIntermediateSteps(steps);\n  }\n\n  /**\n   * Plans the next action or finish state of the agent based on the\n   * provided steps, inputs, and optional callback manager.\n   * @param steps The steps to consider in planning.\n   * @param inputs The inputs to consider in planning.\n   * @param callbackManager Optional CallbackManager to use in planning.\n   * @returns A Promise that resolves to an AgentAction or AgentFinish object representing the planned action or finish state.\n   */\n  async plan(\n    steps: Array<AgentStep>,\n    inputs: ChainValues,\n    callbackManager?: CallbackManager\n  ): Promise<AgentAction | AgentFinish> {\n    // Add scratchpad and stop to inputs\n    const thoughts = await this.constructScratchPad(steps);\n    const newInputs: ChainValues = {\n      ...inputs,\n      agent_scratchpad: thoughts,\n    };\n    if (this._stop().length !== 0) {\n      newInputs.stop = this._stop();\n    }\n\n    // Split inputs between prompt and llm\n    const llm = this.llmChain.llm as\n      | ChatOpenAI\n      | Runnable<\n          BaseLanguageModelInput,\n          BaseMessageChunk,\n          ChatOpenAICallOptions\n        >;\n\n    const valuesForPrompt = { ...newInputs };\n    const valuesForLLM: CallOptionsIfAvailable<typeof llm> = {\n      functions: this.tools.map(convertToOpenAIFunction),\n    };\n    const callKeys =\n      \"callKeys\" in this.llmChain.llm ? this.llmChain.llm.callKeys : [];\n    for (const key of callKeys) {\n      if (key in inputs) {\n        valuesForLLM[key as keyof CallOptionsIfAvailable<typeof llm>] =\n          inputs[key];\n        delete valuesForPrompt[key];\n      }\n    }\n\n    const promptValue = await this.llmChain.prompt.formatPromptValue(\n      valuesForPrompt\n    );\n\n    const message = await (\n      llm as Runnable<\n        BaseLanguageModelInput,\n        BaseMessageChunk,\n        ChatOpenAICallOptions\n      >\n    ).invoke(promptValue.toChatMessages(), {\n      ...valuesForLLM,\n      callbacks: callbackManager,\n    });\n    return this.outputParser.parseAIMessage(message);\n  }\n}\n\n/**\n * Params used by the createOpenAIFunctionsAgent function.\n */\nexport type CreateOpenAIFunctionsAgentParams = {\n  /**\n   * LLM to use as the agent. Should work with OpenAI function calling,\n   * so must either be an OpenAI model that supports that or a wrapper of\n   * a different model that adds in equivalent support.\n   */\n  llm: BaseChatModel<BaseFunctionCallOptions>;\n  /** Tools this agent has access to. */\n  tools: StructuredToolInterface[];\n  /** The prompt to use, must have an input key for `agent_scratchpad`. */\n  prompt: ChatPromptTemplate;\n  /**\n   * Whether to invoke the underlying model in streaming mode,\n   * allowing streaming of intermediate steps. Defaults to true.\n   */\n  streamRunnable?: boolean;\n};\n\n/**\n * Create an agent that uses OpenAI-style function calling.\n * @param params Params required to create the agent. Includes an LLM, tools, and prompt.\n * @returns A runnable sequence representing an agent. It takes as input all the same input\n *     variables as the prompt passed in does. It returns as output either an\n *     AgentAction or AgentFinish.\n *\n * @example\n * ```typescript\n * import { AgentExecutor, createOpenAIFunctionsAgent } from \"langchain/agents\";\n * import { pull } from \"langchain/hub\";\n * import type { ChatPromptTemplate } from \"@langchain/core/prompts\";\n * import { AIMessage, HumanMessage } from \"@langchain/core/messages\";\n *\n * import { ChatOpenAI } from \"@langchain/openai\";\n *\n * // Define the tools the agent will have access to.\n * const tools = [...];\n *\n * // Get the prompt to use - you can modify this!\n * // If you want to see the prompt in full, you can at:\n * // https://smith.langchain.com/hub/hwchase17/openai-functions-agent\n * const prompt = await pull<ChatPromptTemplate>(\n *   \"hwchase17/openai-functions-agent\"\n * );\n *\n * const llm = new ChatOpenAI({\n *   temperature: 0,\n * });\n *\n * const agent = await createOpenAIFunctionsAgent({\n *   llm,\n *   tools,\n *   prompt,\n * });\n *\n * const agentExecutor = new AgentExecutor({\n *   agent,\n *   tools,\n * });\n *\n * const result = await agentExecutor.invoke({\n *   input: \"what is LangChain?\",\n * });\n *\n * // With chat history\n * const result2 = await agentExecutor.invoke({\n *   input: \"what's my name?\",\n *   chat_history: [\n *     new HumanMessage(\"hi! my name is cob\"),\n *     new AIMessage(\"Hello Cob! How can I assist you today?\"),\n *   ],\n * });\n * ```\n */\nexport \nasync function createOpenAIFunctionsAgent({\n  llm,\n  tools,\n  prompt,\n  streamRunnable,\n}: CreateOpenAIFunctionsAgentParams) {\n  if (!prompt.inputVariables.includes(\"agent_scratchpad\")) {\n    throw new Error(\n      [\n        `Prompt must have an input variable named \"agent_scratchpad\".`,\n        `Found ${JSON.stringify(prompt.inputVariables)} instead.`,\n      ].join(\"\\n\")\n    );\n  }\n  const llmWithTools = llm.bind({\n    functions: tools.map(convertToOpenAIFunction),\n  });\n  const agent = AgentRunnableSequence.fromRunnables(\n    [\n      RunnablePassthrough.assign({\n        agent_scratchpad: (input: { steps: AgentStep[] }) =>\n          formatToOpenAIFunctionMessages(input.steps),\n      }),\n      prompt,\n      llmWithTools,\n      new OpenAIFunctionsAgentOutputParser(),\n    ],\n    {\n      name: \"OpenAIFunctionsAgent\",\n      streamRunnable,\n      singleAction: true,\n    }\n  );\n  return agent;\n}\n\n// Path: langchain/src/agents/toolkits/conversational_retrieval/token_buffer_memory.ts\nimport { ChatOpenAI } from \"@langchain/openai\";\nimport {\n  InputValues,\n  MemoryVariables,\n  OutputValues,\n  getInputValue,\n  getOutputValue,\n} from \"@langchain/core/memory\";\nimport { getBufferString } from \"@langchain/core/messages\";\nimport {\n  BaseChatMemory,\n  BaseChatMemoryInput,\n} from \"@langchain/community/memory/chat_memory\";\nimport { _formatIntermediateSteps } from \"../../openai_functions/index.js\";\n\n/**\n * Type definition for the fields required to initialize an instance of\n * OpenAIAgentTokenBufferMemory.\n */\nexport type OpenAIAgentTokenBufferMemoryFields = BaseChatMemoryInput & {\n  llm: ChatOpenAI;\n  humanPrefix?: string;\n  aiPrefix?: string;\n  memoryKey?: string;\n  maxTokenLimit?: number;\n  returnMessages?: boolean;\n  outputKey?: string;\n  intermediateStepsKey?: string;\n};\n\n/**\n * Memory used to save agent output and intermediate steps.\n */\nexport class OpenAIAgentTokenBufferMemory extends BaseChatMemory {\n  humanPrefix = \"Human\";\n\n  aiPrefix = \"AI\";\n\n  llm: ChatOpenAI;\n\n  memoryKey = \"history\";\n\n  maxTokenLimit = 12000;\n\n  returnMessages = true;\n\n  outputKey = \"output\";\n\n  intermediateStepsKey = \"intermediateSteps\";\n\n  constructor(fields: OpenAIAgentTokenBufferMemoryFields) {\n    super(fields);\n    this.humanPrefix = fields.humanPrefix ?? this.humanPrefix;\n    this.aiPrefix = fields.aiPrefix ?? this.aiPrefix;\n    this.llm = fields.llm;\n    this.memoryKey = fields.memoryKey ?? this.memoryKey;\n    this.maxTokenLimit = fields.maxTokenLimit ?? this.maxTokenLimit;\n    this.returnMessages = fields.returnMessages ?? this.returnMessages;\n    this.outputKey = fields.outputKey ?? this.outputKey;\n    this.intermediateStepsKey =\n      fields.intermediateStepsKey ?? this.intermediateStepsKey;\n  }\n\n  get memoryKeys(): string[] {\n    return [this.memoryKey];\n  }\n\n  /**\n   * Retrieves the messages from the chat history.\n   * @returns Promise that resolves with the messages from the chat history.\n   */\n  async getMessages() {\n    return this.chatHistory.getMessages();\n  }\n\n  /**\n   * Loads memory variables from the input values.\n   * @param _values Input values.\n   * @returns Promise that resolves with the loaded memory variables.\n   */\n  async loadMemoryVariables(_values: InputValues): Promise<MemoryVariables> {\n    const buffer = await this.getMessages();\n    if (this.returnMessages) {\n      return { [this.memoryKey]: buffer };\n    } else {\n      const bufferString = getBufferString(\n        buffer,\n        this.humanPrefix,\n        this.aiPrefix\n      );\n      return { [this.memoryKey]: bufferString };\n    }\n  }\n\n  /**\n   * Saves the context of the chat, including user input, AI output, and\n   * intermediate steps. Prunes the chat history if the total token count\n   * exceeds the maximum limit.\n   * @param inputValues Input values.\n   * @param outputValues Output values.\n   * @returns Promise that resolves when the context has been saved.\n   */\n  async saveContext(\n    inputValues: InputValues,\n    outputValues: OutputValues\n  ): Promise<void> {\n    const inputValue = getInputValue(inputValues, this.inputKey);\n    const outputValue = getOutputValue(outputValues, this.outputKey);\n    await this.chatHistory.addUserMessage(inputValue);\n    const intermediateStepMessages = _formatIntermediateSteps(\n      outputValues[this.intermediateStepsKey]\n    );\n    for (const message of intermediateStepMessages) {\n      await this.chatHistory.addMessage(message);\n    }\n    await this.chatHistory.addAIChatMessage(outputValue);\n    const currentMessages = await this.chatHistory.getMessages();\n    let tokenInfo = await this.llm.getNumTokensFromMessages(currentMessages);\n    if (tokenInfo.totalCount > this.maxTokenLimit) {\n      const prunedMemory = [];\n      while (tokenInfo.totalCount > this.maxTokenLimit) {\n        const retainedMessage = currentMessages.pop();\n        if (!retainedMessage) {\n          console.warn(\n            `Could not prune enough messages from chat history to stay under ${this.maxTokenLimit} tokens.`\n          );\n          break;\n        }\n        prunedMemory.push(retainedMessage);\n        tokenInfo = await this.llm.getNumTokensFromMessages(currentMessages);\n      }\n      await this.chatHistory.clear();\n      for (const message of prunedMemory) {\n        await this.chatHistory.addMessage(message);\n      }\n    }\n  }\n}\n\n// Path: langchain/src/agents/toolkits/index.ts\nexport { JsonToolkit, createJsonAgent } from \"./json/json.js\";\nexport {\n  RequestsToolkit,\n  OpenApiToolkit,\n  createOpenApiAgent,\n} from \"./openapi/openapi.js\";\nexport {\n  type VectorStoreInfo,\n  VectorStoreToolkit,\n  VectorStoreRouterToolkit,\n  createVectorStoreAgent,\n  createVectorStoreRouterAgent,\n} from \"./vectorstore/vectorstore.js\";\nexport { ZapierToolKit } from \"./zapier/zapier.js\";\nexport { createRetrieverTool } from \"./conversational_retrieval/tool.js\";\nexport {\n  createConversationalRetrievalAgent,\n  type ConversationalRetrievalAgentOptions,\n} from \"./conversational_retrieval/openai_functions.js\";\nexport { OpenAIAgentTokenBufferMemory } from \"./conversational_retrieval/token_buffer_memory.js\";\n\n// Path: langchain/src/experimental/masking/transformer.ts\n/**\n * Abstract class representing a transformer used for masking and rehydrating messages.\n */\nexport abstract class MaskingTransformer {\n  abstract transform(\n    message: string,\n    state?: Map<string, string>\n  ): Promise<[string, Map<string, string>]>;\n\n  abstract rehydrate(\n    message: string,\n    state: Map<string, string>\n  ): Promise<string>;\n}\n\n// Path: langchain/src/experimental/masking/types.ts\nimport { MaskingTransformer } from \"./transformer.js\";\n/**\n * Configuration type for MaskingParser.\n */\n\nexport type MaskingParserConfig = {\n  transformers?: MaskingTransformer[];\n  defaultHashFunction?: HashFunction;\n  onMaskingStart?: HookFunction;\n  onMaskingEnd?: HookFunction;\n  onRehydratingStart?: HookFunction;\n  onRehydratingEnd?: HookFunction;\n};\n\n/**\n *  Regex Masking Pattern used for masking in PIIMaskingTransformer.\n */\nexport type MaskingPattern = {\n  regex: RegExp;\n  replacement?: string;\n  mask?: (match: string) => string;\n};\n\nexport type HookFunction =\n  | ((message: string) => Promise<void>)\n  | ((message: string) => void);\n\n/**\n * Represents a function that can hash a string input.\n */\nexport type HashFunction = (input: string) => string;\n\n// Path: langchain/src/experimental/masking/regex_masking_transformer.ts\nimport { MaskingTransformer } from \"./transformer.js\";\nimport type { HashFunction, MaskingPattern } from \"./types.js\";\n/**\n * RegexMaskingTransformer class for masking and rehydrating messages with Regex.\n */\nexport class RegexMaskingTransformer extends MaskingTransformer {\n  private patterns: { [key: string]: MaskingPattern };\n\n  private hashFunction: HashFunction;\n\n  /**\n   * Constructs a RegexMaskingTransformer with given patterns and an optional hash function.\n   * Validates the provided patterns to ensure they conform to the expected structure.\n   *\n   * @param patterns - An object containing masking patterns. Each pattern should include\n   *                   a regular expression (`regex`) and optionally a `replacement` string\n   *                   or a `mask` function.\n   * @param hashFunction - An optional custom hash function to be used for masking.\n   */\n  constructor(\n    patterns: { [key: string]: MaskingPattern },\n    hashFunction?: HashFunction\n  ) {\n    super();\n    // Validates the provided masking patterns before initializing the transformer.\n    // This ensures that each pattern has a valid regular expression.\n    this.validatePatterns(patterns);\n\n    // Assigns the validated patterns and the hash function to the transformer.\n    // If no custom hash function is provided, the default hash function is used.\n    this.patterns = patterns;\n    this.hashFunction = hashFunction || this.defaultHashFunction;\n  }\n\n  /**\n   * Validates the given masking patterns to ensure each pattern has a valid regular expression.\n   * Throws an error if any pattern is found to be invalid.\n   *\n   * @param patterns - The patterns object to validate.\n   */\n  private validatePatterns(patterns: { [key: string]: MaskingPattern }) {\n    for (const key of Object.keys(patterns)) {\n      const pattern = patterns[key];\n      // Checks that each pattern is an object and has a regex property that is an instance of RegExp.\n      // Throws an error if these conditions are not met, indicating an invalid pattern configuration.\n      if (\n        !pattern ||\n        typeof pattern !== \"object\" ||\n        // eslint-disable-next-line no-instanceof/no-instanceof\n        !(pattern.regex instanceof RegExp)\n      ) {\n        throw new Error(\"Invalid pattern configuration.\");\n      }\n    }\n  }\n\n  /**\n   * Masks content in a message based on the defined patterns.\n   * @param message - The message to be masked.\n   * @param state - The current state containing original values.\n   * @returns A tuple of the masked message and the updated state.\n   */\n  async transform(\n    message: string,\n    state: Map<string, string>\n  ): Promise<[string, Map<string, string>]> {\n    if (typeof message !== \"string\") {\n      throw new TypeError(\n        \"RegexMaskingTransformer.transform Error: The 'message' argument must be a string.\"\n      );\n    }\n\n    // eslint-disable-next-line no-instanceof/no-instanceof\n    if (!(state instanceof Map)) {\n      throw new TypeError(\n        \"RegexMaskingTransformer.transform Error: The 'state' argument must be an instance of Map.\"\n      );\n    }\n\n    // Holds the progressively masked message\n    let processedMessage = message;\n\n    // Initialize original values map with the current state or a new map\n    const originalValues = state || new Map<string, string>();\n\n    // Iterate over each pattern defined in the transformer\n    for (const key of Object.keys(this.patterns)) {\n      const pattern = this.patterns[key];\n\n      // Apply the current pattern's regex to the message\n      processedMessage = processedMessage.replace(pattern.regex, (match) => {\n        // Determine the masked value: use the mask function if provided, else use the replacement string,\n        // else use the hash function.\n        const maskedValue = pattern.mask\n          ? pattern.mask(match)\n          : pattern.replacement ?? this.hashFunction(match);\n\n        // Store the mapping of the masked value to the original value (match)\n        originalValues.set(maskedValue, match);\n\n        // Return the masked value to replace the original value in the message\n        return maskedValue;\n      });\n    }\n\n    // Return the fully masked message and the state map with all original values\n    // Wrap the synchronous return values in Promise.resolve() to maintain compatibility\n    // with the MaskingParser's expectation of a Promise return type.\n    return [processedMessage, originalValues];\n  }\n\n  /**\n   * Rehydrates a masked message back to its original form using the provided state.\n   * @param message - The masked message to be rehydrated.\n   * @param state - The state map containing mappings of masked values to their original values.\n   * @returns The rehydrated (original) message.\n   */\n  async rehydrate(\n    message: string,\n    state: Map<string, string>\n  ): Promise<string> {\n    if (typeof message !== \"string\") {\n      throw new TypeError(\n        \"RegexMaskingTransformer.rehydrate Error: The 'message' argument must be a string.\"\n      );\n    }\n\n    // eslint-disable-next-line no-instanceof/no-instanceof\n    if (!(state instanceof Map)) {\n      throw new TypeError(\n        \"RegexMaskingTransformer.rehydrate Error: The 'state' argument must be an instance of Map.\"\n      );\n    }\n\n    // Convert the state map to an array and use reduce to sequentially replace masked values with original values.\n    const rehydratedMessage = Array.from(state).reduce(\n      (msg, [masked, original]) => {\n        // Escape special characters in the masked string to ensure it can be used in a regular expression safely.\n        // This is necessary because masked values might contain characters that have special meanings in regex.\n        const escapedMasked = masked.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\n        // Replace all instances of the escaped masked value in the message with the original value.\n        // The 'g' flag in the RegExp ensures that all occurrences of the masked value are replaced.\n        return msg.replace(new RegExp(escapedMasked, \"g\"), original);\n      },\n      message\n    );\n\n    return rehydratedMessage;\n  }\n\n  /**\n   * Default hash function for creating unique hash values.\n   * @param input - The input string to hash.\n   * @returns The resulting hash as a string.\n   */\n  private defaultHashFunction(input: string): string {\n    let hash = 0;\n    // Iterate over each character in the input string\n    for (let i = 0; i < input.length; i += 1) {\n      // Get ASCII value of the character\n      const char = input.charCodeAt(i);\n      // Combine the current hash with the new character and ensure it remains a 32-bit integer\n      hash = (hash << 5) - hash + char;\n      // Bitwise OR operation to convert to a 32-bit integer.\n      // This is a common technique to ensure the final hash value stays within the 32-bit limit,\n      // effectively wrapping the value when it becomes too large.\n      hash |= 0;\n    }\n\n    // Convert the numerical hash value to a string and return\n    return hash.toString();\n  }\n}\n\n// Path: langchain/src/experimental/masking/parser.ts\nimport { MaskingTransformer } from \"./transformer.js\";\nimport type { MaskingParserConfig } from \"./types.js\";\n\n/**\n * MaskingParser class for handling the masking and rehydrating of messages.\n */\nexport class MaskingParser {\n  private transformers: MaskingTransformer[];\n\n  private state: Map<string, string>;\n\n  private config: MaskingParserConfig;\n\n  constructor(config: MaskingParserConfig = {}) {\n    this.transformers = config.transformers ?? [];\n    this.state = new Map<string, string>();\n    this.config = config;\n  }\n\n  /**\n   * Adds a transformer to the parser.\n   * @param transformer - An instance of a class extending MaskingTransformer.\n   */\n  addTransformer(transformer: MaskingTransformer) {\n    this.transformers.push(transformer);\n  }\n\n  /**\n   * Getter method for retrieving the current state.\n   * @returns The current state map.\n   */\n  public getState(): Map<string, string> {\n    return this.state;\n  }\n\n  /**\n   * Masks the provided message using the added transformers.\n   * This method sequentially applies each transformer's masking logic to the message.\n   * It utilizes a state map to track original values corresponding to their masked versions.\n   *\n   * @param message - The message to be masked.\n   * @returns A masked version of the message.\n   * @throws {TypeError} If the message is not a string.\n   * @throws {Error} If no transformers are added.\n   */\n  async mask(message: string): Promise<string> {\n    // If onMaskingStart is a function, handle it accordingly\n    if (this.config.onMaskingStart) {\n      await this.config.onMaskingStart(message);\n    }\n\n    // Check if there are any transformers added to the parser. If not, throw an error\n    // as masking requires at least one transformer to apply its logic.\n    if (this.transformers.length === 0) {\n      throw new Error(\n        \"MaskingParser.mask Error: No transformers have been added. Please add at least one transformer before parsing.\"\n      );\n    }\n\n    if (typeof message !== \"string\") {\n      throw new TypeError(\n        \"MaskingParser.mask Error: The 'message' argument must be a string.\"\n      );\n    }\n\n    // Initialize the variable to hold the progressively masked message.\n    // It starts as the original message and gets transformed by each transformer.\n    let processedMessage = message;\n\n    // Iterate through each transformer and apply their transform method.\n    for (const transformer of this.transformers) {\n      // Transform the message and get the transformer's state changes, ensuring no direct mutation of the shared state.\n      const [transformedMessage, transformerState] =\n        await transformer.transform(processedMessage, new Map(this.state));\n\n      // Update the processed message for subsequent transformers.\n      processedMessage = transformedMessage;\n\n      // Merge state changes from the transformer into the parser's state.\n      // This accumulates all transformations' effects on the state.\n      transformerState.forEach((value, key) => this.state.set(key, value));\n    }\n\n    // Handle onMaskingEnd callback\n    if (this.config.onMaskingEnd) {\n      await this.config.onMaskingEnd(processedMessage);\n    }\n    // Return the fully masked message after all transformers have been applied.\n    return processedMessage;\n  }\n\n  /**\n   * Rehydrates a masked message back to its original form.\n   * This method sequentially applies the rehydration logic of each added transformer in reverse order.\n   * It relies on the state map to correctly map the masked values back to their original values.\n   *\n   * The rehydration process is essential for restoring the original content of a message\n   * that has been transformed (masked) by the transformers. This process is the inverse of the masking process.\n   *\n   * @param message - The masked message to be rehydrated.\n   * @returns The original (rehydrated) version of the message.\n   */\n  async rehydrate(\n    message: string,\n    state?: Map<string, string>\n  ): Promise<string> {\n    // Handle onRehydratingStart callback\n    if (this.config.onRehydratingStart) {\n      await this.config.onRehydratingStart(message);\n    }\n\n    if (typeof message !== \"string\") {\n      throw new TypeError(\n        \"MaskingParser.rehydrate Error: The 'message' argument must be a string.\"\n      );\n    }\n    // Check if any transformers have been added to the parser.\n    // If no transformers are present, throw an error as rehydration requires at least one transformer.\n    if (this.transformers.length === 0) {\n      throw new Error(\n        \"MaskingParser.rehydrate Error: No transformers have been added. Please add at least one transformer before rehydrating.\"\n      );\n    }\n\n    // eslint-disable-next-line no-instanceof/no-instanceof\n    if (state && !(state instanceof Map)) {\n      throw new TypeError(\n        \"MaskingParser.rehydrate Error: The 'state' argument, if provided, must be an instance of Map.\"\n      );\n    }\n\n    const rehydrationState = state || this.state; // Use provided state or fallback to internal state\n    // Initialize the rehydratedMessage with the input masked message.\n    // This variable will undergo rehydration by each transformer in reverse order.\n    let rehydratedMessage = message;\n    // Use a reverse for...of loop to accommodate asynchronous rehydrate methods\n    const reversedTransformers = this.transformers.slice().reverse();\n    for (const transformer of reversedTransformers) {\n      // Check if the result is a Promise and use await, otherwise use it directly\n      rehydratedMessage = await transformer.rehydrate(\n        rehydratedMessage,\n        rehydrationState\n      );\n    }\n\n    // Handle onRehydratingEnd callback\n    if (this.config.onRehydratingEnd) {\n      await this.config.onRehydratingEnd(rehydratedMessage);\n    }\n\n    // Return the fully rehydrated message after all transformers have been applied.\n    return rehydratedMessage;\n  }\n}\n\n// Path: langchain/src/experimental/masking/index.ts\nexport { MaskingParser } from \"./parser.js\";\nexport { RegexMaskingTransformer } from \"./regex_masking_transformer.js\";\nexport { MaskingTransformer } from \"./transformer.js\";\nexport {\n  type MaskingParserConfig,\n  type HashFunction,\n  type HookFunction,\n} from \"./types.js\";\n\n// Path: langchain/src/retrievers/vespa.ts\nexport * from \"@langchain/community/retrievers/vespa\";\n\n// Path: langchain/src/agents/format_scratchpad/xml.ts\nimport { AgentStep } from \"@langchain/core/agents\";\n\nexport function formatXml(intermediateSteps: AgentStep[]) {\n  let log = \"\";\n  for (const step of intermediateSteps) {\n    const { action, observation } = step;\n    log += `<tool>${action.tool}</tool><tool_input>${action.toolInput}\\n</tool_input><observation>${observation}</observation>`;\n  }\n  return log;\n}\n\n// Path: langchain/src/document_loaders/web/sort_xyz_blockchain.ts\nimport { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"../base.js\";\n\n/**\n * See https://docs.sort.xyz/docs/api-keys to get your free Sort API key.\n * See https://docs.sort.xyz for more information on the available queries.\n * See https://docs.sort.xyz/reference for more information about Sort's REST API.\n */\n\nexport interface Query {\n  type: \"NFTMetadata\" | \"latestTransactions\";\n  contractAddress: string;\n  blockchain: \"ethereum\" | \"polygon\" | \"goerli\";\n  limit?: number;\n}\n\n/**\n * Interface representing the parameters for the SortXYZBlockchainLoader\n * class.\n */\nexport interface SortXYZBlockchainLoaderParams {\n  apiKey: string;\n  query: Query | string;\n}\n\n/**\n * Interface representing the response from the SortXYZ API.\n */\nexport interface SortXYZBlockchainAPIResponse {\n  code: number;\n  data: {\n    durationMs: number;\n    id: string;\n    query: string;\n    records: Record<string, unknown>[];\n    recordCount: number;\n  };\n}\n\n/**\n * Class representing a document loader for loading data from the SortXYZ\n * blockchain using the SortXYZ API.\n * @example\n * ```typescript\n * const blockchainLoader = new SortXYZBlockchainLoader({\n *   apiKey: \"YOUR_SORTXYZ_API_KEY\",\n *   query: {\n *     type: \"NFTMetadata\",\n *     blockchain: \"ethereum\",\n *     contractAddress: \"0x887F3909C14DAbd9e9510128cA6cBb448E932d7f\".toLowerCase(),\n *   },\n * });\n *\n * const blockchainData = await blockchainLoader.load();\n *\n * const prompt =\n *   \"Describe the character with the attributes from the following json document in a 4 sentence story. \";\n * const model = new ChatOpenAI({ temperature: 0.9 })\n * const response = await model.invoke(\n *   prompt + JSON.stringify(blockchainData[0], null, 2),\n * );\n * console.log(`user > ${prompt}`);\n * console.log(`chatgpt > ${response}`);\n * ```\n */\nexport class SortXYZBlockchainLoader extends BaseDocumentLoader {\n  public readonly contractAddress: string;\n\n  public readonly blockchain: string;\n\n  public readonly apiKey: string;\n\n  public readonly queryType: string;\n\n  public readonly sql: string;\n\n  public readonly limit: number;\n\n  constructor({ apiKey, query }: SortXYZBlockchainLoaderParams) {\n    super();\n\n    if (!apiKey) {\n      throw new Error(\n        `apiKey is required! Head over to https://docs.sort.xyz/docs/api-keys to get your free Sort API key.`\n      );\n    }\n\n    this.apiKey = apiKey;\n\n    if (typeof query === \"string\") {\n      this.sql = query;\n    } else {\n      this.contractAddress = query.contractAddress.toLowerCase();\n      this.blockchain = query.blockchain;\n      this.queryType = query.type;\n      this.limit = query.limit ?? 100;\n    }\n  }\n\n  /**\n   * Method that loads the data from the SortXYZ blockchain based on the\n   * specified query parameters. It makes requests to the SortXYZ API and\n   * returns an array of Documents representing the retrieved data.\n   * @returns Promise<Document[]> - An array of Documents representing the retrieved data.\n   */\n  public async load(): Promise<Document[]> {\n    if (this.limit > 1000) {\n      throw new Error(\n        `Limit is set too high. Please set limit to 1000 or lower.`\n      );\n    }\n\n    const docs: Document[] = [];\n    let queryOffset = 0;\n\n    // eslint-disable-next-line no-constant-condition\n    while (true) {\n      let query = \"\";\n\n      if (this.sql) {\n        query = this.sql;\n      } else if (this.queryType === \"NFTMetadata\") {\n        // All parameters here are user defined\n        query = `SELECT * FROM ${this.blockchain}.nft_metadata WHERE contract_address = '${this.contractAddress}' ORDER BY token_id DESC LIMIT ${this.limit} OFFSET ${queryOffset}`;\n      } else if (this.queryType === \"latestTransactions\") {\n        // All parameters here are user defined\n        query = `SELECT * FROM ${this.blockchain}.transaction t, ethereum.block b WHERE t.to_address = '${this.contractAddress}' AND b.id=t.block_id ORDER BY b.timestamp DESC LIMIT ${this.limit} OFFSET ${queryOffset}`;\n      }\n\n      try {\n        const response = await fetch(\"https://api.sort.xyz/v1/queries/run\", {\n          method: \"POST\",\n          headers: {\n            \"x-api-key\": this.apiKey as string,\n            Accept: \"application/json\",\n            \"Content-Type\": \"application/json\",\n          },\n          body: JSON.stringify({ query }),\n        });\n\n        const fullResponse = await response.json();\n\n        // Reached the end, no more records\n        if (\n          fullResponse &&\n          fullResponse.data &&\n          fullResponse.data.records.length === 0\n        ) {\n          break;\n        }\n\n        const data = fullResponse?.data || [];\n\n...\n", "needle_token_start": 8640, "needle_token_end": 8926, "code_context_ntokens": 15997, "output": ["```typescript\n/**\n * Create an agent that uses OpenAI-style function calling.\n * @param params Params required to create the agent. Includes an LLM, tools, and prompt.\n * @returns A runnable sequence representing an agent. It takes as input all the same input\n *     variables as the prompt passed in does. It returns as output either an\n *     AgentAction or AgentFinish.\n *\n * @example\n * ```typescript\n * import { AgentExecutor, createOpenAIFunctionsAgent } from \"langchain/agents\";\n * import { pull } from \"langchain/hub\";\n * import type { ChatPromptTemplate } from \"@langchain/core/prompts\";\n * import { AIMessage, HumanMessage } from \"@langchain/core/messages\";\n *\n * import { ChatOpenAI } from \"@langchain/openai\";\n *\n * // Define the tools the agent will have access to.\n * const tools = [...];\n *\n * // Get the prompt to use - you can modify this!\n * // If you want to see the prompt in full, you can at:\n * // https://smith.langchain.com/hub/hwchase17/openai-functions-agent\n * const prompt = await pull<ChatPromptTemplate>(\n *   \"hwchase17/openai-functions-agent\"\n * );\n *\n * const llm = new ChatOpenAI({\n *   temperature: 0,\n * });\n *\n * const agent = await createOpenAIFunctionsAgent({\n *   llm,\n *   tools,\n *   prompt,\n * });\n *\n * const agentExecutor = new AgentExecutor({\n *   agent,\n *   tools,\n * });\n *\n * const result = await agentExecutor.invoke({\n *   input: \"what is LangChain?\",\n * });\n *\n * // With chat history\n * const result2 = await agentExecutor.invoke({\n *   input: \"what's my name?\",\n *   chat_history: [\n *     new HumanMessage(\"hi! my name is cob\"),\n *     new AIMessage(\"Hello Cob! How can I assist you today?\"),\n *   ],\n * });\n * ```\n */\nexport async function createOpenAIFunctionsAgent({\n  llm,\n  tools,\n  prompt,\n  streamRunnable,\n}: CreateOpenAIFunctionsAgentParams) {\n  if (!prompt.inputVariables.includes(\"agent_scratchpad\")) {\n    throw new Error(\n      [\n        `Prompt must have an input variable named \"agent_scratchpad\".`,\n        `Found ${JSON.stringify(prompt.inputVariables)} instead.`,\n      ].join(\"\\n\")\n    );\n  }\n  const llmWithTools = llm.bind({\n    functions: tools.map(convertToOpenAIFunction),\n  });\n  const agent = AgentRunnableSequence.fromRunnables(\n    [\n      RunnablePassthrough.assign({\n        agent_scratchpad: (input: { steps: AgentStep[] }) =>\n          formatToOpenAIFunctionMessages(input.steps),\n      }),\n      prompt,\n      llmWithTools,\n      new OpenAIFunctionsAgentOutputParser(),\n    ],\n    {\n      name: \"OpenAIFunctionsAgent\",\n      streamRunnable,\n      singleAction: true,\n    }\n  );\n  return agent;\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "getEndpoint", "language": "typescript", "path": "langchain/src/util/azure.ts", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate an endpoint URL for Azure OpenAI services based on provided configuration details.\n2. **Input**: An object containing configuration parameters such as deployment name, instance name, API key, base path, and a custom base URL for Azure OpenAI.\n3. **Output**: Returns a string representing the constructed URL or undefined if the necessary parameters are not provided.\n4. **Procedure**: \n   - Checks if the API key, base path, and deployment name are provided. If so, it constructs a URL using the base path and deployment name.\n   - If only the API key is provided, it validates the presence of the instance name and deployment name. If either is missing, it throws an error. If both are present, it constructs a URL using the instance name and deployment name.\n   - If none of the specific Azure parameters are provided, it returns a custom base URL if available.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/types/pdf-parse.d.ts\n/**\n * Type definitions adapted from pdfjs-dist\n * https://github.com/mozilla/pdfjs-dist/blob/master/types/src/display/api.d.ts\n */\n\ndeclare module \"pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js\" {\n  export type TypedArray =\n    | Int8Array\n    | Uint8Array\n    | Uint8ClampedArray\n    | Int16Array\n    | Uint16Array\n    | Int32Array\n    | Uint32Array\n    | Float32Array\n    | Float64Array;\n  export type BinaryData = TypedArray | ArrayBuffer | Array<number> | string;\n  export type RefProxy = {\n    num: number;\n    gen: number;\n  };\n  /**\n   * Document initialization / loading parameters object.\n   */\n  export type DocumentInitParameters = {\n    /**\n     * - The URL of the PDF.\n     */\n    url?: string | URL | undefined;\n    /**\n     * - Binary PDF data.\n     * Use TypedArrays (Uint8Array) to improve the memory usage. If PDF data is\n     * BASE64-encoded, use `atob()` to convert it to a binary string first.\n     *\n     * NOTE: If TypedArrays are used they will generally be transferred to the\n     * worker-thread. This will help reduce main-thread memory usage, however\n     * it will take ownership of the TypedArrays.\n     */\n    data?: BinaryData | undefined;\n    /**\n     * - Basic authentication headers.\n     */\n    httpHeaders?: Object | undefined;\n    /**\n     * - Indicates whether or not\n     * cross-site Access-Control requests should be made using credentials such\n     * as cookies or authorization headers. The default is `false`.\n     */\n    withCredentials?: boolean | undefined;\n    /**\n     * - For decrypting password-protected PDFs.\n     */\n    password?: string | undefined;\n    /**\n     * - The PDF file length. It's used for progress\n     * reports and range requests operations.\n     */\n    length?: number | undefined;\n    /**\n     * - Allows for using a custom range\n     * transport implementation.\n     */\n    range?: PDFDataRangeTransport | undefined;\n    /**\n     * - Specify maximum number of bytes fetched\n     * per range request. The default value is {@link DEFAULT_RANGE_CHUNK_SIZE }.\n     */\n    rangeChunkSize?: number | undefined;\n    /**\n     * - The worker that will be used for loading and\n     * parsing the PDF data.\n     */\n    worker?: PDFWorker | undefined;\n    /**\n     * - Controls the logging level; the constants\n     * from {@link VerbosityLevel } should be used.\n     */\n    verbosity?: number | undefined;\n    /**\n     * - The base URL of the document, used when\n     * attempting to recover valid absolute URLs for annotations, and outline\n     * items, that (incorrectly) only specify relative URLs.\n     */\n    docBaseUrl?: string | undefined;\n    /**\n     * - The URL where the predefined Adobe CMaps are\n     * located. Include the trailing slash.\n     */\n    cMapUrl?: string | undefined;\n    /**\n     * - Specifies if the Adobe CMaps are binary\n     * packed or not. The default value is `true`.\n     */\n    cMapPacked?: boolean | undefined;\n    /**\n     * - The factory that will be used when\n     * reading built-in CMap files. Providing a custom factory is useful for\n     * environments without Fetch API or `XMLHttpRequest` support, such as\n     * Node.js. The default value is {DOMCMapReaderFactory}.\n     */\n    CMapReaderFactory?: Object | undefined;\n    /**\n     * - When `true`, fonts that aren't\n     * embedded in the PDF document will fallback to a system font.\n     * The default value is `true` in web environments and `false` in Node.js;\n     * unless `disableFontFace === true` in which case this defaults to `false`\n     * regardless of the environment (to prevent completely broken fonts).\n     */\n    useSystemFonts?: boolean | undefined;\n    /**\n     * - The URL where the standard font\n     * files are located. Include the trailing slash.\n     */\n    standardFontDataUrl?: string | undefined;\n    /**\n     * - The factory that will be used\n     * when reading the standard font files. Providing a custom factory is useful\n     * for environments without Fetch API or `XMLHttpRequest` support, such as\n     * Node.js. The default value is {DOMStandardFontDataFactory}.\n     */\n    StandardFontDataFactory?: Object | undefined;\n    /**\n     * - Enable using the Fetch API in the\n     * worker-thread when reading CMap and standard font files. When `true`,\n     * the `CMapReaderFactory` and `StandardFontDataFactory` options are ignored.\n     * The default value is `true` in web environments and `false` in Node.js.\n     */\n    useWorkerFetch?: boolean | undefined;\n    /**\n     * - Reject certain promises, e.g.\n     * `getOperatorList`, `getTextContent`, and `RenderTask`, when the associated\n     * PDF data cannot be successfully parsed, instead of attempting to recover\n     * whatever possible of the data. The default value is `false`.\n     */\n    stopAtErrors?: boolean | undefined;\n    /**\n     * - The maximum allowed image size in total\n     * pixels, i.e. width * height. Images above this value will not be rendered.\n     * Use -1 for no limit, which is also the default value.\n     */\n    maxImageSize?: number | undefined;\n    /**\n     * - Determines if we can evaluate strings\n     * as JavaScript. Primarily used to improve performance of font rendering, and\n     * when parsing PDF functions. The default value is `true`.\n     */\n    isEvalSupported?: boolean | undefined;\n    /**\n     * - Determines if we can use\n     * `OffscreenCanvas` in the worker. Primarily used to improve performance of\n     * image conversion/rendering.\n     * The default value is `true` in web environments and `false` in Node.js.\n     */\n    isOffscreenCanvasSupported?: boolean | undefined;\n    /**\n     * - The integer value is used to\n     * know when an image must be resized (uses `OffscreenCanvas` in the worker).\n     * If it's -1 then a possibly slow algorithm is used to guess the max value.\n     */\n    canvasMaxAreaInBytes?: boolean | undefined;\n    /**\n     * - By default fonts are converted to\n     * OpenType fonts and loaded via the Font Loading API or `@font-face` rules.\n     * If disabled, fonts will be rendered using a built-in font renderer that\n     * constructs the glyphs with primitive path commands.\n     * The default value is `false` in web environments and `true` in Node.js.\n     */\n    disableFontFace?: boolean | undefined;\n    /**\n     * - Include additional properties,\n     * which are unused during rendering of PDF documents, when exporting the\n     * parsed font data from the worker-thread. This may be useful for debugging\n     * purposes (and backwards compatibility), but note that it will lead to\n     * increased memory usage. The default value is `false`.\n     */\n    fontExtraProperties?: boolean | undefined;\n    /**\n     * - Render Xfa forms if any.\n     * The default value is `false`.\n     */\n    enableXfa?: boolean | undefined;\n    /**\n     * - Specify an explicit document\n     * context to create elements with and to load resources, such as fonts,\n     * into. Defaults to the current document.\n     */\n    ownerDocument?: HTMLDocument | undefined;\n    /**\n     * - Disable range request loading of PDF\n     * files. When enabled, and if the server supports partial content requests,\n     * then the PDF will be fetched in chunks. The default value is `false`.\n     */\n    disableRange?: boolean | undefined;\n    /**\n     * - Disable streaming of PDF file data.\n     * By default PDF.js attempts to load PDF files in chunks. The default value\n     * is `false`.\n     */\n    disableStream?: boolean | undefined;\n    /**\n     * - Disable pre-fetching of PDF file\n     * data. When range requests are enabled PDF.js will automatically keep\n     * fetching more data even if it isn't needed to display the current page.\n     * The default value is `false`.\n     *\n     * NOTE: It is also necessary to disable streaming, see above, in order for\n     * disabling of pre-fetching to work correctly.\n     */\n    disableAutoFetch?: boolean | undefined;\n    /**\n     * - Enables special hooks for debugging PDF.js\n     * (see `web/debugger.js`). The default value is `false`.\n     */\n    pdfBug?: boolean | undefined;\n    /**\n     * - The factory instance that will be used\n     * when creating canvases. The default value is {new DOMCanvasFactory()}.\n     */\n    canvasFactory?: Object | undefined;\n    /**\n     * - A factory instance that will be used\n     * to create SVG filters when rendering some images on the main canvas.\n     */\n    filterFactory?: Object | undefined;\n  };\n  export type OnProgressParameters = {\n    /**\n     * - Currently loaded number of bytes.\n     */\n    loaded: number;\n    /**\n     * - Total number of bytes in the PDF file.\n     */\n    total: number;\n  };\n  /**\n   * Page getViewport parameters.\n   */\n  export type GetViewportParameters = {\n    /**\n     * - The desired scale of the viewport.\n     */\n    scale: number;\n    /**\n     * - The desired rotation, in degrees, of\n     * the viewport. If omitted it defaults to the page rotation.\n     */\n    rotation?: number | undefined;\n    /**\n     * - The horizontal, i.e. x-axis, offset.\n     * The default value is `0`.\n     */\n    offsetX?: number | undefined;\n    /**\n     * - The vertical, i.e. y-axis, offset.\n     * The default value is `0`.\n     */\n    offsetY?: number | undefined;\n    /**\n     * - If true, the y-axis will not be\n     * flipped. The default value is `false`.\n     */\n    dontFlip?: boolean | undefined;\n  };\n  /**\n   * Page getTextContent parameters.\n   */\n  export type getTextContentParameters = {\n    /**\n     * - When true include marked\n     * content items in the items array of TextContent. The default is `false`.\n     */\n    includeMarkedContent?: boolean | undefined;\n  };\n  /**\n   * Page text content.\n   */\n  export type TextContent = {\n    /**\n     * - Array of\n     * {@link TextItem } and {@link TextMarkedContent } objects. TextMarkedContent\n     * items are included when includeMarkedContent is true.\n     */\n    items: Array<TextItem | TextMarkedContent>;\n    /**\n     * - {@link TextStyle } objects,\n     * indexed by font name.\n     */\n    styles: {\n      [x: string]: TextStyle;\n    };\n  };\n  /**\n   * Page text content part.\n   */\n  export type TextItem = {\n    /**\n     * - Text content.\n     */\n    str: string;\n    /**\n     * - Text direction: 'ttb', 'ltr' or 'rtl'.\n     */\n    dir: string;\n    /**\n     * - Transformation matrix.\n     */\n    transform: Array<any>;\n    /**\n     * - Width in device space.\n     */\n    width: number;\n    /**\n     * - Height in device space.\n     */\n    height: number;\n    /**\n     * - Font name used by PDF.js for converted font.\n     */\n    fontName: string;\n    /**\n     * - Indicating if the text content is followed by a\n     * line-break.\n     */\n    hasEOL: boolean;\n  };\n  /**\n   * Page text marked content part.\n   */\n  export type TextMarkedContent = {\n    /**\n     * - Either 'beginMarkedContent',\n     * 'beginMarkedContentProps', or 'endMarkedContent'.\n     */\n    type: string;\n    /**\n     * - The marked content identifier. Only used for type\n     * 'beginMarkedContentProps'.\n     */\n    id: string;\n  };\n  /**\n   * Text style.\n   */\n  export type TextStyle = {\n    /**\n     * - Font ascent.\n     */\n    ascent: number;\n    /**\n     * - Font descent.\n     */\n    descent: number;\n    /**\n     * - Whether or not the text is in vertical mode.\n     */\n    vertical: boolean;\n    /**\n     * - The possible font family.\n     */\n    fontFamily: string;\n  };\n  /**\n   * Page annotation parameters.\n   */\n  export type GetAnnotationsParameters = {\n    /**\n     * - Determines the annotations that are fetched,\n     * can be 'display' (viewable annotations), 'print' (printable annotations),\n     * or 'any' (all annotations). The default value is 'display'.\n     */\n    intent?: string | undefined;\n  };\n  /**\n   * Page render parameters.\n   */\n  export type RenderParameters = {\n    /**\n     * - A 2D context of a DOM\n     * Canvas object.\n     */\n    canvasContext: CanvasRenderingContext2D;\n    /**\n     * - Rendering viewport obtained by calling\n     * the `PDFPageProxy.getViewport` method.\n     */\n    viewport: PageViewport;\n    /**\n     * - Rendering intent, can be 'display', 'print',\n     * or 'any'. The default value is 'display'.\n     */\n    intent?: string | undefined;\n    /**\n     * Controls which annotations are rendered\n     * onto the canvas, for annotations with appearance-data; the values from\n     * {@link AnnotationMode } should be used. The following values are supported:\n     * - `AnnotationMode.DISABLE`, which disables all annotations.\n     * - `AnnotationMode.ENABLE`, which includes all possible annotations (thus\n     * it also depends on the `intent`-option, see above).\n     * - `AnnotationMode.ENABLE_FORMS`, which excludes annotations that contain\n     * interactive form elements (those will be rendered in the display layer).\n     * - `AnnotationMode.ENABLE_STORAGE`, which includes all possible annotations\n     * (as above) but where interactive form elements are updated with data\n     * from the {@link AnnotationStorage }-instance; useful e.g. for printing.\n     * The default value is `AnnotationMode.ENABLE`.\n     */\n    annotationMode?: number | undefined;\n    /**\n     * - Additional transform, applied just\n     * before viewport transform.\n     */\n    transform?: any[] | undefined;\n    /**\n     * - Background\n     * to use for the canvas.\n     * Any valid `canvas.fillStyle` can be used: a `DOMString` parsed as CSS\n     * <color> value, a `CanvasGradient` object (a linear or radial gradient) or\n     * a `CanvasPattern` object (a repetitive image). The default value is\n     * 'rgb(255,255,255)'.\n     *\n     * NOTE: This option may be partially, or completely, ignored when the\n     * `pageColors`-option is used.\n     */\n    background?: string | CanvasGradient | CanvasPattern | undefined;\n    /**\n     * - Overwrites background and foreground colors\n     * with user defined ones in order to improve readability in high contrast\n     * mode.\n     */\n    pageColors?: Object | undefined;\n    /**\n     * -\n     * A promise that should resolve with an {@link OptionalContentConfig }created from `PDFDocumentProxy.getOptionalContentConfig`. If `null`,\n     * the configuration will be fetched automatically with the default visibility\n     * states set.\n     */\n    optionalContentConfigPromise?: Promise<OptionalContentConfig> | undefined;\n    /**\n     * - Map some\n     * annotation ids with canvases used to render them.\n     */\n    annotationCanvasMap?: Map<string, HTMLCanvasElement> | undefined;\n    printAnnotationStorage?: PrintAnnotationStorage | undefined;\n  };\n  /**\n   * Page getOperatorList parameters.\n   */\n  export type GetOperatorListParameters = {\n    /**\n     * - Rendering intent, can be 'display', 'print',\n     * or 'any'. The default value is 'display'.\n     */\n    intent?: string | undefined;\n    /**\n     * Controls which annotations are included\n     * in the operatorList, for annotations with appearance-data; the values from\n     * {@link AnnotationMode } should be used. The following values are supported:\n     * - `AnnotationMode.DISABLE`, which disables all annotations.\n     * - `AnnotationMode.ENABLE`, which includes all possible annotations (thus\n     * it also depends on the `intent`-option, see above).\n     * - `AnnotationMode.ENABLE_FORMS`, which excludes annotations that contain\n     * interactive form elements (those will be rendered in the display layer).\n     * - `AnnotationMode.ENABLE_STORAGE`, which includes all possible annotations\n     * (as above) but where interactive form elements are updated with data\n     * from the {@link AnnotationStorage }-instance; useful e.g. for printing.\n     * The default value is `AnnotationMode.ENABLE`.\n     */\n    annotationMode?: number | undefined;\n    printAnnotationStorage?: PrintAnnotationStorage | undefined;\n  };\n  /**\n   * Structure tree node. The root node will have a role \"Root\".\n   */\n  export type StructTreeNode = {\n    /**\n     * - Array of\n     * {@link StructTreeNode } and {@link StructTreeContent } objects.\n     */\n    children: Array<StructTreeNode | StructTreeContent>;\n    /**\n     * - element's role, already mapped if a role map exists\n     * in the PDF.\n     */\n    role: string;\n  };\n  /**\n   * Structure tree content.\n   */\n  export type StructTreeContent = {\n    /**\n     * - either \"content\" for page and stream structure\n     * elements or \"object\" for object references.\n     */\n    type: string;\n    /**\n     * - unique id that will map to the text layer.\n     */\n    id: string;\n  };\n  /**\n   * PDF page operator list.\n   */\n  export type PDFOperatorList = {\n    /**\n     * - Array containing the operator functions.\n     */\n    fnArray: Array<number>;\n    /**\n     * - Array containing the arguments of the\n     * functions.\n     */\n    argsArray: Array<any>;\n  };\n  export type PDFWorkerParameters = {\n    /**\n     * - The name of the worker.\n     */\n    name?: string | undefined;\n    /**\n     * - The `workerPort` object.\n     */\n    port?: Worker | undefined;\n    /**\n     * - Controls the logging level;\n     * the constants from {@link VerbosityLevel } should be used.\n     */\n    verbosity?: number | undefined;\n  };\n  /** @type {string} */\n  export const build: string;\n  export let DefaultCanvasFactory: typeof DOMCanvasFactory;\n  export let DefaultCMapReaderFactory: typeof DOMCMapReaderFactory;\n  export let DefaultFilterFactory: typeof DOMFilterFactory;\n  export let DefaultStandardFontDataFactory: typeof DOMStandardFontDataFactory;\n  /**\n   * @typedef { Int8Array | Uint8Array | Uint8ClampedArray |\n   *            Int16Array | Uint16Array |\n   *            Int32Array | Uint32Array | Float32Array |\n   *            Float64Array\n   * } TypedArray\n   */\n  /**\n   * @typedef { TypedArray | ArrayBuffer | Array<number> | string } BinaryData\n   */\n  /**\n   * @typedef {Object} RefProxy\n   * @property {number} num\n   * @property {number} gen\n   */\n  /**\n   * Document initialization / loading parameters object.\n   *\n   * @typedef {Object} DocumentInitParameters\n   * @property {string | URL} [url] - The URL of the PDF.\n   * @property {BinaryData} [data] - Binary PDF data.\n   *   Use TypedArrays (Uint8Array) to improve the memory usage. If PDF data is\n   *   BASE64-encoded, use `atob()` to convert it to a binary string first.\n   *\n   *   NOTE: If TypedArrays are used they will generally be transferred to the\n   *   worker-thread. This will help reduce main-thread memory usage, however\n   *   it will take ownership of the TypedArrays.\n   * @property {Object} [httpHeaders] - Basic authentication headers.\n   * @property {boolean} [withCredentials] - Indicates whether or not\n   *   cross-site Access-Control requests should be made using credentials such\n   *   as cookies or authorization headers. The default is `false`.\n   * @property {string} [password] - For decrypting password-protected PDFs.\n   * @property {number} [length] - The PDF file length. It's used for progress\n   *   reports and range requests operations.\n   * @property {PDFDataRangeTransport} [range] - Allows for using a custom range\n   *   transport implementation.\n   * @property {number} [rangeChunkSize] - Specify maximum number of bytes fetched\n   *   per range request. The default value is {@link DEFAULT_RANGE_CHUNK_SIZE}.\n   * @property {PDFWorker} [worker] - The worker that will be used for loading and\n   *   parsing the PDF data.\n   * @property {number} [verbosity] - Controls the logging level; the constants\n   *   from {@link VerbosityLevel} should be used.\n   * @property {string} [docBaseUrl] - The base URL of the document, used when\n   *   attempting to recover valid absolute URLs for annotations, and outline\n   *   items, that (incorrectly) only specify relative URLs.\n   * @property {string} [cMapUrl] - The URL where the predefined Adobe CMaps are\n   *   located. Include the trailing slash.\n   * @property {boolean} [cMapPacked] - Specifies if the Adobe CMaps are binary\n   *   packed or not. The default value is `true`.\n   * @property {Object} [CMapReaderFactory] - The factory that will be used when\n   *   reading built-in CMap files. Providing a custom factory is useful for\n   *   environments without Fetch API or `XMLHttpRequest` support, such as\n   *   Node.js. The default value is {DOMCMapReaderFactory}.\n   * @property {boolean} [useSystemFonts] - When `true`, fonts that aren't\n   *   embedded in the PDF document will fallback to a system font.\n   *   The default value is `true` in web environments and `false` in Node.js;\n   *   unless `disableFontFace === true` in which case this defaults to `false`\n   *   regardless of the environment (to prevent completely broken fonts).\n   * @property {string} [standardFontDataUrl] - The URL where the standard font\n   *   files are located. Include the trailing slash.\n   * @property {Object} [StandardFontDataFactory] - The factory that will be used\n   *   when reading the standard font files. Providing a custom factory is useful\n   *   for environments without Fetch API or `XMLHttpRequest` support, such as\n   *   Node.js. The default value is {DOMStandardFontDataFactory}.\n   * @property {boolean} [useWorkerFetch] - Enable using the Fetch API in the\n   *   worker-thread when reading CMap and standard font files. When `true`,\n   *   the `CMapReaderFactory` and `StandardFontDataFactory` options are ignored.\n   *   The default value is `true` in web environments and `false` in Node.js.\n   * @property {boolean} [stopAtErrors] - Reject certain promises, e.g.\n   *   `getOperatorList`, `getTextContent`, and `RenderTask`, when the associated\n   *   PDF data cannot be successfully parsed, instead of attempting to recover\n   *   whatever possible of the data. The default value is `false`.\n   * @property {number} [maxImageSize] - The maximum allowed image size in total\n   *   pixels, i.e. width * height. Images above this value will not be rendered.\n   *   Use -1 for no limit, which is also the default value.\n   * @property {boolean} [isEvalSupported] - Determines if we can evaluate strings\n   *   as JavaScript. Primarily used to improve performance of font rendering, and\n   *   when parsing PDF functions. The default value is `true`.\n   * @property {boolean} [isOffscreenCanvasSupported] - Determines if we can use\n   *   `OffscreenCanvas` in the worker. Primarily used to improve performance of\n   *   image conversion/rendering.\n   *   The default value is `true` in web environments and `false` in Node.js.\n   * @property {boolean} [canvasMaxAreaInBytes] - The integer value is used to\n   *   know when an image must be resized (uses `OffscreenCanvas` in the worker).\n   *   If it's -1 then a possibly slow algorithm is used to guess the max value.\n   * @property {boolean} [disableFontFace] - By default fonts are converted to\n   *   OpenType fonts and loaded via the Font Loading API or `@font-face` rules.\n   *   If disabled, fonts will be rendered using a built-in font renderer that\n   *   constructs the glyphs with primitive path commands.\n   *   The default value is `false` in web environments and `true` in Node.js.\n   * @property {boolean} [fontExtraProperties] - Include additional properties,\n   *   which are unused during rendering of PDF documents, when exporting the\n   *   parsed font data from the worker-thread. This may be useful for debugging\n   *   purposes (and backwards compatibility), but note that it will lead to\n   *   increased memory usage. The default value is `false`.\n   * @property {boolean} [enableXfa] - Render Xfa forms if any.\n   *   The default value is `false`.\n   * @property {HTMLDocument} [ownerDocument] - Specify an explicit document\n   *   context to create elements with and to load resources, such as fonts,\n   *   into. Defaults to the current document.\n   * @property {boolean} [disableRange] - Disable range request loading of PDF\n   *   files. When enabled, and if the server supports partial content requests,\n   *   then the PDF will be fetched in chunks. The default value is `false`.\n   * @property {boolean} [disableStream] - Disable streaming of PDF file data.\n   *   By default PDF.js attempts to load PDF files in chunks. The default value\n   *   is `false`.\n   * @property {boolean} [disableAutoFetch] - Disable pre-fetching of PDF file\n   *   data. When range requests are enabled PDF.js will automatically keep\n   *   fetching more data even if it isn't needed to display the current page.\n   *   The default value is `false`.\n   *\n   *   NOTE: It is also necessary to disable streaming, see above, in order for\n   *   disabling of pre-fetching to work correctly.\n   * @property {boolean} [pdfBug] - Enables special hooks for debugging PDF.js\n   *   (see `web/debugger.js`). The default value is `false`.\n   * @property {Object} [canvasFactory] - The factory instance that will be used\n   *   when creating canvases. The default value is {new DOMCanvasFactory()}.\n   * @property {Object} [filterFactory] - A factory instance that will be used\n   *   to create SVG filters when rendering some images on the main canvas.\n   */\n  /**\n   * This is the main entry point for loading a PDF and interacting with it.\n   *\n   * NOTE: If a URL is used to fetch the PDF data a standard Fetch API call (or\n   * XHR as fallback) is used, which means it must follow same origin rules,\n   * e.g. no cross-domain requests without CORS.\n   *\n   * @param {string | URL | TypedArray | ArrayBuffer | DocumentInitParameters}\n   *   src - Can be a URL where a PDF file is located, a typed array (Uint8Array)\n   *         already populated with data, or a parameter object.\n   * @returns {PDFDocumentLoadingTask}\n   */\n  export function getDocument(\n    src: string | URL | TypedArray | ArrayBuffer | DocumentInitParameters\n  ): PDFDocumentLoadingTask;\n  export class LoopbackPort {\n    postMessage(obj: any, transfer: any): void;\n    addEventListener(name: any, listener: any): void;\n    removeEventListener(name: any, listener: any): void;\n    terminate(): void;\n    #private;\n  }\n  /**\n   * @typedef {Object} OnProgressParameters\n   * @property {number} loaded - Currently loaded number of bytes.\n   * @property {number} total - Total number of bytes in the PDF file.\n   */\n  /**\n   * The loading task controls the operations required to load a PDF document\n   * (such as network requests) and provides a way to listen for completion,\n   * after which individual pages can be rendered.\n   */\n  export class PDFDocumentLoadingTask {\n    static \"__#16@#docId\": number;\n    _capability: import(\"../shared/util.js\").PromiseCapability;\n    _transport: any;\n    _worker: any;\n    /**\n     * Unique identifier for the document loading task.\n     * @type {string}\n     */\n    docId: string;\n    /**\n     * Whether the loading task is destroyed or not.\n     * @type {boolean}\n     */\n    destroyed: boolean;\n    /**\n     * Callback to request a password if a wrong or no password was provided.\n     * The callback receives two parameters: a function that should be called\n     * with the new password, and a reason (see {@link PasswordResponses}).\n     * @type {function}\n     */\n    onPassword: Function;\n    /**\n     * Callback to be able to monitor the loading progress of the PDF file\n     * (necessary to implement e.g. a loading bar).\n     * The callback receives an {@link OnProgressParameters} argument.\n     * @type {function}\n     */\n    onProgress: Function;\n    /**\n     * Promise for document loading task completion.\n     * @type {Promise<PDFDocumentProxy>}\n     */\n    get promise(): Promise<PDFDocumentProxy>;\n    /**\n     * Abort all network requests and destroy the worker.\n     * @returns {Promise<void>} A promise that is resolved when destruction is\n     *   completed.\n     */\n    destroy(): Promise<void>;\n  }\n  /**\n   * Proxy to a `PDFDocument` in the worker thread.\n   */\n  export class PDFDocumentProxy {\n    constructor(pdfInfo: any, transport: any);\n    _pdfInfo: any;\n    _transport: any;\n    /**\n     * @type {AnnotationStorage} Storage for annotation data in forms.\n     */\n    get annotationStorage(): AnnotationStorage;\n    /**\n     * @type {Object} The filter factory instance.\n     */\n    get filterFactory(): Object;\n    /**\n     * @type {number} Total number of pages in the PDF file.\n     */\n    get numPages(): number;\n    /**\n     * @type {Array<string, string|null>} A (not guaranteed to be) unique ID to\n     *   identify the PDF document.\n     *   NOTE: The first element will always be defined for all PDF documents,\n     *   whereas the second element is only defined for *modified* PDF documents.\n     */\n    get fingerprints(): string[];\n    /**\n     * @type {boolean} True if only XFA form.\n     */\n    get isPureXfa(): boolean;\n    /**\n     * NOTE: This is (mostly) intended to support printing of XFA forms.\n     *\n     * @type {Object | null} An object representing a HTML tree structure\n     *   to render the XFA, or `null` when no XFA form exists.\n     */\n    get allXfaHtml(): Object | null;\n    /**\n     * @param {number} pageNumber - The page number to get. The first page is 1.\n     * @returns {Promise<PDFPageProxy>} A promise that is resolved with\n     *   a {@link PDFPageProxy} object.\n     */\n    getPage(pageNumber: number): Promise<PDFPageProxy>;\n    /**\n     * @param {RefProxy} ref - The page reference.\n     * @returns {Promise<number>} A promise that is resolved with the page index,\n     *   starting from zero, that is associated with the reference.\n     */\n    getPageIndex(ref: RefProxy): Promise<number>;\n    /**\n     * @returns {Promise<Object<string, Array<any>>>} A promise that is resolved\n     *   with a mapping from named destinations to references.\n     *\n     * This can be slow for large documents. Use `getDestination` instead.\n     */\n    getDestinations(): Promise<{\n      [x: string]: Array<any>;\n    }>;\n    /**\n     * @param {string} id - The named destination to get.\n     * @returns {Promise<Array<any> | null>} A promise that is resolved with all\n     *   information of the given named destination, or `null` when the named\n     *   destination is not present in the PDF file.\n     */\n    getDestination(id: string): Promise<Array<any> | null>;\n    /**\n     * @returns {Promise<Array<string> | null>} A promise that is resolved with\n     *   an {Array} containing the page labels that correspond to the page\n     *   indexes, or `null` when no page labels are present in the PDF file.\n     */\n    getPageLabels(): Promise<Array<string> | null>;\n    /**\n     * @returns {Promise<string>} A promise that is resolved with a {string}\n     *   containing the page layout name.\n     */\n    getPageLayout(): Promise<string>;\n    /**\n     * @returns {Promise<string>} A promise that is resolved with a {string}\n     *   containing the page mode name.\n     */\n    getPageMode(): Promise<string>;\n    /**\n     * @returns {Promise<Object | null>} A promise that is resolved with an\n     *   {Object} containing the viewer preferences, or `null` when no viewer\n     *   preferences are present in the PDF file.\n     */\n    getViewerPreferences(): Promise<Object | null>;\n    /**\n     * @returns {Promise<any | null>} A promise that is resolved with an {Array}\n     *   containing the destination, or `null` when no open action is present\n     *   in the PDF.\n     */\n    getOpenAction(): Promise<any | null>;\n    /**\n     * @returns {Promise<any>} A promise that is resolved with a lookup table\n     *   for mapping named attachments to their content.\n     */\n    getAttachments(): Promise<any>;\n    /**\n     * @returns {Promise<Array<string> | null>} A promise that is resolved with\n     *   an {Array} of all the JavaScript strings in the name tree, or `null`\n     *   if no JavaScript exists.\n     */\n    getJavaScript(): Promise<Array<string> | null>;\n    /**\n     * @returns {Promise<Object | null>} A promise that is resolved with\n     *   an {Object} with the JavaScript actions:\n     *     - from the name tree (like getJavaScript);\n     *     - from A or AA entries in the catalog dictionary.\n     *   , or `null` if no JavaScript exists.\n     */\n    getJSActions(): Promise<Object | null>;\n    /**\n     * @typedef {Object} OutlineNode\n     * @property {string} title\n     * @property {boolean} bold\n     * @property {boolean} italic\n     * @property {Uint8ClampedArray} color - The color in RGB format to use for\n     *   display purposes.\n     * @property {string | Array<any> | null} dest\n     * @property {string | null} url\n     * @property {string | undefined} unsafeUrl\n     * @property {boolean | undefined} newWindow\n...\n// Path: langchain/src/types/expression-parser.d.ts\ndeclare interface ParseOptions {\n  filename?: string;\n  startRule?: \"Start\";\n  tracer?: any;\n  [key: string]: any;\n}\ndeclare type ParseFunction = <Options extends ParseOptions>(\n  input: string,\n  options?: Options\n) => Options extends { startRule: infer StartRule }\n  ? StartRule extends \"Start\"\n    ? Start\n    : Start\n  : Start;\n\n// These types were autogenerated by ts-pegjs\ndeclare type Start = Program;\ndeclare type Identifier = IdentifierName;\ndeclare type IdentifierName = { type: \"Identifier\"; name: string };\ndeclare type Literal =\n  | NullLiteral\n  | BooleanLiteral\n  | NumericLiteral\n  | StringLiteral;\ndeclare type NullLiteral = { type: \"NullLiteral\"; value: null };\ndeclare type BooleanLiteral =\n  | { type: \"BooleanLiteral\"; value: true }\n  | { type: \"BooleanLiteral\"; value: false };\ndeclare type NumericLiteral = DecimalLiteral;\ndeclare type DecimalLiteral = { type: \"NumericLiteral\"; value: number };\ndeclare type StringLiteral = { type: \"StringLiteral\"; value: string };\ndeclare type PrimaryExpression =\n  | Identifier\n  | Literal\n  | ArrayExpression\n  | ObjectExpression\n  | Expression;\ndeclare type ArrayExpression = {\n  type: \"ArrayExpression\";\n  elements: ElementList;\n};\ndeclare type ElementList = PrimaryExpression[];\ndeclare type ObjectExpression =\n  | { type: \"ObjectExpression\"; properties: [] }\n  | { type: \"ObjectExpression\"; properties: PropertyNameAndValueList };\ndeclare type PropertyNameAndValueList = PrimaryExpression[];\ndeclare type PropertyAssignment = {\n  type: \"PropertyAssignment\";\n  key: PropertyName;\n  value: Expression;\n  kind: \"init\";\n};\ndeclare type PropertyName = IdentifierName | StringLiteral | NumericLiteral;\ndeclare type MemberExpression =\n  | {\n      type: \"MemberExpression\";\n      property: StringLiteral;\n      computed: true;\n      object: MemberExpression | Identifier | StringLiteral;\n    }\n  | {\n      type: \"MemberExpression\";\n      property: Identifier;\n      computed: false;\n      object: MemberExpression | Identifier | StringLiteral;\n    };\ndeclare type CallExpression = {\n  type: \"CallExpression\";\n  arguments: Arguments;\n  callee: MemberExpression | Identifier;\n};\ndeclare type Arguments = PrimaryExpression[];\ndeclare type Expression = CallExpression | MemberExpression;\ndeclare type ExpressionStatement = {\n  type: \"ExpressionStatement\";\n  expression: Expression;\n};\ndeclare type Program = { type: \"Program\"; body: ExpressionStatement };\ndeclare type ExpressionNode =\n  | Program\n  | ExpressionStatement\n  | ArrayExpression\n  | BooleanLiteral\n  | CallExpression\n  | Identifier\n  | MemberExpression\n  | NumericLiteral\n  | ObjectExpression\n  | PropertyAssignment\n  | NullLiteral\n  | StringLiteral;\n\n// Path: langchain/src/util/async_caller.ts\nexport * from \"@langchain/core/utils/async_caller\";\n\n// Path: langchain/src/util/types.ts\n/**\n * Represents a string value with autocompleted, but not required, suggestions.\n */\n\nexport type StringWithAutocomplete<T> = T | (string & Record<never, never>);\n\n// Path: langchain/src/util/axios-types.ts\nimport type { AxiosRequestConfig } from \"axios\";\nimport { EventSourceMessage } from \"@langchain/community/utils/event_source_parse\";\n\nexport interface StreamingAxiosRequestConfig extends AxiosRequestConfig {\n  responseType: \"stream\";\n\n  /**\n   * Called when a message is received. NOTE: Unlike the default browser\n   * EventSource.onmessage, this callback is called for _all_ events,\n   * even ones with a custom `event` field.\n   */\n  onmessage?: (ev: EventSourceMessage) => void;\n}\n\nexport type StreamingAxiosConfiguration =\n  | StreamingAxiosRequestConfig\n  | AxiosRequestConfig;\n\n// Path: langchain/src/util/tiktoken.ts\nexport * from \"@langchain/core/utils/tiktoken\";\n\n// Path: langchain/src/util/stream.ts\nexport * from \"@langchain/core/utils/stream\";\n\n// Path: langchain/src/util/azure.ts\nexport interface OpenAIEndpointConfig {\n  azureOpenAIApiDeploymentName?: string;\n  azureOpenAIApiInstanceName?: string;\n  azureOpenAIApiKey?: string;\n  azureOpenAIBasePath?: string;\n  baseURL?: string;\n}\n\n/**\n * This function generates an endpoint URL for (Azure) OpenAI\n * based on the configuration parameters provided.\n *\n * @param {OpenAIEndpointConfig} config - The configuration object for the (Azure) endpoint.\n *\n * @property {string} config.azureOpenAIApiDeploymentName - The deployment name of Azure OpenAI.\n * @property {string} config.azureOpenAIApiInstanceName - The instance name of Azure OpenAI.\n * @property {string} config.azureOpenAIApiKey - The API Key for Azure OpenAI.\n * @property {string} config.azureOpenAIBasePath - The base path for Azure OpenAI.\n * @property {string} config.baseURL - Some other custom base path URL.\n *\n * The function operates as follows:\n * - If both `azureOpenAIBasePath` and `azureOpenAIApiDeploymentName` (plus `azureOpenAIApiKey`) are provided, it returns an URL combining these two parameters (`${azureOpenAIBasePath}/${azureOpenAIApiDeploymentName}`).\n * - If `azureOpenAIApiKey` is provided, it checks for `azureOpenAIApiInstanceName` and `azureOpenAIApiDeploymentName` and throws an error if any of these is missing. If both are provided, it generates an URL incorporating these parameters.\n * - If none of the above conditions are met, return any custom `baseURL`.\n * - The function returns the generated URL as a string, or undefined if no custom paths are specified.\n *\n * @throws Will throw an error if the necessary parameters for generating the URL are missing.\n *\n * @returns {string | undefined} The generated (Azure) OpenAI endpoint URL.\n */\nexport \nfunction getEndpoint(config: OpenAIEndpointConfig) {\n  const {\n    azureOpenAIApiDeploymentName,\n    azureOpenAIApiInstanceName,\n    azureOpenAIApiKey,\n    azureOpenAIBasePath,\n    baseURL,\n  } = config;\n\n  if (\n    azureOpenAIApiKey &&\n    azureOpenAIBasePath &&\n    azureOpenAIApiDeploymentName\n  ) {\n    return `${azureOpenAIBasePath}/${azureOpenAIApiDeploymentName}`;\n  }\n\n  if (azureOpenAIApiKey) {\n    if (!azureOpenAIApiInstanceName) {\n      throw new Error(\n        \"azureOpenAIApiInstanceName is required when using azureOpenAIApiKey\"\n      );\n    }\n    if (!azureOpenAIApiDeploymentName) {\n      throw new Error(\n        \"azureOpenAIApiDeploymentName is a required parameter when using azureOpenAIApiKey\"\n      );\n    }\n    return `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${azureOpenAIApiDeploymentName}`;\n  }\n\n  return baseURL;\n}\n\n// Path: langchain/src/util/convex.ts\nimport { logVersion010MigrationWarning } from \"../util/entrypoint_deprecation.js\";\n\n/* #__PURE__ */ logVersion010MigrationWarning({\n  oldEntrypointName: \"util/convex\",\n});\nexport * from \"@langchain/community/util/convex\";\n\n// Path: langchain/src/util/axios-fetch-adapter.d.ts\n// eslint-disable-next-line import/no-extraneous-dependencies\nimport { AxiosRequestConfig, AxiosPromise } from \"axios\";\n\nexport default function fetchAdapter(config: AxiosRequestConfig): AxiosPromise;\n\n// Path: langchain/src/util/tests/googlevertexai-connection.test.ts\nimport { expect, test } from \"@jest/globals\";\nimport {\n  simpleValue,\n  complexValue,\n  GoogleVertexAIStream,\n} from \"../googlevertexai-connection.js\";\n\ndescribe(\"VertexAI Connection\", () => {\n  describe(\"complexValue\", () => {\n    test(\"int_val\", () => {\n      const result = complexValue(1);\n      expect(result).toEqual({ int_val: 1 });\n    });\n\n    test(\"float_val\", () => {\n      const result = complexValue(2.5);\n      expect(result).toEqual({ float_val: 2.5 });\n    });\n\n    test(\"string_val\", () => {\n      const result = complexValue(\"hello\");\n      expect(result).toEqual({ string_val: [\"hello\"] });\n    });\n\n    test(\"list_val\", () => {\n      const input = [1, 2.5, \"three\"];\n      const result = complexValue(input);\n      expect(result).toEqual({\n        list_val: [\n          { int_val: 1 },\n          { float_val: 2.5 },\n          { string_val: [\"three\"] },\n        ],\n      });\n    });\n\n    test(\"struct_val\", () => {\n      const input = {\n        content: \"prompt\",\n        author: \"the author\",\n      };\n      const result = complexValue(input);\n      expect(result).toEqual({\n        struct_val: {\n          content: { string_val: [\"prompt\"] },\n          author: { string_val: [\"the author\"] },\n        },\n      });\n    });\n  });\n\n  describe(\"simpleValue\", () => {\n    test(\"boolVal true\", () => {\n      const input = {\n        boolVal: [true],\n      };\n      const result = simpleValue(input);\n      expect(result).toEqual(true);\n    });\n\n    test(\"boolVal false\", () => {\n      const input = {\n        boolVal: [false],\n      };\n      const result = simpleValue(input);\n      expect(result).toEqual(false);\n    });\n\n    test(\"stringVal\", () => {\n      const input = {\n        stringVal: [\"they will suffer\"],\n      };\n      const result = simpleValue(input);\n      expect(result).toEqual(\"they will suffer\");\n    });\n\n    test(\"structVal\", () => {\n      const input = {\n        structVal: {\n          citationMetadata: {\n            structVal: {\n              citations: {},\n            },\n          },\n          safetyAttributes: {\n            structVal: {\n              categories: {},\n              scores: {},\n              blocked: {\n                boolVal: [false],\n              },\n            },\n          },\n          content: {\n            stringVal: [\"RESPONSE\"],\n          },\n        },\n      };\n      const result = simpleValue(input);\n      expect(result).toEqual({\n        citationMetadata: {\n          citations: {},\n        },\n        safetyAttributes: {\n          categories: {},\n          scores: {},\n          blocked: false,\n        },\n        content: \"RESPONSE\",\n      });\n    });\n  });\n\n  describe(\"GoogleVertexAIStream\", () => {\n    test(\"skipToOpenBracket\", () => {\n      const gen = new GoogleVertexAIStream();\n      gen._buffer = '[{\"hi\": \"there\"}]';\n      gen._skipTo(\"{\");\n      expect(gen._buffer).toEqual('{\"hi\": \"there\"}]');\n    });\n\n    test(\"getFullObject\", () => {\n      const gen = new GoogleVertexAIStream();\n      gen._buffer = '[{\"hi\": \"there\"}]';\n      gen._skipTo(\"{\");\n      const obj = gen._getFullObject();\n      expect(obj).toEqual({ hi: \"there\" });\n      expect(gen._buffer).toEqual(\"]\");\n    });\n\n    test(\"getFullObject incomplete\", () => {\n      const gen = new GoogleVertexAIStream();\n      gen._buffer = '[{\"hi\": \"ther';\n      gen._skipTo(\"{\");\n      const obj = gen._getFullObject();\n      expect(obj).toEqual(null);\n      expect(gen._buffer).toEqual('{\"hi\": \"ther');\n    });\n\n    test(\"getFullObject strings\", () => {\n      const gen = new GoogleVertexAIStream();\n      gen._buffer = '[{\"frown\": \"}:\"}]';\n      gen._skipTo(\"{\");\n      const obj = gen._getFullObject();\n      expect(obj).toEqual({ frown: \"}:\" });\n      expect(gen._buffer).toEqual(\"]\");\n    });\n\n    test(\"getFullObject complex\", () => {\n      const gen = new GoogleVertexAIStream();\n      gen._buffer = JSON.stringify(\n        [\n          {\n            alpha: {\n              uno: 1,\n              dos: 2,\n            },\n            bravo: {\n              here: 3,\n              there: 4,\n            },\n          },\n        ],\n        null,\n        2\n      );\n      gen._skipTo(\"{\");\n      const obj = gen._getFullObject();\n      expect(obj).toEqual({\n        alpha: {\n          uno: 1,\n          dos: 2,\n        },\n        bravo: {\n          here: 3,\n          there: 4,\n        },\n      });\n      expect(gen._buffer).toEqual(\"\\n]\");\n    });\n\n    test(\"getFullObject bufered\", () => {\n      const gen = new GoogleVertexAIStream();\n      const overall = JSON.stringify(\n        [\n          {\n            alpha: {\n              uno: 1,\n              dos: 2,\n            },\n            bravo: {\n              here: 3,\n              there: 4,\n            },\n          },\n        ],\n        null,\n        2\n      );\n      const part1 = overall.substring(0, overall.length / 2);\n      const part2 = overall.substring(overall.length / 2);\n\n      gen._buffer += part1;\n      gen._skipTo(\"{\");\n      const obj1 = gen._getFullObject();\n      expect(obj1).toEqual(null);\n\n      gen._buffer += part2;\n      gen._skipTo(\"{\");\n      const obj2 = gen._getFullObject();\n      expect(obj2).toEqual({\n        alpha: {\n          uno: 1,\n          dos: 2,\n        },\n        bravo: {\n          here: 3,\n          there: 4,\n        },\n      });\n      expect(gen._buffer).toEqual(\"\\n]\");\n    });\n\n    test(\"end-to-end\", async () => {\n      const src = [\n        `[{\n  \"outputs\": [\n    {\n      \"structVal\": {\n        \"safetyAttributes\": {\n          \"structVal\": {\n            \"scores\": {\n              \"listVal\": [\n                {\n                  \"doubleVal\": [\n                    0.5\n                  ]\n                }\n              ]\n            },\n            \"categories\": {\n              \"listVal\": [\n                {\n                  \"stringVal\": [\n                    \"Religion & Belief\"\n                  ]\n                }\n              ]\n            },\n            \"blocked\": {\n              \"boolVal\": [\n                false\n              ]\n            }\n          }\n        },\n        \"content\": {\n          \"stringVal\": [\n            \" The answer to life, the universe, and everything is 42. This an\"\n          ]\n        },\n        \"citationMetadata\": {\n          \"structVal\": {\n            \"citations\": {}\n          }\n        }\n      }\n    }\n  ]\n},\n`,\n        `{\n  \"outputs\": [\n    {\n      \"structVal\": {\n        \"citationMetadata\": {\n          \"structVal\": {\n            \"citations\": {}\n          }\n        },\n        \"safetyAttributes\": {\n          \"structVal\": {\n            \"categories\": {\n              \"listVal\": [\n                {\n                  \"stringVal\": [\n                    \"Religion & Belief\"\n                  ]\n                }\n              ]\n            },\n            \"blocked\": {\n              \"boolVal\": [\n                false\n              ]\n            },\n            \"scores\": {\n              \"listVal\": [\n                {\n                  \"doubleVal\": [\n                    0.3\n                  ]\n                }\n              ]\n            }\n          }\n        },\n        \"content\": {\n          \"stringVal\": [\n            \"swer was given by Deep Thought, a supercomputer in Douglas Adams\"\n          ]\n        }\n      }\n    }\n  ]\n}\n`,\n        `,{\n  \"outputs\": [\n    {\n      \"structVal\": {\n        \"citationMetadata\": {\n          \"structVal\": {\n            \"citations\": {}\n          }\n        },\n        \"`,\n        `content\": {\n\"stringVal\": [\n  \"`,\n        `'`,\n        `s`,\n        ` The `,\n        `H`,\n        `i`,\n        `tchhiker's Guide to the Galaxy.\"\n]\n      },\n      \"safetyAttributes\": {\n        \"structVal\": {\n          \"blocked\": {\n            \"boolVal\": [\n              false\n            ]\n          },\n          \"categories\": {\n            \"listVal\": [\n              {\n                \"stringVal\": [\n                  \"Religion & Belief\"\n                ]\n              }\n            ]\n          },\n          \"scores\": {\n            \"listVal\": [\n              {\n                \"doubleVal\": [\n                  0.5\n                ]\n              }\n            ]\n          }\n        }\n      }\n    }\n  }\n  ]\n}\n`,\n        `]`,\n      ];\n      const gen = new GoogleVertexAIStream();\n\n      // Set future events to append things to the buffer\n      let co = 0;\n      for (let i = 0; i < src.length; i += 1) {\n        co += 1;\n        setTimeout(() => {\n          gen.appendBuffer(src[i]);\n        }, co * 250);\n      }\n      co += 1;\n      setTimeout(() => {\n        gen.closeBuffer();\n      }, co * 250);\n\n      const chunks = [];\n      while (!gen.streamDone) {\n        const chunk = await gen.nextChunk();\n        chunks.push(chunk);\n      }\n      expect(chunks).toHaveLength(4);\n      expect(chunks[0]?.outputs[0]?.content).toBe(\n        \" The answer to life, the universe, and everything is 42. This an\"\n      );\n      expect(chunks[1]?.outputs[0]?.content).toBe(\n        \"swer was given by Deep Thought, a supercomputer in Douglas Adams\"\n      );\n      expect(chunks[2]?.outputs[0]?.content).toBe(\n        \"'s The Hitchhiker's Guide to the Galaxy.\"\n      );\n      expect(chunks[3]).toBeNull();\n    });\n  });\n});\n\n// Path: langchain/src/util/tests/set.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { difference, intersection, union } from \"../set.js\";\n\ntest(\"difference\", () => {\n  const set1 = new Set([\"a\", \"b\"]);\n  const set2 = new Set([\"b\", \"c\"]);\n\n  const resultSet = difference(set1, set2);\n  expect(resultSet).toMatchInlineSnapshot(`\n      Set {\n        \"a\",\n      }\n    `);\n});\n\ntest(\"intersection\", () => {\n  const set1 = new Set([\"a\", \"b\", \"c\", \"d\"]);\n  const set2 = new Set([\"b\", \"c\", \"e\"]);\n\n  const resultSet = intersection(set1, set2);\n  expect(resultSet).toMatchInlineSnapshot(`\n      Set {\n        \"b\",\n        \"c\",\n      }\n    `);\n});\n\ntest(\"union\", () => {\n  const set1 = new Set([\"a\", \"b\"]);\n  const set2 = new Set([\"c\", \"d\"]);\n\n  const resultSet = union(set1, set2);\n  expect(resultSet).toMatchInlineSnapshot(`\n      Set {\n        \"a\",\n        \"b\",\n        \"c\",\n        \"d\",\n      }\n    `);\n});\n\n// Path: langchain/src/util/tests/async_caller.int.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { OpenAI } from \"@langchain/openai\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\n\ntest(\"AsyncCaller.call passes on arguments and returns return value\", async () => {\n  const caller = new AsyncCaller({});\n  const callable = () => fetch(\"https://langchain.com/\");\n\n  const resultDirect = await callable();\n  const resultWrapped = await caller.call(callable);\n\n  expect(resultDirect.status).toEqual(200);\n  expect(resultWrapped.status).toEqual(200);\n});\n\ntest(\"AsyncCaller doesn't retry on axios error 401\", async () => {\n  const llm = new OpenAI({ openAIApiKey: \"invalid\" });\n\n  await expect(() => llm.call(\"test\")).rejects.toThrowError();\n}, 5000);\n\ntest(\"AsyncCaller doesn't retry on timeout\", async () => {\n  const caller = new AsyncCaller({});\n  const callable = () =>\n    fetch(\"https://langchain.com/?sleep=1000\", {\n      signal: AbortSignal.timeout(10),\n    });\n\n  await expect(() => caller.call(callable)).rejects.toThrowError(\n    \"TimeoutError: The operation was aborted due to timeout\"\n  );\n}, 5000);\n\ntest(\"AsyncCaller doesn't retry on signal abort\", async () => {\n  const controller = new AbortController();\n  const caller = new AsyncCaller({});\n  const callable = () => {\n    const ret = fetch(\"https://langchain.com/?sleep=1000\", {\n      signal: controller.signal,\n    });\n\n    controller.abort();\n\n    return ret;\n  };\n\n  await expect(() => caller.call(callable)).rejects.toThrowError(\n    \"AbortError: This operation was aborted\"\n  );\n}, 5000);\n\n// Path: langchain/src/util/tests/sql_utils.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { DataSource } from \"typeorm\";\nimport {\n  getPromptTemplateFromDataSource,\n  verifyIgnoreTablesExistInDatabase,\n  verifyIncludeTablesExistInDatabase,\n} from \"../sql_utils.js\";\nimport { SQL_SQLITE_PROMPT } from \"../../chains/sql_db/sql_db_prompt.js\";\n\ntest(\"Find include tables when there are there\", () => {\n  const includeTables = [\"user\", \"score\"];\n  const allTables = [\n    { tableName: \"plop\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"score\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"user\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"log\", columns: [{ columnName: \"id\" }] },\n  ];\n\n  expect(() =>\n    verifyIncludeTablesExistInDatabase(allTables, includeTables)\n  ).not.toThrow();\n});\n\ntest(\"Throw Error when include tables are not there\", () => {\n  const includeTables = [\"user\", \"score\"];\n  const allTables = [\n    { tableName: \"plop\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"score\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"log\", columns: [{ columnName: \"id\" }] },\n  ];\n\n  expect(() =>\n    verifyIncludeTablesExistInDatabase(allTables, includeTables)\n  ).toThrow();\n});\n\ntest(\"Find include tables when there are there\", () => {\n  const includeTables = [\"user\", \"score\"];\n  const allTables = [\n    { tableName: \"user\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"plop\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"score\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"log\", columns: [{ columnName: \"id\" }] },\n  ];\n\n  expect(() =>\n    verifyIgnoreTablesExistInDatabase(allTables, includeTables)\n  ).not.toThrow();\n});\n\ntest(\"Throw Error when include tables are not there\", () => {\n  const includeTables = [\"user\", \"score\"];\n  const allTables = [\n    { tableName: \"plop\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"score\", columns: [{ columnName: \"id\" }] },\n    { tableName: \"log\", columns: [{ columnName: \"id\" }] },\n  ];\n\n  expect(() =>\n    verifyIgnoreTablesExistInDatabase(allTables, includeTables)\n  ).toThrow();\n});\n\ntest.skip(\"return sqlite template when the DataSource is sqlite\", () => {\n  const datasource = new DataSource({\n    type: \"sqlite\",\n    database: \"Chinook.db\",\n  });\n\n  const promptTemplate = getPromptTemplateFromDataSource(datasource);\n  expect(promptTemplate).toEqual(SQL_SQLITE_PROMPT);\n});\n\n// Path: langchain/src/util/tests/azure.test.ts\nimport { getEndpoint, OpenAIEndpointConfig } from \"@langchain/openai\";\n\ndescribe(\"getEndpoint\", () => {\n  it(\"should return the correct endpoint with azureOpenAIBasePath and azureOpenAIApiDeploymentName\", () => {\n    const config: OpenAIEndpointConfig = {\n      azureOpenAIApiKey: \"API-KEY\",\n      azureOpenAIApiDeploymentName: \"deploymentName\",\n      azureOpenAIBasePath:\n        \"https://westeurope.api.cognitive.microsoft.com/openai/deployments\",\n    };\n    const result = getEndpoint(config);\n    expect(result).toBe(\n      \"https://westeurope.api.cognitive.microsoft.com/openai/deployments/deploymentName\"\n    );\n  });\n\n  it(\"should return the correct endpoint with azureOpenAIApiKey, azureOpenAIApiInstanceName, and azureOpenAIApiDeploymentName\", () => {\n    const config: OpenAIEndpointConfig = {\n      azureOpenAIApiKey: \"key\",\n      azureOpenAIApiInstanceName: \"instanceName\",\n      azureOpenAIApiDeploymentName: \"deploymentName\",\n    };\n    const result = getEndpoint(config);\n    expect(result).toBe(\n      \"https://instanceName.openai.azure.com/openai/deployments/deploymentName\"\n    );\n  });\n\n  it(\"should throw error when azureOpenAIApiInstanceName is missing with azureOpenAIApiKey\", () => {\n    const config: OpenAIEndpointConfig = {\n      azureOpenAIApiKey: \"key\",\n      azureOpenAIApiDeploymentName: \"deploymentName\",\n    };\n    expect(() => getEndpoint(config)).toThrowError(\n      \"azureOpenAIApiInstanceName is required when using azureOpenAIApiKey\"\n    );\n  });\n\n  it(\"should throw error when azureOpenAIApiDeploymentName is missing with azureOpenAIApiKey\", () => {\n    const config: OpenAIEndpointConfig = {\n      azureOpenAIApiKey: \"key\",\n      azureOpenAIApiInstanceName: \"instanceName\",\n    };\n    expect(() => getEndpoint(config)).toThrowError(\n      \"azureOpenAIApiDeploymentName is a required parameter when using azureOpenAIApiKey\"\n    );\n  });\n\n  it(\"should return the custom basePath when neither azureOpenAIBasePath nor azureOpenAIApiKey is provided\", () => {\n    const config: OpenAIEndpointConfig = {\n      baseURL: \"https://basepath.com\",\n    };\n    const result = getEndpoint(config);\n    expect(result).toBe(\"https://basepath.com\");\n  });\n});\n\n// Path: langchain/src/util/tests/openai-stream.test.ts\nimport { describe, it, expect, jest } from \"@jest/globals\";\nimport { AxiosResponse } from \"axios\";\nimport fetchAdapter from \"../axios-fetch-adapter.js\";\n\nconst mockFetchForOpenAIStream = async ({\n  chunks,\n  status,\n  contentType,\n}: {\n  chunks: Array<string>;\n  status: number;\n  contentType: string;\n}) => {\n  // Mock stream response chunks.\n  const stream = new ReadableStream({\n    async start(controller) {\n      chunks.forEach((chunk) => {\n        controller.enqueue(new TextEncoder().encode(chunk));\n      });\n      controller.close();\n    },\n  });\n\n  // Mock Fetch API call.\n  jest.spyOn(global, \"fetch\").mockImplementation(\n    async () =>\n      new Response(stream, {\n        status,\n        headers: {\n          \"Content-Type\": contentType,\n        },\n      })\n  );\n\n  let error: Error | null = null;\n  let done = false;\n  const receivedChunks: Array<string> = [];\n  const resp = await fetchAdapter({\n    url: \"https://example.com\",\n...\n", "needle_token_start": 10495, "needle_token_end": 10781, "code_context_ntokens": 16305, "output": ["```typescript\n/**\n * This function generates an endpoint URL for (Azure) OpenAI\n * based on the configuration parameters provided.\n *\n * @param {OpenAIEndpointConfig} config - The configuration object for the (Azure) endpoint.\n *\n * @property {string} config.azureOpenAIApiDeploymentName - The deployment name of Azure OpenAI.\n * @property {string} config.azureOpenAIApiInstanceName - The instance name of Azure OpenAI.\n * @property {string} config.azureOpenAIApiKey - The API Key for Azure OpenAI.\n * @property {string} config.azureOpenAIBasePath - The base path for Azure OpenAI.\n * @property {string} config.baseURL - Some other custom base path URL.\n *\n * The function operates as follows:\n * - If both `azureOpenAIBasePath` and `azureOpenAIApiDeploymentName` (plus `azureOpenAIApiKey`) are provided, it returns an URL combining these two parameters (`${azureOpenAIBasePath}/${azureOpenAIApiDeploymentName}`).\n * - If `azureOpenAIApiKey` is provided, it checks for `azureOpenAIApiInstanceName` and `azureOpenAIApiDeploymentName` and throws an error if any of these is missing. If both are provided, it generates an URL incorporating these parameters.\n * - If none of the above conditions are met, return any custom `baseURL`.\n * - The function returns the generated URL as a string, or undefined if no custom paths are specified.\n *\n * @throws Will throw an error if the necessary parameters for generating the URL are missing.\n *\n * @returns {string | undefined} The generated (Azure) OpenAI endpoint URL.\n */\nexport function getEndpoint(config: OpenAIEndpointConfig) {\n  const {\n    azureOpenAIApiDeploymentName,\n    azureOpenAIApiInstanceName,\n    azureOpenAIApiKey,\n    azureOpenAIBasePath,\n    baseURL,\n  } = config;\n\n  if (\n    azureOpenAIApiKey &&\n    azureOpenAIBasePath &&\n    azureOpenAIApiDeploymentName\n  ) {\n    return `${azureOpenAIBasePath}/${azureOpenAIApiDeploymentName}`;\n  }\n\n  if (azureOpenAIApiKey) {\n    if (!azureOpenAIApiInstanceName) {\n      throw new Error(\n        \"azureOpenAIApiInstanceName is required when using azureOpenAIApiKey\"\n      );\n    }\n    if (!azureOpenAIApiDeploymentName) {\n      throw new Error(\n        \"azureOpenAIApiDeploymentName is a required parameter when using azureOpenAIApiKey\"\n      );\n    }\n    return `https://${azureOpenAIApiInstanceName}.openai.azure.com/openai/deployments/${azureOpenAIApiDeploymentName}`;\n  }\n\n  return baseURL;\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "isOffTheShelfEvaluator", "language": "typescript", "path": "langchain/src/smith/config.ts", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine if a given evaluator is a predefined type or a specific configuration object.\n2. **Input**: The function accepts a single parameter which can be a string, a configuration object, or an evaluator function.\n3. **Output**: It returns a boolean value indicating whether the input parameter is a predefined type or a configuration object.\n4. **Procedure**: The function checks the type of the input parameter. If it is a string or an object with a specific property (`evaluatorType`), it returns true, indicating that the input is a predefined type or a configuration object. Otherwise, it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/experimental/openai_assistant/index.ts\nimport { type ClientOptions, OpenAIClient } from \"@langchain/openai\";\nimport { StructuredTool } from \"@langchain/core/tools\";\nimport { Runnable, RunnableConfig } from \"@langchain/core/runnables\";\nimport { formatToOpenAIAssistantTool } from \"@langchain/openai\";\nimport { sleep } from \"../../util/time.js\";\nimport type {\n  OpenAIAssistantFinish,\n  OpenAIAssistantAction,\n  OpenAIToolType,\n} from \"./schema.js\";\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype ThreadMessage = any;\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype RequiredActionFunctionToolCall = any;\n\ntype ExtractRunOutput<AsAgent extends boolean | undefined> =\n  AsAgent extends true\n    ? OpenAIAssistantFinish | OpenAIAssistantAction[]\n    : ThreadMessage[] | RequiredActionFunctionToolCall[];\n\nexport type OpenAIAssistantRunnableInput<\n  AsAgent extends boolean | undefined = undefined\n> = {\n  client?: OpenAIClient;\n  clientOptions?: ClientOptions;\n  assistantId: string;\n  pollIntervalMs?: number;\n  asAgent?: AsAgent;\n};\n\nexport class OpenAIAssistantRunnable<\n  AsAgent extends boolean | undefined,\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  RunInput extends Record<string, any> = Record<string, any>\n> extends Runnable<RunInput, ExtractRunOutput<AsAgent>> {\n  lc_namespace = [\"langchain\", \"experimental\", \"openai_assistant\"];\n\n  private client: OpenAIClient;\n\n  assistantId: string;\n\n  pollIntervalMs = 1000;\n\n  asAgent?: AsAgent;\n\n  constructor(fields: OpenAIAssistantRunnableInput<AsAgent>) {\n    super(fields);\n    this.client = fields.client ?? new OpenAIClient(fields?.clientOptions);\n    this.assistantId = fields.assistantId;\n    this.asAgent = fields.asAgent ?? this.asAgent;\n  }\n\n  static async createAssistant<AsAgent extends boolean>({\n    model,\n    name,\n    instructions,\n    tools,\n    client,\n    clientOptions,\n    asAgent,\n    pollIntervalMs,\n    fileIds,\n  }: Omit<OpenAIAssistantRunnableInput<AsAgent>, \"assistantId\"> & {\n    model: string;\n    name?: string;\n    instructions?: string;\n    tools?: OpenAIToolType | Array<StructuredTool>;\n    fileIds?: string[];\n  }) {\n    const formattedTools =\n      tools?.map((tool) => {\n        // eslint-disable-next-line no-instanceof/no-instanceof\n        if (tool instanceof StructuredTool) {\n          return formatToOpenAIAssistantTool(tool);\n        }\n        return tool;\n      }) ?? [];\n    const oaiClient = client ?? new OpenAIClient(clientOptions);\n    const assistant = await oaiClient.beta.assistants.create({\n      name,\n      instructions,\n      tools: formattedTools,\n      model,\n      file_ids: fileIds,\n    });\n\n    return new this({\n      client: oaiClient,\n      assistantId: assistant.id,\n      asAgent,\n      pollIntervalMs,\n    });\n  }\n\n  async invoke(\n    input: RunInput,\n    _options?: RunnableConfig\n  ): Promise<ExtractRunOutput<AsAgent>> {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    let run: any;\n    if (this.asAgent && input.steps && input.steps.length > 0) {\n      const parsedStepsInput = await this._parseStepsInput(input);\n      run = await this.client.beta.threads.runs.submitToolOutputs(\n        parsedStepsInput.threadId,\n        parsedStepsInput.runId,\n        {\n          tool_outputs: parsedStepsInput.toolOutputs,\n        }\n      );\n    } else if (!(\"threadId\" in input)) {\n      const thread = {\n        messages: [\n          {\n            role: \"user\",\n            content: input.content,\n            file_ids: input.fileIds,\n            metadata: input.messagesMetadata,\n          },\n        ],\n        metadata: input.threadMetadata,\n      };\n      run = await this._createThreadAndRun({\n        ...input,\n        thread,\n      });\n    } else if (!(\"runId\" in input)) {\n      await this.client.beta.threads.messages.create(input.threadId, {\n        content: input.content,\n        role: \"user\",\n        file_ids: input.file_ids,\n        metadata: input.messagesMetadata,\n      });\n      run = await this._createRun(input);\n    } else {\n      // Submitting tool outputs to an existing run, outside the AgentExecutor\n      // framework.\n      run = await this.client.beta.threads.runs.submitToolOutputs(\n        input.threadId,\n        input.runId,\n        {\n          tool_outputs: input.toolOutputs,\n        }\n      );\n    }\n\n    return this._getResponse(run.id, run.thread_id);\n  }\n\n  /**\n   * Delete an assistant.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/assistants/deleteAssistant}\n   * @returns {Promise<AssistantDeleted>}\n   */\n  public async deleteAssistant() {\n    return await this.client.beta.assistants.del(this.assistantId);\n  }\n\n  /**\n   * Retrieves an assistant.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/assistants/getAssistant}\n   * @returns {Promise<OpenAIClient.Beta.Assistants.Assistant>}\n   */\n  public async getAssistant() {\n    return await this.client.beta.assistants.retrieve(this.assistantId);\n  }\n\n  /**\n   * Modifies an assistant.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/assistants/modifyAssistant}\n   * @returns {Promise<OpenAIClient.Beta.Assistants.Assistant>}\n   */\n  public async modifyAssistant<AsAgent extends boolean>({\n    model,\n    name,\n    instructions,\n    fileIds,\n  }: Omit<OpenAIAssistantRunnableInput<AsAgent>, \"assistantId\" | \"tools\"> & {\n    model?: string;\n    name?: string;\n    instructions?: string;\n    fileIds?: string[];\n  }) {\n    return await this.client.beta.assistants.update(this.assistantId, {\n      name,\n      instructions,\n      model,\n      file_ids: fileIds,\n    });\n  }\n\n  private async _parseStepsInput(input: RunInput): Promise<RunInput> {\n    const {\n      action: { runId, threadId },\n    } = input.steps[input.steps.length - 1];\n    const run = await this._waitForRun(runId, threadId);\n    const toolCalls = run.required_action?.submit_tool_outputs.tool_calls;\n    if (!toolCalls) {\n      return input;\n    }\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n...\n// Path: langchain/src/experimental/chat_models/bittensor.ts\nimport { BaseMessage, ChatMessage } from \"@langchain/core/messages\";\nimport { ChatResult, ChatGeneration } from \"@langchain/core/outputs\";\nimport {\n  BaseChatModel,\n  BaseChatModelParams,\n} from \"@langchain/core/language_models/chat_models\";\n\nexport interface BittensorInput extends BaseChatModelParams {\n  systemPrompt?: string | null | undefined;\n}\ninterface Message {\n  role: string;\n  content: string;\n}\n\ninterface ChatPayload {\n  uids?: string[];\n  top_n?: number;\n  messages: Message[];\n}\n\ninterface APIKeyResponse {\n  api_key: string;\n}\n\ninterface ChatResponse {\n  choices?: { message: Message }[];\n}\n/**\n * Class representing the Neural Internet chat model powerd by Bittensor, a decentralized network\n * full of different AI models.s\n * To analyze API_KEYS and logs of you usage visit\n *      https://api.neuralinternet.ai/api-keys\n *      https://api.neuralinternet.ai/logs\n */\nexport class NIBittensorChatModel\n  extends BaseChatModel\n  implements BittensorInput\n{\n  static lc_name(): string {\n    return \"NIBittensorLLM\";\n  }\n\n  systemPrompt: string;\n\n  constructor(fields?: BittensorInput) {\n    super(fields ?? {});\n    this.systemPrompt =\n      fields?.systemPrompt ??\n      \"You are an assistant which is created by Neural Internet(NI) in decentralized network named as a Bittensor. Your task is to provide accurate response based on user prompt\";\n  }\n\n  _combineLLMOutput() {\n    return [];\n  }\n\n  _llmType(): string {\n    return \"NIBittensorLLM\";\n  }\n\n  messageToOpenAIRole(message: BaseMessage) {\n    const type = message._getType();\n    switch (type) {\n      case \"system\":\n        return \"system\";\n      case \"ai\":\n        return \"assistant\";\n      case \"human\":\n        return \"user\";\n      default:\n        return \"user\";\n    }\n  }\n\n  stringToChatMessage(message: string): BaseMessage {\n    return new ChatMessage(message, \"assistant\");\n  }\n\n  /** Call out to NIBittensorChatModel's complete endpoint.\n   Args:\n       messages: The messages to pass into the model.\n\n       Returns: The chat response by the model.\n\n   Example:\n    const chat = new NIBittensorChatModel();\n    const message = new HumanMessage('What is bittensor?');\n    const res = await chat.call([message]);\n   */\n  async _generate(messages: BaseMessage[]): Promise<ChatResult> {\n    const processed_messages = messages.map((message) => {\n      if (typeof message.content !== \"string\") {\n        throw new Error(\"NIBittensorChat does not support non-string output.\");\n      }\n      return {\n        role: this.messageToOpenAIRole(message),\n        content: message.content,\n      };\n    });\n    const generations: ChatGeneration[] = [];\n\n    try {\n      // Retrieve API KEY\n      const apiKeyResponse: Response = await fetch(\n        \"https://test.neuralinternet.ai/admin/api-keys/\"\n      );\n      if (!apiKeyResponse.ok) {\n        throw new Error(\"Network response was not ok\");\n      }\n      const apiKeysData: APIKeyResponse[] = await apiKeyResponse.json();\n      const apiKey: string = apiKeysData[0].api_key;\n\n      const headers = {\n        \"Content-Type\": \"application/json\",\n        Authorization: `Bearer ${apiKey}`,\n        \"Endpoint-Version\": \"2023-05-19\",\n      };\n\n      const minerResponse: Response = await fetch(\n        \"https://test.neuralinternet.ai/top_miner_uids\",\n        { headers }\n      );\n      if (!minerResponse.ok) {\n        throw new Error(\"Network response was not ok\");\n      }\n      const uids: string[] = await minerResponse.json();\n\n      if (Array.isArray(uids) && uids.length) {\n        for (const uid of uids) {\n          try {\n            const payload: ChatPayload = {\n              uids: [uid],\n              messages: [\n                { role: \"system\", content: this.systemPrompt },\n                ...processed_messages,\n              ],\n            };\n\n            const response: Response = await fetch(\n              \"https://test.neuralinternet.ai/chat\",\n              {\n                method: \"POST\",\n                headers,\n                body: JSON.stringify(payload),\n              }\n            );\n\n            if (!response.ok) {\n              throw new Error(\"Network response was not ok\");\n            }\n\n            const chatData: ChatResponse = await response.json();\n\n            if (chatData.choices) {\n              const generation: ChatGeneration = {\n                text: chatData.choices[0].message.content,\n                message: this.stringToChatMessage(\n                  chatData.choices[0].message.content\n                ),\n              };\n              generations.push(generation);\n              return { generations, llmOutput: {} };\n            }\n          } catch (error) {\n            continue;\n          }\n        }\n      }\n    } catch (error) {\n      const generation: ChatGeneration = {\n        text: \"Sorry I am unable to provide response now, Please try again later.\",\n        message: this.stringToChatMessage(\n          \"Sorry I am unable to provide response now, Please try again later.\"\n        ),\n      };\n      generations.push(generation);\n      return { generations, llmOutput: {} };\n    }\n    const generation: ChatGeneration = {\n      text: \"Sorry I am unable to provide response now, Please try again later.\",\n      message: this.stringToChatMessage(\n        \"Sorry I am unable to provide response now, Please try again later.\"\n      ),\n    };\n    generations.push(generation);\n    return { generations, llmOutput: {} };\n  }\n\n  identifyingParams(): {\n    systemPrompt: string | null | undefined;\n  } {\n    return {\n      systemPrompt: this.systemPrompt,\n    };\n  }\n}\n\n// Path: langchain/src/retrievers/hyde.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { Document } from \"@langchain/core/documents\";\nimport { PromptTemplate, BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  StringPromptValue,\n  BasePromptValue,\n} from \"@langchain/core/prompt_values\";\nimport {\n  VectorStore,\n  VectorStoreRetriever,\n  VectorStoreRetrieverInput,\n} from \"@langchain/core/vectorstores\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\n\n/**\n * A string that corresponds to a specific prompt template.\n */\nexport type PromptKey =\n  | \"websearch\"\n  | \"scifact\"\n  | \"arguana\"\n  | \"trec-covid\"\n  | \"fiqa\"\n  | \"dbpedia-entity\"\n  | \"trec-news\"\n  | \"mr-tydi\";\n\n/**\n * Options for the HydeRetriever class, which includes a BaseLanguageModel\n * instance, a VectorStore instance, and an optional promptTemplate which\n * can either be a BasePromptTemplate instance or a PromptKey.\n */\nexport type HydeRetrieverOptions<V extends VectorStore> =\n  VectorStoreRetrieverInput<V> & {\n    llm: BaseLanguageModelInterface;\n    promptTemplate?: BasePromptTemplate | PromptKey;\n  };\n\n/**\n * A class for retrieving relevant documents based on a given query. It\n * extends the VectorStoreRetriever class and uses a BaseLanguageModel to\n * generate a hypothetical answer to the query, which is then used to\n * retrieve relevant documents.\n * @example\n * ```typescript\n * const retriever = new HydeRetriever({\n *   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   llm: new ChatOpenAI(),\n *   k: 1,\n * });\n * await vectorStore.addDocuments(\n *   [\n *     \"My name is John.\",\n *     \"My name is Bob.\",\n *     \"My favourite food is pizza.\",\n *     \"My favourite food is pasta.\",\n *   ].map((pageContent) => new Document({ pageContent })),\n * );\n * const results = await retriever.getRelevantDocuments(\n *   \"What is my favourite food?\",\n * );\n * ```\n */\nexport class HydeRetriever<\n  V extends VectorStore = VectorStore\n> extends VectorStoreRetriever<V> {\n  static lc_name() {\n    return \"HydeRetriever\";\n  }\n\n  get lc_namespace(): string[] {\n    return [\"langchain\", \"retrievers\", \"hyde\"];\n  }\n\n  llm: BaseLanguageModelInterface;\n\n  promptTemplate?: BasePromptTemplate;\n\n  constructor(fields: HydeRetrieverOptions<V>) {\n    super(fields);\n    this.llm = fields.llm;\n    this.promptTemplate =\n      typeof fields.promptTemplate === \"string\"\n        ? getPromptTemplateFromKey(fields.promptTemplate)\n        : fields.promptTemplate;\n    if (this.promptTemplate) {\n      const { inputVariables } = this.promptTemplate;\n      if (inputVariables.length !== 1 && inputVariables[0] !== \"question\") {\n        throw new Error(\n          `Prompt template must accept a single input variable 'question'. Invalid input variables for prompt template: ${inputVariables}`\n        );\n      }\n    }\n  }\n\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Document[]> {\n    let value: BasePromptValue = new StringPromptValue(query);\n\n    // Use a custom template if provided\n    if (this.promptTemplate) {\n      value = await this.promptTemplate.formatPromptValue({ question: query });\n    }\n\n    // Get a hypothetical answer from the LLM\n    const res = await this.llm.generatePrompt([value]);\n    const answer = res.generations[0][0].text;\n\n    // Retrieve relevant documents based on the hypothetical answer\n    const results = await this.vectorStore.similaritySearch(\n      answer,\n      this.k,\n      this.filter,\n      runManager?.getChild(\"vectorstore\")\n    );\n\n    return results;\n  }\n}\n\n/**\n * Returns a BasePromptTemplate instance based on a given PromptKey.\n */\nexport function getPromptTemplateFromKey(key: PromptKey): BasePromptTemplate {\n  let template: string;\n\n  switch (key) {\n    case \"websearch\":\n      template = `Please write a passage to answer the question\nQuestion: {question}\nPassage:`;\n      break;\n    case \"scifact\":\n      template = `Please write a scientific paper passage to support/refute the claim\nClaim: {question}\nPassage:`;\n      break;\n    case \"arguana\":\n      template = `Please write a counter argument for the passage\nPassage: {question}\nCounter Argument:`;\n      break;\n    case \"trec-covid\":\n      template = `Please write a scientific paper passage to answer the question\nQuestion: {question}\nPassage:`;\n      break;\n    case \"fiqa\":\n      template = `Please write a financial article passage to answer the question\nQuestion: {question}\nPassage:`;\n      break;\n    case \"dbpedia-entity\":\n      template = `Please write a passage to answer the question.\nQuestion: {question}\nPassage:`;\n      break;\n    case \"trec-news\":\n      template = `Please write a news passage about the topic.\nTopic: {question}\nPassage:`;\n      break;\n    case \"mr-tydi\":\n      template = `Please write a passage in Swahili/Korean/Japanese/Bengali to answer the question in detail.\nQuestion: {question}\nPassage:`;\n      break;\n    default:\n      throw new Error(`Invalid prompt key: ${key}`);\n  }\n\n  return PromptTemplate.fromTemplate(template);\n}\n\n// Path: langchain/src/util/document.ts\nimport { Document } from \"@langchain/core/documents\";\n\n/**\n * Given a list of documents, this util formats their contents\n * into a string, separated by newlines.\n *\n * @param documents\n * @returns A string of the documents page content, separated by newlines.\n */\nexport const formatDocumentsAsString = (documents: Document[]): string =>\n  documents.map((doc) => doc.pageContent).join(\"\\n\\n\");\n\n// Path: langchain/src/agents/xml/output_parser.ts\nimport { AgentAction, AgentFinish } from \"@langchain/core/agents\";\nimport { OutputParserException } from \"@langchain/core/output_parsers\";\nimport { AgentActionOutputParser } from \"../types.js\";\n\n/**\n * @example\n * ```typescript\n * const prompt = ChatPromptTemplate.fromMessages([\n *   HumanMessagePromptTemplate.fromTemplate(AGENT_INSTRUCTIONS),\n *   new MessagesPlaceholder(\"agent_scratchpad\"),\n * ]);\n * const runnableAgent = RunnableSequence.from([\n *   ...rest of runnable\n *   prompt,\n *   new ChatAnthropic({ modelName: \"claude-2\", temperature: 0 }).bind({\n *     stop: [\"</tool_input>\", \"</final_answer>\"],\n *   }),\n *   new XMLAgentOutputParser(),\n * ]);\n * const result = await executor.invoke({\n *   input: \"What is the weather in Honolulu?\",\n *   tools: [],\n * });\n * ```\n */\nexport class XMLAgentOutputParser extends AgentActionOutputParser {\n  lc_namespace = [\"langchain\", \"agents\", \"xml\"];\n\n  static lc_name() {\n    return \"XMLAgentOutputParser\";\n  }\n\n  /**\n   * Parses the output text from the agent and returns an AgentAction or\n   * AgentFinish object.\n   * @param text The output text from the agent.\n   * @returns An AgentAction or AgentFinish object.\n   */\n  async parse(text: string): Promise<AgentAction | AgentFinish> {\n    if (text.includes(\"</tool>\")) {\n      const [tool, toolInput] = text.split(\"</tool>\");\n      const _tool = tool.split(\"<tool>\")[1];\n      const _toolInput = toolInput.split(\"<tool_input>\")[1];\n      return { tool: _tool, toolInput: _toolInput, log: text };\n    } else if (text.includes(\"<final_answer>\")) {\n      const [, answer] = text.split(\"<final_answer>\");\n      return { returnValues: { output: answer }, log: text };\n    } else {\n      throw new OutputParserException(`Could not parse LLM output: ${text}`);\n    }\n  }\n\n  getFormatInstructions(): string {\n    throw new Error(\n      \"getFormatInstructions not implemented inside OpenAIFunctionsAgentOutputParser.\"\n    );\n  }\n}\n\n// Path: langchain/src/agents/format_scratchpad/log_to_message.ts\nimport type { AgentStep } from \"@langchain/core/agents\";\nimport {\n  type BaseMessage,\n  AIMessage,\n  HumanMessage,\n} from \"@langchain/core/messages\";\nimport { renderTemplate } from \"@langchain/core/prompts\";\n\nexport function formatLogToMessage(\n  intermediateSteps: AgentStep[],\n  templateToolResponse = \"{observation}\"\n): BaseMessage[] {\n  // Get all input variables, if there is more than one, throw an error.\n  const matches = [...templateToolResponse.matchAll(/{([^}]*)}/g)];\n  const stringsInsideBrackets = matches.map((match) => match[1]);\n  if (stringsInsideBrackets.length > 1) {\n    throw new Error(\n      `templateToolResponse must contain one input variable: ${templateToolResponse}`\n    );\n  }\n\n  const thoughts: BaseMessage[] = [];\n\n  for (const step of intermediateSteps) {\n    thoughts.push(new AIMessage(step.action.log));\n    thoughts.push(\n      new HumanMessage(\n        renderTemplate(templateToolResponse, \"f-string\", {\n          [stringsInsideBrackets[0]]: step.observation,\n        })\n      )\n    );\n  }\n  return thoughts;\n}\n\n// Path: langchain/src/stores/message/in_memory.ts\nexport * from \"@langchain/community/stores/message/in_memory\";\n\n// Path: langchain/src/chains/combine_documents/base.ts\nimport { Document } from \"@langchain/core/documents\";\nimport { BasePromptTemplate, PromptTemplate } from \"@langchain/core/prompts\";\nimport { RunnableConfig } from \"@langchain/core/runnables\";\n\nexport const DEFAULT_DOCUMENT_SEPARATOR = \"\\n\\n\";\n\nexport const DOCUMENTS_KEY = \"context\";\nexport const INTERMEDIATE_STEPS_KEY = \"intermediate_steps\";\n\nexport const DEFAULT_DOCUMENT_PROMPT =\n  /* #__PURE__ */ PromptTemplate.fromTemplate(\"{page_content}\");\n\nexport async function formatDocuments({\n  documentPrompt,\n  documentSeparator,\n  documents,\n  config,\n}: {\n  documentPrompt: BasePromptTemplate;\n  documentSeparator: string;\n  documents: Document[];\n  config?: RunnableConfig;\n}) {\n  const formattedDocs = await Promise.all(\n    documents.map((document) =>\n      documentPrompt\n        .withConfig({ runName: \"document_formatter\" })\n        .invoke(\n          { ...document.metadata, page_content: document.pageContent },\n          config\n        )\n    )\n  );\n  return formattedDocs.join(documentSeparator);\n}\n\n// Path: langchain/src/chains/combine_documents/stuff.ts\nimport { LanguageModelLike } from \"@langchain/core/language_models/base\";\nimport {\n  BaseOutputParser,\n  StringOutputParser,\n} from \"@langchain/core/output_parsers\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  RunnablePassthrough,\n  RunnablePick,\n  RunnableSequence,\n} from \"@langchain/core/runnables\";\n\nimport {\n  DEFAULT_DOCUMENT_PROMPT,\n  DEFAULT_DOCUMENT_SEPARATOR,\n  DOCUMENTS_KEY,\n  formatDocuments,\n} from \"./base.js\";\n\n/**\n * Create a chain that passes a list of documents to a model.\n * \n * @param llm Language model to use for responding.\n * @param prompt Prompt template. Must contain input variable \"context\", which will be\n    used for passing in the formatted documents.\n * @param outputParser Output parser. Defaults to `StringOutputParser`.\n * @param documentPrompt Prompt used for formatting each document into a string. Input\n    variables can be \"page_content\" or any metadata keys that are in all documents.\n    \"page_content\" will automatically retrieve the `Document.page_content`, and all \n    other inputs variables will be automatically retrieved from the `Document.metadata` dictionary. Default to a prompt that only contains `Document.page_content`.\n * @param documentSeparator String separator to use between formatted document strings.\n * @returns An LCEL `Runnable` chain.\n    Expects a dictionary as input with a list of `Document`s being passed under\n    the \"context\" key.\n    Return type depends on the `output_parser` used.\n */\nexport async function createStuffDocumentsChain<RunOutput = string>({\n  llm,\n  prompt,\n  outputParser = new StringOutputParser() as unknown as BaseOutputParser<RunOutput>,\n  documentPrompt = DEFAULT_DOCUMENT_PROMPT,\n  documentSeparator = DEFAULT_DOCUMENT_SEPARATOR,\n}: {\n  llm: LanguageModelLike;\n  prompt: BasePromptTemplate;\n  outputParser?: BaseOutputParser<RunOutput>;\n  documentPrompt?: BasePromptTemplate;\n  documentSeparator?: string;\n}) {\n  if (!prompt.inputVariables.includes(DOCUMENTS_KEY)) {\n    throw new Error(`Prompt must include a \"${DOCUMENTS_KEY}\" variable`);\n  }\n\n  return RunnableSequence.from(\n    [\n      RunnablePassthrough.assign({\n        [DOCUMENTS_KEY]: new RunnablePick(DOCUMENTS_KEY).pipe(\n          (documents, metadata) =>\n            formatDocuments({\n              documents,\n              documentPrompt,\n              documentSeparator,\n              config: metadata?.config,\n            })\n        ),\n      }),\n      prompt,\n      llm,\n      outputParser,\n    ],\n    \"stuff_documents_chain\"\n  );\n}\n\n// Path: langchain/src/chains/combine_documents/index.ts\nexport { createStuffDocumentsChain } from \"./stuff.js\";\n\n// Path: langchain/src/retrievers/document_compressors/chain_extract_prompt.ts\nexport const PROMPT_TEMPLATE = (\n  noOutputStr: string\n) => `Given the following question and context, extract any part of the context *AS IS* that is relevant to answer the question. If none of the context is relevant return ${noOutputStr}.\n\nRemember, *DO NOT* edit the extracted parts of the context.\n\n> Question: {question}\n> Context:\n>>>\n{context}\n>>>\nExtracted relevant parts:`;\n\n// Path: langchain/src/retrievers/document_compressors/chain_extract.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { type DocumentInterface, Document } from \"@langchain/core/documents\";\nimport { PromptTemplate } from \"@langchain/core/prompts\";\nimport { BaseOutputParser } from \"@langchain/core/output_parsers\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\nimport { BaseDocumentCompressor } from \"./index.js\";\nimport { PROMPT_TEMPLATE } from \"./chain_extract_prompt.js\";\n\nfunction defaultGetInput(\n  query: string,\n  doc: DocumentInterface\n): Record<string, unknown> {\n  return { question: query, context: doc.pageContent };\n}\n\nclass NoOutputParser extends BaseOutputParser<string> {\n  lc_namespace = [\n    \"langchain\",\n    \"retrievers\",\n    \"document_compressors\",\n    \"chain_extract\",\n  ];\n\n  noOutputStr = \"NO_OUTPUT\";\n\n  parse(text: string): Promise<string> {\n    const cleanedText = text.trim();\n    if (cleanedText === this.noOutputStr) {\n      return Promise.resolve(\"\");\n    }\n    return Promise.resolve(cleanedText);\n  }\n\n  getFormatInstructions(): string {\n    throw new Error(\"Method not implemented.\");\n  }\n}\n\nfunction getDefaultChainPrompt(): PromptTemplate {\n  const outputParser = new NoOutputParser();\n  const template = PROMPT_TEMPLATE(outputParser.noOutputStr);\n  return new PromptTemplate({\n    template,\n    inputVariables: [\"question\", \"context\"],\n    outputParser,\n  });\n}\n\n/**\n * Interface for the arguments required to create an instance of\n * LLMChainExtractor.\n */\nexport interface LLMChainExtractorArgs {\n  llmChain: LLMChain;\n  getInput: (query: string, doc: DocumentInterface) => Record<string, unknown>;\n}\n\n/**\n * A class that uses an LLM chain to extract relevant parts of documents.\n * It extends the BaseDocumentCompressor class.\n */\nexport class LLMChainExtractor extends BaseDocumentCompressor {\n  llmChain: LLMChain;\n\n  getInput: (query: string, doc: DocumentInterface) => Record<string, unknown> =\n    defaultGetInput;\n\n  constructor({ llmChain, getInput }: LLMChainExtractorArgs) {\n    super();\n    this.llmChain = llmChain;\n    this.getInput = getInput;\n  }\n\n  /**\n   * Compresses a list of documents based on the output of an LLM chain.\n   * @param documents The list of documents to be compressed.\n   * @param query The query to be used for document compression.\n   * @returns A list of compressed documents.\n   */\n  async compressDocuments(\n    documents: DocumentInterface[],\n    query: string\n  ): Promise<DocumentInterface[]> {\n    const compressedDocs = await Promise.all(\n      documents.map(async (doc) => {\n        const input = this.getInput(query, doc);\n        const output = await this.llmChain.predict(input);\n        return output.length > 0\n          ? new Document({\n              pageContent: output,\n              metadata: doc.metadata,\n            })\n          : undefined;\n      })\n    );\n    return compressedDocs.filter((doc): doc is Document => doc !== undefined);\n  }\n\n  /**\n   * Creates a new instance of LLMChainExtractor from a given LLM, prompt\n   * template, and getInput function.\n   * @param llm The BaseLanguageModel instance used for document extraction.\n   * @param prompt The PromptTemplate instance used for document extraction.\n   * @param getInput A function used for constructing the chain input from the query and a Document.\n   * @returns A new instance of LLMChainExtractor.\n   */\n  static fromLLM(\n    llm: BaseLanguageModelInterface,\n    prompt?: PromptTemplate,\n    getInput?: (\n      query: string,\n      doc: DocumentInterface\n    ) => Record<string, unknown>\n  ): LLMChainExtractor {\n    const _prompt = prompt || getDefaultChainPrompt();\n    const _getInput = getInput || defaultGetInput;\n    const llmChain = new LLMChain({ llm, prompt: _prompt });\n    return new LLMChainExtractor({ llmChain, getInput: _getInput });\n  }\n}\n\n// Path: langchain/src/schema/prompt_template.ts\nimport { Document } from \"@langchain/core/documents\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\n\n/**\n * Formats a document using a given prompt template.\n *\n * @async\n * @param {Document} document - The document to format.\n * @param {BasePromptTemplate} prompt - The prompt template to use for formatting.\n * @returns {Promise<string>} A Promise that resolves to the formatted document as a string.\n * @throws {Error} If the document is missing required metadata variables specified in the prompt template.\n */\nexport const formatDocument = async (\n  document: Document,\n  prompt: BasePromptTemplate\n): Promise<string> => {\n  const baseInfo = {\n    pageContent: document.pageContent,\n    ...document.metadata,\n  };\n  const variables = new Set(prompt.inputVariables);\n  const requiredMetadata = new Set(\n    prompt.inputVariables\n      .map((v) => (v !== \"pageContent\" ? v : null))\n      .filter((v) => v !== null)\n  );\n  const missingMetadata = [];\n  for (const variable of variables) {\n    if (!(variable in baseInfo) && variable !== \"pageContent\") {\n      missingMetadata.push(variable);\n    }\n  }\n  if (missingMetadata.length) {\n    throw new Error(\n      `Document prompt requires documents to have metadata variables: ${JSON.stringify(\n        requiredMetadata\n      )}. Received document with missing metadata: ${JSON.stringify(\n        missingMetadata\n      )}`\n    );\n  }\n  return prompt.format(baseInfo);\n};\n\n// Path: langchain/src/agents/chat_convo/prompt.ts\nexport const DEFAULT_PREFIX = `Assistant is a large language model trained by OpenAI.\n\nAssistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n\nAssistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n\nOverall, Assistant is a powerful system that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.`;\n\nexport const PREFIX_END = ` However, above all else, all responses must adhere to the format of RESPONSE FORMAT INSTRUCTIONS.`;\n\nexport const FORMAT_INSTRUCTIONS = `RESPONSE FORMAT INSTRUCTIONS\n----------------------------\n\nOutput a JSON markdown code snippet containing a valid JSON object in one of two formats:\n\n**Option 1:**\nUse this if you want the human to use a tool.\nMarkdown code snippet formatted in the following schema:\n\n\\`\\`\\`json\n{{{{\n    \"action\": string, // The action to take. Must be one of [{tool_names}]\n    \"action_input\": string // The input to the action. May be a stringified object.\n}}}}\n\\`\\`\\`\n\n**Option #2:**\nUse this if you want to respond directly and conversationally to the human. Markdown code snippet formatted in the following schema:\n\n\\`\\`\\`json\n{{{{\n    \"action\": \"Final Answer\",\n    \"action_input\": string // You should put what you want to return to use here and make sure to use valid json newline characters.\n}}}}\n\\`\\`\\`\n\nFor both options, remember to always include the surrounding markdown code snippet delimiters (begin with \"\\`\\`\\`json\" and end with \"\\`\\`\\`\")!\n`;\n\nexport const DEFAULT_SUFFIX = `TOOLS\n------\nAssistant can ask the user to use tools to look up information that may be helpful in answering the users original question. The tools the human can use are:\n\n{tools}\n\n{format_instructions}\n\nUSER'S INPUT\n--------------------\nHere is the user's input (remember to respond with a markdown code snippet of a json blob with a single action, and NOTHING else):\n\n{{input}}`;\n\nexport const TEMPLATE_TOOL_RESPONSE = `TOOL RESPONSE:\n---------------------\n{observation}\n\nUSER'S INPUT\n--------------------\n\nOkay, so what is the response to my last comment? If using information obtained from the tools you must mention it explicitly without mentioning the tool names - I have forgotten all TOOL RESPONSES! Remember to respond with a markdown code snippet of a json blob with a single action, and NOTHING else.`;\n\n// Path: langchain/src/agents/format_scratchpad/openai_functions.ts\nimport {\n  AIMessage,\n  type BaseMessage,\n  HumanMessage,\n  FunctionMessage,\n} from \"@langchain/core/messages\";\nimport type { AgentStep } from \"@langchain/core/agents\";\nimport { renderTemplate } from \"@langchain/core/prompts\";\nimport { TEMPLATE_TOOL_RESPONSE } from \"../chat_convo/prompt.js\";\n\n/**\n * Format a list of AgentSteps into a list of BaseMessage instances for\n * agents that use OpenAI's API. Helpful for passing in previous agent\n * step context into new iterations.\n *\n * @deprecated Use formatToOpenAIFunctionMessages instead.\n * @param steps A list of AgentSteps to format.\n * @returns A list of BaseMessages.\n */\nexport function formatForOpenAIFunctions(steps: AgentStep[]): BaseMessage[] {\n  const thoughts: BaseMessage[] = [];\n  for (const step of steps) {\n    thoughts.push(new AIMessage(step.action.log));\n    thoughts.push(\n      new HumanMessage(\n        renderTemplate(TEMPLATE_TOOL_RESPONSE, \"f-string\", {\n          observation: step.observation,\n        })\n      )\n    );\n  }\n  return thoughts;\n}\n\n/**\n * Format a list of AgentSteps into a list of BaseMessage instances for\n * agents that use OpenAI's API. Helpful for passing in previous agent\n * step context into new iterations.\n *\n * @param steps A list of AgentSteps to format.\n * @returns A list of BaseMessages.\n */\nexport function formatToOpenAIFunctionMessages(\n  steps: AgentStep[]\n): BaseMessage[] {\n  return steps.flatMap(({ action, observation }) => {\n    if (\"messageLog\" in action && action.messageLog !== undefined) {\n      const log = action.messageLog as BaseMessage[];\n      return log.concat(new FunctionMessage(observation, action.tool));\n    } else {\n      return [new AIMessage(action.log)];\n    }\n  });\n}\n\n// Path: langchain/src/experimental/openai_files/index.ts\nimport { OpenAIClient, type ClientOptions } from \"@langchain/openai\";\nimport { Serializable } from \"@langchain/core/load/serializable\";\n\nexport type OpenAIFilesInput = {\n  client?: OpenAIClient;\n  clientOptions?: ClientOptions;\n};\n\nexport class OpenAIFiles extends Serializable {\n  lc_namespace = [\"langchain\", \"experimental\"];\n\n  private oaiClient: OpenAIClient;\n\n  constructor(fields?: OpenAIFilesInput) {\n    super(fields);\n    this.oaiClient = fields?.client ?? new OpenAIClient(fields?.clientOptions);\n  }\n\n  /**\n   * Upload file\n   * Upload a file that can be used across various endpoints. The size of all the files uploaded by one organization can be up to 100 GB.\n   *\n   * @note The size of individual files can be a maximum of 512 MB. See the Assistants Tools guide to learn more about the types of files supported. The Fine-tuning API only supports .jsonl files.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/files/create}\n   * @param {OpenAIClient.FileCreateParams['file']} file\n   * @param {OpenAIClient.FileCreateParams['purpose']} purpose\n   * @param {OpenAIClient.RequestOptions | undefined} options\n   * @returns {Promise<OpenAIClient.Files.FileObject>}\n   */\n  async createFile({\n    file,\n    purpose,\n    options,\n  }: OpenAIClient.FileCreateParams & {\n    options?: OpenAIClient.RequestOptions;\n  }) {\n    return this.oaiClient.files.create({ file, purpose }, options);\n  }\n\n  /**\n   * Delete a file.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/files/delete}\n   * @param {string} fileId\n   * @param {OpenAIClient.RequestOptions | undefined} options\n   * @returns {Promise<OpenAIClient.Files.FileDeleted>}\n   */\n  async deleteFile({\n    fileId,\n    options,\n  }: {\n    fileId: string;\n    options?: OpenAIClient.RequestOptions;\n  }) {\n    return this.oaiClient.files.del(fileId, options);\n  }\n\n  /**\n   * List files\n   * Returns a list of files that belong to the user's organization.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/files/list}\n   * @param {OpenAIClient.Files.FileListParams | undefined} query\n   * @param {OpenAIClient.RequestOptions | undefined} options\n   * @returns {Promise<OpenAIClient.Files.FileObjectsPage>}\n   */\n  async listFiles(props?: {\n    query?: OpenAIClient.Files.FileListParams;\n    options?: OpenAIClient.RequestOptions;\n  }) {\n    return this.oaiClient.files.list(props?.query, props?.options);\n  }\n\n  /**\n   * Retrieve file\n   * Returns information about a specific file.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/files/retrieve}\n   * @param {string} fileId\n   * @param {OpenAIClient.RequestOptions | undefined} options\n   * @returns {Promise<OpenAIClient.Files.FileObject>}\n   */\n  async retrieveFile({\n    fileId,\n    options,\n  }: {\n    fileId: string;\n    options?: OpenAIClient.RequestOptions;\n  }) {\n    return this.oaiClient.files.retrieve(fileId, options);\n  }\n\n  /**\n   * Retrieve file content\n   * Returns the contents of the specified file.\n   *\n   * @note You can't retrieve the contents of a file that was uploaded with the \"purpose\": \"assistants\" API.\n   *\n   * @link {https://platform.openai.com/docs/api-reference/files/retrieve-contents}\n   * @param {string} fileId\n   * @param {OpenAIClient.RequestOptions | undefined} options\n   * @returns {Promise<string>}\n   */\n  async retrieveFileContent({\n    fileId,\n    options,\n  }: {\n    fileId: string;\n    options?: OpenAIClient.RequestOptions;\n  }) {\n    return this.oaiClient.files.retrieveContent(fileId, options);\n  }\n}\n\n// Path: langchain/src/smith/config.ts\nimport { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { RunnableConfig } from \"@langchain/core/runnables\";\nimport { Example, Run } from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport {\n  Criteria as CriteriaType,\n  type EmbeddingDistanceEvalChainInput,\n} from \"../evaluation/index.js\";\nimport { LoadEvaluatorOptions } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\n\nexport type EvaluatorInputs = {\n  input?: string | unknown;\n  prediction: string | unknown;\n  reference?: string | unknown;\n};\n\nexport type EvaluatorInputFormatter = ({\n  rawInput,\n  rawPrediction,\n  rawReferenceOutput,\n  run,\n}: {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  rawInput: any;\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  rawPrediction: any;\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  rawReferenceOutput?: any;\n  run: Run;\n}) => EvaluatorInputs;\n\nexport type DynamicRunEvaluatorParams = {\n  input: Record<string, unknown>;\n  prediction?: Record<string, unknown>;\n  reference?: Record<string, unknown>;\n  run: Run;\n  example?: Example;\n};\n\n/**\n * Type of a function that can be coerced into a RunEvaluator function.\n * While we have the class-based RunEvaluator, it's often more convenient to directly\n * pass a function to the runner. This type allows us to do that.\n */\nexport type RunEvaluatorLike =\n  | ((\n      props: DynamicRunEvaluatorParams,\n      options?: { config?: RunnableConfig }\n    ) => Promise<EvaluationResult>)\n  | ((\n      props: DynamicRunEvaluatorParams,\n      options?: { config?: RunnableConfig }\n    ) => EvaluationResult);\n\nexport \nfunction isOffTheShelfEvaluator<\n  T extends keyof EvaluatorType,\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig {\n  return typeof evaluator === \"string\" || \"evaluatorType\" in evaluator;\n}\n\nexport function isCustomEvaluator<\n  T extends keyof EvaluatorType,\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n>(evaluator: T | EvalConfig | U): evaluator is U {\n  return !isOffTheShelfEvaluator(evaluator);\n}\n\nexport type RunEvalType<\n  T extends keyof EvaluatorType =\n    | \"criteria\"\n    | \"labeled_criteria\"\n    | \"embedding_distance\",\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n> = T | EvalConfig | U;\n\n/**\n * Configuration class for running evaluations on datasets.\n *\n * @remarks\n * RunEvalConfig in LangSmith is a configuration class for running evaluations on datasets. Its primary purpose is to define the parameters and evaluators that will be applied during the evaluation of a dataset. This configuration can include various evaluators, custom evaluators, and different keys for inputs, predictions, and references.\n *\n * @typeparam T - The type of evaluators.\n * @typeparam U - The type of custom evaluators.\n */\nexport type RunEvalConfig<\n  T extends keyof EvaluatorType =\n    | \"criteria\"\n    | \"labeled_criteria\"\n    | \"embedding_distance\",\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n> = {\n  /**\n   * Evaluators to apply to a dataset run.\n   * You can optionally specify these by name, or by\n   * configuring them with an EvalConfig object.\n   */\n  evaluators?: RunEvalType<T, U>[];\n\n  /**\n   * Convert the evaluation data into formats that can be used by the evaluator.\n   * This should most commonly be a string.\n   * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n   * @example\n   * ```ts\n   * // Chain input: { input: \"some string\" }\n   * // Chain output: { output: \"some output\" }\n   * // Reference example output format: { output: \"some reference output\" }\n   * const formatEvaluatorInputs = ({\n   *   rawInput,\n   *   rawPrediction,\n   *   rawReferenceOutput,\n   * }) => {\n   *   return {\n   *     input: rawInput.input,\n   *     prediction: rawPrediction.output,\n   *     reference: rawReferenceOutput.output,\n   *   };\n   * };\n   * ```\n   * @returns The prepared data.\n   */\n  formatEvaluatorInputs?: EvaluatorInputFormatter;\n\n  /**\n   * Custom evaluators to apply to a dataset run.\n   * Each evaluator is provided with a run trace containing the model\n   * outputs, as well as an \"example\" object representing a record\n   * in the dataset.\n   *\n   * @deprecated Use `evaluators` instead.\n   */\n  customEvaluators?: U[];\n};\n\nexport interface EvalConfig extends LoadEvaluatorOptions {\n  /**\n   * The name of the evaluator to use.\n   * Example: labeled_criteria, criteria, etc.\n   */\n  evaluatorType: keyof EvaluatorType;\n\n  /**\n   * The feedback (or metric) name to use for the logged\n   * evaluation results. If none provided, we default to\n   * the evaluationName.\n   */\n  feedbackKey?: string;\n\n  /**\n   * Convert the evaluation data into formats that can be used by the evaluator.\n   * This should most commonly be a string.\n   * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n   * @example\n   * ```ts\n   * // Chain input: { input: \"some string\" }\n   * // Chain output: { output: \"some output\" }\n   * // Reference example output format: { output: \"some reference output\" }\n   * const formatEvaluatorInputs = ({\n   *   rawInput,\n   *   rawPrediction,\n   *   rawReferenceOutput,\n   * }) => {\n   *   return {\n   *     input: rawInput.input,\n   *     prediction: rawPrediction.output,\n   *     reference: rawReferenceOutput.output,\n   *   };\n   * };\n   * ```\n   * @returns The prepared data.\n   */\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n}\n\nconst isStringifiableValue = (\n  value: unknown\n): value is string | number | boolean | bigint =>\n  typeof value === \"string\" ||\n  typeof value === \"number\" ||\n  typeof value === \"boolean\" ||\n  typeof value === \"bigint\";\n\nconst getSingleStringifiedValue = (value: unknown) => {\n  if (isStringifiableValue(value)) {\n    return `${value}`;\n  }\n\n  if (typeof value === \"object\" && value != null && !Array.isArray(value)) {\n    const entries = Object.entries(value);\n\n    if (entries.length === 1 && isStringifiableValue(entries[0][1])) {\n      return `${entries[0][1]}`;\n    }\n  }\n\n  console.warn(\"Non-stringifiable value found when coercing\", value);\n  return `${value}`;\n};\n\n/**\n * Configuration to load a \"CriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"criteria\",\n *     criteria: \"helpfulness\"\n *   }]\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"criteria\",\n *     criteria: { \"isCompliant\": \"Does the submission comply with the requirements of XYZ\"\n *   }]\n * };\n */\nexport type Criteria = EvalConfig & {\n  evaluatorType: \"criteria\";\n\n  /**\n   * The \"criteria\" to insert into the prompt template\n   * used for evaluation. See the prompt at\n   * https://smith.langchain.com/hub/langchain-ai/criteria-evaluator\n   * for more information.\n   */\n  criteria?: CriteriaType | Record<string, string>;\n\n  /**\n   * The language model to use as the evaluator, defaults to GPT-4\n   */\n  llm?: BaseLanguageModel;\n};\n\n// for compatibility reasons\nexport type CriteriaEvalChainConfig = Criteria;\n\nexport function Criteria(\n  criteria: CriteriaType,\n  config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\"\n  >\n): EvalConfig {\n  const formatEvaluatorInputs =\n    config?.formatEvaluatorInputs ??\n    ((payload) => ({\n      prediction: getSingleStringifiedValue(payload.rawPrediction),\n      input: getSingleStringifiedValue(payload.rawInput),\n    }));\n\n  return {\n    evaluatorType: \"criteria\",\n    criteria,\n    feedbackKey: config?.feedbackKey ?? criteria,\n    llm: config?.llm,\n    formatEvaluatorInputs,\n  };\n}\n\n/**\n * Configuration to load a \"LabeledCriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria and also\n * provides a \"ground truth\" label for the evaluator to incorporate\n * in its evaluation.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"labeled_criteria\",\n *     criteria: \"correctness\"\n *   }],\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"labeled_criteria\",\n *     criteria: { \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\" }\n *   }],\n * };\n */\nexport type LabeledCriteria = EvalConfig & {\n  evaluatorType: \"labeled_criteria\";\n\n  /**\n   * The \"criteria\" to insert into the prompt template\n   * used for evaluation. See the prompt at\n   * https://smith.langchain.com/hub/langchain-ai/labeled-criteria\n   * for more information.\n   */\n  criteria?: CriteriaType | Record<string, string>;\n\n  /**\n   * The language model to use as the evaluator, defaults to GPT-4\n   */\n  llm?: BaseLanguageModel;\n};\n\nexport function LabeledCriteria(\n  criteria: CriteriaType,\n  config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\"\n  >\n): LabeledCriteria {\n  const formatEvaluatorInputs =\n    config?.formatEvaluatorInputs ??\n    ((payload) => ({\n      prediction: getSingleStringifiedValue(payload.rawPrediction),\n      input: getSingleStringifiedValue(payload.rawInput),\n      reference: getSingleStringifiedValue(payload.rawReferenceOutput),\n    }));\n\n  return {\n    evaluatorType: \"labeled_criteria\",\n    criteria,\n    feedbackKey: config?.feedbackKey ?? criteria,\n    llm: config?.llm,\n    formatEvaluatorInputs,\n  };\n}\n\n/**\n * Configuration to load a \"EmbeddingDistanceEvalChain\" evaluator,\n * which embeds distances to score semantic difference between\n * a prediction and reference.\n */\nexport type EmbeddingDistance = EvalConfig &\n  EmbeddingDistanceEvalChainInput & { evaluatorType: \"embedding_distance\" };\n\nexport function EmbeddingDistance(\n  distanceMetric: EmbeddingDistanceEvalChainInput[\"distanceMetric\"],\n  config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"embedding\" | \"feedbackKey\"\n  >\n): EmbeddingDistance {\n  const formatEvaluatorInputs =\n    config?.formatEvaluatorInputs ??\n    ((payload) => ({\n      prediction: getSingleStringifiedValue(payload.rawPrediction),\n      reference: getSingleStringifiedValue(payload.rawReferenceOutput),\n    }));\n\n  return {\n    evaluatorType: \"embedding_distance\",\n    embedding: config?.embedding,\n    distanceMetric,\n    feedbackKey: config?.feedbackKey ?? \"embedding_distance\",\n    formatEvaluatorInputs,\n  };\n}\n\n// Path: langchain/src/smith/progress.ts\nexport class ProgressBar {\n  total: number;\n\n  current: number;\n\n  barLength: number;\n\n  format: string;\n\n  constructor(props: { total: number; format?: string; barLength?: number }) {\n    const { total, format, barLength } = props;\n    this.total = total;\n    this.current = 0;\n    this.barLength = barLength ?? 40;\n    this.format = format || \"{bar} {percentage}% | {value}/{total}\";\n  }\n\n  initialize(): void {\n    this.update({ current: 0 });\n  }\n\n  update({\n    current,\n    formatArgs,\n  }: {\n    current: number;\n    formatArgs?: Record<string, string>;\n  }): void {\n    this.current = current;\n    const ratio = this.current / this.total;\n    const filledBarLength = Math.round(ratio * this.barLength);\n    const emptyBarLength = this.barLength - filledBarLength;\n\n    const filledBar = \"\u2593\".repeat(filledBarLength);\n    const emptyBar = \"\u2591\".repeat(emptyBarLength);\n    const percentage = (ratio * 100).toFixed(2);\n\n    let formattedString = this.format\n      .replace(\"{bar}\", `${filledBar}${emptyBar}`)\n      .replace(\"{percentage}\", percentage)\n      .replace(\"{value}\", this.current.toString())\n      .replace(\"{total}\", this.total.toString());\n    if (formatArgs) {\n      for (const key in formatArgs) {\n        if (Object.prototype.hasOwnProperty.call(formatArgs, key)) {\n          formattedString = formattedString.replace(\n            `{${key}}`,\n            formatArgs[key].toString()\n          );\n        }\n      }\n    }\n\n    console.log(formattedString);\n  }\n\n  increment({\n    formatArgs,\n  }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.current + 1, formatArgs });\n  }\n\n  complete({ formatArgs }: { formatArgs?: Record<string, string> } = {}): void {\n    this.update({ current: this.total, formatArgs });\n    console.log(\"\\nCompleted\");\n  }\n}\n\n// Path: langchain/src/smith/name_generation.ts\nconst adjectives: string[] = [\n  \"abandoned\",\n  \"aching\",\n  \"advanced\",\n  \"ample\",\n  \"artistic\",\n  \"back\",\n  \"best\",\n  \"bold\",\n  \"brief\",\n  \"clear\",\n  \"cold\",\n  \"complicated\",\n  \"cooked\",\n  \"crazy\",\n  \"crushing\",\n  \"damp\",\n  \"dear\",\n  \"definite\",\n  \"dependable\",\n  \"diligent\",\n  \"drab\",\n  \"earnest\",\n  \"elderly\",\n  \"enchanted\",\n  \"essential\",\n  \"excellent\",\n  \"extraneous\",\n  \"fixed\",\n  \"flowery\",\n  \"formal\",\n  \"fresh\",\n  \"frosty\",\n  \"giving\",\n  \"glossy\",\n  \"healthy\",\n  \"helpful\",\n  \"impressionable\",\n  \"kind\",\n  \"large\",\n  \"left\",\n  \"long\",\n  \"loyal\",\n  \"mealy\",\n  \"memorable\",\n  \"monthly\",\n  \"new\",\n  \"notable\",\n  \"only\",\n  \"ordinary\",\n  \"passionate\",\n  \"perfect\",\n  \"pertinent\",\n  \"proper\",\n  \"puzzled\",\n  \"reflecting\",\n  \"respectful\",\n  \"roasted\",\n  \"scholarly\",\n  \"shiny\",\n  \"slight\",\n  \"sparkling\",\n  \"spotless\",\n  \"stupendous\",\n  \"sunny\",\n  \"tart\",\n  \"terrific\",\n  \"timely\",\n  \"unique\",\n  \"upbeat\",\n  \"vacant\",\n  \"virtual\",\n  \"warm\",\n  \"weary\",\n  \"whispered\",\n  \"worthwhile\",\n  \"yellow\",\n];\n\nconst nouns: string[] = [\n  \"account\",\n  \"acknowledgment\",\n  \"address\",\n  \"advertising\",\n  \"airplane\",\n  \"animal\",\n  \"appointment\",\n  \"arrival\",\n  \"artist\",\n  \"attachment\",\n  \"attitude\",\n  \"availability\",\n  \"backpack\",\n  \"bag\",\n  \"balance\",\n  \"bass\",\n  \"bean\",\n  \"beauty\",\n  \"bibliography\",\n  \"bill\",\n  \"bite\",\n  \"blossom\",\n  \"boat\",\n  \"book\",\n...\n", "needle_token_start": 11982, "needle_token_end": 12072, "code_context_ntokens": 16016, "output": ["```typescript\n/**\n * Type of a function that can be coerced into a RunEvaluator function.\n * While we have the class-based RunEvaluator, it's often more convenient to directly\n * pass a function to the runner. This type allows us to do that.\n */\nexport type RunEvaluatorLike =\n  | ((\n      props: DynamicRunEvaluatorParams,\n      options?: { config?: RunnableConfig }\n    ) => Promise<EvaluationResult>)\n  | ((\n      props: DynamicRunEvaluatorParams,\n      options?: { config?: RunnableConfig }\n    ) => EvaluationResult);\n\nexport \nfunction isOffTheShelfEvaluator<\n  T extends keyof EvaluatorType,\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig {\n  return typeof evaluator === \"string\" || \"evaluatorType\" in evaluator;\n}\n\nexport function isCustomEvaluator<\n  T extends keyof EvaluatorType,\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n>(evaluator: T | EvalConfig | U): evaluator is U {\n  return !isOffTheShelfEvaluator(evaluator);\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "complexValue", "language": "typescript", "path": "langchain/src/util/googlevertexai-connection.ts", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform a given value into a structured format suitable for further processing or interfacing with specific APIs that require complex data structures.\n2. **Input**: Accepts any value, including numbers, strings, objects, arrays, null, or undefined.\n3. **Output**: Returns a structured object that categorizes the input based on its type into specific formats such as integers, floats, strings, lists, or structured objects. If the input is null or undefined, it returns undefined.\n4. **Procedure**: \n   - Checks if the input is null or undefined and returns undefined if true.\n   - If the input is an object, it checks if it's an array. For arrays, it recursively processes each element into a structured list. For non-array objects, it recursively processes each property into a structured object.\n   - For numbers, it distinguishes between integers and floats, returning the appropriate structured type.\n   - For other types (like strings), it wraps them in a structured format.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/experimental/autogpt/agent.ts\nimport type { VectorStoreRetrieverInterface } from \"@langchain/core/vectorstores\";\nimport { Tool } from \"@langchain/core/tools\";\nimport {\n  AIMessage,\n  BaseMessage,\n  HumanMessage,\n  SystemMessage,\n} from \"@langchain/core/messages\";\nimport { BaseChatModel } from \"@langchain/core/language_models/chat_models\";\nimport {\n  getEmbeddingContextSize,\n  getModelContextSize,\n} from \"@langchain/core/language_models/base\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\n\nimport { AutoGPTOutputParser } from \"./output_parser.js\";\nimport { AutoGPTPrompt } from \"./prompt.js\";\n// import { HumanInputRun } from \"./tools/human/tool\"; // TODO\nimport { ObjectTool, FINISH_NAME } from \"./schema.js\";\nimport { TokenTextSplitter } from \"../../text_splitter.js\";\n\n/**\n * Interface for the input parameters of the AutoGPT class.\n */\nexport interface AutoGPTInput {\n  aiName: string;\n  aiRole: string;\n  memory: VectorStoreRetrieverInterface;\n  humanInTheLoop?: boolean;\n  outputParser?: AutoGPTOutputParser;\n  maxIterations?: number;\n}\n\n/**\n * Class representing the AutoGPT concept with LangChain primitives. It is\n * designed to be used with a set of tools such as a search tool,\n * write-file tool, and a read-file tool.\n * @example\n * ```typescript\n * const autogpt = AutoGPT.fromLLMAndTools(\n *   new ChatOpenAI({ temperature: 0 }),\n *   [\n *     new ReadFileTool({ store: new InMemoryFileStore() }),\n *     new WriteFileTool({ store: new InMemoryFileStore() }),\n *     new SerpAPI(\"YOUR_SERPAPI_API_KEY\", {\n *       location: \"San Francisco,California,United States\",\n *       hl: \"en\",\n *       gl: \"us\",\n *     }),\n *   ],\n *   {\n *     memory: new MemoryVectorStore(new OpenAIEmbeddings()).asRetriever(),\n *     aiName: \"Tom\",\n *     aiRole: \"Assistant\",\n *   },\n * );\n * const result = await autogpt.run([\"write a weather report for SF today\"]);\n * ```\n */\nexport class AutoGPT {\n  aiName: string;\n\n  memory: VectorStoreRetrieverInterface;\n\n  fullMessageHistory: BaseMessage[];\n\n  nextActionCount: number;\n\n  chain: LLMChain;\n\n  outputParser: AutoGPTOutputParser;\n\n  tools: ObjectTool[];\n\n  feedbackTool?: Tool;\n\n  maxIterations: number;\n\n  // Currently not generic enough to support any text splitter.\n  textSplitter: TokenTextSplitter;\n\n  constructor({\n    aiName,\n    memory,\n    chain,\n    outputParser,\n    tools,\n    feedbackTool,\n    maxIterations,\n  }: Omit<Required<AutoGPTInput>, \"aiRole\" | \"humanInTheLoop\"> & {\n    chain: LLMChain;\n    tools: ObjectTool[];\n    feedbackTool?: Tool;\n  }) {\n    this.aiName = aiName;\n    this.memory = memory;\n    this.fullMessageHistory = [];\n    this.nextActionCount = 0;\n    this.chain = chain;\n    this.outputParser = outputParser;\n    this.tools = tools;\n    this.feedbackTool = feedbackTool;\n    this.maxIterations = maxIterations;\n    const chunkSize = getEmbeddingContextSize(\n      \"modelName\" in memory.vectorStore.embeddings\n        ? (memory.vectorStore.embeddings.modelName as string)\n        : undefined\n    );\n    this.textSplitter = new TokenTextSplitter({\n      chunkSize,\n      chunkOverlap: Math.round(chunkSize / 10),\n    });\n  }\n\n  /**\n   * Creates a new AutoGPT instance from a given LLM and a set of tools.\n   * @param llm A BaseChatModel object.\n   * @param tools An array of ObjectTool objects.\n   * @param options.aiName The name of the AI.\n   * @param options.aiRole The role of the AI.\n   * @param options.memory A VectorStoreRetriever object that represents the memory of the AI.\n   * @param options.maxIterations The maximum number of iterations the AI can perform.\n   * @param options.outputParser An AutoGPTOutputParser object that parses the output of the AI.\n   * @returns A new instance of the AutoGPT class.\n   */\n  static fromLLMAndTools(\n    llm: BaseChatModel,\n    tools: ObjectTool[],\n    {\n      aiName,\n      aiRole,\n      memory,\n      maxIterations = 100,\n      // humanInTheLoop = false,\n      outputParser = new AutoGPTOutputParser(),\n    }: AutoGPTInput\n  ): AutoGPT {\n    const prompt = new AutoGPTPrompt({\n      aiName,\n      aiRole,\n      tools,\n      tokenCounter: llm.getNumTokens.bind(llm),\n      sendTokenLimit: getModelContextSize(\n        \"modelName\" in llm ? (llm.modelName as string) : \"gpt2\"\n      ),\n    });\n    // const feedbackTool = humanInTheLoop ? new HumanInputRun() : null;\n...\n// Path: langchain/src/experimental/autogpt/tests/output_parser.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { preprocessJsonInput } from \"../output_parser.js\";\n\ntest(\"should parse outputs correctly\", () => {\n  expect(preprocessJsonInput(\"{'escaped':'\\\\a'}\")).toBe(\"{'escaped':'\\\\\\\\a'}\");\n\n  expect(preprocessJsonInput(\"```\\n{}\\n```\")).toBe(\"{}\");\n  expect(preprocessJsonInput(\"```json\\n{}\\n```\")).toBe(\"{}\");\n  expect(\n    preprocessJsonInput(\"I will do the following:\\n\\n```json\\n{}\\n```\")\n  ).toBe(\"{}\");\n});\n\n// Path: langchain/src/tools/fs.ts\nimport { z } from \"zod\";\nimport { StructuredTool, ToolParams } from \"@langchain/core/tools\";\nimport { BaseFileStore } from \"../stores/file/base.js\";\n\n/**\n * Interface for parameters required by the ReadFileTool class.\n */\ninterface ReadFileParams extends ToolParams {\n  store: BaseFileStore;\n}\n\n/**\n * Class for reading files from the disk. Extends the StructuredTool\n * class.\n */\nexport class ReadFileTool extends StructuredTool {\n  static lc_name() {\n    return \"ReadFileTool\";\n  }\n\n  schema = z.object({\n    file_path: z.string().describe(\"name of file\"),\n  });\n\n  name = \"read_file\";\n\n  description = \"Read file from disk\";\n\n  store: BaseFileStore;\n\n  constructor({ store }: ReadFileParams) {\n    super(...arguments);\n\n    this.store = store;\n  }\n\n  async _call({ file_path }: z.infer<typeof this.schema>) {\n    return await this.store.readFile(file_path);\n  }\n}\n\n/**\n * Interface for parameters required by the WriteFileTool class.\n */\ninterface WriteFileParams extends ToolParams {\n  store: BaseFileStore;\n}\n\n/**\n * Class for writing data to files on the disk. Extends the StructuredTool\n * class.\n */\nexport class WriteFileTool extends StructuredTool {\n  static lc_name() {\n    return \"WriteFileTool\";\n  }\n\n  schema = z.object({\n    file_path: z.string().describe(\"name of file\"),\n    text: z.string().describe(\"text to write to file\"),\n  });\n\n  name = \"write_file\";\n\n  description = \"Write file from disk\";\n\n  store: BaseFileStore;\n\n  constructor({ store, ...rest }: WriteFileParams) {\n    super(rest);\n\n    this.store = store;\n  }\n\n  async _call({ file_path, text }: z.infer<typeof this.schema>) {\n    await this.store.writeFile(file_path, text);\n    return \"File written to successfully.\";\n  }\n}\n\n// Path: langchain/src/experimental/autogpt/tests/prompt_generator.test.ts\nimport { test, expect } from \"@jest/globals\";\nimport { z } from \"zod\";\n\nimport { StructuredTool } from \"@langchain/core/tools\";\nimport { getPrompt } from \"../prompt_generator.js\";\nimport { Calculator } from \"../../../tools/calculator.js\";\nimport { ReadFileTool, WriteFileTool } from \"../../../tools/fs.js\";\nimport { InMemoryFileStore } from \"../../../stores/file/in_memory.js\";\n\nclass FakeBrowserTool extends StructuredTool {\n  schema = z.object({\n    url: z.string(),\n    query: z.string().optional(),\n  });\n\n  name = \"fake_browser_tool\";\n\n  description =\n    \"useful for when you need to find something on the web or summarize a webpage.\";\n\n  async _call({\n    url: _url,\n    query: _query,\n  }: z.infer<this[\"schema\"]>): Promise<string> {\n    return \"fake_browser_tool\";\n  }\n}\n\ntest(\"prompt with several tools\", () => {\n  const store = new InMemoryFileStore();\n  const tools = [\n    new FakeBrowserTool(),\n    new Calculator(),\n    new ReadFileTool({ store }),\n    new WriteFileTool({ store }),\n  ];\n  const prompt = getPrompt(tools);\n  expect(prompt).toMatchInlineSnapshot(`\n    \"Constraints:\n    1. ~4000 word limit for short term memory. Your short term memory is short, so immediately save important information to files.\n    2. If you are unsure how you previously did something or want to recall past events, thinking about similar events will help you remember.\n    3. No user assistance\n    4. Exclusively use the commands listed in double quotes e.g. \"command name\"\n\n    Commands:\n    1. \"fake_browser_tool\": useful for when you need to find something on the web or summarize a webpage., args json schema: {\"url\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}\n    2. \"calculator\": Useful for getting the result of a math expression. The input to this tool should be a valid mathematical expression that could be executed by a simple calculator., args json schema: {\"input\":{\"type\":\"string\"}}\n    3. \"read_file\": Read file from disk, args json schema: {\"file_path\":{\"type\":\"string\",\"description\":\"name of file\"}}\n    4. \"write_file\": Write file from disk, args json schema: {\"file_path\":{\"type\":\"string\",\"description\":\"name of file\"},\"text\":{\"type\":\"string\",\"description\":\"text to write to file\"}}\n    5. finish: use this to signal that you have finished all your objectives, args: \"response\": \"final response to let people know you have finished your objectives\"\n\n    Resources:\n    1. Internet access for searches and information gathering.\n    2. Long Term memory management.\n    3. GPT-3.5 powered Agents for delegation of simple tasks.\n    4. File output.\n\n    Performance Evaluation:\n    1. Continuously review and analyze your actions to ensure you are performing to the best of your abilities.\n    2. Constructively self-criticize your big-picture behavior constantly.\n    3. Reflect on past decisions and strategies to refine your approach.\n    4. Every command has a cost, so be smart and efficient. Aim to complete tasks in the least number of steps.\n\n    You should only respond in JSON format as described below \n    Response Format: \n    {\n        \"thoughts\": {\n            \"text\": \"thought\",\n            \"reasoning\": \"reasoning\",\n            \"plan\": \"- short bulleted\\\\n- list that conveys\\\\n- long-term plan\",\n            \"criticism\": \"constructive self-criticism\",\n            \"speak\": \"thoughts summary to say to user\"\n        },\n        \"command\": {\n            \"name\": \"command name\",\n            \"args\": {\n                \"arg name\": \"value\"\n            }\n        }\n    } \n    Ensure the response can be parsed by Python json.loads\"\n  `);\n});\n\n// Path: langchain/src/experimental/autogpt/tests/prompt.test.ts\nimport { HumanMessage, SystemMessage } from \"@langchain/core/messages\";\nimport { AutoGPTPrompt } from \"../prompt.js\";\n\n// Mock token counter function\nconst mockTokenCounter = async (text: string): Promise<number> => text.length;\n\n// Mock vector store retriever interface\n// Todo: replace any with actual interface\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nconst mockMemory: any = {\n  getRelevantDocuments: async () => [\n    { pageContent: \"relevant content\", metadata: {} },\n  ],\n};\n\ndescribe(\"AutoGPTPrompt\", () => {\n  it(\"should construct full prompt correctly\", () => {\n    const prompt = new AutoGPTPrompt({\n      aiName: \"TestAI\",\n      aiRole: \"Assistant\",\n      tools: [],\n      tokenCounter: mockTokenCounter,\n      sendTokenLimit: 2500,\n    });\n\n    const goals = [\"Goal1\", \"Goal2\"];\n    const fullPrompt = prompt.constructFullPrompt(goals);\n    expect(fullPrompt).toContain(\"TestAI\");\n    expect(fullPrompt).toContain(\"Assistant\");\n    expect(fullPrompt).toContain(\"Goal1\");\n    expect(fullPrompt).toContain(\"Goal2\");\n  });\n\n  it(\"should format messages correctly\", async () => {\n    const prompt = new AutoGPTPrompt({\n      aiName: \"TestAI\",\n      aiRole: \"Assistant\",\n      tools: [],\n      tokenCounter: mockTokenCounter,\n      sendTokenLimit: 2500,\n    });\n\n    const formattedMessages = await prompt.formatMessages({\n      goals: [\"Goal1\"],\n      memory: mockMemory,\n      messages: [\n        new HumanMessage(\"Hello\"),\n        new SystemMessage(\"System message\"),\n      ],\n      user_input: \"User input\",\n    });\n\n    expect(formattedMessages).toHaveLength(4); // Base prompt, time prompt, memory message, and 2 previous messages\n\n    // Check the content of the first message (base prompt)\n    expect(formattedMessages[0].content).toContain(\"TestAI\");\n    expect(formattedMessages[0].content).toContain(\"Assistant\");\n    expect(formattedMessages[0].content).toContain(\"Goal1\");\n\n    // Check the content of the second message (time prompt)\n    expect(formattedMessages[1].content).toMatch(\n      /\\d{1,2}\\/\\d{1,2}\\/\\d{4}, \\d{1,2}:\\d{1,2}:\\d{1,2} (AM|PM)/\n    );\n\n    // Check the content of the third message (memory message)\n    expect(formattedMessages[2].content).toContain(\"relevant content\");\n\n    //  Check the content of the previous messages\n    const humanMessage = formattedMessages.find(\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      (msg) => msg instanceof HumanMessage\n    );\n    const systemMessage = formattedMessages.find(\n      // eslint-disable-next-line no-instanceof/no-instanceof\n      (msg) => msg instanceof SystemMessage\n    );\n\n    // Validate HumanMessage\n    expect(humanMessage).toBeDefined();\n\n    // Validate SystemMessage\n    expect(systemMessage).toBeDefined();\n\n    // Validate user_input\n    expect(formattedMessages[3].content).toContain(\"User input\");\n  });\n});\n\n// Path: langchain/src/experimental/openai_files/tests/openai_file.int.test.ts\nimport * as fs from \"fs\";\nimport * as path from \"path\";\nimport { fileURLToPath } from \"url\";\nimport { dirname } from \"path\";\nimport { OpenAIFiles } from \"../index.js\";\n\n/**\n * Otherwise we got the error __dirname doesn't exist\n */\nconst __filename = fileURLToPath(import.meta.url);\nconst __dirname = dirname(__filename);\n\ntest(\"Use file with Open AI\", async () => {\n  const openAIFiles = new OpenAIFiles();\n  const file = await openAIFiles.createFile({\n    file: fs.createReadStream(path.resolve(__dirname, `./test.jsonl`)),\n    purpose: \"fine-tune\",\n  });\n  expect(file.id).toBeDefined();\n  expect(file.object).toBe(\"file\");\n  /**\n   * Output \n    {\n      \"id\": \"file-BK7bzQj3FfZFXr7DbL6xJwfo\",\n      \"object\": \"file\",\n      \"bytes\": 120000,\n      \"created_at\": 1677610602,\n      \"filename\": \"salesOverview.pdf\",\n      \"purpose\": \"assistants\",\n    }\n   */\n  const fileContent = await openAIFiles.retrieveFileContent({\n    fileId: file.id,\n  });\n  console.log(fileContent);\n  expect(fileContent).toBeDefined();\n  /**\n   * Output \n    {\n      \"id\": \"file-BK7bzQj3FfZFXr7DbL6xJwfo\",\n      \"object\": \"file\",\n      \"bytes\": 120000,\n      \"created_at\": 1677610602,\n      \"filename\": \"salesOverview.pdf\",\n      \"purpose\": \"assistants\",\n    }\n   */\n  const retrievedFile = await openAIFiles.retrieveFile({\n    fileId: file.id,\n  });\n  expect(retrievedFile.id).toBeDefined();\n  expect(retrievedFile.object).toBe(\"file\");\n  /**\n   * Output \n    {\n      \"id\": \"file-BK7bzQj3FfZFXr7DbL6xJwfo\",\n      \"object\": \"file\",\n      \"bytes\": 120000,\n      \"created_at\": 1677610602,\n      \"filename\": \"salesOverview.pdf\",\n      \"purpose\": \"assistants\",\n    }\n   */\n  const list = await openAIFiles.listFiles();\n  expect(list).toBeDefined();\n  expect(!!list.data.find((f) => f.id === file.id)).toBeTruthy();\n  /**\n   * Output \n    {\n      \"id\": \"file-BK7bzQj3FfZFXr7DbL6xJwfo\",\n      \"object\": \"file\",\n      \"bytes\": 120000,\n      \"created_at\": 1677610602,\n      \"filename\": \"salesOverview.pdf\",\n      \"purpose\": \"assistants\",\n    }\n   */\n  const result = await openAIFiles.deleteFile({ fileId: file.id });\n  expect(result.id).toBe(file.id);\n  expect(result.deleted).toBeTruthy();\n  /**\n   * Output:\n   {\n      \"id\": \"file-abc123\",\n      \"object\": \"file\",\n      \"deleted\": true\n    }\n   */\n});\n\n// Path: langchain/src/experimental/babyagi/agent.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { CallbackManagerForChainRun } from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, ChainInputs } from \"../../chains/base.js\";\nimport { SerializedBaseChain } from \"../../chains/serde.js\";\nimport { Optional } from \"../../types/type-utils.js\";\nimport { TaskCreationChain } from \"./task_creation.js\";\nimport { TaskExecutionChain } from \"./task_execution.js\";\nimport { TaskPrioritizationChain } from \"./task_prioritization.js\";\n\n/**\n * Interface defining the structure of a task. A task has a `taskID` and a\n * `taskName`.\n */\nexport interface Task {\n  taskID: string;\n  taskName: string;\n}\n\n/**\n * Interface defining the structure of the inputs for the `BabyAGI` class.\n * It extends the `ChainInputs` interface, omitting the 'memory' and\n * 'callbackManager' properties, and adds properties specific to\n * `BabyAGI`.\n */\nexport interface BabyAGIInputs\n  extends Omit<ChainInputs, \"memory\" | \"callbackManager\"> {\n  creationChain: BaseChain;\n  prioritizationChain: BaseChain;\n  executionChain: BaseChain;\n  vectorstore: VectorStoreInterface;\n  maxIterations?: number;\n}\n\n/**\n * Class responsible for managing tasks, including their creation,\n * prioritization, and execution. It uses three chains for these\n * operations: `creationChain`, `prioritizationChain`, and\n * `executionChain`.\n * @example\n * ```typescript\n * const babyAGI = BabyAGI.fromLLM({\n *   llm: new OpenAI({ temperature: 0 }),\n *   vectorstore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   maxIterations: 3,\n * });\n *\n * const result = await babyAGI.call({\n *   objective: \"Write a weather report for SF today\",\n * });\n * ```\n */\nexport class BabyAGI extends BaseChain implements BabyAGIInputs {\n  static lc_name() {\n    return \"BabyAGI\";\n  }\n\n  taskList: Task[];\n\n  creationChain: BaseChain;\n\n  prioritizationChain: BaseChain;\n\n  executionChain: BaseChain;\n\n  taskIDCounter: number;\n\n  vectorstore: VectorStoreInterface;\n\n  maxIterations: number;\n\n  constructor({\n    creationChain,\n    prioritizationChain,\n    executionChain,\n    vectorstore,\n    maxIterations = 100,\n    verbose,\n    callbacks,\n  }: BabyAGIInputs) {\n    super(undefined, verbose, callbacks);\n    this.taskList = [];\n    this.creationChain = creationChain;\n    this.prioritizationChain = prioritizationChain;\n    this.executionChain = executionChain;\n    this.taskIDCounter = 1;\n    this.vectorstore = vectorstore;\n    this.maxIterations = maxIterations;\n  }\n\n  _chainType() {\n    return \"BabyAGI\" as const;\n  }\n\n  get inputKeys() {\n    return [\"objective\", \"firstTask\"];\n  }\n\n  get outputKeys() {\n    return [];\n  }\n\n  /**\n   * Adds a task to the task list.\n   * @param task The task to be added.\n   * @returns Promise resolving to void.\n   */\n  async addTask(task: Task) {\n    this.taskList.push(task);\n  }\n\n  /**\n   * Prints the current task list to the console.\n   * @returns void\n   */\n  printTaskList() {\n    console.log(\"\\x1b[95m\\x1b[1m\\n*****TASK LIST*****\\n\\x1b[0m\\x1b[0m\");\n    for (const t of this.taskList) {\n      console.log(`${t.taskID}: ${t.taskName}`);\n    }\n  }\n\n  /**\n   * Prints the next task to the console.\n   * @param task The next task to be printed.\n   * @returns void\n   */\n  printNextTask(task: Task) {\n    console.log(\"\\x1b[92m\\x1b[1m\\n*****NEXT TASK*****\\n\\x1b[0m\\x1b[0m\");\n    console.log(`${task.taskID}: ${task.taskName}`);\n  }\n\n  /**\n   * Prints the result of a task to the console.\n   * @param result The result of the task.\n   * @returns void\n   */\n  printTaskResult(result: string) {\n    console.log(\"\\x1b[93m\\x1b[1m\\n*****TASK RESULT*****\\n\\x1b[0m\\x1b[0m\");\n    console.log(result.trim());\n  }\n\n  /**\n   * Generates the next tasks based on the result of the previous task, the\n   * task description, and the objective.\n   * @param result The result of the previous task.\n   * @param task_description The description of the task.\n   * @param objective The objective of the task.\n   * @param runManager Optional CallbackManagerForChainRun instance.\n   * @returns Promise resolving to an array of tasks without taskID.\n   */\n  async getNextTasks(\n    result: string,\n    task_description: string,\n    objective: string,\n    runManager?: CallbackManagerForChainRun\n  ): Promise<Optional<Task, \"taskID\">[]> {\n    const taskNames = this.taskList.map((t) => t.taskName);\n    const incomplete_tasks = taskNames.join(\", \");\n    const { [this.creationChain.outputKeys[0]]: text } =\n      await this.creationChain.call(\n        {\n          result,\n          task_description,\n          incomplete_tasks,\n          objective,\n        },\n        runManager?.getChild()\n      );\n    const newTasks = (text as string).split(\"\\n\");\n    return newTasks\n      .filter((taskName) => taskName.trim())\n      .map((taskName) => ({ taskName }));\n  }\n\n  /**\n   * Prioritizes the tasks based on the current task ID and the objective.\n   * @param thisTaskID The ID of the current task.\n   * @param objective The objective of the task.\n   * @param runManager Optional CallbackManagerForChainRun instance.\n   * @returns Promise resolving to an array of prioritized tasks.\n   */\n  async prioritizeTasks(\n    thisTaskID: number,\n    objective: string,\n    runManager?: CallbackManagerForChainRun\n  ) {\n    const taskNames = this.taskList.map((t) => t.taskName);\n    const nextTaskID = thisTaskID + 1;\n    const { [this.prioritizationChain.outputKeys[0]]: text } =\n      await this.prioritizationChain.call(\n        {\n          task_names: taskNames.join(\", \"),\n          next_task_id: String(nextTaskID),\n          objective,\n        },\n        runManager?.getChild()\n      );\n    const newTasks = (text as string).trim().split(\"\\n\");\n    const prioritizedTaskList = [];\n    for (const taskString of newTasks) {\n      const taskParts = taskString.trim().split(\".\", 2);\n      if (taskParts.length === 2) {\n        const taskID = taskParts[0].trim();\n        const taskName = taskParts[1].trim();\n        prioritizedTaskList.push({ taskID, taskName });\n      }\n    }\n    return prioritizedTaskList;\n  }\n\n  /**\n   * Retrieves the top tasks that are most similar to the given query.\n   * @param query The query to search for.\n   * @param k The number of top tasks to retrieve.\n   * @returns Promise resolving to an array of top tasks.\n   */\n  async getTopTasks(query: string, k = 5) {\n    const results = await this.vectorstore.similaritySearch(query, k);\n    if (!results) {\n      return [];\n    }\n    return results.map((item) => String(item.metadata.task));\n  }\n\n  /**\n   * Executes a task based on the objective and the task description.\n   * @param objective The objective of the task.\n   * @param task The task to be executed.\n   * @param runManager Optional CallbackManagerForChainRun instance.\n   * @returns Promise resolving to the result of the task execution as a string.\n   */\n  async executeTask(\n    objective: string,\n    task: string,\n    runManager?: CallbackManagerForChainRun\n  ) {\n    const context = await this.getTopTasks(objective);\n    const { [this.executionChain.outputKeys[0]]: text } =\n      await this.executionChain.call(\n        {\n          objective,\n          context: context.join(\"\\n\"),\n          task,\n        },\n        runManager?.getChild()\n      );\n    return text as string;\n  }\n\n  async _call(\n    { objective, firstTask = \"Make a todo list\" }: ChainValues,\n    runManager?: CallbackManagerForChainRun\n  ) {\n    this.taskList = [];\n    this.taskIDCounter = 1;\n    await this.addTask({ taskID: \"1\", taskName: firstTask });\n\n    let numIters = 0;\n    while (numIters < this.maxIterations && this.taskList.length > 0) {\n      this.printTaskList();\n\n      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n      const task = this.taskList.shift()!;\n      this.printNextTask(task);\n\n      const result = await this.executeTask(\n        objective,\n        task.taskName,\n        runManager\n      );\n      const thisTaskID = parseInt(task.taskID, 10);\n      this.printTaskResult(result);\n\n      await this.vectorstore.addDocuments([\n        new Document({\n          pageContent: result,\n          metadata: { task: task.taskName },\n        }),\n      ]);\n\n      const newTasks = await this.getNextTasks(\n        result,\n        task.taskName,\n        objective,\n        runManager\n      );\n      for (const newTask of newTasks) {\n        this.taskIDCounter += 1;\n        newTask.taskID = this.taskIDCounter.toFixed();\n        await this.addTask(newTask as Task);\n      }\n      this.taskList = await this.prioritizeTasks(\n        thisTaskID,\n        objective,\n        runManager\n      );\n\n      numIters += 1;\n    }\n    return {};\n  }\n\n  serialize(): SerializedBaseChain {\n    throw new Error(\"Method not implemented.\");\n  }\n\n  /**\n   * Static method to create a new BabyAGI instance from a\n   * BaseLanguageModel.\n   * @param llm BaseLanguageModel instance used to generate a new BabyAGI instance.\n   * @param vectorstore VectorStore instance used to store and retrieve vectors.\n   * @param executionChain Optional BaseChain instance used to execute tasks.\n   * @param verbose Optional boolean indicating whether to log verbose output.\n   * @param callbacks Optional callbacks to be used during the execution of tasks.\n   * @param rest Optional additional parameters.\n   * @returns A new instance of BabyAGI.\n   */\n  static fromLLM({\n    llm,\n    vectorstore,\n    executionChain,\n    verbose,\n    callbacks,\n    ...rest\n  }: Optional<\n    BabyAGIInputs,\n    \"executionChain\" | \"creationChain\" | \"prioritizationChain\"\n  > & { llm: BaseLanguageModelInterface }) {\n    const creationChain = TaskCreationChain.fromLLM({\n      llm,\n      verbose,\n      callbacks,\n    });\n    const prioritizationChain = TaskPrioritizationChain.fromLLM({\n      llm,\n      verbose,\n      callbacks,\n    });\n    return new BabyAGI({\n      creationChain,\n      prioritizationChain,\n      executionChain:\n        executionChain ||\n        TaskExecutionChain.fromLLM({ llm, verbose, callbacks }),\n      vectorstore,\n      verbose,\n      callbacks,\n      ...rest,\n    });\n  }\n}\n\n// Path: langchain/src/experimental/llms/bittensor.ts\nimport { BaseLLMParams, LLM } from \"@langchain/core/language_models/llms\";\n\nexport interface BittensorInput extends BaseLLMParams {\n  systemPrompt?: string | null | undefined;\n  topResponses?: number | undefined;\n}\ninterface Message {\n  role: string;\n  content: string;\n}\n\ninterface ChatPayload {\n  uids?: string[];\n  top_n?: number;\n  messages: Message[];\n}\n\ninterface APIKeyResponse {\n  api_key: string;\n}\n\ninterface ChatResponse {\n  choices?: { message: Message }[];\n}\n/**\n * Class representing the Neural Internet language model powerd by Bittensor, a decentralized network\n * full of different AI models.\n * To analyze API_KEYS and logs of you usage visit\n *      https://api.neuralinternet.ai/api-keys\n *      https://api.neuralinternet.ai/logs\n */\nexport class NIBittensorLLM extends LLM implements BittensorInput {\n  static lc_name(): string {\n    return \"NIBittensorLLM\";\n  }\n\n  systemPrompt: string;\n\n  topResponses: number | undefined;\n\n  constructor(fields?: BittensorInput) {\n    super(fields ?? {});\n    this.systemPrompt =\n      fields?.systemPrompt ??\n      \"You are an assistant which is created by Neural Internet(NI) in decentralized network named as a Bittensor. Your task is to provide accurate response based on user prompt\";\n    this.topResponses = fields?.topResponses;\n  }\n\n  _llmType(): string {\n    return \"NIBittensorLLM\";\n  }\n\n  /** Call out to NIBittensorLLM's complete endpoint.\n   Args:\n       prompt: The prompt to pass into the model.\n\n       Returns: The string generated by the model.\n\n   Example:\n   let response = niBittensorLLM.call(\"Tell me a joke.\");\n   */\n  async _call(prompt: string): Promise<string> {\n    try {\n      // Retrieve API KEY\n      const apiKeyResponse: Response = await fetch(\n        \"https://test.neuralinternet.ai/admin/api-keys/\"\n      );\n      if (!apiKeyResponse.ok) {\n        throw new Error(\"Network response was not ok\");\n      }\n      const apiKeysData: APIKeyResponse[] = await apiKeyResponse.json();\n      const apiKey: string = apiKeysData[0].api_key;\n\n      const headers = {\n        \"Content-Type\": \"application/json\",\n        Authorization: `Bearer ${apiKey}`,\n        \"Endpoint-Version\": \"2023-05-19\",\n      };\n      if (this.topResponses !== undefined) {\n        this.topResponses = this.topResponses > 100 ? 100 : this.topResponses;\n      } else {\n        this.topResponses = 0;\n      }\n\n      const minerResponse: Response = await fetch(\n        \"https://test.neuralinternet.ai/top_miner_uids\",\n        { headers }\n      );\n      if (!minerResponse.ok) {\n        throw new Error(\"Network response was not ok\");\n      }\n      const uids: string[] = await minerResponse.json();\n\n      if (Array.isArray(uids) && uids.length && this.topResponses === 0) {\n        for (const uid of uids) {\n          try {\n            const payload: ChatPayload = {\n              uids: [uid],\n              messages: [\n                { role: \"system\", content: this.systemPrompt },\n                { role: \"user\", content: prompt },\n              ],\n            };\n\n            const response: Response = await fetch(\n              \"https://test.neuralinternet.ai/chat\",\n              {\n                method: \"POST\",\n                headers,\n                body: JSON.stringify(payload),\n              }\n            );\n\n            if (!response.ok) {\n              throw new Error(\"Network response was not ok\");\n            }\n\n            const chatData: ChatResponse = await response.json();\n\n            if (chatData.choices) {\n              return chatData.choices[0].message.content;\n            }\n          } catch (error) {\n            continue;\n          }\n        }\n      }\n\n      // For top miner based on bittensor response\n      if (this.topResponses === 0) {\n        this.topResponses = 10;\n      }\n      const payload: ChatPayload = {\n        top_n: this.topResponses,\n        messages: [\n          { role: \"system\", content: this.systemPrompt },\n          { role: \"user\", content: prompt },\n        ],\n      };\n\n      const response: Response = await fetch(\n        \"https://test.neuralinternet.ai/chat\",\n        {\n          method: \"POST\",\n          headers,\n          body: JSON.stringify(payload),\n        }\n      );\n\n      if (!response.ok) {\n        throw new Error(\"Network response was not ok\");\n      }\n\n      const responseData: ChatResponse | string = await response.json();\n\n      if (this.topResponses) {\n        return <string>responseData;\n      } else if ((<ChatResponse>responseData).choices) {\n        // eslint-disable-next-line @typescript-eslint/no-explicit-any\n        const temp: any = (<ChatResponse>responseData).choices;\n        return <string>temp[0].message.content;\n      }\n    } catch (error) {\n      return \"Sorry I am unable to provide response now, Please try again later.\";\n    }\n    return \"default\";\n  }\n\n  identifyingParams(): {\n    systemPrompt: string | null | undefined;\n    topResponses: number | undefined;\n  } {\n    return {\n      systemPrompt: this.systemPrompt,\n      topResponses: this.topResponses,\n    };\n  }\n}\n\n// Path: langchain/src/experimental/llms/tests/bittensor.int.test.ts\nimport { test, describe } from \"@jest/globals\";\nimport { NIBittensorLLM } from \"../bittensor.js\";\n\ndescribe.skip(\"NIBittensorLLM\", () => {\n  test(\"test with no params\", async () => {\n    const niBittensorLLM = new NIBittensorLLM();\n    const result = await niBittensorLLM.call(\"What is Bittensor?\");\n    console.log(\"test with no params: \", result);\n  });\n  test(\"test with system prompt\", async () => {\n    const niBittensorLLM = new NIBittensorLLM({\n      systemPrompt:\n        \"You are an assistant which is created by Neural Internet(NI) in decentralized network named as a Bittensor. Your task is to provide accurate response based on user prompt\",\n    });\n    const result = await niBittensorLLM.call(\"What is Bittensor?\");\n    console.log(\"test with system prompt: \", result);\n  });\n  test(\"test with topResponse parameter\", async () => {\n    const niBittensorLLM = new NIBittensorLLM({\n      systemPrompt:\n        \"You are an assistant which is created by Neural Internet(NI) in decentralized network named as a Bittensor. Your task is to provide accurate response based on user prompt\",\n      topResponses: 10,\n    });\n    const result = await niBittensorLLM.call(\"What is Bittensor?\");\n    console.log(\"test with topResponse parameter: \", result);\n  });\n});\n\n// Path: langchain/src/experimental/plan_and_execute/tests/plan_and_execute.int.test.ts\n/* eslint-disable no-process-env */\nimport { ChatOpenAI } from \"@langchain/openai\";\nimport { SerpAPI } from \"@langchain/community/tools/serpapi\";\nimport { PlanAndExecuteAgentExecutor } from \"../agent_executor.js\";\nimport { Calculator } from \"../../../tools/calculator.js\";\n\ntest.skip(\"Run agent on a simple input\", async () => {\n  const tools = [new Calculator(), new SerpAPI()];\n  const model = new ChatOpenAI({\n    temperature: 0,\n    modelName: \"gpt-3.5-turbo\",\n    verbose: true,\n  });\n  const executor = await PlanAndExecuteAgentExecutor.fromLLMAndTools({\n    llm: model,\n    tools,\n  });\n\n  const result = await executor.call({\n    input: `What is 80 raised to the second power?`,\n  });\n\n  console.log({ result });\n});\n\ntest.skip(\"Run agent\", async () => {\n  const tools = [new Calculator(), new SerpAPI()];\n  const model = new ChatOpenAI({\n    temperature: 0,\n    modelName: \"gpt-3.5-turbo\",\n    verbose: true,\n  });\n  const executor = await PlanAndExecuteAgentExecutor.fromLLMAndTools({\n    llm: model,\n    tools,\n  });\n\n  const result = await executor.call({\n    input: `Who is the current president of the United States? What is their current age raised to the second power?`,\n  });\n\n  console.log({ result });\n});\n\n// TODO: Improve prompt to store compressed context to support this input\ntest.skip(\"Run agent with a sequential math problem\", async () => {\n  const tools = [new Calculator()];\n  const model = new ChatOpenAI({\n    temperature: 0,\n    modelName: \"gpt-3.5-turbo\",\n    verbose: true,\n  });\n  const executor = await PlanAndExecuteAgentExecutor.fromLLMAndTools({\n    llm: model,\n    tools,\n  });\n\n  const result = await executor.call({\n    input: `In a dance class of 20 students, 20% enrolled in contemporary dance, 25% of the remaining enrolled in jazz dance, and the rest enrolled in hip-hop dance. What percentage of the entire students enrolled in hip-hop dance?`,\n  });\n\n  console.log(result);\n});\n\n// Path: langchain/src/experimental/prompts/handlebars.ts\nimport Handlebars from \"handlebars\";\nimport { type ParsedFStringNode } from \"@langchain/core/prompts\";\nimport type { InputValues } from \"@langchain/core/utils/types\";\nimport {\n  CustomFormatPromptTemplate,\n  CustomFormatPromptTemplateInput,\n} from \"./custom_format.js\";\n\nexport const parseHandlebars = (template: string): ParsedFStringNode[] => {\n  const parsed: ParsedFStringNode[] = [];\n  const nodes: { type: string }[] = [...Handlebars.parse(template).body];\n  while (nodes.length) {\n    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion\n    const node = nodes.pop()!;\n    if (node.type === \"ContentStatement\") {\n      // @ts-expect-error - handlebars' hbs.AST.ContentStatement isn't exported\n      const text = node.value;\n      parsed.push({ type: \"literal\", text });\n    } else if (node.type === \"MustacheStatement\") {\n      // @ts-expect-error - handlebars' hbs.AST.MustacheStatement isn't exported\n      const name: string = node.path.parts[0];\n      // @ts-expect-error - handlebars' hbs.AST.MustacheStatement isn't exported\n      const { original } = node.path as { original: string };\n      if (\n        !!name &&\n        !original.startsWith(\"this.\") &&\n        !original.startsWith(\"@\")\n      ) {\n        parsed.push({ type: \"variable\", name });\n      }\n    } else if (node.type === \"PathExpression\") {\n      // @ts-expect-error - handlebars' hbs.AST.PathExpression isn't exported\n      const name: string = node.parts[0];\n      // @ts-expect-error - handlebars' hbs.AST.PathExpression isn't exported\n      const { original } = node;\n      if (\n        !!name &&\n        !original.startsWith(\"this.\") &&\n        !original.startsWith(\"@\")\n      ) {\n        parsed.push({ type: \"variable\", name });\n      }\n    } else if (node.type === \"BlockStatement\") {\n      // @ts-expect-error - handlebars' hbs.AST.BlockStatement isn't exported\n      nodes.push(...node.params, ...node.program.body);\n    }\n  }\n\n  return parsed;\n};\n\nexport const interpolateHandlebars = (\n  template: string,\n  values: InputValues\n) => {\n  const compiled = Handlebars.compile(template, { noEscape: true });\n  return compiled(values);\n};\n\nexport type HandlebarsPromptTemplateInput<RunInput extends InputValues> =\n  CustomFormatPromptTemplateInput<RunInput>;\n\nexport class HandlebarsPromptTemplate<\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  RunInput extends InputValues = any\n> extends CustomFormatPromptTemplate<RunInput> {\n  static lc_name() {\n    return \"HandlebarsPromptTemplate\";\n  }\n\n  /**\n   * Load prompt template from a template\n   */\n  static fromTemplate<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    RunInput extends InputValues = Record<string, any>\n  >(\n    template: string,\n    params?: Omit<\n      HandlebarsPromptTemplateInput<RunInput>,\n      | \"template\"\n      | \"inputVariables\"\n      | \"customParser\"\n      | \"templateValidator\"\n      | \"renderer\"\n    >\n  ) {\n    return super.fromTemplate<RunInput>(template, {\n      ...params,\n      validateTemplate: false,\n      customParser: parseHandlebars,\n      renderer: interpolateHandlebars,\n    });\n  }\n}\n\n// Path: langchain/src/experimental/prompts/tests/handlebars.test.ts\nimport { HandlebarsPromptTemplate } from \"../handlebars.js\";\n\ndescribe.each([\n  [\"{{foo}}\", { foo: \"bar\" }, \"bar\"],\n  [\"pre{{foo}}post\", { foo: \"bar\" }, \"prebarpost\"],\n  [\"{{{foo}}}\", { foo: \"bar\" }, \"bar\"],\n  [\"text\", {}, \"text\"],\n  [\"}}\", {}, \"}}\"],\n  [\"{{first}}_{{second}}\", { first: \"foo\", second: \"bar\" }, \"foo_bar\"],\n])(\"Valid handlebars\", (template, variables, result) => {\n  test(`Interpolation works: ${template}`, async () => {\n    const prompt = HandlebarsPromptTemplate.fromTemplate(template);\n    const invokeResult = await prompt.invoke(variables);\n    expect(invokeResult.value).toBe(result);\n  });\n});\n\ndescribe.each([\n  [\"}}{{\", {}],\n  [\"{{\", {}],\n  [\"{{foo\", {}],\n])(\"Invalid handlebars\", (template) => {\n  test(`Interpolation throws: ${template}`, async () => {\n    expect(() =>\n      HandlebarsPromptTemplate.fromTemplate(template)\n    ).toThrowError();\n  });\n});\n\n// Path: langchain/src/types/googlevertexai-types.ts\nimport { BaseLLMParams } from \"@langchain/core/language_models/llms\";\n\nexport interface GoogleConnectionParams<AuthOptions> {\n  authOptions?: AuthOptions;\n}\n\nexport interface GoogleVertexAIConnectionParams<AuthOptions>\n  extends GoogleConnectionParams<AuthOptions> {\n  /** Hostname for the API call */\n  endpoint?: string;\n\n  /** Region where the LLM is stored */\n  location?: string;\n\n  /** The version of the API functions. Part of the path. */\n  apiVersion?: string;\n}\n\nexport interface GoogleVertexAIModelParams {\n  /** Model to use */\n  model?: string;\n\n  /** Sampling temperature to use */\n  temperature?: number;\n\n  /**\n   * Maximum number of tokens to generate in the completion.\n   */\n  maxOutputTokens?: number;\n\n  /**\n   * Top-p changes how the model selects tokens for output.\n   *\n   * Tokens are selected from most probable to least until the sum\n   * of their probabilities equals the top-p value.\n   *\n   * For example, if tokens A, B, and C have a probability of\n   * .3, .2, and .1 and the top-p value is .5, then the model will\n   * select either A or B as the next token (using temperature).\n   */\n  topP?: number;\n\n  /**\n   * Top-k changes how the model selects tokens for output.\n   *\n   * A top-k of 1 means the selected token is the most probable among\n   * all tokens in the model\u2019s vocabulary (also called greedy decoding),\n   * while a top-k of 3 means that the next token is selected from\n   * among the 3 most probable tokens (using temperature).\n   */\n  topK?: number;\n}\n\nexport interface GoogleVertexAIBaseLLMInput<AuthOptions>\n  extends BaseLLMParams,\n    GoogleVertexAIConnectionParams<AuthOptions>,\n    GoogleVertexAIModelParams {}\n\nexport interface GoogleResponse {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  data: any;\n}\n\nexport interface GoogleVertexAIBasePrediction {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  safetyAttributes?: any;\n}\n\nexport interface GoogleVertexAILLMPredictions<\n  PredictionType extends GoogleVertexAIBasePrediction\n> {\n  predictions: PredictionType[];\n}\n\nexport type GoogleAbstractedClientOpsMethod = \"GET\" | \"POST\";\n\nexport type GoogleAbstractedClientOpsResponseType = \"json\" | \"stream\";\n\nexport type GoogleAbstractedClientOps = {\n  url?: string;\n  method?: GoogleAbstractedClientOpsMethod;\n  data?: unknown;\n  responseType?: GoogleAbstractedClientOpsResponseType;\n};\n\nexport interface GoogleAbstractedClient {\n  request: (opts: GoogleAbstractedClientOps) => unknown;\n  getProjectId: () => Promise<string>;\n}\n\n// Path: langchain/src/util/googlevertexai-connection.ts\nimport { BaseLanguageModelCallOptions } from \"@langchain/core/language_models/base\";\nimport { GenerationChunk } from \"@langchain/core/outputs\";\nimport {\n  AsyncCaller,\n  AsyncCallerCallOptions,\n} from \"@langchain/core/utils/async_caller\";\nimport type {\n  GoogleVertexAIBaseLLMInput,\n  GoogleVertexAIBasePrediction,\n  GoogleVertexAIConnectionParams,\n  GoogleVertexAILLMPredictions,\n  GoogleVertexAIModelParams,\n  GoogleResponse,\n  GoogleAbstractedClient,\n  GoogleAbstractedClientOps,\n  GoogleAbstractedClientOpsMethod,\n} from \"../types/googlevertexai-types.js\";\n\nexport abstract class GoogleConnection<\n  CallOptions extends AsyncCallerCallOptions,\n  ResponseType extends GoogleResponse\n> {\n  caller: AsyncCaller;\n\n  client: GoogleAbstractedClient;\n\n  streaming: boolean;\n\n  constructor(\n    caller: AsyncCaller,\n    client: GoogleAbstractedClient,\n    streaming?: boolean\n  ) {\n    this.caller = caller;\n    this.client = client;\n    this.streaming = streaming ?? false;\n  }\n\n  abstract buildUrl(): Promise<string>;\n\n  abstract buildMethod(): GoogleAbstractedClientOpsMethod;\n\n  async _request(\n    data: unknown | undefined,\n    options: CallOptions\n  ): Promise<ResponseType> {\n    const url = await this.buildUrl();\n    const method = this.buildMethod();\n\n    const opts: GoogleAbstractedClientOps = {\n      url,\n      method,\n    };\n    if (data && method === \"POST\") {\n      opts.data = data;\n    }\n    if (this.streaming) {\n      opts.responseType = \"stream\";\n    } else {\n      opts.responseType = \"json\";\n    }\n\n    const callResponse = await this.caller.callWithOptions(\n      { signal: options?.signal },\n      async () => this.client.request(opts)\n    );\n    const response: unknown = callResponse; // Done for typecast safety, I guess\n    return <ResponseType>response;\n  }\n}\n\nexport abstract class GoogleVertexAIConnection<\n    CallOptions extends AsyncCallerCallOptions,\n    ResponseType extends GoogleResponse,\n    AuthOptions\n  >\n  extends GoogleConnection<CallOptions, ResponseType>\n  implements GoogleVertexAIConnectionParams<AuthOptions>\n{\n  endpoint = \"us-central1-aiplatform.googleapis.com\";\n\n  location = \"us-central1\";\n\n  apiVersion = \"v1\";\n\n  constructor(\n    fields: GoogleVertexAIConnectionParams<AuthOptions> | undefined,\n    caller: AsyncCaller,\n    client: GoogleAbstractedClient,\n    streaming?: boolean\n  ) {\n    super(caller, client, streaming);\n    this.caller = caller;\n\n    this.endpoint = fields?.endpoint ?? this.endpoint;\n    this.location = fields?.location ?? this.location;\n    this.apiVersion = fields?.apiVersion ?? this.apiVersion;\n    this.client = client;\n  }\n\n  buildMethod(): GoogleAbstractedClientOpsMethod {\n    return \"POST\";\n  }\n}\n\nexport \nfunction complexValue(value: unknown): unknown {\n  if (value === null || typeof value === \"undefined\") {\n    // I dunno what to put here. An error, probably\n    return undefined;\n  } else if (typeof value === \"object\") {\n    if (Array.isArray(value)) {\n      return {\n        list_val: value.map((avalue) => complexValue(avalue)),\n      };\n    } else {\n      const ret: Record<string, unknown> = {};\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      const v: Record<string, any> = value;\n      Object.keys(v).forEach((key) => {\n        ret[key] = complexValue(v[key]);\n      });\n      return { struct_val: ret };\n    }\n  } else if (typeof value === \"number\") {\n    if (Number.isInteger(value)) {\n      return { int_val: value };\n    } else {\n      return { float_val: value };\n    }\n  } else {\n    return {\n      string_val: [value],\n    };\n  }\n}\n\nexport function simpleValue(val: unknown): unknown {\n  if (val && typeof val === \"object\" && !Array.isArray(val)) {\n    // eslint-disable-next-line no-prototype-builtins\n    if (val.hasOwnProperty(\"stringVal\")) {\n      return (val as { stringVal: string[] }).stringVal[0];\n\n      // eslint-disable-next-line no-prototype-builtins\n    } else if (val.hasOwnProperty(\"boolVal\")) {\n      return (val as { boolVal: boolean[] }).boolVal[0];\n\n      // eslint-disable-next-line no-prototype-builtins\n    } else if (val.hasOwnProperty(\"listVal\")) {\n      const { listVal } = val as { listVal: unknown[] };\n      return listVal.map((aval) => simpleValue(aval));\n\n      // eslint-disable-next-line no-prototype-builtins\n    } else if (val.hasOwnProperty(\"structVal\")) {\n      const ret: Record<string, unknown> = {};\n      const struct = (val as { structVal: Record<string, unknown> }).structVal;\n      Object.keys(struct).forEach((key) => {\n        ret[key] = simpleValue(struct[key]);\n      });\n      return ret;\n    } else {\n      const ret: Record<string, unknown> = {};\n      const struct = val as Record<string, unknown>;\n      Object.keys(struct).forEach((key) => {\n        ret[key] = simpleValue(struct[key]);\n      });\n      return ret;\n    }\n  } else if (Array.isArray(val)) {\n    return val.map((aval) => simpleValue(aval));\n  } else {\n    return val;\n  }\n}\n\nexport class GoogleVertexAILLMConnection<\n    CallOptions extends BaseLanguageModelCallOptions,\n    InstanceType,\n    PredictionType extends GoogleVertexAIBasePrediction,\n    AuthOptions\n  >\n  extends GoogleVertexAIConnection<\n    CallOptions,\n    GoogleVertexAILLMResponse<PredictionType>,\n    AuthOptions\n  >\n  implements GoogleVertexAIBaseLLMInput<AuthOptions>\n{\n  model: string;\n\n  client: GoogleAbstractedClient;\n\n  constructor(\n    fields: GoogleVertexAIBaseLLMInput<AuthOptions> | undefined,\n    caller: AsyncCaller,\n    client: GoogleAbstractedClient,\n    streaming?: boolean\n  ) {\n    super(fields, caller, client, streaming);\n    this.client = client;\n    this.model = fields?.model ?? this.model;\n  }\n\n  async buildUrl(): Promise<string> {\n    const projectId = await this.client.getProjectId();\n    const method = this.streaming ? \"serverStreamingPredict\" : \"predict\";\n    const url = `https://${this.endpoint}/v1/projects/${projectId}/locations/${this.location}/publishers/google/models/${this.model}:${method}`;\n    return url;\n  }\n\n  formatStreamingData(\n    inputs: InstanceType[],\n    parameters: GoogleVertexAIModelParams\n  ): unknown {\n    return {\n      inputs: [inputs.map((i) => complexValue(i))],\n      parameters: complexValue(parameters),\n    };\n  }\n\n  formatStandardData(\n    instances: InstanceType[],\n    parameters: GoogleVertexAIModelParams\n  ): unknown {\n    return {\n      instances,\n      parameters,\n    };\n  }\n\n  formatData(\n    instances: InstanceType[],\n    parameters: GoogleVertexAIModelParams\n  ): unknown {\n    return this.streaming\n      ? this.formatStreamingData(instances, parameters)\n      : this.formatStandardData(instances, parameters);\n  }\n\n  async request(\n    instances: InstanceType[],\n    parameters: GoogleVertexAIModelParams,\n    options: CallOptions\n  ): Promise<GoogleVertexAILLMResponse<PredictionType>> {\n    const data = this.formatData(instances, parameters);\n    const response = await this._request(data, options);\n    return response;\n  }\n}\n\nexport interface GoogleVertexAILLMResponse<\n  PredictionType extends GoogleVertexAIBasePrediction\n> extends GoogleResponse {\n  data: GoogleVertexAIStream | GoogleVertexAILLMPredictions<PredictionType>;\n}\n\nexport class GoogleVertexAIStream {\n  _buffer = \"\";\n\n  _bufferOpen = true;\n\n  _firstRun = true;\n\n  /**\n   * Add data to the buffer. This may cause chunks to be generated, if available.\n   * @param data\n   */\n  appendBuffer(data: string): void {\n    this._buffer += data;\n    // Our first time, skip to the opening of the array\n    if (this._firstRun) {\n      this._skipTo(\"[\");\n      this._firstRun = false;\n    }\n\n    this._parseBuffer();\n  }\n\n  /**\n   * Indicate there is no more data that will be added to the text buffer.\n   * This should be called when all the data has been read and added to indicate\n   * that we should process everything remaining in the buffer.\n   */\n  closeBuffer(): void {\n    this._bufferOpen = false;\n    this._parseBuffer();\n  }\n\n  /**\n   * Skip characters in the buffer till we get to the start of an object.\n   * Then attempt to read a full object.\n   * If we do read a full object, turn it into a chunk and send it to the chunk handler.\n   * Repeat this for as much as we can.\n   */\n  _parseBuffer(): void {\n    let obj = null;\n    do {\n      this._skipTo(\"{\");\n      obj = this._getFullObject();\n      if (obj !== null) {\n        const chunk = this._simplifyObject(obj);\n        this._handleChunk(chunk);\n      }\n    } while (obj !== null);\n\n    if (!this._bufferOpen) {\n      // No more data will be added, and we have parsed everything we could,\n      // so everything else is garbage.\n      this._handleChunk(null);\n      this._buffer = \"\";\n    }\n  }\n\n  /**\n   * If the string is present, move the start of the buffer to the first occurrence\n   * of that string. This is useful for skipping over elements or parts that we're not\n   * really interested in parsing. (ie - the opening characters, comma separators, etc.)\n   * @param start The string to start the buffer with\n   */\n  _skipTo(start: string): void {\n    const index = this._buffer.indexOf(start);\n    if (index > 0) {\n      this._buffer = this._buffer.slice(index);\n    }\n  }\n\n  /**\n   * Given what is in the buffer, parse a single object out of it.\n   * If a complete object isn't available, return null.\n   * Assumes that we are at the start of an object to parse.\n   */\n  _getFullObject(): object | null {\n    let ret: object | null = null;\n\n    // Loop while we don't have something to return AND we have something in the buffer\n    let index = 0;\n    while (ret === null && this._buffer.length > index) {\n      // Advance to the next close bracket after our current index\n      index = this._buffer.indexOf(\"}\", index + 1);\n\n      // If we don't find one, exit with null\n      if (index === -1) {\n        return null;\n      }\n\n      // If we have one, try to turn it into an object to return\n      try {\n        const objStr = this._buffer.substring(0, index + 1);\n        ret = JSON.parse(objStr);\n\n        // We only get here if it parsed it ok\n        // If we did turn it into an object, remove it from the buffer\n        this._buffer = this._buffer.slice(index + 1);\n      } catch (xx) {\n        // It didn't parse it correctly, so we swallow the exception and continue\n      }\n    }\n\n    return ret;\n  }\n\n  _simplifyObject(obj: unknown): object {\n    return simpleValue(obj) as object;\n  }\n\n  // Set up a potential Promise that the handler can resolve.\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  _chunkResolution: (chunk: any) => void;\n\n  // If there is no Promise (it is null), the handler must add it to the queue\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  _chunkPending: Promise<any> | null = null;\n\n  // A queue that will collect chunks while there is no Promise\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  _chunkQueue: any[] = [];\n\n  /**\n   * Register that we have another chunk available for consumption.\n   * If we are waiting for a chunk, resolve the promise waiting for it immediately.\n   * If not, then add it to the queue.\n   * @param chunk\n   */\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  _handleChunk(chunk: any): void {\n    if (this._chunkPending) {\n      this._chunkResolution(chunk);\n      this._chunkPending = null;\n...\n", "needle_token_start": 13531, "needle_token_end": 13808, "code_context_ntokens": 16095, "output": ["```typescript\nfunction complexValue(value: unknown): unknown {\n  if (value === null || typeof value === \"undefined\") {\n    // I dunno what to put here. An error, probably\n    return undefined;\n  } else if (typeof value === \"object\") {\n    if (Array.isArray(value)) {\n      return {\n        list_val: value.map((avalue) => complexValue(avalue)),\n      };\n    } else {\n      const ret: Record<string, unknown> = {};\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      const v: Record<string, any> = value;\n      Object.keys(v).forEach((key) => {\n        ret[key] = complexValue(v[key]);\n      });\n      return { struct_val: ret };\n    }\n  } else if (typeof value === \"number\") {\n    if (Number.isInteger(value)) {\n      return { int_val: value };\n    } else {\n      return { float_val: value };\n    }\n  } else {\n    return {\n      string_val: [value],\n    };\n  }\n}\n```"]}
{"repo": "langchain-ai/langchainjs", "name": "isAgentAction", "language": "typescript", "path": "langchain/src/agents/agent.ts", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine if a given input is an instance of a specific action type used by agents, excluding arrays and ensuring the presence of a particular property.\n2. **Input**: A variable of unknown type.\n3. **Output**: A boolean value indicating whether the input matches the criteria for a specific action type.\n4. **Procedure**: The function checks if the input is not an array and then verifies if a specific property exists within the input, returning true if both conditions are met, otherwise false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: langchain/src/chains/history_aware_retriever.ts\nimport type { LanguageModelLike } from \"@langchain/core/language_models/base\";\nimport {\n  type Runnable,\n  type RunnableInterface,\n  RunnableSequence,\n  RunnableBranch,\n} from \"@langchain/core/runnables\";\nimport { type BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { StringOutputParser } from \"@langchain/core/output_parsers\";\nimport type { DocumentInterface } from \"@langchain/core/documents\";\nimport type { BaseMessage } from \"@langchain/core/messages\";\n\n/**\n * Params for the createHistoryAwareRetriever method.\n */\nexport type CreateHistoryAwareRetrieverParams = {\n  /**\n   * Language model to use for generating a search term given chat history.\n   */\n  llm: LanguageModelLike;\n  /**\n   * RetrieverLike object that takes a string as input and outputs a list of Documents.\n   */\n  retriever: RunnableInterface<string, DocumentInterface[]>;\n  /**\n   * The prompt used to generate the search query for the retriever.\n   */\n  rephrasePrompt: BasePromptTemplate;\n};\n\n/**\n * Create a chain that takes conversation history and returns documents.\n * If there is no `chat_history`, then the `input` is just passed directly to the\n * retriever. If there is `chat_history`, then the prompt and LLM will be used\n * to generate a search query. That search query is then passed to the retriever.\n * @param {CreateHistoryAwareRetriever} params\n * @returns An LCEL Runnable. The runnable input must take in `input`, and if there\n * is chat history should take it in the form of `chat_history`.\n * The Runnable output is a list of Documents\n * @example\n * ```typescript\n * // yarn add langchain @langchain/openai\n *\n * import { ChatOpenAI } from \"@langchain/openai\";\n * import { pull } from \"langchain/hub\";\n * import { createRetrievalChain } from \"langchain/chains/retrieval\";\n * import { createStuffDocumentsChain } from \"langchain/chains/combine_documents\";\n *\n * const rephrasePrompt = await pull(\"langchain-ai/chat-langchain-rephrase\");\n * const llm = new ChatOpenAI({});\n * const retriever = ...\n * const chain = await createHistoryAwareRetriever({\n *   llm,\n *   retriever,\n *   rephrasePrompt,\n * });\n * const result = await chain.invoke({\"input\": \"...\", \"chat_history\": [] })\n * ```\n */\nexport async function createHistoryAwareRetriever({\n  llm,\n  retriever,\n  rephrasePrompt,\n}: CreateHistoryAwareRetrieverParams): Promise<\n  Runnable<\n    { input: string; chat_history: string | BaseMessage[] },\n    DocumentInterface[]\n  >\n> {\n  if (!rephrasePrompt.inputVariables.includes(\"input\")) {\n    throw new Error(\n      `Expected \"input\" to be a prompt variable, but got ${JSON.stringify(\n        rephrasePrompt.inputVariables\n      )}`\n    );\n  }\n  const retrieveDocuments = RunnableBranch.from([\n    [\n      (input) => !input.chat_history || input.chat_history.length === 0,\n      RunnableSequence.from([(input) => input.input, retriever]),\n    ],\n    RunnableSequence.from([\n      rephrasePrompt,\n      llm,\n      new StringOutputParser(),\n      retriever,\n    ]),\n  ]).withConfig({\n    runName: \"history_aware_retriever\",\n  });\n...\n// Path: langchain/src/retrievers/time_weighted.ts\nimport { BaseRetriever, BaseRetrieverInput } from \"@langchain/core/retrievers\";\nimport type { VectorStoreInterface } from \"@langchain/core/vectorstores\";\nimport type { DocumentInterface } from \"@langchain/core/documents\";\nimport { CallbackManagerForRetrieverRun } from \"@langchain/core/callbacks/manager\";\n\n/**\n * Interface for the fields required to initialize a\n * TimeWeightedVectorStoreRetriever instance.\n */\nexport interface TimeWeightedVectorStoreRetrieverFields\n  extends BaseRetrieverInput {\n  vectorStore: VectorStoreInterface;\n  searchKwargs?: number;\n  memoryStream?: DocumentInterface[];\n  decayRate?: number;\n  k?: number;\n  otherScoreKeys?: string[];\n  defaultSalience?: number;\n}\n\nexport const LAST_ACCESSED_AT_KEY = \"last_accessed_at\";\nexport const BUFFER_IDX = \"buffer_idx\";\n\n/**\n * TimeWeightedVectorStoreRetriever retrieves documents based on their time-weighted relevance.\n * ref: https://github.com/langchain-ai/langchain/blob/master/libs/langchain/langchain/retrievers/time_weighted_retriever.py\n * @example\n * ```typescript\n * const retriever = new TimeWeightedVectorStoreRetriever({\n *   vectorStore: new MemoryVectorStore(new OpenAIEmbeddings()),\n *   memoryStream: [],\n *   searchKwargs: 2,\n * });\n * await retriever.addDocuments([\n *   { pageContent: \"My name is John.\", metadata: {} },\n *   { pageContent: \"My favourite food is pizza.\", metadata: {} },\n *\n * ]);\n * const results = await retriever.getRelevantDocuments(\n *   \"What is my favourite food?\",\n * );\n * ```\n */\nexport class TimeWeightedVectorStoreRetriever extends BaseRetriever {\n  static lc_name() {\n    return \"TimeWeightedVectorStoreRetriever\";\n  }\n\n  get lc_namespace() {\n    return [\"langchain\", \"retrievers\", \"time_weighted\"];\n  }\n\n  /**\n   * The vectorstore to store documents and determine salience.\n   */\n  private vectorStore: VectorStoreInterface;\n\n  /**\n   * The number of top K most relevant documents to consider when searching.\n   */\n  private searchKwargs: number;\n\n  /**\n   * The memory_stream of documents to search through.\n   */\n  private memoryStream: DocumentInterface[];\n\n  /**\n   * The exponential decay factor used as (1.0-decay_rate)**(hrs_passed).\n   */\n  private decayRate: number;\n\n  /**\n   * The maximum number of documents to retrieve in a given call.\n   */\n  private k: number;\n\n  /**\n   * Other keys in the metadata to factor into the score, e.g. 'importance'.\n   */\n  private otherScoreKeys: string[];\n\n  /**\n   * The salience to assign memories not retrieved from the vector store.\n   */\n  private defaultSalience: number | null;\n\n  /**\n   * Constructor to initialize the required fields\n   * @param fields - The fields required for initializing the TimeWeightedVectorStoreRetriever\n   */\n  constructor(fields: TimeWeightedVectorStoreRetrieverFields) {\n    super(fields);\n    this.vectorStore = fields.vectorStore;\n    this.searchKwargs = fields.searchKwargs ?? 100;\n    this.memoryStream = fields.memoryStream ?? [];\n    this.decayRate = fields.decayRate ?? 0.01;\n    this.k = fields.k ?? 4;\n    this.otherScoreKeys = fields.otherScoreKeys ?? [];\n    this.defaultSalience = fields.defaultSalience ?? null;\n  }\n\n  /**\n   * Get the memory stream of documents.\n   * @returns The memory stream of documents.\n   */\n  getMemoryStream(): DocumentInterface[] {\n    return this.memoryStream;\n  }\n\n  /**\n   * Set the memory stream of documents.\n   * @param memoryStream The new memory stream of documents.\n   */\n  setMemoryStream(memoryStream: DocumentInterface[]) {\n    this.memoryStream = memoryStream;\n  }\n\n  /**\n   * Get relevant documents based on time-weighted relevance\n   * @param query - The query to search for\n   * @returns The relevant documents\n   */\n  async _getRelevantDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<DocumentInterface[]> {\n    const now = Math.floor(Date.now() / 1000);\n    const memoryDocsAndScores = this.getMemoryDocsAndScores();\n\n    const salientDocsAndScores = await this.getSalientDocuments(\n      query,\n      runManager\n    );\n    const docsAndScores = { ...memoryDocsAndScores, ...salientDocsAndScores };\n\n    return this.computeResults(docsAndScores, now);\n  }\n\n  /**\n   * NOTE: When adding documents to a vector store, use addDocuments\n   * via retriever instead of directly to the vector store.\n   * This is because it is necessary to process the document\n   * in prepareDocuments.\n   *\n   * @param docs - The documents to add to vector store in the retriever\n   */\n  async addDocuments(docs: DocumentInterface[]): Promise<void> {\n    const now = Math.floor(Date.now() / 1000);\n    const savedDocs = this.prepareDocuments(docs, now);\n\n    this.memoryStream.push(...savedDocs);\n    await this.vectorStore.addDocuments(savedDocs);\n  }\n\n  /**\n   * Get memory documents and their scores\n   * @returns An object containing memory documents and their scores\n   */\n  private getMemoryDocsAndScores(): Record<\n    number,\n    { doc: DocumentInterface; score: number }\n  > {\n    const memoryDocsAndScores: Record<\n      number,\n      { doc: DocumentInterface; score: number }\n    > = {};\n    for (const doc of this.memoryStream.slice(-this.k)) {\n      const bufferIdx = doc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      memoryDocsAndScores[bufferIdx] = {\n        doc,\n        score: this.defaultSalience ?? 0,\n      };\n    }\n    return memoryDocsAndScores;\n  }\n\n  /**\n   * Get salient documents and their scores based on the query\n   * @param query - The query to search for\n   * @returns An object containing salient documents and their scores\n   */\n  private async getSalientDocuments(\n    query: string,\n    runManager?: CallbackManagerForRetrieverRun\n  ): Promise<Record<number, { doc: DocumentInterface; score: number }>> {\n    const docAndScores: [DocumentInterface, number][] =\n      await this.vectorStore.similaritySearchWithScore(\n        query,\n        this.searchKwargs,\n        undefined,\n        runManager?.getChild()\n      );\n    const results: Record<number, { doc: DocumentInterface; score: number }> =\n      {};\n    for (const [fetchedDoc, score] of docAndScores) {\n      const bufferIdx = fetchedDoc.metadata[BUFFER_IDX];\n      if (bufferIdx === undefined) {\n        throw new Error(\n          `Found a document in the vector store that is missing required metadata. This retriever only supports vector stores with documents that have been added through the \"addDocuments\" method on a TimeWeightedVectorStoreRetriever, not directly added or loaded into the backing vector store.`\n        );\n      }\n      const doc = this.memoryStream[bufferIdx];\n      results[bufferIdx] = { doc, score };\n    }\n    return results;\n  }\n\n  /**\n   * Compute the final result set of documents based on the combined scores\n   * @param docsAndScores - An object containing documents and their scores\n   * @param now - The current timestamp\n   * @returns The final set of documents\n   */\n  private computeResults(\n    docsAndScores: Record<number, { doc: DocumentInterface; score: number }>,\n    now: number\n  ): DocumentInterface[] {\n    const recordedDocs = Object.values(docsAndScores)\n      .map(({ doc, score }) => ({\n        doc,\n        score: this.getCombinedScore(doc, score, now),\n      }))\n      .sort((a, b) => b.score - a.score);\n\n    const results: DocumentInterface[] = [];\n    for (const { doc } of recordedDocs) {\n      const bufferedDoc = this.memoryStream[doc.metadata[BUFFER_IDX]];\n      bufferedDoc.metadata[LAST_ACCESSED_AT_KEY] = now;\n      results.push(bufferedDoc);\n      if (results.length > this.k) {\n        break;\n      }\n    }\n    return results;\n  }\n\n  /**\n   * Prepare documents with necessary metadata before saving\n   * @param docs - The documents to prepare\n   * @param now - The current timestamp\n   * @returns The prepared documents\n   */\n  private prepareDocuments(\n    docs: DocumentInterface[],\n    now: number\n  ): DocumentInterface[] {\n    return docs.map((doc, i) => ({\n      ...doc,\n      metadata: {\n        ...doc.metadata,\n        [LAST_ACCESSED_AT_KEY]: doc.metadata[LAST_ACCESSED_AT_KEY] ?? now,\n        created_at: doc.metadata.created_at ?? now,\n        [BUFFER_IDX]: this.memoryStream.length + i,\n      },\n    }));\n  }\n\n  /**\n   * Calculate the combined score based on vector relevance and other factors\n   * @param doc - The document to calculate the score for\n   * @param vectorRelevance - The relevance score from the vector store\n   * @param nowMsec - The current timestamp in milliseconds\n   * @returns The combined score for the document\n   */\n  private getCombinedScore(\n    doc: DocumentInterface,\n    vectorRelevance: number | null,\n    nowMsec: number\n  ): number {\n    const hoursPassed = this.getHoursPassed(\n      nowMsec,\n      doc.metadata[LAST_ACCESSED_AT_KEY]\n    );\n    let score = (1.0 - this.decayRate) ** hoursPassed;\n    for (const key of this.otherScoreKeys) {\n      score += doc.metadata[key];\n    }\n    if (vectorRelevance !== null) {\n      score += vectorRelevance;\n    }\n    return score;\n  }\n\n  /**\n   * Calculate the hours passed between two time points\n   * @param time - The current time in seconds\n   * @param refTime - The reference time in seconds\n   * @returns The number of hours passed between the two time points\n   */\n  private getHoursPassed(time: number, refTime: number): number {\n    return (time - refTime) / 3600;\n  }\n}\n\n// Path: langchain/src/text_splitter.ts\nimport type * as tiktoken from \"js-tiktoken\";\nimport { Document, BaseDocumentTransformer } from \"@langchain/core/documents\";\nimport { getEncoding } from \"@langchain/core/utils/tiktoken\";\n\nexport interface TextSplitterParams {\n  chunkSize: number;\n  chunkOverlap: number;\n  keepSeparator: boolean;\n  lengthFunction?:\n    | ((text: string) => number)\n    | ((text: string) => Promise<number>);\n}\n\nexport type TextSplitterChunkHeaderOptions = {\n  chunkHeader?: string;\n  chunkOverlapHeader?: string;\n  appendChunkOverlapHeader?: boolean;\n};\n\nexport abstract class TextSplitter\n  extends BaseDocumentTransformer\n  implements TextSplitterParams\n{\n  lc_namespace = [\"langchain\", \"document_transformers\", \"text_splitters\"];\n\n  chunkSize = 1000;\n\n  chunkOverlap = 200;\n\n  keepSeparator = false;\n\n  lengthFunction:\n    | ((text: string) => number)\n    | ((text: string) => Promise<number>);\n\n  constructor(fields?: Partial<TextSplitterParams>) {\n    super(fields);\n    this.chunkSize = fields?.chunkSize ?? this.chunkSize;\n    this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;\n    this.keepSeparator = fields?.keepSeparator ?? this.keepSeparator;\n    this.lengthFunction =\n      fields?.lengthFunction ?? ((text: string) => text.length);\n    if (this.chunkOverlap >= this.chunkSize) {\n      throw new Error(\"Cannot have chunkOverlap >= chunkSize\");\n    }\n  }\n\n  async transformDocuments(\n    documents: Document[],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    return this.splitDocuments(documents, chunkHeaderOptions);\n  }\n\n  abstract splitText(text: string): Promise<string[]>;\n\n  protected splitOnSeparator(text: string, separator: string): string[] {\n    let splits;\n    if (separator) {\n      if (this.keepSeparator) {\n        const regexEscapedSeparator = separator.replace(\n          /[/\\-\\\\^$*+?.()|[\\]{}]/g,\n          \"\\\\$&\"\n        );\n        splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));\n      } else {\n        splits = text.split(separator);\n      }\n    } else {\n      splits = text.split(\"\");\n    }\n    return splits.filter((s) => s !== \"\");\n  }\n\n  async createDocuments(\n    texts: string[],\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    metadatas: Record<string, any>[] = [],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    // if no metadata is provided, we create an empty one for each text\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    const _metadatas: Record<string, any>[] =\n      metadatas.length > 0\n        ? metadatas\n        : [...Array(texts.length)].map(() => ({}));\n    const {\n      chunkHeader = \"\",\n      chunkOverlapHeader = \"(cont'd) \",\n      appendChunkOverlapHeader = false,\n    } = chunkHeaderOptions;\n    const documents = new Array<Document>();\n    for (let i = 0; i < texts.length; i += 1) {\n      const text = texts[i];\n      let lineCounterIndex = 1;\n      let prevChunk = null;\n      let indexPrevChunk = -1;\n      for (const chunk of await this.splitText(text)) {\n        let pageContent = chunkHeader;\n\n        // we need to count the \\n that are in the text before getting removed by the splitting\n        const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);\n        if (prevChunk === null) {\n          const newLinesBeforeFirstChunk = this.numberOfNewLines(\n            text,\n            0,\n            indexChunk\n          );\n          lineCounterIndex += newLinesBeforeFirstChunk;\n        } else {\n          const indexEndPrevChunk =\n            indexPrevChunk + (await this.lengthFunction(prevChunk));\n          if (indexEndPrevChunk < indexChunk) {\n            const numberOfIntermediateNewLines = this.numberOfNewLines(\n              text,\n              indexEndPrevChunk,\n              indexChunk\n            );\n            lineCounterIndex += numberOfIntermediateNewLines;\n          } else if (indexEndPrevChunk > indexChunk) {\n            const numberOfIntermediateNewLines = this.numberOfNewLines(\n              text,\n              indexChunk,\n              indexEndPrevChunk\n            );\n            lineCounterIndex -= numberOfIntermediateNewLines;\n          }\n          if (appendChunkOverlapHeader) {\n            pageContent += chunkOverlapHeader;\n          }\n        }\n        const newLinesCount = this.numberOfNewLines(chunk);\n\n        const loc =\n          _metadatas[i].loc && typeof _metadatas[i].loc === \"object\"\n            ? { ..._metadatas[i].loc }\n            : {};\n        loc.lines = {\n          from: lineCounterIndex,\n          to: lineCounterIndex + newLinesCount,\n        };\n        const metadataWithLinesNumber = {\n          ..._metadatas[i],\n          loc,\n        };\n\n        pageContent += chunk;\n        documents.push(\n          new Document({\n            pageContent,\n            metadata: metadataWithLinesNumber,\n          })\n        );\n        lineCounterIndex += newLinesCount;\n        prevChunk = chunk;\n        indexPrevChunk = indexChunk;\n      }\n    }\n    return documents;\n  }\n\n  private numberOfNewLines(text: string, start?: number, end?: number) {\n    const textSection = text.slice(start, end);\n    return (textSection.match(/\\n/g) || []).length;\n  }\n\n  async splitDocuments(\n    documents: Document[],\n    chunkHeaderOptions: TextSplitterChunkHeaderOptions = {}\n  ): Promise<Document[]> {\n    const selectedDocuments = documents.filter(\n      (doc) => doc.pageContent !== undefined\n    );\n    const texts = selectedDocuments.map((doc) => doc.pageContent);\n    const metadatas = selectedDocuments.map((doc) => doc.metadata);\n    return this.createDocuments(texts, metadatas, chunkHeaderOptions);\n  }\n\n  private joinDocs(docs: string[], separator: string): string | null {\n    const text = docs.join(separator).trim();\n    return text === \"\" ? null : text;\n  }\n\n  async mergeSplits(splits: string[], separator: string): Promise<string[]> {\n    const docs: string[] = [];\n    const currentDoc: string[] = [];\n    let total = 0;\n    for (const d of splits) {\n      const _len = await this.lengthFunction(d);\n      if (\n        total + _len + currentDoc.length * separator.length >\n        this.chunkSize\n      ) {\n        if (total > this.chunkSize) {\n          console.warn(\n            `Created a chunk of size ${total}, +\nwhich is longer than the specified ${this.chunkSize}`\n          );\n        }\n        if (currentDoc.length > 0) {\n          const doc = this.joinDocs(currentDoc, separator);\n          if (doc !== null) {\n            docs.push(doc);\n          }\n          // Keep on popping if:\n          // - we have a larger chunk than in the chunk overlap\n          // - or if we still have any chunks and the length is long\n          while (\n            total > this.chunkOverlap ||\n            (total + _len + currentDoc.length * separator.length >\n              this.chunkSize &&\n              total > 0)\n          ) {\n            total -= await this.lengthFunction(currentDoc[0]);\n            currentDoc.shift();\n          }\n        }\n      }\n      currentDoc.push(d);\n      total += _len;\n    }\n    const doc = this.joinDocs(currentDoc, separator);\n    if (doc !== null) {\n      docs.push(doc);\n    }\n    return docs;\n  }\n}\n\nexport interface CharacterTextSplitterParams extends TextSplitterParams {\n  separator: string;\n}\n\nexport class CharacterTextSplitter\n  extends TextSplitter\n  implements CharacterTextSplitterParams\n{\n  static lc_name() {\n    return \"CharacterTextSplitter\";\n  }\n\n  separator = \"\\n\\n\";\n\n  constructor(fields?: Partial<CharacterTextSplitterParams>) {\n    super(fields);\n    this.separator = fields?.separator ?? this.separator;\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    // First we naively split the large input into a bunch of smaller ones.\n    const splits = this.splitOnSeparator(text, this.separator);\n    return this.mergeSplits(splits, this.keepSeparator ? \"\" : this.separator);\n  }\n}\n\nexport interface RecursiveCharacterTextSplitterParams\n  extends TextSplitterParams {\n  separators: string[];\n}\n\nexport const SupportedTextSplitterLanguages = [\n  \"cpp\",\n  \"go\",\n  \"java\",\n  \"js\",\n  \"php\",\n  \"proto\",\n  \"python\",\n  \"rst\",\n  \"ruby\",\n  \"rust\",\n  \"scala\",\n  \"swift\",\n  \"markdown\",\n  \"latex\",\n  \"html\",\n  \"sol\",\n] as const;\n\nexport type SupportedTextSplitterLanguage =\n  (typeof SupportedTextSplitterLanguages)[number];\n\nexport class RecursiveCharacterTextSplitter\n  extends TextSplitter\n  implements RecursiveCharacterTextSplitterParams\n{\n  static lc_name() {\n    return \"RecursiveCharacterTextSplitter\";\n  }\n\n  separators: string[] = [\"\\n\\n\", \"\\n\", \" \", \"\"];\n\n  constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>) {\n    super(fields);\n    this.separators = fields?.separators ?? this.separators;\n    this.keepSeparator = fields?.keepSeparator ?? true;\n  }\n\n  private async _splitText(text: string, separators: string[]) {\n    const finalChunks: string[] = [];\n\n    // Get appropriate separator to use\n    let separator: string = separators[separators.length - 1];\n    let newSeparators;\n    for (let i = 0; i < separators.length; i += 1) {\n      const s = separators[i];\n      if (s === \"\") {\n        separator = s;\n        break;\n      }\n      if (text.includes(s)) {\n        separator = s;\n        newSeparators = separators.slice(i + 1);\n        break;\n      }\n    }\n\n    // Now that we have the separator, split the text\n    const splits = this.splitOnSeparator(text, separator);\n\n    // Now go merging things, recursively splitting longer texts.\n    let goodSplits: string[] = [];\n    const _separator = this.keepSeparator ? \"\" : separator;\n    for (const s of splits) {\n      if ((await this.lengthFunction(s)) < this.chunkSize) {\n        goodSplits.push(s);\n      } else {\n        if (goodSplits.length) {\n          const mergedText = await this.mergeSplits(goodSplits, _separator);\n          finalChunks.push(...mergedText);\n          goodSplits = [];\n        }\n        if (!newSeparators) {\n          finalChunks.push(s);\n        } else {\n          const otherInfo = await this._splitText(s, newSeparators);\n          finalChunks.push(...otherInfo);\n        }\n      }\n    }\n    if (goodSplits.length) {\n      const mergedText = await this.mergeSplits(goodSplits, _separator);\n      finalChunks.push(...mergedText);\n    }\n    return finalChunks;\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    return this._splitText(text, this.separators);\n  }\n\n  static fromLanguage(\n    language: SupportedTextSplitterLanguage,\n    options?: Partial<RecursiveCharacterTextSplitterParams>\n  ) {\n    return new RecursiveCharacterTextSplitter({\n      ...options,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(language),\n    });\n  }\n\n  static getSeparatorsForLanguage(language: SupportedTextSplitterLanguage) {\n    if (language === \"cpp\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along function definitions\n        \"\\nvoid \",\n        \"\\nint \",\n        \"\\nfloat \",\n        \"\\ndouble \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"go\") {\n      return [\n        // Split along function definitions\n        \"\\nfunc \",\n        \"\\nvar \",\n        \"\\nconst \",\n        \"\\ntype \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"java\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along method definitions\n        \"\\npublic \",\n        \"\\nprotected \",\n        \"\\nprivate \",\n        \"\\nstatic \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"js\") {\n      return [\n        // Split along function definitions\n        \"\\nfunction \",\n        \"\\nconst \",\n        \"\\nlet \",\n        \"\\nvar \",\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        \"\\ndefault \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"php\") {\n      return [\n        // Split along function definitions\n        \"\\nfunction \",\n        // Split along class definitions\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nforeach \",\n        \"\\nwhile \",\n        \"\\ndo \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"proto\") {\n      return [\n        // Split along message definitions\n        \"\\nmessage \",\n        // Split along service definitions\n        \"\\nservice \",\n        // Split along enum definitions\n        \"\\nenum \",\n        // Split along option definitions\n        \"\\noption \",\n        // Split along import statements\n        \"\\nimport \",\n        // Split along syntax declarations\n        \"\\nsyntax \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"python\") {\n      return [\n        // First, try to split along class definitions\n        \"\\nclass \",\n        \"\\ndef \",\n        \"\\n\\tdef \",\n        // Now split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"rst\") {\n      return [\n        // Split along section titles\n        \"\\n===\\n\",\n        \"\\n---\\n\",\n        \"\\n***\\n\",\n        // Split along directive markers\n        \"\\n.. \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"ruby\") {\n      return [\n        // Split along method definitions\n        \"\\ndef \",\n        \"\\nclass \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nunless \",\n        \"\\nwhile \",\n        \"\\nfor \",\n        \"\\ndo \",\n        \"\\nbegin \",\n        \"\\nrescue \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"rust\") {\n      return [\n        // Split along function definitions\n        \"\\nfn \",\n        \"\\nconst \",\n        \"\\nlet \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nwhile \",\n        \"\\nfor \",\n        \"\\nloop \",\n        \"\\nmatch \",\n        \"\\nconst \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"scala\") {\n      return [\n        // Split along class definitions\n        \"\\nclass \",\n        \"\\nobject \",\n        // Split along method definitions\n        \"\\ndef \",\n        \"\\nval \",\n        \"\\nvar \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\nmatch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"swift\") {\n      return [\n        // Split along function definitions\n        \"\\nfunc \",\n        // Split along class definitions\n        \"\\nclass \",\n        \"\\nstruct \",\n        \"\\nenum \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\ndo \",\n        \"\\nswitch \",\n        \"\\ncase \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"markdown\") {\n      return [\n        // First, try to split along Markdown headings (starting with level 2)\n        \"\\n## \",\n        \"\\n### \",\n        \"\\n#### \",\n        \"\\n##### \",\n        \"\\n###### \",\n        // Note the alternative syntax for headings (below) is not handled here\n        // Heading level 2\n        // ---------------\n        // End of code block\n        \"```\\n\\n\",\n        // Horizontal lines\n        \"\\n\\n***\\n\\n\",\n        \"\\n\\n---\\n\\n\",\n        \"\\n\\n___\\n\\n\",\n        // Note that this splitter doesn't handle horizontal lines defined\n        // by *three or more* of ***, ---, or ___, but this is not handled\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"latex\") {\n      return [\n        // First, try to split along Latex sections\n        \"\\n\\\\chapter{\",\n        \"\\n\\\\section{\",\n        \"\\n\\\\subsection{\",\n        \"\\n\\\\subsubsection{\",\n\n        // Now split by environments\n        \"\\n\\\\begin{enumerate}\",\n        \"\\n\\\\begin{itemize}\",\n        \"\\n\\\\begin{description}\",\n        \"\\n\\\\begin{list}\",\n        \"\\n\\\\begin{quote}\",\n        \"\\n\\\\begin{quotation}\",\n        \"\\n\\\\begin{verse}\",\n        \"\\n\\\\begin{verbatim}\",\n\n        // Now split by math environments\n        \"\\n\\\\begin{align}\",\n        \"$$\",\n        \"$\",\n\n        // Now split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"html\") {\n      return [\n        // First, try to split along HTML tags\n        \"<body>\",\n        \"<div>\",\n        \"<p>\",\n        \"<br>\",\n        \"<li>\",\n        \"<h1>\",\n        \"<h2>\",\n        \"<h3>\",\n        \"<h4>\",\n        \"<h5>\",\n        \"<h6>\",\n        \"<span>\",\n        \"<table>\",\n        \"<tr>\",\n        \"<td>\",\n        \"<th>\",\n        \"<ul>\",\n        \"<ol>\",\n        \"<header>\",\n        \"<footer>\",\n        \"<nav>\",\n        // Head\n        \"<head>\",\n        \"<style>\",\n        \"<script>\",\n        \"<meta>\",\n        \"<title>\",\n        // Normal type of lines\n        \" \",\n        \"\",\n      ];\n    } else if (language === \"sol\") {\n      return [\n        // Split along compiler informations definitions\n        \"\\npragma \",\n        \"\\nusing \",\n        // Split along contract definitions\n        \"\\ncontract \",\n        \"\\ninterface \",\n        \"\\nlibrary \",\n        // Split along method definitions\n        \"\\nconstructor \",\n        \"\\ntype \",\n        \"\\nfunction \",\n        \"\\nevent \",\n        \"\\nmodifier \",\n        \"\\nerror \",\n        \"\\nstruct \",\n        \"\\nenum \",\n        // Split along control flow statements\n        \"\\nif \",\n        \"\\nfor \",\n        \"\\nwhile \",\n        \"\\ndo while \",\n        \"\\nassembly \",\n        // Split by the normal type of lines\n        \"\\n\\n\",\n        \"\\n\",\n        \" \",\n        \"\",\n      ];\n    } else {\n      throw new Error(`Language ${language} is not supported.`);\n    }\n  }\n}\n\nexport interface TokenTextSplitterParams extends TextSplitterParams {\n  encodingName: tiktoken.TiktokenEncoding;\n  allowedSpecial: \"all\" | Array<string>;\n  disallowedSpecial: \"all\" | Array<string>;\n}\n\n/**\n * Implementation of splitter which looks at tokens.\n */\nexport class TokenTextSplitter\n  extends TextSplitter\n  implements TokenTextSplitterParams\n{\n  static lc_name() {\n    return \"TokenTextSplitter\";\n  }\n\n  encodingName: tiktoken.TiktokenEncoding;\n\n  allowedSpecial: \"all\" | Array<string>;\n\n  disallowedSpecial: \"all\" | Array<string>;\n\n  private tokenizer: tiktoken.Tiktoken;\n\n  constructor(fields?: Partial<TokenTextSplitterParams>) {\n    super(fields);\n\n    this.encodingName = fields?.encodingName ?? \"gpt2\";\n    this.allowedSpecial = fields?.allowedSpecial ?? [];\n    this.disallowedSpecial = fields?.disallowedSpecial ?? \"all\";\n  }\n\n  async splitText(text: string): Promise<string[]> {\n    if (!this.tokenizer) {\n      this.tokenizer = await getEncoding(this.encodingName);\n    }\n\n    const splits: string[] = [];\n\n    const input_ids = this.tokenizer.encode(\n      text,\n      this.allowedSpecial,\n      this.disallowedSpecial\n    );\n\n    let start_idx = 0;\n\n    while (start_idx < input_ids.length) {\n      if (start_idx > 0) {\n        start_idx -= this.chunkOverlap;\n      }\n      const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);\n      const chunk_ids = input_ids.slice(start_idx, end_idx);\n      splits.push(this.tokenizer.decode(chunk_ids));\n      start_idx = end_idx;\n    }\n\n    return splits;\n  }\n}\n\nexport type MarkdownTextSplitterParams = TextSplitterParams;\n\nexport class MarkdownTextSplitter\n  extends RecursiveCharacterTextSplitter\n  implements MarkdownTextSplitterParams\n{\n  constructor(fields?: Partial<MarkdownTextSplitterParams>) {\n    super({\n      ...fields,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(\"markdown\"),\n    });\n  }\n}\n\nexport type LatexTextSplitterParams = TextSplitterParams;\n\nexport class LatexTextSplitter\n  extends RecursiveCharacterTextSplitter\n  implements LatexTextSplitterParams\n{\n  constructor(fields?: Partial<LatexTextSplitterParams>) {\n    super({\n      ...fields,\n      separators:\n        RecursiveCharacterTextSplitter.getSeparatorsForLanguage(\"latex\"),\n    });\n  }\n}\n\n// Path: langchain/src/document_loaders/base.ts\nimport { Document } from \"@langchain/core/documents\";\nimport {\n  RecursiveCharacterTextSplitter,\n  TextSplitter,\n} from \"../text_splitter.js\";\n\n/**\n * Interface that defines the methods for loading and splitting documents.\n */\nexport interface DocumentLoader {\n  load(): Promise<Document[]>;\n  loadAndSplit(textSplitter?: TextSplitter): Promise<Document[]>;\n}\n\n/**\n * Abstract class that provides a default implementation for the\n * loadAndSplit() method from the DocumentLoader interface. The load()\n * method is left abstract and needs to be implemented by subclasses.\n */\nexport abstract class BaseDocumentLoader implements DocumentLoader {\n  /**\n   * Loads the documents.\n   * @returns A Promise that resolves with an array of Document instances.\n   */\n  abstract load(): Promise<Document[]>;\n\n  /**\n   * Loads the documents and splits them using a specified text splitter.\n   * @param textSplitter The TextSplitter instance to use for splitting the loaded documents. Defaults to a RecursiveCharacterTextSplitter instance.\n   * @returns A Promise that resolves with an array of Document instances, each split according to the provided TextSplitter.\n   */\n  async loadAndSplit(\n    splitter: TextSplitter = new RecursiveCharacterTextSplitter()\n  ): Promise<Document[]> {\n    const docs = await this.load();\n    return splitter.splitDocuments(docs);\n  }\n}\n\n// Path: langchain/src/document_loaders/web/serpapi.ts\nimport { Document } from \"@langchain/core/documents\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { BaseDocumentLoader } from \"../base.js\";\n\n/**\n * Interface representing the parameters for the SerpAPI loader. It\n * includes properties such as the search query and the API key.\n */\ninterface SerpAPIParameters {\n  /**\n   * Search Query\n   */\n  q: string;\n  apiKey?: string;\n}\n\n/**\n * Class representing a document loader for loading search results from\n * the SerpAPI. It extends the BaseDocumentLoader class.\n * @example\n * ```typescript\n * const loader = new SerpAPILoader({ q: \"{query}\", apiKey: \"{apiKey}\" });\n * const docs = await loader.load();\n * ```\n */\nexport class SerpAPILoader extends BaseDocumentLoader {\n  private apiKey: string;\n\n  private searchQuery: string;\n\n  constructor(params: SerpAPIParameters) {\n    super();\n    const { apiKey = getEnvironmentVariable(\"SERPAPI_API_KEY\"), q } = params;\n    if (!apiKey) {\n      throw new Error(\n        \"SerpAPI API key not set. You can set it as SERPAPI_API_KEY in your .env file, or pass it to SerpAPI.\"\n      );\n    }\n\n    this.apiKey = apiKey;\n    this.searchQuery = q;\n  }\n\n  /**\n   * Builds the URL for the SerpAPI search request.\n   * @returns The URL for the search request.\n   */\n  public buildUrl(): string {\n    const params = new URLSearchParams();\n    params.append(\"api_key\", this.apiKey);\n    params.append(\"q\", this.searchQuery);\n    return `https://serpapi.com/search?${params.toString()}`;\n  }\n\n  /**\n   * Extracts documents from the provided output.\n   * @param output - The output to extract documents from.\n   * @param responseType - The type of the response to extract documents from.\n   * @returns An array of Documents.\n   */\n  private extractDocuments(output: unknown, responseType: string): Document[] {\n    const documents: Document[] = [];\n    const results = Array.isArray(output) ? output : [output];\n    for (const result of results) {\n      const pageContent = JSON.stringify(result);\n      const metadata = {\n        source: \"SerpAPI\",\n        responseType,\n      };\n      documents.push(new Document({ pageContent, metadata }));\n    }\n    return documents;\n  }\n\n  /**\n   * Processes the response data from the SerpAPI search request and converts it into an array of Documents.\n   * @param data - The response data from the SerpAPI search request.\n   * @returns An array of Documents.\n   */\n  public processResponseData(data: Record<string, unknown>): Document[] {\n    const documents: Document[] = [];\n    const responseTypes = [\n      \"answer_box\",\n      \"sports_results\",\n      \"shopping_results\",\n      \"knowledge_graph\",\n      \"organic_results\",\n    ];\n    for (const responseType of responseTypes) {\n      if (responseType in data) {\n        documents.push(\n          ...this.extractDocuments(data[responseType], responseType)\n        );\n      }\n    }\n    return documents;\n  }\n\n  /**\n   * Fetches the data from the provided URL and returns it as a JSON object.\n   * If an error occurs during the fetch operation, an exception is thrown with the error message.\n   * @param url - The URL to fetch data from.\n   * @returns A promise that resolves to the fetched data as a JSON object.\n   * @throws An error if the fetch operation fails.\n   */\n  private async fetchData(url: string): Promise<Record<string, unknown>> {\n    const response = await fetch(url);\n    const data = await response.json();\n\n    if (data.error) {\n      throw new Error(\n        `Failed to load search results from SerpAPI due to: ${data.error}`\n      );\n    }\n\n    return data;\n  }\n\n  /**\n   * Loads the search results from the SerpAPI.\n   * @returns An array of Documents representing the search results.\n   * @throws An error if the search results could not be loaded.\n   */\n  public async load(): Promise<Document[]> {\n    const url = this.buildUrl();\n\n    const data = await this.fetchData(url);\n    try {\n      return this.processResponseData(data);\n    } catch (error) {\n      console.error(error);\n      throw new Error(\n        `Failed to process search results from SerpAPI: ${error}`\n      );\n    }\n  }\n}\n\n// Path: langchain/src/experimental/chains/violation_of_expectations/index.ts\nexport {\n  type ViolationOfExpectationsChainInput,\n  ViolationOfExpectationsChain,\n} from \"./violation_of_expectations_chain.js\";\n\n// Path: langchain/src/agents/mrkl/prompt.ts\nexport const PREFIX = `Answer the following questions as best you can. You have access to the following tools:`;\nexport const FORMAT_INSTRUCTIONS = `Use the following format in your response:\n\nQuestion: the input question you must answer\nThought: you should always think about what to do\nAction: the action to take, should be one of [{tool_names}]\nAction Input: the input to the action\nObservation: the result of the action\n... (this Thought/Action/Action Input/Observation can repeat N times)\nThought: I now know the final answer\nFinal Answer: the final answer to the original input question`;\nexport const SUFFIX = `Begin!\n\nQuestion: {input}\nThought:{agent_scratchpad}`;\n\n// Path: langchain/src/agents/helpers.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport type { ToolInterface } from \"@langchain/core/tools\";\nimport type { SerializedAgentT, AgentInput } from \"./types.js\";\nimport { LLMChain } from \"../chains/llm_chain.js\";\n\nexport const deserializeHelper = async <\n  T extends string,\n  U extends Record<string, unknown>,\n  V extends AgentInput,\n  Z\n>(\n  llm: BaseLanguageModelInterface | undefined,\n  tools: ToolInterface[] | undefined,\n  data: SerializedAgentT<T, U, V>,\n  fromLLMAndTools: (\n    llm: BaseLanguageModelInterface,\n    tools: ToolInterface[],\n    args: U\n  ) => Z,\n  fromConstructor: (args: V) => Z\n): Promise<Z> => {\n  if (data.load_from_llm_and_tools) {\n    if (!llm) {\n      throw new Error(\"Loading from llm and tools, llm must be provided.\");\n    }\n\n    if (!tools) {\n      throw new Error(\"Loading from llm and tools, tools must be provided.\");\n    }\n\n    return fromLLMAndTools(llm, tools, data);\n  }\n  if (!data.llm_chain) {\n    throw new Error(\"Loading from constructor, llm_chain must be provided.\");\n  }\n\n  const llmChain = await LLMChain.deserialize(data.llm_chain);\n  return fromConstructor({ ...data, llmChain });\n};\n\n// Path: langchain/src/agents/mrkl/index.ts\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { ToolInterface } from \"@langchain/core/tools\";\nimport { PromptTemplate, renderTemplate } from \"@langchain/core/prompts\";\nimport { LLMChain } from \"../../chains/llm_chain.js\";\nimport { Optional } from \"../../types/type-utils.js\";\nimport { Agent, AgentArgs, OutputParserArgs } from \"../agent.js\";\nimport { deserializeHelper } from \"../helpers.js\";\nimport {\n  AgentInput,\n  SerializedFromLLMAndTools,\n  SerializedZeroShotAgent,\n} from \"../types.js\";\nimport { ZeroShotAgentOutputParser } from \"./outputParser.js\";\nimport { FORMAT_INSTRUCTIONS, PREFIX, SUFFIX } from \"./prompt.js\";\n\n/**\n * Interface for creating a prompt for the ZeroShotAgent.\n */\nexport interface ZeroShotCreatePromptArgs {\n  /** String to put after the list of tools. */\n  suffix?: string;\n  /** String to put before the list of tools. */\n  prefix?: string;\n  /** List of input variables the final prompt will expect. */\n  inputVariables?: string[];\n}\n\n/**\n * Type for the input to the ZeroShotAgent, with the 'outputParser'\n * property made optional.\n */\nexport type ZeroShotAgentInput = Optional<AgentInput, \"outputParser\">;\n\n/**\n * Agent for the MRKL chain.\n * @augments Agent\n * @example\n * ```typescript\n *\n * const agent = new ZeroShotAgent({\n *   llmChain: new LLMChain({\n *     llm: new ChatOpenAI({ temperature: 0 }),\n *     prompt: ZeroShotAgent.createPrompt([new SerpAPI(), new Calculator()], {\n *       prefix: `Answer the following questions as best you can, but speaking as a pirate might speak. You have access to the following tools:`,\n *       suffix: `Begin! Remember to speak as a pirate when giving your final answer. Use lots of \"Args\"\n * Question: {input}\n * {agent_scratchpad}`,\n *       inputVariables: [\"input\", \"agent_scratchpad\"],\n *     }),\n *   }),\n *   allowedTools: [\"search\", \"calculator\"],\n * });\n *\n * const result = await agent.invoke({\n *   input: `Who is Olivia Wilde's boyfriend? What is his current age raised to the 0.23 power?`,\n * });\n * ```\n *\n * @deprecated Use the {@link https://api.js.langchain.com/functions/langchain_agents.createReactAgent.html | createReactAgent method instead}.\n */\nexport class ZeroShotAgent extends Agent {\n  static lc_name() {\n    return \"ZeroShotAgent\";\n  }\n\n  lc_namespace = [\"langchain\", \"agents\", \"mrkl\"];\n\n  declare ToolType: ToolInterface;\n\n  constructor(input: ZeroShotAgentInput) {\n    const outputParser =\n      input?.outputParser ?? ZeroShotAgent.getDefaultOutputParser();\n    super({ ...input, outputParser });\n  }\n\n  _agentType() {\n    return \"zero-shot-react-description\" as const;\n  }\n\n  observationPrefix() {\n    return \"Observation: \";\n  }\n\n  llmPrefix() {\n    return \"Thought:\";\n  }\n\n  /**\n   * Returns the default output parser for the ZeroShotAgent.\n   * @param fields Optional arguments for the output parser.\n   * @returns An instance of ZeroShotAgentOutputParser.\n   */\n  static getDefaultOutputParser(fields?: OutputParserArgs) {\n    return new ZeroShotAgentOutputParser(fields);\n  }\n\n  /**\n   * Validates the tools for the ZeroShotAgent. Throws an error if any tool\n   * does not have a description.\n   * @param tools List of tools to validate.\n   */\n  static validateTools(tools: ToolInterface[]) {\n    const descriptionlessTool = tools.find((tool) => !tool.description);\n    if (descriptionlessTool) {\n      const msg =\n        `Got a tool ${descriptionlessTool.name} without a description.` +\n        ` This agent requires descriptions for all tools.`;\n      throw new Error(msg);\n    }\n  }\n\n  /**\n   * Create prompt in the style of the zero shot agent.\n   *\n   * @param tools - List of tools the agent will have access to, used to format the prompt.\n   * @param args - Arguments to create the prompt with.\n   * @param args.suffix - String to put after the list of tools.\n   * @param args.prefix - String to put before the list of tools.\n   * @param args.inputVariables - List of input variables the final prompt will expect.\n   */\n  static createPrompt(tools: ToolInterface[], args?: ZeroShotCreatePromptArgs) {\n    const {\n      prefix = PREFIX,\n      suffix = SUFFIX,\n      inputVariables = [\"input\", \"agent_scratchpad\"],\n    } = args ?? {};\n    const toolStrings = tools\n      .map((tool) => `${tool.name}: ${tool.description}`)\n      .join(\"\\n\");\n\n    const toolNames = tools.map((tool) => tool.name);\n\n    const formatInstructions = renderTemplate(FORMAT_INSTRUCTIONS, \"f-string\", {\n      tool_names: toolNames,\n    });\n\n    const template = [prefix, toolStrings, formatInstructions, suffix].join(\n      \"\\n\\n\"\n    );\n\n    return new PromptTemplate({\n      template,\n      inputVariables,\n    });\n  }\n\n  /**\n   * Creates a ZeroShotAgent from a Large Language Model and a set of tools.\n   * @param llm The Large Language Model to use.\n   * @param tools The tools for the agent to use.\n   * @param args Optional arguments for creating the agent.\n   * @returns A new instance of ZeroShotAgent.\n   */\n  static fromLLMAndTools(\n    llm: BaseLanguageModelInterface,\n    tools: ToolInterface[],\n    args?: ZeroShotCreatePromptArgs & AgentArgs\n  ) {\n    ZeroShotAgent.validateTools(tools);\n    const prompt = ZeroShotAgent.createPrompt(tools, args);\n    const outputParser =\n      args?.outputParser ?? ZeroShotAgent.getDefaultOutputParser();\n    const chain = new LLMChain({\n      prompt,\n      llm,\n      callbacks: args?.callbacks ?? args?.callbackManager,\n    });\n\n    return new ZeroShotAgent({\n      llmChain: chain,\n      allowedTools: tools.map((t) => t.name),\n      outputParser,\n    });\n  }\n\n  static async deserialize(\n    data: SerializedZeroShotAgent & {\n      llm?: BaseLanguageModelInterface;\n      tools?: ToolInterface[];\n    }\n  ): Promise<ZeroShotAgent> {\n    const { llm, tools, ...rest } = data;\n    return deserializeHelper(\n      llm,\n      tools,\n      rest,\n      (\n        llm: BaseLanguageModelInterface,\n        tools: ToolInterface[],\n        args: SerializedFromLLMAndTools\n      ) =>\n        ZeroShotAgent.fromLLMAndTools(llm, tools, {\n          prefix: args.prefix,\n          suffix: args.suffix,\n          inputVariables: args.input_variables,\n        }),\n      (args) => new ZeroShotAgent(args)\n    );\n  }\n}\n\n// Path: langchain/src/agents/agent.ts\nimport type {\n  StructuredToolInterface,\n  ToolInterface,\n} from \"@langchain/core/tools\";\nimport type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { CallbackManager, Callbacks } from \"@langchain/core/callbacks/manager\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport { AgentAction, AgentFinish, AgentStep } from \"@langchain/core/agents\";\nimport { BaseMessage } from \"@langchain/core/messages\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { Serializable } from \"@langchain/core/load/serializable\";\nimport {\n  Runnable,\n  patchConfig,\n  type RunnableConfig,\n  RunnableSequence,\n  RunnableLike,\n} from \"@langchain/core/runnables\";\nimport { LLMChain } from \"../chains/llm_chain.js\";\nimport type {\n  AgentActionOutputParser,\n  AgentInput,\n  RunnableMultiActionAgentInput,\n  RunnableSingleActionAgentInput,\n  SerializedAgent,\n  StoppingMethod,\n} from \"./types.js\";\n\n/**\n * Record type for arguments passed to output parsers.\n */\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport type OutputParserArgs = Record<string, any>;\n\n/**\n * Error class for parse errors in LangChain. Contains information about\n * the error message and the output that caused the error.\n */\nclass ParseError extends Error {\n  output: string;\n\n  constructor(msg: string, output: string) {\n    super(msg);\n    this.output = output;\n  }\n}\n\n/**\n * Abstract base class for agents in LangChain. Provides common\n * functionality for agents, such as handling inputs and outputs.\n */\nexport abstract class BaseAgent extends Serializable {\n  declare ToolType: StructuredToolInterface;\n\n  abstract get inputKeys(): string[];\n\n  get returnValues(): string[] {\n    return [\"output\"];\n  }\n\n  get allowedTools(): string[] | undefined {\n    return undefined;\n  }\n\n  /**\n   * Return the string type key uniquely identifying this class of agent.\n   */\n  _agentType(): string {\n    throw new Error(\"Not implemented\");\n  }\n\n  /**\n   * Return the string type key uniquely identifying multi or single action agents.\n   */\n  abstract _agentActionType(): string;\n\n  /**\n   * Return response when agent has been stopped due to max iterations\n   */\n  returnStoppedResponse(\n    earlyStoppingMethod: StoppingMethod,\n    _steps: AgentStep[],\n    _inputs: ChainValues,\n    _callbackManager?: CallbackManager\n  ): Promise<AgentFinish> {\n    if (earlyStoppingMethod === \"force\") {\n      return Promise.resolve({\n        returnValues: { output: \"Agent stopped due to max iterations.\" },\n        log: \"\",\n      });\n    }\n\n    throw new Error(`Invalid stopping method: ${earlyStoppingMethod}`);\n  }\n\n  /**\n   * Prepare the agent for output, if needed\n   */\n  async prepareForOutput(\n    _returnValues: AgentFinish[\"returnValues\"],\n    _steps: AgentStep[]\n  ): Promise<AgentFinish[\"returnValues\"]> {\n    return {};\n  }\n}\n\n/**\n * Abstract base class for single action agents in LangChain. Extends the\n * BaseAgent class and provides additional functionality specific to\n * single action agents.\n */\nexport abstract class BaseSingleActionAgent extends BaseAgent {\n  _agentActionType(): string {\n    return \"single\" as const;\n  }\n\n  /**\n   * Decide what to do, given some input.\n   *\n   * @param steps - Steps the LLM has taken so far, along with observations from each.\n   * @param inputs - User inputs.\n   * @param callbackManager - Callback manager.\n   *\n   * @returns Action specifying what tool to use.\n   */\n  abstract plan(\n    steps: AgentStep[],\n    inputs: ChainValues,\n    callbackManager?: CallbackManager,\n    config?: RunnableConfig\n  ): Promise<AgentAction | AgentFinish>;\n}\n\n/**\n * Abstract base class for multi-action agents in LangChain. Extends the\n * BaseAgent class and provides additional functionality specific to\n * multi-action agents.\n */\nexport abstract class BaseMultiActionAgent extends BaseAgent {\n  _agentActionType(): string {\n    return \"multi\" as const;\n  }\n\n  /**\n   * Decide what to do, given some input.\n   *\n   * @param steps - Steps the LLM has taken so far, along with observations from each.\n   * @param inputs - User inputs.\n   * @param callbackManager - Callback manager.\n   *\n   * @returns Actions specifying what tools to use.\n   */\n  abstract plan(\n    steps: AgentStep[],\n    inputs: ChainValues,\n    callbackManager?: CallbackManager,\n    config?: RunnableConfig\n  ): Promise<AgentAction[] | AgentFinish>;\n}\n\n\nfunction isAgentAction(input: unknown): input is AgentAction {\n  return !Array.isArray(input) && (input as AgentAction)?.tool !== undefined;\n}\n\nexport function isRunnableAgent(x: BaseAgent) {\n  return (\n    (x as RunnableMultiActionAgent | RunnableSingleActionAgent).runnable !==\n    undefined\n  );\n}\n\n// TODO: Remove in the future. Only for backwards compatibility.\n// Allows for the creation of runnables with properties that will\n// be passed to the agent executor constructor.\nexport class AgentRunnableSequence<\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  RunInput = any,\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  RunOutput = any\n> extends RunnableSequence<RunInput, RunOutput> {\n  streamRunnable?: boolean;\n\n  singleAction: boolean;\n\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  static fromRunnables<RunInput = any, RunOutput = any>(\n    [first, ...runnables]: [\n      RunnableLike<RunInput>,\n      ...RunnableLike[],\n      // eslint-disable-next-line @typescript-eslint/no-explicit-any\n      RunnableLike<any, RunOutput>\n    ],\n    config: { singleAction: boolean; streamRunnable?: boolean; name?: string }\n  ): AgentRunnableSequence<RunInput, Exclude<RunOutput, Error>> {\n    const sequence = RunnableSequence.from(\n      [first, ...runnables],\n      config.name\n    ) as AgentRunnableSequence<RunInput, Exclude<RunOutput, Error>>;\n    sequence.singleAction = config.singleAction;\n    sequence.streamRunnable = config.streamRunnable;\n    return sequence;\n  }\n\n  static isAgentRunnableSequence(x: Runnable): x is AgentRunnableSequence {\n    return typeof (x as AgentRunnableSequence).singleAction === \"boolean\";\n  }\n}\n\n/**\n * Class representing a single-action agent powered by runnables.\n * Extends the BaseSingleActionAgent class and provides methods for\n * planning agent actions with runnables.\n */\nexport class RunnableSingleActionAgent extends BaseSingleActionAgent {\n  lc_namespace = [\"langchain\", \"agents\", \"runnable\"];\n\n  runnable: Runnable<\n    ChainValues & { steps: AgentStep[] },\n    AgentAction | AgentFinish\n  >;\n\n  get inputKeys(): string[] {\n    return [];\n  }\n\n  /**\n   * Whether to stream from the runnable or not.\n   * If true, the underlying LLM is invoked in a streaming fashion to make it\n   * possible to get access to the individual LLM tokens when using\n   * `streamLog` with the Agent Executor. If false then LLM is invoked in a\n   * non-streaming fashion and individual LLM tokens will not be available\n   * in `streamLog`.\n   *\n   * Note that the runnable should still only stream a single action or\n   * finish chunk.\n   */\n  streamRunnable = true;\n\n  defaultRunName = \"RunnableAgent\";\n\n  constructor(fields: RunnableSingleActionAgentInput) {\n    super(fields);\n    this.runnable = fields.runnable;\n...\n", "needle_token_start": 15272, "needle_token_end": 15311, "code_context_ntokens": 16092, "output": ["```typescript\nfunction isAgentAction(input: unknown): input is AgentAction {\n  return !Array.isArray(input) && (input as AgentAction)?.tool !== undefined;\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "setCss", "language": "typescript", "path": "src/api/css.ts", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to modify the CSS properties of a given HTML element, either by setting a single property or multiple properties based on the provided parameters.\n2. **Input**: The function accepts an HTML element, a property name or a map of property names and values, a value for the property which can be a string or a function that dynamically generates the value, and an optional index indicating the position of the element in a collection.\n3. **Output**: There is no direct return value as the function primarily modifies the `style` attribute of the provided HTML element directly.\n4. **Procedure**: The function first checks if the property name is a string or an object. If it is a string, it retrieves the current styles, applies the new value or function result to the specified property, and updates the element's style attribute. If a map of properties is provided, it iterates over each property in the map and recursively applies the changes to each property.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): string | undefined;\n/**\n * Set one CSS property for every matched element.\n *\n * @category CSS\n * @param prop - The name of the property.\n * @param val - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/css/}\n */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  prop: string,\n  val:\n    | string\n    | ((this: Element, i: number, style: string) => string | undefined),\n): Cheerio<T>;\n/**\n * Set multiple CSS properties for every matched element.\n *\n * @category CSS\n * @param map - A map of property names and values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/css/}\n */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  map: Record<string, string>,\n): Cheerio<T>;\n/**\n * Set multiple CSS properties for every matched element.\n *\n * @category CSS\n * @param prop - The names of the properties.\n * @param val - The new values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/css/}\n */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  prop?: string | string[] | Record<string, string>,\n  val?:\n    | string\n    | ((this: Element, i: number, style: string) => string | undefined),\n): Cheerio<T> | Record<string, string> | string | undefined {\n  if (\n    (prop != null && val != null) ||\n    // When `prop` is a \"plain\" object\n    (typeof prop === 'object' && !Array.isArray(prop))\n  ) {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        // `prop` can't be an array here anymore.\n        setCss(el, prop as string, val, i);\n      }\n    });\n  }\n\n  if (this.length === 0) {\n    return undefined;\n  }\n\n  return getCss(this[0], prop as string);\n}\n\n/**\n * Set styles of all elements.\n *\n * @private\n * @param el - Element to set style of.\n * @param prop - Name of property.\n * @param value - Value to set property to.\n * @param idx - Optional index within the selection.\n */\n\nfunction setCss(\n  el: Element,\n  prop: string | Record<string, string>,\n  value:\n    | string\n    | ((this: Element, i: number, style: string) => string | undefined)\n    | undefined,\n  idx: number,\n) {\n  if (typeof prop === 'string') {\n    const styles = getCss(el);\n\n    const val =\n      typeof value === 'function' ? value.call(el, idx, styles[prop]) : value;\n\n    if (val === '') {\n      delete styles[prop];\n    } else if (val != null) {\n      styles[prop] = val;\n    }\n\n    el.attribs['style'] = stringify(styles);\n  } else if (typeof prop === 'object') {\n    const keys = Object.keys(prop);\n    for (let i = 0; i < keys.length; i++) {\n      const k = keys[i];\n      setCss(el, k, prop[k], i);\n    }\n  }\n}\n\n/**\n * Get the parsed styles of the first element.\n *\n * @private\n * @category CSS\n * @param el - Element to get styles from.\n * @param props - Optionally the names of the properties of interest.\n * @returns The parsed styles.\n */\nfunction getCss(el: AnyNode, props?: string[]): Record<string, string>;\n/**\n * Get a property from the parsed styles of the first element.\n *\n * @private\n * @category CSS\n * @param el - Element to get styles from.\n * @param prop - Name of the prop.\n * @returns The value of the property.\n */\nfunction getCss(el: AnyNode, prop: string): string | undefined;\nfunction getCss(\n  el: AnyNode,\n  prop?: string | string[],\n): Record<string, string> | string | undefined {\n  if (!el || !isTag(el)) return;\n\n  const styles = parse(el.attribs['style']);\n  if (typeof prop === 'string') {\n    return styles[prop];\n  }\n  if (Array.isArray(prop)) {\n    const newStyles: Record<string, string> = {};\n    for (const item of prop) {\n      if (styles[item] != null) {\n        newStyles[item] = styles[item];\n      }\n    }\n    return newStyles;\n  }\n  return styles;\n}\n\n/**\n * Stringify `obj` to styles.\n *\n * @private\n * @category CSS\n * @param obj - Object to stringify.\n * @returns The serialized styles.\n */\nfunction stringify(obj: Record<string, string>): string {\n  return Object.keys(obj).reduce(\n    (str, prop) => `${str}${str ? ' ' : ''}${prop}: ${obj[prop]};`,\n    '',\n  );\n}\n\n/**\n * Parse `styles`.\n *\n * @private\n * @category CSS\n * @param styles - Styles to be parsed.\n * @returns The parsed styles.\n */\nfunction parse(styles: string): Record<string, string> {\n  styles = (styles || '').trim();\n\n  if (!styles) return {};\n\n  const obj: Record<string, string> = {};\n\n  let key: string | undefined;\n\n  for (const str of styles.split(';')) {\n    const n = str.indexOf(':');\n    // If there is no :, or if it is the first/last character, add to the previous item's value\n    if (n < 1 || n === str.length - 1) {\n      const trimmed = str.trimEnd();\n      if (trimmed.length > 0 && key !== undefined) {\n        obj[key] += `;${trimmed}`;\n      }\n    } else {\n      key = str.slice(0, n).trim();\n      obj[key] = str.slice(n + 1).trim();\n    }\n  }\n\n  return obj;\n}\n\n// Path: src/api/forms.ts\nimport { isTag, type AnyNode } from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\n\n/*\n * https://github.com/jquery/jquery/blob/2.1.3/src/manipulation/var/rcheckableType.js\n * https://github.com/jquery/jquery/blob/2.1.3/src/serialize.js\n */\nconst submittableSelector = 'input,select,textarea,keygen';\nconst r20 = /%20/g;\nconst rCRLF = /\\r?\\n/g;\n\n/**\n * Encode a set of form elements as a string for submission.\n *\n * @category Forms\n * @example\n *\n * ```js\n * $('<form><input name=\"foo\" value=\"bar\" /></form>').serialize();\n * //=> 'foo=bar'\n * ```\n *\n * @returns The serialized form.\n * @see {@link https://api.jquery.com/serialize/}\n */\nexport function serialize<T extends AnyNode>(this: Cheerio<T>): string {\n  // Convert form elements into name/value objects\n  const arr = this.serializeArray();\n\n  // Serialize each element into a key/value string\n  const retArr = arr.map(\n    (data) =>\n      `${encodeURIComponent(data.name)}=${encodeURIComponent(data.value)}`,\n  );\n\n  // Return the resulting serialization\n  return retArr.join('&').replace(r20, '+');\n}\n\n/**\n * Encode a set of form elements as an array of names and values.\n *\n * @category Forms\n * @example\n *\n * ```js\n * $('<form><input name=\"foo\" value=\"bar\" /></form>').serializeArray();\n * //=> [ { name: 'foo', value: 'bar' } ]\n * ```\n *\n * @returns The serialized form.\n * @see {@link https://api.jquery.com/serializeArray/}\n */\nexport function serializeArray<T extends AnyNode>(\n  this: Cheerio<T>,\n): {\n  name: string;\n  value: string;\n}[] {\n  // Resolve all form elements from either forms or collections of form elements\n  return this.map((_, elem) => {\n    const $elem = this._make(elem);\n    if (isTag(elem) && elem.name === 'form') {\n      return $elem.find(submittableSelector).toArray();\n    }\n    return $elem.filter(submittableSelector).toArray();\n  })\n    .filter(\n      // Verify elements have a name (`attr.name`) and are not disabled (`:enabled`)\n      '[name!=\"\"]:enabled' +\n        // And cannot be clicked (`[type=submit]`) or are used in `x-www-form-urlencoded` (`[type=file]`)\n        ':not(:submit, :button, :image, :reset, :file)' +\n        // And are either checked/don't have a checkable state\n        ':matches([checked], :not(:checkbox, :radio))',\n      // Convert each of the elements to its value(s)\n    )\n    .map<\n      AnyNode,\n      {\n        name: string;\n        value: string;\n      }\n    >((_, elem) => {\n      const $elem = this._make(elem);\n      const name = $elem.attr('name') as string; // We have filtered for elements with a name before.\n      // If there is no value set (e.g. `undefined`, `null`), then default value to empty\n      const value = $elem.val() ?? '';\n\n      // If we have an array of values (e.g. `<select multiple>`), return an array of key/value pairs\n      if (Array.isArray(value)) {\n        return value.map((val) =>\n          /*\n           * We trim replace any line endings (e.g. `\\r` or `\\r\\n` with `\\r\\n`) to guarantee consistency across platforms\n           * These can occur inside of `<textarea>'s`\n           */\n          ({ name, value: val.replace(rCRLF, '\\r\\n') }),\n        );\n      }\n      // Otherwise (e.g. `<input type=\"text\">`, return only one key/value pair\n      return { name, value: value.replace(rCRLF, '\\r\\n') };\n    })\n    .toArray();\n}\n\n// Path: src/cheerio.ts\n/* eslint-disable @typescript-eslint/no-unsafe-declaration-merging */\nimport type { InternalOptions } from './options.js';\nimport type { AnyNode, Document, ParentNode } from 'domhandler';\nimport type { BasicAcceptedElems } from './types.js';\n\nimport * as Attributes from './api/attributes.js';\nimport * as Traversing from './api/traversing.js';\nimport * as Manipulation from './api/manipulation.js';\nimport * as Css from './api/css.js';\nimport * as Forms from './api/forms.js';\nimport * as Extract from './api/extract.js';\n\ntype MethodsType = typeof Attributes &\n  typeof Traversing &\n  typeof Manipulation &\n  typeof Css &\n  typeof Forms &\n  typeof Extract;\n\n/**\n * The cheerio class is the central class of the library. It wraps a set of\n * elements and provides an API for traversing, modifying, and interacting with\n * the set.\n *\n * Loading a document will return the Cheerio class bound to the root element of\n * the document. The class will be instantiated when querying the document (when\n * calling `$('selector')`).\n *\n * @example This is the HTML markup we will be using in all of the API examples:\n *\n * ```html\n * <ul id=\"fruits\">\n *   <li class=\"apple\">Apple</li>\n *   <li class=\"orange\">Orange</li>\n *   <li class=\"pear\">Pear</li>\n * </ul>\n * ```\n */\nexport abstract class Cheerio<T> implements ArrayLike<T> {\n  length = 0;\n  [index: number]: T;\n\n  options: InternalOptions;\n  /**\n   * The root of the document. Can be set by using the `root` argument of the\n   * constructor.\n   *\n   * @private\n   */\n  _root: Cheerio<Document> | null;\n\n  /**\n   * Instance of cheerio. Methods are specified in the modules. Usage of this\n   * constructor is not recommended. Please use `$.load` instead.\n   *\n   * @private\n   * @param elements - The new selection.\n   * @param root - Sets the root node.\n   * @param options - Options for the instance.\n   */\n  constructor(\n    elements: ArrayLike<T> | undefined,\n    root: Cheerio<Document> | null,\n    options: InternalOptions,\n  ) {\n    this.options = options;\n    this._root = root;\n\n    if (elements) {\n      for (let idx = 0; idx < elements.length; idx++) {\n        this[idx] = elements[idx];\n      }\n      this.length = elements.length;\n    }\n  }\n\n  prevObject: Cheerio<any> | undefined;\n  /**\n   * Make a cheerio object.\n   *\n   * @private\n   * @param dom - The contents of the new object.\n   * @param context - The context of the new object.\n   * @returns The new cheerio object.\n   */\n  abstract _make<T>(\n    dom: ArrayLike<T> | T | string,\n    context?: BasicAcceptedElems<AnyNode>,\n  ): Cheerio<T>;\n\n  /**\n   * Parses some content.\n   *\n   * @private\n   * @param content - Content to parse.\n   * @param options - Options for parsing.\n   * @param isDocument - Allows parser to be switched to fragment mode.\n   * @returns A document containing the `content`.\n   */\n  abstract _parse(\n    content: string | Document | AnyNode | AnyNode[] | Buffer,\n    options: InternalOptions,\n    isDocument: boolean,\n    context: ParentNode | null,\n  ): Document;\n\n  /**\n   * Render an element or a set of elements.\n   *\n   * @private\n   * @param dom - DOM to render.\n   * @returns The rendered DOM.\n   */\n  abstract _render(dom: AnyNode | ArrayLike<AnyNode>): string;\n}\n\nexport interface Cheerio<T> extends MethodsType, Iterable<T> {\n  cheerio: '[cheerio object]';\n\n  splice: typeof Array.prototype.splice;\n}\n\n/** Set a signature of the object. */\nCheerio.prototype.cheerio = '[cheerio object]';\n\n/*\n * Make cheerio an array-like object\n */\nCheerio.prototype.splice = Array.prototype.splice;\n\n// Support for (const element of $(...)) iteration:\nCheerio.prototype[Symbol.iterator] = Array.prototype[Symbol.iterator];\n\n// Plug in the API\nObject.assign(\n  Cheerio.prototype,\n  Attributes,\n  Traversing,\n  Manipulation,\n  Css,\n  Forms,\n  Extract,\n);\n\n// Path: src/types.ts\ntype LowercaseLetters =\n  | 'a'\n  | 'b'\n  | 'c'\n  | 'd'\n  | 'e'\n  | 'f'\n  | 'g'\n  | 'h'\n  | 'i'\n  | 'j'\n  | 'k'\n  | 'l'\n  | 'm'\n  | 'n'\n  | 'o'\n  | 'p'\n  | 'q'\n  | 'r'\n  | 's'\n  | 't'\n  | 'u'\n  | 'v'\n  | 'w'\n  | 'x'\n  | 'y'\n  | 'z';\n\ntype AlphaNumeric =\n  | LowercaseLetters\n  | Uppercase<LowercaseLetters>\n  | `${number}`;\n\ntype SelectorSpecial = '.' | '#' | ':' | '|' | '>' | '+' | '~' | '[';\n/**\n * Type for identifying selectors. Allows us to \"upgrade\" queries using\n * selectors to return `Element`s.\n */\nexport type SelectorType =\n  | `${SelectorSpecial}${AlphaNumeric}${string}`\n  | `${AlphaNumeric}${string}`;\n\nimport type { Cheerio } from './cheerio.js';\nimport type { AnyNode } from 'domhandler';\n\n/** Elements that can be passed to manipulation methods. */\nexport type BasicAcceptedElems<T extends AnyNode> = ArrayLike<T> | T | string;\n/** Elements that can be passed to manipulation methods, including functions. */\nexport type AcceptedElems<T extends AnyNode> =\n  | BasicAcceptedElems<T>\n  | ((this: T, i: number, el: T) => BasicAcceptedElems<T>);\n\n/** Function signature, for traversal methods. */\nexport type FilterFunction<T> = (this: T, i: number, el: T) => boolean;\n/** Supported filter types, for traversal methods. */\nexport type AcceptedFilters<T> = string | FilterFunction<T> | T | Cheerio<T>;\n\n// Path: src/load.spec.ts\nimport cheerio from './index.js';\n\ndescribe('.load', () => {\n  it('(html) : should retain original root after creating a new node', () => {\n    const $ = cheerio.load('<body><ul id=\"fruits\"></ul></body>');\n    expect($('body')).toHaveLength(1);\n    $('<script>');\n    expect($('body')).toHaveLength(1);\n  });\n\n  it('(html) : should handle lowercase tag options', () => {\n    const $ = cheerio.load('<BODY><ul id=\"fruits\"></ul></BODY>', {\n      xml: { lowerCaseTags: true },\n    });\n    expect($.html()).toBe('<body><ul id=\"fruits\"/></body>');\n  });\n\n  it('(html) : should handle xml tag option', () => {\n    const $ = cheerio.load('<body><script><foo></script></body>', {\n      xml: true,\n    });\n    expect($('script')[0].children[0].type).toBe('tag');\n  });\n\n  it('(buffer) : should accept a buffer', () => {\n    const html = '<html><head></head><body>foo</body></html>';\n    // eslint-disable-next-line n/no-unsupported-features/node-builtins\n    const $html = cheerio.load(Buffer.from(html));\n    expect($html.html()).toBe(html);\n  });\n});\n\n// Path: src/batteries.ts\n/**\n * @file Batteries-included version of Cheerio. This module includes several\n *   convenience methods for loading documents from various sources.\n */\n\nexport * from './index.js';\n\n/* eslint-disable n/no-unsupported-features/node-builtins */\n\nimport type { CheerioAPI, CheerioOptions } from './index.js';\nimport { load } from './index.js';\nimport { flattenOptions, type InternalOptions } from './options.js';\nimport { adapter as htmlparser2Adapter } from 'parse5-htmlparser2-tree-adapter';\n\nimport * as htmlparser2 from 'htmlparser2';\nimport { ParserStream as Parse5Stream } from 'parse5-parser-stream';\nimport {\n  decodeBuffer,\n  DecodeStream,\n  type SnifferOptions,\n} from 'encoding-sniffer';\nimport * as undici from 'undici';\nimport MIMEType from 'whatwg-mimetype';\nimport { Writable, finished } from 'node:stream';\n\n/**\n * Sniffs the encoding of a buffer, then creates a querying function bound to a\n * document created from the buffer.\n *\n * @category Loading\n * @example\n *\n * ```js\n * import * as cheerio from 'cheerio';\n *\n * const buffer = fs.readFileSync('index.html');\n * const $ = cheerio.fromBuffer(buffer);\n * ```\n *\n * @param buffer - The buffer to sniff the encoding of.\n * @param options - The options to pass to Cheerio.\n * @returns The loaded document.\n */\nexport function loadBuffer(\n  buffer: Buffer,\n  options: DecodeStreamOptions = {},\n): CheerioAPI {\n  const opts = flattenOptions(options);\n  const str = decodeBuffer(buffer, {\n    defaultEncoding: opts?.xmlMode ? 'utf8' : 'windows-1252',\n    ...options.encoding,\n  });\n\n  return load(str, opts);\n}\n\nfunction _stringStream(\n  options: InternalOptions | undefined,\n  cb: (err: Error | null | undefined, $: CheerioAPI) => void,\n): Writable {\n  if (options?._useHtmlParser2) {\n    const parser = htmlparser2.createDocumentStream(\n      (err, document) => cb(err, load(document)),\n      options,\n    );\n\n    return new Writable({\n      decodeStrings: false,\n      write(chunk, _encoding, callback) {\n        if (typeof chunk !== 'string') {\n          throw new TypeError('Expected a string');\n        }\n\n        parser.write(chunk);\n        callback();\n      },\n      final(callback) {\n        parser.end();\n        callback();\n      },\n    });\n  }\n\n  options ??= {};\n  options.treeAdapter ??= htmlparser2Adapter;\n\n  if (options.scriptingEnabled !== false) {\n    options.scriptingEnabled = true;\n  }\n\n  const stream = new Parse5Stream(options);\n\n  finished(stream, (err) => cb(err, load(stream.document)));\n\n  return stream;\n}\n\n/**\n * Creates a stream that parses a sequence of strings into a document.\n *\n * The stream is a `Writable` stream that accepts strings. When the stream is\n * finished, the callback is called with the loaded document.\n *\n * @category Loading\n * @example\n *\n * ```js\n * import * as cheerio from 'cheerio';\n * import * as fs from 'fs';\n *\n * const writeStream = cheerio.stringStream({}, (err, $) => {\n *   if (err) {\n *     // Handle error\n *   }\n *\n *   console.log($('h1').text());\n *   // Output: Hello, world!\n * });\n *\n * fs.createReadStream('my-document.html', { encoding: 'utf8' }).pipe(\n *   writeStream,\n * );\n * ```\n *\n * @param options - The options to pass to Cheerio.\n * @param cb - The callback to call when the stream is finished.\n * @returns The writable stream.\n */\nexport function stringStream(\n  options: CheerioOptions,\n  cb: (err: Error | null | undefined, $: CheerioAPI) => void,\n): Writable {\n  return _stringStream(flattenOptions(options), cb);\n}\n\nexport interface DecodeStreamOptions extends CheerioOptions {\n  encoding?: SnifferOptions;\n}\n\n/**\n * Parses a stream of buffers into a document.\n *\n * The stream is a `Writable` stream that accepts buffers. When the stream is\n * finished, the callback is called with the loaded document.\n *\n * @category Loading\n * @param options - The options to pass to Cheerio.\n * @param cb - The callback to call when the stream is finished.\n * @returns The writable stream.\n */\nexport function decodeStream(\n  options: DecodeStreamOptions,\n  cb: (err: Error | null | undefined, $: CheerioAPI) => void,\n): Writable {\n  const { encoding = {}, ...cheerioOptions } = options;\n  const opts = flattenOptions(cheerioOptions);\n\n  // Set the default encoding to UTF-8 for XML mode\n  encoding.defaultEncoding ??= opts?.xmlMode ? 'utf8' : 'windows-1252';\n\n  const decodeStream = new DecodeStream(encoding);\n  const loadStream = _stringStream(opts, cb);\n\n  decodeStream.pipe(loadStream);\n\n  return decodeStream;\n}\n\ntype UndiciStreamOptions = Parameters<typeof undici.stream>[1];\n\nexport interface CheerioRequestOptions extends DecodeStreamOptions {\n  /** The options passed to `undici`'s `stream` method. */\n  requestOptions?: UndiciStreamOptions;\n}\n\nconst defaultRequestOptions: UndiciStreamOptions = {\n  method: 'GET',\n  // Allow redirects by default\n  maxRedirections: 5,\n  // NOTE: `throwOnError` currently doesn't work https://github.com/nodejs/undici/issues/1753\n  throwOnError: true,\n  // Set an Accept header\n  headers: {\n    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n  },\n};\n\n/**\n * `fromURL` loads a document from a URL.\n *\n * By default, redirects are allowed and non-2xx responses are rejected.\n *\n * @category Loading\n * @example\n *\n * ```js\n * import * as cheerio from 'cheerio';\n *\n * const $ = await cheerio.fromURL('https://example.com');\n * ```\n *\n * @param url - The URL to load the document from.\n * @param options - The options to pass to Cheerio.\n * @returns The loaded document.\n */\nexport async function fromURL(\n  url: string | URL,\n  options: CheerioRequestOptions = {},\n): Promise<CheerioAPI> {\n  const {\n    requestOptions = defaultRequestOptions,\n    encoding = {},\n    ...cheerioOptions\n  } = options;\n  let undiciStream: Promise<undici.Dispatcher.StreamData> | undefined;\n\n  // Add headers if none were supplied.\n  requestOptions.headers ??= defaultRequestOptions.headers;\n\n  const promise = new Promise<CheerioAPI>((resolve, reject) => {\n    undiciStream = undici.stream(url, requestOptions, (res) => {\n      const contentType = res.headers['content-type'] ?? 'text/html';\n      const mimeType = new MIMEType(\n        Array.isArray(contentType) ? contentType[0] : contentType,\n      );\n\n      if (!mimeType.isHTML() && !mimeType.isXML()) {\n        throw new RangeError(\n          `The content-type \"${contentType}\" is neither HTML nor XML.`,\n        );\n      }\n\n      // Forward the charset from the header to the decodeStream.\n      encoding.transportLayerEncodingLabel = mimeType.parameters.get('charset');\n\n      /*\n       * If we allow redirects, we will have entries in the history.\n       * The last entry will be the final URL.\n       */\n      const history = (\n        res.context as\n          | {\n              history?: URL[];\n            }\n          | undefined\n      )?.history;\n\n      const opts = {\n        encoding,\n        // Set XML mode based on the MIME type.\n        xmlMode: mimeType.isXML(),\n        // Set the `baseURL` to the final URL.\n        baseURL: history ? history[history.length - 1] : url,\n        ...cheerioOptions,\n      };\n\n      return decodeStream(opts, (err, $) => (err ? reject(err) : resolve($)));\n    });\n  });\n\n  // Let's make sure the request is completed before returning the promise.\n  await undiciStream;\n\n  return promise;\n}\n\n// Path: src/utils.spec.ts\nimport * as utils from './utils.js';\n\ndescribe('util functions', () => {\n  it('camelCase function test', () => {\n    expect(utils.camelCase('cheerio.js')).toBe('cheerioJs');\n    expect(utils.camelCase('camel-case-')).toBe('camelCase');\n    expect(utils.camelCase('__directory__')).toBe('_directory_');\n    expect(utils.camelCase('_one-two.three')).toBe('OneTwoThree');\n  });\n\n  it('cssCase function test', () => {\n    expect(utils.cssCase('camelCase')).toBe('camel-case');\n    expect(utils.cssCase('jQuery')).toBe('j-query');\n    expect(utils.cssCase('neverSayNever')).toBe('never-say-never');\n    expect(utils.cssCase('CSSCase')).toBe('-c-s-s-case');\n  });\n\n  it('isHtml function test', () => {\n    expect(utils.isHtml('<html>')).toBe(true);\n    expect(utils.isHtml('\\n<html>\\n')).toBe(true);\n    expect(utils.isHtml('#main')).toBe(false);\n    expect(utils.isHtml('\\n<p>foo<p>bar\\n')).toBe(true);\n    expect(utils.isHtml('dog<p>fox<p>cat')).toBe(true);\n    expect(utils.isHtml('<p>fox<p>cat')).toBe(true);\n    expect(utils.isHtml('\\n<p>fox<p>cat\\n')).toBe(true);\n    expect(utils.isHtml('#<p>fox<p>cat#')).toBe(true);\n    expect(utils.isHtml('<!-- comment -->')).toBe(true);\n    expect(utils.isHtml('<!doctype html>')).toBe(true);\n    expect(utils.isHtml('<123>')).toBe(false);\n  });\n});\n\n// Path: src/__fixtures__/fixtures.ts\nexport const fruits = [\n  '<ul id=\"fruits\">',\n  '<li class=\"apple\">Apple</li>',\n  '<li class=\"orange\">Orange</li>',\n  '<li class=\"pear\">Pear</li>',\n  '</ul>',\n].join('');\n\nexport const vegetables = [\n  '<ul id=\"vegetables\">',\n  '<li class=\"carrot\">Carrot</li>',\n  '<li class=\"sweetcorn\">Sweetcorn</li>',\n  '</ul>',\n].join('');\n\nexport const divcontainers = [\n  '<div class=\"container\">',\n  '<div class=\"inner\">First</div>',\n  '<div class=\"inner\">Second</div>',\n  '</div>',\n  '<div class=\"container\">',\n  '<div class=\"inner\">Third</div>',\n  '<div class=\"inner\">Fourth</div>',\n  '</div>',\n  '<div id=\"new\"><div>',\n  '<div>\\n\\n<p><em><b></b></em></p>\\n\\n</div>',\n  '</div>',\n].join('');\n\nexport const chocolates = [\n  '<ul id=\"chocolates\">',\n  '<li class=\"linth\" data-highlight=\"Lindor\" data-origin=\"swiss\">Linth</li>',\n  '<li class=\"frey\" data-taste=\"sweet\" data-best-collection=\"Mahony\">Frey</li>',\n  '<li class=\"cailler\">Cailler</li>',\n  '</ul>',\n].join('');\n\nexport const drinks = [\n  '<ul id=\"drinks\">',\n  '<li class=\"beer\">Beer</li>',\n  '<li class=\"juice\">Juice</li>',\n  '<li class=\"milk\">Milk</li>',\n  '<li class=\"water\">Water</li>',\n  '<li class=\"cider\">Cider</li>',\n  '</ul>',\n].join('');\n\nexport const food = ['<ul id=\"food\">', fruits, vegetables, '</ul>'].join('');\n\nexport const eleven = `\n<html>\n  <body>\n    <ul>\n      <li>One</li>\n      <li>Two</li>\n      <li class=\"blue sel\">Three</li>\n      <li class=\"red\">Four</li>\n    </ul>\n\n    <ul>\n      <li class=\"red\">Five</li>\n      <li>Six</li>\n      <li class=\"blue\">Seven</li>\n    </ul>\n\n    <ul>\n      <li>Eight</li>\n      <li class=\"red sel\">Nine</li>\n      <li>Ten</li>\n      <li class=\"sel\">Eleven</li>\n    </ul>\n  </body>\n</html>\n`;\n\nexport const unwrapspans = [\n  '<div id=unwrap style=\"display: none;\">',\n  '<div id=unwrap1><span class=unwrap>a</span><span class=unwrap>b</span></div>',\n  '<div id=unwrap2><span class=unwrap>c</span><span class=unwrap>d</span></div>',\n  '<div id=unwrap3><b><span class=\"unwrap unwrap3\">e</span></b><b><span class=\"unwrap unwrap3\">f</span></b></div>',\n  '</div>',\n].join('');\n\nexport const inputs = [\n  '<select id=\"one\"><option value=\"option_not_selected\">Option not selected</option><option value=\"option_selected\" selected>Option selected</option></select>',\n  '<select id=\"one-valueless\"><option>Option not selected</option><option selected>Option selected</option></select>',\n  '<select id=\"one-html-entity\"><option>Option not selected</option><option selected>Option &lt;selected&gt;</option></select>',\n  '<select id=\"one-nested\"><option>Option not selected</option><option selected>Option <span>selected</span></option></select>',\n  '<input type=\"text\" value=\"input_text\" />',\n  '<input type=\"checkbox\" name=\"checkbox_off\" value=\"off\" /><input type=\"checkbox\" name=\"checkbox_on\" value=\"on\" checked />',\n  '<input type=\"checkbox\" name=\"checkbox_valueless\" />',\n  '<input type=\"radio\" value=\"off\" name=\"radio\" /><input type=\"radio\" name=\"radio\" value=\"on\" checked />',\n  '<input type=\"radio\" value=\"off\" name=\"radio[brackets]\" /><input type=\"radio\" name=\"radio[brackets]\" value=\"on\" checked />',\n  '<input type=\"radio\" name=\"radio_valueless\" />',\n  '<select id=\"multi\" multiple><option value=\"1\">1</option><option value=\"2\" selected>2</option><option value=\"3\" selected>3</option><option value=\"4\">4</option></select>',\n  '<select id=\"multi-valueless\" multiple><option>1</option><option selected>2</option><option selected>3</option><option>4</option></select>',\n].join('');\n\nexport const text = [\n  '<p>Apples, <b>oranges</b> and pears.</p>',\n  '<p>Carrots and <!-- sweetcorn --></p>',\n].join('');\n\nexport const forms = [\n  '<form id=\"simple\"><input type=\"text\" name=\"fruit\" value=\"Apple\" /></form>',\n  '<form id=\"nested\"><div><input type=\"text\" name=\"fruit\" value=\"Apple\" /></div><input type=\"text\" name=\"vegetable\" value=\"Carrot\" /></form>',\n  '<form id=\"disabled\"><input type=\"text\" name=\"fruit\" value=\"Apple\" disabled /></form>',\n  '<form id=\"submit\"><input type=\"text\" name=\"fruit\" value=\"Apple\" /><input type=\"submit\" name=\"submit\" value=\"Submit\" /></form>',\n  '<form id=\"select\"><select name=\"fruit\"><option value=\"Apple\">Apple</option><option value=\"Orange\" selected>Orange</option></select></form>',\n  '<form id=\"unnamed\"><input type=\"text\" name=\"fruit\" value=\"Apple\" /><input type=\"text\" value=\"Carrot\" /></form>',\n  '<form id=\"multiple\"><select name=\"fruit\" multiple><option value=\"Apple\" selected>Apple</option><option value=\"Orange\" selected>Orange</option><option value=\"Carrot\">Carrot</option></select></form>',\n  '<form id=\"textarea\"><textarea name=\"fruits\">Apple\\nOrange</textarea></form>',\n  '<form id=\"spaces\"><input type=\"text\" name=\"fruit\" value=\"Blood orange\" /></form>',\n].join('');\n\nexport const noscript = [\n  '</body>',\n  '<noscript>',\n  '<!-- anchor linking to external file -->',\n  '<a href=\"https://github.com/cheeriojs/cheerio\">External Link</a>',\n  '</noscript>',\n  '<p>Rocks!</p>',\n  '</body>',\n].join('');\n\nexport const script = [\n  '<div>',\n  '<a>A</a>',\n  '<script>',\n  '  var foo = \"bar\";',\n  '</script>',\n  '<b>B</b>',\n  '</div>',\n].join('');\n\nexport const mixedText = '<a>1</a>TEXT<b>2</b>';\n\n// Path: src/static.spec.ts\nimport * as fixtures from './__fixtures__/fixtures.js';\nimport cheerio, { type CheerioAPI } from './index.js';\n\ndescribe('cheerio', () => {\n  describe('.html', () => {\n    it('() : should return innerHTML; $.html(obj) should return outerHTML', () => {\n      const $div = cheerio(\n        'div',\n        '<div><span>foo</span><span>bar</span></div>',\n      );\n      const span = $div.children()[1];\n      expect(cheerio(span).html()).toBe('bar');\n      expect(cheerio.html(span)).toBe('<span>bar</span>');\n    });\n\n    it('(<obj>) : should accept an object, an array, or a cheerio object', () => {\n      const $span = cheerio('<span>foo</span>');\n      expect(cheerio.html($span[0])).toBe('<span>foo</span>');\n      expect(cheerio.html($span)).toBe('<span>foo</span>');\n    });\n\n    it('(<value>) : should be able to set to an empty string', () => {\n      const $elem = cheerio('<span>foo</span>').html('');\n      expect(cheerio.html($elem)).toBe('<span></span>');\n    });\n\n    it('(<root>) : does not render the root element', () => {\n      const $ = cheerio.load('');\n      expect(cheerio.html($.root())).toBe(\n        '<html><head></head><body></body></html>',\n      );\n    });\n\n    it('(<elem>, <root>, <elem>) : does not render the root element', () => {\n      const $ = cheerio.load('<div>a div</div><span>a span</span>');\n      const $collection = $('div').add($.root()).add('span');\n      const expected =\n        '<html><head></head><body><div>a div</div><span>a span</span></body></html><div>a div</div><span>a span</span>';\n      expect(cheerio.html($collection)).toBe(expected);\n    });\n\n    it('() : does not crash with `null` as `this` value', () => {\n      const { html } = cheerio;\n      expect(html.call(null as never)).toBe('');\n      expect(html.call(null as never, '#nothing')).toBe('');\n    });\n  });\n\n  describe('.text', () => {\n    it('(cheerio object) : should return the text contents of the specified elements', () => {\n      const $ = cheerio.load('<a>This is <em>content</em>.</a>');\n      expect(cheerio.text($('a'))).toBe('This is content.');\n    });\n\n    it('(cheerio object) : should omit comment nodes', () => {\n      const $ = cheerio.load(\n        '<a>This is <!-- a comment --> not a comment.</a>',\n      );\n      expect(cheerio.text($('a'))).toBe('This is  not a comment.');\n    });\n\n    it('(cheerio object) : should include text contents of children recursively', () => {\n      const $ = cheerio.load(\n        '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n      );\n      expect(cheerio.text($('a'))).toBe(\n        'This is a child with another child and  not a comment followed by one last child and some final text.',\n      );\n    });\n\n    it('() : should return the rendered text content of the root', () => {\n      const $ = cheerio.load(\n        '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n      );\n      expect(cheerio.text($.root())).toBe(\n        'This is a child with another child and  not a comment followed by one last child and some final text.',\n      );\n    });\n\n    it('(cheerio object) : should not omit script tags', () => {\n      const $ = cheerio.load('<script>console.log(\"test\")</script>');\n      expect(cheerio.text($.root())).toBe('console.log(\"test\")');\n    });\n\n    it('(cheerio object) : should omit style tags', () => {\n      const $ = cheerio.load(\n        '<style type=\"text/css\">.cf-hidden { display: none; }</style>',\n      );\n      expect($.text()).toBe('.cf-hidden { display: none; }');\n    });\n\n    it('() : does not crash with `null` as `this` value', () => {\n      const { text } = cheerio;\n      expect(text.call(null as never)).toBe('');\n    });\n  });\n\n  describe('.parseHTML', () => {\n    const $ = cheerio.load('');\n\n    it('() : returns null', () => {\n      expect($.parseHTML()).toBe(null);\n    });\n\n    it('(null) : returns null', () => {\n      expect($.parseHTML(null)).toBe(null);\n    });\n\n    it('(\"\") : returns null', () => {\n      expect($.parseHTML('')).toBe(null);\n    });\n\n    it('(largeHtmlString) : parses large HTML strings', () => {\n      const html = '<div></div>'.repeat(10);\n      const nodes = $.parseHTML(html);\n\n      expect(nodes.length).toBe(10);\n      expect(nodes).toBeInstanceOf(Array);\n    });\n\n    it('(\"<script>\") : ignores scripts by default', () => {\n      const html = '<script>undefined()</script>';\n      expect($.parseHTML(html)).toHaveLength(0);\n    });\n\n    it('(\"<script>\", true) : preserves scripts when requested', () => {\n      const html = '<script>undefined()</script>';\n      expect($.parseHTML(html, true)[0]).toHaveProperty('tagName', 'script');\n    });\n\n    it('(\"scriptAndNonScript) : preserves non-script nodes', () => {\n      const html = '<script>undefined()</script><div></div>';\n      expect($.parseHTML(html)[0]).toHaveProperty('tagName', 'div');\n    });\n\n    it('(scriptAndNonScript, true) : Preserves script position', () => {\n      const html = '<script>undefined()</script><div></div>';\n      expect($.parseHTML(html, true)[0]).toHaveProperty('tagName', 'script');\n    });\n\n    it('(text) : returns a text node', () => {\n      expect($.parseHTML('text')[0].type).toBe('text');\n    });\n\n    it('(\\\\ttext) : preserves leading whitespace', () => {\n      expect($.parseHTML('\\t<div></div>')[0]).toHaveProperty('data', '\\t');\n    });\n\n    it('( text) : Leading spaces are treated as text nodes', () => {\n      expect($.parseHTML(' <div/> ')[0].type).toBe('text');\n    });\n\n    it('(html) : should preserve content', () => {\n      const html = '<div>test div</div>';\n      expect(cheerio($.parseHTML(html)[0]).html()).toBe('test div');\n    });\n\n    it('(malformedHtml) : should not break', () => {\n      expect($.parseHTML('<span><span>')).toHaveLength(1);\n    });\n\n    it('(garbageInput) : should not cause an error', () => {\n      expect(\n        $.parseHTML('<#if><tr><p>This is a test.</p></tr><#/if>'),\n      ).toBeTruthy();\n    });\n\n    it('(text) : should return an array that is not effected by DOM manipulation methods', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML('<b></b><i></i>');\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(2);\n    });\n\n    it('(html, context) : should ignore context argument', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML('<script>foo</script><a>', { foo: 123 });\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(1);\n    });\n\n    it('(html, context, keepScripts) : should ignore context argument', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML(\n        '<script>foo</script><a>',\n        { foo: 123 },\n        true,\n      );\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(2);\n    });\n  });\n\n  describe('.merge', () => {\n    const $ = cheerio.load('');\n\n    it('should be a function', () => {\n      expect(typeof $.merge).toBe('function');\n    });\n\n    it('(arraylike, arraylike) : should modify the first array, but not the second', () => {\n      const arr1 = [1, 2, 3];\n      const arr2 = [4, 5, 6];\n\n      const ret = $.merge(arr1, arr2);\n      expect(typeof ret).toBe('object');\n      expect(Array.isArray(ret)).toBe(true);\n      expect(ret).toBe(arr1);\n      expect(arr1).toHaveLength(6);\n      expect(arr2).toHaveLength(3);\n    });\n\n    it('(arraylike, arraylike) : should handle objects that arent arrays, but are arraylike', () => {\n      const arr1: ArrayLike<string> = {\n        length: 3,\n        0: 'a',\n        1: 'b',\n        2: 'c',\n      };\n      const arr2 = {\n        length: 3,\n        0: 'd',\n        1: 'e',\n        2: 'f',\n      };\n\n      $.merge(arr1, arr2);\n      expect(arr1).toHaveLength(6);\n      expect(arr1[3]).toBe('d');\n      expect(arr1[4]).toBe('e');\n      expect(arr1[5]).toBe('f');\n      expect(arr2).toHaveLength(3);\n    });\n\n    it('(?, ?) : should gracefully reject invalid inputs', () => {\n      expect($.merge([4], 3 as never)).toBeFalsy();\n      expect($.merge({} as never, {} as never)).toBeFalsy();\n      expect($.merge([], {} as never)).toBeFalsy();\n      expect($.merge({} as never, [])).toBeFalsy();\n      const fakeArray1 = { length: 3, 0: 'a', 1: 'b', 3: 'd' };\n      expect($.merge(fakeArray1, [])).toBeFalsy();\n      expect($.merge([], fakeArray1)).toBeFalsy();\n      expect($.merge({ length: '7' } as never, [])).toBeFalsy();\n      expect($.merge({ length: -1 }, [])).toBeFalsy();\n    });\n\n    it('(?, ?) : should no-op on invalid inputs', () => {\n      const fakeArray1 = { length: 3, 0: 'a', 1: 'b', 3: 'd' };\n      $.merge(fakeArray1, []);\n      expect(fakeArray1).toHaveLength(3);\n      expect(fakeArray1[0]).toBe('a');\n      expect(fakeArray1[1]).toBe('b');\n      expect(fakeArray1[3]).toBe('d');\n      $.merge([], fakeArray1);\n      expect(fakeArray1).toHaveLength(3);\n      expect(fakeArray1[0]).toBe('a');\n      expect(fakeArray1[1]).toBe('b');\n      expect(fakeArray1[3]).toBe('d');\n    });\n  });\n\n  describe('.contains', () => {\n    let $: CheerioAPI;\n\n    beforeEach(() => {\n      $ = cheerio.load(fixtures.food);\n    });\n\n    it('(container, contained) : should correctly detect the provided element', () => {\n      const $food = $('#food');\n      const $fruits = $('#fruits');\n      const $apple = $('.apple');\n\n      expect($.contains($food[0], $fruits[0])).toBe(true);\n      expect($.contains($food[0], $apple[0])).toBe(true);\n    });\n\n    it('(container, other) : should not detect elements that are not contained', () => {\n      const $fruits = $('#fruits');\n      const $vegetables = $('#vegetables');\n      const $apple = $('.apple');\n\n      expect($.contains($vegetables[0], $apple[0])).toBe(false);\n      expect($.contains($fruits[0], $vegetables[0])).toBe(false);\n      expect($.contains($vegetables[0], $fruits[0])).toBe(false);\n      expect($.contains($fruits[0], $fruits[0])).toBe(false);\n      expect($.contains($vegetables[0], $vegetables[0])).toBe(false);\n    });\n  });\n\n  describe('.root', () => {\n    it('() : should return a cheerio-wrapped root object', () => {\n      const $ = cheerio.load('<html><head></head><body>foo</body></html>');\n      $.root().append('<div id=\"test\"></div>');\n      expect($.html()).toBe(\n        '<html><head></head><body>foo</body></html><div id=\"test\"></div>',\n      );\n    });\n  });\n\n  describe('.extract', () => {\n    it('() : should extract values for selectors', () => {\n      const $ = cheerio.load(fixtures.eleven);\n\n      expect(\n        $.extract({\n          red: [{ selector: '.red', value: 'outerHTML' }],\n        }),\n      ).toStrictEqual({\n        red: [\n          '<li class=\"red\">Four</li>',\n          '<li class=\"red\">Five</li>',\n          '<li class=\"red sel\">Nine</li>',\n        ],\n      });\n    });\n  });\n});\n\n// Path: src/cheerio.spec.ts\nimport { parseDOM } from 'htmlparser2';\nimport cheerio, { type Cheerio } from './index.js';\nimport { fruits, food, noscript } from './__fixtures__/fixtures.js';\nimport type { Element } from 'domhandler';\n\ndeclare module './index.js' {\n  interface Cheerio<T> {\n    myPlugin(...args: unknown[]): {\n      context: Cheerio<T>;\n      args: unknown[];\n    };\n    foo(): void;\n  }\n}\n\nfunction testAppleSelect($apple: ArrayLike<Element>) {\n  expect($apple).toHaveLength(1);\n  const apple = $apple[0];\n  expect(apple.parentNode).toHaveProperty('tagName', 'ul');\n  expect(apple.prev).toBe(null);\n  expect((apple.next as Element).attribs).toHaveProperty('class', 'orange');\n  expect(apple.childNodes).toHaveLength(1);\n  expect(apple.childNodes[0]).toHaveProperty('data', 'Apple');\n}\n\ndescribe('cheerio', () => {\n  it('cheerio(null) should be empty', () => {\n    expect(cheerio(null as never)).toHaveLength(0);\n  });\n\n  it('cheerio(undefined) should be empty', () => {\n    expect(cheerio(undefined)).toHaveLength(0);\n  });\n\n  it(\"cheerio('') should be empty\", () => {\n    expect(cheerio('')).toHaveLength(0);\n  });\n\n  it('cheerio(selector) with no context or root should be empty', () => {\n    expect(cheerio('.h2')).toHaveLength(0);\n    expect(cheerio('#fruits')).toHaveLength(0);\n  });\n\n  it('cheerio(node) : should override previously-loaded nodes', () => {\n    const $ = cheerio.load('<div><span></span></div>');\n    const spanNode = $('span')[0];\n    const $span = $(spanNode);\n    expect($span[0]).toBe(spanNode);\n  });\n\n  it('should be able to create html without a root or context', () => {\n    const $h2 = cheerio('<h2>');\n    expect($h2).not.toHaveLength(0);\n    expect($h2).toHaveLength(1);\n    expect($h2[0]).toHaveProperty('tagName', 'h2');\n  });\n\n  it('should be able to create complicated html', () => {\n    const $script = cheerio(\n      '<script src=\"script.js\" type=\"text/javascript\"></script>',\n    ) as Cheerio<Element>;\n    expect($script).not.toHaveLength(0);\n    expect($script).toHaveLength(1);\n    expect($script[0].attribs).toHaveProperty('src', 'script.js');\n    expect($script[0].attribs).toHaveProperty('type', 'text/javascript');\n    expect($script[0].childNodes).toHaveLength(0);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with only a context', () => {\n    const $apple = cheerio('.apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with a node as context', () => {\n    const $apple = cheerio('.apple', cheerio(fruits)[0]);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with only a root', () => {\n    const $apple = cheerio('.apple', null, fruits);\n    testAppleSelect($apple);\n  });\n\n  it('should be able to select an id', () => {\n    const $fruits = cheerio('#fruits', null, fruits);\n    expect($fruits).toHaveLength(1);\n    expect($fruits[0].attribs).toHaveProperty('id', 'fruits');\n  });\n\n  it('should be able to select a tag', () => {\n    const $ul = cheerio('ul', fruits);\n    expect($ul).toHaveLength(1);\n    expect($ul[0].tagName).toBe('ul');\n  });\n\n  it('should accept a node reference as a context', () => {\n    const $elems = cheerio('<div><span></span></div>');\n    expect(cheerio('span', $elems[0])).toHaveLength(1);\n  });\n\n  it('should accept an array of node references as a context', () => {\n    const $elems = cheerio('<div><span></span></div>');\n    expect(cheerio('span', $elems.toArray())).toHaveLength(1);\n  });\n\n  it('should select only elements inside given context (Issue #193)', () => {\n    const $ = cheerio.load(food);\n    const $fruits = $('#fruits');\n    const fruitElements = $('li', $fruits);\n\n    expect(fruitElements).toHaveLength(3);\n  });\n\n  it('should be able to select multiple tags', () => {\n    const $fruits = cheerio('li', null, fruits);\n    expect($fruits).toHaveLength(3);\n    const classes = ['apple', 'orange', 'pear'];\n    $fruits.each((idx, $fruit) => {\n      expect($fruit.attribs).toHaveProperty('class', classes[idx]);\n    });\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to do: cheerio(\"#fruits .apple\")', () => {\n    const $apple = cheerio('#fruits .apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to do: cheerio(\"li.apple\")', () => {\n    const $apple = cheerio('li.apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select by attributes', () => {\n    const $apple = cheerio('li[class=apple]', fruits);\n    testAppleSelect($apple);\n  });\n\n  it('should be able to select multiple classes: cheerio(\".btn.primary\")', () => {\n    const $a = cheerio(\n      '.btn.primary',\n      '<p><a class=\"btn primary\" href=\"#\">Save</a></p>',\n    );\n    expect($a).toHaveLength(1);\n    expect($a[0].childNodes[0]).toHaveProperty('data', 'Save');\n  });\n\n  it('should not create a top-level node', () => {\n    const $elem = cheerio('* div', '<div>');\n    expect($elem).toHaveLength(0);\n  });\n\n  it('should be able to select multiple elements: cheerio(\".apple, #fruits\")', () => {\n    const $elems = cheerio('.apple, #fruits', fruits);\n    expect($elems).toHaveLength(2);\n\n    const $apple = $elems\n      .toArray()\n      .filter((elem) => elem.attribs['class'] === 'apple');\n    const $fruit = $elems\n      .toArray()\n      .find((elem) => elem.attribs['id'] === 'fruits');\n    testAppleSelect($apple);\n    expect($fruit?.attribs).toHaveProperty('id', 'fruits');\n  });\n\n  it('should select first element cheerio(:first)', () => {\n    const $elem = cheerio('li:first', fruits);\n    expect($elem.attr('class')).toBe('apple');\n\n    const $filtered = cheerio('li', fruits).filter(':even');\n    expect($filtered).toHaveLength(2);\n  });\n\n  it('should be able to select immediate children: cheerio(\"#fruits > .pear\")', () => {\n    const $food = cheerio(food);\n    cheerio('.pear', $food).append('<li class=\"pear\">Another Pear!</li>');\n    expect(cheerio('#fruits .pear', $food)).toHaveLength(2);\n    const $elem = cheerio('#fruits > .pear', $food);\n    expect($elem).toHaveLength(1);\n    expect($elem.attr('class')).toBe('pear');\n  });\n\n  it('should be able to select immediate children: cheerio(\".apple + .pear\")', () => {\n    expect(cheerio('.apple + li', fruits)).toHaveLength(1);\n    expect(cheerio('.apple + .pear', fruits)).toHaveLength(0);\n    const $elem = cheerio('.apple + .orange', fruits);\n    expect($elem).toHaveLength(1);\n    expect($elem.attr('class')).toBe('orange');\n  });\n\n  it('should be able to select immediate children: cheerio(\".apple ~ .pear\")', () => {\n    expect(cheerio('.apple ~ li', fruits)).toHaveLength(2);\n    expect(cheerio('.apple ~ .pear', fruits).attr('class')).toBe('pear');\n  });\n\n  it('should handle wildcards on attributes: cheerio(\"li[class*=r]\")', () => {\n    const $elem = cheerio('li[class*=r]', fruits);\n    expect($elem).toHaveLength(2);\n    expect($elem.eq(0).attr('class')).toBe('orange');\n    expect($elem.eq(1).attr('class')).toBe('pear');\n  });\n\n  it('should handle beginning of attr selectors: cheerio(\"li[class^=o]\")', () => {\n    const $elem = cheerio('li[class^=o]', fruits);\n    expect($elem).toHaveLength(1);\n    expect($elem.eq(0).attr('class')).toBe('orange');\n  });\n\n  it('should handle beginning of attr selectors: cheerio(\"li[class$=e]\")', () => {\n    const $elem = cheerio('li[class$=e]', fruits);\n    expect($elem).toHaveLength(2);\n    expect($elem.eq(0).attr('class')).toBe('apple');\n    expect($elem.eq(1).attr('class')).toBe('orange');\n  });\n\n  it('(extended Array) should not interfere with prototype methods (issue #119)', () => {\n    const extended: any = [];\n    extended.find =\n      extended.children =\n      extended.each =\n        function () {\n          /* Ignore */\n        };\n    const $empty = cheerio(extended);\n\n    expect($empty.find).toBe(cheerio.prototype.find);\n    expect($empty.children).toBe(cheerio.prototype.children);\n    expect($empty.each).toBe(cheerio.prototype.each);\n  });\n\n  it('cheerio.html(null) should return a \"\" string', () => {\n...\n", "needle_token_start": 640, "needle_token_end": 890, "code_context_ntokens": 15953, "output": ["```typescript\n/**\n * Set one CSS property for every matched element.\n *\n * @category CSS\n * @param prop - The name of the property.\n * @param val - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/css/}\n */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  prop: string,\n  val:\n    | string\n    | ((this: Element, i: number, style: string) => string | undefined),\n): Cheerio<T>;\n\n/**\n * Set multiple CSS properties for every matched element.\n *\n * @category CSS\n * @param map - A map of property names and values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/css/}\n */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  map: Record<string, string>,\n): Cheerio<T>;\n\n/**\n * Set multiple CSS properties for every matched element.\n *\n * @category CSS\n * @param prop - The names of the properties.\n * @param val - The new values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/css/}\n */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  prop?: string | string[] | Record<string, string>,\n  val?:\n    | string\n    | ((this: Element, i: number, style: string) => string | undefined),\n): Cheerio<T> | Record<string, string> | string | undefined {\n  if (\n    (prop != null && val != null) ||\n    // When `prop` is a \"plain\" object\n    (typeof prop === 'object' && !Array.isArray(prop))\n  ) {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        // `prop` can't be an array here anymore.\n        setCss(el, prop as string, val, i);\n      }\n    });\n  }\n\n  if (this.length === 0) {\n    return undefined;\n  }\n\n  return getCss(this[0], prop as string);\n}\n\n/**\n * Set styles of all elements.\n *\n * @private\n * @param el - Element to set style of.\n * @param prop - Name of property.\n * @param value - Value to set property to.\n * @param idx - Optional index within the selection.\n */\nfunction setCss(\n  el: Element,\n  prop: string | Record<string, string>,\n  value:\n    | string\n    | ((this: Element, i: number, style: string) => string | undefined)\n    | undefined,\n  idx: number,\n) {\n  if (typeof prop === 'string') {\n    const styles = getCss(el);\n\n    const val =\n      typeof value === 'function' ? value.call(el, idx, styles[prop]) : value;\n\n    if (val === '') {\n      delete styles[prop];\n    } else if (val != null) {\n      styles[prop] = val;\n    }\n\n    el.attribs['style'] = stringify(styles);\n  } else if (typeof prop === 'object') {\n    const keys = Object.keys(prop);\n    for (let i = 0; i < keys.length; i++) {\n      const k = keys[i];\n      setCss(el, k, prop[k], i);\n    }\n  }\n}\n\n/**\n * Get the parsed styles of the first element.\n *\n * @private\n * @category CSS\n * @param el - Element to get styles from.\n * @param props - Optionally the names of the properties of interest.\n * @returns The parsed styles.\n */\nfunction getCss(el: AnyNode, props?: string[]): Record<string, string>;\n/**\n * Get a property from the parsed styles of the first element.\n *\n * @private\n * @category CSS\n * @param el - Element to get styles from.\n * @param prop - Name of the prop.\n * @returns The value of the property.\n */\nfunction getCss(el: AnyNode, prop: string): string | undefined;\nfunction getCss(\n  el: AnyNode,\n  prop?: string | string[],\n): Record<string, string> | string | undefined {\n  if (!el || !isTag(el)) return;\n\n  const styles = parse(el.attribs['style']);\n  if (typeof prop === 'string') {\n    return styles[prop];\n  }\n  if (Array.isArray(prop)) {\n    const newStyles: Record<string, string> = {};\n    for (const item of prop) {\n      if (styles[item] != null) {\n        newStyles[item] = styles[item];\n      }\n    }\n    return newStyles;\n  }\n  return styles;\n}\n\n/**\n * Stringify `obj` to styles.\n *\n * @private\n * @category CSS\n * @param obj - Object to stringify.\n * @returns The serialized styles.\n */\nfunction stringify(obj: Record<string,"]}
{"repo": "cheeriojs/cheerio", "name": "insertAfter", "language": "typescript", "path": "src/api/manipulation.ts", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To place every element in a specified set of matched elements immediately following a target element within the DOM.\n2. **Input**: The target element after which the elements are to be inserted. This can be specified as an HTML string, a DOM element, an array of DOM elements, or a jQuery-like object.\n3. **Output**: The instance itself, containing the newly inserted elements, allowing for method chaining.\n4. **Procedure**: \n   - First, the function determines if the target is specified as a string and converts it to a DOM element if necessary.\n   - It removes the original set of matched elements from their current location in the DOM.\n   - For each element in the target, the function clones the set of matched elements.\n   - It then finds the position of each target element within its parent's children.\n   - The cloned elements are inserted into the parent's children immediately after the target element's position.\n   - The function returns the set of newly inserted elements wrapped in the instance for further chaining.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " * //     </li>\n * //     <li class=\"pear\">\n * //       <div class=\"healthy\">Pear</div>\n * //     </li>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around the content of each element\n *   in the selection.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/wrapInner/}\n */\nexport const wrapInner = _wrap((el, elInsertLocation, wrapperDom) => {\n  if (!hasChildren(el)) return;\n  updateDOM(el.children, elInsertLocation);\n  updateDOM(wrapperDom, el);\n});\n\n/**\n * The .unwrap() function, removes the parents of the set of matched elements\n * from the DOM, leaving the matched elements in their place.\n *\n * @category Manipulation\n * @example <caption>without selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <div><p>Hello</p></div>\\n  <div><p>World</p></div>\\n</div>',\n * );\n * $('#test p').unwrap();\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @example <caption>with selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <p>Hello</p>\\n  <b><p>World</p></b>\\n</div>',\n * );\n * $('#test p').unwrap('b');\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @param selector - A selector to check the parent element against. If an\n *   element's parent does not match the selector, the element won't be\n *   unwrapped.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/unwrap/}\n */\nexport function unwrap<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  this.parent(selector)\n    .not('body')\n    .each((_, el) => {\n      this._make(el).replaceWith(el.children);\n    });\n  return this;\n}\n\n/**\n * The .wrapAll() function can take any string or object that could be passed to\n * the $() function to specify a DOM structure. This structure may be nested\n * several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around all of the elements in the set of matched\n * elements, as a single group.\n *\n * @category Manipulation\n * @example <caption>With markup passed to `wrapAll`</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div class=\"container\"><div class=\"inner\">First</div><div class=\"inner\">Second</div></div>',\n * );\n * $('.inner').wrapAll(\"<div class='new'></div>\");\n *\n * //=> <div class=\"container\">\n * //     <div class='new'>\n * //       <div class=\"inner\">First</div>\n * //       <div class=\"inner\">Second</div>\n * //     </div>\n * //   </div>\n * ```\n *\n * @example <caption>With an existing cheerio instance</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<span>Span 1</span><strong>Strong</strong><span>Span 2</span>',\n * );\n * const wrap = $('<div><p><em><b></b></em></p></div>');\n * $('span').wrapAll(wrap);\n *\n * //=> <div>\n * //     <p>\n * //       <em>\n * //         <b>\n * //           <span>Span 1</span>\n * //           <span>Span 2</span>\n * //         </b>\n * //       </em>\n * //     </p>\n * //   </div>\n * //   <strong>Strong</strong>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around all matched elements in the\n *   selection.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/wrapAll/}\n */\nexport function wrapAll<T extends AnyNode>(\n  this: Cheerio<T>,\n  wrapper: AcceptedElems<T>,\n): Cheerio<T> {\n  const el = this[0];\n  if (el) {\n    const wrap: Cheerio<AnyNode> = this._make(\n      typeof wrapper === 'function' ? wrapper.call(el, 0, el) : wrapper,\n    ).insertBefore(el);\n\n    // If html is given as wrapper, wrap may contain text elements\n    let elInsertLocation: Element | undefined;\n\n    for (let i = 0; i < wrap.length; i++) {\n      if (wrap[i].type === 'tag') elInsertLocation = wrap[i] as Element;\n    }\n\n    let j = 0;\n\n    /*\n     * Find the deepest child. Only consider the first tag child of each node\n     * (ignore text); stop if no children are found.\n     */\n    while (elInsertLocation && j < elInsertLocation.children.length) {\n      const child = elInsertLocation.children[j];\n      if (child.type === 'tag') {\n        elInsertLocation = child as Element;\n        j = 0;\n      } else {\n        j++;\n      }\n    }\n\n    if (elInsertLocation) this._make(elInsertLocation).append(this);\n  }\n  return this;\n}\n\n/**\n * Insert content next to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').after('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert after each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/after/}\n */\nexport function after<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element after `this` element\n    uniqueSplice(siblings, index + 1, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements after the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertAfter('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements after.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertAfter/}\n */\nexport \nfunction insertAfter<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  if (typeof target === 'string') {\n    target = this._make<AnyNode>(target);\n  }\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  for (const el of this._makeDomArray(target)) {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      continue;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) continue;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index + 1, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  }\n\n  return this._make(clones);\n}\n\n/**\n * Insert content previous to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').before('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert before each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/before/}\n */\nexport function before<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element before `el` element\n    uniqueSplice(siblings, index, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements before the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertBefore('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements before.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertBefore/}\n */\nexport function insertBefore<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const targetArr = this._make<AnyNode>(target);\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  domEach(targetArr, (el) => {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  });\n\n  return this._make(clones);\n}\n\n/**\n * Removes the set of matched elements from the DOM and all their children.\n * `selector` filters the set of matched elements to be removed.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.pear').remove();\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //    </ul>\n * ```\n *\n * @param selector - Optional selector for elements to remove.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/remove/}\n */\nexport function remove<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  // Filter if we have selector\n  const elems = selector ? this.filter(selector) : this;\n\n  domEach(elems, (el) => {\n    removeElement(el);\n    el.prev = el.next = el.parent = null;\n  });\n\n  return this;\n}\n\n/**\n * Replaces matched elements with `content`.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const plum = $('<li class=\"plum\">Plum</li>');\n * $('.pear').replaceWith(plum);\n * $.html();\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">Apple</li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n * ```\n *\n * @param content - Replacement for matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/replaceWith/}\n */\nexport function replaceWith<T extends AnyNode>(\n  this: Cheerio<T>,\n  content: AcceptedElems<AnyNode>,\n): Cheerio<T> {\n  return domEach(this, (el, i) => {\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const cont =\n      typeof content === 'function' ? content.call(el, i, el) : content;\n    const dom = this._makeDomArray(cont);\n\n    /*\n     * In the case that `dom` contains nodes that already exist in other\n     * structures, ensure those nodes are properly removed.\n     */\n    updateDOM(dom, null);\n\n    const index = siblings.indexOf(el);\n\n    // Completely remove old element\n    uniqueSplice(siblings, index, 1, dom, parent);\n\n    if (!dom.includes(el)) {\n      el.parent = el.prev = el.next = null;\n    }\n  });\n}\n\n/**\n * Removes all children from each item in the selection. Text nodes and comment\n * nodes are left as is.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').empty();\n * $.html();\n * //=>  <ul id=\"fruits\"></ul>\n * ```\n *\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/empty/}\n */\nexport function empty<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    el.children.length = 0;\n  });\n}\n\n/**\n * Gets an HTML content string from the first selected element.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').html();\n * //=> Orange\n *\n * $('#fruits').html('<li class=\"mango\">Mango</li>').html();\n * //=> <li class=\"mango\">Mango</li>\n * ```\n *\n * @returns The HTML content string.\n * @see {@link https://api.jquery.com/html/}\n */\nexport function html<T extends AnyNode>(this: Cheerio<T>): string | null;\n/**\n * Replaces each selected element's content with the specified content.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').html('<li class=\"mango\">Mango</li>').html();\n * //=> <li class=\"mango\">Mango</li>\n * ```\n *\n * @param str - The content to replace selection's contents with.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/html/}\n */\nexport function html<T extends AnyNode>(\n  this: Cheerio<T>,\n  str: string | Cheerio<T>,\n): Cheerio<T>;\nexport function html<T extends AnyNode>(\n  this: Cheerio<T>,\n  str?: string | Cheerio<AnyNode>,\n): Cheerio<T> | string | null {\n  if (str === undefined) {\n    const el = this[0];\n    if (!el || !hasChildren(el)) return null;\n    return this._render(el.children);\n  }\n\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    const content = isCheerio(str)\n      ? str.toArray()\n      : this._parse(`${str}`, this.options, false, el).children;\n\n    updateDOM(content, el);\n  });\n}\n\n/**\n * Turns the collection to a string. Alias for `.html()`.\n *\n * @category Manipulation\n * @returns The rendered document.\n */\nexport function toString<T extends AnyNode>(this: Cheerio<T>): string {\n  return this._render(this);\n}\n\n/**\n * Get the combined text contents of each element in the set of matched\n * elements, including their descendants.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').text();\n * //=> Orange\n *\n * $('ul').text();\n * //=>  Apple\n * //    Orange\n * //    Pear\n * ```\n *\n * @returns The text contents of the collection.\n * @see {@link https://api.jquery.com/text/}\n */\nexport function text<T extends AnyNode>(this: Cheerio<T>): string;\n/**\n * Set the content of each element in the set of matched elements to the\n * specified text.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').text('Orange');\n * //=> <div class=\"orange\">Orange</div>\n * ```\n *\n * @param str - The text to set as the content of each matched element.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/text/}\n */\nexport function text<T extends AnyNode>(\n  this: Cheerio<T>,\n  str: string | ((this: AnyNode, i: number, text: string) => string),\n): Cheerio<T>;\nexport function text<T extends AnyNode>(\n  this: Cheerio<T>,\n  str?: string | ((this: AnyNode, i: number, text: string) => string),\n): Cheerio<T> | string {\n  // If `str` is undefined, act as a \"getter\"\n  if (str === undefined) {\n    return staticText(this);\n  }\n  if (typeof str === 'function') {\n    // Function support\n    return domEach(this, (el, i) =>\n      this._make(el).text(str.call(el, i, staticText([el]))),\n    );\n  }\n\n  // Append text node to each selected elements\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    const textNode = new Text(`${str}`);\n\n    updateDOM(textNode, el);\n  });\n}\n\n/**\n * Clone the cheerio object.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const moreFruit = $('#fruits').clone();\n * ```\n *\n * @returns The cloned object.\n * @see {@link https://api.jquery.com/clone/}\n */\nexport function clone<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  const clone = Array.prototype.map.call(this.get(), (el) =>\n    cloneNode(el, true),\n  ) as T[];\n\n  // Add a root node around the cloned nodes\n  const root = new Document(clone);\n  for (const node of clone) {\n    node.parent = root;\n  }\n\n  return this._make(clone);\n}\n\n// Path: src/api/traversing.ts\n/**\n * Methods for traversing the DOM structure.\n *\n * @module cheerio/traversing\n */\n\nimport {\n  isTag,\n  type AnyNode,\n  type Element,\n  hasChildren,\n  isDocument,\n  type Document,\n} from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport * as select from 'cheerio-select';\nimport { domEach, isCheerio } from '../utils.js';\nimport { contains } from '../static.js';\nimport {\n  getChildren,\n  getSiblings,\n  nextElementSibling,\n  prevElementSibling,\n  uniqueSort,\n} from 'domutils';\nimport type { FilterFunction, AcceptedFilters } from '../types.js';\nconst reSiblingSelector = /^\\s*[+~]/;\n\n/**\n * Get the descendants of each element in the current set of matched elements,\n * filtered by a selector, jQuery object, or element.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').find('li').length;\n * //=> 3\n * $('#fruits').find($('.apple')).length;\n * //=> 1\n * ```\n *\n * @param selectorOrHaystack - Element to look for.\n * @returns The found elements.\n * @see {@link https://api.jquery.com/find/}\n */\nexport function find<T extends AnyNode>(\n  this: Cheerio<T>,\n  selectorOrHaystack?: string | Cheerio<Element> | Element,\n): Cheerio<Element> {\n  if (!selectorOrHaystack) {\n    return this._make([]);\n  }\n\n  if (typeof selectorOrHaystack !== 'string') {\n    const haystack = isCheerio(selectorOrHaystack)\n      ? selectorOrHaystack.toArray()\n      : [selectorOrHaystack];\n\n    const context = this.toArray();\n\n    return this._make(\n      haystack.filter((elem) => context.some((node) => contains(node, elem))),\n    );\n  }\n\n  return this._findBySelector(selectorOrHaystack, Number.POSITIVE_INFINITY);\n}\n\n/**\n * Find elements by a specific selector.\n *\n * @private\n * @category Traversing\n * @param selector - Selector to filter by.\n * @param limit - Maximum number of elements to match.\n * @returns The found elements.\n */\nexport function _findBySelector<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector: string,\n  limit: number,\n): Cheerio<Element> {\n  const context = this.toArray();\n\n  const elems = reSiblingSelector.test(selector)\n    ? context\n    : this.children().toArray();\n\n  const options = {\n    context,\n    root: this._root?.[0],\n\n    // Pass options that are recognized by `cheerio-select`\n    xmlMode: this.options.xmlMode,\n    lowerCaseTags: this.options.lowerCaseTags,\n    lowerCaseAttributeNames: this.options.lowerCaseAttributeNames,\n    pseudos: this.options.pseudos,\n    quirksMode: this.options.quirksMode,\n  };\n\n  return this._make(select.select(selector, elems, options, limit));\n}\n\n/**\n * Creates a matcher, using a particular mapping function. Matchers provide a\n * function that finds elements using a generating function, supporting\n * filtering.\n *\n * @private\n * @param matchMap - Mapping function.\n * @returns - Function for wrapping generating functions.\n */\nfunction _getMatcher<P>(\n  matchMap: (fn: (elem: AnyNode) => P, elems: Cheerio<AnyNode>) => Element[],\n) {\n  return function (\n    fn: (elem: AnyNode) => P,\n    ...postFns: ((elems: Element[]) => Element[])[]\n  ) {\n    return function <T extends AnyNode>(\n      this: Cheerio<T>,\n      selector?: AcceptedFilters<Element>,\n    ): Cheerio<Element> {\n      let matched: Element[] = matchMap(fn, this);\n\n      if (selector) {\n        matched = filterArray(\n          matched,\n          selector,\n          this.options.xmlMode,\n          this._root?.[0],\n        );\n      }\n\n      return this._make(\n        // Post processing is only necessary if there is more than one element.\n        this.length > 1 && matched.length > 1\n          ? postFns.reduce((elems, fn) => fn(elems), matched)\n          : matched,\n      );\n    };\n  };\n}\n\n/** Matcher that adds multiple elements for each entry in the input. */\nconst _matcher = _getMatcher((fn: (elem: AnyNode) => Element[], elems) => {\n  let ret: Element[] = [];\n\n  for (let i = 0; i < elems.length; i++) {\n    const value = fn(elems[i]);\n    if (value.length > 0) ret = ret.concat(value);\n  }\n\n  return ret;\n});\n\n/** Matcher that adds at most one element for each entry in the input. */\nconst _singleMatcher = _getMatcher(\n  (fn: (elem: AnyNode) => Element | null, elems) => {\n    const ret: Element[] = [];\n\n    for (let i = 0; i < elems.length; i++) {\n      const value = fn(elems[i]);\n      if (value !== null) {\n        ret.push(value);\n      }\n    }\n    return ret;\n  },\n);\n\n/**\n * Matcher that supports traversing until a condition is met.\n *\n * @param nextElem - Function that returns the next element.\n * @param postFns - Post processing functions.\n * @returns A function usable for `*Until` methods.\n */\nfunction _matchUntil(\n  nextElem: (elem: AnyNode) => Element | null,\n  ...postFns: ((elems: Element[]) => Element[])[]\n) {\n  // We use a variable here that is used from within the matcher.\n  let matches: ((el: Element, i: number) => boolean) | null = null;\n\n  const innerMatcher = _getMatcher(\n    (nextElem: (elem: AnyNode) => Element | null, elems) => {\n      const matched: Element[] = [];\n\n      domEach(elems, (elem) => {\n        for (let next; (next = nextElem(elem)); elem = next) {\n          // FIXME: `matched` might contain duplicates here and the index is too large.\n          if (matches?.(next, matched.length)) break;\n          matched.push(next);\n        }\n      });\n\n      return matched;\n    },\n  )(nextElem, ...postFns);\n\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    selector?: AcceptedFilters<Element> | null,\n    filterSelector?: AcceptedFilters<Element>,\n  ): Cheerio<Element> {\n    // Override `matches` variable with the new target.\n    matches =\n      typeof selector === 'string'\n        ? (elem: Element) => select.is(elem, selector, this.options)\n        : selector\n          ? getFilterFn(selector)\n          : null;\n\n    const ret = innerMatcher.call(this, filterSelector);\n\n    // Set `matches` to `null`, so we don't waste memory.\n    matches = null;\n\n    return ret;\n  };\n}\n\nfunction _removeDuplicates<T extends AnyNode>(elems: T[]): T[] {\n  return elems.length > 1 ? Array.from(new Set<T>(elems)) : elems;\n}\n\n/**\n * Get the parent of each element in the current set of matched elements,\n * optionally filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').parent().attr('id');\n * //=> fruits\n * ```\n *\n * @param selector - If specified filter for parent.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parent/}\n */\nexport const parent = _singleMatcher(\n  ({ parent }) => (parent && !isDocument(parent) ? (parent as Element) : null),\n  _removeDuplicates,\n);\n\n/**\n * Get a set of parents filtered by `selector` of each element in the current\n * set of match elements.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').parents().length;\n * //=> 2\n * $('.orange').parents('#fruits').length;\n * //=> 1\n * ```\n *\n * @param selector - If specified filter for parents.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parents/}\n */\nexport const parents = _matcher(\n  (elem) => {\n    const matched = [];\n    while (elem.parent && !isDocument(elem.parent)) {\n      matched.push(elem.parent as Element);\n      elem = elem.parent;\n    }\n    return matched;\n  },\n  uniqueSort,\n  (elems) => elems.reverse(),\n);\n\n/**\n * Get the ancestors of each element in the current set of matched elements, up\n * to but not including the element matched by the selector, DOM node, or\n * cheerio object.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').parentsUntil('#food').length;\n * //=> 1\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - Optional filter for parents.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parentsUntil/}\n */\nexport const parentsUntil = _matchUntil(\n  ({ parent }) => (parent && !isDocument(parent) ? (parent as Element) : null),\n  uniqueSort,\n  (elems) => elems.reverse(),\n);\n\n/**\n * For each element in the set, get the first element that matches the selector\n * by testing the element itself and traversing up through its ancestors in the\n * DOM tree.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').closest();\n * //=> []\n *\n * $('.orange').closest('.apple');\n * // => []\n *\n * $('.orange').closest('li');\n * //=> [<li class=\"orange\">Orange</li>]\n *\n * $('.orange').closest('#fruits');\n * //=> [<ul id=\"fruits\"> ... </ul>]\n * ```\n *\n * @param selector - Selector for the element to find.\n * @returns The closest nodes.\n * @see {@link https://api.jquery.com/closest/}\n */\nexport function closest<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: AcceptedFilters<Element>,\n): Cheerio<AnyNode> {\n  const set: AnyNode[] = [];\n\n  if (!selector) {\n    return this._make(set);\n  }\n\n  const selectOpts = {\n    xmlMode: this.options.xmlMode,\n    root: this._root?.[0],\n  };\n\n  const selectFn =\n    typeof selector === 'string'\n      ? (elem: Element) => select.is(elem, selector, selectOpts)\n      : getFilterFn(selector);\n\n  domEach(this, (elem: AnyNode | null) => {\n    if (elem && !isDocument(elem) && !isTag(elem)) {\n      elem = elem.parent;\n    }\n    while (elem && isTag(elem)) {\n      if (selectFn(elem, 0)) {\n        // Do not add duplicate elements to the set\n        if (!set.includes(elem)) {\n          set.push(elem);\n        }\n        break;\n      }\n      elem = elem.parent;\n    }\n  });\n\n  return this._make(set);\n}\n\n/**\n * Gets the next sibling of each selected element, optionally filtered by a\n * selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').next().hasClass('orange');\n * //=> true\n * ```\n *\n * @param selector - If specified filter for sibling.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/next/}\n */\nexport const next = _singleMatcher((elem) => nextElementSibling(elem));\n\n/**\n * Gets all the following siblings of the each selected element, optionally\n * filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').nextAll();\n * //=> [<li class=\"orange\">Orange</li>, <li class=\"pear\">Pear</li>]\n * $('.apple').nextAll('.orange');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/nextAll/}\n */\nexport const nextAll = _matcher((elem) => {\n  const matched = [];\n  while (elem.next) {\n    elem = elem.next;\n    if (isTag(elem)) matched.push(elem);\n  }\n  return matched;\n}, _removeDuplicates);\n\n/**\n * Gets all the following siblings up to but not including the element matched\n * by the selector, optionally filtered by another selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').nextUntil('.pear');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - If specified filter for siblings.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/nextUntil/}\n */\nexport const nextUntil = _matchUntil(\n  (el) => nextElementSibling(el),\n  _removeDuplicates,\n);\n\n/**\n * Gets the previous sibling of each selected element optionally filtered by a\n * selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').prev().hasClass('apple');\n * //=> true\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prev/}\n */\nexport const prev = _singleMatcher((elem) => prevElementSibling(elem));\n\n/**\n * Gets all the preceding siblings of each selected element, optionally filtered\n * by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').prevAll();\n * //=> [<li class=\"orange\">Orange</li>, <li class=\"apple\">Apple</li>]\n *\n * $('.pear').prevAll('.orange');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prevAll/}\n */\nexport const prevAll = _matcher((elem) => {\n  const matched = [];\n  while (elem.prev) {\n    elem = elem.prev;\n    if (isTag(elem)) matched.push(elem);\n  }\n  return matched;\n}, _removeDuplicates);\n\n/**\n * Gets all the preceding siblings up to but not including the element matched\n * by the selector, optionally filtered by another selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').prevUntil('.apple');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prevUntil/}\n */\nexport const prevUntil = _matchUntil(\n  (el) => prevElementSibling(el),\n  _removeDuplicates,\n);\n\n/**\n * Get the siblings of each element (excluding the element) in the set of\n * matched elements, optionally filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').siblings().length;\n * //=> 2\n *\n * $('.pear').siblings('.orange').length;\n * //=> 1\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The siblings.\n * @see {@link https://api.jquery.com/siblings/}\n */\nexport const siblings = _matcher(\n  (elem) =>\n    getSiblings(elem).filter((el): el is Element => isTag(el) && el !== elem),\n  uniqueSort,\n);\n\n/**\n * Gets the element children of each element in the set of matched elements.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').children().length;\n * //=> 3\n *\n * $('#fruits').children('.pear').text();\n * //=> Pear\n * ```\n *\n * @param selector - If specified filter for children.\n * @returns The children.\n * @see {@link https://api.jquery.com/children/}\n */\nexport const children = _matcher(\n  (elem) => getChildren(elem).filter(isTag),\n  _removeDuplicates,\n);\n\n/**\n * Gets the children of each element in the set of matched elements, including\n * text and comment nodes.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').contents().length;\n * //=> 3\n * ```\n *\n * @returns The children.\n * @see {@link https://api.jquery.com/contents/}\n */\nexport function contents<T extends AnyNode>(\n  this: Cheerio<T>,\n): Cheerio<AnyNode> {\n  const elems = this.toArray().reduce<AnyNode[]>(\n    (newElems, elem) =>\n      hasChildren(elem) ? newElems.concat(elem.children) : newElems,\n    [],\n  );\n  return this._make(elems);\n}\n\n/**\n * Iterates over a cheerio object, executing a function for each matched\n * element. When the callback is fired, the function is fired in the context of\n * the DOM element, so `this` refers to the current element, which is equivalent\n * to the function parameter `element`. To break out of the `each` loop early,\n * return with `false`.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * const fruits = [];\n *\n * $('li').each(function (i, elem) {\n *   fruits[i] = $(this).text();\n * });\n *\n * fruits.join(', ');\n * //=> Apple, Orange, Pear\n * ```\n *\n * @param fn - Function to execute.\n * @returns The instance itself, useful for chaining.\n * @see {@link https://api.jquery.com/each/}\n */\nexport function each<T>(\n  this: Cheerio<T>,\n  fn: (this: T, i: number, el: T) => void | boolean,\n): Cheerio<T> {\n  let i = 0;\n  const len = this.length;\n  while (i < len && fn.call(this[i], i, this[i]) !== false) ++i;\n  return this;\n}\n\n/**\n * Pass each element in the current matched set through a function, producing a\n * new Cheerio object containing the return values. The function can return an\n * individual data item or an array of data items to be inserted into the\n * resulting set. If an array is returned, the elements inside the array are\n * inserted into the set. If the function returns null or undefined, no element\n * will be inserted.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li')\n *   .map(function (i, el) {\n *     // this === el\n *     return $(this).text();\n *   })\n *   .toArray()\n *   .join(' ');\n * //=> \"apple orange pear\"\n * ```\n *\n * @param fn - Function to execute.\n * @returns The mapped elements, wrapped in a Cheerio collection.\n * @see {@link https://api.jquery.com/map/}\n */\nexport function map<T, M>(\n  this: Cheerio<T>,\n  fn: (this: T, i: number, el: T) => M[] | M | null | undefined,\n): Cheerio<M> {\n  let elems: M[] = [];\n  for (let i = 0; i < this.length; i++) {\n    const el = this[i];\n    const val = fn.call(el, i, el);\n    if (val != null) {\n      elems = elems.concat(val);\n    }\n  }\n  return this._make(elems);\n}\n\n/**\n * Creates a function to test if a filter is matched.\n *\n * @param match - A filter.\n * @returns A function that determines if a filter has been matched.\n */\nfunction getFilterFn<T>(\n  match: FilterFunction<T> | Cheerio<T> | T,\n): (el: T, i: number) => boolean {\n  if (typeof match === 'function') {\n    return (el, i) => (match as FilterFunction<T>).call(el, i, el);\n  }\n  if (isCheerio<T>(match)) {\n    return (el) => Array.prototype.includes.call(match, el);\n  }\n  return function (el) {\n    return match === el;\n  };\n}\n\n/**\n * Iterates over a cheerio object, reducing the set of selector elements to\n * those that match the selector or pass the function's test.\n *\n * This is the definition for using type guards; have a look below for other\n * ways to invoke this method. The function is executed in the context of the\n * selected element, so `this` refers to the current element.\n *\n * @category Traversing\n * @example <caption>Function</caption>\n *\n * ```js\n * $('li')\n *   .filter(function (i, el) {\n *     // this === el\n *     return $(this).attr('class') === 'orange';\n *   })\n *   .attr('class'); //=> orange\n * ```\n *\n * @param match - Value to look for, following the rules above.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/filter/}\n */\nexport function filter<T, S extends T>(\n  this: Cheerio<T>,\n  match: (this: T, index: number, value: T) => value is S,\n): Cheerio<S>;\n/**\n * Iterates over a cheerio object, reducing the set of selector elements to\n * those that match the selector or pass the function's test.\n *\n * - When a Cheerio selection is specified, return only the elements contained in\n *   that selection.\n * - When an element is specified, return only that element (if it is contained in\n *   the original selection).\n * - If using the function method, the function is executed in the context of the\n *   selected element, so `this` refers to the current element.\n *\n * @category Traversing\n * @example <caption>Selector</caption>\n *\n * ```js\n * $('li').filter('.orange').attr('class');\n * //=> orange\n * ```\n *\n * @example <caption>Function</caption>\n *\n * ```js\n * $('li')\n *   .filter(function (i, el) {\n *     // this === el\n *     return $(this).attr('class') === 'orange';\n *   })\n *   .attr('class'); //=> orange\n * ```\n *\n * @param match - Value to look for, following the rules above. See\n *   {@link AcceptedFilters}.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/filter/}\n */\nexport function filter<T, S extends AcceptedFilters<T>>(\n  this: Cheerio<T>,\n  match: S,\n): Cheerio<S extends string ? Element : T>;\nexport function filter<T>(\n  this: Cheerio<T>,\n  match: AcceptedFilters<T>,\n): Cheerio<unknown> {\n  return this._make<unknown>(\n    filterArray(this.toArray(), match, this.options.xmlMode, this._root?.[0]),\n  );\n}\n\nexport function filterArray<T>(\n  nodes: T[],\n  match: AcceptedFilters<T>,\n  xmlMode?: boolean,\n  root?: Document,\n): Element[] | T[] {\n  return typeof match === 'string'\n    ? select.filter(match, nodes as unknown as AnyNode[], { xmlMode, root })\n    : nodes.filter(getFilterFn<T>(match));\n}\n\n/**\n * Checks the current list of elements and returns `true` if _any_ of the\n * elements match the selector. If using an element or Cheerio selection,\n * returns `true` if _any_ of the elements match. If using a predicate function,\n * the function is executed in the context of the selected element, so `this`\n * refers to the current element.\n *\n * @category Traversing\n * @param selector - Selector for the selection.\n * @returns Whether or not the selector matches an element of the instance.\n * @see {@link https://api.jquery.com/is/}\n */\nexport function is<T>(\n  this: Cheerio<T>,\n  selector?: AcceptedFilters<T>,\n): boolean {\n  const nodes = this.toArray();\n  return typeof selector === 'string'\n    ? select.some(\n        (nodes as unknown as AnyNode[]).filter(isTag),\n        selector,\n        this.options,\n      )\n    : selector\n      ? nodes.some(getFilterFn<T>(selector))\n      : false;\n}\n\n/**\n * Remove elements from the set of matched elements. Given a Cheerio object that\n * represents a set of DOM elements, the `.not()` method constructs a new\n * Cheerio object from a subset of the matching elements. The supplied selector\n * is tested against each element; the elements that don't match the selector\n * will be included in the result.\n *\n * The `.not()` method can take a function as its argument in the same way that\n * `.filter()` does. Elements for which the function returns `true` are excluded\n * from the filtered set; all other elements are included.\n *\n * @category Traversing\n * @example <caption>Selector</caption>\n *\n * ```js\n * $('li').not('.apple').length;\n * //=> 2\n * ```\n *\n * @example <caption>Function</caption>\n *\n * ```js\n * $('li').not(function (i, el) {\n *   // this === el\n *   return $(this).attr('class') === 'orange';\n * }).length; //=> 2\n * ```\n *\n * @param match - Value to look for, following the rules above.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/not/}\n */\nexport function not<T extends AnyNode>(\n  this: Cheerio<T>,\n  match: AcceptedFilters<T>,\n): Cheerio<T> {\n  let nodes = this.toArray();\n\n  if (typeof match === 'string') {\n    const matches = new Set<AnyNode>(select.filter(match, nodes, this.options));\n    nodes = nodes.filter((el) => !matches.has(el));\n  } else {\n    const filterFn = getFilterFn(match);\n    nodes = nodes.filter((el, i) => !filterFn(el, i));\n  }\n\n  return this._make(nodes);\n}\n\n/**\n * Filters the set of matched elements to only those which have the given DOM\n * element as a descendant or which have a descendant that matches the given\n * selector. Equivalent to `.filter(':has(selector)')`.\n *\n * @category Traversing\n * @example <caption>Selector</caption>\n *\n * ```js\n * $('ul').has('.pear').attr('id');\n * //=> fruits\n * ```\n *\n * @example <caption>Element</caption>\n *\n * ```js\n * $('ul').has($('.pear')[0]).attr('id');\n * //=> fruits\n * ```\n *\n * @param selectorOrHaystack - Element to look for.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/has/}\n */\nexport function has(\n  this: Cheerio<AnyNode | Element>,\n  selectorOrHaystack: string | Cheerio<Element> | Element,\n): Cheerio<AnyNode | Element> {\n  return this.filter(\n    typeof selectorOrHaystack === 'string'\n      ? // Using the `:has` selector here short-circuits searches.\n        `:has(${selectorOrHaystack})`\n      : (_, el) => this._make(el).find(selectorOrHaystack).length > 0,\n  );\n}\n\n/**\n * Will select the first element of a cheerio object.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').children().first().text();\n * //=> Apple\n * ```\n *\n * @returns The first element.\n * @see {@link https://api.jquery.com/first/}\n */\nexport function first<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  return this.length > 1 ? this._make(this[0]) : this;\n}\n\n/**\n * Will select the last element of a cheerio object.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').children().last().text();\n * //=> Pear\n * ```\n *\n * @returns The last element.\n * @see {@link https://api.jquery.com/last/}\n */\nexport function last<T>(this: Cheerio<T>): Cheerio<T> {\n  return this.length > 0 ? this._make(this[this.length - 1]) : this;\n}\n\n/**\n * Reduce the set of matched elements to the one at the specified index. Use\n * `.eq(-i)` to count backwards from the last selected element.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li').eq(0).text();\n * //=> Apple\n *\n * $('li').eq(-1).text();\n * //=> Pear\n * ```\n *\n * @param i - Index of the element to select.\n * @returns The element at the `i`th position.\n * @see {@link https://api.jquery.com/eq/}\n */\nexport function eq<T>(this: Cheerio<T>, i: number): Cheerio<T> {\n  i = +i;\n\n  // Use the first identity optimization if possible\n  if (i === 0 && this.length <= 1) return this;\n\n  if (i < 0) i = this.length + i;\n  return this._make(this[i] ?? []);\n}\n\n/**\n * Retrieve one of the elements matched by the Cheerio object, at the `i`th\n * position.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li').get(0).tagName;\n * //=> li\n * ```\n *\n * @param i - Element to retrieve.\n * @returns The element at the `i`th position.\n * @see {@link https://api.jquery.com/get/}\n */\nexport function get<T>(this: Cheerio<T>, i: number): T | undefined;\n/**\n * Retrieve all elements matched by the Cheerio object, as an array.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li').get().length;\n * //=> 3\n * ```\n *\n * @returns All elements matched by the Cheerio object.\n * @see {@link https://api.jquery.com/get/}\n */\nexport function get<T>(this: Cheerio<T>): T[];\nexport function get<T>(this: Cheerio<T>, i?: number): T | T[] {\n  if (i == null) {\n    return this.toArray();\n  }\n  return this[i < 0 ? this.length + i : i];\n}\n\n/**\n * Retrieve all the DOM elements contained in the jQuery set as an array.\n *\n * @example\n *\n * ```js\n * $('li').toArray();\n * //=> [ {...}, {...}, {...} ]\n * ```\n *\n * @returns The contained items.\n */\nexport function toArray<T>(this: Cheerio<T>): T[] {\n  return Array.prototype.slice.call(this);\n}\n\n/**\n * Search for a given element from among the matched elements.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').index();\n * //=> 2 $('.orange').index('li');\n * //=> 1\n * $('.apple').index($('#fruit, li'));\n * //=> 1\n * ```\n *\n * @param selectorOrNeedle - Element to look for.\n * @returns The index of the element.\n * @see {@link https://api.jquery.com/index/}\n */\nexport function index<T extends AnyNode>(\n  this: Cheerio<T>,\n  selectorOrNeedle?: string | Cheerio<AnyNode> | AnyNode,\n): number {\n  let $haystack: Cheerio<AnyNode>;\n  let needle: AnyNode;\n\n  if (selectorOrNeedle == null) {\n    $haystack = this.parent().children();\n    needle = this[0];\n  } else if (typeof selectorOrNeedle === 'string') {\n    $haystack = this._make<AnyNode>(selectorOrNeedle);\n    needle = this[0];\n  } else {\n    // eslint-disable-next-line @typescript-eslint/no-this-alias, unicorn/no-this-assignment\n    $haystack = this;\n    needle = isCheerio(selectorOrNeedle)\n      ? selectorOrNeedle[0]\n      : selectorOrNeedle;\n  }\n\n  return Array.prototype.indexOf.call($haystack, needle);\n}\n\n/**\n * Gets the elements matching the specified range (0-based position).\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li').slice(1).eq(0).text();\n * //=> 'Orange'\n *\n * $('li').slice(1, 2).length;\n * //=> 1\n * ```\n *\n * @param start - A position at which the elements begin to be selected. If\n *   negative, it indicates an offset from the end of the set.\n * @param end - A position at which the elements stop being selected. If\n *   negative, it indicates an offset from the end of the set. If omitted, the\n *   range continues until the end of the set.\n * @returns The elements matching the specified range.\n * @see {@link https://api.jquery.com/slice/}\n */\nexport function slice<T>(\n  this: Cheerio<T>,\n  start?: number,\n  end?: number,\n): Cheerio<T> {\n  return this._make(Array.prototype.slice.call(this, start, end));\n}\n\n/**\n * End the most recent filtering operation in the current chain and return the\n * set of matched elements to its previous state.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li').eq(0).end().length;\n * //=> 3\n * ```\n *\n * @returns The previous state of the set of matched elements.\n * @see {@link https://api.jquery.com/end/}\n */\nexport function end<T>(this: Cheerio<T>): Cheerio<AnyNode> {\n  return this.prevObject ?? this._make([]);\n}\n\n/**\n * Add elements to the set of matched elements.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').add('.orange').length;\n * //=> 2\n * ```\n *\n * @param other - Elements to add.\n * @param context - Optionally the context of the new selection.\n * @returns The combined set.\n * @see {@link https://api.jquery.com/add/}\n */\nexport function add<S extends AnyNode, T extends AnyNode>(\n  this: Cheerio<T>,\n  other: string | Cheerio<S> | S | S[],\n  context?: Cheerio<S> | string,\n): Cheerio<S | T> {\n  const selection = this._make(other, context);\n  const contents = uniqueSort([...this.get(), ...selection.get()]);\n  return this._make(contents);\n}\n\n/**\n * Add the previous set of elements on the stack to the current set, optionally\n * filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li').eq(0).addBack('.orange').length;\n * //=> 2\n * ```\n *\n * @param selector - Selector for the elements to add.\n * @returns The combined set.\n * @see {@link https://api.jquery.com/addBack/}\n */\nexport function addBack<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<AnyNode> {\n  return this.prevObject\n    ? this.add(selector ? this.prevObject.filter(selector) : this.prevObject)\n    : this;\n}\n\n// Path: src/api/css.ts\nimport { domEach } from '../utils.js';\nimport { isTag, type Element, type AnyNode } from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\n\n/**\n * Get the value of a style property for the first element in the set of matched\n * elements.\n *\n * @category CSS\n * @param names - Optionally the names of the properties of interest.\n * @returns A map of all of the style properties.\n * @see {@link https://api.jquery.com/css/}\n */\nexport function css<T extends AnyNode>(\n  this: Cheerio<T>,\n  names?: string[],\n): Record<string, string> | undefined;\n/**\n * Get the value of a style property for the first element in the set of matched\n * elements.\n *\n * @category CSS\n...\n", "needle_token_start": 2164, "needle_token_end": 2430, "code_context_ntokens": 15290, "output": ["```javascript\n/**\n * Insert every element in the set of matched elements after the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertAfter('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements after.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertAfter/}\n */\nexport function insertAfter<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  if (typeof target === 'string') {\n    target = this._make<AnyNode>(target);\n  }\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  for (const el of this._makeDomArray(target)) {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      continue;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) continue;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index + 1, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  }\n\n  return this._make(clones);\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "testAppleSelect", "language": "typescript", "path": "src/cheerio.spec.ts", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate the structure and properties of a specific HTML element within a DOM tree, ensuring it meets certain criteria related to its position and attributes.\n2. **Input**: The function receives an array-like object containing HTML elements.\n3. **Output**: There is no return value; instead, the function performs assertions to check various properties of the element.\n4. **Procedure**: \n   - The function first checks if the input contains exactly one element.\n   - It then verifies that this element's parent node is a 'ul' tag.\n   - It checks that the element has no previous sibling.\n   - It ensures that the next sibling is an element with a class attribute of 'orange'.\n   - Finally, it checks that the element has exactly one child node, which must contain the text 'Apple'.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/__fixtures__/fixtures.ts\nexport const fruits = [\n  '<ul id=\"fruits\">',\n  '<li class=\"apple\">Apple</li>',\n  '<li class=\"orange\">Orange</li>',\n  '<li class=\"pear\">Pear</li>',\n  '</ul>',\n].join('');\n\nexport const vegetables = [\n  '<ul id=\"vegetables\">',\n...\n// Path: src/static.spec.ts\nimport * as fixtures from './__fixtures__/fixtures.js';\nimport cheerio, { type CheerioAPI } from './index.js';\n\ndescribe('cheerio', () => {\n  describe('.html', () => {\n    it('() : should return innerHTML; $.html(obj) should return outerHTML', () => {\n      const $div = cheerio(\n        'div',\n        '<div><span>foo</span><span>bar</span></div>',\n      );\n      const span = $div.children()[1];\n      expect(cheerio(span).html()).toBe('bar');\n      expect(cheerio.html(span)).toBe('<span>bar</span>');\n    });\n\n    it('(<obj>) : should accept an object, an array, or a cheerio object', () => {\n      const $span = cheerio('<span>foo</span>');\n      expect(cheerio.html($span[0])).toBe('<span>foo</span>');\n      expect(cheerio.html($span)).toBe('<span>foo</span>');\n    });\n\n    it('(<value>) : should be able to set to an empty string', () => {\n      const $elem = cheerio('<span>foo</span>').html('');\n      expect(cheerio.html($elem)).toBe('<span></span>');\n    });\n\n    it('(<root>) : does not render the root element', () => {\n      const $ = cheerio.load('');\n      expect(cheerio.html($.root())).toBe(\n        '<html><head></head><body></body></html>',\n      );\n    });\n\n    it('(<elem>, <root>, <elem>) : does not render the root element', () => {\n      const $ = cheerio.load('<div>a div</div><span>a span</span>');\n      const $collection = $('div').add($.root()).add('span');\n      const expected =\n        '<html><head></head><body><div>a div</div><span>a span</span></body></html><div>a div</div><span>a span</span>';\n      expect(cheerio.html($collection)).toBe(expected);\n    });\n\n    it('() : does not crash with `null` as `this` value', () => {\n      const { html } = cheerio;\n      expect(html.call(null as never)).toBe('');\n      expect(html.call(null as never, '#nothing')).toBe('');\n    });\n  });\n\n  describe('.text', () => {\n    it('(cheerio object) : should return the text contents of the specified elements', () => {\n      const $ = cheerio.load('<a>This is <em>content</em>.</a>');\n      expect(cheerio.text($('a'))).toBe('This is content.');\n    });\n\n    it('(cheerio object) : should omit comment nodes', () => {\n      const $ = cheerio.load(\n        '<a>This is <!-- a comment --> not a comment.</a>',\n      );\n      expect(cheerio.text($('a'))).toBe('This is  not a comment.');\n    });\n\n    it('(cheerio object) : should include text contents of children recursively', () => {\n      const $ = cheerio.load(\n        '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n      );\n      expect(cheerio.text($('a'))).toBe(\n        'This is a child with another child and  not a comment followed by one last child and some final text.',\n      );\n    });\n\n    it('() : should return the rendered text content of the root', () => {\n      const $ = cheerio.load(\n        '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n      );\n      expect(cheerio.text($.root())).toBe(\n        'This is a child with another child and  not a comment followed by one last child and some final text.',\n      );\n    });\n\n    it('(cheerio object) : should not omit script tags', () => {\n      const $ = cheerio.load('<script>console.log(\"test\")</script>');\n      expect(cheerio.text($.root())).toBe('console.log(\"test\")');\n    });\n\n    it('(cheerio object) : should omit style tags', () => {\n      const $ = cheerio.load(\n        '<style type=\"text/css\">.cf-hidden { display: none; }</style>',\n      );\n      expect($.text()).toBe('.cf-hidden { display: none; }');\n    });\n\n    it('() : does not crash with `null` as `this` value', () => {\n      const { text } = cheerio;\n      expect(text.call(null as never)).toBe('');\n    });\n  });\n\n  describe('.parseHTML', () => {\n    const $ = cheerio.load('');\n\n    it('() : returns null', () => {\n      expect($.parseHTML()).toBe(null);\n    });\n\n    it('(null) : returns null', () => {\n      expect($.parseHTML(null)).toBe(null);\n    });\n\n    it('(\"\") : returns null', () => {\n      expect($.parseHTML('')).toBe(null);\n    });\n\n    it('(largeHtmlString) : parses large HTML strings', () => {\n      const html = '<div></div>'.repeat(10);\n      const nodes = $.parseHTML(html);\n\n      expect(nodes.length).toBe(10);\n      expect(nodes).toBeInstanceOf(Array);\n    });\n\n    it('(\"<script>\") : ignores scripts by default', () => {\n      const html = '<script>undefined()</script>';\n      expect($.parseHTML(html)).toHaveLength(0);\n    });\n\n    it('(\"<script>\", true) : preserves scripts when requested', () => {\n      const html = '<script>undefined()</script>';\n      expect($.parseHTML(html, true)[0]).toHaveProperty('tagName', 'script');\n    });\n\n    it('(\"scriptAndNonScript) : preserves non-script nodes', () => {\n      const html = '<script>undefined()</script><div></div>';\n      expect($.parseHTML(html)[0]).toHaveProperty('tagName', 'div');\n    });\n\n    it('(scriptAndNonScript, true) : Preserves script position', () => {\n      const html = '<script>undefined()</script><div></div>';\n      expect($.parseHTML(html, true)[0]).toHaveProperty('tagName', 'script');\n    });\n\n    it('(text) : returns a text node', () => {\n      expect($.parseHTML('text')[0].type).toBe('text');\n    });\n\n    it('(\\\\ttext) : preserves leading whitespace', () => {\n      expect($.parseHTML('\\t<div></div>')[0]).toHaveProperty('data', '\\t');\n    });\n\n    it('( text) : Leading spaces are treated as text nodes', () => {\n      expect($.parseHTML(' <div/> ')[0].type).toBe('text');\n    });\n\n    it('(html) : should preserve content', () => {\n      const html = '<div>test div</div>';\n      expect(cheerio($.parseHTML(html)[0]).html()).toBe('test div');\n    });\n\n    it('(malformedHtml) : should not break', () => {\n      expect($.parseHTML('<span><span>')).toHaveLength(1);\n    });\n\n    it('(garbageInput) : should not cause an error', () => {\n      expect(\n        $.parseHTML('<#if><tr><p>This is a test.</p></tr><#/if>'),\n      ).toBeTruthy();\n    });\n\n    it('(text) : should return an array that is not effected by DOM manipulation methods', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML('<b></b><i></i>');\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(2);\n    });\n\n    it('(html, context) : should ignore context argument', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML('<script>foo</script><a>', { foo: 123 });\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(1);\n    });\n\n    it('(html, context, keepScripts) : should ignore context argument', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML(\n        '<script>foo</script><a>',\n        { foo: 123 },\n        true,\n      );\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(2);\n    });\n  });\n\n  describe('.merge', () => {\n    const $ = cheerio.load('');\n\n    it('should be a function', () => {\n      expect(typeof $.merge).toBe('function');\n    });\n\n    it('(arraylike, arraylike) : should modify the first array, but not the second', () => {\n      const arr1 = [1, 2, 3];\n      const arr2 = [4, 5, 6];\n\n      const ret = $.merge(arr1, arr2);\n      expect(typeof ret).toBe('object');\n      expect(Array.isArray(ret)).toBe(true);\n      expect(ret).toBe(arr1);\n      expect(arr1).toHaveLength(6);\n      expect(arr2).toHaveLength(3);\n    });\n\n    it('(arraylike, arraylike) : should handle objects that arent arrays, but are arraylike', () => {\n      const arr1: ArrayLike<string> = {\n        length: 3,\n        0: 'a',\n        1: 'b',\n        2: 'c',\n      };\n      const arr2 = {\n        length: 3,\n        0: 'd',\n        1: 'e',\n        2: 'f',\n      };\n\n      $.merge(arr1, arr2);\n      expect(arr1).toHaveLength(6);\n      expect(arr1[3]).toBe('d');\n      expect(arr1[4]).toBe('e');\n      expect(arr1[5]).toBe('f');\n      expect(arr2).toHaveLength(3);\n    });\n\n    it('(?, ?) : should gracefully reject invalid inputs', () => {\n      expect($.merge([4], 3 as never)).toBeFalsy();\n      expect($.merge({} as never, {} as never)).toBeFalsy();\n      expect($.merge([], {} as never)).toBeFalsy();\n      expect($.merge({} as never, [])).toBeFalsy();\n      const fakeArray1 = { length: 3, 0: 'a', 1: 'b', 3: 'd' };\n      expect($.merge(fakeArray1, [])).toBeFalsy();\n      expect($.merge([], fakeArray1)).toBeFalsy();\n      expect($.merge({ length: '7' } as never, [])).toBeFalsy();\n      expect($.merge({ length: -1 }, [])).toBeFalsy();\n    });\n\n    it('(?, ?) : should no-op on invalid inputs', () => {\n      const fakeArray1 = { length: 3, 0: 'a', 1: 'b', 3: 'd' };\n      $.merge(fakeArray1, []);\n      expect(fakeArray1).toHaveLength(3);\n      expect(fakeArray1[0]).toBe('a');\n      expect(fakeArray1[1]).toBe('b');\n      expect(fakeArray1[3]).toBe('d');\n      $.merge([], fakeArray1);\n      expect(fakeArray1).toHaveLength(3);\n      expect(fakeArray1[0]).toBe('a');\n      expect(fakeArray1[1]).toBe('b');\n      expect(fakeArray1[3]).toBe('d');\n    });\n  });\n\n  describe('.contains', () => {\n    let $: CheerioAPI;\n\n    beforeEach(() => {\n      $ = cheerio.load(fixtures.food);\n    });\n\n    it('(container, contained) : should correctly detect the provided element', () => {\n      const $food = $('#food');\n      const $fruits = $('#fruits');\n      const $apple = $('.apple');\n\n      expect($.contains($food[0], $fruits[0])).toBe(true);\n      expect($.contains($food[0], $apple[0])).toBe(true);\n    });\n\n    it('(container, other) : should not detect elements that are not contained', () => {\n      const $fruits = $('#fruits');\n      const $vegetables = $('#vegetables');\n      const $apple = $('.apple');\n\n      expect($.contains($vegetables[0], $apple[0])).toBe(false);\n      expect($.contains($fruits[0], $vegetables[0])).toBe(false);\n      expect($.contains($vegetables[0], $fruits[0])).toBe(false);\n      expect($.contains($fruits[0], $fruits[0])).toBe(false);\n      expect($.contains($vegetables[0], $vegetables[0])).toBe(false);\n    });\n  });\n\n  describe('.root', () => {\n    it('() : should return a cheerio-wrapped root object', () => {\n      const $ = cheerio.load('<html><head></head><body>foo</body></html>');\n      $.root().append('<div id=\"test\"></div>');\n      expect($.html()).toBe(\n        '<html><head></head><body>foo</body></html><div id=\"test\"></div>',\n      );\n    });\n  });\n\n  describe('.extract', () => {\n    it('() : should extract values for selectors', () => {\n      const $ = cheerio.load(fixtures.eleven);\n\n      expect(\n        $.extract({\n          red: [{ selector: '.red', value: 'outerHTML' }],\n        }),\n      ).toStrictEqual({\n        red: [\n          '<li class=\"red\">Four</li>',\n          '<li class=\"red\">Five</li>',\n          '<li class=\"red sel\">Nine</li>',\n        ],\n      });\n    });\n  });\n});\n\n// Path: src/cheerio.spec.ts\nimport { parseDOM } from 'htmlparser2';\nimport cheerio, { type Cheerio } from './index.js';\nimport { fruits, food, noscript } from './__fixtures__/fixtures.js';\nimport type { Element } from 'domhandler';\n\ndeclare module './index.js' {\n  interface Cheerio<T> {\n    myPlugin(...args: unknown[]): {\n      context: Cheerio<T>;\n      args: unknown[];\n    };\n    foo(): void;\n  }\n}\n\n\nfunction testAppleSelect($apple: ArrayLike<Element>) {\n  expect($apple).toHaveLength(1);\n  const apple = $apple[0];\n  expect(apple.parentNode).toHaveProperty('tagName', 'ul');\n  expect(apple.prev).toBe(null);\n  expect((apple.next as Element).attribs).toHaveProperty('class', 'orange');\n  expect(apple.childNodes).toHaveLength(1);\n  expect(apple.childNodes[0]).toHaveProperty('data', 'Apple');\n}\n\ndescribe('cheerio', () => {\n  it('cheerio(null) should be empty', () => {\n    expect(cheerio(null as never)).toHaveLength(0);\n  });\n\n  it('cheerio(undefined) should be empty', () => {\n    expect(cheerio(undefined)).toHaveLength(0);\n  });\n\n  it(\"cheerio('') should be empty\", () => {\n    expect(cheerio('')).toHaveLength(0);\n  });\n\n  it('cheerio(selector) with no context or root should be empty', () => {\n    expect(cheerio('.h2')).toHaveLength(0);\n    expect(cheerio('#fruits')).toHaveLength(0);\n  });\n\n  it('cheerio(node) : should override previously-loaded nodes', () => {\n    const $ = cheerio.load('<div><span></span></div>');\n    const spanNode = $('span')[0];\n    const $span = $(spanNode);\n    expect($span[0]).toBe(spanNode);\n  });\n\n  it('should be able to create html without a root or context', () => {\n    const $h2 = cheerio('<h2>');\n    expect($h2).not.toHaveLength(0);\n    expect($h2).toHaveLength(1);\n    expect($h2[0]).toHaveProperty('tagName', 'h2');\n  });\n\n  it('should be able to create complicated html', () => {\n    const $script = cheerio(\n      '<script src=\"script.js\" type=\"text/javascript\"></script>',\n    ) as Cheerio<Element>;\n    expect($script).not.toHaveLength(0);\n    expect($script).toHaveLength(1);\n    expect($script[0].attribs).toHaveProperty('src', 'script.js');\n    expect($script[0].attribs).toHaveProperty('type', 'text/javascript');\n    expect($script[0].childNodes).toHaveLength(0);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with only a context', () => {\n    const $apple = cheerio('.apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with a node as context', () => {\n    const $apple = cheerio('.apple', cheerio(fruits)[0]);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with only a root', () => {\n    const $apple = cheerio('.apple', null, fruits);\n    testAppleSelect($apple);\n  });\n\n  it('should be able to select an id', () => {\n    const $fruits = cheerio('#fruits', null, fruits);\n    expect($fruits).toHaveLength(1);\n    expect($fruits[0].attribs).toHaveProperty('id', 'fruits');\n  });\n\n  it('should be able to select a tag', () => {\n    const $ul = cheerio('ul', fruits);\n    expect($ul).toHaveLength(1);\n    expect($ul[0].tagName).toBe('ul');\n  });\n\n  it('should accept a node reference as a context', () => {\n    const $elems = cheerio('<div><span></span></div>');\n    expect(cheerio('span', $elems[0])).toHaveLength(1);\n  });\n\n  it('should accept an array of node references as a context', () => {\n    const $elems = cheerio('<div><span></span></div>');\n    expect(cheerio('span', $elems.toArray())).toHaveLength(1);\n  });\n\n  it('should select only elements inside given context (Issue #193)', () => {\n    const $ = cheerio.load(food);\n    const $fruits = $('#fruits');\n    const fruitElements = $('li', $fruits);\n\n    expect(fruitElements).toHaveLength(3);\n  });\n\n  it('should be able to select multiple tags', () => {\n    const $fruits = cheerio('li', null, fruits);\n    expect($fruits).toHaveLength(3);\n    const classes = ['apple', 'orange', 'pear'];\n    $fruits.each((idx, $fruit) => {\n      expect($fruit.attribs).toHaveProperty('class', classes[idx]);\n    });\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to do: cheerio(\"#fruits .apple\")', () => {\n    const $apple = cheerio('#fruits .apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to do: cheerio(\"li.apple\")', () => {\n    const $apple = cheerio('li.apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select by attributes', () => {\n    const $apple = cheerio('li[class=apple]', fruits);\n    testAppleSelect($apple);\n  });\n\n  it('should be able to select multiple classes: cheerio(\".btn.primary\")', () => {\n    const $a = cheerio(\n      '.btn.primary',\n      '<p><a class=\"btn primary\" href=\"#\">Save</a></p>',\n    );\n    expect($a).toHaveLength(1);\n    expect($a[0].childNodes[0]).toHaveProperty('data', 'Save');\n  });\n\n  it('should not create a top-level node', () => {\n    const $elem = cheerio('* div', '<div>');\n    expect($elem).toHaveLength(0);\n  });\n\n  it('should be able to select multiple elements: cheerio(\".apple, #fruits\")', () => {\n    const $elems = cheerio('.apple, #fruits', fruits);\n    expect($elems).toHaveLength(2);\n\n    const $apple = $elems\n      .toArray()\n      .filter((elem) => elem.attribs['class'] === 'apple');\n    const $fruit = $elems\n      .toArray()\n      .find((elem) => elem.attribs['id'] === 'fruits');\n    testAppleSelect($apple);\n    expect($fruit?.attribs).toHaveProperty('id', 'fruits');\n  });\n\n  it('should select first element cheerio(:first)', () => {\n    const $elem = cheerio('li:first', fruits);\n    expect($elem.attr('class')).toBe('apple');\n\n    const $filtered = cheerio('li', fruits).filter(':even');\n    expect($filtered).toHaveLength(2);\n  });\n\n  it('should be able to select immediate children: cheerio(\"#fruits > .pear\")', () => {\n    const $food = cheerio(food);\n    cheerio('.pear', $food).append('<li class=\"pear\">Another Pear!</li>');\n    expect(cheerio('#fruits .pear', $food)).toHaveLength(2);\n    const $elem = cheerio('#fruits > .pear', $food);\n    expect($elem).toHaveLength(1);\n    expect($elem.attr('class')).toBe('pear');\n  });\n\n  it('should be able to select immediate children: cheerio(\".apple + .pear\")', () => {\n    expect(cheerio('.apple + li', fruits)).toHaveLength(1);\n    expect(cheerio('.apple + .pear', fruits)).toHaveLength(0);\n    const $elem = cheerio('.apple + .orange', fruits);\n    expect($elem).toHaveLength(1);\n    expect($elem.attr('class')).toBe('orange');\n  });\n\n  it('should be able to select immediate children: cheerio(\".apple ~ .pear\")', () => {\n    expect(cheerio('.apple ~ li', fruits)).toHaveLength(2);\n    expect(cheerio('.apple ~ .pear', fruits).attr('class')).toBe('pear');\n  });\n\n  it('should handle wildcards on attributes: cheerio(\"li[class*=r]\")', () => {\n    const $elem = cheerio('li[class*=r]', fruits);\n    expect($elem).toHaveLength(2);\n    expect($elem.eq(0).attr('class')).toBe('orange');\n    expect($elem.eq(1).attr('class')).toBe('pear');\n  });\n\n  it('should handle beginning of attr selectors: cheerio(\"li[class^=o]\")', () => {\n    const $elem = cheerio('li[class^=o]', fruits);\n    expect($elem).toHaveLength(1);\n    expect($elem.eq(0).attr('class')).toBe('orange');\n  });\n\n  it('should handle beginning of attr selectors: cheerio(\"li[class$=e]\")', () => {\n    const $elem = cheerio('li[class$=e]', fruits);\n    expect($elem).toHaveLength(2);\n    expect($elem.eq(0).attr('class')).toBe('apple');\n    expect($elem.eq(1).attr('class')).toBe('orange');\n  });\n\n  it('(extended Array) should not interfere with prototype methods (issue #119)', () => {\n    const extended: any = [];\n    extended.find =\n      extended.children =\n      extended.each =\n        function () {\n          /* Ignore */\n        };\n    const $empty = cheerio(extended);\n\n    expect($empty.find).toBe(cheerio.prototype.find);\n    expect($empty.children).toBe(cheerio.prototype.children);\n    expect($empty.each).toBe(cheerio.prototype.each);\n  });\n\n  it('cheerio.html(null) should return a \"\" string', () => {\n    expect(cheerio.html(null as never)).toBe('');\n  });\n\n  it('should set html(number) as a string', () => {\n    const $elem = cheerio('<div>');\n    $elem.html(123 as never);\n    expect(typeof $elem.text()).toBe('string');\n  });\n\n  it('should set text(number) as a string', () => {\n    const $elem = cheerio('<div>');\n    $elem.text(123 as never);\n    expect(typeof $elem.text()).toBe('string');\n  });\n\n  describe('.load', () => {\n    it('should generate selections as proper instances', () => {\n      const $ = cheerio.load(fruits);\n\n      expect($('.apple')).toBeInstanceOf($);\n    });\n\n    // Issue #1092\n    it('should handle a character `)` in `:contains` selector', () => {\n      const result = cheerio.load('<p>)aaa</p>')(\":contains('\\\\)aaa')\");\n      expect(result).toHaveLength(3);\n      expect(result.first().prop('tagName')).toBe('HTML');\n      expect(result.eq(1).prop('tagName')).toBe('BODY');\n      expect(result.last().prop('tagName')).toBe('P');\n    });\n\n    it('should be able to filter down using the context', () => {\n      const $ = cheerio.load(fruits);\n      const apple = $('.apple', 'ul');\n      const lis = $('li', 'ul');\n\n      expect(apple).toHaveLength(1);\n      expect(lis).toHaveLength(3);\n    });\n\n    it('should preserve root content', () => {\n      const $ = cheerio.load(fruits);\n      // Root should not be overwritten\n      const el = $('<div></div>');\n      expect(Object.is(el, el._root)).toBe(false);\n      // Query has to have results\n      expect($('li', 'ul')).toHaveLength(3);\n    });\n\n    it('should allow loading a pre-parsed DOM', () => {\n      const dom = parseDOM(food);\n      const $ = cheerio.load(dom);\n\n      expect($('ul')).toHaveLength(3);\n    });\n\n    it('should allow loading a single element', () => {\n      const el = parseDOM(food)[0];\n      const $ = cheerio.load(el);\n\n      expect($('ul')).toHaveLength(3);\n    });\n\n    it('should render xml in html() when options.xml = true', () => {\n      const str = '<MixedCaseTag UPPERCASEATTRIBUTE=\"\"></MixedCaseTag>';\n      const expected = '<MixedCaseTag UPPERCASEATTRIBUTE=\"\"/>';\n      const $ = cheerio.load(str, { xml: true });\n\n      expect($('MixedCaseTag').get(0)).toHaveProperty(\n        'tagName',\n        'MixedCaseTag',\n      );\n      expect($.html()).toBe(expected);\n    });\n\n    it('should render xml in html() when options.xml = true passed to html()', () => {\n      const str = '<MixedCaseTag UPPERCASEATTRIBUTE=\"\"></MixedCaseTag>';\n      // Since parsing done without xml flag, all tags converted to lowercase\n      const expectedXml =\n        '<html><head/><body><mixedcasetag uppercaseattribute=\"\"/></body></html>';\n      const expectedNoXml =\n        '<html><head></head><body><mixedcasetag uppercaseattribute=\"\"></mixedcasetag></body></html>';\n      const $ = cheerio.load(str);\n\n      expect($('MixedCaseTag').get(0)).toHaveProperty(\n        'tagName',\n        'mixedcasetag',\n      );\n      expect($.html()).toBe(expectedNoXml);\n      expect($.html({ xml: true })).toBe(expectedXml);\n    });\n\n    it('should respect options on the element level', () => {\n      const str =\n        '<!doctype html><html><head><title>Some test</title></head><body><footer><p>Copyright &copy; 2003-2014</p></footer></body></html>';\n      const expectedHtml = '<p>Copyright &copy; 2003-2014</p>';\n      const expectedXml = '<p>Copyright \u00a9 2003-2014</p>';\n      const domNotEncoded = cheerio.load(str, {\n        xml: { decodeEntities: false },\n      });\n      const domEncoded = cheerio.load(str);\n\n      expect(domNotEncoded('footer').html()).toBe(expectedHtml);\n      expect(domEncoded('footer').html()).toBe(expectedXml);\n    });\n\n    it('should use htmlparser2 if xml option is used', () => {\n      const str = '<div></div>';\n      const dom = cheerio.load(str, null, false);\n      expect(dom.html()).toBe(str);\n    });\n\n    it('should return a fully-qualified Function', () => {\n      const $ = cheerio.load('<div>');\n\n      expect($).toBeInstanceOf(Function);\n    });\n\n    describe('prototype extensions', () => {\n      it('should honor extensions defined on `prototype` property', () => {\n        const $ = cheerio.load('<div>');\n\n        $.prototype.myPlugin = function (...args: unknown[]) {\n          return {\n            context: this,\n            args,\n          };\n        };\n\n        const $div = $('div');\n\n        expect(typeof $div.myPlugin).toBe('function');\n        expect($div.myPlugin().context).toBe($div);\n        expect($div.myPlugin(1, 2, 3).args).toStrictEqual([1, 2, 3]);\n      });\n\n      it('should honor extensions defined on `fn` property', () => {\n        const $ = cheerio.load('<div>');\n        $.fn.myPlugin = function (...args: unknown[]) {\n          return {\n            context: this,\n            args,\n          };\n        };\n\n        const $div = $('div');\n\n        expect(typeof $div.myPlugin).toBe('function');\n        expect($div.myPlugin().context).toBe($div);\n        expect($div.myPlugin(1, 2, 3).args).toStrictEqual([1, 2, 3]);\n      });\n\n      it('should isolate extensions between loaded functions', () => {\n        const $a = cheerio.load('<div>');\n        const $b = cheerio.load('<div>');\n\n        $a.prototype.foo = function () {\n          /* Ignore */\n        };\n\n        expect($b('div').foo).toBeUndefined();\n      });\n    });\n  });\n\n  describe('parse5 options', () => {\n    // Should parse noscript tags only with false option value\n    test('{scriptingEnabled: ???}', () => {\n      // [default] `scriptingEnabled: true` - tag contains one text element\n      const withScripts = cheerio.load(noscript)('noscript');\n      expect(withScripts).toHaveLength(1);\n      expect(withScripts[0].children).toHaveLength(1);\n      expect(withScripts[0].children[0].type).toBe('text');\n\n      // `scriptingEnabled: false` - content of noscript will parsed\n      const noScripts = cheerio.load(noscript, { scriptingEnabled: false })(\n        'noscript',\n      );\n      expect(noScripts).toHaveLength(1);\n      expect(noScripts[0].children).toHaveLength(2);\n      expect(noScripts[0].children[0].type).toBe('comment');\n      expect(noScripts[0].children[1].type).toBe('tag');\n      expect(noScripts[0].children[1]).toHaveProperty('name', 'a');\n\n      // `scriptingEnabled: ???` - should acts as true\n      for (const val of [undefined, null, 0, '']) {\n        const options = { scriptingEnabled: val as never };\n        const result = cheerio.load(noscript, options)('noscript');\n        expect(result).toHaveLength(1);\n        expect(result[0].children).toHaveLength(1);\n        expect(result[0].children[0].type).toBe('text');\n      }\n    });\n\n    // Should contain location data only with truthful option value\n    test('{sourceCodeLocationInfo: ???}', () => {\n      // Location data should not be present\n      for (const val of [undefined, null, 0, false, '']) {\n        const options = { sourceCodeLocationInfo: val as never };\n        const result = cheerio.load(noscript, options)('noscript');\n        expect(result).toHaveLength(1);\n        expect(result[0]).not.toHaveProperty('sourceCodeLocation');\n      }\n\n      // Location data should be present\n      for (const val of [true, 1, 'test']) {\n        const options = { sourceCodeLocationInfo: val as never };\n        const result = cheerio.load(noscript, options)('noscript');\n        expect(result).toHaveLength(1);\n        expect(result[0]).toHaveProperty('sourceCodeLocation');\n        expect(typeof result[0].sourceCodeLocation).toBe('object');\n      }\n    });\n  });\n});\n\n// Path: src/batteries.spec.ts\n/* eslint-disable jest/no-done-callback */\nimport * as cheerio from './batteries.js';\nimport { Writable } from 'node:stream';\nimport { createServer, type Server } from 'node:http';\n\nfunction noop() {\n  // Ignore\n}\n\nconst TEST_HTML = '<h1>Hello World</h1>';\nconst TEST_HTML_UTF16 = Buffer.from(TEST_HTML, 'utf16le');\nconst TEST_HTML_UTF16_BOM = Buffer.from([\n  // UTF16-LE BOM\n  0xff,\n  0xfe,\n  ...Array.from(TEST_HTML_UTF16),\n]);\n\ndescribe('loadBuffer', () => {\n  it('should parse UTF-8 HTML', () => {\n    const $ = cheerio.loadBuffer(Buffer.from(TEST_HTML));\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n\n  it('should parse UTF-16 HTML', () => {\n    const $ = cheerio.loadBuffer(TEST_HTML_UTF16_BOM);\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n});\n\ndescribe('stringStream', () => {\n  it('should use parse5 by default', (cb) => {\n    const stream = cheerio.stringStream({}, (err, $) => {\n      expect(err).toBeUndefined();\n\n      expect($.html()).toBe(\n        `<html><head></head><body>${TEST_HTML}</body></html>`,\n      );\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML);\n  });\n\n  it('should error from parse5 on buffer', () => {\n    const stream = cheerio.stringStream({}, noop);\n    expect(stream).toBeInstanceOf(Writable);\n\n    expect(() => stream.write(Buffer.from(TEST_HTML))).toThrow(\n      'Parser can work only with string streams.',\n    );\n  });\n\n  it('should use htmlparser2 for XML', (cb) => {\n    const stream = cheerio.stringStream({ xml: true }, (err, $) => {\n      expect(err).toBeNull();\n\n      expect($.html()).toBe(TEST_HTML);\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML);\n  });\n});\n\ndescribe('decodeStream', () => {\n  it('should use parse5 by default', (cb) => {\n    const stream = cheerio.decodeStream({}, (err, $) => {\n      expect(err).toBeUndefined();\n\n      expect($.html()).toBe(\n        `<html><head></head><body>${TEST_HTML}</body></html>`,\n      );\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML_UTF16_BOM);\n  });\n\n  it('should use htmlparser2 for XML', (cb) => {\n    const stream = cheerio.decodeStream({ xmlMode: true }, (err, $) => {\n      expect(err).toBeNull();\n\n      expect($.html()).toBe(TEST_HTML);\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML_UTF16_BOM);\n  });\n});\n\ndescribe('fromURL', () => {\n  let server: Server | undefined;\n\n  function createTestServer(\n    contentType: string,\n    body: string | Buffer,\n  ): Promise<number> {\n    return new Promise((resolve, reject) => {\n      server = createServer((_req, res) => {\n        res.writeHead(200, { 'Content-Type': contentType });\n        res.end(body);\n      });\n\n      server.listen(0, () => {\n        const address = server?.address();\n\n        if (typeof address === 'string' || address == null) {\n          reject(new Error('Failed to get port'));\n        } else {\n          resolve(address.port);\n        }\n      });\n    });\n  }\n\n  afterEach((cb) => {\n    if (server) {\n      server.close(cb);\n      server = undefined;\n    } else {\n      cb();\n    }\n  });\n\n  it('should fetch UTF-8 HTML', async () => {\n    const port = await createTestServer('text/html', TEST_HTML);\n    const $ = await cheerio.fromURL(`http://localhost:${port}`);\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n\n  it('should fetch UTF-16 HTML', async () => {\n    const port = await createTestServer(\n      'text/html; charset=utf-16le',\n      TEST_HTML_UTF16,\n    );\n    const $ = await cheerio.fromURL(`http://localhost:${port}`);\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n\n  it('should parse XML based on Content-Type', async () => {\n    const port = await createTestServer('text/xml', TEST_HTML);\n    const $ = await cheerio.fromURL(`http://localhost:${port}`);\n\n    expect($.html()).toBe(TEST_HTML);\n  });\n});\n\n// Path: src/slim.ts\n/**\n * @file Alternative entry point for Cheerio that always uses htmlparser2. This\n *   way, parse5 won't be loaded, saving some memory.\n */\n\nexport type {\n  Cheerio,\n  CheerioAPI,\n  CheerioOptions,\n  HTMLParser2Options,\n  Node,\n  AnyNode,\n  ParentNode,\n  Element,\n  Document,\n} from './index.js';\n\n/**\n * Types used in signatures of Cheerio methods.\n *\n * @category Cheerio\n */\nexport * from './types.js';\n\nimport { getLoad } from './load.js';\nimport { getParse } from './parse.js';\nimport render from 'dom-serializer';\nimport { parseDocument } from 'htmlparser2';\n\n/**\n * Create a querying function, bound to a document created from the provided\n * markup.\n *\n * @param content - Markup to be loaded.\n * @param options - Options for the created instance.\n * @param isDocument - Always `false` here, as we are always using\n *   `htmlparser2`.\n * @returns The loaded document.\n * @see {@link https://cheerio.js.org#loading} for additional usage information.\n */\nexport const load = getLoad(getParse(parseDocument), render);\n\n// Path: src/index.spec.ts\nimport * as cheerio from './index.js';\nimport * as statics from './static.js';\n\ndescribe('static method re-exports', () => {\n  it('should export expected static methods', () => {\n    for (const key of Object.keys(statics) as (keyof typeof statics)[]) {\n      if (key === 'extract') continue;\n      expect(typeof cheerio[key]).toBe(typeof statics[key]);\n    }\n  });\n\n  it('should have a functional `html` that is bound to the default instance', () => {\n    expect(cheerio.html(cheerio.default('<div>test div</div>'))).toBe(\n      '<div>test div</div>',\n    );\n  });\n\n  it('should have a functional `xml` that is bound to the default instance', () => {\n    expect(cheerio.xml(cheerio.default('<div>test div</div>'))).toBe(\n      '<div>test div</div>',\n    );\n  });\n\n  it('should have a functional `text` that is bound to the default instance', () => {\n    expect(cheerio.text(cheerio.default('<div>test div</div>'))).toBe(\n      'test div',\n    );\n  });\n});\n\n// Path: src/parse.spec.ts\nimport type { Document, Element } from 'domhandler';\nimport { getParse } from './parse.js';\n\nimport { parseDocument as parseWithHtmlparser2 } from 'htmlparser2';\nimport { parseWithParse5 } from './parsers/parse5-adapter.js';\n\nconst defaultOpts = { _useHtmlParser2: false };\n\nconst parse = getParse((content, options, isDocument, context) =>\n  options._useHtmlParser2\n    ? parseWithHtmlparser2(content, options)\n    : parseWithParse5(content, options, isDocument, context),\n);\n\n// Tags\nconst basic = '<html></html>';\nconst siblings = '<h2></h2><p></p>';\n\n// Single Tags\nconst single = '<br/>';\nconst singleWrong = '<br>';\n\n// Children\nconst children = '<html><br/></html>';\nconst li = '<li class=\"durian\">Durian</li>';\n\n// Attributes\nconst attributes = '<img src=\"hello.png\" alt=\"man waving\">';\nconst noValueAttribute = '<textarea disabled></textarea>';\n\n// Comments\nconst comment = '<!-- sexy -->';\nconst conditional =\n  '<!--[if IE 8]><html class=\"no-js ie8\" lang=\"en\"><![endif]-->';\n\n// Text\nconst text = 'lorem ipsum';\n\n// Script\nconst script = '<script type=\"text/javascript\">alert(\"hi world!\");</script>';\nconst scriptEmpty = '<script></script>';\n\n// Style\nconst style = '<style type=\"text/css\"> h2 { color:blue; } </style>';\nconst styleEmpty = '<style></style>';\n\n// Directives\nconst directive = '<!doctype html>';\n\nfunction rootTest(root: Document) {\n  expect(root).toHaveProperty('type', 'root');\n\n  expect(root.nextSibling).toBe(null);\n  expect(root.previousSibling).toBe(null);\n  expect(root.parentNode).toBe(null);\n\n  const child = root.childNodes[0];\n  expect(child.parentNode).toBe(root);\n}\n\ndescribe('parse', () => {\n  describe('evaluate', () => {\n    it(`should parse basic empty tags: ${basic}`, () => {\n      const [tag] = parse(basic, defaultOpts, true, null).children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('html');\n      expect(tag.childNodes).toHaveLength(2);\n    });\n\n    it(`should handle sibling tags: ${siblings}`, () => {\n      const dom = parse(siblings, defaultOpts, false, null)\n        .children as Element[];\n      const [h2, p] = dom;\n\n      expect(dom).toHaveLength(2);\n      expect(h2.tagName).toBe('h2');\n      expect(p.tagName).toBe('p');\n    });\n\n    it(`should handle single tags: ${single}`, () => {\n      const [tag] = parse(single, defaultOpts, false, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('br');\n      expect(tag.childNodes).toHaveLength(0);\n    });\n\n    it(`should handle malformatted single tags: ${singleWrong}`, () => {\n      const [tag] = parse(singleWrong, defaultOpts, false, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('br');\n      expect(tag.childNodes).toHaveLength(0);\n    });\n\n    it(`should handle tags with children: ${children}`, () => {\n      const [tag] = parse(children, defaultOpts, true, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('html');\n      expect(tag.childNodes).toBeTruthy();\n      expect(tag.childNodes[1]).toHaveProperty('tagName', 'body');\n      expect((tag.childNodes[1] as Element).childNodes).toHaveLength(1);\n    });\n\n    it(`should handle tags with children: ${li}`, () => {\n      const [tag] = parse(li, defaultOpts, false, null).children as Element[];\n      expect(tag.childNodes).toHaveLength(1);\n      expect(tag.childNodes[0]).toHaveProperty('data', 'Durian');\n    });\n\n    it(`should handle tags with attributes: ${attributes}`, () => {\n      const attrs = parse(attributes, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(attrs.attribs).toBeTruthy();\n      expect(attrs.attribs).toHaveProperty('src', 'hello.png');\n      expect(attrs.attribs).toHaveProperty('alt', 'man waving');\n    });\n\n    it(`should handle value-less attributes: ${noValueAttribute}`, () => {\n      const attrs = parse(noValueAttribute, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(attrs.attribs).toBeTruthy();\n      expect(attrs.attribs).toHaveProperty('disabled', '');\n    });\n\n    it(`should handle comments: ${comment}`, () => {\n      const elem = parse(comment, defaultOpts, false, null).children[0];\n      expect(elem.type).toBe('comment');\n      expect(elem).toHaveProperty('data', ' sexy ');\n    });\n\n    it(`should handle conditional comments: ${conditional}`, () => {\n      const elem = parse(conditional, defaultOpts, false, null).children[0];\n      expect(elem.type).toBe('comment');\n      expect(elem).toHaveProperty(\n        'data',\n        conditional.replace('<!--', '').replace('-->', ''),\n      );\n    });\n\n    it(`should handle text: ${text}`, () => {\n      const text_ = parse(text, defaultOpts, false, null).children[0];\n      expect(text_.type).toBe('text');\n      expect(text_).toHaveProperty('data', 'lorem ipsum');\n    });\n\n    it(`should handle script tags: ${script}`, () => {\n      const script_ = parse(script, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(script_.type).toBe('script');\n      expect(script_.tagName).toBe('script');\n      expect(script_.attribs).toHaveProperty('type', 'text/javascript');\n      expect(script_.childNodes).toHaveLength(1);\n      expect(script_.childNodes[0].type).toBe('text');\n      expect(script_.childNodes[0]).toHaveProperty(\n        'data',\n        'alert(\"hi world!\");',\n      );\n    });\n\n    it(`should handle style tags: ${style}`, () => {\n      const style_ = parse(style, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(style_.type).toBe('style');\n      expect(style_.tagName).toBe('style');\n      expect(style_.attribs).toHaveProperty('type', 'text/css');\n      expect(style_.childNodes).toHaveLength(1);\n      expect(style_.childNodes[0].type).toBe('text');\n      expect(style_.childNodes[0]).toHaveProperty(\n        'data',\n        ' h2 { color:blue; } ',\n      );\n    });\n\n    it(`should handle directives: ${directive}`, () => {\n      const elem = parse(directive, defaultOpts, true, null).children[0];\n      expect(elem.type).toBe('directive');\n      expect(elem).toHaveProperty('data', '!DOCTYPE html');\n      expect(elem).toHaveProperty('name', '!doctype');\n    });\n  });\n\n  describe('.parse', () => {\n    // Root test utility\n\n    it(`should add root to: ${basic}`, () => {\n      const root = parse(basic, defaultOpts, true, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'html');\n    });\n\n    it(`should add root to: ${siblings}`, () => {\n      const root = parse(siblings, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(2);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'h2');\n      expect(root.childNodes[1]).toHaveProperty('tagName', 'p');\n      expect(root.childNodes[1].parent).toBe(root);\n    });\n\n    it(`should add root to: ${comment}`, () => {\n      const root = parse(comment, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('comment');\n    });\n\n    it(`should add root to: ${text}`, () => {\n      const root = parse(text, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('text');\n    });\n\n    it(`should add root to: ${scriptEmpty}`, () => {\n      const root = parse(scriptEmpty, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('script');\n    });\n\n    it(`should add root to: ${styleEmpty}`, () => {\n      const root = parse(styleEmpty, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('style');\n    });\n\n    it(`should add root to: ${directive}`, () => {\n      const root = parse(directive, defaultOpts, true, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(2);\n      expect(root.childNodes[0].type).toBe('directive');\n    });\n\n    it('should simply return root', () => {\n      const oldroot = parse(basic, defaultOpts, true, null);\n      const root = parse(oldroot, defaultOpts, true, null);\n      expect(root).toBe(oldroot);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'html');\n    });\n\n    it('should expose the DOM level 1 API', () => {\n      const root = parse(\n        '<div><a></a><span></span><p></p></div>',\n        defaultOpts,\n        false,\n        null,\n      ).childNodes[0] as Element;\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes).toHaveLength(3);\n\n      expect(root.tagName).toBe('div');\n      expect(root.firstChild).toBe(childNodes[0]);\n      expect(root.lastChild).toBe(childNodes[2]);\n\n      expect(childNodes[0].tagName).toBe('a');\n      expect(childNodes[0].previousSibling).toBe(null);\n      expect(childNodes[0].nextSibling).toBe(childNodes[1]);\n      expect(childNodes[0].parentNode).toBe(root);\n      expect((childNodes[0] as Element).childNodes).toHaveLength(0);\n      expect(childNodes[0].firstChild).toBe(null);\n      expect(childNodes[0].lastChild).toBe(null);\n\n      expect(childNodes[1].tagName).toBe('span');\n      expect(childNodes[1].previousSibling).toBe(childNodes[0]);\n      expect(childNodes[1].nextSibling).toBe(childNodes[2]);\n      expect(childNodes[1].parentNode).toBe(root);\n      expect(childNodes[1].childNodes).toHaveLength(0);\n      expect(childNodes[1].firstChild).toBe(null);\n      expect(childNodes[1].lastChild).toBe(null);\n\n      expect(childNodes[2].tagName).toBe('p');\n      expect(childNodes[2].previousSibling).toBe(childNodes[1]);\n      expect(childNodes[2].nextSibling).toBe(null);\n      expect(childNodes[2].parentNode).toBe(root);\n      expect(childNodes[2].childNodes).toHaveLength(0);\n      expect(childNodes[2].firstChild).toBe(null);\n      expect(childNodes[2].lastChild).toBe(null);\n    });\n\n    it('Should parse less than or equal sign sign', () => {\n      const root = parse('<i>A</i><=<i>B</i>', defaultOpts, false, null);\n      const { childNodes } = root;\n\n      expect(childNodes[0]).toHaveProperty('tagName', 'i');\n      expect((childNodes[0] as Element).childNodes[0]).toHaveProperty(\n        'data',\n        'A',\n      );\n      expect(childNodes[1]).toHaveProperty('data', '<=');\n      expect(childNodes[2]).toHaveProperty('tagName', 'i');\n      expect((childNodes[2] as Element).childNodes[0]).toHaveProperty(\n        'data',\n        'B',\n      );\n    });\n\n    it('Should ignore unclosed CDATA', () => {\n      const root = parse(\n        '<a></a><script>foo //<![CDATA[ bar</script><b></b>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('a');\n      expect(childNodes[1].tagName).toBe('script');\n      expect(childNodes[1].childNodes[0]).toHaveProperty(\n        'data',\n        'foo //<![CDATA[ bar',\n      );\n      expect(childNodes[2].tagName).toBe('b');\n    });\n\n    it('Should add <head> to documents', () => {\n      const root = parse('<html></html>', defaultOpts, true, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('html');\n      expect(childNodes[0].childNodes[0]).toHaveProperty('tagName', 'head');\n    });\n\n    it('Should implicitly create <tr> around <td>', () => {\n      const root = parse(\n        '<table><td>bar</td></tr></table>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('table');\n      expect(childNodes[0].childNodes.length).toBe(1);\n      expect(childNodes[0].childNodes[0]).toHaveProperty('tagName', 'tbody');\n      expect((childNodes[0] as any).childNodes[0].childNodes[0]).toHaveProperty(\n        'tagName',\n        'tr',\n      );\n      expect(\n        (childNodes[0] as any).childNodes[0].childNodes[0].childNodes[0]\n          .tagName,\n      ).toBe('td');\n      expect(\n        (childNodes[0] as any).childNodes[0].childNodes[0].childNodes[0]\n          .childNodes[0].data,\n      ).toBe('bar');\n    });\n\n    it('Should parse custom tag <line>', () => {\n      const root = parse('<line>test</line>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(1);\n      expect(childNodes[0].tagName).toBe('line');\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', 'test');\n    });\n\n    it('Should properly parse misnested table tags', () => {\n      const root = parse(\n        '<tr><td>i1</td></tr><tr><td>i2</td></td></tr><tr><td>i3</td></td></tr>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(3);\n\n      for (let i = 0; i < childNodes.length; i++) {\n        const child = childNodes[i];\n        expect(child.tagName).toBe('tr');\n        expect(child.childNodes[0]).toHaveProperty('tagName', 'td');\n        expect((child.childNodes[0] as Element).childNodes[0]).toHaveProperty(\n          'data',\n          `i${i + 1}`,\n        );\n      }\n    });\n\n    it('Should correctly parse data url attributes', () => {\n      const html =\n        '<div style=\\'font-family:\"butcherman-caps\"; src:url(data:font/opentype;base64,AAEA...);\\'></div>';\n      const expectedAttr =\n        'font-family:\"butcherman-caps\"; src:url(data:font/opentype;base64,AAEA...);';\n      const root = parse(html, defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].attribs).toHaveProperty('style', expectedAttr);\n    });\n\n    it('Should treat <xmp> tag content as text', () => {\n      const root = parse('<xmp><h2></xmp>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', '<h2>');\n    });\n\n    it('Should correctly parse malformed numbered entities', () => {\n      const root = parse('<p>z&#</p>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', 'z&#');\n    });\n\n    it('Should correctly parse mismatched headings', () => {\n      const root = parse('<h2>Test</h3><div></div>', defaultOpts, false, null);\n      const { childNodes } = root;\n\n      expect(childNodes.length).toBe(2);\n      expect(childNodes[0]).toHaveProperty('tagName', 'h2');\n      expect(childNodes[1]).toHaveProperty('tagName', 'div');\n    });\n\n    it('Should correctly parse tricky <pre> content', () => {\n      const root = parse(\n        '<pre>\\nA <- factor(A, levels = c(\"c\",\"a\",\"b\"))\\n</pre>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(1);\n      expect(childNodes[0].tagName).toBe('pre');\n      expect(childNodes[0].childNodes[0]).toHaveProperty(\n        'data',\n        'A <- factor(A, levels = c(\"c\",\"a\",\"b\"))\\n',\n      );\n    });\n\n    it('should pass the options for including the location info to parse5', () => {\n      const root = parse(\n        '<p>Hello</p>',\n        { ...defaultOpts, sourceCodeLocationInfo: true },\n        false,\n        null,\n      );\n      const location = root.children[0].sourceCodeLocation;\n\n      expect(typeof location).toBe('object');\n      expect(location?.endOffset).toBe(12);\n    });\n  });\n});\n\n// Path: src/__tests__/xml.spec.ts\nimport cheerio from '../index.js';\nimport type { CheerioOptions } from '../options.js';\n\n...\n", "needle_token_start": 3973, "needle_token_end": 4103, "code_context_ntokens": 16124, "output": ["```typescript\nfunction testAppleSelect($apple: ArrayLike<Element>) {\n  expect($apple).toHaveLength(1);\n  const apple = $apple[0];\n  expect(apple.parentNode).toHaveProperty('tagName', 'ul');\n  expect(apple.prev).toBe(null);\n  expect((apple.next as Element).attribs).toHaveProperty('class', 'orange');\n  expect(apple.childNodes).toHaveLength(1);\n  expect(apple.childNodes[0]).toHaveProperty('data', 'Apple');\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "toggleClass", "language": "typescript", "path": "src/api/attributes.ts", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to add or remove specified classes from elements based on their current state or a provided boolean state. It can dynamically alter the class attributes of elements, allowing for conditional styling changes.\n2. **Input**: Accepts either a string of space-separated class names, a function that determines class names based on element-specific logic, and an optional boolean that explicitly specifies whether to add or remove the classes.\n3. **Output**: Returns the instance it was called on, allowing for method chaining.\n4. **Procedure**: Iterates over each element in the collection. For each element, it splits the class attribute into an array and processes each class name from the input. Depending on the presence of the class in the element and the optional boolean state, it either adds or removes the class. If a function is provided as input, it calls the function for each element to determine the class changes dynamically.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n    switch (name) {\n      case 'style': {\n        const property = this.css() as StyleProp;\n        const keys = Object.keys(property);\n        for (let i = 0; i < keys.length; i++) {\n          property[i] = keys[i];\n        }\n\n        property.length = keys.length;\n\n        return property;\n      }\n      case 'tagName':\n      case 'nodeName': {\n        return el.name.toUpperCase();\n      }\n\n      case 'href':\n      case 'src': {\n        const prop = el.attribs?.[name];\n\n        /* eslint-disable n/no-unsupported-features/node-builtins */\n        if (\n          typeof URL !== 'undefined' &&\n          ((name === 'href' && (el.tagName === 'a' || el.name === 'link')) ||\n            (name === 'src' &&\n              (el.tagName === 'img' ||\n                el.tagName === 'iframe' ||\n                el.tagName === 'audio' ||\n                el.tagName === 'video' ||\n                el.tagName === 'source'))) &&\n          prop !== undefined &&\n          this.options.baseURI\n        ) {\n          return new URL(prop, this.options.baseURI).href;\n        }\n        /* eslint-enable n/no-unsupported-features/node-builtins */\n\n        return prop;\n      }\n\n      case 'innerText': {\n        return innerText(el);\n      }\n\n      case 'textContent': {\n        return textContent(el);\n      }\n\n      case 'outerHTML': {\n        return this.clone().wrap('<container />').parent().html();\n      }\n\n      case 'innerHTML': {\n        return this.html();\n      }\n\n      default: {\n        return getProp(el, name, this.options.xmlMode);\n      }\n    }\n  }\n\n  if (typeof name === 'object' || value !== undefined) {\n    if (typeof value === 'function') {\n      if (typeof name === 'object') {\n        throw new TypeError('Bad combination of arguments.');\n      }\n      return domEach(this, (el, i) => {\n        if (isTag(el)) {\n          setProp(\n            el,\n            name,\n            value.call(el, i, getProp(el, name, this.options.xmlMode)),\n            this.options.xmlMode,\n          );\n        }\n      });\n    }\n\n    return domEach(this, (el) => {\n      if (!isTag(el)) return;\n\n      if (typeof name === 'object') {\n        for (const key of Object.keys(name)) {\n          const val = name[key];\n          setProp(el, key, val, this.options.xmlMode);\n        }\n      } else {\n        setProp(el, name, value, this.options.xmlMode);\n      }\n    });\n  }\n\n  return undefined;\n}\n\n/**\n * An element with a data attribute.\n *\n * @private\n */\ninterface DataElement extends Element {\n  /** The data attribute. */\n  data?: Record<string, unknown>;\n}\n\n/**\n * Sets the value of a data attribute.\n *\n * @private\n * @param elem - The element to set the data attribute on.\n * @param name - The data attribute's name.\n * @param value - The data attribute's value.\n */\nfunction setData(\n  elem: DataElement,\n  name: string | Record<string, unknown>,\n  value?: unknown,\n) {\n  elem.data ??= {};\n\n  if (typeof name === 'object') Object.assign(elem.data, name);\n  else if (typeof name === 'string' && value !== undefined) {\n    elem.data[name] = value;\n  }\n}\n\n/**\n * Read _all_ HTML5 `data-*` attributes from the equivalent HTML5 `data-*`\n * attribute, and cache the value in the node's internal data store.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the data attribute of.\n * @returns A map with all of the data attributes.\n */\nfunction readAllData(el: DataElement): unknown {\n  for (const domName of Object.keys(el.attribs)) {\n    if (!domName.startsWith(dataAttrPrefix)) {\n      continue;\n    }\n\n    const jsName = camelCase(domName.slice(dataAttrPrefix.length));\n\n    if (!hasOwn.call(el.data, jsName)) {\n      el.data![jsName] = parseDataValue(el.attribs[domName]);\n    }\n  }\n\n  return el.data;\n}\n\n/**\n * Read the specified attribute from the equivalent HTML5 `data-*` attribute,\n * and (if present) cache the value in the node's internal data store.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the data attribute of.\n * @param name - Name of the data attribute.\n * @returns The data attribute's value.\n */\nfunction readData(el: DataElement, name: string): unknown {\n  const domName = dataAttrPrefix + cssCase(name);\n  const data = el.data!;\n\n  if (hasOwn.call(data, name)) {\n    return data[name];\n  }\n\n  if (hasOwn.call(el.attribs, domName)) {\n    return (data[name] = parseDataValue(el.attribs[domName]));\n  }\n\n  return undefined;\n}\n\n/**\n * Coerce string data-* attributes to their corresponding JavaScript primitives.\n *\n * @private\n * @category Attributes\n * @param value - The value to parse.\n * @returns The parsed value.\n */\nfunction parseDataValue(value: string): unknown {\n  if (value === 'null') return null;\n  if (value === 'true') return true;\n  if (value === 'false') return false;\n  const num = Number(value);\n  if (value === String(num)) return num;\n  if (rbrace.test(value)) {\n    try {\n      return JSON.parse(value);\n    } catch {\n      /* Ignore */\n    }\n  }\n  return value;\n}\n\n/**\n * Method for getting data attributes, for only the first element in the matched\n * set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('<div data-apple-color=\"red\"></div>').data('apple-color');\n * //=> 'red'\n * ```\n *\n * @param name - Name of the data attribute.\n * @returns The data attribute's value, or `undefined` if the attribute does not\n *   exist.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): unknown | undefined;\n/**\n * Method for getting all of an element's data attributes, for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('<div data-apple-color=\"red\"></div>').data();\n * //=> { appleColor: 'red' }\n * ```\n *\n * @returns A map with all of the data attributes.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n): Record<string, unknown>;\n/**\n * Method for setting data attributes, for only the first element in the matched\n * set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * const apple = $('.apple').data('kind', 'mac');\n *\n * apple.data('kind');\n * //=> 'mac'\n * ```\n *\n * @param name - Name of the data attribute.\n * @param value - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value: unknown,\n): Cheerio<T>;\n/**\n * Method for setting multiple data attributes at once, for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * const apple = $('.apple').data({ kind: 'mac' });\n *\n * apple.data('kind');\n * //=> 'mac'\n * ```\n *\n * @param values - Map of names to values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  values: Record<string, unknown>,\n): Cheerio<T>;\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name?: string | Record<string, unknown>,\n  value?: unknown,\n): unknown | Cheerio<T> | undefined | Record<string, unknown> {\n  const elem = this[0];\n\n  if (!elem || !isTag(elem)) return;\n\n  const dataEl: DataElement = elem;\n  dataEl.data ??= {};\n\n  // Return the entire data object if no data specified\n  if (name == null) {\n    return readAllData(dataEl);\n  }\n\n  // Set the value (with attr map support)\n  if (typeof name === 'object' || value !== undefined) {\n    domEach(this, (el) => {\n      if (isTag(el)) {\n        if (typeof name === 'object') setData(el, name);\n        else setData(el, name, value as unknown);\n      }\n    });\n    return this;\n  }\n\n  return readData(dataEl, name);\n}\n\n/**\n * Method for getting the value of input, select, and textarea. Note: Support\n * for `map`, and `function` has not been added yet.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"text\"]').val();\n * //=> input_text\n * ```\n *\n * @returns The value.\n * @see {@link https://api.jquery.com/val/}\n */\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n): string | undefined | string[];\n/**\n * Method for setting the value of input, select, and textarea. Note: Support\n * for `map`, and `function` has not been added yet.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"text\"]').val('test').html();\n * //=> <input type=\"text\" value=\"test\"/>\n * ```\n *\n * @param value - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/val/}\n */\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n  value: string | string[],\n): Cheerio<T>;\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n  value?: string | string[],\n): string | string[] | Cheerio<T> | undefined {\n  const querying = arguments.length === 0;\n  const element = this[0];\n\n  if (!element || !isTag(element)) return querying ? undefined : this;\n\n  switch (element.name) {\n    case 'textarea': {\n      return this.text(value as string);\n    }\n    case 'select': {\n      const option = this.find('option:selected');\n      if (!querying) {\n        if (this.attr('multiple') == null && typeof value === 'object') {\n          return this;\n        }\n\n        this.find('option').removeAttr('selected');\n\n        const values = typeof value === 'object' ? value : [value];\n        for (const val of values) {\n          this.find(`option[value=\"${val}\"]`).attr('selected', '');\n        }\n\n        return this;\n      }\n\n      return this.attr('multiple')\n        ? option.toArray().map((el) => text(el.children))\n        : option.attr('value');\n    }\n    case 'input':\n    case 'option': {\n      return querying\n        ? this.attr('value')\n        : this.attr('value', value as string);\n    }\n  }\n\n  return undefined;\n}\n\n/**\n * Remove an attribute.\n *\n * @private\n * @param elem - Node to remove attribute from.\n * @param name - Name of the attribute to remove.\n */\nfunction removeAttribute(elem: Element, name: string) {\n  if (!elem.attribs || !hasOwn.call(elem.attribs, name)) return;\n\n  delete elem.attribs[name];\n}\n\n/**\n * Splits a space-separated list of names to individual names.\n *\n * @category Attributes\n * @param names - Names to split.\n * @returns - Split names.\n */\nfunction splitNames(names?: string): string[] {\n  return names ? names.trim().split(rspace) : [];\n}\n\n/**\n * Method for removing attributes by `name`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').removeAttr('class').html();\n * //=> <li>Pear</li>\n *\n * $('.apple').attr('id', 'favorite');\n * $('.apple').removeAttr('id class').html();\n * //=> <li>Apple</li>\n * ```\n *\n * @param name - Name of the attribute.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/removeAttr/}\n */\nexport function removeAttr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): Cheerio<T> {\n  const attrNames = splitNames(name);\n\n  for (const attrName of attrNames) {\n    domEach(this, (elem) => {\n      if (isTag(elem)) removeAttribute(elem, attrName);\n    });\n  }\n\n  return this;\n}\n\n/**\n * Check to see if _any_ of the matched elements have the given `className`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').hasClass('pear');\n * //=> true\n *\n * $('apple').hasClass('fruit');\n * //=> false\n *\n * $('li').hasClass('pear');\n * //=> true\n * ```\n *\n * @param className - Name of the class.\n * @returns Indicates if an element has the given `className`.\n * @see {@link https://api.jquery.com/hasClass/}\n */\nexport function hasClass<T extends AnyNode>(\n  this: Cheerio<T>,\n  className: string,\n): boolean {\n  return this.toArray().some((elem) => {\n    const clazz = isTag(elem) && elem.attribs['class'];\n    let idx = -1;\n\n    if (clazz && className.length > 0) {\n      while ((idx = clazz.indexOf(className, idx + 1)) > -1) {\n        const end = idx + className.length;\n\n        if (\n          (idx === 0 || rspace.test(clazz[idx - 1])) &&\n          (end === clazz.length || rspace.test(clazz[end]))\n        ) {\n          return true;\n        }\n      }\n    }\n\n    return false;\n  });\n}\n\n/**\n * Adds class(es) to all of the matched elements. Also accepts a `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').addClass('fruit').html();\n * //=> <li class=\"pear fruit\">Pear</li>\n *\n * $('.apple').addClass('fruit red').html();\n * //=> <li class=\"apple fruit red\">Apple</li>\n * ```\n *\n * @param value - Name of new class.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/addClass/}\n */\nexport function addClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  value?:\n    | string\n    | ((this: Element, i: number, className: string) => string | undefined),\n): R {\n  // Support functions\n  if (typeof value === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        const className = el.attribs['class'] || '';\n        addClass.call([el], value.call(el, i, className));\n      }\n    });\n  }\n\n  // Return if no value or not a string or function\n  if (!value || typeof value !== 'string') return this;\n\n  const classNames = value.split(rspace);\n  const numElements = this.length;\n\n  for (let i = 0; i < numElements; i++) {\n    const el = this[i];\n    // If selected element isn't a tag, move on\n    if (!isTag(el)) continue;\n\n    // If we don't already have classes \u2014 always set xmlMode to false here, as it doesn't matter for classes\n    const className = getAttr(el, 'class', false);\n\n    if (className) {\n      let setClass = ` ${className} `;\n\n      // Check if class already exists\n      for (const cn of classNames) {\n        const appendClass = `${cn} `;\n        if (!setClass.includes(` ${appendClass}`)) setClass += appendClass;\n      }\n\n      setAttr(el, 'class', setClass.trim());\n    } else {\n      setAttr(el, 'class', classNames.join(' ').trim());\n    }\n  }\n\n  return this;\n}\n\n/**\n * Removes one or more space-separated classes from the selected elements. If no\n * `className` is defined, all classes will be removed. Also accepts a\n * `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').removeClass('pear').html();\n * //=> <li class=\"\">Pear</li>\n *\n * $('.apple').addClass('red').removeClass().html();\n * //=> <li class=\"\">Apple</li>\n * ```\n *\n * @param name - Name of the class. If not specified, removes all elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/removeClass/}\n */\nexport function removeClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  name?:\n    | string\n    | ((this: Element, i: number, className: string) => string | undefined),\n): R {\n  // Handle if value is a function\n  if (typeof name === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        removeClass.call([el], name.call(el, i, el.attribs['class'] || ''));\n      }\n    });\n  }\n\n  const classes = splitNames(name);\n  const numClasses = classes.length;\n  const removeAll = arguments.length === 0;\n\n  return domEach(this, (el) => {\n    if (!isTag(el)) return;\n\n    if (removeAll) {\n      // Short circuit the remove all case as this is the nice one\n      el.attribs['class'] = '';\n    } else {\n      const elClasses = splitNames(el.attribs['class']);\n      let changed = false;\n\n      for (let j = 0; j < numClasses; j++) {\n        const index = elClasses.indexOf(classes[j]);\n\n        if (index >= 0) {\n          elClasses.splice(index, 1);\n          changed = true;\n\n          /*\n           * We have to do another pass to ensure that there are not duplicate\n           * classes listed\n           */\n          j--;\n        }\n      }\n      if (changed) {\n        el.attribs['class'] = elClasses.join(' ');\n      }\n    }\n  });\n}\n\n/**\n * Add or remove class(es) from the matched elements, depending on either the\n * class's presence or the value of the switch argument. Also accepts a\n * `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple.green').toggleClass('fruit green red').html();\n * //=> <li class=\"apple fruit red\">Apple</li>\n *\n * $('.apple.green').toggleClass('fruit green red', true).html();\n * //=> <li class=\"apple green fruit red\">Apple</li>\n * ```\n *\n * @param value - Name of the class. Can also be a function.\n * @param stateVal - If specified the state of the class.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/toggleClass/}\n */\nexport fu\nnction toggleClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  value?:\n    | string\n    | ((\n        this: Element,\n        i: number,\n        className: string,\n        stateVal?: boolean,\n      ) => string),\n  stateVal?: boolean,\n): R {\n  // Support functions\n  if (typeof value === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        toggleClass.call(\n          [el],\n          value.call(el, i, el.attribs['class'] || '', stateVal),\n          stateVal,\n        );\n      }\n    });\n  }\n\n  // Return if no value or not a string or function\n  if (!value || typeof value !== 'string') return this;\n\n  const classNames = value.split(rspace);\n  const numClasses = classNames.length;\n  const state = typeof stateVal === 'boolean' ? (stateVal ? 1 : -1) : 0;\n  const numElements = this.length;\n\n  for (let i = 0; i < numElements; i++) {\n    const el = this[i];\n    // If selected element isn't a tag, move on\n    if (!isTag(el)) continue;\n\n    const elementClasses = splitNames(el.attribs['class']);\n\n    // Check if class already exists\n    for (let j = 0; j < numClasses; j++) {\n      // Check if the class name is currently defined\n      const index = elementClasses.indexOf(classNames[j]);\n\n      // Add if stateValue === true or we are toggling and there is no value\n      if (state >= 0 && index < 0) {\n        elementClasses.push(classNames[j]);\n      } else if (state <= 0 && index >= 0) {\n        // Otherwise remove but only if the item exists\n        elementClasses.splice(index, 1);\n      }\n    }\n\n    el.attribs['class'] = elementClasses.join(' ');\n  }\n\n  return this;\n}\n\n// Path: src/api/manipulation.ts\n/**\n * Methods for modifying the DOM structure.\n *\n * @module cheerio/manipulation\n */\n\nimport {\n  isTag,\n  Text,\n  hasChildren,\n  cloneNode,\n  Document,\n  type ParentNode,\n  type AnyNode,\n  type Element,\n} from 'domhandler';\nimport { update as updateDOM } from '../parse.js';\nimport { text as staticText } from '../static.js';\nimport { domEach, isHtml, isCheerio } from '../utils.js';\nimport { removeElement } from 'domutils';\nimport type { Cheerio } from '../cheerio.js';\nimport type { BasicAcceptedElems, AcceptedElems } from '../types.js';\n\n/**\n * Create an array of nodes, recursing into arrays and parsing strings if\n * necessary.\n *\n * @private\n * @category Manipulation\n * @param elem - Elements to make an array of.\n * @param clone - Optionally clone nodes.\n * @returns The array of nodes.\n */\nexport function _makeDomArray<T extends AnyNode>(\n  this: Cheerio<T>,\n  elem?: BasicAcceptedElems<AnyNode> | BasicAcceptedElems<AnyNode>[],\n  clone?: boolean,\n): AnyNode[] {\n  if (elem == null) {\n    return [];\n  }\n\n  if (typeof elem === 'string') {\n    return this._parse(elem, this.options, false, null).children.slice(0);\n  }\n\n  if ('length' in elem) {\n    if (elem.length === 1) {\n      return this._makeDomArray(elem[0], clone);\n    }\n\n    const result: AnyNode[] = [];\n\n    for (let i = 0; i < elem.length; i++) {\n      const el = elem[i];\n\n      if (typeof el === 'object') {\n        if (el == null) {\n          continue;\n        }\n\n        if (!('length' in el)) {\n          result.push(clone ? cloneNode(el, true) : el);\n          continue;\n        }\n      }\n\n      result.push(...this._makeDomArray(el, clone));\n    }\n\n    return result;\n  }\n\n  return [clone ? cloneNode(elem, true) : elem];\n}\n\nfunction _insert(\n  concatenator: (\n    dom: AnyNode[],\n    children: AnyNode[],\n    parent: ParentNode,\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    ...elems:\n      | [\n          (\n            this: AnyNode,\n            i: number,\n            html: string,\n          ) => BasicAcceptedElems<AnyNode>,\n        ]\n      | BasicAcceptedElems<AnyNode>[]\n  ) {\n    const lastIdx = this.length - 1;\n\n    return domEach(this, (el, i) => {\n      if (!hasChildren(el)) return;\n\n      const domSrc =\n        typeof elems[0] === 'function'\n          ? elems[0].call(el, i, this._render(el.children))\n          : (elems as BasicAcceptedElems<AnyNode>[]);\n\n      const dom = this._makeDomArray(domSrc, i < lastIdx);\n      concatenator(dom, el.children, el);\n    });\n  };\n}\n\n/**\n * Modify an array in-place, removing some number of elements and adding new\n * elements directly following them.\n *\n * @private\n * @category Manipulation\n * @param array - Target array to splice.\n * @param spliceIdx - Index at which to begin changing the array.\n * @param spliceCount - Number of elements to remove from the array.\n * @param newElems - Elements to insert into the array.\n * @param parent - The parent of the node.\n * @returns The spliced array.\n */\nfunction uniqueSplice(\n  array: AnyNode[],\n  spliceIdx: number,\n  spliceCount: number,\n  newElems: AnyNode[],\n  parent: ParentNode,\n): AnyNode[] {\n  const spliceArgs: Parameters<typeof Array.prototype.splice> = [\n    spliceIdx,\n    spliceCount,\n    ...newElems,\n  ];\n  const prev = spliceIdx === 0 ? null : array[spliceIdx - 1];\n  const next =\n    spliceIdx + spliceCount >= array.length\n      ? null\n      : array[spliceIdx + spliceCount];\n\n  /*\n   * Before splicing in new elements, ensure they do not already appear in the\n   * current array.\n   */\n  for (let idx = 0; idx < newElems.length; ++idx) {\n    const node = newElems[idx];\n    const oldParent = node.parent;\n\n    if (oldParent) {\n      const oldSiblings: AnyNode[] = oldParent.children;\n      const prevIdx = oldSiblings.indexOf(node);\n\n      if (prevIdx > -1) {\n        oldParent.children.splice(prevIdx, 1);\n        if (parent === oldParent && spliceIdx > prevIdx) {\n          spliceArgs[0]--;\n        }\n      }\n    }\n\n    node.parent = parent;\n\n    if (node.prev) {\n      node.prev.next = node.next ?? null;\n    }\n\n    if (node.next) {\n      node.next.prev = node.prev ?? null;\n    }\n\n    node.prev = idx === 0 ? prev : newElems[idx - 1];\n    node.next = idx === newElems.length - 1 ? next : newElems[idx + 1];\n  }\n\n  if (prev) {\n    prev.next = newElems[0];\n  }\n  if (next) {\n    next.prev = newElems[newElems.length - 1];\n  }\n  return array.splice(...spliceArgs);\n}\n\n/**\n * Insert every element in the set of matched elements to the end of the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').appendTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to append elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/appendTo/}\n */\nexport function appendTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const appendTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  appendTarget.append(this);\n\n  return this;\n}\n\n/**\n * Insert every element in the set of matched elements to the beginning of the\n * target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').prependTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to prepend elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/prependTo/}\n */\nexport function prependTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const prependTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  prependTarget.prepend(this);\n\n  return this;\n}\n\n/**\n * Inserts content as the _last_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').append('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/append/}\n */\nexport const append = _insert((dom, children, parent) => {\n  uniqueSplice(children, children.length, 0, dom, parent);\n});\n\n/**\n * Inserts content as the _first_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').prepend('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/prepend/}\n */\nexport const prepend = _insert((dom, children, parent) => {\n  uniqueSplice(children, 0, 0, dom, parent);\n});\n\nfunction _wrap(\n  insert: (\n    el: AnyNode,\n    elInsertLocation: ParentNode,\n    wrapperDom: ParentNode[],\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    wrapper: AcceptedElems<AnyNode>,\n  ) {\n    const lastIdx = this.length - 1;\n    const lastParent = this.parents().last();\n\n    for (let i = 0; i < this.length; i++) {\n      const el = this[i];\n\n      const wrap =\n        typeof wrapper === 'function'\n          ? wrapper.call(el, i, el)\n          : typeof wrapper === 'string' && !isHtml(wrapper)\n            ? lastParent.find(wrapper).clone()\n            : wrapper;\n\n      const [wrapperDom] = this._makeDomArray(wrap, i < lastIdx);\n\n      if (!wrapperDom || !hasChildren(wrapperDom)) continue;\n\n      let elInsertLocation = wrapperDom;\n\n      /*\n       * Find the deepest child. Only consider the first tag child of each node\n       * (ignore text); stop if no children are found.\n       */\n      let j = 0;\n\n      while (j < elInsertLocation.children.length) {\n        const child = elInsertLocation.children[j];\n        if (isTag(child)) {\n          elInsertLocation = child;\n          j = 0;\n        } else {\n          j++;\n        }\n      }\n\n      insert(el, elInsertLocation, [wrapperDom]);\n    }\n\n    return this;\n  };\n}\n\n/**\n * The .wrap() function can take any string or object that could be passed to\n * the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. A\n * copy of this structure will be wrapped around each of the elements in the set\n * of matched elements. This method returns the original set of elements for\n * chaining purposes.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrap(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"red-fruit\">\n * //      <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrap(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"healthy\">\n * //       <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //       <li class=\"orange\">Orange</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //        <li class=\"plum\">Plum</li>\n * //     </div>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around each element in the\n *   selection.\n * @see {@link https://api.jquery.com/wrap/}\n */\nexport const wrap = _wrap((el, elInsertLocation, wrapperDom) => {\n  const { parent } = el;\n\n  if (!parent) return;\n\n  const siblings: AnyNode[] = parent.children;\n  const index = siblings.indexOf(el);\n\n  updateDOM([el], elInsertLocation);\n  /*\n   * The previous operation removed the current element from the `siblings`\n   * array, so the `dom` array can be inserted without removing any\n   * additional elements.\n   */\n  uniqueSplice(siblings, index, 0, wrapperDom, parent);\n});\n\n/**\n * The .wrapInner() function can take any string or object that could be passed\n * to the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around the content of each of the elements in the\n * set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrapInner(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"red-fruit\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"pear\">Pear</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrapInner(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"healthy\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">\n * //       <div class=\"healthy\">Orange</div>\n * //     </li>\n * //     <li class=\"pear\">\n * //       <div class=\"healthy\">Pear</div>\n * //     </li>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around the content of each element\n *   in the selection.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/wrapInner/}\n */\nexport const wrapInner = _wrap((el, elInsertLocation, wrapperDom) => {\n  if (!hasChildren(el)) return;\n  updateDOM(el.children, elInsertLocation);\n  updateDOM(wrapperDom, el);\n});\n\n/**\n * The .unwrap() function, removes the parents of the set of matched elements\n * from the DOM, leaving the matched elements in their place.\n *\n * @category Manipulation\n * @example <caption>without selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <div><p>Hello</p></div>\\n  <div><p>World</p></div>\\n</div>',\n * );\n * $('#test p').unwrap();\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @example <caption>with selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <p>Hello</p>\\n  <b><p>World</p></b>\\n</div>',\n * );\n * $('#test p').unwrap('b');\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @param selector - A selector to check the parent element against. If an\n *   element's parent does not match the selector, the element won't be\n *   unwrapped.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/unwrap/}\n */\nexport function unwrap<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  this.parent(selector)\n    .not('body')\n    .each((_, el) => {\n      this._make(el).replaceWith(el.children);\n    });\n  return this;\n}\n\n/**\n * The .wrapAll() function can take any string or object that could be passed to\n * the $() function to specify a DOM structure. This structure may be nested\n * several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around all of the elements in the set of matched\n * elements, as a single group.\n *\n * @category Manipulation\n * @example <caption>With markup passed to `wrapAll`</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div class=\"container\"><div class=\"inner\">First</div><div class=\"inner\">Second</div></div>',\n * );\n * $('.inner').wrapAll(\"<div class='new'></div>\");\n *\n * //=> <div class=\"container\">\n * //     <div class='new'>\n * //       <div class=\"inner\">First</div>\n * //       <div class=\"inner\">Second</div>\n * //     </div>\n * //   </div>\n * ```\n *\n * @example <caption>With an existing cheerio instance</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<span>Span 1</span><strong>Strong</strong><span>Span 2</span>',\n * );\n * const wrap = $('<div><p><em><b></b></em></p></div>');\n * $('span').wrapAll(wrap);\n *\n * //=> <div>\n * //     <p>\n * //       <em>\n * //         <b>\n * //           <span>Span 1</span>\n * //           <span>Span 2</span>\n * //         </b>\n * //       </em>\n * //     </p>\n * //   </div>\n * //   <strong>Strong</strong>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around all matched elements in the\n *   selection.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/wrapAll/}\n */\nexport function wrapAll<T extends AnyNode>(\n  this: Cheerio<T>,\n  wrapper: AcceptedElems<T>,\n): Cheerio<T> {\n  const el = this[0];\n  if (el) {\n    const wrap: Cheerio<AnyNode> = this._make(\n      typeof wrapper === 'function' ? wrapper.call(el, 0, el) : wrapper,\n    ).insertBefore(el);\n\n    // If html is given as wrapper, wrap may contain text elements\n    let elInsertLocation: Element | undefined;\n\n    for (let i = 0; i < wrap.length; i++) {\n      if (wrap[i].type === 'tag') elInsertLocation = wrap[i] as Element;\n    }\n\n    let j = 0;\n\n    /*\n     * Find the deepest child. Only consider the first tag child of each node\n     * (ignore text); stop if no children are found.\n     */\n    while (elInsertLocation && j < elInsertLocation.children.length) {\n      const child = elInsertLocation.children[j];\n      if (child.type === 'tag') {\n        elInsertLocation = child as Element;\n        j = 0;\n      } else {\n        j++;\n      }\n    }\n\n    if (elInsertLocation) this._make(elInsertLocation).append(this);\n  }\n  return this;\n}\n\n/**\n * Insert content next to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').after('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert after each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/after/}\n */\nexport function after<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element after `this` element\n    uniqueSplice(siblings, index + 1, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements after the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertAfter('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements after.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertAfter/}\n */\nexport function insertAfter<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  if (typeof target === 'string') {\n    target = this._make<AnyNode>(target);\n  }\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  for (const el of this._makeDomArray(target)) {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      continue;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) continue;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index + 1, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  }\n\n  return this._make(clones);\n}\n\n/**\n * Insert content previous to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').before('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert before each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/before/}\n */\nexport function before<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element before `el` element\n    uniqueSplice(siblings, index, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements before the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertBefore('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements before.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertBefore/}\n */\nexport function insertBefore<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const targetArr = this._make<AnyNode>(target);\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  domEach(targetArr, (el) => {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  });\n\n  return this._make(clones);\n}\n\n/**\n * Removes the set of matched elements from the DOM and all their children.\n * `selector` filters the set of matched elements to be removed.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.pear').remove();\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //    </ul>\n * ```\n *\n * @param selector - Optional selector for elements to remove.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/remove/}\n */\nexport function remove<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  // Filter if we have selector\n  const elems = selector ? this.filter(selector) : this;\n\n  domEach(elems, (el) => {\n    removeElement(el);\n    el.prev = el.next = el.parent = null;\n  });\n\n  return this;\n}\n\n/**\n * Replaces matched elements with `content`.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const plum = $('<li class=\"plum\">Plum</li>');\n * $('.pear').replaceWith(plum);\n * $.html();\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">Apple</li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n * ```\n *\n * @param content - Replacement for matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/replaceWith/}\n */\nexport function replaceWith<T extends AnyNode>(\n  this: Cheerio<T>,\n  content: AcceptedElems<AnyNode>,\n): Cheerio<T> {\n  return domEach(this, (el, i) => {\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const cont =\n      typeof content === 'function' ? content.call(el, i, el) : content;\n    const dom = this._makeDomArray(cont);\n\n    /*\n     * In the case that `dom` contains nodes that already exist in other\n     * structures, ensure those nodes are properly removed.\n     */\n    updateDOM(dom, null);\n\n    const index = siblings.indexOf(el);\n\n    // Completely remove old element\n    uniqueSplice(siblings, index, 1, dom, parent);\n\n    if (!dom.includes(el)) {\n      el.parent = el.prev = el.next = null;\n    }\n  });\n}\n\n/**\n * Removes all children from each item in the selection. Text nodes and comment\n * nodes are left as is.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').empty();\n * $.html();\n * //=>  <ul id=\"fruits\"></ul>\n * ```\n *\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/empty/}\n */\nexport function empty<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    el.children.length = 0;\n  });\n}\n\n/**\n * Gets an HTML content string from the first selected element.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').html();\n * //=> Orange\n *\n * $('#fruits').html('<li class=\"mango\">Mango</li>').html();\n * //=> <li class=\"mango\">Mango</li>\n * ```\n *\n * @returns The HTML content string.\n * @see {@link https://api.jquery.com/html/}\n */\nexport function html<T extends AnyNode>(this: Cheerio<T>): string | null;\n/**\n * Replaces each selected element's content with the specified content.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').html('<li class=\"mango\">Mango</li>').html();\n * //=> <li class=\"mango\">Mango</li>\n * ```\n *\n * @param str - The content to replace selection's contents with.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/html/}\n */\nexport function html<T extends AnyNode>(\n  this: Cheerio<T>,\n  str: string | Cheerio<T>,\n): Cheerio<T>;\nexport function html<T extends AnyNode>(\n  this: Cheerio<T>,\n  str?: string | Cheerio<AnyNode>,\n): Cheerio<T> | string | null {\n  if (str === undefined) {\n    const el = this[0];\n    if (!el || !hasChildren(el)) return null;\n    return this._render(el.children);\n  }\n\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    const content = isCheerio(str)\n      ? str.toArray()\n      : this._parse(`${str}`, this.options, false, el).children;\n\n    updateDOM(content, el);\n  });\n}\n\n/**\n * Turns the collection to a string. Alias for `.html()`.\n *\n * @category Manipulation\n * @returns The rendered document.\n */\nexport function toString<T extends AnyNode>(this: Cheerio<T>): string {\n  return this._render(this);\n}\n\n/**\n * Get the combined text contents of each element in the set of matched\n * elements, including their descendants.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').text();\n * //=> Orange\n *\n * $('ul').text();\n * //=>  Apple\n * //    Orange\n * //    Pear\n * ```\n *\n * @returns The text contents of the collection.\n * @see {@link https://api.jquery.com/text/}\n */\nexport function text<T extends AnyNode>(this: Cheerio<T>): string;\n/**\n * Set the content of each element in the set of matched elements to the\n * specified text.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').text('Orange');\n * //=> <div class=\"orange\">Orange</div>\n * ```\n *\n * @param str - The text to set as the content of each matched element.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/text/}\n */\nexport function text<T extends AnyNode>(\n  this: Cheerio<T>,\n  str: string | ((this: AnyNode, i: number, text: string) => string),\n): Cheerio<T>;\nexport function text<T extends AnyNode>(\n  this: Cheerio<T>,\n  str?: string | ((this: AnyNode, i: number, text: string) => string),\n): Cheerio<T> | string {\n  // If `str` is undefined, act as a \"getter\"\n  if (str === undefined) {\n    return staticText(this);\n  }\n  if (typeof str === 'function') {\n    // Function support\n    return domEach(this, (el, i) =>\n      this._make(el).text(str.call(el, i, staticText([el]))),\n    );\n  }\n\n  // Append text node to each selected elements\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    const textNode = new Text(`${str}`);\n\n    updateDOM(textNode, el);\n  });\n}\n\n/**\n * Clone the cheerio object.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const moreFruit = $('#fruits').clone();\n * ```\n *\n * @returns The cloned object.\n * @see {@link https://api.jquery.com/clone/}\n */\nexport function clone<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  const clone = Array.prototype.map.call(this.get(), (el) =>\n    cloneNode(el, true),\n  ) as T[];\n\n  // Add a root node around the cloned nodes\n  const root = new Document(clone);\n  for (const node of clone) {\n    node.parent = root;\n  }\n\n  return this._make(clone);\n}\n\n// Path: src/api/traversing.ts\n/**\n * Methods for traversing the DOM structure.\n *\n * @module cheerio/traversing\n */\n\nimport {\n  isTag,\n  type AnyNode,\n  type Element,\n  hasChildren,\n  isDocument,\n  type Document,\n} from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport * as select from 'cheerio-select';\nimport { domEach, isCheerio } from '../utils.js';\nimport { contains } from '../static.js';\nimport {\n  getChildren,\n  getSiblings,\n  nextElementSibling,\n  prevElementSibling,\n  uniqueSort,\n} from 'domutils';\nimport type { FilterFunction, AcceptedFilters } from '../types.js';\nconst reSiblingSelector = /^\\s*[+~]/;\n\n/**\n * Get the descendants of each element in the current set of matched elements,\n * filtered by a selector, jQuery object, or element.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').find('li').length;\n * //=> 3\n * $('#fruits').find($('.apple')).length;\n * //=> 1\n * ```\n...\n", "needle_token_start": 5154, "needle_token_end": 5665, "code_context_ntokens": 15429, "output": ["```typescript\n/**\n * Add or remove class(es) from the matched elements, depending on either the\n * class's presence or the value of the switch argument. Also accepts a\n * `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple.green').toggleClass('fruit green red').html();\n * //=> <li class=\"apple fruit red\">Apple</li>\n *\n * $('.apple.green').toggleClass('fruit green red', true).html();\n * //=> <li class=\"apple green fruit red\">Apple</li>\n * ```\n *\n * @param value - Name of the class. Can also be a function.\n * @param stateVal - If specified the state of the class.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/toggleClass/}\n */\nexport function toggleClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  value?:\n    | string\n    | ((\n        this: Element,\n        i: number,\n        className: string,\n        stateVal?: boolean,\n      ) => string),\n  stateVal?: boolean,\n): R {\n  // Support functions\n  if (typeof value === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        toggleClass.call(\n          [el],\n          value.call(el, i, el.attribs['class'] || '', stateVal),\n          stateVal,\n        );\n      }\n    });\n  }\n\n  // Return if no value or not a string or function\n  if (!value || typeof value !== 'string') return this;\n\n  const classNames = value.split(rspace);\n  const numClasses = classNames.length;\n  const state = typeof stateVal === 'boolean' ? (stateVal ? 1 : -1) : 0;\n  const numElements = this.length;\n\n  for (let i = 0; i < numElements; i++) {\n    const el = this[i];\n    // If selected element isn't a tag, move on\n    if (!isTag(el)) continue;\n\n    const elementClasses = splitNames(el.attribs['class']);\n\n    // Check if class already exists\n    for (let j = 0; j < numClasses; j++) {\n      // Check if the class name is currently defined\n      const index = elementClasses.indexOf(classNames[j]);\n\n      // Add if stateValue === true or we are toggling and there is no value\n      if (state >= 0 && index < 0) {\n        elementClasses.push(classNames[j]);\n      } else if (state <= 0 && index >= 0) {\n        // Otherwise remove but only if the item exists\n        elementClasses.splice(index, 1);\n      }\n    }\n\n    el.attribs['class'] = elementClasses.join(' ');\n  }\n\n  return this;\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "attr", "language": "typescript", "path": "src/api/attributes.ts", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to manage attributes of HTML elements in a web document. It can retrieve, set, or remove attributes based on the provided parameters.\n2. **Input**: This function accepts either a single attribute name, a pair of attribute name and value, a map of attribute names and values, or a function to dynamically set the attribute values. It operates on HTML elements wrapped in a specific object type.\n3. **Output**: Depending on the operation, the output can be the value of an attribute, a map of all attributes, the modified HTML element itself, or undefined if the attribute does not exist.\n4. **Procedure**: The function first checks the type and number of arguments to determine the operation mode (get, set, or remove). For setting attributes, it can handle both individual and multiple attributes through different parameter formats. If setting an attribute to `null`, it removes the attribute. For getting attributes, it can return either a single attribute value or all attributes of the first element in the set. The function ensures operations are only performed on valid HTML elements.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/load.ts\nimport {\n  type CheerioOptions,\n  type InternalOptions,\n  flattenOptions,\n} from './options.js';\nimport * as staticMethods from './static.js';\nimport { Cheerio } from './cheerio.js';\nimport { isHtml, isCheerio } from './utils.js';\nimport type { AnyNode, Document, Element, ParentNode } from 'domhandler';\nimport type { SelectorType, BasicAcceptedElems } from './types.js';\n\ntype StaticType = typeof staticMethods;\n\n/**\n * A querying function, bound to a document created from the provided markup.\n *\n * Also provides several helper methods for dealing with the document as a\n * whole.\n */\nexport interface CheerioAPI extends StaticType {\n  /**\n   * This selector method is the starting point for traversing and manipulating\n   * the document. Like jQuery, it's the primary method for selecting elements\n   * in the document.\n   *\n   * `selector` searches within the `context` scope, which searches within the\n   * `root` scope.\n   *\n   * @example\n   *\n   * ```js\n   * $('ul .pear').attr('class');\n   * //=> pear\n   *\n   * $('li[class=orange]').html();\n   * //=> Orange\n   *\n   * $('.apple', '#fruits').text();\n   * //=> Apple\n   * ```\n   *\n   * Optionally, you can also load HTML by passing the string as the selector:\n   *\n   * ```js\n   * $('<ul id=\"fruits\">...</ul>');\n   * ```\n   *\n   * Or the context:\n   *\n   * ```js\n   * $('ul', '<ul id=\"fruits\">...</ul>');\n   * ```\n   *\n   * Or as the root:\n   *\n   * ```js\n   * $('li', 'ul', '<ul id=\"fruits\">...</ul>');\n   * ```\n   *\n   * @param selector - Either a selector to look for within the document, or the\n   *   contents of a new Cheerio instance.\n   * @param context - Either a selector to look for within the root, or the\n   *   contents of the document to query.\n   * @param root - Optional HTML document string.\n   */\n  <T extends AnyNode, S extends string>(\n...\n// Path: src/index.ts\n/**\n * The main types of Cheerio objects.\n *\n * @category Cheerio\n */\nexport type { Cheerio } from './cheerio.js';\n\n/**\n * Types used in signatures of Cheerio methods.\n *\n * @category Cheerio\n */\nexport * from './types.js';\nexport type {\n  CheerioOptions,\n  HTMLParser2Options,\n  Parse5Options,\n} from './options.js';\n/**\n * Re-exporting all of the node types.\n *\n * @category DOM Node\n */\nexport type { Node, AnyNode, ParentNode, Element, Document } from 'domhandler';\n\nexport type { CheerioAPI } from './load.js';\nimport { getLoad } from './load.js';\nimport { getParse } from './parse.js';\nimport { renderWithParse5, parseWithParse5 } from './parsers/parse5-adapter.js';\nimport renderWithHtmlparser2 from 'dom-serializer';\nimport { parseDocument as parseWithHtmlparser2 } from 'htmlparser2';\n\nconst parse = getParse((content, options, isDocument, context) =>\n  options._useHtmlParser2\n    ? parseWithHtmlparser2(content, options)\n    : parseWithParse5(content, options, isDocument, context),\n);\n\n// Duplicate docs due to https://github.com/TypeStrong/typedoc/issues/1616\n/**\n * Create a querying function, bound to a document created from the provided\n * markup.\n *\n * Note that similar to web browser contexts, this operation may introduce\n * `<html>`, `<head>`, and `<body>` elements; set `isDocument` to `false` to\n * switch to fragment mode and disable this.\n *\n * @category Loading\n * @param content - Markup to be loaded.\n * @param options - Options for the created instance.\n * @param isDocument - Allows parser to be switched to fragment mode.\n * @returns The loaded document.\n * @see {@link https://cheerio.js.org#loading} for additional usage information.\n */\nexport const load = getLoad(parse, (dom, options) =>\n  options._useHtmlParser2\n    ? renderWithHtmlparser2(dom, options)\n    : renderWithParse5(dom),\n);\n\nconst defaultInstance = load([]);\n\n/**\n * The default cheerio instance.\n *\n * @deprecated Use the function returned by `load` instead. To access load, make\n *   sure you are importing `* as cheerio` instead of this default export.\n * @category Deprecated\n */\nexport default defaultInstance;\n\nimport * as staticMethods from './static.js';\nimport type { BasicAcceptedElems } from './types.js';\nimport type { CheerioOptions } from './options.js';\nimport type { AnyNode } from 'domhandler';\n\nexport const { contains, merge } = staticMethods;\n\n/**\n * Renders the document.\n *\n * @deprecated Use `html` on the loaded instance instead.\n * @category Deprecated\n * @param dom - Element to render.\n * @param options - Options for the renderer.\n * @returns The rendered document.\n */\nexport const html: (\n  dom: BasicAcceptedElems<AnyNode>,\n  options?: CheerioOptions,\n) => string = staticMethods.html.bind(defaultInstance);\n\n/**\n * Render the document as XML.\n *\n * @deprecated Use `xml` on the loaded instance instead.\n * @category Deprecated\n * @param dom - Element to render.\n * @returns The rendered document.\n */\nexport const xml: (dom: BasicAcceptedElems<AnyNode>) => string =\n  staticMethods.xml.bind(defaultInstance);\n\n/**\n * Render the document as text.\n *\n * This returns the `textContent` of the passed elements. The result will\n * include the contents of `<script>` and `<style>` elements. To avoid this, use\n * `.prop('innerText')` instead.\n *\n * @deprecated Use `text` on the loaded instance instead.\n * @category Deprecated\n * @param elements - Elements to render.\n * @returns The rendered document.\n */\nexport const text: (elements: ArrayLike<AnyNode>) => string =\n  staticMethods.text.bind(defaultInstance);\n\n/**\n * The `.parseHTML` method exported by the Cheerio module is deprecated.\n *\n * In order to promote consistency with the jQuery library, users are encouraged\n * to instead use the static method of the same name as it is defined on the\n * \"loaded\" Cheerio factory function.\n *\n * @deprecated Use `parseHTML` on the loaded instance instead.\n * @category Deprecated\n * @example\n *\n * ```js\n * const $ = cheerio.load('');\n * $.parseHTML('<b>markup</b>');\n * ```\n */\nexport const parseHTML = staticMethods.parseHTML.bind(defaultInstance);\n\n/**\n * The `.root` method exported by the Cheerio module is deprecated.\n *\n * Users seeking to access the top-level element of a parsed document should\n * instead use the `root` static method of a \"loaded\" Cheerio function.\n *\n * @deprecated Use `root` on the loaded instance instead.\n * @category Deprecated\n * @example\n *\n * ```js\n * const $ = cheerio.load('');\n * $.root();\n * ```\n */\nexport const root = staticMethods.root.bind(defaultInstance);\n\n// Path: src/api/extract.ts\nimport type { AnyNode, Element } from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport type { prop } from './attributes.js';\n\ntype ExtractDescriptorFn = (\n  el: Element,\n  key: string,\n  // TODO: This could be typed with ExtractedMap\n  obj: Record<string, unknown>,\n) => unknown;\n\ninterface ExtractDescriptor {\n  selector: string;\n  value?: string | ExtractDescriptorFn | ExtractMap;\n}\n\ntype ExtractValue = string | ExtractDescriptor | [string | ExtractDescriptor];\n\nexport interface ExtractMap {\n  [key: string]: ExtractValue;\n}\n\ntype ExtractedValue<V extends ExtractValue, M extends ExtractMap> = V extends [\n  string | ExtractDescriptor,\n]\n  ? NonNullable<ExtractedValue<V[0], M>>[]\n  : V extends string\n    ? string | undefined\n    : V extends ExtractDescriptor\n      ? V['value'] extends ExtractMap\n        ? ExtractedMap<V['value']> | undefined\n        : V['value'] extends ExtractDescriptorFn\n          ? ReturnType<V['value']> | undefined\n          : ReturnType<typeof prop> | undefined\n      : never;\n\nexport type ExtractedMap<M extends ExtractMap> = {\n  [key in keyof M]: ExtractedValue<M[key], M>;\n};\n\nfunction getExtractDescr(\n  descr: string | ExtractDescriptor,\n): Required<ExtractDescriptor> {\n  if (typeof descr === 'string') {\n    return { selector: descr, value: 'textContent' };\n  }\n\n  return {\n    selector: descr.selector,\n    value: descr.value ?? 'textContent',\n  };\n}\n\n/**\n * Extract multiple values from a document, and store them in an object.\n *\n * @param map - An object containing key-value pairs. The keys are the names of\n *   the properties to be created on the object, and the values are the\n *   selectors to be used to extract the values.\n * @returns An object containing the extracted values.\n */\nexport function extract<M extends ExtractMap, T extends AnyNode>(\n  this: Cheerio<T>,\n  map: M,\n): ExtractedMap<M> {\n  const ret: Record<string, unknown> = {};\n\n  for (const key in map) {\n    const descr = map[key];\n    const isArray = Array.isArray(descr);\n\n    const { selector, value } = getExtractDescr(isArray ? descr[0] : descr);\n\n    const fn: ExtractDescriptorFn =\n      typeof value === 'function'\n        ? value\n        : typeof value === 'string'\n          ? (el: Element) => this._make(el).prop(value)\n          : (el: Element) => this._make(el).extract(value);\n\n    if (isArray) {\n      ret[key] = this._findBySelector(selector, Number.POSITIVE_INFINITY)\n        .map((_, el) => fn(el, key, ret))\n        .get();\n    } else {\n      const $ = this._findBySelector(selector, 1);\n      ret[key] = $.length > 0 ? fn($[0], key, ret) : undefined;\n    }\n  }\n\n  return ret as ExtractedMap<M>;\n}\n\n// Path: src/static.ts\nimport type { BasicAcceptedElems } from './types.js';\nimport type { CheerioAPI, Cheerio } from './index.js';\nimport type { AnyNode, Document } from 'domhandler';\nimport { textContent } from 'domutils';\nimport {\n  type InternalOptions,\n  type CheerioOptions,\n  flattenOptions as flattenOptions,\n} from './options.js';\nimport type { ExtractedMap, ExtractMap } from './api/extract.js';\n\n/**\n * Helper function to render a DOM.\n *\n * @param that - Cheerio instance to render.\n * @param dom - The DOM to render. Defaults to `that`'s root.\n * @param options - Options for rendering.\n * @returns The rendered document.\n */\nfunction render(\n  that: CheerioAPI,\n  dom: BasicAcceptedElems<AnyNode> | undefined,\n  options: InternalOptions,\n): string {\n  if (!that) return '';\n\n  return that(dom ?? that._root.children, null, undefined, options).toString();\n}\n\n/**\n * Checks if a passed object is an options object.\n *\n * @param dom - Object to check if it is an options object.\n * @param options - Options object.\n * @returns Whether the object is an options object.\n */\nfunction isOptions(\n  dom?: BasicAcceptedElems<AnyNode> | CheerioOptions | null,\n  options?: CheerioOptions,\n): dom is CheerioOptions {\n  return (\n    !options &&\n    typeof dom === 'object' &&\n    dom != null &&\n    !('length' in dom) &&\n    !('type' in dom)\n  );\n}\n\n/**\n * Renders the document.\n *\n * @category Static\n * @param options - Options for the renderer.\n * @returns The rendered document.\n */\nexport function html(this: CheerioAPI, options?: CheerioOptions): string;\n/**\n * Renders the document.\n *\n * @category Static\n * @param dom - Element to render.\n * @param options - Options for the renderer.\n * @returns The rendered document.\n */\nexport function html(\n  this: CheerioAPI,\n  dom?: BasicAcceptedElems<AnyNode>,\n  options?: CheerioOptions,\n): string;\nexport function html(\n  this: CheerioAPI,\n  dom?: BasicAcceptedElems<AnyNode> | CheerioOptions,\n  options?: CheerioOptions,\n): string {\n  /*\n   * Be flexible about parameters, sometimes we call html(),\n   * with options as only parameter\n   * check dom argument for dom element specific properties\n   * assume there is no 'length' or 'type' properties in the options object\n   */\n  const toRender = isOptions(dom) ? ((options = dom), undefined) : dom;\n\n  /*\n   * Sometimes `$.html()` is used without preloading html,\n   * so fallback non-existing options to the default ones.\n   */\n  const opts = {\n    ...this?._options,\n    ...flattenOptions(options),\n  };\n\n  return render(this, toRender, opts);\n}\n\n/**\n * Render the document as XML.\n *\n * @category Static\n * @param dom - Element to render.\n * @returns THe rendered document.\n */\nexport function xml(\n  this: CheerioAPI,\n  dom?: BasicAcceptedElems<AnyNode>,\n): string {\n  const options = { ...this._options, xmlMode: true };\n\n  return render(this, dom, options);\n}\n\n/**\n * Render the document as text.\n *\n * This returns the `textContent` of the passed elements. The result will\n * include the contents of `<script>` and `<style>` elements. To avoid this, use\n * `.prop('innerText')` instead.\n *\n * @category Static\n * @param elements - Elements to render.\n * @returns The rendered document.\n */\nexport function text(\n  this: CheerioAPI | void,\n  elements?: ArrayLike<AnyNode>,\n): string {\n  const elems = elements ?? (this ? this.root() : []);\n\n  let ret = '';\n\n  for (let i = 0; i < elems.length; i++) {\n    ret += textContent(elems[i]);\n  }\n\n  return ret;\n}\n\n/**\n * Parses a string into an array of DOM nodes. The `context` argument has no\n * meaning for Cheerio, but it is maintained for API compatibility with jQuery.\n *\n * @category Static\n * @param data - Markup that will be parsed.\n * @param context - Will be ignored. If it is a boolean it will be used as the\n *   value of `keepScripts`.\n * @param keepScripts - If false all scripts will be removed.\n * @returns The parsed DOM.\n * @alias Cheerio.parseHTML\n * @see {@link https://api.jquery.com/jQuery.parseHTML/}\n */\nexport function parseHTML(\n  this: CheerioAPI,\n  data: string,\n  context?: unknown | boolean,\n  keepScripts?: boolean,\n): AnyNode[];\nexport function parseHTML(this: CheerioAPI, data?: '' | null): null;\nexport function parseHTML(\n  this: CheerioAPI,\n  data?: string | null,\n  context?: unknown | boolean,\n  keepScripts = typeof context === 'boolean' ? context : false,\n): AnyNode[] | null {\n  if (!data || typeof data !== 'string') {\n    return null;\n  }\n\n  if (typeof context === 'boolean') {\n    keepScripts = context;\n  }\n\n  const parsed = this.load(data, this._options, false);\n  if (!keepScripts) {\n    parsed('script').remove();\n  }\n\n  /*\n   * The `children` array is used by Cheerio internally to group elements that\n   * share the same parents. When nodes created through `parseHTML` are\n   * inserted into previously-existing DOM structures, they will be removed\n   * from the `children` array. The results of `parseHTML` should remain\n   * constant across these operations, so a shallow copy should be returned.\n   */\n  return [...parsed.root()[0].children];\n}\n\n/**\n * Sometimes you need to work with the top-level root element. To query it, you\n * can use `$.root()`.\n *\n * @category Static\n * @example\n *\n * ```js\n * $.root().append('<ul id=\"vegetables\"></ul>').html();\n * //=> <ul id=\"fruits\">...</ul><ul id=\"vegetables\"></ul>\n * ```\n *\n * @returns Cheerio instance wrapping the root node.\n * @alias Cheerio.root\n */\nexport function root(this: CheerioAPI): Cheerio<Document> {\n  return this(this._root);\n}\n\n/**\n * Checks to see if the `contained` DOM element is a descendant of the\n * `container` DOM element.\n *\n * @category Static\n * @param container - Potential parent node.\n * @param contained - Potential child node.\n * @returns Indicates if the nodes contain one another.\n * @alias Cheerio.contains\n * @see {@link https://api.jquery.com/jQuery.contains/}\n */\nexport function contains(container: AnyNode, contained: AnyNode): boolean {\n  // According to the jQuery API, an element does not \"contain\" itself\n  if (contained === container) {\n    return false;\n  }\n\n  /*\n   * Step up the descendants, stopping when the root element is reached\n   * (signaled by `.parent` returning a reference to the same object)\n   */\n  let next: AnyNode | null = contained;\n  while (next && next !== next.parent) {\n    next = next.parent;\n    if (next === container) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\n/**\n * Extract multiple values from a document, and store them in an object.\n *\n * @category Static\n * @param map - An object containing key-value pairs. The keys are the names of\n *   the properties to be created on the object, and the values are the\n *   selectors to be used to extract the values.\n * @returns An object containing the extracted values.\n */\nexport function extract<M extends ExtractMap>(\n  this: CheerioAPI,\n  map: M,\n): ExtractedMap<M> {\n  return this.root().extract(map);\n}\n\ntype Writable<T> = { -readonly [P in keyof T]: T[P] };\n\n/**\n * $.merge().\n *\n * @category Static\n * @param arr1 - First array.\n * @param arr2 - Second array.\n * @returns `arr1`, with elements of `arr2` inserted.\n * @alias Cheerio.merge\n * @see {@link https://api.jquery.com/jQuery.merge/}\n */\nexport function merge<T>(\n  arr1: Writable<ArrayLike<T>>,\n  arr2: ArrayLike<T>,\n): ArrayLike<T> | undefined {\n  if (!isArrayLike(arr1) || !isArrayLike(arr2)) {\n    return;\n  }\n  let newLength = arr1.length;\n  const len = +arr2.length;\n\n  for (let i = 0; i < len; i++) {\n    arr1[newLength++] = arr2[i];\n  }\n  arr1.length = newLength;\n  return arr1;\n}\n\n/**\n * Checks if an object is array-like.\n *\n * @category Static\n * @param item - Item to check.\n * @returns Indicates if the item is array-like.\n */\nfunction isArrayLike(item: unknown): item is ArrayLike<unknown> {\n  if (Array.isArray(item)) {\n    return true;\n  }\n\n  if (\n    typeof item !== 'object' ||\n    item === null ||\n    !('length' in item) ||\n    typeof item.length !== 'number' ||\n    item.length < 0\n  ) {\n    return false;\n  }\n\n  for (let i = 0; i < item.length; i++) {\n    if (!(i in item)) {\n      return false;\n    }\n  }\n  return true;\n}\n\n// Path: src/api/attributes.ts\n/**\n * Methods for getting and modifying attributes.\n *\n * @module cheerio/attributes\n */\n\nimport { text } from '../static.js';\nimport { domEach, camelCase, cssCase } from '../utils.js';\nimport { isTag, type AnyNode, type Element } from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport { innerText, textContent } from 'domutils';\nconst hasOwn = Object.prototype.hasOwnProperty;\nconst rspace = /\\s+/;\nconst dataAttrPrefix = 'data-';\n\n// Attributes that are booleans\nconst rboolean =\n  /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i;\n// Matches strings that look like JSON objects or arrays\nconst rbrace = /^{[^]*}$|^\\[[^]*]$/;\n\n/**\n * Gets a node's attribute. For boolean attributes, it will return the value's\n * name should it be set.\n *\n * Also supports getting the `value` of several form elements.\n *\n * @private\n * @category Attributes\n * @param elem - Element to get the attribute of.\n * @param name - Name of the attribute.\n * @param xmlMode - Disable handling of special HTML attributes.\n * @returns The attribute's value.\n */\nfunction getAttr(\n  elem: AnyNode,\n  name: undefined,\n  xmlMode?: boolean,\n): Record<string, string> | undefined;\nfunction getAttr(\n  elem: AnyNode,\n  name: string,\n  xmlMode?: boolean,\n): string | undefined;\nfunction getAttr(\n  elem: AnyNode,\n  name: string | undefined,\n  xmlMode?: boolean,\n): Record<string, string> | string | undefined {\n  if (!elem || !isTag(elem)) return undefined;\n\n  elem.attribs ??= {};\n\n  // Return the entire attribs object if no attribute specified\n  if (!name) {\n    return elem.attribs;\n  }\n\n  if (hasOwn.call(elem.attribs, name)) {\n    // Get the (decoded) attribute\n    return !xmlMode && rboolean.test(name) ? name : elem.attribs[name];\n  }\n\n  // Mimic the DOM and return text content as value for `option's`\n  if (elem.name === 'option' && name === 'value') {\n    return text(elem.children);\n  }\n\n  // Mimic DOM with default value for radios/checkboxes\n  if (\n    elem.name === 'input' &&\n    (elem.attribs['type'] === 'radio' || elem.attribs['type'] === 'checkbox') &&\n    name === 'value'\n  ) {\n    return 'on';\n  }\n\n  return undefined;\n}\n\n/**\n * Sets the value of an attribute. The attribute will be deleted if the value is\n * `null`.\n *\n * @private\n * @param el - The element to set the attribute on.\n * @param name - The attribute's name.\n * @param value - The attribute's value.\n */\nfunction setAttr(el: Element, name: string, value: string | null) {\n  if (value === null) {\n    removeAttribute(el, name);\n  } else {\n    el.attribs[name] = `${value}`;\n  }\n}\n\n/**\n * Method for getting attributes. Gets the attribute value for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr('id');\n * //=> fruits\n * ```\n *\n * @param name - Name of the attribute.\n * @returns The attribute's value.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): string | undefined;\n/**\n * Method for getting all attributes and their values of the first element in\n * the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr();\n * //=> { id: 'fruits' }\n * ```\n *\n * @returns The attribute's values.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n): Record<string, string> | undefined;\n/**\n * Method for setting attributes. Sets the attribute value for only the first\n * element in the matched set. If you set an attribute's value to `null`, you\n * remove that attribute. You may also pass a `map` and `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr('id', 'favorite').html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param name - Name of the attribute.\n * @param value - The new value of the attribute.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): Cheerio<T>;\n/**\n * Method for setting multiple attributes at once. Sets the attribute value for\n * only the first element in the matched set. If you set an attribute's value to\n * `null`, you remove that attribute.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr({ id: 'favorite' }).html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param values - Map of attribute names and values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  values: Record<string, string | null>,\n): Cheerio<T>;\nexport \nfunction attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name?: string | Record<string, string | null>,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): string | Cheerio<T> | undefined | Record<string, string> {\n  // Set the value (with attr map support)\n  if (typeof name === 'object' || value !== undefined) {\n    if (typeof value === 'function') {\n      if (typeof name !== 'string') {\n        {\n          throw new Error('Bad combination of arguments.');\n        }\n      }\n      return domEach(this, (el, i) => {\n        if (isTag(el)) setAttr(el, name, value.call(el, i, el.attribs[name]));\n      });\n    }\n    return domEach(this, (el) => {\n      if (!isTag(el)) return;\n\n      if (typeof name === 'object') {\n        for (const objName of Object.keys(name)) {\n          const objValue = name[objName];\n          setAttr(el, objName, objValue);\n        }\n      } else {\n        setAttr(el, name as string, value as string);\n      }\n    });\n  }\n\n  return arguments.length > 1\n    ? this\n    : getAttr(this[0], name as string, this.options.xmlMode);\n}\n\n/**\n * Gets a node's prop.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the prop of.\n * @param name - Name of the prop.\n * @param xmlMode - Disable handling of special HTML attributes.\n * @returns The prop's value.\n */\nfunction getProp(\n  el: Element,\n  name: string,\n  xmlMode?: boolean,\n): string | undefined | Element[keyof Element] {\n  return name in el\n    ? // @ts-expect-error TS doesn't like us accessing the value directly here.\n      el[name]\n    : !xmlMode && rboolean.test(name)\n      ? getAttr(el, name, false) !== undefined\n      : getAttr(el, name, xmlMode);\n}\n\n/**\n * Sets the value of a prop.\n *\n * @private\n * @param el - The element to set the prop on.\n * @param name - The prop's name.\n * @param value - The prop's value.\n * @param xmlMode - Disable handling of special HTML attributes.\n */\nfunction setProp(el: Element, name: string, value: unknown, xmlMode?: boolean) {\n  if (name in el) {\n    // @ts-expect-error Overriding value\n    el[name] = value;\n  } else {\n    setAttr(\n      el,\n      name,\n      !xmlMode && rboolean.test(name) ? (value ? '' : null) : `${value}`,\n    );\n  }\n}\n\ninterface StyleProp {\n  length: number;\n  [key: string]: string | number;\n  [index: number]: string;\n}\n\n/**\n * Method for getting and setting properties. Gets the property value for only\n * the first element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"checkbox\"]').prop('checked');\n * //=> false\n *\n * $('input[type=\"checkbox\"]').prop('checked', true).val();\n * //=> ok\n * ```\n *\n * @param name - Name of the property.\n * @returns If `value` is specified the instance itself, otherwise the prop's\n *   value.\n * @see {@link https://api.jquery.com/prop/}\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'tagName' | 'nodeName',\n): string | undefined;\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'innerHTML' | 'outerHTML' | 'innerText' | 'textContent',\n): string | null;\n/**\n * Get a parsed CSS style object.\n *\n * @param name - Name of the property.\n * @returns The style object, or `undefined` if the element has no `style`\n *   attribute.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'style',\n): StyleProp | undefined;\n/**\n * Resolve `href` or `src` of supported elements. Requires the `baseURI` option\n * to be set, and a global `URL` object to be part of the environment.\n *\n * @example With `baseURI` set to `'https://example.com'`:\n *\n * ```js\n * $('<img src=\"image.png\">').prop('src');\n * //=> 'https://example.com/image.png'\n * ```\n *\n * @param name - Name of the property.\n * @returns The resolved URL, or `undefined` if the element is not supported.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'href' | 'src',\n): string | undefined;\n/**\n * Get a property of an element.\n *\n * @param name - Name of the property.\n * @returns The property's value.\n */\nexport function prop<T extends AnyNode, K extends keyof Element>(\n  this: Cheerio<T>,\n  name: K,\n): Element[K];\n/**\n * Set a property of an element.\n *\n * @param name - Name of the property.\n * @param value - Value to set the property to.\n * @returns The instance itself.\n */\nexport function prop<T extends AnyNode, K extends keyof Element>(\n  this: Cheerio<T>,\n  name: K,\n  value:\n    | Element[K]\n    | ((this: Element, i: number, prop: K) => Element[keyof Element]),\n): Cheerio<T>;\n/**\n * Set multiple properties of an element.\n *\n * @example\n *\n * ```js\n * $('input[type=\"checkbox\"]').prop({\n *   checked: true,\n *   disabled: false,\n * });\n * ```\n *\n * @param map - Object of properties to set.\n * @returns The instance itself.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  map: Record<string, string | Element[keyof Element] | boolean>,\n): Cheerio<T>;\n/**\n * Set a property of an element.\n *\n * @param name - Name of the property.\n * @param value - Value to set the property to.\n * @returns The instance itself.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value:\n    | string\n    | boolean\n    | null\n    | ((this: Element, i: number, prop: string) => string | boolean),\n): Cheerio<T>;\n/**\n * Get a property of an element.\n *\n * @param name - The property's name.\n * @returns The property's value.\n */\nexport function prop<T extends AnyNode>(this: Cheerio<T>, name: string): string;\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string | Record<string, string | Element[keyof Element] | boolean>,\n  value?:\n    | ((\n        this: Element,\n        i: number,\n        prop: string | undefined,\n      ) => string | Element[keyof Element] | boolean)\n    | unknown,\n): Cheerio<T> | string | undefined | null | Element[keyof Element] | StyleProp {\n  if (typeof name === 'string' && value === undefined) {\n    const el = this[0];\n\n    if (!el || !isTag(el)) return undefined;\n\n    switch (name) {\n      case 'style': {\n        const property = this.css() as StyleProp;\n        const keys = Object.keys(property);\n        for (let i = 0; i < keys.length; i++) {\n          property[i] = keys[i];\n        }\n\n        property.length = keys.length;\n\n        return property;\n      }\n      case 'tagName':\n      case 'nodeName': {\n        return el.name.toUpperCase();\n      }\n\n      case 'href':\n      case 'src': {\n        const prop = el.attribs?.[name];\n\n        /* eslint-disable n/no-unsupported-features/node-builtins */\n        if (\n          typeof URL !== 'undefined' &&\n          ((name === 'href' && (el.tagName === 'a' || el.name === 'link')) ||\n            (name === 'src' &&\n              (el.tagName === 'img' ||\n                el.tagName === 'iframe' ||\n                el.tagName === 'audio' ||\n                el.tagName === 'video' ||\n                el.tagName === 'source'))) &&\n          prop !== undefined &&\n          this.options.baseURI\n        ) {\n          return new URL(prop, this.options.baseURI).href;\n        }\n        /* eslint-enable n/no-unsupported-features/node-builtins */\n\n        return prop;\n      }\n\n      case 'innerText': {\n        return innerText(el);\n      }\n\n      case 'textContent': {\n        return textContent(el);\n      }\n\n      case 'outerHTML': {\n        return this.clone().wrap('<container />').parent().html();\n      }\n\n      case 'innerHTML': {\n        return this.html();\n      }\n\n      default: {\n        return getProp(el, name, this.options.xmlMode);\n      }\n    }\n  }\n\n  if (typeof name === 'object' || value !== undefined) {\n    if (typeof value === 'function') {\n      if (typeof name === 'object') {\n        throw new TypeError('Bad combination of arguments.');\n      }\n      return domEach(this, (el, i) => {\n        if (isTag(el)) {\n          setProp(\n            el,\n            name,\n            value.call(el, i, getProp(el, name, this.options.xmlMode)),\n            this.options.xmlMode,\n          );\n        }\n      });\n    }\n\n    return domEach(this, (el) => {\n      if (!isTag(el)) return;\n\n      if (typeof name === 'object') {\n        for (const key of Object.keys(name)) {\n          const val = name[key];\n          setProp(el, key, val, this.options.xmlMode);\n        }\n      } else {\n        setProp(el, name, value, this.options.xmlMode);\n      }\n    });\n  }\n\n  return undefined;\n}\n\n/**\n * An element with a data attribute.\n *\n * @private\n */\ninterface DataElement extends Element {\n  /** The data attribute. */\n  data?: Record<string, unknown>;\n}\n\n/**\n * Sets the value of a data attribute.\n *\n * @private\n * @param elem - The element to set the data attribute on.\n * @param name - The data attribute's name.\n * @param value - The data attribute's value.\n */\nfunction setData(\n  elem: DataElement,\n  name: string | Record<string, unknown>,\n  value?: unknown,\n) {\n  elem.data ??= {};\n\n  if (typeof name === 'object') Object.assign(elem.data, name);\n  else if (typeof name === 'string' && value !== undefined) {\n    elem.data[name] = value;\n  }\n}\n\n/**\n * Read _all_ HTML5 `data-*` attributes from the equivalent HTML5 `data-*`\n * attribute, and cache the value in the node's internal data store.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the data attribute of.\n * @returns A map with all of the data attributes.\n */\nfunction readAllData(el: DataElement): unknown {\n  for (const domName of Object.keys(el.attribs)) {\n    if (!domName.startsWith(dataAttrPrefix)) {\n      continue;\n    }\n\n    const jsName = camelCase(domName.slice(dataAttrPrefix.length));\n\n    if (!hasOwn.call(el.data, jsName)) {\n      el.data![jsName] = parseDataValue(el.attribs[domName]);\n    }\n  }\n\n  return el.data;\n}\n\n/**\n * Read the specified attribute from the equivalent HTML5 `data-*` attribute,\n * and (if present) cache the value in the node's internal data store.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the data attribute of.\n * @param name - Name of the data attribute.\n * @returns The data attribute's value.\n */\nfunction readData(el: DataElement, name: string): unknown {\n  const domName = dataAttrPrefix + cssCase(name);\n  const data = el.data!;\n\n  if (hasOwn.call(data, name)) {\n    return data[name];\n  }\n\n  if (hasOwn.call(el.attribs, domName)) {\n    return (data[name] = parseDataValue(el.attribs[domName]));\n  }\n\n  return undefined;\n}\n\n/**\n * Coerce string data-* attributes to their corresponding JavaScript primitives.\n *\n * @private\n * @category Attributes\n * @param value - The value to parse.\n * @returns The parsed value.\n */\nfunction parseDataValue(value: string): unknown {\n  if (value === 'null') return null;\n  if (value === 'true') return true;\n  if (value === 'false') return false;\n  const num = Number(value);\n  if (value === String(num)) return num;\n  if (rbrace.test(value)) {\n    try {\n      return JSON.parse(value);\n    } catch {\n      /* Ignore */\n    }\n  }\n  return value;\n}\n\n/**\n * Method for getting data attributes, for only the first element in the matched\n * set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('<div data-apple-color=\"red\"></div>').data('apple-color');\n * //=> 'red'\n * ```\n *\n * @param name - Name of the data attribute.\n * @returns The data attribute's value, or `undefined` if the attribute does not\n *   exist.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): unknown | undefined;\n/**\n * Method for getting all of an element's data attributes, for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('<div data-apple-color=\"red\"></div>').data();\n * //=> { appleColor: 'red' }\n * ```\n *\n * @returns A map with all of the data attributes.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n): Record<string, unknown>;\n/**\n * Method for setting data attributes, for only the first element in the matched\n * set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * const apple = $('.apple').data('kind', 'mac');\n *\n * apple.data('kind');\n * //=> 'mac'\n * ```\n *\n * @param name - Name of the data attribute.\n * @param value - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value: unknown,\n): Cheerio<T>;\n/**\n * Method for setting multiple data attributes at once, for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * const apple = $('.apple').data({ kind: 'mac' });\n *\n * apple.data('kind');\n * //=> 'mac'\n * ```\n *\n * @param values - Map of names to values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  values: Record<string, unknown>,\n): Cheerio<T>;\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name?: string | Record<string, unknown>,\n  value?: unknown,\n): unknown | Cheerio<T> | undefined | Record<string, unknown> {\n  const elem = this[0];\n\n  if (!elem || !isTag(elem)) return;\n\n  const dataEl: DataElement = elem;\n  dataEl.data ??= {};\n\n  // Return the entire data object if no data specified\n  if (name == null) {\n    return readAllData(dataEl);\n  }\n\n  // Set the value (with attr map support)\n  if (typeof name === 'object' || value !== undefined) {\n    domEach(this, (el) => {\n      if (isTag(el)) {\n        if (typeof name === 'object') setData(el, name);\n        else setData(el, name, value as unknown);\n      }\n    });\n    return this;\n  }\n\n  return readData(dataEl, name);\n}\n\n/**\n * Method for getting the value of input, select, and textarea. Note: Support\n * for `map`, and `function` has not been added yet.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"text\"]').val();\n * //=> input_text\n * ```\n *\n * @returns The value.\n * @see {@link https://api.jquery.com/val/}\n */\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n): string | undefined | string[];\n/**\n * Method for setting the value of input, select, and textarea. Note: Support\n * for `map`, and `function` has not been added yet.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"text\"]').val('test').html();\n * //=> <input type=\"text\" value=\"test\"/>\n * ```\n *\n * @param value - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/val/}\n */\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n  value: string | string[],\n): Cheerio<T>;\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n  value?: string | string[],\n): string | string[] | Cheerio<T> | undefined {\n  const querying = arguments.length === 0;\n  const element = this[0];\n\n  if (!element || !isTag(element)) return querying ? undefined : this;\n\n  switch (element.name) {\n    case 'textarea': {\n      return this.text(value as string);\n    }\n    case 'select': {\n      const option = this.find('option:selected');\n      if (!querying) {\n        if (this.attr('multiple') == null && typeof value === 'object') {\n          return this;\n        }\n\n        this.find('option').removeAttr('selected');\n\n        const values = typeof value === 'object' ? value : [value];\n        for (const val of values) {\n          this.find(`option[value=\"${val}\"]`).attr('selected', '');\n        }\n\n        return this;\n      }\n\n      return this.attr('multiple')\n        ? option.toArray().map((el) => text(el.children))\n        : option.attr('value');\n    }\n    case 'input':\n    case 'option': {\n      return querying\n        ? this.attr('value')\n        : this.attr('value', value as string);\n    }\n  }\n\n  return undefined;\n}\n\n/**\n * Remove an attribute.\n *\n * @private\n * @param elem - Node to remove attribute from.\n * @param name - Name of the attribute to remove.\n */\nfunction removeAttribute(elem: Element, name: string) {\n  if (!elem.attribs || !hasOwn.call(elem.attribs, name)) return;\n\n  delete elem.attribs[name];\n}\n\n/**\n * Splits a space-separated list of names to individual names.\n *\n * @category Attributes\n * @param names - Names to split.\n * @returns - Split names.\n */\nfunction splitNames(names?: string): string[] {\n  return names ? names.trim().split(rspace) : [];\n}\n\n/**\n * Method for removing attributes by `name`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').removeAttr('class').html();\n * //=> <li>Pear</li>\n *\n * $('.apple').attr('id', 'favorite');\n * $('.apple').removeAttr('id class').html();\n * //=> <li>Apple</li>\n * ```\n *\n * @param name - Name of the attribute.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/removeAttr/}\n */\nexport function removeAttr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): Cheerio<T> {\n  const attrNames = splitNames(name);\n\n  for (const attrName of attrNames) {\n    domEach(this, (elem) => {\n      if (isTag(elem)) removeAttribute(elem, attrName);\n    });\n  }\n\n  return this;\n}\n\n/**\n * Check to see if _any_ of the matched elements have the given `className`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').hasClass('pear');\n * //=> true\n *\n * $('apple').hasClass('fruit');\n * //=> false\n *\n * $('li').hasClass('pear');\n * //=> true\n * ```\n *\n * @param className - Name of the class.\n * @returns Indicates if an element has the given `className`.\n * @see {@link https://api.jquery.com/hasClass/}\n */\nexport function hasClass<T extends AnyNode>(\n  this: Cheerio<T>,\n  className: string,\n): boolean {\n  return this.toArray().some((elem) => {\n    const clazz = isTag(elem) && elem.attribs['class'];\n    let idx = -1;\n\n    if (clazz && className.length > 0) {\n      while ((idx = clazz.indexOf(className, idx + 1)) > -1) {\n        const end = idx + className.length;\n\n        if (\n          (idx === 0 || rspace.test(clazz[idx - 1])) &&\n          (end === clazz.length || rspace.test(clazz[end]))\n        ) {\n          return true;\n        }\n      }\n    }\n\n    return false;\n  });\n}\n\n/**\n * Adds class(es) to all of the matched elements. Also accepts a `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').addClass('fruit').html();\n * //=> <li class=\"pear fruit\">Pear</li>\n *\n * $('.apple').addClass('fruit red').html();\n * //=> <li class=\"apple fruit red\">Apple</li>\n * ```\n *\n * @param value - Name of new class.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/addClass/}\n */\nexport function addClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  value?:\n    | string\n    | ((this: Element, i: number, className: string) => string | undefined),\n): R {\n  // Support functions\n  if (typeof value === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        const className = el.attribs['class'] || '';\n        addClass.call([el], value.call(el, i, className));\n      }\n    });\n  }\n\n  // Return if no value or not a string or function\n  if (!value || typeof value !== 'string') return this;\n\n  const classNames = value.split(rspace);\n  const numElements = this.length;\n\n  for (let i = 0; i < numElements; i++) {\n    const el = this[i];\n    // If selected element isn't a tag, move on\n    if (!isTag(el)) continue;\n\n    // If we don't already have classes \u2014 always set xmlMode to false here, as it doesn't matter for classes\n    const className = getAttr(el, 'class', false);\n\n    if (className) {\n      let setClass = ` ${className} `;\n\n      // Check if class already exists\n      for (const cn of classNames) {\n        const appendClass = `${cn} `;\n        if (!setClass.includes(` ${appendClass}`)) setClass += appendClass;\n      }\n\n      setAttr(el, 'class', setClass.trim());\n    } else {\n      setAttr(el, 'class', classNames.join(' ').trim());\n    }\n  }\n\n  return this;\n}\n\n/**\n * Removes one or more space-separated classes from the selected elements. If no\n * `className` is defined, all classes will be removed. Also accepts a\n * `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').removeClass('pear').html();\n * //=> <li class=\"\">Pear</li>\n *\n * $('.apple').addClass('red').removeClass().html();\n * //=> <li class=\"\">Apple</li>\n * ```\n *\n * @param name - Name of the class. If not specified, removes all elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/removeClass/}\n */\nexport function removeClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  name?:\n    | string\n    | ((this: Element, i: number, className: string) => string | undefined),\n): R {\n  // Handle if value is a function\n  if (typeof name === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        removeClass.call([el], name.call(el, i, el.attribs['class'] || ''));\n      }\n    });\n  }\n\n  const classes = splitNames(name);\n  const numClasses = classes.length;\n  const removeAll = arguments.length === 0;\n\n  return domEach(this, (el) => {\n    if (!isTag(el)) return;\n\n    if (removeAll) {\n      // Short circuit the remove all case as this is the nice one\n      el.attribs['class'] = '';\n    } else {\n      const elClasses = splitNames(el.attribs['class']);\n      let changed = false;\n\n      for (let j = 0; j < numClasses; j++) {\n        const index = elClasses.indexOf(classes[j]);\n\n        if (index >= 0) {\n          elClasses.splice(index, 1);\n          changed = true;\n\n          /*\n           * We have to do another pass to ensure that there are not duplicate\n           * classes listed\n           */\n          j--;\n        }\n      }\n      if (changed) {\n        el.attribs['class'] = elClasses.join(' ');\n      }\n    }\n  });\n}\n\n/**\n * Add or remove class(es) from the matched elements, depending on either the\n * class's presence or the value of the switch argument. Also accepts a\n * `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple.green').toggleClass('fruit green red').html();\n * //=> <li class=\"apple fruit red\">Apple</li>\n *\n * $('.apple.green').toggleClass('fruit green red', true).html();\n * //=> <li class=\"apple green fruit red\">Apple</li>\n * ```\n *\n * @param value - Name of the class. Can also be a function.\n * @param stateVal - If specified the state of the class.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/toggleClass/}\n */\nexport function toggleClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  value?:\n    | string\n    | ((\n        this: Element,\n        i: number,\n        className: string,\n        stateVal?: boolean,\n      ) => string),\n  stateVal?: boolean,\n): R {\n  // Support functions\n  if (typeof value === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        toggleClass.call(\n          [el],\n          value.call(el, i, el.attribs['class'] || '', stateVal),\n          stateVal,\n        );\n      }\n    });\n  }\n\n  // Return if no value or not a string or function\n  if (!value || typeof value !== 'string') return this;\n\n  const classNames = value.split(rspace);\n  const numClasses = classNames.length;\n  const state = typeof stateVal === 'boolean' ? (stateVal ? 1 : -1) : 0;\n  const numElements = this.length;\n\n  for (let i = 0; i < numElements; i++) {\n    const el = this[i];\n    // If selected element isn't a tag, move on\n    if (!isTag(el)) continue;\n\n    const elementClasses = splitNames(el.attribs['class']);\n\n    // Check if class already exists\n    for (let j = 0; j < numClasses; j++) {\n      // Check if the class name is currently defined\n      const index = elementClasses.indexOf(classNames[j]);\n\n      // Add if stateValue === true or we are toggling and there is no value\n      if (state >= 0 && index < 0) {\n        elementClasses.push(classNames[j]);\n      } else if (state <= 0 && index >= 0) {\n        // Otherwise remove but only if the item exists\n        elementClasses.splice(index, 1);\n      }\n    }\n\n    el.attribs['class'] = elementClasses.join(' ');\n  }\n\n  return this;\n}\n\n// Path: src/api/manipulation.ts\n/**\n * Methods for modifying the DOM structure.\n *\n * @module cheerio/manipulation\n */\n\nimport {\n  isTag,\n  Text,\n  hasChildren,\n  cloneNode,\n  Document,\n  type ParentNode,\n  type AnyNode,\n  type Element,\n} from 'domhandler';\nimport { update as updateDOM } from '../parse.js';\nimport { text as staticText } from '../static.js';\nimport { domEach, isHtml, isCheerio } from '../utils.js';\nimport { removeElement } from 'domutils';\nimport type { Cheerio } from '../cheerio.js';\nimport type { BasicAcceptedElems, AcceptedElems } from '../types.js';\n\n/**\n * Create an array of nodes, recursing into arrays and parsing strings if\n * necessary.\n *\n * @private\n * @category Manipulation\n * @param elem - Elements to make an array of.\n * @param clone - Optionally clone nodes.\n * @returns The array of nodes.\n */\nexport function _makeDomArray<T extends AnyNode>(\n  this: Cheerio<T>,\n  elem?: BasicAcceptedElems<AnyNode> | BasicAcceptedElems<AnyNode>[],\n  clone?: boolean,\n): AnyNode[] {\n  if (elem == null) {\n    return [];\n  }\n\n  if (typeof elem === 'string') {\n    return this._parse(elem, this.options, false, null).children.slice(0);\n  }\n\n  if ('length' in elem) {\n    if (elem.length === 1) {\n      return this._makeDomArray(elem[0], clone);\n    }\n\n    const result: AnyNode[] = [];\n\n    for (let i = 0; i < elem.length; i++) {\n      const el = elem[i];\n\n      if (typeof el === 'object') {\n        if (el == null) {\n          continue;\n        }\n\n        if (!('length' in el)) {\n          result.push(clone ? cloneNode(el, true) : el);\n          continue;\n        }\n      }\n\n      result.push(...this._makeDomArray(el, clone));\n    }\n\n    return result;\n  }\n\n  return [clone ? cloneNode(elem, true) : elem];\n}\n\nfunction _insert(\n  concatenator: (\n    dom: AnyNode[],\n    children: AnyNode[],\n    parent: ParentNode,\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    ...elems:\n      | [\n          (\n            this: AnyNode,\n            i: number,\n            html: string,\n          ) => BasicAcceptedElems<AnyNode>,\n        ]\n      | BasicAcceptedElems<AnyNode>[]\n  ) {\n    const lastIdx = this.length - 1;\n\n    return domEach(this, (el, i) => {\n      if (!hasChildren(el)) return;\n\n      const domSrc =\n        typeof elems[0] === 'function'\n          ? elems[0].call(el, i, this._render(el.children))\n          : (elems as BasicAcceptedElems<AnyNode>[]);\n\n      const dom = this._makeDomArray(domSrc, i < lastIdx);\n      concatenator(dom, el.children, el);\n    });\n  };\n}\n\n/**\n * Modify an array in-place, removing some number of elements and adding new\n * elements directly following them.\n *\n * @private\n * @category Manipulation\n * @param array - Target array to splice.\n * @param spliceIdx - Index at which to begin changing the array.\n * @param spliceCount - Number of elements to remove from the array.\n * @param newElems - Elements to insert into the array.\n * @param parent - The parent of the node.\n * @returns The spliced array.\n */\nfunction uniqueSplice(\n  array: AnyNode[],\n  spliceIdx: number,\n  spliceCount: number,\n  newElems: AnyNode[],\n  parent: ParentNode,\n): AnyNode[] {\n  const spliceArgs: Parameters<typeof Array.prototype.splice> = [\n    spliceIdx,\n    spliceCount,\n    ...newElems,\n  ];\n  const prev = spliceIdx === 0 ? null : array[spliceIdx - 1];\n  const next =\n    spliceIdx + spliceCount >= array.length\n      ? null\n...\n", "needle_token_start": 6815, "needle_token_end": 7170, "code_context_ntokens": 15526, "output": ["```typescript\n/**\n * Method for getting attributes. Gets the attribute value for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr('id');\n * //=> fruits\n * ```\n *\n * @param name - Name of the attribute.\n * @returns The attribute's value.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): string | undefined;\n/**\n * Method for getting all attributes and their values of the first element in\n * the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr();\n * //=> { id: 'fruits' }\n * ```\n *\n * @returns The attribute's values.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n): Record<string, string> | undefined;\n/**\n * Method for setting attributes. Sets the attribute value for only the first\n * element in the matched set. If you set an attribute's value to `null`, you\n * remove that attribute. You may also pass a `map` and `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr('id', 'favorite').html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param name - Name of the attribute.\n * @param value - The new value of the attribute.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): Cheerio<T>;\n/**\n * Method for setting multiple attributes at once. Sets the attribute value for\n * only the first element in the matched set. If you set an attribute's value to\n * `null`, you remove that attribute.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr({ id: 'favorite' }).html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param values - Map of attribute names and values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  values: Record<string, string | null>,\n): Cheerio<T>;\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name?: string | Record<string, string | null>,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): string | Cheerio<T> | undefined | Record<string, string> {\n  // Set the value (with attr map support)\n  if (typeof name === 'object' || value !== undefined) {\n    if (typeof value === 'function') {\n      if (typeof name !== 'string') {\n        {\n          throw new Error('Bad combination of arguments.');\n        }\n      }\n      return domEach(this, (el, i) => {\n        if (isTag(el)) setAttr(el, name, value.call(el, i, el.attribs[name]));\n      });\n    }\n    return domEach(this, (el) => {\n      if (!isTag(el)) return;\n\n      if (typeof name === 'object') {\n        for (const objName of Object.keys(name)) {\n          const objValue = name[objName];\n          setAttr(el, objName, objValue);\n        }\n      } else {\n        setAttr(el, name as string, value as string);\n      }\n    });\n  }\n\n  return arguments.length > 1\n    ? this\n    : getAttr(this[0], name as string, this.options.xmlMode);\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "createTestServer", "language": "typescript", "path": "src/batteries.spec.ts", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to set up a temporary HTTP server for testing purposes, which responds with specified content and content type.\n2. **Input**: It takes two parameters: a string specifying the content type of the response, and the content itself which can be either a string or a Buffer.\n3. **Output**: Returns a promise that resolves with the port number on which the server is listening, allowing the caller to interact with the server at that port.\n4. **Procedure**: The function initializes a server that listens on an available port. It configures the server to respond with the provided content type and body content for any incoming requests. Once the server is successfully listening, it resolves the promise with the port number; if it fails to determine the port, it rejects the promise with an error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/static.spec.ts\nimport * as fixtures from './__fixtures__/fixtures.js';\nimport cheerio, { type CheerioAPI } from './index.js';\n\ndescribe('cheerio', () => {\n  describe('.html', () => {\n    it('() : should return innerHTML; $.html(obj) should return outerHTML', () => {\n      const $div = cheerio(\n        'div',\n        '<div><span>foo</span><span>bar</span></div>',\n      );\n      const span = $div.children()[1];\n      expect(cheerio(span).html()).toBe('bar');\n      expect(cheerio.html(span)).toBe('<span>bar</span>');\n    });\n\n    it('(<obj>) : should accept an object, an array, or a cheerio object', () => {\n      const $span = cheerio('<span>foo</span>');\n      expect(cheerio.html($span[0])).toBe('<span>foo</span>');\n      expect(cheerio.html($span)).toBe('<span>foo</span>');\n    });\n\n    it('(<value>) : should be able to set to an empty string', () => {\n      const $elem = cheerio('<span>foo</span>').html('');\n      expect(cheerio.html($elem)).toBe('<span></span>');\n    });\n\n    it('(<root>) : does not render the root element', () => {\n      const $ = cheerio.load('');\n      expect(cheerio.html($.root())).toBe(\n        '<html><head></head><body></body></html>',\n      );\n    });\n\n    it('(<elem>, <root>, <elem>) : does not render the root element', () => {\n      const $ = cheerio.load('<div>a div</div><span>a span</span>');\n      const $collection = $('div').add($.root()).add('span');\n      const expected =\n        '<html><head></head><body><div>a div</div><span>a span</span></body></html><div>a div</div><span>a span</span>';\n      expect(cheerio.html($collection)).toBe(expected);\n    });\n\n    it('() : does not crash with `null` as `this` value', () => {\n      const { html } = cheerio;\n      expect(html.call(null as never)).toBe('');\n      expect(html.call(null as never, '#nothing')).toBe('');\n    });\n  });\n\n  describe('.text', () => {\n    it('(cheerio object) : should return the text contents of the specified elements', () => {\n      const $ = cheerio.load('<a>This is <em>content</em>.</a>');\n      expect(cheerio.text($('a'))).toBe('This is content.');\n    });\n\n    it('(cheerio object) : should omit comment nodes', () => {\n      const $ = cheerio.load(\n        '<a>This is <!-- a comment --> not a comment.</a>',\n      );\n      expect(cheerio.text($('a'))).toBe('This is  not a comment.');\n    });\n\n    it('(cheerio object) : should include text contents of children recursively', () => {\n      const $ = cheerio.load(\n        '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n      );\n      expect(cheerio.text($('a'))).toBe(\n        'This is a child with another child and  not a comment followed by one last child and some final text.',\n      );\n    });\n\n    it('() : should return the rendered text content of the root', () => {\n      const $ = cheerio.load(\n        '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n      );\n      expect(cheerio.text($.root())).toBe(\n        'This is a child with another child and  not a comment followed by one last child and some final text.',\n      );\n    });\n\n    it('(cheerio object) : should not omit script tags', () => {\n      const $ = cheerio.load('<script>console.log(\"test\")</script>');\n      expect(cheerio.text($.root())).toBe('console.log(\"test\")');\n    });\n\n    it('(cheerio object) : should omit style tags', () => {\n      const $ = cheerio.load(\n        '<style type=\"text/css\">.cf-hidden { display: none; }</style>',\n      );\n      expect($.text()).toBe('.cf-hidden { display: none; }');\n    });\n\n    it('() : does not crash with `null` as `this` value', () => {\n      const { text } = cheerio;\n      expect(text.call(null as never)).toBe('');\n    });\n  });\n\n  describe('.parseHTML', () => {\n    const $ = cheerio.load('');\n\n    it('() : returns null', () => {\n      expect($.parseHTML()).toBe(null);\n    });\n\n    it('(null) : returns null', () => {\n      expect($.parseHTML(null)).toBe(null);\n    });\n\n    it('(\"\") : returns null', () => {\n      expect($.parseHTML('')).toBe(null);\n    });\n\n    it('(largeHtmlString) : parses large HTML strings', () => {\n      const html = '<div></div>'.repeat(10);\n      const nodes = $.parseHTML(html);\n\n      expect(nodes.length).toBe(10);\n      expect(nodes).toBeInstanceOf(Array);\n    });\n\n    it('(\"<script>\") : ignores scripts by default', () => {\n      const html = '<script>undefined()</script>';\n      expect($.parseHTML(html)).toHaveLength(0);\n    });\n\n    it('(\"<script>\", true) : preserves scripts when requested', () => {\n      const html = '<script>undefined()</script>';\n      expect($.parseHTML(html, true)[0]).toHaveProperty('tagName', 'script');\n    });\n\n    it('(\"scriptAndNonScript) : preserves non-script nodes', () => {\n      const html = '<script>undefined()</script><div></div>';\n      expect($.parseHTML(html)[0]).toHaveProperty('tagName', 'div');\n    });\n\n    it('(scriptAndNonScript, true) : Preserves script position', () => {\n      const html = '<script>undefined()</script><div></div>';\n      expect($.parseHTML(html, true)[0]).toHaveProperty('tagName', 'script');\n    });\n\n    it('(text) : returns a text node', () => {\n      expect($.parseHTML('text')[0].type).toBe('text');\n    });\n\n    it('(\\\\ttext) : preserves leading whitespace', () => {\n      expect($.parseHTML('\\t<div></div>')[0]).toHaveProperty('data', '\\t');\n    });\n\n    it('( text) : Leading spaces are treated as text nodes', () => {\n      expect($.parseHTML(' <div/> ')[0].type).toBe('text');\n    });\n\n    it('(html) : should preserve content', () => {\n      const html = '<div>test div</div>';\n      expect(cheerio($.parseHTML(html)[0]).html()).toBe('test div');\n    });\n\n    it('(malformedHtml) : should not break', () => {\n      expect($.parseHTML('<span><span>')).toHaveLength(1);\n    });\n\n    it('(garbageInput) : should not cause an error', () => {\n      expect(\n        $.parseHTML('<#if><tr><p>This is a test.</p></tr><#/if>'),\n      ).toBeTruthy();\n    });\n\n    it('(text) : should return an array that is not effected by DOM manipulation methods', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML('<b></b><i></i>');\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(2);\n    });\n\n    it('(html, context) : should ignore context argument', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML('<script>foo</script><a>', { foo: 123 });\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(1);\n    });\n\n    it('(html, context, keepScripts) : should ignore context argument', () => {\n      const $div = cheerio.load('<div>');\n      const elems = $div.parseHTML(\n        '<script>foo</script><a>',\n        { foo: 123 },\n        true,\n      );\n\n      $div('div').append(elems);\n\n      expect(elems).toHaveLength(2);\n    });\n  });\n\n  describe('.merge', () => {\n    const $ = cheerio.load('');\n\n    it('should be a function', () => {\n      expect(typeof $.merge).toBe('function');\n    });\n\n    it('(arraylike, arraylike) : should modify the first array, but not the second', () => {\n      const arr1 = [1, 2, 3];\n      const arr2 = [4, 5, 6];\n\n      const ret = $.merge(arr1, arr2);\n      expect(typeof ret).toBe('object');\n      expect(Array.isArray(ret)).toBe(true);\n      expect(ret).toBe(arr1);\n      expect(arr1).toHaveLength(6);\n      expect(arr2).toHaveLength(3);\n    });\n\n    it('(arraylike, arraylike) : should handle objects that arent arrays, but are arraylike', () => {\n      const arr1: ArrayLike<string> = {\n        length: 3,\n        0: 'a',\n        1: 'b',\n        2: 'c',\n      };\n      const arr2 = {\n        length: 3,\n        0: 'd',\n        1: 'e',\n        2: 'f',\n      };\n\n      $.merge(arr1, arr2);\n      expect(arr1).toHaveLength(6);\n      expect(arr1[3]).toBe('d');\n      expect(arr1[4]).toBe('e');\n      expect(arr1[5]).toBe('f');\n      expect(arr2).toHaveLength(3);\n    });\n\n...\n// Path: src/cheerio.spec.ts\nimport { parseDOM } from 'htmlparser2';\nimport cheerio, { type Cheerio } from './index.js';\nimport { fruits, food, noscript } from './__fixtures__/fixtures.js';\nimport type { Element } from 'domhandler';\n\ndeclare module './index.js' {\n  interface Cheerio<T> {\n    myPlugin(...args: unknown[]): {\n      context: Cheerio<T>;\n      args: unknown[];\n    };\n    foo(): void;\n  }\n}\n\nfunction testAppleSelect($apple: ArrayLike<Element>) {\n  expect($apple).toHaveLength(1);\n  const apple = $apple[0];\n  expect(apple.parentNode).toHaveProperty('tagName', 'ul');\n  expect(apple.prev).toBe(null);\n  expect((apple.next as Element).attribs).toHaveProperty('class', 'orange');\n  expect(apple.childNodes).toHaveLength(1);\n  expect(apple.childNodes[0]).toHaveProperty('data', 'Apple');\n}\n\ndescribe('cheerio', () => {\n  it('cheerio(null) should be empty', () => {\n    expect(cheerio(null as never)).toHaveLength(0);\n  });\n\n  it('cheerio(undefined) should be empty', () => {\n    expect(cheerio(undefined)).toHaveLength(0);\n  });\n\n  it(\"cheerio('') should be empty\", () => {\n    expect(cheerio('')).toHaveLength(0);\n  });\n\n  it('cheerio(selector) with no context or root should be empty', () => {\n    expect(cheerio('.h2')).toHaveLength(0);\n    expect(cheerio('#fruits')).toHaveLength(0);\n  });\n\n  it('cheerio(node) : should override previously-loaded nodes', () => {\n    const $ = cheerio.load('<div><span></span></div>');\n    const spanNode = $('span')[0];\n    const $span = $(spanNode);\n    expect($span[0]).toBe(spanNode);\n  });\n\n  it('should be able to create html without a root or context', () => {\n    const $h2 = cheerio('<h2>');\n    expect($h2).not.toHaveLength(0);\n    expect($h2).toHaveLength(1);\n    expect($h2[0]).toHaveProperty('tagName', 'h2');\n  });\n\n  it('should be able to create complicated html', () => {\n    const $script = cheerio(\n      '<script src=\"script.js\" type=\"text/javascript\"></script>',\n    ) as Cheerio<Element>;\n    expect($script).not.toHaveLength(0);\n    expect($script).toHaveLength(1);\n    expect($script[0].attribs).toHaveProperty('src', 'script.js');\n    expect($script[0].attribs).toHaveProperty('type', 'text/javascript');\n    expect($script[0].childNodes).toHaveLength(0);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with only a context', () => {\n    const $apple = cheerio('.apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with a node as context', () => {\n    const $apple = cheerio('.apple', cheerio(fruits)[0]);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select .apple with only a root', () => {\n    const $apple = cheerio('.apple', null, fruits);\n    testAppleSelect($apple);\n  });\n\n  it('should be able to select an id', () => {\n    const $fruits = cheerio('#fruits', null, fruits);\n    expect($fruits).toHaveLength(1);\n    expect($fruits[0].attribs).toHaveProperty('id', 'fruits');\n  });\n\n  it('should be able to select a tag', () => {\n    const $ul = cheerio('ul', fruits);\n    expect($ul).toHaveLength(1);\n    expect($ul[0].tagName).toBe('ul');\n  });\n\n  it('should accept a node reference as a context', () => {\n    const $elems = cheerio('<div><span></span></div>');\n    expect(cheerio('span', $elems[0])).toHaveLength(1);\n  });\n\n  it('should accept an array of node references as a context', () => {\n    const $elems = cheerio('<div><span></span></div>');\n    expect(cheerio('span', $elems.toArray())).toHaveLength(1);\n  });\n\n  it('should select only elements inside given context (Issue #193)', () => {\n    const $ = cheerio.load(food);\n    const $fruits = $('#fruits');\n    const fruitElements = $('li', $fruits);\n\n    expect(fruitElements).toHaveLength(3);\n  });\n\n  it('should be able to select multiple tags', () => {\n    const $fruits = cheerio('li', null, fruits);\n    expect($fruits).toHaveLength(3);\n    const classes = ['apple', 'orange', 'pear'];\n    $fruits.each((idx, $fruit) => {\n      expect($fruit.attribs).toHaveProperty('class', classes[idx]);\n    });\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to do: cheerio(\"#fruits .apple\")', () => {\n    const $apple = cheerio('#fruits .apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to do: cheerio(\"li.apple\")', () => {\n    const $apple = cheerio('li.apple', fruits);\n    testAppleSelect($apple);\n  });\n\n  // eslint-disable-next-line jest/expect-expect\n  it('should be able to select by attributes', () => {\n    const $apple = cheerio('li[class=apple]', fruits);\n    testAppleSelect($apple);\n  });\n\n  it('should be able to select multiple classes: cheerio(\".btn.primary\")', () => {\n    const $a = cheerio(\n      '.btn.primary',\n      '<p><a class=\"btn primary\" href=\"#\">Save</a></p>',\n    );\n    expect($a).toHaveLength(1);\n    expect($a[0].childNodes[0]).toHaveProperty('data', 'Save');\n  });\n\n  it('should not create a top-level node', () => {\n    const $elem = cheerio('* div', '<div>');\n    expect($elem).toHaveLength(0);\n  });\n\n  it('should be able to select multiple elements: cheerio(\".apple, #fruits\")', () => {\n    const $elems = cheerio('.apple, #fruits', fruits);\n    expect($elems).toHaveLength(2);\n\n    const $apple = $elems\n      .toArray()\n      .filter((elem) => elem.attribs['class'] === 'apple');\n    const $fruit = $elems\n      .toArray()\n      .find((elem) => elem.attribs['id'] === 'fruits');\n    testAppleSelect($apple);\n    expect($fruit?.attribs).toHaveProperty('id', 'fruits');\n  });\n\n  it('should select first element cheerio(:first)', () => {\n    const $elem = cheerio('li:first', fruits);\n    expect($elem.attr('class')).toBe('apple');\n\n    const $filtered = cheerio('li', fruits).filter(':even');\n    expect($filtered).toHaveLength(2);\n  });\n\n  it('should be able to select immediate children: cheerio(\"#fruits > .pear\")', () => {\n    const $food = cheerio(food);\n    cheerio('.pear', $food).append('<li class=\"pear\">Another Pear!</li>');\n    expect(cheerio('#fruits .pear', $food)).toHaveLength(2);\n    const $elem = cheerio('#fruits > .pear', $food);\n    expect($elem).toHaveLength(1);\n    expect($elem.attr('class')).toBe('pear');\n  });\n\n  it('should be able to select immediate children: cheerio(\".apple + .pear\")', () => {\n    expect(cheerio('.apple + li', fruits)).toHaveLength(1);\n    expect(cheerio('.apple + .pear', fruits)).toHaveLength(0);\n    const $elem = cheerio('.apple + .orange', fruits);\n    expect($elem).toHaveLength(1);\n    expect($elem.attr('class')).toBe('orange');\n  });\n\n  it('should be able to select immediate children: cheerio(\".apple ~ .pear\")', () => {\n    expect(cheerio('.apple ~ li', fruits)).toHaveLength(2);\n    expect(cheerio('.apple ~ .pear', fruits).attr('class')).toBe('pear');\n  });\n\n  it('should handle wildcards on attributes: cheerio(\"li[class*=r]\")', () => {\n    const $elem = cheerio('li[class*=r]', fruits);\n    expect($elem).toHaveLength(2);\n    expect($elem.eq(0).attr('class')).toBe('orange');\n    expect($elem.eq(1).attr('class')).toBe('pear');\n  });\n\n  it('should handle beginning of attr selectors: cheerio(\"li[class^=o]\")', () => {\n    const $elem = cheerio('li[class^=o]', fruits);\n    expect($elem).toHaveLength(1);\n    expect($elem.eq(0).attr('class')).toBe('orange');\n  });\n\n  it('should handle beginning of attr selectors: cheerio(\"li[class$=e]\")', () => {\n    const $elem = cheerio('li[class$=e]', fruits);\n    expect($elem).toHaveLength(2);\n    expect($elem.eq(0).attr('class')).toBe('apple');\n    expect($elem.eq(1).attr('class')).toBe('orange');\n  });\n\n  it('(extended Array) should not interfere with prototype methods (issue #119)', () => {\n    const extended: any = [];\n    extended.find =\n      extended.children =\n      extended.each =\n        function () {\n          /* Ignore */\n        };\n    const $empty = cheerio(extended);\n\n    expect($empty.find).toBe(cheerio.prototype.find);\n    expect($empty.children).toBe(cheerio.prototype.children);\n    expect($empty.each).toBe(cheerio.prototype.each);\n  });\n\n  it('cheerio.html(null) should return a \"\" string', () => {\n    expect(cheerio.html(null as never)).toBe('');\n  });\n\n  it('should set html(number) as a string', () => {\n    const $elem = cheerio('<div>');\n    $elem.html(123 as never);\n    expect(typeof $elem.text()).toBe('string');\n  });\n\n  it('should set text(number) as a string', () => {\n    const $elem = cheerio('<div>');\n    $elem.text(123 as never);\n    expect(typeof $elem.text()).toBe('string');\n  });\n\n  describe('.load', () => {\n    it('should generate selections as proper instances', () => {\n      const $ = cheerio.load(fruits);\n\n      expect($('.apple')).toBeInstanceOf($);\n    });\n\n    // Issue #1092\n    it('should handle a character `)` in `:contains` selector', () => {\n      const result = cheerio.load('<p>)aaa</p>')(\":contains('\\\\)aaa')\");\n      expect(result).toHaveLength(3);\n      expect(result.first().prop('tagName')).toBe('HTML');\n      expect(result.eq(1).prop('tagName')).toBe('BODY');\n      expect(result.last().prop('tagName')).toBe('P');\n    });\n\n    it('should be able to filter down using the context', () => {\n      const $ = cheerio.load(fruits);\n      const apple = $('.apple', 'ul');\n      const lis = $('li', 'ul');\n\n      expect(apple).toHaveLength(1);\n      expect(lis).toHaveLength(3);\n    });\n\n    it('should preserve root content', () => {\n      const $ = cheerio.load(fruits);\n      // Root should not be overwritten\n      const el = $('<div></div>');\n      expect(Object.is(el, el._root)).toBe(false);\n      // Query has to have results\n      expect($('li', 'ul')).toHaveLength(3);\n    });\n\n    it('should allow loading a pre-parsed DOM', () => {\n      const dom = parseDOM(food);\n      const $ = cheerio.load(dom);\n\n      expect($('ul')).toHaveLength(3);\n    });\n\n    it('should allow loading a single element', () => {\n      const el = parseDOM(food)[0];\n      const $ = cheerio.load(el);\n\n      expect($('ul')).toHaveLength(3);\n    });\n\n    it('should render xml in html() when options.xml = true', () => {\n      const str = '<MixedCaseTag UPPERCASEATTRIBUTE=\"\"></MixedCaseTag>';\n      const expected = '<MixedCaseTag UPPERCASEATTRIBUTE=\"\"/>';\n      const $ = cheerio.load(str, { xml: true });\n\n      expect($('MixedCaseTag').get(0)).toHaveProperty(\n        'tagName',\n        'MixedCaseTag',\n      );\n      expect($.html()).toBe(expected);\n    });\n\n    it('should render xml in html() when options.xml = true passed to html()', () => {\n      const str = '<MixedCaseTag UPPERCASEATTRIBUTE=\"\"></MixedCaseTag>';\n      // Since parsing done without xml flag, all tags converted to lowercase\n      const expectedXml =\n        '<html><head/><body><mixedcasetag uppercaseattribute=\"\"/></body></html>';\n      const expectedNoXml =\n        '<html><head></head><body><mixedcasetag uppercaseattribute=\"\"></mixedcasetag></body></html>';\n      const $ = cheerio.load(str);\n\n      expect($('MixedCaseTag').get(0)).toHaveProperty(\n        'tagName',\n        'mixedcasetag',\n      );\n      expect($.html()).toBe(expectedNoXml);\n      expect($.html({ xml: true })).toBe(expectedXml);\n    });\n\n    it('should respect options on the element level', () => {\n      const str =\n        '<!doctype html><html><head><title>Some test</title></head><body><footer><p>Copyright &copy; 2003-2014</p></footer></body></html>';\n      const expectedHtml = '<p>Copyright &copy; 2003-2014</p>';\n      const expectedXml = '<p>Copyright \u00a9 2003-2014</p>';\n      const domNotEncoded = cheerio.load(str, {\n        xml: { decodeEntities: false },\n      });\n      const domEncoded = cheerio.load(str);\n\n      expect(domNotEncoded('footer').html()).toBe(expectedHtml);\n      expect(domEncoded('footer').html()).toBe(expectedXml);\n    });\n\n    it('should use htmlparser2 if xml option is used', () => {\n      const str = '<div></div>';\n      const dom = cheerio.load(str, null, false);\n      expect(dom.html()).toBe(str);\n    });\n\n    it('should return a fully-qualified Function', () => {\n      const $ = cheerio.load('<div>');\n\n      expect($).toBeInstanceOf(Function);\n    });\n\n    describe('prototype extensions', () => {\n      it('should honor extensions defined on `prototype` property', () => {\n        const $ = cheerio.load('<div>');\n\n        $.prototype.myPlugin = function (...args: unknown[]) {\n          return {\n            context: this,\n            args,\n          };\n        };\n\n        const $div = $('div');\n\n        expect(typeof $div.myPlugin).toBe('function');\n        expect($div.myPlugin().context).toBe($div);\n        expect($div.myPlugin(1, 2, 3).args).toStrictEqual([1, 2, 3]);\n      });\n\n      it('should honor extensions defined on `fn` property', () => {\n        const $ = cheerio.load('<div>');\n        $.fn.myPlugin = function (...args: unknown[]) {\n          return {\n            context: this,\n            args,\n          };\n        };\n\n        const $div = $('div');\n\n        expect(typeof $div.myPlugin).toBe('function');\n        expect($div.myPlugin().context).toBe($div);\n        expect($div.myPlugin(1, 2, 3).args).toStrictEqual([1, 2, 3]);\n      });\n\n      it('should isolate extensions between loaded functions', () => {\n        const $a = cheerio.load('<div>');\n        const $b = cheerio.load('<div>');\n\n        $a.prototype.foo = function () {\n          /* Ignore */\n        };\n\n        expect($b('div').foo).toBeUndefined();\n      });\n    });\n  });\n\n  describe('parse5 options', () => {\n    // Should parse noscript tags only with false option value\n    test('{scriptingEnabled: ???}', () => {\n      // [default] `scriptingEnabled: true` - tag contains one text element\n      const withScripts = cheerio.load(noscript)('noscript');\n      expect(withScripts).toHaveLength(1);\n      expect(withScripts[0].children).toHaveLength(1);\n      expect(withScripts[0].children[0].type).toBe('text');\n\n      // `scriptingEnabled: false` - content of noscript will parsed\n      const noScripts = cheerio.load(noscript, { scriptingEnabled: false })(\n        'noscript',\n      );\n      expect(noScripts).toHaveLength(1);\n      expect(noScripts[0].children).toHaveLength(2);\n      expect(noScripts[0].children[0].type).toBe('comment');\n      expect(noScripts[0].children[1].type).toBe('tag');\n      expect(noScripts[0].children[1]).toHaveProperty('name', 'a');\n\n      // `scriptingEnabled: ???` - should acts as true\n      for (const val of [undefined, null, 0, '']) {\n        const options = { scriptingEnabled: val as never };\n        const result = cheerio.load(noscript, options)('noscript');\n        expect(result).toHaveLength(1);\n        expect(result[0].children).toHaveLength(1);\n        expect(result[0].children[0].type).toBe('text');\n      }\n    });\n\n    // Should contain location data only with truthful option value\n    test('{sourceCodeLocationInfo: ???}', () => {\n      // Location data should not be present\n      for (const val of [undefined, null, 0, false, '']) {\n        const options = { sourceCodeLocationInfo: val as never };\n        const result = cheerio.load(noscript, options)('noscript');\n        expect(result).toHaveLength(1);\n        expect(result[0]).not.toHaveProperty('sourceCodeLocation');\n      }\n\n      // Location data should be present\n      for (const val of [true, 1, 'test']) {\n        const options = { sourceCodeLocationInfo: val as never };\n        const result = cheerio.load(noscript, options)('noscript');\n        expect(result).toHaveLength(1);\n        expect(result[0]).toHaveProperty('sourceCodeLocation');\n        expect(typeof result[0].sourceCodeLocation).toBe('object');\n      }\n    });\n  });\n});\n\n// Path: src/batteries.spec.ts\n/* eslint-disable jest/no-done-callback */\nimport * as cheerio from './batteries.js';\nimport { Writable } from 'node:stream';\nimport { createServer, type Server } from 'node:http';\n\nfunction noop() {\n  // Ignore\n}\n\nconst TEST_HTML = '<h1>Hello World</h1>';\nconst TEST_HTML_UTF16 = Buffer.from(TEST_HTML, 'utf16le');\nconst TEST_HTML_UTF16_BOM = Buffer.from([\n  // UTF16-LE BOM\n  0xff,\n  0xfe,\n  ...Array.from(TEST_HTML_UTF16),\n]);\n\ndescribe('loadBuffer', () => {\n  it('should parse UTF-8 HTML', () => {\n    const $ = cheerio.loadBuffer(Buffer.from(TEST_HTML));\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n\n  it('should parse UTF-16 HTML', () => {\n    const $ = cheerio.loadBuffer(TEST_HTML_UTF16_BOM);\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n});\n\ndescribe('stringStream', () => {\n  it('should use parse5 by default', (cb) => {\n    const stream = cheerio.stringStream({}, (err, $) => {\n      expect(err).toBeUndefined();\n\n      expect($.html()).toBe(\n        `<html><head></head><body>${TEST_HTML}</body></html>`,\n      );\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML);\n  });\n\n  it('should error from parse5 on buffer', () => {\n    const stream = cheerio.stringStream({}, noop);\n    expect(stream).toBeInstanceOf(Writable);\n\n    expect(() => stream.write(Buffer.from(TEST_HTML))).toThrow(\n      'Parser can work only with string streams.',\n    );\n  });\n\n  it('should use htmlparser2 for XML', (cb) => {\n    const stream = cheerio.stringStream({ xml: true }, (err, $) => {\n      expect(err).toBeNull();\n\n      expect($.html()).toBe(TEST_HTML);\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML);\n  });\n});\n\ndescribe('decodeStream', () => {\n  it('should use parse5 by default', (cb) => {\n    const stream = cheerio.decodeStream({}, (err, $) => {\n      expect(err).toBeUndefined();\n\n      expect($.html()).toBe(\n        `<html><head></head><body>${TEST_HTML}</body></html>`,\n      );\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML_UTF16_BOM);\n  });\n\n  it('should use htmlparser2 for XML', (cb) => {\n    const stream = cheerio.decodeStream({ xmlMode: true }, (err, $) => {\n      expect(err).toBeNull();\n\n      expect($.html()).toBe(TEST_HTML);\n\n      cb();\n    });\n    expect(stream).toBeInstanceOf(Writable);\n\n    stream.end(TEST_HTML_UTF16_BOM);\n  });\n});\n\ndescribe('fromURL', () => {\n  let server: Server | undefined;\n\n  \nfunction createTestServer(\n    contentType: string,\n    body: string | Buffer,\n  ): Promise<number> {\n    return new Promise((resolve, reject) => {\n      server = createServer((_req, res) => {\n        res.writeHead(200, { 'Content-Type': contentType });\n        res.end(body);\n      });\n\n      server.listen(0, () => {\n        const address = server?.address();\n\n        if (typeof address === 'string' || address == null) {\n          reject(new Error('Failed to get port'));\n        } else {\n          resolve(address.port);\n        }\n      });\n    });\n  }\n\n  afterEach((cb) => {\n    if (server) {\n      server.close(cb);\n      server = undefined;\n    } else {\n      cb();\n    }\n  });\n\n  it('should fetch UTF-8 HTML', async () => {\n    const port = await createTestServer('text/html', TEST_HTML);\n    const $ = await cheerio.fromURL(`http://localhost:${port}`);\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n\n  it('should fetch UTF-16 HTML', async () => {\n    const port = await createTestServer(\n      'text/html; charset=utf-16le',\n      TEST_HTML_UTF16,\n    );\n    const $ = await cheerio.fromURL(`http://localhost:${port}`);\n\n    expect($.html()).toBe(\n      `<html><head></head><body>${TEST_HTML}</body></html>`,\n    );\n  });\n\n  it('should parse XML based on Content-Type', async () => {\n    const port = await createTestServer('text/xml', TEST_HTML);\n    const $ = await cheerio.fromURL(`http://localhost:${port}`);\n\n    expect($.html()).toBe(TEST_HTML);\n  });\n});\n\n// Path: src/slim.ts\n/**\n * @file Alternative entry point for Cheerio that always uses htmlparser2. This\n *   way, parse5 won't be loaded, saving some memory.\n */\n\nexport type {\n  Cheerio,\n  CheerioAPI,\n  CheerioOptions,\n  HTMLParser2Options,\n  Node,\n  AnyNode,\n  ParentNode,\n  Element,\n  Document,\n} from './index.js';\n\n/**\n * Types used in signatures of Cheerio methods.\n *\n * @category Cheerio\n */\nexport * from './types.js';\n\nimport { getLoad } from './load.js';\nimport { getParse } from './parse.js';\nimport render from 'dom-serializer';\nimport { parseDocument } from 'htmlparser2';\n\n/**\n * Create a querying function, bound to a document created from the provided\n * markup.\n *\n * @param content - Markup to be loaded.\n * @param options - Options for the created instance.\n * @param isDocument - Always `false` here, as we are always using\n *   `htmlparser2`.\n * @returns The loaded document.\n * @see {@link https://cheerio.js.org#loading} for additional usage information.\n */\nexport const load = getLoad(getParse(parseDocument), render);\n\n// Path: src/index.spec.ts\nimport * as cheerio from './index.js';\nimport * as statics from './static.js';\n\ndescribe('static method re-exports', () => {\n  it('should export expected static methods', () => {\n    for (const key of Object.keys(statics) as (keyof typeof statics)[]) {\n      if (key === 'extract') continue;\n      expect(typeof cheerio[key]).toBe(typeof statics[key]);\n    }\n  });\n\n  it('should have a functional `html` that is bound to the default instance', () => {\n    expect(cheerio.html(cheerio.default('<div>test div</div>'))).toBe(\n      '<div>test div</div>',\n    );\n  });\n\n  it('should have a functional `xml` that is bound to the default instance', () => {\n    expect(cheerio.xml(cheerio.default('<div>test div</div>'))).toBe(\n      '<div>test div</div>',\n    );\n  });\n\n  it('should have a functional `text` that is bound to the default instance', () => {\n    expect(cheerio.text(cheerio.default('<div>test div</div>'))).toBe(\n      'test div',\n    );\n  });\n});\n\n// Path: src/parse.spec.ts\nimport type { Document, Element } from 'domhandler';\nimport { getParse } from './parse.js';\n\nimport { parseDocument as parseWithHtmlparser2 } from 'htmlparser2';\nimport { parseWithParse5 } from './parsers/parse5-adapter.js';\n\nconst defaultOpts = { _useHtmlParser2: false };\n\nconst parse = getParse((content, options, isDocument, context) =>\n  options._useHtmlParser2\n    ? parseWithHtmlparser2(content, options)\n    : parseWithParse5(content, options, isDocument, context),\n);\n\n// Tags\nconst basic = '<html></html>';\nconst siblings = '<h2></h2><p></p>';\n\n// Single Tags\nconst single = '<br/>';\nconst singleWrong = '<br>';\n\n// Children\nconst children = '<html><br/></html>';\nconst li = '<li class=\"durian\">Durian</li>';\n\n// Attributes\nconst attributes = '<img src=\"hello.png\" alt=\"man waving\">';\nconst noValueAttribute = '<textarea disabled></textarea>';\n\n// Comments\nconst comment = '<!-- sexy -->';\nconst conditional =\n  '<!--[if IE 8]><html class=\"no-js ie8\" lang=\"en\"><![endif]-->';\n\n// Text\nconst text = 'lorem ipsum';\n\n// Script\nconst script = '<script type=\"text/javascript\">alert(\"hi world!\");</script>';\nconst scriptEmpty = '<script></script>';\n\n// Style\nconst style = '<style type=\"text/css\"> h2 { color:blue; } </style>';\nconst styleEmpty = '<style></style>';\n\n// Directives\nconst directive = '<!doctype html>';\n\nfunction rootTest(root: Document) {\n  expect(root).toHaveProperty('type', 'root');\n\n  expect(root.nextSibling).toBe(null);\n  expect(root.previousSibling).toBe(null);\n  expect(root.parentNode).toBe(null);\n\n  const child = root.childNodes[0];\n  expect(child.parentNode).toBe(root);\n}\n\ndescribe('parse', () => {\n  describe('evaluate', () => {\n    it(`should parse basic empty tags: ${basic}`, () => {\n      const [tag] = parse(basic, defaultOpts, true, null).children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('html');\n      expect(tag.childNodes).toHaveLength(2);\n    });\n\n    it(`should handle sibling tags: ${siblings}`, () => {\n      const dom = parse(siblings, defaultOpts, false, null)\n        .children as Element[];\n      const [h2, p] = dom;\n\n      expect(dom).toHaveLength(2);\n      expect(h2.tagName).toBe('h2');\n      expect(p.tagName).toBe('p');\n    });\n\n    it(`should handle single tags: ${single}`, () => {\n      const [tag] = parse(single, defaultOpts, false, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('br');\n      expect(tag.childNodes).toHaveLength(0);\n    });\n\n    it(`should handle malformatted single tags: ${singleWrong}`, () => {\n      const [tag] = parse(singleWrong, defaultOpts, false, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('br');\n      expect(tag.childNodes).toHaveLength(0);\n    });\n\n    it(`should handle tags with children: ${children}`, () => {\n      const [tag] = parse(children, defaultOpts, true, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('html');\n      expect(tag.childNodes).toBeTruthy();\n      expect(tag.childNodes[1]).toHaveProperty('tagName', 'body');\n      expect((tag.childNodes[1] as Element).childNodes).toHaveLength(1);\n    });\n\n    it(`should handle tags with children: ${li}`, () => {\n      const [tag] = parse(li, defaultOpts, false, null).children as Element[];\n      expect(tag.childNodes).toHaveLength(1);\n      expect(tag.childNodes[0]).toHaveProperty('data', 'Durian');\n    });\n\n    it(`should handle tags with attributes: ${attributes}`, () => {\n      const attrs = parse(attributes, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(attrs.attribs).toBeTruthy();\n      expect(attrs.attribs).toHaveProperty('src', 'hello.png');\n      expect(attrs.attribs).toHaveProperty('alt', 'man waving');\n    });\n\n    it(`should handle value-less attributes: ${noValueAttribute}`, () => {\n      const attrs = parse(noValueAttribute, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(attrs.attribs).toBeTruthy();\n      expect(attrs.attribs).toHaveProperty('disabled', '');\n    });\n\n    it(`should handle comments: ${comment}`, () => {\n      const elem = parse(comment, defaultOpts, false, null).children[0];\n      expect(elem.type).toBe('comment');\n      expect(elem).toHaveProperty('data', ' sexy ');\n    });\n\n    it(`should handle conditional comments: ${conditional}`, () => {\n      const elem = parse(conditional, defaultOpts, false, null).children[0];\n      expect(elem.type).toBe('comment');\n      expect(elem).toHaveProperty(\n        'data',\n        conditional.replace('<!--', '').replace('-->', ''),\n      );\n    });\n\n    it(`should handle text: ${text}`, () => {\n      const text_ = parse(text, defaultOpts, false, null).children[0];\n      expect(text_.type).toBe('text');\n      expect(text_).toHaveProperty('data', 'lorem ipsum');\n    });\n\n    it(`should handle script tags: ${script}`, () => {\n      const script_ = parse(script, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(script_.type).toBe('script');\n      expect(script_.tagName).toBe('script');\n      expect(script_.attribs).toHaveProperty('type', 'text/javascript');\n      expect(script_.childNodes).toHaveLength(1);\n      expect(script_.childNodes[0].type).toBe('text');\n      expect(script_.childNodes[0]).toHaveProperty(\n        'data',\n        'alert(\"hi world!\");',\n      );\n    });\n\n    it(`should handle style tags: ${style}`, () => {\n      const style_ = parse(style, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(style_.type).toBe('style');\n      expect(style_.tagName).toBe('style');\n      expect(style_.attribs).toHaveProperty('type', 'text/css');\n      expect(style_.childNodes).toHaveLength(1);\n      expect(style_.childNodes[0].type).toBe('text');\n      expect(style_.childNodes[0]).toHaveProperty(\n        'data',\n        ' h2 { color:blue; } ',\n      );\n    });\n\n    it(`should handle directives: ${directive}`, () => {\n      const elem = parse(directive, defaultOpts, true, null).children[0];\n      expect(elem.type).toBe('directive');\n      expect(elem).toHaveProperty('data', '!DOCTYPE html');\n      expect(elem).toHaveProperty('name', '!doctype');\n    });\n  });\n\n  describe('.parse', () => {\n    // Root test utility\n\n    it(`should add root to: ${basic}`, () => {\n      const root = parse(basic, defaultOpts, true, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'html');\n    });\n\n    it(`should add root to: ${siblings}`, () => {\n      const root = parse(siblings, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(2);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'h2');\n      expect(root.childNodes[1]).toHaveProperty('tagName', 'p');\n      expect(root.childNodes[1].parent).toBe(root);\n    });\n\n    it(`should add root to: ${comment}`, () => {\n      const root = parse(comment, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('comment');\n    });\n\n    it(`should add root to: ${text}`, () => {\n      const root = parse(text, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('text');\n    });\n\n    it(`should add root to: ${scriptEmpty}`, () => {\n      const root = parse(scriptEmpty, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('script');\n    });\n\n    it(`should add root to: ${styleEmpty}`, () => {\n      const root = parse(styleEmpty, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('style');\n    });\n\n    it(`should add root to: ${directive}`, () => {\n      const root = parse(directive, defaultOpts, true, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(2);\n      expect(root.childNodes[0].type).toBe('directive');\n    });\n\n    it('should simply return root', () => {\n      const oldroot = parse(basic, defaultOpts, true, null);\n      const root = parse(oldroot, defaultOpts, true, null);\n      expect(root).toBe(oldroot);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'html');\n    });\n\n    it('should expose the DOM level 1 API', () => {\n      const root = parse(\n        '<div><a></a><span></span><p></p></div>',\n        defaultOpts,\n        false,\n        null,\n      ).childNodes[0] as Element;\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes).toHaveLength(3);\n\n      expect(root.tagName).toBe('div');\n      expect(root.firstChild).toBe(childNodes[0]);\n      expect(root.lastChild).toBe(childNodes[2]);\n\n      expect(childNodes[0].tagName).toBe('a');\n      expect(childNodes[0].previousSibling).toBe(null);\n      expect(childNodes[0].nextSibling).toBe(childNodes[1]);\n      expect(childNodes[0].parentNode).toBe(root);\n      expect((childNodes[0] as Element).childNodes).toHaveLength(0);\n      expect(childNodes[0].firstChild).toBe(null);\n      expect(childNodes[0].lastChild).toBe(null);\n\n      expect(childNodes[1].tagName).toBe('span');\n      expect(childNodes[1].previousSibling).toBe(childNodes[0]);\n      expect(childNodes[1].nextSibling).toBe(childNodes[2]);\n      expect(childNodes[1].parentNode).toBe(root);\n      expect(childNodes[1].childNodes).toHaveLength(0);\n      expect(childNodes[1].firstChild).toBe(null);\n      expect(childNodes[1].lastChild).toBe(null);\n\n      expect(childNodes[2].tagName).toBe('p');\n      expect(childNodes[2].previousSibling).toBe(childNodes[1]);\n      expect(childNodes[2].nextSibling).toBe(null);\n      expect(childNodes[2].parentNode).toBe(root);\n      expect(childNodes[2].childNodes).toHaveLength(0);\n      expect(childNodes[2].firstChild).toBe(null);\n      expect(childNodes[2].lastChild).toBe(null);\n    });\n\n    it('Should parse less than or equal sign sign', () => {\n      const root = parse('<i>A</i><=<i>B</i>', defaultOpts, false, null);\n      const { childNodes } = root;\n\n      expect(childNodes[0]).toHaveProperty('tagName', 'i');\n      expect((childNodes[0] as Element).childNodes[0]).toHaveProperty(\n        'data',\n        'A',\n      );\n      expect(childNodes[1]).toHaveProperty('data', '<=');\n      expect(childNodes[2]).toHaveProperty('tagName', 'i');\n      expect((childNodes[2] as Element).childNodes[0]).toHaveProperty(\n        'data',\n        'B',\n      );\n    });\n\n    it('Should ignore unclosed CDATA', () => {\n      const root = parse(\n        '<a></a><script>foo //<![CDATA[ bar</script><b></b>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('a');\n      expect(childNodes[1].tagName).toBe('script');\n      expect(childNodes[1].childNodes[0]).toHaveProperty(\n        'data',\n        'foo //<![CDATA[ bar',\n      );\n      expect(childNodes[2].tagName).toBe('b');\n    });\n\n    it('Should add <head> to documents', () => {\n      const root = parse('<html></html>', defaultOpts, true, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('html');\n      expect(childNodes[0].childNodes[0]).toHaveProperty('tagName', 'head');\n    });\n\n    it('Should implicitly create <tr> around <td>', () => {\n      const root = parse(\n        '<table><td>bar</td></tr></table>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('table');\n      expect(childNodes[0].childNodes.length).toBe(1);\n      expect(childNodes[0].childNodes[0]).toHaveProperty('tagName', 'tbody');\n      expect((childNodes[0] as any).childNodes[0].childNodes[0]).toHaveProperty(\n        'tagName',\n        'tr',\n      );\n      expect(\n        (childNodes[0] as any).childNodes[0].childNodes[0].childNodes[0]\n          .tagName,\n      ).toBe('td');\n      expect(\n        (childNodes[0] as any).childNodes[0].childNodes[0].childNodes[0]\n          .childNodes[0].data,\n      ).toBe('bar');\n    });\n\n    it('Should parse custom tag <line>', () => {\n      const root = parse('<line>test</line>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(1);\n      expect(childNodes[0].tagName).toBe('line');\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', 'test');\n    });\n\n    it('Should properly parse misnested table tags', () => {\n      const root = parse(\n        '<tr><td>i1</td></tr><tr><td>i2</td></td></tr><tr><td>i3</td></td></tr>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(3);\n\n      for (let i = 0; i < childNodes.length; i++) {\n        const child = childNodes[i];\n        expect(child.tagName).toBe('tr');\n        expect(child.childNodes[0]).toHaveProperty('tagName', 'td');\n        expect((child.childNodes[0] as Element).childNodes[0]).toHaveProperty(\n          'data',\n          `i${i + 1}`,\n        );\n      }\n    });\n\n    it('Should correctly parse data url attributes', () => {\n      const html =\n        '<div style=\\'font-family:\"butcherman-caps\"; src:url(data:font/opentype;base64,AAEA...);\\'></div>';\n      const expectedAttr =\n        'font-family:\"butcherman-caps\"; src:url(data:font/opentype;base64,AAEA...);';\n      const root = parse(html, defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].attribs).toHaveProperty('style', expectedAttr);\n    });\n\n    it('Should treat <xmp> tag content as text', () => {\n      const root = parse('<xmp><h2></xmp>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', '<h2>');\n    });\n\n    it('Should correctly parse malformed numbered entities', () => {\n      const root = parse('<p>z&#</p>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', 'z&#');\n    });\n\n    it('Should correctly parse mismatched headings', () => {\n      const root = parse('<h2>Test</h3><div></div>', defaultOpts, false, null);\n      const { childNodes } = root;\n\n      expect(childNodes.length).toBe(2);\n      expect(childNodes[0]).toHaveProperty('tagName', 'h2');\n      expect(childNodes[1]).toHaveProperty('tagName', 'div');\n    });\n\n    it('Should correctly parse tricky <pre> content', () => {\n      const root = parse(\n        '<pre>\\nA <- factor(A, levels = c(\"c\",\"a\",\"b\"))\\n</pre>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(1);\n      expect(childNodes[0].tagName).toBe('pre');\n      expect(childNodes[0].childNodes[0]).toHaveProperty(\n        'data',\n        'A <- factor(A, levels = c(\"c\",\"a\",\"b\"))\\n',\n      );\n    });\n\n    it('should pass the options for including the location info to parse5', () => {\n      const root = parse(\n        '<p>Hello</p>',\n        { ...defaultOpts, sourceCodeLocationInfo: true },\n        false,\n        null,\n      );\n      const location = root.children[0].sourceCodeLocation;\n\n      expect(typeof location).toBe('object');\n      expect(location?.endOffset).toBe(12);\n    });\n  });\n});\n\n// Path: src/__tests__/xml.spec.ts\nimport cheerio from '../index.js';\nimport type { CheerioOptions } from '../options.js';\n\nfunction xml(str: string, options?: CheerioOptions) {\n  options = { xml: true, ...options };\n  const $ = cheerio.load(str, options);\n  return $.xml();\n}\n\nfunction dom(str: string, options?: CheerioOptions) {\n  const $ = cheerio.load('', options);\n  return $(str).html();\n}\n\ndescribe('render', () => {\n  describe('(xml)', () => {\n    it('should render <media:thumbnail /> tags correctly', () => {\n      const str =\n        '<media:thumbnail url=\"http://www.foo.com/keyframe.jpg\" width=\"75\" height=\"50\" time=\"12:05:01.123\" />';\n      expect(xml(str)).toBe(\n        '<media:thumbnail url=\"http://www.foo.com/keyframe.jpg\" width=\"75\" height=\"50\" time=\"12:05:01.123\"/>',\n      );\n    });\n\n    it('should render <link /> tags (RSS) correctly', () => {\n      const str = '<link>http://www.github.com/</link>';\n      expect(xml(str)).toBe('<link>http://www.github.com/</link>');\n    });\n\n    it('should escape entities', () => {\n      const str = '<tag attr=\"foo &amp; bar\"/>';\n      expect(xml(str)).toBe(str);\n    });\n\n    it('should render HTML as XML', () => {\n      const $ = cheerio.load('<foo></foo>', null, false);\n      expect($.xml()).toBe('<foo/>');\n    });\n  });\n\n  describe('(dom)', () => {\n    it('should not keep camelCase for new nodes', () => {\n      const str = '<g><someElem someAttribute=\"something\">hello</someElem></g>';\n      expect(dom(str, { xml: false })).toBe(\n        '<someelem someattribute=\"something\">hello</someelem>',\n      );\n    });\n\n    it('should keep camelCase for new nodes', () => {\n      const str = '<g><someElem someAttribute=\"something\">hello</someElem></g>';\n      expect(dom(str, { xml: true })).toBe(\n        '<someElem someAttribute=\"something\">hello</someElem>',\n      );\n    });\n\n    it('should maintain the parsing options of distinct contexts independently', () => {\n      const str = '<g><someElem someAttribute=\"something\">hello</someElem></g>';\n      const $ = cheerio.load('', { xml: false });\n\n      expect($(str).html()).toBe(\n        '<someelem someattribute=\"something\">hello</someelem>',\n      );\n    });\n  });\n});\n\n// Path: src/__tests__/deprecated.spec.ts\n/**\n * This file includes tests for deprecated APIs. The methods are expected to be\n * removed in the next major release of Cheerio, but their stability should be\n * maintained until that time.\n */\nimport * as fixtures from '../__fixtures__/fixtures.js';\nimport cheerio from '../index.js';\n\ndescribe('deprecated APIs', () => {\n  describe('cheerio module', () => {\n    describe('.parseHTML', () => {\n      it('(html) : should preserve content', () => {\n        const html = '<div>test div</div>';\n        expect(cheerio(cheerio.parseHTML(html)[0]).html()).toBe('test div');\n      });\n    });\n\n    describe('.merge', () => {\n      it('should be a function', () => {\n        expect(typeof cheerio.merge).toBe('function');\n      });\n\n      // #1674 - merge, wont accept Cheerio object\n      it('should be a able merge array and cheerio object', () => {\n        const ret = cheerio.merge<unknown>(cheerio(), ['elem1', 'elem2']);\n        expect(typeof ret).toBe('object');\n        expect(ret).toHaveLength(2);\n      });\n\n      it('(arraylike, arraylike) : should modify the first array, but not the second', () => {\n        const arr1 = [1, 2, 3];\n        const arr2 = [4, 5, 6];\n        const ret = cheerio.merge(arr1, arr2);\n\n        expect(typeof ret).toBe('object');\n        expect(Array.isArray(ret)).toBe(true);\n        expect(ret).toBe(arr1);\n...\n", "needle_token_start": 8778, "needle_token_end": 8938, "code_context_ntokens": 16109, "output": ["```typescript\nfunction createTestServer(\n  contentType: string,\n  body: string | Buffer,\n): Promise<number> {\n  return new Promise((resolve, reject) => {\n    server = createServer((_req, res) => {\n      res.writeHead(200, { 'Content-Type': contentType });\n      res.end(body);\n    });\n\n    server.listen(0, () => {\n      const address = server?.address();\n\n      if (typeof address === 'string' || address == null) {\n        reject(new Error('Failed to get port'));\n      } else {\n        resolve(address.port);\n      }\n    });\n  });\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "empty", "language": "typescript", "path": "src/api/manipulation.ts", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to remove all child elements from each selected element in a document, while leaving text and comment nodes intact.\n2. **Input**: There are no parameters required for this function.\n3. **Output**: Returns the instance itself, allowing for method chaining.\n4. **Procedure**: Iterates over each selected element, checks if it has children, and if so, removes all child elements by clearing the children array and resetting the relationship links (previous, next, parent) of each child to null.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/api/attributes.ts\n/**\n * Methods for getting and modifying attributes.\n *\n * @module cheerio/attributes\n */\n\nimport { text } from '../static.js';\nimport { domEach, camelCase, cssCase } from '../utils.js';\nimport { isTag, type AnyNode, type Element } from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport { innerText, textContent } from 'domutils';\nconst hasOwn = Object.prototype.hasOwnProperty;\nconst rspace = /\\s+/;\nconst dataAttrPrefix = 'data-';\n\n// Attributes that are booleans\nconst rboolean =\n  /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i;\n// Matches strings that look like JSON objects or arrays\nconst rbrace = /^{[^]*}$|^\\[[^]*]$/;\n\n/**\n * Gets a node's attribute. For boolean attributes, it will return the value's\n * name should it be set.\n *\n * Also supports getting the `value` of several form elements.\n *\n * @private\n * @category Attributes\n * @param elem - Element to get the attribute of.\n * @param name - Name of the attribute.\n * @param xmlMode - Disable handling of special HTML attributes.\n * @returns The attribute's value.\n */\nfunction getAttr(\n  elem: AnyNode,\n  name: undefined,\n  xmlMode?: boolean,\n): Record<string, string> | undefined;\nfunction getAttr(\n  elem: AnyNode,\n  name: string,\n  xmlMode?: boolean,\n): string | undefined;\nfunction getAttr(\n  elem: AnyNode,\n  name: string | undefined,\n  xmlMode?: boolean,\n): Record<string, string> | string | undefined {\n  if (!elem || !isTag(elem)) return undefined;\n\n  elem.attribs ??= {};\n\n  // Return the entire attribs object if no attribute specified\n  if (!name) {\n    return elem.attribs;\n  }\n\n  if (hasOwn.call(elem.attribs, name)) {\n    // Get the (decoded) attribute\n    return !xmlMode && rboolean.test(name) ? name : elem.attribs[name];\n  }\n\n  // Mimic the DOM and return text content as value for `option's`\n  if (elem.name === 'option' && name === 'value') {\n    return text(elem.children);\n  }\n\n  // Mimic DOM with default value for radios/checkboxes\n  if (\n    elem.name === 'input' &&\n    (elem.attribs['type'] === 'radio' || elem.attribs['type'] === 'checkbox') &&\n    name === 'value'\n  ) {\n    return 'on';\n  }\n\n  return undefined;\n}\n\n/**\n * Sets the value of an attribute. The attribute will be deleted if the value is\n * `null`.\n *\n * @private\n * @param el - The element to set the attribute on.\n * @param name - The attribute's name.\n * @param value - The attribute's value.\n */\nfunction setAttr(el: Element, name: string, value: string | null) {\n  if (value === null) {\n    removeAttribute(el, name);\n  } else {\n    el.attribs[name] = `${value}`;\n  }\n}\n\n/**\n * Method for getting attributes. Gets the attribute value for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr('id');\n * //=> fruits\n * ```\n *\n * @param name - Name of the attribute.\n * @returns The attribute's value.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): string | undefined;\n/**\n * Method for getting all attributes and their values of the first element in\n * the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr();\n * //=> { id: 'fruits' }\n * ```\n *\n * @returns The attribute's values.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n): Record<string, string> | undefined;\n/**\n * Method for setting attributes. Sets the attribute value for only the first\n * element in the matched set. If you set an attribute's value to `null`, you\n * remove that attribute. You may also pass a `map` and `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr('id', 'favorite').html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param name - Name of the attribute.\n * @param value - The new value of the attribute.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): Cheerio<T>;\n/**\n * Method for setting multiple attributes at once. Sets the attribute value for\n * only the first element in the matched set. If you set an attribute's value to\n * `null`, you remove that attribute.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr({ id: 'favorite' }).html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param values - Map of attribute names and values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  values: Record<string, string | null>,\n): Cheerio<T>;\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name?: string | Record<string, string | null>,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): string | Cheerio<T> | undefined | Record<string, string> {\n  // Set the value (with attr map support)\n  if (typeof name === 'object' || value !== undefined) {\n    if (typeof value === 'function') {\n      if (typeof name !== 'string') {\n        {\n          throw new Error('Bad combination of arguments.');\n        }\n      }\n      return domEach(this, (el, i) => {\n        if (isTag(el)) setAttr(el, name, value.call(el, i, el.attribs[name]));\n      });\n    }\n    return domEach(this, (el) => {\n      if (!isTag(el)) return;\n\n      if (typeof name === 'object') {\n        for (const objName of Object.keys(name)) {\n          const objValue = name[objName];\n          setAttr(el, objName, objValue);\n        }\n      } else {\n        setAttr(el, name as string, value as string);\n      }\n    });\n  }\n\n  return arguments.length > 1\n    ? this\n    : getAttr(this[0], name as string, this.options.xmlMode);\n}\n\n/**\n * Gets a node's prop.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the prop of.\n * @param name - Name of the prop.\n...\n// Path: src/api/manipulation.ts\n/**\n * Methods for modifying the DOM structure.\n *\n * @module cheerio/manipulation\n */\n\nimport {\n  isTag,\n  Text,\n  hasChildren,\n  cloneNode,\n  Document,\n  type ParentNode,\n  type AnyNode,\n  type Element,\n} from 'domhandler';\nimport { update as updateDOM } from '../parse.js';\nimport { text as staticText } from '../static.js';\nimport { domEach, isHtml, isCheerio } from '../utils.js';\nimport { removeElement } from 'domutils';\nimport type { Cheerio } from '../cheerio.js';\nimport type { BasicAcceptedElems, AcceptedElems } from '../types.js';\n\n/**\n * Create an array of nodes, recursing into arrays and parsing strings if\n * necessary.\n *\n * @private\n * @category Manipulation\n * @param elem - Elements to make an array of.\n * @param clone - Optionally clone nodes.\n * @returns The array of nodes.\n */\nexport function _makeDomArray<T extends AnyNode>(\n  this: Cheerio<T>,\n  elem?: BasicAcceptedElems<AnyNode> | BasicAcceptedElems<AnyNode>[],\n  clone?: boolean,\n): AnyNode[] {\n  if (elem == null) {\n    return [];\n  }\n\n  if (typeof elem === 'string') {\n    return this._parse(elem, this.options, false, null).children.slice(0);\n  }\n\n  if ('length' in elem) {\n    if (elem.length === 1) {\n      return this._makeDomArray(elem[0], clone);\n    }\n\n    const result: AnyNode[] = [];\n\n    for (let i = 0; i < elem.length; i++) {\n      const el = elem[i];\n\n      if (typeof el === 'object') {\n        if (el == null) {\n          continue;\n        }\n\n        if (!('length' in el)) {\n          result.push(clone ? cloneNode(el, true) : el);\n          continue;\n        }\n      }\n\n      result.push(...this._makeDomArray(el, clone));\n    }\n\n    return result;\n  }\n\n  return [clone ? cloneNode(elem, true) : elem];\n}\n\nfunction _insert(\n  concatenator: (\n    dom: AnyNode[],\n    children: AnyNode[],\n    parent: ParentNode,\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    ...elems:\n      | [\n          (\n            this: AnyNode,\n            i: number,\n            html: string,\n          ) => BasicAcceptedElems<AnyNode>,\n        ]\n      | BasicAcceptedElems<AnyNode>[]\n  ) {\n    const lastIdx = this.length - 1;\n\n    return domEach(this, (el, i) => {\n      if (!hasChildren(el)) return;\n\n      const domSrc =\n        typeof elems[0] === 'function'\n          ? elems[0].call(el, i, this._render(el.children))\n          : (elems as BasicAcceptedElems<AnyNode>[]);\n\n      const dom = this._makeDomArray(domSrc, i < lastIdx);\n      concatenator(dom, el.children, el);\n    });\n  };\n}\n\n/**\n * Modify an array in-place, removing some number of elements and adding new\n * elements directly following them.\n *\n * @private\n * @category Manipulation\n * @param array - Target array to splice.\n * @param spliceIdx - Index at which to begin changing the array.\n * @param spliceCount - Number of elements to remove from the array.\n * @param newElems - Elements to insert into the array.\n * @param parent - The parent of the node.\n * @returns The spliced array.\n */\nfunction uniqueSplice(\n  array: AnyNode[],\n  spliceIdx: number,\n  spliceCount: number,\n  newElems: AnyNode[],\n  parent: ParentNode,\n): AnyNode[] {\n  const spliceArgs: Parameters<typeof Array.prototype.splice> = [\n    spliceIdx,\n    spliceCount,\n    ...newElems,\n  ];\n  const prev = spliceIdx === 0 ? null : array[spliceIdx - 1];\n  const next =\n    spliceIdx + spliceCount >= array.length\n      ? null\n      : array[spliceIdx + spliceCount];\n\n  /*\n   * Before splicing in new elements, ensure they do not already appear in the\n   * current array.\n   */\n  for (let idx = 0; idx < newElems.length; ++idx) {\n    const node = newElems[idx];\n    const oldParent = node.parent;\n\n    if (oldParent) {\n      const oldSiblings: AnyNode[] = oldParent.children;\n      const prevIdx = oldSiblings.indexOf(node);\n\n      if (prevIdx > -1) {\n        oldParent.children.splice(prevIdx, 1);\n        if (parent === oldParent && spliceIdx > prevIdx) {\n          spliceArgs[0]--;\n        }\n      }\n    }\n\n    node.parent = parent;\n\n    if (node.prev) {\n      node.prev.next = node.next ?? null;\n    }\n\n    if (node.next) {\n      node.next.prev = node.prev ?? null;\n    }\n\n    node.prev = idx === 0 ? prev : newElems[idx - 1];\n    node.next = idx === newElems.length - 1 ? next : newElems[idx + 1];\n  }\n\n  if (prev) {\n    prev.next = newElems[0];\n  }\n  if (next) {\n    next.prev = newElems[newElems.length - 1];\n  }\n  return array.splice(...spliceArgs);\n}\n\n/**\n * Insert every element in the set of matched elements to the end of the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').appendTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to append elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/appendTo/}\n */\nexport function appendTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const appendTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  appendTarget.append(this);\n\n  return this;\n}\n\n/**\n * Insert every element in the set of matched elements to the beginning of the\n * target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').prependTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to prepend elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/prependTo/}\n */\nexport function prependTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const prependTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  prependTarget.prepend(this);\n\n  return this;\n}\n\n/**\n * Inserts content as the _last_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').append('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/append/}\n */\nexport const append = _insert((dom, children, parent) => {\n  uniqueSplice(children, children.length, 0, dom, parent);\n});\n\n/**\n * Inserts content as the _first_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').prepend('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/prepend/}\n */\nexport const prepend = _insert((dom, children, parent) => {\n  uniqueSplice(children, 0, 0, dom, parent);\n});\n\nfunction _wrap(\n  insert: (\n    el: AnyNode,\n    elInsertLocation: ParentNode,\n    wrapperDom: ParentNode[],\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    wrapper: AcceptedElems<AnyNode>,\n  ) {\n    const lastIdx = this.length - 1;\n    const lastParent = this.parents().last();\n\n    for (let i = 0; i < this.length; i++) {\n      const el = this[i];\n\n      const wrap =\n        typeof wrapper === 'function'\n          ? wrapper.call(el, i, el)\n          : typeof wrapper === 'string' && !isHtml(wrapper)\n            ? lastParent.find(wrapper).clone()\n            : wrapper;\n\n      const [wrapperDom] = this._makeDomArray(wrap, i < lastIdx);\n\n      if (!wrapperDom || !hasChildren(wrapperDom)) continue;\n\n      let elInsertLocation = wrapperDom;\n\n      /*\n       * Find the deepest child. Only consider the first tag child of each node\n       * (ignore text); stop if no children are found.\n       */\n      let j = 0;\n\n      while (j < elInsertLocation.children.length) {\n        const child = elInsertLocation.children[j];\n        if (isTag(child)) {\n          elInsertLocation = child;\n          j = 0;\n        } else {\n          j++;\n        }\n      }\n\n      insert(el, elInsertLocation, [wrapperDom]);\n    }\n\n    return this;\n  };\n}\n\n/**\n * The .wrap() function can take any string or object that could be passed to\n * the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. A\n * copy of this structure will be wrapped around each of the elements in the set\n * of matched elements. This method returns the original set of elements for\n * chaining purposes.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrap(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"red-fruit\">\n * //      <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrap(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"healthy\">\n * //       <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //       <li class=\"orange\">Orange</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //        <li class=\"plum\">Plum</li>\n * //     </div>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around each element in the\n *   selection.\n * @see {@link https://api.jquery.com/wrap/}\n */\nexport const wrap = _wrap((el, elInsertLocation, wrapperDom) => {\n  const { parent } = el;\n\n  if (!parent) return;\n\n  const siblings: AnyNode[] = parent.children;\n  const index = siblings.indexOf(el);\n\n  updateDOM([el], elInsertLocation);\n  /*\n   * The previous operation removed the current element from the `siblings`\n   * array, so the `dom` array can be inserted without removing any\n   * additional elements.\n   */\n  uniqueSplice(siblings, index, 0, wrapperDom, parent);\n});\n\n/**\n * The .wrapInner() function can take any string or object that could be passed\n * to the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around the content of each of the elements in the\n * set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrapInner(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"red-fruit\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"pear\">Pear</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrapInner(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"healthy\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">\n * //       <div class=\"healthy\">Orange</div>\n * //     </li>\n * //     <li class=\"pear\">\n * //       <div class=\"healthy\">Pear</div>\n * //     </li>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around the content of each element\n *   in the selection.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/wrapInner/}\n */\nexport const wrapInner = _wrap((el, elInsertLocation, wrapperDom) => {\n  if (!hasChildren(el)) return;\n  updateDOM(el.children, elInsertLocation);\n  updateDOM(wrapperDom, el);\n});\n\n/**\n * The .unwrap() function, removes the parents of the set of matched elements\n * from the DOM, leaving the matched elements in their place.\n *\n * @category Manipulation\n * @example <caption>without selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <div><p>Hello</p></div>\\n  <div><p>World</p></div>\\n</div>',\n * );\n * $('#test p').unwrap();\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @example <caption>with selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <p>Hello</p>\\n  <b><p>World</p></b>\\n</div>',\n * );\n * $('#test p').unwrap('b');\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @param selector - A selector to check the parent element against. If an\n *   element's parent does not match the selector, the element won't be\n *   unwrapped.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/unwrap/}\n */\nexport function unwrap<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  this.parent(selector)\n    .not('body')\n    .each((_, el) => {\n      this._make(el).replaceWith(el.children);\n    });\n  return this;\n}\n\n/**\n * The .wrapAll() function can take any string or object that could be passed to\n * the $() function to specify a DOM structure. This structure may be nested\n * several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around all of the elements in the set of matched\n * elements, as a single group.\n *\n * @category Manipulation\n * @example <caption>With markup passed to `wrapAll`</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div class=\"container\"><div class=\"inner\">First</div><div class=\"inner\">Second</div></div>',\n * );\n * $('.inner').wrapAll(\"<div class='new'></div>\");\n *\n * //=> <div class=\"container\">\n * //     <div class='new'>\n * //       <div class=\"inner\">First</div>\n * //       <div class=\"inner\">Second</div>\n * //     </div>\n * //   </div>\n * ```\n *\n * @example <caption>With an existing cheerio instance</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<span>Span 1</span><strong>Strong</strong><span>Span 2</span>',\n * );\n * const wrap = $('<div><p><em><b></b></em></p></div>');\n * $('span').wrapAll(wrap);\n *\n * //=> <div>\n * //     <p>\n * //       <em>\n * //         <b>\n * //           <span>Span 1</span>\n * //           <span>Span 2</span>\n * //         </b>\n * //       </em>\n * //     </p>\n * //   </div>\n * //   <strong>Strong</strong>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around all matched elements in the\n *   selection.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/wrapAll/}\n */\nexport function wrapAll<T extends AnyNode>(\n  this: Cheerio<T>,\n  wrapper: AcceptedElems<T>,\n): Cheerio<T> {\n  const el = this[0];\n  if (el) {\n    const wrap: Cheerio<AnyNode> = this._make(\n      typeof wrapper === 'function' ? wrapper.call(el, 0, el) : wrapper,\n    ).insertBefore(el);\n\n    // If html is given as wrapper, wrap may contain text elements\n    let elInsertLocation: Element | undefined;\n\n    for (let i = 0; i < wrap.length; i++) {\n      if (wrap[i].type === 'tag') elInsertLocation = wrap[i] as Element;\n    }\n\n    let j = 0;\n\n    /*\n     * Find the deepest child. Only consider the first tag child of each node\n     * (ignore text); stop if no children are found.\n     */\n    while (elInsertLocation && j < elInsertLocation.children.length) {\n      const child = elInsertLocation.children[j];\n      if (child.type === 'tag') {\n        elInsertLocation = child as Element;\n        j = 0;\n      } else {\n        j++;\n      }\n    }\n\n    if (elInsertLocation) this._make(elInsertLocation).append(this);\n  }\n  return this;\n}\n\n/**\n * Insert content next to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').after('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert after each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/after/}\n */\nexport function after<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element after `this` element\n    uniqueSplice(siblings, index + 1, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements after the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertAfter('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements after.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertAfter/}\n */\nexport function insertAfter<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  if (typeof target === 'string') {\n    target = this._make<AnyNode>(target);\n  }\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  for (const el of this._makeDomArray(target)) {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      continue;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) continue;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index + 1, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  }\n\n  return this._make(clones);\n}\n\n/**\n * Insert content previous to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').before('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert before each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/before/}\n */\nexport function before<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element before `el` element\n    uniqueSplice(siblings, index, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements before the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertBefore('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements before.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertBefore/}\n */\nexport function insertBefore<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const targetArr = this._make<AnyNode>(target);\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  domEach(targetArr, (el) => {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  });\n\n  return this._make(clones);\n}\n\n/**\n * Removes the set of matched elements from the DOM and all their children.\n * `selector` filters the set of matched elements to be removed.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.pear').remove();\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //    </ul>\n * ```\n *\n * @param selector - Optional selector for elements to remove.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/remove/}\n */\nexport function remove<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  // Filter if we have selector\n  const elems = selector ? this.filter(selector) : this;\n\n  domEach(elems, (el) => {\n    removeElement(el);\n    el.prev = el.next = el.parent = null;\n  });\n\n  return this;\n}\n\n/**\n * Replaces matched elements with `content`.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const plum = $('<li class=\"plum\">Plum</li>');\n * $('.pear').replaceWith(plum);\n * $.html();\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">Apple</li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n * ```\n *\n * @param content - Replacement for matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/replaceWith/}\n */\nexport function replaceWith<T extends AnyNode>(\n  this: Cheerio<T>,\n  content: AcceptedElems<AnyNode>,\n): Cheerio<T> {\n  return domEach(this, (el, i) => {\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const cont =\n      typeof content === 'function' ? content.call(el, i, el) : content;\n    const dom = this._makeDomArray(cont);\n\n    /*\n     * In the case that `dom` contains nodes that already exist in other\n     * structures, ensure those nodes are properly removed.\n     */\n    updateDOM(dom, null);\n\n    const index = siblings.indexOf(el);\n\n    // Completely remove old element\n    uniqueSplice(siblings, index, 1, dom, parent);\n\n    if (!dom.includes(el)) {\n      el.parent = el.prev = el.next = null;\n    }\n  });\n}\n\n/**\n * Removes all children from each item in the selection. Text nodes and comment\n * nodes are left as is.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').empty();\n * $.html();\n * //=>  <ul id=\"fruits\"></ul>\n * ```\n *\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/empty/}\n */\nexport \nfunction empty<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    el.children.length = 0;\n  });\n}\n\n/**\n * Gets an HTML content string from the first selected element.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').html();\n * //=> Orange\n *\n * $('#fruits').html('<li class=\"mango\">Mango</li>').html();\n * //=> <li class=\"mango\">Mango</li>\n * ```\n *\n * @returns The HTML content string.\n * @see {@link https://api.jquery.com/html/}\n */\nexport function html<T extends AnyNode>(this: Cheerio<T>): string | null;\n/**\n * Replaces each selected element's content with the specified content.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').html('<li class=\"mango\">Mango</li>').html();\n * //=> <li class=\"mango\">Mango</li>\n * ```\n *\n * @param str - The content to replace selection's contents with.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/html/}\n */\nexport function html<T extends AnyNode>(\n  this: Cheerio<T>,\n  str: string | Cheerio<T>,\n): Cheerio<T>;\nexport function html<T extends AnyNode>(\n  this: Cheerio<T>,\n  str?: string | Cheerio<AnyNode>,\n): Cheerio<T> | string | null {\n  if (str === undefined) {\n    const el = this[0];\n    if (!el || !hasChildren(el)) return null;\n    return this._render(el.children);\n  }\n\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    const content = isCheerio(str)\n      ? str.toArray()\n      : this._parse(`${str}`, this.options, false, el).children;\n\n    updateDOM(content, el);\n  });\n}\n\n/**\n * Turns the collection to a string. Alias for `.html()`.\n *\n * @category Manipulation\n * @returns The rendered document.\n */\nexport function toString<T extends AnyNode>(this: Cheerio<T>): string {\n  return this._render(this);\n}\n\n/**\n * Get the combined text contents of each element in the set of matched\n * elements, including their descendants.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').text();\n * //=> Orange\n *\n * $('ul').text();\n * //=>  Apple\n * //    Orange\n * //    Pear\n * ```\n *\n * @returns The text contents of the collection.\n * @see {@link https://api.jquery.com/text/}\n */\nexport function text<T extends AnyNode>(this: Cheerio<T>): string;\n/**\n * Set the content of each element in the set of matched elements to the\n * specified text.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.orange').text('Orange');\n * //=> <div class=\"orange\">Orange</div>\n * ```\n *\n * @param str - The text to set as the content of each matched element.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/text/}\n */\nexport function text<T extends AnyNode>(\n  this: Cheerio<T>,\n  str: string | ((this: AnyNode, i: number, text: string) => string),\n): Cheerio<T>;\nexport function text<T extends AnyNode>(\n  this: Cheerio<T>,\n  str?: string | ((this: AnyNode, i: number, text: string) => string),\n): Cheerio<T> | string {\n  // If `str` is undefined, act as a \"getter\"\n  if (str === undefined) {\n    return staticText(this);\n  }\n  if (typeof str === 'function') {\n    // Function support\n    return domEach(this, (el, i) =>\n      this._make(el).text(str.call(el, i, staticText([el]))),\n    );\n  }\n\n  // Append text node to each selected elements\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    const textNode = new Text(`${str}`);\n\n    updateDOM(textNode, el);\n  });\n}\n\n/**\n * Clone the cheerio object.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const moreFruit = $('#fruits').clone();\n * ```\n *\n * @returns The cloned object.\n * @see {@link https://api.jquery.com/clone/}\n */\nexport function clone<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  const clone = Array.prototype.map.call(this.get(), (el) =>\n    cloneNode(el, true),\n  ) as T[];\n\n  // Add a root node around the cloned nodes\n  const root = new Document(clone);\n  for (const node of clone) {\n    node.parent = root;\n  }\n\n  return this._make(clone);\n}\n\n// Path: src/api/traversing.ts\n/**\n * Methods for traversing the DOM structure.\n *\n * @module cheerio/traversing\n */\n\nimport {\n  isTag,\n  type AnyNode,\n  type Element,\n  hasChildren,\n  isDocument,\n  type Document,\n} from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport * as select from 'cheerio-select';\nimport { domEach, isCheerio } from '../utils.js';\nimport { contains } from '../static.js';\nimport {\n  getChildren,\n  getSiblings,\n  nextElementSibling,\n  prevElementSibling,\n  uniqueSort,\n} from 'domutils';\nimport type { FilterFunction, AcceptedFilters } from '../types.js';\nconst reSiblingSelector = /^\\s*[+~]/;\n\n/**\n * Get the descendants of each element in the current set of matched elements,\n * filtered by a selector, jQuery object, or element.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').find('li').length;\n * //=> 3\n * $('#fruits').find($('.apple')).length;\n * //=> 1\n * ```\n *\n * @param selectorOrHaystack - Element to look for.\n * @returns The found elements.\n * @see {@link https://api.jquery.com/find/}\n */\nexport function find<T extends AnyNode>(\n  this: Cheerio<T>,\n  selectorOrHaystack?: string | Cheerio<Element> | Element,\n): Cheerio<Element> {\n  if (!selectorOrHaystack) {\n    return this._make([]);\n  }\n\n  if (typeof selectorOrHaystack !== 'string') {\n    const haystack = isCheerio(selectorOrHaystack)\n      ? selectorOrHaystack.toArray()\n      : [selectorOrHaystack];\n\n    const context = this.toArray();\n\n    return this._make(\n      haystack.filter((elem) => context.some((node) => contains(node, elem))),\n    );\n  }\n\n  return this._findBySelector(selectorOrHaystack, Number.POSITIVE_INFINITY);\n}\n\n/**\n * Find elements by a specific selector.\n *\n * @private\n * @category Traversing\n * @param selector - Selector to filter by.\n * @param limit - Maximum number of elements to match.\n * @returns The found elements.\n */\nexport function _findBySelector<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector: string,\n  limit: number,\n): Cheerio<Element> {\n  const context = this.toArray();\n\n  const elems = reSiblingSelector.test(selector)\n    ? context\n    : this.children().toArray();\n\n  const options = {\n    context,\n    root: this._root?.[0],\n\n    // Pass options that are recognized by `cheerio-select`\n    xmlMode: this.options.xmlMode,\n    lowerCaseTags: this.options.lowerCaseTags,\n    lowerCaseAttributeNames: this.options.lowerCaseAttributeNames,\n    pseudos: this.options.pseudos,\n    quirksMode: this.options.quirksMode,\n  };\n\n  return this._make(select.select(selector, elems, options, limit));\n}\n\n/**\n * Creates a matcher, using a particular mapping function. Matchers provide a\n * function that finds elements using a generating function, supporting\n * filtering.\n *\n * @private\n * @param matchMap - Mapping function.\n * @returns - Function for wrapping generating functions.\n */\nfunction _getMatcher<P>(\n  matchMap: (fn: (elem: AnyNode) => P, elems: Cheerio<AnyNode>) => Element[],\n) {\n  return function (\n    fn: (elem: AnyNode) => P,\n    ...postFns: ((elems: Element[]) => Element[])[]\n  ) {\n    return function <T extends AnyNode>(\n      this: Cheerio<T>,\n      selector?: AcceptedFilters<Element>,\n    ): Cheerio<Element> {\n      let matched: Element[] = matchMap(fn, this);\n\n      if (selector) {\n        matched = filterArray(\n          matched,\n          selector,\n          this.options.xmlMode,\n          this._root?.[0],\n        );\n      }\n\n      return this._make(\n        // Post processing is only necessary if there is more than one element.\n        this.length > 1 && matched.length > 1\n          ? postFns.reduce((elems, fn) => fn(elems), matched)\n          : matched,\n      );\n    };\n  };\n}\n\n/** Matcher that adds multiple elements for each entry in the input. */\nconst _matcher = _getMatcher((fn: (elem: AnyNode) => Element[], elems) => {\n  let ret: Element[] = [];\n\n  for (let i = 0; i < elems.length; i++) {\n    const value = fn(elems[i]);\n    if (value.length > 0) ret = ret.concat(value);\n  }\n\n  return ret;\n});\n\n/** Matcher that adds at most one element for each entry in the input. */\nconst _singleMatcher = _getMatcher(\n  (fn: (elem: AnyNode) => Element | null, elems) => {\n    const ret: Element[] = [];\n\n    for (let i = 0; i < elems.length; i++) {\n      const value = fn(elems[i]);\n      if (value !== null) {\n        ret.push(value);\n      }\n    }\n    return ret;\n  },\n);\n\n/**\n * Matcher that supports traversing until a condition is met.\n *\n * @param nextElem - Function that returns the next element.\n * @param postFns - Post processing functions.\n * @returns A function usable for `*Until` methods.\n */\nfunction _matchUntil(\n  nextElem: (elem: AnyNode) => Element | null,\n  ...postFns: ((elems: Element[]) => Element[])[]\n) {\n  // We use a variable here that is used from within the matcher.\n  let matches: ((el: Element, i: number) => boolean) | null = null;\n\n  const innerMatcher = _getMatcher(\n    (nextElem: (elem: AnyNode) => Element | null, elems) => {\n      const matched: Element[] = [];\n\n      domEach(elems, (elem) => {\n        for (let next; (next = nextElem(elem)); elem = next) {\n          // FIXME: `matched` might contain duplicates here and the index is too large.\n          if (matches?.(next, matched.length)) break;\n          matched.push(next);\n        }\n      });\n\n      return matched;\n    },\n  )(nextElem, ...postFns);\n\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    selector?: AcceptedFilters<Element> | null,\n    filterSelector?: AcceptedFilters<Element>,\n  ): Cheerio<Element> {\n    // Override `matches` variable with the new target.\n    matches =\n      typeof selector === 'string'\n        ? (elem: Element) => select.is(elem, selector, this.options)\n        : selector\n          ? getFilterFn(selector)\n          : null;\n\n    const ret = innerMatcher.call(this, filterSelector);\n\n    // Set `matches` to `null`, so we don't waste memory.\n    matches = null;\n\n    return ret;\n  };\n}\n\nfunction _removeDuplicates<T extends AnyNode>(elems: T[]): T[] {\n  return elems.length > 1 ? Array.from(new Set<T>(elems)) : elems;\n}\n\n/**\n * Get the parent of each element in the current set of matched elements,\n * optionally filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').parent().attr('id');\n * //=> fruits\n * ```\n *\n * @param selector - If specified filter for parent.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parent/}\n */\nexport const parent = _singleMatcher(\n  ({ parent }) => (parent && !isDocument(parent) ? (parent as Element) : null),\n  _removeDuplicates,\n);\n\n/**\n * Get a set of parents filtered by `selector` of each element in the current\n * set of match elements.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').parents().length;\n * //=> 2\n * $('.orange').parents('#fruits').length;\n * //=> 1\n * ```\n *\n * @param selector - If specified filter for parents.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parents/}\n */\nexport const parents = _matcher(\n  (elem) => {\n    const matched = [];\n    while (elem.parent && !isDocument(elem.parent)) {\n      matched.push(elem.parent as Element);\n      elem = elem.parent;\n    }\n    return matched;\n  },\n  uniqueSort,\n  (elems) => elems.reverse(),\n);\n\n/**\n * Get the ancestors of each element in the current set of matched elements, up\n * to but not including the element matched by the selector, DOM node, or\n * cheerio object.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').parentsUntil('#food').length;\n * //=> 1\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - Optional filter for parents.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parentsUntil/}\n */\nexport const parentsUntil = _matchUntil(\n  ({ parent }) => (parent && !isDocument(parent) ? (parent as Element) : null),\n  uniqueSort,\n  (elems) => elems.reverse(),\n);\n\n/**\n * For each element in the set, get the first element that matches the selector\n * by testing the element itself and traversing up through its ancestors in the\n * DOM tree.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').closest();\n * //=> []\n *\n * $('.orange').closest('.apple');\n * // => []\n *\n * $('.orange').closest('li');\n * //=> [<li class=\"orange\">Orange</li>]\n *\n * $('.orange').closest('#fruits');\n * //=> [<ul id=\"fruits\"> ... </ul>]\n * ```\n *\n * @param selector - Selector for the element to find.\n * @returns The closest nodes.\n * @see {@link https://api.jquery.com/closest/}\n */\nexport function closest<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: AcceptedFilters<Element>,\n): Cheerio<AnyNode> {\n  const set: AnyNode[] = [];\n\n  if (!selector) {\n    return this._make(set);\n  }\n\n  const selectOpts = {\n    xmlMode: this.options.xmlMode,\n    root: this._root?.[0],\n  };\n\n  const selectFn =\n    typeof selector === 'string'\n      ? (elem: Element) => select.is(elem, selector, selectOpts)\n      : getFilterFn(selector);\n\n  domEach(this, (elem: AnyNode | null) => {\n    if (elem && !isDocument(elem) && !isTag(elem)) {\n      elem = elem.parent;\n    }\n    while (elem && isTag(elem)) {\n      if (selectFn(elem, 0)) {\n        // Do not add duplicate elements to the set\n        if (!set.includes(elem)) {\n          set.push(elem);\n        }\n        break;\n      }\n      elem = elem.parent;\n    }\n  });\n\n  return this._make(set);\n}\n\n/**\n * Gets the next sibling of each selected element, optionally filtered by a\n * selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').next().hasClass('orange');\n * //=> true\n * ```\n *\n * @param selector - If specified filter for sibling.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/next/}\n */\nexport const next = _singleMatcher((elem) => nextElementSibling(elem));\n\n/**\n * Gets all the following siblings of the each selected element, optionally\n * filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').nextAll();\n * //=> [<li class=\"orange\">Orange</li>, <li class=\"pear\">Pear</li>]\n * $('.apple').nextAll('.orange');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/nextAll/}\n */\nexport const nextAll = _matcher((elem) => {\n  const matched = [];\n  while (elem.next) {\n    elem = elem.next;\n    if (isTag(elem)) matched.push(elem);\n  }\n  return matched;\n}, _removeDuplicates);\n\n/**\n * Gets all the following siblings up to but not including the element matched\n * by the selector, optionally filtered by another selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').nextUntil('.pear');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - If specified filter for siblings.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/nextUntil/}\n */\nexport const nextUntil = _matchUntil(\n  (el) => nextElementSibling(el),\n  _removeDuplicates,\n);\n\n/**\n * Gets the previous sibling of each selected element optionally filtered by a\n * selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').prev().hasClass('apple');\n * //=> true\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prev/}\n */\nexport const prev = _singleMatcher((elem) => prevElementSibling(elem));\n\n/**\n * Gets all the preceding siblings of each selected element, optionally filtered\n * by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').prevAll();\n * //=> [<li class=\"orange\">Orange</li>, <li class=\"apple\">Apple</li>]\n *\n * $('.pear').prevAll('.orange');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prevAll/}\n */\nexport const prevAll = _matcher((elem) => {\n  const matched = [];\n  while (elem.prev) {\n    elem = elem.prev;\n    if (isTag(elem)) matched.push(elem);\n  }\n  return matched;\n}, _removeDuplicates);\n\n/**\n * Gets all the preceding siblings up to but not including the element matched\n * by the selector, optionally filtered by another selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').prevUntil('.apple');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n...\n", "needle_token_start": 9961, "needle_token_end": 10057, "code_context_ntokens": 15383, "output": ["```typescript\n/**\n * Removes all children from each item in the selection. Text nodes and comment\n * nodes are left as is.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').empty();\n * $.html();\n * //=>  <ul id=\"fruits\"></ul>\n * ```\n *\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/empty/}\n */\nexport function empty<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    el.children.length = 0;\n  });\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "withClass", "language": "typescript", "path": "src/api/attributes.spec.ts", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a new HTML element with a specified class attribute.\n2. **Input**: A single string representing the class name to be assigned to the new HTML element.\n3. **Output**: An object representing a newly created HTML `<div>` element with the specified class attribute.\n4. **Procedure**: The function takes a class name as input, constructs an HTML string for a `<div>` element with that class, and then uses a library to parse this HTML string into an object that represents the `<div>` element with the given class attribute.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/slim.ts\n/**\n * @file Alternative entry point for Cheerio that always uses htmlparser2. This\n *   way, parse5 won't be loaded, saving some memory.\n */\n\nexport type {\n...\n// Path: src/index.spec.ts\nimport * as cheerio from './index.js';\nimport * as statics from './static.js';\n\ndescribe('static method re-exports', () => {\n  it('should export expected static methods', () => {\n    for (const key of Object.keys(statics) as (keyof typeof statics)[]) {\n      if (key === 'extract') continue;\n      expect(typeof cheerio[key]).toBe(typeof statics[key]);\n    }\n  });\n\n  it('should have a functional `html` that is bound to the default instance', () => {\n    expect(cheerio.html(cheerio.default('<div>test div</div>'))).toBe(\n      '<div>test div</div>',\n    );\n  });\n\n  it('should have a functional `xml` that is bound to the default instance', () => {\n    expect(cheerio.xml(cheerio.default('<div>test div</div>'))).toBe(\n      '<div>test div</div>',\n    );\n  });\n\n  it('should have a functional `text` that is bound to the default instance', () => {\n    expect(cheerio.text(cheerio.default('<div>test div</div>'))).toBe(\n      'test div',\n    );\n  });\n});\n\n// Path: src/parse.spec.ts\nimport type { Document, Element } from 'domhandler';\nimport { getParse } from './parse.js';\n\nimport { parseDocument as parseWithHtmlparser2 } from 'htmlparser2';\nimport { parseWithParse5 } from './parsers/parse5-adapter.js';\n\nconst defaultOpts = { _useHtmlParser2: false };\n\nconst parse = getParse((content, options, isDocument, context) =>\n  options._useHtmlParser2\n    ? parseWithHtmlparser2(content, options)\n    : parseWithParse5(content, options, isDocument, context),\n);\n\n// Tags\nconst basic = '<html></html>';\nconst siblings = '<h2></h2><p></p>';\n\n// Single Tags\nconst single = '<br/>';\nconst singleWrong = '<br>';\n\n// Children\nconst children = '<html><br/></html>';\nconst li = '<li class=\"durian\">Durian</li>';\n\n// Attributes\nconst attributes = '<img src=\"hello.png\" alt=\"man waving\">';\nconst noValueAttribute = '<textarea disabled></textarea>';\n\n// Comments\nconst comment = '<!-- sexy -->';\nconst conditional =\n  '<!--[if IE 8]><html class=\"no-js ie8\" lang=\"en\"><![endif]-->';\n\n// Text\nconst text = 'lorem ipsum';\n\n// Script\nconst script = '<script type=\"text/javascript\">alert(\"hi world!\");</script>';\nconst scriptEmpty = '<script></script>';\n\n// Style\nconst style = '<style type=\"text/css\"> h2 { color:blue; } </style>';\nconst styleEmpty = '<style></style>';\n\n// Directives\nconst directive = '<!doctype html>';\n\nfunction rootTest(root: Document) {\n  expect(root).toHaveProperty('type', 'root');\n\n  expect(root.nextSibling).toBe(null);\n  expect(root.previousSibling).toBe(null);\n  expect(root.parentNode).toBe(null);\n\n  const child = root.childNodes[0];\n  expect(child.parentNode).toBe(root);\n}\n\ndescribe('parse', () => {\n  describe('evaluate', () => {\n    it(`should parse basic empty tags: ${basic}`, () => {\n      const [tag] = parse(basic, defaultOpts, true, null).children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('html');\n      expect(tag.childNodes).toHaveLength(2);\n    });\n\n    it(`should handle sibling tags: ${siblings}`, () => {\n      const dom = parse(siblings, defaultOpts, false, null)\n        .children as Element[];\n      const [h2, p] = dom;\n\n      expect(dom).toHaveLength(2);\n      expect(h2.tagName).toBe('h2');\n      expect(p.tagName).toBe('p');\n    });\n\n    it(`should handle single tags: ${single}`, () => {\n      const [tag] = parse(single, defaultOpts, false, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('br');\n      expect(tag.childNodes).toHaveLength(0);\n    });\n\n    it(`should handle malformatted single tags: ${singleWrong}`, () => {\n      const [tag] = parse(singleWrong, defaultOpts, false, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('br');\n      expect(tag.childNodes).toHaveLength(0);\n    });\n\n    it(`should handle tags with children: ${children}`, () => {\n      const [tag] = parse(children, defaultOpts, true, null)\n        .children as Element[];\n      expect(tag.type).toBe('tag');\n      expect(tag.tagName).toBe('html');\n      expect(tag.childNodes).toBeTruthy();\n      expect(tag.childNodes[1]).toHaveProperty('tagName', 'body');\n      expect((tag.childNodes[1] as Element).childNodes).toHaveLength(1);\n    });\n\n    it(`should handle tags with children: ${li}`, () => {\n      const [tag] = parse(li, defaultOpts, false, null).children as Element[];\n      expect(tag.childNodes).toHaveLength(1);\n      expect(tag.childNodes[0]).toHaveProperty('data', 'Durian');\n    });\n\n    it(`should handle tags with attributes: ${attributes}`, () => {\n      const attrs = parse(attributes, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(attrs.attribs).toBeTruthy();\n      expect(attrs.attribs).toHaveProperty('src', 'hello.png');\n      expect(attrs.attribs).toHaveProperty('alt', 'man waving');\n    });\n\n    it(`should handle value-less attributes: ${noValueAttribute}`, () => {\n      const attrs = parse(noValueAttribute, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(attrs.attribs).toBeTruthy();\n      expect(attrs.attribs).toHaveProperty('disabled', '');\n    });\n\n    it(`should handle comments: ${comment}`, () => {\n      const elem = parse(comment, defaultOpts, false, null).children[0];\n      expect(elem.type).toBe('comment');\n      expect(elem).toHaveProperty('data', ' sexy ');\n    });\n\n    it(`should handle conditional comments: ${conditional}`, () => {\n      const elem = parse(conditional, defaultOpts, false, null).children[0];\n      expect(elem.type).toBe('comment');\n      expect(elem).toHaveProperty(\n        'data',\n        conditional.replace('<!--', '').replace('-->', ''),\n      );\n    });\n\n    it(`should handle text: ${text}`, () => {\n      const text_ = parse(text, defaultOpts, false, null).children[0];\n      expect(text_.type).toBe('text');\n      expect(text_).toHaveProperty('data', 'lorem ipsum');\n    });\n\n    it(`should handle script tags: ${script}`, () => {\n      const script_ = parse(script, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(script_.type).toBe('script');\n      expect(script_.tagName).toBe('script');\n      expect(script_.attribs).toHaveProperty('type', 'text/javascript');\n      expect(script_.childNodes).toHaveLength(1);\n      expect(script_.childNodes[0].type).toBe('text');\n      expect(script_.childNodes[0]).toHaveProperty(\n        'data',\n        'alert(\"hi world!\");',\n      );\n    });\n\n    it(`should handle style tags: ${style}`, () => {\n      const style_ = parse(style, defaultOpts, false, null)\n        .children[0] as Element;\n      expect(style_.type).toBe('style');\n      expect(style_.tagName).toBe('style');\n      expect(style_.attribs).toHaveProperty('type', 'text/css');\n      expect(style_.childNodes).toHaveLength(1);\n      expect(style_.childNodes[0].type).toBe('text');\n      expect(style_.childNodes[0]).toHaveProperty(\n        'data',\n        ' h2 { color:blue; } ',\n      );\n    });\n\n    it(`should handle directives: ${directive}`, () => {\n      const elem = parse(directive, defaultOpts, true, null).children[0];\n      expect(elem.type).toBe('directive');\n      expect(elem).toHaveProperty('data', '!DOCTYPE html');\n      expect(elem).toHaveProperty('name', '!doctype');\n    });\n  });\n\n  describe('.parse', () => {\n    // Root test utility\n\n    it(`should add root to: ${basic}`, () => {\n      const root = parse(basic, defaultOpts, true, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'html');\n    });\n\n    it(`should add root to: ${siblings}`, () => {\n      const root = parse(siblings, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(2);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'h2');\n      expect(root.childNodes[1]).toHaveProperty('tagName', 'p');\n      expect(root.childNodes[1].parent).toBe(root);\n    });\n\n    it(`should add root to: ${comment}`, () => {\n      const root = parse(comment, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('comment');\n    });\n\n    it(`should add root to: ${text}`, () => {\n      const root = parse(text, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('text');\n    });\n\n    it(`should add root to: ${scriptEmpty}`, () => {\n      const root = parse(scriptEmpty, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('script');\n    });\n\n    it(`should add root to: ${styleEmpty}`, () => {\n      const root = parse(styleEmpty, defaultOpts, false, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0].type).toBe('style');\n    });\n\n    it(`should add root to: ${directive}`, () => {\n      const root = parse(directive, defaultOpts, true, null);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(2);\n      expect(root.childNodes[0].type).toBe('directive');\n    });\n\n    it('should simply return root', () => {\n      const oldroot = parse(basic, defaultOpts, true, null);\n      const root = parse(oldroot, defaultOpts, true, null);\n      expect(root).toBe(oldroot);\n      rootTest(root);\n      expect(root.childNodes).toHaveLength(1);\n      expect(root.childNodes[0]).toHaveProperty('tagName', 'html');\n    });\n\n    it('should expose the DOM level 1 API', () => {\n      const root = parse(\n        '<div><a></a><span></span><p></p></div>',\n        defaultOpts,\n        false,\n        null,\n      ).childNodes[0] as Element;\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes).toHaveLength(3);\n\n      expect(root.tagName).toBe('div');\n      expect(root.firstChild).toBe(childNodes[0]);\n      expect(root.lastChild).toBe(childNodes[2]);\n\n      expect(childNodes[0].tagName).toBe('a');\n      expect(childNodes[0].previousSibling).toBe(null);\n      expect(childNodes[0].nextSibling).toBe(childNodes[1]);\n      expect(childNodes[0].parentNode).toBe(root);\n      expect((childNodes[0] as Element).childNodes).toHaveLength(0);\n      expect(childNodes[0].firstChild).toBe(null);\n      expect(childNodes[0].lastChild).toBe(null);\n\n      expect(childNodes[1].tagName).toBe('span');\n      expect(childNodes[1].previousSibling).toBe(childNodes[0]);\n      expect(childNodes[1].nextSibling).toBe(childNodes[2]);\n      expect(childNodes[1].parentNode).toBe(root);\n      expect(childNodes[1].childNodes).toHaveLength(0);\n      expect(childNodes[1].firstChild).toBe(null);\n      expect(childNodes[1].lastChild).toBe(null);\n\n      expect(childNodes[2].tagName).toBe('p');\n      expect(childNodes[2].previousSibling).toBe(childNodes[1]);\n      expect(childNodes[2].nextSibling).toBe(null);\n      expect(childNodes[2].parentNode).toBe(root);\n      expect(childNodes[2].childNodes).toHaveLength(0);\n      expect(childNodes[2].firstChild).toBe(null);\n      expect(childNodes[2].lastChild).toBe(null);\n    });\n\n    it('Should parse less than or equal sign sign', () => {\n      const root = parse('<i>A</i><=<i>B</i>', defaultOpts, false, null);\n      const { childNodes } = root;\n\n      expect(childNodes[0]).toHaveProperty('tagName', 'i');\n      expect((childNodes[0] as Element).childNodes[0]).toHaveProperty(\n        'data',\n        'A',\n      );\n      expect(childNodes[1]).toHaveProperty('data', '<=');\n      expect(childNodes[2]).toHaveProperty('tagName', 'i');\n      expect((childNodes[2] as Element).childNodes[0]).toHaveProperty(\n        'data',\n        'B',\n      );\n    });\n\n    it('Should ignore unclosed CDATA', () => {\n      const root = parse(\n        '<a></a><script>foo //<![CDATA[ bar</script><b></b>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('a');\n      expect(childNodes[1].tagName).toBe('script');\n      expect(childNodes[1].childNodes[0]).toHaveProperty(\n        'data',\n        'foo //<![CDATA[ bar',\n      );\n      expect(childNodes[2].tagName).toBe('b');\n    });\n\n    it('Should add <head> to documents', () => {\n      const root = parse('<html></html>', defaultOpts, true, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('html');\n      expect(childNodes[0].childNodes[0]).toHaveProperty('tagName', 'head');\n    });\n\n    it('Should implicitly create <tr> around <td>', () => {\n      const root = parse(\n        '<table><td>bar</td></tr></table>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].tagName).toBe('table');\n      expect(childNodes[0].childNodes.length).toBe(1);\n      expect(childNodes[0].childNodes[0]).toHaveProperty('tagName', 'tbody');\n      expect((childNodes[0] as any).childNodes[0].childNodes[0]).toHaveProperty(\n        'tagName',\n        'tr',\n      );\n      expect(\n        (childNodes[0] as any).childNodes[0].childNodes[0].childNodes[0]\n          .tagName,\n      ).toBe('td');\n      expect(\n        (childNodes[0] as any).childNodes[0].childNodes[0].childNodes[0]\n          .childNodes[0].data,\n      ).toBe('bar');\n    });\n\n    it('Should parse custom tag <line>', () => {\n      const root = parse('<line>test</line>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(1);\n      expect(childNodes[0].tagName).toBe('line');\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', 'test');\n    });\n\n    it('Should properly parse misnested table tags', () => {\n      const root = parse(\n        '<tr><td>i1</td></tr><tr><td>i2</td></td></tr><tr><td>i3</td></td></tr>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(3);\n\n      for (let i = 0; i < childNodes.length; i++) {\n        const child = childNodes[i];\n        expect(child.tagName).toBe('tr');\n        expect(child.childNodes[0]).toHaveProperty('tagName', 'td');\n        expect((child.childNodes[0] as Element).childNodes[0]).toHaveProperty(\n          'data',\n          `i${i + 1}`,\n        );\n      }\n    });\n\n    it('Should correctly parse data url attributes', () => {\n      const html =\n        '<div style=\\'font-family:\"butcherman-caps\"; src:url(data:font/opentype;base64,AAEA...);\\'></div>';\n      const expectedAttr =\n        'font-family:\"butcherman-caps\"; src:url(data:font/opentype;base64,AAEA...);';\n      const root = parse(html, defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].attribs).toHaveProperty('style', expectedAttr);\n    });\n\n    it('Should treat <xmp> tag content as text', () => {\n      const root = parse('<xmp><h2></xmp>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', '<h2>');\n    });\n\n    it('Should correctly parse malformed numbered entities', () => {\n      const root = parse('<p>z&#</p>', defaultOpts, false, null);\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes[0].childNodes[0]).toHaveProperty('data', 'z&#');\n    });\n\n    it('Should correctly parse mismatched headings', () => {\n      const root = parse('<h2>Test</h3><div></div>', defaultOpts, false, null);\n      const { childNodes } = root;\n\n      expect(childNodes.length).toBe(2);\n      expect(childNodes[0]).toHaveProperty('tagName', 'h2');\n      expect(childNodes[1]).toHaveProperty('tagName', 'div');\n    });\n\n    it('Should correctly parse tricky <pre> content', () => {\n      const root = parse(\n        '<pre>\\nA <- factor(A, levels = c(\"c\",\"a\",\"b\"))\\n</pre>',\n        defaultOpts,\n        false,\n        null,\n      );\n      const childNodes = root.childNodes as Element[];\n\n      expect(childNodes.length).toBe(1);\n      expect(childNodes[0].tagName).toBe('pre');\n      expect(childNodes[0].childNodes[0]).toHaveProperty(\n        'data',\n        'A <- factor(A, levels = c(\"c\",\"a\",\"b\"))\\n',\n      );\n    });\n\n    it('should pass the options for including the location info to parse5', () => {\n      const root = parse(\n        '<p>Hello</p>',\n        { ...defaultOpts, sourceCodeLocationInfo: true },\n        false,\n        null,\n      );\n      const location = root.children[0].sourceCodeLocation;\n\n      expect(typeof location).toBe('object');\n      expect(location?.endOffset).toBe(12);\n    });\n  });\n});\n\n// Path: src/__tests__/xml.spec.ts\nimport cheerio from '../index.js';\nimport type { CheerioOptions } from '../options.js';\n\nfunction xml(str: string, options?: CheerioOptions) {\n  options = { xml: true, ...options };\n  const $ = cheerio.load(str, options);\n  return $.xml();\n}\n\nfunction dom(str: string, options?: CheerioOptions) {\n  const $ = cheerio.load('', options);\n  return $(str).html();\n}\n\ndescribe('render', () => {\n  describe('(xml)', () => {\n    it('should render <media:thumbnail /> tags correctly', () => {\n      const str =\n        '<media:thumbnail url=\"http://www.foo.com/keyframe.jpg\" width=\"75\" height=\"50\" time=\"12:05:01.123\" />';\n      expect(xml(str)).toBe(\n        '<media:thumbnail url=\"http://www.foo.com/keyframe.jpg\" width=\"75\" height=\"50\" time=\"12:05:01.123\"/>',\n      );\n    });\n\n    it('should render <link /> tags (RSS) correctly', () => {\n      const str = '<link>http://www.github.com/</link>';\n      expect(xml(str)).toBe('<link>http://www.github.com/</link>');\n    });\n\n    it('should escape entities', () => {\n      const str = '<tag attr=\"foo &amp; bar\"/>';\n      expect(xml(str)).toBe(str);\n    });\n\n    it('should render HTML as XML', () => {\n      const $ = cheerio.load('<foo></foo>', null, false);\n      expect($.xml()).toBe('<foo/>');\n    });\n  });\n\n  describe('(dom)', () => {\n    it('should not keep camelCase for new nodes', () => {\n      const str = '<g><someElem someAttribute=\"something\">hello</someElem></g>';\n      expect(dom(str, { xml: false })).toBe(\n        '<someelem someattribute=\"something\">hello</someelem>',\n      );\n    });\n\n    it('should keep camelCase for new nodes', () => {\n      const str = '<g><someElem someAttribute=\"something\">hello</someElem></g>';\n      expect(dom(str, { xml: true })).toBe(\n        '<someElem someAttribute=\"something\">hello</someElem>',\n      );\n    });\n\n    it('should maintain the parsing options of distinct contexts independently', () => {\n      const str = '<g><someElem someAttribute=\"something\">hello</someElem></g>';\n      const $ = cheerio.load('', { xml: false });\n\n      expect($(str).html()).toBe(\n        '<someelem someattribute=\"something\">hello</someelem>',\n      );\n    });\n  });\n});\n\n// Path: src/__tests__/deprecated.spec.ts\n/**\n * This file includes tests for deprecated APIs. The methods are expected to be\n * removed in the next major release of Cheerio, but their stability should be\n * maintained until that time.\n */\nimport * as fixtures from '../__fixtures__/fixtures.js';\nimport cheerio from '../index.js';\n\ndescribe('deprecated APIs', () => {\n  describe('cheerio module', () => {\n    describe('.parseHTML', () => {\n      it('(html) : should preserve content', () => {\n        const html = '<div>test div</div>';\n        expect(cheerio(cheerio.parseHTML(html)[0]).html()).toBe('test div');\n      });\n    });\n\n    describe('.merge', () => {\n      it('should be a function', () => {\n        expect(typeof cheerio.merge).toBe('function');\n      });\n\n      // #1674 - merge, wont accept Cheerio object\n      it('should be a able merge array and cheerio object', () => {\n        const ret = cheerio.merge<unknown>(cheerio(), ['elem1', 'elem2']);\n        expect(typeof ret).toBe('object');\n        expect(ret).toHaveLength(2);\n      });\n\n      it('(arraylike, arraylike) : should modify the first array, but not the second', () => {\n        const arr1 = [1, 2, 3];\n        const arr2 = [4, 5, 6];\n        const ret = cheerio.merge(arr1, arr2);\n\n        expect(typeof ret).toBe('object');\n        expect(Array.isArray(ret)).toBe(true);\n        expect(ret).toBe(arr1);\n        expect(arr1).toHaveLength(6);\n        expect(arr2).toHaveLength(3);\n      });\n\n      it('(arraylike, arraylike) : should handle objects that arent arrays, but are arraylike', () => {\n        const arr1: ArrayLike<string> = {\n          length: 3,\n          0: 'a',\n          1: 'b',\n          2: 'c',\n        };\n        const arr2 = {\n          length: 3,\n          0: 'd',\n          1: 'e',\n          2: 'f',\n        };\n\n        cheerio.merge(arr1, arr2);\n        expect(arr1).toHaveLength(6);\n        expect(arr1[3]).toBe('d');\n        expect(arr1[4]).toBe('e');\n        expect(arr1[5]).toBe('f');\n        expect(arr2).toHaveLength(3);\n      });\n\n      it('(?, ?) : should gracefully reject invalid inputs', () => {\n        expect(cheerio.merge([4], 3 as never)).toBeUndefined();\n        expect(cheerio.merge({} as never, {} as never)).toBeUndefined();\n        expect(cheerio.merge([], {} as never)).toBeUndefined();\n        expect(cheerio.merge({} as never, [])).toBeUndefined();\n\n        const fakeArray = { length: 3, 0: 'a', 1: 'b', 3: 'd' };\n        expect(cheerio.merge(fakeArray, [])).toBeUndefined();\n        expect(cheerio.merge([], fakeArray)).toBeUndefined();\n\n        expect(cheerio.merge({ length: '7' } as never, [])).toBeUndefined();\n        expect(cheerio.merge({ length: -1 }, [])).toBeUndefined();\n      });\n\n      it('(?, ?) : should no-op on invalid inputs', () => {\n        const fakeArray1 = { length: 3, 0: 'a', 1: 'b', 3: 'd' };\n        cheerio.merge(fakeArray1, []);\n        expect(fakeArray1).toHaveLength(3);\n        expect(fakeArray1[0]).toBe('a');\n        expect(fakeArray1[1]).toBe('b');\n        expect(fakeArray1[3]).toBe('d');\n        cheerio.merge([], fakeArray1);\n        expect(fakeArray1).toHaveLength(3);\n        expect(fakeArray1[0]).toBe('a');\n        expect(fakeArray1[1]).toBe('b');\n        expect(fakeArray1[3]).toBe('d');\n      });\n    });\n\n    describe('.contains', () => {\n      let $: typeof cheerio;\n\n      beforeEach(() => {\n        $ = cheerio.load(fixtures.food);\n      });\n\n      it('(container, contained) : should correctly detect the provided element', () => {\n        const $food = $('#food');\n        const $fruits = $('#fruits');\n        const $apple = $('.apple');\n\n        expect(cheerio.contains($food[0], $fruits[0])).toBe(true);\n        expect(cheerio.contains($food[0], $apple[0])).toBe(true);\n      });\n\n      it('(container, other) : should not detect elements that are not contained', () => {\n        const $fruits = $('#fruits');\n        const $vegetables = $('#vegetables');\n        const $apple = $('.apple');\n\n        expect(cheerio.contains($vegetables[0], $apple[0])).toBe(false);\n        expect(cheerio.contains($fruits[0], $vegetables[0])).toBe(false);\n        expect(cheerio.contains($vegetables[0], $fruits[0])).toBe(false);\n        expect(cheerio.contains($fruits[0], $fruits[0])).toBe(false);\n        expect(cheerio.contains($vegetables[0], $vegetables[0])).toBe(false);\n      });\n    });\n\n    describe('.root', () => {\n      it('returns an empty selection', () => {\n        const $empty = cheerio.root();\n        expect($empty).toHaveLength(1);\n        expect($empty[0].children).toHaveLength(0);\n      });\n    });\n  });\n\n  describe('Cheerio function', () => {\n    it('.load', () => {\n      const $1 = cheerio.load(fixtures.fruits);\n      const $2 = $1.load('<div><p>Some <a>text</a>.</p></div>');\n\n      expect($2('a')).toHaveLength(1);\n    });\n\n    /**\n     * The `.html` static method defined on the \"loaded\" Cheerio factory\n     * function is deprecated.\n     *\n     * In order to promote consistency with the jQuery library, users are\n     * encouraged to instead use the instance method of the same name.\n     *\n     * @example\n     *\n     * ```js\n     * const $ = cheerio.load('<h1>Hello, <span>world</span>.</h1>');\n     *\n     * $('h1').html();\n     * //=> '<h1>Hello, <span>world</span>.'\n     * ```\n     *\n     * @example <caption>To render the markup of an entire document, invoke the\n     * `html` function exported by the Cheerio module with a \"root\"\n     * selection.</caption>\n     *\n     * ```js\n     * cheerio.html($.root());\n     * //=> '<html><head></head><body><h1>Hello, <span>world</span>.</h1></body></html>'\n     * ```\n     */\n    describe('.html - deprecated API', () => {\n      it('() : of empty cheerio object should return null', () => {\n        /*\n         * Note: the direct invocation of the Cheerio constructor function is\n         * also deprecated.\n         */\n        const $ = cheerio();\n        expect($.html()).toBe(null);\n      });\n\n      it('(selector) : should return the outerHTML of the selected element', () => {\n        const $ = cheerio.load(fixtures.fruits);\n        expect($.html('.pear')).toBe('<li class=\"pear\">Pear</li>');\n      });\n    });\n\n    /**\n     * The `.xml` static method defined on the \"loaded\" Cheerio factory function\n     * is deprecated. Users are encouraged to instead use the `xml` function\n     * exported by the Cheerio module.\n     *\n     * @example\n     *\n     * ```js\n     * cheerio.xml($.root());\n     * ```\n     */\n    describe('.xml  - deprecated API', () => {\n      it('() :  renders XML', () => {\n        const $ = cheerio.load('<foo></foo>', { xmlMode: true });\n        expect($.xml()).toBe('<foo/>');\n      });\n    });\n\n    /**\n     * The `.text` static method defined on the \"loaded\" Cheerio factory\n     * function is deprecated.\n     *\n     * In order to promote consistency with the jQuery library, users are\n     * encouraged to instead use the instance method of the same name.\n     *\n     * @example\n     *\n     * ```js\n     * const $ = cheerio.load('<h1>Hello, <span>world</span>.</h1>');\n     * $('h1').text();\n     * //=> 'Hello, world.'\n     * ```\n     *\n     * @example <caption>To render the text content of an entire document,\n     * invoke the `text` function exported by the Cheerio module with a \"root\"\n     * selection. </caption>\n     *\n     * ```js\n     * cheerio.text($.root());\n     * //=> 'Hello, world.'\n     * ```\n     */\n    describe('.text  - deprecated API', () => {\n      it('(cheerio object) : should return the text contents of the specified elements', () => {\n        const $ = cheerio.load('<a>This is <em>content</em>.</a>');\n        expect($.text($('a'))).toBe('This is content.');\n      });\n\n      it('(cheerio object) : should omit comment nodes', () => {\n        const $ = cheerio.load(\n          '<a>This is <!-- a comment --> not a comment.</a>',\n        );\n        expect($.text($('a'))).toBe('This is  not a comment.');\n      });\n\n      it('(cheerio object) : should include text contents of children recursively', () => {\n        const $ = cheerio.load(\n          '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n        );\n        expect($.text($('a'))).toBe(\n          'This is a child with another child and  not a comment followed by one last child and some final text.',\n        );\n      });\n\n      it('() : should return the rendered text content of the root', () => {\n        const $ = cheerio.load(\n          '<a>This is <div>a child with <span>another child and <!-- a comment --> not a comment</span> followed by <em>one last child</em> and some final</div> text.</a>',\n        );\n        expect($.text()).toBe(\n          'This is a child with another child and  not a comment followed by one last child and some final text.',\n        );\n      });\n\n      it('(cheerio object) : should not omit script tags', () => {\n        const $ = cheerio.load('<script>console.log(\"test\")</script>');\n        expect($.text()).toBe('console.log(\"test\")');\n      });\n\n      it('(cheerio object) : should omit style tags', () => {\n        const $ = cheerio.load(\n          '<style type=\"text/css\">.cf-hidden { display: none; }</style>',\n        );\n        expect($.text()).toBe('.cf-hidden { display: none; }');\n      });\n    });\n  });\n});\n\n// Path: src/api/css.spec.ts\nimport cheerio, { load, type Cheerio } from '../index.js';\nimport type { Element } from 'domhandler';\nimport { mixedText } from '../__fixtures__/fixtures.js';\n\ndescribe('$(...)', () => {\n  describe('.css', () => {\n    it('(prop): should return a css property value', () => {\n      const el = cheerio('<li style=\"hai: there\">');\n      expect(el.css('hai')).toBe('there');\n    });\n\n    it('([prop1, prop2]): should return the specified property values as an object', () => {\n      const el = cheerio(\n        '<li style=\"margin: 1px; padding: 2px; color: blue;\">',\n      );\n      expect(el.css(['margin', 'color'])).toStrictEqual({\n        margin: '1px',\n        color: 'blue',\n      });\n    });\n\n    it('(prop, val): should set a css property', () => {\n      const el = cheerio('<li style=\"margin: 0;\"></li><li></li>');\n      el.css('color', 'red');\n      expect(el.attr('style')).toBe('margin: 0; color: red;');\n      expect(el.eq(1).attr('style')).toBe('color: red;');\n    });\n\n    it('(prop, val) : should skip text nodes', () => {\n      const $text = load(mixedText);\n      const $body = $text($text('body')[0].children);\n\n      $body.css('test', 'value');\n\n      expect($text('body').html()).toBe(\n        '<a style=\"test: value;\">1</a>TEXT<b style=\"test: value;\">2</b>',\n      );\n    });\n\n    it('(prop, \"\"): should unset a css property', () => {\n      const el = cheerio('<li style=\"padding: 1px; margin: 0;\">');\n      el.css('padding', '');\n      expect(el.attr('style')).toBe('margin: 0;');\n    });\n\n    it('(any, val): should ignore unsupported prop types', () => {\n      const el = cheerio('<li style=\"padding: 1px;\">');\n      el.css(123 as never, 'test');\n      expect(el.attr('style')).toBe('padding: 1px;');\n    });\n\n    it('(prop): should not mangle embedded urls', () => {\n      const el = cheerio(\n        '<li style=\"background-image:url(http://example.com/img.png);\">',\n      );\n      expect(el.css('background-image')).toBe(\n        'url(http://example.com/img.png)',\n      );\n    });\n\n    it('(prop): should ignore blank properties', () => {\n      const el = cheerio('<li style=\":#ccc;color:#aaa;\">');\n      expect(el.css()).toStrictEqual({ color: '#aaa' });\n    });\n\n    it('(prop): should ignore blank values', () => {\n      const el = cheerio('<li style=\"color:;position:absolute;\">');\n      expect(el.css()).toStrictEqual({ position: 'absolute' });\n    });\n\n    it('(prop): should return undefined for unmatched elements', () => {\n      const $ = load('<li style=\"color:;position:absolute;\">');\n      expect($('ul').css('background-image')).toBeUndefined();\n    });\n\n    it('(prop): should return undefined for unmatched styles', () => {\n      const el = cheerio('<li style=\"color:;position:absolute;\">');\n      expect(el.css('margin')).toBeUndefined();\n    });\n\n    describe('(prop, function):', () => {\n      let $el: Cheerio<Element>;\n      beforeEach(() => {\n        const $ = load(\n          '<div style=\"margin: 0px;\"></div><div style=\"margin: 1px;\"></div><div style=\"margin: 2px;\">',\n        );\n        $el = $('div');\n      });\n\n      it('should iterate over the selection', () => {\n        let count = 0;\n        $el.css('margin', function (idx, value) {\n          expect(idx).toBe(count);\n          expect(value).toBe(`${count}px`);\n          expect(this).toBe($el[count]);\n          count++;\n          return undefined;\n        });\n        expect(count).toBe(3);\n      });\n\n      it('should set each attribute independently', () => {\n        const values = ['4px', '', undefined];\n        $el.css('margin', (idx) => values[idx]);\n        expect($el.eq(0).attr('style')).toBe('margin: 4px;');\n        expect($el.eq(1).attr('style')).toBe('');\n        expect($el.eq(2).attr('style')).toBe('margin: 2px;');\n      });\n    });\n\n    it('(obj): should set each key and val', () => {\n      const el = cheerio('<li style=\"padding: 0;\"></li><li></li>');\n      el.css({ foo: 0 } as never);\n      expect(el.eq(0).attr('style')).toBe('padding: 0; foo: 0;');\n      expect(el.eq(1).attr('style')).toBe('foo: 0;');\n    });\n\n    describe('parser', () => {\n      it('should allow any whitespace between declarations', () => {\n        const el = cheerio('<li style=\"one \\t:\\n 0;\\n two \\f\\r:\\v 1\">');\n        expect(el.css(['one', 'two', 'five'])).toStrictEqual({\n          one: '0',\n          two: '1',\n        });\n      });\n\n      it('should add malformed values to previous field (#1134)', () => {\n        const el = cheerio(\n          '<button style=\"background-image: url(data:image/png;base64,iVBORw0KGgo)\"></button>',\n        );\n        expect(el.css('background-image')).toStrictEqual(\n          'url(data:image/png;base64,iVBORw0KGgo)',\n        );\n      });\n    });\n  });\n});\n\n// Path: src/api/forms.spec.ts\nimport cheerio, { type CheerioAPI } from '../index.js';\nimport { forms } from '../__fixtures__/fixtures.js';\n\ndescribe('$(...)', () => {\n  let $: CheerioAPI;\n\n  beforeEach(() => {\n    $ = cheerio.load(forms);\n  });\n\n  describe('.serializeArray', () => {\n    it('() : should get form controls', () => {\n      expect($('form#simple').serializeArray()).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'Apple',\n        },\n      ]);\n    });\n\n    it('() : should get nested form controls', () => {\n      expect($('form#nested').serializeArray()).toHaveLength(2);\n      const data = $('form#nested').serializeArray();\n      data.sort((a, b) => (a.value > b.value ? 1 : -1));\n      expect(data).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'Apple',\n        },\n        {\n          name: 'vegetable',\n          value: 'Carrot',\n        },\n      ]);\n    });\n\n    it('() : should not get disabled form controls', () => {\n      expect($('form#disabled').serializeArray()).toStrictEqual([]);\n    });\n\n    it('() : should not get form controls with the wrong type', () => {\n      expect($('form#submit').serializeArray()).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'Apple',\n        },\n      ]);\n    });\n\n    it('() : should get selected options', () => {\n      expect($('form#select').serializeArray()).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'Orange',\n        },\n      ]);\n    });\n\n    it('() : should not get unnamed form controls', () => {\n      expect($('form#unnamed').serializeArray()).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'Apple',\n        },\n      ]);\n    });\n\n    it('() : should get multiple selected options', () => {\n      expect($('form#multiple').serializeArray()).toHaveLength(2);\n      const data = $('form#multiple').serializeArray();\n      data.sort((a, b) => (a.value > b.value ? 1 : -1));\n      expect(data).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'Apple',\n        },\n        {\n          name: 'fruit',\n          value: 'Orange',\n        },\n      ]);\n    });\n\n    it('() : should get individually selected elements', () => {\n      const data = $('form#nested input').serializeArray();\n      data.sort((a, b) => (a.value > b.value ? 1 : -1));\n      expect(data).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'Apple',\n        },\n        {\n          name: 'vegetable',\n          value: 'Carrot',\n        },\n      ]);\n    });\n\n    it('() : should standardize line breaks', () => {\n      expect($('form#textarea').serializeArray()).toStrictEqual([\n        {\n          name: 'fruits',\n          value: 'Apple\\r\\nOrange',\n        },\n      ]);\n    });\n\n    it(\"() : shouldn't serialize the empty string\", () => {\n      expect($('<input value=pineapple>').serializeArray()).toStrictEqual([]);\n      expect(\n        $('<input name=\"\" value=pineapple>').serializeArray(),\n      ).toStrictEqual([]);\n      expect(\n        $('<input name=\"fruit\" value=pineapple>').serializeArray(),\n      ).toStrictEqual([\n        {\n          name: 'fruit',\n          value: 'pineapple',\n        },\n      ]);\n    });\n\n    it('() : should serialize inputs without value attributes', () => {\n      expect($('<input name=\"fruit\">').serializeArray()).toStrictEqual([\n        {\n          name: 'fruit',\n          value: '',\n        },\n      ]);\n    });\n  });\n\n  describe('.serialize', () => {\n    it('() : should get form controls', () => {\n      expect($('form#simple').serialize()).toBe('fruit=Apple');\n    });\n\n    it('() : should get nested form controls', () => {\n      expect($('form#nested').serialize()).toBe('fruit=Apple&vegetable=Carrot');\n    });\n\n    it('() : should not get disabled form controls', () => {\n      expect($('form#disabled').serialize()).toBe('');\n    });\n\n    it('() : should get multiple selected options', () => {\n      expect($('form#multiple').serialize()).toBe('fruit=Apple&fruit=Orange');\n    });\n\n    it(\"() : should encode spaces as +'s\", () => {\n      expect($('form#spaces').serialize()).toBe('fruit=Blood+orange');\n    });\n  });\n});\n\n// Path: src/api/attributes.spec.ts\nimport cheerio, { load, type CheerioAPI, type Cheerio } from '../index.js';\nimport type { Element } from 'domhandler';\nimport {\n  script,\n  fruits,\n  vegetables,\n  food,\n  chocolates,\n  inputs,\n  mixedText,\n} from '../__fixtures__/fixtures.js';\n\n\nfunction withClass(attr: string) {\n  return cheerio(`<div class=\"${attr}\"></div>`);\n}\n\ndescribe('$(...)', () => {\n  describe('.attr', () => {\n    let $: CheerioAPI;\n\n    beforeEach(() => {\n      $ = load(fruits);\n    });\n\n    it('() : should get all the attributes', () => {\n      const attrs = $('ul').attr();\n      expect(attrs).toHaveProperty('id', 'fruits');\n    });\n\n    it('(invalid key) : invalid attr should get undefined', () => {\n      const attr = $('.apple').attr('lol');\n      expect(attr).toBeUndefined();\n    });\n\n    it('(valid key) : valid attr should get value', () => {\n      const cls = $('.apple').attr('class');\n      expect(cls).toBe('apple');\n    });\n\n    it('(valid key) : valid attr should get name when boolean', () => {\n      const attr = $('<input name=email autofocus>').attr('autofocus');\n      expect(attr).toBe('autofocus');\n    });\n\n    it('(key, value) : should set one attr', () => {\n      const $pear = $('.pear').attr('id', 'pear');\n      expect($('#pear')).toHaveLength(1);\n      expect($pear).toBeInstanceOf($);\n    });\n\n    it('(key, value) : should set multiple attr', () => {\n      const $el = cheerio('<div></div> <div></div>').attr(\n        'class',\n        'pear',\n      ) as Cheerio<Element>;\n\n      expect($el[0].attribs).toHaveProperty('class', 'pear');\n      expect($el[1].attribs).toBeUndefined();\n      expect($el[2].attribs).toHaveProperty('class', 'pear');\n    });\n\n    it('(key, value) : should return an empty object for an empty object', () => {\n      const $src = $().attr('key', 'value');\n      expect($src.length).toBe(0);\n      expect($src[0]).toBeUndefined();\n    });\n\n    it('(map) : object map should set multiple attributes', () => {\n      $('.apple').attr({\n        id: 'apple',\n        style: 'color:red;',\n        'data-url': 'http://apple.com',\n      });\n      const attrs = $('.apple').attr();\n      expect(attrs).toHaveProperty('id', 'apple');\n      expect(attrs).toHaveProperty('style', 'color:red;');\n      expect(attrs).toHaveProperty('data-url', 'http://apple.com');\n    });\n\n    it('(map, val) : should throw with wrong combination of arguments', () => {\n      expect(() =>\n        $('.apple').attr(\n          {\n            id: 'apple',\n            style: 'color:red;',\n            'data-url': 'http://apple.com',\n          } as never,\n          () => '',\n        ),\n      ).toThrow('Bad combination of arguments.');\n    });\n\n    it('(key, function) : should call the function and update the attribute with the return value', () => {\n      const $fruits = $('#fruits');\n      $fruits.attr('id', (index, value) => {\n        expect(index).toBe(0);\n        expect(value).toBe('fruits');\n        return 'ninja';\n      });\n      const attrs = $fruits.attr();\n      expect(attrs).toHaveProperty('id', 'ninja');\n    });\n\n    it('(key, function) : should ignore text nodes', () => {\n      const $text = $(mixedText);\n      $text.attr('class', () => 'ninja');\n      const className = $text.attr('class');\n      expect(className).toBe('ninja');\n    });\n\n    it('(key, value) : should correctly encode then decode unsafe values', () => {\n      const $apple = $('.apple');\n      $apple.attr(\n        'href',\n        'http://github.com/\"><script>alert(\"XSS!\")</script><br',\n      );\n      expect($apple.attr('href')).toBe(\n        'http://github.com/\"><script>alert(\"XSS!\")</script><br',\n      );\n\n      $apple.attr(\n        'href',\n        'http://github.com/\"><script>alert(\"XSS!\")</script><br',\n      );\n      expect($apple.html()).not.toContain('<script>alert(\"XSS!\")</script>');\n    });\n\n    it('(key, value) : should coerce values to a string', () => {\n      const $apple = $('.apple');\n      $apple.attr('data-test', 1 as never);\n      expect($apple[0].attribs['data-test']).toBe('1');\n      expect($apple.attr('data-test')).toBe('1');\n    });\n\n    it('(key, value) : handle removed boolean attributes', () => {\n      const $apple = $('.apple');\n      $apple.attr('autofocus', 'autofocus');\n      expect($apple.attr('autofocus')).toBe('autofocus');\n      $apple.removeAttr('autofocus');\n      expect($apple.attr('autofocus')).toBeUndefined();\n    });\n\n    it('(key, value) : should remove non-boolean attributes with names or values similar to boolean ones', () => {\n      const $apple = $('.apple');\n      $apple.attr('data-autofocus', 'autofocus');\n      expect($apple.attr('data-autofocus')).toBe('autofocus');\n      $apple.removeAttr('data-autofocus');\n      expect($apple.attr('data-autofocus')).toBeUndefined();\n    });\n\n    it('(key, value) : should remove attributes when called with null value', () => {\n      const $pear = $('.pear').attr('autofocus', 'autofocus');\n      expect($pear.attr('autofocus')).toBe('autofocus');\n      $pear.attr('autofocus', null);\n      expect($pear.attr('autofocus')).toBeUndefined();\n    });\n\n    it('(map) : should remove attributes with null values', () => {\n      const $pear = $('.pear').attr({\n        autofocus: 'autofocus',\n        style: 'color:red',\n      });\n      expect($pear.attr('autofocus')).toBe('autofocus');\n      expect($pear.attr('style')).toBe('color:red');\n      $pear.attr({ autofocus: null, style: 'color:blue' });\n      expect($pear.attr('autofocus')).toBeUndefined();\n      expect($pear.attr('style')).toBe('color:blue');\n    });\n\n    it('(chaining) setting value and calling attr returns result', () => {\n      const pearAttr = $('.pear').attr('foo', 'bar').attr('foo');\n      expect(pearAttr).toBe('bar');\n    });\n\n    it('(chaining) setting attr to null returns a $', () => {\n      const $pear = $('.pear').attr('foo', null);\n      expect($pear).toBeInstanceOf($);\n    });\n\n    it('(chaining) setting attr to undefined returns a $', () => {\n      const $pear = $('.pear').attr('foo', undefined);\n      expect($('.pear')).toHaveLength(1);\n      expect($('.pear').attr('foo')).toBeUndefined();\n      expect($pear).toBeInstanceOf($);\n    });\n\n    it(\"(bool) shouldn't treat boolean attributes differently in XML mode\", () => {\n      const $xml = $.load(`<input checked=checked disabled=yes />`, {\n        xml: true,\n      })('input');\n\n      expect($xml.attr('checked')).toBe('checked');\n      expect($xml.attr('disabled')).toBe('yes');\n    });\n  });\n\n  describe('.prop', () => {\n    let $: CheerioAPI;\n    let checkbox: Cheerio<Element>;\n\n    beforeEach(() => {\n      $ = load(inputs);\n      checkbox = $('input[name=checkbox_on]');\n    });\n\n    it('(valid key) : valid prop should get value', () => {\n      expect(checkbox.prop('checked')).toBe(true);\n      checkbox.css('display', 'none');\n      expect(checkbox.prop('style')).toHaveProperty('display', 'none');\n      expect(checkbox.prop('style')).toHaveLength(1);\n      expect(checkbox.prop('style')).toContain('display');\n      expect(checkbox.prop('tagName')).toBe('INPUT');\n      expect(checkbox.prop('nodeName')).toBe('INPUT');\n    });\n\n    it('(valid key) : should return on empty collection', () => {\n      expect($(undefined).prop('checked')).toBeUndefined();\n      expect($(undefined).prop('style')).toBeUndefined();\n      expect($(undefined).prop('tagName')).toBeUndefined();\n      expect($(undefined).prop('nodeName')).toBeUndefined();\n    });\n\n    it('(invalid key) : invalid prop should get undefined', () => {\n      expect(checkbox.prop('lol')).toBeUndefined();\n      expect(checkbox.prop(4 as never)).toBeUndefined();\n      expect(checkbox.prop(true as never)).toBeUndefined();\n    });\n\n    it('(key, value) : should set prop', () => {\n      expect(checkbox.prop('checked')).toBe(true);\n      checkbox.prop('checked', false);\n      expect(checkbox.prop('checked')).toBe(false);\n      checkbox.prop('checked', true);\n      expect(checkbox.prop('checked')).toBe(true);\n    });\n\n    it('(key, value) : should update attribute', () => {\n      expect(checkbox.prop('checked')).toBe(true);\n      expect(checkbox.attr('checked')).toBe('checked');\n      checkbox.prop('checked', false);\n      expect(checkbox.prop('checked')).toBe(false);\n      expect(checkbox.attr('checked')).toBeUndefined();\n      checkbox.prop('checked', true);\n      expect(checkbox.prop('checked')).toBe(true);\n      expect(checkbox.attr('checked')).toBe('checked');\n    });\n\n    it('(key, value) : should update namespace', () => {\n      const imgs = $('<img>\\n\\n<img>\\n\\n<img>');\n      const nsHtml = 'http://www.w3.org/1999/xhtml';\n      imgs.prop('src', '#').prop('namespace', nsHtml);\n      expect(imgs.prop('namespace')).toBe(nsHtml);\n      imgs.prop('attribs', null);\n      expect(imgs.prop('src')).toBeUndefined();\n      expect(imgs.prop('data-foo')).toBeUndefined();\n    });\n\n    it('(key, value) : should ignore empty collection', () => {\n      expect($(undefined).prop('checked')).toBeUndefined();\n      $(undefined).prop('checked', true);\n      expect($(undefined).prop('checked')).toBeUndefined();\n    });\n\n    it('(map) : object map should set multiple props', () => {\n      checkbox.prop({\n        id: 'check',\n        checked: false,\n      });\n      expect(checkbox.prop('id')).toBe('check');\n      expect(checkbox.prop('checked')).toBe(false);\n    });\n\n    it('(map, val) : should throw with wrong combination of arguments', () => {\n      expect(() =>\n        $('.apple').prop(\n          {\n            id: 'check',\n            checked: false,\n          } as never,\n          () => '',\n        ),\n      ).toThrow('Bad combination of arguments.');\n    });\n\n    it('(key, function) : should call the function and update the prop with the return value', () => {\n      checkbox.prop('checked', (index, value) => {\n        expect(index).toBe(0);\n        expect(value).toBe(true);\n        return false;\n      });\n      expect(checkbox.prop('checked')).toBe(false);\n    });\n\n    it('(key, value) : should support chaining after setting props', () => {\n      expect(checkbox.prop('checked', false)).toBe(checkbox);\n    });\n\n    it('(invalid element/tag) : prop should return undefined', () => {\n      expect($(undefined).prop('prop')).toBeUndefined();\n      expect($(null as never).prop('prop')).toBeUndefined();\n    });\n\n    it('(\"href\") : should resolve links with `baseURI`', () => {\n      const $ = load(\n        `\n          <a id=\"1\" href=\"http://example.org\">example1</a>\n          <a id=\"2\" href=\"//example.org\">example2</a>\n          <a id=\"3\" href=\"/example.org\">example3</a>\n          <a id=\"4\" href=\"example.org\">example4</a>\n        `,\n        { baseURI: 'http://example.com/page/1' },\n      );\n\n      expect($('#1').prop('href')).toBe('http://example.org/');\n      expect($('#2').prop('href')).toBe('http://example.org/');\n      expect($('#3').prop('href')).toBe('http://example.com/example.org');\n      expect($('#4').prop('href')).toBe('http://example.com/page/example.org');\n\n      expect($(undefined).prop('href')).toBeUndefined();\n    });\n\n    it('(\"src\") : should resolve links with `baseURI`', () => {\n      const $ = load(\n        `\n          <img id=\"1\" src=\"http://example.org/image.png\">\n          <iframe id=\"2\" src=\"//example.org/page.html\"></iframe>\n          <audio id=\"3\" src=\"/example.org/song.mp3\"></audio>\n          <source id=\"4\" src=\"example.org/image.png\">\n        `,\n        { baseURI: 'http://example.com/page/1' },\n      );\n\n      expect($('#1').prop('src')).toBe('http://example.org/image.png');\n      expect($('#2').prop('src')).toBe('http://example.org/page.html');\n      expect($('#3').prop('src')).toBe(\n        'http://example.com/example.org/song.mp3',\n      );\n      expect($('#4').prop('src')).toBe(\n        'http://example.com/page/example.org/image.png',\n      );\n\n      expect($(undefined).prop('src')).toBeUndefined();\n    });\n\n    it('(\"outerHTML\") : should render properly', () => {\n      const outerHtml = '<div><a></a></div>';\n      const $a = $(outerHtml);\n\n      expect($a.prop('outerHTML')).toBe(outerHtml);\n\n      expect($(undefined).prop('outerHTML')).toBeUndefined();\n    });\n\n    it('(\"innerHTML\") : should render properly', () => {\n      const $a = $('<div><a></a></div>');\n\n      expect($a.prop('innerHTML')).toBe('<a></a>');\n\n      expect($(undefined).prop('innerHTML')).toBeUndefined();\n    });\n\n    it('(\"textContent\") : should render properly', () => {\n      expect($('select').children().prop('textContent')).toBe(\n        'Option not selected',\n      );\n\n      expect($(script).prop('textContent')).toBe('A  var foo = \"bar\";B');\n\n      expect($(undefined).prop('textContent')).toBeUndefined();\n    });\n\n    it('(\"textContent\") : should include style and script tags', () => {\n      const $ = load(\n        '<body>Welcome <div>Hello, testing text function,<script>console.log(\"hello\")</script></div><style type=\"text/css\">.cf-hidden { display: none; }</style>End of message</body>',\n      );\n      expect($('body').prop('textContent')).toBe(\n...\n", "needle_token_start": 12113, "needle_token_end": 12140, "code_context_ntokens": 16147, "output": ["```typescript\nfunction withClass(attr: string) {\n  return cheerio(`<div class=\"${attr}\"></div>`);\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "appendTo", "language": "typescript", "path": "src/api/manipulation.ts", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to insert a set of elements at the end of a specified target element, effectively modifying the DOM to include new child elements under the specified parent.\n2. **Input**: The function takes a target element where the elements will be appended.\n3. **Output**: Returns the instance of the manipulated elements, allowing for method chaining.\n4. **Procedure**: \n   - First, the function determines if the target is already a suitable type or needs to be converted.\n   - It then appends the current set of elements to the determined target.\n   - Finally, the function returns the instance of the elements that were appended, facilitating further operations on them if needed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/static.ts\nimport type { BasicAcceptedElems } from './types.js';\nimport type { CheerioAPI, Cheerio } from './index.js';\nimport type { AnyNode, Document } from 'domhandler';\nimport { textContent } from 'domutils';\nimport {\n  type InternalOptions,\n  type CheerioOptions,\n  flattenOptions as flattenOptions,\n} from './options.js';\nimport type { ExtractedMap, ExtractMap } from './api/extract.js';\n\n/**\n * Helper function to render a DOM.\n *\n * @param that - Cheerio instance to render.\n * @param dom - The DOM to render. Defaults to `that`'s root.\n * @param options - Options for rendering.\n * @returns The rendered document.\n */\nfunction render(\n  that: CheerioAPI,\n  dom: BasicAcceptedElems<AnyNode> | undefined,\n  options: InternalOptions,\n): string {\n  if (!that) return '';\n\n  return that(dom ?? that._root.children, null, undefined, options).toString();\n}\n\n/**\n * Checks if a passed object is an options object.\n *\n * @param dom - Object to check if it is an options object.\n * @param options - Options object.\n * @returns Whether the object is an options object.\n */\nfunction isOptions(\n  dom?: BasicAcceptedElems<AnyNode> | CheerioOptions | null,\n  options?: CheerioOptions,\n): dom is CheerioOptions {\n  return (\n    !options &&\n    typeof dom === 'object' &&\n    dom != null &&\n    !('length' in dom) &&\n    !('type' in dom)\n  );\n}\n\n/**\n * Renders the document.\n *\n * @category Static\n * @param options - Options for the renderer.\n * @returns The rendered document.\n */\nexport function html(this: CheerioAPI, options?: CheerioOptions): string;\n/**\n * Renders the document.\n *\n * @category Static\n * @param dom - Element to render.\n * @param options - Options for the renderer.\n * @returns The rendered document.\n */\nexport function html(\n  this: CheerioAPI,\n  dom?: BasicAcceptedElems<AnyNode>,\n  options?: CheerioOptions,\n): string;\nexport function html(\n  this: CheerioAPI,\n  dom?: BasicAcceptedElems<AnyNode> | CheerioOptions,\n  options?: CheerioOptions,\n): string {\n  /*\n   * Be flexible about parameters, sometimes we call html(),\n   * with options as only parameter\n   * check dom argument for dom element specific properties\n   * assume there is no 'length' or 'type' properties in the options object\n   */\n  const toRender = isOptions(dom) ? ((options = dom), undefined) : dom;\n\n  /*\n   * Sometimes `$.html()` is used without preloading html,\n   * so fallback non-existing options to the default ones.\n   */\n  const opts = {\n    ...this?._options,\n    ...flattenOptions(options),\n  };\n\n  return render(this, toRender, opts);\n}\n\n/**\n * Render the document as XML.\n *\n * @category Static\n * @param dom - Element to render.\n * @returns THe rendered document.\n */\nexport function xml(\n  this: CheerioAPI,\n  dom?: BasicAcceptedElems<AnyNode>,\n): string {\n  const options = { ...this._options, xmlMode: true };\n\n  return render(this, dom, options);\n}\n\n/**\n * Render the document as text.\n *\n * This returns the `textContent` of the passed elements. The result will\n * include the contents of `<script>` and `<style>` elements. To avoid this, use\n * `.prop('innerText')` instead.\n *\n * @category Static\n * @param elements - Elements to render.\n * @returns The rendered document.\n */\nexport function text(\n  this: CheerioAPI | void,\n  elements?: ArrayLike<AnyNode>,\n): string {\n  const elems = elements ?? (this ? this.root() : []);\n\n  let ret = '';\n\n  for (let i = 0; i < elems.length; i++) {\n    ret += textContent(elems[i]);\n  }\n\n  return ret;\n}\n\n/**\n * Parses a string into an array of DOM nodes. The `context` argument has no\n * meaning for Cheerio, but it is maintained for API compatibility with jQuery.\n *\n * @category Static\n * @param data - Markup that will be parsed.\n * @param context - Will be ignored. If it is a boolean it will be used as the\n *   value of `keepScripts`.\n * @param keepScripts - If false all scripts will be removed.\n * @returns The parsed DOM.\n * @alias Cheerio.parseHTML\n * @see {@link https://api.jquery.com/jQuery.parseHTML/}\n */\nexport function parseHTML(\n  this: CheerioAPI,\n  data: string,\n  context?: unknown | boolean,\n  keepScripts?: boolean,\n): AnyNode[];\nexport function parseHTML(this: CheerioAPI, data?: '' | null): null;\nexport function parseHTML(\n  this: CheerioAPI,\n  data?: string | null,\n  context?: unknown | boolean,\n  keepScripts = typeof context === 'boolean' ? context : false,\n): AnyNode[] | null {\n  if (!data || typeof data !== 'string') {\n    return null;\n  }\n\n  if (typeof context === 'boolean') {\n    keepScripts = context;\n  }\n\n  const parsed = this.load(data, this._options, false);\n  if (!keepScripts) {\n    parsed('script').remove();\n  }\n\n  /*\n   * The `children` array is used by Cheerio internally to group elements that\n   * share the same parents. When nodes created through `parseHTML` are\n   * inserted into previously-existing DOM structures, they will be removed\n   * from the `children` array. The results of `parseHTML` should remain\n   * constant across these operations, so a shallow copy should be returned.\n   */\n  return [...parsed.root()[0].children];\n}\n\n/**\n * Sometimes you need to work with the top-level root element. To query it, you\n * can use `$.root()`.\n *\n * @category Static\n * @example\n *\n * ```js\n * $.root().append('<ul id=\"vegetables\"></ul>').html();\n * //=> <ul id=\"fruits\">...</ul><ul id=\"vegetables\"></ul>\n * ```\n *\n * @returns Cheerio instance wrapping the root node.\n * @alias Cheerio.root\n */\nexport function root(this: CheerioAPI): Cheerio<Document> {\n  return this(this._root);\n}\n\n/**\n * Checks to see if the `contained` DOM element is a descendant of the\n * `container` DOM element.\n *\n * @category Static\n * @param container - Potential parent node.\n * @param contained - Potential child node.\n * @returns Indicates if the nodes contain one another.\n * @alias Cheerio.contains\n * @see {@link https://api.jquery.com/jQuery.contains/}\n */\nexport function contains(container: AnyNode, contained: AnyNode): boolean {\n  // According to the jQuery API, an element does not \"contain\" itself\n  if (contained === container) {\n    return false;\n  }\n\n  /*\n   * Step up the descendants, stopping when the root element is reached\n   * (signaled by `.parent` returning a reference to the same object)\n   */\n  let next: AnyNode | null = contained;\n  while (next && next !== next.parent) {\n    next = next.parent;\n    if (next === container) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\n/**\n * Extract multiple values from a document, and store them in an object.\n *\n * @category Static\n * @param map - An object containing key-value pairs. The keys are the names of\n *   the properties to be created on the object, and the values are the\n *   selectors to be used to extract the values.\n * @returns An object containing the extracted values.\n */\nexport function extract<M extends ExtractMap>(\n  this: CheerioAPI,\n  map: M,\n): ExtractedMap<M> {\n  return this.root().extract(map);\n}\n\ntype Writable<T> = { -readonly [P in keyof T]: T[P] };\n\n/**\n * $.merge().\n *\n * @category Static\n * @param arr1 - First array.\n * @param arr2 - Second array.\n * @returns `arr1`, with elements of `arr2` inserted.\n * @alias Cheerio.merge\n * @see {@link https://api.jquery.com/jQuery.merge/}\n */\nexport function merge<T>(\n  arr1: Writable<ArrayLike<T>>,\n  arr2: ArrayLike<T>,\n): ArrayLike<T> | undefined {\n  if (!isArrayLike(arr1) || !isArrayLike(arr2)) {\n    return;\n  }\n  let newLength = arr1.length;\n  const len = +arr2.length;\n\n  for (let i = 0; i < len; i++) {\n...\n// Path: src/api/attributes.ts\n/**\n * Methods for getting and modifying attributes.\n *\n * @module cheerio/attributes\n */\n\nimport { text } from '../static.js';\nimport { domEach, camelCase, cssCase } from '../utils.js';\nimport { isTag, type AnyNode, type Element } from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport { innerText, textContent } from 'domutils';\nconst hasOwn = Object.prototype.hasOwnProperty;\nconst rspace = /\\s+/;\nconst dataAttrPrefix = 'data-';\n\n// Attributes that are booleans\nconst rboolean =\n  /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i;\n// Matches strings that look like JSON objects or arrays\nconst rbrace = /^{[^]*}$|^\\[[^]*]$/;\n\n/**\n * Gets a node's attribute. For boolean attributes, it will return the value's\n * name should it be set.\n *\n * Also supports getting the `value` of several form elements.\n *\n * @private\n * @category Attributes\n * @param elem - Element to get the attribute of.\n * @param name - Name of the attribute.\n * @param xmlMode - Disable handling of special HTML attributes.\n * @returns The attribute's value.\n */\nfunction getAttr(\n  elem: AnyNode,\n  name: undefined,\n  xmlMode?: boolean,\n): Record<string, string> | undefined;\nfunction getAttr(\n  elem: AnyNode,\n  name: string,\n  xmlMode?: boolean,\n): string | undefined;\nfunction getAttr(\n  elem: AnyNode,\n  name: string | undefined,\n  xmlMode?: boolean,\n): Record<string, string> | string | undefined {\n  if (!elem || !isTag(elem)) return undefined;\n\n  elem.attribs ??= {};\n\n  // Return the entire attribs object if no attribute specified\n  if (!name) {\n    return elem.attribs;\n  }\n\n  if (hasOwn.call(elem.attribs, name)) {\n    // Get the (decoded) attribute\n    return !xmlMode && rboolean.test(name) ? name : elem.attribs[name];\n  }\n\n  // Mimic the DOM and return text content as value for `option's`\n  if (elem.name === 'option' && name === 'value') {\n    return text(elem.children);\n  }\n\n  // Mimic DOM with default value for radios/checkboxes\n  if (\n    elem.name === 'input' &&\n    (elem.attribs['type'] === 'radio' || elem.attribs['type'] === 'checkbox') &&\n    name === 'value'\n  ) {\n    return 'on';\n  }\n\n  return undefined;\n}\n\n/**\n * Sets the value of an attribute. The attribute will be deleted if the value is\n * `null`.\n *\n * @private\n * @param el - The element to set the attribute on.\n * @param name - The attribute's name.\n * @param value - The attribute's value.\n */\nfunction setAttr(el: Element, name: string, value: string | null) {\n  if (value === null) {\n    removeAttribute(el, name);\n  } else {\n    el.attribs[name] = `${value}`;\n  }\n}\n\n/**\n * Method for getting attributes. Gets the attribute value for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr('id');\n * //=> fruits\n * ```\n *\n * @param name - Name of the attribute.\n * @returns The attribute's value.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): string | undefined;\n/**\n * Method for getting all attributes and their values of the first element in\n * the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('ul').attr();\n * //=> { id: 'fruits' }\n * ```\n *\n * @returns The attribute's values.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n): Record<string, string> | undefined;\n/**\n * Method for setting attributes. Sets the attribute value for only the first\n * element in the matched set. If you set an attribute's value to `null`, you\n * remove that attribute. You may also pass a `map` and `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr('id', 'favorite').html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param name - Name of the attribute.\n * @param value - The new value of the attribute.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): Cheerio<T>;\n/**\n * Method for setting multiple attributes at once. Sets the attribute value for\n * only the first element in the matched set. If you set an attribute's value to\n * `null`, you remove that attribute.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple').attr({ id: 'favorite' }).html();\n * //=> <li class=\"apple\" id=\"favorite\">Apple</li>\n * ```\n *\n * @param values - Map of attribute names and values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/attr/}\n */\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  values: Record<string, string | null>,\n): Cheerio<T>;\nexport function attr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name?: string | Record<string, string | null>,\n  value?:\n    | string\n    | null\n    | ((this: Element, i: number, attrib: string) => string | null),\n): string | Cheerio<T> | undefined | Record<string, string> {\n  // Set the value (with attr map support)\n  if (typeof name === 'object' || value !== undefined) {\n    if (typeof value === 'function') {\n      if (typeof name !== 'string') {\n        {\n          throw new Error('Bad combination of arguments.');\n        }\n      }\n      return domEach(this, (el, i) => {\n        if (isTag(el)) setAttr(el, name, value.call(el, i, el.attribs[name]));\n      });\n    }\n    return domEach(this, (el) => {\n      if (!isTag(el)) return;\n\n      if (typeof name === 'object') {\n        for (const objName of Object.keys(name)) {\n          const objValue = name[objName];\n          setAttr(el, objName, objValue);\n        }\n      } else {\n        setAttr(el, name as string, value as string);\n      }\n    });\n  }\n\n  return arguments.length > 1\n    ? this\n    : getAttr(this[0], name as string, this.options.xmlMode);\n}\n\n/**\n * Gets a node's prop.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the prop of.\n * @param name - Name of the prop.\n * @param xmlMode - Disable handling of special HTML attributes.\n * @returns The prop's value.\n */\nfunction getProp(\n  el: Element,\n  name: string,\n  xmlMode?: boolean,\n): string | undefined | Element[keyof Element] {\n  return name in el\n    ? // @ts-expect-error TS doesn't like us accessing the value directly here.\n      el[name]\n    : !xmlMode && rboolean.test(name)\n      ? getAttr(el, name, false) !== undefined\n      : getAttr(el, name, xmlMode);\n}\n\n/**\n * Sets the value of a prop.\n *\n * @private\n * @param el - The element to set the prop on.\n * @param name - The prop's name.\n * @param value - The prop's value.\n * @param xmlMode - Disable handling of special HTML attributes.\n */\nfunction setProp(el: Element, name: string, value: unknown, xmlMode?: boolean) {\n  if (name in el) {\n    // @ts-expect-error Overriding value\n    el[name] = value;\n  } else {\n    setAttr(\n      el,\n      name,\n      !xmlMode && rboolean.test(name) ? (value ? '' : null) : `${value}`,\n    );\n  }\n}\n\ninterface StyleProp {\n  length: number;\n  [key: string]: string | number;\n  [index: number]: string;\n}\n\n/**\n * Method for getting and setting properties. Gets the property value for only\n * the first element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"checkbox\"]').prop('checked');\n * //=> false\n *\n * $('input[type=\"checkbox\"]').prop('checked', true).val();\n * //=> ok\n * ```\n *\n * @param name - Name of the property.\n * @returns If `value` is specified the instance itself, otherwise the prop's\n *   value.\n * @see {@link https://api.jquery.com/prop/}\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'tagName' | 'nodeName',\n): string | undefined;\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'innerHTML' | 'outerHTML' | 'innerText' | 'textContent',\n): string | null;\n/**\n * Get a parsed CSS style object.\n *\n * @param name - Name of the property.\n * @returns The style object, or `undefined` if the element has no `style`\n *   attribute.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'style',\n): StyleProp | undefined;\n/**\n * Resolve `href` or `src` of supported elements. Requires the `baseURI` option\n * to be set, and a global `URL` object to be part of the environment.\n *\n * @example With `baseURI` set to `'https://example.com'`:\n *\n * ```js\n * $('<img src=\"image.png\">').prop('src');\n * //=> 'https://example.com/image.png'\n * ```\n *\n * @param name - Name of the property.\n * @returns The resolved URL, or `undefined` if the element is not supported.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: 'href' | 'src',\n): string | undefined;\n/**\n * Get a property of an element.\n *\n * @param name - Name of the property.\n * @returns The property's value.\n */\nexport function prop<T extends AnyNode, K extends keyof Element>(\n  this: Cheerio<T>,\n  name: K,\n): Element[K];\n/**\n * Set a property of an element.\n *\n * @param name - Name of the property.\n * @param value - Value to set the property to.\n * @returns The instance itself.\n */\nexport function prop<T extends AnyNode, K extends keyof Element>(\n  this: Cheerio<T>,\n  name: K,\n  value:\n    | Element[K]\n    | ((this: Element, i: number, prop: K) => Element[keyof Element]),\n): Cheerio<T>;\n/**\n * Set multiple properties of an element.\n *\n * @example\n *\n * ```js\n * $('input[type=\"checkbox\"]').prop({\n *   checked: true,\n *   disabled: false,\n * });\n * ```\n *\n * @param map - Object of properties to set.\n * @returns The instance itself.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  map: Record<string, string | Element[keyof Element] | boolean>,\n): Cheerio<T>;\n/**\n * Set a property of an element.\n *\n * @param name - Name of the property.\n * @param value - Value to set the property to.\n * @returns The instance itself.\n */\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value:\n    | string\n    | boolean\n    | null\n    | ((this: Element, i: number, prop: string) => string | boolean),\n): Cheerio<T>;\n/**\n * Get a property of an element.\n *\n * @param name - The property's name.\n * @returns The property's value.\n */\nexport function prop<T extends AnyNode>(this: Cheerio<T>, name: string): string;\nexport function prop<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string | Record<string, string | Element[keyof Element] | boolean>,\n  value?:\n    | ((\n        this: Element,\n        i: number,\n        prop: string | undefined,\n      ) => string | Element[keyof Element] | boolean)\n    | unknown,\n): Cheerio<T> | string | undefined | null | Element[keyof Element] | StyleProp {\n  if (typeof name === 'string' && value === undefined) {\n    const el = this[0];\n\n    if (!el || !isTag(el)) return undefined;\n\n    switch (name) {\n      case 'style': {\n        const property = this.css() as StyleProp;\n        const keys = Object.keys(property);\n        for (let i = 0; i < keys.length; i++) {\n          property[i] = keys[i];\n        }\n\n        property.length = keys.length;\n\n        return property;\n      }\n      case 'tagName':\n      case 'nodeName': {\n        return el.name.toUpperCase();\n      }\n\n      case 'href':\n      case 'src': {\n        const prop = el.attribs?.[name];\n\n        /* eslint-disable n/no-unsupported-features/node-builtins */\n        if (\n          typeof URL !== 'undefined' &&\n          ((name === 'href' && (el.tagName === 'a' || el.name === 'link')) ||\n            (name === 'src' &&\n              (el.tagName === 'img' ||\n                el.tagName === 'iframe' ||\n                el.tagName === 'audio' ||\n                el.tagName === 'video' ||\n                el.tagName === 'source'))) &&\n          prop !== undefined &&\n          this.options.baseURI\n        ) {\n          return new URL(prop, this.options.baseURI).href;\n        }\n        /* eslint-enable n/no-unsupported-features/node-builtins */\n\n        return prop;\n      }\n\n      case 'innerText': {\n        return innerText(el);\n      }\n\n      case 'textContent': {\n        return textContent(el);\n      }\n\n      case 'outerHTML': {\n        return this.clone().wrap('<container />').parent().html();\n      }\n\n      case 'innerHTML': {\n        return this.html();\n      }\n\n      default: {\n        return getProp(el, name, this.options.xmlMode);\n      }\n    }\n  }\n\n  if (typeof name === 'object' || value !== undefined) {\n    if (typeof value === 'function') {\n      if (typeof name === 'object') {\n        throw new TypeError('Bad combination of arguments.');\n      }\n      return domEach(this, (el, i) => {\n        if (isTag(el)) {\n          setProp(\n            el,\n            name,\n            value.call(el, i, getProp(el, name, this.options.xmlMode)),\n            this.options.xmlMode,\n          );\n        }\n      });\n    }\n\n    return domEach(this, (el) => {\n      if (!isTag(el)) return;\n\n      if (typeof name === 'object') {\n        for (const key of Object.keys(name)) {\n          const val = name[key];\n          setProp(el, key, val, this.options.xmlMode);\n        }\n      } else {\n        setProp(el, name, value, this.options.xmlMode);\n      }\n    });\n  }\n\n  return undefined;\n}\n\n/**\n * An element with a data attribute.\n *\n * @private\n */\ninterface DataElement extends Element {\n  /** The data attribute. */\n  data?: Record<string, unknown>;\n}\n\n/**\n * Sets the value of a data attribute.\n *\n * @private\n * @param elem - The element to set the data attribute on.\n * @param name - The data attribute's name.\n * @param value - The data attribute's value.\n */\nfunction setData(\n  elem: DataElement,\n  name: string | Record<string, unknown>,\n  value?: unknown,\n) {\n  elem.data ??= {};\n\n  if (typeof name === 'object') Object.assign(elem.data, name);\n  else if (typeof name === 'string' && value !== undefined) {\n    elem.data[name] = value;\n  }\n}\n\n/**\n * Read _all_ HTML5 `data-*` attributes from the equivalent HTML5 `data-*`\n * attribute, and cache the value in the node's internal data store.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the data attribute of.\n * @returns A map with all of the data attributes.\n */\nfunction readAllData(el: DataElement): unknown {\n  for (const domName of Object.keys(el.attribs)) {\n    if (!domName.startsWith(dataAttrPrefix)) {\n      continue;\n    }\n\n    const jsName = camelCase(domName.slice(dataAttrPrefix.length));\n\n    if (!hasOwn.call(el.data, jsName)) {\n      el.data![jsName] = parseDataValue(el.attribs[domName]);\n    }\n  }\n\n  return el.data;\n}\n\n/**\n * Read the specified attribute from the equivalent HTML5 `data-*` attribute,\n * and (if present) cache the value in the node's internal data store.\n *\n * @private\n * @category Attributes\n * @param el - Element to get the data attribute of.\n * @param name - Name of the data attribute.\n * @returns The data attribute's value.\n */\nfunction readData(el: DataElement, name: string): unknown {\n  const domName = dataAttrPrefix + cssCase(name);\n  const data = el.data!;\n\n  if (hasOwn.call(data, name)) {\n    return data[name];\n  }\n\n  if (hasOwn.call(el.attribs, domName)) {\n    return (data[name] = parseDataValue(el.attribs[domName]));\n  }\n\n  return undefined;\n}\n\n/**\n * Coerce string data-* attributes to their corresponding JavaScript primitives.\n *\n * @private\n * @category Attributes\n * @param value - The value to parse.\n * @returns The parsed value.\n */\nfunction parseDataValue(value: string): unknown {\n  if (value === 'null') return null;\n  if (value === 'true') return true;\n  if (value === 'false') return false;\n  const num = Number(value);\n  if (value === String(num)) return num;\n  if (rbrace.test(value)) {\n    try {\n      return JSON.parse(value);\n    } catch {\n      /* Ignore */\n    }\n  }\n  return value;\n}\n\n/**\n * Method for getting data attributes, for only the first element in the matched\n * set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('<div data-apple-color=\"red\"></div>').data('apple-color');\n * //=> 'red'\n * ```\n *\n * @param name - Name of the data attribute.\n * @returns The data attribute's value, or `undefined` if the attribute does not\n *   exist.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): unknown | undefined;\n/**\n * Method for getting all of an element's data attributes, for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('<div data-apple-color=\"red\"></div>').data();\n * //=> { appleColor: 'red' }\n * ```\n *\n * @returns A map with all of the data attributes.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n): Record<string, unknown>;\n/**\n * Method for setting data attributes, for only the first element in the matched\n * set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * const apple = $('.apple').data('kind', 'mac');\n *\n * apple.data('kind');\n * //=> 'mac'\n * ```\n *\n * @param name - Name of the data attribute.\n * @param value - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n  value: unknown,\n): Cheerio<T>;\n/**\n * Method for setting multiple data attributes at once, for only the first\n * element in the matched set.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * const apple = $('.apple').data({ kind: 'mac' });\n *\n * apple.data('kind');\n * //=> 'mac'\n * ```\n *\n * @param values - Map of names to values.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/data/}\n */\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  values: Record<string, unknown>,\n): Cheerio<T>;\nexport function data<T extends AnyNode>(\n  this: Cheerio<T>,\n  name?: string | Record<string, unknown>,\n  value?: unknown,\n): unknown | Cheerio<T> | undefined | Record<string, unknown> {\n  const elem = this[0];\n\n  if (!elem || !isTag(elem)) return;\n\n  const dataEl: DataElement = elem;\n  dataEl.data ??= {};\n\n  // Return the entire data object if no data specified\n  if (name == null) {\n    return readAllData(dataEl);\n  }\n\n  // Set the value (with attr map support)\n  if (typeof name === 'object' || value !== undefined) {\n    domEach(this, (el) => {\n      if (isTag(el)) {\n        if (typeof name === 'object') setData(el, name);\n        else setData(el, name, value as unknown);\n      }\n    });\n    return this;\n  }\n\n  return readData(dataEl, name);\n}\n\n/**\n * Method for getting the value of input, select, and textarea. Note: Support\n * for `map`, and `function` has not been added yet.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"text\"]').val();\n * //=> input_text\n * ```\n *\n * @returns The value.\n * @see {@link https://api.jquery.com/val/}\n */\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n): string | undefined | string[];\n/**\n * Method for setting the value of input, select, and textarea. Note: Support\n * for `map`, and `function` has not been added yet.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('input[type=\"text\"]').val('test').html();\n * //=> <input type=\"text\" value=\"test\"/>\n * ```\n *\n * @param value - The new value.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/val/}\n */\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n  value: string | string[],\n): Cheerio<T>;\nexport function val<T extends AnyNode>(\n  this: Cheerio<T>,\n  value?: string | string[],\n): string | string[] | Cheerio<T> | undefined {\n  const querying = arguments.length === 0;\n  const element = this[0];\n\n  if (!element || !isTag(element)) return querying ? undefined : this;\n\n  switch (element.name) {\n    case 'textarea': {\n      return this.text(value as string);\n    }\n    case 'select': {\n      const option = this.find('option:selected');\n      if (!querying) {\n        if (this.attr('multiple') == null && typeof value === 'object') {\n          return this;\n        }\n\n        this.find('option').removeAttr('selected');\n\n        const values = typeof value === 'object' ? value : [value];\n        for (const val of values) {\n          this.find(`option[value=\"${val}\"]`).attr('selected', '');\n        }\n\n        return this;\n      }\n\n      return this.attr('multiple')\n        ? option.toArray().map((el) => text(el.children))\n        : option.attr('value');\n    }\n    case 'input':\n    case 'option': {\n      return querying\n        ? this.attr('value')\n        : this.attr('value', value as string);\n    }\n  }\n\n  return undefined;\n}\n\n/**\n * Remove an attribute.\n *\n * @private\n * @param elem - Node to remove attribute from.\n * @param name - Name of the attribute to remove.\n */\nfunction removeAttribute(elem: Element, name: string) {\n  if (!elem.attribs || !hasOwn.call(elem.attribs, name)) return;\n\n  delete elem.attribs[name];\n}\n\n/**\n * Splits a space-separated list of names to individual names.\n *\n * @category Attributes\n * @param names - Names to split.\n * @returns - Split names.\n */\nfunction splitNames(names?: string): string[] {\n  return names ? names.trim().split(rspace) : [];\n}\n\n/**\n * Method for removing attributes by `name`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').removeAttr('class').html();\n * //=> <li>Pear</li>\n *\n * $('.apple').attr('id', 'favorite');\n * $('.apple').removeAttr('id class').html();\n * //=> <li>Apple</li>\n * ```\n *\n * @param name - Name of the attribute.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/removeAttr/}\n */\nexport function removeAttr<T extends AnyNode>(\n  this: Cheerio<T>,\n  name: string,\n): Cheerio<T> {\n  const attrNames = splitNames(name);\n\n  for (const attrName of attrNames) {\n    domEach(this, (elem) => {\n      if (isTag(elem)) removeAttribute(elem, attrName);\n    });\n  }\n\n  return this;\n}\n\n/**\n * Check to see if _any_ of the matched elements have the given `className`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').hasClass('pear');\n * //=> true\n *\n * $('apple').hasClass('fruit');\n * //=> false\n *\n * $('li').hasClass('pear');\n * //=> true\n * ```\n *\n * @param className - Name of the class.\n * @returns Indicates if an element has the given `className`.\n * @see {@link https://api.jquery.com/hasClass/}\n */\nexport function hasClass<T extends AnyNode>(\n  this: Cheerio<T>,\n  className: string,\n): boolean {\n  return this.toArray().some((elem) => {\n    const clazz = isTag(elem) && elem.attribs['class'];\n    let idx = -1;\n\n    if (clazz && className.length > 0) {\n      while ((idx = clazz.indexOf(className, idx + 1)) > -1) {\n        const end = idx + className.length;\n\n        if (\n          (idx === 0 || rspace.test(clazz[idx - 1])) &&\n          (end === clazz.length || rspace.test(clazz[end]))\n        ) {\n          return true;\n        }\n      }\n    }\n\n    return false;\n  });\n}\n\n/**\n * Adds class(es) to all of the matched elements. Also accepts a `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').addClass('fruit').html();\n * //=> <li class=\"pear fruit\">Pear</li>\n *\n * $('.apple').addClass('fruit red').html();\n * //=> <li class=\"apple fruit red\">Apple</li>\n * ```\n *\n * @param value - Name of new class.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/addClass/}\n */\nexport function addClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  value?:\n    | string\n    | ((this: Element, i: number, className: string) => string | undefined),\n): R {\n  // Support functions\n  if (typeof value === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        const className = el.attribs['class'] || '';\n        addClass.call([el], value.call(el, i, className));\n      }\n    });\n  }\n\n  // Return if no value or not a string or function\n  if (!value || typeof value !== 'string') return this;\n\n  const classNames = value.split(rspace);\n  const numElements = this.length;\n\n  for (let i = 0; i < numElements; i++) {\n    const el = this[i];\n    // If selected element isn't a tag, move on\n    if (!isTag(el)) continue;\n\n    // If we don't already have classes \u2014 always set xmlMode to false here, as it doesn't matter for classes\n    const className = getAttr(el, 'class', false);\n\n    if (className) {\n      let setClass = ` ${className} `;\n\n      // Check if class already exists\n      for (const cn of classNames) {\n        const appendClass = `${cn} `;\n        if (!setClass.includes(` ${appendClass}`)) setClass += appendClass;\n      }\n\n      setAttr(el, 'class', setClass.trim());\n    } else {\n      setAttr(el, 'class', classNames.join(' ').trim());\n    }\n  }\n\n  return this;\n}\n\n/**\n * Removes one or more space-separated classes from the selected elements. If no\n * `className` is defined, all classes will be removed. Also accepts a\n * `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.pear').removeClass('pear').html();\n * //=> <li class=\"\">Pear</li>\n *\n * $('.apple').addClass('red').removeClass().html();\n * //=> <li class=\"\">Apple</li>\n * ```\n *\n * @param name - Name of the class. If not specified, removes all elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/removeClass/}\n */\nexport function removeClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  name?:\n    | string\n    | ((this: Element, i: number, className: string) => string | undefined),\n): R {\n  // Handle if value is a function\n  if (typeof name === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        removeClass.call([el], name.call(el, i, el.attribs['class'] || ''));\n      }\n    });\n  }\n\n  const classes = splitNames(name);\n  const numClasses = classes.length;\n  const removeAll = arguments.length === 0;\n\n  return domEach(this, (el) => {\n    if (!isTag(el)) return;\n\n    if (removeAll) {\n      // Short circuit the remove all case as this is the nice one\n      el.attribs['class'] = '';\n    } else {\n      const elClasses = splitNames(el.attribs['class']);\n      let changed = false;\n\n      for (let j = 0; j < numClasses; j++) {\n        const index = elClasses.indexOf(classes[j]);\n\n        if (index >= 0) {\n          elClasses.splice(index, 1);\n          changed = true;\n\n          /*\n           * We have to do another pass to ensure that there are not duplicate\n           * classes listed\n           */\n          j--;\n        }\n      }\n      if (changed) {\n        el.attribs['class'] = elClasses.join(' ');\n      }\n    }\n  });\n}\n\n/**\n * Add or remove class(es) from the matched elements, depending on either the\n * class's presence or the value of the switch argument. Also accepts a\n * `function`.\n *\n * @category Attributes\n * @example\n *\n * ```js\n * $('.apple.green').toggleClass('fruit green red').html();\n * //=> <li class=\"apple fruit red\">Apple</li>\n *\n * $('.apple.green').toggleClass('fruit green red', true).html();\n * //=> <li class=\"apple green fruit red\">Apple</li>\n * ```\n *\n * @param value - Name of the class. Can also be a function.\n * @param stateVal - If specified the state of the class.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/toggleClass/}\n */\nexport function toggleClass<T extends AnyNode, R extends ArrayLike<T>>(\n  this: R,\n  value?:\n    | string\n    | ((\n        this: Element,\n        i: number,\n        className: string,\n        stateVal?: boolean,\n      ) => string),\n  stateVal?: boolean,\n): R {\n  // Support functions\n  if (typeof value === 'function') {\n    return domEach(this, (el, i) => {\n      if (isTag(el)) {\n        toggleClass.call(\n          [el],\n          value.call(el, i, el.attribs['class'] || '', stateVal),\n          stateVal,\n        );\n      }\n    });\n  }\n\n  // Return if no value or not a string or function\n  if (!value || typeof value !== 'string') return this;\n\n  const classNames = value.split(rspace);\n  const numClasses = classNames.length;\n  const state = typeof stateVal === 'boolean' ? (stateVal ? 1 : -1) : 0;\n  const numElements = this.length;\n\n  for (let i = 0; i < numElements; i++) {\n    const el = this[i];\n    // If selected element isn't a tag, move on\n    if (!isTag(el)) continue;\n\n    const elementClasses = splitNames(el.attribs['class']);\n\n    // Check if class already exists\n    for (let j = 0; j < numClasses; j++) {\n      // Check if the class name is currently defined\n      const index = elementClasses.indexOf(classNames[j]);\n\n      // Add if stateValue === true or we are toggling and there is no value\n      if (state >= 0 && index < 0) {\n        elementClasses.push(classNames[j]);\n      } else if (state <= 0 && index >= 0) {\n        // Otherwise remove but only if the item exists\n        elementClasses.splice(index, 1);\n      }\n    }\n\n    el.attribs['class'] = elementClasses.join(' ');\n  }\n\n  return this;\n}\n\n// Path: src/api/manipulation.ts\n/**\n * Methods for modifying the DOM structure.\n *\n * @module cheerio/manipulation\n */\n\nimport {\n  isTag,\n  Text,\n  hasChildren,\n  cloneNode,\n  Document,\n  type ParentNode,\n  type AnyNode,\n  type Element,\n} from 'domhandler';\nimport { update as updateDOM } from '../parse.js';\nimport { text as staticText } from '../static.js';\nimport { domEach, isHtml, isCheerio } from '../utils.js';\nimport { removeElement } from 'domutils';\nimport type { Cheerio } from '../cheerio.js';\nimport type { BasicAcceptedElems, AcceptedElems } from '../types.js';\n\n/**\n * Create an array of nodes, recursing into arrays and parsing strings if\n * necessary.\n *\n * @private\n * @category Manipulation\n * @param elem - Elements to make an array of.\n * @param clone - Optionally clone nodes.\n * @returns The array of nodes.\n */\nexport function _makeDomArray<T extends AnyNode>(\n  this: Cheerio<T>,\n  elem?: BasicAcceptedElems<AnyNode> | BasicAcceptedElems<AnyNode>[],\n  clone?: boolean,\n): AnyNode[] {\n  if (elem == null) {\n    return [];\n  }\n\n  if (typeof elem === 'string') {\n    return this._parse(elem, this.options, false, null).children.slice(0);\n  }\n\n  if ('length' in elem) {\n    if (elem.length === 1) {\n      return this._makeDomArray(elem[0], clone);\n    }\n\n    const result: AnyNode[] = [];\n\n    for (let i = 0; i < elem.length; i++) {\n      const el = elem[i];\n\n      if (typeof el === 'object') {\n        if (el == null) {\n          continue;\n        }\n\n        if (!('length' in el)) {\n          result.push(clone ? cloneNode(el, true) : el);\n          continue;\n        }\n      }\n\n      result.push(...this._makeDomArray(el, clone));\n    }\n\n    return result;\n  }\n\n  return [clone ? cloneNode(elem, true) : elem];\n}\n\nfunction _insert(\n  concatenator: (\n    dom: AnyNode[],\n    children: AnyNode[],\n    parent: ParentNode,\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    ...elems:\n      | [\n          (\n            this: AnyNode,\n            i: number,\n            html: string,\n          ) => BasicAcceptedElems<AnyNode>,\n        ]\n      | BasicAcceptedElems<AnyNode>[]\n  ) {\n    const lastIdx = this.length - 1;\n\n    return domEach(this, (el, i) => {\n      if (!hasChildren(el)) return;\n\n      const domSrc =\n        typeof elems[0] === 'function'\n          ? elems[0].call(el, i, this._render(el.children))\n          : (elems as BasicAcceptedElems<AnyNode>[]);\n\n      const dom = this._makeDomArray(domSrc, i < lastIdx);\n      concatenator(dom, el.children, el);\n    });\n  };\n}\n\n/**\n * Modify an array in-place, removing some number of elements and adding new\n * elements directly following them.\n *\n * @private\n * @category Manipulation\n * @param array - Target array to splice.\n * @param spliceIdx - Index at which to begin changing the array.\n * @param spliceCount - Number of elements to remove from the array.\n * @param newElems - Elements to insert into the array.\n * @param parent - The parent of the node.\n * @returns The spliced array.\n */\nfunction uniqueSplice(\n  array: AnyNode[],\n  spliceIdx: number,\n  spliceCount: number,\n  newElems: AnyNode[],\n  parent: ParentNode,\n): AnyNode[] {\n  const spliceArgs: Parameters<typeof Array.prototype.splice> = [\n    spliceIdx,\n    spliceCount,\n    ...newElems,\n  ];\n  const prev = spliceIdx === 0 ? null : array[spliceIdx - 1];\n  const next =\n    spliceIdx + spliceCount >= array.length\n      ? null\n      : array[spliceIdx + spliceCount];\n\n  /*\n   * Before splicing in new elements, ensure they do not already appear in the\n   * current array.\n   */\n  for (let idx = 0; idx < newElems.length; ++idx) {\n    const node = newElems[idx];\n    const oldParent = node.parent;\n\n    if (oldParent) {\n      const oldSiblings: AnyNode[] = oldParent.children;\n      const prevIdx = oldSiblings.indexOf(node);\n\n      if (prevIdx > -1) {\n        oldParent.children.splice(prevIdx, 1);\n        if (parent === oldParent && spliceIdx > prevIdx) {\n          spliceArgs[0]--;\n        }\n      }\n    }\n\n    node.parent = parent;\n\n    if (node.prev) {\n      node.prev.next = node.next ?? null;\n    }\n\n    if (node.next) {\n      node.next.prev = node.prev ?? null;\n    }\n\n    node.prev = idx === 0 ? prev : newElems[idx - 1];\n    node.next = idx === newElems.length - 1 ? next : newElems[idx + 1];\n  }\n\n  if (prev) {\n    prev.next = newElems[0];\n  }\n  if (next) {\n    next.prev = newElems[newElems.length - 1];\n  }\n  return array.splice(...spliceArgs);\n}\n\n/**\n * Insert every element in the set of matched elements to the end of the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').appendTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to append elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/appendTo/}\n */\nexport \nfunction appendTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const appendTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  appendTarget.append(this);\n\n  return this;\n}\n\n/**\n * Insert every element in the set of matched elements to the beginning of the\n * target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').prependTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to prepend elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/prependTo/}\n */\nexport function prependTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const prependTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  prependTarget.prepend(this);\n\n  return this;\n}\n\n/**\n * Inserts content as the _last_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').append('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/append/}\n */\nexport const append = _insert((dom, children, parent) => {\n  uniqueSplice(children, children.length, 0, dom, parent);\n});\n\n/**\n * Inserts content as the _first_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').prepend('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/prepend/}\n */\nexport const prepend = _insert((dom, children, parent) => {\n  uniqueSplice(children, 0, 0, dom, parent);\n});\n\nfunction _wrap(\n  insert: (\n    el: AnyNode,\n    elInsertLocation: ParentNode,\n    wrapperDom: ParentNode[],\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    wrapper: AcceptedElems<AnyNode>,\n  ) {\n    const lastIdx = this.length - 1;\n    const lastParent = this.parents().last();\n\n    for (let i = 0; i < this.length; i++) {\n      const el = this[i];\n\n      const wrap =\n        typeof wrapper === 'function'\n          ? wrapper.call(el, i, el)\n          : typeof wrapper === 'string' && !isHtml(wrapper)\n            ? lastParent.find(wrapper).clone()\n            : wrapper;\n\n      const [wrapperDom] = this._makeDomArray(wrap, i < lastIdx);\n\n      if (!wrapperDom || !hasChildren(wrapperDom)) continue;\n\n      let elInsertLocation = wrapperDom;\n\n      /*\n       * Find the deepest child. Only consider the first tag child of each node\n       * (ignore text); stop if no children are found.\n       */\n      let j = 0;\n\n      while (j < elInsertLocation.children.length) {\n        const child = elInsertLocation.children[j];\n        if (isTag(child)) {\n          elInsertLocation = child;\n          j = 0;\n        } else {\n          j++;\n        }\n      }\n\n      insert(el, elInsertLocation, [wrapperDom]);\n    }\n\n    return this;\n  };\n}\n\n/**\n * The .wrap() function can take any string or object that could be passed to\n * the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. A\n * copy of this structure will be wrapped around each of the elements in the set\n * of matched elements. This method returns the original set of elements for\n * chaining purposes.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrap(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"red-fruit\">\n * //      <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrap(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"healthy\">\n * //       <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //       <li class=\"orange\">Orange</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //        <li class=\"plum\">Plum</li>\n * //     </div>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around each element in the\n *   selection.\n * @see {@link https://api.jquery.com/wrap/}\n */\nexport const wrap = _wrap((el, elInsertLocation, wrapperDom) => {\n  const { parent } = el;\n\n  if (!parent) return;\n\n  const siblings: AnyNode[] = parent.children;\n  const index = siblings.indexOf(el);\n\n  updateDOM([el], elInsertLocation);\n  /*\n   * The previous operation removed the current element from the `siblings`\n   * array, so the `dom` array can be inserted without removing any\n   * additional elements.\n   */\n  uniqueSplice(siblings, index, 0, wrapperDom, parent);\n});\n\n/**\n * The .wrapInner() function can take any string or object that could be passed\n * to the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around the content of each of the elements in the\n * set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrapInner(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"red-fruit\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"pear\">Pear</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrapInner(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"healthy\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">\n * //       <div class=\"healthy\">Orange</div>\n * //     </li>\n * //     <li class=\"pear\">\n * //       <div class=\"healthy\">Pear</div>\n * //     </li>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around the content of each element\n *   in the selection.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/wrapInner/}\n */\nexport const wrapInner = _wrap((el, elInsertLocation, wrapperDom) => {\n  if (!hasChildren(el)) return;\n  updateDOM(el.children, elInsertLocation);\n  updateDOM(wrapperDom, el);\n});\n\n/**\n * The .unwrap() function, removes the parents of the set of matched elements\n * from the DOM, leaving the matched elements in their place.\n *\n * @category Manipulation\n * @example <caption>without selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n...\n", "needle_token_start": 13138, "needle_token_end": 13222, "code_context_ntokens": 15464, "output": ["```typescript\n/**\n * Insert every element in the set of matched elements to the end of the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').appendTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to append elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/appendTo/}\n */\nexport function appendTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const appendTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  appendTarget.append(this);\n\n  return this;\n}\n```"]}
{"repo": "cheeriojs/cheerio", "name": "is", "language": "typescript", "path": "src/api/traversing.ts", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function checks if any elements in a collection match a given selector or pass a specified test.\n2. **Input**: A selector or a function used to test each element.\n3. **Output**: A boolean value indicating whether at least one element in the collection matches the selector or passes the test.\n4. **Procedure**: The function first converts the collection to an array. If the input is a string, it uses a selector engine to check if any elements match the selector. If the input is a function, it iterates over the array and applies the function to each element to determine if any pass the test.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/api/manipulation.ts\n/**\n * Methods for modifying the DOM structure.\n *\n * @module cheerio/manipulation\n */\n\nimport {\n  isTag,\n  Text,\n  hasChildren,\n  cloneNode,\n  Document,\n  type ParentNode,\n  type AnyNode,\n  type Element,\n} from 'domhandler';\nimport { update as updateDOM } from '../parse.js';\nimport { text as staticText } from '../static.js';\nimport { domEach, isHtml, isCheerio } from '../utils.js';\nimport { removeElement } from 'domutils';\nimport type { Cheerio } from '../cheerio.js';\nimport type { BasicAcceptedElems, AcceptedElems } from '../types.js';\n\n/**\n * Create an array of nodes, recursing into arrays and parsing strings if\n * necessary.\n *\n * @private\n * @category Manipulation\n * @param elem - Elements to make an array of.\n * @param clone - Optionally clone nodes.\n * @returns The array of nodes.\n */\nexport function _makeDomArray<T extends AnyNode>(\n  this: Cheerio<T>,\n  elem?: BasicAcceptedElems<AnyNode> | BasicAcceptedElems<AnyNode>[],\n  clone?: boolean,\n): AnyNode[] {\n  if (elem == null) {\n    return [];\n  }\n\n  if (typeof elem === 'string') {\n    return this._parse(elem, this.options, false, null).children.slice(0);\n  }\n\n  if ('length' in elem) {\n    if (elem.length === 1) {\n      return this._makeDomArray(elem[0], clone);\n    }\n\n    const result: AnyNode[] = [];\n\n    for (let i = 0; i < elem.length; i++) {\n      const el = elem[i];\n\n      if (typeof el === 'object') {\n        if (el == null) {\n          continue;\n        }\n\n        if (!('length' in el)) {\n          result.push(clone ? cloneNode(el, true) : el);\n          continue;\n        }\n      }\n\n      result.push(...this._makeDomArray(el, clone));\n    }\n\n    return result;\n  }\n\n  return [clone ? cloneNode(elem, true) : elem];\n}\n\nfunction _insert(\n  concatenator: (\n    dom: AnyNode[],\n    children: AnyNode[],\n    parent: ParentNode,\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    ...elems:\n      | [\n          (\n            this: AnyNode,\n            i: number,\n            html: string,\n          ) => BasicAcceptedElems<AnyNode>,\n        ]\n      | BasicAcceptedElems<AnyNode>[]\n  ) {\n    const lastIdx = this.length - 1;\n\n    return domEach(this, (el, i) => {\n      if (!hasChildren(el)) return;\n\n      const domSrc =\n        typeof elems[0] === 'function'\n          ? elems[0].call(el, i, this._render(el.children))\n          : (elems as BasicAcceptedElems<AnyNode>[]);\n\n      const dom = this._makeDomArray(domSrc, i < lastIdx);\n      concatenator(dom, el.children, el);\n    });\n  };\n}\n\n/**\n * Modify an array in-place, removing some number of elements and adding new\n * elements directly following them.\n *\n * @private\n * @category Manipulation\n * @param array - Target array to splice.\n * @param spliceIdx - Index at which to begin changing the array.\n * @param spliceCount - Number of elements to remove from the array.\n * @param newElems - Elements to insert into the array.\n * @param parent - The parent of the node.\n * @returns The spliced array.\n */\nfunction uniqueSplice(\n  array: AnyNode[],\n  spliceIdx: number,\n  spliceCount: number,\n  newElems: AnyNode[],\n  parent: ParentNode,\n): AnyNode[] {\n  const spliceArgs: Parameters<typeof Array.prototype.splice> = [\n    spliceIdx,\n    spliceCount,\n    ...newElems,\n  ];\n  const prev = spliceIdx === 0 ? null : array[spliceIdx - 1];\n  const next =\n    spliceIdx + spliceCount >= array.length\n      ? null\n      : array[spliceIdx + spliceCount];\n\n  /*\n   * Before splicing in new elements, ensure they do not already appear in the\n   * current array.\n   */\n  for (let idx = 0; idx < newElems.length; ++idx) {\n    const node = newElems[idx];\n    const oldParent = node.parent;\n\n    if (oldParent) {\n      const oldSiblings: AnyNode[] = oldParent.children;\n      const prevIdx = oldSiblings.indexOf(node);\n\n      if (prevIdx > -1) {\n        oldParent.children.splice(prevIdx, 1);\n        if (parent === oldParent && spliceIdx > prevIdx) {\n          spliceArgs[0]--;\n        }\n      }\n    }\n\n    node.parent = parent;\n\n    if (node.prev) {\n      node.prev.next = node.next ?? null;\n    }\n\n    if (node.next) {\n      node.next.prev = node.prev ?? null;\n    }\n\n    node.prev = idx === 0 ? prev : newElems[idx - 1];\n    node.next = idx === newElems.length - 1 ? next : newElems[idx + 1];\n  }\n\n  if (prev) {\n    prev.next = newElems[0];\n  }\n  if (next) {\n    next.prev = newElems[newElems.length - 1];\n  }\n  return array.splice(...spliceArgs);\n}\n\n/**\n * Insert every element in the set of matched elements to the end of the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').appendTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to append elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/appendTo/}\n */\nexport function appendTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const appendTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  appendTarget.append(this);\n\n  return this;\n}\n\n/**\n * Insert every element in the set of matched elements to the beginning of the\n * target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').prependTo('#fruits');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to prepend elements to.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/prependTo/}\n */\nexport function prependTo<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const prependTarget = isCheerio<T>(target) ? target : this._make(target);\n\n  prependTarget.prepend(this);\n\n  return this;\n}\n\n/**\n * Inserts content as the _last_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').append('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //      <li class=\"plum\">Plum</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/append/}\n */\nexport const append = _insert((dom, children, parent) => {\n  uniqueSplice(children, children.length, 0, dom, parent);\n});\n\n/**\n * Inserts content as the _first_ child of each of the selected elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').prepend('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @see {@link https://api.jquery.com/prepend/}\n */\nexport const prepend = _insert((dom, children, parent) => {\n  uniqueSplice(children, 0, 0, dom, parent);\n});\n\nfunction _wrap(\n  insert: (\n    el: AnyNode,\n    elInsertLocation: ParentNode,\n    wrapperDom: ParentNode[],\n  ) => void,\n) {\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    wrapper: AcceptedElems<AnyNode>,\n  ) {\n    const lastIdx = this.length - 1;\n    const lastParent = this.parents().last();\n\n    for (let i = 0; i < this.length; i++) {\n      const el = this[i];\n\n      const wrap =\n        typeof wrapper === 'function'\n          ? wrapper.call(el, i, el)\n          : typeof wrapper === 'string' && !isHtml(wrapper)\n            ? lastParent.find(wrapper).clone()\n            : wrapper;\n\n      const [wrapperDom] = this._makeDomArray(wrap, i < lastIdx);\n\n      if (!wrapperDom || !hasChildren(wrapperDom)) continue;\n\n      let elInsertLocation = wrapperDom;\n\n      /*\n       * Find the deepest child. Only consider the first tag child of each node\n       * (ignore text); stop if no children are found.\n       */\n      let j = 0;\n\n      while (j < elInsertLocation.children.length) {\n        const child = elInsertLocation.children[j];\n        if (isTag(child)) {\n          elInsertLocation = child;\n          j = 0;\n        } else {\n          j++;\n        }\n      }\n\n      insert(el, elInsertLocation, [wrapperDom]);\n    }\n\n    return this;\n  };\n}\n\n/**\n * The .wrap() function can take any string or object that could be passed to\n * the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. A\n * copy of this structure will be wrapped around each of the elements in the set\n * of matched elements. This method returns the original set of elements for\n * chaining purposes.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrap(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"red-fruit\">\n * //      <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrap(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <div class=\"healthy\">\n * //       <li class=\"apple\">Apple</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //       <li class=\"orange\">Orange</li>\n * //     </div>\n * //     <div class=\"healthy\">\n * //        <li class=\"plum\">Plum</li>\n * //     </div>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around each element in the\n *   selection.\n * @see {@link https://api.jquery.com/wrap/}\n */\nexport const wrap = _wrap((el, elInsertLocation, wrapperDom) => {\n  const { parent } = el;\n\n  if (!parent) return;\n\n  const siblings: AnyNode[] = parent.children;\n  const index = siblings.indexOf(el);\n\n  updateDOM([el], elInsertLocation);\n  /*\n   * The previous operation removed the current element from the `siblings`\n   * array, so the `dom` array can be inserted without removing any\n   * additional elements.\n   */\n  uniqueSplice(siblings, index, 0, wrapperDom, parent);\n});\n\n/**\n * The .wrapInner() function can take any string or object that could be passed\n * to the $() factory function to specify a DOM structure. This structure may be\n * nested several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around the content of each of the elements in the\n * set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const redFruit = $('<div class=\"red-fruit\"></div>');\n * $('.apple').wrapInner(redFruit);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"red-fruit\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"pear\">Pear</li>\n * //   </ul>\n *\n * const healthy = $('<div class=\"healthy\"></div>');\n * $('li').wrapInner(healthy);\n *\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">\n * //       <div class=\"healthy\">Apple</div>\n * //     </li>\n * //     <li class=\"orange\">\n * //       <div class=\"healthy\">Orange</div>\n * //     </li>\n * //     <li class=\"pear\">\n * //       <div class=\"healthy\">Pear</div>\n * //     </li>\n * //   </ul>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around the content of each element\n *   in the selection.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/wrapInner/}\n */\nexport const wrapInner = _wrap((el, elInsertLocation, wrapperDom) => {\n  if (!hasChildren(el)) return;\n  updateDOM(el.children, elInsertLocation);\n  updateDOM(wrapperDom, el);\n});\n\n/**\n * The .unwrap() function, removes the parents of the set of matched elements\n * from the DOM, leaving the matched elements in their place.\n *\n * @category Manipulation\n * @example <caption>without selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <div><p>Hello</p></div>\\n  <div><p>World</p></div>\\n</div>',\n * );\n * $('#test p').unwrap();\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @example <caption>with selector</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div id=test>\\n  <p>Hello</p>\\n  <b><p>World</p></b>\\n</div>',\n * );\n * $('#test p').unwrap('b');\n *\n * //=> <div id=test>\n * //     <p>Hello</p>\n * //     <p>World</p>\n * //   </div>\n * ```\n *\n * @param selector - A selector to check the parent element against. If an\n *   element's parent does not match the selector, the element won't be\n *   unwrapped.\n * @returns The instance itself, for chaining.\n * @see {@link https://api.jquery.com/unwrap/}\n */\nexport function unwrap<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  this.parent(selector)\n    .not('body')\n    .each((_, el) => {\n      this._make(el).replaceWith(el.children);\n    });\n  return this;\n}\n\n/**\n * The .wrapAll() function can take any string or object that could be passed to\n * the $() function to specify a DOM structure. This structure may be nested\n * several levels deep, but should contain only one inmost element. The\n * structure will be wrapped around all of the elements in the set of matched\n * elements, as a single group.\n *\n * @category Manipulation\n * @example <caption>With markup passed to `wrapAll`</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<div class=\"container\"><div class=\"inner\">First</div><div class=\"inner\">Second</div></div>',\n * );\n * $('.inner').wrapAll(\"<div class='new'></div>\");\n *\n * //=> <div class=\"container\">\n * //     <div class='new'>\n * //       <div class=\"inner\">First</div>\n * //       <div class=\"inner\">Second</div>\n * //     </div>\n * //   </div>\n * ```\n *\n * @example <caption>With an existing cheerio instance</caption>\n *\n * ```js\n * const $ = cheerio.load(\n *   '<span>Span 1</span><strong>Strong</strong><span>Span 2</span>',\n * );\n * const wrap = $('<div><p><em><b></b></em></p></div>');\n * $('span').wrapAll(wrap);\n *\n * //=> <div>\n * //     <p>\n * //       <em>\n * //         <b>\n * //           <span>Span 1</span>\n * //           <span>Span 2</span>\n * //         </b>\n * //       </em>\n * //     </p>\n * //   </div>\n * //   <strong>Strong</strong>\n * ```\n *\n * @param wrapper - The DOM structure to wrap around all matched elements in the\n *   selection.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/wrapAll/}\n */\nexport function wrapAll<T extends AnyNode>(\n  this: Cheerio<T>,\n  wrapper: AcceptedElems<T>,\n): Cheerio<T> {\n  const el = this[0];\n  if (el) {\n    const wrap: Cheerio<AnyNode> = this._make(\n      typeof wrapper === 'function' ? wrapper.call(el, 0, el) : wrapper,\n    ).insertBefore(el);\n\n    // If html is given as wrapper, wrap may contain text elements\n    let elInsertLocation: Element | undefined;\n\n    for (let i = 0; i < wrap.length; i++) {\n      if (wrap[i].type === 'tag') elInsertLocation = wrap[i] as Element;\n    }\n\n    let j = 0;\n\n    /*\n     * Find the deepest child. Only consider the first tag child of each node\n     * (ignore text); stop if no children are found.\n     */\n    while (elInsertLocation && j < elInsertLocation.children.length) {\n      const child = elInsertLocation.children[j];\n      if (child.type === 'tag') {\n        elInsertLocation = child as Element;\n        j = 0;\n      } else {\n        j++;\n      }\n    }\n\n    if (elInsertLocation) this._make(elInsertLocation).append(this);\n  }\n  return this;\n}\n\n/**\n * Insert content next to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').after('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert after each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/after/}\n */\nexport function after<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element after `this` element\n    uniqueSplice(siblings, index + 1, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements after the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertAfter('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements after.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertAfter/}\n */\nexport function insertAfter<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  if (typeof target === 'string') {\n    target = this._make<AnyNode>(target);\n  }\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  for (const el of this._makeDomArray(target)) {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      continue;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) continue;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index + 1, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  }\n\n  return this._make(clones);\n}\n\n/**\n * Insert content previous to each element in the set of matched elements.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.apple').before('<li class=\"plum\">Plum</li>');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param elems - HTML string, DOM element, array of DOM elements or Cheerio to\n *   insert before each element in the set of matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/before/}\n */\nexport function before<T extends AnyNode>(\n  this: Cheerio<T>,\n  ...elems:\n    | [(this: AnyNode, i: number, html: string) => BasicAcceptedElems<AnyNode>]\n    | BasicAcceptedElems<AnyNode>[]\n): Cheerio<T> {\n  const lastIdx = this.length - 1;\n\n  return domEach(this, (el, i) => {\n    if (!hasChildren(el) || !el.parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = el.parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    const domSrc =\n      typeof elems[0] === 'function'\n        ? elems[0].call(el, i, this._render(el.children))\n        : (elems as BasicAcceptedElems<AnyNode>[]);\n\n    const dom = this._makeDomArray(domSrc, i < lastIdx);\n\n    // Add element before `el` element\n    uniqueSplice(siblings, index, 0, dom, el.parent);\n  });\n}\n\n/**\n * Insert every element in the set of matched elements before the target.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('<li class=\"plum\">Plum</li>').insertBefore('.apple');\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"plum\">Plum</li>\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //      <li class=\"pear\">Pear</li>\n * //    </ul>\n * ```\n *\n * @param target - Element to insert elements before.\n * @returns The set of newly inserted elements.\n * @see {@link https://api.jquery.com/insertBefore/}\n */\nexport function insertBefore<T extends AnyNode>(\n  this: Cheerio<T>,\n  target: BasicAcceptedElems<AnyNode>,\n): Cheerio<T> {\n  const targetArr = this._make<AnyNode>(target);\n\n  this.remove();\n\n  const clones: T[] = [];\n\n  domEach(targetArr, (el) => {\n    const clonedSelf = this.clone().toArray();\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const index = siblings.indexOf(el);\n\n    // If not found, move on\n    /* istanbul ignore next */\n    if (index < 0) return;\n\n    // Add cloned `this` element(s) after target element\n    uniqueSplice(siblings, index, 0, clonedSelf, parent);\n    clones.push(...clonedSelf);\n  });\n\n  return this._make(clones);\n}\n\n/**\n * Removes the set of matched elements from the DOM and all their children.\n * `selector` filters the set of matched elements to be removed.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('.pear').remove();\n * $.html();\n * //=>  <ul id=\"fruits\">\n * //      <li class=\"apple\">Apple</li>\n * //      <li class=\"orange\">Orange</li>\n * //    </ul>\n * ```\n *\n * @param selector - Optional selector for elements to remove.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/remove/}\n */\nexport function remove<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: string,\n): Cheerio<T> {\n  // Filter if we have selector\n  const elems = selector ? this.filter(selector) : this;\n\n  domEach(elems, (el) => {\n    removeElement(el);\n    el.prev = el.next = el.parent = null;\n  });\n\n  return this;\n}\n\n/**\n * Replaces matched elements with `content`.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * const plum = $('<li class=\"plum\">Plum</li>');\n * $('.pear').replaceWith(plum);\n * $.html();\n * //=> <ul id=\"fruits\">\n * //     <li class=\"apple\">Apple</li>\n * //     <li class=\"orange\">Orange</li>\n * //     <li class=\"plum\">Plum</li>\n * //   </ul>\n * ```\n *\n * @param content - Replacement for matched elements.\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/replaceWith/}\n */\nexport function replaceWith<T extends AnyNode>(\n  this: Cheerio<T>,\n  content: AcceptedElems<AnyNode>,\n): Cheerio<T> {\n  return domEach(this, (el, i) => {\n    const { parent } = el;\n    if (!parent) {\n      return;\n    }\n\n    const siblings: AnyNode[] = parent.children;\n    const cont =\n      typeof content === 'function' ? content.call(el, i, el) : content;\n    const dom = this._makeDomArray(cont);\n\n    /*\n     * In the case that `dom` contains nodes that already exist in other\n     * structures, ensure those nodes are properly removed.\n     */\n    updateDOM(dom, null);\n\n    const index = siblings.indexOf(el);\n\n    // Completely remove old element\n    uniqueSplice(siblings, index, 1, dom, parent);\n\n    if (!dom.includes(el)) {\n      el.parent = el.prev = el.next = null;\n    }\n  });\n}\n\n/**\n * Removes all children from each item in the selection. Text nodes and comment\n * nodes are left as is.\n *\n * @category Manipulation\n * @example\n *\n * ```js\n * $('ul').empty();\n * $.html();\n * //=>  <ul id=\"fruits\"></ul>\n * ```\n *\n * @returns The instance itself.\n * @see {@link https://api.jquery.com/empty/}\n */\nexport function empty<T extends AnyNode>(this: Cheerio<T>): Cheerio<T> {\n  return domEach(this, (el) => {\n    if (!hasChildren(el)) return;\n    for (const child of el.children) {\n      child.next = child.prev = child.parent = null;\n    }\n\n    el.children.length = 0;\n  });\n}\n\n/**\n * Gets an HTML content string from the first selected element.\n *\n...\n// Path: src/api/traversing.ts\n/**\n * Methods for traversing the DOM structure.\n *\n * @module cheerio/traversing\n */\n\nimport {\n  isTag,\n  type AnyNode,\n  type Element,\n  hasChildren,\n  isDocument,\n  type Document,\n} from 'domhandler';\nimport type { Cheerio } from '../cheerio.js';\nimport * as select from 'cheerio-select';\nimport { domEach, isCheerio } from '../utils.js';\nimport { contains } from '../static.js';\nimport {\n  getChildren,\n  getSiblings,\n  nextElementSibling,\n  prevElementSibling,\n  uniqueSort,\n} from 'domutils';\nimport type { FilterFunction, AcceptedFilters } from '../types.js';\nconst reSiblingSelector = /^\\s*[+~]/;\n\n/**\n * Get the descendants of each element in the current set of matched elements,\n * filtered by a selector, jQuery object, or element.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').find('li').length;\n * //=> 3\n * $('#fruits').find($('.apple')).length;\n * //=> 1\n * ```\n *\n * @param selectorOrHaystack - Element to look for.\n * @returns The found elements.\n * @see {@link https://api.jquery.com/find/}\n */\nexport function find<T extends AnyNode>(\n  this: Cheerio<T>,\n  selectorOrHaystack?: string | Cheerio<Element> | Element,\n): Cheerio<Element> {\n  if (!selectorOrHaystack) {\n    return this._make([]);\n  }\n\n  if (typeof selectorOrHaystack !== 'string') {\n    const haystack = isCheerio(selectorOrHaystack)\n      ? selectorOrHaystack.toArray()\n      : [selectorOrHaystack];\n\n    const context = this.toArray();\n\n    return this._make(\n      haystack.filter((elem) => context.some((node) => contains(node, elem))),\n    );\n  }\n\n  return this._findBySelector(selectorOrHaystack, Number.POSITIVE_INFINITY);\n}\n\n/**\n * Find elements by a specific selector.\n *\n * @private\n * @category Traversing\n * @param selector - Selector to filter by.\n * @param limit - Maximum number of elements to match.\n * @returns The found elements.\n */\nexport function _findBySelector<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector: string,\n  limit: number,\n): Cheerio<Element> {\n  const context = this.toArray();\n\n  const elems = reSiblingSelector.test(selector)\n    ? context\n    : this.children().toArray();\n\n  const options = {\n    context,\n    root: this._root?.[0],\n\n    // Pass options that are recognized by `cheerio-select`\n    xmlMode: this.options.xmlMode,\n    lowerCaseTags: this.options.lowerCaseTags,\n    lowerCaseAttributeNames: this.options.lowerCaseAttributeNames,\n    pseudos: this.options.pseudos,\n    quirksMode: this.options.quirksMode,\n  };\n\n  return this._make(select.select(selector, elems, options, limit));\n}\n\n/**\n * Creates a matcher, using a particular mapping function. Matchers provide a\n * function that finds elements using a generating function, supporting\n * filtering.\n *\n * @private\n * @param matchMap - Mapping function.\n * @returns - Function for wrapping generating functions.\n */\nfunction _getMatcher<P>(\n  matchMap: (fn: (elem: AnyNode) => P, elems: Cheerio<AnyNode>) => Element[],\n) {\n  return function (\n    fn: (elem: AnyNode) => P,\n    ...postFns: ((elems: Element[]) => Element[])[]\n  ) {\n    return function <T extends AnyNode>(\n      this: Cheerio<T>,\n      selector?: AcceptedFilters<Element>,\n    ): Cheerio<Element> {\n      let matched: Element[] = matchMap(fn, this);\n\n      if (selector) {\n        matched = filterArray(\n          matched,\n          selector,\n          this.options.xmlMode,\n          this._root?.[0],\n        );\n      }\n\n      return this._make(\n        // Post processing is only necessary if there is more than one element.\n        this.length > 1 && matched.length > 1\n          ? postFns.reduce((elems, fn) => fn(elems), matched)\n          : matched,\n      );\n    };\n  };\n}\n\n/** Matcher that adds multiple elements for each entry in the input. */\nconst _matcher = _getMatcher((fn: (elem: AnyNode) => Element[], elems) => {\n  let ret: Element[] = [];\n\n  for (let i = 0; i < elems.length; i++) {\n    const value = fn(elems[i]);\n    if (value.length > 0) ret = ret.concat(value);\n  }\n\n  return ret;\n});\n\n/** Matcher that adds at most one element for each entry in the input. */\nconst _singleMatcher = _getMatcher(\n  (fn: (elem: AnyNode) => Element | null, elems) => {\n    const ret: Element[] = [];\n\n    for (let i = 0; i < elems.length; i++) {\n      const value = fn(elems[i]);\n      if (value !== null) {\n        ret.push(value);\n      }\n    }\n    return ret;\n  },\n);\n\n/**\n * Matcher that supports traversing until a condition is met.\n *\n * @param nextElem - Function that returns the next element.\n * @param postFns - Post processing functions.\n * @returns A function usable for `*Until` methods.\n */\nfunction _matchUntil(\n  nextElem: (elem: AnyNode) => Element | null,\n  ...postFns: ((elems: Element[]) => Element[])[]\n) {\n  // We use a variable here that is used from within the matcher.\n  let matches: ((el: Element, i: number) => boolean) | null = null;\n\n  const innerMatcher = _getMatcher(\n    (nextElem: (elem: AnyNode) => Element | null, elems) => {\n      const matched: Element[] = [];\n\n      domEach(elems, (elem) => {\n        for (let next; (next = nextElem(elem)); elem = next) {\n          // FIXME: `matched` might contain duplicates here and the index is too large.\n          if (matches?.(next, matched.length)) break;\n          matched.push(next);\n        }\n      });\n\n      return matched;\n    },\n  )(nextElem, ...postFns);\n\n  return function <T extends AnyNode>(\n    this: Cheerio<T>,\n    selector?: AcceptedFilters<Element> | null,\n    filterSelector?: AcceptedFilters<Element>,\n  ): Cheerio<Element> {\n    // Override `matches` variable with the new target.\n    matches =\n      typeof selector === 'string'\n        ? (elem: Element) => select.is(elem, selector, this.options)\n        : selector\n          ? getFilterFn(selector)\n          : null;\n\n    const ret = innerMatcher.call(this, filterSelector);\n\n    // Set `matches` to `null`, so we don't waste memory.\n    matches = null;\n\n    return ret;\n  };\n}\n\nfunction _removeDuplicates<T extends AnyNode>(elems: T[]): T[] {\n  return elems.length > 1 ? Array.from(new Set<T>(elems)) : elems;\n}\n\n/**\n * Get the parent of each element in the current set of matched elements,\n * optionally filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').parent().attr('id');\n * //=> fruits\n * ```\n *\n * @param selector - If specified filter for parent.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parent/}\n */\nexport const parent = _singleMatcher(\n  ({ parent }) => (parent && !isDocument(parent) ? (parent as Element) : null),\n  _removeDuplicates,\n);\n\n/**\n * Get a set of parents filtered by `selector` of each element in the current\n * set of match elements.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').parents().length;\n * //=> 2\n * $('.orange').parents('#fruits').length;\n * //=> 1\n * ```\n *\n * @param selector - If specified filter for parents.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parents/}\n */\nexport const parents = _matcher(\n  (elem) => {\n    const matched = [];\n    while (elem.parent && !isDocument(elem.parent)) {\n      matched.push(elem.parent as Element);\n      elem = elem.parent;\n    }\n    return matched;\n  },\n  uniqueSort,\n  (elems) => elems.reverse(),\n);\n\n/**\n * Get the ancestors of each element in the current set of matched elements, up\n * to but not including the element matched by the selector, DOM node, or\n * cheerio object.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').parentsUntil('#food').length;\n * //=> 1\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - Optional filter for parents.\n * @returns The parents.\n * @see {@link https://api.jquery.com/parentsUntil/}\n */\nexport const parentsUntil = _matchUntil(\n  ({ parent }) => (parent && !isDocument(parent) ? (parent as Element) : null),\n  uniqueSort,\n  (elems) => elems.reverse(),\n);\n\n/**\n * For each element in the set, get the first element that matches the selector\n * by testing the element itself and traversing up through its ancestors in the\n * DOM tree.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').closest();\n * //=> []\n *\n * $('.orange').closest('.apple');\n * // => []\n *\n * $('.orange').closest('li');\n * //=> [<li class=\"orange\">Orange</li>]\n *\n * $('.orange').closest('#fruits');\n * //=> [<ul id=\"fruits\"> ... </ul>]\n * ```\n *\n * @param selector - Selector for the element to find.\n * @returns The closest nodes.\n * @see {@link https://api.jquery.com/closest/}\n */\nexport function closest<T extends AnyNode>(\n  this: Cheerio<T>,\n  selector?: AcceptedFilters<Element>,\n): Cheerio<AnyNode> {\n  const set: AnyNode[] = [];\n\n  if (!selector) {\n    return this._make(set);\n  }\n\n  const selectOpts = {\n    xmlMode: this.options.xmlMode,\n    root: this._root?.[0],\n  };\n\n  const selectFn =\n    typeof selector === 'string'\n      ? (elem: Element) => select.is(elem, selector, selectOpts)\n      : getFilterFn(selector);\n\n  domEach(this, (elem: AnyNode | null) => {\n    if (elem && !isDocument(elem) && !isTag(elem)) {\n      elem = elem.parent;\n    }\n    while (elem && isTag(elem)) {\n      if (selectFn(elem, 0)) {\n        // Do not add duplicate elements to the set\n        if (!set.includes(elem)) {\n          set.push(elem);\n        }\n        break;\n      }\n      elem = elem.parent;\n    }\n  });\n\n  return this._make(set);\n}\n\n/**\n * Gets the next sibling of each selected element, optionally filtered by a\n * selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').next().hasClass('orange');\n * //=> true\n * ```\n *\n * @param selector - If specified filter for sibling.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/next/}\n */\nexport const next = _singleMatcher((elem) => nextElementSibling(elem));\n\n/**\n * Gets all the following siblings of the each selected element, optionally\n * filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').nextAll();\n * //=> [<li class=\"orange\">Orange</li>, <li class=\"pear\">Pear</li>]\n * $('.apple').nextAll('.orange');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/nextAll/}\n */\nexport const nextAll = _matcher((elem) => {\n  const matched = [];\n  while (elem.next) {\n    elem = elem.next;\n    if (isTag(elem)) matched.push(elem);\n  }\n  return matched;\n}, _removeDuplicates);\n\n/**\n * Gets all the following siblings up to but not including the element matched\n * by the selector, optionally filtered by another selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.apple').nextUntil('.pear');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - If specified filter for siblings.\n * @returns The next nodes.\n * @see {@link https://api.jquery.com/nextUntil/}\n */\nexport const nextUntil = _matchUntil(\n  (el) => nextElementSibling(el),\n  _removeDuplicates,\n);\n\n/**\n * Gets the previous sibling of each selected element optionally filtered by a\n * selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.orange').prev().hasClass('apple');\n * //=> true\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prev/}\n */\nexport const prev = _singleMatcher((elem) => prevElementSibling(elem));\n\n/**\n * Gets all the preceding siblings of each selected element, optionally filtered\n * by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').prevAll();\n * //=> [<li class=\"orange\">Orange</li>, <li class=\"apple\">Apple</li>]\n *\n * $('.pear').prevAll('.orange');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prevAll/}\n */\nexport const prevAll = _matcher((elem) => {\n  const matched = [];\n  while (elem.prev) {\n    elem = elem.prev;\n    if (isTag(elem)) matched.push(elem);\n  }\n  return matched;\n}, _removeDuplicates);\n\n/**\n * Gets all the preceding siblings up to but not including the element matched\n * by the selector, optionally filtered by another selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').prevUntil('.apple');\n * //=> [<li class=\"orange\">Orange</li>]\n * ```\n *\n * @param selector - Selector for element to stop at.\n * @param filterSelector - If specified filter for siblings.\n * @returns The previous nodes.\n * @see {@link https://api.jquery.com/prevUntil/}\n */\nexport const prevUntil = _matchUntil(\n  (el) => prevElementSibling(el),\n  _removeDuplicates,\n);\n\n/**\n * Get the siblings of each element (excluding the element) in the set of\n * matched elements, optionally filtered by a selector.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('.pear').siblings().length;\n * //=> 2\n *\n * $('.pear').siblings('.orange').length;\n * //=> 1\n * ```\n *\n * @param selector - If specified filter for siblings.\n * @returns The siblings.\n * @see {@link https://api.jquery.com/siblings/}\n */\nexport const siblings = _matcher(\n  (elem) =>\n    getSiblings(elem).filter((el): el is Element => isTag(el) && el !== elem),\n  uniqueSort,\n);\n\n/**\n * Gets the element children of each element in the set of matched elements.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').children().length;\n * //=> 3\n *\n * $('#fruits').children('.pear').text();\n * //=> Pear\n * ```\n *\n * @param selector - If specified filter for children.\n * @returns The children.\n * @see {@link https://api.jquery.com/children/}\n */\nexport const children = _matcher(\n  (elem) => getChildren(elem).filter(isTag),\n  _removeDuplicates,\n);\n\n/**\n * Gets the children of each element in the set of matched elements, including\n * text and comment nodes.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('#fruits').contents().length;\n * //=> 3\n * ```\n *\n * @returns The children.\n * @see {@link https://api.jquery.com/contents/}\n */\nexport function contents<T extends AnyNode>(\n  this: Cheerio<T>,\n): Cheerio<AnyNode> {\n  const elems = this.toArray().reduce<AnyNode[]>(\n    (newElems, elem) =>\n      hasChildren(elem) ? newElems.concat(elem.children) : newElems,\n    [],\n  );\n  return this._make(elems);\n}\n\n/**\n * Iterates over a cheerio object, executing a function for each matched\n * element. When the callback is fired, the function is fired in the context of\n * the DOM element, so `this` refers to the current element, which is equivalent\n * to the function parameter `element`. To break out of the `each` loop early,\n * return with `false`.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * const fruits = [];\n *\n * $('li').each(function (i, elem) {\n *   fruits[i] = $(this).text();\n * });\n *\n * fruits.join(', ');\n * //=> Apple, Orange, Pear\n * ```\n *\n * @param fn - Function to execute.\n * @returns The instance itself, useful for chaining.\n * @see {@link https://api.jquery.com/each/}\n */\nexport function each<T>(\n  this: Cheerio<T>,\n  fn: (this: T, i: number, el: T) => void | boolean,\n): Cheerio<T> {\n  let i = 0;\n  const len = this.length;\n  while (i < len && fn.call(this[i], i, this[i]) !== false) ++i;\n  return this;\n}\n\n/**\n * Pass each element in the current matched set through a function, producing a\n * new Cheerio object containing the return values. The function can return an\n * individual data item or an array of data items to be inserted into the\n * resulting set. If an array is returned, the elements inside the array are\n * inserted into the set. If the function returns null or undefined, no element\n * will be inserted.\n *\n * @category Traversing\n * @example\n *\n * ```js\n * $('li')\n *   .map(function (i, el) {\n *     // this === el\n *     return $(this).text();\n *   })\n *   .toArray()\n *   .join(' ');\n * //=> \"apple orange pear\"\n * ```\n *\n * @param fn - Function to execute.\n * @returns The mapped elements, wrapped in a Cheerio collection.\n * @see {@link https://api.jquery.com/map/}\n */\nexport function map<T, M>(\n  this: Cheerio<T>,\n  fn: (this: T, i: number, el: T) => M[] | M | null | undefined,\n): Cheerio<M> {\n  let elems: M[] = [];\n  for (let i = 0; i < this.length; i++) {\n    const el = this[i];\n    const val = fn.call(el, i, el);\n    if (val != null) {\n      elems = elems.concat(val);\n    }\n  }\n  return this._make(elems);\n}\n\n/**\n * Creates a function to test if a filter is matched.\n *\n * @param match - A filter.\n * @returns A function that determines if a filter has been matched.\n */\nfunction getFilterFn<T>(\n  match: FilterFunction<T> | Cheerio<T> | T,\n): (el: T, i: number) => boolean {\n  if (typeof match === 'function') {\n    return (el, i) => (match as FilterFunction<T>).call(el, i, el);\n  }\n  if (isCheerio<T>(match)) {\n    return (el) => Array.prototype.includes.call(match, el);\n  }\n  return function (el) {\n    return match === el;\n  };\n}\n\n/**\n * Iterates over a cheerio object, reducing the set of selector elements to\n * those that match the selector or pass the function's test.\n *\n * This is the definition for using type guards; have a look below for other\n * ways to invoke this method. The function is executed in the context of the\n * selected element, so `this` refers to the current element.\n *\n * @category Traversing\n * @example <caption>Function</caption>\n *\n * ```js\n * $('li')\n *   .filter(function (i, el) {\n *     // this === el\n *     return $(this).attr('class') === 'orange';\n *   })\n *   .attr('class'); //=> orange\n * ```\n *\n * @param match - Value to look for, following the rules above.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/filter/}\n */\nexport function filter<T, S extends T>(\n  this: Cheerio<T>,\n  match: (this: T, index: number, value: T) => value is S,\n): Cheerio<S>;\n/**\n * Iterates over a cheerio object, reducing the set of selector elements to\n * those that match the selector or pass the function's test.\n *\n * - When a Cheerio selection is specified, return only the elements contained in\n *   that selection.\n * - When an element is specified, return only that element (if it is contained in\n *   the original selection).\n * - If using the function method, the function is executed in the context of the\n *   selected element, so `this` refers to the current element.\n *\n * @category Traversing\n * @example <caption>Selector</caption>\n *\n * ```js\n * $('li').filter('.orange').attr('class');\n * //=> orange\n * ```\n *\n * @example <caption>Function</caption>\n *\n * ```js\n * $('li')\n *   .filter(function (i, el) {\n *     // this === el\n *     return $(this).attr('class') === 'orange';\n *   })\n *   .attr('class'); //=> orange\n * ```\n *\n * @param match - Value to look for, following the rules above. See\n *   {@link AcceptedFilters}.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/filter/}\n */\nexport function filter<T, S extends AcceptedFilters<T>>(\n  this: Cheerio<T>,\n  match: S,\n): Cheerio<S extends string ? Element : T>;\nexport function filter<T>(\n  this: Cheerio<T>,\n  match: AcceptedFilters<T>,\n): Cheerio<unknown> {\n  return this._make<unknown>(\n    filterArray(this.toArray(), match, this.options.xmlMode, this._root?.[0]),\n  );\n}\n\nexport function filterArray<T>(\n  nodes: T[],\n  match: AcceptedFilters<T>,\n  xmlMode?: boolean,\n  root?: Document,\n): Element[] | T[] {\n  return typeof match === 'string'\n    ? select.filter(match, nodes as unknown as AnyNode[], { xmlMode, root })\n    : nodes.filter(getFilterFn<T>(match));\n}\n\n/**\n * Checks the current list of elements and returns `true` if _any_ of the\n * elements match the selector. If using an element or Cheerio selection,\n * returns `true` if _any_ of the elements match. If using a predicate function,\n * the function is executed in the context of the selected element, so `this`\n * refers to the current element.\n *\n * @category Traversing\n * @param selector - Selector for the selection.\n * @returns Whether or not the selector matches an element of the instance.\n * @see {@link https://api.jquery.com/is/}\n */\nexport \nfunction is<T>(\n  this: Cheerio<T>,\n  selector?: AcceptedFilters<T>,\n): boolean {\n  const nodes = this.toArray();\n  return typeof selector === 'string'\n    ? select.some(\n        (nodes as unknown as AnyNode[]).filter(isTag),\n        selector,\n        this.options,\n      )\n    : selector\n      ? nodes.some(getFilterFn<T>(selector))\n      : false;\n}\n\n/**\n * Remove elements from the set of matched elements. Given a Cheerio object that\n * represents a set of DOM elements, the `.not()` method constructs a new\n * Cheerio object from a subset of the matching elements. The supplied selector\n * is tested against each element; the elements that don't match the selector\n * will be included in the result.\n *\n * The `.not()` method can take a function as its argument in the same way that\n * `.filter()` does. Elements for which the function returns `true` are excluded\n * from the filtered set; all other elements are included.\n *\n * @category Traversing\n * @example <caption>Selector</caption>\n *\n * ```js\n * $('li').not('.apple').length;\n * //=> 2\n * ```\n *\n * @example <caption>Function</caption>\n *\n * ```js\n * $('li').not(function (i, el) {\n *   // this === el\n *   return $(this).attr('class') === 'orange';\n * }).length; //=> 2\n * ```\n *\n * @param match - Value to look for, following the rules above.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/not/}\n */\nexport function not<T extends AnyNode>(\n  this: Cheerio<T>,\n  match: AcceptedFilters<T>,\n): Cheerio<T> {\n  let nodes = this.toArray();\n\n  if (typeof match === 'string') {\n    const matches = new Set<AnyNode>(select.filter(match, nodes, this.options));\n    nodes = nodes.filter((el) => !matches.has(el));\n  } else {\n    const filterFn = getFilterFn(match);\n    nodes = nodes.filter((el, i) => !filterFn(el, i));\n  }\n\n  return this._make(nodes);\n}\n\n/**\n * Filters the set of matched elements to only those which have the given DOM\n * element as a descendant or which have a descendant that matches the given\n * selector. Equivalent to `.filter(':has(selector)')`.\n *\n * @category Traversing\n * @example <caption>Selector</caption>\n *\n * ```js\n * $('ul').has('.pear').attr('id');\n * //=> fruits\n * ```\n *\n * @example <caption>Element</caption>\n *\n * ```js\n * $('ul').has($('.pear')[0]).attr('id');\n * //=> fruits\n * ```\n *\n * @param selectorOrHaystack - Element to look for.\n * @returns The filtered collection.\n * @see {@link https://api.jquery.com/has/}\n */\nexport function has(\n  this: Cheerio<AnyNode | Element>,\n  selectorOrHaystack: string | Cheerio<Element> | Element,\n): Cheerio<AnyNode | Element> {\n  return this.filter(\n...\n", "needle_token_start": 14540, "needle_token_end": 14651, "code_context_ntokens": 15354, "output": ["```typescript\n/**\n * Checks the current list of elements and returns `true` if _any_ of the\n * elements match the selector. If using an element or Cheerio selection,\n * returns `true` if _any_ of the elements match. If using a predicate function,\n * the function is executed in the context of the selected element, so `this`\n * refers to the current element.\n *\n * @category Traversing\n * @param selector - Selector for the selection.\n * @returns Whether or not the selector matches an element of the instance.\n * @see {@link https://api.jquery.com/is/}\n */\nexport function is<T>(\n  this: Cheerio<T>,\n  selector?: AcceptedFilters<T>,\n): boolean {\n  const nodes = this.toArray();\n  return typeof selector === 'string'\n    ? select.some(\n        (nodes as unknown as AnyNode[]).filter(isTag),\n        selector,\n        this.options,\n      )\n    : selector\n      ? nodes.some(getFilterFn<T>(selector))\n      : false;\n}\n```"]}
{"repo": "openai/openai-node", "name": "getBrowserInfo", "language": "typescript", "path": "src/core.ts", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to identify the type and version of a web browser based on the user agent string provided by the browser.\n2. **Input**: There are no direct parameters taken by the function; it internally accesses the global `navigator.userAgent` string.\n3. **Output**: The function returns an object containing the browser type and version if a match is found, otherwise, it returns null.\n4. **Procedure**: The function first checks if the `navigator` object is defined and has a `userAgent` property. It then iterates through a predefined list of browser patterns, attempting to match each pattern with the user agent string. If a match is found, it extracts and constructs the browser version from the matched groups and returns this information. If no matches are found after checking all patterns, it returns null.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': 'Unknown',\n      'X-Stainless-Arch': `other:${EdgeRuntime}`,\n      'X-Stainless-Runtime': 'edge',\n      'X-Stainless-Runtime-Version': process.version,\n    };\n  }\n  // Check if Node.js\n  if (Object.prototype.toString.call(typeof process !== 'undefined' ? process : 0) === '[object process]') {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': normalizePlatform(process.platform),\n      'X-Stainless-Arch': normalizeArch(process.arch),\n      'X-Stainless-Runtime': 'node',\n      'X-Stainless-Runtime-Version': process.version,\n    };\n  }\n\n  const browserInfo = getBrowserInfo();\n  if (browserInfo) {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': 'Unknown',\n      'X-Stainless-Arch': 'unknown',\n      'X-Stainless-Runtime': `browser:${browserInfo.browser}`,\n      'X-Stainless-Runtime-Version': browserInfo.version,\n    };\n  }\n\n  // TODO add support for Cloudflare workers, etc.\n  return {\n    'X-Stainless-Lang': 'js',\n    'X-Stainless-Package-Version': VERSION,\n    'X-Stainless-OS': 'Unknown',\n    'X-Stainless-Arch': 'unknown',\n    'X-Stainless-Runtime': 'unknown',\n    'X-Stainless-Runtime-Version': 'unknown',\n  };\n};\n\ntype BrowserInfo = {\n  browser: Browser;\n  version: string;\n};\n\ndeclare const navigator: { userAgent: string } | undefined;\n\n// Note: modified from https://github.com/JS-DevTools/host-environment/blob/b1ab79ecde37db5d6e163c050e54fe7d287d7c92/src/isomorphic.browser.ts\nfunction g\netBrowserInfo(): BrowserInfo | null {\n  if (typeof navigator === 'undefined' || !navigator) {\n    return null;\n  }\n\n  // NOTE: The order matters here!\n  const browserPatterns = [\n    { key: 'edge' as const, pattern: /Edge(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /MSIE(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /Trident(?:.*rv\\:(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'chrome' as const, pattern: /Chrome(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'firefox' as const, pattern: /Firefox(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'safari' as const, pattern: /(?:Version\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?(?:\\W+Mobile\\S*)?\\W+Safari/ },\n  ];\n\n  // Find the FIRST matching browser\n  for (const { key, pattern } of browserPatterns) {\n    const match = pattern.exec(navigator.userAgent);\n    if (match) {\n      const major = match[1] || 0;\n      const minor = match[2] || 0;\n      const patch = match[3] || 0;\n\n      return { browser: key, version: `${major}.${minor}.${patch}` };\n    }\n  }\n\n  return null;\n}\n\nconst normalizeArch = (arch: string): Arch => {\n  // Node docs:\n  // - https://nodejs.org/api/process.html#processarch\n  // Deno docs:\n  // - https://doc.deno.land/deno/stable/~/Deno.build\n  if (arch === 'x32') return 'x32';\n  if (arch === 'x86_64' || arch === 'x64') return 'x64';\n  if (arch === 'arm') return 'arm';\n  if (arch === 'aarch64' || arch === 'arm64') return 'arm64';\n  if (arch) return `other:${arch}`;\n  return 'unknown';\n};\n\nconst normalizePlatform = (platform: string): PlatformName => {\n  // Node platforms:\n  // - https://nodejs.org/api/process.html#processplatform\n  // Deno platforms:\n  // - https://doc.deno.land/deno/stable/~/Deno.build\n  // - https://github.com/denoland/deno/issues/14799\n\n  platform = platform.toLowerCase();\n\n  // NOTE: this iOS check is untested and may not work\n  // Node does not work natively on IOS, there is a fork at\n  // https://github.com/nodejs-mobile/nodejs-mobile\n  // however it is unknown at the time of writing how to detect if it is running\n  if (platform.includes('ios')) return 'iOS';\n  if (platform === 'android') return 'Android';\n  if (platform === 'darwin') return 'MacOS';\n  if (platform === 'win32') return 'Windows';\n  if (platform === 'freebsd') return 'FreeBSD';\n  if (platform === 'openbsd') return 'OpenBSD';\n  if (platform === 'linux') return 'Linux';\n  if (platform) return `Other:${platform}`;\n  return 'Unknown';\n};\n\nlet _platformHeaders: PlatformProperties;\nconst getPlatformHeaders = () => {\n  return (_platformHeaders ??= getPlatformProperties());\n};\n\nexport const safeJSON = (text: string) => {\n  try {\n    return JSON.parse(text);\n  } catch (err) {\n    return undefined;\n  }\n};\n\n// https://stackoverflow.com/a/19709846\nconst startsWithSchemeRegexp = new RegExp('^(?:[a-z]+:)?//', 'i');\nconst isAbsoluteURL = (url: string): boolean => {\n  return startsWithSchemeRegexp.test(url);\n};\n\nexport const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));\n\nconst validatePositiveInteger = (name: string, n: unknown): number => {\n  if (typeof n !== 'number' || !Number.isInteger(n)) {\n    throw new OpenAIError(`${name} must be an integer`);\n  }\n  if (n < 0) {\n    throw new OpenAIError(`${name} must be a positive integer`);\n  }\n  return n;\n};\n\nexport const castToError = (err: any): Error => {\n  if (err instanceof Error) return err;\n  return new Error(err);\n};\n\nexport const ensurePresent = <T>(value: T | null | undefined): T => {\n  if (value == null) throw new OpenAIError(`Expected a value to be given but received ${value} instead.`);\n  return value;\n};\n\n/**\n * Read an environment variable.\n *\n * Trims beginning and trailing whitespace.\n *\n * Will return undefined if the environment variable doesn't exist or cannot be accessed.\n */\nexport const readEnv = (env: string): string | undefined => {\n  if (typeof process !== 'undefined') {\n    return process.env?.[env]?.trim() ?? undefined;\n  }\n  if (typeof Deno !== 'undefined') {\n    return Deno.env?.get?.(env)?.trim();\n  }\n  return undefined;\n};\n\nexport const coerceInteger = (value: unknown): number => {\n  if (typeof value === 'number') return Math.round(value);\n  if (typeof value === 'string') return parseInt(value, 10);\n\n  throw new OpenAIError(`Could not coerce ${value} (type: ${typeof value}) into a number`);\n};\n\nexport const coerceFloat = (value: unknown): number => {\n  if (typeof value === 'number') return value;\n  if (typeof value === 'string') return parseFloat(value);\n\n  throw new OpenAIError(`Could not coerce ${value} (type: ${typeof value}) into a number`);\n};\n\nexport const coerceBoolean = (value: unknown): boolean => {\n  if (typeof value === 'boolean') return value;\n  if (typeof value === 'string') return value === 'true';\n  return Boolean(value);\n};\n\nexport const maybeCoerceInteger = (value: unknown): number | undefined => {\n  if (value === undefined) {\n    return undefined;\n  }\n  return coerceInteger(value);\n};\n\nexport const maybeCoerceFloat = (value: unknown): number | undefined => {\n  if (value === undefined) {\n    return undefined;\n  }\n  return coerceFloat(value);\n};\n\nexport const maybeCoerceBoolean = (value: unknown): boolean | undefined => {\n  if (value === undefined) {\n    return undefined;\n  }\n  return coerceBoolean(value);\n};\n\n// https://stackoverflow.com/a/34491287\nexport function isEmptyObj(obj: Object | null | undefined): boolean {\n  if (!obj) return true;\n  for (const _k in obj) return false;\n  return true;\n}\n\n// https://eslint.org/docs/latest/rules/no-prototype-builtins\nexport function hasOwn(obj: Object, key: string): boolean {\n  return Object.prototype.hasOwnProperty.call(obj, key);\n}\n\n/**\n * Copies headers from \"newHeaders\" onto \"targetHeaders\",\n * using lower-case for all properties,\n * ignoring any keys with undefined values,\n * and deleting any keys with null values.\n */\nfunction applyHeadersMut(targetHeaders: Headers, newHeaders: Headers): void {\n  for (const k in newHeaders) {\n    if (!hasOwn(newHeaders, k)) continue;\n    const lowerKey = k.toLowerCase();\n    if (!lowerKey) continue;\n\n    const val = newHeaders[k];\n\n    if (val === null) {\n      delete targetHeaders[lowerKey];\n    } else if (val !== undefined) {\n      targetHeaders[lowerKey] = val;\n    }\n  }\n}\n\nexport function debug(action: string, ...args: any[]) {\n  if (typeof process !== 'undefined' && process?.env?.['DEBUG'] === 'true') {\n    console.log(`OpenAI:DEBUG:${action}`, ...args);\n  }\n}\n\n/**\n * https://stackoverflow.com/a/2117523\n */\nconst uuid4 = () => {\n  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {\n    const r = (Math.random() * 16) | 0;\n    const v = c === 'x' ? r : (r & 0x3) | 0x8;\n    return v.toString(16);\n  });\n};\n\nexport const isRunningInBrowser = () => {\n  return (\n    // @ts-ignore\n    typeof window !== 'undefined' &&\n    // @ts-ignore\n    typeof window.document !== 'undefined' &&\n    // @ts-ignore\n    typeof navigator !== 'undefined'\n  );\n};\n\nexport interface HeadersProtocol {\n  get: (header: string) => string | null | undefined;\n}\nexport type HeadersLike = Record<string, string | string[] | undefined> | HeadersProtocol;\n\nexport const isHeadersProtocol = (headers: any): headers is HeadersProtocol => {\n  return typeof headers?.get === 'function';\n};\n\nexport const getRequiredHeader = (headers: HeadersLike, header: string): string => {\n  const lowerCasedHeader = header.toLowerCase();\n  if (isHeadersProtocol(headers)) {\n    // to deal with the case where the header looks like Stainless-Event-Id\n    const intercapsHeader =\n      header[0]?.toUpperCase() +\n      header.substring(1).replace(/([^\\w])(\\w)/g, (_m, g1, g2) => g1 + g2.toUpperCase());\n    for (const key of [header, lowerCasedHeader, header.toUpperCase(), intercapsHeader]) {\n      const value = headers.get(key);\n      if (value) {\n        return value;\n      }\n    }\n  }\n\n  for (const [key, value] of Object.entries(headers)) {\n    if (key.toLowerCase() === lowerCasedHeader) {\n      if (Array.isArray(value)) {\n        if (value.length <= 1) return value[0];\n        console.warn(`Received ${value.length} entries for the ${header} header, using the first entry.`);\n        return value[0];\n      }\n      return value;\n    }\n  }\n\n  throw new Error(`Could not find ${header} header`);\n};\n\n/**\n * Encodes a string to Base64 format.\n */\nexport const toBase64 = (str: string | null | undefined): string => {\n  if (!str) return '';\n  if (typeof Buffer !== 'undefined') {\n    return Buffer.from(str).toString('base64');\n  }\n\n  if (typeof btoa !== 'undefined') {\n    return btoa(str);\n  }\n\n  throw new OpenAIError('Cannot generate b64 string; Expected `Buffer` or `btoa` to be defined');\n};\n\nexport function isObj(obj: unknown): obj is Record<string, unknown> {\n  return obj != null && typeof obj === 'object' && !Array.isArray(obj);\n}\n\n// Path: src/_shims/registry.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { type RequestOptions } from '../core';\n\nexport interface Shims {\n  kind: string;\n  fetch: any;\n  Request: any;\n  Response: any;\n  Headers: any;\n  FormData: any;\n  Blob: any;\n  File: any;\n  ReadableStream: any;\n  getMultipartRequestOptions: <T = Record<string, unknown>>(\n    form: Shims['FormData'],\n    opts: RequestOptions<T>,\n  ) => Promise<RequestOptions<T>>;\n  getDefaultAgent: (url: string) => any;\n  fileFromPath:\n    | ((path: string, filename?: string, options?: {}) => Promise<Shims['File']>)\n    | ((path: string, options?: {}) => Promise<Shims['File']>);\n  isFsReadStream: (value: any) => boolean;\n}\n\nexport let auto = false;\nexport let kind: Shims['kind'] | undefined = undefined;\nexport let fetch: Shims['fetch'] | undefined = undefined;\nexport let Request: Shims['Request'] | undefined = undefined;\nexport let Response: Shims['Response'] | undefined = undefined;\nexport let Headers: Shims['Headers'] | undefined = undefined;\nexport let FormData: Shims['FormData'] | undefined = undefined;\nexport let Blob: Shims['Blob'] | undefined = undefined;\nexport let File: Shims['File'] | undefined = undefined;\nexport let ReadableStream: Shims['ReadableStream'] | undefined = undefined;\nexport let getMultipartRequestOptions: Shims['getMultipartRequestOptions'] | undefined = undefined;\nexport let getDefaultAgent: Shims['getDefaultAgent'] | undefined = undefined;\nexport let fileFromPath: Shims['fileFromPath'] | undefined = undefined;\nexport let isFsReadStream: Shims['isFsReadStream'] | undefined = undefined;\n\nexport function setShims(shims: Shims, options: { auto: boolean } = { auto: false }) {\n  if (auto) {\n    throw new Error(\n      `you must \\`import 'openai/shims/${shims.kind}'\\` before importing anything else from openai`,\n    );\n  }\n  if (kind) {\n    throw new Error(`can't \\`import 'openai/shims/${shims.kind}'\\` after \\`import 'openai/shims/${kind}'\\``);\n  }\n  auto = options.auto;\n  kind = shims.kind;\n  fetch = shims.fetch;\n  Request = shims.Request;\n  Response = shims.Response;\n  Headers = shims.Headers;\n  FormData = shims.FormData;\n  Blob = shims.Blob;\n  File = shims.File;\n  ReadableStream = shims.ReadableStream;\n  getMultipartRequestOptions = shims.getMultipartRequestOptions;\n  getDefaultAgent = shims.getDefaultAgent;\n  fileFromPath = shims.fileFromPath;\n  isFsReadStream = shims.isFsReadStream;\n}\n\n// Path: src/_shims/web-runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { MultipartBody } from './MultipartBody';\nimport { type RequestOptions } from '../core';\nimport { type Shims } from './registry';\n\nexport function getRuntime({ manuallyImported }: { manuallyImported?: boolean } = {}): Shims {\n  const recommendation =\n    manuallyImported ?\n      `You may need to use polyfills`\n    : `Add one of these imports before your first \\`import \u2026 from 'openai'\\`:\n- \\`import 'openai/shims/node'\\` (if you're running on Node)\n- \\`import 'openai/shims/web'\\` (otherwise)\n`;\n\n  let _fetch, _Request, _Response, _Headers;\n  try {\n    // @ts-ignore\n    _fetch = fetch;\n    // @ts-ignore\n    _Request = Request;\n    // @ts-ignore\n    _Response = Response;\n    // @ts-ignore\n    _Headers = Headers;\n  } catch (error) {\n    throw new Error(\n      `this environment is missing the following Web Fetch API type: ${\n        (error as any).message\n      }. ${recommendation}`,\n    );\n  }\n\n  return {\n    kind: 'web',\n    fetch: _fetch,\n    Request: _Request,\n    Response: _Response,\n    Headers: _Headers,\n    FormData:\n      // @ts-ignore\n      typeof FormData !== 'undefined' ? FormData : (\n        class FormData {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'FormData' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    Blob:\n      typeof Blob !== 'undefined' ? Blob : (\n        class Blob {\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'Blob' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    File:\n      // @ts-ignore\n      typeof File !== 'undefined' ? File : (\n        class File {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'File' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    ReadableStream:\n      // @ts-ignore\n      typeof ReadableStream !== 'undefined' ? ReadableStream : (\n        class ReadableStream {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `streaming isn't supported in this environment yet as 'ReadableStream' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    getMultipartRequestOptions: async <T = Record<string, unknown>>(\n      // @ts-ignore\n      form: FormData,\n      opts: RequestOptions<T>,\n    ): Promise<RequestOptions<T>> => ({\n      ...opts,\n      body: new MultipartBody(form) as any,\n    }),\n    getDefaultAgent: (url: string) => undefined,\n    fileFromPath: () => {\n      throw new Error(\n        'The `fileFromPath` function is only supported in Node. See the README for more details: https://www.github.com/openai/openai-node#file-uploads',\n      );\n    },\n    isFsReadStream: (value: any) => false,\n  };\n}\n\n// Path: src/_shims/auto/runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../web-runtime';\n\n// Path: src/_shims/index.js\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nconst shims = require('./registry');\nconst auto = require('openai/_shims/auto/runtime');\nif (!shims.kind) shims.setShims(auto.getRuntime(), { auto: true });\nfor (const property of Object.keys(shims)) {\n  Object.defineProperty(exports, property, {\n    get() {\n      return shims[property];\n    },\n  });\n}\n\n// Path: src/streaming.ts\nimport { ReadableStream, type Response } from './_shims/index';\nimport { OpenAIError } from './error';\n\nimport { APIError } from 'openai/error';\n\ntype Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;\n\nexport type ServerSentEvent = {\n  event: string | null;\n  data: string;\n  raw: string[];\n};\n\nexport class Stream<Item> implements AsyncIterable<Item> {\n  controller: AbortController;\n\n  constructor(\n    private iterator: () => AsyncIterator<Item>,\n    controller: AbortController,\n  ) {\n    this.controller = controller;\n  }\n\n  static fromSSEResponse<Item>(response: Response, controller: AbortController) {\n    let consumed = false;\n\n    async function* iterator(): AsyncIterator<Item, any, undefined> {\n      if (consumed) {\n        throw new Error('Cannot iterate over a consumed stream, use `.tee()` to split the stream.');\n      }\n      consumed = true;\n      let done = false;\n      try {\n        for await (const sse of _iterSSEMessages(response, controller)) {\n          if (done) continue;\n\n          if (sse.data.startsWith('[DONE]')) {\n            done = true;\n            continue;\n          }\n\n          if (sse.event === null) {\n            let data;\n\n            try {\n              data = JSON.parse(sse.data);\n            } catch (e) {\n              console.error(`Could not parse message into JSON:`, sse.data);\n              console.error(`From chunk:`, sse.raw);\n              throw e;\n            }\n\n            if (data && data.error) {\n              throw new APIError(undefined, data.error, undefined, undefined);\n            }\n\n            yield data;\n          } else {\n            let data;\n            try {\n              data = JSON.parse(sse.data);\n            } catch (e) {\n              console.error(`Could not parse message into JSON:`, sse.data);\n              console.error(`From chunk:`, sse.raw);\n              throw e;\n            }\n            // TODO: Is this where the error should be thrown?\n            if (sse.event == 'error') {\n              throw new APIError(undefined, data.error, data.message, undefined);\n            }\n            yield { event: sse.event, data: data } as any;\n          }\n        }\n        done = true;\n      } catch (e) {\n        // If the user calls `stream.controller.abort()`, we should exit without throwing.\n        if (e instanceof Error && e.name === 'AbortError') return;\n        throw e;\n      } finally {\n        // If the user `break`s, abort the ongoing request.\n        if (!done) controller.abort();\n      }\n    }\n\n    return new Stream(iterator, controller);\n  }\n\n  /**\n   * Generates a Stream from a newline-separated ReadableStream\n   * where each item is a JSON value.\n   */\n  static fromReadableStream<Item>(readableStream: ReadableStream, controller: AbortController) {\n    let consumed = false;\n\n    async function* iterLines(): AsyncGenerator<string, void, unknown> {\n      const lineDecoder = new LineDecoder();\n\n      const iter = readableStreamAsyncIterable<Bytes>(readableStream);\n      for await (const chunk of iter) {\n        for (const line of lineDecoder.decode(chunk)) {\n          yield line;\n        }\n      }\n\n      for (const line of lineDecoder.flush()) {\n        yield line;\n      }\n    }\n\n    async function* iterator(): AsyncIterator<Item, any, undefined> {\n      if (consumed) {\n        throw new Error('Cannot iterate over a consumed stream, use `.tee()` to split the stream.');\n      }\n      consumed = true;\n      let done = false;\n      try {\n        for await (const line of iterLines()) {\n          if (done) continue;\n          if (line) yield JSON.parse(line);\n        }\n        done = true;\n      } catch (e) {\n        // If the user calls `stream.controller.abort()`, we should exit without throwing.\n        if (e instanceof Error && e.name === 'AbortError') return;\n        throw e;\n      } finally {\n        // If the user `break`s, abort the ongoing request.\n        if (!done) controller.abort();\n      }\n    }\n\n    return new Stream(iterator, controller);\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<Item> {\n    return this.iterator();\n  }\n\n  /**\n   * Splits the stream into two streams which can be\n   * independently read from at different speeds.\n   */\n  tee(): [Stream<Item>, Stream<Item>] {\n    const left: Array<Promise<IteratorResult<Item>>> = [];\n    const right: Array<Promise<IteratorResult<Item>>> = [];\n    const iterator = this.iterator();\n\n    const teeIterator = (queue: Array<Promise<IteratorResult<Item>>>): AsyncIterator<Item> => {\n      return {\n        next: () => {\n          if (queue.length === 0) {\n            const result = iterator.next();\n            left.push(result);\n            right.push(result);\n          }\n          return queue.shift()!;\n        },\n      };\n    };\n\n    return [\n      new Stream(() => teeIterator(left), this.controller),\n      new Stream(() => teeIterator(right), this.controller),\n    ];\n  }\n\n  /**\n   * Converts this stream to a newline-separated ReadableStream of\n   * JSON stringified values in the stream\n   * which can be turned back into a Stream with `Stream.fromReadableStream()`.\n   */\n  toReadableStream(): ReadableStream {\n    const self = this;\n    let iter: AsyncIterator<Item>;\n    const encoder = new TextEncoder();\n\n    return new ReadableStream({\n      async start() {\n        iter = self[Symbol.asyncIterator]();\n      },\n      async pull(ctrl: any) {\n        try {\n          const { value, done } = await iter.next();\n          if (done) return ctrl.close();\n\n          const bytes = encoder.encode(JSON.stringify(value) + '\\n');\n\n          ctrl.enqueue(bytes);\n        } catch (err) {\n          ctrl.error(err);\n        }\n      },\n      async cancel() {\n        await iter.return?.();\n      },\n    });\n  }\n}\n\nexport async function* _iterSSEMessages(\n  response: Response,\n  controller: AbortController,\n): AsyncGenerator<ServerSentEvent, void, unknown> {\n  if (!response.body) {\n    controller.abort();\n    throw new OpenAIError(`Attempted to iterate over a response with no body`);\n  }\n\n  const sseDecoder = new SSEDecoder();\n  const lineDecoder = new LineDecoder();\n\n  const iter = readableStreamAsyncIterable<Bytes>(response.body);\n  for await (const sseChunk of iterSSEChunks(iter)) {\n    for (const line of lineDecoder.decode(sseChunk)) {\n      const sse = sseDecoder.decode(line);\n      if (sse) yield sse;\n    }\n  }\n\n  for (const line of lineDecoder.flush()) {\n    const sse = sseDecoder.decode(line);\n    if (sse) yield sse;\n  }\n}\n\n/**\n * Given an async iterable iterator, iterates over it and yields full\n * SSE chunks, i.e. yields when a double new-line is encountered.\n */\nasync function* iterSSEChunks(iterator: AsyncIterableIterator<Bytes>): AsyncGenerator<Uint8Array> {\n  let data = new Uint8Array();\n\n  for await (const chunk of iterator) {\n    if (chunk == null) {\n      continue;\n    }\n\n    const binaryChunk =\n      chunk instanceof ArrayBuffer ? new Uint8Array(chunk)\n      : typeof chunk === 'string' ? new TextEncoder().encode(chunk)\n      : chunk;\n\n    let newData = new Uint8Array(data.length + binaryChunk.length);\n    newData.set(data);\n    newData.set(binaryChunk, data.length);\n    data = newData;\n\n    let patternIndex;\n    while ((patternIndex = findDoubleNewlineIndex(data)) !== -1) {\n      yield data.slice(0, patternIndex);\n      data = data.slice(patternIndex);\n    }\n  }\n\n  if (data.length > 0) {\n    yield data;\n  }\n}\n\nfunction findDoubleNewlineIndex(buffer: Uint8Array): number {\n  // This function searches the buffer for the end patterns (\\r\\r, \\n\\n, \\r\\n\\r\\n)\n  // and returns the index right after the first occurrence of any pattern,\n  // or -1 if none of the patterns are found.\n  const newline = 0x0a; // \\n\n  const carriage = 0x0d; // \\r\n\n  for (let i = 0; i < buffer.length - 2; i++) {\n    if (buffer[i] === newline && buffer[i + 1] === newline) {\n      // \\n\\n\n      return i + 2;\n    }\n    if (buffer[i] === carriage && buffer[i + 1] === carriage) {\n      // \\r\\r\n      return i + 2;\n    }\n    if (\n      buffer[i] === carriage &&\n      buffer[i + 1] === newline &&\n      i + 3 < buffer.length &&\n      buffer[i + 2] === carriage &&\n      buffer[i + 3] === newline\n    ) {\n      // \\r\\n\\r\\n\n      return i + 4;\n    }\n  }\n\n  return -1;\n}\n\nclass SSEDecoder {\n  private data: string[];\n  private event: string | null;\n  private chunks: string[];\n\n  constructor() {\n    this.event = null;\n    this.data = [];\n    this.chunks = [];\n  }\n\n  decode(line: string) {\n    if (line.endsWith('\\r')) {\n      line = line.substring(0, line.length - 1);\n    }\n\n    if (!line) {\n      // empty line and we didn't previously encounter any messages\n      if (!this.event && !this.data.length) return null;\n\n      const sse: ServerSentEvent = {\n        event: this.event,\n        data: this.data.join('\\n'),\n        raw: this.chunks,\n      };\n\n      this.event = null;\n      this.data = [];\n      this.chunks = [];\n\n      return sse;\n    }\n\n    this.chunks.push(line);\n\n    if (line.startsWith(':')) {\n      return null;\n    }\n\n    let [fieldname, _, value] = partition(line, ':');\n\n    if (value.startsWith(' ')) {\n      value = value.substring(1);\n    }\n\n    if (fieldname === 'event') {\n      this.event = value;\n    } else if (fieldname === 'data') {\n      this.data.push(value);\n    }\n\n    return null;\n  }\n}\n\n/**\n * A re-implementation of httpx's `LineDecoder` in Python that handles incrementally\n * reading lines from text.\n *\n * https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258\n */\nclass LineDecoder {\n  // prettier-ignore\n  static NEWLINE_CHARS = new Set(['\\n', '\\r']);\n  static NEWLINE_REGEXP = /\\r\\n|[\\n\\r]/g;\n\n  buffer: string[];\n  trailingCR: boolean;\n  textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either \"dom\" or \"node\" types.\n\n  constructor() {\n    this.buffer = [];\n    this.trailingCR = false;\n  }\n\n  decode(chunk: Bytes): string[] {\n    let text = this.decodeText(chunk);\n\n    if (this.trailingCR) {\n      text = '\\r' + text;\n      this.trailingCR = false;\n    }\n    if (text.endsWith('\\r')) {\n      this.trailingCR = true;\n      text = text.slice(0, -1);\n    }\n\n    if (!text) {\n      return [];\n    }\n\n    const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');\n    let lines = text.split(LineDecoder.NEWLINE_REGEXP);\n\n    // if there is a trailing new line then the last entry will be an empty\n    // string which we don't care about\n    if (trailingNewline) {\n      lines.pop();\n    }\n\n    if (lines.length === 1 && !trailingNewline) {\n      this.buffer.push(lines[0]!);\n      return [];\n    }\n\n    if (this.buffer.length > 0) {\n      lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];\n      this.buffer = [];\n    }\n\n    if (!trailingNewline) {\n      this.buffer = [lines.pop() || ''];\n    }\n\n    return lines;\n  }\n\n  decodeText(bytes: Bytes): string {\n    if (bytes == null) return '';\n    if (typeof bytes === 'string') return bytes;\n\n    // Node:\n    if (typeof Buffer !== 'undefined') {\n      if (bytes instanceof Buffer) {\n        return bytes.toString();\n      }\n      if (bytes instanceof Uint8Array) {\n        return Buffer.from(bytes).toString();\n      }\n\n      throw new OpenAIError(\n        `Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global \"Buffer\" defined, which this library assumes to be Node. Please report this error.`,\n      );\n    }\n\n    // Browser\n    if (typeof TextDecoder !== 'undefined') {\n      if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {\n        this.textDecoder ??= new TextDecoder('utf8');\n        return this.textDecoder.decode(bytes);\n      }\n\n      throw new OpenAIError(\n        `Unexpected: received non-Uint8Array/ArrayBuffer (${\n          (bytes as any).constructor.name\n        }) in a web platform. Please report this error.`,\n      );\n    }\n\n    throw new OpenAIError(\n      `Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`,\n    );\n  }\n\n  flush(): string[] {\n    if (!this.buffer.length && !this.trailingCR) {\n      return [];\n    }\n\n    const lines = [this.buffer.join('')];\n    this.buffer = [];\n    this.trailingCR = false;\n    return lines;\n  }\n}\n\n/** This is an internal helper function that's just used for testing */\nexport function _decodeChunks(chunks: string[]): string[] {\n  const decoder = new LineDecoder();\n  const lines: string[] = [];\n  for (const chunk of chunks) {\n    lines.push(...decoder.decode(chunk));\n  }\n\n  return lines;\n}\n\nfunction partition(str: string, delimiter: string): [string, string, string] {\n  const index = str.indexOf(delimiter);\n  if (index !== -1) {\n    return [str.substring(0, index), delimiter, str.substring(index + delimiter.length)];\n  }\n\n  return [str, '', ''];\n}\n\n/**\n * Most browsers don't yet have async iterable support for ReadableStream,\n * and Node has a very different way of reading bytes from its \"ReadableStream\".\n *\n * This polyfill was pulled from https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490\n */\nexport function readableStreamAsyncIterable<T>(stream: any): AsyncIterableIterator<T> {\n  if (stream[Symbol.asyncIterator]) return stream;\n\n  const reader = stream.getReader();\n  return {\n    async next() {\n      try {\n        const result = await reader.read();\n        if (result?.done) reader.releaseLock(); // release lock when stream becomes closed\n        return result;\n      } catch (e) {\n        reader.releaseLock(); // release lock when stream becomes errored\n        throw e;\n      }\n    },\n    async return() {\n      const cancelPromise = reader.cancel();\n      reader.releaseLock();\n      await cancelPromise;\n      return { done: true, value: undefined };\n    },\n    [Symbol.asyncIterator]() {\n      return this;\n    },\n  };\n}\n\n// Path: src/pagination.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { AbstractPage, Response, APIClient, FinalRequestOptions, PageInfo } from './core';\n\nexport interface PageResponse<Item> {\n  data: Array<Item>;\n\n  object: string;\n}\n\n/**\n * Note: no pagination actually occurs yet, this is for forwards-compatibility.\n */\nexport class Page<Item> extends AbstractPage<Item> implements PageResponse<Item> {\n  data: Array<Item>;\n\n  object: string;\n\n  constructor(client: APIClient, response: Response, body: PageResponse<Item>, options: FinalRequestOptions) {\n    super(client, response, body, options);\n\n    this.data = body.data || [];\n    this.object = body.object;\n  }\n\n  getPaginatedItems(): Item[] {\n    return this.data ?? [];\n  }\n\n  // @deprecated Please use `nextPageInfo()` instead\n  /**\n   * This page represents a response that isn't actually paginated at the API level\n   * so there will never be any next page params.\n   */\n  nextPageParams(): null {\n    return null;\n  }\n\n  nextPageInfo(): null {\n    return null;\n  }\n}\n\nexport interface CursorPageResponse<Item> {\n  data: Array<Item>;\n}\n\nexport interface CursorPageParams {\n  after?: string;\n\n  limit?: number;\n}\n\nexport class CursorPage<Item extends { id: string }>\n  extends AbstractPage<Item>\n  implements CursorPageResponse<Item>\n{\n  data: Array<Item>;\n\n  constructor(\n    client: APIClient,\n    response: Response,\n    body: CursorPageResponse<Item>,\n    options: FinalRequestOptions,\n  ) {\n    super(client, response, body, options);\n\n    this.data = body.data || [];\n  }\n\n  getPaginatedItems(): Item[] {\n    return this.data ?? [];\n  }\n\n  // @deprecated Please use `nextPageInfo()` instead\n  nextPageParams(): Partial<CursorPageParams> | null {\n    const info = this.nextPageInfo();\n    if (!info) return null;\n    if ('params' in info) return info.params;\n    const params = Object.fromEntries(info.url.searchParams);\n    if (!Object.keys(params).length) return null;\n    return params;\n  }\n\n  nextPageInfo(): PageInfo | null {\n    const data = this.getPaginatedItems();\n    if (!data.length) {\n      return null;\n    }\n\n    const id = data[data.length - 1]?.id;\n    if (!id) {\n      return null;\n    }\n\n    return { params: { after: id } };\n  }\n}\n\n// Path: src/resources/files.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { isRequestOptions } from 'openai/core';\nimport { type Response } from 'openai/_shims/index';\nimport { sleep } from 'openai/core';\nimport { APIConnectionTimeoutError } from 'openai/error';\nimport * as FilesAPI from 'openai/resources/files';\nimport { type Uploadable, multipartFormRequestOptions } from 'openai/core';\nimport { Page } from 'openai/pagination';\n\nexport class Files extends APIResource {\n  /**\n   * Upload a file that can be used across various endpoints. The size of all the\n   * files uploaded by one organization can be up to 100 GB.\n   *\n   * The size of individual files can be a maximum of 512 MB or 2 million tokens for\n   * Assistants. See the\n   * [Assistants Tools guide](https://platform.openai.com/docs/assistants/tools) to\n   * learn more about the types of files supported. The Fine-tuning API only supports\n   * `.jsonl` files.\n   *\n   * Please [contact us](https://help.openai.com/) if you need to increase these\n   * storage limits.\n   */\n  create(body: FileCreateParams, options?: Core.RequestOptions): Core.APIPromise<FileObject> {\n    return this._client.post('/files', multipartFormRequestOptions({ body, ...options }));\n  }\n\n  /**\n   * Returns information about a specific file.\n   */\n  retrieve(fileId: string, options?: Core.RequestOptions): Core.APIPromise<FileObject> {\n    return this._client.get(`/files/${fileId}`, options);\n  }\n\n  /**\n   * Returns a list of files that belong to the user's organization.\n   */\n  list(query?: FileListParams, options?: Core.RequestOptions): Core.PagePromise<FileObjectsPage, FileObject>;\n  list(options?: Core.RequestOptions): Core.PagePromise<FileObjectsPage, FileObject>;\n  list(\n    query: FileListParams | Core.RequestOptions = {},\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<FileObjectsPage, FileObject> {\n    if (isRequestOptions(query)) {\n      return this.list({}, query);\n    }\n    return this._client.getAPIList('/files', FileObjectsPage, { query, ...options });\n  }\n\n  /**\n   * Delete a file.\n   */\n  del(fileId: string, options?: Core.RequestOptions): Core.APIPromise<FileDeleted> {\n    return this._client.delete(`/files/${fileId}`, options);\n  }\n\n  /**\n   * Returns the contents of the specified file.\n   */\n  content(fileId: string, options?: Core.RequestOptions): Core.APIPromise<Response> {\n    return this._client.get(`/files/${fileId}/content`, { ...options, __binaryResponse: true });\n  }\n\n  /**\n   * Returns the contents of the specified file.\n   *\n   * @deprecated The `.content()` method should be used instead\n   */\n  retrieveContent(fileId: string, options?: Core.RequestOptions): Core.APIPromise<string> {\n    return this._client.get(`/files/${fileId}/content`, {\n      ...options,\n      headers: { Accept: 'application/json', ...options?.headers },\n    });\n  }\n\n  /**\n   * Waits for the given file to be processed, default timeout is 30 mins.\n   */\n  async waitForProcessing(\n    id: string,\n    { pollInterval = 5000, maxWait = 30 * 60 * 1000 }: { pollInterval?: number; maxWait?: number } = {},\n  ): Promise<FileObject> {\n    const TERMINAL_STATES = new Set(['processed', 'error', 'deleted']);\n\n    const start = Date.now();\n    let file = await this.retrieve(id);\n\n    while (!file.status || !TERMINAL_STATES.has(file.status)) {\n      await sleep(pollInterval);\n\n      file = await this.retrieve(id);\n      if (Date.now() - start > maxWait) {\n        throw new APIConnectionTimeoutError({\n          message: `Giving up on waiting for file ${id} to finish processing after ${maxWait} milliseconds.`,\n        });\n      }\n    }\n\n    return file;\n  }\n}\n\n/**\n * Note: no pagination actually occurs yet, this is for forwards-compatibility.\n */\nexport class FileObjectsPage extends Page<FileObject> {}\n\nexport type FileContent = string;\n\nexport interface FileDeleted {\n  id: string;\n\n  deleted: boolean;\n\n  object: 'file';\n}\n\n/**\n * The `File` object represents a document that has been uploaded to OpenAI.\n */\nexport interface FileObject {\n  /**\n   * The file identifier, which can be referenced in the API endpoints.\n   */\n  id: string;\n\n  /**\n   * The size of the file, in bytes.\n   */\n  bytes: number;\n\n  /**\n   * The Unix timestamp (in seconds) for when the file was created.\n   */\n  created_at: number;\n\n  /**\n   * The name of the file.\n   */\n  filename: string;\n\n  /**\n   * The object type, which is always `file`.\n   */\n  object: 'file';\n\n  /**\n   * The intended purpose of the file. Supported values are `fine-tune`,\n   * `fine-tune-results`, `assistants`, and `assistants_output`.\n   */\n  purpose: 'fine-tune' | 'fine-tune-results' | 'assistants' | 'assistants_output';\n\n  /**\n   * @deprecated: Deprecated. The current status of the file, which can be either\n   * `uploaded`, `processed`, or `error`.\n   */\n  status: 'uploaded' | 'processed' | 'error';\n\n  /**\n   * @deprecated: Deprecated. For details on why a fine-tuning training file failed\n   * validation, see the `error` field on `fine_tuning.job`.\n   */\n  status_details?: string;\n}\n\nexport interface FileCreateParams {\n  /**\n   * The File object (not file name) to be uploaded.\n   */\n  file: Uploadable;\n\n  /**\n   * The intended purpose of the uploaded file.\n   *\n   * Use \"fine-tune\" for\n   * [Fine-tuning](https://platform.openai.com/docs/api-reference/fine-tuning) and\n   * \"assistants\" for\n   * [Assistants](https://platform.openai.com/docs/api-reference/assistants) and\n   * [Messages](https://platform.openai.com/docs/api-reference/messages). This allows\n   * us to validate the format of the uploaded file is correct for fine-tuning.\n   */\n  purpose: 'fine-tune' | 'assistants';\n}\n\nexport interface FileListParams {\n  /**\n   * Only return files with the given purpose.\n   */\n  purpose?: string;\n}\n\nexport namespace Files {\n  export import FileContent = FilesAPI.FileContent;\n  export import FileDeleted = FilesAPI.FileDeleted;\n  export import FileObject = FilesAPI.FileObject;\n  export import FileObjectsPage = FilesAPI.FileObjectsPage;\n  export import FileCreateParams = FilesAPI.FileCreateParams;\n  export import FileListParams = FilesAPI.FileListParams;\n}\n\n// Path: src/resources/audio/transcriptions.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport * as TranscriptionsAPI from 'openai/resources/audio/transcriptions';\nimport { type Uploadable, multipartFormRequestOptions } from 'openai/core';\n\nexport class Transcriptions extends APIResource {\n  /**\n   * Transcribes audio into the input language.\n   */\n  create(body: TranscriptionCreateParams, options?: Core.RequestOptions): Core.APIPromise<Transcription> {\n    return this._client.post('/audio/transcriptions', multipartFormRequestOptions({ body, ...options }));\n  }\n}\n\n/**\n * Represents a transcription response returned by model, based on the provided\n * input.\n */\nexport interface Transcription {\n  /**\n   * The transcribed text.\n   */\n  text: string;\n}\n\nexport interface TranscriptionCreateParams {\n  /**\n   * The audio file object (not file name) to transcribe, in one of these formats:\n   * flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.\n   */\n  file: Uploadable;\n\n  /**\n   * ID of the model to use. Only `whisper-1` (which is powered by our open source\n   * Whisper V2 model) is currently available.\n   */\n  model: (string & {}) | 'whisper-1';\n\n  /**\n   * The language of the input audio. Supplying the input language in\n   * [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will\n   * improve accuracy and latency.\n   */\n  language?: string;\n\n  /**\n   * An optional text to guide the model's style or continue a previous audio\n   * segment. The\n   * [prompt](https://platform.openai.com/docs/guides/speech-to-text/prompting)\n   * should match the audio language.\n   */\n  prompt?: string;\n\n  /**\n   * The format of the transcript output, in one of these options: `json`, `text`,\n   * `srt`, `verbose_json`, or `vtt`.\n   */\n  response_format?: 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt';\n\n  /**\n   * The sampling temperature, between 0 and 1. Higher values like 0.8 will make the\n   * output more random, while lower values like 0.2 will make it more focused and\n   * deterministic. If set to 0, the model will use\n   * [log probability](https://en.wikipedia.org/wiki/Log_probability) to\n   * automatically increase the temperature until certain thresholds are hit.\n   */\n  temperature?: number;\n\n  /**\n   * The timestamp granularities to populate for this transcription.\n   * `response_format` must be set `verbose_json` to use timestamp granularities.\n   * Either or both of these options are supported: `word`, or `segment`. Note: There\n   * is no additional latency for segment timestamps, but generating word timestamps\n   * incurs additional latency.\n   */\n  timestamp_granularities?: Array<'word' | 'segment'>;\n}\n\nexport namespace Transcriptions {\n  export import Transcription = TranscriptionsAPI.Transcription;\n  export import TranscriptionCreateParams = TranscriptionsAPI.TranscriptionCreateParams;\n}\n\n// Path: src/resources/audio/speech.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { type Response } from 'openai/_shims/index';\nimport * as SpeechAPI from 'openai/resources/audio/speech';\n\nexport class Speech extends APIResource {\n  /**\n   * Generates audio from the input text.\n   */\n  create(body: SpeechCreateParams, options?: Core.RequestOptions): Core.APIPromise<Response> {\n    return this._client.post('/audio/speech', { body, ...options, __binaryResponse: true });\n  }\n}\n\nexport interface SpeechCreateParams {\n  /**\n   * The text to generate audio for. The maximum length is 4096 characters.\n   */\n  input: string;\n\n  /**\n   * One of the available [TTS models](https://platform.openai.com/docs/models/tts):\n   * `tts-1` or `tts-1-hd`\n   */\n  model: (string & {}) | 'tts-1' | 'tts-1-hd';\n\n  /**\n   * The voice to use when generating the audio. Supported voices are `alloy`,\n   * `echo`, `fable`, `onyx`, `nova`, and `shimmer`. Previews of the voices are\n   * available in the\n   * [Text to speech guide](https://platform.openai.com/docs/guides/text-to-speech/voice-options).\n   */\n  voice: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';\n\n  /**\n   * The format to audio in. Supported formats are `mp3`, `opus`, `aac`, `flac`,\n   * `wav`, and `pcm`.\n   */\n  response_format?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';\n\n  /**\n   * The speed of the generated audio. Select a value from `0.25` to `4.0`. `1.0` is\n   * the default.\n   */\n  speed?: number;\n}\n\nexport namespace Speech {\n  export import SpeechCreateParams = SpeechAPI.SpeechCreateParams;\n}\n\n// Path: src/resources/audio/translations.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport * as TranslationsAPI from 'openai/resources/audio/translations';\nimport { type Uploadable, multipartFormRequestOptions } from 'openai/core';\n\nexport class Translations extends APIResource {\n  /**\n   * Translates audio into English.\n   */\n  create(body: TranslationCreateParams, options?: Core.RequestOptions): Core.APIPromise<Translation> {\n    return this._client.post('/audio/translations', multipartFormRequestOptions({ body, ...options }));\n  }\n}\n\nexport interface Translation {\n  text: string;\n}\n\nexport interface TranslationCreateParams {\n  /**\n   * The audio file object (not file name) translate, in one of these formats: flac,\n   * mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.\n   */\n  file: Uploadable;\n\n  /**\n   * ID of the model to use. Only `whisper-1` (which is powered by our open source\n   * Whisper V2 model) is currently available.\n   */\n  model: (string & {}) | 'whisper-1';\n\n  /**\n   * An optional text to guide the model's style or continue a previous audio\n   * segment. The\n   * [prompt](https://platform.openai.com/docs/guides/speech-to-text/prompting)\n   * should be in English.\n   */\n  prompt?: string;\n\n  /**\n   * The format of the transcript output, in one of these options: `json`, `text`,\n   * `srt`, `verbose_json`, or `vtt`.\n   */\n  response_format?: string;\n\n  /**\n   * The sampling temperature, between 0 and 1. Higher values like 0.8 will make the\n   * output more random, while lower values like 0.2 will make it more focused and\n   * deterministic. If set to 0, the model will use\n   * [log probability](https://en.wikipedia.org/wiki/Log_probability) to\n   * automatically increase the temperature until certain thresholds are hit.\n   */\n  temperature?: number;\n}\n\nexport namespace Translations {\n  export import Translation = TranslationsAPI.Translation;\n  export import TranslationCreateParams = TranslationsAPI.TranslationCreateParams;\n}\n\n// Path: src/resources/audio/audio.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { APIResource } from 'openai/resource';\nimport * as SpeechAPI from 'openai/resources/audio/speech';\nimport * as TranscriptionsAPI from 'openai/resources/audio/transcriptions';\nimport * as TranslationsAPI from 'openai/resources/audio/translations';\n\nexport class Audio extends APIResource {\n  transcriptions: TranscriptionsAPI.Transcriptions = new TranscriptionsAPI.Transcriptions(this._client);\n  translations: TranslationsAPI.Translations = new TranslationsAPI.Translations(this._client);\n  speech: SpeechAPI.Speech = new SpeechAPI.Speech(this._client);\n}\n\nexport namespace Audio {\n  export import Transcriptions = TranscriptionsAPI.Transcriptions;\n  export import Transcription = TranscriptionsAPI.Transcription;\n  export import TranscriptionCreateParams = TranscriptionsAPI.TranscriptionCreateParams;\n  export import Translations = TranslationsAPI.Translations;\n  export import Translation = TranslationsAPI.Translation;\n  export import TranslationCreateParams = TranslationsAPI.TranslationCreateParams;\n  export import Speech = SpeechAPI.Speech;\n  export import SpeechCreateParams = SpeechAPI.SpeechCreateParams;\n}\n\n// Path: src/resources/embeddings.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport * as EmbeddingsAPI from 'openai/resources/embeddings';\n\nexport class Embeddings extends APIResource {\n  /**\n   * Creates an embedding vector representing the input text.\n   */\n  create(\n    body: EmbeddingCreateParams,\n    options?: Core.RequestOptions,\n  ): Core.APIPromise<CreateEmbeddingResponse> {\n    return this._client.post('/embeddings', { body, ...options });\n  }\n}\n\nexport interface CreateEmbeddingResponse {\n  /**\n   * The list of embeddings generated by the model.\n   */\n  data: Array<Embedding>;\n\n  /**\n   * The name of the model used to generate the embedding.\n   */\n  model: string;\n\n  /**\n   * The object type, which is always \"list\".\n   */\n  object: 'list';\n\n  /**\n   * The usage information for the request.\n   */\n  usage: CreateEmbeddingResponse.Usage;\n}\n\nexport namespace CreateEmbeddingResponse {\n  /**\n   * The usage information for the request.\n   */\n  export interface Usage {\n    /**\n     * The number of tokens used by the prompt.\n     */\n    prompt_tokens: number;\n\n    /**\n     * The total number of tokens used by the request.\n     */\n    total_tokens: number;\n  }\n}\n\n/**\n * Represents an embedding vector returned by embedding endpoint.\n */\nexport interface Embedding {\n  /**\n   * The embedding vector, which is a list of floats. The length of vector depends on\n   * the model as listed in the\n   * [embedding guide](https://platform.openai.com/docs/guides/embeddings).\n   */\n  embedding: Array<number>;\n\n  /**\n   * The index of the embedding in the list of embeddings.\n   */\n  index: number;\n\n  /**\n   * The object type, which is always \"embedding\".\n   */\n  object: 'embedding';\n}\n\nexport interface EmbeddingCreateParams {\n  /**\n   * Input text to embed, encoded as a string or array of tokens. To embed multiple\n   * inputs in a single request, pass an array of strings or array of token arrays.\n   * The input must not exceed the max input tokens for the model (8192 tokens for\n   * `text-embedding-ada-002`), cannot be an empty string, and any array must be 2048\n   * dimensions or less.\n   * [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken)\n   * for counting tokens.\n   */\n  input: string | Array<string> | Array<number> | Array<Array<number>>;\n\n  /**\n   * ID of the model to use. You can use the\n   * [List models](https://platform.openai.com/docs/api-reference/models/list) API to\n   * see all of your available models, or see our\n   * [Model overview](https://platform.openai.com/docs/models/overview) for\n   * descriptions of them.\n   */\n  model: (string & {}) | 'text-embedding-ada-002' | 'text-embedding-3-small' | 'text-embedding-3-large';\n\n  /**\n   * The number of dimensions the resulting output embeddings should have. Only\n   * supported in `text-embedding-3` and later models.\n   */\n  dimensions?: number;\n\n  /**\n   * The format to return the embeddings in. Can be either `float` or\n   * [`base64`](https://pypi.org/project/pybase64/).\n   */\n  encoding_format?: 'float' | 'base64';\n\n  /**\n   * A unique identifier representing your end-user, which can help OpenAI to monitor\n   * and detect abuse.\n   * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n   */\n  user?: string;\n}\n\nexport namespace Embeddings {\n  export import CreateEmbeddingResponse = EmbeddingsAPI.CreateEmbeddingResponse;\n  export import Embedding = EmbeddingsAPI.Embedding;\n  export import EmbeddingCreateParams = EmbeddingsAPI.EmbeddingCreateParams;\n}\n\n// Path: src/resources/shared.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport interface ErrorObject {\n  code: string | null;\n\n  message: string;\n\n  param: string | null;\n\n  type: string;\n}\n\nexport interface FunctionDefinition {\n  /**\n   * The name of the function to be called. Must be a-z, A-Z, 0-9, or contain\n   * underscores and dashes, with a maximum length of 64.\n   */\n  name: string;\n\n  /**\n   * A description of what the function does, used by the model to choose when and\n   * how to call the function.\n   */\n...\n", "needle_token_start": 589, "needle_token_end": 1034, "code_context_ntokens": 16087, "output": ["```typescript\nconst getBrowserInfo = (): BrowserInfo | null => {\n  if (typeof navigator === 'undefined' || !navigator) {\n    return null;\n  }\n\n  // NOTE: The order matters here!\n  const browserPatterns = [\n    { key: 'edge' as const, pattern: /Edge(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /MSIE(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /Trident(?:.*rv\\:(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'chrome' as const, pattern: /Chrome(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'firefox' as const, pattern: /Firefox(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'safari' as const, pattern: /(?:Version\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?(?:\\W+Mobile\\S*)?\\W+Safari/ },\n  ];\n\n  // Find the FIRST matching browser\n  for (const { key, pattern } of browserPatterns) {\n    const match = pattern.exec(navigator.userAgent);\n    if (match) {\n      const major = match[1] || 0;\n      const minor = match[2] || 0;\n      const patch = match[3] || 0;\n\n      return { browser: key, version: `${major}.${minor}.${patch}` };\n    }\n  }\n\n  return null;\n};\n```"]}
{"repo": "openai/openai-node", "name": "mockFetch", "language": "typescript", "path": "src/lib/ChatCompletionRunFunctions.test.ts", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to simulate network requests for testing purposes without making actual HTTP calls. It allows interception and handling of these simulated requests with custom logic.\n2. **Input**: The function accepts a callback function that defines how to handle the simulated network request.\n3. **Output**: It returns an object containing two functions: one that simulates the network request and another that sets up the callback to handle the next simulated request.\n4. **Procedure**: \n   - Initializes queues for managing request handlers and their promises.\n   - Provides a function to simulate network requests, which waits for a handler from the queue to process the request.\n   - Includes a function to set up a handler for the next request, allowing the user to define how the request should be handled when it is made.\n   - Handles potential abort signals in network requests, allowing the simulation to mimic real-world scenarios where requests might be cancelled.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/index.ts\n...\n// Path: src/resource.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport type { OpenAI } from './index';\n\nexport class APIResource {\n  protected _client: OpenAI;\n\n  constructor(client: OpenAI) {\n    this._client = client;\n  }\n}\n\n// Path: src/resources/audio/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport { Audio } from './audio';\nexport { SpeechCreateParams, Speech } from './speech';\nexport { Transcription, TranscriptionCreateParams, Transcriptions } from './transcriptions';\nexport { Translation, TranslationCreateParams, Translations } from './translations';\n\n// Path: src/resources/fine-tuning/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport { FineTuning } from './fine-tuning';\nexport {\n  FineTuningJob,\n  FineTuningJobEvent,\n  JobCreateParams,\n  JobListParams,\n  JobListEventsParams,\n  FineTuningJobsPage,\n  FineTuningJobEventsPage,\n  Jobs,\n} from './jobs';\n\n// Path: src/resources/beta/chat/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport { Chat } from './chat';\nexport { Completions } from './completions';\n\n// Path: src/resources/beta/threads/runs/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  CodeInterpreterLogs,\n  CodeInterpreterOutputImage,\n  CodeInterpreterToolCall,\n  CodeInterpreterToolCallDelta,\n  FunctionToolCall,\n  FunctionToolCallDelta,\n  MessageCreationStepDetails,\n  RetrievalToolCall,\n  RetrievalToolCallDelta,\n  RunStep,\n  RunStepDelta,\n  RunStepDeltaEvent,\n  RunStepDeltaMessageDelta,\n  ToolCall,\n  ToolCallDelta,\n  ToolCallDeltaObject,\n  ToolCallsStepDetails,\n  StepListParams,\n  RunStepsPage,\n  Steps,\n} from './steps';\nexport {\n  RequiredActionFunctionToolCall,\n  Run,\n  RunStatus,\n  RunCreateParams,\n  RunCreateParamsNonStreaming,\n  RunCreateParamsStreaming,\n  RunUpdateParams,\n  RunListParams,\n  RunCreateAndPollParams,\n  RunCreateAndStreamParams,\n  RunStreamParams,\n  RunSubmitToolOutputsParams,\n  RunSubmitToolOutputsParamsNonStreaming,\n  RunSubmitToolOutputsParamsStreaming,\n  RunSubmitToolOutputsAndPollParams,\n  RunSubmitToolOutputsStreamParams,\n  RunsPage,\n  Runs,\n} from './runs';\n\n// Path: src/resources/beta/threads/messages/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Annotation,\n  AnnotationDelta,\n  FileCitationAnnotation,\n  FileCitationDeltaAnnotation,\n  FilePathAnnotation,\n  FilePathDeltaAnnotation,\n  ImageFile,\n  ImageFileContentBlock,\n  ImageFileDelta,\n  ImageFileDeltaBlock,\n  Message,\n  MessageContent,\n  MessageContentDelta,\n  MessageDeleted,\n  MessageDelta,\n  MessageDeltaEvent,\n  Text,\n  TextContentBlock,\n  TextDelta,\n  TextDeltaBlock,\n  MessageCreateParams,\n  MessageUpdateParams,\n  MessageListParams,\n  MessagesPage,\n  Messages,\n} from './messages';\nexport { MessageFile, FileListParams, MessageFilesPage, Files } from './files';\n\n// Path: src/resources/beta/threads/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Annotation,\n  AnnotationDelta,\n  FileCitationAnnotation,\n  FileCitationDeltaAnnotation,\n  FilePathAnnotation,\n  FilePathDeltaAnnotation,\n  ImageFile,\n  ImageFileContentBlock,\n  ImageFileDelta,\n  ImageFileDeltaBlock,\n  Message,\n  MessageContent,\n  MessageContentDelta,\n  MessageDeleted,\n  MessageDelta,\n  MessageDeltaEvent,\n  Text,\n  TextContentBlock,\n  TextDelta,\n  TextDeltaBlock,\n  MessageCreateParams,\n  MessageUpdateParams,\n  MessageListParams,\n  MessagesPage,\n  Messages,\n} from './messages/index';\nexport {\n  RequiredActionFunctionToolCall,\n  Run,\n  RunStatus,\n  RunCreateParams,\n  RunCreateParamsNonStreaming,\n  RunCreateParamsStreaming,\n  RunUpdateParams,\n  RunListParams,\n  RunCreateAndPollParams,\n  RunCreateAndStreamParams,\n  RunStreamParams,\n  RunSubmitToolOutputsParams,\n  RunSubmitToolOutputsParamsNonStreaming,\n  RunSubmitToolOutputsParamsStreaming,\n  RunSubmitToolOutputsAndPollParams,\n  RunSubmitToolOutputsStreamParams,\n  RunsPage,\n  Runs,\n} from './runs/index';\nexport {\n  Thread,\n  ThreadDeleted,\n  ThreadCreateParams,\n  ThreadUpdateParams,\n  ThreadCreateAndRunParams,\n  ThreadCreateAndRunParamsNonStreaming,\n  ThreadCreateAndRunParamsStreaming,\n  ThreadCreateAndRunPollParams,\n  ThreadCreateAndRunStreamParams,\n  Threads,\n} from './threads';\n\n// Path: src/resources/beta/assistants/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Assistant,\n  AssistantDeleted,\n  AssistantStreamEvent,\n  AssistantTool,\n  CodeInterpreterTool,\n  FunctionTool,\n  MessageStreamEvent,\n  RetrievalTool,\n  RunStepStreamEvent,\n  RunStreamEvent,\n  ThreadStreamEvent,\n  AssistantCreateParams,\n  AssistantUpdateParams,\n  AssistantListParams,\n  AssistantsPage,\n  Assistants,\n} from './assistants';\nexport {\n  AssistantFile,\n  FileDeleteResponse,\n  FileCreateParams,\n  FileListParams,\n  AssistantFilesPage,\n  Files,\n} from './files';\n\n// Path: src/resources/beta/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Assistant,\n  AssistantDeleted,\n  AssistantStreamEvent,\n  AssistantTool,\n  CodeInterpreterTool,\n  FunctionTool,\n  MessageStreamEvent,\n  RetrievalTool,\n  RunStepStreamEvent,\n  RunStreamEvent,\n  ThreadStreamEvent,\n  AssistantCreateParams,\n  AssistantUpdateParams,\n  AssistantListParams,\n  AssistantsPage,\n  Assistants,\n} from './assistants/index';\nexport { Beta } from './beta';\nexport { Chat } from './chat/index';\nexport {\n  Thread,\n  ThreadDeleted,\n  ThreadCreateParams,\n  ThreadUpdateParams,\n  ThreadCreateAndRunParams,\n  ThreadCreateAndRunParamsNonStreaming,\n  ThreadCreateAndRunParamsStreaming,\n  ThreadCreateAndRunPollParams,\n  ThreadCreateAndRunStreamParams,\n  Threads,\n} from './threads/index';\n\n// Path: src/lib/ChatCompletionRunFunctions.test.ts\nimport OpenAI from 'openai';\nimport { OpenAIError, APIConnectionError } from 'openai/error';\nimport { PassThrough } from 'stream';\nimport {\n  ParsingToolFunction,\n  type ChatCompletionRunner,\n  type ChatCompletionFunctionRunnerParams,\n  ChatCompletionStreamingRunner,\n  type ChatCompletionStreamingFunctionRunnerParams,\n} from 'openai/resources/beta/chat/completions';\nimport type { ChatCompletionMessageParam } from 'openai/resources/chat/completions';\n\nimport { type RequestInfo, type RequestInit } from 'openai/_shims/index';\nimport { Response } from 'node-fetch';\nimport { isAssistantMessage } from './chatCompletionUtils';\n\ntype Fetch = (req: string | RequestInfo, init?: RequestInit) => Promise<Response>;\n\n/**\n * Creates a mock `fetch` function and a `handleRequest` function for intercepting `fetch` calls.\n *\n * You call `handleRequest` with a callback function that handles the next `fetch` call.\n * It returns a Promise that:\n * - waits for the next call to `fetch`\n * - calls the callback with the `fetch` arguments\n * - resolves `fetch` with the callback output\n */\n\nfunction mockFetch(): { fetch: Fetch; handleRequest: (handle: Fetch) => Promise<void> } {\n  const fetchQueue: ((handler: typeof fetch) => void)[] = [];\n  const handlerQueue: Promise<typeof fetch>[] = [];\n\n  const enqueueHandler = () => {\n    handlerQueue.push(\n      new Promise<typeof fetch>((resolve) => {\n        fetchQueue.push((handle: typeof fetch) => {\n          enqueueHandler();\n          resolve(handle);\n        });\n      }),\n    );\n  };\n  enqueueHandler();\n\n  async function fetch(req: string | RequestInfo, init?: RequestInit): Promise<Response> {\n    const handler = await handlerQueue.shift();\n    if (!handler) throw new Error('expected handler to be defined');\n    const signal = init?.signal;\n    if (!signal) return await handler(req, init);\n    return await Promise.race([\n      handler(req, init),\n      new Promise<Response>((resolve, reject) => {\n        if (signal.aborted) {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n          return;\n        }\n        signal.addEventListener('abort', (e) => {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n        });\n      }),\n    ]);\n  }\n\n  function handleRequest(handle: typeof fetch): Promise<void> {\n    return new Promise<void>((resolve, reject) => {\n      fetchQueue.shift()?.(async (req, init) => {\n        try {\n          return await handle(req, init);\n        } catch (err) {\n          reject(err);\n          return err as any;\n        } finally {\n          resolve();\n        }\n      });\n    });\n  }\n\n  return { fetch, handleRequest };\n}\n\n// mockChatCompletionFetch is like mockFetch, but with better a more convenient handleRequest to mock\n// chat completion request/responses.\nfunction mockChatCompletionFetch() {\n  const { fetch, handleRequest: handleRawRequest } = mockFetch();\n\n  function handleRequest(\n    handler: (body: ChatCompletionFunctionRunnerParams<any[]>) => Promise<OpenAI.Chat.ChatCompletion>,\n  ): Promise<void> {\n    return handleRawRequest(async (req, init) => {\n      const rawBody = init?.body;\n      if (typeof rawBody !== 'string') throw new Error(`expected init.body to be a string`);\n      const body: ChatCompletionFunctionRunnerParams<any[]> = JSON.parse(rawBody);\n      return new Response(JSON.stringify(await handler(body)), {\n        headers: { 'Content-Type': 'application/json' },\n      });\n    });\n  }\n  return { fetch, handleRequest };\n}\n\n// mockStreamingChatCompletionFetch is like mockFetch, but with better a more convenient handleRequest to mock\n// streaming chat completion request/responses.\nfunction mockStreamingChatCompletionFetch() {\n  const { fetch, handleRequest: handleRawRequest } = mockFetch();\n\n  function handleRequest(\n    handler: (\n      body: ChatCompletionStreamingFunctionRunnerParams<any[]>,\n    ) => AsyncIterable<OpenAI.Chat.ChatCompletionChunk>,\n  ): Promise<void> {\n    return handleRawRequest(async (req, init) => {\n      const rawBody = init?.body;\n      if (typeof rawBody !== 'string') throw new Error(`expected init.body to be a string`);\n      const body: ChatCompletionStreamingFunctionRunnerParams<any[]> = JSON.parse(rawBody);\n      const stream = new PassThrough();\n      (async () => {\n        for await (const chunk of handler(body)) {\n          stream.write(`data: ${JSON.stringify(chunk)}\\n\\n`);\n        }\n        stream.end(`data: [DONE]\\n\\n`);\n      })();\n      return new Response(stream, {\n        headers: {\n          'Content-Type': 'text/event-stream',\n          'Transfer-Encoding': 'chunked',\n        },\n      });\n    });\n  }\n  return { fetch, handleRequest };\n}\n\n// contentChoiceDeltas returns an async iterator which mocks a delta stream of a by splitting the\n// argument into chunks separated by whitespace.\nfunction* contentChoiceDeltas(\n  content: string,\n  {\n    index = 0,\n    role = 'assistant',\n  }: { index?: number; role?: NonNullable<OpenAI.Chat.ChatCompletionChunk.Choice.Delta['role']> } = {},\n): Iterable<OpenAI.Chat.Completions.ChatCompletionChunk.Choice> {\n  const deltas = content.split(/\\s+/g);\n  for (let i = 0; i < deltas.length; i++) {\n    yield {\n      index,\n      finish_reason: i === deltas.length - 1 ? 'stop' : null,\n      logprobs: null,\n      delta: {\n        role,\n        content: deltas[i] ? `${deltas[i]}${i === deltas.length - 1 ? '' : ' '}` : null,\n      },\n    };\n  }\n}\n\n// functionCallDeltas returns an async iterator which mocks a delta stream of a functionCall by splitting\n// the argument into chunks separated by whitespace.\nfunction* functionCallDeltas(\n  args: string,\n  {\n    index = 0,\n    id = '123',\n    name,\n    role = 'assistant',\n  }: {\n    name: string;\n    id?: string;\n    index?: number;\n    role?: NonNullable<OpenAI.Chat.ChatCompletionChunk.Choice.Delta['role']>;\n  },\n): Iterable<OpenAI.Chat.Completions.ChatCompletionChunk.Choice> {\n  const deltas = args.split(/\\s+/g);\n  for (let i = 0; i < deltas.length; i++) {\n    yield {\n      index,\n      finish_reason: i === deltas.length - 1 ? 'function_call' : null,\n      delta: {\n        role,\n        tool_calls: [\n          {\n            type: 'function',\n            index: 0,\n            id,\n            function: {\n              arguments: `${deltas[i] || ''}${i === deltas.length - 1 ? '' : ' '}`,\n              ...(i === deltas.length - 1 ? { name } : null),\n            },\n          },\n        ],\n      },\n    };\n  }\n}\n\nclass RunnerListener {\n  readonly contents: string[] = [];\n  readonly messages: ChatCompletionMessageParam[] = [];\n  readonly chatCompletions: OpenAI.Chat.ChatCompletion[] = [];\n  readonly functionCalls: OpenAI.Chat.ChatCompletionMessage.FunctionCall[] = [];\n  readonly functionCallResults: string[] = [];\n  finalContent: string | null = null;\n  finalMessage: ChatCompletionMessageParam | undefined;\n  finalChatCompletion: OpenAI.Chat.ChatCompletion | undefined;\n  finalFunctionCall: OpenAI.Chat.ChatCompletionMessage.FunctionCall | undefined;\n  finalFunctionCallResult: string | undefined;\n  totalUsage: OpenAI.CompletionUsage | undefined;\n  error: OpenAIError | undefined;\n  gotConnect = false;\n  gotAbort = false;\n  gotEnd = false;\n\n  onceMessageCallCount = 0;\n\n  constructor(public runner: ChatCompletionRunner) {\n    runner\n      .on('connect', () => (this.gotConnect = true))\n      .on('content', (content) => this.contents.push(content))\n      .on('message', (message) => this.messages.push(message))\n      .on('chatCompletion', (completion) => this.chatCompletions.push(completion))\n      .on('functionCall', (functionCall) => this.functionCalls.push(functionCall))\n      .on('functionCallResult', (result) => this.functionCallResults.push(result))\n      .on('finalContent', (content) => (this.finalContent = content))\n      .on('finalMessage', (message) => (this.finalMessage = message))\n      .on('finalChatCompletion', (completion) => (this.finalChatCompletion = completion))\n      .on('finalFunctionCall', (functionCall) => (this.finalFunctionCall = functionCall))\n      .on('finalFunctionCallResult', (result) => (this.finalFunctionCallResult = result))\n      .on('totalUsage', (usage) => (this.totalUsage = usage))\n      .on('error', (error) => (this.error = error))\n      .on('abort', (error) => ((this.error = error), (this.gotAbort = true)))\n      .on('end', () => (this.gotEnd = true))\n      .once('message', () => this.onceMessageCallCount++);\n  }\n\n  async sanityCheck({ error }: { error?: string } = {}) {\n    expect(this.onceMessageCallCount).toBeLessThanOrEqual(1);\n    expect(this.gotAbort).toEqual(this.runner.aborted);\n    if (this.runner.aborted) expect(this.runner.errored).toBe(true);\n    if (error) {\n      expect(this.error?.message).toEqual(error);\n      expect(this.runner.errored).toBe(true);\n      await expect(this.runner.finalChatCompletion()).rejects.toThrow(error);\n      await expect(this.runner.finalMessage()).rejects.toThrow(error);\n      await expect(this.runner.finalContent()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCall()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCallResult()).rejects.toThrow(error);\n      await expect(this.runner.totalUsage()).rejects.toThrow(error);\n      await expect(this.runner.done()).rejects.toThrow(error);\n    } else {\n      expect(this.error).toBeUndefined();\n      expect(this.runner.errored).toBe(false);\n    }\n\n    if (!this.gotConnect) {\n      expect(this.contents).toEqual([]);\n      expect(this.messages).toEqual([]);\n      expect(this.chatCompletions).toEqual([]);\n      expect(this.functionCalls).toEqual([]);\n      expect(this.functionCallResults).toEqual([]);\n      expect(this.finalContent).toBeUndefined();\n      expect(this.finalMessage).toBeUndefined();\n      expect(this.finalChatCompletion).toBeUndefined();\n      expect(this.finalFunctionCall).toBeUndefined();\n      expect(this.finalFunctionCallResult).toBeUndefined();\n      expect(this.totalUsage).toBeUndefined();\n      expect(this.gotEnd).toBe(true);\n      return;\n    }\n\n    if (error) return;\n\n    const expectedContents = this.messages\n      .filter(isAssistantMessage)\n      .map((m) => m.content as string)\n      .filter(Boolean);\n    expect(this.contents).toEqual(expectedContents);\n    expect(this.finalMessage).toEqual([...this.messages].reverse().find((x) => x.role === 'assistant'));\n    expect(await this.runner.finalMessage()).toEqual(this.finalMessage);\n    expect(this.finalContent).toEqual(expectedContents[expectedContents.length - 1] ?? null);\n    expect(await this.runner.finalContent()).toEqual(this.finalContent);\n    expect(this.finalChatCompletion).toEqual(this.chatCompletions[this.chatCompletions.length - 1]);\n    expect(await this.runner.finalChatCompletion()).toEqual(this.finalChatCompletion);\n    expect(this.finalFunctionCall).toEqual(this.functionCalls[this.functionCalls.length - 1]);\n    expect(await this.runner.finalFunctionCall()).toEqual(this.finalFunctionCall);\n    expect(this.finalFunctionCallResult).toEqual(\n      this.functionCallResults[this.functionCallResults.length - 1],\n    );\n    expect(await this.runner.finalFunctionCallResult()).toEqual(this.finalFunctionCallResult);\n    expect(this.chatCompletions).toEqual(this.runner.allChatCompletions());\n    expect(this.messages).toEqual(this.runner.messages.slice(-this.messages.length));\n    if (this.chatCompletions.some((c) => c.usage)) {\n      const totalUsage: OpenAI.CompletionUsage = {\n        completion_tokens: 0,\n        prompt_tokens: 0,\n        total_tokens: 0,\n      };\n      for (const { usage } of this.chatCompletions) {\n        if (usage) {\n          totalUsage.completion_tokens += usage.completion_tokens;\n          totalUsage.prompt_tokens += usage.prompt_tokens;\n          totalUsage.total_tokens += usage.total_tokens;\n        }\n      }\n      expect(this.totalUsage).toEqual(totalUsage);\n      expect(await this.runner.totalUsage()).toEqual(totalUsage);\n    }\n\n    expect(this.gotEnd).toBe(true);\n  }\n}\n\nclass StreamingRunnerListener {\n  readonly eventChunks: OpenAI.Chat.ChatCompletionChunk[] = [];\n  readonly eventContents: [string, string][] = [];\n  readonly eventMessages: ChatCompletionMessageParam[] = [];\n  readonly eventChatCompletions: OpenAI.Chat.ChatCompletion[] = [];\n  readonly eventFunctionCalls: OpenAI.Chat.ChatCompletionMessage.FunctionCall[] = [];\n  readonly eventFunctionCallResults: string[] = [];\n\n  finalContent: string | null = null;\n  finalMessage: ChatCompletionMessageParam | undefined;\n  finalChatCompletion: OpenAI.Chat.ChatCompletion | undefined;\n  finalFunctionCall: OpenAI.Chat.ChatCompletionMessage.FunctionCall | undefined;\n  finalFunctionCallResult: string | undefined;\n  error: OpenAIError | undefined;\n  gotConnect = false;\n  gotEnd = false;\n\n  constructor(public runner: ChatCompletionStreamingRunner) {\n    runner\n      .on('connect', () => (this.gotConnect = true))\n      .on('chunk', (chunk) => this.eventChunks.push(chunk))\n      .on('content', (delta, snapshot) => this.eventContents.push([delta, snapshot]))\n      .on('message', (message) => this.eventMessages.push(message))\n      .on('chatCompletion', (completion) => this.eventChatCompletions.push(completion))\n      .on('functionCall', (functionCall) => this.eventFunctionCalls.push(functionCall))\n      .on('functionCallResult', (result) => this.eventFunctionCallResults.push(result))\n      .on('finalContent', (content) => (this.finalContent = content))\n      .on('finalMessage', (message) => (this.finalMessage = message))\n      .on('finalChatCompletion', (completion) => (this.finalChatCompletion = completion))\n      .on('finalFunctionCall', (functionCall) => (this.finalFunctionCall = functionCall))\n      .on('finalFunctionCallResult', (result) => (this.finalFunctionCallResult = result))\n      .on('error', (error) => (this.error = error))\n      .on('abort', (abort) => (this.error = abort))\n      .on('end', () => (this.gotEnd = true));\n  }\n\n  async sanityCheck({ error }: { error?: string } = {}) {\n    if (error) {\n      expect(this.error?.message).toEqual(error);\n      expect(this.runner.errored).toBe(true);\n      await expect(this.runner.finalChatCompletion()).rejects.toThrow(error);\n      await expect(this.runner.finalMessage()).rejects.toThrow(error);\n      await expect(this.runner.finalContent()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCall()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCallResult()).rejects.toThrow(error);\n      await expect(this.runner.done()).rejects.toThrow(error);\n    } else {\n      expect(this.error).toBeUndefined();\n      expect(this.runner.errored).toBe(false);\n    }\n\n    if (!this.gotConnect) {\n      expect(this.eventContents).toEqual([]);\n      expect(this.eventMessages).toEqual([]);\n      expect(this.eventChatCompletions).toEqual([]);\n      expect(this.eventFunctionCalls).toEqual([]);\n      expect(this.eventFunctionCallResults).toEqual([]);\n      expect(this.finalContent).toBeUndefined();\n      expect(this.finalMessage).toBeUndefined();\n      expect(this.finalChatCompletion).toBeUndefined();\n      expect(this.finalFunctionCall).toBeUndefined();\n      expect(this.finalFunctionCallResult).toBeUndefined();\n      expect(this.gotEnd).toBe(true);\n      return;\n    }\n\n    if (error) return;\n\n    if (this.eventContents.length) expect(this.eventChunks.length).toBeGreaterThan(0);\n    expect(this.finalMessage).toEqual([...this.eventMessages].reverse().find((x) => x.role === 'assistant'));\n    expect(await this.runner.finalMessage()).toEqual(this.finalMessage);\n    expect(this.finalContent).toEqual(this.eventContents[this.eventContents.length - 1]?.[1] ?? null);\n    expect(await this.runner.finalContent()).toEqual(this.finalContent);\n    expect(this.finalChatCompletion).toEqual(this.eventChatCompletions[this.eventChatCompletions.length - 1]);\n    expect(await this.runner.finalChatCompletion()).toEqual(this.finalChatCompletion);\n    expect(this.finalFunctionCall).toEqual(this.eventFunctionCalls[this.eventFunctionCalls.length - 1]);\n    expect(await this.runner.finalFunctionCall()).toEqual(this.finalFunctionCall);\n    expect(this.finalFunctionCallResult).toEqual(\n      this.eventFunctionCallResults[this.eventFunctionCallResults.length - 1],\n    );\n    expect(await this.runner.finalFunctionCallResult()).toEqual(this.finalFunctionCallResult);\n    expect(this.eventChatCompletions).toEqual(this.runner.allChatCompletions());\n    expect(this.eventMessages).toEqual(this.runner.messages.slice(-this.eventMessages.length));\n    if (error) {\n      expect(this.error?.message).toEqual(error);\n      expect(this.runner.errored).toBe(true);\n    } else {\n      expect(this.error).toBeUndefined();\n      expect(this.runner.errored).toBe(false);\n    }\n    expect(this.gotEnd).toBe(true);\n  }\n}\n\nfunction _typeTests() {\n  const openai = new OpenAI();\n\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    tools: [\n      {\n        type: 'function',\n        function: {\n          name: 'numProperties',\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          parse: (str: string): object => {\n            const result = JSON.parse(str);\n            if (!(result instanceof Object) || Array.isArray(result)) {\n              throw new Error('must be an object');\n            }\n            return result;\n          },\n          description: 'gets the number of properties on an object',\n        },\n      },\n      {\n        type: 'function',\n        function: {\n          function: (str: string) => String(str.length),\n          parameters: { type: 'string' },\n          description: 'gets the length of a string',\n        },\n      },\n      {\n        type: 'function',\n        // @ts-expect-error function must accept string if parse is omitted\n        function: {\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n    ],\n  });\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    tools: [\n      new ParsingToolFunction({\n        name: 'numProperties',\n        // @ts-expect-error parse and function don't match\n        parse: (str: string) => str,\n        function: (obj: object) => String(Object.keys(obj).length),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n    ],\n  });\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    tools: [\n      new ParsingToolFunction({\n        name: 'numProperties',\n        parse: (str: string): object => {\n          const result = JSON.parse(str);\n          if (!(result instanceof Object) || Array.isArray(result)) {\n            throw new Error('must be an object');\n          }\n          return result;\n        },\n        function: (obj: object) => String(Object.keys(obj).length),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n      new ParsingToolFunction({\n        name: 'keys',\n        parse: (str: string): object => {\n          const result = JSON.parse(str);\n          if (!(result instanceof Object)) {\n            throw new Error('must be an Object');\n          }\n          return result;\n        },\n        function: (obj: object) => Object.keys(obj).join(', '),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n      new ParsingToolFunction({\n        name: 'len2',\n        // @ts-expect-error parse and function don't match\n        parse: (str: string) => str,\n        function: (obj: object) => String(Object.keys(obj).length),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n    ],\n  });\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    // @ts-ignore error occurs here in TS 4\n    tools: [\n      {\n        type: 'function',\n        function: {\n          name: 'numProperties',\n          parse: (str: string): object => {\n            const result = JSON.parse(str);\n            if (!(result instanceof Object) || Array.isArray(result)) {\n              throw new Error('must be an object');\n            }\n            return result;\n          },\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n      {\n        type: 'function',\n        function: {\n          name: 'keys',\n          parse: (str: string): object => {\n            const result = JSON.parse(str);\n            if (!(result instanceof Object)) {\n              throw new Error('must be an Object');\n            }\n            return result;\n          },\n          function: (obj: object) => Object.keys(obj).join(', '),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n      {\n        type: 'function',\n        function: {\n          name: 'len2',\n          parse: (str: string) => str,\n          // @ts-ignore error occurs here in TS 5\n          // function input doesn't match parse output\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n    ] as const,\n  });\n}\n\ndescribe('resource completions', () => {\n  describe('runTools with stream: false', () => {\n    test('successful flow', async () => {\n      const { fetch, handleRequest } = mockChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const runner = openai.beta.chat.completions.runTools({\n        messages: [{ role: 'user', content: 'tell me what the weather is like' }],\n        model: 'gpt-3.5-turbo',\n        tools: [\n          {\n            type: 'function',\n            function: {\n              function: function getWeather() {\n                return `it's raining`;\n              },\n              parameters: {},\n              description: 'gets the weather',\n            },\n          },\n        ],\n      });\n      const listener = new RunnerListener(runner);\n\n      await handleRequest(async (request) => {\n        expect(request.messages).toEqual([{ role: 'user', content: 'tell me what the weather is like' }]);\n        return {\n          id: '1',\n          choices: [\n            {\n              index: 0,\n              finish_reason: 'function_call',\n              logprobs: null,\n              message: {\n                role: 'assistant',\n                content: null,\n                tool_calls: [\n                  {\n                    type: 'function',\n                    id: '123',\n                    function: {\n                      arguments: '',\n                      name: 'getWeather',\n                    },\n                  },\n                ],\n              },\n            },\n          ],\n          created: Math.floor(Date.now() / 1000),\n          model: 'gpt-3.5-turbo',\n          object: 'chat.completion',\n        };\n      });\n\n      await handleRequest(async (request) => {\n        expect(request.messages).toEqual([\n          { role: 'user', content: 'tell me what the weather is like' },\n          {\n            role: 'assistant',\n            content: null,\n            tool_calls: [\n              {\n                type: 'function',\n                id: '123',\n                function: {\n                  arguments: '',\n                  name: 'getWeather',\n                },\n              },\n            ],\n          },\n          {\n            role: 'tool',\n            content: `it's raining`,\n            tool_call_id: '123',\n          },\n        ]);\n\n        return {\n          id: '2',\n          choices: [\n            {\n              index: 0,\n              finish_reason: 'stop',\n              logprobs: null,\n              message: {\n                role: 'assistant',\n                content: `it's raining`,\n              },\n            },\n          ],\n          created: Math.floor(Date.now() / 1000),\n          model: 'gpt-3.5-turbo',\n          object: 'chat.completion',\n        };\n      });\n\n      await runner.done();\n\n      expect(listener.messages).toEqual([\n        { role: 'user', content: 'tell me what the weather is like' },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [\n            {\n              type: 'function',\n              id: '123',\n              function: {\n                arguments: '',\n                name: 'getWeather',\n              },\n            },\n          ],\n        },\n        { role: 'tool', content: `it's raining`, tool_call_id: '123' },\n        { role: 'assistant', content: \"it's raining\" },\n      ]);\n      expect(listener.functionCallResults).toEqual([`it's raining`]);\n      await listener.sanityCheck();\n    });\n    test('flow with abort', async () => {\n      const { fetch, handleRequest } = mockChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const controller = new AbortController();\n      const runner = openai.beta.chat.completions.runTools(\n        {\n          messages: [{ role: 'user', content: 'tell me what the weather is like' }],\n          model: 'gpt-3.5-turbo',\n          tools: [\n            {\n              type: 'function',\n              function: {\n                function: function getWeather() {\n                  return `it's raining`;\n                },\n                parameters: {},\n                description: 'gets the weather',\n              },\n            },\n          ],\n        },\n        { signal: controller.signal },\n      );\n      const listener = new RunnerListener(runner);\n\n      await handleRequest(async (request) => {\n        expect(request.messages).toEqual([{ role: 'user', content: 'tell me what the weather is like' }]);\n        return {\n          id: '1',\n          choices: [\n            {\n              index: 0,\n              finish_reason: 'function_call',\n              logprobs: null,\n              message: {\n                role: 'assistant',\n                content: null,\n                tool_calls: [\n                  {\n                    type: 'function',\n                    id: '123',\n                    function: {\n                      arguments: '',\n                      name: 'getWeather',\n                    },\n                  },\n                ],\n              },\n            },\n          ],\n          created: Math.floor(Date.now() / 1000),\n          model: 'gpt-3.5-turbo',\n          object: 'chat.completion',\n        };\n      });\n\n      controller.abort();\n\n      await runner.done().catch(() => {});\n\n      expect(listener.messages).toEqual([\n        { role: 'user', content: 'tell me what the weather is like' },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [\n            {\n              type: 'function',\n              id: '123',\n              function: {\n                arguments: '',\n                name: 'getWeather',\n              },\n            },\n          ],\n        },\n        { role: 'tool', content: `it's raining`, tool_call_id: '123' },\n      ]);\n      expect(listener.functionCallResults).toEqual([`it's raining`]);\n      await listener.sanityCheck({ error: 'Request was aborted.' });\n      expect(runner.aborted).toBe(true);\n    });\n    test('successful flow with parse', async () => {\n      const { fetch, handleRequest } = mockChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const runner = openai.beta.chat.completions.runTools({\n        messages: [\n          {\n            role: 'user',\n            content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n          },\n        ],\n        model: 'gpt-3.5-turbo',\n        tools: [\n          new ParsingToolFunction({\n            name: 'numProperties',\n            function: (obj: object) => String(Object.keys(obj).length),\n            parameters: { type: 'object' },\n            parse: (str: string): object => {\n              const result = JSON.parse(str);\n              if (!(result instanceof Object) || Array.isArray(result)) {\n                throw new Error('must be an object');\n              }\n              return result;\n            },\n            description: 'gets the number of properties on an object',\n          }),\n        ],\n      });\n      const listener = new RunnerListener(runner);\n\n      await handleRequest(async (request) => {\n        expect(request.messages).toEqual([\n          {\n            role: 'user',\n            content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n          },\n        ]);\n        return {\n          id: '1',\n          choices: [\n            {\n              index: 0,\n              finish_reason: 'function_call',\n              logprobs: null,\n              message: {\n                role: 'assistant',\n                content: null,\n                tool_calls: [\n                  {\n                    type: 'function',\n                    id: '123',\n                    function: {\n                      arguments: '{\"a\": 1, \"b\": 2, \"c\": 3}',\n                      name: 'numProperties',\n                    },\n                  },\n                ],\n              },\n            },\n          ],\n          created: Math.floor(Date.now() / 1000),\n          model: 'gpt-3.5-turbo',\n          object: 'chat.completion',\n          usage: {\n            completion_tokens: 5,\n            prompt_tokens: 20,\n            total_tokens: 25,\n          },\n        };\n      });\n\n      await handleRequest(async (request) => {\n        expect(request.messages).toEqual([\n          {\n            role: 'user',\n            content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n          },\n          {\n            role: 'assistant',\n            content: null,\n            tool_calls: [\n              {\n                type: 'function',\n                id: '123',\n                function: {\n                  arguments: '{\"a\": 1, \"b\": 2, \"c\": 3}',\n                  name: 'numProperties',\n                },\n              },\n            ],\n          },\n          {\n            role: 'tool',\n            content: '3',\n            tool_call_id: '123',\n          },\n        ]);\n        return {\n          id: '2',\n          choices: [\n            {\n              index: 0,\n              finish_reason: 'stop',\n              logprobs: null,\n              message: {\n                role: 'assistant',\n                content: `there are 3 properties in {\"a\": 1, \"b\": 2, \"c\": 3}`,\n              },\n            },\n          ],\n          created: Math.floor(Date.now() / 1000),\n          model: 'gpt-3.5-turbo',\n          object: 'chat.completion',\n          usage: {\n            completion_tokens: 10,\n            prompt_tokens: 25,\n            total_tokens: 35,\n          },\n        };\n      });\n\n      await runner.done();\n\n      expect(listener.messages).toEqual([\n        {\n          role: 'user',\n          content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n        },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [\n            {\n              type: 'function',\n              id: '123',\n              function: { name: 'numProperties', arguments: '{\"a\": 1, \"b\": 2, \"c\": 3}' },\n            },\n          ],\n        },\n        { role: 'tool', content: '3', tool_call_id: '123' },\n        { role: 'assistant', content: 'there are 3 properties in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n      ]);\n      expect(listener.functionCallResults).toEqual(['3']);\n      await listener.sanityCheck();\n    });\n    test('flow with parse error', async () => {\n      const { fetch, handleRequest } = mockChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const runner = openai.beta.chat.completions.runTools({\n        messages: [\n          {\n            role: 'user',\n            content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n          },\n        ],\n        model: 'gpt-3.5-turbo',\n        tools: [\n          new ParsingToolFunction({\n            name: 'numProperties',\n            function: (obj: object) => String(Object.keys(obj).length),\n            parameters: { type: 'object' },\n            parse: (str: string): object => {\n              const result = JSON.parse(str);\n              if (!(result instanceof Object) || Array.isArray(result)) {\n                throw new Error('must be an object');\n              }\n              return result;\n            },\n            description: 'gets the number of properties on an object',\n          }),\n        ],\n      });\n      const listener = new RunnerListener(runner);\n\n      await Promise.all([\n        handleRequest(async (request) => {\n          expect(request.messages).toEqual([\n            {\n              role: 'user',\n              content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n            },\n          ]);\n          return {\n            id: '1',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'function_call',\n                logprobs: null,\n                message: {\n                  role: 'assistant',\n                  content: null,\n                  tool_calls: [\n                    {\n                      type: 'function',\n                      id: '123',\n                      function: {\n                        arguments: '[{\"a\": 1, \"b\": 2, \"c\": 3}]',\n                        name: 'numProperties',\n                      },\n                    },\n                  ],\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion',\n          };\n        }),\n        handleRequest(async (request) => {\n          expect(request.messages).toEqual([\n            {\n              role: 'user',\n              content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n            },\n            {\n              role: 'assistant',\n              content: null,\n              tool_calls: [\n                {\n                  type: 'function',\n                  id: '123',\n                  function: {\n                    arguments: '[{\"a\": 1, \"b\": 2, \"c\": 3}]',\n                    name: 'numProperties',\n                  },\n                },\n              ],\n            },\n            {\n              role: 'tool',\n              content: `must be an object`,\n              tool_call_id: '123',\n            },\n          ]);\n          return {\n            id: '2',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'function_call',\n                logprobs: null,\n                message: {\n                  role: 'assistant',\n                  content: null,\n                  tool_calls: [\n                    {\n                      type: 'function',\n                      id: '1234',\n                      function: {\n                        arguments: '{\"a\": 1, \"b\": 2, \"c\": 3}',\n                        name: 'numProperties',\n                      },\n                    },\n                  ],\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion',\n          };\n        }),\n        handleRequest(async (request) => {\n          expect(request.messages).toEqual([\n            {\n              role: 'user',\n              content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n            },\n            {\n              role: 'assistant',\n              content: null,\n              tool_calls: [\n                {\n                  type: 'function',\n                  id: '123',\n                  function: {\n                    arguments: '[{\"a\": 1, \"b\": 2, \"c\": 3}]',\n                    name: 'numProperties',\n                  },\n                },\n              ],\n            },\n            {\n              role: 'tool',\n              content: `must be an object`,\n              tool_call_id: '123',\n            },\n            {\n              role: 'assistant',\n              content: null,\n              tool_calls: [\n                {\n                  type: 'function',\n                  id: '1234',\n                  function: {\n                    arguments: '{\"a\": 1, \"b\": 2, \"c\": 3}',\n                    name: 'numProperties',\n                  },\n                },\n              ],\n            },\n            {\n              role: 'tool',\n              content: '3',\n              tool_call_id: '1234',\n            },\n          ]);\n          return {\n            id: '3',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'stop',\n                logprobs: null,\n                message: {\n                  role: 'assistant',\n                  content: `there are 3 properties in {\"a\": 1, \"b\": 2, \"c\": 3}`,\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion',\n          };\n        }),\n        runner.done(),\n      ]);\n\n      expect(listener.messages).toEqual([\n        {\n          role: 'user',\n          content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}',\n        },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [\n            {\n              type: 'function',\n              id: '123',\n              function: { name: 'numProperties', arguments: '[{\"a\": 1, \"b\": 2, \"c\": 3}]' },\n            },\n          ],\n        },\n        { role: 'tool', content: `must be an object`, tool_call_id: '123' },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [\n            {\n              type: 'function',\n              id: '1234',\n              function: { name: 'numProperties', arguments: '{\"a\": 1, \"b\": 2, \"c\": 3}' },\n            },\n          ],\n        },\n        { role: 'tool', content: '3', tool_call_id: '1234' },\n        { role: 'assistant', content: 'there are 3 properties in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n      ]);\n      expect(listener.functionCallResults).toEqual([`must be an object`, '3']);\n      await listener.sanityCheck();\n    });\n    test('single function call', async () => {\n      const { fetch, handleRequest } = mockChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const runner = openai.beta.chat.completions.runTools({\n        messages: [{ role: 'user', content: 'tell me what the weather is like' }],\n        model: 'gpt-3.5-turbo',\n        tool_choice: {\n          type: 'function',\n          function: {\n            name: 'getWeather',\n          },\n        },\n        tools: [\n          {\n            type: 'function',\n            function: {\n              function: function getWeather() {\n                return `it's raining`;\n              },\n              parameters: {},\n              description: 'gets the weather',\n            },\n          },\n        ],\n      });\n      const listener = new RunnerListener(runner);\n\n      await Promise.all([\n        handleRequest(async (request) => {\n          expect(request.messages).toEqual([{ role: 'user', content: 'tell me what the weather is like' }]);\n          return {\n            id: '1',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'function_call',\n                logprobs: null,\n                message: {\n                  role: 'assistant',\n                  content: null,\n                  tool_calls: [\n                    {\n                      type: 'function',\n                      id: '123',\n                      function: {\n                        arguments: '',\n                        name: 'getWeather',\n                      },\n                    },\n                  ],\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion',\n          };\n        }),\n        runner.done(),\n      ]);\n\n      expect(listener.messages).toEqual([\n        { role: 'user', content: 'tell me what the weather is like' },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [\n            {\n              type: 'function',\n              id: '123',\n              function: {\n                arguments: '',\n                name: 'getWeather',\n              },\n            },\n          ],\n        },\n        { role: 'tool', content: `it's raining`, tool_call_id: '123' },\n      ]);\n      expect(listener.functionCallResults).toEqual([`it's raining`]);\n      await listener.sanityCheck();\n    });\n    test('wrong function name', async () => {\n      const { fetch, handleRequest } = mockChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const runner = openai.beta.chat.completions.runTools({\n        messages: [{ role: 'user', content: 'tell me what the weather is like' }],\n        model: 'gpt-3.5-turbo',\n        tools: [\n          {\n            type: 'function',\n            function: {\n              function: function getWeather() {\n                return `it's raining`;\n              },\n              parameters: {},\n              description: 'gets the weather',\n            },\n          },\n        ],\n      });\n      const listener = new RunnerListener(runner);\n\n      await Promise.all([\n        handleRequest(async (request) => {\n          expect(request.messages).toEqual([{ role: 'user', content: 'tell me what the weather is like' }]);\n          return {\n            id: '1',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'function_call',\n                logprobs: null,\n                message: {\n                  role: 'assistant',\n                  content: null,\n                  tool_calls: [\n                    {\n                      type: 'function',\n                      id: '123',\n                      function: {\n                        arguments: '',\n                        name: 'get_weather',\n                      },\n                    },\n                  ],\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion',\n          };\n        }),\n        handleRequest(async (request) => {\n          expect(request.messages).toEqual([\n            { role: 'user', content: 'tell me what the weather is like' },\n            {\n              role: 'assistant',\n              content: null,\n              tool_calls: [\n                {\n                  type: 'function',\n                  id: '123',\n                  function: {\n                    arguments: '',\n                    name: 'get_weather',\n                  },\n                },\n              ],\n            },\n            {\n              role: 'tool',\n              content: `Invalid tool_call: \"get_weather\". Available options are: \"getWeather\". Please try again`,\n              tool_call_id: '123',\n            },\n          ]);\n          return {\n            id: '2',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'function_call',\n                logprobs: null,\n                message: {\n                  role: 'assistant',\n                  content: null,\n                  tool_calls: [\n                    {\n                      type: 'function',\n                      id: '1234',\n                      function: {\n                        arguments: '',\n                        name: 'getWeather',\n                      },\n                    },\n                  ],\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion',\n          };\n        }),\n        handleRequest(async (request) => {\n          expect(request.messages).toEqual([\n            { role: 'user', content: 'tell me what the weather is like' },\n            {\n              role: 'assistant',\n              content: null,\n              tool_calls: [\n                {\n                  type: 'function',\n                  id: '123',\n                  function: {\n                    arguments: '',\n                    name: 'get_weather',\n                  },\n                },\n              ],\n            },\n            {\n              role: 'tool',\n              content: `Invalid tool_call: \"get_weather\". Available options are: \"getWeather\". Please try again`,\n              tool_call_id: '123',\n            },\n            {\n              role: 'assistant',\n              content: null,\n              tool_calls: [\n                {\n                  type: 'function',\n                  id: '1234',\n                  function: {\n                    arguments: '',\n                    name: 'getWeather',\n                  },\n                },\n              ],\n            },\n            {\n              role: 'tool',\n              content: `it's raining`,\n              tool_call_id: '1234',\n            },\n          ]);\n          return {\n            id: '3',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'stop',\n                logprobs: null,\n                message: {\n                  role: 'assistant',\n                  content: `it's raining`,\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion',\n          };\n        }),\n        runner.done(),\n      ]);\n\n      expect(listener.messages).toEqual([\n        { role: 'user', content: 'tell me what the weather is like' },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [{ type: 'function', id: '123', function: { name: 'get_weather', arguments: '' } }],\n        },\n        {\n          role: 'tool',\n          content: `Invalid tool_call: \"get_weather\". Available options are: \"getWeather\". Please try again`,\n          tool_call_id: '123',\n        },\n        {\n          role: 'assistant',\n          content: null,\n          tool_calls: [{ type: 'function', id: '1234', function: { name: 'getWeather', arguments: '' } }],\n        },\n        { role: 'tool', content: `it's raining`, tool_call_id: '1234' },\n        { role: 'assistant', content: \"it's raining\" },\n      ]);\n      expect(listener.functionCallResults).toEqual([\n        `Invalid tool_call: \"get_weather\". Available options are: \"getWeather\". Please try again`,\n        `it's raining`,\n      ]);\n      await listener.sanityCheck();\n    });\n  });\n\n  describe('runTools with stream: true', () => {\n    test('successful flow', async () => {\n      const { fetch, handleRequest } = mockStreamingChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const runner = openai.beta.chat.completions.runTools({\n        stream: true,\n        messages: [{ role: 'user', content: 'tell me what the weather is like' }],\n        model: 'gpt-3.5-turbo',\n        tools: [\n          {\n            type: 'function',\n            function: {\n              function: function getWeather() {\n                return `it's raining`;\n              },\n              parameters: {},\n              description: 'gets the weather',\n            },\n          },\n        ],\n      });\n      const listener = new StreamingRunnerListener(runner);\n\n      await Promise.all([\n        handleRequest(async function* (request): AsyncIterable<OpenAI.Chat.ChatCompletionChunk> {\n          expect(request.messages).toEqual([{ role: 'user', content: 'tell me what the weather is like' }]);\n          yield {\n            id: '1',\n            choices: [\n              {\n                index: 0,\n                finish_reason: 'function_call',\n                logprobs: null,\n                delta: {\n                  role: 'assistant',\n                  content: null,\n                  tool_calls: [\n                    {\n                      type: 'function',\n                      index: 0,\n                      id: '123',\n                      function: {\n                        arguments: '',\n                        name: 'getWeather',\n                      },\n                    },\n                  ],\n                },\n              },\n            ],\n            created: Math.floor(Date.now() / 1000),\n            model: 'gpt-3.5-turbo',\n            object: 'chat.completion.chunk',\n          };\n        }),\n        handleRequest(async function* (request): AsyncIterable<OpenAI.Chat.ChatCompletionChunk> {\n          expect(request.messages).toEqual([\n            { role: 'user', content: 'tell me what the weather is like' },\n            {\n              role: 'assistant',\n              content: null,\n              tool_calls: [\n                {\n                  type: 'function',\n                  id: '123',\n                  function: {\n                    arguments: '',\n                    name: 'getWeather',\n                  },\n                },\n              ],\n            },\n            {\n              role: 'tool',\n              content: `it's raining`,\n              tool_call_id: '123',\n            },\n          ]);\n          for (const choice of contentChoiceDeltas(`it's raining`)) {\n            yield {\n              id: '2',\n              choices: [choice],\n              created: Math.floor(Date.now() / 1000),\n              model: 'gpt-3.5-turbo',\n              object: 'chat.completion.chunk',\n...\n", "needle_token_start": 2190, "needle_token_end": 2654, "code_context_ntokens": 16290, "output": ["```typescript\nfunction mockFetch(): { fetch: Fetch; handleRequest: (handle: Fetch) => Promise<void> } {\n  const fetchQueue: ((handler: typeof fetch) => void)[] = [];\n  const handlerQueue: Promise<typeof fetch>[] = [];\n\n  const enqueueHandler = () => {\n    handlerQueue.push(\n      new Promise<typeof fetch>((resolve) => {\n        fetchQueue.push((handle: typeof fetch) => {\n          enqueueHandler();\n          resolve(handle);\n        });\n      }),\n    );\n  };\n  enqueueHandler();\n\n  async function fetch(req: string | RequestInfo, init?: RequestInit): Promise<Response> {\n    const handler = await handlerQueue.shift();\n    if (!handler) throw new Error('expected handler to be defined');\n    const signal = init?.signal;\n    if (!signal) return await handler(req, init);\n    return await Promise.race([\n      handler(req, init),\n      new Promise<Response>((resolve, reject) => {\n        if (signal.aborted) {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n          return;\n        }\n        signal.addEventListener('abort', (e) => {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n        });\n      }),\n    ]);\n  }\n\n  function handleRequest(handle: typeof fetch): Promise<void> {\n    return new Promise<void>((resolve, reject) => {\n      fetchQueue.shift()?.(async (req, init) => {\n        try {\n          return await handle(req, init);\n        } catch (err) {\n          reject(err);\n          return err as any;\n        } finally {\n          resolve();\n        }\n      });\n    });\n  }\n\n  return { fetch, handleRequest };\n}\n```"]}
{"repo": "openai/openai-node", "name": "applyHeadersMut", "language": "typescript", "path": "src/core.ts", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to modify a set of HTTP headers by incorporating changes from another set of headers, handling case sensitivity and removing or ignoring certain entries based on specific conditions.\n2. **Input**: The function takes two sets of HTTP headers. The first set is the target that will be modified, and the second set contains new values that may update or delete entries in the target.\n3. **Output**: There is no return value; the function directly modifies the first input set of headers.\n4. **Procedure**: The function iterates over each entry in the new headers. For each entry, it checks if the key exists in the new headers set. It then converts the key to lowercase to ensure case insensitivity. If the value associated with a key is `null`, the key is removed from the target headers. If the value is not `undefined`, the target headers are updated with the new value under the converted lowercase key.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": " *\n * It also implements AsyncIterable to allow auto-paginating iteration on an unawaited list call, eg:\n *\n *    for await (const item of client.items.list()) {\n *      console.log(item)\n *    }\n */\nexport class PagePromise<\n    PageClass extends AbstractPage<Item>,\n    Item = ReturnType<PageClass['getPaginatedItems']>[number],\n  >\n  extends APIPromise<PageClass>\n  implements AsyncIterable<Item>\n{\n  constructor(\n    client: APIClient,\n    request: Promise<APIResponseProps>,\n    Page: new (...args: ConstructorParameters<typeof AbstractPage>) => PageClass,\n  ) {\n    super(\n      request,\n      async (props) => new Page(client, props.response, await defaultParseResponse(props), props.options),\n    );\n  }\n\n  /**\n   * Allow auto-paginating iteration on an unawaited list call, eg:\n   *\n   *    for await (const item of client.items.list()) {\n   *      console.log(item)\n   *    }\n   */\n  async *[Symbol.asyncIterator]() {\n    const page = await this;\n    for await (const item of page) {\n      yield item;\n    }\n  }\n}\n\nexport const createResponseHeaders = (\n  headers: Awaited<ReturnType<Fetch>>['headers'],\n): Record<string, string> => {\n  return new Proxy(\n    Object.fromEntries(\n      // @ts-ignore\n      headers.entries(),\n    ),\n    {\n      get(target, name) {\n        const key = name.toString();\n        return target[key.toLowerCase()] || target[key];\n      },\n    },\n  );\n};\n\ntype HTTPMethod = 'get' | 'post' | 'put' | 'patch' | 'delete';\n\nexport type RequestClient = { fetch: Fetch };\nexport type Headers = Record<string, string | null | undefined>;\nexport type DefaultQuery = Record<string, string | undefined>;\nexport type KeysEnum<T> = { [P in keyof Required<T>]: true };\n\nexport type RequestOptions<Req = unknown | Record<string, unknown> | Readable> = {\n  method?: HTTPMethod;\n  path?: string;\n  query?: Req | undefined;\n  body?: Req | null | undefined;\n  headers?: Headers | undefined;\n\n  maxRetries?: number;\n  stream?: boolean | undefined;\n  timeout?: number;\n  httpAgent?: Agent;\n  signal?: AbortSignal | undefined | null;\n  idempotencyKey?: string;\n\n  __binaryResponse?: boolean | undefined;\n  __streamClass?: typeof Stream;\n};\n\n// This is required so that we can determine if a given object matches the RequestOptions\n// type at runtime. While this requires duplication, it is enforced by the TypeScript\n// compiler such that any missing / extraneous keys will cause an error.\nconst requestOptionsKeys: KeysEnum<RequestOptions> = {\n  method: true,\n  path: true,\n  query: true,\n  body: true,\n  headers: true,\n\n  maxRetries: true,\n  stream: true,\n  timeout: true,\n  httpAgent: true,\n  signal: true,\n  idempotencyKey: true,\n\n  __binaryResponse: true,\n  __streamClass: true,\n};\n\nexport const isRequestOptions = (obj: unknown): obj is RequestOptions => {\n  return (\n    typeof obj === 'object' &&\n    obj !== null &&\n    !isEmptyObj(obj) &&\n    Object.keys(obj).every((k) => hasOwn(requestOptionsKeys, k))\n  );\n};\n\nexport type FinalRequestOptions<Req = unknown | Record<string, unknown> | Readable> = RequestOptions<Req> & {\n  method: HTTPMethod;\n  path: string;\n};\n\ndeclare const Deno: any;\ndeclare const EdgeRuntime: any;\ntype Arch = 'x32' | 'x64' | 'arm' | 'arm64' | `other:${string}` | 'unknown';\ntype PlatformName =\n  | 'MacOS'\n  | 'Linux'\n  | 'Windows'\n  | 'FreeBSD'\n  | 'OpenBSD'\n  | 'iOS'\n  | 'Android'\n  | `Other:${string}`\n  | 'Unknown';\ntype Browser = 'ie' | 'edge' | 'chrome' | 'firefox' | 'safari';\ntype PlatformProperties = {\n  'X-Stainless-Lang': 'js';\n  'X-Stainless-Package-Version': string;\n  'X-Stainless-OS': PlatformName;\n  'X-Stainless-Arch': Arch;\n  'X-Stainless-Runtime': 'node' | 'deno' | 'edge' | `browser:${Browser}` | 'unknown';\n  'X-Stainless-Runtime-Version': string;\n};\nconst getPlatformProperties = (): PlatformProperties => {\n  if (typeof Deno !== 'undefined' && Deno.build != null) {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': normalizePlatform(Deno.build.os),\n      'X-Stainless-Arch': normalizeArch(Deno.build.arch),\n      'X-Stainless-Runtime': 'deno',\n      'X-Stainless-Runtime-Version':\n        typeof Deno.version === 'string' ? Deno.version : Deno.version?.deno ?? 'unknown',\n    };\n  }\n  if (typeof EdgeRuntime !== 'undefined') {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': 'Unknown',\n      'X-Stainless-Arch': `other:${EdgeRuntime}`,\n      'X-Stainless-Runtime': 'edge',\n      'X-Stainless-Runtime-Version': process.version,\n    };\n  }\n  // Check if Node.js\n  if (Object.prototype.toString.call(typeof process !== 'undefined' ? process : 0) === '[object process]') {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': normalizePlatform(process.platform),\n      'X-Stainless-Arch': normalizeArch(process.arch),\n      'X-Stainless-Runtime': 'node',\n      'X-Stainless-Runtime-Version': process.version,\n    };\n  }\n\n  const browserInfo = getBrowserInfo();\n  if (browserInfo) {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': 'Unknown',\n      'X-Stainless-Arch': 'unknown',\n      'X-Stainless-Runtime': `browser:${browserInfo.browser}`,\n      'X-Stainless-Runtime-Version': browserInfo.version,\n    };\n  }\n\n  // TODO add support for Cloudflare workers, etc.\n  return {\n    'X-Stainless-Lang': 'js',\n    'X-Stainless-Package-Version': VERSION,\n    'X-Stainless-OS': 'Unknown',\n    'X-Stainless-Arch': 'unknown',\n    'X-Stainless-Runtime': 'unknown',\n    'X-Stainless-Runtime-Version': 'unknown',\n  };\n};\n\ntype BrowserInfo = {\n  browser: Browser;\n  version: string;\n};\n\ndeclare const navigator: { userAgent: string } | undefined;\n\n// Note: modified from https://github.com/JS-DevTools/host-environment/blob/b1ab79ecde37db5d6e163c050e54fe7d287d7c92/src/isomorphic.browser.ts\nfunction getBrowserInfo(): BrowserInfo | null {\n  if (typeof navigator === 'undefined' || !navigator) {\n    return null;\n  }\n\n  // NOTE: The order matters here!\n  const browserPatterns = [\n    { key: 'edge' as const, pattern: /Edge(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /MSIE(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /Trident(?:.*rv\\:(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'chrome' as const, pattern: /Chrome(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'firefox' as const, pattern: /Firefox(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'safari' as const, pattern: /(?:Version\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?(?:\\W+Mobile\\S*)?\\W+Safari/ },\n  ];\n\n  // Find the FIRST matching browser\n  for (const { key, pattern } of browserPatterns) {\n    const match = pattern.exec(navigator.userAgent);\n    if (match) {\n      const major = match[1] || 0;\n      const minor = match[2] || 0;\n      const patch = match[3] || 0;\n\n      return { browser: key, version: `${major}.${minor}.${patch}` };\n    }\n  }\n\n  return null;\n}\n\nconst normalizeArch = (arch: string): Arch => {\n  // Node docs:\n  // - https://nodejs.org/api/process.html#processarch\n  // Deno docs:\n  // - https://doc.deno.land/deno/stable/~/Deno.build\n  if (arch === 'x32') return 'x32';\n  if (arch === 'x86_64' || arch === 'x64') return 'x64';\n  if (arch === 'arm') return 'arm';\n  if (arch === 'aarch64' || arch === 'arm64') return 'arm64';\n  if (arch) return `other:${arch}`;\n  return 'unknown';\n};\n\nconst normalizePlatform = (platform: string): PlatformName => {\n  // Node platforms:\n  // - https://nodejs.org/api/process.html#processplatform\n  // Deno platforms:\n  // - https://doc.deno.land/deno/stable/~/Deno.build\n  // - https://github.com/denoland/deno/issues/14799\n\n  platform = platform.toLowerCase();\n\n  // NOTE: this iOS check is untested and may not work\n  // Node does not work natively on IOS, there is a fork at\n  // https://github.com/nodejs-mobile/nodejs-mobile\n  // however it is unknown at the time of writing how to detect if it is running\n  if (platform.includes('ios')) return 'iOS';\n  if (platform === 'android') return 'Android';\n  if (platform === 'darwin') return 'MacOS';\n  if (platform === 'win32') return 'Windows';\n  if (platform === 'freebsd') return 'FreeBSD';\n  if (platform === 'openbsd') return 'OpenBSD';\n  if (platform === 'linux') return 'Linux';\n  if (platform) return `Other:${platform}`;\n  return 'Unknown';\n};\n\nlet _platformHeaders: PlatformProperties;\nconst getPlatformHeaders = () => {\n  return (_platformHeaders ??= getPlatformProperties());\n};\n\nexport const safeJSON = (text: string) => {\n  try {\n    return JSON.parse(text);\n  } catch (err) {\n    return undefined;\n  }\n};\n\n// https://stackoverflow.com/a/19709846\nconst startsWithSchemeRegexp = new RegExp('^(?:[a-z]+:)?//', 'i');\nconst isAbsoluteURL = (url: string): boolean => {\n  return startsWithSchemeRegexp.test(url);\n};\n\nexport const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));\n\nconst validatePositiveInteger = (name: string, n: unknown): number => {\n  if (typeof n !== 'number' || !Number.isInteger(n)) {\n    throw new OpenAIError(`${name} must be an integer`);\n  }\n  if (n < 0) {\n    throw new OpenAIError(`${name} must be a positive integer`);\n  }\n  return n;\n};\n\nexport const castToError = (err: any): Error => {\n  if (err instanceof Error) return err;\n  return new Error(err);\n};\n\nexport const ensurePresent = <T>(value: T | null | undefined): T => {\n  if (value == null) throw new OpenAIError(`Expected a value to be given but received ${value} instead.`);\n  return value;\n};\n\n/**\n * Read an environment variable.\n *\n * Trims beginning and trailing whitespace.\n *\n * Will return undefined if the environment variable doesn't exist or cannot be accessed.\n */\nexport const readEnv = (env: string): string | undefined => {\n  if (typeof process !== 'undefined') {\n    return process.env?.[env]?.trim() ?? undefined;\n  }\n  if (typeof Deno !== 'undefined') {\n    return Deno.env?.get?.(env)?.trim();\n  }\n  return undefined;\n};\n\nexport const coerceInteger = (value: unknown): number => {\n  if (typeof value === 'number') return Math.round(value);\n  if (typeof value === 'string') return parseInt(value, 10);\n\n  throw new OpenAIError(`Could not coerce ${value} (type: ${typeof value}) into a number`);\n};\n\nexport const coerceFloat = (value: unknown): number => {\n  if (typeof value === 'number') return value;\n  if (typeof value === 'string') return parseFloat(value);\n\n  throw new OpenAIError(`Could not coerce ${value} (type: ${typeof value}) into a number`);\n};\n\nexport const coerceBoolean = (value: unknown): boolean => {\n  if (typeof value === 'boolean') return value;\n  if (typeof value === 'string') return value === 'true';\n  return Boolean(value);\n};\n\nexport const maybeCoerceInteger = (value: unknown): number | undefined => {\n  if (value === undefined) {\n    return undefined;\n  }\n  return coerceInteger(value);\n};\n\nexport const maybeCoerceFloat = (value: unknown): number | undefined => {\n  if (value === undefined) {\n    return undefined;\n  }\n  return coerceFloat(value);\n};\n\nexport const maybeCoerceBoolean = (value: unknown): boolean | undefined => {\n  if (value === undefined) {\n    return undefined;\n  }\n  return coerceBoolean(value);\n};\n\n// https://stackoverflow.com/a/34491287\nexport function isEmptyObj(obj: Object | null | undefined): boolean {\n  if (!obj) return true;\n  for (const _k in obj) return false;\n  return true;\n}\n\n// https://eslint.org/docs/latest/rules/no-prototype-builtins\nexport function hasOwn(obj: Object, key: string): boolean {\n  return Object.prototype.hasOwnProperty.call(obj, key);\n}\n\n/**\n * Copies headers from \"newHeaders\" onto \"targetHeaders\",\n * using lower-case for all properties,\n * ignoring any keys with undefined values,\n * and deleting any keys with null values.\n */\nfunction a\npplyHeadersMut(targetHeaders: Headers, newHeaders: Headers): void {\n  for (const k in newHeaders) {\n    if (!hasOwn(newHeaders, k)) continue;\n    const lowerKey = k.toLowerCase();\n    if (!lowerKey) continue;\n\n    const val = newHeaders[k];\n\n    if (val === null) {\n      delete targetHeaders[lowerKey];\n    } else if (val !== undefined) {\n      targetHeaders[lowerKey] = val;\n    }\n  }\n}\n\nexport function debug(action: string, ...args: any[]) {\n  if (typeof process !== 'undefined' && process?.env?.['DEBUG'] === 'true') {\n    console.log(`OpenAI:DEBUG:${action}`, ...args);\n  }\n}\n\n/**\n * https://stackoverflow.com/a/2117523\n */\nconst uuid4 = () => {\n  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {\n    const r = (Math.random() * 16) | 0;\n    const v = c === 'x' ? r : (r & 0x3) | 0x8;\n    return v.toString(16);\n  });\n};\n\nexport const isRunningInBrowser = () => {\n  return (\n    // @ts-ignore\n    typeof window !== 'undefined' &&\n    // @ts-ignore\n    typeof window.document !== 'undefined' &&\n    // @ts-ignore\n    typeof navigator !== 'undefined'\n  );\n};\n\nexport interface HeadersProtocol {\n  get: (header: string) => string | null | undefined;\n}\nexport type HeadersLike = Record<string, string | string[] | undefined> | HeadersProtocol;\n\nexport const isHeadersProtocol = (headers: any): headers is HeadersProtocol => {\n  return typeof headers?.get === 'function';\n};\n\nexport const getRequiredHeader = (headers: HeadersLike, header: string): string => {\n  const lowerCasedHeader = header.toLowerCase();\n  if (isHeadersProtocol(headers)) {\n    // to deal with the case where the header looks like Stainless-Event-Id\n    const intercapsHeader =\n      header[0]?.toUpperCase() +\n      header.substring(1).replace(/([^\\w])(\\w)/g, (_m, g1, g2) => g1 + g2.toUpperCase());\n    for (const key of [header, lowerCasedHeader, header.toUpperCase(), intercapsHeader]) {\n      const value = headers.get(key);\n      if (value) {\n        return value;\n      }\n    }\n  }\n\n  for (const [key, value] of Object.entries(headers)) {\n    if (key.toLowerCase() === lowerCasedHeader) {\n      if (Array.isArray(value)) {\n        if (value.length <= 1) return value[0];\n        console.warn(`Received ${value.length} entries for the ${header} header, using the first entry.`);\n        return value[0];\n      }\n      return value;\n    }\n  }\n\n  throw new Error(`Could not find ${header} header`);\n};\n\n/**\n * Encodes a string to Base64 format.\n */\nexport const toBase64 = (str: string | null | undefined): string => {\n  if (!str) return '';\n  if (typeof Buffer !== 'undefined') {\n    return Buffer.from(str).toString('base64');\n  }\n\n  if (typeof btoa !== 'undefined') {\n    return btoa(str);\n  }\n\n  throw new OpenAIError('Cannot generate b64 string; Expected `Buffer` or `btoa` to be defined');\n};\n\nexport function isObj(obj: unknown): obj is Record<string, unknown> {\n  return obj != null && typeof obj === 'object' && !Array.isArray(obj);\n}\n\n// Path: src/_shims/registry.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { type RequestOptions } from '../core';\n\nexport interface Shims {\n  kind: string;\n  fetch: any;\n  Request: any;\n  Response: any;\n  Headers: any;\n  FormData: any;\n  Blob: any;\n  File: any;\n  ReadableStream: any;\n  getMultipartRequestOptions: <T = Record<string, unknown>>(\n    form: Shims['FormData'],\n    opts: RequestOptions<T>,\n  ) => Promise<RequestOptions<T>>;\n  getDefaultAgent: (url: string) => any;\n  fileFromPath:\n    | ((path: string, filename?: string, options?: {}) => Promise<Shims['File']>)\n    | ((path: string, options?: {}) => Promise<Shims['File']>);\n  isFsReadStream: (value: any) => boolean;\n}\n\nexport let auto = false;\nexport let kind: Shims['kind'] | undefined = undefined;\nexport let fetch: Shims['fetch'] | undefined = undefined;\nexport let Request: Shims['Request'] | undefined = undefined;\nexport let Response: Shims['Response'] | undefined = undefined;\nexport let Headers: Shims['Headers'] | undefined = undefined;\nexport let FormData: Shims['FormData'] | undefined = undefined;\nexport let Blob: Shims['Blob'] | undefined = undefined;\nexport let File: Shims['File'] | undefined = undefined;\nexport let ReadableStream: Shims['ReadableStream'] | undefined = undefined;\nexport let getMultipartRequestOptions: Shims['getMultipartRequestOptions'] | undefined = undefined;\nexport let getDefaultAgent: Shims['getDefaultAgent'] | undefined = undefined;\nexport let fileFromPath: Shims['fileFromPath'] | undefined = undefined;\nexport let isFsReadStream: Shims['isFsReadStream'] | undefined = undefined;\n\nexport function setShims(shims: Shims, options: { auto: boolean } = { auto: false }) {\n  if (auto) {\n    throw new Error(\n      `you must \\`import 'openai/shims/${shims.kind}'\\` before importing anything else from openai`,\n    );\n  }\n  if (kind) {\n    throw new Error(`can't \\`import 'openai/shims/${shims.kind}'\\` after \\`import 'openai/shims/${kind}'\\``);\n  }\n  auto = options.auto;\n  kind = shims.kind;\n  fetch = shims.fetch;\n  Request = shims.Request;\n  Response = shims.Response;\n  Headers = shims.Headers;\n  FormData = shims.FormData;\n  Blob = shims.Blob;\n  File = shims.File;\n  ReadableStream = shims.ReadableStream;\n  getMultipartRequestOptions = shims.getMultipartRequestOptions;\n  getDefaultAgent = shims.getDefaultAgent;\n  fileFromPath = shims.fileFromPath;\n  isFsReadStream = shims.isFsReadStream;\n}\n\n// Path: src/_shims/web-runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { MultipartBody } from './MultipartBody';\nimport { type RequestOptions } from '../core';\nimport { type Shims } from './registry';\n\nexport function getRuntime({ manuallyImported }: { manuallyImported?: boolean } = {}): Shims {\n  const recommendation =\n    manuallyImported ?\n      `You may need to use polyfills`\n    : `Add one of these imports before your first \\`import \u2026 from 'openai'\\`:\n- \\`import 'openai/shims/node'\\` (if you're running on Node)\n- \\`import 'openai/shims/web'\\` (otherwise)\n`;\n\n  let _fetch, _Request, _Response, _Headers;\n  try {\n    // @ts-ignore\n    _fetch = fetch;\n    // @ts-ignore\n    _Request = Request;\n    // @ts-ignore\n    _Response = Response;\n    // @ts-ignore\n    _Headers = Headers;\n  } catch (error) {\n    throw new Error(\n      `this environment is missing the following Web Fetch API type: ${\n        (error as any).message\n      }. ${recommendation}`,\n    );\n  }\n\n  return {\n    kind: 'web',\n    fetch: _fetch,\n    Request: _Request,\n    Response: _Response,\n    Headers: _Headers,\n    FormData:\n      // @ts-ignore\n      typeof FormData !== 'undefined' ? FormData : (\n        class FormData {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'FormData' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    Blob:\n      typeof Blob !== 'undefined' ? Blob : (\n        class Blob {\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'Blob' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    File:\n      // @ts-ignore\n      typeof File !== 'undefined' ? File : (\n        class File {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'File' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    ReadableStream:\n      // @ts-ignore\n      typeof ReadableStream !== 'undefined' ? ReadableStream : (\n        class ReadableStream {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `streaming isn't supported in this environment yet as 'ReadableStream' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    getMultipartRequestOptions: async <T = Record<string, unknown>>(\n      // @ts-ignore\n      form: FormData,\n      opts: RequestOptions<T>,\n    ): Promise<RequestOptions<T>> => ({\n      ...opts,\n      body: new MultipartBody(form) as any,\n    }),\n    getDefaultAgent: (url: string) => undefined,\n    fileFromPath: () => {\n      throw new Error(\n        'The `fileFromPath` function is only supported in Node. See the README for more details: https://www.github.com/openai/openai-node#file-uploads',\n      );\n    },\n    isFsReadStream: (value: any) => false,\n  };\n}\n\n// Path: src/_shims/auto/runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../web-runtime';\n\n// Path: src/_shims/index.js\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nconst shims = require('./registry');\nconst auto = require('openai/_shims/auto/runtime');\nif (!shims.kind) shims.setShims(auto.getRuntime(), { auto: true });\nfor (const property of Object.keys(shims)) {\n  Object.defineProperty(exports, property, {\n    get() {\n      return shims[property];\n    },\n  });\n}\n\n// Path: src/streaming.ts\nimport { ReadableStream, type Response } from './_shims/index';\nimport { OpenAIError } from './error';\n\nimport { APIError } from 'openai/error';\n\ntype Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;\n\nexport type ServerSentEvent = {\n  event: string | null;\n  data: string;\n  raw: string[];\n};\n\nexport class Stream<Item> implements AsyncIterable<Item> {\n  controller: AbortController;\n\n  constructor(\n    private iterator: () => AsyncIterator<Item>,\n    controller: AbortController,\n  ) {\n    this.controller = controller;\n  }\n\n  static fromSSEResponse<Item>(response: Response, controller: AbortController) {\n    let consumed = false;\n\n    async function* iterator(): AsyncIterator<Item, any, undefined> {\n      if (consumed) {\n        throw new Error('Cannot iterate over a consumed stream, use `.tee()` to split the stream.');\n      }\n      consumed = true;\n      let done = false;\n      try {\n        for await (const sse of _iterSSEMessages(response, controller)) {\n          if (done) continue;\n\n          if (sse.data.startsWith('[DONE]')) {\n            done = true;\n            continue;\n          }\n\n          if (sse.event === null) {\n            let data;\n\n            try {\n              data = JSON.parse(sse.data);\n            } catch (e) {\n              console.error(`Could not parse message into JSON:`, sse.data);\n              console.error(`From chunk:`, sse.raw);\n              throw e;\n            }\n\n            if (data && data.error) {\n              throw new APIError(undefined, data.error, undefined, undefined);\n            }\n\n            yield data;\n          } else {\n            let data;\n            try {\n              data = JSON.parse(sse.data);\n            } catch (e) {\n              console.error(`Could not parse message into JSON:`, sse.data);\n              console.error(`From chunk:`, sse.raw);\n              throw e;\n            }\n            // TODO: Is this where the error should be thrown?\n            if (sse.event == 'error') {\n              throw new APIError(undefined, data.error, data.message, undefined);\n            }\n            yield { event: sse.event, data: data } as any;\n          }\n        }\n        done = true;\n      } catch (e) {\n        // If the user calls `stream.controller.abort()`, we should exit without throwing.\n        if (e instanceof Error && e.name === 'AbortError') return;\n        throw e;\n      } finally {\n        // If the user `break`s, abort the ongoing request.\n        if (!done) controller.abort();\n      }\n    }\n\n    return new Stream(iterator, controller);\n  }\n\n  /**\n   * Generates a Stream from a newline-separated ReadableStream\n   * where each item is a JSON value.\n   */\n  static fromReadableStream<Item>(readableStream: ReadableStream, controller: AbortController) {\n    let consumed = false;\n\n    async function* iterLines(): AsyncGenerator<string, void, unknown> {\n      const lineDecoder = new LineDecoder();\n\n      const iter = readableStreamAsyncIterable<Bytes>(readableStream);\n      for await (const chunk of iter) {\n        for (const line of lineDecoder.decode(chunk)) {\n          yield line;\n        }\n      }\n\n      for (const line of lineDecoder.flush()) {\n        yield line;\n      }\n    }\n\n    async function* iterator(): AsyncIterator<Item, any, undefined> {\n      if (consumed) {\n        throw new Error('Cannot iterate over a consumed stream, use `.tee()` to split the stream.');\n      }\n      consumed = true;\n      let done = false;\n      try {\n        for await (const line of iterLines()) {\n          if (done) continue;\n          if (line) yield JSON.parse(line);\n        }\n        done = true;\n      } catch (e) {\n        // If the user calls `stream.controller.abort()`, we should exit without throwing.\n        if (e instanceof Error && e.name === 'AbortError') return;\n        throw e;\n      } finally {\n        // If the user `break`s, abort the ongoing request.\n        if (!done) controller.abort();\n      }\n    }\n\n    return new Stream(iterator, controller);\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<Item> {\n    return this.iterator();\n  }\n\n  /**\n   * Splits the stream into two streams which can be\n   * independently read from at different speeds.\n   */\n  tee(): [Stream<Item>, Stream<Item>] {\n    const left: Array<Promise<IteratorResult<Item>>> = [];\n    const right: Array<Promise<IteratorResult<Item>>> = [];\n    const iterator = this.iterator();\n\n    const teeIterator = (queue: Array<Promise<IteratorResult<Item>>>): AsyncIterator<Item> => {\n      return {\n        next: () => {\n          if (queue.length === 0) {\n            const result = iterator.next();\n            left.push(result);\n            right.push(result);\n          }\n          return queue.shift()!;\n        },\n      };\n    };\n\n    return [\n      new Stream(() => teeIterator(left), this.controller),\n      new Stream(() => teeIterator(right), this.controller),\n    ];\n  }\n\n  /**\n   * Converts this stream to a newline-separated ReadableStream of\n   * JSON stringified values in the stream\n   * which can be turned back into a Stream with `Stream.fromReadableStream()`.\n   */\n  toReadableStream(): ReadableStream {\n    const self = this;\n    let iter: AsyncIterator<Item>;\n    const encoder = new TextEncoder();\n\n    return new ReadableStream({\n      async start() {\n        iter = self[Symbol.asyncIterator]();\n      },\n      async pull(ctrl: any) {\n        try {\n          const { value, done } = await iter.next();\n          if (done) return ctrl.close();\n\n          const bytes = encoder.encode(JSON.stringify(value) + '\\n');\n\n          ctrl.enqueue(bytes);\n        } catch (err) {\n          ctrl.error(err);\n        }\n      },\n      async cancel() {\n        await iter.return?.();\n      },\n    });\n  }\n}\n\nexport async function* _iterSSEMessages(\n  response: Response,\n  controller: AbortController,\n): AsyncGenerator<ServerSentEvent, void, unknown> {\n  if (!response.body) {\n    controller.abort();\n    throw new OpenAIError(`Attempted to iterate over a response with no body`);\n  }\n\n  const sseDecoder = new SSEDecoder();\n  const lineDecoder = new LineDecoder();\n\n  const iter = readableStreamAsyncIterable<Bytes>(response.body);\n  for await (const sseChunk of iterSSEChunks(iter)) {\n    for (const line of lineDecoder.decode(sseChunk)) {\n      const sse = sseDecoder.decode(line);\n      if (sse) yield sse;\n    }\n  }\n\n  for (const line of lineDecoder.flush()) {\n    const sse = sseDecoder.decode(line);\n    if (sse) yield sse;\n  }\n}\n\n/**\n * Given an async iterable iterator, iterates over it and yields full\n * SSE chunks, i.e. yields when a double new-line is encountered.\n */\nasync function* iterSSEChunks(iterator: AsyncIterableIterator<Bytes>): AsyncGenerator<Uint8Array> {\n  let data = new Uint8Array();\n\n  for await (const chunk of iterator) {\n    if (chunk == null) {\n      continue;\n    }\n\n    const binaryChunk =\n      chunk instanceof ArrayBuffer ? new Uint8Array(chunk)\n      : typeof chunk === 'string' ? new TextEncoder().encode(chunk)\n      : chunk;\n\n    let newData = new Uint8Array(data.length + binaryChunk.length);\n    newData.set(data);\n    newData.set(binaryChunk, data.length);\n    data = newData;\n\n    let patternIndex;\n    while ((patternIndex = findDoubleNewlineIndex(data)) !== -1) {\n      yield data.slice(0, patternIndex);\n      data = data.slice(patternIndex);\n    }\n  }\n\n  if (data.length > 0) {\n    yield data;\n  }\n}\n\nfunction findDoubleNewlineIndex(buffer: Uint8Array): number {\n  // This function searches the buffer for the end patterns (\\r\\r, \\n\\n, \\r\\n\\r\\n)\n  // and returns the index right after the first occurrence of any pattern,\n  // or -1 if none of the patterns are found.\n  const newline = 0x0a; // \\n\n  const carriage = 0x0d; // \\r\n\n  for (let i = 0; i < buffer.length - 2; i++) {\n    if (buffer[i] === newline && buffer[i + 1] === newline) {\n      // \\n\\n\n      return i + 2;\n    }\n    if (buffer[i] === carriage && buffer[i + 1] === carriage) {\n      // \\r\\r\n      return i + 2;\n    }\n    if (\n      buffer[i] === carriage &&\n      buffer[i + 1] === newline &&\n      i + 3 < buffer.length &&\n      buffer[i + 2] === carriage &&\n      buffer[i + 3] === newline\n    ) {\n      // \\r\\n\\r\\n\n      return i + 4;\n    }\n  }\n\n  return -1;\n}\n\nclass SSEDecoder {\n  private data: string[];\n  private event: string | null;\n  private chunks: string[];\n\n  constructor() {\n    this.event = null;\n    this.data = [];\n    this.chunks = [];\n  }\n\n  decode(line: string) {\n    if (line.endsWith('\\r')) {\n      line = line.substring(0, line.length - 1);\n    }\n\n    if (!line) {\n      // empty line and we didn't previously encounter any messages\n      if (!this.event && !this.data.length) return null;\n\n      const sse: ServerSentEvent = {\n        event: this.event,\n        data: this.data.join('\\n'),\n        raw: this.chunks,\n      };\n\n      this.event = null;\n      this.data = [];\n      this.chunks = [];\n\n      return sse;\n    }\n\n    this.chunks.push(line);\n\n    if (line.startsWith(':')) {\n      return null;\n    }\n\n    let [fieldname, _, value] = partition(line, ':');\n\n    if (value.startsWith(' ')) {\n      value = value.substring(1);\n    }\n\n    if (fieldname === 'event') {\n      this.event = value;\n    } else if (fieldname === 'data') {\n      this.data.push(value);\n    }\n\n    return null;\n  }\n}\n\n/**\n * A re-implementation of httpx's `LineDecoder` in Python that handles incrementally\n * reading lines from text.\n *\n * https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258\n */\nclass LineDecoder {\n  // prettier-ignore\n  static NEWLINE_CHARS = new Set(['\\n', '\\r']);\n  static NEWLINE_REGEXP = /\\r\\n|[\\n\\r]/g;\n\n  buffer: string[];\n  trailingCR: boolean;\n  textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either \"dom\" or \"node\" types.\n\n  constructor() {\n    this.buffer = [];\n    this.trailingCR = false;\n  }\n\n  decode(chunk: Bytes): string[] {\n    let text = this.decodeText(chunk);\n\n    if (this.trailingCR) {\n      text = '\\r' + text;\n      this.trailingCR = false;\n    }\n    if (text.endsWith('\\r')) {\n      this.trailingCR = true;\n      text = text.slice(0, -1);\n    }\n\n    if (!text) {\n      return [];\n    }\n\n    const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');\n    let lines = text.split(LineDecoder.NEWLINE_REGEXP);\n\n    // if there is a trailing new line then the last entry will be an empty\n    // string which we don't care about\n    if (trailingNewline) {\n      lines.pop();\n    }\n\n    if (lines.length === 1 && !trailingNewline) {\n      this.buffer.push(lines[0]!);\n      return [];\n    }\n\n    if (this.buffer.length > 0) {\n      lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];\n      this.buffer = [];\n    }\n\n    if (!trailingNewline) {\n      this.buffer = [lines.pop() || ''];\n    }\n\n    return lines;\n  }\n\n  decodeText(bytes: Bytes): string {\n    if (bytes == null) return '';\n    if (typeof bytes === 'string') return bytes;\n\n    // Node:\n    if (typeof Buffer !== 'undefined') {\n      if (bytes instanceof Buffer) {\n        return bytes.toString();\n      }\n      if (bytes instanceof Uint8Array) {\n        return Buffer.from(bytes).toString();\n      }\n\n      throw new OpenAIError(\n        `Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global \"Buffer\" defined, which this library assumes to be Node. Please report this error.`,\n      );\n    }\n\n    // Browser\n    if (typeof TextDecoder !== 'undefined') {\n      if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {\n        this.textDecoder ??= new TextDecoder('utf8');\n        return this.textDecoder.decode(bytes);\n      }\n\n      throw new OpenAIError(\n        `Unexpected: received non-Uint8Array/ArrayBuffer (${\n          (bytes as any).constructor.name\n        }) in a web platform. Please report this error.`,\n      );\n    }\n\n    throw new OpenAIError(\n      `Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`,\n    );\n  }\n\n  flush(): string[] {\n    if (!this.buffer.length && !this.trailingCR) {\n      return [];\n    }\n\n    const lines = [this.buffer.join('')];\n    this.buffer = [];\n    this.trailingCR = false;\n    return lines;\n  }\n}\n\n/** This is an internal helper function that's just used for testing */\nexport function _decodeChunks(chunks: string[]): string[] {\n  const decoder = new LineDecoder();\n  const lines: string[] = [];\n  for (const chunk of chunks) {\n    lines.push(...decoder.decode(chunk));\n  }\n\n  return lines;\n}\n\nfunction partition(str: string, delimiter: string): [string, string, string] {\n  const index = str.indexOf(delimiter);\n  if (index !== -1) {\n    return [str.substring(0, index), delimiter, str.substring(index + delimiter.length)];\n  }\n\n  return [str, '', ''];\n}\n\n/**\n * Most browsers don't yet have async iterable support for ReadableStream,\n * and Node has a very different way of reading bytes from its \"ReadableStream\".\n *\n * This polyfill was pulled from https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490\n */\nexport function readableStreamAsyncIterable<T>(stream: any): AsyncIterableIterator<T> {\n  if (stream[Symbol.asyncIterator]) return stream;\n\n  const reader = stream.getReader();\n  return {\n    async next() {\n      try {\n        const result = await reader.read();\n        if (result?.done) reader.releaseLock(); // release lock when stream becomes closed\n        return result;\n      } catch (e) {\n        reader.releaseLock(); // release lock when stream becomes errored\n        throw e;\n      }\n    },\n    async return() {\n      const cancelPromise = reader.cancel();\n      reader.releaseLock();\n      await cancelPromise;\n      return { done: true, value: undefined };\n    },\n    [Symbol.asyncIterator]() {\n      return this;\n    },\n  };\n}\n\n// Path: src/pagination.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { AbstractPage, Response, APIClient, FinalRequestOptions, PageInfo } from './core';\n\nexport interface PageResponse<Item> {\n  data: Array<Item>;\n\n  object: string;\n}\n\n/**\n * Note: no pagination actually occurs yet, this is for forwards-compatibility.\n */\nexport class Page<Item> extends AbstractPage<Item> implements PageResponse<Item> {\n  data: Array<Item>;\n\n  object: string;\n\n  constructor(client: APIClient, response: Response, body: PageResponse<Item>, options: FinalRequestOptions) {\n    super(client, response, body, options);\n\n    this.data = body.data || [];\n    this.object = body.object;\n  }\n\n  getPaginatedItems(): Item[] {\n    return this.data ?? [];\n  }\n\n  // @deprecated Please use `nextPageInfo()` instead\n  /**\n   * This page represents a response that isn't actually paginated at the API level\n   * so there will never be any next page params.\n   */\n  nextPageParams(): null {\n    return null;\n  }\n\n  nextPageInfo(): null {\n    return null;\n  }\n}\n\nexport interface CursorPageResponse<Item> {\n  data: Array<Item>;\n}\n\nexport interface CursorPageParams {\n  after?: string;\n\n  limit?: number;\n}\n\nexport class CursorPage<Item extends { id: string }>\n  extends AbstractPage<Item>\n  implements CursorPageResponse<Item>\n{\n  data: Array<Item>;\n\n  constructor(\n    client: APIClient,\n    response: Response,\n    body: CursorPageResponse<Item>,\n    options: FinalRequestOptions,\n  ) {\n    super(client, response, body, options);\n\n    this.data = body.data || [];\n  }\n\n  getPaginatedItems(): Item[] {\n    return this.data ?? [];\n  }\n\n  // @deprecated Please use `nextPageInfo()` instead\n  nextPageParams(): Partial<CursorPageParams> | null {\n    const info = this.nextPageInfo();\n    if (!info) return null;\n    if ('params' in info) return info.params;\n    const params = Object.fromEntries(info.url.searchParams);\n    if (!Object.keys(params).length) return null;\n    return params;\n  }\n\n  nextPageInfo(): PageInfo | null {\n    const data = this.getPaginatedItems();\n    if (!data.length) {\n      return null;\n    }\n\n    const id = data[data.length - 1]?.id;\n    if (!id) {\n      return null;\n    }\n\n    return { params: { after: id } };\n  }\n}\n\n// Path: src/resources/files.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { isRequestOptions } from 'openai/core';\nimport { type Response } from 'openai/_shims/index';\nimport { sleep } from 'openai/core';\nimport { APIConnectionTimeoutError } from 'openai/error';\nimport * as FilesAPI from 'openai/resources/files';\nimport { type Uploadable, multipartFormRequestOptions } from 'openai/core';\nimport { Page } from 'openai/pagination';\n\nexport class Files extends APIResource {\n  /**\n   * Upload a file that can be used across various endpoints. The size of all the\n   * files uploaded by one organization can be up to 100 GB.\n   *\n   * The size of individual files can be a maximum of 512 MB or 2 million tokens for\n   * Assistants. See the\n   * [Assistants Tools guide](https://platform.openai.com/docs/assistants/tools) to\n   * learn more about the types of files supported. The Fine-tuning API only supports\n   * `.jsonl` files.\n   *\n   * Please [contact us](https://help.openai.com/) if you need to increase these\n   * storage limits.\n   */\n  create(body: FileCreateParams, options?: Core.RequestOptions): Core.APIPromise<FileObject> {\n    return this._client.post('/files', multipartFormRequestOptions({ body, ...options }));\n  }\n\n  /**\n   * Returns information about a specific file.\n   */\n  retrieve(fileId: string, options?: Core.RequestOptions): Core.APIPromise<FileObject> {\n    return this._client.get(`/files/${fileId}`, options);\n  }\n\n  /**\n   * Returns a list of files that belong to the user's organization.\n   */\n  list(query?: FileListParams, options?: Core.RequestOptions): Core.PagePromise<FileObjectsPage, FileObject>;\n  list(options?: Core.RequestOptions): Core.PagePromise<FileObjectsPage, FileObject>;\n  list(\n    query: FileListParams | Core.RequestOptions = {},\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<FileObjectsPage, FileObject> {\n    if (isRequestOptions(query)) {\n      return this.list({}, query);\n    }\n    return this._client.getAPIList('/files', FileObjectsPage, { query, ...options });\n  }\n\n  /**\n   * Delete a file.\n   */\n  del(fileId: string, options?: Core.RequestOptions): Core.APIPromise<FileDeleted> {\n    return this._client.delete(`/files/${fileId}`, options);\n  }\n\n  /**\n   * Returns the contents of the specified file.\n   */\n  content(fileId: string, options?: Core.RequestOptions): Core.APIPromise<Response> {\n    return this._client.get(`/files/${fileId}/content`, { ...options, __binaryResponse: true });\n  }\n\n  /**\n   * Returns the contents of the specified file.\n   *\n   * @deprecated The `.content()` method should be used instead\n   */\n  retrieveContent(fileId: string, options?: Core.RequestOptions): Core.APIPromise<string> {\n    return this._client.get(`/files/${fileId}/content`, {\n      ...options,\n      headers: { Accept: 'application/json', ...options?.headers },\n    });\n  }\n\n  /**\n   * Waits for the given file to be processed, default timeout is 30 mins.\n   */\n  async waitForProcessing(\n    id: string,\n    { pollInterval = 5000, maxWait = 30 * 60 * 1000 }: { pollInterval?: number; maxWait?: number } = {},\n  ): Promise<FileObject> {\n    const TERMINAL_STATES = new Set(['processed', 'error', 'deleted']);\n\n    const start = Date.now();\n    let file = await this.retrieve(id);\n\n    while (!file.status || !TERMINAL_STATES.has(file.status)) {\n      await sleep(pollInterval);\n\n      file = await this.retrieve(id);\n      if (Date.now() - start > maxWait) {\n        throw new APIConnectionTimeoutError({\n          message: `Giving up on waiting for file ${id} to finish processing after ${maxWait} milliseconds.`,\n        });\n      }\n    }\n\n    return file;\n  }\n}\n\n/**\n * Note: no pagination actually occurs yet, this is for forwards-compatibility.\n */\nexport class FileObjectsPage extends Page<FileObject> {}\n\nexport type FileContent = string;\n\nexport interface FileDeleted {\n  id: string;\n\n  deleted: boolean;\n\n  object: 'file';\n}\n\n/**\n * The `File` object represents a document that has been uploaded to OpenAI.\n */\nexport interface FileObject {\n  /**\n   * The file identifier, which can be referenced in the API endpoints.\n   */\n  id: string;\n\n  /**\n   * The size of the file, in bytes.\n   */\n  bytes: number;\n\n  /**\n   * The Unix timestamp (in seconds) for when the file was created.\n   */\n  created_at: number;\n\n  /**\n   * The name of the file.\n   */\n  filename: string;\n\n  /**\n   * The object type, which is always `file`.\n   */\n  object: 'file';\n\n  /**\n   * The intended purpose of the file. Supported values are `fine-tune`,\n   * `fine-tune-results`, `assistants`, and `assistants_output`.\n   */\n  purpose: 'fine-tune' | 'fine-tune-results' | 'assistants' | 'assistants_output';\n\n  /**\n   * @deprecated: Deprecated. The current status of the file, which can be either\n   * `uploaded`, `processed`, or `error`.\n   */\n  status: 'uploaded' | 'processed' | 'error';\n\n  /**\n   * @deprecated: Deprecated. For details on why a fine-tuning training file failed\n   * validation, see the `error` field on `fine_tuning.job`.\n   */\n  status_details?: string;\n}\n\nexport interface FileCreateParams {\n  /**\n   * The File object (not file name) to be uploaded.\n   */\n  file: Uploadable;\n\n  /**\n   * The intended purpose of the uploaded file.\n   *\n   * Use \"fine-tune\" for\n   * [Fine-tuning](https://platform.openai.com/docs/api-reference/fine-tuning) and\n   * \"assistants\" for\n   * [Assistants](https://platform.openai.com/docs/api-reference/assistants) and\n   * [Messages](https://platform.openai.com/docs/api-reference/messages). This allows\n   * us to validate the format of the uploaded file is correct for fine-tuning.\n   */\n  purpose: 'fine-tune' | 'assistants';\n}\n\nexport interface FileListParams {\n  /**\n   * Only return files with the given purpose.\n   */\n  purpose?: string;\n}\n\nexport namespace Files {\n  export import FileContent = FilesAPI.FileContent;\n  export import FileDeleted = FilesAPI.FileDeleted;\n  export import FileObject = FilesAPI.FileObject;\n  export import FileObjectsPage = FilesAPI.FileObjectsPage;\n  export import FileCreateParams = FilesAPI.FileCreateParams;\n  export import FileListParams = FilesAPI.FileListParams;\n}\n\n// Path: src/resources/audio/transcriptions.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport * as TranscriptionsAPI from 'openai/resources/audio/transcriptions';\nimport { type Uploadable, multipartFormRequestOptions } from 'openai/core';\n\nexport class Transcriptions extends APIResource {\n  /**\n   * Transcribes audio into the input language.\n   */\n  create(body: TranscriptionCreateParams, options?: Core.RequestOptions): Core.APIPromise<Transcription> {\n    return this._client.post('/audio/transcriptions', multipartFormRequestOptions({ body, ...options }));\n  }\n}\n\n/**\n * Represents a transcription response returned by model, based on the provided\n * input.\n */\nexport interface Transcription {\n  /**\n   * The transcribed text.\n   */\n  text: string;\n}\n\nexport interface TranscriptionCreateParams {\n  /**\n   * The audio file object (not file name) to transcribe, in one of these formats:\n   * flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.\n   */\n  file: Uploadable;\n\n  /**\n   * ID of the model to use. Only `whisper-1` (which is powered by our open source\n   * Whisper V2 model) is currently available.\n   */\n  model: (string & {}) | 'whisper-1';\n\n  /**\n   * The language of the input audio. Supplying the input language in\n   * [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will\n   * improve accuracy and latency.\n   */\n  language?: string;\n\n  /**\n   * An optional text to guide the model's style or continue a previous audio\n   * segment. The\n   * [prompt](https://platform.openai.com/docs/guides/speech-to-text/prompting)\n   * should match the audio language.\n   */\n  prompt?: string;\n\n  /**\n   * The format of the transcript output, in one of these options: `json`, `text`,\n   * `srt`, `verbose_json`, or `vtt`.\n   */\n  response_format?: 'json' | 'text' | 'srt' | 'verbose_json' | 'vtt';\n\n  /**\n   * The sampling temperature, between 0 and 1. Higher values like 0.8 will make the\n   * output more random, while lower values like 0.2 will make it more focused and\n   * deterministic. If set to 0, the model will use\n   * [log probability](https://en.wikipedia.org/wiki/Log_probability) to\n   * automatically increase the temperature until certain thresholds are hit.\n   */\n  temperature?: number;\n\n  /**\n   * The timestamp granularities to populate for this transcription.\n   * `response_format` must be set `verbose_json` to use timestamp granularities.\n   * Either or both of these options are supported: `word`, or `segment`. Note: There\n   * is no additional latency for segment timestamps, but generating word timestamps\n   * incurs additional latency.\n   */\n  timestamp_granularities?: Array<'word' | 'segment'>;\n}\n\nexport namespace Transcriptions {\n  export import Transcription = TranscriptionsAPI.Transcription;\n  export import TranscriptionCreateParams = TranscriptionsAPI.TranscriptionCreateParams;\n}\n\n// Path: src/resources/audio/speech.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { type Response } from 'openai/_shims/index';\nimport * as SpeechAPI from 'openai/resources/audio/speech';\n\nexport class Speech extends APIResource {\n  /**\n   * Generates audio from the input text.\n   */\n  create(body: SpeechCreateParams, options?: Core.RequestOptions): Core.APIPromise<Response> {\n    return this._client.post('/audio/speech', { body, ...options, __binaryResponse: true });\n  }\n}\n\nexport interface SpeechCreateParams {\n  /**\n   * The text to generate audio for. The maximum length is 4096 characters.\n   */\n  input: string;\n\n  /**\n   * One of the available [TTS models](https://platform.openai.com/docs/models/tts):\n   * `tts-1` or `tts-1-hd`\n   */\n  model: (string & {}) | 'tts-1' | 'tts-1-hd';\n\n  /**\n   * The voice to use when generating the audio. Supported voices are `alloy`,\n   * `echo`, `fable`, `onyx`, `nova`, and `shimmer`. Previews of the voices are\n   * available in the\n   * [Text to speech guide](https://platform.openai.com/docs/guides/text-to-speech/voice-options).\n   */\n  voice: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer';\n\n  /**\n   * The format to audio in. Supported formats are `mp3`, `opus`, `aac`, `flac`,\n   * `wav`, and `pcm`.\n   */\n  response_format?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm';\n\n  /**\n   * The speed of the generated audio. Select a value from `0.25` to `4.0`. `1.0` is\n   * the default.\n   */\n  speed?: number;\n}\n\nexport namespace Speech {\n  export import SpeechCreateParams = SpeechAPI.SpeechCreateParams;\n}\n\n// Path: src/resources/audio/translations.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport * as TranslationsAPI from 'openai/resources/audio/translations';\nimport { type Uploadable, multipartFormRequestOptions } from 'openai/core';\n\nexport class Translations extends APIResource {\n  /**\n   * Translates audio into English.\n   */\n  create(body: TranslationCreateParams, options?: Core.RequestOptions): Core.APIPromise<Translation> {\n    return this._client.post('/audio/translations', multipartFormRequestOptions({ body, ...options }));\n  }\n}\n\nexport interface Translation {\n  text: string;\n}\n\nexport interface TranslationCreateParams {\n  /**\n   * The audio file object (not file name) translate, in one of these formats: flac,\n   * mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.\n   */\n  file: Uploadable;\n\n  /**\n   * ID of the model to use. Only `whisper-1` (which is powered by our open source\n   * Whisper V2 model) is currently available.\n   */\n  model: (string & {}) | 'whisper-1';\n\n  /**\n   * An optional text to guide the model's style or continue a previous audio\n   * segment. The\n   * [prompt](https://platform.openai.com/docs/guides/speech-to-text/prompting)\n   * should be in English.\n   */\n  prompt?: string;\n\n  /**\n   * The format of the transcript output, in one of these options: `json`, `text`,\n   * `srt`, `verbose_json`, or `vtt`.\n   */\n  response_format?: string;\n\n  /**\n   * The sampling temperature, between 0 and 1. Higher values like 0.8 will make the\n   * output more random, while lower values like 0.2 will make it more focused and\n   * deterministic. If set to 0, the model will use\n   * [log probability](https://en.wikipedia.org/wiki/Log_probability) to\n   * automatically increase the temperature until certain thresholds are hit.\n   */\n  temperature?: number;\n}\n\nexport namespace Translations {\n  export import Translation = TranslationsAPI.Translation;\n  export import TranslationCreateParams = TranslationsAPI.TranslationCreateParams;\n}\n\n// Path: src/resources/audio/audio.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { APIResource } from 'openai/resource';\nimport * as SpeechAPI from 'openai/resources/audio/speech';\nimport * as TranscriptionsAPI from 'openai/resources/audio/transcriptions';\nimport * as TranslationsAPI from 'openai/resources/audio/translations';\n\nexport class Audio extends APIResource {\n  transcriptions: TranscriptionsAPI.Transcriptions = new TranscriptionsAPI.Transcriptions(this._client);\n  translations: TranslationsAPI.Translations = new TranslationsAPI.Translations(this._client);\n  speech: SpeechAPI.Speech = new SpeechAPI.Speech(this._client);\n}\n\nexport namespace Audio {\n  export import Transcriptions = TranscriptionsAPI.Transcriptions;\n  export import Transcription = TranscriptionsAPI.Transcription;\n  export import TranscriptionCreateParams = TranscriptionsAPI.TranscriptionCreateParams;\n...\n", "needle_token_start": 3967, "needle_token_end": 4098, "code_context_ntokens": 16099, "output": ["```typescript\nfunction applyHeadersMut(targetHeaders: Headers, newHeaders: Headers): void {\n  for (const k in newHeaders) {\n    if (!hasOwn(newHeaders, k)) continue;\n    const lowerKey = k.toLowerCase();\n    if (!lowerKey) continue;\n\n    const val = newHeaders[k];\n\n    if (val === null) {\n      delete targetHeaders[lowerKey];\n    } else if (val !== undefined) {\n      targetHeaders[lowerKey] = val;\n    }\n  }\n}\n```"]}
{"repo": "openai/openai-node", "name": "getDefaultAgent", "language": "typescript", "path": "src/_shims/index-deno.ts", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine and return the appropriate network agent based on the URL provided, typically used for network requests.\n2. **Input**: A single string parameter representing a URL.\n3. **Output**: The function returns `undefined`, indicating no specific agent is set or needed for the given URL in this context.\n4. **Procedure**: The function receives a URL as input and directly returns `undefined`, implying that no specific processing or agent determination is performed.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/lib/ChatCompletionRunFunctions.test.ts\nimport OpenAI from 'openai';\nimport { OpenAIError, APIConnectionError } from 'openai/error';\nimport { PassThrough } from 'stream';\nimport {\n  ParsingToolFunction,\n  type ChatCompletionRunner,\n  type ChatCompletionFunctionRunnerParams,\n  ChatCompletionStreamingRunner,\n  type ChatCompletionStreamingFunctionRunnerParams,\n} from 'openai/resources/beta/chat/completions';\nimport type { ChatCompletionMessageParam } from 'openai/resources/chat/completions';\n\nimport { type RequestInfo, type RequestInit } from 'openai/_shims/index';\nimport { Response } from 'node-fetch';\nimport { isAssistantMessage } from './chatCompletionUtils';\n\ntype Fetch = (req: string | RequestInfo, init?: RequestInit) => Promise<Response>;\n\n/**\n * Creates a mock `fetch` function and a `handleRequest` function for intercepting `fetch` calls.\n *\n * You call `handleRequest` with a callback function that handles the next `fetch` call.\n * It returns a Promise that:\n * - waits for the next call to `fetch`\n * - calls the callback with the `fetch` arguments\n * - resolves `fetch` with the callback output\n */\nfunction mockFetch(): { fetch: Fetch; handleRequest: (handle: Fetch) => Promise<void> } {\n  const fetchQueue: ((handler: typeof fetch) => void)[] = [];\n  const handlerQueue: Promise<typeof fetch>[] = [];\n\n  const enqueueHandler = () => {\n    handlerQueue.push(\n      new Promise<typeof fetch>((resolve) => {\n        fetchQueue.push((handle: typeof fetch) => {\n          enqueueHandler();\n          resolve(handle);\n        });\n      }),\n    );\n  };\n  enqueueHandler();\n\n  async function fetch(req: string | RequestInfo, init?: RequestInit): Promise<Response> {\n    const handler = await handlerQueue.shift();\n    if (!handler) throw new Error('expected handler to be defined');\n    const signal = init?.signal;\n    if (!signal) return await handler(req, init);\n    return await Promise.race([\n      handler(req, init),\n      new Promise<Response>((resolve, reject) => {\n        if (signal.aborted) {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n          return;\n        }\n        signal.addEventListener('abort', (e) => {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n        });\n      }),\n    ]);\n  }\n\n  function handleRequest(handle: typeof fetch): Promise<void> {\n    return new Promise<void>((resolve, reject) => {\n      fetchQueue.shift()?.(async (req, init) => {\n        try {\n          return await handle(req, init);\n        } catch (err) {\n          reject(err);\n          return err as any;\n        } finally {\n          resolve();\n        }\n      });\n    });\n  }\n\n  return { fetch, handleRequest };\n}\n\n// mockChatCompletionFetch is like mockFetch, but with better a more convenient handleRequest to mock\n// chat completion request/responses.\nfunction mockChatCompletionFetch() {\n  const { fetch, handleRequest: handleRawRequest } = mockFetch();\n\n  function handleRequest(\n    handler: (body: ChatCompletionFunctionRunnerParams<any[]>) => Promise<OpenAI.Chat.ChatCompletion>,\n  ): Promise<void> {\n    return handleRawRequest(async (req, init) => {\n      const rawBody = init?.body;\n      if (typeof rawBody !== 'string') throw new Error(`expected init.body to be a string`);\n      const body: ChatCompletionFunctionRunnerParams<any[]> = JSON.parse(rawBody);\n      return new Response(JSON.stringify(await handler(body)), {\n        headers: { 'Content-Type': 'application/json' },\n      });\n    });\n  }\n  return { fetch, handleRequest };\n}\n\n// mockStreamingChatCompletionFetch is like mockFetch, but with better a more convenient handleRequest to mock\n// streaming chat completion request/responses.\nfunction mockStreamingChatCompletionFetch() {\n  const { fetch, handleRequest: handleRawRequest } = mockFetch();\n\n  function handleRequest(\n    handler: (\n      body: ChatCompletionStreamingFunctionRunnerParams<any[]>,\n    ) => AsyncIterable<OpenAI.Chat.ChatCompletionChunk>,\n  ): Promise<void> {\n    return handleRawRequest(async (req, init) => {\n      const rawBody = init?.body;\n      if (typeof rawBody !== 'string') throw new Error(`expected init.body to be a string`);\n      const body: ChatCompletionStreamingFunctionRunnerParams<any[]> = JSON.parse(rawBody);\n      const stream = new PassThrough();\n      (async () => {\n        for await (const chunk of handler(body)) {\n          stream.write(`data: ${JSON.stringify(chunk)}\\n\\n`);\n        }\n        stream.end(`data: [DONE]\\n\\n`);\n      })();\n      return new Response(stream, {\n        headers: {\n          'Content-Type': 'text/event-stream',\n          'Transfer-Encoding': 'chunked',\n        },\n      });\n    });\n  }\n  return { fetch, handleRequest };\n}\n\n// contentChoiceDeltas returns an async iterator which mocks a delta stream of a by splitting the\n// argument into chunks separated by whitespace.\nfunction* contentChoiceDeltas(\n  content: string,\n  {\n    index = 0,\n    role = 'assistant',\n  }: { index?: number; role?: NonNullable<OpenAI.Chat.ChatCompletionChunk.Choice.Delta['role']> } = {},\n): Iterable<OpenAI.Chat.Completions.ChatCompletionChunk.Choice> {\n  const deltas = content.split(/\\s+/g);\n  for (let i = 0; i < deltas.length; i++) {\n    yield {\n      index,\n      finish_reason: i === deltas.length - 1 ? 'stop' : null,\n      logprobs: null,\n      delta: {\n        role,\n        content: deltas[i] ? `${deltas[i]}${i === deltas.length - 1 ? '' : ' '}` : null,\n      },\n    };\n  }\n}\n\n// functionCallDeltas returns an async iterator which mocks a delta stream of a functionCall by splitting\n// the argument into chunks separated by whitespace.\nfunction* functionCallDeltas(\n  args: string,\n  {\n    index = 0,\n    id = '123',\n    name,\n    role = 'assistant',\n  }: {\n    name: string;\n    id?: string;\n    index?: number;\n    role?: NonNullable<OpenAI.Chat.ChatCompletionChunk.Choice.Delta['role']>;\n  },\n): Iterable<OpenAI.Chat.Completions.ChatCompletionChunk.Choice> {\n  const deltas = args.split(/\\s+/g);\n  for (let i = 0; i < deltas.length; i++) {\n    yield {\n      index,\n      finish_reason: i === deltas.length - 1 ? 'function_call' : null,\n      delta: {\n        role,\n        tool_calls: [\n          {\n            type: 'function',\n            index: 0,\n            id,\n            function: {\n              arguments: `${deltas[i] || ''}${i === deltas.length - 1 ? '' : ' '}`,\n              ...(i === deltas.length - 1 ? { name } : null),\n            },\n          },\n        ],\n      },\n    };\n  }\n}\n\nclass RunnerListener {\n  readonly contents: string[] = [];\n  readonly messages: ChatCompletionMessageParam[] = [];\n  readonly chatCompletions: OpenAI.Chat.ChatCompletion[] = [];\n  readonly functionCalls: OpenAI.Chat.ChatCompletionMessage.FunctionCall[] = [];\n  readonly functionCallResults: string[] = [];\n  finalContent: string | null = null;\n  finalMessage: ChatCompletionMessageParam | undefined;\n  finalChatCompletion: OpenAI.Chat.ChatCompletion | undefined;\n  finalFunctionCall: OpenAI.Chat.ChatCompletionMessage.FunctionCall | undefined;\n  finalFunctionCallResult: string | undefined;\n  totalUsage: OpenAI.CompletionUsage | undefined;\n  error: OpenAIError | undefined;\n  gotConnect = false;\n  gotAbort = false;\n  gotEnd = false;\n\n  onceMessageCallCount = 0;\n\n  constructor(public runner: ChatCompletionRunner) {\n    runner\n      .on('connect', () => (this.gotConnect = true))\n      .on('content', (content) => this.contents.push(content))\n      .on('message', (message) => this.messages.push(message))\n      .on('chatCompletion', (completion) => this.chatCompletions.push(completion))\n      .on('functionCall', (functionCall) => this.functionCalls.push(functionCall))\n      .on('functionCallResult', (result) => this.functionCallResults.push(result))\n      .on('finalContent', (content) => (this.finalContent = content))\n      .on('finalMessage', (message) => (this.finalMessage = message))\n      .on('finalChatCompletion', (completion) => (this.finalChatCompletion = completion))\n      .on('finalFunctionCall', (functionCall) => (this.finalFunctionCall = functionCall))\n      .on('finalFunctionCallResult', (result) => (this.finalFunctionCallResult = result))\n      .on('totalUsage', (usage) => (this.totalUsage = usage))\n      .on('error', (error) => (this.error = error))\n      .on('abort', (error) => ((this.error = error), (this.gotAbort = true)))\n...\n// Path: src/_shims/web-types.js\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\n\n// Path: src/shims/web.ts\n// @ts-ignore\nimport * as types from '../_shims/web-types';\nimport { setShims } from '../_shims/registry';\nimport { getRuntime } from '../_shims/web-runtime';\nsetShims(getRuntime({ manuallyImported: true }));\n\ndeclare module '../_shims/manual-types' {\n  export namespace manual {\n    // @ts-ignore\n    export type Agent = types.Agent;\n    // @ts-ignore\n    export import fetch = types.fetch;\n    // @ts-ignore\n    export type Request = types.Request;\n    // @ts-ignore\n    export type RequestInfo = types.RequestInfo;\n    // @ts-ignore\n    export type RequestInit = types.RequestInit;\n    // @ts-ignore\n    export type Response = types.Response;\n    // @ts-ignore\n    export type ResponseInit = types.ResponseInit;\n    // @ts-ignore\n    export type ResponseType = types.ResponseType;\n    // @ts-ignore\n    export type BodyInit = types.BodyInit;\n    // @ts-ignore\n    export type Headers = types.Headers;\n    // @ts-ignore\n    export type HeadersInit = types.HeadersInit;\n    // @ts-ignore\n    export type BlobPropertyBag = types.BlobPropertyBag;\n    // @ts-ignore\n    export type FilePropertyBag = types.FilePropertyBag;\n    // @ts-ignore\n    export type FileFromPathOptions = types.FileFromPathOptions;\n    // @ts-ignore\n    export import FormData = types.FormData;\n    // @ts-ignore\n    export import File = types.File;\n    // @ts-ignore\n    export import Blob = types.Blob;\n    // @ts-ignore\n    export type Readable = types.Readable;\n    // @ts-ignore\n    export type FsReadStream = types.FsReadStream;\n    // @ts-ignore\n    export import ReadableStream = types.ReadableStream;\n  }\n}\n\n// Path: src/_shims/node-runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport * as nf from 'node-fetch';\nimport * as fd from 'formdata-node';\nimport { type File, type FilePropertyBag } from 'formdata-node';\nimport KeepAliveAgent from 'agentkeepalive';\nimport { AbortController as AbortControllerPolyfill } from 'abort-controller';\nimport { ReadStream as FsReadStream } from 'node:fs';\nimport { type Agent } from 'node:http';\nimport { FormDataEncoder } from 'form-data-encoder';\nimport { Readable } from 'node:stream';\nimport { type RequestOptions } from '../core';\nimport { MultipartBody } from './MultipartBody';\nimport { type Shims } from './registry';\n\n// @ts-ignore (this package does not have proper export maps for this export)\nimport { ReadableStream } from 'web-streams-polyfill/dist/ponyfill.es2018.js';\n\ntype FileFromPathOptions = Omit<FilePropertyBag, 'lastModified'>;\n\nlet fileFromPathWarned = false;\n\n/**\n * @deprecated use fs.createReadStream('./my/file.txt') instead\n */\nasync function fileFromPath(path: string): Promise<File>;\nasync function fileFromPath(path: string, filename?: string): Promise<File>;\nasync function fileFromPath(path: string, options?: FileFromPathOptions): Promise<File>;\nasync function fileFromPath(path: string, filename?: string, options?: FileFromPathOptions): Promise<File>;\nasync function fileFromPath(path: string, ...args: any[]): Promise<File> {\n  // this import fails in environments that don't handle export maps correctly, like old versions of Jest\n  const { fileFromPath: _fileFromPath } = await import('formdata-node/file-from-path');\n\n  if (!fileFromPathWarned) {\n    console.warn(`fileFromPath is deprecated; use fs.createReadStream(${JSON.stringify(path)}) instead`);\n    fileFromPathWarned = true;\n  }\n  // @ts-ignore\n  return await _fileFromPath(path, ...args);\n}\n\nconst defaultHttpAgent: Agent = new KeepAliveAgent({ keepAlive: true, timeout: 5 * 60 * 1000 });\nconst defaultHttpsAgent: Agent = new KeepAliveAgent.HttpsAgent({ keepAlive: true, timeout: 5 * 60 * 1000 });\n\nasync function getMultipartRequestOptions<T = Record<string, unknown>>(\n  form: fd.FormData,\n  opts: RequestOptions<T>,\n): Promise<RequestOptions<T>> {\n  const encoder = new FormDataEncoder(form);\n  const readable = Readable.from(encoder);\n  const body = new MultipartBody(readable);\n  const headers = {\n    ...opts.headers,\n    ...encoder.headers,\n    'Content-Length': encoder.contentLength,\n  };\n\n  return { ...opts, body: body as any, headers };\n}\n\nexport function getRuntime(): Shims {\n  // Polyfill global object if needed.\n  if (typeof AbortController === 'undefined') {\n    // @ts-expect-error (the types are subtly different, but compatible in practice)\n    globalThis.AbortController = AbortControllerPolyfill;\n  }\n  return {\n    kind: 'node',\n    fetch: nf.default,\n    Request: nf.Request,\n    Response: nf.Response,\n    Headers: nf.Headers,\n    FormData: fd.FormData,\n    Blob: fd.Blob,\n    File: fd.File,\n    ReadableStream,\n    getMultipartRequestOptions,\n    getDefaultAgent: (url: string): Agent => (url.startsWith('https') ? defaultHttpsAgent : defaultHttpAgent),\n    fileFromPath,\n    isFsReadStream: (value: any): value is FsReadStream => value instanceof FsReadStream,\n  };\n}\n\n// Path: src/_shims/node-types.js\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\n\n// Path: src/shims/node.ts\n// @ts-ignore\nimport * as types from '../_shims/node-types';\nimport { setShims } from '../_shims/registry';\nimport { getRuntime } from '../_shims/node-runtime';\nsetShims(getRuntime());\n\ndeclare module '../_shims/manual-types' {\n  export namespace manual {\n    // @ts-ignore\n    export type Agent = types.Agent;\n    // @ts-ignore\n    export import fetch = types.fetch;\n    // @ts-ignore\n    export type Request = types.Request;\n    // @ts-ignore\n    export type RequestInfo = types.RequestInfo;\n    // @ts-ignore\n    export type RequestInit = types.RequestInit;\n    // @ts-ignore\n    export type Response = types.Response;\n    // @ts-ignore\n    export type ResponseInit = types.ResponseInit;\n    // @ts-ignore\n    export type ResponseType = types.ResponseType;\n    // @ts-ignore\n    export type BodyInit = types.BodyInit;\n    // @ts-ignore\n    export type Headers = types.Headers;\n    // @ts-ignore\n    export type HeadersInit = types.HeadersInit;\n    // @ts-ignore\n    export type BlobPropertyBag = types.BlobPropertyBag;\n    // @ts-ignore\n    export type FilePropertyBag = types.FilePropertyBag;\n    // @ts-ignore\n    export type FileFromPathOptions = types.FileFromPathOptions;\n    // @ts-ignore\n    export import FormData = types.FormData;\n    // @ts-ignore\n    export import File = types.File;\n    // @ts-ignore\n    export import Blob = types.Blob;\n    // @ts-ignore\n    export type Readable = types.Readable;\n    // @ts-ignore\n    export type FsReadStream = types.FsReadStream;\n    // @ts-ignore\n    export import ReadableStream = types.ReadableStream;\n  }\n}\n\n// Path: src/_shims/node-types.d.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport * as nf from 'node-fetch';\nimport * as fd from 'formdata-node';\n\nexport { type Agent } from 'node:http';\nexport { type Readable } from 'node:stream';\nexport { type ReadStream as FsReadStream } from 'node:fs';\nexport { ReadableStream } from 'web-streams-polyfill';\n\nexport const fetch: typeof nf.default;\n\nexport type Request = nf.Request;\nexport type RequestInfo = nf.RequestInfo;\nexport type RequestInit = nf.RequestInit;\n\nexport type Response = nf.Response;\nexport type ResponseInit = nf.ResponseInit;\nexport type ResponseType = nf.ResponseType;\nexport type BodyInit = nf.BodyInit;\nexport type Headers = nf.Headers;\nexport type HeadersInit = nf.HeadersInit;\n\ntype EndingType = 'native' | 'transparent';\nexport interface BlobPropertyBag {\n  endings?: EndingType;\n  type?: string;\n}\n\nexport interface FilePropertyBag extends BlobPropertyBag {\n  lastModified?: number;\n}\n\nexport type FileFromPathOptions = Omit<FilePropertyBag, 'lastModified'>;\n\nexport type FormData = fd.FormData;\nexport const FormData: typeof fd.FormData;\nexport type File = fd.File;\nexport const File: typeof fd.File;\nexport type Blob = fd.Blob;\nexport const Blob: typeof fd.Blob;\n\n// Path: src/_shims/manual-types.d.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\n/**\n * Types will get added to this namespace when you import one of the following:\n *\n *   import 'openai/shims/node'\n *   import 'openai/shims/web'\n *\n * Importing more than one will cause type and runtime errors.\n */\nexport namespace manual {}\n\n// Path: src/_shims/index-deno.ts\nimport { MultipartBody } from './MultipartBody';\nimport { type RequestOptions } from '../core';\n\nexport const kind: string = 'web';\n\nexport type Agent = any;\n\nconst _fetch = fetch;\ntype _fetch = typeof fetch;\nexport { _fetch as fetch };\n\nconst _Request = Request;\ntype _Request = Request;\nexport { _Request as Request };\n\ntype _RequestInfo = RequestInfo;\nexport { type _RequestInfo as RequestInfo };\n\ntype _RequestInit = RequestInit;\nexport { type _RequestInit as RequestInit };\n\nconst _Response = Response;\ntype _Response = Response;\nexport { _Response as Response };\n\ntype _ResponseInit = ResponseInit;\nexport { type _ResponseInit as ResponseInit };\n\ntype _ResponseType = ResponseType;\nexport { type _ResponseType as ResponseType };\n\ntype _BodyInit = BodyInit;\nexport { type _BodyInit as BodyInit };\n\nconst _Headers = Headers;\ntype _Headers = Headers;\nexport { _Headers as Headers };\n\ntype _HeadersInit = HeadersInit;\nexport { type _HeadersInit as HeadersInit };\n\ntype EndingType = 'native' | 'transparent';\n\nexport interface BlobPropertyBag {\n  endings?: EndingType;\n  type?: string;\n}\n\nexport interface FilePropertyBag extends BlobPropertyBag {\n  lastModified?: number;\n}\n\nexport type FileFromPathOptions = Omit<FilePropertyBag, 'lastModified'>;\n\nconst _FormData = FormData;\ntype _FormData = FormData;\nexport { _FormData as FormData };\n\nconst _File = File;\ntype _File = File;\nexport { _File as File };\n\nconst _Blob = Blob;\ntype _Blob = Blob;\nexport { _Blob as Blob };\n\nexport async function getMultipartRequestOptions<T = Record<string, unknown>>(\n  form: FormData,\n  opts: RequestOptions<T>,\n): Promise<RequestOptions<T>> {\n  return {\n    ...opts,\n    body: new MultipartBody(form) as any,\n  };\n}\n\nexport \nfunction getDefaultAgent(url: string) {\n  return undefined;\n}\nexport function fileFromPath() {\n  throw new Error(\n    'The `fileFromPath` function is only supported in Node. See the README for more details: https://www.github.com/openai/openai-node#file-uploads',\n  );\n}\n\nexport const isFsReadStream = (value: any) => false;\n\nexport declare class Readable {\n  readable: boolean;\n  readonly readableEnded: boolean;\n  readonly readableFlowing: boolean | null;\n  readonly readableHighWaterMark: number;\n  readonly readableLength: number;\n  readonly readableObjectMode: boolean;\n  destroyed: boolean;\n  read(size?: number): any;\n  pause(): this;\n  resume(): this;\n  isPaused(): boolean;\n  destroy(error?: Error): this;\n  [Symbol.asyncIterator](): AsyncIterableIterator<any>;\n}\n\nexport declare class FsReadStream extends Readable {\n  path: {}; // node type is string | Buffer\n}\n\nconst _ReadableStream = ReadableStream;\ntype _ReadableStream = ReadableStream;\nexport { _ReadableStream as ReadableStream };\n\n// Path: src/_shims/manual-types.js\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\n\n// Path: src/_shims/bun-runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { type Shims } from './registry';\nimport { getRuntime as getWebRuntime } from './web-runtime';\nimport { ReadStream as FsReadStream } from 'node:fs';\n\nexport function getRuntime(): Shims {\n  const runtime = getWebRuntime();\n  function isFsReadStream(value: any): value is FsReadStream {\n    return value instanceof FsReadStream;\n  }\n  return { ...runtime, isFsReadStream };\n}\n\n// Path: src/_shims/web-types.d.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport type Agent = any;\n\ndeclare const _fetch: typeof fetch;\nexport { _fetch as fetch };\n\ntype _Request = Request;\nexport { _Request as Request };\n\ntype _RequestInfo = RequestInfo;\nexport { type _RequestInfo as RequestInfo };\n\ntype _RequestInit = RequestInit;\nexport { type _RequestInit as RequestInit };\n\ntype _Response = Response;\nexport { _Response as Response };\n\ntype _ResponseInit = ResponseInit;\nexport { type _ResponseInit as ResponseInit };\n\ntype _ResponseType = ResponseType;\nexport { type _ResponseType as ResponseType };\n\ntype _BodyInit = BodyInit;\nexport { type _BodyInit as BodyInit };\n\ntype _Headers = Headers;\nexport { _Headers as Headers };\n\ntype _HeadersInit = HeadersInit;\nexport { type _HeadersInit as HeadersInit };\n\ntype EndingType = 'native' | 'transparent';\n\nexport interface BlobPropertyBag {\n  endings?: EndingType;\n  type?: string;\n}\n\nexport interface FilePropertyBag extends BlobPropertyBag {\n  lastModified?: number;\n}\n\nexport type FileFromPathOptions = Omit<FilePropertyBag, 'lastModified'>;\n\ntype _FormData = FormData;\ndeclare const _FormData: typeof FormData;\nexport { _FormData as FormData };\n\ntype _File = File;\ndeclare const _File: typeof File;\nexport { _File as File };\n\ntype _Blob = Blob;\ndeclare const _Blob: typeof Blob;\nexport { _Blob as Blob };\n\nexport declare class Readable {\n  readable: boolean;\n  readonly readableEnded: boolean;\n  readonly readableFlowing: boolean | null;\n  readonly readableHighWaterMark: number;\n  readonly readableLength: number;\n  readonly readableObjectMode: boolean;\n  destroyed: boolean;\n  read(size?: number): any;\n  pause(): this;\n  resume(): this;\n  isPaused(): boolean;\n  destroy(error?: Error): this;\n  [Symbol.asyncIterator](): AsyncIterableIterator<any>;\n}\n\nexport declare class FsReadStream extends Readable {\n  path: {}; // node type is string | Buffer\n}\n\ntype _ReadableStream<R = any> = ReadableStream<R>;\ndeclare const _ReadableStream: typeof ReadableStream;\nexport { _ReadableStream as ReadableStream };\n\n// Path: src/_shims/auto/types.js\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\n\n// Path: src/_shims/index.d.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { manual } from './manual-types';\nimport * as auto from 'openai/_shims/auto/types';\nimport { type RequestOptions } from '../core';\n\ntype SelectType<Manual, Auto> = unknown extends Manual ? Auto : Manual;\n\nexport const kind: string;\n\n// @ts-ignore\nexport type Agent = SelectType<manual.Agent, auto.Agent>;\n\n// @ts-ignore\nexport const fetch: SelectType<typeof manual.fetch, typeof auto.fetch>;\n\n// @ts-ignore\nexport type Request = SelectType<manual.Request, auto.Request>;\n// @ts-ignore\nexport type RequestInfo = SelectType<manual.RequestInfo, auto.RequestInfo>;\n// @ts-ignore\nexport type RequestInit = SelectType<manual.RequestInit, auto.RequestInit>;\n\n// @ts-ignore\nexport type Response = SelectType<manual.Response, auto.Response>;\n// @ts-ignore\nexport type ResponseInit = SelectType<manual.ResponseInit, auto.ResponseInit>;\n// @ts-ignore\nexport type ResponseType = SelectType<manual.ResponseType, auto.ResponseType>;\n// @ts-ignore\nexport type BodyInit = SelectType<manual.BodyInit, auto.BodyInit>;\n// @ts-ignore\nexport type Headers = SelectType<manual.Headers, auto.Headers>;\n// @ts-ignore\nexport const Headers: SelectType<typeof manual.Headers, typeof auto.Headers>;\n// @ts-ignore\nexport type HeadersInit = SelectType<manual.HeadersInit, auto.HeadersInit>;\n\n// @ts-ignore\nexport type BlobPropertyBag = SelectType<manual.BlobPropertyBag, auto.BlobPropertyBag>;\n// @ts-ignore\nexport type FilePropertyBag = SelectType<manual.FilePropertyBag, auto.FilePropertyBag>;\n// @ts-ignore\nexport type FileFromPathOptions = SelectType<manual.FileFromPathOptions, auto.FileFromPathOptions>;\n// @ts-ignore\nexport type FormData = SelectType<manual.FormData, auto.FormData>;\n// @ts-ignore\nexport const FormData: SelectType<typeof manual.FormData, typeof auto.FormData>;\n// @ts-ignore\nexport type File = SelectType<manual.File, auto.File>;\n// @ts-ignore\nexport const File: SelectType<typeof manual.File, typeof auto.File>;\n// @ts-ignore\nexport type Blob = SelectType<manual.Blob, auto.Blob>;\n// @ts-ignore\nexport const Blob: SelectType<typeof manual.Blob, typeof auto.Blob>;\n\n// @ts-ignore\nexport type Readable = SelectType<manual.Readable, auto.Readable>;\n// @ts-ignore\nexport type FsReadStream = SelectType<manual.FsReadStream, auto.FsReadStream>;\n// @ts-ignore\nexport type ReadableStream = SelectType<manual.ReadableStream, auto.ReadableStream>;\n// @ts-ignore\nexport const ReadableStream: SelectType<typeof manual.ReadableStream, typeof auto.ReadableStream>;\n\nexport function getMultipartRequestOptions<T = Record<string, unknown>>(\n  form: FormData,\n  opts: RequestOptions<T>,\n): Promise<RequestOptions<T>>;\n\nexport function getDefaultAgent(url: string): any;\n\n// @ts-ignore\nexport type FileFromPathOptions = SelectType<manual.FileFromPathOptions, auto.FileFromPathOptions>;\n\nexport function fileFromPath(path: string, options?: FileFromPathOptions): Promise<File>;\nexport function fileFromPath(path: string, filename?: string, options?: FileFromPathOptions): Promise<File>;\n\nexport function isFsReadStream(value: any): value is FsReadStream;\n\n// Path: src/_shims/auto/types.d.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport type Agent = any;\n\n// @ts-ignore\ndeclare const _fetch: unknown extends typeof fetch ? never : typeof fetch;\nexport { _fetch as fetch };\n\n// @ts-ignore\ntype _Request = unknown extends Request ? never : Request;\nexport { _Request as Request };\n\n// @ts-ignore\ntype _RequestInfo = unknown extends RequestInfo ? never : RequestInfo;\nexport { type _RequestInfo as RequestInfo };\n\n// @ts-ignore\ntype _RequestInit = unknown extends RequestInit ? never : RequestInit;\nexport { type _RequestInit as RequestInit };\n\n// @ts-ignore\ntype _Response = unknown extends Response ? never : Response;\nexport { _Response as Response };\n\n// @ts-ignore\ntype _ResponseInit = unknown extends ResponseInit ? never : ResponseInit;\nexport { type _ResponseInit as ResponseInit };\n\n// @ts-ignore\ntype _ResponseType = unknown extends ResponseType ? never : ResponseType;\nexport { type _ResponseType as ResponseType };\n\n// @ts-ignore\ntype _BodyInit = unknown extends BodyInit ? never : BodyInit;\nexport { type _BodyInit as BodyInit };\n\n// @ts-ignore\ntype _Headers = unknown extends Headers ? never : Headers;\nexport { _Headers as Headers };\n\n// @ts-ignore\ntype _HeadersInit = unknown extends HeadersInit ? never : HeadersInit;\nexport { type _HeadersInit as HeadersInit };\n\ntype EndingType = 'native' | 'transparent';\n\nexport interface BlobPropertyBag {\n  endings?: EndingType;\n  type?: string;\n}\n\nexport interface FilePropertyBag extends BlobPropertyBag {\n  lastModified?: number;\n}\n\nexport type FileFromPathOptions = Omit<FilePropertyBag, 'lastModified'>;\n\n// @ts-ignore\ntype _FormData = unknown extends FormData ? never : FormData;\n// @ts-ignore\ndeclare const _FormData: unknown extends typeof FormData ? never : typeof FormData;\nexport { _FormData as FormData };\n\n// @ts-ignore\ntype _File = unknown extends File ? never : File;\n// @ts-ignore\ndeclare const _File: unknown extends typeof File ? never : typeof File;\nexport { _File as File };\n\n// @ts-ignore\ntype _Blob = unknown extends Blob ? never : Blob;\n// @ts-ignore\ndeclare const _Blob: unknown extends typeof Blob ? never : typeof Blob;\nexport { _Blob as Blob };\n\nexport declare class Readable {\n  readable: boolean;\n  readonly readableEnded: boolean;\n  readonly readableFlowing: boolean | null;\n  readonly readableHighWaterMark: number;\n  readonly readableLength: number;\n  readonly readableObjectMode: boolean;\n  destroyed: boolean;\n  read(size?: number): any;\n  pause(): this;\n  resume(): this;\n  isPaused(): boolean;\n  destroy(error?: Error): this;\n  [Symbol.asyncIterator](): AsyncIterableIterator<any>;\n}\n\nexport declare class FsReadStream extends Readable {\n  path: {}; // node type is string | Buffer\n}\n\n// @ts-ignore\ntype _ReadableStream<R = any> = unknown extends ReadableStream<R> ? never : ReadableStream<R>;\n// @ts-ignore\ndeclare const _ReadableStream: unknown extends typeof ReadableStream ? never : typeof ReadableStream;\nexport { _ReadableStream as ReadableStream };\n\n// Path: src/_shims/auto/types-node.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../node-types';\n\n// Path: src/_shims/auto/types-deno.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../web-types';\n\n// Path: src/_shims/auto/runtime-deno.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../web-runtime';\n\n// Path: src/_shims/auto/runtime-node.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../node-runtime';\n\n// Path: src/_shims/auto/runtime-bun.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../bun-runtime';\n\n", "needle_token_start": 5619, "needle_token_end": 5636, "code_context_ntokens": 8821, "output": ["```typescript\nfunction getDefaultAgent(url: string) {\n  return undefined;\n}\n```"]}
{"repo": "openai/openai-node", "name": "toFile", "language": "typescript", "path": "src/uploads.ts", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a file object suitable for SDK upload methods from various data formats, including native file types, server responses, or streams.\n2. **Input**: Accepts a variety of data types such as native file objects, server response objects, file streams, individual data pieces like array buffers, or asynchronous iterable sequences of data pieces. Optional parameters include the file name and additional file properties like MIME type and last modified timestamp.\n3. **Output**: Returns a file object that conforms to standard file interfaces, including properties like size, type, and methods for text manipulation and slicing.\n4. **Procedure**: The function first resolves the input if it's a promise. It then checks the type of input to appropriately handle it:\n   - If the input is a server response, it extracts the blob from the response and determines the file name from the URL or assigns a default name if not available.\n   - For other data types, it gathers the data into a format suitable for file creation, determines the file name if not provided, and sets the MIME type if available.\n   - Finally, it creates and returns a new file object using the gathered data, name, and options.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/version.ts\nexport const VERSION = '4.33.0'; // x-release-please-version\n\n// Path: src/error.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { castToError, Headers } from './core';\n\nexport class OpenAIError extends Error {}\n\nexport class APIError extends OpenAIError {\n  readonly status: number | undefined;\n  readonly headers: Headers | undefined;\n  readonly error: Object | undefined;\n\n  readonly code: string | null | undefined;\n  readonly param: string | null | undefined;\n  readonly type: string | undefined;\n\n  constructor(\n    status: number | undefined,\n    error: Object | undefined,\n    message: string | undefined,\n    headers: Headers | undefined,\n  ) {\n    super(`${APIError.makeMessage(status, error, message)}`);\n    this.status = status;\n    this.headers = headers;\n\n    const data = error as Record<string, any>;\n    this.error = data;\n    this.code = data?.['code'];\n    this.param = data?.['param'];\n    this.type = data?.['type'];\n  }\n\n  private static makeMessage(status: number | undefined, error: any, message: string | undefined) {\n    const msg =\n      error?.message ?\n        typeof error.message === 'string' ?\n          error.message\n        : JSON.stringify(error.message)\n      : error ? JSON.stringify(error)\n      : message;\n\n    if (status && msg) {\n      return `${status} ${msg}`;\n    }\n    if (status) {\n      return `${status} status code (no body)`;\n    }\n    if (msg) {\n      return msg;\n    }\n    return '(no status code or body)';\n  }\n\n  static generate(\n    status: number | undefined,\n    errorResponse: Object | undefined,\n    message: string | undefined,\n    headers: Headers | undefined,\n  ) {\n    if (!status) {\n      return new APIConnectionError({ cause: castToError(errorResponse) });\n    }\n\n    const error = (errorResponse as Record<string, any>)?.['error'];\n\n    if (status === 400) {\n      return new BadRequestError(status, error, message, headers);\n    }\n\n    if (status === 401) {\n      return new AuthenticationError(status, error, message, headers);\n    }\n\n    if (status === 403) {\n      return new PermissionDeniedError(status, error, message, headers);\n    }\n\n    if (status === 404) {\n      return new NotFoundError(status, error, message, headers);\n    }\n\n    if (status === 409) {\n      return new ConflictError(status, error, message, headers);\n    }\n\n    if (status === 422) {\n      return new UnprocessableEntityError(status, error, message, headers);\n    }\n\n    if (status === 429) {\n      return new RateLimitError(status, error, message, headers);\n    }\n\n    if (status >= 500) {\n      return new InternalServerError(status, error, message, headers);\n    }\n\n    return new APIError(status, error, message, headers);\n  }\n}\n\nexport class APIUserAbortError extends APIError {\n  override readonly status: undefined = undefined;\n\n  constructor({ message }: { message?: string } = {}) {\n    super(undefined, undefined, message || 'Request was aborted.', undefined);\n  }\n}\n\nexport class APIConnectionError extends APIError {\n  override readonly status: undefined = undefined;\n\n  constructor({ message, cause }: { message?: string; cause?: Error | undefined }) {\n    super(undefined, undefined, message || 'Connection error.', undefined);\n    // in some environments the 'cause' property is already declared\n    // @ts-ignore\n    if (cause) this.cause = cause;\n  }\n}\n\nexport class APIConnectionTimeoutError extends APIConnectionError {\n  constructor({ message }: { message?: string } = {}) {\n    super({ message: message ?? 'Request timed out.' });\n  }\n}\n\nexport class BadRequestError extends APIError {\n  override readonly status: 400 = 400;\n}\n\nexport class AuthenticationError extends APIError {\n  override readonly status: 401 = 401;\n}\n\nexport class PermissionDeniedError extends APIError {\n  override readonly status: 403 = 403;\n}\n\nexport class NotFoundError extends APIError {\n  override readonly status: 404 = 404;\n}\n\nexport class ConflictError extends APIError {\n  override readonly status: 409 = 409;\n}\n\nexport class UnprocessableEntityError extends APIError {\n  override readonly status: 422 = 422;\n}\n\nexport class RateLimitError extends APIError {\n  override readonly status: 429 = 429;\n}\n\nexport class InternalServerError extends APIError {}\n\n// Path: src/_shims/MultipartBody.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport class MultipartBody {\n  constructor(public body: any) {}\n  get [Symbol.toStringTag](): string {\n    return 'MultipartBody';\n  }\n}\n\n// Path: src/uploads.ts\nimport { type RequestOptions } from './core';\nimport {\n  FormData,\n  File,\n  type Blob,\n  type FilePropertyBag,\n  getMultipartRequestOptions,\n  type FsReadStream,\n  isFsReadStream,\n} from './_shims/index';\nimport { MultipartBody } from './_shims/MultipartBody';\nexport { fileFromPath } from './_shims/index';\n\ntype BlobLikePart = string | ArrayBuffer | ArrayBufferView | BlobLike | Uint8Array | DataView;\nexport type BlobPart = string | ArrayBuffer | ArrayBufferView | Blob | Uint8Array | DataView;\n\n/**\n * Typically, this is a native \"File\" class.\n *\n * We provide the {@link toFile} utility to convert a variety of objects\n * into the File class.\n *\n * For convenience, you can also pass a fetch Response, or in Node,\n * the result of fs.createReadStream().\n */\nexport type Uploadable = FileLike | ResponseLike | FsReadStream;\n\n/**\n * Intended to match web.Blob, node.Blob, node-fetch.Blob, etc.\n */\nexport interface BlobLike {\n  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) */\n  readonly size: number;\n  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) */\n  readonly type: string;\n  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) */\n  text(): Promise<string>;\n  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) */\n  slice(start?: number, end?: number): BlobLike;\n  // unfortunately @types/node-fetch@^2.6.4 doesn't type the arrayBuffer method\n}\n\n/**\n * Intended to match web.File, node.File, node-fetch.File, etc.\n */\nexport interface FileLike extends BlobLike {\n  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) */\n  readonly lastModified: number;\n  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) */\n  readonly name: string;\n}\n\n/**\n * Intended to match web.Response, node.Response, node-fetch.Response, etc.\n */\nexport interface ResponseLike {\n  url: string;\n  blob(): Promise<BlobLike>;\n}\n\nexport const isResponseLike = (value: any): value is ResponseLike =>\n  value != null &&\n  typeof value === 'object' &&\n  typeof value.url === 'string' &&\n  typeof value.blob === 'function';\n\nexport const isFileLike = (value: any): value is FileLike =>\n  value != null &&\n  typeof value === 'object' &&\n  typeof value.name === 'string' &&\n  typeof value.lastModified === 'number' &&\n  isBlobLike(value);\n\n/**\n * The BlobLike type omits arrayBuffer() because @types/node-fetch@^2.6.4 lacks it; but this check\n * adds the arrayBuffer() method type because it is available and used at runtime\n */\nexport const isBlobLike = (value: any): value is BlobLike & { arrayBuffer(): Promise<ArrayBuffer> } =>\n  value != null &&\n  typeof value === 'object' &&\n  typeof value.size === 'number' &&\n  typeof value.type === 'string' &&\n  typeof value.text === 'function' &&\n  typeof value.slice === 'function' &&\n  typeof value.arrayBuffer === 'function';\n\nexport const isUploadable = (value: any): value is Uploadable => {\n  return isFileLike(value) || isResponseLike(value) || isFsReadStream(value);\n};\n\nexport type ToFileInput = Uploadable | Exclude<BlobLikePart, string> | AsyncIterable<BlobLikePart>;\n\n/**\n * Helper for creating a {@link File} to pass to an SDK upload method from a variety of different data formats\n * @param value the raw content of the file.  Can be an {@link Uploadable}, {@link BlobLikePart}, or {@link AsyncIterable} of {@link BlobLikePart}s\n * @param {string=} name the name of the file. If omitted, toFile will try to determine a file name from bits if possible\n * @param {Object=} options additional properties\n * @param {string=} options.type the MIME type of the content\n * @param {number=} options.lastModified the last modified timestamp\n * @returns a {@link File} with the given properties\n */\nexport \nasync function toFile(\n  value: ToFileInput | PromiseLike<ToFileInput>,\n  name?: string | null | undefined,\n  options?: FilePropertyBag | undefined,\n): Promise<FileLike> {\n  // If it's a promise, resolve it.\n  value = await value;\n\n  // Use the file's options if there isn't one provided\n  options ??= isFileLike(value) ? { lastModified: value.lastModified, type: value.type } : {};\n\n  if (isResponseLike(value)) {\n    const blob = await value.blob();\n    name ||= new URL(value.url).pathname.split(/[\\\\/]/).pop() ?? 'unknown_file';\n\n    return new File([blob as any], name, options);\n  }\n\n  const bits = await getBytes(value);\n\n  name ||= getName(value) ?? 'unknown_file';\n\n  if (!options.type) {\n    const type = (bits[0] as any)?.type;\n    if (typeof type === 'string') {\n      options = { ...options, type };\n    }\n  }\n\n  return new File(bits, name, options);\n}\n\nasync function getBytes(value: ToFileInput): Promise<Array<BlobPart>> {\n  let parts: Array<BlobPart> = [];\n  if (\n    typeof value === 'string' ||\n    ArrayBuffer.isView(value) || // includes Uint8Array, Buffer, etc.\n    value instanceof ArrayBuffer\n  ) {\n    parts.push(value);\n  } else if (isBlobLike(value)) {\n    parts.push(await value.arrayBuffer());\n  } else if (\n    isAsyncIterableIterator(value) // includes Readable, ReadableStream, etc.\n  ) {\n    for await (const chunk of value) {\n      parts.push(chunk as BlobPart); // TODO, consider validating?\n    }\n  } else {\n    throw new Error(\n      `Unexpected data type: ${typeof value}; constructor: ${value?.constructor\n        ?.name}; props: ${propsForError(value)}`,\n    );\n  }\n\n  return parts;\n}\n\nfunction propsForError(value: any): string {\n  const props = Object.getOwnPropertyNames(value);\n  return `[${props.map((p) => `\"${p}\"`).join(', ')}]`;\n}\n\nfunction getName(value: any): string | undefined {\n  return (\n    getStringFromMaybeBuffer(value.name) ||\n    getStringFromMaybeBuffer(value.filename) ||\n    // For fs.ReadStream\n    getStringFromMaybeBuffer(value.path)?.split(/[\\\\/]/).pop()\n  );\n}\n\nconst getStringFromMaybeBuffer = (x: string | Buffer | unknown): string | undefined => {\n  if (typeof x === 'string') return x;\n  if (typeof Buffer !== 'undefined' && x instanceof Buffer) return String(x);\n  return undefined;\n};\n\nconst isAsyncIterableIterator = (value: any): value is AsyncIterableIterator<unknown> =>\n  value != null && typeof value === 'object' && typeof value[Symbol.asyncIterator] === 'function';\n\nexport const isMultipartBody = (body: any): body is MultipartBody =>\n  body && typeof body === 'object' && body.body && body[Symbol.toStringTag] === 'MultipartBody';\n\n/**\n * Returns a multipart/form-data request if any part of the given request body contains a File / Blob value.\n * Otherwise returns the request as is.\n */\nexport const maybeMultipartFormRequestOptions = async <T = Record<string, unknown>>(\n  opts: RequestOptions<T>,\n): Promise<RequestOptions<T | MultipartBody>> => {\n  if (!hasUploadableValue(opts.body)) return opts;\n\n  const form = await createForm(opts.body);\n  return getMultipartRequestOptions(form, opts);\n};\n\nexport const multipartFormRequestOptions = async <T = Record<string, unknown>>(\n  opts: RequestOptions<T>,\n): Promise<RequestOptions<T | MultipartBody>> => {\n  const form = await createForm(opts.body);\n  return getMultipartRequestOptions(form, opts);\n};\n\nexport const createForm = async <T = Record<string, unknown>>(body: T | undefined): Promise<FormData> => {\n  const form = new FormData();\n  await Promise.all(Object.entries(body || {}).map(([key, value]) => addFormValue(form, key, value)));\n  return form;\n};\n\nconst hasUploadableValue = (value: unknown): boolean => {\n  if (isUploadable(value)) return true;\n  if (Array.isArray(value)) return value.some(hasUploadableValue);\n  if (value && typeof value === 'object') {\n    for (const k in value) {\n      if (hasUploadableValue((value as any)[k])) return true;\n    }\n  }\n  return false;\n};\n\nconst addFormValue = async (form: FormData, key: string, value: unknown): Promise<void> => {\n  if (value === undefined) return;\n  if (value == null) {\n    throw new TypeError(\n      `Received null for \"${key}\"; to pass null in FormData, you must use the string 'null'`,\n    );\n  }\n\n  // TODO: make nested formats configurable\n  if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {\n    form.append(key, String(value));\n  } else if (isUploadable(value)) {\n    const file = await toFile(value);\n    form.append(key, file as File);\n  } else if (Array.isArray(value)) {\n    await Promise.all(value.map((entry) => addFormValue(form, key + '[]', entry)));\n  } else if (typeof value === 'object') {\n    await Promise.all(\n      Object.entries(value).map(([name, prop]) => addFormValue(form, `${key}[${name}]`, prop)),\n    );\n  } else {\n    throw new TypeError(\n      `Invalid value given to form, expected a string, number, boolean, object, Array, File or Blob but got ${value} instead`,\n    );\n  }\n};\n\n// Path: src/core.ts\nimport { VERSION } from './version';\nimport { Stream } from './streaming';\nimport {\n  OpenAIError,\n  APIError,\n  APIConnectionError,\n  APIConnectionTimeoutError,\n  APIUserAbortError,\n} from './error';\nimport {\n  kind as shimsKind,\n  type Readable,\n  getDefaultAgent,\n  type Agent,\n  fetch,\n  type RequestInfo,\n  type RequestInit,\n  type Response,\n  type HeadersInit,\n} from './_shims/index';\nexport { type Response };\nimport { isMultipartBody } from './uploads';\nexport {\n  maybeMultipartFormRequestOptions,\n  multipartFormRequestOptions,\n  createForm,\n  type Uploadable,\n} from './uploads';\n\nexport type Fetch = (url: RequestInfo, init?: RequestInit) => Promise<Response>;\n\ntype PromiseOrValue<T> = T | Promise<T>;\n\ntype APIResponseProps = {\n  response: Response;\n  options: FinalRequestOptions;\n  controller: AbortController;\n};\n\nasync function defaultParseResponse<T>(props: APIResponseProps): Promise<T> {\n  const { response } = props;\n  if (props.options.stream) {\n    debug('response', response.status, response.url, response.headers, response.body);\n\n    // Note: there is an invariant here that isn't represented in the type system\n    // that if you set `stream: true` the response type must also be `Stream<T>`\n\n    if (props.options.__streamClass) {\n      return props.options.__streamClass.fromSSEResponse(response, props.controller) as any;\n    }\n\n    return Stream.fromSSEResponse(response, props.controller) as any;\n  }\n\n  // fetch refuses to read the body when the status code is 204.\n  if (response.status === 204) {\n    return null as T;\n  }\n\n  if (props.options.__binaryResponse) {\n    return response as unknown as T;\n  }\n\n  const contentType = response.headers.get('content-type');\n  const isJSON =\n    contentType?.includes('application/json') || contentType?.includes('application/vnd.api+json');\n  if (isJSON) {\n    const json = await response.json();\n\n    debug('response', response.status, response.url, response.headers, json);\n\n    return json as T;\n  }\n\n  const text = await response.text();\n  debug('response', response.status, response.url, response.headers, text);\n\n  // TODO handle blob, arraybuffer, other content types, etc.\n  return text as unknown as T;\n}\n\n/**\n * A subclass of `Promise` providing additional helper methods\n * for interacting with the SDK.\n */\nexport class APIPromise<T> extends Promise<T> {\n  private parsedPromise: Promise<T> | undefined;\n\n  constructor(\n    private responsePromise: Promise<APIResponseProps>,\n    private parseResponse: (props: APIResponseProps) => PromiseOrValue<T> = defaultParseResponse,\n  ) {\n    super((resolve) => {\n      // this is maybe a bit weird but this has to be a no-op to not implicitly\n      // parse the response body; instead .then, .catch, .finally are overridden\n      // to parse the response\n      resolve(null as any);\n    });\n  }\n\n  _thenUnwrap<U>(transform: (data: T) => U): APIPromise<U> {\n    return new APIPromise(this.responsePromise, async (props) => transform(await this.parseResponse(props)));\n  }\n\n  /**\n   * Gets the raw `Response` instance instead of parsing the response\n   * data.\n   *\n   * If you want to parse the response body but still get the `Response`\n   * instance, you can use {@link withResponse()}.\n   *\n   * \ud83d\udc4b Getting the wrong TypeScript type for `Response`?\n   * Try setting `\"moduleResolution\": \"NodeNext\"` if you can,\n   * or add one of these imports before your first `import \u2026 from 'openai'`:\n   * - `import 'openai/shims/node'` (if you're running on Node)\n   * - `import 'openai/shims/web'` (otherwise)\n   */\n  asResponse(): Promise<Response> {\n    return this.responsePromise.then((p) => p.response);\n  }\n  /**\n   * Gets the parsed response data and the raw `Response` instance.\n   *\n   * If you just want to get the raw `Response` instance without parsing it,\n   * you can use {@link asResponse()}.\n   *\n   *\n   * \ud83d\udc4b Getting the wrong TypeScript type for `Response`?\n   * Try setting `\"moduleResolution\": \"NodeNext\"` if you can,\n   * or add one of these imports before your first `import \u2026 from 'openai'`:\n   * - `import 'openai/shims/node'` (if you're running on Node)\n   * - `import 'openai/shims/web'` (otherwise)\n   */\n  async withResponse(): Promise<{ data: T; response: Response }> {\n    const [data, response] = await Promise.all([this.parse(), this.asResponse()]);\n    return { data, response };\n  }\n\n  private parse(): Promise<T> {\n    if (!this.parsedPromise) {\n      this.parsedPromise = this.responsePromise.then(this.parseResponse);\n    }\n    return this.parsedPromise;\n  }\n\n  override then<TResult1 = T, TResult2 = never>(\n    onfulfilled?: ((value: T) => TResult1 | PromiseLike<TResult1>) | undefined | null,\n    onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | undefined | null,\n  ): Promise<TResult1 | TResult2> {\n    return this.parse().then(onfulfilled, onrejected);\n  }\n\n  override catch<TResult = never>(\n    onrejected?: ((reason: any) => TResult | PromiseLike<TResult>) | undefined | null,\n  ): Promise<T | TResult> {\n    return this.parse().catch(onrejected);\n  }\n\n  override finally(onfinally?: (() => void) | undefined | null): Promise<T> {\n    return this.parse().finally(onfinally);\n  }\n}\n\nexport abstract class APIClient {\n  baseURL: string;\n  maxRetries: number;\n  timeout: number;\n  httpAgent: Agent | undefined;\n\n  private fetch: Fetch;\n  protected idempotencyHeader?: string;\n\n  constructor({\n    baseURL,\n    maxRetries = 2,\n    timeout = 600000, // 10 minutes\n    httpAgent,\n    fetch: overridenFetch,\n  }: {\n    baseURL: string;\n    maxRetries?: number | undefined;\n    timeout: number | undefined;\n    httpAgent: Agent | undefined;\n    fetch: Fetch | undefined;\n  }) {\n    this.baseURL = baseURL;\n    this.maxRetries = validatePositiveInteger('maxRetries', maxRetries);\n    this.timeout = validatePositiveInteger('timeout', timeout);\n    this.httpAgent = httpAgent;\n\n    this.fetch = overridenFetch ?? fetch;\n  }\n\n  protected authHeaders(opts: FinalRequestOptions): Headers {\n    return {};\n  }\n\n  /**\n   * Override this to add your own default headers, for example:\n   *\n   *  {\n   *    ...super.defaultHeaders(),\n   *    Authorization: 'Bearer 123',\n   *  }\n   */\n  protected defaultHeaders(opts: FinalRequestOptions): Headers {\n    return {\n      Accept: 'application/json',\n      'Content-Type': 'application/json',\n      'User-Agent': this.getUserAgent(),\n      ...getPlatformHeaders(),\n      ...this.authHeaders(opts),\n    };\n  }\n\n  protected abstract defaultQuery(): DefaultQuery | undefined;\n\n  /**\n   * Override this to add your own headers validation:\n   */\n  protected validateHeaders(headers: Headers, customHeaders: Headers) {}\n\n  protected defaultIdempotencyKey(): string {\n    return `stainless-node-retry-${uuid4()}`;\n  }\n\n  get<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('get', path, opts);\n  }\n\n  post<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('post', path, opts);\n  }\n\n  patch<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('patch', path, opts);\n  }\n\n  put<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('put', path, opts);\n  }\n\n  delete<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('delete', path, opts);\n  }\n\n  private methodRequest<Req, Rsp>(\n    method: HTTPMethod,\n    path: string,\n    opts?: PromiseOrValue<RequestOptions<Req>>,\n  ): APIPromise<Rsp> {\n    return this.request(Promise.resolve(opts).then((opts) => ({ method, path, ...opts })));\n  }\n\n  getAPIList<Item, PageClass extends AbstractPage<Item> = AbstractPage<Item>>(\n    path: string,\n    Page: new (...args: any[]) => PageClass,\n    opts?: RequestOptions<any>,\n  ): PagePromise<PageClass, Item> {\n    return this.requestAPIList(Page, { method: 'get', path, ...opts });\n  }\n\n  private calculateContentLength(body: unknown): string | null {\n    if (typeof body === 'string') {\n      if (typeof Buffer !== 'undefined') {\n        return Buffer.byteLength(body, 'utf8').toString();\n      }\n\n      if (typeof TextEncoder !== 'undefined') {\n        const encoder = new TextEncoder();\n        const encoded = encoder.encode(body);\n        return encoded.length.toString();\n      }\n    }\n\n    return null;\n  }\n\n  buildRequest<Req>(options: FinalRequestOptions<Req>): { req: RequestInit; url: string; timeout: number } {\n    const { method, path, query, headers: headers = {} } = options;\n\n    const body =\n      isMultipartBody(options.body) ? options.body.body\n      : options.body ? JSON.stringify(options.body, null, 2)\n      : null;\n    const contentLength = this.calculateContentLength(body);\n\n    const url = this.buildURL(path!, query);\n    if ('timeout' in options) validatePositiveInteger('timeout', options.timeout);\n    const timeout = options.timeout ?? this.timeout;\n    const httpAgent = options.httpAgent ?? this.httpAgent ?? getDefaultAgent(url);\n    const minAgentTimeout = timeout + 1000;\n    if (\n      typeof (httpAgent as any)?.options?.timeout === 'number' &&\n      minAgentTimeout > ((httpAgent as any).options.timeout ?? 0)\n    ) {\n      // Allow any given request to bump our agent active socket timeout.\n      // This may seem strange, but leaking active sockets should be rare and not particularly problematic,\n      // and without mutating agent we would need to create more of them.\n      // This tradeoff optimizes for performance.\n      (httpAgent as any).options.timeout = minAgentTimeout;\n    }\n\n    if (this.idempotencyHeader && method !== 'get') {\n      if (!options.idempotencyKey) options.idempotencyKey = this.defaultIdempotencyKey();\n      headers[this.idempotencyHeader] = options.idempotencyKey;\n    }\n\n    const reqHeaders = this.buildHeaders({ options, headers, contentLength });\n\n    const req: RequestInit = {\n      method,\n      ...(body && { body: body as any }),\n      headers: reqHeaders,\n      ...(httpAgent && { agent: httpAgent }),\n      // @ts-ignore node-fetch uses a custom AbortSignal type that is\n      // not compatible with standard web types\n      signal: options.signal ?? null,\n    };\n\n    return { req, url, timeout };\n  }\n\n  private buildHeaders({\n    options,\n    headers,\n    contentLength,\n  }: {\n    options: FinalRequestOptions;\n    headers: Record<string, string | null | undefined>;\n    contentLength: string | null | undefined;\n  }): Record<string, string> {\n    const reqHeaders: Record<string, string> = {};\n    if (contentLength) {\n      reqHeaders['content-length'] = contentLength;\n    }\n\n    const defaultHeaders = this.defaultHeaders(options);\n    applyHeadersMut(reqHeaders, defaultHeaders);\n    applyHeadersMut(reqHeaders, headers);\n\n    // let builtin fetch set the Content-Type for multipart bodies\n    if (isMultipartBody(options.body) && shimsKind !== 'node') {\n      delete reqHeaders['content-type'];\n    }\n\n    this.validateHeaders(reqHeaders, headers);\n\n    return reqHeaders;\n  }\n\n  /**\n   * Used as a callback for mutating the given `FinalRequestOptions` object.\n   */\n  protected async prepareOptions(options: FinalRequestOptions): Promise<void> {}\n\n  /**\n   * Used as a callback for mutating the given `RequestInit` object.\n   *\n   * This is useful for cases where you want to add certain headers based off of\n   * the request properties, e.g. `method` or `url`.\n   */\n  protected async prepareRequest(\n    request: RequestInit,\n    { url, options }: { url: string; options: FinalRequestOptions },\n  ): Promise<void> {}\n\n  protected parseHeaders(headers: HeadersInit | null | undefined): Record<string, string> {\n    return (\n      !headers ? {}\n      : Symbol.iterator in headers ?\n        Object.fromEntries(Array.from(headers as Iterable<string[]>).map((header) => [...header]))\n      : { ...headers }\n    );\n  }\n\n  protected makeStatusError(\n    status: number | undefined,\n    error: Object | undefined,\n    message: string | undefined,\n    headers: Headers | undefined,\n  ) {\n    return APIError.generate(status, error, message, headers);\n  }\n\n  request<Req, Rsp>(\n    options: PromiseOrValue<FinalRequestOptions<Req>>,\n    remainingRetries: number | null = null,\n  ): APIPromise<Rsp> {\n    return new APIPromise(this.makeRequest(options, remainingRetries));\n  }\n\n  private async makeRequest<Req>(\n    optionsInput: PromiseOrValue<FinalRequestOptions<Req>>,\n    retriesRemaining: number | null,\n  ): Promise<APIResponseProps> {\n    const options = await optionsInput;\n    if (retriesRemaining == null) {\n      retriesRemaining = options.maxRetries ?? this.maxRetries;\n    }\n\n    await this.prepareOptions(options);\n\n    const { req, url, timeout } = this.buildRequest(options);\n\n    await this.prepareRequest(req, { url, options });\n\n    debug('request', url, options, req.headers);\n\n    if (options.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n\n    const controller = new AbortController();\n    const response = await this.fetchWithTimeout(url, req, timeout, controller).catch(castToError);\n\n    if (response instanceof Error) {\n      if (options.signal?.aborted) {\n        throw new APIUserAbortError();\n      }\n      if (retriesRemaining) {\n        return this.retryRequest(options, retriesRemaining);\n      }\n      if (response.name === 'AbortError') {\n        throw new APIConnectionTimeoutError();\n      }\n      throw new APIConnectionError({ cause: response });\n    }\n\n    const responseHeaders = createResponseHeaders(response.headers);\n\n    if (!response.ok) {\n      if (retriesRemaining && this.shouldRetry(response)) {\n        const retryMessage = `retrying, ${retriesRemaining} attempts remaining`;\n        debug(`response (error; ${retryMessage})`, response.status, url, responseHeaders);\n        return this.retryRequest(options, retriesRemaining, responseHeaders);\n      }\n\n      const errText = await response.text().catch((e) => castToError(e).message);\n      const errJSON = safeJSON(errText);\n      const errMessage = errJSON ? undefined : errText;\n      const retryMessage = retriesRemaining ? `(error; no more retries left)` : `(error; not retryable)`;\n\n      debug(`response (error; ${retryMessage})`, response.status, url, responseHeaders, errMessage);\n\n      const err = this.makeStatusError(response.status, errJSON, errMessage, responseHeaders);\n      throw err;\n    }\n\n    return { response, options, controller };\n  }\n\n  requestAPIList<Item = unknown, PageClass extends AbstractPage<Item> = AbstractPage<Item>>(\n    Page: new (...args: ConstructorParameters<typeof AbstractPage>) => PageClass,\n    options: FinalRequestOptions,\n  ): PagePromise<PageClass, Item> {\n    const request = this.makeRequest(options, null);\n    return new PagePromise<PageClass, Item>(this, request, Page);\n  }\n\n  buildURL<Req>(path: string, query: Req | null | undefined): string {\n    const url =\n      isAbsoluteURL(path) ?\n        new URL(path)\n      : new URL(this.baseURL + (this.baseURL.endsWith('/') && path.startsWith('/') ? path.slice(1) : path));\n\n    const defaultQuery = this.defaultQuery();\n    if (!isEmptyObj(defaultQuery)) {\n      query = { ...defaultQuery, ...query } as Req;\n    }\n\n    if (typeof query === 'object' && query && !Array.isArray(query)) {\n      url.search = this.stringifyQuery(query as Record<string, unknown>);\n    }\n\n    return url.toString();\n  }\n\n  protected stringifyQuery(query: Record<string, unknown>): string {\n    return Object.entries(query)\n      .filter(([_, value]) => typeof value !== 'undefined')\n      .map(([key, value]) => {\n        if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {\n          return `${encodeURIComponent(key)}=${encodeURIComponent(value)}`;\n        }\n        if (value === null) {\n          return `${encodeURIComponent(key)}=`;\n        }\n        throw new OpenAIError(\n          `Cannot stringify type ${typeof value}; Expected string, number, boolean, or null. If you need to pass nested query parameters, you can manually encode them, e.g. { query: { 'foo[key1]': value1, 'foo[key2]': value2 } }, and please open a GitHub issue requesting better support for your use case.`,\n        );\n      })\n      .join('&');\n  }\n\n  async fetchWithTimeout(\n    url: RequestInfo,\n    init: RequestInit | undefined,\n    ms: number,\n    controller: AbortController,\n  ): Promise<Response> {\n    const { signal, ...options } = init || {};\n    if (signal) signal.addEventListener('abort', () => controller.abort());\n\n    const timeout = setTimeout(() => controller.abort(), ms);\n\n    return (\n      this.getRequestClient()\n        // use undefined this binding; fetch errors if bound to something else in browser/cloudflare\n        .fetch.call(undefined, url, { signal: controller.signal as any, ...options })\n        .finally(() => {\n          clearTimeout(timeout);\n        })\n    );\n  }\n\n  protected getRequestClient(): RequestClient {\n    return { fetch: this.fetch };\n  }\n\n  private shouldRetry(response: Response): boolean {\n    // Note this is not a standard header.\n    const shouldRetryHeader = response.headers.get('x-should-retry');\n\n    // If the server explicitly says whether or not to retry, obey.\n    if (shouldRetryHeader === 'true') return true;\n    if (shouldRetryHeader === 'false') return false;\n\n    // Retry on request timeouts.\n    if (response.status === 408) return true;\n\n    // Retry on lock timeouts.\n    if (response.status === 409) return true;\n\n    // Retry on rate limits.\n    if (response.status === 429) return true;\n\n    // Retry internal errors.\n    if (response.status >= 500) return true;\n\n    return false;\n  }\n\n  private async retryRequest(\n    options: FinalRequestOptions,\n    retriesRemaining: number,\n    responseHeaders?: Headers | undefined,\n  ): Promise<APIResponseProps> {\n    let timeoutMillis: number | undefined;\n\n    // Note the `retry-after-ms` header may not be standard, but is a good idea and we'd like proactive support for it.\n    const retryAfterMillisHeader = responseHeaders?.['retry-after-ms'];\n    if (retryAfterMillisHeader) {\n      const timeoutMs = parseFloat(retryAfterMillisHeader);\n      if (!Number.isNaN(timeoutMs)) {\n        timeoutMillis = timeoutMs;\n      }\n    }\n\n    // About the Retry-After header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After\n    const retryAfterHeader = responseHeaders?.['retry-after'];\n    if (retryAfterHeader && !timeoutMillis) {\n      const timeoutSeconds = parseFloat(retryAfterHeader);\n      if (!Number.isNaN(timeoutSeconds)) {\n        timeoutMillis = timeoutSeconds * 1000;\n      } else {\n        timeoutMillis = Date.parse(retryAfterHeader) - Date.now();\n      }\n    }\n\n    // If the API asks us to wait a certain amount of time (and it's a reasonable amount),\n    // just do what it says, but otherwise calculate a default\n    if (!(timeoutMillis && 0 <= timeoutMillis && timeoutMillis < 60 * 1000)) {\n      const maxRetries = options.maxRetries ?? this.maxRetries;\n      timeoutMillis = this.calculateDefaultRetryTimeoutMillis(retriesRemaining, maxRetries);\n    }\n    await sleep(timeoutMillis);\n\n    return this.makeRequest(options, retriesRemaining - 1);\n  }\n\n  private calculateDefaultRetryTimeoutMillis(retriesRemaining: number, maxRetries: number): number {\n    const initialRetryDelay = 0.5;\n    const maxRetryDelay = 8.0;\n\n    const numRetries = maxRetries - retriesRemaining;\n\n    // Apply exponential backoff, but not more than the max.\n    const sleepSeconds = Math.min(initialRetryDelay * Math.pow(2, numRetries), maxRetryDelay);\n\n    // Apply some jitter, take up to at most 25 percent of the retry time.\n    const jitter = 1 - Math.random() * 0.25;\n\n    return sleepSeconds * jitter * 1000;\n  }\n\n  private getUserAgent(): string {\n    return `${this.constructor.name}/JS ${VERSION}`;\n  }\n}\n\nexport type PageInfo = { url: URL } | { params: Record<string, unknown> | null };\n\nexport abstract class AbstractPage<Item> implements AsyncIterable<Item> {\n  #client: APIClient;\n  protected options: FinalRequestOptions;\n\n  protected response: Response;\n  protected body: unknown;\n\n  constructor(client: APIClient, response: Response, body: unknown, options: FinalRequestOptions) {\n    this.#client = client;\n    this.options = options;\n    this.response = response;\n    this.body = body;\n  }\n\n  /**\n   * @deprecated Use nextPageInfo instead\n   */\n  abstract nextPageParams(): Partial<Record<string, unknown>> | null;\n  abstract nextPageInfo(): PageInfo | null;\n\n  abstract getPaginatedItems(): Item[];\n\n  hasNextPage(): boolean {\n    const items = this.getPaginatedItems();\n    if (!items.length) return false;\n    return this.nextPageInfo() != null;\n  }\n\n  async getNextPage(): Promise<this> {\n    const nextInfo = this.nextPageInfo();\n    if (!nextInfo) {\n      throw new OpenAIError(\n        'No next page expected; please check `.hasNextPage()` before calling `.getNextPage()`.',\n      );\n    }\n    const nextOptions = { ...this.options };\n    if ('params' in nextInfo && typeof nextOptions.query === 'object') {\n      nextOptions.query = { ...nextOptions.query, ...nextInfo.params };\n    } else if ('url' in nextInfo) {\n      const params = [...Object.entries(nextOptions.query || {}), ...nextInfo.url.searchParams.entries()];\n      for (const [key, value] of params) {\n        nextInfo.url.searchParams.set(key, value as any);\n      }\n      nextOptions.query = undefined;\n      nextOptions.path = nextInfo.url.toString();\n    }\n    return await this.#client.requestAPIList(this.constructor as any, nextOptions);\n  }\n\n  async *iterPages() {\n    // eslint-disable-next-line @typescript-eslint/no-this-alias\n    let page: AbstractPage<Item> = this;\n    yield page;\n    while (page.hasNextPage()) {\n      page = await page.getNextPage();\n      yield page;\n    }\n  }\n\n  async *[Symbol.asyncIterator]() {\n    for await (const page of this.iterPages()) {\n      for (const item of page.getPaginatedItems()) {\n        yield item;\n      }\n    }\n  }\n}\n\n/**\n * This subclass of Promise will resolve to an instantiated Page once the request completes.\n *\n * It also implements AsyncIterable to allow auto-paginating iteration on an unawaited list call, eg:\n *\n *    for await (const item of client.items.list()) {\n *      console.log(item)\n *    }\n */\nexport class PagePromise<\n    PageClass extends AbstractPage<Item>,\n    Item = ReturnType<PageClass['getPaginatedItems']>[number],\n  >\n  extends APIPromise<PageClass>\n  implements AsyncIterable<Item>\n{\n  constructor(\n    client: APIClient,\n    request: Promise<APIResponseProps>,\n    Page: new (...args: ConstructorParameters<typeof AbstractPage>) => PageClass,\n  ) {\n    super(\n      request,\n      async (props) => new Page(client, props.response, await defaultParseResponse(props), props.options),\n    );\n  }\n\n  /**\n   * Allow auto-paginating iteration on an unawaited list call, eg:\n   *\n   *    for await (const item of client.items.list()) {\n   *      console.log(item)\n   *    }\n   */\n  async *[Symbol.asyncIterator]() {\n    const page = await this;\n    for await (const item of page) {\n      yield item;\n    }\n  }\n}\n\nexport const createResponseHeaders = (\n  headers: Awaited<ReturnType<Fetch>>['headers'],\n): Record<string, string> => {\n  return new Proxy(\n    Object.fromEntries(\n      // @ts-ignore\n      headers.entries(),\n    ),\n    {\n      get(target, name) {\n        const key = name.toString();\n        return target[key.toLowerCase()] || target[key];\n      },\n    },\n  );\n};\n\ntype HTTPMethod = 'get' | 'post' | 'put' | 'patch' | 'delete';\n\nexport type RequestClient = { fetch: Fetch };\nexport type Headers = Record<string, string | null | undefined>;\nexport type DefaultQuery = Record<string, string | undefined>;\nexport type KeysEnum<T> = { [P in keyof Required<T>]: true };\n\nexport type RequestOptions<Req = unknown | Record<string, unknown> | Readable> = {\n  method?: HTTPMethod;\n  path?: string;\n  query?: Req | undefined;\n  body?: Req | null | undefined;\n  headers?: Headers | undefined;\n\n  maxRetries?: number;\n  stream?: boolean | undefined;\n  timeout?: number;\n  httpAgent?: Agent;\n  signal?: AbortSignal | undefined | null;\n  idempotencyKey?: string;\n\n  __binaryResponse?: boolean | undefined;\n  __streamClass?: typeof Stream;\n};\n\n// This is required so that we can determine if a given object matches the RequestOptions\n// type at runtime. While this requires duplication, it is enforced by the TypeScript\n// compiler such that any missing / extraneous keys will cause an error.\nconst requestOptionsKeys: KeysEnum<RequestOptions> = {\n  method: true,\n  path: true,\n  query: true,\n  body: true,\n  headers: true,\n\n  maxRetries: true,\n  stream: true,\n  timeout: true,\n  httpAgent: true,\n  signal: true,\n  idempotencyKey: true,\n\n  __binaryResponse: true,\n  __streamClass: true,\n};\n\nexport const isRequestOptions = (obj: unknown): obj is RequestOptions => {\n  return (\n    typeof obj === 'object' &&\n    obj !== null &&\n    !isEmptyObj(obj) &&\n...\n", "needle_token_start": 2508, "needle_token_end": 2791, "code_context_ntokens": 11505, "output": ["```typescript\n/**\n * Helper for creating a {@link File} to pass to an SDK upload method from a variety of different data formats\n * @param value the raw content of the file.  Can be an {@link Uploadable}, {@link BlobLikePart}, or {@link AsyncIterable} of {@link BlobLikePart}s\n * @param {string=} name the name of the file. If omitted, toFile will try to determine a file name from bits if possible\n * @param {Object=} options additional properties\n * @param {string=} options.type the MIME type of the content\n * @param {number=} options.lastModified the last modified timestamp\n * @returns a {@link File} with the given properties\n */\nexport \nasync function toFile(\n  value: ToFileInput | PromiseLike<ToFileInput>,\n  name?: string | null | undefined,\n  options?: FilePropertyBag | undefined,\n): Promise<FileLike> {\n  // If it's a promise, resolve it.\n  value = await value;\n\n  // Use the file's options if there isn't one provided\n  options ??= isFileLike(value) ? { lastModified: value.lastModified, type: value.type } : {};\n\n  if (isResponseLike(value)) {\n    const blob = await value.blob();\n    name ||= new URL(value.url).pathname.split(/[\\\\/]/).pop() ?? 'unknown_file';\n\n    return new File([blob as any], name, options);\n  }\n\n  const bits = await getBytes(value);\n\n  name ||= getName(value) ?? 'unknown_file';\n\n  if (!options.type) {\n    const type = (bits[0] as any)?.type;\n    if (typeof type === 'string') {\n      options = { ...options, type };\n    }\n  }\n\n  return new File(bits, name, options);\n}\n\nasync function getBytes(value: ToFileInput): Promise<Array<BlobPart>> {\n  let parts: Array<BlobPart> = [];\n  if (\n    typeof value === 'string' ||\n    ArrayBuffer.isView(value) || // includes Uint8Array, Buffer, etc.\n    value instanceof ArrayBuffer\n  ) {\n    parts.push(value);\n  } else if (isBlobLike(value)) {\n    parts.push(await value.arrayBuffer());\n  } else if (\n    isAsyncIterableIterator(value) // includes Readable, ReadableStream, etc.\n  ) {\n    for await (const chunk of value) {\n      parts.push(chunk as BlobPart); // TODO, consider validating?\n    }\n  } else {\n    throw new Error(\n      `Unexpected data type: ${typeof value}; constructor: ${value?.constructor\n        ?.name}; props: ${propsForError(value)}`,\n    );\n  }\n\n  return parts;\n}\n\nfunction propsForError(value: any): string {\n  const props = Object.getOwnPropertyNames(value);\n  return `[${props.map((p) => `\"${p}\"`).join(', ')}]`;\n}\n\nfunction getName(value: any): string | undefined {\n  return (\n    getStringFromMaybeBuffer(value.name) ||\n    getStringFromMaybeBuffer(value.filename) ||\n    // For fs.ReadStream\n    getStringFromMaybeBuffer(value.path)?.split(/[\\\\/]/).pop()\n  );\n}\n\nconst getStringFromMaybeBuffer = (x: string | Buffer | unknown): string | undefined => {\n  if (typeof x === 'string') return x;\n  if (typeof Buffer !== 'undefined' && x instanceof Buffer) return String(x);\n  return undefined;\n};\n\nconst isAsyncIterableIterator = (value: any): value is AsyncIterableIterator<unknown> =>\n  value != null && typeof value === 'object' && typeof value[Symbol.asyncIterator] === 'function';\n```"]}
{"repo": "openai/openai-node", "name": "isPresent", "language": "typescript", "path": "src/lib/chatCompletionUtils.ts", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function checks if a given value is neither null nor undefined.\n2. **Input**: A value of any type, potentially null or undefined.\n3. **Output**: A boolean value indicating whether the input is not null and not undefined.\n4. **Procedure**: The function evaluates whether the input is not equal to null or undefined using a strict inequality check and returns the result as a boolean.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/resources/beta/assistants/assistants.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { isRequestOptions } from 'openai/core';\nimport * as AssistantsAPI from 'openai/resources/beta/assistants/assistants';\nimport * as Shared from 'openai/resources/shared';\nimport * as FilesAPI from 'openai/resources/beta/assistants/files';\nimport * as ThreadsAPI from 'openai/resources/beta/threads/threads';\nimport * as MessagesAPI from 'openai/resources/beta/threads/messages/messages';\nimport * as RunsAPI from 'openai/resources/beta/threads/runs/runs';\nimport * as StepsAPI from 'openai/resources/beta/threads/runs/steps';\nimport { CursorPage, type CursorPageParams } from 'openai/pagination';\n\nexport class Assistants extends APIResource {\n  files: FilesAPI.Files = new FilesAPI.Files(this._client);\n\n  /**\n   * Create an assistant with a model and instructions.\n   */\n  create(body: AssistantCreateParams, options?: Core.RequestOptions): Core.APIPromise<Assistant> {\n    return this._client.post('/assistants', {\n      body,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Retrieves an assistant.\n   */\n  retrieve(assistantId: string, options?: Core.RequestOptions): Core.APIPromise<Assistant> {\n    return this._client.get(`/assistants/${assistantId}`, {\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Modifies an assistant.\n   */\n  update(\n    assistantId: string,\n    body: AssistantUpdateParams,\n    options?: Core.RequestOptions,\n  ): Core.APIPromise<Assistant> {\n    return this._client.post(`/assistants/${assistantId}`, {\n      body,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Returns a list of assistants.\n   */\n  list(\n    query?: AssistantListParams,\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<AssistantsPage, Assistant>;\n  list(options?: Core.RequestOptions): Core.PagePromise<AssistantsPage, Assistant>;\n  list(\n    query: AssistantListParams | Core.RequestOptions = {},\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<AssistantsPage, Assistant> {\n    if (isRequestOptions(query)) {\n      return this.list({}, query);\n    }\n...\n// Path: src/lib/jsonschema.ts\n// File mostly copied from @types/json-schema, but stripped down a bit for brevity\n// https://github.com/DefinitelyTyped/DefinitelyTyped/blob/817274f3280152ba2929a6067c93df8b34c4c9aa/types/json-schema/index.d.ts\n//\n// ==================================================================================================\n// JSON Schema Draft 07\n// ==================================================================================================\n// https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n// --------------------------------------------------------------------------------------------------\n\n/**\n * Primitive type\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1.1\n */\nexport type JSONSchemaTypeName =\n  | ({} & string)\n  | 'string'\n  | 'number'\n  | 'integer'\n  | 'boolean'\n  | 'object'\n  | 'array'\n  | 'null';\n\n/**\n * Primitive type\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1.1\n */\nexport type JSONSchemaType =\n  | string //\n  | number\n  | boolean\n  | JSONSchemaObject\n  | JSONSchemaArray\n  | null;\n\n// Workaround for infinite type recursion\nexport interface JSONSchemaObject {\n  [key: string]: JSONSchemaType;\n}\n\n// Workaround for infinite type recursion\n// https://github.com/Microsoft/TypeScript/issues/3496#issuecomment-128553540\nexport interface JSONSchemaArray extends Array<JSONSchemaType> {}\n\n/**\n * Meta schema\n *\n * Recommended values:\n * - 'http://json-schema.org/schema#'\n * - 'http://json-schema.org/hyper-schema#'\n * - 'http://json-schema.org/draft-07/schema#'\n * - 'http://json-schema.org/draft-07/hyper-schema#'\n *\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-5\n */\nexport type JSONSchemaVersion = string;\n\n/**\n * JSON Schema v7\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n */\nexport type JSONSchemaDefinition = JSONSchema | boolean;\nexport interface JSONSchema {\n  $id?: string | undefined;\n  $comment?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1\n   */\n  type?: JSONSchemaTypeName | JSONSchemaTypeName[] | undefined;\n  enum?: JSONSchemaType[] | undefined;\n  const?: JSONSchemaType | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.2\n   */\n  multipleOf?: number | undefined;\n  maximum?: number | undefined;\n  exclusiveMaximum?: number | undefined;\n  minimum?: number | undefined;\n  exclusiveMinimum?: number | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.3\n   */\n  maxLength?: number | undefined;\n  minLength?: number | undefined;\n  pattern?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.4\n   */\n  items?: JSONSchemaDefinition | JSONSchemaDefinition[] | undefined;\n  additionalItems?: JSONSchemaDefinition | undefined;\n  maxItems?: number | undefined;\n  minItems?: number | undefined;\n  uniqueItems?: boolean | undefined;\n  contains?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.5\n   */\n  maxProperties?: number | undefined;\n  minProperties?: number | undefined;\n  required?: string[] | undefined;\n  properties?:\n    | {\n        [key: string]: JSONSchemaDefinition;\n      }\n    | undefined;\n  patternProperties?:\n    | {\n        [key: string]: JSONSchemaDefinition;\n      }\n    | undefined;\n  additionalProperties?: JSONSchemaDefinition | undefined;\n  propertyNames?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.6\n   */\n  if?: JSONSchemaDefinition | undefined;\n  then?: JSONSchemaDefinition | undefined;\n  else?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.7\n   */\n  allOf?: JSONSchemaDefinition[] | undefined;\n  anyOf?: JSONSchemaDefinition[] | undefined;\n  oneOf?: JSONSchemaDefinition[] | undefined;\n  not?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-7\n   */\n  format?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-10\n   */\n  title?: string | undefined;\n  description?: string | undefined;\n  default?: JSONSchemaType | undefined;\n  readOnly?: boolean | undefined;\n  writeOnly?: boolean | undefined;\n  examples?: JSONSchemaType | undefined;\n}\n\n// Path: src/lib/ChatCompletionStream.ts\nimport * as Core from 'openai/core';\nimport { OpenAIError, APIUserAbortError } from 'openai/error';\nimport {\n  Completions,\n  type ChatCompletion,\n  type ChatCompletionChunk,\n  type ChatCompletionCreateParams,\n  type ChatCompletionCreateParamsBase,\n} from 'openai/resources/chat/completions';\nimport {\n  AbstractChatCompletionRunner,\n  type AbstractChatCompletionRunnerEvents,\n} from './AbstractChatCompletionRunner';\nimport { type ReadableStream } from 'openai/_shims/index';\nimport { Stream } from 'openai/streaming';\n\nexport interface ChatCompletionStreamEvents extends AbstractChatCompletionRunnerEvents {\n  content: (contentDelta: string, contentSnapshot: string) => void;\n  chunk: (chunk: ChatCompletionChunk, snapshot: ChatCompletionSnapshot) => void;\n}\n\nexport type ChatCompletionStreamParams = Omit<ChatCompletionCreateParamsBase, 'stream'> & {\n  stream?: true;\n};\n\nexport class ChatCompletionStream\n  extends AbstractChatCompletionRunner<ChatCompletionStreamEvents>\n  implements AsyncIterable<ChatCompletionChunk>\n{\n  #currentChatCompletionSnapshot: ChatCompletionSnapshot | undefined;\n\n  get currentChatCompletionSnapshot(): ChatCompletionSnapshot | undefined {\n    return this.#currentChatCompletionSnapshot;\n  }\n\n  /**\n   * Intended for use on the frontend, consuming a stream produced with\n   * `.toReadableStream()` on the backend.\n   *\n   * Note that messages sent to the model do not appear in `.on('message')`\n   * in this context.\n   */\n  static fromReadableStream(stream: ReadableStream): ChatCompletionStream {\n    const runner = new ChatCompletionStream();\n    runner._run(() => runner._fromReadableStream(stream));\n    return runner;\n  }\n\n  static createChatCompletion(\n    completions: Completions,\n    params: ChatCompletionStreamParams,\n    options?: Core.RequestOptions,\n  ): ChatCompletionStream {\n    const runner = new ChatCompletionStream();\n    runner._run(() =>\n      runner._runChatCompletion(\n        completions,\n        { ...params, stream: true },\n        { ...options, headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'stream' } },\n      ),\n    );\n    return runner;\n  }\n\n  #beginRequest() {\n    if (this.ended) return;\n    this.#currentChatCompletionSnapshot = undefined;\n  }\n  #addChunk(chunk: ChatCompletionChunk) {\n    if (this.ended) return;\n    const completion = this.#accumulateChatCompletion(chunk);\n    this._emit('chunk', chunk, completion);\n    const delta = chunk.choices[0]?.delta?.content;\n    const snapshot = completion.choices[0]?.message;\n    if (delta != null && snapshot?.role === 'assistant' && snapshot?.content) {\n      this._emit('content', delta, snapshot.content);\n    }\n  }\n  #endRequest(): ChatCompletion {\n    if (this.ended) {\n      throw new OpenAIError(`stream has ended, this shouldn't happen`);\n    }\n    const snapshot = this.#currentChatCompletionSnapshot;\n    if (!snapshot) {\n      throw new OpenAIError(`request ended without sending any chunks`);\n    }\n    this.#currentChatCompletionSnapshot = undefined;\n    return finalizeChatCompletion(snapshot);\n  }\n\n  protected override async _createChatCompletion(\n    completions: Completions,\n    params: ChatCompletionCreateParams,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    const signal = options?.signal;\n    if (signal) {\n      if (signal.aborted) this.controller.abort();\n      signal.addEventListener('abort', () => this.controller.abort());\n    }\n    this.#beginRequest();\n    const stream = await completions.create(\n      { ...params, stream: true },\n      { ...options, signal: this.controller.signal },\n    );\n    this._connected();\n    for await (const chunk of stream) {\n      this.#addChunk(chunk);\n    }\n    if (stream.controller.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n    return this._addChatCompletion(this.#endRequest());\n  }\n\n  protected async _fromReadableStream(\n    readableStream: ReadableStream,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    const signal = options?.signal;\n    if (signal) {\n      if (signal.aborted) this.controller.abort();\n      signal.addEventListener('abort', () => this.controller.abort());\n    }\n    this.#beginRequest();\n    this._connected();\n    const stream = Stream.fromReadableStream<ChatCompletionChunk>(readableStream, this.controller);\n    let chatId;\n    for await (const chunk of stream) {\n      if (chatId && chatId !== chunk.id) {\n        // A new request has been made.\n        this._addChatCompletion(this.#endRequest());\n      }\n\n      this.#addChunk(chunk);\n      chatId = chunk.id;\n    }\n    if (stream.controller.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n    return this._addChatCompletion(this.#endRequest());\n  }\n\n  #accumulateChatCompletion(chunk: ChatCompletionChunk): ChatCompletionSnapshot {\n    let snapshot = this.#currentChatCompletionSnapshot;\n    const { choices, ...rest } = chunk;\n    if (!snapshot) {\n      snapshot = this.#currentChatCompletionSnapshot = {\n        ...rest,\n        choices: [],\n      };\n    } else {\n      Object.assign(snapshot, rest);\n    }\n\n    for (const { delta, finish_reason, index, logprobs = null, ...other } of chunk.choices) {\n      let choice = snapshot.choices[index];\n      if (!choice) {\n        choice = snapshot.choices[index] = { finish_reason, index, message: {}, logprobs, ...other };\n      }\n\n      if (logprobs) {\n        if (!choice.logprobs) {\n          choice.logprobs = Object.assign({}, logprobs);\n        } else {\n          const { content, ...rest } = logprobs;\n          Object.assign(choice.logprobs, rest);\n          if (content) {\n            choice.logprobs.content ??= [];\n            choice.logprobs.content.push(...content);\n          }\n        }\n      }\n\n      if (finish_reason) choice.finish_reason = finish_reason;\n      Object.assign(choice, other);\n\n      if (!delta) continue; // Shouldn't happen; just in case.\n      const { content, function_call, role, tool_calls, ...rest } = delta;\n      Object.assign(choice.message, rest);\n\n      if (content) choice.message.content = (choice.message.content || '') + content;\n      if (role) choice.message.role = role;\n      if (function_call) {\n        if (!choice.message.function_call) {\n          choice.message.function_call = function_call;\n        } else {\n          if (function_call.name) choice.message.function_call.name = function_call.name;\n          if (function_call.arguments) {\n            choice.message.function_call.arguments ??= '';\n            choice.message.function_call.arguments += function_call.arguments;\n          }\n        }\n      }\n      if (tool_calls) {\n        if (!choice.message.tool_calls) choice.message.tool_calls = [];\n        for (const { index, id, type, function: fn, ...rest } of tool_calls) {\n          const tool_call = (choice.message.tool_calls[index] ??= {});\n          Object.assign(tool_call, rest);\n          if (id) tool_call.id = id;\n          if (type) tool_call.type = type;\n          if (fn) tool_call.function ??= { arguments: '' };\n          if (fn?.name) tool_call.function!.name = fn.name;\n          if (fn?.arguments) tool_call.function!.arguments += fn.arguments;\n        }\n      }\n    }\n    return snapshot;\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<ChatCompletionChunk> {\n    const pushQueue: ChatCompletionChunk[] = [];\n    const readQueue: {\n      resolve: (chunk: ChatCompletionChunk | undefined) => void;\n      reject: (err: unknown) => void;\n    }[] = [];\n    let done = false;\n\n    this.on('chunk', (chunk) => {\n      const reader = readQueue.shift();\n      if (reader) {\n        reader.resolve(chunk);\n      } else {\n        pushQueue.push(chunk);\n      }\n    });\n\n    this.on('end', () => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.resolve(undefined);\n      }\n      readQueue.length = 0;\n    });\n\n    this.on('abort', (err) => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.reject(err);\n      }\n      readQueue.length = 0;\n    });\n\n    this.on('error', (err) => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.reject(err);\n      }\n      readQueue.length = 0;\n    });\n\n    return {\n      next: async (): Promise<IteratorResult<ChatCompletionChunk>> => {\n        if (!pushQueue.length) {\n          if (done) {\n            return { value: undefined, done: true };\n          }\n          return new Promise<ChatCompletionChunk | undefined>((resolve, reject) =>\n            readQueue.push({ resolve, reject }),\n          ).then((chunk) => (chunk ? { value: chunk, done: false } : { value: undefined, done: true }));\n        }\n        const chunk = pushQueue.shift()!;\n        return { value: chunk, done: false };\n      },\n      return: async () => {\n        this.abort();\n        return { value: undefined, done: true };\n      },\n    };\n  }\n\n  toReadableStream(): ReadableStream {\n    const stream = new Stream(this[Symbol.asyncIterator].bind(this), this.controller);\n    return stream.toReadableStream();\n  }\n}\n\nfunction finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletion {\n  const { id, choices, created, model, system_fingerprint, ...rest } = snapshot;\n  return {\n    ...rest,\n    id,\n    choices: choices.map(\n      ({ message, finish_reason, index, logprobs, ...choiceRest }): ChatCompletion.Choice => {\n        if (!finish_reason) throw new OpenAIError(`missing finish_reason for choice ${index}`);\n        const { content = null, function_call, tool_calls, ...messageRest } = message;\n        const role = message.role as 'assistant'; // this is what we expect; in theory it could be different which would make our types a slight lie but would be fine.\n        if (!role) throw new OpenAIError(`missing role for choice ${index}`);\n        if (function_call) {\n          const { arguments: args, name } = function_call;\n          if (args == null) throw new OpenAIError(`missing function_call.arguments for choice ${index}`);\n          if (!name) throw new OpenAIError(`missing function_call.name for choice ${index}`);\n          return {\n            ...choiceRest,\n            message: { content, function_call: { arguments: args, name }, role },\n            finish_reason,\n            index,\n            logprobs,\n          };\n        }\n        if (tool_calls) {\n          return {\n            ...choiceRest,\n            index,\n            finish_reason,\n            logprobs,\n            message: {\n              ...messageRest,\n              role,\n              content,\n              tool_calls: tool_calls.map((tool_call, i) => {\n                const { function: fn, type, id, ...toolRest } = tool_call;\n                const { arguments: args, name, ...fnRest } = fn || {};\n                if (id == null)\n                  throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].id\\n${str(snapshot)}`);\n                if (type == null)\n                  throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].type\\n${str(snapshot)}`);\n                if (name == null)\n                  throw new OpenAIError(\n                    `missing choices[${index}].tool_calls[${i}].function.name\\n${str(snapshot)}`,\n                  );\n                if (args == null)\n                  throw new OpenAIError(\n                    `missing choices[${index}].tool_calls[${i}].function.arguments\\n${str(snapshot)}`,\n                  );\n\n                return { ...toolRest, id, type, function: { ...fnRest, name, arguments: args } };\n              }),\n            },\n          };\n        }\n        return {\n          ...choiceRest,\n          message: { ...messageRest, content, role },\n          finish_reason,\n          index,\n          logprobs,\n        };\n      },\n    ),\n    created,\n    model,\n    object: 'chat.completion',\n    ...(system_fingerprint ? { system_fingerprint } : {}),\n  };\n}\n\nfunction str(x: unknown) {\n  return JSON.stringify(x);\n}\n\n/**\n * Represents a streamed chunk of a chat completion response returned by model,\n * based on the provided input.\n */\nexport interface ChatCompletionSnapshot {\n  /**\n   * A unique identifier for the chat completion.\n   */\n  id: string;\n\n  /**\n   * A list of chat completion choices. Can be more than one if `n` is greater\n   * than 1.\n   */\n  choices: Array<ChatCompletionSnapshot.Choice>;\n\n  /**\n   * The Unix timestamp (in seconds) of when the chat completion was created.\n   */\n  created: number;\n\n  /**\n   * The model to generate the completion.\n   */\n  model: string;\n\n  // Note we do not include an \"object\" type on the snapshot,\n  // because the object is not a valid \"chat.completion\" until finalized.\n  // object: 'chat.completion';\n\n  /**\n   * This fingerprint represents the backend configuration that the model runs with.\n   *\n   * Can be used in conjunction with the `seed` request parameter to understand when\n   * backend changes have been made that might impact determinism.\n   */\n  system_fingerprint?: string;\n}\n\nexport namespace ChatCompletionSnapshot {\n  export interface Choice {\n    /**\n     * A chat completion delta generated by streamed model responses.\n     */\n    message: Choice.Message;\n\n    /**\n     * The reason the model stopped generating tokens. This will be `stop` if the model\n     * hit a natural stop point or a provided stop sequence, `length` if the maximum\n     * number of tokens specified in the request was reached, `content_filter` if\n     * content was omitted due to a flag from our content filters, or `function_call`\n     * if the model called a function.\n     */\n    finish_reason: ChatCompletion.Choice['finish_reason'] | null;\n\n    /**\n     * Log probability information for the choice.\n     */\n    logprobs: ChatCompletion.Choice.Logprobs | null;\n\n    /**\n     * The index of the choice in the list of choices.\n     */\n    index: number;\n  }\n\n  export namespace Choice {\n    /**\n     * A chat completion delta generated by streamed model responses.\n     */\n    export interface Message {\n      /**\n       * The contents of the chunk message.\n       */\n      content?: string | null;\n\n      /**\n       * The name and arguments of a function that should be called, as generated by the\n       * model.\n       */\n      function_call?: Message.FunctionCall;\n\n      tool_calls?: Array<Message.ToolCall>;\n\n      /**\n       * The role of the author of this message.\n       */\n      role?: 'system' | 'user' | 'assistant' | 'function' | 'tool';\n    }\n\n    export namespace Message {\n      export interface ToolCall {\n        /**\n         * The ID of the tool call.\n         */\n        id?: string;\n\n        function?: ToolCall.Function;\n\n        /**\n         * The type of the tool.\n         */\n        type?: 'function';\n      }\n\n      export namespace ToolCall {\n        export interface Function {\n          /**\n           * The arguments to call the function with, as generated by the model in JSON\n           * format. Note that the model does not always generate valid JSON, and may\n           * hallucinate parameters not defined by your function schema. Validate the\n           * arguments in your code before calling your function.\n           */\n          arguments?: string;\n\n          /**\n           * The name of the function to call.\n           */\n          name?: string;\n        }\n      }\n\n      /**\n       * The name and arguments of a function that should be called, as generated by the\n       * model.\n       */\n      export interface FunctionCall {\n        /**\n         * The arguments to call the function with, as generated by the model in JSON\n         * format. Note that the model does not always generate valid JSON, and may\n         * hallucinate parameters not defined by your function schema. Validate the\n         * arguments in your code before calling your function.\n         */\n        arguments?: string;\n\n        /**\n         * The name of the function to call.\n         */\n        name?: string;\n      }\n    }\n  }\n}\n\n// Path: src/lib/ChatCompletionStreamingRunner.ts\nimport {\n  Completions,\n  type ChatCompletionChunk,\n  type ChatCompletionCreateParamsStreaming,\n} from 'openai/resources/chat/completions';\nimport { RunnerOptions, type AbstractChatCompletionRunnerEvents } from './AbstractChatCompletionRunner';\nimport { type ReadableStream } from 'openai/_shims/index';\nimport { RunnableTools, type BaseFunctionsArgs, type RunnableFunctions } from './RunnableFunction';\nimport { ChatCompletionSnapshot, ChatCompletionStream } from './ChatCompletionStream';\n\nexport interface ChatCompletionStreamEvents extends AbstractChatCompletionRunnerEvents {\n  content: (contentDelta: string, contentSnapshot: string) => void;\n  chunk: (chunk: ChatCompletionChunk, snapshot: ChatCompletionSnapshot) => void;\n}\n\nexport type ChatCompletionStreamingFunctionRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsStreaming,\n  'functions'\n> & {\n  functions: RunnableFunctions<FunctionsArgs>;\n};\n\nexport type ChatCompletionStreamingToolRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsStreaming,\n  'tools'\n> & {\n  tools: RunnableTools<FunctionsArgs>;\n};\n\nexport class ChatCompletionStreamingRunner\n  extends ChatCompletionStream\n  implements AsyncIterable<ChatCompletionChunk>\n{\n  static override fromReadableStream(stream: ReadableStream): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    runner._run(() => runner._fromReadableStream(stream));\n    return runner;\n  }\n\n  /** @deprecated - please use `runTools` instead. */\n  static runFunctions<T extends (string | object)[]>(\n    completions: Completions,\n    params: ChatCompletionStreamingFunctionRunnerParams<T>,\n    options?: RunnerOptions,\n  ): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    const opts = {\n      ...options,\n      headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'runFunctions' },\n    };\n    runner._run(() => runner._runFunctions(completions, params, opts));\n    return runner;\n  }\n\n  static runTools<T extends (string | object)[]>(\n    completions: Completions,\n    params: ChatCompletionStreamingToolRunnerParams<T>,\n    options?: RunnerOptions,\n  ): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    const opts = {\n      ...options,\n      headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'runTools' },\n    };\n    runner._run(() => runner._runTools(completions, params, opts));\n    return runner;\n  }\n}\n\n// Path: src/lib/RunnableFunction.ts\nimport { type ChatCompletionRunner } from './ChatCompletionRunner';\nimport { type ChatCompletionStreamingRunner } from './ChatCompletionStreamingRunner';\nimport { JSONSchema } from './jsonschema';\n\ntype PromiseOrValue<T> = T | Promise<T>;\n\nexport type RunnableFunctionWithParse<Args extends object> = {\n  /**\n   * @param args the return value from `parse`.\n   * @param runner the runner evaluating this callback.\n   * @returns a string to send back to OpenAI.\n   */\n  function: (\n    args: Args,\n    runner: ChatCompletionRunner | ChatCompletionStreamingRunner,\n  ) => PromiseOrValue<unknown>;\n  /**\n   * @param input the raw args from the OpenAI function call.\n   * @returns the parsed arguments to pass to `function`\n   */\n  parse: (input: string) => PromiseOrValue<Args>;\n  /**\n   * The parameters the function accepts, describes as a JSON Schema object.\n   */\n  parameters: JSONSchema;\n  /**\n   * A description of what the function does, used by the model to choose when and how to call the function.\n   */\n  description: string;\n  /**\n   * The name of the function to be called. Will default to function.name if omitted.\n   */\n  name?: string | undefined;\n};\n\nexport type RunnableFunctionWithoutParse = {\n  /**\n   * @param args the raw args from the OpenAI function call.\n   * @returns a string to send back to OpenAI\n   */\n  function: (\n    args: string,\n    runner: ChatCompletionRunner | ChatCompletionStreamingRunner,\n  ) => PromiseOrValue<unknown>;\n  /**\n   * The parameters the function accepts, describes as a JSON Schema object.\n   */\n  parameters: JSONSchema;\n  /**\n   * A description of what the function does, used by the model to choose when and how to call the function.\n   */\n  description: string;\n  /**\n   * The name of the function to be called. Will default to function.name if omitted.\n   */\n  name?: string | undefined;\n};\n\nexport type RunnableFunction<Args extends object | string> =\n  Args extends string ? RunnableFunctionWithoutParse\n  : Args extends object ? RunnableFunctionWithParse<Args>\n  : never;\n\nexport type RunnableToolFunction<Args extends object | string> =\n  Args extends string ? RunnableToolFunctionWithoutParse\n  : Args extends object ? RunnableToolFunctionWithParse<Args>\n  : never;\n\nexport type RunnableToolFunctionWithoutParse = {\n  type: 'function';\n  function: RunnableFunctionWithoutParse;\n};\nexport type RunnableToolFunctionWithParse<Args extends object> = {\n  type: 'function';\n  function: RunnableFunctionWithParse<Args>;\n};\n\nexport function isRunnableFunctionWithParse<Args extends object>(\n  fn: any,\n): fn is RunnableFunctionWithParse<Args> {\n  return typeof (fn as any).parse === 'function';\n}\n\nexport type BaseFunctionsArgs = readonly (object | string)[];\n\nexport type RunnableFunctions<FunctionsArgs extends BaseFunctionsArgs> =\n  [any[]] extends [FunctionsArgs] ? readonly RunnableFunction<any>[]\n  : {\n      [Index in keyof FunctionsArgs]: Index extends number ? RunnableFunction<FunctionsArgs[Index]>\n      : FunctionsArgs[Index];\n    };\n\nexport type RunnableTools<FunctionsArgs extends BaseFunctionsArgs> =\n  [any[]] extends [FunctionsArgs] ? readonly RunnableToolFunction<any>[]\n  : {\n      [Index in keyof FunctionsArgs]: Index extends number ? RunnableToolFunction<FunctionsArgs[Index]>\n      : FunctionsArgs[Index];\n    };\n\n/**\n * This is helper class for passing a `function` and `parse` where the `function`\n * argument type matches the `parse` return type.\n *\n * @deprecated - please use ParsingToolFunction instead.\n */\nexport class ParsingFunction<Args extends object> {\n  function: RunnableFunctionWithParse<Args>['function'];\n  parse: RunnableFunctionWithParse<Args>['parse'];\n  parameters: RunnableFunctionWithParse<Args>['parameters'];\n  description: RunnableFunctionWithParse<Args>['description'];\n  name?: RunnableFunctionWithParse<Args>['name'];\n\n  constructor(input: RunnableFunctionWithParse<Args>) {\n    this.function = input.function;\n    this.parse = input.parse;\n    this.parameters = input.parameters;\n    this.description = input.description;\n    this.name = input.name;\n  }\n}\n\n/**\n * This is helper class for passing a `function` and `parse` where the `function`\n * argument type matches the `parse` return type.\n */\nexport class ParsingToolFunction<Args extends object> {\n  type: 'function';\n  function: RunnableFunctionWithParse<Args>;\n\n  constructor(input: RunnableFunctionWithParse<Args>) {\n    this.type = 'function';\n    this.function = input;\n  }\n}\n\n// Path: src/lib/chatCompletionUtils.ts\nimport {\n  type ChatCompletionAssistantMessageParam,\n  type ChatCompletionFunctionMessageParam,\n  type ChatCompletionMessageParam,\n  type ChatCompletionToolMessageParam,\n} from 'openai/resources';\n\nexport const isAssistantMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionAssistantMessageParam => {\n  return message?.role === 'assistant';\n};\n\nexport const isFunctionMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionFunctionMessageParam => {\n  return message?.role === 'function';\n};\n\nexport const isToolMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionToolMessageParam => {\n  return message?.role === 'tool';\n};\n\nexport \nfunction isPresent<T>(obj: T | null | undefined): obj is T {\n  return obj != null;\n}\n\n// Path: src/lib/AbstractChatCompletionRunner.ts\nimport * as Core from 'openai/core';\nimport { type CompletionUsage } from 'openai/resources/completions';\nimport {\n  type Completions,\n  type ChatCompletion,\n  type ChatCompletionMessage,\n  type ChatCompletionMessageParam,\n  type ChatCompletionCreateParams,\n  type ChatCompletionTool,\n} from 'openai/resources/chat/completions';\nimport { APIUserAbortError, OpenAIError } from 'openai/error';\nimport {\n  type RunnableFunction,\n  isRunnableFunctionWithParse,\n  type BaseFunctionsArgs,\n} from './RunnableFunction';\nimport { ChatCompletionFunctionRunnerParams, ChatCompletionToolRunnerParams } from './ChatCompletionRunner';\nimport {\n  ChatCompletionStreamingFunctionRunnerParams,\n  ChatCompletionStreamingToolRunnerParams,\n} from './ChatCompletionStreamingRunner';\nimport { isAssistantMessage, isFunctionMessage, isToolMessage } from './chatCompletionUtils';\n\nconst DEFAULT_MAX_CHAT_COMPLETIONS = 10;\nexport interface RunnerOptions extends Core.RequestOptions {\n  /** How many requests to make before canceling. Default 10. */\n  maxChatCompletions?: number;\n}\n\nexport abstract class AbstractChatCompletionRunner<\n  Events extends CustomEvents<any> = AbstractChatCompletionRunnerEvents,\n> {\n  controller: AbortController = new AbortController();\n\n  #connectedPromise: Promise<void>;\n  #resolveConnectedPromise: () => void = () => {};\n  #rejectConnectedPromise: (error: OpenAIError) => void = () => {};\n\n  #endPromise: Promise<void>;\n  #resolveEndPromise: () => void = () => {};\n  #rejectEndPromise: (error: OpenAIError) => void = () => {};\n\n  #listeners: { [Event in keyof Events]?: ListenersForEvent<Events, Event> } = {};\n\n  protected _chatCompletions: ChatCompletion[] = [];\n  messages: ChatCompletionMessageParam[] = [];\n\n  #ended = false;\n  #errored = false;\n  #aborted = false;\n  #catchingPromiseCreated = false;\n\n  constructor() {\n    this.#connectedPromise = new Promise<void>((resolve, reject) => {\n      this.#resolveConnectedPromise = resolve;\n      this.#rejectConnectedPromise = reject;\n    });\n\n    this.#endPromise = new Promise<void>((resolve, reject) => {\n      this.#resolveEndPromise = resolve;\n      this.#rejectEndPromise = reject;\n    });\n\n    // Don't let these promises cause unhandled rejection errors.\n    // we will manually cause an unhandled rejection error later\n    // if the user hasn't registered any error listener or called\n    // any promise-returning method.\n    this.#connectedPromise.catch(() => {});\n    this.#endPromise.catch(() => {});\n  }\n\n  protected _run(executor: () => Promise<any>) {\n    // Unfortunately if we call `executor()` immediately we get runtime errors about\n    // references to `this` before the `super()` constructor call returns.\n    setTimeout(() => {\n      executor().then(() => {\n        this._emitFinal();\n        this._emit('end');\n      }, this.#handleError);\n    }, 0);\n  }\n\n  protected _addChatCompletion(chatCompletion: ChatCompletion): ChatCompletion {\n    this._chatCompletions.push(chatCompletion);\n    this._emit('chatCompletion', chatCompletion);\n    const message = chatCompletion.choices[0]?.message;\n    if (message) this._addMessage(message as ChatCompletionMessageParam);\n    return chatCompletion;\n  }\n\n  protected _addMessage(message: ChatCompletionMessageParam, emit = true) {\n    if (!('content' in message)) message.content = null;\n\n    this.messages.push(message);\n\n    if (emit) {\n      this._emit('message', message);\n      if ((isFunctionMessage(message) || isToolMessage(message)) && message.content) {\n        // Note, this assumes that {role: 'tool', content: \u2026} is always the result of a call of tool of type=function.\n        this._emit('functionCallResult', message.content as string);\n      } else if (isAssistantMessage(message) && message.function_call) {\n        this._emit('functionCall', message.function_call);\n      } else if (isAssistantMessage(message) && message.tool_calls) {\n        for (const tool_call of message.tool_calls) {\n          if (tool_call.type === 'function') {\n            this._emit('functionCall', tool_call.function);\n          }\n        }\n      }\n    }\n  }\n\n  protected _connected() {\n    if (this.ended) return;\n    this.#resolveConnectedPromise();\n    this._emit('connect');\n  }\n\n  get ended(): boolean {\n    return this.#ended;\n  }\n\n  get errored(): boolean {\n    return this.#errored;\n  }\n\n  get aborted(): boolean {\n    return this.#aborted;\n  }\n\n  abort() {\n    this.controller.abort();\n  }\n\n  /**\n   * Adds the listener function to the end of the listeners array for the event.\n   * No checks are made to see if the listener has already been added. Multiple calls passing\n   * the same combination of event and listener will result in the listener being added, and\n   * called, multiple times.\n   * @returns this ChatCompletionStream, so that calls can be chained\n   */\n  on<Event extends keyof Events>(event: Event, listener: ListenerForEvent<Events, Event>): this {\n    const listeners: ListenersForEvent<Events, Event> =\n      this.#listeners[event] || (this.#listeners[event] = []);\n    listeners.push({ listener });\n    return this;\n  }\n\n  /**\n   * Removes the specified listener from the listener array for the event.\n   * off() will remove, at most, one instance of a listener from the listener array. If any single\n   * listener has been added multiple times to the listener array for the specified event, then\n   * off() must be called multiple times to remove each instance.\n   * @returns this ChatCompletionStream, so that calls can be chained\n   */\n  off<Event extends keyof Events>(event: Event, listener: ListenerForEvent<Events, Event>): this {\n    const listeners = this.#listeners[event];\n    if (!listeners) return this;\n    const index = listeners.findIndex((l) => l.listener === listener);\n    if (index >= 0) listeners.splice(index, 1);\n    return this;\n  }\n\n  /**\n   * Adds a one-time listener function for the event. The next time the event is triggered,\n   * this listener is removed and then invoked.\n   * @returns this ChatCompletionStream, so that calls can be chained\n   */\n  once<Event extends keyof Events>(event: Event, listener: ListenerForEvent<Events, Event>): this {\n    const listeners: ListenersForEvent<Events, Event> =\n      this.#listeners[event] || (this.#listeners[event] = []);\n    listeners.push({ listener, once: true });\n    return this;\n  }\n\n  /**\n   * This is similar to `.once()`, but returns a Promise that resolves the next time\n   * the event is triggered, instead of calling a listener callback.\n   * @returns a Promise that resolves the next time given event is triggered,\n   * or rejects if an error is emitted.  (If you request the 'error' event,\n   * returns a promise that resolves with the error).\n   *\n   * Example:\n   *\n   *   const message = await stream.emitted('message') // rejects if the stream errors\n   */\n  emitted<Event extends keyof Events>(\n    event: Event,\n  ): Promise<\n    EventParameters<Events, Event> extends [infer Param] ? Param\n    : EventParameters<Events, Event> extends [] ? void\n    : EventParameters<Events, Event>\n  > {\n    return new Promise((resolve, reject) => {\n      this.#catchingPromiseCreated = true;\n      if (event !== 'error') this.once('error', reject);\n      this.once(event, resolve as any);\n    });\n  }\n\n  async done(): Promise<void> {\n    this.#catchingPromiseCreated = true;\n    await this.#endPromise;\n  }\n\n  /**\n   * @returns a promise that resolves with the final ChatCompletion, or rejects\n   * if an error occurred or the stream ended prematurely without producing a ChatCompletion.\n   */\n  async finalChatCompletion(): Promise<ChatCompletion> {\n    await this.done();\n    const completion = this._chatCompletions[this._chatCompletions.length - 1];\n    if (!completion) throw new OpenAIError('stream ended without producing a ChatCompletion');\n    return completion;\n  }\n\n  #getFinalContent(): string | null {\n    return this.#getFinalMessage().content ?? null;\n  }\n\n  /**\n   * @returns a promise that resolves with the content of the final ChatCompletionMessage, or rejects\n   * if an error occurred or the stream ended prematurely without producing a ChatCompletionMessage.\n   */\n  async finalContent(): Promise<string | null> {\n    await this.done();\n    return this.#getFinalContent();\n  }\n\n  #getFinalMessage(): ChatCompletionMessage {\n    let i = this.messages.length;\n    while (i-- > 0) {\n      const message = this.messages[i];\n      if (isAssistantMessage(message)) {\n        return { ...message, content: message.content ?? null };\n      }\n    }\n    throw new OpenAIError('stream ended without producing a ChatCompletionMessage with role=assistant');\n  }\n\n  /**\n   * @returns a promise that resolves with the the final assistant ChatCompletionMessage response,\n   * or rejects if an error occurred or the stream ended prematurely without producing a ChatCompletionMessage.\n   */\n  async finalMessage(): Promise<ChatCompletionMessage> {\n    await this.done();\n    return this.#getFinalMessage();\n  }\n\n  #getFinalFunctionCall(): ChatCompletionMessage.FunctionCall | undefined {\n    for (let i = this.messages.length - 1; i >= 0; i--) {\n      const message = this.messages[i];\n      if (isAssistantMessage(message) && message?.function_call) {\n        return message.function_call;\n      }\n      if (isAssistantMessage(message) && message?.tool_calls?.length) {\n        return message.tool_calls.at(-1)?.function;\n      }\n    }\n\n    return;\n  }\n\n  /**\n   * @returns a promise that resolves with the content of the final FunctionCall, or rejects\n   * if an error occurred or the stream ended prematurely without producing a ChatCompletionMessage.\n   */\n  async finalFunctionCall(): Promise<ChatCompletionMessage.FunctionCall | undefined> {\n    await this.done();\n    return this.#getFinalFunctionCall();\n  }\n\n  #getFinalFunctionCallResult(): string | undefined {\n    for (let i = this.messages.length - 1; i >= 0; i--) {\n      const message = this.messages[i];\n      if (isFunctionMessage(message) && message.content != null) {\n        return message.content;\n      }\n      if (\n        isToolMessage(message) &&\n        message.content != null &&\n        this.messages.some(\n          (x) =>\n            x.role === 'assistant' &&\n            x.tool_calls?.some((y) => y.type === 'function' && y.id === message.tool_call_id),\n        )\n      ) {\n        return message.content;\n      }\n    }\n\n    return;\n  }\n\n  async finalFunctionCallResult(): Promise<string | undefined> {\n    await this.done();\n    return this.#getFinalFunctionCallResult();\n  }\n\n  #calculateTotalUsage(): CompletionUsage {\n    const total: CompletionUsage = {\n      completion_tokens: 0,\n      prompt_tokens: 0,\n      total_tokens: 0,\n    };\n    for (const { usage } of this._chatCompletions) {\n      if (usage) {\n        total.completion_tokens += usage.completion_tokens;\n        total.prompt_tokens += usage.prompt_tokens;\n        total.total_tokens += usage.total_tokens;\n      }\n    }\n    return total;\n  }\n\n  async totalUsage(): Promise<CompletionUsage> {\n    await this.done();\n    return this.#calculateTotalUsage();\n  }\n\n  allChatCompletions(): ChatCompletion[] {\n    return [...this._chatCompletions];\n  }\n\n  #handleError = (error: unknown) => {\n    this.#errored = true;\n    if (error instanceof Error && error.name === 'AbortError') {\n      error = new APIUserAbortError();\n    }\n    if (error instanceof APIUserAbortError) {\n      this.#aborted = true;\n      return this._emit('abort', error);\n    }\n    if (error instanceof OpenAIError) {\n      return this._emit('error', error);\n    }\n    if (error instanceof Error) {\n      const openAIError: OpenAIError = new OpenAIError(error.message);\n      // @ts-ignore\n      openAIError.cause = error;\n      return this._emit('error', openAIError);\n    }\n    return this._emit('error', new OpenAIError(String(error)));\n  };\n\n  protected _emit<Event extends keyof Events>(event: Event, ...args: EventParameters<Events, Event>) {\n    // make sure we don't emit any events after end\n    if (this.#ended) {\n      return;\n    }\n\n    if (event === 'end') {\n      this.#ended = true;\n      this.#resolveEndPromise();\n    }\n\n    const listeners: ListenersForEvent<Events, Event> | undefined = this.#listeners[event];\n    if (listeners) {\n      this.#listeners[event] = listeners.filter((l) => !l.once) as any;\n      listeners.forEach(({ listener }: any) => listener(...args));\n    }\n\n    if (event === 'abort') {\n      const error = args[0] as APIUserAbortError;\n      if (!this.#catchingPromiseCreated && !listeners?.length) {\n        Promise.reject(error);\n      }\n      this.#rejectConnectedPromise(error);\n      this.#rejectEndPromise(error);\n      this._emit('end');\n      return;\n    }\n\n    if (event === 'error') {\n      // NOTE: _emit('error', error) should only be called from #handleError().\n\n      const error = args[0] as OpenAIError;\n      if (!this.#catchingPromiseCreated && !listeners?.length) {\n        // Trigger an unhandled rejection if the user hasn't registered any error handlers.\n        // If you are seeing stack traces here, make sure to handle errors via either:\n        // - runner.on('error', () => ...)\n        // - await runner.done()\n        // - await runner.finalChatCompletion()\n        // - etc.\n        Promise.reject(error);\n      }\n      this.#rejectConnectedPromise(error);\n      this.#rejectEndPromise(error);\n      this._emit('end');\n    }\n  }\n\n  protected _emitFinal() {\n    const completion = this._chatCompletions[this._chatCompletions.length - 1];\n    if (completion) this._emit('finalChatCompletion', completion);\n    const finalMessage = this.#getFinalMessage();\n    if (finalMessage) this._emit('finalMessage', finalMessage);\n    const finalContent = this.#getFinalContent();\n    if (finalContent) this._emit('finalContent', finalContent);\n\n    const finalFunctionCall = this.#getFinalFunctionCall();\n    if (finalFunctionCall) this._emit('finalFunctionCall', finalFunctionCall);\n\n    const finalFunctionCallResult = this.#getFinalFunctionCallResult();\n    if (finalFunctionCallResult != null) this._emit('finalFunctionCallResult', finalFunctionCallResult);\n\n    if (this._chatCompletions.some((c) => c.usage)) {\n      this._emit('totalUsage', this.#calculateTotalUsage());\n    }\n  }\n\n  #validateParams(params: ChatCompletionCreateParams): void {\n    if (params.n != null && params.n > 1) {\n      throw new OpenAIError(\n        'ChatCompletion convenience helpers only support n=1 at this time. To use n>1, please use chat.completions.create() directly.',\n      );\n    }\n  }\n\n  protected async _createChatCompletion(\n    completions: Completions,\n    params: ChatCompletionCreateParams,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    const signal = options?.signal;\n    if (signal) {\n      if (signal.aborted) this.controller.abort();\n      signal.addEventListener('abort', () => this.controller.abort());\n    }\n    this.#validateParams(params);\n\n    const chatCompletion = await completions.create(\n      { ...params, stream: false },\n      { ...options, signal: this.controller.signal },\n    );\n    this._connected();\n    return this._addChatCompletion(chatCompletion);\n  }\n\n  protected async _runChatCompletion(\n    completions: Completions,\n    params: ChatCompletionCreateParams,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    for (const message of params.messages) {\n      this._addMessage(message, false);\n    }\n    return await this._createChatCompletion(completions, params, options);\n  }\n\n  protected async _runFunctions<FunctionsArgs extends BaseFunctionsArgs>(\n    completions: Completions,\n    params:\n      | ChatCompletionFunctionRunnerParams<FunctionsArgs>\n      | ChatCompletionStreamingFunctionRunnerParams<FunctionsArgs>,\n    options?: RunnerOptions,\n  ) {\n    const role = 'function' as const;\n    const { function_call = 'auto', stream, ...restParams } = params;\n    const singleFunctionToCall = typeof function_call !== 'string' && function_call?.name;\n    const { maxChatCompletions = DEFAULT_MAX_CHAT_COMPLETIONS } = options || {};\n\n    const functionsByName: Record<string, RunnableFunction<any>> = {};\n    for (const f of params.functions) {\n      functionsByName[f.name || f.function.name] = f;\n    }\n\n    const functions: ChatCompletionCreateParams.Function[] = params.functions.map(\n      (f): ChatCompletionCreateParams.Function => ({\n        name: f.name || f.function.name,\n        parameters: f.parameters as Record<string, unknown>,\n        description: f.description,\n      }),\n    );\n\n    for (const message of params.messages) {\n      this._addMessage(message, false);\n    }\n\n    for (let i = 0; i < maxChatCompletions; ++i) {\n      const chatCompletion: ChatCompletion = await this._createChatCompletion(\n        completions,\n        {\n          ...restParams,\n          function_call,\n          functions,\n          messages: [...this.messages],\n        },\n        options,\n      );\n      const message = chatCompletion.choices[0]?.message;\n      if (!message) {\n        throw new OpenAIError(`missing message in ChatCompletion response`);\n      }\n      if (!message.function_call) return;\n      const { name, arguments: args } = message.function_call;\n      const fn = functionsByName[name];\n      if (!fn) {\n        const content = `Invalid function_call: ${JSON.stringify(name)}. Available options are: ${functions\n          .map((f) => JSON.stringify(f.name))\n          .join(', ')}. Please try again`;\n\n        this._addMessage({ role, name, content });\n        continue;\n      } else if (singleFunctionToCall && singleFunctionToCall !== name) {\n        const content = `Invalid function_call: ${JSON.stringify(name)}. ${JSON.stringify(\n          singleFunctionToCall,\n        )} requested. Please try again`;\n\n        this._addMessage({ role, name, content });\n        continue;\n      }\n\n      let parsed;\n      try {\n        parsed = isRunnableFunctionWithParse(fn) ? await fn.parse(args) : args;\n      } catch (error) {\n        this._addMessage({\n          role,\n          name,\n          content: error instanceof Error ? error.message : String(error),\n        });\n        continue;\n      }\n\n      // @ts-expect-error it can't rule out `never` type.\n      const rawContent = await fn.function(parsed, this);\n      const content = this.#stringifyFunctionCallResult(rawContent);\n\n      this._addMessage({ role, name, content });\n\n      if (singleFunctionToCall) return;\n    }\n  }\n\n  protected async _runTools<FunctionsArgs extends BaseFunctionsArgs>(\n    completions: Completions,\n    params:\n      | ChatCompletionToolRunnerParams<FunctionsArgs>\n      | ChatCompletionStreamingToolRunnerParams<FunctionsArgs>,\n    options?: RunnerOptions,\n  ) {\n    const role = 'tool' as const;\n    const { tool_choice = 'auto', stream, ...restParams } = params;\n    const singleFunctionToCall = typeof tool_choice !== 'string' && tool_choice?.function?.name;\n    const { maxChatCompletions = DEFAULT_MAX_CHAT_COMPLETIONS } = options || {};\n\n    const functionsByName: Record<string, RunnableFunction<any>> = {};\n    for (const f of params.tools) {\n      if (f.type === 'function') {\n        functionsByName[f.function.name || f.function.function.name] = f.function;\n      }\n    }\n\n    const tools: ChatCompletionTool[] =\n      'tools' in params ?\n        params.tools.map((t) =>\n          t.type === 'function' ?\n            {\n              type: 'function',\n              function: {\n                name: t.function.name || t.function.function.name,\n                parameters: t.function.parameters as Record<string, unknown>,\n                description: t.function.description,\n              },\n            }\n          : (t as unknown as ChatCompletionTool),\n        )\n      : (undefined as any);\n\n    for (const message of params.messages) {\n      this._addMessage(message, false);\n    }\n\n    for (let i = 0; i < maxChatCompletions; ++i) {\n      const chatCompletion: ChatCompletion = await this._createChatCompletion(\n        completions,\n        {\n          ...restParams,\n          tool_choice,\n          tools,\n          messages: [...this.messages],\n        },\n        options,\n      );\n      const message = chatCompletion.choices[0]?.message;\n      if (!message) {\n        throw new OpenAIError(`missing message in ChatCompletion response`);\n      }\n      if (!message.tool_calls) {\n        return;\n      }\n\n      for (const tool_call of message.tool_calls) {\n        if (tool_call.type !== 'function') continue;\n        const tool_call_id = tool_call.id;\n        const { name, arguments: args } = tool_call.function;\n        const fn = functionsByName[name];\n\n        if (!fn) {\n          const content = `Invalid tool_call: ${JSON.stringify(name)}. Available options are: ${tools\n            .map((f) => JSON.stringify(f.function.name))\n            .join(', ')}. Please try again`;\n\n          this._addMessage({ role, tool_call_id, content });\n          continue;\n        } else if (singleFunctionToCall && singleFunctionToCall !== name) {\n          const content = `Invalid tool_call: ${JSON.stringify(name)}. ${JSON.stringify(\n            singleFunctionToCall,\n          )} requested. Please try again`;\n\n          this._addMessage({ role, tool_call_id, content });\n          continue;\n        }\n\n        let parsed;\n        try {\n          parsed = isRunnableFunctionWithParse(fn) ? await fn.parse(args) : args;\n        } catch (error) {\n          const content = error instanceof Error ? error.message : String(error);\n          this._addMessage({ role, tool_call_id, content });\n          continue;\n        }\n\n        // @ts-expect-error it can't rule out `never` type.\n        const rawContent = await fn.function(parsed, this);\n        const content = this.#stringifyFunctionCallResult(rawContent);\n        this._addMessage({ role, tool_call_id, content });\n\n        if (singleFunctionToCall) {\n          return;\n        }\n      }\n    }\n\n    return;\n  }\n\n  #stringifyFunctionCallResult(rawContent: unknown): string {\n    return (\n      typeof rawContent === 'string' ? rawContent\n      : rawContent === undefined ? 'undefined'\n      : JSON.stringify(rawContent)\n    );\n  }\n}\n\ntype CustomEvents<Event extends string> = {\n  [k in Event]: k extends keyof AbstractChatCompletionRunnerEvents ? AbstractChatCompletionRunnerEvents[k]\n  : (...args: any[]) => void;\n};\n\ntype ListenerForEvent<Events extends CustomEvents<any>, Event extends keyof Events> = Event extends (\n  keyof AbstractChatCompletionRunnerEvents\n) ?\n  AbstractChatCompletionRunnerEvents[Event]\n: Events[Event];\n\ntype ListenersForEvent<Events extends CustomEvents<any>, Event extends keyof Events> = Array<{\n  listener: ListenerForEvent<Events, Event>;\n  once?: boolean;\n}>;\ntype EventParameters<Events extends CustomEvents<any>, Event extends keyof Events> = Parameters<\n  ListenerForEvent<Events, Event>\n>;\n\nexport interface AbstractChatCompletionRunnerEvents {\n  connect: () => void;\n  functionCall: (functionCall: ChatCompletionMessage.FunctionCall) => void;\n  message: (message: ChatCompletionMessageParam) => void;\n  chatCompletion: (completion: ChatCompletion) => void;\n  finalContent: (contentSnapshot: string) => void;\n  finalMessage: (message: ChatCompletionMessageParam) => void;\n  finalChatCompletion: (completion: ChatCompletion) => void;\n  finalFunctionCall: (functionCall: ChatCompletionMessage.FunctionCall) => void;\n  functionCallResult: (content: string) => void;\n  finalFunctionCallResult: (content: string) => void;\n  error: (error: OpenAIError) => void;\n  abort: (error: APIUserAbortError) => void;\n  end: () => void;\n  totalUsage: (usage: CompletionUsage) => void;\n}\n\n// Path: src/lib/ChatCompletionRunner.ts\nimport {\n  type Completions,\n  type ChatCompletionMessageParam,\n  type ChatCompletionCreateParamsNonStreaming,\n} from 'openai/resources/chat/completions';\nimport { type RunnableFunctions, type BaseFunctionsArgs, RunnableTools } from './RunnableFunction';\nimport {\n  AbstractChatCompletionRunner,\n  AbstractChatCompletionRunnerEvents,\n  RunnerOptions,\n} from './AbstractChatCompletionRunner';\nimport { isAssistantMessage } from './chatCompletionUtils';\n\nexport interface ChatCompletionRunnerEvents extends AbstractChatCompletionRunnerEvents {\n  content: (content: string) => void;\n}\n\nexport type ChatCompletionFunctionRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsNonStreaming,\n  'functions'\n> & {\n  functions: RunnableFunctions<FunctionsArgs>;\n};\n\nexport type ChatCompletionToolRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsNonStreaming,\n  'tools'\n> & {\n  tools: RunnableTools<FunctionsArgs>;\n};\n\nexport class ChatCompletionRunner extends AbstractChatCompletionRunner<ChatCompletionRunnerEvents> {\n  /** @deprecated - please use `runTools` instead. */\n  static runFunctions(\n    completions: Completions,\n    params: ChatCompletionFunctionRunnerParams<any[]>,\n    options?: RunnerOptions,\n  ): ChatCompletionRunner {\n    const runner = new ChatCompletionRunner();\n    const opts = {\n      ...options,\n      headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'runFunctions' },\n    };\n    runner._run(() => runner._runFunctions(completions, params, opts));\n    return runner;\n  }\n\n  static runTools(\n    completions: Completions,\n    params: ChatCompletionToolRunnerParams<any[]>,\n    options?: RunnerOptions,\n  ): ChatCompletionRunner {\n    const runner = new ChatCompletionRunner();\n...\n", "needle_token_start": 8856, "needle_token_end": 8884, "code_context_ntokens": 16149, "output": ["```typescript\nfunction isPresent<T>(obj: T | null | undefined): obj is T {\n  return obj != null;\n}\n```"]}
{"repo": "openai/openai-node", "name": "mockStreamingChatCompletionFetch", "language": "typescript", "path": "src/lib/ChatCompletionRunFunctions.test.ts", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to simulate network requests for streaming chat completions, allowing developers to test and debug streaming chat interfaces without making actual network calls.\n2. **Input**: The input is a handler function that takes parameters for streaming chat completions and returns an asynchronous iterable that yields chat completion chunks.\n3. **Output**: The output is a mocked fetch function that, when called, returns a simulated streaming response mimicking a server sending chat completion data in chunks.\n4. **Procedure**: \n   - The function initializes a mock fetch setup.\n   - It intercepts fetch calls and uses a provided handler to process the input data.\n   - The handler function processes the input and generates chat completion chunks asynchronously.\n   - These chunks are then streamed back as a response in a format suitable for streaming data (using server-sent events).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/resources/beta/chat/completions.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { ChatCompletionRunner, ChatCompletionFunctionRunnerParams } from 'openai/lib/ChatCompletionRunner';\nexport { ChatCompletionRunner, ChatCompletionFunctionRunnerParams } from 'openai/lib/ChatCompletionRunner';\nimport {\n  ChatCompletionStreamingRunner,\n  ChatCompletionStreamingFunctionRunnerParams,\n} from 'openai/lib/ChatCompletionStreamingRunner';\nexport {\n  ChatCompletionStreamingRunner,\n  ChatCompletionStreamingFunctionRunnerParams,\n} from 'openai/lib/ChatCompletionStreamingRunner';\nimport { BaseFunctionsArgs } from 'openai/lib/RunnableFunction';\nexport {\n  RunnableFunction,\n  RunnableFunctions,\n  RunnableFunctionWithParse,\n  RunnableFunctionWithoutParse,\n  ParsingFunction,\n  ParsingToolFunction,\n} from 'openai/lib/RunnableFunction';\nimport { ChatCompletionToolRunnerParams } from 'openai/lib/ChatCompletionRunner';\nexport { ChatCompletionToolRunnerParams } from 'openai/lib/ChatCompletionRunner';\nimport { ChatCompletionStreamingToolRunnerParams } from 'openai/lib/ChatCompletionStreamingRunner';\nexport { ChatCompletionStreamingToolRunnerParams } from 'openai/lib/ChatCompletionStreamingRunner';\nimport { ChatCompletionStream, type ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';\nexport { ChatCompletionStream, type ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';\n\nexport class Completions extends APIResource {\n  /**\n   * @deprecated - use `runTools` instead.\n   */\n  runFunctions<FunctionsArgs extends BaseFunctionsArgs>(\n    body: ChatCompletionFunctionRunnerParams<FunctionsArgs>,\n    options?: Core.RequestOptions,\n  ): ChatCompletionRunner;\n  runFunctions<FunctionsArgs extends BaseFunctionsArgs>(\n    body: ChatCompletionStreamingFunctionRunnerParams<FunctionsArgs>,\n    options?: Core.RequestOptions,\n  ): ChatCompletionStreamingRunner;\n  runFunctions<FunctionsArgs extends BaseFunctionsArgs>(\n    body:\n      | ChatCompletionFunctionRunnerParams<FunctionsArgs>\n      | ChatCompletionStreamingFunctionRunnerParams<FunctionsArgs>,\n    options?: Core.RequestOptions,\n  ): ChatCompletionRunner | ChatCompletionStreamingRunner {\n    if (body.stream) {\n...\n// Path: src/resources/beta/chat/chat.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { APIResource } from 'openai/resource';\nimport * as CompletionsAPI from 'openai/resources/beta/chat/completions';\n\nexport class Chat extends APIResource {\n  completions: CompletionsAPI.Completions = new CompletionsAPI.Completions(this._client);\n}\n\nexport namespace Chat {\n  export import Completions = CompletionsAPI.Completions;\n}\n\n// Path: src/resources/beta/beta.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { APIResource } from 'openai/resource';\nimport * as AssistantsAPI from 'openai/resources/beta/assistants/assistants';\nimport * as ChatAPI from 'openai/resources/beta/chat/chat';\nimport * as ThreadsAPI from 'openai/resources/beta/threads/threads';\n\nexport class Beta extends APIResource {\n  chat: ChatAPI.Chat = new ChatAPI.Chat(this._client);\n  assistants: AssistantsAPI.Assistants = new AssistantsAPI.Assistants(this._client);\n  threads: ThreadsAPI.Threads = new ThreadsAPI.Threads(this._client);\n}\n\nexport namespace Beta {\n  export import Chat = ChatAPI.Chat;\n  export import Assistants = AssistantsAPI.Assistants;\n  export import Assistant = AssistantsAPI.Assistant;\n  export import AssistantDeleted = AssistantsAPI.AssistantDeleted;\n  export import AssistantStreamEvent = AssistantsAPI.AssistantStreamEvent;\n  export import AssistantTool = AssistantsAPI.AssistantTool;\n  export import CodeInterpreterTool = AssistantsAPI.CodeInterpreterTool;\n  export import FunctionTool = AssistantsAPI.FunctionTool;\n  export import MessageStreamEvent = AssistantsAPI.MessageStreamEvent;\n  export import RetrievalTool = AssistantsAPI.RetrievalTool;\n  export import RunStepStreamEvent = AssistantsAPI.RunStepStreamEvent;\n  export import RunStreamEvent = AssistantsAPI.RunStreamEvent;\n  export import ThreadStreamEvent = AssistantsAPI.ThreadStreamEvent;\n  export import AssistantsPage = AssistantsAPI.AssistantsPage;\n  export import AssistantCreateParams = AssistantsAPI.AssistantCreateParams;\n  export import AssistantUpdateParams = AssistantsAPI.AssistantUpdateParams;\n  export import AssistantListParams = AssistantsAPI.AssistantListParams;\n  export import Threads = ThreadsAPI.Threads;\n  export import Thread = ThreadsAPI.Thread;\n  export import ThreadDeleted = ThreadsAPI.ThreadDeleted;\n  export import ThreadCreateParams = ThreadsAPI.ThreadCreateParams;\n  export import ThreadUpdateParams = ThreadsAPI.ThreadUpdateParams;\n  export import ThreadCreateAndRunParams = ThreadsAPI.ThreadCreateAndRunParams;\n  export import ThreadCreateAndRunParamsNonStreaming = ThreadsAPI.ThreadCreateAndRunParamsNonStreaming;\n  export import ThreadCreateAndRunParamsStreaming = ThreadsAPI.ThreadCreateAndRunParamsStreaming;\n  export import ThreadCreateAndRunPollParams = ThreadsAPI.ThreadCreateAndRunPollParams;\n  export import ThreadCreateAndRunStreamParams = ThreadsAPI.ThreadCreateAndRunStreamParams;\n}\n\n// Path: src/resources/images.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport * as ImagesAPI from 'openai/resources/images';\nimport { type Uploadable, multipartFormRequestOptions } from 'openai/core';\n\nexport class Images extends APIResource {\n  /**\n   * Creates a variation of a given image.\n   */\n  createVariation(\n    body: ImageCreateVariationParams,\n    options?: Core.RequestOptions,\n  ): Core.APIPromise<ImagesResponse> {\n    return this._client.post('/images/variations', multipartFormRequestOptions({ body, ...options }));\n  }\n\n  /**\n   * Creates an edited or extended image given an original image and a prompt.\n   */\n  edit(body: ImageEditParams, options?: Core.RequestOptions): Core.APIPromise<ImagesResponse> {\n    return this._client.post('/images/edits', multipartFormRequestOptions({ body, ...options }));\n  }\n\n  /**\n   * Creates an image given a prompt.\n   */\n  generate(body: ImageGenerateParams, options?: Core.RequestOptions): Core.APIPromise<ImagesResponse> {\n    return this._client.post('/images/generations', { body, ...options });\n  }\n}\n\n/**\n * Represents the url or the content of an image generated by the OpenAI API.\n */\nexport interface Image {\n  /**\n   * The base64-encoded JSON of the generated image, if `response_format` is\n   * `b64_json`.\n   */\n  b64_json?: string;\n\n  /**\n   * The prompt that was used to generate the image, if there was any revision to the\n   * prompt.\n   */\n  revised_prompt?: string;\n\n  /**\n   * The URL of the generated image, if `response_format` is `url` (default).\n   */\n  url?: string;\n}\n\nexport interface ImagesResponse {\n  created: number;\n\n  data: Array<Image>;\n}\n\nexport interface ImageCreateVariationParams {\n  /**\n   * The image to use as the basis for the variation(s). Must be a valid PNG file,\n   * less than 4MB, and square.\n   */\n  image: Uploadable;\n\n  /**\n   * The model to use for image generation. Only `dall-e-2` is supported at this\n   * time.\n   */\n  model?: (string & {}) | 'dall-e-2' | null;\n\n  /**\n   * The number of images to generate. Must be between 1 and 10. For `dall-e-3`, only\n   * `n=1` is supported.\n   */\n  n?: number | null;\n\n  /**\n   * The format in which the generated images are returned. Must be one of `url` or\n   * `b64_json`. URLs are only valid for 60 minutes after the image has been\n   * generated.\n   */\n  response_format?: 'url' | 'b64_json' | null;\n\n  /**\n   * The size of the generated images. Must be one of `256x256`, `512x512`, or\n   * `1024x1024`.\n   */\n  size?: '256x256' | '512x512' | '1024x1024' | null;\n\n  /**\n   * A unique identifier representing your end-user, which can help OpenAI to monitor\n   * and detect abuse.\n   * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n   */\n  user?: string;\n}\n\nexport interface ImageEditParams {\n  /**\n   * The image to edit. Must be a valid PNG file, less than 4MB, and square. If mask\n   * is not provided, image must have transparency, which will be used as the mask.\n   */\n  image: Uploadable;\n\n  /**\n   * A text description of the desired image(s). The maximum length is 1000\n   * characters.\n   */\n  prompt: string;\n\n  /**\n   * An additional image whose fully transparent areas (e.g. where alpha is zero)\n   * indicate where `image` should be edited. Must be a valid PNG file, less than\n   * 4MB, and have the same dimensions as `image`.\n   */\n  mask?: Uploadable;\n\n  /**\n   * The model to use for image generation. Only `dall-e-2` is supported at this\n   * time.\n   */\n  model?: (string & {}) | 'dall-e-2' | null;\n\n  /**\n   * The number of images to generate. Must be between 1 and 10.\n   */\n  n?: number | null;\n\n  /**\n   * The format in which the generated images are returned. Must be one of `url` or\n   * `b64_json`. URLs are only valid for 60 minutes after the image has been\n   * generated.\n   */\n  response_format?: 'url' | 'b64_json' | null;\n\n  /**\n   * The size of the generated images. Must be one of `256x256`, `512x512`, or\n   * `1024x1024`.\n   */\n  size?: '256x256' | '512x512' | '1024x1024' | null;\n\n  /**\n   * A unique identifier representing your end-user, which can help OpenAI to monitor\n   * and detect abuse.\n   * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n   */\n  user?: string;\n}\n\nexport interface ImageGenerateParams {\n  /**\n   * A text description of the desired image(s). The maximum length is 1000\n   * characters for `dall-e-2` and 4000 characters for `dall-e-3`.\n   */\n  prompt: string;\n\n  /**\n   * The model to use for image generation.\n   */\n  model?: (string & {}) | 'dall-e-2' | 'dall-e-3' | null;\n\n  /**\n   * The number of images to generate. Must be between 1 and 10. For `dall-e-3`, only\n   * `n=1` is supported.\n   */\n  n?: number | null;\n\n  /**\n   * The quality of the image that will be generated. `hd` creates images with finer\n   * details and greater consistency across the image. This param is only supported\n   * for `dall-e-3`.\n   */\n  quality?: 'standard' | 'hd';\n\n  /**\n   * The format in which the generated images are returned. Must be one of `url` or\n   * `b64_json`. URLs are only valid for 60 minutes after the image has been\n   * generated.\n   */\n  response_format?: 'url' | 'b64_json' | null;\n\n  /**\n   * The size of the generated images. Must be one of `256x256`, `512x512`, or\n   * `1024x1024` for `dall-e-2`. Must be one of `1024x1024`, `1792x1024`, or\n   * `1024x1792` for `dall-e-3` models.\n   */\n  size?: '256x256' | '512x512' | '1024x1024' | '1792x1024' | '1024x1792' | null;\n\n  /**\n   * The style of the generated images. Must be one of `vivid` or `natural`. Vivid\n   * causes the model to lean towards generating hyper-real and dramatic images.\n   * Natural causes the model to produce more natural, less hyper-real looking\n   * images. This param is only supported for `dall-e-3`.\n   */\n  style?: 'vivid' | 'natural' | null;\n\n  /**\n   * A unique identifier representing your end-user, which can help OpenAI to monitor\n   * and detect abuse.\n   * [Learn more](https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids).\n   */\n  user?: string;\n}\n\nexport namespace Images {\n  export import Image = ImagesAPI.Image;\n  export import ImagesResponse = ImagesAPI.ImagesResponse;\n  export import ImageCreateVariationParams = ImagesAPI.ImageCreateVariationParams;\n  export import ImageEditParams = ImagesAPI.ImageEditParams;\n  export import ImageGenerateParams = ImagesAPI.ImageGenerateParams;\n}\n\n// Path: src/resources/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport * from './chat/index';\nexport * from './shared';\nexport { Audio } from './audio/audio';\nexport { Beta } from './beta/beta';\nexport {\n  Completion,\n  CompletionChoice,\n  CompletionUsage,\n  CompletionCreateParams,\n  CompletionCreateParamsNonStreaming,\n  CompletionCreateParamsStreaming,\n  Completions,\n} from './completions';\nexport { CreateEmbeddingResponse, Embedding, EmbeddingCreateParams, Embeddings } from './embeddings';\nexport {\n  FileContent,\n  FileDeleted,\n  FileObject,\n  FileCreateParams,\n  FileListParams,\n  FileObjectsPage,\n  Files,\n} from './files';\nexport { FineTuning } from './fine-tuning/fine-tuning';\nexport {\n  Image,\n  ImagesResponse,\n  ImageCreateVariationParams,\n  ImageEditParams,\n  ImageGenerateParams,\n  Images,\n} from './images';\nexport { Model, ModelDeleted, ModelsPage, Models } from './models';\nexport { Moderation, ModerationCreateResponse, ModerationCreateParams, Moderations } from './moderations';\n\n// Path: src/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from './core';\nimport * as Errors from './error';\nimport { type Agent } from './_shims/index';\nimport * as Uploads from './uploads';\nimport * as Pagination from 'openai/pagination';\nimport * as API from 'openai/resources/index';\n\nexport interface ClientOptions {\n  /**\n   * Defaults to process.env['OPENAI_API_KEY'].\n   */\n  apiKey?: string | undefined;\n\n  /**\n   * Defaults to process.env['OPENAI_ORG_ID'].\n   */\n  organization?: string | null | undefined;\n\n  /**\n   * Override the default base URL for the API, e.g., \"https://api.example.com/v2/\"\n   *\n   * Defaults to process.env['OPENAI_BASE_URL'].\n   */\n  baseURL?: string | null | undefined;\n\n  /**\n   * The maximum amount of time (in milliseconds) that the client should wait for a response\n   * from the server before timing out a single request.\n   *\n   * Note that request timeouts are retried by default, so in a worst-case scenario you may wait\n   * much longer than this timeout before the promise succeeds or fails.\n   */\n  timeout?: number;\n\n  /**\n   * An HTTP agent used to manage HTTP(S) connections.\n   *\n   * If not provided, an agent will be constructed by default in the Node.js environment,\n   * otherwise no agent is used.\n   */\n  httpAgent?: Agent;\n\n  /**\n   * Specify a custom `fetch` function implementation.\n   *\n   * If not provided, we use `node-fetch` on Node.js and otherwise expect that `fetch` is\n   * defined globally.\n   */\n  fetch?: Core.Fetch | undefined;\n\n  /**\n   * The maximum number of times that the client will retry a request in case of a\n   * temporary failure, like a network error or a 5XX error from the server.\n   *\n   * @default 2\n   */\n  maxRetries?: number;\n\n  /**\n   * Default headers to include with every request to the API.\n   *\n   * These can be removed in individual requests by explicitly setting the\n   * header to `undefined` or `null` in request options.\n   */\n  defaultHeaders?: Core.Headers;\n\n  /**\n   * Default query parameters to include with every request to the API.\n   *\n   * These can be removed in individual requests by explicitly setting the\n   * param to `undefined` in request options.\n   */\n  defaultQuery?: Core.DefaultQuery;\n\n  /**\n   * By default, client-side use of this library is not allowed, as it risks exposing your secret API credentials to attackers.\n   * Only set this option to `true` if you understand the risks and have appropriate mitigations in place.\n   */\n  dangerouslyAllowBrowser?: boolean;\n}\n\n/** API Client for interfacing with the OpenAI API. */\nexport class OpenAI extends Core.APIClient {\n  apiKey: string;\n  organization: string | null;\n\n  private _options: ClientOptions;\n\n  /**\n   * API Client for interfacing with the OpenAI API.\n   *\n   * @param {string | undefined} [opts.apiKey=process.env['OPENAI_API_KEY'] ?? undefined]\n   * @param {string | null | undefined} [opts.organization=process.env['OPENAI_ORG_ID'] ?? null]\n   * @param {string} [opts.baseURL=process.env['OPENAI_BASE_URL'] ?? https://api.openai.com/v1] - Override the default base URL for the API.\n   * @param {number} [opts.timeout=10 minutes] - The maximum amount of time (in milliseconds) the client will wait for a response before timing out.\n   * @param {number} [opts.httpAgent] - An HTTP agent used to manage HTTP(s) connections.\n   * @param {Core.Fetch} [opts.fetch] - Specify a custom `fetch` function implementation.\n   * @param {number} [opts.maxRetries=2] - The maximum number of times the client will retry a request.\n   * @param {Core.Headers} opts.defaultHeaders - Default headers to include with every request to the API.\n   * @param {Core.DefaultQuery} opts.defaultQuery - Default query parameters to include with every request to the API.\n   * @param {boolean} [opts.dangerouslyAllowBrowser=false] - By default, client-side use of this library is not allowed, as it risks exposing your secret API credentials to attackers.\n   */\n  constructor({\n    baseURL = Core.readEnv('OPENAI_BASE_URL'),\n    apiKey = Core.readEnv('OPENAI_API_KEY'),\n    organization = Core.readEnv('OPENAI_ORG_ID') ?? null,\n    ...opts\n  }: ClientOptions = {}) {\n    if (apiKey === undefined) {\n      throw new Errors.OpenAIError(\n        \"The OPENAI_API_KEY environment variable is missing or empty; either provide it, or instantiate the OpenAI client with an apiKey option, like new OpenAI({ apiKey: 'My API Key' }).\",\n      );\n    }\n\n    const options: ClientOptions = {\n      apiKey,\n      organization,\n      ...opts,\n      baseURL: baseURL || `https://api.openai.com/v1`,\n    };\n\n    if (!options.dangerouslyAllowBrowser && Core.isRunningInBrowser()) {\n      throw new Errors.OpenAIError(\n        \"It looks like you're running in a browser-like environment.\\n\\nThis is disabled by default, as it risks exposing your secret API credentials to attackers.\\nIf you understand the risks and have appropriate mitigations in place,\\nyou can set the `dangerouslyAllowBrowser` option to `true`, e.g.,\\n\\nnew OpenAI({ apiKey, dangerouslyAllowBrowser: true });\\n\\nhttps://help.openai.com/en/articles/5112595-best-practices-for-api-key-safety\\n\",\n      );\n    }\n\n    super({\n      baseURL: options.baseURL!,\n      timeout: options.timeout ?? 600000 /* 10 minutes */,\n      httpAgent: options.httpAgent,\n      maxRetries: options.maxRetries,\n      fetch: options.fetch,\n    });\n    this._options = options;\n\n    this.apiKey = apiKey;\n    this.organization = organization;\n  }\n\n  completions: API.Completions = new API.Completions(this);\n  chat: API.Chat = new API.Chat(this);\n  embeddings: API.Embeddings = new API.Embeddings(this);\n  files: API.Files = new API.Files(this);\n  images: API.Images = new API.Images(this);\n  audio: API.Audio = new API.Audio(this);\n  moderations: API.Moderations = new API.Moderations(this);\n  models: API.Models = new API.Models(this);\n  fineTuning: API.FineTuning = new API.FineTuning(this);\n  beta: API.Beta = new API.Beta(this);\n\n  protected override defaultQuery(): Core.DefaultQuery | undefined {\n    return this._options.defaultQuery;\n  }\n\n  protected override defaultHeaders(opts: Core.FinalRequestOptions): Core.Headers {\n    return {\n      ...super.defaultHeaders(opts),\n      'OpenAI-Organization': this.organization,\n      ...this._options.defaultHeaders,\n    };\n  }\n\n  protected override authHeaders(opts: Core.FinalRequestOptions): Core.Headers {\n    return { Authorization: `Bearer ${this.apiKey}` };\n  }\n\n  static OpenAI = this;\n\n  static OpenAIError = Errors.OpenAIError;\n  static APIError = Errors.APIError;\n  static APIConnectionError = Errors.APIConnectionError;\n  static APIConnectionTimeoutError = Errors.APIConnectionTimeoutError;\n  static APIUserAbortError = Errors.APIUserAbortError;\n  static NotFoundError = Errors.NotFoundError;\n  static ConflictError = Errors.ConflictError;\n  static RateLimitError = Errors.RateLimitError;\n  static BadRequestError = Errors.BadRequestError;\n  static AuthenticationError = Errors.AuthenticationError;\n  static InternalServerError = Errors.InternalServerError;\n  static PermissionDeniedError = Errors.PermissionDeniedError;\n  static UnprocessableEntityError = Errors.UnprocessableEntityError;\n}\n\nexport const {\n  OpenAIError,\n  APIError,\n  APIConnectionError,\n  APIConnectionTimeoutError,\n  APIUserAbortError,\n  NotFoundError,\n  ConflictError,\n  RateLimitError,\n  BadRequestError,\n  AuthenticationError,\n  InternalServerError,\n  PermissionDeniedError,\n  UnprocessableEntityError,\n} = Errors;\n\nexport import toFile = Uploads.toFile;\nexport import fileFromPath = Uploads.fileFromPath;\n\nexport namespace OpenAI {\n  // Helper functions\n  export import toFile = Uploads.toFile;\n  export import fileFromPath = Uploads.fileFromPath;\n\n  export import RequestOptions = Core.RequestOptions;\n\n  export import Page = Pagination.Page;\n  export import PageResponse = Pagination.PageResponse;\n\n  export import CursorPage = Pagination.CursorPage;\n  export import CursorPageParams = Pagination.CursorPageParams;\n  export import CursorPageResponse = Pagination.CursorPageResponse;\n\n  export import Completions = API.Completions;\n  export import Completion = API.Completion;\n  export import CompletionChoice = API.CompletionChoice;\n  export import CompletionUsage = API.CompletionUsage;\n  export import CompletionCreateParams = API.CompletionCreateParams;\n  export import CompletionCreateParamsNonStreaming = API.CompletionCreateParamsNonStreaming;\n  export import CompletionCreateParamsStreaming = API.CompletionCreateParamsStreaming;\n\n  export import Chat = API.Chat;\n  export import ChatCompletion = API.ChatCompletion;\n  export import ChatCompletionAssistantMessageParam = API.ChatCompletionAssistantMessageParam;\n  export import ChatCompletionChunk = API.ChatCompletionChunk;\n  export import ChatCompletionContentPart = API.ChatCompletionContentPart;\n  export import ChatCompletionContentPartImage = API.ChatCompletionContentPartImage;\n  export import ChatCompletionContentPartText = API.ChatCompletionContentPartText;\n  export import ChatCompletionFunctionCallOption = API.ChatCompletionFunctionCallOption;\n  export import ChatCompletionFunctionMessageParam = API.ChatCompletionFunctionMessageParam;\n  export import ChatCompletionMessage = API.ChatCompletionMessage;\n  export import ChatCompletionMessageParam = API.ChatCompletionMessageParam;\n  export import ChatCompletionMessageToolCall = API.ChatCompletionMessageToolCall;\n  export import ChatCompletionNamedToolChoice = API.ChatCompletionNamedToolChoice;\n  export import ChatCompletionRole = API.ChatCompletionRole;\n  export import ChatCompletionSystemMessageParam = API.ChatCompletionSystemMessageParam;\n  export import ChatCompletionTokenLogprob = API.ChatCompletionTokenLogprob;\n  export import ChatCompletionTool = API.ChatCompletionTool;\n  export import ChatCompletionToolChoiceOption = API.ChatCompletionToolChoiceOption;\n  export import ChatCompletionToolMessageParam = API.ChatCompletionToolMessageParam;\n  export import ChatCompletionUserMessageParam = API.ChatCompletionUserMessageParam;\n  export import ChatCompletionCreateParams = API.ChatCompletionCreateParams;\n  export import ChatCompletionCreateParamsNonStreaming = API.ChatCompletionCreateParamsNonStreaming;\n  export import ChatCompletionCreateParamsStreaming = API.ChatCompletionCreateParamsStreaming;\n\n  export import Embeddings = API.Embeddings;\n  export import CreateEmbeddingResponse = API.CreateEmbeddingResponse;\n  export import Embedding = API.Embedding;\n  export import EmbeddingCreateParams = API.EmbeddingCreateParams;\n\n  export import Files = API.Files;\n  export import FileContent = API.FileContent;\n  export import FileDeleted = API.FileDeleted;\n  export import FileObject = API.FileObject;\n  export import FileObjectsPage = API.FileObjectsPage;\n  export import FileCreateParams = API.FileCreateParams;\n  export import FileListParams = API.FileListParams;\n\n  export import Images = API.Images;\n  export import Image = API.Image;\n  export import ImagesResponse = API.ImagesResponse;\n  export import ImageCreateVariationParams = API.ImageCreateVariationParams;\n  export import ImageEditParams = API.ImageEditParams;\n  export import ImageGenerateParams = API.ImageGenerateParams;\n\n  export import Audio = API.Audio;\n\n  export import Moderations = API.Moderations;\n  export import Moderation = API.Moderation;\n  export import ModerationCreateResponse = API.ModerationCreateResponse;\n  export import ModerationCreateParams = API.ModerationCreateParams;\n\n  export import Models = API.Models;\n  export import Model = API.Model;\n  export import ModelDeleted = API.ModelDeleted;\n  export import ModelsPage = API.ModelsPage;\n\n  export import FineTuning = API.FineTuning;\n\n  export import Beta = API.Beta;\n\n  export import ErrorObject = API.ErrorObject;\n  export import FunctionDefinition = API.FunctionDefinition;\n  export import FunctionParameters = API.FunctionParameters;\n}\n\nexport default OpenAI;\n\n// Path: src/resource.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport type { OpenAI } from './index';\n\nexport class APIResource {\n  protected _client: OpenAI;\n\n  constructor(client: OpenAI) {\n    this._client = client;\n  }\n}\n\n// Path: src/resources/audio/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport { Audio } from './audio';\nexport { SpeechCreateParams, Speech } from './speech';\nexport { Transcription, TranscriptionCreateParams, Transcriptions } from './transcriptions';\nexport { Translation, TranslationCreateParams, Translations } from './translations';\n\n// Path: src/resources/fine-tuning/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport { FineTuning } from './fine-tuning';\nexport {\n  FineTuningJob,\n  FineTuningJobEvent,\n  JobCreateParams,\n  JobListParams,\n  JobListEventsParams,\n  FineTuningJobsPage,\n  FineTuningJobEventsPage,\n  Jobs,\n} from './jobs';\n\n// Path: src/resources/beta/chat/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport { Chat } from './chat';\nexport { Completions } from './completions';\n\n// Path: src/resources/beta/threads/runs/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  CodeInterpreterLogs,\n  CodeInterpreterOutputImage,\n  CodeInterpreterToolCall,\n  CodeInterpreterToolCallDelta,\n  FunctionToolCall,\n  FunctionToolCallDelta,\n  MessageCreationStepDetails,\n  RetrievalToolCall,\n  RetrievalToolCallDelta,\n  RunStep,\n  RunStepDelta,\n  RunStepDeltaEvent,\n  RunStepDeltaMessageDelta,\n  ToolCall,\n  ToolCallDelta,\n  ToolCallDeltaObject,\n  ToolCallsStepDetails,\n  StepListParams,\n  RunStepsPage,\n  Steps,\n} from './steps';\nexport {\n  RequiredActionFunctionToolCall,\n  Run,\n  RunStatus,\n  RunCreateParams,\n  RunCreateParamsNonStreaming,\n  RunCreateParamsStreaming,\n  RunUpdateParams,\n  RunListParams,\n  RunCreateAndPollParams,\n  RunCreateAndStreamParams,\n  RunStreamParams,\n  RunSubmitToolOutputsParams,\n  RunSubmitToolOutputsParamsNonStreaming,\n  RunSubmitToolOutputsParamsStreaming,\n  RunSubmitToolOutputsAndPollParams,\n  RunSubmitToolOutputsStreamParams,\n  RunsPage,\n  Runs,\n} from './runs';\n\n// Path: src/resources/beta/threads/messages/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Annotation,\n  AnnotationDelta,\n  FileCitationAnnotation,\n  FileCitationDeltaAnnotation,\n  FilePathAnnotation,\n  FilePathDeltaAnnotation,\n  ImageFile,\n  ImageFileContentBlock,\n  ImageFileDelta,\n  ImageFileDeltaBlock,\n  Message,\n  MessageContent,\n  MessageContentDelta,\n  MessageDeleted,\n  MessageDelta,\n  MessageDeltaEvent,\n  Text,\n  TextContentBlock,\n  TextDelta,\n  TextDeltaBlock,\n  MessageCreateParams,\n  MessageUpdateParams,\n  MessageListParams,\n  MessagesPage,\n  Messages,\n} from './messages';\nexport { MessageFile, FileListParams, MessageFilesPage, Files } from './files';\n\n// Path: src/resources/beta/threads/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Annotation,\n  AnnotationDelta,\n  FileCitationAnnotation,\n  FileCitationDeltaAnnotation,\n  FilePathAnnotation,\n  FilePathDeltaAnnotation,\n  ImageFile,\n  ImageFileContentBlock,\n  ImageFileDelta,\n  ImageFileDeltaBlock,\n  Message,\n  MessageContent,\n  MessageContentDelta,\n  MessageDeleted,\n  MessageDelta,\n  MessageDeltaEvent,\n  Text,\n  TextContentBlock,\n  TextDelta,\n  TextDeltaBlock,\n  MessageCreateParams,\n  MessageUpdateParams,\n  MessageListParams,\n  MessagesPage,\n  Messages,\n} from './messages/index';\nexport {\n  RequiredActionFunctionToolCall,\n  Run,\n  RunStatus,\n  RunCreateParams,\n  RunCreateParamsNonStreaming,\n  RunCreateParamsStreaming,\n  RunUpdateParams,\n  RunListParams,\n  RunCreateAndPollParams,\n  RunCreateAndStreamParams,\n  RunStreamParams,\n  RunSubmitToolOutputsParams,\n  RunSubmitToolOutputsParamsNonStreaming,\n  RunSubmitToolOutputsParamsStreaming,\n  RunSubmitToolOutputsAndPollParams,\n  RunSubmitToolOutputsStreamParams,\n  RunsPage,\n  Runs,\n} from './runs/index';\nexport {\n  Thread,\n  ThreadDeleted,\n  ThreadCreateParams,\n  ThreadUpdateParams,\n  ThreadCreateAndRunParams,\n  ThreadCreateAndRunParamsNonStreaming,\n  ThreadCreateAndRunParamsStreaming,\n  ThreadCreateAndRunPollParams,\n  ThreadCreateAndRunStreamParams,\n  Threads,\n} from './threads';\n\n// Path: src/resources/beta/assistants/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Assistant,\n  AssistantDeleted,\n  AssistantStreamEvent,\n  AssistantTool,\n  CodeInterpreterTool,\n  FunctionTool,\n  MessageStreamEvent,\n  RetrievalTool,\n  RunStepStreamEvent,\n  RunStreamEvent,\n  ThreadStreamEvent,\n  AssistantCreateParams,\n  AssistantUpdateParams,\n  AssistantListParams,\n  AssistantsPage,\n  Assistants,\n} from './assistants';\nexport {\n  AssistantFile,\n  FileDeleteResponse,\n  FileCreateParams,\n  FileListParams,\n  AssistantFilesPage,\n  Files,\n} from './files';\n\n// Path: src/resources/beta/index.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nexport {\n  Assistant,\n  AssistantDeleted,\n  AssistantStreamEvent,\n  AssistantTool,\n  CodeInterpreterTool,\n  FunctionTool,\n  MessageStreamEvent,\n  RetrievalTool,\n  RunStepStreamEvent,\n  RunStreamEvent,\n  ThreadStreamEvent,\n  AssistantCreateParams,\n  AssistantUpdateParams,\n  AssistantListParams,\n  AssistantsPage,\n  Assistants,\n} from './assistants/index';\nexport { Beta } from './beta';\nexport { Chat } from './chat/index';\nexport {\n  Thread,\n  ThreadDeleted,\n  ThreadCreateParams,\n  ThreadUpdateParams,\n  ThreadCreateAndRunParams,\n  ThreadCreateAndRunParamsNonStreaming,\n  ThreadCreateAndRunParamsStreaming,\n  ThreadCreateAndRunPollParams,\n  ThreadCreateAndRunStreamParams,\n  Threads,\n} from './threads/index';\n\n// Path: src/lib/ChatCompletionRunFunctions.test.ts\nimport OpenAI from 'openai';\nimport { OpenAIError, APIConnectionError } from 'openai/error';\nimport { PassThrough } from 'stream';\nimport {\n  ParsingToolFunction,\n  type ChatCompletionRunner,\n  type ChatCompletionFunctionRunnerParams,\n  ChatCompletionStreamingRunner,\n  type ChatCompletionStreamingFunctionRunnerParams,\n} from 'openai/resources/beta/chat/completions';\nimport type { ChatCompletionMessageParam } from 'openai/resources/chat/completions';\n\nimport { type RequestInfo, type RequestInit } from 'openai/_shims/index';\nimport { Response } from 'node-fetch';\nimport { isAssistantMessage } from './chatCompletionUtils';\n\ntype Fetch = (req: string | RequestInfo, init?: RequestInit) => Promise<Response>;\n\n/**\n * Creates a mock `fetch` function and a `handleRequest` function for intercepting `fetch` calls.\n *\n * You call `handleRequest` with a callback function that handles the next `fetch` call.\n * It returns a Promise that:\n * - waits for the next call to `fetch`\n * - calls the callback with the `fetch` arguments\n * - resolves `fetch` with the callback output\n */\nfunction mockFetch(): { fetch: Fetch; handleRequest: (handle: Fetch) => Promise<void> } {\n  const fetchQueue: ((handler: typeof fetch) => void)[] = [];\n  const handlerQueue: Promise<typeof fetch>[] = [];\n\n  const enqueueHandler = () => {\n    handlerQueue.push(\n      new Promise<typeof fetch>((resolve) => {\n        fetchQueue.push((handle: typeof fetch) => {\n          enqueueHandler();\n          resolve(handle);\n        });\n      }),\n    );\n  };\n  enqueueHandler();\n\n  async function fetch(req: string | RequestInfo, init?: RequestInit): Promise<Response> {\n    const handler = await handlerQueue.shift();\n    if (!handler) throw new Error('expected handler to be defined');\n    const signal = init?.signal;\n    if (!signal) return await handler(req, init);\n    return await Promise.race([\n      handler(req, init),\n      new Promise<Response>((resolve, reject) => {\n        if (signal.aborted) {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n          return;\n        }\n        signal.addEventListener('abort', (e) => {\n          // @ts-ignore does exist in Node\n          reject(new DOMException('The user aborted a request.', 'AbortError'));\n        });\n      }),\n    ]);\n  }\n\n  function handleRequest(handle: typeof fetch): Promise<void> {\n    return new Promise<void>((resolve, reject) => {\n      fetchQueue.shift()?.(async (req, init) => {\n        try {\n          return await handle(req, init);\n        } catch (err) {\n          reject(err);\n          return err as any;\n        } finally {\n          resolve();\n        }\n      });\n    });\n  }\n\n  return { fetch, handleRequest };\n}\n\n// mockChatCompletionFetch is like mockFetch, but with better a more convenient handleRequest to mock\n// chat completion request/responses.\nfunction mockChatCompletionFetch() {\n  const { fetch, handleRequest: handleRawRequest } = mockFetch();\n\n  function handleRequest(\n    handler: (body: ChatCompletionFunctionRunnerParams<any[]>) => Promise<OpenAI.Chat.ChatCompletion>,\n  ): Promise<void> {\n    return handleRawRequest(async (req, init) => {\n      const rawBody = init?.body;\n      if (typeof rawBody !== 'string') throw new Error(`expected init.body to be a string`);\n      const body: ChatCompletionFunctionRunnerParams<any[]> = JSON.parse(rawBody);\n      return new Response(JSON.stringify(await handler(body)), {\n        headers: { 'Content-Type': 'application/json' },\n      });\n    });\n  }\n  return { fetch, handleRequest };\n}\n\n// mockStreamingChatCompletionFetch is like mockFetch, but with better a more convenient handleRequest to mock\n// streaming chat completion request/responses.\n\nfunction mockStreamingChatCompletionFetch() {\n  const { fetch, handleRequest: handleRawRequest } = mockFetch();\n\n  function handleRequest(\n    handler: (\n      body: ChatCompletionStreamingFunctionRunnerParams<any[]>,\n    ) => AsyncIterable<OpenAI.Chat.ChatCompletionChunk>,\n  ): Promise<void> {\n    return handleRawRequest(async (req, init) => {\n      const rawBody = init?.body;\n      if (typeof rawBody !== 'string') throw new Error(`expected init.body to be a string`);\n      const body: ChatCompletionStreamingFunctionRunnerParams<any[]> = JSON.parse(rawBody);\n      const stream = new PassThrough();\n      (async () => {\n        for await (const chunk of handler(body)) {\n          stream.write(`data: ${JSON.stringify(chunk)}\\n\\n`);\n        }\n        stream.end(`data: [DONE]\\n\\n`);\n      })();\n      return new Response(stream, {\n        headers: {\n          'Content-Type': 'text/event-stream',\n          'Transfer-Encoding': 'chunked',\n        },\n      });\n    });\n  }\n  return { fetch, handleRequest };\n}\n\n// contentChoiceDeltas returns an async iterator which mocks a delta stream of a by splitting the\n// argument into chunks separated by whitespace.\nfunction* contentChoiceDeltas(\n  content: string,\n  {\n    index = 0,\n    role = 'assistant',\n  }: { index?: number; role?: NonNullable<OpenAI.Chat.ChatCompletionChunk.Choice.Delta['role']> } = {},\n): Iterable<OpenAI.Chat.Completions.ChatCompletionChunk.Choice> {\n  const deltas = content.split(/\\s+/g);\n  for (let i = 0; i < deltas.length; i++) {\n    yield {\n      index,\n      finish_reason: i === deltas.length - 1 ? 'stop' : null,\n      logprobs: null,\n      delta: {\n        role,\n        content: deltas[i] ? `${deltas[i]}${i === deltas.length - 1 ? '' : ' '}` : null,\n      },\n    };\n  }\n}\n\n// functionCallDeltas returns an async iterator which mocks a delta stream of a functionCall by splitting\n// the argument into chunks separated by whitespace.\nfunction* functionCallDeltas(\n  args: string,\n  {\n    index = 0,\n    id = '123',\n    name,\n    role = 'assistant',\n  }: {\n    name: string;\n    id?: string;\n    index?: number;\n    role?: NonNullable<OpenAI.Chat.ChatCompletionChunk.Choice.Delta['role']>;\n  },\n): Iterable<OpenAI.Chat.Completions.ChatCompletionChunk.Choice> {\n  const deltas = args.split(/\\s+/g);\n  for (let i = 0; i < deltas.length; i++) {\n    yield {\n      index,\n      finish_reason: i === deltas.length - 1 ? 'function_call' : null,\n      delta: {\n        role,\n        tool_calls: [\n          {\n            type: 'function',\n            index: 0,\n            id,\n            function: {\n              arguments: `${deltas[i] || ''}${i === deltas.length - 1 ? '' : ' '}`,\n              ...(i === deltas.length - 1 ? { name } : null),\n            },\n          },\n        ],\n      },\n    };\n  }\n}\n\nclass RunnerListener {\n  readonly contents: string[] = [];\n  readonly messages: ChatCompletionMessageParam[] = [];\n  readonly chatCompletions: OpenAI.Chat.ChatCompletion[] = [];\n  readonly functionCalls: OpenAI.Chat.ChatCompletionMessage.FunctionCall[] = [];\n  readonly functionCallResults: string[] = [];\n  finalContent: string | null = null;\n  finalMessage: ChatCompletionMessageParam | undefined;\n  finalChatCompletion: OpenAI.Chat.ChatCompletion | undefined;\n  finalFunctionCall: OpenAI.Chat.ChatCompletionMessage.FunctionCall | undefined;\n  finalFunctionCallResult: string | undefined;\n  totalUsage: OpenAI.CompletionUsage | undefined;\n  error: OpenAIError | undefined;\n  gotConnect = false;\n  gotAbort = false;\n  gotEnd = false;\n\n  onceMessageCallCount = 0;\n\n  constructor(public runner: ChatCompletionRunner) {\n    runner\n      .on('connect', () => (this.gotConnect = true))\n      .on('content', (content) => this.contents.push(content))\n      .on('message', (message) => this.messages.push(message))\n      .on('chatCompletion', (completion) => this.chatCompletions.push(completion))\n      .on('functionCall', (functionCall) => this.functionCalls.push(functionCall))\n      .on('functionCallResult', (result) => this.functionCallResults.push(result))\n      .on('finalContent', (content) => (this.finalContent = content))\n      .on('finalMessage', (message) => (this.finalMessage = message))\n      .on('finalChatCompletion', (completion) => (this.finalChatCompletion = completion))\n      .on('finalFunctionCall', (functionCall) => (this.finalFunctionCall = functionCall))\n      .on('finalFunctionCallResult', (result) => (this.finalFunctionCallResult = result))\n      .on('totalUsage', (usage) => (this.totalUsage = usage))\n      .on('error', (error) => (this.error = error))\n      .on('abort', (error) => ((this.error = error), (this.gotAbort = true)))\n      .on('end', () => (this.gotEnd = true))\n      .once('message', () => this.onceMessageCallCount++);\n  }\n\n  async sanityCheck({ error }: { error?: string } = {}) {\n    expect(this.onceMessageCallCount).toBeLessThanOrEqual(1);\n    expect(this.gotAbort).toEqual(this.runner.aborted);\n    if (this.runner.aborted) expect(this.runner.errored).toBe(true);\n    if (error) {\n      expect(this.error?.message).toEqual(error);\n      expect(this.runner.errored).toBe(true);\n      await expect(this.runner.finalChatCompletion()).rejects.toThrow(error);\n      await expect(this.runner.finalMessage()).rejects.toThrow(error);\n      await expect(this.runner.finalContent()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCall()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCallResult()).rejects.toThrow(error);\n      await expect(this.runner.totalUsage()).rejects.toThrow(error);\n      await expect(this.runner.done()).rejects.toThrow(error);\n    } else {\n      expect(this.error).toBeUndefined();\n      expect(this.runner.errored).toBe(false);\n    }\n\n    if (!this.gotConnect) {\n      expect(this.contents).toEqual([]);\n      expect(this.messages).toEqual([]);\n      expect(this.chatCompletions).toEqual([]);\n      expect(this.functionCalls).toEqual([]);\n      expect(this.functionCallResults).toEqual([]);\n      expect(this.finalContent).toBeUndefined();\n      expect(this.finalMessage).toBeUndefined();\n      expect(this.finalChatCompletion).toBeUndefined();\n      expect(this.finalFunctionCall).toBeUndefined();\n      expect(this.finalFunctionCallResult).toBeUndefined();\n      expect(this.totalUsage).toBeUndefined();\n      expect(this.gotEnd).toBe(true);\n      return;\n    }\n\n    if (error) return;\n\n    const expectedContents = this.messages\n      .filter(isAssistantMessage)\n      .map((m) => m.content as string)\n      .filter(Boolean);\n    expect(this.contents).toEqual(expectedContents);\n    expect(this.finalMessage).toEqual([...this.messages].reverse().find((x) => x.role === 'assistant'));\n    expect(await this.runner.finalMessage()).toEqual(this.finalMessage);\n    expect(this.finalContent).toEqual(expectedContents[expectedContents.length - 1] ?? null);\n    expect(await this.runner.finalContent()).toEqual(this.finalContent);\n    expect(this.finalChatCompletion).toEqual(this.chatCompletions[this.chatCompletions.length - 1]);\n    expect(await this.runner.finalChatCompletion()).toEqual(this.finalChatCompletion);\n    expect(this.finalFunctionCall).toEqual(this.functionCalls[this.functionCalls.length - 1]);\n    expect(await this.runner.finalFunctionCall()).toEqual(this.finalFunctionCall);\n    expect(this.finalFunctionCallResult).toEqual(\n      this.functionCallResults[this.functionCallResults.length - 1],\n    );\n    expect(await this.runner.finalFunctionCallResult()).toEqual(this.finalFunctionCallResult);\n    expect(this.chatCompletions).toEqual(this.runner.allChatCompletions());\n    expect(this.messages).toEqual(this.runner.messages.slice(-this.messages.length));\n    if (this.chatCompletions.some((c) => c.usage)) {\n      const totalUsage: OpenAI.CompletionUsage = {\n        completion_tokens: 0,\n        prompt_tokens: 0,\n        total_tokens: 0,\n      };\n      for (const { usage } of this.chatCompletions) {\n        if (usage) {\n          totalUsage.completion_tokens += usage.completion_tokens;\n          totalUsage.prompt_tokens += usage.prompt_tokens;\n          totalUsage.total_tokens += usage.total_tokens;\n        }\n      }\n      expect(this.totalUsage).toEqual(totalUsage);\n      expect(await this.runner.totalUsage()).toEqual(totalUsage);\n    }\n\n    expect(this.gotEnd).toBe(true);\n  }\n}\n\nclass StreamingRunnerListener {\n  readonly eventChunks: OpenAI.Chat.ChatCompletionChunk[] = [];\n  readonly eventContents: [string, string][] = [];\n  readonly eventMessages: ChatCompletionMessageParam[] = [];\n  readonly eventChatCompletions: OpenAI.Chat.ChatCompletion[] = [];\n  readonly eventFunctionCalls: OpenAI.Chat.ChatCompletionMessage.FunctionCall[] = [];\n  readonly eventFunctionCallResults: string[] = [];\n\n  finalContent: string | null = null;\n  finalMessage: ChatCompletionMessageParam | undefined;\n  finalChatCompletion: OpenAI.Chat.ChatCompletion | undefined;\n  finalFunctionCall: OpenAI.Chat.ChatCompletionMessage.FunctionCall | undefined;\n  finalFunctionCallResult: string | undefined;\n  error: OpenAIError | undefined;\n  gotConnect = false;\n  gotEnd = false;\n\n  constructor(public runner: ChatCompletionStreamingRunner) {\n    runner\n      .on('connect', () => (this.gotConnect = true))\n      .on('chunk', (chunk) => this.eventChunks.push(chunk))\n      .on('content', (delta, snapshot) => this.eventContents.push([delta, snapshot]))\n      .on('message', (message) => this.eventMessages.push(message))\n      .on('chatCompletion', (completion) => this.eventChatCompletions.push(completion))\n      .on('functionCall', (functionCall) => this.eventFunctionCalls.push(functionCall))\n      .on('functionCallResult', (result) => this.eventFunctionCallResults.push(result))\n      .on('finalContent', (content) => (this.finalContent = content))\n      .on('finalMessage', (message) => (this.finalMessage = message))\n      .on('finalChatCompletion', (completion) => (this.finalChatCompletion = completion))\n      .on('finalFunctionCall', (functionCall) => (this.finalFunctionCall = functionCall))\n      .on('finalFunctionCallResult', (result) => (this.finalFunctionCallResult = result))\n      .on('error', (error) => (this.error = error))\n      .on('abort', (abort) => (this.error = abort))\n      .on('end', () => (this.gotEnd = true));\n  }\n\n  async sanityCheck({ error }: { error?: string } = {}) {\n    if (error) {\n      expect(this.error?.message).toEqual(error);\n      expect(this.runner.errored).toBe(true);\n      await expect(this.runner.finalChatCompletion()).rejects.toThrow(error);\n      await expect(this.runner.finalMessage()).rejects.toThrow(error);\n      await expect(this.runner.finalContent()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCall()).rejects.toThrow(error);\n      await expect(this.runner.finalFunctionCallResult()).rejects.toThrow(error);\n      await expect(this.runner.done()).rejects.toThrow(error);\n    } else {\n      expect(this.error).toBeUndefined();\n      expect(this.runner.errored).toBe(false);\n    }\n\n    if (!this.gotConnect) {\n      expect(this.eventContents).toEqual([]);\n      expect(this.eventMessages).toEqual([]);\n      expect(this.eventChatCompletions).toEqual([]);\n      expect(this.eventFunctionCalls).toEqual([]);\n      expect(this.eventFunctionCallResults).toEqual([]);\n      expect(this.finalContent).toBeUndefined();\n      expect(this.finalMessage).toBeUndefined();\n      expect(this.finalChatCompletion).toBeUndefined();\n      expect(this.finalFunctionCall).toBeUndefined();\n      expect(this.finalFunctionCallResult).toBeUndefined();\n      expect(this.gotEnd).toBe(true);\n      return;\n    }\n\n    if (error) return;\n\n    if (this.eventContents.length) expect(this.eventChunks.length).toBeGreaterThan(0);\n    expect(this.finalMessage).toEqual([...this.eventMessages].reverse().find((x) => x.role === 'assistant'));\n    expect(await this.runner.finalMessage()).toEqual(this.finalMessage);\n    expect(this.finalContent).toEqual(this.eventContents[this.eventContents.length - 1]?.[1] ?? null);\n    expect(await this.runner.finalContent()).toEqual(this.finalContent);\n    expect(this.finalChatCompletion).toEqual(this.eventChatCompletions[this.eventChatCompletions.length - 1]);\n    expect(await this.runner.finalChatCompletion()).toEqual(this.finalChatCompletion);\n    expect(this.finalFunctionCall).toEqual(this.eventFunctionCalls[this.eventFunctionCalls.length - 1]);\n    expect(await this.runner.finalFunctionCall()).toEqual(this.finalFunctionCall);\n    expect(this.finalFunctionCallResult).toEqual(\n      this.eventFunctionCallResults[this.eventFunctionCallResults.length - 1],\n    );\n    expect(await this.runner.finalFunctionCallResult()).toEqual(this.finalFunctionCallResult);\n    expect(this.eventChatCompletions).toEqual(this.runner.allChatCompletions());\n    expect(this.eventMessages).toEqual(this.runner.messages.slice(-this.eventMessages.length));\n    if (error) {\n      expect(this.error?.message).toEqual(error);\n      expect(this.runner.errored).toBe(true);\n    } else {\n      expect(this.error).toBeUndefined();\n      expect(this.runner.errored).toBe(false);\n    }\n    expect(this.gotEnd).toBe(true);\n  }\n}\n\nfunction _typeTests() {\n  const openai = new OpenAI();\n\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    tools: [\n      {\n        type: 'function',\n        function: {\n          name: 'numProperties',\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          parse: (str: string): object => {\n            const result = JSON.parse(str);\n            if (!(result instanceof Object) || Array.isArray(result)) {\n              throw new Error('must be an object');\n            }\n            return result;\n          },\n          description: 'gets the number of properties on an object',\n        },\n      },\n      {\n        type: 'function',\n        function: {\n          function: (str: string) => String(str.length),\n          parameters: { type: 'string' },\n          description: 'gets the length of a string',\n        },\n      },\n      {\n        type: 'function',\n        // @ts-expect-error function must accept string if parse is omitted\n        function: {\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n    ],\n  });\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    tools: [\n      new ParsingToolFunction({\n        name: 'numProperties',\n        // @ts-expect-error parse and function don't match\n        parse: (str: string) => str,\n        function: (obj: object) => String(Object.keys(obj).length),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n    ],\n  });\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    tools: [\n      new ParsingToolFunction({\n        name: 'numProperties',\n        parse: (str: string): object => {\n          const result = JSON.parse(str);\n          if (!(result instanceof Object) || Array.isArray(result)) {\n            throw new Error('must be an object');\n          }\n          return result;\n        },\n        function: (obj: object) => String(Object.keys(obj).length),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n      new ParsingToolFunction({\n        name: 'keys',\n        parse: (str: string): object => {\n          const result = JSON.parse(str);\n          if (!(result instanceof Object)) {\n            throw new Error('must be an Object');\n          }\n          return result;\n        },\n        function: (obj: object) => Object.keys(obj).join(', '),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n      new ParsingToolFunction({\n        name: 'len2',\n        // @ts-expect-error parse and function don't match\n        parse: (str: string) => str,\n        function: (obj: object) => String(Object.keys(obj).length),\n        parameters: { type: 'object' },\n        description: 'gets the number of properties on an object',\n      }),\n    ],\n  });\n  openai.beta.chat.completions.runTools({\n    messages: [\n      { role: 'user', content: 'can you tell me how many properties are in {\"a\": 1, \"b\": 2, \"c\": 3}' },\n    ],\n    model: 'gpt-3.5-turbo',\n    // @ts-ignore error occurs here in TS 4\n    tools: [\n      {\n        type: 'function',\n        function: {\n          name: 'numProperties',\n          parse: (str: string): object => {\n            const result = JSON.parse(str);\n            if (!(result instanceof Object) || Array.isArray(result)) {\n              throw new Error('must be an object');\n            }\n            return result;\n          },\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n      {\n        type: 'function',\n        function: {\n          name: 'keys',\n          parse: (str: string): object => {\n            const result = JSON.parse(str);\n            if (!(result instanceof Object)) {\n              throw new Error('must be an Object');\n            }\n            return result;\n          },\n          function: (obj: object) => Object.keys(obj).join(', '),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n      {\n        type: 'function',\n        function: {\n          name: 'len2',\n          parse: (str: string) => str,\n          // @ts-ignore error occurs here in TS 5\n          // function input doesn't match parse output\n          function: (obj: object) => String(Object.keys(obj).length),\n          parameters: { type: 'object' },\n          description: 'gets the number of properties on an object',\n        },\n      },\n    ] as const,\n  });\n}\n\ndescribe('resource completions', () => {\n  describe('runTools with stream: false', () => {\n    test('successful flow', async () => {\n      const { fetch, handleRequest } = mockChatCompletionFetch();\n\n      const openai = new OpenAI({ apiKey: 'something1234', baseURL: 'http://127.0.0.1:4010', fetch });\n\n      const runner = openai.beta.chat.completions.runTools({\n        messages: [{ role: 'user', content: 'tell me what the weather is like' }],\n        model: 'gpt-3.5-turbo',\n        tools: [\n          {\n            type: 'function',\n            function: {\n              function: function getWeather() {\n                return `it's raining`;\n              },\n              parameters: {},\n              description: 'gets the weather',\n            },\n          },\n        ],\n      });\n      const listener = new RunnerListener(runner);\n\n      await handleRequest(async (request) => {\n        expect(request.messages).toEqual([{ role: 'user', content: 'tell me what the weather is like' }]);\n        return {\n          id: '1',\n          choices: [\n            {\n              index: 0,\n              finish_reason: 'function_call',\n              logprobs: null,\n              message: {\n                role: 'assistant',\n...\n", "needle_token_start": 10363, "needle_token_end": 10659, "code_context_ntokens": 16219, "output": ["```typescript\nfunction mockStreamingChatCompletionFetch() {\n  const { fetch, handleRequest: handleRawRequest } = mockFetch();\n\n  function handleRequest(\n    handler: (\n      body: ChatCompletionStreamingFunctionRunnerParams<any[]>,\n    ) => AsyncIterable<OpenAI.Chat.ChatCompletionChunk>,\n  ): Promise<void> {\n    return handleRawRequest(async (req, init) => {\n      const rawBody = init?.body;\n      if (typeof rawBody !== 'string') throw new Error(`expected init.body to be a string`);\n      const body: ChatCompletionStreamingFunctionRunnerParams<any[]> = JSON.parse(rawBody);\n      const stream = new PassThrough();\n      (async () => {\n        for await (const chunk of handler(body)) {\n          stream.write(`data: ${JSON.stringify(chunk)}\\n\\n`);\n        }\n        stream.end(`data: [DONE]\\n\\n`);\n      })();\n      return new Response(stream, {\n        headers: {\n          'Content-Type': 'text/event-stream',\n          'Transfer-Encoding': 'chunked',\n        },\n      });\n    });\n  }\n  return { fetch, handleRequest };\n}\n```"]}
{"repo": "openai/openai-node", "name": "str", "language": "typescript", "path": "src/lib/ChatCompletionStream.ts", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: Converts any JavaScript value into a JSON string format.\n2. **Input**: Accepts any JavaScript value (e.g., object, array, string, number, etc.).\n3. **Output**: Returns a string that represents the JSON-encoded version of the input.\n4. **Procedure**: The function internally utilizes the JSON.stringify method to convert the input value into a JSON-formatted string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/resources/beta/assistants/assistants.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { isRequestOptions } from 'openai/core';\nimport * as AssistantsAPI from 'openai/resources/beta/assistants/assistants';\nimport * as Shared from 'openai/resources/shared';\nimport * as FilesAPI from 'openai/resources/beta/assistants/files';\nimport * as ThreadsAPI from 'openai/resources/beta/threads/threads';\nimport * as MessagesAPI from 'openai/resources/beta/threads/messages/messages';\nimport * as RunsAPI from 'openai/resources/beta/threads/runs/runs';\nimport * as StepsAPI from 'openai/resources/beta/threads/runs/steps';\nimport { CursorPage, type CursorPageParams } from 'openai/pagination';\n\nexport class Assistants extends APIResource {\n  files: FilesAPI.Files = new FilesAPI.Files(this._client);\n\n  /**\n   * Create an assistant with a model and instructions.\n   */\n  create(body: AssistantCreateParams, options?: Core.RequestOptions): Core.APIPromise<Assistant> {\n    return this._client.post('/assistants', {\n      body,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Retrieves an assistant.\n   */\n  retrieve(assistantId: string, options?: Core.RequestOptions): Core.APIPromise<Assistant> {\n    return this._client.get(`/assistants/${assistantId}`, {\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Modifies an assistant.\n   */\n  update(\n    assistantId: string,\n    body: AssistantUpdateParams,\n    options?: Core.RequestOptions,\n  ): Core.APIPromise<Assistant> {\n    return this._client.post(`/assistants/${assistantId}`, {\n      body,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Returns a list of assistants.\n   */\n  list(\n    query?: AssistantListParams,\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<AssistantsPage, Assistant>;\n  list(options?: Core.RequestOptions): Core.PagePromise<AssistantsPage, Assistant>;\n  list(\n    query: AssistantListParams | Core.RequestOptions = {},\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<AssistantsPage, Assistant> {\n    if (isRequestOptions(query)) {\n      return this.list({}, query);\n    }\n    return this._client.getAPIList('/assistants', AssistantsPage, {\n      query,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Delete an assistant.\n   */\n  del(assistantId: string, options?: Core.RequestOptions): Core.APIPromise<AssistantDeleted> {\n    return this._client.delete(`/assistants/${assistantId}`, {\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n}\n\nexport class AssistantsPage extends CursorPage<Assistant> {}\n\n/**\n * Represents an `assistant` that can call the model and use tools.\n */\nexport interface Assistant {\n  /**\n   * The identifier, which can be referenced in API endpoints.\n   */\n  id: string;\n\n  /**\n   * The Unix timestamp (in seconds) for when the assistant was created.\n   */\n  created_at: number;\n\n  /**\n   * The description of the assistant. The maximum length is 512 characters.\n   */\n  description: string | null;\n\n  /**\n   * A list of [file](https://platform.openai.com/docs/api-reference/files) IDs\n   * attached to this assistant. There can be a maximum of 20 files attached to the\n   * assistant. Files are ordered by their creation date in ascending order.\n   */\n  file_ids: Array<string>;\n\n  /**\n   * The system instructions that the assistant uses. The maximum length is 32768\n   * characters.\n   */\n  instructions: string | null;\n\n  /**\n   * Set of 16 key-value pairs that can be attached to an object. This can be useful\n   * for storing additional information about the object in a structured format. Keys\n   * can be a maximum of 64 characters long and values can be a maxium of 512\n   * characters long.\n   */\n  metadata: unknown | null;\n\n  /**\n   * ID of the model to use. You can use the\n   * [List models](https://platform.openai.com/docs/api-reference/models/list) API to\n   * see all of your available models, or see our\n   * [Model overview](https://platform.openai.com/docs/models/overview) for\n   * descriptions of them.\n   */\n  model: string;\n\n  /**\n   * The name of the assistant. The maximum length is 256 characters.\n   */\n  name: string | null;\n\n  /**\n   * The object type, which is always `assistant`.\n   */\n  object: 'assistant';\n\n  /**\n   * A list of tool enabled on the assistant. There can be a maximum of 128 tools per\n   * assistant. Tools can be of types `code_interpreter`, `retrieval`, or `function`.\n   */\n  tools: Array<AssistantTool>;\n}\n\nexport interface AssistantDeleted {\n  id: string;\n\n  deleted: boolean;\n\n  object: 'assistant.deleted';\n}\n\n/**\n * Represents an event emitted when streaming a Run.\n *\n * Each event in a server-sent events stream has an `event` and `data` property:\n *\n * ```\n * event: thread.created\n * data: {\"id\": \"thread_123\", \"object\": \"thread\", ...}\n * ```\n *\n * We emit events whenever a new object is created, transitions to a new state, or\n * is being streamed in parts (deltas). For example, we emit `thread.run.created`\n * when a new run is created, `thread.run.completed` when a run completes, and so\n * on. When an Assistant chooses to create a message during a run, we emit a\n * `thread.message.created event`, a `thread.message.in_progress` event, many\n * `thread.message.delta` events, and finally a `thread.message.completed` event.\n *\n * We may add additional events over time, so we recommend handling unknown events\n * gracefully in your code. See the\n * [Assistants API quickstart](https://platform.openai.com/docs/assistants/overview)\n * to learn how to integrate the Assistants API with streaming.\n */\nexport type AssistantStreamEvent =\n  | AssistantStreamEvent.ThreadCreated\n  | AssistantStreamEvent.ThreadRunCreated\n  | AssistantStreamEvent.ThreadRunQueued\n  | AssistantStreamEvent.ThreadRunInProgress\n  | AssistantStreamEvent.ThreadRunRequiresAction\n  | AssistantStreamEvent.ThreadRunCompleted\n  | AssistantStreamEvent.ThreadRunFailed\n  | AssistantStreamEvent.ThreadRunCancelling\n  | AssistantStreamEvent.ThreadRunCancelled\n  | AssistantStreamEvent.ThreadRunExpired\n  | AssistantStreamEvent.ThreadRunStepCreated\n  | AssistantStreamEvent.ThreadRunStepInProgress\n  | AssistantStreamEvent.ThreadRunStepDelta\n  | AssistantStreamEvent.ThreadRunStepCompleted\n  | AssistantStreamEvent.ThreadRunStepFailed\n  | AssistantStreamEvent.ThreadRunStepCancelled\n  | AssistantStreamEvent.ThreadRunStepExpired\n  | AssistantStreamEvent.ThreadMessageCreated\n  | AssistantStreamEvent.ThreadMessageInProgress\n  | AssistantStreamEvent.ThreadMessageDelta\n  | AssistantStreamEvent.ThreadMessageCompleted\n  | AssistantStreamEvent.ThreadMessageIncomplete\n  | AssistantStreamEvent.ErrorEvent;\n\nexport namespace AssistantStreamEvent {\n  /**\n   * Occurs when a new\n   * [thread](https://platform.openai.com/docs/api-reference/threads/object) is\n   * created.\n   */\n  export interface ThreadCreated {\n    /**\n     * Represents a thread that contains\n     * [messages](https://platform.openai.com/docs/api-reference/messages).\n     */\n    data: ThreadsAPI.Thread;\n\n    event: 'thread.created';\n  }\n\n  /**\n   * Occurs when a new\n   * [run](https://platform.openai.com/docs/api-reference/runs/object) is created.\n   */\n  export interface ThreadRunCreated {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.created';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `queued` status.\n   */\n  export interface ThreadRunQueued {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.queued';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to an `in_progress` status.\n   */\n  export interface ThreadRunInProgress {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.in_progress';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `requires_action` status.\n   */\n  export interface ThreadRunRequiresAction {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.requires_action';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * is completed.\n   */\n  export interface ThreadRunCompleted {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.completed';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * fails.\n   */\n  export interface ThreadRunFailed {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.failed';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `cancelling` status.\n   */\n  export interface ThreadRunCancelling {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.cancelling';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * is cancelled.\n   */\n  export interface ThreadRunCancelled {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.cancelled';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * expires.\n   */\n  export interface ThreadRunExpired {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.expired';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * created.\n   */\n  export interface ThreadRunStepCreated {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.created';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * moves to an `in_progress` state.\n   */\n  export interface ThreadRunStepInProgress {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) are\n   * being streamed.\n   */\n  export interface ThreadRunStepDelta {\n    /**\n     * Represents a run step delta i.e. any changed fields on a run step during\n     * streaming.\n     */\n    data: StepsAPI.RunStepDeltaEvent;\n\n    event: 'thread.run.step.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * completed.\n   */\n  export interface ThreadRunStepCompleted {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * fails.\n   */\n  export interface ThreadRunStepFailed {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.failed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * cancelled.\n   */\n  export interface ThreadRunStepCancelled {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.cancelled';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * expires.\n   */\n  export interface ThreadRunStepExpired {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.expired';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * created.\n   */\n  export interface ThreadMessageCreated {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.created';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) moves\n   * to an `in_progress` state.\n   */\n  export interface ThreadMessageInProgress {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [Message](https://platform.openai.com/docs/api-reference/messages/object) are\n   * being streamed.\n   */\n  export interface ThreadMessageDelta {\n    /**\n     * Represents a message delta i.e. any changed fields on a message during\n     * streaming.\n     */\n    data: MessagesAPI.MessageDeltaEvent;\n\n    event: 'thread.message.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * completed.\n   */\n  export interface ThreadMessageCompleted {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) ends\n   * before it is completed.\n   */\n  export interface ThreadMessageIncomplete {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.incomplete';\n  }\n\n  /**\n   * Occurs when an\n   * [error](https://platform.openai.com/docs/guides/error-codes/api-errors) occurs.\n   * This can happen due to an internal server error or a timeout.\n   */\n  export interface ErrorEvent {\n    data: Shared.ErrorObject;\n\n    event: 'error';\n  }\n}\n\nexport type AssistantTool = CodeInterpreterTool | RetrievalTool | FunctionTool;\n\nexport interface CodeInterpreterTool {\n  /**\n   * The type of tool being defined: `code_interpreter`\n   */\n  type: 'code_interpreter';\n}\n\nexport interface FunctionTool {\n  function: Shared.FunctionDefinition;\n\n  /**\n   * The type of tool being defined: `function`\n   */\n  type: 'function';\n}\n\n/**\n * Occurs when a\n * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n * created.\n */\nexport type MessageStreamEvent =\n  | MessageStreamEvent.ThreadMessageCreated\n  | MessageStreamEvent.ThreadMessageInProgress\n  | MessageStreamEvent.ThreadMessageDelta\n  | MessageStreamEvent.ThreadMessageCompleted\n  | MessageStreamEvent.ThreadMessageIncomplete;\n\nexport namespace MessageStreamEvent {\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * created.\n   */\n  export interface ThreadMessageCreated {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.created';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) moves\n   * to an `in_progress` state.\n   */\n  export interface ThreadMessageInProgress {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [Message](https://platform.openai.com/docs/api-reference/messages/object) are\n   * being streamed.\n   */\n  export interface ThreadMessageDelta {\n    /**\n     * Represents a message delta i.e. any changed fields on a message during\n     * streaming.\n     */\n    data: MessagesAPI.MessageDeltaEvent;\n\n    event: 'thread.message.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * completed.\n   */\n  export interface ThreadMessageCompleted {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) ends\n   * before it is completed.\n   */\n  export interface ThreadMessageIncomplete {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.incomplete';\n  }\n}\n\nexport interface RetrievalTool {\n  /**\n   * The type of tool being defined: `retrieval`\n   */\n  type: 'retrieval';\n}\n\n/**\n * Occurs when a\n * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n * created.\n */\nexport type RunStepStreamEvent =\n  | RunStepStreamEvent.ThreadRunStepCreated\n  | RunStepStreamEvent.ThreadRunStepInProgress\n  | RunStepStreamEvent.ThreadRunStepDelta\n  | RunStepStreamEvent.ThreadRunStepCompleted\n  | RunStepStreamEvent.ThreadRunStepFailed\n  | RunStepStreamEvent.ThreadRunStepCancelled\n  | RunStepStreamEvent.ThreadRunStepExpired;\n\nexport namespace RunStepStreamEvent {\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * created.\n   */\n  export interface ThreadRunStepCreated {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.created';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * moves to an `in_progress` state.\n   */\n  export interface ThreadRunStepInProgress {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) are\n   * being streamed.\n   */\n  export interface ThreadRunStepDelta {\n    /**\n     * Represents a run step delta i.e. any changed fields on a run step during\n     * streaming.\n     */\n    data: StepsAPI.RunStepDeltaEvent;\n\n    event: 'thread.run.step.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * completed.\n   */\n  export interface ThreadRunStepCompleted {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * fails.\n   */\n  export interface ThreadRunStepFailed {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.failed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * cancelled.\n   */\n  export interface ThreadRunStepCancelled {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.cancelled';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * expires.\n   */\n  export interface ThreadRunStepExpired {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.expired';\n  }\n}\n\n/**\n * Occurs when a new\n * [run](https://platform.openai.com/docs/api-reference/runs/object) is created.\n */\nexport type RunStreamEvent =\n  | RunStreamEvent.ThreadRunCreated\n  | RunStreamEvent.ThreadRunQueued\n  | RunStreamEvent.ThreadRunInProgress\n  | RunStreamEvent.ThreadRunRequiresAction\n  | RunStreamEvent.ThreadRunCompleted\n  | RunStreamEvent.ThreadRunFailed\n  | RunStreamEvent.ThreadRunCancelling\n  | RunStreamEvent.ThreadRunCancelled\n  | RunStreamEvent.ThreadRunExpired;\n\nexport namespace RunStreamEvent {\n  /**\n   * Occurs when a new\n   * [run](https://platform.openai.com/docs/api-reference/runs/object) is created.\n   */\n  export interface ThreadRunCreated {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.created';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `queued` status.\n   */\n  export interface ThreadRunQueued {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.queued';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to an `in_progress` status.\n   */\n  export interface ThreadRunInProgress {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.in_progress';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `requires_action` status.\n   */\n  export interface ThreadRunRequiresAction {\n    /**\n     * Represents an execution run on a\n...\n// Path: src/lib/jsonschema.ts\n// File mostly copied from @types/json-schema, but stripped down a bit for brevity\n// https://github.com/DefinitelyTyped/DefinitelyTyped/blob/817274f3280152ba2929a6067c93df8b34c4c9aa/types/json-schema/index.d.ts\n//\n// ==================================================================================================\n// JSON Schema Draft 07\n// ==================================================================================================\n// https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n// --------------------------------------------------------------------------------------------------\n\n/**\n * Primitive type\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1.1\n */\nexport type JSONSchemaTypeName =\n  | ({} & string)\n  | 'string'\n  | 'number'\n  | 'integer'\n  | 'boolean'\n  | 'object'\n  | 'array'\n  | 'null';\n\n/**\n * Primitive type\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1.1\n */\nexport type JSONSchemaType =\n  | string //\n  | number\n  | boolean\n  | JSONSchemaObject\n  | JSONSchemaArray\n  | null;\n\n// Workaround for infinite type recursion\nexport interface JSONSchemaObject {\n  [key: string]: JSONSchemaType;\n}\n\n// Workaround for infinite type recursion\n// https://github.com/Microsoft/TypeScript/issues/3496#issuecomment-128553540\nexport interface JSONSchemaArray extends Array<JSONSchemaType> {}\n\n/**\n * Meta schema\n *\n * Recommended values:\n * - 'http://json-schema.org/schema#'\n * - 'http://json-schema.org/hyper-schema#'\n * - 'http://json-schema.org/draft-07/schema#'\n * - 'http://json-schema.org/draft-07/hyper-schema#'\n *\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-5\n */\nexport type JSONSchemaVersion = string;\n\n/**\n * JSON Schema v7\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n */\nexport type JSONSchemaDefinition = JSONSchema | boolean;\nexport interface JSONSchema {\n  $id?: string | undefined;\n  $comment?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1\n   */\n  type?: JSONSchemaTypeName | JSONSchemaTypeName[] | undefined;\n  enum?: JSONSchemaType[] | undefined;\n  const?: JSONSchemaType | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.2\n   */\n  multipleOf?: number | undefined;\n  maximum?: number | undefined;\n  exclusiveMaximum?: number | undefined;\n  minimum?: number | undefined;\n  exclusiveMinimum?: number | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.3\n   */\n  maxLength?: number | undefined;\n  minLength?: number | undefined;\n  pattern?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.4\n   */\n  items?: JSONSchemaDefinition | JSONSchemaDefinition[] | undefined;\n  additionalItems?: JSONSchemaDefinition | undefined;\n  maxItems?: number | undefined;\n  minItems?: number | undefined;\n  uniqueItems?: boolean | undefined;\n  contains?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.5\n   */\n  maxProperties?: number | undefined;\n  minProperties?: number | undefined;\n  required?: string[] | undefined;\n  properties?:\n    | {\n        [key: string]: JSONSchemaDefinition;\n      }\n    | undefined;\n  patternProperties?:\n    | {\n        [key: string]: JSONSchemaDefinition;\n      }\n    | undefined;\n  additionalProperties?: JSONSchemaDefinition | undefined;\n  propertyNames?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.6\n   */\n  if?: JSONSchemaDefinition | undefined;\n  then?: JSONSchemaDefinition | undefined;\n  else?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.7\n   */\n  allOf?: JSONSchemaDefinition[] | undefined;\n  anyOf?: JSONSchemaDefinition[] | undefined;\n  oneOf?: JSONSchemaDefinition[] | undefined;\n  not?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-7\n   */\n  format?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-10\n   */\n  title?: string | undefined;\n  description?: string | undefined;\n  default?: JSONSchemaType | undefined;\n  readOnly?: boolean | undefined;\n  writeOnly?: boolean | undefined;\n  examples?: JSONSchemaType | undefined;\n}\n\n// Path: src/lib/ChatCompletionStream.ts\nimport * as Core from 'openai/core';\nimport { OpenAIError, APIUserAbortError } from 'openai/error';\nimport {\n  Completions,\n  type ChatCompletion,\n  type ChatCompletionChunk,\n  type ChatCompletionCreateParams,\n  type ChatCompletionCreateParamsBase,\n} from 'openai/resources/chat/completions';\nimport {\n  AbstractChatCompletionRunner,\n  type AbstractChatCompletionRunnerEvents,\n} from './AbstractChatCompletionRunner';\nimport { type ReadableStream } from 'openai/_shims/index';\nimport { Stream } from 'openai/streaming';\n\nexport interface ChatCompletionStreamEvents extends AbstractChatCompletionRunnerEvents {\n  content: (contentDelta: string, contentSnapshot: string) => void;\n  chunk: (chunk: ChatCompletionChunk, snapshot: ChatCompletionSnapshot) => void;\n}\n\nexport type ChatCompletionStreamParams = Omit<ChatCompletionCreateParamsBase, 'stream'> & {\n  stream?: true;\n};\n\nexport class ChatCompletionStream\n  extends AbstractChatCompletionRunner<ChatCompletionStreamEvents>\n  implements AsyncIterable<ChatCompletionChunk>\n{\n  #currentChatCompletionSnapshot: ChatCompletionSnapshot | undefined;\n\n  get currentChatCompletionSnapshot(): ChatCompletionSnapshot | undefined {\n    return this.#currentChatCompletionSnapshot;\n  }\n\n  /**\n   * Intended for use on the frontend, consuming a stream produced with\n   * `.toReadableStream()` on the backend.\n   *\n   * Note that messages sent to the model do not appear in `.on('message')`\n   * in this context.\n   */\n  static fromReadableStream(stream: ReadableStream): ChatCompletionStream {\n    const runner = new ChatCompletionStream();\n    runner._run(() => runner._fromReadableStream(stream));\n    return runner;\n  }\n\n  static createChatCompletion(\n    completions: Completions,\n    params: ChatCompletionStreamParams,\n    options?: Core.RequestOptions,\n  ): ChatCompletionStream {\n    const runner = new ChatCompletionStream();\n    runner._run(() =>\n      runner._runChatCompletion(\n        completions,\n        { ...params, stream: true },\n        { ...options, headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'stream' } },\n      ),\n    );\n    return runner;\n  }\n\n  #beginRequest() {\n    if (this.ended) return;\n    this.#currentChatCompletionSnapshot = undefined;\n  }\n  #addChunk(chunk: ChatCompletionChunk) {\n    if (this.ended) return;\n    const completion = this.#accumulateChatCompletion(chunk);\n    this._emit('chunk', chunk, completion);\n    const delta = chunk.choices[0]?.delta?.content;\n    const snapshot = completion.choices[0]?.message;\n    if (delta != null && snapshot?.role === 'assistant' && snapshot?.content) {\n      this._emit('content', delta, snapshot.content);\n    }\n  }\n  #endRequest(): ChatCompletion {\n    if (this.ended) {\n      throw new OpenAIError(`stream has ended, this shouldn't happen`);\n    }\n    const snapshot = this.#currentChatCompletionSnapshot;\n    if (!snapshot) {\n      throw new OpenAIError(`request ended without sending any chunks`);\n    }\n    this.#currentChatCompletionSnapshot = undefined;\n    return finalizeChatCompletion(snapshot);\n  }\n\n  protected override async _createChatCompletion(\n    completions: Completions,\n    params: ChatCompletionCreateParams,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    const signal = options?.signal;\n    if (signal) {\n      if (signal.aborted) this.controller.abort();\n      signal.addEventListener('abort', () => this.controller.abort());\n    }\n    this.#beginRequest();\n    const stream = await completions.create(\n      { ...params, stream: true },\n      { ...options, signal: this.controller.signal },\n    );\n    this._connected();\n    for await (const chunk of stream) {\n      this.#addChunk(chunk);\n    }\n    if (stream.controller.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n    return this._addChatCompletion(this.#endRequest());\n  }\n\n  protected async _fromReadableStream(\n    readableStream: ReadableStream,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    const signal = options?.signal;\n    if (signal) {\n      if (signal.aborted) this.controller.abort();\n      signal.addEventListener('abort', () => this.controller.abort());\n    }\n    this.#beginRequest();\n    this._connected();\n    const stream = Stream.fromReadableStream<ChatCompletionChunk>(readableStream, this.controller);\n    let chatId;\n    for await (const chunk of stream) {\n      if (chatId && chatId !== chunk.id) {\n        // A new request has been made.\n        this._addChatCompletion(this.#endRequest());\n      }\n\n      this.#addChunk(chunk);\n      chatId = chunk.id;\n    }\n    if (stream.controller.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n    return this._addChatCompletion(this.#endRequest());\n  }\n\n  #accumulateChatCompletion(chunk: ChatCompletionChunk): ChatCompletionSnapshot {\n    let snapshot = this.#currentChatCompletionSnapshot;\n    const { choices, ...rest } = chunk;\n    if (!snapshot) {\n      snapshot = this.#currentChatCompletionSnapshot = {\n        ...rest,\n        choices: [],\n      };\n    } else {\n      Object.assign(snapshot, rest);\n    }\n\n    for (const { delta, finish_reason, index, logprobs = null, ...other } of chunk.choices) {\n      let choice = snapshot.choices[index];\n      if (!choice) {\n        choice = snapshot.choices[index] = { finish_reason, index, message: {}, logprobs, ...other };\n      }\n\n      if (logprobs) {\n        if (!choice.logprobs) {\n          choice.logprobs = Object.assign({}, logprobs);\n        } else {\n          const { content, ...rest } = logprobs;\n          Object.assign(choice.logprobs, rest);\n          if (content) {\n            choice.logprobs.content ??= [];\n            choice.logprobs.content.push(...content);\n          }\n        }\n      }\n\n      if (finish_reason) choice.finish_reason = finish_reason;\n      Object.assign(choice, other);\n\n      if (!delta) continue; // Shouldn't happen; just in case.\n      const { content, function_call, role, tool_calls, ...rest } = delta;\n      Object.assign(choice.message, rest);\n\n      if (content) choice.message.content = (choice.message.content || '') + content;\n      if (role) choice.message.role = role;\n      if (function_call) {\n        if (!choice.message.function_call) {\n          choice.message.function_call = function_call;\n        } else {\n          if (function_call.name) choice.message.function_call.name = function_call.name;\n          if (function_call.arguments) {\n            choice.message.function_call.arguments ??= '';\n            choice.message.function_call.arguments += function_call.arguments;\n          }\n        }\n      }\n      if (tool_calls) {\n        if (!choice.message.tool_calls) choice.message.tool_calls = [];\n        for (const { index, id, type, function: fn, ...rest } of tool_calls) {\n          const tool_call = (choice.message.tool_calls[index] ??= {});\n          Object.assign(tool_call, rest);\n          if (id) tool_call.id = id;\n          if (type) tool_call.type = type;\n          if (fn) tool_call.function ??= { arguments: '' };\n          if (fn?.name) tool_call.function!.name = fn.name;\n          if (fn?.arguments) tool_call.function!.arguments += fn.arguments;\n        }\n      }\n    }\n    return snapshot;\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<ChatCompletionChunk> {\n    const pushQueue: ChatCompletionChunk[] = [];\n    const readQueue: {\n      resolve: (chunk: ChatCompletionChunk | undefined) => void;\n      reject: (err: unknown) => void;\n    }[] = [];\n    let done = false;\n\n    this.on('chunk', (chunk) => {\n      const reader = readQueue.shift();\n      if (reader) {\n        reader.resolve(chunk);\n      } else {\n        pushQueue.push(chunk);\n      }\n    });\n\n    this.on('end', () => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.resolve(undefined);\n      }\n      readQueue.length = 0;\n    });\n\n    this.on('abort', (err) => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.reject(err);\n      }\n      readQueue.length = 0;\n    });\n\n    this.on('error', (err) => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.reject(err);\n      }\n      readQueue.length = 0;\n    });\n\n    return {\n      next: async (): Promise<IteratorResult<ChatCompletionChunk>> => {\n        if (!pushQueue.length) {\n          if (done) {\n            return { value: undefined, done: true };\n          }\n          return new Promise<ChatCompletionChunk | undefined>((resolve, reject) =>\n            readQueue.push({ resolve, reject }),\n          ).then((chunk) => (chunk ? { value: chunk, done: false } : { value: undefined, done: true }));\n        }\n        const chunk = pushQueue.shift()!;\n        return { value: chunk, done: false };\n      },\n      return: async () => {\n        this.abort();\n        return { value: undefined, done: true };\n      },\n    };\n  }\n\n  toReadableStream(): ReadableStream {\n    const stream = new Stream(this[Symbol.asyncIterator].bind(this), this.controller);\n    return stream.toReadableStream();\n  }\n}\n\nfunction finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletion {\n  const { id, choices, created, model, system_fingerprint, ...rest } = snapshot;\n  return {\n    ...rest,\n    id,\n    choices: choices.map(\n      ({ message, finish_reason, index, logprobs, ...choiceRest }): ChatCompletion.Choice => {\n        if (!finish_reason) throw new OpenAIError(`missing finish_reason for choice ${index}`);\n        const { content = null, function_call, tool_calls, ...messageRest } = message;\n        const role = message.role as 'assistant'; // this is what we expect; in theory it could be different which would make our types a slight lie but would be fine.\n        if (!role) throw new OpenAIError(`missing role for choice ${index}`);\n        if (function_call) {\n          const { arguments: args, name } = function_call;\n          if (args == null) throw new OpenAIError(`missing function_call.arguments for choice ${index}`);\n          if (!name) throw new OpenAIError(`missing function_call.name for choice ${index}`);\n          return {\n            ...choiceRest,\n            message: { content, function_call: { arguments: args, name }, role },\n            finish_reason,\n            index,\n            logprobs,\n          };\n        }\n        if (tool_calls) {\n          return {\n            ...choiceRest,\n            index,\n            finish_reason,\n            logprobs,\n            message: {\n              ...messageRest,\n              role,\n              content,\n              tool_calls: tool_calls.map((tool_call, i) => {\n                const { function: fn, type, id, ...toolRest } = tool_call;\n                const { arguments: args, name, ...fnRest } = fn || {};\n                if (id == null)\n                  throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].id\\n${str(snapshot)}`);\n                if (type == null)\n                  throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].type\\n${str(snapshot)}`);\n                if (name == null)\n                  throw new OpenAIError(\n                    `missing choices[${index}].tool_calls[${i}].function.name\\n${str(snapshot)}`,\n                  );\n                if (args == null)\n                  throw new OpenAIError(\n                    `missing choices[${index}].tool_calls[${i}].function.arguments\\n${str(snapshot)}`,\n                  );\n\n                return { ...toolRest, id, type, function: { ...fnRest, name, arguments: args } };\n              }),\n            },\n          };\n        }\n        return {\n          ...choiceRest,\n          message: { ...messageRest, content, role },\n          finish_reason,\n          index,\n          logprobs,\n        };\n      },\n    ),\n    created,\n    model,\n    object: 'chat.completion',\n    ...(system_fingerprint ? { system_fingerprint } : {}),\n  };\n}\n\n\nfunction str(x: unknown) {\n  return JSON.stringify(x);\n}\n\n/**\n * Represents a streamed chunk of a chat completion response returned by model,\n * based on the provided input.\n */\nexport interface ChatCompletionSnapshot {\n  /**\n   * A unique identifier for the chat completion.\n   */\n  id: string;\n\n  /**\n   * A list of chat completion choices. Can be more than one if `n` is greater\n   * than 1.\n   */\n  choices: Array<ChatCompletionSnapshot.Choice>;\n\n  /**\n   * The Unix timestamp (in seconds) of when the chat completion was created.\n   */\n  created: number;\n\n  /**\n   * The model to generate the completion.\n   */\n  model: string;\n\n  // Note we do not include an \"object\" type on the snapshot,\n  // because the object is not a valid \"chat.completion\" until finalized.\n  // object: 'chat.completion';\n\n  /**\n   * This fingerprint represents the backend configuration that the model runs with.\n   *\n   * Can be used in conjunction with the `seed` request parameter to understand when\n   * backend changes have been made that might impact determinism.\n   */\n  system_fingerprint?: string;\n}\n\nexport namespace ChatCompletionSnapshot {\n  export interface Choice {\n    /**\n     * A chat completion delta generated by streamed model responses.\n     */\n    message: Choice.Message;\n\n    /**\n     * The reason the model stopped generating tokens. This will be `stop` if the model\n     * hit a natural stop point or a provided stop sequence, `length` if the maximum\n     * number of tokens specified in the request was reached, `content_filter` if\n     * content was omitted due to a flag from our content filters, or `function_call`\n     * if the model called a function.\n     */\n    finish_reason: ChatCompletion.Choice['finish_reason'] | null;\n\n    /**\n     * Log probability information for the choice.\n     */\n    logprobs: ChatCompletion.Choice.Logprobs | null;\n\n    /**\n     * The index of the choice in the list of choices.\n     */\n    index: number;\n  }\n\n  export namespace Choice {\n    /**\n     * A chat completion delta generated by streamed model responses.\n     */\n    export interface Message {\n      /**\n       * The contents of the chunk message.\n       */\n      content?: string | null;\n\n      /**\n       * The name and arguments of a function that should be called, as generated by the\n       * model.\n       */\n      function_call?: Message.FunctionCall;\n\n      tool_calls?: Array<Message.ToolCall>;\n\n      /**\n       * The role of the author of this message.\n       */\n      role?: 'system' | 'user' | 'assistant' | 'function' | 'tool';\n    }\n\n    export namespace Message {\n      export interface ToolCall {\n        /**\n         * The ID of the tool call.\n         */\n        id?: string;\n\n        function?: ToolCall.Function;\n\n        /**\n         * The type of the tool.\n         */\n        type?: 'function';\n      }\n\n      export namespace ToolCall {\n        export interface Function {\n          /**\n           * The arguments to call the function with, as generated by the model in JSON\n           * format. Note that the model does not always generate valid JSON, and may\n           * hallucinate parameters not defined by your function schema. Validate the\n           * arguments in your code before calling your function.\n           */\n          arguments?: string;\n\n          /**\n           * The name of the function to call.\n           */\n          name?: string;\n        }\n      }\n\n      /**\n       * The name and arguments of a function that should be called, as generated by the\n       * model.\n       */\n      export interface FunctionCall {\n        /**\n         * The arguments to call the function with, as generated by the model in JSON\n         * format. Note that the model does not always generate valid JSON, and may\n         * hallucinate parameters not defined by your function schema. Validate the\n         * arguments in your code before calling your function.\n         */\n        arguments?: string;\n\n        /**\n         * The name of the function to call.\n         */\n        name?: string;\n      }\n    }\n  }\n}\n\n// Path: src/lib/ChatCompletionStreamingRunner.ts\nimport {\n  Completions,\n  type ChatCompletionChunk,\n  type ChatCompletionCreateParamsStreaming,\n} from 'openai/resources/chat/completions';\nimport { RunnerOptions, type AbstractChatCompletionRunnerEvents } from './AbstractChatCompletionRunner';\nimport { type ReadableStream } from 'openai/_shims/index';\nimport { RunnableTools, type BaseFunctionsArgs, type RunnableFunctions } from './RunnableFunction';\nimport { ChatCompletionSnapshot, ChatCompletionStream } from './ChatCompletionStream';\n\nexport interface ChatCompletionStreamEvents extends AbstractChatCompletionRunnerEvents {\n  content: (contentDelta: string, contentSnapshot: string) => void;\n  chunk: (chunk: ChatCompletionChunk, snapshot: ChatCompletionSnapshot) => void;\n}\n\nexport type ChatCompletionStreamingFunctionRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsStreaming,\n  'functions'\n> & {\n  functions: RunnableFunctions<FunctionsArgs>;\n};\n\nexport type ChatCompletionStreamingToolRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsStreaming,\n  'tools'\n> & {\n  tools: RunnableTools<FunctionsArgs>;\n};\n\nexport class ChatCompletionStreamingRunner\n  extends ChatCompletionStream\n  implements AsyncIterable<ChatCompletionChunk>\n{\n  static override fromReadableStream(stream: ReadableStream): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    runner._run(() => runner._fromReadableStream(stream));\n    return runner;\n  }\n\n  /** @deprecated - please use `runTools` instead. */\n  static runFunctions<T extends (string | object)[]>(\n    completions: Completions,\n    params: ChatCompletionStreamingFunctionRunnerParams<T>,\n    options?: RunnerOptions,\n  ): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    const opts = {\n      ...options,\n      headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'runFunctions' },\n    };\n    runner._run(() => runner._runFunctions(completions, params, opts));\n    return runner;\n  }\n\n  static runTools<T extends (string | object)[]>(\n    completions: Completions,\n    params: ChatCompletionStreamingToolRunnerParams<T>,\n    options?: RunnerOptions,\n  ): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    const opts = {\n      ...options,\n      headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'runTools' },\n    };\n    runner._run(() => runner._runTools(completions, params, opts));\n    return runner;\n  }\n}\n\n// Path: src/lib/RunnableFunction.ts\nimport { type ChatCompletionRunner } from './ChatCompletionRunner';\nimport { type ChatCompletionStreamingRunner } from './ChatCompletionStreamingRunner';\nimport { JSONSchema } from './jsonschema';\n\ntype PromiseOrValue<T> = T | Promise<T>;\n\nexport type RunnableFunctionWithParse<Args extends object> = {\n  /**\n   * @param args the return value from `parse`.\n   * @param runner the runner evaluating this callback.\n   * @returns a string to send back to OpenAI.\n   */\n  function: (\n    args: Args,\n    runner: ChatCompletionRunner | ChatCompletionStreamingRunner,\n  ) => PromiseOrValue<unknown>;\n  /**\n   * @param input the raw args from the OpenAI function call.\n   * @returns the parsed arguments to pass to `function`\n   */\n  parse: (input: string) => PromiseOrValue<Args>;\n  /**\n   * The parameters the function accepts, describes as a JSON Schema object.\n   */\n  parameters: JSONSchema;\n  /**\n   * A description of what the function does, used by the model to choose when and how to call the function.\n   */\n  description: string;\n  /**\n   * The name of the function to be called. Will default to function.name if omitted.\n   */\n  name?: string | undefined;\n};\n\nexport type RunnableFunctionWithoutParse = {\n  /**\n   * @param args the raw args from the OpenAI function call.\n   * @returns a string to send back to OpenAI\n   */\n  function: (\n    args: string,\n    runner: ChatCompletionRunner | ChatCompletionStreamingRunner,\n  ) => PromiseOrValue<unknown>;\n  /**\n   * The parameters the function accepts, describes as a JSON Schema object.\n   */\n  parameters: JSONSchema;\n  /**\n   * A description of what the function does, used by the model to choose when and how to call the function.\n   */\n  description: string;\n  /**\n   * The name of the function to be called. Will default to function.name if omitted.\n   */\n  name?: string | undefined;\n};\n\nexport type RunnableFunction<Args extends object | string> =\n  Args extends string ? RunnableFunctionWithoutParse\n  : Args extends object ? RunnableFunctionWithParse<Args>\n  : never;\n\nexport type RunnableToolFunction<Args extends object | string> =\n  Args extends string ? RunnableToolFunctionWithoutParse\n  : Args extends object ? RunnableToolFunctionWithParse<Args>\n  : never;\n\nexport type RunnableToolFunctionWithoutParse = {\n  type: 'function';\n  function: RunnableFunctionWithoutParse;\n};\nexport type RunnableToolFunctionWithParse<Args extends object> = {\n  type: 'function';\n  function: RunnableFunctionWithParse<Args>;\n};\n\nexport function isRunnableFunctionWithParse<Args extends object>(\n  fn: any,\n): fn is RunnableFunctionWithParse<Args> {\n  return typeof (fn as any).parse === 'function';\n}\n\nexport type BaseFunctionsArgs = readonly (object | string)[];\n\nexport type RunnableFunctions<FunctionsArgs extends BaseFunctionsArgs> =\n  [any[]] extends [FunctionsArgs] ? readonly RunnableFunction<any>[]\n  : {\n      [Index in keyof FunctionsArgs]: Index extends number ? RunnableFunction<FunctionsArgs[Index]>\n      : FunctionsArgs[Index];\n    };\n\nexport type RunnableTools<FunctionsArgs extends BaseFunctionsArgs> =\n  [any[]] extends [FunctionsArgs] ? readonly RunnableToolFunction<any>[]\n  : {\n      [Index in keyof FunctionsArgs]: Index extends number ? RunnableToolFunction<FunctionsArgs[Index]>\n      : FunctionsArgs[Index];\n    };\n\n/**\n * This is helper class for passing a `function` and `parse` where the `function`\n * argument type matches the `parse` return type.\n *\n * @deprecated - please use ParsingToolFunction instead.\n */\nexport class ParsingFunction<Args extends object> {\n  function: RunnableFunctionWithParse<Args>['function'];\n  parse: RunnableFunctionWithParse<Args>['parse'];\n  parameters: RunnableFunctionWithParse<Args>['parameters'];\n  description: RunnableFunctionWithParse<Args>['description'];\n  name?: RunnableFunctionWithParse<Args>['name'];\n\n  constructor(input: RunnableFunctionWithParse<Args>) {\n    this.function = input.function;\n    this.parse = input.parse;\n    this.parameters = input.parameters;\n    this.description = input.description;\n    this.name = input.name;\n  }\n}\n\n/**\n * This is helper class for passing a `function` and `parse` where the `function`\n * argument type matches the `parse` return type.\n */\nexport class ParsingToolFunction<Args extends object> {\n  type: 'function';\n  function: RunnableFunctionWithParse<Args>;\n\n  constructor(input: RunnableFunctionWithParse<Args>) {\n    this.type = 'function';\n    this.function = input;\n  }\n}\n\n// Path: src/lib/chatCompletionUtils.ts\nimport {\n  type ChatCompletionAssistantMessageParam,\n  type ChatCompletionFunctionMessageParam,\n  type ChatCompletionMessageParam,\n  type ChatCompletionToolMessageParam,\n} from 'openai/resources';\n\nexport const isAssistantMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionAssistantMessageParam => {\n  return message?.role === 'assistant';\n};\n\nexport const isFunctionMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionFunctionMessageParam => {\n  return message?.role === 'function';\n};\n\nexport const isToolMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionToolMessageParam => {\n  return message?.role === 'tool';\n};\n\nexport function isPresent<T>(obj: T | null | undefined): obj is T {\n  return obj != null;\n}\n\n// Path: src/lib/AbstractChatCompletionRunner.ts\nimport * as Core from 'openai/core';\nimport { type CompletionUsage } from 'openai/resources/completions';\nimport {\n  type Completions,\n  type ChatCompletion,\n  type ChatCompletionMessage,\n  type ChatCompletionMessageParam,\n  type ChatCompletionCreateParams,\n  type ChatCompletionTool,\n} from 'openai/resources/chat/completions';\nimport { APIUserAbortError, OpenAIError } from 'openai/error';\nimport {\n  type RunnableFunction,\n  isRunnableFunctionWithParse,\n  type BaseFunctionsArgs,\n} from './RunnableFunction';\nimport { ChatCompletionFunctionRunnerParams, ChatCompletionToolRunnerParams } from './ChatCompletionRunner';\nimport {\n  ChatCompletionStreamingFunctionRunnerParams,\n  ChatCompletionStreamingToolRunnerParams,\n} from './ChatCompletionStreamingRunner';\nimport { isAssistantMessage, isFunctionMessage, isToolMessage } from './chatCompletionUtils';\n\nconst DEFAULT_MAX_CHAT_COMPLETIONS = 10;\nexport interface RunnerOptions extends Core.RequestOptions {\n  /** How many requests to make before canceling. Default 10. */\n  maxChatCompletions?: number;\n}\n\nexport abstract class AbstractChatCompletionRunner<\n  Events extends CustomEvents<any> = AbstractChatCompletionRunnerEvents,\n> {\n  controller: AbortController = new AbortController();\n\n  #connectedPromise: Promise<void>;\n  #resolveConnectedPromise: () => void = () => {};\n  #rejectConnectedPromise: (error: OpenAIError) => void = () => {};\n\n  #endPromise: Promise<void>;\n  #resolveEndPromise: () => void = () => {};\n  #rejectEndPromise: (error: OpenAIError) => void = () => {};\n\n  #listeners: { [Event in keyof Events]?: ListenersForEvent<Events, Event> } = {};\n\n  protected _chatCompletions: ChatCompletion[] = [];\n  messages: ChatCompletionMessageParam[] = [];\n\n  #ended = false;\n  #errored = false;\n  #aborted = false;\n  #catchingPromiseCreated = false;\n\n  constructor() {\n    this.#connectedPromise = new Promise<void>((resolve, reject) => {\n      this.#resolveConnectedPromise = resolve;\n      this.#rejectConnectedPromise = reject;\n    });\n\n    this.#endPromise = new Promise<void>((resolve, reject) => {\n      this.#resolveEndPromise = resolve;\n      this.#rejectEndPromise = reject;\n    });\n\n    // Don't let these promises cause unhandled rejection errors.\n    // we will manually cause an unhandled rejection error later\n    // if the user hasn't registered any error listener or called\n    // any promise-returning method.\n    this.#connectedPromise.catch(() => {});\n    this.#endPromise.catch(() => {});\n  }\n\n  protected _run(executor: () => Promise<any>) {\n    // Unfortunately if we call `executor()` immediately we get runtime errors about\n...\n", "needle_token_start": 12066, "needle_token_end": 12085, "code_context_ntokens": 16086, "output": ["```typescript\nfunction str(x: unknown) {\n  return JSON.stringify(x);\n}\n```"]}
{"repo": "openai/openai-node", "name": "findDoubleNewlineIndex", "language": "typescript", "path": "src/streaming.ts", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to locate the index immediately following the first occurrence of a double newline pattern in a byte array. These patterns can be either two newline characters, two carriage return characters, or a combination of both repeated twice.\n2. **Input**: Accepts a byte array (`Uint8Array`) which potentially contains one of the specified double newline patterns.\n3. **Output**: Returns an integer representing the position in the byte array immediately after the first detected double newline pattern. If no such pattern is found, it returns -1.\n4. **Procedure**: Iterates through the byte array, checking for the sequences that represent the double newline patterns (`\\n\\n`, `\\r\\r`, `\\r\\n\\r\\n`). Upon finding the first match, it calculates and returns the index right after the pattern. If the end of the array is reached without finding any pattern, it returns -1.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/core.ts\nimport { VERSION } from './version';\nimport { Stream } from './streaming';\nimport {\n  OpenAIError,\n  APIError,\n  APIConnectionError,\n  APIConnectionTimeoutError,\n  APIUserAbortError,\n} from './error';\nimport {\n  kind as shimsKind,\n  type Readable,\n  getDefaultAgent,\n  type Agent,\n  fetch,\n  type RequestInfo,\n  type RequestInit,\n  type Response,\n  type HeadersInit,\n} from './_shims/index';\nexport { type Response };\nimport { isMultipartBody } from './uploads';\nexport {\n  maybeMultipartFormRequestOptions,\n  multipartFormRequestOptions,\n  createForm,\n  type Uploadable,\n} from './uploads';\n\nexport type Fetch = (url: RequestInfo, init?: RequestInit) => Promise<Response>;\n\ntype PromiseOrValue<T> = T | Promise<T>;\n\ntype APIResponseProps = {\n  response: Response;\n  options: FinalRequestOptions;\n  controller: AbortController;\n};\n\nasync function defaultParseResponse<T>(props: APIResponseProps): Promise<T> {\n  const { response } = props;\n  if (props.options.stream) {\n    debug('response', response.status, response.url, response.headers, response.body);\n\n    // Note: there is an invariant here that isn't represented in the type system\n    // that if you set `stream: true` the response type must also be `Stream<T>`\n\n    if (props.options.__streamClass) {\n      return props.options.__streamClass.fromSSEResponse(response, props.controller) as any;\n    }\n\n    return Stream.fromSSEResponse(response, props.controller) as any;\n  }\n\n  // fetch refuses to read the body when the status code is 204.\n  if (response.status === 204) {\n    return null as T;\n  }\n\n  if (props.options.__binaryResponse) {\n    return response as unknown as T;\n  }\n\n  const contentType = response.headers.get('content-type');\n  const isJSON =\n    contentType?.includes('application/json') || contentType?.includes('application/vnd.api+json');\n  if (isJSON) {\n    const json = await response.json();\n\n    debug('response', response.status, response.url, response.headers, json);\n\n    return json as T;\n  }\n\n  const text = await response.text();\n  debug('response', response.status, response.url, response.headers, text);\n\n  // TODO handle blob, arraybuffer, other content types, etc.\n  return text as unknown as T;\n}\n\n/**\n * A subclass of `Promise` providing additional helper methods\n * for interacting with the SDK.\n */\nexport class APIPromise<T> extends Promise<T> {\n  private parsedPromise: Promise<T> | undefined;\n\n  constructor(\n    private responsePromise: Promise<APIResponseProps>,\n    private parseResponse: (props: APIResponseProps) => PromiseOrValue<T> = defaultParseResponse,\n  ) {\n    super((resolve) => {\n      // this is maybe a bit weird but this has to be a no-op to not implicitly\n      // parse the response body; instead .then, .catch, .finally are overridden\n      // to parse the response\n      resolve(null as any);\n    });\n  }\n\n  _thenUnwrap<U>(transform: (data: T) => U): APIPromise<U> {\n    return new APIPromise(this.responsePromise, async (props) => transform(await this.parseResponse(props)));\n  }\n\n  /**\n   * Gets the raw `Response` instance instead of parsing the response\n   * data.\n   *\n   * If you want to parse the response body but still get the `Response`\n   * instance, you can use {@link withResponse()}.\n   *\n   * \ud83d\udc4b Getting the wrong TypeScript type for `Response`?\n   * Try setting `\"moduleResolution\": \"NodeNext\"` if you can,\n   * or add one of these imports before your first `import \u2026 from 'openai'`:\n   * - `import 'openai/shims/node'` (if you're running on Node)\n   * - `import 'openai/shims/web'` (otherwise)\n   */\n  asResponse(): Promise<Response> {\n    return this.responsePromise.then((p) => p.response);\n  }\n  /**\n   * Gets the parsed response data and the raw `Response` instance.\n   *\n   * If you just want to get the raw `Response` instance without parsing it,\n   * you can use {@link asResponse()}.\n   *\n   *\n   * \ud83d\udc4b Getting the wrong TypeScript type for `Response`?\n   * Try setting `\"moduleResolution\": \"NodeNext\"` if you can,\n   * or add one of these imports before your first `import \u2026 from 'openai'`:\n   * - `import 'openai/shims/node'` (if you're running on Node)\n   * - `import 'openai/shims/web'` (otherwise)\n   */\n  async withResponse(): Promise<{ data: T; response: Response }> {\n    const [data, response] = await Promise.all([this.parse(), this.asResponse()]);\n    return { data, response };\n  }\n\n  private parse(): Promise<T> {\n    if (!this.parsedPromise) {\n      this.parsedPromise = this.responsePromise.then(this.parseResponse);\n    }\n    return this.parsedPromise;\n  }\n\n  override then<TResult1 = T, TResult2 = never>(\n    onfulfilled?: ((value: T) => TResult1 | PromiseLike<TResult1>) | undefined | null,\n    onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | undefined | null,\n  ): Promise<TResult1 | TResult2> {\n    return this.parse().then(onfulfilled, onrejected);\n  }\n\n  override catch<TResult = never>(\n    onrejected?: ((reason: any) => TResult | PromiseLike<TResult>) | undefined | null,\n  ): Promise<T | TResult> {\n    return this.parse().catch(onrejected);\n  }\n\n  override finally(onfinally?: (() => void) | undefined | null): Promise<T> {\n    return this.parse().finally(onfinally);\n  }\n}\n\nexport abstract class APIClient {\n  baseURL: string;\n  maxRetries: number;\n  timeout: number;\n  httpAgent: Agent | undefined;\n\n  private fetch: Fetch;\n  protected idempotencyHeader?: string;\n\n  constructor({\n    baseURL,\n    maxRetries = 2,\n    timeout = 600000, // 10 minutes\n    httpAgent,\n    fetch: overridenFetch,\n  }: {\n    baseURL: string;\n    maxRetries?: number | undefined;\n    timeout: number | undefined;\n    httpAgent: Agent | undefined;\n    fetch: Fetch | undefined;\n  }) {\n    this.baseURL = baseURL;\n    this.maxRetries = validatePositiveInteger('maxRetries', maxRetries);\n    this.timeout = validatePositiveInteger('timeout', timeout);\n    this.httpAgent = httpAgent;\n\n    this.fetch = overridenFetch ?? fetch;\n  }\n\n  protected authHeaders(opts: FinalRequestOptions): Headers {\n    return {};\n  }\n\n  /**\n   * Override this to add your own default headers, for example:\n   *\n   *  {\n   *    ...super.defaultHeaders(),\n   *    Authorization: 'Bearer 123',\n   *  }\n   */\n  protected defaultHeaders(opts: FinalRequestOptions): Headers {\n    return {\n      Accept: 'application/json',\n      'Content-Type': 'application/json',\n      'User-Agent': this.getUserAgent(),\n      ...getPlatformHeaders(),\n      ...this.authHeaders(opts),\n    };\n  }\n\n  protected abstract defaultQuery(): DefaultQuery | undefined;\n\n  /**\n   * Override this to add your own headers validation:\n   */\n  protected validateHeaders(headers: Headers, customHeaders: Headers) {}\n\n  protected defaultIdempotencyKey(): string {\n    return `stainless-node-retry-${uuid4()}`;\n  }\n\n  get<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('get', path, opts);\n  }\n\n  post<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('post', path, opts);\n  }\n\n  patch<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('patch', path, opts);\n  }\n\n  put<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('put', path, opts);\n  }\n\n  delete<Req, Rsp>(path: string, opts?: PromiseOrValue<RequestOptions<Req>>): APIPromise<Rsp> {\n    return this.methodRequest('delete', path, opts);\n  }\n\n  private methodRequest<Req, Rsp>(\n    method: HTTPMethod,\n    path: string,\n    opts?: PromiseOrValue<RequestOptions<Req>>,\n  ): APIPromise<Rsp> {\n    return this.request(Promise.resolve(opts).then((opts) => ({ method, path, ...opts })));\n  }\n\n  getAPIList<Item, PageClass extends AbstractPage<Item> = AbstractPage<Item>>(\n    path: string,\n    Page: new (...args: any[]) => PageClass,\n    opts?: RequestOptions<any>,\n  ): PagePromise<PageClass, Item> {\n    return this.requestAPIList(Page, { method: 'get', path, ...opts });\n  }\n\n  private calculateContentLength(body: unknown): string | null {\n    if (typeof body === 'string') {\n      if (typeof Buffer !== 'undefined') {\n        return Buffer.byteLength(body, 'utf8').toString();\n      }\n\n      if (typeof TextEncoder !== 'undefined') {\n        const encoder = new TextEncoder();\n        const encoded = encoder.encode(body);\n        return encoded.length.toString();\n      }\n    }\n\n    return null;\n  }\n\n  buildRequest<Req>(options: FinalRequestOptions<Req>): { req: RequestInit; url: string; timeout: number } {\n    const { method, path, query, headers: headers = {} } = options;\n\n    const body =\n      isMultipartBody(options.body) ? options.body.body\n      : options.body ? JSON.stringify(options.body, null, 2)\n      : null;\n    const contentLength = this.calculateContentLength(body);\n\n    const url = this.buildURL(path!, query);\n    if ('timeout' in options) validatePositiveInteger('timeout', options.timeout);\n    const timeout = options.timeout ?? this.timeout;\n    const httpAgent = options.httpAgent ?? this.httpAgent ?? getDefaultAgent(url);\n    const minAgentTimeout = timeout + 1000;\n    if (\n      typeof (httpAgent as any)?.options?.timeout === 'number' &&\n      minAgentTimeout > ((httpAgent as any).options.timeout ?? 0)\n    ) {\n      // Allow any given request to bump our agent active socket timeout.\n      // This may seem strange, but leaking active sockets should be rare and not particularly problematic,\n      // and without mutating agent we would need to create more of them.\n      // This tradeoff optimizes for performance.\n      (httpAgent as any).options.timeout = minAgentTimeout;\n    }\n\n    if (this.idempotencyHeader && method !== 'get') {\n      if (!options.idempotencyKey) options.idempotencyKey = this.defaultIdempotencyKey();\n      headers[this.idempotencyHeader] = options.idempotencyKey;\n    }\n\n    const reqHeaders = this.buildHeaders({ options, headers, contentLength });\n\n    const req: RequestInit = {\n      method,\n      ...(body && { body: body as any }),\n      headers: reqHeaders,\n      ...(httpAgent && { agent: httpAgent }),\n      // @ts-ignore node-fetch uses a custom AbortSignal type that is\n      // not compatible with standard web types\n      signal: options.signal ?? null,\n    };\n\n    return { req, url, timeout };\n  }\n\n  private buildHeaders({\n    options,\n    headers,\n    contentLength,\n  }: {\n    options: FinalRequestOptions;\n    headers: Record<string, string | null | undefined>;\n    contentLength: string | null | undefined;\n  }): Record<string, string> {\n    const reqHeaders: Record<string, string> = {};\n    if (contentLength) {\n      reqHeaders['content-length'] = contentLength;\n    }\n\n    const defaultHeaders = this.defaultHeaders(options);\n    applyHeadersMut(reqHeaders, defaultHeaders);\n    applyHeadersMut(reqHeaders, headers);\n\n    // let builtin fetch set the Content-Type for multipart bodies\n    if (isMultipartBody(options.body) && shimsKind !== 'node') {\n      delete reqHeaders['content-type'];\n    }\n\n    this.validateHeaders(reqHeaders, headers);\n\n    return reqHeaders;\n  }\n\n  /**\n   * Used as a callback for mutating the given `FinalRequestOptions` object.\n   */\n  protected async prepareOptions(options: FinalRequestOptions): Promise<void> {}\n\n  /**\n   * Used as a callback for mutating the given `RequestInit` object.\n   *\n   * This is useful for cases where you want to add certain headers based off of\n   * the request properties, e.g. `method` or `url`.\n   */\n  protected async prepareRequest(\n    request: RequestInit,\n    { url, options }: { url: string; options: FinalRequestOptions },\n  ): Promise<void> {}\n\n  protected parseHeaders(headers: HeadersInit | null | undefined): Record<string, string> {\n    return (\n      !headers ? {}\n      : Symbol.iterator in headers ?\n        Object.fromEntries(Array.from(headers as Iterable<string[]>).map((header) => [...header]))\n      : { ...headers }\n    );\n  }\n\n  protected makeStatusError(\n    status: number | undefined,\n    error: Object | undefined,\n    message: string | undefined,\n    headers: Headers | undefined,\n  ) {\n    return APIError.generate(status, error, message, headers);\n  }\n\n  request<Req, Rsp>(\n    options: PromiseOrValue<FinalRequestOptions<Req>>,\n    remainingRetries: number | null = null,\n  ): APIPromise<Rsp> {\n    return new APIPromise(this.makeRequest(options, remainingRetries));\n  }\n\n  private async makeRequest<Req>(\n    optionsInput: PromiseOrValue<FinalRequestOptions<Req>>,\n    retriesRemaining: number | null,\n  ): Promise<APIResponseProps> {\n    const options = await optionsInput;\n    if (retriesRemaining == null) {\n      retriesRemaining = options.maxRetries ?? this.maxRetries;\n    }\n\n    await this.prepareOptions(options);\n\n    const { req, url, timeout } = this.buildRequest(options);\n\n    await this.prepareRequest(req, { url, options });\n\n    debug('request', url, options, req.headers);\n\n    if (options.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n\n    const controller = new AbortController();\n    const response = await this.fetchWithTimeout(url, req, timeout, controller).catch(castToError);\n\n    if (response instanceof Error) {\n      if (options.signal?.aborted) {\n        throw new APIUserAbortError();\n      }\n      if (retriesRemaining) {\n        return this.retryRequest(options, retriesRemaining);\n      }\n      if (response.name === 'AbortError') {\n        throw new APIConnectionTimeoutError();\n      }\n      throw new APIConnectionError({ cause: response });\n    }\n\n    const responseHeaders = createResponseHeaders(response.headers);\n\n    if (!response.ok) {\n      if (retriesRemaining && this.shouldRetry(response)) {\n        const retryMessage = `retrying, ${retriesRemaining} attempts remaining`;\n        debug(`response (error; ${retryMessage})`, response.status, url, responseHeaders);\n        return this.retryRequest(options, retriesRemaining, responseHeaders);\n      }\n\n      const errText = await response.text().catch((e) => castToError(e).message);\n      const errJSON = safeJSON(errText);\n      const errMessage = errJSON ? undefined : errText;\n      const retryMessage = retriesRemaining ? `(error; no more retries left)` : `(error; not retryable)`;\n\n      debug(`response (error; ${retryMessage})`, response.status, url, responseHeaders, errMessage);\n\n      const err = this.makeStatusError(response.status, errJSON, errMessage, responseHeaders);\n      throw err;\n    }\n\n    return { response, options, controller };\n  }\n\n  requestAPIList<Item = unknown, PageClass extends AbstractPage<Item> = AbstractPage<Item>>(\n    Page: new (...args: ConstructorParameters<typeof AbstractPage>) => PageClass,\n    options: FinalRequestOptions,\n  ): PagePromise<PageClass, Item> {\n    const request = this.makeRequest(options, null);\n    return new PagePromise<PageClass, Item>(this, request, Page);\n  }\n\n  buildURL<Req>(path: string, query: Req | null | undefined): string {\n    const url =\n      isAbsoluteURL(path) ?\n        new URL(path)\n      : new URL(this.baseURL + (this.baseURL.endsWith('/') && path.startsWith('/') ? path.slice(1) : path));\n\n    const defaultQuery = this.defaultQuery();\n    if (!isEmptyObj(defaultQuery)) {\n      query = { ...defaultQuery, ...query } as Req;\n    }\n\n    if (typeof query === 'object' && query && !Array.isArray(query)) {\n      url.search = this.stringifyQuery(query as Record<string, unknown>);\n    }\n\n    return url.toString();\n  }\n\n  protected stringifyQuery(query: Record<string, unknown>): string {\n    return Object.entries(query)\n      .filter(([_, value]) => typeof value !== 'undefined')\n      .map(([key, value]) => {\n        if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {\n          return `${encodeURIComponent(key)}=${encodeURIComponent(value)}`;\n        }\n        if (value === null) {\n          return `${encodeURIComponent(key)}=`;\n        }\n        throw new OpenAIError(\n          `Cannot stringify type ${typeof value}; Expected string, number, boolean, or null. If you need to pass nested query parameters, you can manually encode them, e.g. { query: { 'foo[key1]': value1, 'foo[key2]': value2 } }, and please open a GitHub issue requesting better support for your use case.`,\n        );\n      })\n      .join('&');\n  }\n\n  async fetchWithTimeout(\n    url: RequestInfo,\n    init: RequestInit | undefined,\n    ms: number,\n    controller: AbortController,\n  ): Promise<Response> {\n    const { signal, ...options } = init || {};\n    if (signal) signal.addEventListener('abort', () => controller.abort());\n\n    const timeout = setTimeout(() => controller.abort(), ms);\n\n    return (\n      this.getRequestClient()\n        // use undefined this binding; fetch errors if bound to something else in browser/cloudflare\n        .fetch.call(undefined, url, { signal: controller.signal as any, ...options })\n        .finally(() => {\n          clearTimeout(timeout);\n        })\n    );\n  }\n\n  protected getRequestClient(): RequestClient {\n    return { fetch: this.fetch };\n  }\n\n  private shouldRetry(response: Response): boolean {\n    // Note this is not a standard header.\n    const shouldRetryHeader = response.headers.get('x-should-retry');\n\n    // If the server explicitly says whether or not to retry, obey.\n    if (shouldRetryHeader === 'true') return true;\n    if (shouldRetryHeader === 'false') return false;\n\n    // Retry on request timeouts.\n    if (response.status === 408) return true;\n\n    // Retry on lock timeouts.\n    if (response.status === 409) return true;\n\n    // Retry on rate limits.\n    if (response.status === 429) return true;\n\n    // Retry internal errors.\n    if (response.status >= 500) return true;\n\n    return false;\n  }\n\n  private async retryRequest(\n    options: FinalRequestOptions,\n    retriesRemaining: number,\n    responseHeaders?: Headers | undefined,\n  ): Promise<APIResponseProps> {\n    let timeoutMillis: number | undefined;\n\n    // Note the `retry-after-ms` header may not be standard, but is a good idea and we'd like proactive support for it.\n    const retryAfterMillisHeader = responseHeaders?.['retry-after-ms'];\n    if (retryAfterMillisHeader) {\n      const timeoutMs = parseFloat(retryAfterMillisHeader);\n      if (!Number.isNaN(timeoutMs)) {\n        timeoutMillis = timeoutMs;\n      }\n    }\n\n    // About the Retry-After header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After\n    const retryAfterHeader = responseHeaders?.['retry-after'];\n    if (retryAfterHeader && !timeoutMillis) {\n      const timeoutSeconds = parseFloat(retryAfterHeader);\n      if (!Number.isNaN(timeoutSeconds)) {\n        timeoutMillis = timeoutSeconds * 1000;\n      } else {\n        timeoutMillis = Date.parse(retryAfterHeader) - Date.now();\n      }\n    }\n\n    // If the API asks us to wait a certain amount of time (and it's a reasonable amount),\n    // just do what it says, but otherwise calculate a default\n    if (!(timeoutMillis && 0 <= timeoutMillis && timeoutMillis < 60 * 1000)) {\n      const maxRetries = options.maxRetries ?? this.maxRetries;\n      timeoutMillis = this.calculateDefaultRetryTimeoutMillis(retriesRemaining, maxRetries);\n    }\n    await sleep(timeoutMillis);\n\n    return this.makeRequest(options, retriesRemaining - 1);\n  }\n\n  private calculateDefaultRetryTimeoutMillis(retriesRemaining: number, maxRetries: number): number {\n    const initialRetryDelay = 0.5;\n    const maxRetryDelay = 8.0;\n\n    const numRetries = maxRetries - retriesRemaining;\n\n    // Apply exponential backoff, but not more than the max.\n    const sleepSeconds = Math.min(initialRetryDelay * Math.pow(2, numRetries), maxRetryDelay);\n\n    // Apply some jitter, take up to at most 25 percent of the retry time.\n    const jitter = 1 - Math.random() * 0.25;\n\n    return sleepSeconds * jitter * 1000;\n  }\n\n  private getUserAgent(): string {\n    return `${this.constructor.name}/JS ${VERSION}`;\n  }\n}\n\nexport type PageInfo = { url: URL } | { params: Record<string, unknown> | null };\n\nexport abstract class AbstractPage<Item> implements AsyncIterable<Item> {\n  #client: APIClient;\n  protected options: FinalRequestOptions;\n\n  protected response: Response;\n  protected body: unknown;\n\n  constructor(client: APIClient, response: Response, body: unknown, options: FinalRequestOptions) {\n    this.#client = client;\n    this.options = options;\n    this.response = response;\n    this.body = body;\n  }\n\n  /**\n   * @deprecated Use nextPageInfo instead\n   */\n  abstract nextPageParams(): Partial<Record<string, unknown>> | null;\n  abstract nextPageInfo(): PageInfo | null;\n\n  abstract getPaginatedItems(): Item[];\n\n  hasNextPage(): boolean {\n    const items = this.getPaginatedItems();\n    if (!items.length) return false;\n    return this.nextPageInfo() != null;\n  }\n\n  async getNextPage(): Promise<this> {\n    const nextInfo = this.nextPageInfo();\n    if (!nextInfo) {\n      throw new OpenAIError(\n        'No next page expected; please check `.hasNextPage()` before calling `.getNextPage()`.',\n      );\n    }\n    const nextOptions = { ...this.options };\n    if ('params' in nextInfo && typeof nextOptions.query === 'object') {\n      nextOptions.query = { ...nextOptions.query, ...nextInfo.params };\n    } else if ('url' in nextInfo) {\n      const params = [...Object.entries(nextOptions.query || {}), ...nextInfo.url.searchParams.entries()];\n      for (const [key, value] of params) {\n        nextInfo.url.searchParams.set(key, value as any);\n      }\n      nextOptions.query = undefined;\n      nextOptions.path = nextInfo.url.toString();\n    }\n    return await this.#client.requestAPIList(this.constructor as any, nextOptions);\n  }\n\n  async *iterPages() {\n    // eslint-disable-next-line @typescript-eslint/no-this-alias\n    let page: AbstractPage<Item> = this;\n    yield page;\n    while (page.hasNextPage()) {\n      page = await page.getNextPage();\n      yield page;\n    }\n  }\n\n  async *[Symbol.asyncIterator]() {\n    for await (const page of this.iterPages()) {\n      for (const item of page.getPaginatedItems()) {\n        yield item;\n      }\n    }\n  }\n}\n\n/**\n * This subclass of Promise will resolve to an instantiated Page once the request completes.\n *\n * It also implements AsyncIterable to allow auto-paginating iteration on an unawaited list call, eg:\n *\n *    for await (const item of client.items.list()) {\n *      console.log(item)\n *    }\n */\nexport class PagePromise<\n    PageClass extends AbstractPage<Item>,\n    Item = ReturnType<PageClass['getPaginatedItems']>[number],\n  >\n  extends APIPromise<PageClass>\n  implements AsyncIterable<Item>\n{\n  constructor(\n    client: APIClient,\n    request: Promise<APIResponseProps>,\n    Page: new (...args: ConstructorParameters<typeof AbstractPage>) => PageClass,\n  ) {\n    super(\n      request,\n      async (props) => new Page(client, props.response, await defaultParseResponse(props), props.options),\n    );\n  }\n\n  /**\n   * Allow auto-paginating iteration on an unawaited list call, eg:\n   *\n   *    for await (const item of client.items.list()) {\n   *      console.log(item)\n   *    }\n   */\n  async *[Symbol.asyncIterator]() {\n    const page = await this;\n    for await (const item of page) {\n      yield item;\n    }\n  }\n}\n\nexport const createResponseHeaders = (\n  headers: Awaited<ReturnType<Fetch>>['headers'],\n): Record<string, string> => {\n  return new Proxy(\n    Object.fromEntries(\n      // @ts-ignore\n      headers.entries(),\n    ),\n    {\n      get(target, name) {\n        const key = name.toString();\n        return target[key.toLowerCase()] || target[key];\n      },\n    },\n  );\n};\n\ntype HTTPMethod = 'get' | 'post' | 'put' | 'patch' | 'delete';\n\nexport type RequestClient = { fetch: Fetch };\nexport type Headers = Record<string, string | null | undefined>;\nexport type DefaultQuery = Record<string, string | undefined>;\nexport type KeysEnum<T> = { [P in keyof Required<T>]: true };\n\nexport type RequestOptions<Req = unknown | Record<string, unknown> | Readable> = {\n  method?: HTTPMethod;\n  path?: string;\n  query?: Req | undefined;\n  body?: Req | null | undefined;\n  headers?: Headers | undefined;\n\n  maxRetries?: number;\n  stream?: boolean | undefined;\n  timeout?: number;\n  httpAgent?: Agent;\n  signal?: AbortSignal | undefined | null;\n  idempotencyKey?: string;\n\n  __binaryResponse?: boolean | undefined;\n  __streamClass?: typeof Stream;\n};\n\n// This is required so that we can determine if a given object matches the RequestOptions\n// type at runtime. While this requires duplication, it is enforced by the TypeScript\n// compiler such that any missing / extraneous keys will cause an error.\nconst requestOptionsKeys: KeysEnum<RequestOptions> = {\n  method: true,\n  path: true,\n  query: true,\n  body: true,\n  headers: true,\n\n  maxRetries: true,\n  stream: true,\n  timeout: true,\n  httpAgent: true,\n  signal: true,\n  idempotencyKey: true,\n\n  __binaryResponse: true,\n  __streamClass: true,\n};\n\nexport const isRequestOptions = (obj: unknown): obj is RequestOptions => {\n  return (\n    typeof obj === 'object' &&\n    obj !== null &&\n    !isEmptyObj(obj) &&\n    Object.keys(obj).every((k) => hasOwn(requestOptionsKeys, k))\n  );\n};\n\nexport type FinalRequestOptions<Req = unknown | Record<string, unknown> | Readable> = RequestOptions<Req> & {\n  method: HTTPMethod;\n  path: string;\n};\n\ndeclare const Deno: any;\ndeclare const EdgeRuntime: any;\ntype Arch = 'x32' | 'x64' | 'arm' | 'arm64' | `other:${string}` | 'unknown';\ntype PlatformName =\n  | 'MacOS'\n  | 'Linux'\n  | 'Windows'\n  | 'FreeBSD'\n  | 'OpenBSD'\n  | 'iOS'\n  | 'Android'\n  | `Other:${string}`\n  | 'Unknown';\ntype Browser = 'ie' | 'edge' | 'chrome' | 'firefox' | 'safari';\ntype PlatformProperties = {\n  'X-Stainless-Lang': 'js';\n  'X-Stainless-Package-Version': string;\n  'X-Stainless-OS': PlatformName;\n  'X-Stainless-Arch': Arch;\n  'X-Stainless-Runtime': 'node' | 'deno' | 'edge' | `browser:${Browser}` | 'unknown';\n  'X-Stainless-Runtime-Version': string;\n};\nconst getPlatformProperties = (): PlatformProperties => {\n  if (typeof Deno !== 'undefined' && Deno.build != null) {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': normalizePlatform(Deno.build.os),\n      'X-Stainless-Arch': normalizeArch(Deno.build.arch),\n      'X-Stainless-Runtime': 'deno',\n      'X-Stainless-Runtime-Version':\n        typeof Deno.version === 'string' ? Deno.version : Deno.version?.deno ?? 'unknown',\n    };\n  }\n  if (typeof EdgeRuntime !== 'undefined') {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': 'Unknown',\n      'X-Stainless-Arch': `other:${EdgeRuntime}`,\n      'X-Stainless-Runtime': 'edge',\n      'X-Stainless-Runtime-Version': process.version,\n    };\n  }\n  // Check if Node.js\n  if (Object.prototype.toString.call(typeof process !== 'undefined' ? process : 0) === '[object process]') {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': normalizePlatform(process.platform),\n      'X-Stainless-Arch': normalizeArch(process.arch),\n      'X-Stainless-Runtime': 'node',\n      'X-Stainless-Runtime-Version': process.version,\n    };\n  }\n\n  const browserInfo = getBrowserInfo();\n  if (browserInfo) {\n    return {\n      'X-Stainless-Lang': 'js',\n      'X-Stainless-Package-Version': VERSION,\n      'X-Stainless-OS': 'Unknown',\n      'X-Stainless-Arch': 'unknown',\n      'X-Stainless-Runtime': `browser:${browserInfo.browser}`,\n      'X-Stainless-Runtime-Version': browserInfo.version,\n    };\n  }\n\n  // TODO add support for Cloudflare workers, etc.\n  return {\n    'X-Stainless-Lang': 'js',\n    'X-Stainless-Package-Version': VERSION,\n    'X-Stainless-OS': 'Unknown',\n    'X-Stainless-Arch': 'unknown',\n    'X-Stainless-Runtime': 'unknown',\n    'X-Stainless-Runtime-Version': 'unknown',\n  };\n};\n\ntype BrowserInfo = {\n  browser: Browser;\n  version: string;\n};\n\ndeclare const navigator: { userAgent: string } | undefined;\n\n// Note: modified from https://github.com/JS-DevTools/host-environment/blob/b1ab79ecde37db5d6e163c050e54fe7d287d7c92/src/isomorphic.browser.ts\nfunction getBrowserInfo(): BrowserInfo | null {\n  if (typeof navigator === 'undefined' || !navigator) {\n    return null;\n  }\n\n  // NOTE: The order matters here!\n  const browserPatterns = [\n    { key: 'edge' as const, pattern: /Edge(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /MSIE(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'ie' as const, pattern: /Trident(?:.*rv\\:(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'chrome' as const, pattern: /Chrome(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'firefox' as const, pattern: /Firefox(?:\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?/ },\n    { key: 'safari' as const, pattern: /(?:Version\\W+(\\d+)\\.(\\d+)(?:\\.(\\d+))?)?(?:\\W+Mobile\\S*)?\\W+Safari/ },\n  ];\n\n  // Find the FIRST matching browser\n  for (const { key, pattern } of browserPatterns) {\n    const match = pattern.exec(navigator.userAgent);\n    if (match) {\n      const major = match[1] || 0;\n      const minor = match[2] || 0;\n      const patch = match[3] || 0;\n\n      return { browser: key, version: `${major}.${minor}.${patch}` };\n    }\n  }\n\n  return null;\n}\n\nconst normalizeArch = (arch: string): Arch => {\n  // Node docs:\n  // - https://nodejs.org/api/process.html#processarch\n  // Deno docs:\n  // - https://doc.deno.land/deno/stable/~/Deno.build\n  if (arch === 'x32') return 'x32';\n  if (arch === 'x86_64' || arch === 'x64') return 'x64';\n  if (arch === 'arm') return 'arm';\n  if (arch === 'aarch64' || arch === 'arm64') return 'arm64';\n  if (arch) return `other:${arch}`;\n  return 'unknown';\n};\n\nconst normalizePlatform = (platform: string): PlatformName => {\n  // Node platforms:\n  // - https://nodejs.org/api/process.html#processplatform\n  // Deno platforms:\n  // - https://doc.deno.land/deno/stable/~/Deno.build\n  // - https://github.com/denoland/deno/issues/14799\n\n  platform = platform.toLowerCase();\n\n  // NOTE: this iOS check is untested and may not work\n  // Node does not work natively on IOS, there is a fork at\n  // https://github.com/nodejs-mobile/nodejs-mobile\n  // however it is unknown at the time of writing how to detect if it is running\n  if (platform.includes('ios')) return 'iOS';\n  if (platform === 'android') return 'Android';\n  if (platform === 'darwin') return 'MacOS';\n  if (platform === 'win32') return 'Windows';\n  if (platform === 'freebsd') return 'FreeBSD';\n  if (platform === 'openbsd') return 'OpenBSD';\n  if (platform === 'linux') return 'Linux';\n  if (platform) return `Other:${platform}`;\n  return 'Unknown';\n};\n\nlet _platformHeaders: PlatformProperties;\n...\n// Path: src/_shims/registry.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { type RequestOptions } from '../core';\n\nexport interface Shims {\n  kind: string;\n  fetch: any;\n  Request: any;\n  Response: any;\n  Headers: any;\n  FormData: any;\n  Blob: any;\n  File: any;\n  ReadableStream: any;\n  getMultipartRequestOptions: <T = Record<string, unknown>>(\n    form: Shims['FormData'],\n    opts: RequestOptions<T>,\n  ) => Promise<RequestOptions<T>>;\n  getDefaultAgent: (url: string) => any;\n  fileFromPath:\n    | ((path: string, filename?: string, options?: {}) => Promise<Shims['File']>)\n    | ((path: string, options?: {}) => Promise<Shims['File']>);\n  isFsReadStream: (value: any) => boolean;\n}\n\nexport let auto = false;\nexport let kind: Shims['kind'] | undefined = undefined;\nexport let fetch: Shims['fetch'] | undefined = undefined;\nexport let Request: Shims['Request'] | undefined = undefined;\nexport let Response: Shims['Response'] | undefined = undefined;\nexport let Headers: Shims['Headers'] | undefined = undefined;\nexport let FormData: Shims['FormData'] | undefined = undefined;\nexport let Blob: Shims['Blob'] | undefined = undefined;\nexport let File: Shims['File'] | undefined = undefined;\nexport let ReadableStream: Shims['ReadableStream'] | undefined = undefined;\nexport let getMultipartRequestOptions: Shims['getMultipartRequestOptions'] | undefined = undefined;\nexport let getDefaultAgent: Shims['getDefaultAgent'] | undefined = undefined;\nexport let fileFromPath: Shims['fileFromPath'] | undefined = undefined;\nexport let isFsReadStream: Shims['isFsReadStream'] | undefined = undefined;\n\nexport function setShims(shims: Shims, options: { auto: boolean } = { auto: false }) {\n  if (auto) {\n    throw new Error(\n      `you must \\`import 'openai/shims/${shims.kind}'\\` before importing anything else from openai`,\n    );\n  }\n  if (kind) {\n    throw new Error(`can't \\`import 'openai/shims/${shims.kind}'\\` after \\`import 'openai/shims/${kind}'\\``);\n  }\n  auto = options.auto;\n  kind = shims.kind;\n  fetch = shims.fetch;\n  Request = shims.Request;\n  Response = shims.Response;\n  Headers = shims.Headers;\n  FormData = shims.FormData;\n  Blob = shims.Blob;\n  File = shims.File;\n  ReadableStream = shims.ReadableStream;\n  getMultipartRequestOptions = shims.getMultipartRequestOptions;\n  getDefaultAgent = shims.getDefaultAgent;\n  fileFromPath = shims.fileFromPath;\n  isFsReadStream = shims.isFsReadStream;\n}\n\n// Path: src/_shims/web-runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nimport { MultipartBody } from './MultipartBody';\nimport { type RequestOptions } from '../core';\nimport { type Shims } from './registry';\n\nexport function getRuntime({ manuallyImported }: { manuallyImported?: boolean } = {}): Shims {\n  const recommendation =\n    manuallyImported ?\n      `You may need to use polyfills`\n    : `Add one of these imports before your first \\`import \u2026 from 'openai'\\`:\n- \\`import 'openai/shims/node'\\` (if you're running on Node)\n- \\`import 'openai/shims/web'\\` (otherwise)\n`;\n\n  let _fetch, _Request, _Response, _Headers;\n  try {\n    // @ts-ignore\n    _fetch = fetch;\n    // @ts-ignore\n    _Request = Request;\n    // @ts-ignore\n    _Response = Response;\n    // @ts-ignore\n    _Headers = Headers;\n  } catch (error) {\n    throw new Error(\n      `this environment is missing the following Web Fetch API type: ${\n        (error as any).message\n      }. ${recommendation}`,\n    );\n  }\n\n  return {\n    kind: 'web',\n    fetch: _fetch,\n    Request: _Request,\n    Response: _Response,\n    Headers: _Headers,\n    FormData:\n      // @ts-ignore\n      typeof FormData !== 'undefined' ? FormData : (\n        class FormData {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'FormData' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    Blob:\n      typeof Blob !== 'undefined' ? Blob : (\n        class Blob {\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'Blob' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    File:\n      // @ts-ignore\n      typeof File !== 'undefined' ? File : (\n        class File {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `file uploads aren't supported in this environment yet as 'File' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    ReadableStream:\n      // @ts-ignore\n      typeof ReadableStream !== 'undefined' ? ReadableStream : (\n        class ReadableStream {\n          // @ts-ignore\n          constructor() {\n            throw new Error(\n              `streaming isn't supported in this environment yet as 'ReadableStream' is undefined. ${recommendation}`,\n            );\n          }\n        }\n      ),\n    getMultipartRequestOptions: async <T = Record<string, unknown>>(\n      // @ts-ignore\n      form: FormData,\n      opts: RequestOptions<T>,\n    ): Promise<RequestOptions<T>> => ({\n      ...opts,\n      body: new MultipartBody(form) as any,\n    }),\n    getDefaultAgent: (url: string) => undefined,\n    fileFromPath: () => {\n      throw new Error(\n        'The `fileFromPath` function is only supported in Node. See the README for more details: https://www.github.com/openai/openai-node#file-uploads',\n      );\n    },\n    isFsReadStream: (value: any) => false,\n  };\n}\n\n// Path: src/_shims/auto/runtime.ts\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nexport * from '../web-runtime';\n\n// Path: src/_shims/index.js\n/**\n * Disclaimer: modules in _shims aren't intended to be imported by SDK users.\n */\nconst shims = require('./registry');\nconst auto = require('openai/_shims/auto/runtime');\nif (!shims.kind) shims.setShims(auto.getRuntime(), { auto: true });\nfor (const property of Object.keys(shims)) {\n  Object.defineProperty(exports, property, {\n    get() {\n      return shims[property];\n    },\n  });\n}\n\n// Path: src/streaming.ts\nimport { ReadableStream, type Response } from './_shims/index';\nimport { OpenAIError } from './error';\n\nimport { APIError } from 'openai/error';\n\ntype Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;\n\nexport type ServerSentEvent = {\n  event: string | null;\n  data: string;\n  raw: string[];\n};\n\nexport class Stream<Item> implements AsyncIterable<Item> {\n  controller: AbortController;\n\n  constructor(\n    private iterator: () => AsyncIterator<Item>,\n    controller: AbortController,\n  ) {\n    this.controller = controller;\n  }\n\n  static fromSSEResponse<Item>(response: Response, controller: AbortController) {\n    let consumed = false;\n\n    async function* iterator(): AsyncIterator<Item, any, undefined> {\n      if (consumed) {\n        throw new Error('Cannot iterate over a consumed stream, use `.tee()` to split the stream.');\n      }\n      consumed = true;\n      let done = false;\n      try {\n        for await (const sse of _iterSSEMessages(response, controller)) {\n          if (done) continue;\n\n          if (sse.data.startsWith('[DONE]')) {\n            done = true;\n            continue;\n          }\n\n          if (sse.event === null) {\n            let data;\n\n            try {\n              data = JSON.parse(sse.data);\n            } catch (e) {\n              console.error(`Could not parse message into JSON:`, sse.data);\n              console.error(`From chunk:`, sse.raw);\n              throw e;\n            }\n\n            if (data && data.error) {\n              throw new APIError(undefined, data.error, undefined, undefined);\n            }\n\n            yield data;\n          } else {\n            let data;\n            try {\n              data = JSON.parse(sse.data);\n            } catch (e) {\n              console.error(`Could not parse message into JSON:`, sse.data);\n              console.error(`From chunk:`, sse.raw);\n              throw e;\n            }\n            // TODO: Is this where the error should be thrown?\n            if (sse.event == 'error') {\n              throw new APIError(undefined, data.error, data.message, undefined);\n            }\n            yield { event: sse.event, data: data } as any;\n          }\n        }\n        done = true;\n      } catch (e) {\n        // If the user calls `stream.controller.abort()`, we should exit without throwing.\n        if (e instanceof Error && e.name === 'AbortError') return;\n        throw e;\n      } finally {\n        // If the user `break`s, abort the ongoing request.\n        if (!done) controller.abort();\n      }\n    }\n\n    return new Stream(iterator, controller);\n  }\n\n  /**\n   * Generates a Stream from a newline-separated ReadableStream\n   * where each item is a JSON value.\n   */\n  static fromReadableStream<Item>(readableStream: ReadableStream, controller: AbortController) {\n    let consumed = false;\n\n    async function* iterLines(): AsyncGenerator<string, void, unknown> {\n      const lineDecoder = new LineDecoder();\n\n      const iter = readableStreamAsyncIterable<Bytes>(readableStream);\n      for await (const chunk of iter) {\n        for (const line of lineDecoder.decode(chunk)) {\n          yield line;\n        }\n      }\n\n      for (const line of lineDecoder.flush()) {\n        yield line;\n      }\n    }\n\n    async function* iterator(): AsyncIterator<Item, any, undefined> {\n      if (consumed) {\n        throw new Error('Cannot iterate over a consumed stream, use `.tee()` to split the stream.');\n      }\n      consumed = true;\n      let done = false;\n      try {\n        for await (const line of iterLines()) {\n          if (done) continue;\n          if (line) yield JSON.parse(line);\n        }\n        done = true;\n      } catch (e) {\n        // If the user calls `stream.controller.abort()`, we should exit without throwing.\n        if (e instanceof Error && e.name === 'AbortError') return;\n        throw e;\n      } finally {\n        // If the user `break`s, abort the ongoing request.\n        if (!done) controller.abort();\n      }\n    }\n\n    return new Stream(iterator, controller);\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<Item> {\n    return this.iterator();\n  }\n\n  /**\n   * Splits the stream into two streams which can be\n   * independently read from at different speeds.\n   */\n  tee(): [Stream<Item>, Stream<Item>] {\n    const left: Array<Promise<IteratorResult<Item>>> = [];\n    const right: Array<Promise<IteratorResult<Item>>> = [];\n    const iterator = this.iterator();\n\n    const teeIterator = (queue: Array<Promise<IteratorResult<Item>>>): AsyncIterator<Item> => {\n      return {\n        next: () => {\n          if (queue.length === 0) {\n            const result = iterator.next();\n            left.push(result);\n            right.push(result);\n          }\n          return queue.shift()!;\n        },\n      };\n    };\n\n    return [\n      new Stream(() => teeIterator(left), this.controller),\n      new Stream(() => teeIterator(right), this.controller),\n    ];\n  }\n\n  /**\n   * Converts this stream to a newline-separated ReadableStream of\n   * JSON stringified values in the stream\n   * which can be turned back into a Stream with `Stream.fromReadableStream()`.\n   */\n  toReadableStream(): ReadableStream {\n    const self = this;\n    let iter: AsyncIterator<Item>;\n    const encoder = new TextEncoder();\n\n    return new ReadableStream({\n      async start() {\n        iter = self[Symbol.asyncIterator]();\n      },\n      async pull(ctrl: any) {\n        try {\n          const { value, done } = await iter.next();\n          if (done) return ctrl.close();\n\n          const bytes = encoder.encode(JSON.stringify(value) + '\\n');\n\n          ctrl.enqueue(bytes);\n        } catch (err) {\n          ctrl.error(err);\n        }\n      },\n      async cancel() {\n        await iter.return?.();\n      },\n    });\n  }\n}\n\nexport async function* _iterSSEMessages(\n  response: Response,\n  controller: AbortController,\n): AsyncGenerator<ServerSentEvent, void, unknown> {\n  if (!response.body) {\n    controller.abort();\n    throw new OpenAIError(`Attempted to iterate over a response with no body`);\n  }\n\n  const sseDecoder = new SSEDecoder();\n  const lineDecoder = new LineDecoder();\n\n  const iter = readableStreamAsyncIterable<Bytes>(response.body);\n  for await (const sseChunk of iterSSEChunks(iter)) {\n    for (const line of lineDecoder.decode(sseChunk)) {\n      const sse = sseDecoder.decode(line);\n      if (sse) yield sse;\n    }\n  }\n\n  for (const line of lineDecoder.flush()) {\n    const sse = sseDecoder.decode(line);\n    if (sse) yield sse;\n  }\n}\n\n/**\n * Given an async iterable iterator, iterates over it and yields full\n * SSE chunks, i.e. yields when a double new-line is encountered.\n */\nasync function* iterSSEChunks(iterator: AsyncIterableIterator<Bytes>): AsyncGenerator<Uint8Array> {\n  let data = new Uint8Array();\n\n  for await (const chunk of iterator) {\n    if (chunk == null) {\n      continue;\n    }\n\n    const binaryChunk =\n      chunk instanceof ArrayBuffer ? new Uint8Array(chunk)\n      : typeof chunk === 'string' ? new TextEncoder().encode(chunk)\n      : chunk;\n\n    let newData = new Uint8Array(data.length + binaryChunk.length);\n    newData.set(data);\n    newData.set(binaryChunk, data.length);\n    data = newData;\n\n    let patternIndex;\n    while ((patternIndex = findDoubleNewlineIndex(data)) !== -1) {\n      yield data.slice(0, patternIndex);\n      data = data.slice(patternIndex);\n    }\n  }\n\n  if (data.length > 0) {\n    yield data;\n  }\n}\n\n\nfunction findDoubleNewlineIndex(buffer: Uint8Array): number {\n  // This function searches the buffer for the end patterns (\\r\\r, \\n\\n, \\r\\n\\r\\n)\n  // and returns the index right after the first occurrence of any pattern,\n  // or -1 if none of the patterns are found.\n  const newline = 0x0a; // \\n\n  const carriage = 0x0d; // \\r\n\n  for (let i = 0; i < buffer.length - 2; i++) {\n    if (buffer[i] === newline && buffer[i + 1] === newline) {\n      // \\n\\n\n      return i + 2;\n    }\n    if (buffer[i] === carriage && buffer[i + 1] === carriage) {\n      // \\r\\r\n      return i + 2;\n    }\n    if (\n      buffer[i] === carriage &&\n      buffer[i + 1] === newline &&\n      i + 3 < buffer.length &&\n      buffer[i + 2] === carriage &&\n      buffer[i + 3] === newline\n    ) {\n      // \\r\\n\\r\\n\n      return i + 4;\n    }\n  }\n\n  return -1;\n}\n\nclass SSEDecoder {\n  private data: string[];\n  private event: string | null;\n  private chunks: string[];\n\n  constructor() {\n    this.event = null;\n    this.data = [];\n    this.chunks = [];\n  }\n\n  decode(line: string) {\n    if (line.endsWith('\\r')) {\n      line = line.substring(0, line.length - 1);\n    }\n\n    if (!line) {\n      // empty line and we didn't previously encounter any messages\n      if (!this.event && !this.data.length) return null;\n\n      const sse: ServerSentEvent = {\n        event: this.event,\n        data: this.data.join('\\n'),\n        raw: this.chunks,\n      };\n\n      this.event = null;\n      this.data = [];\n      this.chunks = [];\n\n      return sse;\n    }\n\n    this.chunks.push(line);\n\n    if (line.startsWith(':')) {\n      return null;\n    }\n\n    let [fieldname, _, value] = partition(line, ':');\n\n    if (value.startsWith(' ')) {\n      value = value.substring(1);\n    }\n\n    if (fieldname === 'event') {\n      this.event = value;\n    } else if (fieldname === 'data') {\n      this.data.push(value);\n    }\n\n    return null;\n  }\n}\n\n/**\n * A re-implementation of httpx's `LineDecoder` in Python that handles incrementally\n * reading lines from text.\n *\n * https://github.com/encode/httpx/blob/920333ea98118e9cf617f246905d7b202510941c/httpx/_decoders.py#L258\n */\nclass LineDecoder {\n  // prettier-ignore\n  static NEWLINE_CHARS = new Set(['\\n', '\\r']);\n  static NEWLINE_REGEXP = /\\r\\n|[\\n\\r]/g;\n\n  buffer: string[];\n  trailingCR: boolean;\n  textDecoder: any; // TextDecoder found in browsers; not typed to avoid pulling in either \"dom\" or \"node\" types.\n\n  constructor() {\n    this.buffer = [];\n    this.trailingCR = false;\n  }\n\n  decode(chunk: Bytes): string[] {\n    let text = this.decodeText(chunk);\n\n    if (this.trailingCR) {\n      text = '\\r' + text;\n      this.trailingCR = false;\n    }\n    if (text.endsWith('\\r')) {\n      this.trailingCR = true;\n      text = text.slice(0, -1);\n    }\n\n    if (!text) {\n      return [];\n    }\n\n    const trailingNewline = LineDecoder.NEWLINE_CHARS.has(text[text.length - 1] || '');\n    let lines = text.split(LineDecoder.NEWLINE_REGEXP);\n\n    // if there is a trailing new line then the last entry will be an empty\n    // string which we don't care about\n    if (trailingNewline) {\n      lines.pop();\n    }\n\n    if (lines.length === 1 && !trailingNewline) {\n      this.buffer.push(lines[0]!);\n      return [];\n    }\n\n    if (this.buffer.length > 0) {\n      lines = [this.buffer.join('') + lines[0], ...lines.slice(1)];\n      this.buffer = [];\n    }\n\n    if (!trailingNewline) {\n      this.buffer = [lines.pop() || ''];\n    }\n\n    return lines;\n  }\n\n  decodeText(bytes: Bytes): string {\n    if (bytes == null) return '';\n    if (typeof bytes === 'string') return bytes;\n\n    // Node:\n    if (typeof Buffer !== 'undefined') {\n      if (bytes instanceof Buffer) {\n        return bytes.toString();\n      }\n      if (bytes instanceof Uint8Array) {\n        return Buffer.from(bytes).toString();\n      }\n\n      throw new OpenAIError(\n        `Unexpected: received non-Uint8Array (${bytes.constructor.name}) stream chunk in an environment with a global \"Buffer\" defined, which this library assumes to be Node. Please report this error.`,\n      );\n    }\n\n    // Browser\n    if (typeof TextDecoder !== 'undefined') {\n      if (bytes instanceof Uint8Array || bytes instanceof ArrayBuffer) {\n        this.textDecoder ??= new TextDecoder('utf8');\n        return this.textDecoder.decode(bytes);\n      }\n\n      throw new OpenAIError(\n        `Unexpected: received non-Uint8Array/ArrayBuffer (${\n          (bytes as any).constructor.name\n        }) in a web platform. Please report this error.`,\n      );\n    }\n\n    throw new OpenAIError(\n      `Unexpected: neither Buffer nor TextDecoder are available as globals. Please report this error.`,\n    );\n  }\n\n  flush(): string[] {\n    if (!this.buffer.length && !this.trailingCR) {\n      return [];\n    }\n\n    const lines = [this.buffer.join('')];\n    this.buffer = [];\n    this.trailingCR = false;\n    return lines;\n  }\n}\n\n/** This is an internal helper function that's just used for testing */\nexport function _decodeChunks(chunks: string[]): string[] {\n  const decoder = new LineDecoder();\n  const lines: string[] = [];\n  for (const chunk of chunks) {\n    lines.push(...decoder.decode(chunk));\n  }\n\n  return lines;\n}\n\nfunction partition(str: string, delimiter: string): [string, string, string] {\n  const index = str.indexOf(delimiter);\n  if (index !== -1) {\n    return [str.substring(0, index), delimiter, str.substring(index + delimiter.length)];\n  }\n\n  return [str, '', ''];\n}\n\n/**\n * Most browsers don't yet have async iterable support for ReadableStream,\n * and Node has a very different way of reading bytes from its \"ReadableStream\".\n *\n * This polyfill was pulled from https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490\n */\nexport function readableStreamAsyncIterable<T>(stream: any): AsyncIterableIterator<T> {\n  if (stream[Symbol.asyncIterator]) return stream;\n\n  const reader = stream.getReader();\n  return {\n    async next() {\n      try {\n        const result = await reader.read();\n        if (result?.done) reader.releaseLock(); // release lock when stream becomes closed\n        return result;\n      } catch (e) {\n        reader.releaseLock(); // release lock when stream becomes errored\n        throw e;\n      }\n    },\n    async return() {\n      const cancelPromise = reader.cancel();\n      reader.releaseLock();\n      await cancelPromise;\n      return { done: true, value: undefined };\n    },\n    [Symbol.asyncIterator]() {\n      return this;\n    },\n  };\n}\n\n// Path: src/pagination.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport { AbstractPage, Response, APIClient, FinalRequestOptions, PageInfo } from './core';\n\nexport interface PageResponse<Item> {\n  data: Array<Item>;\n\n  object: string;\n}\n\n/**\n * Note: no pagination actually occurs yet, this is for forwards-compatibility.\n */\nexport class Page<Item> extends AbstractPage<Item> implements PageResponse<Item> {\n  data: Array<Item>;\n\n  object: string;\n\n  constructor(client: APIClient, response: Response, body: PageResponse<Item>, options: FinalRequestOptions) {\n    super(client, response, body, options);\n\n    this.data = body.data || [];\n    this.object = body.object;\n  }\n\n  getPaginatedItems(): Item[] {\n    return this.data ?? [];\n  }\n\n  // @deprecated Please use `nextPageInfo()` instead\n  /**\n   * This page represents a response that isn't actually paginated at the API level\n   * so there will never be any next page params.\n   */\n  nextPageParams(): null {\n    return null;\n  }\n\n  nextPageInfo(): null {\n    return null;\n  }\n}\n\nexport interface CursorPageResponse<Item> {\n  data: Array<Item>;\n}\n\nexport interface CursorPageParams {\n  after?: string;\n\n  limit?: number;\n}\n\nexport class CursorPage<Item extends { id: string }>\n  extends AbstractPage<Item>\n  implements CursorPageResponse<Item>\n{\n  data: Array<Item>;\n\n  constructor(\n    client: APIClient,\n    response: Response,\n...\n", "needle_token_start": 13565, "needle_token_end": 13870, "code_context_ntokens": 16107, "output": ["```typescript\nfunction findDoubleNewlineIndex(buffer: Uint8Array): number {\n  // This function searches the buffer for the end patterns (\\r\\r, \\n\\n, \\r\\n\\r\\n)\n  // and returns the index right after the first occurrence of any pattern,\n  // or -1 if none of the patterns are found.\n  const newline = 0x0a; // \\n\n  const carriage = 0x0d; // \\r\n\n  for (let i = 0; i < buffer.length - 2; i++) {\n    if (buffer[i] === newline && buffer[i + 1] === newline) {\n      // \\n\\n\n      return i + 2;\n    }\n    if (buffer[i] === carriage && buffer[i + 1] === carriage) {\n      // \\r\\r\n      return i + 2;\n    }\n    if (\n      buffer[i] === carriage &&\n      buffer[i + 1] === newline &&\n      i + 3 < buffer.length &&\n      buffer[i + 2] === carriage &&\n      buffer[i + 3] === newline\n    ) {\n      // \\r\\n\\r\\n\n      return i + 4;\n    }\n  }\n\n  return -1;\n}\n```"]}
{"repo": "openai/openai-node", "name": "isRunnableFunctionWithParse", "language": "typescript", "path": "src/lib/RunnableFunction.ts", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function checks if a given object includes a specific method, indicating it conforms to a particular interface that includes parsing capabilities.\n2. **Input**: The function takes a single argument, which is an object of any type.\n3. **Output**: The function returns a boolean value. It returns true if the input object contains a method for parsing, otherwise false.\n4. **Procedure**: The function examines the input object to determine if it has a method dedicated to parsing. It specifically checks for the presence of this method and returns true if found, otherwise false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/resources/beta/assistants/assistants.ts\n// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.\n\nimport * as Core from 'openai/core';\nimport { APIResource } from 'openai/resource';\nimport { isRequestOptions } from 'openai/core';\nimport * as AssistantsAPI from 'openai/resources/beta/assistants/assistants';\nimport * as Shared from 'openai/resources/shared';\nimport * as FilesAPI from 'openai/resources/beta/assistants/files';\nimport * as ThreadsAPI from 'openai/resources/beta/threads/threads';\nimport * as MessagesAPI from 'openai/resources/beta/threads/messages/messages';\nimport * as RunsAPI from 'openai/resources/beta/threads/runs/runs';\nimport * as StepsAPI from 'openai/resources/beta/threads/runs/steps';\nimport { CursorPage, type CursorPageParams } from 'openai/pagination';\n\nexport class Assistants extends APIResource {\n  files: FilesAPI.Files = new FilesAPI.Files(this._client);\n\n  /**\n   * Create an assistant with a model and instructions.\n   */\n  create(body: AssistantCreateParams, options?: Core.RequestOptions): Core.APIPromise<Assistant> {\n    return this._client.post('/assistants', {\n      body,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Retrieves an assistant.\n   */\n  retrieve(assistantId: string, options?: Core.RequestOptions): Core.APIPromise<Assistant> {\n    return this._client.get(`/assistants/${assistantId}`, {\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Modifies an assistant.\n   */\n  update(\n    assistantId: string,\n    body: AssistantUpdateParams,\n    options?: Core.RequestOptions,\n  ): Core.APIPromise<Assistant> {\n    return this._client.post(`/assistants/${assistantId}`, {\n      body,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Returns a list of assistants.\n   */\n  list(\n    query?: AssistantListParams,\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<AssistantsPage, Assistant>;\n  list(options?: Core.RequestOptions): Core.PagePromise<AssistantsPage, Assistant>;\n  list(\n    query: AssistantListParams | Core.RequestOptions = {},\n    options?: Core.RequestOptions,\n  ): Core.PagePromise<AssistantsPage, Assistant> {\n    if (isRequestOptions(query)) {\n      return this.list({}, query);\n    }\n    return this._client.getAPIList('/assistants', AssistantsPage, {\n      query,\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n\n  /**\n   * Delete an assistant.\n   */\n  del(assistantId: string, options?: Core.RequestOptions): Core.APIPromise<AssistantDeleted> {\n    return this._client.delete(`/assistants/${assistantId}`, {\n      ...options,\n      headers: { 'OpenAI-Beta': 'assistants=v1', ...options?.headers },\n    });\n  }\n}\n\nexport class AssistantsPage extends CursorPage<Assistant> {}\n\n/**\n * Represents an `assistant` that can call the model and use tools.\n */\nexport interface Assistant {\n  /**\n   * The identifier, which can be referenced in API endpoints.\n   */\n  id: string;\n\n  /**\n   * The Unix timestamp (in seconds) for when the assistant was created.\n   */\n  created_at: number;\n\n  /**\n   * The description of the assistant. The maximum length is 512 characters.\n   */\n  description: string | null;\n\n  /**\n   * A list of [file](https://platform.openai.com/docs/api-reference/files) IDs\n   * attached to this assistant. There can be a maximum of 20 files attached to the\n   * assistant. Files are ordered by their creation date in ascending order.\n   */\n  file_ids: Array<string>;\n\n  /**\n   * The system instructions that the assistant uses. The maximum length is 32768\n   * characters.\n   */\n  instructions: string | null;\n\n  /**\n   * Set of 16 key-value pairs that can be attached to an object. This can be useful\n   * for storing additional information about the object in a structured format. Keys\n   * can be a maximum of 64 characters long and values can be a maxium of 512\n   * characters long.\n   */\n  metadata: unknown | null;\n\n  /**\n   * ID of the model to use. You can use the\n   * [List models](https://platform.openai.com/docs/api-reference/models/list) API to\n   * see all of your available models, or see our\n   * [Model overview](https://platform.openai.com/docs/models/overview) for\n   * descriptions of them.\n   */\n  model: string;\n\n  /**\n   * The name of the assistant. The maximum length is 256 characters.\n   */\n  name: string | null;\n\n  /**\n   * The object type, which is always `assistant`.\n   */\n  object: 'assistant';\n\n  /**\n   * A list of tool enabled on the assistant. There can be a maximum of 128 tools per\n   * assistant. Tools can be of types `code_interpreter`, `retrieval`, or `function`.\n   */\n  tools: Array<AssistantTool>;\n}\n\nexport interface AssistantDeleted {\n  id: string;\n\n  deleted: boolean;\n\n  object: 'assistant.deleted';\n}\n\n/**\n * Represents an event emitted when streaming a Run.\n *\n * Each event in a server-sent events stream has an `event` and `data` property:\n *\n * ```\n * event: thread.created\n * data: {\"id\": \"thread_123\", \"object\": \"thread\", ...}\n * ```\n *\n * We emit events whenever a new object is created, transitions to a new state, or\n * is being streamed in parts (deltas). For example, we emit `thread.run.created`\n * when a new run is created, `thread.run.completed` when a run completes, and so\n * on. When an Assistant chooses to create a message during a run, we emit a\n * `thread.message.created event`, a `thread.message.in_progress` event, many\n * `thread.message.delta` events, and finally a `thread.message.completed` event.\n *\n * We may add additional events over time, so we recommend handling unknown events\n * gracefully in your code. See the\n * [Assistants API quickstart](https://platform.openai.com/docs/assistants/overview)\n * to learn how to integrate the Assistants API with streaming.\n */\nexport type AssistantStreamEvent =\n  | AssistantStreamEvent.ThreadCreated\n  | AssistantStreamEvent.ThreadRunCreated\n  | AssistantStreamEvent.ThreadRunQueued\n  | AssistantStreamEvent.ThreadRunInProgress\n  | AssistantStreamEvent.ThreadRunRequiresAction\n  | AssistantStreamEvent.ThreadRunCompleted\n  | AssistantStreamEvent.ThreadRunFailed\n  | AssistantStreamEvent.ThreadRunCancelling\n  | AssistantStreamEvent.ThreadRunCancelled\n  | AssistantStreamEvent.ThreadRunExpired\n  | AssistantStreamEvent.ThreadRunStepCreated\n  | AssistantStreamEvent.ThreadRunStepInProgress\n  | AssistantStreamEvent.ThreadRunStepDelta\n  | AssistantStreamEvent.ThreadRunStepCompleted\n  | AssistantStreamEvent.ThreadRunStepFailed\n  | AssistantStreamEvent.ThreadRunStepCancelled\n  | AssistantStreamEvent.ThreadRunStepExpired\n  | AssistantStreamEvent.ThreadMessageCreated\n  | AssistantStreamEvent.ThreadMessageInProgress\n  | AssistantStreamEvent.ThreadMessageDelta\n  | AssistantStreamEvent.ThreadMessageCompleted\n  | AssistantStreamEvent.ThreadMessageIncomplete\n  | AssistantStreamEvent.ErrorEvent;\n\nexport namespace AssistantStreamEvent {\n  /**\n   * Occurs when a new\n   * [thread](https://platform.openai.com/docs/api-reference/threads/object) is\n   * created.\n   */\n  export interface ThreadCreated {\n    /**\n     * Represents a thread that contains\n     * [messages](https://platform.openai.com/docs/api-reference/messages).\n     */\n    data: ThreadsAPI.Thread;\n\n    event: 'thread.created';\n  }\n\n  /**\n   * Occurs when a new\n   * [run](https://platform.openai.com/docs/api-reference/runs/object) is created.\n   */\n  export interface ThreadRunCreated {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.created';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `queued` status.\n   */\n  export interface ThreadRunQueued {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.queued';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to an `in_progress` status.\n   */\n  export interface ThreadRunInProgress {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.in_progress';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `requires_action` status.\n   */\n  export interface ThreadRunRequiresAction {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.requires_action';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * is completed.\n   */\n  export interface ThreadRunCompleted {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.completed';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * fails.\n   */\n  export interface ThreadRunFailed {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.failed';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `cancelling` status.\n   */\n  export interface ThreadRunCancelling {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.cancelling';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * is cancelled.\n   */\n  export interface ThreadRunCancelled {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.cancelled';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * expires.\n   */\n  export interface ThreadRunExpired {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.expired';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * created.\n   */\n  export interface ThreadRunStepCreated {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.created';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * moves to an `in_progress` state.\n   */\n  export interface ThreadRunStepInProgress {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) are\n   * being streamed.\n   */\n  export interface ThreadRunStepDelta {\n    /**\n     * Represents a run step delta i.e. any changed fields on a run step during\n     * streaming.\n     */\n    data: StepsAPI.RunStepDeltaEvent;\n\n    event: 'thread.run.step.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * completed.\n   */\n  export interface ThreadRunStepCompleted {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * fails.\n   */\n  export interface ThreadRunStepFailed {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.failed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * cancelled.\n   */\n  export interface ThreadRunStepCancelled {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.cancelled';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * expires.\n   */\n  export interface ThreadRunStepExpired {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.expired';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * created.\n   */\n  export interface ThreadMessageCreated {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.created';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) moves\n   * to an `in_progress` state.\n   */\n  export interface ThreadMessageInProgress {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [Message](https://platform.openai.com/docs/api-reference/messages/object) are\n   * being streamed.\n   */\n  export interface ThreadMessageDelta {\n    /**\n     * Represents a message delta i.e. any changed fields on a message during\n     * streaming.\n     */\n    data: MessagesAPI.MessageDeltaEvent;\n\n    event: 'thread.message.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * completed.\n   */\n  export interface ThreadMessageCompleted {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) ends\n   * before it is completed.\n   */\n  export interface ThreadMessageIncomplete {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.incomplete';\n  }\n\n  /**\n   * Occurs when an\n   * [error](https://platform.openai.com/docs/guides/error-codes/api-errors) occurs.\n   * This can happen due to an internal server error or a timeout.\n   */\n  export interface ErrorEvent {\n    data: Shared.ErrorObject;\n\n    event: 'error';\n  }\n}\n\nexport type AssistantTool = CodeInterpreterTool | RetrievalTool | FunctionTool;\n\nexport interface CodeInterpreterTool {\n  /**\n   * The type of tool being defined: `code_interpreter`\n   */\n  type: 'code_interpreter';\n}\n\nexport interface FunctionTool {\n  function: Shared.FunctionDefinition;\n\n  /**\n   * The type of tool being defined: `function`\n   */\n  type: 'function';\n}\n\n/**\n * Occurs when a\n * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n * created.\n */\nexport type MessageStreamEvent =\n  | MessageStreamEvent.ThreadMessageCreated\n  | MessageStreamEvent.ThreadMessageInProgress\n  | MessageStreamEvent.ThreadMessageDelta\n  | MessageStreamEvent.ThreadMessageCompleted\n  | MessageStreamEvent.ThreadMessageIncomplete;\n\nexport namespace MessageStreamEvent {\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * created.\n   */\n  export interface ThreadMessageCreated {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.created';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) moves\n   * to an `in_progress` state.\n   */\n  export interface ThreadMessageInProgress {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [Message](https://platform.openai.com/docs/api-reference/messages/object) are\n   * being streamed.\n   */\n  export interface ThreadMessageDelta {\n    /**\n     * Represents a message delta i.e. any changed fields on a message during\n     * streaming.\n     */\n    data: MessagesAPI.MessageDeltaEvent;\n\n    event: 'thread.message.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) is\n   * completed.\n   */\n  export interface ThreadMessageCompleted {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [message](https://platform.openai.com/docs/api-reference/messages/object) ends\n   * before it is completed.\n   */\n  export interface ThreadMessageIncomplete {\n    /**\n     * Represents a message within a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: MessagesAPI.Message;\n\n    event: 'thread.message.incomplete';\n  }\n}\n\nexport interface RetrievalTool {\n  /**\n   * The type of tool being defined: `retrieval`\n   */\n  type: 'retrieval';\n}\n\n/**\n * Occurs when a\n * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n * created.\n */\nexport type RunStepStreamEvent =\n  | RunStepStreamEvent.ThreadRunStepCreated\n  | RunStepStreamEvent.ThreadRunStepInProgress\n  | RunStepStreamEvent.ThreadRunStepDelta\n  | RunStepStreamEvent.ThreadRunStepCompleted\n  | RunStepStreamEvent.ThreadRunStepFailed\n  | RunStepStreamEvent.ThreadRunStepCancelled\n  | RunStepStreamEvent.ThreadRunStepExpired;\n\nexport namespace RunStepStreamEvent {\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * created.\n   */\n  export interface ThreadRunStepCreated {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.created';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * moves to an `in_progress` state.\n   */\n  export interface ThreadRunStepInProgress {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.in_progress';\n  }\n\n  /**\n   * Occurs when parts of a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) are\n   * being streamed.\n   */\n  export interface ThreadRunStepDelta {\n    /**\n     * Represents a run step delta i.e. any changed fields on a run step during\n     * streaming.\n     */\n    data: StepsAPI.RunStepDeltaEvent;\n\n    event: 'thread.run.step.delta';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * completed.\n   */\n  export interface ThreadRunStepCompleted {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.completed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * fails.\n   */\n  export interface ThreadRunStepFailed {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.failed';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object) is\n   * cancelled.\n   */\n  export interface ThreadRunStepCancelled {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.cancelled';\n  }\n\n  /**\n   * Occurs when a\n   * [run step](https://platform.openai.com/docs/api-reference/runs/step-object)\n   * expires.\n   */\n  export interface ThreadRunStepExpired {\n    /**\n     * Represents a step in execution of a run.\n     */\n    data: StepsAPI.RunStep;\n\n    event: 'thread.run.step.expired';\n  }\n}\n\n/**\n * Occurs when a new\n * [run](https://platform.openai.com/docs/api-reference/runs/object) is created.\n */\nexport type RunStreamEvent =\n  | RunStreamEvent.ThreadRunCreated\n  | RunStreamEvent.ThreadRunQueued\n  | RunStreamEvent.ThreadRunInProgress\n  | RunStreamEvent.ThreadRunRequiresAction\n  | RunStreamEvent.ThreadRunCompleted\n  | RunStreamEvent.ThreadRunFailed\n  | RunStreamEvent.ThreadRunCancelling\n  | RunStreamEvent.ThreadRunCancelled\n  | RunStreamEvent.ThreadRunExpired;\n\nexport namespace RunStreamEvent {\n  /**\n   * Occurs when a new\n   * [run](https://platform.openai.com/docs/api-reference/runs/object) is created.\n   */\n  export interface ThreadRunCreated {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.created';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `queued` status.\n   */\n  export interface ThreadRunQueued {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.queued';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to an `in_progress` status.\n   */\n  export interface ThreadRunInProgress {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.in_progress';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `requires_action` status.\n   */\n  export interface ThreadRunRequiresAction {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.requires_action';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * is completed.\n   */\n  export interface ThreadRunCompleted {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.completed';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * fails.\n   */\n  export interface ThreadRunFailed {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.failed';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * moves to a `cancelling` status.\n   */\n  export interface ThreadRunCancelling {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.cancelling';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * is cancelled.\n   */\n  export interface ThreadRunCancelled {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.cancelled';\n  }\n\n  /**\n   * Occurs when a [run](https://platform.openai.com/docs/api-reference/runs/object)\n   * expires.\n   */\n  export interface ThreadRunExpired {\n    /**\n     * Represents an execution run on a\n     * [thread](https://platform.openai.com/docs/api-reference/threads).\n     */\n    data: RunsAPI.Run;\n\n    event: 'thread.run.expired';\n  }\n}\n\n/**\n * Occurs when a new\n * [thread](https://platform.openai.com/docs/api-reference/threads/object) is\n * created.\n */\nexport interface ThreadStreamEvent {\n  /**\n   * Represents a thread that contains\n   * [messages](https://platform.openai.com/docs/api-reference/messages).\n   */\n  data: ThreadsAPI.Thread;\n\n  event: 'thread.created';\n}\n\n...\n// Path: src/lib/jsonschema.ts\n// File mostly copied from @types/json-schema, but stripped down a bit for brevity\n// https://github.com/DefinitelyTyped/DefinitelyTyped/blob/817274f3280152ba2929a6067c93df8b34c4c9aa/types/json-schema/index.d.ts\n//\n// ==================================================================================================\n// JSON Schema Draft 07\n// ==================================================================================================\n// https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n// --------------------------------------------------------------------------------------------------\n\n/**\n * Primitive type\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1.1\n */\nexport type JSONSchemaTypeName =\n  | ({} & string)\n  | 'string'\n  | 'number'\n  | 'integer'\n  | 'boolean'\n  | 'object'\n  | 'array'\n  | 'null';\n\n/**\n * Primitive type\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1.1\n */\nexport type JSONSchemaType =\n  | string //\n  | number\n  | boolean\n  | JSONSchemaObject\n  | JSONSchemaArray\n  | null;\n\n// Workaround for infinite type recursion\nexport interface JSONSchemaObject {\n  [key: string]: JSONSchemaType;\n}\n\n// Workaround for infinite type recursion\n// https://github.com/Microsoft/TypeScript/issues/3496#issuecomment-128553540\nexport interface JSONSchemaArray extends Array<JSONSchemaType> {}\n\n/**\n * Meta schema\n *\n * Recommended values:\n * - 'http://json-schema.org/schema#'\n * - 'http://json-schema.org/hyper-schema#'\n * - 'http://json-schema.org/draft-07/schema#'\n * - 'http://json-schema.org/draft-07/hyper-schema#'\n *\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-5\n */\nexport type JSONSchemaVersion = string;\n\n/**\n * JSON Schema v7\n * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01\n */\nexport type JSONSchemaDefinition = JSONSchema | boolean;\nexport interface JSONSchema {\n  $id?: string | undefined;\n  $comment?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.1\n   */\n  type?: JSONSchemaTypeName | JSONSchemaTypeName[] | undefined;\n  enum?: JSONSchemaType[] | undefined;\n  const?: JSONSchemaType | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.2\n   */\n  multipleOf?: number | undefined;\n  maximum?: number | undefined;\n  exclusiveMaximum?: number | undefined;\n  minimum?: number | undefined;\n  exclusiveMinimum?: number | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.3\n   */\n  maxLength?: number | undefined;\n  minLength?: number | undefined;\n  pattern?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.4\n   */\n  items?: JSONSchemaDefinition | JSONSchemaDefinition[] | undefined;\n  additionalItems?: JSONSchemaDefinition | undefined;\n  maxItems?: number | undefined;\n  minItems?: number | undefined;\n  uniqueItems?: boolean | undefined;\n  contains?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.5\n   */\n  maxProperties?: number | undefined;\n  minProperties?: number | undefined;\n  required?: string[] | undefined;\n  properties?:\n    | {\n        [key: string]: JSONSchemaDefinition;\n      }\n    | undefined;\n  patternProperties?:\n    | {\n        [key: string]: JSONSchemaDefinition;\n      }\n    | undefined;\n  additionalProperties?: JSONSchemaDefinition | undefined;\n  propertyNames?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.6\n   */\n  if?: JSONSchemaDefinition | undefined;\n  then?: JSONSchemaDefinition | undefined;\n  else?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-6.7\n   */\n  allOf?: JSONSchemaDefinition[] | undefined;\n  anyOf?: JSONSchemaDefinition[] | undefined;\n  oneOf?: JSONSchemaDefinition[] | undefined;\n  not?: JSONSchemaDefinition | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-7\n   */\n  format?: string | undefined;\n\n  /**\n   * @see https://tools.ietf.org/html/draft-handrews-json-schema-validation-01#section-10\n   */\n  title?: string | undefined;\n  description?: string | undefined;\n  default?: JSONSchemaType | undefined;\n  readOnly?: boolean | undefined;\n  writeOnly?: boolean | undefined;\n  examples?: JSONSchemaType | undefined;\n}\n\n// Path: src/lib/ChatCompletionStream.ts\nimport * as Core from 'openai/core';\nimport { OpenAIError, APIUserAbortError } from 'openai/error';\nimport {\n  Completions,\n  type ChatCompletion,\n  type ChatCompletionChunk,\n  type ChatCompletionCreateParams,\n  type ChatCompletionCreateParamsBase,\n} from 'openai/resources/chat/completions';\nimport {\n  AbstractChatCompletionRunner,\n  type AbstractChatCompletionRunnerEvents,\n} from './AbstractChatCompletionRunner';\nimport { type ReadableStream } from 'openai/_shims/index';\nimport { Stream } from 'openai/streaming';\n\nexport interface ChatCompletionStreamEvents extends AbstractChatCompletionRunnerEvents {\n  content: (contentDelta: string, contentSnapshot: string) => void;\n  chunk: (chunk: ChatCompletionChunk, snapshot: ChatCompletionSnapshot) => void;\n}\n\nexport type ChatCompletionStreamParams = Omit<ChatCompletionCreateParamsBase, 'stream'> & {\n  stream?: true;\n};\n\nexport class ChatCompletionStream\n  extends AbstractChatCompletionRunner<ChatCompletionStreamEvents>\n  implements AsyncIterable<ChatCompletionChunk>\n{\n  #currentChatCompletionSnapshot: ChatCompletionSnapshot | undefined;\n\n  get currentChatCompletionSnapshot(): ChatCompletionSnapshot | undefined {\n    return this.#currentChatCompletionSnapshot;\n  }\n\n  /**\n   * Intended for use on the frontend, consuming a stream produced with\n   * `.toReadableStream()` on the backend.\n   *\n   * Note that messages sent to the model do not appear in `.on('message')`\n   * in this context.\n   */\n  static fromReadableStream(stream: ReadableStream): ChatCompletionStream {\n    const runner = new ChatCompletionStream();\n    runner._run(() => runner._fromReadableStream(stream));\n    return runner;\n  }\n\n  static createChatCompletion(\n    completions: Completions,\n    params: ChatCompletionStreamParams,\n    options?: Core.RequestOptions,\n  ): ChatCompletionStream {\n    const runner = new ChatCompletionStream();\n    runner._run(() =>\n      runner._runChatCompletion(\n        completions,\n        { ...params, stream: true },\n        { ...options, headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'stream' } },\n      ),\n    );\n    return runner;\n  }\n\n  #beginRequest() {\n    if (this.ended) return;\n    this.#currentChatCompletionSnapshot = undefined;\n  }\n  #addChunk(chunk: ChatCompletionChunk) {\n    if (this.ended) return;\n    const completion = this.#accumulateChatCompletion(chunk);\n    this._emit('chunk', chunk, completion);\n    const delta = chunk.choices[0]?.delta?.content;\n    const snapshot = completion.choices[0]?.message;\n    if (delta != null && snapshot?.role === 'assistant' && snapshot?.content) {\n      this._emit('content', delta, snapshot.content);\n    }\n  }\n  #endRequest(): ChatCompletion {\n    if (this.ended) {\n      throw new OpenAIError(`stream has ended, this shouldn't happen`);\n    }\n    const snapshot = this.#currentChatCompletionSnapshot;\n    if (!snapshot) {\n      throw new OpenAIError(`request ended without sending any chunks`);\n    }\n    this.#currentChatCompletionSnapshot = undefined;\n    return finalizeChatCompletion(snapshot);\n  }\n\n  protected override async _createChatCompletion(\n    completions: Completions,\n    params: ChatCompletionCreateParams,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    const signal = options?.signal;\n    if (signal) {\n      if (signal.aborted) this.controller.abort();\n      signal.addEventListener('abort', () => this.controller.abort());\n    }\n    this.#beginRequest();\n    const stream = await completions.create(\n      { ...params, stream: true },\n      { ...options, signal: this.controller.signal },\n    );\n    this._connected();\n    for await (const chunk of stream) {\n      this.#addChunk(chunk);\n    }\n    if (stream.controller.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n    return this._addChatCompletion(this.#endRequest());\n  }\n\n  protected async _fromReadableStream(\n    readableStream: ReadableStream,\n    options?: Core.RequestOptions,\n  ): Promise<ChatCompletion> {\n    const signal = options?.signal;\n    if (signal) {\n      if (signal.aborted) this.controller.abort();\n      signal.addEventListener('abort', () => this.controller.abort());\n    }\n    this.#beginRequest();\n    this._connected();\n    const stream = Stream.fromReadableStream<ChatCompletionChunk>(readableStream, this.controller);\n    let chatId;\n    for await (const chunk of stream) {\n      if (chatId && chatId !== chunk.id) {\n        // A new request has been made.\n        this._addChatCompletion(this.#endRequest());\n      }\n\n      this.#addChunk(chunk);\n      chatId = chunk.id;\n    }\n    if (stream.controller.signal?.aborted) {\n      throw new APIUserAbortError();\n    }\n    return this._addChatCompletion(this.#endRequest());\n  }\n\n  #accumulateChatCompletion(chunk: ChatCompletionChunk): ChatCompletionSnapshot {\n    let snapshot = this.#currentChatCompletionSnapshot;\n    const { choices, ...rest } = chunk;\n    if (!snapshot) {\n      snapshot = this.#currentChatCompletionSnapshot = {\n        ...rest,\n        choices: [],\n      };\n    } else {\n      Object.assign(snapshot, rest);\n    }\n\n    for (const { delta, finish_reason, index, logprobs = null, ...other } of chunk.choices) {\n      let choice = snapshot.choices[index];\n      if (!choice) {\n        choice = snapshot.choices[index] = { finish_reason, index, message: {}, logprobs, ...other };\n      }\n\n      if (logprobs) {\n        if (!choice.logprobs) {\n          choice.logprobs = Object.assign({}, logprobs);\n        } else {\n          const { content, ...rest } = logprobs;\n          Object.assign(choice.logprobs, rest);\n          if (content) {\n            choice.logprobs.content ??= [];\n            choice.logprobs.content.push(...content);\n          }\n        }\n      }\n\n      if (finish_reason) choice.finish_reason = finish_reason;\n      Object.assign(choice, other);\n\n      if (!delta) continue; // Shouldn't happen; just in case.\n      const { content, function_call, role, tool_calls, ...rest } = delta;\n      Object.assign(choice.message, rest);\n\n      if (content) choice.message.content = (choice.message.content || '') + content;\n      if (role) choice.message.role = role;\n      if (function_call) {\n        if (!choice.message.function_call) {\n          choice.message.function_call = function_call;\n        } else {\n          if (function_call.name) choice.message.function_call.name = function_call.name;\n          if (function_call.arguments) {\n            choice.message.function_call.arguments ??= '';\n            choice.message.function_call.arguments += function_call.arguments;\n          }\n        }\n      }\n      if (tool_calls) {\n        if (!choice.message.tool_calls) choice.message.tool_calls = [];\n        for (const { index, id, type, function: fn, ...rest } of tool_calls) {\n          const tool_call = (choice.message.tool_calls[index] ??= {});\n          Object.assign(tool_call, rest);\n          if (id) tool_call.id = id;\n          if (type) tool_call.type = type;\n          if (fn) tool_call.function ??= { arguments: '' };\n          if (fn?.name) tool_call.function!.name = fn.name;\n          if (fn?.arguments) tool_call.function!.arguments += fn.arguments;\n        }\n      }\n    }\n    return snapshot;\n  }\n\n  [Symbol.asyncIterator](): AsyncIterator<ChatCompletionChunk> {\n    const pushQueue: ChatCompletionChunk[] = [];\n    const readQueue: {\n      resolve: (chunk: ChatCompletionChunk | undefined) => void;\n      reject: (err: unknown) => void;\n    }[] = [];\n    let done = false;\n\n    this.on('chunk', (chunk) => {\n      const reader = readQueue.shift();\n      if (reader) {\n        reader.resolve(chunk);\n      } else {\n        pushQueue.push(chunk);\n      }\n    });\n\n    this.on('end', () => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.resolve(undefined);\n      }\n      readQueue.length = 0;\n    });\n\n    this.on('abort', (err) => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.reject(err);\n      }\n      readQueue.length = 0;\n    });\n\n    this.on('error', (err) => {\n      done = true;\n      for (const reader of readQueue) {\n        reader.reject(err);\n      }\n      readQueue.length = 0;\n    });\n\n    return {\n      next: async (): Promise<IteratorResult<ChatCompletionChunk>> => {\n        if (!pushQueue.length) {\n          if (done) {\n            return { value: undefined, done: true };\n          }\n          return new Promise<ChatCompletionChunk | undefined>((resolve, reject) =>\n            readQueue.push({ resolve, reject }),\n          ).then((chunk) => (chunk ? { value: chunk, done: false } : { value: undefined, done: true }));\n        }\n        const chunk = pushQueue.shift()!;\n        return { value: chunk, done: false };\n      },\n      return: async () => {\n        this.abort();\n        return { value: undefined, done: true };\n      },\n    };\n  }\n\n  toReadableStream(): ReadableStream {\n    const stream = new Stream(this[Symbol.asyncIterator].bind(this), this.controller);\n    return stream.toReadableStream();\n  }\n}\n\nfunction finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletion {\n  const { id, choices, created, model, system_fingerprint, ...rest } = snapshot;\n  return {\n    ...rest,\n    id,\n    choices: choices.map(\n      ({ message, finish_reason, index, logprobs, ...choiceRest }): ChatCompletion.Choice => {\n        if (!finish_reason) throw new OpenAIError(`missing finish_reason for choice ${index}`);\n        const { content = null, function_call, tool_calls, ...messageRest } = message;\n        const role = message.role as 'assistant'; // this is what we expect; in theory it could be different which would make our types a slight lie but would be fine.\n        if (!role) throw new OpenAIError(`missing role for choice ${index}`);\n        if (function_call) {\n          const { arguments: args, name } = function_call;\n          if (args == null) throw new OpenAIError(`missing function_call.arguments for choice ${index}`);\n          if (!name) throw new OpenAIError(`missing function_call.name for choice ${index}`);\n          return {\n            ...choiceRest,\n            message: { content, function_call: { arguments: args, name }, role },\n            finish_reason,\n            index,\n            logprobs,\n          };\n        }\n        if (tool_calls) {\n          return {\n            ...choiceRest,\n            index,\n            finish_reason,\n            logprobs,\n            message: {\n              ...messageRest,\n              role,\n              content,\n              tool_calls: tool_calls.map((tool_call, i) => {\n                const { function: fn, type, id, ...toolRest } = tool_call;\n                const { arguments: args, name, ...fnRest } = fn || {};\n                if (id == null)\n                  throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].id\\n${str(snapshot)}`);\n                if (type == null)\n                  throw new OpenAIError(`missing choices[${index}].tool_calls[${i}].type\\n${str(snapshot)}`);\n                if (name == null)\n                  throw new OpenAIError(\n                    `missing choices[${index}].tool_calls[${i}].function.name\\n${str(snapshot)}`,\n                  );\n                if (args == null)\n                  throw new OpenAIError(\n                    `missing choices[${index}].tool_calls[${i}].function.arguments\\n${str(snapshot)}`,\n                  );\n\n                return { ...toolRest, id, type, function: { ...fnRest, name, arguments: args } };\n              }),\n            },\n          };\n        }\n        return {\n          ...choiceRest,\n          message: { ...messageRest, content, role },\n          finish_reason,\n          index,\n          logprobs,\n        };\n      },\n    ),\n    created,\n    model,\n    object: 'chat.completion',\n    ...(system_fingerprint ? { system_fingerprint } : {}),\n  };\n}\n\nfunction str(x: unknown) {\n  return JSON.stringify(x);\n}\n\n/**\n * Represents a streamed chunk of a chat completion response returned by model,\n * based on the provided input.\n */\nexport interface ChatCompletionSnapshot {\n  /**\n   * A unique identifier for the chat completion.\n   */\n  id: string;\n\n  /**\n   * A list of chat completion choices. Can be more than one if `n` is greater\n   * than 1.\n   */\n  choices: Array<ChatCompletionSnapshot.Choice>;\n\n  /**\n   * The Unix timestamp (in seconds) of when the chat completion was created.\n   */\n  created: number;\n\n  /**\n   * The model to generate the completion.\n   */\n  model: string;\n\n  // Note we do not include an \"object\" type on the snapshot,\n  // because the object is not a valid \"chat.completion\" until finalized.\n  // object: 'chat.completion';\n\n  /**\n   * This fingerprint represents the backend configuration that the model runs with.\n   *\n   * Can be used in conjunction with the `seed` request parameter to understand when\n   * backend changes have been made that might impact determinism.\n   */\n  system_fingerprint?: string;\n}\n\nexport namespace ChatCompletionSnapshot {\n  export interface Choice {\n    /**\n     * A chat completion delta generated by streamed model responses.\n     */\n    message: Choice.Message;\n\n    /**\n     * The reason the model stopped generating tokens. This will be `stop` if the model\n     * hit a natural stop point or a provided stop sequence, `length` if the maximum\n     * number of tokens specified in the request was reached, `content_filter` if\n     * content was omitted due to a flag from our content filters, or `function_call`\n     * if the model called a function.\n     */\n    finish_reason: ChatCompletion.Choice['finish_reason'] | null;\n\n    /**\n     * Log probability information for the choice.\n     */\n    logprobs: ChatCompletion.Choice.Logprobs | null;\n\n    /**\n     * The index of the choice in the list of choices.\n     */\n    index: number;\n  }\n\n  export namespace Choice {\n    /**\n     * A chat completion delta generated by streamed model responses.\n     */\n    export interface Message {\n      /**\n       * The contents of the chunk message.\n       */\n      content?: string | null;\n\n      /**\n       * The name and arguments of a function that should be called, as generated by the\n       * model.\n       */\n      function_call?: Message.FunctionCall;\n\n      tool_calls?: Array<Message.ToolCall>;\n\n      /**\n       * The role of the author of this message.\n       */\n      role?: 'system' | 'user' | 'assistant' | 'function' | 'tool';\n    }\n\n    export namespace Message {\n      export interface ToolCall {\n        /**\n         * The ID of the tool call.\n         */\n        id?: string;\n\n        function?: ToolCall.Function;\n\n        /**\n         * The type of the tool.\n         */\n        type?: 'function';\n      }\n\n      export namespace ToolCall {\n        export interface Function {\n          /**\n           * The arguments to call the function with, as generated by the model in JSON\n           * format. Note that the model does not always generate valid JSON, and may\n           * hallucinate parameters not defined by your function schema. Validate the\n           * arguments in your code before calling your function.\n           */\n          arguments?: string;\n\n          /**\n           * The name of the function to call.\n           */\n          name?: string;\n        }\n      }\n\n      /**\n       * The name and arguments of a function that should be called, as generated by the\n       * model.\n       */\n      export interface FunctionCall {\n        /**\n         * The arguments to call the function with, as generated by the model in JSON\n         * format. Note that the model does not always generate valid JSON, and may\n         * hallucinate parameters not defined by your function schema. Validate the\n         * arguments in your code before calling your function.\n         */\n        arguments?: string;\n\n        /**\n         * The name of the function to call.\n         */\n        name?: string;\n      }\n    }\n  }\n}\n\n// Path: src/lib/ChatCompletionStreamingRunner.ts\nimport {\n  Completions,\n  type ChatCompletionChunk,\n  type ChatCompletionCreateParamsStreaming,\n} from 'openai/resources/chat/completions';\nimport { RunnerOptions, type AbstractChatCompletionRunnerEvents } from './AbstractChatCompletionRunner';\nimport { type ReadableStream } from 'openai/_shims/index';\nimport { RunnableTools, type BaseFunctionsArgs, type RunnableFunctions } from './RunnableFunction';\nimport { ChatCompletionSnapshot, ChatCompletionStream } from './ChatCompletionStream';\n\nexport interface ChatCompletionStreamEvents extends AbstractChatCompletionRunnerEvents {\n  content: (contentDelta: string, contentSnapshot: string) => void;\n  chunk: (chunk: ChatCompletionChunk, snapshot: ChatCompletionSnapshot) => void;\n}\n\nexport type ChatCompletionStreamingFunctionRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsStreaming,\n  'functions'\n> & {\n  functions: RunnableFunctions<FunctionsArgs>;\n};\n\nexport type ChatCompletionStreamingToolRunnerParams<FunctionsArgs extends BaseFunctionsArgs> = Omit<\n  ChatCompletionCreateParamsStreaming,\n  'tools'\n> & {\n  tools: RunnableTools<FunctionsArgs>;\n};\n\nexport class ChatCompletionStreamingRunner\n  extends ChatCompletionStream\n  implements AsyncIterable<ChatCompletionChunk>\n{\n  static override fromReadableStream(stream: ReadableStream): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    runner._run(() => runner._fromReadableStream(stream));\n    return runner;\n  }\n\n  /** @deprecated - please use `runTools` instead. */\n  static runFunctions<T extends (string | object)[]>(\n    completions: Completions,\n    params: ChatCompletionStreamingFunctionRunnerParams<T>,\n    options?: RunnerOptions,\n  ): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    const opts = {\n      ...options,\n      headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'runFunctions' },\n    };\n    runner._run(() => runner._runFunctions(completions, params, opts));\n    return runner;\n  }\n\n  static runTools<T extends (string | object)[]>(\n    completions: Completions,\n    params: ChatCompletionStreamingToolRunnerParams<T>,\n    options?: RunnerOptions,\n  ): ChatCompletionStreamingRunner {\n    const runner = new ChatCompletionStreamingRunner();\n    const opts = {\n      ...options,\n      headers: { ...options?.headers, 'X-Stainless-Helper-Method': 'runTools' },\n    };\n    runner._run(() => runner._runTools(completions, params, opts));\n    return runner;\n  }\n}\n\n// Path: src/lib/RunnableFunction.ts\nimport { type ChatCompletionRunner } from './ChatCompletionRunner';\nimport { type ChatCompletionStreamingRunner } from './ChatCompletionStreamingRunner';\nimport { JSONSchema } from './jsonschema';\n\ntype PromiseOrValue<T> = T | Promise<T>;\n\nexport type RunnableFunctionWithParse<Args extends object> = {\n  /**\n   * @param args the return value from `parse`.\n   * @param runner the runner evaluating this callback.\n   * @returns a string to send back to OpenAI.\n   */\n  function: (\n    args: Args,\n    runner: ChatCompletionRunner | ChatCompletionStreamingRunner,\n  ) => PromiseOrValue<unknown>;\n  /**\n   * @param input the raw args from the OpenAI function call.\n   * @returns the parsed arguments to pass to `function`\n   */\n  parse: (input: string) => PromiseOrValue<Args>;\n  /**\n   * The parameters the function accepts, describes as a JSON Schema object.\n   */\n  parameters: JSONSchema;\n  /**\n   * A description of what the function does, used by the model to choose when and how to call the function.\n   */\n  description: string;\n  /**\n   * The name of the function to be called. Will default to function.name if omitted.\n   */\n  name?: string | undefined;\n};\n\nexport type RunnableFunctionWithoutParse = {\n  /**\n   * @param args the raw args from the OpenAI function call.\n   * @returns a string to send back to OpenAI\n   */\n  function: (\n    args: string,\n    runner: ChatCompletionRunner | ChatCompletionStreamingRunner,\n  ) => PromiseOrValue<unknown>;\n  /**\n   * The parameters the function accepts, describes as a JSON Schema object.\n   */\n  parameters: JSONSchema;\n  /**\n   * A description of what the function does, used by the model to choose when and how to call the function.\n   */\n  description: string;\n  /**\n   * The name of the function to be called. Will default to function.name if omitted.\n   */\n  name?: string | undefined;\n};\n\nexport type RunnableFunction<Args extends object | string> =\n  Args extends string ? RunnableFunctionWithoutParse\n  : Args extends object ? RunnableFunctionWithParse<Args>\n  : never;\n\nexport type RunnableToolFunction<Args extends object | string> =\n  Args extends string ? RunnableToolFunctionWithoutParse\n  : Args extends object ? RunnableToolFunctionWithParse<Args>\n  : never;\n\nexport type RunnableToolFunctionWithoutParse = {\n  type: 'function';\n  function: RunnableFunctionWithoutParse;\n};\nexport type RunnableToolFunctionWithParse<Args extends object> = {\n  type: 'function';\n  function: RunnableFunctionWithParse<Args>;\n};\n\nexport \nfunction isRunnableFunctionWithParse<Args extends object>(\n  fn: any,\n): fn is RunnableFunctionWithParse<Args> {\n  return typeof (fn as any).parse === 'function';\n}\n\nexport type BaseFunctionsArgs = readonly (object | string)[];\n\nexport type RunnableFunctions<FunctionsArgs extends BaseFunctionsArgs> =\n  [any[]] extends [FunctionsArgs] ? readonly RunnableFunction<any>[]\n  : {\n      [Index in keyof FunctionsArgs]: Index extends number ? RunnableFunction<FunctionsArgs[Index]>\n      : FunctionsArgs[Index];\n    };\n\nexport type RunnableTools<FunctionsArgs extends BaseFunctionsArgs> =\n  [any[]] extends [FunctionsArgs] ? readonly RunnableToolFunction<any>[]\n  : {\n      [Index in keyof FunctionsArgs]: Index extends number ? RunnableToolFunction<FunctionsArgs[Index]>\n      : FunctionsArgs[Index];\n    };\n\n/**\n * This is helper class for passing a `function` and `parse` where the `function`\n * argument type matches the `parse` return type.\n *\n * @deprecated - please use ParsingToolFunction instead.\n */\nexport class ParsingFunction<Args extends object> {\n  function: RunnableFunctionWithParse<Args>['function'];\n  parse: RunnableFunctionWithParse<Args>['parse'];\n  parameters: RunnableFunctionWithParse<Args>['parameters'];\n  description: RunnableFunctionWithParse<Args>['description'];\n  name?: RunnableFunctionWithParse<Args>['name'];\n\n  constructor(input: RunnableFunctionWithParse<Args>) {\n    this.function = input.function;\n    this.parse = input.parse;\n    this.parameters = input.parameters;\n    this.description = input.description;\n    this.name = input.name;\n  }\n}\n\n/**\n * This is helper class for passing a `function` and `parse` where the `function`\n * argument type matches the `parse` return type.\n */\nexport class ParsingToolFunction<Args extends object> {\n  type: 'function';\n  function: RunnableFunctionWithParse<Args>;\n\n  constructor(input: RunnableFunctionWithParse<Args>) {\n    this.type = 'function';\n    this.function = input;\n  }\n}\n\n// Path: src/lib/chatCompletionUtils.ts\nimport {\n  type ChatCompletionAssistantMessageParam,\n  type ChatCompletionFunctionMessageParam,\n  type ChatCompletionMessageParam,\n  type ChatCompletionToolMessageParam,\n} from 'openai/resources';\n\nexport const isAssistantMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionAssistantMessageParam => {\n  return message?.role === 'assistant';\n};\n\nexport const isFunctionMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionFunctionMessageParam => {\n  return message?.role === 'function';\n};\n\nexport const isToolMessage = (\n  message: ChatCompletionMessageParam | null | undefined,\n): message is ChatCompletionToolMessageParam => {\n  return message?.role === 'tool';\n};\n\nexport function isPresent<T>(obj: T | null | undefined): obj is T {\n  return obj != null;\n}\n\n// Path: src/lib/AbstractChatCompletionRunner.ts\n...\n", "needle_token_start": 15278, "needle_token_end": 15325, "code_context_ntokens": 16088, "output": ["```typescript\nexport function isRunnableFunctionWithParse<Args extends object>(\n  fn: any,\n): fn is RunnableFunctionWithParse<Args> {\n  return typeof (fn as any).parse === 'function';\n}\n```"]}
{"repo": "umami-software/umami", "name": "getKeyName", "language": "typescript", "path": "src/lib/data.ts", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a concatenated key name based on a current key and its parent key, primarily used in hierarchical data structures.\n2. **Input**: Takes two strings: the current key and the parent key.\n3. **Output**: Returns a single string that represents the full path of the key, combining the parent key and the current key with a dot if a parent key exists.\n4. **Procedure**: Checks if a parent key is provided. If not, it returns the current key as is. If a parent key exists, it concatenates the parent key with the current key using a dot as a separator and returns the resulting string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/queries/admin/website.ts\n...\n// Path: src/queries/analytics/sessions/getSession.ts\nimport prisma from 'lib/prisma';\n\nexport async function getSession(id: string) {\n  return prisma.client.session.findUnique({\n    where: {\n      id,\n    },\n  });\n}\n\n// Path: src/lib/data.ts\nimport { DATA_TYPE, DATETIME_REGEX } from './constants';\nimport { DynamicDataType } from './types';\n\nexport function flattenJSON(\n  eventData: { [key: string]: any },\n  keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],\n  parentKey = '',\n): { key: string; value: any; dataType: DynamicDataType }[] {\n  return Object.keys(eventData).reduce(\n    (acc, key) => {\n      const value = eventData[key];\n      const type = typeof eventData[key];\n\n      // nested object\n      if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {\n        flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));\n      } else {\n        createKey(getKeyName(key, parentKey), value, acc);\n      }\n\n      return acc;\n    },\n    { keyValues, parentKey },\n  ).keyValues;\n}\n\nexport function isValidDateValue(value: string) {\n  return typeof value === 'string' && DATETIME_REGEX.test(value);\n}\n\nexport function getDataType(value: any): string {\n  let type: string = typeof value;\n\n  if (isValidDateValue(value)) {\n    type = 'date';\n  }\n\n  return type;\n}\n\nexport function getStringValue(value: string, dataType: number) {\n  if (dataType === DATA_TYPE.number) {\n    return parseFloat(value).toFixed(4);\n  }\n\n  if (dataType === DATA_TYPE.date) {\n    return new Date(value).toISOString();\n  }\n\n  return value;\n}\n\nfunction createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {\n  const type = getDataType(value);\n\n  let dataType = null;\n\n  switch (type) {\n    case 'number':\n      dataType = DATA_TYPE.number;\n      break;\n    case 'string':\n      dataType = DATA_TYPE.string;\n      break;\n    case 'boolean':\n      dataType = DATA_TYPE.boolean;\n      value = value ? 'true' : 'false';\n      break;\n    case 'date':\n      dataType = DATA_TYPE.date;\n      break;\n    case 'object':\n      dataType = DATA_TYPE.array;\n      value = JSON.stringify(value);\n      break;\n    default:\n      dataType = DATA_TYPE.string;\n      break;\n  }\n\n  acc.keyValues.push({ key, value, dataType });\n}\n\n\nfunction getKeyName(key: string, parentKey: string) {\n  if (!parentKey) {\n    return key;\n  }\n\n  return `${parentKey}.${key}`;\n}\n\n// Path: src/queries/analytics/sessions/saveSessionData.ts\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveSessionData(data: {\n  websiteId: string;\n  sessionId: string;\n  sessionData: DynamicData;\n}) {\n  const { client, transaction } = prisma;\n  const { websiteId, sessionId, sessionData } = data;\n\n  const jsonKeys = flattenJSON(sessionData);\n\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteId,\n    sessionId,\n    key: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return transaction([\n    client.sessionData.deleteMany({\n      where: {\n        sessionId,\n      },\n    }),\n    client.sessionData.createMany({\n      data: flattenedData as any,\n    }),\n  ]);\n}\n\n// Path: src/queries/analytics/reports/getUTM.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nexport async function getUTM(\n  ...args: [\n    websiteId: string,\n    filters: {\n      startDate: Date;\n      endDate: Date;\n      timezone?: string;\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = prisma;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n      and coalesce(url_query, '') != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and url_query != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nfunction parseParameters(data: any[]) {\n  return data.reduce((obj, { url_query, num }) => {\n    try {\n      const searchParams = new URLSearchParams(url_query);\n\n      for (const [key, value] of searchParams) {\n        if (key.match(/^utm_(\\w+)$/)) {\n          const name = safeDecodeURIComponent(value);\n          if (!obj[key]) {\n            obj[key] = { [name]: Number(num) };\n          } else if (!obj[key][name]) {\n            obj[key][name] = Number(num);\n          } else {\n            obj[key][name] += Number(num);\n          }\n        }\n      }\n    } catch {\n      // Ignore\n    }\n\n    return obj;\n  }, {});\n}\n\n// Path: src/lib/kafka.ts\nimport dateFormat from 'dateformat';\nimport debug from 'debug';\nimport { Kafka, Mechanism, Producer, RecordMetadata, SASLOptions, logLevel } from 'kafkajs';\nimport { KAFKA, KAFKA_PRODUCER } from 'lib/db';\nimport * as tls from 'tls';\n\nconst log = debug('umami:kafka');\n\nlet kafka: Kafka;\nlet producer: Producer;\nconst enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER);\n\nfunction getClient() {\n  const { username, password } = new URL(process.env.KAFKA_URL);\n  const brokers = process.env.KAFKA_BROKER.split(',');\n\n  const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions | Mechanism } =\n    username && password\n      ? {\n          ssl: {\n            checkServerIdentity: () => undefined,\n            ca: [process.env.CA_CERT],\n            key: process.env.CLIENT_KEY,\n            cert: process.env.CLIENT_CERT,\n          },\n          sasl: {\n            mechanism: 'plain',\n            username,\n            password,\n          },\n        }\n      : {};\n\n  const client: Kafka = new Kafka({\n    clientId: 'umami',\n    brokers: brokers,\n    connectionTimeout: 3000,\n    logLevel: logLevel.ERROR,\n    ...ssl,\n  });\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA] = client;\n  }\n\n  log('Kafka initialized');\n\n  return client;\n}\n\nasync function getProducer(): Promise<Producer> {\n  const producer = kafka.producer();\n  await producer.connect();\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA_PRODUCER] = producer;\n  }\n\n  log('Kafka producer initialized');\n\n  return producer;\n}\n\nfunction getDateFormat(date: Date, format?: string): string {\n  return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');\n}\n\nasync function sendMessage(\n  message: { [key: string]: string | number },\n  topic: string,\n): Promise<RecordMetadata[]> {\n  await connect();\n\n  return producer.send({\n    topic,\n    messages: [\n      {\n        value: JSON.stringify(message),\n      },\n    ],\n    acks: -1,\n  });\n}\n\nasync function sendMessages(messages: { [key: string]: string | number }[], topic: string) {\n  await connect();\n\n  await producer.send({\n    topic,\n    messages: messages.map(a => {\n      return { value: JSON.stringify(a) };\n    }),\n    acks: 1,\n  });\n}\n\nasync function connect(): Promise<Kafka> {\n  if (!kafka) {\n    kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient());\n\n    if (kafka) {\n      producer = global[KAFKA_PRODUCER] || (await getProducer());\n    }\n  }\n\n  return kafka;\n}\n\nexport default {\n  enabled,\n  client: kafka,\n  producer,\n  log,\n  connect,\n  getDateFormat,\n  sendMessage,\n  sendMessages,\n};\n\n// Path: src/queries/analytics/eventData/saveEventData.ts\nimport { Prisma } from '@prisma/client';\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveEventData(args: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  eventId: string;\n  eventData: DynamicData;\n}): Promise<Prisma.BatchPayload> {\n  const { websiteId, eventId, eventData } = data;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  // id, websiteEventId, eventStringValue\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteEventId: eventId,\n    websiteId,\n    eventKey: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return prisma.client.eventData.createMany({\n    data: flattenedData,\n  });\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;\n\n  const { getDateFormat, sendMessages } = kafka;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  const messages = jsonKeys.map(a => ({\n    website_id: websiteId,\n    session_id: sessionId,\n    event_id: eventId,\n    url_path: urlPath,\n    event_name: eventName,\n    event_key: a.key,\n    string_value: getStringValue(a.value, a.dataType),\n    number_value: a.dataType === DATA_TYPE.number ? a.value : null,\n    date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null,\n    data_type: a.dataType,\n    created_at: createdAt,\n  }));\n\n  await sendMessages(messages, 'event_data');\n\n  return data;\n}\n\n// Path: src/queries/analytics/events/saveEvent.ts\nimport { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE, PAGE_TITLE_LENGTH } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { uuid } from 'lib/crypto';\nimport { saveEventData } from 'queries/analytics/eventData/saveEventData';\n\nexport async function saveEvent(args: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    eventName,\n    eventData,\n    pageTitle,\n  } = data;\n  const websiteEventId = uuid();\n\n  const websiteEvent = prisma.client.websiteEvent.create({\n    data: {\n      id: websiteEventId,\n      websiteId,\n      sessionId,\n      visitId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      urlQuery: urlQuery?.substring(0, URL_LENGTH),\n      referrerPath: referrerPath?.substring(0, URL_LENGTH),\n      referrerQuery: referrerQuery?.substring(0, URL_LENGTH),\n      referrerDomain: referrerDomain?.substring(0, URL_LENGTH),\n      pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n      eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n      eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    },\n  });\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId: websiteEventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n    });\n  }\n\n  return websiteEvent;\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    pageTitle,\n    eventName,\n    eventData,\n    country,\n    subdivision1,\n    subdivision2,\n    city,\n    ...args\n  } = data;\n  const { getDateFormat, sendMessage } = kafka;\n  const eventId = uuid();\n  const createdAt = getDateFormat(new Date());\n\n  const message = {\n    ...args,\n    website_id: websiteId,\n    session_id: sessionId,\n    visit_id: visitId,\n    event_id: uuid(),\n    country: country,\n    subdivision1:\n      country && subdivision1\n        ? subdivision1.includes('-')\n          ? subdivision1\n          : `${country}-${subdivision1}`\n        : null,\n    subdivision2: subdivision2,\n    city: city,\n    url_path: urlPath?.substring(0, URL_LENGTH),\n    url_query: urlQuery?.substring(0, URL_LENGTH),\n    referrer_path: referrerPath?.substring(0, URL_LENGTH),\n    referrer_query: referrerQuery?.substring(0, URL_LENGTH),\n    referrer_domain: referrerDomain?.substring(0, URL_LENGTH),\n    page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n    event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n    event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    created_at: createdAt,\n  };\n\n  await sendMessage(message, 'event');\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n      createdAt,\n    });\n  }\n\n  return data;\n}\n\n// Path: src/queries/analytics/sessions/getSessions.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';\n\nexport async function getSessions(...args: [websiteId: string, startAt: Date]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, startDate: Date) {\n  return prisma.client.session.findMany({\n    where: {\n      websiteId,\n      createdAt: {\n        gte: startDate,\n      },\n    },\n    orderBy: {\n      createdAt: 'asc',\n    },\n  });\n}\n\nasync function clickhouseQuery(websiteId: string, startDate: Date) {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select\n      session_id as id,\n      website_id as websiteId,\n      created_at as createdAt,\n      toUnixTimestamp(created_at) as timestamp,\n      hostname,\n      browser,\n      os,\n      device,\n      screen,\n      language,\n      country,\n      subdivision1,\n      subdivision2,\n      city\n    from website_event\n    where website_id = {websiteId:UUID}\n    and created_at >= {startDate:DateTime64}\n    order by created_at asc\n    `,\n    {\n      websiteId,\n      startDate,\n    },\n  );\n}\n\n// Path: src/queries/analytics/reports/getFunnel.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nconst formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {\n  return steps.map((step: { type: string; value: string }, i: number) => {\n    const visitors = Number(results[i]?.count) || 0;\n    const previous = Number(results[i - 1]?.count) || 0;\n    const dropped = previous > 0 ? previous - visitors : 0;\n    const dropoff = 1 - visitors / previous;\n    const remaining = visitors / Number(results[0].count);\n\n    return {\n      ...step,\n      visitors,\n      previous,\n      dropped,\n      dropoff,\n      remaining,\n    };\n  });\n};\n\nexport async function getFunnel(\n  ...args: [\n    websiteId: string,\n    criteria: {\n      windowMinutes: number;\n      startDate: Date;\n      endDate: Date;\n      steps: { type: string; value: string }[];\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery, getAddIntervalQuery } = prisma;\n  const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, windowMinutes);\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    params: string[];\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union ' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\n          WITH level1 AS (\n            select distinct session_id, created_at\n            from website_event\n            where website_id = {{websiteId::uuid}}\n              and created_at between {{startDate}} and {{endDate}}\n              and ${column} ${operator} {{${i}}}\n          )`;\n        } else {\n          pv.levelQuery += `\n          , level${levelNumber} AS (\n            select distinct we.session_id, we.created_at\n            from level${i} l\n            join website_event we\n                on l.session_id = we.session_id\n            where we.website_id = {{websiteId::uuid}}\n                and we.created_at between l.created_at and ${getAddIntervalQuery(\n                  `l.created_at `,\n                  `${windowMinutes} minute`,\n                )}\n                and we.${column} ${operator} {{${i}}}\n                and we.created_at <= {{endDate}}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.params.push(paramValue);\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        params: [],\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    ${levelOneQuery}\n    ${levelQuery}\n    ${sumQuery}\n    ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery } = clickhouse;\n  const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(\n    steps,\n    windowMinutes,\n  );\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    stepFilterQuery: string;\n    params: { [key: string]: string };\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union all ' : '';\n        const startFilter = i > 0 ? 'or' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\\n\n          level1 AS (\n            select *\n            from level0\n            where ${column} ${operator} {param${i}:String}\n          )`;\n        } else {\n          pv.levelQuery += `\\n\n          , level${levelNumber} AS (\n            select distinct y.session_id as session_id,\n                y.url_path as url_path,\n                y.referrer_path as referrer_path,\n                y.event_name,\n                y.created_at as created_at\n            from level${i} x\n            join level0 y\n            on x.session_id = y.session_id\n            where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute\n                and y.${column} ${operator} {param${i}:String}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.stepFilterQuery += `${startFilter} ${column} ${operator} {param${i}:String} `;\n        pv.params[`param${i}`] = paramValue;\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        stepFilterQuery: '',\n        params: {},\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    WITH level0 AS (\n      select distinct session_id, url_path, referrer_path, event_name, created_at\n      from umami.website_event\n      where (${stepFilterQuery})\n        and website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ),\n    ${levelOneQuery}\n    ${levelQuery}\n    select *\n    from (\n      ${sumQuery} \n    ) ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\n// Path: src/queries/analytics/getWebsiteStats.ts\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getWebsiteStats(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(case when t.c = 1 then 1 else 0 end) as \"bounces\",\n      sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as \"totaltime\"\n    from (\n      select\n        website_event.session_id,\n        website_event.visit_id,\n        count(*) as \"c\",\n        min(website_event.created_at) as \"min_time\",\n        max(website_event.created_at) as \"max_time\"\n      from website_event\n      join website \n        on website_event.website_id = website.website_id\n        ${joinSession}\n      where website.website_id = {{websiteId::uuid}}\n        and website_event.created_at between {{startDate}} and {{endDate}}\n        and event_type = {{eventType}}\n        ${filterQuery}\n      group by 1, 2\n    ) as t\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(if(t.c = 1, 1, 0)) as \"bounces\",\n      sum(max_time-min_time) as \"totaltime\"\n    from (\n      select\n        session_id,\n        visit_id,\n        count(*) c,\n        min(created_at) min_time,\n        max(created_at) max_time\n      from website_event\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n        and event_type = {eventType:UInt32}\n        ${filterQuery}\n      group by session_id, visit_id\n    ) as t;\n    `,\n    params,\n  ).then(result => {\n    return Object.values(result).map(n => {\n      return {\n        pageviews: Number(n.pageviews),\n        visitors: Number(n.visitors),\n        visits: Number(n.visits),\n        bounces: Number(n.bounces),\n        totaltime: Number(n.totaltime),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getActiveVisitors.ts\nimport { subMinutes } from 'date-fns';\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\n\nexport async function getActiveVisitors(...args: [websiteId: string]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string) {\n  const { rawQuery } = prisma;\n\n  const result = await rawQuery(\n    `\n    select count(distinct session_id) x\n    from website_event\n    where website_id = {{websiteId::uuid}}\n    and created_at >= {{startAt}}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  );\n\n  return result[0] ?? null;\n}\n\nasync function clickhouseQuery(websiteId: string): Promise<{ x: number }> {\n  const { rawQuery } = clickhouse;\n\n  const result = await rawQuery(\n    `\n    select\n      count(distinct session_id) x\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startAt:DateTime64}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: Number(a.x) };\n    });\n  });\n\n  return result[0] ?? null;\n}\n\n// Path: src/queries/admin/team.ts\nimport { Prisma, Team } from '@prisma/client';\nimport { ROLES } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport prisma from 'lib/prisma';\nimport { FilterResult, TeamSearchFilter } from 'lib/types';\nimport TeamFindManyArgs = Prisma.TeamFindManyArgs;\n\nexport async function findTeam(criteria: Prisma.TeamFindUniqueArgs): Promise<Team> {\n  return prisma.client.team.findUnique(criteria);\n}\n\nexport async function getTeam(teamId: string, options: { includeMembers?: boolean } = {}) {\n  const { includeMembers } = options;\n\n  return findTeam({\n    where: {\n      id: teamId,\n    },\n    ...(includeMembers && { include: { teamUser: true } }),\n  });\n}\n\nexport async function getTeams(\n  criteria: TeamFindManyArgs,\n  filters: TeamSearchFilter = {},\n): Promise<FilterResult<Team[]>> {\n  const { getSearchParameters } = prisma;\n  const { query } = filters;\n\n  const where: Prisma.TeamWhereInput = {\n    ...criteria.where,\n    ...getSearchParameters(query, [{ name: 'contains' }]),\n  };\n\n  return prisma.pagedQuery<TeamFindManyArgs>(\n    'team',\n    {\n      ...criteria,\n      where,\n    },\n    filters,\n  );\n}\n\nexport async function getUserTeams(userId: string, filters: TeamSearchFilter = {}) {\n  return getTeams(\n    {\n      where: {\n        deletedAt: null,\n        teamUser: {\n          some: { userId },\n        },\n      },\n      include: {\n        teamUser: {\n          include: {\n            user: {\n              select: {\n                id: true,\n                username: true,\n              },\n            },\n          },\n        },\n        _count: {\n          select: {\n            website: {\n              where: { deletedAt: null },\n            },\n            teamUser: {\n              where: {\n                user: { deletedAt: null },\n              },\n            },\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createTeam(data: Prisma.TeamCreateInput, userId: string): Promise<any> {\n  const { id } = data;\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.team.create({\n      data,\n    }),\n    client.teamUser.create({\n      data: {\n        id: uuid(),\n        teamId: id,\n        userId,\n        role: ROLES.teamOwner,\n      },\n    }),\n  ]);\n}\n\nexport async function updateTeam(teamId: string, data: Prisma.TeamUpdateInput): Promise<Team> {\n  const { client } = prisma;\n\n  return client.team.update({\n    where: {\n      id: teamId,\n    },\n    data: {\n      ...data,\n      updatedAt: new Date(),\n    },\n  });\n}\n\nexport async function deleteTeam(\n  teamId: string,\n): Promise<Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Team]>> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  if (cloudMode) {\n    return transaction([\n      client.team.update({\n        data: {\n          deletedAt: new Date(),\n        },\n        where: {\n          id: teamId,\n        },\n      }),\n    ]);\n  }\n\n  return transaction([\n    client.teamUser.deleteMany({\n      where: {\n        teamId,\n      },\n    }),\n    client.team.delete({\n      where: {\n        id: teamId,\n      },\n    }),\n  ]);\n}\n\n// Path: src/queries/index.ts\nexport * from './admin/report';\nexport * from './admin/team';\nexport * from './admin/teamUser';\nexport * from './admin/user';\nexport * from './admin/website';\nexport * from './analytics/events/getEventMetrics';\nexport * from './analytics/events/getEventUsage';\nexport * from './analytics/events/getEvents';\nexport * from './analytics/eventData/getEventDataEvents';\nexport * from './analytics/eventData/getEventDataFields';\nexport * from './analytics/eventData/getEventDataStats';\nexport * from './analytics/eventData/getEventDataUsage';\nexport * from './analytics/events/saveEvent';\nexport * from './analytics/reports/getFunnel';\nexport * from './analytics/reports/getRetention';\nexport * from './analytics/reports/getInsights';\nexport * from './analytics/reports/getUTM';\nexport * from './analytics/pageviews/getPageviewMetrics';\nexport * from './analytics/pageviews/getPageviewStats';\nexport * from './analytics/sessions/createSession';\nexport * from './analytics/sessions/getSession';\nexport * from './analytics/sessions/getSessionMetrics';\nexport * from './analytics/sessions/getSessions';\nexport * from './analytics/sessions/getSessionStats';\nexport * from './analytics/sessions/saveSessionData';\nexport * from './analytics/getActiveVisitors';\nexport * from './analytics/getRealtimeData';\nexport * from './analytics/getValues';\nexport * from './analytics/getWebsiteDateRange';\nexport * from './analytics/getWebsiteStats';\n\n// Path: src/lib/cache.ts\nimport { User, Website } from '@prisma/client';\nimport redis from '@umami/redis-client';\nimport { getSession, getUser, getWebsite } from '../queries';\n\nasync function fetchWebsite(websiteId: string): Promise<Website> {\n  return redis.client.getCache(`website:${websiteId}`, () => getWebsite(websiteId), 86400);\n}\n\nasync function storeWebsite(data: { id: any }) {\n  const { id } = data;\n  const key = `website:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteWebsite(id) {\n  return redis.client.deleteCache(`website:${id}`);\n}\n\nasync function fetchUser(id): Promise<User> {\n  return redis.client.getCache(`user:${id}`, () => getUser(id, { includePassword: true }), 86400);\n}\n\nasync function storeUser(data) {\n  const { id } = data;\n  const key = `user:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteUser(id) {\n  return redis.client.deleteCache(`user:${id}`);\n}\n\nasync function fetchSession(id) {\n  return redis.client.getCache(`session:${id}`, () => getSession(id), 86400);\n}\n\nasync function storeSession(data) {\n  const { id } = data;\n  const key = `session:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteSession(id) {\n  return redis.client.deleteCache(`session:${id}`);\n}\n\nasync function fetchUserBlock(userId: string) {\n  const key = `user:block:${userId}`;\n  return redis.client.get(key);\n}\n\nasync function incrementUserBlock(userId: string) {\n  const key = `user:block:${userId}`;\n  return redis.client.incr(key);\n}\n\nexport default {\n  fetchWebsite,\n  storeWebsite,\n  deleteWebsite,\n  fetchUser,\n  storeUser,\n  deleteUser,\n  fetchSession,\n  storeSession,\n  deleteSession,\n  fetchUserBlock,\n  incrementUserBlock,\n  enabled: !!redis.enabled,\n};\n\n// Path: src/lib/detect.ts\nimport path from 'path';\nimport { getClientIp } from 'request-ip';\nimport { browserName, detectOS } from 'detect-browser';\nimport isLocalhost from 'is-localhost-ip';\nimport maxmind from 'maxmind';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nimport {\n  DESKTOP_OS,\n  MOBILE_OS,\n  DESKTOP_SCREEN_WIDTH,\n  LAPTOP_SCREEN_WIDTH,\n  MOBILE_SCREEN_WIDTH,\n} from './constants';\nimport { NextApiRequestCollect } from 'pages/api/send';\n\nlet lookup;\n\nexport function getIpAddress(req: NextApiRequestCollect) {\n  const customHeader = String(process.env.CLIENT_IP_HEADER).toLowerCase();\n\n  // Custom header\n  if (customHeader !== 'undefined' && req.headers[customHeader]) {\n    return req.headers[customHeader];\n  }\n  // Cloudflare\n  else if (req.headers['cf-connecting-ip']) {\n    return req.headers['cf-connecting-ip'];\n  }\n\n  return getClientIp(req);\n}\n\nexport function getDevice(screen: string, os: string) {\n  if (!screen) return;\n\n  const [width] = screen.split('x');\n\n  if (DESKTOP_OS.includes(os)) {\n    if (os === 'Chrome OS' || +width < DESKTOP_SCREEN_WIDTH) {\n      return 'laptop';\n    }\n    return 'desktop';\n  } else if (MOBILE_OS.includes(os)) {\n    if (os === 'Amazon OS' || +width > MOBILE_SCREEN_WIDTH) {\n      return 'tablet';\n    }\n    return 'mobile';\n  }\n\n  if (+width >= DESKTOP_SCREEN_WIDTH) {\n    return 'desktop';\n  } else if (+width >= LAPTOP_SCREEN_WIDTH) {\n    return 'laptop';\n  } else if (+width >= MOBILE_SCREEN_WIDTH) {\n    return 'tablet';\n  } else {\n    return 'mobile';\n  }\n}\n\nfunction getRegionCode(country: string, region: string) {\n  if (!country || !region) {\n    return undefined;\n  }\n\n  return region.includes('-') ? region : `${country}-${region}`;\n}\n\nexport async function getLocation(ip: string, req: NextApiRequestCollect) {\n  // Ignore local ips\n  if (await isLocalhost(ip)) {\n    return;\n  }\n\n  // Cloudflare headers\n  if (req.headers['cf-ipcountry']) {\n    const country = safeDecodeURIComponent(req.headers['cf-ipcountry']);\n    const subdivision1 = safeDecodeURIComponent(req.headers['cf-region-code']);\n    const city = safeDecodeURIComponent(req.headers['cf-ipcity']);\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      city,\n    };\n  }\n\n  // Vercel headers\n  if (req.headers['x-vercel-ip-country']) {\n    const country = safeDecodeURIComponent(req.headers['x-vercel-ip-country']);\n    const subdivision1 = safeDecodeURIComponent(req.headers['x-vercel-ip-country-region']);\n    const city = safeDecodeURIComponent(req.headers['x-vercel-ip-city']);\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      city,\n    };\n  }\n\n  // Database lookup\n  if (!lookup) {\n    const dir = path.join(process.cwd(), 'geo');\n\n    lookup = await maxmind.open(path.resolve(dir, 'GeoLite2-City.mmdb'));\n  }\n\n  const result = lookup.get(ip);\n\n  if (result) {\n    const country = result.country?.iso_code ?? result?.registered_country?.iso_code;\n    const subdivision1 = result.subdivisions?.[0]?.iso_code;\n    const subdivision2 = result.subdivisions?.[1]?.names?.en;\n    const city = result.city?.names?.en;\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      subdivision2,\n      city,\n    };\n  }\n}\n\nexport async function getClientInfo(req: NextApiRequestCollect, { screen }) {\n  const userAgent = req.headers['user-agent'];\n  const ip = req.body.payload.ip || getIpAddress(req);\n  const location = await getLocation(ip, req);\n  const country = location?.country;\n  const subdivision1 = location?.subdivision1;\n  const subdivision2 = location?.subdivision2;\n  const city = location?.city;\n  const browser = browserName(userAgent);\n  const os = detectOS(userAgent);\n  const device = getDevice(screen, os);\n\n  return { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device };\n}\n\n// Path: src/lib/auth.ts\nimport { Report } from '@prisma/client';\nimport redis from '@umami/redis-client';\nimport debug from 'debug';\nimport { PERMISSIONS, ROLE_PERMISSIONS, SHARE_TOKEN_HEADER, ROLES } from 'lib/constants';\nimport { secret } from 'lib/crypto';\nimport { NextApiRequest } from 'next';\nimport { createSecureToken, ensureArray, getRandomChars, parseToken } from 'next-basics';\nimport { getTeamUser, getWebsite } from 'queries';\nimport { Auth } from './types';\n\nconst log = debug('umami:auth');\nconst cloudMode = process.env.CLOUD_MODE;\n\nexport async function saveAuth(data: any, expire = 0) {\n  const authKey = `auth:${getRandomChars(32)}`;\n\n  await redis.client.set(authKey, data);\n\n  if (expire) {\n    await redis.client.expire(authKey, expire);\n  }\n\n  return createSecureToken({ authKey }, secret());\n}\n\nexport function getAuthToken(req: NextApiRequest) {\n  try {\n    return req.headers.authorization.split(' ')[1];\n  } catch {\n    return null;\n  }\n}\n\nexport function parseShareToken(req: Request) {\n  try {\n    return parseToken(req.headers[SHARE_TOKEN_HEADER], secret());\n  } catch (e) {\n    log(e);\n    return null;\n  }\n}\n\nexport async function canViewWebsite({ user, shareToken }: Auth, websiteId: string) {\n  if (user?.isAdmin) {\n    return true;\n  }\n\n  if (shareToken?.websiteId === websiteId) {\n    return true;\n  }\n\n  const website = await getWebsite(websiteId);\n\n  if (website.userId) {\n    return user.id === website.userId;\n  }\n\n  if (website.teamId) {\n    const teamUser = await getTeamUser(website.teamId, user.id);\n\n    return !!teamUser;\n  }\n\n  return false;\n}\n\nexport async function canViewAllWebsites({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function canCreateWebsite({ user, grant }: Auth) {\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.websiteCreate);\n  }\n\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return hasPermission(user.role, PERMISSIONS.websiteCreate);\n}\n\nexport async function canUpdateWebsite({ user }: Auth, websiteId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const website = await getWebsite(websiteId);\n\n  if (website.userId) {\n    return user.id === website.userId;\n  }\n\n  if (website.teamId) {\n    const teamUser = await getTeamUser(website.teamId, user.id);\n\n    return teamUser && hasPermission(teamUser.role, PERMISSIONS.websiteUpdate);\n  }\n\n  return false;\n}\n\nexport async function canTransferWebsiteToUser({ user }: Auth, websiteId: string, userId: string) {\n  const website = await getWebsite(websiteId);\n\n  if (website.teamId && user.id === userId) {\n    const teamUser = await getTeamUser(website.teamId, userId);\n\n    return teamUser?.role === ROLES.teamOwner;\n  }\n\n  return false;\n}\n\nexport async function canTransferWebsiteToTeam({ user }: Auth, websiteId: string, teamId: string) {\n  const website = await getWebsite(websiteId);\n\n  if (website.userId && website.userId === user.id) {\n    const teamUser = await getTeamUser(teamId, user.id);\n\n    return teamUser?.role === ROLES.teamOwner;\n  }\n\n  return false;\n}\n\nexport async function canDeleteWebsite({ user }: Auth, websiteId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const website = await getWebsite(websiteId);\n\n  if (website.userId) {\n    return user.id === website.userId;\n  }\n\n  if (website.teamId) {\n    const teamUser = await getTeamUser(website.teamId, user.id);\n\n    return teamUser && hasPermission(teamUser.role, PERMISSIONS.websiteDelete);\n  }\n\n  return false;\n}\n\nexport async function canViewReport(auth: Auth, report: Report) {\n  if (auth.user.isAdmin) {\n    return true;\n  }\n\n  if (auth.user.id == report.userId) {\n    return true;\n  }\n\n  return !!(await canViewWebsite(auth, report.websiteId));\n}\n\nexport async function canUpdateReport({ user }: Auth, report: Report) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return user.id == report.userId;\n}\n\nexport async function canDeleteReport(auth: Auth, report: Report) {\n  return canUpdateReport(auth, report);\n}\n\nexport async function canCreateTeam({ user, grant }: Auth) {\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.teamCreate);\n  }\n\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return !!user;\n}\n\nexport async function canViewTeam({ user }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return getTeamUser(teamId, user.id);\n}\n\nexport async function canUpdateTeam({ user, grant }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.teamUpdate);\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamUpdate);\n}\n\nexport async function canAddUserToTeam({ user, grant }: Auth) {\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.teamUpdate);\n  }\n\n  return user.isAdmin;\n}\n\nexport async function canDeleteTeam({ user }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamDelete);\n}\n\nexport async function canDeleteTeamUser({ user }: Auth, teamId: string, removeUserId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  if (removeUserId === user.id) {\n    return true;\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamUpdate);\n}\n\nexport async function canCreateTeamWebsite({ user }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.websiteCreate);\n}\n\nexport async function canCreateUser({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function canViewUser({ user }: Auth, viewedUserId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return user.id === viewedUserId;\n}\n\nexport async function canViewUsers({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function canUpdateUser({ user }: Auth, viewedUserId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return user.id === viewedUserId;\n}\n\nexport async function canDeleteUser({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function hasPermission(role: string, permission: string | string[]) {\n  return ensureArray(permission).some(e => ROLE_PERMISSIONS[role]?.includes(e));\n}\n\n// Path: src/lib/middleware.ts\nimport cors from 'cors';\nimport debug from 'debug';\nimport redis from '@umami/redis-client';\nimport { getAuthToken, parseShareToken } from 'lib/auth';\nimport { ROLES } from 'lib/constants';\nimport { secret } from 'lib/crypto';\nimport { findSession } from 'lib/session';\nimport {\n  badRequest,\n  createMiddleware,\n  forbidden,\n  parseSecureToken,\n  tooManyRequest,\n  unauthorized,\n} from 'next-basics';\nimport { NextApiRequestCollect } from 'pages/api/send';\nimport { getUser } from '../queries';\n\nconst log = debug('umami:middleware');\n\nexport const useCors = createMiddleware(\n  cors({\n    // Cache CORS preflight request 24 hours by default\n    maxAge: Number(process.env.CORS_MAX_AGE) || 86400,\n  }),\n);\n\nexport const useSession = createMiddleware(async (req, res, next) => {\n  try {\n    const session = await findSession(req as NextApiRequestCollect);\n\n    if (!session) {\n      log('useSession: Session not found');\n      return badRequest(res, 'Session not found.');\n    }\n\n    (req as any).session = session;\n  } catch (e: any) {\n    if (e.message === 'Usage Limit.') {\n      return tooManyRequest(res, e.message);\n    }\n    if (e.message.startsWith('Website not found:')) {\n      return forbidden(res, e.message);\n    }\n    return badRequest(res, e.message);\n  }\n\n  next();\n});\n\nexport const useAuth = createMiddleware(async (req, res, next) => {\n  const token = getAuthToken(req);\n  const payload = parseSecureToken(token, secret());\n  const shareToken = await parseShareToken(req as any);\n\n  let user = null;\n  const { userId, authKey, grant } = payload || {};\n\n  if (userId) {\n    user = await getUser(userId);\n  } else if (redis.enabled && authKey) {\n    const key = await redis.client.get(authKey);\n\n    if (key?.userId) {\n      user = await getUser(key.userId);\n    }\n  }\n\n  if (process.env.NODE_ENV === 'development') {\n    log('useAuth:', { token, shareToken, payload, user, grant });\n  }\n\n  if (!user?.id && !shareToken) {\n    log('useAuth: User not authorized');\n    return unauthorized(res);\n  }\n\n  if (user) {\n    user.isAdmin = user.role === ROLES.admin;\n  }\n\n  (req as any).auth = {\n    user,\n    grant,\n    token,\n    shareToken,\n    authKey,\n  };\n\n  next();\n});\n\nexport const useValidate = async (schema, req, res) => {\n  return createMiddleware(async (req: any, res, next) => {\n    try {\n      const rules = schema[req.method];\n\n      if (rules) {\n        rules.validateSync({ ...req.query, ...req.body });\n      }\n    } catch (e: any) {\n      return badRequest(res, e.message);\n    }\n\n    next();\n  })(req, res);\n};\n\n// Path: src/pages/api/send.ts\nimport ipaddr from 'ipaddr.js';\nimport { isbot } from 'isbot';\nimport { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants';\nimport { secret, visitSalt, uuid } from 'lib/crypto';\nimport { getIpAddress } from 'lib/detect';\nimport { useCors, useSession, useValidate } from 'lib/middleware';\nimport { CollectionType, YupRequest } from 'lib/types';\nimport { NextApiRequest, NextApiResponse } from 'next';\nimport {\n  badRequest,\n  createToken,\n  forbidden,\n  methodNotAllowed,\n  ok,\n  safeDecodeURI,\n  send,\n} from 'next-basics';\nimport { saveEvent, saveSessionData } from 'queries';\nimport * as yup from 'yup';\n\nexport interface CollectRequestBody {\n  payload: {\n    website: string;\n    data?: { [key: string]: any };\n    hostname?: string;\n    ip?: string;\n    language?: string;\n    name?: string;\n    referrer?: string;\n    screen?: string;\n    tag?: string;\n    title?: string;\n    url: string;\n  };\n  type: CollectionType;\n}\n\nexport interface NextApiRequestCollect extends NextApiRequest {\n  body: CollectRequestBody;\n  session: {\n    id: string;\n    websiteId: string;\n    visitId: string;\n    ownerId: string;\n    hostname: string;\n    browser: string;\n    os: string;\n    device: string;\n    screen: string;\n    language: string;\n    country: string;\n    subdivision1: string;\n    subdivision2: string;\n    city: string;\n    iat: number;\n  };\n  headers: { [key: string]: any };\n  yup: YupRequest;\n}\n\nconst schema = {\n...\n", "needle_token_start": 773, "needle_token_end": 819, "code_context_ntokens": 16105, "output": ["```typescript\nfunction getKeyName(key: string, parentKey: string) {\n  if (!parentKey) {\n    return key;\n  }\n\n  return `${parentKey}.${key}`;\n}\n```"]}
{"repo": "umami-software/umami", "name": "convertFilters", "language": "typescript", "path": "src/pages/api/reports/insights.ts", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform an array of filter objects into a single object where each key corresponds to the name of a filter, facilitating easier access and manipulation of filter data.\n2. **Input**: The function accepts an array of objects, where each object contains properties such as `name`, `type`, `operator`, and `value`.\n3. **Output**: It returns an object where each key is the `name` from the filter objects in the input array, and the value is the corresponding filter object.\n4. **Procedure**: The function iterates over the input array, using the `reduce` method to accumulate the results into a single object. For each filter in the array, it adds a new property to the accumulator object using the filter's `name` as the key and the filter object itself as the value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/pages/api/reports/funnel.ts\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getFunnel } from 'queries';\nimport * as yup from 'yup';\n\nexport interface FunnelRequestBody {\n  websiteId: string;\n  steps: { type: string; value: string }[];\n  window: number;\n  dateRange: {\n    startDate: string;\n    endDate: string;\n  };\n}\n\nexport interface FunnelResponse {\n  steps: { type: string; value: string }[];\n  window: number;\n  startAt: number;\n  endAt: number;\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    steps: yup\n      .array()\n      .of(\n        yup.object().shape({\n          type: yup.string().required(),\n          value: yup.string().required(),\n        }),\n      )\n      .min(2)\n      .required(),\n    window: yup.number().positive().required(),\n    dateRange: yup\n      .object()\n      .shape({\n        startDate: yup.date().required(),\n        endDate: yup.date().required(),\n      })\n      .required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<any, FunnelRequestBody>,\n  res: NextApiResponse<FunnelResponse>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  if (req.method === 'POST') {\n    const {\n      websiteId,\n      steps,\n      window,\n      dateRange: { startDate, endDate },\n    } = req.body;\n\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n...\n// Path: src/pages/api/reports/utm.ts\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { TimezoneTest } from 'lib/yup';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getUTM } from 'queries';\nimport * as yup from 'yup';\n\nexport interface UTMRequestBody {\n  websiteId: string;\n  dateRange: { startDate: string; endDate: string; timezone: string };\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    dateRange: yup\n      .object()\n      .shape({\n        startDate: yup.date().required(),\n        endDate: yup.date().required(),\n        timezone: TimezoneTest,\n      })\n      .required(),\n  }),\n};\n\nexport default async (req: NextApiRequestQueryBody<any, UTMRequestBody>, res: NextApiResponse) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  if (req.method === 'POST') {\n    const {\n      websiteId,\n      dateRange: { startDate, endDate, timezone },\n    } = req.body;\n\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const data = await getUTM(websiteId, {\n      startDate: new Date(startDate),\n      endDate: new Date(endDate),\n      timezone,\n    });\n\n    return ok(res, data);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/reports/index.ts\nimport { uuid } from 'lib/crypto';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { pageInfo } from 'lib/schema';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { createReport, getReports } from 'queries';\nimport * as yup from 'yup';\nimport { canUpdateWebsite, canViewTeam, canViewWebsite } from 'lib/auth';\n\nexport interface ReportRequestBody {\n  websiteId: string;\n  name: string;\n  type: string;\n  description: string;\n  parameters: {\n    [key: string]: any;\n  };\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    ...pageInfo,\n  }),\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    name: yup.string().max(200).required(),\n    type: yup\n      .string()\n      .matches(/funnel|insights|retention|utm/i)\n      .required(),\n    description: yup.string().max(500),\n    parameters: yup\n      .object()\n      .test('len', 'Must not exceed 6000 characters.', val => JSON.stringify(val).length < 6000),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<any, ReportRequestBody>,\n  res: NextApiResponse,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const {\n    user: { id: userId },\n  } = req.auth;\n\n  if (req.method === 'GET') {\n    const { page, query, pageSize, websiteId, teamId } = req.query;\n    const filters = {\n      page,\n      pageSize,\n      query,\n    };\n\n    if (\n      (websiteId && !(await canViewWebsite(req.auth, websiteId))) ||\n      (teamId && !(await canViewTeam(req.auth, teamId)))\n    ) {\n      return unauthorized(res);\n    }\n\n    const data = await getReports(\n      {\n        where: {\n          userId: !teamId && !websiteId ? userId : undefined,\n          websiteId,\n          website: {\n            teamId,\n          },\n        },\n        include: {\n          website: {\n            select: {\n              domain: true,\n            },\n          },\n        },\n      },\n      filters,\n    );\n\n    return ok(res, data);\n  }\n\n  if (req.method === 'POST') {\n    const { websiteId, type, name, description, parameters } = req.body;\n\n    if (!(await canUpdateWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const result = await createReport({\n      id: uuid(),\n      userId,\n      websiteId,\n      type,\n      name,\n      description,\n      parameters: JSON.stringify(parameters),\n    } as any);\n\n    return ok(res, result);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/reports/insights.ts\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getInsights } from 'queries';\nimport * as yup from 'yup';\n\nexport interface InsightsRequestBody {\n  websiteId: string;\n  dateRange: {\n    startDate: string;\n    endDate: string;\n  };\n  fields: { name: string; type: string; label: string }[];\n  filters: { name: string; type: string; operator: string; value: string }[];\n  groups: { name: string; type: string }[];\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    dateRange: yup\n      .object()\n      .shape({\n        startDate: yup.date().required(),\n        endDate: yup.date().required(),\n      })\n      .required(),\n    fields: yup\n      .array()\n      .of(\n        yup.object().shape({\n          name: yup.string().required(),\n          type: yup.string().required(),\n          label: yup.string().required(),\n        }),\n      )\n      .min(1)\n      .required(),\n    filters: yup.array().of(\n      yup.object().shape({\n        name: yup.string().required(),\n        type: yup.string().required(),\n        operator: yup.string().required(),\n        value: yup.string().required(),\n      }),\n    ),\n    groups: yup.array().of(\n      yup.object().shape({\n        name: yup.string().required(),\n        type: yup.string().required(),\n      }),\n    ),\n  }),\n};\n\n\nfunction convertFilters(filters: any[]) {\n  return filters.reduce((obj, filter) => {\n    obj[filter.name] = filter;\n\n    return obj;\n  }, {});\n}\n\nexport default async (\n  req: NextApiRequestQueryBody<any, InsightsRequestBody>,\n  res: NextApiResponse,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  if (req.method === 'POST') {\n    const {\n      websiteId,\n      dateRange: { startDate, endDate },\n      fields,\n      filters,\n    } = req.body;\n\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const data = await getInsights(websiteId, fields, {\n      ...convertFilters(filters),\n      startDate: new Date(startDate),\n      endDate: new Date(endDate),\n    });\n\n    return ok(res, data);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/reports/retention.ts\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { TimezoneTest } from 'lib/yup';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getRetention } from 'queries';\nimport * as yup from 'yup';\n\nexport interface RetentionRequestBody {\n  websiteId: string;\n  dateRange: { startDate: string; endDate: string; timezone: string };\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    dateRange: yup\n      .object()\n      .shape({\n        startDate: yup.date().required(),\n        endDate: yup.date().required(),\n        timezone: TimezoneTest,\n      })\n      .required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<any, RetentionRequestBody>,\n  res: NextApiResponse,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  if (req.method === 'POST') {\n    const {\n      websiteId,\n      dateRange: { startDate, endDate, timezone },\n    } = req.body;\n\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const data = await getRetention(websiteId, {\n      startDate: new Date(startDate),\n      endDate: new Date(endDate),\n      timezone,\n    });\n\n    return ok(res, data);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/share/[shareId].ts\nimport { secret } from 'lib/crypto';\nimport { useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { createToken, methodNotAllowed, notFound, ok } from 'next-basics';\nimport { getSharedWebsite } from 'queries';\nimport * as yup from 'yup';\n\nexport interface ShareRequestQuery {\n  shareId: string;\n}\n\nexport interface ShareResponse {\n  shareId: string;\n  token: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    shareId: yup.string().required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<ShareRequestQuery>,\n  res: NextApiResponse<ShareResponse>,\n) => {\n  await useValidate(schema, req, res);\n\n  const { shareId } = req.query;\n\n  if (req.method === 'GET') {\n    const website = await getSharedWebsite(shareId);\n\n    if (website) {\n      const data = { websiteId: website.id };\n      const token = createToken(data, secret());\n\n      return ok(res, { ...data, token });\n    }\n\n    return notFound(res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/users/index.ts\nimport { canCreateUser } from 'lib/auth';\nimport { ROLES } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { useAuth, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody, Role, SearchFilter, User } from 'lib/types';\nimport { pageInfo } from 'lib/schema';\nimport { NextApiResponse } from 'next';\nimport { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { createUser, getUserByUsername } from 'queries';\nimport * as yup from 'yup';\n\nexport interface UsersRequestQuery extends SearchFilter {}\nexport interface UsersRequestBody {\n  username: string;\n  password: string;\n  id: string;\n  role: Role;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    ...pageInfo,\n  }),\n  POST: yup.object().shape({\n    username: yup.string().max(255).required(),\n    password: yup.string().required(),\n    id: yup.string().uuid(),\n    role: yup\n      .string()\n      .matches(/admin|user|view-only/i)\n      .required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<UsersRequestQuery, UsersRequestBody>,\n  res: NextApiResponse<User[] | User>,\n) => {\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  if (req.method === 'POST') {\n    if (!(await canCreateUser(req.auth))) {\n      return unauthorized(res);\n    }\n\n    const { username, password, role, id } = req.body;\n\n    const existingUser = await getUserByUsername(username, { showDeleted: true });\n\n    if (existingUser) {\n      return badRequest(res, 'User already exists');\n    }\n\n    const created = await createUser({\n      id: id || uuid(),\n      username,\n      password: hashPassword(password),\n      role: role ?? ROLES.user,\n    });\n\n    return ok(res, created);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/users/[userId]/index.ts\nimport * as yup from 'yup';\nimport { canDeleteUser, canUpdateUser, canViewUser } from 'lib/auth';\nimport { useAuth, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody, Role, User } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { deleteUser, getUser, getUserByUsername, updateUser } from 'queries';\n\nexport interface UserRequestQuery {\n  userId: string;\n}\n\nexport interface UserRequestBody {\n  userId: string;\n  username: string;\n  password: string;\n  role: Role;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    userId: yup.string().uuid().required(),\n  }),\n  POST: yup.object().shape({\n    userId: yup.string().uuid().required(),\n    username: yup.string().max(255),\n    password: yup.string(),\n    role: yup.string().matches(/admin|user|view-only/i),\n  }),\n  DELETE: yup.object().shape({\n    userId: yup.string().uuid().required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<UserRequestQuery, UserRequestBody>,\n  res: NextApiResponse<User>,\n) => {\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const {\n    user: { isAdmin },\n  } = req.auth;\n  const userId: string = req.query.userId;\n\n  if (req.method === 'GET') {\n    if (!(await canViewUser(req.auth, userId))) {\n      return unauthorized(res);\n    }\n\n    const user = await getUser(userId);\n\n    return ok(res, user);\n  }\n\n  if (req.method === 'POST') {\n    if (!(await canUpdateUser(req.auth, userId))) {\n      return unauthorized(res);\n    }\n\n    const { username, password, role } = req.body;\n\n    const user = await getUser(userId);\n\n    const data: any = {};\n\n    if (password) {\n      data.password = hashPassword(password);\n    }\n\n    // Only admin can change these fields\n    if (role && isAdmin) {\n      data.role = role;\n    }\n\n    if (username && isAdmin) {\n      data.username = username;\n    }\n\n    // Check when username changes\n    if (data.username && user.username !== data.username) {\n      const user = await getUserByUsername(username);\n\n      if (user) {\n        return badRequest(res, 'User already exists');\n      }\n    }\n\n    const updated = await updateUser(userId, data);\n\n    return ok(res, updated);\n  }\n\n  if (req.method === 'DELETE') {\n    if (!(await canDeleteUser(req.auth))) {\n      return unauthorized(res);\n    }\n\n    if (userId === req.auth.user.id) {\n      return badRequest(res, 'You cannot delete yourself.');\n    }\n\n    await deleteUser(userId);\n\n    return ok(res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/users/[userId]/usage.ts\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getAllWebsites, getEventDataUsage, getEventUsage } from 'queries';\nimport * as yup from 'yup';\n\nexport interface UserUsageRequestQuery {\n  id: string;\n  startAt: string;\n  endAt: string;\n}\n\nexport interface UserUsageRequestResponse {\n  websiteEventUsage: number;\n  eventDataUsage: number;\n  websites: {\n    websiteEventUsage: number;\n    eventDataUsage: number;\n    websiteId: string;\n    websiteName: string;\n  }[];\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    id: yup.string().uuid().required(),\n    startAt: yup.number().integer().required(),\n    endAt: yup.number().integer().moreThan(yup.ref<number>('startAt')).required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<UserUsageRequestQuery>,\n  res: NextApiResponse<UserUsageRequestResponse>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { user } = req.auth;\n\n  if (req.method === 'GET') {\n    if (!user.isAdmin) {\n      return unauthorized(res);\n    }\n\n    const { id: userId, startAt, endAt } = req.query;\n\n    const startDate = new Date(+startAt);\n    const endDate = new Date(+endAt);\n\n    const websites = await getAllWebsites(userId);\n\n    const websiteIds = websites.map(a => a.id);\n\n    const websiteEventUsage = await getEventUsage(websiteIds, startDate, endDate);\n    const eventDataUsage = await getEventDataUsage(websiteIds, startDate, endDate);\n\n    const websiteUsage = websites.map(a => ({\n      websiteId: a.id,\n      websiteName: a.name,\n      websiteEventUsage: websiteEventUsage.find(b => a.id === b.websiteId)?.count || 0,\n      eventDataUsage: eventDataUsage.find(b => a.id === b.websiteId)?.count || 0,\n    }));\n\n    const usage = websiteUsage.reduce(\n      (acc, cv) => {\n        acc.websiteEventUsage += cv.websiteEventUsage;\n        acc.eventDataUsage += cv.eventDataUsage;\n\n        return acc;\n      },\n      { websiteEventUsage: 0, eventDataUsage: 0 },\n    );\n\n    return ok(res, {\n      ...usage,\n      websites: websiteUsage,\n    });\n  }\n\n  return methodNotAllowed(res);\n};\n\n", "needle_token_start": 2396, "needle_token_end": 2443, "code_context_ntokens": 5711, "output": ["```typescript\nfunction convertFilters(filters: any[]) {\n  return filters.reduce((obj, filter) => {\n    obj[filter.name] = filter;\n\n    return obj;\n  }, {});\n}\n```"]}
{"repo": "umami-software/umami", "name": "checkUserBlock", "language": "typescript", "path": "src/lib/session.ts", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To verify if a user has reached a usage limit and block further actions if they have.\n2. **Input**: A unique identifier for a user.\n3. **Output**: Throws an error if the user is blocked due to reaching the usage limit; otherwise, no output.\n4. **Procedure**: The function first checks if the blocking feature is enabled. If it is, it queries a cache to see if the user is already blocked. If the user is blocked, it increments the block count for monitoring purposes and throws an error indicating that the usage limit has been reached.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/lib/auth.ts\nimport { Report } from '@prisma/client';\nimport redis from '@umami/redis-client';\nimport debug from 'debug';\nimport { PERMISSIONS, ROLE_PERMISSIONS, SHARE_TOKEN_HEADER, ROLES } from 'lib/constants';\nimport { secret } from 'lib/crypto';\nimport { NextApiRequest } from 'next';\nimport { createSecureToken, ensureArray, getRandomChars, parseToken } from 'next-basics';\nimport { getTeamUser, getWebsite } from 'queries';\nimport { Auth } from './types';\n\nconst log = debug('umami:auth');\nconst cloudMode = process.env.CLOUD_MODE;\n\nexport async function saveAuth(data: any, expire = 0) {\n  const authKey = `auth:${getRandomChars(32)}`;\n\n  await redis.client.set(authKey, data);\n\n  if (expire) {\n    await redis.client.expire(authKey, expire);\n  }\n\n  return createSecureToken({ authKey }, secret());\n}\n\nexport function getAuthToken(req: NextApiRequest) {\n  try {\n    return req.headers.authorization.split(' ')[1];\n  } catch {\n    return null;\n  }\n}\n\nexport function parseShareToken(req: Request) {\n  try {\n    return parseToken(req.headers[SHARE_TOKEN_HEADER], secret());\n  } catch (e) {\n    log(e);\n    return null;\n  }\n}\n\nexport async function canViewWebsite({ user, shareToken }: Auth, websiteId: string) {\n  if (user?.isAdmin) {\n    return true;\n  }\n\n  if (shareToken?.websiteId === websiteId) {\n    return true;\n  }\n\n  const website = await getWebsite(websiteId);\n\n  if (website.userId) {\n    return user.id === website.userId;\n  }\n\n  if (website.teamId) {\n    const teamUser = await getTeamUser(website.teamId, user.id);\n\n    return !!teamUser;\n  }\n\n  return false;\n}\n\nexport async function canViewAllWebsites({ user }: Auth) {\n  return user.isAdmin;\n}\n\n...\n// Path: src/lib/middleware.ts\nimport cors from 'cors';\nimport debug from 'debug';\nimport redis from '@umami/redis-client';\nimport { getAuthToken, parseShareToken } from 'lib/auth';\nimport { ROLES } from 'lib/constants';\nimport { secret } from 'lib/crypto';\nimport { findSession } from 'lib/session';\nimport {\n  badRequest,\n  createMiddleware,\n  forbidden,\n  parseSecureToken,\n  tooManyRequest,\n  unauthorized,\n} from 'next-basics';\nimport { NextApiRequestCollect } from 'pages/api/send';\nimport { getUser } from '../queries';\n\nconst log = debug('umami:middleware');\n\nexport const useCors = createMiddleware(\n  cors({\n    // Cache CORS preflight request 24 hours by default\n    maxAge: Number(process.env.CORS_MAX_AGE) || 86400,\n  }),\n);\n\nexport const useSession = createMiddleware(async (req, res, next) => {\n  try {\n    const session = await findSession(req as NextApiRequestCollect);\n\n    if (!session) {\n      log('useSession: Session not found');\n      return badRequest(res, 'Session not found.');\n    }\n\n    (req as any).session = session;\n  } catch (e: any) {\n    if (e.message === 'Usage Limit.') {\n      return tooManyRequest(res, e.message);\n    }\n    if (e.message.startsWith('Website not found:')) {\n      return forbidden(res, e.message);\n    }\n    return badRequest(res, e.message);\n  }\n\n  next();\n});\n\nexport const useAuth = createMiddleware(async (req, res, next) => {\n  const token = getAuthToken(req);\n  const payload = parseSecureToken(token, secret());\n  const shareToken = await parseShareToken(req as any);\n\n  let user = null;\n  const { userId, authKey, grant } = payload || {};\n\n  if (userId) {\n    user = await getUser(userId);\n  } else if (redis.enabled && authKey) {\n    const key = await redis.client.get(authKey);\n\n    if (key?.userId) {\n      user = await getUser(key.userId);\n    }\n  }\n\n  if (process.env.NODE_ENV === 'development') {\n    log('useAuth:', { token, shareToken, payload, user, grant });\n  }\n\n  if (!user?.id && !shareToken) {\n    log('useAuth: User not authorized');\n    return unauthorized(res);\n  }\n\n  if (user) {\n    user.isAdmin = user.role === ROLES.admin;\n  }\n\n  (req as any).auth = {\n    user,\n    grant,\n    token,\n    shareToken,\n    authKey,\n  };\n\n  next();\n});\n\nexport const useValidate = async (schema, req, res) => {\n  return createMiddleware(async (req: any, res, next) => {\n    try {\n      const rules = schema[req.method];\n\n      if (rules) {\n        rules.validateSync({ ...req.query, ...req.body });\n      }\n    } catch (e: any) {\n      return badRequest(res, e.message);\n    }\n\n    next();\n  })(req, res);\n};\n\n// Path: src/pages/api/send.ts\nimport ipaddr from 'ipaddr.js';\nimport { isbot } from 'isbot';\nimport { COLLECTION_TYPE, HOSTNAME_REGEX, IP_REGEX } from 'lib/constants';\nimport { secret, visitSalt, uuid } from 'lib/crypto';\nimport { getIpAddress } from 'lib/detect';\nimport { useCors, useSession, useValidate } from 'lib/middleware';\nimport { CollectionType, YupRequest } from 'lib/types';\nimport { NextApiRequest, NextApiResponse } from 'next';\nimport {\n  badRequest,\n  createToken,\n  forbidden,\n  methodNotAllowed,\n  ok,\n  safeDecodeURI,\n  send,\n} from 'next-basics';\nimport { saveEvent, saveSessionData } from 'queries';\nimport * as yup from 'yup';\n\nexport interface CollectRequestBody {\n  payload: {\n    website: string;\n    data?: { [key: string]: any };\n    hostname?: string;\n    ip?: string;\n    language?: string;\n    name?: string;\n    referrer?: string;\n    screen?: string;\n    tag?: string;\n    title?: string;\n    url: string;\n  };\n  type: CollectionType;\n}\n\nexport interface NextApiRequestCollect extends NextApiRequest {\n  body: CollectRequestBody;\n  session: {\n    id: string;\n    websiteId: string;\n    visitId: string;\n    ownerId: string;\n    hostname: string;\n    browser: string;\n    os: string;\n    device: string;\n    screen: string;\n    language: string;\n    country: string;\n    subdivision1: string;\n    subdivision2: string;\n    city: string;\n    iat: number;\n  };\n  headers: { [key: string]: any };\n  yup: YupRequest;\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    payload: yup\n      .object()\n      .shape({\n        data: yup.object(),\n        hostname: yup.string().matches(HOSTNAME_REGEX).max(100),\n        ip: yup.string().matches(IP_REGEX),\n        language: yup.string().max(35),\n        referrer: yup.string(),\n        screen: yup.string().max(11),\n        title: yup.string(),\n        url: yup.string(),\n        website: yup.string().uuid().required(),\n        name: yup.string().max(50),\n        tag: yup.string().max(50).nullable(),\n      })\n      .required(),\n    type: yup\n      .string()\n      .matches(/event|identify/i)\n      .required(),\n  }),\n};\n\nexport default async (req: NextApiRequestCollect, res: NextApiResponse) => {\n  await useCors(req, res);\n\n  if (req.method === 'POST') {\n    if (!process.env.DISABLE_BOT_CHECK && isbot(req.headers['user-agent'])) {\n      return ok(res);\n    }\n\n    await useValidate(schema, req, res);\n\n    if (hasBlockedIp(req)) {\n      return forbidden(res);\n    }\n\n    const { type, payload } = req.body;\n    const { url, referrer, name: eventName, data, title } = payload;\n    const pageTitle = safeDecodeURI(title);\n\n    await useSession(req, res);\n\n    const session = req.session;\n\n    // expire visitId after 30 minutes\n    session.visitId =\n      !!session.iat && Math.floor(new Date().getTime() / 1000) - session.iat > 1800\n        ? uuid(session.id, visitSalt())\n        : session.visitId;\n\n    session.iat = Math.floor(new Date().getTime() / 1000);\n\n    if (type === COLLECTION_TYPE.event) {\n      // eslint-disable-next-line prefer-const\n      let [urlPath, urlQuery] = safeDecodeURI(url)?.split('?') || [];\n      let [referrerPath, referrerQuery] = safeDecodeURI(referrer)?.split('?') || [];\n      let referrerDomain = '';\n\n      if (!urlPath) {\n        urlPath = '/';\n      }\n\n      if (referrerPath?.startsWith('http')) {\n        const refUrl = new URL(referrer);\n        referrerPath = refUrl.pathname;\n        referrerQuery = refUrl.search.substring(1);\n        referrerDomain = refUrl.hostname.replace(/www\\./, '');\n      }\n\n      if (process.env.REMOVE_TRAILING_SLASH) {\n        urlPath = urlPath.replace(/(.+)\\/$/, '$1');\n      }\n\n      await saveEvent({\n        urlPath,\n        urlQuery,\n        referrerPath,\n        referrerQuery,\n        referrerDomain,\n        pageTitle,\n        eventName,\n        eventData: data,\n        ...session,\n        sessionId: session.id,\n        visitId: session.visitId,\n      });\n    }\n\n    if (type === COLLECTION_TYPE.identify) {\n      if (!data) {\n        return badRequest(res, 'Data required.');\n      }\n\n      await saveSessionData({\n        websiteId: session.websiteId,\n        sessionId: session.id,\n        sessionData: data,\n      });\n    }\n\n    const token = createToken(session, secret());\n\n    return send(res, token);\n  }\n\n  return methodNotAllowed(res);\n};\n\nfunction hasBlockedIp(req: NextApiRequestCollect) {\n  const ignoreIps = process.env.IGNORE_IP;\n\n  if (ignoreIps) {\n    const ips = [];\n\n    if (ignoreIps) {\n      ips.push(...ignoreIps.split(',').map(n => n.trim()));\n    }\n\n    const clientIp = getIpAddress(req);\n\n    return ips.find(ip => {\n      if (ip === clientIp) return true;\n\n      // CIDR notation\n      if (ip.indexOf('/') > 0) {\n        const addr = ipaddr.parse(clientIp);\n        const range = ipaddr.parseCIDR(ip);\n\n        if (addr.kind() === range[0].kind() && addr.match(range)) return true;\n      }\n    });\n  }\n\n  return false;\n}\n\n// Path: src/lib/session.ts\nimport { isUuid, secret, uuid, visitSalt } from 'lib/crypto';\nimport { getClientInfo } from 'lib/detect';\nimport { parseToken } from 'next-basics';\nimport { NextApiRequestCollect } from 'pages/api/send';\nimport { createSession } from 'queries';\nimport cache from './cache';\nimport clickhouse from './clickhouse';\nimport { loadSession, loadWebsite } from './load';\n\nexport async function findSession(req: NextApiRequestCollect): Promise<{\n  id: any;\n  websiteId: string;\n  visitId: string;\n  hostname: string;\n  browser: string;\n  os: any;\n  device: string;\n  screen: string;\n  language: string;\n  country: any;\n  subdivision1: any;\n  subdivision2: any;\n  city: any;\n  ownerId: string;\n}> {\n  const { payload } = req.body;\n\n  if (!payload) {\n    throw new Error('Invalid payload.');\n  }\n\n  // Check if cache token is passed\n  const cacheToken = req.headers['x-umami-cache'];\n\n  if (cacheToken) {\n    const result = await parseToken(cacheToken, secret());\n\n    if (result) {\n      await checkUserBlock(result?.ownerId);\n\n      return result;\n    }\n  }\n\n  // Verify payload\n  const { website: websiteId, hostname, screen, language } = payload;\n\n  // Check the hostname value for legality to eliminate dirty data\n  const validHostnameRegex = /^[\\w-.]+$/;\n  if (!validHostnameRegex.test(hostname)) {\n    throw new Error('Invalid hostname.');\n  }\n\n  if (!isUuid(websiteId)) {\n    throw new Error('Invalid website ID.');\n  }\n\n  // Find website\n  const website = await loadWebsite(websiteId);\n\n  if (!website) {\n    throw new Error(`Website not found: ${websiteId}.`);\n  }\n\n  await checkUserBlock(website.userId);\n\n  const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =\n    await getClientInfo(req, payload);\n\n  const sessionId = uuid(websiteId, hostname, ip, userAgent);\n  const visitId = uuid(sessionId, visitSalt());\n\n  // Clickhouse does not require session lookup\n  if (clickhouse.enabled) {\n    return {\n      id: sessionId,\n      websiteId,\n      visitId,\n      hostname,\n      browser,\n      os: os as any,\n      device,\n      screen,\n      language,\n      country,\n      subdivision1,\n      subdivision2,\n      city,\n      ownerId: website.userId,\n    };\n  }\n\n  // Find session\n  let session = await loadSession(sessionId);\n\n  // Create a session if not found\n  if (!session) {\n    try {\n      session = await createSession({\n        id: sessionId,\n        websiteId,\n        hostname,\n        browser,\n        os,\n        device,\n        screen,\n        language,\n        country,\n        subdivision1,\n        subdivision2,\n        city,\n      });\n    } catch (e: any) {\n      if (!e.message.toLowerCase().includes('unique constraint')) {\n        throw e;\n      }\n    }\n  }\n\n  return { ...session, ownerId: website.userId, visitId: visitId };\n}\n\n\nasync function checkUserBlock(userId: string) {\n  if (process.env.ENABLE_BLOCKER && (await cache.fetchUserBlock(userId))) {\n    await cache.incrementUserBlock(userId);\n\n    throw new Error('Usage Limit.');\n  }\n}\n\n// Path: src/lib/request.ts\nimport { NextApiRequest } from 'next';\nimport { getAllowedUnits, getMinimumUnit } from './date';\nimport { getWebsiteDateRange } from '../queries';\nimport { FILTER_COLUMNS } from 'lib/constants';\n\nexport async function getRequestDateRange(req: NextApiRequest) {\n  const { websiteId, startAt, endAt, unit } = req.query;\n\n  // All-time\n  if (+startAt === 0 && +endAt === 1) {\n    const result = await getWebsiteDateRange(websiteId as string);\n    const { min, max } = result[0];\n    const startDate = new Date(min);\n    const endDate = new Date(max);\n\n    return {\n      startDate,\n      endDate,\n      unit: getMinimumUnit(startDate, endDate),\n    };\n  }\n\n  const startDate = new Date(+startAt);\n  const endDate = new Date(+endAt);\n  const minUnit = getMinimumUnit(startDate, endDate);\n\n  return {\n    startDate,\n    endDate,\n    unit: (getAllowedUnits(startDate, endDate).includes(unit as string) ? unit : minUnit) as string,\n  };\n}\n\nexport function getRequestFilters(req: NextApiRequest) {\n  return Object.keys(FILTER_COLUMNS).reduce((obj, key) => {\n    const value = req.query[key];\n\n    if (value !== undefined) {\n      obj[key] = value;\n    }\n\n    return obj;\n  }, {});\n}\n\n// Path: src/lib/charts.ts\nimport { formatDate } from 'lib/date';\nimport { formatLongNumber } from 'lib/format';\n\nexport function renderNumberLabels(label: string) {\n  return +label > 1000 ? formatLongNumber(+label) : label;\n}\n\nexport function renderDateLabels(unit: string, locale: string) {\n  return (label: string, index: number, values: any[]) => {\n    const d = new Date(values[index].value);\n\n    switch (unit) {\n      case 'minute':\n        return formatDate(d, 'h:mm', locale);\n      case 'hour':\n        return formatDate(d, 'p', locale);\n      case 'day':\n        return formatDate(d, 'MMM d', locale);\n      case 'month':\n        return formatDate(d, 'MMM', locale);\n      case 'year':\n        return formatDate(d, 'YYY', locale);\n      default:\n        return label;\n    }\n  };\n}\n\n// Path: src/lib/schema.ts\nimport * as yup from 'yup';\n\nexport const dateRange = {\n  startAt: yup.number().integer().required(),\n  endAt: yup.number().integer().moreThan(yup.ref('startAt')).required(),\n};\n\nexport const pageInfo = {\n  query: yup.string(),\n  page: yup.number().integer().positive(),\n  pageSize: yup.number().integer().positive().min(1).max(200),\n  orderBy: yup.string(),\n};\n\n// Path: src/lib/sql.ts\n\n// Path: src/lib/__tests__/detect.test.ts\nimport * as detect from '../detect';\n\nconst IP = '127.0.0.1';\n\ntest('getIpAddress: Custom header', () => {\n  process.env.CLIENT_IP_HEADER = 'x-custom-ip-header';\n\n  expect(detect.getIpAddress({ headers: { 'x-custom-ip-header': IP } } as any)).toEqual(IP);\n});\n\ntest('getIpAddress: CloudFlare header', () => {\n  expect(detect.getIpAddress({ headers: { 'cf-connecting-ip': IP } } as any)).toEqual(IP);\n});\n\ntest('getIpAddress: Standard header', () => {\n  expect(detect.getIpAddress({ headers: { 'x-forwarded-for': IP } } as any)).toEqual(IP);\n});\n\ntest('getIpAddress: No header', () => {\n  expect(detect.getIpAddress({ headers: {} } as any)).toEqual(null);\n});\n\n// Path: src/lib/__tests__/format.test.ts\nimport * as format from '../format';\n\ntest('parseTime', () => {\n  expect(format.parseTime(86400 + 3600 + 60 + 1)).toEqual({\n    days: 1,\n    hours: 1,\n    minutes: 1,\n    seconds: 1,\n    ms: 0,\n  });\n});\n\ntest('formatTime', () => {\n  expect(format.formatTime(3600 + 60 + 1)).toBe('1:01:01');\n});\n\ntest('formatShortTime', () => {\n  expect(format.formatShortTime(3600 + 60 + 1)).toBe('1m1s');\n\n  expect(format.formatShortTime(3600 + 60 + 1, ['h', 'm', 's'])).toBe('1h1m1s');\n});\n\ntest('formatNumber', () => {\n  expect(format.formatNumber('10.2')).toBe('10');\n  expect(format.formatNumber('10.5')).toBe('11');\n});\n\ntest('formatLongNumber', () => {\n  expect(format.formatLongNumber(1200000)).toBe('1.2m');\n  expect(format.formatLongNumber(575000)).toBe('575k');\n  expect(format.formatLongNumber(10500)).toBe('10.5k');\n  expect(format.formatLongNumber(1200)).toBe('1.20k');\n});\n\ntest('stringToColor', () => {\n  expect(format.stringToColor('hello')).toBe('#d218e9');\n  expect(format.stringToColor('goodbye')).toBe('#11e956');\n});\n\n// Path: src/store/dashboard.ts\nimport { create } from 'zustand';\nimport { DASHBOARD_CONFIG, DEFAULT_WEBSITE_LIMIT } from 'lib/constants';\nimport { getItem, setItem } from 'next-basics';\n\nexport const initialState = {\n  showCharts: true,\n  limit: DEFAULT_WEBSITE_LIMIT,\n  websiteOrder: [],\n  editing: false,\n};\n\nconst store = create(() => ({ ...initialState, ...getItem(DASHBOARD_CONFIG) }));\n\nexport function saveDashboard(settings) {\n  store.setState(settings);\n\n  setItem(DASHBOARD_CONFIG, store.getState());\n}\n\nexport default store;\n\n// Path: src/store/cache.ts\nimport { create } from 'zustand';\n\nconst store = create(() => ({}));\n\nexport function setValue(key: string, value: any) {\n  store.setState({ [key]: value });\n}\n\nexport default store;\n\n// Path: src/store/version.ts\nimport { create } from 'zustand';\nimport { produce } from 'immer';\nimport semver from 'semver';\nimport { CURRENT_VERSION, VERSION_CHECK, UPDATES_URL } from 'lib/constants';\nimport { getItem } from 'next-basics';\n\nconst initialState = {\n  current: CURRENT_VERSION,\n  latest: null,\n  hasUpdate: false,\n  checked: false,\n  releaseUrl: null,\n};\n\nconst store = create(() => ({ ...initialState }));\n\nexport async function checkVersion() {\n  const { current } = store.getState();\n\n  const data = await fetch(`${UPDATES_URL}?v=${current}`, {\n    method: 'GET',\n    headers: {\n      Accept: 'application/json',\n    },\n  }).then(res => {\n    if (res.ok) {\n      return res.json();\n    }\n\n    return null;\n  });\n\n  if (!data) {\n    return;\n  }\n\n  store.setState(\n    produce(state => {\n      const { latest, url } = data;\n      const lastCheck = getItem(VERSION_CHECK);\n\n      const hasUpdate = !!(latest && lastCheck?.version !== latest && semver.gt(latest, current));\n\n      state.current = current;\n      state.latest = latest;\n      state.hasUpdate = hasUpdate;\n      state.checked = true;\n      state.releaseUrl = url;\n\n      return state;\n    }),\n  );\n}\n\nexport default store;\n\n// Path: src/pages/api/heartbeat.ts\nimport { NextApiRequest, NextApiResponse } from 'next';\nimport { ok } from 'next-basics';\n\nexport default async (req: NextApiRequest, res: NextApiResponse) => {\n  return ok(res);\n};\n\n// Path: src/pages/api/config.ts\nimport { NextApiRequest, NextApiResponse } from 'next';\nimport { ok, methodNotAllowed } from 'next-basics';\n\nexport interface ConfigResponse {\n  telemetryDisabled: boolean;\n  trackerScriptName: string;\n  uiDisabled: boolean;\n  updatesDisabled: boolean;\n}\n\nexport default async (req: NextApiRequest, res: NextApiResponse<ConfigResponse>) => {\n  if (req.method === 'GET') {\n    return ok(res, {\n      telemetryDisabled: !!process.env.DISABLE_TELEMETRY,\n      trackerScriptName: process.env.TRACKER_SCRIPT_NAME,\n      uiDisabled: !!process.env.DISABLE_UI,\n      updatesDisabled: !!process.env.DISABLE_UPDATES,\n    });\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/version.ts\nimport { NextApiRequest, NextApiResponse } from 'next';\nimport { ok, methodNotAllowed } from 'next-basics';\nimport { CURRENT_VERSION } from 'lib/constants';\n\nexport interface VersionResponse {\n  version: string;\n}\n\nexport default async (req: NextApiRequest, res: NextApiResponse<VersionResponse>) => {\n  if (req.method === 'GET') {\n    return ok(res, {\n      version: CURRENT_VERSION,\n    });\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/auth/verify.ts\nimport { NextApiRequestAuth } from 'lib/types';\nimport { useAuth } from 'lib/middleware';\nimport { NextApiResponse } from 'next';\nimport { ok } from 'next-basics';\n\nexport default async (req: NextApiRequestAuth, res: NextApiResponse) => {\n  await useAuth(req, res);\n\n  const { user } = req.auth;\n\n  return ok(res, user);\n};\n\n// Path: src/pages/api/auth/login.ts\nimport redis from '@umami/redis-client';\nimport debug from 'debug';\nimport { saveAuth } from 'lib/auth';\nimport { secret } from 'lib/crypto';\nimport { useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody, User } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport {\n  checkPassword,\n  createSecureToken,\n  forbidden,\n  methodNotAllowed,\n  ok,\n  unauthorized,\n} from 'next-basics';\nimport { getUserByUsername } from 'queries';\nimport * as yup from 'yup';\nimport { ROLES } from 'lib/constants';\nimport { getIpAddress } from 'lib/detect';\n\nconst log = debug('umami:auth');\n\nexport interface LoginRequestBody {\n  username: string;\n  password: string;\n}\n\nexport interface LoginResponse {\n  token: string;\n  user: User;\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    username: yup.string().required(),\n    password: yup.string().required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<any, LoginRequestBody>,\n  res: NextApiResponse<LoginResponse>,\n) => {\n  if (process.env.DISABLE_LOGIN) {\n    return forbidden(res);\n  }\n\n  await useValidate(schema, req, res);\n\n  if (req.method === 'POST') {\n    const { username, password } = req.body;\n\n    const user = await getUserByUsername(username, { includePassword: true });\n\n    if (user && checkPassword(password, user.password)) {\n      if (redis.enabled) {\n        const token = await saveAuth({ userId: user.id });\n\n        return ok(res, { token, user });\n      }\n\n      const token = createSecureToken({ userId: user.id }, secret());\n      const { id, username, role, createdAt } = user;\n\n      return ok(res, {\n        token,\n        user: { id, username, role, createdAt, isAdmin: role === ROLES.admin },\n      });\n    }\n\n    log(\n      `Login from ip ${getIpAddress(req)} with username \"${username.replace(\n        /[\"\\r\\n]/g,\n        '',\n      )}\" failed.`,\n    );\n\n    return unauthorized(res, 'message.incorrect-username-password');\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/auth/sso.ts\nimport { NextApiRequestAuth } from 'lib/types';\nimport { useAuth } from 'lib/middleware';\nimport { NextApiResponse } from 'next';\nimport { badRequest, ok } from 'next-basics';\nimport redis from '@umami/redis-client';\nimport { saveAuth } from 'lib/auth';\n\nexport default async (req: NextApiRequestAuth, res: NextApiResponse) => {\n  await useAuth(req, res);\n\n  if (redis.enabled && req.auth.user) {\n    const token = await saveAuth({ userId: req.auth.user.id }, 86400);\n\n    return ok(res, { user: req.auth.user, token });\n  }\n\n  return badRequest(res);\n};\n\n// Path: src/pages/api/auth/logout.ts\nimport { methodNotAllowed, ok } from 'next-basics';\nimport redis from '@umami/redis-client';\nimport { useAuth } from 'lib/middleware';\nimport { getAuthToken } from 'lib/auth';\nimport { NextApiRequest, NextApiResponse } from 'next';\n\nexport default async (req: NextApiRequest, res: NextApiResponse) => {\n  await useAuth(req, res);\n\n  if (req.method === 'POST') {\n    if (redis.enabled) {\n      await redis.client.del(getAuthToken(req));\n    }\n\n    return ok(res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/users/[userId]/websites.ts\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { pageInfo } from 'lib/schema';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getUserWebsites } from 'queries';\nimport * as yup from 'yup';\n\nconst schema = {\n  GET: yup.object().shape({\n    userId: yup.string().uuid().required(),\n    teamId: yup.string().uuid(),\n    ...pageInfo,\n  }),\n};\n\nexport default async (req: NextApiRequestQueryBody, res: NextApiResponse) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { user } = req.auth;\n  const { userId, page = 1, pageSize, query = '', ...rest } = req.query;\n\n  if (req.method === 'GET') {\n    if (!user.isAdmin && user.id !== userId) {\n      return unauthorized(res);\n    }\n\n    const websites = await getUserWebsites(userId, {\n      page,\n      pageSize,\n      query,\n      ...rest,\n    });\n\n    return ok(res, websites);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/index.ts\nimport { canCreateTeamWebsite, canCreateWebsite } from 'lib/auth';\nimport { uuid } from 'lib/crypto';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody, SearchFilter } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { createWebsite } from 'queries';\nimport userWebsitesRoute from 'pages/api/users/[userId]/websites';\nimport * as yup from 'yup';\nimport { pageInfo } from 'lib/schema';\n\nexport interface WebsitesRequestQuery extends SearchFilter {}\n\nexport interface WebsitesRequestBody {\n  name: string;\n  domain: string;\n  shareId: string;\n  teamId: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    ...pageInfo,\n  }),\n  POST: yup.object().shape({\n    name: yup.string().max(100).required(),\n    domain: yup.string().max(500).required(),\n    shareId: yup.string().max(50).nullable(),\n    teamId: yup.string().nullable(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsitesRequestQuery, WebsitesRequestBody>,\n  res: NextApiResponse,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const {\n    user: { id: userId },\n  } = req.auth;\n\n  if (req.method === 'GET') {\n    if (!req.query.userId) {\n      req.query.userId = userId;\n    }\n\n    return userWebsitesRoute(req, res);\n  }\n\n  if (req.method === 'POST') {\n    const { name, domain, shareId, teamId } = req.body;\n\n    if (\n      (teamId && !(await canCreateTeamWebsite(req.auth, teamId))) ||\n      !(await canCreateWebsite(req.auth))\n    ) {\n      return unauthorized(res);\n    }\n\n    const data: any = {\n      id: uuid(),\n      createdBy: userId,\n      name,\n      domain,\n      shareId,\n      teamId,\n    };\n\n    if (!teamId) {\n      data.userId = userId;\n    }\n\n    const website = await createWebsite(data);\n\n    return ok(res, website);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/daterange.ts\nimport { WebsiteActive, NextApiRequestQueryBody } from 'lib/types';\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getWebsiteDateRange } from 'queries';\nimport * as yup from 'yup';\n\nexport interface WebsiteDateRangeRequestQuery {\n  websiteId: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteDateRangeRequestQuery>,\n  res: NextApiResponse<WebsiteActive>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId } = req.query;\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const result = await getWebsiteDateRange(websiteId);\n\n    return ok(res, result);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/reports.ts\nimport * as yup from 'yup';\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody, SearchFilter } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getWebsiteReports } from 'queries';\nimport { pageInfo } from 'lib/schema';\n\nexport interface ReportsRequestQuery extends SearchFilter {\n  websiteId: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    ...pageInfo,\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<ReportsRequestQuery, any>,\n  res: NextApiResponse,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId } = req.query;\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const { page, query, pageSize } = req.query;\n\n    const data = await getWebsiteReports(websiteId, {\n      page,\n      pageSize,\n      query,\n    });\n\n    return ok(res, data);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/reset.ts\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { canUpdateWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { resetWebsite } from 'queries';\nimport * as yup from 'yup';\n\nexport interface WebsiteResetRequestQuery {\n  websiteId: string;\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteResetRequestQuery>,\n  res: NextApiResponse,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId } = req.query;\n\n  if (req.method === 'POST') {\n    if (!(await canUpdateWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    await resetWebsite(websiteId);\n\n    return ok(res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/stats.ts\nimport { subMinutes, differenceInMinutes } from 'date-fns';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody, WebsiteStats } from 'lib/types';\nimport { getRequestFilters, getRequestDateRange } from 'lib/request';\nimport { getWebsiteStats } from 'queries';\n\nexport interface WebsiteStatsRequestQuery {\n  websiteId: string;\n  startAt: number;\n  endAt: number;\n  url?: string;\n  referrer?: string;\n  title?: string;\n  query?: string;\n  event?: string;\n  os?: string;\n  browser?: string;\n  device?: string;\n  country?: string;\n  region?: string;\n  city?: string;\n}\n\nimport * as yup from 'yup';\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    startAt: yup.number().required(),\n    endAt: yup.number().required(),\n    url: yup.string(),\n    referrer: yup.string(),\n    title: yup.string(),\n    query: yup.string(),\n    event: yup.string(),\n    os: yup.string(),\n    browser: yup.string(),\n    device: yup.string(),\n    country: yup.string(),\n    region: yup.string(),\n    city: yup.string(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteStatsRequestQuery>,\n  res: NextApiResponse<WebsiteStats>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId } = req.query;\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const { startDate, endDate } = await getRequestDateRange(req);\n    const diff = differenceInMinutes(endDate, startDate);\n    const prevStartDate = subMinutes(startDate, diff);\n    const prevEndDate = subMinutes(endDate, diff);\n\n    const filters = getRequestFilters(req);\n\n    const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate });\n\n    const prevPeriod = await getWebsiteStats(websiteId, {\n      ...filters,\n      startDate: prevStartDate,\n      endDate: prevEndDate,\n    });\n\n    const stats = Object.keys(metrics[0]).reduce((obj, key) => {\n      obj[key] = {\n        value: Number(metrics[0][key]) || 0,\n        change: Number(metrics[0][key]) - Number(prevPeriod[0][key]) || 0,\n      };\n      return obj;\n    }, {});\n\n    return ok(res, stats);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/pageviews.ts\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { getRequestFilters, getRequestDateRange } from 'lib/request';\nimport { NextApiRequestQueryBody, WebsitePageviews } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getPageviewStats, getSessionStats } from 'queries';\n\nexport interface WebsitePageviewRequestQuery {\n  websiteId: string;\n  startAt: number;\n  endAt: number;\n  unit?: string;\n  timezone?: string;\n  url?: string;\n  referrer?: string;\n  title?: string;\n  os?: string;\n  browser?: string;\n  device?: string;\n  country?: string;\n  region: string;\n  city?: string;\n}\n\nimport { TimezoneTest, UnitTypeTest } from 'lib/yup';\nimport * as yup from 'yup';\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    startAt: yup.number().required(),\n    endAt: yup.number().required(),\n    unit: UnitTypeTest,\n    timezone: TimezoneTest,\n    url: yup.string(),\n    referrer: yup.string(),\n    title: yup.string(),\n    os: yup.string(),\n    browser: yup.string(),\n    device: yup.string(),\n    country: yup.string(),\n    region: yup.string(),\n    city: yup.string(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsitePageviewRequestQuery>,\n  res: NextApiResponse<WebsitePageviews>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId, timezone } = req.query;\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const { startDate, endDate, unit } = await getRequestDateRange(req);\n\n    const filters = {\n      ...getRequestFilters(req),\n      startDate,\n      endDate,\n      timezone,\n      unit,\n    };\n\n    const [pageviews, sessions] = await Promise.all([\n      getPageviewStats(websiteId, filters),\n      getSessionStats(websiteId, filters),\n    ]);\n\n    return ok(res, { pageviews, sessions });\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/values.ts\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiResponse } from 'next';\nimport {\n  badRequest,\n  methodNotAllowed,\n  ok,\n  safeDecodeURIComponent,\n  unauthorized,\n} from 'next-basics';\nimport { EVENT_COLUMNS, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';\nimport { getValues } from 'queries';\nimport { getRequestDateRange } from 'lib/request';\nimport * as yup from 'yup';\n\nexport interface ValuesRequestQuery {\n  websiteId: string;\n  type: string;\n  startAt: number;\n  endAt: number;\n  search?: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    type: yup.string().required(),\n    startAt: yup.number().required(),\n    endAt: yup.number().required(),\n    search: yup.string(),\n  }),\n};\n\nexport default async (req: NextApiRequestQueryBody<ValuesRequestQuery>, res: NextApiResponse) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId, type, search } = req.query;\n  const { startDate, endDate } = await getRequestDateRange(req);\n\n  if (req.method === 'GET') {\n    if (!SESSION_COLUMNS.includes(type as string) && !EVENT_COLUMNS.includes(type as string)) {\n      return badRequest(res);\n    }\n\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const values = await getValues(\n      websiteId,\n      FILTER_COLUMNS[type as string],\n      startDate,\n      endDate,\n      search,\n    );\n\n    return ok(\n      res,\n      values\n        .map(({ value }) => safeDecodeURIComponent(value))\n        .filter(n => n)\n        .sort(),\n    );\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/index.ts\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, serverError, unauthorized } from 'next-basics';\nimport { Website, NextApiRequestQueryBody } from 'lib/types';\nimport { canViewWebsite, canUpdateWebsite, canDeleteWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { deleteWebsite, getWebsite, updateWebsite } from 'queries';\nimport { SHARE_ID_REGEX } from 'lib/constants';\n\nexport interface WebsiteRequestQuery {\n  websiteId: string;\n}\n\nexport interface WebsiteRequestBody {\n  name: string;\n  domain: string;\n  shareId: string;\n}\n\nimport * as yup from 'yup';\n\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n  }),\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    name: yup.string(),\n    domain: yup.string(),\n    shareId: yup.string().matches(SHARE_ID_REGEX, { excludeEmptyString: true }).nullable(),\n  }),\n  DELETE: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteRequestQuery, WebsiteRequestBody>,\n  res: NextApiResponse<Website>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId } = req.query;\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const website = await getWebsite(websiteId);\n\n    return ok(res, website);\n  }\n\n  if (req.method === 'POST') {\n    if (!(await canUpdateWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const { name, domain, shareId } = req.body;\n\n    try {\n      const website = await updateWebsite(websiteId, { name, domain, shareId });\n\n      return ok(res, website);\n    } catch (e: any) {\n      if (e.message.includes('Unique constraint') && e.message.includes('share_id')) {\n        return serverError(res, 'That share ID is already taken.');\n      }\n\n      return serverError(res, e);\n    }\n  }\n\n  if (req.method === 'DELETE') {\n    if (!(await canDeleteWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    await deleteWebsite(websiteId);\n\n    return ok(res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/active.ts\nimport { WebsiteActive, NextApiRequestQueryBody } from 'lib/types';\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getActiveVisitors } from 'queries';\nimport * as yup from 'yup';\n\nexport interface WebsiteActiveRequestQuery {\n  websiteId: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteActiveRequestQuery>,\n  res: NextApiResponse<WebsiteActive>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId } = req.query;\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId as string))) {\n      return unauthorized(res);\n    }\n\n    const result = await getActiveVisitors(websiteId as string);\n\n    return ok(res, result);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/transfer.ts\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { canTransferWebsiteToTeam, canTransferWebsiteToUser } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiResponse } from 'next';\nimport { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { updateWebsite } from 'queries';\nimport * as yup from 'yup';\n\nexport interface WebsiteTransferRequestQuery {\n  websiteId: string;\n}\n\nexport interface WebsiteTransferRequestBody {\n  userId?: string;\n  teamId?: string;\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    userId: yup.string().uuid(),\n    teamId: yup.string().uuid(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteTransferRequestQuery, WebsiteTransferRequestBody>,\n  res: NextApiResponse,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId } = req.query;\n  const { userId, teamId } = req.body;\n\n  if (req.method === 'POST') {\n    if (userId) {\n      if (!(await canTransferWebsiteToUser(req.auth, websiteId, userId))) {\n        return unauthorized(res);\n      }\n\n      const website = await updateWebsite(websiteId, {\n        userId,\n        teamId: null,\n      });\n\n      return ok(res, website);\n    } else if (teamId) {\n      if (!(await canTransferWebsiteToTeam(req.auth, websiteId, teamId))) {\n        return unauthorized(res);\n      }\n\n      const website = await updateWebsite(websiteId, {\n        userId: null,\n        teamId,\n      });\n\n      return ok(res, website);\n    }\n\n    return badRequest(res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/metrics.ts\nimport { NextApiResponse } from 'next';\nimport { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { WebsiteMetric, NextApiRequestQueryBody } from 'lib/types';\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { SESSION_COLUMNS, EVENT_COLUMNS, FILTER_COLUMNS, OPERATORS } from 'lib/constants';\nimport { getPageviewMetrics, getSessionMetrics } from 'queries';\nimport { getRequestFilters, getRequestDateRange } from 'lib/request';\nimport * as yup from 'yup';\n\nexport interface WebsiteMetricsRequestQuery {\n  websiteId: string;\n  type: string;\n  startAt: number;\n  endAt: number;\n  url?: string;\n  referrer?: string;\n  title?: string;\n  query?: string;\n  os?: string;\n  browser?: string;\n  device?: string;\n  country?: string;\n  region?: string;\n  city?: string;\n  language?: string;\n  event?: string;\n  limit?: number;\n  offset?: number;\n  search?: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    type: yup.string().required(),\n    startAt: yup.number().required(),\n    endAt: yup.number().required(),\n    url: yup.string(),\n    referrer: yup.string(),\n    title: yup.string(),\n    query: yup.string(),\n    os: yup.string(),\n    browser: yup.string(),\n    device: yup.string(),\n    country: yup.string(),\n    region: yup.string(),\n    city: yup.string(),\n    language: yup.string(),\n    event: yup.string(),\n    limit: yup.number(),\n    offset: yup.number(),\n    search: yup.string(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteMetricsRequestQuery>,\n  res: NextApiResponse<WebsiteMetric[]>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId, type, limit, offset, search } = req.query;\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const { startDate, endDate } = await getRequestDateRange(req);\n    const column = FILTER_COLUMNS[type] || type;\n    const filters = {\n      ...getRequestFilters(req),\n      startDate,\n      endDate,\n    };\n\n    if (search) {\n      filters[type] = {\n        name: type,\n        column,\n        operator: OPERATORS.contains,\n        value: search,\n      };\n    }\n\n    if (SESSION_COLUMNS.includes(type)) {\n      const data = await getSessionMetrics(websiteId, type, filters, limit, offset);\n\n      if (type === 'language') {\n        const combined = {};\n\n        for (const { x, y } of data) {\n          const key = String(x).toLowerCase().split('-')[0];\n\n          if (combined[key] === undefined) {\n            combined[key] = { x: key, y };\n          } else {\n            combined[key].y += y;\n          }\n        }\n\n        return ok(res, Object.values(combined));\n      }\n\n      return ok(res, data);\n    }\n\n    if (EVENT_COLUMNS.includes(type)) {\n      const data = await getPageviewMetrics(websiteId, column, filters, limit, offset);\n\n      return ok(res, data);\n    }\n\n    return badRequest(res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/websites/[websiteId]/events.ts\nimport { canViewWebsite } from 'lib/auth';\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { getRequestDateRange } from 'lib/request';\nimport { NextApiRequestQueryBody, WebsiteMetric } from 'lib/types';\nimport { TimezoneTest, UnitTypeTest } from 'lib/yup';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed, ok, unauthorized } from 'next-basics';\nimport { getEventMetrics } from 'queries';\nimport * as yup from 'yup';\n\nexport interface WebsiteEventsRequestQuery {\n  websiteId: string;\n  startAt: string;\n  endAt: string;\n  unit?: string;\n  timezone?: string;\n  url: string;\n}\n\nconst schema = {\n  GET: yup.object().shape({\n    websiteId: yup.string().uuid().required(),\n    startAt: yup.number().integer().required(),\n    endAt: yup.number().integer().moreThan(yup.ref('startAt')).required(),\n    unit: UnitTypeTest,\n    timezone: TimezoneTest,\n    url: yup.string(),\n  }),\n};\n\nexport default async (\n  req: NextApiRequestQueryBody<WebsiteEventsRequestQuery>,\n  res: NextApiResponse<WebsiteMetric>,\n) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  const { websiteId, timezone, url } = req.query;\n  const { startDate, endDate, unit } = await getRequestDateRange(req);\n\n  if (req.method === 'GET') {\n    if (!(await canViewWebsite(req.auth, websiteId))) {\n      return unauthorized(res);\n    }\n\n    const events = await getEventMetrics(websiteId, {\n      startDate,\n      endDate,\n      timezone,\n      unit,\n      url,\n    });\n\n    return ok(res, events);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/me/websites.ts\nimport { useAuth, useCors, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody } from 'lib/types';\nimport { pageInfo } from 'lib/schema';\nimport { NextApiResponse } from 'next';\nimport { methodNotAllowed } from 'next-basics';\nimport userWebsitesRoute from 'pages/api/users/[userId]/websites';\nimport * as yup from 'yup';\n\nconst schema = {\n  GET: yup.object().shape({\n    ...pageInfo,\n  }),\n};\n\nexport default async (req: NextApiRequestQueryBody, res: NextApiResponse) => {\n  await useCors(req, res);\n  await useAuth(req, res);\n  await useValidate(schema, req, res);\n\n  if (req.method === 'GET') {\n    req.query.userId = req.auth.user.id;\n\n    return userWebsitesRoute(req, res);\n  }\n\n  return methodNotAllowed(res);\n};\n\n// Path: src/pages/api/me/index.ts\nimport { NextApiResponse } from 'next';\nimport { useAuth } from 'lib/middleware';\nimport { NextApiRequestQueryBody, User } from 'lib/types';\nimport { ok } from 'next-basics';\n\nexport default async (\n  req: NextApiRequestQueryBody<unknown, unknown>,\n  res: NextApiResponse<User>,\n) => {\n  await useAuth(req, res);\n\n  return ok(res, req.auth.user);\n};\n\n// Path: src/pages/api/me/password.ts\nimport { useAuth, useValidate } from 'lib/middleware';\nimport { NextApiRequestQueryBody, User } from 'lib/types';\nimport { NextApiResponse } from 'next';\nimport {\n  badRequest,\n  checkPassword,\n  forbidden,\n  hashPassword,\n  methodNotAllowed,\n  ok,\n} from 'next-basics';\nimport { getUser, updateUser } from 'queries';\nimport * as yup from 'yup';\n\nexport interface UserPasswordRequestBody {\n  currentPassword: string;\n  newPassword: string;\n}\n\nconst schema = {\n  POST: yup.object().shape({\n    currentPassword: yup.string().required(),\n    newPassword: yup.string().min(8).required(),\n  }),\n};\n\nexport default async (\n...\n", "needle_token_start": 3978, "needle_token_end": 4043, "code_context_ntokens": 16035, "output": ["```typescript\nasync function checkUserBlock(userId: string) {\n  if (process.env.ENABLE_BLOCKER && (await cache.fetchUserBlock(userId))) {\n    await cache.incrementUserBlock(userId);\n\n    throw new Error('Usage Limit.');\n  }\n}\n```"]}
{"repo": "umami-software/umami", "name": "getRetention", "language": "typescript", "path": "src/queries/analytics/reports/getRetention.ts", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve retention data for a specific website over a given time period, using different database technologies based on configuration.\n2. **Input**: The function accepts a tuple containing a string identifier for the website and a filters object specifying start and end dates, with an optional timezone.\n3. **Output**: The function returns retention data, the specifics of which depend on the underlying database query results.\n4. **Procedure**: The function determines which database technology to use (relational or ClickHouse) based on configuration. It then executes a corresponding query function with the provided parameters to fetch retention data from the selected database.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/queries/admin/report.ts\nimport { Prisma, Report } from '@prisma/client';\n...\n// Path: src/queries/admin/user.ts\nimport { Prisma } from '@prisma/client';\nimport cache from 'lib/cache';\nimport { ROLES } from 'lib/constants';\nimport prisma from 'lib/prisma';\nimport { FilterResult, Role, User, UserSearchFilter } from 'lib/types';\nimport { getRandomChars } from 'next-basics';\nimport UserFindManyArgs = Prisma.UserFindManyArgs;\n\nexport interface GetUserOptions {\n  includePassword?: boolean;\n  showDeleted?: boolean;\n}\n\nasync function findUser(\n  criteria: Prisma.UserFindUniqueArgs,\n  options: GetUserOptions = {},\n): Promise<User> {\n  const { includePassword = false, showDeleted = false } = options;\n\n  return prisma.client.user.findUnique({\n    ...criteria,\n    where: {\n      ...criteria.where,\n      ...(showDeleted && { deletedAt: null }),\n    },\n    select: {\n      id: true,\n      username: true,\n      password: includePassword,\n      role: true,\n      createdAt: true,\n    },\n  });\n}\n\nexport async function getUser(userId: string, options: GetUserOptions = {}) {\n  return findUser(\n    {\n      where: {\n        id: userId,\n      },\n    },\n    options,\n  );\n}\n\nexport async function getUserByUsername(username: string, options: GetUserOptions = {}) {\n  return findUser({ where: { username } }, options);\n}\n\nexport async function getUsers(\n  criteria: UserFindManyArgs,\n  filters?: UserSearchFilter,\n): Promise<FilterResult<User[]>> {\n  const { query } = filters;\n\n  const where: Prisma.UserWhereInput = {\n    ...criteria.where,\n    ...prisma.getSearchParameters(query, [{ username: 'contains' }]),\n    deletedAt: null,\n  };\n\n  return prisma.pagedQuery(\n    'user',\n    {\n      ...criteria,\n      where,\n    },\n    {\n      orderBy: 'createdAt',\n      sortDescending: true,\n      ...filters,\n    },\n  );\n}\n\nexport async function createUser(data: {\n  id: string;\n  username: string;\n  password: string;\n  role: Role;\n}): Promise<{\n  id: string;\n  username: string;\n  role: string;\n}> {\n  return prisma.client.user.create({\n    data,\n    select: {\n      id: true,\n      username: true,\n      role: true,\n    },\n  });\n}\n\nexport async function updateUser(userId: string, data: Prisma.UserUpdateInput): Promise<User> {\n  return prisma.client.user.update({\n    where: {\n      id: userId,\n    },\n    data,\n    select: {\n      id: true,\n      username: true,\n      role: true,\n      createdAt: true,\n    },\n  });\n}\n\nexport async function deleteUser(\n  userId: string,\n): Promise<\n  [\n    Prisma.BatchPayload,\n    Prisma.BatchPayload,\n    Prisma.BatchPayload,\n    Prisma.BatchPayload,\n    Prisma.BatchPayload,\n    Prisma.BatchPayload,\n    User,\n  ]\n> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  const websites = await client.website.findMany({\n    where: { userId },\n  });\n\n  let websiteIds = [];\n\n  if (websites.length > 0) {\n    websiteIds = websites.map(a => a.id);\n  }\n\n  const teams = await client.team.findMany({\n    where: {\n      teamUser: {\n        some: {\n          userId,\n          role: ROLES.teamOwner,\n        },\n      },\n    },\n  });\n\n  const teamIds = teams.map(a => a.id);\n\n  if (cloudMode) {\n    return transaction([\n      client.website.updateMany({\n        data: {\n          deletedAt: new Date(),\n        },\n        where: { id: { in: websiteIds } },\n      }),\n      client.user.update({\n        data: {\n          username: getRandomChars(32),\n          deletedAt: new Date(),\n        },\n        where: {\n          id: userId,\n        },\n      }),\n    ]);\n  }\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId: { in: websiteIds } },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId: { in: websiteIds } },\n    }),\n    client.session.deleteMany({\n      where: { websiteId: { in: websiteIds } },\n    }),\n    client.teamUser.deleteMany({\n      where: {\n        OR: [\n          {\n            teamId: {\n              in: teamIds,\n            },\n          },\n          {\n            userId,\n          },\n        ],\n      },\n    }),\n    client.team.deleteMany({\n      where: {\n        id: {\n          in: teamIds,\n        },\n      },\n    }),\n    client.report.deleteMany({\n      where: {\n        OR: [\n          {\n            websiteId: {\n              in: websiteIds,\n            },\n          },\n          {\n            userId,\n          },\n        ],\n      },\n    }),\n    client.website.deleteMany({\n      where: { id: { in: websiteIds } },\n    }),\n    client.user.delete({\n      where: {\n        id: userId,\n      },\n    }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      const ids = websites.map(a => a.id);\n\n      for (let i = 0; i < ids.length; i++) {\n        await cache.deleteWebsite(`website:${ids[i]}`);\n      }\n    }\n\n    return data;\n  });\n}\n\n// Path: src/queries/analytics/pageviews/getPageviewStats.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { EVENT_TYPE } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters) {\n  const { timezone = 'utc', unit = 'day' } = filters;\n  const { getDateQuery, parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      ${getDateQuery('website_event.created_at', unit, timezone)} x,\n      count(*) y\n    from website_event\n      ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and event_type = {{eventType}}\n      ${filterQuery}\n    group by 1\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<{ x: string; y: number }[]> {\n  const { timezone = 'UTC', unit = 'day' } = filters;\n  const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      ${getDateStringQuery('g.t', unit)} as x, \n      g.y as y\n    from (\n      select \n        ${getDateQuery('created_at', unit, timezone)} as t,\n        count(*) as y\n      from website_event\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n        and event_type = {eventType:UInt32}\n        ${filterQuery}\n      group by t\n    ) as g\n    order by t\n    `,\n    params,\n  ).then(result => {\n    return Object.values(result).map(a => {\n      return { x: a.x, y: Number(a.y) };\n    });\n  });\n}\n\n// Path: src/queries/analytics/events/getEventUsage.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery, notImplemented } from 'lib/db';\n\nexport function getEventUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {\n  return runQuery({\n    [PRISMA]: notImplemented,\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nfunction clickhouseQuery(\n  websiteIds: string[],\n  startDate: Date,\n  endDate: Date,\n): Promise<{ websiteId: string; count: number }[]> {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select \n      website_id as websiteId,\n      count(*) as count\n    from website_event \n    where website_id in {websiteIds:Array(UUID)}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    group by website_id\n    `,\n    {\n      websiteIds,\n      startDate,\n      endDate,\n    },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { websiteId: a.websiteId, count: Number(a.count) };\n    });\n  });\n}\n\n// Path: src/queries/analytics/events/getEventMetrics.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\nimport { WebsiteEventMetric, QueryFilters } from 'lib/types';\nimport { EVENT_TYPE } from 'lib/constants';\n\nexport async function getEventMetrics(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<WebsiteEventMetric[]> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters) {\n  const { timezone = 'utc', unit = 'day' } = filters;\n  const { rawQuery, getDateQuery, parseFilters } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.customEvent,\n  });\n\n  return rawQuery(\n    `\n    select\n      event_name x,\n      ${getDateQuery('created_at', unit, timezone)} t,\n      count(*) y\n    from website_event\n    ${joinSession}\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n      and event_type = {{eventType}}\n      ${filterQuery}\n    group by 1, 2\n    order by 2\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<{ x: string; t: string; y: number }[]> {\n  const { timezone = 'UTC', unit = 'day' } = filters;\n  const { rawQuery, getDateQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.customEvent,\n  });\n\n  return rawQuery(\n    `\n    select\n      event_name x,\n      ${getDateQuery('created_at', unit, timezone)} t,\n      count(*) y\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and event_type = {eventType:UInt32}\n      ${filterQuery}\n    group by x, t\n    order by t\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: a.x, t: a.t, y: Number(a.y) };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getRealtimeData.ts\nimport { getSessions, getEvents } from 'queries/index';\nimport { EVENT_TYPE } from 'lib/constants';\n\nexport async function getRealtimeData(websiteId: string, startDate: Date) {\n  const [pageviews, sessions, events] = await Promise.all([\n    getEvents(websiteId, startDate, EVENT_TYPE.pageView),\n    getSessions(websiteId, startDate),\n    getEvents(websiteId, startDate, EVENT_TYPE.customEvent),\n  ]);\n\n  const decorate = (type: string, data: any[]) => {\n    return data.map((values: { [key: string]: any }) => ({\n      ...values,\n      __type: type,\n      timestamp: values.timestamp ? values.timestamp * 1000 : new Date(values.createdAt).getTime(),\n    }));\n  };\n\n  const set = new Set();\n  const uniques = (type: string, data: any[]) => {\n    return data.reduce((arr, values: { [key: string]: any }) => {\n      if (!set.has(values.id)) {\n        set.add(values.id);\n\n        return arr.concat({\n          ...values,\n          __type: type,\n          timestamp: values.timestamp\n            ? values.timestamp * 1000\n            : new Date(values.createdAt).getTime(),\n        });\n      }\n      return arr;\n    }, []);\n  };\n\n  return {\n    pageviews: decorate('pageview', pageviews),\n    sessions: uniques('session', sessions),\n    events: decorate('event', events),\n    timestamp: Date.now(),\n  };\n}\n\n// Path: src/queries/analytics/sessions/getSessionStats.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { EVENT_TYPE } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters) {\n  const { timezone = 'utc', unit = 'day' } = filters;\n  const { getDateQuery, parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      ${getDateQuery('website_event.created_at', unit, timezone)} x,\n      count(distinct website_event.session_id) y\n    from website_event\n      ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and event_type = {{eventType}}\n      ${filterQuery}\n    group by 1\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<{ x: string; y: number }[]> {\n  const { timezone = 'UTC', unit = 'day' } = filters;\n  const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      ${getDateStringQuery('g.t', unit)} as x, \n      g.y as y\n    from (\n      select \n        ${getDateQuery('created_at', unit, timezone)} as t,\n        count(distinct session_id) as y\n      from website_event\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n        and event_type = {eventType:UInt32}\n        ${filterQuery}\n      group by t\n    ) as g\n    order by t\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: a.x, y: Number(a.y) };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getWebsiteDateRange.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\nimport { DEFAULT_RESET_DATE } from 'lib/constants';\n\nexport async function getWebsiteDateRange(...args: [websiteId: string]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string) {\n  const { rawQuery, parseFilters } = prisma;\n  const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });\n\n  const result = await rawQuery(\n    `\n    select\n      min(created_at) as mindate,\n      max(created_at) as maxdate\n    from website_event\n    where website_id = {{websiteId::uuid}}\n      and created_at >= {{startDate}}\n    `,\n    params,\n  );\n\n  return result[0] ?? null;\n}\n\nasync function clickhouseQuery(websiteId: string) {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });\n\n  const result = await rawQuery(\n    `\n    select\n      min(created_at) as mindate,\n      max(created_at) as maxdate\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startDate:DateTime64}\n    `,\n    params,\n  );\n\n  return result[0] ?? null;\n}\n\n// Path: src/queries/admin/teamUser.ts\nimport { Prisma, TeamUser } from '@prisma/client';\nimport { uuid } from 'lib/crypto';\nimport prisma from 'lib/prisma';\nimport { FilterResult, TeamUserSearchFilter } from 'lib/types';\nimport TeamUserFindManyArgs = Prisma.TeamUserFindManyArgs;\n\nexport async function findTeamUser(criteria: Prisma.TeamUserFindUniqueArgs): Promise<TeamUser> {\n  return prisma.client.teamUser.findUnique(criteria);\n}\n\nexport async function getTeamUser(teamId: string, userId: string): Promise<TeamUser> {\n  return prisma.client.teamUser.findFirst({\n    where: {\n      teamId,\n      userId,\n    },\n  });\n}\n\nexport async function getTeamUsers(\n  criteria: TeamUserFindManyArgs,\n  filters?: TeamUserSearchFilter,\n): Promise<FilterResult<TeamUser[]>> {\n  const { query } = filters;\n\n  const where: Prisma.TeamUserWhereInput = {\n    ...criteria.where,\n    ...prisma.getSearchParameters(query, [{ user: { username: 'contains' } }]),\n  };\n\n  return prisma.pagedQuery(\n    'teamUser',\n    {\n      ...criteria,\n      where,\n    },\n    filters,\n  );\n}\n\nexport async function createTeamUser(\n  userId: string,\n  teamId: string,\n  role: string,\n): Promise<TeamUser> {\n  return prisma.client.teamUser.create({\n    data: {\n      id: uuid(),\n      userId,\n      teamId,\n      role,\n    },\n  });\n}\n\nexport async function updateTeamUser(\n  teamUserId: string,\n  data: Prisma.TeamUserUpdateInput,\n): Promise<TeamUser> {\n  return prisma.client.teamUser.update({\n    where: {\n      id: teamUserId,\n    },\n    data,\n  });\n}\n\nexport async function deleteTeamUser(teamId: string, userId: string): Promise<Prisma.BatchPayload> {\n  return prisma.client.teamUser.deleteMany({\n    where: {\n      teamId,\n      userId,\n    },\n  });\n}\n\n// Path: src/queries/analytics/reports/getRetention.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nexport \nasync function getRetention(\n  ...args: [\n    websiteId: string,\n    filters: {\n      startDate: Date;\n      endDate: Date;\n      timezone?: string;\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n): Promise<\n  {\n    date: string;\n    day: number;\n    visitors: number;\n    returnVisitors: number;\n    percentage: number;\n  }[]\n> {\n  const { startDate, endDate, timezone = 'UTC' } = filters;\n  const { getDateQuery, getDayDiffQuery, getCastColumnQuery, rawQuery } = prisma;\n  const unit = 'day';\n\n  return rawQuery(\n    `\n    WITH cohort_items AS (\n      select session_id,\n        ${getDateQuery('created_at', unit, timezone)} as cohort_date  \n      from session \n      where website_id = {{websiteId::uuid}}\n        and created_at between {{startDate}} and {{endDate}}\n    ),\n    user_activities AS (\n      select distinct\n        w.session_id,\n        ${getDayDiffQuery(\n          getDateQuery('created_at', unit, timezone),\n          'c.cohort_date',\n        )} as day_number\n      from website_event w\n      join cohort_items c\n      on w.session_id = c.session_id\n      where website_id = {{websiteId::uuid}}\n          and created_at between {{startDate}} and {{endDate}}\n      ),\n    cohort_size as (\n      select cohort_date,\n        count(*) as visitors\n      from cohort_items\n      group by 1\n      order by 1\n    ),\n    cohort_date as (\n      select\n        c.cohort_date,\n        a.day_number,\n        count(*) as visitors\n      from user_activities a\n      join cohort_items c\n      on a.session_id = c.session_id\n      group by 1, 2\n    )\n    select\n      c.cohort_date as date,\n      c.day_number as day,\n      s.visitors,\n      c.visitors as \"returnVisitors\",\n      ${getCastColumnQuery('c.visitors', 'float')} * 100 / s.visitors  as percentage\n    from cohort_date c\n    join cohort_size s\n    on c.cohort_date = s.cohort_date\n    where c.day_number <= 31\n    order by 1, 2`,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(results => {\n    return results.map(i => ({ ...i, percentage: Number(i.percentage) || 0 }));\n  });\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n): Promise<\n  {\n    date: string;\n    day: number;\n    visitors: number;\n    returnVisitors: number;\n    percentage: number;\n  }[]\n> {\n  const { startDate, endDate, timezone = 'UTC' } = filters;\n  const { getDateQuery, getDateStringQuery, rawQuery } = clickhouse;\n  const unit = 'day';\n\n  return rawQuery(\n    `\n    WITH cohort_items AS (\n      select\n        min(${getDateQuery('created_at', unit, timezone)}) as cohort_date,\n        session_id\n      from website_event\n      where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      group by session_id\n    ),\n    user_activities AS (\n      select distinct\n        w.session_id,\n        (${getDateQuery('created_at', unit, timezone)} - c.cohort_date) / 86400 as day_number\n      from website_event w\n      join cohort_items c\n      on w.session_id = c.session_id\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ),\n    cohort_size as (\n      select cohort_date,\n        count(*) as visitors\n      from cohort_items\n      group by 1\n      order by 1\n    ),\n    cohort_date as (\n      select\n        c.cohort_date,\n        a.day_number,\n        count(*) as visitors\n      from user_activities a\n      join cohort_items c\n      on a.session_id = c.session_id\n      group by 1, 2\n    )\n    select\n      ${getDateStringQuery('c.cohort_date', unit)} as date,\n      c.day_number as day,\n      s.visitors as visitors,\n      c.visitors returnVisitors,\n      c.visitors * 100 / s.visitors as percentage\n    from cohort_date c\n    join cohort_size s\n    on c.cohort_date = s.cohort_date\n    where c.day_number <= 31\n    order by 1, 2`,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        date: a.date,\n        day: Number(a.day),\n        visitors: Number(a.visitors),\n        returnVisitors: Number(a.returnVisitors),\n        percentage: Number(a.percentage),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/sessions/createSession.ts\nimport { Prisma } from '@prisma/client';\nimport cache from 'lib/cache';\nimport prisma from 'lib/prisma';\n\nexport async function createSession(data: Prisma.SessionCreateInput) {\n  const {\n    id,\n    websiteId,\n    hostname,\n    browser,\n    os,\n    device,\n    screen,\n    language,\n    country,\n    subdivision1,\n    subdivision2,\n    city,\n  } = data;\n\n  return prisma.client.session\n    .create({\n      data: {\n        id,\n        websiteId,\n        hostname,\n        browser,\n        os,\n        device,\n        screen,\n        language,\n        country,\n        subdivision1,\n        subdivision2,\n        city,\n      },\n    })\n    .then(async data => {\n      if (cache.enabled) {\n        await cache.storeSession(data);\n      }\n\n      return data;\n    });\n}\n\n// Path: src/queries/analytics/pageviews/getPageviewMetrics.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\nimport { EVENT_TYPE, SESSION_COLUMNS } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getPageviewMetrics(\n  ...args: [\n    websiteId: string,\n    column: string,\n    filters: QueryFilters,\n    limit?: number,\n    offset?: number,\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  column: string,\n  filters: QueryFilters,\n  limit: number = 500,\n  offset: number = 0,\n) {\n  const { rawQuery, parseFilters } = prisma;\n\n  const { filterQuery, joinSession, params } = await parseFilters(\n    websiteId,\n    {\n      ...filters,\n      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n    },\n    { joinSession: SESSION_COLUMNS.includes(column) },\n  );\n\n  let excludeDomain = '';\n  if (column === 'referrer_domain') {\n    excludeDomain =\n      'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)';\n  }\n\n  return rawQuery(\n    `\n    select ${column} x, count(*) y\n    from website_event\n    ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and event_type = {{eventType}}\n      ${excludeDomain}\n      ${filterQuery}\n    group by 1\n    order by 2 desc\n    limit ${limit}\n    offset ${offset}\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  column: string,\n  filters: QueryFilters,\n  limit: number = 500,\n  offset: number = 0,\n): Promise<{ x: string; y: number }[]> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n  });\n\n  let excludeDomain = '';\n  if (column === 'referrer_domain') {\n    excludeDomain = 'and referrer_domain != {websiteDomain:String}';\n  }\n\n  return rawQuery(\n    `\n    select ${column} x, count(*) y\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and event_type = {eventType:UInt32}\n      ${excludeDomain}\n      ${filterQuery}\n    group by x\n    order by y desc\n    limit ${limit}\n    offset ${offset}\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: a.x, y: Number(a.y) };\n    });\n  });\n}\n\n// Path: src/queries/analytics/eventData/getEventDataUsage.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery, notImplemented } from 'lib/db';\n\nexport function getEventDataUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {\n  return runQuery({\n    [PRISMA]: notImplemented,\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nfunction clickhouseQuery(\n  websiteIds: string[],\n  startDate: Date,\n  endDate: Date,\n): Promise<{ websiteId: string; count: number }[]> {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select \n      website_id as websiteId,\n      count(*) as count\n    from event_data \n    where created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and website_id in {websiteIds:Array(UUID)}\n    group by website_id\n    `,\n    {\n      websiteIds,\n      startDate,\n      endDate,\n    },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { websiteId: a.websiteId, count: Number(a.count) };\n    });\n  });\n}\n\n// Path: src/queries/analytics/sessions/getSessionMetrics.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\nimport { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getSessionMetrics(\n  ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  type: string,\n  filters: QueryFilters,\n  limit: number = 500,\n  offset: number = 0,\n) {\n  const column = FILTER_COLUMNS[type] || type;\n  const { parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(\n    websiteId,\n    {\n      ...filters,\n      eventType: EVENT_TYPE.pageView,\n    },\n    {\n      joinSession: SESSION_COLUMNS.includes(type),\n    },\n  );\n  const includeCountry = column === 'city' || column === 'subdivision1';\n\n  return rawQuery(\n    `\n    select \n      ${column} x,\n      count(distinct website_event.session_id) y\n      ${includeCountry ? ', country' : ''}\n    from website_event\n    ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and website_event.event_type = {{eventType}}\n    ${filterQuery}\n    group by 1 \n    ${includeCountry ? ', 3' : ''}\n    order by 2 desc\n    limit ${limit}\n    offset ${offset}\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  type: string,\n  filters: QueryFilters,\n  limit: number = 500,\n  offset: number = 0,\n): Promise<{ x: string; y: number }[]> {\n  const column = FILTER_COLUMNS[type] || type;\n  const { parseFilters, rawQuery } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n  const includeCountry = column === 'city' || column === 'subdivision1';\n\n  return rawQuery(\n    `\n    select\n      ${column} x,\n      count(distinct session_id) y\n      ${includeCountry ? ', country' : ''}\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and event_type = {eventType:UInt32}\n      ${filterQuery}\n    group by x \n    ${includeCountry ? ', country' : ''}\n    order by y desc\n    limit ${limit}\n    offset ${offset}\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: a.x, y: Number(a.y), country: a.country };\n    });\n  });\n}\n\n// Path: src/queries/analytics/eventData/getEventDataStats.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getEventDataStats(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<{\n  events: number;\n  fields: number;\n  records: number;\n}> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  }).then(results => results[0]);\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters) {\n  const { rawQuery, parseFilters } = prisma;\n  const { filterQuery, params } = await parseFilters(websiteId, filters);\n\n  return rawQuery(\n    `\n    select \n      count(distinct t.website_event_id) as \"events\",\n      count(distinct t.event_key) as \"fields\",\n      sum(t.total) as \"records\"\n    from (\n      select\n        website_event_id,\n        event_key,\n        count(*) as \"total\"\n      from event_data\n      where website_id = {{websiteId::uuid}}\n        and created_at between {{startDate}} and {{endDate}}\n      ${filterQuery}\n      group by website_event_id, event_key\n      ) as t\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<{ events: number; fields: number; records: number }[]> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, filters);\n\n  return rawQuery(\n    `\n    select \n      count(distinct t.event_id) as \"events\",\n      count(distinct t.event_key) as \"fields\",\n      sum(t.total) as \"records\"\n    from (\n      select\n        event_id,\n        event_key,\n        count(*) as \"total\"\n      from event_data\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      ${filterQuery}\n      group by event_id, event_key\n      ) as t\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        events: Number(a.events),\n        fields: Number(a.fields),\n        records: Number(a.records),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/reports/getInsights.ts\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getInsights(\n  ...args: [websiteId: string, fields: { name: string; type?: string }[], filters: QueryFilters]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  fields: { name: string; type?: string }[],\n  filters: QueryFilters,\n): Promise<\n  {\n    x: string;\n    y: number;\n  }[]\n> {\n  const { parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(\n    websiteId,\n    {\n      ...filters,\n      eventType: EVENT_TYPE.pageView,\n    },\n    {\n      joinSession: !!fields.find(({ name }) => SESSION_COLUMNS.includes(name)),\n    },\n  );\n\n  return rawQuery(\n    `\n    select \n      ${parseFields(fields)}\n    from website_event\n      ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and website_event.event_type = {{eventType}}\n      ${filterQuery}\n    ${parseGroupBy(fields)}\n    order by 1 desc, 2 desc\n    limit 500\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  fields: { name: string; type?: string }[],\n  filters: QueryFilters,\n): Promise<\n  {\n    x: string;\n    y: number;\n  }[]\n> {\n  const { parseFilters, rawQuery } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      ${parseFields(fields)}\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and event_type = {eventType:UInt32}\n      ${filterQuery}\n    ${parseGroupBy(fields)}\n    order by 1 desc, 2 desc\n    limit 500\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        ...a,\n        views: Number(a.views),\n        visitors: Number(a.visitors),\n      };\n    });\n  });\n}\n\nfunction parseFields(fields: any[]) {\n  const query = fields.reduce(\n    (arr, field) => {\n      const { name } = field;\n\n      return arr.concat(`${FILTER_COLUMNS[name]} as \"${name}\"`);\n    },\n    ['count(*) as views', 'count(distinct website_event.session_id) as visitors'],\n  );\n\n  return query.join(',\\n');\n}\n\nfunction parseGroupBy(fields: { name: any }[]) {\n  if (!fields.length) {\n    return '';\n  }\n  return `group by ${fields.map(({ name }) => FILTER_COLUMNS[name]).join(',')}`;\n}\n\n// Path: src/queries/analytics/eventData/getEventDataFields.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { QueryFilters, WebsiteEventData } from 'lib/types';\n\nexport async function getEventDataFields(\n  ...args: [websiteId: string, filters: QueryFilters & { field?: string }]\n): Promise<WebsiteEventData[]> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters & { field?: string }) {\n  const { rawQuery, parseFilters } = prisma;\n  const { filterQuery, params } = await parseFilters(websiteId, filters, {\n    columns: { field: 'event_key' },\n  });\n\n  return rawQuery(\n    `\n    select\n      event_key as \"fieldName\",\n      data_type as \"dataType\",\n      string_value as \"fieldValue\",\n      count(*) as \"total\"\n    from event_data\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n    ${filterQuery}\n    group by event_key, data_type, string_value\n    order by 3 desc, 2 desc, 1 asc\n    limit 500\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters & { field?: string },\n): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, filters, {\n    columns: { field: 'event_key' },\n  });\n\n  return rawQuery(\n    `\n    select\n      event_key as fieldName,\n      data_type as dataType,\n      string_value as fieldValue,\n      count(*) as total\n    from event_data\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ${filterQuery}\n    group by event_key, data_type, string_value\n    order by 3 desc, 2 desc, 1 asc\n    limit 500\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        fieldName: a.fieldName,\n        dataType: Number(a.dataType),\n        fieldValue: a.fieldValue,\n        total: Number(a.total),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getValues.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\n\nexport async function getValues(\n  ...args: [websiteId: string, column: string, startDate: Date, endDate: Date, search: string]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  column: string,\n  startDate: Date,\n  endDate: Date,\n  search: string,\n) {\n  const { rawQuery, getSearchQuery } = prisma;\n  let searchQuery = '';\n\n  if (search) {\n    searchQuery = getSearchQuery(column);\n  }\n\n  return rawQuery(\n    `\n    select ${column} as \"value\", count(*)\n    from website_event\n    inner join session\n      on session.session_id = website_event.session_id\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      ${searchQuery}\n    group by 1\n    order by 2 desc\n    limit 10\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      search: `%${search}%`,\n    },\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  column: string,\n  startDate: Date,\n  endDate: Date,\n  search: string,\n) {\n  const { rawQuery } = clickhouse;\n  let searchQuery = '';\n\n  if (search) {\n    searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;\n  }\n\n  return rawQuery(\n    `\n    select ${column} as value, count(*)\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      ${searchQuery}\n    group by 1\n    order by 2 desc\n    limit 10\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      search,\n    },\n  );\n}\n\n// Path: src/queries/analytics/events/getEvents.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nexport function getEvents(...args: [websiteId: string, startDate: Date, eventType: number]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nfunction relationalQuery(websiteId: string, startDate: Date, eventType: number) {\n  return prisma.client.websiteEvent.findMany({\n    where: {\n      websiteId,\n      eventType,\n      createdAt: {\n        gte: startDate,\n      },\n    },\n    orderBy: {\n      createdAt: 'asc',\n    },\n  });\n}\n\nfunction clickhouseQuery(websiteId: string, startDate: Date, eventType: number) {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select\n      event_id as id,\n      website_id as websiteId, \n      session_id as sessionId,\n      created_at as createdAt,\n      toUnixTimestamp(created_at) as timestamp,\n      url_path as urlPath,\n      referrer_domain as referrerDomain,\n      event_name as eventName\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startDate:DateTime64}\n      and event_type = {eventType:UInt32}\n    order by created_at asc\n    `,\n    {\n      websiteId,\n      startDate,\n      eventType,\n    },\n  );\n}\n\n// Path: src/queries/admin/website.ts\nimport { Prisma, Website } from '@prisma/client';\nimport cache from 'lib/cache';\nimport prisma from 'lib/prisma';\nimport { FilterResult, WebsiteSearchFilter } from 'lib/types';\nimport WebsiteFindManyArgs = Prisma.WebsiteFindManyArgs;\n\nasync function findWebsite(criteria: Prisma.WebsiteFindUniqueArgs): Promise<Website> {\n  return prisma.client.website.findUnique(criteria);\n}\n\nexport async function getWebsite(websiteId: string) {\n  return findWebsite({\n    where: {\n      id: websiteId,\n    },\n  });\n}\n\nexport async function getSharedWebsite(shareId: string) {\n  return findWebsite({\n    where: {\n      shareId,\n    },\n  });\n}\n\nexport async function getWebsites(\n  criteria: WebsiteFindManyArgs,\n  filters: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  const { query } = filters;\n\n  const where: Prisma.WebsiteWhereInput = {\n    ...criteria.where,\n    ...prisma.getSearchParameters(query, [\n      {\n        name: 'contains',\n      },\n      { domain: 'contains' },\n    ]),\n    deletedAt: null,\n  };\n\n  return prisma.pagedQuery('website', { ...criteria, where }, filters);\n}\n\nexport async function getAllWebsites(userId: string) {\n  return prisma.client.website.findMany({\n    where: {\n      userId,\n    },\n  });\n}\n\nexport async function getUserWebsites(\n  userId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        userId,\n      },\n      include: {\n        user: {\n          select: {\n            username: true,\n            id: true,\n          },\n        },\n      },\n    },\n    {\n      orderBy: 'name',\n      ...filters,\n    },\n  );\n}\n\nexport async function getTeamWebsites(\n  teamId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        teamId,\n      },\n      include: {\n        createUser: {\n          select: {\n            id: true,\n            username: true,\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createWebsite(\n  data: Prisma.WebsiteCreateInput | Prisma.WebsiteUncheckedCreateInput,\n): Promise<Website> {\n  return prisma.client.website\n    .create({\n      data,\n    })\n    .then(async data => {\n      if (cache.enabled) {\n        await cache.storeWebsite(data);\n      }\n\n      return data;\n    });\n}\n\nexport async function updateWebsite(\n  websiteId: string,\n  data: Prisma.WebsiteUpdateInput | Prisma.WebsiteUncheckedUpdateInput,\n): Promise<Website> {\n  return prisma.client.website.update({\n    where: {\n      id: websiteId,\n    },\n    data,\n  });\n}\n\nexport async function resetWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId },\n    }),\n    client.session.deleteMany({\n      where: { websiteId },\n    }),\n    client.website.update({\n      where: { id: websiteId },\n      data: {\n        resetAt: new Date(),\n      },\n    }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      await cache.storeWebsite(data[3]);\n    }\n\n    return data;\n  });\n}\n\nexport async function deleteWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId },\n    }),\n    client.session.deleteMany({\n      where: { websiteId },\n    }),\n    client.report.deleteMany({\n      where: {\n        websiteId,\n      },\n    }),\n    cloudMode\n      ? client.website.update({\n          data: {\n            deletedAt: new Date(),\n          },\n          where: { id: websiteId },\n        })\n      : client.website.delete({\n          where: { id: websiteId },\n        }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      await cache.deleteWebsite(websiteId);\n    }\n\n    return data;\n  });\n}\n\n// Path: src/queries/analytics/sessions/getSession.ts\nimport prisma from 'lib/prisma';\n\nexport async function getSession(id: string) {\n  return prisma.client.session.findUnique({\n    where: {\n      id,\n    },\n  });\n}\n\n// Path: src/lib/data.ts\nimport { DATA_TYPE, DATETIME_REGEX } from './constants';\nimport { DynamicDataType } from './types';\n\nexport function flattenJSON(\n  eventData: { [key: string]: any },\n  keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],\n  parentKey = '',\n): { key: string; value: any; dataType: DynamicDataType }[] {\n  return Object.keys(eventData).reduce(\n    (acc, key) => {\n      const value = eventData[key];\n      const type = typeof eventData[key];\n\n      // nested object\n      if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {\n        flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));\n      } else {\n        createKey(getKeyName(key, parentKey), value, acc);\n      }\n\n      return acc;\n    },\n    { keyValues, parentKey },\n  ).keyValues;\n}\n\nexport function isValidDateValue(value: string) {\n  return typeof value === 'string' && DATETIME_REGEX.test(value);\n}\n\nexport function getDataType(value: any): string {\n  let type: string = typeof value;\n\n  if (isValidDateValue(value)) {\n    type = 'date';\n  }\n\n  return type;\n}\n\nexport function getStringValue(value: string, dataType: number) {\n  if (dataType === DATA_TYPE.number) {\n    return parseFloat(value).toFixed(4);\n  }\n\n  if (dataType === DATA_TYPE.date) {\n    return new Date(value).toISOString();\n  }\n\n  return value;\n}\n\nfunction createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {\n  const type = getDataType(value);\n\n  let dataType = null;\n\n  switch (type) {\n    case 'number':\n      dataType = DATA_TYPE.number;\n      break;\n    case 'string':\n      dataType = DATA_TYPE.string;\n      break;\n    case 'boolean':\n      dataType = DATA_TYPE.boolean;\n      value = value ? 'true' : 'false';\n      break;\n    case 'date':\n      dataType = DATA_TYPE.date;\n      break;\n    case 'object':\n      dataType = DATA_TYPE.array;\n      value = JSON.stringify(value);\n      break;\n    default:\n      dataType = DATA_TYPE.string;\n      break;\n  }\n\n  acc.keyValues.push({ key, value, dataType });\n}\n\nfunction getKeyName(key: string, parentKey: string) {\n  if (!parentKey) {\n    return key;\n  }\n\n  return `${parentKey}.${key}`;\n}\n\n// Path: src/queries/analytics/sessions/saveSessionData.ts\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveSessionData(data: {\n  websiteId: string;\n  sessionId: string;\n  sessionData: DynamicData;\n}) {\n  const { client, transaction } = prisma;\n  const { websiteId, sessionId, sessionData } = data;\n\n  const jsonKeys = flattenJSON(sessionData);\n\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteId,\n    sessionId,\n    key: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return transaction([\n    client.sessionData.deleteMany({\n      where: {\n        sessionId,\n      },\n    }),\n    client.sessionData.createMany({\n      data: flattenedData as any,\n    }),\n  ]);\n}\n\n// Path: src/queries/analytics/reports/getUTM.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nexport async function getUTM(\n  ...args: [\n    websiteId: string,\n    filters: {\n      startDate: Date;\n      endDate: Date;\n      timezone?: string;\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = prisma;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n      and coalesce(url_query, '') != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and url_query != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nfunction parseParameters(data: any[]) {\n  return data.reduce((obj, { url_query, num }) => {\n    try {\n      const searchParams = new URLSearchParams(url_query);\n\n      for (const [key, value] of searchParams) {\n        if (key.match(/^utm_(\\w+)$/)) {\n          const name = safeDecodeURIComponent(value);\n          if (!obj[key]) {\n            obj[key] = { [name]: Number(num) };\n          } else if (!obj[key][name]) {\n            obj[key][name] = Number(num);\n          } else {\n            obj[key][name] += Number(num);\n...\n", "needle_token_start": 5605, "needle_token_end": 5701, "code_context_ntokens": 16170, "output": ["```typescript\n// Path: src/queries/analytics/reports/getRetention.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nexport async function getRetention(\n  ...args: [\n    websiteId: string,\n    filters: {\n      startDate: Date;\n      endDate: Date;\n      timezone?: string;\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n): Promise<\n  {\n    date: string;\n    day: number;\n    visitors: number;\n    returnVisitors: number;\n    percentage: number;\n  }[]\n> {\n  const { startDate, endDate, timezone = 'UTC' } = filters;\n  const { getDateQuery, getDayDiffQuery, getCastColumnQuery, rawQuery } = prisma;\n  const unit = 'day';\n\n  return rawQuery(\n    `\n    WITH cohort_items AS (\n      select session_id,\n        ${getDateQuery('created_at', unit, timezone)} as cohort_date  \n      from session \n      where website_id = {{websiteId::uuid}}\n        and created_at between {{startDate}} and {{endDate}}\n    ),\n    user_activities AS (\n      select distinct\n        w.session_id,\n        ${getDayDiffQuery(\n          getDateQuery('created_at', unit, timezone),\n          'c.cohort_date',\n        )} as day_number\n      from website_event w\n      join cohort_items c\n      on w.session_id = c.session_id\n      where website_id = {{websiteId::uuid}}\n          and created_at between {{startDate}} and {{endDate}}\n      ),\n    cohort_size as (\n      select cohort_date,\n        count(*) as visitors\n      from cohort_items\n      group by 1\n      order by 1\n    ),\n    cohort_date as (\n      select\n        c.cohort_date,\n        a.day_number,\n        count(*) as visitors\n      from user_activities a\n      join cohort_items c\n      on a.session_id = c.session_id\n      group by 1, 2\n    )\n    select\n      c.cohort_date as date,\n      c.day_number as day,\n      s.visitors,\n      c.visitors as \"returnVisitors\",\n      ${getCastColumnQuery('c.visitors', 'float')} * 100 / s.visitors  as percentage\n    from cohort_date c\n    join cohort_size s\n    on c.cohort_date = s.cohort_date\n    where c.day_number <= 31\n    order by 1, 2`,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(results => {\n    return results.map(i => ({ ...i, percentage: Number(i.percentage) || 0 }));\n  });\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n): Promise<\n  {\n    date: string;\n    day: number;\n    visitors: number;\n    returnVisitors: number;\n    percentage: number;\n  }[]\n> {\n  const { startDate, endDate, timezone = 'UTC' } = filters;\n  const { getDateQuery, getDateStringQuery, rawQuery } = clickhouse;\n  const unit = 'day';\n\n  return rawQuery(\n    `\n    WITH cohort_items AS (\n      select\n        min(${getDateQuery('created_at', unit, timezone)}) as cohort_date,\n        session_id\n      from website_event\n      where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      group by session_id\n    ),\n    user_activities AS (\n      select distinct\n        w.session_id,\n        (${getDateQuery('created_at', unit, timezone)} - c.cohort_date) / 86400 as day_number\n      from website_event w\n      join cohort_items c\n      on w.session_id = c.session_id\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ),\n    cohort_size as (\n      select cohort_date,\n        count(*) as visitors\n      from cohort_items\n      group by 1\n      order by 1\n    ),\n    cohort_date as (\n      select\n        c.cohort_date,\n        a.day_number,\n        count(*) as visitors\n      from user_activities a\n      join cohort_items c\n      on a.session_id = c.session_id\n      group by 1,"]}
{"repo": "umami-software/umami", "name": "getAllowedUnits", "language": "typescript", "path": "src/lib/date.ts", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function determines which time units are appropriate for use based on the difference between two provided dates.\n2. **Input**: Two date objects representing the start and end dates.\n3. **Output**: An array of strings, each representing a time unit that is suitable given the span between the two dates.\n4. **Procedure**: The function first identifies the smallest relevant time unit for the given date range. It then filters out finer units that are not suitable for representing the span between the two dates, returning only those units that are appropriate.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/lib/constants.ts\n/* eslint-disable no-unused-vars */\nexport const CURRENT_VERSION = process.env.currentVersion;\nexport const AUTH_TOKEN = 'umami.auth';\nexport const LOCALE_CONFIG = 'umami.locale';\nexport const TIMEZONE_CONFIG = 'umami.timezone';\nexport const DATE_RANGE_CONFIG = 'umami.date-range';\nexport const THEME_CONFIG = 'umami.theme';\nexport const DASHBOARD_CONFIG = 'umami.dashboard';\nexport const VERSION_CHECK = 'umami.version-check';\nexport const SHARE_TOKEN_HEADER = 'x-umami-share-token';\nexport const HOMEPAGE_URL = 'https://umami.is';\nexport const REPO_URL = 'https://github.com/umami-software/umami';\nexport const UPDATES_URL = 'https://api.umami.is/v1/updates';\nexport const TELEMETRY_PIXEL = 'https://i.umami.is/a.png';\n\nexport const DEFAULT_LOCALE = process.env.defaultLocale || 'en-US';\nexport const DEFAULT_THEME = 'light';\nexport const DEFAULT_ANIMATION_DURATION = 300;\nexport const DEFAULT_DATE_RANGE = '24hour';\nexport const DEFAULT_WEBSITE_LIMIT = 10;\nexport const DEFAULT_RESET_DATE = '2000-01-01';\nexport const DEFAULT_PAGE_SIZE = 10;\n\nexport const REALTIME_RANGE = 30;\nexport const REALTIME_INTERVAL = 5000;\n\nexport const FILTER_COMBINED = 'filter-combined';\nexport const FILTER_RAW = 'filter-raw';\nexport const FILTER_DAY = 'filter-day';\nexport const FILTER_RANGE = 'filter-range';\nexport const FILTER_REFERRERS = 'filter-referrers';\nexport const FILTER_PAGES = 'filter-pages';\nexport const UNIT_TYPES = ['year', 'month', 'hour', 'day'];\nexport const EVENT_COLUMNS = ['url', 'referrer', 'title', 'query', 'event'];\n\nexport const SESSION_COLUMNS = [\n  'browser',\n  'os',\n  'device',\n  'screen',\n  'language',\n  'country',\n  'region',\n  'city',\n];\n\nexport const FILTER_COLUMNS = {\n  url: 'url_path',\n  referrer: 'referrer_domain',\n  title: 'page_title',\n  query: 'url_query',\n  os: 'os',\n  browser: 'browser',\n  device: 'device',\n  country: 'country',\n  region: 'subdivision1',\n  city: 'city',\n  language: 'language',\n  event: 'event_name',\n};\n\nexport const COLLECTION_TYPE = {\n  event: 'event',\n  identify: 'identify',\n};\n\nexport const EVENT_TYPE = {\n  pageView: 1,\n  customEvent: 2,\n} as const;\n\nexport const DATA_TYPE = {\n  string: 1,\n  number: 2,\n  boolean: 3,\n  date: 4,\n  array: 5,\n} as const;\n\nexport const OPERATORS = {\n  equals: 'eq',\n  notEquals: 'neq',\n  set: 's',\n  notSet: 'ns',\n  contains: 'c',\n  doesNotContain: 'dnc',\n  true: 't',\n  false: 'f',\n  greaterThan: 'gt',\n  lessThan: 'lt',\n  greaterThanEquals: 'gte',\n  lessThanEquals: 'lte',\n  before: 'bf',\n  after: 'af',\n} as const;\n\nexport const OPERATOR_PREFIXES = {\n  [OPERATORS.equals]: '',\n  [OPERATORS.notEquals]: '!',\n  [OPERATORS.contains]: '~',\n  [OPERATORS.doesNotContain]: '!~',\n};\n\nexport const DATA_TYPES = {\n  [DATA_TYPE.string]: 'string',\n  [DATA_TYPE.number]: 'number',\n  [DATA_TYPE.boolean]: 'boolean',\n  [DATA_TYPE.date]: 'date',\n  [DATA_TYPE.array]: 'array',\n};\n\nexport const REPORT_TYPES = {\n  funnel: 'funnel',\n  insights: 'insights',\n  retention: 'retention',\n  utm: 'utm',\n} as const;\n\nexport const REPORT_PARAMETERS = {\n  fields: 'fields',\n  filters: 'filters',\n  groups: 'groups',\n} as const;\n\nexport const KAFKA_TOPIC = {\n  event: 'event',\n  eventData: 'event_data',\n} as const;\n\nexport const ROLES = {\n  admin: 'admin',\n  user: 'user',\n  viewOnly: 'view-only',\n  teamOwner: 'team-owner',\n  teamMember: 'team-member',\n  teamViewOnly: 'team-view-only',\n} as const;\n\nexport const PERMISSIONS = {\n  all: 'all',\n  websiteCreate: 'website:create',\n  websiteUpdate: 'website:update',\n  websiteDelete: 'website:delete',\n  teamCreate: 'team:create',\n  teamUpdate: 'team:update',\n  teamDelete: 'team:delete',\n} as const;\n\nexport const ROLE_PERMISSIONS = {\n  [ROLES.admin]: [PERMISSIONS.all],\n  [ROLES.user]: [\n    PERMISSIONS.websiteCreate,\n    PERMISSIONS.websiteUpdate,\n    PERMISSIONS.websiteDelete,\n    PERMISSIONS.teamCreate,\n  ],\n  [ROLES.viewOnly]: [],\n  [ROLES.teamOwner]: [\n    PERMISSIONS.teamUpdate,\n    PERMISSIONS.teamDelete,\n    PERMISSIONS.websiteCreate,\n    PERMISSIONS.websiteUpdate,\n    PERMISSIONS.websiteDelete,\n  ],\n  [ROLES.teamMember]: [\n    PERMISSIONS.websiteCreate,\n    PERMISSIONS.websiteUpdate,\n    PERMISSIONS.websiteDelete,\n  ],\n  [ROLES.teamViewOnly]: [],\n} as const;\n\nexport const THEME_COLORS = {\n  light: {\n    primary: '#2680eb',\n    gray50: '#ffffff',\n    gray75: '#fafafa',\n    gray100: '#f5f5f5',\n    gray200: '#eaeaea',\n    gray300: '#e1e1e1',\n    gray400: '#cacaca',\n    gray500: '#b3b3b3',\n    gray600: '#8e8e8e',\n    gray700: '#6e6e6e',\n    gray800: '#4b4b4b',\n    gray900: '#2c2c2c',\n  },\n  dark: {\n    primary: '#2680eb',\n    gray50: '#252525',\n    gray75: '#2f2f2f',\n    gray100: '#323232',\n    gray200: '#3e3e3e',\n    gray300: '#4a4a4a',\n    gray400: '#5a5a5a',\n    gray500: '#6e6e6e',\n    gray600: '#909090',\n    gray700: '#b9b9b9',\n    gray800: '#e3e3e3',\n    gray900: '#ffffff',\n  },\n};\n\nexport const CHART_COLORS = [\n  '#2680eb',\n  '#9256d9',\n  '#44b556',\n  '#e68619',\n  '#e34850',\n  '#f7bd12',\n  '#01bad7',\n  '#6734bc',\n  '#89c541',\n  '#ffc301',\n  '#ec1562',\n  '#ffec16',\n];\n\nexport const DOMAIN_REGEX =\n  /^(localhost(:[1-9]\\d{0,4})?|((?=[a-z0-9-]{1,63}\\.)(xn--)?[a-z0-9-]+(-[a-z0-9-]+)*\\.)+(xn--)?[a-z0-9-]{2,63})$/;\nexport const SHARE_ID_REGEX = /^[a-zA-Z0-9]{8,16}$/;\nexport const UUID_REGEX =\n...\n// Path: src/lib/client.ts\nimport { getItem, setItem, removeItem } from 'next-basics';\nimport { AUTH_TOKEN } from './constants';\n\nexport function getClientAuthToken() {\n  return getItem(AUTH_TOKEN);\n}\n\nexport function setClientAuthToken(token: string) {\n  setItem(AUTH_TOKEN, token);\n}\n\nexport function removeClientAuthToken() {\n  removeItem(AUTH_TOKEN);\n}\n\n// Path: src/lib/lang.ts\nimport {\n  arSA,\n  be,\n  bn,\n  cs,\n  sk,\n  da,\n  de,\n  el,\n  enUS,\n  enGB,\n  es,\n  fi,\n  fr,\n  faIR,\n  he,\n  hi,\n  hr,\n  id,\n  it,\n  ja,\n  km,\n  ko,\n  lt,\n  mn,\n  ms,\n  nb,\n  nl,\n  pl,\n  pt,\n  ptBR,\n  ro,\n  ru,\n  sl,\n  sv,\n  ta,\n  th,\n  tr,\n  uk,\n  zhCN,\n  zhTW,\n  ca,\n  hu,\n  vi,\n} from 'date-fns/locale';\n\nexport const languages = {\n  'ar-SA': { label: '\u0627\u0644\u0639\u0631\u0628\u064a\u0629', dateLocale: arSA, dir: 'rtl' },\n  'be-BY': { label: '\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f', dateLocale: be },\n  'bn-BD': { label: '\u09ac\u09be\u0982\u09b2\u09be', dateLocale: bn },\n  'ca-ES': { label: 'Catal\u00e0', dateLocale: ca },\n  'cs-CZ': { label: '\u010ce\u0161tina', dateLocale: cs },\n  'da-DK': { label: 'Dansk', dateLocale: da },\n  'de-CH': { label: 'Schwiizerd\u00fctsch', dateLocale: de },\n  'de-DE': { label: 'Deutsch', dateLocale: de },\n  'el-GR': { label: '\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac', dateLocale: el },\n  'en-GB': { label: 'English (UK)', dateLocale: enGB },\n  'en-US': { label: 'English (US)', dateLocale: enUS },\n  'es-ES': { label: 'Espa\u00f1ol', dateLocale: es },\n  'fa-IR': { label: '\u0641\u0627\u0631\u0633\u06cc', dateLocale: faIR, dir: 'rtl' },\n  'fi-FI': { label: 'Suomi', dateLocale: fi },\n  'fo-FO': { label: 'F\u00f8royskt' },\n  'fr-FR': { label: 'Fran\u00e7ais', dateLocale: fr },\n  'ga-ES': { label: 'Galacian (Spain)', dateLocale: es },\n  'he-IL': { label: '\u05e2\u05d1\u05e8\u05d9\u05ea', dateLocale: he },\n  'hi-IN': { label: '\u0939\u093f\u0928\u094d\u0926\u0940', dateLocale: hi },\n  'hr-HR': { label: 'Hrvatski', dateLocale: hr },\n  'hu-HU': { label: 'Hungarian', dateLocale: hu },\n  'id-ID': { label: 'Bahasa Indonesia', dateLocale: id },\n  'it-IT': { label: 'Italiano', dateLocale: it },\n  'ja-JP': { label: '\u65e5\u672c\u8a9e', dateLocale: ja },\n  'km-KH': { label: '\u1797\u17b6\u179f\u17b6\u1781\u17d2\u1798\u17c2\u179a', dateLocale: km },\n  'ko-KR': { label: '\ud55c\uad6d\uc5b4', dateLocale: ko },\n  'lt-LT': { label: 'Lietuvi\u0173', dateLocale: lt },\n  'mn-MN': { label: '\u041c\u043e\u043d\u0433\u043e\u043b', dateLocale: mn },\n  'ms-MY': { label: 'Malay', dateLocale: ms },\n  'my-MM': { label: '\u1019\u103c\u1014\u103a\u1019\u102c\u1018\u102c\u101e\u102c', dateLocale: enUS },\n  'nl-NL': { label: 'Nederlands', dateLocale: nl },\n  'nb-NO': { label: 'Norsk Bokm\u00e5l', dateLocale: nb },\n  'pl-PL': { label: 'Polski', dateLocale: pl },\n  'pt-BR': { label: 'Portugu\u00eas do Brasil', dateLocale: ptBR },\n  'pt-PT': { label: 'Portugu\u00eas', dateLocale: pt },\n  'ro-RO': { label: 'Rom\u00e2n\u0103', dateLocale: ro },\n  'ru-RU': { label: '\u0420\u0443\u0441\u0441\u043a\u0438\u0439', dateLocale: ru },\n  'si-LK': { label: '\u0dc3\u0dd2\u0d82\u0dc4\u0dbd', dateLocale: id },\n  'sk-SK': { label: 'Sloven\u010dina', dateLocale: sk },\n  'sl-SI': { label: 'Sloven\u0161\u010dina', dateLocale: sl },\n  'sv-SE': { label: 'Svenska', dateLocale: sv },\n  'ta-IN': { label: '\u0ba4\u0bae\u0bbf\u0bb4\u0bcd', dateLocale: ta },\n  'th-TH': { label: '\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22', dateLocale: th },\n  'tr-TR': { label: 'T\u00fcrk\u00e7e', dateLocale: tr },\n  'uk-UA': { label: '\u0443\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430', dateLocale: uk },\n  'ur-PK': { label: 'Urdu (Pakistan)', dateLocale: uk, dir: 'rtl' },\n  'vi-VN': { label: 'Ti\u1ebfng Vi\u1ec7t', dateLocale: vi },\n  'zh-CN': { label: '\u4e2d\u6587', dateLocale: zhCN },\n  'zh-TW': { label: '\u4e2d\u6587(\u7e41\u9ad4)', dateLocale: zhTW },\n};\n\nexport function getDateLocale(locale: string) {\n  return languages[locale]?.dateLocale || enUS;\n}\n\nexport function getTextDirection(locale: string) {\n  return languages[locale]?.dir || 'ltr';\n}\n\n// Path: src/lib/types.ts\nimport { NextApiRequest } from 'next';\nimport {\n  COLLECTION_TYPE,\n  DATA_TYPE,\n  EVENT_TYPE,\n  KAFKA_TOPIC,\n  PERMISSIONS,\n  REPORT_TYPES,\n  ROLES,\n} from './constants';\nimport * as yup from 'yup';\nimport { TIME_UNIT } from './date';\nimport { Dispatch, SetStateAction } from 'react';\n\ntype ObjectValues<T> = T[keyof T];\n\nexport type TimeUnit = ObjectValues<typeof TIME_UNIT>;\nexport type Permission = ObjectValues<typeof PERMISSIONS>;\n\nexport type CollectionType = ObjectValues<typeof COLLECTION_TYPE>;\nexport type Role = ObjectValues<typeof ROLES>;\nexport type EventType = ObjectValues<typeof EVENT_TYPE>;\nexport type DynamicDataType = ObjectValues<typeof DATA_TYPE>;\nexport type KafkaTopic = ObjectValues<typeof KAFKA_TOPIC>;\nexport type ReportType = ObjectValues<typeof REPORT_TYPES>;\n\nexport interface WebsiteSearchFilter extends SearchFilter {\n  userId?: string;\n  teamId?: string;\n  includeTeams?: boolean;\n  onlyTeams?: boolean;\n}\n\nexport interface UserSearchFilter extends SearchFilter {\n  teamId?: string;\n}\n\nexport interface TeamSearchFilter extends SearchFilter {\n  userId?: string;\n}\n\nexport interface TeamUserSearchFilter extends SearchFilter {\n  teamId?: string;\n}\n\nexport interface ReportSearchFilter extends SearchFilter {\n  userId?: string;\n  websiteId?: string;\n}\n\nexport interface SearchFilter {\n  query?: string;\n  page?: number;\n  pageSize?: number;\n  orderBy?: string;\n  sortDescending?: boolean;\n}\n\nexport interface FilterResult<T> {\n  data: T;\n  count: number;\n  page: number;\n  pageSize: number;\n  orderBy?: string;\n  sortDescending?: boolean;\n}\n\nexport interface FilterQueryResult<T> {\n  result: FilterResult<T>;\n  query: any;\n  params: SearchFilter;\n  setParams: Dispatch<SetStateAction<T | SearchFilter>>;\n}\n\nexport interface DynamicData {\n  [key: string]: number | string | DynamicData | number[] | string[] | DynamicData[];\n}\n\nexport interface Auth {\n  user?: {\n    id: string;\n    username: string;\n    role: string;\n    isAdmin: boolean;\n  };\n  grant?: Permission[];\n  shareToken?: {\n    websiteId: string;\n  };\n}\n\nexport interface YupRequest {\n  GET?: yup.ObjectSchema<any>;\n  POST?: yup.ObjectSchema<any>;\n  PUT?: yup.ObjectSchema<any>;\n  DELETE?: yup.ObjectSchema<any>;\n}\n\nexport interface NextApiRequestQueryBody<TQuery = any, TBody = any> extends NextApiRequest {\n  auth?: Auth;\n  query: TQuery & { [key: string]: string | string[] };\n  body: TBody;\n  headers: any;\n  yup: YupRequest;\n}\n\nexport interface NextApiRequestAuth extends NextApiRequest {\n  auth?: Auth;\n  headers: any;\n}\n\nexport interface User {\n  id: string;\n  username: string;\n  password?: string;\n  role: string;\n  createdAt?: Date;\n}\n\nexport interface Website {\n  id: string;\n  userId: string;\n  resetAt: Date;\n  name: string;\n  domain: string;\n  shareId: string;\n  createdAt: Date;\n}\n\nexport interface Share {\n  id: string;\n  token: string;\n}\n\nexport interface WebsiteActive {\n  x: number;\n}\n\nexport interface WebsiteMetric {\n  x: string;\n  y: number;\n}\n\nexport interface WebsiteEventMetric {\n  x: string;\n  t: string;\n  y: number;\n}\n\nexport interface WebsiteEventData {\n  eventName?: string;\n  fieldName: string;\n  dataType: number;\n  fieldValue?: string;\n  total: number;\n}\n\nexport interface WebsitePageviews {\n  pageviews: {\n    t: string;\n    y: number;\n  };\n  sessions: {\n    t: string;\n    y: number;\n  };\n}\n\nexport interface WebsiteStats {\n  pageviews: { value: number; change: number };\n  uniques: { value: number; change: number };\n  bounces: { value: number; change: number };\n  totalTime: { value: number; change: number };\n}\n\nexport interface RealtimeInit {\n  websites: Website[];\n  token: string;\n  data: RealtimeUpdate;\n}\n\nexport interface RealtimeUpdate {\n  pageviews: any[];\n  sessions: any[];\n  events: any[];\n  timestamp: number;\n}\n\nexport interface DateRange {\n  value: string;\n  startDate: Date;\n  endDate: Date;\n  unit?: TimeUnit;\n  num?: number;\n  offset?: number;\n}\n\nexport interface QueryFilters {\n  startDate?: Date;\n  endDate?: Date;\n  timezone?: string;\n  unit?: string;\n  eventType?: number;\n  url?: string;\n  referrer?: string;\n  title?: string;\n  query?: string;\n  os?: string;\n  browser?: string;\n  device?: string;\n  country?: string;\n  region?: string;\n  city?: string;\n  language?: string;\n  event?: string;\n  search?: string;\n}\n\nexport interface QueryOptions {\n  joinSession?: boolean;\n  columns?: { [key: string]: string };\n  limit?: number;\n}\n\nexport interface RealtimeData {\n  pageviews: any[];\n  sessions: any[];\n  events: any[];\n  timestamp: number;\n  countries?: any[];\n  visitors?: any[];\n}\n\n// Path: src/lib/date.ts\nimport moment from 'moment-timezone';\nimport {\n  addMinutes,\n  addHours,\n  addDays,\n  addMonths,\n  addYears,\n  subMinutes,\n  subHours,\n  subDays,\n  subMonths,\n  subYears,\n  startOfMinute,\n  startOfHour,\n  startOfDay,\n  startOfWeek,\n  startOfMonth,\n  startOfYear,\n  endOfHour,\n  endOfDay,\n  endOfWeek,\n  endOfMonth,\n  endOfYear,\n  differenceInMinutes,\n  differenceInHours,\n  differenceInCalendarDays,\n  differenceInCalendarWeeks,\n  differenceInCalendarMonths,\n  differenceInCalendarYears,\n  format,\n  max,\n  min,\n  isDate,\n  addWeeks,\n  subWeeks,\n  endOfMinute,\n} from 'date-fns';\nimport { getDateLocale } from 'lib/lang';\nimport { DateRange } from 'lib/types';\n\nexport const TIME_UNIT = {\n  minute: 'minute',\n  hour: 'hour',\n  day: 'day',\n  week: 'week',\n  month: 'month',\n  year: 'year',\n};\n\nexport const CUSTOM_FORMATS = {\n  'en-US': {\n    p: 'ha',\n    pp: 'h:mm:ss',\n  },\n  'fr-FR': {\n    'M/d': 'd/M',\n    'MMM d': 'd MMM',\n    'EEE M/d': 'EEE d/M',\n  },\n};\n\nconst DATE_FUNCTIONS = {\n  minute: {\n    diff: differenceInMinutes,\n    add: addMinutes,\n    sub: subMinutes,\n    start: startOfMinute,\n    end: endOfMinute,\n  },\n  hour: {\n    diff: differenceInHours,\n    add: addHours,\n    sub: subHours,\n    start: startOfHour,\n    end: endOfHour,\n  },\n  day: {\n    diff: differenceInCalendarDays,\n    add: addDays,\n    sub: subDays,\n    start: startOfDay,\n    end: endOfDay,\n  },\n  week: {\n    diff: differenceInCalendarWeeks,\n    add: addWeeks,\n    sub: subWeeks,\n    start: startOfWeek,\n    end: endOfWeek,\n  },\n  month: {\n    diff: differenceInCalendarMonths,\n    add: addMonths,\n    sub: subMonths,\n    start: startOfMonth,\n    end: endOfMonth,\n  },\n  year: {\n    diff: differenceInCalendarYears,\n    add: addYears,\n    sub: subYears,\n    start: startOfYear,\n    end: endOfYear,\n  },\n};\n\nexport function getTimezone() {\n  return moment.tz.guess();\n}\n\nexport function parseDateValue(value: string) {\n  const match = value.match?.(/^(?<num>[0-9-]+)(?<unit>hour|day|week|month|year)$/);\n\n  if (!match) return null;\n\n  const { num, unit } = match.groups;\n\n  return { num: +num, unit };\n}\n\nexport function parseDateRange(value: string | object, locale = 'en-US'): DateRange {\n  if (typeof value === 'object') {\n    return value as DateRange;\n  }\n\n  if (value === 'all') {\n    return {\n      startDate: new Date(0),\n      endDate: new Date(1),\n      value,\n    };\n  }\n\n  if (value?.startsWith?.('range')) {\n    const [, startTime, endTime] = value.split(':');\n\n    const startDate = new Date(+startTime);\n    const endDate = new Date(+endTime);\n\n    return {\n      startDate,\n      endDate,\n      value,\n      ...parseDateValue(value),\n      offset: 0,\n      unit: getMinimumUnit(startDate, endDate),\n    };\n  }\n\n  const now = new Date();\n  const dateLocale = getDateLocale(locale);\n  const { num, unit } = parseDateValue(value);\n\n  switch (unit) {\n    case 'hour':\n      return {\n        startDate: subHours(startOfHour(now), num),\n        endDate: endOfHour(now),\n        offset: 0,\n        num: num || 1,\n        unit,\n        value,\n      };\n    case 'day':\n      return {\n        startDate: subDays(startOfDay(now), num),\n        endDate: subDays(endOfDay(now), num ? 1 : 0),\n        unit: num ? 'day' : 'hour',\n        offset: 0,\n        num: num || 1,\n        value,\n      };\n    case 'week':\n      return {\n        startDate: subWeeks(startOfWeek(now, { locale: dateLocale }), num),\n        endDate: subWeeks(endOfWeek(now, { locale: dateLocale }), num),\n        unit: 'day',\n        offset: 0,\n        num: num || 1,\n        value,\n      };\n    case 'month':\n      return {\n        startDate: subMonths(startOfMonth(now), num),\n        endDate: subMonths(endOfMonth(now), num ? 1 : 0),\n        unit: num ? 'month' : 'day',\n        offset: 0,\n        num: num || 1,\n        value,\n      };\n    case 'year':\n      return {\n        startDate: subYears(startOfYear(now), num),\n        endDate: subYears(endOfYear(now), num),\n        unit: 'month',\n        offset: 0,\n        num: num || 1,\n        value,\n      };\n  }\n}\n\nexport function getOffsetDateRange(dateRange: DateRange, increment: number) {\n  const { startDate, endDate, unit, num, offset, value } = dateRange;\n\n  const change = num * increment;\n  const { add } = DATE_FUNCTIONS[unit];\n  const { unit: originalUnit } = parseDateValue(value) || {};\n\n  switch (originalUnit) {\n    case 'day':\n      return {\n        ...dateRange,\n        startDate: addDays(startDate, change),\n        endDate: addDays(endDate, change),\n        offset: offset + increment,\n      };\n    case 'week':\n      return {\n        ...dateRange,\n        startDate: addWeeks(startDate, change),\n        endDate: addWeeks(endDate, change),\n        offset: offset + increment,\n      };\n    case 'month':\n      return {\n        ...dateRange,\n        startDate: addMonths(startDate, change),\n        endDate: addMonths(endDate, change),\n        offset: offset + increment,\n      };\n    case 'year':\n      return {\n        ...dateRange,\n        startDate: addYears(startDate, change),\n        endDate: addYears(endDate, change),\n        offset: offset + increment,\n      };\n    default:\n      return {\n        startDate: add(startDate, change),\n        endDate: add(endDate, change),\n        value,\n        unit,\n        num,\n        offset: offset + increment,\n      };\n  }\n}\n\nexport \nfunction getAllowedUnits(startDate: Date, endDate: Date) {\n  const units = ['minute', 'hour', 'day', 'month', 'year'];\n  const minUnit = getMinimumUnit(startDate, endDate);\n  const index = units.indexOf(minUnit === 'year' ? 'month' : minUnit);\n\n  return index >= 0 ? units.splice(index) : [];\n}\n\nexport function getMinimumUnit(startDate: number | Date, endDate: number | Date) {\n  if (differenceInMinutes(endDate, startDate) <= 60) {\n    return 'minute';\n  } else if (differenceInHours(endDate, startDate) <= 48) {\n    return 'hour';\n  } else if (differenceInCalendarMonths(endDate, startDate) <= 6) {\n    return 'day';\n  } else if (differenceInCalendarMonths(endDate, startDate) <= 24) {\n    return 'month';\n  }\n\n  return 'year';\n}\n\nexport function getDateFromString(str: string) {\n  const [ymd, hms] = str.split(' ');\n  const [year, month, day] = ymd.split('-');\n\n  if (hms) {\n    const [hour, min, sec] = hms.split(':');\n\n    return new Date(+year, +month - 1, +day, +hour, +min, +sec);\n  }\n\n  return new Date(+year, +month - 1, +day);\n}\n\nexport function getDateArray(data: any[], startDate: Date, endDate: Date, unit: string) {\n  const arr = [];\n  const { diff, add, start } = DATE_FUNCTIONS[unit];\n  const n = diff(endDate, startDate) + 1;\n\n  function findData(date: Date) {\n    const d = data.find(({ x }) => {\n      return start(getDateFromString(x)).getTime() === date.getTime();\n    });\n\n    return d?.y || 0;\n  }\n\n  for (let i = 0; i < n; i++) {\n    const t = start(add(startDate, i));\n    const y = findData(t);\n\n    arr.push({ x: t, y });\n  }\n\n  return arr;\n}\n\nexport function formatDate(date: string | number | Date, str: string, locale = 'en-US') {\n  return format(\n    typeof date === 'string' ? new Date(date) : date,\n    CUSTOM_FORMATS?.[locale]?.[str] || str,\n    {\n      locale: getDateLocale(locale),\n    },\n  );\n}\n\nexport function maxDate(...args: Date[]) {\n  return max(args.filter(n => isDate(n)));\n}\n\nexport function minDate(...args: any[]) {\n  return min(args.filter(n => isDate(n)));\n}\n\nexport function getLocalTime(t: string | number | Date) {\n  return addMinutes(new Date(t), new Date().getTimezoneOffset());\n}\n\nexport function getDateLength(startDate: Date, endDate: Date, unit: string | number) {\n  const { diff } = DATE_FUNCTIONS[unit];\n  return diff(endDate, startDate) + 1;\n}\n\n// Path: src/store/app.ts\nimport { create } from 'zustand';\nimport {\n  DATE_RANGE_CONFIG,\n  DEFAULT_DATE_RANGE,\n  DEFAULT_LOCALE,\n  DEFAULT_THEME,\n  LOCALE_CONFIG,\n  THEME_CONFIG,\n  TIMEZONE_CONFIG,\n} from 'lib/constants';\nimport { getItem } from 'next-basics';\nimport { getTimezone } from 'lib/date';\n\nfunction getDefaultTheme() {\n  return typeof window !== 'undefined'\n    ? window?.matchMedia('(prefers-color-scheme: dark)')?.matches\n      ? 'dark'\n      : 'light'\n    : 'light';\n}\n\nconst initialState = {\n  locale: getItem(LOCALE_CONFIG) || DEFAULT_LOCALE,\n  theme: getItem(THEME_CONFIG) || getDefaultTheme() || DEFAULT_THEME,\n  timezone: getItem(TIMEZONE_CONFIG) || getTimezone(),\n  dateRange: getItem(DATE_RANGE_CONFIG) || DEFAULT_DATE_RANGE,\n  shareToken: null,\n  user: null,\n  config: null,\n};\n\nconst store = create(() => ({ ...initialState }));\n\nexport function setTheme(theme: string) {\n  store.setState({ theme });\n}\n\nexport function setTimezone(timezone: string) {\n  store.setState({ timezone });\n}\n\nexport function setLocale(locale: string) {\n  store.setState({ locale });\n}\n\nexport function setShareToken(shareToken: string) {\n  store.setState({ shareToken });\n}\n\nexport function setUser(user: object) {\n  store.setState({ user });\n}\n\nexport function setConfig(config: object) {\n  store.setState({ config });\n}\n\nexport function setDateRange(dateRange: string | object) {\n  store.setState({ dateRange });\n}\n\nexport default store;\n\n// Path: src/components/hooks/queries/useApi.ts\nimport * as reactQuery from '@tanstack/react-query';\nimport { useApi as nextUseApi } from 'next-basics';\nimport { getClientAuthToken } from 'lib/client';\nimport { SHARE_TOKEN_HEADER } from 'lib/constants';\nimport useStore from 'store/app';\n\nconst selector = (state: { shareToken: { token?: string } }) => state.shareToken;\n\nexport function useApi() {\n  const shareToken = useStore(selector);\n\n  const { get, post, put, del } = nextUseApi(\n    { authorization: `Bearer ${getClientAuthToken()}`, [SHARE_TOKEN_HEADER]: shareToken?.token },\n    process.env.basePath,\n  );\n\n  return { get, post, put, del, ...reactQuery };\n}\n\nexport default useApi;\n\n// Path: src/components/hooks/queries/useWebsite.ts\nimport useApi from './useApi';\n\nexport function useWebsite(websiteId: string, options?: { [key: string]: any }) {\n  const { get, useQuery } = useApi();\n\n  return useQuery({\n    queryKey: ['website', { websiteId }],\n    queryFn: () => get(`/websites/${websiteId}`),\n    enabled: !!websiteId,\n    ...options,\n  });\n}\n\nexport default useWebsite;\n\n// Path: src/components/hooks/queries/useFilterQuery.ts\nimport { UseQueryOptions } from '@tanstack/react-query';\nimport { useState } from 'react';\nimport { useApi } from './useApi';\nimport { FilterResult, SearchFilter, FilterQueryResult } from 'lib/types';\n\nexport function useFilterQuery<T = any>({\n  queryKey,\n  queryFn,\n  ...options\n}: Omit<UseQueryOptions, 'queryFn'> & { queryFn: (params?: object) => any }): FilterQueryResult<T> {\n  const [params, setParams] = useState<T | SearchFilter>({\n    query: '',\n    page: 1,\n  });\n\n  const { useQuery } = useApi();\n  const { data, ...query } = useQuery({\n    queryKey: [{ ...queryKey, ...params }],\n    queryFn: () => queryFn(params as any),\n    ...options,\n  });\n\n  return {\n    result: data as FilterResult<any>,\n    query,\n    params,\n    setParams,\n  };\n}\n\nexport default useFilterQuery;\n\n// Path: src/components/hooks/useSticky.ts\nimport { useState, useEffect, useRef } from 'react';\n\nexport function useSticky({ enabled = true, threshold = 1 }) {\n  const [isSticky, setIsSticky] = useState(false);\n  const ref = useRef(null);\n\n  useEffect(() => {\n    let observer: IntersectionObserver | undefined;\n    const handler: IntersectionObserverCallback = ([entry]) =>\n      setIsSticky(entry.intersectionRatio < threshold);\n\n    if (enabled && ref.current) {\n      observer = new IntersectionObserver(handler, { threshold: [threshold] });\n      observer.observe(ref.current);\n    }\n    return () => {\n      if (observer) {\n        observer.disconnect();\n      }\n    };\n  }, [ref, enabled, threshold]);\n\n  return { ref, isSticky };\n}\n\nexport default useSticky;\n\n// Path: src/components/hooks/useForceUpdate.ts\nimport { useCallback, useState } from 'react';\n\nexport function useForceUpdate() {\n  const [, update] = useState(Object.create(null));\n\n  return useCallback(() => {\n    update(Object.create(null));\n  }, [update]);\n}\n\nexport default useForceUpdate;\n\n// Path: src/components/hooks/useLocale.ts\nimport { useEffect } from 'react';\nimport { httpGet, setItem } from 'next-basics';\nimport { LOCALE_CONFIG } from 'lib/constants';\nimport { getDateLocale, getTextDirection } from 'lib/lang';\nimport useStore, { setLocale } from 'store/app';\nimport { useForceUpdate } from './useForceUpdate';\nimport enUS from '../../../public/intl/country/en-US.json';\n\nconst messages = {\n  'en-US': enUS,\n};\n\nconst selector = (state: { locale: any }) => state.locale;\n\nexport function useLocale() {\n  const locale = useStore(selector);\n  const forceUpdate = useForceUpdate();\n  const dir = getTextDirection(locale);\n  const dateLocale = getDateLocale(locale);\n\n  async function loadMessages(locale: string) {\n    const { ok, data } = await httpGet(`${process.env.basePath}/intl/messages/${locale}.json`);\n\n    if (ok) {\n      messages[locale] = data;\n    }\n  }\n\n  async function saveLocale(value: string) {\n    if (!messages[value]) {\n      await loadMessages(value);\n    }\n\n    setItem(LOCALE_CONFIG, value);\n\n    document.getElementById('__next')?.setAttribute('dir', getTextDirection(value));\n\n    if (locale !== value) {\n      setLocale(value);\n    } else {\n      forceUpdate();\n    }\n  }\n\n  useEffect(() => {\n    if (!messages[locale]) {\n      saveLocale(locale);\n    }\n  }, [locale]);\n\n  useEffect(() => {\n    const url = new URL(window?.location?.href);\n    const locale = url.searchParams.get('locale');\n\n    if (locale) {\n      saveLocale(locale);\n    }\n  }, []);\n\n  return { locale, saveLocale, messages, dir, dateLocale };\n}\n\nexport default useLocale;\n\n// Path: src/components/hooks/useNavigation.ts\nimport { useMemo } from 'react';\nimport { usePathname, useRouter, useSearchParams } from 'next/navigation';\nimport { buildUrl, safeDecodeURIComponent } from 'next-basics';\n\nexport function useNavigation(): {\n  pathname: string;\n  query: { [key: string]: string };\n  router: any;\n  renderUrl: (params: any, reset?: boolean) => string;\n} {\n  const router = useRouter();\n  const pathname = usePathname();\n  const params = useSearchParams();\n\n  const query = useMemo(() => {\n    const obj = {};\n\n    for (const [key, value] of params.entries()) {\n      obj[key] = safeDecodeURIComponent(value);\n    }\n\n    return obj;\n  }, [params]);\n\n  function renderUrl(params: any, reset?: boolean) {\n    return reset ? pathname : buildUrl(pathname, { ...query, ...params });\n  }\n\n  return { pathname, query, router, renderUrl };\n}\n\nexport default useNavigation;\n\n// Path: src/components/hooks/queries/useShareToken.ts\nimport useStore, { setShareToken } from 'store/app';\nimport useApi from './useApi';\n\nconst selector = (state: { shareToken: string }) => state.shareToken;\n\nexport function useShareToken(shareId: string): {\n  shareToken: any;\n  isLoading?: boolean;\n  error?: Error;\n} {\n  const shareToken = useStore(selector);\n  const { get, useQuery } = useApi();\n  const { isLoading, error } = useQuery({\n    queryKey: ['share', shareId],\n    queryFn: async () => {\n      const data = await get(`/share/${shareId}`);\n\n      setShareToken(data);\n\n      return data;\n    },\n  });\n\n  return { shareToken, isLoading, error };\n}\n\nexport default useShareToken;\n\n// Path: src/store/modified.ts\nimport { create } from 'zustand';\n\nconst store = create(() => ({}));\n\nexport function touch(key: string) {\n  store.setState({ [key]: Date.now() });\n}\n\nexport default store;\n\n// Path: src/components/hooks/useModified.ts\nimport useStore from 'store/modified';\n\nexport function useModified(key?: string) {\n  const modified = useStore(state => state?.[key]);\n\n  const touch = (id?: string) => {\n    if (id || key) {\n      useStore.setState({ [id || key]: Date.now() });\n    }\n  };\n\n  return { modified, touch };\n}\n\nexport default useModified;\n\n// Path: src/components/hooks/queries/useLogin.ts\nimport useStore, { setUser } from 'store/app';\nimport useApi from './useApi';\nimport { UseQueryResult } from '@tanstack/react-query';\n\nconst selector = (state: { user: any }) => state.user;\n\nexport function useLogin(): {\n  user: any;\n  setUser: (data: any) => void;\n} & UseQueryResult {\n  const { get, useQuery } = useApi();\n  const user = useStore(selector);\n\n  const query = useQuery({\n    queryKey: ['login'],\n    queryFn: async () => {\n      const data = await get('/auth/verify');\n\n      setUser(data);\n\n      return data;\n    },\n    enabled: !user,\n  });\n\n  return { user, setUser, ...query };\n}\n\nexport default useLogin;\n\n// Path: src/components/hooks/queries/useWebsites.ts\nimport { useApi } from './useApi';\nimport { useFilterQuery } from './useFilterQuery';\nimport { useLogin } from './useLogin';\nimport useModified from '../useModified';\n\nexport function useWebsites(\n  { userId, teamId }: { userId?: string; teamId?: string },\n  params?: { [key: string]: string | number },\n) {\n  const { get } = useApi();\n  const { user } = useLogin();\n  const { modified } = useModified(`websites`);\n\n  return useFilterQuery({\n    queryKey: ['websites', { userId, teamId, modified, ...params }],\n    queryFn: (data: any) => {\n      return get(teamId ? `/teams/${teamId}/websites` : `/users/${userId || user.id}/websites`, {\n        ...data,\n        ...params,\n      });\n    },\n  });\n}\n\nexport default useWebsites;\n\n// Path: src/components/hooks/queries/useTeam.ts\nimport useApi from './useApi';\n\nexport function useTeam(teamId: string) {\n  const { get, useQuery } = useApi();\n  return useQuery({\n    queryKey: ['teams', teamId],\n    queryFn: () => get(`/teams/${teamId}`),\n    enabled: !!teamId,\n  });\n}\n\nexport default useTeam;\n\n// Path: src/components/messages.ts\nimport { defineMessages } from 'react-intl';\n\nexport const labels = defineMessages({\n  ok: { id: 'label.ok', defaultMessage: 'OK' },\n  unknown: { id: 'label.unknown', defaultMessage: 'Unknown' },\n  required: { id: 'label.required', defaultMessage: 'Required' },\n  save: { id: 'label.save', defaultMessage: 'Save' },\n  cancel: { id: 'label.cancel', defaultMessage: 'Cancel' },\n  continue: { id: 'label.continue', defaultMessage: 'Continue' },\n  delete: { id: 'label.delete', defaultMessage: 'Delete' },\n  leave: { id: 'label.leave', defaultMessage: 'Leave' },\n  users: { id: 'label.users', defaultMessage: 'Users' },\n  createUser: { id: 'label.create-user', defaultMessage: 'Create user' },\n  deleteUser: { id: 'label.delete-user', defaultMessage: 'Delete user' },\n  username: { id: 'label.username', defaultMessage: 'Username' },\n  password: { id: 'label.password', defaultMessage: 'Password' },\n  role: { id: 'label.role', defaultMessage: 'Role' },\n  admin: { id: 'label.admin', defaultMessage: 'Admin' },\n  user: { id: 'label.user', defaultMessage: 'User' },\n  viewOnly: { id: 'label.view-only', defaultMessage: 'View only' },\n  manage: { id: 'label.manage', defaultMessage: 'Manage' },\n  admin: { id: 'label.admin', defaultMessage: 'Administrator' },\n  confirm: { id: 'label.confirm', defaultMessage: 'Confirm' },\n  details: { id: 'label.details', defaultMessage: 'Details' },\n  website: { id: 'label.website', defaultMessage: 'Website' },\n  websites: { id: 'label.websites', defaultMessage: 'Websites' },\n  myWebsites: { id: 'label.my-websites', defaultMessage: 'My websites' },\n  teamWebsites: { id: 'label.team-websites', defaultMessage: 'Team websites' },\n  created: { id: 'label.created', defaultMessage: 'Created' },\n  createdBy: { id: 'label.created-by', defaultMessage: 'Created By' },\n  edit: { id: 'label.edit', defaultMessage: 'Edit' },\n  name: { id: 'label.name', defaultMessage: 'Name' },\n  member: { id: 'label.member', defaultMessage: 'Member' },\n  members: { id: 'label.members', defaultMessage: 'Members' },\n  accessCode: { id: 'label.access-code', defaultMessage: 'Access code' },\n  teamId: { id: 'label.team-id', defaultMessage: 'Team ID' },\n  team: { id: 'label.team', defaultMessage: 'Team' },\n  teamName: { id: 'label.team-name', defaultMessage: 'Team name' },\n  regenerate: { id: 'label.regenerate', defaultMessage: 'Regenerate' },\n  remove: { id: 'label.remove', defaultMessage: 'Remove' },\n  join: { id: 'label.join', defaultMessage: 'Join' },\n  createTeam: { id: 'label.create-team', defaultMessage: 'Create team' },\n  joinTeam: { id: 'label.join-team', defaultMessage: 'Join team' },\n  settings: { id: 'label.settings', defaultMessage: 'Settings' },\n  owner: { id: 'label.owner', defaultMessage: 'Owner' },\n  teamOwner: { id: 'label.team-owner', defaultMessage: 'Team owner' },\n  teamMember: { id: 'label.team-member', defaultMessage: 'Team member' },\n  teamViewOnly: { id: 'label.team-view-only', defaultMessage: 'Team view only' },\n  enableShareUrl: { id: 'label.enable-share-url', defaultMessage: 'Enable share URL' },\n  data: { id: 'label.data', defaultMessage: 'Data' },\n  trackingCode: { id: 'label.tracking-code', defaultMessage: 'Tracking code' },\n  shareUrl: { id: 'label.share-url', defaultMessage: 'Share URL' },\n  actions: { id: 'label.actions', defaultMessage: 'Actions' },\n  domain: { id: 'label.domain', defaultMessage: 'Domain' },\n  websiteId: { id: 'label.website-id', defaultMessage: 'Website ID' },\n  resetWebsite: { id: 'label.reset-website', defaultMessage: 'Reset website' },\n  deleteWebsite: { id: 'label.delete-website', defaultMessage: 'Delete website' },\n  transferWebsite: { id: 'label.transfer-website', defaultMessage: 'Transfer website' },\n  deleteReport: { id: 'label.delete-report', defaultMessage: 'Delete report' },\n  reset: { id: 'label.reset', defaultMessage: 'Reset' },\n  addWebsite: { id: 'label.add-website', defaultMessage: 'Add website' },\n  addMember: { id: 'label.add-member', defaultMessage: 'Add member' },\n  editMember: { id: 'label.edit-member', defaultMessage: 'Edit member' },\n  removeMember: { id: 'label.remove-member', defaultMessage: 'Remove member' },\n  addDescription: { id: 'label.add-description', defaultMessage: 'Add description' },\n  changePassword: { id: 'label.change-password', defaultMessage: 'Change password' },\n  currentPassword: { id: 'label.current-password', defaultMessage: 'Current password' },\n  newPassword: { id: 'label.new-password', defaultMessage: 'New password' },\n  confirmPassword: { id: 'label.confirm-password', defaultMessage: 'Confirm password' },\n  timezone: { id: 'label.timezone', defaultMessage: 'Timezone' },\n  defaultDateRange: { id: 'label.default-date-range', defaultMessage: 'Default date range' },\n  language: { id: 'label.language', defaultMessage: 'Language' },\n  theme: { id: 'label.theme', defaultMessage: 'Theme' },\n  profile: { id: 'label.profile', defaultMessage: 'Profile' },\n  dashboard: { id: 'label.dashboard', defaultMessage: 'Dashboard' },\n  more: { id: 'label.more', defaultMessage: 'More' },\n  realtime: { id: 'label.realtime', defaultMessage: 'Realtime' },\n  queries: { id: 'label.queries', defaultMessage: 'Queries' },\n  teams: { id: 'label.teams', defaultMessage: 'Teams' },\n  analytics: { id: 'label.analytics', defaultMessage: 'Analytics' },\n  login: { id: 'label.login', defaultMessage: 'Login' },\n  logout: { id: 'label.logout', defaultMessage: 'Logout' },\n  singleDay: { id: 'label.single-day', defaultMessage: 'Single day' },\n  dateRange: { id: 'label.date-range', defaultMessage: 'Date range' },\n  viewDetails: { id: 'label.view-details', defaultMessage: 'View details' },\n  deleteTeam: { id: 'label.delete-team', defaultMessage: 'Delete team' },\n  leaveTeam: { id: 'label.leave-team', defaultMessage: 'Leave team' },\n  refresh: { id: 'label.refresh', defaultMessage: 'Refresh' },\n  pages: { id: 'label.pages', defaultMessage: 'Pages' },\n  referrers: { id: 'label.referrers', defaultMessage: 'Referrers' },\n  screens: { id: 'label.screens', defaultMessage: 'Screens' },\n  browsers: { id: 'label.browsers', defaultMessage: 'Browsers' },\n  os: { id: 'label.os', defaultMessage: 'OS' },\n  devices: { id: 'label.devices', defaultMessage: 'Devices' },\n  countries: { id: 'label.countries', defaultMessage: 'Countries' },\n  languages: { id: 'label.languages', defaultMessage: 'Languages' },\n  event: { id: 'label.event', defaultMessage: 'Event' },\n  events: { id: 'label.events', defaultMessage: 'Events' },\n  query: { id: 'label.query', defaultMessage: 'Query' },\n  queryParameters: { id: 'label.query-parameters', defaultMessage: 'Query parameters' },\n  back: { id: 'label.back', defaultMessage: 'Back' },\n  visitors: { id: 'label.visitors', defaultMessage: 'Visitors' },\n  visits: { id: 'label.visits', defaultMessage: 'Visits' },\n  filterCombined: { id: 'label.filter-combined', defaultMessage: 'Combined' },\n  filterRaw: { id: 'label.filter-raw', defaultMessage: 'Raw' },\n  views: { id: 'label.views', defaultMessage: 'Views' },\n  none: { id: 'label.none', defaultMessage: 'None' },\n  clearAll: { id: 'label.clear-all', defaultMessage: 'Clear all' },\n  today: { id: 'label.today', defaultMessage: 'Today' },\n  lastHours: { id: 'label.last-hours', defaultMessage: 'Last {x} hours' },\n  yesterday: { id: 'label.yesterday', defaultMessage: 'Yesterday' },\n  thisWeek: { id: 'label.this-week', defaultMessage: 'This week' },\n  lastDays: { id: 'label.last-days', defaultMessage: 'Last {x} days' },\n  lastMonths: { id: 'label.last-months', defaultMessage: 'Last {x} months' },\n  thisMonth: { id: 'label.this-month', defaultMessage: 'This month' },\n  thisYear: { id: 'label.this-year', defaultMessage: 'This year' },\n  allTime: { id: 'label.all-time', defaultMessage: 'All time' },\n  customRange: { id: 'label.custom-range', defaultMessage: 'Custom range' },\n  selectWebsite: { id: 'label.select-website', defaultMessage: 'Select website' },\n  selectRole: { id: 'label.select-role', defaultMessage: 'Select role' },\n  selectDate: { id: 'label.select-date', defaultMessage: 'Select date' },\n  all: { id: 'label.all', defaultMessage: 'All' },\n  sessions: { id: 'label.sessions', defaultMessage: 'Sessions' },\n  pageNotFound: { id: 'message.page-not-found', defaultMessage: 'Page not found' },\n  activityLog: { id: 'label.activity-log', defaultMessage: 'Activity log' },\n  dismiss: { id: 'label.dismiss', defaultMessage: 'Dismiss' },\n  poweredBy: { id: 'label.powered-by', defaultMessage: 'Powered by {name}' },\n  pageViews: { id: 'label.page-views', defaultMessage: 'Page views' },\n  uniqueVisitors: { id: 'label.unique-visitors', defaultMessage: 'Unique visitors' },\n  bounceRate: { id: 'label.bounce-rate', defaultMessage: 'Bounce rate' },\n  viewsPerVisit: { id: 'label.views-per-visit', defaultMessage: 'Views per visit' },\n  averageVisitTime: { id: 'label.average-visit-time', defaultMessage: 'Average visit time' },\n  desktop: { id: 'label.desktop', defaultMessage: 'Desktop' },\n  laptop: { id: 'label.laptop', defaultMessage: 'Laptop' },\n  tablet: { id: 'label.tablet', defaultMessage: 'Tablet' },\n  mobile: { id: 'label.mobile', defaultMessage: 'Mobile' },\n  toggleCharts: { id: 'label.toggle-charts', defaultMessage: 'Toggle charts' },\n  editDashboard: { id: 'label.edit-dashboard', defaultMessage: 'Edit dashboard' },\n  title: { id: 'label.title', defaultMessage: 'Title' },\n  view: { id: 'label.view', defaultMessage: 'View' },\n  cities: { id: 'label.cities', defaultMessage: 'Cities' },\n  regions: { id: 'label.regions', defaultMessage: 'Regions' },\n  reports: { id: 'label.reports', defaultMessage: 'Reports' },\n  eventData: { id: 'label.event-data', defaultMessage: 'Event data' },\n  funnel: { id: 'label.funnel', defaultMessage: 'Funnel' },\n  funnelDescription: {\n    id: 'label.funnel-description',\n    defaultMessage: 'Understand the conversion and drop-off rate of users.',\n  },\n  url: { id: 'label.url', defaultMessage: 'URL' },\n  urls: { id: 'label.urls', defaultMessage: 'URLs' },\n  add: { id: 'label.add', defaultMessage: 'Add' },\n  update: { id: 'label.update', defaultMessage: 'Update' },\n  window: { id: 'label.window', defaultMessage: 'Window' },\n  runQuery: { id: 'label.run-query', defaultMessage: 'Run query' },\n  field: { id: 'label.field', defaultMessage: 'Field' },\n  fields: { id: 'label.fields', defaultMessage: 'Fields' },\n  createReport: { id: 'label.create-report', defaultMessage: 'Create report' },\n  description: { id: 'label.description', defaultMessage: 'Description' },\n  untitled: { id: 'label.untitled', defaultMessage: 'Untitled' },\n  type: { id: 'label.type', defaultMessage: 'Type' },\n  filter: { id: 'label.filter', defaultMessage: 'Filter' },\n  filters: { id: 'label.filters', defaultMessage: 'Filters' },\n  breakdown: { id: 'label.breakdown', defaultMessage: 'Breakdown' },\n  true: { id: 'label.true', defaultMessage: 'True' },\n  false: { id: 'label.false', defaultMessage: 'False' },\n  is: { id: 'label.is', defaultMessage: 'Is' },\n  isNot: { id: 'label.is-not', defaultMessage: 'Is not' },\n  isSet: { id: 'label.is-set', defaultMessage: 'Is set' },\n  isNotSet: { id: 'label.is-not-set', defaultMessage: 'Is not set' },\n  greaterThan: { id: 'label.greater-than', defaultMessage: 'Greater than' },\n  lessThan: { id: 'label.less-than', defaultMessage: 'Less than' },\n  greaterThanEquals: { id: 'label.greater-than-equals', defaultMessage: 'Greater than or equals' },\n  lessThanEquals: { id: 'label.less-than-equals', defaultMessage: 'Less than or equals' },\n  contains: { id: 'label.contains', defaultMessage: 'Contains' },\n  doesNotContain: { id: 'label.does-not-contain', defaultMessage: 'Does not contain' },\n  before: { id: 'label.before', defaultMessage: 'Before' },\n  after: { id: 'label.after', defaultMessage: 'After' },\n  total: { id: 'label.total', defaultMessage: 'Total' },\n  sum: { id: 'label.sum', defaultMessage: 'Sum' },\n  average: { id: 'label.average', defaultMessage: 'Average' },\n  min: { id: 'label.min', defaultMessage: 'Min' },\n  max: { id: 'label.max', defaultMessage: 'Max' },\n  unique: { id: 'label.unique', defaultMessage: 'Unique' },\n  value: { id: 'label.value', defaultMessage: 'Value' },\n  overview: { id: 'label.overview', defaultMessage: 'Overview' },\n  totalRecords: { id: 'label.total-records', defaultMessage: 'Total records' },\n  insights: { id: 'label.insights', defaultMessage: 'Insights' },\n  insightsDescription: {\n    id: 'label.insights-description',\n    defaultMessage: 'Dive deeper into your data by using segments and filters.',\n  },\n  retention: { id: 'label.retention', defaultMessage: 'Retention' },\n  retentionDescription: {\n    id: 'label.retention-description',\n    defaultMessage: 'Measure your website stickiness by tracking how often users return.',\n  },\n  dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' },\n  referrer: { id: 'label.referrer', defaultMessage: 'Referrer' },\n  country: { id: 'label.country', defaultMessage: 'Country' },\n  region: { id: 'label.region', defaultMessage: 'Region' },\n  city: { id: 'label.city', defaultMessage: 'City' },\n  browser: { id: 'label.browser', defaultMessage: 'Browser' },\n  device: { id: 'label.device', defaultMessage: 'Device' },\n  pageTitle: { id: 'label.pageTitle', defaultMessage: 'Page title' },\n  day: { id: 'label.day', defaultMessage: 'Day' },\n  date: { id: 'label.date', defaultMessage: 'Date' },\n  pageOf: { id: 'label.page-of', defaultMessage: 'Page {current} of {total}' },\n  create: { id: 'label.create', defaultMessage: 'Create' },\n  search: { id: 'label.search', defaultMessage: 'Search' },\n  numberOfRecords: {\n    id: 'label.number-of-records',\n    defaultMessage: '{x} {x, plural, one {record} other {records}}',\n  },\n  select: { id: 'label.select', defaultMessage: 'Select' },\n  myAccount: { id: 'label.my-account', defaultMessage: 'My account' },\n  transfer: { id: 'label.transfer', defaultMessage: 'Transfer' },\n  viewedPage: {\n    id: 'message.viewed-page',\n    defaultMessage: 'Viewed page',\n  },\n  triggeredEvent: {\n    id: 'message.triggered-event',\n    defaultMessage: 'Triggered event',\n  },\n  visitorsDroppedOff: {\n    id: 'message.visitors-dropped-off',\n    defaultMessage: 'Visitors dropped off',\n  },\n  utm: { id: 'label.utm', defaultMessage: 'UTM' },\n  utmDescription: {\n    id: 'label.utm-description',\n    defaultMessage: 'Track your campaigns through UTM parameters.',\n  },\n  steps: { id: 'label.steps', defaultMessage: 'Steps' },\n  addStep: { id: 'label.add-step', defaultMessage: 'Add step' },\n});\n\nexport const messages = defineMessages({\n  error: { id: 'message.error', defaultMessage: 'Something went wrong.' },\n  saved: { id: 'message.saved', defaultMessage: 'Saved.' },\n  noUsers: { id: 'message.no-users', defaultMessage: 'There are no users.' },\n  userDeleted: { id: 'message.user-deleted', defaultMessage: 'User deleted.' },\n  noDataAvailable: { id: 'message.no-data-available', defaultMessage: 'No data available.' },\n  confirmReset: {\n    id: 'message.confirm-reset',\n    defaultMessage: 'Are you sure you want to reset {target}?',\n  },\n  confirmDelete: {\n...\n", "needle_token_start": 7184, "needle_token_end": 7283, "code_context_ntokens": 16126, "output": ["```typescript\nexport function getAllowedUnits(startDate: Date, endDate: Date) {\n  const units = ['minute', 'hour', 'day', 'month', 'year'];\n  const minUnit = getMinimumUnit(startDate, endDate);\n  const index = units.indexOf(minUnit === 'year' ? 'month' : minUnit);\n\n  return index >= 0 ? units.splice(index) : [];\n}\n```"]}
{"repo": "umami-software/umami", "name": "getWebsiteStats", "language": "typescript", "path": "src/queries/analytics/getWebsiteStats.ts", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To gather and return various statistical metrics related to the performance and user engagement of a specific website over a given time period.\n2. **Input**: A unique identifier for the website and a set of filtering criteria which includes date ranges and other parameters relevant to the website's events.\n3. **Output**: A collection of statistics such as page views, unique visitors, total visits, bounce rates, and total engagement time.\n4. **Procedure**: The function selects the appropriate database query method based on the environment (relational or ClickHouse). It then executes a complex SQL query that filters website events within the specified date range and calculates different engagement metrics. The results are formatted and returned as structured data.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/queries/admin/website.ts\nimport { Prisma, Website } from '@prisma/client';\nimport cache from 'lib/cache';\nimport prisma from 'lib/prisma';\nimport { FilterResult, WebsiteSearchFilter } from 'lib/types';\nimport WebsiteFindManyArgs = Prisma.WebsiteFindManyArgs;\n\nasync function findWebsite(criteria: Prisma.WebsiteFindUniqueArgs): Promise<Website> {\n  return prisma.client.website.findUnique(criteria);\n}\n\nexport async function getWebsite(websiteId: string) {\n  return findWebsite({\n    where: {\n      id: websiteId,\n    },\n  });\n}\n\nexport async function getSharedWebsite(shareId: string) {\n  return findWebsite({\n    where: {\n      shareId,\n    },\n  });\n}\n\nexport async function getWebsites(\n  criteria: WebsiteFindManyArgs,\n  filters: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  const { query } = filters;\n\n  const where: Prisma.WebsiteWhereInput = {\n    ...criteria.where,\n    ...prisma.getSearchParameters(query, [\n      {\n        name: 'contains',\n      },\n      { domain: 'contains' },\n    ]),\n    deletedAt: null,\n  };\n\n  return prisma.pagedQuery('website', { ...criteria, where }, filters);\n}\n\nexport async function getAllWebsites(userId: string) {\n  return prisma.client.website.findMany({\n    where: {\n      userId,\n    },\n  });\n}\n\nexport async function getUserWebsites(\n  userId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        userId,\n      },\n      include: {\n        user: {\n          select: {\n            username: true,\n            id: true,\n          },\n        },\n      },\n    },\n    {\n      orderBy: 'name',\n      ...filters,\n    },\n  );\n}\n\nexport async function getTeamWebsites(\n  teamId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        teamId,\n      },\n      include: {\n        createUser: {\n          select: {\n            id: true,\n            username: true,\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createWebsite(\n  data: Prisma.WebsiteCreateInput | Prisma.WebsiteUncheckedCreateInput,\n): Promise<Website> {\n  return prisma.client.website\n    .create({\n      data,\n    })\n    .then(async data => {\n      if (cache.enabled) {\n        await cache.storeWebsite(data);\n      }\n\n      return data;\n    });\n}\n\nexport async function updateWebsite(\n  websiteId: string,\n  data: Prisma.WebsiteUpdateInput | Prisma.WebsiteUncheckedUpdateInput,\n): Promise<Website> {\n  return prisma.client.website.update({\n    where: {\n      id: websiteId,\n    },\n    data,\n  });\n}\n\nexport async function resetWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId },\n    }),\n    client.session.deleteMany({\n      where: { websiteId },\n    }),\n    client.website.update({\n      where: { id: websiteId },\n      data: {\n        resetAt: new Date(),\n      },\n    }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      await cache.storeWebsite(data[3]);\n    }\n\n    return data;\n  });\n}\n\nexport async function deleteWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n...\n// Path: src/queries/analytics/sessions/getSession.ts\nimport prisma from 'lib/prisma';\n\nexport async function getSession(id: string) {\n  return prisma.client.session.findUnique({\n    where: {\n      id,\n    },\n  });\n}\n\n// Path: src/lib/data.ts\nimport { DATA_TYPE, DATETIME_REGEX } from './constants';\nimport { DynamicDataType } from './types';\n\nexport function flattenJSON(\n  eventData: { [key: string]: any },\n  keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],\n  parentKey = '',\n): { key: string; value: any; dataType: DynamicDataType }[] {\n  return Object.keys(eventData).reduce(\n    (acc, key) => {\n      const value = eventData[key];\n      const type = typeof eventData[key];\n\n      // nested object\n      if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {\n        flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));\n      } else {\n        createKey(getKeyName(key, parentKey), value, acc);\n      }\n\n      return acc;\n    },\n    { keyValues, parentKey },\n  ).keyValues;\n}\n\nexport function isValidDateValue(value: string) {\n  return typeof value === 'string' && DATETIME_REGEX.test(value);\n}\n\nexport function getDataType(value: any): string {\n  let type: string = typeof value;\n\n  if (isValidDateValue(value)) {\n    type = 'date';\n  }\n\n  return type;\n}\n\nexport function getStringValue(value: string, dataType: number) {\n  if (dataType === DATA_TYPE.number) {\n    return parseFloat(value).toFixed(4);\n  }\n\n  if (dataType === DATA_TYPE.date) {\n    return new Date(value).toISOString();\n  }\n\n  return value;\n}\n\nfunction createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {\n  const type = getDataType(value);\n\n  let dataType = null;\n\n  switch (type) {\n    case 'number':\n      dataType = DATA_TYPE.number;\n      break;\n    case 'string':\n      dataType = DATA_TYPE.string;\n      break;\n    case 'boolean':\n      dataType = DATA_TYPE.boolean;\n      value = value ? 'true' : 'false';\n      break;\n    case 'date':\n      dataType = DATA_TYPE.date;\n      break;\n    case 'object':\n      dataType = DATA_TYPE.array;\n      value = JSON.stringify(value);\n      break;\n    default:\n      dataType = DATA_TYPE.string;\n      break;\n  }\n\n  acc.keyValues.push({ key, value, dataType });\n}\n\nfunction getKeyName(key: string, parentKey: string) {\n  if (!parentKey) {\n    return key;\n  }\n\n  return `${parentKey}.${key}`;\n}\n\n// Path: src/queries/analytics/sessions/saveSessionData.ts\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveSessionData(data: {\n  websiteId: string;\n  sessionId: string;\n  sessionData: DynamicData;\n}) {\n  const { client, transaction } = prisma;\n  const { websiteId, sessionId, sessionData } = data;\n\n  const jsonKeys = flattenJSON(sessionData);\n\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteId,\n    sessionId,\n    key: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return transaction([\n    client.sessionData.deleteMany({\n      where: {\n        sessionId,\n      },\n    }),\n    client.sessionData.createMany({\n      data: flattenedData as any,\n    }),\n  ]);\n}\n\n// Path: src/queries/analytics/reports/getUTM.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nexport async function getUTM(\n  ...args: [\n    websiteId: string,\n    filters: {\n      startDate: Date;\n      endDate: Date;\n      timezone?: string;\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = prisma;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n      and coalesce(url_query, '') != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and url_query != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nfunction parseParameters(data: any[]) {\n  return data.reduce((obj, { url_query, num }) => {\n    try {\n      const searchParams = new URLSearchParams(url_query);\n\n      for (const [key, value] of searchParams) {\n        if (key.match(/^utm_(\\w+)$/)) {\n          const name = safeDecodeURIComponent(value);\n          if (!obj[key]) {\n            obj[key] = { [name]: Number(num) };\n          } else if (!obj[key][name]) {\n            obj[key][name] = Number(num);\n          } else {\n            obj[key][name] += Number(num);\n          }\n        }\n      }\n    } catch {\n      // Ignore\n    }\n\n    return obj;\n  }, {});\n}\n\n// Path: src/lib/kafka.ts\nimport dateFormat from 'dateformat';\nimport debug from 'debug';\nimport { Kafka, Mechanism, Producer, RecordMetadata, SASLOptions, logLevel } from 'kafkajs';\nimport { KAFKA, KAFKA_PRODUCER } from 'lib/db';\nimport * as tls from 'tls';\n\nconst log = debug('umami:kafka');\n\nlet kafka: Kafka;\nlet producer: Producer;\nconst enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER);\n\nfunction getClient() {\n  const { username, password } = new URL(process.env.KAFKA_URL);\n  const brokers = process.env.KAFKA_BROKER.split(',');\n\n  const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions | Mechanism } =\n    username && password\n      ? {\n          ssl: {\n            checkServerIdentity: () => undefined,\n            ca: [process.env.CA_CERT],\n            key: process.env.CLIENT_KEY,\n            cert: process.env.CLIENT_CERT,\n          },\n          sasl: {\n            mechanism: 'plain',\n            username,\n            password,\n          },\n        }\n      : {};\n\n  const client: Kafka = new Kafka({\n    clientId: 'umami',\n    brokers: brokers,\n    connectionTimeout: 3000,\n    logLevel: logLevel.ERROR,\n    ...ssl,\n  });\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA] = client;\n  }\n\n  log('Kafka initialized');\n\n  return client;\n}\n\nasync function getProducer(): Promise<Producer> {\n  const producer = kafka.producer();\n  await producer.connect();\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA_PRODUCER] = producer;\n  }\n\n  log('Kafka producer initialized');\n\n  return producer;\n}\n\nfunction getDateFormat(date: Date, format?: string): string {\n  return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');\n}\n\nasync function sendMessage(\n  message: { [key: string]: string | number },\n  topic: string,\n): Promise<RecordMetadata[]> {\n  await connect();\n\n  return producer.send({\n    topic,\n    messages: [\n      {\n        value: JSON.stringify(message),\n      },\n    ],\n    acks: -1,\n  });\n}\n\nasync function sendMessages(messages: { [key: string]: string | number }[], topic: string) {\n  await connect();\n\n  await producer.send({\n    topic,\n    messages: messages.map(a => {\n      return { value: JSON.stringify(a) };\n    }),\n    acks: 1,\n  });\n}\n\nasync function connect(): Promise<Kafka> {\n  if (!kafka) {\n    kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient());\n\n    if (kafka) {\n      producer = global[KAFKA_PRODUCER] || (await getProducer());\n    }\n  }\n\n  return kafka;\n}\n\nexport default {\n  enabled,\n  client: kafka,\n  producer,\n  log,\n  connect,\n  getDateFormat,\n  sendMessage,\n  sendMessages,\n};\n\n// Path: src/queries/analytics/eventData/saveEventData.ts\nimport { Prisma } from '@prisma/client';\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveEventData(args: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  eventId: string;\n  eventData: DynamicData;\n}): Promise<Prisma.BatchPayload> {\n  const { websiteId, eventId, eventData } = data;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  // id, websiteEventId, eventStringValue\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteEventId: eventId,\n    websiteId,\n    eventKey: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return prisma.client.eventData.createMany({\n    data: flattenedData,\n  });\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;\n\n  const { getDateFormat, sendMessages } = kafka;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  const messages = jsonKeys.map(a => ({\n    website_id: websiteId,\n    session_id: sessionId,\n    event_id: eventId,\n    url_path: urlPath,\n    event_name: eventName,\n    event_key: a.key,\n    string_value: getStringValue(a.value, a.dataType),\n    number_value: a.dataType === DATA_TYPE.number ? a.value : null,\n    date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null,\n    data_type: a.dataType,\n    created_at: createdAt,\n  }));\n\n  await sendMessages(messages, 'event_data');\n\n  return data;\n}\n\n// Path: src/queries/analytics/events/saveEvent.ts\nimport { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE, PAGE_TITLE_LENGTH } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { uuid } from 'lib/crypto';\nimport { saveEventData } from 'queries/analytics/eventData/saveEventData';\n\nexport async function saveEvent(args: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    eventName,\n    eventData,\n    pageTitle,\n  } = data;\n  const websiteEventId = uuid();\n\n  const websiteEvent = prisma.client.websiteEvent.create({\n    data: {\n      id: websiteEventId,\n      websiteId,\n      sessionId,\n      visitId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      urlQuery: urlQuery?.substring(0, URL_LENGTH),\n      referrerPath: referrerPath?.substring(0, URL_LENGTH),\n      referrerQuery: referrerQuery?.substring(0, URL_LENGTH),\n      referrerDomain: referrerDomain?.substring(0, URL_LENGTH),\n      pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n      eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n      eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    },\n  });\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId: websiteEventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n    });\n  }\n\n  return websiteEvent;\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    pageTitle,\n    eventName,\n    eventData,\n    country,\n    subdivision1,\n    subdivision2,\n    city,\n    ...args\n  } = data;\n  const { getDateFormat, sendMessage } = kafka;\n  const eventId = uuid();\n  const createdAt = getDateFormat(new Date());\n\n  const message = {\n    ...args,\n    website_id: websiteId,\n    session_id: sessionId,\n    visit_id: visitId,\n    event_id: uuid(),\n    country: country,\n    subdivision1:\n      country && subdivision1\n        ? subdivision1.includes('-')\n          ? subdivision1\n          : `${country}-${subdivision1}`\n        : null,\n    subdivision2: subdivision2,\n    city: city,\n    url_path: urlPath?.substring(0, URL_LENGTH),\n    url_query: urlQuery?.substring(0, URL_LENGTH),\n    referrer_path: referrerPath?.substring(0, URL_LENGTH),\n    referrer_query: referrerQuery?.substring(0, URL_LENGTH),\n    referrer_domain: referrerDomain?.substring(0, URL_LENGTH),\n    page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n    event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n    event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    created_at: createdAt,\n  };\n\n  await sendMessage(message, 'event');\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n      createdAt,\n    });\n  }\n\n  return data;\n}\n\n// Path: src/queries/analytics/sessions/getSessions.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';\n\nexport async function getSessions(...args: [websiteId: string, startAt: Date]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, startDate: Date) {\n  return prisma.client.session.findMany({\n    where: {\n      websiteId,\n      createdAt: {\n        gte: startDate,\n      },\n    },\n    orderBy: {\n      createdAt: 'asc',\n    },\n  });\n}\n\nasync function clickhouseQuery(websiteId: string, startDate: Date) {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select\n      session_id as id,\n      website_id as websiteId,\n      created_at as createdAt,\n      toUnixTimestamp(created_at) as timestamp,\n      hostname,\n      browser,\n      os,\n      device,\n      screen,\n      language,\n      country,\n      subdivision1,\n      subdivision2,\n      city\n    from website_event\n    where website_id = {websiteId:UUID}\n    and created_at >= {startDate:DateTime64}\n    order by created_at asc\n    `,\n    {\n      websiteId,\n      startDate,\n    },\n  );\n}\n\n// Path: src/queries/analytics/reports/getFunnel.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nconst formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {\n  return steps.map((step: { type: string; value: string }, i: number) => {\n    const visitors = Number(results[i]?.count) || 0;\n    const previous = Number(results[i - 1]?.count) || 0;\n    const dropped = previous > 0 ? previous - visitors : 0;\n    const dropoff = 1 - visitors / previous;\n    const remaining = visitors / Number(results[0].count);\n\n    return {\n      ...step,\n      visitors,\n      previous,\n      dropped,\n      dropoff,\n      remaining,\n    };\n  });\n};\n\nexport async function getFunnel(\n  ...args: [\n    websiteId: string,\n    criteria: {\n      windowMinutes: number;\n      startDate: Date;\n      endDate: Date;\n      steps: { type: string; value: string }[];\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery, getAddIntervalQuery } = prisma;\n  const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, windowMinutes);\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    params: string[];\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union ' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\n          WITH level1 AS (\n            select distinct session_id, created_at\n            from website_event\n            where website_id = {{websiteId::uuid}}\n              and created_at between {{startDate}} and {{endDate}}\n              and ${column} ${operator} {{${i}}}\n          )`;\n        } else {\n          pv.levelQuery += `\n          , level${levelNumber} AS (\n            select distinct we.session_id, we.created_at\n            from level${i} l\n            join website_event we\n                on l.session_id = we.session_id\n            where we.website_id = {{websiteId::uuid}}\n                and we.created_at between l.created_at and ${getAddIntervalQuery(\n                  `l.created_at `,\n                  `${windowMinutes} minute`,\n                )}\n                and we.${column} ${operator} {{${i}}}\n                and we.created_at <= {{endDate}}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.params.push(paramValue);\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        params: [],\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    ${levelOneQuery}\n    ${levelQuery}\n    ${sumQuery}\n    ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery } = clickhouse;\n  const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(\n    steps,\n    windowMinutes,\n  );\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    stepFilterQuery: string;\n    params: { [key: string]: string };\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union all ' : '';\n        const startFilter = i > 0 ? 'or' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\\n\n          level1 AS (\n            select *\n            from level0\n            where ${column} ${operator} {param${i}:String}\n          )`;\n        } else {\n          pv.levelQuery += `\\n\n          , level${levelNumber} AS (\n            select distinct y.session_id as session_id,\n                y.url_path as url_path,\n                y.referrer_path as referrer_path,\n                y.event_name,\n                y.created_at as created_at\n            from level${i} x\n            join level0 y\n            on x.session_id = y.session_id\n            where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute\n                and y.${column} ${operator} {param${i}:String}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.stepFilterQuery += `${startFilter} ${column} ${operator} {param${i}:String} `;\n        pv.params[`param${i}`] = paramValue;\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        stepFilterQuery: '',\n        params: {},\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    WITH level0 AS (\n      select distinct session_id, url_path, referrer_path, event_name, created_at\n      from umami.website_event\n      where (${stepFilterQuery})\n        and website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ),\n    ${levelOneQuery}\n    ${levelQuery}\n    select *\n    from (\n      ${sumQuery} \n    ) ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\n// Path: src/queries/analytics/getWebsiteStats.ts\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { QueryFilters } from 'lib/types';\n\nexport \nasync function getWebsiteStats(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(case when t.c = 1 then 1 else 0 end) as \"bounces\",\n      sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as \"totaltime\"\n    from (\n      select\n        website_event.session_id,\n        website_event.visit_id,\n        count(*) as \"c\",\n        min(website_event.created_at) as \"min_time\",\n        max(website_event.created_at) as \"max_time\"\n      from website_event\n      join website \n        on website_event.website_id = website.website_id\n        ${joinSession}\n      where website.website_id = {{websiteId::uuid}}\n        and website_event.created_at between {{startDate}} and {{endDate}}\n        and event_type = {{eventType}}\n        ${filterQuery}\n      group by 1, 2\n    ) as t\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(if(t.c = 1, 1, 0)) as \"bounces\",\n      sum(max_time-min_time) as \"totaltime\"\n    from (\n      select\n        session_id,\n        visit_id,\n        count(*) c,\n        min(created_at) min_time,\n        max(created_at) max_time\n      from website_event\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n        and event_type = {eventType:UInt32}\n        ${filterQuery}\n      group by session_id, visit_id\n    ) as t;\n    `,\n    params,\n  ).then(result => {\n    return Object.values(result).map(n => {\n      return {\n        pageviews: Number(n.pageviews),\n        visitors: Number(n.visitors),\n        visits: Number(n.visits),\n        bounces: Number(n.bounces),\n        totaltime: Number(n.totaltime),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getActiveVisitors.ts\nimport { subMinutes } from 'date-fns';\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\n\nexport async function getActiveVisitors(...args: [websiteId: string]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string) {\n  const { rawQuery } = prisma;\n\n  const result = await rawQuery(\n    `\n    select count(distinct session_id) x\n    from website_event\n    where website_id = {{websiteId::uuid}}\n    and created_at >= {{startAt}}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  );\n\n  return result[0] ?? null;\n}\n\nasync function clickhouseQuery(websiteId: string): Promise<{ x: number }> {\n  const { rawQuery } = clickhouse;\n\n  const result = await rawQuery(\n    `\n    select\n      count(distinct session_id) x\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startAt:DateTime64}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: Number(a.x) };\n    });\n  });\n\n  return result[0] ?? null;\n}\n\n// Path: src/queries/admin/team.ts\nimport { Prisma, Team } from '@prisma/client';\nimport { ROLES } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport prisma from 'lib/prisma';\nimport { FilterResult, TeamSearchFilter } from 'lib/types';\nimport TeamFindManyArgs = Prisma.TeamFindManyArgs;\n\nexport async function findTeam(criteria: Prisma.TeamFindUniqueArgs): Promise<Team> {\n  return prisma.client.team.findUnique(criteria);\n}\n\nexport async function getTeam(teamId: string, options: { includeMembers?: boolean } = {}) {\n  const { includeMembers } = options;\n\n  return findTeam({\n    where: {\n      id: teamId,\n    },\n    ...(includeMembers && { include: { teamUser: true } }),\n  });\n}\n\nexport async function getTeams(\n  criteria: TeamFindManyArgs,\n  filters: TeamSearchFilter = {},\n): Promise<FilterResult<Team[]>> {\n  const { getSearchParameters } = prisma;\n  const { query } = filters;\n\n  const where: Prisma.TeamWhereInput = {\n    ...criteria.where,\n    ...getSearchParameters(query, [{ name: 'contains' }]),\n  };\n\n  return prisma.pagedQuery<TeamFindManyArgs>(\n    'team',\n    {\n      ...criteria,\n      where,\n    },\n    filters,\n  );\n}\n\nexport async function getUserTeams(userId: string, filters: TeamSearchFilter = {}) {\n  return getTeams(\n    {\n      where: {\n        deletedAt: null,\n        teamUser: {\n          some: { userId },\n        },\n      },\n      include: {\n        teamUser: {\n          include: {\n            user: {\n              select: {\n                id: true,\n                username: true,\n              },\n            },\n          },\n        },\n        _count: {\n          select: {\n            website: {\n              where: { deletedAt: null },\n            },\n            teamUser: {\n              where: {\n                user: { deletedAt: null },\n              },\n            },\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createTeam(data: Prisma.TeamCreateInput, userId: string): Promise<any> {\n  const { id } = data;\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.team.create({\n      data,\n    }),\n    client.teamUser.create({\n      data: {\n        id: uuid(),\n        teamId: id,\n        userId,\n        role: ROLES.teamOwner,\n      },\n    }),\n  ]);\n}\n\nexport async function updateTeam(teamId: string, data: Prisma.TeamUpdateInput): Promise<Team> {\n  const { client } = prisma;\n\n  return client.team.update({\n    where: {\n      id: teamId,\n    },\n    data: {\n      ...data,\n      updatedAt: new Date(),\n    },\n  });\n}\n\nexport async function deleteTeam(\n  teamId: string,\n): Promise<Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Team]>> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  if (cloudMode) {\n    return transaction([\n      client.team.update({\n        data: {\n          deletedAt: new Date(),\n        },\n        where: {\n          id: teamId,\n        },\n      }),\n    ]);\n  }\n\n  return transaction([\n    client.teamUser.deleteMany({\n      where: {\n        teamId,\n      },\n    }),\n    client.team.delete({\n      where: {\n        id: teamId,\n      },\n    }),\n  ]);\n}\n\n// Path: src/queries/index.ts\nexport * from './admin/report';\nexport * from './admin/team';\nexport * from './admin/teamUser';\nexport * from './admin/user';\nexport * from './admin/website';\nexport * from './analytics/events/getEventMetrics';\nexport * from './analytics/events/getEventUsage';\nexport * from './analytics/events/getEvents';\nexport * from './analytics/eventData/getEventDataEvents';\nexport * from './analytics/eventData/getEventDataFields';\nexport * from './analytics/eventData/getEventDataStats';\nexport * from './analytics/eventData/getEventDataUsage';\nexport * from './analytics/events/saveEvent';\nexport * from './analytics/reports/getFunnel';\nexport * from './analytics/reports/getRetention';\nexport * from './analytics/reports/getInsights';\nexport * from './analytics/reports/getUTM';\nexport * from './analytics/pageviews/getPageviewMetrics';\nexport * from './analytics/pageviews/getPageviewStats';\nexport * from './analytics/sessions/createSession';\nexport * from './analytics/sessions/getSession';\nexport * from './analytics/sessions/getSessionMetrics';\nexport * from './analytics/sessions/getSessions';\nexport * from './analytics/sessions/getSessionStats';\nexport * from './analytics/sessions/saveSessionData';\nexport * from './analytics/getActiveVisitors';\nexport * from './analytics/getRealtimeData';\nexport * from './analytics/getValues';\nexport * from './analytics/getWebsiteDateRange';\nexport * from './analytics/getWebsiteStats';\n\n// Path: src/lib/cache.ts\nimport { User, Website } from '@prisma/client';\nimport redis from '@umami/redis-client';\nimport { getSession, getUser, getWebsite } from '../queries';\n\nasync function fetchWebsite(websiteId: string): Promise<Website> {\n  return redis.client.getCache(`website:${websiteId}`, () => getWebsite(websiteId), 86400);\n}\n\nasync function storeWebsite(data: { id: any }) {\n  const { id } = data;\n  const key = `website:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteWebsite(id) {\n  return redis.client.deleteCache(`website:${id}`);\n}\n\nasync function fetchUser(id): Promise<User> {\n  return redis.client.getCache(`user:${id}`, () => getUser(id, { includePassword: true }), 86400);\n}\n\nasync function storeUser(data) {\n  const { id } = data;\n  const key = `user:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteUser(id) {\n  return redis.client.deleteCache(`user:${id}`);\n}\n\nasync function fetchSession(id) {\n  return redis.client.getCache(`session:${id}`, () => getSession(id), 86400);\n}\n\nasync function storeSession(data) {\n  const { id } = data;\n  const key = `session:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteSession(id) {\n  return redis.client.deleteCache(`session:${id}`);\n}\n\nasync function fetchUserBlock(userId: string) {\n  const key = `user:block:${userId}`;\n  return redis.client.get(key);\n}\n\nasync function incrementUserBlock(userId: string) {\n  const key = `user:block:${userId}`;\n  return redis.client.incr(key);\n}\n\nexport default {\n  fetchWebsite,\n  storeWebsite,\n  deleteWebsite,\n  fetchUser,\n  storeUser,\n  deleteUser,\n  fetchSession,\n  storeSession,\n  deleteSession,\n  fetchUserBlock,\n  incrementUserBlock,\n  enabled: !!redis.enabled,\n};\n\n// Path: src/lib/detect.ts\nimport path from 'path';\nimport { getClientIp } from 'request-ip';\nimport { browserName, detectOS } from 'detect-browser';\nimport isLocalhost from 'is-localhost-ip';\nimport maxmind from 'maxmind';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nimport {\n  DESKTOP_OS,\n  MOBILE_OS,\n  DESKTOP_SCREEN_WIDTH,\n  LAPTOP_SCREEN_WIDTH,\n  MOBILE_SCREEN_WIDTH,\n} from './constants';\nimport { NextApiRequestCollect } from 'pages/api/send';\n\nlet lookup;\n\nexport function getIpAddress(req: NextApiRequestCollect) {\n  const customHeader = String(process.env.CLIENT_IP_HEADER).toLowerCase();\n\n  // Custom header\n  if (customHeader !== 'undefined' && req.headers[customHeader]) {\n    return req.headers[customHeader];\n  }\n  // Cloudflare\n  else if (req.headers['cf-connecting-ip']) {\n    return req.headers['cf-connecting-ip'];\n  }\n\n  return getClientIp(req);\n}\n\nexport function getDevice(screen: string, os: string) {\n  if (!screen) return;\n\n  const [width] = screen.split('x');\n\n  if (DESKTOP_OS.includes(os)) {\n    if (os === 'Chrome OS' || +width < DESKTOP_SCREEN_WIDTH) {\n      return 'laptop';\n    }\n    return 'desktop';\n  } else if (MOBILE_OS.includes(os)) {\n    if (os === 'Amazon OS' || +width > MOBILE_SCREEN_WIDTH) {\n      return 'tablet';\n    }\n    return 'mobile';\n  }\n\n  if (+width >= DESKTOP_SCREEN_WIDTH) {\n    return 'desktop';\n  } else if (+width >= LAPTOP_SCREEN_WIDTH) {\n    return 'laptop';\n  } else if (+width >= MOBILE_SCREEN_WIDTH) {\n    return 'tablet';\n  } else {\n    return 'mobile';\n  }\n}\n\nfunction getRegionCode(country: string, region: string) {\n  if (!country || !region) {\n    return undefined;\n  }\n\n  return region.includes('-') ? region : `${country}-${region}`;\n}\n\nexport async function getLocation(ip: string, req: NextApiRequestCollect) {\n  // Ignore local ips\n  if (await isLocalhost(ip)) {\n    return;\n  }\n\n  // Cloudflare headers\n  if (req.headers['cf-ipcountry']) {\n    const country = safeDecodeURIComponent(req.headers['cf-ipcountry']);\n    const subdivision1 = safeDecodeURIComponent(req.headers['cf-region-code']);\n    const city = safeDecodeURIComponent(req.headers['cf-ipcity']);\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      city,\n    };\n  }\n\n  // Vercel headers\n  if (req.headers['x-vercel-ip-country']) {\n    const country = safeDecodeURIComponent(req.headers['x-vercel-ip-country']);\n    const subdivision1 = safeDecodeURIComponent(req.headers['x-vercel-ip-country-region']);\n    const city = safeDecodeURIComponent(req.headers['x-vercel-ip-city']);\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      city,\n    };\n  }\n\n  // Database lookup\n  if (!lookup) {\n    const dir = path.join(process.cwd(), 'geo');\n\n    lookup = await maxmind.open(path.resolve(dir, 'GeoLite2-City.mmdb'));\n  }\n\n  const result = lookup.get(ip);\n\n  if (result) {\n    const country = result.country?.iso_code ?? result?.registered_country?.iso_code;\n    const subdivision1 = result.subdivisions?.[0]?.iso_code;\n    const subdivision2 = result.subdivisions?.[1]?.names?.en;\n    const city = result.city?.names?.en;\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      subdivision2,\n      city,\n    };\n  }\n}\n\nexport async function getClientInfo(req: NextApiRequestCollect, { screen }) {\n  const userAgent = req.headers['user-agent'];\n  const ip = req.body.payload.ip || getIpAddress(req);\n  const location = await getLocation(ip, req);\n  const country = location?.country;\n  const subdivision1 = location?.subdivision1;\n  const subdivision2 = location?.subdivision2;\n  const city = location?.city;\n  const browser = browserName(userAgent);\n  const os = detectOS(userAgent);\n  const device = getDevice(screen, os);\n\n  return { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device };\n}\n\n// Path: src/lib/auth.ts\nimport { Report } from '@prisma/client';\nimport redis from '@umami/redis-client';\nimport debug from 'debug';\nimport { PERMISSIONS, ROLE_PERMISSIONS, SHARE_TOKEN_HEADER, ROLES } from 'lib/constants';\nimport { secret } from 'lib/crypto';\nimport { NextApiRequest } from 'next';\nimport { createSecureToken, ensureArray, getRandomChars, parseToken } from 'next-basics';\nimport { getTeamUser, getWebsite } from 'queries';\nimport { Auth } from './types';\n\nconst log = debug('umami:auth');\nconst cloudMode = process.env.CLOUD_MODE;\n\nexport async function saveAuth(data: any, expire = 0) {\n  const authKey = `auth:${getRandomChars(32)}`;\n\n  await redis.client.set(authKey, data);\n\n  if (expire) {\n    await redis.client.expire(authKey, expire);\n  }\n\n  return createSecureToken({ authKey }, secret());\n}\n\nexport function getAuthToken(req: NextApiRequest) {\n  try {\n    return req.headers.authorization.split(' ')[1];\n  } catch {\n    return null;\n  }\n}\n\nexport function parseShareToken(req: Request) {\n  try {\n    return parseToken(req.headers[SHARE_TOKEN_HEADER], secret());\n  } catch (e) {\n    log(e);\n    return null;\n  }\n}\n\nexport async function canViewWebsite({ user, shareToken }: Auth, websiteId: string) {\n  if (user?.isAdmin) {\n    return true;\n  }\n\n  if (shareToken?.websiteId === websiteId) {\n    return true;\n  }\n\n  const website = await getWebsite(websiteId);\n\n  if (website.userId) {\n    return user.id === website.userId;\n  }\n\n  if (website.teamId) {\n    const teamUser = await getTeamUser(website.teamId, user.id);\n\n    return !!teamUser;\n  }\n\n  return false;\n}\n\nexport async function canViewAllWebsites({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function canCreateWebsite({ user, grant }: Auth) {\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.websiteCreate);\n  }\n\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return hasPermission(user.role, PERMISSIONS.websiteCreate);\n}\n\nexport async function canUpdateWebsite({ user }: Auth, websiteId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const website = await getWebsite(websiteId);\n\n  if (website.userId) {\n    return user.id === website.userId;\n  }\n\n  if (website.teamId) {\n    const teamUser = await getTeamUser(website.teamId, user.id);\n\n    return teamUser && hasPermission(teamUser.role, PERMISSIONS.websiteUpdate);\n  }\n\n  return false;\n}\n\nexport async function canTransferWebsiteToUser({ user }: Auth, websiteId: string, userId: string) {\n  const website = await getWebsite(websiteId);\n\n  if (website.teamId && user.id === userId) {\n    const teamUser = await getTeamUser(website.teamId, userId);\n\n    return teamUser?.role === ROLES.teamOwner;\n  }\n\n  return false;\n}\n\nexport async function canTransferWebsiteToTeam({ user }: Auth, websiteId: string, teamId: string) {\n  const website = await getWebsite(websiteId);\n\n  if (website.userId && website.userId === user.id) {\n    const teamUser = await getTeamUser(teamId, user.id);\n\n    return teamUser?.role === ROLES.teamOwner;\n  }\n\n  return false;\n}\n\nexport async function canDeleteWebsite({ user }: Auth, websiteId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const website = await getWebsite(websiteId);\n\n  if (website.userId) {\n    return user.id === website.userId;\n  }\n\n  if (website.teamId) {\n    const teamUser = await getTeamUser(website.teamId, user.id);\n\n    return teamUser && hasPermission(teamUser.role, PERMISSIONS.websiteDelete);\n  }\n\n  return false;\n}\n\nexport async function canViewReport(auth: Auth, report: Report) {\n  if (auth.user.isAdmin) {\n    return true;\n  }\n\n  if (auth.user.id == report.userId) {\n    return true;\n  }\n\n  return !!(await canViewWebsite(auth, report.websiteId));\n}\n\nexport async function canUpdateReport({ user }: Auth, report: Report) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return user.id == report.userId;\n}\n\nexport async function canDeleteReport(auth: Auth, report: Report) {\n  return canUpdateReport(auth, report);\n}\n\nexport async function canCreateTeam({ user, grant }: Auth) {\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.teamCreate);\n  }\n\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return !!user;\n}\n\nexport async function canViewTeam({ user }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return getTeamUser(teamId, user.id);\n}\n\nexport async function canUpdateTeam({ user, grant }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.teamUpdate);\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamUpdate);\n}\n\nexport async function canAddUserToTeam({ user, grant }: Auth) {\n  if (cloudMode) {\n    return !!grant?.find(a => a === PERMISSIONS.teamUpdate);\n  }\n\n  return user.isAdmin;\n}\n\nexport async function canDeleteTeam({ user }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamDelete);\n}\n\nexport async function canDeleteTeamUser({ user }: Auth, teamId: string, removeUserId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  if (removeUserId === user.id) {\n    return true;\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.teamUpdate);\n}\n\nexport async function canCreateTeamWebsite({ user }: Auth, teamId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  const teamUser = await getTeamUser(teamId, user.id);\n\n  return teamUser && hasPermission(teamUser.role, PERMISSIONS.websiteCreate);\n}\n\nexport async function canCreateUser({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function canViewUser({ user }: Auth, viewedUserId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return user.id === viewedUserId;\n}\n\nexport async function canViewUsers({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function canUpdateUser({ user }: Auth, viewedUserId: string) {\n  if (user.isAdmin) {\n    return true;\n  }\n\n  return user.id === viewedUserId;\n}\n\nexport async function canDeleteUser({ user }: Auth) {\n  return user.isAdmin;\n}\n\nexport async function hasPermission(role: string, permission: string | string[]) {\n  return ensureArray(permission).some(e => ROLE_PERMISSIONS[role]?.includes(e));\n}\n\n// Path: src/lib/middleware.ts\nimport cors from 'cors';\nimport debug from 'debug';\nimport redis from '@umami/redis-client';\nimport { getAuthToken, parseShareToken } from 'lib/auth';\nimport { ROLES } from 'lib/constants';\nimport { secret } from 'lib/crypto';\nimport { findSession } from 'lib/session';\nimport {\n  badRequest,\n  createMiddleware,\n  forbidden,\n  parseSecureToken,\n  tooManyRequest,\n  unauthorized,\n} from 'next-basics';\nimport { NextApiRequestCollect } from 'pages/api/send';\nimport { getUser } from '../queries';\n\nconst log = debug('umami:middleware');\n\nexport const useCors = createMiddleware(\n  cors({\n    // Cache CORS preflight request 24 hours by default\n    maxAge: Number(process.env.CORS_MAX_AGE) || 86400,\n  }),\n);\n\nexport const useSession = createMiddleware(async (req, res, next) => {\n  try {\n...\n", "needle_token_start": 8824, "needle_token_end": 8928, "code_context_ntokens": 16091, "output": ["```typescript\n// Path: src/queries/analytics/getWebsiteStats.ts\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { QueryFilters } from 'lib/types';\n\nexport \nasync function getWebsiteStats(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(case when t.c = 1 then 1 else 0 end) as \"bounces\",\n      sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as \"totaltime\"\n    from (\n      select\n        website_event.session_id,\n        website_event.visit_id,\n        count(*) as \"c\",\n        min(website_event.created_at) as \"min_time\",\n        max(website_event.created_at) as \"max_time\"\n      from website_event\n      join website \n        on website_event.website_id = website.website_id\n        ${joinSession}\n      where website.website_id = {{websiteId::uuid}}\n        and website_event.created_at between {{startDate}} and {{endDate}}\n        and event_type = {{eventType}}\n        ${filterQuery}\n      group by 1, 2\n    ) as t\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(if(t.c = 1, 1, 0)) as \"bounces\",\n      sum(max_time-min_time) as \"totaltime\"\n    from (\n      select\n        session_id,\n        visit_id,\n        count(*) c,\n        min(created_at) min_time,\n        max(created_at) max_time\n      from website_event\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n        and event_type = {eventType:UInt32}\n        ${filterQuery}\n      group by session_id, visit_id\n    ) as t;\n    `,\n    params,\n  ).then(result => {\n    return Object.values(result).map(n => {\n      return {\n        pageviews: Number(n.pageviews),\n        visitors: Number(n.visitors),\n        visits: Number(n.visits),\n        bounces: Number(n.bounces),\n        totaltime: Number(n.totaltime),\n      };\n    });\n  });\n}\n```"]}
{"repo": "umami-software/umami", "name": "useLanguageNames", "language": "typescript", "path": "src/components/hooks/useLanguageNames.ts", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To provide a list of language names based on the specified locale, ensuring the data is up-to-date by fetching it from a server if not already cached.\n2. **Input**: A string representing the locale (e.g., 'en-US').\n3. **Output**: A list of language names appropriate to the given locale.\n4. **Procedure**: \n   - Initially, the function checks if the language data for the specified locale is already available in a local cache.\n   - If the data is not in the cache, it fetches the data asynchronously from a server using a specified endpoint and updates the cache.\n   - The function then sets the fetched or cached data as the state, which is returned to the component that invoked this function.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/components/hooks/queries/useWebsite.ts\nimport useApi from './useApi';\n\nexport function useWebsite(websiteId: string, options?: { [key: string]: any }) {\n  const { get, useQuery } = useApi();\n\n  return useQuery({\n    queryKey: ['website', { websiteId }],\n...\n// Path: src/components/hooks/queries/useFilterQuery.ts\nimport { UseQueryOptions } from '@tanstack/react-query';\nimport { useState } from 'react';\nimport { useApi } from './useApi';\nimport { FilterResult, SearchFilter, FilterQueryResult } from 'lib/types';\n\nexport function useFilterQuery<T = any>({\n  queryKey,\n  queryFn,\n  ...options\n}: Omit<UseQueryOptions, 'queryFn'> & { queryFn: (params?: object) => any }): FilterQueryResult<T> {\n  const [params, setParams] = useState<T | SearchFilter>({\n    query: '',\n    page: 1,\n  });\n\n  const { useQuery } = useApi();\n  const { data, ...query } = useQuery({\n    queryKey: [{ ...queryKey, ...params }],\n    queryFn: () => queryFn(params as any),\n    ...options,\n  });\n\n  return {\n    result: data as FilterResult<any>,\n    query,\n    params,\n    setParams,\n  };\n}\n\nexport default useFilterQuery;\n\n// Path: src/components/hooks/useSticky.ts\nimport { useState, useEffect, useRef } from 'react';\n\nexport function useSticky({ enabled = true, threshold = 1 }) {\n  const [isSticky, setIsSticky] = useState(false);\n  const ref = useRef(null);\n\n  useEffect(() => {\n    let observer: IntersectionObserver | undefined;\n    const handler: IntersectionObserverCallback = ([entry]) =>\n      setIsSticky(entry.intersectionRatio < threshold);\n\n    if (enabled && ref.current) {\n      observer = new IntersectionObserver(handler, { threshold: [threshold] });\n      observer.observe(ref.current);\n    }\n    return () => {\n      if (observer) {\n        observer.disconnect();\n      }\n    };\n  }, [ref, enabled, threshold]);\n\n  return { ref, isSticky };\n}\n\nexport default useSticky;\n\n// Path: src/components/hooks/useForceUpdate.ts\nimport { useCallback, useState } from 'react';\n\nexport function useForceUpdate() {\n  const [, update] = useState(Object.create(null));\n\n  return useCallback(() => {\n    update(Object.create(null));\n  }, [update]);\n}\n\nexport default useForceUpdate;\n\n// Path: src/components/hooks/useLocale.ts\nimport { useEffect } from 'react';\nimport { httpGet, setItem } from 'next-basics';\nimport { LOCALE_CONFIG } from 'lib/constants';\nimport { getDateLocale, getTextDirection } from 'lib/lang';\nimport useStore, { setLocale } from 'store/app';\nimport { useForceUpdate } from './useForceUpdate';\nimport enUS from '../../../public/intl/country/en-US.json';\n\nconst messages = {\n  'en-US': enUS,\n};\n\nconst selector = (state: { locale: any }) => state.locale;\n\nexport function useLocale() {\n  const locale = useStore(selector);\n  const forceUpdate = useForceUpdate();\n  const dir = getTextDirection(locale);\n  const dateLocale = getDateLocale(locale);\n\n  async function loadMessages(locale: string) {\n    const { ok, data } = await httpGet(`${process.env.basePath}/intl/messages/${locale}.json`);\n\n    if (ok) {\n      messages[locale] = data;\n    }\n  }\n\n  async function saveLocale(value: string) {\n    if (!messages[value]) {\n      await loadMessages(value);\n    }\n\n    setItem(LOCALE_CONFIG, value);\n\n    document.getElementById('__next')?.setAttribute('dir', getTextDirection(value));\n\n    if (locale !== value) {\n      setLocale(value);\n    } else {\n      forceUpdate();\n    }\n  }\n\n  useEffect(() => {\n    if (!messages[locale]) {\n      saveLocale(locale);\n    }\n  }, [locale]);\n\n  useEffect(() => {\n    const url = new URL(window?.location?.href);\n    const locale = url.searchParams.get('locale');\n\n    if (locale) {\n      saveLocale(locale);\n    }\n  }, []);\n\n  return { locale, saveLocale, messages, dir, dateLocale };\n}\n\nexport default useLocale;\n\n// Path: src/components/hooks/useNavigation.ts\nimport { useMemo } from 'react';\nimport { usePathname, useRouter, useSearchParams } from 'next/navigation';\nimport { buildUrl, safeDecodeURIComponent } from 'next-basics';\n\nexport function useNavigation(): {\n  pathname: string;\n  query: { [key: string]: string };\n  router: any;\n  renderUrl: (params: any, reset?: boolean) => string;\n} {\n  const router = useRouter();\n  const pathname = usePathname();\n  const params = useSearchParams();\n\n  const query = useMemo(() => {\n    const obj = {};\n\n    for (const [key, value] of params.entries()) {\n      obj[key] = safeDecodeURIComponent(value);\n    }\n\n    return obj;\n  }, [params]);\n\n  function renderUrl(params: any, reset?: boolean) {\n    return reset ? pathname : buildUrl(pathname, { ...query, ...params });\n  }\n\n  return { pathname, query, router, renderUrl };\n}\n\nexport default useNavigation;\n\n// Path: src/components/hooks/queries/useShareToken.ts\nimport useStore, { setShareToken } from 'store/app';\nimport useApi from './useApi';\n\nconst selector = (state: { shareToken: string }) => state.shareToken;\n\nexport function useShareToken(shareId: string): {\n  shareToken: any;\n  isLoading?: boolean;\n  error?: Error;\n} {\n  const shareToken = useStore(selector);\n  const { get, useQuery } = useApi();\n  const { isLoading, error } = useQuery({\n    queryKey: ['share', shareId],\n    queryFn: async () => {\n      const data = await get(`/share/${shareId}`);\n\n      setShareToken(data);\n\n      return data;\n    },\n  });\n\n  return { shareToken, isLoading, error };\n}\n\nexport default useShareToken;\n\n// Path: src/store/modified.ts\nimport { create } from 'zustand';\n\nconst store = create(() => ({}));\n\nexport function touch(key: string) {\n  store.setState({ [key]: Date.now() });\n}\n\nexport default store;\n\n// Path: src/components/hooks/useModified.ts\nimport useStore from 'store/modified';\n\nexport function useModified(key?: string) {\n  const modified = useStore(state => state?.[key]);\n\n  const touch = (id?: string) => {\n    if (id || key) {\n      useStore.setState({ [id || key]: Date.now() });\n    }\n  };\n\n  return { modified, touch };\n}\n\nexport default useModified;\n\n// Path: src/components/hooks/queries/useLogin.ts\nimport useStore, { setUser } from 'store/app';\nimport useApi from './useApi';\nimport { UseQueryResult } from '@tanstack/react-query';\n\nconst selector = (state: { user: any }) => state.user;\n\nexport function useLogin(): {\n  user: any;\n  setUser: (data: any) => void;\n} & UseQueryResult {\n  const { get, useQuery } = useApi();\n  const user = useStore(selector);\n\n  const query = useQuery({\n    queryKey: ['login'],\n    queryFn: async () => {\n      const data = await get('/auth/verify');\n\n      setUser(data);\n\n      return data;\n    },\n    enabled: !user,\n  });\n\n  return { user, setUser, ...query };\n}\n\nexport default useLogin;\n\n// Path: src/components/hooks/queries/useWebsites.ts\nimport { useApi } from './useApi';\nimport { useFilterQuery } from './useFilterQuery';\nimport { useLogin } from './useLogin';\nimport useModified from '../useModified';\n\nexport function useWebsites(\n  { userId, teamId }: { userId?: string; teamId?: string },\n  params?: { [key: string]: string | number },\n) {\n  const { get } = useApi();\n  const { user } = useLogin();\n  const { modified } = useModified(`websites`);\n\n  return useFilterQuery({\n    queryKey: ['websites', { userId, teamId, modified, ...params }],\n    queryFn: (data: any) => {\n      return get(teamId ? `/teams/${teamId}/websites` : `/users/${userId || user.id}/websites`, {\n        ...data,\n        ...params,\n      });\n    },\n  });\n}\n\nexport default useWebsites;\n\n// Path: src/components/hooks/queries/useTeam.ts\nimport useApi from './useApi';\n\nexport function useTeam(teamId: string) {\n  const { get, useQuery } = useApi();\n  return useQuery({\n    queryKey: ['teams', teamId],\n    queryFn: () => get(`/teams/${teamId}`),\n    enabled: !!teamId,\n  });\n}\n\nexport default useTeam;\n\n// Path: src/components/messages.ts\nimport { defineMessages } from 'react-intl';\n\nexport const labels = defineMessages({\n  ok: { id: 'label.ok', defaultMessage: 'OK' },\n  unknown: { id: 'label.unknown', defaultMessage: 'Unknown' },\n  required: { id: 'label.required', defaultMessage: 'Required' },\n  save: { id: 'label.save', defaultMessage: 'Save' },\n  cancel: { id: 'label.cancel', defaultMessage: 'Cancel' },\n  continue: { id: 'label.continue', defaultMessage: 'Continue' },\n  delete: { id: 'label.delete', defaultMessage: 'Delete' },\n  leave: { id: 'label.leave', defaultMessage: 'Leave' },\n  users: { id: 'label.users', defaultMessage: 'Users' },\n  createUser: { id: 'label.create-user', defaultMessage: 'Create user' },\n  deleteUser: { id: 'label.delete-user', defaultMessage: 'Delete user' },\n  username: { id: 'label.username', defaultMessage: 'Username' },\n  password: { id: 'label.password', defaultMessage: 'Password' },\n  role: { id: 'label.role', defaultMessage: 'Role' },\n  admin: { id: 'label.admin', defaultMessage: 'Admin' },\n  user: { id: 'label.user', defaultMessage: 'User' },\n  viewOnly: { id: 'label.view-only', defaultMessage: 'View only' },\n  manage: { id: 'label.manage', defaultMessage: 'Manage' },\n  admin: { id: 'label.admin', defaultMessage: 'Administrator' },\n  confirm: { id: 'label.confirm', defaultMessage: 'Confirm' },\n  details: { id: 'label.details', defaultMessage: 'Details' },\n  website: { id: 'label.website', defaultMessage: 'Website' },\n  websites: { id: 'label.websites', defaultMessage: 'Websites' },\n  myWebsites: { id: 'label.my-websites', defaultMessage: 'My websites' },\n  teamWebsites: { id: 'label.team-websites', defaultMessage: 'Team websites' },\n  created: { id: 'label.created', defaultMessage: 'Created' },\n  createdBy: { id: 'label.created-by', defaultMessage: 'Created By' },\n  edit: { id: 'label.edit', defaultMessage: 'Edit' },\n  name: { id: 'label.name', defaultMessage: 'Name' },\n  member: { id: 'label.member', defaultMessage: 'Member' },\n  members: { id: 'label.members', defaultMessage: 'Members' },\n  accessCode: { id: 'label.access-code', defaultMessage: 'Access code' },\n  teamId: { id: 'label.team-id', defaultMessage: 'Team ID' },\n  team: { id: 'label.team', defaultMessage: 'Team' },\n  teamName: { id: 'label.team-name', defaultMessage: 'Team name' },\n  regenerate: { id: 'label.regenerate', defaultMessage: 'Regenerate' },\n  remove: { id: 'label.remove', defaultMessage: 'Remove' },\n  join: { id: 'label.join', defaultMessage: 'Join' },\n  createTeam: { id: 'label.create-team', defaultMessage: 'Create team' },\n  joinTeam: { id: 'label.join-team', defaultMessage: 'Join team' },\n  settings: { id: 'label.settings', defaultMessage: 'Settings' },\n  owner: { id: 'label.owner', defaultMessage: 'Owner' },\n  teamOwner: { id: 'label.team-owner', defaultMessage: 'Team owner' },\n  teamMember: { id: 'label.team-member', defaultMessage: 'Team member' },\n  teamViewOnly: { id: 'label.team-view-only', defaultMessage: 'Team view only' },\n  enableShareUrl: { id: 'label.enable-share-url', defaultMessage: 'Enable share URL' },\n  data: { id: 'label.data', defaultMessage: 'Data' },\n  trackingCode: { id: 'label.tracking-code', defaultMessage: 'Tracking code' },\n  shareUrl: { id: 'label.share-url', defaultMessage: 'Share URL' },\n  actions: { id: 'label.actions', defaultMessage: 'Actions' },\n  domain: { id: 'label.domain', defaultMessage: 'Domain' },\n  websiteId: { id: 'label.website-id', defaultMessage: 'Website ID' },\n  resetWebsite: { id: 'label.reset-website', defaultMessage: 'Reset website' },\n  deleteWebsite: { id: 'label.delete-website', defaultMessage: 'Delete website' },\n  transferWebsite: { id: 'label.transfer-website', defaultMessage: 'Transfer website' },\n  deleteReport: { id: 'label.delete-report', defaultMessage: 'Delete report' },\n  reset: { id: 'label.reset', defaultMessage: 'Reset' },\n  addWebsite: { id: 'label.add-website', defaultMessage: 'Add website' },\n  addMember: { id: 'label.add-member', defaultMessage: 'Add member' },\n  editMember: { id: 'label.edit-member', defaultMessage: 'Edit member' },\n  removeMember: { id: 'label.remove-member', defaultMessage: 'Remove member' },\n  addDescription: { id: 'label.add-description', defaultMessage: 'Add description' },\n  changePassword: { id: 'label.change-password', defaultMessage: 'Change password' },\n  currentPassword: { id: 'label.current-password', defaultMessage: 'Current password' },\n  newPassword: { id: 'label.new-password', defaultMessage: 'New password' },\n  confirmPassword: { id: 'label.confirm-password', defaultMessage: 'Confirm password' },\n  timezone: { id: 'label.timezone', defaultMessage: 'Timezone' },\n  defaultDateRange: { id: 'label.default-date-range', defaultMessage: 'Default date range' },\n  language: { id: 'label.language', defaultMessage: 'Language' },\n  theme: { id: 'label.theme', defaultMessage: 'Theme' },\n  profile: { id: 'label.profile', defaultMessage: 'Profile' },\n  dashboard: { id: 'label.dashboard', defaultMessage: 'Dashboard' },\n  more: { id: 'label.more', defaultMessage: 'More' },\n  realtime: { id: 'label.realtime', defaultMessage: 'Realtime' },\n  queries: { id: 'label.queries', defaultMessage: 'Queries' },\n  teams: { id: 'label.teams', defaultMessage: 'Teams' },\n  analytics: { id: 'label.analytics', defaultMessage: 'Analytics' },\n  login: { id: 'label.login', defaultMessage: 'Login' },\n  logout: { id: 'label.logout', defaultMessage: 'Logout' },\n  singleDay: { id: 'label.single-day', defaultMessage: 'Single day' },\n  dateRange: { id: 'label.date-range', defaultMessage: 'Date range' },\n  viewDetails: { id: 'label.view-details', defaultMessage: 'View details' },\n  deleteTeam: { id: 'label.delete-team', defaultMessage: 'Delete team' },\n  leaveTeam: { id: 'label.leave-team', defaultMessage: 'Leave team' },\n  refresh: { id: 'label.refresh', defaultMessage: 'Refresh' },\n  pages: { id: 'label.pages', defaultMessage: 'Pages' },\n  referrers: { id: 'label.referrers', defaultMessage: 'Referrers' },\n  screens: { id: 'label.screens', defaultMessage: 'Screens' },\n  browsers: { id: 'label.browsers', defaultMessage: 'Browsers' },\n  os: { id: 'label.os', defaultMessage: 'OS' },\n  devices: { id: 'label.devices', defaultMessage: 'Devices' },\n  countries: { id: 'label.countries', defaultMessage: 'Countries' },\n  languages: { id: 'label.languages', defaultMessage: 'Languages' },\n  event: { id: 'label.event', defaultMessage: 'Event' },\n  events: { id: 'label.events', defaultMessage: 'Events' },\n  query: { id: 'label.query', defaultMessage: 'Query' },\n  queryParameters: { id: 'label.query-parameters', defaultMessage: 'Query parameters' },\n  back: { id: 'label.back', defaultMessage: 'Back' },\n  visitors: { id: 'label.visitors', defaultMessage: 'Visitors' },\n  visits: { id: 'label.visits', defaultMessage: 'Visits' },\n  filterCombined: { id: 'label.filter-combined', defaultMessage: 'Combined' },\n  filterRaw: { id: 'label.filter-raw', defaultMessage: 'Raw' },\n  views: { id: 'label.views', defaultMessage: 'Views' },\n  none: { id: 'label.none', defaultMessage: 'None' },\n  clearAll: { id: 'label.clear-all', defaultMessage: 'Clear all' },\n  today: { id: 'label.today', defaultMessage: 'Today' },\n  lastHours: { id: 'label.last-hours', defaultMessage: 'Last {x} hours' },\n  yesterday: { id: 'label.yesterday', defaultMessage: 'Yesterday' },\n  thisWeek: { id: 'label.this-week', defaultMessage: 'This week' },\n  lastDays: { id: 'label.last-days', defaultMessage: 'Last {x} days' },\n  lastMonths: { id: 'label.last-months', defaultMessage: 'Last {x} months' },\n  thisMonth: { id: 'label.this-month', defaultMessage: 'This month' },\n  thisYear: { id: 'label.this-year', defaultMessage: 'This year' },\n  allTime: { id: 'label.all-time', defaultMessage: 'All time' },\n  customRange: { id: 'label.custom-range', defaultMessage: 'Custom range' },\n  selectWebsite: { id: 'label.select-website', defaultMessage: 'Select website' },\n  selectRole: { id: 'label.select-role', defaultMessage: 'Select role' },\n  selectDate: { id: 'label.select-date', defaultMessage: 'Select date' },\n  all: { id: 'label.all', defaultMessage: 'All' },\n  sessions: { id: 'label.sessions', defaultMessage: 'Sessions' },\n  pageNotFound: { id: 'message.page-not-found', defaultMessage: 'Page not found' },\n  activityLog: { id: 'label.activity-log', defaultMessage: 'Activity log' },\n  dismiss: { id: 'label.dismiss', defaultMessage: 'Dismiss' },\n  poweredBy: { id: 'label.powered-by', defaultMessage: 'Powered by {name}' },\n  pageViews: { id: 'label.page-views', defaultMessage: 'Page views' },\n  uniqueVisitors: { id: 'label.unique-visitors', defaultMessage: 'Unique visitors' },\n  bounceRate: { id: 'label.bounce-rate', defaultMessage: 'Bounce rate' },\n  viewsPerVisit: { id: 'label.views-per-visit', defaultMessage: 'Views per visit' },\n  averageVisitTime: { id: 'label.average-visit-time', defaultMessage: 'Average visit time' },\n  desktop: { id: 'label.desktop', defaultMessage: 'Desktop' },\n  laptop: { id: 'label.laptop', defaultMessage: 'Laptop' },\n  tablet: { id: 'label.tablet', defaultMessage: 'Tablet' },\n  mobile: { id: 'label.mobile', defaultMessage: 'Mobile' },\n  toggleCharts: { id: 'label.toggle-charts', defaultMessage: 'Toggle charts' },\n  editDashboard: { id: 'label.edit-dashboard', defaultMessage: 'Edit dashboard' },\n  title: { id: 'label.title', defaultMessage: 'Title' },\n  view: { id: 'label.view', defaultMessage: 'View' },\n  cities: { id: 'label.cities', defaultMessage: 'Cities' },\n  regions: { id: 'label.regions', defaultMessage: 'Regions' },\n  reports: { id: 'label.reports', defaultMessage: 'Reports' },\n  eventData: { id: 'label.event-data', defaultMessage: 'Event data' },\n  funnel: { id: 'label.funnel', defaultMessage: 'Funnel' },\n  funnelDescription: {\n    id: 'label.funnel-description',\n    defaultMessage: 'Understand the conversion and drop-off rate of users.',\n  },\n  url: { id: 'label.url', defaultMessage: 'URL' },\n  urls: { id: 'label.urls', defaultMessage: 'URLs' },\n  add: { id: 'label.add', defaultMessage: 'Add' },\n  update: { id: 'label.update', defaultMessage: 'Update' },\n  window: { id: 'label.window', defaultMessage: 'Window' },\n  runQuery: { id: 'label.run-query', defaultMessage: 'Run query' },\n  field: { id: 'label.field', defaultMessage: 'Field' },\n  fields: { id: 'label.fields', defaultMessage: 'Fields' },\n  createReport: { id: 'label.create-report', defaultMessage: 'Create report' },\n  description: { id: 'label.description', defaultMessage: 'Description' },\n  untitled: { id: 'label.untitled', defaultMessage: 'Untitled' },\n  type: { id: 'label.type', defaultMessage: 'Type' },\n  filter: { id: 'label.filter', defaultMessage: 'Filter' },\n  filters: { id: 'label.filters', defaultMessage: 'Filters' },\n  breakdown: { id: 'label.breakdown', defaultMessage: 'Breakdown' },\n  true: { id: 'label.true', defaultMessage: 'True' },\n  false: { id: 'label.false', defaultMessage: 'False' },\n  is: { id: 'label.is', defaultMessage: 'Is' },\n  isNot: { id: 'label.is-not', defaultMessage: 'Is not' },\n  isSet: { id: 'label.is-set', defaultMessage: 'Is set' },\n  isNotSet: { id: 'label.is-not-set', defaultMessage: 'Is not set' },\n  greaterThan: { id: 'label.greater-than', defaultMessage: 'Greater than' },\n  lessThan: { id: 'label.less-than', defaultMessage: 'Less than' },\n  greaterThanEquals: { id: 'label.greater-than-equals', defaultMessage: 'Greater than or equals' },\n  lessThanEquals: { id: 'label.less-than-equals', defaultMessage: 'Less than or equals' },\n  contains: { id: 'label.contains', defaultMessage: 'Contains' },\n  doesNotContain: { id: 'label.does-not-contain', defaultMessage: 'Does not contain' },\n  before: { id: 'label.before', defaultMessage: 'Before' },\n  after: { id: 'label.after', defaultMessage: 'After' },\n  total: { id: 'label.total', defaultMessage: 'Total' },\n  sum: { id: 'label.sum', defaultMessage: 'Sum' },\n  average: { id: 'label.average', defaultMessage: 'Average' },\n  min: { id: 'label.min', defaultMessage: 'Min' },\n  max: { id: 'label.max', defaultMessage: 'Max' },\n  unique: { id: 'label.unique', defaultMessage: 'Unique' },\n  value: { id: 'label.value', defaultMessage: 'Value' },\n  overview: { id: 'label.overview', defaultMessage: 'Overview' },\n  totalRecords: { id: 'label.total-records', defaultMessage: 'Total records' },\n  insights: { id: 'label.insights', defaultMessage: 'Insights' },\n  insightsDescription: {\n    id: 'label.insights-description',\n    defaultMessage: 'Dive deeper into your data by using segments and filters.',\n  },\n  retention: { id: 'label.retention', defaultMessage: 'Retention' },\n  retentionDescription: {\n    id: 'label.retention-description',\n    defaultMessage: 'Measure your website stickiness by tracking how often users return.',\n  },\n  dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' },\n  referrer: { id: 'label.referrer', defaultMessage: 'Referrer' },\n  country: { id: 'label.country', defaultMessage: 'Country' },\n  region: { id: 'label.region', defaultMessage: 'Region' },\n  city: { id: 'label.city', defaultMessage: 'City' },\n  browser: { id: 'label.browser', defaultMessage: 'Browser' },\n  device: { id: 'label.device', defaultMessage: 'Device' },\n  pageTitle: { id: 'label.pageTitle', defaultMessage: 'Page title' },\n  day: { id: 'label.day', defaultMessage: 'Day' },\n  date: { id: 'label.date', defaultMessage: 'Date' },\n  pageOf: { id: 'label.page-of', defaultMessage: 'Page {current} of {total}' },\n  create: { id: 'label.create', defaultMessage: 'Create' },\n  search: { id: 'label.search', defaultMessage: 'Search' },\n  numberOfRecords: {\n    id: 'label.number-of-records',\n    defaultMessage: '{x} {x, plural, one {record} other {records}}',\n  },\n  select: { id: 'label.select', defaultMessage: 'Select' },\n  myAccount: { id: 'label.my-account', defaultMessage: 'My account' },\n  transfer: { id: 'label.transfer', defaultMessage: 'Transfer' },\n  viewedPage: {\n    id: 'message.viewed-page',\n    defaultMessage: 'Viewed page',\n  },\n  triggeredEvent: {\n    id: 'message.triggered-event',\n    defaultMessage: 'Triggered event',\n  },\n  visitorsDroppedOff: {\n    id: 'message.visitors-dropped-off',\n    defaultMessage: 'Visitors dropped off',\n  },\n  utm: { id: 'label.utm', defaultMessage: 'UTM' },\n  utmDescription: {\n    id: 'label.utm-description',\n    defaultMessage: 'Track your campaigns through UTM parameters.',\n  },\n  steps: { id: 'label.steps', defaultMessage: 'Steps' },\n  addStep: { id: 'label.add-step', defaultMessage: 'Add step' },\n});\n\nexport const messages = defineMessages({\n  error: { id: 'message.error', defaultMessage: 'Something went wrong.' },\n  saved: { id: 'message.saved', defaultMessage: 'Saved.' },\n  noUsers: { id: 'message.no-users', defaultMessage: 'There are no users.' },\n  userDeleted: { id: 'message.user-deleted', defaultMessage: 'User deleted.' },\n  noDataAvailable: { id: 'message.no-data-available', defaultMessage: 'No data available.' },\n  confirmReset: {\n    id: 'message.confirm-reset',\n    defaultMessage: 'Are you sure you want to reset {target}?',\n  },\n  confirmDelete: {\n    id: 'message.confirm-delete',\n    defaultMessage: 'Are you sure you want to delete {target}?',\n  },\n  confirmRemove: {\n    id: 'message.confirm-remove',\n    defaultMessage: 'Are you sure you want to remove {target}?',\n  },\n  confirmLeave: {\n    id: 'message.confirm-leave',\n    defaultMessage: 'Are you sure you want to leave {target}?',\n  },\n  minPasswordLength: {\n    id: 'message.min-password-length',\n    defaultMessage: 'Minimum length of {n} characters',\n  },\n  noTeams: {\n    id: 'message.no-teams',\n    defaultMessage: 'You have not created any teams.',\n  },\n  shareUrl: {\n    id: 'message.share-url',\n    defaultMessage: 'Your website stats are publicly available at the following URL:',\n  },\n  trackingCode: {\n    id: 'message.tracking-code',\n    defaultMessage:\n      'To track stats for this website, place the following code in the <head>...</head> section of your HTML.',\n  },\n  joinTeamWarning: {\n    id: 'message.team-already-member',\n    defaultMessage: 'You are already a member of the team.',\n  },\n  actionConfirmation: {\n    id: 'message.action-confirmation',\n    defaultMessage: 'Type {confirmation} in the box below to confirm.',\n  },\n  resetWebsite: {\n    id: 'message.reset-website',\n    defaultMessage: 'To reset this website, type {confirmation} in the box below to confirm.',\n  },\n  invalidDomain: {\n    id: 'message.invalid-domain',\n    defaultMessage: 'Invalid domain. Do not include http/https.',\n  },\n  resetWebsiteWarning: {\n    id: 'message.reset-website-warning',\n    defaultMessage:\n      'All statistics for this website will be deleted, but your settings will remain intact.',\n  },\n  deleteWebsiteWarning: {\n    id: 'message.delete-website-warning',\n    defaultMessage: 'All website data will be deleted.',\n  },\n  deleteTeamWarning: {\n    id: 'message.delete-team-warning',\n    defaultMessage: 'Deleting a team will also delete all team websites.',\n  },\n  noResultsFound: {\n    id: 'message.no-results-found',\n    defaultMessage: 'No results found.',\n  },\n  noWebsitesConfigured: {\n    id: 'message.no-websites-configured',\n    defaultMessage: 'You do not have any websites configured.',\n  },\n  noTeamWebsites: {\n    id: 'message.no-team-websites',\n    defaultMessage: 'This team does not have any websites.',\n  },\n  teamWebsitesInfo: {\n    id: 'message.team-websites-info',\n    defaultMessage: 'Websites can be viewed by anyone on the team.',\n  },\n  noMatchPassword: { id: 'message.no-match-password', defaultMessage: 'Passwords do not match.' },\n  goToSettings: {\n    id: 'message.go-to-settings',\n    defaultMessage: 'Go to settings',\n  },\n  activeUsers: {\n    id: 'message.active-users',\n    defaultMessage: '{x} current {x, plural, one {visitor} other {visitors}}',\n  },\n  teamNotFound: {\n    id: 'message.team-not-found',\n    defaultMessage: 'Team not found.',\n  },\n  visitorLog: {\n    id: 'message.visitor-log',\n    defaultMessage: 'Visitor from {country} using {browser} on {os} {device}',\n  },\n  eventLog: {\n    id: 'message.event-log',\n    defaultMessage: '{event} on {url}',\n  },\n  incorrectUsernamePassword: {\n    id: 'message.incorrect-username-password',\n    defaultMessage: 'Incorrect username and/or password.',\n  },\n  noEventData: {\n    id: 'message.no-event-data',\n    defaultMessage: 'No event data is available.',\n  },\n  newVersionAvailable: {\n    id: 'message.new-version-available',\n    defaultMessage: 'A new version of Umami {version} is available!',\n  },\n  transferWebsite: {\n    id: 'message.transfer-website',\n    defaultMessage: 'Transfer website ownership to your account or another team.',\n  },\n  transferTeamWebsiteToUser: {\n    id: 'message.transfer-team-website-to-user',\n    defaultMessage: 'Transfer this website to your account?',\n  },\n  transferUserWebsiteToTeam: {\n    id: 'message.transfer-user-website-to-team',\n    defaultMessage: 'Select the team to transfer this website to.',\n  },\n});\n\n// Path: src/components/hooks/useMessages.ts\nimport { useIntl, FormattedMessage } from 'react-intl';\nimport { messages, labels } from 'components/messages';\n\nexport function useMessages(): any {\n  const intl = useIntl();\n\n  const getMessage = (id: string) => {\n    const message = Object.values(messages).find(value => value.id === id);\n\n    return message ? formatMessage(message) : id;\n  };\n\n  const formatMessage = (\n    descriptor: {\n      id: string;\n      defaultMessage: string;\n    },\n    values?: { [key: string]: string },\n    opts?: any,\n  ) => {\n    return descriptor ? intl.formatMessage(descriptor, values, opts) : null;\n  };\n\n  return { formatMessage, FormattedMessage, messages, labels, getMessage };\n}\n\nexport default useMessages;\n\n// Path: src/components/hooks/useTimezone.ts\nimport { setItem } from 'next-basics';\nimport { TIMEZONE_CONFIG } from 'lib/constants';\nimport useStore, { setTimezone } from 'store/app';\n\nconst selector = (state: { timezone: string }) => state.timezone;\n\nexport function useTimezone() {\n  const timezone = useStore(selector);\n\n  const saveTimezone = (value: string) => {\n    setItem(TIMEZONE_CONFIG, value);\n    setTimezone(value);\n  };\n\n  return { timezone, saveTimezone };\n}\n\nexport default useTimezone;\n\n// Path: src/components/hooks/queries/useReport.ts\nimport { produce } from 'immer';\nimport { useCallback, useEffect, useState } from 'react';\nimport { useApi } from './useApi';\nimport { useTimezone } from '../useTimezone';\nimport { useMessages } from '../useMessages';\n\nexport function useReport(\n  reportId: string,\n  defaultParameters: { type: string; parameters: { [key: string]: any } },\n) {\n  const [report, setReport] = useState(null);\n  const [isRunning, setIsRunning] = useState(false);\n  const { get, post } = useApi();\n  const { timezone } = useTimezone();\n  const { formatMessage, labels } = useMessages();\n\n  const baseParameters = {\n    name: formatMessage(labels.untitled),\n    description: '',\n    parameters: {},\n  };\n\n  const loadReport = async (id: string) => {\n    const data: any = await get(`/reports/${id}`);\n\n    const { dateRange } = data?.parameters || {};\n    const { startDate, endDate } = dateRange || {};\n\n    if (startDate && endDate) {\n      dateRange.startDate = new Date(startDate);\n      dateRange.endDate = new Date(endDate);\n    }\n\n    data.parameters = { ...defaultParameters?.parameters, ...data.parameters };\n\n    setReport(data);\n  };\n\n  const runReport = useCallback(\n    async (parameters: { [key: string]: any }) => {\n      setIsRunning(true);\n\n      const { type } = report;\n\n      const data = await post(`/reports/${type}`, { ...parameters, timezone });\n\n      setReport(\n        produce((state: any) => {\n          state.parameters = { ...defaultParameters?.parameters, ...parameters };\n          state.data = data;\n\n          return state;\n        }),\n      );\n\n      setIsRunning(false);\n    },\n    [report, timezone],\n  );\n\n  const updateReport = useCallback(\n    async (data: { [x: string]: any; parameters: any }) => {\n      setReport(\n        produce((state: any) => {\n          const { parameters, ...rest } = data;\n\n          if (parameters) {\n            state.parameters = {\n              ...defaultParameters?.parameters,\n              ...state.parameters,\n              ...parameters,\n            };\n          }\n\n          for (const key in rest) {\n            state[key] = rest[key];\n          }\n\n          return state;\n        }),\n      );\n    },\n    [report],\n  );\n\n  useEffect(() => {\n    if (!reportId) {\n      setReport({ ...baseParameters, ...defaultParameters });\n    } else {\n      loadReport(reportId);\n    }\n  }, [reportId]);\n\n  return { report, runReport, updateReport, isRunning };\n}\n\nexport default useReport;\n\n// Path: src/components/hooks/queries/useUser.ts\nimport useApi from './useApi';\n\nexport function useUser(userId: string, options?: { [key: string]: any }) {\n  const { get, useQuery } = useApi();\n  return useQuery({\n    queryKey: ['users', userId],\n    queryFn: () => get(`/users/${userId}`),\n    enabled: !!userId,\n    ...options,\n  });\n}\n\nexport default useUser;\n\n// Path: src/components/hooks/queries/useTeamWebsites.ts\nimport useApi from './useApi';\nimport useFilterQuery from './useFilterQuery';\nimport useModified from '../useModified';\n\nexport function useTeamWebsites(teamId: string) {\n  const { get } = useApi();\n  const { modified } = useModified(`websites`);\n\n  return useFilterQuery({\n    queryKey: ['teams:websites', { teamId, modified }],\n    queryFn: (params: any) => {\n      return get(`/teams/${teamId}/websites`, params);\n    },\n  });\n}\n\nexport default useTeamWebsites;\n\n// Path: src/components/hooks/useDocumentClick.ts\nimport { useEffect } from 'react';\n\nexport function useDocumentClick(handler: (event: MouseEvent) => any) {\n  useEffect(() => {\n    document.addEventListener('click', handler);\n\n    return () => {\n      document.removeEventListener('click', handler);\n    };\n  }, [handler]);\n\n  return null;\n}\n\nexport default useDocumentClick;\n\n// Path: src/components/hooks/useCountryNames.ts\nimport { useState, useEffect } from 'react';\nimport { httpGet } from 'next-basics';\nimport enUS from '../../../public/intl/country/en-US.json';\n\nconst countryNames = {\n  'en-US': enUS,\n};\n\nexport function useCountryNames(locale: string) {\n  const [list, setList] = useState(countryNames[locale] || enUS);\n\n  async function loadData(locale: string) {\n    const { data } = await httpGet(`${process.env.basePath}/intl/country/${locale}.json`);\n\n    if (data) {\n      countryNames[locale] = data;\n      setList(countryNames[locale]);\n    } else {\n      setList(enUS);\n    }\n  }\n\n  useEffect(() => {\n    if (!countryNames[locale]) {\n      loadData(locale);\n    } else {\n      setList(countryNames[locale]);\n    }\n  }, [locale]);\n\n  return list;\n}\n\nexport default useCountryNames;\n\n// Path: src/components/hooks/useLanguageNames.ts\nimport { useState, useEffect } from 'react';\nimport { httpGet } from 'next-basics';\nimport enUS from '../../../public/intl/language/en-US.json';\n\nconst languageNames = {\n  'en-US': enUS,\n};\n\nexport \nfunction useLanguageNames(locale) {\n  const [list, setList] = useState(languageNames[locale] || enUS);\n\n  async function loadData(locale) {\n    const { data } = await httpGet(`${process.env.basePath}/intl/language/${locale}.json`);\n\n    if (data) {\n      languageNames[locale] = data;\n      setList(languageNames[locale]);\n    } else {\n      setList(enUS);\n    }\n  }\n\n  useEffect(() => {\n    if (!languageNames[locale]) {\n      loadData(locale);\n    } else {\n      setList(languageNames[locale]);\n    }\n  }, [locale]);\n\n  return list;\n}\n\nexport default useLanguageNames;\n\n// Path: src/components/hooks/useTeamUrl.ts\nimport { usePathname } from 'next/navigation';\n\nexport function useTeamUrl(): {\n  teamId?: string;\n  renderTeamUrl: (url: string) => string;\n} {\n  const pathname = usePathname();\n  const [, teamId] = pathname.match(/^\\/teams\\/([a-f0-9-]+)/) || [];\n\n  function renderTeamUrl(url: string) {\n    return teamId ? `/teams/${teamId}${url}` : url;\n  }\n\n  return { teamId, renderTeamUrl };\n}\n\nexport default useTeamUrl;\n\n// Path: src/components/hooks/queries/useWebsiteValues.ts\nimport { useApi } from 'components/hooks';\n\nexport function useWebsiteValues({\n  websiteId,\n  type,\n  startDate,\n  endDate,\n  search,\n}: {\n  websiteId: string;\n  type: string;\n  startDate: Date;\n  endDate: Date;\n  search?: string;\n}) {\n  const { get, useQuery } = useApi();\n\n  return useQuery({\n    queryKey: ['websites:values', { websiteId, type, startDate, endDate, search }],\n    queryFn: () =>\n      get(`/websites/${websiteId}/values`, {\n        type,\n        startAt: +startDate,\n        endAt: +endDate,\n        search,\n      }),\n    enabled: !!(websiteId && type && startDate && endDate),\n  });\n}\n\nexport default useWebsiteValues;\n\n// Path: src/components/hooks/useFields.ts\nimport { useMessages } from './useMessages';\n\nexport function useFields() {\n  const { formatMessage, labels } = useMessages();\n\n  const fields = [\n    { name: 'url', type: 'string', label: formatMessage(labels.url) },\n    { name: 'title', type: 'string', label: formatMessage(labels.pageTitle) },\n    { name: 'referrer', type: 'string', label: formatMessage(labels.referrer) },\n    { name: 'query', type: 'string', label: formatMessage(labels.query) },\n    { name: 'browser', type: 'string', label: formatMessage(labels.browser) },\n    { name: 'os', type: 'string', label: formatMessage(labels.os) },\n    { name: 'device', type: 'string', label: formatMessage(labels.device) },\n    { name: 'country', type: 'string', label: formatMessage(labels.country) },\n    { name: 'region', type: 'string', label: formatMessage(labels.region) },\n    { name: 'city', type: 'string', label: formatMessage(labels.city) },\n  ];\n\n  return { fields };\n}\n\nexport default useFields;\n\n// Path: src/components/hooks/queries/useTeams.ts\nimport useApi from './useApi';\nimport useFilterQuery from './useFilterQuery';\nimport useModified from '../useModified';\n\nexport function useTeams(userId: string) {\n  const { get } = useApi();\n  const { modified } = useModified(`teams`);\n\n  return useFilterQuery({\n    queryKey: ['teams', { userId, modified }],\n    queryFn: (params: any) => {\n      return get(`/users/${userId}/teams`, params);\n    },\n  });\n}\n\nexport default useTeams;\n\n// Path: src/store/websites.ts\nimport { create } from 'zustand';\nimport { produce } from 'immer';\nimport { DateRange } from 'lib/types';\n\nconst store = create(() => ({}));\n\nexport function setWebsiteDateRange(websiteId: string, dateRange: DateRange) {\n  store.setState(\n    produce(state => {\n      if (!state[websiteId]) {\n        state[websiteId] = {};\n      }\n\n      state[websiteId].dateRange = { ...dateRange, modified: Date.now() };\n\n      return state;\n    }),\n  );\n}\n\nexport default store;\n\n// Path: src/components/hooks/useDateRange.ts\nimport { getMinimumUnit, parseDateRange } from 'lib/date';\nimport { setItem } from 'next-basics';\nimport { DATE_RANGE_CONFIG, DEFAULT_DATE_RANGE } from 'lib/constants';\nimport websiteStore, { setWebsiteDateRange } from 'store/websites';\nimport appStore, { setDateRange } from 'store/app';\nimport { DateRange } from 'lib/types';\nimport { useLocale } from './useLocale';\nimport { useApi } from './queries/useApi';\n\nexport function useDateRange(websiteId?: string): [DateRange, (value: string | DateRange) => void] {\n  const { get } = useApi();\n  const { locale } = useLocale();\n  const websiteConfig = websiteStore(state => state[websiteId]?.dateRange);\n  const defaultConfig = DEFAULT_DATE_RANGE;\n  const globalConfig = appStore(state => state.dateRange);\n  const dateRange = parseDateRange(websiteConfig || globalConfig || defaultConfig, locale);\n\n  const saveDateRange = async (value: DateRange | string) => {\n    if (websiteId) {\n      let dateRange: DateRange | string = value;\n\n      if (typeof value === 'string') {\n        if (value === 'all') {\n          const result: any = await get(`/websites/${websiteId}/daterange`);\n          const { mindate, maxdate } = result;\n\n          const startDate = new Date(mindate);\n          const endDate = new Date(maxdate);\n\n          dateRange = {\n            startDate,\n            endDate,\n            unit: getMinimumUnit(startDate, endDate),\n            value,\n          };\n        } else {\n          dateRange = parseDateRange(value, locale);\n        }\n      }\n\n      setWebsiteDateRange(websiteId, dateRange as DateRange);\n    } else {\n      setItem(DATE_RANGE_CONFIG, value);\n      setDateRange(value);\n    }\n  };\n\n  return [dateRange, saveDateRange];\n}\n\nexport default useDateRange;\n\n// Path: src/components/hooks/useTheme.ts\nimport { useEffect, useMemo } from 'react';\nimport useStore, { setTheme } from 'store/app';\nimport { getItem, setItem } from 'next-basics';\nimport { DEFAULT_THEME, THEME_COLORS, THEME_CONFIG } from 'lib/constants';\nimport { colord } from 'colord';\n\nconst selector = (state: { theme: string }) => state.theme;\n\nexport function useTheme() {\n  const theme = useStore(selector) || getItem(THEME_CONFIG) || DEFAULT_THEME;\n  const primaryColor = colord(THEME_COLORS[theme].primary);\n\n  const colors = useMemo(() => {\n    return {\n      theme: {\n        ...THEME_COLORS[theme],\n      },\n      chart: {\n        text: THEME_COLORS[theme].gray700,\n        line: THEME_COLORS[theme].gray200,\n        views: {\n          hoverBackgroundColor: primaryColor.alpha(0.7).toRgbString(),\n          backgroundColor: primaryColor.alpha(0.4).toRgbString(),\n          borderColor: primaryColor.alpha(0.7).toRgbString(),\n          hoverBorderColor: primaryColor.toRgbString(),\n        },\n        visitors: {\n          hoverBackgroundColor: primaryColor.alpha(0.9).toRgbString(),\n          backgroundColor: primaryColor.alpha(0.6).toRgbString(),\n          borderColor: primaryColor.alpha(0.9).toRgbString(),\n          hoverBorderColor: primaryColor.toRgbString(),\n        },\n      },\n      map: {\n        baseColor: THEME_COLORS[theme].primary,\n        fillColor: THEME_COLORS[theme].gray100,\n        strokeColor: THEME_COLORS[theme].primary,\n        hoverColor: THEME_COLORS[theme].primary,\n      },\n    };\n  }, [theme]);\n\n  const saveTheme = (value: string) => {\n    setItem(THEME_CONFIG, value);\n    setTheme(value);\n  };\n\n  useEffect(() => {\n    document.body.setAttribute('data-theme', theme);\n  }, [theme]);\n\n  useEffect(() => {\n    const url = new URL(window?.location?.href);\n    const theme = url.searchParams.get('theme');\n\n    if (['light', 'dark'].includes(theme)) {\n      saveTheme(theme);\n    }\n  }, []);\n\n  return { theme, saveTheme, colors };\n}\n\nexport default useTheme;\n\n// Path: src/components/hooks/queries/useConfig.ts\nimport { useEffect } from 'react';\nimport useStore, { setConfig } from 'store/app';\nimport { useApi } from './useApi';\n\nlet loading = false;\n\nexport function useConfig() {\n  const { config } = useStore();\n  const { get } = useApi();\n  const configUrl = process.env.configUrl;\n\n  async function loadConfig() {\n    const data = await get(configUrl);\n    loading = false;\n    setConfig(data);\n  }\n\n  useEffect(() => {\n    if (!config && !loading && configUrl) {\n      loading = true;\n      loadConfig();\n    }\n  }, []);\n\n  return config;\n}\n\nexport default useConfig;\n\n// Path: src/components/hooks/queries/useReports.ts\nimport useApi from './useApi';\nimport useFilterQuery from './useFilterQuery';\nimport useModified from '../useModified';\n\nexport function useReports({ websiteId, teamId }: { websiteId?: string; teamId?: string }) {\n  const { modified } = useModified(`reports`);\n  const { get, del, useMutation } = useApi();\n  const queryResult = useFilterQuery({\n    queryKey: ['reports', { websiteId, teamId, modified }],\n    queryFn: (params: any) => {\n      return get('/reports', { websiteId, teamId, ...params });\n    },\n  });\n  const { mutate } = useMutation({ mutationFn: (reportId: string) => del(`/reports/${reportId}`) });\n\n  const deleteReport = (reportId: any) => {\n    mutate(reportId, {\n      onSuccess: () => {},\n    });\n  };\n\n  return {\n    ...queryResult,\n    deleteReport,\n  };\n}\n\nexport default useReports;\n\n// Path: src/components/hooks/useEscapeKey.ts\nimport { useEffect, useCallback, KeyboardEvent } from 'react';\n\nexport function useEscapeKey(handler: (event: KeyboardEvent) => void) {\n  const escFunction = useCallback((event: KeyboardEvent) => {\n    if (event.key === 'Escape') {\n      handler(event);\n    }\n  }, []);\n\n  useEffect(() => {\n    document.addEventListener('keydown', escFunction as any, false);\n\n    return () => {\n      document.removeEventListener('keydown', escFunction as any, false);\n    };\n  }, [escFunction]);\n\n  return null;\n}\n\nexport default useEscapeKey;\n\n// Path: src/components/hooks/queries/useUsers.ts\nimport useApi from './useApi';\nimport useFilterQuery from './useFilterQuery';\nimport useModified from '../useModified';\n\nexport function useUsers() {\n  const { get } = useApi();\n  const { modified } = useModified(`users`);\n\n  return useFilterQuery({\n    queryKey: ['users', { modified }],\n    queryFn: (params: any) => {\n      return get('/admin/users', {\n        ...params,\n      });\n    },\n  });\n}\n\nexport default useUsers;\n\n// Path: src/lib/filters.ts\nexport const urlFilter = (data: any[]) => {\n  const map = data.reduce((obj, { x, y }) => {\n    if (x) {\n      if (!obj[x]) {\n        obj[x] = y;\n      } else {\n        obj[x] += y;\n      }\n    }\n\n    return obj;\n  }, {});\n\n  return Object.keys(map).map(key => ({ x: key, y: map[key] }));\n};\n\nexport const refFilter = (data: any[]) => {\n  const links = {};\n\n  const map = data.reduce((obj, { x, y }) => {\n    let id;\n\n    try {\n      const url = new URL(x);\n\n      id = url.hostname.replace(/www\\./, '') || url.href;\n    } catch {\n      id = '';\n    }\n\n    links[id] = x;\n\n    if (!obj[id]) {\n      obj[id] = y;\n    } else {\n      obj[id] += y;\n    }\n\n    return obj;\n  }, {});\n\n  return Object.keys(map).map(key => ({ x: key, y: map[key], w: links[key] }));\n};\n\nexport const emptyFilter = (data: any[]) => {\n  return data.map(item => (item.x ? item : null)).filter(n => n);\n};\n\nexport const percentFilter = (data: any[]) => {\n  const total = data.reduce((n, { y }) => n + y, 0);\n  return data.map(({ x, y, ...props }) => ({ x, y, z: total ? (y / total) * 100 : 0, ...props }));\n};\n\nexport const paramFilter = (data: any[]) => {\n  const map = data.reduce((obj, { x, y }) => {\n    try {\n      const searchParams = new URLSearchParams(x);\n\n      for (const [key, value] of searchParams) {\n        if (!obj[key]) {\n          obj[key] = { [value]: y };\n        } else if (!obj[key][value]) {\n          obj[key][value] = y;\n        } else {\n          obj[key][value] += y;\n        }\n      }\n    } catch {\n      // Ignore\n    }\n\n    return obj;\n  }, {});\n\n  return Object.keys(map).flatMap(key =>\n    Object.keys(map[key]).map(n => ({ x: `${key}=${n}`, p: key, v: n, y: map[key][n] })),\n  );\n};\n\n// Path: src/components/hooks/queries/useRealtime.ts\nimport { useMemo, useRef } from 'react';\nimport { RealtimeData } from 'lib/types';\nimport { useApi } from 'components/hooks';\nimport { REALTIME_INTERVAL, REALTIME_RANGE } from 'lib/constants';\nimport { startOfMinute, subMinutes } from 'date-fns';\nimport { percentFilter } from 'lib/filters';\nimport thenby from 'thenby';\n\nfunction mergeData(state = [], data = [], time: number) {\n  const ids = state.map(({ id }) => id);\n  return state\n    .concat(data.filter(({ id }) => !ids.includes(id)))\n    .filter(({ timestamp }) => timestamp >= time);\n}\n\nexport function useRealtime(websiteId: string) {\n  const currentData = useRef({\n    pageviews: [],\n    sessions: [],\n    events: [],\n    countries: [],\n    visitors: [],\n    timestamp: 0,\n  });\n  const { get, useQuery } = useApi();\n  const { data, isLoading, error } = useQuery<RealtimeData>({\n    queryKey: ['realtime', websiteId],\n    queryFn: async () => {\n      const state = currentData.current;\n      const data = await get(`/realtime/${websiteId}`, { startAt: state?.timestamp || 0 });\n      const date = subMinutes(startOfMinute(new Date()), REALTIME_RANGE);\n      const time = date.getTime();\n      const { pageviews, sessions, events, timestamp } = data;\n\n      return {\n        pageviews: mergeData(state?.pageviews, pageviews, time),\n        sessions: mergeData(state?.sessions, sessions, time),\n        events: mergeData(state?.events, events, time),\n        timestamp,\n      };\n    },\n    enabled: !!websiteId,\n    refetchInterval: REALTIME_INTERVAL,\n  });\n\n  const realtimeData: RealtimeData = useMemo(() => {\n    if (!data) {\n      return { pageviews: [], sessions: [], events: [], countries: [], visitors: [], timestamp: 0 };\n    }\n\n    data.countries = percentFilter(\n      data.sessions\n        .reduce((arr, data) => {\n          if (!arr.find(({ id }) => id === data.id)) {\n            return arr.concat(data);\n          }\n          return arr;\n        }, [])\n        .reduce((arr: { x: any; y: number }[], { country }: any) => {\n          if (country) {\n            const row = arr.find(({ x }) => x === country);\n\n            if (!row) {\n              arr.push({ x: country, y: 1 });\n            } else {\n              row.y += 1;\n            }\n          }\n          return arr;\n        }, [])\n        .sort(thenby.firstBy('y', -1)),\n    );\n\n    data.visitors = data.sessions.reduce((arr, val) => {\n      if (!arr.find(({ id }) => id === val.id)) {\n        return arr.concat(val);\n      }\n      return arr;\n    }, []);\n\n    return data;\n  }, [data]);\n\n  return { data: realtimeData, isLoading, error };\n}\n\nexport default useRealtime;\n\n// Path: src/components/hooks/queries/useWebsiteMetrics.ts\nimport useApi from './useApi';\nimport { UseQueryOptions } from '@tanstack/react-query';\n\nexport function useWebsiteMetrics(\n  websiteId: string,\n  params?: { [key: string]: any },\n  options?: Omit<UseQueryOptions & { onDataLoad?: (data: any) => void }, 'queryKey' | 'queryFn'>,\n) {\n  const { get, useQuery } = useApi();\n\n  return useQuery({\n    queryKey: [\n      'websites:metrics',\n      {\n        websiteId,\n        ...params,\n      },\n    ],\n    queryFn: async () => {\n      const filters = { ...params };\n\n      filters[params.type] = undefined;\n\n      const data = await get(`/websites/${websiteId}/metrics`, {\n        ...filters,\n      });\n\n      options?.onDataLoad?.(data);\n\n      return data;\n    },\n    ...options,\n  });\n}\n\nexport default useWebsiteMetrics;\n\n// Path: src/components/hooks/useFilters.ts\nimport { useMessages } from './useMessages';\nimport { OPERATORS } from 'lib/constants';\n\nexport function useFilters() {\n  const { formatMessage, labels } = useMessages();\n\n  const operatorLabels = {\n    [OPERATORS.equals]: formatMessage(labels.is),\n    [OPERATORS.notEquals]: formatMessage(labels.isNot),\n    [OPERATORS.set]: formatMessage(labels.isSet),\n    [OPERATORS.notSet]: formatMessage(labels.isNotSet),\n    [OPERATORS.contains]: formatMessage(labels.contains),\n    [OPERATORS.doesNotContain]: formatMessage(labels.doesNotContain),\n    [OPERATORS.true]: formatMessage(labels.true),\n    [OPERATORS.false]: formatMessage(labels.false),\n    [OPERATORS.greaterThan]: formatMessage(labels.greaterThan),\n    [OPERATORS.lessThan]: formatMessage(labels.lessThan),\n    [OPERATORS.greaterThanEquals]: formatMessage(labels.greaterThanEquals),\n    [OPERATORS.lessThanEquals]: formatMessage(labels.lessThanEquals),\n    [OPERATORS.before]: formatMessage(labels.before),\n    [OPERATORS.after]: formatMessage(labels.after),\n  };\n\n  const typeFilters = {\n    string: [OPERATORS.equals, OPERATORS.notEquals, OPERATORS.contains, OPERATORS.doesNotContain],\n    array: [OPERATORS.contains, OPERATORS.doesNotContain],\n    boolean: [OPERATORS.true, OPERATORS.false],\n    number: [\n      OPERATORS.equals,\n      OPERATORS.notEquals,\n      OPERATORS.greaterThan,\n      OPERATORS.lessThan,\n      OPERATORS.greaterThanEquals,\n      OPERATORS.lessThanEquals,\n    ],\n    date: [OPERATORS.before, OPERATORS.after],\n    uuid: [OPERATORS.equals],\n  };\n\n  const filters = Object.keys(typeFilters).flatMap(key => {\n    return (\n      typeFilters[key]?.map(value => ({ type: key, value, label: operatorLabels[value] })) ?? []\n    );\n  });\n\n  const getFilters = type => {\n    return typeFilters[type]?.map(key => ({ type, value: key, label: operatorLabels[key] })) ?? [];\n  };\n\n  return { filters, operatorLabels, typeFilters, getFilters };\n}\n\nexport default useFilters;\n\n// Path: src/components/hooks/queries/useWebsiteEvents.ts\nimport useApi from './useApi';\nimport { UseQueryOptions } from '@tanstack/react-query';\nimport { useDateRange, useNavigation, useTimezone } from 'components/hooks';\nimport { zonedTimeToUtc } from 'date-fns-tz';\n\nexport function useWebsiteEvents(\n  websiteId: string,\n...\n", "needle_token_start": 10423, "needle_token_end": 10598, "code_context_ntokens": 16133, "output": ["```javascript\nimport { useState, useEffect } from 'react';\nimport { httpGet } from 'next-basics';\nimport enUS from '../../../public/intl/language/en-US.json';\n\nconst languageNames = {\n  'en-US': enUS,\n};\n\nexport function useLanguageNames(locale) {\n  const [list, setList] = useState(languageNames[locale] || enUS);\n\n  async function loadData(locale) {\n    const { data } = await httpGet(`${process.env.basePath}/intl/language/${locale}.json`);\n\n    if (data) {\n      languageNames[locale] = data;\n      setList(languageNames[locale]);\n    } else {\n      setList(enUS);\n    }\n  }\n\n  useEffect(() => {\n    if (!languageNames[locale]) {\n      loadData(locale);\n    } else {\n      setList(languageNames[locale]);\n    }\n  }, [locale]);\n\n  return list;\n}\n\nexport default useLanguageNames;\n```"]}
{"repo": "umami-software/umami", "name": "getFunnel", "language": "typescript", "path": "src/queries/analytics/reports/getFunnel.ts", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to analyze user interactions on a website by tracking the progression through specified steps within a given timeframe, helping to understand user behavior and conversion rates.\n2. **Input**: It accepts a unique identifier for the website, a set of criteria including the time window in minutes, start and end dates, and a sequence of steps each defined by a type and a value.\n3. **Output**: The function returns a list of steps with statistics for each step, including the number of visitors who reached that step, the number who dropped off from the previous step, the drop-off rate, and the percentage of visitors remaining from the initial count.\n4. **Procedure**: The function determines which database to query based on configuration, executes a query to fetch data for each step within the specified timeframe, and then processes the results to calculate the metrics for each step, such as visitor counts and drop-off rates.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/queries/analytics/pageviews/getPageviewMetrics.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\nimport { EVENT_TYPE, SESSION_COLUMNS } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getPageviewMetrics(\n  ...args: [\n    websiteId: string,\n    column: string,\n    filters: QueryFilters,\n    limit?: number,\n    offset?: number,\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  column: string,\n  filters: QueryFilters,\n  limit: number = 500,\n...\n// Path: src/queries/analytics/eventData/getEventDataUsage.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery, notImplemented } from 'lib/db';\n\nexport function getEventDataUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {\n  return runQuery({\n    [PRISMA]: notImplemented,\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nfunction clickhouseQuery(\n  websiteIds: string[],\n  startDate: Date,\n  endDate: Date,\n): Promise<{ websiteId: string; count: number }[]> {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select \n      website_id as websiteId,\n      count(*) as count\n    from event_data \n    where created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and website_id in {websiteIds:Array(UUID)}\n    group by website_id\n    `,\n    {\n      websiteIds,\n      startDate,\n      endDate,\n    },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { websiteId: a.websiteId, count: Number(a.count) };\n    });\n  });\n}\n\n// Path: src/queries/analytics/sessions/getSessionMetrics.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\nimport { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getSessionMetrics(\n  ...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  type: string,\n  filters: QueryFilters,\n  limit: number = 500,\n  offset: number = 0,\n) {\n  const column = FILTER_COLUMNS[type] || type;\n  const { parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(\n    websiteId,\n    {\n      ...filters,\n      eventType: EVENT_TYPE.pageView,\n    },\n    {\n      joinSession: SESSION_COLUMNS.includes(type),\n    },\n  );\n  const includeCountry = column === 'city' || column === 'subdivision1';\n\n  return rawQuery(\n    `\n    select \n      ${column} x,\n      count(distinct website_event.session_id) y\n      ${includeCountry ? ', country' : ''}\n    from website_event\n    ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and website_event.event_type = {{eventType}}\n    ${filterQuery}\n    group by 1 \n    ${includeCountry ? ', 3' : ''}\n    order by 2 desc\n    limit ${limit}\n    offset ${offset}\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  type: string,\n  filters: QueryFilters,\n  limit: number = 500,\n  offset: number = 0,\n): Promise<{ x: string; y: number }[]> {\n  const column = FILTER_COLUMNS[type] || type;\n  const { parseFilters, rawQuery } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n  const includeCountry = column === 'city' || column === 'subdivision1';\n\n  return rawQuery(\n    `\n    select\n      ${column} x,\n      count(distinct session_id) y\n      ${includeCountry ? ', country' : ''}\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and event_type = {eventType:UInt32}\n      ${filterQuery}\n    group by x \n    ${includeCountry ? ', country' : ''}\n    order by y desc\n    limit ${limit}\n    offset ${offset}\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: a.x, y: Number(a.y), country: a.country };\n    });\n  });\n}\n\n// Path: src/queries/analytics/eventData/getEventDataStats.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getEventDataStats(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<{\n  events: number;\n  fields: number;\n  records: number;\n}> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  }).then(results => results[0]);\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters) {\n  const { rawQuery, parseFilters } = prisma;\n  const { filterQuery, params } = await parseFilters(websiteId, filters);\n\n  return rawQuery(\n    `\n    select \n      count(distinct t.website_event_id) as \"events\",\n      count(distinct t.event_key) as \"fields\",\n      sum(t.total) as \"records\"\n    from (\n      select\n        website_event_id,\n        event_key,\n        count(*) as \"total\"\n      from event_data\n      where website_id = {{websiteId::uuid}}\n        and created_at between {{startDate}} and {{endDate}}\n      ${filterQuery}\n      group by website_event_id, event_key\n      ) as t\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<{ events: number; fields: number; records: number }[]> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, filters);\n\n  return rawQuery(\n    `\n    select \n      count(distinct t.event_id) as \"events\",\n      count(distinct t.event_key) as \"fields\",\n      sum(t.total) as \"records\"\n    from (\n      select\n        event_id,\n        event_key,\n        count(*) as \"total\"\n      from event_data\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      ${filterQuery}\n      group by event_id, event_key\n      ) as t\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        events: Number(a.events),\n        fields: Number(a.fields),\n        records: Number(a.records),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/reports/getInsights.ts\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getInsights(\n  ...args: [websiteId: string, fields: { name: string; type?: string }[], filters: QueryFilters]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  fields: { name: string; type?: string }[],\n  filters: QueryFilters,\n): Promise<\n  {\n    x: string;\n    y: number;\n  }[]\n> {\n  const { parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(\n    websiteId,\n    {\n      ...filters,\n      eventType: EVENT_TYPE.pageView,\n    },\n    {\n      joinSession: !!fields.find(({ name }) => SESSION_COLUMNS.includes(name)),\n    },\n  );\n\n  return rawQuery(\n    `\n    select \n      ${parseFields(fields)}\n    from website_event\n      ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and website_event.event_type = {{eventType}}\n      ${filterQuery}\n    ${parseGroupBy(fields)}\n    order by 1 desc, 2 desc\n    limit 500\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  fields: { name: string; type?: string }[],\n  filters: QueryFilters,\n): Promise<\n  {\n    x: string;\n    y: number;\n  }[]\n> {\n  const { parseFilters, rawQuery } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      ${parseFields(fields)}\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and event_type = {eventType:UInt32}\n      ${filterQuery}\n    ${parseGroupBy(fields)}\n    order by 1 desc, 2 desc\n    limit 500\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        ...a,\n        views: Number(a.views),\n        visitors: Number(a.visitors),\n      };\n    });\n  });\n}\n\nfunction parseFields(fields: any[]) {\n  const query = fields.reduce(\n    (arr, field) => {\n      const { name } = field;\n\n      return arr.concat(`${FILTER_COLUMNS[name]} as \"${name}\"`);\n    },\n    ['count(*) as views', 'count(distinct website_event.session_id) as visitors'],\n  );\n\n  return query.join(',\\n');\n}\n\nfunction parseGroupBy(fields: { name: any }[]) {\n  if (!fields.length) {\n    return '';\n  }\n  return `group by ${fields.map(({ name }) => FILTER_COLUMNS[name]).join(',')}`;\n}\n\n// Path: src/queries/analytics/eventData/getEventDataFields.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { QueryFilters, WebsiteEventData } from 'lib/types';\n\nexport async function getEventDataFields(\n  ...args: [websiteId: string, filters: QueryFilters & { field?: string }]\n): Promise<WebsiteEventData[]> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters & { field?: string }) {\n  const { rawQuery, parseFilters } = prisma;\n  const { filterQuery, params } = await parseFilters(websiteId, filters, {\n    columns: { field: 'event_key' },\n  });\n\n  return rawQuery(\n    `\n    select\n      event_key as \"fieldName\",\n      data_type as \"dataType\",\n      string_value as \"fieldValue\",\n      count(*) as \"total\"\n    from event_data\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n    ${filterQuery}\n    group by event_key, data_type, string_value\n    order by 3 desc, 2 desc, 1 asc\n    limit 500\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters & { field?: string },\n): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, filters, {\n    columns: { field: 'event_key' },\n  });\n\n  return rawQuery(\n    `\n    select\n      event_key as fieldName,\n      data_type as dataType,\n      string_value as fieldValue,\n      count(*) as total\n    from event_data\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ${filterQuery}\n    group by event_key, data_type, string_value\n    order by 3 desc, 2 desc, 1 asc\n    limit 500\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        fieldName: a.fieldName,\n        dataType: Number(a.dataType),\n        fieldValue: a.fieldValue,\n        total: Number(a.total),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getValues.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\n\nexport async function getValues(\n  ...args: [websiteId: string, column: string, startDate: Date, endDate: Date, search: string]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  column: string,\n  startDate: Date,\n  endDate: Date,\n  search: string,\n) {\n  const { rawQuery, getSearchQuery } = prisma;\n  let searchQuery = '';\n\n  if (search) {\n    searchQuery = getSearchQuery(column);\n  }\n\n  return rawQuery(\n    `\n    select ${column} as \"value\", count(*)\n    from website_event\n    inner join session\n      on session.session_id = website_event.session_id\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      ${searchQuery}\n    group by 1\n    order by 2 desc\n    limit 10\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      search: `%${search}%`,\n    },\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  column: string,\n  startDate: Date,\n  endDate: Date,\n  search: string,\n) {\n  const { rawQuery } = clickhouse;\n  let searchQuery = '';\n\n  if (search) {\n    searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;\n  }\n\n  return rawQuery(\n    `\n    select ${column} as value, count(*)\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      ${searchQuery}\n    group by 1\n    order by 2 desc\n    limit 10\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      search,\n    },\n  );\n}\n\n// Path: src/queries/analytics/events/getEvents.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nexport function getEvents(...args: [websiteId: string, startDate: Date, eventType: number]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nfunction relationalQuery(websiteId: string, startDate: Date, eventType: number) {\n  return prisma.client.websiteEvent.findMany({\n    where: {\n      websiteId,\n      eventType,\n      createdAt: {\n        gte: startDate,\n      },\n    },\n    orderBy: {\n      createdAt: 'asc',\n    },\n  });\n}\n\nfunction clickhouseQuery(websiteId: string, startDate: Date, eventType: number) {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select\n      event_id as id,\n      website_id as websiteId, \n      session_id as sessionId,\n      created_at as createdAt,\n      toUnixTimestamp(created_at) as timestamp,\n      url_path as urlPath,\n      referrer_domain as referrerDomain,\n      event_name as eventName\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startDate:DateTime64}\n      and event_type = {eventType:UInt32}\n    order by created_at asc\n    `,\n    {\n      websiteId,\n      startDate,\n      eventType,\n    },\n  );\n}\n\n// Path: src/queries/admin/website.ts\nimport { Prisma, Website } from '@prisma/client';\nimport cache from 'lib/cache';\nimport prisma from 'lib/prisma';\nimport { FilterResult, WebsiteSearchFilter } from 'lib/types';\nimport WebsiteFindManyArgs = Prisma.WebsiteFindManyArgs;\n\nasync function findWebsite(criteria: Prisma.WebsiteFindUniqueArgs): Promise<Website> {\n  return prisma.client.website.findUnique(criteria);\n}\n\nexport async function getWebsite(websiteId: string) {\n  return findWebsite({\n    where: {\n      id: websiteId,\n    },\n  });\n}\n\nexport async function getSharedWebsite(shareId: string) {\n  return findWebsite({\n    where: {\n      shareId,\n    },\n  });\n}\n\nexport async function getWebsites(\n  criteria: WebsiteFindManyArgs,\n  filters: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  const { query } = filters;\n\n  const where: Prisma.WebsiteWhereInput = {\n    ...criteria.where,\n    ...prisma.getSearchParameters(query, [\n      {\n        name: 'contains',\n      },\n      { domain: 'contains' },\n    ]),\n    deletedAt: null,\n  };\n\n  return prisma.pagedQuery('website', { ...criteria, where }, filters);\n}\n\nexport async function getAllWebsites(userId: string) {\n  return prisma.client.website.findMany({\n    where: {\n      userId,\n    },\n  });\n}\n\nexport async function getUserWebsites(\n  userId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        userId,\n      },\n      include: {\n        user: {\n          select: {\n            username: true,\n            id: true,\n          },\n        },\n      },\n    },\n    {\n      orderBy: 'name',\n      ...filters,\n    },\n  );\n}\n\nexport async function getTeamWebsites(\n  teamId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        teamId,\n      },\n      include: {\n        createUser: {\n          select: {\n            id: true,\n            username: true,\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createWebsite(\n  data: Prisma.WebsiteCreateInput | Prisma.WebsiteUncheckedCreateInput,\n): Promise<Website> {\n  return prisma.client.website\n    .create({\n      data,\n    })\n    .then(async data => {\n      if (cache.enabled) {\n        await cache.storeWebsite(data);\n      }\n\n      return data;\n    });\n}\n\nexport async function updateWebsite(\n  websiteId: string,\n  data: Prisma.WebsiteUpdateInput | Prisma.WebsiteUncheckedUpdateInput,\n): Promise<Website> {\n  return prisma.client.website.update({\n    where: {\n      id: websiteId,\n    },\n    data,\n  });\n}\n\nexport async function resetWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId },\n    }),\n    client.session.deleteMany({\n      where: { websiteId },\n    }),\n    client.website.update({\n      where: { id: websiteId },\n      data: {\n        resetAt: new Date(),\n      },\n    }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      await cache.storeWebsite(data[3]);\n    }\n\n    return data;\n  });\n}\n\nexport async function deleteWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId },\n    }),\n    client.session.deleteMany({\n      where: { websiteId },\n    }),\n    client.report.deleteMany({\n      where: {\n        websiteId,\n      },\n    }),\n    cloudMode\n      ? client.website.update({\n          data: {\n            deletedAt: new Date(),\n          },\n          where: { id: websiteId },\n        })\n      : client.website.delete({\n          where: { id: websiteId },\n        }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      await cache.deleteWebsite(websiteId);\n    }\n\n    return data;\n  });\n}\n\n// Path: src/queries/analytics/sessions/getSession.ts\nimport prisma from 'lib/prisma';\n\nexport async function getSession(id: string) {\n  return prisma.client.session.findUnique({\n    where: {\n      id,\n    },\n  });\n}\n\n// Path: src/lib/data.ts\nimport { DATA_TYPE, DATETIME_REGEX } from './constants';\nimport { DynamicDataType } from './types';\n\nexport function flattenJSON(\n  eventData: { [key: string]: any },\n  keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],\n  parentKey = '',\n): { key: string; value: any; dataType: DynamicDataType }[] {\n  return Object.keys(eventData).reduce(\n    (acc, key) => {\n      const value = eventData[key];\n      const type = typeof eventData[key];\n\n      // nested object\n      if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {\n        flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));\n      } else {\n        createKey(getKeyName(key, parentKey), value, acc);\n      }\n\n      return acc;\n    },\n    { keyValues, parentKey },\n  ).keyValues;\n}\n\nexport function isValidDateValue(value: string) {\n  return typeof value === 'string' && DATETIME_REGEX.test(value);\n}\n\nexport function getDataType(value: any): string {\n  let type: string = typeof value;\n\n  if (isValidDateValue(value)) {\n    type = 'date';\n  }\n\n  return type;\n}\n\nexport function getStringValue(value: string, dataType: number) {\n  if (dataType === DATA_TYPE.number) {\n    return parseFloat(value).toFixed(4);\n  }\n\n  if (dataType === DATA_TYPE.date) {\n    return new Date(value).toISOString();\n  }\n\n  return value;\n}\n\nfunction createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {\n  const type = getDataType(value);\n\n  let dataType = null;\n\n  switch (type) {\n    case 'number':\n      dataType = DATA_TYPE.number;\n      break;\n    case 'string':\n      dataType = DATA_TYPE.string;\n      break;\n    case 'boolean':\n      dataType = DATA_TYPE.boolean;\n      value = value ? 'true' : 'false';\n      break;\n    case 'date':\n      dataType = DATA_TYPE.date;\n      break;\n    case 'object':\n      dataType = DATA_TYPE.array;\n      value = JSON.stringify(value);\n      break;\n    default:\n      dataType = DATA_TYPE.string;\n      break;\n  }\n\n  acc.keyValues.push({ key, value, dataType });\n}\n\nfunction getKeyName(key: string, parentKey: string) {\n  if (!parentKey) {\n    return key;\n  }\n\n  return `${parentKey}.${key}`;\n}\n\n// Path: src/queries/analytics/sessions/saveSessionData.ts\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveSessionData(data: {\n  websiteId: string;\n  sessionId: string;\n  sessionData: DynamicData;\n}) {\n  const { client, transaction } = prisma;\n  const { websiteId, sessionId, sessionData } = data;\n\n  const jsonKeys = flattenJSON(sessionData);\n\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteId,\n    sessionId,\n    key: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return transaction([\n    client.sessionData.deleteMany({\n      where: {\n        sessionId,\n      },\n    }),\n    client.sessionData.createMany({\n      data: flattenedData as any,\n    }),\n  ]);\n}\n\n// Path: src/queries/analytics/reports/getUTM.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nexport async function getUTM(\n  ...args: [\n    websiteId: string,\n    filters: {\n      startDate: Date;\n      endDate: Date;\n      timezone?: string;\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = prisma;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n      and coalesce(url_query, '') != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and url_query != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nfunction parseParameters(data: any[]) {\n  return data.reduce((obj, { url_query, num }) => {\n    try {\n      const searchParams = new URLSearchParams(url_query);\n\n      for (const [key, value] of searchParams) {\n        if (key.match(/^utm_(\\w+)$/)) {\n          const name = safeDecodeURIComponent(value);\n          if (!obj[key]) {\n            obj[key] = { [name]: Number(num) };\n          } else if (!obj[key][name]) {\n            obj[key][name] = Number(num);\n          } else {\n            obj[key][name] += Number(num);\n          }\n        }\n      }\n    } catch {\n      // Ignore\n    }\n\n    return obj;\n  }, {});\n}\n\n// Path: src/lib/kafka.ts\nimport dateFormat from 'dateformat';\nimport debug from 'debug';\nimport { Kafka, Mechanism, Producer, RecordMetadata, SASLOptions, logLevel } from 'kafkajs';\nimport { KAFKA, KAFKA_PRODUCER } from 'lib/db';\nimport * as tls from 'tls';\n\nconst log = debug('umami:kafka');\n\nlet kafka: Kafka;\nlet producer: Producer;\nconst enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER);\n\nfunction getClient() {\n  const { username, password } = new URL(process.env.KAFKA_URL);\n  const brokers = process.env.KAFKA_BROKER.split(',');\n\n  const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions | Mechanism } =\n    username && password\n      ? {\n          ssl: {\n            checkServerIdentity: () => undefined,\n            ca: [process.env.CA_CERT],\n            key: process.env.CLIENT_KEY,\n            cert: process.env.CLIENT_CERT,\n          },\n          sasl: {\n            mechanism: 'plain',\n            username,\n            password,\n          },\n        }\n      : {};\n\n  const client: Kafka = new Kafka({\n    clientId: 'umami',\n    brokers: brokers,\n    connectionTimeout: 3000,\n    logLevel: logLevel.ERROR,\n    ...ssl,\n  });\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA] = client;\n  }\n\n  log('Kafka initialized');\n\n  return client;\n}\n\nasync function getProducer(): Promise<Producer> {\n  const producer = kafka.producer();\n  await producer.connect();\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA_PRODUCER] = producer;\n  }\n\n  log('Kafka producer initialized');\n\n  return producer;\n}\n\nfunction getDateFormat(date: Date, format?: string): string {\n  return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');\n}\n\nasync function sendMessage(\n  message: { [key: string]: string | number },\n  topic: string,\n): Promise<RecordMetadata[]> {\n  await connect();\n\n  return producer.send({\n    topic,\n    messages: [\n      {\n        value: JSON.stringify(message),\n      },\n    ],\n    acks: -1,\n  });\n}\n\nasync function sendMessages(messages: { [key: string]: string | number }[], topic: string) {\n  await connect();\n\n  await producer.send({\n    topic,\n    messages: messages.map(a => {\n      return { value: JSON.stringify(a) };\n    }),\n    acks: 1,\n  });\n}\n\nasync function connect(): Promise<Kafka> {\n  if (!kafka) {\n    kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient());\n\n    if (kafka) {\n      producer = global[KAFKA_PRODUCER] || (await getProducer());\n    }\n  }\n\n  return kafka;\n}\n\nexport default {\n  enabled,\n  client: kafka,\n  producer,\n  log,\n  connect,\n  getDateFormat,\n  sendMessage,\n  sendMessages,\n};\n\n// Path: src/queries/analytics/eventData/saveEventData.ts\nimport { Prisma } from '@prisma/client';\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveEventData(args: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  eventId: string;\n  eventData: DynamicData;\n}): Promise<Prisma.BatchPayload> {\n  const { websiteId, eventId, eventData } = data;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  // id, websiteEventId, eventStringValue\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteEventId: eventId,\n    websiteId,\n    eventKey: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return prisma.client.eventData.createMany({\n    data: flattenedData,\n  });\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;\n\n  const { getDateFormat, sendMessages } = kafka;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  const messages = jsonKeys.map(a => ({\n    website_id: websiteId,\n    session_id: sessionId,\n    event_id: eventId,\n    url_path: urlPath,\n    event_name: eventName,\n    event_key: a.key,\n    string_value: getStringValue(a.value, a.dataType),\n    number_value: a.dataType === DATA_TYPE.number ? a.value : null,\n    date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null,\n    data_type: a.dataType,\n    created_at: createdAt,\n  }));\n\n  await sendMessages(messages, 'event_data');\n\n  return data;\n}\n\n// Path: src/queries/analytics/events/saveEvent.ts\nimport { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE, PAGE_TITLE_LENGTH } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { uuid } from 'lib/crypto';\nimport { saveEventData } from 'queries/analytics/eventData/saveEventData';\n\nexport async function saveEvent(args: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    eventName,\n    eventData,\n    pageTitle,\n  } = data;\n  const websiteEventId = uuid();\n\n  const websiteEvent = prisma.client.websiteEvent.create({\n    data: {\n      id: websiteEventId,\n      websiteId,\n      sessionId,\n      visitId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      urlQuery: urlQuery?.substring(0, URL_LENGTH),\n      referrerPath: referrerPath?.substring(0, URL_LENGTH),\n      referrerQuery: referrerQuery?.substring(0, URL_LENGTH),\n      referrerDomain: referrerDomain?.substring(0, URL_LENGTH),\n      pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n      eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n      eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    },\n  });\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId: websiteEventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n    });\n  }\n\n  return websiteEvent;\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    pageTitle,\n    eventName,\n    eventData,\n    country,\n    subdivision1,\n    subdivision2,\n    city,\n    ...args\n  } = data;\n  const { getDateFormat, sendMessage } = kafka;\n  const eventId = uuid();\n  const createdAt = getDateFormat(new Date());\n\n  const message = {\n    ...args,\n    website_id: websiteId,\n    session_id: sessionId,\n    visit_id: visitId,\n    event_id: uuid(),\n    country: country,\n    subdivision1:\n      country && subdivision1\n        ? subdivision1.includes('-')\n          ? subdivision1\n          : `${country}-${subdivision1}`\n        : null,\n    subdivision2: subdivision2,\n    city: city,\n    url_path: urlPath?.substring(0, URL_LENGTH),\n    url_query: urlQuery?.substring(0, URL_LENGTH),\n    referrer_path: referrerPath?.substring(0, URL_LENGTH),\n    referrer_query: referrerQuery?.substring(0, URL_LENGTH),\n    referrer_domain: referrerDomain?.substring(0, URL_LENGTH),\n    page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n    event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n    event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    created_at: createdAt,\n  };\n\n  await sendMessage(message, 'event');\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n      createdAt,\n    });\n  }\n\n  return data;\n}\n\n// Path: src/queries/analytics/sessions/getSessions.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';\n\nexport async function getSessions(...args: [websiteId: string, startAt: Date]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, startDate: Date) {\n  return prisma.client.session.findMany({\n    where: {\n      websiteId,\n      createdAt: {\n        gte: startDate,\n      },\n    },\n    orderBy: {\n      createdAt: 'asc',\n    },\n  });\n}\n\nasync function clickhouseQuery(websiteId: string, startDate: Date) {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select\n      session_id as id,\n      website_id as websiteId,\n      created_at as createdAt,\n      toUnixTimestamp(created_at) as timestamp,\n      hostname,\n      browser,\n      os,\n      device,\n      screen,\n      language,\n      country,\n      subdivision1,\n      subdivision2,\n      city\n    from website_event\n    where website_id = {websiteId:UUID}\n    and created_at >= {startDate:DateTime64}\n    order by created_at asc\n    `,\n    {\n      websiteId,\n      startDate,\n    },\n  );\n}\n\n// Path: src/queries/analytics/reports/getFunnel.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nconst formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {\n  return steps.map((step: { type: string; value: string }, i: number) => {\n    const visitors = Number(results[i]?.count) || 0;\n    const previous = Number(results[i - 1]?.count) || 0;\n    const dropped = previous > 0 ? previous - visitors : 0;\n    const dropoff = 1 - visitors / previous;\n    const remaining = visitors / Number(results[0].count);\n\n    return {\n      ...step,\n      visitors,\n      previous,\n      dropped,\n      dropoff,\n      remaining,\n    };\n  });\n};\n\nexport \nasync function getFunnel(\n  ...args: [\n    websiteId: string,\n    criteria: {\n      windowMinutes: number;\n      startDate: Date;\n      endDate: Date;\n      steps: { type: string; value: string }[];\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery, getAddIntervalQuery } = prisma;\n  const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, windowMinutes);\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    params: string[];\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union ' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\n          WITH level1 AS (\n            select distinct session_id, created_at\n            from website_event\n            where website_id = {{websiteId::uuid}}\n              and created_at between {{startDate}} and {{endDate}}\n              and ${column} ${operator} {{${i}}}\n          )`;\n        } else {\n          pv.levelQuery += `\n          , level${levelNumber} AS (\n            select distinct we.session_id, we.created_at\n            from level${i} l\n            join website_event we\n                on l.session_id = we.session_id\n            where we.website_id = {{websiteId::uuid}}\n                and we.created_at between l.created_at and ${getAddIntervalQuery(\n                  `l.created_at `,\n                  `${windowMinutes} minute`,\n                )}\n                and we.${column} ${operator} {{${i}}}\n                and we.created_at <= {{endDate}}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.params.push(paramValue);\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        params: [],\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    ${levelOneQuery}\n    ${levelQuery}\n    ${sumQuery}\n    ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery } = clickhouse;\n  const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(\n    steps,\n    windowMinutes,\n  );\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    stepFilterQuery: string;\n    params: { [key: string]: string };\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union all ' : '';\n        const startFilter = i > 0 ? 'or' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\\n\n          level1 AS (\n            select *\n            from level0\n            where ${column} ${operator} {param${i}:String}\n          )`;\n        } else {\n          pv.levelQuery += `\\n\n          , level${levelNumber} AS (\n            select distinct y.session_id as session_id,\n                y.url_path as url_path,\n                y.referrer_path as referrer_path,\n                y.event_name,\n                y.created_at as created_at\n            from level${i} x\n            join level0 y\n            on x.session_id = y.session_id\n            where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute\n                and y.${column} ${operator} {param${i}:String}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.stepFilterQuery += `${startFilter} ${column} ${operator} {param${i}:String} `;\n        pv.params[`param${i}`] = paramValue;\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        stepFilterQuery: '',\n        params: {},\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    WITH level0 AS (\n      select distinct session_id, url_path, referrer_path, event_name, created_at\n      from umami.website_event\n      where (${stepFilterQuery})\n        and website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ),\n    ${levelOneQuery}\n    ${levelQuery}\n    select *\n    from (\n      ${sumQuery} \n    ) ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\n// Path: src/queries/analytics/getWebsiteStats.ts\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getWebsiteStats(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(case when t.c = 1 then 1 else 0 end) as \"bounces\",\n      sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as \"totaltime\"\n    from (\n      select\n        website_event.session_id,\n        website_event.visit_id,\n        count(*) as \"c\",\n        min(website_event.created_at) as \"min_time\",\n        max(website_event.created_at) as \"max_time\"\n      from website_event\n      join website \n        on website_event.website_id = website.website_id\n        ${joinSession}\n      where website.website_id = {{websiteId::uuid}}\n        and website_event.created_at between {{startDate}} and {{endDate}}\n        and event_type = {{eventType}}\n        ${filterQuery}\n      group by 1, 2\n    ) as t\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(if(t.c = 1, 1, 0)) as \"bounces\",\n      sum(max_time-min_time) as \"totaltime\"\n    from (\n      select\n        session_id,\n        visit_id,\n        count(*) c,\n        min(created_at) min_time,\n        max(created_at) max_time\n      from website_event\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n        and event_type = {eventType:UInt32}\n        ${filterQuery}\n      group by session_id, visit_id\n    ) as t;\n    `,\n    params,\n  ).then(result => {\n    return Object.values(result).map(n => {\n      return {\n        pageviews: Number(n.pageviews),\n        visitors: Number(n.visitors),\n        visits: Number(n.visits),\n        bounces: Number(n.bounces),\n        totaltime: Number(n.totaltime),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getActiveVisitors.ts\nimport { subMinutes } from 'date-fns';\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\n\nexport async function getActiveVisitors(...args: [websiteId: string]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string) {\n  const { rawQuery } = prisma;\n\n  const result = await rawQuery(\n    `\n    select count(distinct session_id) x\n    from website_event\n    where website_id = {{websiteId::uuid}}\n    and created_at >= {{startAt}}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  );\n\n  return result[0] ?? null;\n}\n\nasync function clickhouseQuery(websiteId: string): Promise<{ x: number }> {\n  const { rawQuery } = clickhouse;\n\n  const result = await rawQuery(\n    `\n    select\n      count(distinct session_id) x\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startAt:DateTime64}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: Number(a.x) };\n    });\n  });\n\n  return result[0] ?? null;\n}\n\n// Path: src/queries/admin/team.ts\nimport { Prisma, Team } from '@prisma/client';\nimport { ROLES } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport prisma from 'lib/prisma';\nimport { FilterResult, TeamSearchFilter } from 'lib/types';\nimport TeamFindManyArgs = Prisma.TeamFindManyArgs;\n\nexport async function findTeam(criteria: Prisma.TeamFindUniqueArgs): Promise<Team> {\n  return prisma.client.team.findUnique(criteria);\n}\n\nexport async function getTeam(teamId: string, options: { includeMembers?: boolean } = {}) {\n  const { includeMembers } = options;\n\n  return findTeam({\n    where: {\n      id: teamId,\n    },\n    ...(includeMembers && { include: { teamUser: true } }),\n  });\n}\n\nexport async function getTeams(\n  criteria: TeamFindManyArgs,\n  filters: TeamSearchFilter = {},\n): Promise<FilterResult<Team[]>> {\n  const { getSearchParameters } = prisma;\n  const { query } = filters;\n\n  const where: Prisma.TeamWhereInput = {\n    ...criteria.where,\n    ...getSearchParameters(query, [{ name: 'contains' }]),\n  };\n\n  return prisma.pagedQuery<TeamFindManyArgs>(\n    'team',\n    {\n      ...criteria,\n      where,\n    },\n    filters,\n  );\n}\n\nexport async function getUserTeams(userId: string, filters: TeamSearchFilter = {}) {\n  return getTeams(\n    {\n      where: {\n        deletedAt: null,\n        teamUser: {\n          some: { userId },\n        },\n      },\n      include: {\n        teamUser: {\n          include: {\n            user: {\n              select: {\n                id: true,\n                username: true,\n              },\n            },\n          },\n        },\n        _count: {\n          select: {\n            website: {\n              where: { deletedAt: null },\n            },\n            teamUser: {\n              where: {\n                user: { deletedAt: null },\n              },\n            },\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createTeam(data: Prisma.TeamCreateInput, userId: string): Promise<any> {\n  const { id } = data;\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.team.create({\n      data,\n    }),\n    client.teamUser.create({\n      data: {\n        id: uuid(),\n        teamId: id,\n        userId,\n        role: ROLES.teamOwner,\n      },\n    }),\n  ]);\n}\n\nexport async function updateTeam(teamId: string, data: Prisma.TeamUpdateInput): Promise<Team> {\n  const { client } = prisma;\n\n  return client.team.update({\n    where: {\n...\n", "needle_token_start": 12061, "needle_token_end": 12174, "code_context_ntokens": 16169, "output": ["```typescript\n// Path: src/queries/analytics/reports/getFunnel.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nconst formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {\n  return steps.map((step: { type: string; value: string }, i: number) => {\n    const visitors = Number(results[i]?.count) || 0;\n    const previous = Number(results[i - 1]?.count) || 0;\n    const dropped = previous > 0 ? previous - visitors : 0;\n    const dropoff = 1 - visitors / previous;\n    const remaining = visitors / Number(results[0].count);\n\n    return {\n      ...step,\n      visitors,\n      previous,\n      dropped,\n      dropoff,\n      remaining,\n    };\n  });\n};\n\nexport \nasync function getFunnel(\n  ...args: [\n    websiteId: string,\n    criteria: {\n      windowMinutes: number;\n      startDate: Date;\n      endDate: Date;\n      steps: { type: string; value: string }[];\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery, getAddIntervalQuery } = prisma;\n  const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, windowMinutes);\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    params: string[];\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union ' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\n          WITH level1 AS (\n            select distinct session_id, created_at\n            from website_event\n            where website_id = {{websiteId::uuid}}\n              and created_at between {{startDate}} and {{endDate}}\n              and ${column} ${operator} {{${i}}}\n          )`;\n        } else {\n          pv.levelQuery += `\n          , level${levelNumber} AS (\n            select distinct we.session_id, we.created_at\n            from level${i} l\n            join website_event we\n                on l.session_id = we.session_id\n            where we.website_id = {{websiteId::uuid}}\n                and we.created_at between l.created_at and ${getAddIntervalQuery(\n                  `l.created_at `,\n                  `${windowMinutes} minute`,\n                )}\n                and we.${column} ${operator} {{${i}}}\n                and we.created_at <= {{endDate}}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.params.push(paramValue);\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        params: [],\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    ${levelOneQuery}\n    ${levelQuery}\n    ${sumQuery}\n    ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery } = clickhouse;\n  const { levelOneQuery, levelQuery, sumQuery, stepFilter"]}
{"repo": "umami-software/umami", "name": "updateTeam", "language": "typescript", "path": "src/queries/admin/team.ts", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to modify the details of an existing team entity in the database.\n2. **Input**: It accepts a unique identifier for the team and a set of new values to update in the team's record.\n3. **Output**: Returns the updated team entity after applying the changes.\n4. **Procedure**: The function first specifies the team to be updated using its unique identifier. It then applies the provided updates along with the current timestamp to mark when the update occurred. Finally, it commits these changes to the database and returns the updated team entity.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/queries/analytics/reports/getInsights.ts\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getInsights(\n  ...args: [websiteId: string, fields: { name: string; type?: string }[], filters: QueryFilters]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  fields: { name: string; type?: string }[],\n  filters: QueryFilters,\n): Promise<\n  {\n    x: string;\n    y: number;\n  }[]\n> {\n  const { parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(\n    websiteId,\n    {\n      ...filters,\n      eventType: EVENT_TYPE.pageView,\n    },\n    {\n      joinSession: !!fields.find(({ name }) => SESSION_COLUMNS.includes(name)),\n    },\n  );\n\n  return rawQuery(\n    `\n    select \n      ${parseFields(fields)}\n    from website_event\n      ${joinSession}\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      and website_event.event_type = {{eventType}}\n      ${filterQuery}\n    ${parseGroupBy(fields)}\n    order by 1 desc, 2 desc\n    limit 500\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  fields: { name: string; type?: string }[],\n  filters: QueryFilters,\n): Promise<\n  {\n    x: string;\n    y: number;\n  }[]\n> {\n  const { parseFilters, rawQuery } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      ${parseFields(fields)}\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and event_type = {eventType:UInt32}\n      ${filterQuery}\n    ${parseGroupBy(fields)}\n    order by 1 desc, 2 desc\n    limit 500\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n...\n// Path: src/queries/analytics/eventData/getEventDataFields.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { QueryFilters, WebsiteEventData } from 'lib/types';\n\nexport async function getEventDataFields(\n  ...args: [websiteId: string, filters: QueryFilters & { field?: string }]\n): Promise<WebsiteEventData[]> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, filters: QueryFilters & { field?: string }) {\n  const { rawQuery, parseFilters } = prisma;\n  const { filterQuery, params } = await parseFilters(websiteId, filters, {\n    columns: { field: 'event_key' },\n  });\n\n  return rawQuery(\n    `\n    select\n      event_key as \"fieldName\",\n      data_type as \"dataType\",\n      string_value as \"fieldValue\",\n      count(*) as \"total\"\n    from event_data\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n    ${filterQuery}\n    group by event_key, data_type, string_value\n    order by 3 desc, 2 desc, 1 asc\n    limit 500\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters & { field?: string },\n): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, filters, {\n    columns: { field: 'event_key' },\n  });\n\n  return rawQuery(\n    `\n    select\n      event_key as fieldName,\n      data_type as dataType,\n      string_value as fieldValue,\n      count(*) as total\n    from event_data\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ${filterQuery}\n    group by event_key, data_type, string_value\n    order by 3 desc, 2 desc, 1 asc\n    limit 500\n    `,\n    params,\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return {\n        fieldName: a.fieldName,\n        dataType: Number(a.dataType),\n        fieldValue: a.fieldValue,\n        total: Number(a.total),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getValues.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\n\nexport async function getValues(\n  ...args: [websiteId: string, column: string, startDate: Date, endDate: Date, search: string]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  column: string,\n  startDate: Date,\n  endDate: Date,\n  search: string,\n) {\n  const { rawQuery, getSearchQuery } = prisma;\n  let searchQuery = '';\n\n  if (search) {\n    searchQuery = getSearchQuery(column);\n  }\n\n  return rawQuery(\n    `\n    select ${column} as \"value\", count(*)\n    from website_event\n    inner join session\n      on session.session_id = website_event.session_id\n    where website_event.website_id = {{websiteId::uuid}}\n      and website_event.created_at between {{startDate}} and {{endDate}}\n      ${searchQuery}\n    group by 1\n    order by 2 desc\n    limit 10\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      search: `%${search}%`,\n    },\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  column: string,\n  startDate: Date,\n  endDate: Date,\n  search: string,\n) {\n  const { rawQuery } = clickhouse;\n  let searchQuery = '';\n\n  if (search) {\n    searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;\n  }\n\n  return rawQuery(\n    `\n    select ${column} as value, count(*)\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      ${searchQuery}\n    group by 1\n    order by 2 desc\n    limit 10\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      search,\n    },\n  );\n}\n\n// Path: src/queries/analytics/events/getEvents.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nexport function getEvents(...args: [websiteId: string, startDate: Date, eventType: number]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nfunction relationalQuery(websiteId: string, startDate: Date, eventType: number) {\n  return prisma.client.websiteEvent.findMany({\n    where: {\n      websiteId,\n      eventType,\n      createdAt: {\n        gte: startDate,\n      },\n    },\n    orderBy: {\n      createdAt: 'asc',\n    },\n  });\n}\n\nfunction clickhouseQuery(websiteId: string, startDate: Date, eventType: number) {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select\n      event_id as id,\n      website_id as websiteId, \n      session_id as sessionId,\n      created_at as createdAt,\n      toUnixTimestamp(created_at) as timestamp,\n      url_path as urlPath,\n      referrer_domain as referrerDomain,\n      event_name as eventName\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startDate:DateTime64}\n      and event_type = {eventType:UInt32}\n    order by created_at asc\n    `,\n    {\n      websiteId,\n      startDate,\n      eventType,\n    },\n  );\n}\n\n// Path: src/queries/admin/website.ts\nimport { Prisma, Website } from '@prisma/client';\nimport cache from 'lib/cache';\nimport prisma from 'lib/prisma';\nimport { FilterResult, WebsiteSearchFilter } from 'lib/types';\nimport WebsiteFindManyArgs = Prisma.WebsiteFindManyArgs;\n\nasync function findWebsite(criteria: Prisma.WebsiteFindUniqueArgs): Promise<Website> {\n  return prisma.client.website.findUnique(criteria);\n}\n\nexport async function getWebsite(websiteId: string) {\n  return findWebsite({\n    where: {\n      id: websiteId,\n    },\n  });\n}\n\nexport async function getSharedWebsite(shareId: string) {\n  return findWebsite({\n    where: {\n      shareId,\n    },\n  });\n}\n\nexport async function getWebsites(\n  criteria: WebsiteFindManyArgs,\n  filters: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  const { query } = filters;\n\n  const where: Prisma.WebsiteWhereInput = {\n    ...criteria.where,\n    ...prisma.getSearchParameters(query, [\n      {\n        name: 'contains',\n      },\n      { domain: 'contains' },\n    ]),\n    deletedAt: null,\n  };\n\n  return prisma.pagedQuery('website', { ...criteria, where }, filters);\n}\n\nexport async function getAllWebsites(userId: string) {\n  return prisma.client.website.findMany({\n    where: {\n      userId,\n    },\n  });\n}\n\nexport async function getUserWebsites(\n  userId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        userId,\n      },\n      include: {\n        user: {\n          select: {\n            username: true,\n            id: true,\n          },\n        },\n      },\n    },\n    {\n      orderBy: 'name',\n      ...filters,\n    },\n  );\n}\n\nexport async function getTeamWebsites(\n  teamId: string,\n  filters?: WebsiteSearchFilter,\n): Promise<FilterResult<Website[]>> {\n  return getWebsites(\n    {\n      where: {\n        teamId,\n      },\n      include: {\n        createUser: {\n          select: {\n            id: true,\n            username: true,\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createWebsite(\n  data: Prisma.WebsiteCreateInput | Prisma.WebsiteUncheckedCreateInput,\n): Promise<Website> {\n  return prisma.client.website\n    .create({\n      data,\n    })\n    .then(async data => {\n      if (cache.enabled) {\n        await cache.storeWebsite(data);\n      }\n\n      return data;\n    });\n}\n\nexport async function updateWebsite(\n  websiteId: string,\n  data: Prisma.WebsiteUpdateInput | Prisma.WebsiteUncheckedUpdateInput,\n): Promise<Website> {\n  return prisma.client.website.update({\n    where: {\n      id: websiteId,\n    },\n    data,\n  });\n}\n\nexport async function resetWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId },\n    }),\n    client.session.deleteMany({\n      where: { websiteId },\n    }),\n    client.website.update({\n      where: { id: websiteId },\n      data: {\n        resetAt: new Date(),\n      },\n    }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      await cache.storeWebsite(data[3]);\n    }\n\n    return data;\n  });\n}\n\nexport async function deleteWebsite(\n  websiteId: string,\n): Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Website]> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  return transaction([\n    client.eventData.deleteMany({\n      where: { websiteId },\n    }),\n    client.websiteEvent.deleteMany({\n      where: { websiteId },\n    }),\n    client.session.deleteMany({\n      where: { websiteId },\n    }),\n    client.report.deleteMany({\n      where: {\n        websiteId,\n      },\n    }),\n    cloudMode\n      ? client.website.update({\n          data: {\n            deletedAt: new Date(),\n          },\n          where: { id: websiteId },\n        })\n      : client.website.delete({\n          where: { id: websiteId },\n        }),\n  ]).then(async data => {\n    if (cache.enabled) {\n      await cache.deleteWebsite(websiteId);\n    }\n\n    return data;\n  });\n}\n\n// Path: src/queries/analytics/sessions/getSession.ts\nimport prisma from 'lib/prisma';\n\nexport async function getSession(id: string) {\n  return prisma.client.session.findUnique({\n    where: {\n      id,\n    },\n  });\n}\n\n// Path: src/lib/data.ts\nimport { DATA_TYPE, DATETIME_REGEX } from './constants';\nimport { DynamicDataType } from './types';\n\nexport function flattenJSON(\n  eventData: { [key: string]: any },\n  keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],\n  parentKey = '',\n): { key: string; value: any; dataType: DynamicDataType }[] {\n  return Object.keys(eventData).reduce(\n    (acc, key) => {\n      const value = eventData[key];\n      const type = typeof eventData[key];\n\n      // nested object\n      if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {\n        flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));\n      } else {\n        createKey(getKeyName(key, parentKey), value, acc);\n      }\n\n      return acc;\n    },\n    { keyValues, parentKey },\n  ).keyValues;\n}\n\nexport function isValidDateValue(value: string) {\n  return typeof value === 'string' && DATETIME_REGEX.test(value);\n}\n\nexport function getDataType(value: any): string {\n  let type: string = typeof value;\n\n  if (isValidDateValue(value)) {\n    type = 'date';\n  }\n\n  return type;\n}\n\nexport function getStringValue(value: string, dataType: number) {\n  if (dataType === DATA_TYPE.number) {\n    return parseFloat(value).toFixed(4);\n  }\n\n  if (dataType === DATA_TYPE.date) {\n    return new Date(value).toISOString();\n  }\n\n  return value;\n}\n\nfunction createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {\n  const type = getDataType(value);\n\n  let dataType = null;\n\n  switch (type) {\n    case 'number':\n      dataType = DATA_TYPE.number;\n      break;\n    case 'string':\n      dataType = DATA_TYPE.string;\n      break;\n    case 'boolean':\n      dataType = DATA_TYPE.boolean;\n      value = value ? 'true' : 'false';\n      break;\n    case 'date':\n      dataType = DATA_TYPE.date;\n      break;\n    case 'object':\n      dataType = DATA_TYPE.array;\n      value = JSON.stringify(value);\n      break;\n    default:\n      dataType = DATA_TYPE.string;\n      break;\n  }\n\n  acc.keyValues.push({ key, value, dataType });\n}\n\nfunction getKeyName(key: string, parentKey: string) {\n  if (!parentKey) {\n    return key;\n  }\n\n  return `${parentKey}.${key}`;\n}\n\n// Path: src/queries/analytics/sessions/saveSessionData.ts\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveSessionData(data: {\n  websiteId: string;\n  sessionId: string;\n  sessionData: DynamicData;\n}) {\n  const { client, transaction } = prisma;\n  const { websiteId, sessionId, sessionData } = data;\n\n  const jsonKeys = flattenJSON(sessionData);\n\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteId,\n    sessionId,\n    key: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return transaction([\n    client.sessionData.deleteMany({\n      where: {\n        sessionId,\n      },\n    }),\n    client.sessionData.createMany({\n      data: flattenedData as any,\n    }),\n  ]);\n}\n\n// Path: src/queries/analytics/reports/getUTM.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nexport async function getUTM(\n  ...args: [\n    websiteId: string,\n    filters: {\n      startDate: Date;\n      endDate: Date;\n      timezone?: string;\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = prisma;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {{websiteId::uuid}}\n      and created_at between {{startDate}} and {{endDate}}\n      and coalesce(url_query, '') != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: {\n    startDate: Date;\n    endDate: Date;\n    timezone?: string;\n  },\n) {\n  const { startDate, endDate } = filters;\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select url_query, count(*) as \"num\"\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n      and url_query != ''\n      and event_type = 1\n    group by 1\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n    },\n  ).then(result => parseParameters(result as any[]));\n}\n\nfunction parseParameters(data: any[]) {\n  return data.reduce((obj, { url_query, num }) => {\n    try {\n      const searchParams = new URLSearchParams(url_query);\n\n      for (const [key, value] of searchParams) {\n        if (key.match(/^utm_(\\w+)$/)) {\n          const name = safeDecodeURIComponent(value);\n          if (!obj[key]) {\n            obj[key] = { [name]: Number(num) };\n          } else if (!obj[key][name]) {\n            obj[key][name] = Number(num);\n          } else {\n            obj[key][name] += Number(num);\n          }\n        }\n      }\n    } catch {\n      // Ignore\n    }\n\n    return obj;\n  }, {});\n}\n\n// Path: src/lib/kafka.ts\nimport dateFormat from 'dateformat';\nimport debug from 'debug';\nimport { Kafka, Mechanism, Producer, RecordMetadata, SASLOptions, logLevel } from 'kafkajs';\nimport { KAFKA, KAFKA_PRODUCER } from 'lib/db';\nimport * as tls from 'tls';\n\nconst log = debug('umami:kafka');\n\nlet kafka: Kafka;\nlet producer: Producer;\nconst enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER);\n\nfunction getClient() {\n  const { username, password } = new URL(process.env.KAFKA_URL);\n  const brokers = process.env.KAFKA_BROKER.split(',');\n\n  const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions | Mechanism } =\n    username && password\n      ? {\n          ssl: {\n            checkServerIdentity: () => undefined,\n            ca: [process.env.CA_CERT],\n            key: process.env.CLIENT_KEY,\n            cert: process.env.CLIENT_CERT,\n          },\n          sasl: {\n            mechanism: 'plain',\n            username,\n            password,\n          },\n        }\n      : {};\n\n  const client: Kafka = new Kafka({\n    clientId: 'umami',\n    brokers: brokers,\n    connectionTimeout: 3000,\n    logLevel: logLevel.ERROR,\n    ...ssl,\n  });\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA] = client;\n  }\n\n  log('Kafka initialized');\n\n  return client;\n}\n\nasync function getProducer(): Promise<Producer> {\n  const producer = kafka.producer();\n  await producer.connect();\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[KAFKA_PRODUCER] = producer;\n  }\n\n  log('Kafka producer initialized');\n\n  return producer;\n}\n\nfunction getDateFormat(date: Date, format?: string): string {\n  return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');\n}\n\nasync function sendMessage(\n  message: { [key: string]: string | number },\n  topic: string,\n): Promise<RecordMetadata[]> {\n  await connect();\n\n  return producer.send({\n    topic,\n    messages: [\n      {\n        value: JSON.stringify(message),\n      },\n    ],\n    acks: -1,\n  });\n}\n\nasync function sendMessages(messages: { [key: string]: string | number }[], topic: string) {\n  await connect();\n\n  await producer.send({\n    topic,\n    messages: messages.map(a => {\n      return { value: JSON.stringify(a) };\n    }),\n    acks: 1,\n  });\n}\n\nasync function connect(): Promise<Kafka> {\n  if (!kafka) {\n    kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient());\n\n    if (kafka) {\n      producer = global[KAFKA_PRODUCER] || (await getProducer());\n    }\n  }\n\n  return kafka;\n}\n\nexport default {\n  enabled,\n  client: kafka,\n  producer,\n  log,\n  connect,\n  getDateFormat,\n  sendMessage,\n  sendMessages,\n};\n\n// Path: src/queries/analytics/eventData/saveEventData.ts\nimport { Prisma } from '@prisma/client';\nimport { DATA_TYPE } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport { flattenJSON, getStringValue } from 'lib/data';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { DynamicData } from 'lib/types';\n\nexport async function saveEventData(args: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  eventId: string;\n  eventData: DynamicData;\n}): Promise<Prisma.BatchPayload> {\n  const { websiteId, eventId, eventData } = data;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  // id, websiteEventId, eventStringValue\n  const flattenedData = jsonKeys.map(a => ({\n    id: uuid(),\n    websiteEventId: eventId,\n    websiteId,\n    eventKey: a.key,\n    stringValue: getStringValue(a.value, a.dataType),\n    numberValue: a.dataType === DATA_TYPE.number ? a.value : null,\n    dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,\n    dataType: a.dataType,\n  }));\n\n  return prisma.client.eventData.createMany({\n    data: flattenedData,\n  });\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  eventId: string;\n  sessionId?: string;\n  urlPath?: string;\n  eventName?: string;\n  eventData: DynamicData;\n  createdAt?: string;\n}) {\n  const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;\n\n  const { getDateFormat, sendMessages } = kafka;\n\n  const jsonKeys = flattenJSON(eventData);\n\n  const messages = jsonKeys.map(a => ({\n    website_id: websiteId,\n    session_id: sessionId,\n    event_id: eventId,\n    url_path: urlPath,\n    event_name: eventName,\n    event_key: a.key,\n    string_value: getStringValue(a.value, a.dataType),\n    number_value: a.dataType === DATA_TYPE.number ? a.value : null,\n    date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null,\n    data_type: a.dataType,\n    created_at: createdAt,\n  }));\n\n  await sendMessages(messages, 'event_data');\n\n  return data;\n}\n\n// Path: src/queries/analytics/events/saveEvent.ts\nimport { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE, PAGE_TITLE_LENGTH } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport kafka from 'lib/kafka';\nimport prisma from 'lib/prisma';\nimport { uuid } from 'lib/crypto';\nimport { saveEventData } from 'queries/analytics/eventData/saveEventData';\n\nexport async function saveEvent(args: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(args),\n    [CLICKHOUSE]: () => clickhouseQuery(args),\n  });\n}\n\nasync function relationalQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    eventName,\n    eventData,\n    pageTitle,\n  } = data;\n  const websiteEventId = uuid();\n\n  const websiteEvent = prisma.client.websiteEvent.create({\n    data: {\n      id: websiteEventId,\n      websiteId,\n      sessionId,\n      visitId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      urlQuery: urlQuery?.substring(0, URL_LENGTH),\n      referrerPath: referrerPath?.substring(0, URL_LENGTH),\n      referrerQuery: referrerQuery?.substring(0, URL_LENGTH),\n      referrerDomain: referrerDomain?.substring(0, URL_LENGTH),\n      pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n      eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n      eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    },\n  });\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId: websiteEventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n    });\n  }\n\n  return websiteEvent;\n}\n\nasync function clickhouseQuery(data: {\n  websiteId: string;\n  sessionId: string;\n  visitId: string;\n  urlPath: string;\n  urlQuery?: string;\n  referrerPath?: string;\n  referrerQuery?: string;\n  referrerDomain?: string;\n  pageTitle?: string;\n  eventName?: string;\n  eventData?: any;\n  hostname?: string;\n  browser?: string;\n  os?: string;\n  device?: string;\n  screen?: string;\n  language?: string;\n  country?: string;\n  subdivision1?: string;\n  subdivision2?: string;\n  city?: string;\n}) {\n  const {\n    websiteId,\n    sessionId,\n    visitId,\n    urlPath,\n    urlQuery,\n    referrerPath,\n    referrerQuery,\n    referrerDomain,\n    pageTitle,\n    eventName,\n    eventData,\n    country,\n    subdivision1,\n    subdivision2,\n    city,\n    ...args\n  } = data;\n  const { getDateFormat, sendMessage } = kafka;\n  const eventId = uuid();\n  const createdAt = getDateFormat(new Date());\n\n  const message = {\n    ...args,\n    website_id: websiteId,\n    session_id: sessionId,\n    visit_id: visitId,\n    event_id: uuid(),\n    country: country,\n    subdivision1:\n      country && subdivision1\n        ? subdivision1.includes('-')\n          ? subdivision1\n          : `${country}-${subdivision1}`\n        : null,\n    subdivision2: subdivision2,\n    city: city,\n    url_path: urlPath?.substring(0, URL_LENGTH),\n    url_query: urlQuery?.substring(0, URL_LENGTH),\n    referrer_path: referrerPath?.substring(0, URL_LENGTH),\n    referrer_query: referrerQuery?.substring(0, URL_LENGTH),\n    referrer_domain: referrerDomain?.substring(0, URL_LENGTH),\n    page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH),\n    event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,\n    event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,\n    created_at: createdAt,\n  };\n\n  await sendMessage(message, 'event');\n\n  if (eventData) {\n    await saveEventData({\n      websiteId,\n      sessionId,\n      eventId,\n      urlPath: urlPath?.substring(0, URL_LENGTH),\n      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),\n      eventData,\n      createdAt,\n    });\n  }\n\n  return data;\n}\n\n// Path: src/queries/analytics/sessions/getSessions.ts\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';\n\nexport async function getSessions(...args: [websiteId: string, startAt: Date]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string, startDate: Date) {\n  return prisma.client.session.findMany({\n    where: {\n      websiteId,\n      createdAt: {\n        gte: startDate,\n      },\n    },\n    orderBy: {\n      createdAt: 'asc',\n    },\n  });\n}\n\nasync function clickhouseQuery(websiteId: string, startDate: Date) {\n  const { rawQuery } = clickhouse;\n\n  return rawQuery(\n    `\n    select\n      session_id as id,\n      website_id as websiteId,\n      created_at as createdAt,\n      toUnixTimestamp(created_at) as timestamp,\n      hostname,\n      browser,\n      os,\n      device,\n      screen,\n      language,\n      country,\n      subdivision1,\n      subdivision2,\n      city\n    from website_event\n    where website_id = {websiteId:UUID}\n    and created_at >= {startDate:DateTime64}\n    order by created_at asc\n    `,\n    {\n      websiteId,\n      startDate,\n    },\n  );\n}\n\n// Path: src/queries/analytics/reports/getFunnel.ts\nimport clickhouse from 'lib/clickhouse';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\n\nconst formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {\n  return steps.map((step: { type: string; value: string }, i: number) => {\n    const visitors = Number(results[i]?.count) || 0;\n    const previous = Number(results[i - 1]?.count) || 0;\n    const dropped = previous > 0 ? previous - visitors : 0;\n    const dropoff = 1 - visitors / previous;\n    const remaining = visitors / Number(results[0].count);\n\n    return {\n      ...step,\n      visitors,\n      previous,\n      dropped,\n      dropoff,\n      remaining,\n    };\n  });\n};\n\nexport async function getFunnel(\n  ...args: [\n    websiteId: string,\n    criteria: {\n      windowMinutes: number;\n      startDate: Date;\n      endDate: Date;\n      steps: { type: string; value: string }[];\n    },\n  ]\n) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery, getAddIntervalQuery } = prisma;\n  const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, windowMinutes);\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    params: string[];\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union ' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\n          WITH level1 AS (\n            select distinct session_id, created_at\n            from website_event\n            where website_id = {{websiteId::uuid}}\n              and created_at between {{startDate}} and {{endDate}}\n              and ${column} ${operator} {{${i}}}\n          )`;\n        } else {\n          pv.levelQuery += `\n          , level${levelNumber} AS (\n            select distinct we.session_id, we.created_at\n            from level${i} l\n            join website_event we\n                on l.session_id = we.session_id\n            where we.website_id = {{websiteId::uuid}}\n                and we.created_at between l.created_at and ${getAddIntervalQuery(\n                  `l.created_at `,\n                  `${windowMinutes} minute`,\n                )}\n                and we.${column} ${operator} {{${i}}}\n                and we.created_at <= {{endDate}}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.params.push(paramValue);\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        params: [],\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    ${levelOneQuery}\n    ${levelQuery}\n    ${sumQuery}\n    ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  criteria: {\n    windowMinutes: number;\n    startDate: Date;\n    endDate: Date;\n    steps: { type: string; value: string }[];\n  },\n): Promise<\n  {\n    value: string;\n    visitors: number;\n    dropoff: number;\n  }[]\n> {\n  const { windowMinutes, startDate, endDate, steps } = criteria;\n  const { rawQuery } = clickhouse;\n  const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(\n    steps,\n    windowMinutes,\n  );\n\n  function getFunnelQuery(\n    steps: { type: string; value: string }[],\n    windowMinutes: number,\n  ): {\n    levelOneQuery: string;\n    levelQuery: string;\n    sumQuery: string;\n    stepFilterQuery: string;\n    params: { [key: string]: string };\n  } {\n    return steps.reduce(\n      (pv, cv, i) => {\n        const levelNumber = i + 1;\n        const startSum = i > 0 ? 'union all ' : '';\n        const startFilter = i > 0 ? 'or' : '';\n        const operator = cv.type === 'url' && cv.value.endsWith('*') ? 'like' : '=';\n        const column = cv.type === 'url' ? 'url_path' : 'event_name';\n        const paramValue = cv.value.endsWith('*') ? cv.value.replace('*', '%') : cv.value;\n\n        if (levelNumber === 1) {\n          pv.levelOneQuery = `\\n\n          level1 AS (\n            select *\n            from level0\n            where ${column} ${operator} {param${i}:String}\n          )`;\n        } else {\n          pv.levelQuery += `\\n\n          , level${levelNumber} AS (\n            select distinct y.session_id as session_id,\n                y.url_path as url_path,\n                y.referrer_path as referrer_path,\n                y.event_name,\n                y.created_at as created_at\n            from level${i} x\n            join level0 y\n            on x.session_id = y.session_id\n            where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute\n                and y.${column} ${operator} {param${i}:String}\n          )`;\n        }\n\n        pv.sumQuery += `\\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;\n        pv.stepFilterQuery += `${startFilter} ${column} ${operator} {param${i}:String} `;\n        pv.params[`param${i}`] = paramValue;\n\n        return pv;\n      },\n      {\n        levelOneQuery: '',\n        levelQuery: '',\n        sumQuery: '',\n        stepFilterQuery: '',\n        params: {},\n      },\n    );\n  }\n\n  return rawQuery(\n    `\n    WITH level0 AS (\n      select distinct session_id, url_path, referrer_path, event_name, created_at\n      from umami.website_event\n      where (${stepFilterQuery})\n        and website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n    ),\n    ${levelOneQuery}\n    ${levelQuery}\n    select *\n    from (\n      ${sumQuery} \n    ) ORDER BY level;\n    `,\n    {\n      websiteId,\n      startDate,\n      endDate,\n      ...params,\n    },\n  ).then(formatResults(steps));\n}\n\n// Path: src/queries/analytics/getWebsiteStats.ts\nimport clickhouse from 'lib/clickhouse';\nimport { EVENT_TYPE } from 'lib/constants';\nimport { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';\nimport prisma from 'lib/prisma';\nimport { QueryFilters } from 'lib/types';\n\nexport async function getWebsiteStats(\n  ...args: [websiteId: string, filters: QueryFilters]\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma;\n  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select\n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(case when t.c = 1 then 1 else 0 end) as \"bounces\",\n      sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as \"totaltime\"\n    from (\n      select\n        website_event.session_id,\n        website_event.visit_id,\n        count(*) as \"c\",\n        min(website_event.created_at) as \"min_time\",\n        max(website_event.created_at) as \"max_time\"\n      from website_event\n      join website \n        on website_event.website_id = website.website_id\n        ${joinSession}\n      where website.website_id = {{websiteId::uuid}}\n        and website_event.created_at between {{startDate}} and {{endDate}}\n        and event_type = {{eventType}}\n        ${filterQuery}\n      group by 1, 2\n    ) as t\n    `,\n    params,\n  );\n}\n\nasync function clickhouseQuery(\n  websiteId: string,\n  filters: QueryFilters,\n): Promise<\n  { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]\n> {\n  const { rawQuery, parseFilters } = clickhouse;\n  const { filterQuery, params } = await parseFilters(websiteId, {\n    ...filters,\n    eventType: EVENT_TYPE.pageView,\n  });\n\n  return rawQuery(\n    `\n    select \n      sum(t.c) as \"pageviews\",\n      count(distinct t.session_id) as \"visitors\",\n      count(distinct t.visit_id) as \"visits\",\n      sum(if(t.c = 1, 1, 0)) as \"bounces\",\n      sum(max_time-min_time) as \"totaltime\"\n    from (\n      select\n        session_id,\n        visit_id,\n        count(*) c,\n        min(created_at) min_time,\n        max(created_at) max_time\n      from website_event\n      where website_id = {websiteId:UUID}\n        and created_at between {startDate:DateTime64} and {endDate:DateTime64}\n        and event_type = {eventType:UInt32}\n        ${filterQuery}\n      group by session_id, visit_id\n    ) as t;\n    `,\n    params,\n  ).then(result => {\n    return Object.values(result).map(n => {\n      return {\n        pageviews: Number(n.pageviews),\n        visitors: Number(n.visitors),\n        visits: Number(n.visits),\n        bounces: Number(n.bounces),\n        totaltime: Number(n.totaltime),\n      };\n    });\n  });\n}\n\n// Path: src/queries/analytics/getActiveVisitors.ts\nimport { subMinutes } from 'date-fns';\nimport prisma from 'lib/prisma';\nimport clickhouse from 'lib/clickhouse';\nimport { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';\n\nexport async function getActiveVisitors(...args: [websiteId: string]) {\n  return runQuery({\n    [PRISMA]: () => relationalQuery(...args),\n    [CLICKHOUSE]: () => clickhouseQuery(...args),\n  });\n}\n\nasync function relationalQuery(websiteId: string) {\n  const { rawQuery } = prisma;\n\n  const result = await rawQuery(\n    `\n    select count(distinct session_id) x\n    from website_event\n    where website_id = {{websiteId::uuid}}\n    and created_at >= {{startAt}}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  );\n\n  return result[0] ?? null;\n}\n\nasync function clickhouseQuery(websiteId: string): Promise<{ x: number }> {\n  const { rawQuery } = clickhouse;\n\n  const result = await rawQuery(\n    `\n    select\n      count(distinct session_id) x\n    from website_event\n    where website_id = {websiteId:UUID}\n      and created_at >= {startAt:DateTime64}\n    `,\n    { websiteId, startAt: subMinutes(new Date(), 5) },\n  ).then(a => {\n    return Object.values(a).map(a => {\n      return { x: Number(a.x) };\n    });\n  });\n\n  return result[0] ?? null;\n}\n\n// Path: src/queries/admin/team.ts\nimport { Prisma, Team } from '@prisma/client';\nimport { ROLES } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport prisma from 'lib/prisma';\nimport { FilterResult, TeamSearchFilter } from 'lib/types';\nimport TeamFindManyArgs = Prisma.TeamFindManyArgs;\n\nexport async function findTeam(criteria: Prisma.TeamFindUniqueArgs): Promise<Team> {\n  return prisma.client.team.findUnique(criteria);\n}\n\nexport async function getTeam(teamId: string, options: { includeMembers?: boolean } = {}) {\n  const { includeMembers } = options;\n\n  return findTeam({\n    where: {\n      id: teamId,\n    },\n    ...(includeMembers && { include: { teamUser: true } }),\n  });\n}\n\nexport async function getTeams(\n  criteria: TeamFindManyArgs,\n  filters: TeamSearchFilter = {},\n): Promise<FilterResult<Team[]>> {\n  const { getSearchParameters } = prisma;\n  const { query } = filters;\n\n  const where: Prisma.TeamWhereInput = {\n    ...criteria.where,\n    ...getSearchParameters(query, [{ name: 'contains' }]),\n  };\n\n  return prisma.pagedQuery<TeamFindManyArgs>(\n    'team',\n    {\n      ...criteria,\n      where,\n    },\n    filters,\n  );\n}\n\nexport async function getUserTeams(userId: string, filters: TeamSearchFilter = {}) {\n  return getTeams(\n    {\n      where: {\n        deletedAt: null,\n        teamUser: {\n          some: { userId },\n        },\n      },\n      include: {\n        teamUser: {\n          include: {\n            user: {\n              select: {\n                id: true,\n                username: true,\n              },\n            },\n          },\n        },\n        _count: {\n          select: {\n            website: {\n              where: { deletedAt: null },\n            },\n            teamUser: {\n              where: {\n                user: { deletedAt: null },\n              },\n            },\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createTeam(data: Prisma.TeamCreateInput, userId: string): Promise<any> {\n  const { id } = data;\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.team.create({\n      data,\n    }),\n    client.teamUser.create({\n      data: {\n        id: uuid(),\n        teamId: id,\n        userId,\n        role: ROLES.teamOwner,\n      },\n    }),\n  ]);\n}\n\nexport \nasync function updateTeam(teamId: string, data: Prisma.TeamUpdateInput): Promise<Team> {\n  const { client } = prisma;\n\n  return client.team.update({\n    where: {\n      id: teamId,\n    },\n    data: {\n      ...data,\n      updatedAt: new Date(),\n    },\n  });\n}\n\nexport async function deleteTeam(\n  teamId: string,\n): Promise<Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Team]>> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  if (cloudMode) {\n    return transaction([\n      client.team.update({\n        data: {\n          deletedAt: new Date(),\n        },\n        where: {\n          id: teamId,\n        },\n      }),\n    ]);\n  }\n\n  return transaction([\n    client.teamUser.deleteMany({\n      where: {\n        teamId,\n      },\n    }),\n    client.team.delete({\n      where: {\n        id: teamId,\n      },\n    }),\n  ]);\n}\n\n// Path: src/queries/index.ts\nexport * from './admin/report';\nexport * from './admin/team';\nexport * from './admin/teamUser';\nexport * from './admin/user';\nexport * from './admin/website';\nexport * from './analytics/events/getEventMetrics';\nexport * from './analytics/events/getEventUsage';\nexport * from './analytics/events/getEvents';\nexport * from './analytics/eventData/getEventDataEvents';\nexport * from './analytics/eventData/getEventDataFields';\nexport * from './analytics/eventData/getEventDataStats';\nexport * from './analytics/eventData/getEventDataUsage';\nexport * from './analytics/events/saveEvent';\nexport * from './analytics/reports/getFunnel';\nexport * from './analytics/reports/getRetention';\nexport * from './analytics/reports/getInsights';\nexport * from './analytics/reports/getUTM';\nexport * from './analytics/pageviews/getPageviewMetrics';\nexport * from './analytics/pageviews/getPageviewStats';\nexport * from './analytics/sessions/createSession';\nexport * from './analytics/sessions/getSession';\nexport * from './analytics/sessions/getSessionMetrics';\nexport * from './analytics/sessions/getSessions';\nexport * from './analytics/sessions/getSessionStats';\nexport * from './analytics/sessions/saveSessionData';\nexport * from './analytics/getActiveVisitors';\nexport * from './analytics/getRealtimeData';\nexport * from './analytics/getValues';\nexport * from './analytics/getWebsiteDateRange';\nexport * from './analytics/getWebsiteStats';\n\n// Path: src/lib/cache.ts\nimport { User, Website } from '@prisma/client';\nimport redis from '@umami/redis-client';\nimport { getSession, getUser, getWebsite } from '../queries';\n\nasync function fetchWebsite(websiteId: string): Promise<Website> {\n  return redis.client.getCache(`website:${websiteId}`, () => getWebsite(websiteId), 86400);\n}\n\nasync function storeWebsite(data: { id: any }) {\n  const { id } = data;\n  const key = `website:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteWebsite(id) {\n  return redis.client.deleteCache(`website:${id}`);\n}\n\nasync function fetchUser(id): Promise<User> {\n  return redis.client.getCache(`user:${id}`, () => getUser(id, { includePassword: true }), 86400);\n}\n\nasync function storeUser(data) {\n  const { id } = data;\n  const key = `user:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteUser(id) {\n  return redis.client.deleteCache(`user:${id}`);\n}\n\nasync function fetchSession(id) {\n  return redis.client.getCache(`session:${id}`, () => getSession(id), 86400);\n}\n\nasync function storeSession(data) {\n  const { id } = data;\n  const key = `session:${id}`;\n\n  const obj = await redis.client.setCache(key, data);\n  await redis.client.expire(key, 86400);\n\n  return obj;\n}\n\nasync function deleteSession(id) {\n  return redis.client.deleteCache(`session:${id}`);\n}\n\nasync function fetchUserBlock(userId: string) {\n  const key = `user:block:${userId}`;\n  return redis.client.get(key);\n}\n\nasync function incrementUserBlock(userId: string) {\n  const key = `user:block:${userId}`;\n  return redis.client.incr(key);\n}\n\nexport default {\n  fetchWebsite,\n  storeWebsite,\n  deleteWebsite,\n  fetchUser,\n  storeUser,\n  deleteUser,\n  fetchSession,\n  storeSession,\n  deleteSession,\n  fetchUserBlock,\n  incrementUserBlock,\n  enabled: !!redis.enabled,\n};\n\n// Path: src/lib/detect.ts\nimport path from 'path';\nimport { getClientIp } from 'request-ip';\nimport { browserName, detectOS } from 'detect-browser';\nimport isLocalhost from 'is-localhost-ip';\nimport maxmind from 'maxmind';\nimport { safeDecodeURIComponent } from 'next-basics';\n\nimport {\n  DESKTOP_OS,\n  MOBILE_OS,\n  DESKTOP_SCREEN_WIDTH,\n  LAPTOP_SCREEN_WIDTH,\n  MOBILE_SCREEN_WIDTH,\n} from './constants';\nimport { NextApiRequestCollect } from 'pages/api/send';\n\nlet lookup;\n\nexport function getIpAddress(req: NextApiRequestCollect) {\n  const customHeader = String(process.env.CLIENT_IP_HEADER).toLowerCase();\n\n  // Custom header\n  if (customHeader !== 'undefined' && req.headers[customHeader]) {\n    return req.headers[customHeader];\n  }\n  // Cloudflare\n  else if (req.headers['cf-connecting-ip']) {\n    return req.headers['cf-connecting-ip'];\n  }\n\n  return getClientIp(req);\n}\n\nexport function getDevice(screen: string, os: string) {\n  if (!screen) return;\n\n  const [width] = screen.split('x');\n\n  if (DESKTOP_OS.includes(os)) {\n    if (os === 'Chrome OS' || +width < DESKTOP_SCREEN_WIDTH) {\n      return 'laptop';\n    }\n    return 'desktop';\n  } else if (MOBILE_OS.includes(os)) {\n    if (os === 'Amazon OS' || +width > MOBILE_SCREEN_WIDTH) {\n      return 'tablet';\n    }\n    return 'mobile';\n  }\n\n  if (+width >= DESKTOP_SCREEN_WIDTH) {\n    return 'desktop';\n  } else if (+width >= LAPTOP_SCREEN_WIDTH) {\n    return 'laptop';\n  } else if (+width >= MOBILE_SCREEN_WIDTH) {\n    return 'tablet';\n  } else {\n    return 'mobile';\n  }\n}\n\nfunction getRegionCode(country: string, region: string) {\n  if (!country || !region) {\n    return undefined;\n  }\n\n  return region.includes('-') ? region : `${country}-${region}`;\n}\n\nexport async function getLocation(ip: string, req: NextApiRequestCollect) {\n  // Ignore local ips\n  if (await isLocalhost(ip)) {\n    return;\n  }\n\n  // Cloudflare headers\n  if (req.headers['cf-ipcountry']) {\n    const country = safeDecodeURIComponent(req.headers['cf-ipcountry']);\n    const subdivision1 = safeDecodeURIComponent(req.headers['cf-region-code']);\n    const city = safeDecodeURIComponent(req.headers['cf-ipcity']);\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      city,\n    };\n  }\n\n  // Vercel headers\n  if (req.headers['x-vercel-ip-country']) {\n    const country = safeDecodeURIComponent(req.headers['x-vercel-ip-country']);\n    const subdivision1 = safeDecodeURIComponent(req.headers['x-vercel-ip-country-region']);\n    const city = safeDecodeURIComponent(req.headers['x-vercel-ip-city']);\n\n    return {\n      country,\n      subdivision1: getRegionCode(country, subdivision1),\n      city,\n    };\n  }\n\n  // Database lookup\n  if (!lookup) {\n    const dir = path.join(process.cwd(), 'geo');\n\n    lookup = await maxmind.open(path.resolve(dir, 'GeoLite2-City.mmdb'));\n  }\n\n  const result = lookup.get(ip);\n\n  if (result) {\n    const country = result.country?.iso_code ?? result?.registered_country?.iso_code;\n    const subdivision1 = result.subdivisions?.[0]?.iso_code;\n...\n", "needle_token_start": 13699, "needle_token_end": 13785, "code_context_ntokens": 16157, "output": ["```typescript\n// Path: src/queries/admin/team.ts\nimport { Prisma, Team } from '@prisma/client';\nimport { ROLES } from 'lib/constants';\nimport { uuid } from 'lib/crypto';\nimport prisma from 'lib/prisma';\nimport { FilterResult, TeamSearchFilter } from 'lib/types';\nimport TeamFindManyArgs = Prisma.TeamFindManyArgs;\n\nexport async function findTeam(criteria: Prisma.TeamFindUniqueArgs): Promise<Team> {\n  return prisma.client.team.findUnique(criteria);\n}\n\nexport async function getTeam(teamId: string, options: { includeMembers?: boolean } = {}) {\n  const { includeMembers } = options;\n\n  return findTeam({\n    where: {\n      id: teamId,\n    },\n    ...(includeMembers && { include: { teamUser: true } }),\n  });\n}\n\nexport async function getTeams(\n  criteria: TeamFindManyArgs,\n  filters: TeamSearchFilter = {},\n): Promise<FilterResult<Team[]>> {\n  const { getSearchParameters } = prisma;\n  const { query } = filters;\n\n  const where: Prisma.TeamWhereInput = {\n    ...criteria.where,\n    ...getSearchParameters(query, [{ name: 'contains' }]),\n  };\n\n  return prisma.pagedQuery<TeamFindManyArgs>(\n    'team',\n    {\n      ...criteria,\n      where,\n    },\n    filters,\n  );\n}\n\nexport async function getUserTeams(userId: string, filters: TeamSearchFilter = {}) {\n  return getTeams(\n    {\n      where: {\n        deletedAt: null,\n        teamUser: {\n          some: { userId },\n        },\n      },\n      include: {\n        teamUser: {\n          include: {\n            user: {\n              select: {\n                id: true,\n                username: true,\n              },\n            },\n          },\n        },\n        _count: {\n          select: {\n            website: {\n              where: { deletedAt: null },\n            },\n            teamUser: {\n              where: {\n                user: { deletedAt: null },\n              },\n            },\n          },\n        },\n      },\n    },\n    filters,\n  );\n}\n\nexport async function createTeam(data: Prisma.TeamCreateInput, userId: string): Promise<any> {\n  const { id } = data;\n  const { client, transaction } = prisma;\n\n  return transaction([\n    client.team.create({\n      data,\n    }),\n    client.teamUser.create({\n      data: {\n        id: uuid(),\n        teamId: id,\n        userId,\n        role: ROLES.teamOwner,\n      },\n    }),\n  ]);\n}\n\nexport async function updateTeam(teamId: string, data: Prisma.TeamUpdateInput): Promise<Team> {\n  const { client } = prisma;\n\n  return client.team.update({\n    where: {\n      id: teamId,\n    },\n    data: {\n      ...data,\n      updatedAt: new Date(),\n    },\n  });\n}\n\nexport async function deleteTeam(\n  teamId: string,\n): Promise<Promise<[Prisma.BatchPayload, Prisma.BatchPayload, Team]>> {\n  const { client, transaction } = prisma;\n  const cloudMode = process.env.CLOUD_MODE;\n\n  if (cloudMode) {\n    return transaction([\n      client.team.update({\n        data: {\n          deletedAt: new Date(),\n        },\n        where: {\n          id: teamId,\n        },\n      }),\n    ]);\n  }\n\n  return transaction([\n    client.teamUser.deleteMany({\n      where: {\n        teamId,\n      },\n    }),\n    client.team.delete({\n      where: {\n        id: teamId,\n      },\n    }),\n  ]);\n}\n```"]}
{"repo": "umami-software/umami", "name": "getTimestampDiffQuery", "language": "typescript", "path": "src/lib/prisma.ts", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function calculates the difference in seconds between two timestamp fields.\n2. **Input**: Two timestamp fields (start and end).\n3. **Output**: A string representing a SQL query that when executed, returns the difference in seconds between the two timestamps.\n4. **Procedure**: Depending on the database type (either PostgreSQL or MySQL), the function constructs a SQL query using database-specific functions to calculate the difference in seconds between the two provided timestamp fields.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/components/hooks/useTheme.ts\nimport { useEffect, useMemo } from 'react';\nimport useStore, { setTheme } from 'store/app';\nimport { getItem, setItem } from 'next-basics';\nimport { DEFAULT_THEME, THEME_COLORS, THEME_CONFIG } from 'lib/constants';\nimport { colord } from 'colord';\n\nconst selector = (state: { theme: string }) => state.theme;\n\nexport function useTheme() {\n  const theme = useStore(selector) || getItem(THEME_CONFIG) || DEFAULT_THEME;\n  const primaryColor = colord(THEME_COLORS[theme].primary);\n\n  const colors = useMemo(() => {\n    return {\n      theme: {\n        ...THEME_COLORS[theme],\n      },\n      chart: {\n        text: THEME_COLORS[theme].gray700,\n        line: THEME_COLORS[theme].gray200,\n        views: {\n          hoverBackgroundColor: primaryColor.alpha(0.7).toRgbString(),\n          backgroundColor: primaryColor.alpha(0.4).toRgbString(),\n          borderColor: primaryColor.alpha(0.7).toRgbString(),\n          hoverBorderColor: primaryColor.toRgbString(),\n        },\n        visitors: {\n          hoverBackgroundColor: primaryColor.alpha(0.9).toRgbString(),\n          backgroundColor: primaryColor.alpha(0.6).toRgbString(),\n          borderColor: primaryColor.alpha(0.9).toRgbString(),\n          hoverBorderColor: primaryColor.toRgbString(),\n        },\n      },\n      map: {\n        baseColor: THEME_COLORS[theme].primary,\n        fillColor: THEME_COLORS[theme].gray100,\n        strokeColor: THEME_COLORS[theme].primary,\n        hoverColor: THEME_COLORS[theme].primary,\n      },\n    };\n  }, [theme]);\n\n  const saveTheme = (value: string) => {\n    setItem(THEME_CONFIG, value);\n    setTheme(value);\n  };\n\n  useEffect(() => {\n...\n// Path: src/components/hooks/queries/useConfig.ts\nimport { useEffect } from 'react';\nimport useStore, { setConfig } from 'store/app';\nimport { useApi } from './useApi';\n\nlet loading = false;\n\nexport function useConfig() {\n  const { config } = useStore();\n  const { get } = useApi();\n  const configUrl = process.env.configUrl;\n\n  async function loadConfig() {\n    const data = await get(configUrl);\n    loading = false;\n    setConfig(data);\n  }\n\n  useEffect(() => {\n    if (!config && !loading && configUrl) {\n      loading = true;\n      loadConfig();\n    }\n  }, []);\n\n  return config;\n}\n\nexport default useConfig;\n\n// Path: src/components/hooks/queries/useReports.ts\nimport useApi from './useApi';\nimport useFilterQuery from './useFilterQuery';\nimport useModified from '../useModified';\n\nexport function useReports({ websiteId, teamId }: { websiteId?: string; teamId?: string }) {\n  const { modified } = useModified(`reports`);\n  const { get, del, useMutation } = useApi();\n  const queryResult = useFilterQuery({\n    queryKey: ['reports', { websiteId, teamId, modified }],\n    queryFn: (params: any) => {\n      return get('/reports', { websiteId, teamId, ...params });\n    },\n  });\n  const { mutate } = useMutation({ mutationFn: (reportId: string) => del(`/reports/${reportId}`) });\n\n  const deleteReport = (reportId: any) => {\n    mutate(reportId, {\n      onSuccess: () => {},\n    });\n  };\n\n  return {\n    ...queryResult,\n    deleteReport,\n  };\n}\n\nexport default useReports;\n\n// Path: src/components/hooks/useEscapeKey.ts\nimport { useEffect, useCallback, KeyboardEvent } from 'react';\n\nexport function useEscapeKey(handler: (event: KeyboardEvent) => void) {\n  const escFunction = useCallback((event: KeyboardEvent) => {\n    if (event.key === 'Escape') {\n      handler(event);\n    }\n  }, []);\n\n  useEffect(() => {\n    document.addEventListener('keydown', escFunction as any, false);\n\n    return () => {\n      document.removeEventListener('keydown', escFunction as any, false);\n    };\n  }, [escFunction]);\n\n  return null;\n}\n\nexport default useEscapeKey;\n\n// Path: src/components/hooks/queries/useUsers.ts\nimport useApi from './useApi';\nimport useFilterQuery from './useFilterQuery';\nimport useModified from '../useModified';\n\nexport function useUsers() {\n  const { get } = useApi();\n  const { modified } = useModified(`users`);\n\n  return useFilterQuery({\n    queryKey: ['users', { modified }],\n    queryFn: (params: any) => {\n      return get('/admin/users', {\n        ...params,\n      });\n    },\n  });\n}\n\nexport default useUsers;\n\n// Path: src/lib/filters.ts\nexport const urlFilter = (data: any[]) => {\n  const map = data.reduce((obj, { x, y }) => {\n    if (x) {\n      if (!obj[x]) {\n        obj[x] = y;\n      } else {\n        obj[x] += y;\n      }\n    }\n\n    return obj;\n  }, {});\n\n  return Object.keys(map).map(key => ({ x: key, y: map[key] }));\n};\n\nexport const refFilter = (data: any[]) => {\n  const links = {};\n\n  const map = data.reduce((obj, { x, y }) => {\n    let id;\n\n    try {\n      const url = new URL(x);\n\n      id = url.hostname.replace(/www\\./, '') || url.href;\n    } catch {\n      id = '';\n    }\n\n    links[id] = x;\n\n    if (!obj[id]) {\n      obj[id] = y;\n    } else {\n      obj[id] += y;\n    }\n\n    return obj;\n  }, {});\n\n  return Object.keys(map).map(key => ({ x: key, y: map[key], w: links[key] }));\n};\n\nexport const emptyFilter = (data: any[]) => {\n  return data.map(item => (item.x ? item : null)).filter(n => n);\n};\n\nexport const percentFilter = (data: any[]) => {\n  const total = data.reduce((n, { y }) => n + y, 0);\n  return data.map(({ x, y, ...props }) => ({ x, y, z: total ? (y / total) * 100 : 0, ...props }));\n};\n\nexport const paramFilter = (data: any[]) => {\n  const map = data.reduce((obj, { x, y }) => {\n    try {\n      const searchParams = new URLSearchParams(x);\n\n      for (const [key, value] of searchParams) {\n        if (!obj[key]) {\n          obj[key] = { [value]: y };\n        } else if (!obj[key][value]) {\n          obj[key][value] = y;\n        } else {\n          obj[key][value] += y;\n        }\n      }\n    } catch {\n      // Ignore\n    }\n\n    return obj;\n  }, {});\n\n  return Object.keys(map).flatMap(key =>\n    Object.keys(map[key]).map(n => ({ x: `${key}=${n}`, p: key, v: n, y: map[key][n] })),\n  );\n};\n\n// Path: src/components/hooks/queries/useRealtime.ts\nimport { useMemo, useRef } from 'react';\nimport { RealtimeData } from 'lib/types';\nimport { useApi } from 'components/hooks';\nimport { REALTIME_INTERVAL, REALTIME_RANGE } from 'lib/constants';\nimport { startOfMinute, subMinutes } from 'date-fns';\nimport { percentFilter } from 'lib/filters';\nimport thenby from 'thenby';\n\nfunction mergeData(state = [], data = [], time: number) {\n  const ids = state.map(({ id }) => id);\n  return state\n    .concat(data.filter(({ id }) => !ids.includes(id)))\n    .filter(({ timestamp }) => timestamp >= time);\n}\n\nexport function useRealtime(websiteId: string) {\n  const currentData = useRef({\n    pageviews: [],\n    sessions: [],\n    events: [],\n    countries: [],\n    visitors: [],\n    timestamp: 0,\n  });\n  const { get, useQuery } = useApi();\n  const { data, isLoading, error } = useQuery<RealtimeData>({\n    queryKey: ['realtime', websiteId],\n    queryFn: async () => {\n      const state = currentData.current;\n      const data = await get(`/realtime/${websiteId}`, { startAt: state?.timestamp || 0 });\n      const date = subMinutes(startOfMinute(new Date()), REALTIME_RANGE);\n      const time = date.getTime();\n      const { pageviews, sessions, events, timestamp } = data;\n\n      return {\n        pageviews: mergeData(state?.pageviews, pageviews, time),\n        sessions: mergeData(state?.sessions, sessions, time),\n        events: mergeData(state?.events, events, time),\n        timestamp,\n      };\n    },\n    enabled: !!websiteId,\n    refetchInterval: REALTIME_INTERVAL,\n  });\n\n  const realtimeData: RealtimeData = useMemo(() => {\n    if (!data) {\n      return { pageviews: [], sessions: [], events: [], countries: [], visitors: [], timestamp: 0 };\n    }\n\n    data.countries = percentFilter(\n      data.sessions\n        .reduce((arr, data) => {\n          if (!arr.find(({ id }) => id === data.id)) {\n            return arr.concat(data);\n          }\n          return arr;\n        }, [])\n        .reduce((arr: { x: any; y: number }[], { country }: any) => {\n          if (country) {\n            const row = arr.find(({ x }) => x === country);\n\n            if (!row) {\n              arr.push({ x: country, y: 1 });\n            } else {\n              row.y += 1;\n            }\n          }\n          return arr;\n        }, [])\n        .sort(thenby.firstBy('y', -1)),\n    );\n\n    data.visitors = data.sessions.reduce((arr, val) => {\n      if (!arr.find(({ id }) => id === val.id)) {\n        return arr.concat(val);\n      }\n      return arr;\n    }, []);\n\n    return data;\n  }, [data]);\n\n  return { data: realtimeData, isLoading, error };\n}\n\nexport default useRealtime;\n\n// Path: src/components/hooks/queries/useWebsiteMetrics.ts\nimport useApi from './useApi';\nimport { UseQueryOptions } from '@tanstack/react-query';\n\nexport function useWebsiteMetrics(\n  websiteId: string,\n  params?: { [key: string]: any },\n  options?: Omit<UseQueryOptions & { onDataLoad?: (data: any) => void }, 'queryKey' | 'queryFn'>,\n) {\n  const { get, useQuery } = useApi();\n\n  return useQuery({\n    queryKey: [\n      'websites:metrics',\n      {\n        websiteId,\n        ...params,\n      },\n    ],\n    queryFn: async () => {\n      const filters = { ...params };\n\n      filters[params.type] = undefined;\n\n      const data = await get(`/websites/${websiteId}/metrics`, {\n        ...filters,\n      });\n\n      options?.onDataLoad?.(data);\n\n      return data;\n    },\n    ...options,\n  });\n}\n\nexport default useWebsiteMetrics;\n\n// Path: src/components/hooks/useFilters.ts\nimport { useMessages } from './useMessages';\nimport { OPERATORS } from 'lib/constants';\n\nexport function useFilters() {\n  const { formatMessage, labels } = useMessages();\n\n  const operatorLabels = {\n    [OPERATORS.equals]: formatMessage(labels.is),\n    [OPERATORS.notEquals]: formatMessage(labels.isNot),\n    [OPERATORS.set]: formatMessage(labels.isSet),\n    [OPERATORS.notSet]: formatMessage(labels.isNotSet),\n    [OPERATORS.contains]: formatMessage(labels.contains),\n    [OPERATORS.doesNotContain]: formatMessage(labels.doesNotContain),\n    [OPERATORS.true]: formatMessage(labels.true),\n    [OPERATORS.false]: formatMessage(labels.false),\n    [OPERATORS.greaterThan]: formatMessage(labels.greaterThan),\n    [OPERATORS.lessThan]: formatMessage(labels.lessThan),\n    [OPERATORS.greaterThanEquals]: formatMessage(labels.greaterThanEquals),\n    [OPERATORS.lessThanEquals]: formatMessage(labels.lessThanEquals),\n    [OPERATORS.before]: formatMessage(labels.before),\n    [OPERATORS.after]: formatMessage(labels.after),\n  };\n\n  const typeFilters = {\n    string: [OPERATORS.equals, OPERATORS.notEquals, OPERATORS.contains, OPERATORS.doesNotContain],\n    array: [OPERATORS.contains, OPERATORS.doesNotContain],\n    boolean: [OPERATORS.true, OPERATORS.false],\n    number: [\n      OPERATORS.equals,\n      OPERATORS.notEquals,\n      OPERATORS.greaterThan,\n      OPERATORS.lessThan,\n      OPERATORS.greaterThanEquals,\n      OPERATORS.lessThanEquals,\n    ],\n    date: [OPERATORS.before, OPERATORS.after],\n    uuid: [OPERATORS.equals],\n  };\n\n  const filters = Object.keys(typeFilters).flatMap(key => {\n    return (\n      typeFilters[key]?.map(value => ({ type: key, value, label: operatorLabels[value] })) ?? []\n    );\n  });\n\n  const getFilters = type => {\n    return typeFilters[type]?.map(key => ({ type, value: key, label: operatorLabels[key] })) ?? [];\n  };\n\n  return { filters, operatorLabels, typeFilters, getFilters };\n}\n\nexport default useFilters;\n\n// Path: src/components/hooks/queries/useWebsiteEvents.ts\nimport useApi from './useApi';\nimport { UseQueryOptions } from '@tanstack/react-query';\nimport { useDateRange, useNavigation, useTimezone } from 'components/hooks';\nimport { zonedTimeToUtc } from 'date-fns-tz';\n\nexport function useWebsiteEvents(\n  websiteId: string,\n  options?: Omit<UseQueryOptions, 'queryKey' | 'queryFn'>,\n) {\n  const { get, useQuery } = useApi();\n  const [dateRange] = useDateRange(websiteId);\n  const { startDate, endDate, unit, offset } = dateRange;\n  const { timezone } = useTimezone();\n  const {\n    query: { url, event },\n  } = useNavigation();\n\n  const params = {\n    startAt: +zonedTimeToUtc(startDate, timezone),\n    endAt: +zonedTimeToUtc(endDate, timezone),\n    unit,\n    offset,\n    timezone,\n    url,\n    event,\n  };\n\n  return useQuery({\n    queryKey: ['events', { ...params }],\n    queryFn: () => get(`/websites/${websiteId}/events`, { ...params }),\n    enabled: !!websiteId,\n    ...options,\n  });\n}\n\nexport default useWebsiteEvents;\n\n// Path: src/components/hooks/queries/useTeamMembers.ts\nimport useApi from './useApi';\nimport useFilterQuery from './useFilterQuery';\nimport useModified from '../useModified';\n\nexport function useTeamMembers(teamId: string) {\n  const { get } = useApi();\n  const { modified } = useModified(`teams:members`);\n\n  return useFilterQuery({\n    queryKey: ['teams:members', { teamId, modified }],\n    queryFn: (params: any) => {\n      return get(`/teams/${teamId}/users`, params);\n    },\n    enabled: !!teamId,\n  });\n}\n\nexport default useTeamMembers;\n\n// Path: src/components/hooks/useFormat.ts\nimport useMessages from './useMessages';\nimport { BROWSERS, OS_NAMES } from 'lib/constants';\nimport useLocale from './useLocale';\nimport useCountryNames from './useCountryNames';\nimport regions from '../../../public/iso-3166-2.json';\n\nexport function useFormat() {\n  const { formatMessage, labels } = useMessages();\n  const { locale } = useLocale();\n  const countryNames = useCountryNames(locale);\n\n  const formatOS = (value: string): string => {\n    return OS_NAMES[value] || value;\n  };\n\n  const formatBrowser = (value: string): string => {\n    return BROWSERS[value] || value;\n  };\n\n  const formatDevice = (value: string): string => {\n    return formatMessage(labels[value] || labels.unknown);\n  };\n\n  const formatCountry = (value: string): string => {\n    return countryNames[value] || value;\n  };\n\n  const formatRegion = (value: string): string => {\n    const [country] = value.split('-');\n    return regions[value] ? `${regions[value]}, ${countryNames[country]}` : value;\n  };\n\n  const formatCity = (value: string, country?: string): string => {\n    return countryNames[country] ? `${value}, ${countryNames[country]}` : value;\n  };\n\n  const formatValue = (value: string, type: string, data?: { [key: string]: any }): string => {\n    switch (type) {\n      case 'os':\n        return formatOS(value);\n      case 'browser':\n        return formatBrowser(value);\n      case 'device':\n        return formatDevice(value);\n      case 'country':\n        return formatCountry(value);\n      case 'region':\n        return formatRegion(value);\n      case 'city':\n        return formatCity(value, data?.country);\n      default:\n        return value;\n    }\n  };\n\n  return { formatOS, formatBrowser, formatDevice, formatCountry, formatRegion, formatValue };\n}\n\nexport default useFormat;\n\n// Path: src/components/hooks/index.ts\nexport * from './queries/useApi';\nexport * from './queries/useConfig';\nexport * from './queries/useFilterQuery';\nexport * from './queries/useLogin';\nexport * from './queries/useRealtime';\nexport * from './queries/useReport';\nexport * from './queries/useReports';\nexport * from './queries/useShareToken';\nexport * from './queries/useTeam';\nexport * from './queries/useTeams';\nexport * from './queries/useTeamWebsites';\nexport * from './queries/useTeamMembers';\nexport * from './queries/useUser';\nexport * from './queries/useUsers';\nexport * from './queries/useWebsite';\nexport * from './queries/useWebsites';\nexport * from './queries/useWebsiteEvents';\nexport * from './queries/useWebsiteMetrics';\nexport * from './queries/useWebsiteValues';\nexport * from './useCountryNames';\nexport * from './useDateRange';\nexport * from './useDocumentClick';\nexport * from './useEscapeKey';\nexport * from './useFields';\nexport * from './useFilters';\nexport * from './useForceUpdate';\nexport * from './useFormat';\nexport * from './useLanguageNames';\nexport * from './useLocale';\nexport * from './useMessages';\nexport * from './useModified';\nexport * from './useNavigation';\nexport * from './useSticky';\nexport * from './useTeamUrl';\nexport * from './useTheme';\nexport * from './useTimezone';\n\n// Path: src/index.ts\nexport * from 'components/hooks';\n\nexport * from 'app/(main)/settings/teams/[teamId]/members/TeamMemberEditButton';\nexport * from 'app/(main)/settings/teams/[teamId]/members/TeamMemberEditForm';\nexport * from 'app/(main)/settings/teams/[teamId]/members/TeamMemberRemoveButton';\nexport * from 'app/(main)/settings/teams/[teamId]/members/TeamMembersDataTable';\nexport * from 'app/(main)/settings/teams/[teamId]/members/TeamMembersTable';\n\nexport * from 'app/(main)/settings/teams/[teamId]/team/TeamDeleteForm';\nexport * from 'app/(main)/settings/teams/[teamId]/team/TeamDetails';\nexport * from 'app/(main)/settings/teams/[teamId]/team/TeamEditForm';\nexport * from 'app/(main)/settings/teams/[teamId]/team/TeamManage';\n\nexport * from 'app/(main)/settings/teams/[teamId]/websites/TeamWebsiteRemoveButton';\nexport * from 'app/(main)/settings/teams/[teamId]/websites/TeamWebsitesDataTable';\nexport * from 'app/(main)/settings/teams/[teamId]/websites/TeamWebsitesTable';\n\nexport * from 'app/(main)/settings/teams/TeamAddForm';\nexport * from 'app/(main)/settings/teams/TeamJoinForm';\nexport * from 'app/(main)/settings/teams/TeamLeaveButton';\nexport * from 'app/(main)/settings/teams/TeamLeaveForm';\nexport * from 'app/(main)/settings/teams/TeamsAddButton';\nexport * from 'app/(main)/settings/teams/TeamsDataTable';\nexport * from 'app/(main)/settings/teams/TeamsHeader';\nexport * from 'app/(main)/settings/teams/TeamsJoinButton';\nexport * from 'app/(main)/settings/teams/TeamsTable';\nexport * from 'app/(main)/settings/teams/WebsiteTags';\n\nexport * from 'app/(main)/settings/websites/[websiteId]/ShareUrl';\nexport * from 'app/(main)/settings/websites/[websiteId]/TrackingCode';\nexport * from 'app/(main)/settings/websites/[websiteId]/WebsiteData';\nexport * from 'app/(main)/settings/websites/[websiteId]/WebsiteDeleteForm';\nexport * from 'app/(main)/settings/websites/[websiteId]/WebsiteEditForm';\nexport * from 'app/(main)/settings/websites/[websiteId]/WebsiteResetForm';\nexport * from 'app/(main)/settings/websites/[websiteId]/WebsiteSettings';\n\nexport * from 'app/(main)/settings/websites/WebsiteAddButton';\nexport * from 'app/(main)/settings/websites/WebsiteAddForm';\nexport * from 'app/(main)/settings/websites/WebsitesDataTable';\nexport * from 'app/(main)/settings/websites/WebsitesHeader';\nexport * from 'app/(main)/settings/websites/WebsitesTable';\n\nexport * from 'app/(main)/teams/[teamId]/TeamProvider';\nexport * from 'app/(main)/websites/[websiteId]/WebsiteProvider';\n\nexport * from 'components/common/ConfirmationForm';\nexport * from 'components/common/DataTable';\nexport * from 'components/common/Empty';\nexport * from 'components/common/ErrorBoundary';\nexport * from 'components/common/ErrorMessage';\nexport * from 'components/common/Favicon';\nexport * from 'components/common/FilterButtons';\nexport * from 'components/common/FilterLink';\nexport * from 'components/common/HamburgerButton';\nexport * from 'components/common/HoverTooltip';\nexport * from 'components/common/LinkButton';\nexport * from 'components/common/MobileMenu';\nexport * from 'components/common/Pager';\nexport * from 'components/common/TypeConfirmationForm';\n\nexport * from 'components/input/TeamsButton';\nexport * from 'components/input/ThemeButton';\n\nexport { ROLES } from 'lib/constants';\n\n// Path: src/components/declarations.d.ts\ndeclare module '*.css';\ndeclare module '*.svg';\ndeclare module '*.json';\ndeclare module 'uuid';\n\n// Path: src/components/icons.ts\nimport { Icons } from 'react-basics';\nimport AddUser from 'assets/add-user.svg';\nimport Bars from 'assets/bars.svg';\nimport BarChart from 'assets/bar-chart.svg';\nimport Bolt from 'assets/bolt.svg';\nimport Calendar from 'assets/calendar.svg';\nimport Change from 'assets/change.svg';\nimport Clock from 'assets/clock.svg';\nimport Dashboard from 'assets/dashboard.svg';\nimport Eye from 'assets/eye.svg';\nimport Gear from 'assets/gear.svg';\nimport Globe from 'assets/globe.svg';\nimport Lock from 'assets/lock.svg';\nimport Logo from 'assets/logo.svg';\nimport Magnet from 'assets/magnet.svg';\nimport Moon from 'assets/moon.svg';\nimport Nodes from 'assets/nodes.svg';\nimport Overview from 'assets/overview.svg';\nimport Profile from 'assets/profile.svg';\nimport Reports from 'assets/reports.svg';\nimport Sun from 'assets/sun.svg';\nimport User from 'assets/user.svg';\nimport Users from 'assets/users.svg';\nimport Visitor from 'assets/visitor.svg';\n\nconst icons = {\n  ...Icons,\n  AddUser,\n  Bars,\n  BarChart,\n  Bolt,\n  Calendar,\n  Change,\n  Clock,\n  Dashboard,\n  Eye,\n  Gear,\n  Globe,\n  Lock,\n  Logo,\n  Magnet,\n  Moon,\n  Nodes,\n  Overview,\n  Profile,\n  Reports,\n  Sun,\n  User,\n  Users,\n  Visitor,\n};\n\nexport default icons;\n\n// Path: src/components/hooks/queries/useWebsiteStats.ts\nimport { useApi, useDateRange, useNavigation } from 'components/hooks';\n\nexport function useWebsiteStats(websiteId: string, options?: { [key: string]: string }) {\n  const { get, useQuery } = useApi();\n  const [dateRange] = useDateRange(websiteId);\n  const { startDate, endDate } = dateRange;\n  const {\n    query: { url, referrer, title, os, browser, device, country, region, city },\n  } = useNavigation();\n\n  const params = {\n    startAt: +startDate,\n    endAt: +endDate,\n    url,\n    referrer,\n    title,\n    os,\n    browser,\n    device,\n    country,\n    region,\n    city,\n  };\n\n  return useQuery({\n    queryKey: ['websites:stats', { websiteId, ...params }],\n    queryFn: () => get(`/websites/${websiteId}/stats`, params),\n    ...options,\n  });\n}\n\nexport default useWebsiteStats;\n\n// Path: src/components/hooks/queries/useWebsitePageviews.ts\nimport { zonedTimeToUtc } from 'date-fns-tz';\nimport { useApi, useDateRange, useNavigation, useTimezone } from 'components/hooks';\n\nexport function useWebsitePageviews(websiteId: string, options?: { [key: string]: string }) {\n  const { get, useQuery } = useApi();\n  const [dateRange] = useDateRange(websiteId);\n  const { startDate, endDate, unit } = dateRange;\n  const { timezone } = useTimezone();\n  const {\n    query: { url, referrer, os, browser, device, country, region, city, title },\n  } = useNavigation();\n\n  const params = {\n    startAt: +zonedTimeToUtc(startDate, timezone),\n    endAt: +zonedTimeToUtc(endDate, timezone),\n    unit,\n    timezone,\n    url,\n    referrer,\n    os,\n    browser,\n    device,\n    country,\n    region,\n    city,\n    title,\n  };\n\n  return useQuery({\n    queryKey: ['websites:pageviews', { websiteId, ...params }],\n    queryFn: () => get(`/websites/${websiteId}/pageviews`, params),\n    ...options,\n  });\n}\n\nexport default useWebsitePageviews;\n\n// Path: src/tracker/index.js\n(window => {\n  const {\n    screen: { width, height },\n    navigator: { language },\n    location,\n    localStorage,\n    document,\n    history,\n  } = window;\n  const { hostname, pathname, search } = location;\n  const { currentScript, referrer } = document;\n\n  if (!currentScript) return;\n\n  const _data = 'data-';\n  const _false = 'false';\n  const _true = 'true';\n  const attr = currentScript.getAttribute.bind(currentScript);\n  const website = attr(_data + 'website-id');\n  const hostUrl = attr(_data + 'host-url');\n  const tag = attr(_data + 'tag');\n  const autoTrack = attr(_data + 'auto-track') !== _false;\n  const excludeSearch = attr(_data + 'exclude-search') === _true;\n  const domain = attr(_data + 'domains') || '';\n  const domains = domain.split(',').map(n => n.trim());\n  const host =\n    hostUrl || '__COLLECT_API_HOST__' || currentScript.src.split('/').slice(0, -1).join('/');\n  const endpoint = `${host.replace(/\\/$/, '')}__COLLECT_API_ENDPOINT__`;\n  const screen = `${width}x${height}`;\n  const eventRegex = /data-umami-event-([\\w-_]+)/;\n  const eventNameAttribute = _data + 'umami-event';\n  const delayDuration = 300;\n\n  /* Helper functions */\n\n  const encode = str => {\n    if (!str) {\n      return undefined;\n    }\n\n    try {\n      const result = decodeURI(str);\n\n      if (result !== str) {\n        return result;\n      }\n    } catch {\n      return str;\n    }\n\n    return encodeURI(str);\n  };\n\n  const parseURL = url => {\n    return excludeSearch ? url.split('?')[0] : url;\n  };\n\n  const getPayload = () => ({\n    website,\n    hostname,\n    screen,\n    language,\n    title: encode(title),\n    url: encode(currentUrl),\n    referrer: encode(currentRef),\n    tag: tag ? tag : undefined,\n  });\n\n  /* Event handlers */\n\n  const handlePush = (state, title, url) => {\n    if (!url) return;\n\n    currentRef = currentUrl;\n    currentUrl = parseURL(url.toString());\n\n    if (currentUrl !== currentRef) {\n      setTimeout(track, delayDuration);\n    }\n  };\n\n  const handlePathChanges = () => {\n    const hook = (_this, method, callback) => {\n      const orig = _this[method];\n\n      return (...args) => {\n        callback.apply(null, args);\n\n        return orig.apply(_this, args);\n      };\n    };\n\n    history.pushState = hook(history, 'pushState', handlePush);\n    history.replaceState = hook(history, 'replaceState', handlePush);\n  };\n\n  const handleTitleChanges = () => {\n    const observer = new MutationObserver(([entry]) => {\n      title = entry && entry.target ? entry.target.text : undefined;\n    });\n\n    const node = document.querySelector('head > title');\n\n    if (node) {\n      observer.observe(node, {\n        subtree: true,\n        characterData: true,\n        childList: true,\n      });\n    }\n  };\n\n  const handleClicks = () => {\n    document.addEventListener(\n      'click',\n      async e => {\n        const isSpecialTag = tagName => ['BUTTON', 'A'].includes(tagName);\n\n        const trackElement = async el => {\n          const attr = el.getAttribute.bind(el);\n          const eventName = attr(eventNameAttribute);\n\n          if (eventName) {\n            const eventData = {};\n\n            el.getAttributeNames().forEach(name => {\n              const match = name.match(eventRegex);\n\n              if (match) {\n                eventData[match[1]] = attr(name);\n              }\n            });\n\n            return track(eventName, eventData);\n          }\n        };\n\n        const findParentTag = (rootElem, maxSearchDepth) => {\n          let currentElement = rootElem;\n          for (let i = 0; i < maxSearchDepth; i++) {\n            if (isSpecialTag(currentElement.tagName)) {\n              return currentElement;\n            }\n            currentElement = currentElement.parentElement;\n            if (!currentElement) {\n              return null;\n            }\n          }\n        };\n\n        const el = e.target;\n        const parentElement = isSpecialTag(el.tagName) ? el : findParentTag(el, 10);\n\n        if (parentElement) {\n          const { href, target } = parentElement;\n          const eventName = parentElement.getAttribute(eventNameAttribute);\n\n          if (eventName) {\n            if (parentElement.tagName === 'A') {\n              const external =\n                target === '_blank' ||\n                e.ctrlKey ||\n                e.shiftKey ||\n                e.metaKey ||\n                (e.button && e.button === 1);\n\n              if (eventName && href) {\n                if (!external) {\n                  e.preventDefault();\n                }\n                return trackElement(parentElement).then(() => {\n                  if (!external) location.href = href;\n                });\n              }\n            } else if (parentElement.tagName === 'BUTTON') {\n              return trackElement(parentElement);\n            }\n          }\n        } else {\n          return trackElement(el);\n        }\n      },\n      true,\n    );\n  };\n\n  /* Tracking functions */\n\n  const trackingDisabled = () =>\n    (localStorage && localStorage.getItem('umami.disabled')) ||\n    (domain && !domains.includes(hostname));\n\n  const send = async (payload, type = 'event') => {\n    if (trackingDisabled()) return;\n    const headers = {\n      'Content-Type': 'application/json',\n    };\n    if (typeof cache !== 'undefined') {\n      headers['x-umami-cache'] = cache;\n    }\n    try {\n      const res = await fetch(endpoint, {\n        method: 'POST',\n        body: JSON.stringify({ type, payload }),\n        headers,\n      });\n      const text = await res.text();\n\n      return (cache = text);\n    } catch {\n      /* empty */\n    }\n  };\n\n  const track = (obj, data) => {\n    if (typeof obj === 'string') {\n      return send({\n        ...getPayload(),\n        name: obj,\n        data: typeof data === 'object' ? data : undefined,\n      });\n    } else if (typeof obj === 'object') {\n      return send(obj);\n    } else if (typeof obj === 'function') {\n      return send(obj(getPayload()));\n    }\n    return send(getPayload());\n  };\n\n  const identify = data => send({ ...getPayload(), data }, 'identify');\n\n  /* Start */\n\n  if (!window.umami) {\n    window.umami = {\n      track,\n      identify,\n    };\n  }\n\n  let currentUrl = `${pathname}${search}`;\n  let currentRef = referrer !== hostname ? referrer : '';\n  let title = document.title;\n  let cache;\n  let initialized;\n\n  if (autoTrack && !trackingDisabled()) {\n    handlePathChanges();\n    handleTitleChanges();\n    handleClicks();\n\n    const init = () => {\n      if (document.readyState === 'complete' && !initialized) {\n        track();\n        initialized = true;\n      }\n    };\n\n    document.addEventListener('readystatechange', init, true);\n\n    init();\n  }\n})(window);\n\n// Path: src/tracker/index.d.ts\nexport type TrackedProperties = {\n  /**\n   * Hostname of server\n   *\n   * @description extracted from `window.location.hostname`\n   * @example 'analytics.umami.is'\n   */\n  hostname: string;\n\n  /**\n   * Browser language\n   *\n   * @description extracted from `window.navigator.language`\n   * @example 'en-US', 'fr-FR'\n   */\n  language: string;\n\n  /**\n   * Page referrer\n   *\n   * @description extracted from `window.navigator.language`\n   * @example 'https://analytics.umami.is/docs/getting-started'\n   */\n  referrer: string;\n\n  /**\n   * Screen dimensions\n   *\n   * @description extracted from `window.screen.width` and `window.screen.height`\n   * @example '1920x1080', '2560x1440'\n   */\n  screen: string;\n\n  /**\n   * Page title\n   *\n   * @description extracted from `document.querySelector('head > title')`\n   * @example 'umami'\n   */\n  title: string;\n\n  /**\n   * Page url\n   *\n   * @description built from `${window.location.pathname}${window.location.search}`\n   * @example 'docs/getting-started'\n   */\n  url: string;\n\n  /**\n   * Website ID (required)\n   *\n   * @example 'b59e9c65-ae32-47f1-8400-119fcf4861c4'\n   */\n  website: string;\n};\n\nexport type WithRequired<T, K extends keyof T> = T & { [P in K]-?: T[P] };\n\n/**\n *\n * Event Data can work with any JSON data. There are a few rules in place to maintain performance.\n * - Numbers have a max precision of 4.\n * - Strings have a max length of 500.\n * - Arrays are converted to a String, with the same max length of 500.\n * - Objects have a max of 50 properties. Arrays are considered 1 property.\n */\nexport interface EventData {\n  [key: string]: number | string | EventData | number[] | string[] | EventData[];\n}\n\nexport type EventProperties = {\n  /**\n   * NOTE: event names will be truncated past 50 characters\n   */\n  name: string;\n  data?: EventData;\n} & WithRequired<TrackedProperties, 'website'>;\nexport type PageViewProperties = WithRequired<TrackedProperties, 'website'>;\nexport type CustomEventFunction = (\n  props: PageViewProperties,\n) => EventProperties | PageViewProperties;\n\nexport type UmamiTracker = {\n  track: {\n    /**\n     * Track a page view\n     *\n     * @example ```\n     * umami.track();\n     * ```\n     */\n    (): Promise<string>;\n\n    /**\n     * Track an event with a given name\n     *\n     * NOTE: event names will be truncated past 50 characters\n     *\n     * @example ```\n     * umami.track('signup-button');\n     * ```\n     */\n    (eventName: string): Promise<string>;\n\n    /**\n     * Tracks an event with dynamic data.\n     *\n     * NOTE: event names will be truncated past 50 characters\n     *\n     * When tracking events, the default properties are included in the payload. This is equivalent to running:\n     *\n     * ```js\n     * umami.track(props => ({\n     *   ...props,\n     *   name: 'signup-button',\n     *   data: {\n     *     name: 'newsletter',\n     *     id: 123\n     *   }\n     * }));\n     * ```\n     *\n     * @example ```\n     * umami.track('signup-button', { name: 'newsletter', id: 123 });\n     * ```\n     */\n    (eventName: string, obj: EventData): Promise<string>;\n\n    /**\n     * Tracks a page view with custom properties\n     *\n     * @example ```\n     * umami.track({ website: 'e676c9b4-11e4-4ef1-a4d7-87001773e9f2', url: '/home', title: 'Home page' });\n     * ```\n     */\n    (properties: PageViewProperties): Promise<string>;\n\n    /**\n     * Tracks an event with fully customizable dynamic data\n     * If you don't specify any `name` and/or `data`, it will be treated as a page view\n     *\n     * @example ```\n     * umami.track((props) => ({ ...props, url: path }));\n     * ```\n     */\n    (eventFunction: CustomEventFunction): Promise<string>;\n  };\n};\n\ninterface Window {\n  umami: UmamiTracker;\n}\n\n// Path: src/lib/crypto.ts\nimport { startOfHour, startOfMonth } from 'date-fns';\nimport { hash } from 'next-basics';\nimport { v4, v5, validate } from 'uuid';\n\nexport function secret() {\n  return hash(process.env.APP_SECRET || process.env.DATABASE_URL);\n}\n\nexport function salt() {\n  const ROTATING_SALT = hash(startOfMonth(new Date()).toUTCString());\n\n  return hash(secret(), ROTATING_SALT);\n}\n\nexport function visitSalt() {\n  const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString());\n\n  return hash(secret(), ROTATING_SALT);\n}\n\nexport function uuid(...args: any) {\n  if (!args.length) return v4();\n\n  return v5(hash(...args, salt()), v5.DNS);\n}\n\nexport function isUuid(value: string) {\n  return validate(value);\n}\n\n// Path: src/lib/format.ts\nexport function parseTime(val: number) {\n  const days = ~~(val / 86400);\n  const hours = ~~(val / 3600) - days * 24;\n  const minutes = ~~(val / 60) - days * 1440 - hours * 60;\n  const seconds = ~~val - days * 86400 - hours * 3600 - minutes * 60;\n  const ms = (val - ~~val) * 1000;\n\n  return {\n    days,\n    hours,\n    minutes,\n    seconds,\n    ms,\n  };\n}\n\nexport function formatTime(val: number) {\n  const { hours, minutes, seconds } = parseTime(val);\n  const h = hours > 0 ? `${hours}:` : '';\n  const m = hours > 0 ? minutes.toString().padStart(2, '0') : minutes;\n  const s = seconds.toString().padStart(2, '0');\n\n  return `${h}${m}:${s}`;\n}\n\nexport function formatShortTime(val: number, formats = ['m', 's'], space = '') {\n  const { days, hours, minutes, seconds, ms } = parseTime(val);\n  let t = '';\n\n  if (days > 0 && formats.indexOf('d') !== -1) t += `${days}d${space}`;\n  if (hours > 0 && formats.indexOf('h') !== -1) t += `${hours}h${space}`;\n  if (minutes > 0 && formats.indexOf('m') !== -1) t += `${minutes}m${space}`;\n  if (seconds > 0 && formats.indexOf('s') !== -1) t += `${seconds}s${space}`;\n  if (ms > 0 && formats.indexOf('ms') !== -1) t += `${ms}ms`;\n\n  if (!t) {\n    return `0${formats[formats.length - 1]}`;\n  }\n\n  return t;\n}\n\nexport function formatNumber(n: string | number) {\n  return Number(n).toFixed(0);\n}\n\nexport function formatLongNumber(value: number) {\n  const n = Number(value);\n\n  if (n >= 1000000) {\n    return `${(n / 1000000).toFixed(1)}m`;\n  }\n  if (n >= 100000) {\n    return `${(n / 1000).toFixed(0)}k`;\n  }\n  if (n >= 10000) {\n    return `${(n / 1000).toFixed(1)}k`;\n  }\n  if (n >= 1000) {\n    return `${(n / 1000).toFixed(2)}k`;\n  }\n\n  return formatNumber(n);\n}\n\nexport function stringToColor(str: string) {\n  if (!str) {\n    return '#ffffff';\n  }\n  let hash = 0;\n  for (let i = 0; i < str.length; i++) {\n    hash = str.charCodeAt(i) + ((hash << 5) - hash);\n  }\n  let color = '#';\n  for (let i = 0; i < 3; i++) {\n    const value = (hash >> (i * 8)) & 0xff;\n    color += ('00' + value.toString(16)).slice(-2);\n  }\n  return color;\n}\n\n// Path: src/lib/yup.ts\nimport moment from 'moment-timezone';\nimport * as yup from 'yup';\nimport { UNIT_TYPES } from './constants';\n\nexport const TimezoneTest = yup\n  .string()\n  .default('UTC')\n  .test(\n    'timezone',\n    () => `Invalid timezone`,\n    value => moment.tz.zone(value) !== null,\n  );\n\nexport const UnitTypeTest = yup.string().test(\n  'unit',\n  () => `Invalid unit`,\n  value => UNIT_TYPES.includes(value),\n);\n\n// Path: src/lib/load.ts\nimport cache from 'lib/cache';\nimport { getSession, getUser, getWebsite } from 'queries';\nimport { User, Website, Session } from '@prisma/client';\n\nexport async function loadWebsite(websiteId: string): Promise<Website> {\n  let website;\n\n  if (cache.enabled) {\n    website = await cache.fetchWebsite(websiteId);\n  } else {\n    website = await getWebsite(websiteId);\n  }\n\n  if (!website || website.deletedAt) {\n    return null;\n  }\n\n  return website;\n}\n\nexport async function loadSession(sessionId: string): Promise<Session> {\n  let session;\n\n  if (cache.enabled) {\n    session = await cache.fetchSession(sessionId);\n  } else {\n    session = await getSession(sessionId);\n  }\n\n  if (!session) {\n    return null;\n  }\n\n  return session;\n}\n\nexport async function loadUser(userId: string): Promise<User> {\n  let user;\n\n  if (cache.enabled) {\n    user = await cache.fetchUser(userId);\n  } else {\n    user = await getUser(userId);\n  }\n\n  if (!user || user.deletedAt) {\n    return null;\n  }\n\n  return user;\n}\n\n// Path: src/lib/params.ts\nimport { FILTER_COLUMNS, OPERATOR_PREFIXES, OPERATORS } from 'lib/constants';\nimport { QueryFilters, QueryOptions } from 'lib/types';\n\nexport function parseParameterValue(param: any) {\n  if (typeof param === 'string') {\n    const [, prefix, value] = param.match(/^(!~|!|~)?(.*)$/);\n\n    const operator =\n      Object.keys(OPERATOR_PREFIXES).find(key => OPERATOR_PREFIXES[key] === prefix) ||\n      OPERATORS.equals;\n\n    return { operator, value };\n  }\n  return { operator: OPERATORS.equals, value: param };\n}\n\nexport function isEqualsOperator(operator: any) {\n  return [OPERATORS.equals, OPERATORS.notEquals].includes(operator);\n}\n\nexport function isSearchOperator(operator: any) {\n  return [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator);\n}\n\nexport function filtersToArray(filters: QueryFilters = {}, options: QueryOptions = {}) {\n  return Object.keys(filters).reduce((arr, key) => {\n    const filter = filters[key];\n\n    if (filter === undefined || filter === null) {\n      return arr;\n    }\n\n    if (filter?.name && filter?.value !== undefined) {\n      return arr.concat({ ...filter, column: options?.columns?.[key] ?? FILTER_COLUMNS[key] });\n    }\n\n    const { operator, value } = parseParameterValue(filter);\n\n    return arr.concat({\n      name: key,\n      column: options?.columns?.[key] ?? FILTER_COLUMNS[key],\n      operator,\n      value,\n    });\n  }, []);\n}\n\n// Path: src/lib/db.ts\nexport const PRISMA = 'prisma';\nexport const POSTGRESQL = 'postgresql';\nexport const MYSQL = 'mysql';\nexport const CLICKHOUSE = 'clickhouse';\nexport const KAFKA = 'kafka';\nexport const KAFKA_PRODUCER = 'kafka-producer';\n\n// Fixes issue with converting bigint values\nBigInt.prototype['toJSON'] = function () {\n  return Number(this);\n};\n\nexport function getDatabaseType(url = process.env.DATABASE_URL) {\n  const type = url && url.split(':')[0];\n\n  if (type === 'postgres') {\n    return POSTGRESQL;\n  }\n\n  return type;\n}\n\nexport async function runQuery(queries: any) {\n  if (process.env.CLICKHOUSE_URL) {\n    if (queries[KAFKA]) {\n      return queries[KAFKA]();\n    }\n\n    return queries[CLICKHOUSE]();\n  }\n\n  const db = getDatabaseType();\n\n  if (db === POSTGRESQL || db === MYSQL) {\n    return queries[PRISMA]();\n  }\n}\n\nexport function notImplemented() {\n  throw new Error('Not implemented.');\n}\n\n// Path: src/lib/clickhouse.ts\nimport { ClickHouseClient, createClient } from '@clickhouse/client';\nimport dateFormat from 'dateformat';\nimport debug from 'debug';\nimport { CLICKHOUSE } from 'lib/db';\nimport { QueryFilters, QueryOptions } from './types';\nimport { OPERATORS } from './constants';\nimport { loadWebsite } from './load';\nimport { maxDate } from './date';\nimport { filtersToArray } from './params';\n\nexport const CLICKHOUSE_DATE_FORMATS = {\n  minute: '%Y-%m-%d %H:%M:00',\n  hour: '%Y-%m-%d %H:00:00',\n  day: '%Y-%m-%d',\n  month: '%Y-%m-01',\n  year: '%Y-01-01',\n};\n\nconst log = debug('umami:clickhouse');\n\nlet clickhouse: ClickHouseClient;\nconst enabled = Boolean(process.env.CLICKHOUSE_URL);\n\nfunction getClient() {\n  const {\n    hostname,\n    port,\n    pathname,\n    protocol,\n    username = 'default',\n    password,\n  } = new URL(process.env.CLICKHOUSE_URL);\n\n  const client = createClient({\n    host: `${protocol}//${hostname}:${port}`,\n    database: pathname.replace('/', ''),\n    username: username,\n    password,\n  });\n\n  if (process.env.NODE_ENV !== 'production') {\n    global[CLICKHOUSE] = client;\n  }\n\n  log('Clickhouse initialized');\n\n  return client;\n}\n\nfunction getDateStringQuery(data: any, unit: string | number) {\n  return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;\n}\n\nfunction getDateQuery(field: string, unit: string, timezone?: string) {\n  if (timezone) {\n    return `date_trunc('${unit}', ${field}, '${timezone}')`;\n  }\n  return `date_trunc('${unit}', ${field})`;\n}\n\nfunction getDateFormat(date: Date) {\n  return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;\n}\n\nfunction mapFilter(column: string, operator: string, name: string, type: string = 'String') {\n  const value = `{${name}:${type}}`;\n\n  switch (operator) {\n    case OPERATORS.equals:\n      return `${column} = ${value}`;\n    case OPERATORS.notEquals:\n      return `${column} != ${value}`;\n    case OPERATORS.contains:\n      return `positionCaseInsensitive(${column}, ${value}) > 0`;\n    case OPERATORS.doesNotContain:\n      return `positionCaseInsensitive(${column}, ${value}) = 0`;\n    default:\n      return '';\n  }\n}\n\nfunction getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {\n  const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {\n    if (column) {\n      arr.push(`and ${mapFilter(column, operator, name)}`);\n\n      if (name === 'referrer') {\n        arr.push('and referrer_domain != {websiteDomain:String}');\n      }\n    }\n\n    return arr;\n  }, []);\n\n  return query.join('\\n');\n}\n\nfunction getFilterParams(filters: QueryFilters = {}) {\n  return filtersToArray(filters).reduce((obj, { name, value }) => {\n    if (name && value !== undefined) {\n      obj[name] = value;\n    }\n\n    return obj;\n  }, {});\n}\n\nasync function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) {\n  const website = await loadWebsite(websiteId);\n\n  return {\n    filterQuery: getFilterQuery(filters, options),\n    params: {\n      ...getFilterParams(filters),\n      websiteId,\n      startDate: maxDate(filters.startDate, new Date(website?.resetAt)),\n      websiteDomain: website.domain,\n    },\n  };\n}\n\nasync function rawQuery(query: string, params: Record<string, unknown> = {}): Promise<unknown> {\n  if (process.env.LOG_QUERY) {\n    log('QUERY:\\n', query);\n    log('PARAMETERS:\\n', params);\n  }\n\n  await connect();\n\n  const resultSet = await clickhouse.query({\n    query: query,\n    query_params: params,\n    format: 'JSONEachRow',\n  });\n\n  return resultSet.json();\n}\n\nasync function findUnique(data: any[]) {\n  if (data.length > 1) {\n    throw `${data.length} records found when expecting 1.`;\n  }\n\n  return findFirst(data);\n}\n\nasync function findFirst(data: any[]) {\n  return data[0] ?? null;\n}\n\nasync function connect() {\n  if (enabled && !clickhouse) {\n    clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient());\n  }\n\n  return clickhouse;\n}\n\nexport default {\n  enabled,\n  client: clickhouse,\n  log,\n  connect,\n  getDateStringQuery,\n  getDateQuery,\n  getDateFormat,\n  getFilterQuery,\n  parseFilters,\n  findUnique,\n  findFirst,\n  rawQuery,\n};\n\n// Path: src/lib/prisma.ts\nimport { Prisma } from '@prisma/client';\nimport prisma from '@umami/prisma-client';\nimport moment from 'moment-timezone';\nimport { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';\nimport { SESSION_COLUMNS, OPERATORS, DEFAULT_PAGE_SIZE } from './constants';\nimport { loadWebsite } from './load';\nimport { maxDate } from './date';\nimport { QueryFilters, QueryOptions, SearchFilter } from './types';\nimport { filtersToArray } from './params';\n\nconst MYSQL_DATE_FORMATS = {\n  minute: '%Y-%m-%d %H:%i:00',\n  hour: '%Y-%m-%d %H:00:00',\n  day: '%Y-%m-%d',\n  month: '%Y-%m-01',\n  year: '%Y-01-01',\n};\n\nconst POSTGRESQL_DATE_FORMATS = {\n  minute: 'YYYY-MM-DD HH24:MI:00',\n  hour: 'YYYY-MM-DD HH24:00:00',\n  day: 'YYYY-MM-DD',\n  month: 'YYYY-MM-01',\n  year: 'YYYY-01-01',\n};\n\nfunction getAddIntervalQuery(field: string, interval: string): string {\n  const db = getDatabaseType();\n\n  if (db === POSTGRESQL) {\n    return `${field} + interval '${interval}'`;\n  }\n\n  if (db === MYSQL) {\n    return `DATE_ADD(${field}, interval ${interval})`;\n  }\n}\n\nfunction getDayDiffQuery(field1: string, field2: string): string {\n  const db = getDatabaseType();\n\n  if (db === POSTGRESQL) {\n    return `${field1}::date - ${field2}::date`;\n  }\n\n  if (db === MYSQL) {\n    return `DATEDIFF(${field1}, ${field2})`;\n  }\n}\n\nfunction getCastColumnQuery(field: string, type: string): string {\n  const db = getDatabaseType();\n\n  if (db === POSTGRESQL) {\n    return `${field}::${type}`;\n  }\n\n  if (db === MYSQL) {\n    return `${field}`;\n  }\n}\n\nfunction getDateQuery(field: string, unit: string, timezone?: string): string {\n  const db = getDatabaseType();\n\n  if (db === POSTGRESQL) {\n    if (timezone) {\n      return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;\n    }\n    return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;\n  }\n\n  if (db === MYSQL) {\n    if (timezone) {\n      const tz = moment.tz(timezone).format('Z');\n\n      return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;\n    }\n\n    return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;\n  }\n}\n\n\nfunction getTimestampDiffQuery(field1: string, field2: string): string {\n  const db = getDatabaseType();\n\n  if (db === POSTGRESQL) {\n    return `floor(extract(epoch from (${field2} - ${field1})))`;\n  }\n\n  if (db === MYSQL) {\n    return `timestampdiff(second, ${field1}, ${field2})`;\n  }\n}\n\nfunction getSearchQuery(column: string): string {\n  const db = getDatabaseType();\n  const like = db === POSTGRESQL ? 'ilike' : 'like';\n\n  return `and ${column} ${like} {{search}}`;\n}\n\nfunction mapFilter(column: string, operator: string, name: string, type: string = '') {\n  const db = getDatabaseType();\n  const like = db === POSTGRESQL ? 'ilike' : 'like';\n  const value = `{{${name}${type ? `::${type}` : ''}}}`;\n\n  switch (operator) {\n    case OPERATORS.equals:\n      return `${column} = ${value}`;\n    case OPERATORS.notEquals:\n      return `${column} != ${value}`;\n    case OPERATORS.contains:\n      return `${column} ${like} ${value}`;\n    case OPERATORS.doesNotContain:\n      return `${column} not ${like} ${value}`;\n    default:\n      return '';\n  }\n}\n\nfunction getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string {\n  const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {\n    if (column) {\n      arr.push(`and ${mapFilter(column, operator, name)}`);\n\n      if (name === 'referrer') {\n        arr.push(\n          'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)',\n        );\n      }\n    }\n\n    return arr;\n  }, []);\n\n  return query.join('\\n');\n}\n\nfunction getFilterParams(filters: QueryFilters = {}) {\n  return filtersToArray(filters).reduce((obj, { name, operator, value }) => {\n    obj[name] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator)\n      ? `%${value}%`\n      : value;\n\n    return obj;\n  }, {});\n}\n\nasync function parseFilters(\n  websiteId: string,\n  filters: QueryFilters = {},\n  options: QueryOptions = {},\n) {\n  const website = await loadWebsite(websiteId);\n  const joinSession = Object.keys(filters).find(key => SESSION_COLUMNS.includes(key));\n\n  return {\n    joinSession:\n      options?.joinSession || joinSession\n        ? `inner join session on website_event.session_id = session.session_id`\n        : '',\n    filterQuery: getFilterQuery(filters, options),\n    params: {\n      ...getFilterParams(filters),\n      websiteId,\n      startDate: maxDate(filters.startDate, website?.resetAt),\n      websiteDomain: website.domain,\n    },\n  };\n}\n\nasync function rawQuery(sql: string, data: object): Promise<any> {\n  const db = getDatabaseType();\n  const params = [];\n\n  if (db !== POSTGRESQL && db !== MYSQL) {\n...\n", "needle_token_start": 15226, "needle_token_end": 15326, "code_context_ntokens": 16073, "output": ["```typescript\nfunction getTimestampDiffQuery(field1: string, field2: string): string {\n  const db = getDatabaseType();\n\n  if (db === POSTGRESQL) {\n    return `floor(extract(epoch from (${field2} - ${field1})))`;\n  }\n\n  if (db === MYSQL) {\n    return `timestampdiff(second, ${field1}, ${field2})`;\n  }\n}\n```"]}
{"repo": "mrdoob/three.js", "name": "generateTorso", "language": "typescript", "path": "src/geometries/CylinderGeometry.js", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create the main body of a cylindrical shape, calculating the vertices, normals, and texture coordinates that define the surface of the cylinder.\n2. **Input**: The function does not take any direct parameters; instead, it utilizes properties from its parent object such as the top and bottom radii, height, radial and height segments, and angular parameters.\n3. **Output**: There is no direct return value. Instead, it modifies arrays within its scope to store the vertices, normals, UV coordinates, and indices that describe the geometry of the cylinder's surface.\n4. **Procedure**: \n   - Initializes arrays for storing row indices and calculates the slope for normal vector calculation.\n   - Iterates over the height segments to compute vertices along the height of the cylinder.\n   - For each height segment, iterates over the radial segments to calculate and store vertices, normals, and UVs for each point on the circle using trigonometric functions.\n   - Constructs faces (triangles) using indices from consecutive segments.\n   - Updates the geometry's groups to support multiple materials, ensuring proper rendering and material application.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\t\theight: height,\n\t\t\tradialSegments: radialSegments,\n\t\t\theightSegments: heightSegments,\n\t\t\topenEnded: openEnded,\n\t\t\tthetaStart: thetaStart,\n\t\t\tthetaLength: thetaLength\n\t\t};\n\n\t\tconst scope = this;\n\n\t\tradialSegments = Math.floor( radialSegments );\n\t\theightSegments = Math.floor( heightSegments );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\t// helper variables\n\n\t\tlet index = 0;\n\t\tconst indexArray = [];\n\t\tconst halfHeight = height / 2;\n\t\tlet groupStart = 0;\n\n\t\t// generate geometry\n\n\t\tgenerateTorso();\n\n\t\tif ( openEnded === false ) {\n\n\t\t\tif ( radiusTop > 0 ) generateCap( true );\n\t\t\tif ( radiusBottom > 0 ) generateCap( false );\n\n\t\t}\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t\t\nfunction generateTorso() {\n\n\t\t\tconst normal = new Vector3();\n\t\t\tconst vertex = new Vector3();\n\n\t\t\tlet groupCount = 0;\n\n\t\t\t// this will be used to calculate the normal\n\t\t\tconst slope = ( radiusBottom - radiusTop ) / height;\n\n\t\t\t// generate vertices, normals and uvs\n\n\t\t\tfor ( let y = 0; y <= heightSegments; y ++ ) {\n\n\t\t\t\tconst indexRow = [];\n\n\t\t\t\tconst v = y / heightSegments;\n\n\t\t\t\t// calculate the radius of the current row\n\n\t\t\t\tconst radius = v * ( radiusBottom - radiusTop ) + radiusTop;\n\n\t\t\t\tfor ( let x = 0; x <= radialSegments; x ++ ) {\n\n\t\t\t\t\tconst u = x / radialSegments;\n\n\t\t\t\t\tconst theta = u * thetaLength + thetaStart;\n\n\t\t\t\t\tconst sinTheta = Math.sin( theta );\n\t\t\t\t\tconst cosTheta = Math.cos( theta );\n\n\t\t\t\t\t// vertex\n\n\t\t\t\t\tvertex.x = radius * sinTheta;\n\t\t\t\t\tvertex.y = - v * height + halfHeight;\n\t\t\t\t\tvertex.z = radius * cosTheta;\n\t\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t\t// normal\n\n\t\t\t\t\tnormal.set( sinTheta, slope, cosTheta ).normalize();\n\t\t\t\t\tnormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t\t// uv\n\n\t\t\t\t\tuvs.push( u, 1 - v );\n\n\t\t\t\t\t// save index of vertex in respective row\n\n\t\t\t\t\tindexRow.push( index ++ );\n\n\t\t\t\t}\n\n\t\t\t\t// now save vertices of the row in our index array\n\n\t\t\t\tindexArray.push( indexRow );\n\n\t\t\t}\n\n\t\t\t// generate indices\n\n\t\t\tfor ( let x = 0; x < radialSegments; x ++ ) {\n\n\t\t\t\tfor ( let y = 0; y < heightSegments; y ++ ) {\n\n\t\t\t\t\t// we use the index array to access the correct indices\n\n\t\t\t\t\tconst a = indexArray[ y ][ x ];\n\t\t\t\t\tconst b = indexArray[ y + 1 ][ x ];\n\t\t\t\t\tconst c = indexArray[ y + 1 ][ x + 1 ];\n\t\t\t\t\tconst d = indexArray[ y ][ x + 1 ];\n\n\t\t\t\t\t// faces\n\n\t\t\t\t\tindices.push( a, b, d );\n\t\t\t\t\tindices.push( b, c, d );\n\n\t\t\t\t\t// update group counter\n\n\t\t\t\t\tgroupCount += 6;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// add a group to the geometry. this will ensure multi material support\n\n\t\t\tscope.addGroup( groupStart, groupCount, 0 );\n\n\t\t\t// calculate new start value for groups\n\n\t\t\tgroupStart += groupCount;\n\n\t\t}\n\n\t\tfunction generateCap( top ) {\n\n\t\t\t// save the index of the first center vertex\n\t\t\tconst centerIndexStart = index;\n\n\t\t\tconst uv = new Vector2();\n\t\t\tconst vertex = new Vector3();\n\n\t\t\tlet groupCount = 0;\n\n\t\t\tconst radius = ( top === true ) ? radiusTop : radiusBottom;\n\t\t\tconst sign = ( top === true ) ? 1 : - 1;\n\n\t\t\t// first we generate the center vertex data of the cap.\n\t\t\t// because the geometry needs one set of uvs per face,\n\t\t\t// we must generate a center vertex per face/segment\n\n\t\t\tfor ( let x = 1; x <= radialSegments; x ++ ) {\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertices.push( 0, halfHeight * sign, 0 );\n\n\t\t\t\t// normal\n\n\t\t\t\tnormals.push( 0, sign, 0 );\n\n\t\t\t\t// uv\n\n\t\t\t\tuvs.push( 0.5, 0.5 );\n\n\t\t\t\t// increase index\n\n\t\t\t\tindex ++;\n\n\t\t\t}\n\n\t\t\t// save the index of the last center vertex\n\t\t\tconst centerIndexEnd = index;\n\n\t\t\t// now we generate the surrounding vertices, normals and uvs\n\n\t\t\tfor ( let x = 0; x <= radialSegments; x ++ ) {\n\n\t\t\t\tconst u = x / radialSegments;\n\t\t\t\tconst theta = u * thetaLength + thetaStart;\n\n\t\t\t\tconst cosTheta = Math.cos( theta );\n\t\t\t\tconst sinTheta = Math.sin( theta );\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertex.x = radius * sinTheta;\n\t\t\t\tvertex.y = halfHeight * sign;\n\t\t\t\tvertex.z = radius * cosTheta;\n\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t// normal\n\n\t\t\t\tnormals.push( 0, sign, 0 );\n\n\t\t\t\t// uv\n\n\t\t\t\tuv.x = ( cosTheta * 0.5 ) + 0.5;\n\t\t\t\tuv.y = ( sinTheta * 0.5 * sign ) + 0.5;\n\t\t\t\tuvs.push( uv.x, uv.y );\n\n\t\t\t\t// increase index\n\n\t\t\t\tindex ++;\n\n\t\t\t}\n\n\t\t\t// generate indices\n\n\t\t\tfor ( let x = 0; x < radialSegments; x ++ ) {\n\n\t\t\t\tconst c = centerIndexStart + x;\n\t\t\t\tconst i = centerIndexEnd + x;\n\n\t\t\t\tif ( top === true ) {\n\n\t\t\t\t\t// face top\n\n\t\t\t\t\tindices.push( i, i + 1, c );\n\n\t\t\t\t} else {\n\n\t\t\t\t\t// face bottom\n\n\t\t\t\t\tindices.push( i + 1, i, c );\n\n\t\t\t\t}\n\n\t\t\t\tgroupCount += 3;\n\n\t\t\t}\n\n\t\t\t// add a group to the geometry. this will ensure multi material support\n\n\t\t\tscope.addGroup( groupStart, groupCount, top === true ? 1 : 2 );\n\n\t\t\t// calculate new start value for groups\n\n\t\t\tgroupStart += groupCount;\n\n\t\t}\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new CylinderGeometry( data.radiusTop, data.radiusBottom, data.height, data.radialSegments, data.heightSegments, data.openEnded, data.thetaStart, data.thetaLength );\n\n\t}\n\n}\n\n\nexport { CylinderGeometry };\n\n// Path: src/geometries/ConeGeometry.js\nimport { CylinderGeometry } from './CylinderGeometry.js';\n\nclass ConeGeometry extends CylinderGeometry {\n\n\tconstructor( radius = 1, height = 1, radialSegments = 32, heightSegments = 1, openEnded = false, thetaStart = 0, thetaLength = Math.PI * 2 ) {\n\n\t\tsuper( 0, radius, height, radialSegments, heightSegments, openEnded, thetaStart, thetaLength );\n\n\t\tthis.type = 'ConeGeometry';\n\n\t\tthis.parameters = {\n\t\t\tradius: radius,\n\t\t\theight: height,\n\t\t\tradialSegments: radialSegments,\n\t\t\theightSegments: heightSegments,\n\t\t\topenEnded: openEnded,\n\t\t\tthetaStart: thetaStart,\n\t\t\tthetaLength: thetaLength\n\t\t};\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new ConeGeometry( data.radius, data.height, data.radialSegments, data.heightSegments, data.openEnded, data.thetaStart, data.thetaLength );\n\n\t}\n\n}\n\nexport { ConeGeometry };\n\n// Path: src/geometries/LatheGeometry.js\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Vector2 } from '../math/Vector2.js';\nimport * as MathUtils from '../math/MathUtils.js';\n\nclass LatheGeometry extends BufferGeometry {\n\n\tconstructor( points = [ new Vector2( 0, - 0.5 ), new Vector2( 0.5, 0 ), new Vector2( 0, 0.5 ) ], segments = 12, phiStart = 0, phiLength = Math.PI * 2 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'LatheGeometry';\n\n\t\tthis.parameters = {\n\t\t\tpoints: points,\n\t\t\tsegments: segments,\n\t\t\tphiStart: phiStart,\n\t\t\tphiLength: phiLength\n\t\t};\n\n\t\tsegments = Math.floor( segments );\n\n\t\t// clamp phiLength so it's in range of [ 0, 2PI ]\n\n\t\tphiLength = MathUtils.clamp( phiLength, 0, Math.PI * 2 );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst uvs = [];\n\t\tconst initNormals = [];\n\t\tconst normals = [];\n\n\t\t// helper variables\n\n\t\tconst inverseSegments = 1.0 / segments;\n\t\tconst vertex = new Vector3();\n\t\tconst uv = new Vector2();\n\t\tconst normal = new Vector3();\n\t\tconst curNormal = new Vector3();\n\t\tconst prevNormal = new Vector3();\n\t\tlet dx = 0;\n\t\tlet dy = 0;\n\n\t\t// pre-compute normals for initial \"meridian\"\n\n\t\tfor ( let j = 0; j <= ( points.length - 1 ); j ++ ) {\n\n\t\t\tswitch ( j ) {\n\n\t\t\t\tcase 0:\t\t\t\t// special handling for 1st vertex on path\n\n\t\t\t\t\tdx = points[ j + 1 ].x - points[ j ].x;\n\t\t\t\t\tdy = points[ j + 1 ].y - points[ j ].y;\n\n\t\t\t\t\tnormal.x = dy * 1.0;\n\t\t\t\t\tnormal.y = - dx;\n\t\t\t\t\tnormal.z = dy * 0.0;\n\n\t\t\t\t\tprevNormal.copy( normal );\n\n\t\t\t\t\tnormal.normalize();\n\n\t\t\t\t\tinitNormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase ( points.length - 1 ):\t// special handling for last Vertex on path\n\n\t\t\t\t\tinitNormals.push( prevNormal.x, prevNormal.y, prevNormal.z );\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tdefault:\t\t\t// default handling for all vertices in between\n\n\t\t\t\t\tdx = points[ j + 1 ].x - points[ j ].x;\n\t\t\t\t\tdy = points[ j + 1 ].y - points[ j ].y;\n\n\t\t\t\t\tnormal.x = dy * 1.0;\n\t\t\t\t\tnormal.y = - dx;\n\t\t\t\t\tnormal.z = dy * 0.0;\n\n\t\t\t\t\tcurNormal.copy( normal );\n\n\t\t\t\t\tnormal.x += prevNormal.x;\n\t\t\t\t\tnormal.y += prevNormal.y;\n\t\t\t\t\tnormal.z += prevNormal.z;\n\n\t\t\t\t\tnormal.normalize();\n\n\t\t\t\t\tinitNormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t\tprevNormal.copy( curNormal );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// generate vertices, uvs and normals\n\n\t\tfor ( let i = 0; i <= segments; i ++ ) {\n\n\t\t\tconst phi = phiStart + i * inverseSegments * phiLength;\n\n\t\t\tconst sin = Math.sin( phi );\n\t\t\tconst cos = Math.cos( phi );\n\n\t\t\tfor ( let j = 0; j <= ( points.length - 1 ); j ++ ) {\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertex.x = points[ j ].x * sin;\n\t\t\t\tvertex.y = points[ j ].y;\n\t\t\t\tvertex.z = points[ j ].x * cos;\n\n\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t// uv\n\n\t\t\t\tuv.x = i / segments;\n\t\t\t\tuv.y = j / ( points.length - 1 );\n\n\t\t\t\tuvs.push( uv.x, uv.y );\n\n\t\t\t\t// normal\n\n\t\t\t\tconst x = initNormals[ 3 * j + 0 ] * sin;\n\t\t\t\tconst y = initNormals[ 3 * j + 1 ];\n\t\t\t\tconst z = initNormals[ 3 * j + 0 ] * cos;\n\n\t\t\t\tnormals.push( x, y, z );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// indices\n\n\t\tfor ( let i = 0; i < segments; i ++ ) {\n\n\t\t\tfor ( let j = 0; j < ( points.length - 1 ); j ++ ) {\n\n\t\t\t\tconst base = j + i * points.length;\n\n\t\t\t\tconst a = base;\n\t\t\t\tconst b = base + points.length;\n\t\t\t\tconst c = base + points.length + 1;\n\t\t\t\tconst d = base + 1;\n\n\t\t\t\t// faces\n\n\t\t\t\tindices.push( a, b, d );\n\t\t\t\tindices.push( c, d, b );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new LatheGeometry( data.points, data.segments, data.phiStart, data.phiLength );\n\n\t}\n\n}\n\n\nexport { LatheGeometry };\n\n// Path: src/geometries/BoxGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\n\nclass BoxGeometry extends BufferGeometry {\n\n\tconstructor( width = 1, height = 1, depth = 1, widthSegments = 1, heightSegments = 1, depthSegments = 1 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'BoxGeometry';\n\n\t\tthis.parameters = {\n\t\t\twidth: width,\n\t\t\theight: height,\n\t\t\tdepth: depth,\n\t\t\twidthSegments: widthSegments,\n\t\t\theightSegments: heightSegments,\n\t\t\tdepthSegments: depthSegments\n\t\t};\n\n\t\tconst scope = this;\n\n\t\t// segments\n\n\t\twidthSegments = Math.floor( widthSegments );\n\t\theightSegments = Math.floor( heightSegments );\n\t\tdepthSegments = Math.floor( depthSegments );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\t// helper variables\n\n\t\tlet numberOfVertices = 0;\n\t\tlet groupStart = 0;\n\n\t\t// build each side of the box geometry\n\n\t\tbuildPlane( 'z', 'y', 'x', - 1, - 1, depth, height, width, depthSegments, heightSegments, 0 ); // px\n\t\tbuildPlane( 'z', 'y', 'x', 1, - 1, depth, height, - width, depthSegments, heightSegments, 1 ); // nx\n\t\tbuildPlane( 'x', 'z', 'y', 1, 1, width, depth, height, widthSegments, depthSegments, 2 ); // py\n\t\tbuildPlane( 'x', 'z', 'y', 1, - 1, width, depth, - height, widthSegments, depthSegments, 3 ); // ny\n\t\tbuildPlane( 'x', 'y', 'z', 1, - 1, width, height, depth, widthSegments, heightSegments, 4 ); // pz\n\t\tbuildPlane( 'x', 'y', 'z', - 1, - 1, width, height, - depth, widthSegments, heightSegments, 5 ); // nz\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t\tfunction buildPlane( u, v, w, udir, vdir, width, height, depth, gridX, gridY, materialIndex ) {\n\n\t\t\tconst segmentWidth = width / gridX;\n\t\t\tconst segmentHeight = height / gridY;\n\n\t\t\tconst widthHalf = width / 2;\n\t\t\tconst heightHalf = height / 2;\n\t\t\tconst depthHalf = depth / 2;\n\n\t\t\tconst gridX1 = gridX + 1;\n\t\t\tconst gridY1 = gridY + 1;\n\n\t\t\tlet vertexCounter = 0;\n\t\t\tlet groupCount = 0;\n\n\t\t\tconst vector = new Vector3();\n\n\t\t\t// generate vertices, normals and uvs\n\n\t\t\tfor ( let iy = 0; iy < gridY1; iy ++ ) {\n\n\t\t\t\tconst y = iy * segmentHeight - heightHalf;\n\n\t\t\t\tfor ( let ix = 0; ix < gridX1; ix ++ ) {\n\n\t\t\t\t\tconst x = ix * segmentWidth - widthHalf;\n\n\t\t\t\t\t// set values to correct vector component\n\n\t\t\t\t\tvector[ u ] = x * udir;\n\t\t\t\t\tvector[ v ] = y * vdir;\n\t\t\t\t\tvector[ w ] = depthHalf;\n\n\t\t\t\t\t// now apply vector to vertex buffer\n\n\t\t\t\t\tvertices.push( vector.x, vector.y, vector.z );\n\n\t\t\t\t\t// set values to correct vector component\n\n\t\t\t\t\tvector[ u ] = 0;\n\t\t\t\t\tvector[ v ] = 0;\n\t\t\t\t\tvector[ w ] = depth > 0 ? 1 : - 1;\n\n\t\t\t\t\t// now apply vector to normal buffer\n\n\t\t\t\t\tnormals.push( vector.x, vector.y, vector.z );\n\n\t\t\t\t\t// uvs\n\n\t\t\t\t\tuvs.push( ix / gridX );\n\t\t\t\t\tuvs.push( 1 - ( iy / gridY ) );\n\n\t\t\t\t\t// counters\n\n\t\t\t\t\tvertexCounter += 1;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// indices\n\n\t\t\t// 1. you need three indices to draw a single face\n\t\t\t// 2. a single segment consists of two faces\n\t\t\t// 3. so we need to generate six (2*3) indices per segment\n\n\t\t\tfor ( let iy = 0; iy < gridY; iy ++ ) {\n\n\t\t\t\tfor ( let ix = 0; ix < gridX; ix ++ ) {\n\n\t\t\t\t\tconst a = numberOfVertices + ix + gridX1 * iy;\n\t\t\t\t\tconst b = numberOfVertices + ix + gridX1 * ( iy + 1 );\n\t\t\t\t\tconst c = numberOfVertices + ( ix + 1 ) + gridX1 * ( iy + 1 );\n\t\t\t\t\tconst d = numberOfVertices + ( ix + 1 ) + gridX1 * iy;\n\n\t\t\t\t\t// faces\n\n\t\t\t\t\tindices.push( a, b, d );\n\t\t\t\t\tindices.push( b, c, d );\n\n\t\t\t\t\t// increase counter\n\n\t\t\t\t\tgroupCount += 6;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// add a group to the geometry. this will ensure multi material support\n\n\t\t\tscope.addGroup( groupStart, groupCount, materialIndex );\n\n\t\t\t// calculate new start value for groups\n\n\t\t\tgroupStart += groupCount;\n\n\t\t\t// update total number of vertices\n\n\t\t\tnumberOfVertices += vertexCounter;\n\n\t\t}\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new BoxGeometry( data.width, data.height, data.depth, data.widthSegments, data.heightSegments, data.depthSegments );\n\n\t}\n\n}\n\nexport { BoxGeometry };\n\n// Path: src/geometries/PlaneGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\n\nclass PlaneGeometry extends BufferGeometry {\n\n\tconstructor( width = 1, height = 1, widthSegments = 1, heightSegments = 1 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'PlaneGeometry';\n\n\t\tthis.parameters = {\n\t\t\twidth: width,\n\t\t\theight: height,\n\t\t\twidthSegments: widthSegments,\n\t\t\theightSegments: heightSegments\n\t\t};\n\n\t\tconst width_half = width / 2;\n\t\tconst height_half = height / 2;\n\n\t\tconst gridX = Math.floor( widthSegments );\n\t\tconst gridY = Math.floor( heightSegments );\n\n\t\tconst gridX1 = gridX + 1;\n\t\tconst gridY1 = gridY + 1;\n\n\t\tconst segment_width = width / gridX;\n\t\tconst segment_height = height / gridY;\n\n\t\t//\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\tfor ( let iy = 0; iy < gridY1; iy ++ ) {\n\n\t\t\tconst y = iy * segment_height - height_half;\n\n\t\t\tfor ( let ix = 0; ix < gridX1; ix ++ ) {\n\n\t\t\t\tconst x = ix * segment_width - width_half;\n\n\t\t\t\tvertices.push( x, - y, 0 );\n\n\t\t\t\tnormals.push( 0, 0, 1 );\n\n\t\t\t\tuvs.push( ix / gridX );\n\t\t\t\tuvs.push( 1 - ( iy / gridY ) );\n\n\t\t\t}\n\n\t\t}\n\n\t\tfor ( let iy = 0; iy < gridY; iy ++ ) {\n\n\t\t\tfor ( let ix = 0; ix < gridX; ix ++ ) {\n\n\t\t\t\tconst a = ix + gridX1 * iy;\n\t\t\t\tconst b = ix + gridX1 * ( iy + 1 );\n\t\t\t\tconst c = ( ix + 1 ) + gridX1 * ( iy + 1 );\n\t\t\t\tconst d = ( ix + 1 ) + gridX1 * iy;\n\n\t\t\t\tindices.push( a, b, d );\n\t\t\t\tindices.push( b, c, d );\n\n\t\t\t}\n\n\t\t}\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new PlaneGeometry( data.width, data.height, data.widthSegments, data.heightSegments );\n\n\t}\n\n}\n\nexport { PlaneGeometry };\n\n// Path: src/geometries/WireframeGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\n\nclass WireframeGeometry extends BufferGeometry {\n\n\tconstructor( geometry = null ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'WireframeGeometry';\n\n\t\tthis.parameters = {\n\t\t\tgeometry: geometry\n\t\t};\n\n\t\tif ( geometry !== null ) {\n\n\t\t\t// buffer\n\n\t\t\tconst vertices = [];\n\t\t\tconst edges = new Set();\n\n\t\t\t// helper variables\n\n\t\t\tconst start = new Vector3();\n\t\t\tconst end = new Vector3();\n\n\t\t\tif ( geometry.index !== null ) {\n\n\t\t\t\t// indexed BufferGeometry\n\n\t\t\t\tconst position = geometry.attributes.position;\n\t\t\t\tconst indices = geometry.index;\n\t\t\t\tlet groups = geometry.groups;\n\n\t\t\t\tif ( groups.length === 0 ) {\n\n\t\t\t\t\tgroups = [ { start: 0, count: indices.count, materialIndex: 0 } ];\n\n\t\t\t\t}\n\n\t\t\t\t// create a data structure that contains all edges without duplicates\n\n\t\t\t\tfor ( let o = 0, ol = groups.length; o < ol; ++ o ) {\n\n\t\t\t\t\tconst group = groups[ o ];\n\n\t\t\t\t\tconst groupStart = group.start;\n\t\t\t\t\tconst groupCount = group.count;\n\n\t\t\t\t\tfor ( let i = groupStart, l = ( groupStart + groupCount ); i < l; i += 3 ) {\n\n\t\t\t\t\t\tfor ( let j = 0; j < 3; j ++ ) {\n\n\t\t\t\t\t\t\tconst index1 = indices.getX( i + j );\n\t\t\t\t\t\t\tconst index2 = indices.getX( i + ( j + 1 ) % 3 );\n\n\t\t\t\t\t\t\tstart.fromBufferAttribute( position, index1 );\n\t\t\t\t\t\t\tend.fromBufferAttribute( position, index2 );\n\n\t\t\t\t\t\t\tif ( isUniqueEdge( start, end, edges ) === true ) {\n\n\t\t\t\t\t\t\t\tvertices.push( start.x, start.y, start.z );\n\t\t\t\t\t\t\t\tvertices.push( end.x, end.y, end.z );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\t// non-indexed BufferGeometry\n\n\t\t\t\tconst position = geometry.attributes.position;\n\n\t\t\t\tfor ( let i = 0, l = ( position.count / 3 ); i < l; i ++ ) {\n\n\t\t\t\t\tfor ( let j = 0; j < 3; j ++ ) {\n\n\t\t\t\t\t\t// three edges per triangle, an edge is represented as (index1, index2)\n\t\t\t\t\t\t// e.g. the first triangle has the following edges: (0,1),(1,2),(2,0)\n\n\t\t\t\t\t\tconst index1 = 3 * i + j;\n\t\t\t\t\t\tconst index2 = 3 * i + ( ( j + 1 ) % 3 );\n\n\t\t\t\t\t\tstart.fromBufferAttribute( position, index1 );\n\t\t\t\t\t\tend.fromBufferAttribute( position, index2 );\n\n\t\t\t\t\t\tif ( isUniqueEdge( start, end, edges ) === true ) {\n\n\t\t\t\t\t\t\tvertices.push( start.x, start.y, start.z );\n\t\t\t\t\t\t\tvertices.push( end.x, end.y, end.z );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// build geometry\n\n\t\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\n\t\t}\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n}\n\nfunction isUniqueEdge( start, end, edges ) {\n\n\tconst hash1 = `${start.x},${start.y},${start.z}-${end.x},${end.y},${end.z}`;\n\tconst hash2 = `${end.x},${end.y},${end.z}-${start.x},${start.y},${start.z}`; // coincident edge\n\n\tif ( edges.has( hash1 ) === true || edges.has( hash2 ) === true ) {\n\n\t\treturn false;\n\n\t} else {\n\n\t\tedges.add( hash1 );\n\t\tedges.add( hash2 );\n\t\treturn true;\n\n\t}\n\n}\n\n\nexport { WireframeGeometry };\n\n// Path: src/geometries/TorusGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\n\nclass TorusGeometry extends BufferGeometry {\n\n\tconstructor( radius = 1, tube = 0.4, radialSegments = 12, tubularSegments = 48, arc = Math.PI * 2 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'TorusGeometry';\n\n\t\tthis.parameters = {\n\t\t\tradius: radius,\n\t\t\ttube: tube,\n\t\t\tradialSegments: radialSegments,\n\t\t\ttubularSegments: tubularSegments,\n\t\t\tarc: arc\n\t\t};\n\n\t\tradialSegments = Math.floor( radialSegments );\n\t\ttubularSegments = Math.floor( tubularSegments );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\t// helper variables\n\n\t\tconst center = new Vector3();\n\t\tconst vertex = new Vector3();\n\t\tconst normal = new Vector3();\n\n\t\t// generate vertices, normals and uvs\n\n\t\tfor ( let j = 0; j <= radialSegments; j ++ ) {\n\n\t\t\tfor ( let i = 0; i <= tubularSegments; i ++ ) {\n\n\t\t\t\tconst u = i / tubularSegments * arc;\n\t\t\t\tconst v = j / radialSegments * Math.PI * 2;\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertex.x = ( radius + tube * Math.cos( v ) ) * Math.cos( u );\n\t\t\t\tvertex.y = ( radius + tube * Math.cos( v ) ) * Math.sin( u );\n\t\t\t\tvertex.z = tube * Math.sin( v );\n\n\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t// normal\n\n\t\t\t\tcenter.x = radius * Math.cos( u );\n\t\t\t\tcenter.y = radius * Math.sin( u );\n\t\t\t\tnormal.subVectors( vertex, center ).normalize();\n\n\t\t\t\tnormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t// uv\n\n\t\t\t\tuvs.push( i / tubularSegments );\n\t\t\t\tuvs.push( j / radialSegments );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// generate indices\n\n\t\tfor ( let j = 1; j <= radialSegments; j ++ ) {\n\n\t\t\tfor ( let i = 1; i <= tubularSegments; i ++ ) {\n\n\t\t\t\t// indices\n\n\t\t\t\tconst a = ( tubularSegments + 1 ) * j + i - 1;\n\t\t\t\tconst b = ( tubularSegments + 1 ) * ( j - 1 ) + i - 1;\n\t\t\t\tconst c = ( tubularSegments + 1 ) * ( j - 1 ) + i;\n\t\t\t\tconst d = ( tubularSegments + 1 ) * j + i;\n\n\t\t\t\t// faces\n\n\t\t\t\tindices.push( a, b, d );\n\t\t\t\tindices.push( b, c, d );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new TorusGeometry( data.radius, data.tube, data.radialSegments, data.tubularSegments, data.arc );\n\n\t}\n\n}\n\nexport { TorusGeometry };\n\n// Path: src/extras/Earcut.js\n/**\n * Port from https://github.com/mapbox/earcut (v2.2.4)\n */\n\nconst Earcut = {\n\n\ttriangulate: function ( data, holeIndices, dim = 2 ) {\n\n\t\tconst hasHoles = holeIndices && holeIndices.length;\n\t\tconst outerLen = hasHoles ? holeIndices[ 0 ] * dim : data.length;\n\t\tlet outerNode = linkedList( data, 0, outerLen, dim, true );\n\t\tconst triangles = [];\n\n\t\tif ( ! outerNode || outerNode.next === outerNode.prev ) return triangles;\n\n\t\tlet minX, minY, maxX, maxY, x, y, invSize;\n\n\t\tif ( hasHoles ) outerNode = eliminateHoles( data, holeIndices, outerNode, dim );\n\n\t\t// if the shape is not too simple, we'll use z-order curve hash later; calculate polygon bbox\n\t\tif ( data.length > 80 * dim ) {\n\n\t\t\tminX = maxX = data[ 0 ];\n\t\t\tminY = maxY = data[ 1 ];\n\n\t\t\tfor ( let i = dim; i < outerLen; i += dim ) {\n\n\t\t\t\tx = data[ i ];\n\t\t\t\ty = data[ i + 1 ];\n\t\t\t\tif ( x < minX ) minX = x;\n\t\t\t\tif ( y < minY ) minY = y;\n\t\t\t\tif ( x > maxX ) maxX = x;\n\t\t\t\tif ( y > maxY ) maxY = y;\n\n\t\t\t}\n\n\t\t\t// minX, minY and invSize are later used to transform coords into integers for z-order calculation\n\t\t\tinvSize = Math.max( maxX - minX, maxY - minY );\n\t\t\tinvSize = invSize !== 0 ? 32767 / invSize : 0;\n\n\t\t}\n\n\t\tearcutLinked( outerNode, triangles, dim, minX, minY, invSize, 0 );\n\n\t\treturn triangles;\n\n\t}\n\n};\n\n// create a circular doubly linked list from polygon points in the specified winding order\nfunction linkedList( data, start, end, dim, clockwise ) {\n\n\tlet i, last;\n\n\tif ( clockwise === ( signedArea( data, start, end, dim ) > 0 ) ) {\n\n\t\tfor ( i = start; i < end; i += dim ) last = insertNode( i, data[ i ], data[ i + 1 ], last );\n\n\t} else {\n\n\t\tfor ( i = end - dim; i >= start; i -= dim ) last = insertNode( i, data[ i ], data[ i + 1 ], last );\n\n\t}\n\n\tif ( last && equals( last, last.next ) ) {\n\n\t\tremoveNode( last );\n\t\tlast = last.next;\n\n\t}\n\n\treturn last;\n\n}\n\n// eliminate colinear or duplicate points\nfunction filterPoints( start, end ) {\n\n\tif ( ! start ) return start;\n\tif ( ! end ) end = start;\n\n\tlet p = start,\n\t\tagain;\n\tdo {\n\n\t\tagain = false;\n\n\t\tif ( ! p.steiner && ( equals( p, p.next ) || area( p.prev, p, p.next ) === 0 ) ) {\n\n\t\t\tremoveNode( p );\n\t\t\tp = end = p.prev;\n\t\t\tif ( p === p.next ) break;\n\t\t\tagain = true;\n\n\t\t} else {\n\n\t\t\tp = p.next;\n\n\t\t}\n\n\t} while ( again || p !== end );\n\n\treturn end;\n\n}\n\n// main ear slicing loop which triangulates a polygon (given as a linked list)\nfunction earcutLinked( ear, triangles, dim, minX, minY, invSize, pass ) {\n\n\tif ( ! ear ) return;\n\n\t// interlink polygon nodes in z-order\n\tif ( ! pass && invSize ) indexCurve( ear, minX, minY, invSize );\n\n\tlet stop = ear,\n\t\tprev, next;\n\n\t// iterate through ears, slicing them one by one\n\twhile ( ear.prev !== ear.next ) {\n\n\t\tprev = ear.prev;\n\t\tnext = ear.next;\n\n\t\tif ( invSize ? isEarHashed( ear, minX, minY, invSize ) : isEar( ear ) ) {\n\n\t\t\t// cut off the triangle\n\t\t\ttriangles.push( prev.i / dim | 0 );\n\t\t\ttriangles.push( ear.i / dim | 0 );\n\t\t\ttriangles.push( next.i / dim | 0 );\n\n\t\t\tremoveNode( ear );\n\n\t\t\t// skipping the next vertex leads to less sliver triangles\n\t\t\tear = next.next;\n\t\t\tstop = next.next;\n\n\t\t\tcontinue;\n\n\t\t}\n\n\t\tear = next;\n\n\t\t// if we looped through the whole remaining polygon and can't find any more ears\n\t\tif ( ear === stop ) {\n\n\t\t\t// try filtering points and slicing again\n\t\t\tif ( ! pass ) {\n\n\t\t\t\tearcutLinked( filterPoints( ear ), triangles, dim, minX, minY, invSize, 1 );\n\n\t\t\t\t// if this didn't work, try curing all small self-intersections locally\n\n\t\t\t} else if ( pass === 1 ) {\n\n\t\t\t\tear = cureLocalIntersections( filterPoints( ear ), triangles, dim );\n\t\t\t\tearcutLinked( ear, triangles, dim, minX, minY, invSize, 2 );\n\n\t\t\t\t// as a last resort, try splitting the remaining polygon into two\n\n\t\t\t} else if ( pass === 2 ) {\n\n\t\t\t\tsplitEarcut( ear, triangles, dim, minX, minY, invSize );\n\n\t\t\t}\n\n\t\t\tbreak;\n\n\t\t}\n\n\t}\n\n}\n\n// check whether a polygon node forms a valid ear with adjacent nodes\nfunction isEar( ear ) {\n\n\tconst a = ear.prev,\n\t\tb = ear,\n\t\tc = ear.next;\n\n\tif ( area( a, b, c ) >= 0 ) return false; // reflex, can't be an ear\n\n\t// now make sure we don't have other points inside the potential ear\n\tconst ax = a.x, bx = b.x, cx = c.x, ay = a.y, by = b.y, cy = c.y;\n\n\t// triangle bbox; min & max are calculated like this for speed\n\tconst x0 = ax < bx ? ( ax < cx ? ax : cx ) : ( bx < cx ? bx : cx ),\n\t\ty0 = ay < by ? ( ay < cy ? ay : cy ) : ( by < cy ? by : cy ),\n\t\tx1 = ax > bx ? ( ax > cx ? ax : cx ) : ( bx > cx ? bx : cx ),\n\t\ty1 = ay > by ? ( ay > cy ? ay : cy ) : ( by > cy ? by : cy );\n\n\tlet p = c.next;\n\twhile ( p !== a ) {\n\n\t\tif ( p.x >= x0 && p.x <= x1 && p.y >= y0 && p.y <= y1 &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, p.x, p.y ) &&\n\t\t\tarea( p.prev, p, p.next ) >= 0 ) return false;\n\t\tp = p.next;\n\n\t}\n\n\treturn true;\n\n}\n\nfunction isEarHashed( ear, minX, minY, invSize ) {\n\n\tconst a = ear.prev,\n\t\tb = ear,\n\t\tc = ear.next;\n\n\tif ( area( a, b, c ) >= 0 ) return false; // reflex, can't be an ear\n\n\tconst ax = a.x, bx = b.x, cx = c.x, ay = a.y, by = b.y, cy = c.y;\n\n\t// triangle bbox; min & max are calculated like this for speed\n\tconst x0 = ax < bx ? ( ax < cx ? ax : cx ) : ( bx < cx ? bx : cx ),\n\t\ty0 = ay < by ? ( ay < cy ? ay : cy ) : ( by < cy ? by : cy ),\n\t\tx1 = ax > bx ? ( ax > cx ? ax : cx ) : ( bx > cx ? bx : cx ),\n\t\ty1 = ay > by ? ( ay > cy ? ay : cy ) : ( by > cy ? by : cy );\n\n\t// z-order range for the current triangle bbox;\n\tconst minZ = zOrder( x0, y0, minX, minY, invSize ),\n\t\tmaxZ = zOrder( x1, y1, minX, minY, invSize );\n\n\tlet p = ear.prevZ,\n\t\tn = ear.nextZ;\n\n\t// look for points inside the triangle in both directions\n\twhile ( p && p.z >= minZ && n && n.z <= maxZ ) {\n\n\t\tif ( p.x >= x0 && p.x <= x1 && p.y >= y0 && p.y <= y1 && p !== a && p !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, p.x, p.y ) && area( p.prev, p, p.next ) >= 0 ) return false;\n\t\tp = p.prevZ;\n\n\t\tif ( n.x >= x0 && n.x <= x1 && n.y >= y0 && n.y <= y1 && n !== a && n !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, n.x, n.y ) && area( n.prev, n, n.next ) >= 0 ) return false;\n\t\tn = n.nextZ;\n\n\t}\n\n\t// look for remaining points in decreasing z-order\n\twhile ( p && p.z >= minZ ) {\n\n\t\tif ( p.x >= x0 && p.x <= x1 && p.y >= y0 && p.y <= y1 && p !== a && p !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, p.x, p.y ) && area( p.prev, p, p.next ) >= 0 ) return false;\n\t\tp = p.prevZ;\n\n\t}\n\n\t// look for remaining points in increasing z-order\n\twhile ( n && n.z <= maxZ ) {\n\n\t\tif ( n.x >= x0 && n.x <= x1 && n.y >= y0 && n.y <= y1 && n !== a && n !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, n.x, n.y ) && area( n.prev, n, n.next ) >= 0 ) return false;\n\t\tn = n.nextZ;\n\n\t}\n\n\treturn true;\n\n}\n\n// go through all polygon nodes and cure small local self-intersections\nfunction cureLocalIntersections( start, triangles, dim ) {\n\n\tlet p = start;\n\tdo {\n\n\t\tconst a = p.prev,\n\t\t\tb = p.next.next;\n\n\t\tif ( ! equals( a, b ) && intersects( a, p, p.next, b ) && locallyInside( a, b ) && locallyInside( b, a ) ) {\n\n\t\t\ttriangles.push( a.i / dim | 0 );\n\t\t\ttriangles.push( p.i / dim | 0 );\n\t\t\ttriangles.push( b.i / dim | 0 );\n\n\t\t\t// remove two nodes involved\n\t\t\tremoveNode( p );\n\t\t\tremoveNode( p.next );\n\n\t\t\tp = start = b;\n\n\t\t}\n\n\t\tp = p.next;\n\n\t} while ( p !== start );\n\n\treturn filterPoints( p );\n\n}\n\n// try splitting polygon into two and triangulate them independently\nfunction splitEarcut( start, triangles, dim, minX, minY, invSize ) {\n\n\t// look for a valid diagonal that divides the polygon into two\n\tlet a = start;\n\tdo {\n\n\t\tlet b = a.next.next;\n\t\twhile ( b !== a.prev ) {\n\n\t\t\tif ( a.i !== b.i && isValidDiagonal( a, b ) ) {\n\n\t\t\t\t// split the polygon in two by the diagonal\n\t\t\t\tlet c = splitPolygon( a, b );\n\n\t\t\t\t// filter colinear points around the cuts\n\t\t\t\ta = filterPoints( a, a.next );\n\t\t\t\tc = filterPoints( c, c.next );\n\n\t\t\t\t// run earcut on each half\n\t\t\t\tearcutLinked( a, triangles, dim, minX, minY, invSize, 0 );\n\t\t\t\tearcutLinked( c, triangles, dim, minX, minY, invSize, 0 );\n\t\t\t\treturn;\n\n\t\t\t}\n\n\t\t\tb = b.next;\n\n\t\t}\n\n\t\ta = a.next;\n\n\t} while ( a !== start );\n\n}\n\n// link every hole into the outer loop, producing a single-ring polygon without holes\nfunction eliminateHoles( data, holeIndices, outerNode, dim ) {\n\n\tconst queue = [];\n\tlet i, len, start, end, list;\n\n\tfor ( i = 0, len = holeIndices.length; i < len; i ++ ) {\n\n\t\tstart = holeIndices[ i ] * dim;\n\t\tend = i < len - 1 ? holeIndices[ i + 1 ] * dim : data.length;\n\t\tlist = linkedList( data, start, end, dim, false );\n\t\tif ( list === list.next ) list.steiner = true;\n\t\tqueue.push( getLeftmost( list ) );\n\n\t}\n\n\tqueue.sort( compareX );\n\n\t// process holes from left to right\n\tfor ( i = 0; i < queue.length; i ++ ) {\n\n\t\touterNode = eliminateHole( queue[ i ], outerNode );\n\n\t}\n\n\treturn outerNode;\n\n}\n\nfunction compareX( a, b ) {\n\n\treturn a.x - b.x;\n\n}\n\n// find a bridge between vertices that connects hole with an outer ring and link it\nfunction eliminateHole( hole, outerNode ) {\n\n\tconst bridge = findHoleBridge( hole, outerNode );\n\tif ( ! bridge ) {\n\n\t\treturn outerNode;\n\n\t}\n\n\tconst bridgeReverse = splitPolygon( bridge, hole );\n\n\t// filter collinear points around the cuts\n\tfilterPoints( bridgeReverse, bridgeReverse.next );\n\treturn filterPoints( bridge, bridge.next );\n\n}\n\n// David Eberly's algorithm for finding a bridge between hole and outer polygon\nfunction findHoleBridge( hole, outerNode ) {\n\n\tlet p = outerNode,\n\t\tqx = - Infinity,\n\t\tm;\n\n\tconst hx = hole.x, hy = hole.y;\n\n\t// find a segment intersected by a ray from the hole's leftmost point to the left;\n\t// segment's endpoint with lesser x will be potential connection point\n\tdo {\n\n\t\tif ( hy <= p.y && hy >= p.next.y && p.next.y !== p.y ) {\n\n\t\t\tconst x = p.x + ( hy - p.y ) * ( p.next.x - p.x ) / ( p.next.y - p.y );\n\t\t\tif ( x <= hx && x > qx ) {\n\n\t\t\t\tqx = x;\n\t\t\t\tm = p.x < p.next.x ? p : p.next;\n\t\t\t\tif ( x === hx ) return m; // hole touches outer segment; pick leftmost endpoint\n\n\t\t\t}\n\n\t\t}\n\n\t\tp = p.next;\n\n\t} while ( p !== outerNode );\n\n\tif ( ! m ) return null;\n\n\t// look for points inside the triangle of hole point, segment intersection and endpoint;\n\t// if there are no points found, we have a valid connection;\n\t// otherwise choose the point of the minimum angle with the ray as connection point\n\n\tconst stop = m,\n\t\tmx = m.x,\n\t\tmy = m.y;\n\tlet tanMin = Infinity, tan;\n\n\tp = m;\n\n\tdo {\n\n\t\tif ( hx >= p.x && p.x >= mx && hx !== p.x &&\n\t\t\t\tpointInTriangle( hy < my ? hx : qx, hy, mx, my, hy < my ? qx : hx, hy, p.x, p.y ) ) {\n\n\t\t\ttan = Math.abs( hy - p.y ) / ( hx - p.x ); // tangential\n\n\t\t\tif ( locallyInside( p, hole ) && ( tan < tanMin || ( tan === tanMin && ( p.x > m.x || ( p.x === m.x && sectorContainsSector( m, p ) ) ) ) ) ) {\n\n\t\t\t\tm = p;\n\t\t\t\ttanMin = tan;\n\n\t\t\t}\n\n\t\t}\n\n\t\tp = p.next;\n\n\t} while ( p !== stop );\n\n\treturn m;\n\n}\n\n// whether sector in vertex m contains sector in vertex p in the same coordinates\nfunction sectorContainsSector( m, p ) {\n\n\treturn area( m.prev, m, p.prev ) < 0 && area( p.next, m, m.next ) < 0;\n\n}\n\n// interlink polygon nodes in z-order\nfunction indexCurve( start, minX, minY, invSize ) {\n\n\tlet p = start;\n\tdo {\n\n\t\tif ( p.z === 0 ) p.z = zOrder( p.x, p.y, minX, minY, invSize );\n\t\tp.prevZ = p.prev;\n\t\tp.nextZ = p.next;\n\t\tp = p.next;\n\n\t} while ( p !== start );\n\n\tp.prevZ.nextZ = null;\n\tp.prevZ = null;\n\n\tsortLinked( p );\n\n}\n\n// Simon Tatham's linked list merge sort algorithm\n// http://www.chiark.greenend.org.uk/~sgtatham/algorithms/listsort.html\nfunction sortLinked( list ) {\n\n\tlet i, p, q, e, tail, numMerges, pSize, qSize,\n\t\tinSize = 1;\n\n\tdo {\n\n\t\tp = list;\n\t\tlist = null;\n\t\ttail = null;\n\t\tnumMerges = 0;\n\n\t\twhile ( p ) {\n\n\t\t\tnumMerges ++;\n\t\t\tq = p;\n\t\t\tpSize = 0;\n\t\t\tfor ( i = 0; i < inSize; i ++ ) {\n\n\t\t\t\tpSize ++;\n\t\t\t\tq = q.nextZ;\n\t\t\t\tif ( ! q ) break;\n\n\t\t\t}\n\n\t\t\tqSize = inSize;\n\n\t\t\twhile ( pSize > 0 || ( qSize > 0 && q ) ) {\n\n\t\t\t\tif ( pSize !== 0 && ( qSize === 0 || ! q || p.z <= q.z ) ) {\n\n\t\t\t\t\te = p;\n\t\t\t\t\tp = p.nextZ;\n\t\t\t\t\tpSize --;\n\n\t\t\t\t} else {\n\n\t\t\t\t\te = q;\n\t\t\t\t\tq = q.nextZ;\n\t\t\t\t\tqSize --;\n\n\t\t\t\t}\n\n\t\t\t\tif ( tail ) tail.nextZ = e;\n\t\t\t\telse list = e;\n\n\t\t\t\te.prevZ = tail;\n\t\t\t\ttail = e;\n\n\t\t\t}\n\n\t\t\tp = q;\n\n\t\t}\n\n\t\ttail.nextZ = null;\n\t\tinSize *= 2;\n\n\t} while ( numMerges > 1 );\n\n\treturn list;\n\n}\n\n// z-order of a point given coords and inverse of the longer side of data bbox\nfunction zOrder( x, y, minX, minY, invSize ) {\n\n\t// coords are transformed into non-negative 15-bit integer range\n\tx = ( x - minX ) * invSize | 0;\n\ty = ( y - minY ) * invSize | 0;\n\n\tx = ( x | ( x << 8 ) ) & 0x00FF00FF;\n\tx = ( x | ( x << 4 ) ) & 0x0F0F0F0F;\n\tx = ( x | ( x << 2 ) ) & 0x33333333;\n\tx = ( x | ( x << 1 ) ) & 0x55555555;\n\n\ty = ( y | ( y << 8 ) ) & 0x00FF00FF;\n\ty = ( y | ( y << 4 ) ) & 0x0F0F0F0F;\n\ty = ( y | ( y << 2 ) ) & 0x33333333;\n\ty = ( y | ( y << 1 ) ) & 0x55555555;\n\n\treturn x | ( y << 1 );\n\n}\n\n// find the leftmost node of a polygon ring\nfunction getLeftmost( start ) {\n\n\tlet p = start,\n\t\tleftmost = start;\n\tdo {\n\n\t\tif ( p.x < leftmost.x || ( p.x === leftmost.x && p.y < leftmost.y ) ) leftmost = p;\n\t\tp = p.next;\n\n\t} while ( p !== start );\n\n\treturn leftmost;\n\n}\n\n// check if a point lies within a convex triangle\nfunction pointInTriangle( ax, ay, bx, by, cx, cy, px, py ) {\n\n\treturn ( cx - px ) * ( ay - py ) >= ( ax - px ) * ( cy - py ) &&\n           ( ax - px ) * ( by - py ) >= ( bx - px ) * ( ay - py ) &&\n...\n", "needle_token_start": 359, "needle_token_end": 1169, "code_context_ntokens": 14934, "output": ["```javascript\nfunction generateTorso() {\n\n\t\t\tconst normal = new Vector3();\n\t\t\tconst vertex = new Vector3();\n\n\t\t\tlet groupCount = 0;\n\n\t\t\t// this will be used to calculate the normal\n\t\t\tconst slope = ( radiusBottom - radiusTop ) / height;\n\n\t\t\t// generate vertices, normals and uvs\n\n\t\t\tfor ( let y = 0; y <= heightSegments; y ++ ) {\n\n\t\t\t\tconst indexRow = [];\n\n\t\t\t\tconst v = y / heightSegments;\n\n\t\t\t\t// calculate the radius of the current row\n\n\t\t\t\tconst radius = v * ( radiusBottom - radiusTop ) + radiusTop;\n\n\t\t\t\tfor ( let x = 0; x <= radialSegments; x ++ ) {\n\n\t\t\t\t\tconst u = x / radialSegments;\n\n\t\t\t\t\tconst theta = u * thetaLength + thetaStart;\n\n\t\t\t\t\tconst sinTheta = Math.sin( theta );\n\t\t\t\t\tconst cosTheta = Math.cos( theta );\n\n\t\t\t\t\t// vertex\n\n\t\t\t\t\tvertex.x = radius * sinTheta;\n\t\t\t\t\tvertex.y = - v * height + halfHeight;\n\t\t\t\t\tvertex.z = radius * cosTheta;\n\t\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t\t// normal\n\n\t\t\t\t\tnormal.set( sinTheta, slope, cosTheta ).normalize();\n\t\t\t\t\tnormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t\t// uv\n\n\t\t\t\t\tuvs.push( u, 1 - v );\n\n\t\t\t\t\t// save index of vertex in respective row\n\n\t\t\t\t\tindexRow.push( index ++ );\n\n\t\t\t\t}\n\n\t\t\t\t// now save vertices of the row in our index array\n\n\t\t\t\tindexArray.push( indexRow );\n\n\t\t\t}\n\n\t\t\t// generate indices\n\n\t\t\tfor ( let x = 0; x < radialSegments; x ++ ) {\n\n\t\t\t\tfor ( let y = 0; y < heightSegments; y ++ ) {\n\n\t\t\t\t\t// we use the index array to access the correct indices\n\n\t\t\t\t\tconst a = indexArray[ y ][ x ];\n\t\t\t\t\tconst b = indexArray[ y + 1 ][ x ];\n\t\t\t\t\tconst c = indexArray[ y + 1 ][ x + 1 ];\n\t\t\t\t\tconst d = indexArray[ y ][ x + 1 ];\n\n\t\t\t\t\t// faces\n\n\t\t\t\t\tindices.push( a, b, d );\n\t\t\t\t\tindices.push( b, c, d );\n\n\t\t\t\t\t// update group counter\n\n\t\t\t\t\tgroupCount += 6;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// add a group to the geometry. this will ensure multi material support\n\n\t\t\tscope.addGroup( groupStart, groupCount, 0 );\n\n\t\t\t// calculate new start value for groups\n\n\t\t\tgroupStart += groupCount;\n\n\t\t}\n```"]}
{"repo": "mrdoob/three.js", "name": "sortOpaque", "language": "typescript", "path": "src/objects/BatchedMesh.js", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To compare two objects based on their depth along the z-axis, typically used for sorting purposes in rendering processes where objects need to be drawn in a specific order.\n2. **Input**: Two objects each containing a 'z' property that represents their position along the z-axis.\n3. **Output**: A numerical value indicating the relative position of the two objects along the z-axis. If the result is negative, the first object is closer than the second; if positive, the first is farther; if zero, they are at the same depth.\n4. **Procedure**: The function subtracts the 'z' property of the second object from the 'z' property of the first object and returns the result. This difference determines their order when sorting.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/core/InterleavedBuffer.js\nimport * as MathUtils from '../math/MathUtils.js';\nimport { StaticDrawUsage } from '../constants.js';\nimport { warnOnce } from '../utils.js';\n\nclass InterleavedBuffer {\n\n\tconstructor( array, stride ) {\n\n\t\tthis.isInterleavedBuffer = true;\n\n\t\tthis.array = array;\n\t\tthis.stride = stride;\n\t\tthis.count = array !== undefined ? array.length / stride : 0;\n\n\t\tthis.usage = StaticDrawUsage;\n\t\tthis._updateRange = { offset: 0, count: - 1 };\n\t\tthis.updateRanges = [];\n\n\t\tthis.version = 0;\n\n\t\tthis.uuid = MathUtils.generateUUID();\n\n\t}\n\n\tonUploadCallback() {}\n\n\tset needsUpdate( value ) {\n\n\t\tif ( value === true ) this.version ++;\n\n\t}\n\n\tget updateRange() {\n\n\t\twarnOnce( 'THREE.InterleavedBuffer: updateRange() is deprecated and will be removed in r169. Use addUpdateRange() instead.' ); // @deprecated, r159\n\t\treturn this._updateRange;\n\n\t}\n\n\tsetUsage( value ) {\n\n\t\tthis.usage = value;\n\n\t\treturn this;\n\n\t}\n\n\taddUpdateRange( start, count ) {\n\n\t\tthis.updateRanges.push( { start, count } );\n\n\t}\n\n\tclearUpdateRanges() {\n\n\t\tthis.updateRanges.length = 0;\n\n\t}\n\n\tcopy( source ) {\n\n\t\tthis.array = new source.array.constructor( source.array );\n...\n// Path: src/core/InstancedInterleavedBuffer.js\nimport { InterleavedBuffer } from './InterleavedBuffer.js';\n\nclass InstancedInterleavedBuffer extends InterleavedBuffer {\n\n\tconstructor( array, stride, meshPerAttribute = 1 ) {\n\n\t\tsuper( array, stride );\n\n\t\tthis.isInstancedInterleavedBuffer = true;\n\n\t\tthis.meshPerAttribute = meshPerAttribute;\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.meshPerAttribute = source.meshPerAttribute;\n\n\t\treturn this;\n\n\t}\n\n\tclone( data ) {\n\n\t\tconst ib = super.clone( data );\n\n\t\tib.meshPerAttribute = this.meshPerAttribute;\n\n\t\treturn ib;\n\n\t}\n\n\ttoJSON( data ) {\n\n\t\tconst json = super.toJSON( data );\n\n\t\tjson.isInstancedInterleavedBuffer = true;\n\t\tjson.meshPerAttribute = this.meshPerAttribute;\n\n\t\treturn json;\n\n\t}\n\n}\n\nexport { InstancedInterleavedBuffer };\n\n// Path: src/audio/PositionalAudio.js\nimport { Vector3 } from '../math/Vector3.js';\nimport { Quaternion } from '../math/Quaternion.js';\nimport { Audio } from './Audio.js';\n\nconst _position = /*@__PURE__*/ new Vector3();\nconst _quaternion = /*@__PURE__*/ new Quaternion();\nconst _scale = /*@__PURE__*/ new Vector3();\nconst _orientation = /*@__PURE__*/ new Vector3();\n\nclass PositionalAudio extends Audio {\n\n\tconstructor( listener ) {\n\n\t\tsuper( listener );\n\n\t\tthis.panner = this.context.createPanner();\n\t\tthis.panner.panningModel = 'HRTF';\n\t\tthis.panner.connect( this.gain );\n\n\t}\n\n\tconnect() {\n\n\t\tsuper.connect();\n\n\t\tthis.panner.connect( this.gain );\n\n\t}\n\n\tdisconnect() {\n\n\t\tsuper.disconnect();\n\n\t\tthis.panner.disconnect( this.gain );\n\n\t}\n\n\tgetOutput() {\n\n\t\treturn this.panner;\n\n\t}\n\n\tgetRefDistance() {\n\n\t\treturn this.panner.refDistance;\n\n\t}\n\n\tsetRefDistance( value ) {\n\n\t\tthis.panner.refDistance = value;\n\n\t\treturn this;\n\n\t}\n\n\tgetRolloffFactor() {\n\n\t\treturn this.panner.rolloffFactor;\n\n\t}\n\n\tsetRolloffFactor( value ) {\n\n\t\tthis.panner.rolloffFactor = value;\n\n\t\treturn this;\n\n\t}\n\n\tgetDistanceModel() {\n\n\t\treturn this.panner.distanceModel;\n\n\t}\n\n\tsetDistanceModel( value ) {\n\n\t\tthis.panner.distanceModel = value;\n\n\t\treturn this;\n\n\t}\n\n\tgetMaxDistance() {\n\n\t\treturn this.panner.maxDistance;\n\n\t}\n\n\tsetMaxDistance( value ) {\n\n\t\tthis.panner.maxDistance = value;\n\n\t\treturn this;\n\n\t}\n\n\tsetDirectionalCone( coneInnerAngle, coneOuterAngle, coneOuterGain ) {\n\n\t\tthis.panner.coneInnerAngle = coneInnerAngle;\n\t\tthis.panner.coneOuterAngle = coneOuterAngle;\n\t\tthis.panner.coneOuterGain = coneOuterGain;\n\n\t\treturn this;\n\n\t}\n\n\tupdateMatrixWorld( force ) {\n\n\t\tsuper.updateMatrixWorld( force );\n\n\t\tif ( this.hasPlaybackControl === true && this.isPlaying === false ) return;\n\n\t\tthis.matrixWorld.decompose( _position, _quaternion, _scale );\n\n\t\t_orientation.set( 0, 0, 1 ).applyQuaternion( _quaternion );\n\n\t\tconst panner = this.panner;\n\n\t\tif ( panner.positionX ) {\n\n\t\t\t// code path for Chrome and Firefox (see #14393)\n\n\t\t\tconst endTime = this.context.currentTime + this.listener.timeDelta;\n\n\t\t\tpanner.positionX.linearRampToValueAtTime( _position.x, endTime );\n\t\t\tpanner.positionY.linearRampToValueAtTime( _position.y, endTime );\n\t\t\tpanner.positionZ.linearRampToValueAtTime( _position.z, endTime );\n\t\t\tpanner.orientationX.linearRampToValueAtTime( _orientation.x, endTime );\n\t\t\tpanner.orientationY.linearRampToValueAtTime( _orientation.y, endTime );\n\t\t\tpanner.orientationZ.linearRampToValueAtTime( _orientation.z, endTime );\n\n\t\t} else {\n\n\t\t\tpanner.setPosition( _position.x, _position.y, _position.z );\n\t\t\tpanner.setOrientation( _orientation.x, _orientation.y, _orientation.z );\n\n\t\t}\n\n\t}\n\n}\n\nexport { PositionalAudio };\n\n// Path: src/textures/DataTexture.js\nimport { Texture } from './Texture.js';\nimport { NearestFilter } from '../constants.js';\n\nclass DataTexture extends Texture {\n\n\tconstructor( data = null, width = 1, height = 1, format, type, mapping, wrapS, wrapT, magFilter = NearestFilter, minFilter = NearestFilter, anisotropy, colorSpace ) {\n\n\t\tsuper( null, mapping, wrapS, wrapT, magFilter, minFilter, format, type, anisotropy, colorSpace );\n\n\t\tthis.isDataTexture = true;\n\n\t\tthis.image = { data: data, width: width, height: height };\n\n\t\tthis.generateMipmaps = false;\n\t\tthis.flipY = false;\n\t\tthis.unpackAlignment = 1;\n\n\t}\n\n}\n\nexport { DataTexture };\n\n// Path: src/objects/BatchedMesh.js\nimport { BufferAttribute } from '../core/BufferAttribute.js';\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { DataTexture } from '../textures/DataTexture.js';\nimport { FloatType } from '../constants.js';\nimport { Matrix4 } from '../math/Matrix4.js';\nimport { Mesh } from './Mesh.js';\nimport { RGBAFormat } from '../constants.js';\nimport { Box3 } from '../math/Box3.js';\nimport { Sphere } from '../math/Sphere.js';\nimport { Frustum } from '../math/Frustum.js';\nimport { Vector3 } from '../math/Vector3.js';\n\n\nfunction sortOpaque( a, b ) {\n\n\treturn a.z - b.z;\n\n}\n\nfunction sortTransparent( a, b ) {\n\n\treturn b.z - a.z;\n\n}\n\nclass MultiDrawRenderList {\n\n\tconstructor() {\n\n\t\tthis.index = 0;\n\t\tthis.pool = [];\n\t\tthis.list = [];\n\n\t}\n\n\tpush( drawRange, z ) {\n\n\t\tconst pool = this.pool;\n\t\tconst list = this.list;\n\t\tif ( this.index >= pool.length ) {\n\n\t\t\tpool.push( {\n\n\t\t\t\tstart: - 1,\n\t\t\t\tcount: - 1,\n\t\t\t\tz: - 1,\n\n\t\t\t} );\n\n\t\t}\n\n\t\tconst item = pool[ this.index ];\n\t\tlist.push( item );\n\t\tthis.index ++;\n\n\t\titem.start = drawRange.start;\n\t\titem.count = drawRange.count;\n\t\titem.z = z;\n\n\t}\n\n\treset() {\n\n\t\tthis.list.length = 0;\n\t\tthis.index = 0;\n\n\t}\n\n}\n\nconst ID_ATTR_NAME = 'batchId';\nconst _matrix = /*@__PURE__*/ new Matrix4();\nconst _invMatrixWorld = /*@__PURE__*/ new Matrix4();\nconst _identityMatrix = /*@__PURE__*/ new Matrix4();\nconst _projScreenMatrix = /*@__PURE__*/ new Matrix4();\nconst _frustum = /*@__PURE__*/ new Frustum();\nconst _box = /*@__PURE__*/ new Box3();\nconst _sphere = /*@__PURE__*/ new Sphere();\nconst _vector = /*@__PURE__*/ new Vector3();\nconst _renderList = /*@__PURE__*/ new MultiDrawRenderList();\nconst _mesh = /*@__PURE__*/ new Mesh();\nconst _batchIntersects = [];\n\n// @TODO: SkinnedMesh support?\n// @TODO: geometry.groups support?\n// @TODO: geometry.drawRange support?\n// @TODO: geometry.morphAttributes support?\n// @TODO: Support uniform parameter per geometry\n// @TODO: Add an \"optimize\" function to pack geometry and remove data gaps\n\n// copies data from attribute \"src\" into \"target\" starting at \"targetOffset\"\nfunction copyAttributeData( src, target, targetOffset = 0 ) {\n\n\tconst itemSize = target.itemSize;\n\tif ( src.isInterleavedBufferAttribute || src.array.constructor !== target.array.constructor ) {\n\n\t\t// use the component getters and setters if the array data cannot\n\t\t// be copied directly\n\t\tconst vertexCount = src.count;\n\t\tfor ( let i = 0; i < vertexCount; i ++ ) {\n\n\t\t\tfor ( let c = 0; c < itemSize; c ++ ) {\n\n\t\t\t\ttarget.setComponent( i + targetOffset, c, src.getComponent( i, c ) );\n\n\t\t\t}\n\n\t\t}\n\n\t} else {\n\n\t\t// faster copy approach using typed array set function\n\t\ttarget.array.set( src.array, targetOffset * itemSize );\n\n\t}\n\n\ttarget.needsUpdate = true;\n\n}\n\nclass BatchedMesh extends Mesh {\n\n\tget maxGeometryCount() {\n\n\t\treturn this._maxGeometryCount;\n\n\t}\n\n\tconstructor( maxGeometryCount, maxVertexCount, maxIndexCount = maxVertexCount * 2, material ) {\n\n\t\tsuper( new BufferGeometry(), material );\n\n\t\tthis.isBatchedMesh = true;\n\t\tthis.perObjectFrustumCulled = true;\n\t\tthis.sortObjects = true;\n\t\tthis.boundingBox = null;\n\t\tthis.boundingSphere = null;\n\t\tthis.customSort = null;\n\n\t\tthis._drawRanges = [];\n\t\tthis._reservedRanges = [];\n\n\t\tthis._visibility = [];\n\t\tthis._active = [];\n\t\tthis._bounds = [];\n\n\t\tthis._maxGeometryCount = maxGeometryCount;\n\t\tthis._maxVertexCount = maxVertexCount;\n\t\tthis._maxIndexCount = maxIndexCount;\n\n\t\tthis._geometryInitialized = false;\n\t\tthis._geometryCount = 0;\n\t\tthis._multiDrawCounts = new Int32Array( maxGeometryCount );\n\t\tthis._multiDrawStarts = new Int32Array( maxGeometryCount );\n\t\tthis._multiDrawCount = 0;\n\t\tthis._visibilityChanged = true;\n\n\t\t// Local matrix per geometry by using data texture\n\t\tthis._matricesTexture = null;\n\n\t\tthis._initMatricesTexture();\n\n\t}\n\n\t_initMatricesTexture() {\n\n\t\t// layout (1 matrix = 4 pixels)\n\t\t//      RGBA RGBA RGBA RGBA (=> column1, column2, column3, column4)\n\t\t//  with  8x8  pixel texture max   16 matrices * 4 pixels =  (8 * 8)\n\t\t//       16x16 pixel texture max   64 matrices * 4 pixels = (16 * 16)\n\t\t//       32x32 pixel texture max  256 matrices * 4 pixels = (32 * 32)\n\t\t//       64x64 pixel texture max 1024 matrices * 4 pixels = (64 * 64)\n\n\t\tlet size = Math.sqrt( this._maxGeometryCount * 4 ); // 4 pixels needed for 1 matrix\n\t\tsize = Math.ceil( size / 4 ) * 4;\n\t\tsize = Math.max( size, 4 );\n\n\t\tconst matricesArray = new Float32Array( size * size * 4 ); // 4 floats per RGBA pixel\n\t\tconst matricesTexture = new DataTexture( matricesArray, size, size, RGBAFormat, FloatType );\n\n\t\tthis._matricesTexture = matricesTexture;\n\n\t}\n\n\t_initializeGeometry( reference ) {\n\n\t\tconst geometry = this.geometry;\n\t\tconst maxVertexCount = this._maxVertexCount;\n\t\tconst maxGeometryCount = this._maxGeometryCount;\n\t\tconst maxIndexCount = this._maxIndexCount;\n\t\tif ( this._geometryInitialized === false ) {\n\n\t\t\tfor ( const attributeName in reference.attributes ) {\n\n\t\t\t\tconst srcAttribute = reference.getAttribute( attributeName );\n\t\t\t\tconst { array, itemSize, normalized } = srcAttribute;\n\n\t\t\t\tconst dstArray = new array.constructor( maxVertexCount * itemSize );\n\t\t\t\tconst dstAttribute = new BufferAttribute( dstArray, itemSize, normalized );\n\n\t\t\t\tgeometry.setAttribute( attributeName, dstAttribute );\n\n\t\t\t}\n\n\t\t\tif ( reference.getIndex() !== null ) {\n\n\t\t\t\tconst indexArray = maxVertexCount > 65536\n\t\t\t\t\t? new Uint32Array( maxIndexCount )\n\t\t\t\t\t: new Uint16Array( maxIndexCount );\n\n\t\t\t\tgeometry.setIndex( new BufferAttribute( indexArray, 1 ) );\n\n\t\t\t}\n\n\t\t\tconst idArray = maxGeometryCount > 65536\n\t\t\t\t? new Uint32Array( maxVertexCount )\n\t\t\t\t: new Uint16Array( maxVertexCount );\n\t\t\tgeometry.setAttribute( ID_ATTR_NAME, new BufferAttribute( idArray, 1 ) );\n\n\t\t\tthis._geometryInitialized = true;\n\n\t\t}\n\n\t}\n\n\t// Make sure the geometry is compatible with the existing combined geometry attributes\n\t_validateGeometry( geometry ) {\n\n\t\t// check that the geometry doesn't have a version of our reserved id attribute\n\t\tif ( geometry.getAttribute( ID_ATTR_NAME ) ) {\n\n\t\t\tthrow new Error( `BatchedMesh: Geometry cannot use attribute \"${ ID_ATTR_NAME }\"` );\n\n\t\t}\n\n\t\t// check to ensure the geometries are using consistent attributes and indices\n\t\tconst batchGeometry = this.geometry;\n\t\tif ( Boolean( geometry.getIndex() ) !== Boolean( batchGeometry.getIndex() ) ) {\n\n\t\t\tthrow new Error( 'BatchedMesh: All geometries must consistently have \"index\".' );\n\n\t\t}\n\n\t\tfor ( const attributeName in batchGeometry.attributes ) {\n\n\t\t\tif ( attributeName === ID_ATTR_NAME ) {\n\n\t\t\t\tcontinue;\n\n\t\t\t}\n\n\t\t\tif ( ! geometry.hasAttribute( attributeName ) ) {\n\n\t\t\t\tthrow new Error( `BatchedMesh: Added geometry missing \"${ attributeName }\". All geometries must have consistent attributes.` );\n\n\t\t\t}\n\n\t\t\tconst srcAttribute = geometry.getAttribute( attributeName );\n\t\t\tconst dstAttribute = batchGeometry.getAttribute( attributeName );\n\t\t\tif ( srcAttribute.itemSize !== dstAttribute.itemSize || srcAttribute.normalized !== dstAttribute.normalized ) {\n\n\t\t\t\tthrow new Error( 'BatchedMesh: All attributes must have a consistent itemSize and normalized value.' );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tsetCustomSort( func ) {\n\n\t\tthis.customSort = func;\n\t\treturn this;\n\n\t}\n\n\tcomputeBoundingBox() {\n\n\t\tif ( this.boundingBox === null ) {\n\n\t\t\tthis.boundingBox = new Box3();\n\n\t\t}\n\n\t\tconst geometryCount = this._geometryCount;\n\t\tconst boundingBox = this.boundingBox;\n\t\tconst active = this._active;\n\n\t\tboundingBox.makeEmpty();\n\t\tfor ( let i = 0; i < geometryCount; i ++ ) {\n\n\t\t\tif ( active[ i ] === false ) continue;\n\n\t\t\tthis.getMatrixAt( i, _matrix );\n\t\t\tthis.getBoundingBoxAt( i, _box ).applyMatrix4( _matrix );\n\t\t\tboundingBox.union( _box );\n\n\t\t}\n\n\t}\n\n\tcomputeBoundingSphere() {\n\n\t\tif ( this.boundingSphere === null ) {\n\n\t\t\tthis.boundingSphere = new Sphere();\n\n\t\t}\n\n\t\tconst geometryCount = this._geometryCount;\n\t\tconst boundingSphere = this.boundingSphere;\n\t\tconst active = this._active;\n\n\t\tboundingSphere.makeEmpty();\n\t\tfor ( let i = 0; i < geometryCount; i ++ ) {\n\n\t\t\tif ( active[ i ] === false ) continue;\n\n\t\t\tthis.getMatrixAt( i, _matrix );\n\t\t\tthis.getBoundingSphereAt( i, _sphere ).applyMatrix4( _matrix );\n\t\t\tboundingSphere.union( _sphere );\n\n\t\t}\n\n\t}\n\n\taddGeometry( geometry, vertexCount = - 1, indexCount = - 1 ) {\n\n\t\tthis._initializeGeometry( geometry );\n\n\t\tthis._validateGeometry( geometry );\n\n\t\t// ensure we're not over geometry\n\t\tif ( this._geometryCount >= this._maxGeometryCount ) {\n\n\t\t\tthrow new Error( 'BatchedMesh: Maximum geometry count reached.' );\n\n\t\t}\n\n\t\t// get the necessary range fo the geometry\n\t\tconst reservedRange = {\n\t\t\tvertexStart: - 1,\n\t\t\tvertexCount: - 1,\n\t\t\tindexStart: - 1,\n\t\t\tindexCount: - 1,\n\t\t};\n\n\t\tlet lastRange = null;\n\t\tconst reservedRanges = this._reservedRanges;\n\t\tconst drawRanges = this._drawRanges;\n\t\tconst bounds = this._bounds;\n\t\tif ( this._geometryCount !== 0 ) {\n\n\t\t\tlastRange = reservedRanges[ reservedRanges.length - 1 ];\n\n\t\t}\n\n\t\tif ( vertexCount === - 1 ) {\n\n\t\t\treservedRange.vertexCount = geometry.getAttribute( 'position' ).count;\n\n\t\t} else {\n\n\t\t\treservedRange.vertexCount = vertexCount;\n\n\t\t}\n\n\t\tif ( lastRange === null ) {\n\n\t\t\treservedRange.vertexStart = 0;\n\n\t\t} else {\n\n\t\t\treservedRange.vertexStart = lastRange.vertexStart + lastRange.vertexCount;\n\n\t\t}\n\n\t\tconst index = geometry.getIndex();\n\t\tconst hasIndex = index !== null;\n\t\tif ( hasIndex ) {\n\n\t\t\tif ( indexCount\t=== - 1 ) {\n\n\t\t\t\treservedRange.indexCount = index.count;\n\n\t\t\t} else {\n\n\t\t\t\treservedRange.indexCount = indexCount;\n\n\t\t\t}\n\n\t\t\tif ( lastRange === null ) {\n\n\t\t\t\treservedRange.indexStart = 0;\n\n\t\t\t} else {\n\n\t\t\t\treservedRange.indexStart = lastRange.indexStart + lastRange.indexCount;\n\n\t\t\t}\n\n\t\t}\n\n\t\tif (\n\t\t\treservedRange.indexStart !== - 1 &&\n\t\t\treservedRange.indexStart + reservedRange.indexCount > this._maxIndexCount ||\n\t\t\treservedRange.vertexStart + reservedRange.vertexCount > this._maxVertexCount\n\t\t) {\n\n\t\t\tthrow new Error( 'BatchedMesh: Reserved space request exceeds the maximum buffer size.' );\n\n\t\t}\n\n\t\tconst visibility = this._visibility;\n\t\tconst active = this._active;\n\t\tconst matricesTexture = this._matricesTexture;\n\t\tconst matricesArray = this._matricesTexture.image.data;\n\n\t\t// push new visibility states\n\t\tvisibility.push( true );\n\t\tactive.push( true );\n\n\t\t// update id\n\t\tconst geometryId = this._geometryCount;\n\t\tthis._geometryCount ++;\n\n\t\t// initialize matrix information\n\t\t_identityMatrix.toArray( matricesArray, geometryId * 16 );\n\t\tmatricesTexture.needsUpdate = true;\n\n\t\t// add the reserved range and draw range objects\n\t\treservedRanges.push( reservedRange );\n\t\tdrawRanges.push( {\n\t\t\tstart: hasIndex ? reservedRange.indexStart : reservedRange.vertexStart,\n\t\t\tcount: - 1\n\t\t} );\n\t\tbounds.push( {\n\t\t\tboxInitialized: false,\n\t\t\tbox: new Box3(),\n\n\t\t\tsphereInitialized: false,\n\t\t\tsphere: new Sphere()\n\t\t} );\n\n\t\t// set the id for the geometry\n\t\tconst idAttribute = this.geometry.getAttribute( ID_ATTR_NAME );\n\t\tfor ( let i = 0; i < reservedRange.vertexCount; i ++ ) {\n\n\t\t\tidAttribute.setX( reservedRange.vertexStart + i, geometryId );\n\n\t\t}\n\n\t\tidAttribute.needsUpdate = true;\n\n\t\t// update the geometry\n\t\tthis.setGeometryAt( geometryId, geometry );\n\n\t\treturn geometryId;\n\n\t}\n\n\tsetGeometryAt( id, geometry ) {\n\n\t\tif ( id >= this._geometryCount ) {\n\n\t\t\tthrow new Error( 'BatchedMesh: Maximum geometry count reached.' );\n\n\t\t}\n\n\t\tthis._validateGeometry( geometry );\n\n\t\tconst batchGeometry = this.geometry;\n\t\tconst hasIndex = batchGeometry.getIndex() !== null;\n\t\tconst dstIndex = batchGeometry.getIndex();\n\t\tconst srcIndex = geometry.getIndex();\n\t\tconst reservedRange = this._reservedRanges[ id ];\n\t\tif (\n\t\t\thasIndex &&\n\t\t\tsrcIndex.count > reservedRange.indexCount ||\n\t\t\tgeometry.attributes.position.count > reservedRange.vertexCount\n\t\t) {\n\n\t\t\tthrow new Error( 'BatchedMesh: Reserved space not large enough for provided geometry.' );\n\n\t\t}\n\n\t\t// copy geometry over\n\t\tconst vertexStart = reservedRange.vertexStart;\n\t\tconst vertexCount = reservedRange.vertexCount;\n\t\tfor ( const attributeName in batchGeometry.attributes ) {\n\n\t\t\tif ( attributeName === ID_ATTR_NAME ) {\n\n\t\t\t\tcontinue;\n\n\t\t\t}\n\n\t\t\t// copy attribute data\n\t\t\tconst srcAttribute = geometry.getAttribute( attributeName );\n\t\t\tconst dstAttribute = batchGeometry.getAttribute( attributeName );\n\t\t\tcopyAttributeData( srcAttribute, dstAttribute, vertexStart );\n\n\t\t\t// fill the rest in with zeroes\n\t\t\tconst itemSize = srcAttribute.itemSize;\n\t\t\tfor ( let i = srcAttribute.count, l = vertexCount; i < l; i ++ ) {\n\n\t\t\t\tconst index = vertexStart + i;\n\t\t\t\tfor ( let c = 0; c < itemSize; c ++ ) {\n\n\t\t\t\t\tdstAttribute.setComponent( index, c, 0 );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tdstAttribute.needsUpdate = true;\n\t\t\tdstAttribute.addUpdateRange( vertexStart * itemSize, vertexCount * itemSize );\n\n\t\t}\n\n\t\t// copy index\n\t\tif ( hasIndex ) {\n\n\t\t\tconst indexStart = reservedRange.indexStart;\n\n\t\t\t// copy index data over\n\t\t\tfor ( let i = 0; i < srcIndex.count; i ++ ) {\n\n\t\t\t\tdstIndex.setX( indexStart + i, vertexStart + srcIndex.getX( i ) );\n\n\t\t\t}\n\n\t\t\t// fill the rest in with zeroes\n\t\t\tfor ( let i = srcIndex.count, l = reservedRange.indexCount; i < l; i ++ ) {\n\n\t\t\t\tdstIndex.setX( indexStart + i, vertexStart );\n\n\t\t\t}\n\n\t\t\tdstIndex.needsUpdate = true;\n\t\t\tdstIndex.addUpdateRange( indexStart, reservedRange.indexCount );\n\n\t\t}\n\n\t\t// store the bounding boxes\n\t\tconst bound = this._bounds[ id ];\n\t\tif ( geometry.boundingBox !== null ) {\n\n\t\t\tbound.box.copy( geometry.boundingBox );\n\t\t\tbound.boxInitialized = true;\n\n\t\t} else {\n\n\t\t\tbound.boxInitialized = false;\n\n\t\t}\n\n\t\tif ( geometry.boundingSphere !== null ) {\n\n\t\t\tbound.sphere.copy( geometry.boundingSphere );\n\t\t\tbound.sphereInitialized = true;\n\n\t\t} else {\n\n\t\t\tbound.sphereInitialized = false;\n\n\t\t}\n\n\t\t// set drawRange count\n\t\tconst drawRange = this._drawRanges[ id ];\n\t\tconst posAttr = geometry.getAttribute( 'position' );\n\t\tdrawRange.count = hasIndex ? srcIndex.count : posAttr.count;\n\t\tthis._visibilityChanged = true;\n\n\t\treturn id;\n\n\t}\n\n\tdeleteGeometry( geometryId ) {\n\n\t\t// Note: User needs to call optimize() afterward to pack the data.\n\n\t\tconst active = this._active;\n\t\tif ( geometryId >= active.length || active[ geometryId ] === false ) {\n\n\t\t\treturn this;\n\n\t\t}\n\n\t\tactive[ geometryId ] = false;\n\t\tthis._visibilityChanged = true;\n\n\t\treturn this;\n\n\t}\n\n\t// get bounding box and compute it if it doesn't exist\n\tgetBoundingBoxAt( id, target ) {\n\n\t\tconst active = this._active;\n\t\tif ( active[ id ] === false ) {\n\n\t\t\treturn null;\n\n\t\t}\n\n\t\t// compute bounding box\n\t\tconst bound = this._bounds[ id ];\n\t\tconst box = bound.box;\n\t\tconst geometry = this.geometry;\n\t\tif ( bound.boxInitialized === false ) {\n\n\t\t\tbox.makeEmpty();\n\n\t\t\tconst index = geometry.index;\n\t\t\tconst position = geometry.attributes.position;\n\t\t\tconst drawRange = this._drawRanges[ id ];\n\t\t\tfor ( let i = drawRange.start, l = drawRange.start + drawRange.count; i < l; i ++ ) {\n\n\t\t\t\tlet iv = i;\n\t\t\t\tif ( index ) {\n\n\t\t\t\t\tiv = index.getX( iv );\n\n\t\t\t\t}\n\n\t\t\t\tbox.expandByPoint( _vector.fromBufferAttribute( position, iv ) );\n\n\t\t\t}\n\n\t\t\tbound.boxInitialized = true;\n\n\t\t}\n\n\t\ttarget.copy( box );\n\t\treturn target;\n\n\t}\n\n\t// get bounding sphere and compute it if it doesn't exist\n\tgetBoundingSphereAt( id, target ) {\n\n\t\tconst active = this._active;\n\t\tif ( active[ id ] === false ) {\n\n\t\t\treturn null;\n\n\t\t}\n\n\t\t// compute bounding sphere\n\t\tconst bound = this._bounds[ id ];\n\t\tconst sphere = bound.sphere;\n\t\tconst geometry = this.geometry;\n\t\tif ( bound.sphereInitialized === false ) {\n\n\t\t\tsphere.makeEmpty();\n\n\t\t\tthis.getBoundingBoxAt( id, _box );\n\t\t\t_box.getCenter( sphere.center );\n\n\t\t\tconst index = geometry.index;\n\t\t\tconst position = geometry.attributes.position;\n\t\t\tconst drawRange = this._drawRanges[ id ];\n\n\t\t\tlet maxRadiusSq = 0;\n\t\t\tfor ( let i = drawRange.start, l = drawRange.start + drawRange.count; i < l; i ++ ) {\n\n\t\t\t\tlet iv = i;\n\t\t\t\tif ( index ) {\n\n\t\t\t\t\tiv = index.getX( iv );\n\n\t\t\t\t}\n\n\t\t\t\t_vector.fromBufferAttribute( position, iv );\n\t\t\t\tmaxRadiusSq = Math.max( maxRadiusSq, sphere.center.distanceToSquared( _vector ) );\n\n\t\t\t}\n\n\t\t\tsphere.radius = Math.sqrt( maxRadiusSq );\n\t\t\tbound.sphereInitialized = true;\n\n\t\t}\n\n\t\ttarget.copy( sphere );\n\t\treturn target;\n\n\t}\n\n\tsetMatrixAt( geometryId, matrix ) {\n\n\t\t// @TODO: Map geometryId to index of the arrays because\n\t\t//        optimize() can make geometryId mismatch the index\n\n\t\tconst active = this._active;\n\t\tconst matricesTexture = this._matricesTexture;\n\t\tconst matricesArray = this._matricesTexture.image.data;\n\t\tconst geometryCount = this._geometryCount;\n\t\tif ( geometryId >= geometryCount || active[ geometryId ] === false ) {\n\n\t\t\treturn this;\n\n\t\t}\n\n\t\tmatrix.toArray( matricesArray, geometryId * 16 );\n\t\tmatricesTexture.needsUpdate = true;\n\n\t\treturn this;\n\n\t}\n\n\tgetMatrixAt( geometryId, matrix ) {\n\n\t\tconst active = this._active;\n\t\tconst matricesArray = this._matricesTexture.image.data;\n\t\tconst geometryCount = this._geometryCount;\n\t\tif ( geometryId >= geometryCount || active[ geometryId ] === false ) {\n\n\t\t\treturn null;\n\n\t\t}\n\n\t\treturn matrix.fromArray( matricesArray, geometryId * 16 );\n\n\t}\n\n\tsetVisibleAt( geometryId, value ) {\n\n\t\tconst visibility = this._visibility;\n\t\tconst active = this._active;\n\t\tconst geometryCount = this._geometryCount;\n\n\t\t// if the geometry is out of range, not active, or visibility state\n\t\t// does not change then return early\n\t\tif (\n\t\t\tgeometryId >= geometryCount ||\n\t\t\tactive[ geometryId ] === false ||\n\t\t\tvisibility[ geometryId ] === value\n\t\t) {\n\n\t\t\treturn this;\n\n\t\t}\n\n\t\tvisibility[ geometryId ] = value;\n\t\tthis._visibilityChanged = true;\n\n\t\treturn this;\n\n\t}\n\n\tgetVisibleAt( geometryId ) {\n\n\t\tconst visibility = this._visibility;\n\t\tconst active = this._active;\n\t\tconst geometryCount = this._geometryCount;\n\n\t\t// return early if the geometry is out of range or not active\n\t\tif ( geometryId >= geometryCount || active[ geometryId ] === false ) {\n\n\t\t\treturn false;\n\n\t\t}\n\n\t\treturn visibility[ geometryId ];\n\n\t}\n\n\traycast( raycaster, intersects ) {\n\n\t\tconst visibility = this._visibility;\n\t\tconst active = this._active;\n\t\tconst drawRanges = this._drawRanges;\n\t\tconst geometryCount = this._geometryCount;\n\t\tconst matrixWorld = this.matrixWorld;\n\t\tconst batchGeometry = this.geometry;\n\n\t\t// iterate over each geometry\n\t\t_mesh.material = this.material;\n\t\t_mesh.geometry.index = batchGeometry.index;\n\t\t_mesh.geometry.attributes = batchGeometry.attributes;\n\t\tif ( _mesh.geometry.boundingBox === null ) {\n\n\t\t\t_mesh.geometry.boundingBox = new Box3();\n\n\t\t}\n\n\t\tif ( _mesh.geometry.boundingSphere === null ) {\n\n\t\t\t_mesh.geometry.boundingSphere = new Sphere();\n\n\t\t}\n\n\t\tfor ( let i = 0; i < geometryCount; i ++ ) {\n\n\t\t\tif ( ! visibility[ i ] || ! active[ i ] ) {\n\n\t\t\t\tcontinue;\n\n\t\t\t}\n\n\t\t\tconst drawRange = drawRanges[ i ];\n\t\t\t_mesh.geometry.setDrawRange( drawRange.start, drawRange.count );\n\n\t\t\t// ge the intersects\n\t\t\tthis.getMatrixAt( i, _mesh.matrixWorld ).premultiply( matrixWorld );\n\t\t\tthis.getBoundingBoxAt( i, _mesh.geometry.boundingBox );\n\t\t\tthis.getBoundingSphereAt( i, _mesh.geometry.boundingSphere );\n\t\t\t_mesh.raycast( raycaster, _batchIntersects );\n\n\t\t\t// add batch id to the intersects\n\t\t\tfor ( let j = 0, l = _batchIntersects.length; j < l; j ++ ) {\n\n\t\t\t\tconst intersect = _batchIntersects[ j ];\n\t\t\t\tintersect.object = this;\n\t\t\t\tintersect.batchId = i;\n\t\t\t\tintersects.push( intersect );\n\n\t\t\t}\n\n\t\t\t_batchIntersects.length = 0;\n\n\t\t}\n\n\t\t_mesh.material = null;\n\t\t_mesh.geometry.index = null;\n\t\t_mesh.geometry.attributes = {};\n\t\t_mesh.geometry.setDrawRange( 0, Infinity );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.geometry = source.geometry.clone();\n\t\tthis.perObjectFrustumCulled = source.perObjectFrustumCulled;\n\t\tthis.sortObjects = source.sortObjects;\n\t\tthis.boundingBox = source.boundingBox !== null ? source.boundingBox.clone() : null;\n\t\tthis.boundingSphere = source.boundingSphere !== null ? source.boundingSphere.clone() : null;\n\n\t\tthis._drawRanges = source._drawRanges.map( range => ( { ...range } ) );\n\t\tthis._reservedRanges = source._reservedRanges.map( range => ( { ...range } ) );\n\n\t\tthis._visibility = source._visibility.slice();\n\t\tthis._active = source._active.slice();\n\t\tthis._bounds = source._bounds.map( bound => ( {\n\t\t\tboxInitialized: bound.boxInitialized,\n\t\t\tbox: bound.box.clone(),\n\n\t\t\tsphereInitialized: bound.sphereInitialized,\n\t\t\tsphere: bound.sphere.clone()\n\t\t} ) );\n\n\t\tthis._maxGeometryCount = source._maxGeometryCount;\n\t\tthis._maxVertexCount = source._maxVertexCount;\n\t\tthis._maxIndexCount = source._maxIndexCount;\n\n\t\tthis._geometryInitialized = source._geometryInitialized;\n\t\tthis._geometryCount = source._geometryCount;\n\t\tthis._multiDrawCounts = source._multiDrawCounts.slice();\n\t\tthis._multiDrawStarts = source._multiDrawStarts.slice();\n\n\t\tthis._matricesTexture = source._matricesTexture.clone();\n\t\tthis._matricesTexture.image.data = this._matricesTexture.image.slice();\n\n\t\treturn this;\n\n\t}\n\n\tdispose() {\n\n\t\t// Assuming the geometry is not shared with other meshes\n\t\tthis.geometry.dispose();\n\n\t\tthis._matricesTexture.dispose();\n\t\tthis._matricesTexture = null;\n\t\treturn this;\n\n\t}\n\n\tonBeforeRender( renderer, scene, camera, geometry, material/*, _group*/ ) {\n\n\t\t// if visibility has not changed and frustum culling and object sorting is not required\n\t\t// then skip iterating over all items\n\t\tif ( ! this._visibilityChanged && ! this.perObjectFrustumCulled && ! this.sortObjects ) {\n\n\t\t\treturn;\n\n\t\t}\n\n\t\t// the indexed version of the multi draw function requires specifying the start\n\t\t// offset in bytes.\n\t\tconst index = geometry.getIndex();\n\t\tconst bytesPerElement = index === null ? 1 : index.array.BYTES_PER_ELEMENT;\n\n\t\tconst active = this._active;\n\t\tconst visibility = this._visibility;\n\t\tconst multiDrawStarts = this._multiDrawStarts;\n\t\tconst multiDrawCounts = this._multiDrawCounts;\n\t\tconst drawRanges = this._drawRanges;\n\t\tconst perObjectFrustumCulled = this.perObjectFrustumCulled;\n\n\t\t// prepare the frustum in the local frame\n\t\tif ( perObjectFrustumCulled ) {\n\n\t\t\t_projScreenMatrix\n\t\t\t\t.multiplyMatrices( camera.projectionMatrix, camera.matrixWorldInverse )\n\t\t\t\t.multiply( this.matrixWorld );\n\t\t\t_frustum.setFromProjectionMatrix(\n\t\t\t\t_projScreenMatrix,\n\t\t\t\trenderer.coordinateSystem\n\t\t\t);\n\n\t\t}\n\n\t\tlet count = 0;\n\t\tif ( this.sortObjects ) {\n\n\t\t\t// get the camera position in the local frame\n\t\t\t_invMatrixWorld.copy( this.matrixWorld ).invert();\n\t\t\t_vector.setFromMatrixPosition( camera.matrixWorld ).applyMatrix4( _invMatrixWorld );\n\n\t\t\tfor ( let i = 0, l = visibility.length; i < l; i ++ ) {\n\n\t\t\t\tif ( visibility[ i ] && active[ i ] ) {\n\n\t\t\t\t\t// get the bounds in world space\n\t\t\t\t\tthis.getMatrixAt( i, _matrix );\n\t\t\t\t\tthis.getBoundingSphereAt( i, _sphere ).applyMatrix4( _matrix );\n\n\t\t\t\t\t// determine whether the batched geometry is within the frustum\n\t\t\t\t\tlet culled = false;\n\t\t\t\t\tif ( perObjectFrustumCulled ) {\n\n\t\t\t\t\t\tculled = ! _frustum.intersectsSphere( _sphere );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( ! culled ) {\n\n\t\t\t\t\t\t// get the distance from camera used for sorting\n\t\t\t\t\t\tconst z = _vector.distanceTo( _sphere.center );\n\t\t\t\t\t\t_renderList.push( drawRanges[ i ], z );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// Sort the draw ranges and prep for rendering\n\t\t\tconst list = _renderList.list;\n\t\t\tconst customSort = this.customSort;\n\t\t\tif ( customSort === null ) {\n\n\t\t\t\tlist.sort( material.transparent ? sortTransparent : sortOpaque );\n\n\t\t\t} else {\n\n\t\t\t\tcustomSort.call( this, list, camera );\n\n\t\t\t}\n\n\t\t\tfor ( let i = 0, l = list.length; i < l; i ++ ) {\n\n\t\t\t\tconst item = list[ i ];\n\t\t\t\tmultiDrawStarts[ count ] = item.start * bytesPerElement;\n\t\t\t\tmultiDrawCounts[ count ] = item.count;\n\t\t\t\tcount ++;\n\n\t\t\t}\n\n\t\t\t_renderList.reset();\n\n\t\t} else {\n\n\t\t\tfor ( let i = 0, l = visibility.length; i < l; i ++ ) {\n\n\t\t\t\tif ( visibility[ i ] && active[ i ] ) {\n\n\t\t\t\t\t// determine whether the batched geometry is within the frustum\n\t\t\t\t\tlet culled = false;\n\t\t\t\t\tif ( perObjectFrustumCulled ) {\n\n\t\t\t\t\t\t// get the bounds in world space\n\t\t\t\t\t\tthis.getMatrixAt( i, _matrix );\n\t\t\t\t\t\tthis.getBoundingSphereAt( i, _sphere ).applyMatrix4( _matrix );\n\t\t\t\t\t\tculled = ! _frustum.intersectsSphere( _sphere );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( ! culled ) {\n\n\t\t\t\t\t\tconst range = drawRanges[ i ];\n\t\t\t\t\t\tmultiDrawStarts[ count ] = range.start * bytesPerElement;\n\t\t\t\t\t\tmultiDrawCounts[ count ] = range.count;\n\t\t\t\t\t\tcount ++;\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\tthis._multiDrawCount = count;\n\t\tthis._visibilityChanged = false;\n\n\t}\n\n\tonBeforeShadow( renderer, object, camera, shadowCamera, geometry, depthMaterial/* , group */ ) {\n\n\t\tthis.onBeforeRender( renderer, null, shadowCamera, geometry, depthMaterial );\n\n\t}\n\n}\n\nexport { BatchedMesh };\n\n// Path: src/core/UniformsGroup.js\nimport { EventDispatcher } from './EventDispatcher.js';\nimport { StaticDrawUsage } from '../constants.js';\n\nlet _id = 0;\n\nclass UniformsGroup extends EventDispatcher {\n\n\tconstructor() {\n\n\t\tsuper();\n\n\t\tthis.isUniformsGroup = true;\n\n\t\tObject.defineProperty( this, 'id', { value: _id ++ } );\n\n\t\tthis.name = '';\n\n\t\tthis.usage = StaticDrawUsage;\n\t\tthis.uniforms = [];\n\n\t}\n\n\tadd( uniform ) {\n\n\t\tthis.uniforms.push( uniform );\n\n\t\treturn this;\n\n\t}\n\n\tremove( uniform ) {\n\n\t\tconst index = this.uniforms.indexOf( uniform );\n\n\t\tif ( index !== - 1 ) this.uniforms.splice( index, 1 );\n\n\t\treturn this;\n\n\t}\n\n\tsetName( name ) {\n\n\t\tthis.name = name;\n\n\t\treturn this;\n\n\t}\n\n\tsetUsage( value ) {\n\n\t\tthis.usage = value;\n\n\t\treturn this;\n\n\t}\n\n\tdispose() {\n\n\t\tthis.dispatchEvent( { type: 'dispose' } );\n\n\t\treturn this;\n\n\t}\n\n\tcopy( source ) {\n\n\t\tthis.name = source.name;\n\t\tthis.usage = source.usage;\n\n\t\tconst uniformsSource = source.uniforms;\n\n\t\tthis.uniforms.length = 0;\n\n\t\tfor ( let i = 0, l = uniformsSource.length; i < l; i ++ ) {\n\n\t\t\tconst uniforms = Array.isArray( uniformsSource[ i ] ) ? uniformsSource[ i ] : [ uniformsSource[ i ] ];\n\n\t\t\tfor ( let j = 0; j < uniforms.length; j ++ ) {\n\n\t\t\t\tthis.uniforms.push( uniforms[ j ].clone() );\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tclone() {\n\n\t\treturn new this.constructor().copy( this );\n\n\t}\n\n}\n\nexport { UniformsGroup };\n\n// Path: src/cameras/CubeCamera.js\nimport { WebGLCoordinateSystem, WebGPUCoordinateSystem } from '../constants.js';\nimport { Object3D } from '../core/Object3D.js';\nimport { PerspectiveCamera } from './PerspectiveCamera.js';\n\nconst fov = - 90; // negative fov is not an error\nconst aspect = 1;\n\nclass CubeCamera extends Object3D {\n\n\tconstructor( near, far, renderTarget ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'CubeCamera';\n\n\t\tthis.renderTarget = renderTarget;\n\t\tthis.coordinateSystem = null;\n\t\tthis.activeMipmapLevel = 0;\n\n\t\tconst cameraPX = new PerspectiveCamera( fov, aspect, near, far );\n\t\tcameraPX.layers = this.layers;\n\t\tthis.add( cameraPX );\n\n\t\tconst cameraNX = new PerspectiveCamera( fov, aspect, near, far );\n\t\tcameraNX.layers = this.layers;\n\t\tthis.add( cameraNX );\n\n\t\tconst cameraPY = new PerspectiveCamera( fov, aspect, near, far );\n\t\tcameraPY.layers = this.layers;\n\t\tthis.add( cameraPY );\n\n\t\tconst cameraNY = new PerspectiveCamera( fov, aspect, near, far );\n\t\tcameraNY.layers = this.layers;\n\t\tthis.add( cameraNY );\n\n\t\tconst cameraPZ = new PerspectiveCamera( fov, aspect, near, far );\n\t\tcameraPZ.layers = this.layers;\n\t\tthis.add( cameraPZ );\n\n\t\tconst cameraNZ = new PerspectiveCamera( fov, aspect, near, far );\n\t\tcameraNZ.layers = this.layers;\n\t\tthis.add( cameraNZ );\n\n\t}\n\n\tupdateCoordinateSystem() {\n\n\t\tconst coordinateSystem = this.coordinateSystem;\n\n\t\tconst cameras = this.children.concat();\n\n\t\tconst [ cameraPX, cameraNX, cameraPY, cameraNY, cameraPZ, cameraNZ ] = cameras;\n\n\t\tfor ( const camera of cameras ) this.remove( camera );\n\n\t\tif ( coordinateSystem === WebGLCoordinateSystem ) {\n\n\t\t\tcameraPX.up.set( 0, 1, 0 );\n\t\t\tcameraPX.lookAt( 1, 0, 0 );\n\n\t\t\tcameraNX.up.set( 0, 1, 0 );\n\t\t\tcameraNX.lookAt( - 1, 0, 0 );\n\n\t\t\tcameraPY.up.set( 0, 0, - 1 );\n\t\t\tcameraPY.lookAt( 0, 1, 0 );\n\n\t\t\tcameraNY.up.set( 0, 0, 1 );\n\t\t\tcameraNY.lookAt( 0, - 1, 0 );\n\n\t\t\tcameraPZ.up.set( 0, 1, 0 );\n\t\t\tcameraPZ.lookAt( 0, 0, 1 );\n\n\t\t\tcameraNZ.up.set( 0, 1, 0 );\n\t\t\tcameraNZ.lookAt( 0, 0, - 1 );\n\n\t\t} else if ( coordinateSystem === WebGPUCoordinateSystem ) {\n\n\t\t\tcameraPX.up.set( 0, - 1, 0 );\n\t\t\tcameraPX.lookAt( - 1, 0, 0 );\n\n\t\t\tcameraNX.up.set( 0, - 1, 0 );\n\t\t\tcameraNX.lookAt( 1, 0, 0 );\n\n\t\t\tcameraPY.up.set( 0, 0, 1 );\n\t\t\tcameraPY.lookAt( 0, 1, 0 );\n\n\t\t\tcameraNY.up.set( 0, 0, - 1 );\n\t\t\tcameraNY.lookAt( 0, - 1, 0 );\n\n\t\t\tcameraPZ.up.set( 0, - 1, 0 );\n\t\t\tcameraPZ.lookAt( 0, 0, 1 );\n\n\t\t\tcameraNZ.up.set( 0, - 1, 0 );\n\t\t\tcameraNZ.lookAt( 0, 0, - 1 );\n\n\t\t} else {\n\n\t\t\tthrow new Error( 'THREE.CubeCamera.updateCoordinateSystem(): Invalid coordinate system: ' + coordinateSystem );\n\n\t\t}\n\n\t\tfor ( const camera of cameras ) {\n\n\t\t\tthis.add( camera );\n\n\t\t\tcamera.updateMatrixWorld();\n\n\t\t}\n\n\t}\n\n\tupdate( renderer, scene ) {\n\n\t\tif ( this.parent === null ) this.updateMatrixWorld();\n\n\t\tconst { renderTarget, activeMipmapLevel } = this;\n\n\t\tif ( this.coordinateSystem !== renderer.coordinateSystem ) {\n\n\t\t\tthis.coordinateSystem = renderer.coordinateSystem;\n\n\t\t\tthis.updateCoordinateSystem();\n\n\t\t}\n\n\t\tconst [ cameraPX, cameraNX, cameraPY, cameraNY, cameraPZ, cameraNZ ] = this.children;\n\n\t\tconst currentRenderTarget = renderer.getRenderTarget();\n\t\tconst currentActiveCubeFace = renderer.getActiveCubeFace();\n\t\tconst currentActiveMipmapLevel = renderer.getActiveMipmapLevel();\n\n\t\tconst currentXrEnabled = renderer.xr.enabled;\n\n\t\trenderer.xr.enabled = false;\n\n\t\tconst generateMipmaps = renderTarget.texture.generateMipmaps;\n\n\t\trenderTarget.texture.generateMipmaps = false;\n\n\t\trenderer.setRenderTarget( renderTarget, 0, activeMipmapLevel );\n\t\trenderer.render( scene, cameraPX );\n\n\t\trenderer.setRenderTarget( renderTarget, 1, activeMipmapLevel );\n\t\trenderer.render( scene, cameraNX );\n\n\t\trenderer.setRenderTarget( renderTarget, 2, activeMipmapLevel );\n\t\trenderer.render( scene, cameraPY );\n\n\t\trenderer.setRenderTarget( renderTarget, 3, activeMipmapLevel );\n\t\trenderer.render( scene, cameraNY );\n\n\t\trenderer.setRenderTarget( renderTarget, 4, activeMipmapLevel );\n\t\trenderer.render( scene, cameraPZ );\n\n\t\t// mipmaps are generated during the last call of render()\n\t\t// at this point, all sides of the cube render target are defined\n\n\t\trenderTarget.texture.generateMipmaps = generateMipmaps;\n\n\t\trenderer.setRenderTarget( renderTarget, 5, activeMipmapLevel );\n\t\trenderer.render( scene, cameraNZ );\n\n\t\trenderer.setRenderTarget( currentRenderTarget, currentActiveCubeFace, currentActiveMipmapLevel );\n\n\t\trenderer.xr.enabled = currentXrEnabled;\n\n\t\trenderTarget.texture.needsPMREMUpdate = true;\n\n\t}\n\n}\n\nexport { CubeCamera };\n\n// Path: src/core/InterleavedBufferAttribute.js\nimport { Vector3 } from '../math/Vector3.js';\nimport { BufferAttribute } from './BufferAttribute.js';\nimport { denormalize, normalize } from '../math/MathUtils.js';\n\nconst _vector = /*@__PURE__*/ new Vector3();\n\nclass InterleavedBufferAttribute {\n\n\tconstructor( interleavedBuffer, itemSize, offset, normalized = false ) {\n\n\t\tthis.isInterleavedBufferAttribute = true;\n\n\t\tthis.name = '';\n\n\t\tthis.data = interleavedBuffer;\n\t\tthis.itemSize = itemSize;\n\t\tthis.offset = offset;\n\n\t\tthis.normalized = normalized;\n\n\t}\n\n\tget count() {\n\n\t\treturn this.data.count;\n\n\t}\n\n\tget array() {\n\n\t\treturn this.data.array;\n\n\t}\n\n\tset needsUpdate( value ) {\n\n\t\tthis.data.needsUpdate = value;\n\n\t}\n\n\tapplyMatrix4( m ) {\n\n\t\tfor ( let i = 0, l = this.data.count; i < l; i ++ ) {\n\n\t\t\t_vector.fromBufferAttribute( this, i );\n\n\t\t\t_vector.applyMatrix4( m );\n\n\t\t\tthis.setXYZ( i, _vector.x, _vector.y, _vector.z );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tapplyNormalMatrix( m ) {\n\n\t\tfor ( let i = 0, l = this.count; i < l; i ++ ) {\n\n\t\t\t_vector.fromBufferAttribute( this, i );\n\n\t\t\t_vector.applyNormalMatrix( m );\n\n\t\t\tthis.setXYZ( i, _vector.x, _vector.y, _vector.z );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\ttransformDirection( m ) {\n\n\t\tfor ( let i = 0, l = this.count; i < l; i ++ ) {\n\n\t\t\t_vector.fromBufferAttribute( this, i );\n\n\t\t\t_vector.transformDirection( m );\n\n\t\t\tthis.setXYZ( i, _vector.x, _vector.y, _vector.z );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tgetComponent( index, component ) {\n\n\t\tlet value = this.array[ index * this.data.stride + this.offset + component ];\n\n\t\tif ( this.normalized ) value = denormalize( value, this.array );\n\n\t\treturn value;\n\n\t}\n\n\tsetComponent( index, component, value ) {\n\n\t\tif ( this.normalized ) value = normalize( value, this.array );\n\n\t\tthis.data.array[ index * this.data.stride + this.offset + component ] = value;\n\n\t\treturn this;\n\n\t}\n\n\tsetX( index, x ) {\n\n\t\tif ( this.normalized ) x = normalize( x, this.array );\n\n\t\tthis.data.array[ index * this.data.stride + this.offset ] = x;\n\n\t\treturn this;\n\n\t}\n\n\tsetY( index, y ) {\n\n\t\tif ( this.normalized ) y = normalize( y, this.array );\n\n\t\tthis.data.array[ index * this.data.stride + this.offset + 1 ] = y;\n\n\t\treturn this;\n\n\t}\n\n\tsetZ( index, z ) {\n\n\t\tif ( this.normalized ) z = normalize( z, this.array );\n\n\t\tthis.data.array[ index * this.data.stride + this.offset + 2 ] = z;\n\n\t\treturn this;\n\n\t}\n\n\tsetW( index, w ) {\n\n\t\tif ( this.normalized ) w = normalize( w, this.array );\n\n\t\tthis.data.array[ index * this.data.stride + this.offset + 3 ] = w;\n\n\t\treturn this;\n\n\t}\n\n\tgetX( index ) {\n\n\t\tlet x = this.data.array[ index * this.data.stride + this.offset ];\n\n\t\tif ( this.normalized ) x = denormalize( x, this.array );\n\n\t\treturn x;\n\n\t}\n\n\tgetY( index ) {\n\n\t\tlet y = this.data.array[ index * this.data.stride + this.offset + 1 ];\n\n\t\tif ( this.normalized ) y = denormalize( y, this.array );\n\n\t\treturn y;\n\n\t}\n\n\tgetZ( index ) {\n\n\t\tlet z = this.data.array[ index * this.data.stride + this.offset + 2 ];\n\n\t\tif ( this.normalized ) z = denormalize( z, this.array );\n\n\t\treturn z;\n\n\t}\n\n\tgetW( index ) {\n\n\t\tlet w = this.data.array[ index * this.data.stride + this.offset + 3 ];\n\n\t\tif ( this.normalized ) w = denormalize( w, this.array );\n\n\t\treturn w;\n\n\t}\n\n\tsetXY( index, x, y ) {\n\n\t\tindex = index * this.data.stride + this.offset;\n\n\t\tif ( this.normalized ) {\n\n\t\t\tx = normalize( x, this.array );\n\t\t\ty = normalize( y, this.array );\n\n\t\t}\n\n\t\tthis.data.array[ index + 0 ] = x;\n\t\tthis.data.array[ index + 1 ] = y;\n\n\t\treturn this;\n\n\t}\n\n\tsetXYZ( index, x, y, z ) {\n\n...\n", "needle_token_start": 2186, "needle_token_end": 2212, "code_context_ntokens": 14684, "output": ["```javascript\nfunction sortOpaque( a, b ) {\n\n\treturn a.z - b.z;\n\n}\n```"]}
{"repo": "mrdoob/three.js", "name": "transformVertex", "language": "typescript", "path": "src/objects/Sprite.js", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to adjust the position of a vertex based on its initial position, a transformation matrix, and optional rotation parameters. It is used to correctly position and orient vertices in 3D space relative to a camera's view.\n2. **Input**: The inputs include the initial vertex position, a model-view position, a center point for alignment, a scaling factor, and optional sine and cosine values for rotation.\n3. **Output**: There is no direct output returned by the function; however, it modifies the vertex position in place, updating it to reflect its new position in world space after applying the transformations and rotations.\n4. **Procedure**: The function first aligns the vertex relative to a center point and applies scaling. If a rotation is specified (non-zero), it rotates the vertex around the center. The vertex's position is then adjusted by the model-view position and finally transformed to world space using a transformation matrix.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/objects/Skeleton.js\nimport {\n\tRGBAFormat,\n\tFloatType\n} from '../constants.js';\nimport { Bone } from './Bone.js';\nimport { Matrix4 } from '../math/Matrix4.js';\nimport { DataTexture } from '../textures/DataTexture.js';\nimport * as MathUtils from '../math/MathUtils.js';\n\nconst _offsetMatrix = /*@__PURE__*/ new Matrix4();\nconst _identityMatrix = /*@__PURE__*/ new Matrix4();\n\nclass Skeleton {\n\n\tconstructor( bones = [], boneInverses = [] ) {\n\n\t\tthis.uuid = MathUtils.generateUUID();\n\n\t\tthis.bones = bones.slice( 0 );\n\t\tthis.boneInverses = boneInverses;\n\t\tthis.boneMatrices = null;\n\n\t\tthis.boneTexture = null;\n\n\t\tthis.init();\n\n\t}\n\n\tinit() {\n\n\t\tconst bones = this.bones;\n\t\tconst boneInverses = this.boneInverses;\n\n\t\tthis.boneMatrices = new Float32Array( bones.length * 16 );\n\n\t\t// calculate inverse bone matrices if necessary\n\n\t\tif ( boneInverses.length === 0 ) {\n\n\t\t\tthis.calculateInverses();\n\n\t\t} else {\n\n\t\t\t// handle special case\n\n\t\t\tif ( bones.length !== boneInverses.length ) {\n\n\t\t\t\tconsole.warn( 'THREE.Skeleton: Number of inverse bone matrices does not match amount of bones.' );\n\n\t\t\t\tthis.boneInverses = [];\n\n\t\t\t\tfor ( let i = 0, il = this.bones.length; i < il; i ++ ) {\n\n\t\t\t\t\tthis.boneInverses.push( new Matrix4() );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tcalculateInverses() {\n\n\t\tthis.boneInverses.length = 0;\n\n\t\tfor ( let i = 0, il = this.bones.length; i < il; i ++ ) {\n\n\t\t\tconst inverse = new Matrix4();\n\n\t\t\tif ( this.bones[ i ] ) {\n\n\t\t\t\tinverse.copy( this.bones[ i ].matrixWorld ).invert();\n\n\t\t\t}\n\n\t\t\tthis.boneInverses.push( inverse );\n\n\t\t}\n\n\t}\n\n\tpose() {\n\n\t\t// recover the bind-time world matrices\n\n\t\tfor ( let i = 0, il = this.bones.length; i < il; i ++ ) {\n\n\t\t\tconst bone = this.bones[ i ];\n\n\t\t\tif ( bone ) {\n\n\t\t\t\tbone.matrixWorld.copy( this.boneInverses[ i ] ).invert();\n\n\t\t\t}\n\n\t\t}\n\n\t\t// compute the local matrices, positions, rotations and scales\n\n\t\tfor ( let i = 0, il = this.bones.length; i < il; i ++ ) {\n\n\t\t\tconst bone = this.bones[ i ];\n\n\t\t\tif ( bone ) {\n\n\t\t\t\tif ( bone.parent && bone.parent.isBone ) {\n\n\t\t\t\t\tbone.matrix.copy( bone.parent.matrixWorld ).invert();\n\t\t\t\t\tbone.matrix.multiply( bone.matrixWorld );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tbone.matrix.copy( bone.matrixWorld );\n\n\t\t\t\t}\n\n\t\t\t\tbone.matrix.decompose( bone.position, bone.quaternion, bone.scale );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tupdate() {\n\n\t\tconst bones = this.bones;\n\t\tconst boneInverses = this.boneInverses;\n\t\tconst boneMatrices = this.boneMatrices;\n\t\tconst boneTexture = this.boneTexture;\n\n\t\t// flatten bone matrices to array\n\n\t\tfor ( let i = 0, il = bones.length; i < il; i ++ ) {\n\n\t\t\t// compute the offset between the current and the original transform\n\n\t\t\tconst matrix = bones[ i ] ? bones[ i ].matrixWorld : _identityMatrix;\n\n\t\t\t_offsetMatrix.multiplyMatrices( matrix, boneInverses[ i ] );\n\t\t\t_offsetMatrix.toArray( boneMatrices, i * 16 );\n\n\t\t}\n\n\t\tif ( boneTexture !== null ) {\n\n\t\t\tboneTexture.needsUpdate = true;\n\n\t\t}\n\n\t}\n\n\tclone() {\n\n\t\treturn new Skeleton( this.bones, this.boneInverses );\n\n\t}\n\n\tcomputeBoneTexture() {\n\n\t\t// layout (1 matrix = 4 pixels)\n\t\t//      RGBA RGBA RGBA RGBA (=> column1, column2, column3, column4)\n\t\t//  with  8x8  pixel texture max   16 bones * 4 pixels =  (8 * 8)\n\t\t//       16x16 pixel texture max   64 bones * 4 pixels = (16 * 16)\n\t\t//       32x32 pixel texture max  256 bones * 4 pixels = (32 * 32)\n\t\t//       64x64 pixel texture max 1024 bones * 4 pixels = (64 * 64)\n\n\t\tlet size = Math.sqrt( this.bones.length * 4 ); // 4 pixels needed for 1 matrix\n\t\tsize = Math.ceil( size / 4 ) * 4;\n\t\tsize = Math.max( size, 4 );\n\n\t\tconst boneMatrices = new Float32Array( size * size * 4 ); // 4 floats per RGBA pixel\n\t\tboneMatrices.set( this.boneMatrices ); // copy current values\n\n\t\tconst boneTexture = new DataTexture( boneMatrices, size, size, RGBAFormat, FloatType );\n\t\tboneTexture.needsUpdate = true;\n\n\t\tthis.boneMatrices = boneMatrices;\n\t\tthis.boneTexture = boneTexture;\n\n\t\treturn this;\n\n\t}\n\n\tgetBoneByName( name ) {\n\n\t\tfor ( let i = 0, il = this.bones.length; i < il; i ++ ) {\n\n\t\t\tconst bone = this.bones[ i ];\n\n\t\t\tif ( bone.name === name ) {\n\n\t\t\t\treturn bone;\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn undefined;\n\n\t}\n\n\tdispose( ) {\n\n...\n// Path: src/scenes/Fog.js\nimport { Color } from '../math/Color.js';\n\nclass Fog {\n\n\tconstructor( color, near = 1, far = 1000 ) {\n\n\t\tthis.isFog = true;\n\n\t\tthis.name = '';\n\n\t\tthis.color = new Color( color );\n\n\t\tthis.near = near;\n\t\tthis.far = far;\n\n\t}\n\n\tclone() {\n\n\t\treturn new Fog( this.color, this.near, this.far );\n\n\t}\n\n\ttoJSON( /* meta */ ) {\n\n\t\treturn {\n\t\t\ttype: 'Fog',\n\t\t\tname: this.name,\n\t\t\tcolor: this.color.getHex(),\n\t\t\tnear: this.near,\n\t\t\tfar: this.far\n\t\t};\n\n\t}\n\n}\n\nexport { Fog };\n\n// Path: src/objects/Sprite.js\nimport { Vector2 } from '../math/Vector2.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Matrix4 } from '../math/Matrix4.js';\nimport { Triangle } from '../math/Triangle.js';\nimport { Object3D } from '../core/Object3D.js';\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { InterleavedBuffer } from '../core/InterleavedBuffer.js';\nimport { InterleavedBufferAttribute } from '../core/InterleavedBufferAttribute.js';\nimport { SpriteMaterial } from '../materials/SpriteMaterial.js';\n\nlet _geometry;\n\nconst _intersectPoint = /*@__PURE__*/ new Vector3();\nconst _worldScale = /*@__PURE__*/ new Vector3();\nconst _mvPosition = /*@__PURE__*/ new Vector3();\n\nconst _alignedPosition = /*@__PURE__*/ new Vector2();\nconst _rotatedPosition = /*@__PURE__*/ new Vector2();\nconst _viewWorldMatrix = /*@__PURE__*/ new Matrix4();\n\nconst _vA = /*@__PURE__*/ new Vector3();\nconst _vB = /*@__PURE__*/ new Vector3();\nconst _vC = /*@__PURE__*/ new Vector3();\n\nconst _uvA = /*@__PURE__*/ new Vector2();\nconst _uvB = /*@__PURE__*/ new Vector2();\nconst _uvC = /*@__PURE__*/ new Vector2();\n\nclass Sprite extends Object3D {\n\n\tconstructor( material = new SpriteMaterial() ) {\n\n\t\tsuper();\n\n\t\tthis.isSprite = true;\n\n\t\tthis.type = 'Sprite';\n\n\t\tif ( _geometry === undefined ) {\n\n\t\t\t_geometry = new BufferGeometry();\n\n\t\t\tconst float32Array = new Float32Array( [\n\t\t\t\t- 0.5, - 0.5, 0, 0, 0,\n\t\t\t\t0.5, - 0.5, 0, 1, 0,\n\t\t\t\t0.5, 0.5, 0, 1, 1,\n\t\t\t\t- 0.5, 0.5, 0, 0, 1\n\t\t\t] );\n\n\t\t\tconst interleavedBuffer = new InterleavedBuffer( float32Array, 5 );\n\n\t\t\t_geometry.setIndex( [ 0, 1, 2,\t0, 2, 3 ] );\n\t\t\t_geometry.setAttribute( 'position', new InterleavedBufferAttribute( interleavedBuffer, 3, 0, false ) );\n\t\t\t_geometry.setAttribute( 'uv', new InterleavedBufferAttribute( interleavedBuffer, 2, 3, false ) );\n\n\t\t}\n\n\t\tthis.geometry = _geometry;\n\t\tthis.material = material;\n\n\t\tthis.center = new Vector2( 0.5, 0.5 );\n\n\t}\n\n\traycast( raycaster, intersects ) {\n\n\t\tif ( raycaster.camera === null ) {\n\n\t\t\tconsole.error( 'THREE.Sprite: \"Raycaster.camera\" needs to be set in order to raycast against sprites.' );\n\n\t\t}\n\n\t\t_worldScale.setFromMatrixScale( this.matrixWorld );\n\n\t\t_viewWorldMatrix.copy( raycaster.camera.matrixWorld );\n\t\tthis.modelViewMatrix.multiplyMatrices( raycaster.camera.matrixWorldInverse, this.matrixWorld );\n\n\t\t_mvPosition.setFromMatrixPosition( this.modelViewMatrix );\n\n\t\tif ( raycaster.camera.isPerspectiveCamera && this.material.sizeAttenuation === false ) {\n\n\t\t\t_worldScale.multiplyScalar( - _mvPosition.z );\n\n\t\t}\n\n\t\tconst rotation = this.material.rotation;\n\t\tlet sin, cos;\n\n\t\tif ( rotation !== 0 ) {\n\n\t\t\tcos = Math.cos( rotation );\n\t\t\tsin = Math.sin( rotation );\n\n\t\t}\n\n\t\tconst center = this.center;\n\n\t\ttransformVertex( _vA.set( - 0.5, - 0.5, 0 ), _mvPosition, center, _worldScale, sin, cos );\n\t\ttransformVertex( _vB.set( 0.5, - 0.5, 0 ), _mvPosition, center, _worldScale, sin, cos );\n\t\ttransformVertex( _vC.set( 0.5, 0.5, 0 ), _mvPosition, center, _worldScale, sin, cos );\n\n\t\t_uvA.set( 0, 0 );\n\t\t_uvB.set( 1, 0 );\n\t\t_uvC.set( 1, 1 );\n\n\t\t// check first triangle\n\t\tlet intersect = raycaster.ray.intersectTriangle( _vA, _vB, _vC, false, _intersectPoint );\n\n\t\tif ( intersect === null ) {\n\n\t\t\t// check second triangle\n\t\t\ttransformVertex( _vB.set( - 0.5, 0.5, 0 ), _mvPosition, center, _worldScale, sin, cos );\n\t\t\t_uvB.set( 0, 1 );\n\n\t\t\tintersect = raycaster.ray.intersectTriangle( _vA, _vC, _vB, false, _intersectPoint );\n\t\t\tif ( intersect === null ) {\n\n\t\t\t\treturn;\n\n\t\t\t}\n\n\t\t}\n\n\t\tconst distance = raycaster.ray.origin.distanceTo( _intersectPoint );\n\n\t\tif ( distance < raycaster.near || distance > raycaster.far ) return;\n\n\t\tintersects.push( {\n\n\t\t\tdistance: distance,\n\t\t\tpoint: _intersectPoint.clone(),\n\t\t\tuv: Triangle.getInterpolation( _intersectPoint, _vA, _vB, _vC, _uvA, _uvB, _uvC, new Vector2() ),\n\t\t\tface: null,\n\t\t\tobject: this\n\n\t\t} );\n\n\t}\n\n\tcopy( source, recursive ) {\n\n\t\tsuper.copy( source, recursive );\n\n\t\tif ( source.center !== undefined ) this.center.copy( source.center );\n\n\t\tthis.material = source.material;\n\n\t\treturn this;\n\n\t}\n\n}\n\n\nfunction transformVertex( vertexPosition, mvPosition, center, scale, sin, cos ) {\n\n\t// compute position in camera space\n\t_alignedPosition.subVectors( vertexPosition, center ).addScalar( 0.5 ).multiply( scale );\n\n\t// to check if rotation is not zero\n\tif ( sin !== undefined ) {\n\n\t\t_rotatedPosition.x = ( cos * _alignedPosition.x ) - ( sin * _alignedPosition.y );\n\t\t_rotatedPosition.y = ( sin * _alignedPosition.x ) + ( cos * _alignedPosition.y );\n\n\t} else {\n\n\t\t_rotatedPosition.copy( _alignedPosition );\n\n\t}\n\n\n\tvertexPosition.copy( mvPosition );\n\tvertexPosition.x += _rotatedPosition.x;\n\tvertexPosition.y += _rotatedPosition.y;\n\n\t// transform to world space\n\tvertexPosition.applyMatrix4( _viewWorldMatrix );\n\n}\n\nexport { Sprite };\n\n// Path: src/lights/RectAreaLight.js\nimport { Light } from './Light.js';\n\nclass RectAreaLight extends Light {\n\n\tconstructor( color, intensity, width = 10, height = 10 ) {\n\n\t\tsuper( color, intensity );\n\n\t\tthis.isRectAreaLight = true;\n\n\t\tthis.type = 'RectAreaLight';\n\n\t\tthis.width = width;\n\t\tthis.height = height;\n\n\t}\n\n\tget power() {\n\n\t\t// compute the light's luminous power (in lumens) from its intensity (in nits)\n\t\treturn this.intensity * this.width * this.height * Math.PI;\n\n\t}\n\n\tset power( power ) {\n\n\t\t// set the light's intensity (in nits) from the desired luminous power (in lumens)\n\t\tthis.intensity = power / ( this.width * this.height * Math.PI );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.width = source.width;\n\t\tthis.height = source.height;\n\n\t\treturn this;\n\n\t}\n\n\ttoJSON( meta ) {\n\n\t\tconst data = super.toJSON( meta );\n\n\t\tdata.object.width = this.width;\n\t\tdata.object.height = this.height;\n\n\t\treturn data;\n\n\t}\n\n}\n\nexport { RectAreaLight };\n\n// Path: src/objects/LOD.js\nimport { Vector3 } from '../math/Vector3.js';\nimport { Object3D } from '../core/Object3D.js';\n\nconst _v1 = /*@__PURE__*/ new Vector3();\nconst _v2 = /*@__PURE__*/ new Vector3();\n\nclass LOD extends Object3D {\n\n\tconstructor() {\n\n\t\tsuper();\n\n\t\tthis._currentLevel = 0;\n\n\t\tthis.type = 'LOD';\n\n\t\tObject.defineProperties( this, {\n\t\t\tlevels: {\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: []\n\t\t\t},\n\t\t\tisLOD: {\n\t\t\t\tvalue: true,\n\t\t\t}\n\t\t} );\n\n\t\tthis.autoUpdate = true;\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source, false );\n\n\t\tconst levels = source.levels;\n\n\t\tfor ( let i = 0, l = levels.length; i < l; i ++ ) {\n\n\t\t\tconst level = levels[ i ];\n\n\t\t\tthis.addLevel( level.object.clone(), level.distance, level.hysteresis );\n\n\t\t}\n\n\t\tthis.autoUpdate = source.autoUpdate;\n\n\t\treturn this;\n\n\t}\n\n\taddLevel( object, distance = 0, hysteresis = 0 ) {\n\n\t\tdistance = Math.abs( distance );\n\n\t\tconst levels = this.levels;\n\n\t\tlet l;\n\n\t\tfor ( l = 0; l < levels.length; l ++ ) {\n\n\t\t\tif ( distance < levels[ l ].distance ) {\n\n\t\t\t\tbreak;\n\n\t\t\t}\n\n\t\t}\n\n\t\tlevels.splice( l, 0, { distance: distance, hysteresis: hysteresis, object: object } );\n\n\t\tthis.add( object );\n\n\t\treturn this;\n\n\t}\n\n\tgetCurrentLevel() {\n\n\t\treturn this._currentLevel;\n\n\t}\n\n\n\n\tgetObjectForDistance( distance ) {\n\n\t\tconst levels = this.levels;\n\n\t\tif ( levels.length > 0 ) {\n\n\t\t\tlet i, l;\n\n\t\t\tfor ( i = 1, l = levels.length; i < l; i ++ ) {\n\n\t\t\t\tlet levelDistance = levels[ i ].distance;\n\n\t\t\t\tif ( levels[ i ].object.visible ) {\n\n\t\t\t\t\tlevelDistance -= levelDistance * levels[ i ].hysteresis;\n\n\t\t\t\t}\n\n\t\t\t\tif ( distance < levelDistance ) {\n\n\t\t\t\t\tbreak;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\treturn levels[ i - 1 ].object;\n\n\t\t}\n\n\t\treturn null;\n\n\t}\n\n\traycast( raycaster, intersects ) {\n\n\t\tconst levels = this.levels;\n\n\t\tif ( levels.length > 0 ) {\n\n\t\t\t_v1.setFromMatrixPosition( this.matrixWorld );\n\n\t\t\tconst distance = raycaster.ray.origin.distanceTo( _v1 );\n\n\t\t\tthis.getObjectForDistance( distance ).raycast( raycaster, intersects );\n\n\t\t}\n\n\t}\n\n\tupdate( camera ) {\n\n\t\tconst levels = this.levels;\n\n\t\tif ( levels.length > 1 ) {\n\n\t\t\t_v1.setFromMatrixPosition( camera.matrixWorld );\n\t\t\t_v2.setFromMatrixPosition( this.matrixWorld );\n\n\t\t\tconst distance = _v1.distanceTo( _v2 ) / camera.zoom;\n\n\t\t\tlevels[ 0 ].object.visible = true;\n\n\t\t\tlet i, l;\n\n\t\t\tfor ( i = 1, l = levels.length; i < l; i ++ ) {\n\n\t\t\t\tlet levelDistance = levels[ i ].distance;\n\n\t\t\t\tif ( levels[ i ].object.visible ) {\n\n\t\t\t\t\tlevelDistance -= levelDistance * levels[ i ].hysteresis;\n\n\t\t\t\t}\n\n\t\t\t\tif ( distance >= levelDistance ) {\n\n\t\t\t\t\tlevels[ i - 1 ].object.visible = false;\n\t\t\t\t\tlevels[ i ].object.visible = true;\n\n\t\t\t\t} else {\n\n\t\t\t\t\tbreak;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tthis._currentLevel = i - 1;\n\n\t\t\tfor ( ; i < l; i ++ ) {\n\n\t\t\t\tlevels[ i ].object.visible = false;\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\ttoJSON( meta ) {\n\n\t\tconst data = super.toJSON( meta );\n\n\t\tif ( this.autoUpdate === false ) data.object.autoUpdate = false;\n\n\t\tdata.object.levels = [];\n\n\t\tconst levels = this.levels;\n\n\t\tfor ( let i = 0, l = levels.length; i < l; i ++ ) {\n\n\t\t\tconst level = levels[ i ];\n\n\t\t\tdata.object.levels.push( {\n\t\t\t\tobject: level.object.uuid,\n\t\t\t\tdistance: level.distance,\n\t\t\t\thysteresis: level.hysteresis\n\t\t\t} );\n\n\t\t}\n\n\t\treturn data;\n\n\t}\n\n}\n\n\nexport { LOD };\n\n// Path: src/scenes/Scene.js\nimport { Object3D } from '../core/Object3D.js';\nimport { Euler } from '../math/Euler.js';\n\nclass Scene extends Object3D {\n\n\tconstructor() {\n\n\t\tsuper();\n\n\t\tthis.isScene = true;\n\n\t\tthis.type = 'Scene';\n\n\t\tthis.background = null;\n\t\tthis.environment = null;\n\t\tthis.fog = null;\n\n\t\tthis.backgroundBlurriness = 0;\n\t\tthis.backgroundIntensity = 1;\n\t\tthis.backgroundRotation = new Euler();\n\n\t\tthis.environmentIntensity = 1;\n\t\tthis.environmentRotation = new Euler();\n\n\t\tthis.overrideMaterial = null;\n\n\t\tif ( typeof __THREE_DEVTOOLS__ !== 'undefined' ) {\n\n\t\t\t__THREE_DEVTOOLS__.dispatchEvent( new CustomEvent( 'observe', { detail: this } ) );\n\n\t\t}\n\n\t}\n\n\tcopy( source, recursive ) {\n\n\t\tsuper.copy( source, recursive );\n\n\t\tif ( source.background !== null ) this.background = source.background.clone();\n\t\tif ( source.environment !== null ) this.environment = source.environment.clone();\n\t\tif ( source.fog !== null ) this.fog = source.fog.clone();\n\n\t\tthis.backgroundBlurriness = source.backgroundBlurriness;\n\t\tthis.backgroundIntensity = source.backgroundIntensity;\n\t\tthis.backgroundRotation.copy( source.backgroundRotation );\n\n\t\tthis.environmentIntensity = source.environmentIntensity;\n\t\tthis.environmentRotation.copy( source.environmentRotation );\n\n\t\tif ( source.overrideMaterial !== null ) this.overrideMaterial = source.overrideMaterial.clone();\n\n\t\tthis.matrixAutoUpdate = source.matrixAutoUpdate;\n\n\t\treturn this;\n\n\t}\n\n\ttoJSON( meta ) {\n\n\t\tconst data = super.toJSON( meta );\n\n\t\tif ( this.fog !== null ) data.object.fog = this.fog.toJSON();\n\n\t\tif ( this.backgroundBlurriness > 0 ) data.object.backgroundBlurriness = this.backgroundBlurriness;\n\t\tif ( this.backgroundIntensity !== 1 ) data.object.backgroundIntensity = this.backgroundIntensity;\n\t\tdata.object.backgroundRotation = this.backgroundRotation.toArray();\n\n\t\tif ( this.environmentIntensity !== 1 ) data.object.environmentIntensity = this.environmentIntensity;\n\t\tdata.object.environmentRotation = this.environmentRotation.toArray();\n\n\t\treturn data;\n\n\t}\n\n}\n\nexport { Scene };\n\n// Path: src/math/SphericalHarmonics3.js\nimport { Vector3 } from './Vector3.js';\n\n/**\n * Primary reference:\n *   https://graphics.stanford.edu/papers/envmap/envmap.pdf\n *\n * Secondary reference:\n *   https://www.ppsloan.org/publications/StupidSH36.pdf\n */\n\n// 3-band SH defined by 9 coefficients\n\nclass SphericalHarmonics3 {\n\n\tconstructor() {\n\n\t\tthis.isSphericalHarmonics3 = true;\n\n\t\tthis.coefficients = [];\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tthis.coefficients.push( new Vector3() );\n\n\t\t}\n\n\t}\n\n\tset( coefficients ) {\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tthis.coefficients[ i ].copy( coefficients[ i ] );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tzero() {\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tthis.coefficients[ i ].set( 0, 0, 0 );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\t// get the radiance in the direction of the normal\n\t// target is a Vector3\n\tgetAt( normal, target ) {\n\n\t\t// normal is assumed to be unit length\n\n\t\tconst x = normal.x, y = normal.y, z = normal.z;\n\n\t\tconst coeff = this.coefficients;\n\n\t\t// band 0\n\t\ttarget.copy( coeff[ 0 ] ).multiplyScalar( 0.282095 );\n\n\t\t// band 1\n\t\ttarget.addScaledVector( coeff[ 1 ], 0.488603 * y );\n\t\ttarget.addScaledVector( coeff[ 2 ], 0.488603 * z );\n\t\ttarget.addScaledVector( coeff[ 3 ], 0.488603 * x );\n\n\t\t// band 2\n\t\ttarget.addScaledVector( coeff[ 4 ], 1.092548 * ( x * y ) );\n\t\ttarget.addScaledVector( coeff[ 5 ], 1.092548 * ( y * z ) );\n\t\ttarget.addScaledVector( coeff[ 6 ], 0.315392 * ( 3.0 * z * z - 1.0 ) );\n\t\ttarget.addScaledVector( coeff[ 7 ], 1.092548 * ( x * z ) );\n\t\ttarget.addScaledVector( coeff[ 8 ], 0.546274 * ( x * x - y * y ) );\n\n\t\treturn target;\n\n\t}\n\n\t// get the irradiance (radiance convolved with cosine lobe) in the direction of the normal\n\t// target is a Vector3\n\t// https://graphics.stanford.edu/papers/envmap/envmap.pdf\n\tgetIrradianceAt( normal, target ) {\n\n\t\t// normal is assumed to be unit length\n\n\t\tconst x = normal.x, y = normal.y, z = normal.z;\n\n\t\tconst coeff = this.coefficients;\n\n\t\t// band 0\n\t\ttarget.copy( coeff[ 0 ] ).multiplyScalar( 0.886227 ); // \u03c0 * 0.282095\n\n\t\t// band 1\n\t\ttarget.addScaledVector( coeff[ 1 ], 2.0 * 0.511664 * y ); // ( 2 * \u03c0 / 3 ) * 0.488603\n\t\ttarget.addScaledVector( coeff[ 2 ], 2.0 * 0.511664 * z );\n\t\ttarget.addScaledVector( coeff[ 3 ], 2.0 * 0.511664 * x );\n\n\t\t// band 2\n\t\ttarget.addScaledVector( coeff[ 4 ], 2.0 * 0.429043 * x * y ); // ( \u03c0 / 4 ) * 1.092548\n\t\ttarget.addScaledVector( coeff[ 5 ], 2.0 * 0.429043 * y * z );\n\t\ttarget.addScaledVector( coeff[ 6 ], 0.743125 * z * z - 0.247708 ); // ( \u03c0 / 4 ) * 0.315392 * 3\n\t\ttarget.addScaledVector( coeff[ 7 ], 2.0 * 0.429043 * x * z );\n\t\ttarget.addScaledVector( coeff[ 8 ], 0.429043 * ( x * x - y * y ) ); // ( \u03c0 / 4 ) * 0.546274\n\n\t\treturn target;\n\n\t}\n\n\tadd( sh ) {\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tthis.coefficients[ i ].add( sh.coefficients[ i ] );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\taddScaledSH( sh, s ) {\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tthis.coefficients[ i ].addScaledVector( sh.coefficients[ i ], s );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tscale( s ) {\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tthis.coefficients[ i ].multiplyScalar( s );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tlerp( sh, alpha ) {\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tthis.coefficients[ i ].lerp( sh.coefficients[ i ], alpha );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tequals( sh ) {\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tif ( ! this.coefficients[ i ].equals( sh.coefficients[ i ] ) ) {\n\n\t\t\t\treturn false;\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn true;\n\n\t}\n\n\tcopy( sh ) {\n\n\t\treturn this.set( sh.coefficients );\n\n\t}\n\n\tclone() {\n\n\t\treturn new this.constructor().copy( this );\n\n\t}\n\n\tfromArray( array, offset = 0 ) {\n\n\t\tconst coefficients = this.coefficients;\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tcoefficients[ i ].fromArray( array, offset + ( i * 3 ) );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\ttoArray( array = [], offset = 0 ) {\n\n\t\tconst coefficients = this.coefficients;\n\n\t\tfor ( let i = 0; i < 9; i ++ ) {\n\n\t\t\tcoefficients[ i ].toArray( array, offset + ( i * 3 ) );\n\n\t\t}\n\n\t\treturn array;\n\n\t}\n\n\t// evaluate the basis functions\n\t// shBasis is an Array[ 9 ]\n\tstatic getBasisAt( normal, shBasis ) {\n\n\t\t// normal is assumed to be unit length\n\n\t\tconst x = normal.x, y = normal.y, z = normal.z;\n\n\t\t// band 0\n\t\tshBasis[ 0 ] = 0.282095;\n\n\t\t// band 1\n\t\tshBasis[ 1 ] = 0.488603 * y;\n\t\tshBasis[ 2 ] = 0.488603 * z;\n\t\tshBasis[ 3 ] = 0.488603 * x;\n\n\t\t// band 2\n\t\tshBasis[ 4 ] = 1.092548 * x * y;\n\t\tshBasis[ 5 ] = 1.092548 * y * z;\n\t\tshBasis[ 6 ] = 0.315392 * ( 3 * z * z - 1 );\n\t\tshBasis[ 7 ] = 1.092548 * x * z;\n\t\tshBasis[ 8 ] = 0.546274 * ( x * x - y * y );\n\n\t}\n\n}\n\nexport { SphericalHarmonics3 };\n\n// Path: src/lights/LightProbe.js\nimport { SphericalHarmonics3 } from '../math/SphericalHarmonics3.js';\nimport { Light } from './Light.js';\n\nclass LightProbe extends Light {\n\n\tconstructor( sh = new SphericalHarmonics3(), intensity = 1 ) {\n\n\t\tsuper( undefined, intensity );\n\n\t\tthis.isLightProbe = true;\n\n\t\tthis.sh = sh;\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.sh.copy( source.sh );\n\n\t\treturn this;\n\n\t}\n\n\tfromJSON( json ) {\n\n\t\tthis.intensity = json.intensity; // TODO: Move this bit to Light.fromJSON();\n\t\tthis.sh.fromArray( json.sh );\n\n\t\treturn this;\n\n\t}\n\n\ttoJSON( meta ) {\n\n\t\tconst data = super.toJSON( meta );\n\n\t\tdata.object.sh = this.sh.toArray();\n\n\t\treturn data;\n\n\t}\n\n}\n\nexport { LightProbe };\n\n// Path: src/lights/HemisphereLight.js\nimport { Light } from './Light.js';\nimport { Color } from '../math/Color.js';\nimport { Object3D } from '../core/Object3D.js';\n\nclass HemisphereLight extends Light {\n\n\tconstructor( skyColor, groundColor, intensity ) {\n\n\t\tsuper( skyColor, intensity );\n\n\t\tthis.isHemisphereLight = true;\n\n\t\tthis.type = 'HemisphereLight';\n\n\t\tthis.position.copy( Object3D.DEFAULT_UP );\n\t\tthis.updateMatrix();\n\n\t\tthis.groundColor = new Color( groundColor );\n\n\t}\n\n\tcopy( source, recursive ) {\n\n\t\tsuper.copy( source, recursive );\n\n\t\tthis.groundColor.copy( source.groundColor );\n\n\t\treturn this;\n\n\t}\n\n}\n\nexport { HemisphereLight };\n\n// Path: src/core/InstancedBufferAttribute.js\nimport { BufferAttribute } from './BufferAttribute.js';\n\nclass InstancedBufferAttribute extends BufferAttribute {\n\n\tconstructor( array, itemSize, normalized, meshPerAttribute = 1 ) {\n\n\t\tsuper( array, itemSize, normalized );\n\n\t\tthis.isInstancedBufferAttribute = true;\n\n\t\tthis.meshPerAttribute = meshPerAttribute;\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.meshPerAttribute = source.meshPerAttribute;\n\n\t\treturn this;\n\n\t}\n\n\ttoJSON() {\n\n\t\tconst data = super.toJSON();\n\n\t\tdata.meshPerAttribute = this.meshPerAttribute;\n\n\t\tdata.isInstancedBufferAttribute = true;\n\n\t\treturn data;\n\n\t}\n\n}\n\nexport { InstancedBufferAttribute };\n\n// Path: src/objects/InstancedMesh.js\nimport { InstancedBufferAttribute } from '../core/InstancedBufferAttribute.js';\nimport { Mesh } from './Mesh.js';\nimport { Box3 } from '../math/Box3.js';\nimport { Matrix4 } from '../math/Matrix4.js';\nimport { Sphere } from '../math/Sphere.js';\nimport { DataTexture } from '../textures/DataTexture.js';\nimport { FloatType, RedFormat } from '../constants.js';\n\nconst _instanceLocalMatrix = /*@__PURE__*/ new Matrix4();\nconst _instanceWorldMatrix = /*@__PURE__*/ new Matrix4();\n\nconst _instanceIntersects = [];\n\nconst _box3 = /*@__PURE__*/ new Box3();\nconst _identity = /*@__PURE__*/ new Matrix4();\nconst _mesh = /*@__PURE__*/ new Mesh();\nconst _sphere = /*@__PURE__*/ new Sphere();\n\nclass InstancedMesh extends Mesh {\n\n\tconstructor( geometry, material, count ) {\n\n\t\tsuper( geometry, material );\n\n\t\tthis.isInstancedMesh = true;\n\n\t\tthis.instanceMatrix = new InstancedBufferAttribute( new Float32Array( count * 16 ), 16 );\n\t\tthis.instanceColor = null;\n\t\tthis.morphTexture = null;\n\n\t\tthis.count = count;\n\n\t\tthis.boundingBox = null;\n\t\tthis.boundingSphere = null;\n\n\t\tfor ( let i = 0; i < count; i ++ ) {\n\n\t\t\tthis.setMatrixAt( i, _identity );\n\n\t\t}\n\n\t}\n\n\tcomputeBoundingBox() {\n\n\t\tconst geometry = this.geometry;\n\t\tconst count = this.count;\n\n\t\tif ( this.boundingBox === null ) {\n\n\t\t\tthis.boundingBox = new Box3();\n\n\t\t}\n\n\t\tif ( geometry.boundingBox === null ) {\n\n\t\t\tgeometry.computeBoundingBox();\n\n\t\t}\n\n\t\tthis.boundingBox.makeEmpty();\n\n\t\tfor ( let i = 0; i < count; i ++ ) {\n\n\t\t\tthis.getMatrixAt( i, _instanceLocalMatrix );\n\n\t\t\t_box3.copy( geometry.boundingBox ).applyMatrix4( _instanceLocalMatrix );\n\n\t\t\tthis.boundingBox.union( _box3 );\n\n\t\t}\n\n\t}\n\n\tcomputeBoundingSphere() {\n\n\t\tconst geometry = this.geometry;\n\t\tconst count = this.count;\n\n\t\tif ( this.boundingSphere === null ) {\n\n\t\t\tthis.boundingSphere = new Sphere();\n\n\t\t}\n\n\t\tif ( geometry.boundingSphere === null ) {\n\n\t\t\tgeometry.computeBoundingSphere();\n\n\t\t}\n\n\t\tthis.boundingSphere.makeEmpty();\n\n\t\tfor ( let i = 0; i < count; i ++ ) {\n\n\t\t\tthis.getMatrixAt( i, _instanceLocalMatrix );\n\n\t\t\t_sphere.copy( geometry.boundingSphere ).applyMatrix4( _instanceLocalMatrix );\n\n\t\t\tthis.boundingSphere.union( _sphere );\n\n\t\t}\n\n\t}\n\n\tcopy( source, recursive ) {\n\n\t\tsuper.copy( source, recursive );\n\n\t\tthis.instanceMatrix.copy( source.instanceMatrix );\n\n\t\tif ( source.morphTexture !== null ) this.morphTexture = source.morphTexture.clone();\n\t\tif ( source.instanceColor !== null ) this.instanceColor = source.instanceColor.clone();\n\n\t\tthis.count = source.count;\n\n\t\tif ( source.boundingBox !== null ) this.boundingBox = source.boundingBox.clone();\n\t\tif ( source.boundingSphere !== null ) this.boundingSphere = source.boundingSphere.clone();\n\n\t\treturn this;\n\n\t}\n\n\tgetColorAt( index, color ) {\n\n\t\tcolor.fromArray( this.instanceColor.array, index * 3 );\n\n\t}\n\n\tgetMatrixAt( index, matrix ) {\n\n\t\tmatrix.fromArray( this.instanceMatrix.array, index * 16 );\n\n\t}\n\n\tgetMorphAt( index, object ) {\n\n\t\tconst objectInfluences = object.morphTargetInfluences;\n\n\t\tconst array = this.morphTexture.source.data.data;\n\n\t\tconst len = objectInfluences.length + 1; // All influences + the baseInfluenceSum\n\n\t\tconst dataIndex = index * len + 1; // Skip the baseInfluenceSum at the beginning\n\n\t\tfor ( let i = 0; i < objectInfluences.length; i ++ ) {\n\n\t\t\tobjectInfluences[ i ] = array[ dataIndex + i ];\n\n\t\t}\n\n\t}\n\n\traycast( raycaster, intersects ) {\n\n\t\tconst matrixWorld = this.matrixWorld;\n\t\tconst raycastTimes = this.count;\n\n\t\t_mesh.geometry = this.geometry;\n\t\t_mesh.material = this.material;\n\n\t\tif ( _mesh.material === undefined ) return;\n\n\t\t// test with bounding sphere first\n\n\t\tif ( this.boundingSphere === null ) this.computeBoundingSphere();\n\n\t\t_sphere.copy( this.boundingSphere );\n\t\t_sphere.applyMatrix4( matrixWorld );\n\n\t\tif ( raycaster.ray.intersectsSphere( _sphere ) === false ) return;\n\n\t\t// now test each instance\n\n\t\tfor ( let instanceId = 0; instanceId < raycastTimes; instanceId ++ ) {\n\n\t\t\t// calculate the world matrix for each instance\n\n\t\t\tthis.getMatrixAt( instanceId, _instanceLocalMatrix );\n\n\t\t\t_instanceWorldMatrix.multiplyMatrices( matrixWorld, _instanceLocalMatrix );\n\n\t\t\t// the mesh represents this single instance\n\n\t\t\t_mesh.matrixWorld = _instanceWorldMatrix;\n\n\t\t\t_mesh.raycast( raycaster, _instanceIntersects );\n\n\t\t\t// process the result of raycast\n\n\t\t\tfor ( let i = 0, l = _instanceIntersects.length; i < l; i ++ ) {\n\n\t\t\t\tconst intersect = _instanceIntersects[ i ];\n\t\t\t\tintersect.instanceId = instanceId;\n\t\t\t\tintersect.object = this;\n\t\t\t\tintersects.push( intersect );\n\n\t\t\t}\n\n\t\t\t_instanceIntersects.length = 0;\n\n\t\t}\n\n\t}\n\n\tsetColorAt( index, color ) {\n\n\t\tif ( this.instanceColor === null ) {\n\n\t\t\tthis.instanceColor = new InstancedBufferAttribute( new Float32Array( this.instanceMatrix.count * 3 ), 3 );\n\n\t\t}\n\n\t\tcolor.toArray( this.instanceColor.array, index * 3 );\n\n\t}\n\n\tsetMatrixAt( index, matrix ) {\n\n\t\tmatrix.toArray( this.instanceMatrix.array, index * 16 );\n\n\t}\n\n\tsetMorphAt( index, object ) {\n\n\t\tconst objectInfluences = object.morphTargetInfluences;\n\n\t\tconst len = objectInfluences.length + 1; // morphBaseInfluence + all influences\n\n\t\tif ( this.morphTexture === null ) {\n\n\t\t\tthis.morphTexture = new DataTexture( new Float32Array( len * this.count ), len, this.count, RedFormat, FloatType );\n\n\t\t}\n\n\t\tconst array = this.morphTexture.source.data.data;\n\n\t\tlet morphInfluencesSum = 0;\n\n\t\tfor ( let i = 0; i < objectInfluences.length; i ++ ) {\n\n\t\t\tmorphInfluencesSum += objectInfluences[ i ];\n\n\t\t}\n\n\t\tconst morphBaseInfluence = this.geometry.morphTargetsRelative ? 1 : 1 - morphInfluencesSum;\n\n\t\tconst dataIndex = len * index;\n\n\t\tarray[ dataIndex ] = morphBaseInfluence;\n\n\t\tarray.set( objectInfluences, dataIndex + 1 );\n\n\t}\n\n\tupdateMorphTargets() {\n\n\t}\n\n\tdispose() {\n\n\t\tthis.dispatchEvent( { type: 'dispose' } );\n\n\t\tif ( this.morphTexture !== null ) {\n\n\t\t\tthis.morphTexture.dispose();\n\t\t\tthis.morphTexture = null;\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n}\n\nexport { InstancedMesh };\n\n// Path: src/math/interpolants/QuaternionLinearInterpolant.js\nimport { Interpolant } from '../Interpolant.js';\nimport { Quaternion } from '../Quaternion.js';\n\n/**\n * Spherical linear unit quaternion interpolant.\n */\n\nclass QuaternionLinearInterpolant extends Interpolant {\n\n\tconstructor( parameterPositions, sampleValues, sampleSize, resultBuffer ) {\n\n\t\tsuper( parameterPositions, sampleValues, sampleSize, resultBuffer );\n\n\t}\n\n\tinterpolate_( i1, t0, t, t1 ) {\n\n\t\tconst result = this.resultBuffer,\n\t\t\tvalues = this.sampleValues,\n\t\t\tstride = this.valueSize,\n\n\t\t\talpha = ( t - t0 ) / ( t1 - t0 );\n\n\t\tlet offset = i1 * stride;\n\n\t\tfor ( let end = offset + stride; offset !== end; offset += 4 ) {\n\n\t\t\tQuaternion.slerpFlat( result, 0, values, offset - stride, values, offset, alpha );\n\n\t\t}\n\n\t\treturn result;\n\n\t}\n\n}\n\n\nexport { QuaternionLinearInterpolant };\n\n// Path: src/animation/tracks/QuaternionKeyframeTrack.js\nimport { InterpolateLinear } from '../../constants.js';\nimport { KeyframeTrack } from '../KeyframeTrack.js';\nimport { QuaternionLinearInterpolant } from '../../math/interpolants/QuaternionLinearInterpolant.js';\n\n/**\n * A Track of quaternion keyframe values.\n */\nclass QuaternionKeyframeTrack extends KeyframeTrack {\n\n\tInterpolantFactoryMethodLinear( result ) {\n\n\t\treturn new QuaternionLinearInterpolant( this.times, this.values, this.getValueSize(), result );\n\n\t}\n\n}\n\nQuaternionKeyframeTrack.prototype.ValueTypeName = 'quaternion';\n// ValueBufferType is inherited\nQuaternionKeyframeTrack.prototype.DefaultInterpolation = InterpolateLinear;\nQuaternionKeyframeTrack.prototype.InterpolantFactoryMethodSmooth = undefined;\n\nexport { QuaternionKeyframeTrack };\n\n// Path: src/animation/tracks/BooleanKeyframeTrack.js\nimport { InterpolateDiscrete } from '../../constants.js';\nimport { KeyframeTrack } from '../KeyframeTrack.js';\n\n/**\n * A Track of Boolean keyframe values.\n */\nclass BooleanKeyframeTrack extends KeyframeTrack {}\n\nBooleanKeyframeTrack.prototype.ValueTypeName = 'bool';\nBooleanKeyframeTrack.prototype.ValueBufferType = Array;\nBooleanKeyframeTrack.prototype.DefaultInterpolation = InterpolateDiscrete;\nBooleanKeyframeTrack.prototype.InterpolantFactoryMethodLinear = undefined;\nBooleanKeyframeTrack.prototype.InterpolantFactoryMethodSmooth = undefined;\n\n// Note: Actually this track could have a optimized / compressed\n// representation of a single value and a custom interpolant that\n// computes \"firstValue ^ isOdd( index )\".\n\nexport { BooleanKeyframeTrack };\n\n// Path: src/animation/tracks/VectorKeyframeTrack.js\nimport { KeyframeTrack } from '../KeyframeTrack.js';\n\n/**\n * A Track of vectored keyframe values.\n */\nclass VectorKeyframeTrack extends KeyframeTrack {}\n\nVectorKeyframeTrack.prototype.ValueTypeName = 'vector';\n// ValueBufferType is inherited\n// DefaultInterpolation is inherited\n\nexport { VectorKeyframeTrack };\n\n// Path: src/animation/tracks/NumberKeyframeTrack.js\nimport { KeyframeTrack } from '../KeyframeTrack.js';\n\n/**\n * A Track of numeric keyframe values.\n */\nclass NumberKeyframeTrack extends KeyframeTrack {}\n\nNumberKeyframeTrack.prototype.ValueTypeName = 'number';\n// ValueBufferType is inherited\n// DefaultInterpolation is inherited\n\nexport { NumberKeyframeTrack };\n\n// Path: src/animation/AnimationClip.js\nimport * as AnimationUtils from './AnimationUtils.js';\nimport { KeyframeTrack } from './KeyframeTrack.js';\nimport { BooleanKeyframeTrack } from './tracks/BooleanKeyframeTrack.js';\nimport { ColorKeyframeTrack } from './tracks/ColorKeyframeTrack.js';\nimport { NumberKeyframeTrack } from './tracks/NumberKeyframeTrack.js';\nimport { QuaternionKeyframeTrack } from './tracks/QuaternionKeyframeTrack.js';\nimport { StringKeyframeTrack } from './tracks/StringKeyframeTrack.js';\nimport { VectorKeyframeTrack } from './tracks/VectorKeyframeTrack.js';\nimport * as MathUtils from '../math/MathUtils.js';\nimport { NormalAnimationBlendMode } from '../constants.js';\n\nclass AnimationClip {\n\n\tconstructor( name = '', duration = - 1, tracks = [], blendMode = NormalAnimationBlendMode ) {\n\n\t\tthis.name = name;\n\t\tthis.tracks = tracks;\n\t\tthis.duration = duration;\n\t\tthis.blendMode = blendMode;\n\n\t\tthis.uuid = MathUtils.generateUUID();\n\n\t\t// this means it should figure out its duration by scanning the tracks\n\t\tif ( this.duration < 0 ) {\n\n\t\t\tthis.resetDuration();\n\n\t\t}\n\n\t}\n\n\n\tstatic parse( json ) {\n\n\t\tconst tracks = [],\n\t\t\tjsonTracks = json.tracks,\n\t\t\tframeTime = 1.0 / ( json.fps || 1.0 );\n\n\t\tfor ( let i = 0, n = jsonTracks.length; i !== n; ++ i ) {\n\n\t\t\ttracks.push( parseKeyframeTrack( jsonTracks[ i ] ).scale( frameTime ) );\n\n\t\t}\n\n\t\tconst clip = new this( json.name, json.duration, tracks, json.blendMode );\n\t\tclip.uuid = json.uuid;\n\n\t\treturn clip;\n\n\t}\n\n\tstatic toJSON( clip ) {\n\n\t\tconst tracks = [],\n\t\t\tclipTracks = clip.tracks;\n\n\t\tconst json = {\n\n\t\t\t'name': clip.name,\n\t\t\t'duration': clip.duration,\n\t\t\t'tracks': tracks,\n\t\t\t'uuid': clip.uuid,\n\t\t\t'blendMode': clip.blendMode\n\n\t\t};\n\n\t\tfor ( let i = 0, n = clipTracks.length; i !== n; ++ i ) {\n\n\t\t\ttracks.push( KeyframeTrack.toJSON( clipTracks[ i ] ) );\n\n\t\t}\n\n\t\treturn json;\n\n\t}\n\n\tstatic CreateFromMorphTargetSequence( name, morphTargetSequence, fps, noLoop ) {\n\n\t\tconst numMorphTargets = morphTargetSequence.length;\n\t\tconst tracks = [];\n\n\t\tfor ( let i = 0; i < numMorphTargets; i ++ ) {\n\n\t\t\tlet times = [];\n\t\t\tlet values = [];\n\n\t\t\ttimes.push(\n\t\t\t\t( i + numMorphTargets - 1 ) % numMorphTargets,\n\t\t\t\ti,\n\t\t\t\t( i + 1 ) % numMorphTargets );\n\n\t\t\tvalues.push( 0, 1, 0 );\n\n\t\t\tconst order = AnimationUtils.getKeyframeOrder( times );\n\t\t\ttimes = AnimationUtils.sortedArray( times, 1, order );\n\t\t\tvalues = AnimationUtils.sortedArray( values, 1, order );\n\n\t\t\t// if there is a key at the first frame, duplicate it as the\n\t\t\t// last frame as well for perfect loop.\n\t\t\tif ( ! noLoop && times[ 0 ] === 0 ) {\n\n\t\t\t\ttimes.push( numMorphTargets );\n\t\t\t\tvalues.push( values[ 0 ] );\n\n\t\t\t}\n\n\t\t\ttracks.push(\n\t\t\t\tnew NumberKeyframeTrack(\n\t\t\t\t\t'.morphTargetInfluences[' + morphTargetSequence[ i ].name + ']',\n\t\t\t\t\ttimes, values\n\t\t\t\t).scale( 1.0 / fps ) );\n\n\t\t}\n\n\t\treturn new this( name, - 1, tracks );\n\n\t}\n\n\tstatic findByName( objectOrClipArray, name ) {\n\n\t\tlet clipArray = objectOrClipArray;\n\n\t\tif ( ! Array.isArray( objectOrClipArray ) ) {\n\n\t\t\tconst o = objectOrClipArray;\n\t\t\tclipArray = o.geometry && o.geometry.animations || o.animations;\n\n\t\t}\n\n\t\tfor ( let i = 0; i < clipArray.length; i ++ ) {\n\n\t\t\tif ( clipArray[ i ].name === name ) {\n\n\t\t\t\treturn clipArray[ i ];\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn null;\n\n\t}\n\n\tstatic CreateClipsFromMorphTargetSequences( morphTargets, fps, noLoop ) {\n\n\t\tconst animationToMorphTargets = {};\n\n\t\t// tested with https://regex101.com/ on trick sequences\n\t\t// such flamingo_flyA_003, flamingo_run1_003, crdeath0059\n\t\tconst pattern = /^([\\w-]*?)([\\d]+)$/;\n\n\t\t// sort morph target names into animation groups based\n\t\t// patterns like Walk_001, Walk_002, Run_001, Run_002\n\t\tfor ( let i = 0, il = morphTargets.length; i < il; i ++ ) {\n\n\t\t\tconst morphTarget = morphTargets[ i ];\n\t\t\tconst parts = morphTarget.name.match( pattern );\n\n\t\t\tif ( parts && parts.length > 1 ) {\n\n\t\t\t\tconst name = parts[ 1 ];\n\n\t\t\t\tlet animationMorphTargets = animationToMorphTargets[ name ];\n\n\t\t\t\tif ( ! animationMorphTargets ) {\n\n\t\t\t\t\tanimationToMorphTargets[ name ] = animationMorphTargets = [];\n\n\t\t\t\t}\n\n\t\t\t\tanimationMorphTargets.push( morphTarget );\n\n\t\t\t}\n\n\t\t}\n\n\t\tconst clips = [];\n\n\t\tfor ( const name in animationToMorphTargets ) {\n\n\t\t\tclips.push( this.CreateFromMorphTargetSequence( name, animationToMorphTargets[ name ], fps, noLoop ) );\n\n\t\t}\n\n\t\treturn clips;\n\n\t}\n\n\t// parse the animation.hierarchy format\n\tstatic parseAnimation( animation, bones ) {\n\n\t\tif ( ! animation ) {\n\n\t\t\tconsole.error( 'THREE.AnimationClip: No animation in JSONLoader data.' );\n\t\t\treturn null;\n\n\t\t}\n\n\t\tconst addNonemptyTrack = function ( trackType, trackName, animationKeys, propertyName, destTracks ) {\n\n\t\t\t// only return track if there are actually keys.\n\t\t\tif ( animationKeys.length !== 0 ) {\n\n\t\t\t\tconst times = [];\n\t\t\t\tconst values = [];\n\n\t\t\t\tAnimationUtils.flattenJSON( animationKeys, times, values, propertyName );\n\n\t\t\t\t// empty keys are filtered out, so check again\n\t\t\t\tif ( times.length !== 0 ) {\n\n\t\t\t\t\tdestTracks.push( new trackType( trackName, times, values ) );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t};\n\n\t\tconst tracks = [];\n\n\t\tconst clipName = animation.name || 'default';\n\t\tconst fps = animation.fps || 30;\n\t\tconst blendMode = animation.blendMode;\n\n\t\t// automatic length determination in AnimationClip.\n\t\tlet duration = animation.length || - 1;\n\n\t\tconst hierarchyTracks = animation.hierarchy || [];\n\n\t\tfor ( let h = 0; h < hierarchyTracks.length; h ++ ) {\n\n\t\t\tconst animationKeys = hierarchyTracks[ h ].keys;\n\n\t\t\t// skip empty tracks\n\t\t\tif ( ! animationKeys || animationKeys.length === 0 ) continue;\n\n\t\t\t// process morph targets\n\t\t\tif ( animationKeys[ 0 ].morphTargets ) {\n\n\t\t\t\t// figure out all morph targets used in this track\n\t\t\t\tconst morphTargetNames = {};\n\n\t\t\t\tlet k;\n\n\t\t\t\tfor ( k = 0; k < animationKeys.length; k ++ ) {\n\n\t\t\t\t\tif ( animationKeys[ k ].morphTargets ) {\n\n\t\t\t\t\t\tfor ( let m = 0; m < animationKeys[ k ].morphTargets.length; m ++ ) {\n\n\t\t\t\t\t\t\tmorphTargetNames[ animationKeys[ k ].morphTargets[ m ] ] = - 1;\n\n\t\t\t\t\t\t}\n...\n", "needle_token_start": 3609, "needle_token_end": 3839, "code_context_ntokens": 14762, "output": ["```javascript\nfunction transformVertex( vertexPosition, mvPosition, center, scale, sin, cos ) {\n\n\t// compute position in camera space\n\t_alignedPosition.subVectors( vertexPosition, center ).addScalar( 0.5 ).multiply( scale );\n\n\t// to check if rotation is not zero\n\tif ( sin !== undefined ) {\n\n\t\t_rotatedPosition.x = ( cos * _alignedPosition.x ) - ( sin * _alignedPosition.y );\n\t\t_rotatedPosition.y = ( sin * _alignedPosition.x ) + ( cos * _alignedPosition.y );\n\n\t} else {\n\n\t\t_rotatedPosition.copy( _alignedPosition );\n\n\t}\n\n\n\tvertexPosition.copy( mvPosition );\n\tvertexPosition.x += _rotatedPosition.x;\n\tvertexPosition.y += _rotatedPosition.y;\n\n\t// transform to world space\n\tvertexPosition.applyMatrix4( _viewWorldMatrix );\n\n}\n```"]}
{"repo": "mrdoob/three.js", "name": "handleTriangle", "language": "typescript", "path": "src/core/BufferGeometry.js", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function calculates tangent and bitangent vectors for each vertex of a triangle based on its vertex positions and UV coordinates. These vectors are used in shading calculations to correctly apply textures to 3D models.\n2. **Input**: The function takes indices of three vertices of a triangle.\n3. **Output**: There is no direct output; however, the function updates two arrays with the computed tangent and bitangent vectors for each vertex.\n4. **Procedure**: \n   - Retrieve the position and UV coordinates for each of the three vertices.\n   - Compute the vectors along the sides of the triangle in both position and UV space.\n   - Calculate the reciprocal of the determinant of a 2x2 matrix derived from the UV vectors.\n   - If the determinant is non-zero, compute the tangent and bitangent vectors by solving a linear system formed by the UV and position vectors, scaled by the reciprocal of the determinant.\n   - Add the computed tangent and bitangent vectors to accumulators for each vertex.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/math/Sphere.js\nimport { Box3 } from './Box3.js';\nimport { Vector3 } from './Vector3.js';\n\nconst _box = /*@__PURE__*/ new Box3();\nconst _v1 = /*@__PURE__*/ new Vector3();\nconst _v2 = /*@__PURE__*/ new Vector3();\n\nclass Sphere {\n\n\tconstructor( center = new Vector3(), radius = - 1 ) {\n\n\t\tthis.isSphere = true;\n\n\t\tthis.center = center;\n\t\tthis.radius = radius;\n\n\t}\n\n\tset( center, radius ) {\n\n\t\tthis.center.copy( center );\n\t\tthis.radius = radius;\n\n\t\treturn this;\n\n\t}\n\n\tsetFromPoints( points, optionalCenter ) {\n\n\t\tconst center = this.center;\n\n\t\tif ( optionalCenter !== undefined ) {\n\n\t\t\tcenter.copy( optionalCenter );\n\n\t\t} else {\n\n\t\t\t_box.setFromPoints( points ).getCenter( center );\n\n\t\t}\n\n\t\tlet maxRadiusSq = 0;\n\n\t\tfor ( let i = 0, il = points.length; i < il; i ++ ) {\n\n\t\t\tmaxRadiusSq = Math.max( maxRadiusSq, center.distanceToSquared( points[ i ] ) );\n\n\t\t}\n\n\t\tthis.radius = Math.sqrt( maxRadiusSq );\n\n\t\treturn this;\n\n\t}\n\n\tcopy( sphere ) {\n\n\t\tthis.center.copy( sphere.center );\n\t\tthis.radius = sphere.radius;\n\n\t\treturn this;\n\n\t}\n\n\tisEmpty() {\n\n\t\treturn ( this.radius < 0 );\n\n\t}\n\n\tmakeEmpty() {\n\n\t\tthis.center.set( 0, 0, 0 );\n\t\tthis.radius = - 1;\n\n\t\treturn this;\n\n\t}\n\n\tcontainsPoint( point ) {\n\n\t\treturn ( point.distanceToSquared( this.center ) <= ( this.radius * this.radius ) );\n\n\t}\n\n\tdistanceToPoint( point ) {\n\n\t\treturn ( point.distanceTo( this.center ) - this.radius );\n\n\t}\n\n\tintersectsSphere( sphere ) {\n\n\t\tconst radiusSum = this.radius + sphere.radius;\n\n\t\treturn sphere.center.distanceToSquared( this.center ) <= ( radiusSum * radiusSum );\n\n\t}\n\n\tintersectsBox( box ) {\n\n\t\treturn box.intersectsSphere( this );\n\n\t}\n\n\tintersectsPlane( plane ) {\n\n\t\treturn Math.abs( plane.distanceToPoint( this.center ) ) <= this.radius;\n\n\t}\n\n\tclampPoint( point, target ) {\n\n\t\tconst deltaLengthSq = this.center.distanceToSquared( point );\n\n\t\ttarget.copy( point );\n\n\t\tif ( deltaLengthSq > ( this.radius * this.radius ) ) {\n\n\t\t\ttarget.sub( this.center ).normalize();\n\t\t\ttarget.multiplyScalar( this.radius ).add( this.center );\n\n\t\t}\n\n\t\treturn target;\n\n\t}\n\n\tgetBoundingBox( target ) {\n\n\t\tif ( this.isEmpty() ) {\n\n\t\t\t// Empty sphere produces empty bounding box\n\t\t\ttarget.makeEmpty();\n\t\t\treturn target;\n\n\t\t}\n\n\t\ttarget.set( this.center, this.center );\n\t\ttarget.expandByScalar( this.radius );\n\n\t\treturn target;\n\n\t}\n\n\tapplyMatrix4( matrix ) {\n\n\t\tthis.center.applyMatrix4( matrix );\n\t\tthis.radius = this.radius * matrix.getMaxScaleOnAxis();\n\n\t\treturn this;\n\n\t}\n\n\ttranslate( offset ) {\n\n\t\tthis.center.add( offset );\n\n\t\treturn this;\n\n\t}\n\n...\n// Path: src/core/BufferGeometry.js\nimport { Vector3 } from '../math/Vector3.js';\nimport { Vector2 } from '../math/Vector2.js';\nimport { Box3 } from '../math/Box3.js';\nimport { EventDispatcher } from './EventDispatcher.js';\nimport { BufferAttribute, Float32BufferAttribute, Uint16BufferAttribute, Uint32BufferAttribute } from './BufferAttribute.js';\nimport { Sphere } from '../math/Sphere.js';\nimport { Object3D } from './Object3D.js';\nimport { Matrix4 } from '../math/Matrix4.js';\nimport { Matrix3 } from '../math/Matrix3.js';\nimport * as MathUtils from '../math/MathUtils.js';\nimport { arrayNeedsUint32 } from '../utils.js';\n\nlet _id = 0;\n\nconst _m1 = /*@__PURE__*/ new Matrix4();\nconst _obj = /*@__PURE__*/ new Object3D();\nconst _offset = /*@__PURE__*/ new Vector3();\nconst _box = /*@__PURE__*/ new Box3();\nconst _boxMorphTargets = /*@__PURE__*/ new Box3();\nconst _vector = /*@__PURE__*/ new Vector3();\n\nclass BufferGeometry extends EventDispatcher {\n\n\tconstructor() {\n\n\t\tsuper();\n\n\t\tthis.isBufferGeometry = true;\n\n\t\tObject.defineProperty( this, 'id', { value: _id ++ } );\n\n\t\tthis.uuid = MathUtils.generateUUID();\n\n\t\tthis.name = '';\n\t\tthis.type = 'BufferGeometry';\n\n\t\tthis.index = null;\n\t\tthis.attributes = {};\n\n\t\tthis.morphAttributes = {};\n\t\tthis.morphTargetsRelative = false;\n\n\t\tthis.groups = [];\n\n\t\tthis.boundingBox = null;\n\t\tthis.boundingSphere = null;\n\n\t\tthis.drawRange = { start: 0, count: Infinity };\n\n\t\tthis.userData = {};\n\n\t}\n\n\tgetIndex() {\n\n\t\treturn this.index;\n\n\t}\n\n\tsetIndex( index ) {\n\n\t\tif ( Array.isArray( index ) ) {\n\n\t\t\tthis.index = new ( arrayNeedsUint32( index ) ? Uint32BufferAttribute : Uint16BufferAttribute )( index, 1 );\n\n\t\t} else {\n\n\t\t\tthis.index = index;\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tgetAttribute( name ) {\n\n\t\treturn this.attributes[ name ];\n\n\t}\n\n\tsetAttribute( name, attribute ) {\n\n\t\tthis.attributes[ name ] = attribute;\n\n\t\treturn this;\n\n\t}\n\n\tdeleteAttribute( name ) {\n\n\t\tdelete this.attributes[ name ];\n\n\t\treturn this;\n\n\t}\n\n\thasAttribute( name ) {\n\n\t\treturn this.attributes[ name ] !== undefined;\n\n\t}\n\n\taddGroup( start, count, materialIndex = 0 ) {\n\n\t\tthis.groups.push( {\n\n\t\t\tstart: start,\n\t\t\tcount: count,\n\t\t\tmaterialIndex: materialIndex\n\n\t\t} );\n\n\t}\n\n\tclearGroups() {\n\n\t\tthis.groups = [];\n\n\t}\n\n\tsetDrawRange( start, count ) {\n\n\t\tthis.drawRange.start = start;\n\t\tthis.drawRange.count = count;\n\n\t}\n\n\tapplyMatrix4( matrix ) {\n\n\t\tconst position = this.attributes.position;\n\n\t\tif ( position !== undefined ) {\n\n\t\t\tposition.applyMatrix4( matrix );\n\n\t\t\tposition.needsUpdate = true;\n\n\t\t}\n\n\t\tconst normal = this.attributes.normal;\n\n\t\tif ( normal !== undefined ) {\n\n\t\t\tconst normalMatrix = new Matrix3().getNormalMatrix( matrix );\n\n\t\t\tnormal.applyNormalMatrix( normalMatrix );\n\n\t\t\tnormal.needsUpdate = true;\n\n\t\t}\n\n\t\tconst tangent = this.attributes.tangent;\n\n\t\tif ( tangent !== undefined ) {\n\n\t\t\ttangent.transformDirection( matrix );\n\n\t\t\ttangent.needsUpdate = true;\n\n\t\t}\n\n\t\tif ( this.boundingBox !== null ) {\n\n\t\t\tthis.computeBoundingBox();\n\n\t\t}\n\n\t\tif ( this.boundingSphere !== null ) {\n\n\t\t\tthis.computeBoundingSphere();\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tapplyQuaternion( q ) {\n\n\t\t_m1.makeRotationFromQuaternion( q );\n\n\t\tthis.applyMatrix4( _m1 );\n\n\t\treturn this;\n\n\t}\n\n\trotateX( angle ) {\n\n\t\t// rotate geometry around world x-axis\n\n\t\t_m1.makeRotationX( angle );\n\n\t\tthis.applyMatrix4( _m1 );\n\n\t\treturn this;\n\n\t}\n\n\trotateY( angle ) {\n\n\t\t// rotate geometry around world y-axis\n\n\t\t_m1.makeRotationY( angle );\n\n\t\tthis.applyMatrix4( _m1 );\n\n\t\treturn this;\n\n\t}\n\n\trotateZ( angle ) {\n\n\t\t// rotate geometry around world z-axis\n\n\t\t_m1.makeRotationZ( angle );\n\n\t\tthis.applyMatrix4( _m1 );\n\n\t\treturn this;\n\n\t}\n\n\ttranslate( x, y, z ) {\n\n\t\t// translate geometry\n\n\t\t_m1.makeTranslation( x, y, z );\n\n\t\tthis.applyMatrix4( _m1 );\n\n\t\treturn this;\n\n\t}\n\n\tscale( x, y, z ) {\n\n\t\t// scale geometry\n\n\t\t_m1.makeScale( x, y, z );\n\n\t\tthis.applyMatrix4( _m1 );\n\n\t\treturn this;\n\n\t}\n\n\tlookAt( vector ) {\n\n\t\t_obj.lookAt( vector );\n\n\t\t_obj.updateMatrix();\n\n\t\tthis.applyMatrix4( _obj.matrix );\n\n\t\treturn this;\n\n\t}\n\n\tcenter() {\n\n\t\tthis.computeBoundingBox();\n\n\t\tthis.boundingBox.getCenter( _offset ).negate();\n\n\t\tthis.translate( _offset.x, _offset.y, _offset.z );\n\n\t\treturn this;\n\n\t}\n\n\tsetFromPoints( points ) {\n\n\t\tconst position = [];\n\n\t\tfor ( let i = 0, l = points.length; i < l; i ++ ) {\n\n\t\t\tconst point = points[ i ];\n\t\t\tposition.push( point.x, point.y, point.z || 0 );\n\n\t\t}\n\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( position, 3 ) );\n\n\t\treturn this;\n\n\t}\n\n\tcomputeBoundingBox() {\n\n\t\tif ( this.boundingBox === null ) {\n\n\t\t\tthis.boundingBox = new Box3();\n\n\t\t}\n\n\t\tconst position = this.attributes.position;\n\t\tconst morphAttributesPosition = this.morphAttributes.position;\n\n\t\tif ( position && position.isGLBufferAttribute ) {\n\n\t\t\tconsole.error( 'THREE.BufferGeometry.computeBoundingBox(): GLBufferAttribute requires a manual bounding box.', this );\n\n\t\t\tthis.boundingBox.set(\n\t\t\t\tnew Vector3( - Infinity, - Infinity, - Infinity ),\n\t\t\t\tnew Vector3( + Infinity, + Infinity, + Infinity )\n\t\t\t);\n\n\t\t\treturn;\n\n\t\t}\n\n\t\tif ( position !== undefined ) {\n\n\t\t\tthis.boundingBox.setFromBufferAttribute( position );\n\n\t\t\t// process morph attributes if present\n\n\t\t\tif ( morphAttributesPosition ) {\n\n\t\t\t\tfor ( let i = 0, il = morphAttributesPosition.length; i < il; i ++ ) {\n\n\t\t\t\t\tconst morphAttribute = morphAttributesPosition[ i ];\n\t\t\t\t\t_box.setFromBufferAttribute( morphAttribute );\n\n\t\t\t\t\tif ( this.morphTargetsRelative ) {\n\n\t\t\t\t\t\t_vector.addVectors( this.boundingBox.min, _box.min );\n\t\t\t\t\t\tthis.boundingBox.expandByPoint( _vector );\n\n\t\t\t\t\t\t_vector.addVectors( this.boundingBox.max, _box.max );\n\t\t\t\t\t\tthis.boundingBox.expandByPoint( _vector );\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tthis.boundingBox.expandByPoint( _box.min );\n\t\t\t\t\t\tthis.boundingBox.expandByPoint( _box.max );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t} else {\n\n\t\t\tthis.boundingBox.makeEmpty();\n\n\t\t}\n\n\t\tif ( isNaN( this.boundingBox.min.x ) || isNaN( this.boundingBox.min.y ) || isNaN( this.boundingBox.min.z ) ) {\n\n\t\t\tconsole.error( 'THREE.BufferGeometry.computeBoundingBox(): Computed min/max have NaN values. The \"position\" attribute is likely to have NaN values.', this );\n\n\t\t}\n\n\t}\n\n\tcomputeBoundingSphere() {\n\n\t\tif ( this.boundingSphere === null ) {\n\n\t\t\tthis.boundingSphere = new Sphere();\n\n\t\t}\n\n\t\tconst position = this.attributes.position;\n\t\tconst morphAttributesPosition = this.morphAttributes.position;\n\n\t\tif ( position && position.isGLBufferAttribute ) {\n\n\t\t\tconsole.error( 'THREE.BufferGeometry.computeBoundingSphere(): GLBufferAttribute requires a manual bounding sphere.', this );\n\n\t\t\tthis.boundingSphere.set( new Vector3(), Infinity );\n\n\t\t\treturn;\n\n\t\t}\n\n\t\tif ( position ) {\n\n\t\t\t// first, find the center of the bounding sphere\n\n\t\t\tconst center = this.boundingSphere.center;\n\n\t\t\t_box.setFromBufferAttribute( position );\n\n\t\t\t// process morph attributes if present\n\n\t\t\tif ( morphAttributesPosition ) {\n\n\t\t\t\tfor ( let i = 0, il = morphAttributesPosition.length; i < il; i ++ ) {\n\n\t\t\t\t\tconst morphAttribute = morphAttributesPosition[ i ];\n\t\t\t\t\t_boxMorphTargets.setFromBufferAttribute( morphAttribute );\n\n\t\t\t\t\tif ( this.morphTargetsRelative ) {\n\n\t\t\t\t\t\t_vector.addVectors( _box.min, _boxMorphTargets.min );\n\t\t\t\t\t\t_box.expandByPoint( _vector );\n\n\t\t\t\t\t\t_vector.addVectors( _box.max, _boxMorphTargets.max );\n\t\t\t\t\t\t_box.expandByPoint( _vector );\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\t_box.expandByPoint( _boxMorphTargets.min );\n\t\t\t\t\t\t_box.expandByPoint( _boxMorphTargets.max );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t_box.getCenter( center );\n\n\t\t\t// second, try to find a boundingSphere with a radius smaller than the\n\t\t\t// boundingSphere of the boundingBox: sqrt(3) smaller in the best case\n\n\t\t\tlet maxRadiusSq = 0;\n\n\t\t\tfor ( let i = 0, il = position.count; i < il; i ++ ) {\n\n\t\t\t\t_vector.fromBufferAttribute( position, i );\n\n\t\t\t\tmaxRadiusSq = Math.max( maxRadiusSq, center.distanceToSquared( _vector ) );\n\n\t\t\t}\n\n\t\t\t// process morph attributes if present\n\n\t\t\tif ( morphAttributesPosition ) {\n\n\t\t\t\tfor ( let i = 0, il = morphAttributesPosition.length; i < il; i ++ ) {\n\n\t\t\t\t\tconst morphAttribute = morphAttributesPosition[ i ];\n\t\t\t\t\tconst morphTargetsRelative = this.morphTargetsRelative;\n\n\t\t\t\t\tfor ( let j = 0, jl = morphAttribute.count; j < jl; j ++ ) {\n\n\t\t\t\t\t\t_vector.fromBufferAttribute( morphAttribute, j );\n\n\t\t\t\t\t\tif ( morphTargetsRelative ) {\n\n\t\t\t\t\t\t\t_offset.fromBufferAttribute( position, j );\n\t\t\t\t\t\t\t_vector.add( _offset );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tmaxRadiusSq = Math.max( maxRadiusSq, center.distanceToSquared( _vector ) );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tthis.boundingSphere.radius = Math.sqrt( maxRadiusSq );\n\n\t\t\tif ( isNaN( this.boundingSphere.radius ) ) {\n\n\t\t\t\tconsole.error( 'THREE.BufferGeometry.computeBoundingSphere(): Computed radius is NaN. The \"position\" attribute is likely to have NaN values.', this );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tcomputeTangents() {\n\n\t\tconst index = this.index;\n\t\tconst attributes = this.attributes;\n\n\t\t// based on http://www.terathon.com/code/tangent.html\n\t\t// (per vertex tangents)\n\n\t\tif ( index === null ||\n\t\t\t attributes.position === undefined ||\n\t\t\t attributes.normal === undefined ||\n\t\t\t attributes.uv === undefined ) {\n\n\t\t\tconsole.error( 'THREE.BufferGeometry: .computeTangents() failed. Missing required attributes (index, position, normal or uv)' );\n\t\t\treturn;\n\n\t\t}\n\n\t\tconst positionAttribute = attributes.position;\n\t\tconst normalAttribute = attributes.normal;\n\t\tconst uvAttribute = attributes.uv;\n\n\t\tif ( this.hasAttribute( 'tangent' ) === false ) {\n\n\t\t\tthis.setAttribute( 'tangent', new BufferAttribute( new Float32Array( 4 * positionAttribute.count ), 4 ) );\n\n\t\t}\n\n\t\tconst tangentAttribute = this.getAttribute( 'tangent' );\n\n\t\tconst tan1 = [], tan2 = [];\n\n\t\tfor ( let i = 0; i < positionAttribute.count; i ++ ) {\n\n\t\t\ttan1[ i ] = new Vector3();\n\t\t\ttan2[ i ] = new Vector3();\n\n\t\t}\n\n\t\tconst vA = new Vector3(),\n\t\t\tvB = new Vector3(),\n\t\t\tvC = new Vector3(),\n\n\t\t\tuvA = new Vector2(),\n\t\t\tuvB = new Vector2(),\n\t\t\tuvC = new Vector2(),\n\n\t\t\tsdir = new Vector3(),\n\t\t\ttdir = new Vector3();\n\n\t\t\nfunction handleTriangle( a, b, c ) {\n\n\t\t\tvA.fromBufferAttribute( positionAttribute, a );\n\t\t\tvB.fromBufferAttribute( positionAttribute, b );\n\t\t\tvC.fromBufferAttribute( positionAttribute, c );\n\n\t\t\tuvA.fromBufferAttribute( uvAttribute, a );\n\t\t\tuvB.fromBufferAttribute( uvAttribute, b );\n\t\t\tuvC.fromBufferAttribute( uvAttribute, c );\n\n\t\t\tvB.sub( vA );\n\t\t\tvC.sub( vA );\n\n\t\t\tuvB.sub( uvA );\n\t\t\tuvC.sub( uvA );\n\n\t\t\tconst r = 1.0 / ( uvB.x * uvC.y - uvC.x * uvB.y );\n\n\t\t\t// silently ignore degenerate uv triangles having coincident or colinear vertices\n\n\t\t\tif ( ! isFinite( r ) ) return;\n\n\t\t\tsdir.copy( vB ).multiplyScalar( uvC.y ).addScaledVector( vC, - uvB.y ).multiplyScalar( r );\n\t\t\ttdir.copy( vC ).multiplyScalar( uvB.x ).addScaledVector( vB, - uvC.x ).multiplyScalar( r );\n\n\t\t\ttan1[ a ].add( sdir );\n\t\t\ttan1[ b ].add( sdir );\n\t\t\ttan1[ c ].add( sdir );\n\n\t\t\ttan2[ a ].add( tdir );\n\t\t\ttan2[ b ].add( tdir );\n\t\t\ttan2[ c ].add( tdir );\n\n\t\t}\n\n\t\tlet groups = this.groups;\n\n\t\tif ( groups.length === 0 ) {\n\n\t\t\tgroups = [ {\n\t\t\t\tstart: 0,\n\t\t\t\tcount: index.count\n\t\t\t} ];\n\n\t\t}\n\n\t\tfor ( let i = 0, il = groups.length; i < il; ++ i ) {\n\n\t\t\tconst group = groups[ i ];\n\n\t\t\tconst start = group.start;\n\t\t\tconst count = group.count;\n\n\t\t\tfor ( let j = start, jl = start + count; j < jl; j += 3 ) {\n\n\t\t\t\thandleTriangle(\n\t\t\t\t\tindex.getX( j + 0 ),\n\t\t\t\t\tindex.getX( j + 1 ),\n\t\t\t\t\tindex.getX( j + 2 )\n\t\t\t\t);\n\n\t\t\t}\n\n\t\t}\n\n\t\tconst tmp = new Vector3(), tmp2 = new Vector3();\n\t\tconst n = new Vector3(), n2 = new Vector3();\n\n\t\tfunction handleVertex( v ) {\n\n\t\t\tn.fromBufferAttribute( normalAttribute, v );\n\t\t\tn2.copy( n );\n\n\t\t\tconst t = tan1[ v ];\n\n\t\t\t// Gram-Schmidt orthogonalize\n\n\t\t\ttmp.copy( t );\n\t\t\ttmp.sub( n.multiplyScalar( n.dot( t ) ) ).normalize();\n\n\t\t\t// Calculate handedness\n\n\t\t\ttmp2.crossVectors( n2, t );\n\t\t\tconst test = tmp2.dot( tan2[ v ] );\n\t\t\tconst w = ( test < 0.0 ) ? - 1.0 : 1.0;\n\n\t\t\ttangentAttribute.setXYZW( v, tmp.x, tmp.y, tmp.z, w );\n\n\t\t}\n\n\t\tfor ( let i = 0, il = groups.length; i < il; ++ i ) {\n\n\t\t\tconst group = groups[ i ];\n\n\t\t\tconst start = group.start;\n\t\t\tconst count = group.count;\n\n\t\t\tfor ( let j = start, jl = start + count; j < jl; j += 3 ) {\n\n\t\t\t\thandleVertex( index.getX( j + 0 ) );\n\t\t\t\thandleVertex( index.getX( j + 1 ) );\n\t\t\t\thandleVertex( index.getX( j + 2 ) );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tcomputeVertexNormals() {\n\n\t\tconst index = this.index;\n\t\tconst positionAttribute = this.getAttribute( 'position' );\n\n\t\tif ( positionAttribute !== undefined ) {\n\n\t\t\tlet normalAttribute = this.getAttribute( 'normal' );\n\n\t\t\tif ( normalAttribute === undefined ) {\n\n\t\t\t\tnormalAttribute = new BufferAttribute( new Float32Array( positionAttribute.count * 3 ), 3 );\n\t\t\t\tthis.setAttribute( 'normal', normalAttribute );\n\n\t\t\t} else {\n\n\t\t\t\t// reset existing normals to zero\n\n\t\t\t\tfor ( let i = 0, il = normalAttribute.count; i < il; i ++ ) {\n\n\t\t\t\t\tnormalAttribute.setXYZ( i, 0, 0, 0 );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tconst pA = new Vector3(), pB = new Vector3(), pC = new Vector3();\n\t\t\tconst nA = new Vector3(), nB = new Vector3(), nC = new Vector3();\n\t\t\tconst cb = new Vector3(), ab = new Vector3();\n\n\t\t\t// indexed elements\n\n\t\t\tif ( index ) {\n\n\t\t\t\tfor ( let i = 0, il = index.count; i < il; i += 3 ) {\n\n\t\t\t\t\tconst vA = index.getX( i + 0 );\n\t\t\t\t\tconst vB = index.getX( i + 1 );\n\t\t\t\t\tconst vC = index.getX( i + 2 );\n\n\t\t\t\t\tpA.fromBufferAttribute( positionAttribute, vA );\n\t\t\t\t\tpB.fromBufferAttribute( positionAttribute, vB );\n\t\t\t\t\tpC.fromBufferAttribute( positionAttribute, vC );\n\n\t\t\t\t\tcb.subVectors( pC, pB );\n\t\t\t\t\tab.subVectors( pA, pB );\n\t\t\t\t\tcb.cross( ab );\n\n\t\t\t\t\tnA.fromBufferAttribute( normalAttribute, vA );\n\t\t\t\t\tnB.fromBufferAttribute( normalAttribute, vB );\n\t\t\t\t\tnC.fromBufferAttribute( normalAttribute, vC );\n\n\t\t\t\t\tnA.add( cb );\n\t\t\t\t\tnB.add( cb );\n\t\t\t\t\tnC.add( cb );\n\n\t\t\t\t\tnormalAttribute.setXYZ( vA, nA.x, nA.y, nA.z );\n\t\t\t\t\tnormalAttribute.setXYZ( vB, nB.x, nB.y, nB.z );\n\t\t\t\t\tnormalAttribute.setXYZ( vC, nC.x, nC.y, nC.z );\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\t// non-indexed elements (unconnected triangle soup)\n\n\t\t\t\tfor ( let i = 0, il = positionAttribute.count; i < il; i += 3 ) {\n\n\t\t\t\t\tpA.fromBufferAttribute( positionAttribute, i + 0 );\n\t\t\t\t\tpB.fromBufferAttribute( positionAttribute, i + 1 );\n\t\t\t\t\tpC.fromBufferAttribute( positionAttribute, i + 2 );\n\n\t\t\t\t\tcb.subVectors( pC, pB );\n\t\t\t\t\tab.subVectors( pA, pB );\n\t\t\t\t\tcb.cross( ab );\n\n\t\t\t\t\tnormalAttribute.setXYZ( i + 0, cb.x, cb.y, cb.z );\n\t\t\t\t\tnormalAttribute.setXYZ( i + 1, cb.x, cb.y, cb.z );\n\t\t\t\t\tnormalAttribute.setXYZ( i + 2, cb.x, cb.y, cb.z );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tthis.normalizeNormals();\n\n\t\t\tnormalAttribute.needsUpdate = true;\n\n\t\t}\n\n\t}\n\n\tnormalizeNormals() {\n\n\t\tconst normals = this.attributes.normal;\n\n\t\tfor ( let i = 0, il = normals.count; i < il; i ++ ) {\n\n\t\t\t_vector.fromBufferAttribute( normals, i );\n\n\t\t\t_vector.normalize();\n\n\t\t\tnormals.setXYZ( i, _vector.x, _vector.y, _vector.z );\n\n\t\t}\n\n\t}\n\n\ttoNonIndexed() {\n\n\t\tfunction convertBufferAttribute( attribute, indices ) {\n\n\t\t\tconst array = attribute.array;\n\t\t\tconst itemSize = attribute.itemSize;\n\t\t\tconst normalized = attribute.normalized;\n\n\t\t\tconst array2 = new array.constructor( indices.length * itemSize );\n\n\t\t\tlet index = 0, index2 = 0;\n\n\t\t\tfor ( let i = 0, l = indices.length; i < l; i ++ ) {\n\n\t\t\t\tif ( attribute.isInterleavedBufferAttribute ) {\n\n\t\t\t\t\tindex = indices[ i ] * attribute.data.stride + attribute.offset;\n\n\t\t\t\t} else {\n\n\t\t\t\t\tindex = indices[ i ] * itemSize;\n\n\t\t\t\t}\n\n\t\t\t\tfor ( let j = 0; j < itemSize; j ++ ) {\n\n\t\t\t\t\tarray2[ index2 ++ ] = array[ index ++ ];\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\treturn new BufferAttribute( array2, itemSize, normalized );\n\n\t\t}\n\n\t\t//\n\n\t\tif ( this.index === null ) {\n\n\t\t\tconsole.warn( 'THREE.BufferGeometry.toNonIndexed(): BufferGeometry is already non-indexed.' );\n\t\t\treturn this;\n\n\t\t}\n\n\t\tconst geometry2 = new BufferGeometry();\n\n\t\tconst indices = this.index.array;\n\t\tconst attributes = this.attributes;\n\n\t\t// attributes\n\n\t\tfor ( const name in attributes ) {\n\n\t\t\tconst attribute = attributes[ name ];\n\n\t\t\tconst newAttribute = convertBufferAttribute( attribute, indices );\n\n\t\t\tgeometry2.setAttribute( name, newAttribute );\n\n\t\t}\n\n\t\t// morph attributes\n\n\t\tconst morphAttributes = this.morphAttributes;\n\n\t\tfor ( const name in morphAttributes ) {\n\n\t\t\tconst morphArray = [];\n\t\t\tconst morphAttribute = morphAttributes[ name ]; // morphAttribute: array of Float32BufferAttributes\n\n\t\t\tfor ( let i = 0, il = morphAttribute.length; i < il; i ++ ) {\n\n\t\t\t\tconst attribute = morphAttribute[ i ];\n\n\t\t\t\tconst newAttribute = convertBufferAttribute( attribute, indices );\n\n\t\t\t\tmorphArray.push( newAttribute );\n\n\t\t\t}\n\n\t\t\tgeometry2.morphAttributes[ name ] = morphArray;\n\n\t\t}\n\n\t\tgeometry2.morphTargetsRelative = this.morphTargetsRelative;\n\n\t\t// groups\n\n\t\tconst groups = this.groups;\n\n\t\tfor ( let i = 0, l = groups.length; i < l; i ++ ) {\n\n\t\t\tconst group = groups[ i ];\n\t\t\tgeometry2.addGroup( group.start, group.count, group.materialIndex );\n\n\t\t}\n\n\t\treturn geometry2;\n\n\t}\n\n\ttoJSON() {\n\n\t\tconst data = {\n\t\t\tmetadata: {\n\t\t\t\tversion: 4.6,\n\t\t\t\ttype: 'BufferGeometry',\n\t\t\t\tgenerator: 'BufferGeometry.toJSON'\n\t\t\t}\n\t\t};\n\n\t\t// standard BufferGeometry serialization\n\n\t\tdata.uuid = this.uuid;\n\t\tdata.type = this.type;\n\t\tif ( this.name !== '' ) data.name = this.name;\n\t\tif ( Object.keys( this.userData ).length > 0 ) data.userData = this.userData;\n\n\t\tif ( this.parameters !== undefined ) {\n\n\t\t\tconst parameters = this.parameters;\n\n\t\t\tfor ( const key in parameters ) {\n\n\t\t\t\tif ( parameters[ key ] !== undefined ) data[ key ] = parameters[ key ];\n\n\t\t\t}\n\n\t\t\treturn data;\n\n\t\t}\n\n\t\t// for simplicity the code assumes attributes are not shared across geometries, see #15811\n\n\t\tdata.data = { attributes: {} };\n\n\t\tconst index = this.index;\n\n\t\tif ( index !== null ) {\n\n\t\t\tdata.data.index = {\n\t\t\t\ttype: index.array.constructor.name,\n\t\t\t\tarray: Array.prototype.slice.call( index.array )\n\t\t\t};\n\n\t\t}\n\n\t\tconst attributes = this.attributes;\n\n\t\tfor ( const key in attributes ) {\n\n\t\t\tconst attribute = attributes[ key ];\n\n\t\t\tdata.data.attributes[ key ] = attribute.toJSON( data.data );\n\n\t\t}\n\n\t\tconst morphAttributes = {};\n\t\tlet hasMorphAttributes = false;\n\n\t\tfor ( const key in this.morphAttributes ) {\n\n\t\t\tconst attributeArray = this.morphAttributes[ key ];\n\n\t\t\tconst array = [];\n\n\t\t\tfor ( let i = 0, il = attributeArray.length; i < il; i ++ ) {\n\n\t\t\t\tconst attribute = attributeArray[ i ];\n\n\t\t\t\tarray.push( attribute.toJSON( data.data ) );\n\n\t\t\t}\n\n\t\t\tif ( array.length > 0 ) {\n\n\t\t\t\tmorphAttributes[ key ] = array;\n\n\t\t\t\thasMorphAttributes = true;\n\n\t\t\t}\n\n\t\t}\n\n\t\tif ( hasMorphAttributes ) {\n\n\t\t\tdata.data.morphAttributes = morphAttributes;\n\t\t\tdata.data.morphTargetsRelative = this.morphTargetsRelative;\n\n\t\t}\n\n\t\tconst groups = this.groups;\n\n\t\tif ( groups.length > 0 ) {\n\n\t\t\tdata.data.groups = JSON.parse( JSON.stringify( groups ) );\n\n\t\t}\n\n\t\tconst boundingSphere = this.boundingSphere;\n\n\t\tif ( boundingSphere !== null ) {\n\n\t\t\tdata.data.boundingSphere = {\n\t\t\t\tcenter: boundingSphere.center.toArray(),\n\t\t\t\tradius: boundingSphere.radius\n\t\t\t};\n\n\t\t}\n\n\t\treturn data;\n\n\t}\n\n\tclone() {\n\n\t\treturn new this.constructor().copy( this );\n\n\t}\n\n\tcopy( source ) {\n\n\t\t// reset\n\n\t\tthis.index = null;\n\t\tthis.attributes = {};\n\t\tthis.morphAttributes = {};\n\t\tthis.groups = [];\n\t\tthis.boundingBox = null;\n\t\tthis.boundingSphere = null;\n\n\t\t// used for storing cloned, shared data\n\n\t\tconst data = {};\n\n\t\t// name\n\n\t\tthis.name = source.name;\n\n\t\t// index\n\n\t\tconst index = source.index;\n\n\t\tif ( index !== null ) {\n\n\t\t\tthis.setIndex( index.clone( data ) );\n\n\t\t}\n\n\t\t// attributes\n\n\t\tconst attributes = source.attributes;\n\n\t\tfor ( const name in attributes ) {\n\n\t\t\tconst attribute = attributes[ name ];\n\t\t\tthis.setAttribute( name, attribute.clone( data ) );\n\n\t\t}\n\n\t\t// morph attributes\n\n\t\tconst morphAttributes = source.morphAttributes;\n\n\t\tfor ( const name in morphAttributes ) {\n\n\t\t\tconst array = [];\n\t\t\tconst morphAttribute = morphAttributes[ name ]; // morphAttribute: array of Float32BufferAttributes\n\n\t\t\tfor ( let i = 0, l = morphAttribute.length; i < l; i ++ ) {\n\n\t\t\t\tarray.push( morphAttribute[ i ].clone( data ) );\n\n\t\t\t}\n\n\t\t\tthis.morphAttributes[ name ] = array;\n\n\t\t}\n\n\t\tthis.morphTargetsRelative = source.morphTargetsRelative;\n\n\t\t// groups\n\n\t\tconst groups = source.groups;\n\n\t\tfor ( let i = 0, l = groups.length; i < l; i ++ ) {\n\n\t\t\tconst group = groups[ i ];\n\t\t\tthis.addGroup( group.start, group.count, group.materialIndex );\n\n\t\t}\n\n\t\t// bounding box\n\n\t\tconst boundingBox = source.boundingBox;\n\n\t\tif ( boundingBox !== null ) {\n\n\t\t\tthis.boundingBox = boundingBox.clone();\n\n\t\t}\n\n\t\t// bounding sphere\n\n\t\tconst boundingSphere = source.boundingSphere;\n\n\t\tif ( boundingSphere !== null ) {\n\n\t\t\tthis.boundingSphere = boundingSphere.clone();\n\n\t\t}\n\n\t\t// draw range\n\n\t\tthis.drawRange.start = source.drawRange.start;\n\t\tthis.drawRange.count = source.drawRange.count;\n\n\t\t// user data\n\n\t\tthis.userData = source.userData;\n\n\t\treturn this;\n\n\t}\n\n\tdispose() {\n\n\t\tthis.dispatchEvent( { type: 'dispose' } );\n\n\t}\n\n}\n\nexport { BufferGeometry };\n\n// Path: src/geometries/PolyhedronGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Vector2 } from '../math/Vector2.js';\n\nclass PolyhedronGeometry extends BufferGeometry {\n\n\tconstructor( vertices = [], indices = [], radius = 1, detail = 0 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'PolyhedronGeometry';\n\n\t\tthis.parameters = {\n\t\t\tvertices: vertices,\n\t\t\tindices: indices,\n\t\t\tradius: radius,\n\t\t\tdetail: detail\n\t\t};\n\n\t\t// default buffer data\n\n\t\tconst vertexBuffer = [];\n\t\tconst uvBuffer = [];\n\n\t\t// the subdivision creates the vertex buffer data\n\n\t\tsubdivide( detail );\n\n\t\t// all vertices should lie on a conceptual sphere with a given radius\n\n\t\tapplyRadius( radius );\n\n\t\t// finally, create the uv data\n\n\t\tgenerateUVs();\n\n\t\t// build non-indexed geometry\n\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertexBuffer, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( vertexBuffer.slice(), 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvBuffer, 2 ) );\n\n\t\tif ( detail === 0 ) {\n\n\t\t\tthis.computeVertexNormals(); // flat normals\n\n\t\t} else {\n\n\t\t\tthis.normalizeNormals(); // smooth normals\n\n\t\t}\n\n\t\t// helper functions\n\n\t\tfunction subdivide( detail ) {\n\n\t\t\tconst a = new Vector3();\n\t\t\tconst b = new Vector3();\n\t\t\tconst c = new Vector3();\n\n\t\t\t// iterate over all faces and apply a subdivision with the given detail value\n\n\t\t\tfor ( let i = 0; i < indices.length; i += 3 ) {\n\n\t\t\t\t// get the vertices of the face\n\n\t\t\t\tgetVertexByIndex( indices[ i + 0 ], a );\n\t\t\t\tgetVertexByIndex( indices[ i + 1 ], b );\n\t\t\t\tgetVertexByIndex( indices[ i + 2 ], c );\n\n\t\t\t\t// perform subdivision\n\n\t\t\t\tsubdivideFace( a, b, c, detail );\n\n\t\t\t}\n\n\t\t}\n\n\t\tfunction subdivideFace( a, b, c, detail ) {\n\n\t\t\tconst cols = detail + 1;\n\n\t\t\t// we use this multidimensional array as a data structure for creating the subdivision\n\n\t\t\tconst v = [];\n\n\t\t\t// construct all of the vertices for this subdivision\n\n\t\t\tfor ( let i = 0; i <= cols; i ++ ) {\n\n\t\t\t\tv[ i ] = [];\n\n\t\t\t\tconst aj = a.clone().lerp( c, i / cols );\n\t\t\t\tconst bj = b.clone().lerp( c, i / cols );\n\n\t\t\t\tconst rows = cols - i;\n\n\t\t\t\tfor ( let j = 0; j <= rows; j ++ ) {\n\n\t\t\t\t\tif ( j === 0 && i === cols ) {\n\n\t\t\t\t\t\tv[ i ][ j ] = aj;\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tv[ i ][ j ] = aj.clone().lerp( bj, j / rows );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// construct all of the faces\n\n\t\t\tfor ( let i = 0; i < cols; i ++ ) {\n\n\t\t\t\tfor ( let j = 0; j < 2 * ( cols - i ) - 1; j ++ ) {\n\n\t\t\t\t\tconst k = Math.floor( j / 2 );\n\n\t\t\t\t\tif ( j % 2 === 0 ) {\n\n\t\t\t\t\t\tpushVertex( v[ i ][ k + 1 ] );\n\t\t\t\t\t\tpushVertex( v[ i + 1 ][ k ] );\n\t\t\t\t\t\tpushVertex( v[ i ][ k ] );\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tpushVertex( v[ i ][ k + 1 ] );\n\t\t\t\t\t\tpushVertex( v[ i + 1 ][ k + 1 ] );\n\t\t\t\t\t\tpushVertex( v[ i + 1 ][ k ] );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\tfunction applyRadius( radius ) {\n\n\t\t\tconst vertex = new Vector3();\n\n\t\t\t// iterate over the entire buffer and apply the radius to each vertex\n\n\t\t\tfor ( let i = 0; i < vertexBuffer.length; i += 3 ) {\n\n\t\t\t\tvertex.x = vertexBuffer[ i + 0 ];\n\t\t\t\tvertex.y = vertexBuffer[ i + 1 ];\n\t\t\t\tvertex.z = vertexBuffer[ i + 2 ];\n\n\t\t\t\tvertex.normalize().multiplyScalar( radius );\n\n\t\t\t\tvertexBuffer[ i + 0 ] = vertex.x;\n\t\t\t\tvertexBuffer[ i + 1 ] = vertex.y;\n\t\t\t\tvertexBuffer[ i + 2 ] = vertex.z;\n\n\t\t\t}\n\n\t\t}\n\n\t\tfunction generateUVs() {\n\n\t\t\tconst vertex = new Vector3();\n\n\t\t\tfor ( let i = 0; i < vertexBuffer.length; i += 3 ) {\n\n\t\t\t\tvertex.x = vertexBuffer[ i + 0 ];\n\t\t\t\tvertex.y = vertexBuffer[ i + 1 ];\n\t\t\t\tvertex.z = vertexBuffer[ i + 2 ];\n\n\t\t\t\tconst u = azimuth( vertex ) / 2 / Math.PI + 0.5;\n\t\t\t\tconst v = inclination( vertex ) / Math.PI + 0.5;\n\t\t\t\tuvBuffer.push( u, 1 - v );\n\n\t\t\t}\n\n\t\t\tcorrectUVs();\n\n\t\t\tcorrectSeam();\n\n\t\t}\n\n\t\tfunction correctSeam() {\n\n\t\t\t// handle case when face straddles the seam, see #3269\n\n\t\t\tfor ( let i = 0; i < uvBuffer.length; i += 6 ) {\n\n\t\t\t\t// uv data of a single face\n\n\t\t\t\tconst x0 = uvBuffer[ i + 0 ];\n\t\t\t\tconst x1 = uvBuffer[ i + 2 ];\n\t\t\t\tconst x2 = uvBuffer[ i + 4 ];\n\n\t\t\t\tconst max = Math.max( x0, x1, x2 );\n\t\t\t\tconst min = Math.min( x0, x1, x2 );\n\n\t\t\t\t// 0.9 is somewhat arbitrary\n\n\t\t\t\tif ( max > 0.9 && min < 0.1 ) {\n\n\t\t\t\t\tif ( x0 < 0.2 ) uvBuffer[ i + 0 ] += 1;\n\t\t\t\t\tif ( x1 < 0.2 ) uvBuffer[ i + 2 ] += 1;\n\t\t\t\t\tif ( x2 < 0.2 ) uvBuffer[ i + 4 ] += 1;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\tfunction pushVertex( vertex ) {\n\n\t\t\tvertexBuffer.push( vertex.x, vertex.y, vertex.z );\n\n\t\t}\n\n\t\tfunction getVertexByIndex( index, vertex ) {\n\n\t\t\tconst stride = index * 3;\n\n\t\t\tvertex.x = vertices[ stride + 0 ];\n\t\t\tvertex.y = vertices[ stride + 1 ];\n\t\t\tvertex.z = vertices[ stride + 2 ];\n\n\t\t}\n\n\t\tfunction correctUVs() {\n\n\t\t\tconst a = new Vector3();\n\t\t\tconst b = new Vector3();\n\t\t\tconst c = new Vector3();\n\n\t\t\tconst centroid = new Vector3();\n\n\t\t\tconst uvA = new Vector2();\n\t\t\tconst uvB = new Vector2();\n\t\t\tconst uvC = new Vector2();\n\n\t\t\tfor ( let i = 0, j = 0; i < vertexBuffer.length; i += 9, j += 6 ) {\n\n\t\t\t\ta.set( vertexBuffer[ i + 0 ], vertexBuffer[ i + 1 ], vertexBuffer[ i + 2 ] );\n\t\t\t\tb.set( vertexBuffer[ i + 3 ], vertexBuffer[ i + 4 ], vertexBuffer[ i + 5 ] );\n\t\t\t\tc.set( vertexBuffer[ i + 6 ], vertexBuffer[ i + 7 ], vertexBuffer[ i + 8 ] );\n\n\t\t\t\tuvA.set( uvBuffer[ j + 0 ], uvBuffer[ j + 1 ] );\n\t\t\t\tuvB.set( uvBuffer[ j + 2 ], uvBuffer[ j + 3 ] );\n\t\t\t\tuvC.set( uvBuffer[ j + 4 ], uvBuffer[ j + 5 ] );\n\n\t\t\t\tcentroid.copy( a ).add( b ).add( c ).divideScalar( 3 );\n\n\t\t\t\tconst azi = azimuth( centroid );\n\n\t\t\t\tcorrectUV( uvA, j + 0, a, azi );\n\t\t\t\tcorrectUV( uvB, j + 2, b, azi );\n\t\t\t\tcorrectUV( uvC, j + 4, c, azi );\n\n\t\t\t}\n\n\t\t}\n\n\t\tfunction correctUV( uv, stride, vector, azimuth ) {\n\n\t\t\tif ( ( azimuth < 0 ) && ( uv.x === 1 ) ) {\n\n\t\t\t\tuvBuffer[ stride ] = uv.x - 1;\n\n\t\t\t}\n\n\t\t\tif ( ( vector.x === 0 ) && ( vector.z === 0 ) ) {\n\n\t\t\t\tuvBuffer[ stride ] = azimuth / 2 / Math.PI + 0.5;\n\n\t\t\t}\n\n\t\t}\n\n\t\t// Angle around the Y axis, counter-clockwise when looking from above.\n\n\t\tfunction azimuth( vector ) {\n\n\t\t\treturn Math.atan2( vector.z, - vector.x );\n\n\t\t}\n\n\n\t\t// Angle above the XZ plane.\n\n\t\tfunction inclination( vector ) {\n\n\t\t\treturn Math.atan2( - vector.y, Math.sqrt( ( vector.x * vector.x ) + ( vector.z * vector.z ) ) );\n\n\t\t}\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new PolyhedronGeometry( data.vertices, data.indices, data.radius, data.details );\n\n\t}\n\n}\n\nexport { PolyhedronGeometry };\n\n// Path: src/geometries/OctahedronGeometry.js\nimport { PolyhedronGeometry } from './PolyhedronGeometry.js';\n\nclass OctahedronGeometry extends PolyhedronGeometry {\n\n\tconstructor( radius = 1, detail = 0 ) {\n\n\t\tconst vertices = [\n\t\t\t1, 0, 0, \t- 1, 0, 0,\t0, 1, 0,\n\t\t\t0, - 1, 0, \t0, 0, 1,\t0, 0, - 1\n\t\t];\n\n\t\tconst indices = [\n\t\t\t0, 2, 4,\t0, 4, 3,\t0, 3, 5,\n\t\t\t0, 5, 2,\t1, 2, 5,\t1, 5, 3,\n\t\t\t1, 3, 4,\t1, 4, 2\n\t\t];\n\n\t\tsuper( vertices, indices, radius, detail );\n\n\t\tthis.type = 'OctahedronGeometry';\n\n\t\tthis.parameters = {\n\t\t\tradius: radius,\n\t\t\tdetail: detail\n\t\t};\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new OctahedronGeometry( data.radius, data.detail );\n\n\t}\n\n}\n\nexport { OctahedronGeometry };\n\n// Path: src/math/Color.js\nimport { clamp, euclideanModulo, lerp } from './MathUtils.js';\nimport { ColorManagement, SRGBToLinear, LinearToSRGB } from './ColorManagement.js';\nimport { SRGBColorSpace } from '../constants.js';\n\nconst _colorKeywords = { 'aliceblue': 0xF0F8FF, 'antiquewhite': 0xFAEBD7, 'aqua': 0x00FFFF, 'aquamarine': 0x7FFFD4, 'azure': 0xF0FFFF,\n\t'beige': 0xF5F5DC, 'bisque': 0xFFE4C4, 'black': 0x000000, 'blanchedalmond': 0xFFEBCD, 'blue': 0x0000FF, 'blueviolet': 0x8A2BE2,\n\t'brown': 0xA52A2A, 'burlywood': 0xDEB887, 'cadetblue': 0x5F9EA0, 'chartreuse': 0x7FFF00, 'chocolate': 0xD2691E, 'coral': 0xFF7F50,\n\t'cornflowerblue': 0x6495ED, 'cornsilk': 0xFFF8DC, 'crimson': 0xDC143C, 'cyan': 0x00FFFF, 'darkblue': 0x00008B, 'darkcyan': 0x008B8B,\n\t'darkgoldenrod': 0xB8860B, 'darkgray': 0xA9A9A9, 'darkgreen': 0x006400, 'darkgrey': 0xA9A9A9, 'darkkhaki': 0xBDB76B, 'darkmagenta': 0x8B008B,\n\t'darkolivegreen': 0x556B2F, 'darkorange': 0xFF8C00, 'darkorchid': 0x9932CC, 'darkred': 0x8B0000, 'darksalmon': 0xE9967A, 'darkseagreen': 0x8FBC8F,\n\t'darkslateblue': 0x483D8B, 'darkslategray': 0x2F4F4F, 'darkslategrey': 0x2F4F4F, 'darkturquoise': 0x00CED1, 'darkviolet': 0x9400D3,\n\t'deeppink': 0xFF1493, 'deepskyblue': 0x00BFFF, 'dimgray': 0x696969, 'dimgrey': 0x696969, 'dodgerblue': 0x1E90FF, 'firebrick': 0xB22222,\n\t'floralwhite': 0xFFFAF0, 'forestgreen': 0x228B22, 'fuchsia': 0xFF00FF, 'gainsboro': 0xDCDCDC, 'ghostwhite': 0xF8F8FF, 'gold': 0xFFD700,\n\t'goldenrod': 0xDAA520, 'gray': 0x808080, 'green': 0x008000, 'greenyellow': 0xADFF2F, 'grey': 0x808080, 'honeydew': 0xF0FFF0, 'hotpink': 0xFF69B4,\n\t'indianred': 0xCD5C5C, 'indigo': 0x4B0082, 'ivory': 0xFFFFF0, 'khaki': 0xF0E68C, 'lavender': 0xE6E6FA, 'lavenderblush': 0xFFF0F5, 'lawngreen': 0x7CFC00,\n\t'lemonchiffon': 0xFFFACD, 'lightblue': 0xADD8E6, 'lightcoral': 0xF08080, 'lightcyan': 0xE0FFFF, 'lightgoldenrodyellow': 0xFAFAD2, 'lightgray': 0xD3D3D3,\n\t'lightgreen': 0x90EE90, 'lightgrey': 0xD3D3D3, 'lightpink': 0xFFB6C1, 'lightsalmon': 0xFFA07A, 'lightseagreen': 0x20B2AA, 'lightskyblue': 0x87CEFA,\n\t'lightslategray': 0x778899, 'lightslategrey': 0x778899, 'lightsteelblue': 0xB0C4DE, 'lightyellow': 0xFFFFE0, 'lime': 0x00FF00, 'limegreen': 0x32CD32,\n\t'linen': 0xFAF0E6, 'magenta': 0xFF00FF, 'maroon': 0x800000, 'mediumaquamarine': 0x66CDAA, 'mediumblue': 0x0000CD, 'mediumorchid': 0xBA55D3,\n\t'mediumpurple': 0x9370DB, 'mediumseagreen': 0x3CB371, 'mediumslateblue': 0x7B68EE, 'mediumspringgreen': 0x00FA9A, 'mediumturquoise': 0x48D1CC,\n\t'mediumvioletred': 0xC71585, 'midnightblue': 0x191970, 'mintcream': 0xF5FFFA, 'mistyrose': 0xFFE4E1, 'moccasin': 0xFFE4B5, 'navajowhite': 0xFFDEAD,\n\t'navy': 0x000080, 'oldlace': 0xFDF5E6, 'olive': 0x808000, 'olivedrab': 0x6B8E23, 'orange': 0xFFA500, 'orangered': 0xFF4500, 'orchid': 0xDA70D6,\n\t'palegoldenrod': 0xEEE8AA, 'palegreen': 0x98FB98, 'paleturquoise': 0xAFEEEE, 'palevioletred': 0xDB7093, 'papayawhip': 0xFFEFD5, 'peachpuff': 0xFFDAB9,\n\t'peru': 0xCD853F, 'pink': 0xFFC0CB, 'plum': 0xDDA0DD, 'powderblue': 0xB0E0E6, 'purple': 0x800080, 'rebeccapurple': 0x663399, 'red': 0xFF0000, 'rosybrown': 0xBC8F8F,\n\t'royalblue': 0x4169E1, 'saddlebrown': 0x8B4513, 'salmon': 0xFA8072, 'sandybrown': 0xF4A460, 'seagreen': 0x2E8B57, 'seashell': 0xFFF5EE,\n\t'sienna': 0xA0522D, 'silver': 0xC0C0C0, 'skyblue': 0x87CEEB, 'slateblue': 0x6A5ACD, 'slategray': 0x708090, 'slategrey': 0x708090, 'snow': 0xFFFAFA,\n\t'springgreen': 0x00FF7F, 'steelblue': 0x4682B4, 'tan': 0xD2B48C, 'teal': 0x008080, 'thistle': 0xD8BFD8, 'tomato': 0xFF6347, 'turquoise': 0x40E0D0,\n\t'violet': 0xEE82EE, 'wheat': 0xF5DEB3, 'white': 0xFFFFFF, 'whitesmoke': 0xF5F5F5, 'yellow': 0xFFFF00, 'yellowgreen': 0x9ACD32 };\n\nconst _hslA = { h: 0, s: 0, l: 0 };\nconst _hslB = { h: 0, s: 0, l: 0 };\n\nfunction hue2rgb( p, q, t ) {\n\n...\n", "needle_token_start": 4857, "needle_token_end": 5289, "code_context_ntokens": 14834, "output": ["```javascript\nfunction handleTriangle( a, b, c ) {\n\n\tvA.fromBufferAttribute( positionAttribute, a );\n\tvB.fromBufferAttribute( positionAttribute, b );\n\tvC.fromBufferAttribute( positionAttribute, c );\n\n\tuvA.fromBufferAttribute( uvAttribute, a );\n\tuvB.fromBufferAttribute( uvAttribute, b );\n\tuvC.fromBufferAttribute( uvAttribute, c );\n\n\tvB.sub( vA );\n\tvC.sub( vA );\n\n\tuvB.sub( uvA );\n\tuvC.sub( uvA );\n\n\tconst r = 1.0 / ( uvB.x * uvC.y - uvC.x * uvB.y );\n\n\t// silently ignore degenerate uv triangles having coincident or colinear vertices\n\n\tif ( ! isFinite( r ) ) return;\n\n\tsdir.copy( vB ).multiplyScalar( uvC.y ).addScaledVector( vC, - uvB.y ).multiplyScalar( r );\n\ttdir.copy( vC ).multiplyScalar( uvB.x ).addScaledVector( vB, - uvC.x ).multiplyScalar( r );\n\n\ttan1[ a ].add( sdir );\n\ttan1[ b ].add( sdir );\n\ttan1[ c ].add( sdir );\n\n\ttan2[ a ].add( tdir );\n\ttan2[ b ].add( tdir );\n\ttan2[ c ].add( tdir );\n\n}\n```"]}
{"repo": "mrdoob/three.js", "name": "createCanvas", "language": "typescript", "path": "src/renderers/webgl/WebGLTextures.js", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a canvas element, which can be either an OffscreenCanvas or a regular HTML canvas, depending on the environment's capabilities and availability.\n2. **Input**: Two parameters representing the width and height of the canvas to be created.\n3. **Output**: Returns a canvas object, either an OffscreenCanvas or a regular HTML canvas, based on the execution context and feature support.\n4. **Procedure**: The function first checks if OffscreenCanvas is available and functional for the '2d' context. If so, it creates and returns an OffscreenCanvas with the specified dimensions. If not, it falls back to creating and returning a standard HTML canvas element with the given dimensions.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/renderers/webxr/WebXRManager.js\nimport { ArrayCamera } from '../../cameras/ArrayCamera.js';\nimport { EventDispatcher } from '../../core/EventDispatcher.js';\nimport { PerspectiveCamera } from '../../cameras/PerspectiveCamera.js';\nimport { Vector2 } from '../../math/Vector2.js';\nimport { Vector3 } from '../../math/Vector3.js';\nimport { Vector4 } from '../../math/Vector4.js';\nimport { RAD2DEG } from '../../math/MathUtils.js';\nimport { WebGLAnimation } from '../webgl/WebGLAnimation.js';\nimport { WebGLRenderTarget } from '../WebGLRenderTarget.js';\nimport { WebXRController } from './WebXRController.js';\nimport { DepthTexture } from '../../textures/DepthTexture.js';\nimport { DepthFormat, DepthStencilFormat, RGBAFormat, UnsignedByteType, UnsignedIntType, UnsignedInt248Type } from '../../constants.js';\nimport { WebXRDepthSensing } from './WebXRDepthSensing.js';\n\nclass WebXRManager extends EventDispatcher {\n\n\tconstructor( renderer, gl ) {\n\n\t\tsuper();\n\n\t\tconst scope = this;\n\n\t\tlet session = null;\n\n\t\tlet framebufferScaleFactor = 1.0;\n\n\t\tlet referenceSpace = null;\n\t\tlet referenceSpaceType = 'local-floor';\n\t\t// Set default foveation to maximum.\n\t\tlet foveation = 1.0;\n\t\tlet customReferenceSpace = null;\n\n\t\tlet pose = null;\n\t\tlet glBinding = null;\n\t\tlet glProjLayer = null;\n\t\tlet glBaseLayer = null;\n\t\tlet xrFrame = null;\n\n\t\tconst depthSensing = new WebXRDepthSensing();\n\t\tconst attributes = gl.getContextAttributes();\n\n\t\tlet initialRenderTarget = null;\n\t\tlet newRenderTarget = null;\n\n\t\tconst controllers = [];\n\t\tconst controllerInputSources = [];\n\n\t\tconst currentSize = new Vector2();\n\t\tlet currentPixelRatio = null;\n\n\t\t//\n\n\t\tconst cameraL = new PerspectiveCamera();\n\t\tcameraL.layers.enable( 1 );\n\t\tcameraL.viewport = new Vector4();\n\n\t\tconst cameraR = new PerspectiveCamera();\n\t\tcameraR.layers.enable( 2 );\n\t\tcameraR.viewport = new Vector4();\n\n\t\tconst cameras = [ cameraL, cameraR ];\n\n\t\tconst cameraXR = new ArrayCamera();\n\t\tcameraXR.layers.enable( 1 );\n\t\tcameraXR.layers.enable( 2 );\n\n\t\tlet _currentDepthNear = null;\n\t\tlet _currentDepthFar = null;\n\n\t\t//\n\n\t\tthis.cameraAutoUpdate = true;\n\t\tthis.enabled = false;\n\n\t\tthis.isPresenting = false;\n\n\t\tthis.getController = function ( index ) {\n\n\t\t\tlet controller = controllers[ index ];\n\n\t\t\tif ( controller === undefined ) {\n\n\t\t\t\tcontroller = new WebXRController();\n\t\t\t\tcontrollers[ index ] = controller;\n\n\t\t\t}\n\n\t\t\treturn controller.getTargetRaySpace();\n\n\t\t};\n\n\t\tthis.getControllerGrip = function ( index ) {\n\n\t\t\tlet controller = controllers[ index ];\n\n\t\t\tif ( controller === undefined ) {\n\n\t\t\t\tcontroller = new WebXRController();\n\t\t\t\tcontrollers[ index ] = controller;\n\n\t\t\t}\n\n\t\t\treturn controller.getGripSpace();\n\n\t\t};\n\n\t\tthis.getHand = function ( index ) {\n\n\t\t\tlet controller = controllers[ index ];\n\n\t\t\tif ( controller === undefined ) {\n\n\t\t\t\tcontroller = new WebXRController();\n\t\t\t\tcontrollers[ index ] = controller;\n\n\t\t\t}\n\n\t\t\treturn controller.getHandSpace();\n\n\t\t};\n\n\t\t//\n\n\t\tfunction onSessionEvent( event ) {\n\n\t\t\tconst controllerIndex = controllerInputSources.indexOf( event.inputSource );\n\n\t\t\tif ( controllerIndex === - 1 ) {\n\n\t\t\t\treturn;\n\n\t\t\t}\n\n\t\t\tconst controller = controllers[ controllerIndex ];\n\n\t\t\tif ( controller !== undefined ) {\n\n\t\t\t\tcontroller.update( event.inputSource, event.frame, customReferenceSpace || referenceSpace );\n\t\t\t\tcontroller.dispatchEvent( { type: event.type, data: event.inputSource } );\n\n\t\t\t}\n\n\t\t}\n\n\t\tfunction onSessionEnd() {\n\n\t\t\tsession.removeEventListener( 'select', onSessionEvent );\n\t\t\tsession.removeEventListener( 'selectstart', onSessionEvent );\n\t\t\tsession.removeEventListener( 'selectend', onSessionEvent );\n\t\t\tsession.removeEventListener( 'squeeze', onSessionEvent );\n\t\t\tsession.removeEventListener( 'squeezestart', onSessionEvent );\n...\n// Path: src/renderers/webgl/WebGLUniformsGroups.js\nfunction WebGLUniformsGroups( gl, info, capabilities, state ) {\n\n\tlet buffers = {};\n\tlet updateList = {};\n\tlet allocatedBindingPoints = [];\n\n\tconst maxBindingPoints = gl.getParameter( gl.MAX_UNIFORM_BUFFER_BINDINGS ); // binding points are global whereas block indices are per shader program\n\n\tfunction bind( uniformsGroup, program ) {\n\n\t\tconst webglProgram = program.program;\n\t\tstate.uniformBlockBinding( uniformsGroup, webglProgram );\n\n\t}\n\n\tfunction update( uniformsGroup, program ) {\n\n\t\tlet buffer = buffers[ uniformsGroup.id ];\n\n\t\tif ( buffer === undefined ) {\n\n\t\t\tprepareUniformsGroup( uniformsGroup );\n\n\t\t\tbuffer = createBuffer( uniformsGroup );\n\t\t\tbuffers[ uniformsGroup.id ] = buffer;\n\n\t\t\tuniformsGroup.addEventListener( 'dispose', onUniformsGroupsDispose );\n\n\t\t}\n\n\t\t// ensure to update the binding points/block indices mapping for this program\n\n\t\tconst webglProgram = program.program;\n\t\tstate.updateUBOMapping( uniformsGroup, webglProgram );\n\n\t\t// update UBO once per frame\n\n\t\tconst frame = info.render.frame;\n\n\t\tif ( updateList[ uniformsGroup.id ] !== frame ) {\n\n\t\t\tupdateBufferData( uniformsGroup );\n\n\t\t\tupdateList[ uniformsGroup.id ] = frame;\n\n\t\t}\n\n\t}\n\n\tfunction createBuffer( uniformsGroup ) {\n\n\t\t// the setup of an UBO is independent of a particular shader program but global\n\n\t\tconst bindingPointIndex = allocateBindingPointIndex();\n\t\tuniformsGroup.__bindingPointIndex = bindingPointIndex;\n\n\t\tconst buffer = gl.createBuffer();\n\t\tconst size = uniformsGroup.__size;\n\t\tconst usage = uniformsGroup.usage;\n\n\t\tgl.bindBuffer( gl.UNIFORM_BUFFER, buffer );\n\t\tgl.bufferData( gl.UNIFORM_BUFFER, size, usage );\n\t\tgl.bindBuffer( gl.UNIFORM_BUFFER, null );\n\t\tgl.bindBufferBase( gl.UNIFORM_BUFFER, bindingPointIndex, buffer );\n\n\t\treturn buffer;\n\n\t}\n\n\tfunction allocateBindingPointIndex() {\n\n\t\tfor ( let i = 0; i < maxBindingPoints; i ++ ) {\n\n\t\t\tif ( allocatedBindingPoints.indexOf( i ) === - 1 ) {\n\n\t\t\t\tallocatedBindingPoints.push( i );\n\t\t\t\treturn i;\n\n\t\t\t}\n\n\t\t}\n\n\t\tconsole.error( 'THREE.WebGLRenderer: Maximum number of simultaneously usable uniforms groups reached.' );\n\n\t\treturn 0;\n\n\t}\n\n\tfunction updateBufferData( uniformsGroup ) {\n\n\t\tconst buffer = buffers[ uniformsGroup.id ];\n\t\tconst uniforms = uniformsGroup.uniforms;\n\t\tconst cache = uniformsGroup.__cache;\n\n\t\tgl.bindBuffer( gl.UNIFORM_BUFFER, buffer );\n\n\t\tfor ( let i = 0, il = uniforms.length; i < il; i ++ ) {\n\n\t\t\tconst uniformArray = Array.isArray( uniforms[ i ] ) ? uniforms[ i ] : [ uniforms[ i ] ];\n\n\t\t\tfor ( let j = 0, jl = uniformArray.length; j < jl; j ++ ) {\n\n\t\t\t\tconst uniform = uniformArray[ j ];\n\n\t\t\t\tif ( hasUniformChanged( uniform, i, j, cache ) === true ) {\n\n\t\t\t\t\tconst offset = uniform.__offset;\n\n\t\t\t\t\tconst values = Array.isArray( uniform.value ) ? uniform.value : [ uniform.value ];\n\n\t\t\t\t\tlet arrayOffset = 0;\n\n\t\t\t\t\tfor ( let k = 0; k < values.length; k ++ ) {\n\n\t\t\t\t\t\tconst value = values[ k ];\n\n\t\t\t\t\t\tconst info = getUniformSize( value );\n\n\t\t\t\t\t\t// TODO add integer and struct support\n\t\t\t\t\t\tif ( typeof value === 'number' || typeof value === 'boolean' ) {\n\n\t\t\t\t\t\t\tuniform.__data[ 0 ] = value;\n\t\t\t\t\t\t\tgl.bufferSubData( gl.UNIFORM_BUFFER, offset + arrayOffset, uniform.__data );\n\n\t\t\t\t\t\t} else if ( value.isMatrix3 ) {\n\n\t\t\t\t\t\t\t// manually converting 3x3 to 3x4\n\n\t\t\t\t\t\t\tuniform.__data[ 0 ] = value.elements[ 0 ];\n\t\t\t\t\t\t\tuniform.__data[ 1 ] = value.elements[ 1 ];\n\t\t\t\t\t\t\tuniform.__data[ 2 ] = value.elements[ 2 ];\n\t\t\t\t\t\t\tuniform.__data[ 3 ] = 0;\n\t\t\t\t\t\t\tuniform.__data[ 4 ] = value.elements[ 3 ];\n\t\t\t\t\t\t\tuniform.__data[ 5 ] = value.elements[ 4 ];\n\t\t\t\t\t\t\tuniform.__data[ 6 ] = value.elements[ 5 ];\n\t\t\t\t\t\t\tuniform.__data[ 7 ] = 0;\n\t\t\t\t\t\t\tuniform.__data[ 8 ] = value.elements[ 6 ];\n\t\t\t\t\t\t\tuniform.__data[ 9 ] = value.elements[ 7 ];\n\t\t\t\t\t\t\tuniform.__data[ 10 ] = value.elements[ 8 ];\n\t\t\t\t\t\t\tuniform.__data[ 11 ] = 0;\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tvalue.toArray( uniform.__data, arrayOffset );\n\n\t\t\t\t\t\t\tarrayOffset += info.storage / Float32Array.BYTES_PER_ELEMENT;\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t\tgl.bufferSubData( gl.UNIFORM_BUFFER, offset, uniform.__data );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\tgl.bindBuffer( gl.UNIFORM_BUFFER, null );\n\n\t}\n\n\tfunction hasUniformChanged( uniform, index, indexArray, cache ) {\n\n\t\tconst value = uniform.value;\n\t\tconst indexString = index + '_' + indexArray;\n\n\t\tif ( cache[ indexString ] === undefined ) {\n\n\t\t\t// cache entry does not exist so far\n\n\t\t\tif ( typeof value === 'number' || typeof value === 'boolean' ) {\n\n\t\t\t\tcache[ indexString ] = value;\n\n\t\t\t} else {\n\n\t\t\t\tcache[ indexString ] = value.clone();\n\n\t\t\t}\n\n\t\t\treturn true;\n\n\t\t} else {\n\n\t\t\tconst cachedObject = cache[ indexString ];\n\n\t\t\t// compare current value with cached entry\n\n\t\t\tif ( typeof value === 'number' || typeof value === 'boolean' ) {\n\n\t\t\t\tif ( cachedObject !== value ) {\n\n\t\t\t\t\tcache[ indexString ] = value;\n\t\t\t\t\treturn true;\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\tif ( cachedObject.equals( value ) === false ) {\n\n\t\t\t\t\tcachedObject.copy( value );\n\t\t\t\t\treturn true;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn false;\n\n\t}\n\n\tfunction prepareUniformsGroup( uniformsGroup ) {\n\n\t\t// determine total buffer size according to the STD140 layout\n\t\t// Hint: STD140 is the only supported layout in WebGL 2\n\n\t\tconst uniforms = uniformsGroup.uniforms;\n\n\t\tlet offset = 0; // global buffer offset in bytes\n\t\tconst chunkSize = 16; // size of a chunk in bytes\n\n\t\tfor ( let i = 0, l = uniforms.length; i < l; i ++ ) {\n\n\t\t\tconst uniformArray = Array.isArray( uniforms[ i ] ) ? uniforms[ i ] : [ uniforms[ i ] ];\n\n\t\t\tfor ( let j = 0, jl = uniformArray.length; j < jl; j ++ ) {\n\n\t\t\t\tconst uniform = uniformArray[ j ];\n\n\t\t\t\tconst values = Array.isArray( uniform.value ) ? uniform.value : [ uniform.value ];\n\n\t\t\t\tfor ( let k = 0, kl = values.length; k < kl; k ++ ) {\n\n\t\t\t\t\tconst value = values[ k ];\n\n\t\t\t\t\tconst info = getUniformSize( value );\n\n\t\t\t\t\t// Calculate the chunk offset\n\t\t\t\t\tconst chunkOffsetUniform = offset % chunkSize;\n\n\t\t\t\t\t// Check for chunk overflow\n\t\t\t\t\tif ( chunkOffsetUniform !== 0 && ( chunkSize - chunkOffsetUniform ) < info.boundary ) {\n\n\t\t\t\t\t\t// Add padding and adjust offset\n\t\t\t\t\t\toffset += ( chunkSize - chunkOffsetUniform );\n\n\t\t\t\t\t}\n\n\t\t\t\t\t// the following two properties will be used for partial buffer updates\n\n\t\t\t\t\tuniform.__data = new Float32Array( info.storage / Float32Array.BYTES_PER_ELEMENT );\n\t\t\t\t\tuniform.__offset = offset;\n\n\n\t\t\t\t\t// Update the global offset\n\t\t\t\t\toffset += info.storage;\n\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\t// ensure correct final padding\n\n\t\tconst chunkOffset = offset % chunkSize;\n\n\t\tif ( chunkOffset > 0 ) offset += ( chunkSize - chunkOffset );\n\n\t\t//\n\n\t\tuniformsGroup.__size = offset;\n\t\tuniformsGroup.__cache = {};\n\n\t\treturn this;\n\n\t}\n\n\tfunction getUniformSize( value ) {\n\n\t\tconst info = {\n\t\t\tboundary: 0, // bytes\n\t\t\tstorage: 0 // bytes\n\t\t};\n\n\t\t// determine sizes according to STD140\n\n\t\tif ( typeof value === 'number' || typeof value === 'boolean' ) {\n\n\t\t\t// float/int/bool\n\n\t\t\tinfo.boundary = 4;\n\t\t\tinfo.storage = 4;\n\n\t\t} else if ( value.isVector2 ) {\n\n\t\t\t// vec2\n\n\t\t\tinfo.boundary = 8;\n\t\t\tinfo.storage = 8;\n\n\t\t} else if ( value.isVector3 || value.isColor ) {\n\n\t\t\t// vec3\n\n\t\t\tinfo.boundary = 16;\n\t\t\tinfo.storage = 12; // evil: vec3 must start on a 16-byte boundary but it only consumes 12 bytes\n\n\t\t} else if ( value.isVector4 ) {\n\n\t\t\t// vec4\n\n\t\t\tinfo.boundary = 16;\n\t\t\tinfo.storage = 16;\n\n\t\t} else if ( value.isMatrix3 ) {\n\n\t\t\t// mat3 (in STD140 a 3x3 matrix is represented as 3x4)\n\n\t\t\tinfo.boundary = 48;\n\t\t\tinfo.storage = 48;\n\n\t\t} else if ( value.isMatrix4 ) {\n\n\t\t\t// mat4\n\n\t\t\tinfo.boundary = 64;\n\t\t\tinfo.storage = 64;\n\n\t\t} else if ( value.isTexture ) {\n\n\t\t\tconsole.warn( 'THREE.WebGLRenderer: Texture samplers can not be part of an uniforms group.' );\n\n\t\t} else {\n\n\t\t\tconsole.warn( 'THREE.WebGLRenderer: Unsupported uniform value type.', value );\n\n\t\t}\n\n\t\treturn info;\n\n\t}\n\n\tfunction onUniformsGroupsDispose( event ) {\n\n\t\tconst uniformsGroup = event.target;\n\n\t\tuniformsGroup.removeEventListener( 'dispose', onUniformsGroupsDispose );\n\n\t\tconst index = allocatedBindingPoints.indexOf( uniformsGroup.__bindingPointIndex );\n\t\tallocatedBindingPoints.splice( index, 1 );\n\n\t\tgl.deleteBuffer( buffers[ uniformsGroup.id ] );\n\n\t\tdelete buffers[ uniformsGroup.id ];\n\t\tdelete updateList[ uniformsGroup.id ];\n\n\t}\n\n\tfunction dispose() {\n\n\t\tfor ( const id in buffers ) {\n\n\t\t\tgl.deleteBuffer( buffers[ id ] );\n\n\t\t}\n\n\t\tallocatedBindingPoints = [];\n\t\tbuffers = {};\n\t\tupdateList = {};\n\n\t}\n\n\treturn {\n\n\t\tbind: bind,\n\t\tupdate: update,\n\n\t\tdispose: dispose\n\n\t};\n\n}\n\n\nexport { WebGLUniformsGroups };\n\n// Path: src/renderers/webgl/WebGLGeometries.js\nimport { Uint16BufferAttribute, Uint32BufferAttribute } from '../../core/BufferAttribute.js';\nimport { arrayNeedsUint32 } from '../../utils.js';\n\nfunction WebGLGeometries( gl, attributes, info, bindingStates ) {\n\n\tconst geometries = {};\n\tconst wireframeAttributes = new WeakMap();\n\n\tfunction onGeometryDispose( event ) {\n\n\t\tconst geometry = event.target;\n\n\t\tif ( geometry.index !== null ) {\n\n\t\t\tattributes.remove( geometry.index );\n\n\t\t}\n\n\t\tfor ( const name in geometry.attributes ) {\n\n\t\t\tattributes.remove( geometry.attributes[ name ] );\n\n\t\t}\n\n\t\tfor ( const name in geometry.morphAttributes ) {\n\n\t\t\tconst array = geometry.morphAttributes[ name ];\n\n\t\t\tfor ( let i = 0, l = array.length; i < l; i ++ ) {\n\n\t\t\t\tattributes.remove( array[ i ] );\n\n\t\t\t}\n\n\t\t}\n\n\t\tgeometry.removeEventListener( 'dispose', onGeometryDispose );\n\n\t\tdelete geometries[ geometry.id ];\n\n\t\tconst attribute = wireframeAttributes.get( geometry );\n\n\t\tif ( attribute ) {\n\n\t\t\tattributes.remove( attribute );\n\t\t\twireframeAttributes.delete( geometry );\n\n\t\t}\n\n\t\tbindingStates.releaseStatesOfGeometry( geometry );\n\n\t\tif ( geometry.isInstancedBufferGeometry === true ) {\n\n\t\t\tdelete geometry._maxInstanceCount;\n\n\t\t}\n\n\t\t//\n\n\t\tinfo.memory.geometries --;\n\n\t}\n\n\tfunction get( object, geometry ) {\n\n\t\tif ( geometries[ geometry.id ] === true ) return geometry;\n\n\t\tgeometry.addEventListener( 'dispose', onGeometryDispose );\n\n\t\tgeometries[ geometry.id ] = true;\n\n\t\tinfo.memory.geometries ++;\n\n\t\treturn geometry;\n\n\t}\n\n\tfunction update( geometry ) {\n\n\t\tconst geometryAttributes = geometry.attributes;\n\n\t\t// Updating index buffer in VAO now. See WebGLBindingStates.\n\n\t\tfor ( const name in geometryAttributes ) {\n\n\t\t\tattributes.update( geometryAttributes[ name ], gl.ARRAY_BUFFER );\n\n\t\t}\n\n\t\t// morph targets\n\n\t\tconst morphAttributes = geometry.morphAttributes;\n\n\t\tfor ( const name in morphAttributes ) {\n\n\t\t\tconst array = morphAttributes[ name ];\n\n\t\t\tfor ( let i = 0, l = array.length; i < l; i ++ ) {\n\n\t\t\t\tattributes.update( array[ i ], gl.ARRAY_BUFFER );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tfunction updateWireframeAttribute( geometry ) {\n\n\t\tconst indices = [];\n\n\t\tconst geometryIndex = geometry.index;\n\t\tconst geometryPosition = geometry.attributes.position;\n\t\tlet version = 0;\n\n\t\tif ( geometryIndex !== null ) {\n\n\t\t\tconst array = geometryIndex.array;\n\t\t\tversion = geometryIndex.version;\n\n\t\t\tfor ( let i = 0, l = array.length; i < l; i += 3 ) {\n\n\t\t\t\tconst a = array[ i + 0 ];\n\t\t\t\tconst b = array[ i + 1 ];\n\t\t\t\tconst c = array[ i + 2 ];\n\n\t\t\t\tindices.push( a, b, b, c, c, a );\n\n\t\t\t}\n\n\t\t} else if ( geometryPosition !== undefined ) {\n\n\t\t\tconst array = geometryPosition.array;\n\t\t\tversion = geometryPosition.version;\n\n\t\t\tfor ( let i = 0, l = ( array.length / 3 ) - 1; i < l; i += 3 ) {\n\n\t\t\t\tconst a = i + 0;\n\t\t\t\tconst b = i + 1;\n\t\t\t\tconst c = i + 2;\n\n\t\t\t\tindices.push( a, b, b, c, c, a );\n\n\t\t\t}\n\n\t\t} else {\n\n\t\t\treturn;\n\n\t\t}\n\n\t\tconst attribute = new ( arrayNeedsUint32( indices ) ? Uint32BufferAttribute : Uint16BufferAttribute )( indices, 1 );\n\t\tattribute.version = version;\n\n\t\t// Updating index buffer in VAO now. See WebGLBindingStates\n\n\t\t//\n\n\t\tconst previousAttribute = wireframeAttributes.get( geometry );\n\n\t\tif ( previousAttribute ) attributes.remove( previousAttribute );\n\n\t\t//\n\n\t\twireframeAttributes.set( geometry, attribute );\n\n\t}\n\n\tfunction getWireframeAttribute( geometry ) {\n\n\t\tconst currentAttribute = wireframeAttributes.get( geometry );\n\n\t\tif ( currentAttribute ) {\n\n\t\t\tconst geometryIndex = geometry.index;\n\n\t\t\tif ( geometryIndex !== null ) {\n\n\t\t\t\t// if the attribute is obsolete, create a new one\n\n\t\t\t\tif ( currentAttribute.version < geometryIndex.version ) {\n\n\t\t\t\t\tupdateWireframeAttribute( geometry );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t} else {\n\n\t\t\tupdateWireframeAttribute( geometry );\n\n\t\t}\n\n\t\treturn wireframeAttributes.get( geometry );\n\n\t}\n\n\treturn {\n\n\t\tget: get,\n\t\tupdate: update,\n\n\t\tgetWireframeAttribute: getWireframeAttribute\n\n\t};\n\n}\n\n\nexport { WebGLGeometries };\n\n// Path: src/renderers/webgl/WebGLTextures.js\nimport { LinearFilter, LinearMipmapLinearFilter, LinearMipmapNearestFilter, NearestFilter, NearestMipmapLinearFilter, NearestMipmapNearestFilter, RGBAFormat, DepthFormat, DepthStencilFormat, UnsignedIntType, UnsignedInt248Type, FloatType, MirroredRepeatWrapping, ClampToEdgeWrapping, RepeatWrapping, UnsignedByteType, NoColorSpace, LinearSRGBColorSpace, NeverCompare, AlwaysCompare, LessCompare, LessEqualCompare, EqualCompare, GreaterEqualCompare, GreaterCompare, NotEqualCompare, SRGBTransfer, LinearTransfer } from '../../constants.js';\nimport { createElementNS } from '../../utils.js';\nimport { ColorManagement } from '../../math/ColorManagement.js';\nimport { Vector2 } from '../../math/Vector2.js';\n\nfunction WebGLTextures( _gl, extensions, state, properties, capabilities, utils, info ) {\n\n\tconst multisampledRTTExt = extensions.has( 'WEBGL_multisampled_render_to_texture' ) ? extensions.get( 'WEBGL_multisampled_render_to_texture' ) : null;\n\tconst supportsInvalidateFramebuffer = typeof navigator === 'undefined' ? false : /OculusBrowser/g.test( navigator.userAgent );\n\n\tconst _imageDimensions = new Vector2();\n\tconst _videoTextures = new WeakMap();\n\tlet _canvas;\n\n\tconst _sources = new WeakMap(); // maps WebglTexture objects to instances of Source\n\n\t// cordova iOS (as of 5.0) still uses UIWebView, which provides OffscreenCanvas,\n\t// also OffscreenCanvas.getContext(\"webgl\"), but not OffscreenCanvas.getContext(\"2d\")!\n\t// Some implementations may only implement OffscreenCanvas partially (e.g. lacking 2d).\n\n\tlet useOffscreenCanvas = false;\n\n\ttry {\n\n\t\tuseOffscreenCanvas = typeof OffscreenCanvas !== 'undefined'\n\t\t\t// eslint-disable-next-line compat/compat\n\t\t\t&& ( new OffscreenCanvas( 1, 1 ).getContext( '2d' ) ) !== null;\n\n\t} catch ( err ) {\n\n\t\t// Ignore any errors\n\n\t}\n\n\t\nfunction createCanvas( width, height ) {\n\n\t\t// Use OffscreenCanvas when available. Specially needed in web workers\n\n\t\treturn useOffscreenCanvas ?\n\t\t\t// eslint-disable-next-line compat/compat\n\t\t\tnew OffscreenCanvas( width, height ) : createElementNS( 'canvas' );\n\n\t}\n\n\tfunction resizeImage( image, needsNewCanvas, maxSize ) {\n\n\t\tlet scale = 1;\n\n\t\tconst dimensions = getDimensions( image );\n\n\t\t// handle case if texture exceeds max size\n\n\t\tif ( dimensions.width > maxSize || dimensions.height > maxSize ) {\n\n\t\t\tscale = maxSize / Math.max( dimensions.width, dimensions.height );\n\n\t\t}\n\n\t\t// only perform resize if necessary\n\n\t\tif ( scale < 1 ) {\n\n\t\t\t// only perform resize for certain image types\n\n\t\t\tif ( ( typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement ) ||\n\t\t\t\t( typeof HTMLCanvasElement !== 'undefined' && image instanceof HTMLCanvasElement ) ||\n\t\t\t\t( typeof ImageBitmap !== 'undefined' && image instanceof ImageBitmap ) ||\n\t\t\t\t( typeof VideoFrame !== 'undefined' && image instanceof VideoFrame ) ) {\n\n\t\t\t\tconst width = Math.floor( scale * dimensions.width );\n\t\t\t\tconst height = Math.floor( scale * dimensions.height );\n\n\t\t\t\tif ( _canvas === undefined ) _canvas = createCanvas( width, height );\n\n\t\t\t\t// cube textures can't reuse the same canvas\n\n\t\t\t\tconst canvas = needsNewCanvas ? createCanvas( width, height ) : _canvas;\n\n\t\t\t\tcanvas.width = width;\n\t\t\t\tcanvas.height = height;\n\n\t\t\t\tconst context = canvas.getContext( '2d' );\n\t\t\t\tcontext.drawImage( image, 0, 0, width, height );\n\n\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: Texture has been resized from (' + dimensions.width + 'x' + dimensions.height + ') to (' + width + 'x' + height + ').' );\n\n\t\t\t\treturn canvas;\n\n\t\t\t} else {\n\n\t\t\t\tif ( 'data' in image ) {\n\n\t\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: Image in DataTexture is too big (' + dimensions.width + 'x' + dimensions.height + ').' );\n\n\t\t\t\t}\n\n\t\t\t\treturn image;\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn image;\n\n\t}\n\n\tfunction textureNeedsGenerateMipmaps( texture ) {\n\n\t\treturn texture.generateMipmaps && texture.minFilter !== NearestFilter && texture.minFilter !== LinearFilter;\n\n\t}\n\n\tfunction generateMipmap( target ) {\n\n\t\t_gl.generateMipmap( target );\n\n\t}\n\n\tfunction getInternalFormat( internalFormatName, glFormat, glType, colorSpace, forceLinearTransfer = false ) {\n\n\t\tif ( internalFormatName !== null ) {\n\n\t\t\tif ( _gl[ internalFormatName ] !== undefined ) return _gl[ internalFormatName ];\n\n\t\t\tconsole.warn( 'THREE.WebGLRenderer: Attempt to use non-existing WebGL internal format \\'' + internalFormatName + '\\'' );\n\n\t\t}\n\n\t\tlet internalFormat = glFormat;\n\n\t\tif ( glFormat === _gl.RED ) {\n\n\t\t\tif ( glType === _gl.FLOAT ) internalFormat = _gl.R32F;\n\t\t\tif ( glType === _gl.HALF_FLOAT ) internalFormat = _gl.R16F;\n\t\t\tif ( glType === _gl.UNSIGNED_BYTE ) internalFormat = _gl.R8;\n\n\t\t}\n\n\t\tif ( glFormat === _gl.RED_INTEGER ) {\n\n\t\t\tif ( glType === _gl.UNSIGNED_BYTE ) internalFormat = _gl.R8UI;\n\t\t\tif ( glType === _gl.UNSIGNED_SHORT ) internalFormat = _gl.R16UI;\n\t\t\tif ( glType === _gl.UNSIGNED_INT ) internalFormat = _gl.R32UI;\n\t\t\tif ( glType === _gl.BYTE ) internalFormat = _gl.R8I;\n\t\t\tif ( glType === _gl.SHORT ) internalFormat = _gl.R16I;\n\t\t\tif ( glType === _gl.INT ) internalFormat = _gl.R32I;\n\n\t\t}\n\n\t\tif ( glFormat === _gl.RG ) {\n\n\t\t\tif ( glType === _gl.FLOAT ) internalFormat = _gl.RG32F;\n\t\t\tif ( glType === _gl.HALF_FLOAT ) internalFormat = _gl.RG16F;\n\t\t\tif ( glType === _gl.UNSIGNED_BYTE ) internalFormat = _gl.RG8;\n\n\t\t}\n\n\t\tif ( glFormat === _gl.RG_INTEGER ) {\n\n\t\t\tif ( glType === _gl.UNSIGNED_BYTE ) internalFormat = _gl.RG8UI;\n\t\t\tif ( glType === _gl.UNSIGNED_SHORT ) internalFormat = _gl.RG16UI;\n\t\t\tif ( glType === _gl.UNSIGNED_INT ) internalFormat = _gl.RG32UI;\n\t\t\tif ( glType === _gl.BYTE ) internalFormat = _gl.RG8I;\n\t\t\tif ( glType === _gl.SHORT ) internalFormat = _gl.RG16I;\n\t\t\tif ( glType === _gl.INT ) internalFormat = _gl.RG32I;\n\n\t\t}\n\n\t\tif ( glFormat === _gl.RGB ) {\n\n\t\t\tif ( glType === _gl.UNSIGNED_INT_5_9_9_9_REV ) internalFormat = _gl.RGB9_E5;\n\n\t\t}\n\n\t\tif ( glFormat === _gl.RGBA ) {\n\n\t\t\tconst transfer = forceLinearTransfer ? LinearTransfer : ColorManagement.getTransfer( colorSpace );\n\n\t\t\tif ( glType === _gl.FLOAT ) internalFormat = _gl.RGBA32F;\n\t\t\tif ( glType === _gl.HALF_FLOAT ) internalFormat = _gl.RGBA16F;\n\t\t\tif ( glType === _gl.UNSIGNED_BYTE ) internalFormat = ( transfer === SRGBTransfer ) ? _gl.SRGB8_ALPHA8 : _gl.RGBA8;\n\t\t\tif ( glType === _gl.UNSIGNED_SHORT_4_4_4_4 ) internalFormat = _gl.RGBA4;\n\t\t\tif ( glType === _gl.UNSIGNED_SHORT_5_5_5_1 ) internalFormat = _gl.RGB5_A1;\n\n\t\t}\n\n\t\tif ( internalFormat === _gl.R16F || internalFormat === _gl.R32F ||\n\t\t\tinternalFormat === _gl.RG16F || internalFormat === _gl.RG32F ||\n\t\t\tinternalFormat === _gl.RGBA16F || internalFormat === _gl.RGBA32F ) {\n\n\t\t\textensions.get( 'EXT_color_buffer_float' );\n\n\t\t}\n\n\t\treturn internalFormat;\n\n\t}\n\n\tfunction getMipLevels( texture, image ) {\n\n\t\tif ( textureNeedsGenerateMipmaps( texture ) === true || ( texture.isFramebufferTexture && texture.minFilter !== NearestFilter && texture.minFilter !== LinearFilter ) ) {\n\n\t\t\treturn Math.log2( Math.max( image.width, image.height ) ) + 1;\n\n\t\t} else if ( texture.mipmaps !== undefined && texture.mipmaps.length > 0 ) {\n\n\t\t\t// user-defined mipmaps\n\n\t\t\treturn texture.mipmaps.length;\n\n\t\t} else if ( texture.isCompressedTexture && Array.isArray( texture.image ) ) {\n\n\t\t\treturn image.mipmaps.length;\n\n\t\t} else {\n\n\t\t\t// texture without mipmaps (only base level)\n\n\t\t\treturn 1;\n\n\t\t}\n\n\t}\n\n\t//\n\n\tfunction onTextureDispose( event ) {\n\n\t\tconst texture = event.target;\n\n\t\ttexture.removeEventListener( 'dispose', onTextureDispose );\n\n\t\tdeallocateTexture( texture );\n\n\t\tif ( texture.isVideoTexture ) {\n\n\t\t\t_videoTextures.delete( texture );\n\n\t\t}\n\n\t}\n\n\tfunction onRenderTargetDispose( event ) {\n\n\t\tconst renderTarget = event.target;\n\n\t\trenderTarget.removeEventListener( 'dispose', onRenderTargetDispose );\n\n\t\tdeallocateRenderTarget( renderTarget );\n\n\t}\n\n\t//\n\n\tfunction deallocateTexture( texture ) {\n\n\t\tconst textureProperties = properties.get( texture );\n\n\t\tif ( textureProperties.__webglInit === undefined ) return;\n\n\t\t// check if it's necessary to remove the WebGLTexture object\n\n\t\tconst source = texture.source;\n\t\tconst webglTextures = _sources.get( source );\n\n\t\tif ( webglTextures ) {\n\n\t\t\tconst webglTexture = webglTextures[ textureProperties.__cacheKey ];\n\t\t\twebglTexture.usedTimes --;\n\n\t\t\t// the WebGLTexture object is not used anymore, remove it\n\n\t\t\tif ( webglTexture.usedTimes === 0 ) {\n\n\t\t\t\tdeleteTexture( texture );\n\n\t\t\t}\n\n\t\t\t// remove the weak map entry if no WebGLTexture uses the source anymore\n\n\t\t\tif ( Object.keys( webglTextures ).length === 0 ) {\n\n\t\t\t\t_sources.delete( source );\n\n\t\t\t}\n\n\t\t}\n\n\t\tproperties.remove( texture );\n\n\t}\n\n\tfunction deleteTexture( texture ) {\n\n\t\tconst textureProperties = properties.get( texture );\n\t\t_gl.deleteTexture( textureProperties.__webglTexture );\n\n\t\tconst source = texture.source;\n\t\tconst webglTextures = _sources.get( source );\n\t\tdelete webglTextures[ textureProperties.__cacheKey ];\n\n\t\tinfo.memory.textures --;\n\n\t}\n\n\tfunction deallocateRenderTarget( renderTarget ) {\n\n\t\tconst renderTargetProperties = properties.get( renderTarget );\n\n\t\tif ( renderTarget.depthTexture ) {\n\n\t\t\trenderTarget.depthTexture.dispose();\n\n\t\t}\n\n\t\tif ( renderTarget.isWebGLCubeRenderTarget ) {\n\n\t\t\tfor ( let i = 0; i < 6; i ++ ) {\n\n\t\t\t\tif ( Array.isArray( renderTargetProperties.__webglFramebuffer[ i ] ) ) {\n\n\t\t\t\t\tfor ( let level = 0; level < renderTargetProperties.__webglFramebuffer[ i ].length; level ++ ) _gl.deleteFramebuffer( renderTargetProperties.__webglFramebuffer[ i ][ level ] );\n\n\t\t\t\t} else {\n\n\t\t\t\t\t_gl.deleteFramebuffer( renderTargetProperties.__webglFramebuffer[ i ] );\n\n\t\t\t\t}\n\n\t\t\t\tif ( renderTargetProperties.__webglDepthbuffer ) _gl.deleteRenderbuffer( renderTargetProperties.__webglDepthbuffer[ i ] );\n\n\t\t\t}\n\n\t\t} else {\n\n\t\t\tif ( Array.isArray( renderTargetProperties.__webglFramebuffer ) ) {\n\n\t\t\t\tfor ( let level = 0; level < renderTargetProperties.__webglFramebuffer.length; level ++ ) _gl.deleteFramebuffer( renderTargetProperties.__webglFramebuffer[ level ] );\n\n\t\t\t} else {\n\n\t\t\t\t_gl.deleteFramebuffer( renderTargetProperties.__webglFramebuffer );\n\n\t\t\t}\n\n\t\t\tif ( renderTargetProperties.__webglDepthbuffer ) _gl.deleteRenderbuffer( renderTargetProperties.__webglDepthbuffer );\n\t\t\tif ( renderTargetProperties.__webglMultisampledFramebuffer ) _gl.deleteFramebuffer( renderTargetProperties.__webglMultisampledFramebuffer );\n\n\t\t\tif ( renderTargetProperties.__webglColorRenderbuffer ) {\n\n\t\t\t\tfor ( let i = 0; i < renderTargetProperties.__webglColorRenderbuffer.length; i ++ ) {\n\n\t\t\t\t\tif ( renderTargetProperties.__webglColorRenderbuffer[ i ] ) _gl.deleteRenderbuffer( renderTargetProperties.__webglColorRenderbuffer[ i ] );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tif ( renderTargetProperties.__webglDepthRenderbuffer ) _gl.deleteRenderbuffer( renderTargetProperties.__webglDepthRenderbuffer );\n\n\t\t}\n\n\t\tconst textures = renderTarget.textures;\n\n\t\tfor ( let i = 0, il = textures.length; i < il; i ++ ) {\n\n\t\t\tconst attachmentProperties = properties.get( textures[ i ] );\n\n\t\t\tif ( attachmentProperties.__webglTexture ) {\n\n\t\t\t\t_gl.deleteTexture( attachmentProperties.__webglTexture );\n\n\t\t\t\tinfo.memory.textures --;\n\n\t\t\t}\n\n\t\t\tproperties.remove( textures[ i ] );\n\n\t\t}\n\n\t\tproperties.remove( renderTarget );\n\n\t}\n\n\t//\n\n\tlet textureUnits = 0;\n\n\tfunction resetTextureUnits() {\n\n\t\ttextureUnits = 0;\n\n\t}\n\n\tfunction allocateTextureUnit() {\n\n\t\tconst textureUnit = textureUnits;\n\n\t\tif ( textureUnit >= capabilities.maxTextures ) {\n\n\t\t\tconsole.warn( 'THREE.WebGLTextures: Trying to use ' + textureUnit + ' texture units while this GPU supports only ' + capabilities.maxTextures );\n\n\t\t}\n\n\t\ttextureUnits += 1;\n\n\t\treturn textureUnit;\n\n\t}\n\n\tfunction getTextureCacheKey( texture ) {\n\n\t\tconst array = [];\n\n\t\tarray.push( texture.wrapS );\n\t\tarray.push( texture.wrapT );\n\t\tarray.push( texture.wrapR || 0 );\n\t\tarray.push( texture.magFilter );\n\t\tarray.push( texture.minFilter );\n\t\tarray.push( texture.anisotropy );\n\t\tarray.push( texture.internalFormat );\n\t\tarray.push( texture.format );\n\t\tarray.push( texture.type );\n\t\tarray.push( texture.generateMipmaps );\n\t\tarray.push( texture.premultiplyAlpha );\n\t\tarray.push( texture.flipY );\n\t\tarray.push( texture.unpackAlignment );\n\t\tarray.push( texture.colorSpace );\n\n\t\treturn array.join();\n\n\t}\n\n\t//\n\n\tfunction setTexture2D( texture, slot ) {\n\n\t\tconst textureProperties = properties.get( texture );\n\n\t\tif ( texture.isVideoTexture ) updateVideoTexture( texture );\n\n\t\tif ( texture.isRenderTargetTexture === false && texture.version > 0 && textureProperties.__version !== texture.version ) {\n\n\t\t\tconst image = texture.image;\n\n\t\t\tif ( image === null ) {\n\n\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: Texture marked for update but no image data found.' );\n\n\t\t\t} else if ( image.complete === false ) {\n\n\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: Texture marked for update but image is incomplete' );\n\n\t\t\t} else {\n\n\t\t\t\tuploadTexture( textureProperties, texture, slot );\n\t\t\t\treturn;\n\n\t\t\t}\n\n\t\t}\n\n\t\tstate.bindTexture( _gl.TEXTURE_2D, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );\n\n\t}\n\n\tfunction setTexture2DArray( texture, slot ) {\n\n\t\tconst textureProperties = properties.get( texture );\n\n\t\tif ( texture.version > 0 && textureProperties.__version !== texture.version ) {\n\n\t\t\tuploadTexture( textureProperties, texture, slot );\n\t\t\treturn;\n\n\t\t}\n\n\t\tstate.bindTexture( _gl.TEXTURE_2D_ARRAY, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );\n\n\t}\n\n\tfunction setTexture3D( texture, slot ) {\n\n\t\tconst textureProperties = properties.get( texture );\n\n\t\tif ( texture.version > 0 && textureProperties.__version !== texture.version ) {\n\n\t\t\tuploadTexture( textureProperties, texture, slot );\n\t\t\treturn;\n\n\t\t}\n\n\t\tstate.bindTexture( _gl.TEXTURE_3D, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );\n\n\t}\n\n\tfunction setTextureCube( texture, slot ) {\n\n\t\tconst textureProperties = properties.get( texture );\n\n\t\tif ( texture.version > 0 && textureProperties.__version !== texture.version ) {\n\n\t\t\tuploadCubeTexture( textureProperties, texture, slot );\n\t\t\treturn;\n\n\t\t}\n\n\t\tstate.bindTexture( _gl.TEXTURE_CUBE_MAP, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );\n\n\t}\n\n\tconst wrappingToGL = {\n\t\t[ RepeatWrapping ]: _gl.REPEAT,\n\t\t[ ClampToEdgeWrapping ]: _gl.CLAMP_TO_EDGE,\n\t\t[ MirroredRepeatWrapping ]: _gl.MIRRORED_REPEAT\n\t};\n\n\tconst filterToGL = {\n\t\t[ NearestFilter ]: _gl.NEAREST,\n\t\t[ NearestMipmapNearestFilter ]: _gl.NEAREST_MIPMAP_NEAREST,\n\t\t[ NearestMipmapLinearFilter ]: _gl.NEAREST_MIPMAP_LINEAR,\n\n\t\t[ LinearFilter ]: _gl.LINEAR,\n\t\t[ LinearMipmapNearestFilter ]: _gl.LINEAR_MIPMAP_NEAREST,\n\t\t[ LinearMipmapLinearFilter ]: _gl.LINEAR_MIPMAP_LINEAR\n\t};\n\n\tconst compareToGL = {\n\t\t[ NeverCompare ]: _gl.NEVER,\n\t\t[ AlwaysCompare ]: _gl.ALWAYS,\n\t\t[ LessCompare ]: _gl.LESS,\n\t\t[ LessEqualCompare ]: _gl.LEQUAL,\n\t\t[ EqualCompare ]: _gl.EQUAL,\n\t\t[ GreaterEqualCompare ]: _gl.GEQUAL,\n\t\t[ GreaterCompare ]: _gl.GREATER,\n\t\t[ NotEqualCompare ]: _gl.NOTEQUAL\n\t};\n\n\tfunction setTextureParameters( textureType, texture ) {\n\n\t\tif ( texture.type === FloatType && extensions.has( 'OES_texture_float_linear' ) === false &&\n\t\t\t( texture.magFilter === LinearFilter || texture.magFilter === LinearMipmapNearestFilter || texture.magFilter === NearestMipmapLinearFilter || texture.magFilter === LinearMipmapLinearFilter ||\n\t\t\ttexture.minFilter === LinearFilter || texture.minFilter === LinearMipmapNearestFilter || texture.minFilter === NearestMipmapLinearFilter || texture.minFilter === LinearMipmapLinearFilter ) ) {\n\n\t\t\tconsole.warn( 'THREE.WebGLRenderer: Unable to use linear filtering with floating point textures. OES_texture_float_linear not supported on this device.' );\n\n\t\t}\n\n\t\t_gl.texParameteri( textureType, _gl.TEXTURE_WRAP_S, wrappingToGL[ texture.wrapS ] );\n\t\t_gl.texParameteri( textureType, _gl.TEXTURE_WRAP_T, wrappingToGL[ texture.wrapT ] );\n\n\t\tif ( textureType === _gl.TEXTURE_3D || textureType === _gl.TEXTURE_2D_ARRAY ) {\n\n\t\t\t_gl.texParameteri( textureType, _gl.TEXTURE_WRAP_R, wrappingToGL[ texture.wrapR ] );\n\n\t\t}\n\n\t\t_gl.texParameteri( textureType, _gl.TEXTURE_MAG_FILTER, filterToGL[ texture.magFilter ] );\n\t\t_gl.texParameteri( textureType, _gl.TEXTURE_MIN_FILTER, filterToGL[ texture.minFilter ] );\n\n\t\tif ( texture.compareFunction ) {\n\n\t\t\t_gl.texParameteri( textureType, _gl.TEXTURE_COMPARE_MODE, _gl.COMPARE_REF_TO_TEXTURE );\n\t\t\t_gl.texParameteri( textureType, _gl.TEXTURE_COMPARE_FUNC, compareToGL[ texture.compareFunction ] );\n\n\t\t}\n\n\t\tif ( extensions.has( 'EXT_texture_filter_anisotropic' ) === true ) {\n\n\t\t\tif ( texture.magFilter === NearestFilter ) return;\n\t\t\tif ( texture.minFilter !== NearestMipmapLinearFilter && texture.minFilter !== LinearMipmapLinearFilter ) return;\n\t\t\tif ( texture.type === FloatType && extensions.has( 'OES_texture_float_linear' ) === false ) return; // verify extension\n\n\t\t\tif ( texture.anisotropy > 1 || properties.get( texture ).__currentAnisotropy ) {\n\n\t\t\t\tconst extension = extensions.get( 'EXT_texture_filter_anisotropic' );\n\t\t\t\t_gl.texParameterf( textureType, extension.TEXTURE_MAX_ANISOTROPY_EXT, Math.min( texture.anisotropy, capabilities.getMaxAnisotropy() ) );\n\t\t\t\tproperties.get( texture ).__currentAnisotropy = texture.anisotropy;\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tfunction initTexture( textureProperties, texture ) {\n\n\t\tlet forceUpload = false;\n\n\t\tif ( textureProperties.__webglInit === undefined ) {\n\n\t\t\ttextureProperties.__webglInit = true;\n\n\t\t\ttexture.addEventListener( 'dispose', onTextureDispose );\n\n\t\t}\n\n\t\t// create Source <-> WebGLTextures mapping if necessary\n\n\t\tconst source = texture.source;\n\t\tlet webglTextures = _sources.get( source );\n\n\t\tif ( webglTextures === undefined ) {\n\n\t\t\twebglTextures = {};\n\t\t\t_sources.set( source, webglTextures );\n\n\t\t}\n\n\t\t// check if there is already a WebGLTexture object for the given texture parameters\n\n\t\tconst textureCacheKey = getTextureCacheKey( texture );\n\n\t\tif ( textureCacheKey !== textureProperties.__cacheKey ) {\n\n\t\t\t// if not, create a new instance of WebGLTexture\n\n\t\t\tif ( webglTextures[ textureCacheKey ] === undefined ) {\n\n\t\t\t\t// create new entry\n\n\t\t\t\twebglTextures[ textureCacheKey ] = {\n\t\t\t\t\ttexture: _gl.createTexture(),\n\t\t\t\t\tusedTimes: 0\n\t\t\t\t};\n\n\t\t\t\tinfo.memory.textures ++;\n\n\t\t\t\t// when a new instance of WebGLTexture was created, a texture upload is required\n\t\t\t\t// even if the image contents are identical\n\n\t\t\t\tforceUpload = true;\n\n\t\t\t}\n\n\t\t\twebglTextures[ textureCacheKey ].usedTimes ++;\n\n\t\t\t// every time the texture cache key changes, it's necessary to check if an instance of\n\t\t\t// WebGLTexture can be deleted in order to avoid a memory leak.\n\n\t\t\tconst webglTexture = webglTextures[ textureProperties.__cacheKey ];\n\n\t\t\tif ( webglTexture !== undefined ) {\n\n\t\t\t\twebglTextures[ textureProperties.__cacheKey ].usedTimes --;\n\n\t\t\t\tif ( webglTexture.usedTimes === 0 ) {\n\n\t\t\t\t\tdeleteTexture( texture );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// store references to cache key and WebGLTexture object\n\n\t\t\ttextureProperties.__cacheKey = textureCacheKey;\n\t\t\ttextureProperties.__webglTexture = webglTextures[ textureCacheKey ].texture;\n\n\t\t}\n\n\t\treturn forceUpload;\n\n\t}\n\n\tfunction uploadTexture( textureProperties, texture, slot ) {\n\n\t\tlet textureType = _gl.TEXTURE_2D;\n\n\t\tif ( texture.isDataArrayTexture || texture.isCompressedArrayTexture ) textureType = _gl.TEXTURE_2D_ARRAY;\n\t\tif ( texture.isData3DTexture ) textureType = _gl.TEXTURE_3D;\n\n\t\tconst forceUpload = initTexture( textureProperties, texture );\n\t\tconst source = texture.source;\n\n\t\tstate.bindTexture( textureType, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );\n\n\t\tconst sourceProperties = properties.get( source );\n\n\t\tif ( source.version !== sourceProperties.__version || forceUpload === true ) {\n\n\t\t\tstate.activeTexture( _gl.TEXTURE0 + slot );\n\n\t\t\tconst workingPrimaries = ColorManagement.getPrimaries( ColorManagement.workingColorSpace );\n\t\t\tconst texturePrimaries = texture.colorSpace === NoColorSpace ? null : ColorManagement.getPrimaries( texture.colorSpace );\n\t\t\tconst unpackConversion = texture.colorSpace === NoColorSpace || workingPrimaries === texturePrimaries ? _gl.NONE : _gl.BROWSER_DEFAULT_WEBGL;\n\n\t\t\t_gl.pixelStorei( _gl.UNPACK_FLIP_Y_WEBGL, texture.flipY );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, texture.premultiplyAlpha );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_ALIGNMENT, texture.unpackAlignment );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_COLORSPACE_CONVERSION_WEBGL, unpackConversion );\n\n\t\t\tlet image = resizeImage( texture.image, false, capabilities.maxTextureSize );\n\t\t\timage = verifyColorSpace( texture, image );\n\n\t\t\tconst glFormat = utils.convert( texture.format, texture.colorSpace );\n\n\t\t\tconst glType = utils.convert( texture.type );\n\t\t\tlet glInternalFormat = getInternalFormat( texture.internalFormat, glFormat, glType, texture.colorSpace, texture.isVideoTexture );\n\n\t\t\tsetTextureParameters( textureType, texture );\n\n\t\t\tlet mipmap;\n\t\t\tconst mipmaps = texture.mipmaps;\n\n\t\t\tconst useTexStorage = ( texture.isVideoTexture !== true );\n\t\t\tconst allocateMemory = ( sourceProperties.__version === undefined ) || ( forceUpload === true );\n\t\t\tconst dataReady = source.dataReady;\n\t\t\tconst levels = getMipLevels( texture, image );\n\n\t\t\tif ( texture.isDepthTexture ) {\n\n\t\t\t\t// populate depth texture with dummy data\n\n\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT16;\n\n\t\t\t\tif ( texture.type === FloatType ) {\n\n\t\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT32F;\n\n\t\t\t\t} else if ( texture.type === UnsignedIntType ) {\n\n\t\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT24;\n\n\t\t\t\t} else if ( texture.type === UnsignedInt248Type ) {\n\n\t\t\t\t\tglInternalFormat = _gl.DEPTH24_STENCIL8;\n\n\t\t\t\t}\n\n\t\t\t\t//\n\n\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, 1, glInternalFormat, image.width, image.height );\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, 0, glInternalFormat, image.width, image.height, 0, glFormat, glType, null );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else if ( texture.isDataTexture ) {\n\n\t\t\t\t// use manually created mipmaps if available\n\t\t\t\t// if there are no manual mipmaps\n\t\t\t\t// set 0 level mipmap and then use GL to generate other mipmap levels\n\n\t\t\t\tif ( mipmaps.length > 0 ) {\n\n\t\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, mipmaps[ 0 ].width, mipmaps[ 0 ].height );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tfor ( let i = 0, il = mipmaps.length; i < il; i ++ ) {\n\n\t\t\t\t\t\tmipmap = mipmaps[ i ];\n\n\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_2D, i, 0, 0, mipmap.width, mipmap.height, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, i, glInternalFormat, mipmap.width, mipmap.height, 0, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t\ttexture.generateMipmaps = false;\n\n\t\t\t\t} else {\n\n\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, image.width, image.height );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_2D, 0, 0, 0, image.width, image.height, glFormat, glType, image.data );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, 0, glInternalFormat, image.width, image.height, 0, glFormat, glType, image.data );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else if ( texture.isCompressedTexture ) {\n\n\t\t\t\tif ( texture.isCompressedArrayTexture ) {\n\n\t\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\t\tstate.texStorage3D( _gl.TEXTURE_2D_ARRAY, levels, glInternalFormat, mipmaps[ 0 ].width, mipmaps[ 0 ].height, image.depth );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tfor ( let i = 0, il = mipmaps.length; i < il; i ++ ) {\n\n\t\t\t\t\t\tmipmap = mipmaps[ i ];\n\n\t\t\t\t\t\tif ( texture.format !== RGBAFormat ) {\n\n\t\t\t\t\t\t\tif ( glFormat !== null ) {\n\n\t\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\t\tstate.compressedTexSubImage3D( _gl.TEXTURE_2D_ARRAY, i, 0, 0, 0, mipmap.width, mipmap.height, image.depth, glFormat, mipmap.data, 0, 0 );\n\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\t\tstate.compressedTexImage3D( _gl.TEXTURE_2D_ARRAY, i, glInternalFormat, mipmap.width, mipmap.height, image.depth, 0, mipmap.data, 0, 0 );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n...\n", "needle_token_start": 6566, "needle_token_end": 6645, "code_context_ntokens": 14797, "output": ["```javascript\nfunction createCanvas( width, height ) {\n\n\t// Use OffscreenCanvas when available. Specially needed in web workers\n\n\treturn useOffscreenCanvas ?\n\t\t// eslint-disable-next-line compat/compat\n\t\tnew OffscreenCanvas( width, height ) : createElementNS( 'canvas' );\n\n}\n```"]}
{"repo": "mrdoob/three.js", "name": "setupFrameBufferTexture", "language": "typescript", "path": "src/renderers/webgl/WebGLTextures.js", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to configure a framebuffer by attaching a texture to it, which is essential for rendering scenes directly to a texture instead of the display.\n2. **Input**: It takes six parameters: a framebuffer object, a rendering target object, a texture object, an attachment point, a texture target, and a mipmap level.\n3. **Output**: There is no direct return value; however, the effect is the modification of the WebGL state by binding a texture to a framebuffer.\n4. **Procedure**: The function begins by determining the texture's format, type, and internal format based on its properties. It then calculates the dimensions of the texture at the specified mipmap level. Depending on the texture target, it initializes the texture storage with no data (null) using either 2D or 3D texture image functions. The framebuffer is then bound, and depending on the rendering target's properties, the texture is attached to the framebuffer using appropriate WebGL methods for either multisampled or regular textures. Finally, the framebuffer binding is cleared.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tfunction initTexture( textureProperties, texture ) {\n\n\t\tlet forceUpload = false;\n\n\t\tif ( textureProperties.__webglInit === undefined ) {\n\n\t\t\ttextureProperties.__webglInit = true;\n\n\t\t\ttexture.addEventListener( 'dispose', onTextureDispose );\n\n\t\t}\n\n\t\t// create Source <-> WebGLTextures mapping if necessary\n\n\t\tconst source = texture.source;\n\t\tlet webglTextures = _sources.get( source );\n\n\t\tif ( webglTextures === undefined ) {\n\n\t\t\twebglTextures = {};\n\t\t\t_sources.set( source, webglTextures );\n\n\t\t}\n\n\t\t// check if there is already a WebGLTexture object for the given texture parameters\n\n\t\tconst textureCacheKey = getTextureCacheKey( texture );\n\n\t\tif ( textureCacheKey !== textureProperties.__cacheKey ) {\n\n\t\t\t// if not, create a new instance of WebGLTexture\n\n\t\t\tif ( webglTextures[ textureCacheKey ] === undefined ) {\n\n\t\t\t\t// create new entry\n\n\t\t\t\twebglTextures[ textureCacheKey ] = {\n\t\t\t\t\ttexture: _gl.createTexture(),\n\t\t\t\t\tusedTimes: 0\n\t\t\t\t};\n\n\t\t\t\tinfo.memory.textures ++;\n\n\t\t\t\t// when a new instance of WebGLTexture was created, a texture upload is required\n\t\t\t\t// even if the image contents are identical\n\n\t\t\t\tforceUpload = true;\n\n\t\t\t}\n\n\t\t\twebglTextures[ textureCacheKey ].usedTimes ++;\n\n\t\t\t// every time the texture cache key changes, it's necessary to check if an instance of\n\t\t\t// WebGLTexture can be deleted in order to avoid a memory leak.\n\n\t\t\tconst webglTexture = webglTextures[ textureProperties.__cacheKey ];\n\n\t\t\tif ( webglTexture !== undefined ) {\n\n\t\t\t\twebglTextures[ textureProperties.__cacheKey ].usedTimes --;\n\n\t\t\t\tif ( webglTexture.usedTimes === 0 ) {\n\n\t\t\t\t\tdeleteTexture( texture );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// store references to cache key and WebGLTexture object\n\n\t\t\ttextureProperties.__cacheKey = textureCacheKey;\n\t\t\ttextureProperties.__webglTexture = webglTextures[ textureCacheKey ].texture;\n\n\t\t}\n\n\t\treturn forceUpload;\n\n\t}\n\n\tfunction uploadTexture( textureProperties, texture, slot ) {\n\n\t\tlet textureType = _gl.TEXTURE_2D;\n\n\t\tif ( texture.isDataArrayTexture || texture.isCompressedArrayTexture ) textureType = _gl.TEXTURE_2D_ARRAY;\n\t\tif ( texture.isData3DTexture ) textureType = _gl.TEXTURE_3D;\n\n\t\tconst forceUpload = initTexture( textureProperties, texture );\n\t\tconst source = texture.source;\n\n\t\tstate.bindTexture( textureType, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );\n\n\t\tconst sourceProperties = properties.get( source );\n\n\t\tif ( source.version !== sourceProperties.__version || forceUpload === true ) {\n\n\t\t\tstate.activeTexture( _gl.TEXTURE0 + slot );\n\n\t\t\tconst workingPrimaries = ColorManagement.getPrimaries( ColorManagement.workingColorSpace );\n\t\t\tconst texturePrimaries = texture.colorSpace === NoColorSpace ? null : ColorManagement.getPrimaries( texture.colorSpace );\n\t\t\tconst unpackConversion = texture.colorSpace === NoColorSpace || workingPrimaries === texturePrimaries ? _gl.NONE : _gl.BROWSER_DEFAULT_WEBGL;\n\n\t\t\t_gl.pixelStorei( _gl.UNPACK_FLIP_Y_WEBGL, texture.flipY );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, texture.premultiplyAlpha );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_ALIGNMENT, texture.unpackAlignment );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_COLORSPACE_CONVERSION_WEBGL, unpackConversion );\n\n\t\t\tlet image = resizeImage( texture.image, false, capabilities.maxTextureSize );\n\t\t\timage = verifyColorSpace( texture, image );\n\n\t\t\tconst glFormat = utils.convert( texture.format, texture.colorSpace );\n\n\t\t\tconst glType = utils.convert( texture.type );\n\t\t\tlet glInternalFormat = getInternalFormat( texture.internalFormat, glFormat, glType, texture.colorSpace, texture.isVideoTexture );\n\n\t\t\tsetTextureParameters( textureType, texture );\n\n\t\t\tlet mipmap;\n\t\t\tconst mipmaps = texture.mipmaps;\n\n\t\t\tconst useTexStorage = ( texture.isVideoTexture !== true );\n\t\t\tconst allocateMemory = ( sourceProperties.__version === undefined ) || ( forceUpload === true );\n\t\t\tconst dataReady = source.dataReady;\n\t\t\tconst levels = getMipLevels( texture, image );\n\n\t\t\tif ( texture.isDepthTexture ) {\n\n\t\t\t\t// populate depth texture with dummy data\n\n\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT16;\n\n\t\t\t\tif ( texture.type === FloatType ) {\n\n\t\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT32F;\n\n\t\t\t\t} else if ( texture.type === UnsignedIntType ) {\n\n\t\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT24;\n\n\t\t\t\t} else if ( texture.type === UnsignedInt248Type ) {\n\n\t\t\t\t\tglInternalFormat = _gl.DEPTH24_STENCIL8;\n\n\t\t\t\t}\n\n\t\t\t\t//\n\n\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, 1, glInternalFormat, image.width, image.height );\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, 0, glInternalFormat, image.width, image.height, 0, glFormat, glType, null );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else if ( texture.isDataTexture ) {\n\n\t\t\t\t// use manually created mipmaps if available\n\t\t\t\t// if there are no manual mipmaps\n\t\t\t\t// set 0 level mipmap and then use GL to generate other mipmap levels\n\n\t\t\t\tif ( mipmaps.length > 0 ) {\n\n\t\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, mipmaps[ 0 ].width, mipmaps[ 0 ].height );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tfor ( let i = 0, il = mipmaps.length; i < il; i ++ ) {\n\n\t\t\t\t\t\tmipmap = mipmaps[ i ];\n\n\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_2D, i, 0, 0, mipmap.width, mipmap.height, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, i, glInternalFormat, mipmap.width, mipmap.height, 0, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t\ttexture.generateMipmaps = false;\n\n\t\t\t\t} else {\n\n\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, image.width, image.height );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_2D, 0, 0, 0, image.width, image.height, glFormat, glType, image.data );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, 0, glInternalFormat, image.width, image.height, 0, glFormat, glType, image.data );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else if ( texture.isCompressedTexture ) {\n\n\t\t\t\tif ( texture.isCompressedArrayTexture ) {\n\n\t\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\t\tstate.texStorage3D( _gl.TEXTURE_2D_ARRAY, levels, glInternalFormat, mipmaps[ 0 ].width, mipmaps[ 0 ].height, image.depth );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tfor ( let i = 0, il = mipmaps.length; i < il; i ++ ) {\n\n\t\t\t\t\t\tmipmap = mipmaps[ i ];\n\n\t\t\t\t\t\tif ( texture.format !== RGBAFormat ) {\n\n\t\t\t\t\t\t\tif ( glFormat !== null ) {\n\n\t\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\t\tstate.compressedTexSubImage3D( _gl.TEXTURE_2D_ARRAY, i, 0, 0, 0, mipmap.width, mipmap.height, image.depth, glFormat, mipmap.data, 0, 0 );\n\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\t\tstate.compressedTexImage3D( _gl.TEXTURE_2D_ARRAY, i, glInternalFormat, mipmap.width, mipmap.height, image.depth, 0, mipmap.data, 0, 0 );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: Attempt to load unsupported compressed texture format in .uploadTexture()' );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\tstate.texSubImage3D( _gl.TEXTURE_2D_ARRAY, i, 0, 0, 0, mipmap.width, mipmap.height, image.depth, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tstate.texImage3D( _gl.TEXTURE_2D_ARRAY, i, glInternalFormat, mipmap.width, mipmap.height, image.depth, 0, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t} else {\n\n\t\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, mipmaps[ 0 ].width, mipmaps[ 0 ].height );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tfor ( let i = 0, il = mipmaps.length; i < il; i ++ ) {\n\n\t\t\t\t\t\tmipmap = mipmaps[ i ];\n\n\t\t\t\t\t\tif ( texture.format !== RGBAFormat ) {\n\n\t\t\t\t\t\t\tif ( glFormat !== null ) {\n\n\t\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\t\tstate.compressedTexSubImage2D( _gl.TEXTURE_2D, i, 0, 0, mipmap.width, mipmap.height, glFormat, mipmap.data );\n\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\t\tstate.compressedTexImage2D( _gl.TEXTURE_2D, i, glInternalFormat, mipmap.width, mipmap.height, 0, mipmap.data );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: Attempt to load unsupported compressed texture format in .uploadTexture()' );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_2D, i, 0, 0, mipmap.width, mipmap.height, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, i, glInternalFormat, mipmap.width, mipmap.height, 0, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else if ( texture.isDataArrayTexture ) {\n\n\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\t\tstate.texStorage3D( _gl.TEXTURE_2D_ARRAY, levels, glInternalFormat, image.width, image.height, image.depth );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\tstate.texSubImage3D( _gl.TEXTURE_2D_ARRAY, 0, 0, 0, 0, image.width, image.height, image.depth, glFormat, glType, image.data );\n\n\t\t\t\t\t}\n\n\t\t\t\t} else {\n\n\t\t\t\t\tstate.texImage3D( _gl.TEXTURE_2D_ARRAY, 0, glInternalFormat, image.width, image.height, image.depth, 0, glFormat, glType, image.data );\n\n\t\t\t\t}\n\n\t\t\t} else if ( texture.isData3DTexture ) {\n\n\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\t\tstate.texStorage3D( _gl.TEXTURE_3D, levels, glInternalFormat, image.width, image.height, image.depth );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\tstate.texSubImage3D( _gl.TEXTURE_3D, 0, 0, 0, 0, image.width, image.height, image.depth, glFormat, glType, image.data );\n\n\t\t\t\t\t}\n\n\t\t\t\t} else {\n\n\t\t\t\t\tstate.texImage3D( _gl.TEXTURE_3D, 0, glInternalFormat, image.width, image.height, image.depth, 0, glFormat, glType, image.data );\n\n\t\t\t\t}\n\n\t\t\t} else if ( texture.isFramebufferTexture ) {\n\n\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, image.width, image.height );\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tlet width = image.width, height = image.height;\n\n\t\t\t\t\t\tfor ( let i = 0; i < levels; i ++ ) {\n\n\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, i, glInternalFormat, width, height, 0, glFormat, glType, null );\n\n\t\t\t\t\t\t\twidth >>= 1;\n\t\t\t\t\t\t\theight >>= 1;\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\t// regular Texture (image, video, canvas)\n\n\t\t\t\t// use manually created mipmaps if available\n\t\t\t\t// if there are no manual mipmaps\n\t\t\t\t// set 0 level mipmap and then use GL to generate other mipmap levels\n\n\t\t\t\tif ( mipmaps.length > 0 ) {\n\n\t\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\t\tconst dimensions = getDimensions( mipmaps[ 0 ] );\n\n\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, dimensions.width, dimensions.height );\n\n\t\t\t\t\t}\n\n\t\t\t\t\tfor ( let i = 0, il = mipmaps.length; i < il; i ++ ) {\n\n\t\t\t\t\t\tmipmap = mipmaps[ i ];\n\n\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_2D, i, 0, 0, glFormat, glType, mipmap );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, i, glInternalFormat, glFormat, glType, mipmap );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t\ttexture.generateMipmaps = false;\n\n\t\t\t\t} else {\n\n\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\tif ( allocateMemory ) {\n\n\t\t\t\t\t\t\tconst dimensions = getDimensions( image );\n\n\t\t\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_2D, levels, glInternalFormat, dimensions.width, dimensions.height );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_2D, 0, 0, 0, glFormat, glType, image );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_2D, 0, glInternalFormat, glFormat, glType, image );\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tif ( textureNeedsGenerateMipmaps( texture ) ) {\n\n\t\t\t\tgenerateMipmap( textureType );\n\n\t\t\t}\n\n\t\t\tsourceProperties.__version = source.version;\n\n\t\t\tif ( texture.onUpdate ) texture.onUpdate( texture );\n\n\t\t}\n\n\t\ttextureProperties.__version = texture.version;\n\n\t}\n\n\tfunction uploadCubeTexture( textureProperties, texture, slot ) {\n\n\t\tif ( texture.image.length !== 6 ) return;\n\n\t\tconst forceUpload = initTexture( textureProperties, texture );\n\t\tconst source = texture.source;\n\n\t\tstate.bindTexture( _gl.TEXTURE_CUBE_MAP, textureProperties.__webglTexture, _gl.TEXTURE0 + slot );\n\n\t\tconst sourceProperties = properties.get( source );\n\n\t\tif ( source.version !== sourceProperties.__version || forceUpload === true ) {\n\n\t\t\tstate.activeTexture( _gl.TEXTURE0 + slot );\n\n\t\t\tconst workingPrimaries = ColorManagement.getPrimaries( ColorManagement.workingColorSpace );\n\t\t\tconst texturePrimaries = texture.colorSpace === NoColorSpace ? null : ColorManagement.getPrimaries( texture.colorSpace );\n\t\t\tconst unpackConversion = texture.colorSpace === NoColorSpace || workingPrimaries === texturePrimaries ? _gl.NONE : _gl.BROWSER_DEFAULT_WEBGL;\n\n\t\t\t_gl.pixelStorei( _gl.UNPACK_FLIP_Y_WEBGL, texture.flipY );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, texture.premultiplyAlpha );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_ALIGNMENT, texture.unpackAlignment );\n\t\t\t_gl.pixelStorei( _gl.UNPACK_COLORSPACE_CONVERSION_WEBGL, unpackConversion );\n\n\t\t\tconst isCompressed = ( texture.isCompressedTexture || texture.image[ 0 ].isCompressedTexture );\n\t\t\tconst isDataTexture = ( texture.image[ 0 ] && texture.image[ 0 ].isDataTexture );\n\n\t\t\tconst cubeImage = [];\n\n\t\t\tfor ( let i = 0; i < 6; i ++ ) {\n\n\t\t\t\tif ( ! isCompressed && ! isDataTexture ) {\n\n\t\t\t\t\tcubeImage[ i ] = resizeImage( texture.image[ i ], true, capabilities.maxCubemapSize );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tcubeImage[ i ] = isDataTexture ? texture.image[ i ].image : texture.image[ i ];\n\n\t\t\t\t}\n\n\t\t\t\tcubeImage[ i ] = verifyColorSpace( texture, cubeImage[ i ] );\n\n\t\t\t}\n\n\t\t\tconst image = cubeImage[ 0 ],\n\t\t\t\tglFormat = utils.convert( texture.format, texture.colorSpace ),\n\t\t\t\tglType = utils.convert( texture.type ),\n\t\t\t\tglInternalFormat = getInternalFormat( texture.internalFormat, glFormat, glType, texture.colorSpace );\n\n\t\t\tconst useTexStorage = ( texture.isVideoTexture !== true );\n\t\t\tconst allocateMemory = ( sourceProperties.__version === undefined ) || ( forceUpload === true );\n\t\t\tconst dataReady = source.dataReady;\n\t\t\tlet levels = getMipLevels( texture, image );\n\n\t\t\tsetTextureParameters( _gl.TEXTURE_CUBE_MAP, texture );\n\n\t\t\tlet mipmaps;\n\n\t\t\tif ( isCompressed ) {\n\n\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_CUBE_MAP, levels, glInternalFormat, image.width, image.height );\n\n\t\t\t\t}\n\n\t\t\t\tfor ( let i = 0; i < 6; i ++ ) {\n\n\t\t\t\t\tmipmaps = cubeImage[ i ].mipmaps;\n\n\t\t\t\t\tfor ( let j = 0; j < mipmaps.length; j ++ ) {\n\n\t\t\t\t\t\tconst mipmap = mipmaps[ j ];\n\n\t\t\t\t\t\tif ( texture.format !== RGBAFormat ) {\n\n\t\t\t\t\t\t\tif ( glFormat !== null ) {\n\n\t\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\t\tstate.compressedTexSubImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j, 0, 0, mipmap.width, mipmap.height, glFormat, mipmap.data );\n\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\t\tstate.compressedTexImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j, glInternalFormat, mipmap.width, mipmap.height, 0, mipmap.data );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: Attempt to load unsupported compressed texture format in .setTextureCube()' );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j, 0, 0, mipmap.width, mipmap.height, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j, glInternalFormat, mipmap.width, mipmap.height, 0, glFormat, glType, mipmap.data );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\tmipmaps = texture.mipmaps;\n\n\t\t\t\tif ( useTexStorage && allocateMemory ) {\n\n\t\t\t\t\t// TODO: Uniformly handle mipmap definitions\n\t\t\t\t\t// Normal textures and compressed cube textures define base level + mips with their mipmap array\n\t\t\t\t\t// Uncompressed cube textures use their mipmap array only for mips (no base level)\n\n\t\t\t\t\tif ( mipmaps.length > 0 ) levels ++;\n\n\t\t\t\t\tconst dimensions = getDimensions( cubeImage[ 0 ] );\n\n\t\t\t\t\tstate.texStorage2D( _gl.TEXTURE_CUBE_MAP, levels, glInternalFormat, dimensions.width, dimensions.height );\n\n\t\t\t\t}\n\n\t\t\t\tfor ( let i = 0; i < 6; i ++ ) {\n\n\t\t\t\t\tif ( isDataTexture ) {\n\n\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, 0, 0, 0, cubeImage[ i ].width, cubeImage[ i ].height, glFormat, glType, cubeImage[ i ].data );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, 0, glInternalFormat, cubeImage[ i ].width, cubeImage[ i ].height, 0, glFormat, glType, cubeImage[ i ].data );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tfor ( let j = 0; j < mipmaps.length; j ++ ) {\n\n\t\t\t\t\t\t\tconst mipmap = mipmaps[ j ];\n\t\t\t\t\t\t\tconst mipmapImage = mipmap.image[ i ].image;\n\n\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j + 1, 0, 0, mipmapImage.width, mipmapImage.height, glFormat, glType, mipmapImage.data );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j + 1, glInternalFormat, mipmapImage.width, mipmapImage.height, 0, glFormat, glType, mipmapImage.data );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, 0, 0, 0, glFormat, glType, cubeImage[ i ] );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, 0, glInternalFormat, glFormat, glType, cubeImage[ i ] );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tfor ( let j = 0; j < mipmaps.length; j ++ ) {\n\n\t\t\t\t\t\t\tconst mipmap = mipmaps[ j ];\n\n\t\t\t\t\t\t\tif ( useTexStorage ) {\n\n\t\t\t\t\t\t\t\tif ( dataReady ) {\n\n\t\t\t\t\t\t\t\t\tstate.texSubImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j + 1, 0, 0, glFormat, glType, mipmap.image[ i ] );\n\n\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t} else {\n\n\t\t\t\t\t\t\t\tstate.texImage2D( _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, j + 1, glInternalFormat, glFormat, glType, mipmap.image[ i ] );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tif ( textureNeedsGenerateMipmaps( texture ) ) {\n\n\t\t\t\t// We assume images for cube map have the same size.\n\t\t\t\tgenerateMipmap( _gl.TEXTURE_CUBE_MAP );\n\n\t\t\t}\n\n\t\t\tsourceProperties.__version = source.version;\n\n\t\t\tif ( texture.onUpdate ) texture.onUpdate( texture );\n\n\t\t}\n\n\t\ttextureProperties.__version = texture.version;\n\n\t}\n\n\t// Render targets\n\n\t// Setup storage for target texture and bind it to correct framebuffer\n\t\nfunction setupFrameBufferTexture( framebuffer, renderTarget, texture, attachment, textureTarget, level ) {\n\n\t\tconst glFormat = utils.convert( texture.format, texture.colorSpace );\n\t\tconst glType = utils.convert( texture.type );\n\t\tconst glInternalFormat = getInternalFormat( texture.internalFormat, glFormat, glType, texture.colorSpace );\n\t\tconst renderTargetProperties = properties.get( renderTarget );\n\n\t\tif ( ! renderTargetProperties.__hasExternalTextures ) {\n\n\t\t\tconst width = Math.max( 1, renderTarget.width >> level );\n\t\t\tconst height = Math.max( 1, renderTarget.height >> level );\n\n\t\t\tif ( textureTarget === _gl.TEXTURE_3D || textureTarget === _gl.TEXTURE_2D_ARRAY ) {\n\n\t\t\t\tstate.texImage3D( textureTarget, level, glInternalFormat, width, height, renderTarget.depth, 0, glFormat, glType, null );\n\n\t\t\t} else {\n\n\t\t\t\tstate.texImage2D( textureTarget, level, glInternalFormat, width, height, 0, glFormat, glType, null );\n\n\t\t\t}\n\n\t\t}\n\n\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, framebuffer );\n\n\t\tif ( useMultisampledRTT( renderTarget ) ) {\n\n\t\t\tmultisampledRTTExt.framebufferTexture2DMultisampleEXT( _gl.FRAMEBUFFER, attachment, textureTarget, properties.get( texture ).__webglTexture, 0, getRenderTargetSamples( renderTarget ) );\n\n\t\t} else if ( textureTarget === _gl.TEXTURE_2D || ( textureTarget >= _gl.TEXTURE_CUBE_MAP_POSITIVE_X && textureTarget <= _gl.TEXTURE_CUBE_MAP_NEGATIVE_Z ) ) { // see #24753\n\n\t\t\t_gl.framebufferTexture2D( _gl.FRAMEBUFFER, attachment, textureTarget, properties.get( texture ).__webglTexture, level );\n\n\t\t}\n\n\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, null );\n\n\t}\n\n\n\t// Setup storage for internal depth/stencil buffers and bind to correct framebuffer\n\tfunction setupRenderBufferStorage( renderbuffer, renderTarget, isMultisample ) {\n\n\t\t_gl.bindRenderbuffer( _gl.RENDERBUFFER, renderbuffer );\n\n\t\tif ( renderTarget.depthBuffer && ! renderTarget.stencilBuffer ) {\n\n\t\t\tlet glInternalFormat = _gl.DEPTH_COMPONENT24;\n\n\t\t\tif ( isMultisample || useMultisampledRTT( renderTarget ) ) {\n\n\t\t\t\tconst depthTexture = renderTarget.depthTexture;\n\n\t\t\t\tif ( depthTexture && depthTexture.isDepthTexture ) {\n\n\t\t\t\t\tif ( depthTexture.type === FloatType ) {\n\n\t\t\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT32F;\n\n\t\t\t\t\t} else if ( depthTexture.type === UnsignedIntType ) {\n\n\t\t\t\t\t\tglInternalFormat = _gl.DEPTH_COMPONENT24;\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\tconst samples = getRenderTargetSamples( renderTarget );\n\n\t\t\t\tif ( useMultisampledRTT( renderTarget ) ) {\n\n\t\t\t\t\tmultisampledRTTExt.renderbufferStorageMultisampleEXT( _gl.RENDERBUFFER, samples, glInternalFormat, renderTarget.width, renderTarget.height );\n\n\t\t\t\t} else {\n\n\t\t\t\t\t_gl.renderbufferStorageMultisample( _gl.RENDERBUFFER, samples, glInternalFormat, renderTarget.width, renderTarget.height );\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\t_gl.renderbufferStorage( _gl.RENDERBUFFER, glInternalFormat, renderTarget.width, renderTarget.height );\n\n\t\t\t}\n\n\t\t\t_gl.framebufferRenderbuffer( _gl.FRAMEBUFFER, _gl.DEPTH_ATTACHMENT, _gl.RENDERBUFFER, renderbuffer );\n\n\t\t} else if ( renderTarget.depthBuffer && renderTarget.stencilBuffer ) {\n\n\t\t\tconst samples = getRenderTargetSamples( renderTarget );\n\n\t\t\tif ( isMultisample && useMultisampledRTT( renderTarget ) === false ) {\n\n\t\t\t\t_gl.renderbufferStorageMultisample( _gl.RENDERBUFFER, samples, _gl.DEPTH24_STENCIL8, renderTarget.width, renderTarget.height );\n\n\t\t\t} else if ( useMultisampledRTT( renderTarget ) ) {\n\n\t\t\t\tmultisampledRTTExt.renderbufferStorageMultisampleEXT( _gl.RENDERBUFFER, samples, _gl.DEPTH24_STENCIL8, renderTarget.width, renderTarget.height );\n\n\t\t\t} else {\n\n\t\t\t\t_gl.renderbufferStorage( _gl.RENDERBUFFER, _gl.DEPTH_STENCIL, renderTarget.width, renderTarget.height );\n\n\t\t\t}\n\n\n\t\t\t_gl.framebufferRenderbuffer( _gl.FRAMEBUFFER, _gl.DEPTH_STENCIL_ATTACHMENT, _gl.RENDERBUFFER, renderbuffer );\n\n\t\t} else {\n\n\t\t\tconst textures = renderTarget.textures;\n\n\t\t\tfor ( let i = 0; i < textures.length; i ++ ) {\n\n\t\t\t\tconst texture = textures[ i ];\n\n\t\t\t\tconst glFormat = utils.convert( texture.format, texture.colorSpace );\n\t\t\t\tconst glType = utils.convert( texture.type );\n\t\t\t\tconst glInternalFormat = getInternalFormat( texture.internalFormat, glFormat, glType, texture.colorSpace );\n\t\t\t\tconst samples = getRenderTargetSamples( renderTarget );\n\n\t\t\t\tif ( isMultisample && useMultisampledRTT( renderTarget ) === false ) {\n\n\t\t\t\t\t_gl.renderbufferStorageMultisample( _gl.RENDERBUFFER, samples, glInternalFormat, renderTarget.width, renderTarget.height );\n\n\t\t\t\t} else if ( useMultisampledRTT( renderTarget ) ) {\n\n\t\t\t\t\tmultisampledRTTExt.renderbufferStorageMultisampleEXT( _gl.RENDERBUFFER, samples, glInternalFormat, renderTarget.width, renderTarget.height );\n\n\t\t\t\t} else {\n\n\t\t\t\t\t_gl.renderbufferStorage( _gl.RENDERBUFFER, glInternalFormat, renderTarget.width, renderTarget.height );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\t_gl.bindRenderbuffer( _gl.RENDERBUFFER, null );\n\n\t}\n\n\t// Setup resources for a Depth Texture for a FBO (needs an extension)\n\tfunction setupDepthTexture( framebuffer, renderTarget ) {\n\n\t\tconst isCube = ( renderTarget && renderTarget.isWebGLCubeRenderTarget );\n\t\tif ( isCube ) throw new Error( 'Depth Texture with cube render targets is not supported' );\n\n\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, framebuffer );\n\n\t\tif ( ! ( renderTarget.depthTexture && renderTarget.depthTexture.isDepthTexture ) ) {\n\n\t\t\tthrow new Error( 'renderTarget.depthTexture must be an instance of THREE.DepthTexture' );\n\n\t\t}\n\n\t\t// upload an empty depth texture with framebuffer size\n\t\tif ( ! properties.get( renderTarget.depthTexture ).__webglTexture ||\n\t\t\t\trenderTarget.depthTexture.image.width !== renderTarget.width ||\n\t\t\t\trenderTarget.depthTexture.image.height !== renderTarget.height ) {\n\n\t\t\trenderTarget.depthTexture.image.width = renderTarget.width;\n\t\t\trenderTarget.depthTexture.image.height = renderTarget.height;\n\t\t\trenderTarget.depthTexture.needsUpdate = true;\n\n\t\t}\n\n\t\tsetTexture2D( renderTarget.depthTexture, 0 );\n\n\t\tconst webglDepthTexture = properties.get( renderTarget.depthTexture ).__webglTexture;\n\t\tconst samples = getRenderTargetSamples( renderTarget );\n\n\t\tif ( renderTarget.depthTexture.format === DepthFormat ) {\n\n\t\t\tif ( useMultisampledRTT( renderTarget ) ) {\n\n\t\t\t\tmultisampledRTTExt.framebufferTexture2DMultisampleEXT( _gl.FRAMEBUFFER, _gl.DEPTH_ATTACHMENT, _gl.TEXTURE_2D, webglDepthTexture, 0, samples );\n\n\t\t\t} else {\n\n\t\t\t\t_gl.framebufferTexture2D( _gl.FRAMEBUFFER, _gl.DEPTH_ATTACHMENT, _gl.TEXTURE_2D, webglDepthTexture, 0 );\n\n\t\t\t}\n\n\t\t} else if ( renderTarget.depthTexture.format === DepthStencilFormat ) {\n\n\t\t\tif ( useMultisampledRTT( renderTarget ) ) {\n\n\t\t\t\tmultisampledRTTExt.framebufferTexture2DMultisampleEXT( _gl.FRAMEBUFFER, _gl.DEPTH_STENCIL_ATTACHMENT, _gl.TEXTURE_2D, webglDepthTexture, 0, samples );\n\n\t\t\t} else {\n\n\t\t\t\t_gl.framebufferTexture2D( _gl.FRAMEBUFFER, _gl.DEPTH_STENCIL_ATTACHMENT, _gl.TEXTURE_2D, webglDepthTexture, 0 );\n\n\t\t\t}\n\n\t\t} else {\n\n\t\t\tthrow new Error( 'Unknown depthTexture format' );\n\n\t\t}\n\n\t}\n\n\t// Setup GL resources for a non-texture depth buffer\n\tfunction setupDepthRenderbuffer( renderTarget ) {\n\n\t\tconst renderTargetProperties = properties.get( renderTarget );\n\t\tconst isCube = ( renderTarget.isWebGLCubeRenderTarget === true );\n\n\t\tif ( renderTarget.depthTexture && ! renderTargetProperties.__autoAllocateDepthBuffer ) {\n\n\t\t\tif ( isCube ) throw new Error( 'target.depthTexture not supported in Cube render targets' );\n\n\t\t\tsetupDepthTexture( renderTargetProperties.__webglFramebuffer, renderTarget );\n\n\t\t} else {\n\n\t\t\tif ( isCube ) {\n\n\t\t\t\trenderTargetProperties.__webglDepthbuffer = [];\n\n\t\t\t\tfor ( let i = 0; i < 6; i ++ ) {\n\n\t\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, renderTargetProperties.__webglFramebuffer[ i ] );\n\t\t\t\t\trenderTargetProperties.__webglDepthbuffer[ i ] = _gl.createRenderbuffer();\n\t\t\t\t\tsetupRenderBufferStorage( renderTargetProperties.__webglDepthbuffer[ i ], renderTarget, false );\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, renderTargetProperties.__webglFramebuffer );\n\t\t\t\trenderTargetProperties.__webglDepthbuffer = _gl.createRenderbuffer();\n\t\t\t\tsetupRenderBufferStorage( renderTargetProperties.__webglDepthbuffer, renderTarget, false );\n\n\t\t\t}\n\n\t\t}\n\n\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, null );\n\n\t}\n\n\t// rebind framebuffer with external textures\n\tfunction rebindTextures( renderTarget, colorTexture, depthTexture ) {\n\n\t\tconst renderTargetProperties = properties.get( renderTarget );\n\n\t\tif ( colorTexture !== undefined ) {\n\n\t\t\tsetupFrameBufferTexture( renderTargetProperties.__webglFramebuffer, renderTarget, renderTarget.texture, _gl.COLOR_ATTACHMENT0, _gl.TEXTURE_2D, 0 );\n\n\t\t}\n\n\t\tif ( depthTexture !== undefined ) {\n\n\t\t\tsetupDepthRenderbuffer( renderTarget );\n\n\t\t}\n\n\t}\n\n\t// Set up GL resources for the render target\n\tfunction setupRenderTarget( renderTarget ) {\n\n\t\tconst texture = renderTarget.texture;\n\n\t\tconst renderTargetProperties = properties.get( renderTarget );\n\t\tconst textureProperties = properties.get( texture );\n\n\t\trenderTarget.addEventListener( 'dispose', onRenderTargetDispose );\n\n\t\tconst textures = renderTarget.textures;\n\n\t\tconst isCube = ( renderTarget.isWebGLCubeRenderTarget === true );\n\t\tconst isMultipleRenderTargets = ( textures.length > 1 );\n\n\t\tif ( ! isMultipleRenderTargets ) {\n\n\t\t\tif ( textureProperties.__webglTexture === undefined ) {\n\n\t\t\t\ttextureProperties.__webglTexture = _gl.createTexture();\n\n\t\t\t}\n\n\t\t\ttextureProperties.__version = texture.version;\n\t\t\tinfo.memory.textures ++;\n\n\t\t}\n\n\t\t// Setup framebuffer\n\n\t\tif ( isCube ) {\n\n\t\t\trenderTargetProperties.__webglFramebuffer = [];\n\n\t\t\tfor ( let i = 0; i < 6; i ++ ) {\n\n\t\t\t\tif ( texture.mipmaps && texture.mipmaps.length > 0 ) {\n\n\t\t\t\t\trenderTargetProperties.__webglFramebuffer[ i ] = [];\n\n\t\t\t\t\tfor ( let level = 0; level < texture.mipmaps.length; level ++ ) {\n\n\t\t\t\t\t\trenderTargetProperties.__webglFramebuffer[ i ][ level ] = _gl.createFramebuffer();\n\n\t\t\t\t\t}\n\n\t\t\t\t} else {\n\n\t\t\t\t\trenderTargetProperties.__webglFramebuffer[ i ] = _gl.createFramebuffer();\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t} else {\n\n\t\t\tif ( texture.mipmaps && texture.mipmaps.length > 0 ) {\n\n\t\t\t\trenderTargetProperties.__webglFramebuffer = [];\n\n\t\t\t\tfor ( let level = 0; level < texture.mipmaps.length; level ++ ) {\n\n\t\t\t\t\trenderTargetProperties.__webglFramebuffer[ level ] = _gl.createFramebuffer();\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\trenderTargetProperties.__webglFramebuffer = _gl.createFramebuffer();\n\n\t\t\t}\n\n\t\t\tif ( isMultipleRenderTargets ) {\n\n\t\t\t\tfor ( let i = 0, il = textures.length; i < il; i ++ ) {\n\n\t\t\t\t\tconst attachmentProperties = properties.get( textures[ i ] );\n\n\t\t\t\t\tif ( attachmentProperties.__webglTexture === undefined ) {\n\n\t\t\t\t\t\tattachmentProperties.__webglTexture = _gl.createTexture();\n\n\t\t\t\t\t\tinfo.memory.textures ++;\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tif ( ( renderTarget.samples > 0 ) && useMultisampledRTT( renderTarget ) === false ) {\n\n\t\t\t\trenderTargetProperties.__webglMultisampledFramebuffer = _gl.createFramebuffer();\n\t\t\t\trenderTargetProperties.__webglColorRenderbuffer = [];\n\n\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, renderTargetProperties.__webglMultisampledFramebuffer );\n\n\t\t\t\tfor ( let i = 0; i < textures.length; i ++ ) {\n\n\t\t\t\t\tconst texture = textures[ i ];\n\t\t\t\t\trenderTargetProperties.__webglColorRenderbuffer[ i ] = _gl.createRenderbuffer();\n\n\t\t\t\t\t_gl.bindRenderbuffer( _gl.RENDERBUFFER, renderTargetProperties.__webglColorRenderbuffer[ i ] );\n\n\t\t\t\t\tconst glFormat = utils.convert( texture.format, texture.colorSpace );\n\t\t\t\t\tconst glType = utils.convert( texture.type );\n\t\t\t\t\tconst glInternalFormat = getInternalFormat( texture.internalFormat, glFormat, glType, texture.colorSpace, renderTarget.isXRRenderTarget === true );\n\t\t\t\t\tconst samples = getRenderTargetSamples( renderTarget );\n\t\t\t\t\t_gl.renderbufferStorageMultisample( _gl.RENDERBUFFER, samples, glInternalFormat, renderTarget.width, renderTarget.height );\n\n\t\t\t\t\t_gl.framebufferRenderbuffer( _gl.FRAMEBUFFER, _gl.COLOR_ATTACHMENT0 + i, _gl.RENDERBUFFER, renderTargetProperties.__webglColorRenderbuffer[ i ] );\n\n\t\t\t\t}\n\n\t\t\t\t_gl.bindRenderbuffer( _gl.RENDERBUFFER, null );\n\n\t\t\t\tif ( renderTarget.depthBuffer ) {\n\n\t\t\t\t\trenderTargetProperties.__webglDepthRenderbuffer = _gl.createRenderbuffer();\n\t\t\t\t\tsetupRenderBufferStorage( renderTargetProperties.__webglDepthRenderbuffer, renderTarget, true );\n\n\t\t\t\t}\n\n\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, null );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// Setup color buffer\n\n\t\tif ( isCube ) {\n\n\t\t\tstate.bindTexture( _gl.TEXTURE_CUBE_MAP, textureProperties.__webglTexture );\n\t\t\tsetTextureParameters( _gl.TEXTURE_CUBE_MAP, texture );\n\n\t\t\tfor ( let i = 0; i < 6; i ++ ) {\n\n\t\t\t\tif ( texture.mipmaps && texture.mipmaps.length > 0 ) {\n\n\t\t\t\t\tfor ( let level = 0; level < texture.mipmaps.length; level ++ ) {\n\n\t\t\t\t\t\tsetupFrameBufferTexture( renderTargetProperties.__webglFramebuffer[ i ][ level ], renderTarget, texture, _gl.COLOR_ATTACHMENT0, _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, level );\n\n\t\t\t\t\t}\n\n\t\t\t\t} else {\n\n\t\t\t\t\tsetupFrameBufferTexture( renderTargetProperties.__webglFramebuffer[ i ], renderTarget, texture, _gl.COLOR_ATTACHMENT0, _gl.TEXTURE_CUBE_MAP_POSITIVE_X + i, 0 );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tif ( textureNeedsGenerateMipmaps( texture ) ) {\n\n\t\t\t\tgenerateMipmap( _gl.TEXTURE_CUBE_MAP );\n\n\t\t\t}\n\n\t\t\tstate.unbindTexture();\n\n\t\t} else if ( isMultipleRenderTargets ) {\n\n\t\t\tfor ( let i = 0, il = textures.length; i < il; i ++ ) {\n\n\t\t\t\tconst attachment = textures[ i ];\n\t\t\t\tconst attachmentProperties = properties.get( attachment );\n\n\t\t\t\tstate.bindTexture( _gl.TEXTURE_2D, attachmentProperties.__webglTexture );\n\t\t\t\tsetTextureParameters( _gl.TEXTURE_2D, attachment );\n\t\t\t\tsetupFrameBufferTexture( renderTargetProperties.__webglFramebuffer, renderTarget, attachment, _gl.COLOR_ATTACHMENT0 + i, _gl.TEXTURE_2D, 0 );\n\n\t\t\t\tif ( textureNeedsGenerateMipmaps( attachment ) ) {\n\n\t\t\t\t\tgenerateMipmap( _gl.TEXTURE_2D );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tstate.unbindTexture();\n\n\t\t} else {\n\n\t\t\tlet glTextureType = _gl.TEXTURE_2D;\n\n\t\t\tif ( renderTarget.isWebGL3DRenderTarget || renderTarget.isWebGLArrayRenderTarget ) {\n\n\t\t\t\tglTextureType = renderTarget.isWebGL3DRenderTarget ? _gl.TEXTURE_3D : _gl.TEXTURE_2D_ARRAY;\n\n\t\t\t}\n\n\t\t\tstate.bindTexture( glTextureType, textureProperties.__webglTexture );\n\t\t\tsetTextureParameters( glTextureType, texture );\n\n\t\t\tif ( texture.mipmaps && texture.mipmaps.length > 0 ) {\n\n\t\t\t\tfor ( let level = 0; level < texture.mipmaps.length; level ++ ) {\n\n\t\t\t\t\tsetupFrameBufferTexture( renderTargetProperties.__webglFramebuffer[ level ], renderTarget, texture, _gl.COLOR_ATTACHMENT0, glTextureType, level );\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\tsetupFrameBufferTexture( renderTargetProperties.__webglFramebuffer, renderTarget, texture, _gl.COLOR_ATTACHMENT0, glTextureType, 0 );\n\n\t\t\t}\n\n\t\t\tif ( textureNeedsGenerateMipmaps( texture ) ) {\n\n\t\t\t\tgenerateMipmap( glTextureType );\n\n\t\t\t}\n\n\t\t\tstate.unbindTexture();\n\n\t\t}\n\n\t\t// Setup depth and stencil buffers\n\n\t\tif ( renderTarget.depthBuffer ) {\n\n\t\t\tsetupDepthRenderbuffer( renderTarget );\n\n\t\t}\n\n\t}\n\n\tfunction updateRenderTargetMipmap( renderTarget ) {\n\n\t\tconst textures = renderTarget.textures;\n\n\t\tfor ( let i = 0, il = textures.length; i < il; i ++ ) {\n\n\t\t\tconst texture = textures[ i ];\n\n\t\t\tif ( textureNeedsGenerateMipmaps( texture ) ) {\n\n\t\t\t\tconst target = renderTarget.isWebGLCubeRenderTarget ? _gl.TEXTURE_CUBE_MAP : _gl.TEXTURE_2D;\n\t\t\t\tconst webglTexture = properties.get( texture ).__webglTexture;\n\n\t\t\t\tstate.bindTexture( target, webglTexture );\n\t\t\t\tgenerateMipmap( target );\n\t\t\t\tstate.unbindTexture();\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tfunction updateMultisampleRenderTarget( renderTarget ) {\n\n\t\tif ( ( renderTarget.samples > 0 ) && useMultisampledRTT( renderTarget ) === false ) {\n\n\t\t\tconst textures = renderTarget.textures;\n\t\t\tconst width = renderTarget.width;\n\t\t\tconst height = renderTarget.height;\n\t\t\tlet mask = _gl.COLOR_BUFFER_BIT;\n\t\t\tconst invalidationArray = [];\n\t\t\tconst depthStyle = renderTarget.stencilBuffer ? _gl.DEPTH_STENCIL_ATTACHMENT : _gl.DEPTH_ATTACHMENT;\n\t\t\tconst renderTargetProperties = properties.get( renderTarget );\n\t\t\tconst isMultipleRenderTargets = ( textures.length > 1 );\n\n\t\t\t// If MRT we need to remove FBO attachments\n\t\t\tif ( isMultipleRenderTargets ) {\n\n\t\t\t\tfor ( let i = 0; i < textures.length; i ++ ) {\n\n\t\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, renderTargetProperties.__webglMultisampledFramebuffer );\n\t\t\t\t\t_gl.framebufferRenderbuffer( _gl.FRAMEBUFFER, _gl.COLOR_ATTACHMENT0 + i, _gl.RENDERBUFFER, null );\n\n\t\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, renderTargetProperties.__webglFramebuffer );\n\t\t\t\t\t_gl.framebufferTexture2D( _gl.DRAW_FRAMEBUFFER, _gl.COLOR_ATTACHMENT0 + i, _gl.TEXTURE_2D, null, 0 );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tstate.bindFramebuffer( _gl.READ_FRAMEBUFFER, renderTargetProperties.__webglMultisampledFramebuffer );\n\t\t\tstate.bindFramebuffer( _gl.DRAW_FRAMEBUFFER, renderTargetProperties.__webglFramebuffer );\n\n\t\t\tfor ( let i = 0; i < textures.length; i ++ ) {\n\n\t\t\t\tinvalidationArray.push( _gl.COLOR_ATTACHMENT0 + i );\n\n\t\t\t\tif ( renderTarget.depthBuffer ) {\n\n\t\t\t\t\tinvalidationArray.push( depthStyle );\n\n\t\t\t\t}\n\n\t\t\t\tconst ignoreDepthValues = ( renderTargetProperties.__ignoreDepthValues !== undefined ) ? renderTargetProperties.__ignoreDepthValues : false;\n\n\t\t\t\tif ( ignoreDepthValues === false ) {\n\n\t\t\t\t\tif ( renderTarget.depthBuffer ) mask |= _gl.DEPTH_BUFFER_BIT;\n\n\t\t\t\t\t// resolving stencil is slow with a D3D backend. disable it for all transmission render targets (see #27799)\n\n\t\t\t\t\tif ( renderTarget.stencilBuffer && renderTarget.resolveStencilBuffer ) mask |= _gl.STENCIL_BUFFER_BIT;\n\n\t\t\t\t}\n\n\t\t\t\tif ( isMultipleRenderTargets ) {\n\n\t\t\t\t\t_gl.framebufferRenderbuffer( _gl.READ_FRAMEBUFFER, _gl.COLOR_ATTACHMENT0, _gl.RENDERBUFFER, renderTargetProperties.__webglColorRenderbuffer[ i ] );\n\n\t\t\t\t}\n\n\t\t\t\tif ( ignoreDepthValues === true ) {\n\n\t\t\t\t\t_gl.invalidateFramebuffer( _gl.READ_FRAMEBUFFER, [ depthStyle ] );\n\t\t\t\t\t_gl.invalidateFramebuffer( _gl.DRAW_FRAMEBUFFER, [ depthStyle ] );\n\n\t\t\t\t}\n\n\t\t\t\tif ( isMultipleRenderTargets ) {\n\n\t\t\t\t\tconst webglTexture = properties.get( textures[ i ] ).__webglTexture;\n\t\t\t\t\t_gl.framebufferTexture2D( _gl.DRAW_FRAMEBUFFER, _gl.COLOR_ATTACHMENT0, _gl.TEXTURE_2D, webglTexture, 0 );\n\n\t\t\t\t}\n\n\t\t\t\t_gl.blitFramebuffer( 0, 0, width, height, 0, 0, width, height, mask, _gl.NEAREST );\n\n\t\t\t\tif ( supportsInvalidateFramebuffer ) {\n\n\t\t\t\t\t_gl.invalidateFramebuffer( _gl.READ_FRAMEBUFFER, invalidationArray );\n\n\t\t\t\t}\n\n\n\t\t\t}\n\n\t\t\tstate.bindFramebuffer( _gl.READ_FRAMEBUFFER, null );\n\t\t\tstate.bindFramebuffer( _gl.DRAW_FRAMEBUFFER, null );\n\n\t\t\t// If MRT since pre-blit we removed the FBO we need to reconstruct the attachments\n\t\t\tif ( isMultipleRenderTargets ) {\n\n\t\t\t\tfor ( let i = 0; i < textures.length; i ++ ) {\n\n\t\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, renderTargetProperties.__webglMultisampledFramebuffer );\n\t\t\t\t\t_gl.framebufferRenderbuffer( _gl.FRAMEBUFFER, _gl.COLOR_ATTACHMENT0 + i, _gl.RENDERBUFFER, renderTargetProperties.__webglColorRenderbuffer[ i ] );\n\n\t\t\t\t\tconst webglTexture = properties.get( textures[ i ] ).__webglTexture;\n\n\t\t\t\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, renderTargetProperties.__webglFramebuffer );\n\t\t\t\t\t_gl.framebufferTexture2D( _gl.DRAW_FRAMEBUFFER, _gl.COLOR_ATTACHMENT0 + i, _gl.TEXTURE_2D, webglTexture, 0 );\n\n\t\t\t\t}\n\n\t\t\t}\n\n...\n", "needle_token_start": 7983, "needle_token_end": 8526, "code_context_ntokens": 15043, "output": ["```javascript\nfunction setupFrameBufferTexture( framebuffer, renderTarget, texture, attachment, textureTarget, level ) {\n\n\t\tconst glFormat = utils.convert( texture.format, texture.colorSpace );\n\t\tconst glType = utils.convert( texture.type );\n\t\tconst glInternalFormat = getInternalFormat( texture.internalFormat, glFormat, glType, texture.colorSpace );\n\t\tconst renderTargetProperties = properties.get( renderTarget );\n\n\t\tif ( ! renderTargetProperties.__hasExternalTextures ) {\n\n\t\t\tconst width = Math.max( 1, renderTarget.width >> level );\n\t\t\tconst height = Math.max( 1, renderTarget.height >> level );\n\n\t\t\tif ( textureTarget === _gl.TEXTURE_3D || textureTarget === _gl.TEXTURE_2D_ARRAY ) {\n\n\t\t\t\tstate.texImage3D( textureTarget, level, glInternalFormat, width, height, renderTarget.depth, 0, glFormat, glType, null );\n\n\t\t\t} else {\n\n\t\t\t\tstate.texImage2D( textureTarget, level, glInternalFormat, width, height, 0, glFormat, glType, null );\n\n\t\t\t}\n\n\t\t}\n\n\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, framebuffer );\n\n\t\tif ( useMultisampledRTT( renderTarget ) ) {\n\n\t\t\tmultisampledRTTExt.framebufferTexture2DMultisampleEXT( _gl.FRAMEBUFFER, attachment, textureTarget, properties.get( texture ).__webglTexture, 0, getRenderTargetSamples( renderTarget ) );\n\n\t\t} else if ( textureTarget === _gl.TEXTURE_2D || ( textureTarget >= _gl.TEXTURE_CUBE_MAP_POSITIVE_X && textureTarget <= _gl.TEXTURE_CUBE_MAP_NEGATIVE_Z ) ) { // see #24753\n\n\t\t\t_gl.framebufferTexture2D( _gl.FRAMEBUFFER, attachment, textureTarget, properties.get( texture ).__webglTexture, level );\n\n\t\t}\n\n\t\tstate.bindFramebuffer( _gl.FRAMEBUFFER, null );\n\n\t}\n```"]}
{"repo": "mrdoob/three.js", "name": "convertArray", "language": "typescript", "path": "src/animation/AnimationUtils.js", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert an array into a specified type, either a typed array or a regular array, depending on the type specified and certain conditions.\n2. **Input**: The function takes three parameters: the original array, the desired type for the output array, and a boolean indicating whether to force cloning even if the original array is already of the desired type.\n3. **Output**: Returns a new array of the specified type. If the input array is already of the desired type and cloning is not forced, it returns the original array.\n4. **Procedure**: \n   - First, the function checks if the input array is undefined or null, or if it is already of the desired type and cloning is not forced. If any of these conditions are met, it returns the original array.\n   - If the desired type has a property indicating it is a typed array (checked via `BYTES_PER_ELEMENT`), it creates a new typed array from the input array.\n   - If the desired type is a regular array, it creates a new array using the slice method to copy the elements of the input array.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/core/Object3D.js\nimport { Quaternion } from '../math/Quaternion.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Matrix4 } from '../math/Matrix4.js';\nimport { EventDispatcher } from './EventDispatcher.js';\nimport { Euler } from '../math/Euler.js';\nimport { Layers } from './Layers.js';\nimport { Matrix3 } from '../math/Matrix3.js';\nimport * as MathUtils from '../math/MathUtils.js';\n\nlet _object3DId = 0;\n\nconst _v1 = /*@__PURE__*/ new Vector3();\nconst _q1 = /*@__PURE__*/ new Quaternion();\nconst _m1 = /*@__PURE__*/ new Matrix4();\nconst _target = /*@__PURE__*/ new Vector3();\n\nconst _position = /*@__PURE__*/ new Vector3();\nconst _scale = /*@__PURE__*/ new Vector3();\nconst _quaternion = /*@__PURE__*/ new Quaternion();\n\nconst _xAxis = /*@__PURE__*/ new Vector3( 1, 0, 0 );\nconst _yAxis = /*@__PURE__*/ new Vector3( 0, 1, 0 );\nconst _zAxis = /*@__PURE__*/ new Vector3( 0, 0, 1 );\n\nconst _addedEvent = { type: 'added' };\nconst _removedEvent = { type: 'removed' };\n\nconst _childaddedEvent = { type: 'childadded', child: null };\nconst _childremovedEvent = { type: 'childremoved', child: null };\n\nclass Object3D extends EventDispatcher {\n\n\tconstructor() {\n\n\t\tsuper();\n\n\t\tthis.isObject3D = true;\n\n\t\tObject.defineProperty( this, 'id', { value: _object3DId ++ } );\n\n\t\tthis.uuid = MathUtils.generateUUID();\n\n\t\tthis.name = '';\n\t\tthis.type = 'Object3D';\n\n\t\tthis.parent = null;\n\t\tthis.children = [];\n\n\t\tthis.up = Object3D.DEFAULT_UP.clone();\n\n\t\tconst position = new Vector3();\n\t\tconst rotation = new Euler();\n\t\tconst quaternion = new Quaternion();\n\t\tconst scale = new Vector3( 1, 1, 1 );\n\n\t\tfunction onRotationChange() {\n\n\t\t\tquaternion.setFromEuler( rotation, false );\n\n\t\t}\n\n\t\tfunction onQuaternionChange() {\n\n\t\t\trotation.setFromQuaternion( quaternion, undefined, false );\n\n\t\t}\n\n\t\trotation._onChange( onRotationChange );\n\t\tquaternion._onChange( onQuaternionChange );\n\n\t\tObject.defineProperties( this, {\n\t\t\tposition: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: position\n\t\t\t},\n\t\t\trotation: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: rotation\n\t\t\t},\n\t\t\tquaternion: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: quaternion\n\t\t\t},\n\t\t\tscale: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: scale\n\t\t\t},\n\t\t\tmodelViewMatrix: {\n\t\t\t\tvalue: new Matrix4()\n\t\t\t},\n\t\t\tnormalMatrix: {\n\t\t\t\tvalue: new Matrix3()\n\t\t\t}\n\t\t} );\n\n\t\tthis.matrix = new Matrix4();\n\t\tthis.matrixWorld = new Matrix4();\n\n\t\tthis.matrixAutoUpdate = Object3D.DEFAULT_MATRIX_AUTO_UPDATE;\n\n\t\tthis.matrixWorldAutoUpdate = Object3D.DEFAULT_MATRIX_WORLD_AUTO_UPDATE; // checked by the renderer\n\t\tthis.matrixWorldNeedsUpdate = false;\n\n\t\tthis.layers = new Layers();\n\t\tthis.visible = true;\n\n\t\tthis.castShadow = false;\n\t\tthis.receiveShadow = false;\n\n\t\tthis.frustumCulled = true;\n\t\tthis.renderOrder = 0;\n\n\t\tthis.animations = [];\n\n\t\tthis.userData = {};\n\n\t}\n\n\tonBeforeShadow( /* renderer, object, camera, shadowCamera, geometry, depthMaterial, group */ ) {}\n\n\tonAfterShadow( /* renderer, object, camera, shadowCamera, geometry, depthMaterial, group */ ) {}\n\n\tonBeforeRender( /* renderer, scene, camera, geometry, material, group */ ) {}\n\n\tonAfterRender( /* renderer, scene, camera, geometry, material, group */ ) {}\n\n\tapplyMatrix4( matrix ) {\n\n\t\tif ( this.matrixAutoUpdate ) this.updateMatrix();\n\n\t\tthis.matrix.premultiply( matrix );\n\n\t\tthis.matrix.decompose( this.position, this.quaternion, this.scale );\n\n\t}\n\n\tapplyQuaternion( q ) {\n\n\t\tthis.quaternion.premultiply( q );\n\n\t\treturn this;\n\n\t}\n\n\tsetRotationFromAxisAngle( axis, angle ) {\n\n\t\t// assumes axis is normalized\n\n\t\tthis.quaternion.setFromAxisAngle( axis, angle );\n\n\t}\n\n\tsetRotationFromEuler( euler ) {\n\n\t\tthis.quaternion.setFromEuler( euler, true );\n\n\t}\n\n\tsetRotationFromMatrix( m ) {\n\n\t\t// assumes the upper 3x3 of m is a pure rotation matrix (i.e, unscaled)\n\n\t\tthis.quaternion.setFromRotationMatrix( m );\n\n\t}\n\n\tsetRotationFromQuaternion( q ) {\n\n\t\t// assumes q is normalized\n\n\t\tthis.quaternion.copy( q );\n\n\t}\n\n\trotateOnAxis( axis, angle ) {\n\n\t\t// rotate object on axis in object space\n\t\t// axis is assumed to be normalized\n\n\t\t_q1.setFromAxisAngle( axis, angle );\n\n\t\tthis.quaternion.multiply( _q1 );\n\n\t\treturn this;\n\n\t}\n\n\trotateOnWorldAxis( axis, angle ) {\n\n\t\t// rotate object on axis in world space\n\t\t// axis is assumed to be normalized\n\t\t// method assumes no rotated parent\n\n\t\t_q1.setFromAxisAngle( axis, angle );\n\n\t\tthis.quaternion.premultiply( _q1 );\n\n\t\treturn this;\n\n\t}\n\n\trotateX( angle ) {\n\n\t\treturn this.rotateOnAxis( _xAxis, angle );\n\n\t}\n\n\trotateY( angle ) {\n\n\t\treturn this.rotateOnAxis( _yAxis, angle );\n\n\t}\n\n\trotateZ( angle ) {\n\n\t\treturn this.rotateOnAxis( _zAxis, angle );\n\n\t}\n\n\ttranslateOnAxis( axis, distance ) {\n\n\t\t// translate object by distance along axis in object space\n\t\t// axis is assumed to be normalized\n\n\t\t_v1.copy( axis ).applyQuaternion( this.quaternion );\n\n\t\tthis.position.add( _v1.multiplyScalar( distance ) );\n\n\t\treturn this;\n\n\t}\n\n\ttranslateX( distance ) {\n\n\t\treturn this.translateOnAxis( _xAxis, distance );\n\n\t}\n\n\ttranslateY( distance ) {\n\n\t\treturn this.translateOnAxis( _yAxis, distance );\n\n\t}\n\n\ttranslateZ( distance ) {\n\n\t\treturn this.translateOnAxis( _zAxis, distance );\n\n\t}\n\n\tlocalToWorld( vector ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\treturn vector.applyMatrix4( this.matrixWorld );\n\n\t}\n\n\tworldToLocal( vector ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\treturn vector.applyMatrix4( _m1.copy( this.matrixWorld ).invert() );\n\n\t}\n\n\tlookAt( x, y, z ) {\n\n\t\t// This method does not support objects having non-uniformly-scaled parent(s)\n\n\t\tif ( x.isVector3 ) {\n\n\t\t\t_target.copy( x );\n\n\t\t} else {\n\n\t\t\t_target.set( x, y, z );\n\n\t\t}\n\n\t\tconst parent = this.parent;\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\t_position.setFromMatrixPosition( this.matrixWorld );\n\n\t\tif ( this.isCamera || this.isLight ) {\n\n\t\t\t_m1.lookAt( _position, _target, this.up );\n\n\t\t} else {\n\n\t\t\t_m1.lookAt( _target, _position, this.up );\n\n\t\t}\n\n\t\tthis.quaternion.setFromRotationMatrix( _m1 );\n\n\t\tif ( parent ) {\n\n\t\t\t_m1.extractRotation( parent.matrixWorld );\n\t\t\t_q1.setFromRotationMatrix( _m1 );\n\t\t\tthis.quaternion.premultiply( _q1.invert() );\n\n\t\t}\n\n\t}\n\n\tadd( object ) {\n\n\t\tif ( arguments.length > 1 ) {\n\n\t\t\tfor ( let i = 0; i < arguments.length; i ++ ) {\n\n\t\t\t\tthis.add( arguments[ i ] );\n\n\t\t\t}\n\n\t\t\treturn this;\n\n\t\t}\n\n\t\tif ( object === this ) {\n\n\t\t\tconsole.error( 'THREE.Object3D.add: object can\\'t be added as a child of itself.', object );\n\t\t\treturn this;\n\n\t\t}\n\n\t\tif ( object && object.isObject3D ) {\n\n\t\t\tobject.removeFromParent();\n\t\t\tobject.parent = this;\n\t\t\tthis.children.push( object );\n\n\t\t\tobject.dispatchEvent( _addedEvent );\n\n\t\t\t_childaddedEvent.child = object;\n\t\t\tthis.dispatchEvent( _childaddedEvent );\n\t\t\t_childaddedEvent.child = null;\n\n\t\t} else {\n\n\t\t\tconsole.error( 'THREE.Object3D.add: object not an instance of THREE.Object3D.', object );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tremove( object ) {\n\n\t\tif ( arguments.length > 1 ) {\n\n\t\t\tfor ( let i = 0; i < arguments.length; i ++ ) {\n\n\t\t\t\tthis.remove( arguments[ i ] );\n\n\t\t\t}\n\n\t\t\treturn this;\n\n\t\t}\n\n\t\tconst index = this.children.indexOf( object );\n\n\t\tif ( index !== - 1 ) {\n\n\t\t\tobject.parent = null;\n\t\t\tthis.children.splice( index, 1 );\n\n\t\t\tobject.dispatchEvent( _removedEvent );\n\n\t\t\t_childremovedEvent.child = object;\n\t\t\tthis.dispatchEvent( _childremovedEvent );\n\t\t\t_childremovedEvent.child = null;\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tremoveFromParent() {\n\n\t\tconst parent = this.parent;\n\n\t\tif ( parent !== null ) {\n\n\t\t\tparent.remove( this );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tclear() {\n\n\t\treturn this.remove( ... this.children );\n\n\t}\n\n\tattach( object ) {\n\n\t\t// adds object as a child of this, while maintaining the object's world transform\n\n\t\t// Note: This method does not support scene graphs having non-uniformly-scaled nodes(s)\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\t_m1.copy( this.matrixWorld ).invert();\n\n\t\tif ( object.parent !== null ) {\n\n\t\t\tobject.parent.updateWorldMatrix( true, false );\n\n\t\t\t_m1.multiply( object.parent.matrixWorld );\n\n\t\t}\n\n\t\tobject.applyMatrix4( _m1 );\n\n\t\tobject.removeFromParent();\n\t\tobject.parent = this;\n\t\tthis.children.push( object );\n\n\t\tobject.updateWorldMatrix( false, true );\n\n\t\tobject.dispatchEvent( _addedEvent );\n\n\t\t_childaddedEvent.child = object;\n\t\tthis.dispatchEvent( _childaddedEvent );\n\t\t_childaddedEvent.child = null;\n\n\t\treturn this;\n\n\t}\n\n\tgetObjectById( id ) {\n\n\t\treturn this.getObjectByProperty( 'id', id );\n\n\t}\n\n\tgetObjectByName( name ) {\n\n\t\treturn this.getObjectByProperty( 'name', name );\n\n\t}\n\n\tgetObjectByProperty( name, value ) {\n\n\t\tif ( this[ name ] === value ) return this;\n\n\t\tfor ( let i = 0, l = this.children.length; i < l; i ++ ) {\n\n\t\t\tconst child = this.children[ i ];\n\t\t\tconst object = child.getObjectByProperty( name, value );\n\n\t\t\tif ( object !== undefined ) {\n\n\t\t\t\treturn object;\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn undefined;\n\n\t}\n\n\tgetObjectsByProperty( name, value, result = [] ) {\n\n\t\tif ( this[ name ] === value ) result.push( this );\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tchildren[ i ].getObjectsByProperty( name, value, result );\n\n\t\t}\n\n\t\treturn result;\n\n\t}\n\n\tgetWorldPosition( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\treturn target.setFromMatrixPosition( this.matrixWorld );\n\n\t}\n\n\tgetWorldQuaternion( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\tthis.matrixWorld.decompose( _position, target, _scale );\n\n\t\treturn target;\n\n\t}\n\n\tgetWorldScale( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\tthis.matrixWorld.decompose( _position, _quaternion, target );\n\n\t\treturn target;\n\n\t}\n\n\tgetWorldDirection( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\tconst e = this.matrixWorld.elements;\n\n\t\treturn target.set( e[ 8 ], e[ 9 ], e[ 10 ] ).normalize();\n\n\t}\n\n\traycast( /* raycaster, intersects */ ) {}\n\n\ttraverse( callback ) {\n\n\t\tcallback( this );\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tchildren[ i ].traverse( callback );\n\n\t\t}\n\n\t}\n\n\ttraverseVisible( callback ) {\n\n\t\tif ( this.visible === false ) return;\n\n\t\tcallback( this );\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tchildren[ i ].traverseVisible( callback );\n\n\t\t}\n\n\t}\n\n\ttraverseAncestors( callback ) {\n\n\t\tconst parent = this.parent;\n\n\t\tif ( parent !== null ) {\n\n\t\t\tcallback( parent );\n\n\t\t\tparent.traverseAncestors( callback );\n\n\t\t}\n\n\t}\n\n\tupdateMatrix() {\n\n\t\tthis.matrix.compose( this.position, this.quaternion, this.scale );\n\n\t\tthis.matrixWorldNeedsUpdate = true;\n\n\t}\n\n\tupdateMatrixWorld( force ) {\n\n\t\tif ( this.matrixAutoUpdate ) this.updateMatrix();\n\n\t\tif ( this.matrixWorldNeedsUpdate || force ) {\n\n\t\t\tif ( this.parent === null ) {\n\n\t\t\t\tthis.matrixWorld.copy( this.matrix );\n\n\t\t\t} else {\n\n\t\t\t\tthis.matrixWorld.multiplyMatrices( this.parent.matrixWorld, this.matrix );\n\n\t\t\t}\n\n\t\t\tthis.matrixWorldNeedsUpdate = false;\n\n\t\t\tforce = true;\n\n\t\t}\n\n\t\t// update children\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tconst child = children[ i ];\n\n\t\t\tif ( child.matrixWorldAutoUpdate === true || force === true ) {\n\n\t\t\t\tchild.updateMatrixWorld( force );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tupdateWorldMatrix( updateParents, updateChildren ) {\n\n\t\tconst parent = this.parent;\n\n\t\tif ( updateParents === true && parent !== null && parent.matrixWorldAutoUpdate === true ) {\n\n\t\t\tparent.updateWorldMatrix( true, false );\n\n\t\t}\n\n\t\tif ( this.matrixAutoUpdate ) this.updateMatrix();\n\n\t\tif ( this.parent === null ) {\n\n\t\t\tthis.matrixWorld.copy( this.matrix );\n\n\t\t} else {\n\n\t\t\tthis.matrixWorld.multiplyMatrices( this.parent.matrixWorld, this.matrix );\n\n\t\t}\n\n\t\t// update children\n\n\t\tif ( updateChildren === true ) {\n\n\t\t\tconst children = this.children;\n\n\t\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\t\tconst child = children[ i ];\n...\n// Path: src/audio/Audio.js\nimport { Object3D } from '../core/Object3D.js';\n\nclass Audio extends Object3D {\n\n\tconstructor( listener ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'Audio';\n\n\t\tthis.listener = listener;\n\t\tthis.context = listener.context;\n\n\t\tthis.gain = this.context.createGain();\n\t\tthis.gain.connect( listener.getInput() );\n\n\t\tthis.autoplay = false;\n\n\t\tthis.buffer = null;\n\t\tthis.detune = 0;\n\t\tthis.loop = false;\n\t\tthis.loopStart = 0;\n\t\tthis.loopEnd = 0;\n\t\tthis.offset = 0;\n\t\tthis.duration = undefined;\n\t\tthis.playbackRate = 1;\n\t\tthis.isPlaying = false;\n\t\tthis.hasPlaybackControl = true;\n\t\tthis.source = null;\n\t\tthis.sourceType = 'empty';\n\n\t\tthis._startedAt = 0;\n\t\tthis._progress = 0;\n\t\tthis._connected = false;\n\n\t\tthis.filters = [];\n\n\t}\n\n\tgetOutput() {\n\n\t\treturn this.gain;\n\n\t}\n\n\tsetNodeSource( audioNode ) {\n\n\t\tthis.hasPlaybackControl = false;\n\t\tthis.sourceType = 'audioNode';\n\t\tthis.source = audioNode;\n\t\tthis.connect();\n\n\t\treturn this;\n\n\t}\n\n\tsetMediaElementSource( mediaElement ) {\n\n\t\tthis.hasPlaybackControl = false;\n\t\tthis.sourceType = 'mediaNode';\n\t\tthis.source = this.context.createMediaElementSource( mediaElement );\n\t\tthis.connect();\n\n\t\treturn this;\n\n\t}\n\n\tsetMediaStreamSource( mediaStream ) {\n\n\t\tthis.hasPlaybackControl = false;\n\t\tthis.sourceType = 'mediaStreamNode';\n\t\tthis.source = this.context.createMediaStreamSource( mediaStream );\n\t\tthis.connect();\n\n\t\treturn this;\n\n\t}\n\n\tsetBuffer( audioBuffer ) {\n\n\t\tthis.buffer = audioBuffer;\n\t\tthis.sourceType = 'buffer';\n\n\t\tif ( this.autoplay ) this.play();\n\n\t\treturn this;\n\n\t}\n\n\tplay( delay = 0 ) {\n\n\t\tif ( this.isPlaying === true ) {\n\n\t\t\tconsole.warn( 'THREE.Audio: Audio is already playing.' );\n\t\t\treturn;\n\n\t\t}\n\n\t\tif ( this.hasPlaybackControl === false ) {\n\n\t\t\tconsole.warn( 'THREE.Audio: this Audio has no playback control.' );\n\t\t\treturn;\n\n\t\t}\n\n\t\tthis._startedAt = this.context.currentTime + delay;\n\n\t\tconst source = this.context.createBufferSource();\n\t\tsource.buffer = this.buffer;\n\t\tsource.loop = this.loop;\n\t\tsource.loopStart = this.loopStart;\n\t\tsource.loopEnd = this.loopEnd;\n\t\tsource.onended = this.onEnded.bind( this );\n\t\tsource.start( this._startedAt, this._progress + this.offset, this.duration );\n\n\t\tthis.isPlaying = true;\n\n\t\tthis.source = source;\n\n\t\tthis.setDetune( this.detune );\n\t\tthis.setPlaybackRate( this.playbackRate );\n\n\t\treturn this.connect();\n\n\t}\n\n\tpause() {\n\n\t\tif ( this.hasPlaybackControl === false ) {\n\n\t\t\tconsole.warn( 'THREE.Audio: this Audio has no playback control.' );\n\t\t\treturn;\n\n\t\t}\n\n\t\tif ( this.isPlaying === true ) {\n\n\t\t\t// update current progress\n\n\t\t\tthis._progress += Math.max( this.context.currentTime - this._startedAt, 0 ) * this.playbackRate;\n\n\t\t\tif ( this.loop === true ) {\n\n\t\t\t\t// ensure _progress does not exceed duration with looped audios\n\n\t\t\t\tthis._progress = this._progress % ( this.duration || this.buffer.duration );\n\n\t\t\t}\n\n\t\t\tthis.source.stop();\n\t\t\tthis.source.onended = null;\n\n\t\t\tthis.isPlaying = false;\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tstop() {\n\n\t\tif ( this.hasPlaybackControl === false ) {\n\n\t\t\tconsole.warn( 'THREE.Audio: this Audio has no playback control.' );\n\t\t\treturn;\n\n\t\t}\n\n\t\tthis._progress = 0;\n\n\t\tif ( this.source !== null ) {\n\n\t\t\tthis.source.stop();\n\t\t\tthis.source.onended = null;\n\n\t\t}\n\n\t\tthis.isPlaying = false;\n\n\t\treturn this;\n\n\t}\n\n\tconnect() {\n\n\t\tif ( this.filters.length > 0 ) {\n\n\t\t\tthis.source.connect( this.filters[ 0 ] );\n\n\t\t\tfor ( let i = 1, l = this.filters.length; i < l; i ++ ) {\n\n\t\t\t\tthis.filters[ i - 1 ].connect( this.filters[ i ] );\n\n\t\t\t}\n\n\t\t\tthis.filters[ this.filters.length - 1 ].connect( this.getOutput() );\n\n\t\t} else {\n\n\t\t\tthis.source.connect( this.getOutput() );\n\n\t\t}\n\n\t\tthis._connected = true;\n\n\t\treturn this;\n\n\t}\n\n\tdisconnect() {\n\n\t\tif ( this._connected === false ) {\n\n\t\t\treturn;\n\n\t\t}\n\n\t\tif ( this.filters.length > 0 ) {\n\n\t\t\tthis.source.disconnect( this.filters[ 0 ] );\n\n\t\t\tfor ( let i = 1, l = this.filters.length; i < l; i ++ ) {\n\n\t\t\t\tthis.filters[ i - 1 ].disconnect( this.filters[ i ] );\n\n\t\t\t}\n\n\t\t\tthis.filters[ this.filters.length - 1 ].disconnect( this.getOutput() );\n\n\t\t} else {\n\n\t\t\tthis.source.disconnect( this.getOutput() );\n\n\t\t}\n\n\t\tthis._connected = false;\n\n\t\treturn this;\n\n\t}\n\n\tgetFilters() {\n\n\t\treturn this.filters;\n\n\t}\n\n\tsetFilters( value ) {\n\n\t\tif ( ! value ) value = [];\n\n\t\tif ( this._connected === true ) {\n\n\t\t\tthis.disconnect();\n\t\t\tthis.filters = value.slice();\n\t\t\tthis.connect();\n\n\t\t} else {\n\n\t\t\tthis.filters = value.slice();\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tsetDetune( value ) {\n\n\t\tthis.detune = value;\n\n\t\tif ( this.isPlaying === true && this.source.detune !== undefined ) {\n\n\t\t\tthis.source.detune.setTargetAtTime( this.detune, this.context.currentTime, 0.01 );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tgetDetune() {\n\n\t\treturn this.detune;\n\n\t}\n\n\tgetFilter() {\n\n\t\treturn this.getFilters()[ 0 ];\n\n\t}\n\n\tsetFilter( filter ) {\n\n\t\treturn this.setFilters( filter ? [ filter ] : [] );\n\n\t}\n\n\tsetPlaybackRate( value ) {\n\n\t\tif ( this.hasPlaybackControl === false ) {\n\n\t\t\tconsole.warn( 'THREE.Audio: this Audio has no playback control.' );\n\t\t\treturn;\n\n\t\t}\n\n\t\tthis.playbackRate = value;\n\n\t\tif ( this.isPlaying === true ) {\n\n\t\t\tthis.source.playbackRate.setTargetAtTime( this.playbackRate, this.context.currentTime, 0.01 );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tgetPlaybackRate() {\n\n\t\treturn this.playbackRate;\n\n\t}\n\n\tonEnded() {\n\n\t\tthis.isPlaying = false;\n\n\t}\n\n\tgetLoop() {\n\n\t\tif ( this.hasPlaybackControl === false ) {\n\n\t\t\tconsole.warn( 'THREE.Audio: this Audio has no playback control.' );\n\t\t\treturn false;\n\n\t\t}\n\n\t\treturn this.loop;\n\n\t}\n\n\tsetLoop( value ) {\n\n\t\tif ( this.hasPlaybackControl === false ) {\n\n\t\t\tconsole.warn( 'THREE.Audio: this Audio has no playback control.' );\n\t\t\treturn;\n\n\t\t}\n\n\t\tthis.loop = value;\n\n\t\tif ( this.isPlaying === true ) {\n\n\t\t\tthis.source.loop = this.loop;\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tsetLoopStart( value ) {\n\n\t\tthis.loopStart = value;\n\n\t\treturn this;\n\n\t}\n\n\tsetLoopEnd( value ) {\n\n\t\tthis.loopEnd = value;\n\n\t\treturn this;\n\n\t}\n\n\tgetVolume() {\n\n\t\treturn this.gain.gain.value;\n\n\t}\n\n\tsetVolume( value ) {\n\n\t\tthis.gain.gain.setTargetAtTime( value, this.context.currentTime, 0.01 );\n\n\t\treturn this;\n\n\t}\n\n}\n\nexport { Audio };\n\n// Path: src/math/Interpolant.js\n/**\n * Abstract base class of interpolants over parametric samples.\n *\n * The parameter domain is one dimensional, typically the time or a path\n * along a curve defined by the data.\n *\n * The sample values can have any dimensionality and derived classes may\n * apply special interpretations to the data.\n *\n * This class provides the interval seek in a Template Method, deferring\n * the actual interpolation to derived classes.\n *\n * Time complexity is O(1) for linear access crossing at most two points\n * and O(log N) for random access, where N is the number of positions.\n *\n * References:\n *\n * \t\thttp://www.oodesign.com/template-method-pattern.html\n *\n */\n\nclass Interpolant {\n\n\tconstructor( parameterPositions, sampleValues, sampleSize, resultBuffer ) {\n\n\t\tthis.parameterPositions = parameterPositions;\n\t\tthis._cachedIndex = 0;\n\n\t\tthis.resultBuffer = resultBuffer !== undefined ?\n\t\t\tresultBuffer : new sampleValues.constructor( sampleSize );\n\t\tthis.sampleValues = sampleValues;\n\t\tthis.valueSize = sampleSize;\n\n\t\tthis.settings = null;\n\t\tthis.DefaultSettings_ = {};\n\n\t}\n\n\tevaluate( t ) {\n\n\t\tconst pp = this.parameterPositions;\n\t\tlet i1 = this._cachedIndex,\n\t\t\tt1 = pp[ i1 ],\n\t\t\tt0 = pp[ i1 - 1 ];\n\n\t\tvalidate_interval: {\n\n\t\t\tseek: {\n\n\t\t\t\tlet right;\n\n\t\t\t\tlinear_scan: {\n\n\t\t\t\t\t//- See http://jsperf.com/comparison-to-undefined/3\n\t\t\t\t\t//- slower code:\n\t\t\t\t\t//-\n\t\t\t\t\t//- \t\t\t\tif ( t >= t1 || t1 === undefined ) {\n\t\t\t\t\tforward_scan: if ( ! ( t < t1 ) ) {\n\n\t\t\t\t\t\tfor ( let giveUpAt = i1 + 2; ; ) {\n\n\t\t\t\t\t\t\tif ( t1 === undefined ) {\n\n\t\t\t\t\t\t\t\tif ( t < t0 ) break forward_scan;\n\n\t\t\t\t\t\t\t\t// after end\n\n\t\t\t\t\t\t\t\ti1 = pp.length;\n\t\t\t\t\t\t\t\tthis._cachedIndex = i1;\n\t\t\t\t\t\t\t\treturn this.copySampleValue_( i1 - 1 );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif ( i1 === giveUpAt ) break; // this loop\n\n\t\t\t\t\t\t\tt0 = t1;\n\t\t\t\t\t\t\tt1 = pp[ ++ i1 ];\n\n\t\t\t\t\t\t\tif ( t < t1 ) {\n\n\t\t\t\t\t\t\t\t// we have arrived at the sought interval\n\t\t\t\t\t\t\t\tbreak seek;\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// prepare binary search on the right side of the index\n\t\t\t\t\t\tright = pp.length;\n\t\t\t\t\t\tbreak linear_scan;\n\n\t\t\t\t\t}\n\n\t\t\t\t\t//- slower code:\n\t\t\t\t\t//-\t\t\t\t\tif ( t < t0 || t0 === undefined ) {\n\t\t\t\t\tif ( ! ( t >= t0 ) ) {\n\n\t\t\t\t\t\t// looping?\n\n\t\t\t\t\t\tconst t1global = pp[ 1 ];\n\n\t\t\t\t\t\tif ( t < t1global ) {\n\n\t\t\t\t\t\t\ti1 = 2; // + 1, using the scan for the details\n\t\t\t\t\t\t\tt0 = t1global;\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// linear reverse scan\n\n\t\t\t\t\t\tfor ( let giveUpAt = i1 - 2; ; ) {\n\n\t\t\t\t\t\t\tif ( t0 === undefined ) {\n\n\t\t\t\t\t\t\t\t// before start\n\n\t\t\t\t\t\t\t\tthis._cachedIndex = 0;\n\t\t\t\t\t\t\t\treturn this.copySampleValue_( 0 );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\tif ( i1 === giveUpAt ) break; // this loop\n\n\t\t\t\t\t\t\tt1 = t0;\n\t\t\t\t\t\t\tt0 = pp[ -- i1 - 1 ];\n\n\t\t\t\t\t\t\tif ( t >= t0 ) {\n\n\t\t\t\t\t\t\t\t// we have arrived at the sought interval\n\t\t\t\t\t\t\t\tbreak seek;\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// prepare binary search on the left side of the index\n\t\t\t\t\t\tright = i1;\n\t\t\t\t\t\ti1 = 0;\n\t\t\t\t\t\tbreak linear_scan;\n\n\t\t\t\t\t}\n\n\t\t\t\t\t// the interval is valid\n\n\t\t\t\t\tbreak validate_interval;\n\n\t\t\t\t} // linear scan\n\n\t\t\t\t// binary search\n\n\t\t\t\twhile ( i1 < right ) {\n\n\t\t\t\t\tconst mid = ( i1 + right ) >>> 1;\n\n\t\t\t\t\tif ( t < pp[ mid ] ) {\n\n\t\t\t\t\t\tright = mid;\n\n\t\t\t\t\t} else {\n\n\t\t\t\t\t\ti1 = mid + 1;\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t\tt1 = pp[ i1 ];\n\t\t\t\tt0 = pp[ i1 - 1 ];\n\n\t\t\t\t// check boundary cases, again\n\n\t\t\t\tif ( t0 === undefined ) {\n\n\t\t\t\t\tthis._cachedIndex = 0;\n\t\t\t\t\treturn this.copySampleValue_( 0 );\n\n\t\t\t\t}\n\n\t\t\t\tif ( t1 === undefined ) {\n\n\t\t\t\t\ti1 = pp.length;\n\t\t\t\t\tthis._cachedIndex = i1;\n\t\t\t\t\treturn this.copySampleValue_( i1 - 1 );\n\n\t\t\t\t}\n\n\t\t\t} // seek\n\n\t\t\tthis._cachedIndex = i1;\n\n\t\t\tthis.intervalChanged_( i1, t0, t1 );\n\n\t\t} // validate_interval\n\n\t\treturn this.interpolate_( i1, t0, t, t1 );\n\n\t}\n\n\tgetSettings_() {\n\n\t\treturn this.settings || this.DefaultSettings_;\n\n\t}\n\n\tcopySampleValue_( index ) {\n\n\t\t// copies a sample value to the result buffer\n\n\t\tconst result = this.resultBuffer,\n\t\t\tvalues = this.sampleValues,\n\t\t\tstride = this.valueSize,\n\t\t\toffset = index * stride;\n\n\t\tfor ( let i = 0; i !== stride; ++ i ) {\n\n\t\t\tresult[ i ] = values[ offset + i ];\n\n\t\t}\n\n\t\treturn result;\n\n\t}\n\n\t// Template methods for derived classes:\n\n\tinterpolate_( /* i1, t0, t, t1 */ ) {\n\n\t\tthrow new Error( 'call to abstract method' );\n\t\t// implementations shall return this.resultBuffer\n\n\t}\n\n\tintervalChanged_( /* i1, t0, t1 */ ) {\n\n\t\t// empty\n\n\t}\n\n}\n\nexport { Interpolant };\n\n// Path: src/math/interpolants/LinearInterpolant.js\nimport { Interpolant } from '../Interpolant.js';\n\nclass LinearInterpolant extends Interpolant {\n\n\tconstructor( parameterPositions, sampleValues, sampleSize, resultBuffer ) {\n\n\t\tsuper( parameterPositions, sampleValues, sampleSize, resultBuffer );\n\n\t}\n\n\tinterpolate_( i1, t0, t, t1 ) {\n\n\t\tconst result = this.resultBuffer,\n\t\t\tvalues = this.sampleValues,\n\t\t\tstride = this.valueSize,\n\n\t\t\toffset1 = i1 * stride,\n\t\t\toffset0 = offset1 - stride,\n\n\t\t\tweight1 = ( t - t0 ) / ( t1 - t0 ),\n\t\t\tweight0 = 1 - weight1;\n\n\t\tfor ( let i = 0; i !== stride; ++ i ) {\n\n\t\t\tresult[ i ] =\n\t\t\t\t\tvalues[ offset0 + i ] * weight0 +\n\t\t\t\t\tvalues[ offset1 + i ] * weight1;\n\n\t\t}\n\n\t\treturn result;\n\n\t}\n\n}\n\n\nexport { LinearInterpolant };\n\n// Path: src/animation/AnimationUtils.js\nimport { Quaternion } from '../math/Quaternion.js';\nimport { AdditiveAnimationBlendMode } from '../constants.js';\n\n// converts an array to a specific type\n\nfunction convertArray( array, type, forceClone ) {\n\n\tif ( ! array || // let 'undefined' and 'null' pass\n\t\t! forceClone && array.constructor === type ) return array;\n\n\tif ( typeof type.BYTES_PER_ELEMENT === 'number' ) {\n\n\t\treturn new type( array ); // create typed array\n\n\t}\n\n\treturn Array.prototype.slice.call( array ); // create Array\n\n}\n\nfunction isTypedArray( object ) {\n\n\treturn ArrayBuffer.isView( object ) &&\n\t\t! ( object instanceof DataView );\n\n}\n\n// returns an array by which times and values can be sorted\nfunction getKeyframeOrder( times ) {\n\n\tfunction compareTime( i, j ) {\n\n\t\treturn times[ i ] - times[ j ];\n\n\t}\n\n\tconst n = times.length;\n\tconst result = new Array( n );\n\tfor ( let i = 0; i !== n; ++ i ) result[ i ] = i;\n\n\tresult.sort( compareTime );\n\n\treturn result;\n\n}\n\n// uses the array previously returned by 'getKeyframeOrder' to sort data\nfunction sortedArray( values, stride, order ) {\n\n\tconst nValues = values.length;\n\tconst result = new values.constructor( nValues );\n\n\tfor ( let i = 0, dstOffset = 0; dstOffset !== nValues; ++ i ) {\n\n\t\tconst srcOffset = order[ i ] * stride;\n\n\t\tfor ( let j = 0; j !== stride; ++ j ) {\n\n\t\t\tresult[ dstOffset ++ ] = values[ srcOffset + j ];\n\n\t\t}\n\n\t}\n\n\treturn result;\n\n}\n\n// function for parsing AOS keyframe formats\nfunction flattenJSON( jsonKeys, times, values, valuePropertyName ) {\n\n\tlet i = 1, key = jsonKeys[ 0 ];\n\n\twhile ( key !== undefined && key[ valuePropertyName ] === undefined ) {\n\n\t\tkey = jsonKeys[ i ++ ];\n\n\t}\n\n\tif ( key === undefined ) return; // no data\n\n\tlet value = key[ valuePropertyName ];\n\tif ( value === undefined ) return; // no data\n\n\tif ( Array.isArray( value ) ) {\n\n\t\tdo {\n\n\t\t\tvalue = key[ valuePropertyName ];\n\n\t\t\tif ( value !== undefined ) {\n\n\t\t\t\ttimes.push( key.time );\n\t\t\t\tvalues.push.apply( values, value ); // push all elements\n\n\t\t\t}\n\n\t\t\tkey = jsonKeys[ i ++ ];\n\n\t\t} while ( key !== undefined );\n\n\t} else if ( value.toArray !== undefined ) {\n\n\t\t// ...assume THREE.Math-ish\n\n\t\tdo {\n\n\t\t\tvalue = key[ valuePropertyName ];\n\n\t\t\tif ( value !== undefined ) {\n\n\t\t\t\ttimes.push( key.time );\n\t\t\t\tvalue.toArray( values, values.length );\n\n\t\t\t}\n\n\t\t\tkey = jsonKeys[ i ++ ];\n\n\t\t} while ( key !== undefined );\n\n\t} else {\n\n\t\t// otherwise push as-is\n\n\t\tdo {\n\n\t\t\tvalue = key[ valuePropertyName ];\n\n\t\t\tif ( value !== undefined ) {\n\n\t\t\t\ttimes.push( key.time );\n\t\t\t\tvalues.push( value );\n\n\t\t\t}\n\n\t\t\tkey = jsonKeys[ i ++ ];\n\n\t\t} while ( key !== undefined );\n\n\t}\n\n}\n\nfunction subclip( sourceClip, name, startFrame, endFrame, fps = 30 ) {\n\n\tconst clip = sourceClip.clone();\n\n\tclip.name = name;\n\n\tconst tracks = [];\n\n\tfor ( let i = 0; i < clip.tracks.length; ++ i ) {\n\n\t\tconst track = clip.tracks[ i ];\n\t\tconst valueSize = track.getValueSize();\n\n\t\tconst times = [];\n\t\tconst values = [];\n\n\t\tfor ( let j = 0; j < track.times.length; ++ j ) {\n\n\t\t\tconst frame = track.times[ j ] * fps;\n\n\t\t\tif ( frame < startFrame || frame >= endFrame ) continue;\n\n\t\t\ttimes.push( track.times[ j ] );\n\n\t\t\tfor ( let k = 0; k < valueSize; ++ k ) {\n\n\t\t\t\tvalues.push( track.values[ j * valueSize + k ] );\n\n\t\t\t}\n\n\t\t}\n\n\t\tif ( times.length === 0 ) continue;\n\n\t\ttrack.times = convertArray( times, track.times.constructor );\n\t\ttrack.values = convertArray( values, track.values.constructor );\n\n\t\ttracks.push( track );\n\n\t}\n\n\tclip.tracks = tracks;\n\n\t// find minimum .times value across all tracks in the trimmed clip\n\n\tlet minStartTime = Infinity;\n\n\tfor ( let i = 0; i < clip.tracks.length; ++ i ) {\n\n\t\tif ( minStartTime > clip.tracks[ i ].times[ 0 ] ) {\n\n\t\t\tminStartTime = clip.tracks[ i ].times[ 0 ];\n\n\t\t}\n\n\t}\n\n\t// shift all tracks such that clip begins at t=0\n\n\tfor ( let i = 0; i < clip.tracks.length; ++ i ) {\n\n\t\tclip.tracks[ i ].shift( - 1 * minStartTime );\n\n\t}\n\n\tclip.resetDuration();\n\n\treturn clip;\n\n}\n\nfunction makeClipAdditive( targetClip, referenceFrame = 0, referenceClip = targetClip, fps = 30 ) {\n\n\tif ( fps <= 0 ) fps = 30;\n\n\tconst numTracks = referenceClip.tracks.length;\n\tconst referenceTime = referenceFrame / fps;\n\n\t// Make each track's values relative to the values at the reference frame\n\tfor ( let i = 0; i < numTracks; ++ i ) {\n\n\t\tconst referenceTrack = referenceClip.tracks[ i ];\n\t\tconst referenceTrackType = referenceTrack.ValueTypeName;\n\n\t\t// Skip this track if it's non-numeric\n\t\tif ( referenceTrackType === 'bool' || referenceTrackType === 'string' ) continue;\n\n\t\t// Find the track in the target clip whose name and type matches the reference track\n\t\tconst targetTrack = targetClip.tracks.find( function ( track ) {\n\n\t\t\treturn track.name === referenceTrack.name\n\t\t\t\t&& track.ValueTypeName === referenceTrackType;\n\n\t\t} );\n\n\t\tif ( targetTrack === undefined ) continue;\n\n\t\tlet referenceOffset = 0;\n\t\tconst referenceValueSize = referenceTrack.getValueSize();\n\n\t\tif ( referenceTrack.createInterpolant.isInterpolantFactoryMethodGLTFCubicSpline ) {\n\n\t\t\treferenceOffset = referenceValueSize / 3;\n\n\t\t}\n\n\t\tlet targetOffset = 0;\n\t\tconst targetValueSize = targetTrack.getValueSize();\n\n\t\tif ( targetTrack.createInterpolant.isInterpolantFactoryMethodGLTFCubicSpline ) {\n\n\t\t\ttargetOffset = targetValueSize / 3;\n\n\t\t}\n\n\t\tconst lastIndex = referenceTrack.times.length - 1;\n\t\tlet referenceValue;\n\n\t\t// Find the value to subtract out of the track\n\t\tif ( referenceTime <= referenceTrack.times[ 0 ] ) {\n\n\t\t\t// Reference frame is earlier than the first keyframe, so just use the first keyframe\n\t\t\tconst startIndex = referenceOffset;\n\t\t\tconst endIndex = referenceValueSize - referenceOffset;\n\t\t\treferenceValue = referenceTrack.values.slice( startIndex, endIndex );\n\n\t\t} else if ( referenceTime >= referenceTrack.times[ lastIndex ] ) {\n\n\t\t\t// Reference frame is after the last keyframe, so just use the last keyframe\n\t\t\tconst startIndex = lastIndex * referenceValueSize + referenceOffset;\n\t\t\tconst endIndex = startIndex + referenceValueSize - referenceOffset;\n\t\t\treferenceValue = referenceTrack.values.slice( startIndex, endIndex );\n\n\t\t} else {\n\n\t\t\t// Interpolate to the reference value\n\t\t\tconst interpolant = referenceTrack.createInterpolant();\n\t\t\tconst startIndex = referenceOffset;\n\t\t\tconst endIndex = referenceValueSize - referenceOffset;\n\t\t\tinterpolant.evaluate( referenceTime );\n\t\t\treferenceValue = interpolant.resultBuffer.slice( startIndex, endIndex );\n\n\t\t}\n\n\t\t// Conjugate the quaternion\n\t\tif ( referenceTrackType === 'quaternion' ) {\n\n\t\t\tconst referenceQuat = new Quaternion().fromArray( referenceValue ).normalize().conjugate();\n\t\t\treferenceQuat.toArray( referenceValue );\n\n\t\t}\n\n\t\t// Subtract the reference value from all of the track values\n\n\t\tconst numTimes = targetTrack.times.length;\n\t\tfor ( let j = 0; j < numTimes; ++ j ) {\n\n\t\t\tconst valueStart = j * targetValueSize + targetOffset;\n\n\t\t\tif ( referenceTrackType === 'quaternion' ) {\n\n\t\t\t\t// Multiply the conjugate for quaternion track types\n\t\t\t\tQuaternion.multiplyQuaternionsFlat(\n\t\t\t\t\ttargetTrack.values,\n\t\t\t\t\tvalueStart,\n\t\t\t\t\treferenceValue,\n\t\t\t\t\t0,\n\t\t\t\t\ttargetTrack.values,\n\t\t\t\t\tvalueStart\n\t\t\t\t);\n\n\t\t\t} else {\n\n\t\t\t\tconst valueEnd = targetValueSize - targetOffset * 2;\n\n\t\t\t\t// Subtract each value for all other numeric track types\n\t\t\t\tfor ( let k = 0; k < valueEnd; ++ k ) {\n\n\t\t\t\t\ttargetTrack.values[ valueStart + k ] -= referenceValue[ k ];\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\ttargetClip.blendMode = AdditiveAnimationBlendMode;\n\n\treturn targetClip;\n\n}\n\nconst AnimationUtils = {\n\tconvertArray: convertArray,\n\tisTypedArray: isTypedArray,\n\tgetKeyframeOrder: getKeyframeOrder,\n\tsortedArray: sortedArray,\n\tflattenJSON: flattenJSON,\n\tsubclip: subclip,\n\tmakeClipAdditive: makeClipAdditive\n};\n\nexport {\n\tconvertArray,\n\tisTypedArray,\n\tgetKeyframeOrder,\n\tsortedArray,\n\tflattenJSON,\n\tsubclip,\n\tmakeClipAdditive,\n\tAnimationUtils\n};\n\n// Path: src/math/interpolants/CubicInterpolant.js\nimport { ZeroCurvatureEnding, WrapAroundEnding, ZeroSlopeEnding } from '../../constants.js';\nimport { Interpolant } from '../Interpolant.js';\n\n/**\n * Fast and simple cubic spline interpolant.\n *\n * It was derived from a Hermitian construction setting the first derivative\n * at each sample position to the linear slope between neighboring positions\n * over their parameter interval.\n */\n\nclass CubicInterpolant extends Interpolant {\n\n\tconstructor( parameterPositions, sampleValues, sampleSize, resultBuffer ) {\n\n\t\tsuper( parameterPositions, sampleValues, sampleSize, resultBuffer );\n\n\t\tthis._weightPrev = - 0;\n\t\tthis._offsetPrev = - 0;\n\t\tthis._weightNext = - 0;\n\t\tthis._offsetNext = - 0;\n\n\t\tthis.DefaultSettings_ = {\n\n\t\t\tendingStart: ZeroCurvatureEnding,\n\t\t\tendingEnd: ZeroCurvatureEnding\n\n\t\t};\n\n\t}\n\n\tintervalChanged_( i1, t0, t1 ) {\n\n\t\tconst pp = this.parameterPositions;\n\t\tlet iPrev = i1 - 2,\n\t\t\tiNext = i1 + 1,\n\n\t\t\ttPrev = pp[ iPrev ],\n\t\t\ttNext = pp[ iNext ];\n\n\t\tif ( tPrev === undefined ) {\n\n\t\t\tswitch ( this.getSettings_().endingStart ) {\n\n\t\t\t\tcase ZeroSlopeEnding:\n\n\t\t\t\t\t// f'(t0) = 0\n\t\t\t\t\tiPrev = i1;\n\t\t\t\t\ttPrev = 2 * t0 - t1;\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase WrapAroundEnding:\n\n\t\t\t\t\t// use the other end of the curve\n\t\t\t\t\tiPrev = pp.length - 2;\n\t\t\t\t\ttPrev = t0 + pp[ iPrev ] - pp[ iPrev + 1 ];\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tdefault: // ZeroCurvatureEnding\n\n\t\t\t\t\t// f''(t0) = 0 a.k.a. Natural Spline\n\t\t\t\t\tiPrev = i1;\n\t\t\t\t\ttPrev = t1;\n\n\t\t\t}\n\n\t\t}\n\n\t\tif ( tNext === undefined ) {\n\n\t\t\tswitch ( this.getSettings_().endingEnd ) {\n\n\t\t\t\tcase ZeroSlopeEnding:\n\n\t\t\t\t\t// f'(tN) = 0\n\t\t\t\t\tiNext = i1;\n\t\t\t\t\ttNext = 2 * t1 - t0;\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase WrapAroundEnding:\n\n\t\t\t\t\t// use the other end of the curve\n\t\t\t\t\tiNext = 1;\n\t\t\t\t\ttNext = t1 + pp[ 1 ] - pp[ 0 ];\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tdefault: // ZeroCurvatureEnding\n\n\t\t\t\t\t// f''(tN) = 0, a.k.a. Natural Spline\n\t\t\t\t\tiNext = i1 - 1;\n\t\t\t\t\ttNext = t0;\n\n\t\t\t}\n\n\t\t}\n\n\t\tconst halfDt = ( t1 - t0 ) * 0.5,\n\t\t\tstride = this.valueSize;\n\n\t\tthis._weightPrev = halfDt / ( t0 - tPrev );\n\t\tthis._weightNext = halfDt / ( tNext - t1 );\n\t\tthis._offsetPrev = iPrev * stride;\n\t\tthis._offsetNext = iNext * stride;\n\n\t}\n\n\tinterpolate_( i1, t0, t, t1 ) {\n\n\t\tconst result = this.resultBuffer,\n\t\t\tvalues = this.sampleValues,\n\t\t\tstride = this.valueSize,\n\n\t\t\to1 = i1 * stride,\t\to0 = o1 - stride,\n\t\t\toP = this._offsetPrev, \toN = this._offsetNext,\n\t\t\twP = this._weightPrev,\twN = this._weightNext,\n\n\t\t\tp = ( t - t0 ) / ( t1 - t0 ),\n\t\t\tpp = p * p,\n\t\t\tppp = pp * p;\n\n\t\t// evaluate polynomials\n\n\t\tconst sP = - wP * ppp + 2 * wP * pp - wP * p;\n\t\tconst s0 = ( 1 + wP ) * ppp + ( - 1.5 - 2 * wP ) * pp + ( - 0.5 + wP ) * p + 1;\n\t\tconst s1 = ( - 1 - wN ) * ppp + ( 1.5 + wN ) * pp + 0.5 * p;\n\t\tconst sN = wN * ppp - wN * pp;\n\n\t\t// combine data linearly\n\n\t\tfor ( let i = 0; i !== stride; ++ i ) {\n\n\t\t\tresult[ i ] =\n\t\t\t\t\tsP * values[ oP + i ] +\n\t\t\t\t\ts0 * values[ o0 + i ] +\n\t\t\t\t\ts1 * values[ o1 + i ] +\n\t\t\t\t\tsN * values[ oN + i ];\n\n\t\t}\n\n\t\treturn result;\n\n\t}\n\n}\n\nexport { CubicInterpolant };\n\n// Path: src/math/interpolants/DiscreteInterpolant.js\nimport { Interpolant } from '../Interpolant.js';\n\n/**\n *\n * Interpolant that evaluates to the sample value at the position preceding\n * the parameter.\n */\n\nclass DiscreteInterpolant extends Interpolant {\n\n\tconstructor( parameterPositions, sampleValues, sampleSize, resultBuffer ) {\n\n\t\tsuper( parameterPositions, sampleValues, sampleSize, resultBuffer );\n\n\t}\n\n\tinterpolate_( i1 /*, t0, t, t1 */ ) {\n\n\t\treturn this.copySampleValue_( i1 - 1 );\n\n\t}\n\n}\n\n\nexport { DiscreteInterpolant };\n\n// Path: src/animation/KeyframeTrack.js\nimport {\n\tInterpolateLinear,\n\tInterpolateSmooth,\n\tInterpolateDiscrete\n} from '../constants.js';\nimport { CubicInterpolant } from '../math/interpolants/CubicInterpolant.js';\nimport { LinearInterpolant } from '../math/interpolants/LinearInterpolant.js';\nimport { DiscreteInterpolant } from '../math/interpolants/DiscreteInterpolant.js';\nimport * as AnimationUtils from './AnimationUtils.js';\n\nclass KeyframeTrack {\n\n\tconstructor( name, times, values, interpolation ) {\n\n\t\tif ( name === undefined ) throw new Error( 'THREE.KeyframeTrack: track name is undefined' );\n\t\tif ( times === undefined || times.length === 0 ) throw new Error( 'THREE.KeyframeTrack: no keyframes in track named ' + name );\n\n\t\tthis.name = name;\n\n\t\tthis.times = AnimationUtils.convertArray( times, this.TimeBufferType );\n\t\tthis.values = AnimationUtils.convertArray( values, this.ValueBufferType );\n\n\t\tthis.setInterpolation( interpolation || this.DefaultInterpolation );\n\n\t}\n\n\t// Serialization (in static context, because of constructor invocation\n\t// and automatic invocation of .toJSON):\n\n\tstatic toJSON( track ) {\n\n\t\tconst trackType = track.constructor;\n\n\t\tlet json;\n\n\t\t// derived classes can define a static toJSON method\n\t\tif ( trackType.toJSON !== this.toJSON ) {\n\n\t\t\tjson = trackType.toJSON( track );\n\n\t\t} else {\n\n\t\t\t// by default, we assume the data can be serialized as-is\n\t\t\tjson = {\n\n\t\t\t\t'name': track.name,\n\t\t\t\t'times': AnimationUtils.convertArray( track.times, Array ),\n\t\t\t\t'values': AnimationUtils.convertArray( track.values, Array )\n\n\t\t\t};\n\n\t\t\tconst interpolation = track.getInterpolation();\n\n\t\t\tif ( interpolation !== track.DefaultInterpolation ) {\n\n\t\t\t\tjson.interpolation = interpolation;\n\n\t\t\t}\n\n\t\t}\n\n\t\tjson.type = track.ValueTypeName; // mandatory\n\n\t\treturn json;\n\n\t}\n\n\tInterpolantFactoryMethodDiscrete( result ) {\n\n\t\treturn new DiscreteInterpolant( this.times, this.values, this.getValueSize(), result );\n\n\t}\n\n\tInterpolantFactoryMethodLinear( result ) {\n\n\t\treturn new LinearInterpolant( this.times, this.values, this.getValueSize(), result );\n\n\t}\n\n\tInterpolantFactoryMethodSmooth( result ) {\n\n\t\treturn new CubicInterpolant( this.times, this.values, this.getValueSize(), result );\n\n\t}\n\n\tsetInterpolation( interpolation ) {\n\n\t\tlet factoryMethod;\n\n\t\tswitch ( interpolation ) {\n\n\t\t\tcase InterpolateDiscrete:\n\n\t\t\t\tfactoryMethod = this.InterpolantFactoryMethodDiscrete;\n\n\t\t\t\tbreak;\n\n\t\t\tcase InterpolateLinear:\n\n\t\t\t\tfactoryMethod = this.InterpolantFactoryMethodLinear;\n\n\t\t\t\tbreak;\n\n\t\t\tcase InterpolateSmooth:\n\n\t\t\t\tfactoryMethod = this.InterpolantFactoryMethodSmooth;\n\n\t\t\t\tbreak;\n\n\t\t}\n\n\t\tif ( factoryMethod === undefined ) {\n\n\t\t\tconst message = 'unsupported interpolation for ' +\n...\n", "needle_token_start": 9292, "needle_token_end": 9401, "code_context_ntokens": 14489, "output": ["```javascript\nfunction convertArray( array, type, forceClone ) {\n\n\tif ( ! array || // let 'undefined' and 'null' pass\n\t\t! forceClone && array.constructor === type ) return array;\n\n\tif ( typeof type.BYTES_PER_ELEMENT === 'number' ) {\n\n\t\treturn new type( array ); // create typed array\n\n\t}\n\n\treturn Array.prototype.slice.call( array ); // create Array\n\n}\n```"]}
{"repo": "mrdoob/three.js", "name": "linkedList", "language": "typescript", "path": "src/extras/Earcut.js", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a circular doubly linked list from a set of polygon points, ensuring the points are ordered in a specified winding direction.\n2. **Input**: The function receives an array of numerical data representing coordinates, indices indicating the start and end of the segment of the array to be processed, the dimensionality of the points (e.g., 2D or 3D), and a boolean indicating the desired winding order of the points.\n3. **Output**: The output is the last node of the constructed circular doubly linked list, which allows traversal of the list in both directions.\n4. **Procedure**: The function first checks the winding order of the segment of the array and constructs the linked list in the specified order by inserting nodes either from the start to the end or from the end to the start of the segment. It ensures that no consecutive duplicate nodes are present and returns the last node of the list.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/objects/LineSegments.js\nimport { Line } from './Line.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\n\nconst _start = /*@__PURE__*/ new Vector3();\nconst _end = /*@__PURE__*/ new Vector3();\n\n...\n// Path: src/helpers/PolarGridHelper.js\nimport { LineSegments } from '../objects/LineSegments.js';\nimport { LineBasicMaterial } from '../materials/LineBasicMaterial.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Color } from '../math/Color.js';\n\nclass PolarGridHelper extends LineSegments {\n\n\tconstructor( radius = 10, sectors = 16, rings = 8, divisions = 64, color1 = 0x444444, color2 = 0x888888 ) {\n\n\t\tcolor1 = new Color( color1 );\n\t\tcolor2 = new Color( color2 );\n\n\t\tconst vertices = [];\n\t\tconst colors = [];\n\n\t\t// create the sectors\n\n\t\tif ( sectors > 1 ) {\n\n\t\t\tfor ( let i = 0; i < sectors; i ++ ) {\n\n\t\t\t\tconst v = ( i / sectors ) * ( Math.PI * 2 );\n\n\t\t\t\tconst x = Math.sin( v ) * radius;\n\t\t\t\tconst z = Math.cos( v ) * radius;\n\n\t\t\t\tvertices.push( 0, 0, 0 );\n\t\t\t\tvertices.push( x, 0, z );\n\n\t\t\t\tconst color = ( i & 1 ) ? color1 : color2;\n\n\t\t\t\tcolors.push( color.r, color.g, color.b );\n\t\t\t\tcolors.push( color.r, color.g, color.b );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// create the rings\n\n\t\tfor ( let i = 0; i < rings; i ++ ) {\n\n\t\t\tconst color = ( i & 1 ) ? color1 : color2;\n\n\t\t\tconst r = radius - ( radius / rings * i );\n\n\t\t\tfor ( let j = 0; j < divisions; j ++ ) {\n\n\t\t\t\t// first vertex\n\n\t\t\t\tlet v = ( j / divisions ) * ( Math.PI * 2 );\n\n\t\t\t\tlet x = Math.sin( v ) * r;\n\t\t\t\tlet z = Math.cos( v ) * r;\n\n\t\t\t\tvertices.push( x, 0, z );\n\t\t\t\tcolors.push( color.r, color.g, color.b );\n\n\t\t\t\t// second vertex\n\n\t\t\t\tv = ( ( j + 1 ) / divisions ) * ( Math.PI * 2 );\n\n\t\t\t\tx = Math.sin( v ) * r;\n\t\t\t\tz = Math.cos( v ) * r;\n\n\t\t\t\tvertices.push( x, 0, z );\n\t\t\t\tcolors.push( color.r, color.g, color.b );\n\n\t\t\t}\n\n\t\t}\n\n\t\tconst geometry = new BufferGeometry();\n\t\tgeometry.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tgeometry.setAttribute( 'color', new Float32BufferAttribute( colors, 3 ) );\n\n\t\tconst material = new LineBasicMaterial( { vertexColors: true, toneMapped: false } );\n\n\t\tsuper( geometry, material );\n\n\t\tthis.type = 'PolarGridHelper';\n\n\t}\n\n\tdispose() {\n\n\t\tthis.geometry.dispose();\n\t\tthis.material.dispose();\n\n\t}\n\n}\n\n\nexport { PolarGridHelper };\n\n// Path: src/geometries/CylinderGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Vector2 } from '../math/Vector2.js';\n\nclass CylinderGeometry extends BufferGeometry {\n\n\tconstructor( radiusTop = 1, radiusBottom = 1, height = 1, radialSegments = 32, heightSegments = 1, openEnded = false, thetaStart = 0, thetaLength = Math.PI * 2 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'CylinderGeometry';\n\n\t\tthis.parameters = {\n\t\t\tradiusTop: radiusTop,\n\t\t\tradiusBottom: radiusBottom,\n\t\t\theight: height,\n\t\t\tradialSegments: radialSegments,\n\t\t\theightSegments: heightSegments,\n\t\t\topenEnded: openEnded,\n\t\t\tthetaStart: thetaStart,\n\t\t\tthetaLength: thetaLength\n\t\t};\n\n\t\tconst scope = this;\n\n\t\tradialSegments = Math.floor( radialSegments );\n\t\theightSegments = Math.floor( heightSegments );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\t// helper variables\n\n\t\tlet index = 0;\n\t\tconst indexArray = [];\n\t\tconst halfHeight = height / 2;\n\t\tlet groupStart = 0;\n\n\t\t// generate geometry\n\n\t\tgenerateTorso();\n\n\t\tif ( openEnded === false ) {\n\n\t\t\tif ( radiusTop > 0 ) generateCap( true );\n\t\t\tif ( radiusBottom > 0 ) generateCap( false );\n\n\t\t}\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t\tfunction generateTorso() {\n\n\t\t\tconst normal = new Vector3();\n\t\t\tconst vertex = new Vector3();\n\n\t\t\tlet groupCount = 0;\n\n\t\t\t// this will be used to calculate the normal\n\t\t\tconst slope = ( radiusBottom - radiusTop ) / height;\n\n\t\t\t// generate vertices, normals and uvs\n\n\t\t\tfor ( let y = 0; y <= heightSegments; y ++ ) {\n\n\t\t\t\tconst indexRow = [];\n\n\t\t\t\tconst v = y / heightSegments;\n\n\t\t\t\t// calculate the radius of the current row\n\n\t\t\t\tconst radius = v * ( radiusBottom - radiusTop ) + radiusTop;\n\n\t\t\t\tfor ( let x = 0; x <= radialSegments; x ++ ) {\n\n\t\t\t\t\tconst u = x / radialSegments;\n\n\t\t\t\t\tconst theta = u * thetaLength + thetaStart;\n\n\t\t\t\t\tconst sinTheta = Math.sin( theta );\n\t\t\t\t\tconst cosTheta = Math.cos( theta );\n\n\t\t\t\t\t// vertex\n\n\t\t\t\t\tvertex.x = radius * sinTheta;\n\t\t\t\t\tvertex.y = - v * height + halfHeight;\n\t\t\t\t\tvertex.z = radius * cosTheta;\n\t\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t\t// normal\n\n\t\t\t\t\tnormal.set( sinTheta, slope, cosTheta ).normalize();\n\t\t\t\t\tnormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t\t// uv\n\n\t\t\t\t\tuvs.push( u, 1 - v );\n\n\t\t\t\t\t// save index of vertex in respective row\n\n\t\t\t\t\tindexRow.push( index ++ );\n\n\t\t\t\t}\n\n\t\t\t\t// now save vertices of the row in our index array\n\n\t\t\t\tindexArray.push( indexRow );\n\n\t\t\t}\n\n\t\t\t// generate indices\n\n\t\t\tfor ( let x = 0; x < radialSegments; x ++ ) {\n\n\t\t\t\tfor ( let y = 0; y < heightSegments; y ++ ) {\n\n\t\t\t\t\t// we use the index array to access the correct indices\n\n\t\t\t\t\tconst a = indexArray[ y ][ x ];\n\t\t\t\t\tconst b = indexArray[ y + 1 ][ x ];\n\t\t\t\t\tconst c = indexArray[ y + 1 ][ x + 1 ];\n\t\t\t\t\tconst d = indexArray[ y ][ x + 1 ];\n\n\t\t\t\t\t// faces\n\n\t\t\t\t\tindices.push( a, b, d );\n\t\t\t\t\tindices.push( b, c, d );\n\n\t\t\t\t\t// update group counter\n\n\t\t\t\t\tgroupCount += 6;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// add a group to the geometry. this will ensure multi material support\n\n\t\t\tscope.addGroup( groupStart, groupCount, 0 );\n\n\t\t\t// calculate new start value for groups\n\n\t\t\tgroupStart += groupCount;\n\n\t\t}\n\n\t\tfunction generateCap( top ) {\n\n\t\t\t// save the index of the first center vertex\n\t\t\tconst centerIndexStart = index;\n\n\t\t\tconst uv = new Vector2();\n\t\t\tconst vertex = new Vector3();\n\n\t\t\tlet groupCount = 0;\n\n\t\t\tconst radius = ( top === true ) ? radiusTop : radiusBottom;\n\t\t\tconst sign = ( top === true ) ? 1 : - 1;\n\n\t\t\t// first we generate the center vertex data of the cap.\n\t\t\t// because the geometry needs one set of uvs per face,\n\t\t\t// we must generate a center vertex per face/segment\n\n\t\t\tfor ( let x = 1; x <= radialSegments; x ++ ) {\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertices.push( 0, halfHeight * sign, 0 );\n\n\t\t\t\t// normal\n\n\t\t\t\tnormals.push( 0, sign, 0 );\n\n\t\t\t\t// uv\n\n\t\t\t\tuvs.push( 0.5, 0.5 );\n\n\t\t\t\t// increase index\n\n\t\t\t\tindex ++;\n\n\t\t\t}\n\n\t\t\t// save the index of the last center vertex\n\t\t\tconst centerIndexEnd = index;\n\n\t\t\t// now we generate the surrounding vertices, normals and uvs\n\n\t\t\tfor ( let x = 0; x <= radialSegments; x ++ ) {\n\n\t\t\t\tconst u = x / radialSegments;\n\t\t\t\tconst theta = u * thetaLength + thetaStart;\n\n\t\t\t\tconst cosTheta = Math.cos( theta );\n\t\t\t\tconst sinTheta = Math.sin( theta );\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertex.x = radius * sinTheta;\n\t\t\t\tvertex.y = halfHeight * sign;\n\t\t\t\tvertex.z = radius * cosTheta;\n\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t// normal\n\n\t\t\t\tnormals.push( 0, sign, 0 );\n\n\t\t\t\t// uv\n\n\t\t\t\tuv.x = ( cosTheta * 0.5 ) + 0.5;\n\t\t\t\tuv.y = ( sinTheta * 0.5 * sign ) + 0.5;\n\t\t\t\tuvs.push( uv.x, uv.y );\n\n\t\t\t\t// increase index\n\n\t\t\t\tindex ++;\n\n\t\t\t}\n\n\t\t\t// generate indices\n\n\t\t\tfor ( let x = 0; x < radialSegments; x ++ ) {\n\n\t\t\t\tconst c = centerIndexStart + x;\n\t\t\t\tconst i = centerIndexEnd + x;\n\n\t\t\t\tif ( top === true ) {\n\n\t\t\t\t\t// face top\n\n\t\t\t\t\tindices.push( i, i + 1, c );\n\n\t\t\t\t} else {\n\n\t\t\t\t\t// face bottom\n\n\t\t\t\t\tindices.push( i + 1, i, c );\n\n\t\t\t\t}\n\n\t\t\t\tgroupCount += 3;\n\n\t\t\t}\n\n\t\t\t// add a group to the geometry. this will ensure multi material support\n\n\t\t\tscope.addGroup( groupStart, groupCount, top === true ? 1 : 2 );\n\n\t\t\t// calculate new start value for groups\n\n\t\t\tgroupStart += groupCount;\n\n\t\t}\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new CylinderGeometry( data.radiusTop, data.radiusBottom, data.height, data.radialSegments, data.heightSegments, data.openEnded, data.thetaStart, data.thetaLength );\n\n\t}\n\n}\n\n\nexport { CylinderGeometry };\n\n// Path: src/geometries/ConeGeometry.js\nimport { CylinderGeometry } from './CylinderGeometry.js';\n\nclass ConeGeometry extends CylinderGeometry {\n\n\tconstructor( radius = 1, height = 1, radialSegments = 32, heightSegments = 1, openEnded = false, thetaStart = 0, thetaLength = Math.PI * 2 ) {\n\n\t\tsuper( 0, radius, height, radialSegments, heightSegments, openEnded, thetaStart, thetaLength );\n\n\t\tthis.type = 'ConeGeometry';\n\n\t\tthis.parameters = {\n\t\t\tradius: radius,\n\t\t\theight: height,\n\t\t\tradialSegments: radialSegments,\n\t\t\theightSegments: heightSegments,\n\t\t\topenEnded: openEnded,\n\t\t\tthetaStart: thetaStart,\n\t\t\tthetaLength: thetaLength\n\t\t};\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new ConeGeometry( data.radius, data.height, data.radialSegments, data.heightSegments, data.openEnded, data.thetaStart, data.thetaLength );\n\n\t}\n\n}\n\nexport { ConeGeometry };\n\n// Path: src/geometries/LatheGeometry.js\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Vector2 } from '../math/Vector2.js';\nimport * as MathUtils from '../math/MathUtils.js';\n\nclass LatheGeometry extends BufferGeometry {\n\n\tconstructor( points = [ new Vector2( 0, - 0.5 ), new Vector2( 0.5, 0 ), new Vector2( 0, 0.5 ) ], segments = 12, phiStart = 0, phiLength = Math.PI * 2 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'LatheGeometry';\n\n\t\tthis.parameters = {\n\t\t\tpoints: points,\n\t\t\tsegments: segments,\n\t\t\tphiStart: phiStart,\n\t\t\tphiLength: phiLength\n\t\t};\n\n\t\tsegments = Math.floor( segments );\n\n\t\t// clamp phiLength so it's in range of [ 0, 2PI ]\n\n\t\tphiLength = MathUtils.clamp( phiLength, 0, Math.PI * 2 );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst uvs = [];\n\t\tconst initNormals = [];\n\t\tconst normals = [];\n\n\t\t// helper variables\n\n\t\tconst inverseSegments = 1.0 / segments;\n\t\tconst vertex = new Vector3();\n\t\tconst uv = new Vector2();\n\t\tconst normal = new Vector3();\n\t\tconst curNormal = new Vector3();\n\t\tconst prevNormal = new Vector3();\n\t\tlet dx = 0;\n\t\tlet dy = 0;\n\n\t\t// pre-compute normals for initial \"meridian\"\n\n\t\tfor ( let j = 0; j <= ( points.length - 1 ); j ++ ) {\n\n\t\t\tswitch ( j ) {\n\n\t\t\t\tcase 0:\t\t\t\t// special handling for 1st vertex on path\n\n\t\t\t\t\tdx = points[ j + 1 ].x - points[ j ].x;\n\t\t\t\t\tdy = points[ j + 1 ].y - points[ j ].y;\n\n\t\t\t\t\tnormal.x = dy * 1.0;\n\t\t\t\t\tnormal.y = - dx;\n\t\t\t\t\tnormal.z = dy * 0.0;\n\n\t\t\t\t\tprevNormal.copy( normal );\n\n\t\t\t\t\tnormal.normalize();\n\n\t\t\t\t\tinitNormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase ( points.length - 1 ):\t// special handling for last Vertex on path\n\n\t\t\t\t\tinitNormals.push( prevNormal.x, prevNormal.y, prevNormal.z );\n\n\t\t\t\t\tbreak;\n\n\t\t\t\tdefault:\t\t\t// default handling for all vertices in between\n\n\t\t\t\t\tdx = points[ j + 1 ].x - points[ j ].x;\n\t\t\t\t\tdy = points[ j + 1 ].y - points[ j ].y;\n\n\t\t\t\t\tnormal.x = dy * 1.0;\n\t\t\t\t\tnormal.y = - dx;\n\t\t\t\t\tnormal.z = dy * 0.0;\n\n\t\t\t\t\tcurNormal.copy( normal );\n\n\t\t\t\t\tnormal.x += prevNormal.x;\n\t\t\t\t\tnormal.y += prevNormal.y;\n\t\t\t\t\tnormal.z += prevNormal.z;\n\n\t\t\t\t\tnormal.normalize();\n\n\t\t\t\t\tinitNormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t\tprevNormal.copy( curNormal );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// generate vertices, uvs and normals\n\n\t\tfor ( let i = 0; i <= segments; i ++ ) {\n\n\t\t\tconst phi = phiStart + i * inverseSegments * phiLength;\n\n\t\t\tconst sin = Math.sin( phi );\n\t\t\tconst cos = Math.cos( phi );\n\n\t\t\tfor ( let j = 0; j <= ( points.length - 1 ); j ++ ) {\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertex.x = points[ j ].x * sin;\n\t\t\t\tvertex.y = points[ j ].y;\n\t\t\t\tvertex.z = points[ j ].x * cos;\n\n\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t// uv\n\n\t\t\t\tuv.x = i / segments;\n\t\t\t\tuv.y = j / ( points.length - 1 );\n\n\t\t\t\tuvs.push( uv.x, uv.y );\n\n\t\t\t\t// normal\n\n\t\t\t\tconst x = initNormals[ 3 * j + 0 ] * sin;\n\t\t\t\tconst y = initNormals[ 3 * j + 1 ];\n\t\t\t\tconst z = initNormals[ 3 * j + 0 ] * cos;\n\n\t\t\t\tnormals.push( x, y, z );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// indices\n\n\t\tfor ( let i = 0; i < segments; i ++ ) {\n\n\t\t\tfor ( let j = 0; j < ( points.length - 1 ); j ++ ) {\n\n\t\t\t\tconst base = j + i * points.length;\n\n\t\t\t\tconst a = base;\n\t\t\t\tconst b = base + points.length;\n\t\t\t\tconst c = base + points.length + 1;\n\t\t\t\tconst d = base + 1;\n\n\t\t\t\t// faces\n\n\t\t\t\tindices.push( a, b, d );\n\t\t\t\tindices.push( c, d, b );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new LatheGeometry( data.points, data.segments, data.phiStart, data.phiLength );\n\n\t}\n\n}\n\n\nexport { LatheGeometry };\n\n// Path: src/geometries/BoxGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\n\nclass BoxGeometry extends BufferGeometry {\n\n\tconstructor( width = 1, height = 1, depth = 1, widthSegments = 1, heightSegments = 1, depthSegments = 1 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'BoxGeometry';\n\n\t\tthis.parameters = {\n\t\t\twidth: width,\n\t\t\theight: height,\n\t\t\tdepth: depth,\n\t\t\twidthSegments: widthSegments,\n\t\t\theightSegments: heightSegments,\n\t\t\tdepthSegments: depthSegments\n\t\t};\n\n\t\tconst scope = this;\n\n\t\t// segments\n\n\t\twidthSegments = Math.floor( widthSegments );\n\t\theightSegments = Math.floor( heightSegments );\n\t\tdepthSegments = Math.floor( depthSegments );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\t// helper variables\n\n\t\tlet numberOfVertices = 0;\n\t\tlet groupStart = 0;\n\n\t\t// build each side of the box geometry\n\n\t\tbuildPlane( 'z', 'y', 'x', - 1, - 1, depth, height, width, depthSegments, heightSegments, 0 ); // px\n\t\tbuildPlane( 'z', 'y', 'x', 1, - 1, depth, height, - width, depthSegments, heightSegments, 1 ); // nx\n\t\tbuildPlane( 'x', 'z', 'y', 1, 1, width, depth, height, widthSegments, depthSegments, 2 ); // py\n\t\tbuildPlane( 'x', 'z', 'y', 1, - 1, width, depth, - height, widthSegments, depthSegments, 3 ); // ny\n\t\tbuildPlane( 'x', 'y', 'z', 1, - 1, width, height, depth, widthSegments, heightSegments, 4 ); // pz\n\t\tbuildPlane( 'x', 'y', 'z', - 1, - 1, width, height, - depth, widthSegments, heightSegments, 5 ); // nz\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t\tfunction buildPlane( u, v, w, udir, vdir, width, height, depth, gridX, gridY, materialIndex ) {\n\n\t\t\tconst segmentWidth = width / gridX;\n\t\t\tconst segmentHeight = height / gridY;\n\n\t\t\tconst widthHalf = width / 2;\n\t\t\tconst heightHalf = height / 2;\n\t\t\tconst depthHalf = depth / 2;\n\n\t\t\tconst gridX1 = gridX + 1;\n\t\t\tconst gridY1 = gridY + 1;\n\n\t\t\tlet vertexCounter = 0;\n\t\t\tlet groupCount = 0;\n\n\t\t\tconst vector = new Vector3();\n\n\t\t\t// generate vertices, normals and uvs\n\n\t\t\tfor ( let iy = 0; iy < gridY1; iy ++ ) {\n\n\t\t\t\tconst y = iy * segmentHeight - heightHalf;\n\n\t\t\t\tfor ( let ix = 0; ix < gridX1; ix ++ ) {\n\n\t\t\t\t\tconst x = ix * segmentWidth - widthHalf;\n\n\t\t\t\t\t// set values to correct vector component\n\n\t\t\t\t\tvector[ u ] = x * udir;\n\t\t\t\t\tvector[ v ] = y * vdir;\n\t\t\t\t\tvector[ w ] = depthHalf;\n\n\t\t\t\t\t// now apply vector to vertex buffer\n\n\t\t\t\t\tvertices.push( vector.x, vector.y, vector.z );\n\n\t\t\t\t\t// set values to correct vector component\n\n\t\t\t\t\tvector[ u ] = 0;\n\t\t\t\t\tvector[ v ] = 0;\n\t\t\t\t\tvector[ w ] = depth > 0 ? 1 : - 1;\n\n\t\t\t\t\t// now apply vector to normal buffer\n\n\t\t\t\t\tnormals.push( vector.x, vector.y, vector.z );\n\n\t\t\t\t\t// uvs\n\n\t\t\t\t\tuvs.push( ix / gridX );\n\t\t\t\t\tuvs.push( 1 - ( iy / gridY ) );\n\n\t\t\t\t\t// counters\n\n\t\t\t\t\tvertexCounter += 1;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// indices\n\n\t\t\t// 1. you need three indices to draw a single face\n\t\t\t// 2. a single segment consists of two faces\n\t\t\t// 3. so we need to generate six (2*3) indices per segment\n\n\t\t\tfor ( let iy = 0; iy < gridY; iy ++ ) {\n\n\t\t\t\tfor ( let ix = 0; ix < gridX; ix ++ ) {\n\n\t\t\t\t\tconst a = numberOfVertices + ix + gridX1 * iy;\n\t\t\t\t\tconst b = numberOfVertices + ix + gridX1 * ( iy + 1 );\n\t\t\t\t\tconst c = numberOfVertices + ( ix + 1 ) + gridX1 * ( iy + 1 );\n\t\t\t\t\tconst d = numberOfVertices + ( ix + 1 ) + gridX1 * iy;\n\n\t\t\t\t\t// faces\n\n\t\t\t\t\tindices.push( a, b, d );\n\t\t\t\t\tindices.push( b, c, d );\n\n\t\t\t\t\t// increase counter\n\n\t\t\t\t\tgroupCount += 6;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// add a group to the geometry. this will ensure multi material support\n\n\t\t\tscope.addGroup( groupStart, groupCount, materialIndex );\n\n\t\t\t// calculate new start value for groups\n\n\t\t\tgroupStart += groupCount;\n\n\t\t\t// update total number of vertices\n\n\t\t\tnumberOfVertices += vertexCounter;\n\n\t\t}\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new BoxGeometry( data.width, data.height, data.depth, data.widthSegments, data.heightSegments, data.depthSegments );\n\n\t}\n\n}\n\nexport { BoxGeometry };\n\n// Path: src/geometries/PlaneGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\n\nclass PlaneGeometry extends BufferGeometry {\n\n\tconstructor( width = 1, height = 1, widthSegments = 1, heightSegments = 1 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'PlaneGeometry';\n\n\t\tthis.parameters = {\n\t\t\twidth: width,\n\t\t\theight: height,\n\t\t\twidthSegments: widthSegments,\n\t\t\theightSegments: heightSegments\n\t\t};\n\n\t\tconst width_half = width / 2;\n\t\tconst height_half = height / 2;\n\n\t\tconst gridX = Math.floor( widthSegments );\n\t\tconst gridY = Math.floor( heightSegments );\n\n\t\tconst gridX1 = gridX + 1;\n\t\tconst gridY1 = gridY + 1;\n\n\t\tconst segment_width = width / gridX;\n\t\tconst segment_height = height / gridY;\n\n\t\t//\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\tfor ( let iy = 0; iy < gridY1; iy ++ ) {\n\n\t\t\tconst y = iy * segment_height - height_half;\n\n\t\t\tfor ( let ix = 0; ix < gridX1; ix ++ ) {\n\n\t\t\t\tconst x = ix * segment_width - width_half;\n\n\t\t\t\tvertices.push( x, - y, 0 );\n\n\t\t\t\tnormals.push( 0, 0, 1 );\n\n\t\t\t\tuvs.push( ix / gridX );\n\t\t\t\tuvs.push( 1 - ( iy / gridY ) );\n\n\t\t\t}\n\n\t\t}\n\n\t\tfor ( let iy = 0; iy < gridY; iy ++ ) {\n\n\t\t\tfor ( let ix = 0; ix < gridX; ix ++ ) {\n\n\t\t\t\tconst a = ix + gridX1 * iy;\n\t\t\t\tconst b = ix + gridX1 * ( iy + 1 );\n\t\t\t\tconst c = ( ix + 1 ) + gridX1 * ( iy + 1 );\n\t\t\t\tconst d = ( ix + 1 ) + gridX1 * iy;\n\n\t\t\t\tindices.push( a, b, d );\n\t\t\t\tindices.push( b, c, d );\n\n\t\t\t}\n\n\t\t}\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new PlaneGeometry( data.width, data.height, data.widthSegments, data.heightSegments );\n\n\t}\n\n}\n\nexport { PlaneGeometry };\n\n// Path: src/geometries/WireframeGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\n\nclass WireframeGeometry extends BufferGeometry {\n\n\tconstructor( geometry = null ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'WireframeGeometry';\n\n\t\tthis.parameters = {\n\t\t\tgeometry: geometry\n\t\t};\n\n\t\tif ( geometry !== null ) {\n\n\t\t\t// buffer\n\n\t\t\tconst vertices = [];\n\t\t\tconst edges = new Set();\n\n\t\t\t// helper variables\n\n\t\t\tconst start = new Vector3();\n\t\t\tconst end = new Vector3();\n\n\t\t\tif ( geometry.index !== null ) {\n\n\t\t\t\t// indexed BufferGeometry\n\n\t\t\t\tconst position = geometry.attributes.position;\n\t\t\t\tconst indices = geometry.index;\n\t\t\t\tlet groups = geometry.groups;\n\n\t\t\t\tif ( groups.length === 0 ) {\n\n\t\t\t\t\tgroups = [ { start: 0, count: indices.count, materialIndex: 0 } ];\n\n\t\t\t\t}\n\n\t\t\t\t// create a data structure that contains all edges without duplicates\n\n\t\t\t\tfor ( let o = 0, ol = groups.length; o < ol; ++ o ) {\n\n\t\t\t\t\tconst group = groups[ o ];\n\n\t\t\t\t\tconst groupStart = group.start;\n\t\t\t\t\tconst groupCount = group.count;\n\n\t\t\t\t\tfor ( let i = groupStart, l = ( groupStart + groupCount ); i < l; i += 3 ) {\n\n\t\t\t\t\t\tfor ( let j = 0; j < 3; j ++ ) {\n\n\t\t\t\t\t\t\tconst index1 = indices.getX( i + j );\n\t\t\t\t\t\t\tconst index2 = indices.getX( i + ( j + 1 ) % 3 );\n\n\t\t\t\t\t\t\tstart.fromBufferAttribute( position, index1 );\n\t\t\t\t\t\t\tend.fromBufferAttribute( position, index2 );\n\n\t\t\t\t\t\t\tif ( isUniqueEdge( start, end, edges ) === true ) {\n\n\t\t\t\t\t\t\t\tvertices.push( start.x, start.y, start.z );\n\t\t\t\t\t\t\t\tvertices.push( end.x, end.y, end.z );\n\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t} else {\n\n\t\t\t\t// non-indexed BufferGeometry\n\n\t\t\t\tconst position = geometry.attributes.position;\n\n\t\t\t\tfor ( let i = 0, l = ( position.count / 3 ); i < l; i ++ ) {\n\n\t\t\t\t\tfor ( let j = 0; j < 3; j ++ ) {\n\n\t\t\t\t\t\t// three edges per triangle, an edge is represented as (index1, index2)\n\t\t\t\t\t\t// e.g. the first triangle has the following edges: (0,1),(1,2),(2,0)\n\n\t\t\t\t\t\tconst index1 = 3 * i + j;\n\t\t\t\t\t\tconst index2 = 3 * i + ( ( j + 1 ) % 3 );\n\n\t\t\t\t\t\tstart.fromBufferAttribute( position, index1 );\n\t\t\t\t\t\tend.fromBufferAttribute( position, index2 );\n\n\t\t\t\t\t\tif ( isUniqueEdge( start, end, edges ) === true ) {\n\n\t\t\t\t\t\t\tvertices.push( start.x, start.y, start.z );\n\t\t\t\t\t\t\tvertices.push( end.x, end.y, end.z );\n\n\t\t\t\t\t\t}\n\n\t\t\t\t\t}\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\t// build geometry\n\n\t\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\n\t\t}\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n}\n\nfunction isUniqueEdge( start, end, edges ) {\n\n\tconst hash1 = `${start.x},${start.y},${start.z}-${end.x},${end.y},${end.z}`;\n\tconst hash2 = `${end.x},${end.y},${end.z}-${start.x},${start.y},${start.z}`; // coincident edge\n\n\tif ( edges.has( hash1 ) === true || edges.has( hash2 ) === true ) {\n\n\t\treturn false;\n\n\t} else {\n\n\t\tedges.add( hash1 );\n\t\tedges.add( hash2 );\n\t\treturn true;\n\n\t}\n\n}\n\n\nexport { WireframeGeometry };\n\n// Path: src/geometries/TorusGeometry.js\nimport { BufferGeometry } from '../core/BufferGeometry.js';\nimport { Float32BufferAttribute } from '../core/BufferAttribute.js';\nimport { Vector3 } from '../math/Vector3.js';\n\nclass TorusGeometry extends BufferGeometry {\n\n\tconstructor( radius = 1, tube = 0.4, radialSegments = 12, tubularSegments = 48, arc = Math.PI * 2 ) {\n\n\t\tsuper();\n\n\t\tthis.type = 'TorusGeometry';\n\n\t\tthis.parameters = {\n\t\t\tradius: radius,\n\t\t\ttube: tube,\n\t\t\tradialSegments: radialSegments,\n\t\t\ttubularSegments: tubularSegments,\n\t\t\tarc: arc\n\t\t};\n\n\t\tradialSegments = Math.floor( radialSegments );\n\t\ttubularSegments = Math.floor( tubularSegments );\n\n\t\t// buffers\n\n\t\tconst indices = [];\n\t\tconst vertices = [];\n\t\tconst normals = [];\n\t\tconst uvs = [];\n\n\t\t// helper variables\n\n\t\tconst center = new Vector3();\n\t\tconst vertex = new Vector3();\n\t\tconst normal = new Vector3();\n\n\t\t// generate vertices, normals and uvs\n\n\t\tfor ( let j = 0; j <= radialSegments; j ++ ) {\n\n\t\t\tfor ( let i = 0; i <= tubularSegments; i ++ ) {\n\n\t\t\t\tconst u = i / tubularSegments * arc;\n\t\t\t\tconst v = j / radialSegments * Math.PI * 2;\n\n\t\t\t\t// vertex\n\n\t\t\t\tvertex.x = ( radius + tube * Math.cos( v ) ) * Math.cos( u );\n\t\t\t\tvertex.y = ( radius + tube * Math.cos( v ) ) * Math.sin( u );\n\t\t\t\tvertex.z = tube * Math.sin( v );\n\n\t\t\t\tvertices.push( vertex.x, vertex.y, vertex.z );\n\n\t\t\t\t// normal\n\n\t\t\t\tcenter.x = radius * Math.cos( u );\n\t\t\t\tcenter.y = radius * Math.sin( u );\n\t\t\t\tnormal.subVectors( vertex, center ).normalize();\n\n\t\t\t\tnormals.push( normal.x, normal.y, normal.z );\n\n\t\t\t\t// uv\n\n\t\t\t\tuvs.push( i / tubularSegments );\n\t\t\t\tuvs.push( j / radialSegments );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// generate indices\n\n\t\tfor ( let j = 1; j <= radialSegments; j ++ ) {\n\n\t\t\tfor ( let i = 1; i <= tubularSegments; i ++ ) {\n\n\t\t\t\t// indices\n\n\t\t\t\tconst a = ( tubularSegments + 1 ) * j + i - 1;\n\t\t\t\tconst b = ( tubularSegments + 1 ) * ( j - 1 ) + i - 1;\n\t\t\t\tconst c = ( tubularSegments + 1 ) * ( j - 1 ) + i;\n\t\t\t\tconst d = ( tubularSegments + 1 ) * j + i;\n\n\t\t\t\t// faces\n\n\t\t\t\tindices.push( a, b, d );\n\t\t\t\tindices.push( b, c, d );\n\n\t\t\t}\n\n\t\t}\n\n\t\t// build geometry\n\n\t\tthis.setIndex( indices );\n\t\tthis.setAttribute( 'position', new Float32BufferAttribute( vertices, 3 ) );\n\t\tthis.setAttribute( 'normal', new Float32BufferAttribute( normals, 3 ) );\n\t\tthis.setAttribute( 'uv', new Float32BufferAttribute( uvs, 2 ) );\n\n\t}\n\n\tcopy( source ) {\n\n\t\tsuper.copy( source );\n\n\t\tthis.parameters = Object.assign( {}, source.parameters );\n\n\t\treturn this;\n\n\t}\n\n\tstatic fromJSON( data ) {\n\n\t\treturn new TorusGeometry( data.radius, data.tube, data.radialSegments, data.tubularSegments, data.arc );\n\n\t}\n\n}\n\nexport { TorusGeometry };\n\n// Path: src/extras/Earcut.js\n/**\n * Port from https://github.com/mapbox/earcut (v2.2.4)\n */\n\nconst Earcut = {\n\n\ttriangulate: function ( data, holeIndices, dim = 2 ) {\n\n\t\tconst hasHoles = holeIndices && holeIndices.length;\n\t\tconst outerLen = hasHoles ? holeIndices[ 0 ] * dim : data.length;\n\t\tlet outerNode = linkedList( data, 0, outerLen, dim, true );\n\t\tconst triangles = [];\n\n\t\tif ( ! outerNode || outerNode.next === outerNode.prev ) return triangles;\n\n\t\tlet minX, minY, maxX, maxY, x, y, invSize;\n\n\t\tif ( hasHoles ) outerNode = eliminateHoles( data, holeIndices, outerNode, dim );\n\n\t\t// if the shape is not too simple, we'll use z-order curve hash later; calculate polygon bbox\n\t\tif ( data.length > 80 * dim ) {\n\n\t\t\tminX = maxX = data[ 0 ];\n\t\t\tminY = maxY = data[ 1 ];\n\n\t\t\tfor ( let i = dim; i < outerLen; i += dim ) {\n\n\t\t\t\tx = data[ i ];\n\t\t\t\ty = data[ i + 1 ];\n\t\t\t\tif ( x < minX ) minX = x;\n\t\t\t\tif ( y < minY ) minY = y;\n\t\t\t\tif ( x > maxX ) maxX = x;\n\t\t\t\tif ( y > maxY ) maxY = y;\n\n\t\t\t}\n\n\t\t\t// minX, minY and invSize are later used to transform coords into integers for z-order calculation\n\t\t\tinvSize = Math.max( maxX - minX, maxY - minY );\n\t\t\tinvSize = invSize !== 0 ? 32767 / invSize : 0;\n\n\t\t}\n\n\t\tearcutLinked( outerNode, triangles, dim, minX, minY, invSize, 0 );\n\n\t\treturn triangles;\n\n\t}\n\n};\n\n// create a circular doubly linked list from polygon points in the specified winding order\n\nfunction linkedList( data, start, end, dim, clockwise ) {\n\n\tlet i, last;\n\n\tif ( clockwise === ( signedArea( data, start, end, dim ) > 0 ) ) {\n\n\t\tfor ( i = start; i < end; i += dim ) last = insertNode( i, data[ i ], data[ i + 1 ], last );\n\n\t} else {\n\n\t\tfor ( i = end - dim; i >= start; i -= dim ) last = insertNode( i, data[ i ], data[ i + 1 ], last );\n\n\t}\n\n\tif ( last && equals( last, last.next ) ) {\n\n\t\tremoveNode( last );\n\t\tlast = last.next;\n\n\t}\n\n\treturn last;\n\n}\n\n// eliminate colinear or duplicate points\nfunction filterPoints( start, end ) {\n\n\tif ( ! start ) return start;\n\tif ( ! end ) end = start;\n\n\tlet p = start,\n\t\tagain;\n\tdo {\n\n\t\tagain = false;\n\n\t\tif ( ! p.steiner && ( equals( p, p.next ) || area( p.prev, p, p.next ) === 0 ) ) {\n\n\t\t\tremoveNode( p );\n\t\t\tp = end = p.prev;\n\t\t\tif ( p === p.next ) break;\n\t\t\tagain = true;\n\n\t\t} else {\n\n\t\t\tp = p.next;\n\n\t\t}\n\n\t} while ( again || p !== end );\n\n\treturn end;\n\n}\n\n// main ear slicing loop which triangulates a polygon (given as a linked list)\nfunction earcutLinked( ear, triangles, dim, minX, minY, invSize, pass ) {\n\n\tif ( ! ear ) return;\n\n\t// interlink polygon nodes in z-order\n\tif ( ! pass && invSize ) indexCurve( ear, minX, minY, invSize );\n\n\tlet stop = ear,\n\t\tprev, next;\n\n\t// iterate through ears, slicing them one by one\n\twhile ( ear.prev !== ear.next ) {\n\n\t\tprev = ear.prev;\n\t\tnext = ear.next;\n\n\t\tif ( invSize ? isEarHashed( ear, minX, minY, invSize ) : isEar( ear ) ) {\n\n\t\t\t// cut off the triangle\n\t\t\ttriangles.push( prev.i / dim | 0 );\n\t\t\ttriangles.push( ear.i / dim | 0 );\n\t\t\ttriangles.push( next.i / dim | 0 );\n\n\t\t\tremoveNode( ear );\n\n\t\t\t// skipping the next vertex leads to less sliver triangles\n\t\t\tear = next.next;\n\t\t\tstop = next.next;\n\n\t\t\tcontinue;\n\n\t\t}\n\n\t\tear = next;\n\n\t\t// if we looped through the whole remaining polygon and can't find any more ears\n\t\tif ( ear === stop ) {\n\n\t\t\t// try filtering points and slicing again\n\t\t\tif ( ! pass ) {\n\n\t\t\t\tearcutLinked( filterPoints( ear ), triangles, dim, minX, minY, invSize, 1 );\n\n\t\t\t\t// if this didn't work, try curing all small self-intersections locally\n\n\t\t\t} else if ( pass === 1 ) {\n\n\t\t\t\tear = cureLocalIntersections( filterPoints( ear ), triangles, dim );\n\t\t\t\tearcutLinked( ear, triangles, dim, minX, minY, invSize, 2 );\n\n\t\t\t\t// as a last resort, try splitting the remaining polygon into two\n\n\t\t\t} else if ( pass === 2 ) {\n\n\t\t\t\tsplitEarcut( ear, triangles, dim, minX, minY, invSize );\n\n\t\t\t}\n\n\t\t\tbreak;\n\n\t\t}\n\n\t}\n\n}\n\n// check whether a polygon node forms a valid ear with adjacent nodes\nfunction isEar( ear ) {\n\n\tconst a = ear.prev,\n\t\tb = ear,\n\t\tc = ear.next;\n\n\tif ( area( a, b, c ) >= 0 ) return false; // reflex, can't be an ear\n\n\t// now make sure we don't have other points inside the potential ear\n\tconst ax = a.x, bx = b.x, cx = c.x, ay = a.y, by = b.y, cy = c.y;\n\n\t// triangle bbox; min & max are calculated like this for speed\n\tconst x0 = ax < bx ? ( ax < cx ? ax : cx ) : ( bx < cx ? bx : cx ),\n\t\ty0 = ay < by ? ( ay < cy ? ay : cy ) : ( by < cy ? by : cy ),\n\t\tx1 = ax > bx ? ( ax > cx ? ax : cx ) : ( bx > cx ? bx : cx ),\n\t\ty1 = ay > by ? ( ay > cy ? ay : cy ) : ( by > cy ? by : cy );\n\n\tlet p = c.next;\n\twhile ( p !== a ) {\n\n\t\tif ( p.x >= x0 && p.x <= x1 && p.y >= y0 && p.y <= y1 &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, p.x, p.y ) &&\n\t\t\tarea( p.prev, p, p.next ) >= 0 ) return false;\n\t\tp = p.next;\n\n\t}\n\n\treturn true;\n\n}\n\nfunction isEarHashed( ear, minX, minY, invSize ) {\n\n\tconst a = ear.prev,\n\t\tb = ear,\n\t\tc = ear.next;\n\n\tif ( area( a, b, c ) >= 0 ) return false; // reflex, can't be an ear\n\n\tconst ax = a.x, bx = b.x, cx = c.x, ay = a.y, by = b.y, cy = c.y;\n\n\t// triangle bbox; min & max are calculated like this for speed\n\tconst x0 = ax < bx ? ( ax < cx ? ax : cx ) : ( bx < cx ? bx : cx ),\n\t\ty0 = ay < by ? ( ay < cy ? ay : cy ) : ( by < cy ? by : cy ),\n\t\tx1 = ax > bx ? ( ax > cx ? ax : cx ) : ( bx > cx ? bx : cx ),\n\t\ty1 = ay > by ? ( ay > cy ? ay : cy ) : ( by > cy ? by : cy );\n\n\t// z-order range for the current triangle bbox;\n\tconst minZ = zOrder( x0, y0, minX, minY, invSize ),\n\t\tmaxZ = zOrder( x1, y1, minX, minY, invSize );\n\n\tlet p = ear.prevZ,\n\t\tn = ear.nextZ;\n\n\t// look for points inside the triangle in both directions\n\twhile ( p && p.z >= minZ && n && n.z <= maxZ ) {\n\n\t\tif ( p.x >= x0 && p.x <= x1 && p.y >= y0 && p.y <= y1 && p !== a && p !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, p.x, p.y ) && area( p.prev, p, p.next ) >= 0 ) return false;\n\t\tp = p.prevZ;\n\n\t\tif ( n.x >= x0 && n.x <= x1 && n.y >= y0 && n.y <= y1 && n !== a && n !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, n.x, n.y ) && area( n.prev, n, n.next ) >= 0 ) return false;\n\t\tn = n.nextZ;\n\n\t}\n\n\t// look for remaining points in decreasing z-order\n\twhile ( p && p.z >= minZ ) {\n\n\t\tif ( p.x >= x0 && p.x <= x1 && p.y >= y0 && p.y <= y1 && p !== a && p !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, p.x, p.y ) && area( p.prev, p, p.next ) >= 0 ) return false;\n\t\tp = p.prevZ;\n\n\t}\n\n\t// look for remaining points in increasing z-order\n\twhile ( n && n.z <= maxZ ) {\n\n\t\tif ( n.x >= x0 && n.x <= x1 && n.y >= y0 && n.y <= y1 && n !== a && n !== c &&\n\t\t\tpointInTriangle( ax, ay, bx, by, cx, cy, n.x, n.y ) && area( n.prev, n, n.next ) >= 0 ) return false;\n\t\tn = n.nextZ;\n\n\t}\n\n\treturn true;\n\n}\n\n// go through all polygon nodes and cure small local self-intersections\nfunction cureLocalIntersections( start, triangles, dim ) {\n\n\tlet p = start;\n\tdo {\n\n\t\tconst a = p.prev,\n\t\t\tb = p.next.next;\n\n\t\tif ( ! equals( a, b ) && intersects( a, p, p.next, b ) && locallyInside( a, b ) && locallyInside( b, a ) ) {\n\n\t\t\ttriangles.push( a.i / dim | 0 );\n\t\t\ttriangles.push( p.i / dim | 0 );\n\t\t\ttriangles.push( b.i / dim | 0 );\n\n\t\t\t// remove two nodes involved\n\t\t\tremoveNode( p );\n\t\t\tremoveNode( p.next );\n\n\t\t\tp = start = b;\n\n\t\t}\n\n\t\tp = p.next;\n\n\t} while ( p !== start );\n\n\treturn filterPoints( p );\n\n}\n\n// try splitting polygon into two and triangulate them independently\nfunction splitEarcut( start, triangles, dim, minX, minY, invSize ) {\n\n\t// look for a valid diagonal that divides the polygon into two\n\tlet a = start;\n\tdo {\n\n\t\tlet b = a.next.next;\n\t\twhile ( b !== a.prev ) {\n\n\t\t\tif ( a.i !== b.i && isValidDiagonal( a, b ) ) {\n\n\t\t\t\t// split the polygon in two by the diagonal\n\t\t\t\tlet c = splitPolygon( a, b );\n\n\t\t\t\t// filter colinear points around the cuts\n\t\t\t\ta = filterPoints( a, a.next );\n\t\t\t\tc = filterPoints( c, c.next );\n\n\t\t\t\t// run earcut on each half\n\t\t\t\tearcutLinked( a, triangles, dim, minX, minY, invSize, 0 );\n\t\t\t\tearcutLinked( c, triangles, dim, minX, minY, invSize, 0 );\n\t\t\t\treturn;\n\n\t\t\t}\n\n\t\t\tb = b.next;\n\n\t\t}\n\n\t\ta = a.next;\n\n\t} while ( a !== start );\n\n}\n\n// link every hole into the outer loop, producing a single-ring polygon without holes\nfunction eliminateHoles( data, holeIndices, outerNode, dim ) {\n\n\tconst queue = [];\n\tlet i, len, start, end, list;\n\n\tfor ( i = 0, len = holeIndices.length; i < len; i ++ ) {\n\n\t\tstart = holeIndices[ i ] * dim;\n\t\tend = i < len - 1 ? holeIndices[ i + 1 ] * dim : data.length;\n\t\tlist = linkedList( data, start, end, dim, false );\n\t\tif ( list === list.next ) list.steiner = true;\n\t\tqueue.push( getLeftmost( list ) );\n\n\t}\n\n\tqueue.sort( compareX );\n\n\t// process holes from left to right\n\tfor ( i = 0; i < queue.length; i ++ ) {\n\n\t\touterNode = eliminateHole( queue[ i ], outerNode );\n\n\t}\n\n\treturn outerNode;\n\n}\n\nfunction compareX( a, b ) {\n\n\treturn a.x - b.x;\n\n}\n\n// find a bridge between vertices that connects hole with an outer ring and link it\nfunction eliminateHole( hole, outerNode ) {\n\n\tconst bridge = findHoleBridge( hole, outerNode );\n\tif ( ! bridge ) {\n\n\t\treturn outerNode;\n\n\t}\n\n\tconst bridgeReverse = splitPolygon( bridge, hole );\n\n\t// filter collinear points around the cuts\n\tfilterPoints( bridgeReverse, bridgeReverse.next );\n\treturn filterPoints( bridge, bridge.next );\n\n}\n\n// David Eberly's algorithm for finding a bridge between hole and outer polygon\nfunction findHoleBridge( hole, outerNode ) {\n\n\tlet p = outerNode,\n\t\tqx = - Infinity,\n\t\tm;\n\n\tconst hx = hole.x, hy = hole.y;\n\n\t// find a segment intersected by a ray from the hole's leftmost point to the left;\n\t// segment's endpoint with lesser x will be potential connection point\n\tdo {\n\n\t\tif ( hy <= p.y && hy >= p.next.y && p.next.y !== p.y ) {\n\n\t\t\tconst x = p.x + ( hy - p.y ) * ( p.next.x - p.x ) / ( p.next.y - p.y );\n\t\t\tif ( x <= hx && x > qx ) {\n\n\t\t\t\tqx = x;\n\t\t\t\tm = p.x < p.next.x ? p : p.next;\n\t\t\t\tif ( x === hx ) return m; // hole touches outer segment; pick leftmost endpoint\n\n\t\t\t}\n\n\t\t}\n\n\t\tp = p.next;\n\n\t} while ( p !== outerNode );\n\n\tif ( ! m ) return null;\n\n\t// look for points inside the triangle of hole point, segment intersection and endpoint;\n\t// if there are no points found, we have a valid connection;\n\t// otherwise choose the point of the minimum angle with the ray as connection point\n\n\tconst stop = m,\n\t\tmx = m.x,\n\t\tmy = m.y;\n\tlet tanMin = Infinity, tan;\n\n\tp = m;\n\n\tdo {\n\n\t\tif ( hx >= p.x && p.x >= mx && hx !== p.x &&\n\t\t\t\tpointInTriangle( hy < my ? hx : qx, hy, mx, my, hy < my ? qx : hx, hy, p.x, p.y ) ) {\n\n\t\t\ttan = Math.abs( hy - p.y ) / ( hx - p.x ); // tangential\n\n\t\t\tif ( locallyInside( p, hole ) && ( tan < tanMin || ( tan === tanMin && ( p.x > m.x || ( p.x === m.x && sectorContainsSector( m, p ) ) ) ) ) ) {\n\n\t\t\t\tm = p;\n\t\t\t\ttanMin = tan;\n\n\t\t\t}\n\n\t\t}\n\n\t\tp = p.next;\n\n...\n", "needle_token_start": 11040, "needle_token_end": 11226, "code_context_ntokens": 14882, "output": ["```javascript\n// create a circular doubly linked list from polygon points in the specified winding order\nfunction linkedList( data, start, end, dim, clockwise ) {\n\n\tlet i, last;\n\n\tif ( clockwise === ( signedArea( data, start, end, dim ) > 0 ) ) {\n\n\t\tfor ( i = start; i < end; i += dim ) last = insertNode( i, data[ i ], data[ i + 1 ], last );\n\n\t} else {\n\n\t\tfor ( i = end - dim; i >= start; i -= dim ) last = insertNode( i, data[ i ], data[ i + 1 ], last );\n\n\t}\n\n\tif ( last && equals( last, last.next ) ) {\n\n\t\tremoveNode( last );\n\t\tlast = last.next;\n\n\t}\n\n\treturn last;\n\n}\n```"]}
{"repo": "mrdoob/three.js", "name": "WebGLExtensions", "language": "typescript", "path": "src/renderers/webgl/WebGLExtensions.js", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to manage and access various graphical extensions available in a WebGL context, enhancing the capabilities of WebGL by allowing developers to query and utilize additional features not available in the standard API.\n2. **Input**: The primary input is a WebGL rendering context provided upon instantiation.\n3. **Output**: The output is an object that provides methods to check the availability of specific extensions, initialize certain extensions, and retrieve them if available.\n4. **Procedure**: The function initializes an internal storage for caching extensions. It provides methods to retrieve an extension by name, checking multiple vendor prefixes if necessary. It also includes initialization for a predefined set of extensions and logs a warning if a requested extension is not supported.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/renderers/webgl/WebGLMaterials.js\nimport { BackSide } from '../../constants.js';\nimport { getUnlitUniformColorSpace } from '../shaders/UniformsUtils.js';\nimport { Euler } from '../../math/Euler.js';\nimport { Matrix4 } from '../../math/Matrix4.js';\n\nconst _e1 = /*@__PURE__*/ new Euler();\nconst _m1 = /*@__PURE__*/ new Matrix4();\n\nfunction WebGLMaterials( renderer, properties ) {\n\n\tfunction refreshTransformUniform( map, uniform ) {\n\n\t\tif ( map.matrixAutoUpdate === true ) {\n\n\t\t\tmap.updateMatrix();\n\n\t\t}\n\n\t\tuniform.value.copy( map.matrix );\n\n\t}\n\n\tfunction refreshFogUniforms( uniforms, fog ) {\n\n\t\tfog.color.getRGB( uniforms.fogColor.value, getUnlitUniformColorSpace( renderer ) );\n\n\t\tif ( fog.isFog ) {\n\n\t\t\tuniforms.fogNear.value = fog.near;\n\t\t\tuniforms.fogFar.value = fog.far;\n\n\t\t} else if ( fog.isFogExp2 ) {\n\n\t\t\tuniforms.fogDensity.value = fog.density;\n\n\t\t}\n\n\t}\n\n\tfunction refreshMaterialUniforms( uniforms, material, pixelRatio, height, transmissionRenderTarget ) {\n\n\t\tif ( material.isMeshBasicMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\n\t\t} else if ( material.isMeshLambertMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\n\t\t} else if ( material.isMeshToonMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\t\t\trefreshUniformsToon( uniforms, material );\n\n\t\t} else if ( material.isMeshPhongMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\t\t\trefreshUniformsPhong( uniforms, material );\n\n\t\t} else if ( material.isMeshStandardMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\t\t\trefreshUniformsStandard( uniforms, material );\n\n\t\t\tif ( material.isMeshPhysicalMaterial ) {\n\n\t\t\t\trefreshUniformsPhysical( uniforms, material, transmissionRenderTarget );\n\n\t\t\t}\n\n\t\t} else if ( material.isMeshMatcapMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\t\t\trefreshUniformsMatcap( uniforms, material );\n\n\t\t} else if ( material.isMeshDepthMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\n\t\t} else if ( material.isMeshDistanceMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\t\t\trefreshUniformsDistance( uniforms, material );\n\n\t\t} else if ( material.isMeshNormalMaterial ) {\n\n\t\t\trefreshUniformsCommon( uniforms, material );\n\n\t\t} else if ( material.isLineBasicMaterial ) {\n\n\t\t\trefreshUniformsLine( uniforms, material );\n\n\t\t\tif ( material.isLineDashedMaterial ) {\n\n\t\t\t\trefreshUniformsDash( uniforms, material );\n\n\t\t\t}\n\n\t\t} else if ( material.isPointsMaterial ) {\n\n\t\t\trefreshUniformsPoints( uniforms, material, pixelRatio, height );\n\n\t\t} else if ( material.isSpriteMaterial ) {\n\n\t\t\trefreshUniformsSprites( uniforms, material );\n\n\t\t} else if ( material.isShadowMaterial ) {\n\n\t\t\tuniforms.color.value.copy( material.color );\n\t\t\tuniforms.opacity.value = material.opacity;\n\n\t\t} else if ( material.isShaderMaterial ) {\n\n\t\t\tmaterial.uniformsNeedUpdate = false; // #15581\n\n\t\t}\n\n\t}\n\n\tfunction refreshUniformsCommon( uniforms, material ) {\n\n\t\tuniforms.opacity.value = material.opacity;\n\n\t\tif ( material.color ) {\n\n\t\t\tuniforms.diffuse.value.copy( material.color );\n\n\t\t}\n\n\t\tif ( material.emissive ) {\n\n\t\t\tuniforms.emissive.value.copy( material.emissive ).multiplyScalar( material.emissiveIntensity );\n\n\t\t}\n\n\t\tif ( material.map ) {\n\n\t\t\tuniforms.map.value = material.map;\n\n\t\t\trefreshTransformUniform( material.map, uniforms.mapTransform );\n\n\t\t}\n\n\t\tif ( material.alphaMap ) {\n\n\t\t\tuniforms.alphaMap.value = material.alphaMap;\n\n\t\t\trefreshTransformUniform( material.alphaMap, uniforms.alphaMapTransform );\n\n\t\t}\n\n\t\tif ( material.bumpMap ) {\n\n\t\t\tuniforms.bumpMap.value = material.bumpMap;\n\n\t\t\trefreshTransformUniform( material.bumpMap, uniforms.bumpMapTransform );\n\n\t\t\tuniforms.bumpScale.value = material.bumpScale;\n\n\t\t\tif ( material.side === BackSide ) {\n\n\t\t\t\tuniforms.bumpScale.value *= - 1;\n\n\t\t\t}\n\n\t\t}\n\n\t\tif ( material.normalMap ) {\n\n\t\t\tuniforms.normalMap.value = material.normalMap;\n\n\t\t\trefreshTransformUniform( material.normalMap, uniforms.normalMapTransform );\n\n\t\t\tuniforms.normalScale.value.copy( material.normalScale );\n\n\t\t\tif ( material.side === BackSide ) {\n\n\t\t\t\tuniforms.normalScale.value.negate();\n\n\t\t\t}\n\n\t\t}\n\n\t\tif ( material.displacementMap ) {\n\n\t\t\tuniforms.displacementMap.value = material.displacementMap;\n\n\t\t\trefreshTransformUniform( material.displacementMap, uniforms.displacementMapTransform );\n\n\t\t\tuniforms.displacementScale.value = material.displacementScale;\n\t\t\tuniforms.displacementBias.value = material.displacementBias;\n\n\t\t}\n\n\t\tif ( material.emissiveMap ) {\n\n\t\t\tuniforms.emissiveMap.value = material.emissiveMap;\n\n\t\t\trefreshTransformUniform( material.emissiveMap, uniforms.emissiveMapTransform );\n\n\t\t}\n\n\t\tif ( material.specularMap ) {\n\n\t\t\tuniforms.specularMap.value = material.specularMap;\n\n\t\t\trefreshTransformUniform( material.specularMap, uniforms.specularMapTransform );\n\n\t\t}\n\n\t\tif ( material.alphaTest > 0 ) {\n\n\t\t\tuniforms.alphaTest.value = material.alphaTest;\n\n\t\t}\n\n\t\tconst materialProperties = properties.get( material );\n\n\t\tconst envMap = materialProperties.envMap;\n\t\tconst envMapRotation = materialProperties.envMapRotation;\n\n\t\tif ( envMap ) {\n\n\t\t\tuniforms.envMap.value = envMap;\n\n\t\t\t_e1.copy( envMapRotation );\n\n\t\t\t// accommodate left-handed frame\n\t\t\t_e1.x *= - 1; _e1.y *= - 1; _e1.z *= - 1;\n\n\t\t\tif ( envMap.isCubeTexture && envMap.isRenderTargetTexture === false ) {\n\n\t\t\t\t// environment maps which are not cube render targets or PMREMs follow a different convention\n\t\t\t\t_e1.y *= - 1;\n\t\t\t\t_e1.z *= - 1;\n\n\t\t\t}\n\n\t\t\tuniforms.envMapRotation.value.setFromMatrix4( _m1.makeRotationFromEuler( _e1 ) );\n\n\t\t\tuniforms.flipEnvMap.value = ( envMap.isCubeTexture && envMap.isRenderTargetTexture === false ) ? - 1 : 1;\n\n\t\t\tuniforms.reflectivity.value = material.reflectivity;\n\t\t\tuniforms.ior.value = material.ior;\n\t\t\tuniforms.refractionRatio.value = material.refractionRatio;\n\n\t\t}\n\n\t\tif ( material.lightMap ) {\n\n\t\t\tuniforms.lightMap.value = material.lightMap;\n\n\t\t\t// artist-friendly light intensity scaling factor\n\t\t\tconst scaleFactor = ( renderer._useLegacyLights === true ) ? Math.PI : 1;\n\n\t\t\tuniforms.lightMapIntensity.value = material.lightMapIntensity * scaleFactor;\n\n\t\t\trefreshTransformUniform( material.lightMap, uniforms.lightMapTransform );\n\n\t\t}\n\n\t\tif ( material.aoMap ) {\n\n\t\t\tuniforms.aoMap.value = material.aoMap;\n\t\t\tuniforms.aoMapIntensity.value = material.aoMapIntensity;\n\n\t\t\trefreshTransformUniform( material.aoMap, uniforms.aoMapTransform );\n\n\t\t}\n\n\t}\n\n\tfunction refreshUniformsLine( uniforms, material ) {\n\n\t\tuniforms.diffuse.value.copy( material.color );\n\t\tuniforms.opacity.value = material.opacity;\n\n\t\tif ( material.map ) {\n\n\t\t\tuniforms.map.value = material.map;\n\n\t\t\trefreshTransformUniform( material.map, uniforms.mapTransform );\n\n\t\t}\n\n\t}\n\n\tfunction refreshUniformsDash( uniforms, material ) {\n\n\t\tuniforms.dashSize.value = material.dashSize;\n\t\tuniforms.totalSize.value = material.dashSize + material.gapSize;\n\t\tuniforms.scale.value = material.scale;\n\n\t}\n\n\tfunction refreshUniformsPoints( uniforms, material, pixelRatio, height ) {\n\n\t\tuniforms.diffuse.value.copy( material.color );\n\t\tuniforms.opacity.value = material.opacity;\n\t\tuniforms.size.value = material.size * pixelRatio;\n\t\tuniforms.scale.value = height * 0.5;\n\n\t\tif ( material.map ) {\n\n\t\t\tuniforms.map.value = material.map;\n\n\t\t\trefreshTransformUniform( material.map, uniforms.uvTransform );\n\n\t\t}\n\n\t\tif ( material.alphaMap ) {\n\n\t\t\tuniforms.alphaMap.value = material.alphaMap;\n\n\t\t\trefreshTransformUniform( material.alphaMap, uniforms.alphaMapTransform );\n\n\t\t}\n\n\t\tif ( material.alphaTest > 0 ) {\n\n\t\t\tuniforms.alphaTest.value = material.alphaTest;\n\n\t\t}\n\n\t}\n\n\tfunction refreshUniformsSprites( uniforms, material ) {\n\n\t\tuniforms.diffuse.value.copy( material.color );\n\t\tuniforms.opacity.value = material.opacity;\n\t\tuniforms.rotation.value = material.rotation;\n\n\t\tif ( material.map ) {\n\n\t\t\tuniforms.map.value = material.map;\n\n\t\t\trefreshTransformUniform( material.map, uniforms.mapTransform );\n\n\t\t}\n\n\t\tif ( material.alphaMap ) {\n\n\t\t\tuniforms.alphaMap.value = material.alphaMap;\n\n\t\t\trefreshTransformUniform( material.alphaMap, uniforms.alphaMapTransform );\n\n...\n// Path: src/textures/DataArrayTexture.js\nimport { Texture } from './Texture.js';\nimport { ClampToEdgeWrapping, NearestFilter } from '../constants.js';\n\nclass DataArrayTexture extends Texture {\n\n\tconstructor( data = null, width = 1, height = 1, depth = 1 ) {\n\n\t\tsuper( null );\n\n\t\tthis.isDataArrayTexture = true;\n\n\t\tthis.image = { data, width, height, depth };\n\n\t\tthis.magFilter = NearestFilter;\n\t\tthis.minFilter = NearestFilter;\n\n\t\tthis.wrapR = ClampToEdgeWrapping;\n\n\t\tthis.generateMipmaps = false;\n\t\tthis.flipY = false;\n\t\tthis.unpackAlignment = 1;\n\n\t}\n\n}\n\nexport { DataArrayTexture };\n\n// Path: src/renderers/webgl/WebGLUniforms.js\n/**\n * Uniforms of a program.\n * Those form a tree structure with a special top-level container for the root,\n * which you get by calling 'new WebGLUniforms( gl, program )'.\n *\n *\n * Properties of inner nodes including the top-level container:\n *\n * .seq - array of nested uniforms\n * .map - nested uniforms by name\n *\n *\n * Methods of all nodes except the top-level container:\n *\n * .setValue( gl, value, [textures] )\n *\n * \t\tuploads a uniform value(s)\n *  \tthe 'textures' parameter is needed for sampler uniforms\n *\n *\n * Static methods of the top-level container (textures factorizations):\n *\n * .upload( gl, seq, values, textures )\n *\n * \t\tsets uniforms in 'seq' to 'values[id].value'\n *\n * .seqWithValue( seq, values ) : filteredSeq\n *\n * \t\tfilters 'seq' entries with corresponding entry in values\n *\n *\n * Methods of the top-level container (textures factorizations):\n *\n * .setValue( gl, name, value, textures )\n *\n * \t\tsets uniform with  name 'name' to 'value'\n *\n * .setOptional( gl, obj, prop )\n *\n * \t\tlike .set for an optional property of the object\n *\n */\n\nimport { CubeTexture } from '../../textures/CubeTexture.js';\nimport { Texture } from '../../textures/Texture.js';\nimport { DataArrayTexture } from '../../textures/DataArrayTexture.js';\nimport { Data3DTexture } from '../../textures/Data3DTexture.js';\nimport { DepthTexture } from '../../textures/DepthTexture.js';\nimport { LessEqualCompare } from '../../constants.js';\n\nconst emptyTexture = /*@__PURE__*/ new Texture();\n\nconst emptyShadowTexture = /*@__PURE__*/ new DepthTexture( 1, 1 );\nemptyShadowTexture.compareFunction = LessEqualCompare;\n\nconst emptyArrayTexture = /*@__PURE__*/ new DataArrayTexture();\nconst empty3dTexture = /*@__PURE__*/ new Data3DTexture();\nconst emptyCubeTexture = /*@__PURE__*/ new CubeTexture();\n\n// --- Utilities ---\n\n// Array Caches (provide typed arrays for temporary by size)\n\nconst arrayCacheF32 = [];\nconst arrayCacheI32 = [];\n\n// Float32Array caches used for uploading Matrix uniforms\n\nconst mat4array = new Float32Array( 16 );\nconst mat3array = new Float32Array( 9 );\nconst mat2array = new Float32Array( 4 );\n\n// Flattening for arrays of vectors and matrices\n\nfunction flatten( array, nBlocks, blockSize ) {\n\n\tconst firstElem = array[ 0 ];\n\n\tif ( firstElem <= 0 || firstElem > 0 ) return array;\n\t// unoptimized: ! isNaN( firstElem )\n\t// see http://jacksondunstan.com/articles/983\n\n\tconst n = nBlocks * blockSize;\n\tlet r = arrayCacheF32[ n ];\n\n\tif ( r === undefined ) {\n\n\t\tr = new Float32Array( n );\n\t\tarrayCacheF32[ n ] = r;\n\n\t}\n\n\tif ( nBlocks !== 0 ) {\n\n\t\tfirstElem.toArray( r, 0 );\n\n\t\tfor ( let i = 1, offset = 0; i !== nBlocks; ++ i ) {\n\n\t\t\toffset += blockSize;\n\t\t\tarray[ i ].toArray( r, offset );\n\n\t\t}\n\n\t}\n\n\treturn r;\n\n}\n\nfunction arraysEqual( a, b ) {\n\n\tif ( a.length !== b.length ) return false;\n\n\tfor ( let i = 0, l = a.length; i < l; i ++ ) {\n\n\t\tif ( a[ i ] !== b[ i ] ) return false;\n\n\t}\n\n\treturn true;\n\n}\n\nfunction copyArray( a, b ) {\n\n\tfor ( let i = 0, l = b.length; i < l; i ++ ) {\n\n\t\ta[ i ] = b[ i ];\n\n\t}\n\n}\n\n// Texture unit allocation\n\nfunction allocTexUnits( textures, n ) {\n\n\tlet r = arrayCacheI32[ n ];\n\n\tif ( r === undefined ) {\n\n\t\tr = new Int32Array( n );\n\t\tarrayCacheI32[ n ] = r;\n\n\t}\n\n\tfor ( let i = 0; i !== n; ++ i ) {\n\n\t\tr[ i ] = textures.allocateTextureUnit();\n\n\t}\n\n\treturn r;\n\n}\n\n// --- Setters ---\n\n// Note: Defining these methods externally, because they come in a bunch\n// and this way their names minify.\n\n// Single scalar\n\nfunction setValueV1f( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( cache[ 0 ] === v ) return;\n\n\tgl.uniform1f( this.addr, v );\n\n\tcache[ 0 ] = v;\n\n}\n\n// Single float vector (from flat array or THREE.VectorN)\n\nfunction setValueV2f( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y ) {\n\n\t\t\tgl.uniform2f( this.addr, v.x, v.y );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform2fv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\nfunction setValueV3f( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y || cache[ 2 ] !== v.z ) {\n\n\t\t\tgl.uniform3f( this.addr, v.x, v.y, v.z );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\t\t\tcache[ 2 ] = v.z;\n\n\t\t}\n\n\t} else if ( v.r !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.r || cache[ 1 ] !== v.g || cache[ 2 ] !== v.b ) {\n\n\t\t\tgl.uniform3f( this.addr, v.r, v.g, v.b );\n\n\t\t\tcache[ 0 ] = v.r;\n\t\t\tcache[ 1 ] = v.g;\n\t\t\tcache[ 2 ] = v.b;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform3fv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\nfunction setValueV4f( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y || cache[ 2 ] !== v.z || cache[ 3 ] !== v.w ) {\n\n\t\t\tgl.uniform4f( this.addr, v.x, v.y, v.z, v.w );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\t\t\tcache[ 2 ] = v.z;\n\t\t\tcache[ 3 ] = v.w;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform4fv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\n// Single matrix (from flat array or THREE.MatrixN)\n\nfunction setValueM2( gl, v ) {\n\n\tconst cache = this.cache;\n\tconst elements = v.elements;\n\n\tif ( elements === undefined ) {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniformMatrix2fv( this.addr, false, v );\n\n\t\tcopyArray( cache, v );\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, elements ) ) return;\n\n\t\tmat2array.set( elements );\n\n\t\tgl.uniformMatrix2fv( this.addr, false, mat2array );\n\n\t\tcopyArray( cache, elements );\n\n\t}\n\n}\n\nfunction setValueM3( gl, v ) {\n\n\tconst cache = this.cache;\n\tconst elements = v.elements;\n\n\tif ( elements === undefined ) {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniformMatrix3fv( this.addr, false, v );\n\n\t\tcopyArray( cache, v );\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, elements ) ) return;\n\n\t\tmat3array.set( elements );\n\n\t\tgl.uniformMatrix3fv( this.addr, false, mat3array );\n\n\t\tcopyArray( cache, elements );\n\n\t}\n\n}\n\nfunction setValueM4( gl, v ) {\n\n\tconst cache = this.cache;\n\tconst elements = v.elements;\n\n\tif ( elements === undefined ) {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniformMatrix4fv( this.addr, false, v );\n\n\t\tcopyArray( cache, v );\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, elements ) ) return;\n\n\t\tmat4array.set( elements );\n\n\t\tgl.uniformMatrix4fv( this.addr, false, mat4array );\n\n\t\tcopyArray( cache, elements );\n\n\t}\n\n}\n\n// Single integer / boolean\n\nfunction setValueV1i( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( cache[ 0 ] === v ) return;\n\n\tgl.uniform1i( this.addr, v );\n\n\tcache[ 0 ] = v;\n\n}\n\n// Single integer / boolean vector (from flat array or THREE.VectorN)\n\nfunction setValueV2i( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y ) {\n\n\t\t\tgl.uniform2i( this.addr, v.x, v.y );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform2iv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\nfunction setValueV3i( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y || cache[ 2 ] !== v.z ) {\n\n\t\t\tgl.uniform3i( this.addr, v.x, v.y, v.z );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\t\t\tcache[ 2 ] = v.z;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform3iv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\nfunction setValueV4i( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y || cache[ 2 ] !== v.z || cache[ 3 ] !== v.w ) {\n\n\t\t\tgl.uniform4i( this.addr, v.x, v.y, v.z, v.w );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\t\t\tcache[ 2 ] = v.z;\n\t\t\tcache[ 3 ] = v.w;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform4iv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\n// Single unsigned integer\n\nfunction setValueV1ui( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( cache[ 0 ] === v ) return;\n\n\tgl.uniform1ui( this.addr, v );\n\n\tcache[ 0 ] = v;\n\n}\n\n// Single unsigned integer vector (from flat array or THREE.VectorN)\n\nfunction setValueV2ui( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y ) {\n\n\t\t\tgl.uniform2ui( this.addr, v.x, v.y );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform2uiv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\nfunction setValueV3ui( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y || cache[ 2 ] !== v.z ) {\n\n\t\t\tgl.uniform3ui( this.addr, v.x, v.y, v.z );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\t\t\tcache[ 2 ] = v.z;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform3uiv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\nfunction setValueV4ui( gl, v ) {\n\n\tconst cache = this.cache;\n\n\tif ( v.x !== undefined ) {\n\n\t\tif ( cache[ 0 ] !== v.x || cache[ 1 ] !== v.y || cache[ 2 ] !== v.z || cache[ 3 ] !== v.w ) {\n\n\t\t\tgl.uniform4ui( this.addr, v.x, v.y, v.z, v.w );\n\n\t\t\tcache[ 0 ] = v.x;\n\t\t\tcache[ 1 ] = v.y;\n\t\t\tcache[ 2 ] = v.z;\n\t\t\tcache[ 3 ] = v.w;\n\n\t\t}\n\n\t} else {\n\n\t\tif ( arraysEqual( cache, v ) ) return;\n\n\t\tgl.uniform4uiv( this.addr, v );\n\n\t\tcopyArray( cache, v );\n\n\t}\n\n}\n\n\n// Single texture (2D / Cube)\n\nfunction setValueT1( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\tconst unit = textures.allocateTextureUnit();\n\n\tif ( cache[ 0 ] !== unit ) {\n\n\t\tgl.uniform1i( this.addr, unit );\n\t\tcache[ 0 ] = unit;\n\n\t}\n\n\tconst emptyTexture2D = ( this.type === gl.SAMPLER_2D_SHADOW ) ? emptyShadowTexture : emptyTexture;\n\n\ttextures.setTexture2D( v || emptyTexture2D, unit );\n\n}\n\nfunction setValueT3D1( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\tconst unit = textures.allocateTextureUnit();\n\n\tif ( cache[ 0 ] !== unit ) {\n\n\t\tgl.uniform1i( this.addr, unit );\n\t\tcache[ 0 ] = unit;\n\n\t}\n\n\ttextures.setTexture3D( v || empty3dTexture, unit );\n\n}\n\nfunction setValueT6( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\tconst unit = textures.allocateTextureUnit();\n\n\tif ( cache[ 0 ] !== unit ) {\n\n\t\tgl.uniform1i( this.addr, unit );\n\t\tcache[ 0 ] = unit;\n\n\t}\n\n\ttextures.setTextureCube( v || emptyCubeTexture, unit );\n\n}\n\nfunction setValueT2DArray1( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\tconst unit = textures.allocateTextureUnit();\n\n\tif ( cache[ 0 ] !== unit ) {\n\n\t\tgl.uniform1i( this.addr, unit );\n\t\tcache[ 0 ] = unit;\n\n\t}\n\n\ttextures.setTexture2DArray( v || emptyArrayTexture, unit );\n\n}\n\n// Helper to pick the right setter for the singular case\n\nfunction getSingularSetter( type ) {\n\n\tswitch ( type ) {\n\n\t\tcase 0x1406: return setValueV1f; // FLOAT\n\t\tcase 0x8b50: return setValueV2f; // _VEC2\n\t\tcase 0x8b51: return setValueV3f; // _VEC3\n\t\tcase 0x8b52: return setValueV4f; // _VEC4\n\n\t\tcase 0x8b5a: return setValueM2; // _MAT2\n\t\tcase 0x8b5b: return setValueM3; // _MAT3\n\t\tcase 0x8b5c: return setValueM4; // _MAT4\n\n\t\tcase 0x1404: case 0x8b56: return setValueV1i; // INT, BOOL\n\t\tcase 0x8b53: case 0x8b57: return setValueV2i; // _VEC2\n\t\tcase 0x8b54: case 0x8b58: return setValueV3i; // _VEC3\n\t\tcase 0x8b55: case 0x8b59: return setValueV4i; // _VEC4\n\n\t\tcase 0x1405: return setValueV1ui; // UINT\n\t\tcase 0x8dc6: return setValueV2ui; // _VEC2\n\t\tcase 0x8dc7: return setValueV3ui; // _VEC3\n\t\tcase 0x8dc8: return setValueV4ui; // _VEC4\n\n\t\tcase 0x8b5e: // SAMPLER_2D\n\t\tcase 0x8d66: // SAMPLER_EXTERNAL_OES\n\t\tcase 0x8dca: // INT_SAMPLER_2D\n\t\tcase 0x8dd2: // UNSIGNED_INT_SAMPLER_2D\n\t\tcase 0x8b62: // SAMPLER_2D_SHADOW\n\t\t\treturn setValueT1;\n\n\t\tcase 0x8b5f: // SAMPLER_3D\n\t\tcase 0x8dcb: // INT_SAMPLER_3D\n\t\tcase 0x8dd3: // UNSIGNED_INT_SAMPLER_3D\n\t\t\treturn setValueT3D1;\n\n\t\tcase 0x8b60: // SAMPLER_CUBE\n\t\tcase 0x8dcc: // INT_SAMPLER_CUBE\n\t\tcase 0x8dd4: // UNSIGNED_INT_SAMPLER_CUBE\n\t\tcase 0x8dc5: // SAMPLER_CUBE_SHADOW\n\t\t\treturn setValueT6;\n\n\t\tcase 0x8dc1: // SAMPLER_2D_ARRAY\n\t\tcase 0x8dcf: // INT_SAMPLER_2D_ARRAY\n\t\tcase 0x8dd7: // UNSIGNED_INT_SAMPLER_2D_ARRAY\n\t\tcase 0x8dc4: // SAMPLER_2D_ARRAY_SHADOW\n\t\t\treturn setValueT2DArray1;\n\n\t}\n\n}\n\n\n// Array of scalars\n\nfunction setValueV1fArray( gl, v ) {\n\n\tgl.uniform1fv( this.addr, v );\n\n}\n\n// Array of vectors (from flat array or array of THREE.VectorN)\n\nfunction setValueV2fArray( gl, v ) {\n\n\tconst data = flatten( v, this.size, 2 );\n\n\tgl.uniform2fv( this.addr, data );\n\n}\n\nfunction setValueV3fArray( gl, v ) {\n\n\tconst data = flatten( v, this.size, 3 );\n\n\tgl.uniform3fv( this.addr, data );\n\n}\n\nfunction setValueV4fArray( gl, v ) {\n\n\tconst data = flatten( v, this.size, 4 );\n\n\tgl.uniform4fv( this.addr, data );\n\n}\n\n// Array of matrices (from flat array or array of THREE.MatrixN)\n\nfunction setValueM2Array( gl, v ) {\n\n\tconst data = flatten( v, this.size, 4 );\n\n\tgl.uniformMatrix2fv( this.addr, false, data );\n\n}\n\nfunction setValueM3Array( gl, v ) {\n\n\tconst data = flatten( v, this.size, 9 );\n\n\tgl.uniformMatrix3fv( this.addr, false, data );\n\n}\n\nfunction setValueM4Array( gl, v ) {\n\n\tconst data = flatten( v, this.size, 16 );\n\n\tgl.uniformMatrix4fv( this.addr, false, data );\n\n}\n\n// Array of integer / boolean\n\nfunction setValueV1iArray( gl, v ) {\n\n\tgl.uniform1iv( this.addr, v );\n\n}\n\n// Array of integer / boolean vectors (from flat array)\n\nfunction setValueV2iArray( gl, v ) {\n\n\tgl.uniform2iv( this.addr, v );\n\n}\n\nfunction setValueV3iArray( gl, v ) {\n\n\tgl.uniform3iv( this.addr, v );\n\n}\n\nfunction setValueV4iArray( gl, v ) {\n\n\tgl.uniform4iv( this.addr, v );\n\n}\n\n// Array of unsigned integer\n\nfunction setValueV1uiArray( gl, v ) {\n\n\tgl.uniform1uiv( this.addr, v );\n\n}\n\n// Array of unsigned integer vectors (from flat array)\n\nfunction setValueV2uiArray( gl, v ) {\n\n\tgl.uniform2uiv( this.addr, v );\n\n}\n\nfunction setValueV3uiArray( gl, v ) {\n\n\tgl.uniform3uiv( this.addr, v );\n\n}\n\nfunction setValueV4uiArray( gl, v ) {\n\n\tgl.uniform4uiv( this.addr, v );\n\n}\n\n\n// Array of textures (2D / 3D / Cube / 2DArray)\n\nfunction setValueT1Array( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\n\tconst n = v.length;\n\n\tconst units = allocTexUnits( textures, n );\n\n\tif ( ! arraysEqual( cache, units ) ) {\n\n\t\tgl.uniform1iv( this.addr, units );\n\n\t\tcopyArray( cache, units );\n\n\t}\n\n\tfor ( let i = 0; i !== n; ++ i ) {\n\n\t\ttextures.setTexture2D( v[ i ] || emptyTexture, units[ i ] );\n\n\t}\n\n}\n\nfunction setValueT3DArray( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\n\tconst n = v.length;\n\n\tconst units = allocTexUnits( textures, n );\n\n\tif ( ! arraysEqual( cache, units ) ) {\n\n\t\tgl.uniform1iv( this.addr, units );\n\n\t\tcopyArray( cache, units );\n\n\t}\n\n\tfor ( let i = 0; i !== n; ++ i ) {\n\n\t\ttextures.setTexture3D( v[ i ] || empty3dTexture, units[ i ] );\n\n\t}\n\n}\n\nfunction setValueT6Array( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\n\tconst n = v.length;\n\n\tconst units = allocTexUnits( textures, n );\n\n\tif ( ! arraysEqual( cache, units ) ) {\n\n\t\tgl.uniform1iv( this.addr, units );\n\n\t\tcopyArray( cache, units );\n\n\t}\n\n\tfor ( let i = 0; i !== n; ++ i ) {\n\n\t\ttextures.setTextureCube( v[ i ] || emptyCubeTexture, units[ i ] );\n\n\t}\n\n}\n\nfunction setValueT2DArrayArray( gl, v, textures ) {\n\n\tconst cache = this.cache;\n\n\tconst n = v.length;\n\n\tconst units = allocTexUnits( textures, n );\n\n\tif ( ! arraysEqual( cache, units ) ) {\n\n\t\tgl.uniform1iv( this.addr, units );\n\n\t\tcopyArray( cache, units );\n\n\t}\n\n\tfor ( let i = 0; i !== n; ++ i ) {\n\n\t\ttextures.setTexture2DArray( v[ i ] || emptyArrayTexture, units[ i ] );\n\n\t}\n\n}\n\n\n// Helper to pick the right setter for a pure (bottom-level) array\n\nfunction getPureArraySetter( type ) {\n\n\tswitch ( type ) {\n\n\t\tcase 0x1406: return setValueV1fArray; // FLOAT\n\t\tcase 0x8b50: return setValueV2fArray; // _VEC2\n\t\tcase 0x8b51: return setValueV3fArray; // _VEC3\n\t\tcase 0x8b52: return setValueV4fArray; // _VEC4\n\n\t\tcase 0x8b5a: return setValueM2Array; // _MAT2\n\t\tcase 0x8b5b: return setValueM3Array; // _MAT3\n\t\tcase 0x8b5c: return setValueM4Array; // _MAT4\n\n\t\tcase 0x1404: case 0x8b56: return setValueV1iArray; // INT, BOOL\n\t\tcase 0x8b53: case 0x8b57: return setValueV2iArray; // _VEC2\n\t\tcase 0x8b54: case 0x8b58: return setValueV3iArray; // _VEC3\n\t\tcase 0x8b55: case 0x8b59: return setValueV4iArray; // _VEC4\n\n\t\tcase 0x1405: return setValueV1uiArray; // UINT\n\t\tcase 0x8dc6: return setValueV2uiArray; // _VEC2\n\t\tcase 0x8dc7: return setValueV3uiArray; // _VEC3\n\t\tcase 0x8dc8: return setValueV4uiArray; // _VEC4\n\n\t\tcase 0x8b5e: // SAMPLER_2D\n\t\tcase 0x8d66: // SAMPLER_EXTERNAL_OES\n\t\tcase 0x8dca: // INT_SAMPLER_2D\n\t\tcase 0x8dd2: // UNSIGNED_INT_SAMPLER_2D\n\t\tcase 0x8b62: // SAMPLER_2D_SHADOW\n\t\t\treturn setValueT1Array;\n\n\t\tcase 0x8b5f: // SAMPLER_3D\n\t\tcase 0x8dcb: // INT_SAMPLER_3D\n\t\tcase 0x8dd3: // UNSIGNED_INT_SAMPLER_3D\n\t\t\treturn setValueT3DArray;\n\n\t\tcase 0x8b60: // SAMPLER_CUBE\n\t\tcase 0x8dcc: // INT_SAMPLER_CUBE\n\t\tcase 0x8dd4: // UNSIGNED_INT_SAMPLER_CUBE\n\t\tcase 0x8dc5: // SAMPLER_CUBE_SHADOW\n\t\t\treturn setValueT6Array;\n\n\t\tcase 0x8dc1: // SAMPLER_2D_ARRAY\n\t\tcase 0x8dcf: // INT_SAMPLER_2D_ARRAY\n\t\tcase 0x8dd7: // UNSIGNED_INT_SAMPLER_2D_ARRAY\n\t\tcase 0x8dc4: // SAMPLER_2D_ARRAY_SHADOW\n\t\t\treturn setValueT2DArrayArray;\n\n\t}\n\n}\n\n// --- Uniform Classes ---\n\nclass SingleUniform {\n\n\tconstructor( id, activeInfo, addr ) {\n\n\t\tthis.id = id;\n\t\tthis.addr = addr;\n\t\tthis.cache = [];\n\t\tthis.type = activeInfo.type;\n\t\tthis.setValue = getSingularSetter( activeInfo.type );\n\n\t\t// this.path = activeInfo.name; // DEBUG\n\n\t}\n\n}\n\nclass PureArrayUniform {\n\n\tconstructor( id, activeInfo, addr ) {\n\n\t\tthis.id = id;\n\t\tthis.addr = addr;\n\t\tthis.cache = [];\n\t\tthis.type = activeInfo.type;\n\t\tthis.size = activeInfo.size;\n\t\tthis.setValue = getPureArraySetter( activeInfo.type );\n\n\t\t// this.path = activeInfo.name; // DEBUG\n\n\t}\n\n}\n\nclass StructuredUniform {\n\n\tconstructor( id ) {\n\n\t\tthis.id = id;\n\n\t\tthis.seq = [];\n\t\tthis.map = {};\n\n\t}\n\n\tsetValue( gl, value, textures ) {\n\n\t\tconst seq = this.seq;\n\n\t\tfor ( let i = 0, n = seq.length; i !== n; ++ i ) {\n\n\t\t\tconst u = seq[ i ];\n\t\t\tu.setValue( gl, value[ u.id ], textures );\n\n\t\t}\n\n\t}\n\n}\n\n// --- Top-level ---\n\n// Parser - builds up the property tree from the path strings\n\nconst RePathPart = /(\\w+)(\\])?(\\[|\\.)?/g;\n\n// extracts\n// \t- the identifier (member name or array index)\n//  - followed by an optional right bracket (found when array index)\n//  - followed by an optional left bracket or dot (type of subscript)\n//\n// Note: These portions can be read in a non-overlapping fashion and\n// allow straightforward parsing of the hierarchy that WebGL encodes\n// in the uniform names.\n\nfunction addUniform( container, uniformObject ) {\n\n\tcontainer.seq.push( uniformObject );\n\tcontainer.map[ uniformObject.id ] = uniformObject;\n\n}\n\nfunction parseUniform( activeInfo, addr, container ) {\n\n\tconst path = activeInfo.name,\n\t\tpathLength = path.length;\n\n\t// reset RegExp object, because of the early exit of a previous run\n\tRePathPart.lastIndex = 0;\n\n\twhile ( true ) {\n\n\t\tconst match = RePathPart.exec( path ),\n\t\t\tmatchEnd = RePathPart.lastIndex;\n\n\t\tlet id = match[ 1 ];\n\t\tconst idIsIndex = match[ 2 ] === ']',\n\t\t\tsubscript = match[ 3 ];\n\n\t\tif ( idIsIndex ) id = id | 0; // convert to integer\n\n\t\tif ( subscript === undefined || subscript === '[' && matchEnd + 2 === pathLength ) {\n\n\t\t\t// bare name or \"pure\" bottom-level array \"[0]\" suffix\n\n\t\t\taddUniform( container, subscript === undefined ?\n\t\t\t\tnew SingleUniform( id, activeInfo, addr ) :\n\t\t\t\tnew PureArrayUniform( id, activeInfo, addr ) );\n\n\t\t\tbreak;\n\n\t\t} else {\n\n\t\t\t// step into inner node / create it in case it doesn't exist\n\n\t\t\tconst map = container.map;\n\t\t\tlet next = map[ id ];\n\n\t\t\tif ( next === undefined ) {\n\n\t\t\t\tnext = new StructuredUniform( id );\n\t\t\t\taddUniform( container, next );\n\n\t\t\t}\n\n\t\t\tcontainer = next;\n\n\t\t}\n\n\t}\n\n}\n\n// Root Container\n\nclass WebGLUniforms {\n\n\tconstructor( gl, program ) {\n\n\t\tthis.seq = [];\n\t\tthis.map = {};\n\n\t\tconst n = gl.getProgramParameter( program, gl.ACTIVE_UNIFORMS );\n\n\t\tfor ( let i = 0; i < n; ++ i ) {\n\n\t\t\tconst info = gl.getActiveUniform( program, i ),\n\t\t\t\taddr = gl.getUniformLocation( program, info.name );\n\n\t\t\tparseUniform( info, addr, this );\n\n\t\t}\n\n\t}\n\n\tsetValue( gl, name, value, textures ) {\n\n\t\tconst u = this.map[ name ];\n\n\t\tif ( u !== undefined ) u.setValue( gl, value, textures );\n\n\t}\n\n\tsetOptional( gl, object, name ) {\n\n\t\tconst v = object[ name ];\n\n\t\tif ( v !== undefined ) this.setValue( gl, name, v );\n\n\t}\n\n\tstatic upload( gl, seq, values, textures ) {\n\n\t\tfor ( let i = 0, n = seq.length; i !== n; ++ i ) {\n\n\t\t\tconst u = seq[ i ],\n\t\t\t\tv = values[ u.id ];\n\n\t\t\tif ( v.needsUpdate !== false ) {\n\n\t\t\t\t// note: always updating when .needsUpdate is undefined\n\t\t\t\tu.setValue( gl, v.value, textures );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tstatic seqWithValue( seq, values ) {\n\n\t\tconst r = [];\n\n\t\tfor ( let i = 0, n = seq.length; i !== n; ++ i ) {\n\n\t\t\tconst u = seq[ i ];\n\t\t\tif ( u.id in values ) r.push( u );\n\n\t\t}\n\n\t\treturn r;\n\n\t}\n\n}\n\nexport { WebGLUniforms };\n\n// Path: src/renderers/webgl/WebGLExtensions.js\n\nfunction WebGLExtensions( gl ) {\n\n\tconst extensions = {};\n\n\tfunction getExtension( name ) {\n\n\t\tif ( extensions[ name ] !== undefined ) {\n\n\t\t\treturn extensions[ name ];\n\n\t\t}\n\n\t\tlet extension;\n\n\t\tswitch ( name ) {\n\n\t\t\tcase 'WEBGL_depth_texture':\n\t\t\t\textension = gl.getExtension( 'WEBGL_depth_texture' ) || gl.getExtension( 'MOZ_WEBGL_depth_texture' ) || gl.getExtension( 'WEBKIT_WEBGL_depth_texture' );\n\t\t\t\tbreak;\n\n\t\t\tcase 'EXT_texture_filter_anisotropic':\n\t\t\t\textension = gl.getExtension( 'EXT_texture_filter_anisotropic' ) || gl.getExtension( 'MOZ_EXT_texture_filter_anisotropic' ) || gl.getExtension( 'WEBKIT_EXT_texture_filter_anisotropic' );\n\t\t\t\tbreak;\n\n\t\t\tcase 'WEBGL_compressed_texture_s3tc':\n\t\t\t\textension = gl.getExtension( 'WEBGL_compressed_texture_s3tc' ) || gl.getExtension( 'MOZ_WEBGL_compressed_texture_s3tc' ) || gl.getExtension( 'WEBKIT_WEBGL_compressed_texture_s3tc' );\n\t\t\t\tbreak;\n\n\t\t\tcase 'WEBGL_compressed_texture_pvrtc':\n\t\t\t\textension = gl.getExtension( 'WEBGL_compressed_texture_pvrtc' ) || gl.getExtension( 'WEBKIT_WEBGL_compressed_texture_pvrtc' );\n\t\t\t\tbreak;\n\n\t\t\tdefault:\n\t\t\t\textension = gl.getExtension( name );\n\n\t\t}\n\n\t\textensions[ name ] = extension;\n\n\t\treturn extension;\n\n\t}\n\n\treturn {\n\n\t\thas: function ( name ) {\n\n\t\t\treturn getExtension( name ) !== null;\n\n\t\t},\n\n\t\tinit: function () {\n\n\t\t\tgetExtension( 'EXT_color_buffer_float' );\n\t\t\tgetExtension( 'WEBGL_clip_cull_distance' );\n\t\t\tgetExtension( 'OES_texture_float_linear' );\n\t\t\tgetExtension( 'EXT_color_buffer_half_float' );\n\t\t\tgetExtension( 'WEBGL_multisampled_render_to_texture' );\n\t\t\tgetExtension( 'WEBGL_render_shared_exponent' );\n\n\t\t},\n\n\t\tget: function ( name ) {\n\n\t\t\tconst extension = getExtension( name );\n\n\t\t\tif ( extension === null ) {\n\n\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: ' + name + ' extension not supported.' );\n\n\t\t\t}\n\n\t\t\treturn extension;\n\n\t\t}\n\n\t};\n\n}\n\n\nexport { WebGLExtensions };\n\n// Path: src/renderers/webgl/WebGLRenderLists.js\nfunction painterSortStable( a, b ) {\n\n\tif ( a.groupOrder !== b.groupOrder ) {\n\n\t\treturn a.groupOrder - b.groupOrder;\n\n\t} else if ( a.renderOrder !== b.renderOrder ) {\n\n\t\treturn a.renderOrder - b.renderOrder;\n\n\t} else if ( a.material.id !== b.material.id ) {\n\n\t\treturn a.material.id - b.material.id;\n\n\t} else if ( a.z !== b.z ) {\n\n\t\treturn a.z - b.z;\n\n\t} else {\n\n\t\treturn a.id - b.id;\n\n\t}\n\n}\n\nfunction reversePainterSortStable( a, b ) {\n\n\tif ( a.groupOrder !== b.groupOrder ) {\n\n\t\treturn a.groupOrder - b.groupOrder;\n\n\t} else if ( a.renderOrder !== b.renderOrder ) {\n\n\t\treturn a.renderOrder - b.renderOrder;\n\n\t} else if ( a.z !== b.z ) {\n\n\t\treturn b.z - a.z;\n\n\t} else {\n\n\t\treturn a.id - b.id;\n\n\t}\n\n}\n\n\nfunction WebGLRenderList() {\n\n\tconst renderItems = [];\n\tlet renderItemsIndex = 0;\n\n\tconst opaque = [];\n\tconst transmissive = [];\n\tconst transparent = [];\n\n\tfunction init() {\n\n\t\trenderItemsIndex = 0;\n\n\t\topaque.length = 0;\n\t\ttransmissive.length = 0;\n\t\ttransparent.length = 0;\n\n\t}\n\n\tfunction getNextRenderItem( object, geometry, material, groupOrder, z, group ) {\n\n\t\tlet renderItem = renderItems[ renderItemsIndex ];\n\n\t\tif ( renderItem === undefined ) {\n\n\t\t\trenderItem = {\n\t\t\t\tid: object.id,\n\t\t\t\tobject: object,\n\t\t\t\tgeometry: geometry,\n\t\t\t\tmaterial: material,\n\t\t\t\tgroupOrder: groupOrder,\n\t\t\t\trenderOrder: object.renderOrder,\n\t\t\t\tz: z,\n\t\t\t\tgroup: group\n\t\t\t};\n\n\t\t\trenderItems[ renderItemsIndex ] = renderItem;\n\n\t\t} else {\n\n\t\t\trenderItem.id = object.id;\n\t\t\trenderItem.object = object;\n\t\t\trenderItem.geometry = geometry;\n\t\t\trenderItem.material = material;\n\t\t\trenderItem.groupOrder = groupOrder;\n\t\t\trenderItem.renderOrder = object.renderOrder;\n\t\t\trenderItem.z = z;\n\t\t\trenderItem.group = group;\n\n\t\t}\n\n\t\trenderItemsIndex ++;\n\n\t\treturn renderItem;\n\n\t}\n\n\tfunction push( object, geometry, material, groupOrder, z, group ) {\n\n\t\tconst renderItem = getNextRenderItem( object, geometry, material, groupOrder, z, group );\n\n\t\tif ( material.transmission > 0.0 ) {\n\n\t\t\ttransmissive.push( renderItem );\n\n\t\t} else if ( material.transparent === true ) {\n\n\t\t\ttransparent.push( renderItem );\n\n\t\t} else {\n\n\t\t\topaque.push( renderItem );\n\n\t\t}\n\n\t}\n\n\tfunction unshift( object, geometry, material, groupOrder, z, group ) {\n\n\t\tconst renderItem = getNextRenderItem( object, geometry, material, groupOrder, z, group );\n\n\t\tif ( material.transmission > 0.0 ) {\n\n\t\t\ttransmissive.unshift( renderItem );\n\n\t\t} else if ( material.transparent === true ) {\n\n\t\t\ttransparent.unshift( renderItem );\n\n\t\t} else {\n\n\t\t\topaque.unshift( renderItem );\n\n\t\t}\n\n\t}\n\n\tfunction sort( customOpaqueSort, customTransparentSort ) {\n\n\t\tif ( opaque.length > 1 ) opaque.sort( customOpaqueSort || painterSortStable );\n\t\tif ( transmissive.length > 1 ) transmissive.sort( customTransparentSort || reversePainterSortStable );\n\t\tif ( transparent.length > 1 ) transparent.sort( customTransparentSort || reversePainterSortStable );\n\n\t}\n\n\tfunction finish() {\n\n\t\t// Clear references from inactive renderItems in the list\n\n\t\tfor ( let i = renderItemsIndex, il = renderItems.length; i < il; i ++ ) {\n\n\t\t\tconst renderItem = renderItems[ i ];\n\n\t\t\tif ( renderItem.id === null ) break;\n\n\t\t\trenderItem.id = null;\n\t\t\trenderItem.object = null;\n\t\t\trenderItem.geometry = null;\n\t\t\trenderItem.material = null;\n\t\t\trenderItem.group = null;\n\n\t\t}\n\n\t}\n\n\treturn {\n\n\t\topaque: opaque,\n\t\ttransmissive: transmissive,\n\t\ttransparent: transparent,\n\n\t\tinit: init,\n\t\tpush: push,\n\t\tunshift: unshift,\n\t\tfinish: finish,\n\n\t\tsort: sort\n\t};\n\n}\n\nfunction WebGLRenderLists() {\n\n\tlet lists = new WeakMap();\n\n\tfunction get( scene, renderCallDepth ) {\n\n\t\tconst listArray = lists.get( scene );\n\t\tlet list;\n\n\t\tif ( listArray === undefined ) {\n\n\t\t\tlist = new WebGLRenderList();\n\t\t\tlists.set( scene, [ list ] );\n\n\t\t} else {\n\n\t\t\tif ( renderCallDepth >= listArray.length ) {\n\n\t\t\t\tlist = new WebGLRenderList();\n\t\t\t\tlistArray.push( list );\n\n\t\t\t} else {\n\n\t\t\t\tlist = listArray[ renderCallDepth ];\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn list;\n\n\t}\n\n\tfunction dispose() {\n\n\t\tlists = new WeakMap();\n\n\t}\n\n\treturn {\n\t\tget: get,\n\t\tdispose: dispose\n\t};\n\n}\n\n\nexport { WebGLRenderLists, WebGLRenderList };\n\n// Path: src/renderers/webgl/WebGLState.js\n...\n", "needle_token_start": 12185, "needle_token_end": 12947, "code_context_ntokens": 14565, "output": ["```javascript\nfunction WebGLExtensions( gl ) {\n\n\tconst extensions = {};\n\n\tfunction getExtension( name ) {\n\n\t\tif ( extensions[ name ] !== undefined ) {\n\n\t\t\treturn extensions[ name ];\n\n\t\t}\n\n\t\tlet extension;\n\n\t\tswitch ( name ) {\n\n\t\t\tcase 'WEBGL_depth_texture':\n\t\t\t\textension = gl.getExtension( 'WEBGL_depth_texture' ) || gl.getExtension( 'MOZ_WEBGL_depth_texture' ) || gl.getExtension( 'WEBKIT_WEBGL_depth_texture' );\n\t\t\t\tbreak;\n\n\t\t\tcase 'EXT_texture_filter_anisotropic':\n\t\t\t\textension = gl.getExtension( 'EXT_texture_filter_anisotropic' ) || gl.getExtension( 'MOZ_EXT_texture_filter_anisotropic' ) || gl.getExtension( 'WEBKIT_EXT_texture_filter_anisotropic' );\n\t\t\t\tbreak;\n\n\t\t\tcase 'WEBGL_compressed_texture_s3tc':\n\t\t\t\textension = gl.getExtension( 'WEBGL_compressed_texture_s3tc' ) || gl.getExtension( 'MOZ_WEBGL_compressed_texture_s3tc' ) || gl.getExtension( 'WEBKIT_WEBGL_compressed_texture_s3tc' );\n\t\t\t\tbreak;\n\n\t\t\tcase 'WEBGL_compressed_texture_pvrtc':\n\t\t\t\textension = gl.getExtension( 'WEBGL_compressed_texture_pvrtc' ) || gl.getExtension( 'WEBKIT_WEBGL_compressed_texture_pvrtc' );\n\t\t\t\tbreak;\n\n\t\t\tdefault:\n\t\t\t\textension = gl.getExtension( name );\n\n\t\t}\n\n\t\textensions[ name ] = extension;\n\n\t\treturn extension;\n\n\t}\n\n\treturn {\n\n\t\thas: function ( name ) {\n\n\t\t\treturn getExtension( name ) !== null;\n\n\t\t},\n\n\t\tinit: function () {\n\n\t\t\tgetExtension( 'EXT_color_buffer_float' );\n\t\t\tgetExtension( 'WEBGL_clip_cull_distance' );\n\t\t\tgetExtension( 'OES_texture_float_linear' );\n\t\t\tgetExtension( 'EXT_color_buffer_half_float' );\n\t\t\tgetExtension( 'WEBGL_multisampled_render_to_texture' );\n\t\t\tgetExtension( 'WEBGL_render_shared_exponent' );\n\n\t\t},\n\n\t\tget: function ( name ) {\n\n\t\t\tconst extension = getExtension( name );\n\n\t\t\tif ( extension === null ) {\n\n\t\t\t\tconsole.warn( 'THREE.WebGLRenderer: ' + name + ' extension not supported.' );\n\n\t\t\t}\n\n\t\t\treturn extension;\n\n\t\t}\n\n\t};\n\n}\n\nexport { WebGLExtensions };\n```"]}
{"repo": "mrdoob/three.js", "name": "serialize", "language": "typescript", "path": "src/core/Object3D.js", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a 3D object into a JSON format and store it in a library if it hasn't been stored already, ensuring each object is uniquely serialized based on its identifier.\n2. **Input**: The function takes two parameters: a library (a storage object) and an element (a 3D object).\n3. **Output**: Returns the unique identifier of the 3D object after ensuring it is stored in the library.\n4. **Procedure**: The function checks if the 3D object's identifier already exists in the library. If not, it converts the 3D object into JSON format using a method specific to 3D objects and stores this JSON in the library using the object's identifier as the key. Finally, it returns the object's identifier.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/math/Matrix4.js\nimport { WebGLCoordinateSystem, WebGPUCoordinateSystem } from '../constants.js';\nimport { Vector3 } from './Vector3.js';\n\nclass Matrix4 {\n\n\tconstructor( n11, n12, n13, n14, n21, n22, n23, n24, n31, n32, n33, n34, n41, n42, n43, n44 ) {\n\n\t\tMatrix4.prototype.isMatrix4 = true;\n\n\t\tthis.elements = [\n\n\t\t\t1, 0, 0, 0,\n\t\t\t0, 1, 0, 0,\n\t\t\t0, 0, 1, 0,\n\t\t\t0, 0, 0, 1\n\n\t\t];\n\n\t\tif ( n11 !== undefined ) {\n\n\t\t\tthis.set( n11, n12, n13, n14, n21, n22, n23, n24, n31, n32, n33, n34, n41, n42, n43, n44 );\n\n\t\t}\n\n\t}\n\n\tset( n11, n12, n13, n14, n21, n22, n23, n24, n31, n32, n33, n34, n41, n42, n43, n44 ) {\n\n\t\tconst te = this.elements;\n\n\t\tte[ 0 ] = n11; te[ 4 ] = n12; te[ 8 ] = n13; te[ 12 ] = n14;\n\t\tte[ 1 ] = n21; te[ 5 ] = n22; te[ 9 ] = n23; te[ 13 ] = n24;\n\t\tte[ 2 ] = n31; te[ 6 ] = n32; te[ 10 ] = n33; te[ 14 ] = n34;\n\t\tte[ 3 ] = n41; te[ 7 ] = n42; te[ 11 ] = n43; te[ 15 ] = n44;\n\n\t\treturn this;\n\n\t}\n\n\tidentity() {\n\n\t\tthis.set(\n\n\t\t\t1, 0, 0, 0,\n\t\t\t0, 1, 0, 0,\n\t\t\t0, 0, 1, 0,\n\t\t\t0, 0, 0, 1\n\n\t\t);\n\n\t\treturn this;\n\n\t}\n\n\tclone() {\n\n\t\treturn new Matrix4().fromArray( this.elements );\n\n\t}\n\n\tcopy( m ) {\n\n\t\tconst te = this.elements;\n\t\tconst me = m.elements;\n\n\t\tte[ 0 ] = me[ 0 ]; te[ 1 ] = me[ 1 ]; te[ 2 ] = me[ 2 ]; te[ 3 ] = me[ 3 ];\n\t\tte[ 4 ] = me[ 4 ]; te[ 5 ] = me[ 5 ]; te[ 6 ] = me[ 6 ]; te[ 7 ] = me[ 7 ];\n\t\tte[ 8 ] = me[ 8 ]; te[ 9 ] = me[ 9 ]; te[ 10 ] = me[ 10 ]; te[ 11 ] = me[ 11 ];\n\t\tte[ 12 ] = me[ 12 ]; te[ 13 ] = me[ 13 ]; te[ 14 ] = me[ 14 ]; te[ 15 ] = me[ 15 ];\n\n\t\treturn this;\n\n\t}\n\n\tcopyPosition( m ) {\n\n\t\tconst te = this.elements, me = m.elements;\n\n\t\tte[ 12 ] = me[ 12 ];\n\t\tte[ 13 ] = me[ 13 ];\n\t\tte[ 14 ] = me[ 14 ];\n\n\t\treturn this;\n\n\t}\n\n\tsetFromMatrix3( m ) {\n\n\t\tconst me = m.elements;\n\n\t\tthis.set(\n\n\t\t\tme[ 0 ], me[ 3 ], me[ 6 ], 0,\n\t\t\tme[ 1 ], me[ 4 ], me[ 7 ], 0,\n\t\t\tme[ 2 ], me[ 5 ], me[ 8 ], 0,\n\t\t\t0, 0, 0, 1\n\n\t\t);\n\n\t\treturn this;\n\n\t}\n\n\textractBasis( xAxis, yAxis, zAxis ) {\n\n\t\txAxis.setFromMatrixColumn( this, 0 );\n\t\tyAxis.setFromMatrixColumn( this, 1 );\n\t\tzAxis.setFromMatrixColumn( this, 2 );\n\n\t\treturn this;\n\n\t}\n\n\tmakeBasis( xAxis, yAxis, zAxis ) {\n\n\t\tthis.set(\n\t\t\txAxis.x, yAxis.x, zAxis.x, 0,\n\t\t\txAxis.y, yAxis.y, zAxis.y, 0,\n\t\t\txAxis.z, yAxis.z, zAxis.z, 0,\n\t\t\t0, 0, 0, 1\n\t\t);\n\n\t\treturn this;\n\n\t}\n\n\textractRotation( m ) {\n\n\t\t// this method does not support reflection matrices\n\n\t\tconst te = this.elements;\n\t\tconst me = m.elements;\n\n\t\tconst scaleX = 1 / _v1.setFromMatrixColumn( m, 0 ).length();\n\t\tconst scaleY = 1 / _v1.setFromMatrixColumn( m, 1 ).length();\n\t\tconst scaleZ = 1 / _v1.setFromMatrixColumn( m, 2 ).length();\n\n\t\tte[ 0 ] = me[ 0 ] * scaleX;\n\t\tte[ 1 ] = me[ 1 ] * scaleX;\n\t\tte[ 2 ] = me[ 2 ] * scaleX;\n\t\tte[ 3 ] = 0;\n\n\t\tte[ 4 ] = me[ 4 ] * scaleY;\n\t\tte[ 5 ] = me[ 5 ] * scaleY;\n\t\tte[ 6 ] = me[ 6 ] * scaleY;\n\t\tte[ 7 ] = 0;\n\n\t\tte[ 8 ] = me[ 8 ] * scaleZ;\n\t\tte[ 9 ] = me[ 9 ] * scaleZ;\n\t\tte[ 10 ] = me[ 10 ] * scaleZ;\n\t\tte[ 11 ] = 0;\n\n\t\tte[ 12 ] = 0;\n\t\tte[ 13 ] = 0;\n\t\tte[ 14 ] = 0;\n\t\tte[ 15 ] = 1;\n\n\t\treturn this;\n\n\t}\n\n\tmakeRotationFromEuler( euler ) {\n\n\t\tconst te = this.elements;\n\n\t\tconst x = euler.x, y = euler.y, z = euler.z;\n\t\tconst a = Math.cos( x ), b = Math.sin( x );\n\t\tconst c = Math.cos( y ), d = Math.sin( y );\n\t\tconst e = Math.cos( z ), f = Math.sin( z );\n\n\t\tif ( euler.order === 'XYZ' ) {\n\n\t\t\tconst ae = a * e, af = a * f, be = b * e, bf = b * f;\n\n\t\t\tte[ 0 ] = c * e;\n\t\t\tte[ 4 ] = - c * f;\n\t\t\tte[ 8 ] = d;\n\n\t\t\tte[ 1 ] = af + be * d;\n\t\t\tte[ 5 ] = ae - bf * d;\n\t\t\tte[ 9 ] = - b * c;\n\n\t\t\tte[ 2 ] = bf - ae * d;\n\t\t\tte[ 6 ] = be + af * d;\n\t\t\tte[ 10 ] = a * c;\n\n\t\t} else if ( euler.order === 'YXZ' ) {\n\n\t\t\tconst ce = c * e, cf = c * f, de = d * e, df = d * f;\n\n\t\t\tte[ 0 ] = ce + df * b;\n\t\t\tte[ 4 ] = de * b - cf;\n\t\t\tte[ 8 ] = a * d;\n\n\t\t\tte[ 1 ] = a * f;\n\t\t\tte[ 5 ] = a * e;\n\t\t\tte[ 9 ] = - b;\n\n\t\t\tte[ 2 ] = cf * b - de;\n\t\t\tte[ 6 ] = df + ce * b;\n\t\t\tte[ 10 ] = a * c;\n\n\t\t} else if ( euler.order === 'ZXY' ) {\n\n\t\t\tconst ce = c * e, cf = c * f, de = d * e, df = d * f;\n\n\t\t\tte[ 0 ] = ce - df * b;\n\t\t\tte[ 4 ] = - a * f;\n\t\t\tte[ 8 ] = de + cf * b;\n\n\t\t\tte[ 1 ] = cf + de * b;\n\t\t\tte[ 5 ] = a * e;\n\t\t\tte[ 9 ] = df - ce * b;\n\n\t\t\tte[ 2 ] = - a * d;\n\t\t\tte[ 6 ] = b;\n\t\t\tte[ 10 ] = a * c;\n\n\t\t} else if ( euler.order === 'ZYX' ) {\n\n\t\t\tconst ae = a * e, af = a * f, be = b * e, bf = b * f;\n\n\t\t\tte[ 0 ] = c * e;\n\t\t\tte[ 4 ] = be * d - af;\n\t\t\tte[ 8 ] = ae * d + bf;\n\n\t\t\tte[ 1 ] = c * f;\n\t\t\tte[ 5 ] = bf * d + ae;\n\t\t\tte[ 9 ] = af * d - be;\n\n\t\t\tte[ 2 ] = - d;\n\t\t\tte[ 6 ] = b * c;\n\t\t\tte[ 10 ] = a * c;\n\n\t\t} else if ( euler.order === 'YZX' ) {\n\n\t\t\tconst ac = a * c, ad = a * d, bc = b * c, bd = b * d;\n\n\t\t\tte[ 0 ] = c * e;\n\t\t\tte[ 4 ] = bd - ac * f;\n\t\t\tte[ 8 ] = bc * f + ad;\n\n\t\t\tte[ 1 ] = f;\n\t\t\tte[ 5 ] = a * e;\n\t\t\tte[ 9 ] = - b * e;\n\n\t\t\tte[ 2 ] = - d * e;\n\t\t\tte[ 6 ] = ad * f + bc;\n\t\t\tte[ 10 ] = ac - bd * f;\n\n\t\t} else if ( euler.order === 'XZY' ) {\n\n\t\t\tconst ac = a * c, ad = a * d, bc = b * c, bd = b * d;\n\n\t\t\tte[ 0 ] = c * e;\n\t\t\tte[ 4 ] = - f;\n\t\t\tte[ 8 ] = d * e;\n\n\t\t\tte[ 1 ] = ac * f + bd;\n\t\t\tte[ 5 ] = a * e;\n\t\t\tte[ 9 ] = ad * f - bc;\n\n\t\t\tte[ 2 ] = bc * f - ad;\n\t\t\tte[ 6 ] = b * e;\n\t\t\tte[ 10 ] = bd * f + ac;\n\n\t\t}\n\n\t\t// bottom row\n\t\tte[ 3 ] = 0;\n\t\tte[ 7 ] = 0;\n\t\tte[ 11 ] = 0;\n\n\t\t// last column\n\t\tte[ 12 ] = 0;\n\t\tte[ 13 ] = 0;\n\t\tte[ 14 ] = 0;\n\t\tte[ 15 ] = 1;\n\n\t\treturn this;\n\n\t}\n\n\tmakeRotationFromQuaternion( q ) {\n\n\t\treturn this.compose( _zero, q, _one );\n\n\t}\n\n\tlookAt( eye, target, up ) {\n\n\t\tconst te = this.elements;\n\n\t\t_z.subVectors( eye, target );\n\n\t\tif ( _z.lengthSq() === 0 ) {\n\n\t\t\t// eye and target are in the same position\n\n\t\t\t_z.z = 1;\n\n\t\t}\n\n\t\t_z.normalize();\n\t\t_x.crossVectors( up, _z );\n\n\t\tif ( _x.lengthSq() === 0 ) {\n\n\t\t\t// up and z are parallel\n\n\t\t\tif ( Math.abs( up.z ) === 1 ) {\n\n\t\t\t\t_z.x += 0.0001;\n\n\t\t\t} else {\n\n\t\t\t\t_z.z += 0.0001;\n\n\t\t\t}\n\n\t\t\t_z.normalize();\n\t\t\t_x.crossVectors( up, _z );\n\n\t\t}\n\n\t\t_x.normalize();\n\t\t_y.crossVectors( _z, _x );\n\n\t\tte[ 0 ] = _x.x; te[ 4 ] = _y.x; te[ 8 ] = _z.x;\n\t\tte[ 1 ] = _x.y; te[ 5 ] = _y.y; te[ 9 ] = _z.y;\n\t\tte[ 2 ] = _x.z; te[ 6 ] = _y.z; te[ 10 ] = _z.z;\n\n\t\treturn this;\n\n\t}\n\n\tmultiply( m ) {\n\n\t\treturn this.multiplyMatrices( this, m );\n\n\t}\n\n\tpremultiply( m ) {\n\n\t\treturn this.multiplyMatrices( m, this );\n\n\t}\n\n\tmultiplyMatrices( a, b ) {\n\n\t\tconst ae = a.elements;\n\t\tconst be = b.elements;\n\t\tconst te = this.elements;\n\n\t\tconst a11 = ae[ 0 ], a12 = ae[ 4 ], a13 = ae[ 8 ], a14 = ae[ 12 ];\n\t\tconst a21 = ae[ 1 ], a22 = ae[ 5 ], a23 = ae[ 9 ], a24 = ae[ 13 ];\n\t\tconst a31 = ae[ 2 ], a32 = ae[ 6 ], a33 = ae[ 10 ], a34 = ae[ 14 ];\n\t\tconst a41 = ae[ 3 ], a42 = ae[ 7 ], a43 = ae[ 11 ], a44 = ae[ 15 ];\n\n\t\tconst b11 = be[ 0 ], b12 = be[ 4 ], b13 = be[ 8 ], b14 = be[ 12 ];\n\t\tconst b21 = be[ 1 ], b22 = be[ 5 ], b23 = be[ 9 ], b24 = be[ 13 ];\n\t\tconst b31 = be[ 2 ], b32 = be[ 6 ], b33 = be[ 10 ], b34 = be[ 14 ];\n\t\tconst b41 = be[ 3 ], b42 = be[ 7 ], b43 = be[ 11 ], b44 = be[ 15 ];\n\n\t\tte[ 0 ] = a11 * b11 + a12 * b21 + a13 * b31 + a14 * b41;\n\t\tte[ 4 ] = a11 * b12 + a12 * b22 + a13 * b32 + a14 * b42;\n\t\tte[ 8 ] = a11 * b13 + a12 * b23 + a13 * b33 + a14 * b43;\n\t\tte[ 12 ] = a11 * b14 + a12 * b24 + a13 * b34 + a14 * b44;\n\n\t\tte[ 1 ] = a21 * b11 + a22 * b21 + a23 * b31 + a24 * b41;\n\t\tte[ 5 ] = a21 * b12 + a22 * b22 + a23 * b32 + a24 * b42;\n\t\tte[ 9 ] = a21 * b13 + a22 * b23 + a23 * b33 + a24 * b43;\n\t\tte[ 13 ] = a21 * b14 + a22 * b24 + a23 * b34 + a24 * b44;\n\n\t\tte[ 2 ] = a31 * b11 + a32 * b21 + a33 * b31 + a34 * b41;\n\t\tte[ 6 ] = a31 * b12 + a32 * b22 + a33 * b32 + a34 * b42;\n\t\tte[ 10 ] = a31 * b13 + a32 * b23 + a33 * b33 + a34 * b43;\n\t\tte[ 14 ] = a31 * b14 + a32 * b24 + a33 * b34 + a34 * b44;\n\n\t\tte[ 3 ] = a41 * b11 + a42 * b21 + a43 * b31 + a44 * b41;\n\t\tte[ 7 ] = a41 * b12 + a42 * b22 + a43 * b32 + a44 * b42;\n\t\tte[ 11 ] = a41 * b13 + a42 * b23 + a43 * b33 + a44 * b43;\n\t\tte[ 15 ] = a41 * b14 + a42 * b24 + a43 * b34 + a44 * b44;\n\n\t\treturn this;\n\n\t}\n\n\tmultiplyScalar( s ) {\n\n\t\tconst te = this.elements;\n\n\t\tte[ 0 ] *= s; te[ 4 ] *= s; te[ 8 ] *= s; te[ 12 ] *= s;\n\t\tte[ 1 ] *= s; te[ 5 ] *= s; te[ 9 ] *= s; te[ 13 ] *= s;\n\t\tte[ 2 ] *= s; te[ 6 ] *= s; te[ 10 ] *= s; te[ 14 ] *= s;\n\t\tte[ 3 ] *= s; te[ 7 ] *= s; te[ 11 ] *= s; te[ 15 ] *= s;\n\n\t\treturn this;\n\n\t}\n\n\tdeterminant() {\n\n\t\tconst te = this.elements;\n\n...\n// Path: src/math/Euler.js\nimport { Quaternion } from './Quaternion.js';\nimport { Matrix4 } from './Matrix4.js';\nimport { clamp } from './MathUtils.js';\n\nconst _matrix = /*@__PURE__*/ new Matrix4();\nconst _quaternion = /*@__PURE__*/ new Quaternion();\n\nclass Euler {\n\n\tconstructor( x = 0, y = 0, z = 0, order = Euler.DEFAULT_ORDER ) {\n\n\t\tthis.isEuler = true;\n\n\t\tthis._x = x;\n\t\tthis._y = y;\n\t\tthis._z = z;\n\t\tthis._order = order;\n\n\t}\n\n\tget x() {\n\n\t\treturn this._x;\n\n\t}\n\n\tset x( value ) {\n\n\t\tthis._x = value;\n\t\tthis._onChangeCallback();\n\n\t}\n\n\tget y() {\n\n\t\treturn this._y;\n\n\t}\n\n\tset y( value ) {\n\n\t\tthis._y = value;\n\t\tthis._onChangeCallback();\n\n\t}\n\n\tget z() {\n\n\t\treturn this._z;\n\n\t}\n\n\tset z( value ) {\n\n\t\tthis._z = value;\n\t\tthis._onChangeCallback();\n\n\t}\n\n\tget order() {\n\n\t\treturn this._order;\n\n\t}\n\n\tset order( value ) {\n\n\t\tthis._order = value;\n\t\tthis._onChangeCallback();\n\n\t}\n\n\tset( x, y, z, order = this._order ) {\n\n\t\tthis._x = x;\n\t\tthis._y = y;\n\t\tthis._z = z;\n\t\tthis._order = order;\n\n\t\tthis._onChangeCallback();\n\n\t\treturn this;\n\n\t}\n\n\tclone() {\n\n\t\treturn new this.constructor( this._x, this._y, this._z, this._order );\n\n\t}\n\n\tcopy( euler ) {\n\n\t\tthis._x = euler._x;\n\t\tthis._y = euler._y;\n\t\tthis._z = euler._z;\n\t\tthis._order = euler._order;\n\n\t\tthis._onChangeCallback();\n\n\t\treturn this;\n\n\t}\n\n\tsetFromRotationMatrix( m, order = this._order, update = true ) {\n\n\t\t// assumes the upper 3x3 of m is a pure rotation matrix (i.e, unscaled)\n\n\t\tconst te = m.elements;\n\t\tconst m11 = te[ 0 ], m12 = te[ 4 ], m13 = te[ 8 ];\n\t\tconst m21 = te[ 1 ], m22 = te[ 5 ], m23 = te[ 9 ];\n\t\tconst m31 = te[ 2 ], m32 = te[ 6 ], m33 = te[ 10 ];\n\n\t\tswitch ( order ) {\n\n\t\t\tcase 'XYZ':\n\n\t\t\t\tthis._y = Math.asin( clamp( m13, - 1, 1 ) );\n\n\t\t\t\tif ( Math.abs( m13 ) < 0.9999999 ) {\n\n\t\t\t\t\tthis._x = Math.atan2( - m23, m33 );\n\t\t\t\t\tthis._z = Math.atan2( - m12, m11 );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tthis._x = Math.atan2( m32, m22 );\n\t\t\t\t\tthis._z = 0;\n\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\n\t\t\tcase 'YXZ':\n\n\t\t\t\tthis._x = Math.asin( - clamp( m23, - 1, 1 ) );\n\n\t\t\t\tif ( Math.abs( m23 ) < 0.9999999 ) {\n\n\t\t\t\t\tthis._y = Math.atan2( m13, m33 );\n\t\t\t\t\tthis._z = Math.atan2( m21, m22 );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tthis._y = Math.atan2( - m31, m11 );\n\t\t\t\t\tthis._z = 0;\n\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\n\t\t\tcase 'ZXY':\n\n\t\t\t\tthis._x = Math.asin( clamp( m32, - 1, 1 ) );\n\n\t\t\t\tif ( Math.abs( m32 ) < 0.9999999 ) {\n\n\t\t\t\t\tthis._y = Math.atan2( - m31, m33 );\n\t\t\t\t\tthis._z = Math.atan2( - m12, m22 );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tthis._y = 0;\n\t\t\t\t\tthis._z = Math.atan2( m21, m11 );\n\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\n\t\t\tcase 'ZYX':\n\n\t\t\t\tthis._y = Math.asin( - clamp( m31, - 1, 1 ) );\n\n\t\t\t\tif ( Math.abs( m31 ) < 0.9999999 ) {\n\n\t\t\t\t\tthis._x = Math.atan2( m32, m33 );\n\t\t\t\t\tthis._z = Math.atan2( m21, m11 );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tthis._x = 0;\n\t\t\t\t\tthis._z = Math.atan2( - m12, m22 );\n\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\n\t\t\tcase 'YZX':\n\n\t\t\t\tthis._z = Math.asin( clamp( m21, - 1, 1 ) );\n\n\t\t\t\tif ( Math.abs( m21 ) < 0.9999999 ) {\n\n\t\t\t\t\tthis._x = Math.atan2( - m23, m22 );\n\t\t\t\t\tthis._y = Math.atan2( - m31, m11 );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tthis._x = 0;\n\t\t\t\t\tthis._y = Math.atan2( m13, m33 );\n\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\n\t\t\tcase 'XZY':\n\n\t\t\t\tthis._z = Math.asin( - clamp( m12, - 1, 1 ) );\n\n\t\t\t\tif ( Math.abs( m12 ) < 0.9999999 ) {\n\n\t\t\t\t\tthis._x = Math.atan2( m32, m22 );\n\t\t\t\t\tthis._y = Math.atan2( m13, m11 );\n\n\t\t\t\t} else {\n\n\t\t\t\t\tthis._x = Math.atan2( - m23, m33 );\n\t\t\t\t\tthis._y = 0;\n\n\t\t\t\t}\n\n\t\t\t\tbreak;\n\n\t\t\tdefault:\n\n\t\t\t\tconsole.warn( 'THREE.Euler: .setFromRotationMatrix() encountered an unknown order: ' + order );\n\n\t\t}\n\n\t\tthis._order = order;\n\n\t\tif ( update === true ) this._onChangeCallback();\n\n\t\treturn this;\n\n\t}\n\n\tsetFromQuaternion( q, order, update ) {\n\n\t\t_matrix.makeRotationFromQuaternion( q );\n\n\t\treturn this.setFromRotationMatrix( _matrix, order, update );\n\n\t}\n\n\tsetFromVector3( v, order = this._order ) {\n\n\t\treturn this.set( v.x, v.y, v.z, order );\n\n\t}\n\n\treorder( newOrder ) {\n\n\t\t// WARNING: this discards revolution information -bhouston\n\n\t\t_quaternion.setFromEuler( this );\n\n\t\treturn this.setFromQuaternion( _quaternion, newOrder );\n\n\t}\n\n\tequals( euler ) {\n\n\t\treturn ( euler._x === this._x ) && ( euler._y === this._y ) && ( euler._z === this._z ) && ( euler._order === this._order );\n\n\t}\n\n\tfromArray( array ) {\n\n\t\tthis._x = array[ 0 ];\n\t\tthis._y = array[ 1 ];\n\t\tthis._z = array[ 2 ];\n\t\tif ( array[ 3 ] !== undefined ) this._order = array[ 3 ];\n\n\t\tthis._onChangeCallback();\n\n\t\treturn this;\n\n\t}\n\n\ttoArray( array = [], offset = 0 ) {\n\n\t\tarray[ offset ] = this._x;\n\t\tarray[ offset + 1 ] = this._y;\n\t\tarray[ offset + 2 ] = this._z;\n\t\tarray[ offset + 3 ] = this._order;\n\n\t\treturn array;\n\n\t}\n\n\t_onChange( callback ) {\n\n\t\tthis._onChangeCallback = callback;\n\n\t\treturn this;\n\n\t}\n\n\t_onChangeCallback() {}\n\n\t*[ Symbol.iterator ]() {\n\n\t\tyield this._x;\n\t\tyield this._y;\n\t\tyield this._z;\n\t\tyield this._order;\n\n\t}\n\n}\n\nEuler.DEFAULT_ORDER = 'XYZ';\n\nexport { Euler };\n\n// Path: src/core/EventDispatcher.js\n/**\n * https://github.com/mrdoob/eventdispatcher.js/\n */\n\nclass EventDispatcher {\n\n\taddEventListener( type, listener ) {\n\n\t\tif ( this._listeners === undefined ) this._listeners = {};\n\n\t\tconst listeners = this._listeners;\n\n\t\tif ( listeners[ type ] === undefined ) {\n\n\t\t\tlisteners[ type ] = [];\n\n\t\t}\n\n\t\tif ( listeners[ type ].indexOf( listener ) === - 1 ) {\n\n\t\t\tlisteners[ type ].push( listener );\n\n\t\t}\n\n\t}\n\n\thasEventListener( type, listener ) {\n\n\t\tif ( this._listeners === undefined ) return false;\n\n\t\tconst listeners = this._listeners;\n\n\t\treturn listeners[ type ] !== undefined && listeners[ type ].indexOf( listener ) !== - 1;\n\n\t}\n\n\tremoveEventListener( type, listener ) {\n\n\t\tif ( this._listeners === undefined ) return;\n\n\t\tconst listeners = this._listeners;\n\t\tconst listenerArray = listeners[ type ];\n\n\t\tif ( listenerArray !== undefined ) {\n\n\t\t\tconst index = listenerArray.indexOf( listener );\n\n\t\t\tif ( index !== - 1 ) {\n\n\t\t\t\tlistenerArray.splice( index, 1 );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tdispatchEvent( event ) {\n\n\t\tif ( this._listeners === undefined ) return;\n\n\t\tconst listeners = this._listeners;\n\t\tconst listenerArray = listeners[ event.type ];\n\n\t\tif ( listenerArray !== undefined ) {\n\n\t\t\tevent.target = this;\n\n\t\t\t// Make a copy, in case listeners are removed while iterating.\n\t\t\tconst array = listenerArray.slice( 0 );\n\n\t\t\tfor ( let i = 0, l = array.length; i < l; i ++ ) {\n\n\t\t\t\tarray[ i ].call( this, event );\n\n\t\t\t}\n\n\t\t\tevent.target = null;\n\n\t\t}\n\n\t}\n\n}\n\n\nexport { EventDispatcher };\n\n// Path: src/core/Layers.js\nclass Layers {\n\n\tconstructor() {\n\n\t\tthis.mask = 1 | 0;\n\n\t}\n\n\tset( channel ) {\n\n\t\tthis.mask = ( 1 << channel | 0 ) >>> 0;\n\n\t}\n\n\tenable( channel ) {\n\n\t\tthis.mask |= 1 << channel | 0;\n\n\t}\n\n\tenableAll() {\n\n\t\tthis.mask = 0xffffffff | 0;\n\n\t}\n\n\ttoggle( channel ) {\n\n\t\tthis.mask ^= 1 << channel | 0;\n\n\t}\n\n\tdisable( channel ) {\n\n\t\tthis.mask &= ~ ( 1 << channel | 0 );\n\n\t}\n\n\tdisableAll() {\n\n\t\tthis.mask = 0;\n\n\t}\n\n\ttest( layers ) {\n\n\t\treturn ( this.mask & layers.mask ) !== 0;\n\n\t}\n\n\tisEnabled( channel ) {\n\n\t\treturn ( this.mask & ( 1 << channel | 0 ) ) !== 0;\n\n\t}\n\n}\n\n\nexport { Layers };\n\n// Path: src/core/Object3D.js\nimport { Quaternion } from '../math/Quaternion.js';\nimport { Vector3 } from '../math/Vector3.js';\nimport { Matrix4 } from '../math/Matrix4.js';\nimport { EventDispatcher } from './EventDispatcher.js';\nimport { Euler } from '../math/Euler.js';\nimport { Layers } from './Layers.js';\nimport { Matrix3 } from '../math/Matrix3.js';\nimport * as MathUtils from '../math/MathUtils.js';\n\nlet _object3DId = 0;\n\nconst _v1 = /*@__PURE__*/ new Vector3();\nconst _q1 = /*@__PURE__*/ new Quaternion();\nconst _m1 = /*@__PURE__*/ new Matrix4();\nconst _target = /*@__PURE__*/ new Vector3();\n\nconst _position = /*@__PURE__*/ new Vector3();\nconst _scale = /*@__PURE__*/ new Vector3();\nconst _quaternion = /*@__PURE__*/ new Quaternion();\n\nconst _xAxis = /*@__PURE__*/ new Vector3( 1, 0, 0 );\nconst _yAxis = /*@__PURE__*/ new Vector3( 0, 1, 0 );\nconst _zAxis = /*@__PURE__*/ new Vector3( 0, 0, 1 );\n\nconst _addedEvent = { type: 'added' };\nconst _removedEvent = { type: 'removed' };\n\nconst _childaddedEvent = { type: 'childadded', child: null };\nconst _childremovedEvent = { type: 'childremoved', child: null };\n\nclass Object3D extends EventDispatcher {\n\n\tconstructor() {\n\n\t\tsuper();\n\n\t\tthis.isObject3D = true;\n\n\t\tObject.defineProperty( this, 'id', { value: _object3DId ++ } );\n\n\t\tthis.uuid = MathUtils.generateUUID();\n\n\t\tthis.name = '';\n\t\tthis.type = 'Object3D';\n\n\t\tthis.parent = null;\n\t\tthis.children = [];\n\n\t\tthis.up = Object3D.DEFAULT_UP.clone();\n\n\t\tconst position = new Vector3();\n\t\tconst rotation = new Euler();\n\t\tconst quaternion = new Quaternion();\n\t\tconst scale = new Vector3( 1, 1, 1 );\n\n\t\tfunction onRotationChange() {\n\n\t\t\tquaternion.setFromEuler( rotation, false );\n\n\t\t}\n\n\t\tfunction onQuaternionChange() {\n\n\t\t\trotation.setFromQuaternion( quaternion, undefined, false );\n\n\t\t}\n\n\t\trotation._onChange( onRotationChange );\n\t\tquaternion._onChange( onQuaternionChange );\n\n\t\tObject.defineProperties( this, {\n\t\t\tposition: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: position\n\t\t\t},\n\t\t\trotation: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: rotation\n\t\t\t},\n\t\t\tquaternion: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: quaternion\n\t\t\t},\n\t\t\tscale: {\n\t\t\t\tconfigurable: true,\n\t\t\t\tenumerable: true,\n\t\t\t\tvalue: scale\n\t\t\t},\n\t\t\tmodelViewMatrix: {\n\t\t\t\tvalue: new Matrix4()\n\t\t\t},\n\t\t\tnormalMatrix: {\n\t\t\t\tvalue: new Matrix3()\n\t\t\t}\n\t\t} );\n\n\t\tthis.matrix = new Matrix4();\n\t\tthis.matrixWorld = new Matrix4();\n\n\t\tthis.matrixAutoUpdate = Object3D.DEFAULT_MATRIX_AUTO_UPDATE;\n\n\t\tthis.matrixWorldAutoUpdate = Object3D.DEFAULT_MATRIX_WORLD_AUTO_UPDATE; // checked by the renderer\n\t\tthis.matrixWorldNeedsUpdate = false;\n\n\t\tthis.layers = new Layers();\n\t\tthis.visible = true;\n\n\t\tthis.castShadow = false;\n\t\tthis.receiveShadow = false;\n\n\t\tthis.frustumCulled = true;\n\t\tthis.renderOrder = 0;\n\n\t\tthis.animations = [];\n\n\t\tthis.userData = {};\n\n\t}\n\n\tonBeforeShadow( /* renderer, object, camera, shadowCamera, geometry, depthMaterial, group */ ) {}\n\n\tonAfterShadow( /* renderer, object, camera, shadowCamera, geometry, depthMaterial, group */ ) {}\n\n\tonBeforeRender( /* renderer, scene, camera, geometry, material, group */ ) {}\n\n\tonAfterRender( /* renderer, scene, camera, geometry, material, group */ ) {}\n\n\tapplyMatrix4( matrix ) {\n\n\t\tif ( this.matrixAutoUpdate ) this.updateMatrix();\n\n\t\tthis.matrix.premultiply( matrix );\n\n\t\tthis.matrix.decompose( this.position, this.quaternion, this.scale );\n\n\t}\n\n\tapplyQuaternion( q ) {\n\n\t\tthis.quaternion.premultiply( q );\n\n\t\treturn this;\n\n\t}\n\n\tsetRotationFromAxisAngle( axis, angle ) {\n\n\t\t// assumes axis is normalized\n\n\t\tthis.quaternion.setFromAxisAngle( axis, angle );\n\n\t}\n\n\tsetRotationFromEuler( euler ) {\n\n\t\tthis.quaternion.setFromEuler( euler, true );\n\n\t}\n\n\tsetRotationFromMatrix( m ) {\n\n\t\t// assumes the upper 3x3 of m is a pure rotation matrix (i.e, unscaled)\n\n\t\tthis.quaternion.setFromRotationMatrix( m );\n\n\t}\n\n\tsetRotationFromQuaternion( q ) {\n\n\t\t// assumes q is normalized\n\n\t\tthis.quaternion.copy( q );\n\n\t}\n\n\trotateOnAxis( axis, angle ) {\n\n\t\t// rotate object on axis in object space\n\t\t// axis is assumed to be normalized\n\n\t\t_q1.setFromAxisAngle( axis, angle );\n\n\t\tthis.quaternion.multiply( _q1 );\n\n\t\treturn this;\n\n\t}\n\n\trotateOnWorldAxis( axis, angle ) {\n\n\t\t// rotate object on axis in world space\n\t\t// axis is assumed to be normalized\n\t\t// method assumes no rotated parent\n\n\t\t_q1.setFromAxisAngle( axis, angle );\n\n\t\tthis.quaternion.premultiply( _q1 );\n\n\t\treturn this;\n\n\t}\n\n\trotateX( angle ) {\n\n\t\treturn this.rotateOnAxis( _xAxis, angle );\n\n\t}\n\n\trotateY( angle ) {\n\n\t\treturn this.rotateOnAxis( _yAxis, angle );\n\n\t}\n\n\trotateZ( angle ) {\n\n\t\treturn this.rotateOnAxis( _zAxis, angle );\n\n\t}\n\n\ttranslateOnAxis( axis, distance ) {\n\n\t\t// translate object by distance along axis in object space\n\t\t// axis is assumed to be normalized\n\n\t\t_v1.copy( axis ).applyQuaternion( this.quaternion );\n\n\t\tthis.position.add( _v1.multiplyScalar( distance ) );\n\n\t\treturn this;\n\n\t}\n\n\ttranslateX( distance ) {\n\n\t\treturn this.translateOnAxis( _xAxis, distance );\n\n\t}\n\n\ttranslateY( distance ) {\n\n\t\treturn this.translateOnAxis( _yAxis, distance );\n\n\t}\n\n\ttranslateZ( distance ) {\n\n\t\treturn this.translateOnAxis( _zAxis, distance );\n\n\t}\n\n\tlocalToWorld( vector ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\treturn vector.applyMatrix4( this.matrixWorld );\n\n\t}\n\n\tworldToLocal( vector ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\treturn vector.applyMatrix4( _m1.copy( this.matrixWorld ).invert() );\n\n\t}\n\n\tlookAt( x, y, z ) {\n\n\t\t// This method does not support objects having non-uniformly-scaled parent(s)\n\n\t\tif ( x.isVector3 ) {\n\n\t\t\t_target.copy( x );\n\n\t\t} else {\n\n\t\t\t_target.set( x, y, z );\n\n\t\t}\n\n\t\tconst parent = this.parent;\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\t_position.setFromMatrixPosition( this.matrixWorld );\n\n\t\tif ( this.isCamera || this.isLight ) {\n\n\t\t\t_m1.lookAt( _position, _target, this.up );\n\n\t\t} else {\n\n\t\t\t_m1.lookAt( _target, _position, this.up );\n\n\t\t}\n\n\t\tthis.quaternion.setFromRotationMatrix( _m1 );\n\n\t\tif ( parent ) {\n\n\t\t\t_m1.extractRotation( parent.matrixWorld );\n\t\t\t_q1.setFromRotationMatrix( _m1 );\n\t\t\tthis.quaternion.premultiply( _q1.invert() );\n\n\t\t}\n\n\t}\n\n\tadd( object ) {\n\n\t\tif ( arguments.length > 1 ) {\n\n\t\t\tfor ( let i = 0; i < arguments.length; i ++ ) {\n\n\t\t\t\tthis.add( arguments[ i ] );\n\n\t\t\t}\n\n\t\t\treturn this;\n\n\t\t}\n\n\t\tif ( object === this ) {\n\n\t\t\tconsole.error( 'THREE.Object3D.add: object can\\'t be added as a child of itself.', object );\n\t\t\treturn this;\n\n\t\t}\n\n\t\tif ( object && object.isObject3D ) {\n\n\t\t\tobject.removeFromParent();\n\t\t\tobject.parent = this;\n\t\t\tthis.children.push( object );\n\n\t\t\tobject.dispatchEvent( _addedEvent );\n\n\t\t\t_childaddedEvent.child = object;\n\t\t\tthis.dispatchEvent( _childaddedEvent );\n\t\t\t_childaddedEvent.child = null;\n\n\t\t} else {\n\n\t\t\tconsole.error( 'THREE.Object3D.add: object not an instance of THREE.Object3D.', object );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tremove( object ) {\n\n\t\tif ( arguments.length > 1 ) {\n\n\t\t\tfor ( let i = 0; i < arguments.length; i ++ ) {\n\n\t\t\t\tthis.remove( arguments[ i ] );\n\n\t\t\t}\n\n\t\t\treturn this;\n\n\t\t}\n\n\t\tconst index = this.children.indexOf( object );\n\n\t\tif ( index !== - 1 ) {\n\n\t\t\tobject.parent = null;\n\t\t\tthis.children.splice( index, 1 );\n\n\t\t\tobject.dispatchEvent( _removedEvent );\n\n\t\t\t_childremovedEvent.child = object;\n\t\t\tthis.dispatchEvent( _childremovedEvent );\n\t\t\t_childremovedEvent.child = null;\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tremoveFromParent() {\n\n\t\tconst parent = this.parent;\n\n\t\tif ( parent !== null ) {\n\n\t\t\tparent.remove( this );\n\n\t\t}\n\n\t\treturn this;\n\n\t}\n\n\tclear() {\n\n\t\treturn this.remove( ... this.children );\n\n\t}\n\n\tattach( object ) {\n\n\t\t// adds object as a child of this, while maintaining the object's world transform\n\n\t\t// Note: This method does not support scene graphs having non-uniformly-scaled nodes(s)\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\t_m1.copy( this.matrixWorld ).invert();\n\n\t\tif ( object.parent !== null ) {\n\n\t\t\tobject.parent.updateWorldMatrix( true, false );\n\n\t\t\t_m1.multiply( object.parent.matrixWorld );\n\n\t\t}\n\n\t\tobject.applyMatrix4( _m1 );\n\n\t\tobject.removeFromParent();\n\t\tobject.parent = this;\n\t\tthis.children.push( object );\n\n\t\tobject.updateWorldMatrix( false, true );\n\n\t\tobject.dispatchEvent( _addedEvent );\n\n\t\t_childaddedEvent.child = object;\n\t\tthis.dispatchEvent( _childaddedEvent );\n\t\t_childaddedEvent.child = null;\n\n\t\treturn this;\n\n\t}\n\n\tgetObjectById( id ) {\n\n\t\treturn this.getObjectByProperty( 'id', id );\n\n\t}\n\n\tgetObjectByName( name ) {\n\n\t\treturn this.getObjectByProperty( 'name', name );\n\n\t}\n\n\tgetObjectByProperty( name, value ) {\n\n\t\tif ( this[ name ] === value ) return this;\n\n\t\tfor ( let i = 0, l = this.children.length; i < l; i ++ ) {\n\n\t\t\tconst child = this.children[ i ];\n\t\t\tconst object = child.getObjectByProperty( name, value );\n\n\t\t\tif ( object !== undefined ) {\n\n\t\t\t\treturn object;\n\n\t\t\t}\n\n\t\t}\n\n\t\treturn undefined;\n\n\t}\n\n\tgetObjectsByProperty( name, value, result = [] ) {\n\n\t\tif ( this[ name ] === value ) result.push( this );\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tchildren[ i ].getObjectsByProperty( name, value, result );\n\n\t\t}\n\n\t\treturn result;\n\n\t}\n\n\tgetWorldPosition( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\treturn target.setFromMatrixPosition( this.matrixWorld );\n\n\t}\n\n\tgetWorldQuaternion( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\tthis.matrixWorld.decompose( _position, target, _scale );\n\n\t\treturn target;\n\n\t}\n\n\tgetWorldScale( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\tthis.matrixWorld.decompose( _position, _quaternion, target );\n\n\t\treturn target;\n\n\t}\n\n\tgetWorldDirection( target ) {\n\n\t\tthis.updateWorldMatrix( true, false );\n\n\t\tconst e = this.matrixWorld.elements;\n\n\t\treturn target.set( e[ 8 ], e[ 9 ], e[ 10 ] ).normalize();\n\n\t}\n\n\traycast( /* raycaster, intersects */ ) {}\n\n\ttraverse( callback ) {\n\n\t\tcallback( this );\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tchildren[ i ].traverse( callback );\n\n\t\t}\n\n\t}\n\n\ttraverseVisible( callback ) {\n\n\t\tif ( this.visible === false ) return;\n\n\t\tcallback( this );\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tchildren[ i ].traverseVisible( callback );\n\n\t\t}\n\n\t}\n\n\ttraverseAncestors( callback ) {\n\n\t\tconst parent = this.parent;\n\n\t\tif ( parent !== null ) {\n\n\t\t\tcallback( parent );\n\n\t\t\tparent.traverseAncestors( callback );\n\n\t\t}\n\n\t}\n\n\tupdateMatrix() {\n\n\t\tthis.matrix.compose( this.position, this.quaternion, this.scale );\n\n\t\tthis.matrixWorldNeedsUpdate = true;\n\n\t}\n\n\tupdateMatrixWorld( force ) {\n\n\t\tif ( this.matrixAutoUpdate ) this.updateMatrix();\n\n\t\tif ( this.matrixWorldNeedsUpdate || force ) {\n\n\t\t\tif ( this.parent === null ) {\n\n\t\t\t\tthis.matrixWorld.copy( this.matrix );\n\n\t\t\t} else {\n\n\t\t\t\tthis.matrixWorld.multiplyMatrices( this.parent.matrixWorld, this.matrix );\n\n\t\t\t}\n\n\t\t\tthis.matrixWorldNeedsUpdate = false;\n\n\t\t\tforce = true;\n\n\t\t}\n\n\t\t// update children\n\n\t\tconst children = this.children;\n\n\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\tconst child = children[ i ];\n\n\t\t\tif ( child.matrixWorldAutoUpdate === true || force === true ) {\n\n\t\t\t\tchild.updateMatrixWorld( force );\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\tupdateWorldMatrix( updateParents, updateChildren ) {\n\n\t\tconst parent = this.parent;\n\n\t\tif ( updateParents === true && parent !== null && parent.matrixWorldAutoUpdate === true ) {\n\n\t\t\tparent.updateWorldMatrix( true, false );\n\n\t\t}\n\n\t\tif ( this.matrixAutoUpdate ) this.updateMatrix();\n\n\t\tif ( this.parent === null ) {\n\n\t\t\tthis.matrixWorld.copy( this.matrix );\n\n\t\t} else {\n\n\t\t\tthis.matrixWorld.multiplyMatrices( this.parent.matrixWorld, this.matrix );\n\n\t\t}\n\n\t\t// update children\n\n\t\tif ( updateChildren === true ) {\n\n\t\t\tconst children = this.children;\n\n\t\t\tfor ( let i = 0, l = children.length; i < l; i ++ ) {\n\n\t\t\t\tconst child = children[ i ];\n\n\t\t\t\tif ( child.matrixWorldAutoUpdate === true ) {\n\n\t\t\t\t\tchild.updateWorldMatrix( false, true );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t}\n\n\ttoJSON( meta ) {\n\n\t\t// meta is a string when called from JSON.stringify\n\t\tconst isRootObject = ( meta === undefined || typeof meta === 'string' );\n\n\t\tconst output = {};\n\n\t\t// meta is a hash used to collect geometries, materials.\n\t\t// not providing it implies that this is the root object\n\t\t// being serialized.\n\t\tif ( isRootObject ) {\n\n\t\t\t// initialize meta obj\n\t\t\tmeta = {\n\t\t\t\tgeometries: {},\n\t\t\t\tmaterials: {},\n\t\t\t\ttextures: {},\n\t\t\t\timages: {},\n\t\t\t\tshapes: {},\n\t\t\t\tskeletons: {},\n\t\t\t\tanimations: {},\n\t\t\t\tnodes: {}\n\t\t\t};\n\n\t\t\toutput.metadata = {\n\t\t\t\tversion: 4.6,\n\t\t\t\ttype: 'Object',\n\t\t\t\tgenerator: 'Object3D.toJSON'\n\t\t\t};\n\n\t\t}\n\n\t\t// standard Object3D serialization\n\n\t\tconst object = {};\n\n\t\tobject.uuid = this.uuid;\n\t\tobject.type = this.type;\n\n\t\tif ( this.name !== '' ) object.name = this.name;\n\t\tif ( this.castShadow === true ) object.castShadow = true;\n\t\tif ( this.receiveShadow === true ) object.receiveShadow = true;\n\t\tif ( this.visible === false ) object.visible = false;\n\t\tif ( this.frustumCulled === false ) object.frustumCulled = false;\n\t\tif ( this.renderOrder !== 0 ) object.renderOrder = this.renderOrder;\n\t\tif ( Object.keys( this.userData ).length > 0 ) object.userData = this.userData;\n\n\t\tobject.layers = this.layers.mask;\n\t\tobject.matrix = this.matrix.toArray();\n\t\tobject.up = this.up.toArray();\n\n\t\tif ( this.matrixAutoUpdate === false ) object.matrixAutoUpdate = false;\n\n\t\t// object specific properties\n\n\t\tif ( this.isInstancedMesh ) {\n\n\t\t\tobject.type = 'InstancedMesh';\n\t\t\tobject.count = this.count;\n\t\t\tobject.instanceMatrix = this.instanceMatrix.toJSON();\n\t\t\tif ( this.instanceColor !== null ) object.instanceColor = this.instanceColor.toJSON();\n\n\t\t}\n\n\t\tif ( this.isBatchedMesh ) {\n\n\t\t\tobject.type = 'BatchedMesh';\n\t\t\tobject.perObjectFrustumCulled = this.perObjectFrustumCulled;\n\t\t\tobject.sortObjects = this.sortObjects;\n\n\t\t\tobject.drawRanges = this._drawRanges;\n\t\t\tobject.reservedRanges = this._reservedRanges;\n\n\t\t\tobject.visibility = this._visibility;\n\t\t\tobject.active = this._active;\n\t\t\tobject.bounds = this._bounds.map( bound => ( {\n\t\t\t\tboxInitialized: bound.boxInitialized,\n\t\t\t\tboxMin: bound.box.min.toArray(),\n\t\t\t\tboxMax: bound.box.max.toArray(),\n\n\t\t\t\tsphereInitialized: bound.sphereInitialized,\n\t\t\t\tsphereRadius: bound.sphere.radius,\n\t\t\t\tsphereCenter: bound.sphere.center.toArray()\n\t\t\t} ) );\n\n\t\t\tobject.maxGeometryCount = this._maxGeometryCount;\n\t\t\tobject.maxVertexCount = this._maxVertexCount;\n\t\t\tobject.maxIndexCount = this._maxIndexCount;\n\n\t\t\tobject.geometryInitialized = this._geometryInitialized;\n\t\t\tobject.geometryCount = this._geometryCount;\n\n\t\t\tobject.matricesTexture = this._matricesTexture.toJSON( meta );\n\n\t\t\tif ( this.boundingSphere !== null ) {\n\n\t\t\t\tobject.boundingSphere = {\n\t\t\t\t\tcenter: object.boundingSphere.center.toArray(),\n\t\t\t\t\tradius: object.boundingSphere.radius\n\t\t\t\t};\n\n\t\t\t}\n\n\t\t\tif ( this.boundingBox !== null ) {\n\n\t\t\t\tobject.boundingBox = {\n\t\t\t\t\tmin: object.boundingBox.min.toArray(),\n\t\t\t\t\tmax: object.boundingBox.max.toArray()\n\t\t\t\t};\n\n\t\t\t}\n\n\t\t}\n\n\t\t//\n\n\t\t\nfunction serialize( library, element ) {\n\n\t\t\tif ( library[ element.uuid ] === undefined ) {\n\n\t\t\t\tlibrary[ element.uuid ] = element.toJSON( meta );\n\n\t\t\t}\n\n\t\t\treturn element.uuid;\n\n\t\t}\n\n\t\tif ( this.isScene ) {\n\n\t\t\tif ( this.background ) {\n\n\t\t\t\tif ( this.background.isColor ) {\n\n\t\t\t\t\tobject.background = this.background.toJSON();\n\n\t\t\t\t} else if ( this.background.isTexture ) {\n\n\t\t\t\t\tobject.background = this.background.toJSON( meta ).uuid;\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t\tif ( this.environment && this.environment.isTexture && this.environment.isRenderTargetTexture !== true ) {\n\n\t\t\t\tobject.environment = this.environment.toJSON( meta ).uuid;\n\n\t\t\t}\n\n\t\t} else if ( this.isMesh || this.isLine || this.isPoints ) {\n\n\t\t\tobject.geometry = serialize( meta.geometries, this.geometry );\n\n\t\t\tconst parameters = this.geometry.parameters;\n\n\t\t\tif ( parameters !== undefined && parameters.shapes !== undefined ) {\n\n\t\t\t\tconst shapes = parameters.shapes;\n\n\t\t\t\tif ( Array.isArray( shapes ) ) {\n\n\t\t\t\t\tfor ( let i = 0, l = shapes.length; i < l; i ++ ) {\n\n\t\t\t\t\t\tconst shape = shapes[ i ];\n\n\t\t\t\t\t\tserialize( meta.shapes, shape );\n\n\t\t\t\t\t}\n\n\t\t\t\t} else {\n\n\t\t\t\t\tserialize( meta.shapes, shapes );\n\n\t\t\t\t}\n\n\t\t\t}\n\n\t\t}\n\n\t\tif ( this.isSkinnedMesh ) {\n\n\t\t\tobject.bindMode = this.bindMode;\n\t\t\tobject.bindMatrix = this.bindMatrix.toArray();\n\n\t\t\tif ( this.skeleton !== undefined ) {\n\n\t\t\t\tserialize( meta.skeletons, this.skeleton );\n\n\t\t\t\tobject.skeleton = this.skeleton.uuid;\n\n\t\t\t}\n\n\t\t}\n\n\t\tif ( this.material !== undefined ) {\n\n\t\t\tif ( Array.isArray( this.material ) ) {\n\n\t\t\t\tconst uuids = [];\n\n\t\t\t\tfor ( let i = 0, l = this.material.length; i < l; i ++ ) {\n\n\t\t\t\t\tuuids.push( serialize( meta.materials, this.material[ i ] ) );\n\n\t\t\t\t}\n\n\t\t\t\tobject.material = uuids;\n\n\t\t\t} else {\n\n\t\t\t\tobject.material = serialize( meta.materials, this.material );\n\n\t\t\t}\n\n\t\t}\n\n\t\t//\n\n\t\tif ( this.children.length > 0 ) {\n\n...\n", "needle_token_start": 13920, "needle_token_end": 13986, "code_context_ntokens": 14677, "output": ["```javascript\nfunction serialize( library, element ) {\n\n\tif ( library[ element.uuid ] === undefined ) {\n\n\t\tlibrary[ element.uuid ] = element.toJSON( meta );\n\n\t}\n\n\treturn element.uuid;\n\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "initFirestore", "language": "typescript", "path": "src/firestore/firestore-internal.ts", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To create and configure a Firestore database instance using specific settings and an application context.\n2. **Input**: An application instance, a database identifier, and optional settings for the Firestore database.\n3. **Output**: A new Firestore database instance configured with the provided settings.\n4. **Procedure**: \n   - Retrieve and validate the Firestore configuration options based on the application instance and provided settings.\n   - Dynamically load the Firestore library.\n   - Instantiate the Firestore database with the validated options.\n   - Handle any errors related to missing dependencies or library import failures.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    let database = this.databases.get(databaseId);\n    if (database === undefined) {\n      database = initFirestore(this.app, databaseId, {});\n      this.databases.set(databaseId, database);\n      this.firestoreSettings.set(databaseId, {});\n    }\n    return database;\n  }\n\n  private checkIfSameSettings(settingsA: FirestoreSettings, settingsB: FirestoreSettings): boolean {\n    const a = settingsA ?? {};\n    const b = settingsB ?? {};\n    // If we start passing more settings to Firestore constructor,\n    // replace this with deep equality check.\n    return (a.preferRest === b.preferRest);\n  }\n\n  /**\n   * Returns the app associated with this Storage instance.\n   *\n   * @returns The app associated with this Storage instance.\n   */\n  get app(): App {\n    return this.appInternal;\n  }\n}\n\nexport function getFirestoreOptions(app: App, firestoreSettings?: FirestoreSettings): Settings {\n  if (!validator.isNonNullObject(app) || !('options' in app)) {\n    throw new FirebaseFirestoreError({\n      code: 'invalid-argument',\n      message: 'First argument passed to admin.firestore() must be a valid Firebase app instance.',\n    });\n  }\n\n  const projectId: string | null = utils.getExplicitProjectId(app);\n  const credential = app.options.credential;\n  // eslint-disable-next-line @typescript-eslint/no-var-requires\n  const { version: firebaseVersion } = require('../../package.json');\n  const preferRest = firestoreSettings?.preferRest;\n  if (credential instanceof ServiceAccountCredential) {\n    return {\n      credentials: {\n        private_key: credential.privateKey,\n        client_email: credential.clientEmail,\n      },\n      // When the SDK is initialized with ServiceAccountCredentials an explicit projectId is\n      // guaranteed to be available.\n      projectId: projectId!,\n      firebaseVersion,\n      preferRest,\n    };\n  } else if (isApplicationDefault(app.options.credential)) {\n    // Try to use the Google application default credentials.\n    // If an explicit project ID is not available, let Firestore client discover one from the\n    // environment. This prevents the users from having to set GOOGLE_CLOUD_PROJECT in GCP runtimes.\n    return validator.isNonEmptyString(projectId)\n      ? { projectId, firebaseVersion, preferRest }\n      : { firebaseVersion, preferRest };\n  }\n\n  throw new FirebaseFirestoreError({\n    code: 'invalid-credential',\n    message: 'Failed to initialize Google Cloud Firestore client with the available credentials. ' +\n      'Must initialize the SDK with a certificate credential or application default credentials ' +\n      'to use Cloud Firestore API.',\n  });\n}\n\n\nfunction initFirestore(app: App, databaseId: string, firestoreSettings?: FirestoreSettings): Firestore {\n  const options = getFirestoreOptions(app, firestoreSettings);\n  options.databaseId = databaseId;\n  let firestoreDatabase: typeof Firestore;\n  try {\n    // Lazy-load the Firestore implementation here, which in turns loads gRPC.\n    firestoreDatabase = require('@google-cloud/firestore').Firestore;\n  } catch (err) {\n    throw new FirebaseFirestoreError({\n      code: 'missing-dependencies',\n      message: 'Failed to import the Cloud Firestore client library for Node.js. '\n          + 'Make sure to install the \"@google-cloud/firestore\" npm package. '\n          + `Original error: ${err}`,\n    });\n  }\n\n  return new firestoreDatabase(options);\n}\n\n// Path: src/firestore/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Cloud Firestore.\n *\n * @packageDocumentation\n */\n\nimport { Firestore } from '@google-cloud/firestore';\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { FirestoreService, FirestoreSettings } from './firestore-internal';\nimport { DEFAULT_DATABASE_ID } from '@google-cloud/firestore/build/src/path';\n\nexport {\n  AddPrefixToKeys,\n  AggregateField,\n  AggregateFieldType,\n  AggregateQuery,\n  AggregateQuerySnapshot,\n  AggregateSpecData,\n  AggregateSpec,\n  AggregateType,\n  BulkWriter,\n  BulkWriterOptions,\n  BundleBuilder,\n  ChildUpdateFields,\n  CollectionGroup,\n  CollectionReference,\n  DocumentChange,\n  DocumentChangeType,\n  DocumentData,\n  DocumentReference,\n  DocumentSnapshot,\n  FieldPath,\n  FieldValue,\n  Filter,\n  Firestore,\n  FirestoreDataConverter,\n  GeoPoint,\n  GrpcStatus,\n  NestedUpdateFields,\n  OrderByDirection,\n  PartialWithFieldValue,\n  Precondition,\n  Primitive,\n  Query,\n  QueryDocumentSnapshot,\n  QueryPartition,\n  QuerySnapshot,\n  ReadOptions,\n  ReadOnlyTransactionOptions,\n  ReadWriteTransactionOptions,\n  Settings,\n  SetOptions,\n  Timestamp,\n  Transaction,\n  UpdateData,\n  UnionToIntersection,\n  WhereFilterOp,\n  WithFieldValue,\n  WriteBatch,\n  WriteResult,\n  v1,\n  setLogFunction,\n} from '@google-cloud/firestore';\n\nexport { FirestoreSettings };\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n *\n * @example\n * ```javascript\n * // Get the default Firestore service for the default app\n * const defaultFirestore = getFirestore();\n * ```\n\n * @returns The default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n */\nexport function getFirestore(): Firestore;\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app.\n *\n * @example\n * ```javascript\n * // Get the default Firestore service for a specific app\n * const otherFirestore = getFirestore(app);\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @returns The default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service associated with the provided app.\n */\nexport function getFirestore(app: App): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a named database and default app\n * const otherFirestore = getFirestore('otherDb');\n * ```\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n * @beta\n */\nexport function getFirestore(databaseId: string): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a named database and specific app.\n * const otherFirestore = getFirestore('otherDb');\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service associated with the provided app.\n * @beta\n */\nexport function getFirestore(app: App, databaseId: string): Firestore;\n\nexport function getFirestore(\n  appOrDatabaseId?: App | string,\n  optionalDatabaseId?: string\n): Firestore {\n  const app: App = typeof appOrDatabaseId === 'object' ? appOrDatabaseId : getApp();\n  const databaseId =\n    (typeof appOrDatabaseId === 'string' ? appOrDatabaseId : optionalDatabaseId) || DEFAULT_DATABASE_ID;\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  const firestoreService = firebaseApp.getOrInitService(\n    'firestore', (app) => new FirestoreService(app));\n  return firestoreService.getDatabase(databaseId);\n}\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app, passing extra parameters to its constructor.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a specific app, require HTTP/1.1 REST transport\n * const otherFirestore = initializeFirestore(app, {preferRest: true});\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param settings - Settings object to be passed to the constructor.\n *\n * @returns The default `Firestore` service associated with the provided app and settings.\n */\nexport function initializeFirestore(app: App, settings?: FirestoreSettings): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app, passing extra parameters to its constructor.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a specific app, require HTTP/1.1 REST transport\n * const otherFirestore = initializeFirestore(app, {preferRest: true}, 'otherDb');\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param settings - Settings object to be passed to the constructor.\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named `Firestore` service associated with the provided app and settings.\n * @beta\n */\nexport function initializeFirestore(\n  app: App,\n  settings: FirestoreSettings,\n  databaseId: string\n): Firestore;\n\nexport function initializeFirestore(\n  app: App,\n  settings?: FirestoreSettings,\n  databaseId?: string\n): Firestore {\n  settings ??= {};\n  databaseId ??= DEFAULT_DATABASE_ID;\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  const firestoreService = firebaseApp.getOrInitService(\n    'firestore', (app) => new FirestoreService(app));\n\n  return firestoreService.initializeDatabase(databaseId, settings);\n}\n\n// Path: src/auth/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Firebase Authentication.\n *\n * @packageDocumentation\n */\n\nimport { App, getApp } from '../app/index';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { Auth } from './auth';\n\n/**\n * Gets the {@link Auth} service for the default app or a\n * given app.\n *\n * `getAuth()` can be called with no arguments to access the default app's\n * {@link Auth} service or as `getAuth(app)` to access the\n * {@link Auth} service associated with a specific app.\n *\n * @example\n * ```javascript\n * // Get the Auth service for the default app\n * const defaultAuth = getAuth();\n * ```\n *\n * @example\n * ```javascript\n * // Get the Auth service for a given app\n * const otherAuth = getAuth(otherApp);\n * ```\n *\n */\nexport function getAuth(app?: App): Auth {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('auth', (app) => new Auth(app));\n}\n\nexport { ActionCodeSettings } from './action-code-settings-builder';\n\nexport {\n  Auth,\n} from './auth';\n\nexport {\n  AllowByDefault,\n  AllowByDefaultWrap,\n  AllowlistOnly,\n  AllowlistOnlyWrap,\n  AuthFactorType,\n  AuthProviderConfig,\n  AuthProviderConfigFilter,\n  BaseAuthProviderConfig,\n  BaseCreateMultiFactorInfoRequest,\n  BaseUpdateMultiFactorInfoRequest,\n  CreateMultiFactorInfoRequest,\n  CreatePhoneMultiFactorInfoRequest,\n  CreateRequest,\n  EmailSignInProviderConfig,\n  ListProviderConfigResults,\n  MultiFactorConfig,\n  MultiFactorConfigState,\n  MultiFactorCreateSettings,\n  MultiFactorUpdateSettings,\n  MultiFactorProviderConfig,\n  OAuthResponseType,\n  OIDCAuthProviderConfig,\n  OIDCUpdateAuthProviderRequest,\n  RecaptchaAction,\n  RecaptchaConfig,\n  RecaptchaKey,\n  RecaptchaKeyClientType,\n  RecaptchaManagedRule,\n  RecaptchaProviderEnforcementState,\n  SAMLAuthProviderConfig,\n  SAMLUpdateAuthProviderRequest,\n  SmsRegionConfig,\n  UserProvider,\n  UpdateAuthProviderRequest,\n  UpdateMultiFactorInfoRequest,\n  UpdatePhoneMultiFactorInfoRequest,\n  UpdateRequest,\n  TotpMultiFactorProviderConfig,\n  PasswordPolicyConfig,\n  PasswordPolicyEnforcementState,\n  CustomStrengthOptionsConfig,\n  EmailPrivacyConfig,\n} from './auth-config';\n\nexport {\n  BaseAuth,\n  DeleteUsersResult,\n  GetUsersResult,\n  ListUsersResult,\n  SessionCookieOptions,\n} from './base-auth';\n\nexport {\n  EmailIdentifier,\n  PhoneIdentifier,\n  ProviderIdentifier,\n  UidIdentifier,\n  UserIdentifier,\n} from './identifier';\n\nexport {\n  CreateTenantRequest,\n  Tenant,\n  UpdateTenantRequest,\n} from './tenant';\n\nexport {\n  ListTenantsResult,\n  TenantAwareAuth,\n  TenantManager,\n} from './tenant-manager';\n\nexport {\n  UpdateProjectConfigRequest,\n  ProjectConfig,\n} from './project-config';\n\nexport {\n  ProjectConfigManager,\n} from './project-config-manager';\n\nexport {\n  DecodedIdToken,\n  DecodedAuthBlockingToken\n} from './token-verifier';\n\nexport {\n  HashAlgorithmType,\n  UserImportOptions,\n  UserImportRecord,\n  UserImportResult,\n  UserMetadataRequest,\n  UserProviderRequest,\n} from './user-import-builder';\n\nexport {\n  MultiFactorInfo,\n  MultiFactorSettings,\n  PhoneMultiFactorInfo,\n  UserInfo,\n  UserMetadata,\n  UserRecord,\n} from './user-record';\n\n// Path: src/instance-id/instance-id.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { getInstallations } from '../installations';\nimport { App } from '../app/index';\nimport {\n  FirebaseInstallationsError, FirebaseInstanceIdError,\n  InstallationsClientErrorCode, InstanceIdClientErrorCode,\n} from '../utils/error';\nimport * as validator from '../utils/validator';\n\n/**\n * The `InstanceId` service enables deleting the Firebase instance IDs\n * associated with Firebase client app instances.\n *\n * @deprecated Use {@link firebase-admin.installations#Installations} instead.\n */\nexport class InstanceId {\n\n  private app_: App;\n\n  /**\n   * @param app - The app for this InstanceId service.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseInstanceIdError(\n        InstanceIdClientErrorCode.INVALID_ARGUMENT,\n        'First argument passed to instanceId() must be a valid Firebase app instance.',\n      );\n    }\n\n    this.app_ = app;\n  }\n\n  /**\n   * Deletes the specified instance ID and the associated data from Firebase.\n   *\n   * Note that Google Analytics for Firebase uses its own form of Instance ID to\n   * keep track of analytics data. Therefore deleting a Firebase Instance ID does\n   * not delete Analytics data. See\n   * {@link https://firebase.google.com/support/privacy/manage-iids#delete_an_instance_id |\n   * Delete an Instance ID}\n   * for more information.\n   *\n   * @param instanceId - The instance ID to be deleted.\n   *\n   * @returns A promise fulfilled when the instance ID is deleted.\n   */\n  public deleteInstanceId(instanceId: string): Promise<void> {\n    return getInstallations(this.app).deleteInstallation(instanceId)\n      .catch((err) => {\n        if (err instanceof FirebaseInstallationsError) {\n          let code = err.code.replace('installations/', '');\n          if (code === InstallationsClientErrorCode.INVALID_INSTALLATION_ID.code) {\n            code = InstanceIdClientErrorCode.INVALID_INSTANCE_ID.code;\n          }\n\n          throw new FirebaseInstanceIdError({ code, message: err.message });\n        }\n\n        throw err;\n      });\n  }\n\n  /**\n   * Returns the app associated with this InstanceId instance.\n   *\n   * @returns The app associated with this InstanceId instance.\n   */\n  get app(): App {\n    return this.app_;\n  }\n}\n\n// Path: src/project-management/app-metadata.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Platforms with which a Firebase App can be associated.\n */\nexport enum AppPlatform {\n  /**\n   * Unknown state. This is only used for distinguishing unset values.\n   */\n  PLATFORM_UNKNOWN = 'PLATFORM_UNKNOWN',\n\n  /**\n   * The Firebase App is associated with iOS.\n   */\n  IOS = 'IOS',\n\n  /**\n   * The Firebase App is associated with Android.\n   */\n  ANDROID = 'ANDROID',\n}\n\n/**\n * Metadata about a Firebase app.\n */\nexport interface AppMetadata {\n  /**\n   * The globally unique, Firebase-assigned identifier of the app.\n   *\n   * @example\n   * ```javascript\n   * var appId = appMetadata.appId;\n   * ```\n   */\n  appId: string;\n\n  /**\n   * The optional user-assigned display name of the app.\n   *\n   * @example\n   * ```javascript\n   * var displayName = appMetadata.displayName;\n   * ```\n   */\n  displayName?: string;\n\n  /**\n   * The development platform of the app. Supporting Android and iOS app platforms.\n   *\n   * @example\n   * ```javascript\n   * var platform = AppPlatform.ANDROID;\n   * ```\n   */\n  platform: AppPlatform;\n\n  /**\n   * The globally unique, user-assigned ID of the parent project for the app.\n   *\n   * @example\n   * ```javascript\n   * var projectId = appMetadata.projectId;\n   * ```\n   */\n  projectId: string;\n\n  /**\n   * The fully-qualified resource name that identifies this app.\n   *\n   * This is useful when manually constructing requests for Firebase's public API.\n   *\n   * @example\n   * ```javascript\n   * var resourceName = androidAppMetadata.resourceName;\n   * ```\n   */\n  resourceName: string;\n}\n\n// Path: src/project-management/android-app.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseProjectManagementError } from '../utils/error';\nimport * as validator from '../utils/validator';\nimport { ProjectManagementRequestHandler, assertServerResponse } from './project-management-api-request-internal';\nimport { AppMetadata, AppPlatform } from './app-metadata';\n\n\n/**\n * Metadata about a Firebase Android App.\n */\nexport interface AndroidAppMetadata extends AppMetadata {\n\n  platform: AppPlatform.ANDROID;\n\n  /**\n   * The canonical package name of the Android App, as would appear in the Google Play Developer\n   * Console.\n   *\n   * @example\n   * ```javascript\n   * var packageName = androidAppMetadata.packageName;\n   * ```\n   */\n  packageName: string;\n}\n\n/**\n * A reference to a Firebase Android app.\n *\n * Do not call this constructor directly. Instead, use {@link ProjectManagement.androidApp}.\n */\nexport class AndroidApp {\n\n  private readonly resourceName: string;\n\n  /**\n   * @internal\n   */\n  constructor(\n      public readonly appId: string,\n      private readonly requestHandler: ProjectManagementRequestHandler) {\n    if (!validator.isNonEmptyString(appId)) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'appId must be a non-empty string.');\n    }\n\n    this.resourceName = `projects/-/androidApps/${appId}`;\n  }\n\n  /**\n   * Retrieves metadata about this Android app.\n   *\n   * @returns A promise that resolves to the retrieved metadata about this Android app.\n   */\n  public getMetadata(): Promise<AndroidAppMetadata> {\n    return this.requestHandler.getResource(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getMetadata()\\'s responseData must be a non-null object.');\n\n        const requiredFieldsList = ['name', 'appId', 'projectId', 'packageName'];\n        requiredFieldsList.forEach((requiredField) => {\n          assertServerResponse(\n            validator.isNonEmptyString(responseData[requiredField]),\n            responseData,\n            `getMetadata()'s responseData.${requiredField} must be a non-empty string.`);\n        });\n\n        const metadata: AndroidAppMetadata = {\n          platform: AppPlatform.ANDROID,\n          resourceName: responseData.name,\n          appId: responseData.appId,\n          displayName: responseData.displayName || null,\n          projectId: responseData.projectId,\n          packageName: responseData.packageName,\n        };\n        return metadata;\n      });\n  }\n\n  /**\n   * Sets the optional user-assigned display name of the app.\n   *\n   * @param newDisplayName - The new display name to set.\n   *\n   * @returns A promise that resolves when the display name has been set.\n   */\n  public setDisplayName(newDisplayName: string): Promise<void> {\n    return this.requestHandler.setDisplayName(this.resourceName, newDisplayName);\n  }\n\n  /**\n   * Gets the list of SHA certificates associated with this Android app in Firebase.\n   *\n   * @returns The list of SHA-1 and SHA-256 certificates associated with this Android app in\n   *     Firebase.\n   */\n  public getShaCertificates(): Promise<ShaCertificate[]> {\n    return this.requestHandler.getAndroidShaCertificates(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getShaCertificates()\\'s responseData must be a non-null object.');\n\n        if (!responseData.certificates) {\n          return [];\n        }\n\n        assertServerResponse(\n          validator.isArray(responseData.certificates),\n          responseData,\n          '\"certificates\" field must be present in the getShaCertificates() response data.');\n\n        const requiredFieldsList = ['name', 'shaHash'];\n\n        return responseData.certificates.map((certificateJson: any) => {\n          requiredFieldsList.forEach((requiredField) => {\n            assertServerResponse(\n              validator.isNonEmptyString(certificateJson[requiredField]),\n              responseData,\n              `getShaCertificates()'s responseData.certificates[].${requiredField} must be a `\n                      + 'non-empty string.');\n          });\n\n          return new ShaCertificate(certificateJson.shaHash, certificateJson.name);\n        });\n      });\n  }\n\n  /**\n   * Adds the given SHA certificate to this Android app.\n   *\n   * @param certificateToAdd - The SHA certificate to add.\n   *\n   * @returns A promise that resolves when the given certificate\n   *     has been added to the Android app.\n   */\n  public addShaCertificate(certificateToAdd: ShaCertificate): Promise<void> {\n    return this.requestHandler.addAndroidShaCertificate(this.resourceName, certificateToAdd);\n  }\n\n  /**\n   * Deletes the specified SHA certificate from this Android app.\n   *\n   * @param certificateToDelete - The SHA certificate to delete.\n   *\n   * @returns A promise that resolves when the specified\n   *     certificate has been removed from the Android app.\n   */\n  public deleteShaCertificate(certificateToDelete: ShaCertificate): Promise<void> {\n    if (!certificateToDelete.resourceName) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument',\n        'Specified certificate does not include a resourceName. (Use AndroidApp.getShaCertificates() to retrieve ' +\n              'certificates with a resourceName.');\n    }\n    return this.requestHandler.deleteResource(certificateToDelete.resourceName);\n  }\n\n  /**\n   * Gets the configuration artifact associated with this app.\n   *\n   * @returns A promise that resolves to the Android app's\n   *     Firebase config file, in UTF-8 string format. This string is typically\n   *     intended to be written to a JSON file that gets shipped with your Android\n   *     app.\n   */\n  public getConfig(): Promise<string> {\n    return this.requestHandler.getConfig(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getConfig()\\'s responseData must be a non-null object.');\n\n        const base64ConfigFileContents = responseData.configFileContents;\n        assertServerResponse(\n          validator.isBase64String(base64ConfigFileContents),\n          responseData,\n          'getConfig()\\'s responseData.configFileContents must be a base64 string.');\n\n        return Buffer.from(base64ConfigFileContents, 'base64').toString('utf8');\n      });\n  }\n}\n\n/**\n * A SHA-1 or SHA-256 certificate.\n *\n * Do not call this constructor directly. Instead, use\n * [`projectManagement.shaCertificate()`](projectManagement.ProjectManagement#shaCertificate).\n */\nexport class ShaCertificate {\n  /**\n   * The SHA certificate type.\n   *\n   * @example\n   * ```javascript\n   * var certType = shaCertificate.certType;\n   * ```\n   */\n  public readonly certType: ('sha1' | 'sha256');\n\n  /**\n   * Creates a ShaCertificate using the given hash. The ShaCertificate's type (eg. 'sha256') is\n   * automatically determined from the hash itself.\n   *\n   * @param shaHash - The sha256 or sha1 hash for this certificate.\n   * @example\n   * ```javascript\n   * var shaHash = shaCertificate.shaHash;\n   * ```\n   * @param resourceName - The Firebase resource name for this certificate. This does not need to be\n   *     set when creating a new certificate.\n   * @example\n   * ```javascript\n   * var resourceName = shaCertificate.resourceName;\n   * ```\n   *\n   * @internal\n   */\n  constructor(public readonly shaHash: string, public readonly resourceName?: string) {\n    if (/^[a-fA-F0-9]{40}$/.test(shaHash)) {\n      this.certType = 'sha1';\n    } else if (/^[a-fA-F0-9]{64}$/.test(shaHash)) {\n      this.certType = 'sha256';\n    } else {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'shaHash must be either a sha256 hash or a sha1 hash.');\n    }\n  }\n}\n\n// Path: src/project-management/project-management-api-request-internal.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport {\n  AuthorizedHttpClient, HttpError, HttpMethod, HttpRequestConfig, ExponentialBackoffPoller,\n} from '../utils/api-request';\nimport { FirebaseProjectManagementError, ProjectManagementErrorCode } from '../utils/error';\nimport { getSdkVersion } from '../utils/index';\nimport * as validator from '../utils/validator';\nimport { ShaCertificate } from './android-app';\n\n\n/** Project management backend host and port. */\nconst PROJECT_MANAGEMENT_HOST_AND_PORT = 'firebase.googleapis.com:443';\n/** Project management backend path. */\nconst PROJECT_MANAGEMENT_PATH = '/v1/';\n/** Project management beta backend path. */\nconst PROJECT_MANAGEMENT_BETA_PATH = '/v1beta1/';\n/** Project management request header. */\nconst PROJECT_MANAGEMENT_HEADERS = {\n  'X-Client-Version': `Node/Admin/${getSdkVersion()}`,\n};\n/** Project management request timeout duration in milliseconds. */\nconst PROJECT_MANAGEMENT_TIMEOUT_MILLIS = 10000;\n\nconst LIST_APPS_MAX_PAGE_SIZE = 100;\n\nconst CERT_TYPE_API_MAP = {\n  sha1: 'SHA_1',\n  sha256: 'SHA_256',\n};\n\nexport function assertServerResponse(\n  condition: boolean, responseData: object, message: string): void {\n  if (!condition) {\n    throw new FirebaseProjectManagementError(\n      'invalid-server-response',\n      `${message} Response data: ${JSON.stringify(responseData, null, 2)}`);\n  }\n}\n\n/**\n * Class that provides mechanism to send requests to the Firebase project management backend\n * endpoints.\n *\n * @internal\n */\nexport class ProjectManagementRequestHandler {\n  private readonly baseUrl: string =\n    `https://${PROJECT_MANAGEMENT_HOST_AND_PORT}${PROJECT_MANAGEMENT_PATH}`;\n  private readonly baseBetaUrl: string =\n    `https://${PROJECT_MANAGEMENT_HOST_AND_PORT}${PROJECT_MANAGEMENT_BETA_PATH}`;\n  private readonly httpClient: AuthorizedHttpClient;\n\n  private static wrapAndRethrowHttpError(errStatusCode: number, errText?: string): void {\n    let errorCode: ProjectManagementErrorCode;\n    let errorMessage: string;\n\n    switch (errStatusCode) {\n    case 400:\n      errorCode = 'invalid-argument';\n      errorMessage = 'Invalid argument provided.';\n      break;\n    case 401:\n    case 403:\n      errorCode = 'authentication-error';\n      errorMessage = 'An error occurred when trying to authenticate. Make sure the credential '\n            + 'used to authenticate this SDK has the proper permissions. See '\n            + 'https://firebase.google.com/docs/admin/setup for setup instructions.';\n      break;\n    case 404:\n      errorCode = 'not-found';\n      errorMessage = 'The specified entity could not be found.';\n      break;\n    case 409:\n      errorCode = 'already-exists';\n      errorMessage = 'The specified entity already exists.';\n      break;\n    case 500:\n      errorCode = 'internal-error';\n      errorMessage = 'An internal error has occurred. Please retry the request.';\n      break;\n    case 503:\n      errorCode = 'service-unavailable';\n      errorMessage = 'The server could not process the request in time. See the error '\n            + 'documentation for more details.';\n      break;\n    default:\n      errorCode = 'unknown-error';\n      errorMessage = 'An unknown server error was returned.';\n    }\n\n    if (!errText) {\n      errText = '<missing>';\n    }\n    throw new FirebaseProjectManagementError(\n      errorCode,\n      `${ errorMessage } Status code: ${ errStatusCode }. Raw server response: \"${ errText }\".`);\n  }\n\n  /**\n   * @param app - The app used to fetch access tokens to sign API requests.\n   * @constructor\n   */\n  constructor(app: App) {\n    this.httpClient = new AuthorizedHttpClient(app as FirebaseApp);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose Android\n   *     apps you want to list.\n   */\n  public listAndroidApps(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}/androidApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose iOS apps\n   *     you want to list.\n   */\n  public listIosApps(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}/iosApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose iOS apps\n   *     you want to list.\n   */\n  public listAppMetadata(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}:searchApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project that you want\n   *     to create the Android app within.\n   */\n  public createAndroidApp(\n    parentResourceName: string, packageName: string, displayName?: string): Promise<object> {\n    const requestData: any = {\n      packageName,\n    };\n    if (validator.isNonEmptyString(displayName)) {\n      requestData.displayName = displayName;\n    }\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/androidApps`, requestData, 'v1beta1')\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createAndroidApp\\'s responseData must be a non-null object.');\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.name),\n          responseData,\n          'createAndroidApp\\'s responseData.name must be a non-empty string.');\n        return this.pollRemoteOperationWithExponentialBackoff(responseData.name);\n      });\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project that you want\n   *     to create the iOS app within.\n   */\n  public createIosApp(\n    parentResourceName: string, bundleId: string, displayName?: string): Promise<object> {\n    const requestData: any = {\n      bundleId,\n    };\n    if (validator.isNonEmptyString(displayName)) {\n      requestData.displayName = displayName;\n    }\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/iosApps`, requestData, 'v1beta1')\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createIosApp\\'s responseData must be a non-null object.');\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.name),\n          responseData,\n          'createIosApp\\'s responseData.name must be a non-empty string.');\n        return this.pollRemoteOperationWithExponentialBackoff(responseData.name);\n      });\n  }\n\n  /**\n   * @param resourceName - Fully-qualified resource name of the entity whose display name you\n   *     want to set.\n   */\n  public setDisplayName(resourceName: string, newDisplayName: string): Promise<void> {\n    const requestData = {\n      displayName: newDisplayName,\n    };\n    return this\n      .invokeRequestHandler(\n        'PATCH', `${resourceName}?update_mask=display_name`, requestData, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the Android app whose SHA\n   *     certificates you want to get.\n   */\n  public getAndroidShaCertificates(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET', `${parentResourceName}/sha`, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the Android app that you\n   *     want to add the given SHA certificate to.\n   */\n  public addAndroidShaCertificate(\n    parentResourceName: string, certificate: ShaCertificate): Promise<void> {\n    const requestData = {\n      shaHash: certificate.shaHash,\n      certType: CERT_TYPE_API_MAP[certificate.certType],\n    };\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/sha`, requestData, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the app whose config you\n   *     want to get.\n   */\n  public getConfig(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET', `${parentResourceName}/config`, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the entity that you want to\n   *     get.\n   */\n  public getResource(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler('GET', parentResourceName, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param resourceName - Fully-qualified resource name of the entity that you want to\n   *     delete.\n   */\n  public deleteResource(resourceName: string): Promise<void> {\n    return this\n      .invokeRequestHandler('DELETE', resourceName, /* requestData */ null, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  private pollRemoteOperationWithExponentialBackoff(\n    operationResourceName: string): Promise<object> {\n    const poller = new ExponentialBackoffPoller<object>();\n\n    return poller.poll(() => {\n      return this.invokeRequestHandler('GET', operationResourceName, /* requestData */ null)\n        .then((responseData: any) => {\n          if (responseData.error) {\n            const errStatusCode: number = responseData.error.code || 500;\n            const errText: string =\n                  responseData.error.message || JSON.stringify(responseData.error);\n            ProjectManagementRequestHandler.wrapAndRethrowHttpError(errStatusCode, errText);\n          }\n\n          if (!responseData.done) {\n            // Continue polling.\n            return null;\n          }\n\n          // Polling complete. Resolve with operation response JSON.\n          return responseData.response;\n        });\n    });\n  }\n\n  /**\n   * Invokes the request handler with the provided request data.\n   */\n  private invokeRequestHandler(\n    method: HttpMethod,\n    path: string,\n    requestData: object | null,\n    apiVersion: ('v1' | 'v1beta1') = 'v1'): Promise<object> {\n    const baseUrlToUse = (apiVersion === 'v1') ? this.baseUrl : this.baseBetaUrl;\n    const request: HttpRequestConfig = {\n      method,\n      url: `${baseUrlToUse}${path}`,\n      headers: PROJECT_MANAGEMENT_HEADERS,\n      data: requestData,\n      timeout: PROJECT_MANAGEMENT_TIMEOUT_MILLIS,\n    };\n\n    return this.httpClient.send(request)\n      .then((response) => {\n        // Send non-JSON responses to the catch() below, where they will be treated as errors.\n        if (!response.isJson()) {\n          throw new HttpError(response);\n        }\n\n        return response.data;\n      })\n      .catch((err) => {\n        if (err instanceof HttpError) {\n          ProjectManagementRequestHandler.wrapAndRethrowHttpError(\n            err.response.status, err.response.text);\n        }\n        throw err;\n      });\n  }\n}\n\n// Path: src/project-management/ios-app.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseProjectManagementError } from '../utils/error';\nimport * as validator from '../utils/validator';\nimport { ProjectManagementRequestHandler, assertServerResponse } from './project-management-api-request-internal';\nimport { AppMetadata, AppPlatform } from './app-metadata';\n\n/**\n * Metadata about a Firebase iOS App.\n */\nexport interface IosAppMetadata extends AppMetadata {\n  platform: AppPlatform.IOS;\n\n  /**\n   * The canonical bundle ID of the iOS App as it would appear in the iOS App Store.\n   *\n   * @example\n   * ```javascript\n   * var bundleId = iosAppMetadata.bundleId;\n   *```\n    */\n  bundleId: string;\n}\n\n/**\n * A reference to a Firebase iOS app.\n *\n * Do not call this constructor directly. Instead, use {@link ProjectManagement.iosApp}.\n */\nexport class IosApp {\n\n  private readonly resourceName: string;\n\n  /**\n   * @internal\n   */\n  constructor(\n      public readonly appId: string,\n      private readonly requestHandler: ProjectManagementRequestHandler) {\n    if (!validator.isNonEmptyString(appId)) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'appId must be a non-empty string.');\n    }\n\n    this.resourceName = `projects/-/iosApps/${appId}`;\n  }\n\n  /**\n   * Retrieves metadata about this iOS app.\n   *\n   * @returns A promise that\n   *     resolves to the retrieved metadata about this iOS app.\n   */\n  public getMetadata(): Promise<IosAppMetadata> {\n    return this.requestHandler.getResource(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getMetadata()\\'s responseData must be a non-null object.');\n\n        const requiredFieldsList = ['name', 'appId', 'projectId', 'bundleId'];\n        requiredFieldsList.forEach((requiredField) => {\n          assertServerResponse(\n            validator.isNonEmptyString(responseData[requiredField]),\n            responseData,\n            `getMetadata()'s responseData.${requiredField} must be a non-empty string.`);\n        });\n\n        const metadata: IosAppMetadata = {\n          platform: AppPlatform.IOS,\n          resourceName: responseData.name,\n          appId: responseData.appId,\n          displayName: responseData.displayName || null,\n          projectId: responseData.projectId,\n          bundleId: responseData.bundleId,\n        };\n        return metadata;\n      });\n  }\n\n  /**\n   * Sets the optional user-assigned display name of the app.\n   *\n   * @param newDisplayName - The new display name to set.\n   *\n   * @returns A promise that resolves when the display name has\n   *     been set.\n   */\n  public setDisplayName(newDisplayName: string): Promise<void> {\n    return this.requestHandler.setDisplayName(this.resourceName, newDisplayName);\n  }\n\n  /**\n   * Gets the configuration artifact associated with this app.\n   *\n   * @returns A promise that resolves to the iOS app's Firebase\n   *     config file, in UTF-8 string format. This string is typically intended to\n   *     be written to a plist file that gets shipped with your iOS app.\n   */\n  public getConfig(): Promise<string> {\n    return this.requestHandler.getConfig(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getConfig()\\'s responseData must be a non-null object.');\n\n        const base64ConfigFileContents = responseData.configFileContents;\n        assertServerResponse(\n          validator.isBase64String(base64ConfigFileContents),\n          responseData,\n          'getConfig()\\'s responseData.configFileContents must be a base64 string.');\n\n        return Buffer.from(base64ConfigFileContents, 'base64').toString('utf8');\n      });\n  }\n}\n\n// Path: src/project-management/project-management.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseProjectManagementError } from '../utils/error';\nimport * as utils from '../utils/index';\nimport * as validator from '../utils/validator';\nimport { AndroidApp, ShaCertificate } from './android-app';\nimport { IosApp } from './ios-app';\nimport { ProjectManagementRequestHandler, assertServerResponse } from './project-management-api-request-internal';\nimport { AppMetadata, AppPlatform } from './app-metadata';\n\n/**\n * The Firebase ProjectManagement service interface.\n */\nexport class ProjectManagement {\n\n  private readonly requestHandler: ProjectManagementRequestHandler;\n  private projectId: string;\n\n  /**\n   * @param app - The app for this ProjectManagement service.\n   * @constructor\n   * @internal\n   */\n  constructor(readonly app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument',\n        'First argument passed to admin.projectManagement() must be a valid Firebase app '\n              + 'instance.');\n    }\n\n    this.requestHandler = new ProjectManagementRequestHandler(app);\n  }\n\n  /**\n   * Lists up to 100 Firebase Android apps associated with this Firebase project.\n   *\n   * @returns The list of Android apps.\n   */\n  public listAndroidApps(): Promise<AndroidApp[]> {\n    return this.listPlatformApps<AndroidApp>('android', 'listAndroidApps()');\n  }\n\n  /**\n   * Lists up to 100 Firebase iOS apps associated with this Firebase project.\n   *\n   * @returns The list of iOS apps.\n   */\n  public listIosApps(): Promise<IosApp[]> {\n    return this.listPlatformApps<IosApp>('ios', 'listIosApps()');\n  }\n\n  /**\n   * Creates an `AndroidApp` object, referencing the specified Android app within\n   * this Firebase project.\n   *\n   * This method does not perform an RPC.\n   *\n   * @param appId - The `appId` of the Android app to reference.\n   *\n   * @returns An `AndroidApp` object that references the specified Firebase Android app.\n   */\n  public androidApp(appId: string): AndroidApp {\n    return new AndroidApp(appId, this.requestHandler);\n  }\n\n  /**\n   * Creates an `iOSApp` object, referencing the specified iOS app within\n   * this Firebase project.\n   *\n   * This method does not perform an RPC.\n   *\n   * @param appId - The `appId` of the iOS app to reference.\n   *\n   * @returns An `iOSApp` object that references the specified Firebase iOS app.\n   */\n  public iosApp(appId: string): IosApp {\n    return new IosApp(appId, this.requestHandler);\n  }\n\n  /**\n   * Creates a `ShaCertificate` object.\n   *\n   * This method does not perform an RPC.\n   *\n   * @param shaHash - The SHA-1 or SHA-256 hash for this certificate.\n   *\n   * @returns A `ShaCertificate` object contains the specified SHA hash.\n   */\n  public shaCertificate(shaHash: string): ShaCertificate {\n    return new ShaCertificate(shaHash);\n  }\n\n  /**\n   * Creates a new Firebase Android app associated with this Firebase project.\n   *\n   * @param packageName - The canonical package name of the Android App,\n   *     as would appear in the Google Play Developer Console.\n   * @param displayName - An optional user-assigned display name for this\n   *     new app.\n   *\n   * @returns A promise that resolves to the newly created Android app.\n   */\n  public createAndroidApp(packageName: string, displayName?: string): Promise<AndroidApp> {\n    return this.getResourceName()\n      .then((resourceName) => {\n        return this.requestHandler.createAndroidApp(resourceName, packageName, displayName);\n      })\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createAndroidApp()\\'s responseData must be a non-null object.');\n\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.appId),\n          responseData,\n          '\"responseData.appId\" field must be present in createAndroidApp()\\'s response data.');\n        return new AndroidApp(responseData.appId, this.requestHandler);\n      });\n  }\n\n  /**\n   * Creates a new Firebase iOS app associated with this Firebase project.\n   *\n   * @param bundleId - The iOS app bundle ID to use for this new app.\n   * @param displayName - An optional user-assigned display name for this\n   *     new app.\n   *\n   * @returns A promise that resolves to the newly created iOS app.\n   */\n  public createIosApp(bundleId: string, displayName?: string): Promise<IosApp> {\n    return this.getResourceName()\n      .then((resourceName) => {\n        return this.requestHandler.createIosApp(resourceName, bundleId, displayName);\n      })\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createIosApp()\\'s responseData must be a non-null object.');\n\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.appId),\n          responseData,\n          '\"responseData.appId\" field must be present in createIosApp()\\'s response data.');\n        return new IosApp(responseData.appId, this.requestHandler);\n      });\n  }\n\n  /**\n   * Lists up to 100 Firebase apps associated with this Firebase project.\n   *\n   * @returns A promise that resolves to the metadata list of the apps.\n   */\n  public listAppMetadata(): Promise<AppMetadata[]> {\n    return this.getResourceName()\n      .then((resourceName) => {\n        return this.requestHandler.listAppMetadata(resourceName);\n      })\n      .then((responseData) => {\n        return this.getProjectId()\n          .then((projectId) => {\n            return this.transformResponseToAppMetadata(responseData, projectId);\n          });\n      });\n  }\n\n  /**\n   * Update the display name of this Firebase project.\n   *\n   * @param newDisplayName - The new display name to be updated.\n   *\n   * @returns A promise that resolves when the project display name has been updated.\n   */\n  public setDisplayName(newDisplayName: string): Promise<void> {\n    return this.getResourceName()\n      .then((resourceName) => {\n        return this.requestHandler.setDisplayName(resourceName, newDisplayName);\n      });\n  }\n\n  private transformResponseToAppMetadata(responseData: any, projectId: string): AppMetadata[] {\n    this.assertListAppsResponseData(responseData, 'listAppMetadata()');\n\n    if (!responseData.apps) {\n      return [];\n    }\n\n    return responseData.apps.map((appJson: any) => {\n      assertServerResponse(\n        validator.isNonEmptyString(appJson.appId),\n        responseData,\n        '\"apps[].appId\" field must be present in the listAppMetadata() response data.');\n      assertServerResponse(\n        validator.isNonEmptyString(appJson.platform),\n        responseData,\n        '\"apps[].platform\" field must be present in the listAppMetadata() response data.');\n      const metadata: AppMetadata = {\n        appId: appJson.appId,\n        platform: (AppPlatform as any)[appJson.platform] || AppPlatform.PLATFORM_UNKNOWN,\n        projectId,\n        resourceName: appJson.name,\n      };\n      if (appJson.displayName) {\n        metadata.displayName = appJson.displayName;\n      }\n      return metadata;\n    });\n  }\n\n  private getResourceName(): Promise<string> {\n    return this.getProjectId()\n      .then((projectId) => {\n        return `projects/${projectId}`;\n      });\n  }\n\n  private getProjectId(): Promise<string> {\n    if (this.projectId) {\n      return Promise.resolve(this.projectId);\n    }\n\n    return utils.findProjectId(this.app)\n      .then((projectId) => {\n        // Assert that a specific project ID was provided within the app.\n        if (!validator.isNonEmptyString(projectId)) {\n          throw new FirebaseProjectManagementError(\n            'invalid-project-id',\n            'Failed to determine project ID. Initialize the SDK with service account credentials, or '\n                + 'set project ID as an app option. Alternatively, set the GOOGLE_CLOUD_PROJECT '\n                + 'environment variable.');\n        }\n\n        this.projectId = projectId;\n        return this.projectId;\n      });\n  }\n\n  /**\n   * Lists up to 100 Firebase apps for a specified platform, associated with this Firebase project.\n   */\n  private listPlatformApps<T>(platform: 'android' | 'ios', callerName: string): Promise<T[]> {\n    return this.getResourceName()\n      .then((resourceName) => {\n        return (platform === 'android') ?\n          this.requestHandler.listAndroidApps(resourceName)\n          : this.requestHandler.listIosApps(resourceName);\n      })\n      .then((responseData: any) => {\n        this.assertListAppsResponseData(responseData, callerName);\n\n        if (!responseData.apps) {\n          return [];\n        }\n\n        return responseData.apps.map((appJson: any) => {\n          assertServerResponse(\n            validator.isNonEmptyString(appJson.appId),\n            responseData,\n            `\"apps[].appId\" field must be present in the ${callerName} response data.`);\n          if (platform === 'android') {\n            return new AndroidApp(appJson.appId, this.requestHandler);\n          } else {\n            return new IosApp(appJson.appId, this.requestHandler);\n          }\n        });\n      });\n  }\n\n  private assertListAppsResponseData(responseData: any, callerName: string): void {\n    assertServerResponse(\n      validator.isNonNullObject(responseData),\n      responseData,\n      `${callerName}'s responseData must be a non-null object.`);\n\n    if (responseData.apps) {\n      assertServerResponse(\n        validator.isArray(responseData.apps),\n        responseData,\n        `\"apps\" field must be present in the ${callerName} response data.`);\n    }\n  }\n}\n\n// Path: src/firestore/firestore-namespace.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport * as _firestore from '@google-cloud/firestore';\nimport { App } from '../app';\n\nexport declare function firestore(app?: App): _firestore.Firestore;\n\n/* eslint-disable @typescript-eslint/no-namespace */\nexport namespace firestore {\n  /* eslint-disable @typescript-eslint/no-unused-vars */\n  // See https://github.com/typescript-eslint/typescript-eslint/issues/363\n  export import v1beta1 = _firestore.v1beta1;\n  export import v1 = _firestore.v1;\n\n  export import AggregateField = _firestore.AggregateField;\n  export import AggregateFieldType = _firestore.AggregateFieldType;\n  export import AggregateQuery = _firestore.AggregateQuery;\n  export import AggregateQuerySnapshot = _firestore.AggregateQuerySnapshot;\n  export import AggregateSpecData = _firestore.AggregateSpecData;\n  export import AggregateSpec = _firestore.AggregateSpec;\n  export import AggregateType = _firestore.AggregateType;\n  export import BulkWriter = _firestore.BulkWriter;\n  export import BulkWriterOptions = _firestore.BulkWriterOptions;\n  export import BundleBuilder = _firestore.BundleBuilder;\n  export import CollectionGroup = _firestore.CollectionGroup;\n  export import CollectionReference = _firestore.CollectionReference;\n  export import DocumentChange = _firestore.DocumentChange;\n  export import DocumentChangeType = _firestore.DocumentChangeType;\n  export import DocumentData = _firestore.DocumentData;\n  export import DocumentReference = _firestore.DocumentReference;\n  export import DocumentSnapshot = _firestore.DocumentSnapshot;\n...\n", "needle_token_start": 703, "needle_token_end": 910, "code_context_ntokens": 15925, "output": ["```typescript\nfunction initFirestore(app: App, databaseId: string, firestoreSettings?: FirestoreSettings): Firestore {\n  const options = getFirestoreOptions(app, firestoreSettings);\n  options.databaseId = databaseId;\n  let firestoreDatabase: typeof Firestore;\n  try {\n    // Lazy-load the Firestore implementation here, which in turns loads gRPC.\n    firestoreDatabase = require('@google-cloud/firestore').Firestore;\n  } catch (err) {\n    throw new FirebaseFirestoreError({\n      code: 'missing-dependencies',\n      message: 'Failed to import the Cloud Firestore client library for Node.js. '\n          + 'Make sure to install the \"@google-cloud/firestore\" npm package. '\n          + `Original error: ${err}`,\n    });\n  }\n\n  return new firestoreDatabase(options);\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "createIdTokenVerifier", "language": "typescript", "path": "src/auth/token-verifier.ts", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to initialize and return an object responsible for verifying Firebase ID tokens to ensure they are valid and correctly signed.\n2. **Input**: The function takes a single parameter, which is an instance of a Firebase app.\n3. **Output**: The function returns an object capable of performing ID token verification tasks.\n4. **Procedure**: The function creates a new verifier object by providing it with a URL to the client certificate, an issuer URL, token information specific to ID tokens, and the Firebase app instance. This verifier object is then configured to validate the authenticity and integrity of Firebase ID tokens used in client applications.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "      .then((projectId) => {\n        if (typeof audience === 'undefined') {\n          audience = `${projectId}.cloudfunctions.net/`;\n        }\n        return this.decodeAndVerify(jwtToken, projectId, isEmulator, audience);\n      })\n      .then((decoded) => {\n        const decodedAuthBlockingToken = decoded.payload as DecodedAuthBlockingToken;\n        decodedAuthBlockingToken.uid = decodedAuthBlockingToken.sub;\n        return decodedAuthBlockingToken;\n      });\n  }\n\n  private ensureProjectId(): Promise<string> {\n    return util.findProjectId(this.app)\n      .then((projectId) => {\n        if (!validator.isNonEmptyString(projectId)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_CREDENTIAL,\n            'Must initialize app with a cert credential or set your Firebase project ID as the ' +\n            `GOOGLE_CLOUD_PROJECT environment variable to call ${this.tokenInfo.verifyApiName}.`,\n          );\n        }\n        return Promise.resolve(projectId);\n      })\n  }\n\n  private decodeAndVerify(\n    token: string,\n    projectId: string,\n    isEmulator: boolean,\n    audience?: string): Promise<DecodedToken> {\n    return this.safeDecode(token)\n      .then((decodedToken) => {\n        this.verifyContent(decodedToken, projectId, isEmulator, audience);\n        return this.verifySignature(token, isEmulator)\n          .then(() => decodedToken);\n      });\n  }\n\n  private safeDecode(jwtToken: string): Promise<DecodedToken> {\n    return decodeJwt(jwtToken)\n      .catch((err: JwtError) => {\n        if (err.code === JwtErrorCode.INVALID_ARGUMENT) {\n          const verifyJwtTokenDocsMessage = ` See ${this.tokenInfo.url} ` +\n            `for details on how to retrieve ${this.shortNameArticle} ${this.tokenInfo.shortName}.`;\n          const errorMessage = `Decoding ${this.tokenInfo.jwtName} failed. Make sure you passed ` +\n            `the entire string JWT which represents ${this.shortNameArticle} ` +\n            `${this.tokenInfo.shortName}.` + verifyJwtTokenDocsMessage;\n          throw new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT,\n            errorMessage);\n        }\n        throw new FirebaseAuthError(AuthClientErrorCode.INTERNAL_ERROR, err.message);\n      });\n  }\n\n  /**\n   * Verifies the content of a Firebase Auth JWT.\n   *\n   * @param fullDecodedToken - The decoded JWT.\n   * @param projectId - The Firebase Project Id.\n   * @param isEmulator - Whether the token is an Emulator token.\n   */\n  private verifyContent(\n    fullDecodedToken: DecodedToken,\n    projectId: string | null,\n    isEmulator: boolean,\n    audience: string | undefined): void {\n    const header = fullDecodedToken && fullDecodedToken.header;\n    const payload = fullDecodedToken && fullDecodedToken.payload;\n\n    const projectIdMatchMessage = ` Make sure the ${this.tokenInfo.shortName} comes from the same ` +\n      'Firebase project as the service account used to authenticate this SDK.';\n    const verifyJwtTokenDocsMessage = ` See ${this.tokenInfo.url} ` +\n      `for details on how to retrieve ${this.shortNameArticle} ${this.tokenInfo.shortName}.`;\n\n    let errorMessage: string | undefined;\n    if (!isEmulator && typeof header.kid === 'undefined') {\n      const isCustomToken = (payload.aud === FIREBASE_AUDIENCE);\n      const isLegacyCustomToken = (header.alg === 'HS256' && payload.v === 0 && 'd' in payload && 'uid' in payload.d);\n\n      if (isCustomToken) {\n        errorMessage = `${this.tokenInfo.verifyApiName} expects ${this.shortNameArticle} ` +\n          `${this.tokenInfo.shortName}, but was given a custom token.`;\n      } else if (isLegacyCustomToken) {\n        errorMessage = `${this.tokenInfo.verifyApiName} expects ${this.shortNameArticle} ` +\n          `${this.tokenInfo.shortName}, but was given a legacy custom token.`;\n      } else {\n        errorMessage = `${this.tokenInfo.jwtName} has no \"kid\" claim.`;\n      }\n\n      errorMessage += verifyJwtTokenDocsMessage;\n    } else if (!isEmulator && header.alg !== ALGORITHM_RS256) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect algorithm. Expected \"` + ALGORITHM_RS256 + '\" but got ' +\n        '\"' + header.alg + '\".' + verifyJwtTokenDocsMessage;\n    } else if (typeof audience !== 'undefined' && !(payload.aud as string).includes(audience)) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect \"aud\" (audience) claim. Expected \"` +\n      audience + '\" but got \"' + payload.aud + '\".' + verifyJwtTokenDocsMessage;\n    } else if (typeof audience === 'undefined' && payload.aud !== projectId) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect \"aud\" (audience) claim. Expected \"` +\n      projectId + '\" but got \"' + payload.aud + '\".' + projectIdMatchMessage +\n        verifyJwtTokenDocsMessage;\n    } else if (payload.iss !== this.issuer + projectId) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect \"iss\" (issuer) claim. Expected ` +\n        `\"${this.issuer}` + projectId + '\" but got \"' +\n        payload.iss + '\".' + projectIdMatchMessage + verifyJwtTokenDocsMessage;\n    } else if (!(payload.event_type !== undefined &&\n      (payload.event_type === 'beforeSendSms' || payload.event_type === 'beforeSendEmail'))) {\n      // excluding `beforeSendSms` and `beforeSendEmail` from processing `sub` as there is no user record available.\n      // `sub` is the same as `uid` which is part of the user record.\n      if (typeof payload.sub !== 'string') {\n        errorMessage = `${this.tokenInfo.jwtName} has no \"sub\" (subject) claim.` + verifyJwtTokenDocsMessage;\n      } else if (payload.sub === '') {\n        errorMessage = `${this.tokenInfo.jwtName} has an empty \"sub\" (subject) claim.` +\n          verifyJwtTokenDocsMessage;\n      } else if (payload.sub.length > 128) {\n        errorMessage = `${this.tokenInfo.jwtName} has a \"sub\" (subject) claim longer than 128 characters.` +\n          verifyJwtTokenDocsMessage;\n      }\n    }\n    if (errorMessage) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, errorMessage);\n    }\n  }\n\n  private verifySignature(jwtToken: string, isEmulator: boolean):\n    Promise<void> {\n    const verifier = isEmulator ? EMULATOR_VERIFIER : this.signatureVerifier;\n    return verifier.verify(jwtToken)\n      .catch((error) => {\n        throw this.mapJwtErrorToAuthError(error);\n      });\n  }\n\n  /**\n   * Maps JwtError to FirebaseAuthError\n   *\n   * @param error - JwtError to be mapped.\n   * @returns FirebaseAuthError or Error instance.\n   */\n  private mapJwtErrorToAuthError(error: JwtError): Error {\n    const verifyJwtTokenDocsMessage = ` See ${this.tokenInfo.url} ` +\n      `for details on how to retrieve ${this.shortNameArticle} ${this.tokenInfo.shortName}.`;\n    if (error.code === JwtErrorCode.TOKEN_EXPIRED) {\n      const errorMessage = `${this.tokenInfo.jwtName} has expired. Get a fresh ${this.tokenInfo.shortName}` +\n        ` from your client app and try again (auth/${this.tokenInfo.expiredErrorCode.code}).` +\n        verifyJwtTokenDocsMessage;\n      return new FirebaseAuthError(this.tokenInfo.expiredErrorCode, errorMessage);\n    } else if (error.code === JwtErrorCode.INVALID_SIGNATURE) {\n      const errorMessage = `${this.tokenInfo.jwtName} has invalid signature.` + verifyJwtTokenDocsMessage;\n      return new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, errorMessage);\n    } else if (error.code === JwtErrorCode.NO_MATCHING_KID) {\n      const errorMessage = `${this.tokenInfo.jwtName} has \"kid\" claim which does not ` +\n        `correspond to a known public key. Most likely the ${this.tokenInfo.shortName} ` +\n        'is expired, so get a fresh token from your client app and try again.';\n      return new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, errorMessage);\n    }\n    return new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, error.message);\n  }\n}\n\n/**\n * Creates a new FirebaseTokenVerifier to verify Firebase ID tokens.\n *\n * @internal\n * @param app - Firebase app instance.\n * @returns FirebaseTokenVerifier\n */\nexport \nfunction createIdTokenVerifier(app: App): FirebaseTokenVerifier {\n  return new FirebaseTokenVerifier(\n    CLIENT_CERT_URL,\n    'https://securetoken.google.com/',\n    ID_TOKEN_INFO,\n    app\n  );\n}\n\n/**\n * Creates a new FirebaseTokenVerifier to verify Firebase Auth Blocking tokens.\n *\n * @internal\n * @param app - Firebase app instance.\n * @returns FirebaseTokenVerifier\n */\nexport function createAuthBlockingTokenVerifier(app: App): FirebaseTokenVerifier {\n  return new FirebaseTokenVerifier(\n    CLIENT_CERT_URL,\n    'https://securetoken.google.com/',\n    AUTH_BLOCKING_TOKEN_INFO,\n    app\n  );\n}\n\n/**\n * Creates a new FirebaseTokenVerifier to verify Firebase session cookies.\n *\n * @internal\n * @param app - Firebase app instance.\n * @returns FirebaseTokenVerifier\n */\nexport function createSessionCookieVerifier(app: App): FirebaseTokenVerifier {\n  return new FirebaseTokenVerifier(\n    SESSION_COOKIE_CERT_URL,\n    'https://session.firebase.google.com/',\n    SESSION_COOKIE_INFO,\n    app\n  );\n}\n\n// Path: src/auth/token-generator.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { AuthClientErrorCode, ErrorInfo, FirebaseAuthError } from '../utils/error';\nimport { HttpError } from '../utils/api-request';\nimport { CryptoSigner, CryptoSignerError, CryptoSignerErrorCode } from '../utils/crypto-signer';\n\nimport * as validator from '../utils/validator';\nimport { toWebSafeBase64 } from '../utils';\nimport { Algorithm } from 'jsonwebtoken';\n\nconst ALGORITHM_NONE: Algorithm = 'none' as const;\n\nconst ONE_HOUR_IN_SECONDS = 60 * 60;\n\n// List of blacklisted claims which cannot be provided when creating a custom token\nexport const BLACKLISTED_CLAIMS = [\n  'acr', 'amr', 'at_hash', 'aud', 'auth_time', 'azp', 'cnf', 'c_hash', 'exp', 'iat', 'iss', 'jti',\n  'nbf', 'nonce',\n];\n\n// Audience to use for Firebase Auth Custom tokens\nconst FIREBASE_AUDIENCE = 'https://identitytoolkit.googleapis.com/google.identity.identitytoolkit.v1.IdentityToolkit';\n\n/**\n * Represents the header of a JWT.\n */\ninterface JWTHeader {\n  alg: string;\n  typ: string;\n}\n\n/**\n * Represents the body of a JWT.\n */\ninterface JWTBody {\n  claims?: object;\n  uid: string;\n  aud: string;\n  iat: number;\n  exp: number;\n  iss: string;\n  sub: string;\n  tenant_id?: string;\n}\n\n/**\n * A CryptoSigner implementation that is used when communicating with the Auth emulator.\n * It produces unsigned tokens.\n */\nexport class EmulatedSigner implements CryptoSigner {\n\n  algorithm = ALGORITHM_NONE;\n\n  /**\n   * @inheritDoc\n   */\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  public sign(buffer: Buffer): Promise<Buffer> {\n    return Promise.resolve(Buffer.from(''));\n  }\n\n  /**\n   * @inheritDoc\n   */\n  public getAccountId(): Promise<string> {\n    return Promise.resolve('firebase-auth-emulator@example.com');\n  }\n}\n\n/**\n * Class for generating different types of Firebase Auth tokens (JWTs).\n *\n * @internal\n */\nexport class FirebaseTokenGenerator {\n\n  private readonly signer: CryptoSigner;\n\n  /**\n   * @param tenantId - The tenant ID to use for the generated Firebase Auth\n   *     Custom token. If absent, then no tenant ID claim will be set in the\n   *     resulting JWT.\n   */\n  constructor(signer: CryptoSigner, public readonly tenantId?: string) {\n    if (!validator.isNonNullObject(signer)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CREDENTIAL,\n        'INTERNAL ASSERT: Must provide a CryptoSigner to use FirebaseTokenGenerator.',\n      );\n    }\n    if (typeof this.tenantId !== 'undefined' && !validator.isNonEmptyString(this.tenantId)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '`tenantId` argument must be a non-empty string.');\n    }\n    this.signer = signer;\n  }\n\n  /**\n   * Creates a new Firebase Auth Custom token.\n   *\n   * @param uid - The user ID to use for the generated Firebase Auth Custom token.\n   * @param developerClaims - Optional developer claims to include in the generated Firebase\n   *     Auth Custom token.\n   * @returns A Promise fulfilled with a Firebase Auth Custom token signed with a\n   *     service account key and containing the provided payload.\n   */\n  public createCustomToken(uid: string, developerClaims?: {[key: string]: any}): Promise<string> {\n    let errorMessage: string | undefined;\n    if (!validator.isNonEmptyString(uid)) {\n      errorMessage = '`uid` argument must be a non-empty string uid.';\n    } else if (uid.length > 128) {\n      errorMessage = '`uid` argument must a uid with less than or equal to 128 characters.';\n    } else if (!this.isDeveloperClaimsValid_(developerClaims)) {\n      errorMessage = '`developerClaims` argument must be a valid, non-null object containing the developer claims.';\n    }\n\n    if (errorMessage) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, errorMessage);\n    }\n\n    const claims: {[key: string]: any} = {};\n    if (typeof developerClaims !== 'undefined') {\n      for (const key in developerClaims) {\n        /* istanbul ignore else */\n        if (Object.prototype.hasOwnProperty.call(developerClaims, key)) {\n          if (BLACKLISTED_CLAIMS.indexOf(key) !== -1) {\n            throw new FirebaseAuthError(\n              AuthClientErrorCode.INVALID_ARGUMENT,\n              `Developer claim \"${key}\" is reserved and cannot be specified.`,\n            );\n          }\n          claims[key] = developerClaims[key];\n        }\n      }\n    }\n    return this.signer.getAccountId().then((account) => {\n      const header: JWTHeader = {\n        alg: this.signer.algorithm,\n        typ: 'JWT',\n      };\n      const iat = Math.floor(Date.now() / 1000);\n      const body: JWTBody = {\n        aud: FIREBASE_AUDIENCE,\n        iat,\n        exp: iat + ONE_HOUR_IN_SECONDS,\n        iss: account,\n        sub: account,\n        uid,\n      };\n      if (this.tenantId) {\n        body.tenant_id = this.tenantId;\n      }\n      if (Object.keys(claims).length > 0) {\n        body.claims = claims;\n      }\n      const token = `${this.encodeSegment(header)}.${this.encodeSegment(body)}`;\n      const signPromise = this.signer.sign(Buffer.from(token));\n\n      return Promise.all([token, signPromise]);\n    }).then(([token, signature]) => {\n      return `${token}.${this.encodeSegment(signature)}`;\n    }).catch((err) => {\n      throw handleCryptoSignerError(err);\n    });\n  }\n\n  private encodeSegment(segment: object | Buffer): string {\n    const buffer: Buffer = (segment instanceof Buffer) ? segment : Buffer.from(JSON.stringify(segment));\n    return toWebSafeBase64(buffer).replace(/=+$/, '');\n  }\n\n  /**\n   * Returns whether or not the provided developer claims are valid.\n   *\n   * @param developerClaims - Optional developer claims to validate.\n   * @returns True if the provided claims are valid; otherwise, false.\n   */\n  // eslint-disable-next-line @typescript-eslint/naming-convention\n  private isDeveloperClaimsValid_(developerClaims?: object): boolean {\n    if (typeof developerClaims === 'undefined') {\n      return true;\n    }\n    return validator.isNonNullObject(developerClaims);\n  }\n}\n\n/**\n * Creates a new FirebaseAuthError by extracting the error code, message and other relevant\n * details from a CryptoSignerError.\n *\n * @param err - The Error to convert into a FirebaseAuthError error\n * @returns A Firebase Auth error that can be returned to the user.\n */\nexport function handleCryptoSignerError(err: Error): Error {\n  if (!(err instanceof CryptoSignerError)) {\n    return err;\n  }\n  if (err.code === CryptoSignerErrorCode.SERVER_ERROR && validator.isNonNullObject(err.cause)) {\n    const httpError = err.cause;\n    const errorResponse = (httpError as HttpError).response.data;\n    if (validator.isNonNullObject(errorResponse) && errorResponse.error) {\n      const errorCode = errorResponse.error.status;\n      const description = 'Please refer to https://firebase.google.com/docs/auth/admin/create-custom-tokens ' +\n        'for more details on how to use and troubleshoot this feature.';\n      const errorMsg = `${errorResponse.error.message}; ${description}`;\n\n      return FirebaseAuthError.fromServerError(errorCode, errorMsg, errorResponse);\n    }\n    return new FirebaseAuthError(AuthClientErrorCode.INTERNAL_ERROR,\n      'Error returned from server: ' + errorResponse + '. Additionally, an ' +\n      'internal error occurred while attempting to extract the ' +\n      'errorcode from the error.'\n    );\n  }\n  return new FirebaseAuthError(mapToAuthClientErrorCode(err.code), err.message);\n}\n\nfunction mapToAuthClientErrorCode(code: string): ErrorInfo {\n  switch (code) {\n  case CryptoSignerErrorCode.INVALID_CREDENTIAL:\n    return AuthClientErrorCode.INVALID_CREDENTIAL;\n  case CryptoSignerErrorCode.INVALID_ARGUMENT:\n    return AuthClientErrorCode.INVALID_ARGUMENT;\n  default:\n    return AuthClientErrorCode.INTERNAL_ERROR;\n  }\n}\n\n// Path: src/auth/user-record.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { deepCopy } from '../utils/deep-copy';\nimport { isNonNullObject } from '../utils/validator';\nimport * as utils from '../utils';\nimport { AuthClientErrorCode, FirebaseAuthError } from '../utils/error';\n\n/**\n * 'REDACTED', encoded as a base64 string.\n */\nconst B64_REDACTED = Buffer.from('REDACTED').toString('base64');\n\n/**\n * Parses a time stamp string or number and returns the corresponding date if valid.\n *\n * @param time - The unix timestamp string or number in milliseconds.\n * @returns The corresponding date as a UTC string, if valid. Otherwise, null.\n */\nfunction parseDate(time: any): string | null {\n  try {\n    const date = new Date(parseInt(time, 10));\n    if (!isNaN(date.getTime())) {\n      return date.toUTCString();\n    }\n  } catch (e) {\n    // Do nothing. null will be returned.\n  }\n  return null;\n}\n\nexport interface MultiFactorInfoResponse {\n  mfaEnrollmentId: string;\n  displayName?: string;\n  phoneInfo?: string;\n  totpInfo?: TotpInfoResponse;\n  enrolledAt?: string;\n  [key: string]: unknown;\n}\n\nexport interface TotpInfoResponse {\n  [key: string]: unknown;\n}\n\nexport interface ProviderUserInfoResponse {\n  rawId: string;\n  displayName?: string;\n  email?: string;\n  photoUrl?: string;\n  phoneNumber?: string;\n  providerId: string;\n  federatedId?: string;\n}\n\nexport interface GetAccountInfoUserResponse {\n  localId: string;\n  email?: string;\n  emailVerified?: boolean;\n  phoneNumber?: string;\n  displayName?: string;\n  photoUrl?: string;\n  disabled?: boolean;\n  passwordHash?: string;\n  salt?: string;\n  customAttributes?: string;\n  validSince?: string;\n  tenantId?: string;\n  providerUserInfo?: ProviderUserInfoResponse[];\n  mfaInfo?: MultiFactorInfoResponse[];\n  createdAt?: string;\n  lastLoginAt?: string;\n  lastRefreshAt?: string;\n  [key: string]: any;\n}\n\nenum MultiFactorId {\n  Phone = 'phone',\n  Totp = 'totp',\n}\n\n/**\n * Interface representing the common properties of a user-enrolled second factor.\n */\nexport abstract class MultiFactorInfo {\n\n  /**\n   * The ID of the enrolled second factor. This ID is unique to the user.\n   */\n  public readonly uid: string;\n\n  /**\n   * The optional display name of the enrolled second factor.\n   */\n  public readonly displayName?: string;\n\n  /**\n   * The type identifier of the second factor.\n   * For SMS second factors, this is `phone`.\n   * For TOTP second factors, this is `totp`.\n   */\n  public readonly factorId: string;\n\n  /**\n   * The optional date the second factor was enrolled, formatted as a UTC string.\n   */\n  public readonly enrollmentTime?: string;\n\n  /**\n   * Initializes the MultiFactorInfo associated subclass using the server side.\n   * If no MultiFactorInfo is associated with the response, null is returned.\n   *\n   * @param response - The server side response.\n   * @internal\n   */\n  public static initMultiFactorInfo(response: MultiFactorInfoResponse): MultiFactorInfo | null {\n    let multiFactorInfo: MultiFactorInfo | null = null;\n    // PhoneMultiFactorInfo, TotpMultiFactorInfo currently available.\n    try {\n      if (response.phoneInfo !== undefined) {\n        multiFactorInfo = new PhoneMultiFactorInfo(response);\n      } else if (response.totpInfo !== undefined) {\n        multiFactorInfo = new TotpMultiFactorInfo(response);\n      } else {\n        // Ignore the other SDK unsupported MFA factors to prevent blocking developers using the current SDK.\n      }\n    } catch (e) {\n      // Ignore error.\n    }\n    return multiFactorInfo;\n  }\n\n  /**\n   * Initializes the MultiFactorInfo object using the server side response.\n   *\n   * @param response - The server side response.\n   * @constructor\n   * @internal\n   */\n  constructor(response: MultiFactorInfoResponse) {\n    this.initFromServerResponse(response);\n  }\n\n  /**\n   * Returns a JSON-serializable representation of this object.\n   *\n   * @returns A JSON-serializable representation of this object.\n   */\n  public toJSON(): object {\n    return {\n      uid: this.uid,\n      displayName: this.displayName,\n      factorId: this.factorId,\n      enrollmentTime: this.enrollmentTime,\n    };\n  }\n\n  /**\n   * Returns the factor ID based on the response provided.\n   *\n   * @param response - The server side response.\n   * @returns The multi-factor ID associated with the provided response. If the response is\n   *     not associated with any known multi-factor ID, null is returned.\n   *\n   * @internal\n   */\n  protected abstract getFactorId(response: MultiFactorInfoResponse): string | null;\n\n  /**\n   * Initializes the MultiFactorInfo object using the provided server response.\n   *\n   * @param response - The server side response.\n   */\n  private initFromServerResponse(response: MultiFactorInfoResponse): void {\n    const factorId = response && this.getFactorId(response);\n    if (!factorId || !response || !response.mfaEnrollmentId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid multi-factor info response');\n    }\n    utils.addReadonlyGetter(this, 'uid', response.mfaEnrollmentId);\n    utils.addReadonlyGetter(this, 'factorId', factorId);\n    utils.addReadonlyGetter(this, 'displayName', response.displayName);\n    // Encoded using [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format.\n    // For example, \"2017-01-15T01:30:15.01Z\".\n    // This can be parsed directly via Date constructor.\n    // This can be computed using Data.prototype.toISOString.\n    if (response.enrolledAt) {\n      utils.addReadonlyGetter(\n        this, 'enrollmentTime', new Date(response.enrolledAt).toUTCString());\n    } else {\n      utils.addReadonlyGetter(this, 'enrollmentTime', null);\n    }\n  }\n}\n\n/**\n * Interface representing a phone specific user-enrolled second factor.\n */\nexport class PhoneMultiFactorInfo extends MultiFactorInfo {\n\n  /**\n   * The phone number associated with a phone second factor.\n   */\n  public readonly phoneNumber: string;\n\n  /**\n   * Initializes the PhoneMultiFactorInfo object using the server side response.\n   *\n   * @param response - The server side response.\n   * @constructor\n   * @internal\n   */\n  constructor(response: MultiFactorInfoResponse) {\n    super(response);\n    utils.addReadonlyGetter(this, 'phoneNumber', response.phoneInfo);\n  }\n\n  /**\n   * {@inheritdoc MultiFactorInfo.toJSON}\n   */\n  public toJSON(): object {\n    return Object.assign(\n      super.toJSON(),\n      {\n        phoneNumber: this.phoneNumber,\n      });\n  }\n\n  /**\n   * Returns the factor ID based on the response provided.\n   *\n   * @param response - The server side response.\n   * @returns The multi-factor ID associated with the provided response. If the response is\n   *     not associated with any known multi-factor ID, null is returned.\n   *\n   * @internal\n   */\n  protected getFactorId(response: MultiFactorInfoResponse): string | null {\n    return (response && response.phoneInfo) ? MultiFactorId.Phone : null;\n  }\n}\n\n/**\n * `TotpInfo` struct associated with a second factor\n */\nexport class TotpInfo {\n\n}\n\n/**\n * Interface representing a TOTP specific user-enrolled second factor.\n */\nexport class TotpMultiFactorInfo extends MultiFactorInfo {\n\n  /**\n   * `TotpInfo` struct associated with a second factor\n   */\n  public readonly totpInfo: TotpInfo;\n\n  /**\n   * Initializes the `TotpMultiFactorInfo` object using the server side response.\n   *\n   * @param response - The server side response.\n   * @constructor\n   * @internal\n   */\n  constructor(response: MultiFactorInfoResponse) {\n    super(response);\n    utils.addReadonlyGetter(this, 'totpInfo', response.totpInfo);\n  }\n\n  /**\n   * {@inheritdoc MultiFactorInfo.toJSON}\n   */\n  public toJSON(): object {\n    return Object.assign(\n      super.toJSON(),\n      {\n        totpInfo: this.totpInfo,\n      });\n  }\n\n  /**\n   * Returns the factor ID based on the response provided.\n   *\n   * @param response - The server side response.\n   * @returns The multi-factor ID associated with the provided response. If the response is\n   *     not associated with any known multi-factor ID, `null` is returned.\n   *\n   * @internal\n   */\n  protected getFactorId(response: MultiFactorInfoResponse): string | null {\n    return (response && response.totpInfo) ? MultiFactorId.Totp : null;\n  }\n}\n\n/**\n * The multi-factor related user settings.\n */\nexport class MultiFactorSettings {\n\n  /**\n   * List of second factors enrolled with the current user.\n   * Currently only phone and TOTP second factors are supported.\n   */\n  public enrolledFactors: MultiFactorInfo[];\n\n  /**\n   * Initializes the `MultiFactor` object using the server side or JWT format response.\n   *\n   * @param response - The server side response.\n   * @constructor\n   * @internal\n   */\n  constructor(response: GetAccountInfoUserResponse) {\n    const parsedEnrolledFactors: MultiFactorInfo[] = [];\n    if (!isNonNullObject(response)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid multi-factor response');\n    } else if (response.mfaInfo) {\n      response.mfaInfo.forEach((factorResponse) => {\n        const multiFactorInfo = MultiFactorInfo.initMultiFactorInfo(factorResponse);\n        if (multiFactorInfo) {\n          parsedEnrolledFactors.push(multiFactorInfo);\n        }\n      });\n    }\n    // Make enrolled factors immutable.\n    utils.addReadonlyGetter(\n      this, 'enrolledFactors', Object.freeze(parsedEnrolledFactors));\n  }\n\n  /**\n   * Returns a JSON-serializable representation of this multi-factor object.\n   *\n   * @returns A JSON-serializable representation of this multi-factor object.\n   */\n  public toJSON(): object {\n    return {\n      enrolledFactors: this.enrolledFactors.map((info) => info.toJSON()),\n    };\n  }\n}\n\n/**\n * Represents a user's metadata.\n */\nexport class UserMetadata {\n\n  /**\n   * The date the user was created, formatted as a UTC string.\n   */\n  public readonly creationTime: string;\n\n  /**\n   * The date the user last signed in, formatted as a UTC string.\n   */\n  public readonly lastSignInTime: string;\n\n  /**\n   * The time at which the user was last active (ID token refreshed),\n   * formatted as a UTC Date string (eg 'Sat, 03 Feb 2001 04:05:06 GMT').\n   * Returns null if the user was never active.\n   */\n  public readonly lastRefreshTime?: string | null;\n\n  /**\n   * @param response - The server side response returned from the `getAccountInfo`\n   *     endpoint.\n   * @constructor\n   * @internal\n   */\n  constructor(response: GetAccountInfoUserResponse) {\n    // Creation date should always be available but due to some backend bugs there\n    // were cases in the past where users did not have creation date properly set.\n    // This included legacy Firebase migrating project users and some anonymous users.\n    // These bugs have already been addressed since then.\n    utils.addReadonlyGetter(this, 'creationTime', parseDate(response.createdAt));\n    utils.addReadonlyGetter(this, 'lastSignInTime', parseDate(response.lastLoginAt));\n    const lastRefreshAt = response.lastRefreshAt ? new Date(response.lastRefreshAt).toUTCString() : null;\n    utils.addReadonlyGetter(this, 'lastRefreshTime', lastRefreshAt);\n  }\n\n  /**\n   * Returns a JSON-serializable representation of this object.\n   *\n   * @returns A JSON-serializable representation of this object.\n   */\n  public toJSON(): object {\n    return {\n      lastSignInTime: this.lastSignInTime,\n      creationTime: this.creationTime,\n      lastRefreshTime: this.lastRefreshTime,\n    };\n  }\n}\n\n/**\n * Represents a user's info from a third-party identity provider\n * such as Google or Facebook.\n */\nexport class UserInfo {\n\n  /**\n   * The user identifier for the linked provider.\n   */\n  public readonly uid: string;\n\n  /**\n   * The display name for the linked provider.\n   */\n  public readonly displayName: string;\n\n  /**\n   * The email for the linked provider.\n   */\n  public readonly email: string;\n\n  /**\n   * The photo URL for the linked provider.\n   */\n  public readonly photoURL: string;\n\n  /**\n   * The linked provider ID (for example, \"google.com\" for the Google provider).\n   */\n  public readonly providerId: string;\n\n  /**\n   * The phone number for the linked provider.\n   */\n  public readonly phoneNumber: string;\n\n\n  /**\n   * @param response - The server side response returned from the `getAccountInfo`\n   *     endpoint.\n   * @constructor\n   * @internal\n   */\n  constructor(response: ProviderUserInfoResponse) {\n    // Provider user id and provider id are required.\n    if (!response.rawId || !response.providerId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid user info response');\n    }\n\n    utils.addReadonlyGetter(this, 'uid', response.rawId);\n    utils.addReadonlyGetter(this, 'displayName', response.displayName);\n    utils.addReadonlyGetter(this, 'email', response.email);\n    utils.addReadonlyGetter(this, 'photoURL', response.photoUrl);\n    utils.addReadonlyGetter(this, 'providerId', response.providerId);\n    utils.addReadonlyGetter(this, 'phoneNumber', response.phoneNumber);\n  }\n\n  /**\n   * Returns a JSON-serializable representation of this object.\n   *\n   * @returns A JSON-serializable representation of this object.\n   */\n  public toJSON(): object {\n    return {\n      uid: this.uid,\n      displayName: this.displayName,\n      email: this.email,\n      photoURL: this.photoURL,\n      providerId: this.providerId,\n      phoneNumber: this.phoneNumber,\n    };\n  }\n}\n\n/**\n * Represents a user.\n */\nexport class UserRecord {\n\n  /**\n   * The user's `uid`.\n   */\n  public readonly uid: string;\n\n  /**\n   * The user's primary email, if set.\n   */\n  public readonly email?: string;\n\n  /**\n   * Whether or not the user's primary email is verified.\n   */\n  public readonly emailVerified: boolean;\n\n  /**\n   * The user's display name.\n   */\n  public readonly displayName?: string;\n\n  /**\n   * The user's photo URL.\n   */\n  public readonly photoURL?: string;\n\n  /**\n   * The user's primary phone number, if set.\n   */\n  public readonly phoneNumber?: string;\n\n  /**\n   * Whether or not the user is disabled: `true` for disabled; `false` for\n   * enabled.\n   */\n  public readonly disabled: boolean;\n\n  /**\n   * Additional metadata about the user.\n   */\n  public readonly metadata: UserMetadata;\n\n  /**\n   * An array of providers (for example, Google, Facebook) linked to the user.\n   */\n  public readonly providerData: UserInfo[];\n\n  /**\n   * The user's hashed password (base64-encoded), only if Firebase Auth hashing\n   * algorithm (SCRYPT) is used. If a different hashing algorithm had been used\n   * when uploading this user, as is typical when migrating from another Auth\n   * system, this will be an empty string. If no password is set, this is\n   * null. This is only available when the user is obtained from\n   * {@link BaseAuth.listUsers}.\n   */\n  public readonly passwordHash?: string;\n\n  /**\n   * The user's password salt (base64-encoded), only if Firebase Auth hashing\n   * algorithm (SCRYPT) is used. If a different hashing algorithm had been used to\n   * upload this user, typical when migrating from another Auth system, this will\n   * be an empty string. If no password is set, this is null. This is only\n   * available when the user is obtained from {@link BaseAuth.listUsers}.\n   */\n  public readonly passwordSalt?: string;\n\n  /**\n   * The user's custom claims object if available, typically used to define\n   * user roles and propagated to an authenticated user's ID token.\n   * This is set via {@link BaseAuth.setCustomUserClaims}\n   */\n  public readonly customClaims?: {[key: string]: any};\n\n  /**\n   * The ID of the tenant the user belongs to, if available.\n   */\n  public readonly tenantId?: string | null;\n\n  /**\n   * The date the user's tokens are valid after, formatted as a UTC string.\n   * This is updated every time the user's refresh token are revoked either\n   * from the {@link BaseAuth.revokeRefreshTokens}\n   * API or from the Firebase Auth backend on big account changes (password\n   * resets, password or email updates, etc).\n   */\n  public readonly tokensValidAfterTime?: string;\n\n  /**\n   * The multi-factor related properties for the current user, if available.\n   */\n  public readonly multiFactor?: MultiFactorSettings;\n\n  /**\n   * @param response - The server side response returned from the getAccountInfo\n   *     endpoint.\n   * @constructor\n   * @internal\n   */\n  constructor(response: GetAccountInfoUserResponse) {\n    // The Firebase user id is required.\n    if (!response.localId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid user response');\n    }\n\n    utils.addReadonlyGetter(this, 'uid', response.localId);\n    utils.addReadonlyGetter(this, 'email', response.email);\n    utils.addReadonlyGetter(this, 'emailVerified', !!response.emailVerified);\n    utils.addReadonlyGetter(this, 'displayName', response.displayName);\n    utils.addReadonlyGetter(this, 'photoURL', response.photoUrl);\n    utils.addReadonlyGetter(this, 'phoneNumber', response.phoneNumber);\n    // If disabled is not provided, the account is enabled by default.\n    utils.addReadonlyGetter(this, 'disabled', response.disabled || false);\n    utils.addReadonlyGetter(this, 'metadata', new UserMetadata(response));\n    const providerData: UserInfo[] = [];\n    for (const entry of (response.providerUserInfo || [])) {\n      providerData.push(new UserInfo(entry));\n    }\n    utils.addReadonlyGetter(this, 'providerData', providerData);\n\n    // If the password hash is redacted (probably due to missing permissions)\n    // then clear it out, similar to how the salt is returned. (Otherwise, it\n    // *looks* like a b64-encoded hash is present, which is confusing.)\n    if (response.passwordHash === B64_REDACTED) {\n      utils.addReadonlyGetter(this, 'passwordHash', undefined);\n    } else {\n      utils.addReadonlyGetter(this, 'passwordHash', response.passwordHash);\n    }\n\n    utils.addReadonlyGetter(this, 'passwordSalt', response.salt);\n    if (response.customAttributes) {\n      utils.addReadonlyGetter(\n        this, 'customClaims', JSON.parse(response.customAttributes));\n    }\n\n    let validAfterTime: string | null = null;\n    // Convert validSince first to UTC milliseconds and then to UTC date string.\n    if (typeof response.validSince !== 'undefined') {\n      validAfterTime = parseDate(parseInt(response.validSince, 10) * 1000);\n    }\n    utils.addReadonlyGetter(this, 'tokensValidAfterTime', validAfterTime || undefined);\n    utils.addReadonlyGetter(this, 'tenantId', response.tenantId);\n    const multiFactor = new MultiFactorSettings(response);\n    if (multiFactor.enrolledFactors.length > 0) {\n      utils.addReadonlyGetter(this, 'multiFactor', multiFactor);\n    }\n  }\n\n  /**\n   * Returns a JSON-serializable representation of this object.\n   *\n   * @returns A JSON-serializable representation of this object.\n   */\n  public toJSON(): object {\n    const json: any = {\n      uid: this.uid,\n      email: this.email,\n      emailVerified: this.emailVerified,\n      displayName: this.displayName,\n      photoURL: this.photoURL,\n      phoneNumber: this.phoneNumber,\n      disabled: this.disabled,\n      // Convert metadata to json.\n      metadata: this.metadata.toJSON(),\n      passwordHash: this.passwordHash,\n      passwordSalt: this.passwordSalt,\n      customClaims: deepCopy(this.customClaims),\n      tokensValidAfterTime: this.tokensValidAfterTime,\n      tenantId: this.tenantId,\n    };\n    if (this.multiFactor) {\n      json.multiFactor =  this.multiFactor.toJSON();\n    }\n    json.providerData = [];\n    for (const entry of this.providerData) {\n      // Convert each provider data to json.\n      json.providerData.push(entry.toJSON());\n    }\n    return json;\n  }\n}\n\n// Path: src/auth/base-auth.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App, FirebaseArrayIndexError } from '../app';\nimport { AuthClientErrorCode, ErrorInfo, FirebaseAuthError } from '../utils/error';\nimport { deepCopy } from '../utils/deep-copy';\nimport * as validator from '../utils/validator';\n\nimport { AbstractAuthRequestHandler, useEmulator } from './auth-api-request';\nimport { FirebaseTokenGenerator, EmulatedSigner, handleCryptoSignerError } from './token-generator';\nimport {\n  FirebaseTokenVerifier,\n  createSessionCookieVerifier,\n  createIdTokenVerifier,\n  createAuthBlockingTokenVerifier,\n  DecodedIdToken,\n  DecodedAuthBlockingToken,\n} from './token-verifier';\nimport {\n  AuthProviderConfig, SAMLAuthProviderConfig, AuthProviderConfigFilter, ListProviderConfigResults,\n  SAMLConfig, OIDCConfig, OIDCConfigServerResponse, SAMLConfigServerResponse,\n  UpdateAuthProviderRequest, OIDCAuthProviderConfig, CreateRequest, UpdateRequest,\n} from './auth-config';\nimport { UserRecord } from './user-record';\nimport {\n  UserIdentifier, isUidIdentifier, isEmailIdentifier, isPhoneIdentifier, isProviderIdentifier,\n} from './identifier';\nimport { UserImportOptions, UserImportRecord, UserImportResult } from './user-import-builder';\nimport { ActionCodeSettings } from './action-code-settings-builder';\nimport { cryptoSignerFromApp } from '../utils/crypto-signer';\n\n/** Represents the result of the {@link BaseAuth.getUsers} API. */\nexport interface GetUsersResult {\n  /**\n   * Set of user records, corresponding to the set of users that were\n   * requested. Only users that were found are listed here. The result set is\n   * unordered.\n   */\n  users: UserRecord[];\n\n  /** Set of identifiers that were requested, but not found. */\n  notFound: UserIdentifier[];\n}\n\n/**\n * Interface representing the object returned from a\n * {@link BaseAuth.listUsers} operation. Contains the list\n * of users for the current batch and the next page token if available.\n */\nexport interface ListUsersResult {\n\n  /**\n   * The list of {@link UserRecord} objects for the\n   * current downloaded batch.\n   */\n  users: UserRecord[];\n\n  /**\n   * The next page token if available. This is needed for the next batch download.\n   */\n  pageToken?: string;\n}\n\n/**\n * Represents the result of the {@link BaseAuth.deleteUsers}.\n * API.\n */\nexport interface DeleteUsersResult {\n  /**\n   * The number of user records that failed to be deleted (possibly zero).\n   */\n  failureCount: number;\n\n  /**\n   * The number of users that were deleted successfully (possibly zero).\n   * Users that did not exist prior to calling `deleteUsers()` are\n   * considered to be successfully deleted.\n   */\n  successCount: number;\n\n  /**\n   * A list of `FirebaseArrayIndexError` instances describing the errors that\n   * were encountered during the deletion. Length of this list is equal to\n   * the return value of {@link DeleteUsersResult.failureCount}.\n   */\n  errors: FirebaseArrayIndexError[];\n}\n\n/**\n * Interface representing the session cookie options needed for the\n * {@link BaseAuth.createSessionCookie} method.\n */\nexport interface SessionCookieOptions {\n\n  /**\n   * The session cookie custom expiration in milliseconds. The minimum allowed is\n   * 5 minutes and the maxium allowed is 2 weeks.\n   */\n  expiresIn: number;\n}\n\n/**\n * @internal\n */\nexport function createFirebaseTokenGenerator(app: App,\n  tenantId?: string): FirebaseTokenGenerator {\n  try {\n    const signer = useEmulator() ? new EmulatedSigner() : cryptoSignerFromApp(app);\n    return new FirebaseTokenGenerator(signer, tenantId);\n  } catch (err) {\n    throw handleCryptoSignerError(err);\n  }\n}\n\n/**\n * Common parent interface for both `Auth` and `TenantAwareAuth` APIs.\n */\nexport abstract class BaseAuth {\n\n  /** @internal */\n  protected readonly tokenGenerator: FirebaseTokenGenerator;\n  /** @internal */\n  protected readonly idTokenVerifier: FirebaseTokenVerifier;\n  /** @internal */\n  protected readonly authBlockingTokenVerifier: FirebaseTokenVerifier;\n  /** @internal */\n  protected readonly sessionCookieVerifier: FirebaseTokenVerifier;\n\n  /**\n   * The BaseAuth class constructor.\n   *\n   * @param app - The FirebaseApp to associate with this Auth instance.\n   * @param authRequestHandler - The RPC request handler for this instance.\n   * @param tokenGenerator - Optional token generator. If not specified, a\n   *     (non-tenant-aware) instance will be created. Use this paramter to\n   *     specify a tenant-aware tokenGenerator.\n   * @constructor\n   * @internal\n   */\n  constructor(\n    app: App,\n    /** @internal */ protected readonly authRequestHandler: AbstractAuthRequestHandler,\n    tokenGenerator?: FirebaseTokenGenerator) {\n    if (tokenGenerator) {\n      this.tokenGenerator = tokenGenerator;\n    } else {\n      this.tokenGenerator = createFirebaseTokenGenerator(app);\n    }\n\n    this.sessionCookieVerifier = createSessionCookieVerifier(app);\n    this.idTokenVerifier = createIdTokenVerifier(app);\n    this.authBlockingTokenVerifier = createAuthBlockingTokenVerifier(app);\n  }\n\n  /**\n   * Creates a new Firebase custom token (JWT) that can be sent back to a client\n   * device to use to sign in with the client SDKs' `signInWithCustomToken()`\n   * methods. (Tenant-aware instances will also embed the tenant ID in the\n   * token.)\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/create-custom-tokens | Create Custom Tokens}\n   * for code samples and detailed documentation.\n   *\n   * @param uid - The `uid` to use as the custom token's subject.\n   * @param developerClaims - Optional additional claims to include\n   *   in the custom token's payload.\n   *\n   * @returns A promise fulfilled with a custom token for the\n   *   provided `uid` and payload.\n   */\n  public createCustomToken(uid: string, developerClaims?: object): Promise<string> {\n    return this.tokenGenerator.createCustomToken(uid, developerClaims);\n  }\n\n  /**\n   * Verifies a Firebase ID token (JWT). If the token is valid, the promise is\n   * fulfilled with the token's decoded claims; otherwise, the promise is\n   * rejected.\n   *\n   * If `checkRevoked` is set to true, first verifies whether the corresponding\n   * user is disabled. If yes, an `auth/user-disabled` error is thrown. If no,\n   * verifies if the session corresponding to the ID token was revoked. If the\n   * corresponding user's session was invalidated, an `auth/id-token-revoked`\n   * error is thrown. If not specified the check is not applied.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/verify-id-tokens | Verify ID Tokens}\n   * for code samples and detailed documentation.\n   *\n   * @param idToken - The ID token to verify.\n   * @param checkRevoked - Whether to check if the ID token was revoked.\n   *   This requires an extra request to the Firebase Auth backend to check\n   *   the `tokensValidAfterTime` time for the corresponding user.\n   *   When not specified, this additional check is not applied.\n   *\n   * @returns A promise fulfilled with the\n   *   token's decoded claims if the ID token is valid; otherwise, a rejected\n   *   promise.\n   */\n  public verifyIdToken(idToken: string, checkRevoked = false): Promise<DecodedIdToken> {\n    const isEmulator = useEmulator();\n    return this.idTokenVerifier.verifyJWT(idToken, isEmulator)\n      .then((decodedIdToken: DecodedIdToken) => {\n        // Whether to check if the token was revoked.\n        if (checkRevoked || isEmulator) {\n          return this.verifyDecodedJWTNotRevokedOrDisabled(\n            decodedIdToken,\n            AuthClientErrorCode.ID_TOKEN_REVOKED);\n        }\n        return decodedIdToken;\n      });\n  }\n\n  /**\n   * Gets the user data for the user corresponding to a given `uid`.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/manage-users#retrieve_user_data | Retrieve user data}\n   * for code samples and detailed documentation.\n   *\n   * @param uid - The `uid` corresponding to the user whose data to fetch.\n   *\n   * @returns A promise fulfilled with the user\n   *   data corresponding to the provided `uid`.\n   */\n  public getUser(uid: string): Promise<UserRecord> {\n    return this.authRequestHandler.getAccountInfoByUid(uid)\n      .then((response: any) => {\n        // Returns the user record populated with server response.\n        return new UserRecord(response.users[0]);\n      });\n  }\n\n  /**\n   * Gets the user data for the user corresponding to a given email.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/manage-users#retrieve_user_data | Retrieve user data}\n   * for code samples and detailed documentation.\n   *\n   * @param email - The email corresponding to the user whose data to\n   *   fetch.\n   *\n   * @returns A promise fulfilled with the user\n   *   data corresponding to the provided email.\n   */\n  public getUserByEmail(email: string): Promise<UserRecord> {\n    return this.authRequestHandler.getAccountInfoByEmail(email)\n      .then((response: any) => {\n        // Returns the user record populated with server response.\n        return new UserRecord(response.users[0]);\n      });\n  }\n\n  /**\n   * Gets the user data for the user corresponding to a given phone number. The\n   * phone number has to conform to the E.164 specification.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/manage-users#retrieve_user_data | Retrieve user data}\n   * for code samples and detailed documentation.\n   *\n   * @param phoneNumber - The phone number corresponding to the user whose\n   *   data to fetch.\n   *\n   * @returns A promise fulfilled with the user\n   *   data corresponding to the provided phone number.\n   */\n  public getUserByPhoneNumber(phoneNumber: string): Promise<UserRecord> {\n    return this.authRequestHandler.getAccountInfoByPhoneNumber(phoneNumber)\n      .then((response: any) => {\n        // Returns the user record populated with server response.\n        return new UserRecord(response.users[0]);\n      });\n  }\n\n  /**\n   * Gets the user data for the user corresponding to a given provider id.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/manage-users#retrieve_user_data | Retrieve user data}\n   * for code samples and detailed documentation.\n   *\n   * @param providerId - The provider ID, for example, \"google.com\" for the\n   *   Google provider.\n   * @param uid - The user identifier for the given provider.\n   *\n   * @returns A promise fulfilled with the user data corresponding to the\n   *   given provider id.\n   */\n  public getUserByProviderUid(providerId: string, uid: string): Promise<UserRecord> {\n    // Although we don't really advertise it, we want to also handle\n    // non-federated idps with this call. So if we detect one of them, we'll\n    // reroute this request appropriately.\n    if (providerId === 'phone') {\n      return this.getUserByPhoneNumber(uid);\n    } else if (providerId === 'email') {\n      return this.getUserByEmail(uid);\n    }\n\n    return this.authRequestHandler.getAccountInfoByFederatedUid(providerId, uid)\n      .then((response: any) => {\n        // Returns the user record populated with server response.\n        return new UserRecord(response.users[0]);\n      });\n  }\n\n  /**\n   * Gets the user data corresponding to the specified identifiers.\n   *\n   * There are no ordering guarantees; in particular, the nth entry in the result list is not\n   * guaranteed to correspond to the nth entry in the input parameters list.\n   *\n   * Only a maximum of 100 identifiers may be supplied. If more than 100 identifiers are supplied,\n   * this method throws a FirebaseAuthError.\n   *\n   * @param identifiers - The identifiers used to indicate which user records should be returned.\n   *     Must not have more than 100 entries.\n   * @returns A promise that resolves to the corresponding user records.\n   * @throws FirebaseAuthError If any of the identifiers are invalid or if more than 100\n   *     identifiers are specified.\n   */\n  public getUsers(identifiers: UserIdentifier[]): Promise<GetUsersResult> {\n    if (!validator.isArray(identifiers)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT, '`identifiers` parameter must be an array');\n    }\n    return this.authRequestHandler\n      .getAccountInfoByIdentifiers(identifiers)\n      .then((response: any) => {\n        /**\n         * Checks if the specified identifier is within the list of\n         * UserRecords.\n         */\n        const isUserFound = ((id: UserIdentifier, userRecords: UserRecord[]): boolean => {\n          return !!userRecords.find((userRecord) => {\n            if (isUidIdentifier(id)) {\n              return id.uid === userRecord.uid;\n            } else if (isEmailIdentifier(id)) {\n              return id.email === userRecord.email;\n            } else if (isPhoneIdentifier(id)) {\n              return id.phoneNumber === userRecord.phoneNumber;\n            } else if (isProviderIdentifier(id)) {\n              const matchingUserInfo = userRecord.providerData.find((userInfo) => {\n                return id.providerId === userInfo.providerId;\n              });\n              return !!matchingUserInfo && id.providerUid === matchingUserInfo.uid;\n            } else {\n              throw new FirebaseAuthError(\n                AuthClientErrorCode.INTERNAL_ERROR,\n                'Unhandled identifier type');\n            }\n          });\n        });\n\n        const users = response.users ? response.users.map((user: any) => new UserRecord(user)) : [];\n        const notFound = identifiers.filter((id) => !isUserFound(id, users));\n\n        return { users, notFound };\n      });\n  }\n\n  /**\n   * Retrieves a list of users (single batch only) with a size of `maxResults`\n   * starting from the offset as specified by `pageToken`. This is used to\n   * retrieve all the users of a specified project in batches.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/manage-users#list_all_users | List all users}\n   * for code samples and detailed documentation.\n   *\n   * @param maxResults - The page size, 1000 if undefined. This is also\n   *   the maximum allowed limit.\n   * @param pageToken - The next page token. If not specified, returns\n   *   users starting without any offset.\n   * @returns A promise that resolves with\n   *   the current batch of downloaded users and the next page token.\n   */\n  public listUsers(maxResults?: number, pageToken?: string): Promise<ListUsersResult> {\n    return this.authRequestHandler.downloadAccount(maxResults, pageToken)\n      .then((response: any) => {\n        // List of users to return.\n        const users: UserRecord[] = [];\n        // Convert each user response to a UserRecord.\n        response.users.forEach((userResponse: any) => {\n          users.push(new UserRecord(userResponse));\n        });\n        // Return list of user records and the next page token if available.\n        const result = {\n          users,\n          pageToken: response.nextPageToken,\n        };\n        // Delete result.pageToken if undefined.\n        if (typeof result.pageToken === 'undefined') {\n          delete result.pageToken;\n        }\n        return result;\n      });\n  }\n\n  /**\n   * Creates a new user.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/manage-users#create_a_user | Create a user}\n   * for code samples and detailed documentation.\n   *\n   * @param properties - The properties to set on the\n   *   new user record to be created.\n   *\n   * @returns A promise fulfilled with the user\n   *   data corresponding to the newly created user.\n   */\n  public createUser(properties: CreateRequest): Promise<UserRecord> {\n    return this.authRequestHandler.createNewAccount(properties)\n      .then((uid) => {\n        // Return the corresponding user record.\n        return this.getUser(uid);\n      })\n      .catch((error) => {\n        if (error.code === 'auth/user-not-found') {\n          // Something must have happened after creating the user and then retrieving it.\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INTERNAL_ERROR,\n            'Unable to create the user record provided.');\n        }\n        throw error;\n      });\n  }\n\n  /**\n   * Deletes an existing user.\n   *\n   * See {@link https://firebase.google.com/docs/auth/admin/manage-users#delete_a_user | Delete a user}\n   * for code samples and detailed documentation.\n   *\n   * @param uid - The `uid` corresponding to the user to delete.\n   *\n   * @returns An empty promise fulfilled once the user has been\n   *   deleted.\n   */\n  public deleteUser(uid: string): Promise<void> {\n    return this.authRequestHandler.deleteAccount(uid)\n      .then(() => {\n        // Return nothing on success.\n      });\n  }\n\n  /**\n   * Deletes the users specified by the given uids.\n   *\n   * Deleting a non-existing user won't generate an error (i.e. this method\n   * is idempotent.) Non-existing users are considered to be successfully\n   * deleted, and are therefore counted in the\n   * `DeleteUsersResult.successCount` value.\n   *\n   * Only a maximum of 1000 identifiers may be supplied. If more than 1000\n   * identifiers are supplied, this method throws a FirebaseAuthError.\n   *\n   * This API is currently rate limited at the server to 1 QPS. If you exceed\n...\n", "needle_token_start": 2399, "needle_token_end": 2465, "code_context_ntokens": 16107, "output": ["```typescript\n/**\n * Creates a new FirebaseTokenVerifier to verify Firebase ID tokens.\n *\n * @internal\n * @param app - Firebase app instance.\n * @returns FirebaseTokenVerifier\n */\nexport function createIdTokenVerifier(app: App): FirebaseTokenVerifier {\n  return new FirebaseTokenVerifier(\n    CLIENT_CERT_URL,\n    'https://securetoken.google.com/',\n    ID_TOKEN_INFO,\n    app\n  );\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "getFirestoreOptions", "language": "typescript", "path": "src/firestore/firestore-internal.ts", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to configure and validate settings for initializing a Firestore database connection using a Firebase app instance.\n2. **Input**: It takes a Firebase app instance and optionally specific Firestore settings.\n3. **Output**: Returns a configuration object for Firestore that includes credentials, project ID, Firebase version, and transport preferences.\n4. **Procedure**: \n   - Validates the Firebase app instance.\n   - Retrieves the project ID and credential type from the app configuration.\n   - Depending on the credential type (service account or application default), it constructs the Firestore settings object.\n   - Includes the Firebase SDK version and transport preference in the settings.\n   - Throws an error if the credentials are invalid or insufficient for Firestore initialization.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/auth/tenant-manager.ts\n/*!\n * Copyright 2019 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport * as validator from '../utils/validator';\nimport { App } from '../app';\nimport * as utils from '../utils/index';\nimport { AuthClientErrorCode, FirebaseAuthError } from '../utils/error';\n\nimport { BaseAuth, createFirebaseTokenGenerator, SessionCookieOptions } from './base-auth';\nimport { Tenant, TenantServerResponse, CreateTenantRequest, UpdateTenantRequest } from './tenant';\nimport {\n  AuthRequestHandler, TenantAwareAuthRequestHandler,\n} from './auth-api-request';\nimport { DecodedIdToken } from './token-verifier';\n\n/**\n * Interface representing the object returned from a\n * {@link TenantManager.listTenants}\n * operation.\n * Contains the list of tenants for the current batch and the next page token if available.\n */\nexport interface ListTenantsResult {\n\n  /**\n   * The list of {@link Tenant} objects for the downloaded batch.\n   */\n  tenants: Tenant[];\n\n  /**\n   * The next page token if available. This is needed for the next batch download.\n   */\n  pageToken?: string;\n}\n\n/**\n * Tenant-aware `Auth` interface used for managing users, configuring SAML/OIDC providers,\n * generating email links for password reset, email verification, etc for specific tenants.\n *\n * Multi-tenancy support requires Google Cloud's Identity Platform\n * (GCIP). To learn more about GCIP, including pricing and features,\n * see the {@link https://cloud.google.com/identity-platform | GCIP documentation}.\n *\n * Each tenant contains its own identity providers, settings and sets of users.\n * Using `TenantAwareAuth`, users for a specific tenant and corresponding OIDC/SAML\n * configurations can also be managed, ID tokens for users signed in to a specific tenant\n * can be verified, and email action links can also be generated for users belonging to the\n * tenant.\n *\n * `TenantAwareAuth` instances for a specific `tenantId` can be instantiated by calling\n * {@link TenantManager.authForTenant}.\n */\nexport class TenantAwareAuth extends BaseAuth {\n\n  /**\n   * The tenant identifier corresponding to this `TenantAwareAuth` instance.\n   * All calls to the user management APIs, OIDC/SAML provider management APIs, email link\n   * generation APIs, etc will only be applied within the scope of this tenant.\n   */\n  public readonly tenantId: string;\n\n  /**\n   * The TenantAwareAuth class constructor.\n   *\n   * @param app - The app that created this tenant.\n   * @param tenantId - The corresponding tenant ID.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App, tenantId: string) {\n    super(app, new TenantAwareAuthRequestHandler(\n      app, tenantId), createFirebaseTokenGenerator(app, tenantId));\n    utils.addReadonlyGetter(this, 'tenantId', tenantId);\n  }\n\n  /**\n   * {@inheritdoc BaseAuth.verifyIdToken}\n   */\n  public verifyIdToken(idToken: string, checkRevoked = false): Promise<DecodedIdToken> {\n    return super.verifyIdToken(idToken, checkRevoked)\n      .then((decodedClaims) => {\n        // Validate tenant ID.\n        if (decodedClaims.firebase.tenant !== this.tenantId) {\n          throw new FirebaseAuthError(AuthClientErrorCode.MISMATCHING_TENANT_ID);\n        }\n        return decodedClaims;\n      });\n  }\n\n  /**\n   * {@inheritdoc BaseAuth.createSessionCookie}\n   */\n  public createSessionCookie(\n    idToken: string, sessionCookieOptions: SessionCookieOptions): Promise<string> {\n    // Validate arguments before processing.\n    if (!validator.isNonEmptyString(idToken)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_ID_TOKEN));\n    }\n    if (!validator.isNonNullObject(sessionCookieOptions) ||\n        !validator.isNumber(sessionCookieOptions.expiresIn)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_SESSION_COOKIE_DURATION));\n    }\n    // This will verify the ID token and then match the tenant ID before creating the session cookie.\n    return this.verifyIdToken(idToken)\n      .then(() => {\n        return super.createSessionCookie(idToken, sessionCookieOptions);\n      });\n  }\n\n  /**\n   * {@inheritdoc BaseAuth.verifySessionCookie}\n   */\n  public verifySessionCookie(\n    sessionCookie: string, checkRevoked = false): Promise<DecodedIdToken> {\n    return super.verifySessionCookie(sessionCookie, checkRevoked)\n      .then((decodedClaims) => {\n        if (decodedClaims.firebase.tenant !== this.tenantId) {\n          throw new FirebaseAuthError(AuthClientErrorCode.MISMATCHING_TENANT_ID);\n        }\n        return decodedClaims;\n      });\n  }\n}\n\n/**\n * Defines the tenant manager used to help manage tenant related operations.\n * This includes:\n * <ul>\n * <li>The ability to create, update, list, get and delete tenants for the underlying\n *     project.</li>\n * <li>Getting a `TenantAwareAuth` instance for running Auth related operations\n *     (user management, provider configuration management, token verification,\n *     email link generation, etc) in the context of a specified tenant.</li>\n * </ul>\n */\nexport class TenantManager {\n  private readonly authRequestHandler: AuthRequestHandler;\n  private readonly tenantsMap: {[key: string]: TenantAwareAuth};\n\n  /**\n   * Initializes a TenantManager instance for a specified FirebaseApp.\n   *\n   * @param app - The app for this TenantManager instance.\n   *\n   * @constructor\n   * @internal\n   */\n  constructor(private readonly app: App) {\n    this.authRequestHandler = new AuthRequestHandler(app);\n    this.tenantsMap = {};\n  }\n\n  /**\n   * Returns a `TenantAwareAuth` instance bound to the given tenant ID.\n   *\n   * @param tenantId - The tenant ID whose `TenantAwareAuth` instance is to be returned.\n   *\n   * @returns The `TenantAwareAuth` instance corresponding to this tenant identifier.\n   */\n  public authForTenant(tenantId: string): TenantAwareAuth {\n    if (!validator.isNonEmptyString(tenantId)) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_TENANT_ID);\n    }\n    if (typeof this.tenantsMap[tenantId] === 'undefined') {\n      this.tenantsMap[tenantId] = new TenantAwareAuth(this.app, tenantId);\n    }\n    return this.tenantsMap[tenantId];\n  }\n\n  /**\n   * Gets the tenant configuration for the tenant corresponding to a given `tenantId`.\n   *\n   * @param tenantId - The tenant identifier corresponding to the tenant whose data to fetch.\n   *\n   * @returns A promise fulfilled with the tenant configuration to the provided `tenantId`.\n   */\n  public getTenant(tenantId: string): Promise<Tenant> {\n    return this.authRequestHandler.getTenant(tenantId)\n      .then((response: TenantServerResponse) => {\n        return new Tenant(response);\n      });\n  }\n\n  /**\n   * Retrieves a list of tenants (single batch only) with a size of `maxResults`\n...\n// Path: src/auth/auth.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app/index';\nimport { AuthRequestHandler } from './auth-api-request';\nimport { TenantManager } from './tenant-manager';\nimport { BaseAuth } from './base-auth';\nimport { ProjectConfigManager } from './project-config-manager';\n\n/**\n * Auth service bound to the provided app.\n * An Auth instance can have multiple tenants.\n */\nexport class Auth extends BaseAuth {\n\n  private readonly tenantManager_: TenantManager;\n  private readonly projectConfigManager_: ProjectConfigManager;\n  private readonly app_: App;\n\n  /**\n   * @param app - The app for this Auth service.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    super(app, new AuthRequestHandler(app));\n    this.app_ = app;\n    this.tenantManager_ = new TenantManager(app);\n    this.projectConfigManager_ = new ProjectConfigManager(app);\n  }\n\n  /**\n   * Returns the app associated with this Auth instance.\n   *\n   * @returns The app associated with this Auth instance.\n   */\n  get app(): App {\n    return this.app_;\n  }\n\n  /**\n   * Returns the tenant manager instance associated with the current project.\n   *\n   * @returns The tenant manager instance associated with the current project.\n   */\n  public tenantManager(): TenantManager {\n    return this.tenantManager_;\n  }\n\n  /**\n   * Returns the project config manager instance associated with the current project.\n   *\n   * @returns The project config manager instance associated with the current project.\n   */\n  public projectConfigManager(): ProjectConfigManager {\n    return this.projectConfigManager_;\n  }\n}\n\n// Path: src/firestore/firestore-internal.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseFirestoreError } from '../utils/error';\nimport { ServiceAccountCredential, isApplicationDefault } from '../app/credential-internal';\nimport { Firestore, Settings } from '@google-cloud/firestore';\n\nimport * as validator from '../utils/validator';\nimport * as utils from '../utils/index';\nimport { App } from '../app';\n\n/**\n * Settings to pass to the Firestore constructor.\n * \n * @public\n */\nexport interface FirestoreSettings {\n  /**\n   * Use HTTP/1.1 REST transport where possible.\n   * \n   * `preferRest` will force the use of HTTP/1.1 REST transport until a method\n   * that requires gRPC is called. When a method requires gRPC, this Firestore\n   * client will load dependent gRPC libraries and then use gRPC transport for\n   * all communication from that point forward. Currently the only operation\n   * that requires gRPC is creating a snapshot listener using `onSnapshot()`.\n   * \n   * @defaultValue `undefined`\n   */\n  preferRest?: boolean;\n}\n\nexport class FirestoreService {\n\n  private readonly appInternal: App;\n  private readonly databases: Map<string, Firestore> = new Map();\n  private readonly firestoreSettings: Map<string, FirestoreSettings> = new Map();\n\n  constructor(app: App) {\n    this.appInternal = app;\n  }\n\n  initializeDatabase(databaseId: string, settings: FirestoreSettings): Firestore {\n    const existingInstance = this.databases.get(databaseId);\n    if (existingInstance) {\n      const initialSettings = this.firestoreSettings.get(databaseId) ?? {};\n      if (this.checkIfSameSettings(settings, initialSettings)) {\n        return existingInstance;\n      }\n      throw new FirebaseFirestoreError({\n        code: 'failed-precondition',\n        message: 'initializeFirestore() has already been called with ' +\n          'different options. To avoid this error, call initializeFirestore() with the ' +\n          'same options as when it was originally called, or call getFirestore() to return the' +\n          ' already initialized instance.'\n      });\n    }\n    const newInstance = initFirestore(this.app, databaseId, settings);\n    this.databases.set(databaseId, newInstance);\n    this.firestoreSettings.set(databaseId, settings);\n    return newInstance;\n  }\n\n  getDatabase(databaseId: string): Firestore {\n    let database = this.databases.get(databaseId);\n    if (database === undefined) {\n      database = initFirestore(this.app, databaseId, {});\n      this.databases.set(databaseId, database);\n      this.firestoreSettings.set(databaseId, {});\n    }\n    return database;\n  }\n\n  private checkIfSameSettings(settingsA: FirestoreSettings, settingsB: FirestoreSettings): boolean {\n    const a = settingsA ?? {};\n    const b = settingsB ?? {};\n    // If we start passing more settings to Firestore constructor,\n    // replace this with deep equality check.\n    return (a.preferRest === b.preferRest);\n  }\n\n  /**\n   * Returns the app associated with this Storage instance.\n   *\n   * @returns The app associated with this Storage instance.\n   */\n  get app(): App {\n    return this.appInternal;\n  }\n}\n\nexport \nfunction getFirestoreOptions(app: App, firestoreSettings?: FirestoreSettings): Settings {\n  if (!validator.isNonNullObject(app) || !('options' in app)) {\n    throw new FirebaseFirestoreError({\n      code: 'invalid-argument',\n      message: 'First argument passed to admin.firestore() must be a valid Firebase app instance.',\n    });\n  }\n\n  const projectId: string | null = utils.getExplicitProjectId(app);\n  const credential = app.options.credential;\n  // eslint-disable-next-line @typescript-eslint/no-var-requires\n  const { version: firebaseVersion } = require('../../package.json');\n  const preferRest = firestoreSettings?.preferRest;\n  if (credential instanceof ServiceAccountCredential) {\n    return {\n      credentials: {\n        private_key: credential.privateKey,\n        client_email: credential.clientEmail,\n      },\n      // When the SDK is initialized with ServiceAccountCredentials an explicit projectId is\n      // guaranteed to be available.\n      projectId: projectId!,\n      firebaseVersion,\n      preferRest,\n    };\n  } else if (isApplicationDefault(app.options.credential)) {\n    // Try to use the Google application default credentials.\n    // If an explicit project ID is not available, let Firestore client discover one from the\n    // environment. This prevents the users from having to set GOOGLE_CLOUD_PROJECT in GCP runtimes.\n    return validator.isNonEmptyString(projectId)\n      ? { projectId, firebaseVersion, preferRest }\n      : { firebaseVersion, preferRest };\n  }\n\n  throw new FirebaseFirestoreError({\n    code: 'invalid-credential',\n    message: 'Failed to initialize Google Cloud Firestore client with the available credentials. ' +\n      'Must initialize the SDK with a certificate credential or application default credentials ' +\n      'to use Cloud Firestore API.',\n  });\n}\n\nfunction initFirestore(app: App, databaseId: string, firestoreSettings?: FirestoreSettings): Firestore {\n  const options = getFirestoreOptions(app, firestoreSettings);\n  options.databaseId = databaseId;\n  let firestoreDatabase: typeof Firestore;\n  try {\n    // Lazy-load the Firestore implementation here, which in turns loads gRPC.\n    firestoreDatabase = require('@google-cloud/firestore').Firestore;\n  } catch (err) {\n    throw new FirebaseFirestoreError({\n      code: 'missing-dependencies',\n      message: 'Failed to import the Cloud Firestore client library for Node.js. '\n          + 'Make sure to install the \"@google-cloud/firestore\" npm package. '\n          + `Original error: ${err}`,\n    });\n  }\n\n  return new firestoreDatabase(options);\n}\n\n// Path: src/firestore/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Cloud Firestore.\n *\n * @packageDocumentation\n */\n\nimport { Firestore } from '@google-cloud/firestore';\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { FirestoreService, FirestoreSettings } from './firestore-internal';\nimport { DEFAULT_DATABASE_ID } from '@google-cloud/firestore/build/src/path';\n\nexport {\n  AddPrefixToKeys,\n  AggregateField,\n  AggregateFieldType,\n  AggregateQuery,\n  AggregateQuerySnapshot,\n  AggregateSpecData,\n  AggregateSpec,\n  AggregateType,\n  BulkWriter,\n  BulkWriterOptions,\n  BundleBuilder,\n  ChildUpdateFields,\n  CollectionGroup,\n  CollectionReference,\n  DocumentChange,\n  DocumentChangeType,\n  DocumentData,\n  DocumentReference,\n  DocumentSnapshot,\n  FieldPath,\n  FieldValue,\n  Filter,\n  Firestore,\n  FirestoreDataConverter,\n  GeoPoint,\n  GrpcStatus,\n  NestedUpdateFields,\n  OrderByDirection,\n  PartialWithFieldValue,\n  Precondition,\n  Primitive,\n  Query,\n  QueryDocumentSnapshot,\n  QueryPartition,\n  QuerySnapshot,\n  ReadOptions,\n  ReadOnlyTransactionOptions,\n  ReadWriteTransactionOptions,\n  Settings,\n  SetOptions,\n  Timestamp,\n  Transaction,\n  UpdateData,\n  UnionToIntersection,\n  WhereFilterOp,\n  WithFieldValue,\n  WriteBatch,\n  WriteResult,\n  v1,\n  setLogFunction,\n} from '@google-cloud/firestore';\n\nexport { FirestoreSettings };\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n *\n * @example\n * ```javascript\n * // Get the default Firestore service for the default app\n * const defaultFirestore = getFirestore();\n * ```\n\n * @returns The default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n */\nexport function getFirestore(): Firestore;\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app.\n *\n * @example\n * ```javascript\n * // Get the default Firestore service for a specific app\n * const otherFirestore = getFirestore(app);\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @returns The default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service associated with the provided app.\n */\nexport function getFirestore(app: App): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a named database and default app\n * const otherFirestore = getFirestore('otherDb');\n * ```\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n * @beta\n */\nexport function getFirestore(databaseId: string): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a named database and specific app.\n * const otherFirestore = getFirestore('otherDb');\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service associated with the provided app.\n * @beta\n */\nexport function getFirestore(app: App, databaseId: string): Firestore;\n\nexport function getFirestore(\n  appOrDatabaseId?: App | string,\n  optionalDatabaseId?: string\n): Firestore {\n  const app: App = typeof appOrDatabaseId === 'object' ? appOrDatabaseId : getApp();\n  const databaseId =\n    (typeof appOrDatabaseId === 'string' ? appOrDatabaseId : optionalDatabaseId) || DEFAULT_DATABASE_ID;\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  const firestoreService = firebaseApp.getOrInitService(\n    'firestore', (app) => new FirestoreService(app));\n  return firestoreService.getDatabase(databaseId);\n}\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app, passing extra parameters to its constructor.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a specific app, require HTTP/1.1 REST transport\n * const otherFirestore = initializeFirestore(app, {preferRest: true});\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param settings - Settings object to be passed to the constructor.\n *\n * @returns The default `Firestore` service associated with the provided app and settings.\n */\nexport function initializeFirestore(app: App, settings?: FirestoreSettings): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app, passing extra parameters to its constructor.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a specific app, require HTTP/1.1 REST transport\n * const otherFirestore = initializeFirestore(app, {preferRest: true}, 'otherDb');\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param settings - Settings object to be passed to the constructor.\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named `Firestore` service associated with the provided app and settings.\n * @beta\n */\nexport function initializeFirestore(\n  app: App,\n  settings: FirestoreSettings,\n  databaseId: string\n): Firestore;\n\nexport function initializeFirestore(\n  app: App,\n  settings?: FirestoreSettings,\n  databaseId?: string\n): Firestore {\n  settings ??= {};\n  databaseId ??= DEFAULT_DATABASE_ID;\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  const firestoreService = firebaseApp.getOrInitService(\n    'firestore', (app) => new FirestoreService(app));\n\n  return firestoreService.initializeDatabase(databaseId, settings);\n}\n\n// Path: src/auth/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Firebase Authentication.\n *\n * @packageDocumentation\n */\n\nimport { App, getApp } from '../app/index';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { Auth } from './auth';\n\n/**\n * Gets the {@link Auth} service for the default app or a\n * given app.\n *\n * `getAuth()` can be called with no arguments to access the default app's\n * {@link Auth} service or as `getAuth(app)` to access the\n * {@link Auth} service associated with a specific app.\n *\n * @example\n * ```javascript\n * // Get the Auth service for the default app\n * const defaultAuth = getAuth();\n * ```\n *\n * @example\n * ```javascript\n * // Get the Auth service for a given app\n * const otherAuth = getAuth(otherApp);\n * ```\n *\n */\nexport function getAuth(app?: App): Auth {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('auth', (app) => new Auth(app));\n}\n\nexport { ActionCodeSettings } from './action-code-settings-builder';\n\nexport {\n  Auth,\n} from './auth';\n\nexport {\n  AllowByDefault,\n  AllowByDefaultWrap,\n  AllowlistOnly,\n  AllowlistOnlyWrap,\n  AuthFactorType,\n  AuthProviderConfig,\n  AuthProviderConfigFilter,\n  BaseAuthProviderConfig,\n  BaseCreateMultiFactorInfoRequest,\n  BaseUpdateMultiFactorInfoRequest,\n  CreateMultiFactorInfoRequest,\n  CreatePhoneMultiFactorInfoRequest,\n  CreateRequest,\n  EmailSignInProviderConfig,\n  ListProviderConfigResults,\n  MultiFactorConfig,\n  MultiFactorConfigState,\n  MultiFactorCreateSettings,\n  MultiFactorUpdateSettings,\n  MultiFactorProviderConfig,\n  OAuthResponseType,\n  OIDCAuthProviderConfig,\n  OIDCUpdateAuthProviderRequest,\n  RecaptchaAction,\n  RecaptchaConfig,\n  RecaptchaKey,\n  RecaptchaKeyClientType,\n  RecaptchaManagedRule,\n  RecaptchaProviderEnforcementState,\n  SAMLAuthProviderConfig,\n  SAMLUpdateAuthProviderRequest,\n  SmsRegionConfig,\n  UserProvider,\n  UpdateAuthProviderRequest,\n  UpdateMultiFactorInfoRequest,\n  UpdatePhoneMultiFactorInfoRequest,\n  UpdateRequest,\n  TotpMultiFactorProviderConfig,\n  PasswordPolicyConfig,\n  PasswordPolicyEnforcementState,\n  CustomStrengthOptionsConfig,\n  EmailPrivacyConfig,\n} from './auth-config';\n\nexport {\n  BaseAuth,\n  DeleteUsersResult,\n  GetUsersResult,\n  ListUsersResult,\n  SessionCookieOptions,\n} from './base-auth';\n\nexport {\n  EmailIdentifier,\n  PhoneIdentifier,\n  ProviderIdentifier,\n  UidIdentifier,\n  UserIdentifier,\n} from './identifier';\n\nexport {\n  CreateTenantRequest,\n  Tenant,\n  UpdateTenantRequest,\n} from './tenant';\n\nexport {\n  ListTenantsResult,\n  TenantAwareAuth,\n  TenantManager,\n} from './tenant-manager';\n\nexport {\n  UpdateProjectConfigRequest,\n  ProjectConfig,\n} from './project-config';\n\nexport {\n  ProjectConfigManager,\n} from './project-config-manager';\n\nexport {\n  DecodedIdToken,\n  DecodedAuthBlockingToken\n} from './token-verifier';\n\nexport {\n  HashAlgorithmType,\n  UserImportOptions,\n  UserImportRecord,\n  UserImportResult,\n  UserMetadataRequest,\n  UserProviderRequest,\n} from './user-import-builder';\n\nexport {\n  MultiFactorInfo,\n  MultiFactorSettings,\n  PhoneMultiFactorInfo,\n  UserInfo,\n  UserMetadata,\n  UserRecord,\n} from './user-record';\n\n// Path: src/instance-id/instance-id.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { getInstallations } from '../installations';\nimport { App } from '../app/index';\nimport {\n  FirebaseInstallationsError, FirebaseInstanceIdError,\n  InstallationsClientErrorCode, InstanceIdClientErrorCode,\n} from '../utils/error';\nimport * as validator from '../utils/validator';\n\n/**\n * The `InstanceId` service enables deleting the Firebase instance IDs\n * associated with Firebase client app instances.\n *\n * @deprecated Use {@link firebase-admin.installations#Installations} instead.\n */\nexport class InstanceId {\n\n  private app_: App;\n\n  /**\n   * @param app - The app for this InstanceId service.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseInstanceIdError(\n        InstanceIdClientErrorCode.INVALID_ARGUMENT,\n        'First argument passed to instanceId() must be a valid Firebase app instance.',\n      );\n    }\n\n    this.app_ = app;\n  }\n\n  /**\n   * Deletes the specified instance ID and the associated data from Firebase.\n   *\n   * Note that Google Analytics for Firebase uses its own form of Instance ID to\n   * keep track of analytics data. Therefore deleting a Firebase Instance ID does\n   * not delete Analytics data. See\n   * {@link https://firebase.google.com/support/privacy/manage-iids#delete_an_instance_id |\n   * Delete an Instance ID}\n   * for more information.\n   *\n   * @param instanceId - The instance ID to be deleted.\n   *\n   * @returns A promise fulfilled when the instance ID is deleted.\n   */\n  public deleteInstanceId(instanceId: string): Promise<void> {\n    return getInstallations(this.app).deleteInstallation(instanceId)\n      .catch((err) => {\n        if (err instanceof FirebaseInstallationsError) {\n          let code = err.code.replace('installations/', '');\n          if (code === InstallationsClientErrorCode.INVALID_INSTALLATION_ID.code) {\n            code = InstanceIdClientErrorCode.INVALID_INSTANCE_ID.code;\n          }\n\n          throw new FirebaseInstanceIdError({ code, message: err.message });\n        }\n\n        throw err;\n      });\n  }\n\n  /**\n   * Returns the app associated with this InstanceId instance.\n   *\n   * @returns The app associated with this InstanceId instance.\n   */\n  get app(): App {\n    return this.app_;\n  }\n}\n\n// Path: src/project-management/app-metadata.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Platforms with which a Firebase App can be associated.\n */\nexport enum AppPlatform {\n  /**\n   * Unknown state. This is only used for distinguishing unset values.\n   */\n  PLATFORM_UNKNOWN = 'PLATFORM_UNKNOWN',\n\n  /**\n   * The Firebase App is associated with iOS.\n   */\n  IOS = 'IOS',\n\n  /**\n   * The Firebase App is associated with Android.\n   */\n  ANDROID = 'ANDROID',\n}\n\n/**\n * Metadata about a Firebase app.\n */\nexport interface AppMetadata {\n  /**\n   * The globally unique, Firebase-assigned identifier of the app.\n   *\n   * @example\n   * ```javascript\n   * var appId = appMetadata.appId;\n   * ```\n   */\n  appId: string;\n\n  /**\n   * The optional user-assigned display name of the app.\n   *\n   * @example\n   * ```javascript\n   * var displayName = appMetadata.displayName;\n   * ```\n   */\n  displayName?: string;\n\n  /**\n   * The development platform of the app. Supporting Android and iOS app platforms.\n   *\n   * @example\n   * ```javascript\n   * var platform = AppPlatform.ANDROID;\n   * ```\n   */\n  platform: AppPlatform;\n\n  /**\n   * The globally unique, user-assigned ID of the parent project for the app.\n   *\n   * @example\n   * ```javascript\n   * var projectId = appMetadata.projectId;\n   * ```\n   */\n  projectId: string;\n\n  /**\n   * The fully-qualified resource name that identifies this app.\n   *\n   * This is useful when manually constructing requests for Firebase's public API.\n   *\n   * @example\n   * ```javascript\n   * var resourceName = androidAppMetadata.resourceName;\n   * ```\n   */\n  resourceName: string;\n}\n\n// Path: src/project-management/android-app.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseProjectManagementError } from '../utils/error';\nimport * as validator from '../utils/validator';\nimport { ProjectManagementRequestHandler, assertServerResponse } from './project-management-api-request-internal';\nimport { AppMetadata, AppPlatform } from './app-metadata';\n\n\n/**\n * Metadata about a Firebase Android App.\n */\nexport interface AndroidAppMetadata extends AppMetadata {\n\n  platform: AppPlatform.ANDROID;\n\n  /**\n   * The canonical package name of the Android App, as would appear in the Google Play Developer\n   * Console.\n   *\n   * @example\n   * ```javascript\n   * var packageName = androidAppMetadata.packageName;\n   * ```\n   */\n  packageName: string;\n}\n\n/**\n * A reference to a Firebase Android app.\n *\n * Do not call this constructor directly. Instead, use {@link ProjectManagement.androidApp}.\n */\nexport class AndroidApp {\n\n  private readonly resourceName: string;\n\n  /**\n   * @internal\n   */\n  constructor(\n      public readonly appId: string,\n      private readonly requestHandler: ProjectManagementRequestHandler) {\n    if (!validator.isNonEmptyString(appId)) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'appId must be a non-empty string.');\n    }\n\n    this.resourceName = `projects/-/androidApps/${appId}`;\n  }\n\n  /**\n   * Retrieves metadata about this Android app.\n   *\n   * @returns A promise that resolves to the retrieved metadata about this Android app.\n   */\n  public getMetadata(): Promise<AndroidAppMetadata> {\n    return this.requestHandler.getResource(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getMetadata()\\'s responseData must be a non-null object.');\n\n        const requiredFieldsList = ['name', 'appId', 'projectId', 'packageName'];\n        requiredFieldsList.forEach((requiredField) => {\n          assertServerResponse(\n            validator.isNonEmptyString(responseData[requiredField]),\n            responseData,\n            `getMetadata()'s responseData.${requiredField} must be a non-empty string.`);\n        });\n\n        const metadata: AndroidAppMetadata = {\n          platform: AppPlatform.ANDROID,\n          resourceName: responseData.name,\n          appId: responseData.appId,\n          displayName: responseData.displayName || null,\n          projectId: responseData.projectId,\n          packageName: responseData.packageName,\n        };\n        return metadata;\n      });\n  }\n\n  /**\n   * Sets the optional user-assigned display name of the app.\n   *\n   * @param newDisplayName - The new display name to set.\n   *\n   * @returns A promise that resolves when the display name has been set.\n   */\n  public setDisplayName(newDisplayName: string): Promise<void> {\n    return this.requestHandler.setDisplayName(this.resourceName, newDisplayName);\n  }\n\n  /**\n   * Gets the list of SHA certificates associated with this Android app in Firebase.\n   *\n   * @returns The list of SHA-1 and SHA-256 certificates associated with this Android app in\n   *     Firebase.\n   */\n  public getShaCertificates(): Promise<ShaCertificate[]> {\n    return this.requestHandler.getAndroidShaCertificates(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getShaCertificates()\\'s responseData must be a non-null object.');\n\n        if (!responseData.certificates) {\n          return [];\n        }\n\n        assertServerResponse(\n          validator.isArray(responseData.certificates),\n          responseData,\n          '\"certificates\" field must be present in the getShaCertificates() response data.');\n\n        const requiredFieldsList = ['name', 'shaHash'];\n\n        return responseData.certificates.map((certificateJson: any) => {\n          requiredFieldsList.forEach((requiredField) => {\n            assertServerResponse(\n              validator.isNonEmptyString(certificateJson[requiredField]),\n              responseData,\n              `getShaCertificates()'s responseData.certificates[].${requiredField} must be a `\n                      + 'non-empty string.');\n          });\n\n          return new ShaCertificate(certificateJson.shaHash, certificateJson.name);\n        });\n      });\n  }\n\n  /**\n   * Adds the given SHA certificate to this Android app.\n   *\n   * @param certificateToAdd - The SHA certificate to add.\n   *\n   * @returns A promise that resolves when the given certificate\n   *     has been added to the Android app.\n   */\n  public addShaCertificate(certificateToAdd: ShaCertificate): Promise<void> {\n    return this.requestHandler.addAndroidShaCertificate(this.resourceName, certificateToAdd);\n  }\n\n  /**\n   * Deletes the specified SHA certificate from this Android app.\n   *\n   * @param certificateToDelete - The SHA certificate to delete.\n   *\n   * @returns A promise that resolves when the specified\n   *     certificate has been removed from the Android app.\n   */\n  public deleteShaCertificate(certificateToDelete: ShaCertificate): Promise<void> {\n    if (!certificateToDelete.resourceName) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument',\n        'Specified certificate does not include a resourceName. (Use AndroidApp.getShaCertificates() to retrieve ' +\n              'certificates with a resourceName.');\n    }\n    return this.requestHandler.deleteResource(certificateToDelete.resourceName);\n  }\n\n  /**\n   * Gets the configuration artifact associated with this app.\n   *\n   * @returns A promise that resolves to the Android app's\n   *     Firebase config file, in UTF-8 string format. This string is typically\n   *     intended to be written to a JSON file that gets shipped with your Android\n   *     app.\n   */\n  public getConfig(): Promise<string> {\n    return this.requestHandler.getConfig(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getConfig()\\'s responseData must be a non-null object.');\n\n        const base64ConfigFileContents = responseData.configFileContents;\n        assertServerResponse(\n          validator.isBase64String(base64ConfigFileContents),\n          responseData,\n          'getConfig()\\'s responseData.configFileContents must be a base64 string.');\n\n        return Buffer.from(base64ConfigFileContents, 'base64').toString('utf8');\n      });\n  }\n}\n\n/**\n * A SHA-1 or SHA-256 certificate.\n *\n * Do not call this constructor directly. Instead, use\n * [`projectManagement.shaCertificate()`](projectManagement.ProjectManagement#shaCertificate).\n */\nexport class ShaCertificate {\n  /**\n   * The SHA certificate type.\n   *\n   * @example\n   * ```javascript\n   * var certType = shaCertificate.certType;\n   * ```\n   */\n  public readonly certType: ('sha1' | 'sha256');\n\n  /**\n   * Creates a ShaCertificate using the given hash. The ShaCertificate's type (eg. 'sha256') is\n   * automatically determined from the hash itself.\n   *\n   * @param shaHash - The sha256 or sha1 hash for this certificate.\n   * @example\n   * ```javascript\n   * var shaHash = shaCertificate.shaHash;\n   * ```\n   * @param resourceName - The Firebase resource name for this certificate. This does not need to be\n   *     set when creating a new certificate.\n   * @example\n   * ```javascript\n   * var resourceName = shaCertificate.resourceName;\n   * ```\n   *\n   * @internal\n   */\n  constructor(public readonly shaHash: string, public readonly resourceName?: string) {\n    if (/^[a-fA-F0-9]{40}$/.test(shaHash)) {\n      this.certType = 'sha1';\n    } else if (/^[a-fA-F0-9]{64}$/.test(shaHash)) {\n      this.certType = 'sha256';\n    } else {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'shaHash must be either a sha256 hash or a sha1 hash.');\n    }\n  }\n}\n\n// Path: src/project-management/project-management-api-request-internal.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport {\n  AuthorizedHttpClient, HttpError, HttpMethod, HttpRequestConfig, ExponentialBackoffPoller,\n} from '../utils/api-request';\nimport { FirebaseProjectManagementError, ProjectManagementErrorCode } from '../utils/error';\nimport { getSdkVersion } from '../utils/index';\nimport * as validator from '../utils/validator';\nimport { ShaCertificate } from './android-app';\n\n\n/** Project management backend host and port. */\nconst PROJECT_MANAGEMENT_HOST_AND_PORT = 'firebase.googleapis.com:443';\n/** Project management backend path. */\nconst PROJECT_MANAGEMENT_PATH = '/v1/';\n/** Project management beta backend path. */\nconst PROJECT_MANAGEMENT_BETA_PATH = '/v1beta1/';\n/** Project management request header. */\nconst PROJECT_MANAGEMENT_HEADERS = {\n  'X-Client-Version': `Node/Admin/${getSdkVersion()}`,\n};\n/** Project management request timeout duration in milliseconds. */\nconst PROJECT_MANAGEMENT_TIMEOUT_MILLIS = 10000;\n\nconst LIST_APPS_MAX_PAGE_SIZE = 100;\n\nconst CERT_TYPE_API_MAP = {\n  sha1: 'SHA_1',\n  sha256: 'SHA_256',\n};\n\nexport function assertServerResponse(\n  condition: boolean, responseData: object, message: string): void {\n  if (!condition) {\n    throw new FirebaseProjectManagementError(\n      'invalid-server-response',\n      `${message} Response data: ${JSON.stringify(responseData, null, 2)}`);\n  }\n}\n\n/**\n * Class that provides mechanism to send requests to the Firebase project management backend\n * endpoints.\n *\n * @internal\n */\nexport class ProjectManagementRequestHandler {\n  private readonly baseUrl: string =\n    `https://${PROJECT_MANAGEMENT_HOST_AND_PORT}${PROJECT_MANAGEMENT_PATH}`;\n  private readonly baseBetaUrl: string =\n    `https://${PROJECT_MANAGEMENT_HOST_AND_PORT}${PROJECT_MANAGEMENT_BETA_PATH}`;\n  private readonly httpClient: AuthorizedHttpClient;\n\n  private static wrapAndRethrowHttpError(errStatusCode: number, errText?: string): void {\n    let errorCode: ProjectManagementErrorCode;\n    let errorMessage: string;\n\n    switch (errStatusCode) {\n    case 400:\n      errorCode = 'invalid-argument';\n      errorMessage = 'Invalid argument provided.';\n      break;\n    case 401:\n    case 403:\n      errorCode = 'authentication-error';\n      errorMessage = 'An error occurred when trying to authenticate. Make sure the credential '\n            + 'used to authenticate this SDK has the proper permissions. See '\n            + 'https://firebase.google.com/docs/admin/setup for setup instructions.';\n      break;\n    case 404:\n      errorCode = 'not-found';\n      errorMessage = 'The specified entity could not be found.';\n      break;\n    case 409:\n      errorCode = 'already-exists';\n      errorMessage = 'The specified entity already exists.';\n      break;\n    case 500:\n      errorCode = 'internal-error';\n      errorMessage = 'An internal error has occurred. Please retry the request.';\n      break;\n    case 503:\n      errorCode = 'service-unavailable';\n      errorMessage = 'The server could not process the request in time. See the error '\n            + 'documentation for more details.';\n      break;\n    default:\n      errorCode = 'unknown-error';\n      errorMessage = 'An unknown server error was returned.';\n    }\n\n    if (!errText) {\n      errText = '<missing>';\n    }\n    throw new FirebaseProjectManagementError(\n      errorCode,\n      `${ errorMessage } Status code: ${ errStatusCode }. Raw server response: \"${ errText }\".`);\n  }\n\n  /**\n   * @param app - The app used to fetch access tokens to sign API requests.\n   * @constructor\n   */\n  constructor(app: App) {\n    this.httpClient = new AuthorizedHttpClient(app as FirebaseApp);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose Android\n   *     apps you want to list.\n   */\n  public listAndroidApps(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}/androidApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose iOS apps\n   *     you want to list.\n   */\n  public listIosApps(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}/iosApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose iOS apps\n   *     you want to list.\n   */\n  public listAppMetadata(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}:searchApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project that you want\n   *     to create the Android app within.\n   */\n  public createAndroidApp(\n    parentResourceName: string, packageName: string, displayName?: string): Promise<object> {\n    const requestData: any = {\n      packageName,\n    };\n    if (validator.isNonEmptyString(displayName)) {\n      requestData.displayName = displayName;\n    }\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/androidApps`, requestData, 'v1beta1')\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createAndroidApp\\'s responseData must be a non-null object.');\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.name),\n          responseData,\n          'createAndroidApp\\'s responseData.name must be a non-empty string.');\n        return this.pollRemoteOperationWithExponentialBackoff(responseData.name);\n      });\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project that you want\n   *     to create the iOS app within.\n   */\n  public createIosApp(\n    parentResourceName: string, bundleId: string, displayName?: string): Promise<object> {\n    const requestData: any = {\n      bundleId,\n    };\n    if (validator.isNonEmptyString(displayName)) {\n      requestData.displayName = displayName;\n    }\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/iosApps`, requestData, 'v1beta1')\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createIosApp\\'s responseData must be a non-null object.');\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.name),\n          responseData,\n          'createIosApp\\'s responseData.name must be a non-empty string.');\n        return this.pollRemoteOperationWithExponentialBackoff(responseData.name);\n      });\n  }\n\n  /**\n   * @param resourceName - Fully-qualified resource name of the entity whose display name you\n   *     want to set.\n   */\n  public setDisplayName(resourceName: string, newDisplayName: string): Promise<void> {\n    const requestData = {\n      displayName: newDisplayName,\n    };\n    return this\n      .invokeRequestHandler(\n        'PATCH', `${resourceName}?update_mask=display_name`, requestData, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the Android app whose SHA\n   *     certificates you want to get.\n   */\n  public getAndroidShaCertificates(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET', `${parentResourceName}/sha`, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the Android app that you\n   *     want to add the given SHA certificate to.\n   */\n  public addAndroidShaCertificate(\n    parentResourceName: string, certificate: ShaCertificate): Promise<void> {\n    const requestData = {\n      shaHash: certificate.shaHash,\n      certType: CERT_TYPE_API_MAP[certificate.certType],\n    };\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/sha`, requestData, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the app whose config you\n   *     want to get.\n   */\n  public getConfig(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET', `${parentResourceName}/config`, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the entity that you want to\n   *     get.\n   */\n  public getResource(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler('GET', parentResourceName, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param resourceName - Fully-qualified resource name of the entity that you want to\n   *     delete.\n   */\n  public deleteResource(resourceName: string): Promise<void> {\n    return this\n      .invokeRequestHandler('DELETE', resourceName, /* requestData */ null, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  private pollRemoteOperationWithExponentialBackoff(\n    operationResourceName: string): Promise<object> {\n    const poller = new ExponentialBackoffPoller<object>();\n\n    return poller.poll(() => {\n      return this.invokeRequestHandler('GET', operationResourceName, /* requestData */ null)\n        .then((responseData: any) => {\n          if (responseData.error) {\n            const errStatusCode: number = responseData.error.code || 500;\n            const errText: string =\n                  responseData.error.message || JSON.stringify(responseData.error);\n            ProjectManagementRequestHandler.wrapAndRethrowHttpError(errStatusCode, errText);\n          }\n\n          if (!responseData.done) {\n            // Continue polling.\n            return null;\n          }\n\n          // Polling complete. Resolve with operation response JSON.\n          return responseData.response;\n        });\n    });\n  }\n\n  /**\n   * Invokes the request handler with the provided request data.\n   */\n  private invokeRequestHandler(\n    method: HttpMethod,\n    path: string,\n    requestData: object | null,\n    apiVersion: ('v1' | 'v1beta1') = 'v1'): Promise<object> {\n    const baseUrlToUse = (apiVersion === 'v1') ? this.baseUrl : this.baseBetaUrl;\n    const request: HttpRequestConfig = {\n      method,\n      url: `${baseUrlToUse}${path}`,\n      headers: PROJECT_MANAGEMENT_HEADERS,\n      data: requestData,\n      timeout: PROJECT_MANAGEMENT_TIMEOUT_MILLIS,\n    };\n\n    return this.httpClient.send(request)\n      .then((response) => {\n        // Send non-JSON responses to the catch() below, where they will be treated as errors.\n        if (!response.isJson()) {\n          throw new HttpError(response);\n        }\n\n        return response.data;\n      })\n      .catch((err) => {\n        if (err instanceof HttpError) {\n          ProjectManagementRequestHandler.wrapAndRethrowHttpError(\n            err.response.status, err.response.text);\n        }\n        throw err;\n      });\n  }\n}\n\n// Path: src/project-management/ios-app.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseProjectManagementError } from '../utils/error';\nimport * as validator from '../utils/validator';\nimport { ProjectManagementRequestHandler, assertServerResponse } from './project-management-api-request-internal';\nimport { AppMetadata, AppPlatform } from './app-metadata';\n\n/**\n * Metadata about a Firebase iOS App.\n */\nexport interface IosAppMetadata extends AppMetadata {\n  platform: AppPlatform.IOS;\n\n  /**\n   * The canonical bundle ID of the iOS App as it would appear in the iOS App Store.\n   *\n   * @example\n   * ```javascript\n   * var bundleId = iosAppMetadata.bundleId;\n   *```\n    */\n  bundleId: string;\n}\n\n/**\n * A reference to a Firebase iOS app.\n *\n * Do not call this constructor directly. Instead, use {@link ProjectManagement.iosApp}.\n */\nexport class IosApp {\n\n  private readonly resourceName: string;\n\n  /**\n   * @internal\n   */\n  constructor(\n      public readonly appId: string,\n      private readonly requestHandler: ProjectManagementRequestHandler) {\n    if (!validator.isNonEmptyString(appId)) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'appId must be a non-empty string.');\n    }\n\n    this.resourceName = `projects/-/iosApps/${appId}`;\n  }\n\n  /**\n   * Retrieves metadata about this iOS app.\n   *\n   * @returns A promise that\n   *     resolves to the retrieved metadata about this iOS app.\n   */\n  public getMetadata(): Promise<IosAppMetadata> {\n    return this.requestHandler.getResource(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getMetadata()\\'s responseData must be a non-null object.');\n\n        const requiredFieldsList = ['name', 'appId', 'projectId', 'bundleId'];\n        requiredFieldsList.forEach((requiredField) => {\n          assertServerResponse(\n            validator.isNonEmptyString(responseData[requiredField]),\n            responseData,\n            `getMetadata()'s responseData.${requiredField} must be a non-empty string.`);\n        });\n\n        const metadata: IosAppMetadata = {\n          platform: AppPlatform.IOS,\n          resourceName: responseData.name,\n          appId: responseData.appId,\n          displayName: responseData.displayName || null,\n          projectId: responseData.projectId,\n          bundleId: responseData.bundleId,\n        };\n        return metadata;\n      });\n  }\n\n  /**\n   * Sets the optional user-assigned display name of the app.\n   *\n   * @param newDisplayName - The new display name to set.\n   *\n   * @returns A promise that resolves when the display name has\n   *     been set.\n   */\n  public setDisplayName(newDisplayName: string): Promise<void> {\n    return this.requestHandler.setDisplayName(this.resourceName, newDisplayName);\n  }\n\n  /**\n   * Gets the configuration artifact associated with this app.\n   *\n   * @returns A promise that resolves to the iOS app's Firebase\n   *     config file, in UTF-8 string format. This string is typically intended to\n   *     be written to a plist file that gets shipped with your iOS app.\n   */\n  public getConfig(): Promise<string> {\n    return this.requestHandler.getConfig(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getConfig()\\'s responseData must be a non-null object.');\n\n        const base64ConfigFileContents = responseData.configFileContents;\n        assertServerResponse(\n...\n", "needle_token_start": 3727, "needle_token_end": 4201, "code_context_ntokens": 15871, "output": ["```typescript\nfunction getFirestoreOptions(app: App, firestoreSettings?: FirestoreSettings): Settings {\n  if (!validator.isNonNullObject(app) || !('options' in app)) {\n    throw new FirebaseFirestoreError({\n      code: 'invalid-argument',\n      message: 'First argument passed to admin.firestore() must be a valid Firebase app instance.',\n    });\n  }\n\n  const projectId: string | null = utils.getExplicitProjectId(app);\n  const credential = app.options.credential;\n  // eslint-disable-next-line @typescript-eslint/no-var-requires\n  const { version: firebaseVersion } = require('../../package.json');\n  const preferRest = firestoreSettings?.preferRest;\n  if (credential instanceof ServiceAccountCredential) {\n    return {\n      credentials: {\n        private_key: credential.privateKey,\n        client_email: credential.clientEmail,\n      },\n      // When the SDK is initialized with ServiceAccountCredentials an explicit projectId is\n      // guaranteed to be available.\n      projectId: projectId!,\n      firebaseVersion,\n      preferRest,\n    };\n  } else if (isApplicationDefault(app.options.credential)) {\n    // Try to use the Google application default credentials.\n    // If an explicit project ID is not available, let Firestore client discover one from the\n    // environment. This prevents the users from having to set GOOGLE_CLOUD_PROJECT in GCP runtimes.\n    return validator.isNonEmptyString(projectId)\n      ? { projectId, firebaseVersion, preferRest }\n      : { firebaseVersion, preferRest };\n  }\n\n  throw new FirebaseFirestoreError({\n    code: 'invalid-credential',\n    message: 'Failed to initialize Google Cloud Firestore client with the available credentials. ' +\n      'Must initialize the SDK with a certificate credential or application default credentials ' +\n      'to use Cloud Firestore API.',\n  });\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "getMessaging", "language": "typescript", "path": "src/messaging/index.ts", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve a messaging service instance, either for the default application or a specified application, facilitating the management and sending of messages through Firebase Cloud Messaging (FCM).\n2. **Input**: An optional application instance. If no instance is provided, the function defaults to using the primary application instance.\n3. **Output**: Returns an instance of the messaging service associated with either the default or specified application.\n4. **Procedure**: The function first checks if an application instance is provided. If not, it retrieves the default application instance. It then ensures that a messaging service is initialized for the specified application instance, returning this service for use.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/messaging/messaging.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { deepCopy, deepExtend } from '../utils/deep-copy';\nimport { SubRequest } from './batch-request-internal';\nimport { ErrorInfo, MessagingClientErrorCode, FirebaseMessagingError } from '../utils/error';\nimport * as utils from '../utils';\nimport * as validator from '../utils/validator';\nimport { validateMessage, BLACKLISTED_DATA_PAYLOAD_KEYS, BLACKLISTED_OPTIONS_KEYS } from './messaging-internal';\nimport { FirebaseMessagingRequestHandler } from './messaging-api-request-internal';\n\nimport {\n  BatchResponse,\n  Message,\n  MessagingTopicManagementResponse,\n  MulticastMessage,\n\n  // Legacy API types\n  MessagingDevicesResponse,\n  MessagingDeviceGroupResponse,\n  MessagingPayload,\n  MessagingOptions,\n  MessagingTopicResponse,\n  MessagingConditionResponse,\n  DataMessagePayload,\n  NotificationMessagePayload,\n  SendResponse,\n} from './messaging-api';\n\n// FCM endpoints\nconst FCM_SEND_HOST = 'fcm.googleapis.com';\nconst FCM_SEND_PATH = '/fcm/send';\nconst FCM_TOPIC_MANAGEMENT_HOST = 'iid.googleapis.com';\nconst FCM_TOPIC_MANAGEMENT_ADD_PATH = '/iid/v1:batchAdd';\nconst FCM_TOPIC_MANAGEMENT_REMOVE_PATH = '/iid/v1:batchRemove';\n\n// Maximum messages that can be included in a batch request.\nconst FCM_MAX_BATCH_SIZE = 500;\n\n// Key renames for the messaging notification payload object.\nconst CAMELCASED_NOTIFICATION_PAYLOAD_KEYS_MAP = {\n  bodyLocArgs: 'body_loc_args',\n  bodyLocKey: 'body_loc_key',\n  clickAction: 'click_action',\n  titleLocArgs: 'title_loc_args',\n  titleLocKey: 'title_loc_key',\n};\n\n// Key renames for the messaging options object.\nconst CAMELCASE_OPTIONS_KEYS_MAP = {\n  dryRun: 'dry_run',\n  timeToLive: 'time_to_live',\n  collapseKey: 'collapse_key',\n  mutableContent: 'mutable_content',\n  contentAvailable: 'content_available',\n  restrictedPackageName: 'restricted_package_name',\n};\n\n// Key renames for the MessagingDeviceResult object.\nconst MESSAGING_DEVICE_RESULT_KEYS_MAP = {\n  message_id: 'messageId',\n  registration_id: 'canonicalRegistrationToken',\n};\n\n// Key renames for the MessagingDevicesResponse object.\nconst MESSAGING_DEVICES_RESPONSE_KEYS_MAP = {\n  canonical_ids: 'canonicalRegistrationTokenCount',\n  failure: 'failureCount',\n  success: 'successCount',\n  multicast_id: 'multicastId',\n};\n\n// Key renames for the MessagingDeviceGroupResponse object.\nconst MESSAGING_DEVICE_GROUP_RESPONSE_KEYS_MAP = {\n  success: 'successCount',\n  failure: 'failureCount',\n  failed_registration_ids: 'failedRegistrationTokens',\n};\n\n// Key renames for the MessagingTopicResponse object.\nconst MESSAGING_TOPIC_RESPONSE_KEYS_MAP = {\n  message_id: 'messageId',\n};\n\n// Key renames for the MessagingConditionResponse object.\nconst MESSAGING_CONDITION_RESPONSE_KEYS_MAP = {\n  message_id: 'messageId',\n};\n\n/**\n * Maps a raw FCM server response to a MessagingDevicesResponse object.\n *\n * @param response - The raw FCM server response to map.\n *\n * @returns The mapped MessagingDevicesResponse object.\n */\nfunction mapRawResponseToDevicesResponse(response: object): MessagingDevicesResponse {\n  // Rename properties on the server response\n  utils.renameProperties(response, MESSAGING_DEVICES_RESPONSE_KEYS_MAP);\n  if ('results' in response) {\n    (response as any).results.forEach((messagingDeviceResult: any) => {\n      utils.renameProperties(messagingDeviceResult, MESSAGING_DEVICE_RESULT_KEYS_MAP);\n\n      // Map the FCM server's error strings to actual error objects.\n      if ('error' in messagingDeviceResult) {\n        const newError = FirebaseMessagingError.fromServerError(\n          messagingDeviceResult.error, /* message */ undefined, messagingDeviceResult.error,\n        );\n        messagingDeviceResult.error = newError;\n      }\n    });\n  }\n\n  return response as MessagingDevicesResponse;\n}\n\n/**\n * Maps a raw FCM server response to a MessagingDeviceGroupResponse object.\n *\n * @param response - The raw FCM server response to map.\n *\n * @returns The mapped MessagingDeviceGroupResponse object.\n */\nfunction mapRawResponseToDeviceGroupResponse(response: object): MessagingDeviceGroupResponse {\n  // Rename properties on the server response\n  utils.renameProperties(response, MESSAGING_DEVICE_GROUP_RESPONSE_KEYS_MAP);\n\n  // Add the 'failedRegistrationTokens' property if it does not exist on the response, which\n  // it won't when the 'failureCount' property has a value of 0)\n  (response as any).failedRegistrationTokens = (response as any).failedRegistrationTokens || [];\n\n  return response as MessagingDeviceGroupResponse;\n}\n\n/**\n * Maps a raw FCM server response to a MessagingTopicManagementResponse object.\n *\n * @param {object} response The raw FCM server response to map.\n *\n * @returns {MessagingTopicManagementResponse} The mapped MessagingTopicManagementResponse object.\n */\nfunction mapRawResponseToTopicManagementResponse(response: object): MessagingTopicManagementResponse {\n  // Add the success and failure counts.\n  const result: MessagingTopicManagementResponse = {\n    successCount: 0,\n    failureCount: 0,\n    errors: [],\n  };\n\n  if ('results' in response) {\n    (response as any).results.forEach((tokenManagementResult: any, index: number) => {\n      // Map the FCM server's error strings to actual error objects.\n      if ('error' in tokenManagementResult) {\n        result.failureCount += 1;\n        const newError = FirebaseMessagingError.fromTopicManagementServerError(\n          tokenManagementResult.error, /* message */ undefined, tokenManagementResult.error,\n        );\n\n        result.errors.push({\n          index,\n          error: newError,\n        });\n      } else {\n        result.successCount += 1;\n      }\n    });\n  }\n  return result;\n}\n\n\n/**\n * Messaging service bound to the provided app.\n */\nexport class Messaging {\n\n  private urlPath: string;\n  private readonly appInternal: App;\n  private readonly messagingRequestHandler: FirebaseMessagingRequestHandler;\n\n  /**\n   * @internal\n   */\n  constructor(app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT,\n        'First argument passed to admin.messaging() must be a valid Firebase app instance.',\n      );\n    }\n\n    this.appInternal = app;\n    this.messagingRequestHandler = new FirebaseMessagingRequestHandler(app);\n  }\n\n  /**\n   * The {@link firebase-admin.app#App} associated with the current `Messaging` service\n   * instance.\n   *\n   * @example\n   * ```javascript\n   * var app = messaging.app;\n   * ```\n   */\n  get app(): App {\n    return this.appInternal;\n  }\n\n  /**\n   * Sends the given message via FCM.\n   *\n   * @param message - The message payload.\n   * @param dryRun - Whether to send the message in the dry-run\n   *   (validation only) mode.\n   * @returns A promise fulfilled with a unique message ID\n   *   string after the message has been successfully handed off to the FCM\n   *   service for delivery.\n   */\n  public send(message: Message, dryRun?: boolean): Promise<string> {\n    const copy: Message = deepCopy(message);\n    validateMessage(copy);\n    if (typeof dryRun !== 'undefined' && !validator.isBoolean(dryRun)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT, 'dryRun must be a boolean');\n    }\n    return this.getUrlPath()\n      .then((urlPath) => {\n        const request: { message: Message; validate_only?: boolean } = { message: copy };\n        if (dryRun) {\n          request.validate_only = true;\n        }\n        return this.messagingRequestHandler.invokeRequestHandler(FCM_SEND_HOST, urlPath, request);\n      })\n      .then((response) => {\n        return (response as any).name;\n      });\n  }\n\n  /**\n  * Sends each message in the given array via Firebase Cloud Messaging.\n  *\n  * Unlike {@link Messaging.sendAll}, this method makes a single RPC call for each message\n  * in the given array.\n  *\n  * The responses list obtained from the return value corresponds to the order of `messages`.\n  * An error from this method or a `BatchResponse` with all failures indicates a total failure,\n  * meaning that none of the messages in the list could be sent. Partial failures or no\n  * failures are only indicated by a `BatchResponse` return value.\n  *\n  * @param messages - A non-empty array\n  *   containing up to 500 messages.\n  * @param dryRun - Whether to send the messages in the dry-run\n  *   (validation only) mode.\n  * @returns A Promise fulfilled with an object representing the result of the\n  *   send operation.\n  */\n  public sendEach(messages: Message[], dryRun?: boolean): Promise<BatchResponse> {\n    if (validator.isArray(messages) && messages.constructor !== Array) {\n      // In more recent JS specs, an array-like object might have a constructor that is not of\n      // Array type. Our deepCopy() method doesn't handle them properly. Convert such objects to\n      // a regular array here before calling deepCopy(). See issue #566 for details.\n      messages = Array.from(messages);\n    }\n\n    const copy: Message[] = deepCopy(messages);\n    if (!validator.isNonEmptyArray(copy)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT, 'messages must be a non-empty array');\n    }\n    if (copy.length > FCM_MAX_BATCH_SIZE) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT,\n        `messages list must not contain more than ${FCM_MAX_BATCH_SIZE} items`);\n    }\n    if (typeof dryRun !== 'undefined' && !validator.isBoolean(dryRun)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT, 'dryRun must be a boolean');\n    }\n\n    return this.getUrlPath()\n      .then((urlPath) => {\n        const requests: Promise<SendResponse>[] = copy.map(async (message) => {\n          validateMessage(message);\n          const request: { message: Message; validate_only?: boolean } = { message };\n          if (dryRun) {\n            request.validate_only = true;\n          }\n          return this.messagingRequestHandler.invokeRequestHandlerForSendResponse(FCM_SEND_HOST, urlPath, request);\n        });\n        return Promise.allSettled(requests);\n      }).then((results) => {\n        const responses: SendResponse[] = [];\n        results.forEach(result => {\n          if (result.status === 'fulfilled') {\n            responses.push(result.value);\n          } else { // rejected\n            responses.push({ success: false, error: result.reason })\n          }\n        })\n        const successCount: number = responses.filter((resp) => resp.success).length;\n        return {\n          responses,\n          successCount,\n          failureCount: responses.length - successCount,\n        };\n      });\n  }\n\n  /**\n   * Sends the given multicast message to all the FCM registration tokens\n   * specified in it.\n   *\n   * This method uses the {@link Messaging.sendEach} API under the hood to send the given\n   * message to all the target recipients. The responses list obtained from the\n   * return value corresponds to the order of tokens in the `MulticastMessage`.\n   * An error from this method or a `BatchResponse` with all failures indicates a total\n   * failure, meaning that the messages in the list could be sent. Partial failures or\n   * failures are only indicated by a `BatchResponse` return value.\n   *\n   * @param message - A multicast message\n   *   containing up to 500 tokens.\n   * @param dryRun - Whether to send the message in the dry-run\n   *   (validation only) mode.\n   * @returns A Promise fulfilled with an object representing the result of the\n   *   send operation.\n   */\n  public sendEachForMulticast(message: MulticastMessage, dryRun?: boolean): Promise<BatchResponse> {\n    const copy: MulticastMessage = deepCopy(message);\n    if (!validator.isNonNullObject(copy)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT, 'MulticastMessage must be a non-null object');\n    }\n    if (!validator.isNonEmptyArray(copy.tokens)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT, 'tokens must be a non-empty array');\n    }\n    if (copy.tokens.length > FCM_MAX_BATCH_SIZE) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT,\n        `tokens list must not contain more than ${FCM_MAX_BATCH_SIZE} items`);\n    }\n\n    const messages: Message[] = copy.tokens.map((token) => {\n      return {\n        token,\n        android: copy.android,\n        apns: copy.apns,\n        data: copy.data,\n        notification: copy.notification,\n        webpush: copy.webpush,\n        fcmOptions: copy.fcmOptions,\n      };\n    });\n    return this.sendEach(messages, dryRun);\n  }\n\n  /**\n   * Sends all the messages in the given array via Firebase Cloud Messaging.\n   * Employs batching to send the entire list as a single RPC call. Compared\n   * to the `send()` method, this method is a significantly more efficient way\n   * to send multiple messages.\n   *\n   * The responses list obtained from the return value\n   * corresponds to the order of tokens in the `MulticastMessage`. An error\n   * from this method indicates a total failure, meaning that none of the messages\n   * in the list could be sent. Partial failures are indicated by a `BatchResponse`\n   * return value.\n   *\n   * @param messages - A non-empty array\n   *   containing up to 500 messages.\n   * @param dryRun - Whether to send the messages in the dry-run\n   *   (validation only) mode.\n   * @returns A Promise fulfilled with an object representing the result of the\n   *   send operation.\n   *\n   * @deprecated Use {@link Messaging.sendEach} instead.\n   */\n  public sendAll(messages: Message[], dryRun?: boolean): Promise<BatchResponse> {\n    if (validator.isArray(messages) && messages.constructor !== Array) {\n      // In more recent JS specs, an array-like object might have a constructor that is not of\n      // Array type. Our deepCopy() method doesn't handle them properly. Convert such objects to\n      // a regular array here before calling deepCopy(). See issue #566 for details.\n      messages = Array.from(messages);\n    }\n\n    const copy: Message[] = deepCopy(messages);\n    if (!validator.isNonEmptyArray(copy)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT, 'messages must be a non-empty array');\n    }\n    if (copy.length > FCM_MAX_BATCH_SIZE) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT,\n        `messages list must not contain more than ${FCM_MAX_BATCH_SIZE} items`);\n    }\n    if (typeof dryRun !== 'undefined' && !validator.isBoolean(dryRun)) {\n      throw new FirebaseMessagingError(\n        MessagingClientErrorCode.INVALID_ARGUMENT, 'dryRun must be a boolean');\n    }\n\n    return this.getUrlPath()\n      .then((urlPath) => {\n        const requests: SubRequest[] = copy.map((message) => {\n          validateMessage(message);\n          const request: { message: Message; validate_only?: boolean } = { message };\n          if (dryRun) {\n            request.validate_only = true;\n          }\n          return {\n            url: `https://${FCM_SEND_HOST}${urlPath}`,\n            body: request,\n          };\n        });\n        return this.messagingRequestHandler.sendBatchRequest(requests);\n      });\n  }\n\n  /**\n   * Sends the given multicast message to all the FCM registration tokens\n   * specified in it.\n   *\n   * This method uses the `sendAll()` API under the hood to send the given\n   * message to all the target recipients. The responses list obtained from the\n   * return value corresponds to the order of tokens in the `MulticastMessage`.\n   * An error from this method indicates a total failure, meaning that the message\n   * was not sent to any of the tokens in the list. Partial failures are indicated\n   * by a `BatchResponse` return value.\n   *\n   * @param message - A multicast message\n   *   containing up to 500 tokens.\n   * @param dryRun - Whether to send the message in the dry-run\n   *   (validation only) mode.\n   * @returns A Promise fulfilled with an object representing the result of the\n   *   send operation.\n   *\n   * @deprecated Use {@link Messaging.sendEachForMulticast} instead.\n   */\n...\n// Path: src/messaging/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Firebase Cloud Messaging (FCM).\n *\n * @packageDocumentation\n */\n\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { Messaging } from './messaging';\n\nexport {\n  Messaging,\n} from './messaging';\n\nexport {\n  AndroidConfig,\n  AndroidFcmOptions,\n  AndroidNotification,\n  ApnsConfig,\n  ApnsFcmOptions,\n  ApnsPayload,\n  Aps,\n  ApsAlert,\n  BaseMessage,\n  BatchResponse,\n  CriticalSound,\n  ConditionMessage,\n  FcmOptions,\n  LightSettings,\n  Message,\n  MessagingTopicManagementResponse,\n  MulticastMessage,\n  Notification,\n  SendResponse,\n  TokenMessage,\n  TopicMessage,\n  WebpushConfig,\n  WebpushFcmOptions,\n  WebpushNotification,\n\n  // Legacy APIs\n  DataMessagePayload,\n  MessagingConditionResponse,\n  MessagingDeviceGroupResponse,\n  MessagingDeviceResult,\n  MessagingDevicesResponse,\n  MessagingOptions,\n  MessagingPayload,\n  MessagingTopicResponse,\n  NotificationMessagePayload,\n} from './messaging-api';\n\n/**\n * Gets the {@link Messaging} service for the default app or a given app.\n *\n * `admin.messaging()` can be called with no arguments to access the default\n * app's `Messaging` service or as `admin.messaging(app)` to access the\n * `Messaging` service associated with aspecific app.\n *\n * @example\n * ```javascript\n * // Get the Messaging service for the default app\n * const defaultMessaging = getMessaging();\n * ```\n *\n * @example\n * ```javascript\n * // Get the Messaging service for a given app\n * const otherMessaging = getMessaging(otherApp);\n * ```\n *\n * @param app - Optional app whose `Messaging` service to\n *   return. If not provided, the default `Messaging` service will be returned.\n *\n * @returns The default `Messaging` service if no\n *   app is provided or the `Messaging` service associated with the provided\n *   app.\n */\nexport \nfunction getMessaging(app?: App): Messaging {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('messaging', (app) => new Messaging(app));\n}\n\n// Path: src/remote-config/remote-config-api.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Colors that are associated with conditions for display purposes.\n */\nexport type TagColor = 'BLUE' | 'BROWN' | 'CYAN' | 'DEEP_ORANGE' | 'GREEN' |\n  'INDIGO' | 'LIME' | 'ORANGE' | 'PINK' | 'PURPLE' | 'TEAL';\n\n/**\n * Type representing a Remote Config parameter value data type.\n * Defaults to `STRING` if unspecified.\n */\nexport type ParameterValueType = 'STRING' | 'BOOLEAN' | 'NUMBER' | 'JSON'\n\n/**\n * Interface representing a Remote Config condition.\n * A condition targets a specific group of users. A list of these conditions make up\n * part of a Remote Config template.\n */\nexport interface RemoteConfigCondition {\n\n  /**\n   * A non-empty and unique name of this condition.\n   */\n  name: string;\n\n  /**\n   * The logic of this condition.\n   * See the documentation on\n   * {@link https://firebase.google.com/docs/remote-config/condition-reference | condition expressions}\n   * for the expected syntax of this field.\n   */\n  expression: string;\n\n  /**\n   * The color associated with this condition for display purposes in the Firebase Console.\n   * Not specifying this value results in the console picking an arbitrary color to associate\n   * with the condition.\n   */\n  tagColor?: TagColor;\n}\n\n/**\n * Represents a Remote Config condition in the dataplane.\n * A condition targets a specific group of users. A list of these conditions\n * comprise part of a Remote Config template.\n */\nexport interface NamedCondition {\n\n  /**\n   * A non-empty and unique name of this condition.\n   */\n  name: string;\n\n  /**\n   * The logic of this condition.\n   * See the documentation on\n   * {@link https://firebase.google.com/docs/remote-config/condition-reference | condition expressions}\n   * for the expected syntax of this field.\n   */\n  condition: OneOfCondition;\n}\n\n/**\n * Represents a condition that may be one of several types.\n * Only the first defined field will be processed.\n */\nexport interface OneOfCondition {\n\n  /**\n   * Makes this condition an OR condition.\n   */\n  orCondition?: OrCondition;\n\n  /**\n   * Makes this condition an AND condition.\n   */\n  andCondition?: AndCondition;\n\n  /**\n   * Makes this condition a constant true.\n   */\n  true?: Record<string, never>;\n\n  /**\n   * Makes this condition a constant false.\n   */\n  false?: Record<string, never>;\n\n  /**\n   * Makes this condition a percent condition.\n   */\n  percent?: PercentCondition;\n}\n\n/**\n * Represents a collection of conditions that evaluate to true if all are true.\n */\nexport interface AndCondition {\n\n  /**\n   * The collection of conditions.\n   */\n  conditions?: Array<OneOfCondition>;\n}\n\n/**\n * Represents a collection of conditions that evaluate to true if any are true.\n */\nexport interface OrCondition {\n\n  /**\n   * The collection of conditions.\n   */\n  conditions?: Array<OneOfCondition>;\n}\n\n/**\n * Defines supported operators for percent conditions.\n */\nexport enum PercentConditionOperator {\n\n  /**\n   * A catchall error case.\n   */\n  UNKNOWN = 'UNKNOWN',\n\n  /**\n   * Target percentiles less than or equal to the target percent.\n   * A condition using this operator must specify microPercent.\n   */\n  LESS_OR_EQUAL = 'LESS_OR_EQUAL',\n\n  /**\n   * Target percentiles greater than the target percent.\n   * A condition using this operator must specify microPercent.\n   */\n  GREATER_THAN = 'GREATER_THAN',\n\n  /**\n   * Target percentiles within an interval defined by a lower bound and an\n   * upper bound. The lower bound is an exclusive (open) bound and the\n   * micro_percent_range_upper_bound is an inclusive (closed) bound.\n   * A condition using this operator must specify microPercentRange.\n   */\n  BETWEEN = 'BETWEEN'\n}\n\n/**\n * Represents the limit of percentiles to target in micro-percents.\n * The value must be in the range [0 and 100000000]\n */\nexport interface MicroPercentRange {\n\n  /**\n   * The lower limit of percentiles to target in micro-percents.\n   * The value must be in the range [0 and 100000000].\n   */\n  microPercentLowerBound?: number;\n\n  /**\n   * The upper limit of percentiles to target in micro-percents.\n   * The value must be in the range [0 and 100000000].\n   */\n  microPercentUpperBound?: number;\n}\n\n/**\n * Represents a condition that compares the instance pseudo-random\n * percentile to a given limit.\n */\nexport interface PercentCondition {\n\n  /**\n   * The choice of percent operator to determine how to compare targets\n   * to percent(s).\n   */\n  percentOperator?: PercentConditionOperator;\n\n  /**\n   * The limit of percentiles to target in micro-percents when\n   * using the LESS_OR_EQUAL and GREATER_THAN operators. The value must\n   * be in the range [0 and 100000000].\n   */\n  microPercent?: number;\n\n  /**\n   * The seed used when evaluating the hash function to map an instance to\n   * a value in the hash space. This is a string which can have 0 - 32\n   * characters and can contain ASCII characters [-_.0-9a-zA-Z].The string\n   * is case-sensitive.\n   */\n  seed?: string;\n\n  /**\n   * The micro-percent interval to be used with the\n   * BETWEEN operator.\n   */\n  microPercentRange?: MicroPercentRange;\n}\n\n/**\n * Interface representing an explicit parameter value.\n */\nexport interface ExplicitParameterValue {\n  /**\n   * The `string` value that the parameter is set to.\n   */\n  value: string;\n}\n\n/**\n * Interface representing an in-app-default value.\n */\nexport interface InAppDefaultValue {\n  /**\n   * If `true`, the parameter is omitted from the parameter values returned to a client.\n   */\n  useInAppDefault: boolean;\n}\n\n/**\n * Type representing a Remote Config parameter value.\n * A `RemoteConfigParameterValue` could be either an `ExplicitParameterValue` or\n * an `InAppDefaultValue`.\n */\nexport type RemoteConfigParameterValue = ExplicitParameterValue | InAppDefaultValue;\n\n/**\n * Interface representing a Remote Config parameter.\n * At minimum, a `defaultValue` or a `conditionalValues` entry must be present for the\n * parameter to have any effect.\n */\nexport interface RemoteConfigParameter {\n\n  /**\n   * The value to set the parameter to, when none of the named conditions evaluate to `true`.\n   */\n  defaultValue?: RemoteConfigParameterValue;\n\n  /**\n   * A `(condition name, value)` map. The condition name of the highest priority\n   * (the one listed first in the Remote Config template's conditions list) determines the value of\n   * this parameter.\n   */\n  conditionalValues?: { [key: string]: RemoteConfigParameterValue };\n\n  /**\n   * A description for this parameter. Should not be over 100 characters and may contain any\n   * Unicode characters.\n   */\n  description?: string;\n\n  /**\n   * The data type for all values of this parameter in the current version of the template.\n   * Defaults to `ParameterValueType.STRING` if unspecified.\n   */\n  valueType?: ParameterValueType;\n}\n\n/**\n * Interface representing a Remote Config parameter group.\n * Grouping parameters is only for management purposes and does not affect client-side\n * fetching of parameter values.\n */\nexport interface RemoteConfigParameterGroup {\n  /**\n   * A description for the group. Its length must be less than or equal to 256 characters.\n   * A description may contain any Unicode characters.\n   */\n  description?: string;\n\n  /**\n   * Map of parameter keys to their optional default values and optional conditional values for\n   * parameters that belong to this group. A parameter only appears once per\n   * Remote Config template. An ungrouped parameter appears at the top level, whereas a\n   * parameter organized within a group appears within its group's map of parameters.\n   */\n  parameters: { [key: string]: RemoteConfigParameter };\n}\n\n/**\n * Represents a Remote Config client template.\n */\nexport interface RemoteConfigTemplate {\n  /**\n   * A list of conditions in descending order by priority.\n   */\n  conditions: RemoteConfigCondition[];\n\n  /**\n   * Map of parameter keys to their optional default values and optional conditional values.\n   */\n  parameters: { [key: string]: RemoteConfigParameter };\n\n  /**\n   * Map of parameter group names to their parameter group objects.\n   * A group's name is mutable but must be unique among groups in the Remote Config template.\n   * The name is limited to 256 characters and intended to be human-readable. Any Unicode\n   * characters are allowed.\n   */\n  parameterGroups: { [key: string]: RemoteConfigParameterGroup };\n\n  /**\n   * ETag of the current Remote Config template (readonly).\n   */\n  readonly etag: string;\n\n  /**\n   * Version information for the current Remote Config template.\n   */\n  version?: Version;\n}\n\n/**\n * Represents the data in a Remote Config server template.\n */\nexport interface ServerTemplateData {\n  /**\n   * A list of conditions in descending order by priority.\n   */\n  conditions: NamedCondition[];\n\n  /**\n   * Map of parameter keys to their optional default values and optional conditional values.\n   */\n  parameters: { [key: string]: RemoteConfigParameter };\n\n  /**\n   * Current Remote Config template ETag (read-only).\n   */\n  readonly etag: string;\n\n  /**\n   * Version information for the current Remote Config template.\n   */\n  version?: Version;\n}\n\n/**\n * Represents optional arguments that can be used when instantiating {@link ServerTemplate}.\n */\nexport interface GetServerTemplateOptions {\n\n  /**\n   * Defines in-app default parameter values, so that your app behaves as\n   * intended before it connects to the Remote Config backend, and so that\n   * default values are available if none are set on the backend.\n   */\n  defaultConfig?: DefaultConfig;\n}\n\n/**\n * Represents the type of a Remote Config server template that can be set on\n * {@link ServerTemplate}. This can either be a {@link ServerTemplateData} object\n * or a template JSON string.\n */\nexport type ServerTemplateDataType = ServerTemplateData | string;\n\n/**\n * Represents optional arguments that can be used when instantiating\n * {@link ServerTemplate} synchronously.\n */\nexport interface InitServerTemplateOptions extends GetServerTemplateOptions {\n\n  /**\n   * Enables integrations to use template data loaded independently. For\n   * example, customers can reduce initialization latency by pre-fetching and\n   * caching template data and then using this option to initialize the SDK with\n   * that data.\n   */\n  template?: ServerTemplateDataType,\n}\n\n/**\n * Represents a stateful abstraction for a Remote Config server template.\n */\nexport interface ServerTemplate {\n  /**\n   * Evaluates the current template to produce a {@link ServerConfig}.\n   */\n  evaluate(context?: EvaluationContext): ServerConfig;\n\n  /**\n   * Fetches and caches the current active version of the\n   * project's {@link ServerTemplate}.\n   */\n  load(): Promise<void>;\n\n  /**\n   * Sets and caches a {@link ServerTemplateData} or a JSON string representing\n   * the server template\n   */\n  set(template: ServerTemplateDataType): void;\n\n  /**\n   * Returns a JSON representation of {@link ServerTemplateData}\n   */\n  toJSON(): ServerTemplateData;\n}\n\n/**\n * Represents template evaluation input signals.\n */\nexport type EvaluationContext = {\n\n  /**\n   * Defines the identifier to use when splitting a group. For example,\n   * this is used by the percent condition.\n   */\n  randomizationId?: string\n};\n\n/**\n * Interface representing a Remote Config user.\n */\nexport interface RemoteConfigUser {\n  /**\n   * Email address. Output only.\n   */\n  email: string;\n\n  /**\n   * Display name. Output only.\n   */\n  name?: string;\n\n  /**\n   * Image URL. Output only.\n   */\n  imageUrl?: string;\n}\n\n/**\n * Interface representing a Remote Config template version.\n * Output only, except for the version description. Contains metadata about a particular\n * version of the Remote Config template. All fields are set at the time the specified Remote\n * Config template is published. A version's description field may be specified in\n * `publishTemplate` calls.\n */\nexport interface Version {\n  /**\n   * The version number of a Remote Config template.\n   */\n  versionNumber?: string;\n\n  /**\n   * The timestamp of when this version of the Remote Config template was written to the\n   * Remote Config backend.\n   */\n  updateTime?: string;\n\n  /**\n   * The origin of the template update action.\n   */\n  updateOrigin?: ('REMOTE_CONFIG_UPDATE_ORIGIN_UNSPECIFIED' | 'CONSOLE' |\n    'REST_API' | 'ADMIN_SDK_NODE');\n\n  /**\n   * The type of the template update action.\n   */\n  updateType?: ('REMOTE_CONFIG_UPDATE_TYPE_UNSPECIFIED' |\n    'INCREMENTAL_UPDATE' | 'FORCED_UPDATE' | 'ROLLBACK');\n\n  /**\n   * Aggregation of all metadata fields about the account that performed the update.\n   */\n  updateUser?: RemoteConfigUser;\n\n  /**\n   * The user-provided description of the corresponding Remote Config template.\n   */\n  description?: string;\n\n  /**\n   * The version number of the Remote Config template that has become the current version\n   * due to a rollback. Only present if this version is the result of a rollback.\n   */\n  rollbackSource?: string;\n\n  /**\n   * Indicates whether this Remote Config template was published before version history was\n   * supported.\n   */\n  isLegacy?: boolean;\n}\n\n/**\n * Interface representing a list of Remote Config template versions.\n */\nexport interface ListVersionsResult {\n  /**\n   * A list of version metadata objects, sorted in reverse chronological order.\n   */\n  versions: Version[];\n\n  /**\n   * Token to retrieve the next page of results, or empty if there are no more results\n   * in the list.\n   */\n  nextPageToken?: string;\n}\n\n/**\n * Interface representing options for Remote Config list versions operation.\n */\nexport interface ListVersionsOptions {\n  /**\n   * The maximum number of items to return per page.\n   */\n  pageSize?: number;\n\n  /**\n   * The `nextPageToken` value returned from a previous list versions request, if any.\n   */\n  pageToken?: string;\n\n  /**\n   * Specifies the newest version number to include in the results.\n   * If specified, must be greater than zero. Defaults to the newest version.\n   */\n  endVersionNumber?: string | number;\n\n  /**\n   * Specifies the earliest update time to include in the results. Any entries updated before this\n   * time are omitted.\n   */\n  startTime?: Date | string;\n\n  /**\n   * Specifies the latest update time to include in the results. Any entries updated on or after\n   * this time are omitted.\n   */\n  endTime?: Date | string;\n}\n\n/**\n * Represents the configuration produced by evaluating a server template.\n */\nexport interface ServerConfig {\n\n  /**\n   * Gets the value for the given key as a boolean.\n   *\n   * Convenience method for calling <code>serverConfig.getValue(key).asBoolean()</code>.\n   *\n   * @param key - The name of the parameter.\n   *\n   * @returns The value for the given key as a boolean.\n   */\n  getBoolean(key: string): boolean;\n\n  /**\n   * Gets the value for the given key as a number.\n   *\n   * Convenience method for calling <code>serverConfig.getValue(key).asNumber()</code>.\n   *\n   * @param key - The name of the parameter.\n   *\n   * @returns The value for the given key as a number.\n   */\n  getNumber(key: string): number;\n\n  /**\n   * Gets the value for the given key as a string.\n   * Convenience method for calling <code>serverConfig.getValue(key).asString()</code>.\n   *\n   * @param key - The name of the parameter.\n   *\n   * @returns The value for the given key as a string.\n   */\n  getString(key: string): string;\n\n  /**\n   * Gets the {@link Value} for the given key.\n   *\n   * Ensures application logic will always have a type-safe reference,\n   * even if the parameter is removed remotely.\n   *\n   * @param key - The name of the parameter.\n   *\n   * @returns The value for the given key.\n   */\n  getValue(key: string): Value;\n}\n\n/**\n * Wraps a parameter value with metadata and type-safe getters.\n *\n * Type-safe getters insulate application logic from remote\n * changes to parameter names and types.\n */\nexport interface Value {\n\n  /**\n   * Gets the value as a boolean.\n   *\n   * The following values (case insensitive) are interpreted as true:\n   * \"1\", \"true\", \"t\", \"yes\", \"y\", \"on\". Other values are interpreted as false.\n   */\n  asBoolean(): boolean;\n\n  /**\n   * Gets the value as a number. Comparable to calling <code>Number(value) || 0</code>.\n   */\n  asNumber(): number;\n\n  /**\n   * Gets the value as a string.\n   */\n  asString(): string;\n\n  /**\n   * Gets the {@link ValueSource} for the given key.\n   */\n  getSource(): ValueSource;\n}\n\n/**\n * Indicates the source of a value.\n *\n * <ul>\n *   <li>\"static\" indicates the value was defined by a static constant.</li>\n *   <li>\"default\" indicates the value was defined by default config.</li>\n *   <li>\"remote\" indicates the value was defined by config produced by\n *   evaluating a template.</li>\n * </ul>\n */\nexport type ValueSource = 'static' | 'default' | 'remote';\n\n/**\n * Defines the format for in-app default parameter values.\n */\nexport type DefaultConfig = { [key: string]: string | number | boolean };\n\n// Path: src/remote-config/remote-config-api-client-internal.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { HttpRequestConfig, HttpClient, HttpError, AuthorizedHttpClient, HttpResponse } from '../utils/api-request';\nimport { PrefixedFirebaseError } from '../utils/error';\nimport * as utils from '../utils/index';\nimport * as validator from '../utils/validator';\nimport { deepCopy } from '../utils/deep-copy';\nimport {\n  ListVersionsOptions,\n  ListVersionsResult,\n  RemoteConfigTemplate,\n  ServerTemplateData\n} from './remote-config-api';\n\n// Remote Config backend constants\n/**\n  * Allows the `FIREBASE_REMOTE_CONFIG_URL_BASE` environment\n  * variable to override the default API endpoint URL.\n  */\nconst FIREBASE_REMOTE_CONFIG_URL_BASE = process.env.FIREBASE_REMOTE_CONFIG_URL_BASE || 'https://firebaseremoteconfig.googleapis.com';\nconst FIREBASE_REMOTE_CONFIG_HEADERS = {\n  'X-Firebase-Client': `fire-admin-node/${utils.getSdkVersion()}`,\n  // There is a known issue in which the ETag is not properly returned in cases where the request\n  // does not specify a compression type. Currently, it is required to include the header\n  // `Accept-Encoding: gzip` or equivalent in all requests.\n  // https://firebase.google.com/docs/remote-config/use-config-rest#etag_usage_and_forced_updates\n  'Accept-Encoding': 'gzip',\n};\n\n\n/**\n * Class that facilitates sending requests to the Firebase Remote Config backend API.\n *\n * @internal\n */\nexport class RemoteConfigApiClient {\n  private readonly httpClient: HttpClient;\n  private projectIdPrefix?: string;\n\n  constructor(private readonly app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'First argument passed to admin.remoteConfig() must be a valid Firebase app instance.');\n    }\n\n    this.httpClient = new AuthorizedHttpClient(app as FirebaseApp);\n  }\n\n  public getTemplate(): Promise<RemoteConfigTemplate> {\n    return this.getUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'GET',\n          url: `${url}/remoteConfig`,\n          headers: FIREBASE_REMOTE_CONFIG_HEADERS\n        };\n        return this.httpClient.send(request);\n      })\n      .then((resp) => {\n        return this.toRemoteConfigTemplate(resp);\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  public getTemplateAtVersion(versionNumber: number | string): Promise<RemoteConfigTemplate> {\n    const data = { versionNumber: this.validateVersionNumber(versionNumber) };\n    return this.getUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'GET',\n          url: `${url}/remoteConfig`,\n          headers: FIREBASE_REMOTE_CONFIG_HEADERS,\n          data\n        };\n        return this.httpClient.send(request);\n      })\n      .then((resp) => {\n        return this.toRemoteConfigTemplate(resp);\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  public validateTemplate(template: RemoteConfigTemplate): Promise<RemoteConfigTemplate> {\n    template = this.validateInputRemoteConfigTemplate(template);\n    return this.sendPutRequest(template, template.etag, true)\n      .then((resp) => {\n        // validating a template returns an etag with the suffix -0 means that your update\n        // was successfully validated. We set the etag back to the original etag of the template\n        // to allow future operations.\n        this.validateEtag(resp.headers['etag']);\n        return this.toRemoteConfigTemplate(resp, template.etag);\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  public publishTemplate(template: RemoteConfigTemplate, options?: { force: boolean }): Promise<RemoteConfigTemplate> {\n    template = this.validateInputRemoteConfigTemplate(template);\n    let ifMatch: string = template.etag;\n    if (options && options.force === true) {\n      // setting `If-Match: *` forces the Remote Config template to be updated\n      // and circumvent the ETag, and the protection from that it provides.\n      ifMatch = '*';\n    }\n    return this.sendPutRequest(template, ifMatch)\n      .then((resp) => {\n        return this.toRemoteConfigTemplate(resp);\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  public rollback(versionNumber: number | string): Promise<RemoteConfigTemplate> {\n    const data = { versionNumber: this.validateVersionNumber(versionNumber) };\n    return this.getUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'POST',\n          url: `${url}/remoteConfig:rollback`,\n          headers: FIREBASE_REMOTE_CONFIG_HEADERS,\n          data\n        };\n        return this.httpClient.send(request);\n      })\n      .then((resp) => {\n        return this.toRemoteConfigTemplate(resp);\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  public listVersions(options?: ListVersionsOptions): Promise<ListVersionsResult> {\n    if (typeof options !== 'undefined') {\n      options = this.validateListVersionsOptions(options);\n    }\n    return this.getUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'GET',\n          url: `${url}/remoteConfig:listVersions`,\n          headers: FIREBASE_REMOTE_CONFIG_HEADERS,\n          data: options\n        };\n        return this.httpClient.send(request);\n      })\n      .then((resp) => {\n        return resp.data;\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  public getServerTemplate(): Promise<ServerTemplateData> {\n    return this.getUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'GET',\n          url: `${url}/namespaces/firebase-server/serverRemoteConfig`,\n          headers: FIREBASE_REMOTE_CONFIG_HEADERS\n        };\n        return this.httpClient.send(request);\n      })\n      .then((resp) => {\n        return this.toRemoteConfigServerTemplate(resp);\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  private sendPutRequest(template: RemoteConfigTemplate, etag: string, validateOnly?: boolean): Promise<HttpResponse> {\n    let path = 'remoteConfig';\n    if (validateOnly) {\n      path += '?validate_only=true';\n    }\n    return this.getUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'PUT',\n          url: `${url}/${path}`,\n          headers: { ...FIREBASE_REMOTE_CONFIG_HEADERS, 'If-Match': etag },\n          data: {\n            conditions: template.conditions,\n            parameters: template.parameters,\n            parameterGroups: template.parameterGroups,\n            version: template.version,\n          }\n        };\n        return this.httpClient.send(request);\n      });\n  }\n\n  private getUrl(): Promise<string> {\n    return this.getProjectIdPrefix()\n      .then((projectIdPrefix) => {\n        return `${FIREBASE_REMOTE_CONFIG_URL_BASE}/v1/${projectIdPrefix}`;\n      });\n  }\n\n  private getProjectIdPrefix(): Promise<string> {\n    if (this.projectIdPrefix) {\n      return Promise.resolve(this.projectIdPrefix);\n    }\n\n    return utils.findProjectId(this.app)\n      .then((projectId) => {\n        if (!validator.isNonEmptyString(projectId)) {\n          throw new FirebaseRemoteConfigError(\n            'unknown-error',\n            'Failed to determine project ID. Initialize the SDK with service account credentials, or '\n            + 'set project ID as an app option. Alternatively, set the GOOGLE_CLOUD_PROJECT '\n            + 'environment variable.');\n        }\n\n        this.projectIdPrefix = `projects/${projectId}`;\n        return this.projectIdPrefix;\n      });\n  }\n\n  private toFirebaseError(err: HttpError): PrefixedFirebaseError {\n    if (err instanceof PrefixedFirebaseError) {\n      return err;\n    }\n\n    const response = err.response;\n    if (!response.isJson()) {\n      return new FirebaseRemoteConfigError(\n        'unknown-error',\n        `Unexpected response with status: ${response.status} and body: ${response.text}`);\n    }\n\n    const error: Error = (response.data as ErrorResponse).error || {};\n    let code: RemoteConfigErrorCode = 'unknown-error';\n    if (error.status && error.status in ERROR_CODE_MAPPING) {\n      code = ERROR_CODE_MAPPING[error.status];\n    }\n    const message = error.message || `Unknown server error: ${response.text}`;\n    return new FirebaseRemoteConfigError(code, message);\n  }\n\n  /**\n   * Creates a RemoteConfigTemplate from the API response.\n   * If provided, customEtag is used instead of the etag returned in the API response.\n   *\n   * @param {HttpResponse} resp API response object.\n   * @param {string} customEtag A custom etag to replace the etag fom the API response (Optional).\n   */\n  private toRemoteConfigTemplate(resp: HttpResponse, customEtag?: string): RemoteConfigTemplate {\n    const etag = (typeof customEtag === 'undefined') ? resp.headers['etag'] : customEtag;\n    this.validateEtag(etag);\n    return {\n      conditions: resp.data.conditions,\n      parameters: resp.data.parameters,\n      parameterGroups: resp.data.parameterGroups,\n      etag,\n      version: resp.data.version,\n    };\n  }\n\n  /**\n   * Creates a RemoteConfigServerTemplate from the API response.\n   * If provided, customEtag is used instead of the etag returned in the API response.\n   *\n   * @param {HttpResponse} resp API response object.\n   * @param {string} customEtag A custom etag to replace the etag fom the API response (Optional).\n   */\n  private toRemoteConfigServerTemplate(resp: HttpResponse, customEtag?: string): ServerTemplateData {\n    const etag = (typeof customEtag === 'undefined') ? resp.headers['etag'] : customEtag;\n    this.validateEtag(etag);\n    return {\n      conditions: resp.data.conditions,\n      parameters: resp.data.parameters,\n      etag,\n      version: resp.data.version,\n    };\n  }\n\n  /**\n   * Checks if the given RemoteConfigTemplate object is valid.\n   * The object must have valid parameters, parameter groups, conditions, and an etag.\n   * Removes output only properties from version metadata.\n   *\n   * @param {RemoteConfigTemplate} template A RemoteConfigTemplate object to be validated.\n   *\n   * @returns {RemoteConfigTemplate} The validated RemoteConfigTemplate object.\n   */\n  private validateInputRemoteConfigTemplate(template: RemoteConfigTemplate): RemoteConfigTemplate {\n    const templateCopy = deepCopy(template);\n    if (!validator.isNonNullObject(templateCopy)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        `Invalid Remote Config template: ${JSON.stringify(templateCopy)}`);\n    }\n    if (!validator.isNonEmptyString(templateCopy.etag)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'ETag must be a non-empty string.');\n    }\n    if (!validator.isNonNullObject(templateCopy.parameters)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'Remote Config parameters must be a non-null object');\n    }\n    if (!validator.isNonNullObject(templateCopy.parameterGroups)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'Remote Config parameter groups must be a non-null object');\n    }\n    if (!validator.isArray(templateCopy.conditions)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'Remote Config conditions must be an array');\n    }\n    if (typeof templateCopy.version !== 'undefined') {\n      // exclude output only properties and keep the only input property: description\n      templateCopy.version = { description: templateCopy.version.description };\n    }\n    return templateCopy;\n  }\n\n  /**\n   * Checks if a given version number is valid.\n   * A version number must be an integer or a string in int64 format.\n   * If valid, returns the string representation of the provided version number.\n   *\n   * @param {string|number} versionNumber A version number to be validated.\n   *\n   * @returns {string} The validated version number as a string.\n   */\n  private validateVersionNumber(versionNumber: string | number, propertyName = 'versionNumber'): string {\n    if (!validator.isNonEmptyString(versionNumber) &&\n      !validator.isNumber(versionNumber)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        `${propertyName} must be a non-empty string in int64 format or a number`);\n    }\n    if (!Number.isInteger(Number(versionNumber))) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        `${propertyName} must be an integer or a string in int64 format`);\n    }\n    return versionNumber.toString();\n  }\n\n  private validateEtag(etag?: string): void {\n    if (!validator.isNonEmptyString(etag)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'ETag header is not present in the server response.');\n    }\n  }\n\n  /**\n   * Checks if a given `ListVersionsOptions` object is valid. If successful, creates a copy of the\n   * options object and convert `startTime` and `endTime` to RFC3339 UTC \"Zulu\" format, if present.\n   *\n   * @param {ListVersionsOptions} options An options object to be validated.\n   *\n   * @returns {ListVersionsOptions} A copy of the provided options object with timestamps converted\n   * to UTC Zulu format.\n   */\n  private validateListVersionsOptions(options: ListVersionsOptions): ListVersionsOptions {\n    const optionsCopy = deepCopy(options);\n    if (!validator.isNonNullObject(optionsCopy)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'ListVersionsOptions must be a non-null object.');\n    }\n    if (typeof optionsCopy.pageSize !== 'undefined') {\n      if (!validator.isNumber(optionsCopy.pageSize)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument', 'pageSize must be a number.');\n      }\n      if (optionsCopy.pageSize < 1 || optionsCopy.pageSize > 300) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument', 'pageSize must be a number between 1 and 300 (inclusive).');\n      }\n    }\n    if (typeof optionsCopy.pageToken !== 'undefined' && !validator.isNonEmptyString(optionsCopy.pageToken)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument', 'pageToken must be a string value.');\n    }\n    if (typeof optionsCopy.endVersionNumber !== 'undefined') {\n      optionsCopy.endVersionNumber = this.validateVersionNumber(optionsCopy.endVersionNumber, 'endVersionNumber');\n    }\n    if (typeof optionsCopy.startTime !== 'undefined') {\n      if (!(optionsCopy.startTime instanceof Date) && !validator.isUTCDateString(optionsCopy.startTime)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument', 'startTime must be a valid Date object or a UTC date string.');\n      }\n      // Convert startTime to RFC3339 UTC \"Zulu\" format.\n      if (optionsCopy.startTime instanceof Date) {\n        optionsCopy.startTime = optionsCopy.startTime.toISOString();\n      } else {\n        optionsCopy.startTime = new Date(optionsCopy.startTime).toISOString();\n      }\n    }\n    if (typeof optionsCopy.endTime !== 'undefined') {\n      if (!(optionsCopy.endTime instanceof Date) && !validator.isUTCDateString(optionsCopy.endTime)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument', 'endTime must be a valid Date object or a UTC date string.');\n      }\n      // Convert endTime to RFC3339 UTC \"Zulu\" format.\n      if (optionsCopy.endTime instanceof Date) {\n        optionsCopy.endTime = optionsCopy.endTime.toISOString();\n      } else {\n        optionsCopy.endTime = new Date(optionsCopy.endTime).toISOString();\n      }\n    }\n    // Remove undefined fields from optionsCopy\n    Object.keys(optionsCopy).forEach(key =>\n      (typeof (optionsCopy as any)[key] === 'undefined') && delete (optionsCopy as any)[key]\n    );\n    return optionsCopy;\n  }\n}\n\ninterface ErrorResponse {\n  error?: Error;\n}\n\ninterface Error {\n  code?: number;\n  message?: string;\n  status?: string;\n}\n\nconst ERROR_CODE_MAPPING: { [key: string]: RemoteConfigErrorCode } = {\n  ABORTED: 'aborted',\n  ALREADY_EXISTS: 'already-exists',\n  INVALID_ARGUMENT: 'invalid-argument',\n  INTERNAL: 'internal-error',\n  FAILED_PRECONDITION: 'failed-precondition',\n  NOT_FOUND: 'not-found',\n  OUT_OF_RANGE: 'out-of-range',\n  PERMISSION_DENIED: 'permission-denied',\n  RESOURCE_EXHAUSTED: 'resource-exhausted',\n  UNAUTHENTICATED: 'unauthenticated',\n  UNKNOWN: 'unknown-error',\n};\n\nexport type RemoteConfigErrorCode =\n  'aborted'\n  | 'already-exists'\n  | 'failed-precondition'\n  | 'internal-error'\n  | 'invalid-argument'\n  | 'not-found'\n  | 'out-of-range'\n  | 'permission-denied'\n  | 'resource-exhausted'\n  | 'unauthenticated'\n  | 'unknown-error';\n\n/**\n * Firebase Remote Config error code structure. This extends PrefixedFirebaseError.\n *\n * @param {RemoteConfigErrorCode} code The error code.\n * @param {string} message The error message.\n * @constructor\n */\nexport class FirebaseRemoteConfigError extends PrefixedFirebaseError {\n  constructor(code: RemoteConfigErrorCode, message: string) {\n    super('remote-config', code, message);\n  }\n}\n\n// Path: src/remote-config/condition-evaluator-internal.ts\n/*!\n * Copyright 2024 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n'use strict';\n\nimport {\n  AndCondition,\n  OneOfCondition,\n  EvaluationContext,\n  NamedCondition,\n  OrCondition,\n  PercentCondition,\n  PercentConditionOperator\n} from './remote-config-api';\nimport * as farmhash from 'farmhash';\nimport long = require('long');\n\n/**\n * Encapsulates condition evaluation logic to simplify organization and\n * facilitate testing.\n *\n * @internal\n */\nexport class ConditionEvaluator {\n  private static MAX_CONDITION_RECURSION_DEPTH = 10;\n\n  public evaluateConditions(\n    namedConditions: NamedCondition[],\n...\n", "needle_token_start": 5569, "needle_token_end": 5643, "code_context_ntokens": 15977, "output": ["```typescript\n/**\n * Gets the {@link Messaging} service for the default app or a given app.\n *\n * `admin.messaging()` can be called with no arguments to access the default\n * app's `Messaging` service or as `admin.messaging(app)` to access the\n * `Messaging` service associated with aspecific app.\n *\n * @example\n * ```javascript\n * // Get the Messaging service for the default app\n * const defaultMessaging = getMessaging();\n * ```\n *\n * @example\n * ```javascript\n * // Get the Messaging service for a given app\n * const otherMessaging = getMessaging(otherApp);\n * ```\n *\n * @param app - Optional app whose `Messaging` service to\n *   return. If not provided, the default `Messaging` service will be returned.\n *\n * @returns The default `Messaging` service if no\n *   app is provided or the `Messaging` service associated with the provided\n *   app.\n */\nexport function getMessaging(app?: App): Messaging {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('messaging', (app) => new Messaging(app));\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "emulatorHost", "language": "typescript", "path": "src/auth/auth-api-request.ts", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function retrieves the environment variable that specifies the host address for a Firebase Auth emulator, allowing the application to connect to a simulated Firebase Auth environment for development and testing purposes.\n2. **Input**: There are no parameters taken by the function.\n3. **Output**: Returns the host address as a string if the environment variable is set; otherwise, it returns undefined.\n4. **Procedure**: The function checks the environment for a specific variable that holds the emulator host address and returns its value. If the variable is not set, it returns undefined.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "      } catch (e) {\n        return Promise.reject(e);\n      }\n    }\n    if (requestType === 'VERIFY_AND_CHANGE_EMAIL' && typeof newEmail === 'undefined') {\n      return Promise.reject(\n        new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_ARGUMENT,\n          \"`newEmail` is required when `requestType` === 'VERIFY_AND_CHANGE_EMAIL'\",\n        ),\n      );\n    }\n    return this.invokeRequestHandler(this.getAuthUrlBuilder(), FIREBASE_AUTH_GET_OOB_CODE, request)\n      .then((response: any) => {\n        // Return the link.\n        return response.oobLink as string;\n      });\n  }\n\n  /**\n   * Looks up an OIDC provider configuration by provider ID.\n   *\n   * @param providerId - The provider identifier of the configuration to lookup.\n   * @returns A promise that resolves with the provider configuration information.\n   */\n  public getOAuthIdpConfig(providerId: string): Promise<OIDCConfigServerResponse> {\n    if (!OIDCConfig.isProviderId(providerId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID));\n    }\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), GET_OAUTH_IDP_CONFIG, {}, { providerId });\n  }\n\n  /**\n   * Lists the OIDC configurations (single batch only) with a size of maxResults and starting from\n   * the offset as specified by pageToken.\n   *\n   * @param maxResults - The page size, 100 if undefined. This is also the maximum\n   *     allowed limit.\n   * @param pageToken - The next page token. If not specified, returns OIDC configurations\n   *     without any offset. Configurations are returned in the order they were created from oldest to\n   *     newest, relative to the page token offset.\n   * @returns A promise that resolves with the current batch of downloaded\n   *     OIDC configurations and the next page token if available. For the last page, an empty list of provider\n   *     configuration and no page token are returned.\n   */\n  public listOAuthIdpConfigs(\n    maxResults: number = MAX_LIST_PROVIDER_CONFIGURATION_PAGE_SIZE,\n    pageToken?: string): Promise<object> {\n    const request: {pageSize: number; pageToken?: string} = {\n      pageSize: maxResults,\n    };\n    // Add next page token if provided.\n    if (typeof pageToken !== 'undefined') {\n      request.pageToken = pageToken;\n    }\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), LIST_OAUTH_IDP_CONFIGS, request)\n      .then((response: any) => {\n        if (!response.oauthIdpConfigs) {\n          response.oauthIdpConfigs = [];\n          delete response.nextPageToken;\n        }\n        return response as {oauthIdpConfigs: object[]; nextPageToken?: string};\n      });\n  }\n\n  /**\n   * Deletes an OIDC configuration identified by a providerId.\n   *\n   * @param providerId - The identifier of the OIDC configuration to delete.\n   * @returns A promise that resolves when the OIDC provider is deleted.\n   */\n  public deleteOAuthIdpConfig(providerId: string): Promise<void> {\n    if (!OIDCConfig.isProviderId(providerId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID));\n    }\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), DELETE_OAUTH_IDP_CONFIG, {}, { providerId })\n      .then(() => {\n        // Return nothing.\n      });\n  }\n\n  /**\n   * Creates a new OIDC provider configuration with the properties provided.\n   *\n   * @param options - The properties to set on the new OIDC provider configuration to be created.\n   * @returns A promise that resolves with the newly created OIDC\n   *     configuration.\n   */\n  public createOAuthIdpConfig(options: OIDCAuthProviderConfig): Promise<OIDCConfigServerResponse> {\n    // Construct backend request.\n    let request;\n    try {\n      request = OIDCConfig.buildServerRequest(options) || {};\n    } catch (e) {\n      return Promise.reject(e);\n    }\n    const providerId = options.providerId;\n    return this.invokeRequestHandler(\n      this.getProjectConfigUrlBuilder(), CREATE_OAUTH_IDP_CONFIG, request, { providerId })\n      .then((response: any) => {\n        if (!OIDCConfig.getProviderIdFromResourceName(response.name)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INTERNAL_ERROR,\n            'INTERNAL ASSERT FAILED: Unable to create new OIDC provider configuration');\n        }\n        return response as OIDCConfigServerResponse;\n      });\n  }\n\n  /**\n   * Updates an existing OIDC provider configuration with the properties provided.\n   *\n   * @param providerId - The provider identifier of the OIDC configuration to update.\n   * @param options - The properties to update on the existing configuration.\n   * @returns A promise that resolves with the modified provider\n   *     configuration.\n   */\n  public updateOAuthIdpConfig(\n    providerId: string, options: OIDCUpdateAuthProviderRequest): Promise<OIDCConfigServerResponse> {\n    if (!OIDCConfig.isProviderId(providerId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID));\n    }\n    // Construct backend request.\n    let request: OIDCConfigServerRequest;\n    try {\n      request = OIDCConfig.buildServerRequest(options, true) || {};\n    } catch (e) {\n      return Promise.reject(e);\n    }\n    const updateMask = utils.generateUpdateMask(request);\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), UPDATE_OAUTH_IDP_CONFIG, request,\n      { providerId, updateMask: updateMask.join(',') })\n      .then((response: any) => {\n        if (!OIDCConfig.getProviderIdFromResourceName(response.name)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INTERNAL_ERROR,\n            'INTERNAL ASSERT FAILED: Unable to update OIDC provider configuration');\n        }\n        return response as OIDCConfigServerResponse;\n      });\n  }\n\n  /**\n   * Looks up an SAML provider configuration by provider ID.\n   *\n   * @param providerId - The provider identifier of the configuration to lookup.\n   * @returns A promise that resolves with the provider configuration information.\n   */\n  public getInboundSamlConfig(providerId: string): Promise<SAMLConfigServerResponse> {\n    if (!SAMLConfig.isProviderId(providerId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID));\n    }\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), GET_INBOUND_SAML_CONFIG, {}, { providerId });\n  }\n\n  /**\n   * Lists the SAML configurations (single batch only) with a size of maxResults and starting from\n   * the offset as specified by pageToken.\n   *\n   * @param maxResults - The page size, 100 if undefined. This is also the maximum\n   *     allowed limit.\n   * @param pageToken - The next page token. If not specified, returns SAML configurations starting\n   *     without any offset. Configurations are returned in the order they were created from oldest to\n   *     newest, relative to the page token offset.\n   * @returns A promise that resolves with the current batch of downloaded\n   *     SAML configurations and the next page token if available. For the last page, an empty list of provider\n   *     configuration and no page token are returned.\n   */\n  public listInboundSamlConfigs(\n    maxResults: number = MAX_LIST_PROVIDER_CONFIGURATION_PAGE_SIZE,\n    pageToken?: string): Promise<object> {\n    const request: {pageSize: number; pageToken?: string} = {\n      pageSize: maxResults,\n    };\n    // Add next page token if provided.\n    if (typeof pageToken !== 'undefined') {\n      request.pageToken = pageToken;\n    }\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), LIST_INBOUND_SAML_CONFIGS, request)\n      .then((response: any) => {\n        if (!response.inboundSamlConfigs) {\n          response.inboundSamlConfigs = [];\n          delete response.nextPageToken;\n        }\n        return response as {inboundSamlConfigs: object[]; nextPageToken?: string};\n      });\n  }\n\n  /**\n   * Deletes a SAML configuration identified by a providerId.\n   *\n   * @param providerId - The identifier of the SAML configuration to delete.\n   * @returns A promise that resolves when the SAML provider is deleted.\n   */\n  public deleteInboundSamlConfig(providerId: string): Promise<void> {\n    if (!SAMLConfig.isProviderId(providerId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID));\n    }\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), DELETE_INBOUND_SAML_CONFIG, {}, { providerId })\n      .then(() => {\n        // Return nothing.\n      });\n  }\n\n  /**\n   * Creates a new SAML provider configuration with the properties provided.\n   *\n   * @param options - The properties to set on the new SAML provider configuration to be created.\n   * @returns A promise that resolves with the newly created SAML\n   *     configuration.\n   */\n  public createInboundSamlConfig(options: SAMLAuthProviderConfig): Promise<SAMLConfigServerResponse> {\n    // Construct backend request.\n    let request;\n    try {\n      request = SAMLConfig.buildServerRequest(options) || {};\n    } catch (e) {\n      return Promise.reject(e);\n    }\n    const providerId = options.providerId;\n    return this.invokeRequestHandler(\n      this.getProjectConfigUrlBuilder(), CREATE_INBOUND_SAML_CONFIG, request, { providerId })\n      .then((response: any) => {\n        if (!SAMLConfig.getProviderIdFromResourceName(response.name)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INTERNAL_ERROR,\n            'INTERNAL ASSERT FAILED: Unable to create new SAML provider configuration');\n        }\n        return response as SAMLConfigServerResponse;\n      });\n  }\n\n  /**\n   * Updates an existing SAML provider configuration with the properties provided.\n   *\n   * @param providerId - The provider identifier of the SAML configuration to update.\n   * @param options - The properties to update on the existing configuration.\n   * @returns A promise that resolves with the modified provider\n   *     configuration.\n   */\n  public updateInboundSamlConfig(\n    providerId: string, options: SAMLUpdateAuthProviderRequest): Promise<SAMLConfigServerResponse> {\n    if (!SAMLConfig.isProviderId(providerId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID));\n    }\n    // Construct backend request.\n    let request: SAMLConfigServerRequest;\n    try {\n      request = SAMLConfig.buildServerRequest(options, true) || {};\n    } catch (e) {\n      return Promise.reject(e);\n    }\n    const updateMask = utils.generateUpdateMask(request);\n    return this.invokeRequestHandler(this.getProjectConfigUrlBuilder(), UPDATE_INBOUND_SAML_CONFIG, request,\n      { providerId, updateMask: updateMask.join(',') })\n      .then((response: any) => {\n        if (!SAMLConfig.getProviderIdFromResourceName(response.name)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INTERNAL_ERROR,\n            'INTERNAL ASSERT FAILED: Unable to update SAML provider configuration');\n        }\n        return response as SAMLConfigServerResponse;\n      });\n  }\n\n  /**\n   * Invokes the request handler based on the API settings object passed.\n   *\n   * @param urlBuilder - The URL builder for Auth endpoints.\n   * @param apiSettings - The API endpoint settings to apply to request and response.\n   * @param requestData - The request data.\n   * @param additionalResourceParams - Additional resource related params if needed.\n   * @returns A promise that resolves with the response.\n   */\n  protected invokeRequestHandler(\n    urlBuilder: AuthResourceUrlBuilder, apiSettings: ApiSettings,\n    requestData: object | undefined, additionalResourceParams?: object): Promise<object> {\n    return urlBuilder.getUrl(apiSettings.getEndpoint(), additionalResourceParams)\n      .then((url) => {\n        // Validate request.\n        if (requestData) {\n          const requestValidator = apiSettings.getRequestValidator();\n          requestValidator(requestData);\n        }\n        // Process request.\n        const req: HttpRequestConfig = {\n          method: apiSettings.getHttpMethod(),\n          url,\n          headers: FIREBASE_AUTH_HEADER,\n          data: requestData,\n          timeout: FIREBASE_AUTH_TIMEOUT,\n        };\n        return this.httpClient.send(req);\n      })\n      .then((response) => {\n        // Validate response.\n        const responseValidator = apiSettings.getResponseValidator();\n        responseValidator(response.data);\n        // Return entire response.\n        return response.data;\n      })\n      .catch((err) => {\n        if (err instanceof HttpError) {\n          const error = err.response.data;\n          const errorCode = AbstractAuthRequestHandler.getErrorCode(error);\n          if (!errorCode) {\n            throw new FirebaseAuthError(\n              AuthClientErrorCode.INTERNAL_ERROR,\n              'Error returned from server: ' + error + '. Additionally, an ' +\n              'internal error occurred while attempting to extract the ' +\n              'errorcode from the error.',\n            );\n          }\n          throw FirebaseAuthError.fromServerError(errorCode, /* message */ undefined, error);\n        }\n        throw err;\n      });\n  }\n\n  /**\n   * @returns A new Auth user management resource URL builder instance.\n   */\n  protected abstract newAuthUrlBuilder(): AuthResourceUrlBuilder;\n\n  /**\n   * @returns A new project config resource URL builder instance.\n   */\n  protected abstract newProjectConfigUrlBuilder(): AuthResourceUrlBuilder;\n\n  /**\n   * @returns The current Auth user management resource URL builder.\n   */\n  private getAuthUrlBuilder(): AuthResourceUrlBuilder {\n    if (!this.authUrlBuilder) {\n      this.authUrlBuilder = this.newAuthUrlBuilder();\n    }\n    return this.authUrlBuilder;\n  }\n\n  /**\n   * @returns The current project config resource URL builder.\n   */\n  private getProjectConfigUrlBuilder(): AuthResourceUrlBuilder {\n    if (!this.projectConfigUrlBuilder) {\n      this.projectConfigUrlBuilder = this.newProjectConfigUrlBuilder();\n    }\n    return this.projectConfigUrlBuilder;\n  }\n}\n\n/** Instantiates the getConfig endpoint settings. */\nconst GET_PROJECT_CONFIG = new ApiSettings('/config', 'GET')\n  .setResponseValidator((response: any) => {\n    // Response should always contain at least the config name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to get project config',\n      );\n    }\n  });\n\n/** Instantiates the updateConfig endpoint settings. */\nconst UPDATE_PROJECT_CONFIG = new ApiSettings('/config?updateMask={updateMask}', 'PATCH')\n  .setResponseValidator((response: any) => {\n    // Response should always contain at least the config name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to update project config',\n      );\n    }\n  });\n\n/** Instantiates the getTenant endpoint settings. */\nconst GET_TENANT = new ApiSettings('/tenants/{tenantId}', 'GET')\n// Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain at least the tenant name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to get tenant',\n      );\n    }\n  });\n\n/** Instantiates the deleteTenant endpoint settings. */\nconst DELETE_TENANT = new ApiSettings('/tenants/{tenantId}', 'DELETE');\n\n/** Instantiates the updateTenant endpoint settings. */\nconst UPDATE_TENANT = new ApiSettings('/tenants/{tenantId}?updateMask={updateMask}', 'PATCH')\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain at least the tenant name.\n    if (!validator.isNonEmptyString(response.name) ||\n          !Tenant.getTenantIdFromResourceName(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to update tenant',\n      );\n    }\n  });\n\n/** Instantiates the listTenants endpoint settings. */\nconst LIST_TENANTS = new ApiSettings('/tenants', 'GET')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    // Validate next page token.\n    if (typeof request.pageToken !== 'undefined' &&\n        !validator.isNonEmptyString(request.pageToken)) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PAGE_TOKEN);\n    }\n    // Validate max results.\n    if (!validator.isNumber(request.pageSize) ||\n        request.pageSize <= 0 ||\n        request.pageSize > MAX_LIST_TENANT_PAGE_SIZE) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'Required \"maxResults\" must be a positive non-zero number that does not exceed ' +\n        `the allowed ${MAX_LIST_TENANT_PAGE_SIZE}.`,\n      );\n    }\n  });\n\n/** Instantiates the createTenant endpoint settings. */\nconst CREATE_TENANT = new ApiSettings('/tenants', 'POST')\n// Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain at least the tenant name.\n    if (!validator.isNonEmptyString(response.name) ||\n          !Tenant.getTenantIdFromResourceName(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to create new tenant',\n      );\n    }\n  });\n\n\n/**\n * Utility for sending requests to Auth server that are Auth instance related. This includes user, tenant,\n * and project config management related APIs. This extends the BaseFirebaseAuthRequestHandler class and defines\n * additional tenant management related APIs.\n */\nexport class AuthRequestHandler extends AbstractAuthRequestHandler {\n\n  protected readonly authResourceUrlBuilder: AuthResourceUrlBuilder;\n\n  /**\n   * The FirebaseAuthRequestHandler constructor used to initialize an instance using a FirebaseApp.\n   *\n   * @param app - The app used to fetch access tokens to sign API requests.\n   * @constructor\n   */\n  constructor(app: App) {\n    super(app);\n    this.authResourceUrlBuilder =  new AuthResourceUrlBuilder(app, 'v2');\n  }\n\n  /**\n   * @returns A new Auth user management resource URL builder instance.\n   */\n  protected newAuthUrlBuilder(): AuthResourceUrlBuilder {\n    return new AuthResourceUrlBuilder(this.app, 'v1');\n  }\n\n  /**\n   * @returns A new project config resource URL builder instance.\n   */\n  protected newProjectConfigUrlBuilder(): AuthResourceUrlBuilder {\n    return new AuthResourceUrlBuilder(this.app, 'v2');\n  }\n\n  /**\n   * Get the current project's config\n   * @returns A promise that resolves with the project config information.\n   */\n  public getProjectConfig(): Promise<ProjectConfigServerResponse> {\n    return this.invokeRequestHandler(this.authResourceUrlBuilder, GET_PROJECT_CONFIG, {}, {})\n      .then((response: any) => {\n        return response as ProjectConfigServerResponse;\n      });\n  }\n\n  /**\n   * Update the current project's config.\n   * @returns A promise that resolves with the project config information.\n   */\n  public updateProjectConfig(options: UpdateProjectConfigRequest): Promise<ProjectConfigServerResponse> {\n    try {\n      const request = ProjectConfig.buildServerRequest(options);\n      const updateMask = utils.generateUpdateMask(request);\n      return this.invokeRequestHandler(\n        this.authResourceUrlBuilder, UPDATE_PROJECT_CONFIG, request, { updateMask: updateMask.join(',') })\n        .then((response: any) => {\n          return response as ProjectConfigServerResponse;\n        });\n    } catch (e) {\n      return Promise.reject(e);\n    }\n  }\n\n  /**\n   * Looks up a tenant by tenant ID.\n   *\n   * @param tenantId - The tenant identifier of the tenant to lookup.\n   * @returns A promise that resolves with the tenant information.\n   */\n  public getTenant(tenantId: string): Promise<TenantServerResponse> {\n    if (!validator.isNonEmptyString(tenantId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_TENANT_ID));\n    }\n    return this.invokeRequestHandler(this.authResourceUrlBuilder, GET_TENANT, {}, { tenantId })\n      .then((response: any) => {\n        return response as TenantServerResponse;\n      });\n  }\n\n  /**\n   * Exports the tenants (single batch only) with a size of maxResults and starting from\n   * the offset as specified by pageToken.\n   *\n   * @param maxResults - The page size, 1000 if undefined. This is also the maximum\n   *     allowed limit.\n   * @param pageToken - The next page token. If not specified, returns tenants starting\n   *     without any offset. Tenants are returned in the order they were created from oldest to\n   *     newest, relative to the page token offset.\n   * @returns A promise that resolves with the current batch of downloaded\n   *     tenants and the next page token if available. For the last page, an empty list of tenants\n   *     and no page token are returned.\n   */\n  public listTenants(\n    maxResults: number = MAX_LIST_TENANT_PAGE_SIZE,\n    pageToken?: string): Promise<{tenants: TenantServerResponse[]; nextPageToken?: string}> {\n    const request = {\n      pageSize: maxResults,\n      pageToken,\n    };\n    // Remove next page token if not provided.\n    if (typeof request.pageToken === 'undefined') {\n      delete request.pageToken;\n    }\n    return this.invokeRequestHandler(this.authResourceUrlBuilder, LIST_TENANTS, request)\n      .then((response: any) => {\n        if (!response.tenants) {\n          response.tenants = [];\n          delete response.nextPageToken;\n        }\n        return response as {tenants: TenantServerResponse[]; nextPageToken?: string};\n      });\n  }\n\n  /**\n   * Deletes a tenant identified by a tenantId.\n   *\n   * @param tenantId - The identifier of the tenant to delete.\n   * @returns A promise that resolves when the tenant is deleted.\n   */\n  public deleteTenant(tenantId: string): Promise<void> {\n    if (!validator.isNonEmptyString(tenantId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_TENANT_ID));\n    }\n    return this.invokeRequestHandler(this.authResourceUrlBuilder, DELETE_TENANT, undefined, { tenantId })\n      .then(() => {\n        // Return nothing.\n      });\n  }\n\n  /**\n   * Creates a new tenant with the properties provided.\n   *\n   * @param tenantOptions - The properties to set on the new tenant to be created.\n   * @returns A promise that resolves with the newly created tenant object.\n   */\n  public createTenant(tenantOptions: CreateTenantRequest): Promise<TenantServerResponse> {\n    try {\n      // Construct backend request.\n      const request = Tenant.buildServerRequest(tenantOptions, true);\n      return this.invokeRequestHandler(this.authResourceUrlBuilder, CREATE_TENANT, request)\n        .then((response: any) => {\n          return response as TenantServerResponse;\n        });\n    } catch (e) {\n      return Promise.reject(e);\n    }\n  }\n\n  /**\n   * Updates an existing tenant with the properties provided.\n   *\n   * @param tenantId - The tenant identifier of the tenant to update.\n   * @param tenantOptions - The properties to update on the existing tenant.\n   * @returns A promise that resolves with the modified tenant object.\n   */\n  public updateTenant(tenantId: string, tenantOptions: UpdateTenantRequest): Promise<TenantServerResponse> {\n    if (!validator.isNonEmptyString(tenantId)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_TENANT_ID));\n    }\n    try {\n      // Construct backend request.\n      const request = Tenant.buildServerRequest(tenantOptions, false);\n      // Do not traverse deep into testPhoneNumbers. The entire content should be replaced\n      // and not just specific phone numbers.\n      const updateMask = utils.generateUpdateMask(request, ['testPhoneNumbers']);\n      return this.invokeRequestHandler(this.authResourceUrlBuilder, UPDATE_TENANT, request,\n        { tenantId, updateMask: updateMask.join(',') })\n        .then((response: any) => {\n          return response as TenantServerResponse;\n        });\n    } catch (e) {\n      return Promise.reject(e);\n    }\n  }\n}\n\n/**\n * Utility for sending requests to Auth server that are tenant Auth instance related. This includes user\n * management related APIs for specified tenants.\n * This extends the BaseFirebaseAuthRequestHandler class.\n */\nexport class TenantAwareAuthRequestHandler extends AbstractAuthRequestHandler {\n  /**\n   * The FirebaseTenantRequestHandler constructor used to initialize an instance using a\n   * FirebaseApp and a tenant ID.\n   *\n   * @param app - The app used to fetch access tokens to sign API requests.\n   * @param tenantId - The request handler's tenant ID.\n   * @constructor\n   */\n  constructor(app: App, private readonly tenantId: string) {\n    super(app);\n  }\n\n  /**\n   * @returns A new Auth user management resource URL builder instance.\n   */\n  protected newAuthUrlBuilder(): AuthResourceUrlBuilder {\n    return new TenantAwareAuthResourceUrlBuilder(this.app, 'v1', this.tenantId);\n  }\n\n  /**\n   * @returns A new project config resource URL builder instance.\n   */\n  protected newProjectConfigUrlBuilder(): AuthResourceUrlBuilder {\n    return new TenantAwareAuthResourceUrlBuilder(this.app, 'v2', this.tenantId);\n  }\n\n  /**\n   * Imports the list of users provided to Firebase Auth. This is useful when\n   * migrating from an external authentication system without having to use the Firebase CLI SDK.\n   * At most, 1000 users are allowed to be imported one at a time.\n   * When importing a list of password users, UserImportOptions are required to be specified.\n   *\n   * Overrides the superclass methods by adding an additional check to match tenant IDs of\n   * imported user records if present.\n   *\n   * @param users - The list of user records to import to Firebase Auth.\n   * @param options - The user import options, required when the users provided\n   *     include password credentials.\n   * @returns A promise that resolves when the operation completes\n   *     with the result of the import. This includes the number of successful imports, the number\n   *     of failed uploads and their corresponding errors.\n   */\n  public uploadAccount(\n    users: UserImportRecord[], options?: UserImportOptions): Promise<UserImportResult> {\n    // Add additional check to match tenant ID of imported user records.\n    users.forEach((user: UserImportRecord, index: number) => {\n      if (validator.isNonEmptyString(user.tenantId) &&\n          user.tenantId !== this.tenantId) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.MISMATCHING_TENANT_ID,\n          `UserRecord of index \"${index}\" has mismatching tenant ID \"${user.tenantId}\"`);\n      }\n    });\n    return super.uploadAccount(users, options);\n  }\n}\n\n\nfunction emulatorHost(): string | undefined {\n  return process.env.FIREBASE_AUTH_EMULATOR_HOST\n}\n\n/**\n * When true the SDK should communicate with the Auth Emulator for all API\n * calls and also produce unsigned tokens.\n */\nexport function useEmulator(): boolean {\n  return !!emulatorHost();\n}\n\n// Path: src/auth/project-config-manager.ts\n/*!\n * Copyright 2022 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\nimport { App } from '../app';\nimport { ProjectConfig, ProjectConfigServerResponse, UpdateProjectConfigRequest } from './project-config';\nimport {\n  AuthRequestHandler,\n} from './auth-api-request';\n\n/**\n * Manages (gets and updates) the current project config.\n */\nexport class ProjectConfigManager {\n  private readonly authRequestHandler: AuthRequestHandler;\n  /**\n   * Initializes a ProjectConfigManager instance for a specified FirebaseApp.\n   *\n   * @param app - The app for this ProjectConfigManager instance.\n   *\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    this.authRequestHandler = new AuthRequestHandler(app);\n  }\n\n  /**\n   * Get the project configuration.\n   *\n   * @returns A promise fulfilled with the project configuration.\n   */\n  public getProjectConfig(): Promise<ProjectConfig> {\n    return this.authRequestHandler.getProjectConfig()\n      .then((response: ProjectConfigServerResponse) => {\n        return new ProjectConfig(response);\n      })\n  }\n  /**\n   * Updates an existing project configuration.\n   *\n   * @param projectConfigOptions - The properties to update on the project.\n   *\n   * @returns A promise fulfilled with the updated project config.\n   */\n  public updateProjectConfig(projectConfigOptions: UpdateProjectConfigRequest): Promise<ProjectConfig> {\n    return this.authRequestHandler.updateProjectConfig(projectConfigOptions)\n      .then((response: ProjectConfigServerResponse) => {\n        return new ProjectConfig(response);\n      })\n  }\n}\n\n// Path: src/auth/token-verifier.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { AuthClientErrorCode, FirebaseAuthError, ErrorInfo } from '../utils/error';\nimport * as util from '../utils/index';\nimport * as validator from '../utils/validator';\nimport {\n  DecodedToken, decodeJwt, JwtError, JwtErrorCode, EmulatorSignatureVerifier,\n  PublicKeySignatureVerifier, ALGORITHM_RS256, SignatureVerifier,\n} from '../utils/jwt';\nimport { App } from '../app/index';\n\n/**\n * Interface representing a decoded Firebase ID token, returned from the\n * {@link BaseAuth.verifyIdToken} method.\n *\n * Firebase ID tokens are OpenID Connect spec-compliant JSON Web Tokens (JWTs).\n * See the\n * [ID Token section of the OpenID Connect spec](http://openid.net/specs/openid-connect-core-1_0.html#IDToken)\n * for more information about the specific properties below.\n */\nexport interface DecodedIdToken {\n\n  /**\n   * The audience for which this token is intended.\n   *\n   * This value is a string equal to your Firebase project ID, the unique\n   * identifier for your Firebase project, which can be found in [your project's\n   * settings](https://console.firebase.google.com/project/_/settings/general/android:com.random.android).\n   */\n  aud: string;\n\n  /**\n   * Time, in seconds since the Unix epoch, when the end-user authentication\n   * occurred.\n   *\n   * This value is not set when this particular ID token was created, but when the\n   * user initially logged in to this session. In a single session, the Firebase\n   * SDKs will refresh a user's ID tokens every hour. Each ID token will have a\n   * different [`iat`](#iat) value, but the same `auth_time` value.\n   */\n  auth_time: number;\n\n  /**\n   * The email of the user to whom the ID token belongs, if available.\n   */\n  email?: string;\n\n  /**\n   * Whether or not the email of the user to whom the ID token belongs is\n   * verified, provided the user has an email.\n   */\n  email_verified?: boolean;\n\n  /**\n   * The ID token's expiration time, in seconds since the Unix epoch. That is, the\n   * time at which this ID token expires and should no longer be considered valid.\n   *\n   * The Firebase SDKs transparently refresh ID tokens every hour, issuing a new\n   * ID token with up to a one hour expiration.\n   */\n  exp: number;\n\n  /**\n   * Information about the sign in event, including which sign in provider was\n   * used and provider-specific identity details.\n   *\n   * This data is provided by the Firebase Authentication service and is a\n   * reserved claim in the ID token.\n   */\n  firebase: {\n\n    /**\n     * Provider-specific identity details corresponding\n     * to the provider used to sign in the user.\n     */\n    identities: {\n      [key: string]: any;\n    };\n\n    /**\n     * The ID of the provider used to sign in the user.\n     * One of `\"anonymous\"`, `\"password\"`, `\"facebook.com\"`, `\"github.com\"`,\n     * `\"google.com\"`, `\"twitter.com\"`, `\"apple.com\"`, `\"microsoft.com\"`,\n     * `\"yahoo.com\"`, `\"phone\"`, `\"playgames.google.com\"`, `\"gc.apple.com\"`,\n     * or `\"custom\"`.\n     *\n     * Additional Identity Platform provider IDs include `\"linkedin.com\"`,\n     * OIDC and SAML identity providers prefixed with `\"saml.\"` and `\"oidc.\"`\n     * respectively.\n     */\n    sign_in_provider: string;\n\n    /**\n     * The type identifier or `factorId` of the second factor, provided the\n     * ID token was obtained from a multi-factor authenticated user.\n     * For phone, this is `\"phone\"`.\n     */\n    sign_in_second_factor?: string;\n\n    /**\n     * The `uid` of the second factor used to sign in, provided the\n     * ID token was obtained from a multi-factor authenticated user.\n     */\n    second_factor_identifier?: string;\n\n    /**\n     * The ID of the tenant the user belongs to, if available.\n     */\n    tenant?: string;\n    [key: string]: any;\n  };\n\n  /**\n   * The ID token's issued-at time, in seconds since the Unix epoch. That is, the\n   * time at which this ID token was issued and should start to be considered\n   * valid.\n   *\n   * The Firebase SDKs transparently refresh ID tokens every hour, issuing a new\n   * ID token with a new issued-at time. If you want to get the time at which the\n   * user session corresponding to the ID token initially occurred, see the\n   * [`auth_time`](#auth_time) property.\n   */\n  iat: number;\n\n  /**\n   * The issuer identifier for the issuer of the response.\n   *\n   * This value is a URL with the format\n   * `https://securetoken.google.com/<PROJECT_ID>`, where `<PROJECT_ID>` is the\n   * same project ID specified in the [`aud`](#aud) property.\n   */\n  iss: string;\n\n  /**\n   * The phone number of the user to whom the ID token belongs, if available.\n   */\n  phone_number?: string;\n\n  /**\n   * The photo URL for the user to whom the ID token belongs, if available.\n   */\n  picture?: string;\n\n  /**\n   * The `uid` corresponding to the user who the ID token belonged to.\n   *\n   * As a convenience, this value is copied over to the [`uid`](#uid) property.\n   */\n  sub: string;\n\n  /**\n   * The `uid` corresponding to the user who the ID token belonged to.\n   *\n   * This value is not actually in the JWT token claims itself. It is added as a\n   * convenience, and is set as the value of the [`sub`](#sub) property.\n   */\n  uid: string;\n\n  /**\n   * Other arbitrary claims included in the ID token.\n   */\n  [key: string]: any;\n}\n\n/** @alpha */\nexport interface DecodedAuthBlockingSharedUserInfo {\n  uid: string;\n  display_name?: string;\n  email?: string;\n  photo_url?: string;\n  phone_number?: string;\n}\n\n/** @alpha */\nexport interface DecodedAuthBlockingMetadata {\n  creation_time?: number;\n  last_sign_in_time?: number;\n}\n\n/** @alpha */\nexport interface DecodedAuthBlockingUserInfo extends DecodedAuthBlockingSharedUserInfo {\n  provider_id: string;\n}\n\n/** @alpha */\nexport interface DecodedAuthBlockingMfaInfo {\n  uid: string;\n  display_name?: string;\n  phone_number?: string;\n  enrollment_time?: string;\n  factor_id?: string;\n}\n\n/** @alpha */\nexport interface DecodedAuthBlockingEnrolledFactors {\n  enrolled_factors?: DecodedAuthBlockingMfaInfo[];\n}\n\n/** @alpha */\nexport interface DecodedAuthBlockingUserRecord extends DecodedAuthBlockingSharedUserInfo {\n  email_verified?: boolean;\n  disabled?: boolean;\n  metadata?: DecodedAuthBlockingMetadata;\n  password_hash?: string;\n  password_salt?: string;\n  provider_data?: DecodedAuthBlockingUserInfo[];\n  multi_factor?: DecodedAuthBlockingEnrolledFactors;\n  custom_claims?: any;\n  tokens_valid_after_time?: number;\n  tenant_id?: string;\n  [key: string]: any;\n}\n\n/** @alpha */\nexport interface DecodedAuthBlockingToken {\n  aud: string;\n  exp: number;\n  iat: number;\n  iss: string;\n  sub: string;\n  event_id: string;\n  event_type: string;\n  ip_address: string;\n  user_agent?: string;\n  locale?: string;\n  sign_in_method?: string;\n  user_record?: DecodedAuthBlockingUserRecord;\n  tenant_id?: string;\n  raw_user_info?: string;\n  sign_in_attributes?: {\n    [key: string]: any;\n  };\n  oauth_id_token?: string;\n  oauth_access_token?: string;\n  oauth_refresh_token?: string;\n  oauth_token_secret?: string;\n  oauth_expires_in?: number;\n  [key: string]: any;\n}\n\n// Audience to use for Firebase Auth Custom tokens\nconst FIREBASE_AUDIENCE = 'https://identitytoolkit.googleapis.com/google.identity.identitytoolkit.v1.IdentityToolkit';\n\n// URL containing the public keys for the Google certs (whose private keys are used to sign Firebase\n// Auth ID tokens)\nconst CLIENT_CERT_URL = 'https://www.googleapis.com/robot/v1/metadata/x509/securetoken@system.gserviceaccount.com';\n\n// URL containing the public keys for Firebase session cookies. This will be updated to a different URL soon.\nconst SESSION_COOKIE_CERT_URL = 'https://www.googleapis.com/identitytoolkit/v3/relyingparty/publicKeys';\n\nconst EMULATOR_VERIFIER = new EmulatorSignatureVerifier();\n\n/**\n * User facing token information related to the Firebase ID token.\n *\n * @internal\n */\nexport const ID_TOKEN_INFO: FirebaseTokenInfo = {\n  url: 'https://firebase.google.com/docs/auth/admin/verify-id-tokens',\n  verifyApiName: 'verifyIdToken()',\n  jwtName: 'Firebase ID token',\n  shortName: 'ID token',\n  expiredErrorCode: AuthClientErrorCode.ID_TOKEN_EXPIRED,\n};\n\n/**\n * User facing token information related to the Firebase Auth Blocking token.\n *\n * @internal\n */\nexport const AUTH_BLOCKING_TOKEN_INFO: FirebaseTokenInfo = {\n  url: 'https://cloud.google.com/identity-platform/docs/blocking-functions',\n  verifyApiName: '_verifyAuthBlockingToken()',\n  jwtName: 'Firebase Auth Blocking token',\n  shortName: 'Auth Blocking token',\n  expiredErrorCode: AuthClientErrorCode.AUTH_BLOCKING_TOKEN_EXPIRED,\n};\n\n/**\n * User facing token information related to the Firebase session cookie.\n *\n * @internal\n */\nexport const SESSION_COOKIE_INFO: FirebaseTokenInfo = {\n  url: 'https://firebase.google.com/docs/auth/admin/manage-cookies',\n  verifyApiName: 'verifySessionCookie()',\n  jwtName: 'Firebase session cookie',\n  shortName: 'session cookie',\n  expiredErrorCode: AuthClientErrorCode.SESSION_COOKIE_EXPIRED,\n};\n\n/**\n * Interface that defines token related user facing information.\n *\n * @internal\n */\nexport interface FirebaseTokenInfo {\n  /** Documentation URL. */\n  url: string;\n  /** verify API name. */\n  verifyApiName: string;\n  /** The JWT full name. */\n  jwtName: string;\n  /** The JWT short name. */\n  shortName: string;\n  /** JWT Expiration error code. */\n  expiredErrorCode: ErrorInfo;\n}\n\n/**\n * Class for verifying general purpose Firebase JWTs. This verifies ID tokens and session cookies.\n *\n * @internal\n */\nexport class FirebaseTokenVerifier {\n\n  private readonly shortNameArticle: string;\n  private readonly signatureVerifier: SignatureVerifier;\n\n  constructor(clientCertUrl: string, private issuer: string, private tokenInfo: FirebaseTokenInfo,\n              private readonly app: App) {\n\n    if (!validator.isURL(clientCertUrl)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The provided public client certificate URL is an invalid URL.',\n      );\n    } else if (!validator.isURL(issuer)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The provided JWT issuer is an invalid URL.',\n      );\n    } else if (!validator.isNonNullObject(tokenInfo)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The provided JWT information is not an object or null.',\n      );\n    } else if (!validator.isURL(tokenInfo.url)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The provided JWT verification documentation URL is invalid.',\n      );\n    } else if (!validator.isNonEmptyString(tokenInfo.verifyApiName)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The JWT verify API name must be a non-empty string.',\n      );\n    } else if (!validator.isNonEmptyString(tokenInfo.jwtName)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The JWT public full name must be a non-empty string.',\n      );\n    } else if (!validator.isNonEmptyString(tokenInfo.shortName)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The JWT public short name must be a non-empty string.',\n      );\n    } else if (!validator.isNonNullObject(tokenInfo.expiredErrorCode) || !('code' in tokenInfo.expiredErrorCode)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'The JWT expiration error code must be a non-null ErrorInfo object.',\n      );\n    }\n    this.shortNameArticle = tokenInfo.shortName.charAt(0).match(/[aeiou]/i) ? 'an' : 'a';\n\n    this.signatureVerifier =\n      PublicKeySignatureVerifier.withCertificateUrl(clientCertUrl, app.options.httpAgent);\n\n    // For backward compatibility, the project ID is validated in the verification call.\n  }\n\n  /**\n   * Verifies the format and signature of a Firebase Auth JWT token.\n   *\n   * @param jwtToken - The Firebase Auth JWT token to verify.\n   * @param isEmulator - Whether to accept Auth Emulator tokens.\n   * @returns A promise fulfilled with the decoded claims of the Firebase Auth ID token.\n   */\n  public verifyJWT(jwtToken: string, isEmulator = false): Promise<DecodedIdToken> {\n    if (!validator.isString(jwtToken)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        `First argument to ${this.tokenInfo.verifyApiName} must be a ${this.tokenInfo.jwtName} string.`,\n      );\n    }\n\n    return this.ensureProjectId()\n      .then((projectId) => {\n        return this.decodeAndVerify(jwtToken, projectId, isEmulator);\n      })\n      .then((decoded) => {\n        const decodedIdToken = decoded.payload as DecodedIdToken;\n        decodedIdToken.uid = decodedIdToken.sub;\n        return decodedIdToken;\n      });\n  }\n\n  /** @alpha */\n  // eslint-disable-next-line @typescript-eslint/naming-convention\n  public _verifyAuthBlockingToken(\n    jwtToken: string,\n    isEmulator: boolean,\n    audience: string | undefined): Promise<DecodedAuthBlockingToken> {\n    if (!validator.isString(jwtToken)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        `First argument to ${this.tokenInfo.verifyApiName} must be a ${this.tokenInfo.jwtName} string.`,\n      );\n    }\n\n    return this.ensureProjectId()\n      .then((projectId) => {\n        if (typeof audience === 'undefined') {\n          audience = `${projectId}.cloudfunctions.net/`;\n        }\n        return this.decodeAndVerify(jwtToken, projectId, isEmulator, audience);\n      })\n      .then((decoded) => {\n        const decodedAuthBlockingToken = decoded.payload as DecodedAuthBlockingToken;\n        decodedAuthBlockingToken.uid = decodedAuthBlockingToken.sub;\n        return decodedAuthBlockingToken;\n      });\n  }\n\n  private ensureProjectId(): Promise<string> {\n    return util.findProjectId(this.app)\n      .then((projectId) => {\n        if (!validator.isNonEmptyString(projectId)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_CREDENTIAL,\n            'Must initialize app with a cert credential or set your Firebase project ID as the ' +\n            `GOOGLE_CLOUD_PROJECT environment variable to call ${this.tokenInfo.verifyApiName}.`,\n          );\n        }\n        return Promise.resolve(projectId);\n      })\n  }\n\n  private decodeAndVerify(\n    token: string,\n    projectId: string,\n    isEmulator: boolean,\n    audience?: string): Promise<DecodedToken> {\n    return this.safeDecode(token)\n      .then((decodedToken) => {\n        this.verifyContent(decodedToken, projectId, isEmulator, audience);\n        return this.verifySignature(token, isEmulator)\n          .then(() => decodedToken);\n      });\n  }\n\n  private safeDecode(jwtToken: string): Promise<DecodedToken> {\n    return decodeJwt(jwtToken)\n      .catch((err: JwtError) => {\n        if (err.code === JwtErrorCode.INVALID_ARGUMENT) {\n          const verifyJwtTokenDocsMessage = ` See ${this.tokenInfo.url} ` +\n            `for details on how to retrieve ${this.shortNameArticle} ${this.tokenInfo.shortName}.`;\n          const errorMessage = `Decoding ${this.tokenInfo.jwtName} failed. Make sure you passed ` +\n            `the entire string JWT which represents ${this.shortNameArticle} ` +\n            `${this.tokenInfo.shortName}.` + verifyJwtTokenDocsMessage;\n          throw new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT,\n            errorMessage);\n        }\n        throw new FirebaseAuthError(AuthClientErrorCode.INTERNAL_ERROR, err.message);\n      });\n  }\n\n  /**\n   * Verifies the content of a Firebase Auth JWT.\n   *\n   * @param fullDecodedToken - The decoded JWT.\n   * @param projectId - The Firebase Project Id.\n   * @param isEmulator - Whether the token is an Emulator token.\n   */\n  private verifyContent(\n    fullDecodedToken: DecodedToken,\n    projectId: string | null,\n    isEmulator: boolean,\n    audience: string | undefined): void {\n    const header = fullDecodedToken && fullDecodedToken.header;\n    const payload = fullDecodedToken && fullDecodedToken.payload;\n\n    const projectIdMatchMessage = ` Make sure the ${this.tokenInfo.shortName} comes from the same ` +\n      'Firebase project as the service account used to authenticate this SDK.';\n    const verifyJwtTokenDocsMessage = ` See ${this.tokenInfo.url} ` +\n      `for details on how to retrieve ${this.shortNameArticle} ${this.tokenInfo.shortName}.`;\n\n    let errorMessage: string | undefined;\n    if (!isEmulator && typeof header.kid === 'undefined') {\n      const isCustomToken = (payload.aud === FIREBASE_AUDIENCE);\n      const isLegacyCustomToken = (header.alg === 'HS256' && payload.v === 0 && 'd' in payload && 'uid' in payload.d);\n\n      if (isCustomToken) {\n        errorMessage = `${this.tokenInfo.verifyApiName} expects ${this.shortNameArticle} ` +\n          `${this.tokenInfo.shortName}, but was given a custom token.`;\n      } else if (isLegacyCustomToken) {\n        errorMessage = `${this.tokenInfo.verifyApiName} expects ${this.shortNameArticle} ` +\n          `${this.tokenInfo.shortName}, but was given a legacy custom token.`;\n      } else {\n        errorMessage = `${this.tokenInfo.jwtName} has no \"kid\" claim.`;\n      }\n\n      errorMessage += verifyJwtTokenDocsMessage;\n    } else if (!isEmulator && header.alg !== ALGORITHM_RS256) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect algorithm. Expected \"` + ALGORITHM_RS256 + '\" but got ' +\n        '\"' + header.alg + '\".' + verifyJwtTokenDocsMessage;\n    } else if (typeof audience !== 'undefined' && !(payload.aud as string).includes(audience)) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect \"aud\" (audience) claim. Expected \"` +\n      audience + '\" but got \"' + payload.aud + '\".' + verifyJwtTokenDocsMessage;\n    } else if (typeof audience === 'undefined' && payload.aud !== projectId) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect \"aud\" (audience) claim. Expected \"` +\n      projectId + '\" but got \"' + payload.aud + '\".' + projectIdMatchMessage +\n        verifyJwtTokenDocsMessage;\n    } else if (payload.iss !== this.issuer + projectId) {\n      errorMessage = `${this.tokenInfo.jwtName} has incorrect \"iss\" (issuer) claim. Expected ` +\n        `\"${this.issuer}` + projectId + '\" but got \"' +\n        payload.iss + '\".' + projectIdMatchMessage + verifyJwtTokenDocsMessage;\n    } else if (!(payload.event_type !== undefined &&\n      (payload.event_type === 'beforeSendSms' || payload.event_type === 'beforeSendEmail'))) {\n      // excluding `beforeSendSms` and `beforeSendEmail` from processing `sub` as there is no user record available.\n      // `sub` is the same as `uid` which is part of the user record.\n      if (typeof payload.sub !== 'string') {\n        errorMessage = `${this.tokenInfo.jwtName} has no \"sub\" (subject) claim.` + verifyJwtTokenDocsMessage;\n      } else if (payload.sub === '') {\n        errorMessage = `${this.tokenInfo.jwtName} has an empty \"sub\" (subject) claim.` +\n          verifyJwtTokenDocsMessage;\n      } else if (payload.sub.length > 128) {\n        errorMessage = `${this.tokenInfo.jwtName} has a \"sub\" (subject) claim longer than 128 characters.` +\n          verifyJwtTokenDocsMessage;\n      }\n    }\n    if (errorMessage) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, errorMessage);\n    }\n  }\n\n  private verifySignature(jwtToken: string, isEmulator: boolean):\n    Promise<void> {\n    const verifier = isEmulator ? EMULATOR_VERIFIER : this.signatureVerifier;\n    return verifier.verify(jwtToken)\n      .catch((error) => {\n        throw this.mapJwtErrorToAuthError(error);\n      });\n  }\n\n  /**\n   * Maps JwtError to FirebaseAuthError\n   *\n   * @param error - JwtError to be mapped.\n   * @returns FirebaseAuthError or Error instance.\n   */\n  private mapJwtErrorToAuthError(error: JwtError): Error {\n    const verifyJwtTokenDocsMessage = ` See ${this.tokenInfo.url} ` +\n      `for details on how to retrieve ${this.shortNameArticle} ${this.tokenInfo.shortName}.`;\n    if (error.code === JwtErrorCode.TOKEN_EXPIRED) {\n      const errorMessage = `${this.tokenInfo.jwtName} has expired. Get a fresh ${this.tokenInfo.shortName}` +\n        ` from your client app and try again (auth/${this.tokenInfo.expiredErrorCode.code}).` +\n        verifyJwtTokenDocsMessage;\n      return new FirebaseAuthError(this.tokenInfo.expiredErrorCode, errorMessage);\n    } else if (error.code === JwtErrorCode.INVALID_SIGNATURE) {\n      const errorMessage = `${this.tokenInfo.jwtName} has invalid signature.` + verifyJwtTokenDocsMessage;\n      return new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, errorMessage);\n    } else if (error.code === JwtErrorCode.NO_MATCHING_KID) {\n      const errorMessage = `${this.tokenInfo.jwtName} has \"kid\" claim which does not ` +\n        `correspond to a known public key. Most likely the ${this.tokenInfo.shortName} ` +\n        'is expired, so get a fresh token from your client app and try again.';\n      return new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, errorMessage);\n    }\n    return new FirebaseAuthError(AuthClientErrorCode.INVALID_ARGUMENT, error.message);\n  }\n}\n\n/**\n * Creates a new FirebaseTokenVerifier to verify Firebase ID tokens.\n *\n * @internal\n * @param app - Firebase app instance.\n * @returns FirebaseTokenVerifier\n */\nexport function createIdTokenVerifier(app: App): FirebaseTokenVerifier {\n  return new FirebaseTokenVerifier(\n    CLIENT_CERT_URL,\n    'https://securetoken.google.com/',\n    ID_TOKEN_INFO,\n    app\n  );\n}\n\n/**\n * Creates a new FirebaseTokenVerifier to verify Firebase Auth Blocking tokens.\n *\n * @internal\n * @param app - Firebase app instance.\n * @returns FirebaseTokenVerifier\n */\nexport function createAuthBlockingTokenVerifier(app: App): FirebaseTokenVerifier {\n  return new FirebaseTokenVerifier(\n    CLIENT_CERT_URL,\n    'https://securetoken.google.com/',\n    AUTH_BLOCKING_TOKEN_INFO,\n    app\n  );\n}\n\n/**\n * Creates a new FirebaseTokenVerifier to verify Firebase session cookies.\n *\n * @internal\n * @param app - Firebase app instance.\n * @returns FirebaseTokenVerifier\n */\nexport function createSessionCookieVerifier(app: App): FirebaseTokenVerifier {\n  return new FirebaseTokenVerifier(\n    SESSION_COOKIE_CERT_URL,\n    'https://session.firebase.google.com/',\n    SESSION_COOKIE_INFO,\n    app\n  );\n}\n\n// Path: src/auth/token-generator.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { AuthClientErrorCode, ErrorInfo, FirebaseAuthError } from '../utils/error';\nimport { HttpError } from '../utils/api-request';\nimport { CryptoSigner, CryptoSignerError, CryptoSignerErrorCode } from '../utils/crypto-signer';\n\nimport * as validator from '../utils/validator';\nimport { toWebSafeBase64 } from '../utils';\nimport { Algorithm } from 'jsonwebtoken';\n\nconst ALGORITHM_NONE: Algorithm = 'none' as const;\n\nconst ONE_HOUR_IN_SECONDS = 60 * 60;\n\n// List of blacklisted claims which cannot be provided when creating a custom token\nexport const BLACKLISTED_CLAIMS = [\n  'acr', 'amr', 'at_hash', 'aud', 'auth_time', 'azp', 'cnf', 'c_hash', 'exp', 'iat', 'iss', 'jti',\n  'nbf', 'nonce',\n];\n\n// Audience to use for Firebase Auth Custom tokens\nconst FIREBASE_AUDIENCE = 'https://identitytoolkit.googleapis.com/google.identity.identitytoolkit.v1.IdentityToolkit';\n\n/**\n * Represents the header of a JWT.\n */\ninterface JWTHeader {\n  alg: string;\n  typ: string;\n}\n\n/**\n * Represents the body of a JWT.\n */\ninterface JWTBody {\n  claims?: object;\n  uid: string;\n  aud: string;\n  iat: number;\n  exp: number;\n  iss: string;\n  sub: string;\n  tenant_id?: string;\n}\n\n/**\n * A CryptoSigner implementation that is used when communicating with the Auth emulator.\n * It produces unsigned tokens.\n */\nexport class EmulatedSigner implements CryptoSigner {\n\n  algorithm = ALGORITHM_NONE;\n\n  /**\n   * @inheritDoc\n   */\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  public sign(buffer: Buffer): Promise<Buffer> {\n    return Promise.resolve(Buffer.from(''));\n  }\n\n  /**\n   * @inheritDoc\n   */\n  public getAccountId(): Promise<string> {\n    return Promise.resolve('firebase-auth-emulator@example.com');\n  }\n}\n\n/**\n * Class for generating different types of Firebase Auth tokens (JWTs).\n *\n * @internal\n */\nexport class FirebaseTokenGenerator {\n\n  private readonly signer: CryptoSigner;\n\n  /**\n   * @param tenantId - The tenant ID to use for the generated Firebase Auth\n   *     Custom token. If absent, then no tenant ID claim will be set in the\n   *     resulting JWT.\n   */\n  constructor(signer: CryptoSigner, public readonly tenantId?: string) {\n    if (!validator.isNonNullObject(signer)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CREDENTIAL,\n        'INTERNAL ASSERT: Must provide a CryptoSigner to use FirebaseTokenGenerator.',\n      );\n    }\n    if (typeof this.tenantId !== 'undefined' && !validator.isNonEmptyString(this.tenantId)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '`tenantId` argument must be a non-empty string.');\n    }\n    this.signer = signer;\n  }\n\n  /**\n   * Creates a new Firebase Auth Custom token.\n   *\n   * @param uid - The user ID to use for the generated Firebase Auth Custom token.\n   * @param developerClaims - Optional developer claims to include in the generated Firebase\n   *     Auth Custom token.\n   * @returns A Promise fulfilled with a Firebase Auth Custom token signed with a\n   *     service account key and containing the provided payload.\n   */\n  public createCustomToken(uid: string, developerClaims?: {[key: string]: any}): Promise<string> {\n    let errorMessage: string | undefined;\n    if (!validator.isNonEmptyString(uid)) {\n      errorMessage = '`uid` argument must be a non-empty string uid.';\n    } else if (uid.length > 128) {\n      errorMessage = '`uid` argument must a uid with less than or equal to 128 characters.';\n    } else if (!this.isDeveloperClaimsValid_(developerClaims)) {\n...\n", "needle_token_start": 7304, "needle_token_end": 7334, "code_context_ntokens": 16141, "output": ["```typescript\nfunction emulatorHost(): string | undefined {\n  return process.env.FIREBASE_AUTH_EMULATOR_HOST\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "validateProviderUserInfo", "language": "typescript", "path": "src/auth/auth-api-request.ts", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To ensure that a user information object from a specific provider contains only valid and supported fields, and to enforce data integrity by validating the values of these fields.\n2. **Input**: An object containing user information fields from a provider.\n3. **Output**: There is no direct output as it is a validation function; however, it may throw errors if the input contains invalid or unsupported fields.\n4. **Procedure**: \n   - The function iterates over each field in the input object.\n   - Fields not recognized as valid are removed.\n   - It checks if the provider ID and user ID are non-empty strings; if not, it throws an error.\n   - Optional fields such as display name, email, and photo URL are validated for their respective data types and formats (string, valid email, valid URL).\n   - If any validation fails, an error specific to the issue is thrown.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/auth/user-import-builder.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseArrayIndexError } from '../app/index';\nimport { deepCopy, deepExtend } from '../utils/deep-copy';\nimport * as utils from '../utils';\nimport * as validator from '../utils/validator';\nimport { AuthClientErrorCode, FirebaseAuthError } from '../utils/error';\nimport {\n  UpdateMultiFactorInfoRequest, UpdatePhoneMultiFactorInfoRequest, MultiFactorUpdateSettings\n} from './auth-config';\n\nexport type HashAlgorithmType = 'SCRYPT' | 'STANDARD_SCRYPT' | 'HMAC_SHA512' |\n  'HMAC_SHA256' | 'HMAC_SHA1' | 'HMAC_MD5' | 'MD5' | 'PBKDF_SHA1' | 'BCRYPT' |\n  'PBKDF2_SHA256' | 'SHA512' | 'SHA256' | 'SHA1';\n\n/**\n   * Interface representing the user import options needed for\n   * {@link BaseAuth.importUsers} method. This is used to\n   * provide the password hashing algorithm information.\n   */\nexport interface UserImportOptions {\n\n  /**\n   * The password hashing information.\n   */\n  hash: {\n\n    /**\n     * The password hashing algorithm identifier. The following algorithm\n     * identifiers are supported:\n     * `SCRYPT`, `STANDARD_SCRYPT`, `HMAC_SHA512`, `HMAC_SHA256`, `HMAC_SHA1`,\n     * `HMAC_MD5`, `MD5`, `PBKDF_SHA1`, `BCRYPT`, `PBKDF2_SHA256`, `SHA512`,\n     * `SHA256` and `SHA1`.\n     */\n    algorithm: HashAlgorithmType;\n\n    /**\n     * The signing key used in the hash algorithm in buffer bytes.\n     * Required by hashing algorithms `SCRYPT`, `HMAC_SHA512`, `HMAC_SHA256`,\n     * `HAMC_SHA1` and `HMAC_MD5`.\n     */\n    key?: Buffer;\n\n    /**\n     * The salt separator in buffer bytes which is appended to salt when\n     * verifying a password. This is only used by the `SCRYPT` algorithm.\n     */\n    saltSeparator?: Buffer;\n\n    /**\n     * The number of rounds for hashing calculation.\n     * Required for `SCRYPT`, `MD5`, `SHA512`, `SHA256`, `SHA1`, `PBKDF_SHA1` and\n     * `PBKDF2_SHA256`.\n     */\n    rounds?: number;\n\n    /**\n     * The memory cost required for `SCRYPT` algorithm, or the CPU/memory cost.\n     * Required for `STANDARD_SCRYPT` algorithm.\n     */\n    memoryCost?: number;\n\n    /**\n     * The parallelization of the hashing algorithm. Required for the\n     * `STANDARD_SCRYPT` algorithm.\n     */\n    parallelization?: number;\n\n    /**\n     * The block size (normally 8) of the hashing algorithm. Required for the\n     * `STANDARD_SCRYPT` algorithm.\n     */\n    blockSize?: number;\n    /**\n     * The derived key length of the hashing algorithm. Required for the\n     * `STANDARD_SCRYPT` algorithm.\n     */\n    derivedKeyLength?: number;\n  };\n}\n\n/**\n * Interface representing a user to import to Firebase Auth via the\n * {@link BaseAuth.importUsers} method.\n */\nexport interface UserImportRecord {\n\n  /**\n   * The user's `uid`.\n   */\n  uid: string;\n\n  /**\n   * The user's primary email, if set.\n   */\n  email?: string;\n\n  /**\n   * Whether or not the user's primary email is verified.\n   */\n  emailVerified?: boolean;\n\n  /**\n   * The user's display name.\n   */\n  displayName?: string;\n\n  /**\n   * The user's primary phone number, if set.\n   */\n  phoneNumber?: string;\n\n  /**\n   * The user's photo URL.\n   */\n  photoURL?: string;\n\n  /**\n   * Whether or not the user is disabled: `true` for disabled; `false` for\n   * enabled.\n   */\n  disabled?: boolean;\n\n  /**\n   * Additional metadata about the user.\n   */\n  metadata?: UserMetadataRequest;\n\n  /**\n   * An array of providers (for example, Google, Facebook) linked to the user.\n   */\n  providerData?: UserProviderRequest[];\n\n  /**\n   * The user's custom claims object if available, typically used to define\n   * user roles and propagated to an authenticated user's ID token.\n   */\n  customClaims?: { [key: string]: any };\n\n  /**\n   * The buffer of bytes representing the user's hashed password.\n   * When a user is to be imported with a password hash,\n   * {@link UserImportOptions} are required to be\n   * specified to identify the hashing algorithm used to generate this hash.\n   */\n  passwordHash?: Buffer;\n\n  /**\n   * The buffer of bytes representing the user's password salt.\n   */\n  passwordSalt?: Buffer;\n\n  /**\n   * The identifier of the tenant where user is to be imported to.\n   * When not provided in an `admin.auth.Auth` context, the user is uploaded to\n   * the default parent project.\n   * When not provided in an `admin.auth.TenantAwareAuth` context, the user is uploaded\n   * to the tenant corresponding to that `TenantAwareAuth` instance's tenant ID.\n   */\n  tenantId?: string;\n\n  /**\n   * The user's multi-factor related properties.\n   */\n  multiFactor?: MultiFactorUpdateSettings;\n}\n\n/**\n * User metadata to include when importing a user.\n */\nexport interface UserMetadataRequest {\n\n  /**\n   * The date the user last signed in, formatted as a UTC string.\n   */\n  lastSignInTime?: string;\n\n  /**\n   * The date the user was created, formatted as a UTC string.\n   */\n  creationTime?: string;\n}\n\n/**\n * User provider data to include when importing a user.\n */\nexport interface UserProviderRequest {\n\n  /**\n   * The user identifier for the linked provider.\n   */\n  uid: string;\n\n  /**\n   * The display name for the linked provider.\n   */\n  displayName?: string;\n\n  /**\n   * The email for the linked provider.\n   */\n  email?: string;\n\n  /**\n   * The phone number for the linked provider.\n   */\n  phoneNumber?: string;\n\n  /**\n   * The photo URL for the linked provider.\n   */\n  photoURL?: string;\n\n  /**\n   * The linked provider ID (for example, \"google.com\" for the Google provider).\n   */\n  providerId: string;\n}\n\n/**\n   * Interface representing the response from the\n   * {@link BaseAuth.importUsers} method for batch\n   * importing users to Firebase Auth.\n   */\nexport interface UserImportResult {\n\n  /**\n   * The number of user records that failed to import to Firebase Auth.\n   */\n  failureCount: number;\n\n  /**\n   * The number of user records that successfully imported to Firebase Auth.\n   */\n  successCount: number;\n\n  /**\n   * An array of errors corresponding to the provided users to import. The\n   * length of this array is equal to [`failureCount`](#failureCount).\n   */\n  errors: FirebaseArrayIndexError[];\n}\n\n/** Interface representing an Auth second factor in Auth server format. */\nexport interface AuthFactorInfo {\n  // Not required for signupNewUser endpoint.\n  mfaEnrollmentId?: string;\n  displayName?: string;\n  phoneInfo?: string;\n  enrolledAt?: string;\n  [key: string]: any;\n}\n\n\n/** UploadAccount endpoint request user interface. */\ninterface UploadAccountUser {\n  localId: string;\n  email?: string;\n  emailVerified?: boolean;\n  displayName?: string;\n  disabled?: boolean;\n  photoUrl?: string;\n  phoneNumber?: string;\n  providerUserInfo?: Array<{\n    rawId: string;\n    providerId: string;\n    email?: string;\n    displayName?: string;\n    photoUrl?: string;\n  }>;\n  mfaInfo?: AuthFactorInfo[];\n  passwordHash?: string;\n  salt?: string;\n  lastLoginAt?: number;\n  createdAt?: number;\n  customAttributes?: string;\n  tenantId?: string;\n}\n\n\n/** UploadAccount endpoint request hash options. */\nexport interface UploadAccountOptions {\n  hashAlgorithm?: string;\n  signerKey?: string;\n  rounds?: number;\n  memoryCost?: number;\n  saltSeparator?: string;\n  cpuMemCost?: number;\n  parallelization?: number;\n  blockSize?: number;\n  dkLen?: number;\n}\n\n\n/** UploadAccount endpoint complete request interface. */\nexport interface UploadAccountRequest extends UploadAccountOptions {\n  users?: UploadAccountUser[];\n}\n\n\n/** Callback function to validate an UploadAccountUser object. */\nexport type ValidatorFunction = (data: UploadAccountUser) => void;\n\n\n/**\n * Converts a client format second factor object to server format.\n * @param multiFactorInfo - The client format second factor.\n * @returns The corresponding AuthFactorInfo server request format.\n */\nexport function convertMultiFactorInfoToServerFormat(multiFactorInfo: UpdateMultiFactorInfoRequest): AuthFactorInfo {\n  let enrolledAt;\n  if (typeof multiFactorInfo.enrollmentTime !== 'undefined') {\n    if (validator.isUTCDateString(multiFactorInfo.enrollmentTime)) {\n      // Convert from UTC date string (client side format) to ISO date string (server side format).\n      enrolledAt = new Date(multiFactorInfo.enrollmentTime).toISOString();\n    } else {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ENROLLMENT_TIME,\n        `The second factor \"enrollmentTime\" for \"${multiFactorInfo.uid}\" must be a valid ` +\n        'UTC date string.');\n    }\n  }\n  // Currently only phone second factors are supported.\n  if (isPhoneFactor(multiFactorInfo)) {\n    // If any required field is missing or invalid, validation will still fail later.\n    const authFactorInfo: AuthFactorInfo = {\n      mfaEnrollmentId: multiFactorInfo.uid,\n      displayName: multiFactorInfo.displayName,\n      // Required for all phone second factors.\n      phoneInfo: multiFactorInfo.phoneNumber,\n      enrolledAt,\n    };\n    for (const objKey in authFactorInfo) {\n      if (typeof authFactorInfo[objKey] === 'undefined') {\n        delete authFactorInfo[objKey];\n      }\n    }\n    return authFactorInfo;\n  } else {\n    // Unsupported second factor.\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.UNSUPPORTED_SECOND_FACTOR,\n      `Unsupported second factor \"${JSON.stringify(multiFactorInfo)}\" provided.`);\n  }\n}\n\nfunction isPhoneFactor(multiFactorInfo: UpdateMultiFactorInfoRequest):\n  multiFactorInfo is UpdatePhoneMultiFactorInfoRequest {\n  return multiFactorInfo.factorId === 'phone';\n}\n\n/**\n * @param {any} obj The object to check for number field within.\n * @param {string} key The entry key.\n * @returns {number} The corresponding number if available. Otherwise, NaN.\n */\nfunction getNumberField(obj: any, key: string): number {\n  if (typeof obj[key] !== 'undefined' && obj[key] !== null) {\n    return parseInt(obj[key].toString(), 10);\n  }\n  return NaN;\n}\n\n\n/**\n * Converts a UserImportRecord to a UploadAccountUser object. Throws an error when invalid\n * fields are provided.\n * @param {UserImportRecord} user The UserImportRecord to conver to UploadAccountUser.\n * @param {ValidatorFunction=} userValidator The user validator function.\n * @returns {UploadAccountUser} The corresponding UploadAccountUser to return.\n */\nfunction populateUploadAccountUser(\n  user: UserImportRecord, userValidator?: ValidatorFunction): UploadAccountUser {\n  const result: UploadAccountUser = {\n    localId: user.uid,\n    email: user.email,\n    emailVerified: user.emailVerified,\n    displayName: user.displayName,\n    disabled: user.disabled,\n    photoUrl: user.photoURL,\n    phoneNumber: user.phoneNumber,\n    providerUserInfo: [],\n    mfaInfo: [],\n    tenantId: user.tenantId,\n    customAttributes: user.customClaims && JSON.stringify(user.customClaims),\n  };\n  if (typeof user.passwordHash !== 'undefined') {\n    if (!validator.isBuffer(user.passwordHash)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_PASSWORD_HASH,\n      );\n    }\n    result.passwordHash = utils.toWebSafeBase64(user.passwordHash);\n  }\n  if (typeof user.passwordSalt !== 'undefined') {\n    if (!validator.isBuffer(user.passwordSalt)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_PASSWORD_SALT,\n      );\n    }\n    result.salt = utils.toWebSafeBase64(user.passwordSalt);\n  }\n  if (validator.isNonNullObject(user.metadata)) {\n    if (validator.isNonEmptyString(user.metadata.creationTime)) {\n      result.createdAt = new Date(user.metadata.creationTime).getTime();\n    }\n    if (validator.isNonEmptyString(user.metadata.lastSignInTime)) {\n      result.lastLoginAt = new Date(user.metadata.lastSignInTime).getTime();\n    }\n  }\n  if (validator.isArray(user.providerData)) {\n    user.providerData.forEach((providerData) => {\n      result.providerUserInfo!.push({\n        providerId: providerData.providerId,\n        rawId: providerData.uid,\n        email: providerData.email,\n        displayName: providerData.displayName,\n        photoUrl: providerData.photoURL,\n      });\n    });\n  }\n\n  // Convert user.multiFactor.enrolledFactors to server format.\n  if (validator.isNonNullObject(user.multiFactor) &&\n      validator.isNonEmptyArray(user.multiFactor.enrolledFactors)) {\n    user.multiFactor.enrolledFactors.forEach((multiFactorInfo) => {\n      result.mfaInfo!.push(convertMultiFactorInfoToServerFormat(multiFactorInfo));\n    });\n  }\n\n  // Remove blank fields.\n  let key: keyof UploadAccountUser;\n  for (key in result) {\n    if (typeof result[key] === 'undefined') {\n      delete result[key];\n    }\n  }\n  if (result.providerUserInfo!.length === 0) {\n    delete result.providerUserInfo;\n  }\n  if (result.mfaInfo!.length === 0) {\n    delete result.mfaInfo;\n  }\n  // Validate the constructured user individual request. This will throw if an error\n  // is detected.\n  if (typeof userValidator === 'function') {\n    userValidator(result);\n  }\n  return result;\n}\n\n\n/**\n * Class that provides a helper for building/validating uploadAccount requests and\n * UserImportResult responses.\n */\nexport class UserImportBuilder {\n  private requiresHashOptions: boolean;\n  private validatedUsers: UploadAccountUser[];\n  private validatedOptions: UploadAccountOptions;\n  private indexMap: {[key: number]: number};\n  private userImportResultErrors: FirebaseArrayIndexError[];\n\n  /**\n   * @param {UserImportRecord[]} users The list of user records to import.\n   * @param {UserImportOptions=} options The import options which includes hashing\n   *     algorithm details.\n   * @param {ValidatorFunction=} userRequestValidator The user request validator function.\n   * @constructor\n   */\n  constructor(\n    users: UserImportRecord[],\n    options?: UserImportOptions,\n    userRequestValidator?: ValidatorFunction) {\n    this.requiresHashOptions = false;\n    this.validatedUsers = [];\n    this.userImportResultErrors = [];\n    this.indexMap = {};\n\n    this.validatedUsers = this.populateUsers(users, userRequestValidator);\n    this.validatedOptions = this.populateOptions(options, this.requiresHashOptions);\n  }\n\n  /**\n   * Returns the corresponding constructed uploadAccount request.\n   * @returns {UploadAccountRequest} The constructed uploadAccount request.\n   */\n  public buildRequest(): UploadAccountRequest {\n    const users = this.validatedUsers.map((user) => {\n      return deepCopy(user);\n    });\n    return deepExtend({ users }, deepCopy(this.validatedOptions)) as UploadAccountRequest;\n  }\n\n  /**\n   * Populates the UserImportResult using the client side detected errors and the server\n   * side returned errors.\n   * @returns {UserImportResult} The user import result based on the returned failed\n   *     uploadAccount response.\n   */\n  public buildResponse(\n    failedUploads: Array<{index: number; message: string}>): UserImportResult {\n    // Initialize user import result.\n    const importResult: UserImportResult = {\n      successCount: this.validatedUsers.length,\n      failureCount: this.userImportResultErrors.length,\n      errors: deepCopy(this.userImportResultErrors),\n    };\n    importResult.failureCount += failedUploads.length;\n    importResult.successCount -= failedUploads.length;\n    failedUploads.forEach((failedUpload) => {\n      importResult.errors.push({\n        // Map backend request index to original developer provided array index.\n        index: this.indexMap[failedUpload.index],\n        error: new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_USER_IMPORT,\n          failedUpload.message,\n        ),\n      });\n    });\n    // Sort errors by index.\n    importResult.errors.sort((a, b) => {\n      return a.index - b.index;\n    });\n    // Return sorted result.\n    return importResult;\n  }\n\n  /**\n   * Validates and returns the hashing options of the uploadAccount request.\n   * Throws an error whenever an invalid or missing options is detected.\n   * @param {UserImportOptions} options The UserImportOptions.\n   * @param {boolean} requiresHashOptions Whether to require hash options.\n   * @returns {UploadAccountOptions} The populated UploadAccount options.\n   */\n  private populateOptions(\n    options: UserImportOptions | undefined, requiresHashOptions: boolean): UploadAccountOptions {\n    let populatedOptions: UploadAccountOptions;\n    if (!requiresHashOptions) {\n      return {};\n    }\n    if (!validator.isNonNullObject(options)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"UserImportOptions\" are required when importing users with passwords.',\n      );\n    }\n    if (!validator.isNonNullObject(options.hash)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.MISSING_HASH_ALGORITHM,\n        '\"hash.algorithm\" is missing from the provided \"UserImportOptions\".',\n      );\n    }\n    if (typeof options.hash.algorithm === 'undefined' ||\n        !validator.isNonEmptyString(options.hash.algorithm)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_HASH_ALGORITHM,\n        '\"hash.algorithm\" must be a string matching the list of supported algorithms.',\n      );\n    }\n\n    let rounds: number;\n    switch (options.hash.algorithm) {\n    case 'HMAC_SHA512':\n    case 'HMAC_SHA256':\n    case 'HMAC_SHA1':\n    case 'HMAC_MD5':\n      if (!validator.isBuffer(options.hash.key)) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_HASH_KEY,\n...\n// Path: src/auth/auth-api-request.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport * as validator from '../utils/validator';\n\nimport { App } from '../app/index';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { deepCopy, deepExtend } from '../utils/deep-copy';\nimport { AuthClientErrorCode, FirebaseAuthError } from '../utils/error';\nimport {\n  ApiSettings, AuthorizedHttpClient, HttpRequestConfig, HttpError,\n} from '../utils/api-request';\nimport * as utils from '../utils/index';\n\nimport {\n  UserImportOptions, UserImportRecord, UserImportResult,\n  UserImportBuilder, AuthFactorInfo, convertMultiFactorInfoToServerFormat,\n} from './user-import-builder';\nimport { ActionCodeSettings, ActionCodeSettingsBuilder } from './action-code-settings-builder';\nimport { Tenant, TenantServerResponse, CreateTenantRequest, UpdateTenantRequest } from './tenant';\nimport {\n  isUidIdentifier, isEmailIdentifier, isPhoneIdentifier, isProviderIdentifier,\n  UserIdentifier, UidIdentifier, EmailIdentifier,PhoneIdentifier, ProviderIdentifier,\n} from './identifier';\nimport {\n  SAMLConfig, OIDCConfig, OIDCConfigServerResponse, SAMLConfigServerResponse,\n  OIDCConfigServerRequest, SAMLConfigServerRequest, CreateRequest, UpdateRequest,\n  OIDCAuthProviderConfig, SAMLAuthProviderConfig, OIDCUpdateAuthProviderRequest,\n  SAMLUpdateAuthProviderRequest\n} from './auth-config';\nimport { ProjectConfig, ProjectConfigServerResponse, UpdateProjectConfigRequest } from './project-config';\n\n/** Firebase Auth request header. */\nconst FIREBASE_AUTH_HEADER = {\n  'X-Client-Version': `Node/Admin/${utils.getSdkVersion()}`,\n};\n/** Firebase Auth request timeout duration in milliseconds. */\nconst FIREBASE_AUTH_TIMEOUT = 25000;\n\n\n/** List of reserved claims which cannot be provided when creating a custom token. */\nexport const RESERVED_CLAIMS = [\n  'acr', 'amr', 'at_hash', 'aud', 'auth_time', 'azp', 'cnf', 'c_hash', 'exp', 'iat',\n  'iss', 'jti', 'nbf', 'nonce', 'sub', 'firebase',\n];\n\n/** List of supported email action request types. */\nexport const EMAIL_ACTION_REQUEST_TYPES = [\n  'PASSWORD_RESET', 'VERIFY_EMAIL', 'EMAIL_SIGNIN', 'VERIFY_AND_CHANGE_EMAIL',\n];\n\n/** Maximum allowed number of characters in the custom claims payload. */\nconst MAX_CLAIMS_PAYLOAD_SIZE = 1000;\n\n/** Maximum allowed number of users to batch download at one time. */\nconst MAX_DOWNLOAD_ACCOUNT_PAGE_SIZE = 1000;\n\n/** Maximum allowed number of users to batch upload at one time. */\nconst MAX_UPLOAD_ACCOUNT_BATCH_SIZE = 1000;\n\n/** Maximum allowed number of users to batch get at one time. */\nconst MAX_GET_ACCOUNTS_BATCH_SIZE = 100;\n\n/** Maximum allowed number of users to batch delete at one time. */\nconst MAX_DELETE_ACCOUNTS_BATCH_SIZE = 1000;\n\n/** Minimum allowed session cookie duration in seconds (5 minutes). */\nconst MIN_SESSION_COOKIE_DURATION_SECS = 5 * 60;\n\n/** Maximum allowed session cookie duration in seconds (2 weeks). */\nconst MAX_SESSION_COOKIE_DURATION_SECS = 14 * 24 * 60 * 60;\n\n/** Maximum allowed number of provider configurations to batch download at one time. */\nconst MAX_LIST_PROVIDER_CONFIGURATION_PAGE_SIZE = 100;\n\n/** The Firebase Auth backend base URL format. */\nconst FIREBASE_AUTH_BASE_URL_FORMAT =\n    'https://identitytoolkit.googleapis.com/{version}/projects/{projectId}{api}';\n\n/** Firebase Auth base URlLformat when using the auth emultor. */\nconst FIREBASE_AUTH_EMULATOR_BASE_URL_FORMAT =\n  'http://{host}/identitytoolkit.googleapis.com/{version}/projects/{projectId}{api}';\n\n/** The Firebase Auth backend multi-tenancy base URL format. */\nconst FIREBASE_AUTH_TENANT_URL_FORMAT = FIREBASE_AUTH_BASE_URL_FORMAT.replace(\n  'projects/{projectId}', 'projects/{projectId}/tenants/{tenantId}');\n\n/** Firebase Auth base URL format when using the auth emultor with multi-tenancy. */\nconst FIREBASE_AUTH_EMULATOR_TENANT_URL_FORMAT = FIREBASE_AUTH_EMULATOR_BASE_URL_FORMAT.replace(\n  'projects/{projectId}', 'projects/{projectId}/tenants/{tenantId}');\n\n/** Maximum allowed number of tenants to download at one time. */\nconst MAX_LIST_TENANT_PAGE_SIZE = 1000;\n\n\n/**\n * Enum for the user write operation type.\n */\nenum WriteOperationType {\n  Create = 'create',\n  Update = 'update',\n  Upload = 'upload',\n}\n\n\n/** Defines a base utility to help with resource URL construction. */\nclass AuthResourceUrlBuilder {\n\n  protected urlFormat: string;\n  private projectId: string;\n\n  /**\n   * The resource URL builder constructor.\n   *\n   * @param projectId - The resource project ID.\n   * @param version - The endpoint API version.\n   * @constructor\n   */\n  constructor(protected app: App, protected version: string = 'v1') {\n    if (useEmulator()) {\n      this.urlFormat = utils.formatString(FIREBASE_AUTH_EMULATOR_BASE_URL_FORMAT, {\n        host: emulatorHost()\n      });\n    } else {\n      this.urlFormat = FIREBASE_AUTH_BASE_URL_FORMAT;\n    }\n  }\n\n  /**\n   * Returns the resource URL corresponding to the provided parameters.\n   *\n   * @param api - The backend API name.\n   * @param params - The optional additional parameters to substitute in the\n   *     URL path.\n   * @returns The corresponding resource URL.\n   */\n  public getUrl(api?: string, params?: object): Promise<string> {\n    return this.getProjectId()\n      .then((projectId) => {\n        const baseParams = {\n          version: this.version,\n          projectId,\n          api: api || '',\n        };\n        const baseUrl = utils.formatString(this.urlFormat, baseParams);\n        // Substitute additional api related parameters.\n        return utils.formatString(baseUrl, params || {});\n      });\n  }\n\n  private getProjectId(): Promise<string> {\n    if (this.projectId) {\n      return Promise.resolve(this.projectId);\n    }\n\n    return utils.findProjectId(this.app)\n      .then((projectId) => {\n        if (!validator.isNonEmptyString(projectId)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_CREDENTIAL,\n            'Failed to determine project ID for Auth. Initialize the '\n              + 'SDK with service account credentials or set project ID as an app option. '\n              + 'Alternatively set the GOOGLE_CLOUD_PROJECT environment variable.',\n          );\n        }\n\n        this.projectId = projectId;\n        return projectId;\n      });\n  }\n}\n\n\n/** Tenant aware resource builder utility. */\nclass TenantAwareAuthResourceUrlBuilder extends AuthResourceUrlBuilder {\n  /**\n   * The tenant aware resource URL builder constructor.\n   *\n   * @param projectId - The resource project ID.\n   * @param version - The endpoint API version.\n   * @param tenantId - The tenant ID.\n   * @constructor\n   */\n  constructor(protected app: App, protected version: string, protected tenantId: string) {\n    super(app, version);\n    if (useEmulator()) {\n      this.urlFormat = utils.formatString(FIREBASE_AUTH_EMULATOR_TENANT_URL_FORMAT, {\n        host: emulatorHost()\n      });\n    } else {\n      this.urlFormat = FIREBASE_AUTH_TENANT_URL_FORMAT;\n    }\n  }\n\n  /**\n   * Returns the resource URL corresponding to the provided parameters.\n   *\n   * @param api - The backend API name.\n   * @param params - The optional additional parameters to substitute in the\n   *     URL path.\n   * @returns The corresponding resource URL.\n   */\n  public getUrl(api?: string, params?: object): Promise<string> {\n    return super.getUrl(api, params)\n      .then((url) => {\n        return utils.formatString(url, { tenantId: this.tenantId });\n      });\n  }\n}\n\n/**\n * Auth-specific HTTP client which uses the special \"owner\" token\n * when communicating with the Auth Emulator.\n */\nclass AuthHttpClient extends AuthorizedHttpClient {\n\n  protected getToken(): Promise<string> {\n    if (useEmulator()) {\n      return Promise.resolve('owner');\n    }\n\n    return super.getToken();\n  }\n\n}\n\n/**\n * Validates an AuthFactorInfo object. All unsupported parameters\n * are removed from the original request. If an invalid field is passed\n * an error is thrown.\n *\n * @param request - The AuthFactorInfo request object.\n */\nfunction validateAuthFactorInfo(request: AuthFactorInfo): void {\n  const validKeys = {\n    mfaEnrollmentId: true,\n    displayName: true,\n    phoneInfo: true,\n    enrolledAt: true,\n  };\n  // Remove unsupported keys from the original request.\n  for (const key in request) {\n    if (!(key in validKeys)) {\n      delete request[key];\n    }\n  }\n  // No enrollment ID is available for signupNewUser. Use another identifier.\n  const authFactorInfoIdentifier =\n      request.mfaEnrollmentId || request.phoneInfo || JSON.stringify(request);\n  // Enrollment uid may or may not be specified for update operations.\n  if (typeof request.mfaEnrollmentId !== 'undefined' &&\n      !validator.isNonEmptyString(request.mfaEnrollmentId)) {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_UID,\n      'The second factor \"uid\" must be a valid non-empty string.',\n    );\n  }\n  if (typeof request.displayName !== 'undefined' &&\n      !validator.isString(request.displayName)) {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_DISPLAY_NAME,\n      `The second factor \"displayName\" for \"${authFactorInfoIdentifier}\" must be a valid string.`,\n    );\n  }\n  // enrolledAt must be a valid UTC date string.\n  if (typeof request.enrolledAt !== 'undefined' &&\n      !validator.isISODateString(request.enrolledAt)) {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_ENROLLMENT_TIME,\n      `The second factor \"enrollmentTime\" for \"${authFactorInfoIdentifier}\" must be a valid ` +\n      'UTC date string.');\n  }\n  // Validate required fields depending on second factor type.\n  if (typeof request.phoneInfo !== 'undefined') {\n    // phoneNumber should be a string and a valid phone number.\n    if (!validator.isPhoneNumber(request.phoneInfo)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_PHONE_NUMBER,\n        `The second factor \"phoneNumber\" for \"${authFactorInfoIdentifier}\" must be a non-empty ` +\n        'E.164 standard compliant identifier string.');\n    }\n  } else {\n    // Invalid second factor. For example, a phone second factor may have been provided without\n    // a phone number. A TOTP based second factor may require a secret key, etc.\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_ENROLLED_FACTORS,\n      'MFAInfo object provided is invalid.');\n  }\n}\n\n\n/**\n * Validates a providerUserInfo object. All unsupported parameters\n * are removed from the original request. If an invalid field is passed\n * an error is thrown.\n *\n * @param request - The providerUserInfo request object.\n */\n\nfunction validateProviderUserInfo(request: any): void {\n  const validKeys = {\n    rawId: true,\n    providerId: true,\n    email: true,\n    displayName: true,\n    photoUrl: true,\n  };\n  // Remove invalid keys from original request.\n  for (const key in request) {\n    if (!(key in validKeys)) {\n      delete request[key];\n    }\n  }\n  if (!validator.isNonEmptyString(request.providerId)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID);\n  }\n  if (typeof request.displayName !== 'undefined' &&\n      typeof request.displayName !== 'string') {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_DISPLAY_NAME,\n      `The provider \"displayName\" for \"${request.providerId}\" must be a valid string.`,\n    );\n  }\n  if (!validator.isNonEmptyString(request.rawId)) {\n    // This is called localId on the backend but the developer specifies this as\n    // uid externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_UID,\n      `The provider \"uid\" for \"${request.providerId}\" must be a valid non-empty string.`,\n    );\n  }\n  // email should be a string and a valid email.\n  if (typeof request.email !== 'undefined' && !validator.isEmail(request.email)) {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_EMAIL,\n      `The provider \"email\" for \"${request.providerId}\" must be a valid email string.`,\n    );\n  }\n  // photoUrl should be a URL.\n  if (typeof request.photoUrl !== 'undefined' &&\n      !validator.isURL(request.photoUrl)) {\n    // This is called photoUrl on the backend but the developer specifies this as\n    // photoURL externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_PHOTO_URL,\n      `The provider \"photoURL\" for \"${request.providerId}\" must be a valid URL string.`,\n    );\n  }\n}\n\n\n/**\n * Validates a create/edit request object. All unsupported parameters\n * are removed from the original request. If an invalid field is passed\n * an error is thrown.\n *\n * @param request - The create/edit request object.\n * @param writeOperationType - The write operation type.\n */\nfunction validateCreateEditRequest(request: any, writeOperationType: WriteOperationType): void {\n  const uploadAccountRequest = writeOperationType === WriteOperationType.Upload;\n  // Hash set of whitelisted parameters.\n  const validKeys = {\n    displayName: true,\n    localId: true,\n    email: true,\n    password: true,\n    rawPassword: true,\n    emailVerified: true,\n    photoUrl: true,\n    disabled: true,\n    disableUser: true,\n    deleteAttribute: true,\n    deleteProvider: true,\n    sanityCheck: true,\n    phoneNumber: true,\n    customAttributes: true,\n    validSince: true,\n    // Pass linkProviderUserInfo only for updates (i.e. not for uploads.)\n    linkProviderUserInfo: !uploadAccountRequest,\n    // Pass tenantId only for uploadAccount requests.\n    tenantId: uploadAccountRequest,\n    passwordHash: uploadAccountRequest,\n    salt: uploadAccountRequest,\n    createdAt: uploadAccountRequest,\n    lastLoginAt: uploadAccountRequest,\n    providerUserInfo: uploadAccountRequest,\n    mfaInfo: uploadAccountRequest,\n    // Only for non-uploadAccount requests.\n    mfa: !uploadAccountRequest,\n  };\n  // Remove invalid keys from original request.\n  for (const key in request) {\n    if (!(key in validKeys)) {\n      delete request[key];\n    }\n  }\n  if (typeof request.tenantId !== 'undefined' &&\n      !validator.isNonEmptyString(request.tenantId)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_TENANT_ID);\n  }\n  // For any invalid parameter, use the external key name in the error description.\n  // displayName should be a string.\n  if (typeof request.displayName !== 'undefined' &&\n      !validator.isString(request.displayName)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_DISPLAY_NAME);\n  }\n  if ((typeof request.localId !== 'undefined' || uploadAccountRequest) &&\n      !validator.isUid(request.localId)) {\n    // This is called localId on the backend but the developer specifies this as\n    // uid externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_UID);\n  }\n  // email should be a string and a valid email.\n  if (typeof request.email !== 'undefined' && !validator.isEmail(request.email)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_EMAIL);\n  }\n  // phoneNumber should be a string and a valid phone number.\n  if (typeof request.phoneNumber !== 'undefined' &&\n      !validator.isPhoneNumber(request.phoneNumber)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PHONE_NUMBER);\n  }\n  // password should be a string and a minimum of 6 chars.\n  if (typeof request.password !== 'undefined' &&\n      !validator.isPassword(request.password)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PASSWORD);\n  }\n  // rawPassword should be a string and a minimum of 6 chars.\n  if (typeof request.rawPassword !== 'undefined' &&\n      !validator.isPassword(request.rawPassword)) {\n    // This is called rawPassword on the backend but the developer specifies this as\n    // password externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PASSWORD);\n  }\n  // emailVerified should be a boolean.\n  if (typeof request.emailVerified !== 'undefined' &&\n      typeof request.emailVerified !== 'boolean') {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_EMAIL_VERIFIED);\n  }\n  // photoUrl should be a URL.\n  if (typeof request.photoUrl !== 'undefined' &&\n      !validator.isURL(request.photoUrl)) {\n    // This is called photoUrl on the backend but the developer specifies this as\n    // photoURL externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PHOTO_URL);\n  }\n  // disabled should be a boolean.\n  if (typeof request.disabled !== 'undefined' &&\n      typeof request.disabled !== 'boolean') {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_DISABLED_FIELD);\n  }\n  // validSince should be a number.\n  if (typeof request.validSince !== 'undefined' &&\n      !validator.isNumber(request.validSince)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_TOKENS_VALID_AFTER_TIME);\n  }\n  // createdAt should be a number.\n  if (typeof request.createdAt !== 'undefined' &&\n      !validator.isNumber(request.createdAt)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_CREATION_TIME);\n  }\n  // lastSignInAt should be a number.\n  if (typeof request.lastLoginAt !== 'undefined' &&\n      !validator.isNumber(request.lastLoginAt)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_LAST_SIGN_IN_TIME);\n  }\n  // disableUser should be a boolean.\n  if (typeof request.disableUser !== 'undefined' &&\n      typeof request.disableUser !== 'boolean') {\n    // This is called disableUser on the backend but the developer specifies this as\n    // disabled externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_DISABLED_FIELD);\n  }\n  // customAttributes should be stringified JSON with no blacklisted claims.\n  // The payload should not exceed 1KB.\n  if (typeof request.customAttributes !== 'undefined') {\n    let developerClaims: object;\n    try {\n      developerClaims = JSON.parse(request.customAttributes);\n    } catch (error) {\n      // JSON parsing error. This should never happen as we stringify the claims internally.\n      // However, we still need to check since setAccountInfo via edit requests could pass\n      // this field.\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_CLAIMS, error.message);\n    }\n    const invalidClaims: string[] = [];\n    // Check for any invalid claims.\n    RESERVED_CLAIMS.forEach((blacklistedClaim) => {\n      if (Object.prototype.hasOwnProperty.call(developerClaims, blacklistedClaim)) {\n        invalidClaims.push(blacklistedClaim);\n      }\n    });\n    // Throw an error if an invalid claim is detected.\n    if (invalidClaims.length > 0) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.FORBIDDEN_CLAIM,\n        invalidClaims.length > 1 ?\n          `Developer claims \"${invalidClaims.join('\", \"')}\" are reserved and cannot be specified.` :\n          `Developer claim \"${invalidClaims[0]}\" is reserved and cannot be specified.`,\n      );\n    }\n    // Check claims payload does not exceed maxmimum size.\n    if (request.customAttributes.length > MAX_CLAIMS_PAYLOAD_SIZE) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.CLAIMS_TOO_LARGE,\n        `Developer claims payload should not exceed ${MAX_CLAIMS_PAYLOAD_SIZE} characters.`,\n      );\n    }\n  }\n  // passwordHash has to be a base64 encoded string.\n  if (typeof request.passwordHash !== 'undefined' &&\n      !validator.isString(request.passwordHash)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PASSWORD_HASH);\n  }\n  // salt has to be a base64 encoded string.\n  if (typeof request.salt !== 'undefined' &&\n      !validator.isString(request.salt)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PASSWORD_SALT);\n  }\n  // providerUserInfo has to be an array of valid UserInfo requests.\n  if (typeof request.providerUserInfo !== 'undefined' &&\n      !validator.isArray(request.providerUserInfo)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_DATA);\n  } else if (validator.isArray(request.providerUserInfo)) {\n    request.providerUserInfo.forEach((providerUserInfoEntry: any) => {\n      validateProviderUserInfo(providerUserInfoEntry);\n    });\n  }\n\n  // linkProviderUserInfo must be a (single) UserProvider value.\n  if (typeof request.linkProviderUserInfo !== 'undefined') {\n    validateProviderUserInfo(request.linkProviderUserInfo);\n  }\n\n  // mfaInfo is used for importUsers.\n  // mfa.enrollments is used for setAccountInfo.\n  // enrollments has to be an array of valid AuthFactorInfo requests.\n  let enrollments: AuthFactorInfo[] | null = null;\n  if (request.mfaInfo) {\n    enrollments = request.mfaInfo;\n  } else if (request.mfa && request.mfa.enrollments) {\n    enrollments = request.mfa.enrollments;\n  }\n  if (enrollments) {\n    if (!validator.isArray(enrollments)) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_ENROLLED_FACTORS);\n    }\n    enrollments.forEach((authFactorInfoEntry: AuthFactorInfo) => {\n      validateAuthFactorInfo(authFactorInfoEntry);\n    });\n  }\n}\n\n\n/**\n * Instantiates the createSessionCookie endpoint settings.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_CREATE_SESSION_COOKIE =\n    new ApiSettings(':createSessionCookie', 'POST')\n    // Set request validator.\n      .setRequestValidator((request: any) => {\n        // Validate the ID token is a non-empty string.\n        if (!validator.isNonEmptyString(request.idToken)) {\n          throw new FirebaseAuthError(AuthClientErrorCode.INVALID_ID_TOKEN);\n        }\n        // Validate the custom session cookie duration.\n        if (!validator.isNumber(request.validDuration) ||\n              request.validDuration < MIN_SESSION_COOKIE_DURATION_SECS ||\n              request.validDuration > MAX_SESSION_COOKIE_DURATION_SECS) {\n          throw new FirebaseAuthError(AuthClientErrorCode.INVALID_SESSION_COOKIE_DURATION);\n        }\n      })\n    // Set response validator.\n      .setResponseValidator((response: any) => {\n        // Response should always contain the session cookie.\n        if (!validator.isNonEmptyString(response.sessionCookie)) {\n          throw new FirebaseAuthError(AuthClientErrorCode.INTERNAL_ERROR);\n        }\n      });\n\n\n/**\n * Instantiates the uploadAccount endpoint settings.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_UPLOAD_ACCOUNT = new ApiSettings('/accounts:batchCreate', 'POST');\n\n\n/**\n * Instantiates the downloadAccount endpoint settings.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_DOWNLOAD_ACCOUNT = new ApiSettings('/accounts:batchGet', 'GET')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    // Validate next page token.\n    if (typeof request.nextPageToken !== 'undefined' &&\n        !validator.isNonEmptyString(request.nextPageToken)) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PAGE_TOKEN);\n    }\n    // Validate max results.\n    if (!validator.isNumber(request.maxResults) ||\n        request.maxResults <= 0 ||\n        request.maxResults > MAX_DOWNLOAD_ACCOUNT_PAGE_SIZE) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'Required \"maxResults\" must be a positive integer that does not exceed ' +\n        `${MAX_DOWNLOAD_ACCOUNT_PAGE_SIZE}.`,\n      );\n    }\n  });\n\ninterface GetAccountInfoRequest {\n  localId?: string[];\n  email?: string[];\n  phoneNumber?: string[];\n  federatedUserId?: Array<{\n    providerId: string;\n    rawId: string;\n  }>;\n}\n\n/**\n * Instantiates the getAccountInfo endpoint settings.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_GET_ACCOUNT_INFO = new ApiSettings('/accounts:lookup', 'POST')\n  // Set request validator.\n  .setRequestValidator((request: GetAccountInfoRequest) => {\n    if (!request.localId && !request.email && !request.phoneNumber && !request.federatedUserId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Server request is missing user identifier');\n    }\n  })\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    if (!response.users || !response.users.length) {\n      throw new FirebaseAuthError(AuthClientErrorCode.USER_NOT_FOUND);\n    }\n  });\n\n/**\n * Instantiates the getAccountInfo endpoint settings for use when fetching info\n * for multiple accounts.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_GET_ACCOUNTS_INFO = new ApiSettings('/accounts:lookup', 'POST')\n  // Set request validator.\n  .setRequestValidator((request: GetAccountInfoRequest) => {\n    if (!request.localId && !request.email && !request.phoneNumber && !request.federatedUserId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Server request is missing user identifier');\n    }\n  });\n\n\n/**\n * Instantiates the deleteAccount endpoint settings.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_DELETE_ACCOUNT = new ApiSettings('/accounts:delete', 'POST')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    if (!request.localId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Server request is missing user identifier');\n    }\n  });\n\ninterface BatchDeleteAccountsRequest {\n  localIds?: string[];\n  force?: boolean;\n}\n\ninterface BatchDeleteErrorInfo {\n  index?: number;\n  localId?: string;\n  message?: string;\n}\n\nexport interface BatchDeleteAccountsResponse {\n  errors?: BatchDeleteErrorInfo[];\n}\n\n/**\n * @internal\n */\nexport const FIREBASE_AUTH_BATCH_DELETE_ACCOUNTS = new ApiSettings('/accounts:batchDelete', 'POST')\n  .setRequestValidator((request: BatchDeleteAccountsRequest) => {\n    if (!request.localIds) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Server request is missing user identifiers');\n    }\n    if (typeof request.force === 'undefined' || request.force !== true) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Server request is missing force=true field');\n    }\n  })\n  .setResponseValidator((response: BatchDeleteAccountsResponse) => {\n    const errors = response.errors || [];\n    errors.forEach((batchDeleteErrorInfo) => {\n      if (typeof batchDeleteErrorInfo.index === 'undefined') {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INTERNAL_ERROR,\n          'INTERNAL ASSERT FAILED: Server BatchDeleteAccountResponse is missing an errors.index field');\n      }\n      if (!batchDeleteErrorInfo.localId) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INTERNAL_ERROR,\n          'INTERNAL ASSERT FAILED: Server BatchDeleteAccountResponse is missing an errors.localId field');\n      }\n      // Allow the (error) message to be missing/undef.\n    });\n  });\n\n/**\n * Instantiates the setAccountInfo endpoint settings for updating existing accounts.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_SET_ACCOUNT_INFO = new ApiSettings('/accounts:update', 'POST')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    // localId is a required parameter.\n    if (typeof request.localId === 'undefined') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Server request is missing user identifier');\n    }\n    // Throw error when tenantId is passed in POST body.\n    if (typeof request.tenantId !== 'undefined') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"tenantId\" is an invalid \"UpdateRequest\" property.');\n    }\n    validateCreateEditRequest(request, WriteOperationType.Update);\n  })\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // If the localId is not returned, then the request failed.\n    if (!response.localId) {\n      throw new FirebaseAuthError(AuthClientErrorCode.USER_NOT_FOUND);\n    }\n  });\n\n/**\n * Instantiates the signupNewUser endpoint settings for creating a new user with or without\n * uid being specified. The backend will create a new one if not provided and return it.\n *\n * @internal\n */\nexport const FIREBASE_AUTH_SIGN_UP_NEW_USER = new ApiSettings('/accounts', 'POST')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    // signupNewUser does not support customAttributes.\n    if (typeof request.customAttributes !== 'undefined') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"customAttributes\" cannot be set when creating a new user.',\n      );\n    }\n    // signupNewUser does not support validSince.\n    if (typeof request.validSince !== 'undefined') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"validSince\" cannot be set when creating a new user.',\n      );\n    }\n    // Throw error when tenantId is passed in POST body.\n    if (typeof request.tenantId !== 'undefined') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"tenantId\" is an invalid \"CreateRequest\" property.');\n    }\n    validateCreateEditRequest(request, WriteOperationType.Create);\n  })\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // If the localId is not returned, then the request failed.\n    if (!response.localId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to create new user');\n    }\n  });\n\nconst FIREBASE_AUTH_GET_OOB_CODE = new ApiSettings('/accounts:sendOobCode', 'POST')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    if (!validator.isEmail(request.email)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_EMAIL,\n      );\n    }\n    if (typeof request.newEmail !== 'undefined' && !validator.isEmail(request.newEmail)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_NEW_EMAIL,\n      );\n    }\n    if (EMAIL_ACTION_REQUEST_TYPES.indexOf(request.requestType) === -1) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        `\"${request.requestType}\" is not a supported email action request type.`,\n      );\n    }\n  })\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // If the oobLink is not returned, then the request failed.\n    if (!response.oobLink) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to create the email action link');\n    }\n  });\n\n/**\n * Instantiates the retrieve OIDC configuration endpoint settings.\n *\n * @internal\n */\nconst GET_OAUTH_IDP_CONFIG = new ApiSettings('/oauthIdpConfigs/{providerId}', 'GET')\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain the OIDC provider resource name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to get OIDC configuration',\n      );\n    }\n  });\n\n/**\n * Instantiates the delete OIDC configuration endpoint settings.\n *\n * @internal\n */\nconst DELETE_OAUTH_IDP_CONFIG = new ApiSettings('/oauthIdpConfigs/{providerId}', 'DELETE');\n\n/**\n * Instantiates the create OIDC configuration endpoint settings.\n *\n * @internal\n */\nconst CREATE_OAUTH_IDP_CONFIG = new ApiSettings('/oauthIdpConfigs?oauthIdpConfigId={providerId}', 'POST')\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain the OIDC provider resource name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to create new OIDC configuration',\n      );\n    }\n  });\n\n/**\n * Instantiates the update OIDC configuration endpoint settings.\n *\n * @internal\n */\nconst UPDATE_OAUTH_IDP_CONFIG = new ApiSettings('/oauthIdpConfigs/{providerId}?updateMask={updateMask}', 'PATCH')\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain the configuration resource name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to update OIDC configuration',\n      );\n    }\n  });\n\n/**\n * Instantiates the list OIDC configuration endpoint settings.\n *\n * @internal\n */\nconst LIST_OAUTH_IDP_CONFIGS = new ApiSettings('/oauthIdpConfigs', 'GET')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    // Validate next page token.\n    if (typeof request.pageToken !== 'undefined' &&\n        !validator.isNonEmptyString(request.pageToken)) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PAGE_TOKEN);\n    }\n    // Validate max results.\n    if (!validator.isNumber(request.pageSize) ||\n        request.pageSize <= 0 ||\n        request.pageSize > MAX_LIST_PROVIDER_CONFIGURATION_PAGE_SIZE) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'Required \"maxResults\" must be a positive integer that does not exceed ' +\n        `${MAX_LIST_PROVIDER_CONFIGURATION_PAGE_SIZE}.`,\n      );\n    }\n  });\n\n/**\n * Instantiates the retrieve SAML configuration endpoint settings.\n *\n * @internal\n */\nconst GET_INBOUND_SAML_CONFIG = new ApiSettings('/inboundSamlConfigs/{providerId}', 'GET')\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain the SAML provider resource name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to get SAML configuration',\n      );\n    }\n  });\n\n/**\n * Instantiates the delete SAML configuration endpoint settings.\n *\n * @internal\n */\nconst DELETE_INBOUND_SAML_CONFIG = new ApiSettings('/inboundSamlConfigs/{providerId}', 'DELETE');\n\n/**\n * Instantiates the create SAML configuration endpoint settings.\n *\n * @internal\n */\nconst CREATE_INBOUND_SAML_CONFIG = new ApiSettings('/inboundSamlConfigs?inboundSamlConfigId={providerId}', 'POST')\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain the SAML provider resource name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to create new SAML configuration',\n      );\n    }\n  });\n\n/**\n * Instantiates the update SAML configuration endpoint settings.\n *\n * @internal\n */\nconst UPDATE_INBOUND_SAML_CONFIG = new ApiSettings('/inboundSamlConfigs/{providerId}?updateMask={updateMask}', 'PATCH')\n  // Set response validator.\n  .setResponseValidator((response: any) => {\n    // Response should always contain the configuration resource name.\n    if (!validator.isNonEmptyString(response.name)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Unable to update SAML configuration',\n      );\n    }\n  });\n\n/**\n * Instantiates the list SAML configuration endpoint settings.\n *\n * @internal\n */\nconst LIST_INBOUND_SAML_CONFIGS = new ApiSettings('/inboundSamlConfigs', 'GET')\n  // Set request validator.\n  .setRequestValidator((request: any) => {\n    // Validate next page token.\n    if (typeof request.pageToken !== 'undefined' &&\n        !validator.isNonEmptyString(request.pageToken)) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PAGE_TOKEN);\n    }\n    // Validate max results.\n    if (!validator.isNumber(request.pageSize) ||\n        request.pageSize <= 0 ||\n        request.pageSize > MAX_LIST_PROVIDER_CONFIGURATION_PAGE_SIZE) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        'Required \"maxResults\" must be a positive integer that does not exceed ' +\n        `${MAX_LIST_PROVIDER_CONFIGURATION_PAGE_SIZE}.`,\n      );\n    }\n  });\n\n/**\n * Class that provides the mechanism to send requests to the Firebase Auth backend endpoints.\n *\n * @internal\n */\nexport abstract class AbstractAuthRequestHandler {\n\n  protected readonly httpClient: AuthorizedHttpClient;\n  private authUrlBuilder: AuthResourceUrlBuilder;\n  private projectConfigUrlBuilder: AuthResourceUrlBuilder;\n\n  /**\n   * @param response - The response to check for errors.\n   * @returns The error code if present; null otherwise.\n   */\n  private static getErrorCode(response: any): string | null {\n...\n", "needle_token_start": 8555, "needle_token_end": 9103, "code_context_ntokens": 16072, "output": ["```typescript\n/**\n * Validates a providerUserInfo object. All unsupported parameters\n * are removed from the original request. If an invalid field is passed\n * an error is thrown.\n *\n * @param request - The providerUserInfo request object.\n */\nfunction validateProviderUserInfo(request: any): void {\n  const validKeys = {\n    rawId: true,\n    providerId: true,\n    email: true,\n    displayName: true,\n    photoUrl: true,\n  };\n  // Remove invalid keys from original request.\n  for (const key in request) {\n    if (!(key in validKeys)) {\n      delete request[key];\n    }\n  }\n  if (!validator.isNonEmptyString(request.providerId)) {\n    throw new FirebaseAuthError(AuthClientErrorCode.INVALID_PROVIDER_ID);\n  }\n  if (typeof request.displayName !== 'undefined' &&\n      typeof request.displayName !== 'string') {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_DISPLAY_NAME,\n      `The provider \"displayName\" for \"${request.providerId}\" must be a valid string.`,\n    );\n  }\n  if (!validator.isNonEmptyString(request.rawId)) {\n    // This is called localId on the backend but the developer specifies this as\n    // uid externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_UID,\n      `The provider \"uid\" for \"${request.providerId}\" must be a valid non-empty string.`,\n    );\n  }\n  // email should be a string and a valid email.\n  if (typeof request.email !== 'undefined' && !validator.isEmail(request.email)) {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_EMAIL,\n      `The provider \"email\" for \"${request.providerId}\" must be a valid email string.`,\n    );\n  }\n  // photoUrl should be a URL.\n  if (typeof request.photoUrl !== 'undefined' &&\n      !validator.isURL(request.photoUrl)) {\n    // This is called photoUrl on the backend but the developer specifies this as\n    // photoURL externally. So the error message should use the client facing name.\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_PHOTO_URL,\n      `The provider \"photoURL\" for \"${request.providerId}\" must be a valid URL string.`,\n    );\n  }\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "validateTestPhoneNumbers", "language": "typescript", "path": "src/auth/auth-config.ts", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To ensure that a provided map of phone number and code pairs adheres to specific format and content rules, suitable for testing purposes.\n2. **Input**: A map where keys are phone numbers and values are corresponding codes.\n3. **Output**: No direct output, but throws errors if the input does not meet the required criteria.\n4. **Procedure**: \n   - Checks if the input is a valid object.\n   - Ensures the number of entries does not exceed a predefined maximum limit.\n   - Validates each phone number to confirm it complies with the E.164 standard.\n   - Checks each code to ensure it is a string representing a 6-digit number.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/security-rules/security-rules.ts\n/*!\n * Copyright 2019 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport * as validator from '../utils/validator';\nimport {\n  SecurityRulesApiClient, RulesetResponse, RulesetContent, ListRulesetsResponse,\n} from './security-rules-api-client-internal';\nimport { FirebaseSecurityRulesError } from './security-rules-internal';\n\n/**\n * A source file containing some Firebase security rules. The content includes raw\n * source code including text formatting, indentation and comments. Use the\n * {@link SecurityRules.createRulesFileFromSource} method to create new instances of this type.\n */\nexport interface RulesFile {\n  readonly name: string;\n  readonly content: string;\n}\n\n/**\n * Required metadata associated with a ruleset.\n */\nexport interface RulesetMetadata {\n  /**\n   * Name of the `Ruleset` as a short string. This can be directly passed into APIs\n   * like {@link SecurityRules.getRuleset} and {@link SecurityRules.deleteRuleset}.\n   */\n  readonly name: string;\n  /**\n   * Creation time of the `Ruleset` as a UTC timestamp string.\n   */\n  readonly createTime: string;\n}\n\n/**\n * A page of ruleset metadata.\n */\nexport class RulesetMetadataList {\n\n  /**\n   * A batch of ruleset metadata.\n   */\n  public readonly rulesets: RulesetMetadata[];\n\n  /**\n   * The next page token if available. This is needed to retrieve the next batch.\n   */\n  public readonly nextPageToken?: string;\n\n  /**\n   * @internal\n   */\n  constructor(response: ListRulesetsResponse) {\n    if (!validator.isNonNullObject(response) || !validator.isArray(response.rulesets)) {\n      throw new FirebaseSecurityRulesError(\n        'invalid-argument',\n        `Invalid ListRulesets response: ${JSON.stringify(response)}`);\n    }\n\n    this.rulesets = response.rulesets.map((rs) => {\n      return {\n        name: stripProjectIdPrefix(rs.name),\n        createTime: new Date(rs.createTime).toUTCString(),\n      };\n    });\n\n    if (response.nextPageToken) {\n      this.nextPageToken = response.nextPageToken;\n    }\n  }\n}\n\n/**\n * A set of Firebase security rules.\n */\nexport class Ruleset implements RulesetMetadata {\n\n  /**\n   * {@inheritdoc RulesetMetadata.name}\n   */\n  public readonly name: string;\n\n  /**\n   * {@inheritdoc RulesetMetadata.createTime}\n   */\n  public readonly createTime: string;\n\n  public readonly source: RulesFile[];\n\n  /**\n   * @internal\n   */\n  constructor(ruleset: RulesetResponse) {\n    if (!validator.isNonNullObject(ruleset) ||\n      !validator.isNonEmptyString(ruleset.name) ||\n      !validator.isNonEmptyString(ruleset.createTime) ||\n      !validator.isNonNullObject(ruleset.source)) {\n      throw new FirebaseSecurityRulesError(\n        'invalid-argument',\n        `Invalid Ruleset response: ${JSON.stringify(ruleset)}`);\n    }\n\n    this.name = stripProjectIdPrefix(ruleset.name);\n    this.createTime = new Date(ruleset.createTime).toUTCString();\n    this.source = ruleset.source.files || [];\n  }\n}\n\n/**\n * The Firebase `SecurityRules` service interface.\n */\nexport class SecurityRules {\n\n  private static readonly CLOUD_FIRESTORE = 'cloud.firestore';\n  private static readonly FIREBASE_STORAGE = 'firebase.storage';\n\n  private readonly client: SecurityRulesApiClient;\n\n  /**\n   * @param app - The app for this SecurityRules service.\n   * @constructor\n   * @internal\n   */\n  constructor(readonly app: App) {\n    this.client = new SecurityRulesApiClient(app);\n  }\n\n  /**\n   * Gets the {@link Ruleset} identified by the given\n   * name. The input name should be the short name string without the project ID\n   * prefix. For example, to retrieve the `projects/project-id/rulesets/my-ruleset`,\n   * pass the short name \"my-ruleset\". Rejects with a `not-found` error if the\n   * specified `Ruleset` cannot be found.\n   *\n   * @param name - Name of the `Ruleset` to retrieve.\n   * @returns A promise that fulfills with the specified `Ruleset`.\n   */\n  public getRuleset(name: string): Promise<Ruleset> {\n    return this.client.getRuleset(name)\n      .then((rulesetResponse) => {\n        return new Ruleset(rulesetResponse);\n      });\n  }\n\n  /**\n   * Gets the {@link Ruleset} currently applied to\n   * Cloud Firestore. Rejects with a `not-found` error if no ruleset is applied\n   * on Firestore.\n   *\n   * @returns A promise that fulfills with the Firestore ruleset.\n   */\n  public getFirestoreRuleset(): Promise<Ruleset> {\n    return this.getRulesetForRelease(SecurityRules.CLOUD_FIRESTORE);\n  }\n\n  /**\n   * Creates a new {@link Ruleset} from the given\n   * source, and applies it to Cloud Firestore.\n   *\n   * @param source - Rules source to apply.\n   * @returns A promise that fulfills when the ruleset is created and released.\n   */\n  public releaseFirestoreRulesetFromSource(source: string | Buffer): Promise<Ruleset> {\n    return Promise.resolve()\n      .then(() => {\n        const rulesFile = this.createRulesFileFromSource('firestore.rules', source);\n        return this.createRuleset(rulesFile);\n      })\n      .then((ruleset) => {\n        return this.releaseFirestoreRuleset(ruleset)\n          .then(() => {\n            return ruleset;\n          });\n      });\n  }\n\n  /**\n   * Applies the specified {@link Ruleset} ruleset\n   * to Cloud Firestore.\n   *\n   * @param ruleset - Name of the ruleset to apply or a `RulesetMetadata` object\n   *   containing the name.\n   * @returns A promise that fulfills when the ruleset is released.\n   */\n  public releaseFirestoreRuleset(ruleset: string | RulesetMetadata): Promise<void> {\n    return this.releaseRuleset(ruleset, SecurityRules.CLOUD_FIRESTORE);\n  }\n\n  /**\n   * Gets the {@link Ruleset} currently applied to a\n   * Cloud Storage bucket. Rejects with a `not-found` error if no ruleset is applied\n   * on the bucket.\n   *\n   * @param bucket - Optional name of the Cloud Storage bucket to be retrieved. If not\n   *   specified, retrieves the ruleset applied on the default bucket configured via\n   *   `AppOptions`.\n   * @returns A promise that fulfills with the Cloud Storage ruleset.\n   */\n  public getStorageRuleset(bucket?: string): Promise<Ruleset> {\n    return Promise.resolve()\n      .then(() => {\n        return this.getBucketName(bucket);\n      })\n      .then((bucketName) => {\n        return this.getRulesetForRelease(`${SecurityRules.FIREBASE_STORAGE}/${bucketName}`);\n      });\n  }\n\n  /**\n   * Creates a new {@link Ruleset} from the given\n   * source, and applies it to a Cloud Storage bucket.\n   *\n   * @param source - Rules source to apply.\n   * @param bucket - Optional name of the Cloud Storage bucket to apply the rules on. If\n   *   not specified, applies the ruleset on the default bucket configured via\n   *   {@link firebase-admin.app#AppOptions}.\n   * @returns A promise that fulfills when the ruleset is created and released.\n   */\n  public releaseStorageRulesetFromSource(source: string | Buffer, bucket?: string): Promise<Ruleset> {\n    return Promise.resolve()\n      .then(() => {\n        // Bucket name is not required until the last step. But since there's a createRuleset step\n        // before then, make sure to run this check and fail early if the bucket name is invalid.\n        this.getBucketName(bucket);\n        const rulesFile = this.createRulesFileFromSource('storage.rules', source);\n        return this.createRuleset(rulesFile);\n      })\n      .then((ruleset) => {\n        return this.releaseStorageRuleset(ruleset, bucket)\n          .then(() => {\n            return ruleset;\n          });\n      });\n  }\n\n  /**\n   * Applies the specified {@link Ruleset} ruleset\n   * to a Cloud Storage bucket.\n   *\n   * @param ruleset - Name of the ruleset to apply or a `RulesetMetadata` object\n   *   containing the name.\n   * @param bucket - Optional name of the Cloud Storage bucket to apply the rules on. If\n   *   not specified, applies the ruleset on the default bucket configured via\n   *   {@link firebase-admin.app#AppOptions}.\n   * @returns A promise that fulfills when the ruleset is released.\n   */\n  public releaseStorageRuleset(ruleset: string | RulesetMetadata, bucket?: string): Promise<void> {\n    return Promise.resolve()\n      .then(() => {\n        return this.getBucketName(bucket);\n      })\n      .then((bucketName) => {\n        return this.releaseRuleset(ruleset, `${SecurityRules.FIREBASE_STORAGE}/${bucketName}`);\n      });\n  }\n\n  /**\n   * Creates a {@link RulesFile} with the given name\n   * and source. Throws an error if any of the arguments are invalid. This is a local\n   * operation, and does not involve any network API calls.\n   *\n   * @example\n   * ```javascript\n   * const source = '// Some rules source';\n   * const rulesFile = admin.securityRules().createRulesFileFromSource(\n   *   'firestore.rules', source);\n   * ```\n   *\n   * @param name - Name to assign to the rules file. This is usually a short file name that\n   *   helps identify the file in a ruleset.\n   * @param source - Contents of the rules file.\n   * @returns A new rules file instance.\n   */\n  public createRulesFileFromSource(name: string, source: string | Buffer): RulesFile {\n    if (!validator.isNonEmptyString(name)) {\n      throw new FirebaseSecurityRulesError(\n        'invalid-argument', 'Name must be a non-empty string.');\n    }\n\n    let content: string;\n    if (validator.isNonEmptyString(source)) {\n      content = source;\n    } else if (validator.isBuffer(source)) {\n      content = source.toString('utf-8');\n    } else {\n      throw new FirebaseSecurityRulesError(\n        'invalid-argument', 'Source must be a non-empty string or a Buffer.');\n    }\n\n    return {\n      name,\n      content,\n    };\n  }\n\n  /**\n   * Creates a new {@link Ruleset} from the given {@link RulesFile}.\n   *\n   * @param file - Rules file to include in the new `Ruleset`.\n   * @returns A promise that fulfills with the newly created `Ruleset`.\n   */\n  public createRuleset(file: RulesFile): Promise<Ruleset> {\n    const ruleset: RulesetContent = {\n      source: {\n        files: [ file ],\n      },\n    };\n\n    return this.client.createRuleset(ruleset)\n      .then((rulesetResponse) => {\n        return new Ruleset(rulesetResponse);\n      });\n  }\n\n  /**\n   * Deletes the {@link Ruleset} identified by the given\n   * name. The input name should be the short name string without the project ID\n   * prefix. For example, to delete the `projects/project-id/rulesets/my-ruleset`,\n...\n// Path: src/security-rules/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Security Rules for Cloud Firestore and Cloud Storage.\n *\n * @packageDocumentation\n */\n\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { SecurityRules } from './security-rules';\n\nexport {\n  RulesFile,\n  Ruleset,\n  RulesetMetadata,\n  RulesetMetadataList,\n  SecurityRules,\n} from './security-rules';\n\n/**\n * Gets the {@link SecurityRules} service for the default app or a given app.\n *\n * `admin.securityRules()` can be called with no arguments to access the\n * default app's `SecurityRules` service, or as `admin.securityRules(app)` to access\n * the `SecurityRules` service associated with a specific app.\n *\n * @example\n * ```javascript\n * // Get the SecurityRules service for the default app\n * const defaultSecurityRules = getSecurityRules();\n * ```\n *\n * @example\n * ```javascript\n * // Get the SecurityRules service for a given app\n * const otherSecurityRules = getSecurityRules(otherApp);\n * ```\n *\n * @param app - Optional app to return the `SecurityRules` service\n *     for. If not provided, the default `SecurityRules` service\n *     is returned.\n * @returns The default `SecurityRules` service if no app is provided, or the\n *   `SecurityRules` service associated with the provided app.\n */\nexport function getSecurityRules(app?: App): SecurityRules {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('securityRules', (app) => new SecurityRules(app));\n}\n\n// Path: src/auth/auth-config.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport * as validator from '../utils/validator';\nimport { deepCopy } from '../utils/deep-copy';\nimport { AuthClientErrorCode, FirebaseAuthError } from '../utils/error';\n\n/**\n * Interface representing base properties of a user-enrolled second factor for a\n * `CreateRequest`.\n */\nexport interface BaseCreateMultiFactorInfoRequest {\n\n  /**\n   * The optional display name for an enrolled second factor.\n   */\n  displayName?: string;\n\n  /**\n   * The type identifier of the second factor. For SMS second factors, this is `phone`.\n   */\n  factorId: string;\n}\n\n/**\n * Interface representing a phone specific user-enrolled second factor for a\n * `CreateRequest`.\n */\nexport interface CreatePhoneMultiFactorInfoRequest extends BaseCreateMultiFactorInfoRequest {\n\n  /**\n   * The phone number associated with a phone second factor.\n   */\n  phoneNumber: string;\n}\n\n/**\n * Type representing the properties of a user-enrolled second factor\n * for a `CreateRequest`.\n */\nexport type CreateMultiFactorInfoRequest = | CreatePhoneMultiFactorInfoRequest;\n\n/**\n * Interface representing common properties of a user-enrolled second factor\n * for an `UpdateRequest`.\n */\nexport interface BaseUpdateMultiFactorInfoRequest {\n\n  /**\n   * The ID of the enrolled second factor. This ID is unique to the user. When not provided,\n   * a new one is provisioned by the Auth server.\n   */\n  uid?: string;\n\n  /**\n   * The optional display name for an enrolled second factor.\n   */\n  displayName?: string;\n\n  /**\n   * The optional date the second factor was enrolled, formatted as a UTC string.\n   */\n  enrollmentTime?: string;\n\n  /**\n   * The type identifier of the second factor. For SMS second factors, this is `phone`.\n   */\n  factorId: string;\n}\n\n/**\n * Interface representing a phone specific user-enrolled second factor\n * for an `UpdateRequest`.\n */\nexport interface UpdatePhoneMultiFactorInfoRequest extends BaseUpdateMultiFactorInfoRequest {\n\n  /**\n   * The phone number associated with a phone second factor.\n   */\n  phoneNumber: string;\n}\n\n/**\n * Type representing the properties of a user-enrolled second factor\n * for an `UpdateRequest`.\n */\nexport type UpdateMultiFactorInfoRequest = | UpdatePhoneMultiFactorInfoRequest;\n\n/**\n * The multi-factor related user settings for create operations.\n */\nexport interface MultiFactorCreateSettings {\n\n  /**\n   * The created user's list of enrolled second factors.\n   */\n  enrolledFactors: CreateMultiFactorInfoRequest[];\n}\n\n/**\n * The multi-factor related user settings for update operations.\n */\nexport interface MultiFactorUpdateSettings {\n\n  /**\n   * The updated list of enrolled second factors. The provided list overwrites the user's\n   * existing list of second factors.\n   * When null is passed, all of the user's existing second factors are removed.\n   */\n  enrolledFactors: UpdateMultiFactorInfoRequest[] | null;\n}\n\n/**\n * Interface representing the properties to update on the provided user.\n */\nexport interface UpdateRequest {\n\n  /**\n   * Whether or not the user is disabled: `true` for disabled;\n   * `false` for enabled.\n   */\n  disabled?: boolean;\n\n  /**\n   * The user's display name.\n   */\n  displayName?: string | null;\n\n  /**\n   * The user's primary email.\n   */\n  email?: string;\n\n  /**\n   * Whether or not the user's primary email is verified.\n   */\n  emailVerified?: boolean;\n\n  /**\n   * The user's unhashed password.\n   */\n  password?: string;\n\n  /**\n   * The user's primary phone number.\n   */\n  phoneNumber?: string | null;\n\n  /**\n   * The user's photo URL.\n   */\n  photoURL?: string | null;\n\n  /**\n   * The user's updated multi-factor related properties.\n   */\n  multiFactor?: MultiFactorUpdateSettings;\n\n  /**\n   * Links this user to the specified provider.\n   *\n   * Linking a provider to an existing user account does not invalidate the\n   * refresh token of that account. In other words, the existing account\n   * would continue to be able to access resources, despite not having used\n   * the newly linked provider to log in. If you wish to force the user to\n   * authenticate with this new provider, you need to (a) revoke their\n   * refresh token (see\n   * https://firebase.google.com/docs/auth/admin/manage-sessions#revoke_refresh_tokens),\n   * and (b) ensure no other authentication methods are present on this\n   * account.\n   */\n  providerToLink?: UserProvider;\n\n  /**\n   * Unlinks this user from the specified providers.\n   */\n  providersToUnlink?: string[];\n}\n\n/**\n * Represents a user identity provider that can be associated with a Firebase user.\n */\nexport interface UserProvider {\n\n  /**\n   * The user identifier for the linked provider.\n   */\n  uid?: string;\n\n  /**\n   * The display name for the linked provider.\n   */\n  displayName?: string;\n\n  /**\n   * The email for the linked provider.\n   */\n  email?: string;\n\n  /**\n   * The phone number for the linked provider.\n   */\n  phoneNumber?: string;\n\n  /**\n   * The photo URL for the linked provider.\n   */\n  photoURL?: string;\n\n  /**\n   * The linked provider ID (for example, \"google.com\" for the Google provider).\n   */\n  providerId?: string;\n}\n\n\n/**\n * Interface representing the properties to set on a new user record to be\n * created.\n */\nexport interface CreateRequest extends UpdateRequest {\n\n  /**\n   * The user's `uid`.\n   */\n  uid?: string;\n\n  /**\n   * The user's multi-factor related properties.\n   */\n  multiFactor?: MultiFactorCreateSettings;\n}\n\n/**\n * The response interface for listing provider configs. This is only available\n * when listing all identity providers' configurations via\n * {@link BaseAuth.listProviderConfigs}.\n */\nexport interface ListProviderConfigResults {\n\n  /**\n   * The list of providers for the specified type in the current page.\n   */\n  providerConfigs: AuthProviderConfig[];\n\n  /**\n   * The next page token, if available.\n   */\n  pageToken?: string;\n}\n\n/**\n * The filter interface used for listing provider configurations. This is used\n * when specifying how to list configured identity providers via\n * {@link BaseAuth.listProviderConfigs}.\n */\nexport interface AuthProviderConfigFilter {\n\n  /**\n   * The Auth provider configuration filter. This can be either `saml` or `oidc`.\n   * The former is used to look up SAML providers only, while the latter is used\n   * for OIDC providers.\n   */\n  type: 'saml' | 'oidc';\n\n  /**\n   * The maximum number of results to return per page. The default and maximum is\n   * 100.\n   */\n  maxResults?: number;\n\n  /**\n   * The next page token. When not specified, the lookup starts from the beginning\n   * of the list.\n   */\n  pageToken?: string;\n}\n\n/**\n * The request interface for updating a SAML Auth provider. This is used\n * when updating a SAML provider's configuration via\n * {@link BaseAuth.updateProviderConfig}.\n */\nexport interface SAMLUpdateAuthProviderRequest {\n\n  /**\n   * The SAML provider's updated display name. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  displayName?: string;\n\n  /**\n   * Whether the SAML provider is enabled or not. If not provided, the existing\n   * configuration's setting is not modified.\n   */\n  enabled?: boolean;\n\n  /**\n   * The SAML provider's updated IdP entity ID. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  idpEntityId?: string;\n\n  /**\n   * The SAML provider's updated SSO URL. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  ssoURL?: string;\n\n  /**\n   * The SAML provider's updated list of X.509 certificated. If not provided, the\n   * existing configuration list is not modified.\n   */\n  x509Certificates?: string[];\n\n  /**\n   * The SAML provider's updated RP entity ID. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  rpEntityId?: string;\n\n  /**\n   * The SAML provider's callback URL. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  callbackURL?: string;\n}\n\n/**\n * The request interface for updating an OIDC Auth provider. This is used\n * when updating an OIDC provider's configuration via\n * {@link BaseAuth.updateProviderConfig}.\n */\nexport interface OIDCUpdateAuthProviderRequest {\n\n  /**\n   * The OIDC provider's updated display name. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  displayName?: string;\n\n  /**\n   * Whether the OIDC provider is enabled or not. If not provided, the existing\n   * configuration's setting is not modified.\n   */\n  enabled?: boolean;\n\n  /**\n   * The OIDC provider's updated client ID. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  clientId?: string;\n\n  /**\n   * The OIDC provider's updated issuer. If not provided, the existing\n   * configuration's value is not modified.\n   */\n  issuer?: string;\n\n  /**\n   * The OIDC provider's client secret to enable OIDC code flow.\n   * If not provided, the existing configuration's value is not modified.\n   */\n  clientSecret?: string;\n\n  /**\n   * The OIDC provider's response object for OAuth authorization flow.\n   */\n  responseType?: OAuthResponseType;\n}\n\nexport type UpdateAuthProviderRequest =\n  SAMLUpdateAuthProviderRequest | OIDCUpdateAuthProviderRequest;\n\n/** A maximum of 10 test phone number / code pairs can be configured. */\nexport const MAXIMUM_TEST_PHONE_NUMBERS = 10;\n\n/** The server side SAML configuration request interface. */\nexport interface SAMLConfigServerRequest {\n  idpConfig?: {\n    idpEntityId?: string;\n    ssoUrl?: string;\n    idpCertificates?: Array<{\n      x509Certificate: string;\n    }>;\n    signRequest?: boolean;\n  };\n  spConfig?: {\n    spEntityId?: string;\n    callbackUri?: string;\n  };\n  displayName?: string;\n  enabled?: boolean;\n  [key: string]: any;\n}\n\n/** The server side SAML configuration response interface. */\nexport interface SAMLConfigServerResponse {\n  // Used when getting config.\n  // projects/${projectId}/inboundSamlConfigs/${providerId}\n  name?: string;\n  idpConfig?: {\n    idpEntityId?: string;\n    ssoUrl?: string;\n    idpCertificates?: Array<{\n      x509Certificate: string;\n    }>;\n    signRequest?: boolean;\n  };\n  spConfig?: {\n    spEntityId?: string;\n    callbackUri?: string;\n  };\n  displayName?: string;\n  enabled?: boolean;\n}\n\n/** The server side OIDC configuration request interface. */\nexport interface OIDCConfigServerRequest {\n  clientId?: string;\n  issuer?: string;\n  displayName?: string;\n  enabled?: boolean;\n  clientSecret?: string;\n  responseType?: OAuthResponseType;\n  [key: string]: any;\n}\n\n/** The server side OIDC configuration response interface. */\nexport interface OIDCConfigServerResponse {\n  // Used when getting config.\n  // projects/${projectId}/oauthIdpConfigs/${providerId}\n  name?: string;\n  clientId?: string;\n  issuer?: string;\n  displayName?: string;\n  enabled?: boolean;\n  clientSecret?: string;\n  responseType?: OAuthResponseType;\n}\n\n/** The server side email configuration request interface. */\nexport interface EmailSignInConfigServerRequest {\n  allowPasswordSignup?: boolean;\n  enableEmailLinkSignin?: boolean;\n}\n\n/** Identifies the server side second factor type. */\ntype AuthFactorServerType = 'PHONE_SMS';\n\n/** Client Auth factor type to server auth factor type mapping. */\nconst AUTH_FACTOR_CLIENT_TO_SERVER_TYPE: {[key: string]: AuthFactorServerType} = {\n  phone: 'PHONE_SMS',\n};\n\n/** Server Auth factor type to client auth factor type mapping. */\nconst AUTH_FACTOR_SERVER_TO_CLIENT_TYPE: {[key: string]: AuthFactorType} =\n  Object.keys(AUTH_FACTOR_CLIENT_TO_SERVER_TYPE)\n    .reduce((res: {[key: string]: AuthFactorType}, key) => {\n      res[AUTH_FACTOR_CLIENT_TO_SERVER_TYPE[key]] = key as AuthFactorType;\n      return res;\n    }, {});\n\n/** Server side multi-factor configuration. */\nexport interface MultiFactorAuthServerConfig {\n  state?: MultiFactorConfigState;\n  enabledProviders?: AuthFactorServerType[];\n  providerConfigs?: MultiFactorProviderConfig[];\n}\n\n/**\n * Identifies a second factor type.\n */\nexport type AuthFactorType = 'phone';\n\n/**\n * Identifies a multi-factor configuration state.\n */\nexport type MultiFactorConfigState = 'ENABLED' | 'DISABLED';\n\n/**\n * Interface representing a multi-factor configuration.\n * This can be used to define whether multi-factor authentication is enabled\n * or disabled and the list of second factor challenges that are supported.\n */\nexport interface MultiFactorConfig {\n  /**\n   * The multi-factor config state.\n   */\n  state: MultiFactorConfigState;\n\n  /**\n   * The list of identifiers for enabled second factors.\n   * Currently only \u2018phone\u2019 is supported.\n   */\n  factorIds?: AuthFactorType[];\n\n  /**\n   * A list of multi-factor provider configurations. \n   * MFA providers (except phone) indicate whether they're enabled through this field.   */\n  providerConfigs?: MultiFactorProviderConfig[];\n}\n\n/**\n * Interface representing a multi-factor auth provider configuration. \n * This interface is used for second factor auth providers other than SMS. \n * Currently, only TOTP is supported.\n */export interface MultiFactorProviderConfig {\n  /**\n   * Indicates whether this multi-factor provider is enabled or disabled.    */\n  state: MultiFactorConfigState;\n  /**\n   * TOTP multi-factor provider config.   */\n  totpProviderConfig?: TotpMultiFactorProviderConfig;\n}\n\n/**\n * Interface representing configuration settings for TOTP second factor auth. \n */\nexport interface TotpMultiFactorProviderConfig {\n  /**\n    *  The allowed number of adjacent intervals that will be used for verification\n    *  to compensate for clock skew.   */\n  adjacentIntervals?: number;\n}\n\n/**\n * Defines the multi-factor config class used to convert client side MultiFactorConfig\n * to a format that is understood by the Auth server.\n * \n * @internal\n */\nexport class MultiFactorAuthConfig implements MultiFactorConfig {\n\n  /**\n   * The multi-factor config state.\n   */\n  public readonly state: MultiFactorConfigState;\n  /**\n   * The list of identifiers for enabled second factors.\n   * Currently only \u2018phone\u2019 is supported.\n   */\n  public readonly factorIds: AuthFactorType[];\n  /**\n   * A list of multi-factor provider specific config. \n   * New MFA providers (except phone) will indicate enablement/disablement through this field.\n   */\n  public readonly providerConfigs: MultiFactorProviderConfig[];\n\n  /**\n   * Static method to convert a client side request to a MultiFactorAuthServerConfig.\n   * Throws an error if validation fails.\n   *\n   * @param options - The options object to convert to a server request.\n   * @returns The resulting server request.\n   * @internal\n   */\n  public static buildServerRequest(options: MultiFactorConfig): MultiFactorAuthServerConfig {\n    const request: MultiFactorAuthServerConfig = {};\n    MultiFactorAuthConfig.validate(options);\n    if (Object.prototype.hasOwnProperty.call(options, 'state')) {\n      request.state = options.state;\n    }\n    if (Object.prototype.hasOwnProperty.call(options, 'factorIds')) {\n      (options.factorIds || []).forEach((factorId) => {\n        if (typeof request.enabledProviders === 'undefined') {\n          request.enabledProviders = [];\n        }\n        request.enabledProviders.push(AUTH_FACTOR_CLIENT_TO_SERVER_TYPE[factorId]);\n      });\n      // In case an empty array is passed. Ensure it gets populated so the array is cleared.\n      if (options.factorIds && options.factorIds.length === 0) {\n        request.enabledProviders = [];\n      }\n    }\n    if (Object.prototype.hasOwnProperty.call(options, 'providerConfigs')) {\n      request.providerConfigs = options.providerConfigs;\n    }\n    return request;\n  }\n\n  /**\n   * Validates the MultiFactorConfig options object. Throws an error on failure.\n   *\n   * @param options - The options object to validate.\n   */\n  public static validate(options: MultiFactorConfig): void {\n    const validKeys = {\n      state: true,\n      factorIds: true,\n      providerConfigs: true,\n    };\n    if (!validator.isNonNullObject(options)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"MultiFactorConfig\" must be a non-null object.',\n      );\n    }\n    // Check for unsupported top level attributes.\n    for (const key in options) {\n      if (!(key in validKeys)) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_CONFIG,\n          `\"${key}\" is not a valid MultiFactorConfig parameter.`,\n        );\n      }\n    }\n    // Validate content.\n    if (typeof options.state !== 'undefined' &&\n        options.state !== 'ENABLED' &&\n        options.state !== 'DISABLED') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"MultiFactorConfig.state\" must be either \"ENABLED\" or \"DISABLED\".',\n      );\n    }\n\n    if (typeof options.factorIds !== 'undefined') {\n      if (!validator.isArray(options.factorIds)) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_CONFIG,\n          '\"MultiFactorConfig.factorIds\" must be an array of valid \"AuthFactorTypes\".',\n        );\n      }\n\n      // Validate content of array.\n      options.factorIds.forEach((factorId) => {\n        if (typeof AUTH_FACTOR_CLIENT_TO_SERVER_TYPE[factorId] === 'undefined') {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_CONFIG,\n            `\"${factorId}\" is not a valid \"AuthFactorType\".`,\n          );\n        }\n      });\n    }\n\n    if (typeof options.providerConfigs !== 'undefined') {\n      if (!validator.isArray(options.providerConfigs)) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_CONFIG,\n          '\"MultiFactorConfig.providerConfigs\" must be an array of valid \"MultiFactorProviderConfig.\"',\n        );\n      }\n      //Validate content of array.\n      options.providerConfigs.forEach((multiFactorProviderConfig) => {\n        if (typeof multiFactorProviderConfig === 'undefined' || !validator.isObject(multiFactorProviderConfig)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_CONFIG,\n            `\"${multiFactorProviderConfig}\" is not a valid \"MultiFactorProviderConfig\" type.`\n          )\n        }\n        const validProviderConfigKeys = {\n          state: true,\n          totpProviderConfig: true,\n        };\n        for (const key in multiFactorProviderConfig) {\n          if (!(key in validProviderConfigKeys)) {\n            throw new FirebaseAuthError(\n              AuthClientErrorCode.INVALID_CONFIG,\n              `\"${key}\" is not a valid ProviderConfig parameter.`,\n            );\n          }\n        }\n        if (typeof multiFactorProviderConfig.state === 'undefined' ||\n          (multiFactorProviderConfig.state !== 'ENABLED' &&\n            multiFactorProviderConfig.state !== 'DISABLED')) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_CONFIG,\n            '\"MultiFactorConfig.providerConfigs.state\" must be either \"ENABLED\" or \"DISABLED\".',\n          )\n        }\n        // Since TOTP is the only provider config available right now, not defining it will lead into an error\n        if (typeof multiFactorProviderConfig.totpProviderConfig === 'undefined') {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_CONFIG,\n            '\"MultiFactorConfig.providerConfigs.totpProviderConfig\" must be defined.'\n          )\n        }\n        const validTotpProviderConfigKeys = {\n          adjacentIntervals: true,\n        };\n        for (const key in multiFactorProviderConfig.totpProviderConfig) {\n          if (!(key in validTotpProviderConfigKeys)) {\n            throw new FirebaseAuthError(\n              AuthClientErrorCode.INVALID_CONFIG,\n              `\"${key}\" is not a valid TotpProviderConfig parameter.`,\n            );\n          }\n        }\n        const adjIntervals = multiFactorProviderConfig.totpProviderConfig.adjacentIntervals\n        if (typeof adjIntervals !== 'undefined' &&\n          (!Number.isInteger(adjIntervals) || adjIntervals < 0 || adjIntervals > 10)) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INVALID_ARGUMENT,\n            '\"MultiFactorConfig.providerConfigs.totpProviderConfig.adjacentIntervals\" must' +\n            ' be a valid number between 0 and 10 (both inclusive).'\n          )\n        }\n      });\n    }\n  }\n\n  /**\n   * The MultiFactorAuthConfig constructor.\n   *\n   * @param response - The server side response used to initialize the\n   *     MultiFactorAuthConfig object.\n   * @constructor\n   * @internal\n   */\n  constructor(response: MultiFactorAuthServerConfig) {\n    if (typeof response.state === 'undefined') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid multi-factor configuration response');\n    }\n    this.state = response.state;\n    this.factorIds = [];\n    (response.enabledProviders || []).forEach((enabledProvider) => {\n      // Ignore unsupported types. It is possible the current admin SDK version is\n      // not up to date and newer backend types are supported.\n      if (typeof AUTH_FACTOR_SERVER_TO_CLIENT_TYPE[enabledProvider] !== 'undefined') {\n        this.factorIds.push(AUTH_FACTOR_SERVER_TO_CLIENT_TYPE[enabledProvider]);\n      }\n    })\n    this.providerConfigs = [];\n    (response.providerConfigs || []).forEach((providerConfig) => {\n      if (typeof providerConfig !== 'undefined') {\n        if (typeof providerConfig.state === 'undefined' ||\n          typeof providerConfig.totpProviderConfig === 'undefined' ||\n          (typeof providerConfig.totpProviderConfig.adjacentIntervals !== 'undefined' &&\n            typeof providerConfig.totpProviderConfig.adjacentIntervals !== 'number')) {\n          throw new FirebaseAuthError(\n            AuthClientErrorCode.INTERNAL_ERROR,\n            'INTERNAL ASSERT FAILED: Invalid multi-factor configuration response');\n        }\n        this.providerConfigs.push(providerConfig);\n      }\n    })\n  }\n\n  /** Converts MultiFactorConfig to JSON object\n   * @returns The plain object representation of the multi-factor config instance. */\n  public toJSON(): object {\n    return {\n      state: this.state,\n      factorIds: this.factorIds,\n      providerConfigs: this.providerConfigs,\n    };\n  }\n}\n\n\n/**\n * Validates the provided map of test phone number / code pairs.\n * @param testPhoneNumbers - The phone number / code pairs to validate.\n */\nexport function\n validateTestPhoneNumbers(\n  testPhoneNumbers: {[phoneNumber: string]: string},\n): void {\n  if (!validator.isObject(testPhoneNumbers)) {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_ARGUMENT,\n      '\"testPhoneNumbers\" must be a map of phone number / code pairs.',\n    );\n  }\n  if (Object.keys(testPhoneNumbers).length > MAXIMUM_TEST_PHONE_NUMBERS) {\n    throw new FirebaseAuthError(AuthClientErrorCode.MAXIMUM_TEST_PHONE_NUMBER_EXCEEDED);\n  }\n  for (const phoneNumber in testPhoneNumbers) {\n    // Validate phone number.\n    if (!validator.isPhoneNumber(phoneNumber)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_TESTING_PHONE_NUMBER,\n        `\"${phoneNumber}\" is not a valid E.164 standard compliant phone number.`\n      );\n    }\n\n    // Validate code.\n    if (!validator.isString(testPhoneNumbers[phoneNumber]) ||\n        !/^[\\d]{6}$/.test(testPhoneNumbers[phoneNumber])) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_TESTING_PHONE_NUMBER,\n        `\"${testPhoneNumbers[phoneNumber]}\" is not a valid 6 digit code string.`\n      );\n    }\n  }\n}\n\n/**\n * The email sign in provider configuration.\n */\nexport interface EmailSignInProviderConfig {\n  /**\n   * Whether email provider is enabled.\n   */\n  enabled: boolean;\n\n  /**\n   * Whether password is required for email sign-in. When not required,\n   * email sign-in can be performed with password or via email link sign-in.\n   */\n  passwordRequired?: boolean; // In the backend API, default is true if not provided\n}\n\n\n/**\n * Defines the email sign-in config class used to convert client side EmailSignInConfig\n * to a format that is understood by the Auth server.\n *\n * @internal\n */\nexport class EmailSignInConfig implements EmailSignInProviderConfig {\n  public readonly enabled: boolean;\n  public readonly passwordRequired?: boolean;\n\n  /**\n   * Static method to convert a client side request to a EmailSignInConfigServerRequest.\n   * Throws an error if validation fails.\n   *\n   * @param options - The options object to convert to a server request.\n   * @returns The resulting server request.\n   * @internal\n   */\n  public static buildServerRequest(options: EmailSignInProviderConfig): EmailSignInConfigServerRequest {\n    const request: EmailSignInConfigServerRequest = {};\n    EmailSignInConfig.validate(options);\n    if (Object.prototype.hasOwnProperty.call(options, 'enabled')) {\n      request.allowPasswordSignup = options.enabled;\n    }\n    if (Object.prototype.hasOwnProperty.call(options, 'passwordRequired')) {\n      request.enableEmailLinkSignin = !options.passwordRequired;\n    }\n    return request;\n  }\n\n  /**\n   * Validates the EmailSignInConfig options object. Throws an error on failure.\n   *\n   * @param options - The options object to validate.\n   */\n  private static validate(options: EmailSignInProviderConfig): void {\n    // TODO: Validate the request.\n    const validKeys = {\n      enabled: true,\n      passwordRequired: true,\n    };\n    if (!validator.isNonNullObject(options)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"EmailSignInConfig\" must be a non-null object.',\n      );\n    }\n    // Check for unsupported top level attributes.\n    for (const key in options) {\n      if (!(key in validKeys)) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_ARGUMENT,\n          `\"${key}\" is not a valid EmailSignInConfig parameter.`,\n        );\n      }\n    }\n    // Validate content.\n    if (typeof options.enabled !== 'undefined' &&\n        !validator.isBoolean(options.enabled)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"EmailSignInConfig.enabled\" must be a boolean.',\n      );\n    }\n    if (typeof options.passwordRequired !== 'undefined' &&\n        !validator.isBoolean(options.passwordRequired)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_ARGUMENT,\n        '\"EmailSignInConfig.passwordRequired\" must be a boolean.',\n      );\n    }\n  }\n\n  /**\n   * The EmailSignInConfig constructor.\n   *\n   * @param response - The server side response used to initialize the\n   *     EmailSignInConfig object.\n   * @constructor\n   */\n  constructor(response: {[key: string]: any}) {\n    if (typeof response.allowPasswordSignup === 'undefined') {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid email sign-in configuration response');\n    }\n    this.enabled = response.allowPasswordSignup;\n    this.passwordRequired = !response.enableEmailLinkSignin;\n  }\n\n  /** @returns The plain object representation of the email sign-in config. */\n  public toJSON(): object {\n    return {\n      enabled: this.enabled,\n      passwordRequired: this.passwordRequired,\n    };\n  }\n}\n\n/**\n * The base Auth provider configuration interface.\n */\nexport interface BaseAuthProviderConfig {\n\n  /**\n   * The provider ID defined by the developer.\n   * For a SAML provider, this is always prefixed by `saml.`.\n   * For an OIDC provider, this is always prefixed by `oidc.`.\n   */\n  providerId: string;\n\n  /**\n   * The user-friendly display name to the current configuration. This name is\n   * also used as the provider label in the Cloud Console.\n   */\n  displayName?: string;\n\n  /**\n   * Whether the provider configuration is enabled or disabled. A user\n   * cannot sign in using a disabled provider.\n   */\n  enabled: boolean;\n}\n\n/**\n * The\n * [SAML](http://docs.oasis-open.org/security/saml/Post2.0/sstc-saml-tech-overview-2.0.html)\n * Auth provider configuration interface. A SAML provider can be created via\n * {@link BaseAuth.createProviderConfig}.\n */\nexport interface SAMLAuthProviderConfig extends BaseAuthProviderConfig {\n\n  /**\n   * The SAML IdP entity identifier.\n   */\n  idpEntityId: string;\n\n  /**\n   * The SAML IdP SSO URL. This must be a valid URL.\n   */\n  ssoURL: string;\n\n  /**\n   * The list of SAML IdP X.509 certificates issued by CA for this provider.\n   * Multiple certificates are accepted to prevent outages during\n   * IdP key rotation (for example ADFS rotates every 10 days). When the Auth\n   * server receives a SAML response, it will match the SAML response with the\n   * certificate on record. Otherwise the response is rejected.\n   * Developers are expected to manage the certificate updates as keys are\n   * rotated.\n   */\n  x509Certificates: string[];\n\n  /**\n   * The SAML relying party (service provider) entity ID.\n   * This is defined by the developer but needs to be provided to the SAML IdP.\n   */\n  rpEntityId: string;\n\n  /**\n   * This is fixed and must always be the same as the OAuth redirect URL\n   * provisioned by Firebase Auth,\n   * `https://project-id.firebaseapp.com/__/auth/handler` unless a custom\n   * `authDomain` is used.\n   * The callback URL should also be provided to the SAML IdP during\n   * configuration.\n   */\n  callbackURL?: string;\n}\n\n/**\n * The interface representing OIDC provider's response object for OAuth\n * authorization flow.\n * One of the following settings is required:\n * <ul>\n * <li>Set <code>code</code> to <code>true</code> for the code flow.</li>\n * <li>Set <code>idToken</code> to <code>true</code> for the ID token flow.</li>\n * </ul>\n */\nexport interface OAuthResponseType {\n  /**\n   * Whether ID token is returned from IdP's authorization endpoint.\n   */\n  idToken?: boolean;\n\n  /**\n   * Whether authorization code is returned from IdP's authorization endpoint.\n   */\n  code?: boolean;\n}\n\n/**\n * The [OIDC](https://openid.net/specs/openid-connect-core-1_0-final.html) Auth\n * provider configuration interface. An OIDC provider can be created via\n * {@link BaseAuth.createProviderConfig}.\n */\nexport interface OIDCAuthProviderConfig extends BaseAuthProviderConfig {\n\n  /**\n   * This is the required client ID used to confirm the audience of an OIDC\n   * provider's\n   * [ID token](https://openid.net/specs/openid-connect-core-1_0-final.html#IDToken).\n   */\n  clientId: string;\n\n  /**\n   * This is the required provider issuer used to match the provider issuer of\n   * the ID token and to determine the corresponding OIDC discovery document, eg.\n   * [`/.well-known/openid-configuration`](https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderConfig).\n   * This is needed for the following:\n   * <ul>\n   * <li>To verify the provided issuer.</li>\n   * <li>Determine the authentication/authorization endpoint during the OAuth\n   *     `id_token` authentication flow.</li>\n   * <li>To retrieve the public signing keys via `jwks_uri` to verify the OIDC\n   *     provider's ID token's signature.</li>\n   * <li>To determine the claims_supported to construct the user attributes to be\n   *     returned in the additional user info response.</li>\n   * </ul>\n   * ID token validation will be performed as defined in the\n   * [spec](https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation).\n   */\n  issuer: string;\n\n  /**\n   * The OIDC provider's client secret to enable OIDC code flow.\n   */\n  clientSecret?: string;\n\n  /**\n   * The OIDC provider's response object for OAuth authorization flow.\n   */\n  responseType?: OAuthResponseType;\n}\n\n/**\n * The Auth provider configuration type.\n * {@link BaseAuth.createProviderConfig}.\n */\nexport type AuthProviderConfig = SAMLAuthProviderConfig | OIDCAuthProviderConfig;\n\n/**\n * Defines the SAMLConfig class used to convert a client side configuration to its\n * server side representation.\n *\n * @internal\n */\nexport class SAMLConfig implements SAMLAuthProviderConfig {\n  public readonly enabled: boolean;\n  public readonly displayName?: string;\n  public readonly providerId: string;\n  public readonly idpEntityId: string;\n  public readonly ssoURL: string;\n  public readonly x509Certificates: string[];\n  public readonly rpEntityId: string;\n  public readonly callbackURL?: string;\n  public readonly enableRequestSigning?: boolean;\n\n  /**\n   * Converts a client side request to a SAMLConfigServerRequest which is the format\n   * accepted by the backend server.\n   * Throws an error if validation fails. If the request is not a SAMLConfig request,\n   * returns null.\n   *\n   * @param options - The options object to convert to a server request.\n   * @param ignoreMissingFields - Whether to ignore missing fields.\n   * @returns The resulting server request or null if not valid.\n   */\n  public static buildServerRequest(\n    options: Partial<SAMLAuthProviderConfig>,\n    ignoreMissingFields = false): SAMLConfigServerRequest | null {\n    const makeRequest = validator.isNonNullObject(options) &&\n        (options.providerId || ignoreMissingFields);\n    if (!makeRequest) {\n      return null;\n    }\n    const request: SAMLConfigServerRequest = {};\n    // Validate options.\n    SAMLConfig.validate(options, ignoreMissingFields);\n    request.enabled = options.enabled;\n    request.displayName = options.displayName;\n    // IdP config.\n    if (options.idpEntityId || options.ssoURL || options.x509Certificates) {\n      request.idpConfig = {\n        idpEntityId: options.idpEntityId,\n        ssoUrl: options.ssoURL,\n        signRequest: (options as any).enableRequestSigning,\n        idpCertificates: typeof options.x509Certificates === 'undefined' ? undefined : [],\n      };\n      if (options.x509Certificates) {\n        for (const cert of (options.x509Certificates || [])) {\n          request.idpConfig!.idpCertificates!.push({ x509Certificate: cert });\n        }\n      }\n    }\n    // RP config.\n    if (options.callbackURL || options.rpEntityId) {\n      request.spConfig = {\n        spEntityId: options.rpEntityId,\n        callbackUri: options.callbackURL,\n      };\n    }\n    return request;\n  }\n\n  /**\n   * Returns the provider ID corresponding to the resource name if available.\n   *\n   * @param resourceName - The server side resource name.\n   * @returns The provider ID corresponding to the resource, null otherwise.\n   */\n  public static getProviderIdFromResourceName(resourceName: string): string | null {\n    // name is of form projects/project1/inboundSamlConfigs/providerId1\n    const matchProviderRes = resourceName.match(/\\/inboundSamlConfigs\\/(saml\\..*)$/);\n    if (!matchProviderRes || matchProviderRes.length < 2) {\n      return null;\n    }\n    return matchProviderRes[1];\n  }\n\n  /**\n   * @param providerId - The provider ID to check.\n   * @returns Whether the provider ID corresponds to a SAML provider.\n   */\n  public static isProviderId(providerId: any): providerId is string {\n    return validator.isNonEmptyString(providerId) && providerId.indexOf('saml.') === 0;\n  }\n\n  /**\n   * Validates the SAMLConfig options object. Throws an error on failure.\n   *\n   * @param options - The options object to validate.\n   * @param ignoreMissingFields - Whether to ignore missing fields.\n   */\n  public static validate(options: Partial<SAMLAuthProviderConfig>, ignoreMissingFields = false): void {\n    const validKeys = {\n      enabled: true,\n      displayName: true,\n      providerId: true,\n      idpEntityId: true,\n      ssoURL: true,\n      x509Certificates: true,\n      rpEntityId: true,\n      callbackURL: true,\n      enableRequestSigning: true,\n    };\n    if (!validator.isNonNullObject(options)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig\" must be a valid non-null object.',\n      );\n    }\n    // Check for unsupported top level attributes.\n    for (const key in options) {\n      if (!(key in validKeys)) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_CONFIG,\n          `\"${key}\" is not a valid SAML config parameter.`,\n        );\n      }\n    }\n    // Required fields.\n    if (validator.isNonEmptyString(options.providerId)) {\n      if (options.providerId.indexOf('saml.') !== 0) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_PROVIDER_ID,\n          '\"SAMLAuthProviderConfig.providerId\" must be a valid non-empty string prefixed with \"saml.\".',\n        );\n      }\n    } else if (!ignoreMissingFields) {\n      // providerId is required and not provided correctly.\n      throw new FirebaseAuthError(\n        !options.providerId ? AuthClientErrorCode.MISSING_PROVIDER_ID : AuthClientErrorCode.INVALID_PROVIDER_ID,\n        '\"SAMLAuthProviderConfig.providerId\" must be a valid non-empty string prefixed with \"saml.\".',\n      );\n    }\n    if (!(ignoreMissingFields && typeof options.idpEntityId === 'undefined') &&\n        !validator.isNonEmptyString(options.idpEntityId)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.idpEntityId\" must be a valid non-empty string.',\n      );\n    }\n    if (!(ignoreMissingFields && typeof options.ssoURL === 'undefined') &&\n        !validator.isURL(options.ssoURL)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.ssoURL\" must be a valid URL string.',\n      );\n    }\n    if (!(ignoreMissingFields && typeof options.rpEntityId === 'undefined') &&\n        !validator.isNonEmptyString(options.rpEntityId)) {\n      throw new FirebaseAuthError(\n        !options.rpEntityId ? AuthClientErrorCode.MISSING_SAML_RELYING_PARTY_CONFIG :\n          AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.rpEntityId\" must be a valid non-empty string.',\n      );\n    }\n    if (!(ignoreMissingFields && typeof options.callbackURL === 'undefined') &&\n        !validator.isURL(options.callbackURL)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.callbackURL\" must be a valid URL string.',\n      );\n    }\n    if (!(ignoreMissingFields && typeof options.x509Certificates === 'undefined') &&\n        !validator.isArray(options.x509Certificates)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.x509Certificates\" must be a valid array of X509 certificate strings.',\n      );\n    }\n    (options.x509Certificates || []).forEach((cert: string) => {\n      if (!validator.isNonEmptyString(cert)) {\n        throw new FirebaseAuthError(\n          AuthClientErrorCode.INVALID_CONFIG,\n          '\"SAMLAuthProviderConfig.x509Certificates\" must be a valid array of X509 certificate strings.',\n        );\n      }\n    });\n    if (typeof (options as any).enableRequestSigning !== 'undefined' &&\n        !validator.isBoolean((options as any).enableRequestSigning)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.enableRequestSigning\" must be a boolean.',\n      );\n    }\n    if (typeof options.enabled !== 'undefined' &&\n        !validator.isBoolean(options.enabled)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.enabled\" must be a boolean.',\n      );\n    }\n    if (typeof options.displayName !== 'undefined' &&\n        !validator.isString(options.displayName)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_CONFIG,\n        '\"SAMLAuthProviderConfig.displayName\" must be a valid string.',\n      );\n    }\n  }\n\n  /**\n   * The SAMLConfig constructor.\n   *\n   * @param response - The server side response used to initialize the SAMLConfig object.\n   * @constructor\n   */\n  constructor(response: SAMLConfigServerResponse) {\n    if (!response ||\n        !response.idpConfig ||\n        !response.idpConfig.idpEntityId ||\n        !response.idpConfig.ssoUrl ||\n        !response.spConfig ||\n        !response.spConfig.spEntityId ||\n        !response.name ||\n        !(validator.isString(response.name) &&\n          SAMLConfig.getProviderIdFromResourceName(response.name))) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid SAML configuration response');\n    }\n\n    const providerId = SAMLConfig.getProviderIdFromResourceName(response.name);\n    if (!providerId) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INTERNAL_ERROR,\n        'INTERNAL ASSERT FAILED: Invalid SAML configuration response');\n    }\n    this.providerId = providerId;\n\n    // RP config.\n    this.rpEntityId = response.spConfig.spEntityId;\n    this.callbackURL = response.spConfig.callbackUri;\n    // IdP config.\n    this.idpEntityId = response.idpConfig.idpEntityId;\n    this.ssoURL = response.idpConfig.ssoUrl;\n    this.enableRequestSigning = !!response.idpConfig.signRequest;\n    const x509Certificates: string[] = [];\n    for (const cert of (response.idpConfig.idpCertificates || [])) {\n      if (cert.x509Certificate) {\n        x509Certificates.push(cert.x509Certificate);\n      }\n    }\n    this.x509Certificates = x509Certificates;\n    // When enabled is undefined, it takes its default value of false.\n    this.enabled = !!response.enabled;\n    this.displayName = response.displayName;\n  }\n\n  /** @returns The plain object representation of the SAMLConfig. */\n  public toJSON(): object {\n    return {\n      enabled: this.enabled,\n      displayName: this.displayName,\n      providerId: this.providerId,\n      idpEntityId: this.idpEntityId,\n      ssoURL: this.ssoURL,\n      x509Certificates: deepCopy(this.x509Certificates),\n      rpEntityId: this.rpEntityId,\n      callbackURL: this.callbackURL,\n      enableRequestSigning: this.enableRequestSigning,\n    };\n  }\n}\n\n/**\n * Defines the OIDCConfig class used to convert a client side configuration to its\n * server side representation.\n *\n * @internal\n */\nexport class OIDCConfig implements OIDCAuthProviderConfig {\n  public readonly enabled: boolean;\n  public readonly displayName?: string;\n  public readonly providerId: string;\n  public readonly issuer: string;\n  public readonly clientId: string;\n  public readonly clientSecret?: string;\n  public readonly responseType: OAuthResponseType;\n\n  /**\n   * Converts a client side request to a OIDCConfigServerRequest which is the format\n   * accepted by the backend server.\n   * Throws an error if validation fails. If the request is not a OIDCConfig request,\n   * returns null.\n   *\n   * @param options - The options object to convert to a server request.\n   * @param ignoreMissingFields - Whether to ignore missing fields.\n   * @returns The resulting server request or null if not valid.\n   */\n  public static buildServerRequest(\n...\n", "needle_token_start": 10176, "needle_token_end": 10526, "code_context_ntokens": 16001, "output": ["```typescript\n/**\n * Validates the provided map of test phone number / code pairs.\n * @param testPhoneNumbers - The phone number / code pairs to validate.\n */\nexport function\n validateTestPhoneNumbers(\n  testPhoneNumbers: {[phoneNumber: string]: string},\n): void {\n  if (!validator.isObject(testPhoneNumbers)) {\n    throw new FirebaseAuthError(\n      AuthClientErrorCode.INVALID_ARGUMENT,\n      '\"testPhoneNumbers\" must be a map of phone number / code pairs.',\n    );\n  }\n  if (Object.keys(testPhoneNumbers).length > MAXIMUM_TEST_PHONE_NUMBERS) {\n    throw new FirebaseAuthError(AuthClientErrorCode.MAXIMUM_TEST_PHONE_NUMBER_EXCEEDED);\n  }\n  for (const phoneNumber in testPhoneNumbers) {\n    // Validate phone number.\n    if (!validator.isPhoneNumber(phoneNumber)) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_TESTING_PHONE_NUMBER,\n        `\"${phoneNumber}\" is not a valid E.164 standard compliant phone number.`\n      );\n    }\n\n    // Validate code.\n    if (!validator.isString(testPhoneNumbers[phoneNumber]) ||\n        !/^[\\d]{6}$/.test(testPhoneNumbers[phoneNumber])) {\n      throw new FirebaseAuthError(\n        AuthClientErrorCode.INVALID_TESTING_PHONE_NUMBER,\n        `\"${testPhoneNumbers[phoneNumber]}\" is not a valid 6 digit code string.`\n      );\n    }\n  }\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "createPart", "language": "typescript", "path": "src/messaging/batch-request-internal.ts", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To generate a formatted string representing a single part of a multipart HTTP request, including necessary headers and the serialized body of a sub-request.\n2. **Input**: A sub-request object, a string representing the multipart boundary, and an index number for the part.\n3. **Output**: A string that represents a single part of the multipart request, formatted with appropriate headers and the serialized sub-request.\n4. **Procedure**: The function first serializes the sub-request. It then constructs a string starting with the multipart boundary, followed by headers such as `Content-Length`, `Content-Type`, `content-id`, and `content-transfer-encoding`. The serialized sub-request is appended after these headers, and the entire part is returned as a string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/machine-learning/machine-learning-api-client.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport {\n  HttpRequestConfig, HttpClient, HttpError, AuthorizedHttpClient, ExponentialBackoffPoller\n} from '../utils/api-request';\nimport { PrefixedFirebaseError } from '../utils/error';\nimport * as utils from '../utils/index';\nimport * as validator from '../utils/validator';\nimport { FirebaseMachineLearningError, MachineLearningErrorCode } from './machine-learning-utils';\n\n/**\n * Firebase ML Model input objects\n */\nexport interface ModelOptionsBase {\n  displayName?: string;\n  tags?: string[];\n}\n\nexport interface GcsTfliteModelOptions extends ModelOptionsBase {\n  tfliteModel: {\n    gcsTfliteUri: string;\n  };\n}\n\nexport type ModelOptions = ModelOptionsBase | GcsTfliteModelOptions;\n\n/**\n * Interface representing options for listing Models.\n */\nexport interface ListModelsOptions {\n  /**\n   * An expression that specifies how to filter the results.\n   *\n   * Examples:\n   *\n   * ```\n   * display_name = your_model\n   * display_name : experimental_*\n   * tags: face_detector AND tags: experimental\n   * state.published = true\n   * ```\n   *\n   * See https://firebase.google.com/docs/ml/manage-hosted-models#list_your_projects_models\n   */\n  filter?: string;\n\n  /** The number of results to return in each page. */\n  pageSize?: number;\n\n  /** A token that specifies the result page to return. */\n  pageToken?: string;\n}\n\nconst ML_V1BETA2_API = 'https://firebaseml.googleapis.com/v1beta2';\nconst FIREBASE_VERSION_HEADER = {\n  'X-Firebase-Client': `fire-admin-node/${utils.getSdkVersion()}`,\n};\n\n// Operation polling defaults\nconst POLL_DEFAULT_MAX_TIME_MILLISECONDS = 120000;  // Maximum overall 2 minutes\nconst POLL_BASE_WAIT_TIME_MILLISECONDS = 3000;  // Start with 3 second delay\nconst POLL_MAX_WAIT_TIME_MILLISECONDS = 30000;  // Maximum 30 second delay\n\nexport interface StatusErrorResponse {\n    readonly code: number;\n    readonly message: string;\n}\n\n\nexport type ModelUpdateOptions = ModelOptions & { state?: { published?: boolean }};\n\nexport function isGcsTfliteModelOptions(options: ModelOptions): options is GcsTfliteModelOptions {\n  const gcsUri = (options as GcsTfliteModelOptions)?.tfliteModel?.gcsTfliteUri;\n  return typeof gcsUri !== 'undefined'\n}\n\nexport interface ModelContent {\n  readonly displayName?: string;\n  readonly tags?: string[];\n  readonly state?: {\n    readonly validationError?: StatusErrorResponse;\n    readonly published?: boolean;\n  };\n  readonly tfliteModel?: {\n    readonly gcsTfliteUri?: string;\n\n    readonly sizeBytes: number;\n  };\n}\n\nexport interface ModelResponse extends ModelContent {\n  readonly name: string;\n  readonly createTime: string;\n  readonly updateTime: string;\n  readonly etag: string;\n  readonly modelHash?: string;\n  readonly activeOperations?: OperationResponse[];\n}\n\nexport interface ListModelsResponse {\n  readonly models?: ModelResponse[];\n  readonly nextPageToken?: string;\n}\n\nexport interface OperationResponse {\n  readonly name?: string;\n  readonly metadata?: {[key: string]: any};\n  readonly done: boolean;\n  readonly error?: StatusErrorResponse;\n  readonly response?: ModelResponse;\n}\n\n\n/**\n * Class that facilitates sending requests to the Firebase ML backend API.\n *\n * @internal\n */\nexport class MachineLearningApiClient {\n\n  private readonly httpClient: HttpClient;\n  private projectIdPrefix?: string;\n\n  constructor(private readonly app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseMachineLearningError(\n        'invalid-argument',\n        'First argument passed to admin.machineLearning() must be a valid '\n          + 'Firebase app instance.');\n    }\n\n    this.httpClient = new AuthorizedHttpClient(app as FirebaseApp);\n  }\n\n  public createModel(model: ModelOptions): Promise<OperationResponse> {\n    if (!validator.isNonNullObject(model) ||\n        !validator.isNonEmptyString(model.displayName)) {\n      const err = new FirebaseMachineLearningError('invalid-argument', 'Invalid model content.');\n      return Promise.reject(err);\n    }\n    return this.getProjectUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'POST',\n          url: `${url}/models`,\n          data: model,\n        };\n        return this.sendRequest<OperationResponse>(request);\n      });\n  }\n\n  public updateModel(modelId: string, model: ModelUpdateOptions, updateMask: string[]): Promise<OperationResponse> {\n    if (!validator.isNonEmptyString(modelId) ||\n        !validator.isNonNullObject(model) ||\n        !validator.isNonEmptyArray(updateMask)) {\n      const err = new FirebaseMachineLearningError('invalid-argument', 'Invalid model or mask content.');\n      return Promise.reject(err);\n    }\n    return this.getProjectUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'PATCH',\n          url: `${url}/models/${modelId}?updateMask=${updateMask.join()}`,\n          data: model,\n        };\n        return this.sendRequest<OperationResponse>(request);\n      });\n  }\n\n  public getModel(modelId: string): Promise<ModelResponse> {\n    return Promise.resolve()\n      .then(() => {\n        return this.getModelName(modelId);\n      })\n      .then((modelName) => {\n        return this.getResourceWithShortName<ModelResponse>(modelName);\n      });\n  }\n\n  public getOperation(operationName: string): Promise<OperationResponse> {\n    return Promise.resolve()\n      .then(() => {\n        return this.getResourceWithFullName<OperationResponse>(operationName);\n      });\n  }\n\n  public listModels(options: ListModelsOptions = {}): Promise<ListModelsResponse> {\n    if (!validator.isNonNullObject(options)) {\n      const err = new FirebaseMachineLearningError('invalid-argument', 'Invalid ListModelsOptions');\n      return Promise.reject(err);\n    }\n    if (typeof options.filter !== 'undefined' && !validator.isNonEmptyString(options.filter)) {\n      const err = new FirebaseMachineLearningError('invalid-argument', 'Invalid list filter.');\n      return Promise.reject(err);\n    }\n    if (typeof options.pageSize !== 'undefined') {\n      if (!validator.isNumber(options.pageSize)) {\n        const err = new FirebaseMachineLearningError('invalid-argument', 'Invalid page size.');\n        return Promise.reject(err);\n      }\n      if (options.pageSize < 1 || options.pageSize > 100) {\n        const err = new FirebaseMachineLearningError(\n          'invalid-argument', 'Page size must be between 1 and 100.');\n        return Promise.reject(err);\n      }\n    }\n    if (typeof options.pageToken !== 'undefined' && !validator.isNonEmptyString(options.pageToken)) {\n      const err = new FirebaseMachineLearningError(\n        'invalid-argument', 'Next page token must be a non-empty string.');\n      return Promise.reject(err);\n    }\n    return this.getProjectUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'GET',\n          url: `${url}/models`,\n          data: options,\n        };\n        return this.sendRequest<ListModelsResponse>(request);\n      });\n  }\n\n  public deleteModel(modelId: string): Promise<void> {\n    return this.getProjectUrl()\n      .then((url) => {\n        const modelName = this.getModelName(modelId);\n        const request: HttpRequestConfig = {\n          method: 'DELETE',\n          url: `${url}/${modelName}`,\n        };\n        return this.sendRequest<void>(request);\n      });\n  }\n\n  /**\n   * Handles a Long Running Operation coming back from the server.\n   *\n   * @param op - The operation to handle\n   * @param options - The options for polling\n   */\n  public handleOperation(\n    op: OperationResponse,\n    options?: {\n      wait?: boolean;\n      maxTimeMillis?: number;\n      baseWaitMillis?: number;\n      maxWaitMillis?: number;\n    }):\n    Promise<ModelResponse> {\n    if (op.done) {\n      if (op.response) {\n        return Promise.resolve(op.response);\n      } else if (op.error) {\n        const err = FirebaseMachineLearningError.fromOperationError(\n          op.error.code, op.error.message);\n        return Promise.reject(err);\n      }\n\n      // Done operations must have either a response or an error.\n      throw new FirebaseMachineLearningError('invalid-server-response',\n        'Invalid operation response.');\n    }\n\n    // Operation is not done\n    if (options?.wait) {\n      return this.pollOperationWithExponentialBackoff(op.name!, options);\n    }\n\n    const metadata = op.metadata || {};\n    const metadataType: string = metadata['@type'] || '';\n    if (!metadataType.includes('ModelOperationMetadata')) {\n      throw new FirebaseMachineLearningError('invalid-server-response',\n        `Unknown Metadata type: ${JSON.stringify(metadata)}`);\n    }\n\n    return this.getModel(extractModelId(metadata.name));\n  }\n\n  // baseWaitMillis and maxWaitMillis should only ever be modified by unit tests to run faster.\n  private pollOperationWithExponentialBackoff(\n    opName: string,\n    options?: {\n      maxTimeMillis?: number;\n      baseWaitMillis?: number;\n      maxWaitMillis?: number;\n    }): Promise<ModelResponse> {\n\n    const maxTimeMilliseconds = options?.maxTimeMillis ?? POLL_DEFAULT_MAX_TIME_MILLISECONDS;\n    const baseWaitMillis = options?.baseWaitMillis ?? POLL_BASE_WAIT_TIME_MILLISECONDS;\n    const maxWaitMillis = options?.maxWaitMillis ?? POLL_MAX_WAIT_TIME_MILLISECONDS;\n\n    const poller = new ExponentialBackoffPoller<ModelResponse>(\n      baseWaitMillis,\n      maxWaitMillis,\n      maxTimeMilliseconds);\n\n    return poller.poll(() => {\n      return this.getOperation(opName)\n        .then((responseData: {[key: string]: any}) => {\n          if (!responseData.done) {\n            return null;\n          }\n          if (responseData.error) {\n            const err = FirebaseMachineLearningError.fromOperationError(\n              responseData.error.code, responseData.error.message);\n            throw err;\n          }\n          return responseData.response;\n        });\n    });\n  }\n\n  /**\n   * Gets the specified resource from the ML API. Resource names must be the short names without project\n   * ID prefix (e.g. `models/123456789`).\n   *\n   * @param {string} name Short name of the resource to get. e.g. 'models/12345'\n   * @returns {Promise<T>} A promise that fulfills with the resource.\n   */\n  private getResourceWithShortName<T>(name: string): Promise<T> {\n    return this.getProjectUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'GET',\n          url: `${url}/${name}`,\n        };\n        return this.sendRequest<T>(request);\n      });\n  }\n\n  /**\n   * Gets the specified resource from the ML API. Resource names must be the full names including project\n   * number prefix.\n   * @param fullName - Full resource name of the resource to get. e.g. projects/123465/operations/987654\n   * @returns {Promise<T>} A promise that fulfulls with the resource.\n   */\n  private getResourceWithFullName<T>(fullName: string): Promise<T> {\n    const request: HttpRequestConfig = {\n      method: 'GET',\n      url: `${ML_V1BETA2_API}/${fullName}`\n    };\n    return this.sendRequest<T>(request);\n  }\n\n  private sendRequest<T>(request: HttpRequestConfig): Promise<T> {\n    request.headers = FIREBASE_VERSION_HEADER;\n    return this.httpClient.send(request)\n      .then((resp) => {\n        return resp.data as T;\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  private toFirebaseError(err: HttpError): PrefixedFirebaseError {\n    if (err instanceof PrefixedFirebaseError) {\n      return err;\n    }\n\n    const response = err.response;\n    if (!response.isJson()) {\n      return new FirebaseMachineLearningError(\n        'unknown-error',\n        `Unexpected response with status: ${response.status} and body: ${response.text}`);\n    }\n\n    const error: Error = (response.data as ErrorResponse).error || {};\n    let code: MachineLearningErrorCode = 'unknown-error';\n    if (error.status && error.status in ERROR_CODE_MAPPING) {\n      code = ERROR_CODE_MAPPING[error.status];\n    }\n    const message = error.message || `Unknown server error: ${response.text}`;\n    return new FirebaseMachineLearningError(code, message);\n  }\n\n  private getProjectUrl(): Promise<string> {\n    return this.getProjectIdPrefix()\n      .then((projectIdPrefix) => {\n        return `${ML_V1BETA2_API}/${projectIdPrefix}`;\n      });\n  }\n\n  private getProjectIdPrefix(): Promise<string> {\n    if (this.projectIdPrefix) {\n      return Promise.resolve(this.projectIdPrefix);\n    }\n\n    return utils.findProjectId(this.app)\n...\n// Path: src/storage/storage.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseError } from '../utils/error';\nimport { ServiceAccountCredential, isApplicationDefault } from '../app/credential-internal';\nimport { Bucket, Storage as StorageClient } from '@google-cloud/storage';\nimport * as utils from '../utils/index';\nimport * as validator from '../utils/validator';\n\n/**\n * The default `Storage` service if no\n * app is provided or the `Storage` service associated with the provided\n * app.\n */\nexport class Storage {\n\n  private readonly appInternal: App;\n  private readonly storageClient: StorageClient;\n\n  /**\n   * @param app - The app for this Storage service.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseError({\n        code: 'storage/invalid-argument',\n        message: 'First argument passed to admin.storage() must be a valid Firebase app instance.',\n      });\n    }\n\n    if (!process.env.STORAGE_EMULATOR_HOST && process.env.FIREBASE_STORAGE_EMULATOR_HOST) {\n      const firebaseStorageEmulatorHost = process.env.FIREBASE_STORAGE_EMULATOR_HOST;\n\n      if (firebaseStorageEmulatorHost.match(/https?:\\/\\//)) {\n        throw new FirebaseError({\n          code: 'storage/invalid-emulator-host',\n          message: 'FIREBASE_STORAGE_EMULATOR_HOST should not contain a protocol (http or https).',\n        });\n      }\n\n      process.env.STORAGE_EMULATOR_HOST = `http://${process.env.FIREBASE_STORAGE_EMULATOR_HOST}`;\n    }\n\n    let storage: typeof StorageClient;\n    try {\n      storage = require('@google-cloud/storage').Storage;\n    } catch (err) {\n      throw new FirebaseError({\n        code: 'storage/missing-dependencies',\n        message: 'Failed to import the Cloud Storage client library for Node.js. '\n          + 'Make sure to install the \"@google-cloud/storage\" npm package. '\n          + `Original error: ${err}`,\n      });\n    }\n\n    const projectId: string | null = utils.getExplicitProjectId(app);\n    const credential = app.options.credential;\n    if (credential instanceof ServiceAccountCredential) {\n      this.storageClient = new storage({\n        // When the SDK is initialized with ServiceAccountCredentials an explicit projectId is\n        // guaranteed to be available.\n        projectId: projectId!,\n        credentials: {\n          private_key: credential.privateKey,\n          client_email: credential.clientEmail,\n        },\n      });\n    } else if (isApplicationDefault(app.options.credential)) {\n      // Try to use the Google application default credentials.\n      this.storageClient = new storage();\n    } else {\n      throw new FirebaseError({\n        code: 'storage/invalid-credential',\n        message: 'Failed to initialize Google Cloud Storage client with the available credential. ' +\n          'Must initialize the SDK with a certificate credential or application default credentials ' +\n          'to use Cloud Storage API.',\n      });\n    }\n    this.appInternal = app;\n  }\n\n  /**\n   * Gets a reference to a Cloud Storage bucket.\n   *\n   * @param name - Optional name of the bucket to be retrieved. If name is not specified,\n   * retrieves a reference to the default bucket.\n   * @returns A {@link https://cloud.google.com/nodejs/docs/reference/storage/latest/Bucket | Bucket}\n   * instance as defined in the `@google-cloud/storage` package.\n   */\n  public bucket(name?: string): Bucket {\n    const bucketName = (typeof name !== 'undefined')\n      ? name :  this.appInternal.options.storageBucket;\n    if (validator.isNonEmptyString(bucketName)) {\n      return this.storageClient.bucket(bucketName);\n    }\n    throw new FirebaseError({\n      code: 'storage/invalid-argument',\n      message: 'Bucket name not specified or invalid. Specify a valid bucket name via the ' +\n                'storageBucket option when initializing the app, or specify the bucket name ' +\n                'explicitly when calling the getBucket() method.',\n    });\n  }\n  /**\n   * Optional app whose `Storage` service to\n   * return. If not provided, the default `Storage` service will be returned.\n   */\n  get app(): App {\n    return this.appInternal;\n  }\n}\n\n// Path: src/storage/utils.ts\nimport { File } from '@google-cloud/storage';\nexport interface FirebaseMetadata {\n  name: string;\n  bucket: string;\n  generation: string;\n  metageneration: string;\n  contentType: string;\n  timeCreated: string;\n  updated: string;\n  storageClass: string;\n  size: string;\n  md5Hash: string;\n  contentEncoding: string;\n  contentDisposition: string;\n  crc32c: string;\n  etag: string;\n  downloadTokens?: string;\n}\n\nexport function getFirebaseMetadata(\n  endpoint: string,\n  file: File\n): Promise<FirebaseMetadata> {\n  const uri = `${endpoint}/b/${file.bucket.name}/o/${encodeURIComponent(\n    file.name\n  )}`;\n\n  return new Promise((resolve, reject) => {\n    file.storage.makeAuthenticatedRequest(\n      {\n        method: 'GET',\n        uri,\n      },\n      (err, body) => {\n        if (err) {\n          reject(err);\n        } else {\n          resolve(body);\n        }\n      }\n    );\n  });\n}\n\n// Path: src/storage/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Cloud Storage for Firebase.\n *\n * @packageDocumentation\n */\n\nimport { File } from '@google-cloud/storage';\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { Storage } from './storage';\nimport { FirebaseError } from '../utils/error';\nimport { getFirebaseMetadata } from './utils';\n\nexport { Storage } from './storage';\n\n\n/**\n * Gets the {@link Storage} service for the default app or a given app.\n *\n * `getStorage()` can be called with no arguments to access the default\n * app's `Storage` service or as `getStorage(app)` to access the\n * `Storage` service associated with a specific app.\n *\n * @example\n * ```javascript\n * // Get the Storage service for the default app\n * const defaultStorage = getStorage();\n * ```\n *\n * @example\n * ```javascript\n * // Get the Storage service for a given app\n * const otherStorage = getStorage(otherApp);\n * ```\n */\nexport function getStorage(app?: App): Storage {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('storage', (app) => new Storage(app));\n}\n\n\n\n/**\n * Gets the download URL for the given {@link https://cloud.google.com/nodejs/docs/reference/storage/latest/storage/file | File}.\n * \n * @example\n * ```javascript\n * // Get the downloadUrl for a given file ref\n * const storage = getStorage();\n * const myRef = ref(storage, 'images/mountains.jpg');\n * const downloadUrl = await getDownloadURL(myRef);\n * ```\n */\nexport async function getDownloadURL(file: File): Promise<string> {\n  const endpoint =\n    (process.env.STORAGE_EMULATOR_HOST ||\n      'https://firebasestorage.googleapis.com') + '/v0';\n  const { downloadTokens } = await getFirebaseMetadata(endpoint, file);\n  if (!downloadTokens) {\n    throw new FirebaseError({\n      code: 'storage/no-download-token',\n      message:\n        'No download token available. Please create one in the Firebase Console.',\n    });\n  }\n  const [token] = downloadTokens.split(',');\n  return `${endpoint}/b/${file.bucket.name}/o/${encodeURIComponent(\n    file.name\n  )}?alt=media&token=${token}`;\n}\n\n// Path: src/machine-learning/machine-learning.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { getStorage } from '../storage/index';\nimport { FirebaseError } from '../utils/error';\nimport * as validator from '../utils/validator';\nimport { deepCopy } from '../utils/deep-copy';\nimport * as utils from '../utils';\nimport {\n  MachineLearningApiClient, ModelResponse, ModelUpdateOptions, isGcsTfliteModelOptions,\n  ListModelsOptions, ModelOptions,\n} from './machine-learning-api-client';\nimport { FirebaseMachineLearningError } from './machine-learning-utils';\n\n/** Response object for a listModels operation. */\nexport interface ListModelsResult {\n  /** A list of models in your project. */\n  readonly models: Model[];\n\n  /**\n   * A token you can use to retrieve the next page of results. If null, the\n   * current page is the final page.\n   */\n  readonly pageToken?: string;\n}\n\n/**\n * A TensorFlow Lite Model output object\n */\nexport interface TFLiteModel {\n  /** The size of the model. */\n  readonly sizeBytes: number;\n\n  /** The URI from which the model was originally provided to Firebase. */\n  readonly gcsTfliteUri?: string;\n}\n\n/**\n * The Firebase `MachineLearning` service interface.\n */\nexport class MachineLearning {\n\n  private readonly client: MachineLearningApiClient;\n  private readonly appInternal: App;\n\n  /**\n   * @param app - The app for this ML service.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseError({\n        code: 'machine-learning/invalid-argument',\n        message: 'First argument passed to admin.machineLearning() must be a ' +\n            'valid Firebase app instance.',\n      });\n    }\n\n    this.appInternal = app;\n    this.client = new MachineLearningApiClient(app);\n  }\n\n  /**\n   *  The {@link firebase-admin.app#App} associated with the current `MachineLearning`\n   *  service instance.\n   */\n  public get app(): App {\n    return this.appInternal;\n  }\n\n  /**\n   * Creates a model in the current Firebase project.\n   *\n   * @param model - The model to create.\n   *\n   * @returns A Promise fulfilled with the created model.\n   */\n  public createModel(model: ModelOptions): Promise<Model> {\n    return this.signUrlIfPresent(model)\n      .then((modelContent) => this.client.createModel(modelContent))\n      .then((operation) => this.client.handleOperation(operation))\n      .then((modelResponse) => new Model(modelResponse, this.client));\n  }\n\n  /**\n   * Updates a model's metadata or model file.\n   *\n   * @param modelId - The ID of the model to update.\n   * @param model - The model fields to update.\n   *\n   * @returns A Promise fulfilled with the updated model.\n   */\n  public updateModel(modelId: string, model: ModelOptions): Promise<Model> {\n    const updateMask = utils.generateUpdateMask(model);\n    return this.signUrlIfPresent(model)\n      .then((modelContent) => this.client.updateModel(modelId, modelContent, updateMask))\n      .then((operation) => this.client.handleOperation(operation))\n      .then((modelResponse) => new Model(modelResponse, this.client));\n  }\n\n  /**\n   * Publishes a Firebase ML model.\n   *\n   * A published model can be downloaded to client apps.\n   *\n   * @param modelId - The ID of the model to publish.\n   *\n   * @returns A Promise fulfilled with the published model.\n   */\n  public publishModel(modelId: string): Promise<Model> {\n    return this.setPublishStatus(modelId, true);\n  }\n\n  /**\n   * Unpublishes a Firebase ML model.\n   *\n   * @param modelId - The ID of the model to unpublish.\n   *\n   * @returns A Promise fulfilled with the unpublished model.\n   */\n  public unpublishModel(modelId: string): Promise<Model> {\n    return this.setPublishStatus(modelId, false);\n  }\n\n  /**\n   * Gets the model specified by the given ID.\n   *\n   * @param modelId - The ID of the model to get.\n   *\n   * @returns A Promise fulfilled with the model object.\n   */\n  public getModel(modelId: string): Promise<Model> {\n    return this.client.getModel(modelId)\n      .then((modelResponse) => new Model(modelResponse, this.client));\n  }\n\n  /**\n   * Lists the current project's models.\n   *\n   * @param options - The listing options.\n   *\n   * @returns A promise that\n   *     resolves with the current (filtered) list of models and the next page\n   *     token. For the last page, an empty list of models and no page token\n   *     are returned.\n   */\n  public listModels(options: ListModelsOptions = {}): Promise<ListModelsResult> {\n    return this.client.listModels(options)\n      .then((resp) => {\n        if (!validator.isNonNullObject(resp)) {\n          throw new FirebaseMachineLearningError(\n            'invalid-argument',\n            `Invalid ListModels response: ${JSON.stringify(resp)}`);\n        }\n        let models: Model[] = [];\n        if (resp.models) {\n          models = resp.models.map((rs) =>  new Model(rs, this.client));\n        }\n        const result: { models: Model[]; pageToken?: string } = { models };\n        if (resp.nextPageToken) {\n          result.pageToken = resp.nextPageToken;\n        }\n        return result;\n      });\n  }\n\n  /**\n   * Deletes a model from the current project.\n   *\n   * @param modelId - The ID of the model to delete.\n   */\n  public deleteModel(modelId: string): Promise<void> {\n    return this.client.deleteModel(modelId);\n  }\n\n  private setPublishStatus(modelId: string, publish: boolean): Promise<Model> {\n    const updateMask = ['state.published'];\n    const options: ModelUpdateOptions = { state: { published: publish } };\n    return this.client.updateModel(modelId, options, updateMask)\n      .then((operation) => this.client.handleOperation(operation))\n      .then((modelResponse) => new Model(modelResponse, this.client));\n  }\n\n  private signUrlIfPresent(options: ModelOptions): Promise<ModelOptions> {\n    const modelOptions = deepCopy(options);\n    if (isGcsTfliteModelOptions(modelOptions)) {\n      return this.signUrl(modelOptions.tfliteModel.gcsTfliteUri)\n        .then((uri: string) => {\n          modelOptions.tfliteModel.gcsTfliteUri = uri;\n          return modelOptions;\n        })\n        .catch((err: Error) => {\n          throw new FirebaseMachineLearningError(\n            'internal-error',\n            `Error during signing upload url: ${err.message}`);\n        });\n    }\n    return Promise.resolve(modelOptions);\n  }\n\n  private signUrl(unsignedUrl: string): Promise<string> {\n    const MINUTES_IN_MILLIS = 60 * 1000;\n    const URL_VALID_DURATION = 10 * MINUTES_IN_MILLIS;\n\n    const gcsRegex = /^gs:\\/\\/([a-z0-9_.-]{3,63})\\/(.+)$/;\n    const matches = gcsRegex.exec(unsignedUrl);\n    if (!matches) {\n      throw new FirebaseMachineLearningError(\n        'invalid-argument',\n        `Invalid unsigned url: ${unsignedUrl}`);\n    }\n    const bucketName = matches[1];\n    const blobName = matches[2];\n    const bucket = getStorage(this.app).bucket(bucketName);\n    const blob = bucket.file(blobName);\n    return blob.getSignedUrl({\n      action: 'read',\n      expires: Date.now() + URL_VALID_DURATION,\n    }).then((signUrl) => signUrl[0]);\n  }\n}\n\n/**\n * A Firebase ML Model output object.\n */\nexport class Model {\n  private model: ModelResponse;\n  private readonly client?: MachineLearningApiClient;\n\n  /**\n   * @internal\n   */\n  constructor(model: ModelResponse, client: MachineLearningApiClient) {\n    this.model = Model.validateAndClone(model);\n    this.client = client;\n  }\n\n  /** The ID of the model. */\n  get modelId(): string {\n    return extractModelId(this.model.name);\n  }\n\n  /**\n   * The model's name. This is the name you use from your app to load the\n   * model.\n   */\n  get displayName(): string {\n    return this.model.displayName!;\n  }\n\n  /**\n   * The model's tags, which can be used to group or filter models in list\n   * operations.\n   */\n  get tags(): string[] {\n    return this.model.tags || [];\n  }\n\n  /** The timestamp of the model's creation. */\n  get createTime(): string {\n    return new Date(this.model.createTime).toUTCString();\n  }\n\n  /** The timestamp of the model's most recent update. */\n  get updateTime(): string {\n    return new Date(this.model.updateTime).toUTCString();\n  }\n\n  /** Error message when model validation fails. */\n  get validationError(): string | undefined {\n    return this.model.state?.validationError?.message;\n  }\n\n  /** True if the model is published. */\n  get published(): boolean {\n    return this.model.state?.published || false;\n  }\n\n  /**\n   * The ETag identifier of the current version of the model. This value\n   * changes whenever you update any of the model's properties.\n   */\n  get etag(): string {\n    return this.model.etag;\n  }\n\n  /**\n   * The hash of the model's `tflite` file. This value changes only when\n   * you upload a new TensorFlow Lite model.\n   */\n  get modelHash(): string | undefined {\n    return this.model.modelHash;\n  }\n\n  /** Metadata about the model's TensorFlow Lite model file. */\n  get tfliteModel(): TFLiteModel | undefined {\n    // Make a copy so people can't directly modify the private this.model object.\n    return deepCopy(this.model.tfliteModel);\n  }\n\n  /**\n   * True if the model is locked by a server-side operation. You can't make\n   * changes to a locked model. See {@link Model.waitForUnlocked}.\n   */\n  public get locked(): boolean {\n    return (this.model.activeOperations?.length ?? 0) > 0;\n  }\n\n  /**\n   * Return the model as a JSON object.\n   */\n  public toJSON(): {[key: string]: any} {\n    // We can't just return this.model because it has extra fields and\n    // different formats etc. So we build the expected model object.\n    const jsonModel: {[key: string]: any}  = {\n      modelId: this.modelId,\n      displayName: this.displayName,\n      tags: this.tags,\n      createTime: this.createTime,\n      updateTime: this.updateTime,\n      published: this.published,\n      etag: this.etag,\n      locked: this.locked,\n    };\n\n    // Also add possibly undefined fields if they exist.\n\n    if (this.validationError) {\n      jsonModel['validationError'] = this.validationError;\n    }\n\n    if (this.modelHash) {\n      jsonModel['modelHash'] = this.modelHash;\n    }\n\n    if (this.tfliteModel) {\n      jsonModel['tfliteModel'] = this.tfliteModel;\n    }\n\n    return jsonModel;\n  }\n\n  /**\n   * Wait for the model to be unlocked.\n   *\n   * @param maxTimeMillis - The maximum time in milliseconds to wait.\n   *     If not specified, a default maximum of 2 minutes is used.\n   *\n   * @returns A promise that resolves when the model is unlocked\n   *   or the maximum wait time has passed.\n   */\n  public waitForUnlocked(maxTimeMillis?: number): Promise<void> {\n    if ((this.model.activeOperations?.length ?? 0) > 0) {\n      // The client will always be defined on Models that have activeOperations\n      // because models with active operations came back from the server and\n      // were constructed with a non-empty client.\n      return this.client!.handleOperation(this.model.activeOperations![0], { wait: true, maxTimeMillis })\n        .then((modelResponse) => {\n          this.model = Model.validateAndClone(modelResponse);\n        });\n    }\n    return Promise.resolve();\n  }\n\n  private static validateAndClone(model: ModelResponse): ModelResponse {\n    if (!validator.isNonNullObject(model) ||\n    !validator.isNonEmptyString(model.name) ||\n    !validator.isNonEmptyString(model.createTime) ||\n    !validator.isNonEmptyString(model.updateTime) ||\n    !validator.isNonEmptyString(model.displayName) ||\n    !validator.isNonEmptyString(model.etag)) {\n      throw new FirebaseMachineLearningError(\n        'invalid-server-response',\n        `Invalid Model response: ${JSON.stringify(model)}`);\n    }\n    const tmpModel = deepCopy(model);\n\n    // If tflite Model is specified, it must have a source of {gcsTfliteUri}\n    if (model.tfliteModel &&\n        !validator.isNonEmptyString(model.tfliteModel.gcsTfliteUri)) {\n      // If we have some other source, ignore the whole tfliteModel.\n      delete (tmpModel as any).tfliteModel;\n    }\n\n    // Remove '@type' field. We don't need it.\n    if ((tmpModel as any)['@type']) {\n      delete (tmpModel as any)['@type'];\n    }\n    return tmpModel;\n  }\n}\n\nfunction extractModelId(resourceName: string): string {\n  return resourceName.split('/').pop()!;\n}\n\n// Path: src/machine-learning/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Firebase Machine Learning.\n *\n * @packageDocumentation\n */\n\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { MachineLearning } from './machine-learning';\n\nexport {\n  MachineLearning,\n  ListModelsResult,\n  Model,\n  TFLiteModel,\n} from './machine-learning';\nexport {\n  GcsTfliteModelOptions,\n  ListModelsOptions,\n  ModelOptions,\n  ModelOptionsBase,\n} from './machine-learning-api-client';\n\n/**\n * Gets the {@link MachineLearning} service for the default app or a given app.\n *\n * `getMachineLearning()` can be called with no arguments to access the\n * default app's `MachineLearning` service or as `getMachineLearning(app)` to access\n * the `MachineLearning` service associated with a specific app.\n *\n * @example\n * ```javascript\n * // Get the MachineLearning service for the default app\n * const defaultMachineLearning = getMachineLearning();\n * ```\n *\n * @example\n * ```javascript\n * // Get the MachineLearning service for a given app\n * const otherMachineLearning = getMachineLearning(otherApp);\n * ```\n *\n * @param app - Optional app whose `MachineLearning` service to\n *   return. If not provided, the default `MachineLearning` service\n *   will be returned.\n *\n * @returns The default `MachineLearning` service if no app is provided or the\n *   `MachineLearning` service associated with the provided app.\n */\nexport function getMachineLearning(app?: App): MachineLearning {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('machineLearning', (app) => new MachineLearning(app));\n}\n\n// Path: src/messaging/batch-request-internal.ts\n/*!\n * Copyright 2019 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport {\n  HttpClient, HttpRequestConfig, HttpResponse, parseHttpResponse,\n} from '../utils/api-request';\nimport { FirebaseAppError, AppErrorCodes } from '../utils/error';\n\nconst PART_BOUNDARY = '__END_OF_PART__';\nconst TEN_SECONDS_IN_MILLIS = 15000;\n\n/**\n * Represents a request that can be sent as part of an HTTP batch request.\n */\nexport interface SubRequest {\n  url: string;\n  body: object;\n  headers?: {[key: string]: any};\n}\n\n/**\n * An HTTP client that can be used to make batch requests. This client is not tied to any service\n * (FCM or otherwise). Therefore it can be used to make batch requests to any service that allows\n * it. If this requirement ever arises we can move this implementation to the utils module\n * where it can be easily shared among other modules.\n */\nexport class BatchRequestClient {\n\n  /**\n   * @param {HttpClient} httpClient The client that will be used to make HTTP calls.\n   * @param {string} batchUrl The URL that accepts batch requests.\n   * @param {object=} commonHeaders Optional headers that will be included in all requests.\n   *\n   * @constructor\n   */\n  constructor(\n    private readonly httpClient: HttpClient,\n    private readonly batchUrl: string,\n    private readonly commonHeaders?: object) {\n  }\n\n  /**\n   * Sends the given array of sub requests as a single batch, and parses the results into an array\n   * of HttpResponse objects.\n   *\n   * @param requests - An array of sub requests to send.\n   * @returns A promise that resolves when the send operation is complete.\n   */\n  public send(requests: SubRequest[]): Promise<HttpResponse[]> {\n    requests = requests.map((req) => {\n      req.headers = Object.assign({}, this.commonHeaders, req.headers);\n      return req;\n    });\n    const requestHeaders = {\n      'Content-Type': `multipart/mixed; boundary=${PART_BOUNDARY}`,\n    };\n    const request: HttpRequestConfig = {\n      method: 'POST',\n      url: this.batchUrl,\n      data: this.getMultipartPayload(requests),\n      headers: Object.assign({}, this.commonHeaders, requestHeaders),\n      timeout: TEN_SECONDS_IN_MILLIS,\n    };\n    return this.httpClient.send(request).then((response) => {\n      if (!response.multipart) {\n        throw new FirebaseAppError(AppErrorCodes.INTERNAL_ERROR, 'Expected a multipart response.');\n      }\n      return response.multipart.map((buff) => {\n        return parseHttpResponse(buff, request);\n      });\n    });\n  }\n\n  private getMultipartPayload(requests: SubRequest[]): Buffer {\n    let buffer = '';\n    requests.forEach((request: SubRequest, idx: number) => {\n      buffer += createPart(request, PART_BOUNDARY, idx);\n    });\n    buffer += `--${PART_BOUNDARY}--\\r\\n`;\n    return Buffer.from(buffer, 'utf-8');\n  }\n}\n\n/**\n * Creates a single part in a multipart HTTP request body. The part consists of several headers\n * followed by the serialized sub request as the body. As per the requirements of the FCM batch\n * API, sets the content-type header to application/http, and the content-transfer-encoding to\n * binary.\n *\n * @param request - A sub request that will be used to populate the part.\n * @param boundary - Multipart boundary string.\n * @param idx - An index number that is used to set the content-id header.\n * @returns The part as a string that can be included in the HTTP body.\n */\n\nfunction createPart(request: SubRequest, boundary: string, idx: number): string {\n  const serializedRequest: string = serializeSubRequest(request);\n  let part = `--${boundary}\\r\\n`;\n  part += `Content-Length: ${serializedRequest.length}\\r\\n`;\n  part += 'Content-Type: application/http\\r\\n';\n  part += `content-id: ${idx + 1}\\r\\n`;\n  part += 'content-transfer-encoding: binary\\r\\n';\n  part += '\\r\\n';\n  part += `${serializedRequest}\\r\\n`;\n  return part;\n}\n\n/**\n * Serializes a sub request into a string that can be embedded in a multipart HTTP request. The\n * format of the string is the wire format of a typical HTTP request, consisting of a header and a\n * body.\n *\n * @param request - The sub request to be serialized.\n * @returns String representation of the SubRequest.\n */\nfunction serializeSubRequest(request: SubRequest): string {\n  const requestBody: string = JSON.stringify(request.body);\n  let messagePayload = `POST ${request.url} HTTP/1.1\\r\\n`;\n  messagePayload += `Content-Length: ${requestBody.length}\\r\\n`;\n  messagePayload += 'Content-Type: application/json; charset=UTF-8\\r\\n';\n  if (request.headers) {\n    Object.keys(request.headers).forEach((key) => {\n      messagePayload += `${key}: ${request.headers![key]}\\r\\n`;\n    });\n  }\n  messagePayload += '\\r\\n';\n  messagePayload += requestBody;\n  return messagePayload;\n}\n\n// Path: src/messaging/messaging-api.ts\n/*!\n * @license\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseArrayIndexError, FirebaseError } from '../app/index';\n\nexport interface BaseMessage {\n  data?: { [key: string]: string };\n  notification?: Notification;\n  android?: AndroidConfig;\n  webpush?: WebpushConfig;\n  apns?: ApnsConfig;\n  fcmOptions?: FcmOptions;\n}\n\nexport interface TokenMessage extends BaseMessage {\n  token: string;\n}\n\nexport interface TopicMessage extends BaseMessage {\n  topic: string;\n}\n\nexport interface ConditionMessage extends BaseMessage {\n  condition: string;\n}\n\n/**\n * Payload for the {@link Messaging.send} operation. The payload contains all the fields\n * in the BaseMessage type, and exactly one of token, topic or condition.\n */\nexport type Message = TokenMessage | TopicMessage | ConditionMessage;\n\n/**\n * Payload for the {@link Messaging.sendMulticast} method. The payload contains all the fields\n * in the BaseMessage type, and a list of tokens.\n */\nexport interface MulticastMessage extends BaseMessage {\n  tokens: string[];\n}\n\n/**\n   * A notification that can be included in {@link Message}.\n   */\nexport interface Notification {\n  /**\n   * The title of the notification.\n   */\n  title?: string;\n  /**\n   * The notification body\n   */\n  body?: string;\n  /**\n   * URL of an image to be displayed in the notification.\n   */\n  imageUrl?: string;\n}\n\n/**\n * Represents platform-independent options for features provided by the FCM SDKs.\n */\nexport interface FcmOptions {\n  /**\n   * The label associated with the message's analytics data.\n   */\n  analyticsLabel?: string;\n}\n\n/**\n * Represents the WebPush protocol options that can be included in an\n * {@link Message}.\n */\nexport interface WebpushConfig {\n\n  /**\n   * A collection of WebPush headers. Header values must be strings.\n   *\n   * See {@link https://tools.ietf.org/html/rfc8030#section-5 | WebPush specification}\n   * for supported headers.\n   */\n  headers?: { [key: string]: string };\n\n  /**\n   * A collection of data fields.\n   */\n  data?: { [key: string]: string };\n\n  /**\n   * A WebPush notification payload to be included in the message.\n   */\n  notification?: WebpushNotification;\n\n  /**\n   * Options for features provided by the FCM SDK for Web.\n   */\n  fcmOptions?: WebpushFcmOptions;\n}\n\n/** Represents options for features provided by the FCM SDK for Web\n * (which are not part of the Webpush standard).\n */\nexport interface WebpushFcmOptions {\n\n  /**\n   * The link to open when the user clicks on the notification.\n   * For all URL values, HTTPS is required.\n   */\n  link?: string;\n}\n\n/**\n * Represents the WebPush-specific notification options that can be included in\n * {@link WebpushConfig}. This supports most of the standard\n * options as defined in the Web Notification\n * {@link https://developer.mozilla.org/en-US/docs/Web/API/notification/Notification | specification}.\n */\nexport interface WebpushNotification {\n\n  /**\n   * Title text of the notification.\n   */\n  title?: string;\n\n  /**\n   * An array of notification actions representing the actions\n   * available to the user when the notification is presented.\n   */\n  actions?: Array<{\n\n    /**\n     * An action available to the user when the notification is presented\n     */\n    action: string;\n\n    /**\n     * Optional icon for a notification action.\n     */\n    icon?: string;\n\n    /**\n     * Title of the notification action.\n     */\n    title: string;\n  }>;\n\n  /**\n   * URL of the image used to represent the notification when there is\n   * not enough space to display the notification itself.\n   */\n  badge?: string;\n\n  /**\n   * Body text of the notification.\n   */\n  body?: string;\n\n  /**\n   * Arbitrary data that you want associated with the notification.\n   * This can be of any data type.\n   */\n  data?: any;\n\n  /**\n   * The direction in which to display the notification. Must be one\n   * of `auto`, `ltr` or `rtl`.\n   */\n  dir?: 'auto' | 'ltr' | 'rtl';\n\n  /**\n   * URL to the notification icon.\n   */\n  icon?: string;\n\n  /**\n   * URL of an image to be displayed in the notification.\n   */\n  image?: string;\n\n  /**\n   * The notification's language as a BCP 47 language tag.\n   */\n  lang?: string;\n\n  /**\n   * A boolean specifying whether the user should be notified after a\n   * new notification replaces an old one. Defaults to false.\n   */\n  renotify?: boolean;\n\n  /**\n   * Indicates that a notification should remain active until the user\n   * clicks or dismisses it, rather than closing automatically.\n   * Defaults to false.\n   */\n  requireInteraction?: boolean;\n\n  /**\n   * A boolean specifying whether the notification should be silent.\n   * Defaults to false.\n   */\n  silent?: boolean;\n\n  /**\n   * An identifying tag for the notification.\n   */\n  tag?: string;\n\n  /**\n   * Timestamp of the notification. Refer to\n   * https://developer.mozilla.org/en-US/docs/Web/API/notification/timestamp\n   * for details.\n   */\n  timestamp?: number;\n\n  /**\n   * A vibration pattern for the device's vibration hardware to emit\n   * when the notification fires.\n   */\n  vibrate?: number | number[];\n  [key: string]: any;\n}\n\n/**\n * Represents the APNs-specific options that can be included in an\n * {@link Message}. Refer to\n * {@link https://developer.apple.com/library/content/documentation/NetworkingInternet/Conceptual/RemoteNotificationsPG/CommunicatingwithAPNs.html |\n * Apple documentation} for various headers and payload fields supported by APNs.\n */\nexport interface ApnsConfig {\n  /**\n   * A collection of APNs headers. Header values must be strings.\n   */\n  headers?: { [key: string]: string };\n\n  /**\n   * An APNs payload to be included in the message.\n   */\n  payload?: ApnsPayload;\n\n  /**\n   * Options for features provided by the FCM SDK for iOS.\n   */\n  fcmOptions?: ApnsFcmOptions;\n}\n\n/**\n * Represents the payload of an APNs message. Mainly consists of the `aps`\n * dictionary. But may also contain other arbitrary custom keys.\n */\nexport interface ApnsPayload {\n\n  /**\n   * The `aps` dictionary to be included in the message.\n   */\n  aps: Aps;\n  [customData: string]: any;\n}\n\n/**\n * Represents the {@link https://developer.apple.com/library/content/documentation/NetworkingInternet/Conceptual/RemoteNotificationsPG/PayloadKeyReference.html |\n * aps dictionary} that is part of APNs messages.\n */\nexport interface Aps {\n\n  /**\n   * Alert to be included in the message. This may be a string or an object of\n   * type `admin.messaging.ApsAlert`.\n   */\n  alert?: string | ApsAlert;\n\n  /**\n   * Badge to be displayed with the message. Set to 0 to remove the badge. When\n   * not specified, the badge will remain unchanged.\n   */\n  badge?: number;\n\n  /**\n   * Sound to be played with the message.\n   */\n  sound?: string | CriticalSound;\n\n  /**\n   * Specifies whether to configure a background update notification.\n   */\n  contentAvailable?: boolean;\n\n  /**\n   * Specifies whether to set the `mutable-content` property on the message\n   * so the clients can modify the notification via app extensions.\n   */\n  mutableContent?: boolean;\n\n  /**\n   * Type of the notification.\n   */\n  category?: string;\n\n  /**\n   * An app-specific identifier for grouping notifications.\n   */\n  threadId?: string;\n  [customData: string]: any;\n}\n\nexport interface ApsAlert {\n  title?: string;\n  subtitle?: string;\n  body?: string;\n  locKey?: string;\n  locArgs?: string[];\n  titleLocKey?: string;\n  titleLocArgs?: string[];\n  subtitleLocKey?: string;\n  subtitleLocArgs?: string[];\n  actionLocKey?: string;\n  launchImage?: string;\n}\n\n/**\n * Represents a critical sound configuration that can be included in the\n * `aps` dictionary of an APNs payload.\n */\nexport interface CriticalSound {\n\n  /**\n   * The critical alert flag. Set to `true` to enable the critical alert.\n   */\n  critical?: boolean;\n\n  /**\n   * The name of a sound file in the app's main bundle or in the `Library/Sounds`\n   * folder of the app's container directory. Specify the string \"default\" to play\n   * the system sound.\n   */\n  name: string;\n\n  /**\n   * The volume for the critical alert's sound. Must be a value between 0.0\n   * (silent) and 1.0 (full volume).\n   */\n  volume?: number;\n}\n\n/**\n * Represents options for features provided by the FCM SDK for iOS.\n */\nexport interface ApnsFcmOptions {\n\n  /**\n   * The label associated with the message's analytics data.\n   */\n  analyticsLabel?: string;\n\n  /**\n   * URL of an image to be displayed in the notification.\n   */\n  imageUrl?: string;\n}\n\n\n/**\n * Represents the Android-specific options that can be included in an\n * {@link Message}.\n */\nexport interface AndroidConfig {\n\n  /**\n   * Collapse key for the message. Collapse key serves as an identifier for a\n   * group of messages that can be collapsed, so that only the last message gets\n   * sent when delivery can be resumed. A maximum of four different collapse keys\n   * may be active at any given time.\n   */\n  collapseKey?: string;\n\n  /**\n   * Priority of the message. Must be either `normal` or `high`.\n   */\n  priority?: ('high' | 'normal');\n\n  /**\n   * Time-to-live duration of the message in milliseconds.\n   */\n  ttl?: number;\n\n  /**\n   * Package name of the application where the registration tokens must match\n   * in order to receive the message.\n   */\n  restrictedPackageName?: string;\n\n  /**\n   * A collection of data fields to be included in the message. All values must\n   * be strings. When provided, overrides any data fields set on the top-level\n   * {@link Message}.\n   */\n  data?: { [key: string]: string };\n\n  /**\n   * Android notification to be included in the message.\n   */\n  notification?: AndroidNotification;\n\n  /**\n   * Options for features provided by the FCM SDK for Android.\n   */\n  fcmOptions?: AndroidFcmOptions;\n}\n\n/**\n  * Represents the Android-specific notification options that can be included in\n  * {@link AndroidConfig}.\n  */\nexport interface AndroidNotification {\n  /**\n   * Title of the Android notification. When provided, overrides the title set via\n   * `admin.messaging.Notification`.\n   */\n  title?: string;\n\n  /**\n   * Body of the Android notification. When provided, overrides the body set via\n   * `admin.messaging.Notification`.\n   */\n  body?: string;\n\n  /**\n   * Icon resource for the Android notification.\n   */\n  icon?: string;\n\n  /**\n   * Notification icon color in `#rrggbb` format.\n   */\n  color?: string;\n\n  /**\n   * File name of the sound to be played when the device receives the\n   * notification.\n   */\n  sound?: string;\n\n  /**\n   * Notification tag. This is an identifier used to replace existing\n   * notifications in the notification drawer. If not specified, each request\n   * creates a new notification.\n   */\n  tag?: string;\n\n  /**\n   * URL of an image to be displayed in the notification.\n   */\n  imageUrl?: string;\n\n  /**\n   * Action associated with a user click on the notification. If specified, an\n   * activity with a matching Intent Filter is launched when a user clicks on the\n   * notification.\n   */\n  clickAction?: string;\n\n  /**\n   * Key of the body string in the app's string resource to use to localize the\n   * body text.\n   *\n   */\n  bodyLocKey?: string;\n\n  /**\n   * An array of resource keys that will be used in place of the format\n   * specifiers in `bodyLocKey`.\n   */\n  bodyLocArgs?: string[];\n\n  /**\n   * Key of the title string in the app's string resource to use to localize the\n   * title text.\n   */\n  titleLocKey?: string;\n\n  /**\n   * An array of resource keys that will be used in place of the format\n   * specifiers in `titleLocKey`.\n   */\n  titleLocArgs?: string[];\n\n  /**\n   * The Android notification channel ID (new in Android O). The app must create\n   * a channel with this channel ID before any notification with this channel ID\n   * can be received. If you don't send this channel ID in the request, or if the\n   * channel ID provided has not yet been created by the app, FCM uses the channel\n   * ID specified in the app manifest.\n   */\n  channelId?: string;\n\n  /**\n   * Sets the \"ticker\" text, which is sent to accessibility services. Prior to\n...\n", "needle_token_start": 11894, "needle_token_end": 12046, "code_context_ntokens": 15905, "output": ["```typescript\nfunction createPart(request: SubRequest, boundary: string, idx: number): string {\n  const serializedRequest: string = serializeSubRequest(request);\n  let part = `--${boundary}\\r\\n`;\n  part += `Content-Length: ${serializedRequest.length}\\r\\n`;\n  part += 'Content-Type: application/http\\r\\n';\n  part += `content-id: ${idx + 1}\\r\\n`;\n  part += 'content-transfer-encoding: binary\\r\\n';\n  part += '\\r\\n';\n  part += `${serializedRequest}\\r\\n`;\n  return part;\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "assertServerResponse", "language": "typescript", "path": "src/project-management/project-management-api-request-internal.ts", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate server responses by checking if a specified condition is true. If the condition is false, it throws an error with a detailed message including the server's response data.\n2. **Input**: It takes three parameters: a boolean condition, an object containing the server's response data, and a string message to include in the error if the condition is false.\n3. **Output**: There is no return value; however, it may throw an error if the condition is not met.\n4. **Procedure**: The function checks the provided boolean condition. If the condition is false, it constructs an error message that includes the provided string message and a formatted string of the response data. It then throws an error with this constructed message.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/auth/base-auth.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App, FirebaseArrayIndexError } from '../app';\nimport { AuthClientErrorCode, ErrorInfo, FirebaseAuthError } from '../utils/error';\nimport { deepCopy } from '../utils/deep-copy';\nimport * as validator from '../utils/validator';\n\nimport { AbstractAuthRequestHandler, useEmulator } from './auth-api-request';\nimport { FirebaseTokenGenerator, EmulatedSigner, handleCryptoSignerError } from './token-generator';\nimport {\n  FirebaseTokenVerifier,\n  createSessionCookieVerifier,\n  createIdTokenVerifier,\n  createAuthBlockingTokenVerifier,\n  DecodedIdToken,\n  DecodedAuthBlockingToken,\n} from './token-verifier';\nimport {\n  AuthProviderConfig, SAMLAuthProviderConfig, AuthProviderConfigFilter, ListProviderConfigResults,\n  SAMLConfig, OIDCConfig, OIDCConfigServerResponse, SAMLConfigServerResponse,\n  UpdateAuthProviderRequest, OIDCAuthProviderConfig, CreateRequest, UpdateRequest,\n} from './auth-config';\nimport { UserRecord } from './user-record';\nimport {\n  UserIdentifier, isUidIdentifier, isEmailIdentifier, isPhoneIdentifier, isProviderIdentifier,\n} from './identifier';\nimport { UserImportOptions, UserImportRecord, UserImportResult } from './user-import-builder';\nimport { ActionCodeSettings } from './action-code-settings-builder';\nimport { cryptoSignerFromApp } from '../utils/crypto-signer';\n\n/** Represents the result of the {@link BaseAuth.getUsers} API. */\nexport interface GetUsersResult {\n  /**\n   * Set of user records, corresponding to the set of users that were\n   * requested. Only users that were found are listed here. The result set is\n   * unordered.\n   */\n  users: UserRecord[];\n\n  /** Set of identifiers that were requested, but not found. */\n  notFound: UserIdentifier[];\n}\n\n/**\n * Interface representing the object returned from a\n * {@link BaseAuth.listUsers} operation. Contains the list\n * of users for the current batch and the next page token if available.\n */\nexport interface ListUsersResult {\n\n  /**\n...\n// Path: src/auth/tenant-manager.ts\n/*!\n * Copyright 2019 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport * as validator from '../utils/validator';\nimport { App } from '../app';\nimport * as utils from '../utils/index';\nimport { AuthClientErrorCode, FirebaseAuthError } from '../utils/error';\n\nimport { BaseAuth, createFirebaseTokenGenerator, SessionCookieOptions } from './base-auth';\nimport { Tenant, TenantServerResponse, CreateTenantRequest, UpdateTenantRequest } from './tenant';\nimport {\n  AuthRequestHandler, TenantAwareAuthRequestHandler,\n} from './auth-api-request';\nimport { DecodedIdToken } from './token-verifier';\n\n/**\n * Interface representing the object returned from a\n * {@link TenantManager.listTenants}\n * operation.\n * Contains the list of tenants for the current batch and the next page token if available.\n */\nexport interface ListTenantsResult {\n\n  /**\n   * The list of {@link Tenant} objects for the downloaded batch.\n   */\n  tenants: Tenant[];\n\n  /**\n   * The next page token if available. This is needed for the next batch download.\n   */\n  pageToken?: string;\n}\n\n/**\n * Tenant-aware `Auth` interface used for managing users, configuring SAML/OIDC providers,\n * generating email links for password reset, email verification, etc for specific tenants.\n *\n * Multi-tenancy support requires Google Cloud's Identity Platform\n * (GCIP). To learn more about GCIP, including pricing and features,\n * see the {@link https://cloud.google.com/identity-platform | GCIP documentation}.\n *\n * Each tenant contains its own identity providers, settings and sets of users.\n * Using `TenantAwareAuth`, users for a specific tenant and corresponding OIDC/SAML\n * configurations can also be managed, ID tokens for users signed in to a specific tenant\n * can be verified, and email action links can also be generated for users belonging to the\n * tenant.\n *\n * `TenantAwareAuth` instances for a specific `tenantId` can be instantiated by calling\n * {@link TenantManager.authForTenant}.\n */\nexport class TenantAwareAuth extends BaseAuth {\n\n  /**\n   * The tenant identifier corresponding to this `TenantAwareAuth` instance.\n   * All calls to the user management APIs, OIDC/SAML provider management APIs, email link\n   * generation APIs, etc will only be applied within the scope of this tenant.\n   */\n  public readonly tenantId: string;\n\n  /**\n   * The TenantAwareAuth class constructor.\n   *\n   * @param app - The app that created this tenant.\n   * @param tenantId - The corresponding tenant ID.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App, tenantId: string) {\n    super(app, new TenantAwareAuthRequestHandler(\n      app, tenantId), createFirebaseTokenGenerator(app, tenantId));\n    utils.addReadonlyGetter(this, 'tenantId', tenantId);\n  }\n\n  /**\n   * {@inheritdoc BaseAuth.verifyIdToken}\n   */\n  public verifyIdToken(idToken: string, checkRevoked = false): Promise<DecodedIdToken> {\n    return super.verifyIdToken(idToken, checkRevoked)\n      .then((decodedClaims) => {\n        // Validate tenant ID.\n        if (decodedClaims.firebase.tenant !== this.tenantId) {\n          throw new FirebaseAuthError(AuthClientErrorCode.MISMATCHING_TENANT_ID);\n        }\n        return decodedClaims;\n      });\n  }\n\n  /**\n   * {@inheritdoc BaseAuth.createSessionCookie}\n   */\n  public createSessionCookie(\n    idToken: string, sessionCookieOptions: SessionCookieOptions): Promise<string> {\n    // Validate arguments before processing.\n    if (!validator.isNonEmptyString(idToken)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_ID_TOKEN));\n    }\n    if (!validator.isNonNullObject(sessionCookieOptions) ||\n        !validator.isNumber(sessionCookieOptions.expiresIn)) {\n      return Promise.reject(new FirebaseAuthError(AuthClientErrorCode.INVALID_SESSION_COOKIE_DURATION));\n    }\n    // This will verify the ID token and then match the tenant ID before creating the session cookie.\n    return this.verifyIdToken(idToken)\n      .then(() => {\n        return super.createSessionCookie(idToken, sessionCookieOptions);\n      });\n  }\n\n  /**\n   * {@inheritdoc BaseAuth.verifySessionCookie}\n   */\n  public verifySessionCookie(\n    sessionCookie: string, checkRevoked = false): Promise<DecodedIdToken> {\n    return super.verifySessionCookie(sessionCookie, checkRevoked)\n      .then((decodedClaims) => {\n        if (decodedClaims.firebase.tenant !== this.tenantId) {\n          throw new FirebaseAuthError(AuthClientErrorCode.MISMATCHING_TENANT_ID);\n        }\n        return decodedClaims;\n      });\n  }\n}\n\n/**\n * Defines the tenant manager used to help manage tenant related operations.\n * This includes:\n * <ul>\n * <li>The ability to create, update, list, get and delete tenants for the underlying\n *     project.</li>\n * <li>Getting a `TenantAwareAuth` instance for running Auth related operations\n *     (user management, provider configuration management, token verification,\n *     email link generation, etc) in the context of a specified tenant.</li>\n * </ul>\n */\nexport class TenantManager {\n  private readonly authRequestHandler: AuthRequestHandler;\n  private readonly tenantsMap: {[key: string]: TenantAwareAuth};\n\n  /**\n   * Initializes a TenantManager instance for a specified FirebaseApp.\n   *\n   * @param app - The app for this TenantManager instance.\n   *\n   * @constructor\n   * @internal\n   */\n  constructor(private readonly app: App) {\n    this.authRequestHandler = new AuthRequestHandler(app);\n    this.tenantsMap = {};\n  }\n\n  /**\n   * Returns a `TenantAwareAuth` instance bound to the given tenant ID.\n   *\n   * @param tenantId - The tenant ID whose `TenantAwareAuth` instance is to be returned.\n   *\n   * @returns The `TenantAwareAuth` instance corresponding to this tenant identifier.\n   */\n  public authForTenant(tenantId: string): TenantAwareAuth {\n    if (!validator.isNonEmptyString(tenantId)) {\n      throw new FirebaseAuthError(AuthClientErrorCode.INVALID_TENANT_ID);\n    }\n    if (typeof this.tenantsMap[tenantId] === 'undefined') {\n      this.tenantsMap[tenantId] = new TenantAwareAuth(this.app, tenantId);\n    }\n    return this.tenantsMap[tenantId];\n  }\n\n  /**\n   * Gets the tenant configuration for the tenant corresponding to a given `tenantId`.\n   *\n   * @param tenantId - The tenant identifier corresponding to the tenant whose data to fetch.\n   *\n   * @returns A promise fulfilled with the tenant configuration to the provided `tenantId`.\n   */\n  public getTenant(tenantId: string): Promise<Tenant> {\n    return this.authRequestHandler.getTenant(tenantId)\n      .then((response: TenantServerResponse) => {\n        return new Tenant(response);\n      });\n  }\n\n  /**\n   * Retrieves a list of tenants (single batch only) with a size of `maxResults`\n   * starting from the offset as specified by `pageToken`. This is used to\n   * retrieve all the tenants of a specified project in batches.\n   *\n   * @param maxResults - The page size, 1000 if undefined. This is also\n   *   the maximum allowed limit.\n   * @param pageToken - The next page token. If not specified, returns\n   *   tenants starting without any offset.\n   *\n   * @returns A promise that resolves with\n   *   a batch of downloaded tenants and the next page token.\n   */\n  public listTenants(\n    maxResults?: number,\n    pageToken?: string): Promise<ListTenantsResult> {\n    return this.authRequestHandler.listTenants(maxResults, pageToken)\n      .then((response: {tenants: TenantServerResponse[]; nextPageToken?: string}) => {\n        // List of tenants to return.\n        const tenants: Tenant[] = [];\n        // Convert each user response to a Tenant.\n        response.tenants.forEach((tenantResponse: TenantServerResponse) => {\n          tenants.push(new Tenant(tenantResponse));\n        });\n        // Return list of tenants and the next page token if available.\n        const result = {\n          tenants,\n          pageToken: response.nextPageToken,\n        };\n        // Delete result.pageToken if undefined.\n        if (typeof result.pageToken === 'undefined') {\n          delete result.pageToken;\n        }\n        return result;\n      });\n  }\n\n  /**\n   * Deletes an existing tenant.\n   *\n   * @param tenantId - The `tenantId` corresponding to the tenant to delete.\n   *\n   * @returns An empty promise fulfilled once the tenant has been deleted.\n   */\n  public deleteTenant(tenantId: string): Promise<void> {\n    return this.authRequestHandler.deleteTenant(tenantId);\n  }\n\n  /**\n   * Creates a new tenant.\n   * When creating new tenants, tenants that use separate billing and quota will require their\n   * own project and must be defined as `full_service`.\n   *\n   * @param tenantOptions - The properties to set on the new tenant configuration to be created.\n   *\n   * @returns A promise fulfilled with the tenant configuration corresponding to the newly\n   *   created tenant.\n   */\n  public createTenant(tenantOptions: CreateTenantRequest): Promise<Tenant> {\n    return this.authRequestHandler.createTenant(tenantOptions)\n      .then((response: TenantServerResponse) => {\n        return new Tenant(response);\n      });\n  }\n\n  /**\n   * Updates an existing tenant configuration.\n   *\n   * @param tenantId - The `tenantId` corresponding to the tenant to delete.\n   * @param tenantOptions - The properties to update on the provided tenant.\n   *\n   * @returns A promise fulfilled with the update tenant data.\n   */\n  public updateTenant(tenantId: string, tenantOptions: UpdateTenantRequest): Promise<Tenant> {\n    return this.authRequestHandler.updateTenant(tenantId, tenantOptions)\n      .then((response: TenantServerResponse) => {\n        return new Tenant(response);\n      });\n  }\n}\n\n// Path: src/auth/auth.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app/index';\nimport { AuthRequestHandler } from './auth-api-request';\nimport { TenantManager } from './tenant-manager';\nimport { BaseAuth } from './base-auth';\nimport { ProjectConfigManager } from './project-config-manager';\n\n/**\n * Auth service bound to the provided app.\n * An Auth instance can have multiple tenants.\n */\nexport class Auth extends BaseAuth {\n\n  private readonly tenantManager_: TenantManager;\n  private readonly projectConfigManager_: ProjectConfigManager;\n  private readonly app_: App;\n\n  /**\n   * @param app - The app for this Auth service.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    super(app, new AuthRequestHandler(app));\n    this.app_ = app;\n    this.tenantManager_ = new TenantManager(app);\n    this.projectConfigManager_ = new ProjectConfigManager(app);\n  }\n\n  /**\n   * Returns the app associated with this Auth instance.\n   *\n   * @returns The app associated with this Auth instance.\n   */\n  get app(): App {\n    return this.app_;\n  }\n\n  /**\n   * Returns the tenant manager instance associated with the current project.\n   *\n   * @returns The tenant manager instance associated with the current project.\n   */\n  public tenantManager(): TenantManager {\n    return this.tenantManager_;\n  }\n\n  /**\n   * Returns the project config manager instance associated with the current project.\n   *\n   * @returns The project config manager instance associated with the current project.\n   */\n  public projectConfigManager(): ProjectConfigManager {\n    return this.projectConfigManager_;\n  }\n}\n\n// Path: src/firestore/firestore-internal.ts\n/*!\n * @license\n * Copyright 2017 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseFirestoreError } from '../utils/error';\nimport { ServiceAccountCredential, isApplicationDefault } from '../app/credential-internal';\nimport { Firestore, Settings } from '@google-cloud/firestore';\n\nimport * as validator from '../utils/validator';\nimport * as utils from '../utils/index';\nimport { App } from '../app';\n\n/**\n * Settings to pass to the Firestore constructor.\n * \n * @public\n */\nexport interface FirestoreSettings {\n  /**\n   * Use HTTP/1.1 REST transport where possible.\n   * \n   * `preferRest` will force the use of HTTP/1.1 REST transport until a method\n   * that requires gRPC is called. When a method requires gRPC, this Firestore\n   * client will load dependent gRPC libraries and then use gRPC transport for\n   * all communication from that point forward. Currently the only operation\n   * that requires gRPC is creating a snapshot listener using `onSnapshot()`.\n   * \n   * @defaultValue `undefined`\n   */\n  preferRest?: boolean;\n}\n\nexport class FirestoreService {\n\n  private readonly appInternal: App;\n  private readonly databases: Map<string, Firestore> = new Map();\n  private readonly firestoreSettings: Map<string, FirestoreSettings> = new Map();\n\n  constructor(app: App) {\n    this.appInternal = app;\n  }\n\n  initializeDatabase(databaseId: string, settings: FirestoreSettings): Firestore {\n    const existingInstance = this.databases.get(databaseId);\n    if (existingInstance) {\n      const initialSettings = this.firestoreSettings.get(databaseId) ?? {};\n      if (this.checkIfSameSettings(settings, initialSettings)) {\n        return existingInstance;\n      }\n      throw new FirebaseFirestoreError({\n        code: 'failed-precondition',\n        message: 'initializeFirestore() has already been called with ' +\n          'different options. To avoid this error, call initializeFirestore() with the ' +\n          'same options as when it was originally called, or call getFirestore() to return the' +\n          ' already initialized instance.'\n      });\n    }\n    const newInstance = initFirestore(this.app, databaseId, settings);\n    this.databases.set(databaseId, newInstance);\n    this.firestoreSettings.set(databaseId, settings);\n    return newInstance;\n  }\n\n  getDatabase(databaseId: string): Firestore {\n    let database = this.databases.get(databaseId);\n    if (database === undefined) {\n      database = initFirestore(this.app, databaseId, {});\n      this.databases.set(databaseId, database);\n      this.firestoreSettings.set(databaseId, {});\n    }\n    return database;\n  }\n\n  private checkIfSameSettings(settingsA: FirestoreSettings, settingsB: FirestoreSettings): boolean {\n    const a = settingsA ?? {};\n    const b = settingsB ?? {};\n    // If we start passing more settings to Firestore constructor,\n    // replace this with deep equality check.\n    return (a.preferRest === b.preferRest);\n  }\n\n  /**\n   * Returns the app associated with this Storage instance.\n   *\n   * @returns The app associated with this Storage instance.\n   */\n  get app(): App {\n    return this.appInternal;\n  }\n}\n\nexport function getFirestoreOptions(app: App, firestoreSettings?: FirestoreSettings): Settings {\n  if (!validator.isNonNullObject(app) || !('options' in app)) {\n    throw new FirebaseFirestoreError({\n      code: 'invalid-argument',\n      message: 'First argument passed to admin.firestore() must be a valid Firebase app instance.',\n    });\n  }\n\n  const projectId: string | null = utils.getExplicitProjectId(app);\n  const credential = app.options.credential;\n  // eslint-disable-next-line @typescript-eslint/no-var-requires\n  const { version: firebaseVersion } = require('../../package.json');\n  const preferRest = firestoreSettings?.preferRest;\n  if (credential instanceof ServiceAccountCredential) {\n    return {\n      credentials: {\n        private_key: credential.privateKey,\n        client_email: credential.clientEmail,\n      },\n      // When the SDK is initialized with ServiceAccountCredentials an explicit projectId is\n      // guaranteed to be available.\n      projectId: projectId!,\n      firebaseVersion,\n      preferRest,\n    };\n  } else if (isApplicationDefault(app.options.credential)) {\n    // Try to use the Google application default credentials.\n    // If an explicit project ID is not available, let Firestore client discover one from the\n    // environment. This prevents the users from having to set GOOGLE_CLOUD_PROJECT in GCP runtimes.\n    return validator.isNonEmptyString(projectId)\n      ? { projectId, firebaseVersion, preferRest }\n      : { firebaseVersion, preferRest };\n  }\n\n  throw new FirebaseFirestoreError({\n    code: 'invalid-credential',\n    message: 'Failed to initialize Google Cloud Firestore client with the available credentials. ' +\n      'Must initialize the SDK with a certificate credential or application default credentials ' +\n      'to use Cloud Firestore API.',\n  });\n}\n\nfunction initFirestore(app: App, databaseId: string, firestoreSettings?: FirestoreSettings): Firestore {\n  const options = getFirestoreOptions(app, firestoreSettings);\n  options.databaseId = databaseId;\n  let firestoreDatabase: typeof Firestore;\n  try {\n    // Lazy-load the Firestore implementation here, which in turns loads gRPC.\n    firestoreDatabase = require('@google-cloud/firestore').Firestore;\n  } catch (err) {\n    throw new FirebaseFirestoreError({\n      code: 'missing-dependencies',\n      message: 'Failed to import the Cloud Firestore client library for Node.js. '\n          + 'Make sure to install the \"@google-cloud/firestore\" npm package. '\n          + `Original error: ${err}`,\n    });\n  }\n\n  return new firestoreDatabase(options);\n}\n\n// Path: src/firestore/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Cloud Firestore.\n *\n * @packageDocumentation\n */\n\nimport { Firestore } from '@google-cloud/firestore';\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { FirestoreService, FirestoreSettings } from './firestore-internal';\nimport { DEFAULT_DATABASE_ID } from '@google-cloud/firestore/build/src/path';\n\nexport {\n  AddPrefixToKeys,\n  AggregateField,\n  AggregateFieldType,\n  AggregateQuery,\n  AggregateQuerySnapshot,\n  AggregateSpecData,\n  AggregateSpec,\n  AggregateType,\n  BulkWriter,\n  BulkWriterOptions,\n  BundleBuilder,\n  ChildUpdateFields,\n  CollectionGroup,\n  CollectionReference,\n  DocumentChange,\n  DocumentChangeType,\n  DocumentData,\n  DocumentReference,\n  DocumentSnapshot,\n  FieldPath,\n  FieldValue,\n  Filter,\n  Firestore,\n  FirestoreDataConverter,\n  GeoPoint,\n  GrpcStatus,\n  NestedUpdateFields,\n  OrderByDirection,\n  PartialWithFieldValue,\n  Precondition,\n  Primitive,\n  Query,\n  QueryDocumentSnapshot,\n  QueryPartition,\n  QuerySnapshot,\n  ReadOptions,\n  ReadOnlyTransactionOptions,\n  ReadWriteTransactionOptions,\n  Settings,\n  SetOptions,\n  Timestamp,\n  Transaction,\n  UpdateData,\n  UnionToIntersection,\n  WhereFilterOp,\n  WithFieldValue,\n  WriteBatch,\n  WriteResult,\n  v1,\n  setLogFunction,\n} from '@google-cloud/firestore';\n\nexport { FirestoreSettings };\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n *\n * @example\n * ```javascript\n * // Get the default Firestore service for the default app\n * const defaultFirestore = getFirestore();\n * ```\n\n * @returns The default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n */\nexport function getFirestore(): Firestore;\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app.\n *\n * @example\n * ```javascript\n * // Get the default Firestore service for a specific app\n * const otherFirestore = getFirestore(app);\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @returns The default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service associated with the provided app.\n */\nexport function getFirestore(app: App): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a named database and default app\n * const otherFirestore = getFirestore('otherDb');\n * ```\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the default app.\n * @beta\n */\nexport function getFirestore(databaseId: string): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a named database and specific app.\n * const otherFirestore = getFirestore('otherDb');\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service associated with the provided app.\n * @beta\n */\nexport function getFirestore(app: App, databaseId: string): Firestore;\n\nexport function getFirestore(\n  appOrDatabaseId?: App | string,\n  optionalDatabaseId?: string\n): Firestore {\n  const app: App = typeof appOrDatabaseId === 'object' ? appOrDatabaseId : getApp();\n  const databaseId =\n    (typeof appOrDatabaseId === 'string' ? appOrDatabaseId : optionalDatabaseId) || DEFAULT_DATABASE_ID;\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  const firestoreService = firebaseApp.getOrInitService(\n    'firestore', (app) => new FirestoreService(app));\n  return firestoreService.getDatabase(databaseId);\n}\n\n/**\n * Gets the default {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app, passing extra parameters to its constructor.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a specific app, require HTTP/1.1 REST transport\n * const otherFirestore = initializeFirestore(app, {preferRest: true});\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param settings - Settings object to be passed to the constructor.\n *\n * @returns The default `Firestore` service associated with the provided app and settings.\n */\nexport function initializeFirestore(app: App, settings?: FirestoreSettings): Firestore;\n\n/**\n * Gets the named {@link https://googleapis.dev/nodejs/firestore/latest/Firestore.html | Firestore}\n * service for the given app, passing extra parameters to its constructor.\n *\n * @example\n * ```javascript\n * // Get the Firestore service for a specific app, require HTTP/1.1 REST transport\n * const otherFirestore = initializeFirestore(app, {preferRest: true}, 'otherDb');\n * ```\n *\n * @param app - which `Firestore` service to return.\n *\n * @param settings - Settings object to be passed to the constructor.\n *\n * @param databaseId - name of database to return.\n *\n * @returns The named `Firestore` service associated with the provided app and settings.\n * @beta\n */\nexport function initializeFirestore(\n  app: App,\n  settings: FirestoreSettings,\n  databaseId: string\n): Firestore;\n\nexport function initializeFirestore(\n  app: App,\n  settings?: FirestoreSettings,\n  databaseId?: string\n): Firestore {\n  settings ??= {};\n  databaseId ??= DEFAULT_DATABASE_ID;\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  const firestoreService = firebaseApp.getOrInitService(\n    'firestore', (app) => new FirestoreService(app));\n\n  return firestoreService.initializeDatabase(databaseId, settings);\n}\n\n// Path: src/auth/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Firebase Authentication.\n *\n * @packageDocumentation\n */\n\nimport { App, getApp } from '../app/index';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { Auth } from './auth';\n\n/**\n * Gets the {@link Auth} service for the default app or a\n * given app.\n *\n * `getAuth()` can be called with no arguments to access the default app's\n * {@link Auth} service or as `getAuth(app)` to access the\n * {@link Auth} service associated with a specific app.\n *\n * @example\n * ```javascript\n * // Get the Auth service for the default app\n * const defaultAuth = getAuth();\n * ```\n *\n * @example\n * ```javascript\n * // Get the Auth service for a given app\n * const otherAuth = getAuth(otherApp);\n * ```\n *\n */\nexport function getAuth(app?: App): Auth {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('auth', (app) => new Auth(app));\n}\n\nexport { ActionCodeSettings } from './action-code-settings-builder';\n\nexport {\n  Auth,\n} from './auth';\n\nexport {\n  AllowByDefault,\n  AllowByDefaultWrap,\n  AllowlistOnly,\n  AllowlistOnlyWrap,\n  AuthFactorType,\n  AuthProviderConfig,\n  AuthProviderConfigFilter,\n  BaseAuthProviderConfig,\n  BaseCreateMultiFactorInfoRequest,\n  BaseUpdateMultiFactorInfoRequest,\n  CreateMultiFactorInfoRequest,\n  CreatePhoneMultiFactorInfoRequest,\n  CreateRequest,\n  EmailSignInProviderConfig,\n  ListProviderConfigResults,\n  MultiFactorConfig,\n  MultiFactorConfigState,\n  MultiFactorCreateSettings,\n  MultiFactorUpdateSettings,\n  MultiFactorProviderConfig,\n  OAuthResponseType,\n  OIDCAuthProviderConfig,\n  OIDCUpdateAuthProviderRequest,\n  RecaptchaAction,\n  RecaptchaConfig,\n  RecaptchaKey,\n  RecaptchaKeyClientType,\n  RecaptchaManagedRule,\n  RecaptchaProviderEnforcementState,\n  SAMLAuthProviderConfig,\n  SAMLUpdateAuthProviderRequest,\n  SmsRegionConfig,\n  UserProvider,\n  UpdateAuthProviderRequest,\n  UpdateMultiFactorInfoRequest,\n  UpdatePhoneMultiFactorInfoRequest,\n  UpdateRequest,\n  TotpMultiFactorProviderConfig,\n  PasswordPolicyConfig,\n  PasswordPolicyEnforcementState,\n  CustomStrengthOptionsConfig,\n  EmailPrivacyConfig,\n} from './auth-config';\n\nexport {\n  BaseAuth,\n  DeleteUsersResult,\n  GetUsersResult,\n  ListUsersResult,\n  SessionCookieOptions,\n} from './base-auth';\n\nexport {\n  EmailIdentifier,\n  PhoneIdentifier,\n  ProviderIdentifier,\n  UidIdentifier,\n  UserIdentifier,\n} from './identifier';\n\nexport {\n  CreateTenantRequest,\n  Tenant,\n  UpdateTenantRequest,\n} from './tenant';\n\nexport {\n  ListTenantsResult,\n  TenantAwareAuth,\n  TenantManager,\n} from './tenant-manager';\n\nexport {\n  UpdateProjectConfigRequest,\n  ProjectConfig,\n} from './project-config';\n\nexport {\n  ProjectConfigManager,\n} from './project-config-manager';\n\nexport {\n  DecodedIdToken,\n  DecodedAuthBlockingToken\n} from './token-verifier';\n\nexport {\n  HashAlgorithmType,\n  UserImportOptions,\n  UserImportRecord,\n  UserImportResult,\n  UserMetadataRequest,\n  UserProviderRequest,\n} from './user-import-builder';\n\nexport {\n  MultiFactorInfo,\n  MultiFactorSettings,\n  PhoneMultiFactorInfo,\n  UserInfo,\n  UserMetadata,\n  UserRecord,\n} from './user-record';\n\n// Path: src/instance-id/instance-id.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { getInstallations } from '../installations';\nimport { App } from '../app/index';\nimport {\n  FirebaseInstallationsError, FirebaseInstanceIdError,\n  InstallationsClientErrorCode, InstanceIdClientErrorCode,\n} from '../utils/error';\nimport * as validator from '../utils/validator';\n\n/**\n * The `InstanceId` service enables deleting the Firebase instance IDs\n * associated with Firebase client app instances.\n *\n * @deprecated Use {@link firebase-admin.installations#Installations} instead.\n */\nexport class InstanceId {\n\n  private app_: App;\n\n  /**\n   * @param app - The app for this InstanceId service.\n   * @constructor\n   * @internal\n   */\n  constructor(app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseInstanceIdError(\n        InstanceIdClientErrorCode.INVALID_ARGUMENT,\n        'First argument passed to instanceId() must be a valid Firebase app instance.',\n      );\n    }\n\n    this.app_ = app;\n  }\n\n  /**\n   * Deletes the specified instance ID and the associated data from Firebase.\n   *\n   * Note that Google Analytics for Firebase uses its own form of Instance ID to\n   * keep track of analytics data. Therefore deleting a Firebase Instance ID does\n   * not delete Analytics data. See\n   * {@link https://firebase.google.com/support/privacy/manage-iids#delete_an_instance_id |\n   * Delete an Instance ID}\n   * for more information.\n   *\n   * @param instanceId - The instance ID to be deleted.\n   *\n   * @returns A promise fulfilled when the instance ID is deleted.\n   */\n  public deleteInstanceId(instanceId: string): Promise<void> {\n    return getInstallations(this.app).deleteInstallation(instanceId)\n      .catch((err) => {\n        if (err instanceof FirebaseInstallationsError) {\n          let code = err.code.replace('installations/', '');\n          if (code === InstallationsClientErrorCode.INVALID_INSTALLATION_ID.code) {\n            code = InstanceIdClientErrorCode.INVALID_INSTANCE_ID.code;\n          }\n\n          throw new FirebaseInstanceIdError({ code, message: err.message });\n        }\n\n        throw err;\n      });\n  }\n\n  /**\n   * Returns the app associated with this InstanceId instance.\n   *\n   * @returns The app associated with this InstanceId instance.\n   */\n  get app(): App {\n    return this.app_;\n  }\n}\n\n// Path: src/project-management/app-metadata.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Platforms with which a Firebase App can be associated.\n */\nexport enum AppPlatform {\n  /**\n   * Unknown state. This is only used for distinguishing unset values.\n   */\n  PLATFORM_UNKNOWN = 'PLATFORM_UNKNOWN',\n\n  /**\n   * The Firebase App is associated with iOS.\n   */\n  IOS = 'IOS',\n\n  /**\n   * The Firebase App is associated with Android.\n   */\n  ANDROID = 'ANDROID',\n}\n\n/**\n * Metadata about a Firebase app.\n */\nexport interface AppMetadata {\n  /**\n   * The globally unique, Firebase-assigned identifier of the app.\n   *\n   * @example\n   * ```javascript\n   * var appId = appMetadata.appId;\n   * ```\n   */\n  appId: string;\n\n  /**\n   * The optional user-assigned display name of the app.\n   *\n   * @example\n   * ```javascript\n   * var displayName = appMetadata.displayName;\n   * ```\n   */\n  displayName?: string;\n\n  /**\n   * The development platform of the app. Supporting Android and iOS app platforms.\n   *\n   * @example\n   * ```javascript\n   * var platform = AppPlatform.ANDROID;\n   * ```\n   */\n  platform: AppPlatform;\n\n  /**\n   * The globally unique, user-assigned ID of the parent project for the app.\n   *\n   * @example\n   * ```javascript\n   * var projectId = appMetadata.projectId;\n   * ```\n   */\n  projectId: string;\n\n  /**\n   * The fully-qualified resource name that identifies this app.\n   *\n   * This is useful when manually constructing requests for Firebase's public API.\n   *\n   * @example\n   * ```javascript\n   * var resourceName = androidAppMetadata.resourceName;\n   * ```\n   */\n  resourceName: string;\n}\n\n// Path: src/project-management/android-app.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { FirebaseProjectManagementError } from '../utils/error';\nimport * as validator from '../utils/validator';\nimport { ProjectManagementRequestHandler, assertServerResponse } from './project-management-api-request-internal';\nimport { AppMetadata, AppPlatform } from './app-metadata';\n\n\n/**\n * Metadata about a Firebase Android App.\n */\nexport interface AndroidAppMetadata extends AppMetadata {\n\n  platform: AppPlatform.ANDROID;\n\n  /**\n   * The canonical package name of the Android App, as would appear in the Google Play Developer\n   * Console.\n   *\n   * @example\n   * ```javascript\n   * var packageName = androidAppMetadata.packageName;\n   * ```\n   */\n  packageName: string;\n}\n\n/**\n * A reference to a Firebase Android app.\n *\n * Do not call this constructor directly. Instead, use {@link ProjectManagement.androidApp}.\n */\nexport class AndroidApp {\n\n  private readonly resourceName: string;\n\n  /**\n   * @internal\n   */\n  constructor(\n      public readonly appId: string,\n      private readonly requestHandler: ProjectManagementRequestHandler) {\n    if (!validator.isNonEmptyString(appId)) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'appId must be a non-empty string.');\n    }\n\n    this.resourceName = `projects/-/androidApps/${appId}`;\n  }\n\n  /**\n   * Retrieves metadata about this Android app.\n   *\n   * @returns A promise that resolves to the retrieved metadata about this Android app.\n   */\n  public getMetadata(): Promise<AndroidAppMetadata> {\n    return this.requestHandler.getResource(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getMetadata()\\'s responseData must be a non-null object.');\n\n        const requiredFieldsList = ['name', 'appId', 'projectId', 'packageName'];\n        requiredFieldsList.forEach((requiredField) => {\n          assertServerResponse(\n            validator.isNonEmptyString(responseData[requiredField]),\n            responseData,\n            `getMetadata()'s responseData.${requiredField} must be a non-empty string.`);\n        });\n\n        const metadata: AndroidAppMetadata = {\n          platform: AppPlatform.ANDROID,\n          resourceName: responseData.name,\n          appId: responseData.appId,\n          displayName: responseData.displayName || null,\n          projectId: responseData.projectId,\n          packageName: responseData.packageName,\n        };\n        return metadata;\n      });\n  }\n\n  /**\n   * Sets the optional user-assigned display name of the app.\n   *\n   * @param newDisplayName - The new display name to set.\n   *\n   * @returns A promise that resolves when the display name has been set.\n   */\n  public setDisplayName(newDisplayName: string): Promise<void> {\n    return this.requestHandler.setDisplayName(this.resourceName, newDisplayName);\n  }\n\n  /**\n   * Gets the list of SHA certificates associated with this Android app in Firebase.\n   *\n   * @returns The list of SHA-1 and SHA-256 certificates associated with this Android app in\n   *     Firebase.\n   */\n  public getShaCertificates(): Promise<ShaCertificate[]> {\n    return this.requestHandler.getAndroidShaCertificates(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getShaCertificates()\\'s responseData must be a non-null object.');\n\n        if (!responseData.certificates) {\n          return [];\n        }\n\n        assertServerResponse(\n          validator.isArray(responseData.certificates),\n          responseData,\n          '\"certificates\" field must be present in the getShaCertificates() response data.');\n\n        const requiredFieldsList = ['name', 'shaHash'];\n\n        return responseData.certificates.map((certificateJson: any) => {\n          requiredFieldsList.forEach((requiredField) => {\n            assertServerResponse(\n              validator.isNonEmptyString(certificateJson[requiredField]),\n              responseData,\n              `getShaCertificates()'s responseData.certificates[].${requiredField} must be a `\n                      + 'non-empty string.');\n          });\n\n          return new ShaCertificate(certificateJson.shaHash, certificateJson.name);\n        });\n      });\n  }\n\n  /**\n   * Adds the given SHA certificate to this Android app.\n   *\n   * @param certificateToAdd - The SHA certificate to add.\n   *\n   * @returns A promise that resolves when the given certificate\n   *     has been added to the Android app.\n   */\n  public addShaCertificate(certificateToAdd: ShaCertificate): Promise<void> {\n    return this.requestHandler.addAndroidShaCertificate(this.resourceName, certificateToAdd);\n  }\n\n  /**\n   * Deletes the specified SHA certificate from this Android app.\n   *\n   * @param certificateToDelete - The SHA certificate to delete.\n   *\n   * @returns A promise that resolves when the specified\n   *     certificate has been removed from the Android app.\n   */\n  public deleteShaCertificate(certificateToDelete: ShaCertificate): Promise<void> {\n    if (!certificateToDelete.resourceName) {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument',\n        'Specified certificate does not include a resourceName. (Use AndroidApp.getShaCertificates() to retrieve ' +\n              'certificates with a resourceName.');\n    }\n    return this.requestHandler.deleteResource(certificateToDelete.resourceName);\n  }\n\n  /**\n   * Gets the configuration artifact associated with this app.\n   *\n   * @returns A promise that resolves to the Android app's\n   *     Firebase config file, in UTF-8 string format. This string is typically\n   *     intended to be written to a JSON file that gets shipped with your Android\n   *     app.\n   */\n  public getConfig(): Promise<string> {\n    return this.requestHandler.getConfig(this.resourceName)\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'getConfig()\\'s responseData must be a non-null object.');\n\n        const base64ConfigFileContents = responseData.configFileContents;\n        assertServerResponse(\n          validator.isBase64String(base64ConfigFileContents),\n          responseData,\n          'getConfig()\\'s responseData.configFileContents must be a base64 string.');\n\n        return Buffer.from(base64ConfigFileContents, 'base64').toString('utf8');\n      });\n  }\n}\n\n/**\n * A SHA-1 or SHA-256 certificate.\n *\n * Do not call this constructor directly. Instead, use\n * [`projectManagement.shaCertificate()`](projectManagement.ProjectManagement#shaCertificate).\n */\nexport class ShaCertificate {\n  /**\n   * The SHA certificate type.\n   *\n   * @example\n   * ```javascript\n   * var certType = shaCertificate.certType;\n   * ```\n   */\n  public readonly certType: ('sha1' | 'sha256');\n\n  /**\n   * Creates a ShaCertificate using the given hash. The ShaCertificate's type (eg. 'sha256') is\n   * automatically determined from the hash itself.\n   *\n   * @param shaHash - The sha256 or sha1 hash for this certificate.\n   * @example\n   * ```javascript\n   * var shaHash = shaCertificate.shaHash;\n   * ```\n   * @param resourceName - The Firebase resource name for this certificate. This does not need to be\n   *     set when creating a new certificate.\n   * @example\n   * ```javascript\n   * var resourceName = shaCertificate.resourceName;\n   * ```\n   *\n   * @internal\n   */\n  constructor(public readonly shaHash: string, public readonly resourceName?: string) {\n    if (/^[a-fA-F0-9]{40}$/.test(shaHash)) {\n      this.certType = 'sha1';\n    } else if (/^[a-fA-F0-9]{64}$/.test(shaHash)) {\n      this.certType = 'sha256';\n    } else {\n      throw new FirebaseProjectManagementError(\n        'invalid-argument', 'shaHash must be either a sha256 hash or a sha1 hash.');\n    }\n  }\n}\n\n// Path: src/project-management/project-management-api-request-internal.ts\n/*!\n * Copyright 2018 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport {\n  AuthorizedHttpClient, HttpError, HttpMethod, HttpRequestConfig, ExponentialBackoffPoller,\n} from '../utils/api-request';\nimport { FirebaseProjectManagementError, ProjectManagementErrorCode } from '../utils/error';\nimport { getSdkVersion } from '../utils/index';\nimport * as validator from '../utils/validator';\nimport { ShaCertificate } from './android-app';\n\n\n/** Project management backend host and port. */\nconst PROJECT_MANAGEMENT_HOST_AND_PORT = 'firebase.googleapis.com:443';\n/** Project management backend path. */\nconst PROJECT_MANAGEMENT_PATH = '/v1/';\n/** Project management beta backend path. */\nconst PROJECT_MANAGEMENT_BETA_PATH = '/v1beta1/';\n/** Project management request header. */\nconst PROJECT_MANAGEMENT_HEADERS = {\n  'X-Client-Version': `Node/Admin/${getSdkVersion()}`,\n};\n/** Project management request timeout duration in milliseconds. */\nconst PROJECT_MANAGEMENT_TIMEOUT_MILLIS = 10000;\n\nconst LIST_APPS_MAX_PAGE_SIZE = 100;\n\nconst CERT_TYPE_API_MAP = {\n  sha1: 'SHA_1',\n  sha256: 'SHA_256',\n};\n\nexport \nfunction assertServerResponse(\n  condition: boolean, responseData: object, message: string): void {\n  if (!condition) {\n    throw new FirebaseProjectManagementError(\n      'invalid-server-response',\n      `${message} Response data: ${JSON.stringify(responseData, null, 2)}`);\n  }\n}\n\n/**\n * Class that provides mechanism to send requests to the Firebase project management backend\n * endpoints.\n *\n * @internal\n */\nexport class ProjectManagementRequestHandler {\n  private readonly baseUrl: string =\n    `https://${PROJECT_MANAGEMENT_HOST_AND_PORT}${PROJECT_MANAGEMENT_PATH}`;\n  private readonly baseBetaUrl: string =\n    `https://${PROJECT_MANAGEMENT_HOST_AND_PORT}${PROJECT_MANAGEMENT_BETA_PATH}`;\n  private readonly httpClient: AuthorizedHttpClient;\n\n  private static wrapAndRethrowHttpError(errStatusCode: number, errText?: string): void {\n    let errorCode: ProjectManagementErrorCode;\n    let errorMessage: string;\n\n    switch (errStatusCode) {\n    case 400:\n      errorCode = 'invalid-argument';\n      errorMessage = 'Invalid argument provided.';\n      break;\n    case 401:\n    case 403:\n      errorCode = 'authentication-error';\n      errorMessage = 'An error occurred when trying to authenticate. Make sure the credential '\n            + 'used to authenticate this SDK has the proper permissions. See '\n            + 'https://firebase.google.com/docs/admin/setup for setup instructions.';\n      break;\n    case 404:\n      errorCode = 'not-found';\n      errorMessage = 'The specified entity could not be found.';\n      break;\n    case 409:\n      errorCode = 'already-exists';\n      errorMessage = 'The specified entity already exists.';\n      break;\n    case 500:\n      errorCode = 'internal-error';\n      errorMessage = 'An internal error has occurred. Please retry the request.';\n      break;\n    case 503:\n      errorCode = 'service-unavailable';\n      errorMessage = 'The server could not process the request in time. See the error '\n            + 'documentation for more details.';\n      break;\n    default:\n      errorCode = 'unknown-error';\n      errorMessage = 'An unknown server error was returned.';\n    }\n\n    if (!errText) {\n      errText = '<missing>';\n    }\n    throw new FirebaseProjectManagementError(\n      errorCode,\n      `${ errorMessage } Status code: ${ errStatusCode }. Raw server response: \"${ errText }\".`);\n  }\n\n  /**\n   * @param app - The app used to fetch access tokens to sign API requests.\n   * @constructor\n   */\n  constructor(app: App) {\n    this.httpClient = new AuthorizedHttpClient(app as FirebaseApp);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose Android\n   *     apps you want to list.\n   */\n  public listAndroidApps(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}/androidApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose iOS apps\n   *     you want to list.\n   */\n  public listIosApps(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}/iosApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project whose iOS apps\n   *     you want to list.\n   */\n  public listAppMetadata(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET',\n      `${parentResourceName}:searchApps?page_size=${LIST_APPS_MAX_PAGE_SIZE}`,\n      /* requestData */ null,\n      'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project that you want\n   *     to create the Android app within.\n   */\n  public createAndroidApp(\n    parentResourceName: string, packageName: string, displayName?: string): Promise<object> {\n    const requestData: any = {\n      packageName,\n    };\n    if (validator.isNonEmptyString(displayName)) {\n      requestData.displayName = displayName;\n    }\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/androidApps`, requestData, 'v1beta1')\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createAndroidApp\\'s responseData must be a non-null object.');\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.name),\n          responseData,\n          'createAndroidApp\\'s responseData.name must be a non-empty string.');\n        return this.pollRemoteOperationWithExponentialBackoff(responseData.name);\n      });\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the project that you want\n   *     to create the iOS app within.\n   */\n  public createIosApp(\n    parentResourceName: string, bundleId: string, displayName?: string): Promise<object> {\n    const requestData: any = {\n      bundleId,\n    };\n    if (validator.isNonEmptyString(displayName)) {\n      requestData.displayName = displayName;\n    }\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/iosApps`, requestData, 'v1beta1')\n      .then((responseData: any) => {\n        assertServerResponse(\n          validator.isNonNullObject(responseData),\n          responseData,\n          'createIosApp\\'s responseData must be a non-null object.');\n        assertServerResponse(\n          validator.isNonEmptyString(responseData.name),\n          responseData,\n          'createIosApp\\'s responseData.name must be a non-empty string.');\n        return this.pollRemoteOperationWithExponentialBackoff(responseData.name);\n      });\n  }\n\n  /**\n   * @param resourceName - Fully-qualified resource name of the entity whose display name you\n   *     want to set.\n   */\n  public setDisplayName(resourceName: string, newDisplayName: string): Promise<void> {\n    const requestData = {\n      displayName: newDisplayName,\n    };\n    return this\n      .invokeRequestHandler(\n        'PATCH', `${resourceName}?update_mask=display_name`, requestData, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the Android app whose SHA\n   *     certificates you want to get.\n   */\n  public getAndroidShaCertificates(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET', `${parentResourceName}/sha`, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the Android app that you\n   *     want to add the given SHA certificate to.\n   */\n  public addAndroidShaCertificate(\n    parentResourceName: string, certificate: ShaCertificate): Promise<void> {\n    const requestData = {\n      shaHash: certificate.shaHash,\n      certType: CERT_TYPE_API_MAP[certificate.certType],\n    };\n    return this\n      .invokeRequestHandler('POST', `${parentResourceName}/sha`, requestData, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the app whose config you\n   *     want to get.\n   */\n  public getConfig(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler(\n      'GET', `${parentResourceName}/config`, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param parentResourceName - Fully-qualified resource name of the entity that you want to\n   *     get.\n   */\n  public getResource(parentResourceName: string): Promise<object> {\n    return this.invokeRequestHandler('GET', parentResourceName, /* requestData */ null, 'v1beta1');\n  }\n\n  /**\n   * @param resourceName - Fully-qualified resource name of the entity that you want to\n   *     delete.\n   */\n  public deleteResource(resourceName: string): Promise<void> {\n    return this\n      .invokeRequestHandler('DELETE', resourceName, /* requestData */ null, 'v1beta1')\n      .then(() => undefined);\n  }\n\n  private pollRemoteOperationWithExponentialBackoff(\n    operationResourceName: string): Promise<object> {\n    const poller = new ExponentialBackoffPoller<object>();\n\n    return poller.poll(() => {\n      return this.invokeRequestHandler('GET', operationResourceName, /* requestData */ null)\n        .then((responseData: any) => {\n          if (responseData.error) {\n            const errStatusCode: number = responseData.error.code || 500;\n            const errText: string =\n                  responseData.error.message || JSON.stringify(responseData.error);\n            ProjectManagementRequestHandler.wrapAndRethrowHttpError(errStatusCode, errText);\n          }\n\n          if (!responseData.done) {\n...\n", "needle_token_start": 13422, "needle_token_end": 13498, "code_context_ntokens": 15891, "output": ["```typescript\nfunction assertServerResponse(\n  condition: boolean, responseData: object, message: string): void {\n  if (!condition) {\n    throw new FirebaseProjectManagementError(\n      'invalid-server-response',\n      `${message} Response data: ${JSON.stringify(responseData, null, 2)}`);\n  }\n}\n```"]}
{"repo": "firebase/firebase-admin-node", "name": "appCheckErrorFromCryptoSignerError", "language": "typescript", "path": "src/app-check/token-generator.ts", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To transform errors from a cryptographic signing process into a standardized error format suitable for user feedback within an application integrity checking system.\n2. **Input**: An error object, typically originating from cryptographic operations.\n3. **Output**: A standardized error object tailored to provide meaningful feedback to the application's integrity checking system.\n4. **Procedure**: The function checks if the input error is a specific type of cryptographic error. If it is, and contains additional server error details, it extracts these details and maps them to a corresponding application-specific error code and message. If the error details are not specific, it defaults to a general error message. For non-cryptographic errors, the original error is returned unchanged.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/remote-config/internal/value-impl.ts\n/*!\n * Copyright 2024 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n'use strict';\n\nimport {\n  Value,\n  ValueSource,\n} from '../remote-config-api';\n\n/**\n * Implements type-safe getters for parameter values.\n * \n * Visible for testing.\n * \n * @internal\n */\nexport class ValueImpl implements Value {\n  public static readonly DEFAULT_VALUE_FOR_BOOLEAN = false;\n  public static readonly DEFAULT_VALUE_FOR_STRING = '';\n  public static readonly DEFAULT_VALUE_FOR_NUMBER = 0;\n  public static readonly BOOLEAN_TRUTHY_VALUES = ['1', 'true', 't', 'yes', 'y', 'on'];\n  constructor(\n    private readonly source: ValueSource,\n    private readonly value = ValueImpl.DEFAULT_VALUE_FOR_STRING) { }\n  asString(): string {\n    return this.value;\n  }\n  asBoolean(): boolean {\n    if (this.source === 'static') {\n      return ValueImpl.DEFAULT_VALUE_FOR_BOOLEAN;\n    }\n    return ValueImpl.BOOLEAN_TRUTHY_VALUES.indexOf(this.value.toLowerCase()) >= 0;\n  }\n  asNumber(): number {\n    if (this.source === 'static') {\n      return ValueImpl.DEFAULT_VALUE_FOR_NUMBER;\n    }\n    const num = Number(this.value);\n    if (isNaN(num)) {\n      return ValueImpl.DEFAULT_VALUE_FOR_NUMBER;\n    }\n    return num;\n  }\n  getSource(): ValueSource {\n...\n// Path: src/remote-config/remote-config.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport * as validator from '../utils/validator';\nimport { FirebaseRemoteConfigError, RemoteConfigApiClient } from './remote-config-api-client-internal';\nimport { ConditionEvaluator } from './condition-evaluator-internal';\nimport { ValueImpl } from './internal/value-impl';\nimport {\n  ListVersionsOptions,\n  ListVersionsResult,\n  RemoteConfigCondition,\n  RemoteConfigParameter,\n  RemoteConfigParameterGroup,\n  ServerTemplate,\n  RemoteConfigTemplate,\n  RemoteConfigUser,\n  Version,\n  ExplicitParameterValue,\n  InAppDefaultValue,\n  ServerConfig,\n  RemoteConfigParameterValue,\n  EvaluationContext,\n  ServerTemplateData,\n  NamedCondition,\n  Value,\n  DefaultConfig,\n  GetServerTemplateOptions,\n  InitServerTemplateOptions,\n  ServerTemplateDataType,\n} from './remote-config-api';\n\n/**\n * The Firebase `RemoteConfig` service interface.\n */\nexport class RemoteConfig {\n\n  private readonly client: RemoteConfigApiClient;\n\n  /**\n   * @param app - The app for this RemoteConfig service.\n   * @constructor\n   * @internal\n   */\n  constructor(readonly app: App) {\n    this.client = new RemoteConfigApiClient(app);\n  }\n\n  /**\n   * Gets the current active version of the {@link RemoteConfigTemplate} of the project.\n   *\n   * @returns A promise that fulfills with a `RemoteConfigTemplate`.\n   */\n  public getTemplate(): Promise<RemoteConfigTemplate> {\n    return this.client.getTemplate()\n      .then((templateResponse) => {\n        return new RemoteConfigTemplateImpl(templateResponse);\n      });\n  }\n\n  /**\n   * Gets the requested version of the {@link RemoteConfigTemplate} of the project.\n   *\n   * @param versionNumber - Version number of the Remote Config template to look up.\n   *\n   * @returns A promise that fulfills with a `RemoteConfigTemplate`.\n   */\n  public getTemplateAtVersion(versionNumber: number | string): Promise<RemoteConfigTemplate> {\n    return this.client.getTemplateAtVersion(versionNumber)\n      .then((templateResponse) => {\n        return new RemoteConfigTemplateImpl(templateResponse);\n      });\n  }\n\n  /**\n   * Validates a {@link RemoteConfigTemplate}.\n   *\n   * @param template - The Remote Config template to be validated.\n   * @returns A promise that fulfills with the validated `RemoteConfigTemplate`.\n   */\n  public validateTemplate(template: RemoteConfigTemplate): Promise<RemoteConfigTemplate> {\n    return this.client.validateTemplate(template)\n      .then((templateResponse) => {\n        return new RemoteConfigTemplateImpl(templateResponse);\n      });\n  }\n\n  /**\n   * Publishes a Remote Config template.\n   *\n   * @param template - The Remote Config template to be published.\n   * @param options - Optional options object when publishing a Remote Config template:\n   *    - `force`: Setting this to `true` forces the Remote Config template to\n   *      be updated and circumvent the ETag. This approach is not recommended\n   *      because it risks causing the loss of updates to your Remote Config\n   *      template if multiple clients are updating the Remote Config template.\n   *      See {@link https://firebase.google.com/docs/remote-config/use-config-rest#etag_usage_and_forced_updates |\n   *      ETag usage and forced updates}.\n   *\n   * @returns A Promise that fulfills with the published `RemoteConfigTemplate`.\n   */\n  public publishTemplate(template: RemoteConfigTemplate, options?: { force: boolean }): Promise<RemoteConfigTemplate> {\n    return this.client.publishTemplate(template, options)\n      .then((templateResponse) => {\n        return new RemoteConfigTemplateImpl(templateResponse);\n      });\n  }\n\n  /**\n   * Rolls back a project's published Remote Config template to the specified version.\n   * A rollback is equivalent to getting a previously published Remote Config\n   * template and re-publishing it using a force update.\n   *\n   * @param versionNumber - The version number of the Remote Config template to roll back to.\n   *    The specified version number must be lower than the current version number, and not have\n   *    been deleted due to staleness. Only the last 300 versions are stored.\n   *    All versions that correspond to non-active Remote Config templates (that is, all except the\n   *    template that is being fetched by clients) are also deleted if they are more than 90 days old.\n   * @returns A promise that fulfills with the published `RemoteConfigTemplate`.\n   */\n  public rollback(versionNumber: number | string): Promise<RemoteConfigTemplate> {\n    return this.client.rollback(versionNumber)\n      .then((templateResponse) => {\n        return new RemoteConfigTemplateImpl(templateResponse);\n      });\n  }\n\n  /**\n   * Gets a list of Remote Config template versions that have been published, sorted in reverse\n   * chronological order. Only the last 300 versions are stored.\n   * All versions that correspond to non-active Remote Config templates (i.e., all except the\n   * template that is being fetched by clients) are also deleted if they are older than 90 days.\n   *\n   * @param options - Optional options object for getting a list of versions.\n   * @returns A promise that fulfills with a `ListVersionsResult`.\n   */\n  public listVersions(options?: ListVersionsOptions): Promise<ListVersionsResult> {\n    return this.client.listVersions(options)\n      .then((listVersionsResponse) => {\n        return {\n          versions: listVersionsResponse.versions?.map(version => new VersionImpl(version)) ?? [],\n          nextPageToken: listVersionsResponse.nextPageToken,\n        }\n      });\n  }\n\n  /**\n   * Creates and returns a new Remote Config template from a JSON string.\n   *\n   * @param json - The JSON string to populate a Remote Config template.\n   *\n   * @returns A new template instance.\n   */\n  public createTemplateFromJSON(json: string): RemoteConfigTemplate {\n    if (!validator.isNonEmptyString(json)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        'JSON string must be a valid non-empty string');\n    }\n\n    let template: RemoteConfigTemplate;\n    try {\n      template = JSON.parse(json);\n    } catch (e) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        `Failed to parse the JSON string: ${json}. ` + e\n      );\n    }\n\n    return new RemoteConfigTemplateImpl(template);\n  }\n\n  /**\n   * Instantiates {@link ServerTemplate} and then fetches and caches the latest\n   * template version of the project.\n   */\n  public async getServerTemplate(options?: GetServerTemplateOptions): Promise<ServerTemplate> {\n    const template = this.initServerTemplate(options);\n    await template.load();\n    return template;\n  }\n\n  /**\n   * Synchronously instantiates {@link ServerTemplate}.\n   */\n  public initServerTemplate(options?: InitServerTemplateOptions): ServerTemplate {\n    const template = new ServerTemplateImpl(\n      this.client, new ConditionEvaluator(), options?.defaultConfig);\n\n    if (options?.template) {\n      template.set(options?.template);\n    }\n\n    return template;\n  }\n}\n\n/**\n * Remote Config template internal implementation.\n */\nclass RemoteConfigTemplateImpl implements RemoteConfigTemplate {\n\n  public parameters: { [key: string]: RemoteConfigParameter };\n  public parameterGroups: { [key: string]: RemoteConfigParameterGroup };\n  public conditions: RemoteConfigCondition[];\n  private readonly etagInternal: string;\n  public version?: Version;\n\n  constructor(config: RemoteConfigTemplate) {\n    if (!validator.isNonNullObject(config) ||\n      !validator.isNonEmptyString(config.etag)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        `Invalid Remote Config template: ${JSON.stringify(config)}`);\n    }\n\n    this.etagInternal = config.etag;\n\n    if (typeof config.parameters !== 'undefined') {\n      if (!validator.isNonNullObject(config.parameters)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Remote Config parameters must be a non-null object');\n      }\n      this.parameters = config.parameters;\n    } else {\n      this.parameters = {};\n    }\n\n    if (typeof config.parameterGroups !== 'undefined') {\n      if (!validator.isNonNullObject(config.parameterGroups)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Remote Config parameter groups must be a non-null object');\n      }\n      this.parameterGroups = config.parameterGroups;\n    } else {\n      this.parameterGroups = {};\n    }\n\n    if (typeof config.conditions !== 'undefined') {\n      if (!validator.isArray(config.conditions)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Remote Config conditions must be an array');\n      }\n      this.conditions = config.conditions;\n    } else {\n      this.conditions = [];\n    }\n\n    if (typeof config.version !== 'undefined') {\n      this.version = new VersionImpl(config.version);\n    }\n  }\n\n  /**\n   * Gets the ETag of the template.\n   *\n   * @returns The ETag of the Remote Config template.\n   */\n  get etag(): string {\n    return this.etagInternal;\n  }\n\n  /**\n   * Returns a JSON-serializable representation of this object.\n   *\n   * @returns A JSON-serializable representation of this object.\n   */\n  public toJSON(): object {\n    return {\n      conditions: this.conditions,\n      parameters: this.parameters,\n      parameterGroups: this.parameterGroups,\n      etag: this.etag,\n      version: this.version,\n    }\n  }\n}\n\n/**\n * Remote Config dataplane template data implementation.\n */\nclass ServerTemplateImpl implements ServerTemplate {\n  private cache: ServerTemplateData;\n  private stringifiedDefaultConfig: {[key: string]: string} = {};\n\n  constructor(\n    private readonly apiClient: RemoteConfigApiClient,\n    private readonly conditionEvaluator: ConditionEvaluator,\n    public readonly defaultConfig: DefaultConfig = {}\n  ) {\n    // RC stores all remote values as string, but it's more intuitive\n    // to declare default values with specific types, so this converts\n    // the external declaration to an internal string representation.\n    for (const key in defaultConfig) {\n      this.stringifiedDefaultConfig[key] = String(defaultConfig[key]);\n    }\n  }\n\n  /**\n   * Fetches and caches the current active version of the project's {@link ServerTemplate}.\n   */\n  public load(): Promise<void> {\n    return this.apiClient.getServerTemplate()\n      .then((template) => {\n        this.cache = new ServerTemplateDataImpl(template);\n      });\n  }\n\n  /**\n   * Parses a {@link ServerTemplateDataType} and caches it.\n   */\n  public set(template: ServerTemplateDataType): void {\n    let parsed;\n    if (validator.isString(template)) {\n      try {\n        parsed = JSON.parse(template);\n      } catch (e) {\n        // Transforms JSON parse errors to Firebase error.\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          `Failed to parse the JSON string: ${template}. ` + e);\n      }\n    } else {\n      parsed = template;\n    }\n    // Throws template parse errors.\n    this.cache = new ServerTemplateDataImpl(parsed);\n  }\n\n  /**\n   * Evaluates the current template in cache to produce a {@link ServerConfig}.\n   */\n  public evaluate(context: EvaluationContext = {}): ServerConfig {\n    if (!this.cache) {\n\n      // This is the only place we should throw during evaluation, since it's under the\n      // control of application logic. To preserve forward-compatibility, we should only\n      // return false in cases where the SDK is unsure how to evaluate the fetched template.\n      throw new FirebaseRemoteConfigError(\n        'failed-precondition',\n        'No Remote Config Server template in cache. Call load() before calling evaluate().');\n    }\n\n    const evaluatedConditions = this.conditionEvaluator.evaluateConditions(\n      this.cache.conditions, context);\n\n    const configValues: { [key: string]: Value } = {};\n\n    // Initializes config Value objects with default values.\n    for (const key in this.stringifiedDefaultConfig) {\n      configValues[key] = new ValueImpl('default', this.stringifiedDefaultConfig[key]);\n    }\n\n    // Overlays config Value objects derived by evaluating the template.\n    for (const [key, parameter] of Object.entries(this.cache.parameters)) {\n      const { conditionalValues, defaultValue } = parameter;\n\n      // Supports parameters with no conditional values.\n      const normalizedConditionalValues = conditionalValues || {};\n\n      let parameterValueWrapper: RemoteConfigParameterValue | undefined = undefined;\n\n      // Iterates in order over condition list. If there is a value associated\n      // with a condition, this checks if the condition is true.\n      for (const [conditionName, conditionEvaluation] of evaluatedConditions) {\n        if (normalizedConditionalValues[conditionName] && conditionEvaluation) {\n          parameterValueWrapper = normalizedConditionalValues[conditionName];\n          break;\n        }\n      }\n\n      if (parameterValueWrapper && (parameterValueWrapper as InAppDefaultValue).useInAppDefault) {\n        // TODO: add logging once we have a wrapped logger.\n        continue;\n      }\n\n      if (parameterValueWrapper) {\n        const parameterValue = (parameterValueWrapper as ExplicitParameterValue).value;\n        configValues[key] = new ValueImpl('remote', parameterValue);\n        continue;\n      }\n\n      if (!defaultValue) {\n        // TODO: add logging once we have a wrapped logger.\n        continue;\n      }\n\n      if ((defaultValue as InAppDefaultValue).useInAppDefault) {\n        // TODO: add logging once we have a wrapped logger.\n        continue;\n      }\n\n      const parameterDefaultValue = (defaultValue as ExplicitParameterValue).value;\n      configValues[key] = new ValueImpl('remote', parameterDefaultValue);\n    }\n\n    return new ServerConfigImpl(configValues);\n  }\n\n  /**\n   * @returns JSON representation of the server template\n   */\n  public toJSON(): ServerTemplateData {\n    return this.cache;\n  }\n}\n\nclass ServerConfigImpl implements ServerConfig {\n  constructor(\n    private readonly configValues: { [key: string]: Value },\n  ){}\n  getBoolean(key: string): boolean {\n    return this.getValue(key).asBoolean();\n  }\n  getNumber(key: string): number {\n    return this.getValue(key).asNumber();\n  }\n  getString(key: string): string {\n    return this.getValue(key).asString();\n  }\n  getValue(key: string): Value {\n    return this.configValues[key] || new ValueImpl('static');\n  }\n}\n\n/**\n * Remote Config dataplane template data implementation.\n */\nclass ServerTemplateDataImpl implements ServerTemplateData {\n  public parameters: { [key: string]: RemoteConfigParameter };\n  public parameterGroups: { [key: string]: RemoteConfigParameterGroup };\n  public conditions: NamedCondition[];\n  public readonly etag: string;\n  public version?: Version;\n\n  constructor(template: ServerTemplateData) {\n    if (!validator.isNonNullObject(template) ||\n      !validator.isNonEmptyString(template.etag)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        `Invalid Remote Config template: ${JSON.stringify(template)}`);\n    }\n\n    this.etag = template.etag;\n    if (typeof template.parameters !== 'undefined') {\n      if (!validator.isNonNullObject(template.parameters)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Remote Config parameters must be a non-null object');\n      }\n      this.parameters = template.parameters;\n    } else {\n      this.parameters = {};\n    }\n\n    if (typeof template.conditions !== 'undefined') {\n      if (!validator.isArray(template.conditions)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Remote Config conditions must be an array');\n      }\n      this.conditions = template.conditions;\n    } else {\n      this.conditions = [];\n    }\n\n    if (typeof template.version !== 'undefined') {\n      this.version = new VersionImpl(template.version);\n    }\n  }\n}\n\n/**\n* Remote Config Version internal implementation.\n*/\nclass VersionImpl implements Version {\n  public readonly versionNumber?: string; // int64 format\n  public readonly updateTime?: string; // in UTC\n  public readonly updateOrigin?: ('REMOTE_CONFIG_UPDATE_ORIGIN_UNSPECIFIED' | 'CONSOLE' |\n    'REST_API' | 'ADMIN_SDK_NODE');\n  public readonly updateType?: ('REMOTE_CONFIG_UPDATE_TYPE_UNSPECIFIED' |\n    'INCREMENTAL_UPDATE' | 'FORCED_UPDATE' | 'ROLLBACK');\n  public readonly updateUser?: RemoteConfigUser;\n  public readonly description?: string;\n  public readonly rollbackSource?: string;\n  public readonly isLegacy?: boolean;\n\n  constructor(version: Version) {\n    if (!validator.isNonNullObject(version)) {\n      throw new FirebaseRemoteConfigError(\n        'invalid-argument',\n        `Invalid Remote Config version instance: ${JSON.stringify(version)}`);\n    }\n\n    if (typeof version.versionNumber !== 'undefined') {\n      if (!validator.isNonEmptyString(version.versionNumber) &&\n        !validator.isNumber(version.versionNumber)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version number must be a non-empty string in int64 format or a number');\n      }\n      if (!Number.isInteger(Number(version.versionNumber))) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version number must be an integer or a string in int64 format');\n      }\n      this.versionNumber = version.versionNumber;\n    }\n\n    if (typeof version.updateOrigin !== 'undefined') {\n      if (!validator.isNonEmptyString(version.updateOrigin)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version update origin must be a non-empty string');\n      }\n      this.updateOrigin = version.updateOrigin;\n    }\n\n    if (typeof version.updateType !== 'undefined') {\n      if (!validator.isNonEmptyString(version.updateType)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version update type must be a non-empty string');\n      }\n      this.updateType = version.updateType;\n    }\n\n    if (typeof version.updateUser !== 'undefined') {\n      if (!validator.isNonNullObject(version.updateUser)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version update user must be a non-null object');\n      }\n      this.updateUser = version.updateUser;\n    }\n\n    if (typeof version.description !== 'undefined') {\n      if (!validator.isNonEmptyString(version.description)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version description must be a non-empty string');\n      }\n      this.description = version.description;\n    }\n\n    if (typeof version.rollbackSource !== 'undefined') {\n      if (!validator.isNonEmptyString(version.rollbackSource)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version rollback source must be a non-empty string');\n      }\n      this.rollbackSource = version.rollbackSource;\n    }\n\n    if (typeof version.isLegacy !== 'undefined') {\n      if (!validator.isBoolean(version.isLegacy)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version.isLegacy must be a boolean');\n      }\n      this.isLegacy = version.isLegacy;\n    }\n\n    // The backend API provides timestamps in ISO date strings. The Admin SDK exposes timestamps\n    // in UTC date strings. If a developer uses a previously obtained template with UTC timestamps\n    // we could still validate it below.\n    if (typeof version.updateTime !== 'undefined') {\n      if (!this.isValidTimestamp(version.updateTime)) {\n        throw new FirebaseRemoteConfigError(\n          'invalid-argument',\n          'Version update time must be a valid date string');\n      }\n      this.updateTime = new Date(version.updateTime).toUTCString();\n    }\n  }\n\n  /**\n   * @returns A JSON-serializable representation of this object.\n   */\n  public toJSON(): object {\n    return {\n      versionNumber: this.versionNumber,\n      updateOrigin: this.updateOrigin,\n      updateType: this.updateType,\n      updateUser: this.updateUser,\n      description: this.description,\n      rollbackSource: this.rollbackSource,\n      isLegacy: this.isLegacy,\n      updateTime: this.updateTime,\n    }\n  }\n\n  private isValidTimestamp(timestamp: string): boolean {\n    // This validation fails for timestamps earlier than January 1, 1970 and considers strings\n    // such as \"1.2\" as valid timestamps.\n    return validator.isNonEmptyString(timestamp) && (new Date(timestamp)).getTime() > 0;\n  }\n}\n\n// Path: src/remote-config/index.ts\n/*!\n * Copyright 2020 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Firebase Remote Config.\n *\n * @packageDocumentation\n */\n\nimport { App, getApp } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { RemoteConfig } from './remote-config';\n\nexport {\n  AndCondition,\n  DefaultConfig,\n  EvaluationContext,\n  ExplicitParameterValue,\n  GetServerTemplateOptions,\n  InAppDefaultValue,\n  InitServerTemplateOptions,\n  ListVersionsOptions,\n  ListVersionsResult,\n  MicroPercentRange,\n  NamedCondition,\n  OneOfCondition,\n  OrCondition,\n  ParameterValueType,\n  PercentConditionOperator,\n  PercentCondition,\n  RemoteConfigCondition,\n  RemoteConfigParameter,\n  RemoteConfigParameterGroup,\n  RemoteConfigParameterValue,\n  RemoteConfigTemplate,\n  RemoteConfigUser,\n  ServerConfig,\n  ServerTemplate,\n  ServerTemplateData,\n  ServerTemplateDataType,\n  TagColor,\n  Value,\n  ValueSource,\n  Version,\n} from './remote-config-api';\nexport { RemoteConfig } from './remote-config';\n\n/**\n * Gets the {@link RemoteConfig} service for the default app or a given app.\n *\n * `getRemoteConfig()` can be called with no arguments to access the default\n * app's `RemoteConfig` service or as `getRemoteConfig(app)` to access the\n * `RemoteConfig` service associated with a specific app.\n *\n * @example\n * ```javascript\n * // Get the `RemoteConfig` service for the default app\n * const defaultRemoteConfig = getRemoteConfig();\n * ```\n *\n * @example\n * ```javascript\n * // Get the `RemoteConfig` service for a given app\n * const otherRemoteConfig = getRemoteConfig(otherApp);\n * ```\n *\n * @param app - Optional app for which to return the `RemoteConfig` service.\n *   If not provided, the default `RemoteConfig` service is returned.\n *\n * @returns The default `RemoteConfig` service if no\n *   app is provided, or the `RemoteConfig` service associated with the provided\n *   app.\n */\nexport function getRemoteConfig(app?: App): RemoteConfig {\n  if (typeof app === 'undefined') {\n    app = getApp();\n  }\n\n  const firebaseApp: FirebaseApp = app as FirebaseApp;\n  return firebaseApp.getOrInitService('remoteConfig', (app) => new RemoteConfig(app));\n}\n\n// Path: src/app-check/app-check-api.ts\n/*!\n * @license\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * Interface representing an App Check token.\n */\nexport interface AppCheckToken {\n  /**\n   * The Firebase App Check token.\n   */\n  token: string;\n\n  /**\n   * The time-to-live duration of the token in milliseconds.\n   */\n  ttlMillis: number;\n}\n\n/**\n * Interface representing App Check token options.\n */\nexport interface AppCheckTokenOptions {\n  /**\n   * The length of time, in milliseconds, for which the App Check token will\n   * be valid. This value must be between 30 minutes and 7 days, inclusive.\n   */\n  ttlMillis?: number;\n}\n\n/**\n * Interface representing options for the {@link AppCheck.verifyToken} method.\n */\nexport interface VerifyAppCheckTokenOptions {\n  /**\n   * To use the replay protection feature, set this to `true`. The {@link AppCheck.verifyToken}\n   * method will mark the token as consumed after verifying it.\n   * \n   * Tokens that are found to be already consumed will be marked as such in the response.\n   * \n   * Tokens are only considered to be consumed if it is sent to App Check backend by calling the\n   * {@link AppCheck.verifyToken} method with this field set to `true`; other uses of the token\n   * do not consume it.\n   * \n   * This replay protection feature requires an additional network call to the App Check backend\n   * and forces your clients to obtain a fresh attestation from your chosen attestation providers.\n   * This can therefore negatively impact performance and can potentially deplete your attestation\n   * providers' quotas faster. We recommend that you use this feature only for protecting\n   * low volume, security critical, or expensive operations.\n   */\n  consume?: boolean;\n}\n\n/**\n * Interface representing a decoded Firebase App Check token, returned from the\n * {@link AppCheck.verifyToken} method.\n */\nexport interface DecodedAppCheckToken {\n  /**\n   * The issuer identifier for the issuer of the response.\n   * This value is a URL with the format\n   * `https://firebaseappcheck.googleapis.com/<PROJECT_NUMBER>`, where `<PROJECT_NUMBER>` is the\n   * same project number specified in the {@link DecodedAppCheckToken.aud | aud} property.\n   */\n  iss: string;\n\n  /**\n   * The Firebase App ID corresponding to the app the token belonged to.\n   * As a convenience, this value is copied over to the {@link DecodedAppCheckToken.app_id | app_id} property.\n   */\n  sub: string;\n\n  /**\n   * The audience for which this token is intended.\n   * This value is a JSON array of two strings, the first is the project number of your\n   * Firebase project, and the second is the project ID of the same project.\n   */\n  aud: string[];\n\n  /**\n   * The App Check token's expiration time, in seconds since the Unix epoch. That is, the\n   * time at which this App Check token expires and should no longer be considered valid.\n   */\n  exp: number;\n\n  /**\n   * The App Check token's issued-at time, in seconds since the Unix epoch. That is, the\n   * time at which this App Check token was issued and should start to be considered\n   * valid.\n   */\n  iat: number;\n\n  /**\n   * The App ID corresponding to the App the App Check token belonged to.\n   * This value is not actually one of the JWT token claims. It is added as a\n   * convenience, and is set as the value of the {@link DecodedAppCheckToken.sub | sub} property.\n   */\n  app_id: string;\n  [key: string]: any;\n}\n\n/**\n * Interface representing a verified App Check token response.\n */\nexport interface VerifyAppCheckTokenResponse {\n  /**\n   * The App ID corresponding to the App the App Check token belonged to.\n   */\n  appId: string;\n\n  /**\n   * The decoded Firebase App Check token.\n   */\n  token: DecodedAppCheckToken;\n\n  /**\n   * Indicates weather this token was already consumed.\n   * If this is the first time {@link AppCheck.verifyToken} method has seen this token,\n   * this field will contain the value `false`. The given token will then be\n   * marked as `already_consumed` for all future invocations of this {@link AppCheck.verifyToken}\n   * method for this token.\n   * \n   * When this field is `true`, the caller is attempting to reuse a previously consumed token.\n   * You should take precautions against such a caller; for example, you can take actions such as\n   * rejecting the request or ask the caller to pass additional layers of security checks.\n   */\n  alreadyConsumed?: boolean;\n}\n\n// Path: src/utils/crypto-signer.ts\n/*!\n * @license\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport { ServiceAccountCredential } from '../app/credential-internal';\nimport { AuthorizedHttpClient, HttpRequestConfig, HttpClient, HttpError } from './api-request';\n\nimport { Algorithm } from 'jsonwebtoken';\nimport { ErrorInfo } from '../utils/error';\nimport * as validator from '../utils/validator';\n\nconst ALGORITHM_RS256: Algorithm = 'RS256' as const;\n\n/**\n * CryptoSigner interface represents an object that can be used to sign JWTs.\n */\nexport interface CryptoSigner {\n\n  /**\n   * The name of the signing algorithm.\n   */\n  readonly algorithm: Algorithm;\n\n  /**\n   * Cryptographically signs a buffer of data.\n   *\n   * @param buffer - The data to be signed.\n   * @returns A promise that resolves with the raw bytes of a signature.\n   */\n  sign(buffer: Buffer): Promise<Buffer>;\n\n  /**\n   * Returns the ID of the service account used to sign tokens.\n   *\n   * @returns A promise that resolves with a service account ID.\n   */\n  getAccountId(): Promise<string>;\n}\n\n/**\n * A CryptoSigner implementation that uses an explicitly specified service account private key to\n * sign data. Performs all operations locally, and does not make any RPC calls.\n */\nexport class ServiceAccountSigner implements CryptoSigner {\n\n  algorithm = ALGORITHM_RS256;\n\n  /**\n   * Creates a new CryptoSigner instance from the given service account credential.\n   *\n   * @param credential - A service account credential.\n   */\n  constructor(private readonly credential: ServiceAccountCredential) {\n    if (!credential) {\n      throw new CryptoSignerError({\n        code: CryptoSignerErrorCode.INVALID_CREDENTIAL,\n        message: 'INTERNAL ASSERT: Must provide a service account credential to initialize ServiceAccountSigner.',\n      });\n    }\n  }\n\n  /**\n   * @inheritDoc\n   */\n  public sign(buffer: Buffer): Promise<Buffer> {\n    const crypto = require('crypto'); // eslint-disable-line @typescript-eslint/no-var-requires\n    const sign = crypto.createSign('RSA-SHA256');\n    sign.update(buffer);\n    return Promise.resolve(sign.sign(this.credential.privateKey));\n  }\n\n  /**\n   * @inheritDoc\n   */\n  public getAccountId(): Promise<string> {\n    return Promise.resolve(this.credential.clientEmail);\n  }\n}\n\n/**\n * A CryptoSigner implementation that uses the remote IAM service to sign data. If initialized without\n * a service account ID, attempts to discover a service account ID by consulting the local Metadata\n * service. This will succeed in managed environments like Google Cloud Functions and App Engine.\n *\n * @see https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/signBlob\n * @see https://cloud.google.com/compute/docs/storing-retrieving-metadata\n */\nexport class IAMSigner implements CryptoSigner {\n  algorithm = ALGORITHM_RS256;\n\n  private readonly httpClient: AuthorizedHttpClient;\n  private serviceAccountId?: string;\n\n  constructor(httpClient: AuthorizedHttpClient, serviceAccountId?: string) {\n    if (!httpClient) {\n      throw new CryptoSignerError({\n        code: CryptoSignerErrorCode.INVALID_ARGUMENT,\n        message: 'INTERNAL ASSERT: Must provide a HTTP client to initialize IAMSigner.',\n      });\n    }\n    if (typeof serviceAccountId !== 'undefined' && !validator.isNonEmptyString(serviceAccountId)) {\n      throw new CryptoSignerError({\n        code: CryptoSignerErrorCode.INVALID_ARGUMENT,\n        message: 'INTERNAL ASSERT: Service account ID must be undefined or a non-empty string.',\n      });\n    }\n    this.httpClient = httpClient;\n    this.serviceAccountId = serviceAccountId;\n  }\n\n  /**\n   * @inheritDoc\n   */\n  public sign(buffer: Buffer): Promise<Buffer> {\n    return this.getAccountId().then((serviceAccount) => {\n      const request: HttpRequestConfig = {\n        method: 'POST',\n        url: `https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${serviceAccount}:signBlob`,\n        data: { payload: buffer.toString('base64') },\n      };\n      return this.httpClient.send(request);\n    }).then((response: any) => {\n      // Response from IAM is base64 encoded. Decode it into a buffer and return.\n      return Buffer.from(response.data.signedBlob, 'base64');\n    }).catch((err) => {\n      if (err instanceof HttpError) {\n        throw new CryptoSignerError({\n          code: CryptoSignerErrorCode.SERVER_ERROR,\n          message: err.message,\n          cause: err\n        });\n      }\n      throw err\n    });\n  }\n\n  /**\n   * @inheritDoc\n   */\n  public getAccountId(): Promise<string> {\n    if (validator.isNonEmptyString(this.serviceAccountId)) {\n      return Promise.resolve(this.serviceAccountId);\n    }\n    const request: HttpRequestConfig = {\n      method: 'GET',\n      url: 'http://metadata/computeMetadata/v1/instance/service-accounts/default/email',\n      headers: {\n        'Metadata-Flavor': 'Google',\n      },\n    };\n    const client = new HttpClient();\n    return client.send(request).then((response) => {\n      if (!response.text) {\n        throw new CryptoSignerError({\n          code: CryptoSignerErrorCode.INTERNAL_ERROR,\n          message: 'HTTP Response missing payload',\n        });\n      }\n      this.serviceAccountId = response.text;\n      return response.text;\n    }).catch((err) => {\n      throw new CryptoSignerError({\n        code: CryptoSignerErrorCode.INVALID_CREDENTIAL,\n        message: 'Failed to determine service account. Make sure to initialize ' +\n          'the SDK with a service account credential. Alternatively specify a service ' +\n          `account with iam.serviceAccounts.signBlob permission. Original error: ${err}`,\n      });\n    });\n  }\n}\n\n/**\n * Creates a new CryptoSigner instance for the given app. If the app has been initialized with a\n * service account credential, creates a ServiceAccountSigner.\n *\n * @param app - A FirebaseApp instance.\n * @returns A CryptoSigner instance.\n */\nexport function cryptoSignerFromApp(app: App): CryptoSigner {\n  const credential = app.options.credential;\n  if (credential instanceof ServiceAccountCredential) {\n    return new ServiceAccountSigner(credential);\n  }\n\n  return new IAMSigner(new AuthorizedHttpClient(app as FirebaseApp), app.options.serviceAccountId);\n}\n\n/**\n * Defines extended error info type. This includes a code, message string, and error data.\n */\nexport interface ExtendedErrorInfo extends ErrorInfo {\n  cause?: Error;\n}\n\n/**\n * CryptoSigner error code structure.\n *\n * @param errorInfo - The error information (code and message).\n * @constructor\n */\nexport class CryptoSignerError extends Error {\n  constructor(private errorInfo: ExtendedErrorInfo) {\n    super(errorInfo.message);\n\n    /* tslint:disable:max-line-length */\n    // Set the prototype explicitly. See the following link for more details:\n    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work\n    /* tslint:enable:max-line-length */\n    (this as any).__proto__ = CryptoSignerError.prototype;\n  }\n\n  /** @returns The error code. */\n  public get code(): string {\n    return this.errorInfo.code;\n  }\n\n  /** @returns The error message. */\n  public get message(): string {\n    return this.errorInfo.message;\n  }\n\n  /** @returns The error data. */\n  public get cause(): Error | undefined {\n    return this.errorInfo.cause;\n  }\n}\n\n/**\n * Crypto Signer error codes and their default messages.\n */\nexport class CryptoSignerErrorCode {\n  public static INVALID_ARGUMENT = 'invalid-argument';\n  public static INTERNAL_ERROR = 'internal-error';\n  public static INVALID_CREDENTIAL = 'invalid-credential';\n  public static SERVER_ERROR = 'server-error';\n}\n\n// Path: src/app-check/app-check-api-client-internal.ts\n/*!\n * @license\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { App } from '../app';\nimport { FirebaseApp } from '../app/firebase-app';\nimport {\n  HttpRequestConfig, HttpClient, HttpError, AuthorizedHttpClient, HttpResponse\n} from '../utils/api-request';\nimport { PrefixedFirebaseError } from '../utils/error';\nimport * as utils from '../utils/index';\nimport * as validator from '../utils/validator';\nimport { AppCheckToken } from './app-check-api'\n\n// App Check backend constants\nconst FIREBASE_APP_CHECK_V1_API_URL_FORMAT = 'https://firebaseappcheck.googleapis.com/v1/projects/{projectId}/apps/{appId}:exchangeCustomToken';\nconst ONE_TIME_USE_TOKEN_VERIFICATION_URL_FORMAT = 'https://firebaseappcheck.googleapis.com/v1beta/projects/{projectId}:verifyAppCheckToken';\n\nconst FIREBASE_APP_CHECK_CONFIG_HEADERS = {\n  'X-Firebase-Client': `fire-admin-node/${utils.getSdkVersion()}`\n};\n\n/**\n * Class that facilitates sending requests to the Firebase App Check backend API.\n *\n * @internal\n */\nexport class AppCheckApiClient {\n  private readonly httpClient: HttpClient;\n  private projectId?: string;\n\n  constructor(private readonly app: App) {\n    if (!validator.isNonNullObject(app) || !('options' in app)) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument',\n        'First argument passed to admin.appCheck() must be a valid Firebase app instance.');\n    }\n    this.httpClient = new AuthorizedHttpClient(app as FirebaseApp);\n  }\n\n  /**\n   * Exchange a signed custom token to App Check token\n   *\n   * @param customToken - The custom token to be exchanged.\n   * @param appId - The mobile App ID.\n   * @returns A promise that fulfills with a `AppCheckToken`.\n   */\n  public exchangeToken(customToken: string, appId: string): Promise<AppCheckToken> {\n    if (!validator.isNonEmptyString(appId)) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument',\n        '`appId` must be a non-empty string.');\n    }\n    if (!validator.isNonEmptyString(customToken)) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument',\n        '`customToken` must be a non-empty string.');\n    }\n    return this.getUrl(appId)\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'POST',\n          url,\n          headers: FIREBASE_APP_CHECK_CONFIG_HEADERS,\n          data: { customToken }\n        };\n        return this.httpClient.send(request);\n      })\n      .then((resp) => {\n        return this.toAppCheckToken(resp);\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  public verifyReplayProtection(token: string): Promise<boolean> {\n    if (!validator.isNonEmptyString(token)) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument',\n        '`token` must be a non-empty string.');\n    }\n    return this.getVerifyTokenUrl()\n      .then((url) => {\n        const request: HttpRequestConfig = {\n          method: 'POST',\n          url,\n          headers: FIREBASE_APP_CHECK_CONFIG_HEADERS,\n          data: { app_check_token: token }\n        };\n        return this.httpClient.send(request);\n      })\n      .then((resp) => {\n        if (typeof resp.data.alreadyConsumed !== 'undefined'\n          && !validator.isBoolean(resp.data?.alreadyConsumed)) {\n          throw new FirebaseAppCheckError(\n            'invalid-argument', '`alreadyConsumed` must be a boolean value.');\n        }\n        return resp.data.alreadyConsumed || false;\n      })\n      .catch((err) => {\n        throw this.toFirebaseError(err);\n      });\n  }\n\n  private getUrl(appId: string): Promise<string> {\n    return this.getProjectId()\n      .then((projectId) => {\n        const urlParams = {\n          projectId,\n          appId,\n        };\n        const baseUrl = utils.formatString(FIREBASE_APP_CHECK_V1_API_URL_FORMAT, urlParams);\n        return utils.formatString(baseUrl);\n      });\n  }\n\n  private getVerifyTokenUrl(): Promise<string> {\n    return this.getProjectId()\n      .then((projectId) => {\n        const urlParams = {\n          projectId\n        };\n        const baseUrl = utils.formatString(ONE_TIME_USE_TOKEN_VERIFICATION_URL_FORMAT, urlParams);\n        return utils.formatString(baseUrl);\n      });\n  }\n\n  private getProjectId(): Promise<string> {\n    if (this.projectId) {\n      return Promise.resolve(this.projectId);\n    }\n    return utils.findProjectId(this.app)\n      .then((projectId) => {\n        if (!validator.isNonEmptyString(projectId)) {\n          throw new FirebaseAppCheckError(\n            'unknown-error',\n            'Failed to determine project ID. Initialize the '\n            + 'SDK with service account credentials or set project ID as an app option. '\n            + 'Alternatively, set the GOOGLE_CLOUD_PROJECT environment variable.');\n        }\n        this.projectId = projectId;\n        return projectId;\n      });\n  }\n\n  private toFirebaseError(err: HttpError): PrefixedFirebaseError {\n    if (err instanceof PrefixedFirebaseError) {\n      return err;\n    }\n\n    const response = err.response;\n    if (!response.isJson()) {\n      return new FirebaseAppCheckError(\n        'unknown-error',\n        `Unexpected response with status: ${response.status} and body: ${response.text}`);\n    }\n\n    const error: Error = (response.data as ErrorResponse).error || {};\n    let code: AppCheckErrorCode = 'unknown-error';\n    if (error.status && error.status in APP_CHECK_ERROR_CODE_MAPPING) {\n      code = APP_CHECK_ERROR_CODE_MAPPING[error.status];\n    }\n    const message = error.message || `Unknown server error: ${response.text}`;\n    return new FirebaseAppCheckError(code, message);\n  }\n\n  /**\n   * Creates an AppCheckToken from the API response.\n   *\n   * @param resp - API response object.\n   * @returns An AppCheckToken instance.\n   */\n  private toAppCheckToken(resp: HttpResponse): AppCheckToken {\n    const token = resp.data.token;\n    // `ttl` is a string with the suffix \"s\" preceded by the number of seconds,\n    // with nanoseconds expressed as fractional seconds.\n    const ttlMillis = this.stringToMilliseconds(resp.data.ttl);\n    return {\n      token,\n      ttlMillis\n    }\n  }\n\n  /**\n   * Converts a duration string with the suffix `s` to milliseconds.\n   *\n   * @param duration - The duration as a string with the suffix \"s\" preceded by the\n   * number of seconds, with fractional seconds. For example, 3 seconds with 0 nanoseconds\n   * is expressed as \"3s\", while 3 seconds and 1 nanosecond is expressed as \"3.000000001s\",\n   * and 3 seconds and 1 microsecond is expressed as \"3.000001s\".\n   *\n   * @returns The duration in milliseconds.\n   */\n  private stringToMilliseconds(duration: string): number {\n    if (!validator.isNonEmptyString(duration) || !duration.endsWith('s')) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument', '`ttl` must be a valid duration string with the suffix `s`.');\n    }\n    const seconds = duration.slice(0, -1);\n    return Math.floor(Number(seconds) * 1000);\n  }\n}\n\ninterface ErrorResponse {\n  error?: Error;\n}\n\ninterface Error {\n  code?: number;\n  message?: string;\n  status?: string;\n}\n\nexport const APP_CHECK_ERROR_CODE_MAPPING: { [key: string]: AppCheckErrorCode } = {\n  ABORTED: 'aborted',\n  INVALID_ARGUMENT: 'invalid-argument',\n  INVALID_CREDENTIAL: 'invalid-credential',\n  INTERNAL: 'internal-error',\n  PERMISSION_DENIED: 'permission-denied',\n  UNAUTHENTICATED: 'unauthenticated',\n  NOT_FOUND: 'not-found',\n  UNKNOWN: 'unknown-error',\n};\n\nexport type AppCheckErrorCode =\n  'aborted'\n  | 'invalid-argument'\n  | 'invalid-credential'\n  | 'internal-error'\n  | 'permission-denied'\n  | 'unauthenticated'\n  | 'not-found'\n  | 'app-check-token-expired'\n  | 'unknown-error';\n\n/**\n * Firebase App Check error code structure. This extends PrefixedFirebaseError.\n *\n * @param code - The error code.\n * @param message - The error message.\n * @constructor\n */\nexport class FirebaseAppCheckError extends PrefixedFirebaseError {\n  constructor(code: AppCheckErrorCode, message: string) {\n    super('app-check', code, message);\n\n    /* tslint:disable:max-line-length */\n    // Set the prototype explicitly. See the following link for more details:\n    // https://github.com/Microsoft/TypeScript/wiki/Breaking-Changes#extending-built-ins-like-error-array-and-map-may-no-longer-work\n    /* tslint:enable:max-line-length */\n    (this as any).__proto__ = FirebaseAppCheckError.prototype;\n  }\n}\n\n// Path: src/app-check/token-generator.ts\n/*!\n * @license\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport * as validator from '../utils/validator';\nimport { toWebSafeBase64, transformMillisecondsToSecondsString } from '../utils';\nimport { CryptoSigner, CryptoSignerError, CryptoSignerErrorCode } from '../utils/crypto-signer';\nimport {\n  FirebaseAppCheckError,\n  AppCheckErrorCode,\n  APP_CHECK_ERROR_CODE_MAPPING,\n} from './app-check-api-client-internal';\nimport { AppCheckTokenOptions } from './app-check-api';\nimport { HttpError } from '../utils/api-request';\n\nconst ONE_MINUTE_IN_SECONDS = 60;\nconst ONE_MINUTE_IN_MILLIS = ONE_MINUTE_IN_SECONDS * 1000;\nconst ONE_DAY_IN_MILLIS = 24 * 60 * 60 * 1000;\n\n// Audience to use for Firebase App Check Custom tokens\nconst FIREBASE_APP_CHECK_AUDIENCE = 'https://firebaseappcheck.googleapis.com/google.firebase.appcheck.v1.TokenExchangeService';\n\n/**\n * Class for generating Firebase App Check tokens.\n *\n * @internal\n */\nexport class AppCheckTokenGenerator {\n\n  private readonly signer: CryptoSigner;\n\n  /**\n   * The AppCheckTokenGenerator class constructor.\n   *\n   * @param signer - The CryptoSigner instance for this token generator.\n   * @constructor\n   */\n  constructor(signer: CryptoSigner) {\n    if (!validator.isNonNullObject(signer)) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument',\n        'INTERNAL ASSERT: Must provide a CryptoSigner to use AppCheckTokenGenerator.');\n    }\n    this.signer = signer;\n  }\n\n  /**\n   * Creates a new custom token that can be exchanged to an App Check token.\n   *\n   * @param appId - The Application ID to use for the generated token.\n   *\n   * @returns A Promise fulfilled with a custom token signed with a service account key\n   * that can be exchanged to an App Check token.\n   */\n  public createCustomToken(appId: string, options?: AppCheckTokenOptions): Promise<string> {\n    if (!validator.isNonEmptyString(appId)) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument',\n        '`appId` must be a non-empty string.');\n    }\n    let customOptions = {};\n    if (typeof options !== 'undefined') {\n      customOptions = this.validateTokenOptions(options);\n    }\n    return this.signer.getAccountId().then((account) => {\n      const header = {\n        alg: this.signer.algorithm,\n        typ: 'JWT',\n      };\n      const iat = Math.floor(Date.now() / 1000);\n      const body = {\n        iss: account,\n        sub: account,\n        app_id: appId,\n        aud: FIREBASE_APP_CHECK_AUDIENCE,\n        exp: iat + (ONE_MINUTE_IN_SECONDS * 5),\n        iat,\n        ...customOptions,\n      };\n      const token = `${this.encodeSegment(header)}.${this.encodeSegment(body)}`;\n      return this.signer.sign(Buffer.from(token))\n        .then((signature) => {\n          return `${token}.${this.encodeSegment(signature)}`;\n        });\n    }).catch((err) => {\n      throw appCheckErrorFromCryptoSignerError(err);\n    });\n  }\n\n  private encodeSegment(segment: object | Buffer): string {\n    const buffer: Buffer = (segment instanceof Buffer) ? segment : Buffer.from(JSON.stringify(segment));\n    return toWebSafeBase64(buffer).replace(/=+$/, '');\n  }\n\n  /**\n   * Checks if a given `AppCheckTokenOptions` object is valid. If successful, returns an object with\n   * custom properties.\n   *\n   * @param options - An options object to be validated.\n   * @returns A custom object with ttl converted to protobuf Duration string format.\n   */\n  private validateTokenOptions(options: AppCheckTokenOptions): {[key: string]: any} {\n    if (!validator.isNonNullObject(options)) {\n      throw new FirebaseAppCheckError(\n        'invalid-argument',\n        'AppCheckTokenOptions must be a non-null object.');\n    }\n    if (typeof options.ttlMillis !== 'undefined') {\n      if (!validator.isNumber(options.ttlMillis)) {\n        throw new FirebaseAppCheckError('invalid-argument',\n          'ttlMillis must be a duration in milliseconds.');\n      }\n      // ttlMillis must be between 30 minutes and 7 days (inclusive)\n      if (options.ttlMillis < (ONE_MINUTE_IN_MILLIS * 30) || options.ttlMillis > (ONE_DAY_IN_MILLIS * 7)) {\n        throw new FirebaseAppCheckError(\n          'invalid-argument',\n          'ttlMillis must be a duration in milliseconds between 30 minutes and 7 days (inclusive).');\n      }\n      return { ttl: transformMillisecondsToSecondsString(options.ttlMillis) };\n    }\n    return {};\n  }\n}\n\n/**\n * Creates a new `FirebaseAppCheckError` by extracting the error code, message and other relevant\n * details from a `CryptoSignerError`.\n *\n * @param err - The Error to convert into a `FirebaseAppCheckError` error\n * @returns A Firebase App Check error that can be returned to the user.\n */\nexport \nfunction appCheckErrorFromCryptoSignerError(err: Error): Error {\n  if (!(err instanceof CryptoSignerError)) {\n    return err;\n  }\n  if (err.code === CryptoSignerErrorCode.SERVER_ERROR && validator.isNonNullObject(err.cause)) {\n    const httpError = err.cause as HttpError\n    const errorResponse = httpError.response.data;\n    if (errorResponse?.error) {\n      const status = errorResponse.error.status;\n      const description = errorResponse.error.message || JSON.stringify(httpError.response);\n\n      let code: AppCheckErrorCode = 'unknown-error';\n      if (status && status in APP_CHECK_ERROR_CODE_MAPPING) {\n        code = APP_CHECK_ERROR_CODE_MAPPING[status];\n      }\n      return new FirebaseAppCheckError(code,\n        `Error returned from server while signing a custom token: ${description}`\n      );\n    }\n    return new FirebaseAppCheckError('internal-error',\n      'Error returned from server: ' + JSON.stringify(errorResponse) + '.'\n    );\n  }\n  return new FirebaseAppCheckError(mapToAppCheckErrorCode(err.code), err.message);\n}\n\nfunction mapToAppCheckErrorCode(code: string): AppCheckErrorCode {\n  switch (code) {\n  case CryptoSignerErrorCode.INVALID_CREDENTIAL:\n    return 'invalid-credential';\n  case CryptoSignerErrorCode.INVALID_ARGUMENT:\n    return 'invalid-argument';\n  default:\n    return 'internal-error';\n  }\n}\n\n// Path: src/utils/jwt.ts\n/*!\n * Copyright 2021 Google Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport * as validator from './validator';\nimport * as jwt from 'jsonwebtoken';\nimport * as jwks from 'jwks-rsa';\nimport { HttpClient, HttpRequestConfig, HttpError } from '../utils/api-request';\nimport { Agent } from 'http';\n\nexport const ALGORITHM_RS256: jwt.Algorithm = 'RS256' as const;\n\n// `jsonwebtoken` converts errors from the `getKey` callback to its own `JsonWebTokenError` type\n// and prefixes the error message with the following. Use the prefix to identify errors thrown\n// from the key provider callback.\n// https://github.com/auth0/node-jsonwebtoken/blob/d71e383862fc735991fd2e759181480f066bf138/verify.js#L96\nconst JWT_CALLBACK_ERROR_PREFIX = 'error in secret or public key callback: ';\n\nconst NO_MATCHING_KID_ERROR_MESSAGE = 'no-matching-kid-error';\nconst NO_KID_IN_HEADER_ERROR_MESSAGE = 'no-kid-in-header-error';\n\nconst HOUR_IN_SECONDS = 3600;\n\nexport type Dictionary = { [key: string]: any }\n\nexport type DecodedToken = {\n  header: Dictionary;\n  payload: Dictionary;\n}\n\n...\n", "needle_token_start": 15065, "needle_token_end": 15366, "code_context_ntokens": 16010, "output": ["```typescript\n/**\n * Creates a new `FirebaseAppCheckError` by extracting the error code, message and other relevant\n * details from a `CryptoSignerError`.\n *\n * @param err - The Error to convert into a `FirebaseAppCheckError` error\n * @returns A Firebase App Check error that can be returned to the user.\n */\nexport function appCheckErrorFromCryptoSignerError(err: Error): Error {\n  if (!(err instanceof CryptoSignerError)) {\n    return err;\n  }\n  if (err.code === CryptoSignerErrorCode.SERVER_ERROR && validator.isNonNullObject(err.cause)) {\n    const httpError = err.cause as HttpError;\n    const errorResponse = httpError.response.data;\n    if (errorResponse?.error) {\n      const status = errorResponse.error.status;\n      const description = errorResponse.error.message || JSON.stringify(httpError.response);\n\n      let code: AppCheckErrorCode = 'unknown-error';\n      if (status && status in APP_CHECK_ERROR_CODE_MAPPING) {\n        code = APP_CHECK_ERROR_CODE_MAPPING[status];\n      }\n      return new FirebaseAppCheckError(code,\n        `Error returned from server while signing a custom token: ${description}`\n      );\n    }\n    return new FirebaseAppCheckError('internal-error',\n      'Error returned from server: ' + JSON.stringify(errorResponse) + '.'\n    );\n  }\n  return new FirebaseAppCheckError(mapToAppCheckErrorCode(err.code), err.message);\n}\n\nfunction mapToAppCheckErrorCode(code: string): AppCheckErrorCode {\n  switch (code) {\n  case CryptoSignerErrorCode.INVALID_CREDENTIAL:\n    return 'invalid-credential';\n  case CryptoSignerErrorCode.INVALID_ARGUMENT:\n    return 'invalid-argument';\n  default:\n    return 'internal-error';\n  }\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "yd", "language": "rust", "path": "src/bytes/tests.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to identify and extract a sequence of digit characters from a given byte stream.\n2. **Input**: The function takes a byte array as input.\n3. **Output**: It returns a tuple containing the remaining byte array after the digit sequence and the extracted sequence of digit characters as a byte array.\n4. **Procedure**: The function processes the input byte array to locate and capture a contiguous sequence of digit characters. It stops parsing when a non-digit character is encountered or the input ends.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "  );\n\n  fn esc2(i: &str) -> IResult<&str, String> {\n    escaped_transform(\n      alpha,\n      '&',\n      alt((value(\"\u00e8\", tag(\"egrave;\")), value(\"\u00e0\", tag(\"agrave;\")))),\n    )(i)\n  }\n  assert_eq!(esc2(\"ab&egrave;DEF;\"), Ok((\";\", String::from(\"ab\u00e8DEF\"))));\n  assert_eq!(\n    esc2(\"ab&egrave;D&agrave;EF;\"),\n    Ok((\";\", String::from(\"ab\u00e8D\u00e0EF\")))\n  );\n\n  fn esc3(i: &str) -> IResult<&str, String> {\n    escaped_transform(\n      alpha,\n      '\u241b',\n      alt((value(\"\\0\", tag(\"0\")), value(\"\\n\", tag(\"n\")))),\n    )(i)\n  }\n  assert_eq!(esc3(\"a\u241b0bc\u241bn\"), Ok((\"\", String::from(\"a\\0bc\\n\"))));\n}\n\n#[test]\nfn take_until_incomplete() {\n  use crate::bytes::streaming::take_until;\n  fn y(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    take_until(\"end\")(i)\n  }\n  assert_eq!(y(&b\"nd\"[..]), Err(Err::Incomplete(Needed::Unknown)));\n  assert_eq!(y(&b\"123\"[..]), Err(Err::Incomplete(Needed::Unknown)));\n  assert_eq!(y(&b\"123en\"[..]), Err(Err::Incomplete(Needed::Unknown)));\n}\n\n#[test]\nfn take_until_incomplete_s() {\n  use crate::bytes::streaming::take_until;\n  fn ys(i: &str) -> IResult<&str, &str> {\n    take_until(\"end\")(i)\n  }\n  assert_eq!(ys(\"123en\"), Err(Err::Incomplete(Needed::Unknown)));\n}\n\n#[test]\nfn recognize() {\n  use crate::bytes::streaming::{tag, take};\n  use crate::combinator::recognize;\n  use crate::sequence::delimited;\n\n  fn x(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(delimited(tag(\"<!--\"), take(5_usize), tag(\"-->\"))).parse(i)\n  }\n  let r = x(&b\"<!-- abc --> aaa\"[..]);\n  assert_eq!(r, Ok((&b\" aaa\"[..], &b\"<!-- abc -->\"[..])));\n\n  let semicolon = &b\";\"[..];\n\n  fn ya(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(alpha).parse(i)\n  }\n  let ra = ya(&b\"abc;\"[..]);\n  assert_eq!(ra, Ok((semicolon, &b\"abc\"[..])));\n\n  fn yd(i: &[u8]) -> I\nResult<&[u8], &[u8]> {\n    recognize(digit).parse(i)\n  }\n  let rd = yd(&b\"123;\"[..]);\n  assert_eq!(rd, Ok((semicolon, &b\"123\"[..])));\n\n  fn yhd(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(hex_digit).parse(i)\n  }\n  let rhd = yhd(&b\"123abcDEF;\"[..]);\n  assert_eq!(rhd, Ok((semicolon, &b\"123abcDEF\"[..])));\n\n  fn yod(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(oct_digit).parse(i)\n  }\n  let rod = yod(&b\"1234567;\"[..]);\n  assert_eq!(rod, Ok((semicolon, &b\"1234567\"[..])));\n\n  fn ybd(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(bin_digit).parse(i)\n  }\n  let rbd = ybd(&b\"101010;\"[..]);\n  assert_eq!(rbd, Ok((semicolon, &b\"101010\"[..])));\n\n  fn yan(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(alphanumeric).parse(i)\n  }\n  let ran = yan(&b\"123abc;\"[..]);\n  assert_eq!(ran, Ok((semicolon, &b\"123abc\"[..])));\n\n  fn ys(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(space).parse(i)\n  }\n  let rs = ys(&b\" \\t;\"[..]);\n  assert_eq!(rs, Ok((semicolon, &b\" \\t\"[..])));\n\n  fn yms(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(multispace).parse(i)\n  }\n  let rms = yms(&b\" \\t\\r\\n;\"[..]);\n  assert_eq!(rms, Ok((semicolon, &b\" \\t\\r\\n\"[..])));\n}\n\n#[test]\nfn take_while() {\n  use crate::bytes::streaming::take_while;\n\n  fn f(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    take_while(is_alphabetic)(i)\n  }\n  let a = b\"\";\n  let b = b\"abcd\";\n  let c = b\"abcd123\";\n  let d = b\"123\";\n\n  assert_eq!(f(&a[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(&b[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(&c[..]), Ok((&d[..], &b[..])));\n  assert_eq!(f(&d[..]), Ok((&d[..], &a[..])));\n}\n\n#[test]\nfn take_while1() {\n  use crate::bytes::streaming::take_while1;\n\n  fn f(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    take_while1(is_alphabetic)(i)\n  }\n  let a = b\"\";\n  let b = b\"abcd\";\n  let c = b\"abcd123\";\n  let d = b\"123\";\n\n  assert_eq!(f(&a[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(&b[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(&c[..]), Ok((&b\"123\"[..], &b[..])));\n  assert_eq!(\n    f(&d[..]),\n    Err(Err::Error(error_position!(&d[..], ErrorKind::TakeWhile1)))\n  );\n}\n\n#[test]\nfn take_while_m_n() {\n  use crate::bytes::streaming::take_while_m_n;\n\n  fn x(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    take_while_m_n(2, 4, is_alphabetic)(i)\n  }\n  let a = b\"\";\n  let b = b\"a\";\n  let c = b\"abc\";\n  let d = b\"abc123\";\n  let e = b\"abcde\";\n  let f = b\"123\";\n\n  assert_eq!(x(&a[..]), Err(Err::Incomplete(Needed::new(2))));\n  assert_eq!(x(&b[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(x(&c[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(x(&d[..]), Ok((&b\"123\"[..], &b\"abc\"[..])));\n  assert_eq!(x(&e[..]), Ok((&b\"e\"[..], &b\"abcd\"[..])));\n  assert_eq!(\n    x(&f[..]),\n    Err(Err::Error(error_position!(&f[..], ErrorKind::TakeWhileMN)))\n  );\n}\n\n#[test]\nfn take_till() {\n  use crate::bytes::streaming::take_till;\n\n  fn f(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    take_till(is_alphabetic)(i)\n  }\n  let a = b\"\";\n  let b = b\"abcd\";\n  let c = b\"123abcd\";\n  let d = b\"123\";\n\n  assert_eq!(f(&a[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(&b[..]), Ok((&b\"abcd\"[..], &b\"\"[..])));\n  assert_eq!(f(&c[..]), Ok((&b\"abcd\"[..], &b\"123\"[..])));\n  assert_eq!(f(&d[..]), Err(Err::Incomplete(Needed::new(1))));\n}\n\n#[test]\nfn take_till1() {\n  use crate::bytes::streaming::take_till1;\n\n  fn f(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    take_till1(is_alphabetic)(i)\n  }\n  let a = b\"\";\n  let b = b\"abcd\";\n  let c = b\"123abcd\";\n  let d = b\"123\";\n\n  assert_eq!(f(&a[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(\n    f(&b[..]),\n    Err(Err::Error(error_position!(&b[..], ErrorKind::TakeTill1)))\n  );\n  assert_eq!(f(&c[..]), Ok((&b\"abcd\"[..], &b\"123\"[..])));\n  assert_eq!(f(&d[..]), Err(Err::Incomplete(Needed::new(1))));\n}\n\n#[test]\nfn take_while_utf8() {\n  use crate::bytes::streaming::take_while;\n\n  fn f(i: &str) -> IResult<&str, &str> {\n    take_while(|c| c != '\u9ede')(i)\n  }\n\n  assert_eq!(f(\"\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(\"abcd\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(\"abcd\u9ede\"), Ok((\"\u9ede\", \"abcd\")));\n  assert_eq!(f(\"abcd\u9edea\"), Ok((\"\u9edea\", \"abcd\")));\n\n  fn g(i: &str) -> IResult<&str, &str> {\n    take_while(|c| c == '\u9ede')(i)\n  }\n\n  assert_eq!(g(\"\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(g(\"\u9edeabcd\"), Ok((\"abcd\", \"\u9ede\")));\n  assert_eq!(g(\"\u9ede\u9ede\u9edea\"), Ok((\"a\", \"\u9ede\u9ede\u9ede\")));\n}\n\n#[test]\nfn take_till_utf8() {\n  use crate::bytes::streaming::take_till;\n\n  fn f(i: &str) -> IResult<&str, &str> {\n    take_till(|c| c == '\u9ede')(i)\n  }\n\n  assert_eq!(f(\"\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(\"abcd\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(f(\"abcd\u9ede\"), Ok((\"\u9ede\", \"abcd\")));\n  assert_eq!(f(\"abcd\u9edea\"), Ok((\"\u9edea\", \"abcd\")));\n\n  fn g(i: &str) -> IResult<&str, &str> {\n    take_till(|c| c != '\u9ede')(i)\n  }\n\n  assert_eq!(g(\"\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(g(\"\u9edeabcd\"), Ok((\"abcd\", \"\u9ede\")));\n  assert_eq!(g(\"\u9ede\u9ede\u9edea\"), Ok((\"a\", \"\u9ede\u9ede\u9ede\")));\n}\n\n#[test]\nfn take_utf8() {\n  use crate::bytes::streaming::{take, take_while};\n\n  fn f(i: &str) -> IResult<&str, &str> {\n    take(3_usize)(i)\n  }\n\n  assert_eq!(f(\"\"), Err(Err::Incomplete(Needed::Unknown)));\n  assert_eq!(f(\"ab\"), Err(Err::Incomplete(Needed::Unknown)));\n  assert_eq!(f(\"\u9ede\"), Err(Err::Incomplete(Needed::Unknown)));\n  assert_eq!(f(\"ab\u9edecd\"), Ok((\"cd\", \"ab\u9ede\")));\n  assert_eq!(f(\"a\u9edebcd\"), Ok((\"cd\", \"a\u9edeb\")));\n  assert_eq!(f(\"a\u9edeb\"), Ok((\"\", \"a\u9edeb\")));\n\n  fn g(i: &str) -> IResult<&str, &str> {\n    take_while(|c| c == '\u9ede')(i)\n  }\n\n  assert_eq!(g(\"\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(g(\"\u9edeabcd\"), Ok((\"abcd\", \"\u9ede\")));\n  assert_eq!(g(\"\u9ede\u9ede\u9edea\"), Ok((\"a\", \"\u9ede\u9ede\u9ede\")));\n}\n\n#[test]\nfn take_while_m_n_utf8() {\n  use crate::bytes::streaming::take_while_m_n;\n\n  fn parser(i: &str) -> IResult<&str, &str> {\n    take_while_m_n(1, 1, |c| c == 'A' || c == '\ud83d\ude03')(i)\n  }\n  assert_eq!(parser(\"A!\"), Ok((\"!\", \"A\")));\n  assert_eq!(parser(\"\ud83d\ude03!\"), Ok((\"!\", \"\ud83d\ude03\")));\n}\n\n#[test]\nfn take_while_m_n_utf8_full_match() {\n  use crate::bytes::streaming::take_while_m_n;\n\n  fn parser(i: &str) -> IResult<&str, &str> {\n    take_while_m_n(1, 1, |c: char| c.is_alphabetic())(i)\n  }\n  assert_eq!(parser(\"\u00f8n\"), Ok((\"n\", \"\u00f8\")));\n}\n\n#[test]\n#[cfg(feature = \"std\")]\nfn recognize_take_while() {\n  use crate::bytes::streaming::take_while;\n  use crate::character::is_alphanumeric;\n  use crate::combinator::recognize;\n\n  fn x(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    take_while(is_alphanumeric)(i)\n  }\n  fn y(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    recognize(x).parse(i)\n  }\n  assert_eq!(x(&b\"ab.\"[..]), Ok((&b\".\"[..], &b\"ab\"[..])));\n  println!(\"X: {:?}\", x(&b\"ab\"[..]));\n  assert_eq!(y(&b\"ab.\"[..]), Ok((&b\".\"[..], &b\"ab\"[..])));\n}\n\n#[test]\nfn length_bytes() {\n  use crate::{bytes::streaming::tag, multi::length_data, number::streaming::le_u8};\n\n  fn x(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    length_data(le_u8).parse(i)\n  }\n  assert_eq!(x(b\"\\x02..>>\"), Ok((&b\">>\"[..], &b\"..\"[..])));\n  assert_eq!(x(b\"\\x02..\"), Ok((&[][..], &b\"..\"[..])));\n  assert_eq!(x(b\"\\x02.\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(x(b\"\\x02\"), Err(Err::Incomplete(Needed::new(2))));\n\n  fn y(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    let (i, _) = tag(\"magic\")(i)?;\n    length_data(le_u8).parse(i)\n  }\n  assert_eq!(y(b\"magic\\x02..>>\"), Ok((&b\">>\"[..], &b\"..\"[..])));\n  assert_eq!(y(b\"magic\\x02..\"), Ok((&[][..], &b\"..\"[..])));\n  assert_eq!(y(b\"magic\\x02.\"), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(y(b\"magic\\x02\"), Err(Err::Incomplete(Needed::new(2))));\n}\n\n#[cfg(feature = \"alloc\")]\n#[test]\nfn case_insensitive() {\n  use crate::bytes::streaming::tag_no_case;\n\n  fn test(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    tag_no_case(\"ABcd\")(i)\n  }\n  assert_eq!(test(&b\"aBCdefgh\"[..]), Ok((&b\"efgh\"[..], &b\"aBCd\"[..])));\n  assert_eq!(test(&b\"abcdefgh\"[..]), Ok((&b\"efgh\"[..], &b\"abcd\"[..])));\n  assert_eq!(test(&b\"ABCDefgh\"[..]), Ok((&b\"efgh\"[..], &b\"ABCD\"[..])));\n  assert_eq!(test(&b\"ab\"[..]), Err(Err::Incomplete(Needed::new(2))));\n  assert_eq!(\n    test(&b\"Hello\"[..]),\n    Err(Err::Error(error_position!(&b\"Hello\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    test(&b\"Hel\"[..]),\n    Err(Err::Error(error_position!(&b\"Hel\"[..], ErrorKind::Tag)))\n  );\n\n  fn test2(i: &str) -> IResult<&str, &str> {\n    tag_no_case(\"ABcd\")(i)\n  }\n  assert_eq!(test2(\"aBCdefgh\"), Ok((\"efgh\", \"aBCd\")));\n  assert_eq!(test2(\"abcdefgh\"), Ok((\"efgh\", \"abcd\")));\n  assert_eq!(test2(\"ABCDefgh\"), Ok((\"efgh\", \"ABCD\")));\n  assert_eq!(test2(\"ab\"), Err(Err::Incomplete(Needed::new(2))));\n  assert_eq!(\n    test2(\"Hello\"),\n    Err(Err::Error(error_position!(\"Hello\", ErrorKind::Tag)))\n  );\n  assert_eq!(\n    test2(\"Hel\"),\n    Err(Err::Error(error_position!(\"Hel\", ErrorKind::Tag)))\n  );\n}\n\n#[test]\nfn tag_fixed_size_array() {\n  use crate::bytes::streaming::tag;\n\n  fn test(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    tag([0x42])(i)\n  }\n  fn test2(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    tag(&[0x42])(i)\n  }\n  let input = [0x42, 0x00];\n  assert_eq!(test(&input), Ok((&b\"\\x00\"[..], &b\"\\x42\"[..])));\n  assert_eq!(test2(&input), Ok((&b\"\\x00\"[..], &b\"\\x42\"[..])));\n}\n\n// Path: src/bytes/mod.rs\n//! Parsers recognizing bytes streams\n\npub mod complete;\npub mod streaming;\n#[cfg(test)]\nmod tests;\n\nuse core::marker::PhantomData;\n\nuse crate::error::ErrorKind;\nuse crate::error::ParseError;\nuse crate::internal::{Err, Needed, Parser};\nuse crate::lib::std::result::Result::*;\nuse crate::traits::{Compare, CompareResult, InputLength};\nuse crate::AsChar;\nuse crate::Check;\nuse crate::ExtendInto;\nuse crate::FindSubstring;\nuse crate::FindToken;\nuse crate::Input;\nuse crate::IsStreaming;\nuse crate::Mode;\nuse crate::OutputM;\nuse crate::OutputMode;\nuse crate::ToUsize;\n\n/// Recognizes a pattern.\n///\n/// The input data will be compared to the tag combinator's argument and will return the part of\n/// the input that matches the argument.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::streaming::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, &str> {\n///   tag(\"Hello\")(s)\n/// }\n///\n/// assert_eq!(parser(\"Hello, World!\"), Ok((\", World!\", \"Hello\")));\n/// assert_eq!(parser(\"Something\"), Err(Err::Error(Error::new(\"Something\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"S\"), Err(Err::Error(Error::new(\"S\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"H\"), Err(Err::Incomplete(Needed::new(4))));\n/// ```\npub fn tag<T, I, Error: ParseError<I>>(tag: T) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input + Compare<T>,\n  T: InputLength + Clone,\n{\n  Tag {\n    tag,\n    e: PhantomData,\n  }\n}\n\n/// Tag implementation\npub struct Tag<T, E> {\n  tag: T,\n  e: PhantomData<E>,\n}\n\nimpl<I, Error: ParseError<I>, T> Parser<I> for Tag<T, Error>\nwhere\n  I: Input + Compare<T>,\n  T: InputLength + Clone,\n{\n  type Output = I;\n\n  type Error = Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let tag_len = self.tag.input_len();\n    let t = self.tag.clone();\n\n    match i.compare(t) {\n      CompareResult::Ok => Ok((i.take_from(tag_len), OM::Output::bind(|| i.take(tag_len)))),\n      CompareResult::Incomplete => {\n        if OM::Incomplete::is_streaming() {\n          Err(Err::Incomplete(Needed::new(tag_len - i.input_len())))\n        } else {\n          Err(Err::Error(OM::Error::bind(|| {\n            let e: ErrorKind = ErrorKind::Tag;\n            Error::from_error_kind(i, e)\n          })))\n        }\n      }\n      CompareResult::Error => Err(Err::Error(OM::Error::bind(|| {\n        let e: ErrorKind = ErrorKind::Tag;\n        Error::from_error_kind(i, e)\n      }))),\n    }\n  }\n}\n\n/// Recognizes a case insensitive pattern.\n///\n/// The input data will be compared to the tag combinator's argument and will return the part of\n/// the input that matches the argument with no regard to case.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::streaming::tag_no_case;\n///\n/// fn parser(s: &str) -> IResult<&str, &str> {\n///   tag_no_case(\"hello\")(s)\n/// }\n///\n/// assert_eq!(parser(\"Hello, World!\"), Ok((\", World!\", \"Hello\")));\n/// assert_eq!(parser(\"hello, World!\"), Ok((\", World!\", \"hello\")));\n/// assert_eq!(parser(\"HeLlO, World!\"), Ok((\", World!\", \"HeLlO\")));\n/// assert_eq!(parser(\"Something\"), Err(Err::Error(Error::new(\"Something\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Incomplete(Needed::new(5))));\n/// ```\npub fn tag_no_case<T, I, Error: ParseError<I>>(tag: T) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input + Compare<T>,\n  T: InputLength + Clone,\n{\n  TagNoCase {\n    tag,\n    e: PhantomData,\n  }\n}\n\n/// Case insensitive Tag implementation\npub struct TagNoCase<T, E> {\n  tag: T,\n  e: PhantomData<E>,\n}\n\nimpl<I, Error: ParseError<I>, T> Parser<I> for TagNoCase<T, Error>\nwhere\n  I: Input + Compare<T>,\n  T: InputLength + Clone,\n{\n  type Output = I;\n\n  type Error = Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let tag_len = self.tag.input_len();\n    let t = self.tag.clone();\n\n    match i.compare_no_case(t) {\n      CompareResult::Ok => Ok((i.take_from(tag_len), OM::Output::bind(|| i.take(tag_len)))),\n      CompareResult::Incomplete => {\n        if OM::Incomplete::is_streaming() {\n          Err(Err::Incomplete(Needed::new(tag_len - i.input_len())))\n        } else {\n          Err(Err::Error(OM::Error::bind(|| {\n            let e: ErrorKind = ErrorKind::Tag;\n            Error::from_error_kind(i, e)\n          })))\n        }\n      }\n      CompareResult::Error => Err(Err::Error(OM::Error::bind(|| {\n        let e: ErrorKind = ErrorKind::Tag;\n        Error::from_error_kind(i, e)\n      }))),\n    }\n  }\n}\n\n/// Parser wrapper for `split_at_position`\npub struct SplitPosition<F, E> {\n  predicate: F,\n  error: PhantomData<E>,\n}\n\nimpl<I, Error: ParseError<I>, F> Parser<I> for SplitPosition<F, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  type Output = I;\n\n  type Error = Error;\n\n  #[inline(always)]\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    i.split_at_position_mode::<OM, _, _>(|c| (self.predicate)(c))\n  }\n}\n\n/// Parser wrapper for `split_at_position1`\npub struct SplitPosition1<F, E> {\n  e: ErrorKind,\n  predicate: F,\n  error: PhantomData<E>,\n}\n\nimpl<I, Error: ParseError<I>, F> Parser<I> for SplitPosition1<F, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  type Output = I;\n\n  type Error = Error;\n\n  #[inline(always)]\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    i.split_at_position_mode1::<OM, _, _>(|c| (self.predicate)(c), self.e)\n  }\n}\n\n/// Parse till certain characters are met.\n///\n/// The parser will return the longest slice till one of the characters of the combinator's argument are met.\n///\n/// It doesn't consume the matched character.\n///\n/// It will return a `Err::Error((\"\", ErrorKind::IsNot))` if the pattern wasn't met.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::is_not;\n///\n/// fn not_space(s: &str) -> IResult<&str, &str> {\n///   is_not(\" \\t\\r\\n\")(s)\n/// }\n///\n/// assert_eq!(not_space(\"Hello, World!\"), Ok((\" World!\", \"Hello,\")));\n/// assert_eq!(not_space(\"Sometimes\\t\"), Ok((\"\\t\", \"Sometimes\")));\n/// assert_eq!(not_space(\"Nospace\"), Ok((\"\", \"Nospace\")));\n/// assert_eq!(not_space(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::IsNot))));\n/// ```\npub fn is_not<T, I, Error: ParseError<I>>(arr: T) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  T: FindToken<<I as Input>::Item>,\n{\n  SplitPosition1 {\n    e: ErrorKind::IsNot,\n    predicate: move |c| arr.find_token(c),\n    error: PhantomData,\n  }\n}\n\n/// Returns the longest slice of the matches the pattern.\n///\n/// The parser will return the longest slice consisting of the characters in provided in the\n/// combinator's argument.\n///\n/// It will return a `Err(Err::Error((_, ErrorKind::IsA)))` if the pattern wasn't met.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::is_a;\n///\n/// fn hex(s: &str) -> IResult<&str, &str> {\n///   is_a(\"1234567890ABCDEF\")(s)\n/// }\n///\n/// assert_eq!(hex(\"123 and voila\"), Ok((\" and voila\", \"123\")));\n/// assert_eq!(hex(\"DEADBEEF and others\"), Ok((\" and others\", \"DEADBEEF\")));\n/// assert_eq!(hex(\"BADBABEsomething\"), Ok((\"something\", \"BADBABE\")));\n/// assert_eq!(hex(\"D15EA5E\"), Ok((\"\", \"D15EA5E\")));\n/// assert_eq!(hex(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::IsA))));\n/// ```\npub fn is_a<T, I, Error: ParseError<I>>(arr: T) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  T: FindToken<<I as Input>::Item>,\n{\n  SplitPosition1 {\n    e: ErrorKind::IsA,\n    predicate: move |c| !arr.find_token(c),\n    error: PhantomData,\n  }\n}\n\n/// Returns the longest input slice (if any) that matches the predicate.\n///\n/// The parser will return the longest slice that matches the given predicate *(a function that\n/// takes the input and returns a bool)*.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// use nom::bytes::complete::take_while;\n/// use nom::character::is_alphabetic;\n///\n/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   take_while(is_alphabetic)(s)\n/// }\n///\n/// assert_eq!(alpha(b\"latin123\"), Ok((&b\"123\"[..], &b\"latin\"[..])));\n/// assert_eq!(alpha(b\"12345\"), Ok((&b\"12345\"[..], &b\"\"[..])));\n/// assert_eq!(alpha(b\"latin\"), Ok((&b\"\"[..], &b\"latin\"[..])));\n/// assert_eq!(alpha(b\"\"), Ok((&b\"\"[..], &b\"\"[..])));\n/// ```\npub fn take_while<F, I, Error: ParseError<I>>(cond: F) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  SplitPosition {\n    predicate: move |c| !cond(c),\n    error: PhantomData,\n  }\n}\n\n/// Returns the longest (at least 1) input slice that matches the predicate.\n///\n/// The parser will return the longest slice that matches the given predicate *(a function that\n/// takes the input and returns a bool)*.\n///\n/// It will return an `Err(Err::Error((_, ErrorKind::TakeWhile1)))` if the pattern wasn't met.\n///\n/// # Streaming Specific\n/// *Streaming version* will return a `Err::Incomplete(Needed::new(1))` or if the pattern reaches the end of the input.\n///\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::streaming::take_while1;\n/// use nom::character::is_alphabetic;\n///\n/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   take_while1(is_alphabetic)(s)\n/// }\n///\n/// assert_eq!(alpha(b\"latin123\"), Ok((&b\"123\"[..], &b\"latin\"[..])));\n/// assert_eq!(alpha(b\"latin\"), Err(Err::Incomplete(Needed::new(1))));\n/// assert_eq!(alpha(b\"12345\"), Err(Err::Error(Error::new(&b\"12345\"[..], ErrorKind::TakeWhile1))));\n/// ```\npub fn take_while1<F, I, Error: ParseError<I>>(cond: F) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  SplitPosition1 {\n    e: ErrorKind::TakeWhile1,\n    predicate: move |c| !cond(c),\n    error: PhantomData,\n  }\n}\n\n/// Returns the longest (m <= len <= n) input slice  that matches the predicate.\n///\n/// The parser will return the longest slice that matches the given predicate *(a function that\n/// takes the input and returns a bool)*.\n///\n/// It will return an `Err::Error((_, ErrorKind::TakeWhileMN))` if the pattern wasn't met.\n/// # Streaming Specific\n/// *Streaming version* will return a `Err::Incomplete(Needed::new(1))`  if the pattern reaches the end of the input or is too short.\n///\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::streaming::take_while_m_n;\n/// use nom::character::is_alphabetic;\n///\n/// fn short_alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   take_while_m_n(3, 6, is_alphabetic)(s)\n/// }\n///\n/// assert_eq!(short_alpha(b\"latin123\"), Ok((&b\"123\"[..], &b\"latin\"[..])));\n/// assert_eq!(short_alpha(b\"lengthy\"), Ok((&b\"y\"[..], &b\"length\"[..])));\n/// assert_eq!(short_alpha(b\"latin\"), Err(Err::Incomplete(Needed::new(1))));\n/// assert_eq!(short_alpha(b\"ed\"), Err(Err::Incomplete(Needed::new(1))));\n/// assert_eq!(short_alpha(b\"12345\"), Err(Err::Error(Error::new(&b\"12345\"[..], ErrorKind::TakeWhileMN))));\n/// ```\npub fn take_while_m_n<F, I, Error: ParseError<I>>(\n  m: usize,\n  n: usize,\n  predicate: F,\n) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  TakeWhileMN {\n    m,\n    n,\n    predicate,\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for [take_while_m_n]\npub struct TakeWhileMN<F, E> {\n  m: usize,\n  n: usize,\n  predicate: F,\n  e: PhantomData<E>,\n}\n\nimpl<I, Error: ParseError<I>, F> Parser<I> for TakeWhileMN<F, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  type Output = I;\n  type Error = Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n    for (i, (index, item)) in input.iter_indices().enumerate() {\n      if i == self.n {\n        return Ok((\n          input.take_from(index),\n          OM::Output::bind(|| input.take(index)),\n        ));\n      }\n\n      if !(self.predicate)(item) {\n        if i >= self.m {\n          return Ok((\n            input.take_from(index),\n            OM::Output::bind(|| input.take(index)),\n          ));\n        } else {\n          return Err(Err::Error(OM::Error::bind(|| {\n            Error::from_error_kind(input, ErrorKind::TakeWhileMN)\n          })));\n        }\n      }\n      count += 1;\n    }\n\n    let input_len = input.input_len();\n    if OM::Incomplete::is_streaming() {\n      let needed = if self.m > input_len {\n        self.m - input_len\n      } else {\n        1\n      };\n      Err(Err::Incomplete(Needed::new(needed)))\n    } else if count >= self.m {\n      Ok((\n        input.take_from(input_len),\n        OM::Output::bind(|| input.take(input_len)),\n      ))\n    } else {\n      Err(Err::Error(OM::Error::bind(|| {\n        Error::from_error_kind(input, ErrorKind::TakeWhileMN)\n      })))\n    }\n  }\n}\n\n/// Returns the longest input slice (if any) till a predicate is met.\n///\n/// The parser will return the longest slice till the given predicate *(a function that\n/// takes the input and returns a bool)*.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// use nom::bytes::complete::take_till;\n///\n/// fn till_colon(s: &str) -> IResult<&str, &str> {\n///   take_till(|c| c == ':')(s)\n/// }\n///\n/// assert_eq!(till_colon(\"latin:123\"), Ok((\":123\", \"latin\")));\n/// assert_eq!(till_colon(\":empty matched\"), Ok((\":empty matched\", \"\"))); //allowed\n/// assert_eq!(till_colon(\"12345\"), Ok((\"\", \"12345\")));\n/// assert_eq!(till_colon(\"\"), Ok((\"\", \"\")));\n/// ```\n#[allow(clippy::redundant_closure)]\npub fn take_till<F, I, Error: ParseError<I>>(cond: F) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  SplitPosition {\n    predicate: cond,\n    error: PhantomData,\n  }\n}\n\n/// Returns the longest (at least 1) input slice till a predicate is met.\n///\n/// The parser will return the longest slice till the given predicate *(a function that\n/// takes the input and returns a bool)*.\n///\n/// # Streaming Specific\n/// *Streaming version* will return a `Err::Incomplete(Needed::new(1))` if the match reaches the\n/// end of input or if there was not match.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::streaming::take_till1;\n///\n/// fn till_colon(s: &str) -> IResult<&str, &str> {\n///   take_till1(|c| c == ':')(s)\n/// }\n///\n/// assert_eq!(till_colon(\"latin:123\"), Ok((\":123\", \"latin\")));\n/// assert_eq!(till_colon(\":empty matched\"), Err(Err::Error(Error::new(\":empty matched\", ErrorKind::TakeTill1))));\n/// assert_eq!(till_colon(\"12345\"), Err(Err::Incomplete(Needed::new(1))));\n/// assert_eq!(till_colon(\"\"), Err(Err::Incomplete(Needed::new(1))));\n/// ```\n#[allow(clippy::redundant_closure)]\npub fn take_till1<F, I, Error: ParseError<I>>(cond: F) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  SplitPosition1 {\n    e: ErrorKind::TakeTill1,\n    predicate: cond,\n    error: PhantomData,\n  }\n}\n\n/// Returns an input slice containing the first N input elements (Input[..N]).\n///\n/// # Streaming Specific\n/// *Streaming version* if the input has less than N elements, `take` will\n/// return a `Err::Incomplete(Needed::new(M))` where M is the number of\n/// additional bytes the parser would need to succeed.\n/// It is well defined for `&[u8]` as the number of elements is the byte size,\n/// but for types like `&str`, we cannot know how many bytes correspond for\n/// the next few chars, so the result will be `Err::Incomplete(Needed::Unknown)`\n///\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// use nom::bytes::streaming::take;\n///\n/// fn take6(s: &str) -> IResult<&str, &str> {\n///   take(6usize)(s)\n/// }\n///\n/// assert_eq!(take6(\"1234567\"), Ok((\"7\", \"123456\")));\n/// assert_eq!(take6(\"things\"), Ok((\"\", \"things\")));\n/// assert_eq!(take6(\"short\"), Err(Err::Incomplete(Needed::Unknown)));\n/// ```\npub fn take<C, I, Error: ParseError<I>>(count: C) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input,\n  C: ToUsize,\n{\n  Take {\n    length: count.to_usize(),\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for [take]\npub struct Take<E> {\n  length: usize,\n  e: PhantomData<E>,\n}\n\nimpl<I, Error: ParseError<I>> Parser<I> for Take<Error>\nwhere\n  I: Input,\n{\n  type Output = I;\n  type Error = Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match i.slice_index(self.length) {\n      Err(needed) => {\n        if OM::Incomplete::is_streaming() {\n          Err(Err::Incomplete(needed))\n        } else {\n          Err(Err::Error(OM::Error::bind(|| {\n            let e: ErrorKind = ErrorKind::Eof;\n            Error::from_error_kind(i, e)\n          })))\n        }\n      }\n      Ok(index) => Ok((i.take_from(index), OM::Output::bind(|| i.take(index)))),\n    }\n  }\n}\n\n/// Returns the input slice up to the first occurrence of the pattern.\n///\n/// It doesn't consume the pattern.\n///\n/// # Streaming Specific\n/// *Streaming version* will return a `Err::Incomplete(Needed::new(N))` if the input doesn't\n/// contain the pattern or if the input is smaller than the pattern.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// use nom::bytes::streaming::take_until;\n///\n/// fn until_eof(s: &str) -> IResult<&str, &str> {\n///   take_until(\"eof\")(s)\n/// }\n///\n/// assert_eq!(until_eof(\"hello, worldeof\"), Ok((\"eof\", \"hello, world\")));\n/// assert_eq!(until_eof(\"hello, world\"), Err(Err::Incomplete(Needed::Unknown)));\n/// assert_eq!(until_eof(\"hello, worldeo\"), Err(Err::Incomplete(Needed::Unknown)));\n/// assert_eq!(until_eof(\"1eof2eof\"), Ok((\"eof2eof\", \"1\")));\n/// ```\npub fn take_until<T, I, Error: ParseError<I>>(tag: T) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input + FindSubstring<T>,\n  T: Clone,\n{\n  TakeUntil {\n    tag,\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for [take_until]\npub struct TakeUntil<T, E> {\n  tag: T,\n  e: PhantomData<E>,\n}\n\nimpl<I, T, Error: ParseError<I>> Parser<I> for TakeUntil<T, Error>\nwhere\n  I: Input + FindSubstring<T>,\n  T: Clone,\n{\n  type Output = I;\n  type Error = Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match i.find_substring(self.tag.clone()) {\n      None => {\n        if OM::Incomplete::is_streaming() {\n          Err(Err::Incomplete(Needed::Unknown))\n        } else {\n          Err(Err::Error(OM::Error::bind(|| {\n            let e: ErrorKind = ErrorKind::TakeUntil;\n            Error::from_error_kind(i, e)\n          })))\n        }\n      }\n      Some(index) => Ok((i.take_from(index), OM::Output::bind(|| i.take(index)))),\n    }\n  }\n}\n\n/// Returns the non empty input slice up to the first occurrence of the pattern.\n///\n/// It doesn't consume the pattern.\n///\n/// # Streaming Specific\n/// *Streaming version* will return a `Err::Incomplete(Needed::new(N))` if the input doesn't\n/// contain the pattern or if the input is smaller than the pattern.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::streaming::take_until1;\n///\n/// fn until_eof(s: &str) -> IResult<&str, &str> {\n///   take_until1(\"eof\")(s)\n/// }\n///\n/// assert_eq!(until_eof(\"hello, worldeof\"), Ok((\"eof\", \"hello, world\")));\n/// assert_eq!(until_eof(\"hello, world\"), Err(Err::Incomplete(Needed::Unknown)));\n/// assert_eq!(until_eof(\"hello, worldeo\"), Err(Err::Incomplete(Needed::Unknown)));\n/// assert_eq!(until_eof(\"1eof2eof\"), Ok((\"eof2eof\", \"1\")));\n/// assert_eq!(until_eof(\"eof\"),  Err(Err::Error(Error::new(\"eof\", ErrorKind::TakeUntil))));\n/// ```\npub fn take_until1<T, I, Error: ParseError<I>>(tag: T) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input + FindSubstring<T>,\n  T: Clone,\n{\n  TakeUntil1 {\n    tag,\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for take_until1\npub struct TakeUntil1<T, E> {\n  tag: T,\n  e: PhantomData<E>,\n}\n\nimpl<I, T, Error: ParseError<I>> Parser<I> for TakeUntil1<T, Error>\nwhere\n  I: Input + FindSubstring<T>,\n  T: Clone,\n{\n  type Output = I;\n  type Error = Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match i.find_substring(self.tag.clone()) {\n      None => {\n        if OM::Incomplete::is_streaming() {\n          Err(Err::Incomplete(Needed::Unknown))\n        } else {\n          Err(Err::Error(OM::Error::bind(|| {\n            let e: ErrorKind = ErrorKind::TakeUntil;\n            Error::from_error_kind(i, e)\n          })))\n        }\n      }\n      Some(0) => Err(Err::Error(OM::Error::bind(|| {\n        Error::from_error_kind(i, ErrorKind::TakeUntil)\n      }))),\n\n      Some(index) => Ok((i.take_from(index), OM::Output::bind(|| i.take(index)))),\n    }\n  }\n}\n\n/// Matches a byte string with escaped characters.\n///\n/// * The first argument matches the normal characters (it must not accept the control character)\n/// * The second argument is the control character (like `\\` in most languages)\n/// * The third argument matches the escaped characters\n/// # Example\n/// ```\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// # use nom::character::complete::digit1;\n/// use nom::bytes::streaming::escaped;\n/// use nom::character::streaming::one_of;\n///\n/// fn esc(s: &str) -> IResult<&str, &str> {\n///   escaped(digit1, '\\\\', one_of(\"\\\"n\\\\\"))(s)\n/// }\n///\n/// assert_eq!(esc(\"123;\"), Ok((\";\", \"123\")));\n/// assert_eq!(esc(\"12\\\\\\\"34;\"), Ok((\";\", \"12\\\\\\\"34\")));\n/// ```\n///\npub fn escaped<I, Error, F, G>(\n  normal: F,\n  control_char: char,\n  escapable: G,\n) -> impl Parser<I, Output = I, Error = Error>\nwhere\n  I: Input + Clone + crate::traits::Offset,\n  <I as Input>::Item: crate::traits::AsChar,\n  F: Parser<I, Error = Error>,\n  G: Parser<I, Error = Error>,\n  Error: ParseError<I>,\n{\n  Escaped {\n    normal,\n    escapable,\n    control_char,\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for [escaped]\npub struct Escaped<F, G, E> {\n  normal: F,\n  escapable: G,\n  control_char: char,\n  e: PhantomData<E>,\n}\n\nimpl<I, Error: ParseError<I>, F, G> Parser<I> for Escaped<F, G, Error>\nwhere\n  I: Input + Clone + crate::traits::Offset,\n  <I as Input>::Item: crate::traits::AsChar,\n  F: Parser<I, Error = Error>,\n  G: Parser<I, Error = Error>,\n  Error: ParseError<I>,\n{\n  type Output = I;\n  type Error = Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut i = input.clone();\n\n    while i.input_len() > 0 {\n      let current_len = i.input_len();\n\n      match self\n        .normal\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Ok((i2, _)) => {\n          if i2.input_len() == 0 {\n            if OM::Incomplete::is_streaming() {\n              return Err(Err::Incomplete(Needed::Unknown));\n            } else {\n              let index = input.input_len();\n              return Ok((\n                input.take_from(index),\n                OM::Output::bind(|| input.take(index)),\n              ));\n            }\n          } else if i2.input_len() == current_len {\n            let index = input.offset(&i2);\n            return Ok((\n              input.take_from(index),\n              OM::Output::bind(|| input.take(index)),\n            ));\n          } else {\n            i = i2;\n          }\n        }\n        Err(Err::Error(_)) => {\n          // unwrap() should be safe here since index < $i.input_len()\n          if i.iter_elements().next().unwrap().as_char() == self.control_char {\n            let next = self.control_char.len_utf8();\n            if next >= i.input_len() {\n              if OM::Incomplete::is_streaming() {\n                return Err(Err::Incomplete(Needed::new(1)));\n              } else {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  Error::from_error_kind(input, ErrorKind::Escaped)\n                })));\n              }\n            } else {\n              match self\n                .escapable\n                .process::<OutputM<Check, OM::Error, OM::Incomplete>>(i.take_from(next))\n              {\n                Ok((i2, _)) => {\n                  if i2.input_len() == 0 {\n                    if OM::Incomplete::is_streaming() {\n                      return Err(Err::Incomplete(Needed::Unknown));\n                    } else {\n                      let index = input.input_len();\n                      return Ok((\n                        input.take_from(index),\n                        OM::Output::bind(|| input.take(index)),\n                      ));\n                    }\n                  } else {\n                    i = i2;\n                  }\n                }\n                Err(e) => return Err(e),\n              }\n            }\n          } else {\n            let index = input.offset(&i);\n            if index == 0 {\n              return Err(Err::Error(OM::Error::bind(|| {\n                Error::from_error_kind(input, ErrorKind::Escaped)\n              })));\n            } else {\n              return Ok((\n                input.take_from(index),\n                OM::Output::bind(|| input.take(index)),\n              ));\n            }\n          }\n        }\n        Err(Err::Failure(e)) => {\n          return Err(Err::Failure(e));\n        }\n        Err(Err::Incomplete(i)) => {\n          return Err(Err::Incomplete(i));\n        }\n      }\n    }\n\n    if OM::Incomplete::is_streaming() {\n      Err(Err::Incomplete(Needed::Unknown))\n    } else {\n      let index = input.input_len();\n      Ok((\n        input.take_from(index),\n        OM::Output::bind(|| input.take(index)),\n      ))\n    }\n  }\n}\n\n/// Matches a byte string with escaped characters.\n///\n/// * The first argument matches the normal characters (it must not match the control character)\n/// * The second argument is the control character (like `\\` in most languages)\n/// * The third argument matches the escaped characters and transforms them\n///\n/// As an example, the chain `abc\\tdef` could be `abc    def` (it also consumes the control character)\n///\n/// ```\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// # use std::str::from_utf8;\n/// use nom::bytes::streaming::{escaped_transform, tag};\n/// use nom::character::streaming::alpha1;\n/// use nom::branch::alt;\n/// use nom::combinator::value;\n///\n/// fn parser(input: &str) -> IResult<&str, String> {\n///   escaped_transform(\n///     alpha1,\n///     '\\\\',\n///     alt((\n///       value(\"\\\\\", tag(\"\\\\\")),\n///       value(\"\\\"\", tag(\"\\\"\")),\n///       value(\"\\n\", tag(\"n\")),\n///     ))\n///   )(input)\n/// }\n///\n/// assert_eq!(parser(\"ab\\\\\\\"cd\\\"\"), Ok((\"\\\"\", String::from(\"ab\\\"cd\"))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn escaped_transform<I, Error, F, G, ExtendItem, Output>(\n  normal: F,\n  control_char: char,\n  transform: G,\n) -> impl Parser<I, Output = Output, Error = Error>\nwhere\n  I: Clone + crate::traits::Offset + Input,\n  I: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  <F as Parser<I>>::Output: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  <G as Parser<I>>::Output: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  <I as Input>::Item: crate::traits::AsChar,\n  F: Parser<I, Error = Error>,\n  G: Parser<I, Error = Error>,\n  Error: ParseError<I>,\n{\n  EscapedTransform {\n    normal,\n    control_char,\n    transform,\n    e: PhantomData,\n    extend: PhantomData,\n    o: PhantomData,\n  }\n}\n\n/// Parser implementation for [escaped_transform]\npub struct EscapedTransform<F, G, E, ExtendItem, Output> {\n  normal: F,\n  transform: G,\n  control_char: char,\n  e: PhantomData<E>,\n  extend: PhantomData<ExtendItem>,\n  o: PhantomData<Output>,\n}\n\nimpl<I, Error: ParseError<I>, F, G, ExtendItem, Output> Parser<I>\n  for EscapedTransform<F, G, Error, ExtendItem, Output>\nwhere\n  I: Clone + crate::traits::Offset + Input,\n  I: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  <F as Parser<I>>::Output: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  <G as Parser<I>>::Output: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  <I as Input>::Item: crate::traits::AsChar,\n  F: Parser<I, Error = Error>,\n  G: Parser<I, Error = Error>,\n  Error: ParseError<I>,\n{\n  type Output = Output;\n  type Error = Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut index = 0;\n    let mut res = OM::Output::bind(|| input.new_builder());\n\n    while index < input.input_len() {\n      let current_len = input.input_len();\n      let remainder = input.take_from(index);\n      match self.normal.process::<OM>(remainder.clone()) {\n        Ok((i2, o)) => {\n          res = OM::Output::combine(o, res, |o, mut res| {\n            o.extend_into(&mut res);\n            res\n          });\n          if i2.input_len() == 0 {\n            if OM::Incomplete::is_streaming() {\n              return Err(Err::Incomplete(Needed::Unknown));\n            } else {\n              let index = input.input_len();\n              return Ok((input.take_from(index), res));\n            }\n          } else if i2.input_len() == current_len {\n            return Ok((remainder, res));\n          } else {\n            index = input.offset(&i2);\n          }\n        }\n        Err(Err::Error(_)) => {\n          // unwrap() should be safe here since index < $i.input_len()\n          if remainder.iter_elements().next().unwrap().as_char() == self.control_char {\n            let next = index + self.control_char.len_utf8();\n            let input_len = input.input_len();\n\n            if next >= input_len {\n              if OM::Incomplete::is_streaming() {\n                return Err(Err::Incomplete(Needed::Unknown));\n              } else {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  Error::from_error_kind(remainder, ErrorKind::EscapedTransform)\n                })));\n              }\n            } else {\n              match self.transform.process::<OM>(input.take_from(next)) {\n                Ok((i2, o)) => {\n                  res = OM::Output::combine(o, res, |o, mut res| {\n                    o.extend_into(&mut res);\n                    res\n                  });\n                  if i2.input_len() == 0 {\n                    if OM::Incomplete::is_streaming() {\n                      return Err(Err::Incomplete(Needed::Unknown));\n                    } else {\n                      return Ok((input.take_from(input.input_len()), res));\n                    }\n                  } else {\n                    index = input.offset(&i2);\n                  }\n                }\n                Err(Err::Error(e)) => return Err(Err::Error(e)),\n                Err(Err::Failure(e)) => {\n                  return Err(Err::Failure(e));\n                }\n                Err(Err::Incomplete(i)) => {\n                  return Err(Err::Incomplete(i));\n                }\n              }\n...\n", "needle_token_start": 792, "needle_token_end": 832, "code_context_ntokens": 16288, "output": ["```rust\nfn digit(i: &[u8]) -> IResult<&[u8], &[u8]> {\n  take_while1(is_digit)(i)\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "i16_tests", "language": "rust", "path": "src/number/streaming.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function tests the parsing of 2-byte integers from byte arrays to ensure correct interpretation of both positive and negative values, as well as handling incomplete data.\n2. **Input**: The function takes arrays of bytes representing 16-bit integers in big-endian format.\n3. **Output**: The function returns a tuple containing the remaining unparsed bytes and the parsed 16-bit integer, or an error if the input is incomplete.\n4. **Procedure**: The function checks several cases including the parsing of zero, the maximum positive value, the minimum negative value, and scenarios where the input byte array is incomplete (either completely empty or missing bytes).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "/// data.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\npub fn recognize_float_parts<T, E: ParseError<T>>(input: T) -> IResult<T, (bool, T, T, i32), E>\nwhere\n  T: Clone + Offset,\n  T: Input,\n  <T as Input>::Item: AsChar,\n  T: for<'a> Compare<&'a [u8]>,\n  T: AsBytes,\n{\n  let (i, sign) = sign(input.clone())?;\n\n  //let (i, zeroes) = take_while(|c: <T as InputTakeAtPosition>::Item| c.as_char() == '0')(i)?;\n  let (i, zeroes) = match i.as_bytes().iter().position(|c| *c != b'0') {\n    Some(index) => i.take_split(index),\n    None => i.take_split(i.input_len()),\n  };\n\n  //let (i, mut integer) = digit0(i)?;\n  let (i, mut integer) = match i\n    .as_bytes()\n    .iter()\n    .position(|c| !(*c >= b'0' && *c <= b'9'))\n  {\n    Some(index) => i.take_split(index),\n    None => i.take_split(i.input_len()),\n  };\n\n  if integer.input_len() == 0 && zeroes.input_len() > 0 {\n    // keep the last zero if integer is empty\n    integer = zeroes.take_from(zeroes.input_len() - 1);\n  }\n\n  let (i, opt_dot) = opt(tag(&b\".\"[..])).parse(i)?;\n  let (i, fraction) = if opt_dot.is_none() {\n    let i2 = i.clone();\n    (i2, i.take(0))\n  } else {\n    // match number, trim right zeroes\n    let mut zero_count = 0usize;\n    let mut position = None;\n    for (pos, c) in i.as_bytes().iter().enumerate() {\n      if *c >= b'0' && *c <= b'9' {\n        if *c == b'0' {\n          zero_count += 1;\n        } else {\n          zero_count = 0;\n        }\n      } else {\n        position = Some(pos);\n        break;\n      }\n    }\n\n    let position = match position {\n      Some(p) => p,\n      None => return Err(Err::Incomplete(Needed::new(1))),\n    };\n\n    let index = if zero_count == 0 {\n      position\n    } else if zero_count == position {\n      position - zero_count + 1\n    } else {\n      position - zero_count\n    };\n\n    (i.take_from(position), i.take(index))\n  };\n\n  if integer.input_len() == 0 && fraction.input_len() == 0 {\n    return Err(Err::Error(E::from_error_kind(input, ErrorKind::Float)));\n  }\n\n  let i2 = i.clone();\n  let (i, e) = match i.as_bytes().iter().next() {\n    Some(b'e') => (i.take_from(1), true),\n    Some(b'E') => (i.take_from(1), true),\n    _ => (i, false),\n  };\n\n  let (i, exp) = if e {\n    cut(crate::character::streaming::i32).parse(i)?\n  } else {\n    (i2, 0)\n  };\n\n  Ok((i, (sign, integer, fraction, exp)))\n}\n\n/// Recognizes floating point number in text format and returns a f32.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::float;\n///\n/// let parser = |s| {\n///   float(s)\n/// };\n///\n/// assert_eq!(parser(\"11e-1\"), Ok((\"\", 1.1)));\n/// assert_eq!(parser(\"123E-02\"), Ok((\"\", 1.23)));\n/// assert_eq!(parser(\"123K-01\"), Ok((\"K-01\", 123.0)));\n/// assert_eq!(parser(\"abc\"), Err(Err::Error((\"abc\", ErrorKind::Float))));\n/// ```\npub fn float<T, E: ParseError<T>>(input: T) -> IResult<T, f32, E>\nwhere\n  T: Clone + Offset,\n  T: Input + crate::traits::ParseTo<f32> + Compare<&'static str>,\n  <T as Input>::Item: AsChar + Clone,\n  T: AsBytes,\n  T: for<'a> Compare<&'a [u8]>,\n{\n  /*\n  let (i, (sign, integer, fraction, exponent)) = recognize_float_parts(input)?;\n\n  let mut float: f32 = minimal_lexical::parse_float(\n    integer.as_bytes().iter(),\n    fraction.as_bytes().iter(),\n    exponent,\n  );\n  if !sign {\n    float = -float;\n  }\n\n  Ok((i, float))\n  */\n  let (i, s) = recognize_float_or_exceptions(input)?;\n  match s.parse_to() {\n    Some(f) => Ok((i, f)),\n    None => Err(crate::Err::Error(E::from_error_kind(\n      i,\n      crate::error::ErrorKind::Float,\n    ))),\n  }\n}\n\n/// Recognizes floating point number in text format and returns a f64.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::double;\n///\n/// let parser = |s| {\n///   double(s)\n/// };\n///\n/// assert_eq!(parser(\"11e-1\"), Ok((\"\", 1.1)));\n/// assert_eq!(parser(\"123E-02\"), Ok((\"\", 1.23)));\n/// assert_eq!(parser(\"123K-01\"), Ok((\"K-01\", 123.0)));\n/// assert_eq!(parser(\"abc\"), Err(Err::Error((\"abc\", ErrorKind::Float))));\n/// ```\npub fn double<T, E: ParseError<T>>(input: T) -> IResult<T, f64, E>\nwhere\n  T: Clone + Offset,\n  T: Input + crate::traits::ParseTo<f64> + Compare<&'static str>,\n  <T as Input>::Item: AsChar + Clone,\n  T: AsBytes,\n  T: for<'a> Compare<&'a [u8]>,\n{\n  /*\n  let (i, (sign, integer, fraction, exponent)) = recognize_float_parts(input)?;\n\n  let mut float: f64 = minimal_lexical::parse_float(\n    integer.as_bytes().iter(),\n    fraction.as_bytes().iter(),\n    exponent,\n  );\n  if !sign {\n    float = -float;\n  }\n\n  Ok((i, float))\n  */\n  let (i, s) = recognize_float_or_exceptions(input)?;\n  match s.parse_to() {\n    Some(f) => Ok((i, f)),\n    None => Err(crate::Err::Error(E::from_error_kind(\n      i,\n      crate::error::ErrorKind::Float,\n    ))),\n  }\n}\n\n#[cfg(test)]\nmod tests {\n  use super::*;\n  use crate::error::ErrorKind;\n  use crate::internal::{Err, Needed};\n  use proptest::prelude::*;\n\n  macro_rules! assert_parse(\n    ($left: expr, $right: expr) => {\n      let res: $crate::IResult<_, _, (_, ErrorKind)> = $left;\n      assert_eq!(res, $right);\n    };\n  );\n\n  #[test]\n  fn i8_tests() {\n    assert_parse!(be_i8(&[0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_i8(&[0x7f][..]), Ok((&b\"\"[..], 127)));\n    assert_parse!(be_i8(&[0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(be_i8(&[0x80][..]), Ok((&b\"\"[..], -128)));\n    assert_parse!(be_i8(&[][..]), Err(Err::Incomplete(Needed::new(1))));\n  }\n\n  #[test]\n  \nfn i16_tests() {\n    assert_parse!(be_i16(&[0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_i16(&[0x7f, 0xff][..]), Ok((&b\"\"[..], 32_767_i16)));\n    assert_parse!(be_i16(&[0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(be_i16(&[0x80, 0x00][..]), Ok((&b\"\"[..], -32_768_i16)));\n    assert_parse!(be_i16(&[][..]), Err(Err::Incomplete(Needed::new(2))));\n    assert_parse!(be_i16(&[0x00][..]), Err(Err::Incomplete(Needed::new(1))));\n  }\n\n  #[test]\n  fn u24_tests() {\n    assert_parse!(be_u24(&[0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_u24(&[0x00, 0xFF, 0xFF][..]), Ok((&b\"\"[..], 65_535_u32)));\n    assert_parse!(\n      be_u24(&[0x12, 0x34, 0x56][..]),\n      Ok((&b\"\"[..], 1_193_046_u32))\n    );\n    assert_parse!(be_u24(&[][..]), Err(Err::Incomplete(Needed::new(3))));\n    assert_parse!(be_u24(&[0x00][..]), Err(Err::Incomplete(Needed::new(2))));\n    assert_parse!(\n      be_u24(&[0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(1)))\n    );\n  }\n\n  #[test]\n  fn i24_tests() {\n    assert_parse!(be_i24(&[0xFF, 0xFF, 0xFF][..]), Ok((&b\"\"[..], -1_i32)));\n    assert_parse!(be_i24(&[0xFF, 0x00, 0x00][..]), Ok((&b\"\"[..], -65_536_i32)));\n    assert_parse!(\n      be_i24(&[0xED, 0xCB, 0xAA][..]),\n      Ok((&b\"\"[..], -1_193_046_i32))\n    );\n    assert_parse!(be_i24(&[][..]), Err(Err::Incomplete(Needed::new(3))));\n    assert_parse!(be_i24(&[0x00][..]), Err(Err::Incomplete(Needed::new(2))));\n    assert_parse!(\n      be_i24(&[0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(1)))\n    );\n  }\n\n  #[test]\n  fn i32_tests() {\n    assert_parse!(be_i32(&[0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(\n      be_i32(&[0x7f, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], 2_147_483_647_i32))\n    );\n    assert_parse!(be_i32(&[0xff, 0xff, 0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(\n      be_i32(&[0x80, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], -2_147_483_648_i32))\n    );\n    assert_parse!(be_i32(&[][..]), Err(Err::Incomplete(Needed::new(4))));\n    assert_parse!(be_i32(&[0x00][..]), Err(Err::Incomplete(Needed::new(3))));\n    assert_parse!(\n      be_i32(&[0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(2)))\n    );\n    assert_parse!(\n      be_i32(&[0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(1)))\n    );\n  }\n\n  #[test]\n  fn i64_tests() {\n    assert_parse!(\n      be_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      be_i64(&[0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], 9_223_372_036_854_775_807_i64))\n    );\n    assert_parse!(\n      be_i64(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      be_i64(&[0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], -9_223_372_036_854_775_808_i64))\n    );\n    assert_parse!(be_i64(&[][..]), Err(Err::Incomplete(Needed::new(8))));\n    assert_parse!(be_i64(&[0x00][..]), Err(Err::Incomplete(Needed::new(7))));\n    assert_parse!(\n      be_i64(&[0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(6)))\n    );\n    assert_parse!(\n      be_i64(&[0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(5)))\n    );\n    assert_parse!(\n      be_i64(&[0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(4)))\n    );\n    assert_parse!(\n      be_i64(&[0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(3)))\n    );\n    assert_parse!(\n      be_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(2)))\n    );\n    assert_parse!(\n      be_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(1)))\n    );\n  }\n\n  #[test]\n  fn i128_tests() {\n    assert_parse!(\n      be_i128(\n        &[\n          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x00\n        ][..]\n      ),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      be_i128(\n        &[\n          0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0xff\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        170_141_183_460_469_231_731_687_303_715_884_105_727_i128\n      ))\n    );\n    assert_parse!(\n      be_i128(\n        &[\n          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0xff\n        ][..]\n      ),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      be_i128(\n        &[\n          0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x00\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        -170_141_183_460_469_231_731_687_303_715_884_105_728_i128\n      ))\n    );\n    assert_parse!(be_i128(&[][..]), Err(Err::Incomplete(Needed::new(16))));\n    assert_parse!(be_i128(&[0x00][..]), Err(Err::Incomplete(Needed::new(15))));\n    assert_parse!(\n      be_i128(&[0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(14)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(13)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(12)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(11)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(10)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(9)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(8)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(7)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(6)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(5)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(4)))\n    );\n    assert_parse!(\n      be_i128(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Err(Err::Incomplete(Needed::new(3)))\n    );\n    assert_parse!(\n      be_i128(\n        &[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]\n      ),\n      Err(Err::Incomplete(Needed::new(2)))\n    );\n    assert_parse!(\n      be_i128(\n        &[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]\n          [..]\n      ),\n      Err(Err::Incomplete(Needed::new(1)))\n    );\n  }\n\n  #[test]\n  fn le_i8_tests() {\n    assert_parse!(le_i8(&[0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(le_i8(&[0x7f][..]), Ok((&b\"\"[..], 127)));\n    assert_parse!(le_i8(&[0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(le_i8(&[0x80][..]), Ok((&b\"\"[..], -128)));\n  }\n\n  #[test]\n  fn le_i16_tests() {\n    assert_parse!(le_i16(&[0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(le_i16(&[0xff, 0x7f][..]), Ok((&b\"\"[..], 32_767_i16)));\n    assert_parse!(le_i16(&[0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(le_i16(&[0x00, 0x80][..]), Ok((&b\"\"[..], -32_768_i16)));\n  }\n\n  #[test]\n  fn le_u24_tests() {\n    assert_parse!(le_u24(&[0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(le_u24(&[0xFF, 0xFF, 0x00][..]), Ok((&b\"\"[..], 65_535_u32)));\n    assert_parse!(\n      le_u24(&[0x56, 0x34, 0x12][..]),\n      Ok((&b\"\"[..], 1_193_046_u32))\n    );\n  }\n\n  #[test]\n  fn le_i24_tests() {\n    assert_parse!(le_i24(&[0xFF, 0xFF, 0xFF][..]), Ok((&b\"\"[..], -1_i32)));\n    assert_parse!(le_i24(&[0x00, 0x00, 0xFF][..]), Ok((&b\"\"[..], -65_536_i32)));\n    assert_parse!(\n      le_i24(&[0xAA, 0xCB, 0xED][..]),\n      Ok((&b\"\"[..], -1_193_046_i32))\n    );\n  }\n\n  #[test]\n  fn le_i32_tests() {\n    assert_parse!(le_i32(&[0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(\n      le_i32(&[0xff, 0xff, 0xff, 0x7f][..]),\n      Ok((&b\"\"[..], 2_147_483_647_i32))\n    );\n    assert_parse!(le_i32(&[0xff, 0xff, 0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(\n      le_i32(&[0x00, 0x00, 0x00, 0x80][..]),\n      Ok((&b\"\"[..], -2_147_483_648_i32))\n    );\n  }\n\n  #[test]\n  fn le_i64_tests() {\n    assert_parse!(\n      le_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      le_i64(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f][..]),\n      Ok((&b\"\"[..], 9_223_372_036_854_775_807_i64))\n    );\n    assert_parse!(\n      le_i64(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      le_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80][..]),\n      Ok((&b\"\"[..], -9_223_372_036_854_775_808_i64))\n    );\n  }\n\n  #[test]\n  fn le_i128_tests() {\n    assert_parse!(\n      le_i128(\n        &[\n          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x00\n        ][..]\n      ),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      le_i128(\n        &[\n          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0x7f\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        170_141_183_460_469_231_731_687_303_715_884_105_727_i128\n      ))\n    );\n    assert_parse!(\n      le_i128(\n        &[\n          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0xff\n        ][..]\n      ),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      le_i128(\n        &[\n          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x80\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        -170_141_183_460_469_231_731_687_303_715_884_105_728_i128\n      ))\n    );\n  }\n\n  #[test]\n  fn be_f32_tests() {\n    assert_parse!(be_f32(&[0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0_f32)));\n    assert_parse!(\n      be_f32(&[0x4d, 0x31, 0x1f, 0xd8][..]),\n      Ok((&b\"\"[..], 185_728_392_f32))\n    );\n  }\n\n  #[test]\n  fn be_f64_tests() {\n    assert_parse!(\n      be_f64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], 0_f64))\n    );\n    assert_parse!(\n      be_f64(&[0x41, 0xa6, 0x23, 0xfb, 0x10, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], 185_728_392_f64))\n    );\n  }\n\n  #[test]\n  fn le_f32_tests() {\n    assert_parse!(le_f32(&[0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0_f32)));\n    assert_parse!(\n      le_f32(&[0xd8, 0x1f, 0x31, 0x4d][..]),\n      Ok((&b\"\"[..], 185_728_392_f32))\n    );\n  }\n\n  #[test]\n  fn le_f64_tests() {\n    assert_parse!(\n      le_f64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], 0_f64))\n    );\n    assert_parse!(\n      le_f64(&[0x00, 0x00, 0x00, 0x10, 0xfb, 0x23, 0xa6, 0x41][..]),\n      Ok((&b\"\"[..], 185_728_392_f64))\n    );\n  }\n\n  #[test]\n  fn hex_u32_tests() {\n    assert_parse!(\n      hex_u32(&b\";\"[..]),\n      Err(Err::Error(error_position!(&b\";\"[..], ErrorKind::IsA)))\n    );\n    assert_parse!(hex_u32(&b\"ff;\"[..]), Ok((&b\";\"[..], 255)));\n    assert_parse!(hex_u32(&b\"1be2;\"[..]), Ok((&b\";\"[..], 7138)));\n    assert_parse!(hex_u32(&b\"c5a31be2;\"[..]), Ok((&b\";\"[..], 3_315_801_058)));\n    assert_parse!(hex_u32(&b\"C5A31be2;\"[..]), Ok((&b\";\"[..], 3_315_801_058)));\n    assert_parse!(hex_u32(&b\"00c5a31be2;\"[..]), Ok((&b\"e2;\"[..], 12_952_347)));\n    assert_parse!(\n      hex_u32(&b\"c5a31be201;\"[..]),\n      Ok((&b\"01;\"[..], 3_315_801_058))\n    );\n    assert_parse!(hex_u32(&b\"ffffffff;\"[..]), Ok((&b\";\"[..], 4_294_967_295)));\n    assert_parse!(hex_u32(&b\"0x1be2;\"[..]), Ok((&b\"x1be2;\"[..], 0)));\n    assert_parse!(hex_u32(&b\"12af\"[..]), Err(Err::Incomplete(Needed::new(1))));\n  }\n\n  #[test]\n  #[cfg(feature = \"std\")]\n  fn float_test() {\n    let mut test_cases = vec![\n      \"+3.14\",\n      \"3.14\",\n      \"-3.14\",\n      \"0\",\n      \"0.0\",\n      \"1.\",\n      \".789\",\n      \"-.5\",\n      \"1e7\",\n      \"-1E-7\",\n      \".3e-2\",\n      \"1.e4\",\n      \"1.2e4\",\n      \"12.34\",\n      \"-1.234E-12\",\n      \"-1.234e-12\",\n      \"0.00000000000000000087\",\n    ];\n\n    for test in test_cases.drain(..) {\n      let expected32 = str::parse::<f32>(test).unwrap();\n      let expected64 = str::parse::<f64>(test).unwrap();\n\n      println!(\"now parsing: {} -> {}\", test, expected32);\n\n      let larger = format!(\"{};\", test);\n      assert_parse!(recognize_float(&larger[..]), Ok((\";\", test)));\n\n      assert_parse!(float(larger.as_bytes()), Ok((&b\";\"[..], expected32)));\n      assert_parse!(float(&larger[..]), Ok((\";\", expected32)));\n\n      assert_parse!(double(larger.as_bytes()), Ok((&b\";\"[..], expected64)));\n      assert_parse!(double(&larger[..]), Ok((\";\", expected64)));\n    }\n\n    let remaining_exponent = \"-1.234E-\";\n    assert_parse!(\n      recognize_float(remaining_exponent),\n      Err(Err::Incomplete(Needed::new(1)))\n    );\n\n    let (_i, nan) = float::<_, ()>(\"NaN\").unwrap();\n    assert!(nan.is_nan());\n\n    let (_i, inf) = float::<_, ()>(\"inf\").unwrap();\n    assert!(inf.is_infinite());\n    let (i, inf) = float::<_, ()>(\"infinity\").unwrap();\n    assert!(inf.is_infinite());\n    assert!(i.is_empty());\n  }\n\n  #[test]\n  fn configurable_endianness() {\n    use crate::number::Endianness;\n\n    fn be_tst16(i: &[u8]) -> IResult<&[u8], u16> {\n      u16(Endianness::Big)(i)\n    }\n    fn le_tst16(i: &[u8]) -> IResult<&[u8], u16> {\n      u16(Endianness::Little)(i)\n    }\n    assert_eq!(be_tst16(&[0x80, 0x00]), Ok((&b\"\"[..], 32_768_u16)));\n    assert_eq!(le_tst16(&[0x80, 0x00]), Ok((&b\"\"[..], 128_u16)));\n\n    fn be_tst32(i: &[u8]) -> IResult<&[u8], u32> {\n      u32(Endianness::Big)(i)\n    }\n    fn le_tst32(i: &[u8]) -> IResult<&[u8], u32> {\n      u32(Endianness::Little)(i)\n    }\n    assert_eq!(\n      be_tst32(&[0x12, 0x00, 0x60, 0x00]),\n      Ok((&b\"\"[..], 302_014_464_u32))\n    );\n    assert_eq!(\n      le_tst32(&[0x12, 0x00, 0x60, 0x00]),\n      Ok((&b\"\"[..], 6_291_474_u32))\n    );\n\n    fn be_tst64(i: &[u8]) -> IResult<&[u8], u64> {\n      u64(Endianness::Big)(i)\n    }\n    fn le_tst64(i: &[u8]) -> IResult<&[u8], u64> {\n      u64(Endianness::Little)(i)\n    }\n    assert_eq!(\n      be_tst64(&[0x12, 0x00, 0x60, 0x00, 0x12, 0x00, 0x80, 0x00]),\n      Ok((&b\"\"[..], 1_297_142_246_100_992_000_u64))\n    );\n    assert_eq!(\n      le_tst64(&[0x12, 0x00, 0x60, 0x00, 0x12, 0x00, 0x80, 0x00]),\n      Ok((&b\"\"[..], 36_028_874_334_666_770_u64))\n    );\n\n    fn be_tsti16(i: &[u8]) -> IResult<&[u8], i16> {\n      i16(Endianness::Big)(i)\n    }\n    fn le_tsti16(i: &[u8]) -> IResult<&[u8], i16> {\n      i16(Endianness::Little)(i)\n    }\n    assert_eq!(be_tsti16(&[0x00, 0x80]), Ok((&b\"\"[..], 128_i16)));\n    assert_eq!(le_tsti16(&[0x00, 0x80]), Ok((&b\"\"[..], -32_768_i16)));\n\n    fn be_tsti32(i: &[u8]) -> IResult<&[u8], i32> {\n      i32(Endianness::Big)(i)\n    }\n    fn le_tsti32(i: &[u8]) -> IResult<&[u8], i32> {\n      i32(Endianness::Little)(i)\n    }\n    assert_eq!(\n      be_tsti32(&[0x00, 0x12, 0x60, 0x00]),\n      Ok((&b\"\"[..], 1_204_224_i32))\n    );\n    assert_eq!(\n      le_tsti32(&[0x00, 0x12, 0x60, 0x00]),\n      Ok((&b\"\"[..], 6_296_064_i32))\n    );\n\n    fn be_tsti64(i: &[u8]) -> IResult<&[u8], i64> {\n      i64(Endianness::Big)(i)\n    }\n    fn le_tsti64(i: &[u8]) -> IResult<&[u8], i64> {\n      i64(Endianness::Little)(i)\n    }\n    assert_eq!(\n      be_tsti64(&[0x00, 0xFF, 0x60, 0x00, 0x12, 0x00, 0x80, 0x00]),\n      Ok((&b\"\"[..], 71_881_672_479_506_432_i64))\n    );\n    assert_eq!(\n      le_tsti64(&[0x00, 0xFF, 0x60, 0x00, 0x12, 0x00, 0x80, 0x00]),\n      Ok((&b\"\"[..], 36_028_874_334_732_032_i64))\n    );\n  }\n\n  #[cfg(feature = \"std\")]\n  fn parse_f64(i: &str) -> IResult<&str, f64, ()> {\n    use crate::traits::ParseTo;\n    match recognize_float_or_exceptions(i) {\n      Err(e) => Err(e),\n      Ok((i, s)) => {\n        if s.is_empty() {\n          return Err(Err::Error(()));\n        }\n        match s.parse_to() {\n          Some(n) => Ok((i, n)),\n          None => Err(Err::Error(())),\n        }\n      }\n    }\n  }\n\n  proptest! {\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn floats(s in \"\\\\PC*\") {\n        println!(\"testing {}\", s);\n        let res1 = parse_f64(&s);\n        let res2 = double::<_, ()>(s.as_str());\n        assert_eq!(res1, res2);\n    }\n  }\n}\n\n// Path: src/internal.rs\n//! Basic types to build the parsers\n\nuse self::Needed::*;\nuse crate::error::{self, ErrorKind, FromExternalError, ParseError};\nuse crate::lib::std::fmt;\nuse core::marker::PhantomData;\nuse core::num::NonZeroUsize;\n\n/// Holds the result of parsing functions\n///\n/// It depends on the input type `I`, the output type `O`, and the error type `E`\n/// (by default `(I, nom::ErrorKind)`)\n///\n/// The `Ok` side is a pair containing the remainder of the input (the part of the data that\n/// was not parsed) and the produced value. The `Err` side contains an instance of `nom::Err`.\n///\n/// Outside of the parsing code, you can use the [Finish::finish] method to convert\n/// it to a more common result type\npub type IResult<I, O, E = error::Error<I>> = Result<(I, O), Err<E>>;\n\n/// Helper trait to convert a parser's result to a more manageable type\npub trait Finish<I, O, E> {\n  /// converts the parser's result to a type that is more consumable by error\n  /// management libraries. It keeps the same `Ok` branch, and merges `Err::Error`\n  /// and `Err::Failure` into the `Err` side.\n  ///\n  /// *warning*: if the result is `Err(Err::Incomplete(_))`, this method will panic.\n  /// - \"complete\" parsers: It will not be an issue, `Incomplete` is never used\n  /// - \"streaming\" parsers: `Incomplete` will be returned if there's not enough data\n  /// for the parser to decide, and you should gather more data before parsing again.\n  /// Once the parser returns either `Ok(_)`, `Err(Err::Error(_))` or `Err(Err::Failure(_))`,\n  /// you can get out of the parsing loop and call `finish()` on the parser's result\n  fn finish(self) -> Result<(I, O), E>;\n}\n\nimpl<I, O, E> Finish<I, O, E> for IResult<I, O, E> {\n  fn finish(self) -> Result<(I, O), E> {\n    match self {\n      Ok(res) => Ok(res),\n      Err(Err::Error(e)) | Err(Err::Failure(e)) => Err(e),\n      Err(Err::Incomplete(_)) => {\n        panic!(\"Cannot call `finish()` on `Err(Err::Incomplete(_))`: this result means that the parser does not have enough data to decide, you should gather more data and try to reapply the parser instead\")\n      }\n    }\n  }\n}\n\n/// Contains information on needed data if a parser returned `Incomplete`\n#[derive(Debug, PartialEq, Eq, Clone, Copy)]\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub enum Needed {\n  /// Needs more data, but we do not know how much\n  Unknown,\n  /// Contains the required data size in bytes\n  Size(NonZeroUsize),\n}\n\nimpl Needed {\n  /// Creates `Needed` instance, returns `Needed::Unknown` if the argument is zero\n  pub fn new(s: usize) -> Self {\n    match NonZeroUsize::new(s) {\n      Some(sz) => Needed::Size(sz),\n      None => Needed::Unknown,\n    }\n  }\n\n  /// Indicates if we know how many bytes we need\n  pub fn is_known(&self) -> bool {\n    *self != Unknown\n  }\n\n  /// Maps a `Needed` to `Needed` by applying a function to a contained `Size` value.\n  #[inline]\n  pub fn map<F: Fn(NonZeroUsize) -> usize>(self, f: F) -> Needed {\n    match self {\n      Unknown => Unknown,\n      Size(n) => Needed::new(f(n)),\n    }\n  }\n}\n\n/// The `Err` enum indicates the parser was not successful\n///\n/// It has three cases:\n///\n/// * `Incomplete` indicates that more data is needed to decide. The `Needed` enum\n/// can contain how many additional bytes are necessary. If you are sure your parser\n/// is working on full data, you can wrap your parser with the `complete` combinator\n/// to transform that case in `Error`\n/// * `Error` means some parser did not succeed, but another one might (as an example,\n/// when testing different branches of an `alt` combinator)\n/// * `Failure` indicates an unrecoverable error. For example, when a prefix has been\n/// recognised and the next parser has been confirmed, if that parser fails, then the\n/// entire process fails; there are no more parsers to try.\n///\n/// Distinguishing `Failure` this from `Error` is only relevant inside the parser's code. For\n/// external consumers, both mean that parsing failed.\n///\n/// See also: [`Finish`].\n///\n#[derive(Debug, Clone, PartialEq)]\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub enum Err<Failure, Error = Failure> {\n  /// There was not enough data\n  Incomplete(Needed),\n  /// The parser had an error (recoverable)\n  Error(Error),\n  /// The parser had an unrecoverable error: we got to the right\n  /// branch and we know other branches won't work, so backtrack\n  /// as fast as possible\n  Failure(Failure),\n}\n\nimpl<E> Err<E> {\n  /// Tests if the result is Incomplete\n  pub fn is_incomplete(&self) -> bool {\n    matches!(self, Err::Incomplete(..))\n  }\n\n  /// Applies the given function to the inner error\n  pub fn map<E2, F>(self, f: F) -> Err<E2>\n  where\n    F: FnOnce(E) -> E2,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure(t) => Err::Failure(f(t)),\n      Err::Error(t) => Err::Error(f(t)),\n    }\n  }\n\n  /// Automatically converts between errors if the underlying type supports it\n  pub fn convert<F>(e: Err<F>) -> Self\n  where\n    E: From<F>,\n  {\n    e.map(crate::lib::std::convert::Into::into)\n  }\n}\n\nimpl<T> Err<(T, ErrorKind)> {\n  /// Maps `Err<(T, ErrorKind)>` to `Err<(U, ErrorKind)>` with the given `F: T -> U`\n  pub fn map_input<U, F>(self, f: F) -> Err<(U, ErrorKind)>\n  where\n    F: FnOnce(T) -> U,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure((input, k)) => Err::Failure((f(input), k)),\n      Err::Error((input, k)) => Err::Error((f(input), k)),\n    }\n  }\n}\n\nimpl<T> Err<error::Error<T>> {\n  /// Maps `Err<error::Error<T>>` to `Err<error::Error<U>>` with the given `F: T -> U`\n  pub fn map_input<U, F>(self, f: F) -> Err<error::Error<U>>\n  where\n    F: FnOnce(T) -> U,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure(error::Error { input, code }) => Err::Failure(error::Error {\n        input: f(input),\n        code,\n      }),\n      Err::Error(error::Error { input, code }) => Err::Error(error::Error {\n        input: f(input),\n        code,\n      }),\n    }\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nuse crate::lib::std::{borrow::ToOwned, string::String, vec::Vec};\n#[cfg(feature = \"alloc\")]\nimpl Err<(&[u8], ErrorKind)> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<(Vec<u8>, ErrorKind)> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<(&str, ErrorKind)> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<(String, ErrorKind)> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<error::Error<&[u8]>> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<error::Error<Vec<u8>>> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<error::Error<&str>> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<error::Error<String>> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\nimpl<E: Eq> Eq for Err<E> {}\n\nimpl<E> fmt::Display for Err<E>\nwhere\n  E: fmt::Debug,\n{\n  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n    match self {\n      Err::Incomplete(Needed::Size(u)) => write!(f, \"Parsing requires {} bytes/chars\", u),\n      Err::Incomplete(Needed::Unknown) => write!(f, \"Parsing requires more data\"),\n      Err::Failure(c) => write!(f, \"Parsing Failure: {:?}\", c),\n      Err::Error(c) => write!(f, \"Parsing Error: {:?}\", c),\n    }\n  }\n}\n\n#[cfg(feature = \"std\")]\nuse std::error::Error;\n\n#[cfg(feature = \"std\")]\nimpl<E> Error for Err<E>\nwhere\n  E: fmt::Debug,\n{\n  fn source(&self) -> Option<&(dyn Error + 'static)> {\n    None // no underlying error\n  }\n}\n\n/// Parser mode: influences how combinators build values\n///\n/// the [Parser] trait is generic over types implementing [Mode]. Its method are\n/// called to produce and manipulate output values or errors.\n///\n/// The main implementations of this trait are:\n/// * [Emit]: produce a value\n/// * [Check]: apply the parser but do not generate a value\npub trait Mode {\n  /// The output type that may be generated\n  type Output<T>;\n\n  /// Produces a value\n  fn bind<T, F: FnOnce() -> T>(f: F) -> Self::Output<T>;\n\n  /// Applies a function over the produced value\n  fn map<T, U, F: FnOnce(T) -> U>(x: Self::Output<T>, f: F) -> Self::Output<U>;\n\n  /// Combines two values generated by previous parsers\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    x: Self::Output<T>,\n    y: Self::Output<U>,\n    f: F,\n  ) -> Self::Output<V>;\n}\n\n/// Produces a value. This is the default behaviour for parsers\npub struct Emit;\nimpl Mode for Emit {\n  type Output<T> = T;\n\n  #[inline(always)]\n  fn bind<T, F: FnOnce() -> T>(f: F) -> Self::Output<T> {\n    f()\n  }\n\n  #[inline(always)]\n  fn map<T, U, F: FnOnce(T) -> U>(x: Self::Output<T>, f: F) -> Self::Output<U> {\n    f(x)\n  }\n\n  #[inline(always)]\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    x: Self::Output<T>,\n    y: Self::Output<U>,\n    f: F,\n  ) -> Self::Output<V> {\n    f(x, y)\n  }\n}\n\n/// Applies the parser, but do not a produce a value\n///\n/// This has the effect of greatly reducing the amount of code generated and the\n/// parser memory usage. Some combinators chek for an error in a child parser but\n/// discard the error, and for those, using [Check] makes sure the error is not\n/// even generated, only the fact that an error happened remains\npub struct Check;\nimpl Mode for Check {\n  type Output<T> = ();\n\n  #[inline(always)]\n  fn bind<T, F: FnOnce() -> T>(_: F) -> Self::Output<T> {}\n\n  #[inline(always)]\n  fn map<T, U, F: FnOnce(T) -> U>(_: Self::Output<T>, _: F) -> Self::Output<U> {}\n\n  #[inline(always)]\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    _: Self::Output<T>,\n    _: Self::Output<U>,\n    _: F,\n  ) -> Self::Output<V> {\n  }\n}\n\n/// Parser result type\n///\n/// * `Ok` branch: a tuple of the remaining input data, and the output value.\n/// The output value is of the `O` type if the output mode was [Emit], and `()`\n/// if the mode was [Check]\n/// * `Err` branch: an error of the `E` type if the erroor mode was [Emit], and `()`\n/// if the mode was [Check]\npub type PResult<OM, I, O, E> = Result<\n  (I, <<OM as OutputMode>::Output as Mode>::Output<O>),\n  Err<E, <<OM as OutputMode>::Error as Mode>::Output<E>>,\n>;\n\n/// Trait Defining the parser's execution\n///\n/// The same parser implementation can vary in behaviour according to the chosen\n/// output mode\npub trait OutputMode {\n  /// Defines the [Mode] for the output type. [Emit] will generate the value, [Check] will\n  /// apply the parser but will only generate `()` if successul. This can be used when\n  /// verifying that the input data conforms to the format without having to generate any\n  /// output data\n  type Output: Mode;\n  /// Defines the [Mode] for the output type. [Emit] will generate the value, [Check] will\n  /// apply the parser but will only generate `()` if an error happened. [Emit] should be\n  /// used when we want to handle the error and extract useful information from it. [Check]\n  /// is used when we just want to know if parsing failed and reject the data quickly.\n  type Error: Mode;\n  /// Indicates whether the input data is \"complete\", ie we already have the entire data in the\n  /// buffer, or if it is \"streaming\", where more data can be added later in the buffer. In\n  /// streaming mode, the parser will understand that a failure may mean that we are missing\n  /// data, and will return a specific error branch, [Err::Incomplete] to signal it. In complete\n  /// mode, the parser will generate a normal error\n  type Incomplete: IsStreaming;\n}\n\n/// Specifies the behaviour when a parser encounters an error that could be due to partial ata\npub trait IsStreaming {\n  /// called by parsers on partial data errors\n  /// * `needed` can hold the amount of dditional data the parser would need to decide\n  /// * `err_f`: produces the error when in \"complete\" mode\n  fn incomplete<E, F: FnOnce() -> E>(needed: Needed, err_f: F) -> Err<E>;\n  /// Indicates whether the data is in streaming mode or not\n  fn is_streaming() -> bool;\n}\n\n/// Indicates that the input data is streaming: more data may be available later\npub struct Streaming;\n\nimpl IsStreaming for Streaming {\n  fn incomplete<E, F: FnOnce() -> E>(needed: Needed, _err_f: F) -> Err<E> {\n    Err::Incomplete(needed)\n  }\n\n  #[inline]\n  fn is_streaming() -> bool {\n    true\n  }\n}\n\n/// Indicates that the input data is complete: no more data may be added later\npub struct Complete;\n\nimpl IsStreaming for Complete {\n  fn incomplete<E, F: FnOnce() -> E>(_needed: Needed, err_f: F) -> Err<E> {\n    Err::Error(err_f())\n  }\n\n  #[inline]\n  fn is_streaming() -> bool {\n    false\n  }\n}\n\n/// Holds the parser execution modifiers: output [Mode], error [Mode] and\n/// streaming behaviour for input data\npub struct OutputM<M: Mode, EM: Mode, S: IsStreaming> {\n  m: PhantomData<M>,\n  em: PhantomData<EM>,\n  s: PhantomData<S>,\n}\n\nimpl<M: Mode, EM: Mode, S: IsStreaming> OutputMode for OutputM<M, EM, S> {\n  type Output = M;\n  type Error = EM;\n  type Incomplete = S;\n}\n/// All nom parsers implement this trait\npub trait Parser<Input> {\n  /// Type of the produced value\n  type Output;\n  /// Error type of this parser\n  type Error: ParseError<Input>;\n\n  /// A parser takes in input type, and returns a `Result` containing\n...\n", "needle_token_start": 2310, "needle_token_end": 2550, "code_context_ntokens": 16278, "output": ["```rust\n#[test]\nfn be_i16_tests() {\n    assert_parse!(be_i16(&[0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_i16(&[0x7f, 0xff][..]), Ok((&b\"\"[..], 32_767_i16)));\n    assert_parse!(be_i16(&[0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(be_i16(&[0x80, 0x00][..]), Ok((&b\"\"[..], -32_768_i16)));\n    assert_parse!(be_i16(&[][..]), Err(Err::Incomplete(Needed::new(2))));\n    assert_parse!(be_i16(&[0x00][..]), Err(Err::Incomplete(Needed::new(1))));\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "many1_count", "language": "rust", "path": "src/multi/mod.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to repeatedly apply a given parser to an input until the parser fails, counting the number of successful applications. It requires at least one successful parse to avoid returning an error.\n2. **Input**: The function takes a parser function as input, which is applied to a sequence or stream.\n3. **Output**: It returns either the count of successful parses as an unsigned integer if at least one parse is successful, or an error if the first application of the parser fails.\n4. **Procedure**: The function initializes a counter and repeatedly applies the provided parser to the input. If the parser succeeds, the input is updated, and the counter is incremented. If the parser fails on the first attempt, an error is returned. The process stops when the parser fails after at least one successful application, returning the count of successful applications.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "      Err(Err::Error(_)) => return Ok((i, res)),\n      Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n      Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n      Ok((i1, o)) => {\n        res = OM::Output::combine(res, o, |mut res, o| {\n          res.push(o);\n          res\n        });\n        i = i1;\n      }\n    }\n\n    loop {\n      let len = i.input_len();\n      match self\n        .separator\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, res)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, _)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::SeparatedList)\n            })));\n          }\n\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i1.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, res)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i2, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              i = i2;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Alternates between two parsers to produce a list of elements until [`Err::Error`].\n///\n/// Fails if the element parser does not produce at least one element.$\n///\n/// This stops when either parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `sep` Parses the separator between list elements.\n/// * `f` Parses the elements of the list.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::separated_list1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   separated_list1(tag(\"|\"), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abc|abc|abc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123abc\"), Ok((\"123abc\", vec![\"abc\"])));\n/// assert_eq!(parser(\"abc|def\"), Ok((\"|def\", vec![\"abc\"])));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"def|abc\"), Err(Err::Error(Error::new(\"def|abc\", ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn separated_list1<I, E, F, G>(\n  separator: G,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  SeparatedList1 { parser, separator }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [separated_list1] combinator\npub struct SeparatedList1<F, G> {\n  parser: F,\n  separator: G,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, E: ParseError<I>, F, G> Parser<I> for SeparatedList1<F, G>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n\n    match self.parser.process::<OM>(i.clone()) {\n      Err(e) => return Err(e),\n      Ok((i1, o)) => {\n        res = OM::Output::combine(res, o, |mut res, o| {\n          res.push(o);\n          res\n        });\n        i = i1;\n      }\n    }\n\n    loop {\n      let len = i.input_len();\n      match self\n        .separator\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, res)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, _)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::SeparatedList)\n            })));\n          }\n\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i1.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, res)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i2, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              i = i2;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times\n///\n/// This stops before `n` when the parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many_m_n(0, 2, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many_m_n<I, E, F>(\n  min: usize,\n  max: usize,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  ManyMN { parser, min, max }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many_m_n] combinator\npub struct ManyMN<F> {\n  parser: F,\n  min: usize,\n  max: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for ManyMN<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.min > self.max {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::ManyMN,\n      )));\n    }\n\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.min.min(max_initial_capacity))\n    });\n    for count in 0..self.max {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::ManyMN)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |mut res, value| {\n            res.push(value);\n            res\n          });\n          input = tail;\n        }\n        Err(Err::Error(e)) => {\n          if count < self.min {\n            return Err(Err::Error(OM::Error::map(e, |e| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::ManyMN, e)\n            })));\n          } else {\n            return Ok((input, res));\n          }\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Repeats the embedded parser, counting the results\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many0_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many0_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", 0)));\n/// assert_eq!(parser(\"\"), Ok((\"\", 0)));\n/// ```\npub fn many0_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many0Count { parser }\n}\n\n/// Parser implementation for the [many0_count] combinator\npub struct Many0Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many0Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    loop {\n      let input_ = input.clone();\n      let len = input.input_len();\n      match self\n        .parser\n        .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n      {\n        Ok((i, _)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0Count)\n            })));\n          }\n\n          input = i;\n          count += 1;\n        }\n\n        Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser, counting the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many1_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many1_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1Count))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1Count))));\n/// ```\n\npub fn many1_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many1Count { parser }\n}\n\n/// Parser implementation for the [many1_count] combinator\npub struct Many1Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many1Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    match self\n      .parser\n      .process::<OutputM<Check, Check, OM::Incomplete>>(input.clone())\n    {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(move || {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n      }))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e)),\n      Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),\n      Ok((mut input, _)) => {\n        count += 1;\n\n        loop {\n          let input_ = input.clone();\n          let len = input.input_len();\n          match self\n            .parser\n            .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n          {\n            Ok((i, _)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n                })));\n              }\n\n              input = i;\n              count += 1;\n            }\n\n            Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser `count` times, gathering the results in a `Vec`\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `count` How often to apply the parser.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   count(tag(\"abc\"), 2).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn count<I, F>(\n  parser: F,\n  count: usize,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  Count { parser, count }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [count] combinator\npub struct Count<F> {\n  parser: F,\n  count: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Count<F>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.count.min(max_initial_capacity))\n    });\n\n    for _ in 0..self.count {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          res = OM::Output::combine(res, o, |mut res, o| {\n            res.push(o);\n            res\n          });\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Runs the embedded parser repeatedly, filling the given slice with results.\n///\n/// This parser fails if the input runs out before the given slice is full.\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `buf` The slice to fill\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fill;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, [&str; 2]> {\n///   let mut buf = [\"\", \"\"];\n///   let (rest, ()) = fill(tag(\"abc\"), &mut buf).parse(s)?;\n///   Ok((rest, buf))\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", [\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", [\"abc\", \"abc\"])));\n/// ```\npub fn fill<'a, I, E, F>(\n  parser: F,\n  buf: &'a mut [<F as Parser<I>>::Output],\n) -> impl Parser<I, Output = (), Error = E> + 'a\nwhere\n  I: Clone,\n  F: Parser<I, Error = E> + 'a,\n  E: ParseError<I>,\n{\n  Fill { parser, buf }\n}\n\n/// Parser implementation for the [fill] combinator\npub struct Fill<'a, F, O> {\n  parser: F,\n  buf: &'a mut [O],\n}\n\nimpl<'a, I, F, O> Parser<I> for Fill<'a, F, O>\nwhere\n  I: Clone,\n  F: Parser<I, Output = O>,\n{\n  type Output = ();\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n\n    for elem in self.buf.iter_mut() {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          OM::Output::map(o, |o| *elem = o);\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, OM::Output::bind(|| ())))\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many0(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// ```\npub fn fold_many0<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany0 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many0] combinator\npub struct FoldMany0<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany0<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let mut input = i;\n\n    loop {\n      let i_ = input.clone();\n      let len = input.input_len();\n      match self.parser.process::<OM>(i_) {\n        Ok((i, o)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0)\n            })));\n          }\n\n          res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n          input = i;\n        }\n        Err(Err::Error(_)) => {\n          return Ok((input, res));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fold_many1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many1(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1))));\n/// ```\npub fn fold_many1<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany1 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many1] combinator\npub struct FoldMany1<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany1<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let input = i.clone();\n\n    match self.parser.process::<OM>(input) {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many1)\n      }))),\n      Err(e) => Err(e),\n      Ok((i1, o1)) => {\n        res = OM::Output::combine(res, o1, |res, o| (self.g)(res, o));\n\n        let mut input = i1;\n        loop {\n          let i_ = input.clone();\n          let len = input.input_len();\n          match self.parser.process::<OM>(i_) {\n            Ok((i, o)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1)\n                })));\n              }\n\n              res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n              input = i;\n            }\n            Err(Err::Error(_)) => {\n              return Ok((input, res));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times, calling `g` to gather the results\n///\n/// This stops before `n` when the parser returns [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many_m_n(\n///     0,\n///     2,\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\npub fn fold_many_m_n<I, E, F, G, H, R>(\n  min: usize,\n  max: usize,\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldManyMN {\n    parser,\n    g,\n    init,\n    min,\n    max,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many_m_n] combinator\npub struct FoldManyMN<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n  min: usize,\n  max: usize,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldManyMN<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.min > self.max {\n      return Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::ManyMN)\n      })));\n    }\n\n    let mut res = OM::Output::bind(|| (self.init)());\n    for count in 0..self.max {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(tail, ErrorKind::ManyMN)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |res, o| (self.g)(res, o));\n          input = tail;\n        }\n        Err(Err::Error(err)) => {\n          if count < self.min {\n            return Err(Err::Error(OM::Error::map(err, |err| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::ManyMN, err)\n            })));\n          } else {\n            break;\n          }\n        }\n        Err(e) => return Err(e),\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Gets a number from the parser and returns a\n/// subslice of the input of that size.\n/// If the parser returns `Incomplete`,\n/// `length_data` will return an error.\n/// # Arguments\n/// * `f` The parser to apply.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::number::complete::be_u16;\n/// use nom::multi::length_data;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   length_data(be_u16).parse(s)\n/// }\n///\n/// assert_eq!(parser(b\"\\x00\\x03abcefg\"), Ok((&b\"efg\"[..], &b\"abc\"[..])));\n/// assert_eq!(parser(b\"\\x00\\x03a\"), Err(Err::Incomplete(Needed::new(2))));\n/// ```\npub fn length_data<I, E, F>(f: F) -> impl Parser<I, Output = I, Error = E>\nwhere\n  I: Input,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  f.flat_map(|size| take(size))\n}\n\n/// Gets a number from the first parser,\n/// takes a subslice of the input of that size,\n/// then applies the second parser on that subslice.\n/// If the second parser returns `Incomplete`,\n/// `length_value` will return an error.\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `g` The parser to apply on the subslice.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::number::complete::be_u16;\n/// use nom::multi::length_value;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   length_value(be_u16, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(b\"\\x00\\x03abcefg\"), Ok((&b\"efg\"[..], &b\"abc\"[..])));\n/// assert_eq!(parser(b\"\\x00\\x03123123\"), Err(Err::Error(Error::new(&b\"123\"[..], ErrorKind::Tag))));\n/// assert_eq!(parser(b\"\\x00\\x03a\"), Err(Err::Incomplete(Needed::new(2))));\n/// ```\npub fn length_value<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = <G as Parser<I>>::Output, Error = E>\nwhere\n  I: Clone + Input,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  /*f.flat_map(|size| {\n    println!(\"got size: {size}\");\n    take(size)\n  })\n  .and_then(g)*/\n  LengthValue {\n    length: f,\n    parser: g,\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for the [length_value] combinator\npub struct LengthValue<F, G, E> {\n  length: F,\n  parser: G,\n  e: PhantomData<E>,\n}\n\nimpl<I, F, G, E> Parser<I> for LengthValue<F, G, E>\nwhere\n  I: Clone + Input,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  <F as Parser<I>>::Output: ToUsize,\n  E: ParseError<I>,\n{\n  type Output = <G as Parser<I>>::Output;\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let (i, length) = self\n      .length\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(input)?;\n\n    let length: usize = length.to_usize();\n\n    if let Some(needed) = length\n      .checked_sub(i.input_len())\n      .and_then(NonZeroUsize::new)\n    {\n      Err(Err::Incomplete(Needed::Size(needed)))\n    } else {\n      let (rest, i) = i.take_split(length);\n      match self.parser.process::<OM>(i.clone()) {\n        Err(Err::Incomplete(_)) => Err(Err::Error(OM::Error::bind(|| {\n          E::from_error_kind(i, ErrorKind::Complete)\n        }))),\n        Err(e) => Err(e),\n        Ok((_, o)) => Ok((rest, o)),\n      }\n    }\n  }\n}\n\n/// Gets a number from the first parser,\n/// then applies the second parser that many times.\n/// # Arguments\n/// * `f` The parser to apply to obtain the count.\n/// * `g` The parser to apply repeatedly.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::number::complete::u8;\n/// use nom::multi::length_count;\n/// use nom::bytes::complete::tag;\n/// use nom::combinator::map;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n///   length_count(map(u8, |i| {\n///      println!(\"got number: {}\", i);\n///      i\n///   }), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(&b\"\\x02abcabcabc\"[..]), Ok(((&b\"abc\"[..], vec![&b\"abc\"[..], &b\"abc\"[..]]))));\n/// assert_eq!(parser(b\"\\x03123123123\"), Err(Err::Error(Error::new(&b\"123123123\"[..], ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\npub fn length_count<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = Vec<<G as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  LengthCount {\n    length: f,\n    parser: g,\n    e: PhantomData,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [length_count] combinator\npub struct LengthCount<F, G, E> {\n  length: F,\n  parser: G,\n  e: PhantomData<E>,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F, G, E> Parser<I> for LengthCount<F, G, E>\nwhere\n  I: Clone,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  <F as Parser<I>>::Output: ToUsize,\n  E: ParseError<I>,\n{\n  type Output = Vec<<G as Parser<I>>::Output>;\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match self\n      .length\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(input)\n    {\n      Err(e) => Err(e),\n      Ok((i, count)) => {\n        let count = count.to_usize();\n        let mut input = i.clone();\n        let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n          / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n        let mut res = OM::Output::bind(|| {\n          crate::lib::std::vec::Vec::with_capacity(count.min(max_initial_capacity))\n        });\n\n        for _ in 0..count {\n          let input_ = input.clone();\n          match self.parser.process::<OM>(input_) {\n            Ok((i, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              input = i;\n            }\n            Err(Err::Error(e)) => {\n              return Err(Err::Error(OM::Error::map(e, |e| {\n                <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n              })));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n\n        Ok((input, res))\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser and collects the results in a type implementing `Extend + Default`.\n/// Fails if the amount of time the embedded parser is run is not\n/// within the specified range.\n/// # Arguments\n/// * `range` Constrains the number of iterations.\n///   * A range without an upper bound `a..` is equivalent to a range of `a..=usize::MAX`.\n///   * A single `usize` value is equivalent to `value..=value`.\n///   * An empty range is invalid.\n/// * `parse` The parser to apply.\n///\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many(0..=2, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n///\n/// This is not limited to `Vec`, other collections like `HashMap`\n/// can be used:\n///\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many;\n/// use nom::bytes::complete::{tag, take_while};\n/// use nom::sequence::{separated_pair, terminated};\n/// use nom::AsChar;\n///\n/// use std::collections::HashMap;\n///\n/// fn key_value(s: &str) -> IResult<&str, HashMap<&str, &str>> {\n///   many(0.., terminated(\n///     separated_pair(\n///       take_while(AsChar::is_alpha),\n///       tag(\"=\"),\n///       take_while(AsChar::is_alpha)\n///     ),\n///     tag(\";\")\n///   )).parse(s)\n/// }\n///\n/// assert_eq!(\n///   key_value(\"a=b;c=d;\"),\n///   Ok((\"\", HashMap::from([(\"a\", \"b\"), (\"c\", \"d\")])))\n/// );\n/// ```\n///\n/// If more control is needed on the default value, [fold] can\n/// be used instead:\n///\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold;\n/// use nom::bytes::complete::tag;\n///\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold(\n///     0..=4,\n///     tag(\"abc\"),\n///     // preallocates a vector of the max size\n///     || Vec::with_capacity(4),\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n///\n/// assert_eq!(parser(\"abcabcabcabc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many<I, E, Collection, F, G>(\n  range: G,\n  parser: F,\n) -> impl Parser<I, Output = Collection, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  Collection: Extend<<F as Parser<I>>::Output> + Default,\n  E: ParseError<I>,\n  G: NomRange<usize>,\n{\n  Many {\n    parser,\n    range,\n    c: PhantomData,\n  }\n}\n\n/// Parser implementation for the [many] combinator\npub struct Many<F, R, Collection> {\n  parser: F,\n  range: R,\n  c: PhantomData<Collection>,\n}\n\nimpl<I, F, R, Collection> Parser<I> for Many<F, R, Collection>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  Collection: Extend<<F as Parser<I>>::Output> + Default,\n  R: NomRange<usize>,\n{\n  type Output = Collection;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.range.is_inverted() {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::Many,\n      )));\n    }\n\n    let mut res = OM::Output::bind(Collection::default);\n\n    for count in self.range.bounded_iter() {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |mut res, value| {\n            res.extend(Some(value));\n            res\n          });\n          input = tail;\n        }\n        Err(Err::Error(e)) => {\n          if !self.range.contains(&count) {\n            return Err(Err::Error(OM::Error::map(e, |e| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::Many, e)\n            })));\n          } else {\n            return Ok((input, res));\n          }\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Applies a parser and accumulates the results using a given\n/// function and initial value.\n/// Fails if the amount of time the embedded parser is run is not\n/// within the specified range.\n///\n/// # Arguments\n/// * `range` Constrains the number of iterations.\n///   * A range without an upper bound `a..` allows the parser to run until it fails.\n///   * A single `usize` value is equivalent to `value..=value`.\n///   * An empty range is invalid.\n/// * `parse` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `fold` The function that combines a result of `f` with\n///       the current accumulator.\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold(\n///     0..=2,\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\npub fn fold<I, E, F, G, H, J, R>(\n  range: J,\n  parser: F,\n  init: H,\n  fold: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n  J: NomRange<usize>,\n{\n  Fold {\n    parser,\n    init,\n    fold,\n    range,\n  }\n}\n\n/// Parser implementation for the [fold] combinator\npub struct Fold<F, G, H, Range> {\n  parser: F,\n  init: H,\n  fold: G,\n  range: Range,\n}\n\nimpl<I, F, G, H, Range, Res> Parser<I> for Fold<F, G, H, Range>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(Res, <F as Parser<I>>::Output) -> Res,\n  H: FnMut() -> Res,\n  Range: NomRange<usize>,\n{\n  type Output = Res;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.range.is_inverted() {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::Fold,\n      )));\n    }\n\n    let mut acc = OM::Output::bind(|| (self.init)());\n\n    for count in self.range.saturating_iter() {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(tail, ErrorKind::Fold)\n            })));\n          }\n\n          acc = OM::Output::combine(acc, value, |acc, value| (self.fold)(acc, value));\n          input = tail;\n        }\n        Err(Err::Error(err)) => {\n          if !self.range.contains(&count) {\n            return Err(Err::Error(OM::Error::map(err, |err| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::Fold, err)\n            })));\n          } else {\n            break;\n          }\n        }\n        Err(e) => return Err(e),\n      }\n    }\n\n    Ok((input, acc))\n  }\n}\n\n// Path: src/str.rs\n#[cfg(test)]\nmod test {\n  #[cfg(feature = \"alloc\")]\n  use crate::{branch::alt, bytes::complete::tag_no_case, combinator::recognize, multi::many};\n  use crate::{\n    bytes::complete::{is_a, is_not, tag, take, take_till, take_until},\n    error::{self, ErrorKind},\n    Err, IResult,\n  };\n\n  #[test]\n  fn tagtr_succeed() {\n    const INPUT: &str = \"Hello World!\";\n    const TAG: &str = \"Hello\";\n    fn test(input: &str) -> IResult<&str, &str> {\n      tag(TAG)(input)\n    }\n\n    match test(INPUT) {\n      Ok((extra, output)) => {\n        assert!(extra == \" World!\", \"Parser `tag` consumed leftover input.\");\n        assert!(\n          output == TAG,\n          \"Parser `tag` doesn't return the tag it matched on success. \\\n           Expected `{}`, got `{}`.\",\n          TAG,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `tag` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn tagtr_incomplete() {\n    use crate::bytes::streaming::tag;\n\n    const INPUT: &str = \"Hello\";\n    const TAG: &str = \"Hello World!\";\n\n    let res: IResult<_, _, error::Error<_>> = tag(TAG)(INPUT);\n    match res {\n      Err(Err::Incomplete(_)) => (),\n      other => {\n        panic!(\n          \"Parser `tag` didn't require more input when it should have. \\\n           Got `{:?}`.\",\n          other\n        );\n      }\n    };\n  }\n\n  #[test]\n  fn tagtr_error() {\n    const INPUT: &str = \"Hello World!\";\n    const TAG: &str = \"Random\"; // TAG must be closer than INPUT.\n\n    let res: IResult<_, _, error::Error<_>> = tag(TAG)(INPUT);\n    match res {\n      Err(Err::Error(_)) => (),\n      other => {\n        panic!(\n          \"Parser `tag` didn't fail when it should have. Got `{:?}`.`\",\n          other\n        );\n      }\n    };\n  }\n\n  #[test]\n  fn take_s_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\u0192\u01ad\u00e8\u0159\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\";\n    const LEFTOVER: &str = \"\u00e1\u0192\u01ad\u00e8\u0159\";\n\n    let res: IResult<_, _, error::Error<_>> = take(9_usize)(INPUT);\n    match res {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_s` consumed leftover input. Leftover `{}`.\",\n          extra\n        );\n        assert!(\n          output == CONSUMED,\n          \"Parser `take_s` doesn't return the string it consumed on success. Expected `{}`, got `{}`.\",\n          CONSUMED,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `take_s` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn take_until_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u2202\u00e1\u0192\u01ad\u00e8\u0159\";\n    const FIND: &str = \"\u00c2\u00df\u00c7\u2202\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\";\n    const LEFTOVER: &str = \"\u00c2\u00df\u00c7\u2202\u00e1\u0192\u01ad\u00e8\u0159\";\n\n    let res: IResult<_, _, (_, ErrorKind)> = take_until(FIND)(INPUT);\n    match res {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_until`\\\n           consumed leftover input. Leftover `{}`.\",\n          extra\n        );\n        assert!(\n...\n", "needle_token_start": 4037, "needle_token_end": 4116, "code_context_ntokens": 16292, "output": ["```rust\n/// Runs the embedded parser, counting the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many1_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many1_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1Count))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1Count))));\n/// ```\n\npub fn many1_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many1Count { parser }\n}\n\n/// Parser implementation for the [many1_count] combinator\npub struct Many1Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many1Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    match self\n      .parser\n      .process::<OutputM<Check, Check, OM::Incomplete>>(input.clone())\n    {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(move || {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n      }))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e)),\n      Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),\n      Ok((mut input, _)) => {\n        count += 1;\n\n        loop {\n          let input_ = input.clone();\n          let len = input.input_len();\n          match self\n            .parser\n            .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n          {\n            Ok((i, _)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n                })));\n              }\n\n              input = i;\n              count += 1;\n            }\n\n            Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n          }\n        }\n      }\n    }\n  }\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "test_offset_u8", "language": "rust", "path": "src/traits.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To verify the correctness of calculating the starting index of various sub-slices relative to their original slice.\n2. **Input**: A byte slice derived from a string and its sub-slices.\n3. **Output**: The function does not return any output but asserts the correctness of the index calculations.\n4. **Procedure**: The test initializes a byte slice from a string and creates several sub-slices from it. It then checks if the computed starting indices of these sub-slices relative to the original slice are as expected using assertions.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "/// Look for a substring in self\npub trait FindSubstring<T> {\n  /// Returns the byte position of the substring if it is found\n  fn find_substring(&self, substr: T) -> Option<usize>;\n}\n\nimpl<'a, 'b> FindSubstring<&'b [u8]> for &'a [u8] {\n  fn find_substring(&self, substr: &'b [u8]) -> Option<usize> {\n    if substr.len() > self.len() {\n      return None;\n    }\n\n    let (&substr_first, substr_rest) = match substr.split_first() {\n      Some(split) => split,\n      // an empty substring is found at position 0\n      // This matches the behavior of str.find(\"\").\n      None => return Some(0),\n    };\n\n    if substr_rest.is_empty() {\n      return memchr::memchr(substr_first, self);\n    }\n\n    let mut offset = 0;\n    let haystack = &self[..self.len() - substr_rest.len()];\n\n    while let Some(position) = memchr::memchr(substr_first, &haystack[offset..]) {\n      offset += position;\n      let next_offset = offset + 1;\n      if &self[next_offset..][..substr_rest.len()] == substr_rest {\n        return Some(offset);\n      }\n\n      offset = next_offset;\n    }\n\n    None\n  }\n}\n\nimpl<'a, 'b> FindSubstring<&'b str> for &'a [u8] {\n  fn find_substring(&self, substr: &'b str) -> Option<usize> {\n    self.find_substring(AsBytes::as_bytes(substr))\n  }\n}\n\nimpl<'a, 'b> FindSubstring<&'b str> for &'a str {\n  //returns byte index\n  fn find_substring(&self, substr: &'b str) -> Option<usize> {\n    self.find(substr)\n  }\n}\n\n/// Used to integrate `str`'s `parse()` method\npub trait ParseTo<R> {\n  /// Succeeds if `parse()` succeeded. The byte slice implementation\n  /// will first convert it to a `&str`, then apply the `parse()` function\n  fn parse_to(&self) -> Option<R>;\n}\n\nimpl<'a, R: FromStr> ParseTo<R> for &'a [u8] {\n  fn parse_to(&self) -> Option<R> {\n    from_utf8(self).ok().and_then(|s| s.parse().ok())\n  }\n}\n\nimpl<'a, R: FromStr> ParseTo<R> for &'a str {\n  fn parse_to(&self) -> Option<R> {\n    self.parse().ok()\n  }\n}\n\nimpl<const N: usize> InputLength for [u8; N] {\n  #[inline]\n  fn input_len(&self) -> usize {\n    self.len()\n  }\n}\n\nimpl<'a, const N: usize> InputLength for &'a [u8; N] {\n  #[inline]\n  fn input_len(&self) -> usize {\n    self.len()\n  }\n}\n\nimpl<'a, const N: usize> Compare<[u8; N]> for &'a [u8] {\n  #[inline(always)]\n  fn compare(&self, t: [u8; N]) -> CompareResult {\n    self.compare(&t[..])\n  }\n\n  #[inline(always)]\n  fn compare_no_case(&self, t: [u8; N]) -> CompareResult {\n    self.compare_no_case(&t[..])\n  }\n}\n\nimpl<'a, 'b, const N: usize> Compare<&'b [u8; N]> for &'a [u8] {\n  #[inline(always)]\n  fn compare(&self, t: &'b [u8; N]) -> CompareResult {\n    self.compare(&t[..])\n  }\n\n  #[inline(always)]\n  fn compare_no_case(&self, t: &'b [u8; N]) -> CompareResult {\n    self.compare_no_case(&t[..])\n  }\n}\n\nimpl<const N: usize> FindToken<u8> for [u8; N] {\n  fn find_token(&self, token: u8) -> bool {\n    memchr::memchr(token, &self[..]).is_some()\n  }\n}\n\nimpl<'a, const N: usize> FindToken<&'a u8> for [u8; N] {\n  fn find_token(&self, token: &u8) -> bool {\n    self.find_token(*token)\n  }\n}\n\n/// Abstracts something which can extend an `Extend`.\n/// Used to build modified input slices in `escaped_transform`\npub trait ExtendInto {\n  /// The current input type is a sequence of that `Item` type.\n  ///\n  /// Example: `u8` for `&[u8]` or `char` for `&str`\n  type Item;\n\n  /// The type that will be produced\n  type Extender;\n\n  /// Create a new `Extend` of the correct type\n  fn new_builder(&self) -> Self::Extender;\n  /// Accumulate the input into an accumulator\n  fn extend_into(&self, acc: &mut Self::Extender);\n}\n\n#[cfg(feature = \"alloc\")]\nimpl ExtendInto for [u8] {\n  type Item = u8;\n  type Extender = Vec<u8>;\n\n  #[inline]\n  fn new_builder(&self) -> Vec<u8> {\n    Vec::new()\n  }\n  #[inline]\n  fn extend_into(&self, acc: &mut Vec<u8>) {\n    acc.extend(self.iter().cloned());\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl ExtendInto for &[u8] {\n  type Item = u8;\n  type Extender = Vec<u8>;\n\n  #[inline]\n  fn new_builder(&self) -> Vec<u8> {\n    Vec::new()\n  }\n  #[inline]\n  fn extend_into(&self, acc: &mut Vec<u8>) {\n    acc.extend_from_slice(self);\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl ExtendInto for str {\n  type Item = char;\n  type Extender = String;\n\n  #[inline]\n  fn new_builder(&self) -> String {\n    String::new()\n  }\n  #[inline]\n  fn extend_into(&self, acc: &mut String) {\n    acc.push_str(self);\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl ExtendInto for &str {\n  type Item = char;\n  type Extender = String;\n\n  #[inline]\n  fn new_builder(&self) -> String {\n    String::new()\n  }\n  #[inline]\n  fn extend_into(&self, acc: &mut String) {\n    acc.push_str(self);\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl ExtendInto for char {\n  type Item = char;\n  type Extender = String;\n\n  #[inline]\n  fn new_builder(&self) -> String {\n    String::new()\n  }\n  #[inline]\n  fn extend_into(&self, acc: &mut String) {\n    acc.push(*self);\n  }\n}\n\n/// Helper trait to convert numbers to usize.\n///\n/// By default, usize implements `From<u8>` and `From<u16>` but not\n/// `From<u32>` and `From<u64>` because that would be invalid on some\n/// platforms. This trait implements the conversion for platforms\n/// with 32 and 64 bits pointer platforms\npub trait ToUsize {\n  /// converts self to usize\n  fn to_usize(&self) -> usize;\n}\n\nimpl ToUsize for u8 {\n  #[inline]\n  fn to_usize(&self) -> usize {\n    *self as usize\n  }\n}\n\nimpl ToUsize for u16 {\n  #[inline]\n  fn to_usize(&self) -> usize {\n    *self as usize\n  }\n}\n\nimpl ToUsize for usize {\n  #[inline]\n  fn to_usize(&self) -> usize {\n    *self\n  }\n}\n\n#[cfg(any(target_pointer_width = \"32\", target_pointer_width = \"64\"))]\nimpl ToUsize for u32 {\n  #[inline]\n  fn to_usize(&self) -> usize {\n    *self as usize\n  }\n}\n\n#[cfg(target_pointer_width = \"64\")]\nimpl ToUsize for u64 {\n  #[inline]\n  fn to_usize(&self) -> usize {\n    *self as usize\n  }\n}\n\n/// Equivalent From implementation to avoid orphan rules in bits parsers\npub trait ErrorConvert<E> {\n  /// Transform to another error type\n  fn convert(self) -> E;\n}\n\nimpl<I> ErrorConvert<(I, ErrorKind)> for ((I, usize), ErrorKind) {\n  fn convert(self) -> (I, ErrorKind) {\n    ((self.0).0, self.1)\n  }\n}\n\nimpl<I> ErrorConvert<((I, usize), ErrorKind)> for (I, ErrorKind) {\n  fn convert(self) -> ((I, usize), ErrorKind) {\n    ((self.0, 0), self.1)\n  }\n}\n\nuse crate::error;\nimpl<I> ErrorConvert<error::Error<I>> for error::Error<(I, usize)> {\n  fn convert(self) -> error::Error<I> {\n    error::Error {\n      input: self.input.0,\n      code: self.code,\n    }\n  }\n}\n\nimpl<I> ErrorConvert<error::Error<(I, usize)>> for error::Error<I> {\n  fn convert(self) -> error::Error<(I, usize)> {\n    error::Error {\n      input: (self.input, 0),\n      code: self.code,\n    }\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\nimpl<I> ErrorConvert<error::VerboseError<I>> for error::VerboseError<(I, usize)> {\n  fn convert(self) -> error::VerboseError<I> {\n    error::VerboseError {\n      errors: self.errors.into_iter().map(|(i, e)| (i.0, e)).collect(),\n    }\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\nimpl<I> ErrorConvert<error::VerboseError<(I, usize)>> for error::VerboseError<I> {\n  fn convert(self) -> error::VerboseError<(I, usize)> {\n    error::VerboseError {\n      errors: self.errors.into_iter().map(|(i, e)| ((i, 0), e)).collect(),\n    }\n  }\n}\n\nimpl ErrorConvert<()> for () {\n  fn convert(self) {}\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"std\")))]\n/// Helper trait to show a byte slice as a hex dump\npub trait HexDisplay {\n  /// Converts the value of `self` to a hex dump, returning the owned\n  /// `String`.\n  fn to_hex(&self, chunk_size: usize) -> String;\n\n  /// Converts the value of `self` to a hex dump beginning at `from` address, returning the owned\n  /// `String`.\n  fn to_hex_from(&self, chunk_size: usize, from: usize) -> String;\n}\n\n#[cfg(feature = \"std\")]\nstatic CHARS: &[u8] = b\"0123456789abcdef\";\n\n#[cfg(feature = \"std\")]\nimpl HexDisplay for [u8] {\n  #[allow(unused_variables)]\n  fn to_hex(&self, chunk_size: usize) -> String {\n    self.to_hex_from(chunk_size, 0)\n  }\n\n  #[allow(unused_variables)]\n  fn to_hex_from(&self, chunk_size: usize, from: usize) -> String {\n    let mut v = Vec::with_capacity(self.len() * 3);\n    let mut i = from;\n    for chunk in self.chunks(chunk_size) {\n      let s = format!(\"{:08x}\", i);\n      for &ch in s.as_bytes().iter() {\n        v.push(ch);\n      }\n      v.push(b'\\t');\n\n      i += chunk_size;\n\n      for &byte in chunk {\n        v.push(CHARS[(byte >> 4) as usize]);\n        v.push(CHARS[(byte & 0xf) as usize]);\n        v.push(b' ');\n      }\n      if chunk_size > chunk.len() {\n        for j in 0..(chunk_size - chunk.len()) {\n          v.push(b' ');\n          v.push(b' ');\n          v.push(b' ');\n        }\n      }\n      v.push(b'\\t');\n\n      for &byte in chunk {\n        if matches!(byte, 32..=126 | 128..=255) {\n          v.push(byte);\n        } else {\n          v.push(b'.');\n        }\n      }\n      v.push(b'\\n');\n    }\n\n    String::from_utf8_lossy(&v[..]).into_owned()\n  }\n}\n\n#[cfg(feature = \"std\")]\nimpl HexDisplay for str {\n  #[allow(unused_variables)]\n  fn to_hex(&self, chunk_size: usize) -> String {\n    self.to_hex_from(chunk_size, 0)\n  }\n\n  #[allow(unused_variables)]\n  fn to_hex_from(&self, chunk_size: usize, from: usize) -> String {\n    self.as_bytes().to_hex_from(chunk_size, from)\n  }\n}\n\n/// A saturating iterator for usize.\npub struct SaturatingIterator {\n  count: usize,\n}\n\nimpl Iterator for SaturatingIterator {\n  type Item = usize;\n\n  fn next(&mut self) -> Option<Self::Item> {\n    let old_count = self.count;\n    self.count = self.count.saturating_add(1);\n    Some(old_count)\n  }\n}\n\n/// Abstractions for range-like types.\npub trait NomRange<Idx> {\n  /// The saturating iterator type.\n  type Saturating: Iterator<Item = Idx>;\n  /// The bounded iterator type.\n  type Bounded: Iterator<Item = Idx>;\n\n  /// `true` if `item` is contained in the range.\n  fn contains(&self, item: &Idx) -> bool;\n\n  /// Returns the bounds of this range.\n  fn bounds(&self) -> (Bound<Idx>, Bound<Idx>);\n\n  /// `true` if the range is inverted.\n  fn is_inverted(&self) -> bool;\n\n  /// Creates a saturating iterator.\n  /// A saturating iterator counts the number of iterations starting from 0 up to the upper bound of this range.\n  /// If the upper bound is infinite the iterator saturates at the largest representable value of its type and\n  /// returns it for all further elements.\n  fn saturating_iter(&self) -> Self::Saturating;\n\n  /// Creates a bounded iterator.\n  /// A bounded iterator counts the number of iterations starting from 0 up to the upper bound of this range.\n  /// If the upper bounds is infinite the iterator counts up until the amount of iterations has reached the\n  /// largest representable value of its type and then returns `None` for all further elements.\n  fn bounded_iter(&self) -> Self::Bounded;\n}\n\nimpl NomRange<usize> for Range<usize> {\n  type Saturating = Range<usize>;\n  type Bounded = Range<usize>;\n\n  fn bounds(&self) -> (Bound<usize>, Bound<usize>) {\n    (Bound::Included(self.start), Bound::Excluded(self.end))\n  }\n\n  fn contains(&self, item: &usize) -> bool {\n    RangeBounds::contains(self, item)\n  }\n\n  fn is_inverted(&self) -> bool {\n    self.start >= self.end\n  }\n\n  fn saturating_iter(&self) -> Self::Saturating {\n    if self.end == 0 {\n      Range::default()\n    } else {\n      0..self.end - 1\n    }\n  }\n\n  fn bounded_iter(&self) -> Self::Bounded {\n    if self.end == 0 {\n      Range::default()\n    } else {\n      0..self.end - 1\n    }\n  }\n}\n\nimpl NomRange<usize> for RangeInclusive<usize> {\n  type Saturating = Range<usize>;\n  type Bounded = Range<usize>;\n\n  fn bounds(&self) -> (Bound<usize>, Bound<usize>) {\n    (Bound::Included(*self.start()), Bound::Included(*self.end()))\n  }\n\n  fn contains(&self, item: &usize) -> bool {\n    RangeBounds::contains(self, item)\n  }\n\n  fn is_inverted(&self) -> bool {\n    !RangeInclusive::contains(self, self.start())\n  }\n\n  fn saturating_iter(&self) -> Self::Saturating {\n    0..*self.end()\n  }\n\n  fn bounded_iter(&self) -> Self::Bounded {\n    0..*self.end()\n  }\n}\n\nimpl NomRange<usize> for RangeFrom<usize> {\n  type Saturating = SaturatingIterator;\n  type Bounded = Range<usize>;\n\n  fn bounds(&self) -> (Bound<usize>, Bound<usize>) {\n    (Bound::Included(self.start), Bound::Unbounded)\n  }\n\n  fn contains(&self, item: &usize) -> bool {\n    RangeBounds::contains(self, item)\n  }\n\n  fn is_inverted(&self) -> bool {\n    false\n  }\n\n  fn saturating_iter(&self) -> Self::Saturating {\n    SaturatingIterator { count: 0 }\n  }\n\n  fn bounded_iter(&self) -> Self::Bounded {\n    0..core::usize::MAX\n  }\n}\n\nimpl NomRange<usize> for RangeTo<usize> {\n  type Saturating = Range<usize>;\n  type Bounded = Range<usize>;\n\n  fn bounds(&self) -> (Bound<usize>, Bound<usize>) {\n    (Bound::Unbounded, Bound::Excluded(self.end))\n  }\n\n  fn contains(&self, item: &usize) -> bool {\n    RangeBounds::contains(self, item)\n  }\n\n  fn is_inverted(&self) -> bool {\n    false\n  }\n\n  fn saturating_iter(&self) -> Self::Saturating {\n    if self.end == 0 {\n      Range::default()\n    } else {\n      0..self.end - 1\n    }\n  }\n\n  fn bounded_iter(&self) -> Self::Bounded {\n    if self.end == 0 {\n      Range::default()\n    } else {\n      0..self.end - 1\n    }\n  }\n}\n\nimpl NomRange<usize> for RangeToInclusive<usize> {\n  type Saturating = Range<usize>;\n  type Bounded = Range<usize>;\n\n  fn bounds(&self) -> (Bound<usize>, Bound<usize>) {\n    (Bound::Unbounded, Bound::Included(self.end))\n  }\n\n  fn contains(&self, item: &usize) -> bool {\n    RangeBounds::contains(self, item)\n  }\n\n  fn is_inverted(&self) -> bool {\n    false\n  }\n\n  fn saturating_iter(&self) -> Self::Saturating {\n    0..self.end\n  }\n\n  fn bounded_iter(&self) -> Self::Bounded {\n    0..self.end\n  }\n}\n\nimpl NomRange<usize> for RangeFull {\n  type Saturating = SaturatingIterator;\n  type Bounded = Range<usize>;\n\n  fn bounds(&self) -> (Bound<usize>, Bound<usize>) {\n    (Bound::Unbounded, Bound::Unbounded)\n  }\n\n  fn contains(&self, item: &usize) -> bool {\n    RangeBounds::contains(self, item)\n  }\n\n  fn is_inverted(&self) -> bool {\n    false\n  }\n\n  fn saturating_iter(&self) -> Self::Saturating {\n    SaturatingIterator { count: 0 }\n  }\n\n  fn bounded_iter(&self) -> Self::Bounded {\n    0..core::usize::MAX\n  }\n}\n\nimpl NomRange<usize> for usize {\n  type Saturating = Range<usize>;\n  type Bounded = Range<usize>;\n\n  fn bounds(&self) -> (Bound<usize>, Bound<usize>) {\n    (Bound::Included(*self), Bound::Included(*self))\n  }\n\n  fn contains(&self, item: &usize) -> bool {\n    self == item\n  }\n\n  fn is_inverted(&self) -> bool {\n    false\n  }\n\n  fn saturating_iter(&self) -> Self::Saturating {\n    0..*self\n  }\n\n  fn bounded_iter(&self) -> Self::Bounded {\n    0..*self\n  }\n}\n\n#[cfg(test)]\nmod tests {\n  use super::*;\n\n  #[test]\n  \nfn test_offset_u8() {\n    let s = b\"abcd123\";\n    let a = &s[..];\n    let b = &a[2..];\n    let c = &a[..4];\n    let d = &a[3..5];\n    assert_eq!(a.offset(b), 2);\n    assert_eq!(a.offset(c), 0);\n    assert_eq!(a.offset(d), 3);\n  }\n\n  #[test]\n  fn test_offset_str() {\n    let a = \"abc\u0159\u00e8\u00c2\u00df\u00c7d123\";\n    let b = &a[7..];\n    let c = &a[..5];\n    let d = &a[5..9];\n    assert_eq!(a.offset(b), 7);\n    assert_eq!(a.offset(c), 0);\n    assert_eq!(a.offset(d), 5);\n  }\n\n  #[test]\n  fn test_slice_index() {\n    let a = \"abc\u0159\u00e8\u00c2\u00df\u00c7d123\";\n    assert_eq!(a.slice_index(0), Ok(0));\n    assert_eq!(a.slice_index(2), Ok(2));\n  }\n\n  #[test]\n  fn test_slice_index_utf8() {\n    let a = \"a\u00a1\u20ac\ud83d\udca2\u20ac\u00a1a\";\n\n    for (c, len) in a.chars().zip([1, 2, 3, 4, 3, 2, 1]) {\n      assert_eq!(c.len(), len);\n    }\n\n    assert_eq!(a.slice_index(0), Ok(0));\n    assert_eq!(a.slice_index(1), Ok(1));\n    assert_eq!(a.slice_index(2), Ok(3));\n    assert_eq!(a.slice_index(3), Ok(6));\n    assert_eq!(a.slice_index(4), Ok(10));\n    assert_eq!(a.slice_index(5), Ok(13));\n    assert_eq!(a.slice_index(6), Ok(15));\n    assert_eq!(a.slice_index(7), Ok(16));\n\n    assert!(a.slice_index(8).is_err());\n  }\n}\n\n// Path: src/bytes/complete.rs\n//! Parsers recognizing bytes streams, complete input version\n\nuse core::marker::PhantomData;\n\nuse crate::error::ParseError;\nuse crate::internal::{IResult, Parser};\nuse crate::traits::{Compare, FindSubstring, FindToken, InputLength, ToUsize};\nuse crate::Complete;\nuse crate::Emit;\nuse crate::Input;\nuse crate::OutputM;\n\n/// Recognizes a pattern\n///\n/// The input data will be compared to the tag combinator's argument and will return the part of\n/// the input that matches the argument\n///\n/// It will return `Err(Err::Error((_, ErrorKind::Tag)))` if the input doesn't match the pattern\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, &str> {\n///   tag(\"Hello\")(s)\n/// }\n///\n/// assert_eq!(parser(\"Hello, World!\"), Ok((\", World!\", \"Hello\")));\n/// assert_eq!(parser(\"Something\"), Err(Err::Error(Error::new(\"Something\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// ```\npub fn tag<T, I, Error: ParseError<I>>(tag: T) -> impl Fn(I) -> IResult<I, I, Error>\nwhere\n  I: Input + Compare<T>,\n  T: InputLength + Clone,\n{\n  move |i: I| {\n    let mut parser = super::Tag {\n      tag: tag.clone(),\n      e: PhantomData,\n    };\n\n    parser.process::<OutputM<Emit, Emit, Complete>>(i)\n  }\n}\n\n/// Recognizes a case insensitive pattern.\n///\n/// The input data will be compared to the tag combinator's argument and will return the part of\n/// the input that matches the argument with no regard to case.\n///\n/// It will return `Err(Err::Error((_, ErrorKind::Tag)))` if the input doesn't match the pattern.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::tag_no_case;\n///\n/// fn parser(s: &str) -> IResult<&str, &str> {\n///   tag_no_case(\"hello\")(s)\n/// }\n///\n/// assert_eq!(parser(\"Hello, World!\"), Ok((\", World!\", \"Hello\")));\n/// assert_eq!(parser(\"hello, World!\"), Ok((\", World!\", \"hello\")));\n/// assert_eq!(parser(\"HeLlO, World!\"), Ok((\", World!\", \"HeLlO\")));\n/// assert_eq!(parser(\"Something\"), Err(Err::Error(Error::new(\"Something\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// ```\npub fn tag_no_case<T, I, Error: ParseError<I>>(tag: T) -> impl Fn(I) -> IResult<I, I, Error>\nwhere\n  I: Input + Compare<T>,\n  T: InputLength + Clone,\n{\n  move |i: I| {\n    let mut parser = super::TagNoCase {\n      tag: tag.clone(),\n      e: PhantomData,\n    };\n\n    parser.process::<OutputM<Emit, Emit, Complete>>(i)\n  }\n}\n\n/// Parse till certain characters are met.\n///\n/// The parser will return the longest slice till one of the characters of the combinator's argument are met.\n///\n/// It doesn't consume the matched character.\n///\n/// It will return a `Err::Error((\"\", ErrorKind::IsNot))` if the pattern wasn't met.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::is_not;\n///\n/// fn not_space(s: &str) -> IResult<&str, &str> {\n///   is_not(\" \\t\\r\\n\")(s)\n/// }\n///\n/// assert_eq!(not_space(\"Hello, World!\"), Ok((\" World!\", \"Hello,\")));\n/// assert_eq!(not_space(\"Sometimes\\t\"), Ok((\"\\t\", \"Sometimes\")));\n/// assert_eq!(not_space(\"Nospace\"), Ok((\"\", \"Nospace\")));\n/// assert_eq!(not_space(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::IsNot))));\n/// ```\npub fn is_not<T, I, Error: ParseError<I>>(arr: T) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  T: FindToken<<I as Input>::Item>,\n{\n  let mut parser = super::is_not(arr);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the longest slice of the matches the pattern.\n///\n/// The parser will return the longest slice consisting of the characters in provided in the\n/// combinator's argument.\n///\n/// It will return a `Err(Err::Error((_, ErrorKind::IsA)))` if the pattern wasn't met.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::is_a;\n///\n/// fn hex(s: &str) -> IResult<&str, &str> {\n///   is_a(\"1234567890ABCDEF\")(s)\n/// }\n///\n/// assert_eq!(hex(\"123 and voila\"), Ok((\" and voila\", \"123\")));\n/// assert_eq!(hex(\"DEADBEEF and others\"), Ok((\" and others\", \"DEADBEEF\")));\n/// assert_eq!(hex(\"BADBABEsomething\"), Ok((\"something\", \"BADBABE\")));\n/// assert_eq!(hex(\"D15EA5E\"), Ok((\"\", \"D15EA5E\")));\n/// assert_eq!(hex(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::IsA))));\n/// ```\npub fn is_a<T, I, Error: ParseError<I>>(arr: T) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  T: FindToken<<I as Input>::Item>,\n{\n  let mut parser = super::is_a(arr);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the longest input slice (if any) that matches the predicate.\n///\n/// The parser will return the longest slice that matches the given predicate *(a function that\n/// takes the input and returns a bool)*.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// use nom::bytes::complete::take_while;\n/// use nom::character::is_alphabetic;\n///\n/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   take_while(is_alphabetic)(s)\n/// }\n///\n/// assert_eq!(alpha(b\"latin123\"), Ok((&b\"123\"[..], &b\"latin\"[..])));\n/// assert_eq!(alpha(b\"12345\"), Ok((&b\"12345\"[..], &b\"\"[..])));\n/// assert_eq!(alpha(b\"latin\"), Ok((&b\"\"[..], &b\"latin\"[..])));\n/// assert_eq!(alpha(b\"\"), Ok((&b\"\"[..], &b\"\"[..])));\n/// ```\npub fn take_while<F, I, Error: ParseError<I>>(cond: F) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  let mut parser = super::take_while(cond);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the longest (at least 1) input slice that matches the predicate.\n///\n/// The parser will return the longest slice that matches the given predicate *(a function that\n/// takes the input and returns a bool)*.\n///\n/// It will return an `Err(Err::Error((_, ErrorKind::TakeWhile1)))` if the pattern wasn't met.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::take_while1;\n/// use nom::character::is_alphabetic;\n///\n/// fn alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   take_while1(is_alphabetic)(s)\n/// }\n///\n/// assert_eq!(alpha(b\"latin123\"), Ok((&b\"123\"[..], &b\"latin\"[..])));\n/// assert_eq!(alpha(b\"latin\"), Ok((&b\"\"[..], &b\"latin\"[..])));\n/// assert_eq!(alpha(b\"12345\"), Err(Err::Error(Error::new(&b\"12345\"[..], ErrorKind::TakeWhile1))));\n/// ```\npub fn take_while1<F, I, Error: ParseError<I>>(cond: F) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  let mut parser = super::take_while1(cond);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the longest (m <= len <= n) input slice that matches the predicate.\n///\n/// The parser will return the longest slice that matches the given predicate *(a function that\n/// takes the input and returns a bool)*.\n///\n/// It will return an `Err::Error((_, ErrorKind::TakeWhileMN))` if the pattern wasn't met or is out\n/// of range (m <= len <= n).\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::take_while_m_n;\n/// use nom::character::is_alphabetic;\n///\n/// fn short_alpha(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   take_while_m_n(3, 6, is_alphabetic)(s)\n/// }\n///\n/// assert_eq!(short_alpha(b\"latin123\"), Ok((&b\"123\"[..], &b\"latin\"[..])));\n/// assert_eq!(short_alpha(b\"lengthy\"), Ok((&b\"y\"[..], &b\"length\"[..])));\n/// assert_eq!(short_alpha(b\"latin\"), Ok((&b\"\"[..], &b\"latin\"[..])));\n/// assert_eq!(short_alpha(b\"ed\"), Err(Err::Error(Error::new(&b\"ed\"[..], ErrorKind::TakeWhileMN))));\n/// assert_eq!(short_alpha(b\"12345\"), Err(Err::Error(Error::new(&b\"12345\"[..], ErrorKind::TakeWhileMN))));\n/// ```\npub fn take_while_m_n<F, I, Error: ParseError<I>>(\n  m: usize,\n  n: usize,\n  cond: F,\n) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  let mut parser = super::take_while_m_n(m, n, cond);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the longest input slice (if any) till a predicate is met.\n///\n/// The parser will return the longest slice till the given predicate *(a function that\n/// takes the input and returns a bool)*.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// use nom::bytes::complete::take_till;\n///\n/// fn till_colon(s: &str) -> IResult<&str, &str> {\n///   take_till(|c| c == ':')(s)\n/// }\n///\n/// assert_eq!(till_colon(\"latin:123\"), Ok((\":123\", \"latin\")));\n/// assert_eq!(till_colon(\":empty matched\"), Ok((\":empty matched\", \"\"))); //allowed\n/// assert_eq!(till_colon(\"12345\"), Ok((\"\", \"12345\")));\n/// assert_eq!(till_colon(\"\"), Ok((\"\", \"\")));\n/// ```\n#[allow(clippy::redundant_closure)]\npub fn take_till<F, I, Error: ParseError<I>>(cond: F) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  let mut parser = super::take_till(cond);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the longest (at least 1) input slice till a predicate is met.\n///\n/// The parser will return the longest slice till the given predicate *(a function that\n/// takes the input and returns a bool)*.\n///\n/// It will return `Err(Err::Error((_, ErrorKind::TakeTill1)))` if the input is empty or the\n/// predicate matches the first input.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::take_till1;\n///\n/// fn till_colon(s: &str) -> IResult<&str, &str> {\n///   take_till1(|c| c == ':')(s)\n/// }\n///\n/// assert_eq!(till_colon(\"latin:123\"), Ok((\":123\", \"latin\")));\n/// assert_eq!(till_colon(\":empty matched\"), Err(Err::Error(Error::new(\":empty matched\", ErrorKind::TakeTill1))));\n/// assert_eq!(till_colon(\"12345\"), Ok((\"\", \"12345\")));\n/// assert_eq!(till_colon(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::TakeTill1))));\n/// ```\n#[allow(clippy::redundant_closure)]\npub fn take_till1<F, I, Error: ParseError<I>>(cond: F) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  F: Fn(<I as Input>::Item) -> bool,\n{\n  let mut parser = super::take_till1(cond);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns an input slice containing the first N input elements (Input[..N]).\n///\n/// It will return `Err(Err::Error((_, ErrorKind::Eof)))` if the input is shorter than the argument.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::take;\n///\n/// fn take6(s: &str) -> IResult<&str, &str> {\n///   take(6usize)(s)\n/// }\n///\n/// assert_eq!(take6(\"1234567\"), Ok((\"7\", \"123456\")));\n/// assert_eq!(take6(\"things\"), Ok((\"\", \"things\")));\n/// assert_eq!(take6(\"short\"), Err(Err::Error(Error::new(\"short\", ErrorKind::Eof))));\n/// assert_eq!(take6(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Eof))));\n/// ```\n///\n/// The units that are taken will depend on the input type. For example, for a\n/// `&str` it will take a number of `char`'s, whereas for a `&[u8]` it will\n/// take that many `u8`'s:\n///\n/// ```rust\n/// use nom::error::Error;\n/// use nom::bytes::complete::take;\n///\n/// assert_eq!(take::<_, _, Error<_>>(1usize)(\"\ud83d\udc99\"), Ok((\"\", \"\ud83d\udc99\")));\n/// assert_eq!(take::<_, _, Error<_>>(1usize)(\"\ud83d\udc99\".as_bytes()), Ok((b\"\\x9F\\x92\\x99\".as_ref(), b\"\\xF0\".as_ref())));\n/// ```\npub fn take<C, I, Error: ParseError<I>>(count: C) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input,\n  C: ToUsize,\n{\n  let mut parser = super::take(count);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the input slice up to the first occurrence of the pattern.\n///\n/// It doesn't consume the pattern. It will return `Err(Err::Error((_, ErrorKind::TakeUntil)))`\n/// if the pattern wasn't met.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::take_until;\n///\n/// fn until_eof(s: &str) -> IResult<&str, &str> {\n///   take_until(\"eof\")(s)\n/// }\n///\n/// assert_eq!(until_eof(\"hello, worldeof\"), Ok((\"eof\", \"hello, world\")));\n/// assert_eq!(until_eof(\"hello, world\"), Err(Err::Error(Error::new(\"hello, world\", ErrorKind::TakeUntil))));\n/// assert_eq!(until_eof(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::TakeUntil))));\n/// assert_eq!(until_eof(\"1eof2eof\"), Ok((\"eof2eof\", \"1\")));\n/// ```\npub fn take_until<T, I, Error: ParseError<I>>(tag: T) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input + FindSubstring<T>,\n  T: InputLength + Clone,\n{\n  let mut parser = super::take_until(tag);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Returns the non empty input slice up to the first occurrence of the pattern.\n///\n/// It doesn't consume the pattern. It will return `Err(Err::Error((_, ErrorKind::TakeUntil)))`\n/// if the pattern wasn't met.\n/// # Example\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult};\n/// use nom::bytes::complete::take_until1;\n///\n/// fn until_eof(s: &str) -> IResult<&str, &str> {\n///   take_until1(\"eof\")(s)\n/// }\n///\n/// assert_eq!(until_eof(\"hello, worldeof\"), Ok((\"eof\", \"hello, world\")));\n/// assert_eq!(until_eof(\"hello, world\"), Err(Err::Error(Error::new(\"hello, world\", ErrorKind::TakeUntil))));\n/// assert_eq!(until_eof(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::TakeUntil))));\n/// assert_eq!(until_eof(\"1eof2eof\"), Ok((\"eof2eof\", \"1\")));\n/// assert_eq!(until_eof(\"eof\"), Err(Err::Error(Error::new(\"eof\", ErrorKind::TakeUntil))));\n/// ```\npub fn take_until1<T, I, Error: ParseError<I>>(tag: T) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Input + FindSubstring<T>,\n  T: InputLength + Clone,\n{\n  let mut parser = super::take_until1(tag);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Matches a byte string with escaped characters.\n///\n/// * The first argument matches the normal characters (it must not accept the control character)\n/// * The second argument is the control character (like `\\` in most languages)\n/// * The third argument matches the escaped characters\n/// # Example\n/// ```\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// # use nom::character::complete::digit1;\n/// use nom::bytes::complete::escaped;\n/// use nom::character::complete::one_of;\n///\n/// fn esc(s: &str) -> IResult<&str, &str> {\n///   escaped(digit1, '\\\\', one_of(r#\"\"n\\\"#))(s)\n/// }\n///\n/// assert_eq!(esc(\"123;\"), Ok((\";\", \"123\")));\n/// assert_eq!(esc(r#\"12\\\"34;\"#), Ok((\";\", r#\"12\\\"34\"#)));\n/// ```\n///\npub fn escaped<'a, I: 'a, Error, F, G>(\n  normal: F,\n  control_char: char,\n  escapable: G,\n) -> impl FnMut(I) -> IResult<I, I, Error>\nwhere\n  I: Clone + crate::traits::Offset + Input,\n  <I as Input>::Item: crate::traits::AsChar,\n  F: Parser<I, Error = Error>,\n  G: Parser<I, Error = Error>,\n  Error: ParseError<I>,\n{\n  let mut parser = super::escaped(normal, control_char, escapable);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n/// Matches a byte string with escaped characters.\n///\n/// * The first argument matches the normal characters (it must not match the control character)\n/// * The second argument is the control character (like `\\` in most languages)\n/// * The third argument matches the escaped characters and transforms them\n///\n/// As an example, the chain `abc\\tdef` could be `abc    def` (it also consumes the control character)\n///\n/// ```\n/// # use nom::{Err, error::ErrorKind, Needed, IResult};\n/// # use std::str::from_utf8;\n/// use nom::bytes::complete::{escaped_transform, tag};\n/// use nom::character::complete::alpha1;\n/// use nom::branch::alt;\n/// use nom::combinator::value;\n///\n/// fn parser(input: &str) -> IResult<&str, String> {\n///   escaped_transform(\n///     alpha1,\n///     '\\\\',\n///     alt((\n///       value(\"\\\\\", tag(\"\\\\\")),\n///       value(\"\\\"\", tag(\"\\\"\")),\n///       value(\"\\n\", tag(\"n\")),\n///     ))\n///   )(input)\n/// }\n///\n/// assert_eq!(parser(\"ab\\\\\\\"cd\"), Ok((\"\", String::from(\"ab\\\"cd\"))));\n/// assert_eq!(parser(\"ab\\\\ncd\"), Ok((\"\", String::from(\"ab\\ncd\"))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn escaped_transform<I, Error, F, G, O1, O2, ExtendItem, Output>(\n  normal: F,\n  control_char: char,\n  transform: G,\n) -> impl FnMut(I) -> IResult<I, Output, Error>\nwhere\n  I: Clone + crate::traits::Offset + Input,\n  I: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  O1: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  O2: crate::traits::ExtendInto<Item = ExtendItem, Extender = Output>,\n  <I as Input>::Item: crate::traits::AsChar,\n  F: Parser<I, Output = O1, Error = Error>,\n  G: Parser<I, Output = O2, Error = Error>,\n  Error: ParseError<I>,\n{\n  let mut parser = super::escaped_transform(normal, control_char, transform);\n\n  move |i: I| parser.process::<OutputM<Emit, Emit, Complete>>(i)\n}\n\n#[cfg(test)]\nmod tests {\n  use super::*;\n  use crate::error::ErrorKind;\n  use crate::AsChar;\n  use crate::Err;\n\n  #[test]\n  fn complete_take_while_m_n_utf8_all_matching() {\n    let result: IResult<&str, &str> =\n      super::take_while_m_n(1, 4, |c: char| c.is_alphabetic())(\"\u00f8n\");\n    assert_eq!(result, Ok((\"\", \"\u00f8n\")));\n  }\n\n  #[test]\n  fn complete_take_while_m_n_utf8_all_matching_substring() {\n    let result: IResult<&str, &str> =\n      super::take_while_m_n(1, 1, |c: char| c.is_alphabetic())(\"\u00f8n\");\n    assert_eq!(result, Ok((\"n\", \"\u00f8\")));\n  }\n\n  // issue #1336 \"escaped hangs if normal parser accepts empty\"\n  fn escaped_string(input: &str) -> IResult<&str, &str> {\n    use crate::character::complete::{alpha0, one_of};\n    escaped(alpha0, '\\\\', one_of(\"n\"))(input)\n  }\n\n  // issue #1336 \"escaped hangs if normal parser accepts empty\"\n  #[test]\n  fn escaped_hang() {\n    escaped_string(\"7\").unwrap();\n    escaped_string(\"a7\").unwrap();\n  }\n\n  // issue ##1118 escaped does not work with empty string\n  fn unquote(input: &str) -> IResult<&str, &str> {\n    use crate::bytes::complete::*;\n    use crate::character::complete::*;\n    use crate::combinator::opt;\n    use crate::sequence::delimited;\n\n    delimited(\n      char('\"'),\n      escaped(opt(none_of(r#\"\\\"\"#)), '\\\\', one_of(r#\"\\\"rnt\"#)),\n      char('\"'),\n    )\n    .parse(input)\n  }\n\n  #[test]\n  fn escaped_hang_1118() {\n    assert_eq!(unquote(r#\"\"\"\"#), Ok((\"\", \"\")));\n  }\n\n  // issue #1630 take_while_m_n is invalid for multi-byte UTF-8 characters\n  #[test]\n  fn complete_take_while_m_n_multibyte() {\n    use crate::error::Error;\n\n    fn multi_byte_chars(s: &str, m: usize, n: usize) -> IResult<&str, &str> {\n      take_while_m_n(m, n, |c: char| c.len() > 1)(s)\n    }\n\n    assert_eq!(multi_byte_chars(\"\u20ac latin\", 0, 64), Ok((\" latin\", \"\u20ac\")));\n    assert_eq!(multi_byte_chars(\"\ud834\udd20 latin\", 0, 1), Ok((\" latin\", \"\ud834\udd20\")));\n    assert_eq!(multi_byte_chars(\"\u0628\u0627\u0628 latin\", 0, 64), Ok((\" latin\", \"\u0628\u0627\u0628\")));\n    assert_eq!(\n      multi_byte_chars(\"\ud83d\udca3\ud83d\udca2\u1fa0 latin\", 3, 3),\n      Ok((\" latin\", \"\ud83d\udca3\ud83d\udca2\u1fa0\"))\n    );\n    assert_eq!(multi_byte_chars(\"latin\", 0, 64), Ok((\"latin\", \"\")));\n    assert_eq!(multi_byte_chars(\"\u0628\u0627\u0628\", 1, 3), Ok((\"\", \"\u0628\u0627\u0628\")));\n    assert_eq!(multi_byte_chars(\"\u0628\u0627\u0628\", 1, 2), Ok((\"\u0628\", \"\u0628\u0627\")));\n    assert_eq!(\n      multi_byte_chars(\"latin\", 1, 64),\n      Err(Err::Error(Error::new(\"latin\", ErrorKind::TakeWhileMN)))\n    );\n  }\n}\n\n// Path: src/sequence/tests.rs\nuse super::*;\nuse crate::bytes::streaming::{tag, take};\nuse crate::error::{Error, ErrorKind};\nuse crate::internal::{Err, IResult, Needed};\nuse crate::number::streaming::be_u16;\n\n#[test]\nfn single_element_tuples() {\n  use crate::character::complete::alpha1;\n  use crate::{error::ErrorKind, Err};\n\n  let mut parser = (alpha1,);\n  assert_eq!(\n    crate::Parser::parse(&mut parser, \"abc123def\"),\n    Ok((\"123def\", (\"abc\",)))\n  );\n  assert_eq!(\n    crate::Parser::parse(&mut parser, \"123def\"),\n    Err(Err::Error((\"123def\", ErrorKind::Alpha)))\n  );\n}\n\n#[derive(PartialEq, Eq, Debug)]\nstruct B {\n  a: u8,\n  b: u8,\n}\n\n#[derive(PartialEq, Eq, Debug)]\nstruct C {\n  a: u8,\n  b: Option<u8>,\n}\n\n/*FIXME: convert code examples to new error management\nuse util::{add_error_pattern, error_to_list, print_error};\n\n#[cfg(feature = \"std\")]\n#[rustfmt::skip]\nfn error_to_string<P: Clone + PartialEq>(e: &Context<P, u32>) -> &'static str {\n  let v: Vec<(P, ErrorKind<u32>)> = error_to_list(e);\n  // do it this way if you can use slice patterns\n  //match &v[..] {\n  //  [ErrorKind::Custom(42), ErrorKind::Tag]                         => \"missing `ijkl` tag\",\n  //  [ErrorKind::Custom(42), ErrorKind::Custom(128), ErrorKind::Tag] => \"missing `mnop` tag after `ijkl`\",\n  //  _            => \"unrecognized error\"\n  //}\n\n  let collected: Vec<ErrorKind<u32>> = v.iter().map(|&(_, ref e)| e.clone()).collect();\n  if &collected[..] == [ErrorKind::Custom(42), ErrorKind::Tag] {\n    \"missing `ijkl` tag\"\n  } else if &collected[..] == [ErrorKind::Custom(42), ErrorKind::Custom(128), ErrorKind::Tag] {\n    \"missing `mnop` tag after `ijkl`\"\n  } else {\n    \"unrecognized error\"\n  }\n}\n\n// do it this way if you can use box patterns\n//use $crate::lib::std::str;\n//fn error_to_string(e:Err) -> String\n//  match e {\n//    NodePosition(ErrorKind::Custom(42), i1, box Position(ErrorKind::Tag, i2)) => {\n//      format!(\"missing `ijkl` tag, found '{}' instead\", str::from_utf8(i2).unwrap())\n//    },\n//    NodePosition(ErrorKind::Custom(42), i1, box NodePosition(ErrorKind::Custom(128), i2,  box Position(ErrorKind::Tag, i3))) => {\n//      format!(\"missing `mnop` tag after `ijkl`, found '{}' instead\", str::from_utf8(i3).unwrap())\n//    },\n//    _ => \"unrecognized error\".to_string()\n//  }\n//}\n*/\n\n#[test]\nfn complete() {\n  use crate::bytes::complete::tag;\n  fn err_test(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    let (i, _) = tag(\"ijkl\")(i)?;\n    tag(\"mnop\")(i)\n  }\n  let a = &b\"ijklmn\"[..];\n\n  let res_a = err_test(a);\n  assert_eq!(\n    res_a,\n    Err(Err::Error(error_position!(&b\"mn\"[..], ErrorKind::Tag)))\n  );\n}\n\n#[test]\nfn pair_test() {\n  fn pair_abc_def(i: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> {\n    pair(tag(\"abc\"), tag(\"def\")).parse(i)\n  }\n\n  assert_eq!(\n    pair_abc_def(&b\"abcdefghijkl\"[..]),\n    Ok((&b\"ghijkl\"[..], (&b\"abc\"[..], &b\"def\"[..])))\n  );\n  assert_eq!(\n    pair_abc_def(&b\"ab\"[..]),\n    Err(Err::Incomplete(Needed::new(1)))\n  );\n  assert_eq!(\n    pair_abc_def(&b\"abcd\"[..]),\n    Err(Err::Incomplete(Needed::new(2)))\n  );\n  assert_eq!(\n    pair_abc_def(&b\"xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    pair_abc_def(&b\"xxxdef\"[..]),\n    Err(Err::Error(error_position!(&b\"xxxdef\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    pair_abc_def(&b\"abcxxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n}\n\n#[test]\nfn separated_pair_test() {\n  fn sep_pair_abc_def(i: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> {\n    separated_pair(tag(\"abc\"), tag(\",\"), tag(\"def\")).parse(i)\n  }\n\n  assert_eq!(\n    sep_pair_abc_def(&b\"abc,defghijkl\"[..]),\n    Ok((&b\"ghijkl\"[..], (&b\"abc\"[..], &b\"def\"[..])))\n  );\n  assert_eq!(\n    sep_pair_abc_def(&b\"ab\"[..]),\n    Err(Err::Incomplete(Needed::new(1)))\n  );\n  assert_eq!(\n    sep_pair_abc_def(&b\"abc,d\"[..]),\n    Err(Err::Incomplete(Needed::new(2)))\n  );\n  assert_eq!(\n    sep_pair_abc_def(&b\"xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    sep_pair_abc_def(&b\"xxx,def\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx,def\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    sep_pair_abc_def(&b\"abc,xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n}\n\n#[test]\nfn preceded_test() {\n  fn preceded_abcd_efgh(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    preceded(tag(\"abcd\"), tag(\"efgh\")).parse(i)\n  }\n\n  assert_eq!(\n    preceded_abcd_efgh(&b\"abcdefghijkl\"[..]),\n    Ok((&b\"ijkl\"[..], &b\"efgh\"[..]))\n  );\n  assert_eq!(\n    preceded_abcd_efgh(&b\"ab\"[..]),\n    Err(Err::Incomplete(Needed::new(2)))\n  );\n  assert_eq!(\n    preceded_abcd_efgh(&b\"abcde\"[..]),\n    Err(Err::Incomplete(Needed::new(3)))\n  );\n  assert_eq!(\n    preceded_abcd_efgh(&b\"xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    preceded_abcd_efgh(&b\"xxxxdef\"[..]),\n    Err(Err::Error(error_position!(&b\"xxxxdef\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    preceded_abcd_efgh(&b\"abcdxxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n}\n\n#[test]\nfn terminated_test() {\n  fn terminated_abcd_efgh(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    terminated(tag(\"abcd\"), tag(\"efgh\")).parse(i)\n  }\n\n  assert_eq!(\n    terminated_abcd_efgh(&b\"abcdefghijkl\"[..]),\n    Ok((&b\"ijkl\"[..], &b\"abcd\"[..]))\n  );\n  assert_eq!(\n    terminated_abcd_efgh(&b\"ab\"[..]),\n    Err(Err::Incomplete(Needed::new(2)))\n  );\n  assert_eq!(\n    terminated_abcd_efgh(&b\"abcde\"[..]),\n    Err(Err::Incomplete(Needed::new(3)))\n  );\n  assert_eq!(\n    terminated_abcd_efgh(&b\"xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    terminated_abcd_efgh(&b\"xxxxdef\"[..]),\n    Err(Err::Error(error_position!(&b\"xxxxdef\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    terminated_abcd_efgh(&b\"abcdxxxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxxx\"[..], ErrorKind::Tag)))\n  );\n}\n\n#[test]\nfn delimited_test() {\n  fn delimited_abc_def_ghi(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    delimited(tag(\"abc\"), tag(\"def\"), tag(\"ghi\")).parse(i)\n  }\n\n  assert_eq!(\n    delimited_abc_def_ghi(&b\"abcdefghijkl\"[..]),\n    Ok((&b\"jkl\"[..], &b\"def\"[..]))\n  );\n  assert_eq!(\n    delimited_abc_def_ghi(&b\"ab\"[..]),\n...\n", "needle_token_start": 5582, "needle_token_end": 5696, "code_context_ntokens": 16229, "output": ["```rust\n#[cfg(test)]\nmod tests {\n  use super::*;\n\n  #[test]\n  \nfn test_offset_u8() {\n    let s = b\"abcd123\";\n    let a = &s[..];\n    let b = &a[2..];\n    let c = &a[..4];\n    let d = &a[3..5];\n    assert_eq!(a.offset(b), 2);\n    assert_eq!(a.offset(c), 0);\n    assert_eq!(a.offset(d), 3);\n  }\n\n  #[test]\n  fn test_offset_str() {\n    let a = \"abc\u0159\u00e8\u00c2\u00df\u00c7d123\";\n    let b = &a[7..];\n    let c = &a[..5];\n    let d = &a[5..9];\n    assert_eq!(a.offset(b), 7);\n    assert_eq!(a.offset(c), 0);\n    assert_eq!(a.offset(d), 5);\n  }\n\n  #[test]\n  fn test_slice_index() {\n    let a = \"abc\u0159\u00e8\u00c2\u00df\u00c7d123\";\n    assert_eq!(a.slice_index(0), Ok(0));\n    assert_eq!(a.slice_index(2), Ok(2));\n  }\n\n  #[test]\n  fn test_slice_index_utf8() {\n    let a = \"a\u00a1\u20ac\ud83d\udca2\u20ac\u00a1a\";\n\n    for (c, len) in a.chars().zip([1, 2, 3, 4, 3, 2, 1]) {\n      assert_eq!(c.len(), len);\n    }\n\n    assert_eq!(a.slice_index(0), Ok(0));\n    assert_eq!(a.slice_index(1), Ok(1));\n    assert_eq!(a.slice_index(2), Ok(3));\n    assert_eq!(a.slice_index(3), Ok(6));\n    assert_eq!(a.slice_index(4), Ok(10));\n    assert_eq!(a.slice_index(5), Ok(13));\n    assert_eq!(a.slice_index(6), Ok(15));\n    assert_eq!(a.slice_index(7), Ok(16));\n\n    assert!(a.slice_index(8).is_err());\n  }\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "fold_many1", "language": "rust", "path": "src/multi/mod.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to repeatedly apply a given parser and accumulate the results using a specified function, ensuring that the parser is applied at least once.\n2. **Input**: It takes three parameters: a parser that defines how to parse each segment, an initializer function that provides the initial value for the accumulator, and a function that combines the accumulated result with the output of each successful parse.\n3. **Output**: Returns either the accumulated result of all successful parses or an error if the first parse attempt fails or if the parser accepts empty inputs, which could lead to an infinite loop.\n4. **Procedure**: The function initializes an accumulator with the provided initial value. It then enters a loop where it repeatedly applies the parser to the input. For each successful parse, it updates the accumulator using the combining function. The loop continues until the parser fails to parse the input, at which point it returns the accumulated result or an error if no successful parse has occurred.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "///\n/// # Arguments\n/// * `sep` Parses the separator between list elements.\n/// * `f` Parses the elements of the list.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::separated_list1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   separated_list1(tag(\"|\"), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abc|abc|abc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123abc\"), Ok((\"123abc\", vec![\"abc\"])));\n/// assert_eq!(parser(\"abc|def\"), Ok((\"|def\", vec![\"abc\"])));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"def|abc\"), Err(Err::Error(Error::new(\"def|abc\", ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn separated_list1<I, E, F, G>(\n  separator: G,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  SeparatedList1 { parser, separator }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [separated_list1] combinator\npub struct SeparatedList1<F, G> {\n  parser: F,\n  separator: G,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, E: ParseError<I>, F, G> Parser<I> for SeparatedList1<F, G>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n\n    match self.parser.process::<OM>(i.clone()) {\n      Err(e) => return Err(e),\n      Ok((i1, o)) => {\n        res = OM::Output::combine(res, o, |mut res, o| {\n          res.push(o);\n          res\n        });\n        i = i1;\n      }\n    }\n\n    loop {\n      let len = i.input_len();\n      match self\n        .separator\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, res)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, _)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::SeparatedList)\n            })));\n          }\n\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i1.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, res)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i2, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              i = i2;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times\n///\n/// This stops before `n` when the parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many_m_n(0, 2, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many_m_n<I, E, F>(\n  min: usize,\n  max: usize,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  ManyMN { parser, min, max }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many_m_n] combinator\npub struct ManyMN<F> {\n  parser: F,\n  min: usize,\n  max: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for ManyMN<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.min > self.max {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::ManyMN,\n      )));\n    }\n\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.min.min(max_initial_capacity))\n    });\n    for count in 0..self.max {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::ManyMN)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |mut res, value| {\n            res.push(value);\n            res\n          });\n          input = tail;\n        }\n        Err(Err::Error(e)) => {\n          if count < self.min {\n            return Err(Err::Error(OM::Error::map(e, |e| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::ManyMN, e)\n            })));\n          } else {\n            return Ok((input, res));\n          }\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Repeats the embedded parser, counting the results\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many0_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many0_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", 0)));\n/// assert_eq!(parser(\"\"), Ok((\"\", 0)));\n/// ```\npub fn many0_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many0Count { parser }\n}\n\n/// Parser implementation for the [many0_count] combinator\npub struct Many0Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many0Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    loop {\n      let input_ = input.clone();\n      let len = input.input_len();\n      match self\n        .parser\n        .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n      {\n        Ok((i, _)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0Count)\n            })));\n          }\n\n          input = i;\n          count += 1;\n        }\n\n        Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser, counting the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many1_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many1_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1Count))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1Count))));\n/// ```\npub fn many1_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many1Count { parser }\n}\n\n/// Parser implementation for the [many1_count] combinator\npub struct Many1Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many1Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    match self\n      .parser\n      .process::<OutputM<Check, Check, OM::Incomplete>>(input.clone())\n    {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(move || {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n      }))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e)),\n      Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),\n      Ok((mut input, _)) => {\n        count += 1;\n\n        loop {\n          let input_ = input.clone();\n          let len = input.input_len();\n          match self\n            .parser\n            .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n          {\n            Ok((i, _)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n                })));\n              }\n\n              input = i;\n              count += 1;\n            }\n\n            Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser `count` times, gathering the results in a `Vec`\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `count` How often to apply the parser.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   count(tag(\"abc\"), 2).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn count<I, F>(\n  parser: F,\n  count: usize,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  Count { parser, count }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [count] combinator\npub struct Count<F> {\n  parser: F,\n  count: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Count<F>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.count.min(max_initial_capacity))\n    });\n\n    for _ in 0..self.count {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          res = OM::Output::combine(res, o, |mut res, o| {\n            res.push(o);\n            res\n          });\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Runs the embedded parser repeatedly, filling the given slice with results.\n///\n/// This parser fails if the input runs out before the given slice is full.\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `buf` The slice to fill\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fill;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, [&str; 2]> {\n///   let mut buf = [\"\", \"\"];\n///   let (rest, ()) = fill(tag(\"abc\"), &mut buf).parse(s)?;\n///   Ok((rest, buf))\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", [\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", [\"abc\", \"abc\"])));\n/// ```\npub fn fill<'a, I, E, F>(\n  parser: F,\n  buf: &'a mut [<F as Parser<I>>::Output],\n) -> impl Parser<I, Output = (), Error = E> + 'a\nwhere\n  I: Clone,\n  F: Parser<I, Error = E> + 'a,\n  E: ParseError<I>,\n{\n  Fill { parser, buf }\n}\n\n/// Parser implementation for the [fill] combinator\npub struct Fill<'a, F, O> {\n  parser: F,\n  buf: &'a mut [O],\n}\n\nimpl<'a, I, F, O> Parser<I> for Fill<'a, F, O>\nwhere\n  I: Clone,\n  F: Parser<I, Output = O>,\n{\n  type Output = ();\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n\n    for elem in self.buf.iter_mut() {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          OM::Output::map(o, |o| *elem = o);\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, OM::Output::bind(|| ())))\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many0(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// ```\npub fn fold_many0<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany0 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many0] combinator\npub struct FoldMany0<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany0<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let mut input = i;\n\n    loop {\n      let i_ = input.clone();\n      let len = input.input_len();\n      match self.parser.process::<OM>(i_) {\n        Ok((i, o)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0)\n            })));\n          }\n\n          res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n          input = i;\n        }\n        Err(Err::Error(_)) => {\n          return Ok((input, res));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fold_many1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many1(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1))));\n/// ```\n\npub fn fold_many1<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany1 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many1] combinator\npub struct FoldMany1<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany1<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let input = i.clone();\n\n    match self.parser.process::<OM>(input) {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many1)\n      }))),\n      Err(e) => Err(e),\n      Ok((i1, o1)) => {\n        res = OM::Output::combine(res, o1, |res, o| (self.g)(res, o));\n\n        let mut input = i1;\n        loop {\n          let i_ = input.clone();\n          let len = input.input_len();\n          match self.parser.process::<OM>(i_) {\n            Ok((i, o)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1)\n                })));\n              }\n\n              res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n              input = i;\n            }\n            Err(Err::Error(_)) => {\n              return Ok((input, res));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times, calling `g` to gather the results\n///\n/// This stops before `n` when the parser returns [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many_m_n(\n///     0,\n///     2,\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\npub fn fold_many_m_n<I, E, F, G, H, R>(\n  min: usize,\n  max: usize,\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldManyMN {\n    parser,\n    g,\n    init,\n    min,\n    max,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many_m_n] combinator\npub struct FoldManyMN<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n  min: usize,\n  max: usize,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldManyMN<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.min > self.max {\n      return Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::ManyMN)\n      })));\n    }\n\n    let mut res = OM::Output::bind(|| (self.init)());\n    for count in 0..self.max {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(tail, ErrorKind::ManyMN)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |res, o| (self.g)(res, o));\n          input = tail;\n        }\n        Err(Err::Error(err)) => {\n          if count < self.min {\n            return Err(Err::Error(OM::Error::map(err, |err| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::ManyMN, err)\n            })));\n          } else {\n            break;\n          }\n        }\n        Err(e) => return Err(e),\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Gets a number from the parser and returns a\n/// subslice of the input of that size.\n/// If the parser returns `Incomplete`,\n/// `length_data` will return an error.\n/// # Arguments\n/// * `f` The parser to apply.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::number::complete::be_u16;\n/// use nom::multi::length_data;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   length_data(be_u16).parse(s)\n/// }\n///\n/// assert_eq!(parser(b\"\\x00\\x03abcefg\"), Ok((&b\"efg\"[..], &b\"abc\"[..])));\n/// assert_eq!(parser(b\"\\x00\\x03a\"), Err(Err::Incomplete(Needed::new(2))));\n/// ```\npub fn length_data<I, E, F>(f: F) -> impl Parser<I, Output = I, Error = E>\nwhere\n  I: Input,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  f.flat_map(|size| take(size))\n}\n\n/// Gets a number from the first parser,\n/// takes a subslice of the input of that size,\n/// then applies the second parser on that subslice.\n/// If the second parser returns `Incomplete`,\n/// `length_value` will return an error.\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `g` The parser to apply on the subslice.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::number::complete::be_u16;\n/// use nom::multi::length_value;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   length_value(be_u16, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(b\"\\x00\\x03abcefg\"), Ok((&b\"efg\"[..], &b\"abc\"[..])));\n/// assert_eq!(parser(b\"\\x00\\x03123123\"), Err(Err::Error(Error::new(&b\"123\"[..], ErrorKind::Tag))));\n/// assert_eq!(parser(b\"\\x00\\x03a\"), Err(Err::Incomplete(Needed::new(2))));\n/// ```\npub fn length_value<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = <G as Parser<I>>::Output, Error = E>\nwhere\n  I: Clone + Input,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  /*f.flat_map(|size| {\n    println!(\"got size: {size}\");\n    take(size)\n  })\n  .and_then(g)*/\n  LengthValue {\n    length: f,\n    parser: g,\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for the [length_value] combinator\npub struct LengthValue<F, G, E> {\n  length: F,\n  parser: G,\n  e: PhantomData<E>,\n}\n\nimpl<I, F, G, E> Parser<I> for LengthValue<F, G, E>\nwhere\n  I: Clone + Input,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  <F as Parser<I>>::Output: ToUsize,\n  E: ParseError<I>,\n{\n  type Output = <G as Parser<I>>::Output;\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let (i, length) = self\n      .length\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(input)?;\n\n    let length: usize = length.to_usize();\n\n    if let Some(needed) = length\n      .checked_sub(i.input_len())\n      .and_then(NonZeroUsize::new)\n    {\n      Err(Err::Incomplete(Needed::Size(needed)))\n    } else {\n      let (rest, i) = i.take_split(length);\n      match self.parser.process::<OM>(i.clone()) {\n        Err(Err::Incomplete(_)) => Err(Err::Error(OM::Error::bind(|| {\n          E::from_error_kind(i, ErrorKind::Complete)\n        }))),\n        Err(e) => Err(e),\n        Ok((_, o)) => Ok((rest, o)),\n      }\n    }\n  }\n}\n\n/// Gets a number from the first parser,\n/// then applies the second parser that many times.\n/// # Arguments\n/// * `f` The parser to apply to obtain the count.\n/// * `g` The parser to apply repeatedly.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::number::complete::u8;\n/// use nom::multi::length_count;\n/// use nom::bytes::complete::tag;\n/// use nom::combinator::map;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n///   length_count(map(u8, |i| {\n///      println!(\"got number: {}\", i);\n///      i\n///   }), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(&b\"\\x02abcabcabc\"[..]), Ok(((&b\"abc\"[..], vec![&b\"abc\"[..], &b\"abc\"[..]]))));\n/// assert_eq!(parser(b\"\\x03123123123\"), Err(Err::Error(Error::new(&b\"123123123\"[..], ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\npub fn length_count<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = Vec<<G as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  LengthCount {\n    length: f,\n    parser: g,\n    e: PhantomData,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [length_count] combinator\npub struct LengthCount<F, G, E> {\n  length: F,\n  parser: G,\n  e: PhantomData<E>,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F, G, E> Parser<I> for LengthCount<F, G, E>\nwhere\n  I: Clone,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  <F as Parser<I>>::Output: ToUsize,\n  E: ParseError<I>,\n{\n  type Output = Vec<<G as Parser<I>>::Output>;\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match self\n      .length\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(input)\n    {\n      Err(e) => Err(e),\n      Ok((i, count)) => {\n        let count = count.to_usize();\n        let mut input = i.clone();\n        let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n          / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n        let mut res = OM::Output::bind(|| {\n          crate::lib::std::vec::Vec::with_capacity(count.min(max_initial_capacity))\n        });\n\n        for _ in 0..count {\n          let input_ = input.clone();\n          match self.parser.process::<OM>(input_) {\n            Ok((i, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              input = i;\n            }\n            Err(Err::Error(e)) => {\n              return Err(Err::Error(OM::Error::map(e, |e| {\n                <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n              })));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n\n        Ok((input, res))\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser and collects the results in a type implementing `Extend + Default`.\n/// Fails if the amount of time the embedded parser is run is not\n/// within the specified range.\n/// # Arguments\n/// * `range` Constrains the number of iterations.\n///   * A range without an upper bound `a..` is equivalent to a range of `a..=usize::MAX`.\n///   * A single `usize` value is equivalent to `value..=value`.\n///   * An empty range is invalid.\n/// * `parse` The parser to apply.\n///\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many(0..=2, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n///\n/// This is not limited to `Vec`, other collections like `HashMap`\n/// can be used:\n///\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many;\n/// use nom::bytes::complete::{tag, take_while};\n/// use nom::sequence::{separated_pair, terminated};\n/// use nom::AsChar;\n///\n/// use std::collections::HashMap;\n///\n/// fn key_value(s: &str) -> IResult<&str, HashMap<&str, &str>> {\n///   many(0.., terminated(\n///     separated_pair(\n///       take_while(AsChar::is_alpha),\n///       tag(\"=\"),\n///       take_while(AsChar::is_alpha)\n///     ),\n///     tag(\";\")\n///   )).parse(s)\n/// }\n///\n/// assert_eq!(\n///   key_value(\"a=b;c=d;\"),\n///   Ok((\"\", HashMap::from([(\"a\", \"b\"), (\"c\", \"d\")])))\n/// );\n/// ```\n///\n/// If more control is needed on the default value, [fold] can\n/// be used instead:\n///\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold;\n/// use nom::bytes::complete::tag;\n///\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold(\n///     0..=4,\n///     tag(\"abc\"),\n///     // preallocates a vector of the max size\n///     || Vec::with_capacity(4),\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n///\n/// assert_eq!(parser(\"abcabcabcabc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many<I, E, Collection, F, G>(\n  range: G,\n  parser: F,\n) -> impl Parser<I, Output = Collection, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  Collection: Extend<<F as Parser<I>>::Output> + Default,\n  E: ParseError<I>,\n  G: NomRange<usize>,\n{\n  Many {\n    parser,\n    range,\n    c: PhantomData,\n  }\n}\n\n/// Parser implementation for the [many] combinator\npub struct Many<F, R, Collection> {\n  parser: F,\n  range: R,\n  c: PhantomData<Collection>,\n}\n\nimpl<I, F, R, Collection> Parser<I> for Many<F, R, Collection>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  Collection: Extend<<F as Parser<I>>::Output> + Default,\n  R: NomRange<usize>,\n{\n  type Output = Collection;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.range.is_inverted() {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::Many,\n      )));\n    }\n\n    let mut res = OM::Output::bind(Collection::default);\n\n    for count in self.range.bounded_iter() {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |mut res, value| {\n            res.extend(Some(value));\n            res\n          });\n          input = tail;\n        }\n        Err(Err::Error(e)) => {\n          if !self.range.contains(&count) {\n            return Err(Err::Error(OM::Error::map(e, |e| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::Many, e)\n            })));\n          } else {\n            return Ok((input, res));\n          }\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Applies a parser and accumulates the results using a given\n/// function and initial value.\n/// Fails if the amount of time the embedded parser is run is not\n/// within the specified range.\n///\n/// # Arguments\n/// * `range` Constrains the number of iterations.\n///   * A range without an upper bound `a..` allows the parser to run until it fails.\n///   * A single `usize` value is equivalent to `value..=value`.\n///   * An empty range is invalid.\n/// * `parse` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `fold` The function that combines a result of `f` with\n///       the current accumulator.\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold(\n///     0..=2,\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\npub fn fold<I, E, F, G, H, J, R>(\n  range: J,\n  parser: F,\n  init: H,\n  fold: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n  J: NomRange<usize>,\n{\n  Fold {\n    parser,\n    init,\n    fold,\n    range,\n  }\n}\n\n/// Parser implementation for the [fold] combinator\npub struct Fold<F, G, H, Range> {\n  parser: F,\n  init: H,\n  fold: G,\n  range: Range,\n}\n\nimpl<I, F, G, H, Range, Res> Parser<I> for Fold<F, G, H, Range>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(Res, <F as Parser<I>>::Output) -> Res,\n  H: FnMut() -> Res,\n  Range: NomRange<usize>,\n{\n  type Output = Res;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.range.is_inverted() {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::Fold,\n      )));\n    }\n\n    let mut acc = OM::Output::bind(|| (self.init)());\n\n    for count in self.range.saturating_iter() {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(tail, ErrorKind::Fold)\n            })));\n          }\n\n          acc = OM::Output::combine(acc, value, |acc, value| (self.fold)(acc, value));\n          input = tail;\n        }\n        Err(Err::Error(err)) => {\n          if !self.range.contains(&count) {\n            return Err(Err::Error(OM::Error::map(err, |err| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::Fold, err)\n            })));\n          } else {\n            break;\n          }\n        }\n        Err(e) => return Err(e),\n      }\n    }\n\n    Ok((input, acc))\n  }\n}\n\n// Path: src/str.rs\n#[cfg(test)]\nmod test {\n  #[cfg(feature = \"alloc\")]\n  use crate::{branch::alt, bytes::complete::tag_no_case, combinator::recognize, multi::many};\n  use crate::{\n    bytes::complete::{is_a, is_not, tag, take, take_till, take_until},\n    error::{self, ErrorKind},\n    Err, IResult,\n  };\n\n  #[test]\n  fn tagtr_succeed() {\n    const INPUT: &str = \"Hello World!\";\n    const TAG: &str = \"Hello\";\n    fn test(input: &str) -> IResult<&str, &str> {\n      tag(TAG)(input)\n    }\n\n    match test(INPUT) {\n      Ok((extra, output)) => {\n        assert!(extra == \" World!\", \"Parser `tag` consumed leftover input.\");\n        assert!(\n          output == TAG,\n          \"Parser `tag` doesn't return the tag it matched on success. \\\n           Expected `{}`, got `{}`.\",\n          TAG,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `tag` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn tagtr_incomplete() {\n    use crate::bytes::streaming::tag;\n\n    const INPUT: &str = \"Hello\";\n    const TAG: &str = \"Hello World!\";\n\n    let res: IResult<_, _, error::Error<_>> = tag(TAG)(INPUT);\n    match res {\n      Err(Err::Incomplete(_)) => (),\n      other => {\n        panic!(\n          \"Parser `tag` didn't require more input when it should have. \\\n           Got `{:?}`.\",\n          other\n        );\n      }\n    };\n  }\n\n  #[test]\n  fn tagtr_error() {\n    const INPUT: &str = \"Hello World!\";\n    const TAG: &str = \"Random\"; // TAG must be closer than INPUT.\n\n    let res: IResult<_, _, error::Error<_>> = tag(TAG)(INPUT);\n    match res {\n      Err(Err::Error(_)) => (),\n      other => {\n        panic!(\n          \"Parser `tag` didn't fail when it should have. Got `{:?}`.`\",\n          other\n        );\n      }\n    };\n  }\n\n  #[test]\n  fn take_s_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\u0192\u01ad\u00e8\u0159\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\";\n    const LEFTOVER: &str = \"\u00e1\u0192\u01ad\u00e8\u0159\";\n\n    let res: IResult<_, _, error::Error<_>> = take(9_usize)(INPUT);\n    match res {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_s` consumed leftover input. Leftover `{}`.\",\n          extra\n        );\n        assert!(\n          output == CONSUMED,\n          \"Parser `take_s` doesn't return the string it consumed on success. Expected `{}`, got `{}`.\",\n          CONSUMED,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `take_s` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn take_until_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u2202\u00e1\u0192\u01ad\u00e8\u0159\";\n    const FIND: &str = \"\u00c2\u00df\u00c7\u2202\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\";\n    const LEFTOVER: &str = \"\u00c2\u00df\u00c7\u2202\u00e1\u0192\u01ad\u00e8\u0159\";\n\n    let res: IResult<_, _, (_, ErrorKind)> = take_until(FIND)(INPUT);\n    match res {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_until`\\\n           consumed leftover input. Leftover `{}`.\",\n          extra\n        );\n        assert!(\n          output == CONSUMED,\n          \"Parser `take_until`\\\n           doesn't return the string it consumed on success. Expected `{}`, got `{}`.\",\n          CONSUMED,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `take_until` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn take_s_incomplete() {\n    use crate::bytes::streaming::take;\n\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\";\n\n    let res: IResult<_, _, (_, ErrorKind)> = take(13_usize)(INPUT);\n    match res {\n      Err(Err::Incomplete(_)) => (),\n      other => panic!(\n        \"Parser `take` didn't require more input when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    }\n  }\n\n  use crate::internal::Needed;\n\n  fn is_alphabetic(c: char) -> bool {\n    (c as u8 >= 0x41 && c as u8 <= 0x5A) || (c as u8 >= 0x61 && c as u8 <= 0x7A)\n  }\n\n  #[test]\n  fn take_while() {\n    use crate::bytes::streaming::take_while;\n\n    fn f(i: &str) -> IResult<&str, &str> {\n      take_while(is_alphabetic)(i)\n    }\n    let a = \"\";\n    let b = \"abcd\";\n    let c = \"abcd123\";\n    let d = \"123\";\n\n    assert_eq!(f(a), Err(Err::Incomplete(Needed::new(1))));\n    assert_eq!(f(b), Err(Err::Incomplete(Needed::new(1))));\n    assert_eq!(f(c), Ok((d, b)));\n    assert_eq!(f(d), Ok((d, a)));\n  }\n\n  #[test]\n  fn take_while1() {\n    use crate::bytes::streaming::take_while1;\n\n    fn f(i: &str) -> IResult<&str, &str> {\n...\n", "needle_token_start": 7251, "needle_token_end": 7411, "code_context_ntokens": 16284, "output": ["```rust\n/// Applies a parser and accumulates the results using a given\n/// function and initial value.\n/// Fails if the amount of time the embedded parser is run is not\n/// within the specified range.\n///\n/// # Arguments\n/// * `range` Constrains the number of iterations.\n///   * A range without an upper bound `a..` allows the parser to run until it fails.\n///   * A single `usize` value is equivalent to `value..=value`.\n///   * An empty range is invalid.\n/// * `parse` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `fold` The function that combines a result of `f` with\n///       the current accumulator.\n/// ```rust\n/// # #[macro_use] extern crate nom;\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold(\n///     0..=2,\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\npub fn fold<I, E, F, G, H, J, R>(\n  range: J,\n  parser: F,\n  init: H,\n  fold: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n  J: NomRange<usize>,\n{\n  Fold {\n    parser,\n    init,\n    fold,\n    range,\n  }\n}\n\n/// Parser implementation for the [fold] combinator\npub struct Fold<F, G, H, Range> {\n  parser: F,\n  init: H,\n  fold: G,\n  range: Range,\n}\n\nimpl<I, F, G, H, Range, Res> Parser<I> for Fold<F, G, H, Range>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(Res, <F as Parser<I>>::Output) -> Res,\n  H: FnMut() -> Res,\n  Range: NomRange<usize>,\n{\n  type Output = Res;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.range.is_inverted() {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::Fold,\n      )));\n    }\n\n    let mut acc = OM::Output::bind(|| (self.init)());\n\n    for count in self.range.saturating_iter() {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(tail, ErrorKind::Fold)\n            })));\n          }\n\n          acc = OM::Output::combine(acc, value, |acc, value| (self.fold)(acc, value));\n          input = tail;\n        }\n        Err(Err::Error(err)) => {\n          if !self.range.contains(&count) {\n            return Err(Err::Error(OM::Error::map(err, |err| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::Fold, err)\n            })));\n          } else {\n            break;\n          }\n        }\n        Err(e) => return Err(e),\n      }\n    }\n\n    Ok((input, acc))\n  }\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "parse", "language": "rust", "path": "src/internal.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to analyze a given input according to specific rules and either extract data from it or verify its structure. It adapts its behavior based on whether the output should be generated, errors should be detailed, and whether the input is considered complete or potentially extendable (streaming).\n2. **Input**: This function accepts data of a specified type, which it will attempt to process according to the defined parsing rules.\n3. **Output**: The result is either a tuple containing the unprocessed remainder of the input and the extracted data (if any), or an error detailing why the processing failed. The nature of the output and error can vary based on the operational mode.\n4. **Procedure**: The function processes the input by applying a set of parsing rules. If the input is deemed incomplete and expected to receive more data, it can return a specific error indicating the need for additional data. Otherwise, it processes the input fully, returning either the result of the parsing or an error if the parsing fails.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/number/streaming.rs\n//! Parsers recognizing numbers, streaming version\n\nuse crate::branch::alt;\nuse crate::bytes::streaming::tag;\nuse crate::character::streaming::{char, digit1, sign};\nuse crate::combinator::{cut, map, opt, recognize};\nuse crate::error::{ErrorKind, ParseError};\nuse crate::lib::std::ops::{Add, Shl};\nuse crate::sequence::pair;\nuse crate::traits::{AsBytes, AsChar, Compare, Offset};\nuse crate::{internal::*, Input};\n\n/// Recognizes an unsigned 1 byte integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_u8;\n///\n/// let parser = |s| {\n///   be_u8::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01abcd\"[..]), Ok((&b\"\\x01abcd\"[..], 0x00)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Incomplete(Needed::new(1))));\n/// ```\n#[inline]\npub fn be_u8<I, E: ParseError<I>>(input: I) -> IResult<I, u8, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_uint(input, 1)\n}\n\n/// Recognizes a big endian unsigned 2 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_u16;\n///\n/// let parser = |s| {\n///   be_u16::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01abcd\"[..]), Ok((&b\"abcd\"[..], 0x0001)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(1))));\n/// ```\n#[inline]\npub fn be_u16<I, E: ParseError<I>>(input: I) -> IResult<I, u16, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_uint(input, 2)\n}\n\n/// Recognizes a big endian unsigned 3 byte integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_u24;\n///\n/// let parser = |s| {\n///   be_u24::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02abcd\"[..]), Ok((&b\"abcd\"[..], 0x000102)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(2))));\n/// ```\n#[inline]\npub fn be_u24<I, E: ParseError<I>>(input: I) -> IResult<I, u32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_uint(input, 3)\n}\n\n/// Recognizes a big endian unsigned 4 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_u32;\n///\n/// let parser = |s| {\n///   be_u32::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02\\x03abcd\"[..]), Ok((&b\"abcd\"[..], 0x00010203)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(3))));\n/// ```\n#[inline]\npub fn be_u32<I, E: ParseError<I>>(input: I) -> IResult<I, u32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_uint(input, 4)\n}\n\n/// Recognizes a big endian unsigned 8 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_u64;\n///\n/// let parser = |s| {\n///   be_u64::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcd\"[..]), Ok((&b\"abcd\"[..], 0x0001020304050607)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(7))));\n/// ```\n#[inline]\npub fn be_u64<I, E: ParseError<I>>(input: I) -> IResult<I, u64, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_uint(input, 8)\n}\n\n/// Recognizes a big endian unsigned 16 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_u128;\n///\n/// let parser = |s| {\n///   be_u128::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x10\\x11\\x12\\x13\\x14\\x15abcd\"[..]), Ok((&b\"abcd\"[..], 0x00010203040506070809101112131415)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(15))));\n/// ```\n#[inline]\npub fn be_u128<I, E: ParseError<I>>(input: I) -> IResult<I, u128, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_uint(input, 16)\n}\n\n#[inline]\nfn be_uint<I, Uint, E: ParseError<I>>(input: I, bound: usize) -> IResult<I, Uint, E>\nwhere\n  I: Input<Item = u8>,\n  Uint: Default + Shl<u8, Output = Uint> + Add<Uint, Output = Uint> + From<u8>,\n{\n  super::be_uint(bound).parse(input)\n}\n\n/// Recognizes a signed 1 byte integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_i8;\n///\n/// let parser = be_i8::<_, (_, ErrorKind)>;\n///\n/// assert_eq!(parser(&b\"\\x00\\x01abcd\"[..]), Ok((&b\"\\x01abcd\"[..], 0x00)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Incomplete(Needed::new(1))));\n/// ```\n#[inline]\npub fn be_i8<I, E: ParseError<I>>(input: I) -> IResult<I, i8, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_u8.map(|x| x as i8).parse(input)\n}\n\n/// Recognizes a big endian signed 2 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_i16;\n///\n/// let parser = be_i16::<_, (_, ErrorKind)>;\n///\n/// assert_eq!(parser(&b\"\\x00\\x01abcd\"[..]), Ok((&b\"abcd\"[..], 0x0001)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Incomplete(Needed::new(2))));\n/// ```\n#[inline]\npub fn be_i16<I, E: ParseError<I>>(input: I) -> IResult<I, i16, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_u16.map(|x| x as i16).parse(input)\n}\n\n/// Recognizes a big endian signed 3 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_i24;\n///\n/// let parser = be_i24::<_, (_, ErrorKind)>;\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02abcd\"[..]), Ok((&b\"abcd\"[..], 0x000102)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Incomplete(Needed::new(3))));\n/// ```\n#[inline]\npub fn be_i24<I, E: ParseError<I>>(input: I) -> IResult<I, i32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  // Same as the unsigned version but we need to sign-extend manually here\n  be_u24\n    .map(|x| {\n      if x & 0x80_00_00 != 0 {\n        (x | 0xff_00_00_00) as i32\n      } else {\n        x as i32\n      }\n    })\n    .parse(input)\n}\n\n/// Recognizes a big endian signed 4 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_i32;\n///\n/// let parser = be_i32::<_, (_, ErrorKind)>;\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02\\x03abcd\"[..]), Ok((&b\"abcd\"[..], 0x00010203)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Incomplete(Needed::new(4))));\n/// ```\n#[inline]\npub fn be_i32<I, E: ParseError<I>>(input: I) -> IResult<I, i32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_u32.map(|x| x as i32).parse(input)\n}\n\n/// Recognizes a big endian signed 8 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_i64;\n///\n/// let parser = be_i64::<_, (_, ErrorKind)>;\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcd\"[..]), Ok((&b\"abcd\"[..], 0x0001020304050607)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(7))));\n/// ```\n#[inline]\npub fn be_i64<I, E: ParseError<I>>(input: I) -> IResult<I, i64, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_u64.map(|x| x as i64).parse(input)\n}\n\n/// Recognizes a big endian signed 16 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::be_i128;\n///\n/// let parser = be_i128::<_, (_, ErrorKind)>;\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x10\\x11\\x12\\x13\\x14\\x15abcd\"[..]), Ok((&b\"abcd\"[..], 0x00010203040506070809101112131415)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(15))));\n/// ```\n#[inline]\npub fn be_i128<I, E: ParseError<I>>(input: I) -> IResult<I, i128, E>\nwhere\n  I: Input<Item = u8>,\n{\n  be_u128.map(|x| x as i128).parse(input)\n}\n\n/// Recognizes an unsigned 1 byte integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::le_u8;\n///\n/// let parser = le_u8::<_, (_, ErrorKind)>;\n///\n/// assert_eq!(parser(&b\"\\x00\\x01abcd\"[..]), Ok((&b\"\\x01abcd\"[..], 0x00)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Incomplete(Needed::new(1))));\n/// ```\n#[inline]\npub fn le_u8<I, E: ParseError<I>>(input: I) -> IResult<I, u8, E>\nwhere\n  I: Input<Item = u8>,\n{\n  le_uint(input, 1)\n}\n\n/// Recognizes a little endian unsigned 2 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::le_u16;\n///\n/// let parser = |s| {\n///   le_u16::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01abcd\"[..]), Ok((&b\"abcd\"[..], 0x0100)));\n/// assert_eq!(parser(&b\"\\x01\"[..]), Err(Err::Incomplete(Needed::new(1))));\n/// ```\n#[inline]\npub fn le_u16<I, E: ParseError<I>>(input: I) -> IResult<I, u16, E>\nwhere\n  I: Input<Item = u8>,\n{\n  le_uint(input, 2)\n}\n\n/// Recognizes a little endian unsigned 3 bytes integer.\n///\n/// *Streaming version*: Will return `Err(nom::Err::Incomplete(_))` if there is not enough data.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// use nom::number::streaming::le_u24;\n///\n/// let parser = |s| {\n///   le_u24::<_, (_, ErrorKind)>(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x01\\x02abcd\"[..]), Ok((&b\"abcd\"[..], 0x020100)));\n...\n// Path: src/internal.rs\n//! Basic types to build the parsers\n\nuse self::Needed::*;\nuse crate::error::{self, ErrorKind, FromExternalError, ParseError};\nuse crate::lib::std::fmt;\nuse core::marker::PhantomData;\nuse core::num::NonZeroUsize;\n\n/// Holds the result of parsing functions\n///\n/// It depends on the input type `I`, the output type `O`, and the error type `E`\n/// (by default `(I, nom::ErrorKind)`)\n///\n/// The `Ok` side is a pair containing the remainder of the input (the part of the data that\n/// was not parsed) and the produced value. The `Err` side contains an instance of `nom::Err`.\n///\n/// Outside of the parsing code, you can use the [Finish::finish] method to convert\n/// it to a more common result type\npub type IResult<I, O, E = error::Error<I>> = Result<(I, O), Err<E>>;\n\n/// Helper trait to convert a parser's result to a more manageable type\npub trait Finish<I, O, E> {\n  /// converts the parser's result to a type that is more consumable by error\n  /// management libraries. It keeps the same `Ok` branch, and merges `Err::Error`\n  /// and `Err::Failure` into the `Err` side.\n  ///\n  /// *warning*: if the result is `Err(Err::Incomplete(_))`, this method will panic.\n  /// - \"complete\" parsers: It will not be an issue, `Incomplete` is never used\n  /// - \"streaming\" parsers: `Incomplete` will be returned if there's not enough data\n  /// for the parser to decide, and you should gather more data before parsing again.\n  /// Once the parser returns either `Ok(_)`, `Err(Err::Error(_))` or `Err(Err::Failure(_))`,\n  /// you can get out of the parsing loop and call `finish()` on the parser's result\n  fn finish(self) -> Result<(I, O), E>;\n}\n\nimpl<I, O, E> Finish<I, O, E> for IResult<I, O, E> {\n  fn finish(self) -> Result<(I, O), E> {\n    match self {\n      Ok(res) => Ok(res),\n      Err(Err::Error(e)) | Err(Err::Failure(e)) => Err(e),\n      Err(Err::Incomplete(_)) => {\n        panic!(\"Cannot call `finish()` on `Err(Err::Incomplete(_))`: this result means that the parser does not have enough data to decide, you should gather more data and try to reapply the parser instead\")\n      }\n    }\n  }\n}\n\n/// Contains information on needed data if a parser returned `Incomplete`\n#[derive(Debug, PartialEq, Eq, Clone, Copy)]\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub enum Needed {\n  /// Needs more data, but we do not know how much\n  Unknown,\n  /// Contains the required data size in bytes\n  Size(NonZeroUsize),\n}\n\nimpl Needed {\n  /// Creates `Needed` instance, returns `Needed::Unknown` if the argument is zero\n  pub fn new(s: usize) -> Self {\n    match NonZeroUsize::new(s) {\n      Some(sz) => Needed::Size(sz),\n      None => Needed::Unknown,\n    }\n  }\n\n  /// Indicates if we know how many bytes we need\n  pub fn is_known(&self) -> bool {\n    *self != Unknown\n  }\n\n  /// Maps a `Needed` to `Needed` by applying a function to a contained `Size` value.\n  #[inline]\n  pub fn map<F: Fn(NonZeroUsize) -> usize>(self, f: F) -> Needed {\n    match self {\n      Unknown => Unknown,\n      Size(n) => Needed::new(f(n)),\n    }\n  }\n}\n\n/// The `Err` enum indicates the parser was not successful\n///\n/// It has three cases:\n///\n/// * `Incomplete` indicates that more data is needed to decide. The `Needed` enum\n/// can contain how many additional bytes are necessary. If you are sure your parser\n/// is working on full data, you can wrap your parser with the `complete` combinator\n/// to transform that case in `Error`\n/// * `Error` means some parser did not succeed, but another one might (as an example,\n/// when testing different branches of an `alt` combinator)\n/// * `Failure` indicates an unrecoverable error. For example, when a prefix has been\n/// recognised and the next parser has been confirmed, if that parser fails, then the\n/// entire process fails; there are no more parsers to try.\n///\n/// Distinguishing `Failure` this from `Error` is only relevant inside the parser's code. For\n/// external consumers, both mean that parsing failed.\n///\n/// See also: [`Finish`].\n///\n#[derive(Debug, Clone, PartialEq)]\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub enum Err<Failure, Error = Failure> {\n  /// There was not enough data\n  Incomplete(Needed),\n  /// The parser had an error (recoverable)\n  Error(Error),\n  /// The parser had an unrecoverable error: we got to the right\n  /// branch and we know other branches won't work, so backtrack\n  /// as fast as possible\n  Failure(Failure),\n}\n\nimpl<E> Err<E> {\n  /// Tests if the result is Incomplete\n  pub fn is_incomplete(&self) -> bool {\n    matches!(self, Err::Incomplete(..))\n  }\n\n  /// Applies the given function to the inner error\n  pub fn map<E2, F>(self, f: F) -> Err<E2>\n  where\n    F: FnOnce(E) -> E2,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure(t) => Err::Failure(f(t)),\n      Err::Error(t) => Err::Error(f(t)),\n    }\n  }\n\n  /// Automatically converts between errors if the underlying type supports it\n  pub fn convert<F>(e: Err<F>) -> Self\n  where\n    E: From<F>,\n  {\n    e.map(crate::lib::std::convert::Into::into)\n  }\n}\n\nimpl<T> Err<(T, ErrorKind)> {\n  /// Maps `Err<(T, ErrorKind)>` to `Err<(U, ErrorKind)>` with the given `F: T -> U`\n  pub fn map_input<U, F>(self, f: F) -> Err<(U, ErrorKind)>\n  where\n    F: FnOnce(T) -> U,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure((input, k)) => Err::Failure((f(input), k)),\n      Err::Error((input, k)) => Err::Error((f(input), k)),\n    }\n  }\n}\n\nimpl<T> Err<error::Error<T>> {\n  /// Maps `Err<error::Error<T>>` to `Err<error::Error<U>>` with the given `F: T -> U`\n  pub fn map_input<U, F>(self, f: F) -> Err<error::Error<U>>\n  where\n    F: FnOnce(T) -> U,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure(error::Error { input, code }) => Err::Failure(error::Error {\n        input: f(input),\n        code,\n      }),\n      Err::Error(error::Error { input, code }) => Err::Error(error::Error {\n        input: f(input),\n        code,\n      }),\n    }\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nuse crate::lib::std::{borrow::ToOwned, string::String, vec::Vec};\n#[cfg(feature = \"alloc\")]\nimpl Err<(&[u8], ErrorKind)> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<(Vec<u8>, ErrorKind)> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<(&str, ErrorKind)> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<(String, ErrorKind)> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<error::Error<&[u8]>> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<error::Error<Vec<u8>>> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<error::Error<&str>> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<error::Error<String>> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\nimpl<E: Eq> Eq for Err<E> {}\n\nimpl<E> fmt::Display for Err<E>\nwhere\n  E: fmt::Debug,\n{\n  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n    match self {\n      Err::Incomplete(Needed::Size(u)) => write!(f, \"Parsing requires {} bytes/chars\", u),\n      Err::Incomplete(Needed::Unknown) => write!(f, \"Parsing requires more data\"),\n      Err::Failure(c) => write!(f, \"Parsing Failure: {:?}\", c),\n      Err::Error(c) => write!(f, \"Parsing Error: {:?}\", c),\n    }\n  }\n}\n\n#[cfg(feature = \"std\")]\nuse std::error::Error;\n\n#[cfg(feature = \"std\")]\nimpl<E> Error for Err<E>\nwhere\n  E: fmt::Debug,\n{\n  fn source(&self) -> Option<&(dyn Error + 'static)> {\n    None // no underlying error\n  }\n}\n\n/// Parser mode: influences how combinators build values\n///\n/// the [Parser] trait is generic over types implementing [Mode]. Its method are\n/// called to produce and manipulate output values or errors.\n///\n/// The main implementations of this trait are:\n/// * [Emit]: produce a value\n/// * [Check]: apply the parser but do not generate a value\npub trait Mode {\n  /// The output type that may be generated\n  type Output<T>;\n\n  /// Produces a value\n  fn bind<T, F: FnOnce() -> T>(f: F) -> Self::Output<T>;\n\n  /// Applies a function over the produced value\n  fn map<T, U, F: FnOnce(T) -> U>(x: Self::Output<T>, f: F) -> Self::Output<U>;\n\n  /// Combines two values generated by previous parsers\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    x: Self::Output<T>,\n    y: Self::Output<U>,\n    f: F,\n  ) -> Self::Output<V>;\n}\n\n/// Produces a value. This is the default behaviour for parsers\npub struct Emit;\nimpl Mode for Emit {\n  type Output<T> = T;\n\n  #[inline(always)]\n  fn bind<T, F: FnOnce() -> T>(f: F) -> Self::Output<T> {\n    f()\n  }\n\n  #[inline(always)]\n  fn map<T, U, F: FnOnce(T) -> U>(x: Self::Output<T>, f: F) -> Self::Output<U> {\n    f(x)\n  }\n\n  #[inline(always)]\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    x: Self::Output<T>,\n    y: Self::Output<U>,\n    f: F,\n  ) -> Self::Output<V> {\n    f(x, y)\n  }\n}\n\n/// Applies the parser, but do not a produce a value\n///\n/// This has the effect of greatly reducing the amount of code generated and the\n/// parser memory usage. Some combinators chek for an error in a child parser but\n/// discard the error, and for those, using [Check] makes sure the error is not\n/// even generated, only the fact that an error happened remains\npub struct Check;\nimpl Mode for Check {\n  type Output<T> = ();\n\n  #[inline(always)]\n  fn bind<T, F: FnOnce() -> T>(_: F) -> Self::Output<T> {}\n\n  #[inline(always)]\n  fn map<T, U, F: FnOnce(T) -> U>(_: Self::Output<T>, _: F) -> Self::Output<U> {}\n\n  #[inline(always)]\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    _: Self::Output<T>,\n    _: Self::Output<U>,\n    _: F,\n  ) -> Self::Output<V> {\n  }\n}\n\n/// Parser result type\n///\n/// * `Ok` branch: a tuple of the remaining input data, and the output value.\n/// The output value is of the `O` type if the output mode was [Emit], and `()`\n/// if the mode was [Check]\n/// * `Err` branch: an error of the `E` type if the erroor mode was [Emit], and `()`\n/// if the mode was [Check]\npub type PResult<OM, I, O, E> = Result<\n  (I, <<OM as OutputMode>::Output as Mode>::Output<O>),\n  Err<E, <<OM as OutputMode>::Error as Mode>::Output<E>>,\n>;\n\n/// Trait Defining the parser's execution\n///\n/// The same parser implementation can vary in behaviour according to the chosen\n/// output mode\npub trait OutputMode {\n  /// Defines the [Mode] for the output type. [Emit] will generate the value, [Check] will\n  /// apply the parser but will only generate `()` if successul. This can be used when\n  /// verifying that the input data conforms to the format without having to generate any\n  /// output data\n  type Output: Mode;\n  /// Defines the [Mode] for the output type. [Emit] will generate the value, [Check] will\n  /// apply the parser but will only generate `()` if an error happened. [Emit] should be\n  /// used when we want to handle the error and extract useful information from it. [Check]\n  /// is used when we just want to know if parsing failed and reject the data quickly.\n  type Error: Mode;\n  /// Indicates whether the input data is \"complete\", ie we already have the entire data in the\n  /// buffer, or if it is \"streaming\", where more data can be added later in the buffer. In\n  /// streaming mode, the parser will understand that a failure may mean that we are missing\n  /// data, and will return a specific error branch, [Err::Incomplete] to signal it. In complete\n  /// mode, the parser will generate a normal error\n  type Incomplete: IsStreaming;\n}\n\n/// Specifies the behaviour when a parser encounters an error that could be due to partial ata\npub trait IsStreaming {\n  /// called by parsers on partial data errors\n  /// * `needed` can hold the amount of dditional data the parser would need to decide\n  /// * `err_f`: produces the error when in \"complete\" mode\n  fn incomplete<E, F: FnOnce() -> E>(needed: Needed, err_f: F) -> Err<E>;\n  /// Indicates whether the data is in streaming mode or not\n  fn is_streaming() -> bool;\n}\n\n/// Indicates that the input data is streaming: more data may be available later\npub struct Streaming;\n\nimpl IsStreaming for Streaming {\n  fn incomplete<E, F: FnOnce() -> E>(needed: Needed, _err_f: F) -> Err<E> {\n    Err::Incomplete(needed)\n  }\n\n  #[inline]\n  fn is_streaming() -> bool {\n    true\n  }\n}\n\n/// Indicates that the input data is complete: no more data may be added later\npub struct Complete;\n\nimpl IsStreaming for Complete {\n  fn incomplete<E, F: FnOnce() -> E>(_needed: Needed, err_f: F) -> Err<E> {\n    Err::Error(err_f())\n  }\n\n  #[inline]\n  fn is_streaming() -> bool {\n    false\n  }\n}\n\n/// Holds the parser execution modifiers: output [Mode], error [Mode] and\n/// streaming behaviour for input data\npub struct OutputM<M: Mode, EM: Mode, S: IsStreaming> {\n  m: PhantomData<M>,\n  em: PhantomData<EM>,\n  s: PhantomData<S>,\n}\n\nimpl<M: Mode, EM: Mode, S: IsStreaming> OutputMode for OutputM<M, EM, S> {\n  type Output = M;\n  type Error = EM;\n  type Incomplete = S;\n}\n/// All nom parsers implement this trait\npub trait Parser<Input> {\n  /// Type of the produced value\n  type Output;\n  /// Error type of this parser\n  type Error: ParseError<Input>;\n\n  /// A parser takes in input type, and returns a `Result` containing\n  /// either the remaining input and the output value, or an error\n  #[inline]\n  \nfn parse(&mut self, input: Input) -> IResult<Input, Self::Output, Self::Error> {\n    self.process::<OutputM<Emit, Emit, Streaming>>(input)\n  }\n\n  /// A parser takes in input type, and returns a `Result` containing\n  /// either the remaining input and the output value, or an error\n  #[inline]\n  fn parse_complete(&mut self, input: Input) -> IResult<Input, Self::Output, Self::Error> {\n    self.process::<OutputM<Emit, Emit, Complete>>(input)\n  }\n\n  /// A parser takes in input type, and returns a `Result` containing\n  /// either the remaining input and the output value, or an error\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: Input,\n  ) -> PResult<OM, Input, Self::Output, Self::Error>;\n\n  /// Maps a function over the result of a parser\n  fn map<G, O2>(self, g: G) -> Map<Self, G>\n  where\n    G: FnMut(Self::Output) -> O2,\n    Self: core::marker::Sized,\n  {\n    Map { f: self, g }\n  }\n\n  /// Applies a function returning a `Result` over the result of a parser.\n  fn map_res<G, O2, E2>(self, g: G) -> MapRes<Self, G>\n  where\n    G: FnMut(Self::Output) -> Result<O2, E2>,\n    Self::Error: FromExternalError<Input, E2>,\n    Self: core::marker::Sized,\n  {\n    MapRes { f: self, g }\n  }\n\n  /// Applies a function returning an `Option` over the result of a parser.\n  fn map_opt<G, O2>(self, g: G) -> MapOpt<Self, G>\n  where\n    G: FnMut(Self::Output) -> Option<O2>,\n    Self: core::marker::Sized,\n  {\n    MapOpt { f: self, g }\n  }\n\n  /// Creates a second parser from the output of the first one, then apply over the rest of the input\n  fn flat_map<G, H>(self, g: G) -> FlatMap<Self, G>\n  where\n    G: FnMut(Self::Output) -> H,\n    H: Parser<Input, Error = Self::Error>,\n    Self: core::marker::Sized,\n  {\n    FlatMap { f: self, g }\n  }\n\n  /// Applies a second parser over the output of the first one\n  fn and_then<G>(self, g: G) -> AndThen<Self, G>\n  where\n    G: Parser<Self::Output, Error = Self::Error>,\n    Self: core::marker::Sized,\n  {\n    AndThen { f: self, g }\n  }\n\n  /// Applies a second parser after the first one, return their results as a tuple\n  fn and<G, O2>(self, g: G) -> And<Self, G>\n  where\n    G: Parser<Input, Output = O2, Error = Self::Error>,\n    Self: core::marker::Sized,\n  {\n    And { f: self, g }\n  }\n\n  /// Applies a second parser over the input if the first one failed\n  fn or<G>(self, g: G) -> Or<Self, G>\n  where\n    G: Parser<Input, Output = Self::Output, Error = Self::Error>,\n    Self: core::marker::Sized,\n  {\n    Or { f: self, g }\n  }\n\n  /// automatically converts the parser's output and error values to another type, as long as they\n  /// implement the `From` trait\n  fn into<O2: From<Self::Output>, E2: From<Self::Error>>(self) -> Into<Self, O2, E2>\n  where\n    Self: core::marker::Sized,\n  {\n    Into {\n      f: self,\n      phantom_out2: core::marker::PhantomData,\n      phantom_err2: core::marker::PhantomData,\n    }\n  }\n}\n\nimpl<I, O, E: ParseError<I>, F> Parser<I> for F\nwhere\n  F: FnMut(I) -> IResult<I, O, E>,\n{\n  type Output = O;\n  type Error = E;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    let (i, o) = self(i).map_err(|e| match e {\n      Err::Incomplete(i) => Err::Incomplete(i),\n      Err::Error(e) => Err::Error(OM::Error::bind(|| e)),\n      Err::Failure(e) => Err::Failure(e),\n    })?;\n    Ok((i, OM::Output::bind(|| o)))\n  }\n}\n\nmacro_rules! impl_parser_for_tuple {\n  ($($parser:ident $output:ident),+) => (\n    #[allow(non_snake_case)]\n    impl<I, $($output),+, E: ParseError<I>, $($parser),+> Parser<I> for ($($parser),+,)\n    where\n      $($parser: Parser<I, Output = $output, Error = E>),+\n    {\n      type Output = ($($output),+,);\n      type Error = E;\n\n      #[inline(always)]\n      fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n        let ($(ref mut $parser),+,) = *self;\n\n        // FIXME: is there a way to avoid producing the output values?\n        $(let(i, $output) = $parser.process::<OutputM<Emit, OM::Error, OM::Incomplete>>(i)?;)+\n\n        // ???\n        Ok((i, OM::Output::bind(|| ($($output),+,))))\n      }\n    }\n  )\n}\n\nmacro_rules! impl_parser_for_tuples {\n    ($parser1:ident $output1:ident, $($parser:ident $output:ident),+) => {\n        impl_parser_for_tuples!(__impl $parser1 $output1; $($parser $output),+);\n    };\n    (__impl $($parser:ident $output:ident),+; $parser1:ident $output1:ident $(,$parser2:ident $output2:ident)*) => {\n        impl_parser_for_tuple!($($parser $output),+);\n        impl_parser_for_tuples!(__impl $($parser $output),+, $parser1 $output1; $($parser2 $output2),*);\n    };\n    (__impl $($parser:ident $output:ident),+;) => {\n        impl_parser_for_tuple!($($parser $output),+);\n    }\n}\n\nimpl_parser_for_tuples!(P1 O1, P2 O2, P3 O3, P4 O4, P5 O5, P6 O6, P7 O7, P8 O8, P9 O9, P10 O10, P11 O11, P12 O12, P13 O13, P14 O14, P15 O15, P16 O16, P17 O17, P18 O18, P19 O19, P20 O20, P21 O21);\n\n/*\n#[cfg(feature = \"alloc\")]\nuse alloc::boxed::Box;\n\n#[cfg(feature = \"alloc\")]\nimpl<I, O, E: ParseError<I>> Parser<I> for Box<dyn Parser<I, Output = O, Error = E>> {\n  type Output = O;\n  type Error = E;\n\n  fn process<OM: OutputMode>(&mut self, input: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    (**self).process(input)\n  }\n}\n*/\n/// Implementation of `Parser::map`\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub struct Map<F, G> {\n  f: F,\n  g: G,\n}\n\nimpl<I, O2, E: ParseError<I>, F: Parser<I, Error = E>, G: FnMut(<F as Parser<I>>::Output) -> O2>\n  Parser<I> for Map<F, G>\n{\n  type Output = O2;\n  type Error = E;\n\n  #[inline(always)]\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    match self.f.process::<OM>(i) {\n      Err(e) => Err(e),\n      Ok((i, o)) => Ok((i, OM::Output::map(o, |o| (self.g)(o)))),\n    }\n  }\n}\n\n/// Implementation of `Parser::map_res`\npub struct MapRes<F, G> {\n  f: F,\n  g: G,\n}\n\nimpl<I, O2, E2, F, G> Parser<I> for MapRes<F, G>\nwhere\n  I: Clone,\n  <F as Parser<I>>::Error: FromExternalError<I, E2>,\n  F: Parser<I>,\n  G: FnMut(<F as Parser<I>>::Output) -> Result<O2, E2>,\n{\n  type Output = O2;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    let (input, o1) = self\n      .f\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(i.clone())?;\n\n    match (self.g)(o1) {\n      Ok(o2) => Ok((input, OM::Output::bind(|| o2))),\n      Err(e) => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_external_error(i, ErrorKind::MapRes, e)\n      }))),\n    }\n  }\n}\n\n/// Implementation of `Parser::map_opt`\npub struct MapOpt<F, G> {\n  f: F,\n  g: G,\n}\n\nimpl<I, O2, F, G> Parser<I> for MapOpt<F, G>\nwhere\n  I: Clone,\n  F: Parser<I>,\n  G: FnMut(<F as Parser<I>>::Output) -> Option<O2>,\n{\n  type Output = O2;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    let (input, o1) = self\n      .f\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(i.clone())?;\n\n    match (self.g)(o1) {\n      Some(o2) => Ok((input, OM::Output::bind(|| o2))),\n      None => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::MapOpt)\n      }))),\n    }\n  }\n}\n\n/// Implementation of `Parser::flat_map`\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub struct FlatMap<F, G> {\n  f: F,\n  g: G,\n}\n\nimpl<\n    I,\n    E: ParseError<I>,\n    F: Parser<I, Error = E>,\n    G: FnMut(<F as Parser<I>>::Output) -> H,\n    H: Parser<I, Error = E>,\n  > Parser<I> for FlatMap<F, G>\n{\n  type Output = <H as Parser<I>>::Output;\n  type Error = E;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    let (input, o1) = self\n      .f\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(i)?;\n\n    (self.g)(o1).process::<OM>(input)\n  }\n}\n\n/// Implementation of `Parser::and_then`\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub struct AndThen<F, G> {\n  f: F,\n  g: G,\n}\n\nimpl<I, F: Parser<I>, G: Parser<<F as Parser<I>>::Output, Error = <F as Parser<I>>::Error>>\n  Parser<I> for AndThen<F, G>\n{\n  type Output = <G as Parser<<F as Parser<I>>::Output>>::Output;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    let (input, o1) = self\n      .f\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(i)?;\n\n    let (_, o2) = self.g.process::<OM>(o1)?;\n    Ok((input, o2))\n  }\n}\n\n/// Implementation of `Parser::and`\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub struct And<F, G> {\n  f: F,\n  g: G,\n}\n\nimpl<I, E: ParseError<I>, F: Parser<I, Error = E>, G: Parser<I, Error = E>> Parser<I>\n  for And<F, G>\n{\n  type Output = (<F as Parser<I>>::Output, <G as Parser<I>>::Output);\n  type Error = E;\n\n  #[inline(always)]\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    let (i, o1) = self.f.process::<OM>(i)?;\n    let (i, o2) = self.g.process::<OM>(i)?;\n\n    Ok((i, OM::Output::combine(o1, o2, |o1, o2| (o1, o2))))\n  }\n}\n\n/// Implementation of `Parser::or`\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub struct Or<F, G> {\n  f: F,\n  g: G,\n}\n\nimpl<\n    I: Clone,\n    O,\n    E: ParseError<I>,\n    F: Parser<I, Output = O, Error = E>,\n    G: Parser<I, Output = O, Error = E>,\n  > Parser<I> for Or<F, G>\n{\n  type Output = <F as Parser<I>>::Output;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    match self.f.process::<OM>(i.clone()) {\n      Err(Err::Error(e1)) => match self.g.process::<OM>(i) {\n        Err(Err::Error(e2)) => Err(Err::Error(OM::Error::combine(e1, e2, |e1, e2| e1.or(e2)))),\n        res => res,\n      },\n      res => res,\n    }\n  }\n}\n\n/// Implementation of `Parser::into`\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub struct Into<F, O2, E2> {\n  f: F,\n  phantom_out2: core::marker::PhantomData<O2>,\n  phantom_err2: core::marker::PhantomData<E2>,\n}\n\nimpl<\n    I,\n    O2: From<<F as Parser<I>>::Output>,\n    E2: crate::error::ParseError<I> + From<<F as Parser<I>>::Error>,\n    F: Parser<I>,\n  > Parser<I> for Into<F, O2, E2>\n{\n  type Output = O2;\n  type Error = E2;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    match self.f.process::<OM>(i) {\n      Ok((i, o)) => Ok((i, OM::Output::map(o, |o| o.into()))),\n      Err(Err::Error(e)) => Err(Err::Error(OM::Error::map(e, |e| e.into()))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e.into())),\n      Err(Err::Incomplete(e)) => Err(Err::Incomplete(e)),\n    }\n  }\n}\n\n/// Alternate between two Parser implementations with the same result type.\npub(crate) enum Either<F, G> {\n  Left(F),\n  Right(G),\n}\n\nimpl<\n    I,\n    F: Parser<I>,\n    G: Parser<I, Output = <F as Parser<I>>::Output, Error = <F as Parser<I>>::Error>,\n  > Parser<I> for Either<F, G>\n{\n  type Output = <F as Parser<I>>::Output;\n  type Error = <F as Parser<I>>::Error;\n\n  #[inline]\n  fn process<OM: OutputMode>(&mut self, i: I) -> PResult<OM, I, Self::Output, Self::Error> {\n    match self {\n      Either::Left(f) => f.process::<OM>(i),\n      Either::Right(g) => g.process::<OM>(i),\n    }\n  }\n}\n\n#[cfg(test)]\nmod tests {\n  use super::*;\n  use crate::error::ErrorKind;\n\n  use crate::bytes::streaming::{tag, take};\n  use crate::number::streaming::be_u16;\n  use crate::sequence::terminated;\n\n  #[doc(hidden)]\n  #[macro_export]\n  macro_rules! assert_size (\n    ($t:ty, $sz:expr) => (\n      assert_eq!($crate::lib::std::mem::size_of::<$t>(), $sz);\n    );\n  );\n\n  #[test]\n  #[cfg(target_pointer_width = \"64\")]\n  fn size_test() {\n    assert_size!(IResult<&[u8], &[u8], (&[u8], u32)>, 40);\n    //FIXME: since rust 1.65, this is now 32 bytes, likely thanks to https://github.com/rust-lang/rust/pull/94075\n    // deactivating that test for now because it'll have different values depending on the rust version\n    // assert_size!(IResult<&str, &str, u32>, 40);\n    assert_size!(Needed, 8);\n    assert_size!(Err<u32>, 16);\n    assert_size!(ErrorKind, 1);\n  }\n\n  #[test]\n  fn err_map_test() {\n    let e = Err::Error(1);\n    assert_eq!(e.map(|v| v + 1), Err::Error(2));\n  }\n\n  #[test]\n  fn native_tuple_test() {\n    fn tuple_3(i: &[u8]) -> IResult<&[u8], (u16, &[u8])> {\n      terminated((be_u16, take(3u8)), tag(\"fg\")).parse(i)\n    }\n\n    assert_eq!(\n      tuple_3(&b\"abcdefgh\"[..]),\n      Ok((&b\"h\"[..], (0x6162u16, &b\"cde\"[..])))\n    );\n    assert_eq!(tuple_3(&b\"abcd\"[..]), Err(Err::Incomplete(Needed::new(1))));\n    assert_eq!(tuple_3(&b\"abcde\"[..]), Err(Err::Incomplete(Needed::new(2))));\n    assert_eq!(\n      tuple_3(&b\"abcdejk\"[..]),\n      Err(Err::Error(error_position!(&b\"jk\"[..], ErrorKind::Tag)))\n    );\n  }\n}\n\n// Path: src/bits/complete.rs\n//! Bit level parsers\n//!\n\nuse crate::error::{ErrorKind, ParseError};\nuse crate::internal::{Err, IResult};\nuse crate::lib::std::ops::{AddAssign, Div, Shl, Shr};\nuse crate::traits::{Input, ToUsize};\n\n/// Generates a parser taking `count` bits\n///\n/// # Example\n/// ```rust\n/// # use nom::bits::complete::take;\n/// # use nom::IResult;\n/// # use nom::error::{Error, ErrorKind};\n/// // Input is a tuple of (input: I, bit_offset: usize)\n/// fn parser(input: (&[u8], usize), count: usize)-> IResult<(&[u8], usize), u8> {\n///  take(count)(input)\n/// }\n///\n/// // Consumes 0 bits, returns 0\n/// assert_eq!(parser(([0b00010010].as_ref(), 0), 0), Ok((([0b00010010].as_ref(), 0), 0)));\n///\n/// // Consumes 4 bits, returns their values and increase offset to 4\n/// assert_eq!(parser(([0b00010010].as_ref(), 0), 4), Ok((([0b00010010].as_ref(), 4), 0b00000001)));\n///\n/// // Consumes 4 bits, offset is 4, returns their values and increase offset to 0 of next byte\n/// assert_eq!(parser(([0b00010010].as_ref(), 4), 4), Ok((([].as_ref(), 0), 0b00000010)));\n///\n/// // Tries to consume 12 bits but only 8 are available\n/// assert_eq!(parser(([0b00010010].as_ref(), 0), 12), Err(nom::Err::Error(Error{input: ([0b00010010].as_ref(), 0), code: ErrorKind::Eof })));\n/// ```\npub fn take<I, O, C, E: ParseError<(I, usize)>>(\n  count: C,\n) -> impl Fn((I, usize)) -> IResult<(I, usize), O, E>\nwhere\n  I: Input<Item = u8>,\n  C: ToUsize,\n  O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O>,\n{\n  let count = count.to_usize();\n  move |(input, bit_offset): (I, usize)| {\n    if count == 0 {\n      Ok(((input, bit_offset), 0u8.into()))\n    } else {\n      if input.input_len() * 8 < count + bit_offset {\n        Err(Err::Error(E::from_error_kind(\n          (input, bit_offset),\n          ErrorKind::Eof,\n        )))\n      } else {\n        let cnt = (count + bit_offset).div(8);\n        let mut acc: O = 0_u8.into();\n        let mut offset: usize = bit_offset;\n        let mut remaining: usize = count;\n        let mut end_offset: usize = 0;\n\n        for byte in input.iter_elements().take(cnt + 1) {\n          if remaining == 0 {\n            break;\n          }\n          let val: O = if offset == 0 {\n            byte.into()\n          } else {\n            ((byte << offset) >> offset).into()\n          };\n\n          if remaining < 8 - offset {\n            acc += val >> (8 - offset - remaining);\n            end_offset = remaining + offset;\n            break;\n          } else {\n            acc += val << (remaining - (8 - offset));\n            remaining -= 8 - offset;\n            offset = 0;\n          }\n        }\n        Ok(((input.take_from(cnt), end_offset), acc))\n      }\n    }\n  }\n}\n\n/// Generates a parser taking `count` bits and comparing them to `pattern`\npub fn tag<I, O, C, E: ParseError<(I, usize)>>(\n  pattern: O,\n  count: C,\n) -> impl Fn((I, usize)) -> IResult<(I, usize), O, E>\nwhere\n  I: Input<Item = u8> + Clone,\n  C: ToUsize,\n  O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O> + PartialEq,\n{\n  let count = count.to_usize();\n  move |input: (I, usize)| {\n    let inp = input.clone();\n\n    take(count)(input).and_then(|(i, o)| {\n      if pattern == o {\n        Ok((i, o))\n      } else {\n        Err(Err::Error(error_position!(inp, ErrorKind::TagBits)))\n      }\n    })\n  }\n}\n\n/// Parses one specific bit as a bool.\n///\n/// # Example\n/// ```rust\n/// # use nom::bits::complete::bool;\n/// # use nom::IResult;\n/// # use nom::error::{Error, ErrorKind};\n///\n/// fn parse(input: (&[u8], usize)) -> IResult<(&[u8], usize), bool> {\n///     bool(input)\n/// }\n///\n/// assert_eq!(parse(([0b10000000].as_ref(), 0)), Ok((([0b10000000].as_ref(), 1), true)));\n/// assert_eq!(parse(([0b10000000].as_ref(), 1)), Ok((([0b10000000].as_ref(), 2), false)));\n/// ```\npub fn bool<I, E: ParseError<(I, usize)>>(input: (I, usize)) -> IResult<(I, usize), bool, E>\nwhere\n  I: Input<Item = u8>,\n{\n  let (res, bit): (_, u32) = take(1usize)(input)?;\n  Ok((res, bit != 0))\n}\n\n#[cfg(test)]\nmod test {\n  use super::*;\n\n  #[test]\n  fn test_take_0() {\n    let input = [0b00010010].as_ref();\n    let count = 0usize;\n    assert_eq!(count, 0usize);\n    let offset = 0usize;\n\n    let result: crate::IResult<(&[u8], usize), usize> = take(count)((input, offset));\n\n    assert_eq!(result, Ok(((input, offset), 0)));\n  }\n\n  #[test]\n  fn test_take_eof() {\n    let input = [0b00010010].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), usize> = take(1usize)((input, 8));\n\n    assert_eq!(\n      result,\n      Err(crate::Err::Error(crate::error::Error {\n        input: (input, 8),\n        code: ErrorKind::Eof\n      }))\n    )\n  }\n\n  #[test]\n  fn test_take_span_over_multiple_bytes() {\n    let input = [0b00010010, 0b00110100, 0b11111111, 0b11111111].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), usize> = take(24usize)((input, 4));\n\n    assert_eq!(\n      result,\n      Ok((([0b11111111].as_ref(), 4), 0b1000110100111111111111))\n    );\n  }\n\n...\n", "needle_token_start": 8937, "needle_token_end": 8987, "code_context_ntokens": 16265, "output": ["```rust\n/// Parser result type\n///\n/// * `Ok` branch: a tuple of the remaining input data, and the output value.\n/// The output value is of the `O` type if the output mode was [Emit], and `()`\n/// if the mode was [Check]\n/// * `Err` branch: an error of the `E` type if the erroor mode was [Emit], and `()`\n/// if the mode was [Check]\npub type PResult<OM, I, O, E> = Result<\n  (I, <<OM as OutputMode>::Output as Mode>::Output<O>),\n  Err<E, <<OM as OutputMode>::Error as Mode>::Output<E>>,\n>;\n```"]}
{"repo": "rust-bakery/nom", "name": "test_complete_byte_consumption_bits", "language": "rust", "path": "src/bits/mod.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To verify that the function correctly parses specific bit segments from a byte array and accurately tracks the remaining unconsumed bytes.\n2. **Input**: An array of bytes intended to be parsed into segments of 4, 8, and 4 bits respectively.\n3. **Output**: A tuple containing the parsed values from the specified bit segments and the array of bytes that were not consumed during the parsing process.\n4. **Procedure**: The function processes the input byte array to extract the first 4 bits, then the next 8 bits, and finally another 4 bits. It checks that the remaining bytes in the array are correctly identified and returned, ensuring that bytes fully consumed by the parsing are not included in the output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/internal.rs\n//! Basic types to build the parsers\n\nuse self::Needed::*;\nuse crate::error::{self, ErrorKind, FromExternalError, ParseError};\nuse crate::lib::std::fmt;\nuse core::marker::PhantomData;\nuse core::num::NonZeroUsize;\n\n/// Holds the result of parsing functions\n///\n/// It depends on the input type `I`, the output type `O`, and the error type `E`\n/// (by default `(I, nom::ErrorKind)`)\n///\n/// The `Ok` side is a pair containing the remainder of the input (the part of the data that\n/// was not parsed) and the produced value. The `Err` side contains an instance of `nom::Err`.\n///\n/// Outside of the parsing code, you can use the [Finish::finish] method to convert\n/// it to a more common result type\npub type IResult<I, O, E = error::Error<I>> = Result<(I, O), Err<E>>;\n\n/// Helper trait to convert a parser's result to a more manageable type\npub trait Finish<I, O, E> {\n  /// converts the parser's result to a type that is more consumable by error\n  /// management libraries. It keeps the same `Ok` branch, and merges `Err::Error`\n  /// and `Err::Failure` into the `Err` side.\n  ///\n  /// *warning*: if the result is `Err(Err::Incomplete(_))`, this method will panic.\n  /// - \"complete\" parsers: It will not be an issue, `Incomplete` is never used\n  /// - \"streaming\" parsers: `Incomplete` will be returned if there's not enough data\n  /// for the parser to decide, and you should gather more data before parsing again.\n  /// Once the parser returns either `Ok(_)`, `Err(Err::Error(_))` or `Err(Err::Failure(_))`,\n  /// you can get out of the parsing loop and call `finish()` on the parser's result\n  fn finish(self) -> Result<(I, O), E>;\n}\n\nimpl<I, O, E> Finish<I, O, E> for IResult<I, O, E> {\n  fn finish(self) -> Result<(I, O), E> {\n    match self {\n      Ok(res) => Ok(res),\n      Err(Err::Error(e)) | Err(Err::Failure(e)) => Err(e),\n      Err(Err::Incomplete(_)) => {\n        panic!(\"Cannot call `finish()` on `Err(Err::Incomplete(_))`: this result means that the parser does not have enough data to decide, you should gather more data and try to reapply the parser instead\")\n      }\n    }\n  }\n}\n\n/// Contains information on needed data if a parser returned `Incomplete`\n#[derive(Debug, PartialEq, Eq, Clone, Copy)]\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub enum Needed {\n  /// Needs more data, but we do not know how much\n  Unknown,\n  /// Contains the required data size in bytes\n  Size(NonZeroUsize),\n}\n\nimpl Needed {\n  /// Creates `Needed` instance, returns `Needed::Unknown` if the argument is zero\n  pub fn new(s: usize) -> Self {\n    match NonZeroUsize::new(s) {\n      Some(sz) => Needed::Size(sz),\n      None => Needed::Unknown,\n    }\n  }\n\n  /// Indicates if we know how many bytes we need\n  pub fn is_known(&self) -> bool {\n    *self != Unknown\n  }\n\n  /// Maps a `Needed` to `Needed` by applying a function to a contained `Size` value.\n  #[inline]\n  pub fn map<F: Fn(NonZeroUsize) -> usize>(self, f: F) -> Needed {\n    match self {\n      Unknown => Unknown,\n      Size(n) => Needed::new(f(n)),\n    }\n  }\n}\n\n/// The `Err` enum indicates the parser was not successful\n///\n/// It has three cases:\n///\n/// * `Incomplete` indicates that more data is needed to decide. The `Needed` enum\n/// can contain how many additional bytes are necessary. If you are sure your parser\n/// is working on full data, you can wrap your parser with the `complete` combinator\n/// to transform that case in `Error`\n/// * `Error` means some parser did not succeed, but another one might (as an example,\n/// when testing different branches of an `alt` combinator)\n/// * `Failure` indicates an unrecoverable error. For example, when a prefix has been\n/// recognised and the next parser has been confirmed, if that parser fails, then the\n/// entire process fails; there are no more parsers to try.\n///\n/// Distinguishing `Failure` this from `Error` is only relevant inside the parser's code. For\n/// external consumers, both mean that parsing failed.\n///\n/// See also: [`Finish`].\n///\n#[derive(Debug, Clone, PartialEq)]\n#[cfg_attr(nightly, warn(rustdoc::missing_doc_code_examples))]\npub enum Err<Failure, Error = Failure> {\n  /// There was not enough data\n  Incomplete(Needed),\n  /// The parser had an error (recoverable)\n  Error(Error),\n  /// The parser had an unrecoverable error: we got to the right\n  /// branch and we know other branches won't work, so backtrack\n  /// as fast as possible\n  Failure(Failure),\n}\n\nimpl<E> Err<E> {\n  /// Tests if the result is Incomplete\n  pub fn is_incomplete(&self) -> bool {\n    matches!(self, Err::Incomplete(..))\n  }\n\n  /// Applies the given function to the inner error\n  pub fn map<E2, F>(self, f: F) -> Err<E2>\n  where\n    F: FnOnce(E) -> E2,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure(t) => Err::Failure(f(t)),\n      Err::Error(t) => Err::Error(f(t)),\n    }\n  }\n\n  /// Automatically converts between errors if the underlying type supports it\n  pub fn convert<F>(e: Err<F>) -> Self\n  where\n    E: From<F>,\n  {\n    e.map(crate::lib::std::convert::Into::into)\n  }\n}\n\nimpl<T> Err<(T, ErrorKind)> {\n  /// Maps `Err<(T, ErrorKind)>` to `Err<(U, ErrorKind)>` with the given `F: T -> U`\n  pub fn map_input<U, F>(self, f: F) -> Err<(U, ErrorKind)>\n  where\n    F: FnOnce(T) -> U,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure((input, k)) => Err::Failure((f(input), k)),\n      Err::Error((input, k)) => Err::Error((f(input), k)),\n    }\n  }\n}\n\nimpl<T> Err<error::Error<T>> {\n  /// Maps `Err<error::Error<T>>` to `Err<error::Error<U>>` with the given `F: T -> U`\n  pub fn map_input<U, F>(self, f: F) -> Err<error::Error<U>>\n  where\n    F: FnOnce(T) -> U,\n  {\n    match self {\n      Err::Incomplete(n) => Err::Incomplete(n),\n      Err::Failure(error::Error { input, code }) => Err::Failure(error::Error {\n        input: f(input),\n        code,\n      }),\n      Err::Error(error::Error { input, code }) => Err::Error(error::Error {\n        input: f(input),\n        code,\n      }),\n    }\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nuse crate::lib::std::{borrow::ToOwned, string::String, vec::Vec};\n#[cfg(feature = \"alloc\")]\nimpl Err<(&[u8], ErrorKind)> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<(Vec<u8>, ErrorKind)> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<(&str, ErrorKind)> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<(String, ErrorKind)> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<error::Error<&[u8]>> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<error::Error<Vec<u8>>> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl Err<error::Error<&str>> {\n  /// Obtaining ownership\n  #[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\n  pub fn to_owned(self) -> Err<error::Error<String>> {\n    self.map_input(ToOwned::to_owned)\n  }\n}\n\nimpl<E: Eq> Eq for Err<E> {}\n\nimpl<E> fmt::Display for Err<E>\nwhere\n  E: fmt::Debug,\n{\n  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n    match self {\n      Err::Incomplete(Needed::Size(u)) => write!(f, \"Parsing requires {} bytes/chars\", u),\n      Err::Incomplete(Needed::Unknown) => write!(f, \"Parsing requires more data\"),\n      Err::Failure(c) => write!(f, \"Parsing Failure: {:?}\", c),\n      Err::Error(c) => write!(f, \"Parsing Error: {:?}\", c),\n    }\n  }\n}\n\n#[cfg(feature = \"std\")]\nuse std::error::Error;\n\n#[cfg(feature = \"std\")]\nimpl<E> Error for Err<E>\nwhere\n  E: fmt::Debug,\n{\n  fn source(&self) -> Option<&(dyn Error + 'static)> {\n    None // no underlying error\n  }\n}\n\n/// Parser mode: influences how combinators build values\n///\n/// the [Parser] trait is generic over types implementing [Mode]. Its method are\n/// called to produce and manipulate output values or errors.\n///\n/// The main implementations of this trait are:\n/// * [Emit]: produce a value\n/// * [Check]: apply the parser but do not generate a value\npub trait Mode {\n  /// The output type that may be generated\n  type Output<T>;\n\n  /// Produces a value\n  fn bind<T, F: FnOnce() -> T>(f: F) -> Self::Output<T>;\n\n  /// Applies a function over the produced value\n  fn map<T, U, F: FnOnce(T) -> U>(x: Self::Output<T>, f: F) -> Self::Output<U>;\n\n  /// Combines two values generated by previous parsers\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    x: Self::Output<T>,\n    y: Self::Output<U>,\n    f: F,\n  ) -> Self::Output<V>;\n}\n\n/// Produces a value. This is the default behaviour for parsers\npub struct Emit;\nimpl Mode for Emit {\n  type Output<T> = T;\n\n  #[inline(always)]\n  fn bind<T, F: FnOnce() -> T>(f: F) -> Self::Output<T> {\n    f()\n  }\n\n  #[inline(always)]\n  fn map<T, U, F: FnOnce(T) -> U>(x: Self::Output<T>, f: F) -> Self::Output<U> {\n    f(x)\n  }\n\n  #[inline(always)]\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    x: Self::Output<T>,\n    y: Self::Output<U>,\n    f: F,\n  ) -> Self::Output<V> {\n    f(x, y)\n  }\n}\n\n/// Applies the parser, but do not a produce a value\n///\n/// This has the effect of greatly reducing the amount of code generated and the\n/// parser memory usage. Some combinators chek for an error in a child parser but\n/// discard the error, and for those, using [Check] makes sure the error is not\n/// even generated, only the fact that an error happened remains\npub struct Check;\nimpl Mode for Check {\n  type Output<T> = ();\n\n  #[inline(always)]\n  fn bind<T, F: FnOnce() -> T>(_: F) -> Self::Output<T> {}\n\n  #[inline(always)]\n  fn map<T, U, F: FnOnce(T) -> U>(_: Self::Output<T>, _: F) -> Self::Output<U> {}\n\n  #[inline(always)]\n  fn combine<T, U, V, F: FnOnce(T, U) -> V>(\n    _: Self::Output<T>,\n    _: Self::Output<U>,\n    _: F,\n  ) -> Self::Output<V> {\n  }\n}\n\n/// Parser result type\n///\n/// * `Ok` branch: a tuple of the remaining input data, and the output value.\n/// The output value is of the `O` type if the output mode was [Emit], and `()`\n/// if the mode was [Check]\n/// * `Err` branch: an error of the `E` type if the erroor mode was [Emit], and `()`\n/// if the mode was [Check]\npub type PResult<OM, I, O, E> = Result<\n  (I, <<OM as OutputMode>::Output as Mode>::Output<O>),\n  Err<E, <<OM as OutputMode>::Error as Mode>::Output<E>>,\n>;\n\n/// Trait Defining the parser's execution\n///\n/// The same parser implementation can vary in behaviour according to the chosen\n/// output mode\npub trait OutputMode {\n  /// Defines the [Mode] for the output type. [Emit] will generate the value, [Check] will\n  /// apply the parser but will only generate `()` if successul. This can be used when\n  /// verifying that the input data conforms to the format without having to generate any\n  /// output data\n  type Output: Mode;\n  /// Defines the [Mode] for the output type. [Emit] will generate the value, [Check] will\n  /// apply the parser but will only generate `()` if an error happened. [Emit] should be\n  /// used when we want to handle the error and extract useful information from it. [Check]\n  /// is used when we just want to know if parsing failed and reject the data quickly.\n  type Error: Mode;\n  /// Indicates whether the input data is \"complete\", ie we already have the entire data in the\n  /// buffer, or if it is \"streaming\", where more data can be added later in the buffer. In\n  /// streaming mode, the parser will understand that a failure may mean that we are missing\n  /// data, and will return a specific error branch, [Err::Incomplete] to signal it. In complete\n  /// mode, the parser will generate a normal error\n  type Incomplete: IsStreaming;\n}\n\n/// Specifies the behaviour when a parser encounters an error that could be due to partial ata\npub trait IsStreaming {\n  /// called by parsers on partial data errors\n  /// * `needed` can hold the amount of dditional data the parser would need to decide\n  /// * `err_f`: produces the error when in \"complete\" mode\n  fn incomplete<E, F: FnOnce() -> E>(needed: Needed, err_f: F) -> Err<E>;\n  /// Indicates whether the data is in streaming mode or not\n  fn is_streaming() -> bool;\n}\n\n/// Indicates that the input data is streaming: more data may be available later\npub struct Streaming;\n\nimpl IsStreaming for Streaming {\n  fn incomplete<E, F: FnOnce() -> E>(needed: Needed, _err_f: F) -> Err<E> {\n    Err::Incomplete(needed)\n  }\n\n  #[inline]\n  fn is_streaming() -> bool {\n    true\n  }\n}\n\n/// Indicates that the input data is complete: no more data may be added later\npub struct Complete;\n\nimpl IsStreaming for Complete {\n  fn incomplete<E, F: FnOnce() -> E>(_needed: Needed, err_f: F) -> Err<E> {\n    Err::Error(err_f())\n  }\n\n  #[inline]\n  fn is_streaming() -> bool {\n    false\n  }\n}\n\n/// Holds the parser execution modifiers: output [Mode], error [Mode] and\n/// streaming behaviour for input data\npub struct OutputM<M: Mode, EM: Mode, S: IsStreaming> {\n  m: PhantomData<M>,\n  em: PhantomData<EM>,\n  s: PhantomData<S>,\n}\n\nimpl<M: Mode, EM: Mode, S: IsStreaming> OutputMode for OutputM<M, EM, S> {\n  type Output = M;\n  type Error = EM;\n  type Incomplete = S;\n}\n/// All nom parsers implement this trait\npub trait Parser<Input> {\n  /// Type of the produced value\n  type Output;\n  /// Error type of this parser\n  type Error: ParseError<Input>;\n\n  /// A parser takes in input type, and returns a `Result` containing\n  /// either the remaining input and the output value, or an error\n  #[inline]\n  fn parse(&mut self, input: Input) -> IResult<Input, Self::Output, Self::Error> {\n    self.process::<OutputM<Emit, Emit, Streaming>>(input)\n  }\n\n  /// A parser takes in input type, and returns a `Result` containing\n  /// either the remaining input and the output value, or an error\n  #[inline]\n  fn parse_complete(&mut self, input: Input) -> IResult<Input, Self::Output, Self::Error> {\n    self.process::<OutputM<Emit, Emit, Complete>>(input)\n  }\n\n  /// A parser takes in input type, and returns a `Result` containing\n  /// either the remaining input and the output value, or an error\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: Input,\n  ) -> PResult<OM, Input, Self::Output, Self::Error>;\n\n  /// Maps a function over the result of a parser\n  fn map<G, O2>(self, g: G) -> Map<Self, G>\n  where\n    G: FnMut(Self::Output) -> O2,\n    Self: core::marker::Sized,\n  {\n    Map { f: self, g }\n  }\n\n  /// Applies a function returning a `Result` over the result of a parser.\n  fn map_res<G, O2, E2>(self, g: G) -> MapRes<Self, G>\n  where\n    G: FnMut(Self::Output) -> Result<O2, E2>,\n...\n// Path: src/bits/complete.rs\n//! Bit level parsers\n//!\n\nuse crate::error::{ErrorKind, ParseError};\nuse crate::internal::{Err, IResult};\nuse crate::lib::std::ops::{AddAssign, Div, Shl, Shr};\nuse crate::traits::{Input, ToUsize};\n\n/// Generates a parser taking `count` bits\n///\n/// # Example\n/// ```rust\n/// # use nom::bits::complete::take;\n/// # use nom::IResult;\n/// # use nom::error::{Error, ErrorKind};\n/// // Input is a tuple of (input: I, bit_offset: usize)\n/// fn parser(input: (&[u8], usize), count: usize)-> IResult<(&[u8], usize), u8> {\n///  take(count)(input)\n/// }\n///\n/// // Consumes 0 bits, returns 0\n/// assert_eq!(parser(([0b00010010].as_ref(), 0), 0), Ok((([0b00010010].as_ref(), 0), 0)));\n///\n/// // Consumes 4 bits, returns their values and increase offset to 4\n/// assert_eq!(parser(([0b00010010].as_ref(), 0), 4), Ok((([0b00010010].as_ref(), 4), 0b00000001)));\n///\n/// // Consumes 4 bits, offset is 4, returns their values and increase offset to 0 of next byte\n/// assert_eq!(parser(([0b00010010].as_ref(), 4), 4), Ok((([].as_ref(), 0), 0b00000010)));\n///\n/// // Tries to consume 12 bits but only 8 are available\n/// assert_eq!(parser(([0b00010010].as_ref(), 0), 12), Err(nom::Err::Error(Error{input: ([0b00010010].as_ref(), 0), code: ErrorKind::Eof })));\n/// ```\npub fn take<I, O, C, E: ParseError<(I, usize)>>(\n  count: C,\n) -> impl Fn((I, usize)) -> IResult<(I, usize), O, E>\nwhere\n  I: Input<Item = u8>,\n  C: ToUsize,\n  O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O>,\n{\n  let count = count.to_usize();\n  move |(input, bit_offset): (I, usize)| {\n    if count == 0 {\n      Ok(((input, bit_offset), 0u8.into()))\n    } else {\n      if input.input_len() * 8 < count + bit_offset {\n        Err(Err::Error(E::from_error_kind(\n          (input, bit_offset),\n          ErrorKind::Eof,\n        )))\n      } else {\n        let cnt = (count + bit_offset).div(8);\n        let mut acc: O = 0_u8.into();\n        let mut offset: usize = bit_offset;\n        let mut remaining: usize = count;\n        let mut end_offset: usize = 0;\n\n        for byte in input.iter_elements().take(cnt + 1) {\n          if remaining == 0 {\n            break;\n          }\n          let val: O = if offset == 0 {\n            byte.into()\n          } else {\n            ((byte << offset) >> offset).into()\n          };\n\n          if remaining < 8 - offset {\n            acc += val >> (8 - offset - remaining);\n            end_offset = remaining + offset;\n            break;\n          } else {\n            acc += val << (remaining - (8 - offset));\n            remaining -= 8 - offset;\n            offset = 0;\n          }\n        }\n        Ok(((input.take_from(cnt), end_offset), acc))\n      }\n    }\n  }\n}\n\n/// Generates a parser taking `count` bits and comparing them to `pattern`\npub fn tag<I, O, C, E: ParseError<(I, usize)>>(\n  pattern: O,\n  count: C,\n) -> impl Fn((I, usize)) -> IResult<(I, usize), O, E>\nwhere\n  I: Input<Item = u8> + Clone,\n  C: ToUsize,\n  O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O> + PartialEq,\n{\n  let count = count.to_usize();\n  move |input: (I, usize)| {\n    let inp = input.clone();\n\n    take(count)(input).and_then(|(i, o)| {\n      if pattern == o {\n        Ok((i, o))\n      } else {\n        Err(Err::Error(error_position!(inp, ErrorKind::TagBits)))\n      }\n    })\n  }\n}\n\n/// Parses one specific bit as a bool.\n///\n/// # Example\n/// ```rust\n/// # use nom::bits::complete::bool;\n/// # use nom::IResult;\n/// # use nom::error::{Error, ErrorKind};\n///\n/// fn parse(input: (&[u8], usize)) -> IResult<(&[u8], usize), bool> {\n///     bool(input)\n/// }\n///\n/// assert_eq!(parse(([0b10000000].as_ref(), 0)), Ok((([0b10000000].as_ref(), 1), true)));\n/// assert_eq!(parse(([0b10000000].as_ref(), 1)), Ok((([0b10000000].as_ref(), 2), false)));\n/// ```\npub fn bool<I, E: ParseError<(I, usize)>>(input: (I, usize)) -> IResult<(I, usize), bool, E>\nwhere\n  I: Input<Item = u8>,\n{\n  let (res, bit): (_, u32) = take(1usize)(input)?;\n  Ok((res, bit != 0))\n}\n\n#[cfg(test)]\nmod test {\n  use super::*;\n\n  #[test]\n  fn test_take_0() {\n    let input = [0b00010010].as_ref();\n    let count = 0usize;\n    assert_eq!(count, 0usize);\n    let offset = 0usize;\n\n    let result: crate::IResult<(&[u8], usize), usize> = take(count)((input, offset));\n\n    assert_eq!(result, Ok(((input, offset), 0)));\n  }\n\n  #[test]\n  fn test_take_eof() {\n    let input = [0b00010010].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), usize> = take(1usize)((input, 8));\n\n    assert_eq!(\n      result,\n      Err(crate::Err::Error(crate::error::Error {\n        input: (input, 8),\n        code: ErrorKind::Eof\n      }))\n    )\n  }\n\n  #[test]\n  fn test_take_span_over_multiple_bytes() {\n    let input = [0b00010010, 0b00110100, 0b11111111, 0b11111111].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), usize> = take(24usize)((input, 4));\n\n    assert_eq!(\n      result,\n      Ok((([0b11111111].as_ref(), 4), 0b1000110100111111111111))\n    );\n  }\n\n  #[test]\n  fn test_bool_0() {\n    let input = [0b10000000].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), bool> = bool((input, 0));\n\n    assert_eq!(result, Ok(((input, 1), true)));\n  }\n\n  #[test]\n  fn test_bool_eof() {\n    let input = [0b10000000].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), bool> = bool((input, 8));\n\n    assert_eq!(\n      result,\n      Err(crate::Err::Error(crate::error::Error {\n        input: (input, 8),\n        code: ErrorKind::Eof\n      }))\n    );\n  }\n}\n\n// Path: src/bits/streaming.rs\n//! Bit level parsers\n//!\n\nuse crate::error::{ErrorKind, ParseError};\nuse crate::internal::{Err, IResult, Needed};\nuse crate::lib::std::ops::{AddAssign, Div, Shl, Shr};\nuse crate::traits::{Input, ToUsize};\n\n/// Generates a parser taking `count` bits\npub fn take<I, O, C, E: ParseError<(I, usize)>>(\n  count: C,\n) -> impl Fn((I, usize)) -> IResult<(I, usize), O, E>\nwhere\n  I: Input<Item = u8>,\n  C: ToUsize,\n  O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O>,\n{\n  let count = count.to_usize();\n  move |(input, bit_offset): (I, usize)| {\n    if count == 0 {\n      Ok(((input, bit_offset), 0u8.into()))\n    } else {\n      let cnt = (count + bit_offset).div(8);\n      if input.input_len() * 8 < count + bit_offset {\n        Err(Err::Incomplete(Needed::new(count)))\n      } else {\n        let mut acc: O = 0_u8.into();\n        let mut offset: usize = bit_offset;\n        let mut remaining: usize = count;\n        let mut end_offset: usize = 0;\n\n        for byte in input.iter_elements().take(cnt + 1) {\n          if remaining == 0 {\n            break;\n          }\n          let val: O = if offset == 0 {\n            byte.into()\n          } else {\n            ((byte << offset) >> offset).into()\n          };\n\n          if remaining < 8 - offset {\n            acc += val >> (8 - offset - remaining);\n            end_offset = remaining + offset;\n            break;\n          } else {\n            acc += val << (remaining - (8 - offset));\n            remaining -= 8 - offset;\n            offset = 0;\n          }\n        }\n        Ok(((input.take_from(cnt), end_offset), acc))\n      }\n    }\n  }\n}\n\n/// Generates a parser taking `count` bits and comparing them to `pattern`\npub fn tag<I, O, C, E: ParseError<(I, usize)>>(\n  pattern: O,\n  count: C,\n) -> impl Fn((I, usize)) -> IResult<(I, usize), O, E>\nwhere\n  I: Input<Item = u8> + Clone,\n  C: ToUsize,\n  O: From<u8> + AddAssign + Shl<usize, Output = O> + Shr<usize, Output = O> + PartialEq,\n{\n  let count = count.to_usize();\n  move |input: (I, usize)| {\n    let inp = input.clone();\n\n    take(count)(input).and_then(|(i, o)| {\n      if pattern == o {\n        Ok((i, o))\n      } else {\n        Err(Err::Error(error_position!(inp, ErrorKind::TagBits)))\n      }\n    })\n  }\n}\n\n/// Parses one specific bit as a bool.\n///\n/// # Example\n/// ```rust\n/// # use nom::bits::complete::bool;\n/// # use nom::IResult;\n/// # use nom::error::{Error, ErrorKind};\n///\n/// fn parse(input: (&[u8], usize)) -> IResult<(&[u8], usize), bool> {\n///     bool(input)\n/// }\n///\n/// assert_eq!(parse(([0b10000000].as_ref(), 0)), Ok((([0b10000000].as_ref(), 1), true)));\n/// assert_eq!(parse(([0b10000000].as_ref(), 1)), Ok((([0b10000000].as_ref(), 2), false)));\n/// ```\npub fn bool<I, E: ParseError<(I, usize)>>(input: (I, usize)) -> IResult<(I, usize), bool, E>\nwhere\n  I: Input<Item = u8>,\n{\n  let (res, bit): (_, u32) = take(1usize)(input)?;\n  Ok((res, bit != 0))\n}\n\n#[cfg(test)]\nmod test {\n  use super::*;\n\n  #[test]\n  fn test_take_0() {\n    let input = [].as_ref();\n    let count = 0usize;\n    assert_eq!(count, 0usize);\n    let offset = 0usize;\n\n    let result: crate::IResult<(&[u8], usize), usize> = take(count)((input, offset));\n\n    assert_eq!(result, Ok(((input, offset), 0)));\n  }\n\n  #[test]\n  fn test_tag_ok() {\n    let input = [0b00011111].as_ref();\n    let offset = 0usize;\n    let bits_to_take = 4usize;\n    let value_to_tag = 0b0001;\n\n    let result: crate::IResult<(&[u8], usize), usize> =\n      tag(value_to_tag, bits_to_take)((input, offset));\n\n    assert_eq!(result, Ok(((input, bits_to_take), value_to_tag)));\n  }\n\n  #[test]\n  fn test_tag_err() {\n    let input = [0b00011111].as_ref();\n    let offset = 0usize;\n    let bits_to_take = 4usize;\n    let value_to_tag = 0b1111;\n\n    let result: crate::IResult<(&[u8], usize), usize> =\n      tag(value_to_tag, bits_to_take)((input, offset));\n\n    assert_eq!(\n      result,\n      Err(crate::Err::Error(crate::error::Error {\n        input: (input, offset),\n        code: ErrorKind::TagBits\n      }))\n    );\n  }\n\n  #[test]\n  fn test_bool_0() {\n    let input = [0b10000000].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), bool> = bool((input, 0));\n\n    assert_eq!(result, Ok(((input, 1), true)));\n  }\n\n  #[test]\n  fn test_bool_eof() {\n    let input = [0b10000000].as_ref();\n\n    let result: crate::IResult<(&[u8], usize), bool> = bool((input, 8));\n\n    assert_eq!(result, Err(crate::Err::Incomplete(Needed::new(1))));\n  }\n}\n\n// Path: src/bits/mod.rs\n//! Bit level parsers\n//!\n\npub mod complete;\npub mod streaming;\n\nuse crate::error::{ErrorKind, ParseError};\nuse crate::internal::{Err, IResult, Needed, Parser};\nuse crate::traits::ErrorConvert;\nuse crate::Input;\n\n/// Converts a byte-level input to a bit-level input, for consumption by a parser that uses bits.\n///\n/// Afterwards, the input is converted back to a byte-level parser, with any remaining bits thrown\n/// away.\n///\n/// # Example\n/// ```\n/// use nom::bits::{bits, streaming::take};\n/// use nom::error::Error;\n/// use nom::IResult;\n///\n/// fn parse(input: &[u8]) -> IResult<&[u8], (u8, u8)> {\n///     bits::<_, _, Error<(&[u8], usize)>, _, _>((take(4usize), take(8usize)))(input)\n/// }\n///\n/// let input = &[0x12, 0x34, 0xff, 0xff];\n///\n/// let output = parse(input).expect(\"We take 1.5 bytes and the input is longer than 2 bytes\");\n///\n/// // The first byte is consumed, the second byte is partially consumed and dropped.\n/// let remaining = output.0;\n/// assert_eq!(remaining, [0xff, 0xff]);\n///\n/// let parsed = output.1;\n/// assert_eq!(parsed.0, 0x01);\n/// assert_eq!(parsed.1, 0x23);\n/// ```\npub fn bits<I, O, E1, E2, P>(mut parser: P) -> impl FnMut(I) -> IResult<I, O, E2>\nwhere\n  E1: ParseError<(I, usize)> + ErrorConvert<E2>,\n  E2: ParseError<I>,\n  I: Input,\n  P: Parser<(I, usize), Output = O, Error = E1>,\n{\n  move |input: I| match parser.parse((input, 0)) {\n    Ok(((rest, offset), result)) => {\n      // If the next byte has been partially read, it will be sliced away as well.\n      // The parser functions might already slice away all fully read bytes.\n      // That's why `offset / 8` isn't necessarily needed at all times.\n      let remaining_bytes_index = offset / 8 + if offset % 8 == 0 { 0 } else { 1 };\n      Ok((rest.take_from(remaining_bytes_index), result))\n    }\n    Err(Err::Incomplete(n)) => Err(Err::Incomplete(n.map(|u| u.get() / 8 + 1))),\n    Err(Err::Error(e)) => Err(Err::Error(e.convert())),\n    Err(Err::Failure(e)) => Err(Err::Failure(e.convert())),\n  }\n}\n\n/// Counterpart to `bits`, `bytes` transforms its bit stream input into a byte slice for the underlying\n/// parser, allowing byte-slice parsers to work on bit streams.\n///\n/// A partial byte remaining in the input will be ignored and the given parser will start parsing\n/// at the next full byte.\n///\n/// ```\n/// use nom::bits::{bits, bytes, streaming::take};\n/// use nom::combinator::rest;\n/// use nom::error::Error;\n/// use nom::IResult;\n///\n/// fn parse(input: &[u8]) -> IResult<&[u8], (u8, u8, &[u8])> {\n///   bits::<_, _, Error<(&[u8], usize)>, _, _>((\n///     take(4usize),\n///     take(8usize),\n///     bytes::<_, _, Error<&[u8]>, _, _>(rest)\n///   ))(input)\n/// }\n///\n/// let input = &[0x12, 0x34, 0xff, 0xff];\n///\n/// assert_eq!(parse( input ), Ok(( &[][..], (0x01, 0x23, &[0xff, 0xff][..]) )));\n/// ```\npub fn bytes<I, O, E1, E2, P>(mut parser: P) -> impl FnMut((I, usize)) -> IResult<(I, usize), O, E2>\nwhere\n  E1: ParseError<I> + ErrorConvert<E2>,\n  E2: ParseError<(I, usize)>,\n  I: Input + Clone,\n  P: Parser<I, Output = O, Error = E1>,\n{\n  move |(input, offset): (I, usize)| {\n    let inner = if offset % 8 != 0 {\n      input.take_from(1 + offset / 8)\n    } else {\n      input.take_from(offset / 8)\n    };\n    let i = (input, offset);\n    match parser.parse(inner) {\n      Ok((rest, res)) => Ok(((rest, 0), res)),\n      Err(Err::Incomplete(Needed::Unknown)) => Err(Err::Incomplete(Needed::Unknown)),\n      Err(Err::Incomplete(Needed::Size(sz))) => Err(match sz.get().checked_mul(8) {\n        Some(v) => Err::Incomplete(Needed::new(v)),\n        None => Err::Failure(E2::from_error_kind(i, ErrorKind::TooLarge)),\n      }),\n      Err(Err::Error(e)) => Err(Err::Error(e.convert())),\n      Err(Err::Failure(e)) => Err(Err::Failure(e.convert())),\n    }\n  }\n}\n\n#[cfg(test)]\nmod test {\n  use super::*;\n  use crate::bits::streaming::take;\n  use crate::error::Error;\n\n  #[test]\n  /// Take the `bits` function and assert that remaining bytes are correctly returned, if the\n  /// previous bytes are fully consumed\n  \nfn test_complete_byte_consumption_bits() {\n    let input = &[0x12, 0x34, 0x56, 0x78];\n\n    // Take 3 bit slices with sizes [4, 8, 4].\n    let result: IResult<&[u8], (u8, u8, u8)> =\n      bits::<_, _, Error<(&[u8], usize)>, _, _>((take(4usize), take(8usize), take(4usize)))(input);\n\n    let output = result.expect(\"We take 2 bytes and the input is longer than 2 bytes\");\n\n    let remaining = output.0;\n    assert_eq!(remaining, [0x56, 0x78]);\n\n    let parsed = output.1;\n    assert_eq!(parsed.0, 0x01);\n    assert_eq!(parsed.1, 0x23);\n    assert_eq!(parsed.2, 0x04);\n  }\n\n  #[test]\n  /// Take the `bits` function and assert that remaining bytes are correctly returned, if the\n  /// previous bytes are NOT fully consumed. Partially consumed bytes are supposed to be dropped.\n  /// I.e. if we consume 1.5 bytes of 4 bytes, 2 bytes will be returned, bits 13-16 will be\n  /// dropped.\n  fn test_partial_byte_consumption_bits() {\n    let input = &[0x12, 0x34, 0x56, 0x78];\n\n    // Take bit slices with sizes [4, 8].\n    let result: IResult<&[u8], (u8, u8)> =\n      bits::<_, _, Error<(&[u8], usize)>, _, _>((take(4usize), take(8usize)))(input);\n\n    let output = result.expect(\"We take 1.5 bytes and the input is longer than 2 bytes\");\n\n    let remaining = output.0;\n    assert_eq!(remaining, [0x56, 0x78]);\n\n    let parsed = output.1;\n    assert_eq!(parsed.0, 0x01);\n    assert_eq!(parsed.1, 0x23);\n  }\n\n  #[test]\n  #[cfg(feature = \"std\")]\n  /// Ensure that in Incomplete error is thrown, if too few bytes are passed for a given parser.\n  fn test_incomplete_bits() {\n    let input = &[0x12];\n\n    // Take bit slices with sizes [4, 8].\n    let result: IResult<&[u8], (u8, u8)> =\n      bits::<_, _, Error<(&[u8], usize)>, _, _>((take(4usize), take(8usize)))(input);\n\n    assert!(result.is_err());\n    let error = result.err().unwrap();\n    assert_eq!(\"Parsing requires 2 bytes/chars\", error.to_string());\n  }\n}\n\n// Path: src/multi/tests.rs\nuse super::{length_data, length_value, many0_count, many1_count};\nuse crate::{\n  bytes::streaming::tag,\n  character::streaming::digit1 as digit,\n  error::{ErrorKind, ParseError},\n  internal::{Err, IResult, Needed},\n  lib::std::ops::Range,\n  lib::std::str::{self, FromStr},\n  number::streaming::{be_u16, be_u8},\n  sequence::pair,\n  Parser,\n};\n#[cfg(feature = \"alloc\")]\nuse crate::{\n  lib::std::vec::Vec,\n  multi::{\n    count, fold, fold_many0, fold_many1, fold_many_m_n, length_count, many, many0, many1, many_m_n,\n    many_till, separated_list0, separated_list1,\n  },\n};\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn separated_list0_test() {\n  fn multi(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    separated_list0(tag(\",\"), tag(\"abcd\")).parse(i)\n  }\n  fn multi_empty(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    separated_list0(tag(\",\"), tag(\"\")).parse(i)\n  }\n  fn empty_sep(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    separated_list0(tag(\"\"), tag(\"abc\")).parse(i)\n  }\n  fn multi_longsep(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    separated_list0(tag(\"..\"), tag(\"abcd\")).parse(i)\n  }\n\n  let a = &b\"abcdef\"[..];\n  let b = &b\"abcd,abcdef\"[..];\n  let c = &b\"azerty\"[..];\n  let d = &b\",,abc\"[..];\n  let e = &b\"abcd,abcd,ef\"[..];\n  let f = &b\"abc\"[..];\n  let g = &b\"abcd.\"[..];\n  let h = &b\"abcd,abc\"[..];\n  let i = &b\"abcabc\"[..];\n\n  let res1 = vec![&b\"abcd\"[..]];\n  assert_eq!(multi(a), Ok((&b\"ef\"[..], res1)));\n  let res2 = vec![&b\"abcd\"[..], &b\"abcd\"[..]];\n  assert_eq!(multi(b), Ok((&b\"ef\"[..], res2)));\n  assert_eq!(multi(c), Ok((&b\"azerty\"[..], Vec::new())));\n  let res3 = vec![&b\"\"[..], &b\"\"[..], &b\"\"[..]];\n  assert_eq!(multi_empty(d), Ok((&b\"abc\"[..], res3)));\n  let i_err_pos = &i[3..];\n  assert_eq!(\n    empty_sep(i),\n    Err(Err::Error(error_position!(\n      i_err_pos,\n      ErrorKind::SeparatedList\n    )))\n  );\n  let res4 = vec![&b\"abcd\"[..], &b\"abcd\"[..]];\n  assert_eq!(multi(e), Ok((&b\",ef\"[..], res4)));\n\n  assert_eq!(multi(f), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(multi_longsep(g), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(multi(h), Err(Err::Incomplete(Needed::new(1))));\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn separated_list1_test() {\n  fn multi(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    separated_list1(tag(\",\"), tag(\"abcd\")).parse(i)\n  }\n  fn multi_longsep(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    separated_list1(tag(\"..\"), tag(\"abcd\")).parse(i)\n  }\n\n  let a = &b\"abcdef\"[..];\n  let b = &b\"abcd,abcdef\"[..];\n  let c = &b\"azerty\"[..];\n  let d = &b\"abcd,abcd,ef\"[..];\n\n  let f = &b\"abc\"[..];\n  let g = &b\"abcd.\"[..];\n  let h = &b\"abcd,abc\"[..];\n\n  let res1 = vec![&b\"abcd\"[..]];\n  assert_eq!(multi(a), Ok((&b\"ef\"[..], res1)));\n  let res2 = vec![&b\"abcd\"[..], &b\"abcd\"[..]];\n  assert_eq!(multi(b), Ok((&b\"ef\"[..], res2)));\n  assert_eq!(\n    multi(c),\n    Err(Err::Error(error_position!(c, ErrorKind::Tag)))\n  );\n  let res3 = vec![&b\"abcd\"[..], &b\"abcd\"[..]];\n  assert_eq!(multi(d), Ok((&b\",ef\"[..], res3)));\n\n  assert_eq!(multi(f), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(multi_longsep(g), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(multi(h), Err(Err::Incomplete(Needed::new(1))));\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn many0_test() {\n  fn multi(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    many0(tag(\"abcd\")).parse(i)\n  }\n  fn multi_empty(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    many0(tag(\"\")).parse(i)\n  }\n\n  assert_eq!(multi(&b\"abcdef\"[..]), Ok((&b\"ef\"[..], vec![&b\"abcd\"[..]])));\n  assert_eq!(\n    multi(&b\"abcdabcdefgh\"[..]),\n    Ok((&b\"efgh\"[..], vec![&b\"abcd\"[..], &b\"abcd\"[..]]))\n  );\n  assert_eq!(multi(&b\"azerty\"[..]), Ok((&b\"azerty\"[..], Vec::new())));\n  assert_eq!(multi(&b\"abcdab\"[..]), Err(Err::Incomplete(Needed::new(2))));\n  assert_eq!(multi(&b\"abcd\"[..]), Err(Err::Incomplete(Needed::new(4))));\n  assert_eq!(multi(&b\"\"[..]), Err(Err::Incomplete(Needed::new(4))));\n  assert_eq!(\n    multi_empty(&b\"abcdef\"[..]),\n    Err(Err::Error(error_position!(\n      &b\"abcdef\"[..],\n      ErrorKind::Many0\n    )))\n  );\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn many1_test() {\n  fn multi(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    many1(tag(\"abcd\")).parse(i)\n  }\n\n  let a = &b\"abcdef\"[..];\n  let b = &b\"abcdabcdefgh\"[..];\n  let c = &b\"azerty\"[..];\n  let d = &b\"abcdab\"[..];\n\n  let res1 = vec![&b\"abcd\"[..]];\n  assert_eq!(multi(a), Ok((&b\"ef\"[..], res1)));\n  let res2 = vec![&b\"abcd\"[..], &b\"abcd\"[..]];\n  assert_eq!(multi(b), Ok((&b\"efgh\"[..], res2)));\n  assert_eq!(\n    multi(c),\n    Err(Err::Error(error_position!(c, ErrorKind::Tag)))\n  );\n  assert_eq!(multi(d), Err(Err::Incomplete(Needed::new(2))));\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn many_till_test() {\n  #[allow(clippy::type_complexity)]\n  fn multi(i: &[u8]) -> IResult<&[u8], (Vec<&[u8]>, &[u8])> {\n    many_till(tag(\"abcd\"), tag(\"efgh\")).parse(i)\n  }\n\n  let a = b\"abcdabcdefghabcd\";\n  let b = b\"efghabcd\";\n  let c = b\"azerty\";\n\n  let res_a = (vec![&b\"abcd\"[..], &b\"abcd\"[..]], &b\"efgh\"[..]);\n  let res_b: (Vec<&[u8]>, &[u8]) = (Vec::new(), &b\"efgh\"[..]);\n  assert_eq!(multi(&a[..]), Ok((&b\"abcd\"[..], res_a)));\n  assert_eq!(multi(&b[..]), Ok((&b\"abcd\"[..], res_b)));\n  assert_eq!(\n    multi(&c[..]),\n    Err(Err::Error(error_node_position!(\n      &c[..],\n      ErrorKind::ManyTill,\n      error_position!(&c[..], ErrorKind::Tag)\n    )))\n  );\n}\n\n#[test]\n#[cfg(feature = \"std\")]\nfn infinite_many() {\n  fn tst(input: &[u8]) -> IResult<&[u8], &[u8]> {\n    Err(Err::Error(error_position!(input, ErrorKind::Tag)))\n  }\n\n  // should not go into an infinite loop\n  fn multi0(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    many0(tst).parse(i)\n  }\n  let a = &b\"abcdef\"[..];\n  assert_eq!(multi0(a), Ok((a, Vec::new())));\n\n  fn multi1(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    many1(tst).parse(i)\n  }\n  let a = &b\"abcdef\"[..];\n  assert_eq!(\n    multi1(a),\n    Err(Err::Error(error_position!(a, ErrorKind::Tag)))\n  );\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn many_m_n_test() {\n  fn multi(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    many_m_n(2, 4, tag(\"Abcd\")).parse(i)\n  }\n\n  let a = &b\"Abcdef\"[..];\n  let b = &b\"AbcdAbcdefgh\"[..];\n  let c = &b\"AbcdAbcdAbcdAbcdefgh\"[..];\n  let d = &b\"AbcdAbcdAbcdAbcdAbcdefgh\"[..];\n  let e = &b\"AbcdAb\"[..];\n\n  assert_eq!(\n    multi(a),\n    Err(Err::Error(error_position!(&b\"ef\"[..], ErrorKind::Tag)))\n  );\n  let res1 = vec![&b\"Abcd\"[..], &b\"Abcd\"[..]];\n  assert_eq!(multi(b), Ok((&b\"efgh\"[..], res1)));\n  let res2 = vec![&b\"Abcd\"[..], &b\"Abcd\"[..], &b\"Abcd\"[..], &b\"Abcd\"[..]];\n  assert_eq!(multi(c), Ok((&b\"efgh\"[..], res2)));\n  let res3 = vec![&b\"Abcd\"[..], &b\"Abcd\"[..], &b\"Abcd\"[..], &b\"Abcd\"[..]];\n  assert_eq!(multi(d), Ok((&b\"Abcdefgh\"[..], res3)));\n  assert_eq!(multi(e), Err(Err::Incomplete(Needed::new(2))));\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn count_test() {\n  const TIMES: usize = 2;\n  fn cnt_2(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    count(tag(\"abc\"), TIMES).parse(i)\n  }\n\n  assert_eq!(\n    cnt_2(&b\"abcabcabcdef\"[..]),\n    Ok((&b\"abcdef\"[..], vec![&b\"abc\"[..], &b\"abc\"[..]]))\n  );\n  assert_eq!(cnt_2(&b\"ab\"[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(cnt_2(&b\"abcab\"[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(\n    cnt_2(&b\"xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n  assert_eq!(\n    cnt_2(&b\"xxxabcabcdef\"[..]),\n    Err(Err::Error(error_position!(\n      &b\"xxxabcabcdef\"[..],\n      ErrorKind::Tag\n    )))\n  );\n  assert_eq!(\n    cnt_2(&b\"abcxxxabcdef\"[..]),\n    Err(Err::Error(error_position!(\n      &b\"xxxabcdef\"[..],\n      ErrorKind::Tag\n    )))\n  );\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn count_zero() {\n  const TIMES: usize = 0;\n  fn counter_2(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    count(tag(\"abc\"), TIMES).parse(i)\n  }\n\n  let done = &b\"abcabcabcdef\"[..];\n  let parsed_done = Vec::new();\n  let rest = done;\n  let incomplete_1 = &b\"ab\"[..];\n  let parsed_incompl_1 = Vec::new();\n  let incomplete_2 = &b\"abcab\"[..];\n  let parsed_incompl_2 = Vec::new();\n  let error = &b\"xxx\"[..];\n  let error_remain = &b\"xxx\"[..];\n  let parsed_err = Vec::new();\n  let error_1 = &b\"xxxabcabcdef\"[..];\n  let parsed_err_1 = Vec::new();\n  let error_1_remain = &b\"xxxabcabcdef\"[..];\n  let error_2 = &b\"abcxxxabcdef\"[..];\n  let parsed_err_2 = Vec::new();\n  let error_2_remain = &b\"abcxxxabcdef\"[..];\n\n  assert_eq!(counter_2(done), Ok((rest, parsed_done)));\n  assert_eq!(\n    counter_2(incomplete_1),\n    Ok((incomplete_1, parsed_incompl_1))\n  );\n  assert_eq!(\n    counter_2(incomplete_2),\n    Ok((incomplete_2, parsed_incompl_2))\n  );\n  assert_eq!(counter_2(error), Ok((error_remain, parsed_err)));\n  assert_eq!(counter_2(error_1), Ok((error_1_remain, parsed_err_1)));\n  assert_eq!(counter_2(error_2), Ok((error_2_remain, parsed_err_2)));\n}\n\n#[derive(Debug, Clone, Eq, PartialEq)]\npub struct NilError;\n\nimpl<I> From<(I, ErrorKind)> for NilError {\n  fn from(_: (I, ErrorKind)) -> Self {\n    NilError\n  }\n}\n\nimpl<I> ParseError<I> for NilError {\n  fn from_error_kind(_: I, _: ErrorKind) -> NilError {\n    NilError\n  }\n  fn append(_: I, _: ErrorKind, _: NilError) -> NilError {\n    NilError\n  }\n}\n\nfn number(i: &[u8]) -> IResult<&[u8], u32> {\n  use crate::combinator::map_res;\n\n  map_res(map_res(digit, str::from_utf8), FromStr::from_str).parse(i)\n}\n\n#[test]\n#[cfg(feature = \"alloc\")]\nfn length_count_test() {\n  fn cnt(i: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n    length_count(number, tag(\"abc\")).parse(i)\n  }\n\n  assert_eq!(\n    cnt(&b\"2abcabcabcdef\"[..]),\n    Ok((&b\"abcdef\"[..], vec![&b\"abc\"[..], &b\"abc\"[..]]))\n  );\n  assert_eq!(cnt(&b\"2ab\"[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(cnt(&b\"3abcab\"[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(\n    cnt(&b\"xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Digit)))\n  );\n  assert_eq!(\n    cnt(&b\"2abcxxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Tag)))\n  );\n}\n\n#[test]\nfn length_data_test() {\n  fn take(i: &[u8]) -> IResult<&[u8], &[u8]> {\n    length_data(number).parse(i)\n  }\n\n  assert_eq!(\n    take(&b\"6abcabcabcdef\"[..]),\n    Ok((&b\"abcdef\"[..], &b\"abcabc\"[..]))\n  );\n  assert_eq!(take(&b\"3ab\"[..]), Err(Err::Incomplete(Needed::new(1))));\n  assert_eq!(\n    take(&b\"xxx\"[..]),\n    Err(Err::Error(error_position!(&b\"xxx\"[..], ErrorKind::Digit)))\n  );\n  assert_eq!(take(&b\"2abcxxx\"[..]), Ok((&b\"cxxx\"[..], &b\"ab\"[..])));\n}\n\n#[test]\nfn length_value_test() {\n  fn length_value_1(i: &[u8]) -> IResult<&[u8], u16> {\n    length_value(be_u8, be_u16).parse(i)\n  }\n  fn length_value_2(i: &[u8]) -> IResult<&[u8], (u8, u8)> {\n    length_value(be_u8, (be_u8, be_u8)).parse(i)\n  }\n\n  let i1 = [0, 5, 6];\n  assert_eq!(\n    length_value_1(&i1),\n    Err(Err::Error(error_position!(&b\"\"[..], ErrorKind::Complete)))\n  );\n  assert_eq!(\n    length_value_2(&i1),\n    Err(Err::Error(error_position!(&b\"\"[..], ErrorKind::Complete)))\n  );\n\n  let i2 = [1, 5, 6, 3];\n  assert_eq!(\n...\n", "needle_token_start": 10423, "needle_token_end": 10674, "code_context_ntokens": 16256, "output": ["```rust\nuse nom::bits::complete::take;\nuse nom::IResult;\n\nfn parse_bits(input: &[u8]) -> IResult<&[u8], (u8, u8, u8)> {\n    bits::<_, _, nom::error::Error<(&[u8], usize)>, _, _>((take(4usize), take(8usize), take(4usize)))(input)\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "be_i8_tests", "language": "rust", "path": "src/number/complete.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to test the parsing of single-byte signed integers from a byte array in big-endian format.\n2. **Input**: The function takes an array of bytes representing a big-endian signed integer.\n3. **Output**: It returns a tuple, where the first element is the remaining bytes after parsing (if any), and the second element is the parsed integer value.\n4. **Procedure**: The function processes an array of bytes, interprets the first byte as a big-endian signed integer, and checks if the parsed value matches the expected integer. It verifies several cases including the minimum and maximum values for a signed 8-bit integer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "where\n  I: Input<Item = u8>,\n{\n  le_u128.map(|x| x as i128).parse(input)\n}\n\n/// Recognizes an unsigned 1 byte integer\n///\n/// Note that endianness does not apply to 1 byte numbers.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::u8;\n///\n/// let parser = |s| {\n///   u8(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x03abcefg\"[..]), Ok((&b\"\\x03abcefg\"[..], 0x00)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Error((&[][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn u8<I, E: ParseError<I>>(input: I) -> IResult<I, u8, E>\nwhere\n  I: Input<Item = u8>,\n{\n  super::u8().parse_complete(input)\n}\n\n/// Recognizes an unsigned 2 bytes integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian u16 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian u16 integer.\n/// *complete version*: returns an error if there is not enough input data\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::u16;\n///\n/// let be_u16 = |s| {\n///   u16(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_u16(&b\"\\x00\\x03abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0003)));\n/// assert_eq!(be_u16(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_u16 = |s| {\n///   u16(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_u16(&b\"\\x00\\x03abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0300)));\n/// assert_eq!(le_u16(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn u16<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, u16, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::u16(endian).parse_complete(input)\n}\n\n/// Recognizes an unsigned 3 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian u24 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian u24 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::u24;\n///\n/// let be_u24 = |s| {\n///   u24(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_u24(&b\"\\x00\\x03\\x05abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x000305)));\n/// assert_eq!(be_u24(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_u24 = |s| {\n///   u24(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_u24(&b\"\\x00\\x03\\x05abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x050300)));\n/// assert_eq!(le_u24(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn u24<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, u32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::u24(endian).parse_complete(input)\n}\n\n/// Recognizes an unsigned 4 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian u32 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian u32 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::u32;\n///\n/// let be_u32 = |s| {\n///   u32(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_u32(&b\"\\x00\\x03\\x05\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x00030507)));\n/// assert_eq!(be_u32(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_u32 = |s| {\n///   u32(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_u32(&b\"\\x00\\x03\\x05\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x07050300)));\n/// assert_eq!(le_u32(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn u32<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, u32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::u32(endian).parse_complete(input)\n}\n\n/// Recognizes an unsigned 8 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian u64 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian u64 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::u64;\n///\n/// let be_u64 = |s| {\n///   u64(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_u64(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0001020304050607)));\n/// assert_eq!(be_u64(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_u64 = |s| {\n///   u64(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_u64(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0706050403020100)));\n/// assert_eq!(le_u64(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn u64<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, u64, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::u64(endian).parse_complete(input)\n}\n\n/// Recognizes an unsigned 16 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian u128 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian u128 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::u128;\n///\n/// let be_u128 = |s| {\n///   u128(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_u128(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x00010203040506070001020304050607)));\n/// assert_eq!(be_u128(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_u128 = |s| {\n///   u128(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_u128(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x07060504030201000706050403020100)));\n/// assert_eq!(le_u128(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn u128<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, u128, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::u128(endian).parse_complete(input)\n}\n\n/// Recognizes a signed 1 byte integer\n///\n/// Note that endianness does not apply to 1 byte numbers.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::i8;\n///\n/// let parser = |s| {\n///   i8(s)\n/// };\n///\n/// assert_eq!(parser(&b\"\\x00\\x03abcefg\"[..]), Ok((&b\"\\x03abcefg\"[..], 0x00)));\n/// assert_eq!(parser(&b\"\"[..]), Err(Err::Error((&[][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn i8<I, E: ParseError<I>>(i: I) -> IResult<I, i8, E>\nwhere\n  I: Input<Item = u8>,\n{\n  super::u8().map(|x| x as i8).parse_complete(i)\n}\n\n/// Recognizes a signed 2 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian i16 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian i16 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::i16;\n///\n/// let be_i16 = |s| {\n///   i16(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_i16(&b\"\\x00\\x03abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0003)));\n/// assert_eq!(be_i16(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_i16 = |s| {\n///   i16(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_i16(&b\"\\x00\\x03abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0300)));\n/// assert_eq!(le_i16(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn i16<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, i16, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::i16(endian).parse_complete(input)\n}\n\n/// Recognizes a signed 3 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian i24 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian i24 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::i24;\n///\n/// let be_i24 = |s| {\n///   i24(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_i24(&b\"\\x00\\x03\\x05abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x000305)));\n/// assert_eq!(be_i24(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_i24 = |s| {\n///   i24(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_i24(&b\"\\x00\\x03\\x05abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x050300)));\n/// assert_eq!(le_i24(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn i24<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, i32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::i24(endian).parse_complete(input)\n}\n\n/// Recognizes a signed 4 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian i32 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian i32 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::i32;\n///\n/// let be_i32 = |s| {\n///   i32(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_i32(&b\"\\x00\\x03\\x05\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x00030507)));\n/// assert_eq!(be_i32(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_i32 = |s| {\n///   i32(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_i32(&b\"\\x00\\x03\\x05\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x07050300)));\n/// assert_eq!(le_i32(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn i32<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, i32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::i32(endian).parse_complete(input)\n}\n\n/// Recognizes a signed 8 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian i64 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian i64 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::i64;\n///\n/// let be_i64 = |s| {\n///   i64(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_i64(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0001020304050607)));\n/// assert_eq!(be_i64(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_i64 = |s| {\n///   i64(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_i64(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x0706050403020100)));\n/// assert_eq!(le_i64(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn i64<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, i64, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::i64(endian).parse_complete(input)\n}\n\n/// Recognizes a signed 16 byte integer\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian i128 integer,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian i128 integer.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::i128;\n///\n/// let be_i128 = |s| {\n///   i128(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_i128(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x00010203040506070001020304050607)));\n/// assert_eq!(be_i128(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n///\n/// let le_i128 = |s| {\n///   i128(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_i128(&b\"\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07abcefg\"[..]), Ok((&b\"abcefg\"[..], 0x07060504030201000706050403020100)));\n/// assert_eq!(le_i128(&b\"\\x01\"[..]), Err(Err::Error((&[0x01][..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn i128<I, E: ParseError<I>>(\n  endian: crate::number::Endianness,\n) -> impl Fn(I) -> IResult<I, i128, E>\nwhere\n  I: Input<Item = u8>,\n{\n  move |input| super::i128(endian).parse_complete(input)\n}\n\n/// Recognizes a big endian 4 bytes floating point number.\n///\n/// *Complete version*: Returns an error if there is not enough input data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::be_f32;\n///\n/// let parser = |s| {\n///   be_f32(s)\n/// };\n///\n/// assert_eq!(parser(&[0x41, 0x48, 0x00, 0x00][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(parser(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn be_f32<I, E: ParseError<I>>(input: I) -> IResult<I, f32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  match be_u32(input) {\n    Err(e) => Err(e),\n    Ok((i, o)) => Ok((i, f32::from_bits(o))),\n  }\n}\n\n/// Recognizes a big endian 8 bytes floating point number.\n///\n/// *Complete version*: Returns an error if there is not enough input data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::be_f64;\n///\n/// let parser = |s| {\n///   be_f64(s)\n/// };\n///\n/// assert_eq!(parser(&[0x40, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(parser(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn be_f64<I, E: ParseError<I>>(input: I) -> IResult<I, f64, E>\nwhere\n  I: Input<Item = u8>,\n{\n  match be_u64(input) {\n    Err(e) => Err(e),\n    Ok((i, o)) => Ok((i, f64::from_bits(o))),\n  }\n}\n\n/// Recognizes a little endian 4 bytes floating point number.\n///\n/// *Complete version*: Returns an error if there is not enough input data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::le_f32;\n///\n/// let parser = |s| {\n///   le_f32(s)\n/// };\n///\n/// assert_eq!(parser(&[0x00, 0x00, 0x48, 0x41][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(parser(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn le_f32<I, E: ParseError<I>>(input: I) -> IResult<I, f32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  match le_u32(input) {\n    Err(e) => Err(e),\n    Ok((i, o)) => Ok((i, f32::from_bits(o))),\n  }\n}\n\n/// Recognizes a little endian 8 bytes floating point number.\n///\n/// *Complete version*: Returns an error if there is not enough input data.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::le_f64;\n///\n/// let parser = |s| {\n///   le_f64(s)\n/// };\n///\n/// assert_eq!(parser(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x40][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(parser(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn le_f64<I, E: ParseError<I>>(input: I) -> IResult<I, f64, E>\nwhere\n  I: Input<Item = u8>,\n{\n  match le_u64(input) {\n    Err(e) => Err(e),\n    Ok((i, o)) => Ok((i, f64::from_bits(o))),\n  }\n}\n\n/// Recognizes a 4 byte floating point number\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian f32 float,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian f32 float.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::f32;\n///\n/// let be_f32 = |s| {\n///   f32(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_f32(&[0x41, 0x48, 0x00, 0x00][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(be_f32(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n///\n/// let le_f32 = |s| {\n///   f32(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_f32(&[0x00, 0x00, 0x48, 0x41][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(le_f32(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn f32<I, E: ParseError<I>>(endian: crate::number::Endianness) -> fn(I) -> IResult<I, f32, E>\nwhere\n  I: Input<Item = u8>,\n{\n  match endian {\n    crate::number::Endianness::Big => be_f32,\n    crate::number::Endianness::Little => le_f32,\n    #[cfg(target_endian = \"big\")]\n    crate::number::Endianness::Native => be_f32,\n    #[cfg(target_endian = \"little\")]\n    crate::number::Endianness::Native => le_f32,\n  }\n}\n\n/// Recognizes an 8 byte floating point number\n///\n/// If the parameter is `nom::number::Endianness::Big`, parse a big endian f64 float,\n/// otherwise if `nom::number::Endianness::Little` parse a little endian f64 float.\n/// *complete version*: returns an error if there is not enough input data\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::f64;\n///\n/// let be_f64 = |s| {\n///   f64(nom::number::Endianness::Big)(s)\n/// };\n///\n/// assert_eq!(be_f64(&[0x40, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(be_f64(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n///\n/// let le_f64 = |s| {\n///   f64(nom::number::Endianness::Little)(s)\n/// };\n///\n/// assert_eq!(le_f64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x40][..]), Ok((&b\"\"[..], 12.5)));\n/// assert_eq!(le_f64(&b\"abc\"[..]), Err(Err::Error((&b\"abc\"[..], ErrorKind::Eof))));\n/// ```\n#[inline]\npub fn f64<I, E: ParseError<I>>(endian: crate::number::Endianness) -> fn(I) -> IResult<I, f64, E>\nwhere\n  I: Input<Item = u8>,\n{\n  match endian {\n    crate::number::Endianness::Big => be_f64,\n    crate::number::Endianness::Little => le_f64,\n    #[cfg(target_endian = \"big\")]\n    crate::number::Endianness::Native => be_f64,\n    #[cfg(target_endian = \"little\")]\n    crate::number::Endianness::Native => le_f64,\n  }\n}\n\n/// Recognizes a hex-encoded integer.\n///\n/// *Complete version*: Will parse until the end of input if it has less than 8 bytes.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::hex_u32;\n///\n/// let parser = |s| {\n///   hex_u32(s)\n/// };\n///\n/// assert_eq!(parser(&b\"01AE\"[..]), Ok((&b\"\"[..], 0x01AE)));\n/// assert_eq!(parser(&b\"abc\"[..]), Ok((&b\"\"[..], 0x0ABC)));\n/// assert_eq!(parser(&b\"ggg\"[..]), Err(Err::Error((&b\"ggg\"[..], ErrorKind::IsA))));\n/// ```\n#[inline]\npub fn hex_u32<I, E: ParseError<I>>(input: I) -> IResult<I, u32, E>\nwhere\n  I: Input,\n  <I as Input>::Item: AsChar,\n  I: AsBytes,\n{\n  let e: ErrorKind = ErrorKind::IsA;\n  let (i, o) = input.split_at_position1_complete(\n    |c| {\n      let c = c.as_char();\n      !\"0123456789abcdefABCDEF\".contains(c)\n    },\n    e,\n  )?;\n\n  // Do not parse more than 8 characters for a u32\n  let (remaining, parsed) = if o.input_len() <= 8 {\n    (i, o)\n  } else {\n    input.take_split(8)\n  };\n\n  let res = parsed\n    .as_bytes()\n    .iter()\n    .rev()\n    .enumerate()\n    .map(|(k, &v)| {\n      let digit = v as char;\n      digit.to_digit(16).unwrap_or(0) << (k * 4)\n    })\n    .sum();\n\n  Ok((remaining, res))\n}\n\n/// Recognizes floating point number in a byte string and returns the corresponding slice.\n///\n/// *Complete version*: Can parse until the end of input.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::recognize_float;\n///\n/// let parser = |s| {\n///   recognize_float(s)\n/// };\n///\n/// assert_eq!(parser(\"11e-1\"), Ok((\"\", \"11e-1\")));\n/// assert_eq!(parser(\"123E-02\"), Ok((\"\", \"123E-02\")));\n/// assert_eq!(parser(\"123K-01\"), Ok((\"K-01\", \"123\")));\n/// assert_eq!(parser(\"abc\"), Err(Err::Error((\"abc\", ErrorKind::Char))));\n/// ```\n#[rustfmt::skip]\npub fn recognize_float<T, E:ParseError<T>>(input: T) -> IResult<T, T, E>\nwhere\n  T: Clone + Offset,\n  T: Input,\n  <T as Input>::Item: AsChar,\n{\n  recognize((\n    opt(alt((char('+'), char('-')))),\n      alt((\n        map((digit1, opt(pair(char('.'), opt(digit1)))), |_| ()),\n        map((char('.'), digit1), |_| ())\n      )),\n      opt((\n        alt((char('e'), char('E'))),\n        opt(alt((char('+'), char('-')))),\n        cut(digit1)\n      ))\n  )).parse(input)\n}\n\n// workaround until issues with minimal-lexical are fixed\n#[doc(hidden)]\npub fn recognize_float_or_exceptions<T, E: ParseError<T>>(input: T) -> IResult<T, T, E>\nwhere\n  T: Clone + Offset,\n  T: Input + Compare<&'static str>,\n  <T as Input>::Item: AsChar,\n{\n  alt((\n    |i: T| {\n      recognize_float::<_, E>(i.clone()).map_err(|e| match e {\n        crate::Err::Error(_) => crate::Err::Error(E::from_error_kind(i, ErrorKind::Float)),\n        crate::Err::Failure(_) => crate::Err::Failure(E::from_error_kind(i, ErrorKind::Float)),\n        crate::Err::Incomplete(needed) => crate::Err::Incomplete(needed),\n      })\n    },\n    |i: T| {\n      crate::bytes::complete::tag_no_case::<_, _, E>(\"nan\")(i.clone())\n        .map_err(|_| crate::Err::Error(E::from_error_kind(i, ErrorKind::Float)))\n    },\n    |i: T| {\n      crate::bytes::complete::tag_no_case::<_, _, E>(\"infinity\")(i.clone())\n        .map_err(|_| crate::Err::Error(E::from_error_kind(i, ErrorKind::Float)))\n    },\n    |i: T| {\n      crate::bytes::complete::tag_no_case::<_, _, E>(\"inf\")(i.clone())\n        .map_err(|_| crate::Err::Error(E::from_error_kind(i, ErrorKind::Float)))\n    },\n  ))\n  .parse(input)\n}\n\n/// Recognizes a floating point number in text format\n///\n/// It returns a tuple of (`sign`, `integer part`, `fraction part` and `exponent`) of the input\n/// data.\n///\n/// *Complete version*: Can parse until the end of input.\n///\npub fn recognize_float_parts<T, E: ParseError<T>>(input: T) -> IResult<T, (bool, T, T, i32), E>\nwhere\n  T: Clone + Offset,\n  T: Input,\n  <T as Input>::Item: AsChar,\n  T: for<'a> Compare<&'a [u8]>,\n  T: AsBytes,\n{\n  let (i, sign) = sign(input.clone())?;\n\n  //let (i, zeroes) = take_while(|c: <T as Input>::Item| c.as_char() == '0')(i)?;\n  let (i, zeroes) = match i.as_bytes().iter().position(|c| *c != b'0') {\n    Some(index) => i.take_split(index),\n    None => i.take_split(i.input_len()),\n  };\n  //let (i, mut integer) = digit0(i)?;\n  let (i, mut integer) = match i\n    .as_bytes()\n    .iter()\n    .position(|c| !(*c >= b'0' && *c <= b'9'))\n  {\n    Some(index) => i.take_split(index),\n    None => i.take_split(i.input_len()),\n  };\n\n  if integer.input_len() == 0 && zeroes.input_len() > 0 {\n    // keep the last zero if integer is empty\n    integer = zeroes.take_from(zeroes.input_len() - 1);\n  }\n\n  let (i, opt_dot) = opt(tag(&b\".\"[..])).parse(i)?;\n  let (i, fraction) = if opt_dot.is_none() {\n    let i2 = i.clone();\n    (i2, i.take(0))\n  } else {\n    // match number, trim right zeroes\n    let mut zero_count = 0usize;\n    let mut position = None;\n    for (pos, c) in i.as_bytes().iter().enumerate() {\n      if *c >= b'0' && *c <= b'9' {\n        if *c == b'0' {\n          zero_count += 1;\n        } else {\n          zero_count = 0;\n        }\n      } else {\n        position = Some(pos);\n        break;\n      }\n    }\n\n    #[allow(clippy::or_fun_call)]\n    let position = position.unwrap_or(i.input_len());\n\n    let index = if zero_count == 0 {\n      position\n    } else if zero_count == position {\n      position - zero_count + 1\n    } else {\n      position - zero_count\n    };\n\n    (i.take_from(position), i.take(index))\n  };\n\n  if integer.input_len() == 0 && fraction.input_len() == 0 {\n    return Err(Err::Error(E::from_error_kind(input, ErrorKind::Float)));\n  }\n\n  let i2 = i.clone();\n  let (i, e) = match i.as_bytes().iter().next() {\n    Some(b'e') => (i.take_from(1), true),\n    Some(b'E') => (i.take_from(1), true),\n    _ => (i, false),\n  };\n\n  let (i, exp) = if e {\n    cut(crate::character::complete::i32).parse(i)?\n  } else {\n    (i2, 0)\n  };\n\n  Ok((i, (sign, integer, fraction, exp)))\n}\n\nuse crate::traits::ParseTo;\n\n/// Recognizes floating point number in text format and returns a f32.\n///\n/// *Complete version*: Can parse until the end of input.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::float;\n///\n/// let parser = |s| {\n///   float(s)\n/// };\n///\n/// assert_eq!(parser(\"11e-1\"), Ok((\"\", 1.1)));\n/// assert_eq!(parser(\"123E-02\"), Ok((\"\", 1.23)));\n/// assert_eq!(parser(\"123K-01\"), Ok((\"K-01\", 123.0)));\n/// assert_eq!(parser(\"abc\"), Err(Err::Error((\"abc\", ErrorKind::Float))));\n/// ```\npub fn float<T, E: ParseError<T>>(input: T) -> IResult<T, f32, E>\nwhere\n  T: Clone + Offset + ParseTo<f32> + Compare<&'static str>,\n  T: Input,\n  <T as Input>::Item: AsChar,\n  <T as Input>::Iter: Clone,\n  T: AsBytes,\n  T: for<'a> Compare<&'a [u8]>,\n{\n  /*\n  let (i, (sign, integer, fraction, exponent)) = recognize_float_parts(input)?;\n\n  let mut float: f32 = minimal_lexical::parse_float(\n    integer.as_bytes().iter(),\n    fraction.as_bytes().iter(),\n    exponent,\n  );\n  if !sign {\n    float = -float;\n  }\n\n  Ok((i, float))\n      */\n  let (i, s) = recognize_float_or_exceptions(input)?;\n  match s.parse_to() {\n    Some(f) => Ok((i, f)),\n    None => Err(crate::Err::Error(E::from_error_kind(\n      i,\n      crate::error::ErrorKind::Float,\n    ))),\n  }\n}\n\n/// Recognizes floating point number in text format and returns a f64.\n///\n/// *Complete version*: Can parse until the end of input.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed};\n/// # use nom::Needed::Size;\n/// use nom::number::complete::double;\n///\n/// let parser = |s| {\n///   double(s)\n/// };\n///\n/// assert_eq!(parser(\"11e-1\"), Ok((\"\", 1.1)));\n/// assert_eq!(parser(\"123E-02\"), Ok((\"\", 1.23)));\n/// assert_eq!(parser(\"123K-01\"), Ok((\"K-01\", 123.0)));\n/// assert_eq!(parser(\"abc\"), Err(Err::Error((\"abc\", ErrorKind::Float))));\n/// ```\npub fn double<T, E: ParseError<T>>(input: T) -> IResult<T, f64, E>\nwhere\n  T: Clone + Offset + ParseTo<f64> + Compare<&'static str>,\n  T: Input,\n  <T as Input>::Item: AsChar,\n  <T as Input>::Iter: Clone,\n  T: AsBytes,\n  T: for<'a> Compare<&'a [u8]>,\n{\n  /*\n  let (i, (sign, integer, fraction, exponent)) = recognize_float_parts(input)?;\n\n  let mut float: f64 = minimal_lexical::parse_float(\n    integer.as_bytes().iter(),\n    fraction.as_bytes().iter(),\n    exponent,\n  );\n  if !sign {\n    float = -float;\n  }\n\n  Ok((i, float))\n      */\n  let (i, s) = recognize_float_or_exceptions(input)?;\n  match s.parse_to() {\n    Some(f) => Ok((i, f)),\n    None => Err(crate::Err::Error(E::from_error_kind(\n      i,\n      crate::error::ErrorKind::Float,\n    ))),\n  }\n}\n\n#[cfg(test)]\nmod tests {\n  use super::*;\n  use crate::error::ErrorKind;\n  use crate::internal::Err;\n  use proptest::prelude::*;\n\n  macro_rules! assert_parse(\n    ($left: expr, $right: expr) => {\n      let res: $crate::IResult<_, _, (_, ErrorKind)> = $left;\n      assert_eq!(res, $right);\n    };\n  );\n\n  #[test]\n  fn i8_tests() {\n    assert_parse!(i8(&[0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(i8(&[0x7f][..]), Ok((&b\"\"[..], 127)));\n    assert_parse!(i8(&[0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(i8(&[0x80][..]), Ok((&b\"\"[..], -128)));\n  }\n\n  #[test]\n  \nfn be_i8_tests() {\n    assert_parse!(be_i8(&[0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_i8(&[0x7f][..]), Ok((&b\"\"[..], 127)));\n    assert_parse!(be_i8(&[0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(be_i8(&[0x80][..]), Ok((&b\"\"[..], -128)));\n  }\n\n  #[test]\n  fn be_i16_tests() {\n    assert_parse!(be_i16(&[0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_i16(&[0x7f, 0xff][..]), Ok((&b\"\"[..], 32_767_i16)));\n    assert_parse!(be_i16(&[0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(be_i16(&[0x80, 0x00][..]), Ok((&b\"\"[..], -32_768_i16)));\n  }\n\n  #[test]\n  fn be_u24_tests() {\n    assert_parse!(be_u24(&[0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_u24(&[0x00, 0xFF, 0xFF][..]), Ok((&b\"\"[..], 65_535_u32)));\n    assert_parse!(\n      be_u24(&[0x12, 0x34, 0x56][..]),\n      Ok((&b\"\"[..], 1_193_046_u32))\n    );\n  }\n\n  #[test]\n  fn be_i24_tests() {\n    assert_parse!(be_i24(&[0xFF, 0xFF, 0xFF][..]), Ok((&b\"\"[..], -1_i32)));\n    assert_parse!(be_i24(&[0xFF, 0x00, 0x00][..]), Ok((&b\"\"[..], -65_536_i32)));\n    assert_parse!(\n      be_i24(&[0xED, 0xCB, 0xAA][..]),\n      Ok((&b\"\"[..], -1_193_046_i32))\n    );\n  }\n\n  #[test]\n  fn be_i32_tests() {\n    assert_parse!(be_i32(&[0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(\n      be_i32(&[0x7f, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], 2_147_483_647_i32))\n    );\n    assert_parse!(be_i32(&[0xff, 0xff, 0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(\n      be_i32(&[0x80, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], -2_147_483_648_i32))\n    );\n  }\n\n  #[test]\n  fn be_i64_tests() {\n    assert_parse!(\n      be_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      be_i64(&[0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], 9_223_372_036_854_775_807_i64))\n    );\n    assert_parse!(\n      be_i64(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      be_i64(&[0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], -9_223_372_036_854_775_808_i64))\n    );\n  }\n\n  #[test]\n  fn be_i128_tests() {\n    assert_parse!(\n      be_i128(\n        &[\n          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x00\n        ][..]\n      ),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      be_i128(\n        &[\n          0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0xff\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        170_141_183_460_469_231_731_687_303_715_884_105_727_i128\n      ))\n    );\n    assert_parse!(\n      be_i128(\n        &[\n          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0xff\n        ][..]\n      ),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      be_i128(\n        &[\n          0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x00\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        -170_141_183_460_469_231_731_687_303_715_884_105_728_i128\n      ))\n    );\n  }\n\n  #[test]\n  fn le_i8_tests() {\n    assert_parse!(le_i8(&[0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(le_i8(&[0x7f][..]), Ok((&b\"\"[..], 127)));\n    assert_parse!(le_i8(&[0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(le_i8(&[0x80][..]), Ok((&b\"\"[..], -128)));\n  }\n\n  #[test]\n  fn le_i16_tests() {\n    assert_parse!(le_i16(&[0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(le_i16(&[0xff, 0x7f][..]), Ok((&b\"\"[..], 32_767_i16)));\n    assert_parse!(le_i16(&[0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(le_i16(&[0x00, 0x80][..]), Ok((&b\"\"[..], -32_768_i16)));\n  }\n\n  #[test]\n  fn le_u24_tests() {\n    assert_parse!(le_u24(&[0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(le_u24(&[0xFF, 0xFF, 0x00][..]), Ok((&b\"\"[..], 65_535_u32)));\n    assert_parse!(\n      le_u24(&[0x56, 0x34, 0x12][..]),\n      Ok((&b\"\"[..], 1_193_046_u32))\n    );\n  }\n\n  #[test]\n  fn le_i24_tests() {\n    assert_parse!(le_i24(&[0xFF, 0xFF, 0xFF][..]), Ok((&b\"\"[..], -1_i32)));\n    assert_parse!(le_i24(&[0x00, 0x00, 0xFF][..]), Ok((&b\"\"[..], -65_536_i32)));\n    assert_parse!(\n      le_i24(&[0xAA, 0xCB, 0xED][..]),\n      Ok((&b\"\"[..], -1_193_046_i32))\n    );\n  }\n\n  #[test]\n  fn le_i32_tests() {\n    assert_parse!(le_i32(&[0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(\n      le_i32(&[0xff, 0xff, 0xff, 0x7f][..]),\n      Ok((&b\"\"[..], 2_147_483_647_i32))\n    );\n    assert_parse!(le_i32(&[0xff, 0xff, 0xff, 0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(\n      le_i32(&[0x00, 0x00, 0x00, 0x80][..]),\n      Ok((&b\"\"[..], -2_147_483_648_i32))\n    );\n  }\n\n  #[test]\n  fn le_i64_tests() {\n    assert_parse!(\n      le_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00][..]),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      le_i64(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f][..]),\n      Ok((&b\"\"[..], 9_223_372_036_854_775_807_i64))\n    );\n    assert_parse!(\n      le_i64(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff][..]),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      le_i64(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80][..]),\n      Ok((&b\"\"[..], -9_223_372_036_854_775_808_i64))\n    );\n  }\n\n  #[test]\n  fn le_i128_tests() {\n    assert_parse!(\n      le_i128(\n        &[\n          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x00\n        ][..]\n      ),\n      Ok((&b\"\"[..], 0))\n    );\n    assert_parse!(\n      le_i128(\n        &[\n          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0x7f\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        170_141_183_460_469_231_731_687_303_715_884_105_727_i128\n      ))\n    );\n    assert_parse!(\n      le_i128(\n        &[\n          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,\n          0xff\n        ][..]\n      ),\n      Ok((&b\"\"[..], -1))\n    );\n    assert_parse!(\n      le_i128(\n        &[\n          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n          0x80\n        ][..]\n      ),\n      Ok((\n        &b\"\"[..],\n        -170_141_183_460_469_231_731_687_303_715_884_105_728_i128\n      ))\n    );\n  }\n\n  #[test]\n  fn be_f32_tests() {\n    assert_parse!(be_f32(&[0x00, 0x00, 0x00, 0x00][..]), Ok((&b\"\"[..], 0_f32)));\n    assert_parse!(\n      be_f32(&[0x4d, 0x31, 0x1f, 0xd8][..]),\n      Ok((&b\"\"[..], 185_728_392_f32))\n    );\n  }\n\n  #[test]\n  fn be_f64_tests() {\n    assert_parse!(\n...\n", "needle_token_start": 12167, "needle_token_end": 12302, "code_context_ntokens": 16307, "output": ["```rust\n#[test]\nfn be_i8_tests() {\n    assert_parse!(be_i8(&[0x00][..]), Ok((&b\"\"[..], 0)));\n    assert_parse!(be_i8(&[0x7f][..]), Ok((&b\"\"[..], 127)));\n    assert_parse!(be_i8(&[0xff][..]), Ok((&b\"\"[..], -1)));\n    assert_parse!(be_i8(&[0x80][..]), Ok((&b\"\"[..], -128)));\n}\n```"]}
{"repo": "rust-bakery/nom", "name": "tagtr_succeed", "language": "rust", "path": "src/str.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to test the success of a string matching parser, ensuring it correctly identifies and returns a specified substring from the beginning of a given input string.\n2. **Input**: The function takes a single string input which is expected to start with a predefined substring.\n3. **Output**: The output is a tuple containing the remainder of the string after the matched substring and the matched substring itself.\n4. **Procedure**: The function applies a string matching parser to the input. If the beginning of the input matches the predefined substring, the parser returns the matched substring and the rest of the input string. The function then verifies that the parser correctly consumed the matched substring without affecting the remainder of the input and that the returned matched substring is exactly as expected.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/multi/mod.rs\n//! Combinators applying their child parser multiple times\n\n#[cfg(test)]\nmod tests;\n\nuse core::marker::PhantomData;\n\nuse crate::bytes::take;\nuse crate::error::ErrorKind;\nuse crate::error::ParseError;\nuse crate::internal::{Err, Needed, Parser};\nuse crate::lib::std::num::NonZeroUsize;\n#[cfg(feature = \"alloc\")]\nuse crate::lib::std::vec::Vec;\nuse crate::Check;\nuse crate::Emit;\nuse crate::Input;\nuse crate::Mode;\nuse crate::OutputM;\nuse crate::OutputMode;\nuse crate::{\n  traits::{InputLength, ToUsize},\n  NomRange,\n};\n\n/// Don't pre-allocate more than 64KiB when calling `Vec::with_capacity`.\n///\n/// Pre-allocating memory is a nice optimization but count fields can't\n/// always be trusted. We should clamp initial capacities to some reasonable\n/// amount. This reduces the risk of a bogus count value triggering a panic\n/// due to an OOM error.\n///\n/// This does not affect correctness. Nom will always read the full number\n/// of elements regardless of the capacity cap.\n#[cfg(feature = \"alloc\")]\nconst MAX_INITIAL_CAPACITY_BYTES: usize = 65536;\n\n/// Repeats the embedded parser, gathering the results in a `Vec`.\n///\n/// This stops on [`Err::Error`] and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many0(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many0<I, F>(\n  f: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  Many0 { parser: f }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many0] combinator\npub struct Many0<F> {\n  parser: F,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Many0<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = crate::lib::std::vec::Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut acc = OM::Output::bind(|| crate::lib::std::vec::Vec::with_capacity(4));\n    loop {\n      let len = i.input_len();\n      match self\n        .parser\n        .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, acc)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, o)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many0)\n            })));\n          }\n\n          i = i1;\n\n          acc = OM::Output::combine(acc, o, |mut acc, o| {\n            acc.push(o);\n            acc\n          })\n        }\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser, gathering the results in a `Vec`.\n///\n/// This stops on [`Err::Error`] if there is at least one result,  and returns the results that were accumulated. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many1(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many1<I, F>(\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  Many1 { parser }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many1] combinator\npub struct Many1<F> {\n  parser: F,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Many1<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match self\n      .parser\n      .process::<OutputM<OM::Output, Emit, OM::Incomplete>>(i.clone())\n    {\n      Err(Err::Error(err)) => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::append(i, ErrorKind::Many1, err)\n      }))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e)),\n      Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),\n      Ok((i1, o)) => {\n        let mut acc = OM::Output::map(o, |o| {\n          let mut acc = crate::lib::std::vec::Vec::with_capacity(4);\n          acc.push(o);\n          acc\n        });\n\n        i = i1;\n\n        loop {\n          let len = i.input_len();\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, acc)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i1, o)) => {\n              // infinite loop check: the parser must always consume\n              if i1.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many0)\n                })));\n              }\n\n              i = i1;\n\n              acc = OM::Output::combine(acc, o, |mut acc, o| {\n                acc.push(o);\n                acc\n              })\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Applies the parser `f` until the parser `g` produces a result.\n///\n/// Returns a tuple of the results of `f` in a `Vec` and the result of `g`.\n///\n/// `f` keeps going so long as `g` produces [`Err::Error`]. To instead chain an error up, see [`cut`][crate::combinator::cut].\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many_till;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, (Vec<&str>, &str)> {\n///   many_till(tag(\"abc\"), tag(\"end\")).parse(s)\n/// };\n///\n/// assert_eq!(parser(\"abcabcend\"), Ok((\"\", (vec![\"abc\", \"abc\"], \"end\"))));\n/// assert_eq!(parser(\"abc123end\"), Err(Err::Error(Error::new(\"123end\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123end\"), Err(Err::Error(Error::new(\"123123end\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcendefg\"), Ok((\"efg\", (vec![\"abc\"], \"end\"))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many_till<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = (Vec<<F as Parser<I>>::Output>, <G as Parser<I>>::Output), Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  ManyTill {\n    f,\n    g,\n    e: PhantomData,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many_till] combinator\npub struct ManyTill<F, G, E> {\n  f: F,\n  g: G,\n  e: PhantomData<E>,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F, G, E> Parser<I> for ManyTill<F, G, E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  type Output = (Vec<<F as Parser<I>>::Output>, <G as Parser<I>>::Output);\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n    loop {\n      let len = i.input_len();\n      match self\n        .g\n        .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n      {\n        Ok((i1, o)) => return Ok((i1, OM::Output::combine(res, o, |res, o| (res, o)))),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n        Err(Err::Error(_)) => {\n          match self.f.process::<OM>(i.clone()) {\n            Err(Err::Error(err)) => {\n              return Err(Err::Error(OM::Error::map(err, |err| {\n                E::append(i, ErrorKind::ManyTill, err)\n              })))\n            }\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i1, o)) => {\n              // infinite loop check: the parser must always consume\n              if i1.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  E::from_error_kind(i, ErrorKind::Many0)\n                })));\n              }\n\n              i = i1;\n\n              res = OM::Output::combine(res, o, |mut acc, o| {\n                acc.push(o);\n                acc\n              })\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Alternates between two parsers to produce a list of elements.\n///\n/// This stops when either parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `sep` Parses the separator between list elements.\n/// * `f` Parses the elements of the list.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::separated_list0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   separated_list0(tag(\"|\"), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abc|abc|abc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123abc\"), Ok((\"123abc\", vec![\"abc\"])));\n/// assert_eq!(parser(\"abc|def\"), Ok((\"|def\", vec![\"abc\"])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"def|abc\"), Ok((\"def|abc\", vec![])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn separated_list0<I, E, F, G>(\n  sep: G,\n  f: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  SeparatedList0 {\n    parser: f,\n    separator: sep,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [separated_list0] combinator\npub struct SeparatedList0<F, G> {\n  parser: F,\n  separator: G,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, E: ParseError<I>, F, G> Parser<I> for SeparatedList0<F, G>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n\n    match self\n      .parser\n      .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n    {\n      Err(Err::Error(_)) => return Ok((i, res)),\n      Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n      Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n      Ok((i1, o)) => {\n        res = OM::Output::combine(res, o, |mut res, o| {\n          res.push(o);\n          res\n        });\n        i = i1;\n      }\n    }\n\n    loop {\n      let len = i.input_len();\n      match self\n        .separator\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, res)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, _)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::SeparatedList)\n            })));\n          }\n\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i1.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, res)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i2, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              i = i2;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Alternates between two parsers to produce a list of elements until [`Err::Error`].\n///\n/// Fails if the element parser does not produce at least one element.$\n///\n/// This stops when either parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `sep` Parses the separator between list elements.\n/// * `f` Parses the elements of the list.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::separated_list1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   separated_list1(tag(\"|\"), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abc|abc|abc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123abc\"), Ok((\"123abc\", vec![\"abc\"])));\n/// assert_eq!(parser(\"abc|def\"), Ok((\"|def\", vec![\"abc\"])));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"def|abc\"), Err(Err::Error(Error::new(\"def|abc\", ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn separated_list1<I, E, F, G>(\n  separator: G,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  SeparatedList1 { parser, separator }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [separated_list1] combinator\npub struct SeparatedList1<F, G> {\n  parser: F,\n  separator: G,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, E: ParseError<I>, F, G> Parser<I> for SeparatedList1<F, G>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n\n    match self.parser.process::<OM>(i.clone()) {\n      Err(e) => return Err(e),\n      Ok((i1, o)) => {\n        res = OM::Output::combine(res, o, |mut res, o| {\n          res.push(o);\n          res\n        });\n        i = i1;\n      }\n    }\n\n    loop {\n      let len = i.input_len();\n      match self\n        .separator\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, res)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, _)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::SeparatedList)\n            })));\n          }\n\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i1.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, res)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i2, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              i = i2;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times\n///\n/// This stops before `n` when the parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many_m_n(0, 2, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many_m_n<I, E, F>(\n  min: usize,\n  max: usize,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  ManyMN { parser, min, max }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many_m_n] combinator\npub struct ManyMN<F> {\n  parser: F,\n  min: usize,\n  max: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for ManyMN<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.min > self.max {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::ManyMN,\n      )));\n    }\n\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.min.min(max_initial_capacity))\n    });\n    for count in 0..self.max {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::ManyMN)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |mut res, value| {\n            res.push(value);\n            res\n          });\n          input = tail;\n        }\n        Err(Err::Error(e)) => {\n          if count < self.min {\n            return Err(Err::Error(OM::Error::map(e, |e| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::ManyMN, e)\n            })));\n          } else {\n            return Ok((input, res));\n          }\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Repeats the embedded parser, counting the results\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many0_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many0_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", 0)));\n/// assert_eq!(parser(\"\"), Ok((\"\", 0)));\n/// ```\npub fn many0_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many0Count { parser }\n}\n\n/// Parser implementation for the [many0_count] combinator\npub struct Many0Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many0Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    loop {\n      let input_ = input.clone();\n      let len = input.input_len();\n      match self\n        .parser\n        .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n      {\n        Ok((i, _)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0Count)\n            })));\n          }\n\n          input = i;\n          count += 1;\n        }\n\n        Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser, counting the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many1_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many1_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1Count))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1Count))));\n/// ```\npub fn many1_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many1Count { parser }\n}\n\n/// Parser implementation for the [many1_count] combinator\npub struct Many1Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many1Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    match self\n      .parser\n      .process::<OutputM<Check, Check, OM::Incomplete>>(input.clone())\n    {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(move || {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n      }))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e)),\n      Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),\n      Ok((mut input, _)) => {\n        count += 1;\n\n        loop {\n          let input_ = input.clone();\n          let len = input.input_len();\n          match self\n            .parser\n            .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n          {\n            Ok((i, _)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n                })));\n              }\n\n              input = i;\n              count += 1;\n            }\n\n            Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser `count` times, gathering the results in a `Vec`\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `count` How often to apply the parser.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   count(tag(\"abc\"), 2).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn count<I, F>(\n  parser: F,\n  count: usize,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  Count { parser, count }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [count] combinator\npub struct Count<F> {\n  parser: F,\n  count: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Count<F>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.count.min(max_initial_capacity))\n    });\n\n    for _ in 0..self.count {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          res = OM::Output::combine(res, o, |mut res, o| {\n            res.push(o);\n            res\n          });\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Runs the embedded parser repeatedly, filling the given slice with results.\n///\n/// This parser fails if the input runs out before the given slice is full.\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `buf` The slice to fill\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fill;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, [&str; 2]> {\n///   let mut buf = [\"\", \"\"];\n///   let (rest, ()) = fill(tag(\"abc\"), &mut buf).parse(s)?;\n///   Ok((rest, buf))\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", [\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", [\"abc\", \"abc\"])));\n/// ```\npub fn fill<'a, I, E, F>(\n  parser: F,\n  buf: &'a mut [<F as Parser<I>>::Output],\n) -> impl Parser<I, Output = (), Error = E> + 'a\nwhere\n  I: Clone,\n  F: Parser<I, Error = E> + 'a,\n  E: ParseError<I>,\n{\n  Fill { parser, buf }\n}\n\n/// Parser implementation for the [fill] combinator\npub struct Fill<'a, F, O> {\n  parser: F,\n  buf: &'a mut [O],\n}\n\nimpl<'a, I, F, O> Parser<I> for Fill<'a, F, O>\nwhere\n  I: Clone,\n  F: Parser<I, Output = O>,\n{\n  type Output = ();\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n\n    for elem in self.buf.iter_mut() {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          OM::Output::map(o, |o| *elem = o);\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, OM::Output::bind(|| ())))\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many0(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// ```\npub fn fold_many0<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany0 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many0] combinator\npub struct FoldMany0<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany0<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let mut input = i;\n\n    loop {\n      let i_ = input.clone();\n      let len = input.input_len();\n      match self.parser.process::<OM>(i_) {\n        Ok((i, o)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0)\n            })));\n          }\n\n          res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n          input = i;\n        }\n        Err(Err::Error(_)) => {\n          return Ok((input, res));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fold_many1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many1(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1))));\n/// ```\npub fn fold_many1<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany1 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many1] combinator\npub struct FoldMany1<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany1<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let input = i.clone();\n\n    match self.parser.process::<OM>(input) {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many1)\n      }))),\n      Err(e) => Err(e),\n      Ok((i1, o1)) => {\n        res = OM::Output::combine(res, o1, |res, o| (self.g)(res, o));\n\n        let mut input = i1;\n        loop {\n          let i_ = input.clone();\n          let len = input.input_len();\n          match self.parser.process::<OM>(i_) {\n            Ok((i, o)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1)\n                })));\n              }\n\n              res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n              input = i;\n            }\n            Err(Err::Error(_)) => {\n              return Ok((input, res));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times, calling `g` to gather the results\n///\n/// This stops before `n` when the parser returns [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many_m_n(\n///     0,\n///     2,\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\npub fn fold_many_m_n<I, E, F, G, H, R>(\n  min: usize,\n  max: usize,\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldManyMN {\n    parser,\n    g,\n    init,\n    min,\n    max,\n    r: PhantomData,\n  }\n}\n...\n// Path: src/str.rs\n#[cfg(test)]\nmod test {\n  #[cfg(feature = \"alloc\")]\n  use crate::{branch::alt, bytes::complete::tag_no_case, combinator::recognize, multi::many};\n  use crate::{\n    bytes::complete::{is_a, is_not, tag, take, take_till, take_until},\n    error::{self, ErrorKind},\n    Err, IResult,\n  };\n\n  #[test]\n  \nfn tagtr_succeed() {\n    const INPUT: &str = \"Hello World!\";\n    const TAG: &str = \"Hello\";\n    fn test(input: &str) -> IResult<&str, &str> {\n      tag(TAG)(input)\n    }\n\n    match test(INPUT) {\n      Ok((extra, output)) => {\n        assert!(extra == \" World!\", \"Parser `tag` consumed leftover input.\");\n        assert!(\n          output == TAG,\n          \"Parser `tag` doesn't return the tag it matched on success. \\\n           Expected `{}`, got `{}`.\",\n          TAG,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `tag` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn tagtr_incomplete() {\n    use crate::bytes::streaming::tag;\n\n    const INPUT: &str = \"Hello\";\n    const TAG: &str = \"Hello World!\";\n\n    let res: IResult<_, _, error::Error<_>> = tag(TAG)(INPUT);\n    match res {\n      Err(Err::Incomplete(_)) => (),\n      other => {\n        panic!(\n          \"Parser `tag` didn't require more input when it should have. \\\n           Got `{:?}`.\",\n          other\n        );\n      }\n    };\n  }\n\n  #[test]\n  fn tagtr_error() {\n    const INPUT: &str = \"Hello World!\";\n    const TAG: &str = \"Random\"; // TAG must be closer than INPUT.\n\n    let res: IResult<_, _, error::Error<_>> = tag(TAG)(INPUT);\n    match res {\n      Err(Err::Error(_)) => (),\n      other => {\n        panic!(\n          \"Parser `tag` didn't fail when it should have. Got `{:?}`.`\",\n          other\n        );\n      }\n    };\n  }\n\n  #[test]\n  fn take_s_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\u0192\u01ad\u00e8\u0159\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\";\n    const LEFTOVER: &str = \"\u00e1\u0192\u01ad\u00e8\u0159\";\n\n    let res: IResult<_, _, error::Error<_>> = take(9_usize)(INPUT);\n    match res {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_s` consumed leftover input. Leftover `{}`.\",\n          extra\n        );\n        assert!(\n          output == CONSUMED,\n          \"Parser `take_s` doesn't return the string it consumed on success. Expected `{}`, got `{}`.\",\n          CONSUMED,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `take_s` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn take_until_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u2202\u00e1\u0192\u01ad\u00e8\u0159\";\n    const FIND: &str = \"\u00c2\u00df\u00c7\u2202\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\";\n    const LEFTOVER: &str = \"\u00c2\u00df\u00c7\u2202\u00e1\u0192\u01ad\u00e8\u0159\";\n\n    let res: IResult<_, _, (_, ErrorKind)> = take_until(FIND)(INPUT);\n    match res {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_until`\\\n           consumed leftover input. Leftover `{}`.\",\n          extra\n        );\n        assert!(\n          output == CONSUMED,\n          \"Parser `take_until`\\\n           doesn't return the string it consumed on success. Expected `{}`, got `{}`.\",\n          CONSUMED,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `take_until` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn take_s_incomplete() {\n    use crate::bytes::streaming::take;\n\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\";\n\n    let res: IResult<_, _, (_, ErrorKind)> = take(13_usize)(INPUT);\n    match res {\n      Err(Err::Incomplete(_)) => (),\n      other => panic!(\n        \"Parser `take` didn't require more input when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    }\n  }\n\n  use crate::internal::Needed;\n\n  fn is_alphabetic(c: char) -> bool {\n    (c as u8 >= 0x41 && c as u8 <= 0x5A) || (c as u8 >= 0x61 && c as u8 <= 0x7A)\n  }\n\n  #[test]\n  fn take_while() {\n    use crate::bytes::streaming::take_while;\n\n    fn f(i: &str) -> IResult<&str, &str> {\n      take_while(is_alphabetic)(i)\n    }\n    let a = \"\";\n    let b = \"abcd\";\n    let c = \"abcd123\";\n    let d = \"123\";\n\n    assert_eq!(f(a), Err(Err::Incomplete(Needed::new(1))));\n    assert_eq!(f(b), Err(Err::Incomplete(Needed::new(1))));\n    assert_eq!(f(c), Ok((d, b)));\n    assert_eq!(f(d), Ok((d, a)));\n  }\n\n  #[test]\n  fn take_while1() {\n    use crate::bytes::streaming::take_while1;\n\n    fn f(i: &str) -> IResult<&str, &str> {\n      take_while1(is_alphabetic)(i)\n    }\n    let a = \"\";\n    let b = \"abcd\";\n    let c = \"abcd123\";\n    let d = \"123\";\n\n    assert_eq!(f(a), Err(Err::Incomplete(Needed::new(1))));\n    assert_eq!(f(b), Err(Err::Incomplete(Needed::new(1))));\n    assert_eq!(f(c), Ok((\"123\", b)));\n    assert_eq!(\n      f(d),\n      Err(Err::Error(error_position!(d, ErrorKind::TakeWhile1)))\n    );\n  }\n\n  #[test]\n  fn take_till_s_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\u0192\u01ad\u00e8\u0159\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\";\n    const LEFTOVER: &str = \"\u00e1\u0192\u01ad\u00e8\u0159\";\n    fn till_s(c: char) -> bool {\n      c == '\u00e1'\n    }\n    fn test(input: &str) -> IResult<&str, &str> {\n      take_till(till_s)(input)\n    }\n    match test(INPUT) {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_till` consumed leftover input.\"\n        );\n        assert!(\n          output == CONSUMED,\n          \"Parser `take_till` doesn't return the string it consumed on success. \\\n           Expected `{}`, got `{}`.\",\n          CONSUMED,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `take_till` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn take_while_succeed_none() {\n    use crate::bytes::complete::take_while;\n\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\u0192\u01ad\u00e8\u0159\";\n    const CONSUMED: &str = \"\";\n    const LEFTOVER: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\u0192\u01ad\u00e8\u0159\";\n    fn while_s(c: char) -> bool {\n      c == '9'\n    }\n    fn test(input: &str) -> IResult<&str, &str> {\n      take_while(while_s)(input)\n    }\n    match test(INPUT) {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n          \"Parser `take_while` consumed leftover input.\"\n        );\n        assert!(\n          output == CONSUMED,\n          \"Parser `take_while` doesn't return the string it consumed on success. \\\n           Expected `{}`, got `{}`.\",\n          CONSUMED,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `take_while` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n\n  #[test]\n  fn is_not_succeed() {\n    const INPUT: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\u00e1\u0192\u01ad\u00e8\u0159\";\n    const AVOID: &str = \"\u00a3\u00fa\u00e7\u0199\u00a5\u00e1\";\n    const CONSUMED: &str = \"\u03b2\u00e8\u0192\u00f4\u0159\u00e8\u00c2\u00df\u00c7\";\n    const LEFTOVER: &str = \"\u00e1\u0192\u01ad\u00e8\u0159\";\n    fn test(input: &str) -> IResult<&str, &str> {\n      is_not(AVOID)(input)\n    }\n    match test(INPUT) {\n      Ok((extra, output)) => {\n        assert!(\n          extra == LEFTOVER,\n...\n", "needle_token_start": 13768, "needle_token_end": 13978, "code_context_ntokens": 16293, "output": ["```rust\n#[test]\nfn tagtr_succeed() {\n    const INPUT: &str = \"Hello World!\";\n    const TAG: &str = \"Hello\";\n    fn test(input: &str) -> IResult<&str, &str> {\n      tag(TAG)(input)\n    }\n\n    match test(INPUT) {\n      Ok((extra, output)) => {\n        assert!(extra == \" World!\", \"Parser `tag` consumed leftover input.\");\n        assert!(\n          output == TAG,\n          \"Parser `tag` doesn't return the tag it matched on success. \\\n           Expected `{}`, got `{}`.\",\n          TAG,\n          output\n        );\n      }\n      other => panic!(\n        \"Parser `tag` didn't succeed when it should have. \\\n         Got `{:?}`.\",\n        other\n      ),\n    };\n  }\n```"]}
{"repo": "rust-bakery/nom", "name": "length_count", "language": "rust", "path": "src/multi/mod.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to parse a sequence by first determining the number of elements to parse based on the output of a preliminary parser, and then applying a secondary parser that many times to the input.\n2. **Input**: The input is a byte slice or stream that begins with data that can be parsed to determine the number of elements, followed by the elements themselves.\n3. **Output**: The output is a vector containing the results of applying the secondary parser the specified number of times.\n4. **Procedure**: The function first applies the preliminary parser to the input to determine the number of elements to parse. This number is then used to repeatedly apply the secondary parser to subsequent parts of the input, collecting the results into a vector. If the input does not conform to the expected format (e.g., insufficient data for the number of elements), appropriate errors are returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "/// use nom::multi::many0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many0(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many0<I, F>(\n  f: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  Many0 { parser: f }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many0] combinator\npub struct Many0<F> {\n  parser: F,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Many0<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = crate::lib::std::vec::Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut acc = OM::Output::bind(|| crate::lib::std::vec::Vec::with_capacity(4));\n    loop {\n      let len = i.input_len();\n      match self\n        .parser\n        .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, acc)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, o)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many0)\n            })));\n          }\n\n          i = i1;\n\n          acc = OM::Output::combine(acc, o, |mut acc, o| {\n            acc.push(o);\n            acc\n          })\n        }\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser, gathering the results in a `Vec`.\n///\n/// This stops on [`Err::Error`] if there is at least one result,  and returns the results that were accumulated. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many1(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many1<I, F>(\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  Many1 { parser }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many1] combinator\npub struct Many1<F> {\n  parser: F,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Many1<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match self\n      .parser\n      .process::<OutputM<OM::Output, Emit, OM::Incomplete>>(i.clone())\n    {\n      Err(Err::Error(err)) => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::append(i, ErrorKind::Many1, err)\n      }))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e)),\n      Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),\n      Ok((i1, o)) => {\n        let mut acc = OM::Output::map(o, |o| {\n          let mut acc = crate::lib::std::vec::Vec::with_capacity(4);\n          acc.push(o);\n          acc\n        });\n\n        i = i1;\n\n        loop {\n          let len = i.input_len();\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, acc)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i1, o)) => {\n              // infinite loop check: the parser must always consume\n              if i1.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many0)\n                })));\n              }\n\n              i = i1;\n\n              acc = OM::Output::combine(acc, o, |mut acc, o| {\n                acc.push(o);\n                acc\n              })\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Applies the parser `f` until the parser `g` produces a result.\n///\n/// Returns a tuple of the results of `f` in a `Vec` and the result of `g`.\n///\n/// `f` keeps going so long as `g` produces [`Err::Error`]. To instead chain an error up, see [`cut`][crate::combinator::cut].\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many_till;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, (Vec<&str>, &str)> {\n///   many_till(tag(\"abc\"), tag(\"end\")).parse(s)\n/// };\n///\n/// assert_eq!(parser(\"abcabcend\"), Ok((\"\", (vec![\"abc\", \"abc\"], \"end\"))));\n/// assert_eq!(parser(\"abc123end\"), Err(Err::Error(Error::new(\"123end\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123end\"), Err(Err::Error(Error::new(\"123123end\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcendefg\"), Ok((\"efg\", (vec![\"abc\"], \"end\"))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many_till<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = (Vec<<F as Parser<I>>::Output>, <G as Parser<I>>::Output), Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  ManyTill {\n    f,\n    g,\n    e: PhantomData,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many_till] combinator\npub struct ManyTill<F, G, E> {\n  f: F,\n  g: G,\n  e: PhantomData<E>,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F, G, E> Parser<I> for ManyTill<F, G, E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  type Output = (Vec<<F as Parser<I>>::Output>, <G as Parser<I>>::Output);\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n    loop {\n      let len = i.input_len();\n      match self\n        .g\n        .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n      {\n        Ok((i1, o)) => return Ok((i1, OM::Output::combine(res, o, |res, o| (res, o)))),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n        Err(Err::Error(_)) => {\n          match self.f.process::<OM>(i.clone()) {\n            Err(Err::Error(err)) => {\n              return Err(Err::Error(OM::Error::map(err, |err| {\n                E::append(i, ErrorKind::ManyTill, err)\n              })))\n            }\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i1, o)) => {\n              // infinite loop check: the parser must always consume\n              if i1.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  E::from_error_kind(i, ErrorKind::Many0)\n                })));\n              }\n\n              i = i1;\n\n              res = OM::Output::combine(res, o, |mut acc, o| {\n                acc.push(o);\n                acc\n              })\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Alternates between two parsers to produce a list of elements.\n///\n/// This stops when either parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `sep` Parses the separator between list elements.\n/// * `f` Parses the elements of the list.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::separated_list0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   separated_list0(tag(\"|\"), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abc|abc|abc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123abc\"), Ok((\"123abc\", vec![\"abc\"])));\n/// assert_eq!(parser(\"abc|def\"), Ok((\"|def\", vec![\"abc\"])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"def|abc\"), Ok((\"def|abc\", vec![])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn separated_list0<I, E, F, G>(\n  sep: G,\n  f: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  SeparatedList0 {\n    parser: f,\n    separator: sep,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [separated_list0] combinator\npub struct SeparatedList0<F, G> {\n  parser: F,\n  separator: G,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, E: ParseError<I>, F, G> Parser<I> for SeparatedList0<F, G>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n\n    match self\n      .parser\n      .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i.clone())\n    {\n      Err(Err::Error(_)) => return Ok((i, res)),\n      Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n      Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n      Ok((i1, o)) => {\n        res = OM::Output::combine(res, o, |mut res, o| {\n          res.push(o);\n          res\n        });\n        i = i1;\n      }\n    }\n\n    loop {\n      let len = i.input_len();\n      match self\n        .separator\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, res)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, _)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::SeparatedList)\n            })));\n          }\n\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i1.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, res)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i2, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              i = i2;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Alternates between two parsers to produce a list of elements until [`Err::Error`].\n///\n/// Fails if the element parser does not produce at least one element.$\n///\n/// This stops when either parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `sep` Parses the separator between list elements.\n/// * `f` Parses the elements of the list.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::separated_list1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   separated_list1(tag(\"|\"), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abc|abc|abc\"), Ok((\"\", vec![\"abc\", \"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123abc\"), Ok((\"123abc\", vec![\"abc\"])));\n/// assert_eq!(parser(\"abc|def\"), Ok((\"|def\", vec![\"abc\"])));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"def|abc\"), Err(Err::Error(Error::new(\"def|abc\", ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn separated_list1<I, E, F, G>(\n  separator: G,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  SeparatedList1 { parser, separator }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [separated_list1] combinator\npub struct SeparatedList1<F, G> {\n  parser: F,\n  separator: G,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, E: ParseError<I>, F, G> Parser<I> for SeparatedList1<F, G>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut i: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(crate::lib::std::vec::Vec::new);\n\n    match self.parser.process::<OM>(i.clone()) {\n      Err(e) => return Err(e),\n      Ok((i1, o)) => {\n        res = OM::Output::combine(res, o, |mut res, o| {\n          res.push(o);\n          res\n        });\n        i = i1;\n      }\n    }\n\n    loop {\n      let len = i.input_len();\n      match self\n        .separator\n        .process::<OutputM<Check, Check, OM::Incomplete>>(i.clone())\n      {\n        Err(Err::Error(_)) => return Ok((i, res)),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n        Ok((i1, _)) => {\n          // infinite loop check: the parser must always consume\n          if i1.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::SeparatedList)\n            })));\n          }\n\n          match self\n            .parser\n            .process::<OutputM<OM::Output, Check, OM::Incomplete>>(i1.clone())\n          {\n            Err(Err::Error(_)) => return Ok((i, res)),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(e)) => return Err(Err::Incomplete(e)),\n            Ok((i2, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              i = i2;\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times\n///\n/// This stops before `n` when the parser returns [`Err::Error`]  and returns the results that were accumulated. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   many_m_n(0, 2, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn many_m_n<I, E, F>(\n  min: usize,\n  max: usize,\n  parser: F,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  ManyMN { parser, min, max }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [many_m_n] combinator\npub struct ManyMN<F> {\n  parser: F,\n  min: usize,\n  max: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for ManyMN<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.min > self.max {\n      return Err(Err::Failure(<F as Parser<I>>::Error::from_error_kind(\n        input,\n        ErrorKind::ManyMN,\n      )));\n    }\n\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.min.min(max_initial_capacity))\n    });\n    for count in 0..self.max {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::ManyMN)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |mut res, value| {\n            res.push(value);\n            res\n          });\n          input = tail;\n        }\n        Err(Err::Error(e)) => {\n          if count < self.min {\n            return Err(Err::Error(OM::Error::map(e, |e| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::ManyMN, e)\n            })));\n          } else {\n            return Ok((input, res));\n          }\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Repeats the embedded parser, counting the results\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::many0_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many0_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", 0)));\n/// assert_eq!(parser(\"\"), Ok((\"\", 0)));\n/// ```\npub fn many0_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many0Count { parser }\n}\n\n/// Parser implementation for the [many0_count] combinator\npub struct Many0Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many0Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    loop {\n      let input_ = input.clone();\n      let len = input.input_len();\n      match self\n        .parser\n        .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n      {\n        Ok((i, _)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0Count)\n            })));\n          }\n\n          input = i;\n          count += 1;\n        }\n\n        Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n        Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n        Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser, counting the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::many1_count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, usize> {\n///   many1_count(tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", 2)));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", 1)));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1Count))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1Count))));\n/// ```\npub fn many1_count<I, E, F>(parser: F) -> impl Parser<I, Output = usize, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  Many1Count { parser }\n}\n\n/// Parser implementation for the [many1_count] combinator\npub struct Many1Count<F> {\n  parser: F,\n}\n\nimpl<I, F> Parser<I> for Many1Count<F>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n{\n  type Output = usize;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut count = 0;\n\n    match self\n      .parser\n      .process::<OutputM<Check, Check, OM::Incomplete>>(input.clone())\n    {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(move || {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n      }))),\n      Err(Err::Failure(e)) => Err(Err::Failure(e)),\n      Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),\n      Ok((mut input, _)) => {\n        count += 1;\n\n        loop {\n          let input_ = input.clone();\n          let len = input.input_len();\n          match self\n            .parser\n            .process::<OutputM<Check, Check, OM::Incomplete>>(input_)\n          {\n            Ok((i, _)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1Count)\n                })));\n              }\n\n              input = i;\n              count += 1;\n            }\n\n            Err(Err::Error(_)) => return Ok((input, OM::Output::bind(|| count))),\n            Err(Err::Failure(e)) => return Err(Err::Failure(e)),\n            Err(Err::Incomplete(i)) => return Err(Err::Incomplete(i)),\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Runs the embedded parser `count` times, gathering the results in a `Vec`\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `count` How often to apply the parser.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::count;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   count(tag(\"abc\"), 2).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(feature = \"docsrs\", doc(cfg(feature = \"alloc\")))]\npub fn count<I, F>(\n  parser: F,\n  count: usize,\n) -> impl Parser<I, Output = Vec<<F as Parser<I>>::Output>, Error = <F as Parser<I>>::Error>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  Count { parser, count }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [count] combinator\npub struct Count<F> {\n  parser: F,\n  count: usize,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F> Parser<I> for Count<F>\nwhere\n  I: Clone,\n  F: Parser<I>,\n{\n  type Output = Vec<<F as Parser<I>>::Output>;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n    let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n      / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n    let mut res = OM::Output::bind(|| {\n      crate::lib::std::vec::Vec::with_capacity(self.count.min(max_initial_capacity))\n    });\n\n    for _ in 0..self.count {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          res = OM::Output::combine(res, o, |mut res, o| {\n            res.push(o);\n            res\n          });\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Runs the embedded parser repeatedly, filling the given slice with results.\n///\n/// This parser fails if the input runs out before the given slice is full.\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `buf` The slice to fill\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fill;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, [&str; 2]> {\n///   let mut buf = [\"\", \"\"];\n///   let (rest, ()) = fill(tag(\"abc\"), &mut buf).parse(s)?;\n///   Ok((rest, buf))\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", [\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Err(Err::Error(Error::new(\"123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Tag))));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", [\"abc\", \"abc\"])));\n/// ```\npub fn fill<'a, I, E, F>(\n  parser: F,\n  buf: &'a mut [<F as Parser<I>>::Output],\n) -> impl Parser<I, Output = (), Error = E> + 'a\nwhere\n  I: Clone,\n  F: Parser<I, Error = E> + 'a,\n  E: ParseError<I>,\n{\n  Fill { parser, buf }\n}\n\n/// Parser implementation for the [fill] combinator\npub struct Fill<'a, F, O> {\n  parser: F,\n  buf: &'a mut [O],\n}\n\nimpl<'a, I, F, O> Parser<I> for Fill<'a, F, O>\nwhere\n  I: Clone,\n  F: Parser<I, Output = O>,\n{\n  type Output = ();\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut input = i.clone();\n\n    for elem in self.buf.iter_mut() {\n      let input_ = input.clone();\n      match self.parser.process::<OM>(input_) {\n        Ok((i, o)) => {\n          OM::Output::map(o, |o| *elem = o);\n          input = i;\n        }\n        Err(Err::Error(e)) => {\n          return Err(Err::Error(OM::Error::map(e, |e| {\n            <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n          })));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n\n    Ok((input, OM::Output::bind(|| ())))\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: if the parser passed in accepts empty inputs (like `alpha0` or `digit0`), `many0` will\n/// return an error, to prevent going into an infinite loop\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many0;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many0(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// ```\npub fn fold_many0<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany0 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many0] combinator\npub struct FoldMany0<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany0<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let mut input = i;\n\n    loop {\n      let i_ = input.clone();\n      let len = input.input_len();\n      match self.parser.process::<OM>(i_) {\n        Ok((i, o)) => {\n          // infinite loop check: the parser must always consume\n          if i.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many0)\n            })));\n          }\n\n          res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n          input = i;\n        }\n        Err(Err::Error(_)) => {\n          return Ok((input, res));\n        }\n        Err(e) => {\n          return Err(e);\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser, calling `g` to gather the results.\n///\n/// This stops on [`Err::Error`] if there is at least one result. To instead chain an error up,\n/// see [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::multi::fold_many1;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many1(\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Err(Err::Error(Error::new(\"123123\", ErrorKind::Many1))));\n/// assert_eq!(parser(\"\"), Err(Err::Error(Error::new(\"\", ErrorKind::Many1))));\n/// ```\npub fn fold_many1<I, E, F, G, H, R>(\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldMany1 {\n    parser,\n    g,\n    init,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many1] combinator\npub struct FoldMany1<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldMany1<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(&mut self, i: I) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let mut res = OM::Output::bind(|| (self.init)());\n    let input = i.clone();\n\n    match self.parser.process::<OM>(input) {\n      Err(Err::Error(_)) => Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(i, ErrorKind::Many1)\n      }))),\n      Err(e) => Err(e),\n      Ok((i1, o1)) => {\n        res = OM::Output::combine(res, o1, |res, o| (self.g)(res, o));\n\n        let mut input = i1;\n        loop {\n          let i_ = input.clone();\n          let len = input.input_len();\n          match self.parser.process::<OM>(i_) {\n            Ok((i, o)) => {\n              // infinite loop check: the parser must always consume\n              if i.input_len() == len {\n                return Err(Err::Error(OM::Error::bind(|| {\n                  <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::Many1)\n                })));\n              }\n\n              res = OM::Output::combine(res, o, |res, o| (self.g)(res, o));\n              input = i;\n            }\n            Err(Err::Error(_)) => {\n              return Ok((input, res));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser `m..=n` times, calling `g` to gather the results\n///\n/// This stops before `n` when the parser returns [`Err::Error`]. To instead chain an error up, see\n/// [`cut`][crate::combinator::cut].\n///\n/// # Arguments\n/// * `m` The minimum number of iterations.\n/// * `n` The maximum number of iterations.\n/// * `f` The parser to apply.\n/// * `init` A function returning the initial value.\n/// * `g` The function that combines a result of `f` with\n///       the current accumulator.\n///\n/// *Note*: If the parser passed to `many1` accepts empty inputs\n/// (like `alpha0` or `digit0`), `many1` will return an error,\n/// to prevent going into an infinite loop.\n///\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::multi::fold_many_m_n;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &str) -> IResult<&str, Vec<&str>> {\n///   fold_many_m_n(\n///     0,\n///     2,\n///     tag(\"abc\"),\n///     Vec::new,\n///     |mut acc: Vec<_>, item| {\n///       acc.push(item);\n///       acc\n///     }\n///   ).parse(s)\n/// }\n///\n/// assert_eq!(parser(\"abcabc\"), Ok((\"\", vec![\"abc\", \"abc\"])));\n/// assert_eq!(parser(\"abc123\"), Ok((\"123\", vec![\"abc\"])));\n/// assert_eq!(parser(\"123123\"), Ok((\"123123\", vec![])));\n/// assert_eq!(parser(\"\"), Ok((\"\", vec![])));\n/// assert_eq!(parser(\"abcabcabc\"), Ok((\"abc\", vec![\"abc\", \"abc\"])));\n/// ```\npub fn fold_many_m_n<I, E, F, G, H, R>(\n  min: usize,\n  max: usize,\n  parser: F,\n  init: H,\n  g: G,\n) -> impl Parser<I, Output = R, Error = E>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I, Error = E>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  H: FnMut() -> R,\n  E: ParseError<I>,\n{\n  FoldManyMN {\n    parser,\n    g,\n    init,\n    min,\n    max,\n    r: PhantomData,\n  }\n}\n\n/// Parser implementation for the [fold_many_m_n] combinator\npub struct FoldManyMN<F, G, Init, R> {\n  parser: F,\n  g: G,\n  init: Init,\n  r: PhantomData<R>,\n  min: usize,\n  max: usize,\n}\n\nimpl<I, F, G, Init, R> Parser<I> for FoldManyMN<F, G, Init, R>\nwhere\n  I: Clone + InputLength,\n  F: Parser<I>,\n  G: FnMut(R, <F as Parser<I>>::Output) -> R,\n  Init: FnMut() -> R,\n{\n  type Output = R;\n  type Error = <F as Parser<I>>::Error;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    mut input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    if self.min > self.max {\n      return Err(Err::Error(OM::Error::bind(|| {\n        <F as Parser<I>>::Error::from_error_kind(input, ErrorKind::ManyMN)\n      })));\n    }\n\n    let mut res = OM::Output::bind(|| (self.init)());\n    for count in 0..self.max {\n      let len = input.input_len();\n      match self.parser.process::<OM>(input.clone()) {\n        Ok((tail, value)) => {\n          // infinite loop check: the parser must always consume\n          if tail.input_len() == len {\n            return Err(Err::Error(OM::Error::bind(|| {\n              <F as Parser<I>>::Error::from_error_kind(tail, ErrorKind::ManyMN)\n            })));\n          }\n\n          res = OM::Output::combine(res, value, |res, o| (self.g)(res, o));\n          input = tail;\n        }\n        Err(Err::Error(err)) => {\n          if count < self.min {\n            return Err(Err::Error(OM::Error::map(err, |err| {\n              <F as Parser<I>>::Error::append(input, ErrorKind::ManyMN, err)\n            })));\n          } else {\n            break;\n          }\n        }\n        Err(e) => return Err(e),\n      }\n    }\n\n    Ok((input, res))\n  }\n}\n\n/// Gets a number from the parser and returns a\n/// subslice of the input of that size.\n/// If the parser returns `Incomplete`,\n/// `length_data` will return an error.\n/// # Arguments\n/// * `f` The parser to apply.\n/// ```rust\n/// # use nom::{Err, error::ErrorKind, Needed, IResult, Parser};\n/// use nom::number::complete::be_u16;\n/// use nom::multi::length_data;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   length_data(be_u16).parse(s)\n/// }\n///\n/// assert_eq!(parser(b\"\\x00\\x03abcefg\"), Ok((&b\"efg\"[..], &b\"abc\"[..])));\n/// assert_eq!(parser(b\"\\x00\\x03a\"), Err(Err::Incomplete(Needed::new(2))));\n/// ```\npub fn length_data<I, E, F>(f: F) -> impl Parser<I, Output = I, Error = E>\nwhere\n  I: Input,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  f.flat_map(|size| take(size))\n}\n\n/// Gets a number from the first parser,\n/// takes a subslice of the input of that size,\n/// then applies the second parser on that subslice.\n/// If the second parser returns `Incomplete`,\n/// `length_value` will return an error.\n/// # Arguments\n/// * `f` The parser to apply.\n/// * `g` The parser to apply on the subslice.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::number::complete::be_u16;\n/// use nom::multi::length_value;\n/// use nom::bytes::complete::tag;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], &[u8]> {\n///   length_value(be_u16, tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(b\"\\x00\\x03abcefg\"), Ok((&b\"efg\"[..], &b\"abc\"[..])));\n/// assert_eq!(parser(b\"\\x00\\x03123123\"), Err(Err::Error(Error::new(&b\"123\"[..], ErrorKind::Tag))));\n/// assert_eq!(parser(b\"\\x00\\x03a\"), Err(Err::Incomplete(Needed::new(2))));\n/// ```\npub fn length_value<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = <G as Parser<I>>::Output, Error = E>\nwhere\n  I: Clone + Input,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  /*f.flat_map(|size| {\n    println!(\"got size: {size}\");\n    take(size)\n  })\n  .and_then(g)*/\n  LengthValue {\n    length: f,\n    parser: g,\n    e: PhantomData,\n  }\n}\n\n/// Parser implementation for the [length_value] combinator\npub struct LengthValue<F, G, E> {\n  length: F,\n  parser: G,\n  e: PhantomData<E>,\n}\n\nimpl<I, F, G, E> Parser<I> for LengthValue<F, G, E>\nwhere\n  I: Clone + Input,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  <F as Parser<I>>::Output: ToUsize,\n  E: ParseError<I>,\n{\n  type Output = <G as Parser<I>>::Output;\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    let (i, length) = self\n      .length\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(input)?;\n\n    let length: usize = length.to_usize();\n\n    if let Some(needed) = length\n      .checked_sub(i.input_len())\n      .and_then(NonZeroUsize::new)\n    {\n      Err(Err::Incomplete(Needed::Size(needed)))\n    } else {\n      let (rest, i) = i.take_split(length);\n      match self.parser.process::<OM>(i.clone()) {\n        Err(Err::Incomplete(_)) => Err(Err::Error(OM::Error::bind(|| {\n          E::from_error_kind(i, ErrorKind::Complete)\n        }))),\n        Err(e) => Err(e),\n        Ok((_, o)) => Ok((rest, o)),\n      }\n    }\n  }\n}\n\n/// Gets a number from the first parser,\n/// then applies the second parser that many times.\n/// # Arguments\n/// * `f` The parser to apply to obtain the count.\n/// * `g` The parser to apply repeatedly.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::number::complete::u8;\n/// use nom::multi::length_count;\n/// use nom::bytes::complete::tag;\n/// use nom::combinator::map;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n///   length_count(map(u8, |i| {\n///      println!(\"got number: {}\", i);\n///      i\n///   }), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(&b\"\\x02abcabcabc\"[..]), Ok(((&b\"abc\"[..], vec![&b\"abc\"[..], &b\"abc\"[..]]))));\n/// assert_eq!(parser(b\"\\x03123123123\"), Err(Err::Error(Error::new(&b\"123123123\"[..], ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n\npub fn length_count<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = Vec<<G as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  LengthCount {\n    length: f,\n    parser: g,\n    e: PhantomData,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [length_count] combinator\npub struct LengthCount<F, G, E> {\n  length: F,\n  parser: G,\n  e: PhantomData<E>,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F, G, E> Parser<I> for LengthCount<F, G, E>\nwhere\n  I: Clone,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  <F as Parser<I>>::Output: ToUsize,\n  E: ParseError<I>,\n{\n  type Output = Vec<<G as Parser<I>>::Output>;\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match self\n      .length\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(input)\n    {\n      Err(e) => Err(e),\n      Ok((i, count)) => {\n        let count = count.to_usize();\n        let mut input = i.clone();\n        let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n          / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n        let mut res = OM::Output::bind(|| {\n          crate::lib::std::vec::Vec::with_capacity(count.min(max_initial_capacity))\n        });\n\n        for _ in 0..count {\n          let input_ = input.clone();\n          match self.parser.process::<OM>(input_) {\n            Ok((i, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              input = i;\n            }\n            Err(Err::Error(e)) => {\n              return Err(Err::Error(OM::Error::map(e, |e| {\n                <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n              })));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n\n        Ok((input, res))\n      }\n    }\n  }\n}\n\n/// Repeats the embedded parser and collects the results in a type implementing `Extend + Default`.\n/// Fails if the amount of time the embedded parser is run is not\n/// within the specified range.\n/// # Arguments\n/// * `range` Constrains the number of iterations.\n///   * A range without an upper bound `a..` is equivalent to a range of `a..=usize::MAX`.\n///   * A single `usize` value is equivalent to `value..=value`.\n...\n", "needle_token_start": 15412, "needle_token_end": 15559, "code_context_ntokens": 16294, "output": ["```rust\n/// Gets a number from the first parser,\n/// then applies the second parser that many times.\n/// # Arguments\n/// * `f` The parser to apply to obtain the count.\n/// * `g` The parser to apply repeatedly.\n/// ```rust\n/// # use nom::{Err, error::{Error, ErrorKind}, Needed, IResult, Parser};\n/// use nom::number::complete::u8;\n/// use nom::multi::length_count;\n/// use nom::bytes::complete::tag;\n/// use nom::combinator::map;\n///\n/// fn parser(s: &[u8]) -> IResult<&[u8], Vec<&[u8]>> {\n///   length_count(map(u8, |i| {\n///      println!(\"got number: {}\", i);\n///      i\n///   }), tag(\"abc\")).parse(s)\n/// }\n///\n/// assert_eq!(parser(&b\"\\x02abcabcabc\"[..]), Ok(((&b\"abc\"[..], vec![&b\"abc\"[..], &b\"abc\"[..]]))));\n/// assert_eq!(parser(b\"\\x03123123123\"), Err(Err::Error(Error::new(&b\"123123123\"[..], ErrorKind::Tag))));\n/// ```\n#[cfg(feature = \"alloc\")]\n\npub fn length_count<I, E, F, G>(\n  f: F,\n  g: G,\n) -> impl Parser<I, Output = Vec<<G as Parser<I>>::Output>, Error = E>\nwhere\n  I: Clone,\n  <F as Parser<I>>::Output: ToUsize,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  E: ParseError<I>,\n{\n  LengthCount {\n    length: f,\n    parser: g,\n    e: PhantomData,\n  }\n}\n\n#[cfg(feature = \"alloc\")]\n/// Parser implementation for the [length_count] combinator\npub struct LengthCount<F, G, E> {\n  length: F,\n  parser: G,\n  e: PhantomData<E>,\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<I, F, G, E> Parser<I> for LengthCount<F, G, E>\nwhere\n  I: Clone,\n  F: Parser<I, Error = E>,\n  G: Parser<I, Error = E>,\n  <F as Parser<I>>::Output: ToUsize,\n  E: ParseError<I>,\n{\n  type Output = Vec<<G as Parser<I>>::Output>;\n  type Error = E;\n\n  fn process<OM: OutputMode>(\n    &mut self,\n    input: I,\n  ) -> crate::PResult<OM, I, Self::Output, Self::Error> {\n    match self\n      .length\n      .process::<OutputM<Emit, OM::Error, OM::Incomplete>>(input)\n    {\n      Err(e) => Err(e),\n      Ok((i, count)) => {\n        let count = count.to_usize();\n        let mut input = i.clone();\n        let max_initial_capacity = MAX_INITIAL_CAPACITY_BYTES\n          / crate::lib::std::mem::size_of::<<F as Parser<I>>::Output>().max(1);\n        let mut res = OM::Output::bind(|| {\n          crate::lib::std::vec::Vec::with_capacity(count.min(max_initial_capacity))\n        });\n\n        for _ in 0..count {\n          let input_ = input.clone();\n          match self.parser.process::<OM>(input_) {\n            Ok((i, o)) => {\n              res = OM::Output::combine(res, o, |mut res, o| {\n                res.push(o);\n                res\n              });\n              input = i;\n            }\n            Err(Err::Error(e)) => {\n              return Err(Err::Error(OM::Error::map(e, |e| {\n                <F as Parser<I>>::Error::append(i, ErrorKind::Count, e)\n              })));\n            }\n            Err(e) => {\n              return Err(e);\n            }\n          }\n        }\n\n        Ok((input, res))\n      }\n    }\n  }\n}\n```"]}
{"repo": "helix-editor/helix", "name": "composition", "language": "rust", "path": "helix-core/src/transaction.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To combine two sets of document changes sequentially into a single set that represents the cumulative effect of both sets.\n2. **Input**: Two sets of changes, where each set contains operations like retaining a segment of text, inserting new text, or deleting a segment.\n3. **Output**: A new set of changes that represents the combined effect of the two input sets on a document.\n4. **Procedure**: The function sequentially applies the operations from the first set of changes to the document, and then applies the operations from the second set to the result of the first. This results in a new set of changes that effectively merges both operations into a single transformative action on the document.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            Selection::new(ranges, new_primary_idx.unwrap_or(0)),\n        )\n    }\n\n    /// Generate a transaction with a deletion per selection range.\n    /// Compared to using `change_by_selection` directly these ranges may overlap.\n    /// In that case they are merged\n    pub fn delete_by_selection<F>(doc: &Rope, selection: &Selection, f: F) -> Self\n    where\n        F: FnMut(&Range) -> Deletion,\n    {\n        Self::delete(doc, selection.iter().map(f))\n    }\n\n    /// Insert text at each selection head.\n    pub fn insert(doc: &Rope, selection: &Selection, text: Tendril) -> Self {\n        Self::change_by_selection(doc, selection, |range| {\n            (range.head, range.head, Some(text.clone()))\n        })\n    }\n\n    pub fn changes_iter(&self) -> ChangeIterator {\n        self.changes.changes_iter()\n    }\n}\n\nimpl From<ChangeSet> for Transaction {\n    fn from(changes: ChangeSet) -> Self {\n        Self {\n            changes,\n            selection: None,\n        }\n    }\n}\n\npub struct ChangeIterator<'a> {\n    iter: std::iter::Peekable<std::slice::Iter<'a, Operation>>,\n    pos: usize,\n}\n\nimpl<'a> ChangeIterator<'a> {\n    fn new(changeset: &'a ChangeSet) -> Self {\n        let iter = changeset.changes.iter().peekable();\n        Self { iter, pos: 0 }\n    }\n}\n\nimpl<'a> Iterator for ChangeIterator<'a> {\n    type Item = Change;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        use Operation::*;\n\n        loop {\n            match self.iter.next()? {\n                Retain(len) => {\n                    self.pos += len;\n                }\n                Delete(len) => {\n                    let start = self.pos;\n                    self.pos += len;\n                    return Some((start, self.pos, None));\n                }\n                Insert(s) => {\n                    let start = self.pos;\n                    // a subsequent delete means a replace, consume it\n                    if let Some(Delete(len)) = self.iter.peek() {\n                        self.iter.next();\n\n                        self.pos += len;\n                        return Some((start, self.pos, Some(s.clone())));\n                    } else {\n                        return Some((start, start, Some(s.clone())));\n                    }\n                }\n            }\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::history::State;\n\n    #[test]\n    fn com\nposition() {\n        use Operation::*;\n\n        let a = ChangeSet {\n            changes: vec![\n                Retain(5),\n                Insert(\" test!\".into()),\n                Retain(1),\n                Delete(2),\n                Insert(\"abc\".into()),\n            ],\n            len: 8,\n            len_after: 15,\n        };\n\n        let b = ChangeSet {\n            changes: vec![Delete(10), Insert(\"\u4e16orld\".into()), Retain(5)],\n            len: 15,\n            len_after: 10,\n        };\n\n        let mut text = Rope::from(\"hello xz\");\n\n        // should probably return cloned text\n        let composed = a.compose(b);\n        assert_eq!(composed.len, 8);\n        assert!(composed.apply(&mut text));\n        assert_eq!(text, \"\u4e16orld! abc\");\n    }\n\n    #[test]\n    fn invert() {\n        use Operation::*;\n\n        let changes = ChangeSet {\n            changes: vec![Retain(4), Insert(\"test\".into()), Delete(5), Retain(3)],\n            len: 12,\n            len_after: 11,\n        };\n\n        let doc = Rope::from(\"\u4e16\u754c3 hello xz\");\n        let revert = changes.invert(&doc);\n\n        let mut doc2 = doc.clone();\n        changes.apply(&mut doc2);\n\n        // a revert is different\n        assert_ne!(changes, revert);\n        assert_ne!(doc, doc2);\n\n        // but inverting a revert will give us the original\n        assert_eq!(changes, revert.invert(&doc2));\n\n        // applying a revert gives us back the original\n        revert.apply(&mut doc2);\n        assert_eq!(doc, doc2);\n    }\n\n    #[test]\n    fn map_pos() {\n        use Operation::*;\n\n        // maps inserts\n        let cs = ChangeSet {\n            changes: vec![Retain(4), Insert(\"!!\".into()), Retain(4)],\n            len: 8,\n            len_after: 10,\n        };\n\n        assert_eq!(cs.map_pos(0, Assoc::Before), 0); // before insert region\n        assert_eq!(cs.map_pos(4, Assoc::Before), 4); // at insert, track before\n        assert_eq!(cs.map_pos(4, Assoc::After), 6); // at insert, track after\n        assert_eq!(cs.map_pos(5, Assoc::Before), 7); // after insert region\n\n        // maps deletes\n        let cs = ChangeSet {\n            changes: vec![Retain(4), Delete(4), Retain(4)],\n            len: 12,\n            len_after: 8,\n        };\n        assert_eq!(cs.map_pos(0, Assoc::Before), 0); // at start\n        assert_eq!(cs.map_pos(4, Assoc::Before), 4); // before a delete\n        assert_eq!(cs.map_pos(5, Assoc::Before), 4); // inside a delete\n        assert_eq!(cs.map_pos(5, Assoc::After), 4); // inside a delete\n\n        // TODO: delete tracking\n\n        // stays inbetween replacements\n        let cs = ChangeSet {\n            changes: vec![\n                Insert(\"ab\".into()),\n                Delete(2),\n                Insert(\"cd\".into()),\n                Delete(2),\n            ],\n            len: 4,\n            len_after: 4,\n        };\n        assert_eq!(cs.map_pos(2, Assoc::Before), 2);\n        assert_eq!(cs.map_pos(2, Assoc::After), 2);\n        // unsorted selection\n        let cs = ChangeSet {\n            changes: vec![\n                Insert(\"ab\".into()),\n                Delete(2),\n                Insert(\"cd\".into()),\n                Delete(2),\n            ],\n            len: 4,\n            len_after: 4,\n        };\n        let mut positions = [4, 2];\n        cs.update_positions(positions.iter_mut().map(|pos| (pos, Assoc::After)));\n        assert_eq!(positions, [4, 2]);\n        // stays at word boundary\n        let cs = ChangeSet {\n            changes: vec![\n                Retain(2), // <space><space>\n                Insert(\" ab\".into()),\n                Retain(2), // cd\n                Insert(\"de \".into()),\n            ],\n            len: 4,\n            len_after: 10,\n        };\n        assert_eq!(cs.map_pos(2, Assoc::BeforeWord), 3);\n        assert_eq!(cs.map_pos(4, Assoc::AfterWord), 9);\n        let cs = ChangeSet {\n            changes: vec![\n                Retain(1), // <space>\n                Insert(\" b\".into()),\n                Delete(1), // c\n                Retain(1), // d\n                Insert(\"e \".into()),\n                Delete(1), // <space>\n            ],\n            len: 5,\n            len_after: 7,\n        };\n        assert_eq!(cs.map_pos(1, Assoc::BeforeWord), 2);\n        assert_eq!(cs.map_pos(3, Assoc::AfterWord), 5);\n        let cs = ChangeSet {\n            changes: vec![\n                Retain(1), // <space>\n                Insert(\"a\".into()),\n                Delete(2), // <space>b\n                Retain(1), // d\n                Insert(\"e\".into()),\n                Delete(1), // f\n                Retain(1), // <space>\n            ],\n            len: 5,\n            len_after: 7,\n        };\n        assert_eq!(cs.map_pos(2, Assoc::BeforeWord), 1);\n        assert_eq!(cs.map_pos(4, Assoc::AfterWord), 4);\n    }\n\n    #[test]\n    fn transaction_change() {\n        let mut doc = Rope::from(\"hello world!\\ntest 123\");\n        let transaction = Transaction::change(\n            &doc,\n            // (1, 1, None) is a useless 0-width delete that gets factored out\n            vec![(1, 1, None), (6, 11, Some(\"void\".into())), (12, 17, None)].into_iter(),\n        );\n        transaction.apply(&mut doc);\n        assert_eq!(doc, Rope::from_str(\"hello void! 123\"));\n    }\n\n    #[test]\n    fn changes_iter() {\n        let doc = Rope::from(\"hello world!\\ntest 123\");\n        let changes = vec![(6, 11, Some(\"void\".into())), (12, 17, None)];\n        let transaction = Transaction::change(&doc, changes.clone().into_iter());\n        assert_eq!(transaction.changes_iter().collect::<Vec<_>>(), changes);\n    }\n\n    #[test]\n    fn optimized_composition() {\n        let mut state = State {\n            doc: \"\".into(),\n            selection: Selection::point(0),\n        };\n        let t1 = Transaction::insert(&state.doc, &state.selection, Tendril::from(\"h\"));\n        t1.apply(&mut state.doc);\n        state.selection = state.selection.clone().map(t1.changes());\n        let t2 = Transaction::insert(&state.doc, &state.selection, Tendril::from(\"e\"));\n        t2.apply(&mut state.doc);\n        state.selection = state.selection.clone().map(t2.changes());\n        let t3 = Transaction::insert(&state.doc, &state.selection, Tendril::from(\"l\"));\n        t3.apply(&mut state.doc);\n        state.selection = state.selection.clone().map(t3.changes());\n        let t4 = Transaction::insert(&state.doc, &state.selection, Tendril::from(\"l\"));\n        t4.apply(&mut state.doc);\n        state.selection = state.selection.clone().map(t4.changes());\n        let t5 = Transaction::insert(&state.doc, &state.selection, Tendril::from(\"o\"));\n        t5.apply(&mut state.doc);\n        state.selection = state.selection.clone().map(t5.changes());\n\n        assert_eq!(state.doc, Rope::from_str(\"hello\"));\n\n        // changesets as follows:\n        // h\n        // retain 1, e\n        // retain 2, l\n\n        let changes = t1\n            .changes\n            .compose(t2.changes)\n            .compose(t3.changes)\n            .compose(t4.changes)\n            .compose(t5.changes);\n\n        use Operation::*;\n        assert_eq!(changes.changes, &[Insert(\"hello\".into())]);\n        // instead of insert h, insert e, insert l, insert l, insert o\n    }\n\n    #[test]\n    fn combine_with_empty() {\n        let empty = Rope::from(\"\");\n        let a = ChangeSet::new(empty.slice(..));\n\n        let mut b = ChangeSet::new(empty.slice(..));\n        b.insert(\"a\".into());\n\n        let changes = a.compose(b);\n\n        use Operation::*;\n        assert_eq!(changes.changes, &[Insert(\"a\".into())]);\n    }\n\n    #[test]\n    fn combine_with_utf8() {\n        const TEST_CASE: &str = \"Hello, \u3053\u308c\u306f\u30d8\u30ea\u30c3\u30af\u30b9\u30a8\u30c7\u30a3\u30bf\u30fc\u3067\u3059\uff01\";\n\n        let empty = Rope::from(\"\");\n        let a = ChangeSet::new(empty.slice(..));\n\n        let mut b = ChangeSet::new(empty.slice(..));\n        b.insert(TEST_CASE.into());\n\n        let changes = a.compose(b);\n\n        use Operation::*;\n        assert_eq!(changes.changes, &[Insert(TEST_CASE.into())]);\n        assert_eq!(changes.len_after, TEST_CASE.chars().count());\n    }\n}\n\n// Path: helix-core/src/auto_pairs.rs\n//! When typing the opening character of one of the possible pairs defined below,\n//! this module provides the functionality to insert the paired closing character.\n\nuse crate::{graphemes, movement::Direction, Range, Rope, Selection, Tendril, Transaction};\nuse std::collections::HashMap;\n\nuse smallvec::SmallVec;\n\n// Heavily based on https://github.com/codemirror/closebrackets/\npub const DEFAULT_PAIRS: &[(char, char)] = &[\n    ('(', ')'),\n    ('{', '}'),\n    ('[', ']'),\n    ('\\'', '\\''),\n    ('\"', '\"'),\n    ('`', '`'),\n];\n\n/// The type that represents the collection of auto pairs,\n/// keyed by both opener and closer.\n#[derive(Debug, Clone)]\npub struct AutoPairs(HashMap<char, Pair>);\n\n/// Represents the config for a particular pairing.\n#[derive(Debug, Clone, Copy)]\npub struct Pair {\n    pub open: char,\n    pub close: char,\n}\n\nimpl Pair {\n    /// true if open == close\n    pub fn same(&self) -> bool {\n        self.open == self.close\n    }\n\n    /// true if all of the pair's conditions hold for the given document and range\n    pub fn should_close(&self, doc: &Rope, range: &Range) -> bool {\n        let mut should_close = Self::next_is_not_alpha(doc, range);\n\n        if self.same() {\n            should_close &= Self::prev_is_not_alpha(doc, range);\n        }\n\n        should_close\n    }\n\n    pub fn next_is_not_alpha(doc: &Rope, range: &Range) -> bool {\n        let cursor = range.cursor(doc.slice(..));\n        let next_char = doc.get_char(cursor);\n        next_char.map(|c| !c.is_alphanumeric()).unwrap_or(true)\n    }\n\n    pub fn prev_is_not_alpha(doc: &Rope, range: &Range) -> bool {\n        let cursor = range.cursor(doc.slice(..));\n        let prev_char = prev_char(doc, cursor);\n        prev_char.map(|c| !c.is_alphanumeric()).unwrap_or(true)\n    }\n}\n\nimpl From<&(char, char)> for Pair {\n    fn from(&(open, close): &(char, char)) -> Self {\n        Self { open, close }\n    }\n}\n\nimpl From<(&char, &char)> for Pair {\n    fn from((open, close): (&char, &char)) -> Self {\n        Self {\n            open: *open,\n            close: *close,\n        }\n    }\n}\n\nimpl AutoPairs {\n    /// Make a new AutoPairs set with the given pairs and default conditions.\n    pub fn new<'a, V: 'a, A>(pairs: V) -> Self\n    where\n        V: IntoIterator<Item = A>,\n        A: Into<Pair>,\n    {\n        let mut auto_pairs = HashMap::new();\n\n        for pair in pairs.into_iter() {\n            let auto_pair = pair.into();\n\n            auto_pairs.insert(auto_pair.open, auto_pair);\n\n            if auto_pair.open != auto_pair.close {\n                auto_pairs.insert(auto_pair.close, auto_pair);\n            }\n        }\n\n        Self(auto_pairs)\n    }\n\n    pub fn get(&self, ch: char) -> Option<&Pair> {\n        self.0.get(&ch)\n    }\n}\n\nimpl Default for AutoPairs {\n    fn default() -> Self {\n        AutoPairs::new(DEFAULT_PAIRS.iter())\n    }\n}\n\n// insert hook:\n// Fn(doc, selection, char) => Option<Transaction>\n// problem is, we want to do this per range, so we can call default handler for some ranges\n// so maybe ret Vec<Option<Change>>\n// but we also need to be able to return transactions...\n//\n// to simplify, maybe return Option<Transaction> and just reimplement the default\n\n// [TODO]\n// * delete implementation where it erases the whole bracket (|) -> |\n// * change to multi character pairs to handle cases like placing the cursor in the\n//   middle of triple quotes, and more exotic pairs like Jinja's {% %}\n\n#[must_use]\npub fn hook(doc: &Rope, selection: &Selection, ch: char, pairs: &AutoPairs) -> Option<Transaction> {\n    log::trace!(\"autopairs hook selection: {:#?}\", selection);\n\n    if let Some(pair) = pairs.get(ch) {\n        if pair.same() {\n            return Some(handle_same(doc, selection, pair));\n        } else if pair.open == ch {\n            return Some(handle_open(doc, selection, pair));\n        } else if pair.close == ch {\n            // && char_at pos == close\n            return Some(handle_close(doc, selection, pair));\n        }\n    }\n\n    None\n}\n\nfn prev_char(doc: &Rope, pos: usize) -> Option<char> {\n    if pos == 0 {\n        return None;\n    }\n\n    doc.get_char(pos - 1)\n}\n\n/// calculate what the resulting range should be for an auto pair insertion\nfn get_next_range(doc: &Rope, start_range: &Range, offset: usize, len_inserted: usize) -> Range {\n    // When the character under the cursor changes due to complete pair\n    // insertion, we must look backward a grapheme and then add the length\n    // of the insertion to put the resulting cursor in the right place, e.g.\n    //\n    // foo[\\r\\n] - anchor: 3, head: 5\n    // foo([)]\\r\\n - anchor: 4, head: 5\n    //\n    // foo[\\r\\n] - anchor: 3, head: 5\n    // foo'[\\r\\n] - anchor: 4, head: 6\n    //\n    // foo([)]\\r\\n - anchor: 4, head: 5\n    // foo()[\\r\\n] - anchor: 5, head: 7\n    //\n    // [foo]\\r\\n - anchor: 0, head: 3\n    // [foo(])\\r\\n - anchor: 0, head: 5\n\n    // inserting at the very end of the document after the last newline\n    if start_range.head == doc.len_chars() && start_range.anchor == doc.len_chars() {\n        return Range::new(\n            start_range.anchor + offset + 1,\n            start_range.head + offset + 1,\n        );\n    }\n\n    let doc_slice = doc.slice(..);\n    let single_grapheme = start_range.is_single_grapheme(doc_slice);\n\n    // just skip over graphemes\n    if len_inserted == 0 {\n        let end_anchor = if single_grapheme {\n            graphemes::next_grapheme_boundary(doc_slice, start_range.anchor) + offset\n\n        // even for backward inserts with multiple grapheme selections,\n        // we want the anchor to stay where it is so that the relative\n        // selection does not change, e.g.:\n        //\n        // foo([) wor]d -> insert ) -> foo()[ wor]d\n        } else {\n            start_range.anchor + offset\n        };\n\n        return Range::new(\n            end_anchor,\n            graphemes::next_grapheme_boundary(doc_slice, start_range.head) + offset,\n        );\n    }\n\n    // trivial case: only inserted a single-char opener, just move the selection\n    if len_inserted == 1 {\n        let end_anchor = if single_grapheme || start_range.direction() == Direction::Backward {\n            start_range.anchor + offset + 1\n        } else {\n            start_range.anchor + offset\n        };\n\n        return Range::new(end_anchor, start_range.head + offset + 1);\n    }\n\n    // If the head = 0, then we must be in insert mode with a backward\n    // cursor, which implies the head will just move\n    let end_head = if start_range.head == 0 || start_range.direction() == Direction::Backward {\n        start_range.head + offset + 1\n    } else {\n        // We must have a forward cursor, which means we must move to the\n        // other end of the grapheme to get to where the new characters\n        // are inserted, then move the head to where it should be\n        let prev_bound = graphemes::prev_grapheme_boundary(doc_slice, start_range.head);\n        log::trace!(\n            \"prev_bound: {}, offset: {}, len_inserted: {}\",\n            prev_bound,\n            offset,\n            len_inserted\n        );\n        prev_bound + offset + len_inserted\n    };\n\n    let end_anchor = match (start_range.len(), start_range.direction()) {\n        // if we have a zero width cursor, it shifts to the same number\n        (0, _) => end_head,\n\n        // If we are inserting for a regular one-width cursor, the anchor\n        // moves with the head. This is the fast path for ASCII.\n        (1, Direction::Forward) => end_head - 1,\n        (1, Direction::Backward) => end_head + 1,\n\n        (_, Direction::Forward) => {\n            if single_grapheme {\n                graphemes::prev_grapheme_boundary(doc.slice(..), start_range.head) + 1\n\n            // if we are appending, the anchor stays where it is; only offset\n            // for multiple range insertions\n            } else {\n                start_range.anchor + offset\n            }\n        }\n\n        (_, Direction::Backward) => {\n            if single_grapheme {\n                // if we're backward, then the head is at the first char\n                // of the typed char, so we need to add the length of\n                // the closing char\n                graphemes::prev_grapheme_boundary(doc.slice(..), start_range.anchor)\n                    + len_inserted\n                    + offset\n            } else {\n                // when we are inserting in front of a selection, we need to move\n                // the anchor over by however many characters were inserted overall\n                start_range.anchor + offset + len_inserted\n            }\n        }\n    };\n\n    Range::new(end_anchor, end_head)\n}\n\nfn handle_open(doc: &Rope, selection: &Selection, pair: &Pair) -> Transaction {\n    let mut end_ranges = SmallVec::with_capacity(selection.len());\n    let mut offs = 0;\n\n    let transaction = Transaction::change_by_selection(doc, selection, |start_range| {\n        let cursor = start_range.cursor(doc.slice(..));\n        let next_char = doc.get_char(cursor);\n        let len_inserted;\n\n        // Since auto pairs are currently limited to single chars, we're either\n        // inserting exactly one or two chars. When arbitrary length pairs are\n        // added, these will need to be changed.\n        let change = match next_char {\n            Some(_) if !pair.should_close(doc, start_range) => {\n                len_inserted = 1;\n                let mut tendril = Tendril::new();\n                tendril.push(pair.open);\n                (cursor, cursor, Some(tendril))\n            }\n            _ => {\n                // insert open & close\n                let pair_str = Tendril::from_iter([pair.open, pair.close]);\n                len_inserted = 2;\n                (cursor, cursor, Some(pair_str))\n            }\n        };\n\n        let next_range = get_next_range(doc, start_range, offs, len_inserted);\n        end_ranges.push(next_range);\n        offs += len_inserted;\n\n        change\n    });\n\n    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));\n    log::debug!(\"auto pair transaction: {:#?}\", t);\n    t\n}\n\nfn handle_close(doc: &Rope, selection: &Selection, pair: &Pair) -> Transaction {\n    let mut end_ranges = SmallVec::with_capacity(selection.len());\n    let mut offs = 0;\n\n    let transaction = Transaction::change_by_selection(doc, selection, |start_range| {\n        let cursor = start_range.cursor(doc.slice(..));\n        let next_char = doc.get_char(cursor);\n        let mut len_inserted = 0;\n\n        let change = if next_char == Some(pair.close) {\n            // return transaction that moves past close\n            (cursor, cursor, None) // no-op\n        } else {\n            len_inserted = 1;\n            let mut tendril = Tendril::new();\n            tendril.push(pair.close);\n            (cursor, cursor, Some(tendril))\n        };\n\n        let next_range = get_next_range(doc, start_range, offs, len_inserted);\n        end_ranges.push(next_range);\n        offs += len_inserted;\n\n        change\n    });\n\n    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));\n    log::debug!(\"auto pair transaction: {:#?}\", t);\n    t\n}\n\n/// handle cases where open and close is the same, or in triples (\"\"\"docstring\"\"\")\nfn handle_same(doc: &Rope, selection: &Selection, pair: &Pair) -> Transaction {\n    let mut end_ranges = SmallVec::with_capacity(selection.len());\n\n    let mut offs = 0;\n\n    let transaction = Transaction::change_by_selection(doc, selection, |start_range| {\n        let cursor = start_range.cursor(doc.slice(..));\n        let mut len_inserted = 0;\n        let next_char = doc.get_char(cursor);\n\n        let change = if next_char == Some(pair.open) {\n            //  return transaction that moves past close\n            (cursor, cursor, None) // no-op\n        } else {\n            let mut pair_str = Tendril::new();\n            pair_str.push(pair.open);\n\n            // for equal pairs, don't insert both open and close if either\n            // side has a non-pair char\n            if pair.should_close(doc, start_range) {\n                pair_str.push(pair.close);\n            }\n\n            len_inserted += pair_str.chars().count();\n            (cursor, cursor, Some(pair_str))\n        };\n\n        let next_range = get_next_range(doc, start_range, offs, len_inserted);\n        end_ranges.push(next_range);\n        offs += len_inserted;\n\n        change\n    });\n\n    let t = transaction.with_selection(Selection::new(end_ranges, selection.primary_index()));\n    log::debug!(\"auto pair transaction: {:#?}\", t);\n    t\n}\n\n// Path: helix-core/src/syntax.rs\nmod tree_cursor;\n\nuse crate::{\n    auto_pairs::AutoPairs,\n    chars::char_is_line_ending,\n    diagnostic::Severity,\n    regex::Regex,\n    transaction::{ChangeSet, Operation},\n    RopeSlice, Tendril,\n};\n\nuse ahash::RandomState;\nuse arc_swap::{ArcSwap, Guard};\nuse bitflags::bitflags;\nuse globset::GlobSet;\nuse hashbrown::raw::RawTable;\nuse helix_stdx::rope::{self, RopeSliceExt};\nuse slotmap::{DefaultKey as LayerId, HopSlotMap};\n\nuse std::{\n    borrow::Cow,\n    cell::RefCell,\n    collections::{HashMap, HashSet, VecDeque},\n    fmt::{self, Display},\n    hash::{Hash, Hasher},\n    mem::replace,\n    path::{Path, PathBuf},\n    str::FromStr,\n    sync::Arc,\n};\n\nuse once_cell::sync::{Lazy, OnceCell};\nuse serde::{ser::SerializeSeq, Deserialize, Serialize};\n\nuse helix_loader::grammar::{get_language, load_runtime_file};\n\npub use tree_cursor::TreeCursor;\n\nfn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<String>::deserialize(deserializer)?\n        .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_lsp_config<'de, D>(deserializer: D) -> Result<Option<serde_json::Value>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<toml::Value>::deserialize(deserializer)?\n        .map(|toml| toml.try_into().map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_tab_width<'de, D>(deserializer: D) -> Result<usize, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    usize::deserialize(deserializer).and_then(|n| {\n        if n > 0 && n <= 16 {\n            Ok(n)\n        } else {\n            Err(serde::de::Error::custom(\n                \"tab width must be a value from 1 to 16 inclusive\",\n            ))\n        }\n    })\n}\n\npub fn deserialize_auto_pairs<'de, D>(deserializer: D) -> Result<Option<AutoPairs>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Ok(Option::<AutoPairConfig>::deserialize(deserializer)?.and_then(AutoPairConfig::into))\n}\n\nfn default_timeout() -> u64 {\n    20\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct Configuration {\n    pub language: Vec<LanguageConfiguration>,\n    #[serde(default)]\n    pub language_server: HashMap<String, LanguageServerConfiguration>,\n}\n\n// largely based on tree-sitter/cli/src/loader.rs\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\", deny_unknown_fields)]\npub struct LanguageConfiguration {\n    #[serde(rename = \"name\")]\n    pub language_id: String, // c-sharp, rust, tsx\n    #[serde(rename = \"language-id\")]\n    // see the table under https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocumentItem\n    pub language_server_language_id: Option<String>, // csharp, rust, typescriptreact, for the language-server\n    pub scope: String,             // source.rust\n    pub file_types: Vec<FileType>, // filename extension or ends_with? <Gemfile, rb, etc>\n    #[serde(default)]\n    pub shebangs: Vec<String>, // interpreter(s) associated with language\n    #[serde(default)]\n    pub roots: Vec<String>, // these indicate project roots <.git, Cargo.toml>\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_comment_tokens\",\n        alias = \"comment-token\"\n    )]\n    pub comment_tokens: Option<Vec<String>>,\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_block_comment_tokens\"\n    )]\n    pub block_comment_tokens: Option<Vec<BlockCommentToken>>,\n    pub text_width: Option<usize>,\n    pub soft_wrap: Option<SoftWrap>,\n\n    #[serde(default)]\n    pub auto_format: bool,\n\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub formatter: Option<FormatterConfiguration>,\n\n    #[serde(default)]\n    pub diagnostic_severity: Severity,\n\n    pub grammar: Option<String>, // tree-sitter grammar name, defaults to language_id\n\n    // content_regex\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_regex\")]\n    pub injection_regex: Option<Regex>,\n    // first_line_regex\n    //\n    #[serde(skip)]\n    pub(crate) highlight_config: OnceCell<Option<Arc<HighlightConfiguration>>>,\n    // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583\n    #[serde(\n        default,\n        skip_serializing_if = \"Vec::is_empty\",\n        serialize_with = \"serialize_lang_features\",\n        deserialize_with = \"deserialize_lang_features\"\n    )]\n    pub language_servers: Vec<LanguageServerFeatures>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub indent: Option<IndentationConfiguration>,\n\n    #[serde(skip)]\n    pub(crate) indent_query: OnceCell<Option<Query>>,\n    #[serde(skip)]\n    pub(crate) textobject_query: OnceCell<Option<TextObjectQuery>>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub debugger: Option<DebugAdapterConfig>,\n\n    /// Automatic insertion of pairs to parentheses, brackets,\n    /// etc. Defaults to true. Optionally, this can be a list of 2-tuples\n    /// to specify a list of characters to pair. This overrides the\n    /// global setting.\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_auto_pairs\")]\n    pub auto_pairs: Option<AutoPairs>,\n\n    pub rulers: Option<Vec<u16>>, // if set, override editor's rulers\n\n    /// Hardcoded LSP root directories relative to the workspace root, like `examples` or `tools/fuzz`.\n    /// Falling back to the current working directory if none are configured.\n    pub workspace_lsp_roots: Option<Vec<PathBuf>>,\n    #[serde(default)]\n    pub persistent_diagnostic_sources: Vec<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Hash)]\npub enum FileType {\n    /// The extension of the file, either the `Path::extension` or the full\n    /// filename if the file does not have an extension.\n    Extension(String),\n    /// A Unix-style path glob. This is compared to the file's absolute path, so\n    /// it can be used to detect files based on their directories. If the glob\n    /// is not an absolute path and does not already start with a glob pattern,\n    /// a glob pattern will be prepended to it.\n    Glob(globset::Glob),\n}\n\nimpl Serialize for FileType {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: serde::Serializer,\n    {\n        use serde::ser::SerializeMap;\n\n        match self {\n            FileType::Extension(extension) => serializer.serialize_str(extension),\n            FileType::Glob(glob) => {\n                let mut map = serializer.serialize_map(Some(1))?;\n                map.serialize_entry(\"glob\", glob.glob())?;\n                map.end()\n            }\n        }\n    }\n}\n\nimpl<'de> Deserialize<'de> for FileType {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: serde::de::Deserializer<'de>,\n    {\n        struct FileTypeVisitor;\n\n        impl<'de> serde::de::Visitor<'de> for FileTypeVisitor {\n            type Value = FileType;\n\n            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {\n                formatter.write_str(\"string or table\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>\n            where\n                E: serde::de::Error,\n            {\n                Ok(FileType::Extension(value.to_string()))\n            }\n\n            fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>\n            where\n                M: serde::de::MapAccess<'de>,\n            {\n                match map.next_entry::<String, String>()? {\n                    Some((key, mut glob)) if key == \"glob\" => {\n                        // If the glob isn't an absolute path or already starts\n                        // with a glob pattern, add a leading glob so we\n                        // properly match relative paths.\n                        if !glob.starts_with('/') && !glob.starts_with(\"*/\") {\n                            glob.insert_str(0, \"*/\");\n                        }\n\n                        globset::Glob::new(glob.as_str())\n                            .map(FileType::Glob)\n                            .map_err(|err| {\n                                serde::de::Error::custom(format!(\"invalid `glob` pattern: {}\", err))\n                            })\n                    }\n                    Some((key, _value)) => Err(serde::de::Error::custom(format!(\n                        \"unknown key in `file-types` list: {}\",\n                        key\n                    ))),\n                    None => Err(serde::de::Error::custom(\n                        \"expected a `suffix` key in the `file-types` entry\",\n                    )),\n                }\n            }\n        }\n\n        deserializer.deserialize_any(FileTypeVisitor)\n    }\n}\n\nfn from_comment_tokens<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    #[derive(Deserialize)]\n    #[serde(untagged)]\n    enum CommentTokens {\n        Multiple(Vec<String>),\n        Single(String),\n    }\n    Ok(\n        Option::<CommentTokens>::deserialize(deserializer)?.map(|tokens| match tokens {\n            CommentTokens::Single(val) => vec![val],\n            CommentTokens::Multiple(vals) => vals,\n        }),\n    )\n}\n\n#[derive(Clone, Debug, Serialize, Deserialize)]\npub struct BlockCommentToken {\n    pub start: String,\n    pub end: String,\n}\n\nimpl Default for BlockCommentToken {\n    fn default() -> Self {\n        BlockCommentToken {\n            start: \"/*\".to_string(),\n            end: \"*/\".to_string(),\n        }\n    }\n}\n\nfn from_block_comment_tokens<'de, D>(\n    deserializer: D,\n) -> Result<Option<Vec<BlockCommentToken>>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    #[derive(Deserialize)]\n    #[serde(untagged)]\n    enum BlockCommentTokens {\n        Multiple(Vec<BlockCommentToken>),\n        Single(BlockCommentToken),\n    }\n    Ok(\n        Option::<BlockCommentTokens>::deserialize(deserializer)?.map(|tokens| match tokens {\n            BlockCommentTokens::Single(val) => vec![val],\n            BlockCommentTokens::Multiple(vals) => vals,\n        }),\n    )\n}\n\n#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]\n#[serde(rename_all = \"kebab-case\")]\npub enum LanguageServerFeature {\n    Format,\n    GotoDeclaration,\n    GotoDefinition,\n    GotoTypeDefinition,\n    GotoReference,\n    GotoImplementation,\n    // Goto, use bitflags, combining previous Goto members?\n    SignatureHelp,\n    Hover,\n    DocumentHighlight,\n    Completion,\n    CodeAction,\n    WorkspaceCommand,\n    DocumentSymbols,\n    WorkspaceSymbols,\n    // Symbols, use bitflags, see above?\n    Diagnostics,\n    RenameSymbol,\n    InlayHints,\n}\n\nimpl Display for LanguageServerFeature {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        use LanguageServerFeature::*;\n        let feature = match self {\n            Format => \"format\",\n            GotoDeclaration => \"goto-declaration\",\n            GotoDefinition => \"goto-definition\",\n            GotoTypeDefinition => \"goto-type-definition\",\n            GotoReference => \"goto-reference\",\n            GotoImplementation => \"goto-implementation\",\n            SignatureHelp => \"signature-help\",\n            Hover => \"hover\",\n            DocumentHighlight => \"document-highlight\",\n            Completion => \"completion\",\n            CodeAction => \"code-action\",\n            WorkspaceCommand => \"workspace-command\",\n            DocumentSymbols => \"document-symbols\",\n            WorkspaceSymbols => \"workspace-symbols\",\n            Diagnostics => \"diagnostics\",\n            RenameSymbol => \"rename-symbol\",\n            InlayHints => \"inlay-hints\",\n        };\n        write!(f, \"{feature}\",)\n    }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(untagged, rename_all = \"kebab-case\", deny_unknown_fields)]\nenum LanguageServerFeatureConfiguration {\n    #[serde(rename_all = \"kebab-case\")]\n    Features {\n        #[serde(default, skip_serializing_if = \"HashSet::is_empty\")]\n        only_features: HashSet<LanguageServerFeature>,\n        #[serde(default, skip_serializing_if = \"HashSet::is_empty\")]\n        except_features: HashSet<LanguageServerFeature>,\n        name: String,\n    },\n    Simple(String),\n}\n\n#[derive(Debug, Default)]\npub struct LanguageServerFeatures {\n    pub name: String,\n    pub only: HashSet<LanguageServerFeature>,\n    pub excluded: HashSet<LanguageServerFeature>,\n}\n\nimpl LanguageServerFeatures {\n    pub fn has_feature(&self, feature: LanguageServerFeature) -> bool {\n        (self.only.is_empty() || self.only.contains(&feature)) && !self.excluded.contains(&feature)\n    }\n}\n\nfn deserialize_lang_features<'de, D>(\n    deserializer: D,\n) -> Result<Vec<LanguageServerFeatures>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    let raw: Vec<LanguageServerFeatureConfiguration> = Deserialize::deserialize(deserializer)?;\n    let res = raw\n        .into_iter()\n        .map(|config| match config {\n            LanguageServerFeatureConfiguration::Simple(name) => LanguageServerFeatures {\n                name,\n                ..Default::default()\n            },\n            LanguageServerFeatureConfiguration::Features {\n                only_features,\n                except_features,\n                name,\n            } => LanguageServerFeatures {\n                name,\n                only: only_features,\n                excluded: except_features,\n            },\n        })\n        .collect();\n    Ok(res)\n}\nfn serialize_lang_features<S>(\n    map: &Vec<LanguageServerFeatures>,\n    serializer: S,\n) -> Result<S::Ok, S::Error>\nwhere\n    S: serde::Serializer,\n{\n    let mut serializer = serializer.serialize_seq(Some(map.len()))?;\n    for features in map {\n        let features = if features.only.is_empty() && features.excluded.is_empty() {\n            LanguageServerFeatureConfiguration::Simple(features.name.to_owned())\n        } else {\n            LanguageServerFeatureConfiguration::Features {\n                only_features: features.only.clone(),\n                except_features: features.excluded.clone(),\n                name: features.name.to_owned(),\n            }\n        };\n        serializer.serialize_element(&features)?;\n    }\n    serializer.end()\n}\n\nfn deserialize_required_root_patterns<'de, D>(deserializer: D) -> Result<Option<GlobSet>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    let patterns = Vec::<String>::deserialize(deserializer)?;\n    if patterns.is_empty() {\n        return Ok(None);\n    }\n    let mut builder = globset::GlobSetBuilder::new();\n    for pattern in patterns {\n        let glob = globset::Glob::new(&pattern).map_err(serde::de::Error::custom)?;\n        builder.add(glob);\n    }\n    builder.build().map(Some).map_err(serde::de::Error::custom)\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct LanguageServerConfiguration {\n    pub command: String,\n    #[serde(default)]\n    #[serde(skip_serializing_if = \"Vec::is_empty\")]\n    pub args: Vec<String>,\n    #[serde(default, skip_serializing_if = \"HashMap::is_empty\")]\n    pub environment: HashMap<String, String>,\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_lsp_config\")]\n    pub config: Option<serde_json::Value>,\n    #[serde(default = \"default_timeout\")]\n    pub timeout: u64,\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"deserialize_required_root_patterns\"\n    )]\n    pub required_root_patterns: Option<GlobSet>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct FormatterConfiguration {\n    pub command: String,\n    #[serde(default)]\n    #[serde(skip_serializing_if = \"Vec::is_empty\")]\n    pub args: Vec<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct AdvancedCompletion {\n    pub name: Option<String>,\n    pub completion: Option<String>,\n    pub default: Option<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\", untagged)]\npub enum DebugConfigCompletion {\n    Named(String),\n    Advanced(AdvancedCompletion),\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(untagged)]\npub enum DebugArgumentValue {\n    String(String),\n    Array(Vec<String>),\n    Boolean(bool),\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct DebugTemplate {\n    pub name: String,\n    pub request: String,\n    pub completion: Vec<DebugConfigCompletion>,\n    pub args: HashMap<String, DebugArgumentValue>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct DebugAdapterConfig {\n    pub name: String,\n    pub transport: String,\n    #[serde(default)]\n    pub command: String,\n    #[serde(default)]\n    pub args: Vec<String>,\n    pub port_arg: Option<String>,\n    pub templates: Vec<DebugTemplate>,\n    #[serde(default)]\n    pub quirks: DebuggerQuirks,\n}\n\n// Different workarounds for adapters' differences\n#[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize)]\npub struct DebuggerQuirks {\n    #[serde(default)]\n    pub absolute_paths: bool,\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct IndentationConfiguration {\n    #[serde(deserialize_with = \"deserialize_tab_width\")]\n    pub tab_width: usize,\n    pub unit: String,\n}\n\n/// How the indentation for a newly inserted line should be determined.\n/// If the selected heuristic is not available (e.g. because the current\n/// language has no tree-sitter indent queries), a simpler one will be used.\n#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub enum IndentationHeuristic {\n    /// Just copy the indentation of the line that the cursor is currently on.\n    Simple,\n    /// Use tree-sitter indent queries to compute the expected absolute indentation level of the new line.\n    TreeSitter,\n    /// Use tree-sitter indent queries to compute the expected difference in indentation between the new line\n    /// and the line before. Add this to the actual indentation level of the line before.\n    #[default]\n    Hybrid,\n}\n\n/// Configuration for auto pairs\n#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\", deny_unknown_fields, untagged)]\npub enum AutoPairConfig {\n    /// Enables or disables auto pairing. False means disabled. True means to use the default pairs.\n    Enable(bool),\n\n    /// The mappings of pairs.\n    Pairs(HashMap<char, char>),\n}\n\nimpl Default for AutoPairConfig {\n    fn default() -> Self {\n        AutoPairConfig::Enable(true)\n    }\n}\n\nimpl From<&AutoPairConfig> for Option<AutoPairs> {\n    fn from(auto_pair_config: &AutoPairConfig) -> Self {\n        match auto_pair_config {\n            AutoPairConfig::Enable(false) => None,\n            AutoPairConfig::Enable(true) => Some(AutoPairs::default()),\n            AutoPairConfig::Pairs(pairs) => Some(AutoPairs::new(pairs.iter())),\n        }\n    }\n}\n\nimpl From<AutoPairConfig> for Option<AutoPairs> {\n    fn from(auto_pairs_config: AutoPairConfig) -> Self {\n        (&auto_pairs_config).into()\n    }\n}\n\nimpl FromStr for AutoPairConfig {\n    type Err = std::str::ParseBoolError;\n\n    // only do bool parsing for runtime setting\n    fn from_str(s: &str) -> Result<Self, Self::Err> {\n        let enable: bool = s.parse()?;\n        Ok(AutoPairConfig::Enable(enable))\n    }\n}\n\n#[derive(Debug)]\npub struct TextObjectQuery {\n    pub query: Query,\n}\n\n#[derive(Debug)]\npub enum CapturedNode<'a> {\n    Single(Node<'a>),\n    /// Guaranteed to be not empty\n    Grouped(Vec<Node<'a>>),\n}\n\nimpl<'a> CapturedNode<'a> {\n    pub fn start_byte(&self) -> usize {\n        match self {\n            Self::Single(n) => n.start_byte(),\n            Self::Grouped(ns) => ns[0].start_byte(),\n        }\n    }\n\n    pub fn end_byte(&self) -> usize {\n        match self {\n            Self::Single(n) => n.end_byte(),\n            Self::Grouped(ns) => ns.last().unwrap().end_byte(),\n        }\n    }\n\n    pub fn byte_range(&self) -> std::ops::Range<usize> {\n        self.start_byte()..self.end_byte()\n    }\n}\n\n/// The maximum number of in-progress matches a TS cursor can consider at once.\n/// This is set to a constant in order to avoid performance problems for medium to large files. Set with `set_match_limit`.\n/// Using such a limit means that we lose valid captures, so there is fundamentally a tradeoff here.\n///\n///\n/// Old tree sitter versions used a limit of 32 by default until this limit was removed in version `0.19.5` (must now be set manually).\n/// However, this causes performance issues for medium to large files.\n/// In helix, this problem caused treesitter motions to take multiple seconds to complete in medium-sized rust files (3k loc).\n///\n///\n/// Neovim also encountered this problem and reintroduced this limit after it was removed upstream\n/// (see <https://github.com/neovim/neovim/issues/14897> and <https://github.com/neovim/neovim/pull/14915>).\n/// The number used here is fundamentally a tradeoff between breaking some obscure edge cases and performance.\n///\n///\n/// Neovim chose 64 for this value somewhat arbitrarily (<https://github.com/neovim/neovim/pull/18397>).\n/// 64 is too low for some languages though. In particular, it breaks some highlighting for record fields in Erlang record definitions.\n/// This number can be increased if new syntax highlight breakages are found, as long as the performance penalty is not too high.\nconst TREE_SITTER_MATCH_LIMIT: u32 = 256;\n\nimpl TextObjectQuery {\n    /// Run the query on the given node and return sub nodes which match given\n    /// capture (\"function.inside\", \"class.around\", etc).\n    ///\n    /// Captures may contain multiple nodes by using quantifiers (+, *, etc),\n    /// and support for this is partial and could use improvement.\n    ///\n    /// ```query\n    /// (comment)+ @capture\n    ///\n    /// ; OR\n    /// (\n    ///   (comment)*\n    ///   .\n    ///   (function)\n    /// ) @capture\n    /// ```\n    pub fn capture_nodes<'a>(\n        &'a self,\n        capture_name: &str,\n        node: Node<'a>,\n        slice: RopeSlice<'a>,\n        cursor: &'a mut QueryCursor,\n    ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {\n        self.capture_nodes_any(&[capture_name], node, slice, cursor)\n    }\n\n    /// Find the first capture that exists out of all given `capture_names`\n    /// and return sub nodes that match this capture.\n    pub fn capture_nodes_any<'a>(\n        &'a self,\n        capture_names: &[&str],\n        node: Node<'a>,\n        slice: RopeSlice<'a>,\n        cursor: &'a mut QueryCursor,\n    ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {\n        let capture_idx = capture_names\n            .iter()\n            .find_map(|cap| self.query.capture_index_for_name(cap))?;\n\n        cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT);\n\n        let nodes = cursor\n            .captures(&self.query, node, RopeProvider(slice))\n            .filter_map(move |(mat, _)| {\n                let nodes: Vec<_> = mat\n                    .captures\n                    .iter()\n                    .filter_map(|cap| (cap.index == capture_idx).then_some(cap.node))\n                    .collect();\n\n                if nodes.len() > 1 {\n                    Some(CapturedNode::Grouped(nodes))\n                } else {\n                    nodes.into_iter().map(CapturedNode::Single).next()\n                }\n            });\n\n        Some(nodes)\n    }\n}\n\npub fn read_query(language: &str, filename: &str) -> String {\n    static INHERITS_REGEX: Lazy<Regex> =\n        Lazy::new(|| Regex::new(r\";+\\s*inherits\\s*:?\\s*([a-z_,()-]+)\\s*\").unwrap());\n\n    let query = load_runtime_file(language, filename).unwrap_or_default();\n\n    // replaces all \"; inherits <language>(,<language>)*\" with the queries of the given language(s)\n    INHERITS_REGEX\n        .replace_all(&query, |captures: &regex::Captures| {\n            captures[1]\n                .split(',')\n                .map(|language| format!(\"\\n{}\\n\", read_query(language, filename)))\n                .collect::<String>()\n        })\n        .to_string()\n}\n\nimpl LanguageConfiguration {\n    fn initialize_highlight(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {\n        let highlights_query = read_query(&self.language_id, \"highlights.scm\");\n        // always highlight syntax errors\n        // highlights_query += \"\\n(ERROR) @error\";\n\n        let injections_query = read_query(&self.language_id, \"injections.scm\");\n        let locals_query = read_query(&self.language_id, \"locals.scm\");\n\n        if highlights_query.is_empty() {\n            None\n        } else {\n            let language = get_language(self.grammar.as_deref().unwrap_or(&self.language_id))\n                .map_err(|err| {\n                    log::error!(\n                        \"Failed to load tree-sitter parser for language {:?}: {}\",\n                        self.language_id,\n                        err\n                    )\n                })\n                .ok()?;\n            let config = HighlightConfiguration::new(\n                language,\n                &highlights_query,\n                &injections_query,\n                &locals_query,\n            )\n            .map_err(|err| log::error!(\"Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --grammar fetch' and 'hx --grammar build'. This query could not be parsed: {:?}\", self.language_id, err))\n            .ok()?;\n\n            config.configure(scopes);\n            Some(Arc::new(config))\n        }\n    }\n\n    pub fn reconfigure(&self, scopes: &[String]) {\n        if let Some(Some(config)) = self.highlight_config.get() {\n            config.configure(scopes);\n        }\n    }\n\n    pub fn highlight_config(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {\n        self.highlight_config\n            .get_or_init(|| self.initialize_highlight(scopes))\n            .clone()\n    }\n\n    pub fn is_highlight_initialized(&self) -> bool {\n        self.highlight_config.get().is_some()\n    }\n\n    pub fn indent_query(&self) -> Option<&Query> {\n        self.indent_query\n            .get_or_init(|| self.load_query(\"indents.scm\"))\n            .as_ref()\n    }\n\n    pub fn textobject_query(&self) -> Option<&TextObjectQuery> {\n        self.textobject_query\n            .get_or_init(|| {\n                self.load_query(\"textobjects.scm\")\n                    .map(|query| TextObjectQuery { query })\n            })\n            .as_ref()\n    }\n\n    pub fn scope(&self) -> &str {\n        &self.scope\n    }\n\n    fn load_query(&self, kind: &str) -> Option<Query> {\n        let query_text = read_query(&self.language_id, kind);\n        if query_text.is_empty() {\n            return None;\n        }\n        let lang = &self.highlight_config.get()?.as_ref()?.language;\n        Query::new(lang, &query_text)\n            .map_err(|e| {\n                log::error!(\n                    \"Failed to parse {} queries for {}: {}\",\n                    kind,\n                    self.language_id,\n                    e\n                )\n            })\n            .ok()\n    }\n}\n#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(default, rename_all = \"kebab-case\", deny_unknown_fields)]\npub struct SoftWrap {\n    /// Soft wrap lines that exceed viewport width. Default to off\n    // NOTE: Option on purpose because the struct is shared between language config and global config.\n    // By default the option is None so that the language config falls back to the global config unless explicitly set.\n    pub enable: Option<bool>,\n    /// Maximum space left free at the end of the line.\n    /// This space is used to wrap text at word boundaries. If that is not possible within this limit\n    /// the word is simply split at the end of the line.\n    ///\n    /// This is automatically hard-limited to a quarter of the viewport to ensure correct display on small views.\n    ///\n    /// Default to 20\n    pub max_wrap: Option<u16>,\n    /// Maximum number of indentation that can be carried over from the previous line when softwrapping.\n    /// If a line is indented further then this limit it is rendered at the start of the viewport instead.\n    ///\n    /// This is automatically hard-limited to a quarter of the viewport to ensure correct display on small views.\n    ///\n    /// Default to 40\n    pub max_indent_retain: Option<u16>,\n    /// Indicator placed at the beginning of softwrapped lines\n    ///\n    /// Defaults to \u21aa\n    pub wrap_indicator: Option<String>,\n    /// Softwrap at `text_width` instead of viewport width if it is shorter\n    pub wrap_at_text_width: Option<bool>,\n}\n\n#[derive(Debug)]\nstruct FileTypeGlob {\n    glob: globset::Glob,\n    language_id: usize,\n}\n\nimpl FileTypeGlob {\n    fn new(glob: globset::Glob, language_id: usize) -> Self {\n        Self { glob, language_id }\n    }\n}\n\n#[derive(Debug)]\nstruct FileTypeGlobMatcher {\n    matcher: globset::GlobSet,\n    file_types: Vec<FileTypeGlob>,\n}\n\nimpl FileTypeGlobMatcher {\n    fn new(file_types: Vec<FileTypeGlob>) -> Result<Self, globset::Error> {\n        let mut builder = globset::GlobSetBuilder::new();\n        for file_type in &file_types {\n            builder.add(file_type.glob.clone());\n        }\n\n        Ok(Self {\n            matcher: builder.build()?,\n            file_types,\n        })\n    }\n\n    fn language_id_for_path(&self, path: &Path) -> Option<&usize> {\n        self.matcher\n            .matches(path)\n            .iter()\n            .filter_map(|idx| self.file_types.get(*idx))\n            .max_by_key(|file_type| file_type.glob.glob().len())\n            .map(|file_type| &file_type.language_id)\n    }\n}\n\n// Expose loader as Lazy<> global since it's always static?\n\n#[derive(Debug)]\npub struct Loader {\n    // highlight_names ?\n    language_configs: Vec<Arc<LanguageConfiguration>>,\n...\n", "needle_token_start": 696, "needle_token_end": 920, "code_context_ntokens": 16224, "output": ["```rust\npub fn compose(self, other: ChangeSet) -> ChangeSet {\n    let mut changes = self.changes.clone();\n    let mut other_changes = other.changes;\n\n    for change in other_changes.drain(..) {\n        changes.push(change);\n    }\n\n    ChangeSet {\n        changes,\n        len: self.len + other.len,\n        len_after: self.len_after + other.len_after,\n    }\n}\n```"]}
{"repo": "helix-editor/helix", "name": "tokenize", "language": "rust", "path": "helix-core/src/diff.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a sequence of text into a series of line-based segments, facilitating operations that require line-by-line processing.\n2. **Input**: The function accepts a slice of text.\n3. **Output**: It produces an iterator that yields each line of the input text as a separate segment.\n4. **Procedure**: The function accesses the lines method of the text slice, which internally iterates over the text and separates it into lines based on newline characters, returning each line as a segment for further processing.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: helix-core/src/indent.rs\nuse std::{borrow::Cow, collections::HashMap};\n\nuse helix_stdx::rope::RopeSliceExt;\nuse tree_sitter::{Query, QueryCursor, QueryPredicateArg};\n\nuse crate::{\n    chars::{char_is_line_ending, char_is_whitespace},\n    graphemes::{grapheme_width, tab_width_at},\n    syntax::{IndentationHeuristic, LanguageConfiguration, RopeProvider, Syntax},\n    tree_sitter::Node,\n    Position, Rope, RopeGraphemes, RopeSlice,\n};\n\n/// Enum representing indentation style.\n///\n/// Only values 1-8 are valid for the `Spaces` variant.\n#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]\npub enum IndentStyle {\n    Tabs,\n    Spaces(u8),\n}\n\n// 16 spaces\nconst INDENTS: &str = \"                \";\npub const MAX_INDENT: u8 = 16;\n\nimpl IndentStyle {\n    /// Creates an `IndentStyle` from an indentation string.\n    ///\n    /// For example, passing `\"    \"` (four spaces) will create `IndentStyle::Spaces(4)`.\n    #[allow(clippy::should_implement_trait)]\n    #[inline]\n    pub fn from_str(indent: &str) -> Self {\n        // XXX: do we care about validating the input more than this?  Probably not...?\n        debug_assert!(!indent.is_empty() && indent.len() <= MAX_INDENT as usize);\n\n        if indent.starts_with(' ') {\n            IndentStyle::Spaces(indent.len().clamp(1, MAX_INDENT as usize) as u8)\n        } else {\n            IndentStyle::Tabs\n        }\n    }\n\n    #[inline]\n    pub fn as_str(&self) -> &'static str {\n        match *self {\n            IndentStyle::Tabs => \"\\t\",\n            IndentStyle::Spaces(n) => {\n                // Unsupported indentation style.  This should never happen,\n                debug_assert!(n > 0 && n <= MAX_INDENT);\n\n                // Either way, clamp to the nearest supported value\n                let closest_n = n.clamp(1, MAX_INDENT) as usize;\n                &INDENTS[0..closest_n]\n            }\n        }\n    }\n\n    #[inline]\n    pub fn indent_width(&self, tab_width: usize) -> usize {\n        match *self {\n            IndentStyle::Tabs => tab_width,\n            IndentStyle::Spaces(width) => width as usize,\n        }\n    }\n}\n\n/// Attempts to detect the indentation style used in a document.\n///\n/// Returns the indentation style if the auto-detect confidence is\n/// reasonably high, otherwise returns `None`.\npub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {\n    // Build a histogram of the indentation *increases* between\n...\n// Path: helix-core/src/diff.rs\nuse std::ops::Range;\nuse std::time::Instant;\n\nuse imara_diff::intern::InternedInput;\nuse imara_diff::Algorithm;\nuse ropey::RopeSlice;\n\nuse crate::{ChangeSet, Rope, Tendril, Transaction};\n\n/// A `imara_diff::Sink` that builds a `ChangeSet` for a character diff of a hunk\nstruct CharChangeSetBuilder<'a> {\n    res: &'a mut ChangeSet,\n    hunk: &'a InternedInput<char>,\n    pos: u32,\n}\n\nimpl imara_diff::Sink for CharChangeSetBuilder<'_> {\n    type Out = ();\n    fn process_change(&mut self, before: Range<u32>, after: Range<u32>) {\n        self.res.retain((before.start - self.pos) as usize);\n        self.res.delete(before.len());\n        self.pos = before.end;\n\n        let res = self.hunk.after[after.start as usize..after.end as usize]\n            .iter()\n            .map(|&token| self.hunk.interner[token])\n            .collect();\n\n        self.res.insert(res);\n    }\n\n    fn finish(self) -> Self::Out {\n        self.res.retain(self.hunk.before.len() - self.pos as usize);\n    }\n}\n\nstruct LineChangeSetBuilder<'a> {\n    res: ChangeSet,\n    after: RopeSlice<'a>,\n    file: &'a InternedInput<RopeSlice<'a>>,\n    current_hunk: InternedInput<char>,\n    pos: u32,\n}\n\nimpl imara_diff::Sink for LineChangeSetBuilder<'_> {\n    type Out = ChangeSet;\n\n    fn process_change(&mut self, before: Range<u32>, after: Range<u32>) {\n        let len = self.file.before[self.pos as usize..before.start as usize]\n            .iter()\n            .map(|&it| self.file.interner[it].len_chars())\n            .sum();\n        self.res.retain(len);\n        self.pos = before.end;\n\n        // do not perform diffs on large hunks\n        let len_before = before.end - before.start;\n        let len_after = after.end - after.start;\n\n        // Pure insertions/removals do not require a character diff.\n        // Very large changes are ignored because their character diff is expensive to compute\n        // TODO adjust heuristic to detect large changes?\n        if len_before == 0\n            || len_after == 0\n            || len_after > 5 * len_before\n            || 5 * len_after < len_before && len_before > 10\n            || len_before + len_after > 200\n        {\n            let remove = self.file.before[before.start as usize..before.end as usize]\n                .iter()\n                .map(|&it| self.file.interner[it].len_chars())\n                .sum();\n            self.res.delete(remove);\n            let mut fragment = Tendril::new();\n            if len_after > 500 {\n                // copying a rope line by line is slower then copying the entire\n                // rope. Use to_string for very large changes instead..\n                if self.file.after.len() == after.end as usize {\n                    if after.start == 0 {\n                        fragment = self.after.to_string().into();\n                    } else {\n                        let start = self.after.line_to_char(after.start as usize);\n                        fragment = self.after.slice(start..).to_string().into();\n                    }\n                } else if after.start == 0 {\n                    let end = self.after.line_to_char(after.end as usize);\n                    fragment = self.after.slice(..end).to_string().into();\n                } else {\n                    let start = self.after.line_to_char(after.start as usize);\n                    let end = self.after.line_to_char(after.end as usize);\n                    fragment = self.after.slice(start..end).to_string().into();\n                }\n            } else {\n                for &line in &self.file.after[after.start as usize..after.end as usize] {\n                    for chunk in self.file.interner[line].chunks() {\n                        fragment.push_str(chunk)\n                    }\n                }\n            };\n            self.res.insert(fragment);\n        } else {\n            // for reasonably small hunks, generating a ChangeSet from char diff can save memory\n            // TODO use a tokenizer (word diff?) for improved performance\n            let hunk_before = self.file.before[before.start as usize..before.end as usize]\n                .iter()\n                .flat_map(|&it| self.file.interner[it].chars());\n            let hunk_after = self.file.after[after.start as usize..after.end as usize]\n                .iter()\n                .flat_map(|&it| self.file.interner[it].chars());\n            self.current_hunk.update_before(hunk_before);\n            self.current_hunk.update_after(hunk_after);\n\n            // the histogram heuristic does not work as well\n            // for characters because the same characters often reoccur\n            // use myer diff instead\n            imara_diff::diff(\n                Algorithm::Myers,\n                &self.current_hunk,\n                CharChangeSetBuilder {\n                    res: &mut self.res,\n                    hunk: &self.current_hunk,\n                    pos: 0,\n                },\n            );\n\n            self.current_hunk.clear();\n        }\n    }\n\n    fn finish(mut self) -> Self::Out {\n        let len = self.file.before[self.pos as usize..]\n            .iter()\n            .map(|&it| self.file.interner[it].len_chars())\n            .sum();\n\n        self.res.retain(len);\n        self.res\n    }\n}\n\nstruct RopeLines<'a>(RopeSlice<'a>);\n\nimpl<'a> imara_diff::intern::TokenSource for RopeLines<'a> {\n    type Token = RopeSlice<'a>;\n    type Tokenizer = ropey::iter::Lines<'a>;\n\n    \nfn tokenize(&self) -> Self::Tokenizer {\n        self.0.lines()\n    }\n\n    fn estimate_tokens(&self) -> u32 {\n        // we can provide a perfect estimate which is very nice for performance\n        self.0.len_lines() as u32\n    }\n}\n\n/// Compares `old` and `new` to generate a [`Transaction`] describing\n/// the steps required to get from `old` to `new`.\npub fn compare_ropes(before: &Rope, after: &Rope) -> Transaction {\n    let start = Instant::now();\n    let res = ChangeSet::with_capacity(32);\n    let after = after.slice(..);\n    let file = InternedInput::new(RopeLines(before.slice(..)), RopeLines(after));\n    let builder = LineChangeSetBuilder {\n        res,\n        file: &file,\n        after,\n        pos: 0,\n        current_hunk: InternedInput::default(),\n    };\n\n    let res = imara_diff::diff(Algorithm::Histogram, &file, builder).into();\n\n    log::debug!(\n        \"rope diff took {}s\",\n        Instant::now().duration_since(start).as_secs_f64()\n    );\n    res\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    fn test_identity(a: &str, b: &str) {\n        let mut old = Rope::from(a);\n        let new = Rope::from(b);\n        compare_ropes(&old, &new).apply(&mut old);\n        assert_eq!(old, new);\n    }\n\n    quickcheck::quickcheck! {\n        fn test_compare_ropes(a: String, b: String) -> bool {\n            let mut old = Rope::from(a);\n            let new = Rope::from(b);\n            compare_ropes(&old, &new).apply(&mut old);\n            old == new\n        }\n    }\n\n    #[test]\n    fn equal_files() {\n        test_identity(\"foo\", \"foo\");\n    }\n\n    #[test]\n    fn trailing_newline() {\n        test_identity(\"foo\\n\", \"foo\");\n        test_identity(\"foo\", \"foo\\n\");\n    }\n\n    #[test]\n    fn new_file() {\n        test_identity(\"\", \"foo\");\n    }\n\n    #[test]\n    fn deleted_file() {\n        test_identity(\"foo\", \"\");\n    }\n}\n\n// Path: helix-core/src/shellwords.rs\nuse std::borrow::Cow;\n\n/// Auto escape for shellwords usage.\npub fn escape(input: Cow<str>) -> Cow<str> {\n    if !input.chars().any(|x| x.is_ascii_whitespace()) {\n        input\n    } else if cfg!(unix) {\n        Cow::Owned(input.chars().fold(String::new(), |mut buf, c| {\n            if c.is_ascii_whitespace() {\n                buf.push('\\\\');\n            }\n            buf.push(c);\n            buf\n        }))\n    } else {\n        Cow::Owned(format!(\"\\\"{}\\\"\", input))\n    }\n}\n\nenum State {\n    OnWhitespace,\n    Unquoted,\n    UnquotedEscaped,\n    Quoted,\n    QuoteEscaped,\n    Dquoted,\n    DquoteEscaped,\n}\n\npub struct Shellwords<'a> {\n    state: State,\n    /// Shellwords where whitespace and escapes has been resolved.\n    words: Vec<Cow<'a, str>>,\n    /// The parts of the input that are divided into shellwords. This can be\n    /// used to retrieve the original text for a given word by looking up the\n    /// same index in the Vec as the word in `words`.\n    parts: Vec<&'a str>,\n}\n\nimpl<'a> From<&'a str> for Shellwords<'a> {\n    fn from(input: &'a str) -> Self {\n        use State::*;\n\n        let mut state = Unquoted;\n        let mut words = Vec::new();\n        let mut parts = Vec::new();\n        let mut escaped = String::with_capacity(input.len());\n\n        let mut part_start = 0;\n        let mut unescaped_start = 0;\n        let mut end = 0;\n\n        for (i, c) in input.char_indices() {\n            state = match state {\n                OnWhitespace => match c {\n                    '\"' => {\n                        end = i;\n                        Dquoted\n                    }\n                    '\\'' => {\n                        end = i;\n                        Quoted\n                    }\n                    '\\\\' => {\n                        if cfg!(unix) {\n                            escaped.push_str(&input[unescaped_start..i]);\n                            unescaped_start = i + 1;\n                            UnquotedEscaped\n                        } else {\n                            OnWhitespace\n                        }\n                    }\n                    c if c.is_ascii_whitespace() => {\n                        end = i;\n                        OnWhitespace\n                    }\n                    _ => Unquoted,\n                },\n                Unquoted => match c {\n                    '\\\\' => {\n                        if cfg!(unix) {\n                            escaped.push_str(&input[unescaped_start..i]);\n                            unescaped_start = i + 1;\n                            UnquotedEscaped\n                        } else {\n                            Unquoted\n                        }\n                    }\n                    c if c.is_ascii_whitespace() => {\n                        end = i;\n                        OnWhitespace\n                    }\n                    _ => Unquoted,\n                },\n                UnquotedEscaped => Unquoted,\n                Quoted => match c {\n                    '\\\\' => {\n                        if cfg!(unix) {\n                            escaped.push_str(&input[unescaped_start..i]);\n                            unescaped_start = i + 1;\n                            QuoteEscaped\n                        } else {\n                            Quoted\n                        }\n                    }\n                    '\\'' => {\n                        end = i;\n                        OnWhitespace\n                    }\n                    _ => Quoted,\n                },\n                QuoteEscaped => Quoted,\n                Dquoted => match c {\n                    '\\\\' => {\n                        if cfg!(unix) {\n                            escaped.push_str(&input[unescaped_start..i]);\n                            unescaped_start = i + 1;\n                            DquoteEscaped\n                        } else {\n                            Dquoted\n                        }\n                    }\n                    '\"' => {\n                        end = i;\n                        OnWhitespace\n                    }\n                    _ => Dquoted,\n                },\n                DquoteEscaped => Dquoted,\n            };\n\n            let c_len = c.len_utf8();\n            if i == input.len() - c_len && end == 0 {\n                end = i + c_len;\n            }\n\n            if end > 0 {\n                let esc_trim = escaped.trim();\n                let inp = &input[unescaped_start..end];\n\n                if !(esc_trim.is_empty() && inp.trim().is_empty()) {\n                    if esc_trim.is_empty() {\n                        words.push(inp.into());\n                        parts.push(inp);\n                    } else {\n                        words.push([escaped, inp.into()].concat().into());\n                        parts.push(&input[part_start..end]);\n                        escaped = \"\".to_string();\n                    }\n                }\n                unescaped_start = i + 1;\n                part_start = i + 1;\n                end = 0;\n            }\n        }\n\n        debug_assert!(words.len() == parts.len());\n\n        Self {\n            state,\n            words,\n            parts,\n        }\n    }\n}\n\nimpl<'a> Shellwords<'a> {\n    /// Checks that the input ends with a whitespace character which is not escaped.\n    ///\n    /// # Examples\n    ///\n    /// ```rust\n    /// use helix_core::shellwords::Shellwords;\n    /// assert_eq!(Shellwords::from(\" \").ends_with_whitespace(), true);\n    /// assert_eq!(Shellwords::from(\":open \").ends_with_whitespace(), true);\n    /// assert_eq!(Shellwords::from(\":open foo.txt \").ends_with_whitespace(), true);\n    /// assert_eq!(Shellwords::from(\":open\").ends_with_whitespace(), false);\n    /// #[cfg(unix)]\n    /// assert_eq!(Shellwords::from(\":open a\\\\ \").ends_with_whitespace(), false);\n    /// #[cfg(unix)]\n    /// assert_eq!(Shellwords::from(\":open a\\\\ b.txt\").ends_with_whitespace(), false);\n    /// ```\n    pub fn ends_with_whitespace(&self) -> bool {\n        matches!(self.state, State::OnWhitespace)\n    }\n\n    /// Returns the list of shellwords calculated from the input string.\n    pub fn words(&self) -> &[Cow<'a, str>] {\n        &self.words\n    }\n\n    /// Returns a list of strings which correspond to [`Self::words`] but represent the original\n    /// text in the input string - including escape characters - without separating whitespace.\n    pub fn parts(&self) -> &[&'a str] {\n        &self.parts\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    #[cfg(windows)]\n    fn test_normal() {\n        let input = r#\":o single_word tw\u00f3 w\u00f6rds \\three\\ \\\"with\\ escaping\\\\\"#;\n        let shellwords = Shellwords::from(input);\n        let result = shellwords.words().to_vec();\n        let expected = vec![\n            Cow::from(\":o\"),\n            Cow::from(\"single_word\"),\n            Cow::from(\"tw\u00f3\"),\n            Cow::from(\"w\u00f6rds\"),\n            Cow::from(\"\\\\three\\\\\"),\n            Cow::from(\"\\\\\"),\n            Cow::from(\"with\\\\ escaping\\\\\\\\\"),\n        ];\n        // TODO test is_owned and is_borrowed, once they get stabilized.\n        assert_eq!(expected, result);\n    }\n\n    #[test]\n    #[cfg(unix)]\n    fn test_normal() {\n        let input = r#\":o single_word tw\u00f3 w\u00f6rds \\three\\ \\\"with\\ escaping\\\\\"#;\n        let shellwords = Shellwords::from(input);\n        let result = shellwords.words().to_vec();\n        let expected = vec![\n            Cow::from(\":o\"),\n            Cow::from(\"single_word\"),\n            Cow::from(\"tw\u00f3\"),\n            Cow::from(\"w\u00f6rds\"),\n            Cow::from(r#\"three \"with escaping\\\"#),\n        ];\n        // TODO test is_owned and is_borrowed, once they get stabilized.\n        assert_eq!(expected, result);\n    }\n\n    #[test]\n    #[cfg(unix)]\n    fn test_quoted() {\n        let quoted =\n            r#\":o 'single_word' 'tw\u00f3 w\u00f6rds' '' ' ''\\three\\' \\\"with\\ escaping\\\\' 'quote incomplete\"#;\n        let shellwords = Shellwords::from(quoted);\n        let result = shellwords.words().to_vec();\n        let expected = vec![\n            Cow::from(\":o\"),\n            Cow::from(\"single_word\"),\n            Cow::from(\"tw\u00f3 w\u00f6rds\"),\n            Cow::from(r#\"three' \"with escaping\\\"#),\n            Cow::from(\"quote incomplete\"),\n        ];\n        assert_eq!(expected, result);\n    }\n\n    #[test]\n    #[cfg(unix)]\n    fn test_dquoted() {\n        let dquoted = r#\":o \"single_word\" \"tw\u00f3 w\u00f6rds\" \"\" \"  \"\"\\three\\' \\\"with\\ escaping\\\\\" \"dquote incomplete\"#;\n        let shellwords = Shellwords::from(dquoted);\n        let result = shellwords.words().to_vec();\n        let expected = vec![\n            Cow::from(\":o\"),\n            Cow::from(\"single_word\"),\n            Cow::from(\"tw\u00f3 w\u00f6rds\"),\n            Cow::from(r#\"three' \"with escaping\\\"#),\n            Cow::from(\"dquote incomplete\"),\n        ];\n        assert_eq!(expected, result);\n    }\n\n    #[test]\n    #[cfg(unix)]\n    fn test_mixed() {\n        let dquoted = r#\":o single_word 'tw\u00f3 w\u00f6rds' \"\\three\\' \\\"with\\ escaping\\\\\"\"no space before\"'and after' $#%^@ \"%^&(%^\" ')(*&^%''a\\\\\\\\\\b' '\"#;\n        let shellwords = Shellwords::from(dquoted);\n        let result = shellwords.words().to_vec();\n        let expected = vec![\n            Cow::from(\":o\"),\n            Cow::from(\"single_word\"),\n            Cow::from(\"tw\u00f3 w\u00f6rds\"),\n            Cow::from(\"three' \\\"with escaping\\\\\"),\n            Cow::from(\"no space before\"),\n            Cow::from(\"and after\"),\n            Cow::from(\"$#%^@\"),\n            Cow::from(\"%^&(%^\"),\n            Cow::from(\")(*&^%\"),\n            Cow::from(r#\"a\\\\b\"#),\n            //last ' just changes to quoted but since we dont have anything after it, it should be ignored\n        ];\n        assert_eq!(expected, result);\n    }\n\n    #[test]\n    fn test_lists() {\n        let input =\n            r#\":set statusline.center [\"file-type\",\"file-encoding\"] '[\"list\", \"in\", \"quotes\"]'\"#;\n        let shellwords = Shellwords::from(input);\n        let result = shellwords.words().to_vec();\n        let expected = vec![\n            Cow::from(\":set\"),\n            Cow::from(\"statusline.center\"),\n            Cow::from(r#\"[\"file-type\",\"file-encoding\"]\"#),\n            Cow::from(r#\"[\"list\", \"in\", \"quotes\"]\"#),\n        ];\n        assert_eq!(expected, result);\n    }\n\n    #[test]\n    #[cfg(unix)]\n    fn test_escaping_unix() {\n        assert_eq!(escape(\"foobar\".into()), Cow::Borrowed(\"foobar\"));\n        assert_eq!(escape(\"foo bar\".into()), Cow::Borrowed(\"foo\\\\ bar\"));\n        assert_eq!(escape(\"foo\\tbar\".into()), Cow::Borrowed(\"foo\\\\\\tbar\"));\n    }\n\n    #[test]\n    #[cfg(windows)]\n    fn test_escaping_windows() {\n        assert_eq!(escape(\"foobar\".into()), Cow::Borrowed(\"foobar\"));\n        assert_eq!(escape(\"foo bar\".into()), Cow::Borrowed(\"\\\"foo bar\\\"\"));\n    }\n\n    #[test]\n    #[cfg(unix)]\n    fn test_parts() {\n        assert_eq!(Shellwords::from(\":o a\").parts(), &[\":o\", \"a\"]);\n        assert_eq!(Shellwords::from(\":o a\\\\ \").parts(), &[\":o\", \"a\\\\ \"]);\n    }\n\n    #[test]\n    #[cfg(windows)]\n    fn test_parts() {\n        assert_eq!(Shellwords::from(\":o a\").parts(), &[\":o\", \"a\"]);\n        assert_eq!(Shellwords::from(\":o a\\\\ \").parts(), &[\":o\", \"a\\\\\"]);\n    }\n\n    #[test]\n    fn test_multibyte_at_end() {\n        assert_eq!(Shellwords::from(\"\ud808\udc00\").parts(), &[\"\ud808\udc00\"]);\n        assert_eq!(\n            Shellwords::from(\":sh echo \ud808\udc00\").parts(),\n            &[\":sh\", \"echo\", \"\ud808\udc00\"]\n        );\n        assert_eq!(\n            Shellwords::from(\":sh echo \ud808\udc00 hello world\ud808\udc00\").parts(),\n            &[\":sh\", \"echo\", \"\ud808\udc00\", \"hello\", \"world\ud808\udc00\"]\n        );\n    }\n}\n\n// Path: helix-core/src/macros.rs\n#[macro_export]\nmacro_rules! hashmap {\n    (@single $($x:tt)*) => (());\n    (@count $($rest:expr),*) => (<[()]>::len(&[$(hashmap!(@single $rest)),*]));\n\n    ($($key:expr => $value:expr,)+) => { hashmap!($($key => $value),+) };\n    ($($key:expr => $value:expr),*) => {\n        {\n            let _cap = hashmap!(@count $($key),*);\n            let mut _map = ::std::collections::HashMap::with_capacity(_cap);\n            $(\n                let _ = _map.insert($key, $value);\n            )*\n            _map\n        }\n    };\n}\n\n// Path: helix-core/src/object.rs\nuse crate::{syntax::TreeCursor, Range, RopeSlice, Selection, Syntax};\n\npub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {\n    let cursor = &mut syntax.walk();\n\n    selection.transform(|range| {\n        let from = text.char_to_byte(range.from());\n        let to = text.char_to_byte(range.to());\n\n        let byte_range = from..to;\n        cursor.reset_to_byte_range(from, to);\n\n        while cursor.node().byte_range() == byte_range {\n            if !cursor.goto_parent() {\n                break;\n            }\n        }\n\n        let node = cursor.node();\n        let from = text.byte_to_char(node.start_byte());\n        let to = text.byte_to_char(node.end_byte());\n\n        Range::new(to, from).with_direction(range.direction())\n    })\n}\n\npub fn shrink_selection(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {\n    select_node_impl(syntax, text, selection, |cursor| {\n        cursor.goto_first_child();\n    })\n}\n\npub fn select_next_sibling(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {\n    select_node_impl(syntax, text, selection, |cursor| {\n        while !cursor.goto_next_sibling() {\n            if !cursor.goto_parent() {\n                break;\n            }\n        }\n    })\n}\n\npub fn select_prev_sibling(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {\n    select_node_impl(syntax, text, selection, |cursor| {\n        while !cursor.goto_prev_sibling() {\n            if !cursor.goto_parent() {\n                break;\n            }\n        }\n    })\n}\n\nfn select_node_impl<F>(\n    syntax: &Syntax,\n    text: RopeSlice,\n    selection: Selection,\n    motion: F,\n) -> Selection\nwhere\n    F: Fn(&mut TreeCursor),\n{\n    let cursor = &mut syntax.walk();\n\n    selection.transform(|range| {\n        let from = text.char_to_byte(range.from());\n        let to = text.char_to_byte(range.to());\n\n        cursor.reset_to_byte_range(from, to);\n\n        motion(cursor);\n\n        let node = cursor.node();\n        let from = text.byte_to_char(node.start_byte());\n        let to = text.byte_to_char(node.end_byte());\n\n        Range::new(from, to).with_direction(range.direction())\n    })\n}\n\n// Path: helix-core/src/match_brackets.rs\nuse std::iter;\n\nuse ropey::RopeSlice;\nuse tree_sitter::Node;\n\nuse crate::movement::Direction::{self, Backward, Forward};\nuse crate::Syntax;\n\nconst MAX_PLAINTEXT_SCAN: usize = 10000;\nconst MATCH_LIMIT: usize = 16;\n\n// Limit matching pairs to only ( ) { } [ ] < > ' ' \" \"\nconst PAIRS: &[(char, char)] = &[\n    ('(', ')'),\n    ('{', '}'),\n    ('[', ']'),\n    ('<', '>'),\n    ('\\'', '\\''),\n    ('\\\"', '\\\"'),\n];\n\n/// Returns the position of the matching bracket under cursor.\n///\n/// If the cursor is on the opening bracket, the position of\n/// the closing bracket is returned. If the cursor on the closing\n/// bracket, the position of the opening bracket is returned.\n///\n/// If the cursor is not on a bracket, `None` is returned.\n///\n/// If no matching bracket is found, `None` is returned.\n#[must_use]\npub fn find_matching_bracket(syntax: &Syntax, doc: RopeSlice, pos: usize) -> Option<usize> {\n    if pos >= doc.len_chars() || !is_valid_bracket(doc.char(pos)) {\n        return None;\n    }\n    find_pair(syntax, doc, pos, false)\n}\n\n// Returns the position of the bracket that is closing the current scope.\n//\n// If the cursor is on an opening or closing bracket, the function\n// behaves equivalent to [`find_matching_bracket`].\n//\n// If the cursor position is within a scope, the function searches\n// for the surrounding scope that is surrounded by brackets and\n// returns the position of the closing bracket for that scope.\n//\n// If no surrounding scope is found, the function returns `None`.\n#[must_use]\npub fn find_matching_bracket_fuzzy(syntax: &Syntax, doc: RopeSlice, pos: usize) -> Option<usize> {\n    find_pair(syntax, doc, pos, true)\n}\n\nfn find_pair(\n    syntax: &Syntax,\n    doc: RopeSlice,\n    pos_: usize,\n    traverse_parents: bool,\n) -> Option<usize> {\n    let pos = doc.char_to_byte(pos_);\n\n    let root = syntax.tree_for_byte_range(pos, pos + 1).root_node();\n    let mut node = root.descendant_for_byte_range(pos, pos + 1)?;\n\n    loop {\n        if node.is_named() {\n            let (start_byte, end_byte) = surrounding_bytes(doc, &node)?;\n            let (start_char, end_char) = (doc.byte_to_char(start_byte), doc.byte_to_char(end_byte));\n\n            if is_valid_pair(doc, start_char, end_char) {\n                if end_byte == pos {\n                    return Some(start_char);\n                }\n\n                // We return the end char if the cursor is either on the start char\n                // or at some arbitrary position between start and end char.\n                if traverse_parents || start_byte == pos {\n                    return Some(end_char);\n                }\n            }\n        }\n        // this node itselt wasn't a pair but maybe its siblings are\n\n        // check if we are *on* the pair (special cased so we don't look\n        // at the current node twice and to jump to the start on that case)\n        if let Some(open) = as_close_pair(doc, &node) {\n            if let Some(pair_start) = find_pair_end(doc, node.prev_sibling(), open, Backward) {\n                return Some(pair_start);\n            }\n        }\n\n        if !traverse_parents {\n            // check if we are *on* the opening pair (special cased here as\n            // an opptimization since we only care about bracket on the cursor\n            // here)\n            if let Some(close) = as_open_pair(doc, &node) {\n                if let Some(pair_end) = find_pair_end(doc, node.next_sibling(), close, Forward) {\n                    return Some(pair_end);\n                }\n            }\n            if node.is_named() {\n                break;\n            }\n        }\n\n        for close in\n            iter::successors(node.next_sibling(), |node| node.next_sibling()).take(MATCH_LIMIT)\n        {\n            let Some(open) = as_close_pair(doc, &close) else {\n                continue;\n            };\n            if find_pair_end(doc, Some(node), open, Backward).is_some() {\n                return doc.try_byte_to_char(close.start_byte()).ok();\n            }\n        }\n        let Some(parent) = node.parent() else {\n            break;\n        };\n        node = parent;\n    }\n    let node = root.named_descendant_for_byte_range(pos, pos + 1)?;\n    if node.child_count() != 0 {\n        return None;\n    }\n    let node_start = doc.byte_to_char(node.start_byte());\n    find_matching_bracket_plaintext(doc.byte_slice(node.byte_range()), pos_ - node_start)\n        .map(|pos| pos + node_start)\n}\n\n/// Returns the position of the matching bracket under cursor.\n/// This function works on plain text and ignores tree-sitter grammar.\n/// The search is limited to `MAX_PLAINTEXT_SCAN` characters\n///\n/// If the cursor is on the opening bracket, the position of\n/// the closing bracket is returned. If the cursor on the closing\n/// bracket, the position of the opening bracket is returned.\n///\n/// If the cursor is not on a bracket, `None` is returned.\n///\n/// If no matching bracket is found, `None` is returned.\n#[must_use]\npub fn find_matching_bracket_plaintext(doc: RopeSlice, cursor_pos: usize) -> Option<usize> {\n    // Don't do anything when the cursor is not on top of a bracket.\n    let bracket = doc.get_char(cursor_pos)?;\n    if !is_valid_bracket(bracket) {\n        return None;\n    }\n\n    // Determine the direction of the matching.\n    let is_fwd = is_forward_bracket(bracket);\n    let chars_iter = if is_fwd {\n        doc.chars_at(cursor_pos + 1)\n    } else {\n        doc.chars_at(cursor_pos).reversed()\n    };\n\n    let mut open_cnt = 1;\n\n    for (i, candidate) in chars_iter.take(MAX_PLAINTEXT_SCAN).enumerate() {\n        if candidate == bracket {\n            open_cnt += 1;\n        } else if is_valid_pair(\n            doc,\n            if is_fwd {\n                cursor_pos\n            } else {\n                cursor_pos - i - 1\n            },\n            if is_fwd {\n                cursor_pos + i + 1\n            } else {\n                cursor_pos\n            },\n        ) {\n            // Return when all pending brackets have been closed.\n            if open_cnt == 1 {\n                return Some(if is_fwd {\n                    cursor_pos + i + 1\n                } else {\n                    cursor_pos - i - 1\n                });\n            }\n            open_cnt -= 1;\n        }\n    }\n\n    None\n}\n\nfn is_valid_bracket(c: char) -> bool {\n    PAIRS.iter().any(|(l, r)| *l == c || *r == c)\n}\n\nfn is_forward_bracket(c: char) -> bool {\n    PAIRS.iter().any(|(l, _)| *l == c)\n}\n\nfn is_valid_pair(doc: RopeSlice, start_char: usize, end_char: usize) -> bool {\n    PAIRS.contains(&(doc.char(start_char), doc.char(end_char)))\n}\n\nfn surrounding_bytes(doc: RopeSlice, node: &Node) -> Option<(usize, usize)> {\n    let len = doc.len_bytes();\n\n    let start_byte = node.start_byte();\n    let end_byte = node.end_byte().saturating_sub(1);\n\n    if start_byte >= len || end_byte >= len {\n        return None;\n    }\n\n    Some((start_byte, end_byte))\n}\n\n/// Tests if this node is a pair close char and returns the expected open char\nfn as_close_pair(doc: RopeSlice, node: &Node) -> Option<char> {\n    let close = as_char(doc, node)?.1;\n    PAIRS\n        .iter()\n        .find_map(|&(open, close_)| (close_ == close).then_some(open))\n}\n\n/// Checks if `node` or its siblings (at most MATCH_LIMIT nodes) is the specified closing char\n///\n/// # Returns\n///\n/// The position of the found node or `None` otherwise\nfn find_pair_end(\n    doc: RopeSlice,\n    node: Option<Node>,\n    end_char: char,\n    direction: Direction,\n) -> Option<usize> {\n    let advance = match direction {\n        Forward => Node::next_sibling,\n        Backward => Node::prev_sibling,\n    };\n    iter::successors(node, advance)\n        .take(MATCH_LIMIT)\n        .find_map(|node| {\n            let (pos, c) = as_char(doc, &node)?;\n            (end_char == c).then_some(pos)\n        })\n}\n\n/// Tests if this node is a pair close char and returns the expected open char\nfn as_open_pair(doc: RopeSlice, node: &Node) -> Option<char> {\n    let open = as_char(doc, node)?.1;\n    PAIRS\n        .iter()\n        .find_map(|&(open_, close)| (open_ == open).then_some(close))\n}\n\n/// If node is a single char return it (and its char position)\nfn as_char(doc: RopeSlice, node: &Node) -> Option<(usize, char)> {\n    // TODO: multi char/non ASCII pairs\n    if node.byte_range().len() != 1 {\n        return None;\n    }\n    let pos = doc.try_byte_to_char(node.start_byte()).ok()?;\n    Some((pos, doc.char(pos)))\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn find_matching_bracket_empty_file() {\n        let actual = find_matching_bracket_plaintext(\"\".into(), 0);\n        assert_eq!(actual, None);\n    }\n\n    #[test]\n    fn test_find_matching_bracket_current_line_plaintext() {\n        let assert = |input: &str, pos, expected| {\n            let input = RopeSlice::from(input);\n            let actual = find_matching_bracket_plaintext(input, pos);\n            assert_eq!(expected, actual.unwrap());\n\n            let actual = find_matching_bracket_plaintext(input, expected);\n            assert_eq!(pos, actual.unwrap(), \"expected symmetrical behaviour\");\n        };\n\n        assert(\"(hello)\", 0, 6);\n        assert(\"((hello))\", 0, 8);\n        assert(\"((hello))\", 1, 7);\n        assert(\"(((hello)))\", 2, 8);\n\n        assert(\"key: ${value}\", 6, 12);\n        assert(\"key: ${value} # (some comment)\", 16, 29);\n\n        assert(\"(paren (paren {bracket}))\", 0, 24);\n        assert(\"(paren (paren {bracket}))\", 7, 23);\n        assert(\"(paren (paren {bracket}))\", 14, 22);\n\n        assert(\"(prev line\\n ) (middle) ( \\n next line)\", 0, 12);\n        assert(\"(prev line\\n ) (middle) ( \\n next line)\", 14, 21);\n        assert(\"(prev line\\n ) (middle) ( \\n next line)\", 23, 36);\n    }\n}\n\n// Path: helix-core/src/test.rs\n//! Test helpers.\nuse crate::{Range, Selection};\nuse ropey::Rope;\nuse smallvec::SmallVec;\nuse std::cmp::Reverse;\nuse unicode_segmentation::UnicodeSegmentation;\n\n/// Convert annotated test string to test string and selection.\n///\n/// `#[|` for primary selection with head before anchor followed by `]#`.\n/// `#(|` for secondary selection with head before anchor followed by `)#`.\n/// `#[` for primary selection with head after anchor followed by `|]#`.\n/// `#(` for secondary selection with head after anchor followed by `|)#`.\n///\n/// If the selection contains any LF or CRLF sequences, which are immediately\n/// followed by the same grapheme, then the subsequent one is removed. This is\n/// to allow representing having the cursor over the end of the line.\n///\n/// # Examples\n///\n/// ```\n/// use helix_core::{Range, Selection, test::print};\n/// use smallvec::smallvec;\n///\n/// assert_eq!(\n///     print(\"#[a|]#b#(|c)#\"),\n///     (\"abc\".to_owned(), Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0))\n/// );\n/// ```\n///\n/// # Panics\n///\n/// Panics when missing primary or appeared more than once.\n/// Panics when missing head or anchor.\n/// Panics when head come after head or anchor come after anchor.\npub fn print(s: &str) -> (String, Selection) {\n    let mut primary_idx = None;\n    let mut ranges = SmallVec::new();\n    let mut iter = UnicodeSegmentation::graphemes(s, true).peekable();\n    let mut left = String::with_capacity(s.len());\n\n    'outer: while let Some(c) = iter.next() {\n        let start = left.chars().count();\n\n        if c != \"#\" {\n            left.push_str(c);\n            continue;\n        }\n\n        let (is_primary, close_pair) = match iter.next() {\n            Some(\"[\") => (true, \"]\"),\n            Some(\"(\") => (false, \")\"),\n            Some(ch) => {\n                left.push('#');\n                left.push_str(ch);\n                continue;\n            }\n            None => break,\n        };\n\n        if is_primary && primary_idx.is_some() {\n            panic!(\"primary `#[` already appeared {:?} {:?}\", left, s);\n        }\n\n        let head_at_beg = iter.next_if_eq(&\"|\").is_some();\n        let last_grapheme = |s: &str| {\n            UnicodeSegmentation::graphemes(s, true)\n                .last()\n                .map(String::from)\n        };\n\n        while let Some(c) = iter.next() {\n            let next = iter.peek();\n            let mut prev = last_grapheme(left.as_str());\n\n            if !(c == close_pair && next == Some(&\"#\")) {\n                left.push_str(c);\n                continue;\n            }\n\n            if !head_at_beg {\n                match &prev {\n                    Some(p) if p != \"|\" => {\n                        left.push_str(c);\n                        continue;\n                    }\n                    Some(p) if p == \"|\" => {\n                        left.pop().unwrap(); // pop the |\n                        prev = last_grapheme(left.as_str());\n                    }\n                    _ => (),\n                }\n            }\n\n            iter.next(); // skip \"#\"\n            let next = iter.peek();\n\n            // skip explicit line end inside selection\n            if (prev == Some(String::from(\"\\r\\n\")) || prev == Some(String::from(\"\\n\")))\n                && next.map(|n| String::from(*n)) == prev\n            {\n                iter.next();\n            }\n\n            if is_primary {\n                primary_idx = Some(ranges.len());\n            }\n\n            let (anchor, head) = match head_at_beg {\n                true => (left.chars().count(), start),\n                false => (start, left.chars().count()),\n            };\n\n            ranges.push(Range::new(anchor, head));\n            continue 'outer;\n        }\n\n        if head_at_beg {\n            panic!(\"missing end `{}#` {:?} {:?}\", close_pair, left, s);\n        } else {\n            panic!(\"missing end `|{}#` {:?} {:?}\", close_pair, left, s);\n        }\n    }\n\n    let primary = match primary_idx {\n        Some(i) => i,\n        None => panic!(\"missing primary `#[|]#` {:?}\", s),\n    };\n\n    let selection = Selection::new(ranges, primary);\n    (left, selection)\n}\n\n/// Convert test string and selection to annotated test string.\n///\n/// `#[|` for primary selection with head before anchor followed by `]#`.\n/// `#(|` for secondary selection with head before anchor followed by `)#`.\n/// `#[` for primary selection with head after anchor followed by `|]#`.\n/// `#(` for secondary selection with head after anchor followed by `|)#`.\n///\n/// # Examples\n///\n/// ```\n/// use helix_core::{Range, Selection, test::plain};\n/// use smallvec::smallvec;\n///\n/// assert_eq!(\n///     plain(\"abc\", &Selection::new(smallvec![Range::new(0, 1), Range::new(3, 2)], 0)),\n///     \"#[a|]#b#(|c)#\".to_owned()\n/// );\n/// ```\npub fn plain<R: Into<Rope>>(s: R, selection: &Selection) -> String {\n    let s = s.into();\n    let primary = selection.primary_index();\n    let mut out = String::with_capacity(s.len_bytes() + 5 * selection.len());\n    out.push_str(&s.to_string());\n\n    let mut insertion: Vec<_> = selection\n        .iter()\n        .enumerate()\n        .flat_map(|(i, range)| {\n            // sort like this before reversed so anchor < head later\n            match (range.anchor < range.head, i == primary) {\n                (true, true) => [(range.anchor, \"#[\"), (range.head, \"|]#\")],\n                (true, false) => [(range.anchor, \"#(\"), (range.head, \"|)#\")],\n                (false, true) => [(range.anchor, \"]#\"), (range.head, \"#[|\")],\n                (false, false) => [(range.anchor, \")#\"), (range.head, \"#(|\")],\n            }\n        })\n        .map(|(char_idx, marker)| (s.char_to_byte(char_idx), marker))\n        .collect();\n\n    // insert in reverse order\n    insertion.sort_unstable_by_key(|k| Reverse(k.0));\n    for (i, s) in insertion {\n        out.insert_str(i, s);\n    }\n    out\n}\n\n#[cfg(test)]\n#[allow(clippy::module_inception)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn print_single() {\n        assert_eq!(\n            (String::from(\"hello\"), Selection::single(1, 0)),\n            print(\"#[|h]#ello\")\n        );\n        assert_eq!(\n            (String::from(\"hello\"), Selection::single(0, 1)),\n            print(\"#[h|]#ello\")\n        );\n        assert_eq!(\n            (String::from(\"hello\"), Selection::single(4, 0)),\n            print(\"#[|hell]#o\")\n        );\n        assert_eq!(\n            (String::from(\"hello\"), Selection::single(0, 4)),\n            print(\"#[hell|]#o\")\n        );\n        assert_eq!(\n            (String::from(\"hello\"), Selection::single(5, 0)),\n            print(\"#[|hello]#\")\n        );\n        assert_eq!(\n            (String::from(\"hello\"), Selection::single(0, 5)),\n            print(\"#[hello|]#\")\n        );\n    }\n\n    #[test]\n    fn print_multi() {\n        assert_eq!(\n            (\n                String::from(\"hello\"),\n                Selection::new(\n                    SmallVec::from_slice(&[Range::new(1, 0), Range::new(5, 4)]),\n                    0\n                )\n            ),\n            print(\"#[|h]#ell#(|o)#\")\n        );\n        assert_eq!(\n            (\n                String::from(\"hello\"),\n                Selection::new(\n                    SmallVec::from_slice(&[Range::new(0, 1), Range::new(4, 5)]),\n                    0\n                )\n            ),\n            print(\"#[h|]#ell#(o|)#\")\n        );\n        assert_eq!(\n            (\n                String::from(\"hello\"),\n                Selection::new(\n                    SmallVec::from_slice(&[Range::new(2, 0), Range::new(5, 3)]),\n                    0\n                )\n            ),\n            print(\"#[|he]#l#(|lo)#\")\n        );\n        assert_eq!(\n            (\n                String::from(\"hello\\r\\nhello\\r\\nhello\\r\\n\"),\n                Selection::new(\n                    SmallVec::from_slice(&[\n                        Range::new(7, 5),\n                        Range::new(21, 19),\n                        Range::new(14, 12)\n                    ]),\n                    0\n                )\n            ),\n            print(\"hello#[|\\r\\n]#hello#(|\\r\\n)#hello#(|\\r\\n)#\")\n        );\n    }\n\n    #[test]\n    fn print_multi_byte_code_point() {\n        assert_eq!(\n            (String::from(\"\u201e\u201c\"), Selection::single(1, 0)),\n            print(\"#[|\u201e]#\u201c\")\n        );\n        assert_eq!(\n            (String::from(\"\u201e\u201c\"), Selection::single(2, 1)),\n            print(\"\u201e#[|\u201c]#\")\n        );\n        assert_eq!(\n            (String::from(\"\u201e\u201c\"), Selection::single(0, 1)),\n            print(\"#[\u201e|]#\u201c\")\n        );\n        assert_eq!(\n            (String::from(\"\u201e\u201c\"), Selection::single(1, 2)),\n            print(\"\u201e#[\u201c|]#\")\n        );\n        assert_eq!(\n            (String::from(\"they said \u201ehello\u201c\"), Selection::single(11, 10)),\n            print(\"they said #[|\u201e]#hello\u201c\")\n        );\n    }\n\n    #[test]\n    fn print_multi_code_point_grapheme() {\n        assert_eq!(\n            (\n                String::from(\"hello \ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 goodbye\"),\n                Selection::single(13, 6)\n            ),\n            print(\"hello #[|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66]# goodbye\")\n        );\n    }\n\n    #[test]\n    fn plain_single() {\n        assert_eq!(\"#[|h]#ello\", plain(\"hello\", &Selection::single(1, 0)));\n        assert_eq!(\"#[h|]#ello\", plain(\"hello\", &Selection::single(0, 1)));\n        assert_eq!(\"#[|hell]#o\", plain(\"hello\", &Selection::single(4, 0)));\n        assert_eq!(\"#[hell|]#o\", plain(\"hello\", &Selection::single(0, 4)));\n        assert_eq!(\"#[|hello]#\", plain(\"hello\", &Selection::single(5, 0)));\n        assert_eq!(\"#[hello|]#\", plain(\"hello\", &Selection::single(0, 5)));\n    }\n\n    #[test]\n    fn plain_multi() {\n        assert_eq!(\n            plain(\n                \"hello\",\n                &Selection::new(\n                    SmallVec::from_slice(&[Range::new(1, 0), Range::new(5, 4)]),\n                    0\n                )\n            ),\n            String::from(\"#[|h]#ell#(|o)#\")\n        );\n        assert_eq!(\n            plain(\n                \"hello\",\n                &Selection::new(\n                    SmallVec::from_slice(&[Range::new(0, 1), Range::new(4, 5)]),\n                    0\n                )\n            ),\n            String::from(\"#[h|]#ell#(o|)#\")\n        );\n        assert_eq!(\n            plain(\n                \"hello\",\n                &Selection::new(\n                    SmallVec::from_slice(&[Range::new(2, 0), Range::new(5, 3)]),\n                    0\n                )\n            ),\n            String::from(\"#[|he]#l#(|lo)#\")\n        );\n        assert_eq!(\n            plain(\n                \"hello\\r\\nhello\\r\\nhello\\r\\n\",\n                &Selection::new(\n                    SmallVec::from_slice(&[\n                        Range::new(7, 5),\n                        Range::new(21, 19),\n                        Range::new(14, 12)\n                    ]),\n                    0\n                )\n            ),\n            String::from(\"hello#[|\\r\\n]#hello#(|\\r\\n)#hello#(|\\r\\n)#\")\n        );\n    }\n\n    #[test]\n    fn plain_multi_byte_code_point() {\n        assert_eq!(\n            plain(\"\u201e\u201c\", &Selection::single(1, 0)),\n            String::from(\"#[|\u201e]#\u201c\")\n        );\n        assert_eq!(\n            plain(\"\u201e\u201c\", &Selection::single(2, 1)),\n            String::from(\"\u201e#[|\u201c]#\")\n        );\n        assert_eq!(\n            plain(\"\u201e\u201c\", &Selection::single(0, 1)),\n            String::from(\"#[\u201e|]#\u201c\")\n        );\n        assert_eq!(\n            plain(\"\u201e\u201c\", &Selection::single(1, 2)),\n            String::from(\"\u201e#[\u201c|]#\")\n        );\n        assert_eq!(\n            plain(\"they said \u201ehello\u201c\", &Selection::single(11, 10)),\n            String::from(\"they said #[|\u201e]#hello\u201c\")\n        );\n    }\n\n    #[test]\n    fn plain_multi_code_point_grapheme() {\n        assert_eq!(\n            plain(\"hello \ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66 goodbye\", &Selection::single(13, 6)),\n            String::from(\"hello #[|\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc67\u200d\ud83d\udc66]# goodbye\")\n        );\n    }\n}\n\n// Path: helix-core/src/lib.rs\npub use encoding_rs as encoding;\n\npub mod auto_pairs;\npub mod chars;\npub mod comment;\npub mod config;\npub mod diagnostic;\npub mod diff;\npub mod doc_formatter;\npub mod fuzzy;\npub mod graphemes;\npub mod history;\npub mod increment;\npub mod indent;\npub mod line_ending;\npub mod macros;\npub mod match_brackets;\npub mod movement;\npub mod object;\nmod position;\npub mod search;\npub mod selection;\npub mod shellwords;\npub mod surround;\npub mod syntax;\npub mod test;\npub mod text_annotations;\npub mod textobject;\nmod transaction;\npub mod wrap;\n\npub mod unicode {\n    pub use unicode_general_category as category;\n    pub use unicode_segmentation as segmentation;\n    pub use unicode_width as width;\n}\n\npub use helix_loader::find_workspace;\n\nmod rope_reader;\n\npub use rope_reader::RopeReader;\npub use ropey::{self, str_utils, Rope, RopeBuilder, RopeSlice};\n\n// pub use tendril::StrTendril as Tendril;\npub use smartstring::SmartString;\n\npub type Tendril = SmartString<smartstring::LazyCompact>;\n\n#[doc(inline)]\npub use {regex, tree_sitter};\n\npub use graphemes::RopeGraphemes;\npub use position::{\n    char_idx_at_visual_offset, coords_at_pos, pos_at_coords, visual_offset_from_anchor,\n    visual_offset_from_block, Position, VisualOffsetError,\n};\n#[allow(deprecated)]\npub use position::{pos_at_visual_coords, visual_coords_at_pos};\n\npub use selection::{Range, Selection};\npub use smallvec::{smallvec, SmallVec};\npub use syntax::Syntax;\n\npub use diagnostic::Diagnostic;\n\npub use line_ending::{LineEnding, NATIVE_LINE_ENDING};\npub use transaction::{Assoc, Change, ChangeSet, Deletion, Operation, Transaction};\n\n", "needle_token_start": 2411, "needle_token_end": 2434, "code_context_ntokens": 14840, "output": ["```rust\nstruct RopeLines<'a>(RopeSlice<'a>);\n\nimpl<'a> imara_diff::intern::TokenSource for RopeLines<'a> {\n    type Token = RopeSlice<'a>;\n    type Tokenizer = ropey::iter::Lines<'a>;\n\n    fn tokenize(&self) -> Self::Tokenizer {\n        self.0.lines()\n    }\n\n    fn estimate_tokens(&self) -> u32 {\n        self.0.len_lines() as u32\n    }\n}\n```"]}
{"repo": "helix-editor/helix", "name": "serialize_lang_features", "language": "rust", "path": "helix-core/src/syntax.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a collection of language server feature configurations into a serialized format suitable for storage or transmission.\n2. **Input**: It accepts a vector containing multiple configurations, each specifying a set of enabled and excluded features for a language server, along with a name identifier.\n3. **Output**: The output is a serialized sequence of these configurations, formatted according to specified serialization rules.\n4. **Procedure**: The function iterates over each configuration in the input vector. For each configuration, it checks if any features are specifically enabled or excluded. If no specific features are set, it uses a simplified format with just the name. Otherwise, it includes the detailed lists of enabled and excluded features along with the name. Each configuration is then serialized into the sequence.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    fmt::{self, Display},\n    hash::{Hash, Hasher},\n    mem::replace,\n    path::{Path, PathBuf},\n    str::FromStr,\n    sync::Arc,\n};\n\nuse once_cell::sync::{Lazy, OnceCell};\nuse serde::{ser::SerializeSeq, Deserialize, Serialize};\n\nuse helix_loader::grammar::{get_language, load_runtime_file};\n\npub use tree_cursor::TreeCursor;\n\nfn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<String>::deserialize(deserializer)?\n        .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_lsp_config<'de, D>(deserializer: D) -> Result<Option<serde_json::Value>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<toml::Value>::deserialize(deserializer)?\n        .map(|toml| toml.try_into().map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_tab_width<'de, D>(deserializer: D) -> Result<usize, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    usize::deserialize(deserializer).and_then(|n| {\n        if n > 0 && n <= 16 {\n            Ok(n)\n        } else {\n            Err(serde::de::Error::custom(\n                \"tab width must be a value from 1 to 16 inclusive\",\n            ))\n        }\n    })\n}\n\npub fn deserialize_auto_pairs<'de, D>(deserializer: D) -> Result<Option<AutoPairs>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Ok(Option::<AutoPairConfig>::deserialize(deserializer)?.and_then(AutoPairConfig::into))\n}\n\nfn default_timeout() -> u64 {\n    20\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct Configuration {\n    pub language: Vec<LanguageConfiguration>,\n    #[serde(default)]\n    pub language_server: HashMap<String, LanguageServerConfiguration>,\n}\n\n// largely based on tree-sitter/cli/src/loader.rs\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\", deny_unknown_fields)]\npub struct LanguageConfiguration {\n    #[serde(rename = \"name\")]\n    pub language_id: String, // c-sharp, rust, tsx\n    #[serde(rename = \"language-id\")]\n    // see the table under https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocumentItem\n    pub language_server_language_id: Option<String>, // csharp, rust, typescriptreact, for the language-server\n    pub scope: String,             // source.rust\n    pub file_types: Vec<FileType>, // filename extension or ends_with? <Gemfile, rb, etc>\n    #[serde(default)]\n    pub shebangs: Vec<String>, // interpreter(s) associated with language\n    #[serde(default)]\n    pub roots: Vec<String>, // these indicate project roots <.git, Cargo.toml>\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_comment_tokens\",\n        alias = \"comment-token\"\n    )]\n    pub comment_tokens: Option<Vec<String>>,\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_block_comment_tokens\"\n    )]\n    pub block_comment_tokens: Option<Vec<BlockCommentToken>>,\n    pub text_width: Option<usize>,\n    pub soft_wrap: Option<SoftWrap>,\n\n    #[serde(default)]\n    pub auto_format: bool,\n\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub formatter: Option<FormatterConfiguration>,\n\n    #[serde(default)]\n    pub diagnostic_severity: Severity,\n\n    pub grammar: Option<String>, // tree-sitter grammar name, defaults to language_id\n\n    // content_regex\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_regex\")]\n    pub injection_regex: Option<Regex>,\n    // first_line_regex\n    //\n    #[serde(skip)]\n    pub(crate) highlight_config: OnceCell<Option<Arc<HighlightConfiguration>>>,\n    // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583\n    #[serde(\n        default,\n        skip_serializing_if = \"Vec::is_empty\",\n        serialize_with = \"serialize_lang_features\",\n        deserialize_with = \"deserialize_lang_features\"\n    )]\n    pub language_servers: Vec<LanguageServerFeatures>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub indent: Option<IndentationConfiguration>,\n\n    #[serde(skip)]\n    pub(crate) indent_query: OnceCell<Option<Query>>,\n    #[serde(skip)]\n    pub(crate) textobject_query: OnceCell<Option<TextObjectQuery>>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub debugger: Option<DebugAdapterConfig>,\n\n    /// Automatic insertion of pairs to parentheses, brackets,\n    /// etc. Defaults to true. Optionally, this can be a list of 2-tuples\n    /// to specify a list of characters to pair. This overrides the\n    /// global setting.\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_auto_pairs\")]\n    pub auto_pairs: Option<AutoPairs>,\n\n    pub rulers: Option<Vec<u16>>, // if set, override editor's rulers\n\n    /// Hardcoded LSP root directories relative to the workspace root, like `examples` or `tools/fuzz`.\n    /// Falling back to the current working directory if none are configured.\n    pub workspace_lsp_roots: Option<Vec<PathBuf>>,\n    #[serde(default)]\n    pub persistent_diagnostic_sources: Vec<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Hash)]\npub enum FileType {\n    /// The extension of the file, either the `Path::extension` or the full\n    /// filename if the file does not have an extension.\n    Extension(String),\n    /// A Unix-style path glob. This is compared to the file's absolute path, so\n    /// it can be used to detect files based on their directories. If the glob\n    /// is not an absolute path and does not already start with a glob pattern,\n    /// a glob pattern will be prepended to it.\n    Glob(globset::Glob),\n}\n\nimpl Serialize for FileType {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: serde::Serializer,\n    {\n        use serde::ser::SerializeMap;\n\n        match self {\n            FileType::Extension(extension) => serializer.serialize_str(extension),\n            FileType::Glob(glob) => {\n                let mut map = serializer.serialize_map(Some(1))?;\n                map.serialize_entry(\"glob\", glob.glob())?;\n                map.end()\n            }\n        }\n    }\n}\n\nimpl<'de> Deserialize<'de> for FileType {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: serde::de::Deserializer<'de>,\n    {\n        struct FileTypeVisitor;\n\n        impl<'de> serde::de::Visitor<'de> for FileTypeVisitor {\n            type Value = FileType;\n\n            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {\n                formatter.write_str(\"string or table\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>\n            where\n                E: serde::de::Error,\n            {\n                Ok(FileType::Extension(value.to_string()))\n            }\n\n            fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>\n            where\n                M: serde::de::MapAccess<'de>,\n            {\n                match map.next_entry::<String, String>()? {\n                    Some((key, mut glob)) if key == \"glob\" => {\n                        // If the glob isn't an absolute path or already starts\n                        // with a glob pattern, add a leading glob so we\n                        // properly match relative paths.\n                        if !glob.starts_with('/') && !glob.starts_with(\"*/\") {\n                            glob.insert_str(0, \"*/\");\n                        }\n\n                        globset::Glob::new(glob.as_str())\n                            .map(FileType::Glob)\n                            .map_err(|err| {\n                                serde::de::Error::custom(format!(\"invalid `glob` pattern: {}\", err))\n                            })\n                    }\n                    Some((key, _value)) => Err(serde::de::Error::custom(format!(\n                        \"unknown key in `file-types` list: {}\",\n                        key\n                    ))),\n                    None => Err(serde::de::Error::custom(\n                        \"expected a `suffix` key in the `file-types` entry\",\n                    )),\n                }\n            }\n        }\n\n        deserializer.deserialize_any(FileTypeVisitor)\n    }\n}\n\nfn from_comment_tokens<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    #[derive(Deserialize)]\n    #[serde(untagged)]\n    enum CommentTokens {\n        Multiple(Vec<String>),\n        Single(String),\n    }\n    Ok(\n        Option::<CommentTokens>::deserialize(deserializer)?.map(|tokens| match tokens {\n            CommentTokens::Single(val) => vec![val],\n            CommentTokens::Multiple(vals) => vals,\n        }),\n    )\n}\n\n#[derive(Clone, Debug, Serialize, Deserialize)]\npub struct BlockCommentToken {\n    pub start: String,\n    pub end: String,\n}\n\nimpl Default for BlockCommentToken {\n    fn default() -> Self {\n        BlockCommentToken {\n            start: \"/*\".to_string(),\n            end: \"*/\".to_string(),\n        }\n    }\n}\n\nfn from_block_comment_tokens<'de, D>(\n    deserializer: D,\n) -> Result<Option<Vec<BlockCommentToken>>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    #[derive(Deserialize)]\n    #[serde(untagged)]\n    enum BlockCommentTokens {\n        Multiple(Vec<BlockCommentToken>),\n        Single(BlockCommentToken),\n    }\n    Ok(\n        Option::<BlockCommentTokens>::deserialize(deserializer)?.map(|tokens| match tokens {\n            BlockCommentTokens::Single(val) => vec![val],\n            BlockCommentTokens::Multiple(vals) => vals,\n        }),\n    )\n}\n\n#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]\n#[serde(rename_all = \"kebab-case\")]\npub enum LanguageServerFeature {\n    Format,\n    GotoDeclaration,\n    GotoDefinition,\n    GotoTypeDefinition,\n    GotoReference,\n    GotoImplementation,\n    // Goto, use bitflags, combining previous Goto members?\n    SignatureHelp,\n    Hover,\n    DocumentHighlight,\n    Completion,\n    CodeAction,\n    WorkspaceCommand,\n    DocumentSymbols,\n    WorkspaceSymbols,\n    // Symbols, use bitflags, see above?\n    Diagnostics,\n    RenameSymbol,\n    InlayHints,\n}\n\nimpl Display for LanguageServerFeature {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        use LanguageServerFeature::*;\n        let feature = match self {\n            Format => \"format\",\n            GotoDeclaration => \"goto-declaration\",\n            GotoDefinition => \"goto-definition\",\n            GotoTypeDefinition => \"goto-type-definition\",\n            GotoReference => \"goto-reference\",\n            GotoImplementation => \"goto-implementation\",\n            SignatureHelp => \"signature-help\",\n            Hover => \"hover\",\n            DocumentHighlight => \"document-highlight\",\n            Completion => \"completion\",\n            CodeAction => \"code-action\",\n            WorkspaceCommand => \"workspace-command\",\n            DocumentSymbols => \"document-symbols\",\n            WorkspaceSymbols => \"workspace-symbols\",\n            Diagnostics => \"diagnostics\",\n            RenameSymbol => \"rename-symbol\",\n            InlayHints => \"inlay-hints\",\n        };\n        write!(f, \"{feature}\",)\n    }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(untagged, rename_all = \"kebab-case\", deny_unknown_fields)]\nenum LanguageServerFeatureConfiguration {\n    #[serde(rename_all = \"kebab-case\")]\n    Features {\n        #[serde(default, skip_serializing_if = \"HashSet::is_empty\")]\n        only_features: HashSet<LanguageServerFeature>,\n        #[serde(default, skip_serializing_if = \"HashSet::is_empty\")]\n        except_features: HashSet<LanguageServerFeature>,\n        name: String,\n    },\n    Simple(String),\n}\n\n#[derive(Debug, Default)]\npub struct LanguageServerFeatures {\n    pub name: String,\n    pub only: HashSet<LanguageServerFeature>,\n    pub excluded: HashSet<LanguageServerFeature>,\n}\n\nimpl LanguageServerFeatures {\n    pub fn has_feature(&self, feature: LanguageServerFeature) -> bool {\n        (self.only.is_empty() || self.only.contains(&feature)) && !self.excluded.contains(&feature)\n    }\n}\n\nfn deserialize_lang_features<'de, D>(\n    deserializer: D,\n) -> Result<Vec<LanguageServerFeatures>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    let raw: Vec<LanguageServerFeatureConfiguration> = Deserialize::deserialize(deserializer)?;\n    let res = raw\n        .into_iter()\n        .map(|config| match config {\n            LanguageServerFeatureConfiguration::Simple(name) => LanguageServerFeatures {\n                name,\n                ..Default::default()\n            },\n            LanguageServerFeatureConfiguration::Features {\n                only_features,\n                except_features,\n                name,\n            } => LanguageServerFeatures {\n                name,\n                only: only_features,\n                excluded: except_features,\n            },\n        })\n        .collect();\n    Ok(res)\n}\n\nfn serialize_lang_features<S>(\n    map: &Vec<LanguageServerFeatures>,\n    serializer: S,\n) -> Result<S::Ok, S::Error>\nwhere\n    S: serde::Serializer,\n{\n    let mut serializer = serializer.serialize_seq(Some(map.len()))?;\n    for features in map {\n        let features = if features.only.is_empty() && features.excluded.is_empty() {\n            LanguageServerFeatureConfiguration::Simple(features.name.to_owned())\n        } else {\n            LanguageServerFeatureConfiguration::Features {\n                only_features: features.only.clone(),\n                except_features: features.excluded.clone(),\n                name: features.name.to_owned(),\n            }\n        };\n        serializer.serialize_element(&features)?;\n    }\n    serializer.end()\n}\n\nfn deserialize_required_root_patterns<'de, D>(deserializer: D) -> Result<Option<GlobSet>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    let patterns = Vec::<String>::deserialize(deserializer)?;\n    if patterns.is_empty() {\n        return Ok(None);\n    }\n    let mut builder = globset::GlobSetBuilder::new();\n    for pattern in patterns {\n        let glob = globset::Glob::new(&pattern).map_err(serde::de::Error::custom)?;\n        builder.add(glob);\n    }\n    builder.build().map(Some).map_err(serde::de::Error::custom)\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct LanguageServerConfiguration {\n    pub command: String,\n    #[serde(default)]\n    #[serde(skip_serializing_if = \"Vec::is_empty\")]\n    pub args: Vec<String>,\n    #[serde(default, skip_serializing_if = \"HashMap::is_empty\")]\n    pub environment: HashMap<String, String>,\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_lsp_config\")]\n    pub config: Option<serde_json::Value>,\n    #[serde(default = \"default_timeout\")]\n    pub timeout: u64,\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"deserialize_required_root_patterns\"\n    )]\n    pub required_root_patterns: Option<GlobSet>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct FormatterConfiguration {\n    pub command: String,\n    #[serde(default)]\n    #[serde(skip_serializing_if = \"Vec::is_empty\")]\n    pub args: Vec<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct AdvancedCompletion {\n    pub name: Option<String>,\n    pub completion: Option<String>,\n    pub default: Option<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\", untagged)]\npub enum DebugConfigCompletion {\n    Named(String),\n    Advanced(AdvancedCompletion),\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(untagged)]\npub enum DebugArgumentValue {\n    String(String),\n    Array(Vec<String>),\n    Boolean(bool),\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct DebugTemplate {\n    pub name: String,\n    pub request: String,\n    pub completion: Vec<DebugConfigCompletion>,\n    pub args: HashMap<String, DebugArgumentValue>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct DebugAdapterConfig {\n    pub name: String,\n    pub transport: String,\n    #[serde(default)]\n    pub command: String,\n    #[serde(default)]\n    pub args: Vec<String>,\n    pub port_arg: Option<String>,\n    pub templates: Vec<DebugTemplate>,\n    #[serde(default)]\n    pub quirks: DebuggerQuirks,\n}\n\n// Different workarounds for adapters' differences\n#[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize)]\npub struct DebuggerQuirks {\n    #[serde(default)]\n    pub absolute_paths: bool,\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct IndentationConfiguration {\n    #[serde(deserialize_with = \"deserialize_tab_width\")]\n    pub tab_width: usize,\n    pub unit: String,\n}\n\n/// How the indentation for a newly inserted line should be determined.\n/// If the selected heuristic is not available (e.g. because the current\n/// language has no tree-sitter indent queries), a simpler one will be used.\n#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub enum IndentationHeuristic {\n    /// Just copy the indentation of the line that the cursor is currently on.\n    Simple,\n    /// Use tree-sitter indent queries to compute the expected absolute indentation level of the new line.\n    TreeSitter,\n    /// Use tree-sitter indent queries to compute the expected difference in indentation between the new line\n    /// and the line before. Add this to the actual indentation level of the line before.\n    #[default]\n    Hybrid,\n}\n\n/// Configuration for auto pairs\n#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\", deny_unknown_fields, untagged)]\npub enum AutoPairConfig {\n    /// Enables or disables auto pairing. False means disabled. True means to use the default pairs.\n    Enable(bool),\n\n    /// The mappings of pairs.\n    Pairs(HashMap<char, char>),\n}\n\nimpl Default for AutoPairConfig {\n    fn default() -> Self {\n        AutoPairConfig::Enable(true)\n    }\n}\n\nimpl From<&AutoPairConfig> for Option<AutoPairs> {\n    fn from(auto_pair_config: &AutoPairConfig) -> Self {\n        match auto_pair_config {\n            AutoPairConfig::Enable(false) => None,\n            AutoPairConfig::Enable(true) => Some(AutoPairs::default()),\n            AutoPairConfig::Pairs(pairs) => Some(AutoPairs::new(pairs.iter())),\n        }\n    }\n}\n\nimpl From<AutoPairConfig> for Option<AutoPairs> {\n    fn from(auto_pairs_config: AutoPairConfig) -> Self {\n        (&auto_pairs_config).into()\n    }\n}\n\nimpl FromStr for AutoPairConfig {\n    type Err = std::str::ParseBoolError;\n\n    // only do bool parsing for runtime setting\n    fn from_str(s: &str) -> Result<Self, Self::Err> {\n        let enable: bool = s.parse()?;\n        Ok(AutoPairConfig::Enable(enable))\n    }\n}\n\n#[derive(Debug)]\npub struct TextObjectQuery {\n    pub query: Query,\n}\n\n#[derive(Debug)]\npub enum CapturedNode<'a> {\n    Single(Node<'a>),\n    /// Guaranteed to be not empty\n    Grouped(Vec<Node<'a>>),\n}\n\nimpl<'a> CapturedNode<'a> {\n    pub fn start_byte(&self) -> usize {\n        match self {\n            Self::Single(n) => n.start_byte(),\n            Self::Grouped(ns) => ns[0].start_byte(),\n        }\n    }\n\n    pub fn end_byte(&self) -> usize {\n        match self {\n            Self::Single(n) => n.end_byte(),\n            Self::Grouped(ns) => ns.last().unwrap().end_byte(),\n        }\n    }\n\n    pub fn byte_range(&self) -> std::ops::Range<usize> {\n        self.start_byte()..self.end_byte()\n    }\n}\n\n/// The maximum number of in-progress matches a TS cursor can consider at once.\n/// This is set to a constant in order to avoid performance problems for medium to large files. Set with `set_match_limit`.\n/// Using such a limit means that we lose valid captures, so there is fundamentally a tradeoff here.\n///\n///\n/// Old tree sitter versions used a limit of 32 by default until this limit was removed in version `0.19.5` (must now be set manually).\n/// However, this causes performance issues for medium to large files.\n/// In helix, this problem caused treesitter motions to take multiple seconds to complete in medium-sized rust files (3k loc).\n///\n///\n/// Neovim also encountered this problem and reintroduced this limit after it was removed upstream\n/// (see <https://github.com/neovim/neovim/issues/14897> and <https://github.com/neovim/neovim/pull/14915>).\n/// The number used here is fundamentally a tradeoff between breaking some obscure edge cases and performance.\n///\n///\n/// Neovim chose 64 for this value somewhat arbitrarily (<https://github.com/neovim/neovim/pull/18397>).\n/// 64 is too low for some languages though. In particular, it breaks some highlighting for record fields in Erlang record definitions.\n/// This number can be increased if new syntax highlight breakages are found, as long as the performance penalty is not too high.\nconst TREE_SITTER_MATCH_LIMIT: u32 = 256;\n\nimpl TextObjectQuery {\n    /// Run the query on the given node and return sub nodes which match given\n    /// capture (\"function.inside\", \"class.around\", etc).\n    ///\n    /// Captures may contain multiple nodes by using quantifiers (+, *, etc),\n    /// and support for this is partial and could use improvement.\n    ///\n    /// ```query\n    /// (comment)+ @capture\n    ///\n    /// ; OR\n    /// (\n    ///   (comment)*\n    ///   .\n    ///   (function)\n    /// ) @capture\n    /// ```\n    pub fn capture_nodes<'a>(\n        &'a self,\n        capture_name: &str,\n        node: Node<'a>,\n        slice: RopeSlice<'a>,\n        cursor: &'a mut QueryCursor,\n    ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {\n        self.capture_nodes_any(&[capture_name], node, slice, cursor)\n    }\n\n    /// Find the first capture that exists out of all given `capture_names`\n    /// and return sub nodes that match this capture.\n    pub fn capture_nodes_any<'a>(\n        &'a self,\n        capture_names: &[&str],\n        node: Node<'a>,\n        slice: RopeSlice<'a>,\n        cursor: &'a mut QueryCursor,\n    ) -> Option<impl Iterator<Item = CapturedNode<'a>>> {\n        let capture_idx = capture_names\n            .iter()\n            .find_map(|cap| self.query.capture_index_for_name(cap))?;\n\n        cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT);\n\n        let nodes = cursor\n            .captures(&self.query, node, RopeProvider(slice))\n            .filter_map(move |(mat, _)| {\n                let nodes: Vec<_> = mat\n                    .captures\n                    .iter()\n                    .filter_map(|cap| (cap.index == capture_idx).then_some(cap.node))\n                    .collect();\n\n                if nodes.len() > 1 {\n                    Some(CapturedNode::Grouped(nodes))\n                } else {\n                    nodes.into_iter().map(CapturedNode::Single).next()\n                }\n            });\n\n        Some(nodes)\n    }\n}\n\npub fn read_query(language: &str, filename: &str) -> String {\n    static INHERITS_REGEX: Lazy<Regex> =\n        Lazy::new(|| Regex::new(r\";+\\s*inherits\\s*:?\\s*([a-z_,()-]+)\\s*\").unwrap());\n\n    let query = load_runtime_file(language, filename).unwrap_or_default();\n\n    // replaces all \"; inherits <language>(,<language>)*\" with the queries of the given language(s)\n    INHERITS_REGEX\n        .replace_all(&query, |captures: &regex::Captures| {\n            captures[1]\n                .split(',')\n                .map(|language| format!(\"\\n{}\\n\", read_query(language, filename)))\n                .collect::<String>()\n        })\n        .to_string()\n}\n\nimpl LanguageConfiguration {\n    fn initialize_highlight(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {\n        let highlights_query = read_query(&self.language_id, \"highlights.scm\");\n        // always highlight syntax errors\n        // highlights_query += \"\\n(ERROR) @error\";\n\n        let injections_query = read_query(&self.language_id, \"injections.scm\");\n        let locals_query = read_query(&self.language_id, \"locals.scm\");\n\n        if highlights_query.is_empty() {\n            None\n        } else {\n            let language = get_language(self.grammar.as_deref().unwrap_or(&self.language_id))\n                .map_err(|err| {\n                    log::error!(\n                        \"Failed to load tree-sitter parser for language {:?}: {}\",\n                        self.language_id,\n                        err\n                    )\n                })\n                .ok()?;\n            let config = HighlightConfiguration::new(\n                language,\n                &highlights_query,\n                &injections_query,\n                &locals_query,\n            )\n            .map_err(|err| log::error!(\"Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --grammar fetch' and 'hx --grammar build'. This query could not be parsed: {:?}\", self.language_id, err))\n            .ok()?;\n\n            config.configure(scopes);\n            Some(Arc::new(config))\n        }\n    }\n\n    pub fn reconfigure(&self, scopes: &[String]) {\n        if let Some(Some(config)) = self.highlight_config.get() {\n            config.configure(scopes);\n        }\n    }\n\n    pub fn highlight_config(&self, scopes: &[String]) -> Option<Arc<HighlightConfiguration>> {\n        self.highlight_config\n            .get_or_init(|| self.initialize_highlight(scopes))\n            .clone()\n    }\n\n    pub fn is_highlight_initialized(&self) -> bool {\n        self.highlight_config.get().is_some()\n    }\n\n    pub fn indent_query(&self) -> Option<&Query> {\n        self.indent_query\n            .get_or_init(|| self.load_query(\"indents.scm\"))\n            .as_ref()\n    }\n\n    pub fn textobject_query(&self) -> Option<&TextObjectQuery> {\n        self.textobject_query\n            .get_or_init(|| {\n                self.load_query(\"textobjects.scm\")\n                    .map(|query| TextObjectQuery { query })\n            })\n            .as_ref()\n    }\n\n    pub fn scope(&self) -> &str {\n        &self.scope\n    }\n\n    fn load_query(&self, kind: &str) -> Option<Query> {\n        let query_text = read_query(&self.language_id, kind);\n        if query_text.is_empty() {\n            return None;\n        }\n        let lang = &self.highlight_config.get()?.as_ref()?.language;\n        Query::new(lang, &query_text)\n            .map_err(|e| {\n                log::error!(\n                    \"Failed to parse {} queries for {}: {}\",\n                    kind,\n                    self.language_id,\n                    e\n                )\n            })\n            .ok()\n    }\n}\n#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(default, rename_all = \"kebab-case\", deny_unknown_fields)]\npub struct SoftWrap {\n    /// Soft wrap lines that exceed viewport width. Default to off\n    // NOTE: Option on purpose because the struct is shared between language config and global config.\n    // By default the option is None so that the language config falls back to the global config unless explicitly set.\n    pub enable: Option<bool>,\n    /// Maximum space left free at the end of the line.\n    /// This space is used to wrap text at word boundaries. If that is not possible within this limit\n    /// the word is simply split at the end of the line.\n    ///\n    /// This is automatically hard-limited to a quarter of the viewport to ensure correct display on small views.\n    ///\n    /// Default to 20\n    pub max_wrap: Option<u16>,\n    /// Maximum number of indentation that can be carried over from the previous line when softwrapping.\n    /// If a line is indented further then this limit it is rendered at the start of the viewport instead.\n    ///\n    /// This is automatically hard-limited to a quarter of the viewport to ensure correct display on small views.\n    ///\n    /// Default to 40\n    pub max_indent_retain: Option<u16>,\n    /// Indicator placed at the beginning of softwrapped lines\n    ///\n    /// Defaults to \u21aa\n    pub wrap_indicator: Option<String>,\n    /// Softwrap at `text_width` instead of viewport width if it is shorter\n    pub wrap_at_text_width: Option<bool>,\n}\n\n#[derive(Debug)]\nstruct FileTypeGlob {\n    glob: globset::Glob,\n    language_id: usize,\n}\n\nimpl FileTypeGlob {\n    fn new(glob: globset::Glob, language_id: usize) -> Self {\n        Self { glob, language_id }\n    }\n}\n\n#[derive(Debug)]\nstruct FileTypeGlobMatcher {\n    matcher: globset::GlobSet,\n    file_types: Vec<FileTypeGlob>,\n}\n\nimpl FileTypeGlobMatcher {\n    fn new(file_types: Vec<FileTypeGlob>) -> Result<Self, globset::Error> {\n        let mut builder = globset::GlobSetBuilder::new();\n        for file_type in &file_types {\n            builder.add(file_type.glob.clone());\n        }\n\n        Ok(Self {\n            matcher: builder.build()?,\n            file_types,\n        })\n    }\n\n    fn language_id_for_path(&self, path: &Path) -> Option<&usize> {\n        self.matcher\n            .matches(path)\n            .iter()\n            .filter_map(|idx| self.file_types.get(*idx))\n            .max_by_key(|file_type| file_type.glob.glob().len())\n            .map(|file_type| &file_type.language_id)\n    }\n}\n\n// Expose loader as Lazy<> global since it's always static?\n\n#[derive(Debug)]\npub struct Loader {\n    // highlight_names ?\n    language_configs: Vec<Arc<LanguageConfiguration>>,\n    language_config_ids_by_extension: HashMap<String, usize>, // Vec<usize>\n    language_config_ids_glob_matcher: FileTypeGlobMatcher,\n    language_config_ids_by_shebang: HashMap<String, usize>,\n\n    language_server_configs: HashMap<String, LanguageServerConfiguration>,\n\n    scopes: ArcSwap<Vec<String>>,\n}\n\npub type LoaderError = globset::Error;\n\nimpl Loader {\n    pub fn new(config: Configuration) -> Result<Self, LoaderError> {\n        let mut language_configs = Vec::new();\n        let mut language_config_ids_by_extension = HashMap::new();\n        let mut language_config_ids_by_shebang = HashMap::new();\n        let mut file_type_globs = Vec::new();\n\n        for config in config.language {\n            // get the next id\n            let language_id = language_configs.len();\n\n            for file_type in &config.file_types {\n                // entry().or_insert(Vec::new).push(language_id);\n                match file_type {\n                    FileType::Extension(extension) => {\n                        language_config_ids_by_extension.insert(extension.clone(), language_id);\n                    }\n                    FileType::Glob(glob) => {\n                        file_type_globs.push(FileTypeGlob::new(glob.to_owned(), language_id));\n                    }\n                };\n            }\n            for shebang in &config.shebangs {\n                language_config_ids_by_shebang.insert(shebang.clone(), language_id);\n            }\n\n            language_configs.push(Arc::new(config));\n        }\n\n        Ok(Self {\n            language_configs,\n            language_config_ids_by_extension,\n            language_config_ids_glob_matcher: FileTypeGlobMatcher::new(file_type_globs)?,\n            language_config_ids_by_shebang,\n            language_server_configs: config.language_server,\n            scopes: ArcSwap::from_pointee(Vec::new()),\n        })\n    }\n\n    pub fn language_config_for_file_name(&self, path: &Path) -> Option<Arc<LanguageConfiguration>> {\n        // Find all the language configurations that match this file name\n        // or a suffix of the file name.\n        let configuration_id = self\n            .language_config_ids_glob_matcher\n            .language_id_for_path(path)\n            .or_else(|| {\n                path.extension()\n                    .and_then(|extension| extension.to_str())\n                    .and_then(|extension| self.language_config_ids_by_extension.get(extension))\n            });\n\n        configuration_id.and_then(|&id| self.language_configs.get(id).cloned())\n\n        // TODO: content_regex handling conflict resolution\n    }\n\n    pub fn language_config_for_shebang(\n        &self,\n        source: RopeSlice,\n    ) -> Option<Arc<LanguageConfiguration>> {\n        let line = Cow::from(source.line(0));\n        static SHEBANG_REGEX: Lazy<Regex> =\n            Lazy::new(|| Regex::new(&[\"^\", SHEBANG].concat()).unwrap());\n        let configuration_id = SHEBANG_REGEX\n            .captures(&line)\n            .and_then(|cap| self.language_config_ids_by_shebang.get(&cap[1]));\n\n        configuration_id.and_then(|&id| self.language_configs.get(id).cloned())\n    }\n\n    pub fn language_config_for_scope(&self, scope: &str) -> Option<Arc<LanguageConfiguration>> {\n        self.language_configs\n            .iter()\n            .find(|config| config.scope == scope)\n            .cloned()\n    }\n\n    pub fn language_config_for_language_id(&self, id: &str) -> Option<Arc<LanguageConfiguration>> {\n        self.language_configs\n            .iter()\n            .find(|config| config.language_id == id)\n            .cloned()\n    }\n\n    /// Unlike language_config_for_language_id, which only returns Some for an exact id, this\n    /// function will perform a regex match on the given string to find the closest language match.\n    pub fn language_config_for_name(&self, name: &str) -> Option<Arc<LanguageConfiguration>> {\n        let mut best_match_length = 0;\n        let mut best_match_position = None;\n        for (i, configuration) in self.language_configs.iter().enumerate() {\n            if let Some(injection_regex) = &configuration.injection_regex {\n                if let Some(mat) = injection_regex.find(name) {\n                    let length = mat.end() - mat.start();\n                    if length > best_match_length {\n                        best_match_position = Some(i);\n                        best_match_length = length;\n                    }\n                }\n            }\n        }\n\n        best_match_position.map(|i| self.language_configs[i].clone())\n    }\n\n    pub fn language_configuration_for_injection_string(\n        &self,\n        capture: &InjectionLanguageMarker,\n    ) -> Option<Arc<LanguageConfiguration>> {\n        match capture {\n            InjectionLanguageMarker::Name(string) => self.language_config_for_name(string),\n            InjectionLanguageMarker::Filename(file) => self.language_config_for_file_name(file),\n            InjectionLanguageMarker::Shebang(shebang) => {\n                self.language_config_for_language_id(shebang)\n            }\n        }\n    }\n\n    pub fn language_configs(&self) -> impl Iterator<Item = &Arc<LanguageConfiguration>> {\n        self.language_configs.iter()\n    }\n\n    pub fn language_server_configs(&self) -> &HashMap<String, LanguageServerConfiguration> {\n        &self.language_server_configs\n    }\n\n    pub fn set_scopes(&self, scopes: Vec<String>) {\n        self.scopes.store(Arc::new(scopes));\n\n        // Reconfigure existing grammars\n        for config in self\n            .language_configs\n            .iter()\n            .filter(|cfg| cfg.is_highlight_initialized())\n        {\n            config.reconfigure(&self.scopes());\n        }\n    }\n\n    pub fn scopes(&self) -> Guard<Arc<Vec<String>>> {\n        self.scopes.load()\n    }\n}\n\npub struct TsParser {\n    parser: tree_sitter::Parser,\n    pub cursors: Vec<QueryCursor>,\n}\n\n// could also just use a pool, or a single instance?\nthread_local! {\n    pub static PARSER: RefCell<TsParser> = RefCell::new(TsParser {\n        parser: Parser::new(),\n        cursors: Vec::new(),\n    })\n}\n\n#[derive(Debug)]\npub struct Syntax {\n    layers: HopSlotMap<LayerId, LanguageLayer>,\n    root: LayerId,\n    loader: Arc<ArcSwap<Loader>>,\n}\n\nfn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> {\n    Cow::from(source.byte_slice(range))\n}\n\nimpl Syntax {\n    pub fn new(\n        source: RopeSlice,\n        config: Arc<HighlightConfiguration>,\n        loader: Arc<ArcSwap<Loader>>,\n    ) -> Option<Self> {\n        let root_layer = LanguageLayer {\n            tree: None,\n            config,\n            depth: 0,\n            flags: LayerUpdateFlags::empty(),\n            ranges: vec![Range {\n                start_byte: 0,\n                end_byte: usize::MAX,\n                start_point: Point::new(0, 0),\n                end_point: Point::new(usize::MAX, usize::MAX),\n            }],\n            parent: None,\n        };\n\n        // track scope_descriptor: a Vec of scopes for item in tree\n\n        let mut layers = HopSlotMap::default();\n        let root = layers.insert(root_layer);\n\n        let mut syntax = Self {\n            root,\n            layers,\n            loader,\n        };\n\n        let res = syntax.update(source, source, &ChangeSet::new(source));\n\n        if res.is_err() {\n            log::error!(\"TS parser failed, disabling TS for the current buffer: {res:?}\");\n            return None;\n        }\n        Some(syntax)\n    }\n\n    pub fn update(\n        &mut self,\n        old_source: RopeSlice,\n        source: RopeSlice,\n        changeset: &ChangeSet,\n    ) -> Result<(), Error> {\n        let mut queue = VecDeque::new();\n        queue.push_back(self.root);\n\n        let loader = self.loader.load();\n        let scopes = loader.scopes.load();\n        let injection_callback = |language: &InjectionLanguageMarker| {\n            loader\n                .language_configuration_for_injection_string(language)\n                .and_then(|language_config| language_config.highlight_config(&scopes))\n        };\n\n        // Convert the changeset into tree sitter edits.\n        let edits = generate_edits(old_source, changeset);\n\n        // This table allows inverse indexing of `layers`.\n        // That is by hashing a `Layer` you can find\n        // the `LayerId` of an existing equivalent `Layer` in `layers`.\n        //\n        // It is used to determine if a new layer exists for an injection\n        // or if an existing layer needs to be updated.\n        let mut layers_table = RawTable::with_capacity(self.layers.len());\n        let layers_hasher = RandomState::new();\n        // Use the edits to update all layers markers\n        fn point_add(a: Point, b: Point) -> Point {\n            if b.row > 0 {\n                Point::new(a.row.saturating_add(b.row), b.column)\n            } else {\n                Point::new(0, a.column.saturating_add(b.column))\n            }\n        }\n        fn point_sub(a: Point, b: Point) -> Point {\n            if a.row > b.row {\n                Point::new(a.row.saturating_sub(b.row), a.column)\n            } else {\n                Point::new(0, a.column.saturating_sub(b.column))\n            }\n        }\n\n        for (layer_id, layer) in self.layers.iter_mut() {\n            // The root layer always covers the whole range (0..usize::MAX)\n            if layer.depth == 0 {\n                layer.flags = LayerUpdateFlags::MODIFIED;\n                continue;\n            }\n\n            if !edits.is_empty() {\n                for range in &mut layer.ranges {\n                    // Roughly based on https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720\n                    for edit in edits.iter().rev() {\n                        let is_pure_insertion = edit.old_end_byte == edit.start_byte;\n\n                        // if edit is after range, skip\n                        if edit.start_byte > range.end_byte {\n                            // TODO: || (is_noop && edit.start_byte == range.end_byte)\n                            continue;\n                        }\n\n                        // if edit is before range, shift entire range by len\n                        if edit.old_end_byte < range.start_byte {\n                            range.start_byte =\n                                edit.new_end_byte + (range.start_byte - edit.old_end_byte);\n                            range.start_point = point_add(\n                                edit.new_end_position,\n                                point_sub(range.start_point, edit.old_end_position),\n                            );\n\n                            range.end_byte = edit\n                                .new_end_byte\n                                .saturating_add(range.end_byte - edit.old_end_byte);\n                            range.end_point = point_add(\n                                edit.new_end_position,\n                                point_sub(range.end_point, edit.old_end_position),\n                            );\n\n                            layer.flags |= LayerUpdateFlags::MOVED;\n                        }\n                        // if the edit starts in the space before and extends into the range\n                        else if edit.start_byte < range.start_byte {\n                            range.start_byte = edit.new_end_byte;\n                            range.start_point = edit.new_end_position;\n\n                            range.end_byte = range\n                                .end_byte\n                                .saturating_sub(edit.old_end_byte)\n                                .saturating_add(edit.new_end_byte);\n                            range.end_point = point_add(\n                                edit.new_end_position,\n                                point_sub(range.end_point, edit.old_end_position),\n                            );\n                            layer.flags = LayerUpdateFlags::MODIFIED;\n                        }\n                        // If the edit is an insertion at the start of the tree, shift\n                        else if edit.start_byte == range.start_byte && is_pure_insertion {\n                            range.start_byte = edit.new_end_byte;\n                            range.start_point = edit.new_end_position;\n                            layer.flags |= LayerUpdateFlags::MOVED;\n                        } else {\n                            range.end_byte = range\n                                .end_byte\n                                .saturating_sub(edit.old_end_byte)\n                                .saturating_add(edit.new_end_byte);\n                            range.end_point = point_add(\n                                edit.new_end_position,\n                                point_sub(range.end_point, edit.old_end_position),\n                            );\n                            layer.flags = LayerUpdateFlags::MODIFIED;\n                        }\n                    }\n                }\n            }\n\n            let hash = layers_hasher.hash_one(layer);\n            // Safety: insert_no_grow is unsafe because it assumes that the table\n            // has enough capacity to hold additional elements.\n            // This is always the case as we reserved enough capacity above.\n            unsafe { layers_table.insert_no_grow(hash, layer_id) };\n        }\n\n        PARSER.with(|ts_parser| {\n            let ts_parser = &mut ts_parser.borrow_mut();\n            ts_parser.parser.set_timeout_micros(1000 * 500); // half a second is pretty generours\n            let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);\n            // TODO: might need to set cursor range\n            cursor.set_byte_range(0..usize::MAX);\n            cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT);\n\n            let source_slice = source.slice(..);\n\n            while let Some(layer_id) = queue.pop_front() {\n                let layer = &mut self.layers[layer_id];\n\n                // Mark the layer as touched\n                layer.flags |= LayerUpdateFlags::TOUCHED;\n\n                // If a tree already exists, notify it of changes.\n                if let Some(tree) = &mut layer.tree {\n                    if layer\n                        .flags\n                        .intersects(LayerUpdateFlags::MODIFIED | LayerUpdateFlags::MOVED)\n                    {\n                        for edit in edits.iter().rev() {\n                            // Apply the edits in reverse.\n                            // If we applied them in order then edit 1 would disrupt the positioning of edit 2.\n                            tree.edit(edit);\n                        }\n                    }\n\n                    if layer.flags.contains(LayerUpdateFlags::MODIFIED) {\n                        // Re-parse the tree.\n                        layer.parse(&mut ts_parser.parser, source)?;\n                    }\n                } else {\n                    // always parse if this layer has never been parsed before\n                    layer.parse(&mut ts_parser.parser, source)?;\n                }\n\n                // Switch to an immutable borrow.\n                let layer = &self.layers[layer_id];\n\n                // Process injections.\n                let matches = cursor.matches(\n                    &layer.config.injections_query,\n                    layer.tree().root_node(),\n                    RopeProvider(source_slice),\n                );\n                let mut combined_injections = vec![\n                    (None, Vec::new(), IncludedChildren::default());\n                    layer.config.combined_injections_patterns.len()\n                ];\n                let mut injections = Vec::new();\n                let mut last_injection_end = 0;\n                for mat in matches {\n                    let (injection_capture, content_node, included_children) = layer\n                        .config\n                        .injection_for_match(&layer.config.injections_query, &mat, source_slice);\n\n                    // in case this is a combined injection save it for more processing later\n                    if let Some(combined_injection_idx) = layer\n                        .config\n                        .combined_injections_patterns\n                        .iter()\n                        .position(|&pattern| pattern == mat.pattern_index)\n                    {\n                        let entry = &mut combined_injections[combined_injection_idx];\n                        if injection_capture.is_some() {\n                            entry.0 = injection_capture;\n                        }\n                        if let Some(content_node) = content_node {\n                            if content_node.start_byte() >= last_injection_end {\n                                entry.1.push(content_node);\n                                last_injection_end = content_node.end_byte();\n                            }\n                        }\n                        entry.2 = included_children;\n                        continue;\n                    }\n\n                    // Explicitly remove this match so that none of its other captures will remain\n                    // in the stream of captures.\n                    mat.remove();\n\n                    // If a language is found with the given name, then add a new language layer\n                    // to the highlighted document.\n                    if let (Some(injection_capture), Some(content_node)) =\n                        (injection_capture, content_node)\n                    {\n                        if let Some(config) = (injection_callback)(&injection_capture) {\n                            let ranges =\n                                intersect_ranges(&layer.ranges, &[content_node], included_children);\n\n                            if !ranges.is_empty() {\n                                if content_node.start_byte() < last_injection_end {\n                                    continue;\n                                }\n                                last_injection_end = content_node.end_byte();\n                                injections.push((config, ranges));\n                            }\n                        }\n                    }\n                }\n\n                for (lang_name, content_nodes, included_children) in combined_injections {\n                    if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {\n                        if let Some(config) = (injection_callback)(&lang_name) {\n                            let ranges =\n                                intersect_ranges(&layer.ranges, &content_nodes, included_children);\n                            if !ranges.is_empty() {\n                                injections.push((config, ranges));\n                            }\n                        }\n                    }\n                }\n\n                let depth = layer.depth + 1;\n                // TODO: can't inline this since matches borrows self.layers\n                for (config, ranges) in injections {\n                    let new_layer = LanguageLayer {\n                        tree: None,\n                        config,\n                        depth,\n                        ranges,\n                        flags: LayerUpdateFlags::empty(),\n                        parent: Some(layer_id),\n                    };\n\n                    // Find an identical existing layer\n                    let layer = layers_table\n                        .get(layers_hasher.hash_one(&new_layer), |&it| {\n                            self.layers[it] == new_layer\n                        })\n                        .copied();\n\n                    // ...or insert a new one.\n                    let layer_id = layer.unwrap_or_else(|| self.layers.insert(new_layer));\n\n                    queue.push_back(layer_id);\n                }\n\n                // TODO: pre-process local scopes at this time, rather than highlight?\n                // would solve problems with locals not working across boundaries\n            }\n\n            // Return the cursor back in the pool.\n            ts_parser.cursors.push(cursor);\n\n            // Reset all `LayerUpdateFlags` and remove all untouched layers\n            self.layers.retain(|_, layer| {\n                replace(&mut layer.flags, LayerUpdateFlags::empty())\n                    .contains(LayerUpdateFlags::TOUCHED)\n            });\n\n            Ok(())\n        })\n    }\n\n    pub fn tree(&self) -> &Tree {\n        self.layers[self.root].tree()\n    }\n\n    /// Iterate over the highlighted regions for a given slice of source code.\n    pub fn highlight_iter<'a>(\n        &'a self,\n        source: RopeSlice<'a>,\n        range: Option<std::ops::Range<usize>>,\n        cancellation_flag: Option<&'a AtomicUsize>,\n    ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {\n        let mut layers = self\n            .layers\n            .iter()\n            .filter_map(|(_, layer)| {\n                // TODO: if range doesn't overlap layer range, skip it\n\n                // Reuse a cursor from the pool if available.\n                let mut cursor = PARSER.with(|ts_parser| {\n                    let highlighter = &mut ts_parser.borrow_mut();\n                    highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)\n                });\n\n                // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which\n                // prevents them from being moved. But both of these values are really just\n                // pointers, so it's actually ok to move them.\n                let cursor_ref =\n                    unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };\n\n                // if reusing cursors & no range this resets to whole range\n                cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));\n                cursor_ref.set_match_limit(TREE_SITTER_MATCH_LIMIT);\n\n                let mut captures = cursor_ref\n                    .captures(\n                        &layer.config.query,\n                        layer.tree().root_node(),\n                        RopeProvider(source),\n                    )\n                    .peekable();\n\n                // If there's no captures, skip the layer\n                captures.peek()?;\n\n                Some(HighlightIterLayer {\n                    highlight_end_stack: Vec::new(),\n                    scope_stack: vec![LocalScope {\n                        inherits: false,\n                        range: 0..usize::MAX,\n                        local_defs: Vec::new(),\n                    }],\n                    cursor,\n                    _tree: None,\n                    captures: RefCell::new(captures),\n                    config: layer.config.as_ref(), // TODO: just reuse `layer`\n                    depth: layer.depth,            // TODO: just reuse `layer`\n                })\n            })\n            .collect::<Vec<_>>();\n\n        layers.sort_unstable_by_key(|layer| layer.sort_key());\n\n        let mut result = HighlightIter {\n            source,\n            byte_offset: range.map_or(0, |r| r.start),\n            cancellation_flag,\n            iter_count: 0,\n            layers,\n            next_event: None,\n            last_highlight_range: None,\n        };\n        result.sort_layers();\n        result\n    }\n\n    pub fn tree_for_byte_range(&self, start: usize, end: usize) -> &Tree {\n        let mut container_id = self.root;\n\n        for (layer_id, layer) in self.layers.iter() {\n            if layer.depth > self.layers[container_id].depth\n                && layer.contains_byte_range(start, end)\n            {\n                container_id = layer_id;\n            }\n        }\n\n        self.layers[container_id].tree()\n    }\n\n    pub fn named_descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {\n        self.tree_for_byte_range(start, end)\n            .root_node()\n            .named_descendant_for_byte_range(start, end)\n    }\n\n    pub fn descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {\n        self.tree_for_byte_range(start, end)\n            .root_node()\n            .descendant_for_byte_range(start, end)\n    }\n\n    pub fn walk(&self) -> TreeCursor<'_> {\n        // data structure to find the smallest range that contains a point\n        // when some of the ranges in the structure can overlap.\n        TreeCursor::new(&self.layers, self.root)\n    }\n\n    // Commenting\n    // comment_strings_for_pos\n    // is_commented\n\n    // Indentation\n    // suggested_indent_for_line_at_buffer_row\n    // suggested_indent_for_buffer_row\n    // indent_level_for_line\n\n    // TODO: Folding\n}\n\nbitflags! {\n    /// Flags that track the status of a layer\n    /// in the `Sytaxn::update` function\n    #[derive(Debug)]\n    struct LayerUpdateFlags : u32{\n        const MODIFIED = 0b001;\n        const MOVED = 0b010;\n        const TOUCHED = 0b100;\n    }\n}\n\n#[derive(Debug)]\npub struct LanguageLayer {\n    // mode\n    // grammar\n    pub config: Arc<HighlightConfiguration>,\n    pub(crate) tree: Option<Tree>,\n    pub ranges: Vec<Range>,\n    pub depth: u32,\n    flags: LayerUpdateFlags,\n    parent: Option<LayerId>,\n}\n\n/// This PartialEq implementation only checks if that\n/// two layers are theoretically identical (meaning they highlight the same text range with the same language).\n/// It does not check whether the layers have the same internal treesitter\n/// state.\nimpl PartialEq for LanguageLayer {\n    fn eq(&self, other: &Self) -> bool {\n        self.depth == other.depth\n            && self.config.language == other.config.language\n            && self.ranges == other.ranges\n    }\n}\n\n/// Hash implementation belongs to PartialEq implementation above.\n/// See its documentation for details.\nimpl Hash for LanguageLayer {\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        self.depth.hash(state);\n        self.config.language.hash(state);\n        self.ranges.hash(state);\n    }\n}\n\nimpl LanguageLayer {\n    pub fn tree(&self) -> &Tree {\n        // TODO: no unwrap\n        self.tree.as_ref().unwrap()\n    }\n\n    fn parse(&mut self, parser: &mut Parser, source: RopeSlice) -> Result<(), Error> {\n        parser\n            .set_included_ranges(&self.ranges)\n            .map_err(|_| Error::InvalidRanges)?;\n\n        parser\n            .set_language(&self.config.language)\n            .map_err(|_| Error::InvalidLanguage)?;\n\n        // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };\n        let tree = parser\n            .parse_with(\n                &mut |byte, _| {\n                    if byte <= source.len_bytes() {\n                        let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);\n                        &chunk.as_bytes()[byte - start_byte..]\n                    } else {\n                        // out of range\n                        &[]\n                    }\n                },\n                self.tree.as_ref(),\n            )\n            .ok_or(Error::Cancelled)?;\n        // unsafe { ts_parser.parser.set_cancellation_flag(None) };\n...\n", "needle_token_start": 3949, "needle_token_end": 4158, "code_context_ntokens": 16226, "output": ["```rust\nfn serialize_lang_features<S>(\n    map: &Vec<LanguageServerFeatures>,\n    serializer: S,\n) -> Result<S::Ok, S::Error>\nwhere\n    S: serde::Serializer,\n{\n    let mut serializer = serializer.serialize_seq(Some(map.len()))?;\n    for features in map {\n        let features = if features.only.is_empty() && features.excluded.is_empty() {\n            LanguageServerFeatureConfiguration::Simple(features.name.to_owned())\n        } else {\n            LanguageServerFeatureConfiguration::Features {\n                only_features: features.only.clone(),\n                except_features: features.excluded.clone(),\n                name: features.name.to_owned(),\n            }\n        };\n        serializer.serialize_element(&features)?;\n    }\n    serializer.end()\n}\n```"]}
{"repo": "helix-editor/helix", "name": "injection_for_match", "language": "rust", "path": "helix-core/src/syntax.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the language injection details for a specific match within a source code document, including the type of language and whether certain child nodes should be included in the syntax highlighting.\n2. **Input**: The function takes three parameters: a query object, a match result from the query, and a slice of the source code.\n3. **Output**: It returns a tuple containing an optional language marker, an optional node representing the content for injection, and a setting that specifies which child nodes should be included.\n4. **Procedure**: \n   - First, it calls another function to get initial language and content node based on predefined capture indices.\n   - It then iterates over property settings from the query related to the match's pattern index to adjust the language marker and determine how child nodes should be handled based on specific properties like hard-coded language settings or inclusion rules for child nodes.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            && self.config.language == other.config.language\n            && self.ranges == other.ranges\n    }\n}\n\n/// Hash implementation belongs to PartialEq implementation above.\n/// See its documentation for details.\nimpl Hash for LanguageLayer {\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        self.depth.hash(state);\n        self.config.language.hash(state);\n        self.ranges.hash(state);\n    }\n}\n\nimpl LanguageLayer {\n    pub fn tree(&self) -> &Tree {\n        // TODO: no unwrap\n        self.tree.as_ref().unwrap()\n    }\n\n    fn parse(&mut self, parser: &mut Parser, source: RopeSlice) -> Result<(), Error> {\n        parser\n            .set_included_ranges(&self.ranges)\n            .map_err(|_| Error::InvalidRanges)?;\n\n        parser\n            .set_language(&self.config.language)\n            .map_err(|_| Error::InvalidLanguage)?;\n\n        // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };\n        let tree = parser\n            .parse_with(\n                &mut |byte, _| {\n                    if byte <= source.len_bytes() {\n                        let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);\n                        &chunk.as_bytes()[byte - start_byte..]\n                    } else {\n                        // out of range\n                        &[]\n                    }\n                },\n                self.tree.as_ref(),\n            )\n            .ok_or(Error::Cancelled)?;\n        // unsafe { ts_parser.parser.set_cancellation_flag(None) };\n        self.tree = Some(tree);\n        Ok(())\n    }\n\n    /// Whether the layer contains the given byte range.\n    ///\n    /// If the layer has multiple ranges (i.e. combined injections), the\n    /// given range is considered contained if it is within the start and\n    /// end bytes of the first and last ranges **and** if the given range\n    /// starts or ends within any of the layer's ranges.\n    fn contains_byte_range(&self, start: usize, end: usize) -> bool {\n        let layer_start = self\n            .ranges\n            .first()\n            .expect(\"ranges should not be empty\")\n            .start_byte;\n        let layer_end = self\n            .ranges\n            .last()\n            .expect(\"ranges should not be empty\")\n            .end_byte;\n\n        layer_start <= start\n            && layer_end >= end\n            && self.ranges.iter().any(|range| {\n                let byte_range = range.start_byte..range.end_byte;\n                byte_range.contains(&start) || byte_range.contains(&end)\n            })\n    }\n}\n\npub(crate) fn generate_edits(\n    old_text: RopeSlice,\n    changeset: &ChangeSet,\n) -> Vec<tree_sitter::InputEdit> {\n    use Operation::*;\n    let mut old_pos = 0;\n\n    let mut edits = Vec::new();\n\n    if changeset.changes.is_empty() {\n        return edits;\n    }\n\n    let mut iter = changeset.changes.iter().peekable();\n\n    // TODO; this is a lot easier with Change instead of Operation.\n\n    fn point_at_pos(text: RopeSlice, pos: usize) -> (usize, Point) {\n        let byte = text.char_to_byte(pos); // <- attempted to index past end\n        let line = text.char_to_line(pos);\n        let line_start_byte = text.line_to_byte(line);\n        let col = byte - line_start_byte;\n\n        (byte, Point::new(line, col))\n    }\n\n    fn traverse(point: Point, text: &Tendril) -> Point {\n        let Point {\n            mut row,\n            mut column,\n        } = point;\n\n        // TODO: there should be a better way here.\n        let mut chars = text.chars().peekable();\n        while let Some(ch) = chars.next() {\n            if char_is_line_ending(ch) && !(ch == '\\r' && chars.peek() == Some(&'\\n')) {\n                row += 1;\n                column = 0;\n            } else {\n                column += 1;\n            }\n        }\n        Point { row, column }\n    }\n\n    while let Some(change) = iter.next() {\n        let len = match change {\n            Delete(i) | Retain(i) => *i,\n            Insert(_) => 0,\n        };\n        let mut old_end = old_pos + len;\n\n        match change {\n            Retain(_) => {}\n            Delete(_) => {\n                let (start_byte, start_position) = point_at_pos(old_text, old_pos);\n                let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);\n\n                // deletion\n                edits.push(tree_sitter::InputEdit {\n                    start_byte,                       // old_pos to byte\n                    old_end_byte,                     // old_end to byte\n                    new_end_byte: start_byte,         // old_pos to byte\n                    start_position,                   // old pos to coords\n                    old_end_position,                 // old_end to coords\n                    new_end_position: start_position, // old pos to coords\n                });\n            }\n            Insert(s) => {\n                let (start_byte, start_position) = point_at_pos(old_text, old_pos);\n\n                // a subsequent delete means a replace, consume it\n                if let Some(Delete(len)) = iter.peek() {\n                    old_end = old_pos + len;\n                    let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);\n\n                    iter.next();\n\n                    // replacement\n                    edits.push(tree_sitter::InputEdit {\n                        start_byte,                                    // old_pos to byte\n                        old_end_byte,                                  // old_end to byte\n                        new_end_byte: start_byte + s.len(),            // old_pos to byte + s.len()\n                        start_position,                                // old pos to coords\n                        old_end_position,                              // old_end to coords\n                        new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)\n                    });\n                } else {\n                    // insert\n                    edits.push(tree_sitter::InputEdit {\n                        start_byte,                                    // old_pos to byte\n                        old_end_byte: start_byte,                      // same\n                        new_end_byte: start_byte + s.len(),            // old_pos + s.len()\n                        start_position,                                // old pos to coords\n                        old_end_position: start_position,              // same\n                        new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)\n                    });\n                }\n            }\n        }\n        old_pos = old_end;\n    }\n    edits\n}\n\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{iter, mem, ops, str, usize};\nuse tree_sitter::{\n    Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,\n    QueryMatch, Range, TextProvider, Tree,\n};\n\nconst CANCELLATION_CHECK_INTERVAL: usize = 100;\n\n/// Indicates which highlight should be applied to a region of source code.\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\npub struct Highlight(pub usize);\n\n/// Represents the reason why syntax highlighting failed.\n#[derive(Debug, PartialEq, Eq)]\npub enum Error {\n    Cancelled,\n    InvalidLanguage,\n    InvalidRanges,\n    Unknown,\n}\n\n/// Represents a single step in rendering a syntax-highlighted document.\n#[derive(Copy, Clone, Debug)]\npub enum HighlightEvent {\n    Source { start: usize, end: usize },\n    HighlightStart(Highlight),\n    HighlightEnd,\n}\n\n/// Contains the data needed to highlight code written in a particular language.\n///\n/// This struct is immutable and can be shared between threads.\n#[derive(Debug)]\npub struct HighlightConfiguration {\n    pub language: Grammar,\n    pub query: Query,\n    injections_query: Query,\n    combined_injections_patterns: Vec<usize>,\n    highlights_pattern_index: usize,\n    highlight_indices: ArcSwap<Vec<Option<Highlight>>>,\n    non_local_variable_patterns: Vec<bool>,\n    injection_content_capture_index: Option<u32>,\n    injection_language_capture_index: Option<u32>,\n    injection_filename_capture_index: Option<u32>,\n    injection_shebang_capture_index: Option<u32>,\n    local_scope_capture_index: Option<u32>,\n    local_def_capture_index: Option<u32>,\n    local_def_value_capture_index: Option<u32>,\n    local_ref_capture_index: Option<u32>,\n}\n\n#[derive(Debug)]\nstruct LocalDef<'a> {\n    name: Cow<'a, str>,\n    value_range: ops::Range<usize>,\n    highlight: Option<Highlight>,\n}\n\n#[derive(Debug)]\nstruct LocalScope<'a> {\n    inherits: bool,\n    range: ops::Range<usize>,\n    local_defs: Vec<LocalDef<'a>>,\n}\n\n#[derive(Debug)]\nstruct HighlightIter<'a> {\n    source: RopeSlice<'a>,\n    byte_offset: usize,\n    cancellation_flag: Option<&'a AtomicUsize>,\n    layers: Vec<HighlightIterLayer<'a>>,\n    iter_count: usize,\n    next_event: Option<HighlightEvent>,\n    last_highlight_range: Option<(usize, usize, u32)>,\n}\n\n// Adapter to convert rope chunks to bytes\npub struct ChunksBytes<'a> {\n    chunks: ropey::iter::Chunks<'a>,\n}\nimpl<'a> Iterator for ChunksBytes<'a> {\n    type Item = &'a [u8];\n    fn next(&mut self) -> Option<Self::Item> {\n        self.chunks.next().map(str::as_bytes)\n    }\n}\n\npub struct RopeProvider<'a>(pub RopeSlice<'a>);\nimpl<'a> TextProvider<&'a [u8]> for RopeProvider<'a> {\n    type I = ChunksBytes<'a>;\n\n    fn text(&mut self, node: Node) -> Self::I {\n        let fragment = self.0.byte_slice(node.start_byte()..node.end_byte());\n        ChunksBytes {\n            chunks: fragment.chunks(),\n        }\n    }\n}\n\nstruct HighlightIterLayer<'a> {\n    _tree: Option<Tree>,\n    cursor: QueryCursor,\n    captures: RefCell<iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>, &'a [u8]>>>,\n    config: &'a HighlightConfiguration,\n    highlight_end_stack: Vec<usize>,\n    scope_stack: Vec<LocalScope<'a>>,\n    depth: u32,\n}\n\nimpl<'a> fmt::Debug for HighlightIterLayer<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"HighlightIterLayer\").finish()\n    }\n}\n\nimpl HighlightConfiguration {\n    /// Creates a `HighlightConfiguration` for a given `Grammar` and set of highlighting\n    /// queries.\n    ///\n    /// # Parameters\n    ///\n    /// * `language`  - The Tree-sitter `Grammar` that should be used for parsing.\n    /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This\n    ///   should be non-empty, otherwise no syntax highlights will be added.\n    /// * `injections_query` -  A string containing tree patterns for injecting other languages\n    ///   into the document. This can be empty if no injections are desired.\n    /// * `locals_query` - A string containing tree patterns for tracking local variable\n    ///   definitions and references. This can be empty if local variable tracking is not needed.\n    ///\n    /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.\n    pub fn new(\n        language: Grammar,\n        highlights_query: &str,\n        injection_query: &str,\n        locals_query: &str,\n    ) -> Result<Self, QueryError> {\n        // Concatenate the query strings, keeping track of the start offset of each section.\n        let mut query_source = String::new();\n        query_source.push_str(locals_query);\n        let highlights_query_offset = query_source.len();\n        query_source.push_str(highlights_query);\n\n        // Construct a single query by concatenating the three query strings, but record the\n        // range of pattern indices that belong to each individual string.\n        let query = Query::new(&language, &query_source)?;\n        let mut highlights_pattern_index = 0;\n        for i in 0..(query.pattern_count()) {\n            let pattern_offset = query.start_byte_for_pattern(i);\n            if pattern_offset < highlights_query_offset {\n                highlights_pattern_index += 1;\n            }\n        }\n\n        let injections_query = Query::new(&language, injection_query)?;\n        let combined_injections_patterns = (0..injections_query.pattern_count())\n            .filter(|&i| {\n                injections_query\n                    .property_settings(i)\n                    .iter()\n                    .any(|s| &*s.key == \"injection.combined\")\n            })\n            .collect();\n\n        // Find all of the highlighting patterns that are disabled for nodes that\n        // have been identified as local variables.\n        let non_local_variable_patterns = (0..query.pattern_count())\n            .map(|i| {\n                query\n                    .property_predicates(i)\n                    .iter()\n                    .any(|(prop, positive)| !*positive && prop.key.as_ref() == \"local\")\n            })\n            .collect();\n\n        // Store the numeric ids for all of the special captures.\n        let mut injection_content_capture_index = None;\n        let mut injection_language_capture_index = None;\n        let mut injection_filename_capture_index = None;\n        let mut injection_shebang_capture_index = None;\n        let mut local_def_capture_index = None;\n        let mut local_def_value_capture_index = None;\n        let mut local_ref_capture_index = None;\n        let mut local_scope_capture_index = None;\n        for (i, name) in query.capture_names().iter().enumerate() {\n            let i = Some(i as u32);\n            match *name {\n                \"local.definition\" => local_def_capture_index = i,\n                \"local.definition-value\" => local_def_value_capture_index = i,\n                \"local.reference\" => local_ref_capture_index = i,\n                \"local.scope\" => local_scope_capture_index = i,\n                _ => {}\n            }\n        }\n\n        for (i, name) in injections_query.capture_names().iter().enumerate() {\n            let i = Some(i as u32);\n            match *name {\n                \"injection.content\" => injection_content_capture_index = i,\n                \"injection.language\" => injection_language_capture_index = i,\n                \"injection.filename\" => injection_filename_capture_index = i,\n                \"injection.shebang\" => injection_shebang_capture_index = i,\n                _ => {}\n            }\n        }\n\n        let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);\n        Ok(Self {\n            language,\n            query,\n            injections_query,\n            combined_injections_patterns,\n            highlights_pattern_index,\n            highlight_indices,\n            non_local_variable_patterns,\n            injection_content_capture_index,\n            injection_language_capture_index,\n            injection_filename_capture_index,\n            injection_shebang_capture_index,\n            local_scope_capture_index,\n            local_def_capture_index,\n            local_def_value_capture_index,\n            local_ref_capture_index,\n        })\n    }\n\n    /// Get a slice containing all of the highlight names used in the configuration.\n    pub fn names(&self) -> &[&str] {\n        self.query.capture_names()\n    }\n\n    /// Set the list of recognized highlight names.\n    ///\n    /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated\n    /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of\n    /// these queries can choose to recognize highlights with different levels of specificity.\n    /// For example, the string `function.builtin` will match against `function.builtin.constructor`\n    /// but will not match `function.method.builtin` and `function.method`.\n    ///\n    /// When highlighting, results are returned as `Highlight` values, which contain the index\n    /// of the matched highlight this list of highlight names.\n    pub fn configure(&self, recognized_names: &[String]) {\n        let mut capture_parts = Vec::new();\n        let indices: Vec<_> = self\n            .query\n            .capture_names()\n            .iter()\n            .map(move |capture_name| {\n                capture_parts.clear();\n                capture_parts.extend(capture_name.split('.'));\n\n                let mut best_index = None;\n                let mut best_match_len = 0;\n                for (i, recognized_name) in recognized_names.iter().enumerate() {\n                    let mut len = 0;\n                    let mut matches = true;\n                    for (i, part) in recognized_name.split('.').enumerate() {\n                        match capture_parts.get(i) {\n                            Some(capture_part) if *capture_part == part => len += 1,\n                            _ => {\n                                matches = false;\n                                break;\n                            }\n                        }\n                    }\n                    if matches && len > best_match_len {\n                        best_index = Some(i);\n                        best_match_len = len;\n                    }\n                }\n                best_index.map(Highlight)\n            })\n            .collect();\n\n        self.highlight_indices.store(Arc::new(indices));\n    }\n\n    fn injection_pair<'a>(\n        &self,\n        query_match: &QueryMatch<'a, 'a>,\n        source: RopeSlice<'a>,\n    ) -> (Option<InjectionLanguageMarker<'a>>, Option<Node<'a>>) {\n        let mut injection_capture = None;\n        let mut content_node = None;\n\n        for capture in query_match.captures {\n            let index = Some(capture.index);\n            if index == self.injection_language_capture_index {\n                let name = byte_range_to_str(capture.node.byte_range(), source);\n                injection_capture = Some(InjectionLanguageMarker::Name(name));\n            } else if index == self.injection_filename_capture_index {\n                let name = byte_range_to_str(capture.node.byte_range(), source);\n                let path = Path::new(name.as_ref()).to_path_buf();\n                injection_capture = Some(InjectionLanguageMarker::Filename(path.into()));\n            } else if index == self.injection_shebang_capture_index {\n                let node_slice = source.byte_slice(capture.node.byte_range());\n\n                // some languages allow space and newlines before the actual string content\n                // so a shebang could be on either the first or second line\n                let lines = if let Ok(end) = node_slice.try_line_to_byte(2) {\n                    node_slice.byte_slice(..end)\n                } else {\n                    node_slice\n                };\n\n                static SHEBANG_REGEX: Lazy<rope::Regex> =\n                    Lazy::new(|| rope::Regex::new(SHEBANG).unwrap());\n\n                injection_capture = SHEBANG_REGEX\n                    .captures_iter(lines.regex_input())\n                    .map(|cap| {\n                        let cap = lines.byte_slice(cap.get_group(1).unwrap().range());\n                        InjectionLanguageMarker::Shebang(cap.into())\n                    })\n                    .next()\n            } else if index == self.injection_content_capture_index {\n                content_node = Some(capture.node);\n            }\n        }\n        (injection_capture, content_node)\n    }\n\n    fn\n injection_for_match<'a>(\n        &self,\n        query: &'a Query,\n        query_match: &QueryMatch<'a, 'a>,\n        source: RopeSlice<'a>,\n    ) -> (\n        Option<InjectionLanguageMarker<'a>>,\n        Option<Node<'a>>,\n        IncludedChildren,\n    ) {\n        let (mut injection_capture, content_node) = self.injection_pair(query_match, source);\n\n        let mut included_children = IncludedChildren::default();\n        for prop in query.property_settings(query_match.pattern_index) {\n            match prop.key.as_ref() {\n                // In addition to specifying the language name via the text of a\n                // captured node, it can also be hard-coded via a `#set!` predicate\n                // that sets the injection.language key.\n                \"injection.language\" if injection_capture.is_none() => {\n                    injection_capture = prop\n                        .value\n                        .as_ref()\n                        .map(|s| InjectionLanguageMarker::Name(s.as_ref().into()));\n                }\n\n                // By default, injections do not include the *children* of an\n                // `injection.content` node - only the ranges that belong to the\n                // node itself. This can be changed using a `#set!` predicate that\n                // sets the `injection.include-children` key.\n                \"injection.include-children\" => included_children = IncludedChildren::All,\n\n                // Some queries might only exclude named children but include unnamed\n                // children in their `injection.content` node. This can be enabled using\n                // a `#set!` predicate that sets the `injection.include-unnamed-children` key.\n                \"injection.include-unnamed-children\" => {\n                    included_children = IncludedChildren::Unnamed\n                }\n                _ => {}\n            }\n        }\n\n        (injection_capture, content_node, included_children)\n    }\n}\n\nimpl<'a> HighlightIterLayer<'a> {\n    // First, sort scope boundaries by their byte offset in the document. At a\n    // given position, emit scope endings before scope beginnings. Finally, emit\n    // scope boundaries from deeper layers first.\n    fn sort_key(&self) -> Option<(usize, bool, isize)> {\n        let depth = -(self.depth as isize);\n        let next_start = self\n            .captures\n            .borrow_mut()\n            .peek()\n            .map(|(m, i)| m.captures[*i].node.start_byte());\n        let next_end = self.highlight_end_stack.last().cloned();\n        match (next_start, next_end) {\n            (Some(start), Some(end)) => {\n                if start < end {\n                    Some((start, true, depth))\n                } else {\n                    Some((end, false, depth))\n                }\n            }\n            (Some(i), None) => Some((i, true, depth)),\n            (None, Some(j)) => Some((j, false, depth)),\n            _ => None,\n        }\n    }\n}\n\n#[derive(Clone)]\nenum IncludedChildren {\n    None,\n    All,\n    Unnamed,\n}\n\nimpl Default for IncludedChildren {\n    fn default() -> Self {\n        Self::None\n    }\n}\n\n// Compute the ranges that should be included when parsing an injection.\n// This takes into account three things:\n// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.\n// * `nodes` - Every injection takes place within a set of nodes. The injection ranges\n//   are the ranges of those nodes.\n// * `includes_children` - For some injections, the content nodes' children should be\n//   excluded from the nested document, so that only the content nodes' *own* content\n//   is reparsed. For other injections, the content nodes' entire ranges should be\n//   reparsed, including the ranges of their children.\nfn intersect_ranges(\n    parent_ranges: &[Range],\n    nodes: &[Node],\n    included_children: IncludedChildren,\n) -> Vec<Range> {\n    let mut cursor = nodes[0].walk();\n    let mut result = Vec::new();\n    let mut parent_range_iter = parent_ranges.iter();\n    let mut parent_range = parent_range_iter\n        .next()\n        .expect(\"Layers should only be constructed with non-empty ranges vectors\");\n    for node in nodes.iter() {\n        let mut preceding_range = Range {\n            start_byte: 0,\n            start_point: Point::new(0, 0),\n            end_byte: node.start_byte(),\n            end_point: node.start_position(),\n        };\n        let following_range = Range {\n            start_byte: node.end_byte(),\n            start_point: node.end_position(),\n            end_byte: usize::MAX,\n            end_point: Point::new(usize::MAX, usize::MAX),\n        };\n\n        for excluded_range in node\n            .children(&mut cursor)\n            .filter_map(|child| match included_children {\n                IncludedChildren::None => Some(child.range()),\n                IncludedChildren::All => None,\n                IncludedChildren::Unnamed => {\n                    if child.is_named() {\n                        Some(child.range())\n                    } else {\n                        None\n                    }\n                }\n            })\n            .chain([following_range].iter().cloned())\n        {\n            let mut range = Range {\n                start_byte: preceding_range.end_byte,\n                start_point: preceding_range.end_point,\n                end_byte: excluded_range.start_byte,\n                end_point: excluded_range.start_point,\n            };\n            preceding_range = excluded_range;\n\n            if range.end_byte < parent_range.start_byte {\n                continue;\n            }\n\n            while parent_range.start_byte <= range.end_byte {\n                if parent_range.end_byte > range.start_byte {\n                    if range.start_byte < parent_range.start_byte {\n                        range.start_byte = parent_range.start_byte;\n                        range.start_point = parent_range.start_point;\n                    }\n\n                    if parent_range.end_byte < range.end_byte {\n                        if range.start_byte < parent_range.end_byte {\n                            result.push(Range {\n                                start_byte: range.start_byte,\n                                start_point: range.start_point,\n                                end_byte: parent_range.end_byte,\n                                end_point: parent_range.end_point,\n                            });\n                        }\n                        range.start_byte = parent_range.end_byte;\n                        range.start_point = parent_range.end_point;\n                    } else {\n                        if range.start_byte < range.end_byte {\n                            result.push(range);\n                        }\n                        break;\n                    }\n                }\n\n                if let Some(next_range) = parent_range_iter.next() {\n                    parent_range = next_range;\n                } else {\n                    return result;\n                }\n            }\n        }\n    }\n    result\n}\n\nimpl<'a> HighlightIter<'a> {\n    fn emit_event(\n        &mut self,\n        offset: usize,\n        event: Option<HighlightEvent>,\n    ) -> Option<Result<HighlightEvent, Error>> {\n        let result;\n        if self.byte_offset < offset {\n            result = Some(Ok(HighlightEvent::Source {\n                start: self.byte_offset,\n                end: offset,\n            }));\n            self.byte_offset = offset;\n            self.next_event = event;\n        } else {\n            result = event.map(Ok);\n        }\n        self.sort_layers();\n        result\n    }\n\n    fn sort_layers(&mut self) {\n        while !self.layers.is_empty() {\n            if let Some(sort_key) = self.layers[0].sort_key() {\n                let mut i = 0;\n                while i + 1 < self.layers.len() {\n                    if let Some(next_offset) = self.layers[i + 1].sort_key() {\n                        if next_offset < sort_key {\n                            i += 1;\n                            continue;\n                        }\n                    } else {\n                        let layer = self.layers.remove(i + 1);\n                        PARSER.with(|ts_parser| {\n                            let highlighter = &mut ts_parser.borrow_mut();\n                            highlighter.cursors.push(layer.cursor);\n                        });\n                    }\n                    break;\n                }\n                if i > 0 {\n                    self.layers[0..(i + 1)].rotate_left(1);\n                }\n                break;\n            } else {\n                let layer = self.layers.remove(0);\n                PARSER.with(|ts_parser| {\n                    let highlighter = &mut ts_parser.borrow_mut();\n                    highlighter.cursors.push(layer.cursor);\n                });\n            }\n        }\n    }\n}\n\nimpl<'a> Iterator for HighlightIter<'a> {\n    type Item = Result<HighlightEvent, Error>;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        'main: loop {\n            // If we've already determined the next highlight boundary, just return it.\n            if let Some(e) = self.next_event.take() {\n                return Some(Ok(e));\n            }\n\n            // Periodically check for cancellation, returning `Cancelled` error if the\n            // cancellation flag was flipped.\n            if let Some(cancellation_flag) = self.cancellation_flag {\n                self.iter_count += 1;\n                if self.iter_count >= CANCELLATION_CHECK_INTERVAL {\n                    self.iter_count = 0;\n                    if cancellation_flag.load(Ordering::Relaxed) != 0 {\n                        return Some(Err(Error::Cancelled));\n                    }\n                }\n            }\n\n            // If none of the layers have any more highlight boundaries, terminate.\n            if self.layers.is_empty() {\n                let len = self.source.len_bytes();\n                return if self.byte_offset < len {\n                    let result = Some(Ok(HighlightEvent::Source {\n                        start: self.byte_offset,\n                        end: len,\n                    }));\n                    self.byte_offset = len;\n                    result\n                } else {\n                    None\n                };\n            }\n\n            // Get the next capture from whichever layer has the earliest highlight boundary.\n            let range;\n            let layer = &mut self.layers[0];\n            let captures = layer.captures.get_mut();\n            if let Some((next_match, capture_index)) = captures.peek() {\n                let next_capture = next_match.captures[*capture_index];\n                range = next_capture.node.byte_range();\n\n                // If any previous highlight ends before this node starts, then before\n                // processing this capture, emit the source code up until the end of the\n                // previous highlight, and an end event for that highlight.\n                if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {\n                    if end_byte <= range.start {\n                        layer.highlight_end_stack.pop();\n                        return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));\n                    }\n                }\n            }\n            // If there are no more captures, then emit any remaining highlight end events.\n            // And if there are none of those, then just advance to the end of the document.\n            else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {\n                layer.highlight_end_stack.pop();\n                return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));\n            } else {\n                return self.emit_event(self.source.len_bytes(), None);\n            };\n\n            let (mut match_, capture_index) = captures.next().unwrap();\n            let mut capture = match_.captures[capture_index];\n\n            // Remove from the local scope stack any local scopes that have already ended.\n            while range.start > layer.scope_stack.last().unwrap().range.end {\n                layer.scope_stack.pop();\n            }\n\n            // If this capture is for tracking local variables, then process the\n            // local variable info.\n            let mut reference_highlight = None;\n            let mut definition_highlight = None;\n            while match_.pattern_index < layer.config.highlights_pattern_index {\n                // If the node represents a local scope, push a new local scope onto\n                // the scope stack.\n                if Some(capture.index) == layer.config.local_scope_capture_index {\n                    definition_highlight = None;\n                    let mut scope = LocalScope {\n                        inherits: true,\n                        range: range.clone(),\n                        local_defs: Vec::new(),\n                    };\n                    for prop in layer.config.query.property_settings(match_.pattern_index) {\n                        if let \"local.scope-inherits\" = prop.key.as_ref() {\n                            scope.inherits =\n                                prop.value.as_ref().map_or(true, |r| r.as_ref() == \"true\");\n                        }\n                    }\n                    layer.scope_stack.push(scope);\n                }\n                // If the node represents a definition, add a new definition to the\n                // local scope at the top of the scope stack.\n                else if Some(capture.index) == layer.config.local_def_capture_index {\n                    reference_highlight = None;\n                    let scope = layer.scope_stack.last_mut().unwrap();\n\n                    let mut value_range = 0..0;\n                    for capture in match_.captures {\n                        if Some(capture.index) == layer.config.local_def_value_capture_index {\n                            value_range = capture.node.byte_range();\n                        }\n                    }\n\n                    let name = byte_range_to_str(range.clone(), self.source);\n                    scope.local_defs.push(LocalDef {\n                        name,\n                        value_range,\n                        highlight: None,\n                    });\n                    definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);\n                }\n                // If the node represents a reference, then try to find the corresponding\n                // definition in the scope stack.\n                else if Some(capture.index) == layer.config.local_ref_capture_index\n                    && definition_highlight.is_none()\n                {\n                    definition_highlight = None;\n                    let name = byte_range_to_str(range.clone(), self.source);\n                    for scope in layer.scope_stack.iter().rev() {\n                        if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {\n                            if def.name == name && range.start >= def.value_range.end {\n                                Some(def.highlight)\n                            } else {\n                                None\n                            }\n                        }) {\n                            reference_highlight = highlight;\n                            break;\n                        }\n                        if !scope.inherits {\n                            break;\n                        }\n                    }\n                }\n\n                // Continue processing any additional matches for the same node.\n                if let Some((next_match, next_capture_index)) = captures.peek() {\n                    let next_capture = next_match.captures[*next_capture_index];\n                    if next_capture.node == capture.node {\n                        capture = next_capture;\n                        match_ = captures.next().unwrap().0;\n                        continue;\n                    }\n                }\n\n                self.sort_layers();\n                continue 'main;\n            }\n\n            // Otherwise, this capture must represent a highlight.\n            // If this exact range has already been highlighted by an earlier pattern, or by\n            // a different layer, then skip over this one.\n            if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {\n                if range.start == last_start && range.end == last_end && layer.depth < last_depth {\n                    self.sort_layers();\n                    continue 'main;\n                }\n            }\n\n            // If the current node was found to be a local variable, then skip over any\n            // highlighting patterns that are disabled for local variables.\n            if definition_highlight.is_some() || reference_highlight.is_some() {\n                while layer.config.non_local_variable_patterns[match_.pattern_index] {\n                    match_.remove();\n                    if let Some((next_match, next_capture_index)) = captures.peek() {\n                        let next_capture = next_match.captures[*next_capture_index];\n                        if next_capture.node == capture.node {\n                            capture = next_capture;\n                            match_ = captures.next().unwrap().0;\n                            continue;\n                        }\n                    }\n\n                    self.sort_layers();\n                    continue 'main;\n                }\n            }\n\n            // Once a highlighting pattern is found for the current node, skip over\n            // any later highlighting patterns that also match this node. Captures\n            // for a given node are ordered by pattern index, so these subsequent\n            // captures are guaranteed to be for highlighting, not injections or\n            // local variables.\n            while let Some((next_match, next_capture_index)) = captures.peek() {\n                let next_capture = next_match.captures[*next_capture_index];\n                if next_capture.node == capture.node {\n                    captures.next();\n                } else {\n                    break;\n                }\n            }\n\n            let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];\n\n            // If this node represents a local definition, then store the current\n            // highlight value on the local scope entry representing this node.\n            if let Some(definition_highlight) = definition_highlight {\n                *definition_highlight = current_highlight;\n            }\n\n            // Emit a scope start event and push the node's end position to the stack.\n            if let Some(highlight) = reference_highlight.or(current_highlight) {\n                self.last_highlight_range = Some((range.start, range.end, layer.depth));\n                layer.highlight_end_stack.push(range.end);\n                return self\n                    .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));\n            }\n\n            self.sort_layers();\n        }\n    }\n}\n\n#[derive(Debug, Clone)]\npub enum InjectionLanguageMarker<'a> {\n    Name(Cow<'a, str>),\n    Filename(Cow<'a, Path>),\n    Shebang(String),\n}\n\nconst SHEBANG: &str = r\"#!\\s*(?:\\S*[/\\\\](?:env\\s+(?:\\-\\S+\\s+)*)?)?([^\\s\\.\\d]+)\";\n\npub struct Merge<I> {\n    iter: I,\n    spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,\n\n    next_event: Option<HighlightEvent>,\n    next_span: Option<(usize, std::ops::Range<usize>)>,\n\n    queue: Vec<HighlightEvent>,\n}\n\n/// Merge a list of spans into the highlight event stream.\npub fn merge<I: Iterator<Item = HighlightEvent>>(\n    iter: I,\n    spans: Vec<(usize, std::ops::Range<usize>)>,\n) -> Merge<I> {\n    let spans = Box::new(spans.into_iter());\n    let mut merge = Merge {\n        iter,\n        spans,\n        next_event: None,\n        next_span: None,\n        queue: Vec::new(),\n    };\n    merge.next_event = merge.iter.next();\n    merge.next_span = merge.spans.next();\n    merge\n}\n\nimpl<I: Iterator<Item = HighlightEvent>> Iterator for Merge<I> {\n    type Item = HighlightEvent;\n    fn next(&mut self) -> Option<Self::Item> {\n        use HighlightEvent::*;\n        if let Some(event) = self.queue.pop() {\n            return Some(event);\n        }\n\n        loop {\n            match (self.next_event, &self.next_span) {\n                // this happens when range is partially or fully offscreen\n                (Some(Source { start, .. }), Some((span, range))) if start > range.start => {\n                    if start > range.end {\n                        self.next_span = self.spans.next();\n                    } else {\n                        self.next_span = Some((*span, start..range.end));\n                    };\n                }\n                _ => break,\n            }\n        }\n\n        match (self.next_event, &self.next_span) {\n            (Some(HighlightStart(i)), _) => {\n                self.next_event = self.iter.next();\n                Some(HighlightStart(i))\n            }\n            (Some(HighlightEnd), _) => {\n                self.next_event = self.iter.next();\n                Some(HighlightEnd)\n            }\n            (Some(Source { start, end }), Some((_, range))) if start < range.start => {\n                let intersect = range.start.min(end);\n                let event = Source {\n                    start,\n                    end: intersect,\n                };\n\n                if end == intersect {\n                    // the event is complete\n                    self.next_event = self.iter.next();\n                } else {\n                    // subslice the event\n                    self.next_event = Some(Source {\n                        start: intersect,\n                        end,\n                    });\n                };\n\n                Some(event)\n            }\n            (Some(Source { start, end }), Some((span, range))) if start == range.start => {\n                let intersect = range.end.min(end);\n                let event = HighlightStart(Highlight(*span));\n\n                // enqueue in reverse order\n                self.queue.push(HighlightEnd);\n                self.queue.push(Source {\n                    start,\n                    end: intersect,\n                });\n\n                if end == intersect {\n                    // the event is complete\n                    self.next_event = self.iter.next();\n                } else {\n                    // subslice the event\n                    self.next_event = Some(Source {\n                        start: intersect,\n                        end,\n                    });\n                };\n\n                if intersect == range.end {\n                    self.next_span = self.spans.next();\n                } else {\n                    self.next_span = Some((*span, intersect..range.end));\n                }\n\n                Some(event)\n            }\n            (Some(event), None) => {\n                self.next_event = self.iter.next();\n                Some(event)\n            }\n            // Can happen if cursor at EOF and/or diagnostic reaches past the end.\n            // We need to actually emit events for the cursor-at-EOF situation,\n            // even though the range is past the end of the text.  This needs to be\n            // handled appropriately by the drawing code by not assuming that\n            // all `Source` events point to valid indices in the rope.\n            (None, Some((span, range))) => {\n                let event = HighlightStart(Highlight(*span));\n                self.queue.push(HighlightEnd);\n                self.queue.push(Source {\n                    start: range.start,\n                    end: range.end,\n                });\n                self.next_span = self.spans.next();\n                Some(event)\n            }\n            (None, None) => None,\n            e => unreachable!(\"{:?}\", e),\n        }\n    }\n}\n\nfn node_is_visible(node: &Node) -> bool {\n    node.is_missing() || (node.is_named() && node.language().node_kind_is_visible(node.kind_id()))\n}\n\npub fn pretty_print_tree<W: fmt::Write>(fmt: &mut W, node: Node) -> fmt::Result {\n    if node.child_count() == 0 {\n        if node_is_visible(&node) {\n            write!(fmt, \"({})\", node.kind())\n        } else {\n            write!(fmt, \"\\\"{}\\\"\", node.kind())\n        }\n    } else {\n        pretty_print_tree_impl(fmt, &mut node.walk(), 0)\n    }\n}\n\nfn pretty_print_tree_impl<W: fmt::Write>(\n    fmt: &mut W,\n    cursor: &mut tree_sitter::TreeCursor,\n    depth: usize,\n) -> fmt::Result {\n    let node = cursor.node();\n    let visible = node_is_visible(&node);\n\n    if visible {\n        let indentation_columns = depth * 2;\n        write!(fmt, \"{:indentation_columns$}\", \"\")?;\n\n        if let Some(field_name) = cursor.field_name() {\n            write!(fmt, \"{}: \", field_name)?;\n        }\n\n        write!(fmt, \"({}\", node.kind())?;\n    }\n\n    // Handle children.\n    if cursor.goto_first_child() {\n        loop {\n            if node_is_visible(&cursor.node()) {\n                fmt.write_char('\\n')?;\n            }\n\n            pretty_print_tree_impl(fmt, cursor, depth + 1)?;\n\n            if !cursor.goto_next_sibling() {\n                break;\n            }\n        }\n\n        let moved = cursor.goto_parent();\n        // The parent of the first child must exist, and must be `node`.\n        debug_assert!(moved);\n        debug_assert!(cursor.node() == node);\n    }\n\n    if visible {\n        fmt.write_char(')')?;\n    }\n\n    Ok(())\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::{Rope, Transaction};\n\n    #[test]\n    fn test_textobject_queries() {\n        let query_str = r#\"\n        (line_comment)+ @quantified_nodes\n        ((line_comment)+) @quantified_nodes_grouped\n        ((line_comment) (line_comment)) @multiple_nodes_grouped\n        \"#;\n        let source = Rope::from_str(\n            r#\"\n/// a comment on\n/// multiple lines\n        \"#,\n        );\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n        let language = get_language(\"rust\").unwrap();\n\n        let query = Query::new(&language, query_str).unwrap();\n        let textobject = TextObjectQuery { query };\n        let mut cursor = QueryCursor::new();\n\n        let config = HighlightConfiguration::new(language, \"\", \"\", \"\").unwrap();\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n\n        let root = syntax.tree().root_node();\n        let mut test = |capture, range| {\n            let matches: Vec<_> = textobject\n                .capture_nodes(capture, root, source.slice(..), &mut cursor)\n                .unwrap()\n                .collect();\n\n            assert_eq!(\n                matches[0].byte_range(),\n                range,\n                \"@{} expected {:?}\",\n                capture,\n                range\n            )\n        };\n\n        test(\"quantified_nodes\", 1..36);\n        // NOTE: Enable after implementing proper node group capturing\n        // test(\"quantified_nodes_grouped\", 1..36);\n        // test(\"multiple_nodes_grouped\", 1..36);\n    }\n\n    #[test]\n    fn test_parser() {\n        let highlight_names: Vec<String> = [\n            \"attribute\",\n            \"constant\",\n            \"function.builtin\",\n            \"function\",\n            \"keyword\",\n            \"operator\",\n            \"property\",\n            \"punctuation\",\n            \"punctuation.bracket\",\n            \"punctuation.delimiter\",\n            \"string\",\n            \"string.special\",\n            \"tag\",\n            \"type\",\n            \"type.builtin\",\n            \"variable\",\n            \"variable.builtin\",\n            \"variable.parameter\",\n        ]\n        .iter()\n        .cloned()\n        .map(String::from)\n        .collect();\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n\n        let language = get_language(\"rust\").unwrap();\n        let config = HighlightConfiguration::new(\n            language,\n            &std::fs::read_to_string(\"../runtime/grammars/sources/rust/queries/highlights.scm\")\n                .unwrap(),\n            &std::fs::read_to_string(\"../runtime/grammars/sources/rust/queries/injections.scm\")\n                .unwrap(),\n            \"\", // locals.scm\n        )\n        .unwrap();\n        config.configure(&highlight_names);\n\n        let source = Rope::from_str(\n            \"\n            struct Stuff {}\n            fn main() {}\n        \",\n        );\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n        let tree = syntax.tree();\n        let root = tree.root_node();\n        assert_eq!(root.kind(), \"source_file\");\n\n        assert_eq!(\n            root.to_sexp(),\n            concat!(\n                \"(source_file \",\n                \"(struct_item name: (type_identifier) body: (field_declaration_list)) \",\n                \"(function_item name: (identifier) parameters: (parameters) body: (block)))\"\n            )\n        );\n\n        let struct_node = root.child(0).unwrap();\n        assert_eq!(struct_node.kind(), \"struct_item\");\n    }\n\n    #[test]\n    fn test_input_edits() {\n        use tree_sitter::InputEdit;\n\n        let doc = Rope::from(\"hello world!\\ntest 123\");\n        let transaction = Transaction::change(\n            &doc,\n            vec![(6, 11, Some(\"test\".into())), (12, 17, None)].into_iter(),\n        );\n        let edits = generate_edits(doc.slice(..), transaction.changes());\n        // transaction.apply(&mut state);\n\n        assert_eq!(\n            edits,\n            &[\n                InputEdit {\n                    start_byte: 6,\n                    old_end_byte: 11,\n                    new_end_byte: 10,\n                    start_position: Point { row: 0, column: 6 },\n                    old_end_position: Point { row: 0, column: 11 },\n                    new_end_position: Point { row: 0, column: 10 }\n                },\n                InputEdit {\n                    start_byte: 12,\n                    old_end_byte: 17,\n                    new_end_byte: 12,\n                    start_position: Point { row: 0, column: 12 },\n                    old_end_position: Point { row: 1, column: 4 },\n                    new_end_position: Point { row: 0, column: 12 }\n                }\n            ]\n        );\n\n        // Testing with the official example from tree-sitter\n        let mut doc = Rope::from(\"fn test() {}\");\n        let transaction =\n            Transaction::change(&doc, vec![(8, 8, Some(\"a: u32\".into()))].into_iter());\n        let edits = generate_edits(doc.slice(..), transaction.changes());\n        transaction.apply(&mut doc);\n\n        assert_eq!(doc, \"fn test(a: u32) {}\");\n        assert_eq!(\n            edits,\n            &[InputEdit {\n                start_byte: 8,\n                old_end_byte: 8,\n                new_end_byte: 14,\n                start_position: Point { row: 0, column: 8 },\n                old_end_position: Point { row: 0, column: 8 },\n                new_end_position: Point { row: 0, column: 14 }\n            }]\n        );\n    }\n\n    #[track_caller]\n    fn assert_pretty_print(\n        language_name: &str,\n        source: &str,\n        expected: &str,\n        start: usize,\n        end: usize,\n    ) {\n        let source = Rope::from_str(source);\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n        let language = get_language(language_name).unwrap();\n\n        let config = HighlightConfiguration::new(language, \"\", \"\", \"\").unwrap();\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n\n        let root = syntax\n            .tree()\n            .root_node()\n            .descendant_for_byte_range(start, end)\n            .unwrap();\n\n        let mut output = String::new();\n        pretty_print_tree(&mut output, root).unwrap();\n\n        assert_eq!(expected, output);\n    }\n\n    #[test]\n    fn test_pretty_print() {\n        let source = r#\"/// Hello\"#;\n        assert_pretty_print(\"rust\", source, \"(line_comment)\", 0, source.len());\n\n        // A large tree should be indented with fields:\n        let source = r#\"fn main() {\n            println!(\"Hello, World!\");\n        }\"#;\n        assert_pretty_print(\n            \"rust\",\n            source,\n            concat!(\n                \"(function_item\\n\",\n                \"  name: (identifier)\\n\",\n                \"  parameters: (parameters)\\n\",\n                \"  body: (block\\n\",\n                \"    (expression_statement\\n\",\n                \"      (macro_invocation\\n\",\n                \"        macro: (identifier)\\n\",\n                \"        (token_tree\\n\",\n                \"          (string_literal))))))\",\n            ),\n            0,\n            source.len(),\n        );\n\n        // Selecting a token should print just that token:\n        let source = r#\"fn main() {}\"#;\n        assert_pretty_print(\"rust\", source, r#\"\"fn\"\"#, 0, 1);\n\n        // Error nodes are printed as errors:\n        let source = r#\"}{\"#;\n        assert_pretty_print(\"rust\", source, \"(ERROR)\", 0, source.len());\n\n        // Fields broken under unnamed nodes are determined correctly.\n        // In the following source, `object` belongs to the `singleton_method`\n        // rule but `name` and `body` belong to an unnamed helper `_method_rest`.\n        // This can cause a bug with a pretty-printing implementation that\n        // uses `Node::field_name_for_child` to determine field names but is\n        // fixed when using `tree_sitter::TreeCursor::field_name`.\n        let source = \"def self.method_name\n          true\n        end\";\n        assert_pretty_print(\n            \"ruby\",\n            source,\n            concat!(\n                \"(singleton_method\\n\",\n                \"  object: (self)\\n\",\n                \"  name: (identifier)\\n\",\n                \"  body: (body_statement\\n\",\n                \"    (true)))\"\n            ),\n            0,\n            source.len(),\n        );\n    }\n\n    #[test]\n    fn test_load_runtime_file() {\n        // Test to make sure we can load some data from the runtime directory.\n        let contents = load_runtime_file(\"rust\", \"indents.scm\").unwrap();\n        assert!(!contents.is_empty());\n\n        let results = load_runtime_file(\"rust\", \"does-not-exist\");\n        assert!(results.is_err());\n    }\n}\n\n// Path: helix-core/src/config.rs\nuse crate::syntax::{Configuration, Loader, LoaderError};\n\n/// Language configuration based on built-in languages.toml.\npub fn default_lang_config() -> Configuration {\n    helix_loader::config::default_lang_config()\n        .try_into()\n        .expect(\"Could not deserialize built-in languages.toml\")\n}\n\n/// Language configuration loader based on built-in languages.toml.\npub fn default_lang_loader() -> Loader {\n    Loader::new(default_lang_config()).expect(\"Could not compile loader for default config\")\n}\n\n#[derive(Debug)]\npub enum LanguageLoaderError {\n    DeserializeError(toml::de::Error),\n    LoaderError(LoaderError),\n}\n\nimpl std::fmt::Display for LanguageLoaderError {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        match self {\n            Self::DeserializeError(err) => write!(f, \"Failed to parse language config: {err}\"),\n            Self::LoaderError(err) => write!(f, \"Failed to compile language config: {err}\"),\n        }\n    }\n}\n\nimpl std::error::Error for LanguageLoaderError {}\n\n/// Language configuration based on user configured languages.toml.\npub fn user_lang_config() -> Result<Configuration, toml::de::Error> {\n    helix_loader::config::user_lang_config()?.try_into()\n}\n\n/// Language configuration loader based on user configured languages.toml.\npub fn user_lang_loader() -> Result<Loader, LanguageLoaderError> {\n    let config: Configuration = helix_loader::config::user_lang_config()\n        .map_err(LanguageLoaderError::DeserializeError)?\n        .try_into()\n        .map_err(LanguageLoaderError::DeserializeError)?;\n\n    Loader::new(config).map_err(LanguageLoaderError::LoaderError)\n}\n\n// Path: helix-core/src/comment.rs\n//! This module contains the functionality toggle comments on lines over the selection\n//! using the comment character defined in the user's `languages.toml`\n\nuse smallvec::SmallVec;\n\nuse crate::{\n    syntax::BlockCommentToken, Change, Range, Rope, RopeSlice, Selection, Tendril, Transaction,\n};\nuse helix_stdx::rope::RopeSliceExt;\nuse std::borrow::Cow;\n\n/// Given text, a comment token, and a set of line indices, returns the following:\n/// - Whether the given lines should be considered commented\n///     - If any of the lines are uncommented, all lines are considered as such.\n/// - The lines to change for toggling comments\n///     - This is all provided lines excluding blanks lines.\n/// - The column of the comment tokens\n///     - Column of existing tokens, if the lines are commented; column to place tokens at otherwise.\n/// - The margin to the right of the comment tokens\n///     - Defaults to `1`. If any existing comment token is not followed by a space, changes to `0`.\nfn find_line_comment(\n    token: &str,\n    text: RopeSlice,\n    lines: impl IntoIterator<Item = usize>,\n) -> (bool, Vec<usize>, usize, usize) {\n    let mut commented = true;\n    let mut to_change = Vec::new();\n    let mut min = usize::MAX; // minimum col for first_non_whitespace_char\n    let mut margin = 1;\n    let token_len = token.chars().count();\n    for line in lines {\n        let line_slice = text.line(line);\n        if let Some(pos) = line_slice.first_non_whitespace_char() {\n            let len = line_slice.len_chars();\n\n            if pos < min {\n                min = pos;\n            }\n\n            // line can be shorter than pos + token len\n            let fragment = Cow::from(line_slice.slice(pos..std::cmp::min(pos + token.len(), len)));\n\n            if fragment != token {\n                // as soon as one of the non-blank lines doesn't have a comment, the whole block is\n                // considered uncommented.\n                commented = false;\n            }\n\n            // determine margin of 0 or 1 for uncommenting; if any comment token is not followed by a space,\n            // a margin of 0 is used for all lines.\n            if !matches!(line_slice.get_char(pos + token_len), Some(c) if c == ' ') {\n                margin = 0;\n            }\n\n            // blank lines don't get pushed.\n            to_change.push(line);\n        }\n    }\n    (commented, to_change, min, margin)\n}\n\n#[must_use]\npub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&str>) -> Transaction {\n    let text = doc.slice(..);\n\n    let token = token.unwrap_or(\"//\");\n    let comment = Tendril::from(format!(\"{} \", token));\n\n    let mut lines: Vec<usize> = Vec::with_capacity(selection.len());\n\n    let mut min_next_line = 0;\n    for selection in selection {\n        let (start, end) = selection.line_range(text);\n        let start = start.clamp(min_next_line, text.len_lines());\n        let end = (end + 1).min(text.len_lines());\n\n        lines.extend(start..end);\n        min_next_line = end;\n    }\n\n    let (commented, to_change, min, margin) = find_line_comment(token, text, lines);\n\n    let mut changes: Vec<Change> = Vec::with_capacity(to_change.len());\n\n    for line in to_change {\n        let pos = text.line_to_char(line) + min;\n\n        if !commented {\n            // comment line\n            changes.push((pos, pos, Some(comment.clone())));\n        } else {\n            // uncomment line\n            changes.push((pos, pos + token.len() + margin, None));\n        }\n    }\n\n    Transaction::change(doc, changes.into_iter())\n}\n\n#[derive(Debug, PartialEq, Eq)]\npub enum CommentChange {\n    Commented {\n        range: Range,\n...\n", "needle_token_start": 5437, "needle_token_end": 5925, "code_context_ntokens": 16218, "output": ["```rust\nfn injection_for_match<'a>(\n    &self,\n    query: &'a Query,\n    query_match: &QueryMatch<'a, 'a>,\n    source: RopeSlice<'a>,\n) -> (\n    Option<InjectionLanguageMarker<'a>>,\n    Option<Node<'a>>,\n    IncludedChildren,\n) {\n    let (mut injection_capture, content_node) = self.injection_pair(query_match, source);\n\n    let mut included_children = IncludedChildren::default();\n    for prop in query.property_settings(query_match.pattern_index) {\n        match prop.key.as_ref() {\n            // In addition to specifying the language name via the text of a\n            // captured node, it can also be hard-coded via a `#set!` predicate\n            // that sets the injection.language key.\n            \"injection.language\" if injection_capture.is_none() => {\n                injection_capture = prop\n                    .value\n                    .as_ref()\n                    .map(|s| InjectionLanguageMarker::Name(s.as_ref().into()));\n            }\n\n            // By default, injections do not include the *children* of an\n            // `injection.content` node - only the ranges that belong to the\n            // node itself. This can be changed using a `#set!` predicate that\n            // sets the `injection.include-children` key.\n            \"injection.include-children\" => included_children = IncludedChildren::All,\n\n            // Some queries might only exclude named children but include unnamed\n            // children in their `injection.content` node. This can be enabled using\n            // a `#set!` predicate that sets the `injection.include-unnamed-children` key.\n            \"injection.include-unnamed-children\" => {\n                included_children = IncludedChildren::Unnamed\n            }\n            _ => {}\n        }\n    }\n\n    (injection_capture, content_node, included_children)\n}\n```"]}
{"repo": "helix-editor/helix", "name": "names", "language": "rust", "path": "helix-core/src/syntax.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To retrieve a list of all the identifiers used for capturing specific patterns in a syntax highlighting configuration.\n2. **Input**: The method does not require any external inputs as it operates on the internal state of the instance.\n3. **Output**: Returns a slice of string references, each representing a unique identifier used in the syntax highlighting patterns.\n4. **Procedure**: The method accesses a stored list of identifiers from the internal query configuration and returns a reference to this list.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n            while let Some(layer_id) = queue.pop_front() {\n                let layer = &mut self.layers[layer_id];\n\n                // Mark the layer as touched\n                layer.flags |= LayerUpdateFlags::TOUCHED;\n\n                // If a tree already exists, notify it of changes.\n                if let Some(tree) = &mut layer.tree {\n                    if layer\n                        .flags\n                        .intersects(LayerUpdateFlags::MODIFIED | LayerUpdateFlags::MOVED)\n                    {\n                        for edit in edits.iter().rev() {\n                            // Apply the edits in reverse.\n                            // If we applied them in order then edit 1 would disrupt the positioning of edit 2.\n                            tree.edit(edit);\n                        }\n                    }\n\n                    if layer.flags.contains(LayerUpdateFlags::MODIFIED) {\n                        // Re-parse the tree.\n                        layer.parse(&mut ts_parser.parser, source)?;\n                    }\n                } else {\n                    // always parse if this layer has never been parsed before\n                    layer.parse(&mut ts_parser.parser, source)?;\n                }\n\n                // Switch to an immutable borrow.\n                let layer = &self.layers[layer_id];\n\n                // Process injections.\n                let matches = cursor.matches(\n                    &layer.config.injections_query,\n                    layer.tree().root_node(),\n                    RopeProvider(source_slice),\n                );\n                let mut combined_injections = vec![\n                    (None, Vec::new(), IncludedChildren::default());\n                    layer.config.combined_injections_patterns.len()\n                ];\n                let mut injections = Vec::new();\n                let mut last_injection_end = 0;\n                for mat in matches {\n                    let (injection_capture, content_node, included_children) = layer\n                        .config\n                        .injection_for_match(&layer.config.injections_query, &mat, source_slice);\n\n                    // in case this is a combined injection save it for more processing later\n                    if let Some(combined_injection_idx) = layer\n                        .config\n                        .combined_injections_patterns\n                        .iter()\n                        .position(|&pattern| pattern == mat.pattern_index)\n                    {\n                        let entry = &mut combined_injections[combined_injection_idx];\n                        if injection_capture.is_some() {\n                            entry.0 = injection_capture;\n                        }\n                        if let Some(content_node) = content_node {\n                            if content_node.start_byte() >= last_injection_end {\n                                entry.1.push(content_node);\n                                last_injection_end = content_node.end_byte();\n                            }\n                        }\n                        entry.2 = included_children;\n                        continue;\n                    }\n\n                    // Explicitly remove this match so that none of its other captures will remain\n                    // in the stream of captures.\n                    mat.remove();\n\n                    // If a language is found with the given name, then add a new language layer\n                    // to the highlighted document.\n                    if let (Some(injection_capture), Some(content_node)) =\n                        (injection_capture, content_node)\n                    {\n                        if let Some(config) = (injection_callback)(&injection_capture) {\n                            let ranges =\n                                intersect_ranges(&layer.ranges, &[content_node], included_children);\n\n                            if !ranges.is_empty() {\n                                if content_node.start_byte() < last_injection_end {\n                                    continue;\n                                }\n                                last_injection_end = content_node.end_byte();\n                                injections.push((config, ranges));\n                            }\n                        }\n                    }\n                }\n\n                for (lang_name, content_nodes, included_children) in combined_injections {\n                    if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {\n                        if let Some(config) = (injection_callback)(&lang_name) {\n                            let ranges =\n                                intersect_ranges(&layer.ranges, &content_nodes, included_children);\n                            if !ranges.is_empty() {\n                                injections.push((config, ranges));\n                            }\n                        }\n                    }\n                }\n\n                let depth = layer.depth + 1;\n                // TODO: can't inline this since matches borrows self.layers\n                for (config, ranges) in injections {\n                    let new_layer = LanguageLayer {\n                        tree: None,\n                        config,\n                        depth,\n                        ranges,\n                        flags: LayerUpdateFlags::empty(),\n                        parent: Some(layer_id),\n                    };\n\n                    // Find an identical existing layer\n                    let layer = layers_table\n                        .get(layers_hasher.hash_one(&new_layer), |&it| {\n                            self.layers[it] == new_layer\n                        })\n                        .copied();\n\n                    // ...or insert a new one.\n                    let layer_id = layer.unwrap_or_else(|| self.layers.insert(new_layer));\n\n                    queue.push_back(layer_id);\n                }\n\n                // TODO: pre-process local scopes at this time, rather than highlight?\n                // would solve problems with locals not working across boundaries\n            }\n\n            // Return the cursor back in the pool.\n            ts_parser.cursors.push(cursor);\n\n            // Reset all `LayerUpdateFlags` and remove all untouched layers\n            self.layers.retain(|_, layer| {\n                replace(&mut layer.flags, LayerUpdateFlags::empty())\n                    .contains(LayerUpdateFlags::TOUCHED)\n            });\n\n            Ok(())\n        })\n    }\n\n    pub fn tree(&self) -> &Tree {\n        self.layers[self.root].tree()\n    }\n\n    /// Iterate over the highlighted regions for a given slice of source code.\n    pub fn highlight_iter<'a>(\n        &'a self,\n        source: RopeSlice<'a>,\n        range: Option<std::ops::Range<usize>>,\n        cancellation_flag: Option<&'a AtomicUsize>,\n    ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {\n        let mut layers = self\n            .layers\n            .iter()\n            .filter_map(|(_, layer)| {\n                // TODO: if range doesn't overlap layer range, skip it\n\n                // Reuse a cursor from the pool if available.\n                let mut cursor = PARSER.with(|ts_parser| {\n                    let highlighter = &mut ts_parser.borrow_mut();\n                    highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)\n                });\n\n                // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which\n                // prevents them from being moved. But both of these values are really just\n                // pointers, so it's actually ok to move them.\n                let cursor_ref =\n                    unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };\n\n                // if reusing cursors & no range this resets to whole range\n                cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));\n                cursor_ref.set_match_limit(TREE_SITTER_MATCH_LIMIT);\n\n                let mut captures = cursor_ref\n                    .captures(\n                        &layer.config.query,\n                        layer.tree().root_node(),\n                        RopeProvider(source),\n                    )\n                    .peekable();\n\n                // If there's no captures, skip the layer\n                captures.peek()?;\n\n                Some(HighlightIterLayer {\n                    highlight_end_stack: Vec::new(),\n                    scope_stack: vec![LocalScope {\n                        inherits: false,\n                        range: 0..usize::MAX,\n                        local_defs: Vec::new(),\n                    }],\n                    cursor,\n                    _tree: None,\n                    captures: RefCell::new(captures),\n                    config: layer.config.as_ref(), // TODO: just reuse `layer`\n                    depth: layer.depth,            // TODO: just reuse `layer`\n                })\n            })\n            .collect::<Vec<_>>();\n\n        layers.sort_unstable_by_key(|layer| layer.sort_key());\n\n        let mut result = HighlightIter {\n            source,\n            byte_offset: range.map_or(0, |r| r.start),\n            cancellation_flag,\n            iter_count: 0,\n            layers,\n            next_event: None,\n            last_highlight_range: None,\n        };\n        result.sort_layers();\n        result\n    }\n\n    pub fn tree_for_byte_range(&self, start: usize, end: usize) -> &Tree {\n        let mut container_id = self.root;\n\n        for (layer_id, layer) in self.layers.iter() {\n            if layer.depth > self.layers[container_id].depth\n                && layer.contains_byte_range(start, end)\n            {\n                container_id = layer_id;\n            }\n        }\n\n        self.layers[container_id].tree()\n    }\n\n    pub fn named_descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {\n        self.tree_for_byte_range(start, end)\n            .root_node()\n            .named_descendant_for_byte_range(start, end)\n    }\n\n    pub fn descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {\n        self.tree_for_byte_range(start, end)\n            .root_node()\n            .descendant_for_byte_range(start, end)\n    }\n\n    pub fn walk(&self) -> TreeCursor<'_> {\n        // data structure to find the smallest range that contains a point\n        // when some of the ranges in the structure can overlap.\n        TreeCursor::new(&self.layers, self.root)\n    }\n\n    // Commenting\n    // comment_strings_for_pos\n    // is_commented\n\n    // Indentation\n    // suggested_indent_for_line_at_buffer_row\n    // suggested_indent_for_buffer_row\n    // indent_level_for_line\n\n    // TODO: Folding\n}\n\nbitflags! {\n    /// Flags that track the status of a layer\n    /// in the `Sytaxn::update` function\n    #[derive(Debug)]\n    struct LayerUpdateFlags : u32{\n        const MODIFIED = 0b001;\n        const MOVED = 0b010;\n        const TOUCHED = 0b100;\n    }\n}\n\n#[derive(Debug)]\npub struct LanguageLayer {\n    // mode\n    // grammar\n    pub config: Arc<HighlightConfiguration>,\n    pub(crate) tree: Option<Tree>,\n    pub ranges: Vec<Range>,\n    pub depth: u32,\n    flags: LayerUpdateFlags,\n    parent: Option<LayerId>,\n}\n\n/// This PartialEq implementation only checks if that\n/// two layers are theoretically identical (meaning they highlight the same text range with the same language).\n/// It does not check whether the layers have the same internal treesitter\n/// state.\nimpl PartialEq for LanguageLayer {\n    fn eq(&self, other: &Self) -> bool {\n        self.depth == other.depth\n            && self.config.language == other.config.language\n            && self.ranges == other.ranges\n    }\n}\n\n/// Hash implementation belongs to PartialEq implementation above.\n/// See its documentation for details.\nimpl Hash for LanguageLayer {\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        self.depth.hash(state);\n        self.config.language.hash(state);\n        self.ranges.hash(state);\n    }\n}\n\nimpl LanguageLayer {\n    pub fn tree(&self) -> &Tree {\n        // TODO: no unwrap\n        self.tree.as_ref().unwrap()\n    }\n\n    fn parse(&mut self, parser: &mut Parser, source: RopeSlice) -> Result<(), Error> {\n        parser\n            .set_included_ranges(&self.ranges)\n            .map_err(|_| Error::InvalidRanges)?;\n\n        parser\n            .set_language(&self.config.language)\n            .map_err(|_| Error::InvalidLanguage)?;\n\n        // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };\n        let tree = parser\n            .parse_with(\n                &mut |byte, _| {\n                    if byte <= source.len_bytes() {\n                        let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);\n                        &chunk.as_bytes()[byte - start_byte..]\n                    } else {\n                        // out of range\n                        &[]\n                    }\n                },\n                self.tree.as_ref(),\n            )\n            .ok_or(Error::Cancelled)?;\n        // unsafe { ts_parser.parser.set_cancellation_flag(None) };\n        self.tree = Some(tree);\n        Ok(())\n    }\n\n    /// Whether the layer contains the given byte range.\n    ///\n    /// If the layer has multiple ranges (i.e. combined injections), the\n    /// given range is considered contained if it is within the start and\n    /// end bytes of the first and last ranges **and** if the given range\n    /// starts or ends within any of the layer's ranges.\n    fn contains_byte_range(&self, start: usize, end: usize) -> bool {\n        let layer_start = self\n            .ranges\n            .first()\n            .expect(\"ranges should not be empty\")\n            .start_byte;\n        let layer_end = self\n            .ranges\n            .last()\n            .expect(\"ranges should not be empty\")\n            .end_byte;\n\n        layer_start <= start\n            && layer_end >= end\n            && self.ranges.iter().any(|range| {\n                let byte_range = range.start_byte..range.end_byte;\n                byte_range.contains(&start) || byte_range.contains(&end)\n            })\n    }\n}\n\npub(crate) fn generate_edits(\n    old_text: RopeSlice,\n    changeset: &ChangeSet,\n) -> Vec<tree_sitter::InputEdit> {\n    use Operation::*;\n    let mut old_pos = 0;\n\n    let mut edits = Vec::new();\n\n    if changeset.changes.is_empty() {\n        return edits;\n    }\n\n    let mut iter = changeset.changes.iter().peekable();\n\n    // TODO; this is a lot easier with Change instead of Operation.\n\n    fn point_at_pos(text: RopeSlice, pos: usize) -> (usize, Point) {\n        let byte = text.char_to_byte(pos); // <- attempted to index past end\n        let line = text.char_to_line(pos);\n        let line_start_byte = text.line_to_byte(line);\n        let col = byte - line_start_byte;\n\n        (byte, Point::new(line, col))\n    }\n\n    fn traverse(point: Point, text: &Tendril) -> Point {\n        let Point {\n            mut row,\n            mut column,\n        } = point;\n\n        // TODO: there should be a better way here.\n        let mut chars = text.chars().peekable();\n        while let Some(ch) = chars.next() {\n            if char_is_line_ending(ch) && !(ch == '\\r' && chars.peek() == Some(&'\\n')) {\n                row += 1;\n                column = 0;\n            } else {\n                column += 1;\n            }\n        }\n        Point { row, column }\n    }\n\n    while let Some(change) = iter.next() {\n        let len = match change {\n            Delete(i) | Retain(i) => *i,\n            Insert(_) => 0,\n        };\n        let mut old_end = old_pos + len;\n\n        match change {\n            Retain(_) => {}\n            Delete(_) => {\n                let (start_byte, start_position) = point_at_pos(old_text, old_pos);\n                let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);\n\n                // deletion\n                edits.push(tree_sitter::InputEdit {\n                    start_byte,                       // old_pos to byte\n                    old_end_byte,                     // old_end to byte\n                    new_end_byte: start_byte,         // old_pos to byte\n                    start_position,                   // old pos to coords\n                    old_end_position,                 // old_end to coords\n                    new_end_position: start_position, // old pos to coords\n                });\n            }\n            Insert(s) => {\n                let (start_byte, start_position) = point_at_pos(old_text, old_pos);\n\n                // a subsequent delete means a replace, consume it\n                if let Some(Delete(len)) = iter.peek() {\n                    old_end = old_pos + len;\n                    let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);\n\n                    iter.next();\n\n                    // replacement\n                    edits.push(tree_sitter::InputEdit {\n                        start_byte,                                    // old_pos to byte\n                        old_end_byte,                                  // old_end to byte\n                        new_end_byte: start_byte + s.len(),            // old_pos to byte + s.len()\n                        start_position,                                // old pos to coords\n                        old_end_position,                              // old_end to coords\n                        new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)\n                    });\n                } else {\n                    // insert\n                    edits.push(tree_sitter::InputEdit {\n                        start_byte,                                    // old_pos to byte\n                        old_end_byte: start_byte,                      // same\n                        new_end_byte: start_byte + s.len(),            // old_pos + s.len()\n                        start_position,                                // old pos to coords\n                        old_end_position: start_position,              // same\n                        new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)\n                    });\n                }\n            }\n        }\n        old_pos = old_end;\n    }\n    edits\n}\n\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{iter, mem, ops, str, usize};\nuse tree_sitter::{\n    Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,\n    QueryMatch, Range, TextProvider, Tree,\n};\n\nconst CANCELLATION_CHECK_INTERVAL: usize = 100;\n\n/// Indicates which highlight should be applied to a region of source code.\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\npub struct Highlight(pub usize);\n\n/// Represents the reason why syntax highlighting failed.\n#[derive(Debug, PartialEq, Eq)]\npub enum Error {\n    Cancelled,\n    InvalidLanguage,\n    InvalidRanges,\n    Unknown,\n}\n\n/// Represents a single step in rendering a syntax-highlighted document.\n#[derive(Copy, Clone, Debug)]\npub enum HighlightEvent {\n    Source { start: usize, end: usize },\n    HighlightStart(Highlight),\n    HighlightEnd,\n}\n\n/// Contains the data needed to highlight code written in a particular language.\n///\n/// This struct is immutable and can be shared between threads.\n#[derive(Debug)]\npub struct HighlightConfiguration {\n    pub language: Grammar,\n    pub query: Query,\n    injections_query: Query,\n    combined_injections_patterns: Vec<usize>,\n    highlights_pattern_index: usize,\n    highlight_indices: ArcSwap<Vec<Option<Highlight>>>,\n    non_local_variable_patterns: Vec<bool>,\n    injection_content_capture_index: Option<u32>,\n    injection_language_capture_index: Option<u32>,\n    injection_filename_capture_index: Option<u32>,\n    injection_shebang_capture_index: Option<u32>,\n    local_scope_capture_index: Option<u32>,\n    local_def_capture_index: Option<u32>,\n    local_def_value_capture_index: Option<u32>,\n    local_ref_capture_index: Option<u32>,\n}\n\n#[derive(Debug)]\nstruct LocalDef<'a> {\n    name: Cow<'a, str>,\n    value_range: ops::Range<usize>,\n    highlight: Option<Highlight>,\n}\n\n#[derive(Debug)]\nstruct LocalScope<'a> {\n    inherits: bool,\n    range: ops::Range<usize>,\n    local_defs: Vec<LocalDef<'a>>,\n}\n\n#[derive(Debug)]\nstruct HighlightIter<'a> {\n    source: RopeSlice<'a>,\n    byte_offset: usize,\n    cancellation_flag: Option<&'a AtomicUsize>,\n    layers: Vec<HighlightIterLayer<'a>>,\n    iter_count: usize,\n    next_event: Option<HighlightEvent>,\n    last_highlight_range: Option<(usize, usize, u32)>,\n}\n\n// Adapter to convert rope chunks to bytes\npub struct ChunksBytes<'a> {\n    chunks: ropey::iter::Chunks<'a>,\n}\nimpl<'a> Iterator for ChunksBytes<'a> {\n    type Item = &'a [u8];\n    fn next(&mut self) -> Option<Self::Item> {\n        self.chunks.next().map(str::as_bytes)\n    }\n}\n\npub struct RopeProvider<'a>(pub RopeSlice<'a>);\nimpl<'a> TextProvider<&'a [u8]> for RopeProvider<'a> {\n    type I = ChunksBytes<'a>;\n\n    fn text(&mut self, node: Node) -> Self::I {\n        let fragment = self.0.byte_slice(node.start_byte()..node.end_byte());\n        ChunksBytes {\n            chunks: fragment.chunks(),\n        }\n    }\n}\n\nstruct HighlightIterLayer<'a> {\n    _tree: Option<Tree>,\n    cursor: QueryCursor,\n    captures: RefCell<iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>, &'a [u8]>>>,\n    config: &'a HighlightConfiguration,\n    highlight_end_stack: Vec<usize>,\n    scope_stack: Vec<LocalScope<'a>>,\n    depth: u32,\n}\n\nimpl<'a> fmt::Debug for HighlightIterLayer<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"HighlightIterLayer\").finish()\n    }\n}\n\nimpl HighlightConfiguration {\n    /// Creates a `HighlightConfiguration` for a given `Grammar` and set of highlighting\n    /// queries.\n    ///\n    /// # Parameters\n    ///\n    /// * `language`  - The Tree-sitter `Grammar` that should be used for parsing.\n    /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This\n    ///   should be non-empty, otherwise no syntax highlights will be added.\n    /// * `injections_query` -  A string containing tree patterns for injecting other languages\n    ///   into the document. This can be empty if no injections are desired.\n    /// * `locals_query` - A string containing tree patterns for tracking local variable\n    ///   definitions and references. This can be empty if local variable tracking is not needed.\n    ///\n    /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.\n    pub fn new(\n        language: Grammar,\n        highlights_query: &str,\n        injection_query: &str,\n        locals_query: &str,\n    ) -> Result<Self, QueryError> {\n        // Concatenate the query strings, keeping track of the start offset of each section.\n        let mut query_source = String::new();\n        query_source.push_str(locals_query);\n        let highlights_query_offset = query_source.len();\n        query_source.push_str(highlights_query);\n\n        // Construct a single query by concatenating the three query strings, but record the\n        // range of pattern indices that belong to each individual string.\n        let query = Query::new(&language, &query_source)?;\n        let mut highlights_pattern_index = 0;\n        for i in 0..(query.pattern_count()) {\n            let pattern_offset = query.start_byte_for_pattern(i);\n            if pattern_offset < highlights_query_offset {\n                highlights_pattern_index += 1;\n            }\n        }\n\n        let injections_query = Query::new(&language, injection_query)?;\n        let combined_injections_patterns = (0..injections_query.pattern_count())\n            .filter(|&i| {\n                injections_query\n                    .property_settings(i)\n                    .iter()\n                    .any(|s| &*s.key == \"injection.combined\")\n            })\n            .collect();\n\n        // Find all of the highlighting patterns that are disabled for nodes that\n        // have been identified as local variables.\n        let non_local_variable_patterns = (0..query.pattern_count())\n            .map(|i| {\n                query\n                    .property_predicates(i)\n                    .iter()\n                    .any(|(prop, positive)| !*positive && prop.key.as_ref() == \"local\")\n            })\n            .collect();\n\n        // Store the numeric ids for all of the special captures.\n        let mut injection_content_capture_index = None;\n        let mut injection_language_capture_index = None;\n        let mut injection_filename_capture_index = None;\n        let mut injection_shebang_capture_index = None;\n        let mut local_def_capture_index = None;\n        let mut local_def_value_capture_index = None;\n        let mut local_ref_capture_index = None;\n        let mut local_scope_capture_index = None;\n        for (i, name) in query.capture_names().iter().enumerate() {\n            let i = Some(i as u32);\n            match *name {\n                \"local.definition\" => local_def_capture_index = i,\n                \"local.definition-value\" => local_def_value_capture_index = i,\n                \"local.reference\" => local_ref_capture_index = i,\n                \"local.scope\" => local_scope_capture_index = i,\n                _ => {}\n            }\n        }\n\n        for (i, name) in injections_query.capture_names().iter().enumerate() {\n            let i = Some(i as u32);\n            match *name {\n                \"injection.content\" => injection_content_capture_index = i,\n                \"injection.language\" => injection_language_capture_index = i,\n                \"injection.filename\" => injection_filename_capture_index = i,\n                \"injection.shebang\" => injection_shebang_capture_index = i,\n                _ => {}\n            }\n        }\n\n        let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);\n        Ok(Self {\n            language,\n            query,\n            injections_query,\n            combined_injections_patterns,\n            highlights_pattern_index,\n            highlight_indices,\n            non_local_variable_patterns,\n            injection_content_capture_index,\n            injection_language_capture_index,\n            injection_filename_capture_index,\n            injection_shebang_capture_index,\n            local_scope_capture_index,\n            local_def_capture_index,\n            local_def_value_capture_index,\n            local_ref_capture_index,\n        })\n    }\n\n    /// Get a slice containing all of the highlight names used in the configuration.\n    pu\nb fn names(&self) -> &[&str] {\n        self.query.capture_names()\n    }\n\n    /// Set the list of recognized highlight names.\n    ///\n    /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated\n    /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of\n    /// these queries can choose to recognize highlights with different levels of specificity.\n    /// For example, the string `function.builtin` will match against `function.builtin.constructor`\n    /// but will not match `function.method.builtin` and `function.method`.\n    ///\n    /// When highlighting, results are returned as `Highlight` values, which contain the index\n    /// of the matched highlight this list of highlight names.\n    pub fn configure(&self, recognized_names: &[String]) {\n        let mut capture_parts = Vec::new();\n        let indices: Vec<_> = self\n            .query\n            .capture_names()\n            .iter()\n            .map(move |capture_name| {\n                capture_parts.clear();\n                capture_parts.extend(capture_name.split('.'));\n\n                let mut best_index = None;\n                let mut best_match_len = 0;\n                for (i, recognized_name) in recognized_names.iter().enumerate() {\n                    let mut len = 0;\n                    let mut matches = true;\n                    for (i, part) in recognized_name.split('.').enumerate() {\n                        match capture_parts.get(i) {\n                            Some(capture_part) if *capture_part == part => len += 1,\n                            _ => {\n                                matches = false;\n                                break;\n                            }\n                        }\n                    }\n                    if matches && len > best_match_len {\n                        best_index = Some(i);\n                        best_match_len = len;\n                    }\n                }\n                best_index.map(Highlight)\n            })\n            .collect();\n\n        self.highlight_indices.store(Arc::new(indices));\n    }\n\n    fn injection_pair<'a>(\n        &self,\n        query_match: &QueryMatch<'a, 'a>,\n        source: RopeSlice<'a>,\n    ) -> (Option<InjectionLanguageMarker<'a>>, Option<Node<'a>>) {\n        let mut injection_capture = None;\n        let mut content_node = None;\n\n        for capture in query_match.captures {\n            let index = Some(capture.index);\n            if index == self.injection_language_capture_index {\n                let name = byte_range_to_str(capture.node.byte_range(), source);\n                injection_capture = Some(InjectionLanguageMarker::Name(name));\n            } else if index == self.injection_filename_capture_index {\n                let name = byte_range_to_str(capture.node.byte_range(), source);\n                let path = Path::new(name.as_ref()).to_path_buf();\n                injection_capture = Some(InjectionLanguageMarker::Filename(path.into()));\n            } else if index == self.injection_shebang_capture_index {\n                let node_slice = source.byte_slice(capture.node.byte_range());\n\n                // some languages allow space and newlines before the actual string content\n                // so a shebang could be on either the first or second line\n                let lines = if let Ok(end) = node_slice.try_line_to_byte(2) {\n                    node_slice.byte_slice(..end)\n                } else {\n                    node_slice\n                };\n\n                static SHEBANG_REGEX: Lazy<rope::Regex> =\n                    Lazy::new(|| rope::Regex::new(SHEBANG).unwrap());\n\n                injection_capture = SHEBANG_REGEX\n                    .captures_iter(lines.regex_input())\n                    .map(|cap| {\n                        let cap = lines.byte_slice(cap.get_group(1).unwrap().range());\n                        InjectionLanguageMarker::Shebang(cap.into())\n                    })\n                    .next()\n            } else if index == self.injection_content_capture_index {\n                content_node = Some(capture.node);\n            }\n        }\n        (injection_capture, content_node)\n    }\n\n    fn injection_for_match<'a>(\n        &self,\n        query: &'a Query,\n        query_match: &QueryMatch<'a, 'a>,\n        source: RopeSlice<'a>,\n    ) -> (\n        Option<InjectionLanguageMarker<'a>>,\n        Option<Node<'a>>,\n        IncludedChildren,\n    ) {\n        let (mut injection_capture, content_node) = self.injection_pair(query_match, source);\n\n        let mut included_children = IncludedChildren::default();\n        for prop in query.property_settings(query_match.pattern_index) {\n            match prop.key.as_ref() {\n                // In addition to specifying the language name via the text of a\n                // captured node, it can also be hard-coded via a `#set!` predicate\n                // that sets the injection.language key.\n                \"injection.language\" if injection_capture.is_none() => {\n                    injection_capture = prop\n                        .value\n                        .as_ref()\n                        .map(|s| InjectionLanguageMarker::Name(s.as_ref().into()));\n                }\n\n                // By default, injections do not include the *children* of an\n                // `injection.content` node - only the ranges that belong to the\n                // node itself. This can be changed using a `#set!` predicate that\n                // sets the `injection.include-children` key.\n                \"injection.include-children\" => included_children = IncludedChildren::All,\n\n                // Some queries might only exclude named children but include unnamed\n                // children in their `injection.content` node. This can be enabled using\n                // a `#set!` predicate that sets the `injection.include-unnamed-children` key.\n                \"injection.include-unnamed-children\" => {\n                    included_children = IncludedChildren::Unnamed\n                }\n                _ => {}\n            }\n        }\n\n        (injection_capture, content_node, included_children)\n    }\n}\n\nimpl<'a> HighlightIterLayer<'a> {\n    // First, sort scope boundaries by their byte offset in the document. At a\n    // given position, emit scope endings before scope beginnings. Finally, emit\n    // scope boundaries from deeper layers first.\n    fn sort_key(&self) -> Option<(usize, bool, isize)> {\n        let depth = -(self.depth as isize);\n        let next_start = self\n            .captures\n            .borrow_mut()\n            .peek()\n            .map(|(m, i)| m.captures[*i].node.start_byte());\n        let next_end = self.highlight_end_stack.last().cloned();\n        match (next_start, next_end) {\n            (Some(start), Some(end)) => {\n                if start < end {\n                    Some((start, true, depth))\n                } else {\n                    Some((end, false, depth))\n                }\n            }\n            (Some(i), None) => Some((i, true, depth)),\n            (None, Some(j)) => Some((j, false, depth)),\n            _ => None,\n        }\n    }\n}\n\n#[derive(Clone)]\nenum IncludedChildren {\n    None,\n    All,\n    Unnamed,\n}\n\nimpl Default for IncludedChildren {\n    fn default() -> Self {\n        Self::None\n    }\n}\n\n// Compute the ranges that should be included when parsing an injection.\n// This takes into account three things:\n// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.\n// * `nodes` - Every injection takes place within a set of nodes. The injection ranges\n//   are the ranges of those nodes.\n// * `includes_children` - For some injections, the content nodes' children should be\n//   excluded from the nested document, so that only the content nodes' *own* content\n//   is reparsed. For other injections, the content nodes' entire ranges should be\n//   reparsed, including the ranges of their children.\nfn intersect_ranges(\n    parent_ranges: &[Range],\n    nodes: &[Node],\n    included_children: IncludedChildren,\n) -> Vec<Range> {\n    let mut cursor = nodes[0].walk();\n    let mut result = Vec::new();\n    let mut parent_range_iter = parent_ranges.iter();\n    let mut parent_range = parent_range_iter\n        .next()\n        .expect(\"Layers should only be constructed with non-empty ranges vectors\");\n    for node in nodes.iter() {\n        let mut preceding_range = Range {\n            start_byte: 0,\n            start_point: Point::new(0, 0),\n            end_byte: node.start_byte(),\n            end_point: node.start_position(),\n        };\n        let following_range = Range {\n            start_byte: node.end_byte(),\n            start_point: node.end_position(),\n            end_byte: usize::MAX,\n            end_point: Point::new(usize::MAX, usize::MAX),\n        };\n\n        for excluded_range in node\n            .children(&mut cursor)\n            .filter_map(|child| match included_children {\n                IncludedChildren::None => Some(child.range()),\n                IncludedChildren::All => None,\n                IncludedChildren::Unnamed => {\n                    if child.is_named() {\n                        Some(child.range())\n                    } else {\n                        None\n                    }\n                }\n            })\n            .chain([following_range].iter().cloned())\n        {\n            let mut range = Range {\n                start_byte: preceding_range.end_byte,\n                start_point: preceding_range.end_point,\n                end_byte: excluded_range.start_byte,\n                end_point: excluded_range.start_point,\n            };\n            preceding_range = excluded_range;\n\n            if range.end_byte < parent_range.start_byte {\n                continue;\n            }\n\n            while parent_range.start_byte <= range.end_byte {\n                if parent_range.end_byte > range.start_byte {\n                    if range.start_byte < parent_range.start_byte {\n                        range.start_byte = parent_range.start_byte;\n                        range.start_point = parent_range.start_point;\n                    }\n\n                    if parent_range.end_byte < range.end_byte {\n                        if range.start_byte < parent_range.end_byte {\n                            result.push(Range {\n                                start_byte: range.start_byte,\n                                start_point: range.start_point,\n                                end_byte: parent_range.end_byte,\n                                end_point: parent_range.end_point,\n                            });\n                        }\n                        range.start_byte = parent_range.end_byte;\n                        range.start_point = parent_range.end_point;\n                    } else {\n                        if range.start_byte < range.end_byte {\n                            result.push(range);\n                        }\n                        break;\n                    }\n                }\n\n                if let Some(next_range) = parent_range_iter.next() {\n                    parent_range = next_range;\n                } else {\n                    return result;\n                }\n            }\n        }\n    }\n    result\n}\n\nimpl<'a> HighlightIter<'a> {\n    fn emit_event(\n        &mut self,\n        offset: usize,\n        event: Option<HighlightEvent>,\n    ) -> Option<Result<HighlightEvent, Error>> {\n        let result;\n        if self.byte_offset < offset {\n            result = Some(Ok(HighlightEvent::Source {\n                start: self.byte_offset,\n                end: offset,\n            }));\n            self.byte_offset = offset;\n            self.next_event = event;\n        } else {\n            result = event.map(Ok);\n        }\n        self.sort_layers();\n        result\n    }\n\n    fn sort_layers(&mut self) {\n        while !self.layers.is_empty() {\n            if let Some(sort_key) = self.layers[0].sort_key() {\n                let mut i = 0;\n                while i + 1 < self.layers.len() {\n                    if let Some(next_offset) = self.layers[i + 1].sort_key() {\n                        if next_offset < sort_key {\n                            i += 1;\n                            continue;\n                        }\n                    } else {\n                        let layer = self.layers.remove(i + 1);\n                        PARSER.with(|ts_parser| {\n                            let highlighter = &mut ts_parser.borrow_mut();\n                            highlighter.cursors.push(layer.cursor);\n                        });\n                    }\n                    break;\n                }\n                if i > 0 {\n                    self.layers[0..(i + 1)].rotate_left(1);\n                }\n                break;\n            } else {\n                let layer = self.layers.remove(0);\n                PARSER.with(|ts_parser| {\n                    let highlighter = &mut ts_parser.borrow_mut();\n                    highlighter.cursors.push(layer.cursor);\n                });\n            }\n        }\n    }\n}\n\nimpl<'a> Iterator for HighlightIter<'a> {\n    type Item = Result<HighlightEvent, Error>;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        'main: loop {\n            // If we've already determined the next highlight boundary, just return it.\n            if let Some(e) = self.next_event.take() {\n                return Some(Ok(e));\n            }\n\n            // Periodically check for cancellation, returning `Cancelled` error if the\n            // cancellation flag was flipped.\n            if let Some(cancellation_flag) = self.cancellation_flag {\n                self.iter_count += 1;\n                if self.iter_count >= CANCELLATION_CHECK_INTERVAL {\n                    self.iter_count = 0;\n                    if cancellation_flag.load(Ordering::Relaxed) != 0 {\n                        return Some(Err(Error::Cancelled));\n                    }\n                }\n            }\n\n            // If none of the layers have any more highlight boundaries, terminate.\n            if self.layers.is_empty() {\n                let len = self.source.len_bytes();\n                return if self.byte_offset < len {\n                    let result = Some(Ok(HighlightEvent::Source {\n                        start: self.byte_offset,\n                        end: len,\n                    }));\n                    self.byte_offset = len;\n                    result\n                } else {\n                    None\n                };\n            }\n\n            // Get the next capture from whichever layer has the earliest highlight boundary.\n            let range;\n            let layer = &mut self.layers[0];\n            let captures = layer.captures.get_mut();\n            if let Some((next_match, capture_index)) = captures.peek() {\n                let next_capture = next_match.captures[*capture_index];\n                range = next_capture.node.byte_range();\n\n                // If any previous highlight ends before this node starts, then before\n                // processing this capture, emit the source code up until the end of the\n                // previous highlight, and an end event for that highlight.\n                if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {\n                    if end_byte <= range.start {\n                        layer.highlight_end_stack.pop();\n                        return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));\n                    }\n                }\n            }\n            // If there are no more captures, then emit any remaining highlight end events.\n            // And if there are none of those, then just advance to the end of the document.\n            else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {\n                layer.highlight_end_stack.pop();\n                return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));\n            } else {\n                return self.emit_event(self.source.len_bytes(), None);\n            };\n\n            let (mut match_, capture_index) = captures.next().unwrap();\n            let mut capture = match_.captures[capture_index];\n\n            // Remove from the local scope stack any local scopes that have already ended.\n            while range.start > layer.scope_stack.last().unwrap().range.end {\n                layer.scope_stack.pop();\n            }\n\n            // If this capture is for tracking local variables, then process the\n            // local variable info.\n            let mut reference_highlight = None;\n            let mut definition_highlight = None;\n            while match_.pattern_index < layer.config.highlights_pattern_index {\n                // If the node represents a local scope, push a new local scope onto\n                // the scope stack.\n                if Some(capture.index) == layer.config.local_scope_capture_index {\n                    definition_highlight = None;\n                    let mut scope = LocalScope {\n                        inherits: true,\n                        range: range.clone(),\n                        local_defs: Vec::new(),\n                    };\n                    for prop in layer.config.query.property_settings(match_.pattern_index) {\n                        if let \"local.scope-inherits\" = prop.key.as_ref() {\n                            scope.inherits =\n                                prop.value.as_ref().map_or(true, |r| r.as_ref() == \"true\");\n                        }\n                    }\n                    layer.scope_stack.push(scope);\n                }\n                // If the node represents a definition, add a new definition to the\n                // local scope at the top of the scope stack.\n                else if Some(capture.index) == layer.config.local_def_capture_index {\n                    reference_highlight = None;\n                    let scope = layer.scope_stack.last_mut().unwrap();\n\n                    let mut value_range = 0..0;\n                    for capture in match_.captures {\n                        if Some(capture.index) == layer.config.local_def_value_capture_index {\n                            value_range = capture.node.byte_range();\n                        }\n                    }\n\n                    let name = byte_range_to_str(range.clone(), self.source);\n                    scope.local_defs.push(LocalDef {\n                        name,\n                        value_range,\n                        highlight: None,\n                    });\n                    definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);\n                }\n                // If the node represents a reference, then try to find the corresponding\n                // definition in the scope stack.\n                else if Some(capture.index) == layer.config.local_ref_capture_index\n                    && definition_highlight.is_none()\n                {\n                    definition_highlight = None;\n                    let name = byte_range_to_str(range.clone(), self.source);\n                    for scope in layer.scope_stack.iter().rev() {\n                        if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {\n                            if def.name == name && range.start >= def.value_range.end {\n                                Some(def.highlight)\n                            } else {\n                                None\n                            }\n                        }) {\n                            reference_highlight = highlight;\n                            break;\n                        }\n                        if !scope.inherits {\n                            break;\n                        }\n                    }\n                }\n\n                // Continue processing any additional matches for the same node.\n                if let Some((next_match, next_capture_index)) = captures.peek() {\n                    let next_capture = next_match.captures[*next_capture_index];\n                    if next_capture.node == capture.node {\n                        capture = next_capture;\n                        match_ = captures.next().unwrap().0;\n                        continue;\n                    }\n                }\n\n                self.sort_layers();\n                continue 'main;\n            }\n\n            // Otherwise, this capture must represent a highlight.\n            // If this exact range has already been highlighted by an earlier pattern, or by\n            // a different layer, then skip over this one.\n            if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {\n                if range.start == last_start && range.end == last_end && layer.depth < last_depth {\n                    self.sort_layers();\n                    continue 'main;\n                }\n            }\n\n            // If the current node was found to be a local variable, then skip over any\n            // highlighting patterns that are disabled for local variables.\n            if definition_highlight.is_some() || reference_highlight.is_some() {\n                while layer.config.non_local_variable_patterns[match_.pattern_index] {\n                    match_.remove();\n                    if let Some((next_match, next_capture_index)) = captures.peek() {\n                        let next_capture = next_match.captures[*next_capture_index];\n                        if next_capture.node == capture.node {\n                            capture = next_capture;\n                            match_ = captures.next().unwrap().0;\n                            continue;\n                        }\n                    }\n\n                    self.sort_layers();\n                    continue 'main;\n                }\n            }\n\n            // Once a highlighting pattern is found for the current node, skip over\n            // any later highlighting patterns that also match this node. Captures\n            // for a given node are ordered by pattern index, so these subsequent\n            // captures are guaranteed to be for highlighting, not injections or\n            // local variables.\n            while let Some((next_match, next_capture_index)) = captures.peek() {\n                let next_capture = next_match.captures[*next_capture_index];\n                if next_capture.node == capture.node {\n                    captures.next();\n                } else {\n                    break;\n                }\n            }\n\n            let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];\n\n            // If this node represents a local definition, then store the current\n            // highlight value on the local scope entry representing this node.\n            if let Some(definition_highlight) = definition_highlight {\n                *definition_highlight = current_highlight;\n            }\n\n            // Emit a scope start event and push the node's end position to the stack.\n            if let Some(highlight) = reference_highlight.or(current_highlight) {\n                self.last_highlight_range = Some((range.start, range.end, layer.depth));\n                layer.highlight_end_stack.push(range.end);\n                return self\n                    .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));\n            }\n\n            self.sort_layers();\n        }\n    }\n}\n\n#[derive(Debug, Clone)]\npub enum InjectionLanguageMarker<'a> {\n    Name(Cow<'a, str>),\n    Filename(Cow<'a, Path>),\n    Shebang(String),\n}\n\nconst SHEBANG: &str = r\"#!\\s*(?:\\S*[/\\\\](?:env\\s+(?:\\-\\S+\\s+)*)?)?([^\\s\\.\\d]+)\";\n\npub struct Merge<I> {\n    iter: I,\n    spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,\n\n    next_event: Option<HighlightEvent>,\n    next_span: Option<(usize, std::ops::Range<usize>)>,\n\n    queue: Vec<HighlightEvent>,\n}\n\n/// Merge a list of spans into the highlight event stream.\npub fn merge<I: Iterator<Item = HighlightEvent>>(\n    iter: I,\n    spans: Vec<(usize, std::ops::Range<usize>)>,\n) -> Merge<I> {\n    let spans = Box::new(spans.into_iter());\n    let mut merge = Merge {\n        iter,\n        spans,\n        next_event: None,\n        next_span: None,\n        queue: Vec::new(),\n    };\n    merge.next_event = merge.iter.next();\n    merge.next_span = merge.spans.next();\n    merge\n}\n\nimpl<I: Iterator<Item = HighlightEvent>> Iterator for Merge<I> {\n    type Item = HighlightEvent;\n    fn next(&mut self) -> Option<Self::Item> {\n        use HighlightEvent::*;\n        if let Some(event) = self.queue.pop() {\n            return Some(event);\n        }\n\n        loop {\n            match (self.next_event, &self.next_span) {\n                // this happens when range is partially or fully offscreen\n                (Some(Source { start, .. }), Some((span, range))) if start > range.start => {\n                    if start > range.end {\n                        self.next_span = self.spans.next();\n                    } else {\n                        self.next_span = Some((*span, start..range.end));\n                    };\n                }\n                _ => break,\n            }\n        }\n\n        match (self.next_event, &self.next_span) {\n            (Some(HighlightStart(i)), _) => {\n                self.next_event = self.iter.next();\n                Some(HighlightStart(i))\n            }\n            (Some(HighlightEnd), _) => {\n                self.next_event = self.iter.next();\n                Some(HighlightEnd)\n            }\n            (Some(Source { start, end }), Some((_, range))) if start < range.start => {\n                let intersect = range.start.min(end);\n                let event = Source {\n                    start,\n                    end: intersect,\n                };\n\n                if end == intersect {\n                    // the event is complete\n                    self.next_event = self.iter.next();\n                } else {\n                    // subslice the event\n                    self.next_event = Some(Source {\n                        start: intersect,\n                        end,\n                    });\n                };\n\n                Some(event)\n            }\n            (Some(Source { start, end }), Some((span, range))) if start == range.start => {\n                let intersect = range.end.min(end);\n                let event = HighlightStart(Highlight(*span));\n\n                // enqueue in reverse order\n                self.queue.push(HighlightEnd);\n                self.queue.push(Source {\n                    start,\n                    end: intersect,\n                });\n\n                if end == intersect {\n                    // the event is complete\n                    self.next_event = self.iter.next();\n                } else {\n                    // subslice the event\n                    self.next_event = Some(Source {\n                        start: intersect,\n                        end,\n                    });\n                };\n\n                if intersect == range.end {\n                    self.next_span = self.spans.next();\n                } else {\n                    self.next_span = Some((*span, intersect..range.end));\n                }\n\n                Some(event)\n            }\n            (Some(event), None) => {\n                self.next_event = self.iter.next();\n                Some(event)\n            }\n            // Can happen if cursor at EOF and/or diagnostic reaches past the end.\n            // We need to actually emit events for the cursor-at-EOF situation,\n            // even though the range is past the end of the text.  This needs to be\n            // handled appropriately by the drawing code by not assuming that\n            // all `Source` events point to valid indices in the rope.\n            (None, Some((span, range))) => {\n                let event = HighlightStart(Highlight(*span));\n                self.queue.push(HighlightEnd);\n                self.queue.push(Source {\n                    start: range.start,\n                    end: range.end,\n                });\n                self.next_span = self.spans.next();\n                Some(event)\n            }\n            (None, None) => None,\n            e => unreachable!(\"{:?}\", e),\n        }\n    }\n}\n\nfn node_is_visible(node: &Node) -> bool {\n    node.is_missing() || (node.is_named() && node.language().node_kind_is_visible(node.kind_id()))\n}\n\npub fn pretty_print_tree<W: fmt::Write>(fmt: &mut W, node: Node) -> fmt::Result {\n    if node.child_count() == 0 {\n        if node_is_visible(&node) {\n            write!(fmt, \"({})\", node.kind())\n        } else {\n            write!(fmt, \"\\\"{}\\\"\", node.kind())\n        }\n    } else {\n        pretty_print_tree_impl(fmt, &mut node.walk(), 0)\n    }\n}\n\nfn pretty_print_tree_impl<W: fmt::Write>(\n    fmt: &mut W,\n    cursor: &mut tree_sitter::TreeCursor,\n    depth: usize,\n) -> fmt::Result {\n    let node = cursor.node();\n    let visible = node_is_visible(&node);\n\n    if visible {\n        let indentation_columns = depth * 2;\n        write!(fmt, \"{:indentation_columns$}\", \"\")?;\n\n        if let Some(field_name) = cursor.field_name() {\n            write!(fmt, \"{}: \", field_name)?;\n        }\n\n        write!(fmt, \"({}\", node.kind())?;\n    }\n\n    // Handle children.\n    if cursor.goto_first_child() {\n        loop {\n            if node_is_visible(&cursor.node()) {\n                fmt.write_char('\\n')?;\n            }\n\n            pretty_print_tree_impl(fmt, cursor, depth + 1)?;\n\n            if !cursor.goto_next_sibling() {\n                break;\n            }\n        }\n\n        let moved = cursor.goto_parent();\n        // The parent of the first child must exist, and must be `node`.\n        debug_assert!(moved);\n        debug_assert!(cursor.node() == node);\n    }\n\n    if visible {\n        fmt.write_char(')')?;\n    }\n\n    Ok(())\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::{Rope, Transaction};\n\n    #[test]\n    fn test_textobject_queries() {\n        let query_str = r#\"\n        (line_comment)+ @quantified_nodes\n        ((line_comment)+) @quantified_nodes_grouped\n        ((line_comment) (line_comment)) @multiple_nodes_grouped\n        \"#;\n        let source = Rope::from_str(\n            r#\"\n/// a comment on\n/// multiple lines\n        \"#,\n        );\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n        let language = get_language(\"rust\").unwrap();\n\n        let query = Query::new(&language, query_str).unwrap();\n        let textobject = TextObjectQuery { query };\n        let mut cursor = QueryCursor::new();\n\n        let config = HighlightConfiguration::new(language, \"\", \"\", \"\").unwrap();\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n\n        let root = syntax.tree().root_node();\n        let mut test = |capture, range| {\n            let matches: Vec<_> = textobject\n                .capture_nodes(capture, root, source.slice(..), &mut cursor)\n                .unwrap()\n                .collect();\n\n            assert_eq!(\n                matches[0].byte_range(),\n                range,\n                \"@{} expected {:?}\",\n                capture,\n                range\n            )\n        };\n\n        test(\"quantified_nodes\", 1..36);\n        // NOTE: Enable after implementing proper node group capturing\n        // test(\"quantified_nodes_grouped\", 1..36);\n        // test(\"multiple_nodes_grouped\", 1..36);\n    }\n\n    #[test]\n    fn test_parser() {\n        let highlight_names: Vec<String> = [\n            \"attribute\",\n            \"constant\",\n            \"function.builtin\",\n            \"function\",\n            \"keyword\",\n            \"operator\",\n            \"property\",\n            \"punctuation\",\n            \"punctuation.bracket\",\n            \"punctuation.delimiter\",\n            \"string\",\n            \"string.special\",\n            \"tag\",\n            \"type\",\n            \"type.builtin\",\n            \"variable\",\n            \"variable.builtin\",\n            \"variable.parameter\",\n        ]\n        .iter()\n        .cloned()\n        .map(String::from)\n        .collect();\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n\n        let language = get_language(\"rust\").unwrap();\n        let config = HighlightConfiguration::new(\n            language,\n            &std::fs::read_to_string(\"../runtime/grammars/sources/rust/queries/highlights.scm\")\n                .unwrap(),\n            &std::fs::read_to_string(\"../runtime/grammars/sources/rust/queries/injections.scm\")\n                .unwrap(),\n            \"\", // locals.scm\n        )\n        .unwrap();\n        config.configure(&highlight_names);\n\n        let source = Rope::from_str(\n            \"\n            struct Stuff {}\n            fn main() {}\n        \",\n        );\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n        let tree = syntax.tree();\n        let root = tree.root_node();\n        assert_eq!(root.kind(), \"source_file\");\n\n        assert_eq!(\n            root.to_sexp(),\n            concat!(\n                \"(source_file \",\n                \"(struct_item name: (type_identifier) body: (field_declaration_list)) \",\n                \"(function_item name: (identifier) parameters: (parameters) body: (block)))\"\n            )\n        );\n\n        let struct_node = root.child(0).unwrap();\n        assert_eq!(struct_node.kind(), \"struct_item\");\n    }\n\n    #[test]\n    fn test_input_edits() {\n        use tree_sitter::InputEdit;\n\n        let doc = Rope::from(\"hello world!\\ntest 123\");\n        let transaction = Transaction::change(\n            &doc,\n            vec![(6, 11, Some(\"test\".into())), (12, 17, None)].into_iter(),\n        );\n        let edits = generate_edits(doc.slice(..), transaction.changes());\n        // transaction.apply(&mut state);\n\n        assert_eq!(\n            edits,\n            &[\n                InputEdit {\n                    start_byte: 6,\n                    old_end_byte: 11,\n                    new_end_byte: 10,\n...\n", "needle_token_start": 7278, "needle_token_end": 7307, "code_context_ntokens": 16213, "output": ["```rust\npub fn names(&self) -> &[&str] {\n    self.query.capture_names()\n}\n```"]}
{"repo": "helix-editor/helix", "name": "stay_at_gaps", "language": "rust", "path": "helix-core/src/transaction.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function determines whether the cursor behavior should remain fixed between characters or not, based on the association mode provided.\n2. **Input**: The function takes an enumeration value representing the cursor's association mode.\n3. **Output**: Returns a boolean value; `true` if the cursor should stay fixed between characters, and `false` if it should move based on word boundaries.\n4. **Procedure**: The function checks the association mode. If the mode is neither related to preceding nor following word characters specifically, it returns `true`, indicating the cursor should stay fixed. Otherwise, it returns `false`, allowing the cursor to move based on adjacent word characters.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: helix-core/src/selection.rs\n//! Selections are the primary editing construct. Even cursors are\n//! defined as a selection range.\n//!\n//! All positioning is done via `char` offsets into the buffer.\nuse crate::{\n    graphemes::{\n        ensure_grapheme_boundary_next, ensure_grapheme_boundary_prev, next_grapheme_boundary,\n        prev_grapheme_boundary,\n    },\n    line_ending::get_line_ending,\n    movement::Direction,\n    Assoc, ChangeSet, RopeGraphemes, RopeSlice,\n};\nuse helix_stdx::rope::{self, RopeSliceExt};\nuse smallvec::{smallvec, SmallVec};\nuse std::borrow::Cow;\n\n/// A single selection range.\n///\n/// A range consists of an \"anchor\" and \"head\" position in\n/// the text.  The head is the part that the user moves when\n/// directly extending a selection.  The head and anchor\n/// can be in any order, or even share the same position.\n///\n/// The anchor and head positions use gap indexing, meaning\n/// that their indices represent the gaps *between* `char`s\n/// rather than the `char`s themselves. For example, 1\n/// represents the position between the first and second `char`.\n///\n/// Below are some examples of `Range` configurations.\n/// The anchor and head indices are shown as \"(anchor, head)\"\n/// tuples, followed by example text with \"[\" and \"]\" symbols\n/// representing the anchor and head positions:\n///\n/// - (0, 3): `[Som]e text`.\n/// - (3, 0): `]Som[e text`.\n/// - (2, 7): `So[me te]xt`.\n/// - (1, 1): `S[]ome text`.\n///\n/// Ranges are considered to be inclusive on the left and\n/// exclusive on the right, regardless of anchor-head ordering.\n/// This means, for example, that non-zero-width ranges that\n/// are directly adjacent, sharing an edge, do not overlap.\n/// However, a zero-width range will overlap with the shared\n/// left-edge of another range.\n///\n/// By convention, user-facing ranges are considered to have\n/// a block cursor on the head-side of the range that spans a\n/// single grapheme inward from the range's edge.  There are a\n/// variety of helper methods on `Range` for working in terms of\n/// that block cursor, all of which have `cursor` in their name.\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub struct Range {\n    /// The anchor of the range: the side that doesn't move when extending.\n    pub anchor: usize,\n    /// The head of the range, moved when extending.\n    pub head: usize,\n    /// The previous visual offset (softwrapped lines and columns) from\n    /// the start of the line\n    pub old_visual_position: Option<(u32, u32)>,\n}\n\nimpl Range {\n    pub fn new(anchor: usize, head: usize) -> Self {\n        Self {\n            anchor,\n            head,\n            old_visual_position: None,\n        }\n    }\n\n    pub fn point(head: usize) -> Self {\n        Self::new(head, head)\n    }\n\n    /// Start of the range.\n    #[inline]\n    #[must_use]\n    pub fn from(&self) -> usize {\n        std::cmp::min(self.anchor, self.head)\n    }\n\n    /// End of the range.\n    #[inline]\n    #[must_use]\n    pub fn to(&self) -> usize {\n        std::cmp::max(self.anchor, self.head)\n    }\n\n    /// Total length of the range.\n    #[inline]\n    #[must_use]\n    pub fn len(&self) -> usize {\n        self.to() - self.from()\n    }\n\n    /// The (inclusive) range of lines that the range overlaps.\n    #[inline]\n    #[must_use]\n    pub fn line_range(&self, text: RopeSlice) -> (usize, usize) {\n        let from = self.from();\n        let to = if self.is_empty() {\n            self.to()\n        } else {\n            prev_grapheme_boundary(text, self.to()).max(from)\n        };\n\n        (text.char_to_line(from), text.char_to_line(to))\n    }\n\n    /// `true` when head and anchor are at the same position.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.anchor == self.head\n    }\n\n    /// `Direction::Backward` when head < anchor.\n    /// `Direction::Backward` otherwise.\n    #[inline]\n    #[must_use]\n    pub fn direction(&self) -> Direction {\n        if self.head < self.anchor {\n            Direction::Backward\n        } else {\n            Direction::Forward\n        }\n    }\n\n    /// Flips the direction of the selection\n    pub fn flip(&self) -> Self {\n        Self {\n            anchor: self.head,\n            head: self.anchor,\n            old_visual_position: self.old_visual_position,\n        }\n    }\n\n    /// Returns the selection if it goes in the direction of `direction`,\n    /// flipping the selection otherwise.\n    pub fn with_direction(self, direction: Direction) -> Self {\n        if self.direction() == direction {\n            self\n        } else {\n            self.flip()\n        }\n    }\n\n    /// Check two ranges for overlap.\n    #[must_use]\n    pub fn overlaps(&self, other: &Self) -> bool {\n        // To my eye, it's non-obvious why this works, but I arrived\n        // at it after transforming the slower version that explicitly\n        // enumerated more cases.  The unit tests are thorough.\n        self.from() == other.from() || (self.to() > other.from() && other.to() > self.from())\n    }\n\n    #[inline]\n    pub fn contains_range(&self, other: &Self) -> bool {\n        self.from() <= other.from() && self.to() >= other.to()\n    }\n\n    pub fn contains(&self, pos: usize) -> bool {\n        self.from() <= pos && pos < self.to()\n    }\n\n    /// Map a range through a set of changes. Returns a new range representing\n    /// the same position after the changes are applied. Note that this\n    /// function runs in O(N) (N is number of changes) and can therefore\n    /// cause performance problems if run for a large number of ranges as the\n    /// complexity is then O(MN) (for multicuror M=N usually). Instead use\n    /// [Selection::map] or [ChangeSet::update_positions] instead\n    pub fn map(mut self, changes: &ChangeSet) -> Self {\n        use std::cmp::Ordering;\n        if changes.is_empty() {\n            return self;\n        }\n\n...\n// Path: helix-core/src/history.rs\nuse crate::{Assoc, ChangeSet, Range, Rope, Selection, Transaction};\nuse once_cell::sync::Lazy;\nuse regex::Regex;\nuse std::num::NonZeroUsize;\nuse std::time::{Duration, Instant};\n\n#[derive(Debug, Clone)]\npub struct State {\n    pub doc: Rope,\n    pub selection: Selection,\n}\n\n/// Stores the history of changes to a buffer.\n///\n/// Currently the history is represented as a vector of revisions. The vector\n/// always has at least one element: the empty root revision. Each revision\n/// with the exception of the root has a parent revision, a [Transaction]\n/// that can be applied to its parent to transition from the parent to itself,\n/// and an inversion of that transaction to transition from the parent to its\n/// latest child.\n///\n/// When using `u` to undo a change, an inverse of the stored transaction will\n/// be applied which will transition the buffer to the parent state.\n///\n/// Each revision with the exception of the last in the vector also has a\n/// last child revision. When using `U` to redo a change, the last child transaction\n/// will be applied to the current state of the buffer.\n///\n/// The current revision is the one currently displayed in the buffer.\n///\n/// Committing a new revision to the history will update the last child of the\n/// current revision, and push a new revision to the end of the vector.\n///\n/// Revisions are committed with a timestamp. :earlier and :later can be used\n/// to jump to the closest revision to a moment in time relative to the timestamp\n/// of the current revision plus (:later) or minus (:earlier) the duration\n/// given to the command. If a single integer is given, the editor will instead\n/// jump the given number of revisions in the vector.\n///\n/// Limitations:\n///  * Changes in selections currently don't commit history changes. The selection\n///    will only be updated to the state after a committed buffer change.\n///  * The vector of history revisions is currently unbounded. This might\n///    cause the memory consumption to grow significantly large during long\n///    editing sessions.\n///  * Because delete transactions currently don't store the text that they\n///    delete, we also store an inversion of the transaction.\n///\n/// Using time to navigate the history: <https://github.com/helix-editor/helix/pull/194>\n#[derive(Debug)]\npub struct History {\n    revisions: Vec<Revision>,\n    current: usize,\n}\n\n/// A single point in history. See [History] for more information.\n#[derive(Debug, Clone)]\nstruct Revision {\n    parent: usize,\n    last_child: Option<NonZeroUsize>,\n    transaction: Transaction,\n    // We need an inversion for undos because delete transactions don't store\n    // the deleted text.\n    inversion: Transaction,\n    timestamp: Instant,\n}\n\nimpl Default for History {\n    fn default() -> Self {\n        // Add a dummy root revision with empty transaction\n        Self {\n            revisions: vec![Revision {\n                parent: 0,\n                last_child: None,\n                transaction: Transaction::from(ChangeSet::new(\"\".into())),\n                inversion: Transaction::from(ChangeSet::new(\"\".into())),\n                timestamp: Instant::now(),\n            }],\n            current: 0,\n        }\n    }\n}\n\nimpl History {\n    pub fn commit_revision(&mut self, transaction: &Transaction, original: &State) {\n        self.commit_revision_at_timestamp(transaction, original, Instant::now());\n    }\n\n    pub fn commit_revision_at_timestamp(\n        &mut self,\n        transaction: &Transaction,\n        original: &State,\n        timestamp: Instant,\n    ) {\n        let inversion = transaction\n            .invert(&original.doc)\n            // Store the current cursor position\n            .with_selection(original.selection.clone());\n\n        let new_current = self.revisions.len();\n        self.revisions[self.current].last_child = NonZeroUsize::new(new_current);\n        self.revisions.push(Revision {\n            parent: self.current,\n            last_child: None,\n            transaction: transaction.clone(),\n            inversion,\n            timestamp,\n        });\n        self.current = new_current;\n    }\n\n    #[inline]\n    pub fn current_revision(&self) -> usize {\n        self.current\n    }\n\n    #[inline]\n    pub const fn at_root(&self) -> bool {\n        self.current == 0\n    }\n\n    /// Returns the changes since the given revision composed into a transaction.\n    /// Returns None if there are no changes between the current and given revisions.\n    pub fn changes_since(&self, revision: usize) -> Option<Transaction> {\n        let lca = self.lowest_common_ancestor(revision, self.current);\n        let up = self.path_up(revision, lca);\n        let down = self.path_up(self.current, lca);\n        let up_txns = up\n            .iter()\n            .rev()\n            .map(|&n| self.revisions[n].inversion.clone());\n        let down_txns = down.iter().map(|&n| self.revisions[n].transaction.clone());\n\n        down_txns.chain(up_txns).reduce(|acc, tx| tx.compose(acc))\n    }\n\n    /// Undo the last edit.\n    pub fn undo(&mut self) -> Option<&Transaction> {\n        if self.at_root() {\n            return None;\n        }\n\n        let current_revision = &self.revisions[self.current];\n        self.current = current_revision.parent;\n        Some(&current_revision.inversion)\n    }\n\n    /// Redo the last edit.\n    pub fn redo(&mut self) -> Option<&Transaction> {\n        let current_revision = &self.revisions[self.current];\n        let last_child = current_revision.last_child?;\n        self.current = last_child.get();\n\n        Some(&self.revisions[last_child.get()].transaction)\n    }\n\n    // Get the position of last change\n    pub fn last_edit_pos(&self) -> Option<usize> {\n        if self.current == 0 {\n            return None;\n        }\n        let current_revision = &self.revisions[self.current];\n        let primary_selection = current_revision\n            .inversion\n            .selection()\n            .expect(\"inversion always contains a selection\")\n            .primary();\n        let (_from, to, _fragment) = current_revision\n            .transaction\n            .changes_iter()\n            // find a change that matches the primary selection\n            .find(|(from, to, _fragment)| Range::new(*from, *to).overlaps(&primary_selection))\n            // or use the first change\n            .or_else(|| current_revision.transaction.changes_iter().next())\n            .unwrap();\n        let pos = current_revision\n            .transaction\n            .changes()\n            .map_pos(to, Assoc::After);\n        Some(pos)\n    }\n\n    fn lowest_common_ancestor(&self, mut a: usize, mut b: usize) -> usize {\n        use std::collections::HashSet;\n        let mut a_path_set = HashSet::new();\n        let mut b_path_set = HashSet::new();\n        loop {\n            a_path_set.insert(a);\n            b_path_set.insert(b);\n            if a_path_set.contains(&b) {\n                return b;\n            }\n            if b_path_set.contains(&a) {\n                return a;\n            }\n            a = self.revisions[a].parent; // Relies on the parent of 0 being 0.\n            b = self.revisions[b].parent; // Same as above.\n        }\n    }\n\n    /// List of nodes on the way from `n` to 'a`. Doesn't include `a`.\n    /// Includes `n` unless `a == n`. `a` must be an ancestor of `n`.\n    fn path_up(&self, mut n: usize, a: usize) -> Vec<usize> {\n        let mut path = Vec::new();\n        while n != a {\n            path.push(n);\n            n = self.revisions[n].parent;\n        }\n        path\n    }\n\n    /// Create a [`Transaction`] that will jump to a specific revision in the history.\n    fn jump_to(&mut self, to: usize) -> Vec<Transaction> {\n        let lca = self.lowest_common_ancestor(self.current, to);\n        let up = self.path_up(self.current, lca);\n        let down = self.path_up(to, lca);\n        self.current = to;\n        let up_txns = up.iter().map(|&n| self.revisions[n].inversion.clone());\n        let down_txns = down\n            .iter()\n            .rev()\n            .map(|&n| self.revisions[n].transaction.clone());\n        up_txns.chain(down_txns).collect()\n    }\n\n    /// Creates a [`Transaction`] that will undo `delta` revisions.\n    fn jump_backward(&mut self, delta: usize) -> Vec<Transaction> {\n        self.jump_to(self.current.saturating_sub(delta))\n    }\n\n    /// Creates a [`Transaction`] that will redo `delta` revisions.\n    fn jump_forward(&mut self, delta: usize) -> Vec<Transaction> {\n        self.jump_to(\n            self.current\n                .saturating_add(delta)\n                .min(self.revisions.len() - 1),\n        )\n    }\n\n    /// Helper for a binary search case below.\n    fn revision_closer_to_instant(&self, i: usize, instant: Instant) -> usize {\n        let dur_im1 = instant.duration_since(self.revisions[i - 1].timestamp);\n        let dur_i = self.revisions[i].timestamp.duration_since(instant);\n        use std::cmp::Ordering::*;\n        match dur_im1.cmp(&dur_i) {\n            Less => i - 1,\n            Equal | Greater => i,\n        }\n    }\n\n    /// Creates a [`Transaction`] that will match a revision created at around\n    /// `instant`.\n    fn jump_instant(&mut self, instant: Instant) -> Vec<Transaction> {\n        let search_result = self\n            .revisions\n            .binary_search_by(|rev| rev.timestamp.cmp(&instant));\n        let revision = match search_result {\n            Ok(revision) => revision,\n            Err(insert_point) => match insert_point {\n                0 => 0,\n                n if n == self.revisions.len() => n - 1,\n                i => self.revision_closer_to_instant(i, instant),\n            },\n        };\n        self.jump_to(revision)\n    }\n\n    /// Creates a [`Transaction`] that will match a revision created `duration` ago\n    /// from the timestamp of current revision.\n    fn jump_duration_backward(&mut self, duration: Duration) -> Vec<Transaction> {\n        match self.revisions[self.current].timestamp.checked_sub(duration) {\n            Some(instant) => self.jump_instant(instant),\n            None => self.jump_to(0),\n        }\n    }\n\n    /// Creates a [`Transaction`] that will match a revision created `duration` in\n    /// the future from the timestamp of the current revision.\n    fn jump_duration_forward(&mut self, duration: Duration) -> Vec<Transaction> {\n        match self.revisions[self.current].timestamp.checked_add(duration) {\n            Some(instant) => self.jump_instant(instant),\n            None => self.jump_to(self.revisions.len() - 1),\n        }\n    }\n\n    /// Creates an undo [`Transaction`].\n    pub fn earlier(&mut self, uk: UndoKind) -> Vec<Transaction> {\n        use UndoKind::*;\n        match uk {\n            Steps(n) => self.jump_backward(n),\n            TimePeriod(d) => self.jump_duration_backward(d),\n        }\n    }\n\n    /// Creates a redo [`Transaction`].\n    pub fn later(&mut self, uk: UndoKind) -> Vec<Transaction> {\n        use UndoKind::*;\n        match uk {\n            Steps(n) => self.jump_forward(n),\n            TimePeriod(d) => self.jump_duration_forward(d),\n        }\n    }\n}\n\n/// Whether to undo by a number of edits or a duration of time.\n#[derive(Debug, PartialEq, Eq, Clone, Copy)]\npub enum UndoKind {\n    Steps(usize),\n    TimePeriod(std::time::Duration),\n}\n\n/// A subset of systemd.time time span syntax units.\nconst TIME_UNITS: &[(&[&str], &str, u64)] = &[\n    (&[\"seconds\", \"second\", \"sec\", \"s\"], \"seconds\", 1),\n    (&[\"minutes\", \"minute\", \"min\", \"m\"], \"minutes\", 60),\n    (&[\"hours\", \"hour\", \"hr\", \"h\"], \"hours\", 60 * 60),\n    (&[\"days\", \"day\", \"d\"], \"days\", 24 * 60 * 60),\n];\n\n/// Checks if the duration input can be turned into a valid duration. It must be a\n/// positive integer and denote the [unit of time.](`TIME_UNITS`)\n/// Examples of valid durations:\n///  * `5 sec`\n///  * `5 min`\n///  * `5 hr`\n///  * `5 days`\nstatic DURATION_VALIDATION_REGEX: Lazy<Regex> =\n    Lazy::new(|| Regex::new(r\"^(?:\\d+\\s*[a-z]+\\s*)+$\").unwrap());\n\n/// Captures both the number and unit as separate capture groups.\nstatic NUMBER_UNIT_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r\"(\\d+)\\s*([a-z]+)\").unwrap());\n\n/// Parse a string (e.g. \"5 sec\") and try to convert it into a [`Duration`].\nfn parse_human_duration(s: &str) -> Result<Duration, String> {\n    if !DURATION_VALIDATION_REGEX.is_match(s) {\n        return Err(\"duration should be composed \\\n        of positive integers followed by time units\"\n            .to_string());\n    }\n\n    let mut specified = [false; TIME_UNITS.len()];\n    let mut seconds = 0u64;\n    for cap in NUMBER_UNIT_REGEX.captures_iter(s) {\n        let (n, unit_str) = (&cap[1], &cap[2]);\n\n        let n: u64 = n.parse().map_err(|_| format!(\"integer too large: {}\", n))?;\n\n        let time_unit = TIME_UNITS\n            .iter()\n            .enumerate()\n            .find(|(_, (forms, _, _))| forms.iter().any(|f| f == &unit_str));\n\n        if let Some((i, (_, unit, mul))) = time_unit {\n            if specified[i] {\n                return Err(format!(\"{} specified more than once\", unit));\n            }\n            specified[i] = true;\n\n            let new_seconds = n.checked_mul(*mul).and_then(|s| seconds.checked_add(s));\n            match new_seconds {\n                Some(ns) => seconds = ns,\n                None => return Err(\"duration too large\".to_string()),\n            }\n        } else {\n            return Err(format!(\"incorrect time unit: {}\", unit_str));\n        }\n    }\n\n    Ok(Duration::from_secs(seconds))\n}\n\nimpl std::str::FromStr for UndoKind {\n    type Err = String;\n\n    fn from_str(s: &str) -> Result<Self, Self::Err> {\n        let s = s.trim();\n        if s.is_empty() {\n            Ok(Self::Steps(1usize))\n        } else if let Ok(n) = s.parse::<usize>() {\n            Ok(UndoKind::Steps(n))\n        } else {\n            Ok(Self::TimePeriod(parse_human_duration(s)?))\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::Selection;\n\n    #[test]\n    fn test_undo_redo() {\n        let mut history = History::default();\n        let doc = Rope::from(\"hello\");\n        let mut state = State {\n            doc,\n            selection: Selection::point(0),\n        };\n\n        let transaction1 =\n            Transaction::change(&state.doc, vec![(5, 5, Some(\" world!\".into()))].into_iter());\n\n        // Need to commit before applying!\n        history.commit_revision(&transaction1, &state);\n        transaction1.apply(&mut state.doc);\n        assert_eq!(\"hello world!\", state.doc);\n\n        // ---\n\n        let transaction2 =\n            Transaction::change(&state.doc, vec![(6, 11, Some(\"\u4e16\u754c\".into()))].into_iter());\n\n        // Need to commit before applying!\n        history.commit_revision(&transaction2, &state);\n        transaction2.apply(&mut state.doc);\n        assert_eq!(\"hello \u4e16\u754c!\", state.doc);\n\n        // ---\n        fn undo(history: &mut History, state: &mut State) {\n            if let Some(transaction) = history.undo() {\n                transaction.apply(&mut state.doc);\n            }\n        }\n        fn redo(history: &mut History, state: &mut State) {\n            if let Some(transaction) = history.redo() {\n                transaction.apply(&mut state.doc);\n            }\n        }\n\n        undo(&mut history, &mut state);\n        assert_eq!(\"hello world!\", state.doc);\n        redo(&mut history, &mut state);\n        assert_eq!(\"hello \u4e16\u754c!\", state.doc);\n        undo(&mut history, &mut state);\n        undo(&mut history, &mut state);\n        assert_eq!(\"hello\", state.doc);\n\n        // undo at root is a no-op\n        undo(&mut history, &mut state);\n        assert_eq!(\"hello\", state.doc);\n    }\n\n    #[test]\n    fn test_earlier_later() {\n        let mut history = History::default();\n        let doc = Rope::from(\"a\\n\");\n        let mut state = State {\n            doc,\n            selection: Selection::point(0),\n        };\n\n        fn undo(history: &mut History, state: &mut State) {\n            if let Some(transaction) = history.undo() {\n                transaction.apply(&mut state.doc);\n            }\n        }\n\n        fn earlier(history: &mut History, state: &mut State, uk: UndoKind) {\n            let txns = history.earlier(uk);\n            for txn in txns {\n                txn.apply(&mut state.doc);\n            }\n        }\n\n        fn later(history: &mut History, state: &mut State, uk: UndoKind) {\n            let txns = history.later(uk);\n            for txn in txns {\n                txn.apply(&mut state.doc);\n            }\n        }\n\n        fn commit_change(\n            history: &mut History,\n            state: &mut State,\n            change: crate::transaction::Change,\n            instant: Instant,\n        ) {\n            let txn = Transaction::change(&state.doc, vec![change].into_iter());\n            history.commit_revision_at_timestamp(&txn, state, instant);\n            txn.apply(&mut state.doc);\n        }\n\n        let t0 = Instant::now();\n        let t = |n| t0.checked_add(Duration::from_secs(n)).unwrap();\n\n        commit_change(&mut history, &mut state, (1, 1, Some(\" b\".into())), t(0));\n        assert_eq!(\"a b\\n\", state.doc);\n\n        commit_change(&mut history, &mut state, (3, 3, Some(\" c\".into())), t(10));\n        assert_eq!(\"a b c\\n\", state.doc);\n\n        commit_change(&mut history, &mut state, (5, 5, Some(\" d\".into())), t(20));\n        assert_eq!(\"a b c d\\n\", state.doc);\n\n        undo(&mut history, &mut state);\n        assert_eq!(\"a b c\\n\", state.doc);\n\n        commit_change(&mut history, &mut state, (5, 5, Some(\" e\".into())), t(30));\n        assert_eq!(\"a b c e\\n\", state.doc);\n\n        undo(&mut history, &mut state);\n        undo(&mut history, &mut state);\n        assert_eq!(\"a b\\n\", state.doc);\n\n        commit_change(&mut history, &mut state, (1, 3, None), t(40));\n        assert_eq!(\"a\\n\", state.doc);\n\n        commit_change(&mut history, &mut state, (1, 1, Some(\" f\".into())), t(50));\n        assert_eq!(\"a f\\n\", state.doc);\n\n        use UndoKind::*;\n\n        earlier(&mut history, &mut state, Steps(3));\n        assert_eq!(\"a b c d\\n\", state.doc);\n\n        later(&mut history, &mut state, TimePeriod(Duration::new(20, 0)));\n        assert_eq!(\"a\\n\", state.doc);\n\n        earlier(&mut history, &mut state, TimePeriod(Duration::new(19, 0)));\n        assert_eq!(\"a b c d\\n\", state.doc);\n\n        earlier(\n            &mut history,\n            &mut state,\n            TimePeriod(Duration::new(10000, 0)),\n        );\n        assert_eq!(\"a\\n\", state.doc);\n\n        later(&mut history, &mut state, Steps(50));\n        assert_eq!(\"a f\\n\", state.doc);\n\n        earlier(&mut history, &mut state, Steps(4));\n        assert_eq!(\"a b c\\n\", state.doc);\n\n        later(&mut history, &mut state, TimePeriod(Duration::new(1, 0)));\n        assert_eq!(\"a b c\\n\", state.doc);\n\n        later(&mut history, &mut state, TimePeriod(Duration::new(5, 0)));\n        assert_eq!(\"a b c d\\n\", state.doc);\n\n        later(&mut history, &mut state, TimePeriod(Duration::new(6, 0)));\n        assert_eq!(\"a b c e\\n\", state.doc);\n\n        later(&mut history, &mut state, Steps(1));\n        assert_eq!(\"a\\n\", state.doc);\n    }\n\n    #[test]\n    fn test_parse_undo_kind() {\n        use UndoKind::*;\n\n        // Default is one step.\n        assert_eq!(\"\".parse(), Ok(Steps(1)));\n\n        // An integer means the number of steps.\n        assert_eq!(\"1\".parse(), Ok(Steps(1)));\n        assert_eq!(\"  16 \".parse(), Ok(Steps(16)));\n\n        // Duration has a strict format.\n        let validation_err = Err(\"duration should be composed \\\n         of positive integers followed by time units\"\n            .to_string());\n        assert_eq!(\"  16 33\".parse::<UndoKind>(), validation_err);\n        assert_eq!(\"  seconds 22  \".parse::<UndoKind>(), validation_err);\n        assert_eq!(\"  -4 m\".parse::<UndoKind>(), validation_err);\n        assert_eq!(\"5s 3\".parse::<UndoKind>(), validation_err);\n\n        // Units are u64.\n        assert_eq!(\n            \"18446744073709551616minutes\".parse::<UndoKind>(),\n            Err(\"integer too large: 18446744073709551616\".to_string())\n        );\n\n        // Units are validated.\n        assert_eq!(\n            \"1 millennium\".parse::<UndoKind>(),\n            Err(\"incorrect time unit: millennium\".to_string())\n        );\n\n        // Units can't be specified twice.\n        assert_eq!(\n            \"2 seconds 6s\".parse::<UndoKind>(),\n            Err(\"seconds specified more than once\".to_string())\n        );\n\n        // Various formats are correctly handled.\n        assert_eq!(\n            \"4s\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(4)))\n        );\n        assert_eq!(\n            \"2m\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(120)))\n        );\n        assert_eq!(\n            \"5h\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(5 * 60 * 60)))\n        );\n        assert_eq!(\n            \"3d\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(3 * 24 * 60 * 60)))\n        );\n        assert_eq!(\n            \"1m30s\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(90)))\n        );\n        assert_eq!(\n            \"1m 20 seconds\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(80)))\n        );\n        assert_eq!(\n            \"  2 minute 1day\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(24 * 60 * 60 + 2 * 60)))\n        );\n        assert_eq!(\n            \"3 d 2hour 5 minutes 30sec\".parse::<UndoKind>(),\n            Ok(TimePeriod(Duration::from_secs(\n                3 * 24 * 60 * 60 + 2 * 60 * 60 + 5 * 60 + 30\n            )))\n        );\n\n        // Sum overflow is handled.\n        assert_eq!(\n            \"18446744073709551615minutes\".parse::<UndoKind>(),\n            Err(\"duration too large\".to_string())\n        );\n        assert_eq!(\n            \"1 minute 18446744073709551615 seconds\".parse::<UndoKind>(),\n            Err(\"duration too large\".to_string())\n        );\n    }\n}\n\n// Path: helix-core/src/transaction.rs\nuse ropey::RopeSlice;\nuse smallvec::SmallVec;\n\nuse crate::{chars::char_is_word, Range, Rope, Selection, Tendril};\nuse std::{borrow::Cow, iter::once};\n\n/// (from, to, replacement)\npub type Change = (usize, usize, Option<Tendril>);\npub type Deletion = (usize, usize);\n\n// TODO: pub(crate)\n#[derive(Debug, Clone, PartialEq, Eq)]\npub enum Operation {\n    /// Move cursor by n characters.\n    Retain(usize),\n    /// Delete n characters.\n    Delete(usize),\n    /// Insert text at position.\n    Insert(Tendril),\n}\n\n#[derive(Debug, Copy, Clone, PartialEq, Eq)]\npub enum Assoc {\n    Before,\n    After,\n    /// Acts like `After` if a word character is inserted\n    /// after the position, otherwise acts like `Before`\n    AfterWord,\n    /// Acts like `Before` if a word character is inserted\n    /// before the position, otherwise acts like `After`\n    BeforeWord,\n}\n\nimpl Assoc {\n    /// Whether to stick to gaps\n    \nfn stay_at_gaps(self) -> bool {\n        !matches!(self, Self::BeforeWord | Self::AfterWord)\n    }\n\n    fn insert_offset(self, s: &str) -> usize {\n        let chars = s.chars().count();\n        match self {\n            Assoc::After => chars,\n            Assoc::AfterWord => s.chars().take_while(|&c| char_is_word(c)).count(),\n            // return position before inserted text\n            Assoc::Before => 0,\n            Assoc::BeforeWord => chars - s.chars().rev().take_while(|&c| char_is_word(c)).count(),\n        }\n    }\n}\n\n#[derive(Debug, Default, Clone, PartialEq, Eq)]\npub struct ChangeSet {\n    pub(crate) changes: Vec<Operation>,\n    /// The required document length. Will refuse to apply changes unless it matches.\n    len: usize,\n    len_after: usize,\n}\n\nimpl ChangeSet {\n    pub fn with_capacity(capacity: usize) -> Self {\n        Self {\n            changes: Vec::with_capacity(capacity),\n            len: 0,\n            len_after: 0,\n        }\n    }\n\n    #[must_use]\n    pub fn new(doc: RopeSlice) -> Self {\n        let len = doc.len_chars();\n        Self {\n            changes: Vec::new(),\n            len,\n            len_after: len,\n        }\n    }\n\n    // TODO: from iter\n\n    #[doc(hidden)] // used by lsp to convert to LSP changes\n    pub fn changes(&self) -> &[Operation] {\n        &self.changes\n    }\n\n    // Changeset builder operations: delete/insert/retain\n    pub(crate) fn delete(&mut self, n: usize) {\n        use Operation::*;\n        if n == 0 {\n            return;\n        }\n\n        self.len += n;\n\n        if let Some(Delete(count)) = self.changes.last_mut() {\n            *count += n;\n        } else {\n            self.changes.push(Delete(n));\n        }\n    }\n\n    pub(crate) fn insert(&mut self, fragment: Tendril) {\n        use Operation::*;\n\n        if fragment.is_empty() {\n            return;\n        }\n\n        // Avoiding std::str::len() to account for UTF-8 characters.\n        self.len_after += fragment.chars().count();\n\n        let new_last = match self.changes.as_mut_slice() {\n            [.., Insert(prev)] | [.., Insert(prev), Delete(_)] => {\n                prev.push_str(&fragment);\n                return;\n            }\n            [.., last @ Delete(_)] => std::mem::replace(last, Insert(fragment)),\n            _ => Insert(fragment),\n        };\n\n        self.changes.push(new_last);\n    }\n\n    pub(crate) fn retain(&mut self, n: usize) {\n        use Operation::*;\n        if n == 0 {\n            return;\n        }\n\n        self.len += n;\n        self.len_after += n;\n\n        if let Some(Retain(count)) = self.changes.last_mut() {\n            *count += n;\n        } else {\n            self.changes.push(Retain(n));\n        }\n    }\n\n    /// Combine two changesets together.\n    /// In other words,  If `this` goes `docA` \u2192 `docB` and `other` represents `docB` \u2192 `docC`, the\n    /// returned value will represent the change `docA` \u2192 `docC`.\n    pub fn compose(self, other: Self) -> Self {\n        assert!(self.len_after == other.len);\n\n        // composing fails in weird ways if one of the sets is empty\n        // a: [] len: 0 len_after: 1 | b: [Insert(Tendril<UTF8>(inline: \"\\n\")), Retain(1)] len 1\n        if self.changes.is_empty() {\n            return other;\n        }\n        if other.changes.is_empty() {\n            return self;\n        }\n\n        let len = self.changes.len();\n\n        let mut changes_a = self.changes.into_iter();\n        let mut changes_b = other.changes.into_iter();\n\n        let mut head_a = changes_a.next();\n        let mut head_b = changes_b.next();\n\n        let mut changes = Self::with_capacity(len); // TODO: max(a, b), shrink_to_fit() afterwards\n\n        loop {\n            use std::cmp::Ordering;\n            use Operation::*;\n            match (head_a, head_b) {\n                // we are done\n                (None, None) => {\n                    break;\n                }\n                // deletion in A\n                (Some(Delete(i)), b) => {\n                    changes.delete(i);\n                    head_a = changes_a.next();\n                    head_b = b;\n                }\n                // insertion in B\n                (a, Some(Insert(current))) => {\n                    changes.insert(current);\n                    head_a = a;\n                    head_b = changes_b.next();\n                }\n                (None, val) | (val, None) => unreachable!(\"({:?})\", val),\n                (Some(Retain(i)), Some(Retain(j))) => match i.cmp(&j) {\n                    Ordering::Less => {\n                        changes.retain(i);\n                        head_a = changes_a.next();\n                        head_b = Some(Retain(j - i));\n                    }\n                    Ordering::Equal => {\n                        changes.retain(i);\n                        head_a = changes_a.next();\n                        head_b = changes_b.next();\n                    }\n                    Ordering::Greater => {\n                        changes.retain(j);\n                        head_a = Some(Retain(i - j));\n                        head_b = changes_b.next();\n                    }\n                },\n                (Some(Insert(mut s)), Some(Delete(j))) => {\n                    let len = s.chars().count();\n                    match len.cmp(&j) {\n                        Ordering::Less => {\n                            head_a = changes_a.next();\n                            head_b = Some(Delete(j - len));\n                        }\n                        Ordering::Equal => {\n                            head_a = changes_a.next();\n                            head_b = changes_b.next();\n                        }\n                        Ordering::Greater => {\n                            // TODO: cover this with a test\n                            // figure out the byte index of the truncated string end\n                            let (pos, _) = s.char_indices().nth(j).unwrap();\n                            s.replace_range(0..pos, \"\");\n                            head_a = Some(Insert(s));\n                            head_b = changes_b.next();\n                        }\n                    }\n                }\n                (Some(Insert(s)), Some(Retain(j))) => {\n                    let len = s.chars().count();\n                    match len.cmp(&j) {\n                        Ordering::Less => {\n                            changes.insert(s);\n                            head_a = changes_a.next();\n                            head_b = Some(Retain(j - len));\n                        }\n                        Ordering::Equal => {\n                            changes.insert(s);\n                            head_a = changes_a.next();\n                            head_b = changes_b.next();\n                        }\n                        Ordering::Greater => {\n                            // figure out the byte index of the truncated string end\n                            let (pos, _) = s.char_indices().nth(j).unwrap();\n                            let mut before = s;\n                            let after = before.split_off(pos);\n\n                            changes.insert(before);\n                            head_a = Some(Insert(after));\n                            head_b = changes_b.next();\n                        }\n                    }\n                }\n                (Some(Retain(i)), Some(Delete(j))) => match i.cmp(&j) {\n                    Ordering::Less => {\n                        changes.delete(i);\n                        head_a = changes_a.next();\n                        head_b = Some(Delete(j - i));\n                    }\n                    Ordering::Equal => {\n                        changes.delete(j);\n                        head_a = changes_a.next();\n                        head_b = changes_b.next();\n                    }\n                    Ordering::Greater => {\n                        changes.delete(j);\n                        head_a = Some(Retain(i - j));\n                        head_b = changes_b.next();\n                    }\n                },\n            };\n        }\n\n        // starting len should still equal original starting len\n        debug_assert!(changes.len == self.len);\n\n        changes\n    }\n\n    /// Given another change set starting in the same document, maps this\n    /// change set over the other, producing a new change set that can be\n    /// applied to the document produced by applying `other`. When\n    /// `before` is `true`, order changes as if `this` comes before\n    /// `other`, otherwise (the default) treat `other` as coming first.\n    ///\n    /// Given two changes `A` and `B`, `A.compose(B.map(A))` and\n    /// `B.compose(A.map(B, true))` will produce the same document. This\n    /// provides a basic form of [operational\n    /// transformation](https://en.wikipedia.org/wiki/Operational_transformation),\n    /// and can be used for collaborative editing.\n    pub fn map(self, _other: Self) -> Self {\n        unimplemented!()\n    }\n\n    /// Returns a new changeset that reverts this one. Useful for `undo` implementation.\n    /// The document parameter expects the original document before this change was applied.\n    pub fn invert(&self, original_doc: &Rope) -> Self {\n        assert!(original_doc.len_chars() == self.len);\n\n        let mut changes = Self::with_capacity(self.changes.len());\n\n        let mut pos = 0;\n\n        for change in &self.changes {\n            use Operation::*;\n            match change {\n                Retain(n) => {\n                    changes.retain(*n);\n                    pos += n;\n                }\n                Delete(n) => {\n                    let text = Cow::from(original_doc.slice(pos..pos + *n));\n                    changes.insert(Tendril::from(text.as_ref()));\n                    pos += n;\n                }\n                Insert(s) => {\n                    let chars = s.chars().count();\n                    changes.delete(chars);\n                }\n            }\n        }\n\n        changes\n    }\n\n    /// Returns true if applied successfully.\n    pub fn apply(&self, text: &mut Rope) -> bool {\n        if text.len_chars() != self.len {\n            return false;\n        }\n\n        let mut pos = 0;\n\n        for change in &self.changes {\n            use Operation::*;\n            match change {\n                Retain(n) => {\n                    pos += n;\n                }\n                Delete(n) => {\n                    text.remove(pos..pos + *n);\n                    // pos += n;\n                }\n                Insert(s) => {\n                    text.insert(pos, s);\n                    pos += s.chars().count();\n                }\n            }\n        }\n        true\n    }\n\n    /// `true` when the set is empty.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.changes.is_empty() || self.changes == [Operation::Retain(self.len)]\n    }\n\n    /// Map a (mostly) *sorted* list of positions through the changes.\n    ///\n    /// This is equivalent to updating each position with `map_pos`:\n    ///\n    /// ``` no-compile\n    /// for (pos, assoc) in positions {\n    ///     *pos = changes.map_pos(*pos, assoc);\n    /// }\n    /// ```\n    /// However this function is significantly faster for sorted lists running\n    /// in `O(N+M)` instead of `O(NM)`. This function also handles unsorted/\n    /// partially sorted lists. However, in that case worst case complexity is\n    /// again `O(MN)`.  For lists that are often/mostly sorted (like the end of diagnostic ranges)\n    /// performance is usally close to `O(N + M)`\n    pub fn update_positions<'a>(&self, positions: impl Iterator<Item = (&'a mut usize, Assoc)>) {\n        use Operation::*;\n\n        let mut positions = positions.peekable();\n\n        let mut old_pos = 0;\n        let mut new_pos = 0;\n        let mut iter = self.changes.iter().enumerate().peekable();\n\n        'outer: loop {\n            macro_rules! map {\n                ($map: expr, $i: expr) => {\n                    loop {\n                        let Some((pos, assoc)) = positions.peek_mut() else { return; };\n                        if **pos < old_pos {\n                            // Positions are not sorted, revert to the last Operation that\n                            // contains this position and continue iterating from there.\n                            // We can unwrap here since `pos` can not be negative\n                            // (unsigned integer) and iterating backwards to the start\n                            // should always move us back to the start\n                            for (i, change) in self.changes[..$i].iter().enumerate().rev() {\n                                match change {\n                                    Retain(i) => {\n                                        old_pos -= i;\n                                        new_pos -= i;\n                                    }\n                                    Delete(i) => {\n                                        old_pos -= i;\n                                    }\n                                    Insert(ins) => {\n                                        new_pos -= ins.chars().count();\n                                    }\n                                }\n                                if old_pos <= **pos {\n                                    iter = self.changes[i..].iter().enumerate().peekable();\n                                }\n                            }\n                            debug_assert!(old_pos <= **pos, \"Reverse Iter across changeset works\");\n                            continue 'outer;\n                        }\n                        let Some(new_pos) = $map(**pos, *assoc) else { break; };\n                        **pos = new_pos;\n                        positions.next();\n                    }\n                };\n            }\n\n            let Some((i, change)) = iter.next() else {\n                map!(\n                    |pos, _| (old_pos == pos).then_some(new_pos),\n                    self.changes.len()\n                );\n                break;\n            };\n\n            let len = match change {\n                Delete(i) | Retain(i) => *i,\n                Insert(_) => 0,\n            };\n            let mut old_end = old_pos + len;\n\n            match change {\n                Retain(_) => {\n                    map!(\n                        |pos, _| (old_end > pos).then_some(new_pos + (pos - old_pos)),\n                        i\n                    );\n                    new_pos += len;\n                }\n                Delete(_) => {\n                    // in range\n                    map!(|pos, _| (old_end > pos).then_some(new_pos), i);\n                }\n                Insert(s) => {\n                    // a subsequent delete means a replace, consume it\n                    if let Some((_, Delete(len))) = iter.peek() {\n                        iter.next();\n\n                        old_end = old_pos + len;\n                        // in range of replaced text\n                        map!(\n                            |pos, assoc: Assoc| (old_end > pos).then(|| {\n                                // at point or tracking before\n                                if pos == old_pos && assoc.stay_at_gaps() {\n                                    new_pos\n                                } else {\n                                    // place to end of insert\n                                    new_pos + assoc.insert_offset(s)\n                                }\n                            }),\n                            i\n                        );\n                    } else {\n                        // at insert point\n                        map!(\n                            |pos, assoc: Assoc| (old_pos == pos).then(|| {\n                                // return position before inserted text\n                                new_pos + assoc.insert_offset(s)\n                            }),\n                            i\n                        );\n                    }\n\n                    new_pos += s.chars().count();\n                }\n            }\n            old_pos = old_end;\n        }\n        let out_of_bounds: Vec<_> = positions.collect();\n\n        panic!(\"Positions {out_of_bounds:?} are out of range for changeset len {old_pos}!\",)\n    }\n\n    /// Map a position through the changes.\n    ///\n    /// `assoc` indicates which side to associate the position with. `Before` will keep the\n    /// position close to the character before, and will place it before insertions over that\n    /// range, or at that point. `After` will move it forward, placing it at the end of such\n    /// insertions.\n    pub fn map_pos(&self, mut pos: usize, assoc: Assoc) -> usize {\n        self.update_positions(once((&mut pos, assoc)));\n        pos\n    }\n\n    pub fn changes_iter(&self) -> ChangeIterator {\n        ChangeIterator::new(self)\n    }\n}\n\n/// Transaction represents a single undoable unit of changes. Several changes can be grouped into\n/// a single transaction.\n#[derive(Debug, Default, Clone, PartialEq, Eq)]\npub struct Transaction {\n    changes: ChangeSet,\n    selection: Option<Selection>,\n}\n\nimpl Transaction {\n    /// Create a new, empty transaction.\n    pub fn new(doc: &Rope) -> Self {\n        Self {\n            changes: ChangeSet::new(doc.slice(..)),\n            selection: None,\n        }\n    }\n\n    /// Changes made to the buffer.\n    pub fn changes(&self) -> &ChangeSet {\n        &self.changes\n    }\n\n    /// When set, explicitly updates the selection.\n    pub fn selection(&self) -> Option<&Selection> {\n        self.selection.as_ref()\n    }\n\n    /// Returns true if applied successfully.\n    pub fn apply(&self, doc: &mut Rope) -> bool {\n        if self.changes.is_empty() {\n            return true;\n        }\n\n        // apply changes to the document\n        self.changes.apply(doc)\n    }\n\n    /// Generate a transaction that reverts this one.\n    pub fn invert(&self, original: &Rope) -> Self {\n        let changes = self.changes.invert(original);\n\n        Self {\n            changes,\n            selection: None,\n        }\n    }\n\n    pub fn compose(mut self, other: Self) -> Self {\n        self.changes = self.changes.compose(other.changes);\n        // Other selection takes precedence\n        self.selection = other.selection;\n        self\n    }\n\n    pub fn with_selection(mut self, selection: Selection) -> Self {\n        self.selection = Some(selection);\n        self\n    }\n\n    /// Generate a transaction from a set of potentially overlapping changes. The `change_ranges`\n    /// iterator yield the range (of removed text) in the old document for each edit. If any change\n    /// overlaps with a range overlaps with a previous range then that range is ignored.\n    ///\n    /// The `process_change` callback is called for each edit that is not ignored (in the order\n    /// yielded by `changes`) and should return the new text that the associated range will be\n    /// replaced with.\n    ///\n    /// To make this function more flexible the iterator can yield additional data for each change\n    /// that is passed to `process_change`\n    pub fn change_ignore_overlapping<T>(\n        doc: &Rope,\n        change_ranges: impl Iterator<Item = (usize, usize, T)>,\n        mut process_change: impl FnMut(usize, usize, T) -> Option<Tendril>,\n    ) -> Self {\n        let mut last = 0;\n        let changes = change_ranges.filter_map(|(from, to, data)| {\n            if from < last {\n                return None;\n            }\n            let tendril = process_change(from, to, data);\n            last = to;\n            Some((from, to, tendril))\n        });\n        Self::change(doc, changes)\n    }\n\n    /// Generate a transaction from a set of changes.\n    pub fn change<I>(doc: &Rope, changes: I) -> Self\n    where\n        I: Iterator<Item = Change>,\n    {\n        let len = doc.len_chars();\n\n        let (lower, upper) = changes.size_hint();\n        let size = upper.unwrap_or(lower);\n        let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate\n\n        let mut last = 0;\n        for (from, to, tendril) in changes {\n            // Verify ranges are ordered and not overlapping\n            debug_assert!(last <= from);\n            // Verify ranges are correct\n            debug_assert!(\n                from <= to,\n                \"Edit end must end before it starts (should {from} <= {to})\"\n            );\n\n            // Retain from last \"to\" to current \"from\"\n            changeset.retain(from - last);\n            let span = to - from;\n            match tendril {\n                Some(text) => {\n                    changeset.insert(text);\n                    changeset.delete(span);\n                }\n                None => changeset.delete(span),\n            }\n            last = to;\n        }\n\n        changeset.retain(len - last);\n\n        Self::from(changeset)\n    }\n\n    /// Generate a transaction from a set of potentially overlapping deletions\n    /// by merging overlapping deletions together.\n    pub fn delete<I>(doc: &Rope, deletions: I) -> Self\n    where\n        I: Iterator<Item = Deletion>,\n    {\n        let len = doc.len_chars();\n\n        let (lower, upper) = deletions.size_hint();\n        let size = upper.unwrap_or(lower);\n        let mut changeset = ChangeSet::with_capacity(2 * size + 1); // rough estimate\n\n        let mut last = 0;\n        for (mut from, to) in deletions {\n            if last > to {\n                continue;\n            }\n            if last > from {\n                from = last\n            }\n            debug_assert!(\n                from <= to,\n                \"Edit end must end before it starts (should {from} <= {to})\"\n            );\n            // Retain from last \"to\" to current \"from\"\n            changeset.retain(from - last);\n            changeset.delete(to - from);\n            last = to;\n        }\n\n        changeset.retain(len - last);\n\n        Self::from(changeset)\n    }\n\n    pub fn insert_at_eof(mut self, text: Tendril) -> Transaction {\n        self.changes.insert(text);\n        self\n    }\n\n    /// Generate a transaction with a change per selection range.\n    pub fn change_by_selection<F>(doc: &Rope, selection: &Selection, f: F) -> Self\n    where\n        F: FnMut(&Range) -> Change,\n    {\n        Self::change(doc, selection.iter().map(f))\n    }\n\n    pub fn change_by_selection_ignore_overlapping(\n        doc: &Rope,\n        selection: &Selection,\n        mut change_range: impl FnMut(&Range) -> (usize, usize),\n        mut create_tendril: impl FnMut(usize, usize) -> Option<Tendril>,\n    ) -> (Transaction, Selection) {\n        let mut last_selection_idx = None;\n        let mut new_primary_idx = None;\n        let mut ranges: SmallVec<[Range; 1]> = SmallVec::new();\n        let process_change = |change_start, change_end, (idx, range): (usize, &Range)| {\n            // update the primary idx\n            if idx == selection.primary_index() {\n                new_primary_idx = Some(idx);\n            } else if new_primary_idx.is_none() {\n                if idx > selection.primary_index() {\n                    new_primary_idx = last_selection_idx;\n                } else {\n                    last_selection_idx = Some(idx);\n                }\n            }\n            ranges.push(*range);\n            create_tendril(change_start, change_end)\n        };\n        let transaction = Self::change_ignore_overlapping(\n            doc,\n            selection.iter().enumerate().map(|range| {\n                let (change_start, change_end) = change_range(range.1);\n                (change_start, change_end, range)\n            }),\n            process_change,\n        );\n\n        (\n            transaction,\n            Selection::new(ranges, new_primary_idx.unwrap_or(0)),\n        )\n    }\n\n    /// Generate a transaction with a deletion per selection range.\n    /// Compared to using `change_by_selection` directly these ranges may overlap.\n    /// In that case they are merged\n    pub fn delete_by_selection<F>(doc: &Rope, selection: &Selection, f: F) -> Self\n    where\n        F: FnMut(&Range) -> Deletion,\n    {\n        Self::delete(doc, selection.iter().map(f))\n    }\n\n    /// Insert text at each selection head.\n    pub fn insert(doc: &Rope, selection: &Selection, text: Tendril) -> Self {\n        Self::change_by_selection(doc, selection, |range| {\n            (range.head, range.head, Some(text.clone()))\n        })\n    }\n\n    pub fn changes_iter(&self) -> ChangeIterator {\n        self.changes.changes_iter()\n    }\n}\n\nimpl From<ChangeSet> for Transaction {\n    fn from(changes: ChangeSet) -> Self {\n        Self {\n            changes,\n            selection: None,\n        }\n    }\n}\n\npub struct ChangeIterator<'a> {\n    iter: std::iter::Peekable<std::slice::Iter<'a, Operation>>,\n    pos: usize,\n}\n\nimpl<'a> ChangeIterator<'a> {\n    fn new(changeset: &'a ChangeSet) -> Self {\n        let iter = changeset.changes.iter().peekable();\n        Self { iter, pos: 0 }\n    }\n}\n\nimpl<'a> Iterator for ChangeIterator<'a> {\n    type Item = Change;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        use Operation::*;\n\n        loop {\n            match self.iter.next()? {\n                Retain(len) => {\n                    self.pos += len;\n                }\n                Delete(len) => {\n                    let start = self.pos;\n                    self.pos += len;\n                    return Some((start, self.pos, None));\n                }\n                Insert(s) => {\n                    let start = self.pos;\n                    // a subsequent delete means a replace, consume it\n                    if let Some(Delete(len)) = self.iter.peek() {\n                        self.iter.next();\n\n                        self.pos += len;\n                        return Some((start, self.pos, Some(s.clone())));\n                    } else {\n                        return Some((start, start, Some(s.clone())));\n                    }\n                }\n            }\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::history::State;\n\n    #[test]\n    fn composition() {\n        use Operation::*;\n\n        let a = ChangeSet {\n            changes: vec![\n                Retain(5),\n                Insert(\" test!\".into()),\n                Retain(1),\n                Delete(2),\n                Insert(\"abc\".into()),\n            ],\n            len: 8,\n            len_after: 15,\n        };\n\n        let b = ChangeSet {\n            changes: vec![Delete(10), Insert(\"\u4e16orld\".into()), Retain(5)],\n            len: 15,\n            len_after: 10,\n        };\n\n        let mut text = Rope::from(\"hello xz\");\n\n        // should probably return cloned text\n        let composed = a.compose(b);\n        assert_eq!(composed.len, 8);\n        assert!(composed.apply(&mut text));\n        assert_eq!(text, \"\u4e16orld! abc\");\n    }\n\n    #[test]\n    fn invert() {\n        use Operation::*;\n\n        let changes = ChangeSet {\n            changes: vec![Retain(4), Insert(\"test\".into()), Delete(5), Retain(3)],\n            len: 12,\n            len_after: 11,\n        };\n\n        let doc = Rope::from(\"\u4e16\u754c3 hello xz\");\n        let revert = changes.invert(&doc);\n\n...\n", "needle_token_start": 8885, "needle_token_end": 8918, "code_context_ntokens": 16175, "output": ["```rust\nfn stay_at_gaps(self) -> bool {\n    !matches!(self, Self::BeforeWord | Self::AfterWord)\n}\n```"]}
{"repo": "helix-editor/helix", "name": "test_invalid_date_times", "language": "rust", "path": "helix-core/src/increment/date_time.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To verify that the function correctly identifies and handles invalid date and time formats by not performing any increment operations on them.\n2. **Input**: A list of strings representing various incorrectly formatted or logically incorrect dates and times.\n3. **Output**: The function should return `None` for each invalid date or time string, indicating that no valid increment operation could be performed.\n4. **Procedure**: The function iterates over each test case, applies the increment operation with a fixed amount, and checks if the result is `None`, confirming the input was recognized as invalid.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: helix-core/src/syntax.rs\nmod tree_cursor;\n\nuse crate::{\n    auto_pairs::AutoPairs,\n    chars::char_is_line_ending,\n    diagnostic::Severity,\n    regex::Regex,\n    transaction::{ChangeSet, Operation},\n    RopeSlice, Tendril,\n};\n\nuse ahash::RandomState;\nuse arc_swap::{ArcSwap, Guard};\nuse bitflags::bitflags;\nuse globset::GlobSet;\nuse hashbrown::raw::RawTable;\nuse helix_stdx::rope::{self, RopeSliceExt};\nuse slotmap::{DefaultKey as LayerId, HopSlotMap};\n\nuse std::{\n    borrow::Cow,\n    cell::RefCell,\n    collections::{HashMap, HashSet, VecDeque},\n    fmt::{self, Display},\n    hash::{Hash, Hasher},\n    mem::replace,\n    path::{Path, PathBuf},\n    str::FromStr,\n    sync::Arc,\n};\n\nuse once_cell::sync::{Lazy, OnceCell};\nuse serde::{ser::SerializeSeq, Deserialize, Serialize};\n\nuse helix_loader::grammar::{get_language, load_runtime_file};\n\npub use tree_cursor::TreeCursor;\n\nfn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<String>::deserialize(deserializer)?\n        .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_lsp_config<'de, D>(deserializer: D) -> Result<Option<serde_json::Value>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<toml::Value>::deserialize(deserializer)?\n        .map(|toml| toml.try_into().map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_tab_width<'de, D>(deserializer: D) -> Result<usize, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    usize::deserialize(deserializer).and_then(|n| {\n        if n > 0 && n <= 16 {\n            Ok(n)\n        } else {\n            Err(serde::de::Error::custom(\n                \"tab width must be a value from 1 to 16 inclusive\",\n            ))\n        }\n    })\n}\n\npub fn deserialize_auto_pairs<'de, D>(deserializer: D) -> Result<Option<AutoPairs>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Ok(Option::<AutoPairConfig>::deserialize(deserializer)?.and_then(AutoPairConfig::into))\n}\n\nfn default_timeout() -> u64 {\n    20\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct Configuration {\n    pub language: Vec<LanguageConfiguration>,\n    #[serde(default)]\n    pub language_server: HashMap<String, LanguageServerConfiguration>,\n}\n\n// largely based on tree-sitter/cli/src/loader.rs\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\", deny_unknown_fields)]\npub struct LanguageConfiguration {\n    #[serde(rename = \"name\")]\n    pub language_id: String, // c-sharp, rust, tsx\n    #[serde(rename = \"language-id\")]\n    // see the table under https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocumentItem\n    pub language_server_language_id: Option<String>, // csharp, rust, typescriptreact, for the language-server\n    pub scope: String,             // source.rust\n    pub file_types: Vec<FileType>, // filename extension or ends_with? <Gemfile, rb, etc>\n    #[serde(default)]\n    pub shebangs: Vec<String>, // interpreter(s) associated with language\n    #[serde(default)]\n    pub roots: Vec<String>, // these indicate project roots <.git, Cargo.toml>\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_comment_tokens\",\n        alias = \"comment-token\"\n    )]\n    pub comment_tokens: Option<Vec<String>>,\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_block_comment_tokens\"\n    )]\n    pub block_comment_tokens: Option<Vec<BlockCommentToken>>,\n    pub text_width: Option<usize>,\n    pub soft_wrap: Option<SoftWrap>,\n\n    #[serde(default)]\n    pub auto_format: bool,\n\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub formatter: Option<FormatterConfiguration>,\n\n    #[serde(default)]\n    pub diagnostic_severity: Severity,\n\n    pub grammar: Option<String>, // tree-sitter grammar name, defaults to language_id\n\n    // content_regex\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_regex\")]\n    pub injection_regex: Option<Regex>,\n    // first_line_regex\n    //\n    #[serde(skip)]\n    pub(crate) highlight_config: OnceCell<Option<Arc<HighlightConfiguration>>>,\n    // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583\n    #[serde(\n        default,\n        skip_serializing_if = \"Vec::is_empty\",\n        serialize_with = \"serialize_lang_features\",\n        deserialize_with = \"deserialize_lang_features\"\n    )]\n    pub language_servers: Vec<LanguageServerFeatures>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub indent: Option<IndentationConfiguration>,\n\n    #[serde(skip)]\n    pub(crate) indent_query: OnceCell<Option<Query>>,\n    #[serde(skip)]\n    pub(crate) textobject_query: OnceCell<Option<TextObjectQuery>>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub debugger: Option<DebugAdapterConfig>,\n\n...\n// Path: helix-core/src/config.rs\nuse crate::syntax::{Configuration, Loader, LoaderError};\n\n/// Language configuration based on built-in languages.toml.\npub fn default_lang_config() -> Configuration {\n    helix_loader::config::default_lang_config()\n        .try_into()\n        .expect(\"Could not deserialize built-in languages.toml\")\n}\n\n/// Language configuration loader based on built-in languages.toml.\npub fn default_lang_loader() -> Loader {\n    Loader::new(default_lang_config()).expect(\"Could not compile loader for default config\")\n}\n\n#[derive(Debug)]\npub enum LanguageLoaderError {\n    DeserializeError(toml::de::Error),\n    LoaderError(LoaderError),\n}\n\nimpl std::fmt::Display for LanguageLoaderError {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        match self {\n            Self::DeserializeError(err) => write!(f, \"Failed to parse language config: {err}\"),\n            Self::LoaderError(err) => write!(f, \"Failed to compile language config: {err}\"),\n        }\n    }\n}\n\nimpl std::error::Error for LanguageLoaderError {}\n\n/// Language configuration based on user configured languages.toml.\npub fn user_lang_config() -> Result<Configuration, toml::de::Error> {\n    helix_loader::config::user_lang_config()?.try_into()\n}\n\n/// Language configuration loader based on user configured languages.toml.\npub fn user_lang_loader() -> Result<Loader, LanguageLoaderError> {\n    let config: Configuration = helix_loader::config::user_lang_config()\n        .map_err(LanguageLoaderError::DeserializeError)?\n        .try_into()\n        .map_err(LanguageLoaderError::DeserializeError)?;\n\n    Loader::new(config).map_err(LanguageLoaderError::LoaderError)\n}\n\n// Path: helix-core/src/comment.rs\n//! This module contains the functionality toggle comments on lines over the selection\n//! using the comment character defined in the user's `languages.toml`\n\nuse smallvec::SmallVec;\n\nuse crate::{\n    syntax::BlockCommentToken, Change, Range, Rope, RopeSlice, Selection, Tendril, Transaction,\n};\nuse helix_stdx::rope::RopeSliceExt;\nuse std::borrow::Cow;\n\n/// Given text, a comment token, and a set of line indices, returns the following:\n/// - Whether the given lines should be considered commented\n///     - If any of the lines are uncommented, all lines are considered as such.\n/// - The lines to change for toggling comments\n///     - This is all provided lines excluding blanks lines.\n/// - The column of the comment tokens\n///     - Column of existing tokens, if the lines are commented; column to place tokens at otherwise.\n/// - The margin to the right of the comment tokens\n///     - Defaults to `1`. If any existing comment token is not followed by a space, changes to `0`.\nfn find_line_comment(\n    token: &str,\n    text: RopeSlice,\n    lines: impl IntoIterator<Item = usize>,\n) -> (bool, Vec<usize>, usize, usize) {\n    let mut commented = true;\n    let mut to_change = Vec::new();\n    let mut min = usize::MAX; // minimum col for first_non_whitespace_char\n    let mut margin = 1;\n    let token_len = token.chars().count();\n    for line in lines {\n        let line_slice = text.line(line);\n        if let Some(pos) = line_slice.first_non_whitespace_char() {\n            let len = line_slice.len_chars();\n\n            if pos < min {\n                min = pos;\n            }\n\n            // line can be shorter than pos + token len\n            let fragment = Cow::from(line_slice.slice(pos..std::cmp::min(pos + token.len(), len)));\n\n            if fragment != token {\n                // as soon as one of the non-blank lines doesn't have a comment, the whole block is\n                // considered uncommented.\n                commented = false;\n            }\n\n            // determine margin of 0 or 1 for uncommenting; if any comment token is not followed by a space,\n            // a margin of 0 is used for all lines.\n            if !matches!(line_slice.get_char(pos + token_len), Some(c) if c == ' ') {\n                margin = 0;\n            }\n\n            // blank lines don't get pushed.\n            to_change.push(line);\n        }\n    }\n    (commented, to_change, min, margin)\n}\n\n#[must_use]\npub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&str>) -> Transaction {\n    let text = doc.slice(..);\n\n    let token = token.unwrap_or(\"//\");\n    let comment = Tendril::from(format!(\"{} \", token));\n\n    let mut lines: Vec<usize> = Vec::with_capacity(selection.len());\n\n    let mut min_next_line = 0;\n    for selection in selection {\n        let (start, end) = selection.line_range(text);\n        let start = start.clamp(min_next_line, text.len_lines());\n        let end = (end + 1).min(text.len_lines());\n\n        lines.extend(start..end);\n        min_next_line = end;\n    }\n\n    let (commented, to_change, min, margin) = find_line_comment(token, text, lines);\n\n    let mut changes: Vec<Change> = Vec::with_capacity(to_change.len());\n\n    for line in to_change {\n        let pos = text.line_to_char(line) + min;\n\n        if !commented {\n            // comment line\n            changes.push((pos, pos, Some(comment.clone())));\n        } else {\n            // uncomment line\n            changes.push((pos, pos + token.len() + margin, None));\n        }\n    }\n\n    Transaction::change(doc, changes.into_iter())\n}\n\n#[derive(Debug, PartialEq, Eq)]\npub enum CommentChange {\n    Commented {\n        range: Range,\n        start_pos: usize,\n        end_pos: usize,\n        start_margin: bool,\n        end_margin: bool,\n        start_token: String,\n        end_token: String,\n    },\n    Uncommented {\n        range: Range,\n        start_pos: usize,\n        end_pos: usize,\n        start_token: String,\n        end_token: String,\n    },\n    Whitespace {\n        range: Range,\n    },\n}\n\npub fn find_block_comments(\n    tokens: &[BlockCommentToken],\n    text: RopeSlice,\n    selection: &Selection,\n) -> (bool, Vec<CommentChange>) {\n    let mut commented = true;\n    let mut only_whitespace = true;\n    let mut comment_changes = Vec::with_capacity(selection.len());\n    let default_tokens = tokens.first().cloned().unwrap_or_default();\n    // TODO: check if this can be removed on MSRV bump\n    #[allow(clippy::redundant_clone)]\n    let mut start_token = default_tokens.start.clone();\n    #[allow(clippy::redundant_clone)]\n    let mut end_token = default_tokens.end.clone();\n\n    let mut tokens = tokens.to_vec();\n    // sort the tokens by length, so longer tokens will match first\n    tokens.sort_by(|a, b| {\n        if a.start.len() == b.start.len() {\n            b.end.len().cmp(&a.end.len())\n        } else {\n            b.start.len().cmp(&a.start.len())\n        }\n    });\n    for range in selection {\n        let selection_slice = range.slice(text);\n        if let (Some(start_pos), Some(end_pos)) = (\n            selection_slice.first_non_whitespace_char(),\n            selection_slice.last_non_whitespace_char(),\n        ) {\n            let mut line_commented = false;\n            let mut after_start = 0;\n            let mut before_end = 0;\n            let len = (end_pos + 1) - start_pos;\n\n            for BlockCommentToken { start, end } in &tokens {\n                let start_len = start.chars().count();\n                let end_len = end.chars().count();\n                after_start = start_pos + start_len;\n                before_end = end_pos.saturating_sub(end_len);\n\n                if len >= start_len + end_len {\n                    let start_fragment = selection_slice.slice(start_pos..after_start);\n                    let end_fragment = selection_slice.slice(before_end + 1..end_pos + 1);\n\n                    // block commented with these tokens\n                    if start_fragment == start.as_str() && end_fragment == end.as_str() {\n                        start_token = start.to_string();\n                        end_token = end.to_string();\n                        line_commented = true;\n                        break;\n                    }\n                }\n            }\n\n            if !line_commented {\n                comment_changes.push(CommentChange::Uncommented {\n                    range: *range,\n                    start_pos,\n                    end_pos,\n                    start_token: default_tokens.start.clone(),\n                    end_token: default_tokens.end.clone(),\n                });\n                commented = false;\n            } else {\n                comment_changes.push(CommentChange::Commented {\n                    range: *range,\n                    start_pos,\n                    end_pos,\n                    start_margin: selection_slice\n                        .get_char(after_start)\n                        .map_or(false, |c| c == ' '),\n                    end_margin: after_start != before_end\n                        && selection_slice\n                            .get_char(before_end)\n                            .map_or(false, |c| c == ' '),\n                    start_token: start_token.to_string(),\n                    end_token: end_token.to_string(),\n                });\n            }\n            only_whitespace = false;\n        } else {\n            comment_changes.push(CommentChange::Whitespace { range: *range });\n        }\n    }\n    if only_whitespace {\n        commented = false;\n    }\n    (commented, comment_changes)\n}\n\n#[must_use]\npub fn create_block_comment_transaction(\n    doc: &Rope,\n    selection: &Selection,\n    commented: bool,\n    comment_changes: Vec<CommentChange>,\n) -> (Transaction, SmallVec<[Range; 1]>) {\n    let mut changes: Vec<Change> = Vec::with_capacity(selection.len() * 2);\n    let mut ranges: SmallVec<[Range; 1]> = SmallVec::with_capacity(selection.len());\n    let mut offs = 0;\n    for change in comment_changes {\n        if commented {\n            if let CommentChange::Commented {\n                range,\n                start_pos,\n                end_pos,\n                start_token,\n                end_token,\n                start_margin,\n                end_margin,\n            } = change\n            {\n                let from = range.from();\n                changes.push((\n                    from + start_pos,\n                    from + start_pos + start_token.len() + start_margin as usize,\n                    None,\n                ));\n                changes.push((\n                    from + end_pos - end_token.len() - end_margin as usize + 1,\n                    from + end_pos + 1,\n                    None,\n                ));\n            }\n        } else {\n            // uncommented so manually map ranges through changes\n            match change {\n                CommentChange::Uncommented {\n                    range,\n                    start_pos,\n                    end_pos,\n                    start_token,\n                    end_token,\n                } => {\n                    let from = range.from();\n                    changes.push((\n                        from + start_pos,\n                        from + start_pos,\n                        Some(Tendril::from(format!(\"{} \", start_token))),\n                    ));\n                    changes.push((\n                        from + end_pos + 1,\n                        from + end_pos + 1,\n                        Some(Tendril::from(format!(\" {}\", end_token))),\n                    ));\n\n                    let offset = start_token.chars().count() + end_token.chars().count() + 2;\n                    ranges.push(\n                        Range::new(from + offs, from + offs + end_pos + 1 + offset)\n                            .with_direction(range.direction()),\n                    );\n                    offs += offset;\n                }\n                CommentChange::Commented { range, .. } | CommentChange::Whitespace { range } => {\n                    ranges.push(Range::new(range.from() + offs, range.to() + offs));\n                }\n            }\n        }\n    }\n    (Transaction::change(doc, changes.into_iter()), ranges)\n}\n\n#[must_use]\npub fn toggle_block_comments(\n    doc: &Rope,\n    selection: &Selection,\n    tokens: &[BlockCommentToken],\n) -> Transaction {\n    let text = doc.slice(..);\n    let (commented, comment_changes) = find_block_comments(tokens, text, selection);\n    let (mut transaction, ranges) =\n        create_block_comment_transaction(doc, selection, commented, comment_changes);\n    if !commented {\n        transaction = transaction.with_selection(Selection::new(ranges, selection.primary_index()));\n    }\n    transaction\n}\n\npub fn split_lines_of_selection(text: RopeSlice, selection: &Selection) -> Selection {\n    let mut ranges = SmallVec::new();\n    for range in selection.ranges() {\n        let (line_start, line_end) = range.line_range(text.slice(..));\n        let mut pos = text.line_to_char(line_start);\n        for line in text.slice(pos..text.line_to_char(line_end + 1)).lines() {\n            let start = pos;\n            pos += line.len_chars();\n            ranges.push(Range::new(start, pos));\n        }\n    }\n    Selection::new(ranges, 0)\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn test_find_line_comment() {\n        // four lines, two space indented, except for line 1 which is blank.\n        let mut doc = Rope::from(\"  1\\n\\n  2\\n  3\");\n        // select whole document\n        let mut selection = Selection::single(0, doc.len_chars() - 1);\n\n        let text = doc.slice(..);\n\n        let res = find_line_comment(\"//\", text, 0..3);\n        // (commented = true, to_change = [line 0, line 2], min = col 2, margin = 0)\n        assert_eq!(res, (false, vec![0, 2], 2, 0));\n\n        // comment\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n\n        assert_eq!(doc, \"  // 1\\n\\n  // 2\\n  // 3\");\n\n        // uncomment\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n        assert_eq!(doc, \"  1\\n\\n  2\\n  3\");\n        assert!(selection.len() == 1); // to ignore the selection unused warning\n\n        // 0 margin comments\n        doc = Rope::from(\"  //1\\n\\n  //2\\n  //3\");\n        // reset the selection.\n        selection = Selection::single(0, doc.len_chars() - 1);\n\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n        assert_eq!(doc, \"  1\\n\\n  2\\n  3\");\n        assert!(selection.len() == 1); // to ignore the selection unused warning\n\n        // 0 margin comments, with no space\n        doc = Rope::from(\"//\");\n        // reset the selection.\n        selection = Selection::single(0, doc.len_chars() - 1);\n\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n        assert_eq!(doc, \"\");\n        assert!(selection.len() == 1); // to ignore the selection unused warning\n\n        // TODO: account for uncommenting with uneven comment indentation\n    }\n\n    #[test]\n    fn test_find_block_comments() {\n        // three lines 5 characters.\n        let mut doc = Rope::from(\"1\\n2\\n3\");\n        // select whole document\n        let selection = Selection::single(0, doc.len_chars());\n\n        let text = doc.slice(..);\n\n        let res = find_block_comments(&[BlockCommentToken::default()], text, &selection);\n\n        assert_eq!(\n            res,\n            (\n                false,\n                vec![CommentChange::Uncommented {\n                    range: Range::new(0, 5),\n                    start_pos: 0,\n                    end_pos: 4,\n                    start_token: \"/*\".to_string(),\n                    end_token: \"*/\".to_string(),\n                }]\n            )\n        );\n\n        // comment\n        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);\n        transaction.apply(&mut doc);\n\n        assert_eq!(doc, \"/* 1\\n2\\n3 */\");\n\n        // uncomment\n        let selection = Selection::single(0, doc.len_chars());\n        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);\n        transaction.apply(&mut doc);\n        assert_eq!(doc, \"1\\n2\\n3\");\n\n        // don't panic when there is just a space in comment\n        doc = Rope::from(\"/* */\");\n        let selection = Selection::single(0, doc.len_chars());\n        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);\n        transaction.apply(&mut doc);\n        assert_eq!(doc, \"\");\n    }\n}\n\n// Path: helix-core/src/fuzzy.rs\nuse std::ops::DerefMut;\n\nuse nucleo::pattern::{Atom, AtomKind, CaseMatching};\nuse nucleo::Config;\nuse parking_lot::Mutex;\n\npub struct LazyMutex<T> {\n    inner: Mutex<Option<T>>,\n    init: fn() -> T,\n}\n\nimpl<T> LazyMutex<T> {\n    pub const fn new(init: fn() -> T) -> Self {\n        Self {\n            inner: Mutex::new(None),\n            init,\n        }\n    }\n\n    pub fn lock(&self) -> impl DerefMut<Target = T> + '_ {\n        parking_lot::MutexGuard::map(self.inner.lock(), |val| val.get_or_insert_with(self.init))\n    }\n}\n\npub static MATCHER: LazyMutex<nucleo::Matcher> = LazyMutex::new(nucleo::Matcher::default);\n\n/// convenience function to easily fuzzy match\n/// on a (relatively small list of inputs). This is not recommended for building a full tui\n/// application that can match large numbers of matches as all matching is done on the current\n/// thread, effectively blocking the UI\npub fn fuzzy_match<T: AsRef<str>>(\n    pattern: &str,\n    items: impl IntoIterator<Item = T>,\n    path: bool,\n) -> Vec<(T, u16)> {\n    let mut matcher = MATCHER.lock();\n    matcher.config = Config::DEFAULT;\n    if path {\n        matcher.config.set_match_paths();\n    }\n    let pattern = Atom::new(pattern, CaseMatching::Smart, AtomKind::Fuzzy, false);\n    pattern.match_list(items, &mut matcher)\n}\n\n// Path: helix-core/src/increment/date_time.rs\nuse chrono::{Duration, NaiveDate, NaiveDateTime, NaiveTime};\nuse once_cell::sync::Lazy;\nuse regex::Regex;\nuse std::fmt::Write;\n\n/// Increment a Date or DateTime\n///\n/// If just a Date is selected the day will be incremented.\n/// If a DateTime is selected the second will be incremented.\npub fn increment(selected_text: &str, amount: i64) -> Option<String> {\n    if selected_text.is_empty() {\n        return None;\n    }\n\n    FORMATS.iter().find_map(|format| {\n        let captures = format.regex.captures(selected_text)?;\n        if captures.len() - 1 != format.fields.len() {\n            return None;\n        }\n\n        let date_time = captures.get(0)?;\n        let has_date = format.fields.iter().any(|f| f.unit.is_date());\n        let has_time = format.fields.iter().any(|f| f.unit.is_time());\n        let date_time = &selected_text[date_time.start()..date_time.end()];\n        match (has_date, has_time) {\n            (true, true) => {\n                let date_time = NaiveDateTime::parse_from_str(date_time, format.fmt).ok()?;\n                Some(\n                    date_time\n                        .checked_add_signed(Duration::try_minutes(amount)?)?\n                        .format(format.fmt)\n                        .to_string(),\n                )\n            }\n            (true, false) => {\n                let date = NaiveDate::parse_from_str(date_time, format.fmt).ok()?;\n                Some(\n                    date.checked_add_signed(Duration::try_days(amount)?)?\n                        .format(format.fmt)\n                        .to_string(),\n                )\n            }\n            (false, true) => {\n                let time = NaiveTime::parse_from_str(date_time, format.fmt).ok()?;\n                let (adjusted_time, _) =\n                    time.overflowing_add_signed(Duration::try_minutes(amount)?);\n                Some(adjusted_time.format(format.fmt).to_string())\n            }\n            (false, false) => None,\n        }\n    })\n}\n\nstatic FORMATS: Lazy<Vec<Format>> = Lazy::new(|| {\n    vec![\n        Format::new(\"%Y-%m-%d %H:%M:%S\"), // 2021-11-24 07:12:23\n        Format::new(\"%Y/%m/%d %H:%M:%S\"), // 2021/11/24 07:12:23\n        Format::new(\"%Y-%m-%d %H:%M\"),    // 2021-11-24 07:12\n        Format::new(\"%Y/%m/%d %H:%M\"),    // 2021/11/24 07:12\n        Format::new(\"%Y-%m-%d\"),          // 2021-11-24\n        Format::new(\"%Y/%m/%d\"),          // 2021/11/24\n        Format::new(\"%a %b %d %Y\"),       // Wed Nov 24 2021\n        Format::new(\"%d-%b-%Y\"),          // 24-Nov-2021\n        Format::new(\"%Y %b %d\"),          // 2021 Nov 24\n        Format::new(\"%b %d, %Y\"),         // Nov 24, 2021\n        Format::new(\"%-I:%M:%S %P\"),      // 7:21:53 am\n        Format::new(\"%-I:%M %P\"),         // 7:21 am\n        Format::new(\"%-I:%M:%S %p\"),      // 7:21:53 AM\n        Format::new(\"%-I:%M %p\"),         // 7:21 AM\n        Format::new(\"%H:%M:%S\"),          // 23:24:23\n        Format::new(\"%H:%M\"),             // 23:24\n    ]\n});\n\n#[derive(Debug)]\nstruct Format {\n    fmt: &'static str,\n    fields: Vec<DateField>,\n    regex: Regex,\n    max_len: usize,\n}\n\nimpl Format {\n    fn new(fmt: &'static str) -> Self {\n        let mut remaining = fmt;\n        let mut fields = Vec::new();\n        let mut regex = \"^\".to_string();\n        let mut max_len = 0;\n\n        while let Some(i) = remaining.find('%') {\n            let after = &remaining[i + 1..];\n            let mut chars = after.chars();\n            let c = chars.next().unwrap();\n\n            let spec_len = if c == '-' {\n                1 + chars.next().unwrap().len_utf8()\n            } else {\n                c.len_utf8()\n            };\n\n            let specifier = &after[..spec_len];\n            let field = DateField::from_specifier(specifier).unwrap();\n            fields.push(field);\n            max_len += field.max_len + remaining[..i].len();\n            regex += &remaining[..i];\n            write!(regex, \"({})\", field.regex).unwrap();\n            remaining = &after[spec_len..];\n        }\n        regex += \"$\";\n\n        let regex = Regex::new(&regex).unwrap();\n\n        Self {\n            fmt,\n            fields,\n            regex,\n            max_len,\n        }\n    }\n}\n\nimpl PartialEq for Format {\n    fn eq(&self, other: &Self) -> bool {\n        self.fmt == other.fmt && self.fields == other.fields && self.max_len == other.max_len\n    }\n}\n\nimpl Eq for Format {}\n\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\nstruct DateField {\n    regex: &'static str,\n    unit: DateUnit,\n    max_len: usize,\n}\n\nimpl DateField {\n    fn from_specifier(specifier: &str) -> Option<Self> {\n        match specifier {\n            \"Y\" => Some(Self {\n                regex: r\"\\d{4}\",\n                unit: DateUnit::Years,\n                max_len: 5,\n            }),\n            \"y\" => Some(Self {\n                regex: r\"\\d\\d\",\n                unit: DateUnit::Years,\n                max_len: 2,\n            }),\n            \"m\" => Some(Self {\n                regex: r\"[0-1]\\d\",\n                unit: DateUnit::Months,\n                max_len: 2,\n            }),\n            \"d\" => Some(Self {\n                regex: r\"[0-3]\\d\",\n                unit: DateUnit::Days,\n                max_len: 2,\n            }),\n            \"-d\" => Some(Self {\n                regex: r\"[1-3]?\\d\",\n                unit: DateUnit::Days,\n                max_len: 2,\n            }),\n            \"a\" => Some(Self {\n                regex: r\"Sun|Mon|Tue|Wed|Thu|Fri|Sat\",\n                unit: DateUnit::Days,\n                max_len: 3,\n            }),\n            \"A\" => Some(Self {\n                regex: r\"Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday\",\n                unit: DateUnit::Days,\n                max_len: 9,\n            }),\n            \"b\" | \"h\" => Some(Self {\n                regex: r\"Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec\",\n                unit: DateUnit::Months,\n                max_len: 3,\n            }),\n            \"B\" => Some(Self {\n                regex: r\"January|February|March|April|May|June|July|August|September|October|November|December\",\n                unit: DateUnit::Months,\n                max_len: 9,\n            }),\n            \"H\" => Some(Self {\n                regex: r\"[0-2]\\d\",\n                unit: DateUnit::Hours,\n                max_len: 2,\n            }),\n            \"M\" => Some(Self {\n                regex: r\"[0-5]\\d\",\n                unit: DateUnit::Minutes,\n                max_len: 2,\n            }),\n            \"S\" => Some(Self {\n                regex: r\"[0-5]\\d\",\n                unit: DateUnit::Seconds,\n                max_len: 2,\n            }),\n            \"I\" => Some(Self {\n                regex: r\"[0-1]\\d\",\n                unit: DateUnit::Hours,\n                max_len: 2,\n            }),\n            \"-I\" => Some(Self {\n                regex: r\"1?\\d\",\n                unit: DateUnit::Hours,\n                max_len: 2,\n            }),\n            \"P\" => Some(Self {\n                regex: r\"am|pm\",\n                unit: DateUnit::AmPm,\n                max_len: 2,\n            }),\n            \"p\" => Some(Self {\n                regex: r\"AM|PM\",\n                unit: DateUnit::AmPm,\n                max_len: 2,\n            }),\n            _ => None,\n        }\n    }\n}\n\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\nenum DateUnit {\n    Years,\n    Months,\n    Days,\n    Hours,\n    Minutes,\n    Seconds,\n    AmPm,\n}\n\nimpl DateUnit {\n    fn is_date(self) -> bool {\n        matches!(self, DateUnit::Years | DateUnit::Months | DateUnit::Days)\n    }\n\n    fn is_time(self) -> bool {\n        matches!(\n            self,\n            DateUnit::Hours | DateUnit::Minutes | DateUnit::Seconds\n        )\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn test_increment_date_times() {\n        let tests = [\n            // (original, cursor, amount, expected)\n            (\"2020-02-28\", 1, \"2020-02-29\"),\n            (\"2020-02-29\", 1, \"2020-03-01\"),\n            (\"2020-01-31\", 1, \"2020-02-01\"),\n            (\"2020-01-20\", 1, \"2020-01-21\"),\n            (\"2021-01-01\", -1, \"2020-12-31\"),\n            (\"2021-01-31\", -2, \"2021-01-29\"),\n            (\"2020-02-28\", 1, \"2020-02-29\"),\n            (\"2021-02-28\", 1, \"2021-03-01\"),\n            (\"2021-03-01\", -1, \"2021-02-28\"),\n            (\"2020-02-29\", -1, \"2020-02-28\"),\n            (\"2020-02-20\", -1, \"2020-02-19\"),\n            (\"2021-03-01\", -1, \"2021-02-28\"),\n            (\"1980/12/21\", 100, \"1981/03/31\"),\n            (\"1980/12/21\", -100, \"1980/09/12\"),\n            (\"1980/12/21\", 1000, \"1983/09/17\"),\n            (\"1980/12/21\", -1000, \"1978/03/27\"),\n            (\"2021-11-24 07:12:23\", 1, \"2021-11-24 07:13:23\"),\n            (\"2021-11-24 07:12\", 1, \"2021-11-24 07:13\"),\n            (\"Wed Nov 24 2021\", 1, \"Thu Nov 25 2021\"),\n            (\"24-Nov-2021\", 1, \"25-Nov-2021\"),\n            (\"2021 Nov 24\", 1, \"2021 Nov 25\"),\n            (\"Nov 24, 2021\", 1, \"Nov 25, 2021\"),\n            (\"7:21:53 am\", 1, \"7:22:53 am\"),\n            (\"7:21:53 AM\", 1, \"7:22:53 AM\"),\n            (\"7:21 am\", 1, \"7:22 am\"),\n            (\"23:24:23\", 1, \"23:25:23\"),\n            (\"23:24\", 1, \"23:25\"),\n            (\"23:59\", 1, \"00:00\"),\n            (\"23:59:59\", 1, \"00:00:59\"),\n        ];\n\n        for (original, amount, expected) in tests {\n            assert_eq!(increment(original, amount).unwrap(), expected);\n        }\n    }\n\n    #[test]\n    \nfn test_invalid_date_times() {\n        let tests = [\n            \"0000-00-00\",\n            \"1980-2-21\",\n            \"1980-12-1\",\n            \"12345\",\n            \"2020-02-30\",\n            \"1999-12-32\",\n            \"19-12-32\",\n            \"1-2-3\",\n            \"0000/00/00\",\n            \"1980/2/21\",\n            \"1980/12/1\",\n            \"12345\",\n            \"2020/02/30\",\n            \"1999/12/32\",\n            \"19/12/32\",\n            \"1/2/3\",\n            \"123:456:789\",\n            \"11:61\",\n            \"2021-55-12 08:12:54\",\n        ];\n\n        for invalid in tests {\n            assert_eq!(increment(invalid, 1), None)\n        }\n    }\n}\n\n// Path: helix-core/src/increment/integer.rs\nconst SEPARATOR: char = '_';\n\n/// Increment an integer.\n///\n/// Supported bases:\n///     2 with prefix 0b\n///     8 with prefix 0o\n///     10 with no prefix\n///     16 with prefix 0x\n///\n/// An integer can contain `_` as a separator but may not start or end with a separator.\n/// Base 10 integers can go negative, but bases 2, 8, and 16 cannot.\n/// All addition and subtraction is saturating.\npub fn increment(selected_text: &str, amount: i64) -> Option<String> {\n    if selected_text.is_empty()\n        || selected_text.ends_with(SEPARATOR)\n        || selected_text.starts_with(SEPARATOR)\n    {\n        return None;\n    }\n\n    let radix = if selected_text.starts_with(\"0x\") {\n        16\n    } else if selected_text.starts_with(\"0o\") {\n        8\n    } else if selected_text.starts_with(\"0b\") {\n        2\n    } else {\n        10\n    };\n\n    // Get separator indexes from right to left.\n    let separator_rtl_indexes: Vec<usize> = selected_text\n        .chars()\n        .rev()\n        .enumerate()\n        .filter_map(|(i, c)| if c == SEPARATOR { Some(i) } else { None })\n        .collect();\n\n    let word: String = selected_text.chars().filter(|&c| c != SEPARATOR).collect();\n\n    let mut new_text = if radix == 10 {\n        let number = &word;\n        let value = i128::from_str_radix(number, radix).ok()?;\n        let new_value = value.saturating_add(amount as i128);\n\n        let format_length = match (value.is_negative(), new_value.is_negative()) {\n            (true, false) => number.len() - 1,\n            (false, true) => number.len() + 1,\n            _ => number.len(),\n        } - separator_rtl_indexes.len();\n\n        if number.starts_with('0') || number.starts_with(\"-0\") {\n            format!(\"{:01$}\", new_value, format_length)\n        } else {\n            format!(\"{}\", new_value)\n        }\n    } else {\n        let number = &word[2..];\n        let value = u128::from_str_radix(number, radix).ok()?;\n        let new_value = (value as i128).saturating_add(amount as i128);\n        let new_value = if new_value < 0 { 0 } else { new_value };\n        let format_length = selected_text.len() - 2 - separator_rtl_indexes.len();\n\n        match radix {\n            2 => format!(\"0b{:01$b}\", new_value, format_length),\n            8 => format!(\"0o{:01$o}\", new_value, format_length),\n            16 => {\n                let (lower_count, upper_count): (usize, usize) =\n                    number.chars().fold((0, 0), |(lower, upper), c| {\n                        (\n                            lower + c.is_ascii_lowercase() as usize,\n                            upper + c.is_ascii_uppercase() as usize,\n                        )\n                    });\n                if upper_count > lower_count {\n                    format!(\"0x{:01$X}\", new_value, format_length)\n                } else {\n                    format!(\"0x{:01$x}\", new_value, format_length)\n                }\n            }\n            _ => unimplemented!(\"radix not supported: {}\", radix),\n        }\n    };\n\n    // Add separators from original number.\n    for &rtl_index in &separator_rtl_indexes {\n        if rtl_index < new_text.len() {\n            let new_index = new_text.len().saturating_sub(rtl_index);\n            if new_index > 0 {\n                new_text.insert(new_index, SEPARATOR);\n            }\n        }\n    }\n\n    // Add in additional separators if necessary.\n    if new_text.len() > selected_text.len() && !separator_rtl_indexes.is_empty() {\n        let spacing = match separator_rtl_indexes.as_slice() {\n            [.., b, a] => a - b - 1,\n            _ => separator_rtl_indexes[0],\n        };\n\n        let prefix_length = if radix == 10 { 0 } else { 2 };\n        if let Some(mut index) = new_text.find(SEPARATOR) {\n            while index - prefix_length > spacing {\n                index -= spacing;\n                new_text.insert(index, SEPARATOR);\n            }\n        }\n    }\n\n    Some(new_text)\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn test_increment_basic_decimal_numbers() {\n        let tests = [\n            (\"100\", 1, \"101\"),\n            (\"100\", -1, \"99\"),\n            (\"99\", 1, \"100\"),\n            (\"100\", 1000, \"1100\"),\n            (\"100\", -1000, \"-900\"),\n            (\"-1\", 1, \"0\"),\n            (\"-1\", 2, \"1\"),\n            (\"1\", -1, \"0\"),\n            (\"1\", -2, \"-1\"),\n        ];\n\n        for (original, amount, expected) in tests {\n            assert_eq!(increment(original, amount).unwrap(), expected);\n        }\n    }\n\n    #[test]\n    fn test_increment_basic_hexadecimal_numbers() {\n        let tests = [\n            (\"0x0100\", 1, \"0x0101\"),\n            (\"0x0100\", -1, \"0x00ff\"),\n            (\"0x0001\", -1, \"0x0000\"),\n            (\"0x0000\", -1, \"0x0000\"),\n            (\"0xffffffffffffffff\", 1, \"0x10000000000000000\"),\n            (\"0xffffffffffffffff\", 2, \"0x10000000000000001\"),\n            (\"0xffffffffffffffff\", -1, \"0xfffffffffffffffe\"),\n            (\"0xABCDEF1234567890\", 1, \"0xABCDEF1234567891\"),\n            (\"0xabcdef1234567890\", 1, \"0xabcdef1234567891\"),\n        ];\n\n        for (original, amount, expected) in tests {\n            assert_eq!(increment(original, amount).unwrap(), expected);\n        }\n    }\n\n    #[test]\n    fn test_increment_basic_octal_numbers() {\n        let tests = [\n            (\"0o0107\", 1, \"0o0110\"),\n            (\"0o0110\", -1, \"0o0107\"),\n            (\"0o0001\", -1, \"0o0000\"),\n            (\"0o7777\", 1, \"0o10000\"),\n            (\"0o1000\", -1, \"0o0777\"),\n            (\"0o0107\", 10, \"0o0121\"),\n            (\"0o0000\", -1, \"0o0000\"),\n            (\"0o1777777777777777777777\", 1, \"0o2000000000000000000000\"),\n            (\"0o1777777777777777777777\", 2, \"0o2000000000000000000001\"),\n            (\"0o1777777777777777777777\", -1, \"0o1777777777777777777776\"),\n        ];\n\n        for (original, amount, expected) in tests {\n            assert_eq!(increment(original, amount).unwrap(), expected);\n        }\n    }\n\n    #[test]\n    fn test_increment_basic_binary_numbers() {\n        let tests = [\n            (\"0b00000100\", 1, \"0b00000101\"),\n            (\"0b00000100\", -1, \"0b00000011\"),\n            (\"0b00000100\", 2, \"0b00000110\"),\n            (\"0b00000100\", -2, \"0b00000010\"),\n            (\"0b00000001\", -1, \"0b00000000\"),\n            (\"0b00111111\", 10, \"0b01001001\"),\n            (\"0b11111111\", 1, \"0b100000000\"),\n            (\"0b10000000\", -1, \"0b01111111\"),\n            (\"0b0000\", -1, \"0b0000\"),\n            (\n                \"0b1111111111111111111111111111111111111111111111111111111111111111\",\n                1,\n                \"0b10000000000000000000000000000000000000000000000000000000000000000\",\n            ),\n            (\n                \"0b1111111111111111111111111111111111111111111111111111111111111111\",\n                2,\n                \"0b10000000000000000000000000000000000000000000000000000000000000001\",\n            ),\n            (\n                \"0b1111111111111111111111111111111111111111111111111111111111111111\",\n                -1,\n                \"0b1111111111111111111111111111111111111111111111111111111111111110\",\n            ),\n        ];\n\n        for (original, amount, expected) in tests {\n            assert_eq!(increment(original, amount).unwrap(), expected);\n        }\n    }\n\n    #[test]\n    fn test_increment_with_separators() {\n        let tests = [\n            (\"999_999\", 1, \"1_000_000\"),\n            (\"1_000_000\", -1, \"999_999\"),\n            (\"-999_999\", -1, \"-1_000_000\"),\n            (\"0x0000_0000_0001\", 0x1_ffff_0000, \"0x0001_ffff_0001\"),\n            (\"0x0000_0000\", -1, \"0x0000_0000\"),\n            (\"0x0000_0000_0000\", -1, \"0x0000_0000_0000\"),\n            (\"0b01111111_11111111\", 1, \"0b10000000_00000000\"),\n            (\"0b11111111_11111111\", 1, \"0b1_00000000_00000000\"),\n        ];\n\n        for (original, amount, expected) in tests {\n            assert_eq!(increment(original, amount).unwrap(), expected);\n        }\n    }\n\n    #[test]\n    fn test_leading_and_trailing_separators_arent_a_match() {\n        assert_eq!(increment(\"9_\", 1), None);\n        assert_eq!(increment(\"_9\", 1), None);\n        assert_eq!(increment(\"_9_\", 1), None);\n    }\n}\n\n// Path: helix-core/src/increment/mod.rs\nmod date_time;\nmod integer;\n\npub fn integer(selected_text: &str, amount: i64) -> Option<String> {\n    integer::increment(selected_text, amount)\n}\n\npub fn date_time(selected_text: &str, amount: i64) -> Option<String> {\n    date_time::increment(selected_text, amount)\n}\n\n// Path: helix-core/src/rope_reader.rs\nuse std::io;\n\nuse ropey::iter::Chunks;\nuse ropey::RopeSlice;\n\npub struct RopeReader<'a> {\n    current_chunk: &'a [u8],\n    chunks: Chunks<'a>,\n}\n\nimpl<'a> RopeReader<'a> {\n    pub fn new(rope: RopeSlice<'a>) -> RopeReader<'a> {\n        RopeReader {\n            current_chunk: &[],\n            chunks: rope.chunks(),\n        }\n    }\n}\n\nimpl io::Read for RopeReader<'_> {\n    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {\n        let buf_len = buf.len();\n        loop {\n            let read_bytes = self.current_chunk.read(buf)?;\n            buf = &mut buf[read_bytes..];\n            if buf.is_empty() {\n                return Ok(buf_len);\n            }\n\n            if let Some(next_chunk) = self.chunks.next() {\n                self.current_chunk = next_chunk.as_bytes();\n            } else {\n                return Ok(buf_len - buf.len());\n            }\n        }\n    }\n}\n\n// Path: helix-core/src/wrap.rs\nuse smartstring::{LazyCompact, SmartString};\nuse textwrap::{Options, WordSplitter::NoHyphenation};\n\n/// Given a slice of text, return the text re-wrapped to fit it\n/// within the given width.\npub fn reflow_hard_wrap(text: &str, text_width: usize) -> SmartString<LazyCompact> {\n    let options = Options::new(text_width).word_splitter(NoHyphenation);\n    textwrap::refill(text, options).into()\n}\n\n// Path: helix-core/src/indent.rs\nuse std::{borrow::Cow, collections::HashMap};\n\nuse helix_stdx::rope::RopeSliceExt;\nuse tree_sitter::{Query, QueryCursor, QueryPredicateArg};\n\nuse crate::{\n    chars::{char_is_line_ending, char_is_whitespace},\n    graphemes::{grapheme_width, tab_width_at},\n    syntax::{IndentationHeuristic, LanguageConfiguration, RopeProvider, Syntax},\n    tree_sitter::Node,\n    Position, Rope, RopeGraphemes, RopeSlice,\n};\n\n/// Enum representing indentation style.\n///\n/// Only values 1-8 are valid for the `Spaces` variant.\n#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]\npub enum IndentStyle {\n    Tabs,\n    Spaces(u8),\n}\n\n// 16 spaces\nconst INDENTS: &str = \"                \";\npub const MAX_INDENT: u8 = 16;\n\nimpl IndentStyle {\n    /// Creates an `IndentStyle` from an indentation string.\n    ///\n    /// For example, passing `\"    \"` (four spaces) will create `IndentStyle::Spaces(4)`.\n    #[allow(clippy::should_implement_trait)]\n    #[inline]\n    pub fn from_str(indent: &str) -> Self {\n        // XXX: do we care about validating the input more than this?  Probably not...?\n        debug_assert!(!indent.is_empty() && indent.len() <= MAX_INDENT as usize);\n\n        if indent.starts_with(' ') {\n            IndentStyle::Spaces(indent.len().clamp(1, MAX_INDENT as usize) as u8)\n        } else {\n            IndentStyle::Tabs\n        }\n    }\n\n    #[inline]\n    pub fn as_str(&self) -> &'static str {\n        match *self {\n            IndentStyle::Tabs => \"\\t\",\n            IndentStyle::Spaces(n) => {\n                // Unsupported indentation style.  This should never happen,\n                debug_assert!(n > 0 && n <= MAX_INDENT);\n\n                // Either way, clamp to the nearest supported value\n                let closest_n = n.clamp(1, MAX_INDENT) as usize;\n                &INDENTS[0..closest_n]\n            }\n        }\n    }\n\n    #[inline]\n    pub fn indent_width(&self, tab_width: usize) -> usize {\n        match *self {\n            IndentStyle::Tabs => tab_width,\n            IndentStyle::Spaces(width) => width as usize,\n        }\n    }\n}\n\n/// Attempts to detect the indentation style used in a document.\n///\n/// Returns the indentation style if the auto-detect confidence is\n/// reasonably high, otherwise returns `None`.\npub fn auto_detect_indent_style(document_text: &Rope) -> Option<IndentStyle> {\n    // Build a histogram of the indentation *increases* between\n    // subsequent lines, ignoring lines that are all whitespace.\n    //\n    // Index 0 is for tabs, the rest are 1-MAX_INDENT spaces.\n    let histogram: [usize; MAX_INDENT as usize + 1] = {\n        let mut histogram = [0; MAX_INDENT as usize + 1];\n        let mut prev_line_is_tabs = false;\n        let mut prev_line_leading_count = 0usize;\n\n        // Loop through the lines, checking for and recording indentation\n        // increases as we go.\n        'outer: for line in document_text.lines().take(1000) {\n            let mut c_iter = line.chars();\n\n            // Is first character a tab or space?\n            let is_tabs = match c_iter.next() {\n                Some('\\t') => true,\n                Some(' ') => false,\n\n                // Ignore blank lines.\n                Some(c) if char_is_line_ending(c) => continue,\n\n                _ => {\n                    prev_line_is_tabs = false;\n                    prev_line_leading_count = 0;\n                    continue;\n                }\n            };\n\n            // Count the line's total leading tab/space characters.\n            let mut leading_count = 1;\n            let mut count_is_done = false;\n            for c in c_iter {\n                match c {\n                    '\\t' if is_tabs && !count_is_done => leading_count += 1,\n                    ' ' if !is_tabs && !count_is_done => leading_count += 1,\n\n                    // We stop counting if we hit whitespace that doesn't\n                    // qualify as indent or doesn't match the leading\n                    // whitespace, but we don't exit the loop yet because\n                    // we still want to determine if the line is blank.\n                    c if char_is_whitespace(c) => count_is_done = true,\n\n                    // Ignore blank lines.\n                    c if char_is_line_ending(c) => continue 'outer,\n\n                    _ => break,\n                }\n\n                // Bound the worst-case execution time for weird text files.\n                if leading_count > 256 {\n                    continue 'outer;\n                }\n            }\n\n            // If there was an increase in indentation over the previous\n            // line, update the histogram with that increase.\n            if (prev_line_is_tabs == is_tabs || prev_line_leading_count == 0)\n                && prev_line_leading_count < leading_count\n            {\n                if is_tabs {\n                    histogram[0] += 1;\n                } else {\n                    let amount = leading_count - prev_line_leading_count;\n                    if amount <= MAX_INDENT as usize {\n                        histogram[amount] += 1;\n                    }\n                }\n            }\n\n            // Store this line's leading whitespace info for use with\n            // the next line.\n            prev_line_is_tabs = is_tabs;\n            prev_line_leading_count = leading_count;\n        }\n\n...\n", "needle_token_start": 10402, "needle_token_end": 10693, "code_context_ntokens": 16220, "output": ["```rust\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn test_invalid_date_times() {\n        let tests = [\n            \"0000-00-00\",\n            \"1980-2-21\",\n            \"1980-12-1\",\n            \"12345\",\n            \"2020-02-30\",\n            \"1999-12-32\",\n            \"19-12-32\",\n            \"1-2-3\",\n            \"0000/00/00\",\n            \"1980/2/21\",\n            \"1980/12/1\",\n            \"12345\",\n            \"2020/02/30\",\n            \"1999/12/32\",\n            \"19/12/32\",\n            \"1/2/3\",\n            \"123:456:789\",\n            \"11:61\",\n            \"2021-55-12 08:12:54\",\n        ];\n\n        for invalid in tests {\n            assert_eq!(increment(invalid, 1), None)\n        }\n    }\n}\n```"]}
{"repo": "helix-editor/helix", "name": "move_horizontally", "language": "rust", "path": "helix-core/src/movement.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to adjust the cursor position within a text document either forward or backward by a specified number of grapheme clusters, optionally extending the current text selection.\n2. **Input**: It takes a segment of text, a current cursor position or selection range, a direction to move (forward or backward), the number of grapheme clusters to move, a movement behavior (move or extend the selection), and additional formatting and annotation parameters.\n3. **Output**: It returns the new cursor position or text selection range after moving the specified number of grapheme clusters in the given direction.\n4. **Procedure**: The function first determines the current cursor position within the text. Based on the specified direction, it calculates the new cursor position by moving the required number of grapheme boundaries forward or backward. It then updates the selection range based on whether the movement should extend the current selection or just move the cursor, and returns this updated range.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: helix-core/src/doc_formatter.rs\n//! The `DocumentFormatter` forms the bridge between the raw document text\n//! and onscreen positioning. It yields the text graphemes as an iterator\n//! and traverses (part) of the document text. During that traversal it\n//! handles grapheme detection, softwrapping and annotations.\n//! It yields `FormattedGrapheme`s and their corresponding visual coordinates.\n//!\n//! As both virtual text and softwrapping can insert additional lines into the document\n//! it is generally not possible to find the start of the previous visual line.\n//! Instead the `DocumentFormatter` starts at the last \"checkpoint\" (usually a linebreak)\n//! called a \"block\" and the caller must advance it as needed.\n\nuse std::borrow::Cow;\nuse std::fmt::Debug;\nuse std::mem::{replace, take};\n\n#[cfg(test)]\nmod test;\n\nuse unicode_segmentation::{Graphemes, UnicodeSegmentation};\n\nuse crate::graphemes::{Grapheme, GraphemeStr};\nuse crate::syntax::Highlight;\nuse crate::text_annotations::TextAnnotations;\nuse crate::{Position, RopeGraphemes, RopeSlice};\n\n/// TODO make Highlight a u32 to reduce the size of this enum to a single word.\n#[derive(Debug, Clone, Copy)]\npub enum GraphemeSource {\n    Document {\n        codepoints: u32,\n    },\n    /// Inline virtual text can not be highlighted with a `Highlight` iterator\n    /// because it's not part of the document. Instead the `Highlight`\n    /// is emitted right by the document formatter\n    VirtualText {\n        highlight: Option<Highlight>,\n    },\n}\n\n#[derive(Debug, Clone)]\npub struct FormattedGrapheme<'a> {\n    pub grapheme: Grapheme<'a>,\n    pub source: GraphemeSource,\n}\n\nimpl<'a> FormattedGrapheme<'a> {\n    pub fn new(\n        g: GraphemeStr<'a>,\n        visual_x: usize,\n        tab_width: u16,\n        source: GraphemeSource,\n    ) -> FormattedGrapheme<'a> {\n        FormattedGrapheme {\n            grapheme: Grapheme::new(g, visual_x, tab_width),\n            source,\n        }\n    }\n    /// Returns whether this grapheme is virtual inline text\n    pub fn is_virtual(&self) -> bool {\n        matches!(self.source, GraphemeSource::VirtualText { .. })\n    }\n\n    pub fn placeholder() -> Self {\n        FormattedGrapheme {\n            grapheme: Grapheme::Other { g: \" \".into() },\n            source: GraphemeSource::Document { codepoints: 0 },\n        }\n    }\n\n    pub fn doc_chars(&self) -> usize {\n        match self.source {\n...\n// Path: helix-core/src/search.rs\nuse crate::RopeSlice;\n\n// TODO: switch to std::str::Pattern when it is stable.\npub trait CharMatcher {\n    fn char_match(&self, ch: char) -> bool;\n}\n\nimpl CharMatcher for char {\n    fn char_match(&self, ch: char) -> bool {\n        *self == ch\n    }\n}\n\nimpl<F: Fn(&char) -> bool> CharMatcher for F {\n    fn char_match(&self, ch: char) -> bool {\n        (*self)(&ch)\n    }\n}\n\npub fn find_nth_next<M: CharMatcher>(\n    text: RopeSlice,\n    char_matcher: M,\n    mut pos: usize,\n    n: usize,\n) -> Option<usize> {\n    if pos >= text.len_chars() || n == 0 {\n        return None;\n    }\n\n    let mut chars = text.chars_at(pos);\n\n    for _ in 0..n {\n        loop {\n            let c = chars.next()?;\n\n            pos += 1;\n\n            if char_matcher.char_match(c) {\n                break;\n            }\n        }\n    }\n\n    Some(pos - 1)\n}\n\npub fn find_nth_prev(text: RopeSlice, ch: char, mut pos: usize, n: usize) -> Option<usize> {\n    if pos == 0 || n == 0 {\n        return None;\n    }\n\n    let mut chars = text.chars_at(pos);\n\n    for _ in 0..n {\n        loop {\n            let c = chars.prev()?;\n\n            pos -= 1;\n\n            if c == ch {\n                break;\n            }\n        }\n    }\n\n    Some(pos)\n}\n\n// Path: helix-core/src/surround.rs\nuse std::fmt::Display;\n\nuse crate::{movement::Direction, search, Range, Selection};\nuse ropey::RopeSlice;\n\npub const PAIRS: &[(char, char)] = &[\n    ('(', ')'),\n    ('[', ']'),\n    ('{', '}'),\n    ('<', '>'),\n    ('\u00ab', '\u00bb'),\n    ('\u300c', '\u300d'),\n    ('\uff08', '\uff09'),\n];\n\n#[derive(Debug, PartialEq, Eq)]\npub enum Error {\n    PairNotFound,\n    CursorOverlap,\n    RangeExceedsText,\n    CursorOnAmbiguousPair,\n}\n\nimpl Display for Error {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        f.write_str(match *self {\n            Error::PairNotFound => \"Surround pair not found around all cursors\",\n            Error::CursorOverlap => \"Cursors overlap for a single surround pair range\",\n            Error::RangeExceedsText => \"Cursor range exceeds text length\",\n            Error::CursorOnAmbiguousPair => \"Cursor on ambiguous surround pair\",\n        })\n    }\n}\n\ntype Result<T> = std::result::Result<T, Error>;\n\n/// Given any char in [PAIRS], return the open and closing chars. If not found in\n/// [PAIRS] return (ch, ch).\n///\n/// ```\n/// use helix_core::surround::get_pair;\n///\n/// assert_eq!(get_pair('['), ('[', ']'));\n/// assert_eq!(get_pair('}'), ('{', '}'));\n/// assert_eq!(get_pair('\"'), ('\"', '\"'));\n/// ```\npub fn get_pair(ch: char) -> (char, char) {\n    PAIRS\n        .iter()\n        .find(|(open, close)| *open == ch || *close == ch)\n        .copied()\n        .unwrap_or((ch, ch))\n}\n\npub fn find_nth_closest_pairs_pos(\n    text: RopeSlice,\n    range: Range,\n    mut skip: usize,\n) -> Result<(usize, usize)> {\n    let is_open_pair = |ch| PAIRS.iter().any(|(open, _)| *open == ch);\n    let is_close_pair = |ch| PAIRS.iter().any(|(_, close)| *close == ch);\n\n    let mut stack = Vec::with_capacity(2);\n    let pos = range.from();\n    let mut close_pos = pos.saturating_sub(1);\n\n    for ch in text.chars_at(pos) {\n        close_pos += 1;\n\n        if is_open_pair(ch) {\n            // Track open pairs encountered so that we can step over\n            // the corresponding close pairs that will come up further\n            // down the loop. We want to find a lone close pair whose\n            // open pair is before the cursor position.\n            stack.push(ch);\n            continue;\n        }\n\n        if !is_close_pair(ch) {\n            // We don't care if this character isn't a brace pair item,\n            // so short circuit here.\n            continue;\n        }\n\n        let (open, close) = get_pair(ch);\n\n        if stack.last() == Some(&open) {\n            // If we are encountering the closing pair for an opener\n            // we just found while traversing, then its inside the\n            // selection and should be skipped over.\n            stack.pop();\n            continue;\n        }\n\n        match find_nth_open_pair(text, open, close, close_pos, 1) {\n            // Before we accept this pair, we want to ensure that the\n            // pair encloses the range rather than just the cursor.\n            Some(open_pos)\n                if open_pos <= pos.saturating_add(1)\n                    && close_pos >= range.to().saturating_sub(1) =>\n            {\n                // Since we have special conditions for when to\n                // accept, we can't just pass the skip parameter on\n                // through to the find_nth_*_pair methods, so we\n                // track skips manually here.\n                if skip > 1 {\n                    skip -= 1;\n                    continue;\n                }\n\n                return match range.direction() {\n                    Direction::Forward => Ok((open_pos, close_pos)),\n                    Direction::Backward => Ok((close_pos, open_pos)),\n                };\n            }\n            _ => continue,\n        }\n    }\n\n    Err(Error::PairNotFound)\n}\n\n/// Find the position of surround pairs of `ch` which can be either a closing\n/// or opening pair. `n` will skip n - 1 pairs (eg. n=2 will discard (only)\n/// the first pair found and keep looking)\npub fn find_nth_pairs_pos(\n    text: RopeSlice,\n    ch: char,\n    range: Range,\n    n: usize,\n) -> Result<(usize, usize)> {\n    if text.len_chars() < 2 {\n        return Err(Error::PairNotFound);\n    }\n    if range.to() >= text.len_chars() {\n        return Err(Error::RangeExceedsText);\n    }\n\n    let (open, close) = get_pair(ch);\n    let pos = range.cursor(text);\n\n    let (open, close) = if open == close {\n        if Some(open) == text.get_char(pos) {\n            // Cursor is directly on match char. We return no match\n            // because there's no way to know which side of the char\n            // we should be searching on.\n            return Err(Error::CursorOnAmbiguousPair);\n        }\n        (\n            search::find_nth_prev(text, open, pos, n),\n            search::find_nth_next(text, close, pos, n),\n        )\n    } else {\n        (\n            find_nth_open_pair(text, open, close, pos, n),\n            find_nth_close_pair(text, open, close, pos, n),\n        )\n    };\n\n    Option::zip(open, close).ok_or(Error::PairNotFound)\n}\n\nfn find_nth_open_pair(\n    text: RopeSlice,\n    open: char,\n    close: char,\n    mut pos: usize,\n    n: usize,\n) -> Option<usize> {\n    if pos >= text.len_chars() {\n        return None;\n    }\n\n    let mut chars = text.chars_at(pos + 1);\n\n    // Adjusts pos for the first iteration, and handles the case of the\n    // cursor being *on* the close character which will get falsely stepped over\n    // if not skipped here\n    if chars.prev()? == open {\n        return Some(pos);\n    }\n\n    for _ in 0..n {\n        let mut step_over: usize = 0;\n\n        loop {\n            let c = chars.prev()?;\n            pos = pos.saturating_sub(1);\n\n            // ignore other surround pairs that are enclosed *within* our search scope\n            if c == close {\n                step_over += 1;\n            } else if c == open {\n                if step_over == 0 {\n                    break;\n                }\n\n                step_over = step_over.saturating_sub(1);\n            }\n        }\n    }\n\n    Some(pos)\n}\n\nfn find_nth_close_pair(\n    text: RopeSlice,\n    open: char,\n    close: char,\n    mut pos: usize,\n    n: usize,\n) -> Option<usize> {\n    if pos >= text.len_chars() {\n        return None;\n    }\n\n    let mut chars = text.chars_at(pos);\n\n    if chars.next()? == close {\n        return Some(pos);\n    }\n\n    for _ in 0..n {\n        let mut step_over: usize = 0;\n\n        loop {\n            let c = chars.next()?;\n            pos += 1;\n\n            if c == open {\n                step_over += 1;\n            } else if c == close {\n                if step_over == 0 {\n                    break;\n                }\n\n                step_over = step_over.saturating_sub(1);\n            }\n        }\n    }\n\n    Some(pos)\n}\n\n/// Find position of surround characters around every cursor. Returns None\n/// if any positions overlap. Note that the positions are in a flat Vec.\n/// Use get_surround_pos().chunks(2) to get matching pairs of surround positions.\n/// `ch` can be either closing or opening pair. If `ch` is None, surround pairs\n/// are automatically detected around each cursor (note that this may result\n/// in them selecting different surround characters for each selection).\npub fn get_surround_pos(\n    text: RopeSlice,\n    selection: &Selection,\n    ch: Option<char>,\n    skip: usize,\n) -> Result<Vec<usize>> {\n    let mut change_pos = Vec::new();\n\n    for &range in selection {\n        let (open_pos, close_pos) = match ch {\n            Some(ch) => find_nth_pairs_pos(text, ch, range, skip)?,\n            None => find_nth_closest_pairs_pos(text, range, skip)?,\n        };\n        if change_pos.contains(&open_pos) || change_pos.contains(&close_pos) {\n            return Err(Error::CursorOverlap);\n        }\n        // ensure the positions are always paired in the forward direction\n        change_pos.extend_from_slice(&[open_pos.min(close_pos), close_pos.max(open_pos)]);\n    }\n    Ok(change_pos)\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::Range;\n\n    use ropey::Rope;\n    use smallvec::SmallVec;\n\n    #[test]\n    fn test_get_surround_pos() {\n        #[rustfmt::skip]\n        let (doc, selection, expectations) =\n            rope_with_selections_and_expectations(\n                \"(some) (chars)\\n(newline)\",\n                \"_ ^  _ _ ^   _\\n_    ^  _\"\n            );\n\n        assert_eq!(\n            get_surround_pos(doc.slice(..), &selection, Some('('), 1).unwrap(),\n            expectations\n        );\n    }\n\n    #[test]\n    fn test_get_surround_pos_bail_different_surround_chars() {\n        #[rustfmt::skip]\n        let (doc, selection, _) =\n            rope_with_selections_and_expectations(\n                \"[some]\\n(chars)xx\\n(newline)\",\n                \"  ^   \\n  ^      \\n         \"\n            );\n\n        assert_eq!(\n            get_surround_pos(doc.slice(..), &selection, Some('('), 1),\n            Err(Error::PairNotFound)\n        );\n    }\n\n    #[test]\n    fn test_get_surround_pos_bail_overlapping_surround_chars() {\n        #[rustfmt::skip]\n        let (doc, selection, _) =\n            rope_with_selections_and_expectations(\n                \"[some]\\n(chars)xx\\n(newline)\",\n                \"      \\n       ^ \\n      ^  \"\n            );\n\n        assert_eq!(\n            get_surround_pos(doc.slice(..), &selection, Some('('), 1),\n            Err(Error::PairNotFound) // overlapping surround chars\n        );\n    }\n\n    #[test]\n    fn test_get_surround_pos_bail_cursor_overlap() {\n        #[rustfmt::skip]\n        let (doc, selection, _) =\n            rope_with_selections_and_expectations(\n                \"[some]\\n(chars)xx\\n(newline)\",\n                \"  ^^  \\n         \\n         \"\n            );\n\n        assert_eq!(\n            get_surround_pos(doc.slice(..), &selection, Some('['), 1),\n            Err(Error::CursorOverlap)\n        );\n    }\n\n    #[test]\n    fn test_find_nth_pairs_pos_quote_success() {\n        #[rustfmt::skip]\n        let (doc, selection, expectations) =\n            rope_with_selections_and_expectations(\n                \"some 'quoted text' on this 'line'\\n'and this one'\",\n                \"     _        ^  _               \\n              \"\n            );\n\n        assert_eq!(2, expectations.len());\n        assert_eq!(\n            find_nth_pairs_pos(doc.slice(..), '\\'', selection.primary(), 1)\n                .expect(\"find should succeed\"),\n            (expectations[0], expectations[1])\n        )\n    }\n\n    #[test]\n    fn test_find_nth_pairs_pos_nested_quote_success() {\n        #[rustfmt::skip]\n        let (doc, selection, expectations) =\n            rope_with_selections_and_expectations(\n                \"some 'nested 'quoted' text' on this 'line'\\n'and this one'\",\n                \"     _           ^        _               \\n              \"\n            );\n\n        assert_eq!(2, expectations.len());\n        assert_eq!(\n            find_nth_pairs_pos(doc.slice(..), '\\'', selection.primary(), 2)\n                .expect(\"find should succeed\"),\n            (expectations[0], expectations[1])\n        )\n    }\n\n    #[test]\n    fn test_find_nth_pairs_pos_inside_quote_ambiguous() {\n        #[rustfmt::skip]\n        let (doc, selection, _) =\n            rope_with_selections_and_expectations(\n                \"some 'nested 'quoted' text' on this 'line'\\n'and this one'\",\n                \"                    ^                     \\n              \"\n            );\n\n        assert_eq!(\n            find_nth_pairs_pos(doc.slice(..), '\\'', selection.primary(), 1),\n            Err(Error::CursorOnAmbiguousPair)\n        )\n    }\n\n    #[test]\n    fn test_find_nth_closest_pairs_pos_index_range_panic() {\n        #[rustfmt::skip]\n        let (doc, selection, _) =\n            rope_with_selections_and_expectations(\n                \"(a)c)\",\n                \"^^^^^\"\n            );\n\n        assert_eq!(\n            find_nth_closest_pairs_pos(doc.slice(..), selection.primary(), 1),\n            Err(Error::PairNotFound)\n        )\n    }\n\n    // Create a Rope and a matching Selection using a specification language.\n    // ^ is a single-point selection.\n    // _ is an expected index. These are returned as a Vec<usize> for use in assertions.\n    fn rope_with_selections_and_expectations(\n        text: &str,\n        spec: &str,\n    ) -> (Rope, Selection, Vec<usize>) {\n        if text.len() != spec.len() {\n            panic!(\"specification must match text length -- are newlines aligned?\");\n        }\n\n        let rope = Rope::from(text);\n\n        let selections: SmallVec<[Range; 1]> = spec\n            .match_indices('^')\n            .map(|(i, _)| Range::point(i))\n            .collect();\n\n        let expectations: Vec<usize> = spec.match_indices('_').map(|(i, _)| i).collect();\n\n        (rope, Selection::new(selections, 0), expectations)\n    }\n}\n\n// Path: helix-core/src/textobject.rs\nuse std::fmt::Display;\n\nuse ropey::RopeSlice;\nuse tree_sitter::{Node, QueryCursor};\n\nuse crate::chars::{categorize_char, char_is_whitespace, CharCategory};\nuse crate::graphemes::{next_grapheme_boundary, prev_grapheme_boundary};\nuse crate::line_ending::rope_is_line_ending;\nuse crate::movement::Direction;\nuse crate::surround;\nuse crate::syntax::LanguageConfiguration;\nuse crate::Range;\n\nfn find_word_boundary(slice: RopeSlice, mut pos: usize, direction: Direction, long: bool) -> usize {\n    use CharCategory::{Eol, Whitespace};\n\n    let iter = match direction {\n        Direction::Forward => slice.chars_at(pos),\n        Direction::Backward => {\n            let mut iter = slice.chars_at(pos);\n            iter.reverse();\n            iter\n        }\n    };\n\n    let mut prev_category = match direction {\n        Direction::Forward if pos == 0 => Whitespace,\n        Direction::Forward => categorize_char(slice.char(pos - 1)),\n        Direction::Backward if pos == slice.len_chars() => Whitespace,\n        Direction::Backward => categorize_char(slice.char(pos)),\n    };\n\n    for ch in iter {\n        match categorize_char(ch) {\n            Eol | Whitespace => return pos,\n            category => {\n                if !long && category != prev_category && pos != 0 && pos != slice.len_chars() {\n                    return pos;\n                } else {\n                    match direction {\n                        Direction::Forward => pos += 1,\n                        Direction::Backward => pos = pos.saturating_sub(1),\n                    }\n                    prev_category = category;\n                }\n            }\n        }\n    }\n\n    pos\n}\n\n#[derive(Copy, Clone, PartialEq, Eq, Debug)]\npub enum TextObject {\n    Around,\n    Inside,\n    /// Used for moving between objects.\n    Movement,\n}\n\nimpl Display for TextObject {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        f.write_str(match self {\n            Self::Around => \"around\",\n            Self::Inside => \"inside\",\n            Self::Movement => \"movement\",\n        })\n    }\n}\n\n// count doesn't do anything yet\npub fn textobject_word(\n    slice: RopeSlice,\n    range: Range,\n    textobject: TextObject,\n    _count: usize,\n    long: bool,\n) -> Range {\n    let pos = range.cursor(slice);\n\n    let word_start = find_word_boundary(slice, pos, Direction::Backward, long);\n    let word_end = match slice.get_char(pos).map(categorize_char) {\n        None | Some(CharCategory::Whitespace | CharCategory::Eol) => pos,\n        _ => find_word_boundary(slice, pos + 1, Direction::Forward, long),\n    };\n\n    // Special case.\n    if word_start == word_end {\n        return Range::new(word_start, word_end);\n    }\n\n    match textobject {\n        TextObject::Inside => Range::new(word_start, word_end),\n        TextObject::Around => {\n            let whitespace_count_right = slice\n                .chars_at(word_end)\n                .take_while(|c| char_is_whitespace(*c))\n                .count();\n\n            if whitespace_count_right > 0 {\n                Range::new(word_start, word_end + whitespace_count_right)\n            } else {\n                let whitespace_count_left = {\n                    let mut iter = slice.chars_at(word_start);\n                    iter.reverse();\n                    iter.take_while(|c| char_is_whitespace(*c)).count()\n                };\n                Range::new(word_start - whitespace_count_left, word_end)\n            }\n        }\n        TextObject::Movement => unreachable!(),\n    }\n}\n\npub fn textobject_paragraph(\n    slice: RopeSlice,\n    range: Range,\n    textobject: TextObject,\n    count: usize,\n) -> Range {\n    let mut line = range.cursor_line(slice);\n    let prev_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));\n    let curr_line_empty = rope_is_line_ending(slice.line(line));\n    let next_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));\n    let last_char =\n        prev_grapheme_boundary(slice, slice.line_to_char(line + 1)) == range.cursor(slice);\n    let prev_empty_to_line = prev_line_empty && !curr_line_empty;\n    let curr_empty_to_line = curr_line_empty && !next_line_empty;\n\n    // skip character before paragraph boundary\n    let mut line_back = line; // line but backwards\n    if prev_empty_to_line || curr_empty_to_line {\n        line_back += 1;\n    }\n    // do not include current paragraph on paragraph end (include next)\n    if !(curr_empty_to_line && last_char) {\n        let mut lines = slice.lines_at(line_back);\n        lines.reverse();\n        let mut lines = lines.map(rope_is_line_ending).peekable();\n        while lines.next_if(|&e| e).is_some() {\n            line_back -= 1;\n        }\n        while lines.next_if(|&e| !e).is_some() {\n            line_back -= 1;\n        }\n    }\n\n    // skip character after paragraph boundary\n    if curr_empty_to_line && last_char {\n        line += 1;\n    }\n    let mut lines = slice.lines_at(line).map(rope_is_line_ending).peekable();\n    let mut count_done = 0; // count how many non-whitespace paragraphs done\n    for _ in 0..count {\n        let mut done = false;\n        while lines.next_if(|&e| !e).is_some() {\n            line += 1;\n            done = true;\n        }\n        while lines.next_if(|&e| e).is_some() {\n            line += 1;\n        }\n        count_done += done as usize;\n    }\n\n    // search one paragraph backwards for last paragraph\n    // makes `map` at the end of the paragraph with trailing newlines useful\n    let last_paragraph = count_done != count && lines.peek().is_none();\n    if last_paragraph {\n        let mut lines = slice.lines_at(line_back);\n        lines.reverse();\n        let mut lines = lines.map(rope_is_line_ending).peekable();\n        while lines.next_if(|&e| e).is_some() {\n            line_back -= 1;\n        }\n        while lines.next_if(|&e| !e).is_some() {\n            line_back -= 1;\n        }\n    }\n\n    // handle last whitespaces part separately depending on textobject\n    match textobject {\n        TextObject::Around => {}\n        TextObject::Inside => {\n            // remove last whitespace paragraph\n            let mut lines = slice.lines_at(line);\n            lines.reverse();\n            let mut lines = lines.map(rope_is_line_ending).peekable();\n            while lines.next_if(|&e| e).is_some() {\n                line -= 1;\n            }\n        }\n        TextObject::Movement => unreachable!(),\n    }\n\n    let anchor = slice.line_to_char(line_back);\n    let head = slice.line_to_char(line);\n    Range::new(anchor, head)\n}\n\npub fn textobject_pair_surround(\n    slice: RopeSlice,\n    range: Range,\n    textobject: TextObject,\n    ch: char,\n    count: usize,\n) -> Range {\n    textobject_pair_surround_impl(slice, range, textobject, Some(ch), count)\n}\n\npub fn textobject_pair_surround_closest(\n    slice: RopeSlice,\n    range: Range,\n    textobject: TextObject,\n    count: usize,\n) -> Range {\n    textobject_pair_surround_impl(slice, range, textobject, None, count)\n}\n\nfn textobject_pair_surround_impl(\n    slice: RopeSlice,\n    range: Range,\n    textobject: TextObject,\n    ch: Option<char>,\n    count: usize,\n) -> Range {\n    let pair_pos = match ch {\n        Some(ch) => surround::find_nth_pairs_pos(slice, ch, range, count),\n        // Automatically find the closest surround pairs\n        None => surround::find_nth_closest_pairs_pos(slice, range, count),\n    };\n    pair_pos\n        .map(|(anchor, head)| match textobject {\n            TextObject::Inside => {\n                if anchor < head {\n                    Range::new(next_grapheme_boundary(slice, anchor), head)\n                } else {\n                    Range::new(anchor, next_grapheme_boundary(slice, head))\n                }\n            }\n            TextObject::Around => {\n                if anchor < head {\n                    Range::new(anchor, next_grapheme_boundary(slice, head))\n                } else {\n                    Range::new(next_grapheme_boundary(slice, anchor), head)\n                }\n            }\n            TextObject::Movement => unreachable!(),\n        })\n        .unwrap_or(range)\n}\n\n/// Transform the given range to select text objects based on tree-sitter.\n/// `object_name` is a query capture base name like \"function\", \"class\", etc.\n/// `slice_tree` is the tree-sitter node corresponding to given text slice.\npub fn textobject_treesitter(\n    slice: RopeSlice,\n    range: Range,\n    textobject: TextObject,\n    object_name: &str,\n    slice_tree: Node,\n    lang_config: &LanguageConfiguration,\n    _count: usize,\n) -> Range {\n    let get_range = move || -> Option<Range> {\n        let byte_pos = slice.char_to_byte(range.cursor(slice));\n\n        let capture_name = format!(\"{}.{}\", object_name, textobject); // eg. function.inner\n        let mut cursor = QueryCursor::new();\n        let node = lang_config\n            .textobject_query()?\n            .capture_nodes(&capture_name, slice_tree, slice, &mut cursor)?\n            .filter(|node| node.byte_range().contains(&byte_pos))\n            .min_by_key(|node| node.byte_range().len())?;\n\n        let len = slice.len_bytes();\n        let start_byte = node.start_byte();\n        let end_byte = node.end_byte();\n        if start_byte >= len || end_byte >= len {\n            return None;\n        }\n\n        let start_char = slice.byte_to_char(start_byte);\n        let end_char = slice.byte_to_char(end_byte);\n\n        Some(Range::new(start_char, end_char))\n    };\n    get_range().unwrap_or(range)\n}\n\n#[cfg(test)]\nmod test {\n    use super::TextObject::*;\n    use super::*;\n\n    use crate::Range;\n    use ropey::Rope;\n\n    #[test]\n    fn test_textobject_word() {\n        // (text, [(char position, textobject, final range), ...])\n        let tests = &[\n            (\n                \"cursor at beginning of doc\",\n                vec![(0, Inside, (0, 6)), (0, Around, (0, 7))],\n            ),\n            (\n                \"cursor at middle of word\",\n                vec![\n                    (13, Inside, (10, 16)),\n                    (10, Inside, (10, 16)),\n                    (15, Inside, (10, 16)),\n                    (13, Around, (10, 17)),\n                    (10, Around, (10, 17)),\n                    (15, Around, (10, 17)),\n                ],\n            ),\n            (\n                \"cursor between word whitespace\",\n                vec![(6, Inside, (6, 6)), (6, Around, (6, 6))],\n            ),\n            (\n                \"cursor on word before newline\\n\",\n                vec![\n                    (22, Inside, (22, 29)),\n                    (28, Inside, (22, 29)),\n                    (25, Inside, (22, 29)),\n                    (22, Around, (21, 29)),\n                    (28, Around, (21, 29)),\n                    (25, Around, (21, 29)),\n                ],\n            ),\n            (\n                \"cursor on newline\\nnext line\",\n                vec![(17, Inside, (17, 17)), (17, Around, (17, 17))],\n            ),\n            (\n                \"cursor on word after newline\\nnext line\",\n                vec![\n                    (29, Inside, (29, 33)),\n                    (30, Inside, (29, 33)),\n                    (32, Inside, (29, 33)),\n                    (29, Around, (29, 34)),\n                    (30, Around, (29, 34)),\n                    (32, Around, (29, 34)),\n                ],\n            ),\n            (\n                \"cursor on #$%:;* punctuation\",\n                vec![\n                    (13, Inside, (10, 16)),\n                    (10, Inside, (10, 16)),\n                    (15, Inside, (10, 16)),\n                    (13, Around, (10, 17)),\n                    (10, Around, (10, 17)),\n                    (15, Around, (10, 17)),\n                ],\n            ),\n            (\n                \"cursor on punc%^#$:;.tuation\",\n                vec![\n                    (14, Inside, (14, 21)),\n                    (20, Inside, (14, 21)),\n                    (17, Inside, (14, 21)),\n                    (14, Around, (14, 21)),\n                    (20, Around, (14, 21)),\n                    (17, Around, (14, 21)),\n                ],\n            ),\n            (\n                \"cursor in   extra whitespace\",\n                vec![\n                    (9, Inside, (9, 9)),\n                    (10, Inside, (10, 10)),\n                    (11, Inside, (11, 11)),\n                    (9, Around, (9, 9)),\n                    (10, Around, (10, 10)),\n                    (11, Around, (11, 11)),\n                ],\n            ),\n            (\n                \"cursor on word   with extra whitespace\",\n                vec![(11, Inside, (10, 14)), (11, Around, (10, 17))],\n            ),\n            (\n                \"cursor at end with extra   whitespace\",\n                vec![(28, Inside, (27, 37)), (28, Around, (24, 37))],\n            ),\n            (\n                \"cursor at end of doc\",\n                vec![(19, Inside, (17, 20)), (19, Around, (16, 20))],\n            ),\n        ];\n\n        for (sample, scenario) in tests {\n            let doc = Rope::from(*sample);\n            let slice = doc.slice(..);\n            for &case in scenario {\n                let (pos, objtype, expected_range) = case;\n                // cursor is a single width selection\n                let range = Range::new(pos, pos + 1);\n                let result = textobject_word(slice, range, objtype, 1, false);\n                assert_eq!(\n                    result,\n                    expected_range.into(),\n                    \"\\nCase failed: {:?} - {:?}\",\n                    sample,\n                    case\n                );\n            }\n        }\n    }\n\n    #[test]\n    fn test_textobject_paragraph_inside_single() {\n        let tests = [\n            (\"#[|]#\", \"#[|]#\"),\n            (\"firs#[t|]#\\n\\nparagraph\\n\\n\", \"#[first\\n|]#\\nparagraph\\n\\n\"),\n            (\n                \"second\\n\\npa#[r|]#agraph\\n\\n\",\n                \"second\\n\\n#[paragraph\\n|]#\\n\",\n            ),\n            (\"#[f|]#irst char\\n\\n\", \"#[first char\\n|]#\\n\"),\n            (\"last char\\n#[\\n|]#\", \"#[last char\\n|]#\\n\"),\n            (\n                \"empty to line\\n#[\\n|]#paragraph boundary\\n\\n\",\n                \"empty to line\\n\\n#[paragraph boundary\\n|]#\\n\",\n            ),\n            (\n                \"line to empty\\n\\n#[p|]#aragraph boundary\\n\\n\",\n                \"line to empty\\n\\n#[paragraph boundary\\n|]#\\n\",\n            ),\n        ];\n\n        for (before, expected) in tests {\n            let (s, selection) = crate::test::print(before);\n            let text = Rope::from(s.as_str());\n            let selection = selection\n                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 1));\n            let actual = crate::test::plain(s.as_ref(), &selection);\n            assert_eq!(actual, expected, \"\\nbefore: `{:?}`\", before);\n        }\n    }\n\n    #[test]\n    fn test_textobject_paragraph_inside_double() {\n        let tests = [\n            (\n                \"last two\\n\\n#[p|]#aragraph\\n\\nwithout whitespaces\\n\\n\",\n                \"last two\\n\\n#[paragraph\\n\\nwithout whitespaces\\n|]#\\n\",\n            ),\n            (\n                \"last two\\n#[\\n|]#paragraph\\n\\nwithout whitespaces\\n\\n\",\n                \"last two\\n\\n#[paragraph\\n\\nwithout whitespaces\\n|]#\\n\",\n            ),\n        ];\n\n        for (before, expected) in tests {\n            let (s, selection) = crate::test::print(before);\n            let text = Rope::from(s.as_str());\n            let selection = selection\n                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Inside, 2));\n            let actual = crate::test::plain(s.as_ref(), &selection);\n            assert_eq!(actual, expected, \"\\nbefore: `{:?}`\", before);\n        }\n    }\n\n    #[test]\n    fn test_textobject_paragraph_around_single() {\n        let tests = [\n            (\"#[|]#\", \"#[|]#\"),\n            (\"firs#[t|]#\\n\\nparagraph\\n\\n\", \"#[first\\n\\n|]#paragraph\\n\\n\"),\n            (\n                \"second\\n\\npa#[r|]#agraph\\n\\n\",\n                \"second\\n\\n#[paragraph\\n\\n|]#\",\n            ),\n            (\"#[f|]#irst char\\n\\n\", \"#[first char\\n\\n|]#\"),\n            (\"last char\\n#[\\n|]#\", \"#[last char\\n\\n|]#\"),\n            (\n                \"empty to line\\n#[\\n|]#paragraph boundary\\n\\n\",\n                \"empty to line\\n\\n#[paragraph boundary\\n\\n|]#\",\n            ),\n            (\n                \"line to empty\\n\\n#[p|]#aragraph boundary\\n\\n\",\n                \"line to empty\\n\\n#[paragraph boundary\\n\\n|]#\",\n            ),\n        ];\n\n        for (before, expected) in tests {\n            let (s, selection) = crate::test::print(before);\n            let text = Rope::from(s.as_str());\n            let selection = selection\n                .transform(|r| textobject_paragraph(text.slice(..), r, TextObject::Around, 1));\n            let actual = crate::test::plain(s.as_ref(), &selection);\n            assert_eq!(actual, expected, \"\\nbefore: `{:?}`\", before);\n        }\n    }\n\n    #[test]\n    fn test_textobject_surround() {\n        // (text, [(cursor position, textobject, final range, surround char, count), ...])\n        let tests = &[\n            (\n                \"simple (single) surround pairs\",\n                vec![\n                    (3, Inside, (3, 3), '(', 1),\n                    (7, Inside, (8, 14), ')', 1),\n                    (10, Inside, (8, 14), '(', 1),\n                    (14, Inside, (8, 14), ')', 1),\n                    (3, Around, (3, 3), '(', 1),\n                    (7, Around, (7, 15), ')', 1),\n                    (10, Around, (7, 15), '(', 1),\n                    (14, Around, (7, 15), ')', 1),\n                ],\n            ),\n            (\n                \"samexx 'single' surround pairs\",\n                vec![\n                    (3, Inside, (3, 3), '\\'', 1),\n                    (7, Inside, (7, 7), '\\'', 1),\n                    (10, Inside, (8, 14), '\\'', 1),\n                    (14, Inside, (14, 14), '\\'', 1),\n                    (3, Around, (3, 3), '\\'', 1),\n                    (7, Around, (7, 7), '\\'', 1),\n                    (10, Around, (7, 15), '\\'', 1),\n                    (14, Around, (14, 14), '\\'', 1),\n                ],\n            ),\n            (\n                \"(nested (surround (pairs)) 3 levels)\",\n                vec![\n                    (0, Inside, (1, 35), '(', 1),\n                    (6, Inside, (1, 35), ')', 1),\n                    (8, Inside, (9, 25), '(', 1),\n                    (8, Inside, (9, 35), ')', 2),\n                    (20, Inside, (9, 25), '(', 2),\n                    (20, Inside, (1, 35), ')', 3),\n                    (0, Around, (0, 36), '(', 1),\n                    (6, Around, (0, 36), ')', 1),\n                    (8, Around, (8, 26), '(', 1),\n                    (8, Around, (8, 36), ')', 2),\n                    (20, Around, (8, 26), '(', 2),\n                    (20, Around, (0, 36), ')', 3),\n                ],\n            ),\n            (\n                \"(mixed {surround [pair] same} line)\",\n                vec![\n                    (2, Inside, (1, 34), '(', 1),\n                    (9, Inside, (8, 28), '{', 1),\n                    (18, Inside, (18, 22), '[', 1),\n                    (2, Around, (0, 35), '(', 1),\n                    (9, Around, (7, 29), '{', 1),\n                    (18, Around, (17, 23), '[', 1),\n                ],\n            ),\n            (\n                \"(stepped (surround) pairs (should) skip)\",\n                vec![(22, Inside, (1, 39), '(', 1), (22, Around, (0, 40), '(', 1)],\n            ),\n            (\n                \"[surround pairs{\\non different]\\nlines}\",\n                vec![\n                    (7, Inside, (1, 29), '[', 1),\n                    (15, Inside, (16, 36), '{', 1),\n                    (7, Around, (0, 30), '[', 1),\n                    (15, Around, (15, 37), '{', 1),\n                ],\n            ),\n        ];\n\n        for (sample, scenario) in tests {\n            let doc = Rope::from(*sample);\n            let slice = doc.slice(..);\n            for &case in scenario {\n                let (pos, objtype, expected_range, ch, count) = case;\n                let result = textobject_pair_surround(slice, Range::point(pos), objtype, ch, count);\n                assert_eq!(\n                    result,\n                    expected_range.into(),\n                    \"\\nCase failed: {:?} - {:?}\",\n                    sample,\n                    case\n                );\n            }\n        }\n    }\n}\n\n// Path: helix-core/src/movement.rs\nuse std::{cmp::Reverse, iter};\n\nuse ropey::iter::Chars;\nuse tree_sitter::{Node, QueryCursor};\n\nuse crate::{\n    char_idx_at_visual_offset,\n    chars::{categorize_char, char_is_line_ending, CharCategory},\n    doc_formatter::TextFormat,\n    graphemes::{\n        next_grapheme_boundary, nth_next_grapheme_boundary, nth_prev_grapheme_boundary,\n        prev_grapheme_boundary,\n    },\n    line_ending::rope_is_line_ending,\n    position::char_idx_at_visual_block_offset,\n    syntax::LanguageConfiguration,\n    text_annotations::TextAnnotations,\n    textobject::TextObject,\n    visual_offset_from_block, Range, RopeSlice, Selection, Syntax,\n};\n\n#[derive(Debug, Copy, Clone, PartialEq, Eq)]\npub enum Direction {\n    Forward,\n    Backward,\n}\n\n#[derive(Copy, Clone, PartialEq, Eq)]\npub enum Movement {\n    Extend,\n    Move,\n}\n\n\npub fn move_horizontally(\n    slice: RopeSlice,\n    range: Range,\n    dir: Direction,\n    count: usize,\n    behaviour: Movement,\n    _: &TextFormat,\n    _: &mut TextAnnotations,\n) -> Range {\n    let pos = range.cursor(slice);\n\n    // Compute the new position.\n    let new_pos = match dir {\n        Direction::Forward => nth_next_grapheme_boundary(slice, pos, count),\n        Direction::Backward => nth_prev_grapheme_boundary(slice, pos, count),\n    };\n\n    // Compute the final new range.\n    range.put_cursor(slice, new_pos, behaviour == Movement::Extend)\n}\n\npub fn move_vertically_visual(\n    slice: RopeSlice,\n    range: Range,\n    dir: Direction,\n    count: usize,\n    behaviour: Movement,\n    text_fmt: &TextFormat,\n    annotations: &mut TextAnnotations,\n) -> Range {\n    if !text_fmt.soft_wrap {\n        return move_vertically(slice, range, dir, count, behaviour, text_fmt, annotations);\n    }\n    annotations.clear_line_annotations();\n    let pos = range.cursor(slice);\n\n    // Compute the current position's 2d coordinates.\n    let (visual_pos, block_off) = visual_offset_from_block(slice, pos, pos, text_fmt, annotations);\n    let new_col = range\n        .old_visual_position\n        .map_or(visual_pos.col as u32, |(_, col)| col);\n\n    // Compute the new position.\n    let mut row_off = match dir {\n        Direction::Forward => count as isize,\n        Direction::Backward => -(count as isize),\n    };\n\n    // TODO how to handle inline annotations that span an entire visual line (very unlikely).\n\n    // Compute visual offset relative to block start to avoid trasversing the block twice\n    row_off += visual_pos.row as isize;\n    let new_pos = char_idx_at_visual_offset(\n        slice,\n        block_off,\n        row_off,\n        new_col as usize,\n        text_fmt,\n        annotations,\n    )\n    .0;\n\n    // Special-case to avoid moving to the end of the last non-empty line.\n    if behaviour == Movement::Extend && slice.line(slice.char_to_line(new_pos)).len_chars() == 0 {\n        return range;\n    }\n\n    let mut new_range = range.put_cursor(slice, new_pos, behaviour == Movement::Extend);\n    new_range.old_visual_position = Some((0, new_col));\n    new_range\n}\n\npub fn move_vertically(\n    slice: RopeSlice,\n    range: Range,\n    dir: Direction,\n    count: usize,\n    behaviour: Movement,\n    text_fmt: &TextFormat,\n    annotations: &mut TextAnnotations,\n) -> Range {\n    annotations.clear_line_annotations();\n    let pos = range.cursor(slice);\n    let line_idx = slice.char_to_line(pos);\n    let line_start = slice.line_to_char(line_idx);\n\n    // Compute the current position's 2d coordinates.\n    let visual_pos = visual_offset_from_block(slice, line_start, pos, text_fmt, annotations).0;\n    let (mut new_row, new_col) = range\n        .old_visual_position\n        .map_or((visual_pos.row as u32, visual_pos.col as u32), |pos| pos);\n    new_row = new_row.max(visual_pos.row as u32);\n    let line_idx = slice.char_to_line(pos);\n\n    // Compute the new position.\n    let mut new_line_idx = match dir {\n        Direction::Forward => line_idx.saturating_add(count),\n        Direction::Backward => line_idx.saturating_sub(count),\n    };\n\n    let line = if new_line_idx >= slice.len_lines() - 1 {\n        // there is no line terminator for the last line\n        // so the logic below is not necessary here\n        new_line_idx = slice.len_lines() - 1;\n        slice\n    } else {\n        // char_idx_at_visual_block_offset returns a one-past-the-end index\n        // in case it reaches the end of the slice\n        // to avoid moving to the nextline in that case the line terminator is removed from the line\n        let new_line_end = prev_grapheme_boundary(slice, slice.line_to_char(new_line_idx + 1));\n        slice.slice(..new_line_end)\n    };\n\n    let new_line_start = line.line_to_char(new_line_idx);\n\n    let (new_pos, _) = char_idx_at_visual_block_offset(\n        line,\n        new_line_start,\n        new_row as usize,\n        new_col as usize,\n        text_fmt,\n        annotations,\n    );\n\n    // Special-case to avoid moving to the end of the last non-empty line.\n    if behaviour == Movement::Extend && slice.line(new_line_idx).len_chars() == 0 {\n        return range;\n    }\n\n    let mut new_range = range.put_cursor(slice, new_pos, behaviour == Movement::Extend);\n    new_range.old_visual_position = Some((new_row, new_col));\n    new_range\n}\n\npub fn move_next_word_start(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::NextWordStart)\n}\n\npub fn move_next_word_end(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::NextWordEnd)\n}\n\npub fn move_prev_word_start(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::PrevWordStart)\n}\n\npub fn move_prev_word_end(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::PrevWordEnd)\n}\n\npub fn move_next_long_word_start(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::NextLongWordStart)\n}\n\npub fn move_next_long_word_end(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::NextLongWordEnd)\n}\n\npub fn move_prev_long_word_start(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::PrevLongWordStart)\n}\n\npub fn move_prev_long_word_end(slice: RopeSlice, range: Range, count: usize) -> Range {\n    word_move(slice, range, count, WordMotionTarget::PrevLongWordEnd)\n}\n\nfn word_move(slice: RopeSlice, range: Range, count: usize, target: WordMotionTarget) -> Range {\n    let is_prev = matches!(\n        target,\n        WordMotionTarget::PrevWordStart\n            | WordMotionTarget::PrevLongWordStart\n            | WordMotionTarget::PrevWordEnd\n            | WordMotionTarget::PrevLongWordEnd\n    );\n\n    // Special-case early-out.\n    if (is_prev && range.head == 0) || (!is_prev && range.head == slice.len_chars()) {\n        return range;\n    }\n\n    // Prepare the range appropriately based on the target movement\n    // direction.  This is addressing two things at once:\n    //\n    //   1. Block-cursor semantics.\n    //   2. The anchor position being irrelevant to the output result.\n    #[allow(clippy::collapsible_else_if)] // Makes the structure clearer in this case.\n    let start_range = if is_prev {\n        if range.anchor < range.head {\n            Range::new(range.head, prev_grapheme_boundary(slice, range.head))\n        } else {\n            Range::new(next_grapheme_boundary(slice, range.head), range.head)\n        }\n    } else {\n        if range.anchor < range.head {\n            Range::new(prev_grapheme_boundary(slice, range.head), range.head)\n        } else {\n            Range::new(range.head, next_grapheme_boundary(slice, range.head))\n        }\n    };\n\n    // Do the main work.\n    let mut range = start_range;\n    for _ in 0..count {\n        let next_range = slice.chars_at(range.head).range_to_target(target, range);\n        if range == next_range {\n            break;\n        }\n        range = next_range;\n    }\n    range\n}\n\npub fn move_prev_paragraph(\n    slice: RopeSlice,\n    range: Range,\n    count: usize,\n    behavior: Movement,\n) -> Range {\n    let mut line = range.cursor_line(slice);\n    let first_char = slice.line_to_char(line) == range.cursor(slice);\n    let prev_line_empty = rope_is_line_ending(slice.line(line.saturating_sub(1)));\n    let curr_line_empty = rope_is_line_ending(slice.line(line));\n    let prev_empty_to_line = prev_line_empty && !curr_line_empty;\n\n    // skip character before paragraph boundary\n    if prev_empty_to_line && !first_char {\n        line += 1;\n    }\n    let mut lines = slice.lines_at(line);\n    lines.reverse();\n    let mut lines = lines.map(rope_is_line_ending).peekable();\n    let mut last_line = line;\n    for _ in 0..count {\n        while lines.next_if(|&e| e).is_some() {\n            line -= 1;\n        }\n        while lines.next_if(|&e| !e).is_some() {\n            line -= 1;\n        }\n        if line == last_line {\n            break;\n        }\n        last_line = line;\n    }\n\n    let head = slice.line_to_char(line);\n    let anchor = if behavior == Movement::Move {\n        // exclude first character after paragraph boundary\n        if prev_empty_to_line && first_char {\n            range.cursor(slice)\n        } else {\n            range.head\n        }\n    } else {\n        range.put_cursor(slice, head, true).anchor\n    };\n    Range::new(anchor, head)\n}\n\npub fn move_next_paragraph(\n    slice: RopeSlice,\n    range: Range,\n    count: usize,\n    behavior: Movement,\n) -> Range {\n    let mut line = range.cursor_line(slice);\n    let last_char =\n        prev_grapheme_boundary(slice, slice.line_to_char(line + 1)) == range.cursor(slice);\n    let curr_line_empty = rope_is_line_ending(slice.line(line));\n    let next_line_empty =\n        rope_is_line_ending(slice.line(slice.len_lines().saturating_sub(1).min(line + 1)));\n    let curr_empty_to_line = curr_line_empty && !next_line_empty;\n\n    // skip character after paragraph boundary\n    if curr_empty_to_line && last_char {\n        line += 1;\n    }\n    let mut lines = slice.lines_at(line).map(rope_is_line_ending).peekable();\n    let mut last_line = line;\n    for _ in 0..count {\n        while lines.next_if(|&e| !e).is_some() {\n            line += 1;\n        }\n        while lines.next_if(|&e| e).is_some() {\n            line += 1;\n        }\n        if line == last_line {\n            break;\n        }\n        last_line = line;\n    }\n    let head = slice.line_to_char(line);\n    let anchor = if behavior == Movement::Move {\n        if curr_empty_to_line && last_char {\n            range.head\n        } else {\n            range.cursor(slice)\n        }\n    } else {\n        range.put_cursor(slice, head, true).anchor\n    };\n    Range::new(anchor, head)\n}\n\n// ---- util ------------\n\n#[inline]\n/// Returns first index that doesn't satisfy a given predicate when\n/// advancing the character index.\n///\n/// Returns none if all characters satisfy the predicate.\npub fn skip_while<F>(slice: RopeSlice, pos: usize, fun: F) -> Option<usize>\nwhere\n    F: Fn(char) -> bool,\n{\n    let mut chars = slice.chars_at(pos).enumerate();\n    chars.find_map(|(i, c)| if !fun(c) { Some(pos + i) } else { None })\n}\n\n#[inline]\n/// Returns first index that doesn't satisfy a given predicate when\n/// retreating the character index, saturating if all elements satisfy\n/// the condition.\npub fn backwards_skip_while<F>(slice: RopeSlice, pos: usize, fun: F) -> Option<usize>\nwhere\n    F: Fn(char) -> bool,\n{\n    let mut chars_starting_from_next = slice.chars_at(pos);\n    let mut backwards = iter::from_fn(|| chars_starting_from_next.prev()).enumerate();\n    backwards.find_map(|(i, c)| {\n        if !fun(c) {\n            Some(pos.saturating_sub(i))\n        } else {\n            None\n        }\n    })\n}\n\n/// Possible targets of a word motion\n#[derive(Copy, Clone, Debug)]\npub enum WordMotionTarget {\n    NextWordStart,\n    NextWordEnd,\n    PrevWordStart,\n    PrevWordEnd,\n    // A \"Long word\" (also known as a WORD in Vim/Kakoune) is strictly\n    // delimited by whitespace, and can consist of punctuation as well\n    // as alphanumerics.\n    NextLongWordStart,\n    NextLongWordEnd,\n    PrevLongWordStart,\n    PrevLongWordEnd,\n}\n\npub trait CharHelpers {\n    fn range_to_target(&mut self, target: WordMotionTarget, origin: Range) -> Range;\n}\n\nimpl CharHelpers for Chars<'_> {\n    /// Note: this only changes the anchor of the range if the head is effectively\n    /// starting on a boundary (either directly or after skipping newline characters).\n    /// Any other changes to the anchor should be handled by the calling code.\n    fn range_to_target(&mut self, target: WordMotionTarget, origin: Range) -> Range {\n        let is_prev = matches!(\n            target,\n            WordMotionTarget::PrevWordStart\n                | WordMotionTarget::PrevLongWordStart\n                | WordMotionTarget::PrevWordEnd\n                | WordMotionTarget::PrevLongWordEnd\n        );\n\n        // Reverse the iterator if needed for the motion direction.\n        if is_prev {\n            self.reverse();\n        }\n\n        // Function to advance index in the appropriate motion direction.\n        let advance: &dyn Fn(&mut usize) = if is_prev {\n            &|idx| *idx = idx.saturating_sub(1)\n        } else {\n            &|idx| *idx += 1\n        };\n\n        // Initialize state variables.\n        let mut anchor = origin.anchor;\n        let mut head = origin.head;\n        let mut prev_ch = {\n            let ch = self.prev();\n            if ch.is_some() {\n                self.next();\n            }\n            ch\n        };\n\n        // Skip any initial newline characters.\n        while let Some(ch) = self.next() {\n            if char_is_line_ending(ch) {\n...\n", "needle_token_start": 12057, "needle_token_end": 12244, "code_context_ntokens": 16210, "output": ["```rust\npub fn move_horizontally(\n    slice: RopeSlice,\n    range: Range,\n    dir: Direction,\n    count: usize,\n    behaviour: Movement,\n    _: &TextFormat,\n    _: &mut TextAnnotations,\n) -> Range {\n    let pos = range.cursor(slice);\n\n    // Compute the new position.\n    let new_pos = match dir {\n        Direction::Forward => nth_next_grapheme_boundary(slice, pos, count),\n        Direction::Backward => nth_prev_grapheme_boundary(slice, pos, count),\n    };\n\n    // Compute the final new range.\n    range.put_cursor(slice, new_pos, behaviour == Movement::Extend)\n}\n```"]}
{"repo": "helix-editor/helix", "name": "is_date", "language": "rust", "path": "helix-core/src/increment/date_time.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function determines if a given enumeration value represents a component typically associated with a date (year, month, or day).\n2. **Input**: The function takes an enumeration value representing a unit of time.\n3. **Output**: The function returns a boolean value, where `true` indicates that the input is a date-related unit (year, month, or day), and `false` otherwise.\n4. **Procedure**: The function checks if the input enumeration value matches any of the date-related units (years, months, or days) using a pattern matching technique and returns the corresponding boolean result.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: helix-core/src/syntax.rs\nmod tree_cursor;\n\nuse crate::{\n    auto_pairs::AutoPairs,\n    chars::char_is_line_ending,\n    diagnostic::Severity,\n    regex::Regex,\n    transaction::{ChangeSet, Operation},\n    RopeSlice, Tendril,\n};\n\nuse ahash::RandomState;\nuse arc_swap::{ArcSwap, Guard};\nuse bitflags::bitflags;\nuse globset::GlobSet;\nuse hashbrown::raw::RawTable;\nuse helix_stdx::rope::{self, RopeSliceExt};\nuse slotmap::{DefaultKey as LayerId, HopSlotMap};\n\nuse std::{\n    borrow::Cow,\n    cell::RefCell,\n    collections::{HashMap, HashSet, VecDeque},\n    fmt::{self, Display},\n    hash::{Hash, Hasher},\n    mem::replace,\n    path::{Path, PathBuf},\n    str::FromStr,\n    sync::Arc,\n};\n\nuse once_cell::sync::{Lazy, OnceCell};\nuse serde::{ser::SerializeSeq, Deserialize, Serialize};\n\nuse helix_loader::grammar::{get_language, load_runtime_file};\n\npub use tree_cursor::TreeCursor;\n\nfn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<String>::deserialize(deserializer)?\n        .map(|buf| Regex::new(&buf).map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_lsp_config<'de, D>(deserializer: D) -> Result<Option<serde_json::Value>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Option::<toml::Value>::deserialize(deserializer)?\n        .map(|toml| toml.try_into().map_err(serde::de::Error::custom))\n        .transpose()\n}\n\nfn deserialize_tab_width<'de, D>(deserializer: D) -> Result<usize, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    usize::deserialize(deserializer).and_then(|n| {\n        if n > 0 && n <= 16 {\n            Ok(n)\n        } else {\n            Err(serde::de::Error::custom(\n                \"tab width must be a value from 1 to 16 inclusive\",\n            ))\n        }\n    })\n}\n\npub fn deserialize_auto_pairs<'de, D>(deserializer: D) -> Result<Option<AutoPairs>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    Ok(Option::<AutoPairConfig>::deserialize(deserializer)?.and_then(AutoPairConfig::into))\n}\n\nfn default_timeout() -> u64 {\n    20\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct Configuration {\n    pub language: Vec<LanguageConfiguration>,\n    #[serde(default)]\n    pub language_server: HashMap<String, LanguageServerConfiguration>,\n}\n\n// largely based on tree-sitter/cli/src/loader.rs\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\", deny_unknown_fields)]\npub struct LanguageConfiguration {\n    #[serde(rename = \"name\")]\n    pub language_id: String, // c-sharp, rust, tsx\n    #[serde(rename = \"language-id\")]\n    // see the table under https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocumentItem\n    pub language_server_language_id: Option<String>, // csharp, rust, typescriptreact, for the language-server\n    pub scope: String,             // source.rust\n    pub file_types: Vec<FileType>, // filename extension or ends_with? <Gemfile, rb, etc>\n    #[serde(default)]\n    pub shebangs: Vec<String>, // interpreter(s) associated with language\n    #[serde(default)]\n    pub roots: Vec<String>, // these indicate project roots <.git, Cargo.toml>\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_comment_tokens\",\n        alias = \"comment-token\"\n    )]\n    pub comment_tokens: Option<Vec<String>>,\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"from_block_comment_tokens\"\n    )]\n    pub block_comment_tokens: Option<Vec<BlockCommentToken>>,\n    pub text_width: Option<usize>,\n    pub soft_wrap: Option<SoftWrap>,\n\n    #[serde(default)]\n    pub auto_format: bool,\n\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub formatter: Option<FormatterConfiguration>,\n\n    #[serde(default)]\n    pub diagnostic_severity: Severity,\n\n    pub grammar: Option<String>, // tree-sitter grammar name, defaults to language_id\n\n    // content_regex\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_regex\")]\n    pub injection_regex: Option<Regex>,\n    // first_line_regex\n    //\n    #[serde(skip)]\n    pub(crate) highlight_config: OnceCell<Option<Arc<HighlightConfiguration>>>,\n    // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583\n    #[serde(\n        default,\n        skip_serializing_if = \"Vec::is_empty\",\n        serialize_with = \"serialize_lang_features\",\n        deserialize_with = \"deserialize_lang_features\"\n    )]\n    pub language_servers: Vec<LanguageServerFeatures>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub indent: Option<IndentationConfiguration>,\n\n    #[serde(skip)]\n    pub(crate) indent_query: OnceCell<Option<Query>>,\n    #[serde(skip)]\n    pub(crate) textobject_query: OnceCell<Option<TextObjectQuery>>,\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub debugger: Option<DebugAdapterConfig>,\n\n    /// Automatic insertion of pairs to parentheses, brackets,\n    /// etc. Defaults to true. Optionally, this can be a list of 2-tuples\n    /// to specify a list of characters to pair. This overrides the\n    /// global setting.\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_auto_pairs\")]\n    pub auto_pairs: Option<AutoPairs>,\n\n    pub rulers: Option<Vec<u16>>, // if set, override editor's rulers\n\n    /// Hardcoded LSP root directories relative to the workspace root, like `examples` or `tools/fuzz`.\n    /// Falling back to the current working directory if none are configured.\n    pub workspace_lsp_roots: Option<Vec<PathBuf>>,\n    #[serde(default)]\n    pub persistent_diagnostic_sources: Vec<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Hash)]\npub enum FileType {\n    /// The extension of the file, either the `Path::extension` or the full\n    /// filename if the file does not have an extension.\n    Extension(String),\n    /// A Unix-style path glob. This is compared to the file's absolute path, so\n    /// it can be used to detect files based on their directories. If the glob\n    /// is not an absolute path and does not already start with a glob pattern,\n    /// a glob pattern will be prepended to it.\n    Glob(globset::Glob),\n}\n\nimpl Serialize for FileType {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: serde::Serializer,\n    {\n        use serde::ser::SerializeMap;\n\n        match self {\n            FileType::Extension(extension) => serializer.serialize_str(extension),\n            FileType::Glob(glob) => {\n                let mut map = serializer.serialize_map(Some(1))?;\n                map.serialize_entry(\"glob\", glob.glob())?;\n                map.end()\n            }\n        }\n    }\n}\n\nimpl<'de> Deserialize<'de> for FileType {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: serde::de::Deserializer<'de>,\n    {\n        struct FileTypeVisitor;\n\n        impl<'de> serde::de::Visitor<'de> for FileTypeVisitor {\n            type Value = FileType;\n\n            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {\n                formatter.write_str(\"string or table\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>\n            where\n                E: serde::de::Error,\n            {\n                Ok(FileType::Extension(value.to_string()))\n            }\n\n            fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>\n            where\n                M: serde::de::MapAccess<'de>,\n            {\n                match map.next_entry::<String, String>()? {\n                    Some((key, mut glob)) if key == \"glob\" => {\n                        // If the glob isn't an absolute path or already starts\n                        // with a glob pattern, add a leading glob so we\n                        // properly match relative paths.\n                        if !glob.starts_with('/') && !glob.starts_with(\"*/\") {\n                            glob.insert_str(0, \"*/\");\n                        }\n\n                        globset::Glob::new(glob.as_str())\n                            .map(FileType::Glob)\n                            .map_err(|err| {\n                                serde::de::Error::custom(format!(\"invalid `glob` pattern: {}\", err))\n                            })\n                    }\n                    Some((key, _value)) => Err(serde::de::Error::custom(format!(\n                        \"unknown key in `file-types` list: {}\",\n                        key\n                    ))),\n                    None => Err(serde::de::Error::custom(\n                        \"expected a `suffix` key in the `file-types` entry\",\n                    )),\n                }\n            }\n        }\n\n        deserializer.deserialize_any(FileTypeVisitor)\n    }\n}\n\nfn from_comment_tokens<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    #[derive(Deserialize)]\n    #[serde(untagged)]\n    enum CommentTokens {\n        Multiple(Vec<String>),\n        Single(String),\n    }\n    Ok(\n        Option::<CommentTokens>::deserialize(deserializer)?.map(|tokens| match tokens {\n            CommentTokens::Single(val) => vec![val],\n            CommentTokens::Multiple(vals) => vals,\n        }),\n    )\n}\n\n#[derive(Clone, Debug, Serialize, Deserialize)]\npub struct BlockCommentToken {\n    pub start: String,\n    pub end: String,\n}\n\nimpl Default for BlockCommentToken {\n    fn default() -> Self {\n        BlockCommentToken {\n            start: \"/*\".to_string(),\n            end: \"*/\".to_string(),\n        }\n    }\n}\n\nfn from_block_comment_tokens<'de, D>(\n    deserializer: D,\n) -> Result<Option<Vec<BlockCommentToken>>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    #[derive(Deserialize)]\n    #[serde(untagged)]\n    enum BlockCommentTokens {\n        Multiple(Vec<BlockCommentToken>),\n        Single(BlockCommentToken),\n    }\n    Ok(\n        Option::<BlockCommentTokens>::deserialize(deserializer)?.map(|tokens| match tokens {\n            BlockCommentTokens::Single(val) => vec![val],\n            BlockCommentTokens::Multiple(vals) => vals,\n        }),\n    )\n}\n\n#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]\n#[serde(rename_all = \"kebab-case\")]\npub enum LanguageServerFeature {\n    Format,\n    GotoDeclaration,\n    GotoDefinition,\n    GotoTypeDefinition,\n    GotoReference,\n    GotoImplementation,\n    // Goto, use bitflags, combining previous Goto members?\n    SignatureHelp,\n    Hover,\n    DocumentHighlight,\n    Completion,\n    CodeAction,\n    WorkspaceCommand,\n    DocumentSymbols,\n    WorkspaceSymbols,\n    // Symbols, use bitflags, see above?\n    Diagnostics,\n    RenameSymbol,\n    InlayHints,\n}\n\nimpl Display for LanguageServerFeature {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        use LanguageServerFeature::*;\n        let feature = match self {\n            Format => \"format\",\n            GotoDeclaration => \"goto-declaration\",\n            GotoDefinition => \"goto-definition\",\n            GotoTypeDefinition => \"goto-type-definition\",\n            GotoReference => \"goto-reference\",\n            GotoImplementation => \"goto-implementation\",\n            SignatureHelp => \"signature-help\",\n            Hover => \"hover\",\n            DocumentHighlight => \"document-highlight\",\n            Completion => \"completion\",\n            CodeAction => \"code-action\",\n            WorkspaceCommand => \"workspace-command\",\n            DocumentSymbols => \"document-symbols\",\n            WorkspaceSymbols => \"workspace-symbols\",\n            Diagnostics => \"diagnostics\",\n            RenameSymbol => \"rename-symbol\",\n            InlayHints => \"inlay-hints\",\n        };\n        write!(f, \"{feature}\",)\n    }\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(untagged, rename_all = \"kebab-case\", deny_unknown_fields)]\nenum LanguageServerFeatureConfiguration {\n    #[serde(rename_all = \"kebab-case\")]\n    Features {\n        #[serde(default, skip_serializing_if = \"HashSet::is_empty\")]\n        only_features: HashSet<LanguageServerFeature>,\n        #[serde(default, skip_serializing_if = \"HashSet::is_empty\")]\n        except_features: HashSet<LanguageServerFeature>,\n        name: String,\n    },\n    Simple(String),\n}\n\n#[derive(Debug, Default)]\npub struct LanguageServerFeatures {\n    pub name: String,\n    pub only: HashSet<LanguageServerFeature>,\n    pub excluded: HashSet<LanguageServerFeature>,\n}\n\nimpl LanguageServerFeatures {\n    pub fn has_feature(&self, feature: LanguageServerFeature) -> bool {\n        (self.only.is_empty() || self.only.contains(&feature)) && !self.excluded.contains(&feature)\n    }\n}\n\nfn deserialize_lang_features<'de, D>(\n    deserializer: D,\n) -> Result<Vec<LanguageServerFeatures>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    let raw: Vec<LanguageServerFeatureConfiguration> = Deserialize::deserialize(deserializer)?;\n    let res = raw\n        .into_iter()\n        .map(|config| match config {\n            LanguageServerFeatureConfiguration::Simple(name) => LanguageServerFeatures {\n                name,\n                ..Default::default()\n            },\n            LanguageServerFeatureConfiguration::Features {\n                only_features,\n                except_features,\n                name,\n            } => LanguageServerFeatures {\n                name,\n                only: only_features,\n                excluded: except_features,\n            },\n        })\n        .collect();\n    Ok(res)\n}\nfn serialize_lang_features<S>(\n    map: &Vec<LanguageServerFeatures>,\n    serializer: S,\n) -> Result<S::Ok, S::Error>\nwhere\n    S: serde::Serializer,\n{\n    let mut serializer = serializer.serialize_seq(Some(map.len()))?;\n    for features in map {\n        let features = if features.only.is_empty() && features.excluded.is_empty() {\n            LanguageServerFeatureConfiguration::Simple(features.name.to_owned())\n        } else {\n            LanguageServerFeatureConfiguration::Features {\n                only_features: features.only.clone(),\n                except_features: features.excluded.clone(),\n                name: features.name.to_owned(),\n            }\n        };\n        serializer.serialize_element(&features)?;\n    }\n    serializer.end()\n}\n\nfn deserialize_required_root_patterns<'de, D>(deserializer: D) -> Result<Option<GlobSet>, D::Error>\nwhere\n    D: serde::Deserializer<'de>,\n{\n    let patterns = Vec::<String>::deserialize(deserializer)?;\n    if patterns.is_empty() {\n        return Ok(None);\n    }\n    let mut builder = globset::GlobSetBuilder::new();\n    for pattern in patterns {\n        let glob = globset::Glob::new(&pattern).map_err(serde::de::Error::custom)?;\n        builder.add(glob);\n    }\n    builder.build().map(Some).map_err(serde::de::Error::custom)\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct LanguageServerConfiguration {\n    pub command: String,\n    #[serde(default)]\n    #[serde(skip_serializing_if = \"Vec::is_empty\")]\n    pub args: Vec<String>,\n    #[serde(default, skip_serializing_if = \"HashMap::is_empty\")]\n    pub environment: HashMap<String, String>,\n    #[serde(default, skip_serializing, deserialize_with = \"deserialize_lsp_config\")]\n    pub config: Option<serde_json::Value>,\n    #[serde(default = \"default_timeout\")]\n    pub timeout: u64,\n    #[serde(\n        default,\n        skip_serializing,\n        deserialize_with = \"deserialize_required_root_patterns\"\n    )]\n    pub required_root_patterns: Option<GlobSet>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct FormatterConfiguration {\n    pub command: String,\n    #[serde(default)]\n    #[serde(skip_serializing_if = \"Vec::is_empty\")]\n    pub args: Vec<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct AdvancedCompletion {\n    pub name: Option<String>,\n    pub completion: Option<String>,\n    pub default: Option<String>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\", untagged)]\npub enum DebugConfigCompletion {\n    Named(String),\n    Advanced(AdvancedCompletion),\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(untagged)]\npub enum DebugArgumentValue {\n    String(String),\n    Array(Vec<String>),\n    Boolean(bool),\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct DebugTemplate {\n    pub name: String,\n    pub request: String,\n    pub completion: Vec<DebugConfigCompletion>,\n    pub args: HashMap<String, DebugArgumentValue>,\n}\n\n#[derive(Debug, PartialEq, Eq, Clone, Deserialize, Serialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct DebugAdapterConfig {\n    pub name: String,\n    pub transport: String,\n    #[serde(default)]\n    pub command: String,\n    #[serde(default)]\n    pub args: Vec<String>,\n    pub port_arg: Option<String>,\n    pub templates: Vec<DebugTemplate>,\n    #[serde(default)]\n    pub quirks: DebuggerQuirks,\n}\n\n// Different workarounds for adapters' differences\n#[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize)]\npub struct DebuggerQuirks {\n    #[serde(default)]\n    pub absolute_paths: bool,\n}\n\n#[derive(Debug, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub struct IndentationConfiguration {\n    #[serde(deserialize_with = \"deserialize_tab_width\")]\n    pub tab_width: usize,\n    pub unit: String,\n}\n\n/// How the indentation for a newly inserted line should be determined.\n/// If the selected heuristic is not available (e.g. because the current\n/// language has no tree-sitter indent queries), a simpler one will be used.\n#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\")]\npub enum IndentationHeuristic {\n    /// Just copy the indentation of the line that the cursor is currently on.\n    Simple,\n    /// Use tree-sitter indent queries to compute the expected absolute indentation level of the new line.\n    TreeSitter,\n    /// Use tree-sitter indent queries to compute the expected difference in indentation between the new line\n    /// and the line before. Add this to the actual indentation level of the line before.\n    #[default]\n    Hybrid,\n}\n\n/// Configuration for auto pairs\n#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(rename_all = \"kebab-case\", deny_unknown_fields, untagged)]\npub enum AutoPairConfig {\n    /// Enables or disables auto pairing. False means disabled. True means to use the default pairs.\n    Enable(bool),\n\n    /// The mappings of pairs.\n    Pairs(HashMap<char, char>),\n}\n\nimpl Default for AutoPairConfig {\n    fn default() -> Self {\n        AutoPairConfig::Enable(true)\n    }\n}\n\nimpl From<&AutoPairConfig> for Option<AutoPairs> {\n    fn from(auto_pair_config: &AutoPairConfig) -> Self {\n        match auto_pair_config {\n            AutoPairConfig::Enable(false) => None,\n            AutoPairConfig::Enable(true) => Some(AutoPairs::default()),\n            AutoPairConfig::Pairs(pairs) => Some(AutoPairs::new(pairs.iter())),\n        }\n    }\n}\n\nimpl From<AutoPairConfig> for Option<AutoPairs> {\n    fn from(auto_pairs_config: AutoPairConfig) -> Self {\n        (&auto_pairs_config).into()\n    }\n}\n\nimpl FromStr for AutoPairConfig {\n    type Err = std::str::ParseBoolError;\n\n    // only do bool parsing for runtime setting\n    fn from_str(s: &str) -> Result<Self, Self::Err> {\n        let enable: bool = s.parse()?;\n        Ok(AutoPairConfig::Enable(enable))\n    }\n}\n\n#[derive(Debug)]\n...\n// Path: helix-core/src/config.rs\nuse crate::syntax::{Configuration, Loader, LoaderError};\n\n/// Language configuration based on built-in languages.toml.\npub fn default_lang_config() -> Configuration {\n    helix_loader::config::default_lang_config()\n        .try_into()\n        .expect(\"Could not deserialize built-in languages.toml\")\n}\n\n/// Language configuration loader based on built-in languages.toml.\npub fn default_lang_loader() -> Loader {\n    Loader::new(default_lang_config()).expect(\"Could not compile loader for default config\")\n}\n\n#[derive(Debug)]\npub enum LanguageLoaderError {\n    DeserializeError(toml::de::Error),\n    LoaderError(LoaderError),\n}\n\nimpl std::fmt::Display for LanguageLoaderError {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        match self {\n            Self::DeserializeError(err) => write!(f, \"Failed to parse language config: {err}\"),\n            Self::LoaderError(err) => write!(f, \"Failed to compile language config: {err}\"),\n        }\n    }\n}\n\nimpl std::error::Error for LanguageLoaderError {}\n\n/// Language configuration based on user configured languages.toml.\npub fn user_lang_config() -> Result<Configuration, toml::de::Error> {\n    helix_loader::config::user_lang_config()?.try_into()\n}\n\n/// Language configuration loader based on user configured languages.toml.\npub fn user_lang_loader() -> Result<Loader, LanguageLoaderError> {\n    let config: Configuration = helix_loader::config::user_lang_config()\n        .map_err(LanguageLoaderError::DeserializeError)?\n        .try_into()\n        .map_err(LanguageLoaderError::DeserializeError)?;\n\n    Loader::new(config).map_err(LanguageLoaderError::LoaderError)\n}\n\n// Path: helix-core/src/comment.rs\n//! This module contains the functionality toggle comments on lines over the selection\n//! using the comment character defined in the user's `languages.toml`\n\nuse smallvec::SmallVec;\n\nuse crate::{\n    syntax::BlockCommentToken, Change, Range, Rope, RopeSlice, Selection, Tendril, Transaction,\n};\nuse helix_stdx::rope::RopeSliceExt;\nuse std::borrow::Cow;\n\n/// Given text, a comment token, and a set of line indices, returns the following:\n/// - Whether the given lines should be considered commented\n///     - If any of the lines are uncommented, all lines are considered as such.\n/// - The lines to change for toggling comments\n///     - This is all provided lines excluding blanks lines.\n/// - The column of the comment tokens\n///     - Column of existing tokens, if the lines are commented; column to place tokens at otherwise.\n/// - The margin to the right of the comment tokens\n///     - Defaults to `1`. If any existing comment token is not followed by a space, changes to `0`.\nfn find_line_comment(\n    token: &str,\n    text: RopeSlice,\n    lines: impl IntoIterator<Item = usize>,\n) -> (bool, Vec<usize>, usize, usize) {\n    let mut commented = true;\n    let mut to_change = Vec::new();\n    let mut min = usize::MAX; // minimum col for first_non_whitespace_char\n    let mut margin = 1;\n    let token_len = token.chars().count();\n    for line in lines {\n        let line_slice = text.line(line);\n        if let Some(pos) = line_slice.first_non_whitespace_char() {\n            let len = line_slice.len_chars();\n\n            if pos < min {\n                min = pos;\n            }\n\n            // line can be shorter than pos + token len\n            let fragment = Cow::from(line_slice.slice(pos..std::cmp::min(pos + token.len(), len)));\n\n            if fragment != token {\n                // as soon as one of the non-blank lines doesn't have a comment, the whole block is\n                // considered uncommented.\n                commented = false;\n            }\n\n            // determine margin of 0 or 1 for uncommenting; if any comment token is not followed by a space,\n            // a margin of 0 is used for all lines.\n            if !matches!(line_slice.get_char(pos + token_len), Some(c) if c == ' ') {\n                margin = 0;\n            }\n\n            // blank lines don't get pushed.\n            to_change.push(line);\n        }\n    }\n    (commented, to_change, min, margin)\n}\n\n#[must_use]\npub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&str>) -> Transaction {\n    let text = doc.slice(..);\n\n    let token = token.unwrap_or(\"//\");\n    let comment = Tendril::from(format!(\"{} \", token));\n\n    let mut lines: Vec<usize> = Vec::with_capacity(selection.len());\n\n    let mut min_next_line = 0;\n    for selection in selection {\n        let (start, end) = selection.line_range(text);\n        let start = start.clamp(min_next_line, text.len_lines());\n        let end = (end + 1).min(text.len_lines());\n\n        lines.extend(start..end);\n        min_next_line = end;\n    }\n\n    let (commented, to_change, min, margin) = find_line_comment(token, text, lines);\n\n    let mut changes: Vec<Change> = Vec::with_capacity(to_change.len());\n\n    for line in to_change {\n        let pos = text.line_to_char(line) + min;\n\n        if !commented {\n            // comment line\n            changes.push((pos, pos, Some(comment.clone())));\n        } else {\n            // uncomment line\n            changes.push((pos, pos + token.len() + margin, None));\n        }\n    }\n\n    Transaction::change(doc, changes.into_iter())\n}\n\n#[derive(Debug, PartialEq, Eq)]\npub enum CommentChange {\n    Commented {\n        range: Range,\n        start_pos: usize,\n        end_pos: usize,\n        start_margin: bool,\n        end_margin: bool,\n        start_token: String,\n        end_token: String,\n    },\n    Uncommented {\n        range: Range,\n        start_pos: usize,\n        end_pos: usize,\n        start_token: String,\n        end_token: String,\n    },\n    Whitespace {\n        range: Range,\n    },\n}\n\npub fn find_block_comments(\n    tokens: &[BlockCommentToken],\n    text: RopeSlice,\n    selection: &Selection,\n) -> (bool, Vec<CommentChange>) {\n    let mut commented = true;\n    let mut only_whitespace = true;\n    let mut comment_changes = Vec::with_capacity(selection.len());\n    let default_tokens = tokens.first().cloned().unwrap_or_default();\n    // TODO: check if this can be removed on MSRV bump\n    #[allow(clippy::redundant_clone)]\n    let mut start_token = default_tokens.start.clone();\n    #[allow(clippy::redundant_clone)]\n    let mut end_token = default_tokens.end.clone();\n\n    let mut tokens = tokens.to_vec();\n    // sort the tokens by length, so longer tokens will match first\n    tokens.sort_by(|a, b| {\n        if a.start.len() == b.start.len() {\n            b.end.len().cmp(&a.end.len())\n        } else {\n            b.start.len().cmp(&a.start.len())\n        }\n    });\n    for range in selection {\n        let selection_slice = range.slice(text);\n        if let (Some(start_pos), Some(end_pos)) = (\n            selection_slice.first_non_whitespace_char(),\n            selection_slice.last_non_whitespace_char(),\n        ) {\n            let mut line_commented = false;\n            let mut after_start = 0;\n            let mut before_end = 0;\n            let len = (end_pos + 1) - start_pos;\n\n            for BlockCommentToken { start, end } in &tokens {\n                let start_len = start.chars().count();\n                let end_len = end.chars().count();\n                after_start = start_pos + start_len;\n                before_end = end_pos.saturating_sub(end_len);\n\n                if len >= start_len + end_len {\n                    let start_fragment = selection_slice.slice(start_pos..after_start);\n                    let end_fragment = selection_slice.slice(before_end + 1..end_pos + 1);\n\n                    // block commented with these tokens\n                    if start_fragment == start.as_str() && end_fragment == end.as_str() {\n                        start_token = start.to_string();\n                        end_token = end.to_string();\n                        line_commented = true;\n                        break;\n                    }\n                }\n            }\n\n            if !line_commented {\n                comment_changes.push(CommentChange::Uncommented {\n                    range: *range,\n                    start_pos,\n                    end_pos,\n                    start_token: default_tokens.start.clone(),\n                    end_token: default_tokens.end.clone(),\n                });\n                commented = false;\n            } else {\n                comment_changes.push(CommentChange::Commented {\n                    range: *range,\n                    start_pos,\n                    end_pos,\n                    start_margin: selection_slice\n                        .get_char(after_start)\n                        .map_or(false, |c| c == ' '),\n                    end_margin: after_start != before_end\n                        && selection_slice\n                            .get_char(before_end)\n                            .map_or(false, |c| c == ' '),\n                    start_token: start_token.to_string(),\n                    end_token: end_token.to_string(),\n                });\n            }\n            only_whitespace = false;\n        } else {\n            comment_changes.push(CommentChange::Whitespace { range: *range });\n        }\n    }\n    if only_whitespace {\n        commented = false;\n    }\n    (commented, comment_changes)\n}\n\n#[must_use]\npub fn create_block_comment_transaction(\n    doc: &Rope,\n    selection: &Selection,\n    commented: bool,\n    comment_changes: Vec<CommentChange>,\n) -> (Transaction, SmallVec<[Range; 1]>) {\n    let mut changes: Vec<Change> = Vec::with_capacity(selection.len() * 2);\n    let mut ranges: SmallVec<[Range; 1]> = SmallVec::with_capacity(selection.len());\n    let mut offs = 0;\n    for change in comment_changes {\n        if commented {\n            if let CommentChange::Commented {\n                range,\n                start_pos,\n                end_pos,\n                start_token,\n                end_token,\n                start_margin,\n                end_margin,\n            } = change\n            {\n                let from = range.from();\n                changes.push((\n                    from + start_pos,\n                    from + start_pos + start_token.len() + start_margin as usize,\n                    None,\n                ));\n                changes.push((\n                    from + end_pos - end_token.len() - end_margin as usize + 1,\n                    from + end_pos + 1,\n                    None,\n                ));\n            }\n        } else {\n            // uncommented so manually map ranges through changes\n            match change {\n                CommentChange::Uncommented {\n                    range,\n                    start_pos,\n                    end_pos,\n                    start_token,\n                    end_token,\n                } => {\n                    let from = range.from();\n                    changes.push((\n                        from + start_pos,\n                        from + start_pos,\n                        Some(Tendril::from(format!(\"{} \", start_token))),\n                    ));\n                    changes.push((\n                        from + end_pos + 1,\n                        from + end_pos + 1,\n                        Some(Tendril::from(format!(\" {}\", end_token))),\n                    ));\n\n                    let offset = start_token.chars().count() + end_token.chars().count() + 2;\n                    ranges.push(\n                        Range::new(from + offs, from + offs + end_pos + 1 + offset)\n                            .with_direction(range.direction()),\n                    );\n                    offs += offset;\n                }\n                CommentChange::Commented { range, .. } | CommentChange::Whitespace { range } => {\n                    ranges.push(Range::new(range.from() + offs, range.to() + offs));\n                }\n            }\n        }\n    }\n    (Transaction::change(doc, changes.into_iter()), ranges)\n}\n\n#[must_use]\npub fn toggle_block_comments(\n    doc: &Rope,\n    selection: &Selection,\n    tokens: &[BlockCommentToken],\n) -> Transaction {\n    let text = doc.slice(..);\n    let (commented, comment_changes) = find_block_comments(tokens, text, selection);\n    let (mut transaction, ranges) =\n        create_block_comment_transaction(doc, selection, commented, comment_changes);\n    if !commented {\n        transaction = transaction.with_selection(Selection::new(ranges, selection.primary_index()));\n    }\n    transaction\n}\n\npub fn split_lines_of_selection(text: RopeSlice, selection: &Selection) -> Selection {\n    let mut ranges = SmallVec::new();\n    for range in selection.ranges() {\n        let (line_start, line_end) = range.line_range(text.slice(..));\n        let mut pos = text.line_to_char(line_start);\n        for line in text.slice(pos..text.line_to_char(line_end + 1)).lines() {\n            let start = pos;\n            pos += line.len_chars();\n            ranges.push(Range::new(start, pos));\n        }\n    }\n    Selection::new(ranges, 0)\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn test_find_line_comment() {\n        // four lines, two space indented, except for line 1 which is blank.\n        let mut doc = Rope::from(\"  1\\n\\n  2\\n  3\");\n        // select whole document\n        let mut selection = Selection::single(0, doc.len_chars() - 1);\n\n        let text = doc.slice(..);\n\n        let res = find_line_comment(\"//\", text, 0..3);\n        // (commented = true, to_change = [line 0, line 2], min = col 2, margin = 0)\n        assert_eq!(res, (false, vec![0, 2], 2, 0));\n\n        // comment\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n\n        assert_eq!(doc, \"  // 1\\n\\n  // 2\\n  // 3\");\n\n        // uncomment\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n        assert_eq!(doc, \"  1\\n\\n  2\\n  3\");\n        assert!(selection.len() == 1); // to ignore the selection unused warning\n\n        // 0 margin comments\n        doc = Rope::from(\"  //1\\n\\n  //2\\n  //3\");\n        // reset the selection.\n        selection = Selection::single(0, doc.len_chars() - 1);\n\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n        assert_eq!(doc, \"  1\\n\\n  2\\n  3\");\n        assert!(selection.len() == 1); // to ignore the selection unused warning\n\n        // 0 margin comments, with no space\n        doc = Rope::from(\"//\");\n        // reset the selection.\n        selection = Selection::single(0, doc.len_chars() - 1);\n\n        let transaction = toggle_line_comments(&doc, &selection, None);\n        transaction.apply(&mut doc);\n        selection = selection.map(transaction.changes());\n        assert_eq!(doc, \"\");\n        assert!(selection.len() == 1); // to ignore the selection unused warning\n\n        // TODO: account for uncommenting with uneven comment indentation\n    }\n\n    #[test]\n    fn test_find_block_comments() {\n        // three lines 5 characters.\n        let mut doc = Rope::from(\"1\\n2\\n3\");\n        // select whole document\n        let selection = Selection::single(0, doc.len_chars());\n\n        let text = doc.slice(..);\n\n        let res = find_block_comments(&[BlockCommentToken::default()], text, &selection);\n\n        assert_eq!(\n            res,\n            (\n                false,\n                vec![CommentChange::Uncommented {\n                    range: Range::new(0, 5),\n                    start_pos: 0,\n                    end_pos: 4,\n                    start_token: \"/*\".to_string(),\n                    end_token: \"*/\".to_string(),\n                }]\n            )\n        );\n\n        // comment\n        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);\n        transaction.apply(&mut doc);\n\n        assert_eq!(doc, \"/* 1\\n2\\n3 */\");\n\n        // uncomment\n        let selection = Selection::single(0, doc.len_chars());\n        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);\n        transaction.apply(&mut doc);\n        assert_eq!(doc, \"1\\n2\\n3\");\n\n        // don't panic when there is just a space in comment\n        doc = Rope::from(\"/* */\");\n        let selection = Selection::single(0, doc.len_chars());\n        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);\n        transaction.apply(&mut doc);\n        assert_eq!(doc, \"\");\n    }\n}\n\n// Path: helix-core/src/fuzzy.rs\nuse std::ops::DerefMut;\n\nuse nucleo::pattern::{Atom, AtomKind, CaseMatching};\nuse nucleo::Config;\nuse parking_lot::Mutex;\n\npub struct LazyMutex<T> {\n    inner: Mutex<Option<T>>,\n    init: fn() -> T,\n}\n\nimpl<T> LazyMutex<T> {\n    pub const fn new(init: fn() -> T) -> Self {\n        Self {\n            inner: Mutex::new(None),\n            init,\n        }\n    }\n\n    pub fn lock(&self) -> impl DerefMut<Target = T> + '_ {\n        parking_lot::MutexGuard::map(self.inner.lock(), |val| val.get_or_insert_with(self.init))\n    }\n}\n\npub static MATCHER: LazyMutex<nucleo::Matcher> = LazyMutex::new(nucleo::Matcher::default);\n\n/// convenience function to easily fuzzy match\n/// on a (relatively small list of inputs). This is not recommended for building a full tui\n/// application that can match large numbers of matches as all matching is done on the current\n/// thread, effectively blocking the UI\npub fn fuzzy_match<T: AsRef<str>>(\n    pattern: &str,\n    items: impl IntoIterator<Item = T>,\n    path: bool,\n) -> Vec<(T, u16)> {\n    let mut matcher = MATCHER.lock();\n    matcher.config = Config::DEFAULT;\n    if path {\n        matcher.config.set_match_paths();\n    }\n    let pattern = Atom::new(pattern, CaseMatching::Smart, AtomKind::Fuzzy, false);\n    pattern.match_list(items, &mut matcher)\n}\n\n// Path: helix-core/src/increment/date_time.rs\nuse chrono::{Duration, NaiveDate, NaiveDateTime, NaiveTime};\nuse once_cell::sync::Lazy;\nuse regex::Regex;\nuse std::fmt::Write;\n\n/// Increment a Date or DateTime\n///\n/// If just a Date is selected the day will be incremented.\n/// If a DateTime is selected the second will be incremented.\npub fn increment(selected_text: &str, amount: i64) -> Option<String> {\n    if selected_text.is_empty() {\n        return None;\n    }\n\n    FORMATS.iter().find_map(|format| {\n        let captures = format.regex.captures(selected_text)?;\n        if captures.len() - 1 != format.fields.len() {\n            return None;\n        }\n\n        let date_time = captures.get(0)?;\n        let has_date = format.fields.iter().any(|f| f.unit.is_date());\n        let has_time = format.fields.iter().any(|f| f.unit.is_time());\n        let date_time = &selected_text[date_time.start()..date_time.end()];\n        match (has_date, has_time) {\n            (true, true) => {\n                let date_time = NaiveDateTime::parse_from_str(date_time, format.fmt).ok()?;\n                Some(\n                    date_time\n                        .checked_add_signed(Duration::try_minutes(amount)?)?\n                        .format(format.fmt)\n                        .to_string(),\n                )\n            }\n            (true, false) => {\n                let date = NaiveDate::parse_from_str(date_time, format.fmt).ok()?;\n                Some(\n                    date.checked_add_signed(Duration::try_days(amount)?)?\n                        .format(format.fmt)\n                        .to_string(),\n                )\n            }\n            (false, true) => {\n                let time = NaiveTime::parse_from_str(date_time, format.fmt).ok()?;\n                let (adjusted_time, _) =\n                    time.overflowing_add_signed(Duration::try_minutes(amount)?);\n                Some(adjusted_time.format(format.fmt).to_string())\n            }\n            (false, false) => None,\n        }\n    })\n}\n\nstatic FORMATS: Lazy<Vec<Format>> = Lazy::new(|| {\n    vec![\n        Format::new(\"%Y-%m-%d %H:%M:%S\"), // 2021-11-24 07:12:23\n        Format::new(\"%Y/%m/%d %H:%M:%S\"), // 2021/11/24 07:12:23\n        Format::new(\"%Y-%m-%d %H:%M\"),    // 2021-11-24 07:12\n        Format::new(\"%Y/%m/%d %H:%M\"),    // 2021/11/24 07:12\n        Format::new(\"%Y-%m-%d\"),          // 2021-11-24\n        Format::new(\"%Y/%m/%d\"),          // 2021/11/24\n        Format::new(\"%a %b %d %Y\"),       // Wed Nov 24 2021\n        Format::new(\"%d-%b-%Y\"),          // 24-Nov-2021\n        Format::new(\"%Y %b %d\"),          // 2021 Nov 24\n        Format::new(\"%b %d, %Y\"),         // Nov 24, 2021\n        Format::new(\"%-I:%M:%S %P\"),      // 7:21:53 am\n        Format::new(\"%-I:%M %P\"),         // 7:21 am\n        Format::new(\"%-I:%M:%S %p\"),      // 7:21:53 AM\n        Format::new(\"%-I:%M %p\"),         // 7:21 AM\n        Format::new(\"%H:%M:%S\"),          // 23:24:23\n        Format::new(\"%H:%M\"),             // 23:24\n    ]\n});\n\n#[derive(Debug)]\nstruct Format {\n    fmt: &'static str,\n    fields: Vec<DateField>,\n    regex: Regex,\n    max_len: usize,\n}\n\nimpl Format {\n    fn new(fmt: &'static str) -> Self {\n        let mut remaining = fmt;\n        let mut fields = Vec::new();\n        let mut regex = \"^\".to_string();\n        let mut max_len = 0;\n\n        while let Some(i) = remaining.find('%') {\n            let after = &remaining[i + 1..];\n            let mut chars = after.chars();\n            let c = chars.next().unwrap();\n\n            let spec_len = if c == '-' {\n                1 + chars.next().unwrap().len_utf8()\n            } else {\n                c.len_utf8()\n            };\n\n            let specifier = &after[..spec_len];\n            let field = DateField::from_specifier(specifier).unwrap();\n            fields.push(field);\n            max_len += field.max_len + remaining[..i].len();\n            regex += &remaining[..i];\n            write!(regex, \"({})\", field.regex).unwrap();\n            remaining = &after[spec_len..];\n        }\n        regex += \"$\";\n\n        let regex = Regex::new(&regex).unwrap();\n\n        Self {\n            fmt,\n            fields,\n            regex,\n            max_len,\n        }\n    }\n}\n\nimpl PartialEq for Format {\n    fn eq(&self, other: &Self) -> bool {\n        self.fmt == other.fmt && self.fields == other.fields && self.max_len == other.max_len\n    }\n}\n\nimpl Eq for Format {}\n\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\nstruct DateField {\n    regex: &'static str,\n    unit: DateUnit,\n    max_len: usize,\n}\n\nimpl DateField {\n    fn from_specifier(specifier: &str) -> Option<Self> {\n        match specifier {\n            \"Y\" => Some(Self {\n                regex: r\"\\d{4}\",\n                unit: DateUnit::Years,\n                max_len: 5,\n            }),\n            \"y\" => Some(Self {\n                regex: r\"\\d\\d\",\n                unit: DateUnit::Years,\n                max_len: 2,\n            }),\n            \"m\" => Some(Self {\n                regex: r\"[0-1]\\d\",\n                unit: DateUnit::Months,\n                max_len: 2,\n            }),\n            \"d\" => Some(Self {\n                regex: r\"[0-3]\\d\",\n                unit: DateUnit::Days,\n                max_len: 2,\n            }),\n            \"-d\" => Some(Self {\n                regex: r\"[1-3]?\\d\",\n                unit: DateUnit::Days,\n                max_len: 2,\n            }),\n            \"a\" => Some(Self {\n                regex: r\"Sun|Mon|Tue|Wed|Thu|Fri|Sat\",\n                unit: DateUnit::Days,\n                max_len: 3,\n            }),\n            \"A\" => Some(Self {\n                regex: r\"Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday\",\n                unit: DateUnit::Days,\n                max_len: 9,\n            }),\n            \"b\" | \"h\" => Some(Self {\n                regex: r\"Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec\",\n                unit: DateUnit::Months,\n                max_len: 3,\n            }),\n            \"B\" => Some(Self {\n                regex: r\"January|February|March|April|May|June|July|August|September|October|November|December\",\n                unit: DateUnit::Months,\n                max_len: 9,\n            }),\n            \"H\" => Some(Self {\n                regex: r\"[0-2]\\d\",\n                unit: DateUnit::Hours,\n                max_len: 2,\n            }),\n            \"M\" => Some(Self {\n                regex: r\"[0-5]\\d\",\n                unit: DateUnit::Minutes,\n                max_len: 2,\n            }),\n            \"S\" => Some(Self {\n                regex: r\"[0-5]\\d\",\n                unit: DateUnit::Seconds,\n                max_len: 2,\n            }),\n            \"I\" => Some(Self {\n                regex: r\"[0-1]\\d\",\n                unit: DateUnit::Hours,\n                max_len: 2,\n            }),\n            \"-I\" => Some(Self {\n                regex: r\"1?\\d\",\n                unit: DateUnit::Hours,\n                max_len: 2,\n            }),\n            \"P\" => Some(Self {\n                regex: r\"am|pm\",\n                unit: DateUnit::AmPm,\n                max_len: 2,\n            }),\n            \"p\" => Some(Self {\n                regex: r\"AM|PM\",\n                unit: DateUnit::AmPm,\n                max_len: 2,\n            }),\n            _ => None,\n        }\n    }\n}\n\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\nenum DateUnit {\n    Years,\n    Months,\n    Days,\n    Hours,\n    Minutes,\n    Seconds,\n    AmPm,\n}\n\nimpl DateUnit {\n    \nfn is_date(self) -> bool {\n        matches!(self, DateUnit::Years | DateUnit::Months | DateUnit::Days)\n    }\n\n    fn is_time(self) -> bool {\n        matches!(\n            self,\n            DateUnit::Hours | DateUnit::Minutes | DateUnit::Seconds\n        )\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn test_increment_date_times() {\n        let tests = [\n            // (original, cursor, amount, expected)\n            (\"2020-02-28\", 1, \"2020-02-29\"),\n            (\"2020-02-29\", 1, \"2020-03-01\"),\n            (\"2020-01-31\", 1, \"2020-02-01\"),\n            (\"2020-01-20\", 1, \"2020-01-21\"),\n            (\"2021-01-01\", -1, \"2020-12-31\"),\n            (\"2021-01-31\", -2, \"2021-01-29\"),\n            (\"2020-02-28\", 1, \"2020-02-29\"),\n            (\"2021-02-28\", 1, \"2021-03-01\"),\n            (\"2021-03-01\", -1, \"2021-02-28\"),\n            (\"2020-02-29\", -1, \"2020-02-28\"),\n            (\"2020-02-20\", -1, \"2020-02-19\"),\n            (\"2021-03-01\", -1, \"2021-02-28\"),\n            (\"1980/12/21\", 100, \"1981/03/31\"),\n            (\"1980/12/21\", -100, \"1980/09/12\"),\n            (\"1980/12/21\", 1000, \"1983/09/17\"),\n            (\"1980/12/21\", -1000, \"1978/03/27\"),\n            (\"2021-11-24 07:12:23\", 1, \"2021-11-24 07:13:23\"),\n            (\"2021-11-24 07:12\", 1, \"2021-11-24 07:13\"),\n            (\"Wed Nov 24 2021\", 1, \"Thu Nov 25 2021\"),\n            (\"24-Nov-2021\", 1, \"25-Nov-2021\"),\n            (\"2021 Nov 24\", 1, \"2021 Nov 25\"),\n            (\"Nov 24, 2021\", 1, \"Nov 25, 2021\"),\n            (\"7:21:53 am\", 1, \"7:22:53 am\"),\n            (\"7:21:53 AM\", 1, \"7:22:53 AM\"),\n            (\"7:21 am\", 1, \"7:22 am\"),\n            (\"23:24:23\", 1, \"23:25:23\"),\n            (\"23:24\", 1, \"23:25\"),\n            (\"23:59\", 1, \"00:00\"),\n            (\"23:59:59\", 1, \"00:00:59\"),\n        ];\n\n        for (original, amount, expected) in tests {\n            assert_eq!(increment(original, amount).unwrap(), expected);\n        }\n    }\n\n    #[test]\n    fn test_invalid_date_times() {\n        let tests = [\n            \"0000-00-00\",\n            \"1980-2-21\",\n            \"1980-12-1\",\n            \"12345\",\n            \"2020-02-30\",\n            \"1999-12-32\",\n            \"19-12-32\",\n            \"1-2-3\",\n            \"0000/00/00\",\n            \"1980/2/21\",\n            \"1980/12/1\",\n            \"12345\",\n            \"2020/02/30\",\n            \"1999/12/32\",\n            \"19/12/32\",\n            \"1/2/3\",\n            \"123:456:789\",\n            \"11:61\",\n            \"2021-55-12 08:12:54\",\n        ];\n\n        for invalid in tests {\n            assert_eq!(increment(invalid, 1), None)\n        }\n    }\n}\n\n// Path: helix-core/src/increment/integer.rs\nconst SEPARATOR: char = '_';\n\n/// Increment an integer.\n///\n/// Supported bases:\n///     2 with prefix 0b\n///     8 with prefix 0o\n///     10 with no prefix\n///     16 with prefix 0x\n///\n/// An integer can contain `_` as a separator but may not start or end with a separator.\n/// Base 10 integers can go negative, but bases 2, 8, and 16 cannot.\n/// All addition and subtraction is saturating.\npub fn increment(selected_text: &str, amount: i64) -> Option<String> {\n    if selected_text.is_empty()\n        || selected_text.ends_with(SEPARATOR)\n        || selected_text.starts_with(SEPARATOR)\n    {\n        return None;\n    }\n\n    let radix = if selected_text.starts_with(\"0x\") {\n        16\n    } else if selected_text.starts_with(\"0o\") {\n        8\n    } else if selected_text.starts_with(\"0b\") {\n        2\n    } else {\n        10\n    };\n\n    // Get separator indexes from right to left.\n    let separator_rtl_indexes: Vec<usize> = selected_text\n        .chars()\n        .rev()\n        .enumerate()\n        .filter_map(|(i, c)| if c == SEPARATOR { Some(i) } else { None })\n        .collect();\n\n    let word: String = selected_text.chars().filter(|&c| c != SEPARATOR).collect();\n\n    let mut new_text = if radix == 10 {\n        let number = &word;\n        let value = i128::from_str_radix(number, radix).ok()?;\n        let new_value = value.saturating_add(amount as i128);\n\n        let format_length = match (value.is_negative(), new_value.is_negative()) {\n            (true, false) => number.len() - 1,\n            (false, true) => number.len() + 1,\n            _ => number.len(),\n        } - separator_rtl_indexes.len();\n\n        if number.starts_with('0') || number.starts_with(\"-0\") {\n            format!(\"{:01$}\", new_value, format_length)\n        } else {\n            format!(\"{}\", new_value)\n        }\n    } else {\n        let number = &word[2..];\n        let value = u128::from_str_radix(number, radix).ok()?;\n        let new_value = (value as i128).saturating_add(amount as i128);\n        let new_value = if new_value < 0 { 0 } else { new_value };\n        let format_length = selected_text.len() - 2 - separator_rtl_indexes.len();\n\n        match radix {\n            2 => format!(\"0b{:01$b}\", new_value, format_length),\n            8 => format!(\"0o{:01$o}\", new_value, format_length),\n            16 => {\n                let (lower_count, upper_count): (usize, usize) =\n                    number.chars().fold((0, 0), |(lower, upper), c| {\n                        (\n                            lower + c.is_ascii_lowercase() as usize,\n                            upper + c.is_ascii_uppercase() as usize,\n                        )\n                    });\n                if upper_count > lower_count {\n                    format!(\"0x{:01$X}\", new_value, format_length)\n                } else {\n                    format!(\"0x{:01$x}\", new_value, format_length)\n                }\n            }\n            _ => unimplemented!(\"radix not supported: {}\", radix),\n        }\n    };\n\n    // Add separators from original number.\n    for &rtl_index in &separator_rtl_indexes {\n        if rtl_index < new_text.len() {\n            let new_index = new_text.len().saturating_sub(rtl_index);\n            if new_index > 0 {\n                new_text.insert(new_index, SEPARATOR);\n            }\n        }\n    }\n\n    // Add in additional separators if necessary.\n    if new_text.len() > selected_text.len() && !separator_rtl_indexes.is_empty() {\n        let spacing = match separator_rtl_indexes.as_slice() {\n...\n", "needle_token_start": 13801, "needle_token_end": 13837, "code_context_ntokens": 16259, "output": ["```rust\nimpl DateUnit {\n    fn is_date(self) -> bool {\n        matches!(self, DateUnit::Years | DateUnit::Months | DateUnit::Days)\n    }\n}\n```"]}
{"repo": "helix-editor/helix", "name": "test_input_edits", "language": "rust", "path": "helix-core/src/syntax.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to verify the correctness of generating and applying text edits to a document using a specific syntax tree library. It ensures that the edits adjust the document content accurately and that the positions and byte indices are updated correctly.\n2. **Input**: The function takes no direct inputs as it is a test function. It internally uses predefined text strings and edit operations (like insertions and deletions).\n3. **Output**: There is no direct output from this function. However, it internally checks the results using assertions to ensure that the text edits are applied as expected and that the resulting document state matches the anticipated state.\n4. **Procedure**: The function performs the following steps:\n   - Initializes a text document.\n   - Specifies changes to be made to the document (like replacing or deleting text).\n   - Generates edit objects based on these changes.\n   - Applies these edits to the document.\n   - Uses assertions to verify that the document's content and the edit details (like start and end positions) are correct after the edits.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                                    continue;\n                                }\n                                last_injection_end = content_node.end_byte();\n                                injections.push((config, ranges));\n                            }\n                        }\n                    }\n                }\n\n                for (lang_name, content_nodes, included_children) in combined_injections {\n                    if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {\n                        if let Some(config) = (injection_callback)(&lang_name) {\n                            let ranges =\n                                intersect_ranges(&layer.ranges, &content_nodes, included_children);\n                            if !ranges.is_empty() {\n                                injections.push((config, ranges));\n                            }\n                        }\n                    }\n                }\n\n                let depth = layer.depth + 1;\n                // TODO: can't inline this since matches borrows self.layers\n                for (config, ranges) in injections {\n                    let new_layer = LanguageLayer {\n                        tree: None,\n                        config,\n                        depth,\n                        ranges,\n                        flags: LayerUpdateFlags::empty(),\n                        parent: Some(layer_id),\n                    };\n\n                    // Find an identical existing layer\n                    let layer = layers_table\n                        .get(layers_hasher.hash_one(&new_layer), |&it| {\n                            self.layers[it] == new_layer\n                        })\n                        .copied();\n\n                    // ...or insert a new one.\n                    let layer_id = layer.unwrap_or_else(|| self.layers.insert(new_layer));\n\n                    queue.push_back(layer_id);\n                }\n\n                // TODO: pre-process local scopes at this time, rather than highlight?\n                // would solve problems with locals not working across boundaries\n            }\n\n            // Return the cursor back in the pool.\n            ts_parser.cursors.push(cursor);\n\n            // Reset all `LayerUpdateFlags` and remove all untouched layers\n            self.layers.retain(|_, layer| {\n                replace(&mut layer.flags, LayerUpdateFlags::empty())\n                    .contains(LayerUpdateFlags::TOUCHED)\n            });\n\n            Ok(())\n        })\n    }\n\n    pub fn tree(&self) -> &Tree {\n        self.layers[self.root].tree()\n    }\n\n    /// Iterate over the highlighted regions for a given slice of source code.\n    pub fn highlight_iter<'a>(\n        &'a self,\n        source: RopeSlice<'a>,\n        range: Option<std::ops::Range<usize>>,\n        cancellation_flag: Option<&'a AtomicUsize>,\n    ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {\n        let mut layers = self\n            .layers\n            .iter()\n            .filter_map(|(_, layer)| {\n                // TODO: if range doesn't overlap layer range, skip it\n\n                // Reuse a cursor from the pool if available.\n                let mut cursor = PARSER.with(|ts_parser| {\n                    let highlighter = &mut ts_parser.borrow_mut();\n                    highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)\n                });\n\n                // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which\n                // prevents them from being moved. But both of these values are really just\n                // pointers, so it's actually ok to move them.\n                let cursor_ref =\n                    unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };\n\n                // if reusing cursors & no range this resets to whole range\n                cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));\n                cursor_ref.set_match_limit(TREE_SITTER_MATCH_LIMIT);\n\n                let mut captures = cursor_ref\n                    .captures(\n                        &layer.config.query,\n                        layer.tree().root_node(),\n                        RopeProvider(source),\n                    )\n                    .peekable();\n\n                // If there's no captures, skip the layer\n                captures.peek()?;\n\n                Some(HighlightIterLayer {\n                    highlight_end_stack: Vec::new(),\n                    scope_stack: vec![LocalScope {\n                        inherits: false,\n                        range: 0..usize::MAX,\n                        local_defs: Vec::new(),\n                    }],\n                    cursor,\n                    _tree: None,\n                    captures: RefCell::new(captures),\n                    config: layer.config.as_ref(), // TODO: just reuse `layer`\n                    depth: layer.depth,            // TODO: just reuse `layer`\n                })\n            })\n            .collect::<Vec<_>>();\n\n        layers.sort_unstable_by_key(|layer| layer.sort_key());\n\n        let mut result = HighlightIter {\n            source,\n            byte_offset: range.map_or(0, |r| r.start),\n            cancellation_flag,\n            iter_count: 0,\n            layers,\n            next_event: None,\n            last_highlight_range: None,\n        };\n        result.sort_layers();\n        result\n    }\n\n    pub fn tree_for_byte_range(&self, start: usize, end: usize) -> &Tree {\n        let mut container_id = self.root;\n\n        for (layer_id, layer) in self.layers.iter() {\n            if layer.depth > self.layers[container_id].depth\n                && layer.contains_byte_range(start, end)\n            {\n                container_id = layer_id;\n            }\n        }\n\n        self.layers[container_id].tree()\n    }\n\n    pub fn named_descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {\n        self.tree_for_byte_range(start, end)\n            .root_node()\n            .named_descendant_for_byte_range(start, end)\n    }\n\n    pub fn descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {\n        self.tree_for_byte_range(start, end)\n            .root_node()\n            .descendant_for_byte_range(start, end)\n    }\n\n    pub fn walk(&self) -> TreeCursor<'_> {\n        // data structure to find the smallest range that contains a point\n        // when some of the ranges in the structure can overlap.\n        TreeCursor::new(&self.layers, self.root)\n    }\n\n    // Commenting\n    // comment_strings_for_pos\n    // is_commented\n\n    // Indentation\n    // suggested_indent_for_line_at_buffer_row\n    // suggested_indent_for_buffer_row\n    // indent_level_for_line\n\n    // TODO: Folding\n}\n\nbitflags! {\n    /// Flags that track the status of a layer\n    /// in the `Sytaxn::update` function\n    #[derive(Debug)]\n    struct LayerUpdateFlags : u32{\n        const MODIFIED = 0b001;\n        const MOVED = 0b010;\n        const TOUCHED = 0b100;\n    }\n}\n\n#[derive(Debug)]\npub struct LanguageLayer {\n    // mode\n    // grammar\n    pub config: Arc<HighlightConfiguration>,\n    pub(crate) tree: Option<Tree>,\n    pub ranges: Vec<Range>,\n    pub depth: u32,\n    flags: LayerUpdateFlags,\n    parent: Option<LayerId>,\n}\n\n/// This PartialEq implementation only checks if that\n/// two layers are theoretically identical (meaning they highlight the same text range with the same language).\n/// It does not check whether the layers have the same internal treesitter\n/// state.\nimpl PartialEq for LanguageLayer {\n    fn eq(&self, other: &Self) -> bool {\n        self.depth == other.depth\n            && self.config.language == other.config.language\n            && self.ranges == other.ranges\n    }\n}\n\n/// Hash implementation belongs to PartialEq implementation above.\n/// See its documentation for details.\nimpl Hash for LanguageLayer {\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        self.depth.hash(state);\n        self.config.language.hash(state);\n        self.ranges.hash(state);\n    }\n}\n\nimpl LanguageLayer {\n    pub fn tree(&self) -> &Tree {\n        // TODO: no unwrap\n        self.tree.as_ref().unwrap()\n    }\n\n    fn parse(&mut self, parser: &mut Parser, source: RopeSlice) -> Result<(), Error> {\n        parser\n            .set_included_ranges(&self.ranges)\n            .map_err(|_| Error::InvalidRanges)?;\n\n        parser\n            .set_language(&self.config.language)\n            .map_err(|_| Error::InvalidLanguage)?;\n\n        // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };\n        let tree = parser\n            .parse_with(\n                &mut |byte, _| {\n                    if byte <= source.len_bytes() {\n                        let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);\n                        &chunk.as_bytes()[byte - start_byte..]\n                    } else {\n                        // out of range\n                        &[]\n                    }\n                },\n                self.tree.as_ref(),\n            )\n            .ok_or(Error::Cancelled)?;\n        // unsafe { ts_parser.parser.set_cancellation_flag(None) };\n        self.tree = Some(tree);\n        Ok(())\n    }\n\n    /// Whether the layer contains the given byte range.\n    ///\n    /// If the layer has multiple ranges (i.e. combined injections), the\n    /// given range is considered contained if it is within the start and\n    /// end bytes of the first and last ranges **and** if the given range\n    /// starts or ends within any of the layer's ranges.\n    fn contains_byte_range(&self, start: usize, end: usize) -> bool {\n        let layer_start = self\n            .ranges\n            .first()\n            .expect(\"ranges should not be empty\")\n            .start_byte;\n        let layer_end = self\n            .ranges\n            .last()\n            .expect(\"ranges should not be empty\")\n            .end_byte;\n\n        layer_start <= start\n            && layer_end >= end\n            && self.ranges.iter().any(|range| {\n                let byte_range = range.start_byte..range.end_byte;\n                byte_range.contains(&start) || byte_range.contains(&end)\n            })\n    }\n}\n\npub(crate) fn generate_edits(\n    old_text: RopeSlice,\n    changeset: &ChangeSet,\n) -> Vec<tree_sitter::InputEdit> {\n    use Operation::*;\n    let mut old_pos = 0;\n\n    let mut edits = Vec::new();\n\n    if changeset.changes.is_empty() {\n        return edits;\n    }\n\n    let mut iter = changeset.changes.iter().peekable();\n\n    // TODO; this is a lot easier with Change instead of Operation.\n\n    fn point_at_pos(text: RopeSlice, pos: usize) -> (usize, Point) {\n        let byte = text.char_to_byte(pos); // <- attempted to index past end\n        let line = text.char_to_line(pos);\n        let line_start_byte = text.line_to_byte(line);\n        let col = byte - line_start_byte;\n\n        (byte, Point::new(line, col))\n    }\n\n    fn traverse(point: Point, text: &Tendril) -> Point {\n        let Point {\n            mut row,\n            mut column,\n        } = point;\n\n        // TODO: there should be a better way here.\n        let mut chars = text.chars().peekable();\n        while let Some(ch) = chars.next() {\n            if char_is_line_ending(ch) && !(ch == '\\r' && chars.peek() == Some(&'\\n')) {\n                row += 1;\n                column = 0;\n            } else {\n                column += 1;\n            }\n        }\n        Point { row, column }\n    }\n\n    while let Some(change) = iter.next() {\n        let len = match change {\n            Delete(i) | Retain(i) => *i,\n            Insert(_) => 0,\n        };\n        let mut old_end = old_pos + len;\n\n        match change {\n            Retain(_) => {}\n            Delete(_) => {\n                let (start_byte, start_position) = point_at_pos(old_text, old_pos);\n                let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);\n\n                // deletion\n                edits.push(tree_sitter::InputEdit {\n                    start_byte,                       // old_pos to byte\n                    old_end_byte,                     // old_end to byte\n                    new_end_byte: start_byte,         // old_pos to byte\n                    start_position,                   // old pos to coords\n                    old_end_position,                 // old_end to coords\n                    new_end_position: start_position, // old pos to coords\n                });\n            }\n            Insert(s) => {\n                let (start_byte, start_position) = point_at_pos(old_text, old_pos);\n\n                // a subsequent delete means a replace, consume it\n                if let Some(Delete(len)) = iter.peek() {\n                    old_end = old_pos + len;\n                    let (old_end_byte, old_end_position) = point_at_pos(old_text, old_end);\n\n                    iter.next();\n\n                    // replacement\n                    edits.push(tree_sitter::InputEdit {\n                        start_byte,                                    // old_pos to byte\n                        old_end_byte,                                  // old_end to byte\n                        new_end_byte: start_byte + s.len(),            // old_pos to byte + s.len()\n                        start_position,                                // old pos to coords\n                        old_end_position,                              // old_end to coords\n                        new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)\n                    });\n                } else {\n                    // insert\n                    edits.push(tree_sitter::InputEdit {\n                        start_byte,                                    // old_pos to byte\n                        old_end_byte: start_byte,                      // same\n                        new_end_byte: start_byte + s.len(),            // old_pos + s.len()\n                        start_position,                                // old pos to coords\n                        old_end_position: start_position,              // same\n                        new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)\n                    });\n                }\n            }\n        }\n        old_pos = old_end;\n    }\n    edits\n}\n\nuse std::sync::atomic::{AtomicUsize, Ordering};\nuse std::{iter, mem, ops, str, usize};\nuse tree_sitter::{\n    Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError,\n    QueryMatch, Range, TextProvider, Tree,\n};\n\nconst CANCELLATION_CHECK_INTERVAL: usize = 100;\n\n/// Indicates which highlight should be applied to a region of source code.\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\npub struct Highlight(pub usize);\n\n/// Represents the reason why syntax highlighting failed.\n#[derive(Debug, PartialEq, Eq)]\npub enum Error {\n    Cancelled,\n    InvalidLanguage,\n    InvalidRanges,\n    Unknown,\n}\n\n/// Represents a single step in rendering a syntax-highlighted document.\n#[derive(Copy, Clone, Debug)]\npub enum HighlightEvent {\n    Source { start: usize, end: usize },\n    HighlightStart(Highlight),\n    HighlightEnd,\n}\n\n/// Contains the data needed to highlight code written in a particular language.\n///\n/// This struct is immutable and can be shared between threads.\n#[derive(Debug)]\npub struct HighlightConfiguration {\n    pub language: Grammar,\n    pub query: Query,\n    injections_query: Query,\n    combined_injections_patterns: Vec<usize>,\n    highlights_pattern_index: usize,\n    highlight_indices: ArcSwap<Vec<Option<Highlight>>>,\n    non_local_variable_patterns: Vec<bool>,\n    injection_content_capture_index: Option<u32>,\n    injection_language_capture_index: Option<u32>,\n    injection_filename_capture_index: Option<u32>,\n    injection_shebang_capture_index: Option<u32>,\n    local_scope_capture_index: Option<u32>,\n    local_def_capture_index: Option<u32>,\n    local_def_value_capture_index: Option<u32>,\n    local_ref_capture_index: Option<u32>,\n}\n\n#[derive(Debug)]\nstruct LocalDef<'a> {\n    name: Cow<'a, str>,\n    value_range: ops::Range<usize>,\n    highlight: Option<Highlight>,\n}\n\n#[derive(Debug)]\nstruct LocalScope<'a> {\n    inherits: bool,\n    range: ops::Range<usize>,\n    local_defs: Vec<LocalDef<'a>>,\n}\n\n#[derive(Debug)]\nstruct HighlightIter<'a> {\n    source: RopeSlice<'a>,\n    byte_offset: usize,\n    cancellation_flag: Option<&'a AtomicUsize>,\n    layers: Vec<HighlightIterLayer<'a>>,\n    iter_count: usize,\n    next_event: Option<HighlightEvent>,\n    last_highlight_range: Option<(usize, usize, u32)>,\n}\n\n// Adapter to convert rope chunks to bytes\npub struct ChunksBytes<'a> {\n    chunks: ropey::iter::Chunks<'a>,\n}\nimpl<'a> Iterator for ChunksBytes<'a> {\n    type Item = &'a [u8];\n    fn next(&mut self) -> Option<Self::Item> {\n        self.chunks.next().map(str::as_bytes)\n    }\n}\n\npub struct RopeProvider<'a>(pub RopeSlice<'a>);\nimpl<'a> TextProvider<&'a [u8]> for RopeProvider<'a> {\n    type I = ChunksBytes<'a>;\n\n    fn text(&mut self, node: Node) -> Self::I {\n        let fragment = self.0.byte_slice(node.start_byte()..node.end_byte());\n        ChunksBytes {\n            chunks: fragment.chunks(),\n        }\n    }\n}\n\nstruct HighlightIterLayer<'a> {\n    _tree: Option<Tree>,\n    cursor: QueryCursor,\n    captures: RefCell<iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>, &'a [u8]>>>,\n    config: &'a HighlightConfiguration,\n    highlight_end_stack: Vec<usize>,\n    scope_stack: Vec<LocalScope<'a>>,\n    depth: u32,\n}\n\nimpl<'a> fmt::Debug for HighlightIterLayer<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"HighlightIterLayer\").finish()\n    }\n}\n\nimpl HighlightConfiguration {\n    /// Creates a `HighlightConfiguration` for a given `Grammar` and set of highlighting\n    /// queries.\n    ///\n    /// # Parameters\n    ///\n    /// * `language`  - The Tree-sitter `Grammar` that should be used for parsing.\n    /// * `highlights_query` - A string containing tree patterns for syntax highlighting. This\n    ///   should be non-empty, otherwise no syntax highlights will be added.\n    /// * `injections_query` -  A string containing tree patterns for injecting other languages\n    ///   into the document. This can be empty if no injections are desired.\n    /// * `locals_query` - A string containing tree patterns for tracking local variable\n    ///   definitions and references. This can be empty if local variable tracking is not needed.\n    ///\n    /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.\n    pub fn new(\n        language: Grammar,\n        highlights_query: &str,\n        injection_query: &str,\n        locals_query: &str,\n    ) -> Result<Self, QueryError> {\n        // Concatenate the query strings, keeping track of the start offset of each section.\n        let mut query_source = String::new();\n        query_source.push_str(locals_query);\n        let highlights_query_offset = query_source.len();\n        query_source.push_str(highlights_query);\n\n        // Construct a single query by concatenating the three query strings, but record the\n        // range of pattern indices that belong to each individual string.\n        let query = Query::new(&language, &query_source)?;\n        let mut highlights_pattern_index = 0;\n        for i in 0..(query.pattern_count()) {\n            let pattern_offset = query.start_byte_for_pattern(i);\n            if pattern_offset < highlights_query_offset {\n                highlights_pattern_index += 1;\n            }\n        }\n\n        let injections_query = Query::new(&language, injection_query)?;\n        let combined_injections_patterns = (0..injections_query.pattern_count())\n            .filter(|&i| {\n                injections_query\n                    .property_settings(i)\n                    .iter()\n                    .any(|s| &*s.key == \"injection.combined\")\n            })\n            .collect();\n\n        // Find all of the highlighting patterns that are disabled for nodes that\n        // have been identified as local variables.\n        let non_local_variable_patterns = (0..query.pattern_count())\n            .map(|i| {\n                query\n                    .property_predicates(i)\n                    .iter()\n                    .any(|(prop, positive)| !*positive && prop.key.as_ref() == \"local\")\n            })\n            .collect();\n\n        // Store the numeric ids for all of the special captures.\n        let mut injection_content_capture_index = None;\n        let mut injection_language_capture_index = None;\n        let mut injection_filename_capture_index = None;\n        let mut injection_shebang_capture_index = None;\n        let mut local_def_capture_index = None;\n        let mut local_def_value_capture_index = None;\n        let mut local_ref_capture_index = None;\n        let mut local_scope_capture_index = None;\n        for (i, name) in query.capture_names().iter().enumerate() {\n            let i = Some(i as u32);\n            match *name {\n                \"local.definition\" => local_def_capture_index = i,\n                \"local.definition-value\" => local_def_value_capture_index = i,\n                \"local.reference\" => local_ref_capture_index = i,\n                \"local.scope\" => local_scope_capture_index = i,\n                _ => {}\n            }\n        }\n\n        for (i, name) in injections_query.capture_names().iter().enumerate() {\n            let i = Some(i as u32);\n            match *name {\n                \"injection.content\" => injection_content_capture_index = i,\n                \"injection.language\" => injection_language_capture_index = i,\n                \"injection.filename\" => injection_filename_capture_index = i,\n                \"injection.shebang\" => injection_shebang_capture_index = i,\n                _ => {}\n            }\n        }\n\n        let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]);\n        Ok(Self {\n            language,\n            query,\n            injections_query,\n            combined_injections_patterns,\n            highlights_pattern_index,\n            highlight_indices,\n            non_local_variable_patterns,\n            injection_content_capture_index,\n            injection_language_capture_index,\n            injection_filename_capture_index,\n            injection_shebang_capture_index,\n            local_scope_capture_index,\n            local_def_capture_index,\n            local_def_value_capture_index,\n            local_ref_capture_index,\n        })\n    }\n\n    /// Get a slice containing all of the highlight names used in the configuration.\n    pub fn names(&self) -> &[&str] {\n        self.query.capture_names()\n    }\n\n    /// Set the list of recognized highlight names.\n    ///\n    /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated\n    /// highlight names like `punctuation.bracket` and `function.method.builtin`. Consumers of\n    /// these queries can choose to recognize highlights with different levels of specificity.\n    /// For example, the string `function.builtin` will match against `function.builtin.constructor`\n    /// but will not match `function.method.builtin` and `function.method`.\n    ///\n    /// When highlighting, results are returned as `Highlight` values, which contain the index\n    /// of the matched highlight this list of highlight names.\n    pub fn configure(&self, recognized_names: &[String]) {\n        let mut capture_parts = Vec::new();\n        let indices: Vec<_> = self\n            .query\n            .capture_names()\n            .iter()\n            .map(move |capture_name| {\n                capture_parts.clear();\n                capture_parts.extend(capture_name.split('.'));\n\n                let mut best_index = None;\n                let mut best_match_len = 0;\n                for (i, recognized_name) in recognized_names.iter().enumerate() {\n                    let mut len = 0;\n                    let mut matches = true;\n                    for (i, part) in recognized_name.split('.').enumerate() {\n                        match capture_parts.get(i) {\n                            Some(capture_part) if *capture_part == part => len += 1,\n                            _ => {\n                                matches = false;\n                                break;\n                            }\n                        }\n                    }\n                    if matches && len > best_match_len {\n                        best_index = Some(i);\n                        best_match_len = len;\n                    }\n                }\n                best_index.map(Highlight)\n            })\n            .collect();\n\n        self.highlight_indices.store(Arc::new(indices));\n    }\n\n    fn injection_pair<'a>(\n        &self,\n        query_match: &QueryMatch<'a, 'a>,\n        source: RopeSlice<'a>,\n    ) -> (Option<InjectionLanguageMarker<'a>>, Option<Node<'a>>) {\n        let mut injection_capture = None;\n        let mut content_node = None;\n\n        for capture in query_match.captures {\n            let index = Some(capture.index);\n            if index == self.injection_language_capture_index {\n                let name = byte_range_to_str(capture.node.byte_range(), source);\n                injection_capture = Some(InjectionLanguageMarker::Name(name));\n            } else if index == self.injection_filename_capture_index {\n                let name = byte_range_to_str(capture.node.byte_range(), source);\n                let path = Path::new(name.as_ref()).to_path_buf();\n                injection_capture = Some(InjectionLanguageMarker::Filename(path.into()));\n            } else if index == self.injection_shebang_capture_index {\n                let node_slice = source.byte_slice(capture.node.byte_range());\n\n                // some languages allow space and newlines before the actual string content\n                // so a shebang could be on either the first or second line\n                let lines = if let Ok(end) = node_slice.try_line_to_byte(2) {\n                    node_slice.byte_slice(..end)\n                } else {\n                    node_slice\n                };\n\n                static SHEBANG_REGEX: Lazy<rope::Regex> =\n                    Lazy::new(|| rope::Regex::new(SHEBANG).unwrap());\n\n                injection_capture = SHEBANG_REGEX\n                    .captures_iter(lines.regex_input())\n                    .map(|cap| {\n                        let cap = lines.byte_slice(cap.get_group(1).unwrap().range());\n                        InjectionLanguageMarker::Shebang(cap.into())\n                    })\n                    .next()\n            } else if index == self.injection_content_capture_index {\n                content_node = Some(capture.node);\n            }\n        }\n        (injection_capture, content_node)\n    }\n\n    fn injection_for_match<'a>(\n        &self,\n        query: &'a Query,\n        query_match: &QueryMatch<'a, 'a>,\n        source: RopeSlice<'a>,\n    ) -> (\n        Option<InjectionLanguageMarker<'a>>,\n        Option<Node<'a>>,\n        IncludedChildren,\n    ) {\n        let (mut injection_capture, content_node) = self.injection_pair(query_match, source);\n\n        let mut included_children = IncludedChildren::default();\n        for prop in query.property_settings(query_match.pattern_index) {\n            match prop.key.as_ref() {\n                // In addition to specifying the language name via the text of a\n                // captured node, it can also be hard-coded via a `#set!` predicate\n                // that sets the injection.language key.\n                \"injection.language\" if injection_capture.is_none() => {\n                    injection_capture = prop\n                        .value\n                        .as_ref()\n                        .map(|s| InjectionLanguageMarker::Name(s.as_ref().into()));\n                }\n\n                // By default, injections do not include the *children* of an\n                // `injection.content` node - only the ranges that belong to the\n                // node itself. This can be changed using a `#set!` predicate that\n                // sets the `injection.include-children` key.\n                \"injection.include-children\" => included_children = IncludedChildren::All,\n\n                // Some queries might only exclude named children but include unnamed\n                // children in their `injection.content` node. This can be enabled using\n                // a `#set!` predicate that sets the `injection.include-unnamed-children` key.\n                \"injection.include-unnamed-children\" => {\n                    included_children = IncludedChildren::Unnamed\n                }\n                _ => {}\n            }\n        }\n\n        (injection_capture, content_node, included_children)\n    }\n}\n\nimpl<'a> HighlightIterLayer<'a> {\n    // First, sort scope boundaries by their byte offset in the document. At a\n    // given position, emit scope endings before scope beginnings. Finally, emit\n    // scope boundaries from deeper layers first.\n    fn sort_key(&self) -> Option<(usize, bool, isize)> {\n        let depth = -(self.depth as isize);\n        let next_start = self\n            .captures\n            .borrow_mut()\n            .peek()\n            .map(|(m, i)| m.captures[*i].node.start_byte());\n        let next_end = self.highlight_end_stack.last().cloned();\n        match (next_start, next_end) {\n            (Some(start), Some(end)) => {\n                if start < end {\n                    Some((start, true, depth))\n                } else {\n                    Some((end, false, depth))\n                }\n            }\n            (Some(i), None) => Some((i, true, depth)),\n            (None, Some(j)) => Some((j, false, depth)),\n            _ => None,\n        }\n    }\n}\n\n#[derive(Clone)]\nenum IncludedChildren {\n    None,\n    All,\n    Unnamed,\n}\n\nimpl Default for IncludedChildren {\n    fn default() -> Self {\n        Self::None\n    }\n}\n\n// Compute the ranges that should be included when parsing an injection.\n// This takes into account three things:\n// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.\n// * `nodes` - Every injection takes place within a set of nodes. The injection ranges\n//   are the ranges of those nodes.\n// * `includes_children` - For some injections, the content nodes' children should be\n//   excluded from the nested document, so that only the content nodes' *own* content\n//   is reparsed. For other injections, the content nodes' entire ranges should be\n//   reparsed, including the ranges of their children.\nfn intersect_ranges(\n    parent_ranges: &[Range],\n    nodes: &[Node],\n    included_children: IncludedChildren,\n) -> Vec<Range> {\n    let mut cursor = nodes[0].walk();\n    let mut result = Vec::new();\n    let mut parent_range_iter = parent_ranges.iter();\n    let mut parent_range = parent_range_iter\n        .next()\n        .expect(\"Layers should only be constructed with non-empty ranges vectors\");\n    for node in nodes.iter() {\n        let mut preceding_range = Range {\n            start_byte: 0,\n            start_point: Point::new(0, 0),\n            end_byte: node.start_byte(),\n            end_point: node.start_position(),\n        };\n        let following_range = Range {\n            start_byte: node.end_byte(),\n            start_point: node.end_position(),\n            end_byte: usize::MAX,\n            end_point: Point::new(usize::MAX, usize::MAX),\n        };\n\n        for excluded_range in node\n            .children(&mut cursor)\n            .filter_map(|child| match included_children {\n                IncludedChildren::None => Some(child.range()),\n                IncludedChildren::All => None,\n                IncludedChildren::Unnamed => {\n                    if child.is_named() {\n                        Some(child.range())\n                    } else {\n                        None\n                    }\n                }\n            })\n            .chain([following_range].iter().cloned())\n        {\n            let mut range = Range {\n                start_byte: preceding_range.end_byte,\n                start_point: preceding_range.end_point,\n                end_byte: excluded_range.start_byte,\n                end_point: excluded_range.start_point,\n            };\n            preceding_range = excluded_range;\n\n            if range.end_byte < parent_range.start_byte {\n                continue;\n            }\n\n            while parent_range.start_byte <= range.end_byte {\n                if parent_range.end_byte > range.start_byte {\n                    if range.start_byte < parent_range.start_byte {\n                        range.start_byte = parent_range.start_byte;\n                        range.start_point = parent_range.start_point;\n                    }\n\n                    if parent_range.end_byte < range.end_byte {\n                        if range.start_byte < parent_range.end_byte {\n                            result.push(Range {\n                                start_byte: range.start_byte,\n                                start_point: range.start_point,\n                                end_byte: parent_range.end_byte,\n                                end_point: parent_range.end_point,\n                            });\n                        }\n                        range.start_byte = parent_range.end_byte;\n                        range.start_point = parent_range.end_point;\n                    } else {\n                        if range.start_byte < range.end_byte {\n                            result.push(range);\n                        }\n                        break;\n                    }\n                }\n\n                if let Some(next_range) = parent_range_iter.next() {\n                    parent_range = next_range;\n                } else {\n                    return result;\n                }\n            }\n        }\n    }\n    result\n}\n\nimpl<'a> HighlightIter<'a> {\n    fn emit_event(\n        &mut self,\n        offset: usize,\n        event: Option<HighlightEvent>,\n    ) -> Option<Result<HighlightEvent, Error>> {\n        let result;\n        if self.byte_offset < offset {\n            result = Some(Ok(HighlightEvent::Source {\n                start: self.byte_offset,\n                end: offset,\n            }));\n            self.byte_offset = offset;\n            self.next_event = event;\n        } else {\n            result = event.map(Ok);\n        }\n        self.sort_layers();\n        result\n    }\n\n    fn sort_layers(&mut self) {\n        while !self.layers.is_empty() {\n            if let Some(sort_key) = self.layers[0].sort_key() {\n                let mut i = 0;\n                while i + 1 < self.layers.len() {\n                    if let Some(next_offset) = self.layers[i + 1].sort_key() {\n                        if next_offset < sort_key {\n                            i += 1;\n                            continue;\n                        }\n                    } else {\n                        let layer = self.layers.remove(i + 1);\n                        PARSER.with(|ts_parser| {\n                            let highlighter = &mut ts_parser.borrow_mut();\n                            highlighter.cursors.push(layer.cursor);\n                        });\n                    }\n                    break;\n                }\n                if i > 0 {\n                    self.layers[0..(i + 1)].rotate_left(1);\n                }\n                break;\n            } else {\n                let layer = self.layers.remove(0);\n                PARSER.with(|ts_parser| {\n                    let highlighter = &mut ts_parser.borrow_mut();\n                    highlighter.cursors.push(layer.cursor);\n                });\n            }\n        }\n    }\n}\n\nimpl<'a> Iterator for HighlightIter<'a> {\n    type Item = Result<HighlightEvent, Error>;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        'main: loop {\n            // If we've already determined the next highlight boundary, just return it.\n            if let Some(e) = self.next_event.take() {\n                return Some(Ok(e));\n            }\n\n            // Periodically check for cancellation, returning `Cancelled` error if the\n            // cancellation flag was flipped.\n            if let Some(cancellation_flag) = self.cancellation_flag {\n                self.iter_count += 1;\n                if self.iter_count >= CANCELLATION_CHECK_INTERVAL {\n                    self.iter_count = 0;\n                    if cancellation_flag.load(Ordering::Relaxed) != 0 {\n                        return Some(Err(Error::Cancelled));\n                    }\n                }\n            }\n\n            // If none of the layers have any more highlight boundaries, terminate.\n            if self.layers.is_empty() {\n                let len = self.source.len_bytes();\n                return if self.byte_offset < len {\n                    let result = Some(Ok(HighlightEvent::Source {\n                        start: self.byte_offset,\n                        end: len,\n                    }));\n                    self.byte_offset = len;\n                    result\n                } else {\n                    None\n                };\n            }\n\n            // Get the next capture from whichever layer has the earliest highlight boundary.\n            let range;\n            let layer = &mut self.layers[0];\n            let captures = layer.captures.get_mut();\n            if let Some((next_match, capture_index)) = captures.peek() {\n                let next_capture = next_match.captures[*capture_index];\n                range = next_capture.node.byte_range();\n\n                // If any previous highlight ends before this node starts, then before\n                // processing this capture, emit the source code up until the end of the\n                // previous highlight, and an end event for that highlight.\n                if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {\n                    if end_byte <= range.start {\n                        layer.highlight_end_stack.pop();\n                        return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));\n                    }\n                }\n            }\n            // If there are no more captures, then emit any remaining highlight end events.\n            // And if there are none of those, then just advance to the end of the document.\n            else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {\n                layer.highlight_end_stack.pop();\n                return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));\n            } else {\n                return self.emit_event(self.source.len_bytes(), None);\n            };\n\n            let (mut match_, capture_index) = captures.next().unwrap();\n            let mut capture = match_.captures[capture_index];\n\n            // Remove from the local scope stack any local scopes that have already ended.\n            while range.start > layer.scope_stack.last().unwrap().range.end {\n                layer.scope_stack.pop();\n            }\n\n            // If this capture is for tracking local variables, then process the\n            // local variable info.\n            let mut reference_highlight = None;\n            let mut definition_highlight = None;\n            while match_.pattern_index < layer.config.highlights_pattern_index {\n                // If the node represents a local scope, push a new local scope onto\n                // the scope stack.\n                if Some(capture.index) == layer.config.local_scope_capture_index {\n                    definition_highlight = None;\n                    let mut scope = LocalScope {\n                        inherits: true,\n                        range: range.clone(),\n                        local_defs: Vec::new(),\n                    };\n                    for prop in layer.config.query.property_settings(match_.pattern_index) {\n                        if let \"local.scope-inherits\" = prop.key.as_ref() {\n                            scope.inherits =\n                                prop.value.as_ref().map_or(true, |r| r.as_ref() == \"true\");\n                        }\n                    }\n                    layer.scope_stack.push(scope);\n                }\n                // If the node represents a definition, add a new definition to the\n                // local scope at the top of the scope stack.\n                else if Some(capture.index) == layer.config.local_def_capture_index {\n                    reference_highlight = None;\n                    let scope = layer.scope_stack.last_mut().unwrap();\n\n                    let mut value_range = 0..0;\n                    for capture in match_.captures {\n                        if Some(capture.index) == layer.config.local_def_value_capture_index {\n                            value_range = capture.node.byte_range();\n                        }\n                    }\n\n                    let name = byte_range_to_str(range.clone(), self.source);\n                    scope.local_defs.push(LocalDef {\n                        name,\n                        value_range,\n                        highlight: None,\n                    });\n                    definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);\n                }\n                // If the node represents a reference, then try to find the corresponding\n                // definition in the scope stack.\n                else if Some(capture.index) == layer.config.local_ref_capture_index\n                    && definition_highlight.is_none()\n                {\n                    definition_highlight = None;\n                    let name = byte_range_to_str(range.clone(), self.source);\n                    for scope in layer.scope_stack.iter().rev() {\n                        if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {\n                            if def.name == name && range.start >= def.value_range.end {\n                                Some(def.highlight)\n                            } else {\n                                None\n                            }\n                        }) {\n                            reference_highlight = highlight;\n                            break;\n                        }\n                        if !scope.inherits {\n                            break;\n                        }\n                    }\n                }\n\n                // Continue processing any additional matches for the same node.\n                if let Some((next_match, next_capture_index)) = captures.peek() {\n                    let next_capture = next_match.captures[*next_capture_index];\n                    if next_capture.node == capture.node {\n                        capture = next_capture;\n                        match_ = captures.next().unwrap().0;\n                        continue;\n                    }\n                }\n\n                self.sort_layers();\n                continue 'main;\n            }\n\n            // Otherwise, this capture must represent a highlight.\n            // If this exact range has already been highlighted by an earlier pattern, or by\n            // a different layer, then skip over this one.\n            if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {\n                if range.start == last_start && range.end == last_end && layer.depth < last_depth {\n                    self.sort_layers();\n                    continue 'main;\n                }\n            }\n\n            // If the current node was found to be a local variable, then skip over any\n            // highlighting patterns that are disabled for local variables.\n            if definition_highlight.is_some() || reference_highlight.is_some() {\n                while layer.config.non_local_variable_patterns[match_.pattern_index] {\n                    match_.remove();\n                    if let Some((next_match, next_capture_index)) = captures.peek() {\n                        let next_capture = next_match.captures[*next_capture_index];\n                        if next_capture.node == capture.node {\n                            capture = next_capture;\n                            match_ = captures.next().unwrap().0;\n                            continue;\n                        }\n                    }\n\n                    self.sort_layers();\n                    continue 'main;\n                }\n            }\n\n            // Once a highlighting pattern is found for the current node, skip over\n            // any later highlighting patterns that also match this node. Captures\n            // for a given node are ordered by pattern index, so these subsequent\n            // captures are guaranteed to be for highlighting, not injections or\n            // local variables.\n            while let Some((next_match, next_capture_index)) = captures.peek() {\n                let next_capture = next_match.captures[*next_capture_index];\n                if next_capture.node == capture.node {\n                    captures.next();\n                } else {\n                    break;\n                }\n            }\n\n            let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];\n\n            // If this node represents a local definition, then store the current\n            // highlight value on the local scope entry representing this node.\n            if let Some(definition_highlight) = definition_highlight {\n                *definition_highlight = current_highlight;\n            }\n\n            // Emit a scope start event and push the node's end position to the stack.\n            if let Some(highlight) = reference_highlight.or(current_highlight) {\n                self.last_highlight_range = Some((range.start, range.end, layer.depth));\n                layer.highlight_end_stack.push(range.end);\n                return self\n                    .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));\n            }\n\n            self.sort_layers();\n        }\n    }\n}\n\n#[derive(Debug, Clone)]\npub enum InjectionLanguageMarker<'a> {\n    Name(Cow<'a, str>),\n    Filename(Cow<'a, Path>),\n    Shebang(String),\n}\n\nconst SHEBANG: &str = r\"#!\\s*(?:\\S*[/\\\\](?:env\\s+(?:\\-\\S+\\s+)*)?)?([^\\s\\.\\d]+)\";\n\npub struct Merge<I> {\n    iter: I,\n    spans: Box<dyn Iterator<Item = (usize, std::ops::Range<usize>)>>,\n\n    next_event: Option<HighlightEvent>,\n    next_span: Option<(usize, std::ops::Range<usize>)>,\n\n    queue: Vec<HighlightEvent>,\n}\n\n/// Merge a list of spans into the highlight event stream.\npub fn merge<I: Iterator<Item = HighlightEvent>>(\n    iter: I,\n    spans: Vec<(usize, std::ops::Range<usize>)>,\n) -> Merge<I> {\n    let spans = Box::new(spans.into_iter());\n    let mut merge = Merge {\n        iter,\n        spans,\n        next_event: None,\n        next_span: None,\n        queue: Vec::new(),\n    };\n    merge.next_event = merge.iter.next();\n    merge.next_span = merge.spans.next();\n    merge\n}\n\nimpl<I: Iterator<Item = HighlightEvent>> Iterator for Merge<I> {\n    type Item = HighlightEvent;\n    fn next(&mut self) -> Option<Self::Item> {\n        use HighlightEvent::*;\n        if let Some(event) = self.queue.pop() {\n            return Some(event);\n        }\n\n        loop {\n            match (self.next_event, &self.next_span) {\n                // this happens when range is partially or fully offscreen\n                (Some(Source { start, .. }), Some((span, range))) if start > range.start => {\n                    if start > range.end {\n                        self.next_span = self.spans.next();\n                    } else {\n                        self.next_span = Some((*span, start..range.end));\n                    };\n                }\n                _ => break,\n            }\n        }\n\n        match (self.next_event, &self.next_span) {\n            (Some(HighlightStart(i)), _) => {\n                self.next_event = self.iter.next();\n                Some(HighlightStart(i))\n            }\n            (Some(HighlightEnd), _) => {\n                self.next_event = self.iter.next();\n                Some(HighlightEnd)\n            }\n            (Some(Source { start, end }), Some((_, range))) if start < range.start => {\n                let intersect = range.start.min(end);\n                let event = Source {\n                    start,\n                    end: intersect,\n                };\n\n                if end == intersect {\n                    // the event is complete\n                    self.next_event = self.iter.next();\n                } else {\n                    // subslice the event\n                    self.next_event = Some(Source {\n                        start: intersect,\n                        end,\n                    });\n                };\n\n                Some(event)\n            }\n            (Some(Source { start, end }), Some((span, range))) if start == range.start => {\n                let intersect = range.end.min(end);\n                let event = HighlightStart(Highlight(*span));\n\n                // enqueue in reverse order\n                self.queue.push(HighlightEnd);\n                self.queue.push(Source {\n                    start,\n                    end: intersect,\n                });\n\n                if end == intersect {\n                    // the event is complete\n                    self.next_event = self.iter.next();\n                } else {\n                    // subslice the event\n                    self.next_event = Some(Source {\n                        start: intersect,\n                        end,\n                    });\n                };\n\n                if intersect == range.end {\n                    self.next_span = self.spans.next();\n                } else {\n                    self.next_span = Some((*span, intersect..range.end));\n                }\n\n                Some(event)\n            }\n            (Some(event), None) => {\n                self.next_event = self.iter.next();\n                Some(event)\n            }\n            // Can happen if cursor at EOF and/or diagnostic reaches past the end.\n            // We need to actually emit events for the cursor-at-EOF situation,\n            // even though the range is past the end of the text.  This needs to be\n            // handled appropriately by the drawing code by not assuming that\n            // all `Source` events point to valid indices in the rope.\n            (None, Some((span, range))) => {\n                let event = HighlightStart(Highlight(*span));\n                self.queue.push(HighlightEnd);\n                self.queue.push(Source {\n                    start: range.start,\n                    end: range.end,\n                });\n                self.next_span = self.spans.next();\n                Some(event)\n            }\n            (None, None) => None,\n            e => unreachable!(\"{:?}\", e),\n        }\n    }\n}\n\nfn node_is_visible(node: &Node) -> bool {\n    node.is_missing() || (node.is_named() && node.language().node_kind_is_visible(node.kind_id()))\n}\n\npub fn pretty_print_tree<W: fmt::Write>(fmt: &mut W, node: Node) -> fmt::Result {\n    if node.child_count() == 0 {\n        if node_is_visible(&node) {\n            write!(fmt, \"({})\", node.kind())\n        } else {\n            write!(fmt, \"\\\"{}\\\"\", node.kind())\n        }\n    } else {\n        pretty_print_tree_impl(fmt, &mut node.walk(), 0)\n    }\n}\n\nfn pretty_print_tree_impl<W: fmt::Write>(\n    fmt: &mut W,\n    cursor: &mut tree_sitter::TreeCursor,\n    depth: usize,\n) -> fmt::Result {\n    let node = cursor.node();\n    let visible = node_is_visible(&node);\n\n    if visible {\n        let indentation_columns = depth * 2;\n        write!(fmt, \"{:indentation_columns$}\", \"\")?;\n\n        if let Some(field_name) = cursor.field_name() {\n            write!(fmt, \"{}: \", field_name)?;\n        }\n\n        write!(fmt, \"({}\", node.kind())?;\n    }\n\n    // Handle children.\n    if cursor.goto_first_child() {\n        loop {\n            if node_is_visible(&cursor.node()) {\n                fmt.write_char('\\n')?;\n            }\n\n            pretty_print_tree_impl(fmt, cursor, depth + 1)?;\n\n            if !cursor.goto_next_sibling() {\n                break;\n            }\n        }\n\n        let moved = cursor.goto_parent();\n        // The parent of the first child must exist, and must be `node`.\n        debug_assert!(moved);\n        debug_assert!(cursor.node() == node);\n    }\n\n    if visible {\n        fmt.write_char(')')?;\n    }\n\n    Ok(())\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::{Rope, Transaction};\n\n    #[test]\n    fn test_textobject_queries() {\n        let query_str = r#\"\n        (line_comment)+ @quantified_nodes\n        ((line_comment)+) @quantified_nodes_grouped\n        ((line_comment) (line_comment)) @multiple_nodes_grouped\n        \"#;\n        let source = Rope::from_str(\n            r#\"\n/// a comment on\n/// multiple lines\n        \"#,\n        );\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n        let language = get_language(\"rust\").unwrap();\n\n        let query = Query::new(&language, query_str).unwrap();\n        let textobject = TextObjectQuery { query };\n        let mut cursor = QueryCursor::new();\n\n        let config = HighlightConfiguration::new(language, \"\", \"\", \"\").unwrap();\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n\n        let root = syntax.tree().root_node();\n        let mut test = |capture, range| {\n            let matches: Vec<_> = textobject\n                .capture_nodes(capture, root, source.slice(..), &mut cursor)\n                .unwrap()\n                .collect();\n\n            assert_eq!(\n                matches[0].byte_range(),\n                range,\n                \"@{} expected {:?}\",\n                capture,\n                range\n            )\n        };\n\n        test(\"quantified_nodes\", 1..36);\n        // NOTE: Enable after implementing proper node group capturing\n        // test(\"quantified_nodes_grouped\", 1..36);\n        // test(\"multiple_nodes_grouped\", 1..36);\n    }\n\n    #[test]\n    fn test_parser() {\n        let highlight_names: Vec<String> = [\n            \"attribute\",\n            \"constant\",\n            \"function.builtin\",\n            \"function\",\n            \"keyword\",\n            \"operator\",\n            \"property\",\n            \"punctuation\",\n            \"punctuation.bracket\",\n            \"punctuation.delimiter\",\n            \"string\",\n            \"string.special\",\n            \"tag\",\n            \"type\",\n            \"type.builtin\",\n            \"variable\",\n            \"variable.builtin\",\n            \"variable.parameter\",\n        ]\n        .iter()\n        .cloned()\n        .map(String::from)\n        .collect();\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n\n        let language = get_language(\"rust\").unwrap();\n        let config = HighlightConfiguration::new(\n            language,\n            &std::fs::read_to_string(\"../runtime/grammars/sources/rust/queries/highlights.scm\")\n                .unwrap(),\n            &std::fs::read_to_string(\"../runtime/grammars/sources/rust/queries/injections.scm\")\n                .unwrap(),\n            \"\", // locals.scm\n        )\n        .unwrap();\n        config.configure(&highlight_names);\n\n        let source = Rope::from_str(\n            \"\n            struct Stuff {}\n            fn main() {}\n        \",\n        );\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n        let tree = syntax.tree();\n        let root = tree.root_node();\n        assert_eq!(root.kind(), \"source_file\");\n\n        assert_eq!(\n            root.to_sexp(),\n            concat!(\n                \"(source_file \",\n                \"(struct_item name: (type_identifier) body: (field_declaration_list)) \",\n                \"(function_item name: (identifier) parameters: (parameters) body: (block)))\"\n            )\n        );\n\n        let struct_node = root.child(0).unwrap();\n        assert_eq!(struct_node.kind(), \"struct_item\");\n    }\n\n    #[test]\n    fn\n test_input_edits() {\n        use tree_sitter::InputEdit;\n\n        let doc = Rope::from(\"hello world!\\ntest 123\");\n        let transaction = Transaction::change(\n            &doc,\n            vec![(6, 11, Some(\"test\".into())), (12, 17, None)].into_iter(),\n        );\n        let edits = generate_edits(doc.slice(..), transaction.changes());\n        // transaction.apply(&mut state);\n\n        assert_eq!(\n            edits,\n            &[\n                InputEdit {\n                    start_byte: 6,\n                    old_end_byte: 11,\n                    new_end_byte: 10,\n                    start_position: Point { row: 0, column: 6 },\n                    old_end_position: Point { row: 0, column: 11 },\n                    new_end_position: Point { row: 0, column: 10 }\n                },\n                InputEdit {\n                    start_byte: 12,\n                    old_end_byte: 17,\n                    new_end_byte: 12,\n                    start_position: Point { row: 0, column: 12 },\n                    old_end_position: Point { row: 1, column: 4 },\n                    new_end_position: Point { row: 0, column: 12 }\n                }\n            ]\n        );\n\n        // Testing with the official example from tree-sitter\n        let mut doc = Rope::from(\"fn test() {}\");\n        let transaction =\n            Transaction::change(&doc, vec![(8, 8, Some(\"a: u32\".into()))].into_iter());\n        let edits = generate_edits(doc.slice(..), transaction.changes());\n        transaction.apply(&mut doc);\n\n        assert_eq!(doc, \"fn test(a: u32) {}\");\n        assert_eq!(\n            edits,\n            &[InputEdit {\n                start_byte: 8,\n                old_end_byte: 8,\n                new_end_byte: 14,\n                start_position: Point { row: 0, column: 8 },\n                old_end_position: Point { row: 0, column: 8 },\n                new_end_position: Point { row: 0, column: 14 }\n            }]\n        );\n    }\n\n    #[track_caller]\n    fn assert_pretty_print(\n        language_name: &str,\n        source: &str,\n        expected: &str,\n        start: usize,\n        end: usize,\n    ) {\n        let source = Rope::from_str(source);\n\n        let loader = Loader::new(Configuration {\n            language: vec![],\n            language_server: HashMap::new(),\n        })\n        .unwrap();\n        let language = get_language(language_name).unwrap();\n\n        let config = HighlightConfiguration::new(language, \"\", \"\", \"\").unwrap();\n        let syntax = Syntax::new(\n            source.slice(..),\n            Arc::new(config),\n            Arc::new(ArcSwap::from_pointee(loader)),\n        )\n        .unwrap();\n\n        let root = syntax\n            .tree()\n            .root_node()\n            .descendant_for_byte_range(start, end)\n            .unwrap();\n\n        let mut output = String::new();\n        pretty_print_tree(&mut output, root).unwrap();\n\n        assert_eq!(expected, output);\n    }\n\n    #[test]\n    fn test_pretty_print() {\n        let source = r#\"/// Hello\"#;\n        assert_pretty_print(\"rust\", source, \"(line_comment)\", 0, source.len());\n\n        // A large tree should be indented with fields:\n        let source = r#\"fn main() {\n            println!(\"Hello, World!\");\n        }\"#;\n        assert_pretty_print(\n            \"rust\",\n            source,\n            concat!(\n                \"(function_item\\n\",\n                \"  name: (identifier)\\n\",\n                \"  parameters: (parameters)\\n\",\n                \"  body: (block\\n\",\n                \"    (expression_statement\\n\",\n                \"      (macro_invocation\\n\",\n                \"        macro: (identifier)\\n\",\n                \"        (token_tree\\n\",\n                \"          (string_literal))))))\",\n            ),\n            0,\n            source.len(),\n        );\n\n...\n", "needle_token_start": 15106, "needle_token_end": 15706, "code_context_ntokens": 16220, "output": ["```rust\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::{Rope, Transaction};\n\n    #[test]\n    fn test_input_edits() {\n        use tree_sitter::InputEdit;\n\n        let doc = Rope::from(\"hello world!\\ntest 123\");\n        let transaction = Transaction::change(\n            &doc,\n            vec![(6, 11, Some(\"test\".into())), (12, 17, None)].into_iter(),\n        );\n        let edits = generate_edits(doc.slice(..), transaction.changes());\n        // transaction.apply(&mut state);\n\n        assert_eq!(\n            edits,\n            &[\n                InputEdit {\n                    start_byte: 6,\n                    old_end_byte: 11,\n                    new_end_byte: 10,\n                    start_position: Point { row: 0, column: 6 },\n                    old_end_position: Point { row: 0, column: 11 },\n                    new_end_position: Point { row: 0, column: 10 }\n                },\n                InputEdit {\n                    start_byte: 12,\n                    old_end_byte: 17,\n                    new_end_byte: 12,\n                    start_position: Point { row: 0, column: 12 },\n                    old_end_position: Point { row: 1, column: 4 },\n                    new_end_position: Point { row: 0, column: 12 }\n                }\n            ]\n        );\n\n        // Testing with the official example from tree-sitter\n        let mut doc = Rope::from(\"fn test() {}\");\n        let transaction =\n            Transaction::change(&doc, vec![(8, 8, Some(\"a: u32\".into()))].into_iter());\n        let edits = generate_edits(doc.slice(..), transaction.changes());\n        transaction.apply(&mut doc);\n\n        assert_eq!(doc, \"fn test(a: u32) {}\");\n        assert_eq!(\n            edits,\n            &[InputEdit {\n                start_byte: 8,\n                old_end_byte: 8,\n                new_end_byte: 14,\n                start_position: Point { row: 0, column: 8 },\n                old_end_position: Point { row: 0, column: 8 },\n                new_end_position: Point { row: 0, column: 14 }\n            }]\n        );\n    }\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "get_config_profile", "language": "rust", "path": "src/cargo/core/profiles.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To retrieve a specific configuration profile from a workspace's global context based on a given profile name.\n2. **Input**: The function takes two parameters: a reference to a workspace and a string representing the name of the profile.\n3. **Output**: Returns an optional profile configuration object if found, wrapped in a result to handle potential errors.\n4. **Procedure**: \n   - The function constructs a key by appending the profile name to a base string and attempts to fetch the corresponding profile from the workspace's global context.\n   - If the profile is not found, it returns `None`.\n   - If found, the profile is validated for correctness based on certain workspace settings and unstable features.\n   - Any warnings generated during validation are displayed.\n   - Finally, if the profile is valid, it is returned; otherwise, an error is raised.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                    .and_then(|t| t.to_resolved_compile_target(self.root_compile_kind)),\n                None => self.artifact_target_for_features,\n            },\n        )\n    }\n\n    pub(crate) fn root_compile_kind(&self) -> CompileKind {\n        self.root_compile_kind\n    }\n}\n\n/// Takes the manifest profiles, and overlays the config profiles on-top.\n///\n/// Returns a new copy of the profile map with all the mergers complete.\nfn merge_config_profiles(\n    ws: &Workspace<'_>,\n    requested_profile: InternedString,\n) -> CargoResult<BTreeMap<InternedString, TomlProfile>> {\n    let mut profiles = match ws.profiles() {\n        Some(profiles) => profiles\n            .get_all()\n            .iter()\n            .map(|(k, v)| (InternedString::new(k), v.clone()))\n            .collect(),\n        None => BTreeMap::new(),\n    };\n    // Set of profile names to check if defined in config only.\n    let mut check_to_add = HashSet::new();\n    check_to_add.insert(requested_profile);\n    // Merge config onto manifest profiles.\n    for (name, profile) in &mut profiles {\n        if let Some(config_profile) = get_config_profile(ws, name)? {\n            profile.merge(&config_profile);\n        }\n        if let Some(inherits) = &profile.inherits {\n            check_to_add.insert(InternedString::new(inherits));\n        }\n    }\n    // Add the built-in profiles. This is important for things like `cargo\n    // test` which implicitly use the \"dev\" profile for dependencies.\n    for name in &[\"dev\", \"release\", \"test\", \"bench\"] {\n        check_to_add.insert(InternedString::new(name));\n    }\n    // Add config-only profiles.\n    // Need to iterate repeatedly to get all the inherits values.\n    let mut current = HashSet::new();\n    while !check_to_add.is_empty() {\n        std::mem::swap(&mut current, &mut check_to_add);\n        for name in current.drain() {\n            if !profiles.contains_key(name.as_str()) {\n                if let Some(config_profile) = get_config_profile(ws, &name)? {\n                    if let Some(inherits) = &config_profile.inherits {\n                        check_to_add.insert(InternedString::new(inherits));\n                    }\n                    profiles.insert(name, config_profile);\n                }\n            }\n        }\n    }\n    Ok(profiles)\n}\n\n/// Helper for fetching a profile from config.\nfn get_config_profile(ws: &Workspace\n<'_>, name: &str) -> CargoResult<Option<TomlProfile>> {\n    let profile: Option<context::Value<TomlProfile>> =\n        ws.gctx().get(&format!(\"profile.{}\", name))?;\n    let Some(profile) = profile else {\n        return Ok(None);\n    };\n    let mut warnings = Vec::new();\n    validate_profile(\n        &profile.val,\n        name,\n        ws.gctx().cli_unstable(),\n        ws.unstable_features(),\n        &mut warnings,\n    )\n    .with_context(|| {\n        format!(\n            \"config profile `{}` is not valid (defined in `{}`)\",\n            name, profile.definition\n        )\n    })?;\n    for warning in warnings {\n        ws.gctx().shell().warn(warning)?;\n    }\n    Ok(Some(profile.val))\n}\n\n/// Validate that a package does not match multiple package override specs.\n///\n/// For example `[profile.dev.package.bar]` and `[profile.dev.package.\"bar:0.5.0\"]`\n/// would both match `bar:0.5.0` which would be ambiguous.\nfn validate_packages_unique(\n    resolve: &Resolve,\n    name: &str,\n    toml: &Option<TomlProfile>,\n) -> CargoResult<HashSet<PackageIdSpec>> {\n    let Some(toml) = toml else {\n        return Ok(HashSet::new());\n    };\n    let Some(overrides) = toml.package.as_ref() else {\n        return Ok(HashSet::new());\n    };\n    // Verify that a package doesn't match multiple spec overrides.\n    let mut found = HashSet::new();\n    for pkg_id in resolve.iter() {\n        let matches: Vec<&PackageIdSpec> = overrides\n            .keys()\n            .filter_map(|key| match *key {\n                ProfilePackageSpec::All => None,\n                ProfilePackageSpec::Spec(ref spec) => {\n                    if spec.matches(pkg_id) {\n                        Some(spec)\n                    } else {\n                        None\n                    }\n                }\n            })\n            .collect();\n        match matches.len() {\n            0 => {}\n            1 => {\n                found.insert(matches[0].clone());\n            }\n            _ => {\n                let specs = matches\n                    .iter()\n                    .map(|spec| spec.to_string())\n                    .collect::<Vec<_>>()\n                    .join(\", \");\n                bail!(\n                    \"multiple package overrides in profile `{}` match package `{}`\\n\\\n                     found package specs: {}\",\n                    name,\n                    pkg_id,\n                    specs\n                );\n            }\n        }\n    }\n    Ok(found)\n}\n\n/// Check for any profile override specs that do not match any known packages.\n///\n/// This helps check for typos and mistakes.\nfn validate_packages_unmatched(\n    shell: &mut Shell,\n    resolve: &Resolve,\n    name: &str,\n    toml: &TomlProfile,\n    found: &HashSet<PackageIdSpec>,\n) -> CargoResult<()> {\n    let Some(overrides) = toml.package.as_ref() else {\n        return Ok(());\n    };\n\n    // Verify every override matches at least one package.\n    let missing_specs = overrides.keys().filter_map(|key| {\n        if let ProfilePackageSpec::Spec(ref spec) = *key {\n            if !found.contains(spec) {\n                return Some(spec);\n            }\n        }\n        None\n    });\n    for spec in missing_specs {\n        // See if there is an exact name match.\n        let name_matches: Vec<String> = resolve\n            .iter()\n            .filter_map(|pkg_id| {\n                if pkg_id.name() == spec.name() {\n                    Some(pkg_id.to_string())\n                } else {\n                    None\n                }\n            })\n            .collect();\n        if name_matches.is_empty() {\n            let suggestion = closest_msg(&spec.name(), resolve.iter(), |p| p.name().as_str());\n            shell.warn(format!(\n                \"profile package spec `{}` in profile `{}` did not match any packages{}\",\n                spec, name, suggestion\n            ))?;\n        } else {\n            shell.warn(format!(\n                \"profile package spec `{}` in profile `{}` \\\n                 has a version or URL that does not match any of the packages: {}\",\n                spec,\n                name,\n                name_matches.join(\", \")\n            ))?;\n        }\n    }\n    Ok(())\n}\n\n/// Returns `true` if a string is a toggle that turns an option off.\nfn is_off(s: &str) -> bool {\n    matches!(s, \"off\" | \"n\" | \"no\" | \"none\")\n}\n\n// Path: src/cargo/core/compiler/build_context/mod.rs\n//! [`BuildContext`] is a (mostly) static information about a build task.\n\nuse crate::core::compiler::unit_graph::UnitGraph;\nuse crate::core::compiler::{BuildConfig, CompileKind, Unit};\nuse crate::core::profiles::Profiles;\nuse crate::core::PackageSet;\nuse crate::core::Workspace;\nuse crate::util::context::GlobalContext;\nuse crate::util::errors::CargoResult;\nuse crate::util::interning::InternedString;\nuse crate::util::Rustc;\nuse std::collections::{HashMap, HashSet};\n\nmod target_info;\npub use self::target_info::{\n    FileFlavor, FileType, RustDocFingerprint, RustcTargetData, TargetInfo,\n};\n\n/// The build context, containing complete information needed for a build task\n/// before it gets started.\n///\n/// It is intended that this is mostly static information. Stuff that mutates\n/// during the build can be found in the parent [`BuildRunner`]. (I say mostly,\n/// because this has internal caching, but nothing that should be observable\n/// or require &mut.)\n///\n/// As a result, almost every field on `BuildContext` is public, including\n///\n/// * a resolved [`UnitGraph`] of your dependencies,\n/// * a [`Profiles`] containing compiler flags presets,\n/// * a [`RustcTargetData`] containing host and target platform information,\n/// * and a [`PackageSet`] for further package downloads,\n///\n/// just to name a few. Learn more on each own documentation.\n///\n/// # How to use\n///\n/// To prepare a build task, you may not want to use [`BuildContext::new`] directly,\n/// since it is often too lower-level.\n/// Instead, [`ops::create_bcx`] is usually what you are looking for.\n///\n/// After a `BuildContext` is built, the next stage of building is handled in [`BuildRunner`].\n///\n/// [`BuildRunner`]: crate::core::compiler::BuildRunner\n/// [`ops::create_bcx`]: crate::ops::create_bcx\npub struct BuildContext<'a, 'gctx> {\n    /// The workspace the build is for.\n    pub ws: &'a Workspace<'gctx>,\n\n    /// The cargo context.\n    pub gctx: &'gctx GlobalContext,\n\n    /// This contains a collection of compiler flags presets.\n    pub profiles: Profiles,\n\n    /// Configuration information for a rustc build.\n    pub build_config: &'a BuildConfig,\n\n    /// Extra compiler args for either `rustc` or `rustdoc`.\n    pub extra_compiler_args: HashMap<Unit, Vec<String>>,\n\n    /// Package downloader.\n    ///\n    /// This holds ownership of the `Package` objects.\n    pub packages: PackageSet<'gctx>,\n\n    /// Information about rustc and the target platform.\n    pub target_data: RustcTargetData<'gctx>,\n\n    /// The root units of `unit_graph` (units requested on the command-line).\n    pub roots: Vec<Unit>,\n\n    /// The dependency graph of units to compile.\n    pub unit_graph: UnitGraph,\n\n    /// Reverse-dependencies of documented units, used by the `rustdoc --scrape-examples` flag.\n    pub scrape_units: Vec<Unit>,\n\n    /// The list of all kinds that are involved in this build\n    pub all_kinds: HashSet<CompileKind>,\n}\n\nimpl<'a, 'gctx> BuildContext<'a, 'gctx> {\n    pub fn new(\n        ws: &'a Workspace<'gctx>,\n        packages: PackageSet<'gctx>,\n        build_config: &'a BuildConfig,\n        profiles: Profiles,\n        extra_compiler_args: HashMap<Unit, Vec<String>>,\n        target_data: RustcTargetData<'gctx>,\n        roots: Vec<Unit>,\n        unit_graph: UnitGraph,\n        scrape_units: Vec<Unit>,\n    ) -> CargoResult<BuildContext<'a, 'gctx>> {\n        let all_kinds = unit_graph\n            .keys()\n            .map(|u| u.kind)\n            .chain(build_config.requested_kinds.iter().copied())\n            .chain(std::iter::once(CompileKind::Host))\n            .collect();\n\n        Ok(BuildContext {\n            ws,\n            gctx: ws.gctx(),\n            packages,\n            build_config,\n            profiles,\n            extra_compiler_args,\n            target_data,\n            roots,\n            unit_graph,\n            scrape_units,\n            all_kinds,\n        })\n    }\n\n    /// Information of the `rustc` this build task will use.\n    pub fn rustc(&self) -> &Rustc {\n        &self.target_data.rustc\n    }\n\n    /// Gets the host architecture triple.\n    ///\n    /// For example, x86_64-unknown-linux-gnu, would be\n    /// - machine: x86_64,\n    /// - hardware-platform: unknown,\n    /// - operating system: linux-gnu.\n    pub fn host_triple(&self) -> InternedString {\n        self.target_data.rustc.host\n    }\n\n    /// Gets the number of jobs specified for this build.\n    pub fn jobs(&self) -> u32 {\n        self.build_config.jobs\n    }\n\n    /// Extra compiler flags to pass to `rustc` for a given unit.\n    ///\n    /// Although it depends on the caller, in the current Cargo implementation,\n    /// these flags take precedence over those from [`BuildContext::extra_args_for`].\n    ///\n    /// As of now, these flags come from environment variables and configurations.\n    /// See [`TargetInfo.rustflags`] for more on how Cargo collects them.\n    ///\n    /// [`TargetInfo.rustflags`]: TargetInfo::rustflags\n    pub fn rustflags_args(&self, unit: &Unit) -> &[String] {\n        &self.target_data.info(unit.kind).rustflags\n    }\n\n    /// Extra compiler flags to pass to `rustdoc` for a given unit.\n    ///\n    /// Although it depends on the caller, in the current Cargo implementation,\n    /// these flags take precedence over those from [`BuildContext::extra_args_for`].\n    ///\n    /// As of now, these flags come from environment variables and configurations.\n    /// See [`TargetInfo.rustdocflags`] for more on how Cargo collects them.\n    ///\n    /// [`TargetInfo.rustdocflags`]: TargetInfo::rustdocflags\n    pub fn rustdocflags_args(&self, unit: &Unit) -> &[String] {\n        &self.target_data.info(unit.kind).rustdocflags\n    }\n\n    /// Extra compiler args for either `rustc` or `rustdoc`.\n    ///\n    /// As of now, these flags come from the trailing args of either\n    /// `cargo rustc` or `cargo rustdoc`.\n    pub fn extra_args_for(&self, unit: &Unit) -> Option<&Vec<String>> {\n        self.extra_compiler_args.get(unit)\n    }\n}\n\n// Path: src/cargo/core/compiler/lto.rs\nuse crate::core::compiler::{BuildContext, CompileMode, CrateType, Unit};\nuse crate::core::profiles;\nuse crate::util::interning::InternedString;\n\nuse crate::util::errors::CargoResult;\nuse std::collections::hash_map::{Entry, HashMap};\n\n/// Possible ways to run rustc and request various parts of [LTO].\n///\n/// Variant            | Flag                   | Object Code | Bitcode\n/// -------------------|------------------------|-------------|--------\n/// `Run`              | `-C lto=foo`           | n/a         | n/a\n/// `Off`              | `-C lto=off`           | n/a         | n/a\n/// `OnlyBitcode`      | `-C linker-plugin-lto` |             | \u2713\n/// `ObjectAndBitcode` |                        | \u2713           | \u2713\n/// `OnlyObject`       | `-C embed-bitcode=no`  | \u2713           |\n///\n/// [LTO]: https://doc.rust-lang.org/nightly/cargo/reference/profiles.html#lto\n#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]\npub enum Lto {\n    /// LTO is run for this rustc, and it's `-Clto=foo`. If the given value is\n    /// None, that corresponds to `-Clto` with no argument, which means do\n    /// \"fat\" LTO.\n    Run(Option<InternedString>),\n\n    /// LTO has been explicitly listed as \"off\". This means no thin-local-LTO,\n    /// no LTO anywhere, I really mean it!\n    Off,\n\n    /// This rustc invocation only needs to produce bitcode (it is *only* used\n    /// for LTO), there's no need to produce object files, so we can pass\n    /// `-Clinker-plugin-lto`\n    OnlyBitcode,\n\n    /// This rustc invocation needs to embed bitcode in object files. This means\n    /// that object files may be used for a normal link, and the crate may be\n    /// loaded for LTO later, so both are required.\n    ObjectAndBitcode,\n\n    /// This should not include bitcode. This is primarily to reduce disk\n    /// space usage.\n    OnlyObject,\n}\n\npub fn generate(bcx: &BuildContext<'_, '_>) -> CargoResult<HashMap<Unit, Lto>> {\n    let mut map = HashMap::new();\n    for unit in bcx.roots.iter() {\n        let root_lto = match unit.profile.lto {\n            // LTO not requested, no need for bitcode.\n            profiles::Lto::Bool(false) => Lto::OnlyObject,\n            profiles::Lto::Off => Lto::Off,\n            _ => {\n                let crate_types = unit.target.rustc_crate_types();\n                if unit.target.for_host() {\n                    Lto::OnlyObject\n                } else if needs_object(&crate_types) {\n                    lto_when_needs_object(&crate_types)\n                } else {\n                    // This may or may not participate in LTO, let's start\n                    // with the minimum requirements. This may be expanded in\n                    // `calculate` below if necessary.\n                    Lto::OnlyBitcode\n                }\n            }\n        };\n        calculate(bcx, &mut map, unit, root_lto)?;\n    }\n    Ok(map)\n}\n\n/// Whether or not any of these crate types need object code.\nfn needs_object(crate_types: &[CrateType]) -> bool {\n    crate_types.iter().any(|k| k.can_lto() || k.is_dynamic())\n}\n\n/// Lto setting to use when this unit needs object code.\nfn lto_when_needs_object(crate_types: &[CrateType]) -> Lto {\n    if crate_types.iter().all(|ct| *ct == CrateType::Dylib) {\n        // A dylib whose parent is running LTO. rustc currently\n        // doesn't support LTO with dylibs, so bitcode is not\n        // needed.\n        Lto::OnlyObject\n    } else {\n        // Mixed rlib with a dylib or cdylib whose parent is running LTO. This\n        // needs both: bitcode for the rlib (for LTO) and object code for the\n        // dylib.\n        Lto::ObjectAndBitcode\n    }\n}\n\nfn calculate(\n    bcx: &BuildContext<'_, '_>,\n    map: &mut HashMap<Unit, Lto>,\n    unit: &Unit,\n    parent_lto: Lto,\n) -> CargoResult<()> {\n    let crate_types = match unit.mode {\n        // Note: Doctest ignores LTO, but for now we'll compute it as-if it is\n        // a Bin, in case it is ever supported in the future.\n        CompileMode::Test | CompileMode::Bench | CompileMode::Doctest => vec![CrateType::Bin],\n        // Notes on other modes:\n        // - Check: Treat as the underlying type, it doesn't really matter.\n        // - Doc: LTO is N/A for the Doc unit itself since rustdoc does not\n        //   support codegen flags. We still compute the dependencies, which\n        //   are mostly `Check`.\n        // - RunCustomBuild is ignored because it is always \"for_host\".\n        _ => unit.target.rustc_crate_types(),\n    };\n    // LTO can only be performed if *all* of the crate types support it.\n    // For example, a cdylib/rlib combination won't allow LTO.\n    let all_lto_types = crate_types.iter().all(CrateType::can_lto);\n    // Compute the LTO based on the profile, and what our parent requires.\n    let lto = if unit.target.for_host() {\n        // Disable LTO for host builds since we only really want to perform LTO\n        // for the final binary, and LTO on plugins/build scripts/proc macros is\n        // largely not desired.\n        Lto::OnlyObject\n    } else if all_lto_types {\n        // Note that this ignores the `parent_lto` because this isn't a\n        // linkable crate type; this unit is not being embedded in the parent.\n        match unit.profile.lto {\n            profiles::Lto::Named(s) => Lto::Run(Some(s)),\n            profiles::Lto::Off => Lto::Off,\n            profiles::Lto::Bool(true) => Lto::Run(None),\n            profiles::Lto::Bool(false) => Lto::OnlyObject,\n        }\n    } else {\n        match (parent_lto, needs_object(&crate_types)) {\n            // An rlib whose parent is running LTO, we only need bitcode.\n            (Lto::Run(_), false) => Lto::OnlyBitcode,\n            // LTO when something needs object code.\n            (Lto::Run(_), true) | (Lto::OnlyBitcode, true) => lto_when_needs_object(&crate_types),\n            // LTO is disabled, continue to disable it.\n            (Lto::Off, _) => Lto::Off,\n            // If this doesn't have any requirements, or the requirements are\n            // already satisfied, then stay with our parent.\n            (_, false) | (Lto::OnlyObject, true) | (Lto::ObjectAndBitcode, true) => parent_lto,\n        }\n    };\n\n    // Merge the computed LTO. If this unit appears multiple times in the\n    // graph, the merge may expand the requirements.\n    let merged_lto = match map.entry(unit.clone()) {\n        // If we haven't seen this unit before then insert our value and keep\n        // going.\n        Entry::Vacant(v) => *v.insert(lto),\n\n        Entry::Occupied(mut v) => {\n            let result = match (lto, v.get()) {\n                // No change in requirements.\n                (Lto::OnlyBitcode, Lto::OnlyBitcode) => Lto::OnlyBitcode,\n                (Lto::OnlyObject, Lto::OnlyObject) => Lto::OnlyObject,\n\n                // Once we're running LTO we keep running LTO. We should always\n                // calculate the same thing here each iteration because if we\n                // see this twice then it means, for example, two unit tests\n                // depend on a binary, which is normal.\n                (Lto::Run(s), _) | (_, &Lto::Run(s)) => Lto::Run(s),\n\n                // Off means off! This has the same reasoning as `Lto::Run`.\n                (Lto::Off, _) | (_, Lto::Off) => Lto::Off,\n\n                // Once a target has requested both, that's the maximal amount\n                // of work that can be done, so we just keep doing that work.\n                (Lto::ObjectAndBitcode, _) | (_, Lto::ObjectAndBitcode) => Lto::ObjectAndBitcode,\n\n                // Upgrade so that both requirements can be met.\n                //\n                // This is where the trickiness happens. This unit needs\n                // bitcode and the previously calculated value for this unit\n                // says it didn't need bitcode (or vice versa). This means that\n                // we're a shared dependency between some targets which require\n                // LTO and some which don't. This means that instead of being\n                // either only-objects or only-bitcode we have to embed both in\n                // rlibs (used for different compilations), so we switch to\n                // including both.\n                (Lto::OnlyObject, Lto::OnlyBitcode) | (Lto::OnlyBitcode, Lto::OnlyObject) => {\n                    Lto::ObjectAndBitcode\n                }\n            };\n            // No need to recurse if we calculated the same value as before.\n            if result == *v.get() {\n                return Ok(());\n            }\n            v.insert(result);\n            result\n        }\n    };\n\n    for dep in &bcx.unit_graph[unit] {\n        calculate(bcx, map, &dep.unit, merged_lto)?;\n    }\n    Ok(())\n}\n\n// Path: src/cargo/core/compiler/layout.rs\n//! Management of the directory layout of a build\n//!\n//! The directory layout is a little tricky at times, hence a separate file to\n//! house this logic. The current layout looks like this:\n//!\n//! ```text\n//! # This is the root directory for all output, the top-level package\n//! # places all of its output here.\n//! target/\n//!\n//!     # Cache of `rustc -Vv` output for performance.\n//!     .rustc-info.json\n//!\n//!     # All final artifacts are linked into this directory from `deps`.\n//!     # Note that named profiles will soon be included as separate directories\n//!     # here. They have a restricted format, similar to Rust identifiers, so\n//!     # Cargo-specific directories added in the future should use some prefix\n//!     # like `.` to avoid name collisions.\n//!     debug/  # or release/\n//!\n//!         # File used to lock the directory to prevent multiple cargo processes\n//!         # from using it at the same time.\n//!         .cargo-lock\n//!\n//!         # Hidden directory that holds all of the fingerprint files for all\n//!         # packages\n//!         .fingerprint/\n//!             # Each package is in a separate directory.\n//!             # Note that different target kinds have different filename prefixes.\n//!             $pkgname-$META/\n//!                 # Set of source filenames for this package.\n//!                 dep-lib-$targetname\n//!                 # Timestamp when this package was last built.\n//!                 invoked.timestamp\n//!                 # The fingerprint hash.\n//!                 lib-$targetname\n//!                 # Detailed information used for logging the reason why\n//!                 # something is being recompiled.\n//!                 lib-$targetname.json\n//!                 # The console output from the compiler. This is cached\n//!                 # so that warnings can be redisplayed for \"fresh\" units.\n//!                 output-lib-$targetname\n//!\n//!         # This is the root directory for all rustc artifacts except build\n//!         # scripts, examples, and test and bench executables. Almost every\n//!         # artifact should have a metadata hash added to its filename to\n//!         # prevent collisions. One notable exception is dynamic libraries.\n//!         deps/\n//!\n//!             # Each artifact dependency gets in its own directory.\n//!             /artifact/$pkgname-$META/$kind\n//!\n//!         # Root directory for all compiled examples.\n//!         examples/\n//!\n//!         # Directory used to store incremental data for the compiler (when\n//!         # incremental is enabled.\n//!         incremental/\n//!\n//!         # This is the location at which the output of all custom build\n//!         # commands are rooted.\n//!         build/\n//!\n//!             # Each package gets its own directory where its build script and\n//!             # script output are placed\n//!             $pkgname-$META/    # For the build script itself.\n//!                 # The build script executable (name may be changed by user).\n//!                 build-script-build-$META\n//!                 # Hard link to build-script-build-$META.\n//!                 build-script-build\n//!                 # Dependency information generated by rustc.\n//!                 build-script-build-$META.d\n//!                 # Debug information, depending on platform and profile\n//!                 # settings.\n//!                 <debug symbols>\n//!\n//!             # The package shows up twice with two different metadata hashes.\n//!             $pkgname-$META/  # For the output of the build script.\n//!                 # Timestamp when the build script was last executed.\n//!                 invoked.timestamp\n//!                 # Directory where script can output files ($OUT_DIR).\n//!                 out/\n//!                 # Output from the build script.\n//!                 output\n//!                 # Path to `out`, used to help when the target directory is\n//!                 # moved.\n//!                 root-output\n//!                 # Stderr output from the build script.\n//!                 stderr\n//!\n//!     # Output from rustdoc\n//!     doc/\n//!\n//!     # Used by `cargo package` and `cargo publish` to build a `.crate` file.\n//!     package/\n//!\n//!     # Experimental feature for generated build scripts.\n//!     .metabuild/\n//! ```\n//!\n//! When cross-compiling, the layout is the same, except it appears in\n//! `target/$TRIPLE`.\n\nuse crate::core::compiler::CompileTarget;\nuse crate::core::Workspace;\nuse crate::util::{CargoResult, FileLock};\nuse cargo_util::paths;\nuse std::path::{Path, PathBuf};\n\n/// Contains the paths of all target output locations.\n///\n/// See module docs for more information.\npub struct Layout {\n    /// The root directory: `/path/to/target`.\n    /// If cross compiling: `/path/to/target/$TRIPLE`.\n    root: PathBuf,\n    /// The final artifact destination: `$root/debug` (or `release`).\n    dest: PathBuf,\n    /// The directory with rustc artifacts: `$dest/deps`\n    deps: PathBuf,\n    /// The directory for build scripts: `$dest/build`\n    build: PathBuf,\n    /// The directory for artifacts, i.e. binaries, cdylibs, staticlibs: `$dest/deps/artifact`\n    artifact: PathBuf,\n    /// The directory for incremental files: `$dest/incremental`\n    incremental: PathBuf,\n    /// The directory for fingerprints: `$dest/.fingerprint`\n    fingerprint: PathBuf,\n    /// The directory for examples: `$dest/examples`\n    examples: PathBuf,\n    /// The directory for rustdoc output: `$root/doc`\n    doc: PathBuf,\n    /// The directory for temporary data of integration tests and benches: `$dest/tmp`\n    tmp: PathBuf,\n    /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this\n    /// struct is `drop`ped.\n    _lock: FileLock,\n}\n\nimpl Layout {\n    /// Calculate the paths for build output, lock the build directory, and return as a Layout.\n    ///\n    /// This function will block if the directory is already locked.\n    ///\n    /// `dest` should be the final artifact directory name. Currently either\n    /// \"debug\" or \"release\".\n    pub fn new(\n        ws: &Workspace<'_>,\n        target: Option<CompileTarget>,\n        dest: &str,\n    ) -> CargoResult<Layout> {\n        let mut root = ws.target_dir();\n        if let Some(target) = target {\n            root.push(target.short_name());\n        }\n        let dest = root.join(dest);\n        // If the root directory doesn't already exist go ahead and create it\n        // here. Use this opportunity to exclude it from backups as well if the\n        // system supports it since this is a freshly created folder.\n        //\n        paths::create_dir_all_excluded_from_backups_atomic(root.as_path_unlocked())?;\n        // Now that the excluded from backups target root is created we can create the\n        // actual destination (sub)subdirectory.\n        paths::create_dir_all(dest.as_path_unlocked())?;\n\n        // For now we don't do any more finer-grained locking on the artifact\n        // directory, so just lock the entire thing for the duration of this\n        // compile.\n        let lock = dest.open_rw_exclusive_create(\".cargo-lock\", ws.gctx(), \"build directory\")?;\n        let root = root.into_path_unlocked();\n        let dest = dest.into_path_unlocked();\n        let deps = dest.join(\"deps\");\n        let artifact = deps.join(\"artifact\");\n\n        Ok(Layout {\n            deps,\n            build: dest.join(\"build\"),\n            artifact,\n            incremental: dest.join(\"incremental\"),\n            fingerprint: dest.join(\".fingerprint\"),\n            examples: dest.join(\"examples\"),\n            doc: root.join(\"doc\"),\n            tmp: root.join(\"tmp\"),\n            root,\n            dest,\n            _lock: lock,\n        })\n    }\n\n    /// Makes sure all directories stored in the Layout exist on the filesystem.\n    pub fn prepare(&mut self) -> CargoResult<()> {\n        paths::create_dir_all(&self.deps)?;\n        paths::create_dir_all(&self.incremental)?;\n        paths::create_dir_all(&self.fingerprint)?;\n        paths::create_dir_all(&self.examples)?;\n        paths::create_dir_all(&self.build)?;\n\n        Ok(())\n    }\n\n    /// Fetch the destination path for final artifacts  (`/\u2026/target/debug`).\n    pub fn dest(&self) -> &Path {\n        &self.dest\n    }\n    /// Fetch the deps path.\n    pub fn deps(&self) -> &Path {\n        &self.deps\n    }\n    /// Fetch the examples path.\n    pub fn examples(&self) -> &Path {\n        &self.examples\n    }\n    /// Fetch the doc path.\n    pub fn doc(&self) -> &Path {\n        &self.doc\n    }\n    /// Fetch the root path (`/\u2026/target`).\n    pub fn root(&self) -> &Path {\n        &self.root\n    }\n    /// Fetch the incremental path.\n    pub fn incremental(&self) -> &Path {\n        &self.incremental\n    }\n    /// Fetch the fingerprint path.\n    pub fn fingerprint(&self) -> &Path {\n        &self.fingerprint\n    }\n    /// Fetch the build script path.\n    pub fn build(&self) -> &Path {\n        &self.build\n    }\n    /// Fetch the artifact path.\n    pub fn artifact(&self) -> &Path {\n        &self.artifact\n    }\n    /// Create and return the tmp path.\n    pub fn prepare_tmp(&self) -> CargoResult<&Path> {\n        paths::create_dir_all(&self.tmp)?;\n        Ok(&self.tmp)\n    }\n}\n\n// Path: src/cargo/core/compiler/build_runner/compilation_files.rs\n//! See [`CompilationFiles`].\n\nuse std::collections::HashMap;\nuse std::fmt;\nuse std::hash::{Hash, Hasher};\nuse std::path::{Path, PathBuf};\nuse std::sync::Arc;\n\nuse lazycell::LazyCell;\nuse tracing::debug;\n\nuse super::{BuildContext, BuildRunner, CompileKind, FileFlavor, Layout};\nuse crate::core::compiler::{CompileMode, CompileTarget, CrateType, FileType, Unit};\nuse crate::core::{Target, TargetKind, Workspace};\nuse crate::util::{self, CargoResult, StableHasher};\n\n/// This is a generic version number that can be changed to make\n/// backwards-incompatible changes to any file structures in the output\n/// directory. For example, the fingerprint files or the build-script\n/// output files.\n///\n/// Normally cargo updates ship with rustc updates which will\n/// cause a new hash due to the rustc version changing, but this allows\n/// cargo to be extra careful to deal with different versions of cargo that\n/// use the same rustc version.\nconst METADATA_VERSION: u8 = 2;\n\n/// The `Metadata` is a hash used to make unique file names for each unit in a\n/// build. It is also used for symbol mangling.\n///\n/// For example:\n/// - A project may depend on crate `A` and crate `B`, so the package name must be in the file name.\n/// - Similarly a project may depend on two versions of `A`, so the version must be in the file name.\n///\n/// In general this must include all things that need to be distinguished in different parts of\n/// the same build. This is absolutely required or we override things before\n/// we get chance to use them.\n///\n/// It is also used for symbol mangling, because if you have two versions of\n/// the same crate linked together, their symbols need to be differentiated.\n///\n/// We use a hash because it is an easy way to guarantee\n/// that all the inputs can be converted to a valid path.\n///\n/// This also acts as the main layer of caching provided by Cargo.\n/// For example, we want to cache `cargo build` and `cargo doc` separately, so that running one\n/// does not invalidate the artifacts for the other. We do this by including [`CompileMode`] in the\n/// hash, thus the artifacts go in different folders and do not override each other.\n/// If we don't add something that we should have, for this reason, we get the\n/// correct output but rebuild more than is needed.\n///\n/// Some things that need to be tracked to ensure the correct output should definitely *not*\n/// go in the `Metadata`. For example, the modification time of a file, should be tracked to make a\n/// rebuild when the file changes. However, it would be wasteful to include in the `Metadata`. The\n/// old artifacts are never going to be needed again. We can save space by just overwriting them.\n/// If we add something that we should not have, for this reason, we get the correct output but take\n/// more space than needed. This makes not including something in `Metadata`\n/// a form of cache invalidation.\n///\n/// You should also avoid anything that would interfere with reproducible\n/// builds. For example, *any* absolute path should be avoided. This is one\n/// reason that `RUSTFLAGS` is not in `Metadata`, because it often has\n/// absolute paths (like `--remap-path-prefix` which is fundamentally used for\n/// reproducible builds and has absolute paths in it). Also, in some cases the\n/// mangled symbols need to be stable between different builds with different\n/// settings. For example, profile-guided optimizations need to swap\n/// `RUSTFLAGS` between runs, but needs to keep the same symbol names.\n///\n/// Note that the `Fingerprint` is in charge of tracking everything needed to determine if a\n/// rebuild is needed.\n#[derive(Copy, Clone, Hash, Eq, PartialEq, Ord, PartialOrd)]\npub struct Metadata(u64);\n\nimpl fmt::Display for Metadata {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"{:016x}\", self.0)\n    }\n}\n\nimpl fmt::Debug for Metadata {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Metadata({:016x})\", self.0)\n    }\n}\n\n/// Information about the metadata hashes used for a `Unit`.\nstruct MetaInfo {\n    /// The symbol hash to use.\n    meta_hash: Metadata,\n    /// Whether or not the `-C extra-filename` flag is used to generate unique\n    /// output filenames for this `Unit`.\n    ///\n    /// If this is `true`, the `meta_hash` is used for the filename.\n    use_extra_filename: bool,\n}\n\n/// Collection of information about the files emitted by the compiler, and the\n/// output directory structure.\npub struct CompilationFiles<'a, 'gctx> {\n    /// The target directory layout for the host (and target if it is the same as host).\n    pub(super) host: Layout,\n    /// The target directory layout for the target (if different from then host).\n    pub(super) target: HashMap<CompileTarget, Layout>,\n    /// Additional directory to include a copy of the outputs.\n    export_dir: Option<PathBuf>,\n    /// The root targets requested by the user on the command line (does not\n    /// include dependencies).\n    roots: Vec<Unit>,\n    ws: &'a Workspace<'gctx>,\n    /// Metadata hash to use for each unit.\n    metas: HashMap<Unit, MetaInfo>,\n    /// For each Unit, a list all files produced.\n    outputs: HashMap<Unit, LazyCell<Arc<Vec<OutputFile>>>>,\n}\n\n/// Info about a single file emitted by the compiler.\n#[derive(Debug)]\npub struct OutputFile {\n    /// Absolute path to the file that will be produced by the build process.\n    pub path: PathBuf,\n    /// If it should be linked into `target`, and what it should be called\n    /// (e.g., without metadata).\n    pub hardlink: Option<PathBuf>,\n    /// If `--out-dir` is specified, the absolute path to the exported file.\n    pub export_path: Option<PathBuf>,\n    /// Type of the file (library / debug symbol / else).\n    pub flavor: FileFlavor,\n}\n\nimpl OutputFile {\n    /// Gets the hard link if present; otherwise, returns the path.\n    pub fn bin_dst(&self) -> &PathBuf {\n        match self.hardlink {\n            Some(ref link_dst) => link_dst,\n            None => &self.path,\n        }\n    }\n}\n\nimpl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {\n    pub(super) fn new(\n        build_runner: &BuildRunner<'a, 'gctx>,\n        host: Layout,\n        target: HashMap<CompileTarget, Layout>,\n    ) -> CompilationFiles<'a, 'gctx> {\n        let mut metas = HashMap::new();\n        for unit in &build_runner.bcx.roots {\n            metadata_of(unit, build_runner, &mut metas);\n        }\n        let outputs = metas\n            .keys()\n            .cloned()\n            .map(|unit| (unit, LazyCell::new()))\n            .collect();\n        CompilationFiles {\n            ws: build_runner.bcx.ws,\n            host,\n            target,\n            export_dir: build_runner.bcx.build_config.export_dir.clone(),\n            roots: build_runner.bcx.roots.clone(),\n            metas,\n            outputs,\n        }\n    }\n\n    /// Returns the appropriate directory layout for either a plugin or not.\n    pub fn layout(&self, kind: CompileKind) -> &Layout {\n        match kind {\n            CompileKind::Host => &self.host,\n            CompileKind::Target(target) => &self.target[&target],\n        }\n    }\n\n    /// Gets the metadata for the given unit.\n    ///\n    /// See [`Metadata`] and [`fingerprint`] module for more.\n    ///\n    /// [`fingerprint`]: ../../fingerprint/index.html#fingerprints-and-metadata\n    pub fn metadata(&self, unit: &Unit) -> Metadata {\n        self.metas[unit].meta_hash\n    }\n\n    /// Returns whether or not `-C extra-filename` is used to extend the\n    /// output filenames to make them unique.\n    pub fn use_extra_filename(&self, unit: &Unit) -> bool {\n        self.metas[unit].use_extra_filename\n    }\n\n    /// Gets the short hash based only on the `PackageId`.\n    /// Used for the metadata when `metadata` returns `None`.\n    pub fn target_short_hash(&self, unit: &Unit) -> String {\n        let hashable = unit.pkg.package_id().stable_hash(self.ws.root());\n        util::short_hash(&(METADATA_VERSION, hashable))\n    }\n\n    /// Returns the directory where the artifacts for the given unit are\n    /// initially created.\n    pub fn out_dir(&self, unit: &Unit) -> PathBuf {\n        // Docscrape units need to have doc/ set as the out_dir so sources for reverse-dependencies\n        // will be put into doc/ and not into deps/ where the *.examples files are stored.\n        if unit.mode.is_doc() || unit.mode.is_doc_scrape() {\n            self.layout(unit.kind).doc().to_path_buf()\n        } else if unit.mode.is_doc_test() {\n            panic!(\"doc tests do not have an out dir\");\n        } else if unit.target.is_custom_build() {\n            self.build_script_dir(unit)\n        } else if unit.target.is_example() {\n            self.layout(unit.kind).examples().to_path_buf()\n        } else if unit.artifact.is_true() {\n            self.artifact_dir(unit)\n        } else {\n            self.deps_dir(unit).to_path_buf()\n        }\n    }\n\n    /// Additional export directory from `--out-dir`.\n    pub fn export_dir(&self) -> Option<PathBuf> {\n        self.export_dir.clone()\n    }\n\n    /// Directory name to use for a package in the form `NAME-HASH`.\n    ///\n    /// Note that some units may share the same directory, so care should be\n    /// taken in those cases!\n    fn pkg_dir(&self, unit: &Unit) -> String {\n        let name = unit.pkg.package_id().name();\n        let meta = &self.metas[unit];\n        if meta.use_extra_filename {\n            format!(\"{}-{}\", name, meta.meta_hash)\n        } else {\n            format!(\"{}-{}\", name, self.target_short_hash(unit))\n        }\n    }\n\n    /// Returns the final artifact path for the host (`/\u2026/target/debug`)\n    pub fn host_dest(&self) -> &Path {\n        self.host.dest()\n    }\n\n    /// Returns the root of the build output tree for the host (`/\u2026/target`)\n    pub fn host_root(&self) -> &Path {\n        self.host.root()\n    }\n\n    /// Returns the host `deps` directory path.\n    pub fn host_deps(&self) -> &Path {\n        self.host.deps()\n    }\n\n    /// Returns the directories where Rust crate dependencies are found for the\n    /// specified unit.\n    pub fn deps_dir(&self, unit: &Unit) -> &Path {\n        self.layout(unit.kind).deps()\n    }\n\n    /// Directory where the fingerprint for the given unit should go.\n    pub fn fingerprint_dir(&self, unit: &Unit) -> PathBuf {\n        let dir = self.pkg_dir(unit);\n        self.layout(unit.kind).fingerprint().join(dir)\n    }\n\n    /// Returns the path for a file in the fingerprint directory.\n    ///\n    /// The \"prefix\" should be something to distinguish the file from other\n    /// files in the fingerprint directory.\n    pub fn fingerprint_file_path(&self, unit: &Unit, prefix: &str) -> PathBuf {\n        // Different targets need to be distinguished in the\n        let kind = unit.target.kind().description();\n        let flavor = if unit.mode.is_any_test() {\n            \"test-\"\n        } else if unit.mode.is_doc() {\n            \"doc-\"\n        } else if unit.mode.is_run_custom_build() {\n            \"run-\"\n        } else {\n            \"\"\n        };\n        let name = format!(\"{}{}{}-{}\", prefix, flavor, kind, unit.target.name());\n        self.fingerprint_dir(unit).join(name)\n    }\n\n    /// Path where compiler output is cached.\n    pub fn message_cache_path(&self, unit: &Unit) -> PathBuf {\n        self.fingerprint_file_path(unit, \"output-\")\n    }\n\n    /// Returns the directory where a compiled build script is stored.\n    /// `/path/to/target/{debug,release}/build/PKG-HASH`\n    pub fn build_script_dir(&self, unit: &Unit) -> PathBuf {\n        assert!(unit.target.is_custom_build());\n        assert!(!unit.mode.is_run_custom_build());\n        assert!(self.metas.contains_key(unit));\n        let dir = self.pkg_dir(unit);\n        self.layout(CompileKind::Host).build().join(dir)\n    }\n\n    /// Returns the directory for compiled artifacts files.\n    /// `/path/to/target/{debug,release}/deps/artifact/KIND/PKG-HASH`\n    fn artifact_dir(&self, unit: &Unit) -> PathBuf {\n        assert!(self.metas.contains_key(unit));\n        assert!(unit.artifact.is_true());\n        let dir = self.pkg_dir(unit);\n        let kind = match unit.target.kind() {\n            TargetKind::Bin => \"bin\",\n            TargetKind::Lib(lib_kinds) => match lib_kinds.as_slice() {\n                &[CrateType::Cdylib] => \"cdylib\",\n                &[CrateType::Staticlib] => \"staticlib\",\n                invalid => unreachable!(\n                    \"BUG: unexpected artifact library type(s): {:?} - these should have been split\",\n                    invalid\n                ),\n            },\n            invalid => unreachable!(\n                \"BUG: {:?} are not supposed to be used as artifacts\",\n                invalid\n            ),\n        };\n        self.layout(unit.kind).artifact().join(dir).join(kind)\n    }\n\n    /// Returns the directory where information about running a build script\n    /// is stored.\n    /// `/path/to/target/{debug,release}/build/PKG-HASH`\n    pub fn build_script_run_dir(&self, unit: &Unit) -> PathBuf {\n        assert!(unit.target.is_custom_build());\n        assert!(unit.mode.is_run_custom_build());\n        let dir = self.pkg_dir(unit);\n        self.layout(unit.kind).build().join(dir)\n    }\n\n    /// Returns the \"OUT_DIR\" directory for running a build script.\n    /// `/path/to/target/{debug,release}/build/PKG-HASH/out`\n    pub fn build_script_out_dir(&self, unit: &Unit) -> PathBuf {\n        self.build_script_run_dir(unit).join(\"out\")\n    }\n\n    /// Returns the path to the executable binary for the given bin target.\n    ///\n    /// This should only to be used when a `Unit` is not available.\n    pub fn bin_link_for_target(\n        &self,\n        target: &Target,\n        kind: CompileKind,\n        bcx: &BuildContext<'_, '_>,\n    ) -> CargoResult<PathBuf> {\n        assert!(target.is_bin());\n        let dest = self.layout(kind).dest();\n        let info = bcx.target_data.info(kind);\n        let (file_types, _) = info\n            .rustc_outputs(\n                CompileMode::Build,\n                &TargetKind::Bin,\n                bcx.target_data.short_name(&kind),\n            )\n            .expect(\"target must support `bin`\");\n\n        let file_type = file_types\n            .iter()\n            .find(|file_type| file_type.flavor == FileFlavor::Normal)\n            .expect(\"target must support `bin`\");\n\n        Ok(dest.join(file_type.uplift_filename(target)))\n    }\n\n    /// Returns the filenames that the given unit will generate.\n    ///\n    /// Note: It is not guaranteed that all of the files will be generated.\n    pub(super) fn outputs(\n        &self,\n        unit: &Unit,\n        bcx: &BuildContext<'a, 'gctx>,\n    ) -> CargoResult<Arc<Vec<OutputFile>>> {\n        self.outputs[unit]\n            .try_borrow_with(|| self.calc_outputs(unit, bcx))\n            .map(Arc::clone)\n    }\n\n    /// Returns the path where the output for the given unit and FileType\n    /// should be uplifted to.\n    ///\n    /// Returns `None` if the unit shouldn't be uplifted (for example, a\n    /// dependent rlib).\n    fn uplift_to(&self, unit: &Unit, file_type: &FileType, from_path: &Path) -> Option<PathBuf> {\n        // Tests, check, doc, etc. should not be uplifted.\n        if unit.mode != CompileMode::Build || file_type.flavor == FileFlavor::Rmeta {\n            return None;\n        }\n\n        // Artifact dependencies are never uplifted.\n        if unit.artifact.is_true() {\n            return None;\n        }\n\n        // - Binaries: The user always wants to see these, even if they are\n        //   implicitly built (for example for integration tests).\n        // - dylibs: This ensures that the dynamic linker pulls in all the\n        //   latest copies (even if the dylib was built from a previous cargo\n        //   build). There are complex reasons for this, see #8139, #6167, #6162.\n        // - Things directly requested from the command-line (the \"roots\").\n        //   This one is a little questionable for rlibs (see #6131), but is\n        //   historically how Cargo has operated. This is primarily useful to\n        //   give the user access to staticlibs and cdylibs.\n        if !unit.target.is_bin()\n            && !unit.target.is_custom_build()\n            && file_type.crate_type != Some(CrateType::Dylib)\n            && !self.roots.contains(unit)\n        {\n            return None;\n        }\n\n        let filename = file_type.uplift_filename(&unit.target);\n        let uplift_path = if unit.target.is_example() {\n            // Examples live in their own little world.\n            self.layout(unit.kind).examples().join(filename)\n        } else if unit.target.is_custom_build() {\n            self.build_script_dir(unit).join(filename)\n        } else {\n            self.layout(unit.kind).dest().join(filename)\n        };\n        if from_path == uplift_path {\n            // This can happen with things like examples that reside in the\n            // same directory, do not have a metadata hash (like on Windows),\n            // and do not have hyphens.\n            return None;\n        }\n        Some(uplift_path)\n    }\n\n    /// Calculates the filenames that the given unit will generate.\n    /// Should use [`CompilationFiles::outputs`] instead\n    /// as it caches the result of this function.\n    fn calc_outputs(\n        &self,\n        unit: &Unit,\n        bcx: &BuildContext<'a, 'gctx>,\n    ) -> CargoResult<Arc<Vec<OutputFile>>> {\n        let ret = match unit.mode {\n            CompileMode::Doc { json, .. } => {\n                let path = if json {\n                    self.out_dir(unit)\n                        .join(format!(\"{}.json\", unit.target.crate_name()))\n                } else {\n                    self.out_dir(unit)\n                        .join(unit.target.crate_name())\n                        .join(\"index.html\")\n                };\n\n                vec![OutputFile {\n                    path,\n                    hardlink: None,\n                    export_path: None,\n                    flavor: FileFlavor::Normal,\n                }]\n            }\n            CompileMode::RunCustomBuild => {\n                // At this time, this code path does not handle build script\n                // outputs.\n                vec![]\n            }\n            CompileMode::Doctest => {\n                // Doctests are built in a temporary directory and then\n                // deleted. There is the `--persist-doctests` unstable flag,\n                // but Cargo does not know about that.\n                vec![]\n            }\n            CompileMode::Docscrape => {\n                // The file name needs to be stable across Cargo sessions.\n                // This originally used unit.buildkey(), but that isn't stable,\n                // so we use metadata instead (prefixed with name for debugging).\n                let file_name = format!(\"{}-{}.examples\", unit.pkg.name(), self.metadata(unit));\n                let path = self.deps_dir(unit).join(file_name);\n                vec![OutputFile {\n                    path,\n                    hardlink: None,\n                    export_path: None,\n                    flavor: FileFlavor::Normal,\n                }]\n            }\n            CompileMode::Test\n            | CompileMode::Build\n            | CompileMode::Bench\n            | CompileMode::Check { .. } => self.calc_outputs_rustc(unit, bcx)?,\n        };\n        debug!(\"Target filenames: {:?}\", ret);\n\n        Ok(Arc::new(ret))\n    }\n\n    /// Computes the actual, full pathnames for all the files generated by rustc.\n    ///\n    /// The `OutputFile` also contains the paths where those files should be\n    /// \"uplifted\" to.\n    fn calc_outputs_rustc(\n        &self,\n        unit: &Unit,\n        bcx: &BuildContext<'a, 'gctx>,\n    ) -> CargoResult<Vec<OutputFile>> {\n        let out_dir = self.out_dir(unit);\n\n        let info = bcx.target_data.info(unit.kind);\n        let triple = bcx.target_data.short_name(&unit.kind);\n        let (file_types, unsupported) =\n            info.rustc_outputs(unit.mode, unit.target.kind(), triple)?;\n        if file_types.is_empty() {\n            if !unsupported.is_empty() {\n                let unsupported_strs: Vec<_> = unsupported.iter().map(|ct| ct.as_str()).collect();\n                anyhow::bail!(\n                    \"cannot produce {} for `{}` as the target `{}` \\\n                     does not support these crate types\",\n                    unsupported_strs.join(\", \"),\n                    unit.pkg,\n                    triple,\n                )\n            }\n            anyhow::bail!(\n                \"cannot compile `{}` as the target `{}` does not \\\n                 support any of the output crate types\",\n                unit.pkg,\n                triple,\n            );\n        }\n\n        // Convert FileType to OutputFile.\n        let mut outputs = Vec::new();\n        for file_type in file_types {\n            let meta = &self.metas[unit];\n            let meta_opt = meta.use_extra_filename.then(|| meta.meta_hash.to_string());\n            let path = out_dir.join(file_type.output_filename(&unit.target, meta_opt.as_deref()));\n\n            // If, the `different_binary_name` feature is enabled, the name of the hardlink will\n            // be the name of the binary provided by the user in `Cargo.toml`.\n            let hardlink = self.uplift_to(unit, &file_type, &path);\n            let export_path = if unit.target.is_custom_build() {\n                None\n            } else {\n                self.export_dir.as_ref().and_then(|export_dir| {\n                    hardlink\n                        .as_ref()\n                        .map(|hardlink| export_dir.join(hardlink.file_name().unwrap()))\n                })\n            };\n            outputs.push(OutputFile {\n                path,\n                hardlink,\n                export_path,\n                flavor: file_type.flavor,\n            });\n        }\n        Ok(outputs)\n    }\n}\n\n/// Gets the metadata hash for the given [`Unit`].\n///\n/// When a metadata hash doesn't exist for the given unit,\n/// this calls itself recursively to compute metadata hashes of all its dependencies.\n/// See [`compute_metadata`] for how a single metadata hash is computed.\nfn metadata_of<'a>(\n    unit: &Unit,\n    build_runner: &BuildRunner<'_, '_>,\n    metas: &'a mut HashMap<Unit, MetaInfo>,\n) -> &'a MetaInfo {\n    if !metas.contains_key(unit) {\n        let meta = compute_metadata(unit, build_runner, metas);\n        metas.insert(unit.clone(), meta);\n        for dep in build_runner.unit_deps(unit) {\n            metadata_of(&dep.unit, build_runner, metas);\n        }\n    }\n    &metas[unit]\n}\n\n/// Computes the metadata hash for the given [`Unit`].\nfn compute_metadata(\n    unit: &Unit,\n    build_runner: &BuildRunner<'_, '_>,\n    metas: &mut HashMap<Unit, MetaInfo>,\n) -> MetaInfo {\n    let bcx = &build_runner.bcx;\n    let mut hasher = StableHasher::new();\n\n    METADATA_VERSION.hash(&mut hasher);\n\n    // Unique metadata per (name, source, version) triple. This'll allow us\n    // to pull crates from anywhere without worrying about conflicts.\n    unit.pkg\n        .package_id()\n        .stable_hash(bcx.ws.root())\n        .hash(&mut hasher);\n\n    // Also mix in enabled features to our metadata. This'll ensure that\n    // when changing feature sets each lib is separately cached.\n    unit.features.hash(&mut hasher);\n\n    // Mix in the target-metadata of all the dependencies of this target.\n    let mut deps_metadata = build_runner\n        .unit_deps(unit)\n        .iter()\n        .map(|dep| metadata_of(&dep.unit, build_runner, metas).meta_hash)\n        .collect::<Vec<_>>();\n    deps_metadata.sort();\n    deps_metadata.hash(&mut hasher);\n\n    // Throw in the profile we're compiling with. This helps caching\n    // `panic=abort` and `panic=unwind` artifacts, additionally with various\n    // settings like debuginfo and whatnot.\n    unit.profile.hash(&mut hasher);\n    unit.mode.hash(&mut hasher);\n    build_runner.lto[unit].hash(&mut hasher);\n\n    // Artifacts compiled for the host should have a different\n    // metadata piece than those compiled for the target, so make sure\n    // we throw in the unit's `kind` as well.  Use `fingerprint_hash`\n    // so that the StableHash doesn't change based on the pathnames\n    // of the custom target JSON spec files.\n    unit.kind.fingerprint_hash().hash(&mut hasher);\n\n    // Finally throw in the target name/kind. This ensures that concurrent\n    // compiles of targets in the same crate don't collide.\n    unit.target.name().hash(&mut hasher);\n    unit.target.kind().hash(&mut hasher);\n\n    hash_rustc_version(bcx, &mut hasher);\n\n    if build_runner.bcx.ws.is_member(&unit.pkg) {\n        // This is primarily here for clippy. This ensures that the clippy\n        // artifacts are separate from the `check` ones.\n        if let Some(path) = &build_runner.bcx.rustc().workspace_wrapper {\n            path.hash(&mut hasher);\n        }\n    }\n\n    // Seed the contents of `__CARGO_DEFAULT_LIB_METADATA` to the hasher if present.\n    // This should be the release channel, to get a different hash for each channel.\n    if let Ok(ref channel) = build_runner\n        .bcx\n        .gctx\n        .get_env(\"__CARGO_DEFAULT_LIB_METADATA\")\n    {\n        channel.hash(&mut hasher);\n    }\n\n    // std units need to be kept separate from user dependencies. std crates\n    // are differentiated in the Unit with `is_std` (for things like\n    // `-Zforce-unstable-if-unmarked`), so they are always built separately.\n    // This isn't strictly necessary for build dependencies which probably\n    // don't need unstable support. A future experiment might be to set\n    // `is_std` to false for build dependencies so that they can be shared\n    // with user dependencies.\n    unit.is_std.hash(&mut hasher);\n\n    MetaInfo {\n        meta_hash: Metadata(hasher.finish()),\n        use_extra_filename: should_use_metadata(bcx, unit),\n    }\n}\n\n...\n", "needle_token_start": 701, "needle_token_end": 926, "code_context_ntokens": 16255, "output": ["```rust\n/// Helper for fetching a profile from config.\nfn get_config_profile(ws: &Workspace<'_>, name: &str) -> CargoResult<Option<TomlProfile>> {\n    let profile: Option<context::Value<TomlProfile>> =\n        ws.gctx().get(&format!(\"profile.{}\", name))?;\n    let Some(profile) = profile else {\n        return Ok(None);\n    };\n    let mut warnings = Vec::new();\n    validate_profile(\n        &profile.val,\n        name,\n        ws.gctx().cli_unstable(),\n        ws.unstable_features(),\n        &mut warnings,\n    )\n    .with_context(|| {\n        format!(\n            \"config profile `{}` is not valid (defined in `{}`)\",\n            name, profile.definition\n        )\n    })?;\n    for warning in warnings {\n        ws.gctx().shell().warn(warning)?;\n    }\n    Ok(Some(profile.val))\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "package", "language": "rust", "path": "src/cargo/ops/cargo_package.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to prepare and validate a software package from a project workspace, ensuring it meets specified conditions and configurations before deployment or distribution.\n2. **Input**: It takes a reference to a workspace, which includes all necessary project data and settings, and a set of packaging options that dictate how the packaging should be handled, such as whether to allow dirty states in version control systems, whether to list files instead of packaging, and various other operational flags.\n3. **Output**: The function returns either a list of file locks for the packaged files or none, depending on whether the operation was to list files or to actually package them. Each file lock represents a packaged file that has been successfully prepared and verified according to the specified options.\n4. **Procedure**: The function iterates over each package in the workspace, updating sources, checking metadata, and validating the state of the version control system. It then compiles a list of files to be included in the package, optionally lists these files, and checks package dependencies. If not just listing, it proceeds to create a tarball for each package, verifies the tarball if required, and then moves the tarball to a final location, logging the outcome of the packaging process.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/cargo/ops/lockfile.rs\nuse std::io::prelude::*;\n\nuse crate::core::{resolver, Resolve, ResolveVersion, Workspace};\nuse crate::util::errors::CargoResult;\nuse crate::util::Filesystem;\n\nuse anyhow::Context as _;\n\n#[tracing::instrument(skip_all)]\npub fn load_pkg_lockfile(ws: &Workspace<'_>) -> CargoResult<Option<Resolve>> {\n    let lock_root = lock_root(ws);\n    if !lock_root.as_path_unlocked().join(\"Cargo.lock\").exists() {\n        return Ok(None);\n    }\n\n    let mut f = lock_root.open_ro_shared(\"Cargo.lock\", ws.gctx(), \"Cargo.lock file\")?;\n\n    let mut s = String::new();\n    f.read_to_string(&mut s)\n        .with_context(|| format!(\"failed to read file: {}\", f.path().display()))?;\n\n    let resolve = (|| -> CargoResult<Option<Resolve>> {\n        let v: resolver::EncodableResolve = toml::from_str(&s)?;\n        Ok(Some(v.into_resolve(&s, ws)?))\n    })()\n...\n// Path: src/cargo/ops/cargo_package.rs\nuse std::collections::{BTreeSet, HashMap};\nuse std::fs::{self, File};\nuse std::io::prelude::*;\nuse std::io::SeekFrom;\nuse std::path::{Path, PathBuf};\nuse std::sync::Arc;\nuse std::task::Poll;\n\nuse crate::core::compiler::{BuildConfig, CompileMode, DefaultExecutor, Executor};\nuse crate::core::manifest::Target;\nuse crate::core::resolver::CliFeatures;\nuse crate::core::{registry::PackageRegistry, resolver::HasDevUnits};\nuse crate::core::{Feature, Shell, Verbosity, Workspace};\nuse crate::core::{Package, PackageId, PackageSet, Resolve, SourceId};\nuse crate::sources::PathSource;\nuse crate::util::cache_lock::CacheLockMode;\nuse crate::util::context::JobsConfig;\nuse crate::util::errors::CargoResult;\nuse crate::util::toml::prepare_for_publish;\nuse crate::util::{self, human_readable_bytes, restricted_names, FileLock, GlobalContext};\nuse crate::{drop_println, ops};\nuse anyhow::Context as _;\nuse cargo_util::paths;\nuse flate2::read::GzDecoder;\nuse flate2::{Compression, GzBuilder};\nuse serde::Serialize;\nuse tar::{Archive, Builder, EntryType, Header, HeaderMode};\nuse tracing::debug;\nuse unicase::Ascii as UncasedAscii;\n\npub struct PackageOpts<'gctx> {\n    pub gctx: &'gctx GlobalContext,\n    pub list: bool,\n    pub check_metadata: bool,\n    pub allow_dirty: bool,\n    pub verify: bool,\n    pub jobs: Option<JobsConfig>,\n    pub keep_going: bool,\n    pub to_package: ops::Packages,\n    pub targets: Vec<String>,\n    pub cli_features: CliFeatures,\n}\n\nconst ORIGINAL_MANIFEST_FILE: &str = \"Cargo.toml.orig\";\nconst VCS_INFO_FILE: &str = \".cargo_vcs_info.json\";\n\nstruct ArchiveFile {\n    /// The relative path in the archive (not including the top-level package\n    /// name directory).\n    rel_path: PathBuf,\n    /// String variant of `rel_path`, for convenience.\n    rel_str: String,\n    /// The contents to add to the archive.\n    contents: FileContents,\n}\n\nenum FileContents {\n    /// Absolute path to the file on disk to add to the archive.\n    OnDisk(PathBuf),\n    /// Generates a file.\n    Generated(GeneratedFile),\n}\n\nenum GeneratedFile {\n    /// Generates `Cargo.toml` by rewriting the original.\n    Manifest,\n    /// Generates `Cargo.lock` in some cases (like if there is a binary).\n    Lockfile,\n    /// Adds a `.cargo_vcs_info.json` file if in a (clean) git repo.\n    VcsInfo(VcsInfo),\n}\n\n#[derive(Serialize)]\nstruct VcsInfo {\n    git: GitVcsInfo,\n    /// Path to the package within repo (empty string if root). / not \\\n    path_in_vcs: String,\n}\n\n#[derive(Serialize)]\nstruct GitVcsInfo {\n    sha1: String,\n}\n\npub fn package_one(\n    ws: &Workspace<'_>,\n    pkg: &Package,\n    opts: &PackageOpts<'_>,\n) -> CargoResult<Option<FileLock>> {\n    let gctx = ws.gctx();\n    let mut src = PathSource::new(pkg.root(), pkg.package_id().source_id(), gctx);\n    src.update()?;\n\n    if opts.check_metadata {\n        check_metadata(pkg, gctx)?;\n    }\n\n    if !pkg.manifest().exclude().is_empty() && !pkg.manifest().include().is_empty() {\n        gctx.shell().warn(\n            \"both package.include and package.exclude are specified; \\\n             the exclude list will be ignored\",\n        )?;\n    }\n    let src_files = src.list_files(pkg)?;\n\n    // Check (git) repository state, getting the current commit hash if not\n    // dirty.\n    let vcs_info = if !opts.allow_dirty {\n        // This will error if a dirty repo is found.\n        check_repo_state(pkg, &src_files, gctx)?\n    } else {\n        None\n    };\n\n    let ar_files = build_ar_list(ws, pkg, src_files, vcs_info)?;\n\n    let filecount = ar_files.len();\n\n    if opts.list {\n        for ar_file in ar_files {\n            drop_println!(gctx, \"{}\", ar_file.rel_str);\n        }\n\n        return Ok(None);\n    }\n\n    // Check that the package dependencies are safe to deploy.\n    for dep in pkg.dependencies() {\n        super::check_dep_has_version(dep, false)?;\n    }\n\n    let filename = pkg.package_id().tarball_name();\n    let dir = ws.target_dir().join(\"package\");\n    let mut dst = {\n        let tmp = format!(\".{}\", filename);\n        dir.open_rw_exclusive_create(&tmp, gctx, \"package scratch space\")?\n    };\n\n    // Package up and test a temporary tarball and only move it to the final\n    // location if it actually passes all our tests. Any previously existing\n    // tarball can be assumed as corrupt or invalid, so we just blow it away if\n    // it exists.\n    gctx.shell()\n        .status(\"Packaging\", pkg.package_id().to_string())?;\n    dst.file().set_len(0)?;\n    let uncompressed_size = tar(ws, pkg, ar_files, dst.file(), &filename)\n        .with_context(|| \"failed to prepare local package for uploading\")?;\n    if opts.verify {\n        dst.seek(SeekFrom::Start(0))?;\n        run_verify(ws, pkg, &dst, opts).with_context(|| \"failed to verify package tarball\")?\n    }\n\n    dst.seek(SeekFrom::Start(0))?;\n    let src_path = dst.path();\n    let dst_path = dst.parent().join(&filename);\n    fs::rename(&src_path, &dst_path)\n        .with_context(|| \"failed to move temporary tarball into final location\")?;\n\n    let dst_metadata = dst\n        .file()\n        .metadata()\n        .with_context(|| format!(\"could not learn metadata for: `{}`\", dst_path.display()))?;\n    let compressed_size = dst_metadata.len();\n\n    let uncompressed = human_readable_bytes(uncompressed_size);\n    let compressed = human_readable_bytes(compressed_size);\n\n    let message = format!(\n        \"{} files, {:.1}{} ({:.1}{} compressed)\",\n        filecount, uncompressed.0, uncompressed.1, compressed.0, compressed.1,\n    );\n    // It doesn't really matter if this fails.\n    drop(gctx.shell().status(\"Packaged\", message));\n\n    return Ok(Some(dst));\n}\n\n\npub fn package(ws: &Workspace<'_>, opts: &PackageOpts<'_>) -> CargoResult<Option<Vec<FileLock>>> {\n    let pkgs = ws.members_with_features(\n        &opts.to_package.to_package_id_specs(ws)?,\n        &opts.cli_features,\n    )?;\n\n    let mut dsts = Vec::with_capacity(pkgs.len());\n\n    if ws.root().join(\"Cargo.lock\").exists() {\n        // Make sure the Cargo.lock is up-to-date and valid.\n        let _ = ops::resolve_ws(ws)?;\n        // If Cargo.lock does not exist, it will be generated by `build_lock`\n        // below, and will be validated during the verification step.\n    }\n\n    for (pkg, cli_features) in pkgs {\n        let result = package_one(\n            ws,\n            pkg,\n            &PackageOpts {\n                gctx: opts.gctx,\n                list: opts.list,\n                check_metadata: opts.check_metadata,\n                allow_dirty: opts.allow_dirty,\n                verify: opts.verify,\n                jobs: opts.jobs.clone(),\n                keep_going: opts.keep_going,\n                to_package: ops::Packages::Default,\n                targets: opts.targets.clone(),\n                cli_features: cli_features,\n            },\n        )?;\n\n        if !opts.list {\n            dsts.push(result.unwrap());\n        }\n    }\n\n    if opts.list {\n        // We're just listing, so there's no file output\n        Ok(None)\n    } else {\n        Ok(Some(dsts))\n    }\n}\n\n/// Builds list of files to archive.\nfn build_ar_list(\n    ws: &Workspace<'_>,\n    pkg: &Package,\n    src_files: Vec<PathBuf>,\n    vcs_info: Option<VcsInfo>,\n) -> CargoResult<Vec<ArchiveFile>> {\n    let mut result = HashMap::new();\n    let root = pkg.root();\n\n    for src_file in &src_files {\n        let rel_path = src_file.strip_prefix(&root)?;\n        check_filename(rel_path, &mut ws.gctx().shell())?;\n        let rel_str = rel_path.to_str().ok_or_else(|| {\n            anyhow::format_err!(\"non-utf8 path in source directory: {}\", rel_path.display())\n        })?;\n        match rel_str {\n            \"Cargo.lock\" => continue,\n            VCS_INFO_FILE | ORIGINAL_MANIFEST_FILE => anyhow::bail!(\n                \"invalid inclusion of reserved file name {} in package source\",\n                rel_str\n            ),\n            _ => {\n                result\n                    .entry(UncasedAscii::new(rel_str))\n                    .or_insert_with(Vec::new)\n                    .push(ArchiveFile {\n                        rel_path: rel_path.to_owned(),\n                        rel_str: rel_str.to_owned(),\n                        contents: FileContents::OnDisk(src_file.clone()),\n                    });\n            }\n        }\n    }\n\n    // Ensure we normalize for case insensitive filesystems (like on Windows) by removing the\n    // existing entry, regardless of case, and adding in with the correct case\n    if result.remove(&UncasedAscii::new(\"Cargo.toml\")).is_some() {\n        result\n            .entry(UncasedAscii::new(ORIGINAL_MANIFEST_FILE))\n            .or_insert_with(Vec::new)\n            .push(ArchiveFile {\n                rel_path: PathBuf::from(ORIGINAL_MANIFEST_FILE),\n                rel_str: ORIGINAL_MANIFEST_FILE.to_string(),\n                contents: FileContents::OnDisk(pkg.manifest_path().to_owned()),\n            });\n        result\n            .entry(UncasedAscii::new(\"Cargo.toml\"))\n            .or_insert_with(Vec::new)\n            .push(ArchiveFile {\n                rel_path: PathBuf::from(\"Cargo.toml\"),\n                rel_str: \"Cargo.toml\".to_string(),\n                contents: FileContents::Generated(GeneratedFile::Manifest),\n            });\n    } else {\n        ws.gctx().shell().warn(&format!(\n            \"no `Cargo.toml` file found when packaging `{}` (note the case of the file name).\",\n            pkg.name()\n        ))?;\n    }\n\n    if pkg.include_lockfile() {\n        let rel_str = \"Cargo.lock\";\n        result\n            .entry(UncasedAscii::new(rel_str))\n            .or_insert_with(Vec::new)\n            .push(ArchiveFile {\n                rel_path: PathBuf::from(rel_str),\n                rel_str: rel_str.to_string(),\n                contents: FileContents::Generated(GeneratedFile::Lockfile),\n            });\n    }\n    if let Some(vcs_info) = vcs_info {\n        let rel_str = VCS_INFO_FILE;\n        result\n            .entry(UncasedAscii::new(rel_str))\n            .or_insert_with(Vec::new)\n            .push(ArchiveFile {\n                rel_path: PathBuf::from(rel_str),\n                rel_str: rel_str.to_string(),\n                contents: FileContents::Generated(GeneratedFile::VcsInfo(vcs_info)),\n            });\n    }\n\n    let mut result = result.into_values().flatten().collect();\n    if let Some(license_file) = &pkg.manifest().metadata().license_file {\n        let license_path = Path::new(license_file);\n        let abs_file_path = paths::normalize_path(&pkg.root().join(license_path));\n        if abs_file_path.is_file() {\n            check_for_file_and_add(\n                \"license-file\",\n                license_path,\n                abs_file_path,\n                pkg,\n                &mut result,\n                ws,\n            )?;\n        } else {\n            warn_on_nonexistent_file(&pkg, &license_path, \"license-file\", &ws)?;\n        }\n    }\n    if let Some(readme) = &pkg.manifest().metadata().readme {\n        let readme_path = Path::new(readme);\n        let abs_file_path = paths::normalize_path(&pkg.root().join(readme_path));\n        if abs_file_path.is_file() {\n            check_for_file_and_add(\"readme\", readme_path, abs_file_path, pkg, &mut result, ws)?;\n        } else {\n            warn_on_nonexistent_file(&pkg, &readme_path, \"readme\", &ws)?;\n        }\n    }\n\n    for t in pkg\n        .manifest()\n        .targets()\n        .iter()\n        .filter(|t| t.is_custom_build())\n    {\n        if let Some(custome_build_path) = t.src_path().path() {\n            let abs_custome_build_path =\n                paths::normalize_path(&pkg.root().join(custome_build_path));\n            if !abs_custome_build_path.is_file() || !abs_custome_build_path.starts_with(pkg.root())\n            {\n                error_custom_build_file_not_in_package(pkg, &abs_custome_build_path, t)?;\n            }\n        }\n    }\n\n    result.sort_unstable_by(|a, b| a.rel_path.cmp(&b.rel_path));\n\n    Ok(result)\n}\n\nfn check_for_file_and_add(\n    label: &str,\n    file_path: &Path,\n    abs_file_path: PathBuf,\n    pkg: &Package,\n    result: &mut Vec<ArchiveFile>,\n    ws: &Workspace<'_>,\n) -> CargoResult<()> {\n    match abs_file_path.strip_prefix(&pkg.root()) {\n        Ok(rel_file_path) => {\n            if !result.iter().any(|ar| ar.rel_path == rel_file_path) {\n                result.push(ArchiveFile {\n                    rel_path: rel_file_path.to_path_buf(),\n                    rel_str: rel_file_path\n                        .to_str()\n                        .expect(\"everything was utf8\")\n                        .to_string(),\n                    contents: FileContents::OnDisk(abs_file_path),\n                })\n            }\n        }\n        Err(_) => {\n            // The file exists somewhere outside of the package.\n            let file_name = file_path.file_name().unwrap();\n            if result.iter().any(|ar| ar.rel_path == file_name) {\n                ws.gctx().shell().warn(&format!(\n                    \"{} `{}` appears to be a path outside of the package, \\\n                            but there is already a file named `{}` in the root of the package. \\\n                            The archived crate will contain the copy in the root of the package. \\\n                            Update the {} to point to the path relative \\\n                            to the root of the package to remove this warning.\",\n                    label,\n                    file_path.display(),\n                    file_name.to_str().unwrap(),\n                    label,\n                ))?;\n            } else {\n                result.push(ArchiveFile {\n                    rel_path: PathBuf::from(file_name),\n                    rel_str: file_name.to_str().unwrap().to_string(),\n                    contents: FileContents::OnDisk(abs_file_path),\n                })\n            }\n        }\n    }\n    Ok(())\n}\n\nfn warn_on_nonexistent_file(\n    pkg: &Package,\n    path: &Path,\n    manifest_key_name: &'static str,\n    ws: &Workspace<'_>,\n) -> CargoResult<()> {\n    let rel_msg = if path.is_absolute() {\n        \"\".to_string()\n    } else {\n        format!(\" (relative to `{}`)\", pkg.root().display())\n    };\n    ws.gctx().shell().warn(&format!(\n        \"{manifest_key_name} `{}` does not appear to exist{}.\\n\\\n                Please update the {manifest_key_name} setting in the manifest at `{}`\\n\\\n                This may become a hard error in the future.\",\n        path.display(),\n        rel_msg,\n        pkg.manifest_path().display()\n    ))\n}\n\nfn error_custom_build_file_not_in_package(\n    pkg: &Package,\n    path: &Path,\n    target: &Target,\n) -> CargoResult<Vec<ArchiveFile>> {\n    let tip = {\n        let description_name = target.description_named();\n        if path.is_file() {\n            format!(\"the source file of {description_name} doesn't appear to be a path inside of the package.\\n\\\n            It is at `{}`, whereas the root the package is `{}`.\\n\",\n            path.display(), pkg.root().display()\n            )\n        } else {\n            format!(\"the source file of {description_name} doesn't appear to exist.\\n\",)\n        }\n    };\n    let msg = format!(\n        \"{}\\\n        This may cause issue during packaging, as modules resolution and resources included via macros are often relative to the path of source files.\\n\\\n        Please update the `build` setting in the manifest at `{}` and point to a path inside the root of the package.\",\n        tip,  pkg.manifest_path().display()\n    );\n    anyhow::bail!(msg)\n}\n\n/// Construct `Cargo.lock` for the package to be published.\nfn build_lock(ws: &Workspace<'_>, publish_pkg: &Package) -> CargoResult<String> {\n    let gctx = ws.gctx();\n    let orig_resolve = ops::load_pkg_lockfile(ws)?;\n\n    let tmp_ws = Workspace::ephemeral(publish_pkg.clone(), ws.gctx(), None, true)?;\n    let mut tmp_reg = PackageRegistry::new(ws.gctx())?;\n    let mut new_resolve = ops::resolve_with_previous(\n        &mut tmp_reg,\n        &tmp_ws,\n        &CliFeatures::new_all(true),\n        HasDevUnits::Yes,\n        orig_resolve.as_ref(),\n        None,\n        &[],\n        true,\n    )?;\n    let pkg_set = ops::get_resolved_packages(&new_resolve, tmp_reg)?;\n\n    if let Some(orig_resolve) = orig_resolve {\n        compare_resolve(gctx, tmp_ws.current()?, &orig_resolve, &new_resolve)?;\n    }\n    check_yanked(\n        gctx,\n        &pkg_set,\n        &new_resolve,\n        \"consider updating to a version that is not yanked\",\n    )?;\n\n    ops::resolve_to_string(&tmp_ws, &mut new_resolve)\n}\n\n// Checks that the package has some piece of metadata that a human can\n// use to tell what the package is about.\nfn check_metadata(pkg: &Package, gctx: &GlobalContext) -> CargoResult<()> {\n    let md = pkg.manifest().metadata();\n\n    let mut missing = vec![];\n\n    macro_rules! lacking {\n        ($( $($field: ident)||* ),*) => {{\n            $(\n                if $(md.$field.as_ref().map_or(true, |s| s.is_empty()))&&* {\n                    $(missing.push(stringify!($field).replace(\"_\", \"-\"));)*\n                }\n            )*\n        }}\n    }\n    lacking!(\n        description,\n        license || license_file,\n        documentation || homepage || repository\n    );\n\n    if !missing.is_empty() {\n        let mut things = missing[..missing.len() - 1].join(\", \");\n        // `things` will be empty if and only if its length is 1 (i.e., the only case\n        // to have no `or`).\n        if !things.is_empty() {\n            things.push_str(\" or \");\n        }\n        things.push_str(missing.last().unwrap());\n\n        gctx.shell().warn(&format!(\n            \"manifest has no {things}.\\n\\\n             See https://doc.rust-lang.org/cargo/reference/manifest.html#package-metadata for more info.\",\n            things = things\n        ))?\n    }\n\n    Ok(())\n}\n\n/// Checks if the package source is in a *git* DVCS repository. If *git*, and\n/// the source is *dirty* (e.g., has uncommitted changes) then `bail!` with an\n/// informative message. Otherwise return the sha1 hash of the current *HEAD*\n/// commit, or `None` if no repo is found.\nfn check_repo_state(\n    p: &Package,\n    src_files: &[PathBuf],\n    gctx: &GlobalContext,\n) -> CargoResult<Option<VcsInfo>> {\n    if let Ok(repo) = git2::Repository::discover(p.root()) {\n        if let Some(workdir) = repo.workdir() {\n            debug!(\"found a git repo at {:?}\", workdir);\n            let path = p.manifest_path();\n            let path =\n                paths::strip_prefix_canonical(path, workdir).unwrap_or_else(|_| path.to_path_buf());\n            if let Ok(status) = repo.status_file(&path) {\n                if (status & git2::Status::IGNORED).is_empty() {\n                    debug!(\n                        \"found (git) Cargo.toml at {:?} in workdir {:?}\",\n                        path, workdir\n                    );\n                    let path_in_vcs = path\n                        .parent()\n                        .and_then(|p| p.to_str())\n                        .unwrap_or(\"\")\n                        .replace(\"\\\\\", \"/\");\n                    return Ok(Some(VcsInfo {\n                        git: git(p, src_files, &repo)?,\n                        path_in_vcs,\n                    }));\n                }\n            }\n            gctx.shell().verbose(|shell| {\n                shell.warn(format!(\n                    \"no (git) Cargo.toml found at `{}` in workdir `{}`\",\n                    path.display(),\n                    workdir.display()\n                ))\n            })?;\n        }\n    } else {\n        gctx.shell().verbose(|shell| {\n            shell.warn(format!(\"no (git) VCS found for `{}`\", p.root().display()))\n        })?;\n    }\n\n    // No VCS with a checked in `Cargo.toml` found, so we don't know if the\n    // directory is dirty or not, thus we have to assume that it's clean.\n    return Ok(None);\n\n    fn git(p: &Package, src_files: &[PathBuf], repo: &git2::Repository) -> CargoResult<GitVcsInfo> {\n        // This is a collection of any dirty or untracked files. This covers:\n        // - new/modified/deleted/renamed/type change (index or worktree)\n        // - untracked files (which are \"new\" worktree files)\n        // - ignored (in case the user has an `include` directive that\n        //   conflicts with .gitignore).\n        let mut dirty_files = Vec::new();\n        collect_statuses(repo, &mut dirty_files)?;\n        // Include each submodule so that the error message can provide\n        // specifically *which* files in a submodule are modified.\n        status_submodules(repo, &mut dirty_files)?;\n\n        // Find the intersection of dirty in git, and the src_files that would\n        // be packaged. This is a lazy n^2 check, but seems fine with\n        // thousands of files.\n        let dirty_src_files: Vec<String> = src_files\n            .iter()\n            .filter(|src_file| dirty_files.iter().any(|path| src_file.starts_with(path)))\n            .map(|path| {\n                path.strip_prefix(p.root())\n                    .unwrap_or(path)\n                    .display()\n                    .to_string()\n            })\n            .collect();\n        if dirty_src_files.is_empty() {\n            let rev_obj = repo.revparse_single(\"HEAD\")?;\n            Ok(GitVcsInfo {\n                sha1: rev_obj.id().to_string(),\n            })\n        } else {\n            anyhow::bail!(\n                \"{} files in the working directory contain changes that were \\\n                 not yet committed into git:\\n\\n{}\\n\\n\\\n                 to proceed despite this and include the uncommitted changes, pass the `--allow-dirty` flag\",\n                dirty_src_files.len(),\n                dirty_src_files.join(\"\\n\")\n            )\n        }\n    }\n\n    // Helper to collect dirty statuses for a single repo.\n    fn collect_statuses(\n        repo: &git2::Repository,\n        dirty_files: &mut Vec<PathBuf>,\n    ) -> CargoResult<()> {\n        let mut status_opts = git2::StatusOptions::new();\n        // Exclude submodules, as they are being handled manually by recursing\n        // into each one so that details about specific files can be\n        // retrieved.\n        status_opts\n            .exclude_submodules(true)\n            .include_ignored(true)\n            .include_untracked(true);\n        let repo_statuses = repo.statuses(Some(&mut status_opts)).with_context(|| {\n            format!(\n                \"failed to retrieve git status from repo {}\",\n                repo.path().display()\n            )\n        })?;\n        let workdir = repo.workdir().unwrap();\n        let this_dirty = repo_statuses.iter().filter_map(|entry| {\n            let path = entry.path().expect(\"valid utf-8 path\");\n            if path.ends_with(\"Cargo.lock\") && entry.status() == git2::Status::IGNORED {\n                // It is OK to include Cargo.lock even if it is ignored.\n                return None;\n            }\n            // Use an absolute path, so that comparing paths is easier\n            // (particularly with submodules).\n            Some(workdir.join(path))\n        });\n        dirty_files.extend(this_dirty);\n        Ok(())\n    }\n\n    // Helper to collect dirty statuses while recursing into submodules.\n    fn status_submodules(\n        repo: &git2::Repository,\n        dirty_files: &mut Vec<PathBuf>,\n    ) -> CargoResult<()> {\n        for submodule in repo.submodules()? {\n            // Ignore submodules that don't open, they are probably not initialized.\n            // If its files are required, then the verification step should fail.\n            if let Ok(sub_repo) = submodule.open() {\n                status_submodules(&sub_repo, dirty_files)?;\n                collect_statuses(&sub_repo, dirty_files)?;\n            }\n        }\n        Ok(())\n    }\n}\n\n/// Compresses and packages a list of [`ArchiveFile`]s and writes into the given file.\n///\n/// Returns the uncompressed size of the contents of the new archive file.\nfn tar(\n    ws: &Workspace<'_>,\n    pkg: &Package,\n    ar_files: Vec<ArchiveFile>,\n    dst: &File,\n    filename: &str,\n) -> CargoResult<u64> {\n    // Prepare the encoder and its header.\n    let filename = Path::new(filename);\n    let encoder = GzBuilder::new()\n        .filename(paths::path2bytes(filename)?)\n        .write(dst, Compression::best());\n\n    // Put all package files into a compressed archive.\n    let mut ar = Builder::new(encoder);\n    let gctx = ws.gctx();\n\n    let base_name = format!(\"{}-{}\", pkg.name(), pkg.version());\n    let base_path = Path::new(&base_name);\n    let publish_pkg = prepare_for_publish(pkg, ws)?;\n\n    let mut uncompressed_size = 0;\n    for ar_file in ar_files {\n        let ArchiveFile {\n            rel_path,\n            rel_str,\n            contents,\n        } = ar_file;\n        let ar_path = base_path.join(&rel_path);\n        gctx.shell()\n            .verbose(|shell| shell.status(\"Archiving\", &rel_str))?;\n        let mut header = Header::new_gnu();\n        match contents {\n            FileContents::OnDisk(disk_path) => {\n                let mut file = File::open(&disk_path).with_context(|| {\n                    format!(\"failed to open for archiving: `{}`\", disk_path.display())\n                })?;\n                let metadata = file.metadata().with_context(|| {\n                    format!(\"could not learn metadata for: `{}`\", disk_path.display())\n                })?;\n                header.set_metadata_in_mode(&metadata, HeaderMode::Deterministic);\n                header.set_cksum();\n                ar.append_data(&mut header, &ar_path, &mut file)\n                    .with_context(|| {\n                        format!(\"could not archive source file `{}`\", disk_path.display())\n                    })?;\n                uncompressed_size += metadata.len() as u64;\n            }\n            FileContents::Generated(generated_kind) => {\n                let contents = match generated_kind {\n                    GeneratedFile::Manifest => publish_pkg.manifest().to_resolved_contents()?,\n                    GeneratedFile::Lockfile => build_lock(ws, &publish_pkg)?,\n                    GeneratedFile::VcsInfo(ref s) => serde_json::to_string_pretty(s)?,\n                };\n                header.set_entry_type(EntryType::file());\n                header.set_mode(0o644);\n                header.set_size(contents.len() as u64);\n                // use something nonzero to avoid rust-lang/cargo#9512\n                header.set_mtime(1);\n                header.set_cksum();\n                ar.append_data(&mut header, &ar_path, contents.as_bytes())\n                    .with_context(|| format!(\"could not archive source file `{}`\", rel_str))?;\n                uncompressed_size += contents.len() as u64;\n            }\n        }\n    }\n\n    let encoder = ar.into_inner()?;\n    encoder.finish()?;\n    Ok(uncompressed_size)\n}\n\n/// Generate warnings when packaging Cargo.lock, and the resolve have changed.\nfn compare_resolve(\n    gctx: &GlobalContext,\n    current_pkg: &Package,\n    orig_resolve: &Resolve,\n    new_resolve: &Resolve,\n) -> CargoResult<()> {\n    if gctx.shell().verbosity() != Verbosity::Verbose {\n        return Ok(());\n    }\n    let new_set: BTreeSet<PackageId> = new_resolve.iter().collect();\n    let orig_set: BTreeSet<PackageId> = orig_resolve.iter().collect();\n    let added = new_set.difference(&orig_set);\n    // Removed entries are ignored, this is used to quickly find hints for why\n    // an entry changed.\n    let removed: Vec<&PackageId> = orig_set.difference(&new_set).collect();\n    for pkg_id in added {\n        if pkg_id.name() == current_pkg.name() && pkg_id.version() == current_pkg.version() {\n            // Skip the package that is being created, since its SourceId\n            // (directory) changes.\n            continue;\n        }\n        // Check for candidates where the source has changed (such as [patch]\n        // or a dependency with multiple sources like path/version).\n        let removed_candidates: Vec<&PackageId> = removed\n            .iter()\n            .filter(|orig_pkg_id| {\n                orig_pkg_id.name() == pkg_id.name() && orig_pkg_id.version() == pkg_id.version()\n            })\n            .cloned()\n            .collect();\n        let extra = match removed_candidates.len() {\n            0 => {\n                // This can happen if the original was out of date.\n                let previous_versions: Vec<&PackageId> = removed\n                    .iter()\n                    .filter(|orig_pkg_id| orig_pkg_id.name() == pkg_id.name())\n                    .cloned()\n                    .collect();\n                match previous_versions.len() {\n                    0 => String::new(),\n                    1 => format!(\n                        \", previous version was `{}`\",\n                        previous_versions[0].version()\n                    ),\n                    _ => format!(\n                        \", previous versions were: {}\",\n                        previous_versions\n                            .iter()\n                            .map(|pkg_id| format!(\"`{}`\", pkg_id.version()))\n                            .collect::<Vec<_>>()\n                            .join(\", \")\n                    ),\n                }\n            }\n            1 => {\n                // This can happen for multi-sourced dependencies like\n                // `{path=\"...\", version=\"...\"}` or `[patch]` replacement.\n                // `[replace]` is not captured in Cargo.lock.\n                format!(\n                    \", was originally sourced from `{}`\",\n                    removed_candidates[0].source_id()\n                )\n            }\n            _ => {\n                // I don't know if there is a way to actually trigger this,\n                // but handle it just in case.\n                let comma_list = removed_candidates\n                    .iter()\n                    .map(|pkg_id| format!(\"`{}`\", pkg_id.source_id()))\n                    .collect::<Vec<_>>()\n                    .join(\", \");\n                format!(\n                    \", was originally sourced from one of these sources: {}\",\n                    comma_list\n                )\n            }\n        };\n        let msg = format!(\n            \"package `{}` added to the packaged Cargo.lock file{}\",\n            pkg_id, extra\n        );\n        gctx.shell().note(msg)?;\n    }\n    Ok(())\n}\n\npub fn check_yanked(\n    gctx: &GlobalContext,\n    pkg_set: &PackageSet<'_>,\n    resolve: &Resolve,\n    hint: &str,\n) -> CargoResult<()> {\n    // Checking the yanked status involves taking a look at the registry and\n    // maybe updating files, so be sure to lock it here.\n    let _lock = gctx.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;\n\n    let mut sources = pkg_set.sources_mut();\n    let mut pending: Vec<PackageId> = resolve.iter().collect();\n    let mut results = Vec::new();\n    for (_id, source) in sources.sources_mut() {\n        source.invalidate_cache();\n    }\n    while !pending.is_empty() {\n        pending.retain(|pkg_id| {\n            if let Some(source) = sources.get_mut(pkg_id.source_id()) {\n                match source.is_yanked(*pkg_id) {\n                    Poll::Ready(result) => results.push((*pkg_id, result)),\n                    Poll::Pending => return true,\n                }\n            }\n            false\n        });\n        for (_id, source) in sources.sources_mut() {\n            source.block_until_ready()?;\n        }\n    }\n\n    for (pkg_id, is_yanked) in results {\n        if is_yanked? {\n            gctx.shell().warn(format!(\n                \"package `{}` in Cargo.lock is yanked in registry `{}`, {}\",\n                pkg_id,\n                pkg_id.source_id().display_registry_name(),\n                hint\n            ))?;\n        }\n    }\n    Ok(())\n}\n\nfn run_verify(\n    ws: &Workspace<'_>,\n    pkg: &Package,\n    tar: &FileLock,\n    opts: &PackageOpts<'_>,\n) -> CargoResult<()> {\n    let gctx = ws.gctx();\n\n    gctx.shell().status(\"Verifying\", pkg)?;\n\n    let f = GzDecoder::new(tar.file());\n    let dst = tar\n        .parent()\n        .join(&format!(\"{}-{}\", pkg.name(), pkg.version()));\n    if dst.exists() {\n        paths::remove_dir_all(&dst)?;\n    }\n    let mut archive = Archive::new(f);\n    // We don't need to set the Modified Time, as it's not relevant to verification\n    // and it errors on filesystems that don't support setting a modified timestamp\n    archive.set_preserve_mtime(false);\n    archive.unpack(dst.parent().unwrap())?;\n\n    // Manufacture an ephemeral workspace to ensure that even if the top-level\n    // package has a workspace we can still build our new crate.\n    let id = SourceId::for_path(&dst)?;\n    let mut src = PathSource::new(&dst, id, ws.gctx());\n    let new_pkg = src.root_package()?;\n    let pkg_fingerprint = hash_all(&dst)?;\n    let ws = Workspace::ephemeral(new_pkg, gctx, None, true)?;\n\n    let rustc_args = if pkg\n        .manifest()\n        .unstable_features()\n        .require(Feature::public_dependency())\n        .is_ok()\n        || ws.gctx().cli_unstable().public_dependency\n    {\n        // FIXME: Turn this on at some point in the future\n        //Some(vec![\"-D exported_private_dependencies\".to_string()])\n        Some(vec![])\n    } else {\n        None\n    };\n\n    let exec: Arc<dyn Executor> = Arc::new(DefaultExecutor);\n    ops::compile_with_exec(\n        &ws,\n        &ops::CompileOptions {\n            build_config: BuildConfig::new(\n                gctx,\n                opts.jobs.clone(),\n                opts.keep_going,\n                &opts.targets,\n                CompileMode::Build,\n            )?,\n            cli_features: opts.cli_features.clone(),\n            spec: ops::Packages::Packages(Vec::new()),\n            filter: ops::CompileFilter::Default {\n                required_features_filterable: true,\n            },\n            target_rustdoc_args: None,\n            target_rustc_args: rustc_args,\n            target_rustc_crate_types: None,\n            rustdoc_document_private_items: false,\n            honor_rust_version: true,\n        },\n        &exec,\n    )?;\n\n    // Check that `build.rs` didn't modify any files in the `src` directory.\n    let ws_fingerprint = hash_all(&dst)?;\n    if pkg_fingerprint != ws_fingerprint {\n        let changes = report_hash_difference(&pkg_fingerprint, &ws_fingerprint);\n        anyhow::bail!(\n            \"Source directory was modified by build.rs during cargo publish. \\\n             Build scripts should not modify anything outside of OUT_DIR.\\n\\\n             {}\\n\\n\\\n             To proceed despite this, pass the `--no-verify` flag.\",\n            changes\n        )\n    }\n\n    Ok(())\n}\n\nfn hash_all(path: &Path) -> CargoResult<HashMap<PathBuf, u64>> {\n    fn wrap(path: &Path) -> CargoResult<HashMap<PathBuf, u64>> {\n        let mut result = HashMap::new();\n        let walker = walkdir::WalkDir::new(path).into_iter();\n        for entry in walker.filter_entry(|e| !(e.depth() == 1 && e.file_name() == \"target\")) {\n            let entry = entry?;\n            let file_type = entry.file_type();\n            if file_type.is_file() {\n                let file = File::open(entry.path())?;\n                let hash = util::hex::hash_u64_file(&file)?;\n                result.insert(entry.path().to_path_buf(), hash);\n            } else if file_type.is_symlink() {\n                let hash = util::hex::hash_u64(&fs::read_link(entry.path())?);\n                result.insert(entry.path().to_path_buf(), hash);\n            } else if file_type.is_dir() {\n                let hash = util::hex::hash_u64(&());\n                result.insert(entry.path().to_path_buf(), hash);\n            }\n        }\n        Ok(result)\n    }\n    let result = wrap(path).with_context(|| format!(\"failed to verify output at {:?}\", path))?;\n    Ok(result)\n}\n\nfn report_hash_difference(orig: &HashMap<PathBuf, u64>, after: &HashMap<PathBuf, u64>) -> String {\n    let mut changed = Vec::new();\n    let mut removed = Vec::new();\n    for (key, value) in orig {\n        match after.get(key) {\n            Some(after_value) => {\n                if value != after_value {\n                    changed.push(key.to_string_lossy());\n                }\n            }\n            None => removed.push(key.to_string_lossy()),\n        }\n    }\n    let mut added: Vec<_> = after\n        .keys()\n        .filter(|key| !orig.contains_key(*key))\n        .map(|key| key.to_string_lossy())\n        .collect();\n    let mut result = Vec::new();\n    if !changed.is_empty() {\n        changed.sort_unstable();\n        result.push(format!(\"Changed: {}\", changed.join(\"\\n\\t\")));\n    }\n    if !added.is_empty() {\n        added.sort_unstable();\n        result.push(format!(\"Added: {}\", added.join(\"\\n\\t\")));\n    }\n    if !removed.is_empty() {\n        removed.sort_unstable();\n        result.push(format!(\"Removed: {}\", removed.join(\"\\n\\t\")));\n    }\n    assert!(!result.is_empty(), \"unexpected empty change detection\");\n    result.join(\"\\n\")\n}\n\n// It can often be the case that files of a particular name on one platform\n// can't actually be created on another platform. For example files with colons\n// in the name are allowed on Unix but not on Windows.\n//\n// To help out in situations like this, issue about weird filenames when\n// packaging as a \"heads up\" that something may not work on other platforms.\nfn check_filename(file: &Path, shell: &mut Shell) -> CargoResult<()> {\n    let Some(name) = file.file_name() else {\n        return Ok(());\n    };\n    let Some(name) = name.to_str() else {\n        anyhow::bail!(\n            \"path does not have a unicode filename which may not unpack \\\n             on all platforms: {}\",\n            file.display()\n        )\n    };\n    let bad_chars = ['/', '\\\\', '<', '>', ':', '\"', '|', '?', '*'];\n    if let Some(c) = bad_chars.iter().find(|c| name.contains(**c)) {\n        anyhow::bail!(\n            \"cannot package a filename with a special character `{}`: {}\",\n            c,\n            file.display()\n        )\n    }\n    if restricted_names::is_windows_reserved_path(file) {\n        shell.warn(format!(\n            \"file {} is a reserved Windows filename, \\\n                it will not work on Windows platforms\",\n            file.display()\n        ))?;\n    }\n    Ok(())\n}\n\n// Path: src/cargo/ops/cargo_doc.rs\nuse crate::core::compiler::{Compilation, CompileKind};\nuse crate::core::{shell::Verbosity, Shell, Workspace};\nuse crate::ops;\nuse crate::util::context::{GlobalContext, PathAndArgs};\nuse crate::util::CargoResult;\nuse anyhow::{bail, Error};\nuse std::path::Path;\nuse std::path::PathBuf;\nuse std::process::Command;\nuse std::str::FromStr;\n\n/// Format of rustdoc [`--output-format`][1].\n///\n/// [1]: https://doc.rust-lang.org/nightly/rustdoc/unstable-features.html#-w--output-format-output-format\n#[derive(Debug, Default, Clone)]\npub enum OutputFormat {\n    #[default]\n    Html,\n    Json,\n}\n\nimpl OutputFormat {\n    pub const POSSIBLE_VALUES: [&'static str; 2] = [\"html\", \"json\"];\n}\n\nimpl FromStr for OutputFormat {\n    // bail! error instead of string error like impl FromStr for Edition {\n    type Err = Error;\n\n    fn from_str(s: &str) -> Result<Self, Self::Err> {\n        match s {\n            \"json\" => Ok(OutputFormat::Json),\n            \"html\" => Ok(OutputFormat::Html),\n            _ => bail!(\n                \"supported values for --output-format are `json` and `html`, \\\n\t\t\t\t\t\t but `{}` is unknown\",\n                s\n            ),\n        }\n    }\n}\n\n/// Strongly typed options for the `cargo doc` command.\n#[derive(Debug)]\npub struct DocOptions {\n    /// Whether to attempt to open the browser after compiling the docs\n    pub open_result: bool,\n    /// Same as `rustdoc --output-format`\n    pub output_format: OutputFormat,\n    /// Options to pass through to the compiler\n    pub compile_opts: ops::CompileOptions,\n}\n\n/// Main method for `cargo doc`.\npub fn doc(ws: &Workspace<'_>, options: &DocOptions) -> CargoResult<()> {\n    let compilation = ops::compile(ws, &options.compile_opts)?;\n\n    if options.open_result {\n        let name = &compilation\n            .root_crate_names\n            .get(0)\n            .ok_or_else(|| anyhow::anyhow!(\"no crates with documentation\"))?;\n        let kind = options.compile_opts.build_config.single_requested_kind()?;\n\n        let path = path_by_output_format(&compilation, &kind, &name, &options.output_format);\n\n        if path.exists() {\n            let config_browser = {\n                let cfg: Option<PathAndArgs> = ws.gctx().get(\"doc.browser\")?;\n                cfg.map(|path_args| (path_args.path.resolve_program(ws.gctx()), path_args.args))\n            };\n            let mut shell = ws.gctx().shell();\n            let link = shell.err_file_hyperlink(&path);\n            shell.status(\"Opening\", format!(\"{link}{}{link:#}\", path.display()))?;\n            open_docs(&path, &mut shell, config_browser, ws.gctx())?;\n        }\n    } else if ws.gctx().shell().verbosity() == Verbosity::Verbose {\n        for name in &compilation.root_crate_names {\n            for kind in &options.compile_opts.build_config.requested_kinds {\n                let path =\n                    path_by_output_format(&compilation, &kind, &name, &options.output_format);\n                if path.exists() {\n                    let mut shell = ws.gctx().shell();\n                    let link = shell.err_file_hyperlink(&path);\n                    shell.status(\"Generated\", format!(\"{link}{}{link:#}\", path.display()))?;\n                }\n            }\n        }\n    } else {\n        let mut output = compilation.root_crate_names.iter().flat_map(|name| {\n            options\n                .compile_opts\n                .build_config\n                .requested_kinds\n                .iter()\n                .map(|kind| path_by_output_format(&compilation, kind, name, &options.output_format))\n                .filter(|path| path.exists())\n        });\n        if let Some(first_path) = output.next() {\n            let remaining = output.count();\n            let remaining = match remaining {\n                0 => \"\".to_owned(),\n                1 => \" and 1 other file\".to_owned(),\n                n => format!(\" and {n} other files\"),\n            };\n\n            let mut shell = ws.gctx().shell();\n            let link = shell.err_file_hyperlink(&first_path);\n            shell.status(\n                \"Generated\",\n                format!(\"{link}{}{link:#}{remaining}\", first_path.display(),),\n            )?;\n        }\n    }\n\n    Ok(())\n}\n\nfn path_by_output_format(\n    compilation: &Compilation<'_>,\n    kind: &CompileKind,\n    name: &str,\n    output_format: &OutputFormat,\n) -> PathBuf {\n    if matches!(output_format, OutputFormat::Json) {\n        compilation.root_output[kind]\n            .with_file_name(\"doc\")\n            .join(format!(\"{}.json\", name))\n    } else {\n        compilation.root_output[kind]\n            .with_file_name(\"doc\")\n            .join(name)\n            .join(\"index.html\")\n    }\n}\n\nfn open_docs(\n    path: &Path,\n    shell: &mut Shell,\n    config_browser: Option<(PathBuf, Vec<String>)>,\n    gctx: &GlobalContext,\n) -> CargoResult<()> {\n    let browser =\n        config_browser.or_else(|| Some((PathBuf::from(gctx.get_env_os(\"BROWSER\")?), Vec::new())));\n\n    match browser {\n        Some((browser, initial_args)) => {\n            if let Err(e) = Command::new(&browser).args(initial_args).arg(path).status() {\n                shell.warn(format!(\n                    \"Couldn't open docs with {}: {}\",\n                    browser.to_string_lossy(),\n                    e\n                ))?;\n            }\n        }\n        None => {\n            if let Err(e) = opener::open(&path) {\n                let e = e.into();\n                crate::display_warning_with_error(\"couldn't open docs\", &e, shell);\n            }\n        }\n    };\n\n    Ok(())\n}\n\n// Path: src/cargo/ops/vendor.rs\nuse crate::core::shell::Verbosity;\nuse crate::core::{GitReference, Package, Workspace};\nuse crate::ops;\nuse crate::sources::path::PathSource;\nuse crate::sources::CRATES_IO_REGISTRY;\nuse crate::util::cache_lock::CacheLockMode;\nuse crate::util::{try_canonicalize, CargoResult, GlobalContext};\nuse anyhow::{bail, Context as _};\nuse cargo_util::{paths, Sha256};\nuse serde::Serialize;\nuse std::collections::HashSet;\nuse std::collections::{BTreeMap, BTreeSet, HashMap};\nuse std::ffi::OsStr;\nuse std::fs::{self, File, OpenOptions};\nuse std::io::{Read, Write};\nuse std::path::{Path, PathBuf};\n\npub struct VendorOptions<'a> {\n    pub no_delete: bool,\n    pub versioned_dirs: bool,\n    pub destination: &'a Path,\n    pub extra: Vec<PathBuf>,\n}\n\npub fn vendor(ws: &Workspace<'_>, opts: &VendorOptions<'_>) -> CargoResult<()> {\n    let gctx = ws.gctx();\n    let mut extra_workspaces = Vec::new();\n    for extra in opts.extra.iter() {\n        let extra = gctx.cwd().join(extra);\n        let ws = Workspace::new(&extra, gctx)?;\n        extra_workspaces.push(ws);\n    }\n    let workspaces = extra_workspaces.iter().chain(Some(ws)).collect::<Vec<_>>();\n    let _lock = gctx.acquire_package_cache_lock(CacheLockMode::MutateExclusive)?;\n    let vendor_config = sync(gctx, &workspaces, opts).with_context(|| \"failed to sync\")?;\n\n    if gctx.shell().verbosity() != Verbosity::Quiet {\n        if vendor_config.source.is_empty() {\n            crate::drop_eprintln!(gctx, \"There is no dependency to vendor in this project.\");\n        } else {\n            crate::drop_eprint!(\n                gctx,\n                \"To use vendored sources, add this to your .cargo/config.toml for this project:\\n\\n\"\n            );\n            crate::drop_print!(gctx, \"{}\", &toml::to_string_pretty(&vendor_config).unwrap());\n        }\n    }\n\n    Ok(())\n}\n\n#[derive(Serialize)]\nstruct VendorConfig {\n    source: BTreeMap<String, VendorSource>,\n}\n\n#[derive(Serialize)]\n#[serde(rename_all = \"lowercase\", untagged)]\nenum VendorSource {\n    Directory {\n        directory: String,\n    },\n    Registry {\n        registry: Option<String>,\n        #[serde(rename = \"replace-with\")]\n        replace_with: String,\n    },\n    Git {\n        git: String,\n        branch: Option<String>,\n        tag: Option<String>,\n        rev: Option<String>,\n        #[serde(rename = \"replace-with\")]\n        replace_with: String,\n    },\n}\n\nfn sync(\n    gctx: &GlobalContext,\n    workspaces: &[&Workspace<'_>],\n    opts: &VendorOptions<'_>,\n) -> CargoResult<VendorConfig> {\n    let canonical_destination = try_canonicalize(opts.destination);\n    let canonical_destination = canonical_destination.as_deref().unwrap_or(opts.destination);\n    let dest_dir_already_exists = canonical_destination.exists();\n\n    paths::create_dir_all(&canonical_destination)?;\n    let mut to_remove = HashSet::new();\n    if !opts.no_delete {\n        for entry in canonical_destination.read_dir()? {\n            let entry = entry?;\n            if !entry\n                .file_name()\n                .to_str()\n                .map_or(false, |s| s.starts_with('.'))\n            {\n                to_remove.insert(entry.path());\n            }\n        }\n    }\n\n    // First up attempt to work around rust-lang/cargo#5956. Apparently build\n    // artifacts sprout up in Cargo's global cache for whatever reason, although\n    // it's unsure what tool is causing these issues at this time. For now we\n    // apply a heavy-hammer approach which is to delete Cargo's unpacked version\n    // of each crate to start off with. After we do this we'll re-resolve and\n    // redownload again, which should trigger Cargo to re-extract all the\n    // crates.\n    //\n    // Note that errors are largely ignored here as this is a best-effort\n    // attempt. If anything fails here we basically just move on to the next\n    // crate to work with.\n    for ws in workspaces {\n        let (packages, resolve) =\n            ops::resolve_ws(ws).with_context(|| \"failed to load pkg lockfile\")?;\n\n        packages\n            .get_many(resolve.iter())\n            .with_context(|| \"failed to download packages\")?;\n\n        for pkg in resolve.iter() {\n            // Don't delete actual source code!\n            if pkg.source_id().is_path() {\n                if let Ok(path) = pkg.source_id().url().to_file_path() {\n                    if let Ok(path) = try_canonicalize(path) {\n                        to_remove.remove(&path);\n                    }\n                }\n                continue;\n            }\n            if pkg.source_id().is_git() {\n                continue;\n            }\n            if let Ok(pkg) = packages.get_one(pkg) {\n                drop(fs::remove_dir_all(pkg.root()));\n            }\n        }\n    }\n\n    let mut checksums = HashMap::new();\n    let mut ids = BTreeMap::new();\n\n    // Next up let's actually download all crates and start storing internal\n    // tables about them.\n    for ws in workspaces {\n        let (packages, resolve) =\n            ops::resolve_ws(ws).with_context(|| \"failed to load pkg lockfile\")?;\n\n        packages\n            .get_many(resolve.iter())\n            .with_context(|| \"failed to download packages\")?;\n\n        for pkg in resolve.iter() {\n            // No need to vendor path crates since they're already in the\n            // repository\n            if pkg.source_id().is_path() {\n                continue;\n            }\n            ids.insert(\n                pkg,\n                packages\n                    .get_one(pkg)\n                    .with_context(|| \"failed to fetch package\")?\n                    .clone(),\n            );\n\n            checksums.insert(pkg, resolve.checksums().get(&pkg).cloned());\n        }\n    }\n\n    let mut versions = HashMap::new();\n    for id in ids.keys() {\n        let map = versions.entry(id.name()).or_insert_with(BTreeMap::default);\n        if let Some(prev) = map.get(&id.version()) {\n            bail!(\n                \"found duplicate version of package `{} v{}` \\\n                 vendored from two sources:\\n\\\n                 \\n\\\n                 \\tsource 1: {}\\n\\\n                 \\tsource 2: {}\",\n                id.name(),\n                id.version(),\n                prev,\n                id.source_id()\n            );\n        }\n        map.insert(id.version(), id.source_id());\n    }\n\n    let mut sources = BTreeSet::new();\n    let mut tmp_buf = [0; 64 * 1024];\n    for (id, pkg) in ids.iter() {\n        // Next up, copy it to the vendor directory\n        let src = pkg.root();\n        let max_version = *versions[&id.name()].iter().rev().next().unwrap().0;\n        let dir_has_version_suffix = opts.versioned_dirs || id.version() != max_version;\n        let dst_name = if dir_has_version_suffix {\n            // Eg vendor/futures-0.1.13\n            format!(\"{}-{}\", id.name(), id.version())\n        } else {\n            // Eg vendor/futures\n            id.name().to_string()\n        };\n\n        sources.insert(id.source_id());\n        let dst = canonical_destination.join(&dst_name);\n        to_remove.remove(&dst);\n        let cksum = dst.join(\".cargo-checksum.json\");\n        if dir_has_version_suffix && cksum.exists() {\n            // Always re-copy directory without version suffix in case the version changed\n            continue;\n        }\n\n        gctx.shell().status(\n            \"Vendoring\",\n            &format!(\"{} ({}) to {}\", id, src.to_string_lossy(), dst.display()),\n        )?;\n\n        let _ = fs::remove_dir_all(&dst);\n        let pathsource = PathSource::new(src, id.source_id(), gctx);\n        let paths = pathsource.list_files(pkg)?;\n        let mut map = BTreeMap::new();\n        cp_sources(pkg, src, &paths, &dst, &mut map, &mut tmp_buf)\n            .with_context(|| format!(\"failed to copy over vendored sources for: {}\", id))?;\n\n        // Finally, emit the metadata about this package\n        let json = serde_json::json!({\n            \"package\": checksums.get(id),\n            \"files\": map,\n        });\n\n        paths::write(&cksum, json.to_string())?;\n    }\n\n    for path in to_remove {\n        if path.is_dir() {\n            paths::remove_dir_all(&path)?;\n        } else {\n            paths::remove_file(&path)?;\n        }\n    }\n\n    // add our vendored source\n    let mut config = BTreeMap::new();\n\n    let merged_source_name = \"vendored-sources\";\n\n    // replace original sources with vendor\n    for source_id in sources {\n        let name = if source_id.is_crates_io() {\n            CRATES_IO_REGISTRY.to_string()\n        } else {\n            // Remove `precise` since that makes the source name very long,\n            // and isn't needed to disambiguate multiple sources.\n            source_id.without_precise().as_url().to_string()\n        };\n\n        let source = if source_id.is_crates_io() {\n            VendorSource::Registry {\n                registry: None,\n                replace_with: merged_source_name.to_string(),\n            }\n        } else if source_id.is_remote_registry() {\n            let registry = source_id.url().to_string();\n            VendorSource::Registry {\n...\n", "needle_token_start": 2209, "needle_token_end": 2641, "code_context_ntokens": 16253, "output": ["```rust\npub fn package(ws: &Workspace<'_>, opts: &PackageOpts<'_>) -> CargoResult<Option<Vec<FileLock>>> {\n    let pkgs = ws.members_with_features(\n        &opts.to_package.to_package_id_specs(ws)?,\n        &opts.cli_features,\n    )?;\n\n    let mut dsts = Vec::with_capacity(pkgs.len());\n\n    if ws.root().join(\"Cargo.lock\").exists() {\n        // Make sure the Cargo.lock is up-to-date and valid.\n        let _ = ops::resolve_ws(ws)?;\n        // If Cargo.lock does not exist, it will be generated by `build_lock`\n        // below, and will be validated during the verification step.\n    }\n\n    for (pkg, cli_features) in pkgs {\n        let result = package_one(\n            ws,\n            pkg,\n            &PackageOpts {\n                gctx: opts.gctx,\n                list: opts.list,\n                check_metadata: opts.check_metadata,\n                allow_dirty: opts.allow_dirty,\n                verify: opts.verify,\n                jobs: opts.jobs.clone(),\n                keep_going: opts.keep_going,\n                to_package: ops::Packages::Default,\n                targets: opts.targets.clone(),\n                cli_features: cli_features,\n            },\n        )?;\n\n        if !opts.list {\n            dsts.push(result.unwrap());\n        }\n    }\n\n    if opts.list {\n        // We're just listing, so there's no file output\n        Ok(None)\n    } else {\n        Ok(Some(dsts))\n    }\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "test_cratesio_hash", "language": "rust", "path": "src/cargo/core/source_id.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To verify the consistency of the hash value generated for a specific package source across different versions of a package manager. This ensures that users do not need to redownload package indices and packages unnecessarily when updating the package manager.\n2. **Input**: No direct inputs are provided by the user; the function internally generates the required source identifier using a default global context.\n3. **Output**: The function does not return a value but asserts that the computed hash matches a predefined, expected hash value.\n4. **Procedure**: The function initializes a global context and uses it to create a source identifier for a well-known package registry. It then computes the hash of this source identifier and checks if this hash matches a specific, known hash value. The test is conditionally compiled and run only on systems with little-endian architecture and 64-bit pointers to ensure consistency.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                let path = self\n                    .inner\n                    .url\n                    .to_file_path()\n                    .expect(\"path sources cannot be remote\");\n                Ok(Box::new(RegistrySource::local(\n                    self,\n                    &path,\n                    yanked_whitelist,\n                    gctx,\n                )))\n            }\n            SourceKind::Directory => {\n                let path = self\n                    .inner\n                    .url\n                    .to_file_path()\n                    .expect(\"path sources cannot be remote\");\n                Ok(Box::new(DirectorySource::new(&path, self, gctx)))\n            }\n        }\n    }\n\n    /// Gets the Git reference if this is a git source, otherwise `None`.\n    pub fn git_reference(self) -> Option<&'static GitReference> {\n        match self.inner.kind {\n            SourceKind::Git(ref s) => Some(s),\n            _ => None,\n        }\n    }\n\n    /// Check if the precise data field has bean set\n    pub fn has_precise(self) -> bool {\n        self.inner.precise.is_some()\n    }\n\n    /// Check if the precise data field has bean set to \"locked\"\n    pub fn has_locked_precise(self) -> bool {\n        self.inner.precise == Some(Precise::Locked)\n    }\n\n    /// Check if two sources have the same precise data field\n    pub fn has_same_precise_as(self, other: Self) -> bool {\n        self.inner.precise == other.inner.precise\n    }\n\n    /// Check if the precise data field stores information for this `name`\n    /// from a call to [SourceId::with_precise_registry_version].\n    ///\n    /// If so return the version currently in the lock file and the version to be updated to.\n    pub fn precise_registry_version(\n        self,\n        pkg: &str,\n    ) -> Option<(&semver::Version, &semver::Version)> {\n        match &self.inner.precise {\n            Some(Precise::Updated { name, from, to }) if name == pkg => Some((from, to)),\n            _ => None,\n        }\n    }\n\n    pub fn precise_git_fragment(self) -> Option<&'static str> {\n        match &self.inner.precise {\n            Some(Precise::GitUrlFragment(s)) => Some(&s),\n            _ => None,\n        }\n    }\n\n    /// Creates a new `SourceId` from this source with the given `precise`.\n    pub fn with_git_precise(self, fragment: Option<String>) -> SourceId {\n        SourceId::wrap(SourceIdInner {\n            precise: fragment.map(|f| Precise::GitUrlFragment(f)),\n            ..(*self.inner).clone()\n        })\n    }\n\n    /// Creates a new `SourceId` from this source without a `precise`.\n    pub fn without_precise(self) -> SourceId {\n        SourceId::wrap(SourceIdInner {\n            precise: None,\n            ..(*self.inner).clone()\n        })\n    }\n\n    /// Creates a new `SourceId` from this source without a `precise`.\n    pub fn with_locked_precise(self) -> SourceId {\n        SourceId::wrap(SourceIdInner {\n            precise: Some(Precise::Locked),\n            ..(*self.inner).clone()\n        })\n    }\n\n    /// Creates a new `SourceId` from this source with the `precise` from some other `SourceId`.\n    pub fn with_precise_from(self, v: Self) -> SourceId {\n        SourceId::wrap(SourceIdInner {\n            precise: v.inner.precise.clone(),\n            ..(*self.inner).clone()\n        })\n    }\n\n    /// When updating a lock file on a version using `cargo update --precise`\n    /// the requested version is stored in the precise field.\n    /// On a registry dependency we also need to keep track of the package that\n    /// should be updated and even which of the versions should be updated.\n    /// All of this gets encoded in the precise field using this method.\n    /// The data can be read with [SourceId::precise_registry_version]\n    pub fn with_precise_registry_version(\n        self,\n        name: InternedString,\n        version: semver::Version,\n        precise: &str,\n    ) -> CargoResult<SourceId> {\n        let precise = semver::Version::parse(precise)\n            .with_context(|| format!(\"invalid version format for precise version `{precise}`\"))?;\n\n        Ok(SourceId::wrap(SourceIdInner {\n            precise: Some(Precise::Updated {\n                name,\n                from: version,\n                to: precise,\n            }),\n            ..(*self.inner).clone()\n        }))\n    }\n\n    /// Returns `true` if the remote registry is the standard <https://crates.io>.\n    pub fn is_crates_io(self) -> bool {\n        match self.inner.kind {\n            SourceKind::Registry | SourceKind::SparseRegistry => {}\n            _ => return false,\n        }\n        let url = self.inner.url.as_str();\n        url == CRATES_IO_INDEX || url == CRATES_IO_HTTP_INDEX || is_overridden_crates_io_url(url)\n    }\n\n    /// Hashes `self`.\n    ///\n    /// For paths, remove the workspace prefix so the same source will give the\n    /// same hash in different locations, helping reproducible builds.\n    pub fn stable_hash<S: hash::Hasher>(self, workspace: &Path, into: &mut S) {\n        if self.is_path() {\n            if let Ok(p) = self\n                .inner\n                .url\n                .to_file_path()\n                .unwrap()\n                .strip_prefix(workspace)\n            {\n                self.inner.kind.hash(into);\n                p.to_str().unwrap().hash(into);\n                return;\n            }\n        }\n        self.hash(into)\n    }\n\n    pub fn full_eq(self, other: SourceId) -> bool {\n        ptr::eq(self.inner, other.inner)\n    }\n\n    pub fn full_hash<S: hash::Hasher>(self, into: &mut S) {\n        ptr::NonNull::from(self.inner).hash(into)\n    }\n}\n\nimpl PartialEq for SourceId {\n    fn eq(&self, other: &SourceId) -> bool {\n        self.cmp(other) == Ordering::Equal\n    }\n}\n\nimpl PartialOrd for SourceId {\n    fn partial_cmp(&self, other: &SourceId) -> Option<Ordering> {\n        Some(self.cmp(other))\n    }\n}\n\n// Custom comparison defined as canonical URL equality for git sources and URL\n// equality for other sources, ignoring the `precise` and `name` fields.\nimpl Ord for SourceId {\n    fn cmp(&self, other: &SourceId) -> Ordering {\n        // If our interior pointers are to the exact same `SourceIdInner` then\n        // we're guaranteed to be equal.\n        if ptr::eq(self.inner, other.inner) {\n            return Ordering::Equal;\n        }\n\n        // Sort first based on `kind`, deferring to the URL comparison below if\n        // the kinds are equal.\n        match self.inner.kind.cmp(&other.inner.kind) {\n            Ordering::Equal => {}\n            other => return other,\n        }\n\n        // If the `kind` and the `url` are equal, then for git sources we also\n        // ensure that the canonical urls are equal.\n        match (&self.inner.kind, &other.inner.kind) {\n            (SourceKind::Git(_), SourceKind::Git(_)) => {\n                self.inner.canonical_url.cmp(&other.inner.canonical_url)\n            }\n            _ => self.inner.url.cmp(&other.inner.url),\n        }\n    }\n}\n\nimpl ser::Serialize for SourceId {\n    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>\n    where\n        S: ser::Serializer,\n    {\n        if self.is_path() {\n            None::<String>.serialize(s)\n        } else {\n            s.collect_str(&self.as_url())\n        }\n    }\n}\n\nimpl<'de> de::Deserialize<'de> for SourceId {\n    fn deserialize<D>(d: D) -> Result<SourceId, D::Error>\n    where\n        D: de::Deserializer<'de>,\n    {\n        let string = String::deserialize(d)?;\n        SourceId::from_url(&string).map_err(de::Error::custom)\n    }\n}\n\nfn url_display(url: &Url) -> String {\n    if url.scheme() == \"file\" {\n        if let Ok(path) = url.to_file_path() {\n            if let Some(path_str) = path.to_str() {\n                return path_str.to_string();\n            }\n        }\n    }\n\n    url.as_str().to_string()\n}\n\nimpl fmt::Display for SourceId {\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        match self.inner.kind {\n            SourceKind::Git(ref reference) => {\n                // Don't replace the URL display for git references,\n                // because those are kind of expected to be URLs.\n                write!(f, \"{}\", self.inner.url)?;\n                // TODO(-Znext-lockfile-bump): set it to true when the default is\n                // lockfile v4, because we want Source ID serialization to be\n                // consistent with lockfile.\n                if let Some(pretty) = reference.pretty_ref(false) {\n                    write!(f, \"?{}\", pretty)?;\n                }\n\n                if let Some(s) = &self.inner.precise {\n                    let s = s.to_string();\n                    let len = cmp::min(s.len(), 8);\n                    write!(f, \"#{}\", &s[..len])?;\n                }\n                Ok(())\n            }\n            SourceKind::Path => write!(f, \"{}\", url_display(&self.inner.url)),\n            SourceKind::Registry | SourceKind::SparseRegistry => {\n                write!(f, \"registry `{}`\", self.display_registry_name())\n            }\n            SourceKind::LocalRegistry => write!(f, \"registry `{}`\", url_display(&self.inner.url)),\n            SourceKind::Directory => write!(f, \"dir {}\", url_display(&self.inner.url)),\n        }\n    }\n}\n\n/// The hash of SourceId is used in the name of some Cargo folders, so shouldn't\n/// vary. `as_str` gives the serialisation of a url (which has a spec) and so\n/// insulates against possible changes in how the url crate does hashing.\nimpl Hash for SourceId {\n    fn hash<S: hash::Hasher>(&self, into: &mut S) {\n        self.inner.kind.hash(into);\n        match self.inner.kind {\n            SourceKind::Git(_) => self.inner.canonical_url.hash(into),\n            _ => self.inner.url.as_str().hash(into),\n        }\n    }\n}\n\n/// The hash of `SourceIdInner` is used to retrieve its interned value from\n/// `SOURCE_ID_CACHE`. We only care about fields that make `SourceIdInner`\n/// unique. Optional fields not affecting the uniqueness must be excluded,\n/// such as [`registry_key`]. That's why this is not derived.\n///\n/// [`registry_key`]: SourceIdInner::registry_key\nimpl Hash for SourceIdInner {\n    fn hash<S: hash::Hasher>(&self, into: &mut S) {\n        self.kind.hash(into);\n        self.precise.hash(into);\n        self.canonical_url.hash(into);\n    }\n}\n\n/// This implementation must be synced with [`SourceIdInner::hash`].\nimpl PartialEq for SourceIdInner {\n    fn eq(&self, other: &Self) -> bool {\n        self.kind == other.kind\n            && self.precise == other.precise\n            && self.canonical_url == other.canonical_url\n    }\n}\n\n/// A `Display`able view into a `SourceId` that will write it as a url\npub struct SourceIdAsUrl<'a> {\n    inner: &'a SourceIdInner,\n    encoded: bool,\n}\n\nimpl<'a> fmt::Display for SourceIdAsUrl<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        if let Some(protocol) = self.inner.kind.protocol() {\n            write!(f, \"{protocol}+\")?;\n        }\n        write!(f, \"{}\", self.inner.url)?;\n        if let SourceIdInner {\n            kind: SourceKind::Git(ref reference),\n            ref precise,\n            ..\n        } = *self.inner\n        {\n            if let Some(pretty) = reference.pretty_ref(self.encoded) {\n                write!(f, \"?{}\", pretty)?;\n            }\n            if let Some(precise) = precise.as_ref() {\n                write!(f, \"#{}\", precise)?;\n            }\n        }\n        Ok(())\n    }\n}\n\nimpl KeyOf {\n    /// Gets the underlying key.\n    fn key(&self) -> &str {\n        match self {\n            KeyOf::Registry(k) | KeyOf::Source(k) => k,\n        }\n    }\n\n    /// Gets the key if it's from an alternative registry.\n    fn alternative_registry(&self) -> Option<&str> {\n        match self {\n            KeyOf::Registry(k) => Some(k),\n            _ => None,\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::{GitReference, SourceId, SourceKind};\n    use crate::util::{GlobalContext, IntoUrl};\n\n    #[test]\n    fn github_sources_equal() {\n        let loc = \"https://github.com/foo/bar\".into_url().unwrap();\n        let default = SourceKind::Git(GitReference::DefaultBranch);\n        let s1 = SourceId::new(default.clone(), loc, None).unwrap();\n\n        let loc = \"git://github.com/foo/bar\".into_url().unwrap();\n        let s2 = SourceId::new(default, loc.clone(), None).unwrap();\n\n        assert_eq!(s1, s2);\n\n        let foo = SourceKind::Git(GitReference::Branch(\"foo\".to_string()));\n        let s3 = SourceId::new(foo, loc, None).unwrap();\n        assert_ne!(s1, s3);\n    }\n\n    // This is a test that the hash of the `SourceId` for crates.io is a well-known\n    // value.\n    //\n    // Note that the hash value matches what the crates.io source id has hashed\n    // since long before Rust 1.30. We strive to keep this value the same across\n    // versions of Cargo because changing it means that users will need to\n    // redownload the index and all crates they use when using a new Cargo version.\n    //\n    // This isn't to say that this hash can *never* change, only that when changing\n    // this it should be explicitly done. If this hash changes accidentally and\n    // you're able to restore the hash to its original value, please do so!\n    // Otherwise please just leave a comment in your PR as to why the hash value is\n    // changing and why the old value can't be easily preserved.\n    //\n    // The hash value depends on endianness and bit-width, so we only run this test on\n    // little-endian 64-bit CPUs (such as x86-64 and ARM64) where it matches the\n    // well-known value.\n    #[test]\n    #[cfg(all(target_endian = \"little\", target_pointer_width = \"64\"))]\n    \nfn test_cratesio_hash() {\n        let gctx = GlobalContext::default().unwrap();\n        let crates_io = SourceId::crates_io(&gctx).unwrap();\n        assert_eq!(crate::util::hex::short_hash(&crates_io), \"1ecc6299db9ec823\");\n    }\n\n    // See the comment in `test_cratesio_hash`.\n    //\n    // Only test on non-Windows as paths on Windows will get different hashes.\n    #[test]\n    #[cfg(all(target_endian = \"little\", target_pointer_width = \"64\", not(windows)))]\n    fn test_stable_hash() {\n        use std::hash::Hasher;\n        use std::path::Path;\n\n        let gen_hash = |source_id: SourceId| {\n            let mut hasher = std::collections::hash_map::DefaultHasher::new();\n            source_id.stable_hash(Path::new(\"/tmp/ws\"), &mut hasher);\n            hasher.finish()\n        };\n\n        let url = \"https://my-crates.io\".into_url().unwrap();\n        let source_id = SourceId::for_registry(&url).unwrap();\n        assert_eq!(gen_hash(source_id), 18108075011063494626);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"fb60813d6cb8df79\");\n\n        let url = \"https://your-crates.io\".into_url().unwrap();\n        let source_id = SourceId::for_alt_registry(&url, \"alt\").unwrap();\n        assert_eq!(gen_hash(source_id), 12862859764592646184);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"09c10fd0cbd74bce\");\n\n        let url = \"sparse+https://my-crates.io\".into_url().unwrap();\n        let source_id = SourceId::for_registry(&url).unwrap();\n        assert_eq!(gen_hash(source_id), 8763561830438022424);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"d1ea0d96f6f759b5\");\n\n        let url = \"sparse+https://your-crates.io\".into_url().unwrap();\n        let source_id = SourceId::for_alt_registry(&url, \"alt\").unwrap();\n        assert_eq!(gen_hash(source_id), 5159702466575482972);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"135d23074253cb78\");\n\n        let url = \"file:///tmp/ws/crate\".into_url().unwrap();\n        let source_id = SourceId::for_git(&url, GitReference::DefaultBranch).unwrap();\n        assert_eq!(gen_hash(source_id), 15332537265078583985);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"73a808694abda756\");\n\n        let path = Path::new(\"/tmp/ws/crate\");\n\n        let source_id = SourceId::for_local_registry(path).unwrap();\n        assert_eq!(gen_hash(source_id), 18446533307730842837);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"52a84cc73f6fd48b\");\n\n        let source_id = SourceId::for_path(path).unwrap();\n        assert_eq!(gen_hash(source_id), 8764714075439899829);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"e1ddd48578620fc1\");\n\n        let source_id = SourceId::for_directory(path).unwrap();\n        assert_eq!(gen_hash(source_id), 17459999773908528552);\n        assert_eq!(crate::util::hex::short_hash(&source_id), \"6568fe2c2fab5bfe\");\n    }\n\n    #[test]\n    fn serde_roundtrip() {\n        let url = \"sparse+https://my-crates.io/\".into_url().unwrap();\n        let source_id = SourceId::for_registry(&url).unwrap();\n        let formatted = format!(\"{}\", source_id.as_url());\n        let deserialized = SourceId::from_url(&formatted).unwrap();\n        assert_eq!(formatted, \"sparse+https://my-crates.io/\");\n        assert_eq!(source_id, deserialized);\n    }\n\n    #[test]\n    fn gitrefs_roundtrip() {\n        let base = \"https://host/path\".into_url().unwrap();\n        let branch = GitReference::Branch(\"*-._+20%30 Z/z#foo=bar&zap[]?to\\\\()'\\\"\".to_string());\n        let s1 = SourceId::for_git(&base, branch).unwrap();\n        let ser1 = format!(\"{}\", s1.as_encoded_url());\n        let s2 = SourceId::from_url(&ser1).expect(\"Failed to deserialize\");\n        let ser2 = format!(\"{}\", s2.as_encoded_url());\n        // Serializing twice should yield the same result\n        assert_eq!(ser1, ser2, \"Serialized forms don't match\");\n        // SourceId serializing the same should have the same semantics\n        // This used to not be the case (# was ambiguous)\n        assert_eq!(s1, s2, \"SourceId doesn't round-trip\");\n        // Freeze the format to match an x-www-form-urlencoded query string\n        // https://url.spec.whatwg.org/#application/x-www-form-urlencoded\n        assert_eq!(\n            ser1,\n            \"git+https://host/path?branch=*-._%2B20%2530+Z%2Fz%23foo%3Dbar%26zap%5B%5D%3Fto%5C%28%29%27%22\"\n        );\n    }\n}\n\n/// Check if `url` equals to the overridden crates.io URL.\n// ALLOWED: For testing Cargo itself only.\n#[allow(clippy::disallowed_methods)]\nfn is_overridden_crates_io_url(url: &str) -> bool {\n    std::env::var(\"__CARGO_TEST_CRATES_IO_URL_DO_NOT_USE_THIS\").map_or(false, |v| v == url)\n}\n\n// Path: src/cargo/core/compiler/unit.rs\n//! Types and impls for [`Unit`].\n\nuse crate::core::compiler::unit_dependencies::IsArtifact;\nuse crate::core::compiler::{CompileKind, CompileMode, CompileTarget, CrateType};\nuse crate::core::manifest::{Target, TargetKind};\nuse crate::core::profiles::Profile;\nuse crate::core::Package;\nuse crate::util::hex::short_hash;\nuse crate::util::interning::InternedString;\nuse crate::util::GlobalContext;\nuse std::cell::RefCell;\nuse std::collections::HashSet;\nuse std::fmt;\nuse std::hash::{Hash, Hasher};\nuse std::ops::Deref;\nuse std::rc::Rc;\n\n/// All information needed to define a unit.\n///\n/// A unit is an object that has enough information so that cargo knows how to build it.\n/// For example, if your package has dependencies, then every dependency will be built as a library\n/// unit. If your package is a library, then it will be built as a library unit as well, or if it\n/// is a binary with `main.rs`, then a binary will be output. There are also separate unit types\n/// for `test`ing and `check`ing, amongst others.\n///\n/// The unit also holds information about all possible metadata about the package in `pkg`.\n///\n/// A unit needs to know extra information in addition to the type and root source file. For\n/// example, it needs to know the target architecture (OS, chip arch etc.) and it needs to know\n/// whether you want a debug or release build. There is enough information in this struct to figure\n/// all that out.\n#[derive(Clone, PartialOrd, Ord)]\npub struct Unit {\n    inner: Rc<UnitInner>,\n}\n\n/// Internal fields of `Unit` which `Unit` will dereference to.\n#[derive(Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]\npub struct UnitInner {\n    /// Information about available targets, which files to include/exclude, etc. Basically stuff in\n    /// `Cargo.toml`.\n    pub pkg: Package,\n    /// Information about the specific target to build, out of the possible targets in `pkg`. Not\n    /// to be confused with *target-triple* (or *target architecture* ...), the target arch for a\n    /// build.\n    pub target: Target,\n    /// The profile contains information about *how* the build should be run, including debug\n    /// level, etc.\n    pub profile: Profile,\n    /// Whether this compilation unit is for the host or target architecture.\n    ///\n    /// For example, when\n    /// cross compiling and using a custom build script, the build script needs to be compiled for\n    /// the host architecture so the host rustc can use it (when compiling to the target\n    /// architecture).\n    pub kind: CompileKind,\n    /// The \"mode\" this unit is being compiled for. See [`CompileMode`] for more details.\n    pub mode: CompileMode,\n    /// The `cfg` features to enable for this unit.\n    /// This must be sorted.\n    pub features: Vec<InternedString>,\n    // if `true`, the dependency is an artifact dependency, requiring special handling when\n    // calculating output directories, linkage and environment variables provided to builds.\n    pub artifact: IsArtifact,\n    /// Whether this is a standard library unit.\n    pub is_std: bool,\n    /// A hash of all dependencies of this unit.\n    ///\n    /// This is used to keep the `Unit` unique in the situation where two\n    /// otherwise identical units need to link to different dependencies. This\n    /// can happen, for example, when there are shared dependencies that need\n    /// to be built with different features between normal and build\n    /// dependencies. See `rebuild_unit_graph_shared` for more on why this is\n    /// done.\n    ///\n    /// This value initially starts as 0, and then is filled in via a\n    /// second-pass after all the unit dependencies have been computed.\n    pub dep_hash: u64,\n\n    /// This is used for target-dependent feature resolution and is copied from\n    /// [`FeaturesFor::ArtifactDep`], if the enum matches the variant.\n    ///\n    /// [`FeaturesFor::ArtifactDep`]: crate::core::resolver::features::FeaturesFor::ArtifactDep\n    pub artifact_target_for_features: Option<CompileTarget>,\n}\n\nimpl UnitInner {\n    /// Returns whether compilation of this unit requires all upstream artifacts\n    /// to be available.\n    ///\n    /// This effectively means that this unit is a synchronization point (if the\n    /// return value is `true`) that all previously pipelined units need to\n    /// finish in their entirety before this one is started.\n    pub fn requires_upstream_objects(&self) -> bool {\n        self.mode.is_any_test() || self.target.kind().requires_upstream_objects()\n    }\n\n    /// Returns whether or not this is a \"local\" package.\n    ///\n    /// A \"local\" package is one that the user can likely edit, or otherwise\n    /// wants warnings, etc.\n    pub fn is_local(&self) -> bool {\n        self.pkg.package_id().source_id().is_path() && !self.is_std\n    }\n\n    /// Returns whether or not warnings should be displayed for this unit.\n    pub fn show_warnings(&self, gctx: &GlobalContext) -> bool {\n        self.is_local() || gctx.extra_verbose()\n    }\n}\n\nimpl Unit {\n    /// Gets the unique key for [`-Zbuild-plan`].\n    ///\n    /// [`-Zbuild-plan`]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-plan\n    pub fn buildkey(&self) -> String {\n        format!(\"{}-{}\", self.pkg.name(), short_hash(self))\n    }\n}\n\n// Just hash the pointer for fast hashing\nimpl Hash for Unit {\n    fn hash<H: Hasher>(&self, hasher: &mut H) {\n        std::ptr::hash(&*self.inner, hasher)\n    }\n}\n\n// Just equate the pointer since these are interned\nimpl PartialEq for Unit {\n    fn eq(&self, other: &Unit) -> bool {\n        std::ptr::eq(&*self.inner, &*other.inner)\n    }\n}\n\nimpl Eq for Unit {}\n\nimpl Deref for Unit {\n    type Target = UnitInner;\n\n    fn deref(&self) -> &UnitInner {\n        &*self.inner\n    }\n}\n\nimpl fmt::Debug for Unit {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"Unit\")\n            .field(\"pkg\", &self.pkg)\n            .field(\"target\", &self.target)\n            .field(\"profile\", &self.profile)\n            .field(\"kind\", &self.kind)\n            .field(\"mode\", &self.mode)\n            .field(\"features\", &self.features)\n            .field(\"artifact\", &self.artifact.is_true())\n            .field(\n                \"artifact_target_for_features\",\n                &self.artifact_target_for_features,\n            )\n            .field(\"is_std\", &self.is_std)\n            .field(\"dep_hash\", &self.dep_hash)\n            .finish()\n    }\n}\n\n/// A small structure used to \"intern\" `Unit` values.\n///\n/// A `Unit` is just a thin pointer to an internal `UnitInner`. This is done to\n/// ensure that `Unit` itself is quite small as well as enabling a very\n/// efficient hash/equality implementation for `Unit`. All units are\n/// manufactured through an interner which guarantees that each equivalent value\n/// is only produced once.\npub struct UnitInterner {\n    state: RefCell<InternerState>,\n}\n\nstruct InternerState {\n    cache: HashSet<Rc<UnitInner>>,\n}\n\nimpl UnitInterner {\n    /// Creates a new blank interner\n    pub fn new() -> UnitInterner {\n        UnitInterner {\n            state: RefCell::new(InternerState {\n                cache: HashSet::new(),\n            }),\n        }\n    }\n\n    /// Creates a new `unit` from its components. The returned `Unit`'s fields\n    /// will all be equivalent to the provided arguments, although they may not\n    /// be the exact same instance.\n    pub fn intern(\n        &self,\n        pkg: &Package,\n        target: &Target,\n        profile: Profile,\n        kind: CompileKind,\n        mode: CompileMode,\n        features: Vec<InternedString>,\n        is_std: bool,\n        dep_hash: u64,\n        artifact: IsArtifact,\n        artifact_target_for_features: Option<CompileTarget>,\n    ) -> Unit {\n        let target = match (is_std, target.kind()) {\n            // This is a horrible hack to support build-std. `libstd` declares\n            // itself with both rlib and dylib. We don't want the dylib for a\n            // few reasons:\n            //\n            // - dylibs don't have a hash in the filename. If you do something\n            //   (like switch rustc versions), it will stomp on the dylib\n            //   file, invalidating the entire cache (because std is a dep of\n            //   everything).\n            // - We don't want to publicize the presence of dylib for the\n            //   standard library.\n            //\n            // At some point in the future, it would be nice to have a\n            // first-class way of overriding or specifying crate-types.\n            (true, TargetKind::Lib(crate_types)) if crate_types.contains(&CrateType::Dylib) => {\n                let mut new_target = Target::clone(target);\n                new_target.set_kind(TargetKind::Lib(vec![CrateType::Rlib]));\n                new_target\n            }\n            _ => target.clone(),\n        };\n        let inner = self.intern_inner(&UnitInner {\n            pkg: pkg.clone(),\n            target,\n            profile,\n            kind,\n            mode,\n            features,\n            is_std,\n            dep_hash,\n            artifact,\n            artifact_target_for_features,\n        });\n        Unit { inner }\n    }\n\n    fn intern_inner(&self, item: &UnitInner) -> Rc<UnitInner> {\n        let mut me = self.state.borrow_mut();\n        if let Some(item) = me.cache.get(item) {\n            return item.clone();\n        }\n        let item = Rc::new(item.clone());\n        me.cache.insert(item.clone());\n        item\n    }\n}\n\n// Path: src/cargo/util/hostname.rs\n// Copied from https://github.com/BurntSushi/ripgrep/blob/7099e174acbcbd940f57e4ab4913fee4040c826e/crates/cli/src/hostname.rs\n\nuse std::{ffi::OsString, io};\n\n/// Returns the hostname of the current system.\n///\n/// It is unusual, although technically possible, for this routine to return\n/// an error. It is difficult to list out the error conditions, but one such\n/// possibility is platform support.\n///\n/// # Platform specific behavior\n///\n/// On Unix, this returns the result of the `gethostname` function from the\n/// `libc` linked into the program.\npub fn hostname() -> io::Result<OsString> {\n    #[cfg(unix)]\n    {\n        gethostname()\n    }\n    #[cfg(not(unix))]\n    {\n        Err(io::Error::new(\n            io::ErrorKind::Other,\n            \"hostname could not be found on unsupported platform\",\n        ))\n    }\n}\n\n#[cfg(unix)]\nfn gethostname() -> io::Result<OsString> {\n    use std::os::unix::ffi::OsStringExt;\n\n    // SAFETY: There don't appear to be any safety requirements for calling\n    // sysconf.\n    let limit = unsafe { libc::sysconf(libc::_SC_HOST_NAME_MAX) };\n    if limit == -1 {\n        // It is in theory possible for sysconf to return -1 for a limit but\n        // *not* set errno, in which case, io::Error::last_os_error is\n        // indeterminate. But untangling that is super annoying because std\n        // doesn't expose any unix-specific APIs for inspecting the errno. (We\n        // could do it ourselves, but it just doesn't seem worth doing?)\n        return Err(io::Error::last_os_error());\n    }\n    let Ok(maxlen) = usize::try_from(limit) else {\n        let msg = format!(\"host name max limit ({}) overflowed usize\", limit);\n        return Err(io::Error::new(io::ErrorKind::Other, msg));\n    };\n    // maxlen here includes the NUL terminator.\n    let mut buf = vec![0; maxlen];\n    // SAFETY: The pointer we give is valid as it is derived directly from a\n    // Vec. Similarly, `maxlen` is the length of our Vec, and is thus valid\n    // to write to.\n    let rc = unsafe { libc::gethostname(buf.as_mut_ptr().cast::<libc::c_char>(), maxlen) };\n    if rc == -1 {\n        return Err(io::Error::last_os_error());\n    }\n    // POSIX says that if the hostname is bigger than `maxlen`, then it may\n    // write a truncate name back that is not necessarily NUL terminated (wtf,\n    // lol). So if we can't find a NUL terminator, then just give up.\n    let Some(zeropos) = buf.iter().position(|&b| b == 0) else {\n        let msg = \"could not find NUL terminator in hostname\";\n        return Err(io::Error::new(io::ErrorKind::Other, msg));\n    };\n    buf.truncate(zeropos);\n    buf.shrink_to_fit();\n    Ok(OsString::from_vec(buf))\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn print_hostname() {\n        println!(\"{:?}\", hostname());\n    }\n}\n\n// Path: src/cargo/core/shell.rs\nuse std::fmt;\nuse std::io::prelude::*;\nuse std::io::IsTerminal;\n\nuse anstream::AutoStream;\nuse anstyle::Style;\n\nuse crate::util::errors::CargoResult;\nuse crate::util::hostname;\nuse crate::util::style::*;\n\n/// An abstraction around console output that remembers preferences for output\n/// verbosity and color.\npub struct Shell {\n    /// Wrapper around stdout/stderr. This helps with supporting sending\n    /// output to a memory buffer which is useful for tests.\n    output: ShellOut,\n    /// How verbose messages should be.\n    verbosity: Verbosity,\n    /// Flag that indicates the current line needs to be cleared before\n    /// printing. Used when a progress bar is currently displayed.\n    needs_clear: bool,\n    hostname: Option<String>,\n}\n\nimpl fmt::Debug for Shell {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self.output {\n            ShellOut::Write(_) => f\n                .debug_struct(\"Shell\")\n                .field(\"verbosity\", &self.verbosity)\n                .finish(),\n            ShellOut::Stream { color_choice, .. } => f\n                .debug_struct(\"Shell\")\n                .field(\"verbosity\", &self.verbosity)\n                .field(\"color_choice\", &color_choice)\n                .finish(),\n        }\n    }\n}\n\nimpl Shell {\n    /// Creates a new shell (color choice and verbosity), defaulting to 'auto' color and verbose\n    /// output.\n    pub fn new() -> Shell {\n        let auto_clr = ColorChoice::CargoAuto;\n        let stdout_choice = auto_clr.to_anstream_color_choice();\n        let stderr_choice = auto_clr.to_anstream_color_choice();\n        Shell {\n            output: ShellOut::Stream {\n                stdout: AutoStream::new(std::io::stdout(), stdout_choice),\n                stderr: AutoStream::new(std::io::stderr(), stderr_choice),\n                color_choice: auto_clr,\n                hyperlinks: supports_hyperlinks(),\n                stderr_tty: std::io::stderr().is_terminal(),\n                stdout_unicode: supports_unicode(&std::io::stdout()),\n                stderr_unicode: supports_unicode(&std::io::stderr()),\n            },\n            verbosity: Verbosity::Verbose,\n            needs_clear: false,\n            hostname: None,\n        }\n    }\n\n    /// Creates a shell from a plain writable object, with no color, and max verbosity.\n    pub fn from_write(out: Box<dyn Write>) -> Shell {\n        Shell {\n            output: ShellOut::Write(AutoStream::never(out)), // strip all formatting on write\n            verbosity: Verbosity::Verbose,\n            needs_clear: false,\n            hostname: None,\n        }\n    }\n\n    /// Prints a message, where the status will have `color` color, and can be justified. The\n    /// messages follows without color.\n    fn print(\n        &mut self,\n        status: &dyn fmt::Display,\n        message: Option<&dyn fmt::Display>,\n        color: &Style,\n        justified: bool,\n    ) -> CargoResult<()> {\n        match self.verbosity {\n            Verbosity::Quiet => Ok(()),\n            _ => {\n                if self.needs_clear {\n                    self.err_erase_line();\n                }\n                self.output\n                    .message_stderr(status, message, color, justified)\n            }\n        }\n    }\n\n    /// Sets whether the next print should clear the current line.\n    pub fn set_needs_clear(&mut self, needs_clear: bool) {\n        self.needs_clear = needs_clear;\n    }\n\n    /// Returns `true` if the `needs_clear` flag is unset.\n    pub fn is_cleared(&self) -> bool {\n        !self.needs_clear\n    }\n\n    /// Returns the width of the terminal in spaces, if any.\n    pub fn err_width(&self) -> TtyWidth {\n        match self.output {\n            ShellOut::Stream {\n                stderr_tty: true, ..\n            } => imp::stderr_width(),\n            _ => TtyWidth::NoTty,\n        }\n    }\n\n    /// Returns `true` if stderr is a tty.\n    pub fn is_err_tty(&self) -> bool {\n        match self.output {\n            ShellOut::Stream { stderr_tty, .. } => stderr_tty,\n            _ => false,\n        }\n    }\n\n    /// Gets a reference to the underlying stdout writer.\n    pub fn out(&mut self) -> &mut dyn Write {\n        if self.needs_clear {\n            self.err_erase_line();\n        }\n        self.output.stdout()\n    }\n\n    /// Gets a reference to the underlying stderr writer.\n    pub fn err(&mut self) -> &mut dyn Write {\n        if self.needs_clear {\n            self.err_erase_line();\n        }\n        self.output.stderr()\n    }\n\n    /// Erase from cursor to end of line.\n    pub fn err_erase_line(&mut self) {\n        if self.err_supports_color() {\n            imp::err_erase_line(self);\n            self.needs_clear = false;\n        }\n    }\n\n    /// Shortcut to right-align and color green a status message.\n    pub fn status<T, U>(&mut self, status: T, message: U) -> CargoResult<()>\n    where\n        T: fmt::Display,\n        U: fmt::Display,\n    {\n        self.print(&status, Some(&message), &HEADER, true)\n    }\n\n    pub fn status_header<T>(&mut self, status: T) -> CargoResult<()>\n    where\n        T: fmt::Display,\n    {\n        self.print(&status, None, &NOTE, true)\n    }\n\n    /// Shortcut to right-align a status message.\n    pub fn status_with_color<T, U>(\n        &mut self,\n        status: T,\n        message: U,\n        color: &Style,\n    ) -> CargoResult<()>\n    where\n        T: fmt::Display,\n        U: fmt::Display,\n    {\n        self.print(&status, Some(&message), color, true)\n    }\n\n    /// Runs the callback only if we are in verbose mode.\n    pub fn verbose<F>(&mut self, mut callback: F) -> CargoResult<()>\n    where\n        F: FnMut(&mut Shell) -> CargoResult<()>,\n    {\n        match self.verbosity {\n            Verbosity::Verbose => callback(self),\n            _ => Ok(()),\n        }\n    }\n\n    /// Runs the callback if we are not in verbose mode.\n    pub fn concise<F>(&mut self, mut callback: F) -> CargoResult<()>\n    where\n        F: FnMut(&mut Shell) -> CargoResult<()>,\n    {\n        match self.verbosity {\n            Verbosity::Verbose => Ok(()),\n            _ => callback(self),\n        }\n    }\n\n    /// Prints a red 'error' message.\n    pub fn error<T: fmt::Display>(&mut self, message: T) -> CargoResult<()> {\n        if self.needs_clear {\n            self.err_erase_line();\n        }\n        self.output\n            .message_stderr(&\"error\", Some(&message), &ERROR, false)\n    }\n\n    /// Prints an amber 'warning' message.\n    pub fn warn<T: fmt::Display>(&mut self, message: T) -> CargoResult<()> {\n        match self.verbosity {\n            Verbosity::Quiet => Ok(()),\n            _ => self.print(&\"warning\", Some(&message), &WARN, false),\n        }\n    }\n\n    /// Prints a cyan 'note' message.\n    pub fn note<T: fmt::Display>(&mut self, message: T) -> CargoResult<()> {\n        self.print(&\"note\", Some(&message), &NOTE, false)\n    }\n\n    /// Updates the verbosity of the shell.\n    pub fn set_verbosity(&mut self, verbosity: Verbosity) {\n        self.verbosity = verbosity;\n    }\n\n    /// Gets the verbosity of the shell.\n    pub fn verbosity(&self) -> Verbosity {\n        self.verbosity\n    }\n\n    /// Updates the color choice (always, never, or auto) from a string..\n    pub fn set_color_choice(&mut self, color: Option<&str>) -> CargoResult<()> {\n        if let ShellOut::Stream {\n            stdout,\n            stderr,\n            color_choice,\n            ..\n        } = &mut self.output\n        {\n            let cfg = color\n                .map(|c| c.parse())\n                .transpose()?\n                .unwrap_or(ColorChoice::CargoAuto);\n            *color_choice = cfg;\n            let stdout_choice = cfg.to_anstream_color_choice();\n            let stderr_choice = cfg.to_anstream_color_choice();\n            *stdout = AutoStream::new(std::io::stdout(), stdout_choice);\n            *stderr = AutoStream::new(std::io::stderr(), stderr_choice);\n        }\n        Ok(())\n    }\n\n    pub fn set_unicode(&mut self, yes: bool) -> CargoResult<()> {\n        if let ShellOut::Stream {\n            stdout_unicode,\n            stderr_unicode,\n            ..\n        } = &mut self.output\n        {\n            *stdout_unicode = yes;\n            *stderr_unicode = yes;\n        }\n        Ok(())\n    }\n\n    pub fn set_hyperlinks(&mut self, yes: bool) -> CargoResult<()> {\n        if let ShellOut::Stream { hyperlinks, .. } = &mut self.output {\n            *hyperlinks = yes;\n        }\n        Ok(())\n    }\n\n    pub fn out_unicode(&self) -> bool {\n        match &self.output {\n            ShellOut::Write(_) => true,\n            ShellOut::Stream { stdout_unicode, .. } => *stdout_unicode,\n        }\n    }\n\n    pub fn err_unicode(&self) -> bool {\n        match &self.output {\n            ShellOut::Write(_) => true,\n            ShellOut::Stream { stderr_unicode, .. } => *stderr_unicode,\n        }\n    }\n\n    /// Gets the current color choice.\n    ///\n    /// If we are not using a color stream, this will always return `Never`, even if the color\n    /// choice has been set to something else.\n    pub fn color_choice(&self) -> ColorChoice {\n        match self.output {\n            ShellOut::Stream { color_choice, .. } => color_choice,\n            ShellOut::Write(_) => ColorChoice::Never,\n        }\n    }\n\n    /// Whether the shell supports color.\n    pub fn err_supports_color(&self) -> bool {\n        match &self.output {\n            ShellOut::Write(_) => false,\n            ShellOut::Stream { stderr, .. } => supports_color(stderr.current_choice()),\n        }\n    }\n\n    pub fn out_supports_color(&self) -> bool {\n        match &self.output {\n            ShellOut::Write(_) => false,\n            ShellOut::Stream { stdout, .. } => supports_color(stdout.current_choice()),\n        }\n    }\n\n    pub fn out_hyperlink<D: fmt::Display>(&self, url: D) -> Hyperlink<D> {\n        let supports_hyperlinks = match &self.output {\n            ShellOut::Write(_) => false,\n            ShellOut::Stream {\n                stdout, hyperlinks, ..\n            } => stdout.current_choice() == anstream::ColorChoice::AlwaysAnsi && *hyperlinks,\n        };\n        Hyperlink {\n            url: supports_hyperlinks.then_some(url),\n        }\n    }\n\n    pub fn err_hyperlink<D: fmt::Display>(&self, url: D) -> Hyperlink<D> {\n        let supports_hyperlinks = match &self.output {\n            ShellOut::Write(_) => false,\n            ShellOut::Stream {\n                stderr, hyperlinks, ..\n            } => stderr.current_choice() == anstream::ColorChoice::AlwaysAnsi && *hyperlinks,\n        };\n        if supports_hyperlinks {\n            Hyperlink { url: Some(url) }\n        } else {\n            Hyperlink { url: None }\n        }\n    }\n\n    pub fn out_file_hyperlink(&mut self, path: &std::path::Path) -> Hyperlink<url::Url> {\n        let url = self.file_hyperlink(path);\n        url.map(|u| self.out_hyperlink(u)).unwrap_or_default()\n    }\n\n    pub fn err_file_hyperlink(&mut self, path: &std::path::Path) -> Hyperlink<url::Url> {\n        let url = self.file_hyperlink(path);\n        url.map(|u| self.err_hyperlink(u)).unwrap_or_default()\n    }\n\n    fn file_hyperlink(&mut self, path: &std::path::Path) -> Option<url::Url> {\n        let mut url = url::Url::from_file_path(path).ok()?;\n        // Do a best-effort of setting the host in the URL to avoid issues with opening a link\n        // scoped to the computer you've SSHed into\n        let hostname = if cfg!(windows) {\n            // Not supported correctly on windows\n            None\n        } else {\n            if let Some(hostname) = self.hostname.as_deref() {\n                Some(hostname)\n            } else {\n                self.hostname = hostname().ok().and_then(|h| h.into_string().ok());\n                self.hostname.as_deref()\n            }\n        };\n        let _ = url.set_host(hostname);\n        Some(url)\n    }\n\n    /// Prints a message to stderr and translates ANSI escape code into console colors.\n    pub fn print_ansi_stderr(&mut self, message: &[u8]) -> CargoResult<()> {\n        if self.needs_clear {\n            self.err_erase_line();\n        }\n        self.err().write_all(message)?;\n        Ok(())\n    }\n\n    /// Prints a message to stdout and translates ANSI escape code into console colors.\n    pub fn print_ansi_stdout(&mut self, message: &[u8]) -> CargoResult<()> {\n        if self.needs_clear {\n            self.err_erase_line();\n        }\n        self.out().write_all(message)?;\n        Ok(())\n    }\n\n    pub fn print_json<T: serde::ser::Serialize>(&mut self, obj: &T) -> CargoResult<()> {\n        // Path may fail to serialize to JSON ...\n        let encoded = serde_json::to_string(&obj)?;\n        // ... but don't fail due to a closed pipe.\n        drop(writeln!(self.out(), \"{}\", encoded));\n        Ok(())\n    }\n}\n\nimpl Default for Shell {\n    fn default() -> Self {\n        Self::new()\n    }\n}\n\n/// A `Write`able object, either with or without color support\nenum ShellOut {\n    /// A plain write object without color support\n    Write(AutoStream<Box<dyn Write>>),\n    /// Color-enabled stdio, with information on whether color should be used\n    Stream {\n        stdout: AutoStream<std::io::Stdout>,\n        stderr: AutoStream<std::io::Stderr>,\n        stderr_tty: bool,\n        color_choice: ColorChoice,\n        hyperlinks: bool,\n        stdout_unicode: bool,\n        stderr_unicode: bool,\n    },\n}\n\nimpl ShellOut {\n    /// Prints out a message with a status. The status comes first, and is bold plus the given\n    /// color. The status can be justified, in which case the max width that will right align is\n    /// 12 chars.\n    fn message_stderr(\n        &mut self,\n        status: &dyn fmt::Display,\n        message: Option<&dyn fmt::Display>,\n        style: &Style,\n        justified: bool,\n    ) -> CargoResult<()> {\n        let bold = anstyle::Style::new() | anstyle::Effects::BOLD;\n\n        let mut buffer = Vec::new();\n        if justified {\n            write!(&mut buffer, \"{style}{status:>12}{style:#}\")?;\n        } else {\n            write!(&mut buffer, \"{style}{status}{style:#}{bold}:{bold:#}\")?;\n        }\n        match message {\n            Some(message) => writeln!(buffer, \" {message}\")?,\n            None => write!(buffer, \" \")?,\n        }\n        self.stderr().write_all(&buffer)?;\n        Ok(())\n    }\n\n    /// Gets stdout as a `io::Write`.\n    fn stdout(&mut self) -> &mut dyn Write {\n        match self {\n            ShellOut::Stream { stdout, .. } => stdout,\n            ShellOut::Write(w) => w,\n        }\n    }\n\n    /// Gets stderr as a `io::Write`.\n    fn stderr(&mut self) -> &mut dyn Write {\n        match self {\n            ShellOut::Stream { stderr, .. } => stderr,\n            ShellOut::Write(w) => w,\n        }\n    }\n}\n\npub enum TtyWidth {\n    NoTty,\n    Known(usize),\n    Guess(usize),\n}\n\nimpl TtyWidth {\n    /// Returns the width of the terminal to use for diagnostics (which is\n    /// relayed to rustc via `--diagnostic-width`).\n    pub fn diagnostic_terminal_width(&self) -> Option<usize> {\n        // ALLOWED: For testing cargo itself only.\n        #[allow(clippy::disallowed_methods)]\n        if let Ok(width) = std::env::var(\"__CARGO_TEST_TTY_WIDTH_DO_NOT_USE_THIS\") {\n            return Some(width.parse().unwrap());\n        }\n        match *self {\n            TtyWidth::NoTty | TtyWidth::Guess(_) => None,\n            TtyWidth::Known(width) => Some(width),\n        }\n    }\n\n    /// Returns the width used by progress bars for the tty.\n    pub fn progress_max_width(&self) -> Option<usize> {\n        match *self {\n            TtyWidth::NoTty => None,\n            TtyWidth::Known(width) | TtyWidth::Guess(width) => Some(width),\n        }\n    }\n}\n\n/// The requested verbosity of output.\n#[derive(Debug, Clone, Copy, PartialEq)]\npub enum Verbosity {\n    Verbose,\n    Normal,\n    Quiet,\n}\n\n/// Whether messages should use color output\n#[derive(Debug, PartialEq, Clone, Copy)]\npub enum ColorChoice {\n    /// Force color output\n    Always,\n    /// Force disable color output\n    Never,\n    /// Intelligently guess whether to use color output\n    CargoAuto,\n}\n\nimpl ColorChoice {\n    /// Converts our color choice to anstream's version.\n    fn to_anstream_color_choice(self) -> anstream::ColorChoice {\n        match self {\n            ColorChoice::Always => anstream::ColorChoice::Always,\n            ColorChoice::Never => anstream::ColorChoice::Never,\n            ColorChoice::CargoAuto => anstream::ColorChoice::Auto,\n        }\n    }\n}\n\nimpl std::str::FromStr for ColorChoice {\n    type Err = anyhow::Error;\n    fn from_str(color: &str) -> Result<Self, Self::Err> {\n        let cfg = match color {\n            \"always\" => ColorChoice::Always,\n            \"never\" => ColorChoice::Never,\n\n            \"auto\" => ColorChoice::CargoAuto,\n\n            arg => anyhow::bail!(\n                \"argument for --color must be auto, always, or \\\n                     never, but found `{}`\",\n                arg\n            ),\n        };\n        Ok(cfg)\n    }\n}\n\nfn supports_color(choice: anstream::ColorChoice) -> bool {\n    match choice {\n        anstream::ColorChoice::Always\n        | anstream::ColorChoice::AlwaysAnsi\n        | anstream::ColorChoice::Auto => true,\n        anstream::ColorChoice::Never => false,\n    }\n}\n\nfn supports_unicode(stream: &dyn IsTerminal) -> bool {\n    !stream.is_terminal() || supports_unicode::supports_unicode()\n}\n\nfn supports_hyperlinks() -> bool {\n    #[allow(clippy::disallowed_methods)] // We are reading the state of the system, not config\n    if std::env::var_os(\"TERM_PROGRAM\").as_deref() == Some(std::ffi::OsStr::new(\"iTerm.app\")) {\n        // Override `supports_hyperlinks` as we have an unknown incompatibility with iTerm2\n        return false;\n    }\n\n    supports_hyperlinks::supports_hyperlinks()\n}\n\npub struct Hyperlink<D: fmt::Display> {\n    url: Option<D>,\n}\n\nimpl<D: fmt::Display> Default for Hyperlink<D> {\n    fn default() -> Self {\n        Self { url: None }\n    }\n}\n\nimpl<D: fmt::Display> fmt::Display for Hyperlink<D> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let Some(url) = self.url.as_ref() else {\n            return Ok(());\n        };\n        if f.alternate() {\n            write!(f, \"\\x1B]8;;\\x1B\\\\\")\n        } else {\n            write!(f, \"\\x1B]8;;{url}\\x1B\\\\\")\n        }\n    }\n}\n\n#[cfg(unix)]\nmod imp {\n    use super::{Shell, TtyWidth};\n    use std::mem;\n\n    pub fn stderr_width() -> TtyWidth {\n        unsafe {\n            let mut winsize: libc::winsize = mem::zeroed();\n            // The .into() here is needed for FreeBSD which defines TIOCGWINSZ\n            // as c_uint but ioctl wants c_ulong.\n            if libc::ioctl(libc::STDERR_FILENO, libc::TIOCGWINSZ.into(), &mut winsize) < 0 {\n                return TtyWidth::NoTty;\n            }\n            if winsize.ws_col > 0 {\n                TtyWidth::Known(winsize.ws_col as usize)\n            } else {\n                TtyWidth::NoTty\n            }\n        }\n    }\n\n    pub fn err_erase_line(shell: &mut Shell) {\n        // This is the \"EL - Erase in Line\" sequence. It clears from the cursor\n        // to the end of line.\n        // https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences\n        let _ = shell.output.stderr().write_all(b\"\\x1B[K\");\n    }\n}\n\n#[cfg(windows)]\nmod imp {\n    use std::{cmp, mem, ptr};\n\n    use windows_sys::core::PCSTR;\n    use windows_sys::Win32::Foundation::CloseHandle;\n    use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE;\n    use windows_sys::Win32::Foundation::{GENERIC_READ, GENERIC_WRITE};\n    use windows_sys::Win32::Storage::FileSystem::{\n        CreateFileA, FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING,\n    };\n    use windows_sys::Win32::System::Console::{\n        GetConsoleScreenBufferInfo, GetStdHandle, CONSOLE_SCREEN_BUFFER_INFO, STD_ERROR_HANDLE,\n    };\n\n    pub(super) use super::{default_err_erase_line as err_erase_line, TtyWidth};\n\n    pub fn stderr_width() -> TtyWidth {\n        unsafe {\n            let stdout = GetStdHandle(STD_ERROR_HANDLE);\n            let mut csbi: CONSOLE_SCREEN_BUFFER_INFO = mem::zeroed();\n            if GetConsoleScreenBufferInfo(stdout, &mut csbi) != 0 {\n                return TtyWidth::Known((csbi.srWindow.Right - csbi.srWindow.Left) as usize);\n            }\n\n            // On mintty/msys/cygwin based terminals, the above fails with\n            // INVALID_HANDLE_VALUE. Use an alternate method which works\n            // in that case as well.\n            let h = CreateFileA(\n                \"CONOUT$\\0\".as_ptr() as PCSTR,\n                GENERIC_READ | GENERIC_WRITE,\n                FILE_SHARE_READ | FILE_SHARE_WRITE,\n                ptr::null_mut(),\n                OPEN_EXISTING,\n                0,\n                0,\n            );\n            if h == INVALID_HANDLE_VALUE {\n                return TtyWidth::NoTty;\n            }\n\n            let mut csbi: CONSOLE_SCREEN_BUFFER_INFO = mem::zeroed();\n            let rc = GetConsoleScreenBufferInfo(h, &mut csbi);\n            CloseHandle(h);\n            if rc != 0 {\n                let width = (csbi.srWindow.Right - csbi.srWindow.Left) as usize;\n                // Unfortunately cygwin/mintty does not set the size of the\n                // backing console to match the actual window size. This\n                // always reports a size of 80 or 120 (not sure what\n                // determines that). Use a conservative max of 60 which should\n                // work in most circumstances. ConEmu does some magic to\n                // resize the console correctly, but there's no reasonable way\n                // to detect which kind of terminal we are running in, or if\n                // GetConsoleScreenBufferInfo returns accurate information.\n                return TtyWidth::Guess(cmp::min(60, width));\n            }\n\n            TtyWidth::NoTty\n        }\n    }\n}\n\n#[cfg(windows)]\nfn default_err_erase_line(shell: &mut Shell) {\n    match imp::stderr_width() {\n        TtyWidth::Known(max_width) | TtyWidth::Guess(max_width) => {\n            let blank = \" \".repeat(max_width);\n            drop(write!(shell.output.stderr(), \"{}\\r\", blank));\n        }\n        _ => (),\n    }\n}\n\n// Path: src/cargo/sources/mod.rs\n//! The trait for sources of Cargo packages and its built-in implementations.\n//!\n//! A source is a provider that contains source files and metadata of packages.\n...\n", "needle_token_start": 4013, "needle_token_end": 4097, "code_context_ntokens": 16227, "output": ["```rust\n#[cfg(test)]\nmod tests {\n    use super::{GitReference, SourceId, SourceKind};\n    use crate::util::{GlobalContext, IntoUrl};\n\n    // This is a test that the hash of the `SourceId` for crates.io is a well-known\n    // value.\n    //\n    // Note that the hash value matches what the crates.io source id has hashed\n    // since long before Rust 1.30. We strive to keep this value the same across\n    // versions of Cargo because changing it means that users will need to\n    // redownload the index and all crates they use when using a new Cargo version.\n    //\n    // This isn't to say that this hash can *never* change, only that when changing\n    // this it should be explicitly done. If this hash changes accidentally and\n    // you're able to restore the hash to its original value, please do so!\n    // Otherwise please just leave a comment in your PR as to why the hash value is\n    // changing and why the old value can't be easily preserved.\n    //\n    // The hash value depends on endianness and bit-width, so we only run this test on\n    // little-endian 64-bit CPUs (such as x86-64 and ARM64) where it matches the\n    // well-known value.\n    #[test]\n    #[cfg(all(target_endian = \"little\", target_pointer_width = \"64\"))]\n    fn test_cratesio_hash() {\n        let gctx = GlobalContext::default().unwrap();\n        let crates_io = SourceId::crates_io(&gctx).unwrap();\n        assert_eq!(crate::util::hex::short_hash(&crates_io), \"1ecc6299db9ec823\");\n    }\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "rebuild_unit_graph_shared", "language": "rust", "path": "src/cargo/ops/cargo_compile/mod.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to optimize and adjust a graph of build units by sharing host dependencies where possible, and applying specific transformations to units based on the entire graph's context. This helps in managing dependencies that might conflict due to different required features or compile kinds, and ensures efficient compilation by reducing redundancy.\n\n2. **Input**: The function takes several parameters including an interner for units, the original graph of units, an array of root units, an array of units that need to be scraped, an optional compile kind to transform target dependencies to host dependencies, and target data containing information about the Rust compiler version.\n\n3. **Output**: It returns a tuple containing an updated list of root units, an updated list of units that need scraping, and the newly constructed unit graph which has been optimized and adjusted.\n\n4. **Procedure**: \n   - Initializes a new, empty graph and a memoization map to track already processed units.\n   - Iterates over the root units, transforming each by potentially sharing dependencies and adjusting compile kinds as specified, updating the new graph accordingly.\n   - Ensures that units which require scraping but were not attached as dependencies in the original graph traversal are included by fetching them from the memoization map.\n   - The result is a new graph where shared dependencies are merged safely, compile kinds are adjusted where applicable, and units are optimized for compilation without unnecessary redundancy or conflicts.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "mod packages;\n\npub use packages::Packages;\n\n/// Contains information about how a package should be compiled.\n///\n/// Note on distinction between `CompileOptions` and [`BuildConfig`]:\n/// `BuildConfig` contains values that need to be retained after\n/// [`BuildContext`] is created. The other fields are no longer necessary. Think\n/// of it as `CompileOptions` are high-level settings requested on the\n/// command-line, and `BuildConfig` are low-level settings for actually\n/// driving `rustc`.\n#[derive(Debug, Clone)]\npub struct CompileOptions {\n    /// Configuration information for a rustc build\n    pub build_config: BuildConfig,\n    /// Feature flags requested by the user.\n    pub cli_features: CliFeatures,\n    /// A set of packages to build.\n    pub spec: Packages,\n    /// Filter to apply to the root package to select which targets will be\n    /// built.\n    pub filter: CompileFilter,\n    /// Extra arguments to be passed to rustdoc (single target only)\n    pub target_rustdoc_args: Option<Vec<String>>,\n    /// The specified target will be compiled with all the available arguments,\n    /// note that this only accounts for the *final* invocation of rustc\n    pub target_rustc_args: Option<Vec<String>>,\n    /// Crate types to be passed to rustc (single target only)\n    pub target_rustc_crate_types: Option<Vec<String>>,\n    /// Whether the `--document-private-items` flags was specified and should\n    /// be forwarded to `rustdoc`.\n    pub rustdoc_document_private_items: bool,\n    /// Whether the build process should check the minimum Rust version\n    /// defined in the cargo metadata for a crate.\n    pub honor_rust_version: bool,\n}\n\nimpl CompileOptions {\n    pub fn new(gctx: &GlobalContext, mode: CompileMode) -> CargoResult<CompileOptions> {\n        let jobs = None;\n        let keep_going = false;\n        Ok(CompileOptions {\n            build_config: BuildConfig::new(gctx, jobs, keep_going, &[], mode)?,\n            cli_features: CliFeatures::new_all(false),\n            spec: ops::Packages::Packages(Vec::new()),\n            filter: CompileFilter::Default {\n                required_features_filterable: false,\n            },\n            target_rustdoc_args: None,\n            target_rustc_args: None,\n            target_rustc_crate_types: None,\n            rustdoc_document_private_items: false,\n            honor_rust_version: true,\n        })\n    }\n}\n\n/// Compiles!\n///\n/// This uses the [`DefaultExecutor`]. To use a custom [`Executor`], see [`compile_with_exec`].\npub fn compile<'a>(ws: &Workspace<'a>, options: &CompileOptions) -> CargoResult<Compilation<'a>> {\n    let exec: Arc<dyn Executor> = Arc::new(DefaultExecutor);\n    compile_with_exec(ws, options, &exec)\n}\n\n/// Like [`compile`] but allows specifying a custom [`Executor`]\n/// that will be able to intercept build calls and add custom logic.\n///\n/// [`compile`] uses [`DefaultExecutor`] which just passes calls through.\npub fn compile_with_exec<'a>(\n    ws: &Workspace<'a>,\n    options: &CompileOptions,\n    exec: &Arc<dyn Executor>,\n) -> CargoResult<Compilation<'a>> {\n    ws.emit_warnings()?;\n    compile_ws(ws, options, exec)\n}\n\n/// Like [`compile_with_exec`] but without warnings from manifest parsing.\n#[tracing::instrument(skip_all)]\npub fn compile_ws<'a>(\n    ws: &Workspace<'a>,\n    options: &CompileOptions,\n    exec: &Arc<dyn Executor>,\n) -> CargoResult<Compilation<'a>> {\n    let interner = UnitInterner::new();\n    let bcx = create_bcx(ws, options, &interner)?;\n    if options.build_config.unit_graph {\n        unit_graph::emit_serialized_unit_graph(&bcx.roots, &bcx.unit_graph, ws.gctx())?;\n        return Compilation::new(&bcx);\n    }\n    crate::core::gc::auto_gc(bcx.gctx);\n    let build_runner = BuildRunner::new(&bcx)?;\n    build_runner.compile(exec)\n}\n\n/// Executes `rustc --print <VALUE>`.\n///\n/// * `print_opt_value` is the VALUE passed through.\npub fn print<'a>(\n    ws: &Workspace<'a>,\n    options: &CompileOptions,\n    print_opt_value: &str,\n) -> CargoResult<()> {\n    let CompileOptions {\n        ref build_config,\n        ref target_rustc_args,\n        ..\n    } = *options;\n    let gctx = ws.gctx();\n    let rustc = gctx.load_global_rustc(Some(ws))?;\n    for (index, kind) in build_config.requested_kinds.iter().enumerate() {\n        if index != 0 {\n            drop_println!(gctx);\n        }\n        let target_info = TargetInfo::new(gctx, &build_config.requested_kinds, &rustc, *kind)?;\n        let mut process = rustc.process();\n        process.args(&target_info.rustflags);\n        if let Some(args) = target_rustc_args {\n            process.args(args);\n        }\n        if let CompileKind::Target(t) = kind {\n            process.arg(\"--target\").arg(t.rustc_target());\n        }\n        process.arg(\"--print\").arg(print_opt_value);\n        process.exec()?;\n    }\n    Ok(())\n}\n\n/// Prepares all required information for the actual compilation.\n///\n/// For how it works and what data it collects,\n/// please see the [module-level documentation](self).\n#[tracing::instrument(skip_all)]\npub fn create_bcx<'a, 'gctx>(\n    ws: &'a Workspace<'gctx>,\n    options: &'a CompileOptions,\n    interner: &'a UnitInterner,\n) -> CargoResult<BuildContext<'a, 'gctx>> {\n    let CompileOptions {\n        ref build_config,\n        ref spec,\n        ref cli_features,\n        ref filter,\n        ref target_rustdoc_args,\n        ref target_rustc_args,\n        ref target_rustc_crate_types,\n        rustdoc_document_private_items,\n        honor_rust_version,\n    } = *options;\n    let gctx = ws.gctx();\n\n    // Perform some pre-flight validation.\n    match build_config.mode {\n        CompileMode::Test\n        | CompileMode::Build\n        | CompileMode::Check { .. }\n        | CompileMode::Bench\n        | CompileMode::RunCustomBuild => {\n            if ws.gctx().get_env(\"RUST_FLAGS\").is_ok() {\n                gctx.shell().warn(\n                    \"Cargo does not read `RUST_FLAGS` environment variable. Did you mean `RUSTFLAGS`?\",\n                )?;\n            }\n        }\n        CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::Docscrape => {\n            if ws.gctx().get_env(\"RUSTDOC_FLAGS\").is_ok() {\n                gctx.shell().warn(\n                    \"Cargo does not read `RUSTDOC_FLAGS` environment variable. Did you mean `RUSTDOCFLAGS`?\"\n                )?;\n            }\n        }\n    }\n    gctx.validate_term_config()?;\n\n    let mut target_data = RustcTargetData::new(ws, &build_config.requested_kinds)?;\n\n    let specs = spec.to_package_id_specs(ws)?;\n    let has_dev_units = {\n        // Rustdoc itself doesn't need dev-dependencies. But to scrape examples from packages in the\n        // workspace, if any of those packages need dev-dependencies, then we need include dev-dependencies\n        // to scrape those packages.\n        let any_pkg_has_scrape_enabled = ws\n            .members_with_features(&specs, cli_features)?\n            .iter()\n            .any(|(pkg, _)| {\n                pkg.targets()\n                    .iter()\n                    .any(|target| target.is_example() && target.doc_scrape_examples().is_enabled())\n            });\n\n        if filter.need_dev_deps(build_config.mode)\n            || (build_config.mode.is_doc() && any_pkg_has_scrape_enabled)\n        {\n            HasDevUnits::Yes\n        } else {\n            HasDevUnits::No\n        }\n    };\n    let resolve = ops::resolve_ws_with_opts(\n        ws,\n        &mut target_data,\n        &build_config.requested_kinds,\n        cli_features,\n        &specs,\n        has_dev_units,\n        crate::core::resolver::features::ForceAllTargets::No,\n    )?;\n    let WorkspaceResolve {\n        mut pkg_set,\n        workspace_resolve,\n        targeted_resolve: resolve,\n        resolved_features,\n    } = resolve;\n\n    let std_resolve_features = if let Some(crates) = &gctx.cli_unstable().build_std {\n        let (std_package_set, std_resolve, std_features) =\n            standard_lib::resolve_std(ws, &mut target_data, &build_config, crates)?;\n        pkg_set.add_set(std_package_set);\n        Some((std_resolve, std_features))\n    } else {\n        None\n    };\n\n    // Find the packages in the resolver that the user wants to build (those\n    // passed in with `-p` or the defaults from the workspace), and convert\n    // Vec<PackageIdSpec> to a Vec<PackageId>.\n    let to_build_ids = resolve.specs_to_ids(&specs)?;\n    // Now get the `Package` for each `PackageId`. This may trigger a download\n    // if the user specified `-p` for a dependency that is not downloaded.\n    // Dependencies will be downloaded during build_unit_dependencies.\n    let mut to_builds = pkg_set.get_many(to_build_ids)?;\n\n    // The ordering here affects some error messages coming out of cargo, so\n    // let's be test and CLI friendly by always printing in the same order if\n    // there's an error.\n    to_builds.sort_by_key(|p| p.package_id());\n\n    for pkg in to_builds.iter() {\n        pkg.manifest().print_teapot(gctx);\n\n        if build_config.mode.is_any_test()\n            && !ws.is_member(pkg)\n            && pkg.dependencies().iter().any(|dep| !dep.is_transitive())\n        {\n            anyhow::bail!(\n                \"package `{}` cannot be tested because it requires dev-dependencies \\\n                 and is not a member of the workspace\",\n                pkg.name()\n            );\n        }\n    }\n\n    let (extra_args, extra_args_name) = match (target_rustc_args, target_rustdoc_args) {\n        (Some(args), _) => (Some(args.clone()), \"rustc\"),\n        (_, Some(args)) => (Some(args.clone()), \"rustdoc\"),\n        _ => (None, \"\"),\n    };\n\n    if extra_args.is_some() && to_builds.len() != 1 {\n        panic!(\n            \"`{}` should not accept multiple `-p` flags\",\n            extra_args_name\n        );\n    }\n\n    let profiles = Profiles::new(ws, build_config.requested_profile)?;\n    profiles.validate_packages(\n        ws.profiles(),\n        &mut gctx.shell(),\n        workspace_resolve.as_ref().unwrap_or(&resolve),\n    )?;\n\n    // If `--target` has not been specified, then the unit graph is built\n    // assuming `--target $HOST` was specified. See\n    // `rebuild_unit_graph_shared` for more on why this is done.\n    let explicit_host_kind = CompileKind::Target(CompileTarget::new(&target_data.rustc.host)?);\n    let explicit_host_kinds: Vec<_> = build_config\n        .requested_kinds\n        .iter()\n        .map(|kind| match kind {\n            CompileKind::Host => explicit_host_kind,\n            CompileKind::Target(t) => CompileKind::Target(*t),\n        })\n        .collect();\n\n    // Passing `build_config.requested_kinds` instead of\n    // `explicit_host_kinds` here so that `generate_root_units` can do\n    // its own special handling of `CompileKind::Host`. It will\n    // internally replace the host kind by the `explicit_host_kind`\n    // before setting as a unit.\n    let generator = UnitGenerator {\n        ws,\n        packages: &to_builds,\n        filter,\n        requested_kinds: &build_config.requested_kinds,\n        explicit_host_kind,\n        mode: build_config.mode,\n        resolve: &resolve,\n        workspace_resolve: &workspace_resolve,\n        resolved_features: &resolved_features,\n        package_set: &pkg_set,\n        profiles: &profiles,\n        interner,\n        has_dev_units,\n    };\n    let mut units = generator.generate_root_units()?;\n\n    if let Some(args) = target_rustc_crate_types {\n        override_rustc_crate_types(&mut units, args, interner)?;\n    }\n\n    let should_scrape = build_config.mode.is_doc() && gctx.cli_unstable().rustdoc_scrape_examples;\n    let mut scrape_units = if should_scrape {\n        UnitGenerator {\n            mode: CompileMode::Docscrape,\n            ..generator\n        }\n        .generate_scrape_units(&units)?\n    } else {\n        Vec::new()\n    };\n\n    let std_roots = if let Some(crates) = standard_lib::std_crates(gctx, Some(&units)) {\n        let (std_resolve, std_features) = std_resolve_features.as_ref().unwrap();\n        standard_lib::generate_std_roots(\n            &crates,\n            std_resolve,\n            std_features,\n            &explicit_host_kinds,\n            &pkg_set,\n            interner,\n            &profiles,\n        )?\n    } else {\n        Default::default()\n    };\n\n    let mut unit_graph = build_unit_dependencies(\n        ws,\n        &pkg_set,\n        &resolve,\n        &resolved_features,\n        std_resolve_features.as_ref(),\n        &units,\n        &scrape_units,\n        &std_roots,\n        build_config.mode,\n        &target_data,\n        &profiles,\n        interner,\n    )?;\n\n    // TODO: In theory, Cargo should also dedupe the roots, but I'm uncertain\n    // what heuristics to use in that case.\n    if matches!(build_config.mode, CompileMode::Doc { deps: true, .. }) {\n        remove_duplicate_doc(build_config, &units, &mut unit_graph);\n    }\n\n    let host_kind_requested = build_config\n        .requested_kinds\n        .iter()\n        .any(CompileKind::is_host);\n    // Rebuild the unit graph, replacing the explicit host targets with\n    // CompileKind::Host, removing `artifact_target_for_features` and merging any dependencies\n    // shared with build and artifact dependencies.\n    (units, scrape_units, unit_graph) = rebuild_unit_graph_shared(\n        interner,\n        unit_graph,\n        &units,\n        &scrape_units,\n        host_kind_requested.then_some(explicit_host_kind),\n        &target_data,\n    );\n\n    let mut extra_compiler_args = HashMap::new();\n    if let Some(args) = extra_args {\n        if units.len() != 1 {\n            anyhow::bail!(\n                \"extra arguments to `{}` can only be passed to one \\\n                 target, consider filtering\\nthe package by passing, \\\n                 e.g., `--lib` or `--bin NAME` to specify a single target\",\n                extra_args_name\n            );\n        }\n        extra_compiler_args.insert(units[0].clone(), args);\n    }\n\n    for unit in units\n        .iter()\n        .filter(|unit| unit.mode.is_doc() || unit.mode.is_doc_test())\n        .filter(|unit| rustdoc_document_private_items || unit.target.is_bin())\n    {\n        // Add `--document-private-items` rustdoc flag if requested or if\n        // the target is a binary. Binary crates get their private items\n        // documented by default.\n        let mut args = vec![\"--document-private-items\".into()];\n        if unit.target.is_bin() {\n            // This warning only makes sense if it's possible to document private items\n            // sometimes and ignore them at other times. But cargo consistently passes\n            // `--document-private-items`, so the warning isn't useful.\n            args.push(\"-Arustdoc::private-intra-doc-links\".into());\n        }\n        extra_compiler_args\n            .entry(unit.clone())\n            .or_default()\n            .extend(args);\n    }\n\n    if honor_rust_version {\n        let rustc_version = target_data.rustc.version.clone().into();\n\n        let mut incompatible = Vec::new();\n        let mut local_incompatible = false;\n        for unit in unit_graph.keys() {\n            let Some(pkg_msrv) = unit.pkg.rust_version() else {\n                continue;\n            };\n\n            if pkg_msrv.is_compatible_with(&rustc_version) {\n                continue;\n            }\n\n            local_incompatible |= unit.is_local();\n            incompatible.push((unit, pkg_msrv));\n        }\n        if !incompatible.is_empty() {\n            use std::fmt::Write as _;\n\n            let plural = if incompatible.len() == 1 { \"\" } else { \"s\" };\n            let mut message = format!(\n                \"rustc {rustc_version} is not supported by the following package{plural}:\\n\"\n            );\n            incompatible.sort_by_key(|(unit, _)| (unit.pkg.name(), unit.pkg.version()));\n            for (unit, msrv) in incompatible {\n                let name = &unit.pkg.name();\n                let version = &unit.pkg.version();\n                writeln!(&mut message, \"  {name}@{version} requires rustc {msrv}\").unwrap();\n            }\n            if ws.is_ephemeral() {\n                if ws.ignore_lock() {\n                    writeln!(\n                        &mut message,\n                        \"Try re-running `cargo install` with `--locked`\"\n                    )\n                    .unwrap();\n                }\n            } else if !local_incompatible {\n                writeln!(\n                    &mut message,\n                    \"Either upgrade rustc or select compatible dependency versions with\n`cargo update <name>@<current-ver> --precise <compatible-ver>`\nwhere `<compatible-ver>` is the latest version supporting rustc {rustc_version}\",\n                )\n                .unwrap();\n            }\n            return Err(anyhow::Error::msg(message));\n        }\n    }\n\n    let bcx = BuildContext::new(\n        ws,\n        pkg_set,\n        build_config,\n        profiles,\n        extra_compiler_args,\n        target_data,\n        units,\n        unit_graph,\n        scrape_units,\n    )?;\n\n    Ok(bcx)\n}\n\n/// This is used to rebuild the unit graph, sharing host dependencies if possible,\n/// and applying other unit adjustments based on the whole graph.\n///\n/// This will translate any unit's `CompileKind::Target(host)` to\n/// `CompileKind::Host` if `to_host` is not `None` and the kind is equal to `to_host`.\n/// This also handles generating the unit `dep_hash`, and merging shared units if possible.\n///\n/// This is necessary because if normal dependencies used `CompileKind::Host`,\n/// there would be no way to distinguish those units from build-dependency\n/// units or artifact dependency units.\n/// This can cause a problem if a shared normal/build/artifact dependency needs\n/// to link to another dependency whose features differ based on whether or\n/// not it is a normal, build or artifact dependency. If all units used\n/// `CompileKind::Host`, then they would end up being identical, causing a\n/// collision in the `UnitGraph`, and Cargo would end up randomly choosing one\n/// value or the other.\n///\n/// The solution is to keep normal, build and artifact dependencies separate when\n/// building the unit graph, and then run this second pass which will try to\n/// combine shared dependencies safely. By adding a hash of the dependencies\n/// to the `Unit`, this allows the `CompileKind` to be changed back to `Host`\n/// and `artifact_target_for_features` to be removed without fear of an unwanted\n/// collision for build or artifact dependencies.\n///\n/// This is also responsible for adjusting the `strip` profile option to\n/// opportunistically strip if debug is 0 for all dependencies. This helps\n/// remove debuginfo added by the standard library.\n///\n/// This is also responsible for adjusting the `debug` setting for host\n/// dependencies, turning off debug if the user has not explicitly enabled it,\n/// and the unit is not shared with a target unit.\n\nfn rebuild_unit_graph_shared(\n    interner: &UnitInterner,\n    unit_graph: UnitGraph,\n    roots: &[Unit],\n    scrape_units: &[Unit],\n    to_host: Option<CompileKind>,\n    target_data: &RustcTargetData<'_>,\n) -> (Vec<Unit>, Vec<Unit>, UnitGraph) {\n    let mut result = UnitGraph::new();\n    // Map of the old unit to the new unit, used to avoid recursing into units\n    // that have already been computed to improve performance.\n    let mut memo = HashMap::new();\n    let new_roots = roots\n        .iter()\n        .map(|root| {\n            traverse_and_share(\n                interner,\n                &mut memo,\n                &mut result,\n                &unit_graph,\n                root,\n                false,\n                to_host,\n                target_data,\n            )\n        })\n        .collect();\n    // If no unit in the unit graph ended up having scrape units attached as dependencies,\n    // then they won't have been discovered in traverse_and_share and hence won't be in\n    // memo. So we filter out missing scrape units.\n    let new_scrape_units = scrape_units\n        .iter()\n        .map(|unit| memo.get(unit).unwrap().clone())\n        .collect();\n    (new_roots, new_scrape_units, result)\n}\n\n/// Recursive function for rebuilding the graph.\n///\n/// This walks `unit_graph`, starting at the given `unit`. It inserts the new\n/// units into `new_graph`, and returns a new updated version of the given\n/// unit (`dep_hash` is filled in, and `kind` switched if necessary).\nfn traverse_and_share(\n    interner: &UnitInterner,\n    memo: &mut HashMap<Unit, Unit>,\n    new_graph: &mut UnitGraph,\n    unit_graph: &UnitGraph,\n    unit: &Unit,\n    unit_is_for_host: bool,\n    to_host: Option<CompileKind>,\n    target_data: &RustcTargetData<'_>,\n) -> Unit {\n    if let Some(new_unit) = memo.get(unit) {\n        // Already computed, no need to recompute.\n        return new_unit.clone();\n    }\n    let mut dep_hash = StableHasher::new();\n    let new_deps: Vec<_> = unit_graph[unit]\n        .iter()\n        .map(|dep| {\n            let new_dep_unit = traverse_and_share(\n                interner,\n                memo,\n                new_graph,\n                unit_graph,\n                &dep.unit,\n                dep.unit_for.is_for_host(),\n                to_host,\n                target_data,\n            );\n            new_dep_unit.hash(&mut dep_hash);\n            UnitDep {\n                unit: new_dep_unit,\n                ..dep.clone()\n            }\n        })\n        .collect();\n    // Here, we have recursively traversed this unit's dependencies, and hashed them: we can\n    // finalize the dep hash.\n    let new_dep_hash = dep_hash.finish();\n\n    // This is the key part of the sharing process: if the unit is a runtime dependency, whose\n    // target is the same as the host, we canonicalize the compile kind to `CompileKind::Host`.\n    // A possible host dependency counterpart to this unit would have that kind, and if such a unit\n    // exists in the current `unit_graph`, they will unify in the new unit graph map `new_graph`.\n    // The resulting unit graph will be optimized with less units, thanks to sharing these host\n    // dependencies.\n    let canonical_kind = match to_host {\n        Some(to_host) if to_host == unit.kind => CompileKind::Host,\n        _ => unit.kind,\n    };\n\n    let cfg = target_data.cfg(unit.kind);\n    let is_target_windows_msvc = cfg.contains(&Cfg::Name(\"windows\".to_string()))\n        && cfg.contains(&Cfg::KeyPair(\"target_env\".to_string(), \"msvc\".to_string()));\n    let mut profile = unit.profile.clone();\n    // For MSVC, rustc currently treats -Cstrip=debuginfo same as -Cstrip=symbols, which causes\n    // this optimization to also remove symbols and thus break backtraces.\n    if profile.strip.is_deferred() && !is_target_windows_msvc {\n        // If strip was not manually set, and all dependencies of this unit together\n        // with this unit have debuginfo turned off, we enable debuginfo stripping.\n        // This will remove pre-existing debug symbols coming from the standard library.\n        if !profile.debuginfo.is_turned_on()\n            && new_deps\n                .iter()\n                .all(|dep| !dep.unit.profile.debuginfo.is_turned_on())\n        {\n            profile.strip = profile.strip.strip_debuginfo();\n        }\n    }\n\n    // If this is a build dependency, and it's not shared with runtime dependencies, we can weaken\n    // its debuginfo level to optimize build times. We do nothing if it's an artifact dependency,\n    // as it and its debuginfo may end up embedded in the main program.\n    if unit_is_for_host\n        && to_host.is_some()\n        && profile.debuginfo.is_deferred()\n        && !unit.artifact.is_true()\n    {\n        // We create a \"probe\" test to see if a unit with the same explicit debuginfo level exists\n        // in the graph. This is the level we'd expect if it was set manually or the default value\n        // set by a profile for a runtime dependency: its canonical value.\n        let canonical_debuginfo = profile.debuginfo.finalize();\n        let mut canonical_profile = profile.clone();\n        canonical_profile.debuginfo = canonical_debuginfo;\n        let unit_probe = interner.intern(\n            &unit.pkg,\n            &unit.target,\n            canonical_profile,\n            to_host.unwrap(),\n            unit.mode,\n            unit.features.clone(),\n            unit.is_std,\n            unit.dep_hash,\n            unit.artifact,\n            unit.artifact_target_for_features,\n        );\n\n        // We can now turn the deferred value into its actual final value.\n        profile.debuginfo = if unit_graph.contains_key(&unit_probe) {\n            // The unit is present in both build time and runtime subgraphs: we canonicalize its\n            // level to the other unit's, thus ensuring reuse between the two to optimize build times.\n            canonical_debuginfo\n        } else {\n            // The unit is only present in the build time subgraph, we can weaken its debuginfo\n            // level to optimize build times.\n            canonical_debuginfo.weaken()\n        }\n    }\n\n    let new_unit = interner.intern(\n        &unit.pkg,\n        &unit.target,\n        profile,\n        canonical_kind,\n        unit.mode,\n        unit.features.clone(),\n        unit.is_std,\n        new_dep_hash,\n        unit.artifact,\n        // Since `dep_hash` is now filled in, there's no need to specify the artifact target\n        // for target-dependent feature resolution\n        None,\n    );\n    assert!(memo.insert(unit.clone(), new_unit.clone()).is_none());\n    new_graph.entry(new_unit.clone()).or_insert(new_deps);\n    new_unit\n}\n\n/// Removes duplicate CompileMode::Doc units that would cause problems with\n/// filename collisions.\n///\n/// Rustdoc only separates units by crate name in the file directory\n/// structure. If any two units with the same crate name exist, this would\n/// cause a filename collision, causing different rustdoc invocations to stomp\n/// on one another's files.\n///\n/// Unfortunately this does not remove all duplicates, as some of them are\n/// either user error, or difficult to remove. Cases that I can think of:\n///\n/// - Same target name in different packages. See the `collision_doc` test.\n/// - Different sources. See `collision_doc_sources` test.\n///\n/// Ideally this would not be necessary.\nfn remove_duplicate_doc(\n    build_config: &BuildConfig,\n    root_units: &[Unit],\n    unit_graph: &mut UnitGraph,\n) {\n    // First, create a mapping of crate_name -> Unit so we can see where the\n    // duplicates are.\n    let mut all_docs: HashMap<String, Vec<Unit>> = HashMap::new();\n    for unit in unit_graph.keys() {\n        if unit.mode.is_doc() {\n            all_docs\n                .entry(unit.target.crate_name())\n                .or_default()\n                .push(unit.clone());\n        }\n    }\n    // Keep track of units to remove so that they can be efficiently removed\n    // from the unit_deps.\n    let mut removed_units: HashSet<Unit> = HashSet::new();\n    let mut remove = |units: Vec<Unit>, reason: &str, cb: &dyn Fn(&Unit) -> bool| -> Vec<Unit> {\n        let (to_remove, remaining_units): (Vec<Unit>, Vec<Unit>) = units\n            .into_iter()\n            .partition(|unit| cb(unit) && !root_units.contains(unit));\n        for unit in to_remove {\n            tracing::debug!(\n                \"removing duplicate doc due to {} for package {} target `{}`\",\n                reason,\n                unit.pkg,\n                unit.target.name()\n            );\n            unit_graph.remove(&unit);\n            removed_units.insert(unit);\n        }\n        remaining_units\n    };\n    // Iterate over the duplicates and try to remove them from unit_graph.\n    for (_crate_name, mut units) in all_docs {\n        if units.len() == 1 {\n            continue;\n        }\n        // Prefer target over host if --target was not specified.\n        if build_config\n            .requested_kinds\n            .iter()\n            .all(CompileKind::is_host)\n        {\n            // Note these duplicates may not be real duplicates, since they\n            // might get merged in rebuild_unit_graph_shared. Either way, it\n            // shouldn't hurt to remove them early (although the report in the\n            // log might be confusing).\n            units = remove(units, \"host/target merger\", &|unit| unit.kind.is_host());\n            if units.len() == 1 {\n                continue;\n            }\n        }\n        // Prefer newer versions over older.\n        let mut source_map: HashMap<(InternedString, SourceId, CompileKind), Vec<Unit>> =\n            HashMap::new();\n        for unit in units {\n            let pkg_id = unit.pkg.package_id();\n            // Note, this does not detect duplicates from different sources.\n            source_map\n                .entry((pkg_id.name(), pkg_id.source_id(), unit.kind))\n                .or_default()\n                .push(unit);\n        }\n        let mut remaining_units = Vec::new();\n        for (_key, mut units) in source_map {\n            if units.len() > 1 {\n                units.sort_by(|a, b| a.pkg.version().partial_cmp(b.pkg.version()).unwrap());\n                // Remove any entries with version < newest.\n                let newest_version = units.last().unwrap().pkg.version().clone();\n                let keep_units = remove(units, \"older version\", &|unit| {\n                    unit.pkg.version() < &newest_version\n                });\n                remaining_units.extend(keep_units);\n            } else {\n                remaining_units.extend(units);\n            }\n        }\n        if remaining_units.len() == 1 {\n            continue;\n        }\n        // Are there other heuristics to remove duplicates that would make\n        // sense? Maybe prefer path sources over all others?\n    }\n    // Also remove units from the unit_deps so there aren't any dangling edges.\n    for unit_deps in unit_graph.values_mut() {\n        unit_deps.retain(|unit_dep| !removed_units.contains(&unit_dep.unit));\n    }\n    // Remove any orphan units that were detached from the graph.\n    let mut visited = HashSet::new();\n    fn visit(unit: &Unit, graph: &UnitGraph, visited: &mut HashSet<Unit>) {\n        if !visited.insert(unit.clone()) {\n            return;\n        }\n        for dep in &graph[unit] {\n            visit(&dep.unit, graph, visited);\n        }\n    }\n    for unit in root_units {\n        visit(unit, unit_graph, &mut visited);\n    }\n    unit_graph.retain(|unit, _| visited.contains(unit));\n}\n\n/// Override crate types for given units.\n///\n/// This is primarily used by `cargo rustc --crate-type`.\nfn override_rustc_crate_types(\n    units: &mut [Unit],\n    args: &[String],\n    interner: &UnitInterner,\n) -> CargoResult<()> {\n    if units.len() != 1 {\n        anyhow::bail!(\n            \"crate types to rustc can only be passed to one \\\n            target, consider filtering\\nthe package by passing, \\\n            e.g., `--lib` or `--example` to specify a single target\"\n        );\n    }\n\n    let unit = &units[0];\n    let override_unit = |f: fn(Vec<CrateType>) -> TargetKind| {\n        let crate_types = args.iter().map(|s| s.into()).collect();\n        let mut target = unit.target.clone();\n        target.set_kind(f(crate_types));\n        interner.intern(\n            &unit.pkg,\n            &target,\n            unit.profile.clone(),\n            unit.kind,\n            unit.mode,\n            unit.features.clone(),\n            unit.is_std,\n            unit.dep_hash,\n            unit.artifact,\n            unit.artifact_target_for_features,\n        )\n    };\n    units[0] = match unit.target.kind() {\n        TargetKind::Lib(_) => override_unit(TargetKind::Lib),\n        TargetKind::ExampleLib(_) => override_unit(TargetKind::ExampleLib),\n        _ => {\n            anyhow::bail!(\n                \"crate types can only be specified for libraries and example libraries.\\n\\\n                Binaries, tests, and benchmarks are always the `bin` crate type\"\n            );\n        }\n    };\n\n    Ok(())\n}\n\n/// Gets all of the features enabled for a package, plus its dependencies'\n/// features.\n///\n/// Dependencies are added as `dep_name/feat_name` because `required-features`\n/// wants to support that syntax.\npub fn resolve_all_features(\n    resolve_with_overrides: &Resolve,\n    resolved_features: &features::ResolvedFeatures,\n    package_set: &PackageSet<'_>,\n    package_id: PackageId,\n) -> HashSet<String> {\n    let mut features: HashSet<String> = resolved_features\n        .activated_features(package_id, FeaturesFor::NormalOrDev)\n        .iter()\n        .map(|s| s.to_string())\n        .collect();\n\n    // Include features enabled for use by dependencies so targets can also use them with the\n    // required-features field when deciding whether to be built or skipped.\n    for (dep_id, deps) in resolve_with_overrides.deps(package_id) {\n        let is_proc_macro = package_set\n            .get_one(dep_id)\n            .expect(\"packages downloaded\")\n            .proc_macro();\n        for dep in deps {\n            let features_for = FeaturesFor::from_for_host(is_proc_macro || dep.is_build());\n            for feature in resolved_features\n                .activated_features_unverified(dep_id, features_for)\n                .unwrap_or_default()\n            {\n                features.insert(format!(\"{}/{}\", dep.name_in_toml(), feature));\n            }\n        }\n    }\n\n    features\n}\n\n// Path: src/cargo/util/workspace.rs\nuse crate::core::compiler::Unit;\nuse crate::core::manifest::TargetSourcePath;\nuse crate::core::{Target, Workspace};\nuse crate::ops::CompileOptions;\nuse crate::util::CargoResult;\nuse anyhow::bail;\nuse cargo_util::ProcessBuilder;\nuse std::fmt::Write;\nuse std::path::PathBuf;\n\nfn get_available_targets<'a>(\n    filter_fn: fn(&Target) -> bool,\n    ws: &'a Workspace<'_>,\n    options: &'a CompileOptions,\n) -> CargoResult<Vec<&'a str>> {\n    let packages = options.spec.get_packages(ws)?;\n\n    let mut targets: Vec<_> = packages\n        .into_iter()\n        .flat_map(|pkg| {\n            pkg.manifest()\n                .targets()\n                .iter()\n                .filter(|target| filter_fn(target))\n        })\n        .map(Target::name)\n        .collect();\n\n    targets.sort();\n\n    Ok(targets)\n}\n\nfn print_available_targets(\n    filter_fn: fn(&Target) -> bool,\n    ws: &Workspace<'_>,\n    options: &CompileOptions,\n    option_name: &str,\n    plural_name: &str,\n) -> CargoResult<()> {\n    let targets = get_available_targets(filter_fn, ws, options)?;\n\n    let mut output = String::new();\n    writeln!(output, \"\\\"{}\\\" takes one argument.\", option_name)?;\n\n    if targets.is_empty() {\n        writeln!(output, \"No {} available.\", plural_name)?;\n    } else {\n        writeln!(output, \"Available {}:\", plural_name)?;\n        for target in targets {\n            writeln!(output, \"    {}\", target)?;\n        }\n    }\n    bail!(\"{}\", output)\n}\n\npub fn print_available_packages(ws: &Workspace<'_>) -> CargoResult<()> {\n    let packages = ws\n        .members()\n        .map(|pkg| pkg.name().as_str())\n        .collect::<Vec<_>>();\n\n    let mut output = \"\\\"--package <SPEC>\\\" requires a SPEC format value, \\\n        which can be any package ID specifier in the dependency graph.\\n\\\n        Run `cargo help pkgid` for more information about SPEC format.\\n\\n\"\n        .to_string();\n\n    if packages.is_empty() {\n        // This would never happen.\n        // Just in case something regresses we covers it here.\n        writeln!(output, \"No packages available.\")?;\n    } else {\n        writeln!(output, \"Possible packages/workspace members:\")?;\n        for package in packages {\n            writeln!(output, \"    {}\", package)?;\n        }\n    }\n    bail!(\"{}\", output)\n}\n\npub fn print_available_examples(ws: &Workspace<'_>, options: &CompileOptions) -> CargoResult<()> {\n    print_available_targets(Target::is_example, ws, options, \"--example\", \"examples\")\n}\n\npub fn print_available_binaries(ws: &Workspace<'_>, options: &CompileOptions) -> CargoResult<()> {\n    print_available_targets(Target::is_bin, ws, options, \"--bin\", \"binaries\")\n}\n\npub fn print_available_benches(ws: &Workspace<'_>, options: &CompileOptions) -> CargoResult<()> {\n    print_available_targets(Target::is_bench, ws, options, \"--bench\", \"bench targets\")\n}\n\npub fn print_available_tests(ws: &Workspace<'_>, options: &CompileOptions) -> CargoResult<()> {\n    print_available_targets(Target::is_test, ws, options, \"--test\", \"test targets\")\n}\n\n/// The path that we pass to rustc is actually fairly important because it will\n/// show up in error messages (important for readability), debug information\n/// (important for caching), etc. As a result we need to be pretty careful how we\n/// actually invoke rustc.\n///\n/// In general users don't expect `cargo build` to cause rebuilds if you change\n/// directories. That could be if you just change directories in the package or\n/// if you literally move the whole package wholesale to a new directory. As a\n/// result we mostly don't factor in `cwd` to this calculation. Instead we try to\n/// track the workspace as much as possible and we update the current directory\n/// of rustc/rustdoc where appropriate.\n///\n/// The first returned value here is the argument to pass to rustc, and the\n/// second is the cwd that rustc should operate in.\npub fn path_args(ws: &Workspace<'_>, unit: &Unit) -> (PathBuf, PathBuf) {\n    let ws_root = ws.root();\n    let src = match unit.target.src_path() {\n        TargetSourcePath::Path(path) => path.to_path_buf(),\n        TargetSourcePath::Metabuild => unit.pkg.manifest().metabuild_path(ws.target_dir()),\n    };\n    assert!(src.is_absolute());\n    if unit.pkg.package_id().source_id().is_path() {\n        if let Ok(path) = src.strip_prefix(ws_root) {\n            return (path.to_path_buf(), ws_root.to_path_buf());\n        }\n    }\n    (src, unit.pkg.root().to_path_buf())\n}\n\npub fn add_path_args(ws: &Workspace<'_>, unit: &Unit, cmd: &mut ProcessBuilder) {\n    let (arg, cwd) = path_args(ws, unit);\n    cmd.arg(arg);\n    cmd.cwd(cwd);\n}\n\n// Path: src/cargo/core/compiler/fingerprint/dirty_reason.rs\nuse std::fmt;\nuse std::fmt::Debug;\n\nuse super::*;\nuse crate::core::Shell;\n\n/// Tells a better story of why a build is considered \"dirty\" that leads\n/// to a recompile. Usually constructed via [`Fingerprint::compare`].\n///\n/// [`Fingerprint::compare`]: super::Fingerprint::compare\n#[derive(Clone, Debug)]\npub enum DirtyReason {\n    RustcChanged,\n    FeaturesChanged {\n        old: String,\n        new: String,\n    },\n    DeclaredFeaturesChanged {\n        old: String,\n        new: String,\n    },\n    TargetConfigurationChanged,\n    PathToSourceChanged,\n    ProfileConfigurationChanged,\n    RustflagsChanged {\n        old: Vec<String>,\n        new: Vec<String>,\n    },\n    MetadataChanged,\n    ConfigSettingsChanged,\n    CompileKindChanged,\n    LocalLengthsChanged,\n    PrecalculatedComponentsChanged {\n        old: String,\n        new: String,\n    },\n    DepInfoOutputChanged {\n        old: PathBuf,\n        new: PathBuf,\n    },\n    RerunIfChangedOutputFileChanged {\n        old: PathBuf,\n        new: PathBuf,\n    },\n    RerunIfChangedOutputPathsChanged {\n        old: Vec<PathBuf>,\n        new: Vec<PathBuf>,\n    },\n    EnvVarsChanged {\n        old: String,\n        new: String,\n    },\n    EnvVarChanged {\n        name: String,\n        old_value: Option<String>,\n        new_value: Option<String>,\n    },\n    LocalFingerprintTypeChanged {\n        old: &'static str,\n        new: &'static str,\n    },\n    NumberOfDependenciesChanged {\n        old: usize,\n        new: usize,\n    },\n    UnitDependencyNameChanged {\n        old: InternedString,\n        new: InternedString,\n    },\n    UnitDependencyInfoChanged {\n        old_name: InternedString,\n        old_fingerprint: u64,\n\n        new_name: InternedString,\n        new_fingerprint: u64,\n    },\n    FsStatusOutdated(FsStatus),\n    NothingObvious,\n    Forced,\n    /// First time to build something.\n    FreshBuild,\n}\n\ntrait ShellExt {\n    fn dirty_because(&mut self, unit: &Unit, s: impl fmt::Display) -> CargoResult<()>;\n}\n\nimpl ShellExt for Shell {\n    fn dirty_because(&mut self, unit: &Unit, s: impl fmt::Display) -> CargoResult<()> {\n        self.status(\"Dirty\", format_args!(\"{}: {s}\", &unit.pkg))\n    }\n}\n\nstruct FileTimeDiff {\n    old_time: FileTime,\n    new_time: FileTime,\n}\n\nimpl fmt::Display for FileTimeDiff {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let s_diff = self.new_time.seconds() - self.old_time.seconds();\n        if s_diff >= 1 {\n            fmt::Display::fmt(\n                &humantime::Duration::from(std::time::Duration::from_secs(s_diff as u64)),\n                f,\n            )\n        } else {\n            // format nanoseconds as it is, humantime would display ms, us and ns\n            let ns_diff = self.new_time.nanoseconds() - self.old_time.nanoseconds();\n            write!(f, \"{ns_diff}ns\")\n        }\n    }\n}\n\n#[derive(Copy, Clone)]\nstruct After {\n    old_time: FileTime,\n    new_time: FileTime,\n    what: &'static str,\n}\n\nimpl fmt::Display for After {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let Self {\n            old_time,\n            new_time,\n            what,\n        } = *self;\n        let diff = FileTimeDiff { old_time, new_time };\n\n        write!(f, \"{new_time}, {diff} after {what} at {old_time}\")\n    }\n}\n\nimpl DirtyReason {\n    /// Whether a build is dirty because it is a fresh build being kicked off.\n    pub fn is_fresh_build(&self) -> bool {\n        matches!(self, DirtyReason::FreshBuild)\n    }\n\n    fn after(old_time: FileTime, new_time: FileTime, what: &'static str) -> After {\n        After {\n            old_time,\n            new_time,\n            what,\n        }\n    }\n\n    pub fn present_to(&self, s: &mut Shell, unit: &Unit, root: &Path) -> CargoResult<()> {\n        match self {\n            DirtyReason::RustcChanged => s.dirty_because(unit, \"the toolchain changed\"),\n            DirtyReason::FeaturesChanged { .. } => {\n                s.dirty_because(unit, \"the list of features changed\")\n            }\n            DirtyReason::DeclaredFeaturesChanged { .. } => {\n                s.dirty_because(unit, \"the list of declared features changed\")\n            }\n            DirtyReason::TargetConfigurationChanged => {\n                s.dirty_because(unit, \"the target configuration changed\")\n            }\n            DirtyReason::PathToSourceChanged => {\n                s.dirty_because(unit, \"the path to the source changed\")\n            }\n            DirtyReason::ProfileConfigurationChanged => {\n                s.dirty_because(unit, \"the profile configuration changed\")\n            }\n            DirtyReason::RustflagsChanged { .. } => s.dirty_because(unit, \"the rustflags changed\"),\n            DirtyReason::MetadataChanged => s.dirty_because(unit, \"the metadata changed\"),\n            DirtyReason::ConfigSettingsChanged => {\n                s.dirty_because(unit, \"the config settings changed\")\n            }\n            DirtyReason::CompileKindChanged => {\n                s.dirty_because(unit, \"the rustc compile kind changed\")\n            }\n            DirtyReason::LocalLengthsChanged => {\n                s.dirty_because(unit, \"the local lengths changed\")?;\n                s.note(\n                    \"this could happen because of added/removed `cargo::rerun-if` instructions in the build script\",\n                )?;\n\n                Ok(())\n            }\n            DirtyReason::PrecalculatedComponentsChanged { .. } => {\n                s.dirty_because(unit, \"the precalculated components changed\")\n            }\n            DirtyReason::DepInfoOutputChanged { .. } => {\n                s.dirty_because(unit, \"the dependency info output changed\")\n            }\n            DirtyReason::RerunIfChangedOutputFileChanged { .. } => {\n                s.dirty_because(unit, \"rerun-if-changed output file path changed\")\n            }\n            DirtyReason::RerunIfChangedOutputPathsChanged { .. } => {\n                s.dirty_because(unit, \"the rerun-if-changed instructions changed\")\n            }\n            DirtyReason::EnvVarsChanged { .. } => {\n                s.dirty_because(unit, \"the environment variables changed\")\n            }\n            DirtyReason::EnvVarChanged { name, .. } => {\n                s.dirty_because(unit, format_args!(\"the env variable {name} changed\"))\n            }\n            DirtyReason::LocalFingerprintTypeChanged { .. } => {\n                s.dirty_because(unit, \"the local fingerprint type changed\")\n            }\n            DirtyReason::NumberOfDependenciesChanged { old, new } => s.dirty_because(\n                unit,\n                format_args!(\"number of dependencies changed ({old} => {new})\",),\n            ),\n            DirtyReason::UnitDependencyNameChanged { old, new } => s.dirty_because(\n                unit,\n                format_args!(\"name of dependency changed ({old} => {new})\"),\n            ),\n            DirtyReason::UnitDependencyInfoChanged { .. } => {\n                s.dirty_because(unit, \"dependency info changed\")\n            }\n            DirtyReason::FsStatusOutdated(status) => match status {\n                FsStatus::Stale => s.dirty_because(unit, \"stale, unknown reason\"),\n                FsStatus::StaleItem(item) => match item {\n                    StaleItem::MissingFile(missing_file) => {\n                        let file = missing_file.strip_prefix(root).unwrap_or(&missing_file);\n                        s.dirty_because(\n                            unit,\n                            format_args!(\"the file `{}` is missing\", file.display()),\n                        )\n                    }\n                    StaleItem::ChangedFile {\n                        stale,\n                        stale_mtime,\n                        reference_mtime,\n                        ..\n                    } => {\n                        let file = stale.strip_prefix(root).unwrap_or(&stale);\n                        let after = Self::after(*reference_mtime, *stale_mtime, \"last build\");\n                        s.dirty_because(\n                            unit,\n                            format_args!(\"the file `{}` has changed ({after})\", file.display()),\n                        )\n                    }\n                    StaleItem::ChangedEnv { var, .. } => s.dirty_because(\n                        unit,\n                        format_args!(\"the environment variable {var} changed\"),\n                    ),\n                },\n                FsStatus::StaleDependency {\n                    name,\n                    dep_mtime,\n                    max_mtime,\n                    ..\n                } => {\n                    let after = Self::after(*max_mtime, *dep_mtime, \"last build\");\n                    s.dirty_because(\n                        unit,\n                        format_args!(\"the dependency {name} was rebuilt ({after})\"),\n                    )\n                }\n                FsStatus::StaleDepFingerprint { name } => {\n                    s.dirty_because(unit, format_args!(\"the dependency {name} was rebuilt\"))\n                }\n                FsStatus::UpToDate { .. } => {\n                    unreachable!()\n                }\n            },\n            DirtyReason::NothingObvious => {\n                // See comment in fingerprint compare method.\n                s.dirty_because(unit, \"the fingerprint comparison turned up nothing obvious\")\n            }\n            DirtyReason::Forced => s.dirty_because(unit, \"forced\"),\n            DirtyReason::FreshBuild => s.dirty_because(unit, \"fresh build\"),\n        }\n    }\n}\n\n// Path: src/cargo/core/compiler/job_queue/job.rs\n//! See [`Job`] and [`Work`].\n\nuse std::fmt;\nuse std::mem;\n\nuse super::JobState;\nuse crate::core::compiler::fingerprint::DirtyReason;\nuse crate::util::CargoResult;\n\n/// Represents a unit of [`Work`] with a [`Freshness`] for caller\n/// to determine whether to re-execute or not.\npub struct Job {\n    work: Work,\n    fresh: Freshness,\n}\n\n/// The basic unit of work.\n///\n/// Each proc should send its description before starting.\n/// It should send either once or close immediately.\npub struct Work {\n    inner: Box<dyn FnOnce(&JobState<'_, '_>) -> CargoResult<()> + Send>,\n}\n\nimpl Work {\n    /// Creates a unit of work.\n    pub fn new<F>(f: F) -> Work\n    where\n        F: FnOnce(&JobState<'_, '_>) -> CargoResult<()> + Send + 'static,\n    {\n        Work { inner: Box::new(f) }\n    }\n\n    /// Creates a unit of work that does nothing.\n    pub fn noop() -> Work {\n        Work::new(|_| Ok(()))\n    }\n\n    /// Consumes this work by running it.\n    pub fn call(self, tx: &JobState<'_, '_>) -> CargoResult<()> {\n        (self.inner)(tx)\n    }\n\n    /// Creates a new unit of work that chains `next` after ourself.\n    pub fn then(self, next: Work) -> Work {\n        Work::new(move |state| {\n            self.call(state)?;\n            next.call(state)\n        })\n    }\n}\n\nimpl Job {\n    /// Creates a new job that does nothing.\n    pub fn new_fresh() -> Job {\n        Job {\n            work: Work::noop(),\n            fresh: Freshness::Fresh,\n        }\n    }\n\n    /// Creates a new job representing a unit of work.\n    pub fn new_dirty(work: Work, dirty_reason: DirtyReason) -> Job {\n        Job {\n            work,\n            fresh: Freshness::Dirty(dirty_reason),\n        }\n    }\n\n    /// Consumes this job by running it, returning the result of the\n    /// computation.\n    pub fn run(self, state: &JobState<'_, '_>) -> CargoResult<()> {\n        self.work.call(state)\n    }\n\n    /// Returns whether this job was fresh/dirty, where \"fresh\" means we're\n    /// likely to perform just some small bookkeeping where \"dirty\" means we'll\n    /// probably do something slow like invoke rustc.\n    pub fn freshness(&self) -> &Freshness {\n        &self.fresh\n    }\n\n    /// Chains the given work by putting it in front of our own unit of work.\n    pub fn before(&mut self, next: Work) {\n        let prev = mem::replace(&mut self.work, Work::noop());\n        self.work = next.then(prev);\n    }\n}\n\nimpl fmt::Debug for Job {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Job {{ ... }}\")\n    }\n}\n\n/// Indication of the freshness of a package.\n///\n/// A fresh package does not necessarily need to be rebuilt (unless a dependency\n/// was also rebuilt), and a dirty package must always be rebuilt.\n#[derive(Debug, Clone)]\npub enum Freshness {\n    Fresh,\n    Dirty(DirtyReason),\n}\n\nimpl Freshness {\n    pub fn is_dirty(&self) -> bool {\n        matches!(self, Freshness::Dirty(_))\n    }\n\n    pub fn is_fresh(&self) -> bool {\n        matches!(self, Freshness::Fresh)\n    }\n}\n\n// Path: src/cargo/core/compiler/fingerprint/mod.rs\n//! Tracks changes to determine if something needs to be recompiled.\n//!\n//! This module implements change-tracking so that Cargo can know whether or\n//! not something needs to be recompiled. A Cargo [`Unit`] can be either \"dirty\"\n//! (needs to be recompiled) or \"fresh\" (it does not need to be recompiled).\n//!\n//! ## Mechanisms affecting freshness\n//!\n//! There are several mechanisms that influence a Unit's freshness:\n//!\n//! - The [`Fingerprint`] is a hash, saved to the filesystem in the\n//!   `.fingerprint` directory, that tracks information about the Unit. If the\n//!   fingerprint is missing (such as the first time the unit is being\n//!   compiled), then the unit is dirty. If any of the fingerprint fields\n//!   change (like the name of the source file), then the Unit is considered\n//!   dirty.\n//!\n//!   The `Fingerprint` also tracks the fingerprints of all its dependencies,\n//!   so a change in a dependency will propagate the \"dirty\" status up.\n//!\n//! - Filesystem mtime tracking is also used to check if a unit is dirty.\n//!   See the section below on \"Mtime comparison\" for more details. There\n//!   are essentially two parts to mtime tracking:\n//!\n//!   1. The mtime of a Unit's output files is compared to the mtime of all\n//!      its dependencies' output file mtimes (see\n//!      [`check_filesystem`]). If any output is missing, or is\n//!      older than a dependency's output, then the unit is dirty.\n//!   2. The mtime of a Unit's source files is compared to the mtime of its\n//!      dep-info file in the fingerprint directory (see [`find_stale_file`]).\n//!      The dep-info file is used as an anchor to know when the last build of\n//!      the unit was done. See the \"dep-info files\" section below for more\n//!      details. If any input files are missing, or are newer than the\n//!      dep-info, then the unit is dirty.\n//!\n//! Note: Fingerprinting is not a perfect solution. Filesystem mtime tracking\n//! is notoriously imprecise and problematic. Only a small part of the\n//! environment is captured. This is a balance of performance, simplicity, and\n//! completeness. Sandboxing, hashing file contents, tracking every file\n//! access, environment variable, and network operation would ensure more\n//! reliable and reproducible builds at the cost of being complex, slow, and\n//! platform-dependent.\n//!\n//! ## Fingerprints and Metadata\n//!\n//! The [`Metadata`] hash is a hash added to the output filenames to isolate\n//! each unit. See its documentationfor more details.\n//! NOTE: Not all output files are isolated via filename hashes (like dylibs).\n//! The fingerprint directory uses a hash, but sometimes units share the same\n//! fingerprint directory (when they don't have Metadata) so care should be\n//! taken to handle this!\n//!\n//! Fingerprints and Metadata are similar, and track some of the same things.\n//! The Metadata contains information that is required to keep Units separate.\n//! The Fingerprint includes additional information that should cause a\n//! recompile, but it is desired to reuse the same filenames. A comparison\n//! of what is tracked:\n//!\n//! Value                                      | Fingerprint | Metadata\n//! -------------------------------------------|-------------|----------\n//! rustc                                      | \u2713           | \u2713\n//! [`Profile`]                                | \u2713           | \u2713\n//! `cargo rustc` extra args                   | \u2713           | \u2713\n//! [`CompileMode`]                            | \u2713           | \u2713\n//! Target Name                                | \u2713           | \u2713\n//! TargetKind (bin/lib/etc.)                  | \u2713           | \u2713\n//! Enabled Features                           | \u2713           | \u2713\n//! Declared Features                          | \u2713           |\n//! Immediate dependency\u2019s hashes              | \u2713[^1]       | \u2713\n//! [`CompileKind`] (host/target)              | \u2713           | \u2713\n//! __CARGO_DEFAULT_LIB_METADATA[^4]           |             | \u2713\n//! package_id                                 |             | \u2713\n//! authors, description, homepage, repo       | \u2713           |\n//! Target src path relative to ws             | \u2713           |\n//! Target flags (test/bench/for_host/edition) | \u2713           |\n//! -C incremental=\u2026 flag                      | \u2713           |\n//! mtime of sources                           | \u2713[^3]       |\n//! RUSTFLAGS/RUSTDOCFLAGS                     | \u2713           |\n//! [`Lto`] flags                              | \u2713           | \u2713\n//! config settings[^5]                        | \u2713           |\n//! is_std                                     |             | \u2713\n//! `[lints]` table[^6]                        | \u2713           |\n//!\n//! [^1]: Build script and bin dependencies are not included.\n//!\n//! [^3]: See below for details on mtime tracking.\n//!\n//! [^4]: `__CARGO_DEFAULT_LIB_METADATA` is set by rustbuild to embed the\n//!        release channel (bootstrap/stable/beta/nightly) in libstd.\n//!\n//! [^5]: Config settings that are not otherwise captured anywhere else.\n//!       Currently, this is only `doc.extern-map`.\n//!\n//! [^6]: Via [`Manifest::lint_rustflags`][crate::core::Manifest::lint_rustflags]\n//!\n//! When deciding what should go in the Metadata vs the Fingerprint, consider\n//! that some files (like dylibs) do not have a hash in their filename. Thus,\n//! if a value changes, only the fingerprint will detect the change (consider,\n//! for example, swapping between different features). Fields that are only in\n...\n", "needle_token_start": 5516, "needle_token_end": 5873, "code_context_ntokens": 16276, "output": ["```rust\nfn rebuild_unit_graph_shared(\n    interner: &UnitInterner,\n    unit_graph: UnitGraph,\n    roots: &[Unit],\n    scrape_units: &[Unit],\n    to_host: Option<CompileKind>,\n    target_data: &RustcTargetData<'_>,\n) -> (Vec<Unit>, Vec<Unit>, UnitGraph) {\n    let mut result = UnitGraph::new();\n    // Map of the old unit to the new unit, used to avoid recursing into units\n    // that have already been computed to improve performance.\n    let mut memo = HashMap::new();\n    let new_roots = roots\n        .iter()\n        .map(|root| {\n            traverse_and_share(\n                interner,\n                &mut memo,\n                &mut result,\n                &unit_graph,\n                root,\n                false,\n                to_host,\n                target_data,\n            )\n        })\n        .collect();\n    // If no unit in the unit graph ended up having scrape units attached as dependencies,\n    // then they won't have been discovered in traverse_and_share and hence won't be in\n    // memo. So we filter out missing scrape units.\n    let new_scrape_units = scrape_units\n        .iter()\n        .map(|unit| memo.get(unit).unwrap().clone())\n        .collect();\n    (new_roots, new_scrape_units, result)\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "predefined_dir_names", "language": "rust", "path": "src/cargo/core/profiles.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide a mapping between certain predefined configuration names and their corresponding directory names where build artifacts should be stored.\n2. **Input**: This function does not take any parameters.\n3. **Output**: It returns a mapping (hash map) where keys and values are both strings. The keys represent configuration names, and the values are the directory names associated with these configurations.\n4. **Procedure**: The function initializes a fixed list of key-value pairs where each pair corresponds to a configuration and its associated directory name. This list is then converted into a hash map which is returned as the output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/cargo/util/diagnostic_server.rs\n//! A small TCP server to handle collection of diagnostics information in a\n//! cross-platform way for the `cargo fix` command.\n\nuse std::collections::HashSet;\nuse std::io::{BufReader, Read, Write};\nuse std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};\nuse std::path::PathBuf;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::Arc;\nuse std::thread::{self, JoinHandle};\n\nuse anyhow::{Context as _, Error};\nuse cargo_util::ProcessBuilder;\nuse serde::{Deserialize, Serialize};\nuse tracing::warn;\n\nuse crate::core::Edition;\nuse crate::util::errors::CargoResult;\nuse crate::util::GlobalContext;\n\nconst DIAGNOSTICS_SERVER_VAR: &str = \"__CARGO_FIX_DIAGNOSTICS_SERVER\";\n\n#[derive(Deserialize, Serialize, Hash, Eq, PartialEq, Clone)]\npub enum Message {\n    Migrating {\n        file: String,\n        from_edition: Edition,\n        to_edition: Edition,\n    },\n    Fixing {\n        file: String,\n    },\n    Fixed {\n        file: String,\n        fixes: u32,\n    },\n    FixFailed {\n        files: Vec<String>,\n        krate: Option<String>,\n        errors: Vec<String>,\n        abnormal_exit: Option<String>,\n    },\n    ReplaceFailed {\n        file: String,\n        message: String,\n    },\n    EditionAlreadyEnabled {\n        message: String,\n        edition: Edition,\n    },\n}\n\nimpl Message {\n    pub fn post(&self, gctx: &GlobalContext) -> Result<(), Error> {\n        let addr = gctx\n            .get_env(DIAGNOSTICS_SERVER_VAR)\n            .context(\"diagnostics collector misconfigured\")?;\n        let mut client =\n            TcpStream::connect(&addr).context(\"failed to connect to parent diagnostics target\")?;\n\n        let s = serde_json::to_string(self).context(\"failed to serialize message\")?;\n        client\n            .write_all(s.as_bytes())\n            .context(\"failed to write message to diagnostics target\")?;\n        client\n            .shutdown(Shutdown::Write)\n            .context(\"failed to shutdown\")?;\n\n        client\n            .read_to_end(&mut Vec::new())\n            .context(\"failed to receive a disconnect\")?;\n\n        Ok(())\n    }\n}\n\n/// A printer that will print diagnostics messages to the shell.\npub struct DiagnosticPrinter<'a> {\n    /// The context to get the shell to print to.\n    gctx: &'a GlobalContext,\n    /// An optional wrapper to be used in addition to `rustc.wrapper` for workspace crates.\n    /// This is used to get the correct bug report URL. For instance,\n    /// if `clippy-driver` is set as the value for the wrapper,\n    /// then the correct bug report URL for `clippy` can be obtained.\n    workspace_wrapper: &'a Option<PathBuf>,\n    // A set of messages that have already been printed.\n    dedupe: HashSet<Message>,\n}\n\nimpl<'a> DiagnosticPrinter<'a> {\n    pub fn new(\n        gctx: &'a GlobalContext,\n        workspace_wrapper: &'a Option<PathBuf>,\n    ) -> DiagnosticPrinter<'a> {\n        DiagnosticPrinter {\n            gctx,\n            workspace_wrapper,\n            dedupe: HashSet::new(),\n        }\n    }\n\n    pub fn print(&mut self, msg: &Message) -> CargoResult<()> {\n        match msg {\n            Message::Migrating {\n                file,\n                from_edition,\n                to_edition,\n            } => {\n                if !self.dedupe.insert(msg.clone()) {\n                    return Ok(());\n                }\n                self.gctx.shell().status(\n                    \"Migrating\",\n                    &format!(\"{} from {} edition to {}\", file, from_edition, to_edition),\n                )\n            }\n            Message::Fixing { file } => self\n                .gctx\n                .shell()\n                .verbose(|shell| shell.status(\"Fixing\", file)),\n            Message::Fixed { file, fixes } => {\n                let msg = if *fixes == 1 { \"fix\" } else { \"fixes\" };\n                let msg = format!(\"{} ({} {})\", file, fixes, msg);\n                self.gctx.shell().status(\"Fixed\", msg)\n            }\n            Message::ReplaceFailed { file, message } => {\n                let msg = format!(\"error applying suggestions to `{}`\\n\", file);\n                self.gctx.shell().warn(&msg)?;\n                write!(\n                    self.gctx.shell().err(),\n                    \"The full error message was:\\n\\n> {}\\n\\n\",\n                    message,\n                )?;\n                let issue_link = get_bug_report_url(self.workspace_wrapper);\n                write!(\n                    self.gctx.shell().err(),\n                    \"{}\",\n                    gen_please_report_this_bug_text(issue_link)\n                )?;\n                Ok(())\n            }\n            Message::FixFailed {\n                files,\n                krate,\n                errors,\n                abnormal_exit,\n            } => {\n                if let Some(ref krate) = *krate {\n                    self.gctx.shell().warn(&format!(\n                        \"failed to automatically apply fixes suggested by rustc \\\n                         to crate `{}`\",\n                        krate,\n                    ))?;\n                } else {\n                    self.gctx\n                        .shell()\n                        .warn(\"failed to automatically apply fixes suggested by rustc\")?;\n                }\n                if !files.is_empty() {\n                    writeln!(\n                        self.gctx.shell().err(),\n                        \"\\nafter fixes were automatically applied the compiler \\\n                         reported errors within these files:\\n\"\n                    )?;\n                    for file in files {\n                        writeln!(self.gctx.shell().err(), \"  * {}\", file)?;\n                    }\n                    writeln!(self.gctx.shell().err())?;\n                }\n                let issue_link = get_bug_report_url(self.workspace_wrapper);\n                write!(\n                    self.gctx.shell().err(),\n                    \"{}\",\n                    gen_please_report_this_bug_text(issue_link)\n                )?;\n                if !errors.is_empty() {\n                    writeln!(\n                        self.gctx.shell().err(),\n                        \"The following errors were reported:\"\n                    )?;\n                    for error in errors {\n                        write!(self.gctx.shell().err(), \"{}\", error)?;\n                        if !error.ends_with('\\n') {\n                            writeln!(self.gctx.shell().err())?;\n                        }\n                    }\n                }\n                if let Some(exit) = abnormal_exit {\n                    writeln!(self.gctx.shell().err(), \"rustc exited abnormally: {}\", exit)?;\n                }\n                writeln!(\n                    self.gctx.shell().err(),\n                    \"Original diagnostics will follow.\\n\"\n                )?;\n                Ok(())\n            }\n            Message::EditionAlreadyEnabled { message, edition } => {\n                if !self.dedupe.insert(msg.clone()) {\n                    return Ok(());\n                }\n                // Don't give a really verbose warning if it has already been issued.\n                if self.dedupe.insert(Message::EditionAlreadyEnabled {\n                    message: \"\".to_string(), // Dummy, so that this only long-warns once.\n                    edition: *edition,\n                }) {\n                    self.gctx.shell().warn(&format!(\"\\\n{}\n\nIf you are trying to migrate from the previous edition ({prev_edition}), the\nprocess requires following these steps:\n\n1. Start with `edition = \\\"{prev_edition}\\\"` in `Cargo.toml`\n2. Run `cargo fix --edition`\n3. Modify `Cargo.toml` to set `edition = \\\"{this_edition}\\\"`\n4. Run `cargo build` or `cargo test` to verify the fixes worked\n\nMore details may be found at\nhttps://doc.rust-lang.org/edition-guide/editions/transitioning-an-existing-project-to-a-new-edition.html\n\",\n                        message, this_edition=edition, prev_edition=edition.previous().unwrap()\n                    ))\n                } else {\n                    self.gctx.shell().warn(message)\n                }\n            }\n        }\n    }\n}\n\nfn gen_please_report_this_bug_text(url: &str) -> String {\n    format!(\n        \"This likely indicates a bug in either rustc or cargo itself,\\n\\\n     and we would appreciate a bug report! You're likely to see\\n\\\n     a number of compiler warnings after this message which cargo\\n\\\n     attempted to fix but failed. If you could open an issue at\\n\\\n     {}\\n\\\n     quoting the full output of this command we'd be very appreciative!\\n\\\n     Note that you may be able to make some more progress in the near-term\\n\\\n     fixing code with the `--broken-code` flag\\n\\n\\\n     \",\n        url\n    )\n}\n\nfn get_bug_report_url(rustc_workspace_wrapper: &Option<PathBuf>) -> &str {\n    let clippy = std::ffi::OsStr::new(\"clippy-driver\");\n    let issue_link = match rustc_workspace_wrapper.as_ref().and_then(|x| x.file_stem()) {\n        Some(wrapper) if wrapper == clippy => \"https://github.com/rust-lang/rust-clippy/issues\",\n        _ => \"https://github.com/rust-lang/rust/issues\",\n    };\n\n    issue_link\n}\n\n#[derive(Debug)]\npub struct RustfixDiagnosticServer {\n    listener: TcpListener,\n    addr: SocketAddr,\n}\n\npub struct StartedServer {\n    addr: SocketAddr,\n    done: Arc<AtomicBool>,\n    thread: Option<JoinHandle<()>>,\n}\n\nimpl RustfixDiagnosticServer {\n    pub fn new() -> Result<Self, Error> {\n        let listener = TcpListener::bind(\"127.0.0.1:0\")\n            .with_context(|| \"failed to bind TCP listener to manage locking\")?;\n        let addr = listener.local_addr()?;\n\n        Ok(RustfixDiagnosticServer { listener, addr })\n    }\n\n    pub fn configure(&self, process: &mut ProcessBuilder) {\n        process.env(DIAGNOSTICS_SERVER_VAR, self.addr.to_string());\n    }\n\n    pub fn start<F>(self, on_message: F) -> Result<StartedServer, Error>\n    where\n        F: Fn(Message) + Send + 'static,\n    {\n        let addr = self.addr;\n        let done = Arc::new(AtomicBool::new(false));\n        let done2 = done.clone();\n...\n// Path: src/cargo/core/compiler/build_config.rs\nuse crate::core::compiler::CompileKind;\nuse crate::util::context::JobsConfig;\nuse crate::util::interning::InternedString;\nuse crate::util::{CargoResult, GlobalContext, RustfixDiagnosticServer};\nuse anyhow::{bail, Context as _};\nuse cargo_util::ProcessBuilder;\nuse serde::ser;\nuse std::cell::RefCell;\nuse std::path::PathBuf;\nuse std::rc::Rc;\nuse std::thread::available_parallelism;\n\n/// Configuration information for a rustc build.\n#[derive(Debug, Clone)]\npub struct BuildConfig {\n    /// The requested kind of compilation for this session\n    pub requested_kinds: Vec<CompileKind>,\n    /// Number of rustc jobs to run in parallel.\n    pub jobs: u32,\n    /// Do not abort the build as soon as there is an error.\n    pub keep_going: bool,\n    /// Build profile\n    pub requested_profile: InternedString,\n    /// The mode we are compiling in.\n    pub mode: CompileMode,\n    /// `true` to print stdout in JSON format (for machine reading).\n    pub message_format: MessageFormat,\n    /// Force Cargo to do a full rebuild and treat each target as changed.\n    pub force_rebuild: bool,\n    /// Output a build plan to stdout instead of actually compiling.\n    pub build_plan: bool,\n    /// Output the unit graph to stdout instead of actually compiling.\n    pub unit_graph: bool,\n    /// An optional override of the rustc process for primary units\n    pub primary_unit_rustc: Option<ProcessBuilder>,\n    /// A thread used by `cargo fix` to receive messages on a socket regarding\n    /// the success/failure of applying fixes.\n    pub rustfix_diagnostic_server: Rc<RefCell<Option<RustfixDiagnosticServer>>>,\n    /// The directory to copy final artifacts to. Note that even if `out_dir` is\n    /// set, a copy of artifacts still could be found a `target/(debug\\release)`\n    /// as usual.\n    // Note that, although the cmd-line flag name is `out-dir`, in code we use\n    // `export_dir`, to avoid confusion with out dir at `target/debug/deps`.\n    pub export_dir: Option<PathBuf>,\n    /// `true` to output a future incompatibility report at the end of the build\n    pub future_incompat_report: bool,\n    /// Which kinds of build timings to output (empty if none).\n    pub timing_outputs: Vec<TimingOutput>,\n}\n\nfn default_parallelism() -> CargoResult<u32> {\n    Ok(available_parallelism()\n        .context(\"failed to determine the amount of parallelism available\")?\n        .get() as u32)\n}\n\nimpl BuildConfig {\n    /// Parses all config files to learn about build configuration. Currently\n    /// configured options are:\n    ///\n    /// * `build.jobs`\n    /// * `build.target`\n    /// * `target.$target.ar`\n    /// * `target.$target.linker`\n    /// * `target.$target.libfoo.metadata`\n    pub fn new(\n        gctx: &GlobalContext,\n        jobs: Option<JobsConfig>,\n        keep_going: bool,\n        requested_targets: &[String],\n        mode: CompileMode,\n    ) -> CargoResult<BuildConfig> {\n        let cfg = gctx.build_config()?;\n        let requested_kinds = CompileKind::from_requested_targets(gctx, requested_targets)?;\n        if jobs.is_some() && gctx.jobserver_from_env().is_some() {\n            gctx.shell().warn(\n                \"a `-j` argument was passed to Cargo but Cargo is \\\n                 also configured with an external jobserver in \\\n                 its environment, ignoring the `-j` parameter\",\n            )?;\n        }\n        let jobs = match jobs.or(cfg.jobs.clone()) {\n            None => default_parallelism()?,\n            Some(value) => match value {\n                JobsConfig::Integer(j) => match j {\n                    0 => anyhow::bail!(\"jobs may not be 0\"),\n                    j if j < 0 => (default_parallelism()? as i32 + j).max(1) as u32,\n                    j => j as u32,\n                },\n                JobsConfig::String(j) => match j.as_str() {\n                    \"default\" => default_parallelism()?,\n                    _ => {\n                        anyhow::bail!(\n\t\t\t    format!(\"could not parse `{j}`. Number of parallel jobs should be `default` or a number.\"))\n                    }\n                },\n            },\n        };\n\n        if gctx.cli_unstable().build_std.is_some() && requested_kinds[0].is_host() {\n            // TODO: This should eventually be fixed.\n            anyhow::bail!(\"-Zbuild-std requires --target\");\n        }\n\n        Ok(BuildConfig {\n            requested_kinds,\n            jobs,\n            keep_going,\n            requested_profile: InternedString::new(\"dev\"),\n            mode,\n            message_format: MessageFormat::Human,\n            force_rebuild: false,\n            build_plan: false,\n            unit_graph: false,\n            primary_unit_rustc: None,\n            rustfix_diagnostic_server: Rc::new(RefCell::new(None)),\n            export_dir: None,\n            future_incompat_report: false,\n            timing_outputs: Vec::new(),\n        })\n    }\n\n    /// Whether or not the *user* wants JSON output. Whether or not rustc\n    /// actually uses JSON is decided in `add_error_format`.\n    pub fn emit_json(&self) -> bool {\n        matches!(self.message_format, MessageFormat::Json { .. })\n    }\n\n    pub fn test(&self) -> bool {\n        self.mode == CompileMode::Test || self.mode == CompileMode::Bench\n    }\n\n    pub fn single_requested_kind(&self) -> CargoResult<CompileKind> {\n        match self.requested_kinds.len() {\n            1 => Ok(self.requested_kinds[0]),\n            _ => bail!(\"only one `--target` argument is supported\"),\n        }\n    }\n}\n\n#[derive(Clone, Copy, Debug, PartialEq, Eq)]\npub enum MessageFormat {\n    Human,\n    Json {\n        /// Whether rustc diagnostics are rendered by cargo or included into the\n        /// output stream.\n        render_diagnostics: bool,\n        /// Whether the `rendered` field of rustc diagnostics are using the\n        /// \"short\" rendering.\n        short: bool,\n        /// Whether the `rendered` field of rustc diagnostics embed ansi color\n        /// codes.\n        ansi: bool,\n    },\n    Short,\n}\n\n/// The general \"mode\" for what to do.\n/// This is used for two purposes. The commands themselves pass this in to\n/// `compile_ws` to tell it the general execution strategy. This influences\n/// the default targets selected. The other use is in the `Unit` struct\n/// to indicate what is being done with a specific target.\n#[derive(Clone, Copy, PartialEq, Debug, Eq, Hash, PartialOrd, Ord)]\npub enum CompileMode {\n    /// A target being built for a test.\n    Test,\n    /// Building a target with `rustc` (lib or bin).\n    Build,\n    /// Building a target with `rustc` to emit `rmeta` metadata only. If\n    /// `test` is true, then it is also compiled with `--test` to check it like\n    /// a test.\n    Check { test: bool },\n    /// Used to indicate benchmarks should be built. This is not used in\n    /// `Unit`, because it is essentially the same as `Test` (indicating\n    /// `--test` should be passed to rustc) and by using `Test` instead it\n    /// allows some de-duping of Units to occur.\n    Bench,\n    /// A target that will be documented with `rustdoc`.\n\n    /// If `deps` is true, then it will also document all dependencies.\n    /// if `json` is true, the documentation output is in json format.\n    Doc { deps: bool, json: bool },\n    /// A target that will be tested with `rustdoc`.\n    Doctest,\n    /// An example or library that will be scraped for function calls by `rustdoc`.\n    Docscrape,\n    /// A marker for Units that represent the execution of a `build.rs` script.\n    RunCustomBuild,\n}\n\nimpl ser::Serialize for CompileMode {\n    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>\n    where\n        S: ser::Serializer,\n    {\n        use self::CompileMode::*;\n        match *self {\n            Test => \"test\".serialize(s),\n            Build => \"build\".serialize(s),\n            Check { .. } => \"check\".serialize(s),\n            Bench => \"bench\".serialize(s),\n            Doc { .. } => \"doc\".serialize(s),\n            Doctest => \"doctest\".serialize(s),\n            Docscrape => \"docscrape\".serialize(s),\n            RunCustomBuild => \"run-custom-build\".serialize(s),\n        }\n    }\n}\n\nimpl CompileMode {\n    /// Returns `true` if the unit is being checked.\n    pub fn is_check(self) -> bool {\n        matches!(self, CompileMode::Check { .. })\n    }\n\n    /// Returns `true` if this is generating documentation.\n    pub fn is_doc(self) -> bool {\n        matches!(self, CompileMode::Doc { .. })\n    }\n\n    /// Returns `true` if this a doc test.\n    pub fn is_doc_test(self) -> bool {\n        self == CompileMode::Doctest\n    }\n\n    /// Returns `true` if this is scraping examples for documentation.\n    pub fn is_doc_scrape(self) -> bool {\n        self == CompileMode::Docscrape\n    }\n\n    /// Returns `true` if this is any type of test (test, benchmark, doc test, or\n    /// check test).\n    pub fn is_any_test(self) -> bool {\n        matches!(\n            self,\n            CompileMode::Test\n                | CompileMode::Bench\n                | CompileMode::Check { test: true }\n                | CompileMode::Doctest\n        )\n    }\n\n    /// Returns `true` if this is something that passes `--test` to rustc.\n    pub fn is_rustc_test(self) -> bool {\n        matches!(\n            self,\n            CompileMode::Test | CompileMode::Bench | CompileMode::Check { test: true }\n        )\n    }\n\n    /// Returns `true` if this is the *execution* of a `build.rs` script.\n    pub fn is_run_custom_build(self) -> bool {\n        self == CompileMode::RunCustomBuild\n    }\n\n    /// Returns `true` if this mode may generate an executable.\n    ///\n    /// Note that this also returns `true` for building libraries, so you also\n    /// have to check the target.\n    pub fn generates_executable(self) -> bool {\n        matches!(\n            self,\n            CompileMode::Test | CompileMode::Bench | CompileMode::Build\n        )\n    }\n}\n\n/// Kinds of build timings we can output.\n#[derive(Clone, Copy, PartialEq, Debug, Eq, Hash, PartialOrd, Ord)]\npub enum TimingOutput {\n    /// Human-readable HTML report\n    Html,\n    /// Machine-readable JSON (unstable)\n    Json,\n}\n\n// Path: src/cargo/core/profiles.rs\n//! Handles built-in and customizable compiler flag presets.\n//!\n//! [`Profiles`] is a collections of built-in profiles, and profiles defined\n//! in the root manifest and configurations.\n//!\n//! To start using a profile, most of the time you start from [`Profiles::new`],\n//! which does the followings:\n//!\n//! - Create a `Profiles` by merging profiles from configs onto the profile\n//!   from root manifest (see [`merge_config_profiles`]).\n//! - Add built-in profiles onto it (see [`Profiles::add_root_profiles`]).\n//! - Process profile inheritance for each profiles. (see [`Profiles::add_maker`]).\n//!\n//! Then you can query a [`Profile`] via [`Profiles::get_profile`], which respects\n//! the profile overridden hierarchy described in below. The [`Profile`] you get\n//! is basically an immutable struct containing the compiler flag presets.\n//!\n//! ## Profile overridden hierarchy\n//!\n//! Profile settings can be overridden for specific packages and build-time crates.\n//! The precedence is explained in [`ProfileMaker`].\n//! The algorithm happens within [`ProfileMaker::get_profile`].\n\nuse crate::core::compiler::{CompileKind, CompileTarget, Unit};\nuse crate::core::dependency::Artifact;\nuse crate::core::resolver::features::FeaturesFor;\nuse crate::core::Feature;\nuse crate::core::{\n    PackageId, PackageIdSpec, PackageIdSpecQuery, Resolve, Shell, Target, Workspace,\n};\nuse crate::util::interning::InternedString;\nuse crate::util::toml::validate_profile;\nuse crate::util::{closest_msg, context, CargoResult, GlobalContext};\nuse anyhow::{bail, Context as _};\nuse cargo_util_schemas::manifest::TomlTrimPaths;\nuse cargo_util_schemas::manifest::TomlTrimPathsValue;\nuse cargo_util_schemas::manifest::{\n    ProfilePackageSpec, StringOrBool, TomlDebugInfo, TomlProfile, TomlProfiles,\n};\nuse std::collections::{BTreeMap, HashMap, HashSet};\nuse std::hash::Hash;\nuse std::{cmp, fmt, hash};\n\n/// Collection of all profiles.\n///\n/// To get a specific [`Profile`], you usually create this and call [`get_profile`] then.\n///\n/// [`get_profile`]: Profiles::get_profile\n#[derive(Clone, Debug)]\npub struct Profiles {\n    /// Incremental compilation can be overridden globally via:\n    /// - `CARGO_INCREMENTAL` environment variable.\n    /// - `build.incremental` config value.\n    incremental: Option<bool>,\n    /// Map of profile name to directory name for that profile.\n    dir_names: HashMap<InternedString, InternedString>,\n    /// The profile makers. Key is the profile name.\n    by_name: HashMap<InternedString, ProfileMaker>,\n    /// The original profiles written by the user in the manifest and config.\n    ///\n    /// This is here to assist with error reporting, as the `ProfileMaker`\n    /// values have the inherits chains all merged together.\n    original_profiles: BTreeMap<InternedString, TomlProfile>,\n    /// The profile the user requested to use.\n    requested_profile: InternedString,\n    /// The host target for rustc being used by this `Profiles`.\n    rustc_host: InternedString,\n}\n\nimpl Profiles {\n    pub fn new(ws: &Workspace<'_>, requested_profile: InternedString) -> CargoResult<Profiles> {\n        let gctx = ws.gctx();\n        let incremental = match gctx.get_env_os(\"CARGO_INCREMENTAL\") {\n            Some(v) => Some(v == \"1\"),\n            None => gctx.build_config()?.incremental,\n        };\n        let mut profiles = merge_config_profiles(ws, requested_profile)?;\n        let rustc_host = ws.gctx().load_global_rustc(Some(ws))?.host;\n\n        let mut profile_makers = Profiles {\n            incremental,\n            dir_names: Self::predefined_dir_names(),\n            by_name: HashMap::new(),\n            original_profiles: profiles.clone(),\n            requested_profile,\n            rustc_host,\n        };\n\n        let trim_paths_enabled = ws.unstable_features().is_enabled(Feature::trim_paths())\n            || gctx.cli_unstable().trim_paths;\n        Self::add_root_profiles(&mut profile_makers, &profiles, trim_paths_enabled);\n\n        // Merge with predefined profiles.\n        use std::collections::btree_map::Entry;\n        for (predef_name, mut predef_prof) in Self::predefined_profiles().into_iter() {\n            match profiles.entry(InternedString::new(predef_name)) {\n                Entry::Vacant(vac) => {\n                    vac.insert(predef_prof);\n                }\n                Entry::Occupied(mut oc) => {\n                    // Override predefined with the user-provided Toml.\n                    let r = oc.get_mut();\n                    predef_prof.merge(r);\n                    *r = predef_prof;\n                }\n            }\n        }\n\n        for (name, profile) in &profiles {\n            profile_makers.add_maker(*name, profile, &profiles)?;\n        }\n        // Verify that the requested profile is defined *somewhere*.\n        // This simplifies the API (no need for CargoResult), and enforces\n        // assumptions about how config profiles are loaded.\n        profile_makers.get_profile_maker(&requested_profile)?;\n        Ok(profile_makers)\n    }\n\n    /// Returns the hard-coded directory names for built-in profiles.\n    \nfn predefined_dir_names() -> HashMap<InternedString, InternedString> {\n        [\n            (InternedString::new(\"dev\"), InternedString::new(\"debug\")),\n            (InternedString::new(\"test\"), InternedString::new(\"debug\")),\n            (InternedString::new(\"bench\"), InternedString::new(\"release\")),\n        ]\n        .into()\n    }\n\n    /// Initialize `by_name` with the two \"root\" profiles, `dev`, and\n    /// `release` given the user's definition.\n    fn add_root_profiles(\n        profile_makers: &mut Profiles,\n        profiles: &BTreeMap<InternedString, TomlProfile>,\n        trim_paths_enabled: bool,\n    ) {\n        profile_makers.by_name.insert(\n            InternedString::new(\"dev\"),\n            ProfileMaker::new(Profile::default_dev(), profiles.get(\"dev\").cloned()),\n        );\n\n        profile_makers.by_name.insert(\n            InternedString::new(\"release\"),\n            ProfileMaker::new(\n                Profile::default_release(trim_paths_enabled),\n                profiles.get(\"release\").cloned(),\n            ),\n        );\n    }\n\n    /// Returns the built-in profiles (not including dev/release, which are\n    /// \"root\" profiles).\n    fn predefined_profiles() -> Vec<(&'static str, TomlProfile)> {\n        vec![\n            (\n                \"bench\",\n                TomlProfile {\n                    inherits: Some(String::from(\"release\")),\n                    ..TomlProfile::default()\n                },\n            ),\n            (\n                \"test\",\n                TomlProfile {\n                    inherits: Some(String::from(\"dev\")),\n                    ..TomlProfile::default()\n                },\n            ),\n            (\n                \"doc\",\n                TomlProfile {\n                    inherits: Some(String::from(\"dev\")),\n                    ..TomlProfile::default()\n                },\n            ),\n        ]\n    }\n\n    /// Creates a `ProfileMaker`, and inserts it into `self.by_name`.\n    fn add_maker(\n        &mut self,\n        name: InternedString,\n        profile: &TomlProfile,\n        profiles: &BTreeMap<InternedString, TomlProfile>,\n    ) -> CargoResult<()> {\n        match &profile.dir_name {\n            None => {}\n            Some(dir_name) => {\n                self.dir_names.insert(name, InternedString::new(dir_name));\n            }\n        }\n\n        // dev/release are \"roots\" and don't inherit.\n        if name == \"dev\" || name == \"release\" {\n            if profile.inherits.is_some() {\n                bail!(\n                    \"`inherits` must not be specified in root profile `{}`\",\n                    name\n                );\n            }\n            // Already inserted from `add_root_profiles`, no need to do anything.\n            return Ok(());\n        }\n\n        // Keep track for inherits cycles.\n        let mut set = HashSet::new();\n        set.insert(name);\n        let maker = self.process_chain(name, profile, &mut set, profiles)?;\n        self.by_name.insert(name, maker);\n        Ok(())\n    }\n\n    /// Build a `ProfileMaker` by recursively following the `inherits` setting.\n    ///\n    /// * `name`: The name of the profile being processed.\n    /// * `profile`: The TOML profile being processed.\n    /// * `set`: Set of profiles that have been visited, used to detect cycles.\n    /// * `profiles`: Map of all TOML profiles.\n    ///\n    /// Returns a `ProfileMaker` to be used for the given named profile.\n    fn process_chain(\n        &mut self,\n        name: InternedString,\n        profile: &TomlProfile,\n        set: &mut HashSet<InternedString>,\n        profiles: &BTreeMap<InternedString, TomlProfile>,\n    ) -> CargoResult<ProfileMaker> {\n        let mut maker = match &profile.inherits {\n            Some(inherits_name) if inherits_name == \"dev\" || inherits_name == \"release\" => {\n                // These are the root profiles added in `add_root_profiles`.\n                self.get_profile_maker(&inherits_name).unwrap().clone()\n            }\n            Some(inherits_name) => {\n                let inherits_name = InternedString::new(&inherits_name);\n                if !set.insert(inherits_name) {\n                    bail!(\n                        \"profile inheritance loop detected with profile `{}` inheriting `{}`\",\n                        name,\n                        inherits_name\n                    );\n                }\n\n                match profiles.get(&inherits_name) {\n                    None => {\n                        bail!(\n                            \"profile `{}` inherits from `{}`, but that profile is not defined\",\n                            name,\n                            inherits_name\n                        );\n                    }\n                    Some(parent) => self.process_chain(inherits_name, parent, set, profiles)?,\n                }\n            }\n            None => {\n                bail!(\n                    \"profile `{}` is missing an `inherits` directive \\\n                     (`inherits` is required for all profiles except `dev` or `release`)\",\n                    name\n                );\n            }\n        };\n        match &mut maker.toml {\n            Some(toml) => toml.merge(profile),\n            None => maker.toml = Some(profile.clone()),\n        };\n        Ok(maker)\n    }\n\n    /// Retrieves the profile for a target.\n    /// `is_member` is whether or not this package is a member of the\n    /// workspace.\n    pub fn get_profile(\n        &self,\n        pkg_id: PackageId,\n        is_member: bool,\n        is_local: bool,\n        unit_for: UnitFor,\n        kind: CompileKind,\n    ) -> Profile {\n        let maker = self.get_profile_maker(&self.requested_profile).unwrap();\n        let mut profile = maker.get_profile(Some(pkg_id), is_member, unit_for.is_for_host());\n\n        // Dealing with `panic=abort` and `panic=unwind` requires some special\n        // treatment. Be sure to process all the various options here.\n        match unit_for.panic_setting() {\n            PanicSetting::AlwaysUnwind => profile.panic = PanicStrategy::Unwind,\n            PanicSetting::ReadProfile => {}\n        }\n\n        // Default macOS debug information to being stored in the \"unpacked\"\n        // split-debuginfo format. At the time of this writing that's the only\n        // platform which has a stable `-Csplit-debuginfo` option for rustc,\n        // and it's typically much faster than running `dsymutil` on all builds\n        // in incremental cases.\n        if profile.debuginfo.is_turned_on() && profile.split_debuginfo.is_none() {\n            let target = match &kind {\n                CompileKind::Host => self.rustc_host.as_str(),\n                CompileKind::Target(target) => target.short_name(),\n            };\n            if target.contains(\"-apple-\") {\n                profile.split_debuginfo = Some(InternedString::new(\"unpacked\"));\n            }\n        }\n\n        // Incremental can be globally overridden.\n        if let Some(v) = self.incremental {\n            profile.incremental = v;\n        }\n\n        // Only enable incremental compilation for sources the user can\n        // modify (aka path sources). For things that change infrequently,\n        // non-incremental builds yield better performance in the compiler\n        // itself (aka crates.io / git dependencies)\n        //\n        // (see also https://github.com/rust-lang/cargo/issues/3972)\n        if !is_local {\n            profile.incremental = false;\n        }\n        profile.name = self.requested_profile;\n        profile\n    }\n\n    /// The profile for *running* a `build.rs` script is only used for setting\n    /// a few environment variables. To ensure proper de-duplication of the\n    /// running `Unit`, this uses a stripped-down profile (so that unrelated\n    /// profile flags don't cause `build.rs` to needlessly run multiple\n    /// times).\n    pub fn get_profile_run_custom_build(&self, for_unit_profile: &Profile) -> Profile {\n        let mut result = Profile::default();\n        result.name = for_unit_profile.name;\n        result.root = for_unit_profile.root;\n        result.debuginfo = for_unit_profile.debuginfo;\n        result.opt_level = for_unit_profile.opt_level;\n        result.trim_paths = for_unit_profile.trim_paths.clone();\n        result\n    }\n\n    /// This returns the base profile. This is currently used for the\n    /// `[Finished]` line. It is not entirely accurate, since it doesn't\n    /// select for the package that was actually built.\n    pub fn base_profile(&self) -> Profile {\n        let profile_name = self.requested_profile;\n        let maker = self.get_profile_maker(&profile_name).unwrap();\n        maker.get_profile(None, /*is_member*/ true, /*is_for_host*/ false)\n    }\n\n    /// Gets the directory name for a profile, like `debug` or `release`.\n    pub fn get_dir_name(&self) -> InternedString {\n        *self\n            .dir_names\n            .get(&self.requested_profile)\n            .unwrap_or(&self.requested_profile)\n    }\n\n    /// Used to check for overrides for non-existing packages.\n    pub fn validate_packages(\n        &self,\n        profiles: Option<&TomlProfiles>,\n        shell: &mut Shell,\n        resolve: &Resolve,\n    ) -> CargoResult<()> {\n        for (name, profile) in &self.by_name {\n            // If the user did not specify an override, skip this. This is here\n            // to avoid generating errors for inherited profiles which don't\n            // specify package overrides. The `by_name` profile has had the inherits\n            // chain merged, so we need to look at the original source to check\n            // if an override was specified.\n            if self\n                .original_profiles\n                .get(name)\n                .and_then(|orig| orig.package.as_ref())\n                .is_none()\n            {\n                continue;\n            }\n            let found = validate_packages_unique(resolve, name, &profile.toml)?;\n            // We intentionally do not validate unmatched packages for config\n            // profiles, in case they are defined in a central location. This\n            // iterates over the manifest profiles only.\n            if let Some(profiles) = profiles {\n                if let Some(toml_profile) = profiles.get(name) {\n                    validate_packages_unmatched(shell, resolve, name, toml_profile, &found)?;\n                }\n            }\n        }\n        Ok(())\n    }\n\n    /// Returns the profile maker for the given profile name.\n    fn get_profile_maker(&self, name: &str) -> CargoResult<&ProfileMaker> {\n        self.by_name\n            .get(name)\n            .ok_or_else(|| anyhow::format_err!(\"profile `{}` is not defined\", name))\n    }\n}\n\n/// An object used for handling the profile hierarchy.\n///\n/// The precedence of profiles are (first one wins):\n///\n/// - Profiles in `.cargo/config` files (using same order as below).\n/// - `[profile.dev.package.name]` -- a named package.\n/// - `[profile.dev.package.\"*\"]` -- this cannot apply to workspace members.\n/// - `[profile.dev.build-override]` -- this can only apply to `build.rs` scripts\n///   and their dependencies.\n/// - `[profile.dev]`\n/// - Default (hard-coded) values.\n#[derive(Debug, Clone)]\nstruct ProfileMaker {\n    /// The starting, hard-coded defaults for the profile.\n    default: Profile,\n    /// The TOML profile defined in `Cargo.toml` or config.\n    ///\n    /// This is None if the user did not specify one, in which case the\n    /// `default` is used. Note that the built-in defaults for test/bench/doc\n    /// always set this since they need to declare the `inherits` value.\n    toml: Option<TomlProfile>,\n}\n\nimpl ProfileMaker {\n    /// Creates a new `ProfileMaker`.\n    ///\n    /// Note that this does not process `inherits`, the caller is responsible for that.\n    fn new(default: Profile, toml: Option<TomlProfile>) -> ProfileMaker {\n        ProfileMaker { default, toml }\n    }\n\n    /// Generates a new `Profile`.\n    fn get_profile(\n        &self,\n        pkg_id: Option<PackageId>,\n        is_member: bool,\n        is_for_host: bool,\n    ) -> Profile {\n        let mut profile = self.default.clone();\n\n        // First apply profile-specific settings, things like\n        // `[profile.release]`\n        if let Some(toml) = &self.toml {\n            merge_profile(&mut profile, toml);\n        }\n\n        // Next start overriding those settings. First comes build dependencies\n        // which default to opt-level 0...\n        if is_for_host {\n            // For-host units are things like procedural macros, build scripts, and\n            // their dependencies. For these units most projects simply want them\n            // to compile quickly and the runtime doesn't matter too much since\n            // they tend to process very little data. For this reason we default\n            // them to a \"compile as quickly as possible\" mode which for now means\n            // basically turning down the optimization level and avoid limiting\n            // codegen units. This ensures that we spend little time optimizing as\n            // well as enabling parallelism by not constraining codegen units.\n            profile.opt_level = InternedString::new(\"0\");\n            profile.codegen_units = None;\n\n            // For build dependencies, we usually don't need debuginfo, and\n            // removing it will compile faster. However, that can conflict with\n            // a unit graph optimization, reusing units that are shared between\n            // build dependencies and runtime dependencies: when the runtime\n            // target is the same as the build host, we only need to build a\n            // dependency once and reuse the results, instead of building twice.\n            // We defer the choice of the debuginfo level until we can check if\n            // a unit is shared. If that's the case, we'll use the deferred value\n            // below so the unit can be reused, otherwise we can avoid emitting\n            // the unit's debuginfo.\n            profile.debuginfo = DebugInfo::Deferred(profile.debuginfo.into_inner());\n        }\n        // ... and next comes any other sorts of overrides specified in\n        // profiles, such as `[profile.release.build-override]` or\n        // `[profile.release.package.foo]`\n        if let Some(toml) = &self.toml {\n            merge_toml_overrides(pkg_id, is_member, is_for_host, &mut profile, toml);\n        }\n        profile\n    }\n}\n\n/// Merge package and build overrides from the given TOML profile into the given `Profile`.\nfn merge_toml_overrides(\n    pkg_id: Option<PackageId>,\n    is_member: bool,\n    is_for_host: bool,\n    profile: &mut Profile,\n    toml: &TomlProfile,\n) {\n    if is_for_host {\n        if let Some(build_override) = &toml.build_override {\n            merge_profile(profile, build_override);\n        }\n    }\n    if let Some(overrides) = toml.package.as_ref() {\n        if !is_member {\n            if let Some(all) = overrides.get(&ProfilePackageSpec::All) {\n                merge_profile(profile, all);\n            }\n        }\n        if let Some(pkg_id) = pkg_id {\n            let mut matches = overrides\n                .iter()\n                .filter_map(|(key, spec_profile)| match *key {\n                    ProfilePackageSpec::All => None,\n                    ProfilePackageSpec::Spec(ref s) => {\n                        if s.matches(pkg_id) {\n                            Some(spec_profile)\n                        } else {\n                            None\n                        }\n                    }\n                });\n            if let Some(spec_profile) = matches.next() {\n                merge_profile(profile, spec_profile);\n                // `validate_packages` should ensure that there are\n                // no additional matches.\n                assert!(\n                    matches.next().is_none(),\n                    \"package `{}` matched multiple package profile overrides\",\n                    pkg_id\n                );\n            }\n        }\n    }\n}\n\n/// Merge the given TOML profile into the given `Profile`.\n///\n/// Does not merge overrides (see `merge_toml_overrides`).\nfn merge_profile(profile: &mut Profile, toml: &TomlProfile) {\n    if let Some(ref opt_level) = toml.opt_level {\n        profile.opt_level = InternedString::new(&opt_level.0);\n    }\n    match toml.lto {\n        Some(StringOrBool::Bool(b)) => profile.lto = Lto::Bool(b),\n        Some(StringOrBool::String(ref n)) if is_off(n.as_str()) => profile.lto = Lto::Off,\n        Some(StringOrBool::String(ref n)) => profile.lto = Lto::Named(InternedString::new(n)),\n        None => {}\n    }\n    if toml.codegen_backend.is_some() {\n        profile.codegen_backend = toml.codegen_backend.as_ref().map(InternedString::from);\n    }\n    if toml.codegen_units.is_some() {\n        profile.codegen_units = toml.codegen_units;\n    }\n    if let Some(debuginfo) = toml.debug {\n        profile.debuginfo = DebugInfo::Resolved(debuginfo);\n    }\n    if let Some(debug_assertions) = toml.debug_assertions {\n        profile.debug_assertions = debug_assertions;\n    }\n    if let Some(split_debuginfo) = &toml.split_debuginfo {\n        profile.split_debuginfo = Some(InternedString::new(split_debuginfo));\n    }\n    if let Some(rpath) = toml.rpath {\n        profile.rpath = rpath;\n    }\n    if let Some(panic) = &toml.panic {\n        profile.panic = match panic.as_str() {\n            \"unwind\" => PanicStrategy::Unwind,\n            \"abort\" => PanicStrategy::Abort,\n            // This should be validated in TomlProfile::validate\n            _ => panic!(\"Unexpected panic setting `{}`\", panic),\n        };\n    }\n    if let Some(overflow_checks) = toml.overflow_checks {\n        profile.overflow_checks = overflow_checks;\n    }\n    if let Some(incremental) = toml.incremental {\n        profile.incremental = incremental;\n    }\n    if let Some(flags) = &toml.rustflags {\n        profile.rustflags = flags.iter().map(InternedString::from).collect();\n    }\n    if let Some(trim_paths) = &toml.trim_paths {\n        profile.trim_paths = Some(trim_paths.clone());\n    }\n    profile.strip = match toml.strip {\n        Some(StringOrBool::Bool(true)) => {\n            Strip::Resolved(StripInner::Named(InternedString::new(\"symbols\")))\n        }\n        Some(StringOrBool::Bool(false)) => Strip::Resolved(StripInner::None),\n        Some(StringOrBool::String(ref n)) if n.as_str() == \"none\" => {\n            Strip::Resolved(StripInner::None)\n        }\n        Some(StringOrBool::String(ref n)) => {\n            Strip::Resolved(StripInner::Named(InternedString::new(n)))\n        }\n        None => Strip::Deferred(StripInner::None),\n    };\n}\n\n/// The root profile (dev/release).\n///\n/// This is currently only used for the `PROFILE` env var for build scripts\n/// for backwards compatibility. We should probably deprecate `PROFILE` and\n/// encourage using things like `DEBUG` and `OPT_LEVEL` instead.\n#[derive(Clone, Copy, Eq, PartialOrd, Ord, PartialEq, Debug)]\npub enum ProfileRoot {\n    Release,\n    Debug,\n}\n\n/// Profile settings used to determine which compiler flags to use for a\n/// target.\n#[derive(Clone, Eq, PartialOrd, Ord, serde::Serialize)]\npub struct Profile {\n    pub name: InternedString,\n    pub opt_level: InternedString,\n    #[serde(skip)] // named profiles are unstable\n    pub root: ProfileRoot,\n    pub lto: Lto,\n    // `None` means use rustc default.\n    pub codegen_backend: Option<InternedString>,\n    // `None` means use rustc default.\n    pub codegen_units: Option<u32>,\n    pub debuginfo: DebugInfo,\n    pub split_debuginfo: Option<InternedString>,\n    pub debug_assertions: bool,\n    pub overflow_checks: bool,\n    pub rpath: bool,\n    pub incremental: bool,\n    pub panic: PanicStrategy,\n    pub strip: Strip,\n    #[serde(skip_serializing_if = \"Vec::is_empty\")] // remove when `rustflags` is stablized\n    // Note that `rustflags` is used for the cargo-feature `profile_rustflags`\n    pub rustflags: Vec<InternedString>,\n    // remove when `-Ztrim-paths` is stablized\n    #[serde(skip_serializing_if = \"Option::is_none\")]\n    pub trim_paths: Option<TomlTrimPaths>,\n}\n\nimpl Default for Profile {\n    fn default() -> Profile {\n        Profile {\n            name: InternedString::new(\"\"),\n            opt_level: InternedString::new(\"0\"),\n            root: ProfileRoot::Debug,\n            lto: Lto::Bool(false),\n            codegen_backend: None,\n            codegen_units: None,\n            debuginfo: DebugInfo::Resolved(TomlDebugInfo::None),\n            debug_assertions: false,\n            split_debuginfo: None,\n            overflow_checks: false,\n            rpath: false,\n            incremental: false,\n            panic: PanicStrategy::Unwind,\n            strip: Strip::Deferred(StripInner::None),\n            rustflags: vec![],\n            trim_paths: None,\n        }\n    }\n}\n\ncompact_debug! {\n    impl fmt::Debug for Profile {\n        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n            let (default, default_name) = match self.name.as_str() {\n                \"dev\" => (Profile::default_dev(), \"default_dev()\"),\n                \"release\" => (Profile::default_release(false), \"default_release()\"),\n                _ => (Profile::default(), \"default()\"),\n            };\n            [debug_the_fields(\n                name\n                opt_level\n                lto\n                root\n                codegen_backend\n                codegen_units\n                debuginfo\n                split_debuginfo\n                debug_assertions\n                overflow_checks\n                rpath\n                incremental\n                panic\n                strip\n                rustflags\n                trim_paths\n            )]\n        }\n    }\n}\n\nimpl fmt::Display for Profile {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Profile({})\", self.name)\n    }\n}\n\nimpl hash::Hash for Profile {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: hash::Hasher,\n    {\n        self.comparable().hash(state);\n    }\n}\n\nimpl cmp::PartialEq for Profile {\n    fn eq(&self, other: &Self) -> bool {\n        self.comparable() == other.comparable()\n    }\n}\n\nimpl Profile {\n    /// Returns a built-in `dev` profile.\n    fn default_dev() -> Profile {\n        Profile {\n            name: InternedString::new(\"dev\"),\n            root: ProfileRoot::Debug,\n            debuginfo: DebugInfo::Resolved(TomlDebugInfo::Full),\n            debug_assertions: true,\n            overflow_checks: true,\n            incremental: true,\n            ..Profile::default()\n        }\n    }\n\n    /// Returns a built-in `release` profile.\n    fn default_release(trim_paths_enabled: bool) -> Profile {\n        let trim_paths = trim_paths_enabled.then(|| TomlTrimPathsValue::Object.into());\n        Profile {\n            name: InternedString::new(\"release\"),\n            root: ProfileRoot::Release,\n            opt_level: InternedString::new(\"3\"),\n            trim_paths,\n            ..Profile::default()\n        }\n    }\n\n    /// Compares all fields except `name`, which doesn't affect compilation.\n    /// This is necessary for `Unit` deduplication for things like \"test\" and\n    /// \"dev\" which are essentially the same.\n    fn comparable(&self) -> impl Hash + Eq + '_ {\n        (\n            self.opt_level,\n            self.lto,\n            self.codegen_backend,\n            self.codegen_units,\n            self.debuginfo,\n            self.split_debuginfo,\n            self.debug_assertions,\n            self.overflow_checks,\n            self.rpath,\n            (self.incremental, self.panic, self.strip),\n            &self.rustflags,\n            &self.trim_paths,\n        )\n    }\n}\n\n/// The debuginfo level setting.\n///\n/// This is semantically a [`TomlDebugInfo`], and should be used as so via the\n/// [`DebugInfo::into_inner`] method for all intents and purposes.\n///\n/// Internally, it's used to model a debuginfo level whose value can be deferred\n/// for optimization purposes: host dependencies usually don't need the same\n/// level as target dependencies. For dependencies that are shared between the\n/// two however, that value also affects reuse: different debuginfo levels would\n/// cause to build a unit twice. By deferring the choice until we know\n/// whether to choose the optimized value or the default value, we can make sure\n/// the unit is only built once and the unit graph is still optimized.\n#[derive(Debug, Copy, Clone, serde::Serialize)]\n#[serde(untagged)]\npub enum DebugInfo {\n    /// A debuginfo level that is fixed and will not change.\n    ///\n    /// This can be set by a profile, user, or default value.\n    Resolved(TomlDebugInfo),\n    /// For internal purposes: a deferred debuginfo level that can be optimized\n    /// away, but has this value otherwise.\n    ///\n    /// Behaves like `Resolved` in all situations except for the default build\n    /// dependencies profile: whenever a build dependency is not shared with\n    /// runtime dependencies, this level is weakened to a lower level that is\n    /// faster to build (see [`DebugInfo::weaken`]).\n    ///\n    /// In all other situations, this level value will be the one to use.\n    Deferred(TomlDebugInfo),\n}\n\nimpl DebugInfo {\n    /// The main way to interact with this debuginfo level, turning it into a [`TomlDebugInfo`].\n    pub fn into_inner(self) -> TomlDebugInfo {\n        match self {\n            DebugInfo::Resolved(v) | DebugInfo::Deferred(v) => v,\n        }\n    }\n\n    /// Returns true if any debuginfo will be generated. Helper\n    /// for a common operation on the usual `Option` representation.\n    pub(crate) fn is_turned_on(&self) -> bool {\n        !matches!(self.into_inner(), TomlDebugInfo::None)\n    }\n\n    pub(crate) fn is_deferred(&self) -> bool {\n        matches!(self, DebugInfo::Deferred(_))\n    }\n\n    /// Force the deferred, preferred, debuginfo level to a finalized explicit value.\n    pub(crate) fn finalize(self) -> Self {\n        match self {\n            DebugInfo::Deferred(v) => DebugInfo::Resolved(v),\n            _ => self,\n        }\n    }\n\n    /// Reset to the lowest level: no debuginfo.\n    pub(crate) fn weaken(self) -> Self {\n        DebugInfo::Resolved(TomlDebugInfo::None)\n    }\n}\n\nimpl PartialEq for DebugInfo {\n    fn eq(&self, other: &DebugInfo) -> bool {\n        self.into_inner().eq(&other.into_inner())\n    }\n}\n\nimpl Eq for DebugInfo {}\n\nimpl Hash for DebugInfo {\n    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {\n        self.into_inner().hash(state);\n    }\n}\n\nimpl PartialOrd for DebugInfo {\n    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {\n        self.into_inner().partial_cmp(&other.into_inner())\n    }\n}\n\nimpl Ord for DebugInfo {\n    fn cmp(&self, other: &Self) -> std::cmp::Ordering {\n        self.into_inner().cmp(&other.into_inner())\n    }\n}\n\n/// The link-time-optimization setting.\n#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, PartialOrd, Ord)]\npub enum Lto {\n    /// Explicitly no LTO, disables thin-LTO.\n    Off,\n    /// True = \"Fat\" LTO\n    /// False = rustc default (no args), currently \"thin LTO\"\n    Bool(bool),\n    /// Named LTO settings like \"thin\".\n    Named(InternedString),\n}\n\nimpl serde::ser::Serialize for Lto {\n    fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>\n    where\n        S: serde::ser::Serializer,\n    {\n        match self {\n            Lto::Off => \"off\".serialize(s),\n            Lto::Bool(b) => b.to_string().serialize(s),\n            Lto::Named(n) => n.serialize(s),\n        }\n    }\n}\n\n/// The `panic` setting.\n#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, PartialOrd, Ord, serde::Serialize)]\n#[serde(rename_all = \"lowercase\")]\npub enum PanicStrategy {\n    Unwind,\n    Abort,\n}\n\nimpl fmt::Display for PanicStrategy {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match *self {\n            PanicStrategy::Unwind => \"unwind\",\n            PanicStrategy::Abort => \"abort\",\n        }\n        .fmt(f)\n    }\n}\n\n#[derive(\n    Clone, Copy, PartialEq, Eq, Debug, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,\n)]\npub enum StripInner {\n    /// Don't remove any symbols\n    None,\n    /// Named Strip settings\n    Named(InternedString),\n}\n\nimpl fmt::Display for StripInner {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match *self {\n            StripInner::None => \"none\",\n            StripInner::Named(s) => s.as_str(),\n        }\n        .fmt(f)\n    }\n}\n\n/// The setting for choosing which symbols to strip.\n///\n/// This is semantically a [`StripInner`], and should be used as so via the\n/// [`Strip::into_inner`] method for all intents and purposes.\n///\n/// Internally, it's used to model a strip option whose value can be deferred\n/// for optimization purposes: when no package being compiled requires debuginfo,\n/// then we can strip debuginfo to remove pre-existing debug symbols from the\n/// standard library.\n#[derive(Clone, Copy, Debug, Eq, serde::Serialize, serde::Deserialize)]\n#[serde(rename_all = \"lowercase\")]\npub enum Strip {\n    /// A strip option that is fixed and will not change.\n    Resolved(StripInner),\n    /// A strip option that might be overridden by Cargo for optimization\n    /// purposes.\n    Deferred(StripInner),\n}\n\nimpl Strip {\n    /// The main way to interact with this strip option, turning it into a [`StripInner`].\n    pub fn into_inner(self) -> StripInner {\n        match self {\n            Strip::Resolved(v) | Strip::Deferred(v) => v,\n        }\n    }\n\n    pub(crate) fn is_deferred(&self) -> bool {\n        matches!(self, Strip::Deferred(_))\n    }\n\n    /// Reset to stripping debuginfo.\n    pub(crate) fn strip_debuginfo(self) -> Self {\n        Strip::Resolved(StripInner::Named(\"debuginfo\".into()))\n    }\n}\n\nimpl PartialEq for Strip {\n    fn eq(&self, other: &Self) -> bool {\n        self.into_inner().eq(&other.into_inner())\n    }\n}\n\nimpl Hash for Strip {\n    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {\n        self.into_inner().hash(state);\n    }\n}\n\nimpl PartialOrd for Strip {\n    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {\n        self.into_inner().partial_cmp(&other.into_inner())\n    }\n}\n\nimpl Ord for Strip {\n    fn cmp(&self, other: &Self) -> std::cmp::Ordering {\n        self.into_inner().cmp(&other.into_inner())\n    }\n}\n\n/// Flags used in creating `Unit`s to indicate the purpose for the target, and\n/// to ensure the target's dependencies have the correct settings.\n///\n/// This means these are passed down from the root of the dependency tree to apply\n/// to most child dependencies.\n#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]\npub struct UnitFor {\n    /// A target for `build.rs` or any of its dependencies, or a proc-macro or\n    /// any of its dependencies. This enables `build-override` profiles for\n    /// these targets.\n    ///\n    /// An invariant is that if `host_features` is true, `host` must be true.\n    ///\n    /// Note that this is `true` for `RunCustomBuild` units, even though that\n    /// unit should *not* use build-override profiles. This is a bit of a\n    /// special case. When computing the `RunCustomBuild` unit, it manually\n    /// uses the `get_profile_run_custom_build` method to get the correct\n    /// profile information for the unit. `host` needs to be true so that all\n    /// of the dependencies of that `RunCustomBuild` unit have this flag be\n    /// sticky (and forced to `true` for all further dependencies) \u2014 which is\n    /// the whole point of `UnitFor`.\n    host: bool,\n    /// A target for a build dependency or proc-macro (or any of its\n    /// dependencies). This is used for computing features of build\n    /// dependencies and proc-macros independently of other dependency kinds.\n    ///\n...\n", "needle_token_start": 7263, "needle_token_end": 7358, "code_context_ntokens": 16270, "output": ["```rust\nfn predefined_dir_names() -> HashMap<InternedString, InternedString> {\n    [\n        (InternedString::new(\"dev\"), InternedString::new(\"debug\")),\n        (InternedString::new(\"test\"), InternedString::new(\"debug\")),\n        (InternedString::new(\"bench\"), InternedString::new(\"release\")),\n    ]\n    .into()\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "uninstall_one", "language": "rust", "path": "src/cargo/ops/cargo_uninstall.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to remove a specified package and its associated binaries from a given location.\n2. **Input**: It takes a filesystem path, a package specification string, a list of binary names, and a global context object.\n3. **Output**: Returns a result indicating success or failure of the uninstallation process.\n4. **Procedure**: \n   - Loads the installation tracking information from the specified filesystem path.\n   - Retrieves a list of all installed packages and binaries.\n   - Identifies the package ID based on the provided specification string.\n   - Proceeds to uninstall the identified package and specified binaries using the loaded tracking information.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/cargo/ops/common_for_install_and_uninstall.rs\nuse std::collections::{btree_map, BTreeMap, BTreeSet};\nuse std::env;\nuse std::io::prelude::*;\nuse std::io::SeekFrom;\nuse std::path::{Path, PathBuf};\nuse std::rc::Rc;\nuse std::task::Poll;\n\nuse anyhow::{bail, format_err, Context as _};\nuse cargo_util::paths;\nuse cargo_util_schemas::core::PartialVersion;\nuse ops::FilterRule;\nuse serde::{Deserialize, Serialize};\n\nuse crate::core::compiler::{DirtyReason, Freshness};\nuse crate::core::Target;\nuse crate::core::{Dependency, FeatureValue, Package, PackageId, SourceId};\nuse crate::ops::{self, CompileFilter, CompileOptions};\nuse crate::sources::source::QueryKind;\nuse crate::sources::source::Source;\nuse crate::sources::PathSource;\nuse crate::util::cache_lock::CacheLockMode;\nuse crate::util::errors::CargoResult;\nuse crate::util::GlobalContext;\nuse crate::util::{FileLock, Filesystem};\n\n/// On-disk tracking for which package installed which binary.\n///\n/// v1 is an older style, v2 is a new style that tracks more information, and\n/// is both backwards and forwards compatible. Cargo keeps both files in sync,\n/// updating both v1 and v2 at the same time. Additionally, if it detects\n/// changes in v1 that are not in v2 (such as when an older version of Cargo\n/// is used), it will automatically propagate those changes to v2.\n///\n/// This maintains a filesystem lock, preventing other instances of Cargo from\n/// modifying at the same time. Drop the value to unlock.\n///\n/// It is intended that v1 should be retained for a while during a longish\n/// transition period, and then v1 can be removed.\npub struct InstallTracker {\n    v1: CrateListingV1,\n    v2: CrateListingV2,\n    v1_lock: FileLock,\n    v2_lock: FileLock,\n}\n\n/// Tracking information for the set of installed packages.\n#[derive(Default, Deserialize, Serialize)]\nstruct CrateListingV2 {\n    /// Map of every installed package.\n    installs: BTreeMap<PackageId, InstallInfo>,\n    /// Forwards compatibility. Unknown keys from future versions of Cargo\n    /// will be stored here and retained when the file is saved.\n    #[serde(flatten)]\n    other: BTreeMap<String, serde_json::Value>,\n}\n\n/// Tracking information for the installation of a single package.\n///\n/// This tracks the settings that were used when the package was installed.\n/// Future attempts to install the same package will check these settings to\n/// determine if it needs to be rebuilt/reinstalled. If nothing has changed,\n/// then Cargo will inform the user that it is \"up to date\".\n///\n/// This is only used for the v2 format.\n#[derive(Debug, Deserialize, Serialize)]\nstruct InstallInfo {\n    /// Version requested via `--version`.\n    /// None if `--version` not specified. Currently not used, possibly may be\n    /// used in the future.\n    version_req: Option<String>,\n    /// Set of binary names installed.\n    bins: BTreeSet<String>,\n    /// Set of features explicitly enabled.\n    features: BTreeSet<String>,\n    all_features: bool,\n    no_default_features: bool,\n    /// Either \"debug\" or \"release\".\n    profile: String,\n    /// The installation target.\n    /// Either the host or the value specified in `--target`.\n    /// None if unknown (when loading from v1).\n    target: Option<String>,\n    /// Output of `rustc -V`.\n    /// None if unknown (when loading from v1).\n    /// Currently not used, possibly may be used in the future.\n    rustc: Option<String>,\n    /// Forwards compatibility.\n    #[serde(flatten)]\n    other: BTreeMap<String, serde_json::Value>,\n}\n\n/// Tracking information for the set of installed packages.\n#[derive(Default, Deserialize, Serialize)]\npub struct CrateListingV1 {\n    /// Map of installed package id to the set of binary names for that package.\n    v1: BTreeMap<PackageId, BTreeSet<String>>,\n}\n\nimpl InstallTracker {\n    /// Create an InstallTracker from information on disk.\n    pub fn load(gctx: &GlobalContext, root: &Filesystem) -> CargoResult<InstallTracker> {\n        let v1_lock =\n            root.open_rw_exclusive_create(Path::new(\".crates.toml\"), gctx, \"crate metadata\")?;\n        let v2_lock =\n            root.open_rw_exclusive_create(Path::new(\".crates2.json\"), gctx, \"crate metadata\")?;\n\n        let v1 = (|| -> CargoResult<_> {\n            let mut contents = String::new();\n            v1_lock.file().read_to_string(&mut contents)?;\n            if contents.is_empty() {\n                Ok(CrateListingV1::default())\n            } else {\n                Ok(toml::from_str(&contents).with_context(|| \"invalid TOML found for metadata\")?)\n            }\n        })()\n        .with_context(|| {\n            format!(\n                \"failed to parse crate metadata at `{}`\",\n                v1_lock.path().to_string_lossy()\n            )\n        })?;\n\n        let v2 = (|| -> CargoResult<_> {\n            let mut contents = String::new();\n            v2_lock.file().read_to_string(&mut contents)?;\n            let mut v2 = if contents.is_empty() {\n                CrateListingV2::default()\n            } else {\n                serde_json::from_str(&contents)\n                    .with_context(|| \"invalid JSON found for metadata\")?\n            };\n            v2.sync_v1(&v1);\n            Ok(v2)\n        })()\n        .with_context(|| {\n            format!(\n                \"failed to parse crate metadata at `{}`\",\n                v2_lock.path().to_string_lossy()\n            )\n        })?;\n\n        Ok(InstallTracker {\n            v1,\n            v2,\n            v1_lock,\n            v2_lock,\n        })\n    }\n\n    /// Checks if the given package should be built, and checks if executables\n    /// already exist in the destination directory.\n    ///\n    /// Returns a tuple `(freshness, map)`. `freshness` indicates if the\n    /// package should be built (`Dirty`) or if it is already up-to-date\n    /// (`Fresh`) and should be skipped. The map maps binary names to the\n    /// PackageId that installed it (which is None if not known).\n    ///\n    /// If there are no duplicates, then it will be considered `Dirty` (i.e.,\n    /// it is OK to build/install).\n    ///\n    /// `force=true` will always be considered `Dirty` (i.e., it will always\n    /// be rebuilt/reinstalled).\n    ///\n    /// Returns an error if there is a duplicate and `--force` is not used.\n    pub fn check_upgrade(\n        &self,\n        dst: &Path,\n        pkg: &Package,\n        force: bool,\n        opts: &CompileOptions,\n        target: &str,\n        _rustc: &str,\n    ) -> CargoResult<(Freshness, BTreeMap<String, Option<PackageId>>)> {\n        let exes = exe_names(pkg, &opts.filter);\n        // Check if any tracked exe's are already installed.\n        let duplicates = self.find_duplicates(dst, &exes);\n        if force || duplicates.is_empty() {\n            return Ok((Freshness::Dirty(DirtyReason::Forced), duplicates));\n        }\n        // Check if all duplicates come from packages of the same name. If\n        // there are duplicates from other packages, then --force will be\n        // required.\n        //\n        // There may be multiple matching duplicates if different versions of\n        // the same package installed different binaries.\n        //\n        // This does not check the source_id in order to allow the user to\n        // switch between different sources. For example, installing from git,\n        // and then switching to the official crates.io release or vice-versa.\n        // If the source_id were included, then the user would get possibly\n        // confusing errors like \"package `foo 1.0.0` is already installed\"\n        // and the change of source may not be obvious why it fails.\n        let matching_duplicates: Vec<PackageId> = duplicates\n            .values()\n            .filter_map(|v| match v {\n                Some(dupe_pkg_id) if dupe_pkg_id.name() == pkg.name() => Some(*dupe_pkg_id),\n                _ => None,\n            })\n            .collect();\n\n        // If both sets are the same length, that means all duplicates come\n        // from packages with the same name.\n        if matching_duplicates.len() == duplicates.len() {\n            // Determine if it is dirty or fresh.\n            let source_id = pkg.package_id().source_id();\n            if source_id.is_path() {\n                // `cargo install --path ...` is always rebuilt.\n                return Ok((Freshness::Dirty(DirtyReason::Forced), duplicates));\n            }\n            let is_up_to_date = |dupe_pkg_id| {\n                let info = self\n                    .v2\n                    .installs\n                    .get(dupe_pkg_id)\n                    .expect(\"dupes must be in sync\");\n                let precise_equal = if source_id.is_git() {\n                    // Git sources must have the exact same hash to be\n                    // considered \"fresh\".\n                    dupe_pkg_id.source_id().has_same_precise_as(source_id)\n                } else {\n                    true\n                };\n\n                dupe_pkg_id.version() == pkg.version()\n                    && dupe_pkg_id.source_id() == source_id\n                    && precise_equal\n                    && info.is_up_to_date(opts, target, &exes)\n            };\n            if matching_duplicates.iter().all(is_up_to_date) {\n                Ok((Freshness::Fresh, duplicates))\n            } else {\n                Ok((Freshness::Dirty(DirtyReason::Forced), duplicates))\n            }\n        } else {\n            // Format the error message.\n            let mut msg = String::new();\n            for (bin, p) in duplicates.iter() {\n                msg.push_str(&format!(\"binary `{}` already exists in destination\", bin));\n                if let Some(p) = p.as_ref() {\n                    msg.push_str(&format!(\" as part of `{}`\\n\", p));\n                } else {\n                    msg.push('\\n');\n                }\n            }\n            msg.push_str(\"Add --force to overwrite\");\n            bail!(\"{}\", msg);\n        }\n    }\n\n    /// Check if any executables are already installed.\n    ///\n    /// Returns a map of duplicates, the key is the executable name and the\n    /// value is the PackageId that is already installed. The PackageId is\n    /// None if it is an untracked executable.\n    fn find_duplicates(\n        &self,\n        dst: &Path,\n        exes: &BTreeSet<String>,\n    ) -> BTreeMap<String, Option<PackageId>> {\n        exes.iter()\n            .filter_map(|name| {\n                if !dst.join(&name).exists() {\n                    None\n                } else {\n                    let p = self.v2.package_for_bin(name);\n                    Some((name.clone(), p))\n                }\n            })\n            .collect()\n    }\n\n    /// Mark that a package was installed.\n    pub fn mark_installed(\n        &mut self,\n        package: &Package,\n        bins: &BTreeSet<String>,\n        version_req: Option<String>,\n        opts: &CompileOptions,\n        target: &str,\n        rustc: &str,\n    ) {\n        self.v2\n            .mark_installed(package, bins, version_req, opts, target, rustc);\n        self.v1.mark_installed(package, bins);\n    }\n\n    /// Save tracking information to disk.\n    pub fn save(&self) -> CargoResult<()> {\n        self.v1.save(&self.v1_lock).with_context(|| {\n            format!(\n                \"failed to write crate metadata at `{}`\",\n                self.v1_lock.path().to_string_lossy()\n            )\n        })?;\n\n        self.v2.save(&self.v2_lock).with_context(|| {\n            format!(\n                \"failed to write crate metadata at `{}`\",\n                self.v2_lock.path().to_string_lossy()\n            )\n        })?;\n        Ok(())\n    }\n\n    /// Iterator of all installed binaries.\n    /// Items are `(pkg_id, bins)` where `bins` is the set of binaries that\n    /// package installed.\n    pub fn all_installed_bins(&self) -> impl Iterator<Item = (&PackageId, &BTreeSet<String>)> {\n        self.v1.v1.iter()\n    }\n\n    /// Set of binaries installed by a particular package.\n    /// Returns None if the package is not installed.\n    pub fn installed_bins(&self, pkg_id: PackageId) -> Option<&BTreeSet<String>> {\n        self.v1.v1.get(&pkg_id)\n    }\n\n    /// Remove a package from the tracker.\n    pub fn remove(&mut self, pkg_id: PackageId, bins: &BTreeSet<String>) {\n        self.v1.remove(pkg_id, bins);\n        self.v2.remove(pkg_id, bins);\n    }\n\n    /// Remove a bin after it successfully had been removed in disk and then save the tracker at last.\n    pub fn remove_bin_then_save(\n        &mut self,\n        pkg_id: PackageId,\n        bin: &str,\n        bin_path: &PathBuf,\n    ) -> CargoResult<()> {\n        paths::remove_file(bin_path)?;\n        self.v1.remove_bin(pkg_id, bin);\n        self.v2.remove_bin(pkg_id, bin);\n        self.save()?;\n        Ok(())\n    }\n}\n\nimpl CrateListingV1 {\n    fn mark_installed(&mut self, pkg: &Package, bins: &BTreeSet<String>) {\n        // Remove bins from any other packages.\n        for other_bins in self.v1.values_mut() {\n            for bin in bins {\n                other_bins.remove(bin);\n            }\n        }\n        // Remove entries where `bins` is empty.\n        let to_remove = self\n            .v1\n            .iter()\n            .filter_map(|(&p, set)| if set.is_empty() { Some(p) } else { None })\n            .collect::<Vec<_>>();\n        for p in to_remove.iter() {\n            self.v1.remove(p);\n        }\n        // Add these bins.\n        self.v1\n            .entry(pkg.package_id())\n            .or_insert_with(BTreeSet::new)\n            .append(&mut bins.clone());\n    }\n\n    fn remove(&mut self, pkg_id: PackageId, bins: &BTreeSet<String>) {\n        let mut installed = match self.v1.entry(pkg_id) {\n            btree_map::Entry::Occupied(e) => e,\n            btree_map::Entry::Vacant(..) => panic!(\"v1 unexpected missing `{}`\", pkg_id),\n        };\n\n        for bin in bins {\n            installed.get_mut().remove(bin);\n        }\n        if installed.get().is_empty() {\n            installed.remove();\n        }\n    }\n\n    fn remove_bin(&mut self, pkg_id: PackageId, bin: &str) {\n        let mut installed = match self.v1.entry(pkg_id) {\n            btree_map::Entry::Occupied(e) => e,\n            btree_map::Entry::Vacant(..) => panic!(\"v1 unexpected missing `{}`\", pkg_id),\n        };\n        installed.get_mut().remove(bin);\n        if installed.get().is_empty() {\n            installed.remove();\n        }\n    }\n\n    fn save(&self, lock: &FileLock) -> CargoResult<()> {\n        let mut file = lock.file();\n        file.seek(SeekFrom::Start(0))?;\n        file.set_len(0)?;\n        let data = toml::to_string_pretty(self)?;\n        file.write_all(data.as_bytes())?;\n        Ok(())\n    }\n}\n\nimpl CrateListingV2 {\n    /// Incorporate any changes from v1 into self.\n    /// This handles the initial upgrade to v2, *and* handles the case\n    /// where v2 is in use, and a v1 update is made, then v2 is used again.\n    /// i.e., `cargo +new install foo ; cargo +old install bar ; cargo +new install bar`\n    /// For now, v1 is the source of truth, so its values are trusted over v2.\n    fn sync_v1(&mut self, v1: &CrateListingV1) {\n        // Make the `bins` entries the same.\n        for (pkg_id, bins) in &v1.v1 {\n            self.installs\n                .entry(*pkg_id)\n                .and_modify(|info| info.bins = bins.clone())\n                .or_insert_with(|| InstallInfo::from_v1(bins));\n        }\n        // Remove any packages that aren't present in v1.\n        let to_remove: Vec<_> = self\n            .installs\n            .keys()\n            .filter(|pkg_id| !v1.v1.contains_key(pkg_id))\n            .cloned()\n            .collect();\n        for pkg_id in to_remove {\n            self.installs.remove(&pkg_id);\n        }\n    }\n\n    fn package_for_bin(&self, bin_name: &str) -> Option<PackageId> {\n        self.installs\n            .iter()\n            .find(|(_, info)| info.bins.contains(bin_name))\n            .map(|(pkg_id, _)| *pkg_id)\n    }\n\n    fn mark_installed(\n        &mut self,\n        pkg: &Package,\n        bins: &BTreeSet<String>,\n        version_req: Option<String>,\n        opts: &CompileOptions,\n        target: &str,\n        rustc: &str,\n    ) {\n        // Remove bins from any other packages.\n        for info in &mut self.installs.values_mut() {\n            for bin in bins {\n                info.bins.remove(bin);\n            }\n        }\n        // Remove entries where `bins` is empty.\n        let to_remove = self\n            .installs\n            .iter()\n            .filter_map(|(&p, info)| if info.bins.is_empty() { Some(p) } else { None })\n            .collect::<Vec<_>>();\n        for p in to_remove.iter() {\n            self.installs.remove(p);\n        }\n        // Add these bins.\n        if let Some(info) = self.installs.get_mut(&pkg.package_id()) {\n            info.bins.append(&mut bins.clone());\n            info.version_req = version_req;\n            info.features = feature_set(&opts.cli_features.features);\n            info.all_features = opts.cli_features.all_features;\n            info.no_default_features = !opts.cli_features.uses_default_features;\n            info.profile = opts.build_config.requested_profile.to_string();\n            info.target = Some(target.to_string());\n            info.rustc = Some(rustc.to_string());\n        } else {\n            self.installs.insert(\n                pkg.package_id(),\n                InstallInfo {\n                    version_req,\n                    bins: bins.clone(),\n                    features: feature_set(&opts.cli_features.features),\n                    all_features: opts.cli_features.all_features,\n                    no_default_features: !opts.cli_features.uses_default_features,\n                    profile: opts.build_config.requested_profile.to_string(),\n                    target: Some(target.to_string()),\n                    rustc: Some(rustc.to_string()),\n                    other: BTreeMap::new(),\n                },\n            );\n        }\n    }\n\n    fn remove(&mut self, pkg_id: PackageId, bins: &BTreeSet<String>) {\n        let mut info_entry = match self.installs.entry(pkg_id) {\n            btree_map::Entry::Occupied(e) => e,\n            btree_map::Entry::Vacant(..) => panic!(\"v2 unexpected missing `{}`\", pkg_id),\n        };\n\n        for bin in bins {\n            info_entry.get_mut().bins.remove(bin);\n        }\n        if info_entry.get().bins.is_empty() {\n            info_entry.remove();\n        }\n    }\n\n    fn remove_bin(&mut self, pkg_id: PackageId, bin: &str) {\n        let mut info_entry = match self.installs.entry(pkg_id) {\n            btree_map::Entry::Occupied(e) => e,\n            btree_map::Entry::Vacant(..) => panic!(\"v1 unexpected missing `{}`\", pkg_id),\n        };\n        info_entry.get_mut().bins.remove(bin);\n        if info_entry.get().bins.is_empty() {\n            info_entry.remove();\n        }\n    }\n\n    fn save(&self, lock: &FileLock) -> CargoResult<()> {\n        let mut file = lock.file();\n        file.seek(SeekFrom::Start(0))?;\n        file.set_len(0)?;\n        let data = serde_json::to_string(self)?;\n        file.write_all(data.as_bytes())?;\n        Ok(())\n    }\n}\n\nimpl InstallInfo {\n    fn from_v1(set: &BTreeSet<String>) -> InstallInfo {\n        InstallInfo {\n            version_req: None,\n            bins: set.clone(),\n            features: BTreeSet::new(),\n            all_features: false,\n            no_default_features: false,\n            profile: \"release\".to_string(),\n            target: None,\n            rustc: None,\n            other: BTreeMap::new(),\n        }\n    }\n\n    /// Determine if this installation is \"up to date\", or if it needs to be reinstalled.\n    ///\n    /// This does not do Package/Source/Version checking.\n    fn is_up_to_date(&self, opts: &CompileOptions, target: &str, exes: &BTreeSet<String>) -> bool {\n        self.features == feature_set(&opts.cli_features.features)\n            && self.all_features == opts.cli_features.all_features\n            && self.no_default_features != opts.cli_features.uses_default_features\n            && self.profile.as_str() == opts.build_config.requested_profile.as_str()\n            && (self.target.is_none() || self.target.as_deref() == Some(target))\n            && &self.bins == exes\n    }\n}\n\n/// Determines the root directory where installation is done.\npub fn resolve_root(flag: Option<&str>, gctx: &GlobalContext) -> CargoResult<Filesystem> {\n    let config_root = gctx.get_path(\"install.root\")?;\n    Ok(flag\n        .map(PathBuf::from)\n        .or_else(|| gctx.get_env_os(\"CARGO_INSTALL_ROOT\").map(PathBuf::from))\n        .or_else(move || config_root.map(|v| v.val))\n        .map(Filesystem::new)\n        .unwrap_or_else(|| gctx.home().clone()))\n}\n\n/// Determines the `PathSource` from a `SourceId`.\npub fn path_source(source_id: SourceId, gctx: &GlobalContext) -> CargoResult<PathSource<'_>> {\n    let path = source_id\n        .url()\n        .to_file_path()\n        .map_err(|()| format_err!(\"path sources must have a valid path\"))?;\n    Ok(PathSource::new(&path, source_id, gctx))\n}\n\n/// Gets a Package based on command-line requirements.\npub fn select_dep_pkg<T>(\n    source: &mut T,\n    dep: Dependency,\n    gctx: &GlobalContext,\n    needs_update: bool,\n    current_rust_version: Option<&PartialVersion>,\n) -> CargoResult<Package>\nwhere\n    T: Source,\n{\n    // This operation may involve updating some sources or making a few queries\n    // which may involve frobbing caches, as a result make sure we synchronize\n    // with other global Cargos\n    let _lock = gctx.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;\n\n    if needs_update {\n        source.invalidate_cache();\n    }\n\n    let deps = loop {\n        match source.query_vec(&dep, QueryKind::Exact)? {\n            Poll::Ready(deps) => break deps,\n            Poll::Pending => source.block_until_ready()?,\n        }\n    };\n    match deps\n        .iter()\n        .map(|s| s.as_summary())\n        .max_by_key(|p| p.package_id())\n    {\n        Some(summary) => {\n            if let (Some(current), Some(msrv)) = (current_rust_version, summary.rust_version()) {\n                if !msrv.is_compatible_with(current) {\n                    let name = summary.name();\n                    let ver = summary.version();\n                    let extra = if dep.source_id().is_registry() {\n                        // Match any version, not just the selected\n                        let msrv_dep =\n                            Dependency::parse(dep.package_name(), None, dep.source_id())?;\n                        let msrv_deps = loop {\n                            match source.query_vec(&msrv_dep, QueryKind::Exact)? {\n                                Poll::Ready(deps) => break deps,\n                                Poll::Pending => source.block_until_ready()?,\n                            }\n                        };\n                        if let Some(alt) = msrv_deps\n                            .iter()\n                            .map(|s| s.as_summary())\n                            .filter(|summary| {\n                                summary\n                                    .rust_version()\n                                    .map(|msrv| msrv.is_compatible_with(current))\n                                    .unwrap_or(true)\n                            })\n                            .max_by_key(|s| s.package_id())\n                        {\n                            if let Some(rust_version) = alt.rust_version() {\n                                format!(\n                                    \"\\n`{name} {}` supports rustc {rust_version}\",\n                                    alt.version()\n                                )\n                            } else {\n                                format!(\n                                    \"\\n`{name} {}` has an unspecified minimum rustc version\",\n                                    alt.version()\n                                )\n                            }\n                        } else {\n                            String::new()\n                        }\n                    } else {\n                        String::new()\n                    };\n                    bail!(\"\\\ncannot install package `{name} {ver}`, it requires rustc {msrv} or newer, while the currently active rustc version is {current}{extra}\"\n)\n                }\n            }\n            let pkg = Box::new(source).download_now(summary.package_id(), gctx)?;\n            Ok(pkg)\n        }\n        None => {\n            let is_yanked: bool = if dep.version_req().is_exact() {\n                let version: String = dep.version_req().to_string();\n                if let Ok(pkg_id) =\n                    PackageId::try_new(dep.package_name(), &version[1..], source.source_id())\n                {\n                    source.invalidate_cache();\n                    loop {\n                        match source.is_yanked(pkg_id) {\n                            Poll::Ready(Ok(is_yanked)) => break is_yanked,\n                            Poll::Ready(Err(_)) => break false,\n                            Poll::Pending => source.block_until_ready()?,\n                        }\n                    }\n                } else {\n                    false\n                }\n            } else {\n                false\n            };\n            if is_yanked {\n                bail!(\n                    \"cannot install package `{}`, it has been yanked from {}\",\n                    dep.package_name(),\n                    source.source_id()\n                )\n            } else {\n                bail!(\n                    \"could not find `{}` in {} with version `{}`\",\n                    dep.package_name(),\n                    source.source_id(),\n                    dep.version_req(),\n                )\n            }\n        }\n    }\n}\n\npub fn select_pkg<T, F>(\n    source: &mut T,\n    dep: Option<Dependency>,\n    mut list_all: F,\n    gctx: &GlobalContext,\n    current_rust_version: Option<&PartialVersion>,\n) -> CargoResult<Package>\nwhere\n    T: Source,\n    F: FnMut(&mut T) -> CargoResult<Vec<Package>>,\n{\n    // This operation may involve updating some sources or making a few queries\n    // which may involve frobbing caches, as a result make sure we synchronize\n    // with other global Cargos\n    let _lock = gctx.acquire_package_cache_lock(CacheLockMode::DownloadExclusive)?;\n\n    source.invalidate_cache();\n\n    return if let Some(dep) = dep {\n        select_dep_pkg(source, dep, gctx, false, current_rust_version)\n    } else {\n        let candidates = list_all(source)?;\n        let binaries = candidates\n            .iter()\n            .filter(|cand| cand.targets().iter().filter(|t| t.is_bin()).count() > 0);\n        let examples = candidates\n            .iter()\n            .filter(|cand| cand.targets().iter().filter(|t| t.is_example()).count() > 0);\n        let git_url = source.source_id().url().to_string();\n        let pkg = match one(binaries, |v| multi_err(\"binaries\", &git_url, v))? {\n            Some(p) => p,\n            None => match one(examples, |v| multi_err(\"examples\", &git_url, v))? {\n                Some(p) => p,\n                None => bail!(\n                    \"no packages found with binaries or \\\n                         examples\"\n                ),\n            },\n        };\n        Ok(pkg.clone())\n    };\n\n    fn multi_err(kind: &str, git_url: &str, mut pkgs: Vec<&Package>) -> String {\n        pkgs.sort_unstable_by_key(|a| a.name());\n        let first_pkg = pkgs[0];\n        format!(\n            \"multiple packages with {} found: {}. When installing a git repository, \\\n            cargo will always search the entire repo for any Cargo.toml.\\n\\\n            Please specify a package, e.g. `cargo install --git {} {}`.\",\n            kind,\n            pkgs.iter()\n                .map(|p| p.name().as_str())\n                .collect::<Vec<_>>()\n                .join(\", \"),\n            git_url,\n            first_pkg.name()\n        )\n    }\n}\n\n/// Get one element from the iterator.\n/// Returns None if none left.\n/// Returns error if there is more than one item in the iterator.\nfn one<I, F>(mut i: I, f: F) -> CargoResult<Option<I::Item>>\nwhere\n    I: Iterator,\n    F: FnOnce(Vec<I::Item>) -> String,\n{\n...\n// Path: src/cargo/ops/cargo_uninstall.rs\nuse crate::core::PackageId;\nuse crate::core::{PackageIdSpec, PackageIdSpecQuery, SourceId};\nuse crate::ops::common_for_install_and_uninstall::*;\nuse crate::sources::PathSource;\nuse crate::util::errors::CargoResult;\nuse crate::util::Filesystem;\nuse crate::util::GlobalContext;\nuse anyhow::bail;\nuse std::collections::BTreeSet;\nuse std::env;\n\npub fn uninstall(\n    root: Option<&str>,\n    specs: Vec<&str>,\n    bins: &[String],\n    gctx: &GlobalContext,\n) -> CargoResult<()> {\n    if specs.len() > 1 && !bins.is_empty() {\n        bail!(\"A binary can only be associated with a single installed package, specifying multiple specs with --bin is redundant.\");\n    }\n\n    let root = resolve_root(root, gctx)?;\n    let scheduled_error = if specs.len() == 1 {\n        uninstall_one(&root, specs[0], bins, gctx)?;\n        false\n    } else if specs.is_empty() {\n        uninstall_cwd(&root, bins, gctx)?;\n        false\n    } else {\n        let mut succeeded = vec![];\n        let mut failed = vec![];\n        for spec in specs {\n            let root = root.clone();\n            match uninstall_one(&root, spec, bins, gctx) {\n                Ok(()) => succeeded.push(spec),\n                Err(e) => {\n                    crate::display_error(&e, &mut gctx.shell());\n                    failed.push(spec)\n                }\n            }\n        }\n\n        let mut summary = vec![];\n        if !succeeded.is_empty() {\n            summary.push(format!(\n                \"Successfully uninstalled {}!\",\n                succeeded.join(\", \")\n            ));\n        }\n        if !failed.is_empty() {\n            summary.push(format!(\n                \"Failed to uninstall {} (see error(s) above).\",\n                failed.join(\", \")\n            ));\n        }\n\n        if !succeeded.is_empty() || !failed.is_empty() {\n            gctx.shell().status(\"Summary\", summary.join(\" \"))?;\n        }\n\n        !failed.is_empty()\n    };\n\n    if scheduled_error {\n        bail!(\"some packages failed to uninstall\");\n    }\n\n    Ok(())\n}\n\n\npub fn uninstall_one(\n    root: &Filesystem,\n    spec: &str,\n    bins: &[String],\n    gctx: &GlobalContext,\n) -> CargoResult<()> {\n    let tracker = InstallTracker::load(gctx, root)?;\n    let all_pkgs = tracker.all_installed_bins().map(|(pkg_id, _set)| *pkg_id);\n    let pkgid = PackageIdSpec::query_str(spec, all_pkgs)?;\n    uninstall_pkgid(root, tracker, pkgid, bins, gctx)\n}\n\nfn uninstall_cwd(root: &Filesystem, bins: &[String], gctx: &GlobalContext) -> CargoResult<()> {\n    let tracker = InstallTracker::load(gctx, root)?;\n    let source_id = SourceId::for_path(gctx.cwd())?;\n    let mut src = path_source(source_id, gctx)?;\n    let pkg = select_pkg(\n        &mut src,\n        None,\n        |path: &mut PathSource<'_>| path.read_packages(),\n        gctx,\n        None,\n    )?;\n    let pkgid = pkg.package_id();\n    uninstall_pkgid(root, tracker, pkgid, bins, gctx)\n}\n\nfn uninstall_pkgid(\n    root: &Filesystem,\n    mut tracker: InstallTracker,\n    pkgid: PackageId,\n    bins: &[String],\n    gctx: &GlobalContext,\n) -> CargoResult<()> {\n    let installed = match tracker.installed_bins(pkgid) {\n        Some(bins) => bins.clone(),\n        None => bail!(\"package `{}` is not installed\", pkgid),\n    };\n\n    let dst = root.join(\"bin\").into_path_unlocked();\n    for bin in &installed {\n        let bin = dst.join(bin);\n        if !bin.exists() {\n            bail!(\n                \"corrupt metadata, `{}` does not exist when it should\",\n                bin.display()\n            )\n        }\n    }\n\n    let bins = bins\n        .iter()\n        .map(|s| {\n            if s.ends_with(env::consts::EXE_SUFFIX) {\n                s.to_string()\n            } else {\n                format!(\"{}{}\", s, env::consts::EXE_SUFFIX)\n            }\n        })\n        .collect::<BTreeSet<_>>();\n\n    for bin in bins.iter() {\n        if !installed.contains(bin) {\n            bail!(\"binary `{}` not installed as part of `{}`\", bin, pkgid)\n        }\n    }\n\n    let to_remove = {\n        if bins.is_empty() {\n            installed\n        } else {\n            bins\n        }\n    };\n\n    for bin in to_remove {\n        let bin_path = dst.join(&bin);\n        gctx.shell().status(\"Removing\", bin_path.display())?;\n        tracker.remove_bin_then_save(pkgid, &bin, &bin_path)?;\n    }\n\n    Ok(())\n}\n\n// Path: src/cargo/util/credential/mod.rs\n//! Built-in Cargo credential providers\n\n#![allow(clippy::print_stderr)]\n\npub mod adaptor;\npub mod paseto;\npub mod process;\npub mod token;\n\n// Path: src/cargo/ops/mod.rs\nuse crate::sources::CRATES_IO_DOMAIN;\n\npub use self::cargo_clean::{clean, CleanContext, CleanOptions};\npub use self::cargo_compile::{\n    compile, compile_with_exec, compile_ws, create_bcx, print, resolve_all_features, CompileOptions,\n};\npub use self::cargo_compile::{CompileFilter, FilterRule, LibRule, Packages};\npub use self::cargo_doc::{doc, DocOptions, OutputFormat};\npub use self::cargo_fetch::{fetch, FetchOptions};\npub use self::cargo_generate_lockfile::generate_lockfile;\npub use self::cargo_generate_lockfile::print_lockfile_changes;\npub use self::cargo_generate_lockfile::update_lockfile;\npub use self::cargo_generate_lockfile::UpdateOptions;\npub use self::cargo_install::{install, install_list};\npub use self::cargo_new::{init, new, NewOptions, NewProjectKind, VersionControl};\npub use self::cargo_output_metadata::{output_metadata, ExportInfo, OutputMetadataOptions};\npub use self::cargo_package::{check_yanked, package, package_one, PackageOpts};\npub use self::cargo_pkgid::pkgid;\npub use self::cargo_read_manifest::{read_package, read_packages};\npub use self::cargo_run::run;\npub use self::cargo_test::{run_benches, run_tests, TestOptions};\npub use self::cargo_uninstall::uninstall;\npub use self::fix::{fix, fix_exec_rustc, fix_get_proxy_lock_addr, FixOptions};\npub use self::lockfile::{load_pkg_lockfile, resolve_to_string, write_pkg_lockfile};\npub use self::registry::modify_owners;\npub use self::registry::publish;\npub use self::registry::registry_login;\npub use self::registry::registry_logout;\npub use self::registry::search;\npub use self::registry::yank;\npub use self::registry::OwnersOptions;\npub use self::registry::PublishOpts;\npub use self::registry::RegistryCredentialConfig;\npub use self::registry::RegistryOrIndex;\npub use self::resolve::{\n    add_overrides, get_resolved_packages, resolve_with_previous, resolve_ws, resolve_ws_with_opts,\n    WorkspaceResolve,\n};\npub use self::vendor::{vendor, VendorOptions};\n\npub mod cargo_add;\nmod cargo_clean;\npub(crate) mod cargo_compile;\npub mod cargo_config;\nmod cargo_doc;\nmod cargo_fetch;\nmod cargo_generate_lockfile;\nmod cargo_install;\nmod cargo_new;\nmod cargo_output_metadata;\nmod cargo_package;\nmod cargo_pkgid;\nmod cargo_read_manifest;\npub mod cargo_remove;\nmod cargo_run;\nmod cargo_test;\nmod cargo_uninstall;\nmod common_for_install_and_uninstall;\nmod fix;\npub(crate) mod lockfile;\npub(crate) mod registry;\npub(crate) mod resolve;\npub mod tree;\nmod vendor;\n\n/// Returns true if the dependency is either git or path, false otherwise\n/// Error if a git/path dep is transitive, but has no version (registry source).\n/// This check is performed on dependencies before publishing or packaging\nfn check_dep_has_version(dep: &crate::core::Dependency, publish: bool) -> crate::CargoResult<bool> {\n    let which = if dep.source_id().is_path() {\n        \"path\"\n    } else if dep.source_id().is_git() {\n        \"git\"\n    } else {\n        return Ok(false);\n    };\n\n    if !dep.specified_req() && dep.is_transitive() {\n        let dep_version_source = dep.registry_id().map_or_else(\n            || CRATES_IO_DOMAIN.to_string(),\n            |registry_id| registry_id.display_registry_name(),\n        );\n        anyhow::bail!(\n            \"all dependencies must have a version specified when {}.\\n\\\n             dependency `{}` does not specify a version\\n\\\n             Note: The {} dependency will use the version from {},\\n\\\n             the `{}` specification will be removed from the dependency declaration.\",\n            if publish { \"publishing\" } else { \"packaging\" },\n            dep.package_name(),\n            if publish { \"published\" } else { \"packaged\" },\n            dep_version_source,\n            which,\n        )\n    }\n    Ok(true)\n}\n\n// Path: src/cargo/ops/cargo_fetch.rs\nuse crate::core::compiler::standard_lib;\nuse crate::core::compiler::{BuildConfig, CompileMode, RustcTargetData};\nuse crate::core::{PackageSet, Resolve, Workspace};\nuse crate::ops;\nuse crate::util::context::JobsConfig;\nuse crate::util::CargoResult;\nuse crate::util::GlobalContext;\nuse std::collections::HashSet;\n\npub struct FetchOptions<'a> {\n    pub gctx: &'a GlobalContext,\n    /// The target arch triple to fetch dependencies for\n    pub targets: Vec<String>,\n}\n\n/// Executes `cargo fetch`.\npub fn fetch<'a>(\n    ws: &Workspace<'a>,\n    options: &FetchOptions<'a>,\n) -> CargoResult<(Resolve, PackageSet<'a>)> {\n    ws.emit_warnings()?;\n    let (mut packages, resolve) = ops::resolve_ws(ws)?;\n\n    let jobs = Some(JobsConfig::Integer(1));\n    let keep_going = false;\n    let gctx = ws.gctx();\n    let build_config =\n        BuildConfig::new(gctx, jobs, keep_going, &options.targets, CompileMode::Build)?;\n    let mut data = RustcTargetData::new(ws, &build_config.requested_kinds)?;\n    let mut fetched_packages = HashSet::new();\n    let mut deps_to_fetch = ws.members().map(|p| p.package_id()).collect::<Vec<_>>();\n    let mut to_download = Vec::new();\n\n    while let Some(id) = deps_to_fetch.pop() {\n        if !fetched_packages.insert(id) {\n            continue;\n        }\n\n        to_download.push(id);\n        let deps = resolve\n            .deps(id)\n            .filter(|&(_id, deps)| {\n                deps.iter().any(|d| {\n                    // If no target was specified then all dependencies are\n                    // fetched.\n                    if options.targets.is_empty() {\n                        return true;\n                    }\n\n                    // Otherwise we only download this dependency if any of the\n                    // requested platforms would match this dependency. Note\n                    // that this is a bit lossy because not all dependencies are\n                    // always compiled for all platforms, but it should be\n                    // \"close enough\" for now.\n                    build_config\n                        .requested_kinds\n                        .iter()\n                        .any(|kind| data.dep_platform_activated(d, *kind))\n                })\n            })\n            .map(|(id, _deps)| id);\n        deps_to_fetch.extend(deps);\n    }\n\n    // If -Zbuild-std was passed, download dependencies for the standard library.\n    // We don't know ahead of time what jobs we'll be running, so tell `std_crates` that.\n    if let Some(crates) = standard_lib::std_crates(gctx, None) {\n        let (std_package_set, _, _) =\n            standard_lib::resolve_std(ws, &mut data, &build_config, &crates)?;\n        packages.add_set(std_package_set);\n    }\n\n    packages.get_many(to_download)?;\n    crate::core::gc::auto_gc(gctx);\n\n    Ok((resolve, packages))\n}\n\n// Path: src/cargo/ops/cargo_remove.rs\n//! Core of cargo-remove command\n\nuse crate::core::Package;\nuse crate::util::toml_mut::manifest::DepTable;\nuse crate::util::toml_mut::manifest::LocalManifest;\nuse crate::CargoResult;\nuse crate::GlobalContext;\n\n/// Remove a dependency from a Cargo.toml manifest file.\n#[derive(Debug)]\npub struct RemoveOptions<'a> {\n    /// Configuration information for Cargo operations\n    pub gctx: &'a GlobalContext,\n    /// Package to remove dependencies from\n    pub spec: &'a Package,\n    /// Dependencies to remove\n    pub dependencies: Vec<String>,\n    /// Which dependency section to remove these from\n    pub section: DepTable,\n    /// Whether or not to actually write the manifest\n    pub dry_run: bool,\n}\n\n/// Remove dependencies from a manifest\npub fn remove(options: &RemoveOptions<'_>) -> CargoResult<()> {\n    let dep_table = options\n        .section\n        .to_table()\n        .into_iter()\n        .map(String::from)\n        .collect::<Vec<_>>();\n\n    let manifest_path = options.spec.manifest_path().to_path_buf();\n    let mut manifest = LocalManifest::try_new(&manifest_path)?;\n\n    for dep in &options.dependencies {\n        let section = if dep_table.len() >= 3 {\n            format!(\"{} for target `{}`\", &dep_table[2], &dep_table[1])\n        } else {\n            dep_table[0].clone()\n        };\n        options\n            .gctx\n            .shell()\n            .status(\"Removing\", format!(\"{dep} from {section}\"))?;\n\n        manifest.remove_from_table(&dep_table, dep)?;\n\n        // Now that we have removed the crate, if that was the last reference to that\n        // crate, then we need to drop any explicitly activated features on\n        // that crate.\n        manifest.gc_dep(dep);\n    }\n\n    if options.dry_run {\n        options\n            .gctx\n            .shell()\n            .warn(\"aborting remove due to dry run\")?;\n    } else {\n        manifest.write()?;\n    }\n\n    Ok(())\n}\n\n// Path: src/cargo/macros.rs\nuse std::fmt;\n\nmacro_rules! compact_debug {\n    (\n        impl fmt::Debug for $ty:ident {\n            fn fmt(&$this:ident, f: &mut fmt::Formatter) -> fmt::Result {\n                let (default, default_name) = $e:expr;\n                [debug_the_fields($($field:ident)*)]\n            }\n        }\n    ) => (\n\n        impl fmt::Debug for $ty {\n            fn fmt(&$this, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                // Try printing a pretty version where we collapse as many fields as\n                // possible, indicating that they're equivalent to a function call\n                // that's hopefully enough to indicate what each value is without\n                // actually dumping everything so verbosely.\n                let mut s = f.debug_struct(stringify!($ty));\n                let (default, default_name) = $e;\n                let mut any_default = false;\n\n                // Exhaustively match so when fields are added we get a compile\n                // failure\n                let $ty { $($field),* } = $this;\n                $(\n                    if *$field == default.$field {\n                        any_default = true;\n                    } else {\n                        s.field(stringify!($field), $field);\n                    }\n                )*\n\n                if any_default {\n                    s.field(\"..\", &crate::macros::DisplayAsDebug(default_name));\n                }\n                s.finish()\n            }\n        }\n    )\n}\n\npub struct DisplayAsDebug<T>(pub T);\n\nimpl<T: fmt::Display> fmt::Debug for DisplayAsDebug<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Display::fmt(&self.0, f)\n    }\n}\n\n// Path: src/cargo/ops/cargo_new.rs\nuse crate::core::{Edition, Shell, Workspace};\nuse crate::util::errors::CargoResult;\nuse crate::util::important_paths::find_root_manifest_for_wd;\nuse crate::util::toml_mut::is_sorted;\nuse crate::util::{existing_vcs_repo, FossilRepo, GitRepo, HgRepo, PijulRepo};\nuse crate::util::{restricted_names, GlobalContext};\nuse anyhow::{anyhow, Context as _};\nuse cargo_util::paths::{self, write_atomic};\nuse cargo_util_schemas::manifest::PackageName;\nuse serde::de;\nuse serde::Deserialize;\nuse std::collections::BTreeMap;\nuse std::ffi::OsStr;\nuse std::io::{BufRead, BufReader, ErrorKind};\nuse std::path::{Path, PathBuf};\nuse std::str::FromStr;\nuse std::{fmt, slice};\nuse toml_edit::{Array, Value};\n\n#[derive(Clone, Copy, Debug, PartialEq)]\npub enum VersionControl {\n    Git,\n    Hg,\n    Pijul,\n    Fossil,\n    NoVcs,\n}\n\nimpl FromStr for VersionControl {\n    type Err = anyhow::Error;\n\n    fn from_str(s: &str) -> Result<Self, anyhow::Error> {\n        match s {\n            \"git\" => Ok(VersionControl::Git),\n            \"hg\" => Ok(VersionControl::Hg),\n            \"pijul\" => Ok(VersionControl::Pijul),\n            \"fossil\" => Ok(VersionControl::Fossil),\n            \"none\" => Ok(VersionControl::NoVcs),\n            other => anyhow::bail!(\"unknown vcs specification: `{}`\", other),\n        }\n    }\n}\n\nimpl<'de> de::Deserialize<'de> for VersionControl {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: de::Deserializer<'de>,\n    {\n        let s = String::deserialize(deserializer)?;\n        FromStr::from_str(&s).map_err(de::Error::custom)\n    }\n}\n\n#[derive(Debug)]\npub struct NewOptions {\n    pub version_control: Option<VersionControl>,\n    pub kind: NewProjectKind,\n    pub auto_detect_kind: bool,\n    /// Absolute path to the directory for the new package\n    pub path: PathBuf,\n    pub name: Option<String>,\n    pub edition: Option<String>,\n    pub registry: Option<String>,\n}\n\n#[derive(Clone, Copy, Debug, PartialEq, Eq)]\npub enum NewProjectKind {\n    Bin,\n    Lib,\n}\n\nimpl NewProjectKind {\n    fn is_bin(self) -> bool {\n        self == NewProjectKind::Bin\n    }\n}\n\nimpl fmt::Display for NewProjectKind {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match *self {\n            NewProjectKind::Bin => \"binary (application)\",\n            NewProjectKind::Lib => \"library\",\n        }\n        .fmt(f)\n    }\n}\n\nstruct SourceFileInformation {\n    relative_path: String,\n    bin: bool,\n}\n\nstruct MkOptions<'a> {\n    version_control: Option<VersionControl>,\n    path: &'a Path,\n    name: &'a str,\n    source_files: Vec<SourceFileInformation>,\n    edition: Option<&'a str>,\n    registry: Option<&'a str>,\n}\n\nimpl NewOptions {\n    pub fn new(\n        version_control: Option<VersionControl>,\n        bin: bool,\n        lib: bool,\n        path: PathBuf,\n        name: Option<String>,\n        edition: Option<String>,\n        registry: Option<String>,\n    ) -> CargoResult<NewOptions> {\n        let auto_detect_kind = !bin && !lib;\n\n        let kind = match (bin, lib) {\n            (true, true) => anyhow::bail!(\"can't specify both lib and binary outputs\"),\n            (false, true) => NewProjectKind::Lib,\n            (_, false) => NewProjectKind::Bin,\n        };\n\n        let opts = NewOptions {\n            version_control,\n            kind,\n            auto_detect_kind,\n            path,\n            name,\n            edition,\n            registry,\n        };\n        Ok(opts)\n    }\n}\n\n#[derive(Deserialize)]\nstruct CargoNewConfig {\n    #[deprecated = \"cargo-new no longer supports adding the authors field\"]\n    #[allow(dead_code)]\n    name: Option<String>,\n\n    #[deprecated = \"cargo-new no longer supports adding the authors field\"]\n    #[allow(dead_code)]\n    email: Option<String>,\n\n    #[serde(rename = \"vcs\")]\n    version_control: Option<VersionControl>,\n}\n\nfn get_name<'a>(path: &'a Path, opts: &'a NewOptions) -> CargoResult<&'a str> {\n    if let Some(ref name) = opts.name {\n        return Ok(name);\n    }\n\n    let file_name = path.file_name().ok_or_else(|| {\n        anyhow::format_err!(\n            \"cannot auto-detect package name from path {:?} ; use --name to override\",\n            path.as_os_str()\n        )\n    })?;\n\n    file_name.to_str().ok_or_else(|| {\n        anyhow::format_err!(\n            \"cannot create package with a non-unicode name: {:?}\",\n            file_name\n        )\n    })\n}\n\n/// See also `util::toml::embedded::sanitize_name`\nfn check_name(\n    name: &str,\n    show_name_help: bool,\n    has_bin: bool,\n    shell: &mut Shell,\n) -> CargoResult<()> {\n    // If --name is already used to override, no point in suggesting it\n    // again as a fix.\n    let name_help = if show_name_help {\n        \"\\nIf you need a package name to not match the directory name, consider using --name flag.\"\n    } else {\n        \"\"\n    };\n    let bin_help = || {\n        let mut help = String::from(name_help);\n        if has_bin && !name.is_empty() {\n            help.push_str(&format!(\n                \"\\n\\\n                If you need a binary with the name \\\"{name}\\\", use a valid package \\\n                name, and set the binary name to be different from the package. \\\n                This can be done by setting the binary filename to `src/bin/{name}.rs` \\\n                or change the name in Cargo.toml with:\\n\\\n                \\n    \\\n                [[bin]]\\n    \\\n                name = \\\"{name}\\\"\\n    \\\n                path = \\\"src/main.rs\\\"\\n\\\n            \",\n                name = name\n            ));\n        }\n        help\n    };\n    PackageName::new(name).map_err(|err| {\n        let help = bin_help();\n        anyhow::anyhow!(\"{err}{help}\")\n    })?;\n\n    if restricted_names::is_keyword(name) {\n        anyhow::bail!(\n            \"the name `{}` cannot be used as a package name, it is a Rust keyword{}\",\n            name,\n            bin_help()\n        );\n    }\n    if restricted_names::is_conflicting_artifact_name(name) {\n        if has_bin {\n            anyhow::bail!(\n                \"the name `{}` cannot be used as a package name, \\\n                it conflicts with cargo's build directory names{}\",\n                name,\n                name_help\n            );\n        } else {\n            shell.warn(format!(\n                \"the name `{}` will not support binary \\\n                executables with that name, \\\n                it conflicts with cargo's build directory names\",\n                name\n            ))?;\n        }\n    }\n    if name == \"test\" {\n        anyhow::bail!(\n            \"the name `test` cannot be used as a package name, \\\n            it conflicts with Rust's built-in test library{}\",\n            bin_help()\n        );\n    }\n    if [\"core\", \"std\", \"alloc\", \"proc_macro\", \"proc-macro\"].contains(&name) {\n        shell.warn(format!(\n            \"the name `{}` is part of Rust's standard library\\n\\\n            It is recommended to use a different name to avoid problems.{}\",\n            name,\n            bin_help()\n        ))?;\n    }\n    if restricted_names::is_windows_reserved(name) {\n        if cfg!(windows) {\n            anyhow::bail!(\n                \"cannot use name `{}`, it is a reserved Windows filename{}\",\n                name,\n                name_help\n            );\n        } else {\n            shell.warn(format!(\n                \"the name `{}` is a reserved Windows filename\\n\\\n                This package will not work on Windows platforms.\",\n                name\n            ))?;\n        }\n    }\n    if restricted_names::is_non_ascii_name(name) {\n        shell.warn(format!(\n            \"the name `{}` contains non-ASCII characters\\n\\\n            Non-ASCII crate names are not supported by Rust.\",\n            name\n        ))?;\n    }\n    let name_in_lowercase = name.to_lowercase();\n    if name != name_in_lowercase {\n        shell.warn(format!(\n            \"the name `{name}` is not snake_case or kebab-case which is recommended for package names, consider `{name_in_lowercase}`\"\n        ))?;\n    }\n\n    Ok(())\n}\n\n/// Checks if the path contains any invalid PATH env characters.\nfn check_path(path: &Path, shell: &mut Shell) -> CargoResult<()> {\n    // warn if the path contains characters that will break `env::join_paths`\n    if let Err(_) = paths::join_paths(slice::from_ref(&OsStr::new(path)), \"\") {\n        let path = path.to_string_lossy();\n        shell.warn(format!(\n            \"the path `{path}` contains invalid PATH characters (usually `:`, `;`, or `\\\"`)\\n\\\n            It is recommended to use a different name to avoid problems.\"\n        ))?;\n    }\n    Ok(())\n}\n\nfn detect_source_paths_and_types(\n    package_path: &Path,\n    package_name: &str,\n    detected_files: &mut Vec<SourceFileInformation>,\n) -> CargoResult<()> {\n    let path = package_path;\n    let name = package_name;\n\n    enum H {\n        Bin,\n        Lib,\n        Detect,\n    }\n\n    struct Test {\n        proposed_path: String,\n        handling: H,\n    }\n\n    let tests = vec![\n        Test {\n            proposed_path: \"src/main.rs\".to_string(),\n            handling: H::Bin,\n        },\n        Test {\n            proposed_path: \"main.rs\".to_string(),\n            handling: H::Bin,\n        },\n        Test {\n            proposed_path: format!(\"src/{}.rs\", name),\n            handling: H::Detect,\n        },\n        Test {\n            proposed_path: format!(\"{}.rs\", name),\n            handling: H::Detect,\n        },\n        Test {\n            proposed_path: \"src/lib.rs\".to_string(),\n            handling: H::Lib,\n        },\n        Test {\n            proposed_path: \"lib.rs\".to_string(),\n            handling: H::Lib,\n        },\n    ];\n\n    for i in tests {\n        let pp = i.proposed_path;\n\n        // path/pp does not exist or is not a file\n        if !path.join(&pp).is_file() {\n            continue;\n        }\n\n        let sfi = match i.handling {\n            H::Bin => SourceFileInformation {\n                relative_path: pp,\n                bin: true,\n            },\n            H::Lib => SourceFileInformation {\n                relative_path: pp,\n                bin: false,\n            },\n            H::Detect => {\n                let content = paths::read(&path.join(pp.clone()))?;\n                let isbin = content.contains(\"fn main\");\n                SourceFileInformation {\n                    relative_path: pp,\n                    bin: isbin,\n                }\n            }\n        };\n        detected_files.push(sfi);\n    }\n\n    // Check for duplicate lib attempt\n\n    let mut previous_lib_relpath: Option<&str> = None;\n    let mut duplicates_checker: BTreeMap<&str, &SourceFileInformation> = BTreeMap::new();\n\n    for i in detected_files {\n        if i.bin {\n            if let Some(x) = BTreeMap::get::<str>(&duplicates_checker, &name) {\n                anyhow::bail!(\n                    \"\\\nmultiple possible binary sources found:\n  {}\n  {}\ncannot automatically generate Cargo.toml as the main target would be ambiguous\",\n                    &x.relative_path,\n                    &i.relative_path\n                );\n            }\n            duplicates_checker.insert(name, i);\n        } else {\n            if let Some(plp) = previous_lib_relpath {\n                anyhow::bail!(\n                    \"cannot have a package with \\\n                     multiple libraries, \\\n                     found both `{}` and `{}`\",\n...\n", "needle_token_start": 8877, "needle_token_end": 9024, "code_context_ntokens": 16255, "output": ["```rust\npub fn uninstall(\n    root: Option<&str>,\n    specs: Vec<&str>,\n    bins: &[String],\n    gctx: &GlobalContext,\n) -> CargoResult<()> {\n    if specs.len() > 1 && !bins.is_empty() {\n        bail!(\"A binary can only be associated with a single installed package, specifying multiple specs with --bin is redundant.\");\n    }\n\n    let root = resolve_root(root, gctx)?;\n    let scheduled_error = if specs.len() == 1 {\n        uninstall_one(&root, specs[0], bins, gctx)?;\n        false\n    } else if specs.is_empty() {\n        uninstall_cwd(&root, bins, gctx)?;\n        false\n    } else {\n        let mut succeeded = vec![];\n        let mut failed = vec![];\n        for spec in specs {\n            let root = root.clone();\n            match uninstall_one(&root, spec, bins, gctx) {\n                Ok(()) => succeeded.push(spec),\n                Err(e) => {\n                    crate::display_error(&e, &mut gctx.shell());\n                    failed.push(spec)\n                }\n            }\n        }\n\n        let mut summary = vec![];\n        if !succeeded.is_empty() {\n            summary.push(format!(\n                \"Successfully uninstalled {}!\",\n                succeeded.join(\", \")\n            ));\n        }\n        if !failed.is_empty() {\n            summary.push(format!(\n                \"Failed to uninstall {} (see error(s) above).\",\n                failed.join(\", \")\n            ));\n        }\n\n        if !succeeded.is_empty() || !failed.is_empty() {\n            gctx.shell().status(\"Summary\", summary.join(\" \"))?;\n        }\n\n        !failed.is_empty()\n    };\n\n    if scheduled_error {\n        bail!(\"some packages failed to uninstall\");\n    }\n\n    Ok(())\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "unit_needs_doc_scrape", "language": "rust", "path": "src/cargo/core/workspace.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: Determines if a specific unit within a project should have documentation-related processing applied, excluding certain types of units that either do not support documentation or cause issues with the build system.\n2. **Input**: A unit representing a component of a project, which includes details about the package and target.\n3. **Output**: A boolean value indicating whether documentation-related processing should be applied to the given unit.\n4. **Procedure**: Checks if the unit belongs to a member of the project and ensures it is neither a host-targeted unit (like build scripts or procedural macros) nor a procedural macro package, as these do not support or require documentation scraping.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        self.ignore_lock = ignore_lock;\n        self\n    }\n\n    /// Get the lowest-common denominator `package.rust-version` within the workspace, if specified\n    /// anywhere\n    pub fn rust_version(&self) -> Option<&RustVersion> {\n        self.members().filter_map(|pkg| pkg.rust_version()).min()\n    }\n\n    pub fn custom_metadata(&self) -> Option<&toml::Value> {\n        self.custom_metadata.as_ref()\n    }\n\n    pub fn load_workspace_config(&mut self) -> CargoResult<Option<WorkspaceRootConfig>> {\n        // If we didn't find a root, it must mean there is no [workspace] section, and thus no\n        // metadata.\n        if let Some(root_path) = &self.root_manifest {\n            let root_package = self.packages.load(root_path)?;\n            match root_package.workspace_config() {\n                WorkspaceConfig::Root(ref root_config) => {\n                    return Ok(Some(root_config.clone()));\n                }\n\n                _ => bail!(\n                    \"root of a workspace inferred but wasn't a root: {}\",\n                    root_path.display()\n                ),\n            }\n        }\n\n        Ok(None)\n    }\n\n    /// Finds the root of a workspace for the crate whose manifest is located\n    /// at `manifest_path`.\n    ///\n    /// This will parse the `Cargo.toml` at `manifest_path` and then interpret\n    /// the workspace configuration, optionally walking up the filesystem\n    /// looking for other workspace roots.\n    ///\n    /// Returns an error if `manifest_path` isn't actually a valid manifest or\n    /// if some other transient error happens.\n    fn find_root(&mut self, manifest_path: &Path) -> CargoResult<Option<PathBuf>> {\n        let current = self.packages.load(manifest_path)?;\n        match current\n            .workspace_config()\n            .get_ws_root(manifest_path, manifest_path)\n        {\n            Some(root_path) => {\n                debug!(\"find_root - is root {}\", manifest_path.display());\n                Ok(Some(root_path))\n            }\n            None => find_workspace_root_with_loader(manifest_path, self.gctx, |self_path| {\n                Ok(self\n                    .packages\n                    .load(self_path)?\n                    .workspace_config()\n                    .get_ws_root(self_path, manifest_path))\n            }),\n        }\n    }\n\n    /// After the root of a workspace has been located, probes for all members\n    /// of a workspace.\n    ///\n    /// If the `workspace.members` configuration is present, then this just\n    /// verifies that those are all valid packages to point to. Otherwise, this\n    /// will transitively follow all `path` dependencies looking for members of\n    /// the workspace.\n    fn find_members(&mut self) -> CargoResult<()> {\n        let Some(workspace_config) = self.load_workspace_config()? else {\n            debug!(\"find_members - only me as a member\");\n            self.members.push(self.current_manifest.clone());\n            self.default_members.push(self.current_manifest.clone());\n            if let Ok(pkg) = self.current() {\n                let id = pkg.package_id();\n                self.member_ids.insert(id);\n            }\n            return Ok(());\n        };\n\n        // self.root_manifest must be Some to have retrieved workspace_config\n        let root_manifest_path = self.root_manifest.clone().unwrap();\n\n        let members_paths =\n            workspace_config.members_paths(workspace_config.members.as_ref().unwrap_or(&vec![]))?;\n        let default_members_paths = if root_manifest_path == self.current_manifest {\n            if let Some(ref default) = workspace_config.default_members {\n                Some(workspace_config.members_paths(default)?)\n            } else {\n                None\n            }\n        } else {\n            None\n        };\n\n        for path in &members_paths {\n            self.find_path_deps(&path.join(\"Cargo.toml\"), &root_manifest_path, false)\n                .with_context(|| {\n                    format!(\n                        \"failed to load manifest for workspace member `{}`\\n\\\n                        referenced by workspace at `{}`\",\n                        path.display(),\n                        root_manifest_path.display()\n                    )\n                })?;\n        }\n\n        self.find_path_deps(&root_manifest_path, &root_manifest_path, false)?;\n\n        if let Some(default) = default_members_paths {\n            for path in default {\n                let normalized_path = paths::normalize_path(&path);\n                let manifest_path = normalized_path.join(\"Cargo.toml\");\n                if !self.members.contains(&manifest_path) {\n                    // default-members are allowed to be excluded, but they\n                    // still must be referred to by the original (unfiltered)\n                    // members list. Note that we aren't testing against the\n                    // manifest path, both because `members_paths` doesn't\n                    // include `/Cargo.toml`, and because excluded paths may not\n                    // be crates.\n                    let exclude = members_paths.contains(&normalized_path)\n                        && workspace_config.is_excluded(&normalized_path);\n                    if exclude {\n                        continue;\n                    }\n                    bail!(\n                        \"package `{}` is listed in default-members but is not a member\\n\\\n                        for workspace at {}.\",\n                        path.display(),\n                        root_manifest_path.display()\n                    )\n                }\n                self.default_members.push(manifest_path)\n            }\n        } else if self.is_virtual() {\n            self.default_members = self.members.clone()\n        } else {\n            self.default_members.push(self.current_manifest.clone())\n        }\n\n        Ok(())\n    }\n\n    fn find_path_deps(\n        &mut self,\n        manifest_path: &Path,\n        root_manifest: &Path,\n        is_path_dep: bool,\n    ) -> CargoResult<()> {\n        let manifest_path = paths::normalize_path(manifest_path);\n        if self.members.contains(&manifest_path) {\n            return Ok(());\n        }\n        if is_path_dep && self.root_maybe().is_embedded() {\n            // Embedded manifests cannot have workspace members\n            return Ok(());\n        }\n        if is_path_dep\n            && !manifest_path.parent().unwrap().starts_with(self.root())\n            && self.find_root(&manifest_path)? != self.root_manifest\n        {\n            // If `manifest_path` is a path dependency outside of the workspace,\n            // don't add it, or any of its dependencies, as a members.\n            return Ok(());\n        }\n\n        if let WorkspaceConfig::Root(ref root_config) =\n            *self.packages.load(root_manifest)?.workspace_config()\n        {\n            if root_config.is_excluded(&manifest_path) {\n                return Ok(());\n            }\n        }\n\n        debug!(\"find_members - {}\", manifest_path.display());\n        self.members.push(manifest_path.clone());\n\n        let candidates = {\n            let pkg = match *self.packages.load(&manifest_path)? {\n                MaybePackage::Package(ref p) => p,\n                MaybePackage::Virtual(_) => return Ok(()),\n            };\n            self.member_ids.insert(pkg.package_id());\n            pkg.dependencies()\n                .iter()\n                .map(|d| (d.source_id(), d.package_name()))\n                .filter(|(s, _)| s.is_path())\n                .filter_map(|(s, n)| s.url().to_file_path().ok().map(|p| (p, n)))\n                .map(|(p, n)| (p.join(\"Cargo.toml\"), n))\n                .collect::<Vec<_>>()\n        };\n        for (path, name) in candidates {\n            self.find_path_deps(&path, root_manifest, true)\n                .with_context(|| format!(\"failed to load manifest for dependency `{}`\", name))\n                .map_err(|err| ManifestError::new(err, manifest_path.clone()))?;\n        }\n        Ok(())\n    }\n\n    /// Returns the unstable nightly-only features enabled via `cargo-features` in the manifest.\n    pub fn unstable_features(&self) -> &Features {\n        match self.root_maybe() {\n            MaybePackage::Package(p) => p.manifest().unstable_features(),\n            MaybePackage::Virtual(vm) => vm.unstable_features(),\n        }\n    }\n\n    pub fn resolve_behavior(&self) -> ResolveBehavior {\n        self.resolve_behavior\n    }\n\n    /// Returns `true` if this workspace uses the new CLI features behavior.\n    ///\n    /// The old behavior only allowed choosing the features from the package\n    /// in the current directory, regardless of which packages were chosen\n    /// with the -p flags. The new behavior allows selecting features from the\n    /// packages chosen on the command line (with -p or --workspace flags),\n    /// ignoring whatever is in the current directory.\n    pub fn allows_new_cli_feature_behavior(&self) -> bool {\n        self.is_virtual()\n            || match self.resolve_behavior() {\n                ResolveBehavior::V1 => false,\n                ResolveBehavior::V2 => true,\n            }\n    }\n\n    /// Validates a workspace, ensuring that a number of invariants are upheld:\n    ///\n    /// 1. A workspace only has one root.\n    /// 2. All workspace members agree on this one root as the root.\n    /// 3. The current crate is a member of this workspace.\n    fn validate(&mut self) -> CargoResult<()> {\n        // The rest of the checks require a VirtualManifest or multiple members.\n        if self.root_manifest.is_none() {\n            return Ok(());\n        }\n\n        self.validate_unique_names()?;\n        self.validate_workspace_roots()?;\n        self.validate_members()?;\n        self.error_if_manifest_not_in_members()?;\n        self.validate_manifest()\n    }\n\n    fn validate_unique_names(&self) -> CargoResult<()> {\n        let mut names = BTreeMap::new();\n        for member in self.members.iter() {\n            let package = self.packages.get(member);\n            let name = match *package {\n                MaybePackage::Package(ref p) => p.name(),\n                MaybePackage::Virtual(_) => continue,\n            };\n            if let Some(prev) = names.insert(name, member) {\n                bail!(\n                    \"two packages named `{}` in this workspace:\\n\\\n                         - {}\\n\\\n                         - {}\",\n                    name,\n                    prev.display(),\n                    member.display()\n                );\n            }\n        }\n        Ok(())\n    }\n\n    fn validate_workspace_roots(&self) -> CargoResult<()> {\n        let roots: Vec<PathBuf> = self\n            .members\n            .iter()\n            .filter(|&member| {\n                let config = self.packages.get(member).workspace_config();\n                matches!(config, WorkspaceConfig::Root(_))\n            })\n            .map(|member| member.parent().unwrap().to_path_buf())\n            .collect();\n        match roots.len() {\n            1 => Ok(()),\n            0 => bail!(\n                \"`package.workspace` configuration points to a crate \\\n                 which is not configured with [workspace]: \\n\\\n                 configuration at: {}\\n\\\n                 points to: {}\",\n                self.current_manifest.display(),\n                self.root_manifest.as_ref().unwrap().display()\n            ),\n            _ => {\n                bail!(\n                    \"multiple workspace roots found in the same workspace:\\n{}\",\n                    roots\n                        .iter()\n                        .map(|r| format!(\"  {}\", r.display()))\n                        .collect::<Vec<_>>()\n                        .join(\"\\n\")\n                );\n            }\n        }\n    }\n\n    fn validate_members(&mut self) -> CargoResult<()> {\n        for member in self.members.clone() {\n            let root = self.find_root(&member)?;\n            if root == self.root_manifest {\n                continue;\n            }\n\n            match root {\n                Some(root) => {\n                    bail!(\n                        \"package `{}` is a member of the wrong workspace\\n\\\n                         expected: {}\\n\\\n                         actual:   {}\",\n                        member.display(),\n                        self.root_manifest.as_ref().unwrap().display(),\n                        root.display()\n                    );\n                }\n                None => {\n                    bail!(\n                        \"workspace member `{}` is not hierarchically below \\\n                         the workspace root `{}`\",\n                        member.display(),\n                        self.root_manifest.as_ref().unwrap().display()\n                    );\n                }\n            }\n        }\n        Ok(())\n    }\n\n    fn error_if_manifest_not_in_members(&mut self) -> CargoResult<()> {\n        if self.members.contains(&self.current_manifest) {\n            return Ok(());\n        }\n\n        let root = self.root_manifest.as_ref().unwrap();\n        let root_dir = root.parent().unwrap();\n        let current_dir = self.current_manifest.parent().unwrap();\n        let root_pkg = self.packages.get(root);\n\n        // FIXME: Make this more generic by using a relative path resolver between member and root.\n        let members_msg = match current_dir.strip_prefix(root_dir) {\n            Ok(rel) => format!(\n                \"this may be fixable by adding `{}` to the \\\n                     `workspace.members` array of the manifest \\\n                     located at: {}\",\n                rel.display(),\n                root.display()\n            ),\n            Err(_) => format!(\n                \"this may be fixable by adding a member to \\\n                     the `workspace.members` array of the \\\n                     manifest located at: {}\",\n                root.display()\n            ),\n        };\n        let extra = match *root_pkg {\n            MaybePackage::Virtual(_) => members_msg,\n            MaybePackage::Package(ref p) => {\n                let has_members_list = match *p.manifest().workspace_config() {\n                    WorkspaceConfig::Root(ref root_config) => root_config.has_members_list(),\n                    WorkspaceConfig::Member { .. } => unreachable!(),\n                };\n                if !has_members_list {\n                    format!(\n                        \"this may be fixable by ensuring that this \\\n                             crate is depended on by the workspace \\\n                             root: {}\",\n                        root.display()\n                    )\n                } else {\n                    members_msg\n                }\n            }\n        };\n        bail!(\n            \"current package believes it's in a workspace when it's not:\\n\\\n                 current:   {}\\n\\\n                 workspace: {}\\n\\n{}\\n\\\n                 Alternatively, to keep it out of the workspace, add the package \\\n                 to the `workspace.exclude` array, or add an empty `[workspace]` \\\n                 table to the package's manifest.\",\n            self.current_manifest.display(),\n            root.display(),\n            extra\n        );\n    }\n\n    fn validate_manifest(&mut self) -> CargoResult<()> {\n        if let Some(ref root_manifest) = self.root_manifest {\n            for pkg in self\n                .members()\n                .filter(|p| p.manifest_path() != root_manifest)\n            {\n                let manifest = pkg.manifest();\n                let emit_warning = |what| -> CargoResult<()> {\n                    let msg = format!(\n                        \"{} for the non root package will be ignored, \\\n                         specify {} at the workspace root:\\n\\\n                         package:   {}\\n\\\n                         workspace: {}\",\n                        what,\n                        what,\n                        pkg.manifest_path().display(),\n                        root_manifest.display(),\n                    );\n                    self.gctx.shell().warn(&msg)\n                };\n                if manifest.resolved_toml().has_profiles() {\n                    emit_warning(\"profiles\")?;\n                }\n                if !manifest.replace().is_empty() {\n                    emit_warning(\"replace\")?;\n                }\n                if !manifest.patch().is_empty() {\n                    emit_warning(\"patch\")?;\n                }\n                if let Some(behavior) = manifest.resolve_behavior() {\n                    if behavior != self.resolve_behavior {\n                        // Only warn if they don't match.\n                        emit_warning(\"resolver\")?;\n                    }\n                }\n            }\n            if let MaybePackage::Virtual(vm) = self.root_maybe() {\n                if vm.resolve_behavior().is_none() {\n                    if let Some(edition) = self\n                        .members()\n                        .filter(|p| p.manifest_path() != root_manifest)\n                        .map(|p| p.manifest().edition())\n                        .filter(|&e| e >= Edition::Edition2021)\n                        .max()\n                    {\n                        let resolver = edition.default_resolve_behavior().to_manifest();\n                        self.gctx.shell().warn(format_args!(\n                            \"virtual workspace defaulting to `resolver = \\\"1\\\"` despite one or more workspace members being on edition {edition} which implies `resolver = \\\"{resolver}\\\"`\"\n                        ))?;\n                        self.gctx.shell().note(\n                            \"to keep the current resolver, specify `workspace.resolver = \\\"1\\\"` in the workspace root's manifest\",\n                        )?;\n                        self.gctx.shell().note(format_args!(\n                            \"to use the edition {edition} resolver, specify `workspace.resolver = \\\"{resolver}\\\"` in the workspace root's manifest\"\n                        ))?;\n                        self.gctx.shell().note(\n                            \"for more details see https://doc.rust-lang.org/cargo/reference/resolver.html#resolver-versions\",\n                        )?;\n                    }\n                }\n            }\n        }\n        Ok(())\n    }\n\n    pub fn load(&self, manifest_path: &Path) -> CargoResult<Package> {\n        match self.packages.maybe_get(manifest_path) {\n            Some(MaybePackage::Package(p)) => return Ok(p.clone()),\n            Some(&MaybePackage::Virtual(_)) => bail!(\"cannot load workspace root\"),\n            None => {}\n        }\n\n        let mut loaded = self.loaded_packages.borrow_mut();\n        if let Some(p) = loaded.get(manifest_path).cloned() {\n            return Ok(p);\n        }\n        let source_id = SourceId::for_path(manifest_path.parent().unwrap())?;\n        let package = ops::read_package(manifest_path, source_id, self.gctx)?;\n        loaded.insert(manifest_path.to_path_buf(), package.clone());\n        Ok(package)\n    }\n\n    /// Preload the provided registry with already loaded packages.\n    ///\n    /// A workspace may load packages during construction/parsing/early phases\n    /// for various operations, and this preload step avoids doubly-loading and\n    /// parsing crates on the filesystem by inserting them all into the registry\n    /// with their in-memory formats.\n    pub fn preload(&self, registry: &mut PackageRegistry<'gctx>) {\n        // These can get weird as this generally represents a workspace during\n        // `cargo install`. Things like git repositories will actually have a\n        // `PathSource` with multiple entries in it, so the logic below is\n        // mostly just an optimization for normal `cargo build` in workspaces\n        // during development.\n        if self.is_ephemeral {\n            return;\n        }\n\n        for pkg in self.packages.packages.values() {\n            let pkg = match *pkg {\n                MaybePackage::Package(ref p) => p.clone(),\n                MaybePackage::Virtual(_) => continue,\n            };\n            let mut src = PathSource::new(pkg.root(), pkg.package_id().source_id(), self.gctx);\n            src.preload_with(pkg);\n            registry.add_preloaded(Box::new(src));\n        }\n    }\n\n    pub fn emit_warnings(&self) -> CargoResult<()> {\n        for (path, maybe_pkg) in &self.packages.packages {\n            let path = path.join(\"Cargo.toml\");\n            if let MaybePackage::Package(pkg) = maybe_pkg {\n                self.emit_lints(pkg, &path)?\n            }\n            let warnings = match maybe_pkg {\n                MaybePackage::Package(pkg) => pkg.manifest().warnings().warnings(),\n                MaybePackage::Virtual(vm) => vm.warnings().warnings(),\n            };\n            for warning in warnings {\n                if warning.is_critical {\n                    let err = anyhow::format_err!(\"{}\", warning.message);\n                    let cx =\n                        anyhow::format_err!(\"failed to parse manifest at `{}`\", path.display());\n                    return Err(err.context(cx));\n                } else {\n                    let msg = if self.root_manifest.is_none() {\n                        warning.message.to_string()\n                    } else {\n                        // In a workspace, it can be confusing where a warning\n                        // originated, so include the path.\n                        format!(\"{}: {}\", path.display(), warning.message)\n                    };\n                    self.gctx.shell().warn(msg)?\n                }\n            }\n        }\n        Ok(())\n    }\n\n    pub fn emit_lints(&self, pkg: &Package, path: &Path) -> CargoResult<()> {\n        let mut error_count = 0;\n        let toml_lints = pkg\n            .manifest()\n            .resolved_toml()\n            .lints\n            .clone()\n            .map(|lints| lints.lints)\n            .unwrap_or(manifest::TomlLints::default());\n        let cargo_lints = toml_lints\n            .get(\"cargo\")\n            .cloned()\n            .unwrap_or(manifest::TomlToolLints::default());\n        let normalized_lints = cargo_lints\n            .into_iter()\n            .map(|(name, lint)| (name.replace('-', \"_\"), lint))\n            .collect();\n\n        check_implicit_features(pkg, &path, &normalized_lints, &mut error_count, self.gctx)?;\n        if error_count > 0 {\n            Err(crate::util::errors::AlreadyPrintedError::new(anyhow!(\n                \"encountered {error_count} errors(s) while running lints\"\n            ))\n            .into())\n        } else {\n            Ok(())\n        }\n    }\n\n    pub fn set_target_dir(&mut self, target_dir: Filesystem) {\n        self.target_dir = Some(target_dir);\n    }\n\n    /// Returns a Vec of `(&Package, RequestedFeatures)` tuples that\n    /// represent the workspace members that were requested on the command-line.\n    ///\n    /// `specs` may be empty, which indicates it should return all workspace\n    /// members. In this case, `requested_features.all_features` must be\n    /// `true`. This is used for generating `Cargo.lock`, which must include\n    /// all members with all features enabled.\n    pub fn members_with_features(\n        &self,\n        specs: &[PackageIdSpec],\n        cli_features: &CliFeatures,\n    ) -> CargoResult<Vec<(&Package, CliFeatures)>> {\n        assert!(\n            !specs.is_empty() || cli_features.all_features,\n            \"no specs requires all_features\"\n        );\n        if specs.is_empty() {\n            // When resolving the entire workspace, resolve each member with\n            // all features enabled.\n            return Ok(self\n                .members()\n                .map(|m| (m, CliFeatures::new_all(true)))\n                .collect());\n        }\n        if self.allows_new_cli_feature_behavior() {\n            self.members_with_features_new(specs, cli_features)\n        } else {\n            Ok(self.members_with_features_old(specs, cli_features))\n        }\n    }\n\n    /// Returns the requested features for the given member.\n    /// This filters out any named features that the member does not have.\n    fn collect_matching_features(\n        member: &Package,\n        cli_features: &CliFeatures,\n        found_features: &mut BTreeSet<FeatureValue>,\n    ) -> CliFeatures {\n        if cli_features.features.is_empty() {\n            return cli_features.clone();\n        }\n\n        // Only include features this member defines.\n        let summary = member.summary();\n\n        // Features defined in the manifest\n        let summary_features = summary.features();\n\n        // Dependency name -> dependency\n        let dependencies: BTreeMap<InternedString, &Dependency> = summary\n            .dependencies()\n            .iter()\n            .map(|dep| (dep.name_in_toml(), dep))\n            .collect();\n\n        // Features that enable optional dependencies\n        let optional_dependency_names: BTreeSet<_> = dependencies\n            .iter()\n            .filter(|(_, dep)| dep.is_optional())\n            .map(|(name, _)| name)\n            .copied()\n            .collect();\n\n        let mut features = BTreeSet::new();\n\n        // Checks if a member contains the given feature.\n        let summary_or_opt_dependency_feature = |feature: &InternedString| -> bool {\n            summary_features.contains_key(feature) || optional_dependency_names.contains(feature)\n        };\n\n        for feature in cli_features.features.iter() {\n            match feature {\n                FeatureValue::Feature(f) => {\n                    if summary_or_opt_dependency_feature(f) {\n                        // feature exists in this member.\n                        features.insert(feature.clone());\n                        found_features.insert(feature.clone());\n                    }\n                }\n                // This should be enforced by CliFeatures.\n                FeatureValue::Dep { .. } => panic!(\"unexpected dep: syntax {}\", feature),\n                FeatureValue::DepFeature {\n                    dep_name,\n                    dep_feature,\n                    weak: _,\n                } => {\n                    if dependencies.contains_key(dep_name) {\n                        // pkg/feat for a dependency.\n                        // Will rely on the dependency resolver to validate `dep_feature`.\n                        features.insert(feature.clone());\n                        found_features.insert(feature.clone());\n                    } else if *dep_name == member.name()\n                        && summary_or_opt_dependency_feature(dep_feature)\n                    {\n                        // member/feat where \"feat\" is a feature in member.\n                        //\n                        // `weak` can be ignored here, because the member\n                        // either is or isn't being built.\n                        features.insert(FeatureValue::Feature(*dep_feature));\n                        found_features.insert(feature.clone());\n                    }\n                }\n            }\n        }\n        CliFeatures {\n            features: Rc::new(features),\n            all_features: cli_features.all_features,\n            uses_default_features: cli_features.uses_default_features,\n        }\n    }\n\n    fn report_unknown_features_error(\n        &self,\n        specs: &[PackageIdSpec],\n        cli_features: &CliFeatures,\n        found_features: &BTreeSet<FeatureValue>,\n    ) -> CargoResult<()> {\n        // Keeps track of which features were contained in summary of `member` to suggest similar features in errors\n        let mut summary_features: Vec<InternedString> = Default::default();\n\n        // Keeps track of `member` dependencies (`dep/feature`) and their features names to suggest similar features in error\n        let mut dependencies_features: BTreeMap<InternedString, &[InternedString]> =\n            Default::default();\n\n        // Keeps track of `member` optional dependencies names (which can be enabled with feature) to suggest similar features in error\n        let mut optional_dependency_names: Vec<InternedString> = Default::default();\n\n        // Keeps track of which features were contained in summary of `member` to suggest similar features in errors\n        let mut summary_features_per_member: BTreeMap<&Package, BTreeSet<InternedString>> =\n            Default::default();\n\n        // Keeps track of `member` optional dependencies (which can be enabled with feature) to suggest similar features in error\n        let mut optional_dependency_names_per_member: BTreeMap<&Package, BTreeSet<InternedString>> =\n            Default::default();\n\n        for member in self\n            .members()\n            .filter(|m| specs.iter().any(|spec| spec.matches(m.package_id())))\n        {\n            // Only include features this member defines.\n            let summary = member.summary();\n\n            // Features defined in the manifest\n            summary_features.extend(summary.features().keys());\n            summary_features_per_member\n                .insert(member, summary.features().keys().copied().collect());\n\n            // Dependency name -> dependency\n            let dependencies: BTreeMap<InternedString, &Dependency> = summary\n                .dependencies()\n                .iter()\n                .map(|dep| (dep.name_in_toml(), dep))\n                .collect();\n\n            dependencies_features.extend(\n                dependencies\n                    .iter()\n                    .map(|(name, dep)| (*name, dep.features())),\n            );\n\n            // Features that enable optional dependencies\n            let optional_dependency_names_raw: BTreeSet<_> = dependencies\n                .iter()\n                .filter(|(_, dep)| dep.is_optional())\n                .map(|(name, _)| name)\n                .copied()\n                .collect();\n\n            optional_dependency_names.extend(optional_dependency_names_raw.iter());\n            optional_dependency_names_per_member.insert(member, optional_dependency_names_raw);\n        }\n\n        let edit_distance_test = |a: InternedString, b: InternedString| {\n            edit_distance(a.as_str(), b.as_str(), 3).is_some()\n        };\n\n        let suggestions: Vec<_> = cli_features\n            .features\n            .difference(found_features)\n            .map(|feature| match feature {\n                // Simple feature, check if any of the optional dependency features or member features are close enough\n                FeatureValue::Feature(typo) => {\n                    // Finds member features which are similar to the requested feature.\n                    let summary_features = summary_features\n                        .iter()\n                        .filter(move |feature| edit_distance_test(**feature, *typo));\n\n                    // Finds optional dependencies which name is similar to the feature\n                    let optional_dependency_features = optional_dependency_names\n                        .iter()\n                        .filter(move |feature| edit_distance_test(**feature, *typo));\n\n                    summary_features\n                        .chain(optional_dependency_features)\n                        .map(|s| s.to_string())\n                        .collect::<Vec<_>>()\n                }\n                FeatureValue::Dep { .. } => panic!(\"unexpected dep: syntax {}\", feature),\n                FeatureValue::DepFeature {\n                    dep_name,\n                    dep_feature,\n                    weak: _,\n                } => {\n                    // Finds set of `pkg/feat` that are very similar to current `pkg/feat`.\n                    let pkg_feat_similar = dependencies_features\n                        .iter()\n                        .filter(|(name, _)| edit_distance_test(**name, *dep_name))\n                        .map(|(name, features)| {\n                            (\n                                name,\n                                features\n                                    .iter()\n                                    .filter(|feature| edit_distance_test(**feature, *dep_feature))\n                                    .collect::<Vec<_>>(),\n                            )\n                        })\n                        .map(|(name, features)| {\n                            features\n                                .into_iter()\n                                .map(move |feature| format!(\"{}/{}\", name, feature))\n                        })\n                        .flatten();\n\n                    // Finds set of `member/optional_dep` features which name is similar to current `pkg/feat`.\n                    let optional_dependency_features = optional_dependency_names_per_member\n                        .iter()\n                        .filter(|(package, _)| edit_distance_test(package.name(), *dep_name))\n                        .map(|(package, optional_dependencies)| {\n                            optional_dependencies\n                                .into_iter()\n                                .filter(|optional_dependency| {\n                                    edit_distance_test(**optional_dependency, *dep_name)\n                                })\n                                .map(move |optional_dependency| {\n                                    format!(\"{}/{}\", package.name(), optional_dependency)\n                                })\n                        })\n                        .flatten();\n\n                    // Finds set of `member/feat` features which name is similar to current `pkg/feat`.\n                    let summary_features = summary_features_per_member\n                        .iter()\n                        .filter(|(package, _)| edit_distance_test(package.name(), *dep_name))\n                        .map(|(package, summary_features)| {\n                            summary_features\n                                .into_iter()\n                                .filter(|summary_feature| {\n                                    edit_distance_test(**summary_feature, *dep_feature)\n                                })\n                                .map(move |summary_feature| {\n                                    format!(\"{}/{}\", package.name(), summary_feature)\n                                })\n                        })\n                        .flatten();\n\n                    pkg_feat_similar\n                        .chain(optional_dependency_features)\n                        .chain(summary_features)\n                        .collect::<Vec<_>>()\n                }\n            })\n            .map(|v| v.into_iter())\n            .flatten()\n            .unique()\n            .filter(|element| {\n                let feature = FeatureValue::new(InternedString::new(element));\n                !cli_features.features.contains(&feature) && !found_features.contains(&feature)\n            })\n            .sorted()\n            .take(5)\n            .collect();\n\n        let unknown: Vec<_> = cli_features\n            .features\n            .difference(found_features)\n            .map(|feature| feature.to_string())\n            .sorted()\n            .collect();\n\n        if suggestions.is_empty() {\n            bail!(\n                \"none of the selected packages contains these features: {}\",\n                unknown.join(\", \")\n            );\n        } else {\n            bail!(\n                \"none of the selected packages contains these features: {}, did you mean: {}?\",\n                unknown.join(\", \"),\n                suggestions.join(\", \")\n            );\n        }\n    }\n\n    /// New command-line feature selection behavior with resolver = \"2\" or the\n    /// root of a virtual workspace. See `allows_new_cli_feature_behavior`.\n    fn members_with_features_new(\n        &self,\n        specs: &[PackageIdSpec],\n        cli_features: &CliFeatures,\n    ) -> CargoResult<Vec<(&Package, CliFeatures)>> {\n        // Keeps track of which features matched `member` to produce an error\n        // if any of them did not match anywhere.\n        let mut found_features = Default::default();\n\n        let members: Vec<(&Package, CliFeatures)> = self\n            .members()\n            .filter(|m| specs.iter().any(|spec| spec.matches(m.package_id())))\n            .map(|m| {\n                (\n                    m,\n                    Workspace::collect_matching_features(m, cli_features, &mut found_features),\n                )\n            })\n            .collect();\n\n        if members.is_empty() {\n            // `cargo build -p foo`, where `foo` is not a member.\n            // Do not allow any command-line flags (defaults only).\n            if !(cli_features.features.is_empty()\n                && !cli_features.all_features\n                && cli_features.uses_default_features)\n            {\n                bail!(\"cannot specify features for packages outside of workspace\");\n            }\n            // Add all members from the workspace so we can ensure `-p nonmember`\n            // is in the resolve graph.\n            return Ok(self\n                .members()\n                .map(|m| (m, CliFeatures::new_all(false)))\n                .collect());\n        }\n        if *cli_features.features != found_features {\n            self.report_unknown_features_error(specs, cli_features, &found_features)?;\n        }\n        Ok(members)\n    }\n\n    /// This is the \"old\" behavior for command-line feature selection.\n    /// See `allows_new_cli_feature_behavior`.\n    fn members_with_features_old(\n        &self,\n        specs: &[PackageIdSpec],\n        cli_features: &CliFeatures,\n    ) -> Vec<(&Package, CliFeatures)> {\n        // Split off any features with the syntax `member-name/feature-name` into a map\n        // so that those features can be applied directly to those workspace-members.\n        let mut member_specific_features: HashMap<InternedString, BTreeSet<FeatureValue>> =\n            HashMap::new();\n        // Features for the member in the current directory.\n        let mut cwd_features = BTreeSet::new();\n        for feature in cli_features.features.iter() {\n            match feature {\n                FeatureValue::Feature(_) => {\n                    cwd_features.insert(feature.clone());\n                }\n                // This should be enforced by CliFeatures.\n                FeatureValue::Dep { .. } => panic!(\"unexpected dep: syntax {}\", feature),\n                FeatureValue::DepFeature {\n                    dep_name,\n                    dep_feature,\n                    weak: _,\n                } => {\n                    // I think weak can be ignored here.\n                    // * With `--features member?/feat -p member`, the ? doesn't\n                    //   really mean anything (either the member is built or it isn't).\n                    // * With `--features nonmember?/feat`, cwd_features will\n                    //   handle processing it correctly.\n                    let is_member = self.members().any(|member| {\n                        // Check if `dep_name` is member of the workspace, but isn't associated with current package.\n                        self.current_opt() != Some(member) && member.name() == *dep_name\n                    });\n                    if is_member && specs.iter().any(|spec| spec.name() == dep_name.as_str()) {\n                        member_specific_features\n                            .entry(*dep_name)\n                            .or_default()\n                            .insert(FeatureValue::Feature(*dep_feature));\n                    } else {\n                        cwd_features.insert(feature.clone());\n                    }\n                }\n            }\n        }\n\n        let ms: Vec<_> = self\n            .members()\n            .filter_map(|member| {\n                let member_id = member.package_id();\n                match self.current_opt() {\n                    // The features passed on the command-line only apply to\n                    // the \"current\" package (determined by the cwd).\n                    Some(current) if member_id == current.package_id() => {\n                        let feats = CliFeatures {\n                            features: Rc::new(cwd_features.clone()),\n                            all_features: cli_features.all_features,\n                            uses_default_features: cli_features.uses_default_features,\n                        };\n                        Some((member, feats))\n                    }\n                    _ => {\n                        // Ignore members that are not enabled on the command-line.\n                        if specs.iter().any(|spec| spec.matches(member_id)) {\n                            // -p for a workspace member that is not the \"current\"\n                            // one.\n                            //\n                            // The odd behavior here is due to backwards\n                            // compatibility. `--features` and\n                            // `--no-default-features` used to only apply to the\n                            // \"current\" package. As an extension, this allows\n                            // member-name/feature-name to set member-specific\n                            // features, which should be backwards-compatible.\n                            let feats = CliFeatures {\n                                features: Rc::new(\n                                    member_specific_features\n                                        .remove(member.name().as_str())\n                                        .unwrap_or_default(),\n                                ),\n                                uses_default_features: true,\n                                all_features: cli_features.all_features,\n                            };\n                            Some((member, feats))\n                        } else {\n                            // This member was not requested on the command-line, skip.\n                            None\n                        }\n                    }\n                }\n            })\n            .collect();\n\n        // If any member specific features were not removed while iterating over members\n        // some features will be ignored.\n        assert!(member_specific_features.is_empty());\n\n        ms\n    }\n\n    /// Returns true if `unit` should depend on the output of Docscrape units.\n    \npub fn unit_needs_doc_scrape(&self, unit: &Unit) -> bool {\n        // We do not add scraped units for Host units, as they're either build scripts\n        // (not documented) or proc macros (have no scrape-able exports). Additionally,\n        // naively passing a proc macro's unit_for to new_unit_dep will currently cause\n        // Cargo to panic, see issue #10545.\n        self.is_member(&unit.pkg) && !(unit.target.for_host() || unit.pkg.proc_macro())\n    }\n}\n\nimpl<'gctx> Packages<'gctx> {\n    fn get(&self, manifest_path: &Path) -> &MaybePackage {\n        self.maybe_get(manifest_path).unwrap()\n    }\n\n    fn get_mut(&mut self, manifest_path: &Path) -> &mut MaybePackage {\n        self.maybe_get_mut(manifest_path).unwrap()\n    }\n\n    fn maybe_get(&self, manifest_path: &Path) -> Option<&MaybePackage> {\n        self.packages.get(manifest_path.parent().unwrap())\n    }\n\n    fn maybe_get_mut(&mut self, manifest_path: &Path) -> Option<&mut MaybePackage> {\n        self.packages.get_mut(manifest_path.parent().unwrap())\n    }\n\n    fn load(&mut self, manifest_path: &Path) -> CargoResult<&MaybePackage> {\n        let key = manifest_path.parent().unwrap();\n        match self.packages.entry(key.to_path_buf()) {\n            Entry::Occupied(e) => Ok(e.into_mut()),\n            Entry::Vacant(v) => {\n                let source_id = SourceId::for_path(key)?;\n                let manifest = read_manifest(manifest_path, source_id, self.gctx)?;\n                Ok(v.insert(match manifest {\n                    EitherManifest::Real(manifest) => {\n                        MaybePackage::Package(Package::new(manifest, manifest_path))\n                    }\n                    EitherManifest::Virtual(vm) => MaybePackage::Virtual(vm),\n                }))\n            }\n        }\n    }\n}\n\nimpl MaybePackage {\n    fn workspace_config(&self) -> &WorkspaceConfig {\n        match *self {\n            MaybePackage::Package(ref p) => p.manifest().workspace_config(),\n            MaybePackage::Virtual(ref vm) => vm.workspace_config(),\n        }\n    }\n\n    /// Has an embedded manifest (single-file package)\n    pub fn is_embedded(&self) -> bool {\n        match self {\n            MaybePackage::Package(p) => p.manifest().is_embedded(),\n            MaybePackage::Virtual(_) => false,\n        }\n    }\n}\n\nimpl WorkspaceRootConfig {\n    /// Creates a new Intermediate Workspace Root configuration.\n    pub fn new(\n        root_dir: &Path,\n        members: &Option<Vec<String>>,\n        default_members: &Option<Vec<String>>,\n        exclude: &Option<Vec<String>>,\n        inheritable: &Option<InheritableFields>,\n        custom_metadata: &Option<toml::Value>,\n    ) -> WorkspaceRootConfig {\n        WorkspaceRootConfig {\n            root_dir: root_dir.to_path_buf(),\n            members: members.clone(),\n            default_members: default_members.clone(),\n            exclude: exclude.clone().unwrap_or_default(),\n            inheritable_fields: inheritable.clone().unwrap_or_default(),\n            custom_metadata: custom_metadata.clone(),\n        }\n    }\n    /// Checks the path against the `excluded` list.\n    ///\n    /// This method does **not** consider the `members` list.\n    fn is_excluded(&self, manifest_path: &Path) -> bool {\n        let excluded = self\n            .exclude\n            .iter()\n            .any(|ex| manifest_path.starts_with(self.root_dir.join(ex)));\n\n        let explicit_member = match self.members {\n            Some(ref members) => members\n                .iter()\n                .any(|mem| manifest_path.starts_with(self.root_dir.join(mem))),\n            None => false,\n        };\n\n        !explicit_member && excluded\n    }\n\n    fn has_members_list(&self) -> bool {\n        self.members.is_some()\n    }\n\n    fn members_paths(&self, globs: &[String]) -> CargoResult<Vec<PathBuf>> {\n        let mut expanded_list = Vec::new();\n\n        for glob in globs {\n            let pathbuf = self.root_dir.join(glob);\n            let expanded_paths = Self::expand_member_path(&pathbuf)?;\n\n            // If glob does not find any valid paths, then put the original\n            // path in the expanded list to maintain backwards compatibility.\n            if expanded_paths.is_empty() {\n                expanded_list.push(pathbuf);\n            } else {\n                // Some OS can create system support files anywhere.\n                // (e.g. macOS creates `.DS_Store` file if you visit a directory using Finder.)\n                // Such files can be reported as a member path unexpectedly.\n                // Check and filter out non-directory paths to prevent pushing such accidental unwanted path\n                // as a member.\n                for expanded_path in expanded_paths {\n                    if expanded_path.is_dir() {\n                        expanded_list.push(expanded_path);\n                    }\n                }\n            }\n        }\n\n        Ok(expanded_list)\n    }\n\n    fn expand_member_path(path: &Path) -> CargoResult<Vec<PathBuf>> {\n        let Some(path) = path.to_str() else {\n            return Ok(Vec::new());\n        };\n        let res = glob(path).with_context(|| format!(\"could not parse pattern `{}`\", &path))?;\n        let res = res\n            .map(|p| p.with_context(|| format!(\"unable to match path to pattern `{}`\", &path)))\n            .collect::<Result<Vec<_>, _>>()?;\n        Ok(res)\n    }\n\n    pub fn inheritable(&self) -> &InheritableFields {\n        &self.inheritable_fields\n    }\n}\n\npub fn resolve_relative_path(\n    label: &str,\n    old_root: &Path,\n    new_root: &Path,\n    rel_path: &str,\n) -> CargoResult<String> {\n    let joined_path = normalize_path(&old_root.join(rel_path));\n    match diff_paths(joined_path, new_root) {\n        None => Err(anyhow!(\n            \"`{}` was defined in {} but could not be resolved with {}\",\n            label,\n            old_root.display(),\n            new_root.display()\n        )),\n        Some(path) => Ok(path\n            .to_str()\n            .ok_or_else(|| {\n                anyhow!(\n                    \"`{}` resolved to non-UTF value (`{}`)\",\n                    label,\n                    path.display()\n                )\n            })?\n            .to_owned()),\n    }\n}\n\n/// Finds the path of the root of the workspace.\npub fn find_workspace_root(\n    manifest_path: &Path,\n    gctx: &GlobalContext,\n) -> CargoResult<Option<PathBuf>> {\n    find_workspace_root_with_loader(manifest_path, gctx, |self_path| {\n        let key = self_path.parent().unwrap();\n        let source_id = SourceId::for_path(key)?;\n        let manifest = read_manifest(self_path, source_id, gctx)?;\n        Ok(manifest\n            .workspace_config()\n            .get_ws_root(self_path, manifest_path))\n    })\n}\n\n/// Finds the path of the root of the workspace.\n///\n/// This uses a callback to determine if the given path tells us what the\n/// workspace root is.\nfn find_workspace_root_with_loader(\n    manifest_path: &Path,\n    gctx: &GlobalContext,\n    mut loader: impl FnMut(&Path) -> CargoResult<Option<PathBuf>>,\n) -> CargoResult<Option<PathBuf>> {\n    // Check if there are any workspace roots that have already been found that would work\n    {\n        let roots = gctx.ws_roots.borrow();\n        // Iterate through the manifests parent directories until we find a workspace\n        // root. Note we skip the first item since that is just the path itself\n        for current in manifest_path.ancestors().skip(1) {\n            if let Some(ws_config) = roots.get(current) {\n                if !ws_config.is_excluded(manifest_path) {\n                    // Add `Cargo.toml` since ws_root is the root and not the file\n                    return Ok(Some(current.join(\"Cargo.toml\")));\n                }\n            }\n        }\n    }\n\n    for ances_manifest_path in find_root_iter(manifest_path, gctx) {\n        debug!(\"find_root - trying {}\", ances_manifest_path.display());\n        if let Some(ws_root_path) = loader(&ances_manifest_path)? {\n            return Ok(Some(ws_root_path));\n        }\n    }\n    Ok(None)\n}\n\nfn read_root_pointer(member_manifest: &Path, root_link: &str) -> PathBuf {\n    let path = member_manifest\n        .parent()\n        .unwrap()\n        .join(root_link)\n        .join(\"Cargo.toml\");\n    debug!(\"find_root - pointer {}\", path.display());\n    paths::normalize_path(&path)\n}\n\nfn find_root_iter<'a>(\n    manifest_path: &'a Path,\n    gctx: &'a GlobalContext,\n) -> impl Iterator<Item = PathBuf> + 'a {\n    LookBehind::new(paths::ancestors(manifest_path, None).skip(2))\n        .take_while(|path| !path.curr.ends_with(\"target/package\"))\n        // Don't walk across `CARGO_HOME` when we're looking for the\n        // workspace root. Sometimes a package will be organized with\n        // `CARGO_HOME` pointing inside of the workspace root or in the\n        // current package, but we don't want to mistakenly try to put\n        // crates.io crates into the workspace by accident.\n        .take_while(|path| {\n            if let Some(last) = path.last {\n                gctx.home() != last\n            } else {\n                true\n            }\n        })\n        .map(|path| path.curr.join(\"Cargo.toml\"))\n        .filter(|ances_manifest_path| ances_manifest_path.exists())\n}\n\nstruct LookBehindWindow<'a, T: ?Sized> {\n    curr: &'a T,\n    last: Option<&'a T>,\n}\n\nstruct LookBehind<'a, T: ?Sized, K: Iterator<Item = &'a T>> {\n    iter: K,\n    last: Option<&'a T>,\n}\n\nimpl<'a, T: ?Sized, K: Iterator<Item = &'a T>> LookBehind<'a, T, K> {\n    fn new(items: K) -> Self {\n        Self {\n            iter: items,\n            last: None,\n        }\n    }\n}\n\nimpl<'a, T: ?Sized, K: Iterator<Item = &'a T>> Iterator for LookBehind<'a, T, K> {\n    type Item = LookBehindWindow<'a, T>;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        match self.iter.next() {\n            None => None,\n            Some(next) => {\n                let last = self.last;\n                self.last = Some(next);\n                Some(LookBehindWindow { curr: next, last })\n            }\n        }\n    }\n}\n\n// Path: src/cargo/util/rustc.rs\nuse std::collections::hash_map::HashMap;\nuse std::env;\nuse std::hash::{Hash, Hasher};\nuse std::path::{Path, PathBuf};\nuse std::sync::Mutex;\n\nuse anyhow::Context as _;\nuse cargo_util::{paths, ProcessBuilder, ProcessError};\nuse serde::{Deserialize, Serialize};\nuse tracing::{debug, info, warn};\n\nuse crate::util::interning::InternedString;\nuse crate::util::{CargoResult, GlobalContext, StableHasher};\n\n/// Information on the `rustc` executable\n#[derive(Debug)]\npub struct Rustc {\n    /// The location of the exe\n    pub path: PathBuf,\n    /// An optional program that will be passed the path of the rust exe as its first argument, and\n    /// rustc args following this.\n    pub wrapper: Option<PathBuf>,\n    /// An optional wrapper to be used in addition to `rustc.wrapper` for workspace crates\n    pub workspace_wrapper: Option<PathBuf>,\n    /// Verbose version information (the output of `rustc -vV`)\n    pub verbose_version: String,\n    /// The rustc version (`1.23.4-beta.2`), this comes from verbose_version.\n    pub version: semver::Version,\n    /// The host triple (arch-platform-OS), this comes from verbose_version.\n    pub host: InternedString,\n    /// The rustc full commit hash, this comes from `verbose_version`.\n    pub commit_hash: Option<String>,\n    cache: Mutex<Cache>,\n}\n\nimpl Rustc {\n    /// Runs the compiler at `path` to learn various pieces of information about\n    /// it, with an optional wrapper.\n    ///\n    /// If successful this function returns a description of the compiler along\n    /// with a list of its capabilities.\n    #[tracing::instrument(skip(gctx))]\n    pub fn new(\n        path: PathBuf,\n        wrapper: Option<PathBuf>,\n        workspace_wrapper: Option<PathBuf>,\n        rustup_rustc: &Path,\n        cache_location: Option<PathBuf>,\n        gctx: &GlobalContext,\n    ) -> CargoResult<Rustc> {\n        let mut cache = Cache::load(\n            wrapper.as_deref(),\n            workspace_wrapper.as_deref(),\n            &path,\n            rustup_rustc,\n            cache_location,\n            gctx,\n        );\n\n        let mut cmd = ProcessBuilder::new(&path);\n        cmd.arg(\"-vV\");\n        let verbose_version = cache.cached_output(&cmd, 0)?.0;\n\n        let extract = |field: &str| -> CargoResult<&str> {\n            verbose_version\n                .lines()\n                .find_map(|l| l.strip_prefix(field))\n                .ok_or_else(|| {\n                    anyhow::format_err!(\n                        \"`rustc -vV` didn't have a line for `{}`, got:\\n{}\",\n                        field.trim(),\n                        verbose_version\n                    )\n                })\n        };\n\n        let host = InternedString::new(extract(\"host: \")?);\n        let version = semver::Version::parse(extract(\"release: \")?).with_context(|| {\n            format!(\n                \"rustc version does not appear to be a valid semver version, from:\\n{}\",\n                verbose_version\n            )\n        })?;\n        let commit_hash = extract(\"commit-hash: \").ok().map(|hash| {\n            // Possible commit-hash values from rustc are SHA hex string and \"unknown\". See:\n            // * https://github.com/rust-lang/rust/blob/531cb83fc/src/bootstrap/src/utils/channel.rs#L73\n            // * https://github.com/rust-lang/rust/blob/531cb83fc/compiler/rustc_driver_impl/src/lib.rs#L911-L913\n            #[cfg(debug_assertions)]\n            if hash != \"unknown\" {\n                debug_assert!(\n                    hash.chars().all(|ch| ch.is_ascii_hexdigit()),\n                    \"commit hash must be a hex string, got: {hash:?}\"\n                );\n                debug_assert!(\n                    hash.len() == 40 || hash.len() == 64,\n                    \"hex string must be generated from sha1 or sha256 (i.e., it must be 40 or 64 characters long)\\ngot: {hash:?}\"\n                );\n            }\n            hash.to_string()\n        });\n\n        Ok(Rustc {\n            path,\n            wrapper,\n            workspace_wrapper,\n            verbose_version,\n            version,\n            host,\n            commit_hash,\n            cache: Mutex::new(cache),\n        })\n    }\n\n    /// Gets a process builder set up to use the found rustc version, with a wrapper if `Some`.\n    pub fn process(&self) -> ProcessBuilder {\n        let mut cmd = ProcessBuilder::new(self.path.as_path()).wrapped(self.wrapper.as_ref());\n        cmd.retry_with_argfile(true);\n        cmd\n    }\n\n    /// Gets a process builder set up to use the found rustc version, with a wrapper if `Some`.\n    pub fn workspace_process(&self) -> ProcessBuilder {\n        let mut cmd = ProcessBuilder::new(self.path.as_path())\n            .wrapped(self.workspace_wrapper.as_ref())\n            .wrapped(self.wrapper.as_ref());\n        cmd.retry_with_argfile(true);\n        cmd\n    }\n\n    pub fn process_no_wrapper(&self) -> ProcessBuilder {\n        let mut cmd = ProcessBuilder::new(&self.path);\n        cmd.retry_with_argfile(true);\n        cmd\n    }\n\n    /// Gets the output for the given command.\n    ///\n    /// This will return the cached value if available, otherwise it will run\n    /// the command and cache the output.\n    ///\n    /// `extra_fingerprint` is extra data to include in the cache fingerprint.\n    /// Use this if there is other information about the environment that may\n    /// affect the output that is not part of `cmd`.\n    ///\n    /// Returns a tuple of strings `(stdout, stderr)`.\n    pub fn cached_output(\n        &self,\n        cmd: &ProcessBuilder,\n        extra_fingerprint: u64,\n    ) -> CargoResult<(String, String)> {\n        self.cache\n            .lock()\n            .unwrap()\n            .cached_output(cmd, extra_fingerprint)\n    }\n}\n\n/// It is a well known fact that `rustc` is not the fastest compiler in the\n/// world.  What is less known is that even `rustc --version --verbose` takes\n/// about a hundred milliseconds! Because we need compiler version info even\n/// for no-op builds, we cache it here, based on compiler's mtime and rustup's\n/// current toolchain.\n///\n/// <https://github.com/rust-lang/cargo/issues/5315>\n/// <https://github.com/rust-lang/rust/issues/49761>\n#[derive(Debug)]\nstruct Cache {\n    cache_location: Option<PathBuf>,\n    dirty: bool,\n    data: CacheData,\n}\n\n#[derive(Serialize, Deserialize, Debug, Default)]\nstruct CacheData {\n    rustc_fingerprint: u64,\n    outputs: HashMap<u64, Output>,\n    successes: HashMap<u64, bool>,\n}\n\n#[derive(Serialize, Deserialize, Debug)]\nstruct Output {\n    success: bool,\n    status: String,\n    code: Option<i32>,\n    stdout: String,\n    stderr: String,\n}\n\nimpl Cache {\n    fn load(\n        wrapper: Option<&Path>,\n        workspace_wrapper: Option<&Path>,\n        rustc: &Path,\n        rustup_rustc: &Path,\n        cache_location: Option<PathBuf>,\n        gctx: &GlobalContext,\n    ) -> Cache {\n        match (\n            cache_location,\n            rustc_fingerprint(wrapper, workspace_wrapper, rustc, rustup_rustc, gctx),\n        ) {\n            (Some(cache_location), Ok(rustc_fingerprint)) => {\n                let empty = CacheData {\n                    rustc_fingerprint,\n                    outputs: HashMap::new(),\n                    successes: HashMap::new(),\n                };\n                let mut dirty = true;\n                let data = match read(&cache_location) {\n                    Ok(data) => {\n                        if data.rustc_fingerprint == rustc_fingerprint {\n                            debug!(\"reusing existing rustc info cache\");\n                            dirty = false;\n                            data\n                        } else {\n                            debug!(\"different compiler, creating new rustc info cache\");\n                            empty\n                        }\n                    }\n                    Err(e) => {\n                        debug!(\"failed to read rustc info cache: {}\", e);\n                        empty\n                    }\n                };\n                return Cache {\n                    cache_location: Some(cache_location),\n                    dirty,\n                    data,\n                };\n\n                fn read(path: &Path) -> CargoResult<CacheData> {\n                    let json = paths::read(path)?;\n                    Ok(serde_json::from_str(&json)?)\n                }\n            }\n            (_, fingerprint) => {\n                if let Err(e) = fingerprint {\n                    warn!(\"failed to calculate rustc fingerprint: {}\", e);\n                }\n                debug!(\"rustc info cache disabled\");\n                Cache {\n                    cache_location: None,\n                    dirty: false,\n                    data: CacheData::default(),\n                }\n            }\n        }\n    }\n\n    fn cached_output(\n        &mut self,\n        cmd: &ProcessBuilder,\n        extra_fingerprint: u64,\n    ) -> CargoResult<(String, String)> {\n        let key = process_fingerprint(cmd, extra_fingerprint);\n        if self.data.outputs.contains_key(&key) {\n            debug!(\"rustc info cache hit\");\n        } else {\n            debug!(\"rustc info cache miss\");\n            debug!(\"running {}\", cmd);\n            let output = cmd.output()?;\n            let stdout = String::from_utf8(output.stdout)\n                .map_err(|e| anyhow::anyhow!(\"{}: {:?}\", e, e.as_bytes()))\n                .with_context(|| format!(\"`{}` didn't return utf8 output\", cmd))?;\n            let stderr = String::from_utf8(output.stderr)\n                .map_err(|e| anyhow::anyhow!(\"{}: {:?}\", e, e.as_bytes()))\n                .with_context(|| format!(\"`{}` didn't return utf8 output\", cmd))?;\n            self.data.outputs.insert(\n                key,\n                Output {\n...\n", "needle_token_start": 10491, "needle_token_end": 10634, "code_context_ntokens": 16254, "output": ["```rust\npub fn unit_needs_doc_scrape(&self, unit: &Unit) -> bool {\n    // We do not add scraped units for Host units, as they're either build scripts\n    // (not documented) or proc macros (have no scrape-able exports). Additionally,\n    // naively passing a proc macro's unit_for to new_unit_dep will currently cause\n    // Cargo to panic, see issue #10545.\n    self.is_member(&unit.pkg) && !(unit.target.for_host() || unit.pkg.proc_macro())\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "unsupported", "language": "rust", "path": "src/bin/cargo/commands/vendor.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle command-line arguments that are no longer supported after a software integration, providing users with helpful error messages and suggestions for alternatives.\n2. **Input**: It takes a single string representing the name of a command-line argument that is not supported.\n3. **Output**: It returns a command-line argument configuration that, when used, will display error messages and suggestions instead of executing the intended functionality.\n4. **Procedure**: The function configures a command-line argument to use a custom value parser that generates a series of suggestions and error messages. These messages inform the user that the argument is no longer supported and guide them on how to proceed, such as using a different command or filing an issue for support.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/bin/cargo/commands/remove.rs\nuse cargo::core::dependency::DepKind;\nuse cargo::core::PackageIdSpec;\nuse cargo::core::PackageIdSpecQuery;\nuse cargo::core::Resolve;\nuse cargo::core::Workspace;\nuse cargo::ops::cargo_remove::remove;\nuse cargo::ops::cargo_remove::RemoveOptions;\nuse cargo::ops::resolve_ws;\nuse cargo::util::command_prelude::*;\nuse cargo::util::print_available_packages;\nuse cargo::util::toml_mut::dependency::Dependency;\nuse cargo::util::toml_mut::dependency::MaybeWorkspace;\nuse cargo::util::toml_mut::dependency::Source;\nuse cargo::util::toml_mut::manifest::DepTable;\nuse cargo::util::toml_mut::manifest::LocalManifest;\nuse cargo::CargoResult;\n\npub fn cli() -> clap::Command {\n    clap::Command::new(\"remove\")\n        // Subcommand aliases are handled in `aliased_command()`.\n        // .alias(\"rm\")\n        .about(\"Remove dependencies from a Cargo.toml manifest file\")\n        .args([clap::Arg::new(\"dependencies\")\n            .action(clap::ArgAction::Append)\n            .required(true)\n            .num_args(1..)\n            .value_name(\"DEP_ID\")\n            .help(\"Dependencies to be removed\")])\n        .arg_dry_run(\"Don't actually write the manifest\")\n        .arg_silent_suggestion()\n        .next_help_heading(\"Section\")\n        .args([\n            clap::Arg::new(\"dev\")\n                .long(\"dev\")\n                .conflicts_with(\"build\")\n                .action(clap::ArgAction::SetTrue)\n                .group(\"section\")\n                .help(\"Remove from dev-dependencies\"),\n            clap::Arg::new(\"build\")\n                .long(\"build\")\n                .conflicts_with(\"dev\")\n                .action(clap::ArgAction::SetTrue)\n                .group(\"section\")\n                .help(\"Remove from build-dependencies\"),\n            clap::Arg::new(\"target\")\n                .long(\"target\")\n                .num_args(1)\n                .value_name(\"TARGET\")\n                .value_parser(clap::builder::NonEmptyStringValueParser::new())\n                .help(\"Remove from target-dependencies\"),\n        ])\n        .arg_package(\"Package to remove from\")\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help remove</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let dry_run = args.dry_run();\n\n    let workspace = args.workspace(gctx)?;\n\n    if args.is_present_with_zero_values(\"package\") {\n        print_available_packages(&workspace)?;\n    }\n\n    let packages = args.packages_from_flags()?;\n    let packages = packages.get_packages(&workspace)?;\n    let spec = match packages.len() {\n        0 => {\n            return Err(CliError::new(\n                anyhow::format_err!(\n                    \"no packages selected to modify.  Please specify one with `-p <PKGID>`\"\n                ),\n                101,\n            ));\n        }\n        1 => packages[0],\n        _ => {\n            let names = packages.iter().map(|p| p.name()).collect::<Vec<_>>();\n            return Err(CliError::new(\n                anyhow::format_err!(\n                    \"`cargo remove` could not determine which package to modify. \\\n                    Use the `--package` option to specify a package. \\n\\\n                    available packages: {}\",\n                    names.join(\", \")\n                ),\n                101,\n            ));\n        }\n    };\n\n    let dependencies = args\n        .get_many::<String>(\"dependencies\")\n        .expect(\"required(true)\")\n        .cloned()\n        .collect::<Vec<_>>();\n\n    let section = parse_section(args);\n\n    let options = RemoveOptions {\n        gctx,\n        spec,\n        dependencies,\n        section,\n...\n// Path: src/bin/cargo/commands/help.rs\nuse crate::aliased_command;\nuse crate::command_prelude::*;\nuse cargo::drop_println;\nuse cargo::util::errors::CargoResult;\nuse cargo_util::paths::resolve_executable;\nuse flate2::read::GzDecoder;\nuse std::ffi::OsStr;\nuse std::ffi::OsString;\nuse std::io::Read;\nuse std::io::Write;\nuse std::path::Path;\n\nconst COMPRESSED_MAN: &[u8] = include_bytes!(concat!(env!(\"OUT_DIR\"), \"/man.tgz\"));\n\npub fn cli() -> Command {\n    subcommand(\"help\")\n        .about(\"Displays help for a cargo subcommand\")\n        .arg(Arg::new(\"COMMAND\").action(ArgAction::Set))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let subcommand = args.get_one::<String>(\"COMMAND\");\n    if let Some(subcommand) = subcommand {\n        if !try_help(gctx, subcommand)? {\n            match check_builtin(&subcommand) {\n                Some(s) => {\n                    crate::execute_internal_subcommand(\n                        gctx,\n                        &[OsStr::new(s), OsStr::new(\"--help\")],\n                    )?;\n                }\n                None => {\n                    crate::execute_external_subcommand(\n                        gctx,\n                        subcommand,\n                        &[OsStr::new(subcommand), OsStr::new(\"--help\")],\n                    )?;\n                }\n            }\n        }\n    } else {\n        let mut cmd = crate::cli::cli(gctx);\n        let _ = cmd.print_help();\n    }\n    Ok(())\n}\n\nfn try_help(gctx: &GlobalContext, subcommand: &str) -> CargoResult<bool> {\n    let subcommand = match check_alias(gctx, subcommand) {\n        // If this alias is more than a simple subcommand pass-through, show the alias.\n        Some(argv) if argv.len() > 1 => {\n            let alias = argv.join(\" \");\n            drop_println!(gctx, \"`{}` is aliased to `{}`\", subcommand, alias);\n            return Ok(true);\n        }\n        // Otherwise, resolve the alias into its subcommand.\n        Some(argv) => {\n            // An alias with an empty argv can be created via `\"empty-alias\" = \"\"`.\n            let first = argv.get(0).map(String::as_str).unwrap_or(subcommand);\n            first.to_string()\n        }\n        None => subcommand.to_string(),\n    };\n\n    let subcommand = match check_builtin(&subcommand) {\n        Some(s) => s,\n        None => return Ok(false),\n    };\n\n    if resolve_executable(Path::new(\"man\")).is_ok() {\n        let man = match extract_man(subcommand, \"1\") {\n            Some(man) => man,\n            None => return Ok(false),\n        };\n        write_and_spawn(subcommand, &man, \"man\")?;\n    } else {\n        let txt = match extract_man(subcommand, \"txt\") {\n            Some(txt) => txt,\n            None => return Ok(false),\n        };\n        if resolve_executable(Path::new(\"less\")).is_ok() {\n            write_and_spawn(subcommand, &txt, \"less\")?;\n        } else if resolve_executable(Path::new(\"more\")).is_ok() {\n            write_and_spawn(subcommand, &txt, \"more\")?;\n        } else {\n            drop(std::io::stdout().write_all(&txt));\n        }\n    }\n    Ok(true)\n}\n\n/// Checks if the given subcommand is an alias.\n///\n/// Returns None if it is not an alias.\nfn check_alias(gctx: &GlobalContext, subcommand: &str) -> Option<Vec<String>> {\n    aliased_command(gctx, subcommand).ok().flatten()\n}\n\n/// Checks if the given subcommand is a built-in command (not via an alias).\n///\n/// Returns None if it is not a built-in command.\nfn check_builtin(subcommand: &str) -> Option<&str> {\n    super::builtin_exec(subcommand).map(|_| subcommand)\n}\n\n/// Extracts the given man page from the compressed archive.\n///\n/// Returns None if the command wasn't found.\nfn extract_man(subcommand: &str, extension: &str) -> Option<Vec<u8>> {\n    let extract_name = OsString::from(format!(\"cargo-{}.{}\", subcommand, extension));\n    let gz = GzDecoder::new(COMPRESSED_MAN);\n    let mut ar = tar::Archive::new(gz);\n    // Unwraps should be safe here, since this is a static archive generated\n    // by our build script. It should never be an invalid format!\n    for entry in ar.entries().unwrap() {\n        let mut entry = entry.unwrap();\n        let path = entry.path().unwrap();\n        if path.file_name().unwrap() != extract_name {\n            continue;\n        }\n        let mut result = Vec::new();\n        entry.read_to_end(&mut result).unwrap();\n        return Some(result);\n    }\n    None\n}\n\n/// Write the contents of a man page to disk and spawn the given command to\n/// display it.\nfn write_and_spawn(name: &str, contents: &[u8], command: &str) -> CargoResult<()> {\n    let prefix = format!(\"cargo-{}.\", name);\n    let mut tmp = tempfile::Builder::new().prefix(&prefix).tempfile()?;\n    let f = tmp.as_file_mut();\n    f.write_all(contents)?;\n    f.flush()?;\n    let path = tmp.path();\n    // Use a path relative to the temp directory so that it can work on\n    // cygwin/msys systems which don't handle windows-style paths.\n    let mut relative_name = std::ffi::OsString::from(\"./\");\n    relative_name.push(path.file_name().unwrap());\n    let mut cmd = std::process::Command::new(command)\n        .arg(relative_name)\n        .current_dir(path.parent().unwrap())\n        .spawn()?;\n    drop(cmd.wait());\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/git_checkout.rs\nuse crate::command_prelude::*;\n\nconst REMOVED: &str = \"The `git-checkout` command has been removed.\";\n\npub fn cli() -> Command {\n    subcommand(\"git-checkout\")\n        .about(\"This command has been removed\")\n        .hide(true)\n        .override_help(REMOVED)\n}\n\npub fn exec(_gctx: &mut GlobalContext, _args: &ArgMatches) -> CliResult {\n    Err(anyhow::format_err!(REMOVED).into())\n}\n\n// Path: src/bin/cargo/commands/build.rs\nuse crate::command_prelude::*;\n\nuse cargo::ops;\n\npub fn cli() -> Command {\n    subcommand(\"build\")\n        // subcommand aliases are handled in aliased_command()\n        // .alias(\"b\")\n        .about(\"Compile a local package and all of its dependencies\")\n        .arg_ignore_rust_version()\n        .arg_future_incompat_report()\n        .arg_message_format()\n        .arg_silent_suggestion()\n        .arg_package_spec(\n            \"Package to build (see `cargo help pkgid`)\",\n            \"Build all packages in the workspace\",\n            \"Exclude packages from the build\",\n        )\n        .arg_targets_all(\n            \"Build only this package's library\",\n            \"Build only the specified binary\",\n            \"Build all binaries\",\n            \"Build only the specified example\",\n            \"Build all examples\",\n            \"Build only the specified test target\",\n            \"Build all test targets\",\n            \"Build only the specified bench target\",\n            \"Build all bench targets\",\n            \"Build all targets\",\n        )\n        .arg_features()\n        .arg_release(\"Build artifacts in release mode, with optimizations\")\n        .arg_redundant_default_mode(\"debug\", \"build\", \"release\")\n        .arg_profile(\"Build artifacts with the specified profile\")\n        .arg_parallel()\n        .arg_target_triple(\"Build for the target triple\")\n        .arg_target_dir()\n        .arg_out_dir()\n        .arg_build_plan()\n        .arg_unit_graph()\n        .arg_timings()\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help build</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let ws = args.workspace(gctx)?;\n    let mut compile_opts =\n        args.compile_options(gctx, CompileMode::Build, Some(&ws), ProfileChecking::Custom)?;\n\n    if let Some(out_dir) = args.value_of_path(\"out-dir\", gctx) {\n        compile_opts.build_config.export_dir = Some(out_dir);\n    } else if let Some(out_dir) = gctx.build_config()?.out_dir.as_ref() {\n        let out_dir = out_dir.resolve_path(gctx);\n        compile_opts.build_config.export_dir = Some(out_dir);\n    }\n    if compile_opts.build_config.export_dir.is_some() {\n        gctx.cli_unstable().fail_if_stable_opt(\"--out-dir\", 6790)?;\n    }\n    ops::compile(&ws, &compile_opts)?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/tree.rs\nuse crate::cli;\nuse crate::command_prelude::*;\nuse anyhow::{bail, format_err};\nuse cargo::core::dependency::DepKind;\nuse cargo::ops::tree::{self, EdgeKind};\nuse cargo::ops::Packages;\nuse cargo::util::print_available_packages;\nuse cargo::util::CargoResult;\nuse std::collections::HashSet;\nuse std::str::FromStr;\n\npub fn cli() -> Command {\n    subcommand(\"tree\")\n        .about(\"Display a tree visualization of a dependency graph\")\n        .arg(\n            flag(\"all\", \"Deprecated, use --no-dedupe instead\")\n                .short('a')\n                .hide(true),\n        )\n        .arg_silent_suggestion()\n        .arg(flag(\"no-dev-dependencies\", \"Deprecated, use -e=no-dev instead\").hide(true))\n        .arg(\n            multi_opt(\n                \"edges\",\n                \"KINDS\",\n                \"The kinds of dependencies to display \\\n                 (features, normal, build, dev, all, \\\n                 no-normal, no-build, no-dev, no-proc-macro)\",\n            )\n            .short('e'),\n        )\n        .arg(\n            optional_multi_opt(\n                \"invert\",\n                \"SPEC\",\n                \"Invert the tree direction and focus on the given package\",\n            )\n            .short('i'),\n        )\n        .arg(multi_opt(\n            \"prune\",\n            \"SPEC\",\n            \"Prune the given package from the display of the dependency tree\",\n        ))\n        .arg(opt(\"depth\", \"Maximum display depth of the dependency tree\").value_name(\"DEPTH\"))\n        .arg(flag(\"no-indent\", \"Deprecated, use --prefix=none instead\").hide(true))\n        .arg(flag(\"prefix-depth\", \"Deprecated, use --prefix=depth instead\").hide(true))\n        .arg(\n            opt(\n                \"prefix\",\n                \"Change the prefix (indentation) of how each entry is displayed\",\n            )\n            .value_name(\"PREFIX\")\n            .value_parser([\"depth\", \"indent\", \"none\"])\n            .default_value(\"indent\"),\n        )\n        .arg(flag(\n            \"no-dedupe\",\n            \"Do not de-duplicate (repeats all shared dependencies)\",\n        ))\n        .arg(\n            flag(\n                \"duplicates\",\n                \"Show only dependencies which come in multiple versions (implies -i)\",\n            )\n            .short('d')\n            .alias(\"duplicate\"),\n        )\n        .arg(\n            opt(\"charset\", \"Character set to use in output\")\n                .value_name(\"CHARSET\")\n                .value_parser([\"utf8\", \"ascii\"]),\n        )\n        .arg(\n            opt(\"format\", \"Format string used for printing dependencies\")\n                .value_name(\"FORMAT\")\n                .short('f')\n                .default_value(\"{p}\"),\n        )\n        .arg(\n            // Backwards compatibility with old cargo-tree.\n            flag(\"version\", \"Print version info and exit\")\n                .short('V')\n                .hide(true),\n        )\n        .arg_package_spec_no_all(\n            \"Package to be used as the root of the tree\",\n            \"Display the tree for all packages in the workspace\",\n            \"Exclude specific workspace members\",\n        )\n        .arg_features()\n        .arg(flag(\"all-targets\", \"Deprecated, use --target=all instead\").hide(true))\n        .arg_target_triple(\n            \"Filter dependencies matching the given target-triple (default host platform). \\\n            Pass `all` to include all targets.\",\n        )\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help tree</>` for more detailed information.\\n\"\n        ))\n}\n\n#[derive(Copy, Clone)]\npub enum Charset {\n    Utf8,\n    Ascii,\n}\n\nimpl FromStr for Charset {\n    type Err = &'static str;\n\n    fn from_str(s: &str) -> Result<Charset, &'static str> {\n        match s {\n            \"utf8\" => Ok(Charset::Utf8),\n            \"ascii\" => Ok(Charset::Ascii),\n            _ => Err(\"invalid charset\"),\n        }\n    }\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    if args.flag(\"version\") {\n        let verbose = args.verbose() > 0;\n        let version = cli::get_version_string(verbose);\n        cargo::drop_print!(gctx, \"{}\", version);\n        return Ok(());\n    }\n    let prefix = if args.flag(\"no-indent\") {\n        gctx.shell()\n            .warn(\"the --no-indent flag has been changed to --prefix=none\")?;\n        \"none\"\n    } else if args.flag(\"prefix-depth\") {\n        gctx.shell()\n            .warn(\"the --prefix-depth flag has been changed to --prefix=depth\")?;\n        \"depth\"\n    } else {\n        args.get_one::<String>(\"prefix\").unwrap().as_str()\n    };\n    let prefix = tree::Prefix::from_str(prefix).map_err(|e| anyhow::anyhow!(\"{}\", e))?;\n\n    let no_dedupe = args.flag(\"no-dedupe\") || args.flag(\"all\");\n    if args.flag(\"all\") {\n        gctx.shell().warn(\n            \"The `cargo tree` --all flag has been changed to --no-dedupe, \\\n             and may be removed in a future version.\\n\\\n             If you are looking to display all workspace members, use the --workspace flag.\",\n        )?;\n    }\n\n    let targets = if args.flag(\"all-targets\") {\n        gctx.shell()\n            .warn(\"the --all-targets flag has been changed to --target=all\")?;\n        vec![\"all\".to_string()]\n    } else {\n        args.targets()?\n    };\n    let target = tree::Target::from_cli(targets);\n\n    let (edge_kinds, no_proc_macro) = parse_edge_kinds(gctx, args)?;\n    let graph_features = edge_kinds.contains(&EdgeKind::Feature);\n\n    let pkgs_to_prune = args._values_of(\"prune\");\n\n    let packages = args.packages_from_flags()?;\n    let mut invert = args\n        .get_many::<String>(\"invert\")\n        .map_or_else(|| Vec::new(), |is| is.map(|s| s.to_string()).collect());\n    if args.is_present_with_zero_values(\"invert\") {\n        match &packages {\n            Packages::Packages(ps) => {\n                // Backwards compatibility with old syntax of `cargo tree -i -p foo`.\n                invert.extend(ps.clone());\n            }\n            _ => {\n                return Err(format_err!(\n                    \"The `-i` flag requires a package name.\\n\\\n\\n\\\nThe `-i` flag is used to inspect the reverse dependencies of a specific\\n\\\npackage. It will invert the tree and display the packages that depend on the\\n\\\ngiven package.\\n\\\n\\n\\\nNote that in a workspace, by default it will only display the package's\\n\\\nreverse dependencies inside the tree of the workspace member in the current\\n\\\ndirectory. The --workspace flag can be used to extend it so that it will show\\n\\\nthe package's reverse dependencies across the entire workspace. The -p flag\\n\\\ncan be used to display the package's reverse dependencies only with the\\n\\\nsubtree of the package given to -p.\\n\\\n\"\n                )\n                .into());\n            }\n        }\n    }\n\n    let ws = args.workspace(gctx)?;\n\n    if args.is_present_with_zero_values(\"package\") {\n        print_available_packages(&ws)?;\n    }\n\n    let charset = args.get_one::<String>(\"charset\");\n    if let Some(charset) = charset\n        .map(|c| Charset::from_str(c))\n        .transpose()\n        .map_err(|e| anyhow::anyhow!(\"{}\", e))?\n    {\n        match charset {\n            Charset::Utf8 => gctx.shell().set_unicode(true)?,\n            Charset::Ascii => gctx.shell().set_unicode(false)?,\n        }\n    }\n    let opts = tree::TreeOptions {\n        cli_features: args.cli_features()?,\n        packages,\n        target,\n        edge_kinds,\n        invert,\n        pkgs_to_prune,\n        prefix,\n        no_dedupe,\n        duplicates: args.flag(\"duplicates\"),\n        format: args.get_one::<String>(\"format\").cloned().unwrap(),\n        graph_features,\n        max_display_depth: args.value_of_u32(\"depth\")?.unwrap_or(u32::MAX),\n        no_proc_macro,\n    };\n\n    if opts.graph_features && opts.duplicates {\n        return Err(format_err!(\"the `-e features` flag does not support `--duplicates`\").into());\n    }\n\n    tree::build_and_print(&ws, &opts)?;\n    Ok(())\n}\n\n/// Parses `--edges` option.\n///\n/// Returns a tuple of `EdgeKind` map and `no_proc_marco` flag.\nfn parse_edge_kinds(\n    gctx: &GlobalContext,\n    args: &ArgMatches,\n) -> CargoResult<(HashSet<EdgeKind>, bool)> {\n    let (kinds, no_proc_macro) = {\n        let mut no_proc_macro = false;\n        let mut kinds = args.get_many::<String>(\"edges\").map_or_else(\n            || Vec::new(),\n            |es| {\n                es.flat_map(|e| e.split(','))\n                    .filter(|e| {\n                        no_proc_macro = *e == \"no-proc-macro\";\n                        !no_proc_macro\n                    })\n                    .collect()\n            },\n        );\n\n        if args.flag(\"no-dev-dependencies\") {\n            gctx.shell()\n                .warn(\"the --no-dev-dependencies flag has changed to -e=no-dev\")?;\n            kinds.push(\"no-dev\");\n        }\n\n        if kinds.is_empty() {\n            kinds.extend(&[\"normal\", \"build\", \"dev\"]);\n        }\n\n        (kinds, no_proc_macro)\n    };\n\n    let mut result = HashSet::new();\n    let insert_defaults = |result: &mut HashSet<EdgeKind>| {\n        result.insert(EdgeKind::Dep(DepKind::Normal));\n        result.insert(EdgeKind::Dep(DepKind::Build));\n        result.insert(EdgeKind::Dep(DepKind::Development));\n    };\n    let unknown = |k| {\n        bail!(\n            \"unknown edge kind `{}`, valid values are \\\n                \\\"normal\\\", \\\"build\\\", \\\"dev\\\", \\\n                \\\"no-normal\\\", \\\"no-build\\\", \\\"no-dev\\\", \\\"no-proc-macro\\\", \\\n                \\\"features\\\", or \\\"all\\\"\",\n            k\n        )\n    };\n    if kinds.iter().any(|k| k.starts_with(\"no-\")) {\n        insert_defaults(&mut result);\n        for kind in &kinds {\n            match *kind {\n                \"no-normal\" => result.remove(&EdgeKind::Dep(DepKind::Normal)),\n                \"no-build\" => result.remove(&EdgeKind::Dep(DepKind::Build)),\n                \"no-dev\" => result.remove(&EdgeKind::Dep(DepKind::Development)),\n                \"features\" => result.insert(EdgeKind::Feature),\n                \"normal\" | \"build\" | \"dev\" | \"all\" => {\n                    bail!(\n                        \"`{}` dependency kind cannot be mixed with \\\n                            \\\"no-normal\\\", \\\"no-build\\\", or \\\"no-dev\\\" \\\n                            dependency kinds\",\n                        kind\n                    )\n                }\n                k => return unknown(k),\n            };\n        }\n        return Ok((result, no_proc_macro));\n    }\n    for kind in &kinds {\n        match *kind {\n            \"all\" => {\n                insert_defaults(&mut result);\n                result.insert(EdgeKind::Feature);\n            }\n            \"features\" => {\n                result.insert(EdgeKind::Feature);\n            }\n            \"normal\" => {\n                result.insert(EdgeKind::Dep(DepKind::Normal));\n            }\n            \"build\" => {\n                result.insert(EdgeKind::Dep(DepKind::Build));\n            }\n            \"dev\" => {\n                result.insert(EdgeKind::Dep(DepKind::Development));\n            }\n            k => return unknown(k),\n        }\n    }\n    if kinds.len() == 1 && kinds[0] == \"features\" {\n        insert_defaults(&mut result);\n    }\n    Ok((result, no_proc_macro))\n}\n\n// Path: src/bin/cargo/commands/publish.rs\nuse crate::command_prelude::*;\n\nuse cargo::ops::{self, PublishOpts};\n\npub fn cli() -> Command {\n    subcommand(\"publish\")\n        .about(\"Upload a package to the registry\")\n        .arg_dry_run(\"Perform all checks without uploading\")\n        .arg_index(\"Registry index URL to upload the package to\")\n        .arg_registry(\"Registry to upload the package to\")\n        .arg(opt(\"token\", \"Token to use when uploading\").value_name(\"TOKEN\"))\n        .arg(flag(\n            \"no-verify\",\n            \"Don't verify the contents by building them\",\n        ))\n        .arg(flag(\n            \"allow-dirty\",\n            \"Allow dirty working directories to be packaged\",\n        ))\n        .arg_silent_suggestion()\n        .arg_package(\"Package to publish\")\n        .arg_features()\n        .arg_parallel()\n        .arg_target_triple(\"Build for the target triple\")\n        .arg_target_dir()\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help publish</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let reg_or_index = args.registry_or_index(gctx)?;\n    let ws = args.workspace(gctx)?;\n    if ws.root_maybe().is_embedded() {\n        return Err(anyhow::format_err!(\n            \"{} is unsupported by `cargo publish`\",\n            ws.root_manifest().display()\n        )\n        .into());\n    }\n\n    ops::publish(\n        &ws,\n        &PublishOpts {\n            gctx,\n            token: args\n                .get_one::<String>(\"token\")\n                .map(|s| s.to_string().into()),\n            reg_or_index,\n            verify: !args.flag(\"no-verify\"),\n            allow_dirty: args.flag(\"allow-dirty\"),\n            to_publish: args.packages_from_flags()?,\n            targets: args.targets()?,\n            jobs: args.jobs()?,\n            keep_going: args.keep_going(),\n            dry_run: args.dry_run(),\n            cli_features: args.cli_features()?,\n        },\n    )?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/package.rs\nuse crate::command_prelude::*;\n\nuse cargo::ops::{self, PackageOpts};\n\npub fn cli() -> Command {\n    subcommand(\"package\")\n        .about(\"Assemble the local package into a distributable tarball\")\n        .arg(\n            flag(\n                \"list\",\n                \"Print files included in a package without making one\",\n            )\n            .short('l'),\n        )\n        .arg(flag(\n            \"no-verify\",\n            \"Don't verify the contents by building them\",\n        ))\n        .arg(flag(\n            \"no-metadata\",\n            \"Ignore warnings about a lack of human-usable metadata\",\n        ))\n        .arg(flag(\n            \"allow-dirty\",\n            \"Allow dirty working directories to be packaged\",\n        ))\n        .arg_silent_suggestion()\n        .arg_package_spec_no_all(\n            \"Package(s) to assemble\",\n            \"Assemble all packages in the workspace\",\n            \"Don't assemble specified packages\",\n        )\n        .arg_features()\n        .arg_target_triple(\"Build for the target triple\")\n        .arg_target_dir()\n        .arg_parallel()\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help package</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let ws = args.workspace(gctx)?;\n    if ws.root_maybe().is_embedded() {\n        return Err(anyhow::format_err!(\n            \"{} is unsupported by `cargo package`\",\n            ws.root_manifest().display()\n        )\n        .into());\n    }\n    let specs = args.packages_from_flags()?;\n\n    ops::package(\n        &ws,\n        &PackageOpts {\n            gctx,\n            verify: !args.flag(\"no-verify\"),\n            list: args.flag(\"list\"),\n            check_metadata: !args.flag(\"no-metadata\"),\n            allow_dirty: args.flag(\"allow-dirty\"),\n            to_package: specs,\n            targets: args.targets()?,\n            jobs: args.jobs()?,\n            keep_going: args.keep_going(),\n            cli_features: args.cli_features()?,\n        },\n    )?;\n\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/report.rs\nuse crate::command_prelude::*;\nuse cargo::core::compiler::future_incompat::{OnDiskReports, REPORT_PREAMBLE};\nuse cargo::drop_println;\n\npub fn cli() -> Command {\n    subcommand(\"report\")\n        .about(\"Generate and display various kinds of reports\")\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help report</>` for more detailed information.\\n\"\n        ))\n        .subcommand_required(true)\n        .arg_required_else_help(true)\n        .subcommand(\n            subcommand(\"future-incompatibilities\")\n                .alias(\"future-incompat\")\n                .about(\"Reports any crates which will eventually stop compiling\")\n                .arg(\n                    opt(\n                        \"id\",\n                        \"identifier of the report generated by a Cargo command invocation\",\n                    )\n                    .value_name(\"id\"),\n                )\n                .arg_package(\"Package to display a report for\"),\n        )\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    match args.subcommand() {\n        Some((\"future-incompatibilities\", args)) => report_future_incompatibilities(gctx, args),\n        Some((cmd, _)) => {\n            unreachable!(\"unexpected command {}\", cmd)\n        }\n        None => {\n            unreachable!(\"unexpected command\")\n        }\n    }\n}\n\nfn report_future_incompatibilities(gctx: &GlobalContext, args: &ArgMatches) -> CliResult {\n    let ws = args.workspace(gctx)?;\n    let reports = OnDiskReports::load(&ws)?;\n    let id = args\n        .value_of_u32(\"id\")?\n        .unwrap_or_else(|| reports.last_id());\n    let krate = args.get_one::<String>(\"package\").map(String::as_str);\n    let report = reports.get_report(id, krate)?;\n    drop_println!(gctx, \"{}\", REPORT_PREAMBLE);\n    drop(gctx.shell().print_ansi_stdout(report.as_bytes()));\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/check.rs\nuse crate::command_prelude::*;\n\nuse cargo::ops;\n\npub fn cli() -> Command {\n    subcommand(\"check\")\n        // subcommand aliases are handled in aliased_command()\n        // .alias(\"c\")\n        .about(\"Check a local package and all of its dependencies for errors\")\n        .arg_ignore_rust_version()\n        .arg_future_incompat_report()\n        .arg_message_format()\n        .arg_silent_suggestion()\n        .arg_package_spec(\n            \"Package(s) to check\",\n            \"Check all packages in the workspace\",\n            \"Exclude packages from the check\",\n        )\n        .arg_targets_all(\n            \"Check only this package's library\",\n            \"Check only the specified binary\",\n            \"Check all binaries\",\n            \"Check only the specified example\",\n            \"Check all examples\",\n            \"Check only the specified test target\",\n            \"Check all test targets\",\n            \"Check only the specified bench target\",\n            \"Check all bench targets\",\n            \"Check all targets\",\n        )\n        .arg_features()\n        .arg_parallel()\n        .arg_release(\"Check artifacts in release mode, with optimizations\")\n        .arg_profile(\"Check artifacts with the specified profile\")\n        .arg_target_triple(\"Check for the target triple\")\n        .arg_target_dir()\n        .arg_unit_graph()\n        .arg_timings()\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help check</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let ws = args.workspace(gctx)?;\n    // This is a legacy behavior that causes `cargo check` to pass `--test`.\n    let test = matches!(\n        args.get_one::<String>(\"profile\").map(String::as_str),\n        Some(\"test\")\n    );\n    let mode = CompileMode::Check { test };\n    let compile_opts =\n        args.compile_options(gctx, mode, Some(&ws), ProfileChecking::LegacyTestOnly)?;\n\n    ops::compile(&ws, &compile_opts)?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/owner.rs\nuse crate::command_prelude::*;\n\nuse cargo::ops::{self, OwnersOptions};\nuse cargo_credential::Secret;\n\npub fn cli() -> Command {\n    subcommand(\"owner\")\n        .about(\"Manage the owners of a crate on the registry\")\n        .arg(Arg::new(\"crate\").value_name(\"CRATE\").action(ArgAction::Set))\n        .arg(\n            multi_opt(\n                \"add\",\n                \"LOGIN\",\n                \"Name of a user or team to invite as an owner\",\n            )\n            .short('a'),\n        )\n        .arg(\n            multi_opt(\n                \"remove\",\n                \"LOGIN\",\n                \"Name of a user or team to remove as an owner\",\n            )\n            .short('r'),\n        )\n        .arg(flag(\"list\", \"List owners of a crate\").short('l'))\n        .arg_index(\"Registry index URL to modify owners for\")\n        .arg_registry(\"Registry to modify owners for\")\n        .arg(opt(\"token\", \"API token to use when authenticating\").value_name(\"TOKEN\"))\n        .arg_silent_suggestion()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help owner</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let opts = OwnersOptions {\n        krate: args.get_one::<String>(\"crate\").cloned(),\n        token: args.get_one::<String>(\"token\").cloned().map(Secret::from),\n        reg_or_index: args.registry_or_index(gctx)?,\n        to_add: args\n            .get_many::<String>(\"add\")\n            .map(|xs| xs.cloned().collect()),\n        to_remove: args\n            .get_many::<String>(\"remove\")\n            .map(|xs| xs.cloned().collect()),\n        list: args.flag(\"list\"),\n    };\n    ops::modify_owners(gctx, &opts)?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/verify_project.rs\nuse crate::command_prelude::*;\n\nuse std::collections::HashMap;\nuse std::process;\n\npub fn cli() -> Command {\n    subcommand(\"verify-project\")\n        .about(\"Check correctness of crate manifest\")\n        .arg_silent_suggestion()\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help verify-project</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    if let Err(e) = args.workspace(gctx) {\n        gctx.shell()\n            .print_json(&HashMap::from([(\"invalid\", e.to_string())]))?;\n        process::exit(1)\n    }\n\n    gctx.shell()\n        .print_json(&HashMap::from([(\"success\", \"true\")]))?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/test.rs\nuse crate::command_prelude::*;\nuse cargo::ops;\n\npub fn cli() -> Command {\n    subcommand(\"test\")\n        // Subcommand aliases are handled in `aliased_command()`.\n        // .alias(\"t\")\n        .about(\"Execute all unit and integration tests and build examples of a local package\")\n        .arg(\n            Arg::new(\"TESTNAME\")\n                .action(ArgAction::Set)\n                .help(\"If specified, only run tests containing this string in their names\"),\n        )\n        .arg(\n            Arg::new(\"args\")\n                .value_name(\"ARGS\")\n                .help(\"Arguments for the test binary\")\n                .num_args(0..)\n                .last(true),\n        )\n        .arg(flag(\"doc\", \"Test only this library's documentation\"))\n        .arg(flag(\"no-run\", \"Compile, but don't run tests\"))\n        .arg(flag(\"no-fail-fast\", \"Run all tests regardless of failure\"))\n        .arg_ignore_rust_version()\n        .arg_future_incompat_report()\n        .arg_message_format()\n        .arg(\n            flag(\n                \"quiet\",\n                \"Display one character per test instead of one line\",\n            )\n            .short('q'),\n        )\n        .arg_package_spec(\n            \"Package to run tests for\",\n            \"Test all packages in the workspace\",\n            \"Exclude packages from the test\",\n        )\n        .arg_targets_all(\n            \"Test only this package's library\",\n            \"Test only the specified binary\",\n            \"Test all binaries\",\n            \"Test only the specified example\",\n            \"Test all examples\",\n            \"Test only the specified test target\",\n            \"Test all test targets\",\n            \"Test only the specified bench target\",\n            \"Test all bench targets\",\n            \"Test all targets (does not include doctests)\",\n        )\n        .arg_features()\n        .arg_jobs()\n        .arg_unsupported_keep_going()\n        .arg_release(\"Build artifacts in release mode, with optimizations\")\n        .arg_profile(\"Build artifacts with the specified profile\")\n        .arg_target_triple(\"Build for the target triple\")\n        .arg_target_dir()\n        .arg_unit_graph()\n        .arg_timings()\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help test</>` for more detailed information.\\n\\\n             Run `<cyan,bold>cargo test -- --help</>` for test binary options.\\n\",\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let ws = args.workspace(gctx)?;\n\n    let mut compile_opts =\n        args.compile_options(gctx, CompileMode::Test, Some(&ws), ProfileChecking::Custom)?;\n\n    compile_opts.build_config.requested_profile =\n        args.get_profile_name(gctx, \"test\", ProfileChecking::Custom)?;\n\n    // `TESTNAME` is actually an argument of the test binary, but it's\n    // important, so we explicitly mention it and reconfigure.\n    let test_name = args.get_one::<String>(\"TESTNAME\");\n    let test_args = args.get_one::<String>(\"TESTNAME\").into_iter();\n    let test_args = test_args.chain(args.get_many::<String>(\"args\").unwrap_or_default());\n    let test_args = test_args.map(String::as_str).collect::<Vec<_>>();\n\n    let no_run = args.flag(\"no-run\");\n    let doc = args.flag(\"doc\");\n    if doc {\n        if compile_opts.filter.is_specific() {\n            return Err(\n                anyhow::format_err!(\"Can't mix --doc with other target selecting options\").into(),\n            );\n        }\n        if no_run {\n            return Err(anyhow::format_err!(\"Can't skip running doc tests with --no-run\").into());\n        }\n        compile_opts.build_config.mode = CompileMode::Doctest;\n        compile_opts.filter = ops::CompileFilter::lib_only();\n    } else if test_name.is_some() && !compile_opts.filter.is_specific() {\n        // If arg `TESTNAME` is provided, assumed that the user knows what\n        // exactly they wants to test, so we use `all_test_targets` to\n        // avoid compiling unnecessary targets such as examples, which are\n        // included by the logic of default target filter.\n        compile_opts.filter = ops::CompileFilter::all_test_targets();\n    }\n\n    let ops = ops::TestOptions {\n        no_run,\n        no_fail_fast: args.flag(\"no-fail-fast\"),\n        compile_opts,\n    };\n\n    ops::run_tests(&ws, &ops, &test_args)\n}\n\n// Path: src/bin/cargo/commands/init.rs\nuse crate::command_prelude::*;\n\nuse cargo::ops;\n\npub fn cli() -> Command {\n    subcommand(\"init\")\n        .about(\"Create a new cargo package in an existing directory\")\n        .arg(\n            Arg::new(\"path\")\n                .value_name(\"PATH\")\n                .action(ArgAction::Set)\n                .default_value(\".\"),\n        )\n        .arg_new_opts()\n        .arg_registry(\"Registry to use\")\n        .arg_silent_suggestion()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help init</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let opts = args.new_options(gctx)?;\n    ops::init(&opts, gctx)?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/vendor.rs\nuse crate::command_prelude::*;\nuse cargo::ops;\nuse std::path::PathBuf;\n\npub fn cli() -> Command {\n    subcommand(\"vendor\")\n        .about(\"Vendor all dependencies for a project locally\")\n        .arg(\n            Arg::new(\"path\")\n                .action(ArgAction::Set)\n                .value_parser(clap::value_parser!(PathBuf))\n                .help(\"Where to vendor crates (`vendor` by default)\"),\n        )\n        .arg(flag(\n            \"no-delete\",\n            \"Don't delete older crates in the vendor directory\",\n        ))\n        .arg(\n            Arg::new(\"tomls\")\n                .short('s')\n                .long(\"sync\")\n                .help(\"Additional `Cargo.toml` to sync and vendor\")\n                .value_name(\"TOML\")\n                .value_parser(clap::value_parser!(PathBuf))\n                .action(clap::ArgAction::Append),\n        )\n        .arg(flag(\n            \"respect-source-config\",\n            \"Respect `[source]` config in `.cargo/config`\",\n        ))\n        .arg(flag(\n            \"versioned-dirs\",\n            \"Always include version in subdir name\",\n        ))\n        .arg(unsupported(\"no-merge-sources\"))\n        .arg(unsupported(\"relative-path\"))\n        .arg(unsupported(\"only-git-deps\"))\n        .arg(unsupported(\"disallow-duplicates\"))\n        .arg_manifest_path()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help vendor</>` for more detailed information.\\n\"\n        ))\n}\n\n\nfn unsupported(name: &'static str) -> Arg {\n    // When we moved `cargo vendor` into Cargo itself we didn't stabilize a few\n    // flags, so try to provide a helpful error message in that case to ensure\n    // that users currently using the flag aren't tripped up.\n    let value_parser = clap::builder::UnknownArgumentValueParser::suggest(\"the crates.io `cargo vendor` command has been merged into Cargo\")\n        .and_suggest(format!(\"and the flag `--{name}` isn't supported currently\"))\n        .and_suggest(\"to continue using the flag, execute `cargo-vendor vendor ...`\")\n        .and_suggest(\"to suggest this flag supported in Cargo, file an issue at <https://github.com/rust-lang/cargo/issues/new>\");\n\n    flag(name, \"\").value_parser(value_parser).hide(true)\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    // We're doing the vendoring operation ourselves, so we don't actually want\n    // to respect any of the `source` configuration in Cargo itself. That's\n    // intended for other consumers of Cargo, but we want to go straight to the\n    // source, e.g. crates.io, to fetch crates.\n    if !args.flag(\"respect-source-config\") {\n        gctx.values_mut()?.remove(\"source\");\n    }\n\n    let ws = args.workspace(gctx)?;\n    let path = args\n        .get_one::<PathBuf>(\"path\")\n        .cloned()\n        .unwrap_or_else(|| PathBuf::from(\"vendor\"));\n    ops::vendor(\n        &ws,\n        &ops::VendorOptions {\n            no_delete: args.flag(\"no-delete\"),\n            destination: &path,\n            versioned_dirs: args.flag(\"versioned-dirs\"),\n            extra: args\n                .get_many::<PathBuf>(\"tomls\")\n                .unwrap_or_default()\n                .cloned()\n                .collect(),\n        },\n    )?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/clean.rs\nuse crate::command_prelude::*;\nuse crate::util::cache_lock::CacheLockMode;\nuse cargo::core::gc::Gc;\nuse cargo::core::gc::{parse_human_size, parse_time_span, GcOpts};\nuse cargo::core::global_cache_tracker::GlobalCacheTracker;\nuse cargo::ops::CleanContext;\nuse cargo::ops::{self, CleanOptions};\nuse cargo::util::print_available_packages;\nuse std::time::Duration;\n\npub fn cli() -> Command {\n    subcommand(\"clean\")\n        .about(\"Remove artifacts that cargo has generated in the past\")\n        .arg_doc(\"Whether or not to clean just the documentation directory\")\n        .arg_silent_suggestion()\n        .arg_package_spec_simple(\"Package to clean artifacts for\")\n        .arg_release(\"Whether or not to clean release artifacts\")\n        .arg_profile(\"Clean artifacts of the specified profile\")\n        .arg_target_triple(\"Target triple to clean output for\")\n        .arg_target_dir()\n        .arg_manifest_path()\n        .arg_dry_run(\"Display what would be deleted without deleting anything\")\n        .args_conflicts_with_subcommands(true)\n        .subcommand(\n            subcommand(\"gc\")\n                .about(\"Clean global caches\")\n                .hide(true)\n                .arg_silent_suggestion()\n                .arg_dry_run(\"Display what would be deleted without deleting anything\")\n                // NOTE: Not all of these options may get stabilized. Some of them are\n                // very low-level details, and may not be something typical users need.\n                .arg(\n                    opt(\n                        \"max-src-age\",\n                        \"Deletes source cache files that have not been used \\\n                        since the given age (unstable)\",\n                    )\n                    .value_name(\"DURATION\")\n                    .value_parser(parse_time_span),\n                )\n                .arg(\n                    opt(\n                        \"max-crate-age\",\n                        \"Deletes crate cache files that have not been used \\\n                        since the given age (unstable)\",\n                    )\n                    .value_name(\"DURATION\")\n                    .value_parser(parse_time_span),\n                )\n                .arg(\n                    opt(\n                        \"max-index-age\",\n                        \"Deletes registry indexes that have not been used \\\n                        since the given age (unstable)\",\n                    )\n                    .value_name(\"DURATION\")\n                    .value_parser(parse_time_span),\n                )\n                .arg(\n                    opt(\n                        \"max-git-co-age\",\n                        \"Deletes git dependency checkouts that have not been used \\\n                        since the given age (unstable)\",\n                    )\n                    .value_name(\"DURATION\")\n                    .value_parser(parse_time_span),\n                )\n                .arg(\n                    opt(\n                        \"max-git-db-age\",\n                        \"Deletes git dependency clones that have not been used \\\n                        since the given age (unstable)\",\n                    )\n                    .value_name(\"DURATION\")\n                    .value_parser(parse_time_span),\n                )\n                .arg(\n                    opt(\n                        \"max-download-age\",\n                        \"Deletes any downloaded cache data that has not been used \\\n                        since the given age (unstable)\",\n                    )\n                    .value_name(\"DURATION\")\n                    .value_parser(parse_time_span),\n                )\n                .arg(\n                    opt(\n                        \"max-src-size\",\n                        \"Deletes source cache files until the cache is under the \\\n                        given size (unstable)\",\n                    )\n                    .value_name(\"SIZE\")\n                    .value_parser(parse_human_size),\n                )\n                .arg(\n                    opt(\n                        \"max-crate-size\",\n                        \"Deletes crate cache files until the cache is under the \\\n                        given size (unstable)\",\n                    )\n                    .value_name(\"SIZE\")\n                    .value_parser(parse_human_size),\n                )\n                .arg(\n                    opt(\n                        \"max-git-size\",\n                        \"Deletes git dependency caches until the cache is under \\\n                        the given size (unstable)\",\n                    )\n                    .value_name(\"SIZE\")\n                    .value_parser(parse_human_size),\n                )\n                .arg(\n                    opt(\n                        \"max-download-size\",\n                        \"Deletes downloaded cache data until the cache is under \\\n                        the given size (unstable)\",\n                    )\n                    .value_name(\"SIZE\")\n                    .value_parser(parse_human_size),\n                ),\n        )\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help clean</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    match args.subcommand() {\n        Some((\"gc\", args)) => {\n            return gc(gctx, args);\n        }\n        Some((cmd, _)) => {\n            unreachable!(\"unexpected command {}\", cmd)\n        }\n        None => {}\n    }\n\n    let ws = args.workspace(gctx)?;\n\n    if args.is_present_with_zero_values(\"package\") {\n        print_available_packages(&ws)?;\n    }\n\n    let opts = CleanOptions {\n        gctx,\n        spec: values(args, \"package\"),\n        targets: args.targets()?,\n        requested_profile: args.get_profile_name(gctx, \"dev\", ProfileChecking::Custom)?,\n        profile_specified: args.contains_id(\"profile\") || args.flag(\"release\"),\n        doc: args.flag(\"doc\"),\n        dry_run: args.dry_run(),\n    };\n    ops::clean(&ws, &opts)?;\n    Ok(())\n}\n\nfn gc(gctx: &GlobalContext, args: &ArgMatches) -> CliResult {\n    gctx.cli_unstable().fail_if_stable_command(\n        gctx,\n        \"clean gc\",\n        12633,\n        \"gc\",\n        gctx.cli_unstable().gc,\n    )?;\n\n    let size_opt = |opt| -> Option<u64> { args.get_one::<u64>(opt).copied() };\n    let duration_opt = |opt| -> Option<Duration> { args.get_one::<Duration>(opt).copied() };\n    let mut gc_opts = GcOpts {\n        max_src_age: duration_opt(\"max-src-age\"),\n        max_crate_age: duration_opt(\"max-crate-age\"),\n        max_index_age: duration_opt(\"max-index-age\"),\n        max_git_co_age: duration_opt(\"max-git-co-age\"),\n        max_git_db_age: duration_opt(\"max-git-db-age\"),\n        max_src_size: size_opt(\"max-src-size\"),\n        max_crate_size: size_opt(\"max-crate-size\"),\n        max_git_size: size_opt(\"max-git-size\"),\n        max_download_size: size_opt(\"max-download-size\"),\n    };\n    if let Some(age) = duration_opt(\"max-download-age\") {\n        gc_opts.set_max_download_age(age);\n    }\n    // If the user sets any options, then only perform the options requested.\n    // If no options are set, do the default behavior.\n    if !gc_opts.is_download_cache_opt_set() {\n        gc_opts.update_for_auto_gc(gctx)?;\n    }\n\n    let _lock = gctx.acquire_package_cache_lock(CacheLockMode::MutateExclusive)?;\n    let mut cache_track = GlobalCacheTracker::new(&gctx)?;\n    let mut gc = Gc::new(gctx, &mut cache_track)?;\n    let mut clean_ctx = CleanContext::new(gctx);\n    clean_ctx.dry_run = args.dry_run();\n    gc.gc(&mut clean_ctx, &gc_opts)?;\n    clean_ctx.display_summary()?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/logout.rs\nuse cargo::ops;\nuse cargo::ops::RegistryOrIndex;\n\nuse crate::command_prelude::*;\n\npub fn cli() -> Command {\n    subcommand(\"logout\")\n        .about(\"Remove an API token from the registry locally\")\n        .arg_registry(\"Registry to use\")\n        .arg_silent_suggestion()\n        .after_help(color_print::cstr!(\n            \"Run `<cyan,bold>cargo help logout</>` for more detailed information.\\n\"\n        ))\n}\n\npub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {\n    let reg = args.registry_or_index(gctx)?;\n    assert!(\n        !matches!(reg, Some(RegistryOrIndex::Index(..))),\n        \"must not be index URL\"\n    );\n\n    ops::registry_logout(gctx, reg)?;\n    Ok(())\n}\n\n// Path: src/bin/cargo/commands/mod.rs\nuse crate::command_prelude::*;\n\npub fn builtin() -> Vec<Command> {\n    vec![\n        add::cli(),\n        bench::cli(),\n        build::cli(),\n        check::cli(),\n        clean::cli(),\n        config::cli(),\n        doc::cli(),\n        fetch::cli(),\n        fix::cli(),\n        generate_lockfile::cli(),\n        git_checkout::cli(),\n        help::cli(),\n        init::cli(),\n        install::cli(),\n        locate_project::cli(),\n        login::cli(),\n        logout::cli(),\n        metadata::cli(),\n        new::cli(),\n        owner::cli(),\n        package::cli(),\n        pkgid::cli(),\n        publish::cli(),\n        read_manifest::cli(),\n        remove::cli(),\n        report::cli(),\n        run::cli(),\n        rustc::cli(),\n        rustdoc::cli(),\n        search::cli(),\n        test::cli(),\n        tree::cli(),\n        uninstall::cli(),\n        update::cli(),\n        vendor::cli(),\n        verify_project::cli(),\n        version::cli(),\n        yank::cli(),\n    ]\n}\n\npub type Exec = fn(&mut GlobalContext, &ArgMatches) -> CliResult;\n\npub fn builtin_exec(cmd: &str) -> Option<Exec> {\n    let f = match cmd {\n        \"add\" => add::exec,\n        \"bench\" => bench::exec,\n        \"build\" => build::exec,\n        \"check\" => check::exec,\n        \"clean\" => clean::exec,\n        \"config\" => config::exec,\n        \"doc\" => doc::exec,\n        \"fetch\" => fetch::exec,\n        \"fix\" => fix::exec,\n        \"generate-lockfile\" => generate_lockfile::exec,\n        \"git-checkout\" => git_checkout::exec,\n        \"help\" => help::exec,\n        \"init\" => init::exec,\n        \"install\" => install::exec,\n        \"locate-project\" => locate_project::exec,\n        \"login\" => login::exec,\n        \"logout\" => logout::exec,\n        \"metadata\" => metadata::exec,\n        \"new\" => new::exec,\n        \"owner\" => owner::exec,\n        \"package\" => package::exec,\n        \"pkgid\" => pkgid::exec,\n        \"publish\" => publish::exec,\n        \"read-manifest\" => read_manifest::exec,\n        \"remove\" => remove::exec,\n        \"report\" => report::exec,\n        \"run\" => run::exec,\n        \"rustc\" => rustc::exec,\n        \"rustdoc\" => rustdoc::exec,\n        \"search\" => search::exec,\n        \"test\" => test::exec,\n        \"tree\" => tree::exec,\n        \"uninstall\" => uninstall::exec,\n        \"update\" => update::exec,\n        \"vendor\" => vendor::exec,\n        \"verify-project\" => verify_project::exec,\n        \"version\" => version::exec,\n        \"yank\" => yank::exec,\n        _ => return None,\n    };\n    Some(f)\n}\n\npub mod add;\npub mod bench;\npub mod build;\npub mod check;\npub mod clean;\npub mod config;\npub mod doc;\npub mod fetch;\npub mod fix;\npub mod generate_lockfile;\npub mod git_checkout;\npub mod help;\npub mod init;\npub mod install;\npub mod locate_project;\npub mod login;\npub mod logout;\npub mod metadata;\npub mod new;\npub mod owner;\npub mod package;\npub mod pkgid;\npub mod publish;\npub mod read_manifest;\npub mod remove;\npub mod report;\npub mod run;\npub mod rustc;\npub mod rustdoc;\npub mod search;\npub mod test;\npub mod tree;\npub mod uninstall;\npub mod update;\npub mod vendor;\npub mod verify_project;\npub mod version;\npub mod yank;\n\n// Path: src/cargo/ops/common_for_install_and_uninstall.rs\nuse std::collections::{btree_map, BTreeMap, BTreeSet};\nuse std::env;\nuse std::io::prelude::*;\nuse std::io::SeekFrom;\nuse std::path::{Path, PathBuf};\nuse std::rc::Rc;\nuse std::task::Poll;\n\nuse anyhow::{bail, format_err, Context as _};\nuse cargo_util::paths;\nuse cargo_util_schemas::core::PartialVersion;\nuse ops::FilterRule;\nuse serde::{Deserialize, Serialize};\n\nuse crate::core::compiler::{DirtyReason, Freshness};\nuse crate::core::Target;\nuse crate::core::{Dependency, FeatureValue, Package, PackageId, SourceId};\nuse crate::ops::{self, CompileFilter, CompileOptions};\nuse crate::sources::source::QueryKind;\n...\n", "needle_token_start": 12081, "needle_token_end": 12304, "code_context_ntokens": 16257, "output": ["```rust\nfn unsupported(name: &'static str) -> Arg {\n    // When we moved `cargo vendor` into Cargo itself we didn't stabilize a few\n    // flags, so try to provide a helpful error message in that case to ensure\n    // that users currently using the flag aren't tripped up.\n    let value_parser = clap::builder::UnknownArgumentValueParser::suggest(\"the crates.io `cargo vendor` command has been merged into Cargo\")\n        .and_suggest(format!(\"and the flag `--{name}` isn't supported currently\"))\n        .and_suggest(\"to continue using the flag, execute `cargo-vendor vendor ...`\")\n        .and_suggest(\"to suggest this flag supported in Cargo, file an issue at <https://github.com/rust-lang/cargo/issues/new>\");\n\n    flag(name, \"\").value_parser(value_parser).hide(true)\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "raw_canonicalized_url", "language": "rust", "path": "src/cargo/util/canonical_url.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve a URL that has been standardized for internal comparison and hashing purposes, ensuring consistency in URL handling within a specific system.\n2. **Input**: There are no parameters required as the function operates on an instance of the object containing the standardized URL.\n3. **Output**: Returns a reference to a URL object that has been modified to a canonical form.\n4. **Procedure**: Simply accesses and returns the URL stored within the object, which has previously been processed to meet certain canonicalization criteria (like lowercasing and removing specific extensions) for uniformity.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/cargo/util/into_url.rs\nuse std::path::{Path, PathBuf};\n\nuse url::Url;\n\nuse crate::util::CargoResult;\n\n/// A type that can be converted to a Url\npub trait IntoUrl {\n    /// Performs the conversion\n    fn into_url(self) -> CargoResult<Url>;\n}\n\nimpl<'a> IntoUrl for &'a str {\n    fn into_url(self) -> CargoResult<Url> {\n        Url::parse(self).map_err(|s| anyhow::format_err!(\"invalid url `{}`: {}\", self, s))\n    }\n}\n\nimpl<'a> IntoUrl for &'a Path {\n    fn into_url(self) -> CargoResult<Url> {\n        Url::from_file_path(self)\n            .map_err(|()| anyhow::format_err!(\"invalid path url `{}`\", self.display()))\n    }\n}\n\nimpl<'a> IntoUrl for &'a PathBuf {\n    fn into_url(self) -> CargoResult<Url> {\n        self.as_path().into_url()\n    }\n}\n\n// Path: src/cargo/util/into_url_with_base.rs\nuse crate::util::{CargoResult, IntoUrl};\n\nuse url::Url;\n\n/// A type that can be interpreted as a relative Url and converted to\n/// a Url.\npub trait IntoUrlWithBase {\n    /// Performs the conversion\n    fn into_url_with_base<U: IntoUrl>(self, base: Option<U>) -> CargoResult<Url>;\n}\n\nimpl<'a> IntoUrlWithBase for &'a str {\n    fn into_url_with_base<U: IntoUrl>(self, base: Option<U>) -> CargoResult<Url> {\n        let base_url = match base {\n            Some(base) => Some(\n                base.into_url()\n                    .map_err(|s| anyhow::format_err!(\"invalid url `{}`: {}\", self, s))?,\n            ),\n            None => None,\n        };\n\n        Url::options()\n            .base_url(base_url.as_ref())\n            .parse(self)\n            .map_err(|s| anyhow::format_err!(\"invalid url `{}`: {}\", self, s))\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use crate::util::IntoUrlWithBase;\n\n    #[test]\n    fn into_url_with_base() {\n        assert_eq!(\n            \"rel/path\"\n                .into_url_with_base(Some(\"file:///abs/path/\"))\n                .unwrap()\n                .to_string(),\n            \"file:///abs/path/rel/path\"\n        );\n        assert_eq!(\n            \"rel/path\"\n                .into_url_with_base(Some(\"file:///abs/path/popped-file\"))\n                .unwrap()\n                .to_string(),\n            \"file:///abs/path/rel/path\"\n        );\n    }\n}\n\n// Path: src/cargo/util/canonical_url.rs\nuse crate::util::errors::CargoResult;\nuse std::hash::{self, Hash};\nuse url::Url;\n\n/// A newtype wrapper around `Url` which represents a \"canonical\" version of an\n/// original URL.\n///\n/// A \"canonical\" url is only intended for internal comparison purposes in\n/// Cargo. It's to help paper over mistakes such as depending on\n/// `github.com/foo/bar` vs `github.com/foo/bar.git`. This is **only** for\n/// internal purposes within Cargo and provides no means to actually read the\n/// underlying string value of the `Url` it contains. This is intentional,\n/// because all fetching should still happen within the context of the original\n/// URL.\n#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]\npub struct CanonicalUrl(Url);\n\nimpl CanonicalUrl {\n    pub fn new(url: &Url) -> CargoResult<CanonicalUrl> {\n        let mut url = url.clone();\n\n        // cannot-be-a-base-urls (e.g., `github.com:rust-lang/rustfmt.git`)\n        // are not supported.\n        if url.cannot_be_a_base() {\n            anyhow::bail!(\n                \"invalid url `{}`: cannot-be-a-base-URLs are not supported\",\n                url\n            )\n        }\n\n        // Strip a trailing slash.\n        if url.path().ends_with('/') {\n            url.path_segments_mut().unwrap().pop_if_empty();\n        }\n\n        // For GitHub URLs specifically, just lower-case everything. GitHub\n        // treats both the same, but they hash differently, and we're gonna be\n        // hashing them. This wants a more general solution, and also we're\n        // almost certainly not using the same case conversion rules that GitHub\n        // does. (See issue #84)\n        if url.host_str() == Some(\"github.com\") {\n            url = format!(\"https{}\", &url[url::Position::AfterScheme..])\n                .parse()\n                .unwrap();\n            let path = url.path().to_lowercase();\n            url.set_path(&path);\n        }\n\n        // Repos can generally be accessed with or without `.git` extension.\n        let needs_chopping = url.path().ends_with(\".git\");\n        if needs_chopping {\n            let last = {\n                let last = url.path_segments().unwrap().next_back().unwrap();\n                last[..last.len() - 4].to_owned()\n            };\n            url.path_segments_mut().unwrap().pop().push(&last);\n        }\n\n        Ok(CanonicalUrl(url))\n    }\n\n    /// Returns the raw canonicalized URL, although beware that this should\n    /// never be used/displayed/etc, it should only be used for internal data\n    /// structures and hashes and such.\n    \npub fn raw_canonicalized_url(&self) -> &Url {\n        &self.0\n    }\n}\n\n// See comment in `source_id.rs` for why we explicitly use `as_str()` here.\nimpl Hash for CanonicalUrl {\n    fn hash<S: hash::Hasher>(&self, into: &mut S) {\n        self.0.as_str().hash(into);\n    }\n}\n\n// Path: src/cargo/util/interning.rs\nuse serde::{Serialize, Serializer};\nuse serde_untagged::UntaggedEnumVisitor;\nuse std::borrow::Borrow;\nuse std::cmp::Ordering;\nuse std::collections::HashSet;\nuse std::ffi::OsStr;\nuse std::fmt;\nuse std::hash::{Hash, Hasher};\nuse std::ops::Deref;\nuse std::path::Path;\nuse std::ptr;\nuse std::str;\nuse std::sync::Mutex;\nuse std::sync::OnceLock;\n\nstatic STRING_CACHE: OnceLock<Mutex<HashSet<&'static str>>> = OnceLock::new();\n\n#[derive(Clone, Copy)]\npub struct InternedString {\n    inner: &'static str,\n}\n\nimpl<'a> From<&'a str> for InternedString {\n    fn from(item: &'a str) -> Self {\n        InternedString::new(item)\n    }\n}\n\nimpl<'a> From<&'a String> for InternedString {\n    fn from(item: &'a String) -> Self {\n        InternedString::new(item)\n    }\n}\n\nimpl From<String> for InternedString {\n    fn from(item: String) -> Self {\n        InternedString::new(&item)\n    }\n}\n\nimpl PartialEq for InternedString {\n    fn eq(&self, other: &InternedString) -> bool {\n        ptr::eq(self.as_str(), other.as_str())\n    }\n}\n\nimpl PartialEq<str> for InternedString {\n    fn eq(&self, other: &str) -> bool {\n        *self == other\n    }\n}\n\nimpl<'a> PartialEq<&'a str> for InternedString {\n    fn eq(&self, other: &&str) -> bool {\n        **self == **other\n    }\n}\n\nimpl Eq for InternedString {}\n\nimpl InternedString {\n    pub fn new(str: &str) -> InternedString {\n        let mut cache = STRING_CACHE.get_or_init(Default::default).lock().unwrap();\n        let s = cache.get(str).cloned().unwrap_or_else(|| {\n            let s = str.to_string().leak();\n            cache.insert(s);\n            s\n        });\n\n        InternedString { inner: s }\n    }\n\n    pub fn as_str(&self) -> &'static str {\n        self.inner\n    }\n}\n\nimpl Deref for InternedString {\n    type Target = str;\n\n    fn deref(&self) -> &'static str {\n        self.as_str()\n    }\n}\n\nimpl AsRef<str> for InternedString {\n    fn as_ref(&self) -> &str {\n        self.as_str()\n    }\n}\n\nimpl AsRef<OsStr> for InternedString {\n    fn as_ref(&self) -> &OsStr {\n        self.as_str().as_ref()\n    }\n}\n\nimpl AsRef<Path> for InternedString {\n    fn as_ref(&self) -> &Path {\n        self.as_str().as_ref()\n    }\n}\n\nimpl Hash for InternedString {\n    // N.B., we can't implement this as `identity(self).hash(state)`,\n    // because we use this for on-disk fingerprints and so need\n    // stability across Cargo invocations.\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        self.as_str().hash(state);\n    }\n}\n\nimpl Borrow<str> for InternedString {\n    // If we implement Hash as `identity(self).hash(state)`,\n    // then this will need to be removed.\n    fn borrow(&self) -> &str {\n        self.as_str()\n    }\n}\n\nimpl fmt::Debug for InternedString {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Debug::fmt(self.as_str(), f)\n    }\n}\n\nimpl fmt::Display for InternedString {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Display::fmt(self.as_str(), f)\n    }\n}\n\nimpl Ord for InternedString {\n    fn cmp(&self, other: &InternedString) -> Ordering {\n        self.as_str().cmp(other.as_str())\n    }\n}\n\nimpl PartialOrd for InternedString {\n    fn partial_cmp(&self, other: &InternedString) -> Option<Ordering> {\n        Some(self.cmp(other))\n    }\n}\n\nimpl Serialize for InternedString {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_str(self.inner)\n    }\n}\n\nimpl<'de> serde::Deserialize<'de> for InternedString {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: serde::Deserializer<'de>,\n    {\n        UntaggedEnumVisitor::new()\n            .expecting(\"an String like thing\")\n            .string(|value| Ok(InternedString::new(value)))\n            .deserialize(deserializer)\n    }\n}\n\n// Path: src/cargo/util/semver_ext.rs\nuse std::fmt::{self, Display};\n\nuse semver::{Comparator, Op, Version, VersionReq};\n\npub trait VersionExt {\n    fn is_prerelease(&self) -> bool;\n\n    fn to_exact_req(&self) -> VersionReq;\n}\n\nimpl VersionExt for Version {\n    fn is_prerelease(&self) -> bool {\n        !self.pre.is_empty()\n    }\n\n    fn to_exact_req(&self) -> VersionReq {\n        VersionReq {\n            comparators: vec![Comparator {\n                op: Op::Exact,\n                major: self.major,\n                minor: Some(self.minor),\n                patch: Some(self.patch),\n                pre: self.pre.clone(),\n            }],\n        }\n    }\n}\n\n#[derive(PartialEq, Eq, Hash, Clone, Debug)]\npub enum OptVersionReq {\n    Any,\n    Req(VersionReq),\n    /// The exact locked version and the original version requirement.\n    Locked(Version, VersionReq),\n    /// The exact requested version and the original version requirement.\n    ///\n    /// This looks identical to [`OptVersionReq::Locked`] but has a different\n    /// meaning, and is used for the `--precise` field of `cargo update`.\n    /// See comments in [`OptVersionReq::matches`] for more.\n    Precise(Version, VersionReq),\n}\n\nimpl OptVersionReq {\n    pub fn exact(version: &Version) -> Self {\n        OptVersionReq::Req(version.to_exact_req())\n    }\n\n    // Since some registries have allowed crate versions to differ only by build metadata,\n    // A query using OptVersionReq::exact return nondeterministic results.\n    // So we `lock_to` the exact version were interested in.\n    pub fn lock_to_exact(version: &Version) -> Self {\n        OptVersionReq::Locked(version.clone(), version.to_exact_req())\n    }\n\n    pub fn is_exact(&self) -> bool {\n        match self {\n            OptVersionReq::Any => false,\n            OptVersionReq::Req(req) | OptVersionReq::Precise(_, req) => {\n                req.comparators.len() == 1 && {\n                    let cmp = &req.comparators[0];\n                    cmp.op == Op::Exact && cmp.minor.is_some() && cmp.patch.is_some()\n                }\n            }\n            OptVersionReq::Locked(..) => true,\n        }\n    }\n\n    pub fn lock_to(&mut self, version: &Version) {\n        assert!(self.matches(version), \"cannot lock {} to {}\", self, version);\n        use OptVersionReq::*;\n        let version = version.clone();\n        *self = match self {\n            Any => Locked(version, VersionReq::STAR),\n            Req(req) | Locked(_, req) | Precise(_, req) => Locked(version, req.clone()),\n        };\n    }\n\n    /// Makes the requirement precise to the requested version.\n    ///\n    /// This is used for the `--precise` field of `cargo update`.\n    pub fn precise_to(&mut self, version: &Version) {\n        use OptVersionReq::*;\n        let version = version.clone();\n        *self = match self {\n            Any => Precise(version, VersionReq::STAR),\n            Req(req) | Locked(_, req) | Precise(_, req) => Precise(version, req.clone()),\n        };\n    }\n\n    pub fn is_precise(&self) -> bool {\n        matches!(self, OptVersionReq::Precise(..))\n    }\n...\n", "needle_token_start": 1504, "needle_token_end": 1530, "code_context_ntokens": 3939, "output": ["```rust\npub fn raw_canonicalized_url(&self) -> &Url {\n    &self.0\n}\n```"]}
{"repo": "rust-lang/cargo", "name": "max_unpack_size", "language": "rust", "path": "src/cargo/sources/registry/mod.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To determine the maximum allowable size for unpacking a compressed file in Cargo, ensuring it is within safe limits to prevent issues like zip bombs.\n2. **Input**: A context object containing environmental settings and a `u64` representing the size of the compressed file.\n3. **Output**: A `u64` value representing the maximum size to which the file can be safely unpacked.\n4. **Procedure**: \n   - Retrieve custom settings for maximum unpack size and compression ratio if available (primarily for testing purposes).\n   - Use predefined constants for maximum unpack size and compression ratio if custom settings are not provided.\n   - Calculate the maximum unpack size by comparing the predefined maximum size and the product of the input size and the maximum compression ratio, returning the larger value.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/cargo/sources/registry/http_remote.rs\n//! Access to a HTTP-based crate registry. See [`HttpRegistry`] for details.\n\nuse crate::core::global_cache_tracker;\nuse crate::core::{PackageId, SourceId};\nuse crate::sources::registry::download;\nuse crate::sources::registry::MaybeLock;\nuse crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};\nuse crate::util::cache_lock::CacheLockMode;\nuse crate::util::errors::{CargoResult, HttpNotSuccessful};\nuse crate::util::interning::InternedString;\nuse crate::util::network::http::http_handle;\nuse crate::util::network::retry::{Retry, RetryResult};\nuse crate::util::network::sleep::SleepTracker;\nuse crate::util::{auth, Filesystem, GlobalContext, IntoUrl, Progress, ProgressStyle};\nuse anyhow::Context as _;\nuse cargo_credential::Operation;\nuse cargo_util::paths;\nuse curl::easy::{Easy, List};\nuse curl::multi::{EasyHandle, Multi};\nuse std::cell::RefCell;\nuse std::collections::{HashMap, HashSet};\nuse std::fs::{self, File};\nuse std::io::ErrorKind;\nuse std::path::{Path, PathBuf};\nuse std::str;\nuse std::task::{ready, Poll};\nuse std::time::Duration;\nuse tracing::{debug, trace};\nuse url::Url;\n\n// HTTP headers\nconst ETAG: &'static str = \"etag\";\nconst LAST_MODIFIED: &'static str = \"last-modified\";\nconst WWW_AUTHENTICATE: &'static str = \"www-authenticate\";\nconst IF_NONE_MATCH: &'static str = \"if-none-match\";\nconst IF_MODIFIED_SINCE: &'static str = \"if-modified-since\";\n\nconst UNKNOWN: &'static str = \"Unknown\";\n\n/// A registry served by the HTTP-based registry API.\n///\n/// This type is primarily accessed through the [`RegistryData`] trait.\n///\n/// `HttpRegistry` implements the HTTP-based registry API outlined in [RFC 2789]. Read the RFC for\n/// the complete protocol, but _roughly_ the implementation loads each index file (e.g.,\n/// config.json or re/ge/regex) from an HTTP service rather than from a locally cloned git\n/// repository. The remote service can more or less be a static file server that simply serves the\n/// contents of the origin git repository.\n///\n/// Implemented naively, this leads to a significant amount of network traffic, as a lookup of any\n/// index file would need to check with the remote backend if the index file has changed. This\n/// cost is somewhat mitigated by the use of HTTP conditional fetches (`If-Modified-Since` and\n/// `If-None-Match` for `ETag`s) which can be efficiently handled by HTTP/2.\n///\n/// [RFC 2789]: https://github.com/rust-lang/rfcs/pull/2789\npub struct HttpRegistry<'gctx> {\n    /// The name of this source, a unique string (across all sources) used as\n    /// the directory name where its cached content is stored.\n    name: InternedString,\n    /// Path to the registry index (`$CARGO_HOME/registry/index/$REG-HASH`).\n    ///\n    /// To be fair, `HttpRegistry` doesn't store the registry index it\n    /// downloads on the file system, but other cached data like registry\n    /// configuration could be stored here.\n    index_path: Filesystem,\n    /// Path to the cache of `.crate` files (`$CARGO_HOME/registry/cache/$REG-HASH`).\n    cache_path: Filesystem,\n    /// The unique identifier of this registry source.\n    source_id: SourceId,\n    gctx: &'gctx GlobalContext,\n\n    /// Store the server URL without the protocol prefix (sparse+)\n    url: Url,\n\n    /// HTTP multi-handle for asynchronous/parallel requests.\n    multi: Multi,\n\n    /// Has the client requested a cache update?\n    ///\n    /// Only if they have do we double-check the freshness of each locally-stored index file.\n    requested_update: bool,\n\n    /// State for currently pending index downloads.\n    downloads: Downloads<'gctx>,\n\n    /// Does the config say that we can use HTTP multiplexing?\n    multiplexing: bool,\n\n    /// What paths have we already fetched since the last index update?\n    ///\n    /// We do not need to double-check any of these index files since we have already done so.\n    fresh: HashSet<PathBuf>,\n\n    /// Have we started to download any index files?\n    fetch_started: bool,\n\n    /// Cached registry configuration.\n    registry_config: Option<RegistryConfig>,\n\n    /// Should we include the authorization header?\n    auth_required: bool,\n\n    /// Url to get a token for the registry.\n    login_url: Option<Url>,\n\n    /// Headers received with an HTTP 401.\n    auth_error_headers: Vec<String>,\n\n    /// Disables status messages.\n    quiet: bool,\n}\n\n/// State for currently pending index file downloads.\nstruct Downloads<'gctx> {\n    /// When a download is started, it is added to this map. The key is a\n    /// \"token\" (see [`Download::token`]). It is removed once the download is\n    /// finished.\n    pending: HashMap<usize, (Download<'gctx>, EasyHandle)>,\n    /// Set of paths currently being downloaded.\n    /// This should stay in sync with the `pending` field.\n    pending_paths: HashSet<PathBuf>,\n    /// Downloads that have failed and are waiting to retry again later.\n    sleeping: SleepTracker<(Download<'gctx>, Easy)>,\n    /// The final result of each download.\n    results: HashMap<PathBuf, CargoResult<CompletedDownload>>,\n    /// The next ID to use for creating a token (see [`Download::token`]).\n    next: usize,\n    /// Progress bar.\n    progress: RefCell<Option<Progress<'gctx>>>,\n    /// Number of downloads that have successfully finished.\n    downloads_finished: usize,\n    /// Number of times the caller has requested blocking. This is used for\n    /// an estimate of progress.\n    blocking_calls: usize,\n}\n\n/// Represents a single index file download, including its progress and retry.\nstruct Download<'gctx> {\n    /// The token for this download, used as the key of the\n    /// [`Downloads::pending`] map and stored in [`EasyHandle`] as well.\n    token: usize,\n\n    /// The path of the package that we're downloading.\n    path: PathBuf,\n\n    /// Actual downloaded data, updated throughout the lifetime of this download.\n    data: RefCell<Vec<u8>>,\n\n    /// HTTP headers.\n    header_map: RefCell<Headers>,\n\n    /// Logic used to track retrying this download if it's a spurious failure.\n    retry: Retry<'gctx>,\n}\n\n/// HTTPS headers [`HttpRegistry`] cares about.\n#[derive(Default)]\nstruct Headers {\n    last_modified: Option<String>,\n    etag: Option<String>,\n    www_authenticate: Vec<String>,\n    /// All headers, including explicit headers above.\n    all: Vec<String>,\n}\n\n/// HTTP status code [`HttpRegistry`] cares about.\nenum StatusCode {\n    Success,\n    NotModified,\n    NotFound,\n    Unauthorized,\n}\n\n/// Represents a complete [`Download`] from an HTTP request.\n///\n/// Usually it is constructed in [`HttpRegistry::handle_completed_downloads`],\n/// and then returns to the caller of [`HttpRegistry::load()`].\nstruct CompletedDownload {\n    response_code: StatusCode,\n    data: Vec<u8>,\n    header_map: Headers,\n}\n\nimpl<'gctx> HttpRegistry<'gctx> {\n    /// Creates a HTTP-rebased remote registry for `source_id`.\n    ///\n    /// * `name` --- Name of a path segment where `.crate` tarballs and the\n    ///   registry index are stored. Expect to be unique.\n    pub fn new(\n        source_id: SourceId,\n        gctx: &'gctx GlobalContext,\n        name: &str,\n    ) -> CargoResult<HttpRegistry<'gctx>> {\n        let url = source_id.url().as_str();\n        // Ensure the url ends with a slash so we can concatenate paths.\n        if !url.ends_with('/') {\n            anyhow::bail!(\"sparse registry url must end in a slash `/`: {url}\")\n        }\n        assert!(source_id.is_sparse());\n        let url = url\n            .strip_prefix(\"sparse+\")\n            .expect(\"sparse registry needs sparse+ prefix\")\n            .into_url()\n            .expect(\"a url with the sparse+ stripped should still be valid\");\n\n        Ok(HttpRegistry {\n            name: name.into(),\n            index_path: gctx.registry_index_path().join(name),\n            cache_path: gctx.registry_cache_path().join(name),\n            source_id,\n            gctx,\n            url,\n            multi: Multi::new(),\n            multiplexing: false,\n            downloads: Downloads {\n                next: 0,\n                pending: HashMap::new(),\n                pending_paths: HashSet::new(),\n                sleeping: SleepTracker::new(),\n                results: HashMap::new(),\n                progress: RefCell::new(Some(Progress::with_style(\n                    \"Fetch\",\n                    ProgressStyle::Indeterminate,\n                    gctx,\n                ))),\n                downloads_finished: 0,\n                blocking_calls: 0,\n            },\n            fresh: HashSet::new(),\n            requested_update: false,\n            fetch_started: false,\n            registry_config: None,\n            auth_required: false,\n            login_url: None,\n            auth_error_headers: vec![],\n            quiet: false,\n        })\n    }\n\n    /// Splits HTTP `HEADER: VALUE` to a tuple.\n    fn handle_http_header(buf: &[u8]) -> Option<(&str, &str)> {\n        if buf.is_empty() {\n            return None;\n        }\n        let buf = std::str::from_utf8(buf).ok()?.trim_end();\n        // Don't let server sneak extra lines anywhere.\n        if buf.contains('\\n') {\n            return None;\n        }\n        let (tag, value) = buf.split_once(':')?;\n        let value = value.trim();\n        Some((tag, value))\n    }\n\n    /// Setup the necessary works before the first fetch gets started.\n    ///\n    /// This is a no-op if called more than one time.\n    fn start_fetch(&mut self) -> CargoResult<()> {\n        if self.fetch_started {\n            // We only need to run the setup code once.\n            return Ok(());\n        }\n        self.fetch_started = true;\n\n        // We've enabled the `http2` feature of `curl` in Cargo, so treat\n        // failures here as fatal as it would indicate a build-time problem.\n        self.multiplexing = self.gctx.http_config()?.multiplexing.unwrap_or(true);\n\n        self.multi\n            .pipelining(false, self.multiplexing)\n            .with_context(|| \"failed to enable multiplexing/pipelining in curl\")?;\n\n        // let's not flood the server with connections\n        self.multi.set_max_host_connections(2)?;\n\n        if !self.quiet {\n            self.gctx\n                .shell()\n                .status(\"Updating\", self.source_id.display_index())?;\n        }\n\n        Ok(())\n    }\n\n    /// Checks the results inside the [`HttpRegistry::multi`] handle, and\n    /// updates relevant state in [`HttpRegistry::downloads`] accordingly.\n    fn handle_completed_downloads(&mut self) -> CargoResult<()> {\n        assert_eq!(\n            self.downloads.pending.len(),\n            self.downloads.pending_paths.len()\n        );\n\n        // Collect the results from the Multi handle.\n        let results = {\n...\n// Path: src/cargo/util/io.rs\nuse std::io::{self, Read, Take};\n\n#[derive(Debug)]\npub struct LimitErrorReader<R> {\n    inner: Take<R>,\n}\n\nimpl<R: Read> LimitErrorReader<R> {\n    pub fn new(r: R, limit: u64) -> LimitErrorReader<R> {\n        LimitErrorReader {\n            inner: r.take(limit),\n        }\n    }\n}\n\nimpl<R: Read> Read for LimitErrorReader<R> {\n    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {\n        match self.inner.read(buf) {\n            Ok(0) if self.inner.limit() == 0 => Err(io::Error::new(\n                io::ErrorKind::Other,\n                \"maximum limit reached when reading\",\n            )),\n            e => e,\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::LimitErrorReader;\n\n    use std::io::Read;\n\n    #[test]\n    fn under_the_limit() {\n        let buf = &[1; 7][..];\n        let mut r = LimitErrorReader::new(buf, 8);\n        let mut out = Vec::new();\n        assert!(matches!(r.read_to_end(&mut out), Ok(7)));\n        assert_eq!(buf, out.as_slice());\n    }\n\n    #[test]\n    #[should_panic = \"maximum limit reached when reading\"]\n    fn over_the_limit() {\n        let buf = &[1; 8][..];\n        let mut r = LimitErrorReader::new(buf, 8);\n        let mut out = Vec::new();\n        r.read_to_end(&mut out).unwrap();\n    }\n}\n\n// Path: src/cargo/sources/registry/mod.rs\n//! A `Source` for registry-based packages.\n//!\n//! # What's a Registry?\n//!\n//! [Registries] are central locations where packages can be uploaded to,\n//! discovered, and searched for. The purpose of a registry is to have a\n//! location that serves as permanent storage for versions of a crate over time.\n//!\n//! Compared to git sources (see [`GitSource`]), a registry provides many\n//! packages as well as many versions simultaneously. Git sources can also\n//! have commits deleted through rebasings where registries cannot have their\n//! versions deleted.\n//!\n//! In Cargo, [`RegistryData`] is an abstraction over each kind of actual\n//! registry, and [`RegistrySource`] connects those implementations to\n//! [`Source`] trait. Two prominent features these abstractions provide are\n//!\n//! * A way to query the metadata of a package from a registry. The metadata\n//!   comes from the index.\n//! * A way to download package contents (a.k.a source files) that are required\n//!   when building the package itself.\n//!\n//! We'll cover each functionality later.\n//!\n//! [Registries]: https://doc.rust-lang.org/nightly/cargo/reference/registries.html\n//! [`GitSource`]: super::GitSource\n//!\n//! # Different Kinds of Registries\n//!\n//! Cargo provides multiple kinds of registries. Each of them serves the index\n//! and package contents in a slightly different way. Namely,\n//!\n//! * [`LocalRegistry`] --- Serves the index and package contents entirely on\n//!   a local filesystem.\n//! * [`RemoteRegistry`] --- Serves the index ahead of time from a Git\n//!   repository, and package contents are downloaded as needed.\n//! * [`HttpRegistry`] --- Serves both the index and package contents on demand\n//!   over a HTTP-based registry API. This is the default starting from 1.70.0.\n//!\n//! Each registry has its own [`RegistryData`] implementation, and can be\n//! created from either [`RegistrySource::local`] or [`RegistrySource::remote`].\n//!\n//! [`LocalRegistry`]: local::LocalRegistry\n//! [`RemoteRegistry`]: remote::RemoteRegistry\n//! [`HttpRegistry`]: http_remote::HttpRegistry\n//!\n//! # The Index of a Registry\n//!\n//! One of the major difficulties with a registry is that hosting so many\n//! packages may quickly run into performance problems when dealing with\n//! dependency graphs. It's infeasible for cargo to download the entire contents\n//! of the registry just to resolve one package's dependencies, for example. As\n//! a result, cargo needs some efficient method of querying what packages are\n//! available on a registry, what versions are available, and what the\n//! dependencies for each version is.\n//!\n//! To solve the problem, a registry must provide an index of package metadata.\n//! The index of a registry is essentially an easily query-able version of the\n//! registry's database for a list of versions of a package as well as a list\n//! of dependencies for each version. The exact format of the index is\n//! described later.\n//!\n//! See the [`index`] module for topics about the management, parsing, caching,\n//! and versioning for the on-disk index.\n//!\n//! ## The Format of The Index\n//!\n//! The index is a store for the list of versions for all packages known, so its\n//! format on disk is optimized slightly to ensure that `ls registry` doesn't\n//! produce a list of all packages ever known. The index also wants to ensure\n//! that there's not a million files which may actually end up hitting\n//! filesystem limits at some point. To this end, a few decisions were made\n//! about the format of the registry:\n//!\n//! 1. Each crate will have one file corresponding to it. Each version for a\n//!    crate will just be a line in this file (see [`IndexPackage`] for its\n//!    representation).\n//! 2. There will be two tiers of directories for crate names, under which\n//!    crates corresponding to those tiers will be located.\n//!    (See [`cargo_util::registry::make_dep_path`] for the implementation of\n//!    this layout hierarchy.)\n//!\n//! As an example, this is an example hierarchy of an index:\n//!\n//! ```notrust\n//! .\n//! \u251c\u2500\u2500 3\n//! \u2502\u00a0\u00a0 \u2514\u2500\u2500 u\n//! \u2502\u00a0\u00a0     \u2514\u2500\u2500 url\n//! \u251c\u2500\u2500 bz\n//! \u2502\u00a0\u00a0 \u2514\u2500\u2500 ip\n//! \u2502\u00a0\u00a0     \u2514\u2500\u2500 bzip2\n//! \u251c\u2500\u2500 config.json\n//! \u251c\u2500\u2500 en\n//! \u2502\u00a0\u00a0 \u2514\u2500\u2500 co\n//! \u2502\u00a0\u00a0     \u2514\u2500\u2500 encoding\n//! \u2514\u2500\u2500 li\n//!  \u00a0\u00a0 \u251c\u2500\u2500 bg\n//!  \u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 libgit2\n//!  \u00a0\u00a0 \u2514\u2500\u2500 nk\n//!  \u00a0\u00a0     \u2514\u2500\u2500 link-config\n//! ```\n//!\n//! The root of the index contains a `config.json` file with a few entries\n//! corresponding to the registry (see [`RegistryConfig`] below).\n//!\n//! Otherwise, there are three numbered directories (1, 2, 3) for crates with\n//! names 1, 2, and 3 characters in length. The 1/2 directories simply have the\n//! crate files underneath them, while the 3 directory is sharded by the first\n//! letter of the crate name.\n//!\n//! Otherwise the top-level directory contains many two-letter directory names,\n//! each of which has many sub-folders with two letters. At the end of all these\n//! are the actual crate files themselves.\n//!\n//! The purpose of this layout is to hopefully cut down on `ls` sizes as well as\n//! efficient lookup based on the crate name itself.\n//!\n//! See [The Cargo Book: Registry Index][registry-index] for the public\n//! interface on the index format.\n//!\n//! [registry-index]: https://doc.rust-lang.org/nightly/cargo/reference/registry-index.html\n//!\n//! ## The Index Files\n//!\n//! Each file in the index is the history of one crate over time. Each line in\n//! the file corresponds to one version of a crate, stored in JSON format (see\n//! the [`IndexPackage`] structure).\n//!\n//! As new versions are published, new lines are appended to this file. **The\n//! only modifications to this file that should happen over time are yanks of a\n//! particular version.**\n//!\n//! # Downloading Packages\n//!\n//! The purpose of the index was to provide an efficient method to resolve the\n//! dependency graph for a package. After resolution has been performed, we need\n//! to download the contents of packages so we can read the full manifest and\n//! build the source code.\n//!\n//! To accomplish this, [`RegistryData::download`] will \"make\" an HTTP request\n//! per-package requested to download tarballs into a local cache. These\n//! tarballs will then be unpacked into a destination folder.\n//!\n//! Note that because versions uploaded to the registry are frozen forever that\n//! the HTTP download and unpacking can all be skipped if the version has\n//! already been downloaded and unpacked. This caching allows us to only\n//! download a package when absolutely necessary.\n//!\n//! # Filesystem Hierarchy\n//!\n//! Overall, the `$HOME/.cargo` looks like this when talking about the registry\n//! (remote registries, specifically):\n//!\n//! ```notrust\n//! # A folder under which all registry metadata is hosted (similar to\n//! # $HOME/.cargo/git)\n//! $HOME/.cargo/registry/\n//!\n//!     # For each registry that cargo knows about (keyed by hostname + hash)\n//!     # there is a folder which is the checked out version of the index for\n//!     # the registry in this location. Note that this is done so cargo can\n//!     # support multiple registries simultaneously\n//!     index/\n//!         registry1-<hash>/\n//!         registry2-<hash>/\n//!         ...\n//!\n//!     # This folder is a cache for all downloaded tarballs (`.crate` file)\n//!     # from a registry. Once downloaded and verified, a tarball never changes.\n//!     cache/\n//!         registry1-<hash>/<pkg>-<version>.crate\n//!         ...\n//!\n//!     # Location in which all tarballs are unpacked. Each tarball is known to\n//!     # be frozen after downloading, so transitively this folder is also\n//!     # frozen once its unpacked (it's never unpacked again)\n//!     # CAVEAT: They are not read-only. See rust-lang/cargo#9455.\n//!     src/\n//!         registry1-<hash>/<pkg>-<version>/...\n//!         ...\n//! ```\n//!\n//! [`IndexPackage`]: index::IndexPackage\n\nuse std::collections::HashSet;\nuse std::fs;\nuse std::fs::{File, OpenOptions};\nuse std::io;\nuse std::io::Read;\nuse std::io::Write;\nuse std::path::{Path, PathBuf};\nuse std::task::{ready, Poll};\n\nuse anyhow::Context as _;\nuse cargo_util::paths::{self, exclude_from_backups_and_indexing};\nuse flate2::read::GzDecoder;\nuse serde::Deserialize;\nuse serde::Serialize;\nuse tar::Archive;\nuse tracing::debug;\n\nuse crate::core::dependency::Dependency;\nuse crate::core::global_cache_tracker;\nuse crate::core::{Package, PackageId, SourceId};\nuse crate::sources::source::MaybePackage;\nuse crate::sources::source::QueryKind;\nuse crate::sources::source::Source;\nuse crate::sources::PathSource;\nuse crate::util::cache_lock::CacheLockMode;\nuse crate::util::interning::InternedString;\nuse crate::util::network::PollExt;\nuse crate::util::{hex, VersionExt};\nuse crate::util::{restricted_names, CargoResult, Filesystem, GlobalContext, LimitErrorReader};\n\n/// The `.cargo-ok` file is used to track if the source is already unpacked.\n/// See [`RegistrySource::unpack_package`] for more.\n///\n/// Not to be confused with `.cargo-ok` file in git sources.\nconst PACKAGE_SOURCE_LOCK: &str = \".cargo-ok\";\n\npub const CRATES_IO_INDEX: &str = \"https://github.com/rust-lang/crates.io-index\";\npub const CRATES_IO_HTTP_INDEX: &str = \"sparse+https://index.crates.io/\";\npub const CRATES_IO_REGISTRY: &str = \"crates-io\";\npub const CRATES_IO_DOMAIN: &str = \"crates.io\";\n\n/// The content inside `.cargo-ok`.\n/// See [`RegistrySource::unpack_package`] for more.\n#[derive(Deserialize, Serialize)]\nstruct LockMetadata {\n    /// The version of `.cargo-ok` file\n    v: u32,\n}\n\n/// A [`Source`] implementation for a local or a remote registry.\n///\n/// This contains common functionality that is shared between each registry\n/// kind, with the registry-specific logic implemented as part of the\n/// [`RegistryData`] trait referenced via the `ops` field.\n///\n/// For general concepts of registries, see the [module-level documentation](crate::sources::registry).\npub struct RegistrySource<'gctx> {\n    /// A unique name of the source (typically used as the directory name\n    /// where its cached content is stored).\n    name: InternedString,\n    /// The unique identifier of this source.\n    source_id: SourceId,\n    /// The path where crate files are extracted (`$CARGO_HOME/registry/src/$REG-HASH`).\n    src_path: Filesystem,\n    /// Local reference to [`GlobalContext`] for convenience.\n    gctx: &'gctx GlobalContext,\n    /// Abstraction for interfacing to the different registry kinds.\n    ops: Box<dyn RegistryData + 'gctx>,\n    /// Interface for managing the on-disk index.\n    index: index::RegistryIndex<'gctx>,\n    /// A set of packages that should be allowed to be used, even if they are\n    /// yanked.\n    ///\n    /// This is populated from the entries in `Cargo.lock` to ensure that\n    /// `cargo update somepkg` won't unlock yanked entries in `Cargo.lock`.\n    /// Otherwise, the resolver would think that those entries no longer\n    /// exist, and it would trigger updates to unrelated packages.\n    yanked_whitelist: HashSet<PackageId>,\n    /// Yanked versions that have already been selected during queries.\n    ///\n    /// As of this writing, this is for not emitting the `--precise <yanked>`\n    /// warning twice, with the assumption of (`dep.package_name()` + `--precise`\n    /// version) being sufficient to uniquely identify the same query result.\n    selected_precise_yanked: HashSet<(InternedString, semver::Version)>,\n}\n\n/// The [`config.json`] file stored in the index.\n///\n/// The config file may look like:\n///\n/// ```json\n/// {\n///     \"dl\": \"https://example.com/api/{crate}/{version}/download\",\n///     \"api\": \"https://example.com/api\",\n///     \"auth-required\": false             # unstable feature (RFC 3139)\n/// }\n/// ```\n///\n/// [`config.json`]: https://doc.rust-lang.org/nightly/cargo/reference/registry-index.html#index-configuration\n#[derive(Deserialize, Debug, Clone)]\n#[serde(rename_all = \"kebab-case\")]\npub struct RegistryConfig {\n    /// Download endpoint for all crates.\n    ///\n    /// The string is a template which will generate the download URL for the\n    /// tarball of a specific version of a crate. The substrings `{crate}` and\n    /// `{version}` will be replaced with the crate's name and version\n    /// respectively.  The substring `{prefix}` will be replaced with the\n    /// crate's prefix directory name, and the substring `{lowerprefix}` will\n    /// be replaced with the crate's prefix directory name converted to\n    /// lowercase. The substring `{sha256-checksum}` will be replaced with the\n    /// crate's sha256 checksum.\n    ///\n    /// For backwards compatibility, if the string does not contain any\n    /// markers (`{crate}`, `{version}`, `{prefix}`, or `{lowerprefix}`), it\n    /// will be extended with `/{crate}/{version}/download` to\n    /// support registries like crates.io which were created before the\n    /// templating setup was created.\n    ///\n    /// For more on the template of the download URL, see [Index Configuration](\n    /// https://doc.rust-lang.org/nightly/cargo/reference/registry-index.html#index-configuration).\n    pub dl: String,\n\n    /// API endpoint for the registry. This is what's actually hit to perform\n    /// operations like yanks, owner modifications, publish new crates, etc.\n    /// If this is None, the registry does not support API commands.\n    pub api: Option<String>,\n\n    /// Whether all operations require authentication. See [RFC 3139].\n    ///\n    /// [RFC 3139]: https://rust-lang.github.io/rfcs/3139-cargo-alternative-registry-auth.html\n    #[serde(default)]\n    pub auth_required: bool,\n}\n\n/// Result from loading data from a registry.\npub enum LoadResponse {\n    /// The cache is valid. The cached data should be used.\n    CacheValid,\n\n    /// The cache is out of date. Returned data should be used.\n    Data {\n        raw_data: Vec<u8>,\n        /// Version of this data to determine whether it is out of date.\n        index_version: Option<String>,\n    },\n\n    /// The requested crate was found.\n    NotFound,\n}\n\n/// An abstract interface to handle both a local and remote registry.\n///\n/// This allows [`RegistrySource`] to abstractly handle each registry kind.\n///\n/// For general concepts of registries, see the [module-level documentation](crate::sources::registry).\npub trait RegistryData {\n    /// Performs initialization for the registry.\n    ///\n    /// This should be safe to call multiple times, the implementation is\n    /// expected to not do any work if it is already prepared.\n    fn prepare(&self) -> CargoResult<()>;\n\n    /// Returns the path to the index.\n    ///\n    /// Note that different registries store the index in different formats\n    /// (remote = git, http & local = files).\n    fn index_path(&self) -> &Filesystem;\n\n    /// Loads the JSON for a specific named package from the index.\n    ///\n    /// * `root` is the root path to the index.\n    /// * `path` is the relative path to the package to load (like `ca/rg/cargo`).\n    /// * `index_version` is the version of the requested crate data currently\n    ///    in cache. This is useful for checking if a local cache is outdated.\n    fn load(\n        &mut self,\n        root: &Path,\n        path: &Path,\n        index_version: Option<&str>,\n    ) -> Poll<CargoResult<LoadResponse>>;\n\n    /// Loads the `config.json` file and returns it.\n    ///\n    /// Local registries don't have a config, and return `None`.\n    fn config(&mut self) -> Poll<CargoResult<Option<RegistryConfig>>>;\n\n    /// Invalidates locally cached data.\n    fn invalidate_cache(&mut self);\n\n    /// If quiet, the source should not display any progress or status messages.\n    fn set_quiet(&mut self, quiet: bool);\n\n    /// Is the local cached data up-to-date?\n    fn is_updated(&self) -> bool;\n\n    /// Prepare to start downloading a `.crate` file.\n    ///\n    /// Despite the name, this doesn't actually download anything. If the\n    /// `.crate` is already downloaded, then it returns [`MaybeLock::Ready`].\n    /// If it hasn't been downloaded, then it returns [`MaybeLock::Download`]\n    /// which contains the URL to download. The [`crate::core::package::Downloads`]\n    /// system handles the actual download process. After downloading, it\n    /// calls [`Self::finish_download`] to save the downloaded file.\n    ///\n    /// `checksum` is currently only used by local registries to verify the\n    /// file contents (because local registries never actually download\n    /// anything). Remote registries will validate the checksum in\n    /// `finish_download`. For already downloaded `.crate` files, it does not\n    /// validate the checksum, assuming the filesystem does not suffer from\n    /// corruption or manipulation.\n    fn download(&mut self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock>;\n\n    /// Finish a download by saving a `.crate` file to disk.\n    ///\n    /// After [`crate::core::package::Downloads`] has finished a download,\n    /// it will call this to save the `.crate` file. This is only relevant\n    /// for remote registries. This should validate the checksum and save\n    /// the given data to the on-disk cache.\n    ///\n    /// Returns a [`File`] handle to the `.crate` file, positioned at the start.\n    fn finish_download(&mut self, pkg: PackageId, checksum: &str, data: &[u8])\n        -> CargoResult<File>;\n\n    /// Returns whether or not the `.crate` file is already downloaded.\n    fn is_crate_downloaded(&self, _pkg: PackageId) -> bool {\n        true\n    }\n\n    /// Validates that the global package cache lock is held.\n    ///\n    /// Given the [`Filesystem`], this will make sure that the package cache\n    /// lock is held. If not, it will panic. See\n    /// [`GlobalContext::acquire_package_cache_lock`] for acquiring the global lock.\n    ///\n    /// Returns the [`Path`] to the [`Filesystem`].\n    fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path;\n\n    /// Block until all outstanding Poll::Pending requests are Poll::Ready.\n    fn block_until_ready(&mut self) -> CargoResult<()>;\n}\n\n/// The status of [`RegistryData::download`] which indicates if a `.crate`\n/// file has already been downloaded, or if not then the URL to download.\npub enum MaybeLock {\n    /// The `.crate` file is already downloaded. [`File`] is a handle to the\n    /// opened `.crate` file on the filesystem.\n    Ready(File),\n    /// The `.crate` file is not downloaded, here's the URL to download it from.\n    ///\n    /// `descriptor` is just a text string to display to the user of what is\n    /// being downloaded.\n    Download {\n        url: String,\n        descriptor: String,\n        authorization: Option<String>,\n    },\n}\n\nmod download;\nmod http_remote;\nmod index;\npub use index::IndexSummary;\nmod local;\nmod remote;\n\n/// Generates a unique name for [`SourceId`] to have a unique path to put their\n/// index files.\nfn short_name(id: SourceId, is_shallow: bool) -> String {\n    // CAUTION: This should not change between versions. If you change how\n    // this is computed, it will orphan previously cached data, forcing the\n    // cache to be rebuilt and potentially wasting significant disk space. If\n    // you change it, be cautious of the impact. See `test_cratesio_hash` for\n    // a similar discussion.\n    let hash = hex::short_hash(&id);\n    let ident = id.url().host_str().unwrap_or(\"\").to_string();\n    let mut name = format!(\"{}-{}\", ident, hash);\n    if is_shallow {\n        name.push_str(\"-shallow\");\n    }\n    name\n}\n\nimpl<'gctx> RegistrySource<'gctx> {\n    /// Creates a [`Source`] of a \"remote\" registry.\n    /// It could be either an HTTP-based [`http_remote::HttpRegistry`] or\n    /// a Git-based [`remote::RemoteRegistry`].\n    ///\n    /// * `yanked_whitelist` --- Packages allowed to be used, even if they are yanked.\n    pub fn remote(\n        source_id: SourceId,\n        yanked_whitelist: &HashSet<PackageId>,\n        gctx: &'gctx GlobalContext,\n    ) -> CargoResult<RegistrySource<'gctx>> {\n        assert!(source_id.is_remote_registry());\n        let name = short_name(\n            source_id,\n            gctx.cli_unstable()\n                .git\n                .map_or(false, |features| features.shallow_index)\n                && !source_id.is_sparse(),\n        );\n        let ops = if source_id.is_sparse() {\n            Box::new(http_remote::HttpRegistry::new(source_id, gctx, &name)?) as Box<_>\n        } else {\n            Box::new(remote::RemoteRegistry::new(source_id, gctx, &name)) as Box<_>\n        };\n\n        Ok(RegistrySource::new(\n            source_id,\n            gctx,\n            &name,\n            ops,\n            yanked_whitelist,\n        ))\n    }\n\n    /// Creates a [`Source`] of a local registry, with [`local::LocalRegistry`] under the hood.\n    ///\n    /// * `path` --- The root path of a local registry on the file system.\n    /// * `yanked_whitelist` --- Packages allowed to be used, even if they are yanked.\n    pub fn local(\n        source_id: SourceId,\n        path: &Path,\n        yanked_whitelist: &HashSet<PackageId>,\n        gctx: &'gctx GlobalContext,\n    ) -> RegistrySource<'gctx> {\n        let name = short_name(source_id, false);\n        let ops = local::LocalRegistry::new(path, gctx, &name);\n        RegistrySource::new(source_id, gctx, &name, Box::new(ops), yanked_whitelist)\n    }\n\n    /// Creates a source of a registry. This is a inner helper function.\n    ///\n    /// * `name` --- Name of a path segment which may affect where `.crate`\n    ///   tarballs, the registry index and cache are stored. Expect to be unique.\n    /// * `ops` --- The underlying [`RegistryData`] type.\n    /// * `yanked_whitelist` --- Packages allowed to be used, even if they are yanked.\n    fn new(\n        source_id: SourceId,\n        gctx: &'gctx GlobalContext,\n        name: &str,\n        ops: Box<dyn RegistryData + 'gctx>,\n        yanked_whitelist: &HashSet<PackageId>,\n    ) -> RegistrySource<'gctx> {\n        RegistrySource {\n            name: name.into(),\n            src_path: gctx.registry_source_path().join(name),\n            gctx,\n            source_id,\n            index: index::RegistryIndex::new(source_id, ops.index_path(), gctx),\n            yanked_whitelist: yanked_whitelist.clone(),\n            ops,\n            selected_precise_yanked: HashSet::new(),\n        }\n    }\n\n    /// Decode the [configuration](RegistryConfig) stored within the registry.\n    ///\n    /// This requires that the index has been at least checked out.\n    pub fn config(&mut self) -> Poll<CargoResult<Option<RegistryConfig>>> {\n        self.ops.config()\n    }\n\n    /// Unpacks a downloaded package into a location where it's ready to be\n    /// compiled.\n    ///\n    /// No action is taken if the source looks like it's already unpacked.\n    ///\n    /// # History of interruption detection with `.cargo-ok` file\n    ///\n    /// Cargo has always included a `.cargo-ok` file ([`PACKAGE_SOURCE_LOCK`])\n    /// to detect if extraction was interrupted, but it was originally empty.\n    ///\n    /// In 1.34, Cargo was changed to create the `.cargo-ok` file before it\n    /// started extraction to implement fine-grained locking. After it was\n    /// finished extracting, it wrote two bytes to indicate it was complete.\n    /// It would use the length check to detect if it was possibly interrupted.\n    ///\n    /// In 1.36, Cargo changed to not use fine-grained locking, and instead used\n    /// a global lock. The use of `.cargo-ok` was no longer needed for locking\n    /// purposes, but was kept to detect when extraction was interrupted.\n    ///\n    /// In 1.49, Cargo changed to not create the `.cargo-ok` file before it\n    /// started extraction to deal with `.crate` files that inexplicably had\n    /// a `.cargo-ok` file in them.\n    ///\n    /// In 1.64, Cargo changed to detect `.crate` files with `.cargo-ok` files\n    /// in them in response to [CVE-2022-36113], which dealt with malicious\n    /// `.crate` files making `.cargo-ok` a symlink causing cargo to write \"ok\"\n    /// to any arbitrary file on the filesystem it has permission to.\n    ///\n    /// In 1.71, `.cargo-ok` changed to contain a JSON `{ v: 1 }` to indicate\n    /// the version of it. A failure of parsing will result in a heavy-hammer\n    /// approach that unpacks the `.crate` file again. This is in response to a\n    /// security issue that the unpacking didn't respect umask on Unix systems.\n    ///\n    /// This is all a long-winded way of explaining the circumstances that might\n    /// cause a directory to contain a `.cargo-ok` file that is empty or\n    /// otherwise corrupted. Either this was extracted by a version of Rust\n    /// before 1.34, in which case everything should be fine. However, an empty\n    /// file created by versions 1.36 to 1.49 indicates that the extraction was\n    /// interrupted and that we need to start again.\n    ///\n    /// Another possibility is that the filesystem is simply corrupted, in\n    /// which case deleting the directory might be the safe thing to do. That\n    /// is probably unlikely, though.\n    ///\n    /// To be safe, we deletes the directory and starts over again if an empty\n    /// `.cargo-ok` file is found.\n    ///\n    /// [CVE-2022-36113]: https://blog.rust-lang.org/2022/09/14/cargo-cves.html#arbitrary-file-corruption-cve-2022-36113\n    fn unpack_package(&self, pkg: PackageId, tarball: &File) -> CargoResult<PathBuf> {\n        let package_dir = format!(\"{}-{}\", pkg.name(), pkg.version());\n        let dst = self.src_path.join(&package_dir);\n        let path = dst.join(PACKAGE_SOURCE_LOCK);\n        let path = self\n            .gctx\n            .assert_package_cache_locked(CacheLockMode::DownloadExclusive, &path);\n        let unpack_dir = path.parent().unwrap();\n        match fs::read_to_string(path) {\n            Ok(ok) => match serde_json::from_str::<LockMetadata>(&ok) {\n                Ok(lock_meta) if lock_meta.v == 1 => {\n                    self.gctx\n                        .deferred_global_last_use()?\n                        .mark_registry_src_used(global_cache_tracker::RegistrySrc {\n                            encoded_registry_name: self.name,\n                            package_dir: package_dir.into(),\n                            size: None,\n                        });\n                    return Ok(unpack_dir.to_path_buf());\n                }\n                _ => {\n                    if ok == \"ok\" {\n                        tracing::debug!(\"old `ok` content found, clearing cache\");\n                    } else {\n                        tracing::warn!(\"unrecognized .cargo-ok content, clearing cache: {ok}\");\n                    }\n                    // See comment of `unpack_package` about why removing all stuff.\n                    paths::remove_dir_all(dst.as_path_unlocked())?;\n                }\n            },\n            Err(e) if e.kind() == io::ErrorKind::NotFound => {}\n            Err(e) => anyhow::bail!(\"unable to read .cargo-ok file at {path:?}: {e}\"),\n        }\n        dst.create_dir()?;\n        let mut tar = {\n            let size_limit = max_unpack_size(self.gctx, tarball.metadata()?.len());\n            let gz = GzDecoder::new(tarball);\n            let gz = LimitErrorReader::new(gz, size_limit);\n            let mut tar = Archive::new(gz);\n            set_mask(&mut tar);\n            tar\n        };\n        let mut bytes_written = 0;\n        let prefix = unpack_dir.file_name().unwrap();\n        let parent = unpack_dir.parent().unwrap();\n        for entry in tar.entries()? {\n            let mut entry = entry.with_context(|| \"failed to iterate over archive\")?;\n            let entry_path = entry\n                .path()\n                .with_context(|| \"failed to read entry path\")?\n                .into_owned();\n\n            // We're going to unpack this tarball into the global source\n            // directory, but we want to make sure that it doesn't accidentally\n            // (or maliciously) overwrite source code from other crates. Cargo\n            // itself should never generate a tarball that hits this error, and\n            // crates.io should also block uploads with these sorts of tarballs,\n            // but be extra sure by adding a check here as well.\n            if !entry_path.starts_with(prefix) {\n                anyhow::bail!(\n                    \"invalid tarball downloaded, contains \\\n                     a file at {:?} which isn't under {:?}\",\n                    entry_path,\n                    prefix\n                )\n            }\n            // Prevent unpacking the lockfile from the crate itself.\n            if entry_path\n                .file_name()\n                .map_or(false, |p| p == PACKAGE_SOURCE_LOCK)\n            {\n                continue;\n            }\n            // Unpacking failed\n            bytes_written += entry.size();\n            let mut result = entry.unpack_in(parent).map_err(anyhow::Error::from);\n            if cfg!(windows) && restricted_names::is_windows_reserved_path(&entry_path) {\n                result = result.with_context(|| {\n                    format!(\n                        \"`{}` appears to contain a reserved Windows path, \\\n                        it cannot be extracted on Windows\",\n                        entry_path.display()\n                    )\n                });\n            }\n            result\n                .with_context(|| format!(\"failed to unpack entry at `{}`\", entry_path.display()))?;\n        }\n\n        // Now that we've finished unpacking, create and write to the lock file to indicate that\n        // unpacking was successful.\n        let mut ok = OpenOptions::new()\n            .create_new(true)\n            .read(true)\n            .write(true)\n            .open(&path)\n            .with_context(|| format!(\"failed to open `{}`\", path.display()))?;\n\n        let lock_meta = LockMetadata { v: 1 };\n        write!(ok, \"{}\", serde_json::to_string(&lock_meta).unwrap())?;\n\n        self.gctx\n            .deferred_global_last_use()?\n            .mark_registry_src_used(global_cache_tracker::RegistrySrc {\n                encoded_registry_name: self.name,\n                package_dir: package_dir.into(),\n                size: Some(bytes_written),\n            });\n\n        Ok(unpack_dir.to_path_buf())\n    }\n\n    /// Turns the downloaded `.crate` tarball file into a [`Package`].\n    ///\n    /// This unconditionally sets checksum for the returned package, so it\n    /// should only be called after doing integrity check. That is to say,\n    /// you need to call either [`RegistryData::download`] or\n    /// [`RegistryData::finish_download`] before calling this method.\n    fn get_pkg(&mut self, package: PackageId, path: &File) -> CargoResult<Package> {\n        let path = self\n            .unpack_package(package, path)\n            .with_context(|| format!(\"failed to unpack package `{}`\", package))?;\n        let mut src = PathSource::new(&path, self.source_id, self.gctx);\n        src.update()?;\n        let mut pkg = match src.download(package)? {\n            MaybePackage::Ready(pkg) => pkg,\n            MaybePackage::Download { .. } => unreachable!(),\n        };\n\n        // After we've loaded the package configure its summary's `checksum`\n        // field with the checksum we know for this `PackageId`.\n        let cksum = self\n            .index\n            .hash(package, &mut *self.ops)\n            .expect(\"a downloaded dep now pending!?\")\n            .expect(\"summary not found\");\n        pkg.manifest_mut()\n            .summary_mut()\n            .set_checksum(cksum.to_string());\n\n        Ok(pkg)\n    }\n}\n\nimpl<'gctx> Source for RegistrySource<'gctx> {\n    fn query(\n        &mut self,\n        dep: &Dependency,\n        kind: QueryKind,\n        f: &mut dyn FnMut(IndexSummary),\n    ) -> Poll<CargoResult<()>> {\n        let mut req = dep.version_req().clone();\n\n        // Handle `cargo update --precise` here.\n        if let Some((_, requested)) = self\n            .source_id\n            .precise_registry_version(dep.package_name().as_str())\n            .filter(|(c, to)| {\n                if to.is_prerelease() && self.gctx.cli_unstable().unstable_options {\n                    req.matches_prerelease(c)\n                } else {\n                    req.matches(c)\n                }\n            })\n        {\n            req.precise_to(&requested);\n        }\n\n        let mut called = false;\n        let callback = &mut |s| {\n            called = true;\n            f(s);\n        };\n\n        // If this is a locked dependency, then it came from a lock file and in\n        // theory the registry is known to contain this version. If, however, we\n        // come back with no summaries, then our registry may need to be\n        // updated, so we fall back to performing a lazy update.\n        if kind == QueryKind::Exact && req.is_locked() && !self.ops.is_updated() {\n            debug!(\"attempting query without update\");\n            ready!(self\n                .index\n                .query_inner(dep.package_name(), &req, &mut *self.ops, &mut |s| {\n                    if dep.matches(s.as_summary()) {\n                        // We are looking for a package from a lock file so we do not care about yank\n                        callback(s)\n                    }\n                },))?;\n            if called {\n                Poll::Ready(Ok(()))\n            } else {\n                debug!(\"falling back to an update\");\n                self.invalidate_cache();\n                Poll::Pending\n            }\n        } else {\n            let mut precise_yanked_in_use = false;\n            ready!(self\n                .index\n                .query_inner(dep.package_name(), &req, &mut *self.ops, &mut |s| {\n                    let matched = match kind {\n                        QueryKind::Exact => {\n                            if req.is_precise() && self.gctx.cli_unstable().unstable_options {\n                                dep.matches_prerelease(s.as_summary())\n                            } else {\n                                dep.matches(s.as_summary())\n                            }\n                        }\n                        QueryKind::Alternatives => true,\n                        QueryKind::Normalized => true,\n                    };\n                    if !matched {\n                        return;\n                    }\n                    // Next filter out all yanked packages. Some yanked packages may\n                    // leak through if they're in a whitelist (aka if they were\n                    // previously in `Cargo.lock`\n                    if !s.is_yanked() {\n                        callback(s);\n                    } else if self.yanked_whitelist.contains(&s.package_id()) {\n                        callback(s);\n                    } else if req.is_precise() {\n                        precise_yanked_in_use = true;\n                        if self.gctx.cli_unstable().unstable_options {\n                            callback(s);\n                        }\n                    }\n                }))?;\n            if precise_yanked_in_use {\n                self.gctx\n                    .cli_unstable()\n                    .fail_if_stable_opt(\"--precise <yanked-version>\", 4225)?;\n                let name = dep.package_name();\n                let version = req\n                    .precise_version()\n                    .expect(\"--precise <yanked-version> in use\");\n                if self.selected_precise_yanked.insert((name, version.clone())) {\n                    let mut shell = self.gctx.shell();\n                    shell.warn(format_args!(\n                        \"selected package `{name}@{version}` was yanked by the author\"\n                    ))?;\n                    shell.note(\"if possible, try a compatible non-yanked version\")?;\n                }\n            }\n            if called {\n                return Poll::Ready(Ok(()));\n            }\n            let mut any_pending = false;\n            if kind == QueryKind::Alternatives || kind == QueryKind::Normalized {\n                // Attempt to handle misspellings by searching for a chain of related\n                // names to the original name. The resolver will later\n                // reject any candidates that have the wrong name, and with this it'll\n                // along the way produce helpful \"did you mean?\" suggestions.\n                // For now we only try the canonical lysing `-` to `_` and vice versa.\n                // More advanced fuzzy searching become in the future.\n                for name_permutation in [\n                    dep.package_name().replace('-', \"_\"),\n                    dep.package_name().replace('_', \"-\"),\n                ] {\n                    let name_permutation = InternedString::new(&name_permutation);\n                    if name_permutation == dep.package_name() {\n                        continue;\n                    }\n                    any_pending |= self\n                        .index\n                        .query_inner(name_permutation, &req, &mut *self.ops, f)?\n                        .is_pending();\n                }\n            }\n            if any_pending {\n                Poll::Pending\n            } else {\n                Poll::Ready(Ok(()))\n            }\n        }\n    }\n\n    fn supports_checksums(&self) -> bool {\n        true\n    }\n\n    fn requires_precise(&self) -> bool {\n        false\n    }\n\n    fn source_id(&self) -> SourceId {\n        self.source_id\n    }\n\n    fn invalidate_cache(&mut self) {\n        self.index.clear_summaries_cache();\n        self.ops.invalidate_cache();\n    }\n\n    fn set_quiet(&mut self, quiet: bool) {\n        self.ops.set_quiet(quiet);\n    }\n\n    fn download(&mut self, package: PackageId) -> CargoResult<MaybePackage> {\n        let hash = loop {\n            match self.index.hash(package, &mut *self.ops)? {\n                Poll::Pending => self.block_until_ready()?,\n                Poll::Ready(hash) => break hash,\n            }\n        };\n        match self.ops.download(package, hash)? {\n            MaybeLock::Ready(file) => self.get_pkg(package, &file).map(MaybePackage::Ready),\n            MaybeLock::Download {\n                url,\n                descriptor,\n                authorization,\n            } => Ok(MaybePackage::Download {\n                url,\n                descriptor,\n                authorization,\n            }),\n        }\n    }\n\n    fn finish_download(&mut self, package: PackageId, data: Vec<u8>) -> CargoResult<Package> {\n        let hash = loop {\n            match self.index.hash(package, &mut *self.ops)? {\n                Poll::Pending => self.block_until_ready()?,\n                Poll::Ready(hash) => break hash,\n            }\n        };\n        let file = self.ops.finish_download(package, hash, &data)?;\n        self.get_pkg(package, &file)\n    }\n\n    fn fingerprint(&self, pkg: &Package) -> CargoResult<String> {\n        Ok(pkg.package_id().version().to_string())\n    }\n\n    fn describe(&self) -> String {\n        self.source_id.display_index()\n    }\n\n    fn add_to_yanked_whitelist(&mut self, pkgs: &[PackageId]) {\n        self.yanked_whitelist.extend(pkgs);\n    }\n\n    fn is_yanked(&mut self, pkg: PackageId) -> Poll<CargoResult<bool>> {\n        self.index.is_yanked(pkg, &mut *self.ops)\n    }\n\n    fn block_until_ready(&mut self) -> CargoResult<()> {\n        // Before starting to work on the registry, make sure that\n        // `<cargo_home>/registry` is marked as excluded from indexing and\n        // backups. Older versions of Cargo didn't do this, so we do it here\n        // regardless of whether `<cargo_home>` exists.\n        //\n        // This does not use `create_dir_all_excluded_from_backups_atomic` for\n        // the same reason: we want to exclude it even if the directory already\n        // exists.\n        //\n        // IO errors in creating and marking it are ignored, e.g. in case we're on a\n        // read-only filesystem.\n        let registry_base = self.gctx.registry_base_path();\n        let _ = registry_base.create_dir();\n        exclude_from_backups_and_indexing(&registry_base.into_path_unlocked());\n\n        self.ops.block_until_ready()\n    }\n}\n\nimpl RegistryConfig {\n    /// File name of [`RegistryConfig`].\n    const NAME: &'static str = \"config.json\";\n}\n\n/// Get the maximum upack size that Cargo permits\n/// based on a given `size` of your compressed file.\n///\n/// Returns the larger one between `size * max compression ratio`\n/// and a fixed max unpacked size.\n///\n/// In reality, the compression ratio usually falls in the range of 2:1 to 10:1.\n/// We choose 20:1 to cover almost all possible cases hopefully.\n/// Any ratio higher than this is considered as a zip bomb.\n///\n/// In the future we might want to introduce a configurable size.\n///\n/// Some of the real world data from common compression algorithms:\n///\n/// * <https://www.zlib.net/zlib_tech.html>\n/// * <https://cran.r-project.org/web/packages/brotli/vignettes/brotli-2015-09-22.pdf>\n/// * <https://blog.cloudflare.com/results-experimenting-brotli/>\n/// * <https://tukaani.org/lzma/benchmarks.html>\nfn max_unpack_size(gctx: &GlobalContext, size: u64) -> u64 {\n    const SIZE_VAR: &str = \"__CARGO_TEST_MAX_UNPACK_SIZE\";\n    co\nnst RATIO_VAR: &str = \"__CARGO_TEST_MAX_UNPACK_RATIO\";\n    const MAX_UNPACK_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB\n    const MAX_COMPRESSION_RATIO: usize = 20; // 20:1\n\n    let max_unpack_size = if cfg!(debug_assertions) && gctx.get_env(SIZE_VAR).is_ok() {\n        // For integration test only.\n        gctx.get_env(SIZE_VAR)\n            .unwrap()\n            .parse()\n            .expect(\"a max unpack size in bytes\")\n    } else {\n        MAX_UNPACK_SIZE\n    };\n    let max_compression_ratio = if cfg!(debug_assertions) && gctx.get_env(RATIO_VAR).is_ok() {\n        // For integration test only.\n        gctx.get_env(RATIO_VAR)\n            .unwrap()\n            .parse()\n            .expect(\"a max compression ratio in bytes\")\n    } else {\n        MAX_COMPRESSION_RATIO\n    };\n\n    u64::max(max_unpack_size, size * max_compression_ratio as u64)\n}\n\n/// Set the current [`umask`] value for the given tarball. No-op on non-Unix\n/// platforms.\n///\n/// On Windows, tar only looks at user permissions and tries to set the \"read\n/// only\" attribute, so no-op as well.\n///\n/// [`umask`]: https://man7.org/linux/man-pages/man2/umask.2.html\n#[allow(unused_variables)]\nfn set_mask<R: Read>(tar: &mut Archive<R>) {\n    #[cfg(unix)]\n    tar.set_mask(crate::util::get_umask());\n}\n\n// Path: src/cargo/sources/registry/index/mod.rs\n//! Management of the index of a registry source.\n//!\n//! This module contains management of the index and various operations, such as\n//! actually parsing the index, looking for crates, etc. This is intended to be\n//! abstract over remote indices (downloaded via Git or HTTP) and local registry\n//! indices (which are all just present on the filesystem).\n//!\n//! ## How the index works\n//!\n//! Here is a simple flow when loading a [`Summary`] (metadata) from the index:\n//!\n//! 1. A query is fired via [`RegistryIndex::query_inner`].\n//! 2. Tries loading all summaries via [`RegistryIndex::load_summaries`], and\n//!    under the hood calling [`Summaries::parse`] to parse an index file.\n//!     1. If an on-disk index cache is present, loads it via\n//!        [`Summaries::parse_cache`].\n//!     2. Otherwise goes to the slower path [`RegistryData::load`] to get the\n//!        specific index file.\n//! 3. A [`Summary`] is now ready in callback `f` in [`RegistryIndex::query_inner`].\n//!\n//! To learn the rationale behind this multi-layer index metadata loading,\n//! see [the documentation of the on-disk index cache](cache).\nuse crate::core::dependency::{Artifact, DepKind};\nuse crate::core::Dependency;\nuse crate::core::{PackageId, SourceId, Summary};\nuse crate::sources::registry::{LoadResponse, RegistryData};\nuse crate::util::interning::InternedString;\nuse crate::util::IntoUrl;\nuse crate::util::{internal, CargoResult, Filesystem, GlobalContext, OptVersionReq};\nuse cargo_util::registry::make_dep_path;\nuse cargo_util_schemas::manifest::RustVersion;\nuse semver::Version;\nuse serde::Deserialize;\nuse std::borrow::Cow;\nuse std::collections::BTreeMap;\n...\n", "needle_token_start": 15279, "needle_token_end": 15647, "code_context_ntokens": 16279, "output": ["```rust\n/// Get the maximum upack size that Cargo permits\n/// based on a given `size` of your compressed file.\n///\n/// Returns the larger one between `size * max compression ratio`\n/// and a fixed max unpacked size.\n///\n/// In reality, the compression ratio usually falls in the range of 2:1 to 10:1.\n/// We choose 20:1 to cover almost all possible cases hopefully.\n/// Any ratio higher than this is considered as a zip bomb.\n///\n/// In the future we might want to introduce a configurable size.\n///\n/// Some of the real world data from common compression algorithms:\n///\n/// * <https://www.zlib.net/zlib_tech.html>\n/// * <https://cran.r-project.org/web/packages/brotli/vignettes/brotli-2015-09-22.pdf>\n/// * <https://blog.cloudflare.com/results-experimenting-brotli/>\n/// * <https://tukaani.org/lzma/benchmarks.html>\nfn max_unpack_size(gctx: &GlobalContext, size: u64) -> u64 {\n    const SIZE_VAR: &str = \"__CARGO_TEST_MAX_UNPACK_SIZE\";\n    const RATIO_VAR: &str = \"__CARGO_TEST_MAX_UNPACK_RATIO\";\n    const MAX_UNPACK_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB\n    const MAX_COMPRESSION_RATIO: usize = 20; // 20:1\n\n    let max_unpack_size = if cfg!(debug_assertions) && gctx.get_env(SIZE_VAR).is_ok() {\n        // For integration test only.\n        gctx.get_env(SIZE_VAR)\n            .unwrap()\n            .parse()\n            .expect(\"a max unpack size in bytes\")\n    } else {\n        MAX_UNPACK_SIZE\n    };\n    let max_compression_ratio = if cfg!(debug_assertions) && gctx.get_env(RATIO_VAR).is_ok() {\n        // For integration test only.\n        gctx.get_env(RATIO_VAR)\n            .unwrap()\n            .parse()\n            .expect(\"a max compression ratio in bytes\")\n    } else {\n        MAX_COMPRESSION_RATIO\n    };\n\n    u64::max(max_unpack_size, size * max_compression_ratio as u64)\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "h2_options", "language": "rust", "path": "pingora-core/src/apps/mod.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To provide configuration options for establishing HTTP/2 connections within an HTTP server application.\n2. **Input**: No direct inputs are taken as it is a method that operates based on the internal state of the server application.\n3. **Output**: Returns optional configuration settings specific to HTTP/2 connections, which can modify default behaviors or enhance performance and security features.\n4. **Procedure**: This method checks the internal configurations of the HTTP server application and determines if custom settings for HTTP/2 connections are necessary. If custom settings are defined, it returns these settings; otherwise, it returns `None`, indicating that default settings should be used.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! The abstraction and implementation interface for service application logic\n\npub mod http_app;\npub mod prometheus_http_app;\n\nuse crate::server::ShutdownWatch;\nuse async_trait::async_trait;\nuse log::{debug, error};\nuse std::sync::Arc;\n\nuse crate::protocols::http::v2::server;\nuse crate::protocols::http::ServerSession;\nuse crate::protocols::Digest;\nuse crate::protocols::Stream;\nuse crate::protocols::ALPN;\n\n#[cfg_attr(not(doc_async_trait), async_trait)]\n/// This trait defines the interface of a transport layer (TCP or TLS) application.\npub trait ServerApp {\n    /// Whenever a new connection is established, this function will be called with the established\n    /// [`Stream`] object provided.\n    ///\n    /// The application can do whatever it wants with the `session`.\n    ///\n    /// After processing the `session`, if the `session`'s connection is reusable, This function\n    /// can return it to the service by returning `Some(session)`. The returned `session` will be\n    /// fed to another [`Self::process_new()`] for another round of processing.\n    /// If not reusable, `None` should be returned.\n    ///\n    /// The `shutdown` argument will change from `false` to `true` when the server receives a\n    /// signal to shutdown. This argument allows the application to react accordingly.\n    async fn process_new(\n        self: &Arc<Self>,\n        mut session: Stream,\n        // TODO: make this ShutdownWatch so that all task can await on this event\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream>;\n\n    /// This callback will be called once after the service stops listening to its endpoints.\n    fn cleanup(&self) {}\n}\n\n/// This trait defines the interface of an HTTP application.\n#[cfg_attr(not(doc_async_trait), async_trait)]\npub trait HttpServerApp {\n    /// Similar to the [`ServerApp`], this function is called whenever a new HTTP session is established.\n    ///\n    /// After successful processing, [`ServerSession::finish()`] can be called to return an optionally reusable\n    /// connection back to the service. The caller needs to make sure that the connection is in a reusable state\n    /// i.e., no error or incomplete read or write headers or bodies. Otherwise a `None` should be returned.\n    async fn process_new_http(\n        self: &Arc<Self>,\n        mut session: ServerSession,\n        // TODO: make this ShutdownWatch so that all task can await on this event\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream>;\n\n    /// Provide options on how HTTP/2 connection should be established. This function will be called\n    /// every time a new HTTP/2 **connection** needs to be established.\n    ///\n    /// A `None` means to use the built-in default options. See [`server::H2Options`] for more details.\n    \nfn h2_options(&self) -> Option<server::H2Options> {\n        None\n    }\n\n    fn http_cleanup(&self) {}\n}\n\n#[cfg_attr(not(doc_async_trait), async_trait)]\nimpl<T> ServerApp for T\nwhere\n    T: HttpServerApp + Send + Sync + 'static,\n{\n    async fn process_new(\n        self: &Arc<Self>,\n        stream: Stream,\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream> {\n        match stream.selected_alpn_proto() {\n            Some(ALPN::H2) => {\n                // create a shared connection digest\n                let digest = Arc::new(Digest {\n                    ssl_digest: stream.get_ssl_digest(),\n                    // TODO: log h2 handshake time\n                    timing_digest: stream.get_timing_digest(),\n                    proxy_digest: stream.get_proxy_digest(),\n                    socket_digest: stream.get_socket_digest(),\n                });\n\n                let h2_options = self.h2_options();\n                let h2_conn = server::handshake(stream, h2_options).await;\n                let mut h2_conn = match h2_conn {\n                    Err(e) => {\n                        error!(\"H2 handshake error {e}\");\n                        return None;\n                    }\n                    Ok(c) => c,\n                };\n\n                loop {\n                    // this loop ends when the client decides to close the h2 conn\n                    // TODO: add a timeout?\n                    let h2_stream =\n                        server::HttpSession::from_h2_conn(&mut h2_conn, digest.clone()).await;\n                    let h2_stream = match h2_stream {\n                        Err(e) => {\n                            // It is common for the client to just disconnect TCP without properly\n                            // closing H2. So we don't log the errors here\n                            debug!(\"H2 error when accepting new stream {e}\");\n                            return None;\n                        }\n                        Ok(s) => s?, // None means the connection is ready to be closed\n                    };\n                    let app = self.clone();\n                    let shutdown = shutdown.clone();\n                    pingora_runtime::current_handle().spawn(async move {\n                        app.process_new_http(ServerSession::new_http2(h2_stream), &shutdown)\n                            .await;\n                    });\n                }\n            }\n            _ => {\n                // No ALPN or ALPN::H1 or something else, just try Http1\n                self.process_new_http(ServerSession::new_http1(stream), shutdown)\n                    .await\n            }\n        }\n    }\n\n    fn cleanup(&self) {\n        self.http_cleanup()\n    }\n}\n\n// Path: pingora-core/src/listeners/tls.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::debug;\nuse pingora_error::{ErrorType, OrErr, Result};\nuse std::ops::{Deref, DerefMut};\n\nuse crate::protocols::ssl::{\n    server::{handshake, handshake_with_callback, TlsAcceptCallbacks},\n    SslStream,\n};\nuse crate::protocols::IO;\nuse crate::tls::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod};\n\npub use crate::protocols::ssl::ALPN;\n\npub const TLS_CONF_ERR: ErrorType = ErrorType::Custom(\"TLSConfigError\");\n\npub(crate) struct Acceptor {\n    ssl_acceptor: SslAcceptor,\n    callbacks: Option<TlsAcceptCallbacks>,\n}\n\n/// The TLS settings of a listening endpoint\npub struct TlsSettings {\n    accept_builder: SslAcceptorBuilder,\n    callbacks: Option<TlsAcceptCallbacks>,\n}\n\nimpl Deref for TlsSettings {\n    type Target = SslAcceptorBuilder;\n\n    fn deref(&self) -> &Self::Target {\n        &self.accept_builder\n    }\n}\n\nimpl DerefMut for TlsSettings {\n    fn deref_mut(&mut self) -> &mut Self::Target {\n        &mut self.accept_builder\n    }\n}\n\nimpl TlsSettings {\n    /// Create a new [`TlsSettings`] with the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)\n    /// server side TLS settings. Users can adjust the TLS settings after this object is created.\n    /// Return error if the provided certificate and private key are invalid or not found.\n    pub fn intermediate(cert_path: &str, key_path: &str) -> Result<Self> {\n        let mut accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err(\n            TLS_CONF_ERR,\n            \"fail to create mozilla_intermediate_v5 Acceptor\",\n        )?;\n        accept_builder\n            .set_private_key_file(key_path, SslFiletype::PEM)\n            .or_err(TLS_CONF_ERR, \"fail to read key file {key_path}\")?;\n        accept_builder\n            .set_certificate_chain_file(cert_path)\n            .or_err(TLS_CONF_ERR, \"fail to read cert file {cert_path}\")?;\n        Ok(TlsSettings {\n            accept_builder,\n            callbacks: None,\n        })\n    }\n\n    /// Create a new [`TlsSettings`] similar to [TlsSettings::intermediate()]. A struct that implements [TlsAcceptCallbacks]\n    /// is needed to provide the certificate during the TLS handshake.\n    pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result<Self> {\n        let accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err(\n            TLS_CONF_ERR,\n            \"fail to create mozilla_intermediate_v5 Acceptor\",\n        )?;\n        Ok(TlsSettings {\n            accept_builder,\n            callbacks: Some(callbacks),\n        })\n    }\n\n    /// Enable HTTP/2 support for this endpoint, which is default off.\n    /// This effectively sets the ALPN to prefer HTTP/2 with HTTP/1.1 allowed\n    pub fn enable_h2(&mut self) {\n        self.set_alpn(ALPN::H2H1);\n    }\n\n    /// Set the ALPN preference of this endpoint. See [`ALPN`] for more details\n    pub fn set_alpn(&mut self, alpn: ALPN) {\n        match alpn {\n            ALPN::H2H1 => self\n                .accept_builder\n                .set_alpn_select_callback(alpn::prefer_h2),\n            ALPN::H1 => self.accept_builder.set_alpn_select_callback(alpn::h1_only),\n            ALPN::H2 => self.accept_builder.set_alpn_select_callback(alpn::h2_only),\n        }\n    }\n\n    pub(crate) fn build(self) -> Acceptor {\n        Acceptor {\n            ssl_acceptor: self.accept_builder.build(),\n            callbacks: self.callbacks,\n        }\n    }\n}\n\nimpl Acceptor {\n    pub async fn tls_handshake<S: IO>(&self, stream: S) -> Result<SslStream<S>> {\n        debug!(\"new ssl session\");\n        // TODO: be able to offload this handshake in a thread pool\n        if let Some(cb) = self.callbacks.as_ref() {\n            handshake_with_callback(&self.ssl_acceptor, stream, cb).await\n        } else {\n            handshake(&self.ssl_acceptor, stream).await\n        }\n    }\n}\n\nmod alpn {\n    use super::*;\n    use crate::tls::ssl::{select_next_proto, AlpnError, SslRef};\n\n    // A standard implementation provided by the SSL lib is used below\n\n    pub fn prefer_h2<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> {\n        match select_next_proto(ALPN::H2H1.to_wire_preference(), alpn_in) {\n            Some(p) => Ok(p),\n            _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1\n        }\n    }\n\n    pub fn h1_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> {\n        match select_next_proto(ALPN::H1.to_wire_preference(), alpn_in) {\n            Some(p) => Ok(p),\n            _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1\n        }\n    }\n\n    pub fn h2_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> {\n        match select_next_proto(ALPN::H2.to_wire_preference(), alpn_in) {\n            Some(p) => Ok(p),\n            _ => Err(AlpnError::ALERT_FATAL), // cannot agree\n        }\n    }\n}\n\n// Path: pingora-core/src/listeners/l4.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::warn;\nuse pingora_error::{\n    ErrorType::{AcceptError, BindError},\n    OrErr, Result,\n};\nuse std::fs::Permissions;\nuse std::io::ErrorKind;\nuse std::net::{SocketAddr, ToSocketAddrs};\nuse std::os::unix::io::{AsRawFd, FromRawFd};\nuse std::os::unix::net::UnixListener as StdUnixListener;\nuse std::time::Duration;\nuse tokio::net::TcpSocket;\n\nuse crate::protocols::l4::listener::Listener;\npub use crate::protocols::l4::stream::Stream;\nuse crate::server::ListenFds;\n\nconst TCP_LISTENER_MAX_TRY: usize = 30;\nconst TCP_LISTENER_TRY_STEP: Duration = Duration::from_secs(1);\n// TODO: configurable backlog\nconst LISTENER_BACKLOG: u32 = 65535;\n\n/// Address for listening server, either TCP/UDS socket.\n#[derive(Clone, Debug)]\npub enum ServerAddress {\n    Tcp(String, Option<TcpSocketOptions>),\n    Uds(String, Option<Permissions>),\n}\n\nimpl AsRef<str> for ServerAddress {\n    fn as_ref(&self) -> &str {\n        match &self {\n            Self::Tcp(l, _) => l,\n            Self::Uds(l, _) => l,\n        }\n    }\n}\n\n/// TCP socket configuration options.\n#[derive(Clone, Debug)]\npub struct TcpSocketOptions {\n    /// IPV6_V6ONLY flag (if true, limit socket to IPv6 communication only).\n    /// This is mostly useful when binding to `[::]`, which on most Unix distributions\n    /// will bind to both IPv4 and IPv6 addresses by default.\n    pub ipv6_only: bool,\n    // TODO: allow configuring reuseaddr, backlog, etc. from here?\n}\n\nmod uds {\n    use super::{OrErr, Result};\n    use crate::protocols::l4::listener::Listener;\n    use log::{debug, error};\n    use pingora_error::ErrorType::BindError;\n    use std::fs::{self, Permissions};\n    use std::io::ErrorKind;\n    use std::os::unix::fs::PermissionsExt;\n    use std::os::unix::net::UnixListener as StdUnixListener;\n    use tokio::net::UnixListener;\n\n    use super::LISTENER_BACKLOG;\n\n    pub(super) fn set_perms(path: &str, perms: Option<Permissions>) -> Result<()> {\n        // set read/write permissions for all users on the socket by default\n        let perms = perms.unwrap_or(Permissions::from_mode(0o666));\n        fs::set_permissions(path, perms).or_err_with(BindError, || {\n            format!(\"Fail to bind to {path}, could not set permissions\")\n        })\n    }\n\n    pub(super) fn set_backlog(l: StdUnixListener, backlog: u32) -> Result<UnixListener> {\n        let socket: socket2::Socket = l.into();\n        // Note that we call listen on an already listening socket\n        // POSIX undefined but on Linux it will update the backlog size\n        socket\n            .listen(backlog as i32)\n            .or_err_with(BindError, || format!(\"listen() failed on {socket:?}\"))?;\n        UnixListener::from_std(socket.into()).or_err(BindError, \"Failed to convert to tokio socket\")\n    }\n\n    pub(super) fn bind(addr: &str, perms: Option<Permissions>) -> Result<Listener> {\n        /*\n          We remove the filename/address in case there is a dangling reference.\n\n          \"Binding to a socket with a filename creates a socket in the\n          filesystem that must be deleted by the caller when it is no\n          longer needed (using unlink(2))\"\n        */\n        match std::fs::remove_file(addr) {\n            Ok(()) => {\n                debug!(\"unlink {addr} done\");\n            }\n            Err(e) => match e.kind() {\n                ErrorKind::NotFound => debug!(\"unlink {addr} not found: {e}\"),\n                _ => error!(\"unlink {addr} failed: {e}\"),\n            },\n        }\n        let listener_socket = UnixListener::bind(addr)\n            .or_err_with(BindError, || format!(\"Bind() failed on {addr}\"))?;\n        set_perms(addr, perms)?;\n        let std_listener = listener_socket.into_std().unwrap();\n        Ok(set_backlog(std_listener, LISTENER_BACKLOG)?.into())\n    }\n}\n\n// currently, these options can only apply on sockets prior to calling bind()\nfn apply_tcp_socket_options(sock: &TcpSocket, opt: Option<&TcpSocketOptions>) -> Result<()> {\n    let Some(opt) = opt else {\n        return Ok(());\n    };\n    let socket_ref = socket2::SockRef::from(sock);\n    socket_ref\n        .set_only_v6(opt.ipv6_only)\n        .or_err(BindError, \"failed to set IPV6_V6ONLY\")\n}\n\nfn from_raw_fd(address: &ServerAddress, fd: i32) -> Result<Listener> {\n    match address {\n        ServerAddress::Uds(addr, perm) => {\n            let std_listener = unsafe { StdUnixListener::from_raw_fd(fd) };\n            // set permissions just in case\n            uds::set_perms(addr, perm.clone())?;\n            Ok(uds::set_backlog(std_listener, LISTENER_BACKLOG)?.into())\n        }\n        ServerAddress::Tcp(_, _) => {\n            let std_listener_socket = unsafe { std::net::TcpStream::from_raw_fd(fd) };\n            let listener_socket = TcpSocket::from_std_stream(std_listener_socket);\n            // Note that we call listen on an already listening socket\n            // POSIX undefined but on Linux it will update the backlog size\n            Ok(listener_socket\n                .listen(LISTENER_BACKLOG)\n                .or_err_with(BindError, || format!(\"Listen() failed on {address:?}\"))?\n                .into())\n        }\n    }\n}\n\nasync fn bind_tcp(addr: &str, opt: Option<TcpSocketOptions>) -> Result<Listener> {\n    let mut try_count = 0;\n    loop {\n        let sock_addr = addr\n            .to_socket_addrs() // NOTE: this could invoke a blocking network lookup\n            .or_err_with(BindError, || format!(\"Invalid listen address {addr}\"))?\n            .next() // take the first one for now\n            .unwrap(); // assume there is always at least one\n\n        let listener_socket = match sock_addr {\n            SocketAddr::V4(_) => TcpSocket::new_v4(),\n            SocketAddr::V6(_) => TcpSocket::new_v6(),\n        }\n        .or_err_with(BindError, || format!(\"fail to create address {sock_addr}\"))?;\n\n        // NOTE: this is to preserve the current TcpListener::bind() behavior.\n        // We have a few tests relying on this behavior to allow multiple identical\n        // test servers to coexist.\n        listener_socket\n            .set_reuseaddr(true)\n            .or_err(BindError, \"fail to set_reuseaddr(true)\")?;\n\n        apply_tcp_socket_options(&listener_socket, opt.as_ref())?;\n\n        match listener_socket.bind(sock_addr) {\n            Ok(()) => {\n                break Ok(listener_socket\n                    .listen(LISTENER_BACKLOG)\n                    .or_err(BindError, \"bind() failed\")?\n                    .into())\n            }\n            Err(e) => {\n                if e.kind() != ErrorKind::AddrInUse {\n                    break Err(e).or_err_with(BindError, || format!(\"bind() failed on {addr}\"));\n                }\n                try_count += 1;\n                if try_count >= TCP_LISTENER_MAX_TRY {\n                    break Err(e).or_err_with(BindError, || {\n                        format!(\"bind() failed, after retries, {addr} still in use\")\n                    });\n                }\n                warn!(\"{addr} is in use, will try again\");\n                tokio::time::sleep(TCP_LISTENER_TRY_STEP).await;\n            }\n        }\n    }\n}\n\nasync fn bind(addr: &ServerAddress) -> Result<Listener> {\n    match addr {\n        ServerAddress::Uds(l, perm) => uds::bind(l, perm.clone()),\n        ServerAddress::Tcp(l, opt) => bind_tcp(l, opt.clone()).await,\n    }\n}\n\npub struct ListenerEndpoint {\n    listen_addr: ServerAddress,\n    listener: Option<Listener>,\n}\n\nimpl ListenerEndpoint {\n    pub fn new(listen_addr: ServerAddress) -> Self {\n        ListenerEndpoint {\n            listen_addr,\n            listener: None,\n        }\n    }\n\n    pub fn as_str(&self) -> &str {\n        self.listen_addr.as_ref()\n    }\n\n    pub async fn listen(&mut self, fds: Option<ListenFds>) -> Result<()> {\n        if self.listener.is_some() {\n            return Ok(());\n        }\n\n        let listener = if let Some(fds_table) = fds {\n            let addr = self.listen_addr.as_ref();\n            // consider make this mutex std::sync::Mutex or OnceCell\n            let mut table = fds_table.lock().await;\n            if let Some(fd) = table.get(addr.as_ref()) {\n                from_raw_fd(&self.listen_addr, *fd)?\n            } else {\n                // not found\n                let listener = bind(&self.listen_addr).await?;\n                table.add(addr.to_string(), listener.as_raw_fd());\n                listener\n            }\n        } else {\n            // not found, no fd table\n            bind(&self.listen_addr).await?\n        };\n        self.listener = Some(listener);\n        Ok(())\n    }\n\n    pub async fn accept(&mut self) -> Result<Stream> {\n        let Some(listener) = self.listener.as_mut() else {\n            // panic otherwise this thing dead loop\n            panic!(\"Need to call listen() first\");\n        };\n        let mut stream = listener\n            .accept()\n            .await\n            .or_err(AcceptError, \"Fail to accept()\")?;\n        stream.set_nodelay()?;\n        Ok(stream)\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[tokio::test]\n    async fn test_listen_tcp() {\n        let addr = \"127.0.0.1:7100\";\n        let mut listener = ListenerEndpoint::new(ServerAddress::Tcp(addr.into(), None));\n        listener.listen(None).await.unwrap();\n        tokio::spawn(async move {\n            // just try to accept once\n            listener.accept().await.unwrap();\n        });\n        tokio::net::TcpStream::connect(addr)\n            .await\n            .expect(\"can connect to TCP listener\");\n    }\n\n    #[tokio::test]\n    async fn test_listen_tcp_ipv6_only() {\n        let sock_opt = Some(TcpSocketOptions { ipv6_only: true });\n        let mut listener = ListenerEndpoint::new(ServerAddress::Tcp(\"[::]:7101\".into(), sock_opt));\n        listener.listen(None).await.unwrap();\n        tokio::spawn(async move {\n            // just try to accept twice\n            listener.accept().await.unwrap();\n            listener.accept().await.unwrap();\n        });\n        tokio::net::TcpStream::connect(\"127.0.0.1:7101\")\n            .await\n            .expect_err(\"cannot connect to v4 addr\");\n        tokio::net::TcpStream::connect(\"[::1]:7101\")\n            .await\n            .expect(\"can connect to v6 addr\");\n    }\n\n    #[tokio::test]\n    async fn test_listen_uds() {\n        let addr = \"/tmp/test_listen_uds\";\n        let mut listener = ListenerEndpoint::new(ServerAddress::Uds(addr.into(), None));\n        listener.listen(None).await.unwrap();\n        tokio::spawn(async move {\n            // just try to accept once\n            listener.accept().await.unwrap();\n        });\n        tokio::net::UnixStream::connect(addr)\n            .await\n            .expect(\"can connect to UDS listener\");\n    }\n}\n\n// Path: pingora-core/src/services/listening.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! The listening service\n//!\n//! A [Service] (listening service) responds to incoming requests on its endpoints.\n//! Each [Service] can be configured with custom application logic (e.g. an `HTTPProxy`) and one or\n//! more endpoints to listen to.\n\nuse crate::apps::ServerApp;\nuse crate::listeners::{Listeners, ServerAddress, TcpSocketOptions, TlsSettings, TransportStack};\nuse crate::protocols::Stream;\nuse crate::server::{ListenFds, ShutdownWatch};\nuse crate::services::Service as ServiceTrait;\n\nuse async_trait::async_trait;\nuse log::{debug, error, info};\nuse pingora_error::Result;\nuse pingora_runtime::current_handle;\nuse std::fs::Permissions;\nuse std::sync::Arc;\n\n/// The type of service that is associated with a list of listening endpoints and a particular application\npub struct Service<A> {\n    name: String,\n    listeners: Listeners,\n    app_logic: Arc<A>,\n    /// The number of preferred threads. `None` to follow global setting.\n    pub threads: Option<usize>,\n}\n\nimpl<A> Service<A> {\n    /// Create a new [`Service`] with the given application (see [`crate::apps`]).\n    pub fn new(name: String, app_logic: Arc<A>) -> Self {\n        Service {\n            name,\n            listeners: Listeners::new(),\n            app_logic,\n            threads: None,\n        }\n    }\n\n    /// Create a new [`Service`] with the given application (see [`crate::apps`]) and the given\n    /// [`Listeners`].\n    pub fn with_listeners(name: String, listeners: Listeners, app_logic: Arc<A>) -> Self {\n        Service {\n            name,\n            listeners,\n            app_logic,\n            threads: None,\n        }\n    }\n\n    /// Get the [`Listeners`], mostly to add more endpoints.\n    pub fn endpoints(&mut self) -> &mut Listeners {\n        &mut self.listeners\n    }\n\n    // the follow add* function has no effect if the server is already started\n\n    /// Add a TCP listening endpoint with the given address (e.g., `127.0.0.1:8000`).\n    pub fn add_tcp(&mut self, addr: &str) {\n        self.listeners.add_tcp(addr);\n    }\n\n    /// Add a TCP listening endpoint with the given [`TcpSocketOptions`].\n    pub fn add_tcp_with_settings(&mut self, addr: &str, sock_opt: TcpSocketOptions) {\n        self.listeners.add_tcp_with_settings(addr, sock_opt);\n    }\n\n    /// Add a Unix domain socket listening endpoint with the given path.\n    ///\n    /// Optionally take a permission of the socket file. The default is read and write access for\n    /// everyone (0o666).\n    pub fn add_uds(&mut self, addr: &str, perm: Option<Permissions>) {\n        self.listeners.add_uds(addr, perm);\n    }\n\n    /// Add a TLS listening endpoint with the given certificate and key paths.\n    pub fn add_tls(&mut self, addr: &str, cert_path: &str, key_path: &str) -> Result<()> {\n        self.listeners.add_tls(addr, cert_path, key_path)\n    }\n\n    /// Add a TLS listening endpoint with the given [`TlsSettings`] and [`TcpSocketOptions`].\n    pub fn add_tls_with_settings(\n        &mut self,\n        addr: &str,\n        sock_opt: Option<TcpSocketOptions>,\n        settings: TlsSettings,\n    ) {\n        self.listeners\n            .add_tls_with_settings(addr, sock_opt, settings)\n    }\n\n    /// Add an endpoint according to the given [`ServerAddress`]\n    pub fn add_address(&mut self, addr: ServerAddress) {\n        self.listeners.add_address(addr);\n    }\n}\n\nimpl<A: ServerApp + Send + Sync + 'static> Service<A> {\n    pub async fn handle_event(event: Stream, app_logic: Arc<A>, shutdown: ShutdownWatch) {\n        debug!(\"new event!\");\n        let mut reuse_event = app_logic.process_new(event, &shutdown).await;\n        while let Some(event) = reuse_event {\n            // TODO: with no steal runtime, consider spawn() the next event on\n            // another thread for more evenly load balancing\n            debug!(\"new reusable event!\");\n            reuse_event = app_logic.process_new(event, &shutdown).await;\n        }\n    }\n\n    async fn run_endpoint(\n        app_logic: Arc<A>,\n        mut stack: TransportStack,\n        mut shutdown: ShutdownWatch,\n    ) {\n        if let Err(e) = stack.listen().await {\n            error!(\"Listen() failed: {e}\");\n            return;\n        }\n\n        // the accept loop, until the system is shutting down\n        loop {\n            let new_io = tokio::select! { // TODO: consider biased for perf reason?\n                new_io = stack.accept() => new_io,\n                shutdown_signal = shutdown.changed() => {\n                    match shutdown_signal {\n                        Ok(()) => {\n                            if !*shutdown.borrow() {\n                                // happen in the initial read\n                                continue;\n                            }\n                            info!(\"Shutting down {}\", stack.as_str());\n                            break;\n                        }\n                        Err(e) => {\n                            error!(\"shutdown_signal error {e}\");\n                            break;\n                        }\n                    }\n                }\n            };\n            match new_io {\n                Ok(io) => {\n                    let app = app_logic.clone();\n                    let shutdown = shutdown.clone();\n                    current_handle().spawn(async move {\n                        match io.handshake().await {\n                            Ok(io) => Self::handle_event(io, app, shutdown).await,\n                            Err(e) => {\n                                // TODO: Maybe IOApp trait needs a fn to handle/filter our this error\n                                error!(\"Downstream handshake error {e}\");\n                            }\n                        }\n                    });\n                }\n                Err(e) => {\n                    error!(\"Accept() failed {e}\");\n                    if let Some(io_error) = e\n                        .root_cause()\n                        .downcast_ref::<std::io::Error>()\n                        .and_then(|e| e.raw_os_error())\n                    {\n                        // 24: too many open files. In this case accept() will continue return this\n                        // error without blocking, which could use up all the resources\n                        if io_error == 24 {\n                            // call sleep to calm the thread down and wait for others to release\n                            // some resources\n                            tokio::time::sleep(std::time::Duration::from_secs(1)).await;\n                        }\n                    }\n                }\n            }\n        }\n\n        stack.cleanup();\n    }\n}\n\n#[async_trait]\nimpl<A: ServerApp + Send + Sync + 'static> ServiceTrait for Service<A> {\n    async fn start_service(&mut self, fds: Option<ListenFds>, shutdown: ShutdownWatch) {\n        let runtime = current_handle();\n        let endpoints = self.listeners.build(fds);\n\n        let handlers = endpoints.into_iter().map(|endpoint| {\n            let app_logic = self.app_logic.clone();\n            let shutdown = shutdown.clone();\n            runtime.spawn(async move {\n                Self::run_endpoint(app_logic, endpoint, shutdown).await;\n            })\n        });\n\n        futures::future::join_all(handlers).await;\n        self.listeners.cleanup();\n        self.app_logic.cleanup();\n    }\n\n    fn name(&self) -> &str {\n        &self.name\n    }\n\n    fn threads(&self) -> Option<usize> {\n        self.threads\n    }\n}\n\nuse crate::apps::prometheus_http_app::PrometheusServer;\n\nimpl Service<PrometheusServer> {\n    /// The Prometheus HTTP server\n    ///\n    /// The HTTP server endpoint that reports Prometheus metrics collected in the entire service\n    pub fn prometheus_http_service() -> Self {\n        Service::new(\n            \"Prometheus metric HTTP\".to_string(),\n            Arc::new(PrometheusServer::new()),\n        )\n    }\n}\n\n// Path: pingora-core/src/services/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! The service interface\n//!\n//! A service to the pingora server is just something runs forever until the server is shutting\n//! down.\n//!\n//! Two types of services are particularly useful\n//! - services that are listening to some (TCP) endpoints\n//! - services that are just running in the background.\n\nuse async_trait::async_trait;\n\nuse crate::server::{ListenFds, ShutdownWatch};\n\npub mod background;\npub mod listening;\n\n/// The service interface\n#[async_trait]\npub trait Service: Sync + Send {\n    /// This function will be called when the server is ready to start the service.\n    ///\n    /// - `fds`: a collection of listening file descriptors. During zero downtime restart\n    /// the `fds` would contain the listening sockets passed from the old service, services should\n    /// take the sockets they need to use then. If the sockets the service looks for don't appear in\n    /// the collection, the service should create its own listening sockets and then put them into\n    /// the collection in order for them to be passed to the next server.\n    /// - `shutdown`: the shutdown signal this server would receive.\n    async fn start_service(&mut self, fds: Option<ListenFds>, mut shutdown: ShutdownWatch);\n\n    /// The name of the service, just for logging and naming the threads assigned to this service\n    ///\n    /// Note that due to the limit of the underlying system, only the first 16 chars will be used\n    fn name(&self) -> &str;\n\n    /// The preferred number of threads to run this service\n    ///\n    /// If `None`, the global setting will be used\n    fn threads(&self) -> Option<usize> {\n        None\n    }\n}\n\n// Path: pingora-core/src/server/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Server process and configuration management\n\npub mod configuration;\nmod daemon;\npub(crate) mod transfer_fd;\n\nuse daemon::daemonize;\nuse log::{debug, error, info, warn};\nuse pingora_runtime::Runtime;\nuse pingora_timeout::fast_timeout;\nuse std::sync::Arc;\nuse std::thread;\nuse tokio::signal::unix;\nuse tokio::sync::{watch, Mutex};\nuse tokio::time::{sleep, Duration};\n\nuse crate::services::Service;\nuse configuration::{Opt, ServerConf};\npub use transfer_fd::Fds;\n\nuse pingora_error::{Error, ErrorType, Result};\n\n/* Time to wait before exiting the program.\nThis is the graceful period for all existing sessions to finish */\nconst EXIT_TIMEOUT: u64 = 60 * 5;\n/* Time to wait before shutting down listening sockets.\nThis is the graceful period for the new service to get ready */\nconst CLOSE_TIMEOUT: u64 = 5;\n\nenum ShutdownType {\n    Graceful,\n    Quick,\n}\n\n/// The receiver for server's shutdown event. The value will turn to true once the server starts\n/// to shutdown\npub type ShutdownWatch = watch::Receiver<bool>;\npub type ListenFds = Arc<Mutex<Fds>>;\n\n/// The server object\n///\n/// This object represents an entire pingora server process which may have multiple independent\n/// services (see [crate::services]). The server object handles signals, reading configuration,\n/// zero downtime upgrade and error reporting.\npub struct Server {\n    services: Vec<Box<dyn Service>>,\n    listen_fds: Option<ListenFds>,\n    shutdown_watch: watch::Sender<bool>,\n    // TODO: we many want to drop this copy to let sender call closed()\n    shutdown_recv: ShutdownWatch,\n    /// the parsed server configuration\n    pub configuration: Arc<ServerConf>,\n    /// the parser command line options\n    pub options: Option<Opt>,\n    /// the Sentry DSN\n    ///\n    /// Panics and other events sentry captures will send to this DSN **only in release mode**\n    pub sentry: Option<String>,\n}\n\n// TODO: delete the pid when exit\n\nimpl Server {\n    async fn main_loop(&self) -> ShutdownType {\n        // waiting for exit signal\n        // TODO: there should be a signal handling function\n        let mut graceful_upgrade_signal = unix::signal(unix::SignalKind::quit()).unwrap();\n        let mut graceful_terminate_signal = unix::signal(unix::SignalKind::terminate()).unwrap();\n        let mut fast_shutdown_signal = unix::signal(unix::SignalKind::interrupt()).unwrap();\n        tokio::select! {\n            _ = fast_shutdown_signal.recv() => {\n                info!(\"SIGINT received, exiting\");\n                ShutdownType::Quick\n            },\n            _ = graceful_terminate_signal.recv() => {\n                // we receive a graceful terminate, all instances are instructed to stop\n                info!(\"SIGTERM received, gracefully exiting\");\n                // graceful shutdown if there are listening sockets\n                info!(\"Broadcasting graceful shutdown\");\n                match self.shutdown_watch.send(true) {\n                    Ok(_) => { info!(\"Graceful shutdown started!\"); }\n                    Err(e) => {\n                        error!(\"Graceful shutdown broadcast failed: {e}\");\n                    }\n                }\n                info!(\"Broadcast graceful shutdown complete\");\n                ShutdownType::Graceful\n            }\n            _ = graceful_upgrade_signal.recv() => {\n                // TODO: still need to select! on signals in case a fast shutdown is needed\n                // aka: move below to another task and only kick it off here\n                info!(\"SIGQUIT received, sending socks and gracefully exiting\");\n                if let Some(fds) = &self.listen_fds {\n                    let fds = fds.lock().await;\n                    info!(\"Trying to send socks\");\n                    // XXX: this is blocking IO\n                    match fds.send_to_sock(\n                        self.configuration.as_ref().upgrade_sock.as_str())\n                    {\n                        Ok(_) => {info!(\"listener sockets sent\");},\n                        Err(e) => {\n                            error!(\"Unable to send listener sockets to new process: {e}\");\n                            // sentry log error on fd send failure\n                            #[cfg(not(debug_assertions))]\n                            sentry::capture_error(&e);\n                        }\n                    }\n                    sleep(Duration::from_secs(CLOSE_TIMEOUT)).await;\n                    info!(\"Broadcasting graceful shutdown\");\n                    // gracefully exiting\n                    match self.shutdown_watch.send(true) {\n                        Ok(_) => { info!(\"Graceful shutdown started!\"); }\n                        Err(e) => {\n                            error!(\"Graceful shutdown broadcast failed: {e}\");\n                            // switch to fast shutdown\n                            return ShutdownType::Graceful;\n                        }\n                    }\n                    info!(\"Broadcast graceful shutdown complete\");\n                    ShutdownType::Graceful\n                } else {\n                    info!(\"No socks to send, shutting down.\");\n                    ShutdownType::Graceful\n                }\n            },\n        }\n    }\n\n    fn run_service(\n        mut service: Box<dyn Service>,\n        fds: Option<ListenFds>,\n        shutdown: ShutdownWatch,\n        threads: usize,\n        work_stealing: bool,\n    ) -> Runtime\n// NOTE: we need to keep the runtime outside async since\n        // otherwise the runtime will be dropped.\n    {\n        let service_runtime = Server::create_runtime(service.name(), threads, work_stealing);\n        service_runtime.get_handle().spawn(async move {\n            service.start_service(fds, shutdown).await;\n            info!(\"service exited.\")\n        });\n        service_runtime\n    }\n\n    fn load_fds(&mut self, upgrade: bool) -> Result<(), nix::Error> {\n        let mut fds = Fds::new();\n        if upgrade {\n            debug!(\"Trying to receive socks\");\n            fds.get_from_sock(self.configuration.as_ref().upgrade_sock.as_str())?\n        }\n        self.listen_fds = Some(Arc::new(Mutex::new(fds)));\n        Ok(())\n    }\n\n    /// Create a new [`Server`], using the [`Opt`] and [`ServerConf`] values provided\n    ///\n    /// This method is intended for pingora frontends that are NOT using the built-in\n    /// command line and configuration file parsing, and are instead using their own.\n    ///\n    /// If a configuration file path is provided as part of `opt`, it will be ignored\n    /// and a warning will be logged.\n    pub fn new_with_opt_and_conf(opt: Opt, mut conf: ServerConf) -> Server {\n        if let Some(c) = opt.conf.as_ref() {\n            warn!(\"Ignoring command line argument using '{c}' as configuration, and using provided configuration instead.\");\n        }\n        conf.merge_with_opt(&opt);\n\n        let (tx, rx) = watch::channel(false);\n\n        Server {\n            services: vec![],\n            listen_fds: None,\n            shutdown_watch: tx,\n            shutdown_recv: rx,\n            configuration: Arc::new(conf),\n            options: Some(opt),\n            sentry: None,\n        }\n    }\n\n    /// Create a new [`Server`].\n    ///\n    /// Only one [`Server`] needs to be created for a process. A [`Server`] can hold multiple\n    /// independent services.\n    ///\n    /// Command line options can either be passed by parsing the command line arguments via\n    /// `Opt::from_args()`, or be generated by other means.\n    pub fn new(opt: impl Into<Option<Opt>>) -> Result<Server> {\n        let opt = opt.into();\n        let (tx, rx) = watch::channel(false);\n\n        let conf = if let Some(opt) = opt.as_ref() {\n            opt.conf.as_ref().map_or_else(\n                || {\n                    // options, no conf, generated\n                    ServerConf::new_with_opt_override(opt).ok_or_else(|| {\n                        Error::explain(ErrorType::ReadError, \"Conf generation failed\")\n                    })\n                },\n                |_| {\n                    // options and conf loaded\n                    ServerConf::load_yaml_with_opt_override(opt)\n                },\n            )\n        } else {\n            ServerConf::new()\n                .ok_or_else(|| Error::explain(ErrorType::ReadError, \"Conf generation failed\"))\n        }?;\n\n        Ok(Server {\n            services: vec![],\n            listen_fds: None,\n            shutdown_watch: tx,\n            shutdown_recv: rx,\n            configuration: Arc::new(conf),\n            options: opt,\n            sentry: None,\n        })\n    }\n\n    /// Add a service to this server.\n    ///\n    /// A service is anything that implements [`Service`].\n    pub fn add_service(&mut self, service: impl Service + 'static) {\n        self.services.push(Box::new(service));\n    }\n\n    /// Similar to [`Self::add_service()`], but take a list of services\n    pub fn add_services(&mut self, services: Vec<Box<dyn Service>>) {\n        self.services.extend(services);\n    }\n\n    /// Prepare the server to start\n    ///\n    /// When trying to zero downtime upgrade from an older version of the server which is already\n    /// running, this function will try to get all its listening sockets in order to take them over.\n    pub fn bootstrap(&mut self) {\n        info!(\"Bootstrap starting\");\n        debug!(\"{:#?}\", self.options);\n\n        /* only init sentry in release builds */\n        #[cfg(not(debug_assertions))]\n        let _guard = match self.sentry.as_ref() {\n            Some(uri) => Some(sentry::init(uri.as_str())),\n            None => None,\n        };\n\n        if self.options.as_ref().map_or(false, |o| o.test) {\n            info!(\"Server Test passed, exiting\");\n            std::process::exit(0);\n        }\n\n        // load fds\n        match self.load_fds(self.options.as_ref().map_or(false, |o| o.upgrade)) {\n            Ok(_) => {\n                info!(\"Bootstrap done\");\n            }\n            Err(e) => {\n                // sentry log error on fd load failure\n                #[cfg(not(debug_assertions))]\n                sentry::capture_error(&e);\n\n                error!(\"Bootstrap failed on error: {:?}, exiting.\", e);\n                std::process::exit(1);\n            }\n        }\n    }\n\n    /// Start the server\n    ///\n    /// This function will block forever until the server needs to quit. So this would be the last\n    /// function to call for this object.\n    ///\n    /// Note: this function may fork the process for daemonization, so any additional threads created\n    /// before this function will be lost to any service logic once this function is called.\n    pub fn run_forever(&mut self) {\n        info!(\"Server starting\");\n\n        let conf = self.configuration.as_ref();\n\n        if conf.daemon {\n            info!(\"Daemonizing the server\");\n            fast_timeout::pause_for_fork();\n            daemonize(&self.configuration);\n            fast_timeout::unpause();\n        }\n\n        /* only init sentry in release builds */\n        #[cfg(not(debug_assertions))]\n        let _guard = match self.sentry.as_ref() {\n            Some(uri) => Some(sentry::init(uri.as_str())),\n            None => None,\n        };\n\n        let mut runtimes: Vec<Runtime> = Vec::new();\n\n        while let Some(service) = self.services.pop() {\n            let threads = service.threads().unwrap_or(conf.threads);\n            let runtime = Server::run_service(\n                service,\n                self.listen_fds.clone(),\n                self.shutdown_recv.clone(),\n                threads,\n                conf.work_stealing,\n            );\n            runtimes.push(runtime);\n        }\n\n        // blocked on main loop so that it runs forever\n        // Only work steal runtime can use block_on()\n        let server_runtime = Server::create_runtime(\"Server\", 1, true);\n        let shutdown_type = server_runtime.get_handle().block_on(self.main_loop());\n\n        if matches!(shutdown_type, ShutdownType::Graceful) {\n            let exit_timeout = self\n                .configuration\n                .as_ref()\n                .grace_period_seconds\n                .unwrap_or(EXIT_TIMEOUT);\n            info!(\"Graceful shutdown: grace period {}s starts\", exit_timeout);\n            thread::sleep(Duration::from_secs(exit_timeout));\n            info!(\"Graceful shutdown: grace period ends\");\n        }\n\n        // Give tokio runtimes time to exit\n        let shutdown_timeout = match shutdown_type {\n            ShutdownType::Quick => Duration::from_secs(0),\n            ShutdownType::Graceful => Duration::from_secs(\n                self.configuration\n                    .as_ref()\n                    .graceful_shutdown_timeout_seconds\n                    .unwrap_or(5),\n            ),\n        };\n        let shutdowns: Vec<_> = runtimes\n            .into_iter()\n            .map(|rt| {\n                info!(\"Waiting for runtimes to exit!\");\n                thread::spawn(move || {\n                    rt.shutdown_timeout(shutdown_timeout);\n                    thread::sleep(shutdown_timeout)\n                })\n            })\n            .collect();\n        for shutdown in shutdowns {\n            if let Err(e) = shutdown.join() {\n                error!(\"Failed to shutdown runtime: {:?}\", e);\n            }\n        }\n        info!(\"All runtimes exited, exiting now\");\n        std::process::exit(0);\n    }\n\n    fn create_runtime(name: &str, threads: usize, work_steal: bool) -> Runtime {\n        if work_steal {\n            Runtime::new_steal(threads, name)\n        } else {\n            Runtime::new_no_steal(threads, name)\n        }\n    }\n}\n\n// Path: pingora-core/src/listeners/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! The listening endpoints (TCP and TLS) and their configurations.\n\nmod l4;\nmod tls;\n\nuse crate::protocols::Stream;\nuse crate::server::ListenFds;\n\nuse pingora_error::Result;\nuse std::{fs::Permissions, sync::Arc};\n\nuse l4::{ListenerEndpoint, Stream as L4Stream};\nuse tls::Acceptor;\n\npub use crate::protocols::ssl::server::TlsAccept;\npub use l4::{ServerAddress, TcpSocketOptions};\npub use tls::{TlsSettings, ALPN};\n\nstruct TransportStackBuilder {\n    l4: ServerAddress,\n    tls: Option<TlsSettings>,\n}\n\nimpl TransportStackBuilder {\n    pub fn build(&mut self, upgrade_listeners: Option<ListenFds>) -> TransportStack {\n        TransportStack {\n            l4: ListenerEndpoint::new(self.l4.clone()),\n            tls: self.tls.take().map(|tls| Arc::new(tls.build())),\n            upgrade_listeners,\n        }\n    }\n}\n\npub(crate) struct TransportStack {\n    l4: ListenerEndpoint,\n    tls: Option<Arc<Acceptor>>,\n    // listeners sent from the old process for graceful upgrade\n    upgrade_listeners: Option<ListenFds>,\n}\n\nimpl TransportStack {\n    pub fn as_str(&self) -> &str {\n        self.l4.as_str()\n    }\n\n    pub async fn listen(&mut self) -> Result<()> {\n        self.l4.listen(self.upgrade_listeners.take()).await\n    }\n\n    pub async fn accept(&mut self) -> Result<UninitializedStream> {\n        let stream = self.l4.accept().await?;\n        Ok(UninitializedStream {\n            l4: stream,\n            tls: self.tls.clone(),\n        })\n    }\n\n    pub fn cleanup(&mut self) {\n        // placeholder\n    }\n}\n\npub(crate) struct UninitializedStream {\n    l4: L4Stream,\n    tls: Option<Arc<Acceptor>>,\n}\n\nimpl UninitializedStream {\n    pub async fn handshake(self) -> Result<Stream> {\n        if let Some(tls) = self.tls {\n            let tls_stream = tls.tls_handshake(self.l4).await?;\n            Ok(Box::new(tls_stream))\n        } else {\n            Ok(Box::new(self.l4))\n        }\n    }\n}\n\n/// The struct to hold one more multiple listening endpoints\npub struct Listeners {\n    stacks: Vec<TransportStackBuilder>,\n}\n\nimpl Listeners {\n    /// Create a new [`Listeners`] with no listening endpoints.\n    pub fn new() -> Self {\n        Listeners { stacks: vec![] }\n    }\n\n    /// Create a new [`Listeners`] with a TCP server endpoint from the given string.\n    pub fn tcp(addr: &str) -> Self {\n        let mut listeners = Self::new();\n        listeners.add_tcp(addr);\n        listeners\n    }\n\n    /// Create a new [`Listeners`] with a Unix domain socket endpoint from the given string.\n    pub fn uds(addr: &str, perm: Option<Permissions>) -> Self {\n        let mut listeners = Self::new();\n        listeners.add_uds(addr, perm);\n        listeners\n    }\n\n    /// Create a new [`Listeners`] with a TLS (TCP) endpoint with the given address string,\n    /// and path to the certificate/private key pairs.\n    /// This endpoint will adopt the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)\n    /// server side TLS settings.\n    pub fn tls(addr: &str, cert_path: &str, key_path: &str) -> Result<Self> {\n        let mut listeners = Self::new();\n        listeners.add_tls(addr, cert_path, key_path)?;\n        Ok(listeners)\n    }\n\n    /// Add a TCP endpoint to `self`.\n    pub fn add_tcp(&mut self, addr: &str) {\n        self.add_address(ServerAddress::Tcp(addr.into(), None));\n    }\n\n    /// Add a TCP endpoint to `self`, with the given [`TcpSocketOptions`].\n    pub fn add_tcp_with_settings(&mut self, addr: &str, sock_opt: TcpSocketOptions) {\n        self.add_address(ServerAddress::Tcp(addr.into(), Some(sock_opt)));\n    }\n\n    /// Add a Unix domain socket endpoint to `self`.\n    pub fn add_uds(&mut self, addr: &str, perm: Option<Permissions>) {\n        self.add_address(ServerAddress::Uds(addr.into(), perm));\n    }\n\n    /// Add a TLS endpoint to `self` with the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)\n    /// server side TLS settings.\n    pub fn add_tls(&mut self, addr: &str, cert_path: &str, key_path: &str) -> Result<()> {\n        self.add_tls_with_settings(addr, None, TlsSettings::intermediate(cert_path, key_path)?);\n        Ok(())\n    }\n\n    /// Add a TLS endpoint to `self` with the given socket and server side TLS settings.\n    /// See [`TlsSettings`] and [`TcpSocketOptions`] for more details.\n    pub fn add_tls_with_settings(\n        &mut self,\n        addr: &str,\n        sock_opt: Option<TcpSocketOptions>,\n        settings: TlsSettings,\n    ) {\n        self.add_endpoint(ServerAddress::Tcp(addr.into(), sock_opt), Some(settings));\n    }\n\n    /// Add the given [`ServerAddress`] to `self`.\n    pub fn add_address(&mut self, addr: ServerAddress) {\n        self.add_endpoint(addr, None);\n    }\n\n    /// Add the given [`ServerAddress`] to `self` with the given [`TlsSettings`] if provided\n    pub fn add_endpoint(&mut self, l4: ServerAddress, tls: Option<TlsSettings>) {\n        self.stacks.push(TransportStackBuilder { l4, tls })\n    }\n\n    pub(crate) fn build(&mut self, upgrade_listeners: Option<ListenFds>) -> Vec<TransportStack> {\n        self.stacks\n            .iter_mut()\n            .map(|b| b.build(upgrade_listeners.clone()))\n            .collect()\n    }\n\n    pub(crate) fn cleanup(&self) {\n        // placeholder\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use tokio::io::AsyncWriteExt;\n    use tokio::net::TcpStream;\n    use tokio::time::{sleep, Duration};\n\n    #[tokio::test]\n    async fn test_listen_tcp() {\n        let addr1 = \"127.0.0.1:7101\";\n        let addr2 = \"127.0.0.1:7102\";\n        let mut listeners = Listeners::tcp(addr1);\n        listeners.add_tcp(addr2);\n\n        let listeners = listeners.build(None);\n        assert_eq!(listeners.len(), 2);\n        for mut listener in listeners {\n            tokio::spawn(async move {\n                listener.listen().await.unwrap();\n                // just try to accept once\n                let stream = listener.accept().await.unwrap();\n                stream.handshake().await.unwrap();\n            });\n        }\n\n        // make sure the above starts before the lines below\n        sleep(Duration::from_millis(10)).await;\n\n        TcpStream::connect(addr1).await.unwrap();\n        TcpStream::connect(addr2).await.unwrap();\n    }\n\n    #[tokio::test]\n    async fn test_listen_tls() {\n        use tokio::io::AsyncReadExt;\n\n        let addr = \"127.0.0.1:7103\";\n        let cert_path = format!(\"{}/tests/keys/server.crt\", env!(\"CARGO_MANIFEST_DIR\"));\n        let key_path = format!(\"{}/tests/keys/key.pem\", env!(\"CARGO_MANIFEST_DIR\"));\n        let mut listeners = Listeners::tls(addr, &cert_path, &key_path).unwrap();\n        let mut listener = listeners.build(None).pop().unwrap();\n\n        tokio::spawn(async move {\n            listener.listen().await.unwrap();\n            // just try to accept once\n            let stream = listener.accept().await.unwrap();\n            let mut stream = stream.handshake().await.unwrap();\n            let mut buf = [0; 1024];\n            let _ = stream.read(&mut buf).await.unwrap();\n            stream\n                .write_all(b\"HTTP/1.1 200 OK\\r\\nContent-Length: 1\\r\\n\\r\\na\")\n                .await\n                .unwrap();\n        });\n...\n", "needle_token_start": 787, "needle_token_end": 811, "code_context_ntokens": 16197, "output": ["```rust\nfn h2_options(&self) -> Option<server::H2Options> {\n    None\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "do_read_body_until_closed", "language": "rust", "path": "pingora-core/src/protocols/http/v1/body.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To continuously read data from a stream until the stream is closed, specifically for HTTP 1.0 connections where the end of the body is determined by the closure of the connection.\n2. **Input**: A mutable reference to a stream that implements asynchronous reading.\n3. **Output**: A result containing an optional buffer reference indicating the portion of the buffer that has new data, or `None` if the stream is closed or an error occurs.\n4. **Procedure**: \n   - The function first checks if there is any data left from a previous operation that needs to be processed; if not, it attempts to read from the stream into a buffer.\n   - If data is successfully read, it updates the state to reflect the new total amount of data read and returns a reference to the new data.\n   - If no data is read (i.e., the stream is closed), it updates the state to indicate that reading is complete and returns `None`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "const PARTIAL_CHUNK_HEAD_LIMIT: usize = 1024 * 8;\n\nconst LAST_CHUNK: &[u8; 5] = b\"0\\r\\n\\r\\n\";\n\npub const INVALID_CHUNK: ErrorType = ErrorType::new(\"InvalidChunk\");\npub const PREMATURE_BODY_END: ErrorType = ErrorType::new(\"PrematureBodyEnd\");\n\n#[derive(Clone, Debug, PartialEq, Eq)]\npub enum ParseState {\n    ToStart,\n    Complete(usize),                     // total size\n    Partial(usize, usize),               // size read, remaining size\n    Chunked(usize, usize, usize, usize), // size read, next to read in current buf start, read in current buf start, remaining chucked size to read from IO\n    Done(usize),                         // done but there is error, size read\n    HTTP1_0(usize),                      // read until connection closed, size read\n}\n\ntype PS = ParseState;\n\nimpl ParseState {\n    pub fn finish(&self, additional_bytes: usize) -> Self {\n        match self {\n            PS::Partial(read, to_read) => PS::Complete(read + to_read),\n            PS::Chunked(read, _, _, _) => PS::Complete(read + additional_bytes),\n            PS::HTTP1_0(read) => PS::Complete(read + additional_bytes),\n            _ => self.clone(), /* invalid transaction */\n        }\n    }\n\n    pub fn done(&self, additional_bytes: usize) -> Self {\n        match self {\n            PS::Partial(read, _) => PS::Done(read + additional_bytes),\n            PS::Chunked(read, _, _, _) => PS::Done(read + additional_bytes),\n            PS::HTTP1_0(read) => PS::Done(read + additional_bytes),\n            _ => self.clone(), /* invalid transaction */\n        }\n    }\n\n    pub fn partial_chunk(&self, bytes_read: usize, bytes_to_read: usize) -> Self {\n        match self {\n            PS::Chunked(read, _, _, _) => PS::Chunked(read + bytes_read, 0, 0, bytes_to_read),\n            _ => self.clone(), /* invalid transaction */\n        }\n    }\n\n    pub fn multi_chunk(&self, bytes_read: usize, buf_start_index: usize) -> Self {\n        match self {\n            PS::Chunked(read, _, buf_end, _) => {\n                PS::Chunked(read + bytes_read, buf_start_index, *buf_end, 0)\n            }\n            _ => self.clone(), /* invalid transaction */\n        }\n    }\n\n    pub fn partial_chunk_head(&self, head_end: usize, head_size: usize) -> Self {\n        match self {\n            /* inform reader to read more to form a legal chunk */\n            PS::Chunked(read, _, _, _) => PS::Chunked(*read, 0, head_end, head_size),\n            _ => self.clone(), /* invalid transaction */\n        }\n    }\n\n    pub fn new_buf(&self, buf_end: usize) -> Self {\n        match self {\n            PS::Chunked(read, _, _, _) => PS::Chunked(*read, 0, buf_end, 0),\n            _ => self.clone(), /* invalid transaction */\n        }\n    }\n}\n\npub struct BodyReader {\n    pub body_state: ParseState,\n    pub body_buf: Option<BytesMut>,\n    pub body_buf_size: usize,\n    rewind_buf_len: usize,\n}\n\nimpl BodyReader {\n    pub fn new() -> Self {\n        BodyReader {\n            body_state: PS::ToStart,\n            body_buf: None,\n            body_buf_size: BODY_BUFFER_SIZE,\n            rewind_buf_len: 0,\n        }\n    }\n\n    pub fn need_init(&self) -> bool {\n        matches!(self.body_state, PS::ToStart)\n    }\n\n    pub fn reinit(&mut self) {\n        self.body_state = PS::ToStart;\n    }\n\n    fn prepare_buf(&mut self, buf_to_rewind: &[u8]) {\n        let mut body_buf = BytesMut::with_capacity(self.body_buf_size);\n        if !buf_to_rewind.is_empty() {\n            self.rewind_buf_len = buf_to_rewind.len();\n            // TODO: this is still 1 copy. Make it zero\n            body_buf.put_slice(buf_to_rewind);\n        }\n        if self.body_buf_size > buf_to_rewind.len() {\n            //body_buf.resize(self.body_buf_size, 0);\n            unsafe {\n                body_buf.set_len(self.body_buf_size);\n            }\n        }\n        self.body_buf = Some(body_buf);\n    }\n\n    pub fn init_chunked(&mut self, buf_to_rewind: &[u8]) {\n        self.body_state = PS::Chunked(0, 0, 0, 0);\n        self.prepare_buf(buf_to_rewind);\n    }\n\n    pub fn init_content_length(&mut self, cl: usize, buf_to_rewind: &[u8]) {\n        match cl {\n            0 => self.body_state = PS::Complete(0),\n            _ => {\n                self.prepare_buf(buf_to_rewind);\n                self.body_state = PS::Partial(0, cl);\n            }\n        }\n    }\n\n    pub fn init_http10(&mut self, buf_to_rewind: &[u8]) {\n        self.prepare_buf(buf_to_rewind);\n        self.body_state = PS::HTTP1_0(0);\n    }\n\n    pub fn get_body(&self, buf_ref: &BufRef) -> &[u8] {\n        // TODO: these get_*() could panic. handle them better\n        buf_ref.get(self.body_buf.as_ref().unwrap())\n    }\n\n    pub fn body_done(&self) -> bool {\n        matches!(self.body_state, PS::Complete(_) | PS::Done(_))\n    }\n\n    pub fn body_empty(&self) -> bool {\n        self.body_state == PS::Complete(0)\n    }\n\n    pub async fn read_body<S>(&mut self, stream: &mut S) -> Result<Option<BufRef>>\n    where\n        S: AsyncRead + Unpin + Send,\n    {\n        match self.body_state {\n            PS::Complete(_) => Ok(None),\n            PS::Done(_) => Ok(None),\n            PS::Partial(_, _) => self.do_read_body(stream).await,\n            PS::Chunked(_, _, _, _) => self.do_read_chunked_body(stream).await,\n            PS::HTTP1_0(_) => self.do_read_body_until_closed(stream).await,\n            PS::ToStart => panic!(\"need to init BodyReader first\"),\n        }\n    }\n\n    pub async fn do_read_body<S>(&mut self, stream: &mut S) -> Result<Option<BufRef>>\n    where\n        S: AsyncRead + Unpin + Send,\n    {\n        let body_buf = self.body_buf.as_deref_mut().unwrap();\n        let mut n = self.rewind_buf_len;\n        self.rewind_buf_len = 0; // we only need to read rewind data once\n        if n == 0 {\n            /* Need to actually read */\n            n = stream\n                .read(body_buf)\n                .await\n                .or_err(ReadError, \"when reading body\")?;\n        }\n        match self.body_state {\n            PS::Partial(read, to_read) => {\n                debug!(\n                    \"BodyReader body_state: {:?}, read data from IO: {n}\",\n                    self.body_state\n                );\n                if n == 0 {\n                    self.body_state = PS::Done(read);\n                    Error::e_explain(ConnectionClosed, format!(\n                        \"Peer prematurely closed connection with {} bytes of body remaining to read\",\n                        to_read\n                    ))\n                } else if n >= to_read {\n                    if n > to_read {\n                        warn!(\n                            \"Peer sent more data then expected: extra {}\\\n                               bytes, discarding them\",\n                            n - to_read\n                        )\n                    }\n                    self.body_state = PS::Complete(read + to_read);\n                    Ok(Some(BufRef::new(0, to_read)))\n                } else {\n                    self.body_state = PS::Partial(read + n, to_read - n);\n                    Ok(Some(BufRef::new(0, n)))\n                }\n            }\n            _ => panic!(\"wrong body state: {:?}\", self.body_state),\n        }\n    }\n\n    \npub async fn do_read_body_until_closed<S>(&mut self, stream: &mut S) -> Result<Option<BufRef>>\n    where\n        S: AsyncRead + Unpin + Send,\n    {\n        let body_buf = self.body_buf.as_deref_mut().unwrap();\n        let mut n = self.rewind_buf_len;\n        self.rewind_buf_len = 0; // we only need to read rewind data once\n        if n == 0 {\n            /* Need to actually read */\n            n = stream\n                .read(body_buf)\n                .await\n                .or_err(ReadError, \"when reading body\")?;\n        }\n        match self.body_state {\n            PS::HTTP1_0(read) => {\n                if n == 0 {\n                    self.body_state = PS::Complete(read);\n                    Ok(None)\n                } else {\n                    self.body_state = PS::HTTP1_0(read + n);\n                    Ok(Some(BufRef::new(0, n)))\n                }\n            }\n            _ => panic!(\"wrong body state: {:?}\", self.body_state),\n        }\n    }\n\n    pub async fn do_read_chunked_body<S>(&mut self, stream: &mut S) -> Result<Option<BufRef>>\n    where\n        S: AsyncRead + Unpin + Send,\n    {\n        match self.body_state {\n            PS::Chunked(\n                total_read,\n                existing_buf_start,\n                mut existing_buf_end,\n                mut expecting_from_io,\n            ) => {\n                if existing_buf_start == 0 {\n                    // read a new buf from IO\n                    let body_buf = self.body_buf.as_deref_mut().unwrap();\n                    if existing_buf_end == 0 {\n                        existing_buf_end = self.rewind_buf_len;\n                        self.rewind_buf_len = 0; // we only need to read rewind data once\n                        if existing_buf_end == 0 {\n                            existing_buf_end = stream\n                                .read(body_buf)\n                                .await\n                                .or_err(ReadError, \"when reading body\")?;\n                        }\n                    } else {\n                        /* existing_buf_end != 0 this is partial chunk head */\n                        /* copy the #expecting_from_io bytes until index existing_buf_end\n                         * to the front and read more to form a valid chunk head.\n                         * existing_buf_end is the end of the partial head and\n                         * expecting_from_io is the len of it */\n                        body_buf\n                            .copy_within(existing_buf_end - expecting_from_io..existing_buf_end, 0);\n                        let new_bytes = stream\n                            .read(&mut body_buf[expecting_from_io..])\n                            .await\n                            .or_err(ReadError, \"when reading body\")?;\n                        /* more data is read, extend the buffer */\n                        existing_buf_end = expecting_from_io + new_bytes;\n                        expecting_from_io = 0;\n                    }\n                    self.body_state = self.body_state.new_buf(existing_buf_end);\n                }\n                if existing_buf_end == 0 {\n                    self.body_state = self.body_state.done(0);\n                    Error::e_explain(\n                        ConnectionClosed,\n                        format!(\n                            \"Connection prematurely closed without the termination chunk, \\\n                            read {total_read} bytes\"\n                        ),\n                    )\n                } else {\n                    if expecting_from_io > 0 {\n                        trace!(\n                            \"partial chunk payload, expecting_from_io: {}, \\\n                                existing_buf_end {}, buf: {:?}\",\n                            expecting_from_io,\n                            existing_buf_end,\n                            String::from_utf8_lossy(\n                                &self.body_buf.as_ref().unwrap()[..existing_buf_end]\n                            )\n                        );\n                        // partial chunk payload, will read more\n                        if expecting_from_io >= existing_buf_end + 2 {\n                            // not enough\n                            self.body_state = self.body_state.partial_chunk(\n                                existing_buf_end,\n                                expecting_from_io - existing_buf_end,\n                            );\n                            return Ok(Some(BufRef::new(0, existing_buf_end)));\n                        }\n                        /* could be expecting DATA + CRLF or just CRLF */\n                        let payload_size = if expecting_from_io > 2 {\n                            expecting_from_io - 2\n                        } else {\n                            0\n                        };\n                        /* expecting_from_io < existing_buf_end + 2 */\n                        if expecting_from_io >= existing_buf_end {\n                            self.body_state = self\n                                .body_state\n                                .partial_chunk(payload_size, expecting_from_io - existing_buf_end);\n                            return Ok(Some(BufRef::new(0, payload_size)));\n                        }\n\n                        /* expecting_from_io < existing_buf_end */\n                        self.body_state =\n                            self.body_state.multi_chunk(payload_size, expecting_from_io);\n                        return Ok(Some(BufRef::new(0, payload_size)));\n                    }\n                    self.parse_chunked_buf(existing_buf_start, existing_buf_end)\n                }\n            }\n            _ => panic!(\"wrong body state: {:?}\", self.body_state),\n        }\n    }\n\n    fn parse_chunked_buf(\n        &mut self,\n        buf_index_start: usize,\n        buf_index_end: usize,\n    ) -> Result<Option<BufRef>> {\n        let buf = &self.body_buf.as_ref().unwrap()[buf_index_start..buf_index_end];\n        let chunk_status = httparse::parse_chunk_size(buf);\n        match chunk_status {\n            Ok(status) => {\n                match status {\n                    httparse::Status::Complete((payload_index, chunk_size)) => {\n                        // TODO: Check chunk_size overflow\n                        trace!(\n                            \"Got size {chunk_size}, payload_index: {payload_index}, chunk: {:?}\",\n                            String::from_utf8_lossy(buf)\n                        );\n                        let chunk_size = chunk_size as usize;\n                        if chunk_size == 0 {\n                            /* terminating chunk. TODO: trailer */\n                            self.body_state = self.body_state.finish(0);\n                            return Ok(None);\n                        }\n                        // chunk-size CRLF [payload_index] byte*[chunk_size] CRLF\n                        let data_end_index = payload_index + chunk_size;\n                        let chunk_end_index = data_end_index + 2;\n                        if chunk_end_index >= buf.len() {\n                            // no multi chunk in this buf\n                            let actual_size = if data_end_index > buf.len() {\n                                buf.len() - payload_index\n                            } else {\n                                chunk_size\n                            };\n                            self.body_state = self\n                                .body_state\n                                .partial_chunk(actual_size, chunk_end_index - buf.len());\n                            return Ok(Some(BufRef::new(\n                                buf_index_start + payload_index,\n                                actual_size,\n                            )));\n                        }\n                        /* got multiple chunks, return the first */\n                        self.body_state = self\n                            .body_state\n                            .multi_chunk(chunk_size, buf_index_start + chunk_end_index);\n                        Ok(Some(BufRef::new(\n                            buf_index_start + payload_index,\n                            chunk_size,\n                        )))\n                    }\n                    httparse::Status::Partial => {\n                        if buf.len() > PARTIAL_CHUNK_HEAD_LIMIT {\n                            // https://datatracker.ietf.org/doc/html/rfc9112#name-chunk-extensions\n                            // \"A server ought to limit the total length of chunk extensions received\"\n                            // The buf.len() here is the total length of chunk-size + chunk-ext seen\n                            // so far. This check applies to both server and client\n                            self.body_state = self.body_state.done(0);\n                            Error::e_explain(INVALID_CHUNK, \"Chunk ext over limit\")\n                        } else {\n                            self.body_state =\n                                self.body_state.partial_chunk_head(buf_index_end, buf.len());\n                            Ok(Some(BufRef::new(0, 0)))\n                        }\n                    }\n                }\n            }\n            Err(e) => {\n                let context = format!(\"Invalid chucked encoding: {e:?}\");\n                debug!(\"{context}, {:?}\", String::from_utf8_lossy(buf));\n                self.body_state = self.body_state.done(0);\n                Error::e_explain(INVALID_CHUNK, context)\n            }\n        }\n    }\n}\n\n#[derive(Clone, Debug, PartialEq, Eq)]\npub enum BodyMode {\n    ToSelect,\n    ContentLength(usize, usize), // total length to write, bytes already written\n    ChunkedEncoding(usize),      //bytes written\n    HTTP1_0(usize),              //bytes written\n    Complete(usize),             //bytes written\n}\n\ntype BM = BodyMode;\n\npub struct BodyWriter {\n    pub body_mode: BodyMode,\n}\n\nimpl BodyWriter {\n    pub fn new() -> Self {\n        BodyWriter {\n            body_mode: BM::ToSelect,\n        }\n    }\n\n    pub fn init_chunked(&mut self) {\n        self.body_mode = BM::ChunkedEncoding(0);\n    }\n\n    pub fn init_http10(&mut self) {\n        self.body_mode = BM::HTTP1_0(0);\n    }\n\n    pub fn init_content_length(&mut self, cl: usize) {\n        self.body_mode = BM::ContentLength(cl, 0);\n    }\n\n    // NOTE on buffering/flush stream when writing the body\n    // Buffering writes can reduce the syscalls hence improves efficiency of the system\n    // But it hurts real time communication\n    // So we only allow buffering when the body size is known ahead, which is less likely\n    // to be real time interaction\n\n    pub async fn write_body<S>(&mut self, stream: &mut S, buf: &[u8]) -> Result<Option<usize>>\n    where\n        S: AsyncWrite + Unpin + Send,\n    {\n        trace!(\"Writing Body, size: {}\", buf.len());\n        match self.body_mode {\n            BM::Complete(_) => Ok(None),\n            BM::ContentLength(_, _) => self.do_write_body(stream, buf).await,\n            BM::ChunkedEncoding(_) => self.do_write_chunked_body(stream, buf).await,\n            BM::HTTP1_0(_) => self.do_write_http1_0_body(stream, buf).await,\n            BM::ToSelect => Ok(None), // Error here?\n        }\n    }\n\n    pub fn finished(&self) -> bool {\n        match self.body_mode {\n            BM::Complete(_) => true,\n            BM::ContentLength(total, written) => written >= total,\n            _ => false,\n        }\n    }\n\n    async fn do_write_body<S>(&mut self, stream: &mut S, buf: &[u8]) -> Result<Option<usize>>\n    where\n        S: AsyncWrite + Unpin + Send,\n    {\n        match self.body_mode {\n            BM::ContentLength(total, written) => {\n                if written >= total {\n                    // already written full length\n                    return Ok(None);\n                }\n                let mut to_write = total - written;\n                if to_write < buf.len() {\n                    warn!(\"Trying to write data over content-length: {total}\");\n                } else {\n                    to_write = buf.len();\n                }\n                let res = stream.write_all(&buf[..to_write]).await;\n                match res {\n                    Ok(()) => {\n                        self.body_mode = BM::ContentLength(total, written + to_write);\n                        if self.finished() {\n                            stream.flush().await.or_err(WriteError, \"flushing body\")?;\n                        }\n                        Ok(Some(to_write))\n                    }\n                    Err(e) => Error::e_because(WriteError, \"while writing body\", e),\n                }\n            }\n            _ => panic!(\"wrong body mode: {:?}\", self.body_mode),\n        }\n    }\n\n    async fn do_write_chunked_body<S>(\n        &mut self,\n        stream: &mut S,\n        buf: &[u8],\n    ) -> Result<Option<usize>>\n    where\n        S: AsyncWrite + Unpin + Send,\n    {\n        match self.body_mode {\n            BM::ChunkedEncoding(written) => {\n                let chunk_size = buf.len();\n\n                let chuck_size_buf = format!(\"{:X}\\r\\n\", chunk_size);\n                let mut output_buf = Bytes::from(chuck_size_buf).chain(buf).chain(&b\"\\r\\n\"[..]);\n\n                while output_buf.has_remaining() {\n                    let res = stream.write_vec(&mut output_buf).await;\n                    match res {\n                        Ok(n) => {\n                            if n == 0 {\n                                return Error::e_explain(ConnectionClosed, \"while writing body\");\n                            }\n                        }\n                        Err(e) => {\n                            return Error::e_because(WriteError, \"while writing body\", e);\n                        }\n                    }\n                }\n                stream.flush().await.or_err(WriteError, \"flushing body\")?;\n                self.body_mode = BM::ChunkedEncoding(written + chunk_size);\n                Ok(Some(chunk_size))\n            }\n            _ => panic!(\"wrong body mode: {:?}\", self.body_mode),\n        }\n    }\n\n    async fn do_write_http1_0_body<S>(\n        &mut self,\n        stream: &mut S,\n        buf: &[u8],\n    ) -> Result<Option<usize>>\n    where\n        S: AsyncWrite + Unpin + Send,\n    {\n        match self.body_mode {\n            BM::HTTP1_0(written) => {\n                let res = stream.write_all(buf).await;\n                match res {\n                    Ok(()) => {\n                        self.body_mode = BM::HTTP1_0(written + buf.len());\n                        stream.flush().await.or_err(WriteError, \"flushing body\")?;\n                        Ok(Some(buf.len()))\n                    }\n                    Err(e) => Error::e_because(WriteError, \"while writing body\", e),\n                }\n            }\n            _ => panic!(\"wrong body mode: {:?}\", self.body_mode),\n        }\n    }\n\n    pub async fn finish<S>(&mut self, stream: &mut S) -> Result<Option<usize>>\n    where\n        S: AsyncWrite + Unpin + Send,\n    {\n        match self.body_mode {\n            BM::Complete(_) => Ok(None),\n            BM::ContentLength(_, _) => self.do_finish_body(stream),\n            BM::ChunkedEncoding(_) => self.do_finish_chunked_body(stream).await,\n            BM::HTTP1_0(_) => self.do_finish_http1_0_body(stream),\n            BM::ToSelect => Ok(None),\n        }\n    }\n\n    fn do_finish_body<S>(&mut self, _stream: S) -> Result<Option<usize>> {\n        match self.body_mode {\n            BM::ContentLength(total, written) => {\n                self.body_mode = BM::Complete(written);\n                if written < total {\n                    return Error::e_explain(\n                        PREMATURE_BODY_END,\n                        format!(\"Content-length: {total} bytes written: {written}\"),\n                    );\n                }\n                Ok(Some(written))\n            }\n            _ => panic!(\"wrong body mode: {:?}\", self.body_mode),\n        }\n    }\n\n    async fn do_finish_chunked_body<S>(&mut self, stream: &mut S) -> Result<Option<usize>>\n    where\n        S: AsyncWrite + Unpin + Send,\n    {\n        match self.body_mode {\n            BM::ChunkedEncoding(written) => {\n                let res = stream.write_all(&LAST_CHUNK[..]).await;\n                self.body_mode = BM::Complete(written);\n                match res {\n                    Ok(()) => Ok(Some(written)),\n                    Err(e) => Error::e_because(WriteError, \"while writing body\", e),\n                }\n            }\n            _ => panic!(\"wrong body mode: {:?}\", self.body_mode),\n        }\n    }\n\n    fn do_finish_http1_0_body<S>(&mut self, _stream: &mut S) -> Result<Option<usize>> {\n        match self.body_mode {\n            BM::HTTP1_0(written) => {\n                self.body_mode = BM::Complete(written);\n                Ok(Some(written))\n            }\n            _ => panic!(\"wrong body mode: {:?}\", self.body_mode),\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use tokio_test::io::Builder;\n\n    fn init_log() {\n        let _ = env_logger::builder().is_test(true).try_init();\n    }\n\n    #[tokio::test]\n    async fn read_with_body_content_length() {\n        init_log();\n        let input = b\"abc\";\n        let mut mock_io = Builder::new().read(&input[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_content_length(3, b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 3));\n        assert_eq!(body_reader.body_state, ParseState::Complete(3));\n        assert_eq!(input, body_reader.get_body(&res));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_content_length_2() {\n        init_log();\n        let input1 = b\"a\";\n        let input2 = b\"bc\";\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_content_length(3, b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 1));\n        assert_eq!(body_reader.body_state, ParseState::Partial(1, 2));\n        assert_eq!(input1, body_reader.get_body(&res));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 2));\n        assert_eq!(body_reader.body_state, ParseState::Complete(3));\n        assert_eq!(input2, body_reader.get_body(&res));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_content_length_less() {\n        init_log();\n        let input1 = b\"a\";\n        let input2 = b\"\"; // simulating close\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_content_length(3, b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 1));\n        assert_eq!(body_reader.body_state, ParseState::Partial(1, 2));\n        assert_eq!(input1, body_reader.get_body(&res));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap_err();\n        assert_eq!(&ConnectionClosed, res.etype());\n        assert_eq!(body_reader.body_state, ParseState::Done(1));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_content_length_more() {\n        init_log();\n        let input1 = b\"a\";\n        let input2 = b\"bcd\";\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_content_length(3, b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 1));\n        assert_eq!(body_reader.body_state, ParseState::Partial(1, 2));\n        assert_eq!(input1, body_reader.get_body(&res));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 2));\n        assert_eq!(body_reader.body_state, ParseState::Complete(3));\n        assert_eq!(&input2[0..2], body_reader.get_body(&res));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_content_length_rewind() {\n        init_log();\n        let rewind = b\"ab\";\n        let input = b\"c\";\n        let mut mock_io = Builder::new().read(&input[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_content_length(3, rewind);\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 2));\n        assert_eq!(body_reader.body_state, ParseState::Partial(2, 1));\n        assert_eq!(rewind, body_reader.get_body(&res));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 1));\n        assert_eq!(body_reader.body_state, ParseState::Complete(3));\n        assert_eq!(input, body_reader.get_body(&res));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_http10() {\n        init_log();\n        let input1 = b\"a\";\n        let input2 = b\"\"; // simulating close\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_http10(b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 1));\n        assert_eq!(body_reader.body_state, ParseState::HTTP1_0(1));\n        assert_eq!(input1, body_reader.get_body(&res));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(1));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_http10_rewind() {\n        init_log();\n        let rewind = b\"ab\";\n        let input1 = b\"c\";\n        let input2 = b\"\"; // simulating close\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_http10(rewind);\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 2));\n        assert_eq!(body_reader.body_state, ParseState::HTTP1_0(2));\n        assert_eq!(rewind, body_reader.get_body(&res));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 1));\n        assert_eq!(body_reader.body_state, ParseState::HTTP1_0(3));\n        assert_eq!(input1, body_reader.get_body(&res));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(3));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_zero_chunk() {\n        init_log();\n        let input = b\"0\\r\\n\\r\\n\";\n        let mut mock_io = Builder::new().read(&input[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(0));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_chunk_ext() {\n        init_log();\n        let input = b\"0;aaaa\\r\\n\\r\\n\";\n        let mut mock_io = Builder::new().read(&input[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(0));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_chunk_ext_oversize() {\n        init_log();\n        let chunk_size = b\"0;\";\n        let ext1 = [b'a'; 1024 * 5];\n        let ext2 = [b'a'; 1024 * 3];\n        let mut mock_io = Builder::new()\n            .read(&chunk_size[..])\n            .read(&ext1[..])\n            .read(&ext2[..])\n            .build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(b\"\");\n        // read chunk-size, chunk incomplete\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, Some(BufRef::new(0, 0)));\n        // read ext1, chunk incomplete\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, Some(BufRef::new(0, 0)));\n        // read ext2, now oversized\n        let res = body_reader.read_body(&mut mock_io).await;\n        assert!(res.is_err());\n        assert_eq!(body_reader.body_state, ParseState::Done(0));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_1_chunk() {\n        init_log();\n        let input1 = b\"1\\r\\na\\r\\n\";\n        let input2 = b\"0\\r\\n\\r\\n\";\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(3, 1));\n        assert_eq!(&input1[3..4], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(1, 0, 0, 0));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(1));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_1_chunk_rewind() {\n        init_log();\n        let rewind = b\"1\\r\\nx\\r\\n\";\n        let input1 = b\"1\\r\\na\\r\\n\";\n        let input2 = b\"0\\r\\n\\r\\n\";\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(rewind);\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(3, 1));\n        assert_eq!(&rewind[3..4], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(1, 0, 0, 0));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(3, 1));\n        assert_eq!(&input1[3..4], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(2, 0, 0, 0));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(2));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_multi_chunk() {\n        init_log();\n        let input1 = b\"1\\r\\na\\r\\n2\\r\\nbc\\r\\n\";\n        let input2 = b\"0\\r\\n\\r\\n\";\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(3, 1));\n        assert_eq!(&input1[3..4], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(1, 6, 13, 0));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(9, 2));\n        assert_eq!(&input1[9..11], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(3, 0, 0, 0));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(3));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_partial_chunk() {\n        init_log();\n        let input1 = b\"3\\r\\na\";\n        let input2 = b\"bc\\r\\n0\\r\\n\\r\\n\";\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(3, 1));\n        assert_eq!(&input1[3..4], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(1, 0, 0, 4));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 2));\n        assert_eq!(&input2[0..2], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(3, 4, 9, 0));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(3));\n    }\n\n    #[tokio::test]\n    async fn read_with_body_partial_head_chunk() {\n        init_log();\n        let input1 = b\"1\\r\";\n        let input2 = b\"\\na\\r\\n0\\r\\n\\r\\n\";\n        let mut mock_io = Builder::new().read(&input1[..]).read(&input2[..]).build();\n        let mut body_reader = BodyReader::new();\n        body_reader.init_chunked(b\"\");\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(0, 0));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(0, 0, 2, 2));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, BufRef::new(3, 1)); // input1 concat input2\n        assert_eq!(&input2[1..2], body_reader.get_body(&res));\n        assert_eq!(body_reader.body_state, ParseState::Chunked(1, 6, 11, 0));\n        let res = body_reader.read_body(&mut mock_io).await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_reader.body_state, ParseState::Complete(1));\n    }\n\n    #[tokio::test]\n    async fn write_body_cl() {\n        init_log();\n        let output = b\"a\";\n        let mut mock_io = Builder::new().write(&output[..]).build();\n        let mut body_writer = BodyWriter::new();\n        body_writer.init_content_length(1);\n        assert_eq!(body_writer.body_mode, BodyMode::ContentLength(1, 0));\n        let res = body_writer\n            .write_body(&mut mock_io, &output[..])\n            .await\n            .unwrap()\n            .unwrap();\n        assert_eq!(res, 1);\n        assert_eq!(body_writer.body_mode, BodyMode::ContentLength(1, 1));\n        // write again, over the limit\n        let res = body_writer\n            .write_body(&mut mock_io, &output[..])\n            .await\n            .unwrap();\n        assert_eq!(res, None);\n        assert_eq!(body_writer.body_mode, BodyMode::ContentLength(1, 1));\n        let res = body_writer.finish(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, 1);\n        assert_eq!(body_writer.body_mode, BodyMode::Complete(1));\n    }\n\n    #[tokio::test]\n    async fn write_body_chunked() {\n        init_log();\n        let data = b\"abcdefghij\";\n        let output = b\"A\\r\\nabcdefghij\\r\\n\";\n        let mut mock_io = Builder::new()\n            .write(&output[..])\n            .write(&output[..])\n            .write(&LAST_CHUNK[..])\n            .build();\n        let mut body_writer = BodyWriter::new();\n        body_writer.init_chunked();\n        assert_eq!(body_writer.body_mode, BodyMode::ChunkedEncoding(0));\n        let res = body_writer\n            .write_body(&mut mock_io, &data[..])\n            .await\n            .unwrap()\n            .unwrap();\n        assert_eq!(res, data.len());\n        assert_eq!(body_writer.body_mode, BodyMode::ChunkedEncoding(data.len()));\n        let res = body_writer\n            .write_body(&mut mock_io, &data[..])\n            .await\n            .unwrap()\n            .unwrap();\n        assert_eq!(res, data.len());\n        assert_eq!(\n            body_writer.body_mode,\n            BodyMode::ChunkedEncoding(data.len() * 2)\n        );\n        let res = body_writer.finish(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, data.len() * 2);\n        assert_eq!(body_writer.body_mode, BodyMode::Complete(data.len() * 2));\n    }\n\n    #[tokio::test]\n    async fn write_body_http10() {\n        init_log();\n        let data = b\"a\";\n        let mut mock_io = Builder::new().write(&data[..]).write(&data[..]).build();\n        let mut body_writer = BodyWriter::new();\n        body_writer.init_http10();\n        assert_eq!(body_writer.body_mode, BodyMode::HTTP1_0(0));\n        let res = body_writer\n            .write_body(&mut mock_io, &data[..])\n            .await\n            .unwrap()\n            .unwrap();\n        assert_eq!(res, 1);\n        assert_eq!(body_writer.body_mode, BodyMode::HTTP1_0(1));\n        let res = body_writer\n            .write_body(&mut mock_io, &data[..])\n            .await\n            .unwrap()\n            .unwrap();\n        assert_eq!(res, 1);\n        assert_eq!(body_writer.body_mode, BodyMode::HTTP1_0(2));\n        let res = body_writer.finish(&mut mock_io).await.unwrap().unwrap();\n        assert_eq!(res, 2);\n        assert_eq!(body_writer.body_mode, BodyMode::Complete(2));\n    }\n}\n\n// Path: pingora-core/src/protocols/http/v1/common.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Common functions and constants\n\nuse http::header;\nuse log::warn;\nuse pingora_http::{HMap, RequestHeader, ResponseHeader};\nuse std::str;\nuse std::time::Duration;\n\nuse super::body::BodyWriter;\nuse crate::utils::KVRef;\n\npub(super) const MAX_HEADERS: usize = 256;\n\npub(super) const INIT_HEADER_BUF_SIZE: usize = 4096;\npub(super) const MAX_HEADER_SIZE: usize = 1048575;\n\npub(super) const BODY_BUF_LIMIT: usize = 1024 * 64;\n\npub const CRLF: &[u8; 2] = b\"\\r\\n\";\npub const HEADER_KV_DELIMITER: &[u8; 2] = b\": \";\n\npub(super) enum HeaderParseState {\n    Complete(usize),\n    Partial,\n    Invalid(httparse::Error),\n}\n\n#[derive(Clone, Debug, PartialEq, Eq)]\npub(super) enum KeepaliveStatus {\n    Timeout(Duration),\n    Infinite,\n    Off,\n}\n\nstruct ConnectionValue {\n    keep_alive: bool,\n    upgrade: bool,\n    close: bool,\n}\n\nimpl ConnectionValue {\n    fn new() -> Self {\n        ConnectionValue {\n            keep_alive: false,\n            upgrade: false,\n            close: false,\n        }\n    }\n\n    fn close(mut self) -> Self {\n        self.close = true;\n        self\n    }\n    fn upgrade(mut self) -> Self {\n        self.upgrade = true;\n        self\n    }\n    fn keep_alive(mut self) -> Self {\n        self.keep_alive = true;\n        self\n    }\n}\n\nfn parse_connection_header(value: &[u8]) -> ConnectionValue {\n    // only parse keep-alive, close, and upgrade tokens\n    // https://www.rfc-editor.org/rfc/rfc9110.html#section-7.6.1\n\n    const KEEP_ALIVE: &str = \"keep-alive\";\n    const CLOSE: &str = \"close\";\n    const UPGRADE: &str = \"upgrade\";\n\n    // fast path\n    if value.eq_ignore_ascii_case(CLOSE.as_bytes()) {\n        ConnectionValue::new().close()\n    } else if value.eq_ignore_ascii_case(KEEP_ALIVE.as_bytes()) {\n        ConnectionValue::new().keep_alive()\n    } else if value.eq_ignore_ascii_case(UPGRADE.as_bytes()) {\n        ConnectionValue::new().upgrade()\n    } else {\n        // slow path, parse the connection value\n        let mut close = false;\n        let mut upgrade = false;\n        let value = str::from_utf8(value).unwrap_or(\"\");\n        for token in value\n            .split(',')\n            .map(|s| s.trim())\n            .filter(|&x| !x.is_empty())\n        {\n            if token.eq_ignore_ascii_case(CLOSE) {\n                close = true;\n            } else if token.eq_ignore_ascii_case(UPGRADE) {\n                upgrade = true;\n            }\n            if upgrade && close {\n                return ConnectionValue::new().upgrade().close();\n            }\n        }\n        if close {\n            ConnectionValue::new().close()\n        } else if upgrade {\n            ConnectionValue::new().upgrade()\n        } else {\n            ConnectionValue::new()\n        }\n    }\n}\n\npub(crate) fn init_body_writer_comm(body_writer: &mut BodyWriter, headers: &HMap) {\n    let te_value = headers.get(http::header::TRANSFER_ENCODING);\n    if is_header_value_chunked_encoding(te_value) {\n        // transfer-encoding takes priority over content-length\n        body_writer.init_chunked();\n    } else {\n        let content_length = header_value_content_length(headers.get(http::header::CONTENT_LENGTH));\n        match content_length {\n            Some(length) => {\n                body_writer.init_content_length(length);\n            }\n            None => {\n                /* TODO: 1. connection: keepalive cannot be used,\n                2. mark connection must be closed */\n                body_writer.init_http10();\n            }\n        }\n    }\n}\n\n#[inline]\npub(super) fn is_header_value_chunked_encoding(\n    header_value: Option<&http::header::HeaderValue>,\n) -> bool {\n    match header_value {\n        Some(value) => value.as_bytes().eq_ignore_ascii_case(b\"chunked\"),\n        None => false,\n    }\n}\n\npub(super) fn is_upgrade_req(req: &RequestHeader) -> bool {\n    req.version == http::Version::HTTP_11 && req.headers.get(header::UPGRADE).is_some()\n}\n\n// Unlike the upgrade check on request, this function doesn't check the Upgrade or Connection header\n// because when seeing 101, we assume the server accepts to switch protocol.\n// In reality it is not common that some servers don't send all the required headers to establish\n// websocket connections.\npub(super) fn is_upgrade_resp(header: &ResponseHeader) -> bool {\n    header.status == 101 && header.version == http::Version::HTTP_11\n}\n\n#[inline]\npub fn header_value_content_length(\n    header_value: Option<&http::header::HeaderValue>,\n) -> Option<usize> {\n    match header_value {\n        Some(value) => buf_to_content_length(Some(value.as_bytes())),\n        None => None,\n    }\n}\n\n#[inline]\npub(super) fn buf_to_content_length(header_value: Option<&[u8]>) -> Option<usize> {\n    match header_value {\n        Some(buf) => {\n            match str::from_utf8(buf) {\n                // check valid string\n                Ok(str_cl_value) => match str_cl_value.parse::<i64>() {\n                    Ok(cl_length) => {\n                        if cl_length >= 0 {\n                            Some(cl_length as usize)\n                        } else {\n                            warn!(\"negative content-length header value {cl_length}\");\n                            None\n                        }\n                    }\n                    Err(_) => {\n                        warn!(\"invalid content-length header value {str_cl_value}\");\n                        None\n                    }\n                },\n                Err(_) => {\n                    warn!(\"invalid content-length header encoding\");\n                    None\n                }\n            }\n        }\n        None => None,\n    }\n}\n\n#[inline]\npub(super) fn is_buf_keepalive(header_value: Option<&[u8]>) -> Option<bool> {\n    header_value.and_then(|value| {\n        let value = parse_connection_header(value);\n        if value.keep_alive {\n            Some(true)\n        } else if value.close {\n            Some(false)\n        } else {\n            None\n        }\n    })\n}\n\n#[inline]\npub(super) fn populate_headers(\n    base: usize,\n    header_ref: &mut Vec<KVRef>,\n    headers: &[httparse::Header],\n) -> usize {\n    let mut used_header_index = 0;\n    for header in headers.iter() {\n        if !header.name.is_empty() {\n            header_ref.push(KVRef::new(\n                header.name.as_ptr() as usize - base,\n                header.name.as_bytes().len(),\n                header.value.as_ptr() as usize - base,\n                header.value.len(),\n            ));\n            used_header_index += 1;\n        }\n    }\n    used_header_index\n}\n\n// Path: pingora-core/src/protocols/http/v1/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/1.x client session\n\nuse bytes::{BufMut, Bytes, BytesMut};\nuse http::{header, header::AsHeaderName, HeaderValue, StatusCode, Version};\nuse log::{debug, trace};\nuse pingora_error::{Error, ErrorType::*, OrErr, Result, RetryType};\nuse pingora_http::{HMap, IntoCaseHeaderName, RequestHeader, ResponseHeader};\nuse pingora_timeout::timeout;\nuse std::io::ErrorKind;\nuse std::str;\nuse std::time::Duration;\nuse tokio::io::{AsyncReadExt, AsyncWriteExt};\n\nuse super::body::{BodyReader, BodyWriter};\nuse super::common::*;\nuse crate::protocols::http::HttpTask;\nuse crate::protocols::{Digest, SocketAddr, Stream, UniqueID};\nuse crate::utils::{BufRef, KVRef};\n\n/// The HTTP 1.x client session\npub struct HttpSession {\n    buf: Bytes,\n    pub(crate) underlying_stream: Stream,\n    raw_header: Option<BufRef>,\n    preread_body: Option<BufRef>,\n    body_reader: BodyReader,\n    body_writer: BodyWriter,\n    // timeouts:\n    /// The read timeout, which will be applied to both reading the header and the body.\n    /// The timeout is reset on every read. This is not a timeout on the overall duration of the\n    /// response.\n    pub read_timeout: Option<Duration>,\n    /// The write timeout which will be applied to both writing request header and body.\n    /// The timeout is reset on every write. This is not a timeout on the overall duration of the\n    /// request.\n    pub write_timeout: Option<Duration>,\n    keepalive_timeout: KeepaliveStatus,\n    pub(crate) digest: Box<Digest>,\n    response_header: Option<Box<ResponseHeader>>,\n    request_written: Option<Box<RequestHeader>>,\n    bytes_sent: usize,\n    upgraded: bool,\n}\n\n/// HTTP 1.x client session\nimpl HttpSession {\n    /// Create a new http client session from an established (TCP or TLS) [`Stream`].\n    pub fn new(stream: Stream) -> Self {\n        // TODO: maybe we should put digest in the connection itself\n        let digest = Box::new(Digest {\n            ssl_digest: stream.get_ssl_digest(),\n            timing_digest: stream.get_timing_digest(),\n            proxy_digest: stream.get_proxy_digest(),\n            socket_digest: stream.get_socket_digest(),\n        });\n        HttpSession {\n            underlying_stream: stream,\n            buf: Bytes::new(), // zero size, will be replaced by parsed header later\n            raw_header: None,\n            preread_body: None,\n            body_reader: BodyReader::new(),\n            body_writer: BodyWriter::new(),\n            keepalive_timeout: KeepaliveStatus::Off,\n            response_header: None,\n            request_written: None,\n            read_timeout: None,\n            write_timeout: None,\n            digest,\n            bytes_sent: 0,\n            upgraded: false,\n        }\n    }\n    /// Write the request header to the server\n    /// After the request header is sent. The caller can either start reading the response or\n    /// sending request body if any.\n    pub async fn write_request_header(&mut self, req: Box<RequestHeader>) -> Result<usize> {\n        // TODO: make sure this can only be called once\n        // init body writer\n        self.init_req_body_writer(&req);\n\n        let to_wire = http_req_header_to_wire(&req).unwrap();\n        trace!(\"Writing request header: {to_wire:?}\");\n\n        let write_fut = self.underlying_stream.write_all(to_wire.as_ref());\n        match self.write_timeout {\n            Some(t) => match timeout(t, write_fut).await {\n                Ok(res) => res,\n                Err(_) => Err(std::io::Error::from(ErrorKind::TimedOut)),\n            },\n            None => write_fut.await,\n        }\n        .map_err(|e| match e.kind() {\n            ErrorKind::TimedOut => {\n                Error::because(WriteTimedout, \"while writing request headers (timeout)\", e)\n            }\n            _ => Error::because(WriteError, \"while writing request headers\", e),\n        })?;\n\n        self.underlying_stream\n            .flush()\n            .await\n            .or_err(WriteError, \"flushing request header\")?;\n\n        // write was successful\n        self.request_written = Some(req);\n        Ok(to_wire.len())\n    }\n\n    async fn do_write_body(&mut self, buf: &[u8]) -> Result<Option<usize>> {\n        let written = self\n            .body_writer\n            .write_body(&mut self.underlying_stream, buf)\n            .await;\n\n        if let Ok(Some(num_bytes)) = written {\n            self.bytes_sent += num_bytes;\n        }\n\n        written\n    }\n\n    /// Write request body. Return Ok(None) if no more body should be written, either due to\n    /// Content-Length or the last chunk is already sent\n    pub async fn write_body(&mut self, buf: &[u8]) -> Result<Option<usize>> {\n        // TODO: verify that request header is sent already\n        match self.write_timeout {\n            Some(t) => match timeout(t, self.do_write_body(buf)).await {\n                Ok(res) => res,\n                Err(_) => Error::e_explain(WriteTimedout, format!(\"writing body, timeout: {t:?}\")),\n            },\n            None => self.do_write_body(buf).await,\n        }\n    }\n\n    fn maybe_force_close_body_reader(&mut self) {\n        if self.upgraded && !self.body_reader.body_done() {\n            // request is done, reset the response body to close\n            self.body_reader.init_content_length(0, b\"\");\n        }\n    }\n\n    /// Flush local buffer and notify the server by sending the last chunk if chunked encoding is\n    /// used.\n    pub async fn finish_body(&mut self) -> Result<Option<usize>> {\n        let res = self.body_writer.finish(&mut self.underlying_stream).await?;\n        self.underlying_stream\n            .flush()\n            .await\n...\n", "needle_token_start": 2285, "needle_token_end": 2580, "code_context_ntokens": 16277, "output": ["```rust\npub async fn do_read_body_until_closed<S>(&mut self, stream: &mut S) -> Result<Option<BufRef>>\nwhere\n    S: AsyncRead + Unpin + Send,\n{\n    let body_buf = self.body_buf.as_deref_mut().unwrap();\n    let mut n = self.rewind_buf_len;\n    self.rewind_buf_len = 0; // we only need to read rewind data once\n    if n == 0 {\n        /* Need to actually read */\n        n = stream\n            .read(body_buf)\n            .await\n            .or_err(ReadError, \"when reading body\")?;\n    }\n    match self.body_state {\n        PS::HTTP1_0(read) => {\n            if n == 0 {\n                self.body_state = PS::Complete(read);\n                Ok(None)\n            } else {\n                self.body_state = PS::HTTP1_0(read + n);\n                Ok(Some(BufRef::new(0, n)))\n            }\n        }\n        _ => panic!(\"wrong body state: {:?}\", self.body_state),\n    }\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "write_to_buffer", "language": "rust", "path": "pingora-core/src/protocols/http/body_buffer.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To append data to an internal storage if the total size after appending does not exceed a predefined limit, otherwise mark the storage as truncated.\n2. **Input**: A sequence of bytes to be added to the buffer.\n3. **Output**: There is no direct output returned by the method; however, it modifies the internal state of the buffer by either appending data or setting a truncation flag.\n4. **Procedure**: The method first checks if adding the new data would exceed the buffer's capacity. If not, the data is appended to the buffer. If the addition would exceed the capacity, the buffer is marked as truncated, and no data is added.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/protocols/http/v2/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/2 client session and connection\n// TODO: this module needs a refactor\n\nuse bytes::Bytes;\nuse h2::client::{self, ResponseFuture, SendRequest};\nuse h2::{Reason, RecvStream, SendStream};\nuse http::HeaderMap;\nuse log::{debug, error, warn};\nuse pingora_error::{Error, ErrorType, ErrorType::*, OrErr, Result, RetryType};\nuse pingora_http::{RequestHeader, ResponseHeader};\nuse pingora_timeout::timeout;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::Arc;\nuse std::time::Duration;\nuse tokio::io::{AsyncRead, AsyncWrite};\nuse tokio::sync::watch;\n\nuse crate::connectors::http::v2::ConnectionRef;\nuse crate::protocols::{Digest, SocketAddr};\n\npub const PING_TIMEDOUT: ErrorType = ErrorType::new(\"PingTimedout\");\n\npub struct Http2Session {\n    send_req: SendRequest<Bytes>,\n    send_body: Option<SendStream<Bytes>>,\n    resp_fut: Option<ResponseFuture>,\n    req_sent: Option<Box<RequestHeader>>,\n    response_header: Option<ResponseHeader>,\n    response_body_reader: Option<RecvStream>,\n    /// The read timeout, which will be applied to both reading the header and the body.\n    /// The timeout is reset on every read. This is not a timeout on the overall duration of the\n    /// response.\n    pub read_timeout: Option<Duration>,\n    pub(crate) conn: ConnectionRef,\n    // Indicate that whether a END_STREAM is already sent\n    ended: bool,\n}\n\nimpl Drop for Http2Session {\n    fn drop(&mut self) {\n        self.conn.release_stream();\n    }\n}\n\nimpl Http2Session {\n    pub(crate) fn new(send_req: SendRequest<Bytes>, conn: ConnectionRef) -> Self {\n        Http2Session {\n            send_req,\n            send_body: None,\n            resp_fut: None,\n            req_sent: None,\n            response_header: None,\n            response_body_reader: None,\n            read_timeout: None,\n            conn,\n            ended: false,\n        }\n    }\n\n    fn sanitize_request_header(req: &mut RequestHeader) -> Result<()> {\n        req.set_version(http::Version::HTTP_2);\n        if req.uri.authority().is_some() {\n            return Ok(());\n        }\n        // use host header to populate :authority field\n        let Some(authority) = req.headers.get(http::header::HOST).map(|v| v.as_bytes()) else {\n            return Error::e_explain(InvalidHTTPHeader, \"no authority header for h2\");\n        };\n        let uri = http::uri::Builder::new()\n            .scheme(\"https\") // fixed for now\n            .authority(authority)\n            .path_and_query(req.uri.path_and_query().as_ref().unwrap().as_str())\n            .build();\n        match uri {\n            Ok(uri) => {\n                req.set_uri(uri);\n                Ok(())\n            }\n            Err(_) => Error::e_explain(\n                InvalidHTTPHeader,\n                format!(\"invalid authority from host {authority:?}\"),\n            ),\n        }\n    }\n\n    /// Write the request header to the server\n    pub fn write_request_header(&mut self, mut req: Box<RequestHeader>, end: bool) -> Result<()> {\n        if self.req_sent.is_some() {\n            // cannot send again, TODO: warn\n            return Ok(());\n        }\n        Self::sanitize_request_header(&mut req)?;\n        let parts = req.as_owned_parts();\n        let request = http::Request::from_parts(parts, ());\n        // There is no write timeout for h2 because the actual write happens async from this fn\n        let (resp_fut, send_body) = self\n            .send_req\n            .send_request(request, end)\n            .or_err(H2Error, \"while sending request\")\n            .map_err(|e| self.handle_err(e))?;\n        self.req_sent = Some(req);\n        self.send_body = Some(send_body);\n        self.resp_fut = Some(resp_fut);\n        self.ended = self.ended || end;\n\n        Ok(())\n    }\n\n    /// Write a request body chunk\n    pub fn write_request_body(&mut self, data: Bytes, end: bool) -> Result<()> {\n        if self.ended {\n            warn!(\"Try to write request body after end of stream, dropping the extra data\");\n            return Ok(());\n        }\n\n        let body_writer = self\n            .send_body\n            .as_mut()\n            .expect(\"Try to write request body before sending request header\");\n\n        write_body(body_writer, data, end).map_err(|e| self.handle_err(e))?;\n        self.ended = self.ended || end;\n        Ok(())\n    }\n\n    /// Signal that the request body has ended\n    pub fn finish_request_body(&mut self) -> Result<()> {\n        if self.ended {\n            return Ok(());\n        }\n\n        let body_writer = self\n            .send_body\n            .as_mut()\n            .expect(\"Try to finish request stream before sending request header\");\n\n        // Just send an empty data frame with end of stream set\n        body_writer\n            .send_data(\"\".into(), true)\n            .or_err(WriteError, \"while writing empty h2 request body\")\n            .map_err(|e| self.handle_err(e))?;\n        self.ended = true;\n        Ok(())\n    }\n\n    /// Read the response header\n    pub async fn read_response_header(&mut self) -> Result<()> {\n        // TODO: how to read 1xx headers?\n        // https://github.com/hyperium/h2/issues/167\n\n        if self.response_header.is_some() {\n            panic!(\"H2 response header is already read\")\n        }\n\n        let Some(resp_fut) = self.resp_fut.take() else {\n            panic!(\"Try to  response header is already read\")\n        };\n\n        let res = match self.read_timeout {\n            Some(t) => timeout(t, resp_fut)\n                .await\n                .map_err(|_| Error::explain(ReadTimedout, \"while reading h2 response header\"))\n...\n// Path: pingora-core/src/protocols/http/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse bytes::Bytes;\nuse pingora_error::Result;\nuse pingora_http::{RequestHeader, ResponseHeader};\nuse std::time::Duration;\n\nuse super::v1::client::HttpSession as Http1Session;\nuse super::v2::client::Http2Session;\nuse crate::protocols::{Digest, SocketAddr};\n\n/// A type for Http client session. It can be either an Http1 connection or an Http2 stream.\npub enum HttpSession {\n    H1(Http1Session),\n    H2(Http2Session),\n}\n\nimpl HttpSession {\n    pub fn as_http1(&self) -> Option<&Http1Session> {\n        match self {\n            Self::H1(s) => Some(s),\n            Self::H2(_) => None,\n        }\n    }\n\n    pub fn as_http2(&self) -> Option<&Http2Session> {\n        match self {\n            Self::H1(_) => None,\n            Self::H2(s) => Some(s),\n        }\n    }\n    /// Write the request header to the server\n    /// After the request header is sent. The caller can either start reading the response or\n    /// sending request body if any.\n    pub async fn write_request_header(&mut self, req: Box<RequestHeader>) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                h1.write_request_header(req).await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.write_request_header(req, false),\n        }\n    }\n\n    /// Write a chunk of the request body.\n    pub async fn write_request_body(&mut self, data: Bytes, end: bool) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                // TODO: maybe h1 should also have the concept of `end`\n                h1.write_body(&data).await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.write_request_body(data, end),\n        }\n    }\n\n    /// Signal that the request body has ended\n    pub async fn finish_request_body(&mut self) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                h1.finish_body().await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.finish_request_body(),\n        }\n    }\n\n    /// Set the read timeout for reading header and body.\n    ///\n    /// The timeout is per read operation, not on the overall time reading the entire response\n    pub fn set_read_timeout(&mut self, timeout: Duration) {\n        match self {\n            HttpSession::H1(h1) => h1.read_timeout = Some(timeout),\n            HttpSession::H2(h2) => h2.read_timeout = Some(timeout),\n        }\n    }\n\n    /// Set the write timeout for writing header and body.\n    ///\n    /// The timeout is per write operation, not on the overall time writing the entire request\n    pub fn set_write_timeout(&mut self, timeout: Duration) {\n        match self {\n            HttpSession::H1(h1) => h1.write_timeout = Some(timeout),\n            HttpSession::H2(_) => { /* no write timeout because the actual write happens async*/ }\n        }\n    }\n\n    /// Read the response header from the server\n    /// For http1, this function can be called multiple times, if the headers received are just\n    /// informational headers.\n    pub async fn read_response_header(&mut self) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                h1.read_response().await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.read_response_header().await,\n        }\n    }\n\n    /// Read response body\n    ///\n    /// `None` when no more body to read.\n    pub async fn read_response_body(&mut self) -> Result<Option<Bytes>> {\n        match self {\n            HttpSession::H1(h1) => h1.read_body_bytes().await,\n            HttpSession::H2(h2) => h2.read_response_body().await,\n        }\n    }\n\n    /// No (more) body to read\n    pub fn response_done(&mut self) -> bool {\n        match self {\n            HttpSession::H1(h1) => h1.is_body_done(),\n            HttpSession::H2(h2) => h2.response_finished(),\n        }\n    }\n\n    /// Give up the http session abruptly.\n    /// For H1 this will close the underlying connection\n    /// For H2 this will send RST_STREAM frame to end this stream if the stream has not ended at all\n    pub async fn shutdown(&mut self) {\n        match self {\n            Self::H1(s) => s.shutdown().await,\n            Self::H2(s) => s.shutdown(),\n        }\n    }\n\n    /// Get the response header of the server\n    ///\n    /// `None` if the response header is not read yet.\n    pub fn response_header(&self) -> Option<&ResponseHeader> {\n        match self {\n            Self::H1(s) => s.resp_header(),\n            Self::H2(s) => s.response_header(),\n        }\n    }\n\n    /// Return the [Digest] of the connection\n    ///\n    /// For reused connection, the timing in the digest will reflect its initial handshakes\n    /// The caller should check if the connection is reused to avoid misuse of the timing field\n    pub fn digest(&self) -> Option<&Digest> {\n        match self {\n            Self::H1(s) => Some(s.digest()),\n            Self::H2(s) => s.digest(),\n        }\n    }\n\n    /// Return the server (peer) address of the connection.\n    pub fn server_addr(&self) -> Option<&SocketAddr> {\n        match self {\n            Self::H1(s) => s.server_addr(),\n            Self::H2(s) => s.server_addr(),\n        }\n    }\n\n    /// Return the client (local) address of the connection.\n    pub fn client_addr(&self) -> Option<&SocketAddr> {\n        match self {\n            Self::H1(s) => s.client_addr(),\n            Self::H2(s) => s.client_addr(),\n        }\n    }\n}\n\n// Path: pingora-core/src/protocols/http/body_buffer.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse bytes::{Bytes, BytesMut};\n\n/// A buffer with size limit. When the total amount of data written to the buffer is below the limit\n/// all the data will be held in the buffer. Otherwise, the buffer will report to be truncated.\npub(crate) struct FixedBuffer {\n    buffer: BytesMut,\n    capacity: usize,\n    truncated: bool,\n}\n\nimpl FixedBuffer {\n    pub fn new(capacity: usize) -> Self {\n        FixedBuffer {\n            buffer: BytesMut::new(),\n            capacity,\n            truncated: false,\n        }\n    }\n\n    // TODO: maybe store a Vec of Bytes for zero-copy\n    \npub fn write_to_buffer(&mut self, data: &Bytes) {\n        if !self.truncated && (self.buffer.len() + data.len() <= self.capacity) {\n            self.buffer.extend_from_slice(data);\n        } else {\n            // TODO: clear data because the data held here is useless anyway?\n            self.truncated = true;\n        }\n    }\n    pub fn clear(&mut self) {\n        self.truncated = false;\n        self.buffer.clear();\n    }\n    pub fn is_empty(&self) -> bool {\n        self.buffer.len() == 0\n    }\n    pub fn is_truncated(&self) -> bool {\n        self.truncated\n    }\n    pub fn get_buffer(&self) -> Option<Bytes> {\n        // TODO: return None if truncated?\n        if !self.is_empty() {\n            Some(self.buffer.clone().freeze())\n        } else {\n            None\n        }\n    }\n}\n\n// Path: pingora-core/src/protocols/http/date.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse chrono::DateTime;\nuse http::header::HeaderValue;\nuse std::cell::RefCell;\nuse std::time::{Duration, SystemTime};\n\nfn to_date_string(epoch_sec: i64) -> String {\n    let dt = DateTime::from_timestamp(epoch_sec, 0).unwrap();\n    dt.format(\"%a, %d %b %Y %H:%M:%S GMT\").to_string()\n}\n\nstruct CacheableDate {\n    h1_date: HeaderValue,\n    epoch: Duration,\n}\n\nimpl CacheableDate {\n    pub fn new() -> Self {\n        let d = SystemTime::now()\n            .duration_since(SystemTime::UNIX_EPOCH)\n            .unwrap();\n        CacheableDate {\n            h1_date: HeaderValue::from_str(&to_date_string(d.as_secs() as i64)).unwrap(),\n            epoch: d,\n        }\n    }\n\n    pub fn update(&mut self, d_now: Duration) {\n        if d_now.as_secs() != self.epoch.as_secs() {\n            self.epoch = d_now;\n            self.h1_date = HeaderValue::from_str(&to_date_string(d_now.as_secs() as i64)).unwrap();\n        }\n    }\n\n    pub fn get_date(&mut self) -> HeaderValue {\n        let d = SystemTime::now()\n            .duration_since(SystemTime::UNIX_EPOCH)\n            .unwrap();\n        self.update(d);\n        self.h1_date.clone()\n    }\n}\n\nthread_local! {\n    static CACHED_DATE: RefCell<CacheableDate>\n        = RefCell::new(CacheableDate::new());\n}\n\npub fn get_cached_date() -> HeaderValue {\n    CACHED_DATE.with(|cache_date| (*cache_date.borrow_mut()).get_date())\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    fn now_date_header() -> HeaderValue {\n        HeaderValue::from_str(&to_date_string(\n            SystemTime::now()\n                .duration_since(SystemTime::UNIX_EPOCH)\n                .unwrap()\n                .as_secs() as i64,\n        ))\n        .unwrap()\n    }\n\n    #[test]\n    fn test_date_string() {\n        let date_str = to_date_string(1);\n        assert_eq!(\"Thu, 01 Jan 1970 00:00:01 GMT\", date_str);\n    }\n\n    #[test]\n    fn test_date_cached() {\n        assert_eq!(get_cached_date(), now_date_header());\n    }\n}\n\n// Path: pingora-core/src/protocols/http/v1/server.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/1.x server session\n\nuse bytes::Bytes;\nuse bytes::{BufMut, BytesMut};\nuse http::HeaderValue;\nuse http::{header, header::AsHeaderName, Method, Version};\nuse log::{debug, error, warn};\nuse once_cell::sync::Lazy;\nuse percent_encoding::{percent_encode, AsciiSet, CONTROLS};\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_http::{IntoCaseHeaderName, RequestHeader, ResponseHeader};\nuse pingora_timeout::timeout;\nuse regex::bytes::Regex;\nuse std::time::Duration;\nuse tokio::io::{AsyncReadExt, AsyncWriteExt};\n\nuse super::body::{BodyReader, BodyWriter};\nuse super::common::*;\nuse crate::protocols::http::{body_buffer::FixedBuffer, date, error_resp, HttpTask};\nuse crate::protocols::{Digest, SocketAddr, Stream};\nuse crate::utils::{BufRef, KVRef};\n\n/// The HTTP 1.x server session\npub struct HttpSession {\n    underlying_stream: Stream,\n    /// The buf that holds the raw request header + possibly a portion of request body\n    /// Request body can appear here because they could arrive with the same read() that\n    /// sends the request header.\n    buf: Bytes,\n    /// A slice reference to `buf` which points to the exact range of request header\n    raw_header: Option<BufRef>,\n    /// A slice reference to `buf` which points to the range of a portion of request body if any\n    preread_body: Option<BufRef>,\n    /// A state machine to track how to read the request body\n    body_reader: BodyReader,\n    /// A state machine to track how to write the response body\n    body_writer: BodyWriter,\n    /// An internal buffer to buf multiple body writes to reduce the underlying syscalls\n    body_write_buf: BytesMut,\n    /// Track how many application (not on the wire) body bytes already sent\n    body_bytes_sent: usize,\n    /// Whether to update headers like connection, Date\n    update_resp_headers: bool,\n    /// timeouts:\n    keepalive_timeout: KeepaliveStatus,\n    read_timeout: Option<Duration>,\n    write_timeout: Option<Duration>,\n    /// A copy of the response that is already written to the client\n    response_written: Option<Box<ResponseHeader>>,\n    /// The parse request header\n    request_header: Option<Box<RequestHeader>>,\n    /// An internal buffer that holds a copy of the request body up to a certain size\n    retry_buffer: Option<FixedBuffer>,\n    /// Whether this session is an upgraded session. This flag is calculated when sending the\n    /// response header to the client.\n    upgraded: bool,\n    /// Digest to track underlying connection metrics\n    digest: Box<Digest>,\n}\n\nimpl HttpSession {\n    /// Create a new http server session from an established (TCP or TLS) [`Stream`].\n    /// The created session needs to call [`Self::read_request()`] first before performing\n    /// any other operations.\n    pub fn new(underlying_stream: Stream) -> Self {\n        // TODO: maybe we should put digest in the connection itself\n        let digest = Box::new(Digest {\n            ssl_digest: underlying_stream.get_ssl_digest(),\n            timing_digest: underlying_stream.get_timing_digest(),\n            proxy_digest: underlying_stream.get_proxy_digest(),\n            socket_digest: underlying_stream.get_socket_digest(),\n        });\n\n        HttpSession {\n            underlying_stream,\n            buf: Bytes::new(), // zero size, with be replaced by parsed header later\n            raw_header: None,\n            preread_body: None,\n            body_reader: BodyReader::new(),\n            body_writer: BodyWriter::new(),\n            body_write_buf: BytesMut::new(),\n            keepalive_timeout: KeepaliveStatus::Off,\n            update_resp_headers: true,\n            response_written: None,\n            request_header: None,\n            read_timeout: None,\n            write_timeout: None,\n            body_bytes_sent: 0,\n            retry_buffer: None,\n            upgraded: false,\n            digest,\n        }\n    }\n\n    /// Read the request header. Return `Ok(Some(n))` where the read and parsing are successful.\n    /// Return `Ok(None)` when the client closed the connection without sending any data, which\n    /// is common on a reused connection.\n    pub async fn read_request(&mut self) -> Result<Option<usize>> {\n        const MAX_ERR_BUF_LEN: usize = 2048;\n\n        self.buf.clear();\n        let mut buf = BytesMut::with_capacity(INIT_HEADER_BUF_SIZE);\n        let mut already_read: usize = 0;\n        loop {\n            if already_read > MAX_HEADER_SIZE {\n                /* NOTE: this check only blocks second read. The first large read is allowed\n                since the buf is already allocated. The goal is to avoid slowly bloating\n                this buffer */\n                return Error::e_explain(\n                    InvalidHTTPHeader,\n                    format!(\"Request header larger than {MAX_HEADER_SIZE}\"),\n                );\n            }\n\n            let read_result = {\n                let read_event = self.underlying_stream.read_buf(&mut buf);\n                match self.keepalive_timeout {\n                    KeepaliveStatus::Timeout(d) => match timeout(d, read_event).await {\n                        Ok(res) => res,\n                        Err(e) => {\n                            debug!(\"keepalive timeout {d:?} reached, {e}\");\n                            return Ok(None);\n                        }\n                    },\n                    _ => read_event.await,\n                }\n            };\n            let n = match read_result {\n                Ok(n_read) => {\n                    if n_read == 0 {\n                        if already_read > 0 {\n                            return Error::e_explain(\n                                ConnectionClosed,\n                                format!(\n                                    \"while reading request headers, bytes already read: {}\",\n                                    already_read\n                                ),\n                            );\n                        } else {\n                            /* common when client decides to close a keepalived session */\n                            debug!(\"Client prematurely closed connection with 0 byte sent\");\n                            return Ok(None);\n                        }\n                    }\n                    n_read\n                }\n\n                Err(e) => {\n                    if already_read > 0 {\n                        return Error::e_because(ReadError, \"while reading request headers\", e);\n                    }\n                    /* nothing harmful since we have not ready any thing yet */\n                    return Ok(None);\n                }\n            };\n            already_read += n;\n\n            // Use loop as GOTO to retry escaped request buffer, not a real loop\n            loop {\n                let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS];\n                let mut req = httparse::Request::new(&mut headers);\n                let parsed = parse_req_buffer(&mut req, &buf);\n                match parsed {\n                    HeaderParseState::Complete(s) => {\n                        self.raw_header = Some(BufRef(0, s));\n                        self.preread_body = Some(BufRef(s, already_read));\n\n                        // We have the header name and values we parsed to be just 0 copy Bytes\n                        // referencing the original buf. That requires we convert the buf from\n                        // BytesMut to Bytes. But `req` holds a reference to `buf`. So we use the\n                        // `KVRef`s to record the offset of each piece of data, drop `req`, convert\n                        // buf, the do the 0 copy update\n                        let base = buf.as_ptr() as usize;\n                        let mut header_refs = Vec::<KVRef>::with_capacity(req.headers.len());\n                        // Note: req.headers has the correct number of headers\n                        // while header_refs doesn't as it is still empty\n                        let _num_headers = populate_headers(base, &mut header_refs, req.headers);\n\n                        let mut request_header = Box::new(RequestHeader::build(\n                            req.method.unwrap_or(\"\"),\n                            // we path httparse to allow unsafe bytes in the str\n                            req.path.unwrap_or(\"\").as_bytes(),\n                            Some(req.headers.len()),\n                        )?);\n\n                        request_header.set_version(match req.version {\n                            Some(1) => Version::HTTP_11,\n                            Some(0) => Version::HTTP_10,\n                            _ => Version::HTTP_09,\n                        });\n\n                        let buf = buf.freeze();\n\n                        for header in header_refs {\n                            let header_name = header.get_name_bytes(&buf);\n                            let header_name = header_name.into_case_header_name();\n                            let value_bytes = header.get_value_bytes(&buf);\n                            // safe because this is from what we parsed\n                            let header_value = unsafe {\n                                http::HeaderValue::from_maybe_shared_unchecked(value_bytes)\n                            };\n                            request_header\n                                .append_header(header_name, header_value)\n                                .or_err(InvalidHTTPHeader, \"while parsing request header\")?;\n                        }\n\n                        self.buf = buf;\n                        self.request_header = Some(request_header);\n\n                        self.body_reader.reinit();\n                        self.response_written = None;\n                        self.respect_keepalive();\n\n                        return Ok(Some(s));\n                    }\n                    HeaderParseState::Partial => {\n                        break; /* continue the read loop */\n                    }\n                    HeaderParseState::Invalid(e) => match e {\n                        httparse::Error::Token | httparse::Error::Version => {\n                            // try to escape URI\n                            if let Some(new_buf) = escape_illegal_request_line(&buf) {\n                                buf = new_buf;\n                                already_read = buf.len();\n                            } else {\n                                debug!(\"Invalid request header from {:?}\", self.underlying_stream);\n                                buf.truncate(MAX_ERR_BUF_LEN);\n                                return Error::e_because(\n                                    InvalidHTTPHeader,\n                                    format!(\n                                        \"buf: {}\",\n                                        String::from_utf8_lossy(&buf).escape_default()\n                                    ),\n                                    e,\n                                );\n                            }\n                        }\n                        _ => {\n                            debug!(\"Invalid request header from {:?}\", self.underlying_stream);\n                            buf.truncate(MAX_ERR_BUF_LEN);\n                            return Error::e_because(\n                                InvalidHTTPHeader,\n                                format!(\"buf: {}\", String::from_utf8_lossy(&buf).escape_default()),\n                                e,\n                            );\n                        }\n                    },\n                }\n            }\n        }\n    }\n\n    /// Return a reference of the `RequestHeader` this session read\n    /// # Panics\n    /// this function and most other functions will panic if called before [`Self::read_request()`]\n    pub fn req_header(&self) -> &RequestHeader {\n        self.request_header\n            .as_ref()\n            .expect(\"Request header is not read yet\")\n    }\n\n    /// Return a mutable reference of the `RequestHeader` this session read\n    /// # Panics\n    /// this function and most other functions will panic if called before [`Self::read_request()`]\n    pub fn req_header_mut(&mut self) -> &mut RequestHeader {\n        self.request_header\n            .as_mut()\n            .expect(\"Request header is not read yet\")\n    }\n\n    /// Get the header value for the given header name\n    /// If there are multiple headers under the same name, the first one will be returned\n    /// Use `self.req_header().header.get_all(name)` to get all the headers under the same name\n    pub fn get_header(&self, name: impl AsHeaderName) -> Option<&HeaderValue> {\n        self.request_header\n            .as_ref()\n            .and_then(|h| h.headers.get(name))\n    }\n\n    /// Return the method of this request. None if the request is not read yet.\n    pub(super) fn get_method(&self) -> Option<&http::Method> {\n        self.request_header.as_ref().map(|r| &r.method)\n    }\n\n    /// Return the path of the request (i.e., the `/hello?1` of `GET /hello?1 HTTP1.1`)\n    /// An empty slice will be used if there is no path or the request is not read yet\n    pub(super) fn get_path(&self) -> &[u8] {\n        self.request_header.as_ref().map_or(b\"\", |r| r.raw_path())\n    }\n\n    /// Return the host header of the request. An empty slice will be used if there is no host header\n    pub(super) fn get_host(&self) -> &[u8] {\n        self.request_header\n            .as_ref()\n            .and_then(|h| h.headers.get(header::HOST))\n            .map_or(b\"\", |h| h.as_bytes())\n    }\n\n    /// Return a string `$METHOD $PATH $HOST`. Mostly for logging and debug purpose\n    pub fn request_summary(&self) -> String {\n        format!(\n            \"{} {}, Host: {}\",\n            self.get_method().map_or(\"-\", |r| r.as_str()),\n            String::from_utf8_lossy(self.get_path()),\n            String::from_utf8_lossy(self.get_host())\n        )\n    }\n\n    /// Is the request a upgrade request\n    pub fn is_upgrade_req(&self) -> bool {\n        match self.request_header.as_deref() {\n            Some(req) => is_upgrade_req(req),\n            None => false,\n        }\n    }\n\n    /// Get the request header as raw bytes, `b\"\"` when the header doesn't exist\n    pub fn get_header_bytes(&self, name: impl AsHeaderName) -> &[u8] {\n        self.get_header(name).map_or(b\"\", |v| v.as_bytes())\n    }\n\n    /// Read the request body. `Ok(None)` when there is no (more) body to read.\n    pub async fn read_body_bytes(&mut self) -> Result<Option<Bytes>> {\n        let read = self.read_body().await?;\n        Ok(read.map(|b| {\n            let bytes = Bytes::copy_from_slice(self.get_body(&b));\n            if let Some(buffer) = self.retry_buffer.as_mut() {\n                buffer.write_to_buffer(&bytes);\n            }\n            bytes\n        }))\n    }\n\n    async fn do_read_body(&mut self) -> Result<Option<BufRef>> {\n        self.init_body_reader();\n        self.body_reader\n            .read_body(&mut self.underlying_stream)\n            .await\n    }\n\n    /// Read the body into the internal buffer\n    async fn read_body(&mut self) -> Result<Option<BufRef>> {\n        match self.read_timeout {\n            Some(t) => match timeout(t, self.do_read_body()).await {\n                Ok(res) => res,\n                Err(_) => Error::e_explain(ReadTimedout, format!(\"reading body, timeout: {t:?}\")),\n            },\n            None => self.do_read_body().await,\n        }\n    }\n\n    /// Whether there is no (more) body need to be read.\n    pub fn is_body_done(&mut self) -> bool {\n        self.init_body_reader();\n        self.body_reader.body_done()\n    }\n\n    /// Whether the request has an empty body\n    /// Because HTTP 1.1 clients have to send either `Content-Length` or `Transfer-Encoding` in order\n    /// to signal the server that it will send the body, this function returns accurate results even\n    /// only when the request header is just read.\n    pub fn is_body_empty(&mut self) -> bool {\n        self.init_body_reader();\n        self.body_reader.body_empty()\n    }\n\n    /// Write the response header to the client.\n    /// This function can be called more than once to send 1xx informational headers excluding 101.\n    pub async fn write_response_header(&mut self, mut header: Box<ResponseHeader>) -> Result<()> {\n        if let Some(resp) = self.response_written.as_ref() {\n            if !resp.status.is_informational() {\n                warn!(\"Respond header is already sent, cannot send again\");\n                return Ok(());\n            }\n        }\n\n        // no need to add these headers to 1xx responses\n        if !header.status.is_informational() && self.update_resp_headers {\n            /* update headers */\n            header.insert_header(header::DATE, date::get_cached_date())?;\n\n            // TODO: make these lazy static\n            let connection_value = if self.will_keepalive() {\n                \"keep-alive\"\n            } else {\n                \"close\"\n            };\n            header.insert_header(header::CONNECTION, connection_value)?;\n        }\n\n        if header.status.as_u16() == 101 {\n            // make sure the connection is closed at the end when 101/upgrade is used\n            self.set_keepalive(None);\n        }\n\n        // Allow informational header (excluding 101) to pass through without affecting the state\n        // of the request\n        if header.status == 101 || !header.status.is_informational() {\n            // reset request body to done for incomplete upgrade handshakes\n            if let Some(upgrade_ok) = self.is_upgrade(&header) {\n                if upgrade_ok {\n                    debug!(\"ok upgrade handshake\");\n                    // For ws we use HTTP1_0 do_read_body_until_closed\n                    //\n                    // On ws close the initiator sends a close frame and\n                    // then waits for a response from the peer, once it receives\n                    // a response it closes the conn. After receiving a\n                    // control frame indicating the connection should be closed,\n                    // a peer discards any further data received.\n                    // https://www.rfc-editor.org/rfc/rfc6455#section-1.4\n                    self.upgraded = true;\n                } else {\n                    debug!(\"bad upgrade handshake!\");\n                    // reset request body buf and mark as done\n                    // safe to reset an upgrade because it doesn't have body\n                    self.body_reader.init_content_length(0, b\"\");\n                }\n            }\n            self.init_body_writer(&header);\n        }\n\n        // Don't have to flush response with content length because it is less\n        // likely to be real time communication. So do flush when\n        // 1.1xx response: client needs to see it before the rest of response\n        // 2.No content length: the response could be generated in real time\n        let flush = header.status.is_informational()\n            || header.headers.get(header::CONTENT_LENGTH).is_none();\n\n        let mut write_buf = BytesMut::with_capacity(INIT_HEADER_BUF_SIZE);\n        http_resp_header_to_buf(&header, &mut write_buf).unwrap();\n        match self.underlying_stream.write_all(&write_buf).await {\n            Ok(()) => {\n                // flush the stream if 1xx header or there is no response body\n                if flush || self.body_writer.finished() {\n                    self.underlying_stream\n                        .flush()\n                        .await\n                        .or_err(WriteError, \"flushing response header\")?;\n                }\n                self.response_written = Some(header);\n                self.body_bytes_sent += write_buf.len();\n                Ok(())\n            }\n            Err(e) => Error::e_because(WriteError, \"writing response header\", e),\n        }\n    }\n\n    /// Return the response header if it is already sent.\n    pub fn response_written(&self) -> Option<&ResponseHeader> {\n        self.response_written.as_deref()\n    }\n\n    /// `Some(true)` if the this is a successful upgrade\n    /// `Some(false)` if the request is an upgrade but the response refuses it\n    /// `None` if the request is not an upgrade.\n    pub fn is_upgrade(&self, header: &ResponseHeader) -> Option<bool> {\n        if self.is_upgrade_req() {\n            Some(is_upgrade_resp(header))\n        } else {\n            None\n        }\n    }\n\n    fn set_keepalive(&mut self, seconds: Option<u64>) {\n        match seconds {\n            Some(sec) => {\n                if sec > 0 {\n                    self.keepalive_timeout = KeepaliveStatus::Timeout(Duration::from_secs(sec));\n                } else {\n                    self.keepalive_timeout = KeepaliveStatus::Infinite;\n                }\n            }\n            None => {\n                self.keepalive_timeout = KeepaliveStatus::Off;\n            }\n        }\n    }\n\n    /// Return whether the session will be keepalived for connection reuse.\n    pub fn will_keepalive(&self) -> bool {\n        // TODO: check self.body_writer. If it is http1.0 type then keepalive\n        // cannot be used because the connection close is the signal of end body\n        !matches!(self.keepalive_timeout, KeepaliveStatus::Off)\n    }\n\n    // `Keep-Alive: timeout=5, max=1000` => 5, 1000\n    fn get_keepalive_values(&self) -> (Option<u64>, Option<usize>) {\n        // TODO: implement this parsing\n        (None, None)\n    }\n\n    fn is_connection_keepalive(&self) -> Option<bool> {\n        is_buf_keepalive(self.get_header(header::CONNECTION).map(|v| v.as_bytes()))\n    }\n\n    /// Apply keepalive settings according to the client\n    /// For HTTP 1.1, assume keepalive as long as there is no `Connection: Close` request header.\n    /// For HTTP 1.0, only keepalive if there is an explicit header `Connection: keep-alive`.\n    pub fn respect_keepalive(&mut self) {\n        if let Some(keepalive) = self.is_connection_keepalive() {\n            if keepalive {\n                let (timeout, _max_use) = self.get_keepalive_values();\n                // TODO: respect max_use\n                match timeout {\n                    Some(d) => self.set_keepalive(Some(d)),\n                    None => self.set_keepalive(Some(0)), // infinite\n                }\n            } else {\n                self.set_keepalive(None);\n            }\n        } else if self.req_header().version == Version::HTTP_11 {\n            self.set_keepalive(Some(0)); // on by default for http 1.1\n        } else {\n            self.set_keepalive(None); // off by default for http 1.0\n        }\n    }\n\n    fn init_body_writer(&mut self, header: &ResponseHeader) {\n        use http::StatusCode;\n        /* the following responses don't have body 204, 304, and HEAD */\n        if matches!(\n            header.status,\n            StatusCode::NO_CONTENT | StatusCode::NOT_MODIFIED\n        ) || self.get_method() == Some(&Method::HEAD)\n        {\n            self.body_writer.init_content_length(0);\n            return;\n        }\n\n        if header.status.is_informational() && header.status != StatusCode::SWITCHING_PROTOCOLS {\n            // 1xx response, not enough to init body\n            return;\n        }\n\n        if self.is_upgrade(header) == Some(true) {\n            self.body_writer.init_http10();\n        } else {\n            init_body_writer_comm(&mut self.body_writer, &header.headers);\n        }\n    }\n\n    /// Same as [`Self::write_response_header()`] but takes a reference.\n    pub async fn write_response_header_ref(&mut self, resp: &ResponseHeader) -> Result<()> {\n        self.write_response_header(Box::new(resp.clone())).await\n    }\n\n    async fn do_write_body(&mut self, buf: &[u8]) -> Result<Option<usize>> {\n        let written = self\n            .body_writer\n            .write_body(&mut self.underlying_stream, buf)\n            .await;\n\n        if let Ok(Some(num_bytes)) = written {\n            self.body_bytes_sent += num_bytes;\n        }\n\n        written\n    }\n\n    /// Write response body to the client. Return `Ok(None)` when there shouldn't be more body\n    /// to be written, e.g., writing more bytes than what the `Content-Length` header suggests\n    pub async fn write_body(&mut self, buf: &[u8]) -> Result<Option<usize>> {\n        // TODO: check if the response header is written\n        match self.write_timeout {\n            Some(t) => match timeout(t, self.do_write_body(buf)).await {\n                Ok(res) => res,\n                Err(_) => Error::e_explain(WriteTimedout, format!(\"writing body, timeout: {t:?}\")),\n            },\n            None => self.do_write_body(buf).await,\n        }\n    }\n\n    async fn write_body_buf(&mut self) -> Result<Option<usize>> {\n        // Don't flush empty chunks, they are considered end of body for chunks\n        if self.body_write_buf.is_empty() {\n            return Ok(None);\n        }\n\n        let written = self\n            .body_writer\n            .write_body(&mut self.underlying_stream, &self.body_write_buf)\n            .await;\n\n        if let Ok(Some(num_bytes)) = written {\n            self.body_bytes_sent += num_bytes;\n        }\n\n        // make sure this buf is safe to reuse\n        self.body_write_buf.clear();\n\n        written\n    }\n\n    fn maybe_force_close_body_reader(&mut self) {\n        if self.upgraded && !self.body_reader.body_done() {\n            // response is done, reset the request body to close\n            self.body_reader.init_content_length(0, b\"\");\n        }\n    }\n\n    /// Signal that there is no more body to write.\n    /// This call will try to flush the buffer if there is any un-flushed data.\n    /// For chunked encoding response, this call will also send the last chunk.\n    /// For upgraded sessions, this call will also close the reading of the client body.\n    pub async fn finish_body(&mut self) -> Result<Option<usize>> {\n        let res = self.body_writer.finish(&mut self.underlying_stream).await?;\n        self.underlying_stream\n            .flush()\n            .await\n            .or_err(WriteError, \"flushing body\")?;\n\n        self.maybe_force_close_body_reader();\n        Ok(res)\n    }\n\n    /// Return how many (application, not wire) body bytes that have been written\n    pub fn body_bytes_sent(&self) -> usize {\n        self.body_bytes_sent\n    }\n\n    fn is_chunked_encoding(&self) -> bool {\n        is_header_value_chunked_encoding(self.get_header(header::TRANSFER_ENCODING))\n    }\n\n    fn get_content_length(&self) -> Option<usize> {\n        buf_to_content_length(\n            self.get_header(header::CONTENT_LENGTH)\n                .map(|v| v.as_bytes()),\n        )\n    }\n\n    fn init_body_reader(&mut self) {\n        if self.body_reader.need_init() {\n            // reset retry buffer\n            if let Some(buffer) = self.retry_buffer.as_mut() {\n                buffer.clear();\n            }\n\n            /* follow https://tools.ietf.org/html/rfc7230#section-3.3.3 */\n            let preread_body = self.preread_body.as_ref().unwrap().get(&self.buf[..]);\n\n            if self.req_header().version == Version::HTTP_11 && self.is_upgrade_req() {\n                self.body_reader.init_http10(preread_body);\n                return;\n            }\n\n            if self.is_chunked_encoding() {\n                // if chunked encoding, content-length should be ignored\n                self.body_reader.init_chunked(preread_body);\n            } else {\n                let cl = self.get_content_length();\n                match cl {\n                    Some(i) => {\n                        self.body_reader.init_content_length(i, preread_body);\n                    }\n                    None => {\n                        match self.req_header().version {\n                            Version::HTTP_11 => {\n                                // Per RFC assume no body by default in HTTP 1.1\n                                self.body_reader.init_content_length(0, preread_body);\n                            }\n                            _ => {\n                                self.body_reader.init_http10(preread_body);\n                            }\n                        }\n                    }\n                }\n            }\n        }\n    }\n\n    pub fn retry_buffer_truncated(&self) -> bool {\n        self.retry_buffer\n            .as_ref()\n            .map_or_else(|| false, |r| r.is_truncated())\n    }\n\n    pub fn enable_retry_buffering(&mut self) {\n        if self.retry_buffer.is_none() {\n            self.retry_buffer = Some(FixedBuffer::new(BODY_BUF_LIMIT))\n        }\n    }\n\n    pub fn get_retry_buffer(&self) -> Option<Bytes> {\n        self.retry_buffer.as_ref().and_then(|b| {\n            if b.is_truncated() {\n                None\n            } else {\n                b.get_buffer()\n            }\n        })\n    }\n\n    fn get_body(&self, buf_ref: &BufRef) -> &[u8] {\n        // TODO: these get_*() could panic. handle them better\n        self.body_reader.get_body(buf_ref)\n    }\n\n    /// This function will (async) block forever until the client closes the connection.\n    pub async fn idle(&mut self) -> Result<usize> {\n        // NOTE: this implementation breaks http pipelining, ideally we need poll_error\n        // NOTE: buf cannot be empty, openssl-rs read() requires none empty buf.\n        let mut buf: [u8; 1] = [0; 1];\n        self.underlying_stream\n            .read(&mut buf)\n            .await\n            .or_err(ReadError, \"during HTTP idle state\")\n    }\n\n    /// This function will return body bytes (same as [`Self::read_body_bytes()`]), but after\n    /// the client body finishes (`Ok(None)` is returned), calling this function again will block\n    /// forever, same as [`Self::idle()`].\n    pub async fn read_body_or_idle(&mut self, no_body_expected: bool) -> Result<Option<Bytes>> {\n        if no_body_expected || self.is_body_done() {\n            let read = self.idle().await?;\n            if read == 0 {\n                Error::e_explain(\n                    ConnectionClosed,\n                    if self.response_written.is_none() {\n                        \"Prematurely before response header is sent\"\n                    } else {\n                        \"Prematurely before response body is complete\"\n                    },\n                )\n            } else {\n                Error::e_explain(ConnectError, \"Sent data after end of body\")\n            }\n        } else {\n            self.read_body_bytes().await\n        }\n    }\n\n    /// Return the raw bytes of the request header.\n    pub fn get_headers_raw_bytes(&self) -> Bytes {\n        self.raw_header.as_ref().unwrap().get_bytes(&self.buf)\n    }\n\n    /// Close the connection abruptly. This allows to signal the client that the connection is closed\n    /// before dropping [`HttpSession`]\n    pub async fn shutdown(&mut self) {\n        let _ = self.underlying_stream.shutdown().await;\n    }\n\n    /// Set the server keepalive timeout.\n    /// `None`: disable keepalive, this session cannot be reused.\n    /// `Some(0)`: reusing this session is allowed and there is no timeout.\n    /// `Some(>0)`: reusing this session is allowed within the given timeout in seconds.\n    /// If the client disallows connection reuse, then `keepalive` will be ignored.\n    pub fn set_server_keepalive(&mut self, keepalive: Option<u64>) {\n        if let Some(false) = self.is_connection_keepalive() {\n            // connection: close is set\n            self.set_keepalive(None);\n        } else {\n            self.set_keepalive(keepalive);\n        }\n    }\n\n    /// Return the [Digest] of the connection.\n    pub fn digest(&self) -> &Digest {\n        &self.digest\n    }\n\n    /// Return the client (peer) address of the underlying connnection.\n    pub fn client_addr(&self) -> Option<&SocketAddr> {\n        self.digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.peer_addr())?\n    }\n\n    /// Return the server (local) address of the underlying connnection.\n    pub fn server_addr(&self) -> Option<&SocketAddr> {\n        self.digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.local_addr())?\n    }\n\n    /// Consume `self`, if the connection can be reused, the underlying stream will be returned\n    /// to be fed to the next [`Self::new()`]. The next session can just call [`Self::read_request()`].\n    /// If the connection cannot be reused, the underlying stream will be closed and `None` will be\n    /// returned.\n    pub async fn reuse(mut self) -> Option<Stream> {\n        // TODO: this function is unnecessarily slow for keepalive case\n        // because that case does not need async\n        match self.keepalive_timeout {\n            KeepaliveStatus::Off => {\n                debug!(\"HTTP shutdown connection\");\n                self.shutdown().await;\n                None\n            }\n            _ => Some(self.underlying_stream),\n        }\n    }\n\n    /// Return a error response to the client. This default error response comes with `cache-control: private, no-store`.\n    /// It has no response body.\n    pub async fn respond_error(&mut self, error_status_code: u16) {\n        let (resp, resp_tmp) = match error_status_code {\n            /* commmon error responses are pre-generated */\n            502 => (Some(&*error_resp::HTTP_502_RESPONSE), None),\n            400 => (Some(&*error_resp::HTTP_400_RESPONSE), None),\n            _ => (\n                None,\n                Some(error_resp::gen_error_response(error_status_code)),\n            ),\n        };\n\n        let resp = match resp {\n            Some(r) => r,\n            None => resp_tmp.as_ref().unwrap(),\n        };\n\n        self.write_response_header_ref(resp)\n            .await\n            .unwrap_or_else(|e| {\n                error!(\"failed to send error response to downstream: {}\", e);\n            });\n    }\n\n    /// Write a `100 Continue` response to the client.\n    pub async fn write_continue_response(&mut self) -> Result<()> {\n        // only send if we haven't already\n        if self.response_written.is_none() {\n            // size hint Some(0) because default is 8\n            return self\n                .write_response_header(Box::new(ResponseHeader::build(100, Some(0)).unwrap()))\n                .await;\n        }\n        Ok(())\n    }\n\n    async fn response_duplex(&mut self, task: HttpTask) -> Result<bool> {\n        match task {\n            HttpTask::Header(header, end_stream) => {\n                self.write_response_header(header)\n                    .await\n                    .map_err(|e| e.into_down())?;\n                Ok(end_stream)\n            }\n            HttpTask::Body(data, end_stream) => match data {\n                Some(d) => {\n                    if !d.is_empty() {\n                        self.write_body(&d).await.map_err(|e| e.into_down())?;\n                    }\n                    Ok(end_stream)\n                }\n                None => Ok(end_stream),\n            },\n            HttpTask::Trailer(_) => Ok(true), // h1 trailer is not supported yet\n            HttpTask::Done => {\n                self.finish_body().await.map_err(|e| e.into_down())?;\n                Ok(true)\n            }\n            HttpTask::Failed(e) => Err(e),\n        }\n    }\n\n    // TODO: use vectored write to avoid copying\n    pub async fn response_duplex_vec(&mut self, mut tasks: Vec<HttpTask>) -> Result<bool> {\n        let n_tasks = tasks.len();\n        if n_tasks == 1 {\n            // fallback to single operation to avoid copy\n            return self.response_duplex(tasks.pop().unwrap()).await;\n        }\n        let mut end_stream = false;\n        for task in tasks.into_iter() {\n            end_stream = match task {\n                HttpTask::Header(header, end_stream) => {\n                    self.write_response_header(header)\n                        .await\n                        .map_err(|e| e.into_down())?;\n                    end_stream\n                }\n                HttpTask::Body(data, end_stream) => match data {\n                    Some(d) => {\n                        if !d.is_empty() && !self.body_writer.finished() {\n                            self.body_write_buf.put_slice(&d);\n                        }\n                        end_stream\n                    }\n                    None => end_stream,\n                },\n                HttpTask::Trailer(_) => true, // h1 trailer is not supported yet\n                HttpTask::Done => {\n                    // flush body first\n                    self.write_body_buf().await.map_err(|e| e.into_down())?;\n                    self.finish_body().await.map_err(|e| e.into_down())?;\n                    return Ok(true);\n                }\n                HttpTask::Failed(e) => {\n                    // flush the data we have and quit\n                    self.write_body_buf().await.map_err(|e| e.into_down())?;\n                    self.underlying_stream\n                        .flush()\n                        .await\n                        .or_err(WriteError, \"flushing response\")?;\n                    return Err(e);\n                }\n            }\n        }\n        self.write_body_buf().await.map_err(|e| e.into_down())?;\n        Ok(end_stream)\n    }\n}\n\n// Regex to parse request line that has illegal chars in it\nstatic REQUEST_LINE_REGEX: Lazy<Regex> =\n    Lazy::new(|| Regex::new(r\"^\\w+ (?P<uri>.+) HTTP/\\d(?:\\.\\d)?\").unwrap());\n\n// the chars httparse considers illegal in URL\n// Almost https://url.spec.whatwg.org/#query-percent-encode-set + {}\nconst URI_ESC_CHARSET: &AsciiSet = &CONTROLS.add(b' ').add(b'<').add(b'>').add(b'\"');\n\nfn escape_illegal_request_line(buf: &BytesMut) -> Option<BytesMut> {\n    if let Some(captures) = REQUEST_LINE_REGEX.captures(buf) {\n        // return if nothing matches: not a request line at all\n        let uri = captures.name(\"uri\")?;\n\n        let escaped_uri = percent_encode(uri.as_bytes(), URI_ESC_CHARSET);\n\n        // rebuild the entire request buf in a new buffer\n        // TODO: this might be able to be done in place\n\n        // need to be slightly bigger than the current buf;\n        let mut new_buf = BytesMut::with_capacity(buf.len() + 32);\n        new_buf.extend_from_slice(&buf[..uri.start()]);\n\n        for s in escaped_uri {\n            new_buf.extend_from_slice(s.as_bytes());\n        }\n\n        if new_buf.len() == uri.end() {\n            // buf unchanged, nothing is escaped, return None to avoid loop\n            return None;\n        }\n\n        new_buf.extend_from_slice(&buf[uri.end()..]);\n\n        Some(new_buf)\n    } else {\n        None\n    }\n}\n\n#[inline]\nfn parse_req_buffer<'buf>(\n    req: &mut httparse::Request<'_, 'buf>,\n    buf: &'buf [u8],\n) -> HeaderParseState {\n    use httparse::Result;\n\n    #[cfg(feature = \"patched_http1\")]\n    fn parse<'buf>(req: &mut httparse::Request<'_, 'buf>, buf: &'buf [u8]) -> Result<usize> {\n        req.parse_unchecked(buf)\n    }\n\n    #[cfg(not(feature = \"patched_http1\"))]\n    fn parse<'buf>(req: &mut httparse::Request<'_, 'buf>, buf: &'buf [u8]) -> Result<usize> {\n        req.parse(buf)\n    }\n\n    let res = match parse(req, buf) {\n        Ok(s) => s,\n        Err(e) => {\n            return HeaderParseState::Invalid(e);\n        }\n    };\n    match res {\n        httparse::Status::Complete(s) => HeaderParseState::Complete(s),\n        _ => HeaderParseState::Partial,\n    }\n}\n\n#[inline]\nfn http_resp_header_to_buf(\n    resp: &ResponseHeader,\n    buf: &mut BytesMut,\n) -> std::result::Result<(), ()> {\n    // Status-Line\n    let version = match resp.version {\n        Version::HTTP_09 => \"HTTP/0.9 \",\n        Version::HTTP_10 => \"HTTP/1.0 \",\n        Version::HTTP_11 => \"HTTP/1.1 \",\n        _ => {\n            return Err(()); /*TODO: unsupported version */\n...\n", "needle_token_start": 4001, "needle_token_end": 4098, "code_context_ntokens": 16224, "output": ["```rust\npub fn write_to_buffer(&mut self, data: &Bytes) {\n    if !self.truncated && (self.buffer.len() + data.len() <= self.capacity) {\n        self.buffer.extend_from_slice(data);\n    } else {\n        // TODO: clear data because the data held here is useless anyway?\n        self.truncated = true;\n    }\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "test_connect_h1_plaintext", "language": "rust", "path": "pingora-core/src/connectors/http/v2.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To verify that the system correctly establishes an HTTP/1 connection using plaintext (non-SSL) when explicitly configured to do so, despite an attempt to set an incompatible HTTP version.\n2. **Input**: A configuration for a peer specifying an IP address, port number for HTTP (not HTTPS), and an empty domain string, with the HTTP version set to an incompatible setting for plaintext.\n3. **Output**: The test checks that the session established is indeed HTTP/1, and not HTTP/2, which would be incorrect under the given settings.\n4. **Procedure**: The test initializes a connection setup with specific parameters (IP, port, and domain). It then forces the HTTP version to an incompatible setting to test the system's handling of such a scenario. The session type is then checked to ensure it is HTTP/1, and if not, the test fails, indicating an error in protocol handling or session establishment logic.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/connectors/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Connecting to servers\n\npub mod http;\nmod l4;\nmod offload;\nmod tls;\n\nuse crate::protocols::Stream;\nuse crate::server::configuration::ServerConf;\nuse crate::tls::ssl::SslConnector;\nuse crate::upstreams::peer::{Peer, ALPN};\n\nuse l4::connect as l4_connect;\nuse log::{debug, error, warn};\nuse offload::OffloadRuntime;\nuse parking_lot::RwLock;\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_pool::{ConnectionMeta, ConnectionPool};\nuse std::collections::HashMap;\nuse std::net::SocketAddr;\nuse std::sync::Arc;\nuse tokio::sync::Mutex;\n\n/// The options to configure a [TransportConnector]\n#[derive(Clone)]\npub struct ConnectorOptions {\n    /// Path to the CA file used to validate server certs.\n    ///\n    /// If `None`, the CA in the [default](https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_default_verify_paths.html)\n    /// locations will be loaded\n    pub ca_file: Option<String>,\n    /// The default client cert and key to use for mTLS\n    ///\n    /// Each individual connection can use their own cert key to override this.\n    pub cert_key_file: Option<(String, String)>,\n    /// How many connections to keepalive\n    pub keepalive_pool_size: usize,\n...\n// Path: pingora-core/src/connectors/http/v2.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse super::HttpSession;\nuse crate::connectors::{ConnectorOptions, TransportConnector};\nuse crate::protocols::http::v1::client::HttpSession as Http1Session;\nuse crate::protocols::http::v2::client::{drive_connection, Http2Session};\nuse crate::protocols::{Digest, Stream};\nuse crate::upstreams::peer::{Peer, ALPN};\n\nuse bytes::Bytes;\nuse h2::client::SendRequest;\nuse log::debug;\nuse parking_lot::RwLock;\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_pool::{ConnectionMeta, ConnectionPool, PoolNode};\nuse std::collections::HashMap;\nuse std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};\nuse std::sync::Arc;\nuse std::time::Duration;\nuse tokio::sync::watch;\n\nstruct Stub(SendRequest<Bytes>);\n\nimpl Stub {\n    async fn new_stream(&self) -> Result<SendRequest<Bytes>> {\n        let send_req = self.0.clone();\n        send_req\n            .ready()\n            .await\n            .or_err(H2Error, \"while creating new stream\")\n    }\n}\n\npub(crate) struct ConnectionRefInner {\n    connection_stub: Stub,\n    closed: watch::Receiver<bool>,\n    ping_timeout_occurred: Arc<AtomicBool>,\n    id: i32,\n    // max concurrent streams this connection is allowed to create\n    max_streams: usize,\n    // how many concurrent streams already active\n    current_streams: AtomicUsize,\n    // because `SendRequest` doesn't actually have access to the underlying Stream,\n    // we log info about timing and tcp info here.\n    pub(crate) digest: Digest,\n}\n\n#[derive(Clone)]\npub(crate) struct ConnectionRef(Arc<ConnectionRefInner>);\n\nimpl ConnectionRef {\n    pub fn new(\n        send_req: SendRequest<Bytes>,\n        closed: watch::Receiver<bool>,\n        ping_timeout_occurred: Arc<AtomicBool>,\n        id: i32,\n        max_streams: usize,\n        digest: Digest,\n    ) -> Self {\n        ConnectionRef(Arc::new(ConnectionRefInner {\n            connection_stub: Stub(send_req),\n            closed,\n            ping_timeout_occurred,\n            id,\n            max_streams,\n            current_streams: AtomicUsize::new(0),\n            digest,\n        }))\n    }\n    pub fn more_streams_allowed(&self) -> bool {\n        self.0.max_streams > self.0.current_streams.load(Ordering::Relaxed)\n    }\n\n    pub fn is_idle(&self) -> bool {\n        self.0.current_streams.load(Ordering::Relaxed) == 0\n    }\n\n    pub fn release_stream(&self) {\n        self.0.current_streams.fetch_sub(1, Ordering::SeqCst);\n    }\n\n    pub fn id(&self) -> i32 {\n        self.0.id\n    }\n\n    pub fn digest(&self) -> &Digest {\n        &self.0.digest\n    }\n\n    pub fn ping_timedout(&self) -> bool {\n        self.0.ping_timeout_occurred.load(Ordering::Relaxed)\n    }\n\n    pub fn is_closed(&self) -> bool {\n        *self.0.closed.borrow()\n    }\n\n    // spawn a stream if more stream is allowed, otherwise return Ok(None)\n    pub async fn spawn_stream(&self) -> Result<Option<Http2Session>> {\n        // Atomically check if the current_stream is over the limit\n        // load(), compare and then fetch_add() cannot guarantee the same\n        let current_streams = self.0.current_streams.fetch_add(1, Ordering::SeqCst);\n        if current_streams >= self.0.max_streams {\n            // already over the limit, reset the counter to the previous value\n            self.0.current_streams.fetch_sub(1, Ordering::SeqCst);\n            return Ok(None);\n        }\n        let send_req = self.0.connection_stub.new_stream().await.map_err(|e| {\n            // fail to create the stream, reset the counter\n            self.0.current_streams.fetch_sub(1, Ordering::SeqCst);\n            e\n        })?;\n\n        Ok(Some(Http2Session::new(send_req, self.clone())))\n    }\n}\n\nstruct InUsePool {\n    // TODO: use pingora hashmap to shard the lock contention\n    pools: RwLock<HashMap<u64, PoolNode<ConnectionRef>>>,\n}\n\nimpl InUsePool {\n    fn new() -> Self {\n        InUsePool {\n            pools: RwLock::new(HashMap::new()),\n        }\n    }\n\n    fn insert(&self, reuse_hash: u64, conn: ConnectionRef) {\n        {\n            let pools = self.pools.read();\n            if let Some(pool) = pools.get(&reuse_hash) {\n                pool.insert(conn.id(), conn);\n                return;\n            }\n        } // drop read lock\n\n        let pool = PoolNode::new();\n        pool.insert(conn.id(), conn);\n        let mut pools = self.pools.write();\n        pools.insert(reuse_hash, pool);\n    }\n\n    // retrieve a h2 conn ref to create a new stream\n    // the caller should return the conn ref to this pool if there are still\n    // capacity left for more streams\n    fn get(&self, reuse_hash: u64) -> Option<ConnectionRef> {\n        let pools = self.pools.read();\n        pools.get(&reuse_hash)?.get_any().map(|v| v.1)\n    }\n\n    // release a h2_stream, this functional will cause an ConnectionRef to be returned (if exist)\n    // the caller should update the ref and then decide where to put it (in use pool or idle)\n    fn release(&self, reuse_hash: u64, id: i32) -> Option<ConnectionRef> {\n        let pools = self.pools.read();\n        if let Some(pool) = pools.get(&reuse_hash) {\n            pool.remove(id)\n        } else {\n            None\n        }\n    }\n}\n\nconst DEFAULT_POOL_SIZE: usize = 128;\n\n/// Http2 connector\npub struct Connector {\n    // just for creating connections, the Stream of h2 should be reused\n    transport: TransportConnector,\n    // the h2 connection idle pool\n    idle_pool: Arc<ConnectionPool<ConnectionRef>>,\n    // the pool of h2 connections that have ongoing streams\n    in_use_pool: InUsePool,\n}\n\nimpl Connector {\n    /// Create a new [Connector] from the given [ConnectorOptions]\n    pub fn new(options: Option<ConnectorOptions>) -> Self {\n        let pool_size = options\n            .as_ref()\n            .map_or(DEFAULT_POOL_SIZE, |o| o.keepalive_pool_size);\n        // connection offload is handled by the [TransportConnector]\n        Connector {\n            transport: TransportConnector::new(options),\n            idle_pool: Arc::new(ConnectionPool::new(pool_size)),\n            in_use_pool: InUsePool::new(),\n        }\n    }\n\n    /// Create a new Http2 connection to the given server\n    ///\n    /// Either an Http2 or Http1 session can be returned depending on the server's preference.\n    pub async fn new_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<HttpSession> {\n        let stream = self.transport.new_stream(peer).await?;\n\n        // check alpn\n        match stream.selected_alpn_proto() {\n            Some(ALPN::H2) => { /* continue */ }\n            Some(_) => {\n                // H2 not supported\n                return Ok(HttpSession::H1(Http1Session::new(stream)));\n            }\n            None => {\n                // if tls but no ALPN, default to h1\n                // else if plaintext and min http version is 1, this is most likely h1\n                if peer.tls()\n                    || peer\n                        .get_peer_options()\n                        .map_or(true, |o| o.alpn.get_min_http_version() == 1)\n                {\n                    return Ok(HttpSession::H1(Http1Session::new(stream)));\n                }\n                // else: min http version=H2 over plaintext, there is no ALPN anyways, we trust\n                // the caller that the server speaks h2c\n            }\n        }\n        let max_h2_stream = peer.get_peer_options().map_or(1, |o| o.max_h2_streams);\n        let conn = handshake(stream, max_h2_stream, peer.h2_ping_interval()).await?;\n        let h2_stream = conn\n            .spawn_stream()\n            .await?\n            .expect(\"newly created connections should have at least one free stream\");\n        if conn.more_streams_allowed() {\n            self.in_use_pool.insert(peer.reuse_hash(), conn);\n        }\n        Ok(HttpSession::H2(h2_stream))\n    }\n\n    /// Try to create a new http2 stream from any existing H2 connection.\n    ///\n    /// None means there is no \"free\" connection left.\n    pub async fn reused_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<Option<Http2Session>> {\n        // check in use pool first so that we use fewer total connections\n        // then idle pool\n        let reuse_hash = peer.reuse_hash();\n\n        // NOTE: We grab a conn from the pools, create a new stream and put the conn back if the\n        // conn has more free streams. During this process another caller could arrive but is not\n        // able to find the conn even the conn has free stream to use.\n        // We accept this false negative to keep the implementation simple. This false negative\n        // makes an actual impact when there are only a few connection.\n        // Alternative design 1. given each free stream a conn object: a lot of Arc<>\n        // Alternative design 2. mutex the pool, which creates lock contention when concurrency is high\n        // Alternative design 3. do not pop conn from the pool so that multiple callers can grab it\n        // which will cause issue where spawn_stream() could return None because others call it\n        // first. Thus a caller might have to retry or give up. This issue is more likely to happen\n        // when concurrency is high.\n        let maybe_conn = self\n            .in_use_pool\n            .get(reuse_hash)\n            .or_else(|| self.idle_pool.get(&reuse_hash));\n        if let Some(conn) = maybe_conn {\n            let h2_stream = conn\n                .spawn_stream()\n                .await?\n                .expect(\"connection from the pools should have free stream to allocate\");\n            if conn.more_streams_allowed() {\n                self.in_use_pool.insert(reuse_hash, conn);\n            }\n            Ok(Some(h2_stream))\n        } else {\n            Ok(None)\n        }\n    }\n\n    /// Release a finished h2 stream.\n    ///\n    /// This function will terminate the [Http2Session]. The corresponding h2 connection will now\n    /// have one more free stream to use.\n    ///\n    /// The h2 connection will be closed after `idle_timeout` if it has no active streams.\n    pub fn release_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        session: Http2Session,\n        peer: &P,\n        idle_timeout: Option<Duration>,\n    ) {\n        let id = session.conn.id();\n        let reuse_hash = peer.reuse_hash();\n        // get a ref to the connection, which we might need below, before dropping the h2\n        let conn = session.conn();\n        // this drop() will both drop the actual stream and call the conn.release_stream()\n        drop(session);\n        // find and remove the conn stored in in_use_pool so that it could be put in the idle pool\n        // if necessary\n        let conn = self.in_use_pool.release(reuse_hash, id).unwrap_or(conn);\n        if conn.is_closed() {\n            // Already dead h2 connection\n            return;\n        }\n        if conn.is_idle() {\n            let meta = ConnectionMeta {\n                key: reuse_hash,\n                id,\n            };\n            let closed = conn.0.closed.clone();\n            let (notify_evicted, watch_use) = self.idle_pool.put(&meta, conn);\n            if let Some(to) = idle_timeout {\n                let pool = self.idle_pool.clone(); //clone the arc\n                let rt = pingora_runtime::current_handle();\n                rt.spawn(async move {\n                    pool.idle_timeout(&meta, to, notify_evicted, closed, watch_use)\n                        .await;\n                });\n            }\n        } else {\n            self.in_use_pool.insert(reuse_hash, conn);\n        }\n    }\n\n    /// Tell the connector to always send h1 for ALPN for the given peer in the future.\n    pub fn prefer_h1(&self, peer: &impl Peer) {\n        self.transport.prefer_h1(peer);\n    }\n\n    pub(crate) fn h1_is_preferred(&self, peer: &impl Peer) -> bool {\n        self.transport\n            .preferred_http_version\n            .get(peer)\n            .map_or(false, |v| matches!(v, ALPN::H1))\n    }\n}\n\n// The h2 library we use has unbounded internal buffering, which will cause excessive memory\n// consumption when the downstream is slower than upstream. This window size caps the buffering by\n// limiting how much data can be inflight. However, setting this value will also cap the max\n// download speed by limiting the bandwidth-delay product of a link.\n// Long term, we should advertising large window but shrink it when a small buffer is full.\n// 8 Mbytes = 80 Mbytes X 100ms, which should be enough for most links.\nconst H2_WINDOW_SIZE: u32 = 1 << 23;\n\nasync fn handshake(\n    stream: Stream,\n    max_streams: usize,\n    h2_ping_interval: Option<Duration>,\n) -> Result<ConnectionRef> {\n    use h2::client::Builder;\n    use pingora_runtime::current_handle;\n\n    // Safe guard: new_http_session() assumes there should be at least one free stream\n    if max_streams == 0 {\n        return Error::e_explain(H2Error, \"zero max_stream configured\");\n    }\n\n    let id = stream.id();\n    let digest = Digest {\n        // NOTE: this field is always false because the digest is shared across all streams\n        // The streams should log their own reuse info\n        ssl_digest: stream.get_ssl_digest(),\n        // TODO: log h2 handshake time\n        timing_digest: stream.get_timing_digest(),\n        proxy_digest: stream.get_proxy_digest(),\n        socket_digest: stream.get_socket_digest(),\n    };\n    // TODO: make these configurable\n    let (send_req, connection) = Builder::new()\n        .enable_push(false)\n        .initial_max_send_streams(max_streams)\n        // The limit for the server. Server push is not allowed, so this value doesn't matter\n        .max_concurrent_streams(1)\n        .max_frame_size(64 * 1024) // advise server to send larger frames\n        .initial_window_size(H2_WINDOW_SIZE)\n        // should this be max_streams * H2_WINDOW_SIZE?\n        .initial_connection_window_size(H2_WINDOW_SIZE)\n        .handshake(stream)\n        .await\n        .or_err(HandshakeError, \"during H2 handshake\")?;\n    debug!(\"H2 handshake to server done.\");\n    let ping_timeout_occurred = Arc::new(AtomicBool::new(false));\n    let ping_timeout_clone = ping_timeout_occurred.clone();\n    let max_allowed_streams = std::cmp::min(max_streams, connection.max_concurrent_send_streams());\n\n    // Safe guard: new_http_session() assumes there should be at least one free stream\n    // The server won't commonly advertise 0 max stream.\n    if max_allowed_streams == 0 {\n        return Error::e_explain(H2Error, \"zero max_concurrent_send_streams received\");\n    }\n\n    let (closed_tx, closed_rx) = watch::channel(false);\n\n    current_handle().spawn(async move {\n        drive_connection(\n            connection,\n            id,\n            closed_tx,\n            h2_ping_interval,\n            ping_timeout_clone,\n        )\n        .await;\n    });\n    Ok(ConnectionRef::new(\n        send_req,\n        closed_rx,\n        ping_timeout_occurred,\n        id,\n        max_allowed_streams,\n        digest,\n    ))\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::upstreams::peer::HttpPeer;\n\n    #[tokio::test]\n    async fn test_connect_h2() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        match h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => assert!(!h2_stream.ping_timedout()),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_connect_h1() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        // a hack to force h1, new_http_session() in the future might validate this setting\n        peer.options.set_http_version(1, 1);\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        match h2 {\n            HttpSession::H1(_) => {}\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n    }\n\n    #[tokio::test]\n    \nasync fn test_connect_h1_plaintext() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 80), false, \"\".into());\n        peer.options.set_http_version(2, 1);\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        match h2 {\n            HttpSession::H1(_) => {}\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_h2_single_stream() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        peer.options.max_h2_streams = 1;\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        let h2_1 = match h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => h2_stream,\n        };\n\n        let id = h2_1.conn.id();\n\n        assert!(connector\n            .reused_http_session(&peer)\n            .await\n            .unwrap()\n            .is_none());\n\n        connector.release_http_session(h2_1, &peer, None);\n\n        let h2_2 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_2.conn.id());\n\n        connector.release_http_session(h2_2, &peer, None);\n\n        let h2_3 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_3.conn.id());\n    }\n\n    #[tokio::test]\n    async fn test_h2_multiple_stream() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        peer.options.max_h2_streams = 3;\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        let h2_1 = match h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => h2_stream,\n        };\n\n        let id = h2_1.conn.id();\n\n        let h2_2 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_2.conn.id());\n        let h2_3 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_3.conn.id());\n\n        // max stream is 3 for now\n        assert!(connector\n            .reused_http_session(&peer)\n            .await\n            .unwrap()\n            .is_none());\n\n        connector.release_http_session(h2_1, &peer, None);\n\n        let h2_4 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_4.conn.id());\n\n        connector.release_http_session(h2_2, &peer, None);\n        connector.release_http_session(h2_3, &peer, None);\n        connector.release_http_session(h2_4, &peer, None);\n\n        // all streams are released, now the connection is idle\n        let h2_5 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_5.conn.id());\n    }\n}\n\n// Path: pingora-core/src/protocols/http/v2/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/2 client session and connection\n// TODO: this module needs a refactor\n\nuse bytes::Bytes;\nuse h2::client::{self, ResponseFuture, SendRequest};\nuse h2::{Reason, RecvStream, SendStream};\nuse http::HeaderMap;\nuse log::{debug, error, warn};\nuse pingora_error::{Error, ErrorType, ErrorType::*, OrErr, Result, RetryType};\nuse pingora_http::{RequestHeader, ResponseHeader};\nuse pingora_timeout::timeout;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::Arc;\nuse std::time::Duration;\nuse tokio::io::{AsyncRead, AsyncWrite};\nuse tokio::sync::watch;\n\nuse crate::connectors::http::v2::ConnectionRef;\nuse crate::protocols::{Digest, SocketAddr};\n\npub const PING_TIMEDOUT: ErrorType = ErrorType::new(\"PingTimedout\");\n\npub struct Http2Session {\n    send_req: SendRequest<Bytes>,\n    send_body: Option<SendStream<Bytes>>,\n    resp_fut: Option<ResponseFuture>,\n    req_sent: Option<Box<RequestHeader>>,\n    response_header: Option<ResponseHeader>,\n    response_body_reader: Option<RecvStream>,\n    /// The read timeout, which will be applied to both reading the header and the body.\n    /// The timeout is reset on every read. This is not a timeout on the overall duration of the\n    /// response.\n    pub read_timeout: Option<Duration>,\n    pub(crate) conn: ConnectionRef,\n    // Indicate that whether a END_STREAM is already sent\n    ended: bool,\n}\n\nimpl Drop for Http2Session {\n    fn drop(&mut self) {\n        self.conn.release_stream();\n    }\n}\n\nimpl Http2Session {\n    pub(crate) fn new(send_req: SendRequest<Bytes>, conn: ConnectionRef) -> Self {\n        Http2Session {\n            send_req,\n            send_body: None,\n            resp_fut: None,\n            req_sent: None,\n            response_header: None,\n            response_body_reader: None,\n            read_timeout: None,\n            conn,\n            ended: false,\n        }\n    }\n\n    fn sanitize_request_header(req: &mut RequestHeader) -> Result<()> {\n        req.set_version(http::Version::HTTP_2);\n        if req.uri.authority().is_some() {\n            return Ok(());\n        }\n        // use host header to populate :authority field\n        let Some(authority) = req.headers.get(http::header::HOST).map(|v| v.as_bytes()) else {\n            return Error::e_explain(InvalidHTTPHeader, \"no authority header for h2\");\n        };\n        let uri = http::uri::Builder::new()\n            .scheme(\"https\") // fixed for now\n            .authority(authority)\n            .path_and_query(req.uri.path_and_query().as_ref().unwrap().as_str())\n            .build();\n        match uri {\n            Ok(uri) => {\n                req.set_uri(uri);\n                Ok(())\n            }\n            Err(_) => Error::e_explain(\n                InvalidHTTPHeader,\n                format!(\"invalid authority from host {authority:?}\"),\n            ),\n        }\n    }\n\n    /// Write the request header to the server\n    pub fn write_request_header(&mut self, mut req: Box<RequestHeader>, end: bool) -> Result<()> {\n        if self.req_sent.is_some() {\n            // cannot send again, TODO: warn\n            return Ok(());\n        }\n        Self::sanitize_request_header(&mut req)?;\n        let parts = req.as_owned_parts();\n        let request = http::Request::from_parts(parts, ());\n        // There is no write timeout for h2 because the actual write happens async from this fn\n        let (resp_fut, send_body) = self\n            .send_req\n            .send_request(request, end)\n            .or_err(H2Error, \"while sending request\")\n            .map_err(|e| self.handle_err(e))?;\n        self.req_sent = Some(req);\n        self.send_body = Some(send_body);\n        self.resp_fut = Some(resp_fut);\n        self.ended = self.ended || end;\n\n        Ok(())\n    }\n\n    /// Write a request body chunk\n    pub fn write_request_body(&mut self, data: Bytes, end: bool) -> Result<()> {\n        if self.ended {\n            warn!(\"Try to write request body after end of stream, dropping the extra data\");\n            return Ok(());\n        }\n\n        let body_writer = self\n            .send_body\n            .as_mut()\n            .expect(\"Try to write request body before sending request header\");\n\n        write_body(body_writer, data, end).map_err(|e| self.handle_err(e))?;\n        self.ended = self.ended || end;\n        Ok(())\n    }\n\n    /// Signal that the request body has ended\n    pub fn finish_request_body(&mut self) -> Result<()> {\n        if self.ended {\n            return Ok(());\n        }\n\n        let body_writer = self\n            .send_body\n            .as_mut()\n            .expect(\"Try to finish request stream before sending request header\");\n\n        // Just send an empty data frame with end of stream set\n        body_writer\n            .send_data(\"\".into(), true)\n            .or_err(WriteError, \"while writing empty h2 request body\")\n            .map_err(|e| self.handle_err(e))?;\n        self.ended = true;\n        Ok(())\n    }\n\n    /// Read the response header\n    pub async fn read_response_header(&mut self) -> Result<()> {\n        // TODO: how to read 1xx headers?\n        // https://github.com/hyperium/h2/issues/167\n\n        if self.response_header.is_some() {\n            panic!(\"H2 response header is already read\")\n        }\n\n        let Some(resp_fut) = self.resp_fut.take() else {\n            panic!(\"Try to  response header is already read\")\n        };\n\n        let res = match self.read_timeout {\n            Some(t) => timeout(t, resp_fut)\n                .await\n                .map_err(|_| Error::explain(ReadTimedout, \"while reading h2 response header\"))\n                .map_err(|e| self.handle_err(e))?,\n            None => resp_fut.await,\n        };\n        let (resp, body_reader) = res.map_err(handle_read_header_error)?.into_parts();\n        self.response_header = Some(resp.into());\n        self.response_body_reader = Some(body_reader);\n\n        Ok(())\n    }\n\n    /// Read the response body\n    ///\n    /// `None` means, no more body to read\n    pub async fn read_response_body(&mut self) -> Result<Option<Bytes>> {\n        let Some(body_reader) = self.response_body_reader.as_mut() else {\n            // req is not sent or response is already read\n            // TODO: warn\n            return Ok(None);\n        };\n\n        if body_reader.is_end_stream() {\n            return Ok(None);\n        }\n\n        let fut = body_reader.data();\n        let res = match self.read_timeout {\n            Some(t) => timeout(t, fut)\n                .await\n                .map_err(|_| Error::explain(ReadTimedout, \"while reading h2 response body\"))?,\n            None => fut.await,\n        };\n        let body = res\n            .transpose()\n            .or_err(ReadError, \"while read h2 response body\")\n            .map_err(|mut e| {\n                // cannot use handle_err() because of borrow checker\n                if self.conn.ping_timedout() {\n                    e.etype = PING_TIMEDOUT;\n                }\n                e\n            })?;\n\n        if let Some(data) = body.as_ref() {\n            body_reader\n                .flow_control()\n                .release_capacity(data.len())\n                .or_err(ReadError, \"while releasing h2 response body capacity\")?;\n        }\n\n        Ok(body)\n    }\n\n    /// Whether the response has ended\n    pub fn response_finished(&self) -> bool {\n        // if response_body_reader doesn't exist, the response is not even read yet\n        self.response_body_reader\n            .as_ref()\n            .map_or(false, |reader| reader.is_end_stream())\n    }\n\n    /// Read the optional trailer headers\n    pub async fn read_trailers(&mut self) -> Result<Option<HeaderMap>> {\n        let Some(reader) = self.response_body_reader.as_mut() else {\n            // response is not even read\n            // TODO: warn\n            return Ok(None);\n        };\n        let fut = reader.trailers();\n\n        let res = match self.read_timeout {\n            Some(t) => timeout(t, fut)\n                .await\n                .map_err(|_| Error::explain(ReadTimedout, \"while reading h2 trailer\"))\n                .map_err(|e| self.handle_err(e))?,\n            None => fut.await,\n        };\n        match res {\n            Ok(t) => Ok(t),\n            Err(e) => {\n                // GOAWAY with no error: this is graceful shutdown, continue as if no trailer\n                // RESET_STREAM with no error: https://datatracker.ietf.org/doc/html/rfc9113#section-8.1:\n                // this is to signal client to stop uploading request without breaking the response.\n                // TODO: should actually stop uploading\n                // TODO: should we try reading again?\n                // TODO: handle this when reading headers and body as well\n                // https://github.com/hyperium/h2/issues/741\n\n                if (e.is_go_away() || e.is_reset())\n                    && e.is_remote()\n                    && e.reason() == Some(Reason::NO_ERROR)\n                {\n                    Ok(None)\n                } else {\n                    Err(e)\n                }\n            }\n        }\n        .or_err(ReadError, \"while reading h2 trailers\")\n    }\n\n    /// The response header if it is already read\n    pub fn response_header(&self) -> Option<&ResponseHeader> {\n        self.response_header.as_ref()\n    }\n\n    /// Give up the http session abruptly.\n    pub fn shutdown(&mut self) {\n        if !self.ended || !self.response_finished() {\n            if let Some(send_body) = self.send_body.as_mut() {\n                send_body.send_reset(h2::Reason::INTERNAL_ERROR)\n            }\n        }\n    }\n\n    /// Drop everything in this h2 stream. Return the connection ref.\n    /// After this function the underlying h2 connection should already notify the closure of this\n    /// stream so that another stream can be created if needed.\n    pub(crate) fn conn(&self) -> ConnectionRef {\n        self.conn.clone()\n    }\n\n    /// Whether ping timeout occurred. After a ping timeout, the h2 connection will be terminated.\n    /// Ongoing h2 streams will receive an stream/connection error. The streams should check this\n    /// flag to tell whether the error is triggered by the timeout.\n    pub(crate) fn ping_timedout(&self) -> bool {\n        self.conn.ping_timedout()\n    }\n\n    /// Return the [Digest] of the connection\n    ///\n    /// For reused connection, the timing in the digest will reflect its initial handshakes\n    /// The caller should check if the connection is reused to avoid misuse the timing field\n    pub fn digest(&self) -> Option<&Digest> {\n        Some(self.conn.digest())\n    }\n\n    /// Return the server (peer) address recorded in the connection digest.\n    pub fn server_addr(&self) -> Option<&SocketAddr> {\n        self.conn\n            .digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.peer_addr())?\n    }\n\n    /// Return the client (local) address recorded in the connection digest.\n    pub fn client_addr(&self) -> Option<&SocketAddr> {\n        self.conn\n            .digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.local_addr())?\n    }\n\n    /// the FD of the underlying connection\n    pub fn fd(&self) -> i32 {\n        self.conn.id()\n    }\n\n    /// take the body sender to another task to perform duplex read and write\n    pub fn take_request_body_writer(&mut self) -> Option<SendStream<Bytes>> {\n        self.send_body.take()\n    }\n\n    fn handle_err(&self, mut e: Box<Error>) -> Box<Error> {\n        if self.ping_timedout() {\n            e.etype = PING_TIMEDOUT;\n        }\n        e\n    }\n}\n\n/// A helper function to write the request body\npub fn write_body(send_body: &mut SendStream<Bytes>, data: Bytes, end: bool) -> Result<()> {\n    let data_len = data.len();\n    send_body.reserve_capacity(data_len);\n    send_body\n        .send_data(data, end)\n        .or_err(WriteError, \"while writing h2 request body\")\n}\n\n/* helper functions */\n\n/* Types of errors during h2 header read\n 1. peer requests to downgrade to h1, mostly IIS server for NTLM: we will downgrade and retry\n 2. peer sends invalid h2 frames, usually sending h1 only header: we will downgrade and retry\n 3. peer sends GO_AWAY(NO_ERROR) on reused conn, usually hit http2_max_requests: we will retry\n 4. peer IO error on reused conn, usually firewall kills old conn: we will retry\n 5. All other errors will terminate the request\n*/\nfn handle_read_header_error(e: h2::Error) -> Box<Error> {\n    if e.is_remote()\n        && e.reason()\n            .map_or(false, |r| r == h2::Reason::HTTP_1_1_REQUIRED)\n    {\n        let mut err = Error::because(H2Downgrade, \"while reading h2 header\", e);\n        err.retry = true.into();\n        err\n    } else if e.is_go_away()\n        && e.is_library()\n        && e.reason()\n            .map_or(false, |r| r == h2::Reason::PROTOCOL_ERROR)\n    {\n        // remote send invalid H2 responses\n        let mut err = Error::because(InvalidH2, \"while reading h2 header\", e);\n        err.retry = true.into();\n        err\n    } else if e.is_go_away()\n        && e.is_remote()\n        && e.reason().map_or(false, |r| r == h2::Reason::NO_ERROR)\n    {\n        // is_go_away: retry via another connection, this connection is being teardown\n        // only retry if the connection is reused\n        let mut err = Error::because(H2Error, \"while reading h2 header\", e);\n        err.retry = RetryType::ReusedOnly;\n        err\n    } else if e.is_io() {\n        // is_io: typical if a previously reused connection silently drops it\n        // only retry if the connection is reused\n        let true_io_error = e.get_io().unwrap().raw_os_error().is_some();\n        let mut err = Error::because(ReadError, \"while reading h2 header\", e);\n        if true_io_error {\n            err.retry = RetryType::ReusedOnly;\n        } // else could be TLS error, which is unsafe to retry\n        err\n    } else {\n        Error::because(H2Error, \"while reading h2 header\", e)\n    }\n}\n\nuse tokio::sync::oneshot;\n\npub async fn drive_connection<S>(\n    mut c: client::Connection<S>,\n    id: i32,\n    closed: watch::Sender<bool>,\n    ping_interval: Option<Duration>,\n    ping_timeout_occurred: Arc<AtomicBool>,\n) where\n    S: AsyncRead + AsyncWrite + Send + Unpin,\n{\n    let interval = ping_interval.unwrap_or(Duration::ZERO);\n    if !interval.is_zero() {\n        // for ping to inform this fn to drop the connection\n        let (tx, rx) = oneshot::channel::<()>();\n        // for this fn to inform ping to give up when it is already dropped\n        let dropped = Arc::new(AtomicBool::new(false));\n        let dropped2 = dropped.clone();\n\n        if let Some(ping_pong) = c.ping_pong() {\n            pingora_runtime::current_handle().spawn(async move {\n                do_ping_pong(ping_pong, interval, tx, dropped2, id).await;\n            });\n        } else {\n            warn!(\"Cannot get ping-pong handler from h2 connection\");\n        }\n\n        tokio::select! {\n            r = c => match r {\n                Ok(_) => debug!(\"H2 connection finished fd: {id}\"),\n                Err(e) => debug!(\"H2 connection fd: {id} errored: {e:?}\"),\n            },\n            r = rx => match r {\n                Ok(_) => {\n                    ping_timeout_occurred.store(true, Ordering::Relaxed);\n                    warn!(\"H2 connection Ping timeout/Error fd: {id}, closing conn\");\n                },\n                Err(e) => warn!(\"H2 connection Ping Rx error {e:?}\"),\n            },\n        };\n\n        dropped.store(true, Ordering::Relaxed);\n    } else {\n        match c.await {\n            Ok(_) => debug!(\"H2 connection finished fd: {id}\"),\n            Err(e) => debug!(\"H2 connection fd: {id} errored: {e:?}\"),\n        }\n    }\n    let _ = closed.send(true);\n}\n\nconst PING_TIMEOUT: Duration = Duration::from_secs(5);\n\nasync fn do_ping_pong(\n    mut ping_pong: h2::PingPong,\n    interval: Duration,\n    tx: oneshot::Sender<()>,\n    dropped: Arc<AtomicBool>,\n    id: i32,\n) {\n    // delay before sending the first ping, no need to race with the first request\n    tokio::time::sleep(interval).await;\n    loop {\n        if dropped.load(Ordering::Relaxed) {\n            break;\n        }\n        let ping_fut = ping_pong.ping(h2::Ping::opaque());\n        debug!(\"H2 fd: {id} ping sent\");\n        match tokio::time::timeout(PING_TIMEOUT, ping_fut).await {\n            Err(_) => {\n                error!(\"H2 fd: {id} ping timeout\");\n                let _ = tx.send(());\n                break;\n            }\n            Ok(r) => match r {\n                Ok(_) => {\n                    debug!(\"H2 fd: {} pong received\", id);\n                    tokio::time::sleep(interval).await;\n                }\n                Err(e) => {\n                    if dropped.load(Ordering::Relaxed) {\n                        // drive_connection() exits first, no need to error again\n                        break;\n                    }\n                    error!(\"H2 fd: {id} ping error: {e}\");\n                    let _ = tx.send(());\n                    break;\n                }\n            },\n        }\n    }\n}\n\n// Path: pingora-core/src/protocols/http/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse bytes::Bytes;\nuse pingora_error::Result;\nuse pingora_http::{RequestHeader, ResponseHeader};\nuse std::time::Duration;\n\nuse super::v1::client::HttpSession as Http1Session;\nuse super::v2::client::Http2Session;\nuse crate::protocols::{Digest, SocketAddr};\n\n/// A type for Http client session. It can be either an Http1 connection or an Http2 stream.\npub enum HttpSession {\n    H1(Http1Session),\n    H2(Http2Session),\n}\n\nimpl HttpSession {\n    pub fn as_http1(&self) -> Option<&Http1Session> {\n        match self {\n            Self::H1(s) => Some(s),\n            Self::H2(_) => None,\n        }\n    }\n\n    pub fn as_http2(&self) -> Option<&Http2Session> {\n        match self {\n            Self::H1(_) => None,\n            Self::H2(s) => Some(s),\n        }\n    }\n    /// Write the request header to the server\n    /// After the request header is sent. The caller can either start reading the response or\n    /// sending request body if any.\n    pub async fn write_request_header(&mut self, req: Box<RequestHeader>) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                h1.write_request_header(req).await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.write_request_header(req, false),\n        }\n    }\n\n    /// Write a chunk of the request body.\n    pub async fn write_request_body(&mut self, data: Bytes, end: bool) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                // TODO: maybe h1 should also have the concept of `end`\n                h1.write_body(&data).await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.write_request_body(data, end),\n        }\n    }\n\n    /// Signal that the request body has ended\n    pub async fn finish_request_body(&mut self) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                h1.finish_body().await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.finish_request_body(),\n        }\n    }\n\n    /// Set the read timeout for reading header and body.\n    ///\n    /// The timeout is per read operation, not on the overall time reading the entire response\n    pub fn set_read_timeout(&mut self, timeout: Duration) {\n        match self {\n            HttpSession::H1(h1) => h1.read_timeout = Some(timeout),\n            HttpSession::H2(h2) => h2.read_timeout = Some(timeout),\n        }\n    }\n\n    /// Set the write timeout for writing header and body.\n    ///\n    /// The timeout is per write operation, not on the overall time writing the entire request\n    pub fn set_write_timeout(&mut self, timeout: Duration) {\n        match self {\n            HttpSession::H1(h1) => h1.write_timeout = Some(timeout),\n            HttpSession::H2(_) => { /* no write timeout because the actual write happens async*/ }\n        }\n    }\n\n    /// Read the response header from the server\n    /// For http1, this function can be called multiple times, if the headers received are just\n    /// informational headers.\n    pub async fn read_response_header(&mut self) -> Result<()> {\n        match self {\n            HttpSession::H1(h1) => {\n                h1.read_response().await?;\n                Ok(())\n            }\n            HttpSession::H2(h2) => h2.read_response_header().await,\n        }\n    }\n\n    /// Read response body\n    ///\n    /// `None` when no more body to read.\n    pub async fn read_response_body(&mut self) -> Result<Option<Bytes>> {\n        match self {\n            HttpSession::H1(h1) => h1.read_body_bytes().await,\n            HttpSession::H2(h2) => h2.read_response_body().await,\n        }\n    }\n\n    /// No (more) body to read\n    pub fn response_done(&mut self) -> bool {\n        match self {\n            HttpSession::H1(h1) => h1.is_body_done(),\n            HttpSession::H2(h2) => h2.response_finished(),\n        }\n    }\n\n    /// Give up the http session abruptly.\n    /// For H1 this will close the underlying connection\n    /// For H2 this will send RST_STREAM frame to end this stream if the stream has not ended at all\n    pub async fn shutdown(&mut self) {\n        match self {\n            Self::H1(s) => s.shutdown().await,\n            Self::H2(s) => s.shutdown(),\n        }\n    }\n\n    /// Get the response header of the server\n    ///\n    /// `None` if the response header is not read yet.\n    pub fn response_header(&self) -> Option<&ResponseHeader> {\n        match self {\n            Self::H1(s) => s.resp_header(),\n            Self::H2(s) => s.response_header(),\n        }\n    }\n\n    /// Return the [Digest] of the connection\n    ///\n    /// For reused connection, the timing in the digest will reflect its initial handshakes\n    /// The caller should check if the connection is reused to avoid misuse of the timing field\n    pub fn digest(&self) -> Option<&Digest> {\n        match self {\n            Self::H1(s) => Some(s.digest()),\n            Self::H2(s) => s.digest(),\n        }\n    }\n\n    /// Return the server (peer) address of the connection.\n    pub fn server_addr(&self) -> Option<&SocketAddr> {\n        match self {\n            Self::H1(s) => s.server_addr(),\n            Self::H2(s) => s.server_addr(),\n        }\n    }\n\n    /// Return the client (local) address of the connection.\n    pub fn client_addr(&self) -> Option<&SocketAddr> {\n        match self {\n            Self::H1(s) => s.client_addr(),\n            Self::H2(s) => s.client_addr(),\n        }\n    }\n}\n\n// Path: pingora-core/src/protocols/http/body_buffer.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse bytes::{Bytes, BytesMut};\n\n/// A buffer with size limit. When the total amount of data written to the buffer is below the limit\n/// all the data will be held in the buffer. Otherwise, the buffer will report to be truncated.\npub(crate) struct FixedBuffer {\n    buffer: BytesMut,\n    capacity: usize,\n    truncated: bool,\n}\n\nimpl FixedBuffer {\n    pub fn new(capacity: usize) -> Self {\n        FixedBuffer {\n            buffer: BytesMut::new(),\n            capacity,\n            truncated: false,\n        }\n    }\n\n    // TODO: maybe store a Vec of Bytes for zero-copy\n    pub fn write_to_buffer(&mut self, data: &Bytes) {\n        if !self.truncated && (self.buffer.len() + data.len() <= self.capacity) {\n            self.buffer.extend_from_slice(data);\n        } else {\n            // TODO: clear data because the data held here is useless anyway?\n            self.truncated = true;\n        }\n    }\n    pub fn clear(&mut self) {\n        self.truncated = false;\n        self.buffer.clear();\n    }\n    pub fn is_empty(&self) -> bool {\n        self.buffer.len() == 0\n    }\n    pub fn is_truncated(&self) -> bool {\n        self.truncated\n    }\n    pub fn get_buffer(&self) -> Option<Bytes> {\n        // TODO: return None if truncated?\n        if !self.is_empty() {\n            Some(self.buffer.clone().freeze())\n        } else {\n            None\n        }\n    }\n}\n\n// Path: pingora-core/src/protocols/http/date.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse chrono::DateTime;\nuse http::header::HeaderValue;\nuse std::cell::RefCell;\nuse std::time::{Duration, SystemTime};\n\nfn to_date_string(epoch_sec: i64) -> String {\n    let dt = DateTime::from_timestamp(epoch_sec, 0).unwrap();\n    dt.format(\"%a, %d %b %Y %H:%M:%S GMT\").to_string()\n}\n\nstruct CacheableDate {\n    h1_date: HeaderValue,\n    epoch: Duration,\n}\n\nimpl CacheableDate {\n    pub fn new() -> Self {\n        let d = SystemTime::now()\n            .duration_since(SystemTime::UNIX_EPOCH)\n            .unwrap();\n        CacheableDate {\n            h1_date: HeaderValue::from_str(&to_date_string(d.as_secs() as i64)).unwrap(),\n            epoch: d,\n        }\n    }\n\n    pub fn update(&mut self, d_now: Duration) {\n        if d_now.as_secs() != self.epoch.as_secs() {\n            self.epoch = d_now;\n            self.h1_date = HeaderValue::from_str(&to_date_string(d_now.as_secs() as i64)).unwrap();\n        }\n    }\n\n    pub fn get_date(&mut self) -> HeaderValue {\n        let d = SystemTime::now()\n            .duration_since(SystemTime::UNIX_EPOCH)\n            .unwrap();\n        self.update(d);\n        self.h1_date.clone()\n    }\n}\n\nthread_local! {\n    static CACHED_DATE: RefCell<CacheableDate>\n        = RefCell::new(CacheableDate::new());\n}\n\npub fn get_cached_date() -> HeaderValue {\n    CACHED_DATE.with(|cache_date| (*cache_date.borrow_mut()).get_date())\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    fn now_date_header() -> HeaderValue {\n        HeaderValue::from_str(&to_date_string(\n            SystemTime::now()\n                .duration_since(SystemTime::UNIX_EPOCH)\n                .unwrap()\n                .as_secs() as i64,\n        ))\n        .unwrap()\n    }\n\n    #[test]\n    fn test_date_string() {\n        let date_str = to_date_string(1);\n        assert_eq!(\"Thu, 01 Jan 1970 00:00:01 GMT\", date_str);\n    }\n\n    #[test]\n    fn test_date_cached() {\n        assert_eq!(get_cached_date(), now_date_header());\n    }\n}\n\n// Path: pingora-core/src/protocols/http/v1/server.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/1.x server session\n\nuse bytes::Bytes;\nuse bytes::{BufMut, BytesMut};\nuse http::HeaderValue;\nuse http::{header, header::AsHeaderName, Method, Version};\nuse log::{debug, error, warn};\nuse once_cell::sync::Lazy;\nuse percent_encoding::{percent_encode, AsciiSet, CONTROLS};\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_http::{IntoCaseHeaderName, RequestHeader, ResponseHeader};\nuse pingora_timeout::timeout;\nuse regex::bytes::Regex;\nuse std::time::Duration;\nuse tokio::io::{AsyncReadExt, AsyncWriteExt};\n\nuse super::body::{BodyReader, BodyWriter};\nuse super::common::*;\nuse crate::protocols::http::{body_buffer::FixedBuffer, date, error_resp, HttpTask};\nuse crate::protocols::{Digest, SocketAddr, Stream};\nuse crate::utils::{BufRef, KVRef};\n\n/// The HTTP 1.x server session\npub struct HttpSession {\n    underlying_stream: Stream,\n    /// The buf that holds the raw request header + possibly a portion of request body\n    /// Request body can appear here because they could arrive with the same read() that\n    /// sends the request header.\n    buf: Bytes,\n    /// A slice reference to `buf` which points to the exact range of request header\n    raw_header: Option<BufRef>,\n    /// A slice reference to `buf` which points to the range of a portion of request body if any\n    preread_body: Option<BufRef>,\n    /// A state machine to track how to read the request body\n    body_reader: BodyReader,\n    /// A state machine to track how to write the response body\n    body_writer: BodyWriter,\n    /// An internal buffer to buf multiple body writes to reduce the underlying syscalls\n    body_write_buf: BytesMut,\n    /// Track how many application (not on the wire) body bytes already sent\n    body_bytes_sent: usize,\n    /// Whether to update headers like connection, Date\n    update_resp_headers: bool,\n    /// timeouts:\n    keepalive_timeout: KeepaliveStatus,\n    read_timeout: Option<Duration>,\n    write_timeout: Option<Duration>,\n    /// A copy of the response that is already written to the client\n    response_written: Option<Box<ResponseHeader>>,\n    /// The parse request header\n    request_header: Option<Box<RequestHeader>>,\n    /// An internal buffer that holds a copy of the request body up to a certain size\n    retry_buffer: Option<FixedBuffer>,\n    /// Whether this session is an upgraded session. This flag is calculated when sending the\n    /// response header to the client.\n    upgraded: bool,\n    /// Digest to track underlying connection metrics\n    digest: Box<Digest>,\n}\n\nimpl HttpSession {\n    /// Create a new http server session from an established (TCP or TLS) [`Stream`].\n    /// The created session needs to call [`Self::read_request()`] first before performing\n    /// any other operations.\n    pub fn new(underlying_stream: Stream) -> Self {\n        // TODO: maybe we should put digest in the connection itself\n        let digest = Box::new(Digest {\n            ssl_digest: underlying_stream.get_ssl_digest(),\n...\n", "needle_token_start": 5618, "needle_token_end": 5755, "code_context_ntokens": 16223, "output": ["```rust\n#[tokio::test]\nasync fn test_connect_h1_plaintext() {\n    let connector = Connector::new(None);\n    let mut peer = HttpPeer::new((\"1.1.1.1\", 80), false, \"\".into());\n    peer.options.set_http_version(2, 1);\n    let h2 = connector.new_http_session(&peer).await.unwrap();\n    match h2 {\n        HttpSession::H1(_) => {}\n        HttpSession::H2(_) => panic!(\"expect h1\"),\n    }\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "serialize_vec_string", "language": "rust", "path": "pingora-core/src/server/transfer_fd/mod.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a list of textual elements into a single space-separated string and store this string in a buffer.\n2. **Input**: It accepts a list of strings and a mutable byte buffer.\n3. **Output**: The function returns the number of bytes written to the buffer.\n4. **Procedure**: The function first joins all the strings in the list using a space as a separator to create a single string. This string is then converted to bytes and written into the provided buffer. The function returns the count of bytes that were written.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/server/configuration/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Server configurations\n//!\n//! Server configurations define startup settings such as:\n//! * User and group to run as after daemonization\n//! * Number of threads per service\n//! * Error log file path\n\nuse log::{debug, trace};\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse serde::{Deserialize, Serialize};\nuse std::fs;\nuse structopt::StructOpt;\n\n/// The configuration file\n///\n/// Pingora configuration files are by default YAML files, but any key value format can potentially\n/// be used.\n///\n/// # Extension\n/// New keys can be added to the configuration files which this configuration object will ignore.\n/// Then, users can parse these key-values to pass to their code to use.\n#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(default)]\npub struct ServerConf {\n    /// Version\n    pub version: usize,\n    /// Whether to run this process in the background.\n    pub daemon: bool,\n    /// When configured, error log will be written to the given file. Otherwise StdErr will be used.\n    pub error_log: Option<String>,\n    /// The pid (process ID) file of this server\n    pub pid_file: String,\n    /// the path to the upgrade socket\n    ///\n    /// In order to perform zero downtime restart, both the new and old process need to agree on the\n    /// path to this sock in order to coordinate the upgrade.\n    pub upgrade_sock: String,\n    /// If configured, after daemonization, this process will switch to the given user before\n    /// starting to serve traffic.\n    pub user: Option<String>,\n    /// Similar to `user`, the group this process should switch to.\n    pub group: Option<String>,\n    /// How many threads **each** service should get. The threads are not shared across services.\n    pub threads: usize,\n    /// Allow work stealing between threads of the same service. Default `true`.\n    pub work_stealing: bool,\n    /// The path to CA file the SSL library should use. If empty, the default trust store location\n    /// defined by the SSL library will be used.\n    pub ca_file: Option<String>,\n    /// Grace period in seconds before starting the final step of the graceful shutdown after signaling shutdown.\n    pub grace_period_seconds: Option<u64>,\n    /// Timeout in seconds of the final step for the graceful shutdown.\n    pub graceful_shutdown_timeout_seconds: Option<u64>,\n    // These options don't belong here as they are specific to certain services\n    /// IPv4 addresses for a client connector to bind to. See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub client_bind_to_ipv4: Vec<String>,\n    /// IPv6 addresses for a client connector to bind to. See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub client_bind_to_ipv6: Vec<String>,\n    /// Keepalive pool size for client connections to upstream. See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub upstream_keepalive_pool_size: usize,\n    /// Number of dedicated thread pools to use for upstream connection establishment.\n    /// See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub upstream_connect_offload_threadpools: Option<usize>,\n    /// Number of threads per dedicated upstream connection establishment pool.\n    /// See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub upstream_connect_offload_thread_per_pool: Option<usize>,\n}\n\nimpl Default for ServerConf {\n    fn default() -> Self {\n        ServerConf {\n            version: 0,\n            client_bind_to_ipv4: vec![],\n            client_bind_to_ipv6: vec![],\n            ca_file: None,\n            daemon: false,\n            error_log: None,\n            pid_file: \"/tmp/pingora.pid\".to_string(),\n            upgrade_sock: \"/tmp/pingora_upgrade.sock\".to_string(),\n            user: None,\n            group: None,\n            threads: 1,\n            work_stealing: true,\n            upstream_keepalive_pool_size: 128,\n            upstream_connect_offload_threadpools: None,\n            upstream_connect_offload_thread_per_pool: None,\n            grace_period_seconds: None,\n            graceful_shutdown_timeout_seconds: None,\n        }\n    }\n}\n\n/// Command-line options\n///\n/// Call `Opt::from_args()` to build this object from the process's command line arguments.\n#[derive(StructOpt, Debug)]\n#[structopt(name = \"basic\")]\npub struct Opt {\n    /// Whether this server should try to upgrade from a running old server\n    ///\n    /// `-u` or `--upgrade` can be used\n    #[structopt(short, long)]\n    pub upgrade: bool,\n    /// Whether should run this server in the background\n    ///\n    /// `-d` or `--daemon` can be used\n    #[structopt(short, long)]\n    pub daemon: bool,\n    /// Not actually used. This flag is there so that the server is not upset seeing this flag\n    /// passed from `cargo test` sometimes\n    #[structopt(long)]\n    pub nocapture: bool,\n    /// Test the configuration and exit\n    ///\n    /// When this flag is set, calling `server.bootstrap()` will exit the process without errors\n    ///\n    /// This flag is useful for upgrading service where the user wants to make sure the new\n    /// service can start before shutting down the old server process.\n    ///\n    /// `-t` or `--test` can be used\n    #[structopt(short, long)]\n    pub test: bool,\n    /// The path to the configuration file.\n    ///\n    /// See [`ServerConf`] for more details of the configuration file.\n    ///\n    /// `-c` or `--conf` can be used\n    #[structopt(short, long)]\n    pub conf: Option<String>,\n}\n\n/// Create the default instance of Opt based on the current command-line args.\n/// This is equivalent to running `Opt::from_args` but does not require the\n/// caller to have included the `structopt::StructOpt`\nimpl Default for Opt {\n    fn default() -> Self {\n        Opt::from_args()\n    }\n}\n\nimpl ServerConf {\n    // Does not has to be async until we want runtime reload\n    pub fn load_from_yaml<P>(path: P) -> Result<Self>\n    where\n        P: AsRef<std::path::Path> + std::fmt::Display,\n    {\n        let conf_str = fs::read_to_string(&path).or_err_with(ReadError, || {\n            format!(\"Unable to read conf file from {path}\")\n        })?;\n        debug!(\"Conf file read from {path}\");\n        Self::from_yaml(&conf_str)\n    }\n\n    pub fn load_yaml_with_opt_override(opt: &Opt) -> Result<Self> {\n        if let Some(path) = &opt.conf {\n            let mut conf = Self::load_from_yaml(path)?;\n            conf.merge_with_opt(opt);\n            Ok(conf)\n        } else {\n            Error::e_explain(ReadError, \"No path specified\")\n        }\n    }\n\n    pub fn new() -> Option<Self> {\n        Self::from_yaml(\"---\\nversion: 1\").ok()\n    }\n\n    pub fn new_with_opt_override(opt: &Opt) -> Option<Self> {\n        let conf = Self::new();\n        match conf {\n            Some(mut c) => {\n                c.merge_with_opt(opt);\n                Some(c)\n            }\n            None => None,\n        }\n    }\n\n    pub fn from_yaml(conf_str: &str) -> Result<Self> {\n        trace!(\"Read conf file: {conf_str}\");\n        let conf: ServerConf = serde_yaml::from_str(conf_str).or_err_with(ReadError, || {\n            format!(\"Unable to parse yaml conf {conf_str}\")\n        })?;\n\n        trace!(\"Loaded conf: {conf:?}\");\n        conf.validate()\n    }\n\n    pub fn to_yaml(&self) -> String {\n        serde_yaml::to_string(self).unwrap()\n    }\n\n    pub fn validate(self) -> Result<Self> {\n        // TODO: do the validation\n        Ok(self)\n    }\n\n    pub fn merge_with_opt(&mut self, opt: &Opt) {\n        if opt.daemon {\n            self.daemon = true;\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    fn init_log() {\n        let _ = env_logger::builder().is_test(true).try_init();\n    }\n\n    #[test]\n    fn not_a_test_i_cannot_write_yaml_by_hand() {\n        init_log();\n        let conf = ServerConf {\n            version: 1,\n            client_bind_to_ipv4: vec![\"1.2.3.4\".to_string(), \"5.6.7.8\".to_string()],\n            client_bind_to_ipv6: vec![],\n            ca_file: None,\n            daemon: false,\n            error_log: None,\n            pid_file: \"\".to_string(),\n            upgrade_sock: \"\".to_string(),\n            user: None,\n            group: None,\n            threads: 1,\n            work_stealing: true,\n            upstream_keepalive_pool_size: 4,\n            upstream_connect_offload_threadpools: None,\n            upstream_connect_offload_thread_per_pool: None,\n            grace_period_seconds: None,\n            graceful_shutdown_timeout_seconds: None,\n        };\n        // cargo test -- --nocapture not_a_test_i_cannot_write_yaml_by_hand\n        println!(\"{}\", conf.to_yaml());\n    }\n\n    #[test]\n    fn test_load_file() {\n        init_log();\n        let conf_str = r#\"\n---\nversion: 1\nclient_bind_to_ipv4:\n    - 1.2.3.4\n    - 5.6.7.8\nclient_bind_to_ipv6: []\n        \"#\n        .to_string();\n        let conf = ServerConf::from_yaml(&conf_str).unwrap();\n        assert_eq!(2, conf.client_bind_to_ipv4.len());\n        assert_eq!(0, conf.client_bind_to_ipv6.len());\n        assert_eq!(1, conf.version);\n    }\n\n    #[test]\n    fn test_default() {\n        init_log();\n        let conf_str = r#\"\n---\nversion: 1\n        \"#\n        .to_string();\n        let conf = ServerConf::from_yaml(&conf_str).unwrap();\n        assert_eq!(0, conf.client_bind_to_ipv4.len());\n        assert_eq!(0, conf.client_bind_to_ipv6.len());\n        assert_eq!(1, conf.version);\n        assert_eq!(\"/tmp/pingora.pid\", conf.pid_file);\n    }\n}\n\n// Path: pingora-core/src/server/daemon.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse daemonize::Daemonize;\nuse log::{debug, error};\nuse std::ffi::CString;\nuse std::fs::{self, OpenOptions};\nuse std::os::unix::prelude::OpenOptionsExt;\nuse std::path::Path;\n\nuse crate::server::configuration::ServerConf;\n\n// Utilities to daemonize a pingora server, i.e. run the process in the background, possibly\n// under a different running user and/or group.\n\n// XXX: this operation should have been done when the old service is exiting.\n// Now the new pid file just kick the old one out of the way\nfn move_old_pid(path: &str) {\n    if !Path::new(path).exists() {\n        debug!(\"Old pid file does not exist\");\n        return;\n    }\n    let new_path = format!(\"{path}.old\");\n    match fs::rename(path, &new_path) {\n        Ok(()) => {\n            debug!(\"Old pid file renamed\");\n        }\n        Err(e) => {\n            error!(\n                \"failed to rename pid file from {} to {}: {}\",\n                path, new_path, e\n            );\n        }\n    }\n}\n\nunsafe fn gid_for_username(name: &CString) -> Option<libc::gid_t> {\n    let passwd = libc::getpwnam(name.as_ptr() as *const libc::c_char);\n    if !passwd.is_null() {\n        return Some((*passwd).pw_gid);\n    }\n    None\n}\n\n/// Start a server instance as a daemon.\npub fn daemonize(conf: &ServerConf) {\n    // TODO: customize working dir\n\n    let daemonize = Daemonize::new()\n        .umask(0o007) // allow same group to access files but not everyone else\n        .pid_file(&conf.pid_file);\n\n    let daemonize = if let Some(error_log) = conf.error_log.as_ref() {\n        let err = OpenOptions::new()\n            .append(true)\n            .create(true)\n            // open read() in case there are no readers\n            // available otherwise we will panic with\n            // an ENXIO since O_NONBLOCK is set\n            .read(true)\n            .custom_flags(libc::O_NONBLOCK)\n            .open(error_log)\n            .unwrap();\n        daemonize.stderr(err)\n    } else {\n        daemonize\n    };\n\n    let daemonize = match conf.user.as_ref() {\n        Some(user) => {\n            let user_cstr = CString::new(user.as_str()).unwrap();\n\n            #[cfg(target_os = \"macos\")]\n            let group_id = unsafe { gid_for_username(&user_cstr).map(|gid| gid as i32) };\n            #[cfg(target_os = \"linux\")]\n            let group_id = unsafe { gid_for_username(&user_cstr) };\n\n            daemonize\n                .privileged_action(move || {\n                    if let Some(gid) = group_id {\n                        // Set the supplemental group privileges for the child process.\n                        unsafe {\n                            libc::initgroups(user_cstr.as_ptr() as *const libc::c_char, gid);\n                        }\n                    }\n                })\n                .user(user.as_str())\n                .chown_pid_file(true)\n        }\n        None => daemonize,\n    };\n\n    let daemonize = match conf.group.as_ref() {\n        Some(group) => daemonize.group(group.as_str()),\n        None => daemonize,\n    };\n\n    move_old_pid(&conf.pid_file);\n\n    daemonize.start().unwrap(); // hard crash when fail\n}\n\n// Path: pingora-core/src/server/transfer_fd/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#[cfg(target_os = \"linux\")]\nuse log::{debug, error, warn};\nuse nix::errno::Errno;\n#[cfg(target_os = \"linux\")]\nuse nix::sys::socket::{self, AddressFamily, RecvMsg, SockFlag, SockType, UnixAddr};\n#[cfg(target_os = \"linux\")]\nuse nix::sys::stat;\nuse nix::{Error, NixPath};\nuse std::collections::HashMap;\nuse std::io::Write;\n#[cfg(target_os = \"linux\")]\nuse std::io::{IoSlice, IoSliceMut};\nuse std::os::unix::io::RawFd;\n#[cfg(target_os = \"linux\")]\nuse std::{thread, time};\n\n// Utilities to transfer file descriptors between sockets, e.g. during graceful upgrades.\n\n/// Container for open file descriptors and their associated bind addresses.\npub struct Fds {\n    map: HashMap<String, RawFd>,\n}\n\nimpl Fds {\n    pub fn new() -> Self {\n        Fds {\n            map: HashMap::new(),\n        }\n    }\n\n    pub fn add(&mut self, bind: String, fd: RawFd) {\n        self.map.insert(bind, fd);\n    }\n\n    pub fn get(&self, bind: &str) -> Option<&RawFd> {\n        self.map.get(bind)\n    }\n\n    pub fn serialize(&self) -> (Vec<String>, Vec<RawFd>) {\n        let serialized: Vec<(String, RawFd)> = self\n            .map\n            .iter()\n            .map(|(key, value)| (key.clone(), *value))\n            .collect();\n\n        (\n            serialized.iter().map(|v| v.0.clone()).collect(),\n            serialized.iter().map(|v| v.1).collect(),\n        )\n        // Surely there is a better way of doing this\n    }\n\n    pub fn deserialize(&mut self, binds: Vec<String>, fds: Vec<RawFd>) {\n        assert!(binds.len() == fds.len());\n        // TODO: use zip()\n        for i in 0..binds.len() {\n            self.map.insert(binds[i].clone(), fds[i]);\n        }\n    }\n\n    pub fn send_to_sock<P>(&self, path: &P) -> Result<usize, Error>\n    where\n        P: ?Sized + NixPath + std::fmt::Display,\n    {\n        let (vec_key, vec_fds) = self.serialize();\n        let mut ser_buf: [u8; 2048] = [0; 2048];\n        let ser_key_size = serialize_vec_string(&vec_key, &mut ser_buf);\n        send_fds_to(vec_fds, &ser_buf[..ser_key_size], path)\n    }\n\n    pub fn get_from_sock<P>(&mut self, path: &P) -> Result<(), Error>\n    where\n        P: ?Sized + NixPath + std::fmt::Display,\n    {\n        let mut de_buf: [u8; 2048] = [0; 2048];\n        let (fds, bytes) = get_fds_from(path, &mut de_buf)?;\n        let keys = deserialize_vec_string(&de_buf[..bytes]);\n        self.deserialize(keys, fds);\n        Ok(())\n    }\n}\n\n\nfn serialize_vec_string(vec_string: &[String], mut buf: &mut [u8]) -> usize {\n    // There are many ways to do this. Serde is probably the way to go\n    // But let's start with something simple: space separated strings\n    let joined = vec_string.join(\" \");\n    // TODO: check the buf is large enough\n    buf.write(joined.as_bytes()).unwrap()\n}\n\nfn deserialize_vec_string(buf: &[u8]) -> Vec<String> {\n    let joined = std::str::from_utf8(buf).unwrap(); // TODO: handle error\n    let mut results: Vec<String> = Vec::new();\n    for iter in joined.split_ascii_whitespace() {\n        results.push(String::from(iter));\n    }\n    results\n}\n\n#[cfg(target_os = \"linux\")]\npub fn get_fds_from<P>(path: &P, payload: &mut [u8]) -> Result<(Vec<RawFd>, usize), Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    const MAX_FDS: usize = 32;\n\n    let listen_fd = socket::socket(\n        AddressFamily::Unix,\n        SockType::Stream,\n        SockFlag::SOCK_NONBLOCK,\n        None,\n    )\n    .unwrap();\n    let unix_addr = UnixAddr::new(path).unwrap();\n    // clean up old sock\n    match nix::unistd::unlink(path) {\n        Ok(()) => {\n            debug!(\"unlink {} done\", path);\n        }\n        Err(e) => {\n            // Normal if file does not exist\n            debug!(\"unlink {} failed: {}\", path, e);\n            // TODO: warn if exist but not able to unlink\n        }\n    };\n    socket::bind(listen_fd, &unix_addr).unwrap();\n\n    /* sock is created before we change user, need to give permission to all */\n    stat::fchmodat(\n        None,\n        path,\n        stat::Mode::all(),\n        stat::FchmodatFlags::FollowSymlink,\n    )\n    .unwrap();\n\n    socket::listen(listen_fd, 8).unwrap();\n\n    let fd = match accept_with_retry(listen_fd) {\n        Ok(fd) => fd,\n        Err(e) => {\n            error!(\"Giving up reading socket from: {path}, error: {e:?}\");\n            //cleanup\n            if nix::unistd::close(listen_fd).is_ok() {\n                nix::unistd::unlink(path).unwrap();\n            }\n            return Err(e);\n        }\n    };\n\n    let mut io_vec = [IoSliceMut::new(payload); 1];\n    let mut cmsg_buf = nix::cmsg_space!([RawFd; MAX_FDS]);\n    let msg: RecvMsg<UnixAddr> = socket::recvmsg(\n        fd,\n        &mut io_vec,\n        Some(&mut cmsg_buf),\n        socket::MsgFlags::empty(),\n    )\n    .unwrap();\n\n    let mut fds: Vec<RawFd> = Vec::new();\n    for cmsg in msg.cmsgs() {\n        if let socket::ControlMessageOwned::ScmRights(mut vec_fds) = cmsg {\n            fds.append(&mut vec_fds)\n        } else {\n            warn!(\"Unexpected control messages: {cmsg:?}\")\n        }\n    }\n\n    //cleanup\n    if nix::unistd::close(listen_fd).is_ok() {\n        nix::unistd::unlink(path).unwrap();\n    }\n\n    Ok((fds, msg.bytes))\n}\n\n#[cfg(not(target_os = \"linux\"))]\npub fn get_fds_from<P>(_path: &P, _payload: &mut [u8]) -> Result<(Vec<RawFd>, usize), Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    Err(Errno::ECONNREFUSED)\n}\n\n#[cfg(target_os = \"linux\")]\nconst MAX_RETRY: usize = 5;\n#[cfg(target_os = \"linux\")]\nconst RETRY_INTERVAL: time::Duration = time::Duration::from_secs(1);\n\n#[cfg(target_os = \"linux\")]\nfn accept_with_retry(listen_fd: i32) -> Result<i32, Error> {\n    let mut retried = 0;\n    loop {\n        match socket::accept(listen_fd) {\n            Ok(fd) => return Ok(fd),\n            Err(e) => {\n                if retried > MAX_RETRY {\n                    return Err(e);\n                }\n                match e {\n                    Errno::EAGAIN => {\n                        error!(\n                            \"No incoming socket transfer, sleep {RETRY_INTERVAL:?} and try again\"\n                        );\n                        retried += 1;\n                        thread::sleep(RETRY_INTERVAL);\n                    }\n                    _ => {\n                        error!(\"Error accepting socket transfer: {e}\");\n                        return Err(e);\n                    }\n                }\n            }\n        }\n    }\n}\n\n#[cfg(target_os = \"linux\")]\npub fn send_fds_to<P>(fds: Vec<RawFd>, payload: &[u8], path: &P) -> Result<usize, Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    const MAX_NONBLOCKING_POLLS: usize = 20;\n    const NONBLOCKING_POLL_INTERVAL: time::Duration = time::Duration::from_millis(500);\n\n    let send_fd = socket::socket(\n        AddressFamily::Unix,\n        SockType::Stream,\n        SockFlag::SOCK_NONBLOCK,\n        None,\n    )?;\n    let unix_addr = UnixAddr::new(path)?;\n    let mut retried = 0;\n    let mut nonblocking_polls = 0;\n\n    let conn_result: Result<usize, Error> = loop {\n        match socket::connect(send_fd, &unix_addr) {\n            Ok(_) => break Ok(0),\n            Err(e) => match e {\n                /* If the new process hasn't created the upgrade sock we'll get an ENOENT.\n                ECONNREFUSED may happen if the sock wasn't cleaned up\n                and the old process tries sending before the new one is listening.\n                EACCES may happen if connect() happen before the correct permission is set */\n                Errno::ENOENT | Errno::ECONNREFUSED | Errno::EACCES => {\n                    /*the server is not ready yet*/\n                    retried += 1;\n                    if retried > MAX_RETRY {\n                        error!(\n                            \"Max retry: {} reached. Giving up sending socket to: {}, error: {:?}\",\n                            MAX_RETRY, path, e\n                        );\n                        break Err(e);\n                    }\n                    warn!(\"server not ready, will try again in {RETRY_INTERVAL:?}\");\n                    thread::sleep(RETRY_INTERVAL);\n                }\n                /* handle nonblocking IO */\n                Errno::EINPROGRESS => {\n                    nonblocking_polls += 1;\n                    if nonblocking_polls >= MAX_NONBLOCKING_POLLS {\n                        error!(\"Connect() not ready after retries when sending socket to: {path}\",);\n                        break Err(e);\n                    }\n                    warn!(\"Connect() not ready, will try again in {NONBLOCKING_POLL_INTERVAL:?}\",);\n                    thread::sleep(NONBLOCKING_POLL_INTERVAL);\n                }\n                _ => {\n                    error!(\"Error sending socket to: {path}, error: {e:?}\");\n                    break Err(e);\n                }\n            },\n        }\n    };\n\n    let result = match conn_result {\n        Ok(_) => {\n            let io_vec = [IoSlice::new(payload); 1];\n            let scm = socket::ControlMessage::ScmRights(fds.as_slice());\n            let cmsg = [scm; 1];\n            loop {\n                match socket::sendmsg(\n                    send_fd,\n                    &io_vec,\n                    &cmsg,\n                    socket::MsgFlags::empty(),\n                    None::<&UnixAddr>,\n                ) {\n                    Ok(result) => break Ok(result),\n                    Err(e) => match e {\n                        /* handle nonblocking IO */\n                        Errno::EAGAIN => {\n                            nonblocking_polls += 1;\n                            if nonblocking_polls >= MAX_NONBLOCKING_POLLS {\n                                error!(\n                                    \"Sendmsg() not ready after retries when sending socket to: {}\",\n                                    path\n                                );\n                                break Err(e);\n                            }\n                            warn!(\n                                \"Sendmsg() not ready, will try again in {:?}\",\n                                NONBLOCKING_POLL_INTERVAL\n                            );\n                            thread::sleep(NONBLOCKING_POLL_INTERVAL);\n                        }\n                        _ => break Err(e),\n                    },\n                }\n            }\n        }\n        Err(_) => conn_result,\n    };\n\n    nix::unistd::close(send_fd).unwrap();\n    result\n}\n\n#[cfg(not(target_os = \"linux\"))]\npub fn send_fds_to<P>(_fds: Vec<RawFd>, _payload: &[u8], _path: &P) -> Result<usize, Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    Ok(0)\n}\n\n#[cfg(test)]\n#[cfg(target_os = \"linux\")]\nmod tests {\n    use super::*;\n    use log::{debug, error};\n\n    fn init_log() {\n        let _ = env_logger::builder().is_test(true).try_init();\n    }\n\n    #[test]\n    fn test_add_get() {\n        init_log();\n        let mut fds = Fds::new();\n        let key = \"1.1.1.1:80\".to_string();\n        fds.add(key.clone(), 128);\n        assert_eq!(128, *fds.get(&key).unwrap());\n    }\n\n    #[test]\n    fn test_table_serde() {\n        init_log();\n        let mut fds = Fds::new();\n        let key1 = \"1.1.1.1:80\".to_string();\n        fds.add(key1.clone(), 128);\n        let key2 = \"1.1.1.1:443\".to_string();\n        fds.add(key2.clone(), 129);\n\n        let (k, v) = fds.serialize();\n        let mut fds2 = Fds::new();\n        fds2.deserialize(k, v);\n\n        assert_eq!(128, *fds2.get(&key1).unwrap());\n        assert_eq!(129, *fds2.get(&key2).unwrap());\n    }\n\n    #[test]\n    fn test_vec_string_serde() {\n        init_log();\n        let vec_str: Vec<String> = vec![\"aaaa\".to_string(), \"bbb\".to_string()];\n        let mut ser_buf: [u8; 1024] = [0; 1024];\n        let size = serialize_vec_string(&vec_str, &mut ser_buf);\n        let de_vec_string = deserialize_vec_string(&ser_buf[..size]);\n        assert_eq!(de_vec_string.len(), 2);\n        assert_eq!(de_vec_string[0], \"aaaa\");\n        assert_eq!(de_vec_string[1], \"bbb\");\n    }\n\n    #[test]\n    fn test_send_receive_fds() {\n        init_log();\n        let dumb_fd = socket::socket(\n            AddressFamily::Unix,\n            SockType::Stream,\n            SockFlag::empty(),\n            None,\n        )\n        .unwrap();\n\n        // receiver need to start in another thread since it is blocking\n        let child = thread::spawn(move || {\n            let mut buf: [u8; 32] = [0; 32];\n            let (fds, bytes) = get_fds_from(\"/tmp/pingora_fds_receive.sock\", &mut buf).unwrap();\n            debug!(\"{:?}\", fds);\n            assert_eq!(1, fds.len());\n            assert_eq!(32, bytes);\n            assert_eq!(1, buf[0]);\n            assert_eq!(1, buf[31]);\n        });\n\n        let fds = vec![dumb_fd];\n        let buf: [u8; 128] = [1; 128];\n        match send_fds_to(fds, &buf, \"/tmp/pingora_fds_receive.sock\") {\n            Ok(sent) => {\n                assert!(sent > 0);\n            }\n            Err(e) => {\n                error!(\"{:?}\", e);\n                panic!()\n            }\n        }\n\n        child.join().unwrap();\n    }\n\n    #[test]\n    fn test_serde_via_socket() {\n        init_log();\n        let mut fds = Fds::new();\n        let key1 = \"1.1.1.1:80\".to_string();\n        let dumb_fd1 = socket::socket(\n            AddressFamily::Unix,\n            SockType::Stream,\n            SockFlag::empty(),\n            None,\n        )\n        .unwrap();\n        fds.add(key1.clone(), dumb_fd1);\n        let key2 = \"1.1.1.1:443\".to_string();\n        let dumb_fd2 = socket::socket(\n            AddressFamily::Unix,\n            SockType::Stream,\n            SockFlag::empty(),\n            None,\n        )\n        .unwrap();\n        fds.add(key2.clone(), dumb_fd2);\n\n        let child = thread::spawn(move || {\n            let mut fds2 = Fds::new();\n            fds2.get_from_sock(\"/tmp/pingora_fds_receive2.sock\")\n                .unwrap();\n            assert!(*fds2.get(&key1).unwrap() > 0);\n            assert!(*fds2.get(&key2).unwrap() > 0);\n        });\n\n        fds.send_to_sock(\"/tmp/pingora_fds_receive2.sock\").unwrap();\n        child.join().unwrap();\n    }\n}\n\n// Path: pingora-core/src/services/background.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! The background service\n//!\n//! A [BackgroundService] can be run as part of a Pingora application to add supporting logic that\n//! exists outside of the request/response lifecycle.\n//! Examples might include service discovery (load balancing) and background updates such as\n//! push-style metrics.\n\nuse async_trait::async_trait;\nuse std::sync::Arc;\n\nuse super::Service;\nuse crate::server::{ListenFds, ShutdownWatch};\n\n/// The background service interface\n#[cfg_attr(not(doc_async_trait), async_trait)]\npub trait BackgroundService {\n    /// This function is called when the pingora server tries to start all the\n    /// services. The background service can return at anytime or wait for the\n    /// `shutdown` signal.\n    async fn start(&self, mut shutdown: ShutdownWatch);\n}\n\n/// A generic type of background service\npub struct GenBackgroundService<A> {\n    // Name of the service\n    name: String,\n    // Task the service will execute\n    task: Arc<A>,\n    /// The number of threads. Default is 1\n    pub threads: Option<usize>,\n}\n\nimpl<A> GenBackgroundService<A> {\n    /// Generates a background service that can run in the pingora runtime\n    pub fn new(name: String, task: Arc<A>) -> Self {\n        Self {\n            name,\n            task,\n            threads: Some(1),\n        }\n    }\n\n    /// Return the task behind [Arc] to be shared other logic.\n    pub fn task(&self) -> Arc<A> {\n        self.task.clone()\n    }\n}\n\n#[async_trait]\nimpl<A> Service for GenBackgroundService<A>\nwhere\n    A: BackgroundService + Send + Sync + 'static,\n{\n    async fn start_service(&mut self, _fds: Option<ListenFds>, shutdown: ShutdownWatch) {\n        self.task.start(shutdown).await;\n    }\n\n    fn name(&self) -> &str {\n        &self.name\n    }\n\n    fn threads(&self) -> Option<usize> {\n        self.threads\n    }\n}\n\n// Helper function to create a background service with a human readable name\npub fn background_service<SV>(name: &str, task: SV) -> GenBackgroundService<SV> {\n    GenBackgroundService::new(format!(\"BG {name}\"), Arc::new(task))\n}\n\n// Path: pingora-core/src/protocols/http/error_resp.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Error response generating utilities.\n\nuse http::header;\nuse once_cell::sync::Lazy;\nuse pingora_http::ResponseHeader;\n\nuse super::SERVER_NAME;\n\n/// Generate an error response with the given status code.\n///\n/// This error response has a zero `Content-Length` and `Cache-Control: private, no-store`.\npub fn gen_error_response(code: u16) -> ResponseHeader {\n    let mut resp = ResponseHeader::build(code, Some(4)).unwrap();\n    resp.insert_header(header::SERVER, &SERVER_NAME[..])\n        .unwrap();\n    resp.insert_header(header::DATE, \"Sun, 06 Nov 1994 08:49:37 GMT\")\n        .unwrap(); // placeholder\n    resp.insert_header(header::CONTENT_LENGTH, \"0\").unwrap();\n    resp.insert_header(header::CACHE_CONTROL, \"private, no-store\")\n        .unwrap();\n    resp\n}\n\n/// Pre-generated 502 response\npub static HTTP_502_RESPONSE: Lazy<ResponseHeader> = Lazy::new(|| gen_error_response(502));\n/// Pre-generated 400 response\npub static HTTP_400_RESPONSE: Lazy<ResponseHeader> = Lazy::new(|| gen_error_response(400));\n\n// Path: pingora-core/src/protocols/l4/socket.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Generic socket type\n\nuse crate::{Error, OrErr};\nuse nix::sys::socket::{getpeername, getsockname, SockaddrStorage};\nuse std::cmp::Ordering;\nuse std::hash::{Hash, Hasher};\nuse std::net::SocketAddr as StdSockAddr;\nuse std::os::unix::net::SocketAddr as StdUnixSockAddr;\nuse tokio::net::unix::SocketAddr as TokioUnixSockAddr;\n\n/// [`SocketAddr`] is a storage type that contains either a Internet (IP address)\n/// socket address or a Unix domain socket address.\n#[derive(Debug, Clone)]\npub enum SocketAddr {\n    Inet(StdSockAddr),\n    Unix(StdUnixSockAddr),\n}\n\nimpl SocketAddr {\n    /// Get a reference to the IP socket if it is one\n    pub fn as_inet(&self) -> Option<&StdSockAddr> {\n        if let SocketAddr::Inet(addr) = self {\n            Some(addr)\n        } else {\n            None\n        }\n    }\n\n    /// Get a reference to the Unix domain socket if it is one\n    pub fn as_unix(&self) -> Option<&StdUnixSockAddr> {\n        if let SocketAddr::Unix(addr) = self {\n            Some(addr)\n        } else {\n            None\n        }\n    }\n\n    /// Set the port if the address is an IP socket.\n    pub fn set_port(&mut self, port: u16) {\n        if let SocketAddr::Inet(addr) = self {\n            addr.set_port(port)\n        }\n    }\n\n    fn from_sockaddr_storage(sock: &SockaddrStorage) -> Option<SocketAddr> {\n        if let Some(v4) = sock.as_sockaddr_in() {\n            return Some(SocketAddr::Inet(StdSockAddr::V4(\n                std::net::SocketAddrV4::new(v4.ip().into(), v4.port()),\n            )));\n        } else if let Some(v6) = sock.as_sockaddr_in6() {\n            return Some(SocketAddr::Inet(StdSockAddr::V6(\n                std::net::SocketAddrV6::new(v6.ip(), v6.port(), v6.flowinfo(), v6.scope_id()),\n            )));\n        }\n\n        // TODO: don't set abstract / unnamed for now,\n        // for parity with how we treat these types in TryFrom<TokioUnixSockAddr>\n        Some(SocketAddr::Unix(\n            sock.as_unix_addr()\n                .map(|addr| addr.path().map(StdUnixSockAddr::from_pathname))??\n                .ok()?,\n        ))\n    }\n\n    pub fn from_raw_fd(fd: std::os::unix::io::RawFd, peer_addr: bool) -> Option<SocketAddr> {\n        let sockaddr_storage = if peer_addr {\n            getpeername(fd)\n        } else {\n            getsockname(fd)\n        };\n        match sockaddr_storage {\n            Ok(sockaddr) => Self::from_sockaddr_storage(&sockaddr),\n            // could be errors such as EBADF, i.e. fd is no longer a valid socket\n            // fail open in this case\n            Err(_e) => None,\n        }\n    }\n}\n\nimpl std::fmt::Display for SocketAddr {\n    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {\n        match self {\n            SocketAddr::Inet(addr) => write!(f, \"{addr}\"),\n            SocketAddr::Unix(addr) => {\n                if let Some(path) = addr.as_pathname() {\n                    write!(f, \"{}\", path.display())\n                } else {\n                    write!(f, \"{addr:?}\")\n                }\n            }\n        }\n    }\n}\n\nimpl Hash for SocketAddr {\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        match self {\n            Self::Inet(sockaddr) => sockaddr.hash(state),\n            Self::Unix(sockaddr) => {\n                if let Some(path) = sockaddr.as_pathname() {\n                    // use the underlying path as the hash\n                    path.hash(state);\n                } else {\n                    // unnamed or abstract UDS\n                    // abstract UDS name not yet exposed by std API\n                    // panic for now, we can decide on the right way to hash them later\n                    panic!(\"Unnamed and abstract UDS types not yet supported for hashing\")\n                }\n            }\n        }\n    }\n}\n\nimpl PartialEq for SocketAddr {\n    fn eq(&self, other: &Self) -> bool {\n        match self {\n            Self::Inet(addr) => Some(addr) == other.as_inet(),\n            Self::Unix(addr) => {\n                let path = addr.as_pathname();\n                // can only compare UDS with path, assume false on all unnamed UDS\n                path.is_some() && path == other.as_unix().and_then(|addr| addr.as_pathname())\n            }\n        }\n    }\n}\n\nimpl PartialOrd for SocketAddr {\n    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {\n        Some(self.cmp(other))\n    }\n}\n\nimpl Ord for SocketAddr {\n    fn cmp(&self, other: &Self) -> Ordering {\n        match self {\n            Self::Inet(addr) => {\n                if let Some(o) = other.as_inet() {\n                    addr.cmp(o)\n                } else {\n                    // always make Inet < Unix \"smallest for variants at the top\"\n                    Ordering::Less\n                }\n            }\n            Self::Unix(addr) => {\n                if let Some(o) = other.as_unix() {\n                    // NOTE: unnamed UDS are consider the same\n                    addr.as_pathname().cmp(&o.as_pathname())\n                } else {\n                    // always make Inet < Unix \"smallest for variants at the top\"\n                    Ordering::Greater\n                }\n            }\n        }\n    }\n}\n\nimpl Eq for SocketAddr {}\n\nimpl std::str::FromStr for SocketAddr {\n    type Err = Box<Error>;\n\n    // This is very basic parsing logic, it might treat invalid IP:PORT str as UDS path\n    // TODO: require UDS to have some prefix\n    fn from_str(s: &str) -> Result<Self, Self::Err> {\n        match StdSockAddr::from_str(s) {\n            Ok(addr) => Ok(SocketAddr::Inet(addr)),\n            Err(_) => {\n                let uds_socket = StdUnixSockAddr::from_pathname(s)\n                    .or_err(crate::BindError, \"invalid UDS path\")?;\n                Ok(SocketAddr::Unix(uds_socket))\n            }\n        }\n    }\n}\n\nimpl std::net::ToSocketAddrs for SocketAddr {\n    type Iter = std::iter::Once<StdSockAddr>;\n\n    // Error if UDS addr\n    fn to_socket_addrs(&self) -> std::io::Result<Self::Iter> {\n        if let Some(inet) = self.as_inet() {\n            Ok(std::iter::once(*inet))\n        } else {\n            Err(std::io::Error::new(\n                std::io::ErrorKind::Other,\n                \"UDS socket cannot be used as inet socket\",\n            ))\n        }\n    }\n}\n\nimpl From<StdSockAddr> for SocketAddr {\n    fn from(sockaddr: StdSockAddr) -> Self {\n        SocketAddr::Inet(sockaddr)\n    }\n}\n\nimpl From<StdUnixSockAddr> for SocketAddr {\n    fn from(sockaddr: StdUnixSockAddr) -> Self {\n        SocketAddr::Unix(sockaddr)\n    }\n}\n\n// TODO: ideally mio/tokio will start using the std version of the unix `SocketAddr`\n// so we can avoid a fallible conversion\n// https://github.com/tokio-rs/mio/issues/1527\nimpl TryFrom<TokioUnixSockAddr> for SocketAddr {\n    type Error = String;\n\n    fn try_from(value: TokioUnixSockAddr) -> Result<Self, Self::Error> {\n        if let Some(Ok(addr)) = value.as_pathname().map(StdUnixSockAddr::from_pathname) {\n            Ok(addr.into())\n        } else {\n            // may be unnamed/abstract UDS\n            Err(format!(\"could not convert {value:?} to SocketAddr\"))\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn parse_ip() {\n        let ip: SocketAddr = \"127.0.0.1:80\".parse().unwrap();\n        assert!(ip.as_inet().is_some());\n    }\n\n    #[test]\n    fn parse_uds() {\n        let uds: SocketAddr = \"/tmp/my.sock\".parse().unwrap();\n        assert!(uds.as_unix().is_some());\n    }\n}\n\n// Path: pingora-core/src/utils/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! This module contains various types that make it easier to work with bytes and X509\n//! certificates.\n\n// TODO: move below to its own mod\nuse crate::tls::{nid::Nid, pkey::PKey, pkey::Private, x509::X509};\nuse crate::Result;\nuse bytes::Bytes;\nuse pingora_error::{ErrorType::*, OrErr};\nuse std::hash::{Hash, Hasher};\n\n/// A `BufRef` is a reference to a buffer of bytes. It removes the need for self-referential data\n/// structures. It is safe to use as long as the underlying buffer does not get mutated.\n///\n/// # Panics\n///\n/// This will panic if an index is out of bounds.\n#[derive(Clone, PartialEq, Eq, Debug)]\npub struct BufRef(pub usize, pub usize);\n\nimpl BufRef {\n    /// Return a sub-slice of `buf`.\n    pub fn get<'a>(&self, buf: &'a [u8]) -> &'a [u8] {\n        &buf[self.0..self.1]\n    }\n\n    /// Return a slice of `buf`. This operation is O(1) and increases the reference count of `buf`.\n    pub fn get_bytes(&self, buf: &Bytes) -> Bytes {\n        buf.slice(self.0..self.1)\n    }\n\n    /// Return the size of the slice reference.\n    pub fn len(&self) -> usize {\n        self.1 - self.0\n    }\n\n    /// Return true if the length is zero.\n    pub fn is_empty(&self) -> bool {\n        self.1 == self.0\n    }\n}\n\nimpl BufRef {\n    /// Initialize a `BufRef` that can reference a slice beginning at index `start` and has a\n    /// length of `len`.\n    pub fn new(start: usize, len: usize) -> Self {\n        BufRef(start, start + len)\n    }\n}\n\n/// A `KVRef` contains a key name and value pair, stored as two [BufRef] types.\n#[derive(Clone)]\npub struct KVRef {\n    name: BufRef,\n    value: BufRef,\n}\n\nimpl KVRef {\n    /// Like [BufRef::get] for the name.\n    pub fn get_name<'a>(&self, buf: &'a [u8]) -> &'a [u8] {\n        self.name.get(buf)\n    }\n\n    /// Like [BufRef::get] for the value.\n    pub fn get_value<'a>(&self, buf: &'a [u8]) -> &'a [u8] {\n        self.value.get(buf)\n    }\n\n    /// Like [BufRef::get_bytes] for the name.\n    pub fn get_name_bytes(&self, buf: &Bytes) -> Bytes {\n        self.name.get_bytes(buf)\n    }\n\n    /// Like [BufRef::get_bytes] for the value.\n    pub fn get_value_bytes(&self, buf: &Bytes) -> Bytes {\n        self.value.get_bytes(buf)\n    }\n\n    /// Return a new `KVRef` with name and value start indices and lengths.\n    pub fn new(name_s: usize, name_len: usize, value_s: usize, value_len: usize) -> Self {\n        KVRef {\n            name: BufRef(name_s, name_s + name_len),\n            value: BufRef(value_s, value_s + value_len),\n        }\n    }\n\n    /// Return a reference to the value.\n    pub fn value(&self) -> &BufRef {\n        &self.value\n    }\n}\n\n/// A [KVRef] which contains empty sub-slices.\npub const EMPTY_KV_REF: KVRef = KVRef {\n    name: BufRef(0, 0),\n    value: BufRef(0, 0),\n};\n\nfn get_subject_name(cert: &X509, name_type: Nid) -> Option<String> {\n    cert.subject_name()\n        .entries_by_nid(name_type)\n        .next()\n        .map(|name| {\n            name.data()\n                .as_utf8()\n                .map(|s| s.to_string())\n                .unwrap_or_default()\n        })\n}\n\n...\n", "needle_token_start": 5311, "needle_token_end": 5413, "code_context_ntokens": 14294, "output": ["```rust\nfn serialize_vec_string(vec_string: &[String], mut buf: &mut [u8]) -> usize {\n    // There are many ways to do this. Serde is probably the way to go\n    // But let's start with something simple: space separated strings\n    let joined = vec_string.join(\" \");\n    // TODO: check the buf is large enough\n    buf.write(joined.as_bytes()).unwrap()\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "h1_is_preferred", "language": "rust", "path": "pingora-core/src/connectors/http/v2.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To determine if the HTTP/1 protocol is the preferred communication protocol for a specific network peer.\n2. **Input**: A network peer which implements a specific interface.\n3. **Output**: A boolean value indicating whether HTTP/1 is the preferred protocol for the given peer.\n4. **Procedure**: The function checks a map within the transport layer for the preferred HTTP version associated with the given peer. If the preferred version is HTTP/1, it returns true; otherwise, it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/connectors/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Connecting to servers\n\npub mod http;\nmod l4;\nmod offload;\nmod tls;\n\nuse crate::protocols::Stream;\nuse crate::server::configuration::ServerConf;\nuse crate::tls::ssl::SslConnector;\nuse crate::upstreams::peer::{Peer, ALPN};\n\nuse l4::connect as l4_connect;\nuse log::{debug, error, warn};\nuse offload::OffloadRuntime;\nuse parking_lot::RwLock;\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_pool::{ConnectionMeta, ConnectionPool};\nuse std::collections::HashMap;\nuse std::net::SocketAddr;\nuse std::sync::Arc;\nuse tokio::sync::Mutex;\n\n/// The options to configure a [TransportConnector]\n#[derive(Clone)]\npub struct ConnectorOptions {\n    /// Path to the CA file used to validate server certs.\n    ///\n    /// If `None`, the CA in the [default](https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_default_verify_paths.html)\n    /// locations will be loaded\n    pub ca_file: Option<String>,\n    /// The default client cert and key to use for mTLS\n    ///\n    /// Each individual connection can use their own cert key to override this.\n    pub cert_key_file: Option<(String, String)>,\n    /// How many connections to keepalive\n    pub keepalive_pool_size: usize,\n    /// Optionally offload the connection establishment to dedicated thread pools\n    ///\n    /// TCP and TLS connection establishment can be CPU intensive. Sometimes such tasks can slow\n    /// down the entire service, which causes timeouts which leads to more connections which\n    /// snowballs the issue. Use this option to isolate these CPU intensive tasks from impacting\n    /// other traffic.\n    ///\n    /// Syntax: (#pools, #thread in each pool)\n    pub offload_threadpool: Option<(usize, usize)>,\n    /// Bind to any of the given source IPv6 addresses\n    pub bind_to_v4: Vec<SocketAddr>,\n    /// Bind to any of the given source IPv4 addresses\n    pub bind_to_v6: Vec<SocketAddr>,\n}\n\nimpl ConnectorOptions {\n    /// Derive the [ConnectorOptions] from a [ServerConf]\n    pub fn from_server_conf(server_conf: &ServerConf) -> Self {\n        // if both pools and threads are Some(>0)\n        let offload_threadpool = server_conf\n            .upstream_connect_offload_threadpools\n            .zip(server_conf.upstream_connect_offload_thread_per_pool)\n            .filter(|(pools, threads)| *pools > 0 && *threads > 0);\n\n        // create SocketAddrs with port 0 for src addr bind\n\n        let bind_to_v4 = server_conf\n            .client_bind_to_ipv4\n            .iter()\n            .map(|v4| {\n                let ip = v4.parse().unwrap();\n                SocketAddr::new(ip, 0)\n            })\n            .collect();\n\n        let bind_to_v6 = server_conf\n            .client_bind_to_ipv6\n            .iter()\n            .map(|v6| {\n                let ip = v6.parse().unwrap();\n                SocketAddr::new(ip, 0)\n            })\n            .collect();\n        ConnectorOptions {\n            ca_file: server_conf.ca_file.clone(),\n            cert_key_file: None, // TODO: use it\n            keepalive_pool_size: server_conf.upstream_keepalive_pool_size,\n            offload_threadpool,\n            bind_to_v4,\n            bind_to_v6,\n        }\n    }\n\n    /// Create a new [ConnectorOptions] with the given keepalive pool size\n    pub fn new(keepalive_pool_size: usize) -> Self {\n        ConnectorOptions {\n            ca_file: None,\n            cert_key_file: None,\n            keepalive_pool_size,\n            offload_threadpool: None,\n            bind_to_v4: vec![],\n            bind_to_v6: vec![],\n        }\n    }\n}\n\n/// [TransportConnector] provides APIs to connect to servers via TCP or TLS with connection reuse\npub struct TransportConnector {\n    tls_ctx: tls::Connector,\n    connection_pool: Arc<ConnectionPool<Arc<Mutex<Stream>>>>,\n    offload: Option<OffloadRuntime>,\n    bind_to_v4: Vec<SocketAddr>,\n    bind_to_v6: Vec<SocketAddr>,\n    preferred_http_version: PreferredHttpVersion,\n}\n\nconst DEFAULT_POOL_SIZE: usize = 128;\n\nimpl TransportConnector {\n    /// Create a new [TransportConnector] with the given [ConnectorOptions]\n    pub fn new(mut options: Option<ConnectorOptions>) -> Self {\n        let pool_size = options\n            .as_ref()\n            .map_or(DEFAULT_POOL_SIZE, |c| c.keepalive_pool_size);\n        // Take the offloading setting there because this layer has implement offloading,\n        // so no need for stacks at lower layer to offload again.\n        let offload = options.as_mut().and_then(|o| o.offload_threadpool.take());\n        let bind_to_v4 = options\n            .as_ref()\n            .map_or_else(Vec::new, |o| o.bind_to_v4.clone());\n        let bind_to_v6 = options\n            .as_ref()\n            .map_or_else(Vec::new, |o| o.bind_to_v6.clone());\n        TransportConnector {\n            tls_ctx: tls::Connector::new(options),\n            connection_pool: Arc::new(ConnectionPool::new(pool_size)),\n            offload: offload.map(|v| OffloadRuntime::new(v.0, v.1)),\n            bind_to_v4,\n            bind_to_v6,\n            preferred_http_version: PreferredHttpVersion::new(),\n        }\n    }\n\n    /// Connect to the given server [Peer]\n    ///\n    /// No connection is reused.\n    pub async fn new_stream<P: Peer + Send + Sync + 'static>(&self, peer: &P) -> Result<Stream> {\n        let rt = self\n            .offload\n            .as_ref()\n            .map(|o| o.get_runtime(peer.reuse_hash()));\n        let bind_to = l4::bind_to_random(peer, &self.bind_to_v4, &self.bind_to_v6);\n        let alpn_override = self.preferred_http_version.get(peer);\n        let stream = if let Some(rt) = rt {\n            let peer = peer.clone();\n            let tls_ctx = self.tls_ctx.clone();\n            rt.spawn(async move { do_connect(&peer, bind_to, alpn_override, &tls_ctx.ctx).await })\n                .await\n                .or_err(InternalError, \"offload runtime failure\")??\n        } else {\n            do_connect(peer, bind_to, alpn_override, &self.tls_ctx.ctx).await?\n        };\n\n        Ok(stream)\n    }\n\n    /// Try to find a reusable connection to the given server [Peer]\n    pub async fn reused_stream<P: Peer + Send + Sync>(&self, peer: &P) -> Option<Stream> {\n        match self.connection_pool.get(&peer.reuse_hash()) {\n            Some(s) => {\n                debug!(\"find reusable stream, trying to acquire it\");\n                {\n                    let _ = s.lock().await;\n                } // wait for the idle poll to release it\n                match Arc::try_unwrap(s) {\n                    Ok(l) => {\n                        let mut stream = l.into_inner();\n                        // test_reusable_stream: we assume server would never actively send data\n                        // first on an idle stream.\n                        if peer.matches_fd(stream.id()) && test_reusable_stream(&mut stream) {\n                            Some(stream)\n                        } else {\n                            None\n                        }\n                    }\n                    Err(_) => {\n                        error!(\"failed to acquire reusable stream\");\n                        None\n                    }\n                }\n            }\n            None => {\n                debug!(\"No reusable connection found for {peer}\");\n                None\n            }\n        }\n    }\n\n    /// Return the [Stream] to the [TransportConnector] for connection reuse.\n    ///\n    /// Not all TCP/TLS connections can be reused. It is the caller's responsibility to make sure\n    /// that protocol over the [Stream] supports connection reuse and the [Stream] itself is ready\n    /// to be reused.\n    ///\n    /// If a [Stream] is dropped instead of being returned via this function. it will be closed.\n    pub fn release_stream(\n        &self,\n        mut stream: Stream,\n        key: u64, // usually peer.reuse_hash()\n        idle_timeout: Option<std::time::Duration>,\n    ) {\n        if !test_reusable_stream(&mut stream) {\n            return;\n        }\n        let id = stream.id();\n        let meta = ConnectionMeta::new(key, id);\n        debug!(\"Try to keepalive client session\");\n        let stream = Arc::new(Mutex::new(stream));\n        let locked_stream = stream.clone().try_lock_owned().unwrap(); // safe as we just created it\n        let (notify_close, watch_use) = self.connection_pool.put(&meta, stream);\n        let pool = self.connection_pool.clone(); //clone the arc\n        let rt = pingora_runtime::current_handle();\n        rt.spawn(async move {\n            pool.idle_poll(locked_stream, &meta, idle_timeout, notify_close, watch_use)\n                .await;\n        });\n    }\n\n    /// Get a stream to the given server [Peer]\n    ///\n    /// This function will try to find a reusable [Stream] first. If there is none, a new connection\n    /// will be made to the server.\n    ///\n    /// The returned boolean will indicate whether the stream is reused.\n    pub async fn get_stream<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<(Stream, bool)> {\n        let reused_stream = self.reused_stream(peer).await;\n        if let Some(s) = reused_stream {\n            Ok((s, true))\n        } else {\n            let s = self.new_stream(peer).await?;\n            Ok((s, false))\n        }\n    }\n\n    /// Tell the connector to always send h1 for ALPN for the given peer in the future.\n    pub fn prefer_h1(&self, peer: &impl Peer) {\n        self.preferred_http_version.add(peer, 1);\n    }\n}\n\n// Perform the actual L4 and tls connection steps while respecting the peer's\n// connection timeout if there is one\nasync fn do_connect<P: Peer + Send + Sync>(\n    peer: &P,\n    bind_to: Option<SocketAddr>,\n    alpn_override: Option<ALPN>,\n    tls_ctx: &SslConnector,\n) -> Result<Stream> {\n    // Create the future that does the connections, but don't evaluate it until\n    // we decide if we need a timeout or not\n    let connect_future = do_connect_inner(peer, bind_to, alpn_override, tls_ctx);\n\n    match peer.total_connection_timeout() {\n        Some(t) => match pingora_timeout::timeout(t, connect_future).await {\n            Ok(res) => res,\n            Err(_) => Error::e_explain(\n                ConnectTimedout,\n                format!(\"connecting to server {peer}, total-connection timeout {t:?}\"),\n            ),\n        },\n        None => connect_future.await,\n    }\n}\n\n// Perform the actual L4 and tls connection steps with no timeout\nasync fn do_connect_inner<P: Peer + Send + Sync>(\n    peer: &P,\n    bind_to: Option<SocketAddr>,\n    alpn_override: Option<ALPN>,\n    tls_ctx: &SslConnector,\n) -> Result<Stream> {\n    let stream = l4_connect(peer, bind_to).await?;\n    if peer.tls() {\n        let tls_stream = tls::connect(stream, peer, alpn_override, tls_ctx).await?;\n        Ok(Box::new(tls_stream))\n    } else {\n        Ok(Box::new(stream))\n    }\n}\n\nstruct PreferredHttpVersion {\n    // TODO: shard to avoid the global lock\n    versions: RwLock<HashMap<u64, u8>>, // <hash of peer, version>\n}\n\n// TODO: limit the size of this\n\nimpl PreferredHttpVersion {\n    pub fn new() -> Self {\n        PreferredHttpVersion {\n            versions: RwLock::default(),\n        }\n    }\n\n    pub fn add(&self, peer: &impl Peer, version: u8) {\n        let key = peer.reuse_hash();\n        let mut v = self.versions.write();\n        v.insert(key, version);\n    }\n\n    pub fn get(&self, peer: &impl Peer) -> Option<ALPN> {\n        let key = peer.reuse_hash();\n        let v = self.versions.read();\n        v.get(&key)\n            .copied()\n            .map(|v| if v == 1 { ALPN::H1 } else { ALPN::H2H1 })\n    }\n}\n\nuse futures::future::FutureExt;\nuse tokio::io::AsyncReadExt;\n\n/// Test whether a stream is already closed or not reusable (server sent unexpected data)\nfn test_reusable_stream(stream: &mut Stream) -> bool {\n    let mut buf = [0; 1];\n    let result = stream.read(&mut buf[..]).now_or_never();\n    if let Some(data_result) = result {\n        match data_result {\n            Ok(n) => {\n                if n == 0 {\n                    debug!(\"Idle connection is closed\");\n                } else {\n                    warn!(\"Unexpected data read in idle connection\");\n                }\n            }\n            Err(e) => {\n                debug!(\"Idle connection is broken: {e:?}\");\n            }\n        }\n        false\n    } else {\n        true\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use pingora_error::ErrorType;\n\n    use super::*;\n    use crate::tls::ssl::SslMethod;\n    use crate::upstreams::peer::BasicPeer;\n\n    // 192.0.2.1 is effectively a black hole\n    const BLACK_HOLE: &str = \"192.0.2.1:79\";\n\n    #[tokio::test]\n    async fn test_connect() {\n        let connector = TransportConnector::new(None);\n        let peer = BasicPeer::new(\"1.1.1.1:80\");\n        // make a new connection to 1.1.1.1\n        let stream = connector.new_stream(&peer).await.unwrap();\n        connector.release_stream(stream, peer.reuse_hash(), None);\n\n        let (_, reused) = connector.get_stream(&peer).await.unwrap();\n        assert!(reused);\n    }\n\n    #[tokio::test]\n    async fn test_connect_tls() {\n        let connector = TransportConnector::new(None);\n        let mut peer = BasicPeer::new(\"1.1.1.1:443\");\n        // BasicPeer will use tls when SNI is set\n        peer.sni = \"one.one.one.one\".to_string();\n        // make a new connection to https://1.1.1.1\n        let stream = connector.new_stream(&peer).await.unwrap();\n        connector.release_stream(stream, peer.reuse_hash(), None);\n\n        let (_, reused) = connector.get_stream(&peer).await.unwrap();\n        assert!(reused);\n    }\n\n    async fn do_test_conn_timeout(conf: Option<ConnectorOptions>) {\n        let connector = TransportConnector::new(conf);\n        let mut peer = BasicPeer::new(BLACK_HOLE);\n        peer.options.connection_timeout = Some(std::time::Duration::from_millis(1));\n        let stream = connector.new_stream(&peer).await;\n        match stream {\n            Ok(_) => panic!(\"should throw an error\"),\n            Err(e) => assert_eq!(e.etype(), &ConnectTimedout),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_conn_timeout() {\n        do_test_conn_timeout(None).await;\n    }\n\n    #[tokio::test]\n    async fn test_conn_timeout_with_offload() {\n        let mut conf = ConnectorOptions::new(8);\n        conf.offload_threadpool = Some((2, 2));\n        do_test_conn_timeout(Some(conf)).await;\n    }\n\n    #[tokio::test]\n    async fn test_connector_bind_to() {\n        // connect to remote while bind to localhost will fail\n        let peer = BasicPeer::new(\"240.0.0.1:80\");\n        let mut conf = ConnectorOptions::new(1);\n        conf.bind_to_v4.push(\"127.0.0.1:0\".parse().unwrap());\n        let connector = TransportConnector::new(Some(conf));\n\n        let stream = connector.new_stream(&peer).await;\n        let error = stream.unwrap_err();\n        // XXX: some systems will allow the socket to bind and connect without error, only to timeout\n        assert!(error.etype() == &ConnectError || error.etype() == &ConnectTimedout)\n    }\n\n    /// Helper function for testing error handling in the `do_connect` function.\n    /// This assumes that the connection will fail to on the peer and returns\n    /// the decomposed error type and message\n    async fn get_do_connect_failure_with_peer(peer: &BasicPeer) -> (ErrorType, String) {\n        let ssl_connector = SslConnector::builder(SslMethod::tls()).unwrap().build();\n        let stream = do_connect(peer, None, None, &ssl_connector).await;\n        match stream {\n            Ok(_) => panic!(\"should throw an error\"),\n            Err(e) => (\n                e.etype().clone(),\n                e.context\n                    .as_ref()\n                    .map(|ctx| ctx.as_str().to_owned())\n                    .unwrap_or_default(),\n            ),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_do_connect_with_total_timeout() {\n        let mut peer = BasicPeer::new(BLACK_HOLE);\n        peer.options.total_connection_timeout = Some(std::time::Duration::from_millis(1));\n        let (etype, context) = get_do_connect_failure_with_peer(&peer).await;\n        assert_eq!(etype, ConnectTimedout);\n        assert!(context.contains(\"total-connection timeout\"));\n    }\n\n    #[tokio::test]\n    async fn test_tls_connect_timeout_supersedes_total() {\n        let mut peer = BasicPeer::new(BLACK_HOLE);\n        peer.options.total_connection_timeout = Some(std::time::Duration::from_millis(10));\n        peer.options.connection_timeout = Some(std::time::Duration::from_millis(1));\n...\n// Path: pingora-core/src/connectors/http/v2.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse super::HttpSession;\nuse crate::connectors::{ConnectorOptions, TransportConnector};\nuse crate::protocols::http::v1::client::HttpSession as Http1Session;\nuse crate::protocols::http::v2::client::{drive_connection, Http2Session};\nuse crate::protocols::{Digest, Stream};\nuse crate::upstreams::peer::{Peer, ALPN};\n\nuse bytes::Bytes;\nuse h2::client::SendRequest;\nuse log::debug;\nuse parking_lot::RwLock;\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_pool::{ConnectionMeta, ConnectionPool, PoolNode};\nuse std::collections::HashMap;\nuse std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};\nuse std::sync::Arc;\nuse std::time::Duration;\nuse tokio::sync::watch;\n\nstruct Stub(SendRequest<Bytes>);\n\nimpl Stub {\n    async fn new_stream(&self) -> Result<SendRequest<Bytes>> {\n        let send_req = self.0.clone();\n        send_req\n            .ready()\n            .await\n            .or_err(H2Error, \"while creating new stream\")\n    }\n}\n\npub(crate) struct ConnectionRefInner {\n    connection_stub: Stub,\n    closed: watch::Receiver<bool>,\n    ping_timeout_occurred: Arc<AtomicBool>,\n    id: i32,\n    // max concurrent streams this connection is allowed to create\n    max_streams: usize,\n    // how many concurrent streams already active\n    current_streams: AtomicUsize,\n    // because `SendRequest` doesn't actually have access to the underlying Stream,\n    // we log info about timing and tcp info here.\n    pub(crate) digest: Digest,\n}\n\n#[derive(Clone)]\npub(crate) struct ConnectionRef(Arc<ConnectionRefInner>);\n\nimpl ConnectionRef {\n    pub fn new(\n        send_req: SendRequest<Bytes>,\n        closed: watch::Receiver<bool>,\n        ping_timeout_occurred: Arc<AtomicBool>,\n        id: i32,\n        max_streams: usize,\n        digest: Digest,\n    ) -> Self {\n        ConnectionRef(Arc::new(ConnectionRefInner {\n            connection_stub: Stub(send_req),\n            closed,\n            ping_timeout_occurred,\n            id,\n            max_streams,\n            current_streams: AtomicUsize::new(0),\n            digest,\n        }))\n    }\n    pub fn more_streams_allowed(&self) -> bool {\n        self.0.max_streams > self.0.current_streams.load(Ordering::Relaxed)\n    }\n\n    pub fn is_idle(&self) -> bool {\n        self.0.current_streams.load(Ordering::Relaxed) == 0\n    }\n\n    pub fn release_stream(&self) {\n        self.0.current_streams.fetch_sub(1, Ordering::SeqCst);\n    }\n\n    pub fn id(&self) -> i32 {\n        self.0.id\n    }\n\n    pub fn digest(&self) -> &Digest {\n        &self.0.digest\n    }\n\n    pub fn ping_timedout(&self) -> bool {\n        self.0.ping_timeout_occurred.load(Ordering::Relaxed)\n    }\n\n    pub fn is_closed(&self) -> bool {\n        *self.0.closed.borrow()\n    }\n\n    // spawn a stream if more stream is allowed, otherwise return Ok(None)\n    pub async fn spawn_stream(&self) -> Result<Option<Http2Session>> {\n        // Atomically check if the current_stream is over the limit\n        // load(), compare and then fetch_add() cannot guarantee the same\n        let current_streams = self.0.current_streams.fetch_add(1, Ordering::SeqCst);\n        if current_streams >= self.0.max_streams {\n            // already over the limit, reset the counter to the previous value\n            self.0.current_streams.fetch_sub(1, Ordering::SeqCst);\n            return Ok(None);\n        }\n        let send_req = self.0.connection_stub.new_stream().await.map_err(|e| {\n            // fail to create the stream, reset the counter\n            self.0.current_streams.fetch_sub(1, Ordering::SeqCst);\n            e\n        })?;\n\n        Ok(Some(Http2Session::new(send_req, self.clone())))\n    }\n}\n\nstruct InUsePool {\n    // TODO: use pingora hashmap to shard the lock contention\n    pools: RwLock<HashMap<u64, PoolNode<ConnectionRef>>>,\n}\n\nimpl InUsePool {\n    fn new() -> Self {\n        InUsePool {\n            pools: RwLock::new(HashMap::new()),\n        }\n    }\n\n    fn insert(&self, reuse_hash: u64, conn: ConnectionRef) {\n        {\n            let pools = self.pools.read();\n            if let Some(pool) = pools.get(&reuse_hash) {\n                pool.insert(conn.id(), conn);\n                return;\n            }\n        } // drop read lock\n\n        let pool = PoolNode::new();\n        pool.insert(conn.id(), conn);\n        let mut pools = self.pools.write();\n        pools.insert(reuse_hash, pool);\n    }\n\n    // retrieve a h2 conn ref to create a new stream\n    // the caller should return the conn ref to this pool if there are still\n    // capacity left for more streams\n    fn get(&self, reuse_hash: u64) -> Option<ConnectionRef> {\n        let pools = self.pools.read();\n        pools.get(&reuse_hash)?.get_any().map(|v| v.1)\n    }\n\n    // release a h2_stream, this functional will cause an ConnectionRef to be returned (if exist)\n    // the caller should update the ref and then decide where to put it (in use pool or idle)\n    fn release(&self, reuse_hash: u64, id: i32) -> Option<ConnectionRef> {\n        let pools = self.pools.read();\n        if let Some(pool) = pools.get(&reuse_hash) {\n            pool.remove(id)\n        } else {\n            None\n        }\n    }\n}\n\nconst DEFAULT_POOL_SIZE: usize = 128;\n\n/// Http2 connector\npub struct Connector {\n    // just for creating connections, the Stream of h2 should be reused\n    transport: TransportConnector,\n    // the h2 connection idle pool\n    idle_pool: Arc<ConnectionPool<ConnectionRef>>,\n    // the pool of h2 connections that have ongoing streams\n    in_use_pool: InUsePool,\n}\n\nimpl Connector {\n    /// Create a new [Connector] from the given [ConnectorOptions]\n    pub fn new(options: Option<ConnectorOptions>) -> Self {\n        let pool_size = options\n            .as_ref()\n            .map_or(DEFAULT_POOL_SIZE, |o| o.keepalive_pool_size);\n        // connection offload is handled by the [TransportConnector]\n        Connector {\n            transport: TransportConnector::new(options),\n            idle_pool: Arc::new(ConnectionPool::new(pool_size)),\n            in_use_pool: InUsePool::new(),\n        }\n    }\n\n    /// Create a new Http2 connection to the given server\n    ///\n    /// Either an Http2 or Http1 session can be returned depending on the server's preference.\n    pub async fn new_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<HttpSession> {\n        let stream = self.transport.new_stream(peer).await?;\n\n        // check alpn\n        match stream.selected_alpn_proto() {\n            Some(ALPN::H2) => { /* continue */ }\n            Some(_) => {\n                // H2 not supported\n                return Ok(HttpSession::H1(Http1Session::new(stream)));\n            }\n            None => {\n                // if tls but no ALPN, default to h1\n                // else if plaintext and min http version is 1, this is most likely h1\n                if peer.tls()\n                    || peer\n                        .get_peer_options()\n                        .map_or(true, |o| o.alpn.get_min_http_version() == 1)\n                {\n                    return Ok(HttpSession::H1(Http1Session::new(stream)));\n                }\n                // else: min http version=H2 over plaintext, there is no ALPN anyways, we trust\n                // the caller that the server speaks h2c\n            }\n        }\n        let max_h2_stream = peer.get_peer_options().map_or(1, |o| o.max_h2_streams);\n        let conn = handshake(stream, max_h2_stream, peer.h2_ping_interval()).await?;\n        let h2_stream = conn\n            .spawn_stream()\n            .await?\n            .expect(\"newly created connections should have at least one free stream\");\n        if conn.more_streams_allowed() {\n            self.in_use_pool.insert(peer.reuse_hash(), conn);\n        }\n        Ok(HttpSession::H2(h2_stream))\n    }\n\n    /// Try to create a new http2 stream from any existing H2 connection.\n    ///\n    /// None means there is no \"free\" connection left.\n    pub async fn reused_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<Option<Http2Session>> {\n        // check in use pool first so that we use fewer total connections\n        // then idle pool\n        let reuse_hash = peer.reuse_hash();\n\n        // NOTE: We grab a conn from the pools, create a new stream and put the conn back if the\n        // conn has more free streams. During this process another caller could arrive but is not\n        // able to find the conn even the conn has free stream to use.\n        // We accept this false negative to keep the implementation simple. This false negative\n        // makes an actual impact when there are only a few connection.\n        // Alternative design 1. given each free stream a conn object: a lot of Arc<>\n        // Alternative design 2. mutex the pool, which creates lock contention when concurrency is high\n        // Alternative design 3. do not pop conn from the pool so that multiple callers can grab it\n        // which will cause issue where spawn_stream() could return None because others call it\n        // first. Thus a caller might have to retry or give up. This issue is more likely to happen\n        // when concurrency is high.\n        let maybe_conn = self\n            .in_use_pool\n            .get(reuse_hash)\n            .or_else(|| self.idle_pool.get(&reuse_hash));\n        if let Some(conn) = maybe_conn {\n            let h2_stream = conn\n                .spawn_stream()\n                .await?\n                .expect(\"connection from the pools should have free stream to allocate\");\n            if conn.more_streams_allowed() {\n                self.in_use_pool.insert(reuse_hash, conn);\n            }\n            Ok(Some(h2_stream))\n        } else {\n            Ok(None)\n        }\n    }\n\n    /// Release a finished h2 stream.\n    ///\n    /// This function will terminate the [Http2Session]. The corresponding h2 connection will now\n    /// have one more free stream to use.\n    ///\n    /// The h2 connection will be closed after `idle_timeout` if it has no active streams.\n    pub fn release_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        session: Http2Session,\n        peer: &P,\n        idle_timeout: Option<Duration>,\n    ) {\n        let id = session.conn.id();\n        let reuse_hash = peer.reuse_hash();\n        // get a ref to the connection, which we might need below, before dropping the h2\n        let conn = session.conn();\n        // this drop() will both drop the actual stream and call the conn.release_stream()\n        drop(session);\n        // find and remove the conn stored in in_use_pool so that it could be put in the idle pool\n        // if necessary\n        let conn = self.in_use_pool.release(reuse_hash, id).unwrap_or(conn);\n        if conn.is_closed() {\n            // Already dead h2 connection\n            return;\n        }\n        if conn.is_idle() {\n            let meta = ConnectionMeta {\n                key: reuse_hash,\n                id,\n            };\n            let closed = conn.0.closed.clone();\n            let (notify_evicted, watch_use) = self.idle_pool.put(&meta, conn);\n            if let Some(to) = idle_timeout {\n                let pool = self.idle_pool.clone(); //clone the arc\n                let rt = pingora_runtime::current_handle();\n                rt.spawn(async move {\n                    pool.idle_timeout(&meta, to, notify_evicted, closed, watch_use)\n                        .await;\n                });\n            }\n        } else {\n            self.in_use_pool.insert(reuse_hash, conn);\n        }\n    }\n\n    /// Tell the connector to always send h1 for ALPN for the given peer in the future.\n    pub fn prefer_h1(&self, peer: &impl Peer) {\n        self.transport.prefer_h1(peer);\n    }\n\n    \npub(crate) fn h1_is_preferred(&self, peer: &impl Peer) -> bool {\n        self.transport\n            .preferred_http_version\n            .get(peer)\n            .map_or(false, |v| matches!(v, ALPN::H1))\n    }\n}\n\n// The h2 library we use has unbounded internal buffering, which will cause excessive memory\n// consumption when the downstream is slower than upstream. This window size caps the buffering by\n// limiting how much data can be inflight. However, setting this value will also cap the max\n// download speed by limiting the bandwidth-delay product of a link.\n// Long term, we should advertising large window but shrink it when a small buffer is full.\n// 8 Mbytes = 80 Mbytes X 100ms, which should be enough for most links.\nconst H2_WINDOW_SIZE: u32 = 1 << 23;\n\nasync fn handshake(\n    stream: Stream,\n    max_streams: usize,\n    h2_ping_interval: Option<Duration>,\n) -> Result<ConnectionRef> {\n    use h2::client::Builder;\n    use pingora_runtime::current_handle;\n\n    // Safe guard: new_http_session() assumes there should be at least one free stream\n    if max_streams == 0 {\n        return Error::e_explain(H2Error, \"zero max_stream configured\");\n    }\n\n    let id = stream.id();\n    let digest = Digest {\n        // NOTE: this field is always false because the digest is shared across all streams\n        // The streams should log their own reuse info\n        ssl_digest: stream.get_ssl_digest(),\n        // TODO: log h2 handshake time\n        timing_digest: stream.get_timing_digest(),\n        proxy_digest: stream.get_proxy_digest(),\n        socket_digest: stream.get_socket_digest(),\n    };\n    // TODO: make these configurable\n    let (send_req, connection) = Builder::new()\n        .enable_push(false)\n        .initial_max_send_streams(max_streams)\n        // The limit for the server. Server push is not allowed, so this value doesn't matter\n        .max_concurrent_streams(1)\n        .max_frame_size(64 * 1024) // advise server to send larger frames\n        .initial_window_size(H2_WINDOW_SIZE)\n        // should this be max_streams * H2_WINDOW_SIZE?\n        .initial_connection_window_size(H2_WINDOW_SIZE)\n        .handshake(stream)\n        .await\n        .or_err(HandshakeError, \"during H2 handshake\")?;\n    debug!(\"H2 handshake to server done.\");\n    let ping_timeout_occurred = Arc::new(AtomicBool::new(false));\n    let ping_timeout_clone = ping_timeout_occurred.clone();\n    let max_allowed_streams = std::cmp::min(max_streams, connection.max_concurrent_send_streams());\n\n    // Safe guard: new_http_session() assumes there should be at least one free stream\n    // The server won't commonly advertise 0 max stream.\n    if max_allowed_streams == 0 {\n        return Error::e_explain(H2Error, \"zero max_concurrent_send_streams received\");\n    }\n\n    let (closed_tx, closed_rx) = watch::channel(false);\n\n    current_handle().spawn(async move {\n        drive_connection(\n            connection,\n            id,\n            closed_tx,\n            h2_ping_interval,\n            ping_timeout_clone,\n        )\n        .await;\n    });\n    Ok(ConnectionRef::new(\n        send_req,\n        closed_rx,\n        ping_timeout_occurred,\n        id,\n        max_allowed_streams,\n        digest,\n    ))\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::upstreams::peer::HttpPeer;\n\n    #[tokio::test]\n    async fn test_connect_h2() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        match h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => assert!(!h2_stream.ping_timedout()),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_connect_h1() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        // a hack to force h1, new_http_session() in the future might validate this setting\n        peer.options.set_http_version(1, 1);\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        match h2 {\n            HttpSession::H1(_) => {}\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_connect_h1_plaintext() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 80), false, \"\".into());\n        peer.options.set_http_version(2, 1);\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        match h2 {\n            HttpSession::H1(_) => {}\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_h2_single_stream() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        peer.options.max_h2_streams = 1;\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        let h2_1 = match h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => h2_stream,\n        };\n\n        let id = h2_1.conn.id();\n\n        assert!(connector\n            .reused_http_session(&peer)\n            .await\n            .unwrap()\n            .is_none());\n\n        connector.release_http_session(h2_1, &peer, None);\n\n        let h2_2 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_2.conn.id());\n\n        connector.release_http_session(h2_2, &peer, None);\n\n        let h2_3 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_3.conn.id());\n    }\n\n    #[tokio::test]\n    async fn test_h2_multiple_stream() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        peer.options.max_h2_streams = 3;\n        let h2 = connector.new_http_session(&peer).await.unwrap();\n        let h2_1 = match h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => h2_stream,\n        };\n\n        let id = h2_1.conn.id();\n\n        let h2_2 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_2.conn.id());\n        let h2_3 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_3.conn.id());\n\n        // max stream is 3 for now\n        assert!(connector\n            .reused_http_session(&peer)\n            .await\n            .unwrap()\n            .is_none());\n\n        connector.release_http_session(h2_1, &peer, None);\n\n        let h2_4 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_4.conn.id());\n\n        connector.release_http_session(h2_2, &peer, None);\n        connector.release_http_session(h2_3, &peer, None);\n        connector.release_http_session(h2_4, &peer, None);\n\n        // all streams are released, now the connection is idle\n        let h2_5 = connector.reused_http_session(&peer).await.unwrap().unwrap();\n        assert_eq!(id, h2_5.conn.id());\n    }\n}\n\n// Path: pingora-core/src/protocols/http/v2/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/2 client session and connection\n// TODO: this module needs a refactor\n\nuse bytes::Bytes;\nuse h2::client::{self, ResponseFuture, SendRequest};\nuse h2::{Reason, RecvStream, SendStream};\nuse http::HeaderMap;\nuse log::{debug, error, warn};\nuse pingora_error::{Error, ErrorType, ErrorType::*, OrErr, Result, RetryType};\nuse pingora_http::{RequestHeader, ResponseHeader};\nuse pingora_timeout::timeout;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::Arc;\nuse std::time::Duration;\nuse tokio::io::{AsyncRead, AsyncWrite};\nuse tokio::sync::watch;\n\nuse crate::connectors::http::v2::ConnectionRef;\nuse crate::protocols::{Digest, SocketAddr};\n\npub const PING_TIMEDOUT: ErrorType = ErrorType::new(\"PingTimedout\");\n\npub struct Http2Session {\n    send_req: SendRequest<Bytes>,\n    send_body: Option<SendStream<Bytes>>,\n    resp_fut: Option<ResponseFuture>,\n    req_sent: Option<Box<RequestHeader>>,\n    response_header: Option<ResponseHeader>,\n    response_body_reader: Option<RecvStream>,\n    /// The read timeout, which will be applied to both reading the header and the body.\n    /// The timeout is reset on every read. This is not a timeout on the overall duration of the\n    /// response.\n    pub read_timeout: Option<Duration>,\n    pub(crate) conn: ConnectionRef,\n    // Indicate that whether a END_STREAM is already sent\n    ended: bool,\n}\n\nimpl Drop for Http2Session {\n    fn drop(&mut self) {\n        self.conn.release_stream();\n    }\n}\n\nimpl Http2Session {\n    pub(crate) fn new(send_req: SendRequest<Bytes>, conn: ConnectionRef) -> Self {\n        Http2Session {\n            send_req,\n            send_body: None,\n            resp_fut: None,\n            req_sent: None,\n            response_header: None,\n            response_body_reader: None,\n            read_timeout: None,\n            conn,\n            ended: false,\n        }\n    }\n\n    fn sanitize_request_header(req: &mut RequestHeader) -> Result<()> {\n        req.set_version(http::Version::HTTP_2);\n        if req.uri.authority().is_some() {\n            return Ok(());\n        }\n        // use host header to populate :authority field\n        let Some(authority) = req.headers.get(http::header::HOST).map(|v| v.as_bytes()) else {\n            return Error::e_explain(InvalidHTTPHeader, \"no authority header for h2\");\n        };\n        let uri = http::uri::Builder::new()\n            .scheme(\"https\") // fixed for now\n            .authority(authority)\n            .path_and_query(req.uri.path_and_query().as_ref().unwrap().as_str())\n            .build();\n        match uri {\n            Ok(uri) => {\n                req.set_uri(uri);\n                Ok(())\n            }\n            Err(_) => Error::e_explain(\n                InvalidHTTPHeader,\n                format!(\"invalid authority from host {authority:?}\"),\n            ),\n        }\n    }\n\n    /// Write the request header to the server\n    pub fn write_request_header(&mut self, mut req: Box<RequestHeader>, end: bool) -> Result<()> {\n        if self.req_sent.is_some() {\n            // cannot send again, TODO: warn\n            return Ok(());\n        }\n        Self::sanitize_request_header(&mut req)?;\n        let parts = req.as_owned_parts();\n        let request = http::Request::from_parts(parts, ());\n        // There is no write timeout for h2 because the actual write happens async from this fn\n        let (resp_fut, send_body) = self\n            .send_req\n            .send_request(request, end)\n            .or_err(H2Error, \"while sending request\")\n            .map_err(|e| self.handle_err(e))?;\n        self.req_sent = Some(req);\n        self.send_body = Some(send_body);\n        self.resp_fut = Some(resp_fut);\n        self.ended = self.ended || end;\n\n        Ok(())\n    }\n\n    /// Write a request body chunk\n    pub fn write_request_body(&mut self, data: Bytes, end: bool) -> Result<()> {\n        if self.ended {\n            warn!(\"Try to write request body after end of stream, dropping the extra data\");\n            return Ok(());\n        }\n\n        let body_writer = self\n            .send_body\n            .as_mut()\n            .expect(\"Try to write request body before sending request header\");\n\n        write_body(body_writer, data, end).map_err(|e| self.handle_err(e))?;\n        self.ended = self.ended || end;\n        Ok(())\n    }\n\n    /// Signal that the request body has ended\n    pub fn finish_request_body(&mut self) -> Result<()> {\n        if self.ended {\n            return Ok(());\n        }\n\n        let body_writer = self\n            .send_body\n            .as_mut()\n            .expect(\"Try to finish request stream before sending request header\");\n\n        // Just send an empty data frame with end of stream set\n        body_writer\n            .send_data(\"\".into(), true)\n            .or_err(WriteError, \"while writing empty h2 request body\")\n            .map_err(|e| self.handle_err(e))?;\n        self.ended = true;\n        Ok(())\n    }\n\n    /// Read the response header\n    pub async fn read_response_header(&mut self) -> Result<()> {\n        // TODO: how to read 1xx headers?\n        // https://github.com/hyperium/h2/issues/167\n\n        if self.response_header.is_some() {\n            panic!(\"H2 response header is already read\")\n        }\n\n        let Some(resp_fut) = self.resp_fut.take() else {\n            panic!(\"Try to  response header is already read\")\n        };\n\n        let res = match self.read_timeout {\n            Some(t) => timeout(t, resp_fut)\n                .await\n                .map_err(|_| Error::explain(ReadTimedout, \"while reading h2 response header\"))\n                .map_err(|e| self.handle_err(e))?,\n            None => resp_fut.await,\n        };\n        let (resp, body_reader) = res.map_err(handle_read_header_error)?.into_parts();\n        self.response_header = Some(resp.into());\n        self.response_body_reader = Some(body_reader);\n\n        Ok(())\n    }\n\n    /// Read the response body\n    ///\n    /// `None` means, no more body to read\n    pub async fn read_response_body(&mut self) -> Result<Option<Bytes>> {\n        let Some(body_reader) = self.response_body_reader.as_mut() else {\n            // req is not sent or response is already read\n            // TODO: warn\n            return Ok(None);\n        };\n\n        if body_reader.is_end_stream() {\n            return Ok(None);\n        }\n\n        let fut = body_reader.data();\n        let res = match self.read_timeout {\n            Some(t) => timeout(t, fut)\n                .await\n                .map_err(|_| Error::explain(ReadTimedout, \"while reading h2 response body\"))?,\n            None => fut.await,\n        };\n        let body = res\n            .transpose()\n            .or_err(ReadError, \"while read h2 response body\")\n            .map_err(|mut e| {\n                // cannot use handle_err() because of borrow checker\n                if self.conn.ping_timedout() {\n                    e.etype = PING_TIMEDOUT;\n                }\n                e\n            })?;\n\n        if let Some(data) = body.as_ref() {\n            body_reader\n                .flow_control()\n                .release_capacity(data.len())\n                .or_err(ReadError, \"while releasing h2 response body capacity\")?;\n        }\n\n        Ok(body)\n    }\n\n    /// Whether the response has ended\n    pub fn response_finished(&self) -> bool {\n        // if response_body_reader doesn't exist, the response is not even read yet\n        self.response_body_reader\n            .as_ref()\n            .map_or(false, |reader| reader.is_end_stream())\n    }\n\n    /// Read the optional trailer headers\n    pub async fn read_trailers(&mut self) -> Result<Option<HeaderMap>> {\n        let Some(reader) = self.response_body_reader.as_mut() else {\n            // response is not even read\n            // TODO: warn\n            return Ok(None);\n        };\n        let fut = reader.trailers();\n\n        let res = match self.read_timeout {\n            Some(t) => timeout(t, fut)\n                .await\n                .map_err(|_| Error::explain(ReadTimedout, \"while reading h2 trailer\"))\n                .map_err(|e| self.handle_err(e))?,\n            None => fut.await,\n        };\n        match res {\n            Ok(t) => Ok(t),\n            Err(e) => {\n                // GOAWAY with no error: this is graceful shutdown, continue as if no trailer\n                // RESET_STREAM with no error: https://datatracker.ietf.org/doc/html/rfc9113#section-8.1:\n                // this is to signal client to stop uploading request without breaking the response.\n                // TODO: should actually stop uploading\n                // TODO: should we try reading again?\n                // TODO: handle this when reading headers and body as well\n                // https://github.com/hyperium/h2/issues/741\n\n                if (e.is_go_away() || e.is_reset())\n                    && e.is_remote()\n                    && e.reason() == Some(Reason::NO_ERROR)\n                {\n                    Ok(None)\n                } else {\n                    Err(e)\n                }\n            }\n        }\n        .or_err(ReadError, \"while reading h2 trailers\")\n    }\n\n    /// The response header if it is already read\n    pub fn response_header(&self) -> Option<&ResponseHeader> {\n        self.response_header.as_ref()\n    }\n\n    /// Give up the http session abruptly.\n    pub fn shutdown(&mut self) {\n        if !self.ended || !self.response_finished() {\n            if let Some(send_body) = self.send_body.as_mut() {\n                send_body.send_reset(h2::Reason::INTERNAL_ERROR)\n            }\n        }\n    }\n\n    /// Drop everything in this h2 stream. Return the connection ref.\n    /// After this function the underlying h2 connection should already notify the closure of this\n    /// stream so that another stream can be created if needed.\n    pub(crate) fn conn(&self) -> ConnectionRef {\n        self.conn.clone()\n    }\n\n    /// Whether ping timeout occurred. After a ping timeout, the h2 connection will be terminated.\n    /// Ongoing h2 streams will receive an stream/connection error. The streams should check this\n    /// flag to tell whether the error is triggered by the timeout.\n    pub(crate) fn ping_timedout(&self) -> bool {\n        self.conn.ping_timedout()\n    }\n\n    /// Return the [Digest] of the connection\n    ///\n    /// For reused connection, the timing in the digest will reflect its initial handshakes\n    /// The caller should check if the connection is reused to avoid misuse the timing field\n    pub fn digest(&self) -> Option<&Digest> {\n        Some(self.conn.digest())\n    }\n\n    /// Return the server (peer) address recorded in the connection digest.\n    pub fn server_addr(&self) -> Option<&SocketAddr> {\n        self.conn\n            .digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.peer_addr())?\n    }\n\n    /// Return the client (local) address recorded in the connection digest.\n    pub fn client_addr(&self) -> Option<&SocketAddr> {\n        self.conn\n            .digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.local_addr())?\n    }\n\n    /// the FD of the underlying connection\n    pub fn fd(&self) -> i32 {\n        self.conn.id()\n    }\n\n    /// take the body sender to another task to perform duplex read and write\n    pub fn take_request_body_writer(&mut self) -> Option<SendStream<Bytes>> {\n        self.send_body.take()\n    }\n\n    fn handle_err(&self, mut e: Box<Error>) -> Box<Error> {\n        if self.ping_timedout() {\n            e.etype = PING_TIMEDOUT;\n        }\n        e\n    }\n}\n\n/// A helper function to write the request body\npub fn write_body(send_body: &mut SendStream<Bytes>, data: Bytes, end: bool) -> Result<()> {\n    let data_len = data.len();\n    send_body.reserve_capacity(data_len);\n    send_body\n        .send_data(data, end)\n        .or_err(WriteError, \"while writing h2 request body\")\n}\n\n/* helper functions */\n\n/* Types of errors during h2 header read\n 1. peer requests to downgrade to h1, mostly IIS server for NTLM: we will downgrade and retry\n 2. peer sends invalid h2 frames, usually sending h1 only header: we will downgrade and retry\n 3. peer sends GO_AWAY(NO_ERROR) on reused conn, usually hit http2_max_requests: we will retry\n 4. peer IO error on reused conn, usually firewall kills old conn: we will retry\n 5. All other errors will terminate the request\n*/\nfn handle_read_header_error(e: h2::Error) -> Box<Error> {\n    if e.is_remote()\n        && e.reason()\n            .map_or(false, |r| r == h2::Reason::HTTP_1_1_REQUIRED)\n    {\n        let mut err = Error::because(H2Downgrade, \"while reading h2 header\", e);\n        err.retry = true.into();\n        err\n    } else if e.is_go_away()\n        && e.is_library()\n        && e.reason()\n            .map_or(false, |r| r == h2::Reason::PROTOCOL_ERROR)\n    {\n        // remote send invalid H2 responses\n        let mut err = Error::because(InvalidH2, \"while reading h2 header\", e);\n        err.retry = true.into();\n        err\n    } else if e.is_go_away()\n        && e.is_remote()\n        && e.reason().map_or(false, |r| r == h2::Reason::NO_ERROR)\n    {\n        // is_go_away: retry via another connection, this connection is being teardown\n        // only retry if the connection is reused\n        let mut err = Error::because(H2Error, \"while reading h2 header\", e);\n        err.retry = RetryType::ReusedOnly;\n        err\n    } else if e.is_io() {\n        // is_io: typical if a previously reused connection silently drops it\n        // only retry if the connection is reused\n        let true_io_error = e.get_io().unwrap().raw_os_error().is_some();\n        let mut err = Error::because(ReadError, \"while reading h2 header\", e);\n        if true_io_error {\n            err.retry = RetryType::ReusedOnly;\n        } // else could be TLS error, which is unsafe to retry\n        err\n    } else {\n        Error::because(H2Error, \"while reading h2 header\", e)\n    }\n}\n\nuse tokio::sync::oneshot;\n\npub async fn drive_connection<S>(\n    mut c: client::Connection<S>,\n    id: i32,\n    closed: watch::Sender<bool>,\n    ping_interval: Option<Duration>,\n    ping_timeout_occurred: Arc<AtomicBool>,\n) where\n    S: AsyncRead + AsyncWrite + Send + Unpin,\n{\n    let interval = ping_interval.unwrap_or(Duration::ZERO);\n    if !interval.is_zero() {\n        // for ping to inform this fn to drop the connection\n        let (tx, rx) = oneshot::channel::<()>();\n        // for this fn to inform ping to give up when it is already dropped\n        let dropped = Arc::new(AtomicBool::new(false));\n        let dropped2 = dropped.clone();\n\n        if let Some(ping_pong) = c.ping_pong() {\n            pingora_runtime::current_handle().spawn(async move {\n                do_ping_pong(ping_pong, interval, tx, dropped2, id).await;\n            });\n        } else {\n            warn!(\"Cannot get ping-pong handler from h2 connection\");\n        }\n\n        tokio::select! {\n            r = c => match r {\n                Ok(_) => debug!(\"H2 connection finished fd: {id}\"),\n                Err(e) => debug!(\"H2 connection fd: {id} errored: {e:?}\"),\n            },\n            r = rx => match r {\n                Ok(_) => {\n                    ping_timeout_occurred.store(true, Ordering::Relaxed);\n                    warn!(\"H2 connection Ping timeout/Error fd: {id}, closing conn\");\n                },\n                Err(e) => warn!(\"H2 connection Ping Rx error {e:?}\"),\n            },\n        };\n\n        dropped.store(true, Ordering::Relaxed);\n    } else {\n        match c.await {\n            Ok(_) => debug!(\"H2 connection finished fd: {id}\"),\n            Err(e) => debug!(\"H2 connection fd: {id} errored: {e:?}\"),\n        }\n    }\n    let _ = closed.send(true);\n}\n\nconst PING_TIMEOUT: Duration = Duration::from_secs(5);\n\nasync fn do_ping_pong(\n    mut ping_pong: h2::PingPong,\n    interval: Duration,\n    tx: oneshot::Sender<()>,\n    dropped: Arc<AtomicBool>,\n...\n", "needle_token_start": 8897, "needle_token_end": 8971, "code_context_ntokens": 16233, "output": ["```rust\npub(crate) fn h1_is_preferred(&self, peer: &impl Peer) -> bool {\n    self.transport\n        .preferred_http_version\n        .get(peer)\n        .map_or(false, |v| matches!(v, ALPN::H1))\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "test_conn_error_refused", "language": "rust", "path": "pingora-core/src/connectors/l4.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To verify that the system correctly handles and reports an error when attempting to connect to a port that is expected to be closed or not in use.\n2. **Input**: The test simulates a connection attempt to a specific IP address and port that is likely not open.\n3. **Output**: The expected output is an error indicating that the connection was refused.\n4. **Procedure**: The test initiates a connection to a designated IP address and port. It then checks whether the error returned upon failure matches the expected \"connection refused\" error type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/protocols/http/v1/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/1.x client session\n\nuse bytes::{BufMut, Bytes, BytesMut};\nuse http::{header, header::AsHeaderName, HeaderValue, StatusCode, Version};\nuse log::{debug, trace};\nuse pingora_error::{Error, ErrorType::*, OrErr, Result, RetryType};\nuse pingora_http::{HMap, IntoCaseHeaderName, RequestHeader, ResponseHeader};\nuse pingora_timeout::timeout;\nuse std::io::ErrorKind;\nuse std::str;\nuse std::time::Duration;\nuse tokio::io::{AsyncReadExt, AsyncWriteExt};\n\nuse super::body::{BodyReader, BodyWriter};\nuse super::common::*;\nuse crate::protocols::http::HttpTask;\nuse crate::protocols::{Digest, SocketAddr, Stream, UniqueID};\nuse crate::utils::{BufRef, KVRef};\n\n/// The HTTP 1.x client session\npub struct HttpSession {\n    buf: Bytes,\n    pub(crate) underlying_stream: Stream,\n    raw_header: Option<BufRef>,\n    preread_body: Option<BufRef>,\n    body_reader: BodyReader,\n    body_writer: BodyWriter,\n    // timeouts:\n    /// The read timeout, which will be applied to both reading the header and the body.\n    /// The timeout is reset on every read. This is not a timeout on the overall duration of the\n    /// response.\n    pub read_timeout: Option<Duration>,\n    /// The write timeout which will be applied to both writing request header and body.\n    /// The timeout is reset on every write. This is not a timeout on the overall duration of the\n    /// request.\n    pub write_timeout: Option<Duration>,\n    keepalive_timeout: KeepaliveStatus,\n    pub(crate) digest: Box<Digest>,\n    response_header: Option<Box<ResponseHeader>>,\n    request_written: Option<Box<RequestHeader>>,\n    bytes_sent: usize,\n    upgraded: bool,\n}\n\n/// HTTP 1.x client session\nimpl HttpSession {\n    /// Create a new http client session from an established (TCP or TLS) [`Stream`].\n    pub fn new(stream: Stream) -> Self {\n        // TODO: maybe we should put digest in the connection itself\n        let digest = Box::new(Digest {\n            ssl_digest: stream.get_ssl_digest(),\n            timing_digest: stream.get_timing_digest(),\n            proxy_digest: stream.get_proxy_digest(),\n            socket_digest: stream.get_socket_digest(),\n        });\n        HttpSession {\n            underlying_stream: stream,\n            buf: Bytes::new(), // zero size, will be replaced by parsed header later\n            raw_header: None,\n            preread_body: None,\n            body_reader: BodyReader::new(),\n            body_writer: BodyWriter::new(),\n            keepalive_timeout: KeepaliveStatus::Off,\n            response_header: None,\n            request_written: None,\n            read_timeout: None,\n            write_timeout: None,\n            digest,\n            bytes_sent: 0,\n            upgraded: false,\n        }\n    }\n    /// Write the request header to the server\n    /// After the request header is sent. The caller can either start reading the response or\n    /// sending request body if any.\n    pub async fn write_request_header(&mut self, req: Box<RequestHeader>) -> Result<usize> {\n        // TODO: make sure this can only be called once\n        // init body writer\n        self.init_req_body_writer(&req);\n\n        let to_wire = http_req_header_to_wire(&req).unwrap();\n        trace!(\"Writing request header: {to_wire:?}\");\n\n        let write_fut = self.underlying_stream.write_all(to_wire.as_ref());\n        match self.write_timeout {\n            Some(t) => match timeout(t, write_fut).await {\n                Ok(res) => res,\n                Err(_) => Err(std::io::Error::from(ErrorKind::TimedOut)),\n            },\n            None => write_fut.await,\n        }\n        .map_err(|e| match e.kind() {\n            ErrorKind::TimedOut => {\n                Error::because(WriteTimedout, \"while writing request headers (timeout)\", e)\n            }\n            _ => Error::because(WriteError, \"while writing request headers\", e),\n        })?;\n\n        self.underlying_stream\n            .flush()\n            .await\n            .or_err(WriteError, \"flushing request header\")?;\n\n        // write was successful\n        self.request_written = Some(req);\n        Ok(to_wire.len())\n    }\n\n    async fn do_write_body(&mut self, buf: &[u8]) -> Result<Option<usize>> {\n        let written = self\n            .body_writer\n            .write_body(&mut self.underlying_stream, buf)\n            .await;\n\n        if let Ok(Some(num_bytes)) = written {\n            self.bytes_sent += num_bytes;\n        }\n\n        written\n    }\n\n    /// Write request body. Return Ok(None) if no more body should be written, either due to\n    /// Content-Length or the last chunk is already sent\n    pub async fn write_body(&mut self, buf: &[u8]) -> Result<Option<usize>> {\n        // TODO: verify that request header is sent already\n        match self.write_timeout {\n            Some(t) => match timeout(t, self.do_write_body(buf)).await {\n                Ok(res) => res,\n                Err(_) => Error::e_explain(WriteTimedout, format!(\"writing body, timeout: {t:?}\")),\n            },\n            None => self.do_write_body(buf).await,\n        }\n    }\n\n    fn maybe_force_close_body_reader(&mut self) {\n        if self.upgraded && !self.body_reader.body_done() {\n            // request is done, reset the response body to close\n            self.body_reader.init_content_length(0, b\"\");\n        }\n    }\n\n    /// Flush local buffer and notify the server by sending the last chunk if chunked encoding is\n    /// used.\n    pub async fn finish_body(&mut self) -> Result<Option<usize>> {\n        let res = self.body_writer.finish(&mut self.underlying_stream).await?;\n        self.underlying_stream\n            .flush()\n            .await\n            .or_err(WriteError, \"flushing body\")?;\n\n        self.maybe_force_close_body_reader();\n        Ok(res)\n    }\n\n    /// Read the response header from the server\n    /// This function can be called multiple times, if the headers received are just informational\n    /// headers.\n    pub async fn read_response(&mut self) -> Result<usize> {\n        self.buf.clear();\n        let mut buf = BytesMut::with_capacity(INIT_HEADER_BUF_SIZE);\n        let mut already_read: usize = 0;\n        loop {\n            if already_read > MAX_HEADER_SIZE {\n                /* NOTE: this check only blocks second read. The first large read is allowed\n                since the buf is already allocated. The goal is to avoid slowly bloating\n                this buffer */\n                return Error::e_explain(\n                    InvalidHTTPHeader,\n                    format!(\"Response header larger than {MAX_HEADER_SIZE}\"),\n                );\n            }\n\n            let read_fut = self.underlying_stream.read_buf(&mut buf);\n            let read_result = match self.read_timeout {\n                Some(t) => timeout(t, read_fut)\n                    .await\n                    .map_err(|_| Error::explain(ReadTimedout, \"while reading response headers\"))?,\n                None => read_fut.await,\n            };\n            let n = match read_result {\n                Ok(n) => match n {\n                    0 => {\n                        let mut e = Error::explain(\n                            ConnectionClosed,\n                            format!(\n                                \"while reading response headers, bytes already read: {already_read}\",\n                            ),\n                        );\n                        e.retry = RetryType::ReusedOnly;\n                        return Err(e);\n                    }\n                    _ => {\n                        n /* read n bytes, continue */\n                    }\n                },\n                Err(e) => {\n                    let true_io_error = e.raw_os_error().is_some();\n                    let mut e = Error::because(\n                        ReadError,\n                        format!(\n                            \"while reading response headers, bytes already read: {already_read}\",\n                        ),\n                        e,\n                    );\n                    // Likely OSError, typical if a previously reused connection drops it\n                    if true_io_error {\n                        e.retry = RetryType::ReusedOnly;\n                    } // else: not safe to retry TLS error\n                    return Err(e);\n                }\n            };\n            already_read += n;\n            let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS];\n            let mut resp = httparse::Response::new(&mut headers);\n            let parsed = parse_resp_buffer(&mut resp, &buf);\n            match parsed {\n                HeaderParseState::Complete(s) => {\n                    self.raw_header = Some(BufRef(0, s));\n                    self.preread_body = Some(BufRef(s, already_read));\n                    let base = buf.as_ptr() as usize;\n                    let mut header_refs = Vec::<KVRef>::with_capacity(resp.headers.len());\n\n                    // Note: resp.headers has the correct number of headers\n                    // while header_refs doesn't as it is still empty\n                    let _num_headers = populate_headers(base, &mut header_refs, resp.headers);\n\n                    let mut response_header = Box::new(ResponseHeader::build(\n                        resp.code.unwrap(),\n                        Some(resp.headers.len()),\n                    )?);\n\n                    response_header.set_version(match resp.version {\n                        Some(1) => Version::HTTP_11,\n                        Some(0) => Version::HTTP_10,\n                        _ => Version::HTTP_09,\n                    });\n\n                    let buf = buf.freeze();\n\n                    for header in header_refs {\n                        let header_name = header.get_name_bytes(&buf);\n                        let header_name = header_name.into_case_header_name();\n                        let value_bytes = header.get_value_bytes(&buf);\n                        let header_value = if cfg!(debug_assertions) {\n                            // from_maybe_shared_unchecked() in debug mode still checks whether\n                            // the header value is valid, which breaks the _obsolete_multiline\n                            // support. To work around this, in debug mode, we replace CRLF with\n                            // whitespace\n                            if let Some(p) = value_bytes.windows(CRLF.len()).position(|w| w == CRLF)\n                            {\n                                let mut new_header = Vec::from_iter(value_bytes);\n                                new_header[p] = b' ';\n                                new_header[p + 1] = b' ';\n                                unsafe {\n                                    http::HeaderValue::from_maybe_shared_unchecked(new_header)\n                                }\n                            } else {\n                                unsafe {\n                                    http::HeaderValue::from_maybe_shared_unchecked(value_bytes)\n                                }\n                            }\n                        } else {\n                            // safe because this is from what we parsed\n                            unsafe { http::HeaderValue::from_maybe_shared_unchecked(value_bytes) }\n                        };\n                        response_header\n                            .append_header(header_name, header_value)\n                            .or_err(InvalidHTTPHeader, \"while parsing request header\")?;\n                    }\n\n                    self.buf = buf;\n                    self.upgraded = self.is_upgrade(&response_header).unwrap_or(false);\n                    self.response_header = Some(response_header);\n                    return Ok(s);\n                }\n                HeaderParseState::Partial => { /* continue the loop */ }\n                HeaderParseState::Invalid(e) => {\n                    return Error::e_because(\n                        InvalidHTTPHeader,\n                        format!(\"buf: {}\", String::from_utf8_lossy(&buf).escape_default()),\n                        e,\n                    );\n                }\n            }\n        }\n    }\n\n    /// Similar to [`Self::read_response()`], read the response header and then return a copy of it.\n    pub async fn read_resp_header_parts(&mut self) -> Result<Box<ResponseHeader>> {\n        self.read_response().await?;\n        // safe to unwrap because it is just read\n        Ok(Box::new(self.resp_header().unwrap().clone()))\n    }\n\n    /// Return a reference of the [`ResponseHeader`] if the response is read\n    pub fn resp_header(&self) -> Option<&ResponseHeader> {\n        self.response_header.as_deref()\n    }\n\n    /// Get the header value for the given header name from the response header\n    /// If there are multiple headers under the same name, the first one will be returned\n    /// Use `self.resp_header().header.get_all(name)` to get all the headers under the same name\n    /// Always return `None` if the response is not read yet.\n    pub fn get_header(&self, name: impl AsHeaderName) -> Option<&HeaderValue> {\n        self.response_header\n            .as_ref()\n            .and_then(|h| h.headers.get(name))\n    }\n\n    /// Get the request header as raw bytes, `b\"\"` when the header doesn't exist or response not read\n    pub fn get_header_bytes(&self, name: impl AsHeaderName) -> &[u8] {\n        self.get_header(name).map_or(b\"\", |v| v.as_bytes())\n    }\n\n    /// Return the status code of the response if read\n    pub fn get_status(&self) -> Option<StatusCode> {\n        self.response_header.as_ref().map(|h| h.status)\n    }\n\n    async fn do_read_body(&mut self) -> Result<Option<BufRef>> {\n        self.init_body_reader();\n        self.body_reader\n            .read_body(&mut self.underlying_stream)\n            .await\n    }\n\n    /// Read the response body into the internal buffer.\n    /// Return `Ok(Some(ref)) after a successful read.\n    /// Return `Ok(None)` if there is no more body to read.\n    pub async fn read_body_ref(&mut self) -> Result<Option<&[u8]>> {\n        let result = match self.read_timeout {\n            Some(t) => match timeout(t, self.do_read_body()).await {\n                Ok(res) => res,\n                Err(_) => Error::e_explain(ReadTimedout, format!(\"reading body, timeout: {t:?}\")),\n            },\n            None => self.do_read_body().await,\n        };\n\n        result.map(|maybe_body| maybe_body.map(|body_ref| self.body_reader.get_body(&body_ref)))\n    }\n\n    /// Similar to [`Self::read_body_ref`] but return `Bytes` instead of a slice reference.\n    pub async fn read_body_bytes(&mut self) -> Result<Option<Bytes>> {\n        let read = self.read_body_ref().await?;\n        Ok(read.map(Bytes::copy_from_slice))\n    }\n\n    /// Whether there is no more body to read.\n    pub fn is_body_done(&mut self) -> bool {\n        self.init_body_reader();\n        self.body_reader.body_done()\n    }\n\n    pub(super) fn get_headers_raw(&self) -> &[u8] {\n        // TODO: these get_*() could panic. handle them better\n        self.raw_header.as_ref().unwrap().get(&self.buf[..])\n    }\n\n    /// Get the raw response header bytes\n    pub fn get_headers_raw_bytes(&self) -> Bytes {\n        self.raw_header.as_ref().unwrap().get_bytes(&self.buf)\n    }\n\n    fn set_keepalive(&mut self, seconds: Option<u64>) {\n        match seconds {\n            Some(sec) => {\n                if sec > 0 {\n                    self.keepalive_timeout = KeepaliveStatus::Timeout(Duration::from_secs(sec));\n                } else {\n                    self.keepalive_timeout = KeepaliveStatus::Infinite;\n                }\n            }\n            None => {\n                self.keepalive_timeout = KeepaliveStatus::Off;\n            }\n        }\n    }\n\n    /// Apply keepalive settings according to the server's response\n    /// For HTTP 1.1, assume keepalive as long as there is no `Connection: Close` request header.\n    /// For HTTP 1.0, only keepalive if there is an explicit header `Connection: keep-alive`.\n    pub fn respect_keepalive(&mut self) {\n        if self.get_status() == Some(StatusCode::SWITCHING_PROTOCOLS) {\n            // make sure the connection is closed at the end when 101/upgrade is used\n            self.set_keepalive(None);\n            return;\n        }\n        if let Some(keepalive) = self.is_connection_keepalive() {\n            if keepalive {\n                let (timeout, _max_use) = self.get_keepalive_values();\n                // TODO: respect max_use\n                match timeout {\n                    Some(d) => self.set_keepalive(Some(d)),\n                    None => self.set_keepalive(Some(0)), // infinite\n                }\n            } else {\n                self.set_keepalive(None);\n            }\n        } else if self.resp_header().map(|h| h.version) == Some(Version::HTTP_11) {\n            self.set_keepalive(Some(0)); // on by default for http 1.1\n        } else {\n            self.set_keepalive(None); // off by default for http 1.0\n        }\n    }\n\n    // Whether this session will be kept alive\n    pub fn will_keepalive(&self) -> bool {\n        // TODO: check self.body_writer. If it is http1.0 type then keepalive\n        // cannot be used because the connection close is the signal of end body\n        !matches!(self.keepalive_timeout, KeepaliveStatus::Off)\n    }\n\n    fn is_connection_keepalive(&self) -> Option<bool> {\n        is_buf_keepalive(self.get_header(header::CONNECTION).map(|v| v.as_bytes()))\n    }\n\n    /// `Keep-Alive: timeout=5, max=1000` => 5, 1000\n    /// This is defined in the below spec, this not part of any RFC, so\n    /// it's behavior is different on different platforms.\n    /// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Keep-Alive\n    fn get_keepalive_values(&self) -> (Option<u64>, Option<usize>) {\n        let Some(keep_alive_header) = self.get_header(\"Keep-Alive\") else {\n            return (None, None);\n        };\n\n        let Ok(header_value) = str::from_utf8(keep_alive_header.as_bytes()) else {\n            return (None, None);\n        };\n\n        let mut timeout = None;\n        let mut max = None;\n\n        for param in header_value.split(',') {\n            let mut parts = param.splitn(2, '=').map(|s| s.trim());\n            match (parts.next(), parts.next()) {\n                (Some(\"timeout\"), Some(timeout_value)) => {\n                    timeout = timeout_value.trim().parse().ok()\n                }\n                (Some(\"max\"), Some(max_value)) => max = max_value.trim().parse().ok(),\n                _ => {}\n            }\n        }\n\n        (timeout, max)\n    }\n\n    /// Close the connection abruptly. This allows to signal the server that the connection is closed\n    /// before dropping [`HttpSession`]\n    pub async fn shutdown(&mut self) {\n        let _ = self.underlying_stream.shutdown().await;\n    }\n\n    /// Consume `self`, if the connection can be reused, the underlying stream will be returned.\n    /// The returned connection can be kept in a connection pool so that next time the same\n    /// server is being contacted. A new client session can be created via [`Self::new()`].\n    /// If the connection cannot be reused, the underlying stream will be closed and `None` will be\n    /// returned.\n    pub async fn reuse(mut self) -> Option<Stream> {\n        // TODO: this function is unnecessarily slow for keepalive case\n        // because that case does not need async\n        match self.keepalive_timeout {\n            KeepaliveStatus::Off => {\n                debug!(\"HTTP shutdown connection\");\n                self.shutdown().await;\n                None\n            }\n            _ => Some(self.underlying_stream),\n        }\n    }\n\n    fn init_body_reader(&mut self) {\n        if self.body_reader.need_init() {\n            /* follow https://tools.ietf.org/html/rfc7230#section-3.3.3 */\n            let preread_body = self.preread_body.as_ref().unwrap().get(&self.buf[..]);\n\n            if let Some(req) = self.request_written.as_ref() {\n                if req.method == http::method::Method::HEAD {\n                    self.body_reader.init_content_length(0, preread_body);\n                    return;\n                }\n            }\n\n            let upgraded = if let Some(code) = self.get_status() {\n                match code.as_u16() {\n                    101 => self.is_upgrade_req(),\n                    100..=199 => {\n                        // informational headers, not enough to init body reader\n                        return;\n                    }\n                    204 | 304 => {\n                        // no body by definition\n                        self.body_reader.init_content_length(0, preread_body);\n                        return;\n                    }\n                    _ => false,\n                }\n            } else {\n                false\n            };\n\n            if upgraded {\n                self.body_reader.init_http10(preread_body);\n            } else if self.is_chunked_encoding() {\n                // if chunked encoding, content-length should be ignored\n                self.body_reader.init_chunked(preread_body);\n            } else if let Some(cl) = self.get_content_length() {\n                self.body_reader.init_content_length(cl, preread_body);\n            } else {\n                self.body_reader.init_http10(preread_body);\n            }\n        }\n    }\n\n    /// Whether this request is for upgrade\n    pub fn is_upgrade_req(&self) -> bool {\n        match self.request_written.as_deref() {\n            Some(req) => is_upgrade_req(req),\n            None => false,\n        }\n    }\n\n    /// `Some(true)` if the this is a successful upgrade\n    /// `Some(false)` if the request is an upgrade but the response refuses it\n    /// `None` if the request is not an upgrade.\n    fn is_upgrade(&self, header: &ResponseHeader) -> Option<bool> {\n        if self.is_upgrade_req() {\n            Some(is_upgrade_resp(header))\n        } else {\n            None\n        }\n    }\n\n    fn get_content_length(&self) -> Option<usize> {\n        buf_to_content_length(\n            self.get_header(header::CONTENT_LENGTH)\n                .map(|v| v.as_bytes()),\n        )\n    }\n\n    fn is_chunked_encoding(&self) -> bool {\n        is_header_value_chunked_encoding(self.get_header(header::TRANSFER_ENCODING))\n    }\n\n    fn init_req_body_writer(&mut self, header: &RequestHeader) {\n        if is_upgrade_req(header) {\n            self.body_writer.init_http10();\n        } else {\n            self.init_body_writer_comm(&header.headers)\n        }\n    }\n\n    fn init_body_writer_comm(&mut self, headers: &HMap) {\n        let te_value = headers.get(http::header::TRANSFER_ENCODING);\n        if is_header_value_chunked_encoding(te_value) {\n            // transfer-encoding takes priority over content-length\n            self.body_writer.init_chunked();\n        } else {\n            let content_length =\n                header_value_content_length(headers.get(http::header::CONTENT_LENGTH));\n            match content_length {\n                Some(length) => {\n                    self.body_writer.init_content_length(length);\n                }\n                None => {\n                    /* TODO: 1. connection: keepalive cannot be used,\n                    2. mark connection must be closed */\n                    self.body_writer.init_http10();\n                }\n            }\n        }\n    }\n\n    // should (continue to) try to read response header or start reading response body\n    fn should_read_resp_header(&self) -> bool {\n        match self.get_status().map(|s| s.as_u16()) {\n            Some(101) => false,      // switching protocol successful, no more header to read\n            Some(100..=199) => true, // only informational header read\n            Some(_) => false,\n            None => true, // no response code, no header read yet\n        }\n    }\n\n    pub async fn read_response_task(&mut self) -> Result<HttpTask> {\n        if self.should_read_resp_header() {\n            let resp_header = self.read_resp_header_parts().await?;\n            let end_of_body = self.is_body_done();\n            debug!(\"Response header: {:?}\", resp_header);\n            trace!(\n                \"Raw Response header: {:?}\",\n                str::from_utf8(self.get_headers_raw()).unwrap()\n            );\n            Ok(HttpTask::Header(resp_header, end_of_body))\n        } else if self.is_body_done() {\n            debug!(\"Response is done\");\n            Ok(HttpTask::Done)\n        } else {\n            /* need to read body */\n            let data = self.read_body_bytes().await?;\n            let end_of_body = self.is_body_done();\n            if let Some(body) = data {\n                debug!(\"Response body: {} bytes\", body.len());\n                trace!(\"Response body: {:?}\", body);\n                Ok(HttpTask::Body(Some(body), end_of_body))\n            } else {\n                debug!(\"Response is done\");\n                Ok(HttpTask::Done)\n            }\n        }\n        // TODO: support h1 trailer\n    }\n\n    pub fn digest(&self) -> &Digest {\n        &self.digest\n    }\n\n    /// Return the server (peer) address recorded in the connection digest.\n    pub fn server_addr(&self) -> Option<&SocketAddr> {\n        self.digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.peer_addr())?\n    }\n\n    /// Return the client (local) address recorded in the connection digest.\n    pub fn client_addr(&self) -> Option<&SocketAddr> {\n        self.digest()\n            .socket_digest\n            .as_ref()\n            .map(|d| d.local_addr())?\n    }\n}\n\n#[inline]\nfn parse_resp_buffer<'buf>(\n    resp: &mut httparse::Response<'_, 'buf>,\n    buf: &'buf [u8],\n) -> HeaderParseState {\n    let mut parser = httparse::ParserConfig::default();\n    parser.allow_spaces_after_header_name_in_responses(true);\n    parser.allow_obsolete_multiline_headers_in_responses(true);\n    let res = match parser.parse_response(resp, buf) {\n        Ok(s) => s,\n        Err(e) => {\n            return HeaderParseState::Invalid(e);\n        }\n    };\n    match res {\n        httparse::Status::Complete(s) => HeaderParseState::Complete(s),\n        _ => HeaderParseState::Partial,\n    }\n}\n\n// TODO: change it to to_buf\n#[inline]\npub(crate) fn http_req_header_to_wire(req: &RequestHeader) -> Option<BytesMut> {\n    let mut buf = BytesMut::with_capacity(512);\n\n    // Request-Line\n    let method = req.method.as_str().as_bytes();\n    buf.put_slice(method);\n    buf.put_u8(b' ');\n    buf.put_slice(req.raw_path());\n    buf.put_u8(b' ');\n\n    let version = match req.version {\n        Version::HTTP_09 => \"HTTP/0.9\",\n        Version::HTTP_10 => \"HTTP/1.0\",\n        Version::HTTP_11 => \"HTTP/1.1\",\n        Version::HTTP_2 => \"HTTP/2\",\n        _ => {\n            return None; /*TODO: unsupported version */\n        }\n    };\n    buf.put_slice(version.as_bytes());\n    buf.put_slice(CRLF);\n\n    // headers\n    req.header_to_h1_wire(&mut buf);\n    buf.put_slice(CRLF);\n    Some(buf)\n}\n\nimpl UniqueID for HttpSession {\n    fn id(&self) -> i32 {\n        self.underlying_stream.id()\n    }\n}\n\n#[cfg(test)]\nmod tests_stream {\n    use super::*;\n    use crate::protocols::http::v1::body::ParseState;\n    use crate::ErrorType;\n    use tokio_test::io::Builder;\n\n    fn init_log() {\n        let _ = env_logger::builder().is_test(true).try_init();\n    }\n\n    #[tokio::test]\n    async fn read_basic_response() {\n        init_log();\n        let input = b\"HTTP/1.1 200 OK\\r\\n\\r\\n\";\n        let mock_io = Builder::new().read(&input[..]).build();\n        let mut http_stream = HttpSession::new(Box::new(mock_io));\n        let res = http_stream.read_response().await;\n        assert_eq!(input.len(), res.unwrap());\n        assert_eq!(0, http_stream.resp_header().unwrap().headers.len());\n    }\n\n    #[tokio::test]\n    async fn read_response_default() {\n        init_log();\n        let input_header = b\"HTTP/1.1 200 OK\\r\\n\\r\\n\";\n        let input_body = b\"abc\";\n        let input_close = b\"\"; // simulating close\n        let mock_io = Builder::new()\n            .read(&input_header[..])\n            .read(&input_body[..])\n            .read(&input_close[..])\n            .build();\n        let mut http_stream = HttpSession::new(Box::new(mock_io));\n        let res = http_stream.read_response().await;\n        assert_eq!(input_header.len(), res.unwrap());\n        let res = http_stream.read_body_ref().await.unwrap();\n        assert_eq!(res.unwrap(), input_body);\n        assert_eq!(http_stream.body_reader.body_state, ParseState::HTTP1_0(3));\n        let res = http_stream.read_body_ref().await.unwrap();\n        assert_eq!(res, None);\n        assert_eq!(http_stream.body_reader.body_state, ParseState::Complete(3));\n    }\n\n    #[tokio::test]\n    async fn read_resp_header_with_space() {\n        init_log();\n        let input = b\"HTTP/1.1 200 OK\\r\\nServer : pingora\\r\\n\\r\\n\";\n        let mock_io = Builder::new().read(&input[..]).build();\n...\n// Path: pingora-core/src/connectors/l4.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::debug;\nuse pingora_error::{Context, Error, ErrorType::*, OrErr, Result};\nuse rand::seq::SliceRandom;\nuse std::net::SocketAddr as InetSocketAddr;\nuse std::os::unix::io::AsRawFd;\n\nuse crate::protocols::l4::ext::{connect as tcp_connect, connect_uds, set_tcp_keepalive};\nuse crate::protocols::l4::socket::SocketAddr;\nuse crate::protocols::l4::stream::Stream;\nuse crate::protocols::{GetSocketDigest, SocketDigest};\nuse crate::upstreams::peer::Peer;\n\n/// Establish a connection (l4) to the given peer using its settings and an optional bind address.\npub async fn connect<P>(peer: &P, bind_to: Option<InetSocketAddr>) -> Result<Stream>\nwhere\n    P: Peer + Send + Sync,\n{\n    if peer.get_proxy().is_some() {\n        return proxy_connect(peer)\n            .await\n            .err_context(|| format!(\"Fail to establish CONNECT proxy: {}\", peer));\n    }\n    let peer_addr = peer.address();\n    let mut stream: Stream = match peer_addr {\n        SocketAddr::Inet(addr) => {\n            let connect_future = tcp_connect(addr, bind_to.as_ref());\n            let conn_res = match peer.connection_timeout() {\n                Some(t) => pingora_timeout::timeout(t, connect_future)\n                    .await\n                    .explain_err(ConnectTimedout, |_| {\n                        format!(\"timeout {t:?} connecting to server {peer}\")\n                    })?,\n                None => connect_future.await,\n            };\n            match conn_res {\n                Ok(socket) => {\n                    debug!(\"connected to new server: {}\", peer.address());\n                    if let Some(ka) = peer.tcp_keepalive() {\n                        debug!(\"Setting tcp keepalive\");\n                        set_tcp_keepalive(&socket, ka)?;\n                    }\n                    Ok(socket.into())\n                }\n                Err(e) => {\n                    let c = format!(\"Fail to connect to {peer}\");\n                    match e.etype() {\n                        SocketError | BindError => Error::e_because(InternalError, c, e),\n                        _ => Err(e.more_context(c)),\n                    }\n                }\n            }\n        }\n        SocketAddr::Unix(addr) => {\n            let connect_future = connect_uds(\n                addr.as_pathname()\n                    .expect(\"non-pathname unix sockets not supported as peer\"),\n            );\n            let conn_res = match peer.connection_timeout() {\n                Some(t) => pingora_timeout::timeout(t, connect_future)\n                    .await\n                    .explain_err(ConnectTimedout, |_| {\n                        format!(\"timeout {t:?} connecting to server {peer}\")\n                    })?,\n                None => connect_future.await,\n            };\n            match conn_res {\n                Ok(socket) => {\n                    debug!(\"connected to new server: {}\", peer.address());\n                    // no SO_KEEPALIVE for UDS\n                    Ok(socket.into())\n                }\n                Err(e) => {\n                    let c = format!(\"Fail to connect to {peer}\");\n                    match e.etype() {\n                        SocketError | BindError => Error::e_because(InternalError, c, e),\n                        _ => Err(e.more_context(c)),\n                    }\n                }\n            }\n        }\n    }?;\n    let tracer = peer.get_tracer();\n    if let Some(t) = tracer {\n        t.0.on_connected();\n        stream.tracer = Some(t);\n    }\n\n    stream.set_nodelay()?;\n\n    let digest = SocketDigest::from_raw_fd(stream.as_raw_fd());\n    digest\n        .peer_addr\n        .set(Some(peer_addr.clone()))\n        .expect(\"newly created OnceCell must be empty\");\n    stream.set_socket_digest(digest);\n\n    Ok(stream)\n}\n\npub(crate) fn bind_to_random<P: Peer>(\n    peer: &P,\n    v4_list: &[InetSocketAddr],\n    v6_list: &[InetSocketAddr],\n) -> Option<InetSocketAddr> {\n    let selected = peer.get_peer_options().and_then(|o| o.bind_to);\n    if selected.is_some() {\n        return selected;\n    }\n\n    fn bind_to_ips(ips: &[InetSocketAddr]) -> Option<InetSocketAddr> {\n        match ips.len() {\n            0 => None,\n            1 => Some(ips[0]),\n            _ => {\n                // pick a random bind ip\n                ips.choose(&mut rand::thread_rng()).copied()\n            }\n        }\n    }\n\n    match peer.address() {\n        SocketAddr::Inet(sockaddr) => match sockaddr {\n            InetSocketAddr::V4(_) => bind_to_ips(v4_list),\n            InetSocketAddr::V6(_) => bind_to_ips(v6_list),\n        },\n        SocketAddr::Unix(_) => None,\n    }\n}\n\nuse crate::protocols::raw_connect;\n\nasync fn proxy_connect<P: Peer>(peer: &P) -> Result<Stream> {\n    // safe to unwrap\n    let proxy = peer.get_proxy().unwrap();\n    let options = peer.get_peer_options().unwrap();\n\n    // combine required and optional headers\n    let mut headers = proxy\n        .headers\n        .iter()\n        .chain(options.extra_proxy_headers.iter());\n\n    // not likely to timeout during connect() to UDS\n    let stream: Box<Stream> = Box::new(\n        connect_uds(&proxy.next_hop)\n            .await\n            .or_err_with(ConnectError, || {\n                format!(\"CONNECT proxy connect() error to {:?}\", &proxy.next_hop)\n            })?\n            .into(),\n    );\n\n    let req_header = raw_connect::generate_connect_header(&proxy.host, proxy.port, &mut headers)?;\n    let fut = raw_connect::connect(stream, &req_header);\n    let (mut stream, digest) = match peer.connection_timeout() {\n        Some(t) => pingora_timeout::timeout(t, fut)\n            .await\n            .explain_err(ConnectTimedout, |_| \"establishing CONNECT proxy\")?,\n        None => fut.await,\n    }\n    .map_err(|mut e| {\n        // http protocol may ask to retry if reused client\n        e.retry.decide_reuse(false);\n        e\n    })?;\n    debug!(\"CONNECT proxy established: {:?}\", proxy);\n    stream.set_proxy_digest(digest);\n    let stream = stream.into_any().downcast::<Stream>().unwrap(); // safe, it is Stream from above\n    Ok(*stream)\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::upstreams::peer::{BasicPeer, HttpPeer, Proxy};\n    use std::collections::BTreeMap;\n    use std::path::PathBuf;\n    use tokio::io::AsyncWriteExt;\n    use tokio::net::UnixListener;\n\n    #[tokio::test]\n    \nasync fn test_conn_error_refused() {\n        let peer = BasicPeer::new(\"127.0.0.1:79\"); // hopefully port 79 is not used\n        let new_session = connect(&peer, None).await;\n        assert_eq!(new_session.unwrap_err().etype(), &ConnectRefused)\n    }\n\n    // TODO broken on arm64\n    #[ignore]\n    #[tokio::test]\n    async fn test_conn_error_no_route() {\n        let peer = BasicPeer::new(\"[::3]:79\"); // no route\n        let new_session = connect(&peer, None).await;\n        assert_eq!(new_session.unwrap_err().etype(), &ConnectNoRoute)\n    }\n\n    #[tokio::test]\n    async fn test_conn_error_addr_not_avail() {\n        let peer = HttpPeer::new(\"127.0.0.1:121\".to_string(), false, \"\".to_string());\n        let new_session = connect(&peer, Some(\"192.0.2.2:0\".parse().unwrap())).await;\n        assert_eq!(new_session.unwrap_err().etype(), &InternalError)\n    }\n\n    #[tokio::test]\n    async fn test_conn_error_other() {\n        let peer = HttpPeer::new(\"240.0.0.1:80\".to_string(), false, \"\".to_string()); // non localhost\n\n        // create an error: cannot send from src addr: localhost to dst addr: a public IP\n        let new_session = connect(&peer, Some(\"127.0.0.1:0\".parse().unwrap())).await;\n        let error = new_session.unwrap_err();\n        // XXX: some system will allow the socket to bind and connect without error, only to timeout\n        assert!(error.etype() == &ConnectError || error.etype() == &ConnectTimedout)\n    }\n\n    #[tokio::test]\n    async fn test_conn_timeout() {\n        // 192.0.2.1 is effectively a blackhole\n        let mut peer = BasicPeer::new(\"192.0.2.1:79\");\n        peer.options.connection_timeout = Some(std::time::Duration::from_millis(1)); //1ms\n        let new_session = connect(&peer, None).await;\n        assert_eq!(new_session.unwrap_err().etype(), &ConnectTimedout)\n    }\n\n    #[tokio::test]\n    async fn test_connect_proxy_fail() {\n        let mut peer = HttpPeer::new(\"1.1.1.1:80\".to_string(), false, \"\".to_string());\n        let mut path = PathBuf::new();\n        path.push(\"/tmp/123\");\n        peer.proxy = Some(Proxy {\n            next_hop: path.into(),\n            host: \"1.1.1.1\".into(),\n            port: 80,\n            headers: BTreeMap::new(),\n        });\n        let new_session = connect(&peer, None).await;\n        let e = new_session.unwrap_err();\n        assert_eq!(e.etype(), &ConnectError);\n        assert!(!e.retry());\n    }\n\n    const MOCK_UDS_PATH: &str = \"/tmp/test_unix_connect_proxy.sock\";\n\n    // one-off mock server\n    async fn mock_connect_server() {\n        let _ = std::fs::remove_file(MOCK_UDS_PATH);\n        let listener = UnixListener::bind(MOCK_UDS_PATH).unwrap();\n        if let Ok((mut stream, _addr)) = listener.accept().await {\n            stream.write_all(b\"HTTP/1.1 200 OK\\r\\n\\r\\n\").await.unwrap();\n            // wait a bit so that the client can read\n            tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        }\n        let _ = std::fs::remove_file(MOCK_UDS_PATH);\n    }\n\n    #[tokio::test(flavor = \"multi_thread\")]\n    async fn test_connect_proxy_work() {\n        tokio::spawn(async {\n            mock_connect_server().await;\n        });\n        // wait for the server to start\n        tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        let mut peer = HttpPeer::new(\"1.1.1.1:80\".to_string(), false, \"\".to_string());\n        let mut path = PathBuf::new();\n        path.push(MOCK_UDS_PATH);\n        peer.proxy = Some(Proxy {\n            next_hop: path.into(),\n            host: \"1.1.1.1\".into(),\n            port: 80,\n            headers: BTreeMap::new(),\n        });\n        let new_session = connect(&peer, None).await;\n        assert!(new_session.is_ok());\n    }\n\n    const MOCK_BAD_UDS_PATH: &str = \"/tmp/test_unix_bad_connect_proxy.sock\";\n\n    // one-off mock bad proxy\n    // closes connection upon accepting\n    async fn mock_connect_bad_server() {\n        let _ = std::fs::remove_file(MOCK_BAD_UDS_PATH);\n        let listener = UnixListener::bind(MOCK_BAD_UDS_PATH).unwrap();\n        if let Ok((mut stream, _addr)) = listener.accept().await {\n            stream.shutdown().await.unwrap();\n            tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        }\n        let _ = std::fs::remove_file(MOCK_BAD_UDS_PATH);\n    }\n\n    #[tokio::test(flavor = \"multi_thread\")]\n    async fn test_connect_proxy_conn_closed() {\n        tokio::spawn(async {\n            mock_connect_bad_server().await;\n        });\n        // wait for the server to start\n        tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        let mut peer = HttpPeer::new(\"1.1.1.1:80\".to_string(), false, \"\".to_string());\n        let mut path = PathBuf::new();\n        path.push(MOCK_BAD_UDS_PATH);\n        peer.proxy = Some(Proxy {\n            next_hop: path.into(),\n            host: \"1.1.1.1\".into(),\n            port: 80,\n            headers: BTreeMap::new(),\n        });\n        let new_session = connect(&peer, None).await;\n        let err = new_session.unwrap_err();\n        assert_eq!(err.etype(), &ConnectionClosed);\n        assert!(!err.retry());\n    }\n}\n\n// Path: pingora-core/src/connectors/offload.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::debug;\nuse once_cell::sync::OnceCell;\nuse rand::Rng;\nuse tokio::runtime::{Builder, Handle};\nuse tokio::sync::oneshot::{channel, Sender};\n\n// TODO: use pingora_runtime\n// a shared runtime (thread pools)\npub(crate) struct OffloadRuntime {\n    shards: usize,\n    thread_per_shard: usize,\n    // Lazily init the runtimes so that they are created after pingora\n    // daemonize itself. Otherwise the runtime threads are lost.\n    pools: OnceCell<Box<[(Handle, Sender<()>)]>>,\n}\n\nimpl OffloadRuntime {\n    pub fn new(shards: usize, thread_per_shard: usize) -> Self {\n        assert!(shards != 0);\n        assert!(thread_per_shard != 0);\n        OffloadRuntime {\n            shards,\n            thread_per_shard,\n            pools: OnceCell::new(),\n        }\n    }\n\n    fn init_pools(&self) -> Box<[(Handle, Sender<()>)]> {\n        let threads = self.shards * self.thread_per_shard;\n        let mut pools = Vec::with_capacity(threads);\n        for _ in 0..threads {\n            // We use single thread runtimes to reduce the scheduling overhead of multithread\n            // tokio runtime, which can be 50% of the on CPU time of the runtimes\n            let rt = Builder::new_current_thread().enable_all().build().unwrap();\n            let handler = rt.handle().clone();\n            let (tx, rx) = channel::<()>();\n            std::thread::Builder::new()\n                .name(\"Offload thread\".to_string())\n                .spawn(move || {\n                    debug!(\"Offload thread started\");\n                    // the thread that calls block_on() will drive the runtime\n                    // rx will return when tx is dropped so this runtime and thread will exit\n                    rt.block_on(rx)\n                })\n                .unwrap();\n            pools.push((handler, tx));\n        }\n\n        pools.into_boxed_slice()\n    }\n\n    pub fn get_runtime(&self, hash: u64) -> &Handle {\n        let mut rng = rand::thread_rng();\n\n        // choose a shard based on hash and a random thread with in that shard\n        // e.g. say thread_per_shard=2, shard 1 thread 1 is 1 * 2 + 1 = 3\n        // [[th0, th1], [th2, th3], ...]\n        let shard = hash as usize % self.shards;\n        let thread_in_shard = rng.gen_range(0..self.thread_per_shard);\n        let pools = self.pools.get_or_init(|| self.init_pools());\n        &pools[shard * self.thread_per_shard + thread_in_shard].0\n    }\n}\n\n// Path: pingora-core/src/connectors/http/v1.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse crate::connectors::{ConnectorOptions, TransportConnector};\nuse crate::protocols::http::v1::client::HttpSession;\nuse crate::upstreams::peer::Peer;\n\nuse pingora_error::Result;\nuse std::time::Duration;\n\npub struct Connector {\n    transport: TransportConnector,\n}\n\nimpl Connector {\n    pub fn new(options: Option<ConnectorOptions>) -> Self {\n        Connector {\n            transport: TransportConnector::new(options),\n        }\n    }\n\n    pub async fn get_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<(HttpSession, bool)> {\n        let (stream, reused) = self.transport.get_stream(peer).await?;\n        let http = HttpSession::new(stream);\n        Ok((http, reused))\n    }\n\n    pub async fn reused_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Option<HttpSession> {\n        self.transport\n            .reused_stream(peer)\n            .await\n            .map(HttpSession::new)\n    }\n\n    pub async fn release_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        session: HttpSession,\n        peer: &P,\n        idle_timeout: Option<Duration>,\n    ) {\n        if let Some(stream) = session.reuse().await {\n            self.transport\n                .release_stream(stream, peer.reuse_hash(), idle_timeout);\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::protocols::l4::socket::SocketAddr;\n    use crate::upstreams::peer::HttpPeer;\n    use pingora_http::RequestHeader;\n\n    async fn get_http(http: &mut HttpSession, expected_status: u16) {\n        let mut req = Box::new(RequestHeader::build(\"GET\", b\"/\", None).unwrap());\n        req.append_header(\"Host\", \"one.one.one.one\").unwrap();\n        http.write_request_header(req).await.unwrap();\n        http.read_response().await.unwrap();\n        http.respect_keepalive();\n\n        assert_eq!(http.get_status().unwrap(), expected_status);\n        while http.read_body_bytes().await.unwrap().is_some() {}\n    }\n\n    #[tokio::test]\n    async fn test_connect() {\n        let connector = Connector::new(None);\n        let peer = HttpPeer::new((\"1.1.1.1\", 80), false, \"\".into());\n        // make a new connection to 1.1.1.1\n        let (http, reused) = connector.get_http_session(&peer).await.unwrap();\n        let server_addr = http.server_addr().unwrap();\n        assert_eq!(*server_addr, \"1.1.1.1:80\".parse::<SocketAddr>().unwrap());\n        assert!(!reused);\n\n        // this http is not even used, so not be able to reuse\n        connector.release_http_session(http, &peer, None).await;\n        let (mut http, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n\n        get_http(&mut http, 301).await;\n        connector.release_http_session(http, &peer, None).await;\n        let (_, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(reused);\n    }\n\n    #[tokio::test]\n    async fn test_connect_tls() {\n        let connector = Connector::new(None);\n        let peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        // make a new connection to https://1.1.1.1\n        let (http, reused) = connector.get_http_session(&peer).await.unwrap();\n        let server_addr = http.server_addr().unwrap();\n        assert_eq!(*server_addr, \"1.1.1.1:443\".parse::<SocketAddr>().unwrap());\n        assert!(!reused);\n\n        // this http is not even used, so not be able to reuse\n        connector.release_http_session(http, &peer, None).await;\n        let (mut http, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n\n        get_http(&mut http, 200).await;\n        connector.release_http_session(http, &peer, None).await;\n        let (_, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(reused);\n    }\n}\n\n// Path: pingora-core/src/connectors/http/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Connecting to HTTP servers\n\nuse crate::connectors::ConnectorOptions;\nuse crate::protocols::http::client::HttpSession;\nuse crate::upstreams::peer::Peer;\nuse pingora_error::Result;\nuse std::time::Duration;\n\npub mod v1;\npub mod v2;\n\npub struct Connector {\n    h1: v1::Connector,\n    h2: v2::Connector,\n}\n\nimpl Connector {\n    pub fn new(options: Option<ConnectorOptions>) -> Self {\n        Connector {\n            h1: v1::Connector::new(options.clone()),\n            h2: v2::Connector::new(options),\n        }\n    }\n\n    /// Get an [HttpSession] to the given server.\n    ///\n    /// The second return value indicates whether the session is connected via a reused stream.\n    pub async fn get_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<(HttpSession, bool)> {\n        // NOTE: maybe TODO: we do not yet enforce that only TLS traffic can use h2, which is the\n        // de facto requirement for h2, because non TLS traffic lack the negotiation mechanism.\n\n        // We assume no peer option == no ALPN == h1 only\n        let h1_only = peer\n            .get_peer_options()\n            .map_or(true, |o| o.alpn.get_max_http_version() == 1);\n        if h1_only {\n            let (h1, reused) = self.h1.get_http_session(peer).await?;\n            Ok((HttpSession::H1(h1), reused))\n        } else {\n            // the peer allows h2, we first check the h2 reuse pool\n            let reused_h2 = self.h2.reused_http_session(peer).await?;\n            if let Some(h2) = reused_h2 {\n                return Ok((HttpSession::H2(h2), true));\n            }\n            let h2_only = peer\n                .get_peer_options()\n                .map_or(false, |o| o.alpn.get_min_http_version() == 2)\n                && !self.h2.h1_is_preferred(peer);\n            if !h2_only {\n                // We next check the reuse pool for h1 before creating a new h2 connection.\n                // This is because the server may not support h2 at all, connections to\n                // the server could all be h1.\n                if let Some(h1) = self.h1.reused_http_session(peer).await {\n                    return Ok((HttpSession::H1(h1), true));\n                }\n            }\n            let session = self.h2.new_http_session(peer).await?;\n            Ok((session, false))\n        }\n    }\n\n    pub async fn release_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        session: HttpSession,\n        peer: &P,\n        idle_timeout: Option<Duration>,\n    ) {\n        match session {\n            HttpSession::H1(h1) => self.h1.release_http_session(h1, peer, idle_timeout).await,\n            HttpSession::H2(h2) => self.h2.release_http_session(h2, peer, idle_timeout),\n        }\n    }\n\n    /// Tell the connector to always send h1 for ALPN for the given peer in the future.\n    pub fn prefer_h1(&self, peer: &impl Peer) {\n        self.h2.prefer_h1(peer);\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::protocols::http::v1::client::HttpSession as Http1Session;\n    use crate::upstreams::peer::HttpPeer;\n    use pingora_http::RequestHeader;\n\n    async fn get_http(http: &mut Http1Session, expected_status: u16) {\n        let mut req = Box::new(RequestHeader::build(\"GET\", b\"/\", None).unwrap());\n        req.append_header(\"Host\", \"one.one.one.one\").unwrap();\n        http.write_request_header(req).await.unwrap();\n        http.read_response().await.unwrap();\n        http.respect_keepalive();\n\n        assert_eq!(http.get_status().unwrap(), expected_status);\n        while http.read_body_bytes().await.unwrap().is_some() {}\n    }\n\n    #[tokio::test]\n    async fn test_connect_h2() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        let (h2, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n        match &h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => assert!(!h2_stream.ping_timedout()),\n        }\n\n        connector.release_http_session(h2, &peer, None).await;\n\n        let (h2, reused) = connector.get_http_session(&peer).await.unwrap();\n        // reused this time\n        assert!(reused);\n...\n", "needle_token_start": 10512, "needle_token_end": 10598, "code_context_ntokens": 16235, "output": ["```rust\n#[tokio::test]\nasync fn test_conn_error_refused() {\n    let peer = BasicPeer::new(\"127.0.0.1:79\"); // hopefully port 79 is not used\n    let new_session = connect(&peer, None).await;\n    assert_eq!(new_session.unwrap_err().etype(), &ConnectRefused)\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "get_bytes", "language": "rust", "path": "pingora-core/src/utils/mod.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to extract a specific segment of a byte buffer based on predefined start and end indices, returning this segment as a new, separate byte buffer.\n2. **Input**: The function takes a reference to a byte buffer that encapsulates binary data.\n3. **Output**: It outputs a new byte buffer that contains a portion of the original buffer, determined by the specified indices.\n4. **Procedure**: The function first accesses the original byte buffer using the provided indices to determine the segment of interest. It then creates and returns a new byte buffer that contains just this specific segment.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/server/configuration/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Server configurations\n//!\n//! Server configurations define startup settings such as:\n//! * User and group to run as after daemonization\n//! * Number of threads per service\n//! * Error log file path\n\nuse log::{debug, trace};\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse serde::{Deserialize, Serialize};\nuse std::fs;\nuse structopt::StructOpt;\n\n/// The configuration file\n///\n/// Pingora configuration files are by default YAML files, but any key value format can potentially\n/// be used.\n///\n/// # Extension\n/// New keys can be added to the configuration files which this configuration object will ignore.\n/// Then, users can parse these key-values to pass to their code to use.\n#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]\n#[serde(default)]\npub struct ServerConf {\n    /// Version\n    pub version: usize,\n    /// Whether to run this process in the background.\n    pub daemon: bool,\n    /// When configured, error log will be written to the given file. Otherwise StdErr will be used.\n    pub error_log: Option<String>,\n    /// The pid (process ID) file of this server\n    pub pid_file: String,\n    /// the path to the upgrade socket\n    ///\n    /// In order to perform zero downtime restart, both the new and old process need to agree on the\n    /// path to this sock in order to coordinate the upgrade.\n    pub upgrade_sock: String,\n    /// If configured, after daemonization, this process will switch to the given user before\n    /// starting to serve traffic.\n    pub user: Option<String>,\n    /// Similar to `user`, the group this process should switch to.\n    pub group: Option<String>,\n    /// How many threads **each** service should get. The threads are not shared across services.\n    pub threads: usize,\n    /// Allow work stealing between threads of the same service. Default `true`.\n    pub work_stealing: bool,\n    /// The path to CA file the SSL library should use. If empty, the default trust store location\n    /// defined by the SSL library will be used.\n    pub ca_file: Option<String>,\n    /// Grace period in seconds before starting the final step of the graceful shutdown after signaling shutdown.\n    pub grace_period_seconds: Option<u64>,\n    /// Timeout in seconds of the final step for the graceful shutdown.\n    pub graceful_shutdown_timeout_seconds: Option<u64>,\n    // These options don't belong here as they are specific to certain services\n    /// IPv4 addresses for a client connector to bind to. See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub client_bind_to_ipv4: Vec<String>,\n    /// IPv6 addresses for a client connector to bind to. See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub client_bind_to_ipv6: Vec<String>,\n    /// Keepalive pool size for client connections to upstream. See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub upstream_keepalive_pool_size: usize,\n    /// Number of dedicated thread pools to use for upstream connection establishment.\n    /// See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub upstream_connect_offload_threadpools: Option<usize>,\n    /// Number of threads per dedicated upstream connection establishment pool.\n    /// See [`ConnectorOptions`].\n    /// Note: this is an _unstable_ field that may be renamed or removed in the future.\n    pub upstream_connect_offload_thread_per_pool: Option<usize>,\n}\n\nimpl Default for ServerConf {\n    fn default() -> Self {\n        ServerConf {\n            version: 0,\n            client_bind_to_ipv4: vec![],\n            client_bind_to_ipv6: vec![],\n            ca_file: None,\n            daemon: false,\n            error_log: None,\n            pid_file: \"/tmp/pingora.pid\".to_string(),\n            upgrade_sock: \"/tmp/pingora_upgrade.sock\".to_string(),\n            user: None,\n            group: None,\n            threads: 1,\n            work_stealing: true,\n            upstream_keepalive_pool_size: 128,\n            upstream_connect_offload_threadpools: None,\n            upstream_connect_offload_thread_per_pool: None,\n            grace_period_seconds: None,\n            graceful_shutdown_timeout_seconds: None,\n        }\n    }\n}\n\n/// Command-line options\n///\n/// Call `Opt::from_args()` to build this object from the process's command line arguments.\n#[derive(StructOpt, Debug)]\n#[structopt(name = \"basic\")]\npub struct Opt {\n    /// Whether this server should try to upgrade from a running old server\n    ///\n    /// `-u` or `--upgrade` can be used\n    #[structopt(short, long)]\n    pub upgrade: bool,\n    /// Whether should run this server in the background\n    ///\n    /// `-d` or `--daemon` can be used\n    #[structopt(short, long)]\n    pub daemon: bool,\n    /// Not actually used. This flag is there so that the server is not upset seeing this flag\n    /// passed from `cargo test` sometimes\n    #[structopt(long)]\n    pub nocapture: bool,\n    /// Test the configuration and exit\n    ///\n    /// When this flag is set, calling `server.bootstrap()` will exit the process without errors\n    ///\n    /// This flag is useful for upgrading service where the user wants to make sure the new\n    /// service can start before shutting down the old server process.\n    ///\n    /// `-t` or `--test` can be used\n    #[structopt(short, long)]\n    pub test: bool,\n    /// The path to the configuration file.\n    ///\n    /// See [`ServerConf`] for more details of the configuration file.\n    ///\n    /// `-c` or `--conf` can be used\n    #[structopt(short, long)]\n    pub conf: Option<String>,\n}\n\n...\n// Path: pingora-core/src/server/daemon.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse daemonize::Daemonize;\nuse log::{debug, error};\nuse std::ffi::CString;\nuse std::fs::{self, OpenOptions};\nuse std::os::unix::prelude::OpenOptionsExt;\nuse std::path::Path;\n\nuse crate::server::configuration::ServerConf;\n\n// Utilities to daemonize a pingora server, i.e. run the process in the background, possibly\n// under a different running user and/or group.\n\n// XXX: this operation should have been done when the old service is exiting.\n// Now the new pid file just kick the old one out of the way\nfn move_old_pid(path: &str) {\n    if !Path::new(path).exists() {\n        debug!(\"Old pid file does not exist\");\n        return;\n    }\n    let new_path = format!(\"{path}.old\");\n    match fs::rename(path, &new_path) {\n        Ok(()) => {\n            debug!(\"Old pid file renamed\");\n        }\n        Err(e) => {\n            error!(\n                \"failed to rename pid file from {} to {}: {}\",\n                path, new_path, e\n            );\n        }\n    }\n}\n\nunsafe fn gid_for_username(name: &CString) -> Option<libc::gid_t> {\n    let passwd = libc::getpwnam(name.as_ptr() as *const libc::c_char);\n    if !passwd.is_null() {\n        return Some((*passwd).pw_gid);\n    }\n    None\n}\n\n/// Start a server instance as a daemon.\npub fn daemonize(conf: &ServerConf) {\n    // TODO: customize working dir\n\n    let daemonize = Daemonize::new()\n        .umask(0o007) // allow same group to access files but not everyone else\n        .pid_file(&conf.pid_file);\n\n    let daemonize = if let Some(error_log) = conf.error_log.as_ref() {\n        let err = OpenOptions::new()\n            .append(true)\n            .create(true)\n            // open read() in case there are no readers\n            // available otherwise we will panic with\n            // an ENXIO since O_NONBLOCK is set\n            .read(true)\n            .custom_flags(libc::O_NONBLOCK)\n            .open(error_log)\n            .unwrap();\n        daemonize.stderr(err)\n    } else {\n        daemonize\n    };\n\n    let daemonize = match conf.user.as_ref() {\n        Some(user) => {\n            let user_cstr = CString::new(user.as_str()).unwrap();\n\n            #[cfg(target_os = \"macos\")]\n            let group_id = unsafe { gid_for_username(&user_cstr).map(|gid| gid as i32) };\n            #[cfg(target_os = \"linux\")]\n            let group_id = unsafe { gid_for_username(&user_cstr) };\n\n            daemonize\n                .privileged_action(move || {\n                    if let Some(gid) = group_id {\n                        // Set the supplemental group privileges for the child process.\n                        unsafe {\n                            libc::initgroups(user_cstr.as_ptr() as *const libc::c_char, gid);\n                        }\n                    }\n                })\n                .user(user.as_str())\n                .chown_pid_file(true)\n        }\n        None => daemonize,\n    };\n\n    let daemonize = match conf.group.as_ref() {\n        Some(group) => daemonize.group(group.as_str()),\n        None => daemonize,\n    };\n\n    move_old_pid(&conf.pid_file);\n\n    daemonize.start().unwrap(); // hard crash when fail\n}\n\n// Path: pingora-core/src/server/transfer_fd/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n#[cfg(target_os = \"linux\")]\nuse log::{debug, error, warn};\nuse nix::errno::Errno;\n#[cfg(target_os = \"linux\")]\nuse nix::sys::socket::{self, AddressFamily, RecvMsg, SockFlag, SockType, UnixAddr};\n#[cfg(target_os = \"linux\")]\nuse nix::sys::stat;\nuse nix::{Error, NixPath};\nuse std::collections::HashMap;\nuse std::io::Write;\n#[cfg(target_os = \"linux\")]\nuse std::io::{IoSlice, IoSliceMut};\nuse std::os::unix::io::RawFd;\n#[cfg(target_os = \"linux\")]\nuse std::{thread, time};\n\n// Utilities to transfer file descriptors between sockets, e.g. during graceful upgrades.\n\n/// Container for open file descriptors and their associated bind addresses.\npub struct Fds {\n    map: HashMap<String, RawFd>,\n}\n\nimpl Fds {\n    pub fn new() -> Self {\n        Fds {\n            map: HashMap::new(),\n        }\n    }\n\n    pub fn add(&mut self, bind: String, fd: RawFd) {\n        self.map.insert(bind, fd);\n    }\n\n    pub fn get(&self, bind: &str) -> Option<&RawFd> {\n        self.map.get(bind)\n    }\n\n    pub fn serialize(&self) -> (Vec<String>, Vec<RawFd>) {\n        let serialized: Vec<(String, RawFd)> = self\n            .map\n            .iter()\n            .map(|(key, value)| (key.clone(), *value))\n            .collect();\n\n        (\n            serialized.iter().map(|v| v.0.clone()).collect(),\n            serialized.iter().map(|v| v.1).collect(),\n        )\n        // Surely there is a better way of doing this\n    }\n\n    pub fn deserialize(&mut self, binds: Vec<String>, fds: Vec<RawFd>) {\n        assert!(binds.len() == fds.len());\n        // TODO: use zip()\n        for i in 0..binds.len() {\n            self.map.insert(binds[i].clone(), fds[i]);\n        }\n    }\n\n    pub fn send_to_sock<P>(&self, path: &P) -> Result<usize, Error>\n    where\n        P: ?Sized + NixPath + std::fmt::Display,\n    {\n        let (vec_key, vec_fds) = self.serialize();\n        let mut ser_buf: [u8; 2048] = [0; 2048];\n        let ser_key_size = serialize_vec_string(&vec_key, &mut ser_buf);\n        send_fds_to(vec_fds, &ser_buf[..ser_key_size], path)\n    }\n\n    pub fn get_from_sock<P>(&mut self, path: &P) -> Result<(), Error>\n    where\n        P: ?Sized + NixPath + std::fmt::Display,\n    {\n        let mut de_buf: [u8; 2048] = [0; 2048];\n        let (fds, bytes) = get_fds_from(path, &mut de_buf)?;\n        let keys = deserialize_vec_string(&de_buf[..bytes]);\n        self.deserialize(keys, fds);\n        Ok(())\n    }\n}\n\nfn serialize_vec_string(vec_string: &[String], mut buf: &mut [u8]) -> usize {\n    // There are many ways to do this. Serde is probably the way to go\n    // But let's start with something simple: space separated strings\n    let joined = vec_string.join(\" \");\n    // TODO: check the buf is large enough\n    buf.write(joined.as_bytes()).unwrap()\n}\n\nfn deserialize_vec_string(buf: &[u8]) -> Vec<String> {\n    let joined = std::str::from_utf8(buf).unwrap(); // TODO: handle error\n    let mut results: Vec<String> = Vec::new();\n    for iter in joined.split_ascii_whitespace() {\n        results.push(String::from(iter));\n    }\n    results\n}\n\n#[cfg(target_os = \"linux\")]\npub fn get_fds_from<P>(path: &P, payload: &mut [u8]) -> Result<(Vec<RawFd>, usize), Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    const MAX_FDS: usize = 32;\n\n    let listen_fd = socket::socket(\n        AddressFamily::Unix,\n        SockType::Stream,\n        SockFlag::SOCK_NONBLOCK,\n        None,\n    )\n    .unwrap();\n    let unix_addr = UnixAddr::new(path).unwrap();\n    // clean up old sock\n    match nix::unistd::unlink(path) {\n        Ok(()) => {\n            debug!(\"unlink {} done\", path);\n        }\n        Err(e) => {\n            // Normal if file does not exist\n            debug!(\"unlink {} failed: {}\", path, e);\n            // TODO: warn if exist but not able to unlink\n        }\n    };\n    socket::bind(listen_fd, &unix_addr).unwrap();\n\n    /* sock is created before we change user, need to give permission to all */\n    stat::fchmodat(\n        None,\n        path,\n        stat::Mode::all(),\n        stat::FchmodatFlags::FollowSymlink,\n    )\n    .unwrap();\n\n    socket::listen(listen_fd, 8).unwrap();\n\n    let fd = match accept_with_retry(listen_fd) {\n        Ok(fd) => fd,\n        Err(e) => {\n            error!(\"Giving up reading socket from: {path}, error: {e:?}\");\n            //cleanup\n            if nix::unistd::close(listen_fd).is_ok() {\n                nix::unistd::unlink(path).unwrap();\n            }\n            return Err(e);\n        }\n    };\n\n    let mut io_vec = [IoSliceMut::new(payload); 1];\n    let mut cmsg_buf = nix::cmsg_space!([RawFd; MAX_FDS]);\n    let msg: RecvMsg<UnixAddr> = socket::recvmsg(\n        fd,\n        &mut io_vec,\n        Some(&mut cmsg_buf),\n        socket::MsgFlags::empty(),\n    )\n    .unwrap();\n\n    let mut fds: Vec<RawFd> = Vec::new();\n    for cmsg in msg.cmsgs() {\n        if let socket::ControlMessageOwned::ScmRights(mut vec_fds) = cmsg {\n            fds.append(&mut vec_fds)\n        } else {\n            warn!(\"Unexpected control messages: {cmsg:?}\")\n        }\n    }\n\n    //cleanup\n    if nix::unistd::close(listen_fd).is_ok() {\n        nix::unistd::unlink(path).unwrap();\n    }\n\n    Ok((fds, msg.bytes))\n}\n\n#[cfg(not(target_os = \"linux\"))]\npub fn get_fds_from<P>(_path: &P, _payload: &mut [u8]) -> Result<(Vec<RawFd>, usize), Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    Err(Errno::ECONNREFUSED)\n}\n\n#[cfg(target_os = \"linux\")]\nconst MAX_RETRY: usize = 5;\n#[cfg(target_os = \"linux\")]\nconst RETRY_INTERVAL: time::Duration = time::Duration::from_secs(1);\n\n#[cfg(target_os = \"linux\")]\nfn accept_with_retry(listen_fd: i32) -> Result<i32, Error> {\n    let mut retried = 0;\n    loop {\n        match socket::accept(listen_fd) {\n            Ok(fd) => return Ok(fd),\n            Err(e) => {\n                if retried > MAX_RETRY {\n                    return Err(e);\n                }\n                match e {\n                    Errno::EAGAIN => {\n                        error!(\n                            \"No incoming socket transfer, sleep {RETRY_INTERVAL:?} and try again\"\n                        );\n                        retried += 1;\n                        thread::sleep(RETRY_INTERVAL);\n                    }\n                    _ => {\n                        error!(\"Error accepting socket transfer: {e}\");\n                        return Err(e);\n                    }\n                }\n            }\n        }\n    }\n}\n\n#[cfg(target_os = \"linux\")]\npub fn send_fds_to<P>(fds: Vec<RawFd>, payload: &[u8], path: &P) -> Result<usize, Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    const MAX_NONBLOCKING_POLLS: usize = 20;\n    const NONBLOCKING_POLL_INTERVAL: time::Duration = time::Duration::from_millis(500);\n\n    let send_fd = socket::socket(\n        AddressFamily::Unix,\n        SockType::Stream,\n        SockFlag::SOCK_NONBLOCK,\n        None,\n    )?;\n    let unix_addr = UnixAddr::new(path)?;\n    let mut retried = 0;\n    let mut nonblocking_polls = 0;\n\n    let conn_result: Result<usize, Error> = loop {\n        match socket::connect(send_fd, &unix_addr) {\n            Ok(_) => break Ok(0),\n            Err(e) => match e {\n                /* If the new process hasn't created the upgrade sock we'll get an ENOENT.\n                ECONNREFUSED may happen if the sock wasn't cleaned up\n                and the old process tries sending before the new one is listening.\n                EACCES may happen if connect() happen before the correct permission is set */\n                Errno::ENOENT | Errno::ECONNREFUSED | Errno::EACCES => {\n                    /*the server is not ready yet*/\n                    retried += 1;\n                    if retried > MAX_RETRY {\n                        error!(\n                            \"Max retry: {} reached. Giving up sending socket to: {}, error: {:?}\",\n                            MAX_RETRY, path, e\n                        );\n                        break Err(e);\n                    }\n                    warn!(\"server not ready, will try again in {RETRY_INTERVAL:?}\");\n                    thread::sleep(RETRY_INTERVAL);\n                }\n                /* handle nonblocking IO */\n                Errno::EINPROGRESS => {\n                    nonblocking_polls += 1;\n                    if nonblocking_polls >= MAX_NONBLOCKING_POLLS {\n                        error!(\"Connect() not ready after retries when sending socket to: {path}\",);\n                        break Err(e);\n                    }\n                    warn!(\"Connect() not ready, will try again in {NONBLOCKING_POLL_INTERVAL:?}\",);\n                    thread::sleep(NONBLOCKING_POLL_INTERVAL);\n                }\n                _ => {\n                    error!(\"Error sending socket to: {path}, error: {e:?}\");\n                    break Err(e);\n                }\n            },\n        }\n    };\n\n    let result = match conn_result {\n        Ok(_) => {\n            let io_vec = [IoSlice::new(payload); 1];\n            let scm = socket::ControlMessage::ScmRights(fds.as_slice());\n            let cmsg = [scm; 1];\n            loop {\n                match socket::sendmsg(\n                    send_fd,\n                    &io_vec,\n                    &cmsg,\n                    socket::MsgFlags::empty(),\n                    None::<&UnixAddr>,\n                ) {\n                    Ok(result) => break Ok(result),\n                    Err(e) => match e {\n                        /* handle nonblocking IO */\n                        Errno::EAGAIN => {\n                            nonblocking_polls += 1;\n                            if nonblocking_polls >= MAX_NONBLOCKING_POLLS {\n                                error!(\n                                    \"Sendmsg() not ready after retries when sending socket to: {}\",\n                                    path\n                                );\n                                break Err(e);\n                            }\n                            warn!(\n                                \"Sendmsg() not ready, will try again in {:?}\",\n                                NONBLOCKING_POLL_INTERVAL\n                            );\n                            thread::sleep(NONBLOCKING_POLL_INTERVAL);\n                        }\n                        _ => break Err(e),\n                    },\n                }\n            }\n        }\n        Err(_) => conn_result,\n    };\n\n    nix::unistd::close(send_fd).unwrap();\n    result\n}\n\n#[cfg(not(target_os = \"linux\"))]\npub fn send_fds_to<P>(_fds: Vec<RawFd>, _payload: &[u8], _path: &P) -> Result<usize, Error>\nwhere\n    P: ?Sized + NixPath + std::fmt::Display,\n{\n    Ok(0)\n}\n\n#[cfg(test)]\n#[cfg(target_os = \"linux\")]\nmod tests {\n    use super::*;\n    use log::{debug, error};\n\n    fn init_log() {\n        let _ = env_logger::builder().is_test(true).try_init();\n    }\n\n    #[test]\n    fn test_add_get() {\n        init_log();\n        let mut fds = Fds::new();\n        let key = \"1.1.1.1:80\".to_string();\n        fds.add(key.clone(), 128);\n        assert_eq!(128, *fds.get(&key).unwrap());\n    }\n\n    #[test]\n    fn test_table_serde() {\n        init_log();\n        let mut fds = Fds::new();\n        let key1 = \"1.1.1.1:80\".to_string();\n        fds.add(key1.clone(), 128);\n        let key2 = \"1.1.1.1:443\".to_string();\n        fds.add(key2.clone(), 129);\n\n        let (k, v) = fds.serialize();\n        let mut fds2 = Fds::new();\n        fds2.deserialize(k, v);\n\n        assert_eq!(128, *fds2.get(&key1).unwrap());\n        assert_eq!(129, *fds2.get(&key2).unwrap());\n    }\n\n    #[test]\n    fn test_vec_string_serde() {\n        init_log();\n        let vec_str: Vec<String> = vec![\"aaaa\".to_string(), \"bbb\".to_string()];\n        let mut ser_buf: [u8; 1024] = [0; 1024];\n        let size = serialize_vec_string(&vec_str, &mut ser_buf);\n        let de_vec_string = deserialize_vec_string(&ser_buf[..size]);\n        assert_eq!(de_vec_string.len(), 2);\n        assert_eq!(de_vec_string[0], \"aaaa\");\n        assert_eq!(de_vec_string[1], \"bbb\");\n    }\n\n    #[test]\n    fn test_send_receive_fds() {\n        init_log();\n        let dumb_fd = socket::socket(\n            AddressFamily::Unix,\n            SockType::Stream,\n            SockFlag::empty(),\n            None,\n        )\n        .unwrap();\n\n        // receiver need to start in another thread since it is blocking\n        let child = thread::spawn(move || {\n            let mut buf: [u8; 32] = [0; 32];\n            let (fds, bytes) = get_fds_from(\"/tmp/pingora_fds_receive.sock\", &mut buf).unwrap();\n            debug!(\"{:?}\", fds);\n            assert_eq!(1, fds.len());\n            assert_eq!(32, bytes);\n            assert_eq!(1, buf[0]);\n            assert_eq!(1, buf[31]);\n        });\n\n        let fds = vec![dumb_fd];\n        let buf: [u8; 128] = [1; 128];\n        match send_fds_to(fds, &buf, \"/tmp/pingora_fds_receive.sock\") {\n            Ok(sent) => {\n                assert!(sent > 0);\n            }\n            Err(e) => {\n                error!(\"{:?}\", e);\n                panic!()\n            }\n        }\n\n        child.join().unwrap();\n    }\n\n    #[test]\n    fn test_serde_via_socket() {\n        init_log();\n        let mut fds = Fds::new();\n        let key1 = \"1.1.1.1:80\".to_string();\n        let dumb_fd1 = socket::socket(\n            AddressFamily::Unix,\n            SockType::Stream,\n            SockFlag::empty(),\n            None,\n        )\n        .unwrap();\n        fds.add(key1.clone(), dumb_fd1);\n        let key2 = \"1.1.1.1:443\".to_string();\n        let dumb_fd2 = socket::socket(\n            AddressFamily::Unix,\n            SockType::Stream,\n            SockFlag::empty(),\n            None,\n        )\n        .unwrap();\n        fds.add(key2.clone(), dumb_fd2);\n\n        let child = thread::spawn(move || {\n            let mut fds2 = Fds::new();\n            fds2.get_from_sock(\"/tmp/pingora_fds_receive2.sock\")\n                .unwrap();\n            assert!(*fds2.get(&key1).unwrap() > 0);\n            assert!(*fds2.get(&key2).unwrap() > 0);\n        });\n\n        fds.send_to_sock(\"/tmp/pingora_fds_receive2.sock\").unwrap();\n        child.join().unwrap();\n    }\n}\n\n// Path: pingora-core/src/services/background.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! The background service\n//!\n//! A [BackgroundService] can be run as part of a Pingora application to add supporting logic that\n//! exists outside of the request/response lifecycle.\n//! Examples might include service discovery (load balancing) and background updates such as\n//! push-style metrics.\n\nuse async_trait::async_trait;\nuse std::sync::Arc;\n\nuse super::Service;\nuse crate::server::{ListenFds, ShutdownWatch};\n\n/// The background service interface\n#[cfg_attr(not(doc_async_trait), async_trait)]\npub trait BackgroundService {\n    /// This function is called when the pingora server tries to start all the\n    /// services. The background service can return at anytime or wait for the\n    /// `shutdown` signal.\n    async fn start(&self, mut shutdown: ShutdownWatch);\n}\n\n/// A generic type of background service\npub struct GenBackgroundService<A> {\n    // Name of the service\n    name: String,\n    // Task the service will execute\n    task: Arc<A>,\n    /// The number of threads. Default is 1\n    pub threads: Option<usize>,\n}\n\nimpl<A> GenBackgroundService<A> {\n    /// Generates a background service that can run in the pingora runtime\n    pub fn new(name: String, task: Arc<A>) -> Self {\n        Self {\n            name,\n            task,\n            threads: Some(1),\n        }\n    }\n\n    /// Return the task behind [Arc] to be shared other logic.\n    pub fn task(&self) -> Arc<A> {\n        self.task.clone()\n    }\n}\n\n#[async_trait]\nimpl<A> Service for GenBackgroundService<A>\nwhere\n    A: BackgroundService + Send + Sync + 'static,\n{\n    async fn start_service(&mut self, _fds: Option<ListenFds>, shutdown: ShutdownWatch) {\n        self.task.start(shutdown).await;\n    }\n\n    fn name(&self) -> &str {\n        &self.name\n    }\n\n    fn threads(&self) -> Option<usize> {\n        self.threads\n    }\n}\n\n// Helper function to create a background service with a human readable name\npub fn background_service<SV>(name: &str, task: SV) -> GenBackgroundService<SV> {\n    GenBackgroundService::new(format!(\"BG {name}\"), Arc::new(task))\n}\n\n// Path: pingora-core/src/protocols/http/error_resp.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Error response generating utilities.\n\nuse http::header;\nuse once_cell::sync::Lazy;\nuse pingora_http::ResponseHeader;\n\nuse super::SERVER_NAME;\n\n/// Generate an error response with the given status code.\n///\n/// This error response has a zero `Content-Length` and `Cache-Control: private, no-store`.\npub fn gen_error_response(code: u16) -> ResponseHeader {\n    let mut resp = ResponseHeader::build(code, Some(4)).unwrap();\n    resp.insert_header(header::SERVER, &SERVER_NAME[..])\n        .unwrap();\n    resp.insert_header(header::DATE, \"Sun, 06 Nov 1994 08:49:37 GMT\")\n        .unwrap(); // placeholder\n    resp.insert_header(header::CONTENT_LENGTH, \"0\").unwrap();\n    resp.insert_header(header::CACHE_CONTROL, \"private, no-store\")\n        .unwrap();\n    resp\n}\n\n/// Pre-generated 502 response\npub static HTTP_502_RESPONSE: Lazy<ResponseHeader> = Lazy::new(|| gen_error_response(502));\n/// Pre-generated 400 response\npub static HTTP_400_RESPONSE: Lazy<ResponseHeader> = Lazy::new(|| gen_error_response(400));\n\n// Path: pingora-core/src/protocols/l4/socket.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Generic socket type\n\nuse crate::{Error, OrErr};\nuse nix::sys::socket::{getpeername, getsockname, SockaddrStorage};\nuse std::cmp::Ordering;\nuse std::hash::{Hash, Hasher};\nuse std::net::SocketAddr as StdSockAddr;\nuse std::os::unix::net::SocketAddr as StdUnixSockAddr;\nuse tokio::net::unix::SocketAddr as TokioUnixSockAddr;\n\n/// [`SocketAddr`] is a storage type that contains either a Internet (IP address)\n/// socket address or a Unix domain socket address.\n#[derive(Debug, Clone)]\npub enum SocketAddr {\n    Inet(StdSockAddr),\n    Unix(StdUnixSockAddr),\n}\n\nimpl SocketAddr {\n    /// Get a reference to the IP socket if it is one\n    pub fn as_inet(&self) -> Option<&StdSockAddr> {\n        if let SocketAddr::Inet(addr) = self {\n            Some(addr)\n        } else {\n            None\n        }\n    }\n\n    /// Get a reference to the Unix domain socket if it is one\n    pub fn as_unix(&self) -> Option<&StdUnixSockAddr> {\n        if let SocketAddr::Unix(addr) = self {\n            Some(addr)\n        } else {\n            None\n        }\n    }\n\n    /// Set the port if the address is an IP socket.\n    pub fn set_port(&mut self, port: u16) {\n        if let SocketAddr::Inet(addr) = self {\n            addr.set_port(port)\n        }\n    }\n\n    fn from_sockaddr_storage(sock: &SockaddrStorage) -> Option<SocketAddr> {\n        if let Some(v4) = sock.as_sockaddr_in() {\n            return Some(SocketAddr::Inet(StdSockAddr::V4(\n                std::net::SocketAddrV4::new(v4.ip().into(), v4.port()),\n            )));\n        } else if let Some(v6) = sock.as_sockaddr_in6() {\n            return Some(SocketAddr::Inet(StdSockAddr::V6(\n                std::net::SocketAddrV6::new(v6.ip(), v6.port(), v6.flowinfo(), v6.scope_id()),\n            )));\n        }\n\n        // TODO: don't set abstract / unnamed for now,\n        // for parity with how we treat these types in TryFrom<TokioUnixSockAddr>\n        Some(SocketAddr::Unix(\n            sock.as_unix_addr()\n                .map(|addr| addr.path().map(StdUnixSockAddr::from_pathname))??\n                .ok()?,\n        ))\n    }\n\n    pub fn from_raw_fd(fd: std::os::unix::io::RawFd, peer_addr: bool) -> Option<SocketAddr> {\n        let sockaddr_storage = if peer_addr {\n            getpeername(fd)\n        } else {\n            getsockname(fd)\n        };\n        match sockaddr_storage {\n            Ok(sockaddr) => Self::from_sockaddr_storage(&sockaddr),\n            // could be errors such as EBADF, i.e. fd is no longer a valid socket\n            // fail open in this case\n            Err(_e) => None,\n        }\n    }\n}\n\nimpl std::fmt::Display for SocketAddr {\n    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {\n        match self {\n            SocketAddr::Inet(addr) => write!(f, \"{addr}\"),\n            SocketAddr::Unix(addr) => {\n                if let Some(path) = addr.as_pathname() {\n                    write!(f, \"{}\", path.display())\n                } else {\n                    write!(f, \"{addr:?}\")\n                }\n            }\n        }\n    }\n}\n\nimpl Hash for SocketAddr {\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        match self {\n            Self::Inet(sockaddr) => sockaddr.hash(state),\n            Self::Unix(sockaddr) => {\n                if let Some(path) = sockaddr.as_pathname() {\n                    // use the underlying path as the hash\n                    path.hash(state);\n                } else {\n                    // unnamed or abstract UDS\n                    // abstract UDS name not yet exposed by std API\n                    // panic for now, we can decide on the right way to hash them later\n                    panic!(\"Unnamed and abstract UDS types not yet supported for hashing\")\n                }\n            }\n        }\n    }\n}\n\nimpl PartialEq for SocketAddr {\n    fn eq(&self, other: &Self) -> bool {\n        match self {\n            Self::Inet(addr) => Some(addr) == other.as_inet(),\n            Self::Unix(addr) => {\n                let path = addr.as_pathname();\n                // can only compare UDS with path, assume false on all unnamed UDS\n                path.is_some() && path == other.as_unix().and_then(|addr| addr.as_pathname())\n            }\n        }\n    }\n}\n\nimpl PartialOrd for SocketAddr {\n    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {\n        Some(self.cmp(other))\n    }\n}\n\nimpl Ord for SocketAddr {\n    fn cmp(&self, other: &Self) -> Ordering {\n        match self {\n            Self::Inet(addr) => {\n                if let Some(o) = other.as_inet() {\n                    addr.cmp(o)\n                } else {\n                    // always make Inet < Unix \"smallest for variants at the top\"\n                    Ordering::Less\n                }\n            }\n            Self::Unix(addr) => {\n                if let Some(o) = other.as_unix() {\n                    // NOTE: unnamed UDS are consider the same\n                    addr.as_pathname().cmp(&o.as_pathname())\n                } else {\n                    // always make Inet < Unix \"smallest for variants at the top\"\n                    Ordering::Greater\n                }\n            }\n        }\n    }\n}\n\nimpl Eq for SocketAddr {}\n\nimpl std::str::FromStr for SocketAddr {\n    type Err = Box<Error>;\n\n    // This is very basic parsing logic, it might treat invalid IP:PORT str as UDS path\n    // TODO: require UDS to have some prefix\n    fn from_str(s: &str) -> Result<Self, Self::Err> {\n        match StdSockAddr::from_str(s) {\n            Ok(addr) => Ok(SocketAddr::Inet(addr)),\n            Err(_) => {\n                let uds_socket = StdUnixSockAddr::from_pathname(s)\n                    .or_err(crate::BindError, \"invalid UDS path\")?;\n                Ok(SocketAddr::Unix(uds_socket))\n            }\n        }\n    }\n}\n\nimpl std::net::ToSocketAddrs for SocketAddr {\n    type Iter = std::iter::Once<StdSockAddr>;\n\n    // Error if UDS addr\n    fn to_socket_addrs(&self) -> std::io::Result<Self::Iter> {\n        if let Some(inet) = self.as_inet() {\n            Ok(std::iter::once(*inet))\n        } else {\n            Err(std::io::Error::new(\n                std::io::ErrorKind::Other,\n                \"UDS socket cannot be used as inet socket\",\n            ))\n        }\n    }\n}\n\nimpl From<StdSockAddr> for SocketAddr {\n    fn from(sockaddr: StdSockAddr) -> Self {\n        SocketAddr::Inet(sockaddr)\n    }\n}\n\nimpl From<StdUnixSockAddr> for SocketAddr {\n    fn from(sockaddr: StdUnixSockAddr) -> Self {\n        SocketAddr::Unix(sockaddr)\n    }\n}\n\n// TODO: ideally mio/tokio will start using the std version of the unix `SocketAddr`\n// so we can avoid a fallible conversion\n// https://github.com/tokio-rs/mio/issues/1527\nimpl TryFrom<TokioUnixSockAddr> for SocketAddr {\n    type Error = String;\n\n    fn try_from(value: TokioUnixSockAddr) -> Result<Self, Self::Error> {\n        if let Some(Ok(addr)) = value.as_pathname().map(StdUnixSockAddr::from_pathname) {\n            Ok(addr.into())\n        } else {\n            // may be unnamed/abstract UDS\n            Err(format!(\"could not convert {value:?} to SocketAddr\"))\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n\n    #[test]\n    fn parse_ip() {\n        let ip: SocketAddr = \"127.0.0.1:80\".parse().unwrap();\n        assert!(ip.as_inet().is_some());\n    }\n\n    #[test]\n    fn parse_uds() {\n        let uds: SocketAddr = \"/tmp/my.sock\".parse().unwrap();\n        assert!(uds.as_unix().is_some());\n    }\n}\n\n// Path: pingora-core/src/utils/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! This module contains various types that make it easier to work with bytes and X509\n//! certificates.\n\n// TODO: move below to its own mod\nuse crate::tls::{nid::Nid, pkey::PKey, pkey::Private, x509::X509};\nuse crate::Result;\nuse bytes::Bytes;\nuse pingora_error::{ErrorType::*, OrErr};\nuse std::hash::{Hash, Hasher};\n\n/// A `BufRef` is a reference to a buffer of bytes. It removes the need for self-referential data\n/// structures. It is safe to use as long as the underlying buffer does not get mutated.\n///\n/// # Panics\n///\n/// This will panic if an index is out of bounds.\n#[derive(Clone, PartialEq, Eq, Debug)]\npub struct BufRef(pub usize, pub usize);\n\nimpl BufRef {\n    /// Return a sub-slice of `buf`.\n    pub fn get<'a>(&self, buf: &'a [u8]) -> &'a [u8] {\n        &buf[self.0..self.1]\n    }\n\n    /// Return a slice of `buf`. This operation is O(1) and increases the reference count of `buf`.\n    \npub fn get_bytes(&self, buf: &Bytes) -> Bytes {\n        buf.slice(self.0..self.1)\n    }\n\n    /// Return the size of the slice reference.\n    pub fn len(&self) -> usize {\n        self.1 - self.0\n    }\n\n    /// Return true if the length is zero.\n    pub fn is_empty(&self) -> bool {\n        self.1 == self.0\n    }\n}\n\nimpl BufRef {\n    /// Initialize a `BufRef` that can reference a slice beginning at index `start` and has a\n    /// length of `len`.\n    pub fn new(start: usize, len: usize) -> Self {\n        BufRef(start, start + len)\n    }\n}\n\n/// A `KVRef` contains a key name and value pair, stored as two [BufRef] types.\n#[derive(Clone)]\npub struct KVRef {\n    name: BufRef,\n    value: BufRef,\n}\n\nimpl KVRef {\n    /// Like [BufRef::get] for the name.\n    pub fn get_name<'a>(&self, buf: &'a [u8]) -> &'a [u8] {\n        self.name.get(buf)\n    }\n\n    /// Like [BufRef::get] for the value.\n    pub fn get_value<'a>(&self, buf: &'a [u8]) -> &'a [u8] {\n        self.value.get(buf)\n    }\n\n    /// Like [BufRef::get_bytes] for the name.\n    pub fn get_name_bytes(&self, buf: &Bytes) -> Bytes {\n        self.name.get_bytes(buf)\n    }\n\n    /// Like [BufRef::get_bytes] for the value.\n    pub fn get_value_bytes(&self, buf: &Bytes) -> Bytes {\n        self.value.get_bytes(buf)\n    }\n\n    /// Return a new `KVRef` with name and value start indices and lengths.\n    pub fn new(name_s: usize, name_len: usize, value_s: usize, value_len: usize) -> Self {\n        KVRef {\n            name: BufRef(name_s, name_s + name_len),\n            value: BufRef(value_s, value_s + value_len),\n        }\n    }\n\n    /// Return a reference to the value.\n    pub fn value(&self) -> &BufRef {\n        &self.value\n    }\n}\n\n/// A [KVRef] which contains empty sub-slices.\npub const EMPTY_KV_REF: KVRef = KVRef {\n    name: BufRef(0, 0),\n    value: BufRef(0, 0),\n};\n\nfn get_subject_name(cert: &X509, name_type: Nid) -> Option<String> {\n    cert.subject_name()\n        .entries_by_nid(name_type)\n        .next()\n        .map(|name| {\n            name.data()\n                .as_utf8()\n                .map(|s| s.to_string())\n                .unwrap_or_default()\n        })\n}\n\n/// Return the organization associated with the X509 certificate.\npub fn get_organization(cert: &X509) -> Option<String> {\n    get_subject_name(cert, Nid::ORGANIZATIONNAME)\n}\n\n/// Return the common name associated with the X509 certificate.\npub fn get_common_name(cert: &X509) -> Option<String> {\n    get_subject_name(cert, Nid::COMMONNAME)\n}\n\n/// Return the common name associated with the X509 certificate.\npub fn get_organization_unit(cert: &X509) -> Option<String> {\n    get_subject_name(cert, Nid::ORGANIZATIONALUNITNAME)\n}\n\n/// Return the serial number associated with the X509 certificate as a hexadecimal value.\npub fn get_serial(cert: &X509) -> Result<String> {\n    let bn = cert\n        .serial_number()\n        .to_bn()\n        .or_err(InvalidCert, \"Invalid serial\")?;\n    let hex = bn.to_hex_str().or_err(InvalidCert, \"Invalid serial\")?;\n\n    let hex_str: &str = hex.as_ref();\n    Ok(hex_str.to_owned())\n}\n\n/// This type contains a list of one or more certificates and an associated private key. The leaf\n/// certificate should always be first.\n#[derive(Clone)]\npub struct CertKey {\n    certificates: Vec<X509>,\n    key: PKey<Private>,\n}\n\nimpl CertKey {\n    /// Create a new `CertKey` given a list of certificates and a private key.\n    pub fn new(certificates: Vec<X509>, key: PKey<Private>) -> CertKey {\n        assert!(\n            !certificates.is_empty(),\n            \"expected a non-empty vector of certificates in CertKey::new\"\n        );\n\n        CertKey { certificates, key }\n    }\n\n    /// Peek at the leaf certificate.\n    pub fn leaf(&self) -> &X509 {\n        // This is safe due to the assertion above.\n        &self.certificates[0]\n    }\n\n    /// Return the key.\n    pub fn key(&self) -> &PKey<Private> {\n        &self.key\n    }\n\n    /// Return a slice of intermediate certificates. An empty slice means there are none.\n    pub fn intermediates(&self) -> &[X509] {\n        if self.certificates.len() <= 1 {\n            return &[];\n        }\n        &self.certificates[1..]\n    }\n\n    /// Return the organization from the leaf certificate.\n    pub fn organization(&self) -> Option<String> {\n        get_organization(self.leaf())\n    }\n\n    /// Return the serial from the leaf certificate.\n    pub fn serial(&self) -> Result<String> {\n        get_serial(self.leaf())\n    }\n}\n\nimpl Hash for CertKey {\n    fn hash<H: Hasher>(&self, state: &mut H) {\n        for certificate in &self.certificates {\n            if let Ok(serial) = get_serial(certificate) {\n                serial.hash(state)\n            }\n        }\n    }\n}\n\n// hide private key\nimpl std::fmt::Debug for CertKey {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        f.debug_struct(\"CertKey\")\n            .field(\"X509\", &self.leaf())\n            .finish()\n    }\n}\n\nimpl std::fmt::Display for CertKey {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        let leaf = self.leaf();\n        if let Some(cn) = get_common_name(leaf) {\n            // Write CN if it exists\n            write!(f, \"CN: {cn},\")?;\n        } else if let Some(org_unit) = get_organization_unit(leaf) {\n            // CA cert might not have CN, so print its unit name instead\n            write!(f, \"Org Unit: {org_unit},\")?;\n        }\n        write!(f, \", expire: {}\", leaf.not_after())\n        // ignore the details of the private key\n    }\n}\n\n// Path: pingora-core/src/protocols/ssl/digest.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! TLS information from the TLS connection\n\nuse crate::tls::{hash::MessageDigest, ssl::SslRef};\nuse crate::utils;\n\n/// The TLS connection information\n#[derive(Clone, Debug)]\npub struct SslDigest {\n    /// The cipher used\n    pub cipher: &'static str,\n    /// The TLS version of this connection\n    pub version: &'static str,\n    /// The organization of the peer's certificate\n    pub organization: Option<String>,\n    /// The serial number of the peer's certificate\n    pub serial_number: Option<String>,\n    /// The digest of the peer's certificate\n    pub cert_digest: Vec<u8>,\n}\n\nimpl SslDigest {\n    pub fn from_ssl(ssl: &SslRef) -> Self {\n        let cipher = match ssl.current_cipher() {\n            Some(c) => c.name(),\n            None => \"\",\n        };\n\n        let (cert_digest, org, sn) = match ssl.peer_certificate() {\n            Some(cert) => {\n                let cert_digest = match cert.digest(MessageDigest::sha256()) {\n                    Ok(c) => c.as_ref().to_vec(),\n                    Err(_) => Vec::new(),\n                };\n                (\n                    cert_digest,\n                    utils::get_organization(&cert),\n                    utils::get_serial(&cert).ok(),\n                )\n            }\n            None => (Vec::new(), None, None),\n        };\n\n        SslDigest {\n            cipher,\n            version: ssl.version_str(),\n            organization: org,\n            serial_number: sn,\n            cert_digest,\n        }\n    }\n}\n\n// Path: pingora-core/src/protocols/raw_connect.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! CONNECT protocol over http 1.1 via raw Unix domain socket\n//!\n//! This mod implements the most rudimentary CONNECT client over raw stream.\n//! The idea is to yield raw stream once the CONNECT handshake is complete\n//! so that the protocol encapsulated can use the stream directly.\n//! This idea only works for CONNECT over HTTP 1.1 and localhost (or where the server is close by).\n\nuse super::http::v1::client::HttpSession;\nuse super::http::v1::common::*;\nuse super::Stream;\n\nuse bytes::{BufMut, BytesMut};\nuse http::request::Parts as ReqHeader;\nuse http::Version;\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_http::ResponseHeader;\nuse tokio::io::AsyncWriteExt;\n\n/// Try to establish a CONNECT proxy via the given `stream`.\n///\n/// `request_header` should include the necessary request headers for the CONNECT protocol.\n///\n/// When successful, a [`Stream`] will be returned which is the established CONNECT proxy connection.\npub async fn connect(stream: Stream, request_header: &ReqHeader) -> Result<(Stream, ProxyDigest)> {\n    let mut http = HttpSession::new(stream);\n\n    // We write to stream directly because HttpSession doesn't write req header in auth form\n    let to_wire = http_req_header_to_wire_auth_form(request_header);\n    http.underlying_stream\n        .write_all(to_wire.as_ref())\n        .await\n        .or_err(WriteError, \"while writing request headers\")?;\n    http.underlying_stream\n        .flush()\n        .await\n        .or_err(WriteError, \"while flushing request headers\")?;\n\n    // TODO: set http.read_timeout\n    let resp_header = http.read_resp_header_parts().await?;\n    Ok((\n        http.underlying_stream,\n        validate_connect_response(resp_header)?,\n    ))\n}\n\n/// Generate the CONNECT header for the given destination\npub fn generate_connect_header<'a, H, S>(\n    host: &str,\n    port: u16,\n    headers: H,\n) -> Result<Box<ReqHeader>>\nwhere\n    S: AsRef<[u8]>,\n    H: Iterator<Item = (S, &'a Vec<u8>)>,\n{\n    // TODO: valid that host doesn't have port\n    // TODO: support adding ad-hoc headers\n\n    let authority = format!(\"{host}:{port}\");\n    let req = http::request::Builder::new()\n        .version(http::Version::HTTP_11)\n        .method(http::method::Method::CONNECT)\n        .uri(format!(\"https://{authority}/\")) // scheme doesn't matter\n        .header(http::header::HOST, &authority);\n\n    let (mut req, _) = match req.body(()) {\n        Ok(r) => r.into_parts(),\n        Err(e) => {\n            return Err(e).or_err(InvalidHTTPHeader, \"Invalid CONNECT request\");\n        }\n    };\n\n    for (k, v) in headers {\n        let header_name = http::header::HeaderName::from_bytes(k.as_ref())\n            .or_err(InvalidHTTPHeader, \"Invalid CONNECT request\")?;\n        let header_value = http::header::HeaderValue::from_bytes(v.as_slice())\n            .or_err(InvalidHTTPHeader, \"Invalid CONNECT request\")?;\n        req.headers.insert(header_name, header_value);\n    }\n\n    Ok(Box::new(req))\n}\n\n/// The information about the CONNECT proxy.\n#[derive(Debug)]\npub struct ProxyDigest {\n    /// The response header the proxy returns\n    pub response: Box<ResponseHeader>,\n}\n\nimpl ProxyDigest {\n    pub fn new(response: Box<ResponseHeader>) -> Self {\n        ProxyDigest { response }\n    }\n}\n\n/// The error returned when the CONNECT proxy fails to establish.\n#[derive(Debug)]\npub struct ConnectProxyError {\n    /// The response header the proxy returns\n    pub response: Box<ResponseHeader>,\n}\n\nimpl ConnectProxyError {\n    pub fn boxed_new(response: Box<ResponseHeader>) -> Box<Self> {\n        Box::new(ConnectProxyError { response })\n    }\n}\n\nimpl std::fmt::Display for ConnectProxyError {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        const PROXY_STATUS: &str = \"proxy-status\";\n\n        let reason = self\n            .response\n            .headers\n            .get(PROXY_STATUS)\n            .and_then(|s| s.to_str().ok())\n            .unwrap_or(\"missing proxy-status header value\");\n        write!(\n            f,\n            \"Failed CONNECT Response: status {}, proxy-status {reason}\",\n            &self.response.status\n        )\n    }\n}\n\nimpl std::error::Error for ConnectProxyError {}\n\n#[inline]\nfn http_req_header_to_wire_auth_form(req: &ReqHeader) -> BytesMut {\n    let mut buf = BytesMut::with_capacity(512);\n\n    // Request-Line\n    let method = req.method.as_str().as_bytes();\n    buf.put_slice(method);\n...\n", "needle_token_start": 12168, "needle_token_end": 12202, "code_context_ntokens": 16224, "output": ["```rust\n/// A `BufRef` is a reference to a buffer of bytes. It removes the need for self-referential data\n/// structures. It is safe to use as long as the underlying buffer does not get mutated.\n///\n/// # Panics\n///\n/// This will panic if an index is out of bounds.\n#[derive(Clone, PartialEq, Eq, Debug)]\npub struct BufRef(pub usize, pub usize);\n\nimpl BufRef {\n    /// Return a sub-slice of `buf`.\n    pub fn get<'a>(&self, buf: &'a [u8]) -> &'a [u8] {\n        &buf[self.0..self.1]\n    }\n\n    /// Return a slice of `buf`. This operation is O(1) and increases the reference count of `buf`.\n    pub fn get_bytes(&self, buf: &Bytes) -> Bytes {\n        buf.slice(self.0..self.1)\n    }\n\n    /// Return the size of the slice reference.\n    pub fn len(&self) -> usize {\n        self.1 - self.0\n    }\n\n    /// Return true if the length is zero.\n    pub fn is_empty(&self) -> bool {\n        self.1 == self.0\n    }\n}\n\nimpl BufRef {\n    /// Initialize a `BufRef` that can reference a slice beginning at index `start` and has a\n    /// length of `len`.\n    pub fn new(start: usize, len: usize) -> Self {\n        BufRef(start, start + len)\n    }\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "from_server_conf", "language": "rust", "path": "pingora-core/src/connectors/mod.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate configuration options for a transport connector based on server settings, facilitating the customization of connection parameters such as SSL/TLS settings, connection pooling, and network binding preferences.\n2. **Input**: It takes a configuration object that contains various settings related to server connections.\n3. **Output**: Returns a new configuration object specifically tailored for establishing and managing network connections, including SSL/TLS parameters, connection pooling sizes, and specific network addresses for binding.\n4. **Procedure**: The function extracts relevant settings from the input configuration object, such as SSL/TLS file paths, connection pooling sizes, and IP addresses for binding. It then checks and applies conditions to determine the appropriate settings for offloading connection tasks to thread pools. The output is a fully formed configuration object that encapsulates all these settings, ready to be used for network operations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/protocols/http/v1/client.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n...\n// Path: pingora-core/src/connectors/l4.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::debug;\nuse pingora_error::{Context, Error, ErrorType::*, OrErr, Result};\nuse rand::seq::SliceRandom;\nuse std::net::SocketAddr as InetSocketAddr;\nuse std::os::unix::io::AsRawFd;\n\nuse crate::protocols::l4::ext::{connect as tcp_connect, connect_uds, set_tcp_keepalive};\nuse crate::protocols::l4::socket::SocketAddr;\nuse crate::protocols::l4::stream::Stream;\nuse crate::protocols::{GetSocketDigest, SocketDigest};\nuse crate::upstreams::peer::Peer;\n\n/// Establish a connection (l4) to the given peer using its settings and an optional bind address.\npub async fn connect<P>(peer: &P, bind_to: Option<InetSocketAddr>) -> Result<Stream>\nwhere\n    P: Peer + Send + Sync,\n{\n    if peer.get_proxy().is_some() {\n        return proxy_connect(peer)\n            .await\n            .err_context(|| format!(\"Fail to establish CONNECT proxy: {}\", peer));\n    }\n    let peer_addr = peer.address();\n    let mut stream: Stream = match peer_addr {\n        SocketAddr::Inet(addr) => {\n            let connect_future = tcp_connect(addr, bind_to.as_ref());\n            let conn_res = match peer.connection_timeout() {\n                Some(t) => pingora_timeout::timeout(t, connect_future)\n                    .await\n                    .explain_err(ConnectTimedout, |_| {\n                        format!(\"timeout {t:?} connecting to server {peer}\")\n                    })?,\n                None => connect_future.await,\n            };\n            match conn_res {\n                Ok(socket) => {\n                    debug!(\"connected to new server: {}\", peer.address());\n                    if let Some(ka) = peer.tcp_keepalive() {\n                        debug!(\"Setting tcp keepalive\");\n                        set_tcp_keepalive(&socket, ka)?;\n                    }\n                    Ok(socket.into())\n                }\n                Err(e) => {\n                    let c = format!(\"Fail to connect to {peer}\");\n                    match e.etype() {\n                        SocketError | BindError => Error::e_because(InternalError, c, e),\n                        _ => Err(e.more_context(c)),\n                    }\n                }\n            }\n        }\n        SocketAddr::Unix(addr) => {\n            let connect_future = connect_uds(\n                addr.as_pathname()\n                    .expect(\"non-pathname unix sockets not supported as peer\"),\n            );\n            let conn_res = match peer.connection_timeout() {\n                Some(t) => pingora_timeout::timeout(t, connect_future)\n                    .await\n                    .explain_err(ConnectTimedout, |_| {\n                        format!(\"timeout {t:?} connecting to server {peer}\")\n                    })?,\n                None => connect_future.await,\n            };\n            match conn_res {\n                Ok(socket) => {\n                    debug!(\"connected to new server: {}\", peer.address());\n                    // no SO_KEEPALIVE for UDS\n                    Ok(socket.into())\n                }\n                Err(e) => {\n                    let c = format!(\"Fail to connect to {peer}\");\n                    match e.etype() {\n                        SocketError | BindError => Error::e_because(InternalError, c, e),\n                        _ => Err(e.more_context(c)),\n                    }\n                }\n            }\n        }\n    }?;\n    let tracer = peer.get_tracer();\n    if let Some(t) = tracer {\n        t.0.on_connected();\n        stream.tracer = Some(t);\n    }\n\n    stream.set_nodelay()?;\n\n    let digest = SocketDigest::from_raw_fd(stream.as_raw_fd());\n    digest\n        .peer_addr\n        .set(Some(peer_addr.clone()))\n        .expect(\"newly created OnceCell must be empty\");\n    stream.set_socket_digest(digest);\n\n    Ok(stream)\n}\n\npub(crate) fn bind_to_random<P: Peer>(\n    peer: &P,\n    v4_list: &[InetSocketAddr],\n    v6_list: &[InetSocketAddr],\n) -> Option<InetSocketAddr> {\n    let selected = peer.get_peer_options().and_then(|o| o.bind_to);\n    if selected.is_some() {\n        return selected;\n    }\n\n    fn bind_to_ips(ips: &[InetSocketAddr]) -> Option<InetSocketAddr> {\n        match ips.len() {\n            0 => None,\n            1 => Some(ips[0]),\n            _ => {\n                // pick a random bind ip\n                ips.choose(&mut rand::thread_rng()).copied()\n            }\n        }\n    }\n\n    match peer.address() {\n        SocketAddr::Inet(sockaddr) => match sockaddr {\n            InetSocketAddr::V4(_) => bind_to_ips(v4_list),\n            InetSocketAddr::V6(_) => bind_to_ips(v6_list),\n        },\n        SocketAddr::Unix(_) => None,\n    }\n}\n\nuse crate::protocols::raw_connect;\n\nasync fn proxy_connect<P: Peer>(peer: &P) -> Result<Stream> {\n    // safe to unwrap\n    let proxy = peer.get_proxy().unwrap();\n    let options = peer.get_peer_options().unwrap();\n\n    // combine required and optional headers\n    let mut headers = proxy\n        .headers\n        .iter()\n        .chain(options.extra_proxy_headers.iter());\n\n    // not likely to timeout during connect() to UDS\n    let stream: Box<Stream> = Box::new(\n        connect_uds(&proxy.next_hop)\n            .await\n            .or_err_with(ConnectError, || {\n                format!(\"CONNECT proxy connect() error to {:?}\", &proxy.next_hop)\n            })?\n            .into(),\n    );\n\n    let req_header = raw_connect::generate_connect_header(&proxy.host, proxy.port, &mut headers)?;\n    let fut = raw_connect::connect(stream, &req_header);\n    let (mut stream, digest) = match peer.connection_timeout() {\n        Some(t) => pingora_timeout::timeout(t, fut)\n            .await\n            .explain_err(ConnectTimedout, |_| \"establishing CONNECT proxy\")?,\n        None => fut.await,\n    }\n    .map_err(|mut e| {\n        // http protocol may ask to retry if reused client\n        e.retry.decide_reuse(false);\n        e\n    })?;\n    debug!(\"CONNECT proxy established: {:?}\", proxy);\n    stream.set_proxy_digest(digest);\n    let stream = stream.into_any().downcast::<Stream>().unwrap(); // safe, it is Stream from above\n    Ok(*stream)\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::upstreams::peer::{BasicPeer, HttpPeer, Proxy};\n    use std::collections::BTreeMap;\n    use std::path::PathBuf;\n    use tokio::io::AsyncWriteExt;\n    use tokio::net::UnixListener;\n\n    #[tokio::test]\n    async fn test_conn_error_refused() {\n        let peer = BasicPeer::new(\"127.0.0.1:79\"); // hopefully port 79 is not used\n        let new_session = connect(&peer, None).await;\n        assert_eq!(new_session.unwrap_err().etype(), &ConnectRefused)\n    }\n\n    // TODO broken on arm64\n    #[ignore]\n    #[tokio::test]\n    async fn test_conn_error_no_route() {\n        let peer = BasicPeer::new(\"[::3]:79\"); // no route\n        let new_session = connect(&peer, None).await;\n        assert_eq!(new_session.unwrap_err().etype(), &ConnectNoRoute)\n    }\n\n    #[tokio::test]\n    async fn test_conn_error_addr_not_avail() {\n        let peer = HttpPeer::new(\"127.0.0.1:121\".to_string(), false, \"\".to_string());\n        let new_session = connect(&peer, Some(\"192.0.2.2:0\".parse().unwrap())).await;\n        assert_eq!(new_session.unwrap_err().etype(), &InternalError)\n    }\n\n    #[tokio::test]\n    async fn test_conn_error_other() {\n        let peer = HttpPeer::new(\"240.0.0.1:80\".to_string(), false, \"\".to_string()); // non localhost\n\n        // create an error: cannot send from src addr: localhost to dst addr: a public IP\n        let new_session = connect(&peer, Some(\"127.0.0.1:0\".parse().unwrap())).await;\n        let error = new_session.unwrap_err();\n        // XXX: some system will allow the socket to bind and connect without error, only to timeout\n        assert!(error.etype() == &ConnectError || error.etype() == &ConnectTimedout)\n    }\n\n    #[tokio::test]\n    async fn test_conn_timeout() {\n        // 192.0.2.1 is effectively a blackhole\n        let mut peer = BasicPeer::new(\"192.0.2.1:79\");\n        peer.options.connection_timeout = Some(std::time::Duration::from_millis(1)); //1ms\n        let new_session = connect(&peer, None).await;\n        assert_eq!(new_session.unwrap_err().etype(), &ConnectTimedout)\n    }\n\n    #[tokio::test]\n    async fn test_connect_proxy_fail() {\n        let mut peer = HttpPeer::new(\"1.1.1.1:80\".to_string(), false, \"\".to_string());\n        let mut path = PathBuf::new();\n        path.push(\"/tmp/123\");\n        peer.proxy = Some(Proxy {\n            next_hop: path.into(),\n            host: \"1.1.1.1\".into(),\n            port: 80,\n            headers: BTreeMap::new(),\n        });\n        let new_session = connect(&peer, None).await;\n        let e = new_session.unwrap_err();\n        assert_eq!(e.etype(), &ConnectError);\n        assert!(!e.retry());\n    }\n\n    const MOCK_UDS_PATH: &str = \"/tmp/test_unix_connect_proxy.sock\";\n\n    // one-off mock server\n    async fn mock_connect_server() {\n        let _ = std::fs::remove_file(MOCK_UDS_PATH);\n        let listener = UnixListener::bind(MOCK_UDS_PATH).unwrap();\n        if let Ok((mut stream, _addr)) = listener.accept().await {\n            stream.write_all(b\"HTTP/1.1 200 OK\\r\\n\\r\\n\").await.unwrap();\n            // wait a bit so that the client can read\n            tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        }\n        let _ = std::fs::remove_file(MOCK_UDS_PATH);\n    }\n\n    #[tokio::test(flavor = \"multi_thread\")]\n    async fn test_connect_proxy_work() {\n        tokio::spawn(async {\n            mock_connect_server().await;\n        });\n        // wait for the server to start\n        tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        let mut peer = HttpPeer::new(\"1.1.1.1:80\".to_string(), false, \"\".to_string());\n        let mut path = PathBuf::new();\n        path.push(MOCK_UDS_PATH);\n        peer.proxy = Some(Proxy {\n            next_hop: path.into(),\n            host: \"1.1.1.1\".into(),\n            port: 80,\n            headers: BTreeMap::new(),\n        });\n        let new_session = connect(&peer, None).await;\n        assert!(new_session.is_ok());\n    }\n\n    const MOCK_BAD_UDS_PATH: &str = \"/tmp/test_unix_bad_connect_proxy.sock\";\n\n    // one-off mock bad proxy\n    // closes connection upon accepting\n    async fn mock_connect_bad_server() {\n        let _ = std::fs::remove_file(MOCK_BAD_UDS_PATH);\n        let listener = UnixListener::bind(MOCK_BAD_UDS_PATH).unwrap();\n        if let Ok((mut stream, _addr)) = listener.accept().await {\n            stream.shutdown().await.unwrap();\n            tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        }\n        let _ = std::fs::remove_file(MOCK_BAD_UDS_PATH);\n    }\n\n    #[tokio::test(flavor = \"multi_thread\")]\n    async fn test_connect_proxy_conn_closed() {\n        tokio::spawn(async {\n            mock_connect_bad_server().await;\n        });\n        // wait for the server to start\n        tokio::time::sleep(std::time::Duration::from_millis(100)).await;\n        let mut peer = HttpPeer::new(\"1.1.1.1:80\".to_string(), false, \"\".to_string());\n        let mut path = PathBuf::new();\n        path.push(MOCK_BAD_UDS_PATH);\n        peer.proxy = Some(Proxy {\n            next_hop: path.into(),\n            host: \"1.1.1.1\".into(),\n            port: 80,\n            headers: BTreeMap::new(),\n        });\n        let new_session = connect(&peer, None).await;\n        let err = new_session.unwrap_err();\n        assert_eq!(err.etype(), &ConnectionClosed);\n        assert!(!err.retry());\n    }\n}\n\n// Path: pingora-core/src/connectors/offload.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::debug;\nuse once_cell::sync::OnceCell;\nuse rand::Rng;\nuse tokio::runtime::{Builder, Handle};\nuse tokio::sync::oneshot::{channel, Sender};\n\n// TODO: use pingora_runtime\n// a shared runtime (thread pools)\npub(crate) struct OffloadRuntime {\n    shards: usize,\n    thread_per_shard: usize,\n    // Lazily init the runtimes so that they are created after pingora\n    // daemonize itself. Otherwise the runtime threads are lost.\n    pools: OnceCell<Box<[(Handle, Sender<()>)]>>,\n}\n\nimpl OffloadRuntime {\n    pub fn new(shards: usize, thread_per_shard: usize) -> Self {\n        assert!(shards != 0);\n        assert!(thread_per_shard != 0);\n        OffloadRuntime {\n            shards,\n            thread_per_shard,\n            pools: OnceCell::new(),\n        }\n    }\n\n    fn init_pools(&self) -> Box<[(Handle, Sender<()>)]> {\n        let threads = self.shards * self.thread_per_shard;\n        let mut pools = Vec::with_capacity(threads);\n        for _ in 0..threads {\n            // We use single thread runtimes to reduce the scheduling overhead of multithread\n            // tokio runtime, which can be 50% of the on CPU time of the runtimes\n            let rt = Builder::new_current_thread().enable_all().build().unwrap();\n            let handler = rt.handle().clone();\n            let (tx, rx) = channel::<()>();\n            std::thread::Builder::new()\n                .name(\"Offload thread\".to_string())\n                .spawn(move || {\n                    debug!(\"Offload thread started\");\n                    // the thread that calls block_on() will drive the runtime\n                    // rx will return when tx is dropped so this runtime and thread will exit\n                    rt.block_on(rx)\n                })\n                .unwrap();\n            pools.push((handler, tx));\n        }\n\n        pools.into_boxed_slice()\n    }\n\n    pub fn get_runtime(&self, hash: u64) -> &Handle {\n        let mut rng = rand::thread_rng();\n\n        // choose a shard based on hash and a random thread with in that shard\n        // e.g. say thread_per_shard=2, shard 1 thread 1 is 1 * 2 + 1 = 3\n        // [[th0, th1], [th2, th3], ...]\n        let shard = hash as usize % self.shards;\n        let thread_in_shard = rng.gen_range(0..self.thread_per_shard);\n        let pools = self.pools.get_or_init(|| self.init_pools());\n        &pools[shard * self.thread_per_shard + thread_in_shard].0\n    }\n}\n\n// Path: pingora-core/src/connectors/http/v1.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse crate::connectors::{ConnectorOptions, TransportConnector};\nuse crate::protocols::http::v1::client::HttpSession;\nuse crate::upstreams::peer::Peer;\n\nuse pingora_error::Result;\nuse std::time::Duration;\n\npub struct Connector {\n    transport: TransportConnector,\n}\n\nimpl Connector {\n    pub fn new(options: Option<ConnectorOptions>) -> Self {\n        Connector {\n            transport: TransportConnector::new(options),\n        }\n    }\n\n    pub async fn get_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<(HttpSession, bool)> {\n        let (stream, reused) = self.transport.get_stream(peer).await?;\n        let http = HttpSession::new(stream);\n        Ok((http, reused))\n    }\n\n    pub async fn reused_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Option<HttpSession> {\n        self.transport\n            .reused_stream(peer)\n            .await\n            .map(HttpSession::new)\n    }\n\n    pub async fn release_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        session: HttpSession,\n        peer: &P,\n        idle_timeout: Option<Duration>,\n    ) {\n        if let Some(stream) = session.reuse().await {\n            self.transport\n                .release_stream(stream, peer.reuse_hash(), idle_timeout);\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::protocols::l4::socket::SocketAddr;\n    use crate::upstreams::peer::HttpPeer;\n    use pingora_http::RequestHeader;\n\n    async fn get_http(http: &mut HttpSession, expected_status: u16) {\n        let mut req = Box::new(RequestHeader::build(\"GET\", b\"/\", None).unwrap());\n        req.append_header(\"Host\", \"one.one.one.one\").unwrap();\n        http.write_request_header(req).await.unwrap();\n        http.read_response().await.unwrap();\n        http.respect_keepalive();\n\n        assert_eq!(http.get_status().unwrap(), expected_status);\n        while http.read_body_bytes().await.unwrap().is_some() {}\n    }\n\n    #[tokio::test]\n    async fn test_connect() {\n        let connector = Connector::new(None);\n        let peer = HttpPeer::new((\"1.1.1.1\", 80), false, \"\".into());\n        // make a new connection to 1.1.1.1\n        let (http, reused) = connector.get_http_session(&peer).await.unwrap();\n        let server_addr = http.server_addr().unwrap();\n        assert_eq!(*server_addr, \"1.1.1.1:80\".parse::<SocketAddr>().unwrap());\n        assert!(!reused);\n\n        // this http is not even used, so not be able to reuse\n        connector.release_http_session(http, &peer, None).await;\n        let (mut http, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n\n        get_http(&mut http, 301).await;\n        connector.release_http_session(http, &peer, None).await;\n        let (_, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(reused);\n    }\n\n    #[tokio::test]\n    async fn test_connect_tls() {\n        let connector = Connector::new(None);\n        let peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        // make a new connection to https://1.1.1.1\n        let (http, reused) = connector.get_http_session(&peer).await.unwrap();\n        let server_addr = http.server_addr().unwrap();\n        assert_eq!(*server_addr, \"1.1.1.1:443\".parse::<SocketAddr>().unwrap());\n        assert!(!reused);\n\n        // this http is not even used, so not be able to reuse\n        connector.release_http_session(http, &peer, None).await;\n        let (mut http, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n\n        get_http(&mut http, 200).await;\n        connector.release_http_session(http, &peer, None).await;\n        let (_, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(reused);\n    }\n}\n\n// Path: pingora-core/src/connectors/http/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Connecting to HTTP servers\n\nuse crate::connectors::ConnectorOptions;\nuse crate::protocols::http::client::HttpSession;\nuse crate::upstreams::peer::Peer;\nuse pingora_error::Result;\nuse std::time::Duration;\n\npub mod v1;\npub mod v2;\n\npub struct Connector {\n    h1: v1::Connector,\n    h2: v2::Connector,\n}\n\nimpl Connector {\n    pub fn new(options: Option<ConnectorOptions>) -> Self {\n        Connector {\n            h1: v1::Connector::new(options.clone()),\n            h2: v2::Connector::new(options),\n        }\n    }\n\n    /// Get an [HttpSession] to the given server.\n    ///\n    /// The second return value indicates whether the session is connected via a reused stream.\n    pub async fn get_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<(HttpSession, bool)> {\n        // NOTE: maybe TODO: we do not yet enforce that only TLS traffic can use h2, which is the\n        // de facto requirement for h2, because non TLS traffic lack the negotiation mechanism.\n\n        // We assume no peer option == no ALPN == h1 only\n        let h1_only = peer\n            .get_peer_options()\n            .map_or(true, |o| o.alpn.get_max_http_version() == 1);\n        if h1_only {\n            let (h1, reused) = self.h1.get_http_session(peer).await?;\n            Ok((HttpSession::H1(h1), reused))\n        } else {\n            // the peer allows h2, we first check the h2 reuse pool\n            let reused_h2 = self.h2.reused_http_session(peer).await?;\n            if let Some(h2) = reused_h2 {\n                return Ok((HttpSession::H2(h2), true));\n            }\n            let h2_only = peer\n                .get_peer_options()\n                .map_or(false, |o| o.alpn.get_min_http_version() == 2)\n                && !self.h2.h1_is_preferred(peer);\n            if !h2_only {\n                // We next check the reuse pool for h1 before creating a new h2 connection.\n                // This is because the server may not support h2 at all, connections to\n                // the server could all be h1.\n                if let Some(h1) = self.h1.reused_http_session(peer).await {\n                    return Ok((HttpSession::H1(h1), true));\n                }\n            }\n            let session = self.h2.new_http_session(peer).await?;\n            Ok((session, false))\n        }\n    }\n\n    pub async fn release_http_session<P: Peer + Send + Sync + 'static>(\n        &self,\n        session: HttpSession,\n        peer: &P,\n        idle_timeout: Option<Duration>,\n    ) {\n        match session {\n            HttpSession::H1(h1) => self.h1.release_http_session(h1, peer, idle_timeout).await,\n            HttpSession::H2(h2) => self.h2.release_http_session(h2, peer, idle_timeout),\n        }\n    }\n\n    /// Tell the connector to always send h1 for ALPN for the given peer in the future.\n    pub fn prefer_h1(&self, peer: &impl Peer) {\n        self.h2.prefer_h1(peer);\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n    use crate::protocols::http::v1::client::HttpSession as Http1Session;\n    use crate::upstreams::peer::HttpPeer;\n    use pingora_http::RequestHeader;\n\n    async fn get_http(http: &mut Http1Session, expected_status: u16) {\n        let mut req = Box::new(RequestHeader::build(\"GET\", b\"/\", None).unwrap());\n        req.append_header(\"Host\", \"one.one.one.one\").unwrap();\n        http.write_request_header(req).await.unwrap();\n        http.read_response().await.unwrap();\n        http.respect_keepalive();\n\n        assert_eq!(http.get_status().unwrap(), expected_status);\n        while http.read_body_bytes().await.unwrap().is_some() {}\n    }\n\n    #[tokio::test]\n    async fn test_connect_h2() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 2);\n        let (h2, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n        match &h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => assert!(!h2_stream.ping_timedout()),\n        }\n\n        connector.release_http_session(h2, &peer, None).await;\n\n        let (h2, reused) = connector.get_http_session(&peer).await.unwrap();\n        // reused this time\n        assert!(reused);\n        match &h2 {\n            HttpSession::H1(_) => panic!(\"expect h2\"),\n            HttpSession::H2(h2_stream) => assert!(!h2_stream.ping_timedout()),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_connect_h1() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(1, 1);\n        let (mut h1, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n        match &mut h1 {\n            HttpSession::H1(http) => {\n                get_http(http, 200).await;\n            }\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n        connector.release_http_session(h1, &peer, None).await;\n\n        let (mut h1, reused) = connector.get_http_session(&peer).await.unwrap();\n        // reused this time\n        assert!(reused);\n        match &mut h1 {\n            HttpSession::H1(_) => {}\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_connect_h2_fallback_h1_reuse() {\n        // this test verify that if the server doesn't support h2, the Connector will reuse the\n        // h1 session instead.\n\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        // As it is hard to find a server that support only h1, we use the following hack to trick\n        // the connector to think the server supports only h1. We force ALPN to use h1 and then\n        // return the connection to the Connector. And then we use a Peer that allows h2\n        peer.options.set_http_version(1, 1);\n        let (mut h1, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n        match &mut h1 {\n            HttpSession::H1(http) => {\n                get_http(http, 200).await;\n            }\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n        connector.release_http_session(h1, &peer, None).await;\n\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 1);\n\n        let (mut h1, reused) = connector.get_http_session(&peer).await.unwrap();\n        // reused this time\n        assert!(reused);\n        match &mut h1 {\n            HttpSession::H1(_) => {}\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n    }\n\n    #[tokio::test]\n    async fn test_connect_prefer_h1() {\n        let connector = Connector::new(None);\n        let mut peer = HttpPeer::new((\"1.1.1.1\", 443), true, \"one.one.one.one\".into());\n        peer.options.set_http_version(2, 1);\n        connector.prefer_h1(&peer);\n\n        let (mut h1, reused) = connector.get_http_session(&peer).await.unwrap();\n        assert!(!reused);\n        match &mut h1 {\n            HttpSession::H1(http) => {\n                get_http(http, 200).await;\n            }\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n        connector.release_http_session(h1, &peer, None).await;\n\n        peer.options.set_http_version(2, 2);\n        let (mut h1, reused) = connector.get_http_session(&peer).await.unwrap();\n        // reused this time\n        assert!(reused);\n        match &mut h1 {\n            HttpSession::H1(_) => {}\n            HttpSession::H2(_) => panic!(\"expect h1\"),\n        }\n    }\n}\n\n// Path: pingora-core/src/connectors/tls.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::debug;\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse std::sync::{Arc, Once};\n\nuse super::ConnectorOptions;\nuse crate::protocols::ssl::client::handshake;\nuse crate::protocols::ssl::SslStream;\nuse crate::protocols::IO;\nuse crate::tls::ext::{\n    add_host, clear_error_stack, ssl_add_chain_cert, ssl_set_groups_list,\n    ssl_set_renegotiate_mode_freely, ssl_set_verify_cert_store, ssl_use_certificate,\n    ssl_use_private_key, ssl_use_second_key_share,\n};\n#[cfg(feature = \"boringssl\")]\nuse crate::tls::ssl::SslCurve;\nuse crate::tls::ssl::{SslConnector, SslFiletype, SslMethod, SslVerifyMode, SslVersion};\nuse crate::tls::x509::store::X509StoreBuilder;\nuse crate::upstreams::peer::{Peer, ALPN};\n\nconst CIPHER_LIST: &str = \"AES-128-GCM-SHA256\\\n    :AES-256-GCM-SHA384\\\n    :CHACHA20-POLY1305-SHA256\\\n    :ECDHE-ECDSA-AES128-GCM-SHA256\\\n    :ECDHE-ECDSA-AES256-GCM-SHA384\\\n    :ECDHE-RSA-AES128-GCM-SHA256\\\n    :ECDHE-RSA-AES256-GCM-SHA384\\\n    :ECDHE-RSA-AES128-SHA\\\n    :ECDHE-RSA-AES256-SHA384\\\n    :AES128-GCM-SHA256\\\n    :AES256-GCM-SHA384\\\n    :AES128-SHA\\\n    :AES256-SHA\\\n    :DES-CBC3-SHA\";\n\n/**\n * Enabled signature algorithms for signing/verification (ECDSA).\n * As of 4/10/2023, the only addition to boringssl's defaults is ECDSA_SECP521R1_SHA512.\n */\nconst SIGALG_LIST: &str = \"ECDSA_SECP256R1_SHA256\\\n    :RSA_PSS_RSAE_SHA256\\\n    :RSA_PKCS1_SHA256\\\n    :ECDSA_SECP384R1_SHA384\\\n    :RSA_PSS_RSAE_SHA384\\\n    :RSA_PKCS1_SHA384\\\n    :RSA_PSS_RSAE_SHA512\\\n    :RSA_PKCS1_SHA512\\\n    :RSA_PKCS1_SHA1\\\n    :ECDSA_SECP521R1_SHA512\";\n/**\n * Enabled curves for ECDHE (signature key exchange).\n * As of 4/10/2023, the only addition to boringssl's defaults is SECP521R1.\n *\n * N.B. The ordering of these curves is important. The boringssl library will select the first one\n * as a guess when negotiating a handshake with a server using TLSv1.3. We should opt for curves\n * that are both computationally cheaper and more supported.\n */\n#[cfg(feature = \"boringssl\")]\nconst BORINGSSL_CURVE_LIST: &[SslCurve] = &[\n    SslCurve::X25519,\n    SslCurve::SECP256R1,\n    SslCurve::SECP384R1,\n    SslCurve::SECP521R1,\n];\n\nstatic INIT_CA_ENV: Once = Once::new();\nfn init_ssl_cert_env_vars() {\n    // this sets env vars to pick up the root certs\n    // it is universal across openssl and boringssl\n    INIT_CA_ENV.call_once(openssl_probe::init_ssl_cert_env_vars);\n}\n\n#[derive(Clone)]\npub struct Connector {\n    pub(crate) ctx: Arc<SslConnector>, // Arc to support clone\n}\n\nimpl Connector {\n    pub fn new(options: Option<ConnectorOptions>) -> Self {\n        let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();\n        // TODO: make these conf\n        // Set supported ciphers.\n        builder.set_cipher_list(CIPHER_LIST).unwrap();\n        // Set supported signature algorithms and ECDH (key exchange) curves.\n        builder\n            .set_sigalgs_list(&SIGALG_LIST.to_lowercase())\n            .unwrap();\n        #[cfg(feature = \"boringssl\")]\n        builder.set_curves(BORINGSSL_CURVE_LIST).unwrap();\n        builder\n            .set_max_proto_version(Some(SslVersion::TLS1_3))\n            .unwrap();\n        builder\n            .set_min_proto_version(Some(SslVersion::TLS1))\n            .unwrap();\n        if let Some(conf) = options.as_ref() {\n            if let Some(ca_file_path) = conf.ca_file.as_ref() {\n                builder.set_ca_file(ca_file_path).unwrap();\n            } else {\n                init_ssl_cert_env_vars();\n                // load from default system wide trust location. (the name is misleading)\n                builder.set_default_verify_paths().unwrap();\n            }\n            if let Some((cert, key)) = conf.cert_key_file.as_ref() {\n                builder\n                    .set_certificate_file(cert, SslFiletype::PEM)\n                    .unwrap();\n\n                builder.set_private_key_file(key, SslFiletype::PEM).unwrap();\n            }\n        } else {\n            init_ssl_cert_env_vars();\n            builder.set_default_verify_paths().unwrap();\n        }\n\n        Connector {\n            ctx: Arc::new(builder.build()),\n        }\n    }\n}\n\n/*\n    OpenSSL considers underscores in hostnames non-compliant.\n    We replace the underscore in the leftmost label as we must support these\n    hostnames for wildcard matches and we have not patched OpenSSL.\n\n    https://github.com/openssl/openssl/issues/12566\n\n    > The labels must follow the rules for ARPANET host names.  They must\n    > start with a letter, end with a letter or digit, and have as interior\n    > characters only letters, digits, and hyphen.  There are also some\n    > restrictions on the length.  Labels must be 63 characters or less.\n    - https://datatracker.ietf.org/doc/html/rfc1034#section-3.5\n*/\nfn replace_leftmost_underscore(sni: &str) -> Option<String> {\n    // wildcard is only leftmost label\n    let mut s = sni.splitn(2, '.');\n    if let (Some(leftmost), Some(rest)) = (s.next(), s.next()) {\n        // if not a subdomain or leftmost does not contain underscore return\n        if !rest.contains('.') || !leftmost.contains('_') {\n            return None;\n        }\n        // we have a subdomain, replace underscores\n        let leftmost = leftmost.replace('_', \"-\");\n        return Some(format!(\"{leftmost}.{rest}\"));\n    }\n    None\n}\n\npub(crate) async fn connect<T, P>(\n    stream: T,\n    peer: &P,\n    alpn_override: Option<ALPN>,\n    tls_ctx: &SslConnector,\n) -> Result<SslStream<T>>\nwhere\n    T: IO,\n    P: Peer + Send + Sync,\n{\n    let mut ssl_conf = tls_ctx.configure().unwrap();\n\n    ssl_set_renegotiate_mode_freely(&mut ssl_conf);\n\n    // Set up CA/verify cert store\n    // TODO: store X509Store in the peer directly\n    if let Some(ca_list) = peer.get_ca() {\n        let mut store_builder = X509StoreBuilder::new().unwrap();\n        for ca in &***ca_list {\n            store_builder.add_cert(ca.clone()).unwrap();\n        }\n        ssl_set_verify_cert_store(&mut ssl_conf, &store_builder.build())\n            .or_err(InternalError, \"failed to load cert store\")?;\n    }\n\n    // Set up client cert/key\n    if let Some(key_pair) = peer.get_client_cert_key() {\n        debug!(\"setting client cert and key\");\n        ssl_use_certificate(&mut ssl_conf, key_pair.leaf())\n            .or_err(InternalError, \"invalid client cert\")?;\n        ssl_use_private_key(&mut ssl_conf, key_pair.key())\n            .or_err(InternalError, \"invalid client key\")?;\n\n        let intermediates = key_pair.intermediates();\n        if !intermediates.is_empty() {\n            debug!(\"adding intermediate certificates for mTLS chain\");\n            for int in intermediates {\n                ssl_add_chain_cert(&mut ssl_conf, int)\n                    .or_err(InternalError, \"invalid intermediate client cert\")?;\n            }\n        }\n    }\n\n    if let Some(curve) = peer.get_peer_options().and_then(|o| o.curves) {\n        ssl_set_groups_list(&mut ssl_conf, curve).or_err(InternalError, \"invalid curves\")?;\n    }\n\n    // second_keyshare is default true\n    if !peer.get_peer_options().map_or(true, |o| o.second_keyshare) {\n        ssl_use_second_key_share(&mut ssl_conf, false);\n    }\n\n    // disable verification if sni does not exist\n    // XXX: verify on empty string cause null string seg fault\n    if peer.sni().is_empty() {\n        ssl_conf.set_use_server_name_indication(false);\n        /* NOTE: technically we can still verify who signs the cert but turn it off to be\n        consistent with nginx's behavior */\n        ssl_conf.set_verify(SslVerifyMode::NONE);\n    } else if peer.verify_cert() {\n        if peer.verify_hostname() {\n            let verify_param = ssl_conf.param_mut();\n            add_host(verify_param, peer.sni()).or_err(InternalError, \"failed to add host\")?;\n            // if sni had underscores in leftmost label replace and add\n            if let Some(sni_s) = replace_leftmost_underscore(peer.sni()) {\n                add_host(verify_param, sni_s.as_ref()).unwrap();\n            }\n            if let Some(alt_cn) = peer.alternative_cn() {\n                if !alt_cn.is_empty() {\n                    add_host(verify_param, alt_cn).unwrap();\n                    // if alt_cn had underscores in leftmost label replace and add\n                    if let Some(alt_cn_s) = replace_leftmost_underscore(alt_cn) {\n                        add_host(verify_param, alt_cn_s.as_ref()).unwrap();\n                    }\n                }\n            }\n        }\n        ssl_conf.set_verify(SslVerifyMode::PEER);\n    } else {\n        ssl_conf.set_verify(SslVerifyMode::NONE);\n    }\n\n    /*\n       We always set set_verify_hostname(false) here because:\n        - verify case.)  otherwise ssl.connect calls X509_VERIFY_PARAM_set1_host\n                         which overrides the names added by add_host. Verify is\n                         essentially on as long as the names are added.\n        - off case.)    the non verify hostname case should have it disabled\n    */\n    ssl_conf.set_verify_hostname(false);\n\n    if let Some(alpn) = alpn_override.as_ref().or(peer.get_alpn()) {\n        ssl_conf.set_alpn_protos(alpn.to_wire_preference()).unwrap();\n    }\n\n    clear_error_stack();\n    let connect_future = handshake(ssl_conf, peer.sni(), stream);\n\n    match peer.connection_timeout() {\n        Some(t) => match pingora_timeout::timeout(t, connect_future).await {\n            Ok(res) => res,\n            Err(_) => Error::e_explain(\n                ConnectTimedout,\n                format!(\"connecting to server {}, timeout {:?}\", peer, t),\n            ),\n        },\n        None => connect_future.await,\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn test_replace_leftmost_underscore() {\n        let none_cases = [\n            \"\",\n            \"some\",\n            \"some.com\",\n            \"1.1.1.1:5050\",\n            \"dog.dot.com\",\n            \"dog.d_t.com\",\n            \"dog.dot.c_m\",\n            \"d_g.com\",\n            \"_\",\n            \"dog.c_m\",\n        ];\n\n        for case in none_cases {\n            assert!(replace_leftmost_underscore(case).is_none(), \"{}\", case);\n        }\n\n        assert_eq!(\n            Some(\"bb-b.some.com\".to_string()),\n            replace_leftmost_underscore(\"bb_b.some.com\")\n        );\n        assert_eq!(\n            Some(\"a-a-a.some.com\".to_string()),\n            replace_leftmost_underscore(\"a_a_a.some.com\")\n        );\n        assert_eq!(\n            Some(\"-.some.com\".to_string()),\n            replace_leftmost_underscore(\"_.some.com\")\n        );\n    }\n}\n\n// Path: pingora-core/src/connectors/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Connecting to servers\n\npub mod http;\nmod l4;\nmod offload;\nmod tls;\n\nuse crate::protocols::Stream;\nuse crate::server::configuration::ServerConf;\nuse crate::tls::ssl::SslConnector;\nuse crate::upstreams::peer::{Peer, ALPN};\n\nuse l4::connect as l4_connect;\nuse log::{debug, error, warn};\nuse offload::OffloadRuntime;\nuse parking_lot::RwLock;\nuse pingora_error::{Error, ErrorType::*, OrErr, Result};\nuse pingora_pool::{ConnectionMeta, ConnectionPool};\nuse std::collections::HashMap;\nuse std::net::SocketAddr;\nuse std::sync::Arc;\nuse tokio::sync::Mutex;\n\n/// The options to configure a [TransportConnector]\n#[derive(Clone)]\npub struct ConnectorOptions {\n    /// Path to the CA file used to validate server certs.\n    ///\n    /// If `None`, the CA in the [default](https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_default_verify_paths.html)\n    /// locations will be loaded\n    pub ca_file: Option<String>,\n    /// The default client cert and key to use for mTLS\n    ///\n    /// Each individual connection can use their own cert key to override this.\n    pub cert_key_file: Option<(String, String)>,\n    /// How many connections to keepalive\n    pub keepalive_pool_size: usize,\n    /// Optionally offload the connection establishment to dedicated thread pools\n    ///\n    /// TCP and TLS connection establishment can be CPU intensive. Sometimes such tasks can slow\n    /// down the entire service, which causes timeouts which leads to more connections which\n    /// snowballs the issue. Use this option to isolate these CPU intensive tasks from impacting\n    /// other traffic.\n    ///\n    /// Syntax: (#pools, #thread in each pool)\n    pub offload_threadpool: Option<(usize, usize)>,\n    /// Bind to any of the given source IPv6 addresses\n    pub bind_to_v4: Vec<SocketAddr>,\n    /// Bind to any of the given source IPv4 addresses\n    pub bind_to_v6: Vec<SocketAddr>,\n}\n\nimpl ConnectorOptions {\n    /// Derive the [ConnectorOptions] from a [ServerConf]\n    \npub fn from_server_conf(server_conf: &ServerConf) -> Self {\n        // if both pools and threads are Some(>0)\n        let offload_threadpool = server_conf\n            .upstream_connect_offload_threadpools\n            .zip(server_conf.upstream_connect_offload_thread_per_pool)\n            .filter(|(pools, threads)| *pools > 0 && *threads > 0);\n\n        // create SocketAddrs with port 0 for src addr bind\n\n        let bind_to_v4 = server_conf\n            .client_bind_to_ipv4\n            .iter()\n            .map(|v4| {\n                let ip = v4.parse().unwrap();\n                SocketAddr::new(ip, 0)\n            })\n            .collect();\n\n        let bind_to_v6 = server_conf\n            .client_bind_to_ipv6\n            .iter()\n            .map(|v6| {\n                let ip = v6.parse().unwrap();\n                SocketAddr::new(ip, 0)\n            })\n            .collect();\n        ConnectorOptions {\n            ca_file: server_conf.ca_file.clone(),\n            cert_key_file: None, // TODO: use it\n            keepalive_pool_size: server_conf.upstream_keepalive_pool_size,\n            offload_threadpool,\n            bind_to_v4,\n            bind_to_v6,\n        }\n    }\n\n    /// Create a new [ConnectorOptions] with the given keepalive pool size\n    pub fn new(keepalive_pool_size: usize) -> Self {\n        ConnectorOptions {\n            ca_file: None,\n            cert_key_file: None,\n            keepalive_pool_size,\n            offload_threadpool: None,\n            bind_to_v4: vec![],\n            bind_to_v6: vec![],\n        }\n    }\n}\n\n/// [TransportConnector] provides APIs to connect to servers via TCP or TLS with connection reuse\npub struct TransportConnector {\n    tls_ctx: tls::Connector,\n    connection_pool: Arc<ConnectionPool<Arc<Mutex<Stream>>>>,\n    offload: Option<OffloadRuntime>,\n    bind_to_v4: Vec<SocketAddr>,\n    bind_to_v6: Vec<SocketAddr>,\n    preferred_http_version: PreferredHttpVersion,\n}\n\nconst DEFAULT_POOL_SIZE: usize = 128;\n\nimpl TransportConnector {\n    /// Create a new [TransportConnector] with the given [ConnectorOptions]\n    pub fn new(mut options: Option<ConnectorOptions>) -> Self {\n        let pool_size = options\n            .as_ref()\n            .map_or(DEFAULT_POOL_SIZE, |c| c.keepalive_pool_size);\n        // Take the offloading setting there because this layer has implement offloading,\n        // so no need for stacks at lower layer to offload again.\n        let offload = options.as_mut().and_then(|o| o.offload_threadpool.take());\n        let bind_to_v4 = options\n            .as_ref()\n            .map_or_else(Vec::new, |o| o.bind_to_v4.clone());\n        let bind_to_v6 = options\n            .as_ref()\n            .map_or_else(Vec::new, |o| o.bind_to_v6.clone());\n        TransportConnector {\n            tls_ctx: tls::Connector::new(options),\n            connection_pool: Arc::new(ConnectionPool::new(pool_size)),\n            offload: offload.map(|v| OffloadRuntime::new(v.0, v.1)),\n            bind_to_v4,\n            bind_to_v6,\n            preferred_http_version: PreferredHttpVersion::new(),\n        }\n    }\n\n    /// Connect to the given server [Peer]\n    ///\n    /// No connection is reused.\n    pub async fn new_stream<P: Peer + Send + Sync + 'static>(&self, peer: &P) -> Result<Stream> {\n        let rt = self\n            .offload\n            .as_ref()\n            .map(|o| o.get_runtime(peer.reuse_hash()));\n        let bind_to = l4::bind_to_random(peer, &self.bind_to_v4, &self.bind_to_v6);\n        let alpn_override = self.preferred_http_version.get(peer);\n        let stream = if let Some(rt) = rt {\n            let peer = peer.clone();\n            let tls_ctx = self.tls_ctx.clone();\n            rt.spawn(async move { do_connect(&peer, bind_to, alpn_override, &tls_ctx.ctx).await })\n                .await\n                .or_err(InternalError, \"offload runtime failure\")??\n        } else {\n            do_connect(peer, bind_to, alpn_override, &self.tls_ctx.ctx).await?\n        };\n\n        Ok(stream)\n    }\n\n    /// Try to find a reusable connection to the given server [Peer]\n    pub async fn reused_stream<P: Peer + Send + Sync>(&self, peer: &P) -> Option<Stream> {\n        match self.connection_pool.get(&peer.reuse_hash()) {\n            Some(s) => {\n                debug!(\"find reusable stream, trying to acquire it\");\n                {\n                    let _ = s.lock().await;\n                } // wait for the idle poll to release it\n                match Arc::try_unwrap(s) {\n                    Ok(l) => {\n                        let mut stream = l.into_inner();\n                        // test_reusable_stream: we assume server would never actively send data\n                        // first on an idle stream.\n                        if peer.matches_fd(stream.id()) && test_reusable_stream(&mut stream) {\n                            Some(stream)\n                        } else {\n                            None\n                        }\n                    }\n                    Err(_) => {\n                        error!(\"failed to acquire reusable stream\");\n                        None\n                    }\n                }\n            }\n            None => {\n                debug!(\"No reusable connection found for {peer}\");\n                None\n            }\n        }\n    }\n\n    /// Return the [Stream] to the [TransportConnector] for connection reuse.\n    ///\n    /// Not all TCP/TLS connections can be reused. It is the caller's responsibility to make sure\n    /// that protocol over the [Stream] supports connection reuse and the [Stream] itself is ready\n    /// to be reused.\n    ///\n    /// If a [Stream] is dropped instead of being returned via this function. it will be closed.\n    pub fn release_stream(\n        &self,\n        mut stream: Stream,\n        key: u64, // usually peer.reuse_hash()\n        idle_timeout: Option<std::time::Duration>,\n    ) {\n        if !test_reusable_stream(&mut stream) {\n            return;\n        }\n        let id = stream.id();\n        let meta = ConnectionMeta::new(key, id);\n        debug!(\"Try to keepalive client session\");\n        let stream = Arc::new(Mutex::new(stream));\n        let locked_stream = stream.clone().try_lock_owned().unwrap(); // safe as we just created it\n        let (notify_close, watch_use) = self.connection_pool.put(&meta, stream);\n        let pool = self.connection_pool.clone(); //clone the arc\n        let rt = pingora_runtime::current_handle();\n        rt.spawn(async move {\n            pool.idle_poll(locked_stream, &meta, idle_timeout, notify_close, watch_use)\n                .await;\n        });\n    }\n\n    /// Get a stream to the given server [Peer]\n    ///\n    /// This function will try to find a reusable [Stream] first. If there is none, a new connection\n    /// will be made to the server.\n    ///\n    /// The returned boolean will indicate whether the stream is reused.\n    pub async fn get_stream<P: Peer + Send + Sync + 'static>(\n        &self,\n        peer: &P,\n    ) -> Result<(Stream, bool)> {\n        let reused_stream = self.reused_stream(peer).await;\n        if let Some(s) = reused_stream {\n            Ok((s, true))\n        } else {\n            let s = self.new_stream(peer).await?;\n            Ok((s, false))\n        }\n    }\n\n    /// Tell the connector to always send h1 for ALPN for the given peer in the future.\n    pub fn prefer_h1(&self, peer: &impl Peer) {\n        self.preferred_http_version.add(peer, 1);\n    }\n}\n\n// Perform the actual L4 and tls connection steps while respecting the peer's\n// connection timeout if there is one\nasync fn do_connect<P: Peer + Send + Sync>(\n    peer: &P,\n    bind_to: Option<SocketAddr>,\n    alpn_override: Option<ALPN>,\n    tls_ctx: &SslConnector,\n) -> Result<Stream> {\n    // Create the future that does the connections, but don't evaluate it until\n    // we decide if we need a timeout or not\n    let connect_future = do_connect_inner(peer, bind_to, alpn_override, tls_ctx);\n\n    match peer.total_connection_timeout() {\n        Some(t) => match pingora_timeout::timeout(t, connect_future).await {\n            Ok(res) => res,\n            Err(_) => Error::e_explain(\n                ConnectTimedout,\n                format!(\"connecting to server {peer}, total-connection timeout {t:?}\"),\n            ),\n        },\n        None => connect_future.await,\n    }\n}\n\n// Perform the actual L4 and tls connection steps with no timeout\nasync fn do_connect_inner<P: Peer + Send + Sync>(\n    peer: &P,\n    bind_to: Option<SocketAddr>,\n    alpn_override: Option<ALPN>,\n    tls_ctx: &SslConnector,\n) -> Result<Stream> {\n    let stream = l4_connect(peer, bind_to).await?;\n    if peer.tls() {\n        let tls_stream = tls::connect(stream, peer, alpn_override, tls_ctx).await?;\n...\n", "needle_token_start": 13616, "needle_token_end": 13980, "code_context_ntokens": 16239, "output": ["```rust\n/// Derive the [ConnectorOptions] from a [ServerConf]\npub fn from_server_conf(server_conf: &ServerConf) -> Self {\n    // if both pools and threads are Some(>0)\n    let offload_threadpool = server_conf\n        .upstream_connect_offload_threadpools\n        .zip(server_conf.upstream_connect_offload_thread_per_pool)\n        .filter(|(pools, threads)| *pools > 0 && *threads > 0);\n\n    // create SocketAddrs with port 0 for src addr bind\n\n    let bind_to_v4 = server_conf\n        .client_bind_to_ipv4\n        .iter()\n        .map(|v4| {\n            let ip = v4.parse().unwrap();\n            SocketAddr::new(ip, 0)\n        })\n        .collect();\n\n    let bind_to_v6 = server_conf\n        .client_bind_to_ipv6\n        .iter()\n        .map(|v6| {\n            let ip = v6.parse().unwrap();\n            SocketAddr::new(ip, 0)\n        })\n        .collect();\n    ConnectorOptions {\n        ca_file: server_conf.ca_file.clone(),\n        cert_key_file: None, // TODO: use it\n        keepalive_pool_size: server_conf.upstream_keepalive_pool_size,\n        offload_threadpool,\n        bind_to_v4,\n        bind_to_v6,\n    }\n}\n```"]}
{"repo": "cloudflare/pingora", "name": "enable_h2", "language": "rust", "path": "pingora-core/src/listeners/tls.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To configure a TLS endpoint to support HTTP/2, setting it as the preferred protocol while still allowing HTTP/1.1.\n2. **Input**: None (the method modifies the instance it is called on).\n3. **Output**: None (the method updates the instance state).\n4. **Procedure**: The method updates the Application-Layer Protocol Negotiation (ALPN) settings of the TLS configuration to prioritize HTTP/2 over HTTP/1.1.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pingora-core/src/protocols/http/compression/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP response (de)compression libraries\n//!\n//! Brotli and Gzip and partially supported.\n\nuse super::HttpTask;\n\nuse bytes::Bytes;\nuse log::warn;\nuse pingora_error::{ErrorType, Result};\nuse pingora_http::{RequestHeader, ResponseHeader};\nuse std::time::Duration;\n\nmod brotli;\nmod gzip;\nmod zstd;\n\n/// The type of error to return when (de)compression fails\npub const COMPRESSION_ERROR: ErrorType = ErrorType::new(\"CompressionError\");\n\n/// The trait for both compress and decompress because the interface and syntax are the same:\n/// encode some bytes to other bytes\npub trait Encode {\n    /// Encode the input bytes. The `end` flag signals the end of the entire input. The `end` flag\n    /// helps the encoder to flush out the remaining buffered encoded data because certain compression\n    /// algorithms prefer to collect large enough data to compress all together.\n    fn encode(&mut self, input: &[u8], end: bool) -> Result<Bytes>;\n    /// Return the Encoder's name, the total input bytes, the total output bytes and the total\n    /// duration spent on encoding the data.\n    fn stat(&self) -> (&'static str, usize, usize, Duration);\n}\n\n/// The response compression object. Currently support gzip compression and brotli decompression.\n///\n/// To use it, the caller should create a [`ResponseCompressionCtx`] per HTTP session.\n/// The caller should call the corresponding filters for the request header, response header and\n/// response body. If the algorithms are supported, the output response body will be encoded.\n/// The response header will be adjusted accordingly as well. If the algorithm is not supported\n/// or no encoding needed, the response is untouched.\n///\n/// If configured and if the request's `accept-encoding` header contains the algorithm supported and the\n/// incoming response doesn't have that encoding, the filter will compress the response.\n/// If configured and supported, and if the incoming response's `content-encoding` isn't one of the\n/// request's `accept-encoding` supported algorithm, the ctx will decompress the response.\n///\n/// # Currently supported algorithms and actions\n/// - Brotli decompression: if the response is br compressed, this ctx can decompress it\n/// - Gzip compression: if the response is uncompressed, this ctx can compress it with gzip\npub struct ResponseCompressionCtx(CtxInner);\n\nenum CtxInner {\n    HeaderPhase {\n        compression_level: u32,\n        decompress_enable: bool,\n        // Store the preferred list to compare with content-encoding\n        accept_encoding: Vec<Algorithm>,\n    },\n    BodyPhase(Option<Box<dyn Encode + Send + Sync>>),\n}\n\nimpl ResponseCompressionCtx {\n    /// Create a new [`ResponseCompressionCtx`] with the expected compression level. `0` will disable\n    /// the compression.\n    /// The `decompress_enable` flag will tell the ctx to decompress if needed.\n    pub fn new(compression_level: u32, decompress_enable: bool) -> Self {\n        Self(CtxInner::HeaderPhase {\n            compression_level,\n            decompress_enable,\n            accept_encoding: Vec::new(),\n        })\n    }\n\n    /// Whether the encoder is enabled.\n    /// The enablement will change according to the request and response filter by this ctx.\n    pub fn is_enabled(&self) -> bool {\n        match &self.0 {\n            CtxInner::HeaderPhase {\n                compression_level,\n                decompress_enable,\n                accept_encoding: _,\n            } => *compression_level != 0 || *decompress_enable,\n            CtxInner::BodyPhase(c) => c.is_some(),\n        }\n    }\n\n    /// Return the stat of this ctx:\n    /// algorithm name, in bytes, out bytes, time took for the compression\n    pub fn get_info(&self) -> Option<(&'static str, usize, usize, Duration)> {\n        match &self.0 {\n            CtxInner::HeaderPhase {\n                compression_level: _,\n                decompress_enable: _,\n                accept_encoding: _,\n            } => None,\n            CtxInner::BodyPhase(c) => c.as_ref().map(|c| c.stat()),\n        }\n    }\n\n    /// Adjust the compression level.\n    /// # Panic\n    /// This function will panic if it has already started encoding the response body.\n    pub fn adjust_level(&mut self, new_level: u32) {\n        match &mut self.0 {\n            CtxInner::HeaderPhase {\n                compression_level,\n                decompress_enable: _,\n                accept_encoding: _,\n            } => {\n                *compression_level = new_level;\n            }\n            CtxInner::BodyPhase(_) => panic!(\"Wrong phase: BodyPhase\"),\n        }\n    }\n\n    /// Adjust the decompression flag.\n    /// # Panic\n    /// This function will panic if it has already started encoding the response body.\n    pub fn adjust_decompression(&mut self, enabled: bool) {\n        match &mut self.0 {\n            CtxInner::HeaderPhase {\n                compression_level: _,\n                decompress_enable,\n                accept_encoding: _,\n            } => {\n                *decompress_enable = enabled;\n            }\n            CtxInner::BodyPhase(_) => panic!(\"Wrong phase: BodyPhase\"),\n        }\n    }\n\n    /// Feed the request header into this ctx.\n    pub fn request_filter(&mut self, req: &RequestHeader) {\n        if !self.is_enabled() {\n            return;\n        }\n        match &mut self.0 {\n            CtxInner::HeaderPhase {\n                compression_level: _,\n                decompress_enable: _,\n                accept_encoding,\n            } => parse_accept_encoding(\n                req.headers.get(http::header::ACCEPT_ENCODING),\n                accept_encoding,\n            ),\n            CtxInner::BodyPhase(_) => panic!(\"Wrong phase: BodyPhase\"),\n        }\n    }\n\n    fn response_header_filter(&mut self, resp: &mut ResponseHeader, end: bool) {\n        match &self.0 {\n            CtxInner::HeaderPhase {\n                compression_level,\n                decompress_enable,\n                accept_encoding,\n            } => {\n                if resp.status.is_informational() {\n                    if resp.status == http::status::StatusCode::SWITCHING_PROTOCOLS {\n                        // no transformation for websocket (TODO: cite RFC)\n                        self.0 = CtxInner::BodyPhase(None);\n                    }\n                    // else, wait for the final response header for decision\n                    return;\n                }\n                // do nothing if no body\n                if end {\n                    self.0 = CtxInner::BodyPhase(None);\n                    return;\n                }\n\n                let action = decide_action(resp, accept_encoding);\n                let encoder = match action {\n                    Action::Noop => None,\n                    Action::Compress(algorithm) => algorithm.compressor(*compression_level),\n                    Action::Decompress(algorithm) => algorithm.decompressor(*decompress_enable),\n                };\n                if encoder.is_some() {\n                    adjust_response_header(resp, &action);\n                }\n                self.0 = CtxInner::BodyPhase(encoder);\n            }\n            CtxInner::BodyPhase(_) => panic!(\"Wrong phase: BodyPhase\"),\n        }\n    }\n\n    fn response_body_filter(&mut self, data: Option<&Bytes>, end: bool) -> Option<Bytes> {\n        match &mut self.0 {\n            CtxInner::HeaderPhase {\n                compression_level: _,\n                decompress_enable: _,\n                accept_encoding: _,\n            } => panic!(\"Wrong phase: HeaderPhase\"),\n            CtxInner::BodyPhase(compressor) => {\n                let result = compressor\n                    .as_mut()\n                    .map(|c| {\n                        // Feed even empty slice to compressor because it might yield data\n                        // when `end` is true\n                        let data = if let Some(b) = data { b.as_ref() } else { &[] };\n                        c.encode(data, end)\n                    })\n                    .transpose();\n                result.unwrap_or_else(|e| {\n                    warn!(\"Failed to compress, compression disabled, {}\", e);\n                    // no point to transcode further data because bad data is already seen\n                    self.0 = CtxInner::BodyPhase(None);\n                    None\n                })\n            }\n        }\n    }\n\n    /// Feed the response into this ctx.\n    /// This filter will mutate the response accordingly if encoding is needed.\n    pub fn response_filter(&mut self, t: &mut HttpTask) {\n        if !self.is_enabled() {\n            return;\n        }\n        match t {\n            HttpTask::Header(resp, end) => self.response_header_filter(resp, *end),\n            HttpTask::Body(data, end) => {\n                let compressed = self.response_body_filter(data.as_ref(), *end);\n                if compressed.is_some() {\n                    *t = HttpTask::Body(compressed, *end);\n                }\n            }\n...\n// Path: pingora-core/src/protocols/http/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP/1.x and HTTP/2 implementation APIs\n\nmod body_buffer;\npub mod client;\npub mod compression;\npub(crate) mod date;\npub mod error_resp;\npub mod server;\npub mod v1;\npub mod v2;\n\npub use server::Session as ServerSession;\n\n/// The Pingora server name string\npub const SERVER_NAME: &[u8; 7] = b\"Pingora\";\n\n/// An enum to hold all possible HTTP response events.\n#[derive(Debug)]\npub enum HttpTask {\n    /// the response header and the boolean end of response flag\n    Header(Box<pingora_http::ResponseHeader>, bool),\n    /// A piece of response header and the end of response boolean flag\n    Body(Option<bytes::Bytes>, bool),\n    /// HTTP response trailer\n    Trailer(Option<Box<http::HeaderMap>>),\n    /// Signal that the response is already finished\n    Done,\n    /// Signal that the reading of the response encounters errors.\n    Failed(pingora_error::BError),\n}\n\nimpl HttpTask {\n    /// Whether this [`HttpTask`] means the end of the response\n    pub fn is_end(&self) -> bool {\n        match self {\n            HttpTask::Header(_, end) => *end,\n            HttpTask::Body(_, end) => *end,\n            HttpTask::Trailer(_) => true,\n            HttpTask::Done => true,\n            HttpTask::Failed(_) => true,\n        }\n    }\n}\n\n// Path: pingora-core/src/protocols/http/server.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP server session APIs\n\nuse super::error_resp;\nuse super::v1::server::HttpSession as SessionV1;\nuse super::v2::server::HttpSession as SessionV2;\nuse super::HttpTask;\nuse crate::protocols::{SocketAddr, Stream};\nuse bytes::Bytes;\nuse http::header::AsHeaderName;\nuse http::HeaderValue;\nuse log::error;\nuse pingora_error::Result;\nuse pingora_http::{RequestHeader, ResponseHeader};\n\n/// HTTP server session object for both HTTP/1.x and HTTP/2\npub enum Session {\n    H1(SessionV1),\n    H2(SessionV2),\n}\n\nimpl Session {\n    /// Create a new [`Session`] from an established connection for HTTP/1.x\n    pub fn new_http1(stream: Stream) -> Self {\n        Self::H1(SessionV1::new(stream))\n    }\n\n    /// Create a new [`Session`] from an established HTTP/2 stream\n    pub fn new_http2(session: SessionV2) -> Self {\n        Self::H2(session)\n    }\n\n    /// Whether the session is HTTP/2. If not it is HTTP/1.x\n    pub fn is_http2(&self) -> bool {\n        matches!(self, Self::H2(_))\n    }\n\n    /// Read the request header. This method is required to be called first before doing anything\n    /// else with the session.\n    /// - `Ok(true)`: successful\n    /// - `Ok(false)`: client exit without sending any bytes. This is normal on reused connection.\n    /// In this case the user should give up this session.\n    pub async fn read_request(&mut self) -> Result<bool> {\n        match self {\n            Self::H1(s) => {\n                let read = s.read_request().await?;\n                Ok(read.is_some())\n            }\n            // This call will always return `Ok(true)` for Http2 because the request is already read\n            Self::H2(_) => Ok(true),\n        }\n    }\n\n    /// Return the request header it just read.\n    /// # Panic\n    /// This function will panic if [`Self::read_request()`] is not called.\n    pub fn req_header(&self) -> &RequestHeader {\n        match self {\n            Self::H1(s) => s.req_header(),\n            Self::H2(s) => s.req_header(),\n        }\n    }\n\n    /// Return a mutable reference to request header it just read.\n    /// # Panic\n    /// This function will panic if [`Self::read_request()`] is not called.\n    pub fn req_header_mut(&mut self) -> &mut RequestHeader {\n        match self {\n            Self::H1(s) => s.req_header_mut(),\n            Self::H2(s) => s.req_header_mut(),\n        }\n    }\n\n    /// Return the header by name. None if the header doesn't exist.\n    ///\n    /// In case there are multiple headers under the same name, the first one will be returned. To\n    /// get all the headers: use `self.req_header().headers.get_all()`.\n    pub fn get_header<K: AsHeaderName>(&self, key: K) -> Option<&HeaderValue> {\n        self.req_header().headers.get(key)\n    }\n\n    /// Get the header value in its raw format.\n    /// If the header doesn't exist, return an empty slice.\n    pub fn get_header_bytes<K: AsHeaderName>(&self, key: K) -> &[u8] {\n        self.get_header(key).map_or(b\"\", |v| v.as_bytes())\n    }\n\n    /// Read the request body. Ok(None) if no (more) body to read\n    pub async fn read_request_body(&mut self) -> Result<Option<Bytes>> {\n        match self {\n            Self::H1(s) => s.read_body_bytes().await,\n            Self::H2(s) => s.read_body_bytes().await,\n        }\n    }\n\n    /// Write the response header to client\n    /// Informational headers (status code 100-199, excluding 101) can be written multiple times the final\n    /// response header (status code 200+ or 101) is written.\n    pub async fn write_response_header(&mut self, resp: Box<ResponseHeader>) -> Result<()> {\n        match self {\n            Self::H1(s) => {\n                s.write_response_header(resp).await?;\n                Ok(())\n            }\n            Self::H2(s) => s.write_response_header(resp, false),\n        }\n    }\n\n    /// Similar to `write_response_header()`, this fn will clone the `resp` internally\n    pub async fn write_response_header_ref(&mut self, resp: &ResponseHeader) -> Result<()> {\n        match self {\n            Self::H1(s) => {\n                s.write_response_header_ref(resp).await?;\n                Ok(())\n            }\n            Self::H2(s) => s.write_response_header_ref(resp, false),\n        }\n    }\n\n    /// Write the response body to client\n    pub async fn write_response_body(&mut self, data: Bytes) -> Result<()> {\n        match self {\n            Self::H1(s) => {\n                s.write_body(&data).await?;\n                Ok(())\n            }\n            Self::H2(s) => s.write_body(data, false),\n        }\n    }\n\n    /// Finish the life of this request.\n    /// For H1, if connection reuse is supported, a Some(Stream) will be returned, otherwise None.\n    /// For H2, always return None because H2 stream is not reusable.\n    pub async fn finish(self) -> Result<Option<Stream>> {\n        match self {\n            Self::H1(mut s) => {\n                // need to flush body due to buffering\n                s.finish_body().await?;\n                Ok(s.reuse().await)\n            }\n            Self::H2(mut s) => {\n                s.finish()?;\n                Ok(None)\n            }\n        }\n    }\n\n    pub async fn response_duplex_vec(&mut self, tasks: Vec<HttpTask>) -> Result<bool> {\n        match self {\n            Self::H1(s) => s.response_duplex_vec(tasks).await,\n            Self::H2(s) => s.response_duplex_vec(tasks),\n        }\n    }\n\n    /// Set connection reuse. `duration` defines how long the connection is kept open for the next\n    /// request to reuse. Noop for h2\n    pub fn set_keepalive(&mut self, duration: Option<u64>) {\n        match self {\n            Self::H1(s) => s.set_server_keepalive(duration),\n            Self::H2(_) => {}\n        }\n    }\n\n    /// Return a digest of the request including the method, path and Host header\n    // TODO: make this use a `Formatter`\n    pub fn request_summary(&self) -> String {\n        match self {\n            Self::H1(s) => s.request_summary(),\n            Self::H2(s) => s.request_summary(),\n        }\n    }\n\n    /// Return the written response header. `None` if it is not written yet.\n    /// Only the final (status code >= 200 or 101) response header will be returned\n    pub fn response_written(&self) -> Option<&ResponseHeader> {\n        match self {\n            Self::H1(s) => s.response_written(),\n            Self::H2(s) => s.response_written(),\n        }\n    }\n\n    /// Give up the http session abruptly.\n    /// For H1 this will close the underlying connection\n    /// For H2 this will send RESET frame to end this stream without impacting the connection\n    pub async fn shutdown(&mut self) {\n        match self {\n            Self::H1(s) => s.shutdown().await,\n            Self::H2(s) => s.shutdown(),\n        }\n    }\n\n    pub fn to_h1_raw(&self) -> Bytes {\n        match self {\n            Self::H1(s) => s.get_headers_raw_bytes(),\n            Self::H2(s) => s.pseudo_raw_h1_request_header(),\n        }\n    }\n\n    /// Whether the whole request body is sent\n    pub fn is_body_done(&mut self) -> bool {\n        match self {\n            Self::H1(s) => s.is_body_done(),\n            Self::H2(s) => s.is_body_done(),\n        }\n    }\n\n    /// Notify the client that the entire body is sent\n    /// for H1 chunked encoding, this will end the last empty chunk\n    /// for H1 content-length, this has no effect.\n    /// for H2, this will send an empty DATA frame with END_STREAM flag\n    pub async fn finish_body(&mut self) -> Result<()> {\n        match self {\n            Self::H1(s) => s.finish_body().await.map(|_| ()),\n            Self::H2(s) => s.finish(),\n        }\n    }\n\n    /// Send error response to client\n    pub async fn respond_error(&mut self, error: u16) {\n        let resp = match error {\n            /* common error responses are pre-generated */\n            502 => error_resp::HTTP_502_RESPONSE.clone(),\n            400 => error_resp::HTTP_400_RESPONSE.clone(),\n            _ => error_resp::gen_error_response(error),\n        };\n\n        // TODO: we shouldn't be closing downstream connections on internally generated errors\n        // and possibly other upstream connect() errors (connection refused, timeout, etc)\n        //\n        // This change is only here because we DO NOT re-use downstream connections\n        // today on these errors and we should signal to the client that pingora is dropping it\n        // rather than a misleading the client with 'keep-alive'\n        self.set_keepalive(None);\n\n        self.write_response_header(Box::new(resp))\n            .await\n            .unwrap_or_else(|e| {\n                error!(\"failed to send error response to downstream: {e}\");\n            });\n    }\n\n    /// Whether there is no request body\n    pub fn is_body_empty(&mut self) -> bool {\n        match self {\n            Self::H1(s) => s.is_body_empty(),\n            Self::H2(s) => s.is_body_empty(),\n        }\n    }\n\n    pub fn retry_buffer_truncated(&self) -> bool {\n        match self {\n            Self::H1(s) => s.retry_buffer_truncated(),\n            Self::H2(s) => s.retry_buffer_truncated(),\n        }\n    }\n\n    pub fn enable_retry_buffering(&mut self) {\n        match self {\n            Self::H1(s) => s.enable_retry_buffering(),\n            Self::H2(s) => s.enable_retry_buffering(),\n        }\n    }\n\n    pub fn get_retry_buffer(&self) -> Option<Bytes> {\n        match self {\n            Self::H1(s) => s.get_retry_buffer(),\n            Self::H2(s) => s.get_retry_buffer(),\n        }\n    }\n\n    /// Read body (same as `read_request_body()`) or pending forever until downstream\n    /// terminates the session.\n    pub async fn read_body_or_idle(&mut self, no_body_expected: bool) -> Result<Option<Bytes>> {\n        match self {\n            Self::H1(s) => s.read_body_or_idle(no_body_expected).await,\n            Self::H2(s) => s.read_body_or_idle(no_body_expected).await,\n        }\n    }\n\n    pub fn as_http1(&self) -> Option<&SessionV1> {\n        match self {\n            Self::H1(s) => Some(s),\n            Self::H2(_) => None,\n        }\n    }\n\n    pub fn as_http2(&self) -> Option<&SessionV2> {\n        match self {\n            Self::H1(_) => None,\n            Self::H2(s) => Some(s),\n        }\n    }\n\n    /// Write a 100 Continue response to the client.\n    pub async fn write_continue_response(&mut self) -> Result<()> {\n        match self {\n            Self::H1(s) => s.write_continue_response().await,\n            Self::H2(s) => s.write_response_header(\n                Box::new(ResponseHeader::build(100, Some(0)).unwrap()),\n                false,\n            ),\n        }\n    }\n\n    /// Whether this request is for upgrade (e.g., websocket)\n    pub fn is_upgrade_req(&self) -> bool {\n        match self {\n            Self::H1(s) => s.is_upgrade_req(),\n            Self::H2(_) => false,\n        }\n    }\n\n    /// How many response body bytes already sent\n    pub fn body_bytes_sent(&self) -> usize {\n        match self {\n            Self::H1(s) => s.body_bytes_sent(),\n            Self::H2(s) => s.body_bytes_sent(),\n        }\n    }\n\n    /// Return the client (peer) address of the connnection.\n    pub fn client_addr(&self) -> Option<&SocketAddr> {\n        match self {\n            Self::H1(s) => s.client_addr(),\n            Self::H2(s) => s.client_addr(),\n        }\n    }\n\n    /// Return the server (local) address of the connection.\n    pub fn server_addr(&self) -> Option<&SocketAddr> {\n        match self {\n            Self::H1(s) => s.server_addr(),\n            Self::H2(s) => s.server_addr(),\n        }\n    }\n}\n\n// Path: pingora-core/src/modules/http/compression.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! HTTP compression filter\n\nuse super::*;\nuse crate::protocols::http::compression::ResponseCompressionCtx;\n\n/// HTTP response compression module\npub struct ResponseCompression(ResponseCompressionCtx);\n\nimpl HttpModule for ResponseCompression {\n    fn as_any(&self) -> &dyn std::any::Any {\n        self\n    }\n    fn as_any_mut(&mut self) -> &mut dyn std::any::Any {\n        self\n    }\n\n    fn request_header_filter(&mut self, req: &mut RequestHeader) -> Result<()> {\n        self.0.request_filter(req);\n        Ok(())\n    }\n\n    fn response_filter(&mut self, t: &mut HttpTask) -> Result<()> {\n        self.0.response_filter(t);\n        Ok(())\n    }\n}\n\n/// The builder for HTTP response compression module\npub struct ResponseCompressionBuilder {\n    level: u32,\n}\n\nimpl ResponseCompressionBuilder {\n    /// Return a [ModuleBuilder] for [ResponseCompression] with the given compression level\n    pub fn enable(level: u32) -> ModuleBuilder {\n        Box::new(ResponseCompressionBuilder { level })\n    }\n}\n\nimpl HttpModuleBuilder for ResponseCompressionBuilder {\n    fn init(&self) -> Module {\n        Box::new(ResponseCompression(ResponseCompressionCtx::new(\n            self.level, false,\n        )))\n    }\n\n    fn order(&self) -> i16 {\n        // run the response filter later than most others filters\n        i16::MIN / 2\n    }\n}\n\n// Path: pingora-core/src/modules/http/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! Modules for HTTP traffic.\n//!\n//! [HttpModule]s define request and response filters to use while running an [HttpServer]\n//! application.\n//! See the [ResponseCompression] module for an example of how to implement a basic module.\n\npub mod compression;\n\nuse crate::protocols::http::HttpTask;\nuse bytes::Bytes;\nuse once_cell::sync::OnceCell;\nuse pingora_error::Result;\nuse pingora_http::RequestHeader;\nuse std::any::Any;\nuse std::any::TypeId;\nuse std::collections::HashMap;\nuse std::sync::Arc;\n\n/// The trait an HTTP traffic module needs to implement\n// TODO: * async filters for, e.g., 3rd party auth server; * access the connection for, e.g., GeoIP\npub trait HttpModule {\n    fn request_header_filter(&mut self, _req: &mut RequestHeader) -> Result<()> {\n        Ok(())\n    }\n\n    fn request_body_filter(&mut self, body: Option<Bytes>) -> Result<Option<Bytes>> {\n        Ok(body)\n    }\n\n    fn response_filter(&mut self, _t: &mut HttpTask) -> Result<()> {\n        Ok(())\n    }\n\n    fn as_any(&self) -> &dyn Any;\n    fn as_any_mut(&mut self) -> &mut dyn Any;\n}\n\ntype Module = Box<dyn HttpModule + 'static + Send + Sync>;\n\n/// Trait to init the http module ctx for each request\npub trait HttpModuleBuilder {\n    /// The order the module will run\n    ///\n    /// The lower the value, the later it runs relative to other filters.\n    /// If the order of the filter is not important, leave it to the default 0.\n    fn order(&self) -> i16 {\n        0\n    }\n\n    /// Initialize and return the per request module context\n    fn init(&self) -> Module;\n}\n\npub type ModuleBuilder = Box<dyn HttpModuleBuilder + 'static + Send + Sync>;\n\n/// The object to hold multiple http modules\npub struct HttpModules {\n    modules: Vec<ModuleBuilder>,\n    module_index: OnceCell<Arc<HashMap<TypeId, usize>>>,\n}\n\nimpl HttpModules {\n    /// Create a new [HttpModules]\n    pub fn new() -> Self {\n        HttpModules {\n            modules: vec![],\n            module_index: OnceCell::new(),\n        }\n    }\n\n    /// Add a new [ModuleBuilder] to [HttpModules]\n    ///\n    /// Each type of [HttpModule] can be only added once.\n    /// # Panic\n    /// Panic if any [HttpModule] is added more than once.\n    pub fn add_module(&mut self, builder: ModuleBuilder) {\n        if self.module_index.get().is_some() {\n            // We use a shared module_index the index would be out of sync if we\n            // add more modules.\n            panic!(\"cannot add module after ctx is already built\")\n        }\n        self.modules.push(builder);\n        // not the most efficient way but should be fine\n        // largest order first\n        self.modules.sort_by_key(|m| -m.order());\n    }\n\n    /// Build the contexts of all the modules added to this [HttpModules]\n    pub fn build_ctx(&self) -> HttpModuleCtx {\n        let module_ctx: Vec<_> = self.modules.iter().map(|b| b.init()).collect();\n        let module_index = self\n            .module_index\n            .get_or_init(|| {\n                let mut module_index = HashMap::with_capacity(self.modules.len());\n                for (i, c) in module_ctx.iter().enumerate() {\n                    let exist = module_index.insert(c.as_any().type_id(), i);\n                    if exist.is_some() {\n                        panic!(\"duplicated filters found\")\n                    }\n                }\n                Arc::new(module_index)\n            })\n            .clone();\n\n        HttpModuleCtx {\n            module_ctx,\n            module_index,\n        }\n    }\n}\n\n/// The Contexts of multiple modules\n///\n/// This is the object that will apply all the included modules to a certain HTTP request.\n/// The modules are ordered according to their `order()`.\npub struct HttpModuleCtx {\n    // the modules in the order of execution\n    module_ctx: Vec<Module>,\n    // find the module in the vec with its type ID\n    module_index: Arc<HashMap<TypeId, usize>>,\n}\n\nimpl HttpModuleCtx {\n    /// Create a placeholder empty [HttpModuleCtx].\n    ///\n    /// [HttpModules] should be used to create nonempty [HttpModuleCtx].\n    pub fn empty() -> Self {\n        HttpModuleCtx {\n            module_ctx: vec![],\n            module_index: Arc::new(HashMap::new()),\n        }\n    }\n\n    /// Get a ref to [HttpModule] if any.\n    pub fn get<T: 'static>(&self) -> Option<&T> {\n        let idx = self.module_index.get(&TypeId::of::<T>())?;\n        let ctx = &self.module_ctx[*idx];\n        Some(\n            ctx.as_any()\n                .downcast_ref::<T>()\n                .expect(\"type should always match\"),\n        )\n    }\n\n    /// Get a mut ref to [HttpModule] if any.\n    pub fn get_mut<T: 'static>(&mut self) -> Option<&mut T> {\n        let idx = self.module_index.get(&TypeId::of::<T>())?;\n        let ctx = &mut self.module_ctx[*idx];\n        Some(\n            ctx.as_any_mut()\n                .downcast_mut::<T>()\n                .expect(\"type should always match\"),\n        )\n    }\n\n    /// Run the `request_header_filter` for all the modules according to their orders.\n    pub fn request_header_filter(&mut self, req: &mut RequestHeader) -> Result<()> {\n        for filter in self.module_ctx.iter_mut() {\n            filter.request_header_filter(req)?;\n        }\n        Ok(())\n    }\n\n    /// Run the `request_body_filter` for all the modules according to their orders.\n    pub fn request_body_filter(&mut self, mut body: Option<Bytes>) -> Result<Option<Bytes>> {\n        for filter in self.module_ctx.iter_mut() {\n            body = filter.request_body_filter(body)?;\n        }\n        Ok(body)\n    }\n\n    /// Run the `response_filter` for all the modules according to their orders.\n    pub fn response_filter(&mut self, t: &mut HttpTask) -> Result<()> {\n        for filter in self.module_ctx.iter_mut() {\n            filter.response_filter(t)?;\n        }\n        Ok(())\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    struct MyModule;\n    impl HttpModule for MyModule {\n        fn as_any(&self) -> &dyn Any {\n            self\n        }\n        fn as_any_mut(&mut self) -> &mut dyn Any {\n            self\n        }\n        fn request_header_filter(&mut self, req: &mut RequestHeader) -> Result<()> {\n            req.insert_header(\"my-filter\", \"1\")\n        }\n    }\n    struct MyModuleBuilder;\n    impl HttpModuleBuilder for MyModuleBuilder {\n        fn order(&self) -> i16 {\n            1\n        }\n\n        fn init(&self) -> Module {\n            Box::new(MyModule)\n        }\n    }\n\n    struct MyOtherModule;\n    impl HttpModule for MyOtherModule {\n        fn as_any(&self) -> &dyn Any {\n            self\n        }\n        fn as_any_mut(&mut self) -> &mut dyn Any {\n            self\n        }\n        fn request_header_filter(&mut self, req: &mut RequestHeader) -> Result<()> {\n            if req.headers.get(\"my-filter\").is_some() {\n                // if this MyOtherModule runs after MyModule\n                req.insert_header(\"my-filter\", \"2\")\n            } else {\n                // if this MyOtherModule runs before MyModule\n                req.insert_header(\"my-other-filter\", \"1\")\n            }\n        }\n    }\n    struct MyOtherModuleBuilder;\n    impl HttpModuleBuilder for MyOtherModuleBuilder {\n        fn order(&self) -> i16 {\n            -1\n        }\n\n        fn init(&self) -> Module {\n            Box::new(MyOtherModule)\n        }\n    }\n\n    #[test]\n    fn test_module_get() {\n        let mut http_module = HttpModules::new();\n        http_module.add_module(Box::new(MyModuleBuilder));\n        http_module.add_module(Box::new(MyOtherModuleBuilder));\n        let mut ctx = http_module.build_ctx();\n        assert!(ctx.get::<MyModule>().is_some());\n        assert!(ctx.get::<MyOtherModule>().is_some());\n        assert!(ctx.get::<usize>().is_none());\n        assert!(ctx.get_mut::<MyModule>().is_some());\n        assert!(ctx.get_mut::<MyOtherModule>().is_some());\n        assert!(ctx.get_mut::<usize>().is_none());\n    }\n\n    #[test]\n    fn test_module_filter() {\n        let mut http_module = HttpModules::new();\n        http_module.add_module(Box::new(MyOtherModuleBuilder));\n        http_module.add_module(Box::new(MyModuleBuilder));\n        let mut ctx = http_module.build_ctx();\n        let mut req = RequestHeader::build(\"Get\", b\"/\", None).unwrap();\n        ctx.request_header_filter(&mut req).unwrap();\n        // MyModule runs before MyOtherModule\n        assert_eq!(req.headers.get(\"my-filter\").unwrap(), \"2\");\n        assert!(req.headers.get(\"my-other-filter\").is_none());\n    }\n}\n\n// Path: pingora-core/src/apps/http_app.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! A simple HTTP application trait that maps a request to a response\n\nuse async_trait::async_trait;\nuse http::Response;\nuse log::{debug, error, trace};\nuse pingora_http::ResponseHeader;\nuse std::sync::Arc;\n\nuse crate::apps::HttpServerApp;\nuse crate::modules::http::{HttpModules, ModuleBuilder};\nuse crate::protocols::http::HttpTask;\nuse crate::protocols::http::ServerSession;\nuse crate::protocols::Stream;\nuse crate::server::ShutdownWatch;\n\n/// This trait defines how to map a request to a response\n#[cfg_attr(not(doc_async_trait), async_trait)]\npub trait ServeHttp {\n    /// Define the mapping from a request to a response.\n    /// Note that the request header is already read, but the implementation needs to read the\n    /// request body if any.\n    ///\n    /// # Limitation\n    /// In this API, the entire response has to be generated before the end of this call.\n    /// So it is not suitable for streaming response or interactive communications.\n    /// Users need to implement their own [`super::HttpServerApp`] for those use cases.\n    async fn response(&self, http_session: &mut ServerSession) -> Response<Vec<u8>>;\n}\n\n// TODO: remove this in favor of HttpServer?\n#[cfg_attr(not(doc_async_trait), async_trait)]\nimpl<SV> HttpServerApp for SV\nwhere\n    SV: ServeHttp + Send + Sync,\n{\n    async fn process_new_http(\n        self: &Arc<Self>,\n        mut http: ServerSession,\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream> {\n        match http.read_request().await {\n            Ok(res) => match res {\n                false => {\n                    debug!(\"Failed to read request header\");\n                    return None;\n                }\n                true => {\n                    debug!(\"Successfully get a new request\");\n                }\n            },\n            Err(e) => {\n                error!(\"HTTP server fails to read from downstream: {e}\");\n                return None;\n            }\n        }\n        trace!(\"{:?}\", http.req_header());\n        if *shutdown.borrow() {\n            http.set_keepalive(None);\n        } else {\n            http.set_keepalive(Some(60));\n        }\n        let new_response = self.response(&mut http).await;\n        let (parts, body) = new_response.into_parts();\n        let resp_header: ResponseHeader = parts.into();\n        match http.write_response_header(Box::new(resp_header)).await {\n            Ok(()) => {\n                debug!(\"HTTP response header done.\");\n            }\n            Err(e) => {\n                error!(\n                    \"HTTP server fails to write to downstream: {e}, {}\",\n                    http.request_summary()\n                );\n            }\n        }\n        if !body.is_empty() {\n            // TODO: check if chunked encoding is needed\n            match http.write_response_body(body.into()).await {\n                Ok(_) => debug!(\"HTTP response written.\"),\n                Err(e) => error!(\n                    \"HTTP server fails to write to downstream: {e}, {}\",\n                    http.request_summary()\n                ),\n            }\n        }\n        match http.finish().await {\n            Ok(c) => c,\n            Err(e) => {\n                error!(\"HTTP server fails to finish the request: {e}\");\n                None\n            }\n        }\n    }\n}\n\n/// A helper struct for HTTP server with http modules embedded\npub struct HttpServer<SV> {\n    app: SV,\n    modules: HttpModules,\n}\n\nimpl<SV> HttpServer<SV> {\n    /// Create a new [HttpServer] with the given app which implements [ServeHttp]\n    pub fn new_app(app: SV) -> Self {\n        HttpServer {\n            app,\n            modules: HttpModules::new(),\n        }\n    }\n\n    /// Add [ModuleBuilder] to this [HttpServer]\n    pub fn add_module(&mut self, module: ModuleBuilder) {\n        self.modules.add_module(module)\n    }\n}\n\n#[cfg_attr(not(doc_async_trait), async_trait)]\nimpl<SV> HttpServerApp for HttpServer<SV>\nwhere\n    SV: ServeHttp + Send + Sync,\n{\n    async fn process_new_http(\n        self: &Arc<Self>,\n        mut http: ServerSession,\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream> {\n        match http.read_request().await {\n            Ok(res) => match res {\n                false => {\n                    debug!(\"Failed to read request header\");\n                    return None;\n                }\n                true => {\n                    debug!(\"Successfully get a new request\");\n                }\n            },\n            Err(e) => {\n                error!(\"HTTP server fails to read from downstream: {e}\");\n                return None;\n            }\n        }\n        trace!(\"{:?}\", http.req_header());\n        if *shutdown.borrow() {\n            http.set_keepalive(None);\n        } else {\n            http.set_keepalive(Some(60));\n        }\n        let mut module_ctx = self.modules.build_ctx();\n        let req = http.req_header_mut();\n        module_ctx.request_header_filter(req).ok()?;\n        let new_response = self.app.response(&mut http).await;\n        let (parts, body) = new_response.into_parts();\n        let resp_header: ResponseHeader = parts.into();\n        let mut task = HttpTask::Header(Box::new(resp_header), body.is_empty());\n        module_ctx.response_filter(&mut task).ok()?;\n\n        trace!(\"{task:?}\");\n\n        match http.response_duplex_vec(vec![task]).await {\n            Ok(_) => {\n                debug!(\"HTTP response header done.\");\n            }\n            Err(e) => {\n                error!(\n                    \"HTTP server fails to write to downstream: {e}, {}\",\n                    http.request_summary()\n                );\n            }\n        }\n        let mut task = if !body.is_empty() {\n            HttpTask::Body(Some(body.into()), true)\n        } else {\n            HttpTask::Body(None, true)\n        };\n\n        trace!(\"{task:?}\");\n\n        module_ctx.response_filter(&mut task).ok()?;\n\n        // TODO: check if chunked encoding is needed\n        match http.response_duplex_vec(vec![task]).await {\n            Ok(_) => debug!(\"HTTP response written.\"),\n            Err(e) => error!(\n                \"HTTP server fails to write to downstream: {e}, {}\",\n                http.request_summary()\n            ),\n        }\n        match http.finish().await {\n            Ok(c) => c,\n            Err(e) => {\n                error!(\"HTTP server fails to finish the request: {e}\");\n                None\n            }\n        }\n    }\n}\n\n// Path: pingora-core/src/apps/prometheus_http_app.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! An HTTP application that reports Prometheus metrics.\n\nuse async_trait::async_trait;\nuse http::Response;\nuse prometheus::{Encoder, TextEncoder};\n\nuse super::http_app::HttpServer;\nuse crate::apps::http_app::ServeHttp;\nuse crate::modules::http::compression::ResponseCompressionBuilder;\nuse crate::protocols::http::ServerSession;\n\n/// An HTTP application that reports Prometheus metrics.\n///\n/// This application will report all the [static metrics](https://docs.rs/prometheus/latest/prometheus/index.html#static-metrics)\n/// collected via the [Prometheus](https://docs.rs/prometheus/) crate;\npub struct PrometheusHttpApp;\n\n#[cfg_attr(not(doc_async_trait), async_trait)]\nimpl ServeHttp for PrometheusHttpApp {\n    async fn response(&self, _http_session: &mut ServerSession) -> Response<Vec<u8>> {\n        let encoder = TextEncoder::new();\n        let metric_families = prometheus::gather();\n        let mut buffer = vec![];\n        encoder.encode(&metric_families, &mut buffer).unwrap();\n        Response::builder()\n            .status(200)\n            .header(http::header::CONTENT_TYPE, encoder.format_type())\n            .header(http::header::CONTENT_LENGTH, buffer.len())\n            .body(buffer)\n            .unwrap()\n    }\n}\n\n/// The [HttpServer] for [PrometheusHttpApp]\n///\n/// This type provides the functionality of [PrometheusHttpApp] with compression enabled\npub type PrometheusServer = HttpServer<PrometheusHttpApp>;\n\nimpl PrometheusServer {\n    pub fn new() -> Self {\n        let mut server = Self::new_app(PrometheusHttpApp);\n        // enable gzip level 7 compression\n        server.add_module(ResponseCompressionBuilder::enable(7));\n        server\n    }\n}\n\n// Path: pingora-core/src/apps/mod.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//! The abstraction and implementation interface for service application logic\n\npub mod http_app;\npub mod prometheus_http_app;\n\nuse crate::server::ShutdownWatch;\nuse async_trait::async_trait;\nuse log::{debug, error};\nuse std::sync::Arc;\n\nuse crate::protocols::http::v2::server;\nuse crate::protocols::http::ServerSession;\nuse crate::protocols::Digest;\nuse crate::protocols::Stream;\nuse crate::protocols::ALPN;\n\n#[cfg_attr(not(doc_async_trait), async_trait)]\n/// This trait defines the interface of a transport layer (TCP or TLS) application.\npub trait ServerApp {\n    /// Whenever a new connection is established, this function will be called with the established\n    /// [`Stream`] object provided.\n    ///\n    /// The application can do whatever it wants with the `session`.\n    ///\n    /// After processing the `session`, if the `session`'s connection is reusable, This function\n    /// can return it to the service by returning `Some(session)`. The returned `session` will be\n    /// fed to another [`Self::process_new()`] for another round of processing.\n    /// If not reusable, `None` should be returned.\n    ///\n    /// The `shutdown` argument will change from `false` to `true` when the server receives a\n    /// signal to shutdown. This argument allows the application to react accordingly.\n    async fn process_new(\n        self: &Arc<Self>,\n        mut session: Stream,\n        // TODO: make this ShutdownWatch so that all task can await on this event\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream>;\n\n    /// This callback will be called once after the service stops listening to its endpoints.\n    fn cleanup(&self) {}\n}\n\n/// This trait defines the interface of an HTTP application.\n#[cfg_attr(not(doc_async_trait), async_trait)]\npub trait HttpServerApp {\n    /// Similar to the [`ServerApp`], this function is called whenever a new HTTP session is established.\n    ///\n    /// After successful processing, [`ServerSession::finish()`] can be called to return an optionally reusable\n    /// connection back to the service. The caller needs to make sure that the connection is in a reusable state\n    /// i.e., no error or incomplete read or write headers or bodies. Otherwise a `None` should be returned.\n    async fn process_new_http(\n        self: &Arc<Self>,\n        mut session: ServerSession,\n        // TODO: make this ShutdownWatch so that all task can await on this event\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream>;\n\n    /// Provide options on how HTTP/2 connection should be established. This function will be called\n    /// every time a new HTTP/2 **connection** needs to be established.\n    ///\n    /// A `None` means to use the built-in default options. See [`server::H2Options`] for more details.\n    fn h2_options(&self) -> Option<server::H2Options> {\n        None\n    }\n\n    fn http_cleanup(&self) {}\n}\n\n#[cfg_attr(not(doc_async_trait), async_trait)]\nimpl<T> ServerApp for T\nwhere\n    T: HttpServerApp + Send + Sync + 'static,\n{\n    async fn process_new(\n        self: &Arc<Self>,\n        stream: Stream,\n        shutdown: &ShutdownWatch,\n    ) -> Option<Stream> {\n        match stream.selected_alpn_proto() {\n            Some(ALPN::H2) => {\n                // create a shared connection digest\n                let digest = Arc::new(Digest {\n                    ssl_digest: stream.get_ssl_digest(),\n                    // TODO: log h2 handshake time\n                    timing_digest: stream.get_timing_digest(),\n                    proxy_digest: stream.get_proxy_digest(),\n                    socket_digest: stream.get_socket_digest(),\n                });\n\n                let h2_options = self.h2_options();\n                let h2_conn = server::handshake(stream, h2_options).await;\n                let mut h2_conn = match h2_conn {\n                    Err(e) => {\n                        error!(\"H2 handshake error {e}\");\n                        return None;\n                    }\n                    Ok(c) => c,\n                };\n\n                loop {\n                    // this loop ends when the client decides to close the h2 conn\n                    // TODO: add a timeout?\n                    let h2_stream =\n                        server::HttpSession::from_h2_conn(&mut h2_conn, digest.clone()).await;\n                    let h2_stream = match h2_stream {\n                        Err(e) => {\n                            // It is common for the client to just disconnect TCP without properly\n                            // closing H2. So we don't log the errors here\n                            debug!(\"H2 error when accepting new stream {e}\");\n                            return None;\n                        }\n                        Ok(s) => s?, // None means the connection is ready to be closed\n                    };\n                    let app = self.clone();\n                    let shutdown = shutdown.clone();\n                    pingora_runtime::current_handle().spawn(async move {\n                        app.process_new_http(ServerSession::new_http2(h2_stream), &shutdown)\n                            .await;\n                    });\n                }\n            }\n            _ => {\n                // No ALPN or ALPN::H1 or something else, just try Http1\n                self.process_new_http(ServerSession::new_http1(stream), shutdown)\n                    .await\n            }\n        }\n    }\n\n    fn cleanup(&self) {\n        self.http_cleanup()\n    }\n}\n\n// Path: pingora-core/src/listeners/tls.rs\n// Copyright 2024 Cloudflare, Inc.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n// http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\nuse log::debug;\nuse pingora_error::{ErrorType, OrErr, Result};\nuse std::ops::{Deref, DerefMut};\n\nuse crate::protocols::ssl::{\n    server::{handshake, handshake_with_callback, TlsAcceptCallbacks},\n    SslStream,\n};\nuse crate::protocols::IO;\nuse crate::tls::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod};\n\npub use crate::protocols::ssl::ALPN;\n\npub const TLS_CONF_ERR: ErrorType = ErrorType::Custom(\"TLSConfigError\");\n\npub(crate) struct Acceptor {\n    ssl_acceptor: SslAcceptor,\n    callbacks: Option<TlsAcceptCallbacks>,\n}\n\n/// The TLS settings of a listening endpoint\npub struct TlsSettings {\n    accept_builder: SslAcceptorBuilder,\n    callbacks: Option<TlsAcceptCallbacks>,\n}\n\nimpl Deref for TlsSettings {\n    type Target = SslAcceptorBuilder;\n\n    fn deref(&self) -> &Self::Target {\n        &self.accept_builder\n    }\n}\n\nimpl DerefMut for TlsSettings {\n    fn deref_mut(&mut self) -> &mut Self::Target {\n        &mut self.accept_builder\n    }\n}\n\nimpl TlsSettings {\n    /// Create a new [`TlsSettings`] with the [Mozilla Intermediate](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)\n    /// server side TLS settings. Users can adjust the TLS settings after this object is created.\n    /// Return error if the provided certificate and private key are invalid or not found.\n    pub fn intermediate(cert_path: &str, key_path: &str) -> Result<Self> {\n        let mut accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err(\n            TLS_CONF_ERR,\n            \"fail to create mozilla_intermediate_v5 Acceptor\",\n        )?;\n        accept_builder\n            .set_private_key_file(key_path, SslFiletype::PEM)\n            .or_err(TLS_CONF_ERR, \"fail to read key file {key_path}\")?;\n        accept_builder\n            .set_certificate_chain_file(cert_path)\n            .or_err(TLS_CONF_ERR, \"fail to read cert file {cert_path}\")?;\n        Ok(TlsSettings {\n            accept_builder,\n            callbacks: None,\n        })\n    }\n\n    /// Create a new [`TlsSettings`] similar to [TlsSettings::intermediate()]. A struct that implements [TlsAcceptCallbacks]\n    /// is needed to provide the certificate during the TLS handshake.\n    pub fn with_callbacks(callbacks: TlsAcceptCallbacks) -> Result<Self> {\n        let accept_builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).or_err(\n            TLS_CONF_ERR,\n            \"fail to create mozilla_intermediate_v5 Acceptor\",\n        )?;\n        Ok(TlsSettings {\n            accept_builder,\n            callbacks: Some(callbacks),\n        })\n    }\n\n    /// Enable HTTP/2 support for this endpoint, which is default off.\n    /// This effectively sets the ALPN to prefer HTTP/2 with HTTP/1.1 allowed\n    \npub fn enable_h2(&mut self) {\n        self.set_alpn(ALPN::H2H1);\n    }\n\n    /// Set the ALPN preference of this endpoint. See [`ALPN`] for more details\n    pub fn set_alpn(&mut self, alpn: ALPN) {\n        match alpn {\n            ALPN::H2H1 => self\n                .accept_builder\n                .set_alpn_select_callback(alpn::prefer_h2),\n            ALPN::H1 => self.accept_builder.set_alpn_select_callback(alpn::h1_only),\n            ALPN::H2 => self.accept_builder.set_alpn_select_callback(alpn::h2_only),\n        }\n    }\n\n    pub(crate) fn build(self) -> Acceptor {\n        Acceptor {\n            ssl_acceptor: self.accept_builder.build(),\n            callbacks: self.callbacks,\n        }\n    }\n}\n\nimpl Acceptor {\n    pub async fn tls_handshake<S: IO>(&self, stream: S) -> Result<SslStream<S>> {\n        debug!(\"new ssl session\");\n        // TODO: be able to offload this handshake in a thread pool\n        if let Some(cb) = self.callbacks.as_ref() {\n            handshake_with_callback(&self.ssl_acceptor, stream, cb).await\n        } else {\n            handshake(&self.ssl_acceptor, stream).await\n        }\n    }\n}\n\nmod alpn {\n    use super::*;\n    use crate::tls::ssl::{select_next_proto, AlpnError, SslRef};\n\n    // A standard implementation provided by the SSL lib is used below\n\n    pub fn prefer_h2<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> {\n        match select_next_proto(ALPN::H2H1.to_wire_preference(), alpn_in) {\n            Some(p) => Ok(p),\n            _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1\n        }\n    }\n\n    pub fn h1_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> {\n        match select_next_proto(ALPN::H1.to_wire_preference(), alpn_in) {\n            Some(p) => Ok(p),\n            _ => Err(AlpnError::NOACK), // unknown ALPN, just ignore it. Most clients will fallback to h1\n        }\n    }\n\n    pub fn h2_only<'a>(_ssl: &mut SslRef, alpn_in: &'a [u8]) -> Result<&'a [u8], AlpnError> {\n        match select_next_proto(ALPN::H2.to_wire_preference(), alpn_in) {\n            Some(p) => Ok(p),\n            _ => Err(AlpnError::ALERT_FATAL), // cannot agree\n        }\n    }\n}\n\n// Path: pingora-core/src/listeners/l4.rs\n...\n", "needle_token_start": 15401, "needle_token_end": 15432, "code_context_ntokens": 16222, "output": ["```rust\npub fn enable_h2(&mut self) {\n    self.set_alpn(ALPN::H2H1);\n}\n```"]}
{"repo": "huggingface/candle", "name": "squeeze_dims", "language": "rust", "path": "candle-core/src/tensor.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to reduce the dimensions of a tensor by removing specified dimensions if they are of size one, or by reshaping the tensor according to the remaining dimensions if multiple dimensions are specified.\n2. **Input**: This function takes a list of dimension indices as input.\n3. **Output**: It returns a new tensor with the specified dimensions removed or the tensor reshaped according to the remaining dimensions.\n4. **Procedure**: The function first checks if the input list of dimensions is empty, in which case it returns the tensor as is. If the list contains a single dimension, it applies a dimension reduction operation for that specific dimension. If multiple dimensions are specified, it filters out these dimensions and collects the sizes of the remaining dimensions to reshape the tensor accordingly.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "                op,\n            }\n            .bt())?\n        } else {\n            Ok(())\n        }\n    }\n\n    /// Split a tensor into the specified number of chunks, this may return less chunks than\n    /// specified.\n    pub fn chunk<D: Dim>(&self, chunks: usize, dim: D) -> Result<Vec<Self>> {\n        let dim = dim.to_index(self.shape(), \"chunk\")?;\n        let size = self.dim(dim)?;\n        if size < chunks {\n            (0..size).map(|i| self.narrow(dim, i, 1)).collect()\n        } else {\n            let chunk_size = size / chunks;\n            let cnt_additional = size % chunks;\n            let mut tensors = vec![];\n            let mut sum_chunk_size = 0;\n            for i in 0..chunks {\n                let chunk_size = if i < cnt_additional {\n                    chunk_size + 1\n                } else {\n                    chunk_size\n                };\n                let tensor = self.narrow(dim, sum_chunk_size, chunk_size)?;\n                tensors.push(tensor);\n                sum_chunk_size += chunk_size\n            }\n            Ok(tensors)\n        }\n    }\n\n    /// Returns a new tensor that is a narrowed version of the input, the dimension `dim`\n    /// ranges from `start` to `start + len`.\n    pub fn narrow<D: Dim>(&self, dim: D, start: usize, len: usize) -> Result<Self> {\n        let dims = self.dims();\n        let dim = dim.to_index(self.shape(), \"narrow\")?;\n        let err = |msg| {\n            Err::<(), _>(\n                Error::NarrowInvalidArgs {\n                    shape: self.shape().clone(),\n                    dim,\n                    start,\n                    len,\n                    msg,\n                }\n                .bt(),\n            )\n        };\n        if start > dims[dim] {\n            err(\"start > dim_len\")?\n        }\n        if start.saturating_add(len) > dims[dim] {\n            err(\"start + len > dim_len\")?\n        }\n        if start == 0 && dims[dim] == len {\n            Ok(self.clone())\n        } else {\n            let op = BackpropOp::new1(self, |t| Op::Narrow(t, dim, start, len));\n            let layout = self.layout().narrow(dim, start, len)?;\n            let tensor_ = Tensor_ {\n                id: TensorId::new(),\n                storage: self.storage.clone(),\n                layout,\n                op,\n                is_variable: false,\n                dtype: self.dtype,\n                device: self.device.clone(),\n            };\n            Ok(Tensor(Arc::new(tensor_)))\n        }\n    }\n\n    \nfn squeeze_dims(self, dims: &[usize]) -> Result<Self> {\n        match dims {\n            [] => Ok(self),\n            [i] => self.squeeze(*i),\n            dims => {\n                let dims = self\n                    .dims()\n                    .iter()\n                    .enumerate()\n                    .filter_map(|(dim_idx, &v)| {\n                        if dims.contains(&dim_idx) {\n                            None\n                        } else {\n                            Some(v)\n                        }\n                    })\n                    .collect::<Vec<_>>();\n                self.reshape(dims)\n            }\n        }\n    }\n\n    fn reduce_impl<D: Dim>(&self, dim: D, keepdim: bool, op: ReduceOp) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), op.name())?;\n        let storage = self.storage().reduce_op(op, self.layout(), &[dim])?;\n        let mut dims = self.dims().to_vec();\n        dims[dim] = 1;\n        let op = match op {\n            ReduceOp::Sum | ReduceOp::Min | ReduceOp::Max => {\n                BackpropOp::new1(self, |arg| Op::Reduce(arg, op, dims.to_vec()))\n            }\n            ReduceOp::ArgMin | ReduceOp::ArgMax => BackpropOp::none(),\n        };\n        let res = from_storage(storage, dims, op, false);\n        if keepdim {\n            Ok(res)\n        } else {\n            res.squeeze_dims(&[dim])\n        }\n    }\n\n    fn sum_impl<D: Dims>(&self, sum_dims: D, keepdim: bool) -> Result<Self> {\n        let sum_dims = sum_dims.to_indexes(self.shape(), \"sum\")?;\n        let storage = self\n            .storage()\n            .reduce_op(ReduceOp::Sum, self.layout(), &sum_dims)?;\n        let mut dims = self.dims().to_vec();\n        for &sum_dim in sum_dims.iter() {\n            dims[sum_dim] = 1\n        }\n        let op = BackpropOp::new1(self, |a| Op::Reduce(a, ReduceOp::Sum, dims.to_vec()));\n        let sum = from_storage(storage, dims, op, false);\n        if keepdim {\n            Ok(sum)\n        } else {\n            sum.squeeze_dims(&sum_dims)\n        }\n    }\n\n    /// Roll the tensor input along the given dimension.\n    /// Elements that are shifted beyond the last position are re-introduced at the first position.\n    ///\n    /// ```rust\n    /// # use candle_core::{Tensor, Device};\n    /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let tensor = tensor.roll(1, 0)?;\n    /// assert_eq!(tensor.to_vec2::<f32>()?, &[[4., 5.], [0., 1.], [2., 3.]]);\n    /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let tensor = tensor.roll(-1, 0)?;\n    /// assert_eq!(tensor.to_vec2::<f32>()?, &[[2., 3.], [4., 5.], [0., 1.]]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn roll<D>(&self, shift: i32, dim: D) -> Result<Self>\n    where\n        D: Dim + Clone,\n    {\n        let dim = dim.to_index(self.shape(), \"roll\")?;\n        let dim_size = self.dim(dim)?;\n        let shift = shift.rem_euclid(dim_size as i32) as usize;\n        if shift == 0 {\n            Ok(self.clone())\n        } else {\n            let a = self.narrow(dim, 0, dim_size - shift)?;\n            let b = self.narrow(dim, dim_size - shift, shift)?;\n            Tensor::cat(&[&b, &a], dim)\n        }\n    }\n\n    /// Returns the sum of all elements in the input tensor. The sum is performed over all the\n    /// input dimensions.\n    ///\n    /// The resulting tensor has a shape that is similar to the shape of the input tensor, except\n    /// that the number of elements for each dimension index in `sum_dims` is 1.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?;\n    /// let s = a.sum_keepdim(0)?;\n    /// assert_eq!(s.to_vec2::<f32>()?, &[[2., 4.]]);\n    /// let s = a.sum_keepdim(1)?;\n    /// assert_eq!(s.to_vec2::<f32>()?, &[[1.], [5.]]);\n    /// let s = a.sum_keepdim((0, 1))?;\n    /// assert_eq!(s.to_vec2::<f32>()?, &[[6.]]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn sum_keepdim<D: Dims>(&self, sum_dims: D) -> Result<Self> {\n        self.sum_impl(sum_dims, true)\n    }\n\n    /// Returns the sum of all elements in the input tensor. The sum is performed over all the\n    /// input dimensions and compared to `sum_keepdim` these dimensions are squeezed rather than\n    /// kept.\n    pub fn sum<D: Dims>(&self, sum_dims: D) -> Result<Self> {\n        self.sum_impl(sum_dims, false)\n    }\n\n    /// Returns the mean of all elements in the input tensor. The mean is performed over all the\n    /// input dimensions.\n    ///\n    /// The resulting tensor has a shape that is similar to the shape of the input tensor, except\n    /// that the number of elements for each dimension index in `mean_dims` is 1.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?;\n    /// let s = a.mean_keepdim(0)?;\n    /// assert_eq!(s.to_vec2::<f32>()?, &[[1., 2.]]);\n    /// let s = a.mean_keepdim(1)?;\n    /// assert_eq!(s.to_vec2::<f32>()?, &[[0.5], [2.5]]);\n    /// let s = a.mean_keepdim((0, 1))?;\n    /// assert_eq!(s.to_vec2::<f32>()?, &[[1.5]]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn mean_keepdim<D: Dims>(&self, mean_dims: D) -> Result<Self> {\n        let mean_dims = mean_dims.to_indexes(self.shape(), \"mean-keepdim\")?;\n        let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product();\n        let scale = 1f64 / (reduced_dim as f64);\n        self.sum_impl(mean_dims, true)? * scale\n    }\n\n    /// Returns the mean of all elements in the input tensor. The mean is performed over all the\n    /// input dimensions and compared to `mean_keepdim` these dimensions are squeezed rather than\n    /// kept.\n    pub fn mean<D: Dims>(&self, mean_dims: D) -> Result<Self> {\n        let mean_dims = mean_dims.to_indexes(self.shape(), \"mean\")?;\n        let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product();\n        let scale = 1f64 / (reduced_dim as f64);\n        self.sum_impl(mean_dims, false)? * scale\n    }\n\n    /// Returns the unbiased variance over the selected dimension.\n    pub fn var_keepdim<D: Dim>(&self, dim: D) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), \"var\")?;\n        let mean = self.mean_keepdim(dim)?;\n        let squares = self.broadcast_sub(&mean)?.sqr()?;\n        squares.sum_impl(dim, true)? / (self.dim(dim)? - 1) as f64\n    }\n\n    /// Returns the unbiased variance over the selected dimension.\n    pub fn var<D: Dim>(&self, dim: D) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), \"var\")?;\n        self.var_keepdim(dim)?.squeeze(dim)\n    }\n\n    /// Gathers the maximum value across the selected dimension. The resulting shape has the same\n    /// number of dimensions as the original tensor and the select dimension has a single element.\n    pub fn max_keepdim<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, true, ReduceOp::Max)\n    }\n\n    /// Similar to `max_keepdim` but the target dimension is squeezed.\n    pub fn max<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, false, ReduceOp::Max)\n    }\n\n    /// Gathers the minimum value across the selected dimension. The resulting shape has the same\n    /// number of dimensions as the original tensor and the select dimension has a single element.\n    pub fn min_keepdim<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, true, ReduceOp::Min)\n    }\n\n    /// Similar to `min_keepdim` but the target dimension is squeezed.\n    pub fn min<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, false, ReduceOp::Min)\n    }\n\n    pub fn argmax_keepdim<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, true, ReduceOp::ArgMax)\n    }\n\n    /// Similar to `argmax_keepdim` but the target dimension is squeezed.\n    pub fn argmax<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, false, ReduceOp::ArgMax)\n    }\n\n    pub fn argmin_keepdim<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, true, ReduceOp::ArgMin)\n    }\n\n    /// Similar to `argmin_keepdim` but the target dimension is squeezed.\n    pub fn argmin<D: Dim>(&self, dim: D) -> Result<Self> {\n        self.reduce_impl(dim, false, ReduceOp::ArgMin)\n    }\n\n    /// Element-wise comparison between two tensors, e.g. equality, greater than, ... The actual\n    /// comparison operation is specified by the `op` argument.\n    ///\n    /// The returned tensor has the same shape as the original tensors and uses `u8` elements.\n    pub fn cmp<T: TensorOrScalar>(&self, rhs: T, op: CmpOp) -> Result<Self> {\n        let rhs = match rhs.to_tensor_scalar()? {\n            crate::scalar::TensorScalar::Tensor(rhs) => rhs,\n            crate::scalar::TensorScalar::Scalar(rhs) => rhs\n                .to_dtype(self.dtype())?\n                .to_device(self.device())?\n                .broadcast_as(self.shape())?,\n        };\n        let shape = self.same_shape_binary_op(&rhs, \"cmp\")?;\n        let storage = self\n            .storage()\n            .cmp(op, &rhs.storage(), self.layout(), rhs.layout())?;\n        let op = BackpropOp::new1(self, |a| Op::Cmp(a, op));\n        Ok(from_storage(storage, shape.dims(), op, false))\n    }\n\n    /// Element-wise equality.\n    pub fn eq<T: TensorOrScalar>(&self, rhs: T) -> Result<Self> {\n        self.cmp(rhs, CmpOp::Eq)\n    }\n\n    /// Element-wise non-equality.\n    pub fn ne<T: TensorOrScalar>(&self, rhs: T) -> Result<Self> {\n        self.cmp(rhs, CmpOp::Ne)\n    }\n\n    /// Element-wise comparison with lower-than, the returned tensor uses value 1 where `self <\n    /// rhs` and 0 otherwise.\n    pub fn lt<T: TensorOrScalar>(&self, rhs: T) -> Result<Self> {\n        self.cmp(rhs, CmpOp::Lt)\n    }\n\n    /// Element-wise comparison with greater-than, the returned tensor uses value 1 where `self >\n    /// rhs` and 0 otherwise.\n    pub fn gt<T: TensorOrScalar>(&self, rhs: T) -> Result<Self> {\n        self.cmp(rhs, CmpOp::Gt)\n    }\n\n    /// Element-wise comparison with greater-equal, the returned tensor uses value 1 where `self >=\n    /// rhs` and 0 otherwise.\n    pub fn ge<T: TensorOrScalar>(&self, rhs: T) -> Result<Self> {\n        self.cmp(rhs, CmpOp::Ge)\n    }\n\n    /// Element-wise comparison with lower-equal, the returned tensor uses value 1 where `self <=\n    /// rhs` and 0 otherwise.\n    pub fn le<T: TensorOrScalar>(&self, rhs: T) -> Result<Self> {\n        self.cmp(rhs, CmpOp::Le)\n    }\n\n    /// Clamp the tensor values to be between `min` and `max`.\n    pub fn clamp<T1: TensorOrScalar, T2: TensorOrScalar>(&self, min: T1, max: T2) -> Result<Self> {\n        self.maximum(min)?.minimum(max)\n    }\n\n    /// Interpolate the input tensor to the `target_size` size, taking the value of the nearest element.\n    ///\n    /// The input tensor should have three dimensions, `(batch, channels, l)`, the returned\n    /// tensor also has three dimensions, `(batch, channels, target_size)`.\n    pub fn interpolate1d(&self, target_size: usize) -> Result<Self> {\n        let (n, c, _l) = self.dims3()?;\n        let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest1D { arg, target_size });\n        let storage = self\n            .storage()\n            .upsample_nearest1d(self.layout(), target_size)?;\n        Ok(from_storage(storage, (n, c, target_size), op, false))\n    }\n\n    /// Alias for `interpolate1d`.\n    pub fn upsample_nearest1d(&self, target_size: usize) -> Result<Self> {\n        self.interpolate1d(target_size)\n    }\n\n    /// Interpolate the input tensor to the `(target_h, target_w)` size, taking the value of the\n    /// nearest element.\n    ///\n    /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned\n    /// tensor also has four dimensions, `(batch, channels, target_h, target_w)`.\n    pub fn interpolate2d(&self, target_h: usize, target_w: usize) -> Result<Self> {\n        let (n, c, _h, _w) = self.dims4()?;\n        let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest2D {\n            arg,\n            target_h,\n            target_w,\n        });\n        let storage = self\n            .storage()\n            .upsample_nearest2d(self.layout(), target_h, target_w)?;\n        Ok(from_storage(storage, (n, c, target_h, target_w), op, false))\n    }\n\n    /// Alias for `interpolate2d`.\n    pub fn upsample_nearest2d(&self, target_h: usize, target_w: usize) -> Result<Self> {\n        self.interpolate2d(target_h, target_w)\n    }\n\n    /// 2D average pooling over an input tensor with multiple channels.\n    ///\n    /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned\n    /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on\n    /// the two last dimensions using a kernel of size `sz`. The returned element is the average\n    /// value over the kernel window.\n    pub fn avg_pool2d<T: crate::ToUsize2>(&self, sz: T) -> Result<Self> {\n        let sz = sz.to_usize2();\n        self.avg_pool2d_with_stride(sz, sz)\n    }\n\n    /// Same as `avg_pool2d` but with a `stride` that can be set to a value different from the\n    /// kernel size.\n    pub fn avg_pool2d_with_stride<T: crate::ToUsize2>(\n        &self,\n        kernel_size: T,\n        stride: T,\n    ) -> Result<Self> {\n        let kernel_size = kernel_size.to_usize2();\n        let stride = stride.to_usize2();\n        let (n, c, h, w) = self.dims4()?;\n        if h < kernel_size.0 || w < kernel_size.1 {\n            bail!(\"kernel-size {kernel_size:?} is larger than the input size {h},{w}\")\n        }\n        // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html#torch.nn.AvgPool2d\n        let h_out = (h - kernel_size.0) / stride.0 + 1;\n        let w_out = (w - kernel_size.1) / stride.1 + 1;\n        let op = BackpropOp::new1(self, |arg| Op::AvgPool2D {\n            arg,\n            kernel_size,\n            stride,\n        });\n        let storage = self\n            .storage()\n            .avg_pool2d(self.layout(), kernel_size, stride)?;\n        Ok(from_storage(storage, (n, c, h_out, w_out), op, false))\n    }\n\n    /// 2D max pooling over an input tensor with multiple channels.\n    ///\n    /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned\n    /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on\n    /// the two last dimensions using a kernel of size `sz`, the returned element is the maximum\n    /// value over the kernel window.\n    pub fn max_pool2d<T: crate::ToUsize2>(&self, sz: T) -> Result<Self> {\n        let sz = sz.to_usize2();\n        self.max_pool2d_with_stride(sz, sz)\n    }\n\n    /// Same as `max_pool2d` but with a `stride` that can be set to a value different from the\n    /// kernel size.\n    pub fn max_pool2d_with_stride<T: crate::ToUsize2>(\n        &self,\n        kernel_size: T,\n        stride: T,\n    ) -> Result<Self> {\n        let kernel_size = kernel_size.to_usize2();\n        let stride = stride.to_usize2();\n        let (n, c, h, w) = self.dims4()?;\n        if h < kernel_size.0 || w < kernel_size.1 {\n            bail!(\"kernel-size {kernel_size:?} is larger than the input size {h},{w}\")\n        }\n        // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html#torch.nn.MaxPool2d\n        let h_out = (h - kernel_size.0) / stride.0 + 1;\n        let w_out = (w - kernel_size.1) / stride.1 + 1;\n        let op = BackpropOp::new1(self, |arg| Op::MaxPool2D {\n            arg,\n            kernel_size,\n            stride,\n        });\n        let storage = self\n            .storage()\n            .max_pool2d(self.layout(), kernel_size, stride)?;\n        Ok(from_storage(storage, (n, c, h_out, w_out), op, false))\n    }\n\n    /// Returns the matrix-multiplication of the input tensor with the other provided tensor.\n    ///\n    /// # Arguments\n    ///\n    /// * `self` - A tensor with dimensions `b1, b2, ..., bi, m, k`.\n    /// * `rhs` - A tensor with dimensions `b1, b2, ..., bi, k, n`.\n    ///\n    /// The resulting tensor has dimensions `b1, b2, ..., bi, m, n`.\n    pub fn matmul(&self, rhs: &Self) -> Result<Self> {\n        let a_dims = self.shape().dims();\n        let b_dims = rhs.shape().dims();\n\n        let dim = a_dims.len();\n\n        if dim < 2 || b_dims.len() != dim {\n            Err(Error::ShapeMismatchBinaryOp {\n                lhs: self.shape().clone(),\n                rhs: rhs.shape().clone(),\n                op: \"matmul\",\n            }\n            .bt())?\n        }\n\n        let m = a_dims[dim - 2];\n        let k = a_dims[dim - 1];\n        let k2 = b_dims[dim - 2];\n        let n = b_dims[dim - 1];\n\n        let c_shape = Shape::from(&a_dims[..dim - 2]).extend(&[m, n]);\n        let batching: usize = a_dims[..dim - 2].iter().product();\n        let batching_b: usize = b_dims[..dim - 2].iter().product();\n        if k != k2 || batching != batching_b {\n            Err(Error::ShapeMismatchBinaryOp {\n                lhs: self.shape().clone(),\n                rhs: rhs.shape().clone(),\n                op: \"matmul\",\n            }\n            .bt())?\n        }\n\n        let storage = self.storage().matmul(\n            &rhs.storage(),\n            (batching, m, n, k),\n            self.layout(),\n            rhs.layout(),\n        )?;\n        let op = BackpropOp::new2(self, rhs, Op::Matmul);\n        Ok(from_storage(storage, c_shape, op, false))\n    }\n\n    /// Matrix-multiplication with broadcasting support.\n    ///\n    /// Compared to `matmul` the two matrixes are allowed to have different dimensions as long as\n    /// they are compatible for broadcast. E.g. if `self` has shape `(j, 1, n, k)` and `rhs` has\n    /// shape `(l, k, m)`, the output will have shape `(j, l, n, m)`.\n    pub fn broadcast_matmul(&self, rhs: &Self) -> Result<Self> {\n        let lhs = self;\n        let (l_shape, r_shape) = lhs.shape().broadcast_shape_matmul(rhs.shape())?;\n        let l_broadcast = l_shape != *lhs.shape();\n        let r_broadcast = r_shape != *rhs.shape();\n        // TODO: Avoid concretising the broadcasted matrixes via contiguous.\n        match (l_broadcast, r_broadcast) {\n            (true, true) => lhs\n                .broadcast_as(&l_shape)?\n                .contiguous()?\n                .matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?),\n            (false, true) => lhs.matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?),\n            (true, false) => lhs.broadcast_as(&l_shape)?.contiguous()?.matmul(rhs),\n            (false, false) => lhs.matmul(rhs),\n        }\n    }\n\n    /// Returns a tensor with the same shape as the input tensor, the values are taken from\n    /// `on_true` if the input tensor value is not zero, and `on_false` at the positions where the\n    /// input tensor is equal to zero.\n    pub fn where_cond(&self, on_true: &Self, on_false: &Self) -> Result<Self> {\n        let _shap = self.same_shape_binary_op(on_true, \"where_cond\")?;\n        let shape = self.same_shape_binary_op(on_false, \"where_cond\")?;\n        let storage = self.storage().where_cond(\n            self.layout(),\n            &on_true.storage(),\n            on_true.layout(),\n            &on_false.storage(),\n            on_false.layout(),\n        )?;\n        let op = BackpropOp::new3(self, on_true, on_false, Op::WhereCond);\n        Ok(from_storage(storage, shape, op, false))\n    }\n\n    /// Returns a tensor with the values from the `self` tensor at the index corresponding to the\n    /// values hold in the `ids` tensor.\n    ///\n    /// # Arguments\n    ///\n    /// * `self` - A tensor with dimensions `v, h`.\n    /// * `ids` - A tensor with dimensions `s` and with integer values between 0 and v (exclusive).\n    ///\n    /// The resulting tensor has dimensions `s, h`. `s` is called the sequence length, `v` the\n    /// vocabulary size, and `h` the hidden size.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let values = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let ids = Tensor::new(&[2u32, 1u32, 2u32], &Device::Cpu)?;\n    /// let emb = values.embedding(&ids)?;\n    /// assert_eq!(emb.to_vec2::<f32>()?, &[[4., 5.], [2., 3.], [4., 5.]]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn embedding(&self, ids: &Self) -> Result<Self> {\n        if self.rank() != 2 || ids.rank() != 1 {\n            Err(Error::ShapeMismatchBinaryOp {\n                lhs: self.shape().clone(),\n                rhs: ids.shape().clone(),\n                op: \"embedding\",\n            }\n            .bt())?\n        }\n        self.index_select(ids, 0)\n    }\n\n    pub fn scatter_add<D: Dim>(&self, indexes: &Self, source: &Self, dim: D) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), \"scatter-add\")?;\n        let source_dims = source.dims();\n        let self_dims = self.dims();\n        let mismatch = if source_dims.len() != self_dims.len() {\n            true\n        } else {\n            let mut mismatch = false;\n            for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() {\n                if i != dim && d1 != d2 {\n                    mismatch = true;\n                    break;\n                }\n            }\n            mismatch\n        };\n        if mismatch {\n            Err(Error::ShapeMismatchBinaryOp {\n                op: \"scatter-add (self, src)\",\n                lhs: self.shape().clone(),\n                rhs: source.shape().clone(),\n            }\n            .bt())?\n        }\n        if indexes.dims() != source.dims() {\n            Err(Error::ShapeMismatchBinaryOp {\n                op: \"scatter-add (indexes, src)\",\n                lhs: indexes.shape().clone(),\n                rhs: source.shape().clone(),\n            }\n            .bt())?\n        }\n        let storage = self.storage().scatter_add(\n            self.layout(),\n            &indexes.storage(),\n            indexes.layout(),\n            &source.storage(),\n            source.layout(),\n            dim,\n        )?;\n        let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| {\n            Op::ScatterAdd(t1, t2, t3, dim)\n        });\n        Ok(from_storage(storage, self.shape(), op, false))\n    }\n\n    /// Embeds the values of the `src` tensor into the `self` tensor on the specified dimension.\n    pub fn slice_scatter<D: Dim>(&self, src: &Self, dim: D, start: usize) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), \"slice-scatter\")?;\n        if dim == 0 {\n            self.slice_scatter0(src, start)\n        } else {\n            // TODO: Maybe we want to add a more efficient implementation at some point.\n            self.transpose(0, dim)?\n                .slice_scatter0(&src.transpose(0, dim)?, start)?\n                .transpose(0, dim)\n        }\n    }\n\n    /// Embeds the values of the `src` tensor into the `self` tensor on the first dimension.\n    pub fn slice_scatter0(&self, src: &Self, start: usize) -> Result<Self> {\n        if self.dtype() != src.dtype() {\n            Err(Error::DTypeMismatchBinaryOp {\n                lhs: self.dtype(),\n                rhs: src.dtype(),\n                op: \"slice-scatter\",\n            }\n            .bt())?\n        }\n        if self.device().location() != src.device.location() {\n            Err(Error::DeviceMismatchBinaryOp {\n                lhs: self.device().location(),\n                rhs: src.device().location(),\n                op: \"slice-scatter\",\n            }\n            .bt())?\n        }\n        if self.rank() != src.rank() {\n            Err(Error::UnexpectedNumberOfDims {\n                expected: self.rank(),\n                got: src.rank(),\n                shape: src.shape().clone(),\n            }\n            .bt())?\n        }\n        let shape_ok =\n            self.dims()\n                .iter()\n                .zip(src.dims().iter())\n                .enumerate()\n                .all(|(dim_idx, (&d1, &d2))| {\n                    if 0 == dim_idx {\n                        d2 + start <= d1\n                    } else {\n                        d1 == d2\n                    }\n                });\n        if !shape_ok {\n            Err(Error::ShapeMismatchBinaryOp {\n                op: \"slice-scatter (self, src)\",\n                lhs: self.shape().clone(),\n                rhs: src.shape().clone(),\n            }\n            .bt())?\n        }\n        let mut storage = unsafe { self.device().alloc_uninit(self.shape(), self.dtype())? };\n        self.storage()\n            .copy_strided_src(&mut storage, 0, self.layout())?;\n        let offset = start * src.dims()[1..].iter().product::<usize>();\n        src.storage()\n            .copy_strided_src(&mut storage, offset, src.layout())?;\n        let op = BackpropOp::new2(self, src, |t1, t2| Op::SliceScatter0(t1, t2, start));\n        Ok(from_storage(storage, self.shape(), op, false))\n    }\n\n    /// Accumulate element from `source` at indexes `indexes` and add them to `self`.\n    pub fn index_add<D: Dim>(&self, indexes: &Self, source: &Self, dim: D) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), \"index-add\")?;\n        let source_dims = source.dims();\n        let self_dims = self.dims();\n        let mismatch = if source_dims.len() != self_dims.len() {\n            true\n        } else {\n            let mut mismatch = false;\n            for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() {\n                if i != dim && d1 != d2 {\n                    mismatch = true;\n                    break;\n                }\n            }\n            mismatch\n        };\n        if mismatch {\n            Err(Error::ShapeMismatchBinaryOp {\n                op: \"index-add (self, source)\",\n                lhs: self.shape().clone(),\n                rhs: source.shape().clone(),\n            }\n            .bt())?\n        }\n        // The number of element in indexes must match the dimension on which the add is\n        // performed on the source tensor (and the index values from `indexes` are taken from\n        // the target tensor self)\n        let indexes_len = indexes.dims1()?;\n        if source_dims[dim] != indexes_len {\n            Err(Error::ShapeMismatchBinaryOp {\n                op: \"index-add (ids, source))\",\n                lhs: indexes.shape().clone(),\n                rhs: source.shape().clone(),\n            }\n            .bt())?\n        }\n        let storage = self.storage().index_add(\n            self.layout(),\n            &indexes.storage(),\n            indexes.layout(),\n            &source.storage(),\n            source.layout(),\n            dim,\n        )?;\n        let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| {\n            Op::IndexAdd(t1, t2, t3, dim)\n        });\n        Ok(from_storage(storage, self.shape(), op, false))\n    }\n\n    /// Gather values across the target dimension.\n    ///\n    /// # Arguments\n    ///\n    /// * `self` - The input tensor.\n    /// * `indexes` - The indices of elements to gather, this should have the same shape as `self`\n    ///   but can have a different number of elements on the target dimension.\n    /// * `dim` - the target dimension.\n    ///\n    /// The resulting tensor has the same shape as `indexes` and use values from `self` indexed on\n    /// dimension `dim` by the values in `indexes`.\n    pub fn gather<D: Dim>(&self, indexes: &Self, dim: D) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), \"gather\")?;\n        let self_dims = self.dims();\n        let indexes_dims = indexes.dims();\n        let mismatch = if indexes_dims.len() != self_dims.len() {\n            true\n        } else {\n            let mut mismatch = false;\n            for (i, (&d1, &d2)) in self_dims.iter().zip(indexes_dims.iter()).enumerate() {\n                if i != dim && d1 != d2 {\n                    mismatch = true;\n                    break;\n                }\n            }\n            mismatch\n        };\n        if mismatch {\n            Err(Error::ShapeMismatchBinaryOp {\n                op: \"gather\",\n                lhs: self.shape().clone(),\n                rhs: indexes.shape().clone(),\n            }\n            .bt())?\n        }\n        let storage =\n            self.storage()\n                .gather(self.layout(), &indexes.storage(), indexes.layout(), dim)?;\n        let op = BackpropOp::new2(self, indexes, |t1, t2| Op::Gather(t1, t2, dim));\n        Ok(from_storage(storage, indexes.shape(), op, false))\n    }\n\n    /// Select values for the input tensor at the target indexes across the specified dimension.\n    ///\n    /// The `indexes` is argument is an int tensor with a single dimension.\n    /// The output has the same number of dimension as the `self` input. The target dimension of\n    /// the output has length the length of `indexes` and the values are taken from `self` using\n    /// the index from `indexes`. Other dimensions have the same number of elements as the input\n    /// tensor.\n    pub fn index_select<D: Dim>(&self, indexes: &Self, dim: D) -> Result<Self> {\n        let dim = dim.to_index(self.shape(), \"index-select\")?;\n        let indexes_len = match indexes.dims() {\n            [l] => *l,\n            _ => Err(Error::ShapeMismatchBinaryOp {\n                lhs: self.shape().clone(),\n                rhs: indexes.shape().clone(),\n                op: \"index-select\",\n            }\n            .bt())?,\n        };\n        let storage = self.storage().index_select(\n            &indexes.storage(),\n            self.layout(),\n            indexes.layout(),\n            dim,\n        )?;\n        let mut dims = self.dims().to_vec();\n        dims[dim] = indexes_len;\n        let op = BackpropOp::new2(self, indexes, |t1, t2| Op::IndexSelect(t1, t2, dim));\n        Ok(from_storage(storage, dims, op, false))\n    }\n\n    /// Returns an iterator over position of the elements in the storage when ranging over the\n    /// index tuples in lexicographic order.\n    pub fn strided_index(&self) -> crate::StridedIndex {\n        self.layout.strided_index()\n    }\n\n    /// Similar to `strided_index` but returns the position of the start of each contiguous block\n    /// as well as the length of the contiguous blocks. For a contiguous tensor, the index iterator\n    /// will only return the start offset and the size would be the number of elements in the\n    /// tensor.\n    pub fn strided_blocks(&self) -> crate::StridedBlocks {\n        self.layout.strided_blocks()\n    }\n\n    /// Returns the data contained in a 1D tensor as a vector of scalar values.\n    pub fn to_vec1<S: crate::WithDType>(&self) -> Result<Vec<S>> {\n        if self.rank() != 1 {\n            Err(Error::UnexpectedNumberOfDims {\n                expected: 1,\n                got: self.rank(),\n                shape: self.shape().clone(),\n            }\n            .bt())?\n        }\n        let from_cpu_storage = |cpu_storage: &crate::CpuStorage| {\n            let data = S::cpu_storage_as_slice(cpu_storage)?;\n            let data = match self.layout.contiguous_offsets() {\n                Some((o1, o2)) => data[o1..o2].to_vec(),\n                None => self.strided_index().map(|i| data[i]).collect(),\n            };\n            Ok::<Vec<_>, Error>(data)\n        };\n        match &*self.storage() {\n            Storage::Cpu(storage) => from_cpu_storage(storage),\n            Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?),\n            Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?),\n        }\n    }\n\n    /// Returns the data contained in a 2D tensor as a vector of vector of scalar values.\n    pub fn to_vec2<S: crate::WithDType>(&self) -> Result<Vec<Vec<S>>> {\n        let (dim1, dim2) = self.dims2()?;\n        let from_cpu_storage = |cpu_storage: &crate::CpuStorage| {\n            let data = S::cpu_storage_as_slice(cpu_storage)?;\n            let mut rows = vec![];\n            match self.layout.contiguous_offsets() {\n                Some((o1, o2)) => {\n                    let data = &data[o1..o2];\n                    for idx_row in 0..dim1 {\n                        rows.push(data[idx_row * dim2..(idx_row + 1) * dim2].to_vec())\n                    }\n                }\n                None => {\n                    let mut src_index = self.strided_index();\n                    for _idx_row in 0..dim1 {\n                        let row = (0..dim2).map(|_| data[src_index.next().unwrap()]).collect();\n                        rows.push(row)\n                    }\n                    assert!(src_index.next().is_none());\n                }\n            }\n            Ok(rows)\n        };\n        match &*self.storage() {\n            Storage::Cpu(storage) => from_cpu_storage(storage),\n            Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?),\n            Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?),\n        }\n    }\n\n    /// Returns the data contained in a 3D tensor.\n    pub fn to_vec3<S: crate::WithDType>(&self) -> Result<Vec<Vec<Vec<S>>>> {\n        let (dim1, dim2, dim3) = self.dims3()?;\n        let from_cpu_storage = |cpu_storage: &crate::CpuStorage| {\n            let data = S::cpu_storage_as_slice(cpu_storage)?;\n            let mut top_rows = vec![];\n            match self.layout.contiguous_offsets() {\n                Some((o1, o2)) => {\n                    let data = &data[o1..o2];\n                    let dim23 = dim2 * dim3;\n                    for idx1 in 0..dim1 {\n                        let data = &data[idx1 * dim23..(idx1 + 1) * dim23];\n                        let mut rows = vec![];\n                        for idx2 in 0..dim2 {\n                            rows.push(data[idx2 * dim3..(idx2 + 1) * dim3].to_vec())\n                        }\n                        top_rows.push(rows);\n                    }\n                }\n                None => {\n                    let mut src_index = self.strided_index();\n                    for _idx in 0..dim1 {\n                        let mut rows = vec![];\n                        for _jdx in 0..dim2 {\n                            let row = (0..dim3).map(|_| data[src_index.next().unwrap()]).collect();\n                            rows.push(row)\n                        }\n                        top_rows.push(rows);\n                    }\n                    assert!(src_index.next().is_none());\n                }\n            }\n            Ok(top_rows)\n        };\n        match &*self.storage() {\n            Storage::Cpu(storage) => from_cpu_storage(storage),\n            Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?),\n            Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?),\n        }\n    }\n\n    /// The dtype for the elements stored in the input tensor.\n    pub fn dtype(&self) -> DType {\n        self.dtype\n    }\n\n    /// The device on which the input tensor is located.\n    pub fn device(&self) -> &Device {\n        &self.device\n    }\n\n    /// The tensor shape, i.e. dimension sizes on each axis.\n    pub fn shape(&self) -> &Shape {\n        self.layout().shape()\n    }\n\n    /// The dimension size for this tensor on each axis.\n    pub fn dims(&self) -> &[usize] {\n        self.shape().dims()\n    }\n\n    /// The dimension size for a specified dimension index.\n    pub fn dim<D: Dim>(&self, dim: D) -> Result<usize> {\n        let dim = dim.to_index(self.shape(), \"dim\")?;\n        Ok(self.dims()[dim])\n    }\n\n    /// The layout of the input tensor, this stores both the shape of the tensor as well as the\n    /// strides and the start offset to apply to the underlying storage.\n    pub fn layout(&self) -> &Layout {\n        &self.layout\n    }\n\n    pub fn stride(&self) -> &[usize] {\n        self.layout.stride()\n    }\n\n    /// The number of dimensions for this tensor, 0 for a scalar tensor, 1 for a 1D tensor, etc.\n    pub fn rank(&self) -> usize {\n        self.shape().rank()\n    }\n\n    /// The number of elements stored in this tensor.\n    pub fn elem_count(&self) -> usize {\n        self.shape().elem_count()\n    }\n\n    /// The unique identifier for this tensor.\n    pub fn id(&self) -> TensorId {\n        self.id\n    }\n\n    /// Whether this tensor is a variable or not. A variable is a tensor for which gradient is\n    /// tracked and on which backpropagation can be performed.\n    pub fn is_variable(&self) -> bool {\n        self.is_variable\n    }\n\n    pub(crate) fn op(&self) -> &Option<Op> {\n        &self.op\n    }\n\n    /// Computes the sum of all the elements in this tensor and returns a tensor holding this\n    /// scalar with zero dimensions.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let tensor = tensor.sum_all()?;\n    /// assert_eq!(tensor.to_scalar::<f32>()?, 15.);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn sum_all(&self) -> Result<Tensor> {\n        let dims: Vec<_> = (0..self.rank()).collect();\n        self.sum(dims)\n    }\n\n    pub fn mean_all(&self) -> Result<Tensor> {\n        self.sum_all()? / self.elem_count() as f64\n    }\n\n    fn flatten_<D1: Dim, D2: Dim>(\n        &self,\n        start_dim: Option<D1>,\n        end_dim: Option<D2>,\n    ) -> Result<Tensor> {\n        if self.rank() == 0 {\n            self.reshape(1)\n        } else {\n            let start_dim = match start_dim {\n                None => 0,\n                Some(dim) => dim.to_index(self.shape(), \"flatten\")?,\n            };\n            let end_dim = match end_dim {\n                None => self.rank() - 1,\n                Some(dim) => dim.to_index(self.shape(), \"flatten\")?,\n            };\n            if start_dim < end_dim {\n                let dims = self.dims();\n                let mut dst_dims = dims[..start_dim].to_vec();\n                dst_dims.push(dims[start_dim..end_dim + 1].iter().product::<usize>());\n                if end_dim + 1 < dims.len() {\n                    dst_dims.extend(&dims[end_dim + 1..]);\n                }\n                self.reshape(dst_dims)\n            } else {\n                Ok(self.clone())\n            }\n        }\n    }\n\n    /// Flattens the input tensor on the dimension indexes from `start_dim` to `end_dim` (both\n    /// inclusive).\n    pub fn flatten<D1: Dim, D2: Dim>(&self, start_dim: D1, end_dim: D2) -> Result<Tensor> {\n        self.flatten_(Some(start_dim), Some(end_dim))\n    }\n\n    /// Flattens the input tensor on the dimension indexes from `0` to `end_dim` (inclusive).\n    pub fn flatten_to<D: Dim>(&self, end_dim: D) -> Result<Tensor> {\n        self.flatten_(None::<usize>, Some(end_dim))\n    }\n\n    /// Flattens the input tensor on the dimension indexes from `start_dim` (inclusive) to the last\n    /// dimension.\n    pub fn flatten_from<D: Dim>(&self, start_dim: D) -> Result<Tensor> {\n        self.flatten_(Some(start_dim), None::<usize>)\n    }\n\n    /// Flattens the input tensor by reshaping it into a one dimension tensor.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let tensor = tensor.flatten_all()?;\n    /// assert_eq!(tensor.to_vec1::<f32>()?, &[0., 1., 2., 3., 4., 5.]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn flatten_all(&self) -> Result<Tensor> {\n        self.flatten_(None::<usize>, None::<usize>)\n    }\n\n    /// Returns the sub-tensor fixing the index at `i` on the first dimension.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let t = tensor.get(0)?;\n    /// assert_eq!(t.to_vec1::<f32>()?, &[0., 1.]);\n    /// let t = tensor.get(1)?;\n    /// assert_eq!(t.to_vec1::<f32>()?, &[2., 3.]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn get(&self, i: usize) -> Result<Tensor> {\n        let dims = self.dims();\n        if dims.is_empty() {\n            Ok(self.clone())\n        } else {\n            self.narrow(0, i, 1)?.reshape(&dims[1..])\n        }\n    }\n\n    /// Returns the sub-tensor fixing the index at `index` on the dimension `dim`.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let t = tensor.get_on_dim(1, 0)?;\n    /// assert_eq!(t.to_vec1::<f32>()?, &[0., 2., 4.]);\n    /// let t = tensor.get_on_dim(1, 1)?;\n    /// assert_eq!(t.to_vec1::<f32>()?, &[1., 3., 5.]);\n    /// let t = tensor.get_on_dim(0, 1)?;\n    /// assert_eq!(t.to_vec1::<f32>()?, &[2., 3.]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn get_on_dim<D: Dim>(&self, dim: D, index: usize) -> Result<Tensor> {\n        let dim = dim.to_index(self.shape(), \"get_on_dim\")?;\n        self.narrow(dim, index, 1)?.squeeze(dim)\n    }\n\n    /// Returns a tensor that is a transposed version of the input, the two last dimensions of the\n    /// input are swapped.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?;\n    /// let tensor = tensor.t()?;\n    /// assert_eq!(tensor.to_vec2::<f32>()?, &[[0.0, 2.0, 4.0], [1.0, 3.0, 5.0]]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn t(&self) -> Result<Tensor> {\n        let rank = self.rank();\n        if rank < 2 {\n            Err(Error::UnexpectedNumberOfDims {\n                expected: 2,\n                got: rank,\n                shape: self.shape().clone(),\n            }\n            .bt())?\n        }\n        self.transpose(rank - 2, rank - 1)\n    }\n\n    /// Returns a tensor that is a transposed version of the input, the given dimensions are\n    /// swapped.\n    pub fn transpose<D1: Dim, D2: Dim>(&self, dim1: D1, dim2: D2) -> Result<Tensor> {\n        let dim1 = dim1.to_index(self.shape(), \"transpose\")?;\n        let dim2 = dim2.to_index(self.shape(), \"transpose\")?;\n        if dim1 == dim2 {\n            return Ok(self.clone());\n        }\n        let op = BackpropOp::new1(self, |t| Op::Transpose(t, dim1, dim2));\n        let tensor_ = Tensor_ {\n            id: TensorId::new(),\n            storage: self.storage.clone(),\n            layout: self.layout.transpose(dim1, dim2)?,\n            op,\n            is_variable: false,\n            dtype: self.dtype,\n            device: self.device.clone(),\n        };\n        Ok(Tensor(Arc::new(tensor_)))\n    }\n\n    /// Returns a tensor with the same data as the input where the dimensions have been permuted.\n    /// dims must be a permutation, i.e. include each dimension index exactly once.\n    ///\n    /// ```rust\n    /// use candle_core::{Tensor, Device};\n    /// let tensor = Tensor::arange(0u32, 120u32, &Device::Cpu)?.reshape((2, 3, 4, 5))?;\n    /// assert_eq!(tensor.dims(), &[2, 3, 4, 5]);\n    /// let tensor = tensor.permute((2, 3, 1, 0))?;\n    /// assert_eq!(tensor.dims(), &[4, 5, 3, 2]);\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    pub fn permute<D: Dims>(&self, dims: D) -> Result<Tensor> {\n        let dims = dims.to_indexes(self.shape(), \"permute\")?;\n        // O(n^2) permutation check but these arrays are small.\n        let is_permutation =\n            dims.len() == self.rank() && (0..dims.len()).all(|i| dims.contains(&i));\n        if !is_permutation {\n            bail!(\n                \"dimension mismatch in permute, tensor {:?}, dims: {:?}\",\n                self.dims(),\n                dims\n            )\n        }\n        let op = BackpropOp::new1(self, |t| Op::Permute(t, dims.clone()));\n        let tensor_ = Tensor_ {\n            id: TensorId::new(),\n            storage: self.storage.clone(),\n            layout: self.layout.permute(&dims)?,\n            op,\n            is_variable: false,\n            dtype: self.dtype,\n            device: self.device.clone(),\n        };\n        Ok(Tensor(Arc::new(tensor_)))\n    }\n\n    /// Returns true if the data is stored in a C contiguous (aka row major) way.\n    pub fn is_contiguous(&self) -> bool {\n        self.layout.is_contiguous()\n    }\n\n    /// Returns true if the data is stored in a Fortran contiguous (aka column major) way.\n    pub fn is_fortran_contiguous(&self) -> bool {\n        self.layout.is_fortran_contiguous()\n    }\n\n    /// Compared to clone, this copies the actual storage but may fail because of running out of\n    /// memory.\n    pub fn copy(&self) -> Result<Tensor> {\n        let op = BackpropOp::new1(self, Op::Copy);\n        let tensor_ = Tensor_ {\n            id: TensorId::new(),\n            storage: Arc::new(RwLock::new(self.storage().try_clone(self.layout())?)),\n            layout: self.layout.clone(),\n            op,\n            is_variable: false,\n            dtype: self.dtype,\n            device: self.device.clone(),\n        };\n        Ok(Tensor(Arc::new(tensor_)))\n    }\n\n    /// Returns a new tensor detached from the current graph, gradient are not propagated through\n    /// this new node. The storage of this tensor is shared with the initial tensor.\n    ///\n    /// If the tensor is already detached from the computation graph, the same tensor is returned.\n    pub fn detach(&self) -> Tensor {\n        if self.op.is_none() && !self.is_variable {\n            self.clone()\n        } else {\n            let tensor_ = Tensor_ {\n                id: TensorId::new(),\n                storage: self.storage.clone(),\n                layout: self.layout.clone(),\n                op: BackpropOp::none(),\n                is_variable: false,\n                dtype: self.dtype,\n                device: self.device.clone(),\n            };\n            Tensor(Arc::new(tensor_))\n        }\n    }\n\n    /// If the target device is the same as the tensor device, only a shallow copy is performed.\n    pub fn to_device(&self, device: &Device) -> Result<Tensor> {\n        if self.device().same_device(device) {\n            Ok(self.clone())\n        } else {\n            let storage = match (&*self.storage(), device) {\n                (Storage::Cpu(storage), Device::Cuda(cuda)) => {\n                    Storage::Cuda(cuda.storage_from_cpu_storage(storage)?)\n                }\n                (Storage::Cpu(storage), Device::Metal(metal)) => {\n                    Storage::Metal(metal.storage_from_cpu_storage(storage)?)\n                }\n                (Storage::Cuda(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?),\n                (Storage::Metal(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?),\n                (Storage::Cuda(storage), Device::Cuda(cuda)) => {\n                    // TODO: Avoid passing through the cpu storage here, especially if the gpu ids\n                    // are the same.\n                    let cpu_storage = storage.to_cpu_storage()?;\n                    Storage::Cuda(cuda.storage_from_cpu_storage(&cpu_storage)?)\n                }\n                (Storage::Cpu(storage), Device::Cpu) => Storage::Cpu(storage.clone()),\n                _ => {\n                    bail!(\"not implemented yet\")\n                }\n            };\n            let op = BackpropOp::new1(self, Op::ToDevice);\n            let tensor_ = Tensor_ {\n                id: TensorId::new(),\n                storage: Arc::new(RwLock::new(storage)),\n                layout: self.layout.clone(),\n                op,\n                is_variable: false,\n                dtype: self.dtype,\n                device: device.clone(),\n            };\n            Ok(Tensor(Arc::new(tensor_)))\n        }\n    }\n\n    /// Returns a new tensor duplicating data from the original tensor. New dimensions are inserted\n    /// on the left.\n    pub fn broadcast_left<S: Into<Shape>>(&self, left_shape: S) -> Result<Self> {\n        let left_shape = left_shape.into();\n        let mut dims = left_shape.into_dims();\n        dims.extend(self.dims());\n        self.broadcast_as(dims)\n    }\n\n    /// Broadcast the input tensor to the target shape. This returns an error if the input shape is\n    /// not compatible with the target shape.\n    ///\n    /// If the input shape is `i_1, i_2, ... i_k`, the target shape has to have `k` dimensions or\n    /// more and shape `j_1, ..., j_l, t_1, t_2, ..., t_k`. The dimensions `j_1` to `j_l` can have\n    /// any value, the dimension `t_a` must be equal to `i_a` if `i_a` is different from 1. If\n...\n", "needle_token_start": 720, "needle_token_end": 893, "code_context_ntokens": 16285, "output": ["```rust\nfn squeeze_dims(self, dims: &[usize]) -> Result<Self> {\n    match dims {\n        [] => Ok(self),\n        [i] => self.squeeze(*i),\n        dims => {\n            let dims = self\n                .dims()\n                .iter()\n                .enumerate()\n                .filter_map(|(dim_idx, &v)| {\n                    if dims.contains(&dim_idx) {\n                        None\n                    } else {\n                        Some(v)\n                    }\n                })\n                .collect::<Vec<_>>();\n            self.reshape(dims)\n        }\n    }\n}\n```"]}
{"repo": "huggingface/candle", "name": "striding_error", "language": "rust", "path": "candle-core/src/cpu_backend/mod.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a specific error when the memory layout of matrices involved in a matrix multiplication does not meet expected striding conditions.\n2. **Input**: The function takes two layout descriptions of the matrices involved in the operation and a message describing the context or error.\n3. **Output**: It returns a custom error object that encapsulates details about the unexpected striding, including the layouts of both matrices and a tuple representing the dimensions involved in the matrix multiplication.\n4. **Procedure**: The function constructs an error object by packaging the provided matrix layouts, the dimensions tuple, and the error message. This error object is then enhanced with backtrace information to aid in debugging.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        let cont_s0 = p.i_h * p.i_w * p.c_in;\n        let cont_s1 = p.i_w * p.c_in;\n        let cont_s2 = p.c_in;\n        for b_idx in 0..p.b_size {\n            for h_idx in 0..p.i_h {\n                for w_idx in 0..p.i_w {\n                    for c_idx in 0..p.c_in {\n                        let src_idx =\n                            b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3;\n                        let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx;\n                        inp_cont[dst_idx] = inp[src_idx]\n                    }\n                }\n            }\n        }\n\n        for offset_h in 0..p.k_h {\n            for offset_w in 0..p.k_w {\n                (0..p.c_out).into_par_iter().for_each(|dst_c_idx| {\n                    let dst_idx = dst_c_idx * out_w * out_h;\n                    let k_cont = (0..p.c_in)\n                        .map(|c_in_idx| {\n                            k[dst_c_idx * k_s0\n                                + c_in_idx * k_s1\n                                + offset_h * k_s2\n                                + offset_w * k_s3]\n                        })\n                        .collect::<Vec<_>>();\n                    for b_idx in 0..p.b_size {\n                        let dst_idx = dst_idx + b_idx * p.c_out * out_h * out_w;\n                        for dst_h in 0..out_h {\n                            let dst_idx = dst_idx + dst_h * out_w;\n                            let src_h = p.stride * dst_h + offset_h * p.dilation;\n                            if src_h < p.padding || src_h >= p.i_h + p.padding {\n                                continue;\n                            }\n                            let src_h = src_h - p.padding;\n                            for dst_w in 0..out_w {\n                                let dst_idx = dst_idx + dst_w;\n                                let src_w = p.stride * dst_w + offset_w * p.dilation;\n                                if src_w < p.padding || src_w >= p.i_w + p.padding {\n                                    continue;\n                                }\n                                let src_w = src_w - p.padding;\n                                let inp_cont = &inp_cont\n                                    [b_idx * cont_s0 + src_h * cont_s1 + src_w * cont_s2..];\n                                assert!(inp_cont.len() >= p.c_in);\n                                assert!(k_cont.len() >= p.c_in);\n                                let mut d = T::zero();\n                                unsafe {\n                                    T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in)\n                                }\n                                let dst_p = dst.as_ptr();\n                                // Safety: dst_idx are uniques per dst_c_idx which is used to parallelise\n                                // the different tasks so no two threads can try to write at the same\n                                // location.\n                                unsafe {\n                                    let ptr = dst_p.add(dst_idx) as *mut T;\n                                    *ptr += d\n                                }\n                            }\n                        }\n                    }\n                });\n            }\n        }\n\n        Ok(dst)\n    }\n}\n\nstruct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D);\n\nimpl<'a> Map2 for ConvTranspose2D<'a> {\n    const OP: &'static str = \"conv_transpose2d\";\n    fn f<T: WithDType>(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result<Vec<T>> {\n        let p = self.0;\n        let inp = &inp[inp_l.start_offset()..];\n        let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?;\n        let k = &k[k_l.start_offset()..];\n        let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?;\n        let (out_h, out_w) = (p.out_h(), p.out_w());\n\n        // Output shape: [b_size, c_out, out_h, out_w].\n        let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w];\n        let dst_s0 = p.c_out * out_h * out_w;\n        let dst_s1 = out_h * out_w;\n        let dst_s2 = out_w;\n        let dst_s3 = 1;\n\n        // TODO: Avoid making this copy if `inp` already has the appropriate layout.\n        let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w];\n        let cont_s0 = p.i_h * p.i_w * p.c_in;\n        let cont_s1 = p.i_w * p.c_in;\n        let cont_s2 = p.c_in;\n        for b_idx in 0..p.b_size {\n            for h_idx in 0..p.i_h {\n                for w_idx in 0..p.i_w {\n                    for c_idx in 0..p.c_in {\n                        let src_idx =\n                            b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3;\n                        let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx;\n                        inp_cont[dst_idx] = inp[src_idx]\n                    }\n                }\n            }\n        }\n\n        for k_y in 0..p.k_h {\n            for k_x in 0..p.k_w {\n                (0..p.c_out).into_par_iter().for_each(|dst_c_idx| {\n                    let k_cont = (0..p.c_in)\n                        .map(|c_in_idx| {\n                            k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_y * k_s2 + k_x * k_s3]\n                        })\n                        .collect::<Vec<_>>();\n                    for b_idx in 0..p.b_size {\n                        for inp_y in 0..p.i_h {\n                            for inp_x in 0..p.i_w {\n                                let out_x = inp_x * p.stride + k_x * p.dilation;\n                                let out_y = inp_y * p.stride + k_y * p.dilation;\n                                if out_x < p.padding || out_y < p.padding {\n                                    continue;\n                                }\n                                let out_x = out_x - p.padding;\n                                let out_y = out_y - p.padding;\n                                if out_x < out_w && out_y < out_h {\n                                    let inp_cont = &inp_cont\n                                        [b_idx * cont_s0 + inp_y * cont_s1 + inp_x * cont_s2..];\n                                    let dst_idx = b_idx * dst_s0\n                                        + out_y * dst_s2\n                                        + out_x * dst_s3\n                                        + dst_c_idx * dst_s1;\n                                    let mut d = T::zero();\n                                    unsafe {\n                                        T::vec_dot(\n                                            inp_cont.as_ptr(),\n                                            k_cont.as_ptr(),\n                                            &mut d,\n                                            p.c_in,\n                                        )\n                                    }\n                                    let dst_p = dst.as_ptr();\n                                    // Safety: dst_idx are uniques per dst_c_idx which is used to\n                                    // parallelise the different tasks so no two threads can try to\n                                    // write at the same location.\n                                    unsafe {\n                                        let ptr = dst_p.add(dst_idx) as *mut T;\n                                        *ptr += d\n                                    }\n                                }\n                            }\n                        }\n                    }\n                })\n            }\n        }\n        Ok(dst)\n    }\n}\n\nstruct MatMul((usize, usize, usize, usize));\n\nimpl MatMul {\n    \nfn striding_error(&self, lhs_l: &Layout, rhs_l: &Layout, msg: &'static str) -> Error {\n        Error::MatMulUnexpectedStriding(Box::new(crate::error::MatMulUnexpectedStriding {\n            lhs_l: lhs_l.clone(),\n            rhs_l: rhs_l.clone(),\n            bmnk: self.0,\n            msg,\n        }))\n        .bt()\n    }\n\n    fn ab_skip(&self, lhs_l: &Layout, rhs_l: &Layout) -> Result<(usize, usize)> {\n        let lhs_stride = lhs_l.stride();\n        let rhs_stride = rhs_l.stride();\n        let rank = lhs_stride.len();\n        let (_b, m, n, k) = self.0;\n        let a_skip: usize = match lhs_stride[..rank - 2] {\n            [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride,\n            [_, stride] if lhs_l.dims()[0] == 1 => stride,\n            [stride, _] if lhs_l.dims()[1] == 1 => stride,\n            [stride] => stride,\n            [] => m * k,\n            _ => Err(self.striding_error(lhs_l, rhs_l, \"non-contiguous lhs\"))?,\n        };\n        let b_skip: usize = match rhs_stride[..rank - 2] {\n            [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride,\n            [_, stride] if rhs_l.dims()[0] == 1 => stride,\n            [stride, _] if rhs_l.dims()[1] == 1 => stride,\n            [stride] => stride,\n            [] => n * k,\n            _ => Err(self.striding_error(lhs_l, rhs_l, \"non-contiguous rhs\"))?,\n        };\n        Ok((a_skip, b_skip))\n    }\n}\n\nimpl Map2 for MatMul {\n    const OP: &'static str = \"mat_mul\";\n\n    #[cfg(all(not(feature = \"mkl\"), not(feature = \"accelerate\")))]\n    fn f<T: 'static + WithDType + num_traits::Num + Copy>(\n        &self,\n        lhs: &[T],\n        lhs_l: &Layout,\n        rhs: &[T],\n        rhs_l: &Layout,\n    ) -> Result<Vec<T>> {\n        use gemm::{gemm, Parallelism};\n\n        match T::DTYPE {\n            DType::F16 | DType::F32 | DType::F64 => {}\n            _ => Err(Error::UnsupportedDTypeForOp(T::DTYPE, \"matmul\").bt())?,\n        }\n\n        let (b, m, n, k) = self.0;\n        let lhs = &lhs[lhs_l.start_offset()..];\n        let rhs = &rhs[rhs_l.start_offset()..];\n\n        let lhs_stride = lhs_l.stride();\n        let rhs_stride = rhs_l.stride();\n        let rank = lhs_stride.len();\n        let lhs_cs = lhs_stride[rank - 1];\n        let lhs_rs = lhs_stride[rank - 2];\n\n        let rhs_cs = rhs_stride[rank - 1];\n        let rhs_rs = rhs_stride[rank - 2];\n\n        let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?;\n        let c_skip: usize = m * n;\n\n        let dst_shape: Shape = (m, n).into();\n        let dst_strides = dst_shape.stride_contiguous();\n        let dst_rs = dst_strides[0];\n        let dst_cs = dst_strides[1];\n\n        let mut dst = vec![T::zero(); b * m * n];\n        let num_threads = crate::utils::get_num_threads();\n        let parallelism = if num_threads > 1 {\n            Parallelism::Rayon(num_threads)\n        } else {\n            Parallelism::None\n        };\n        for step in 0..b {\n            let lhs_p = &lhs[step * a_skip..];\n            let rhs_p = &rhs[step * b_skip..];\n            let dst_p = &mut dst[step * c_skip..];\n            unsafe {\n                gemm(\n                    /* m: usize = */ m,\n                    /* n: usize = */ n,\n                    /* k: usize = */ k,\n                    /* dst: *mut T = */ dst_p.as_mut_ptr(),\n                    /* dst_cs: isize = */ dst_cs as isize,\n                    /* dst_rs: isize = */ dst_rs as isize,\n                    /* read_dst: bool = */ false,\n                    /* lhs: *const T = */ lhs_p.as_ptr(),\n                    /* lhs_cs: isize = */ lhs_cs as isize,\n                    /* lhs_rs: isize = */ lhs_rs as isize,\n                    /* rhs: *const T = */ rhs_p.as_ptr(),\n                    /* rhs_cs: isize = */ rhs_cs as isize,\n                    /* rhs_rs: isize = */ rhs_rs as isize,\n                    /* alpha: T = */ T::zero(),\n                    /* beta: T = */ T::one(),\n                    /* conj_dst: bool = */ false,\n                    /* conj_lhs: bool = */ false,\n                    /* conj_rhs: bool = */ false,\n                    parallelism,\n                )\n            }\n        }\n        Ok(dst)\n    }\n\n    #[cfg(feature = \"accelerate\")]\n    fn f<T: 'static + WithDType + num_traits::Num + Copy>(\n        &self,\n        lhs: &[T],\n        lhs_l: &Layout,\n        rhs: &[T],\n        rhs_l: &Layout,\n    ) -> Result<Vec<T>> {\n        let (b, m, n, k) = self.0;\n        let lhs = &lhs[lhs_l.start_offset()..];\n        let rhs = &rhs[rhs_l.start_offset()..];\n\n        let lhs_stride = lhs_l.stride();\n        let rhs_stride = rhs_l.stride();\n\n        let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?;\n        let c_skip: usize = m * n;\n\n        let rhs_m1 = rhs_stride[rhs_stride.len() - 1];\n        let rhs_m2 = rhs_stride[rhs_stride.len() - 2];\n        let lhs_m1 = lhs_stride[lhs_stride.len() - 1];\n        let lhs_m2 = lhs_stride[lhs_stride.len() - 2];\n\n        let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) {\n            (n as i32, b'N')\n        } else if rhs_m1 == k && rhs_m2 == 1 {\n            (k as i32, b'T')\n        } else {\n            Err(self.striding_error(lhs_l, rhs_l, \"non-contiguous rhs\"))?\n        };\n        // The b tensor has dims batching, m, k (lhs)\n        let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) {\n            (k as i32, b'N')\n        } else if lhs_m1 == m && lhs_m2 == 1 {\n            (m as i32, b'T')\n        } else {\n            Err(self.striding_error(lhs_l, rhs_l, \"non-contiguous lhs\"))?\n        };\n\n        let mut dst = vec![T::zero(); b * m * n];\n        match T::DTYPE {\n            DType::F16 => {\n                crate::bail!(\"the accelerate backend does not support f16 matmul\")\n            }\n            DType::F32 => {\n                for step in 0..b {\n                    let lhs_p = &lhs[step * a_skip..];\n                    let rhs_p = &rhs[step * b_skip..];\n                    let dst_p = &mut dst[step * c_skip..];\n                    unsafe {\n                        let a = rhs_p.as_ptr() as *const f32;\n                        let b = lhs_p.as_ptr() as *const f32;\n                        let c = dst_p.as_mut_ptr() as *mut f32;\n                        let a = std::slice::from_raw_parts(a, a_skip);\n                        let b = std::slice::from_raw_parts(b, b_skip);\n                        let c = std::slice::from_raw_parts_mut(c, c_skip);\n                        crate::accelerate::sgemm(\n                            transa, transb, /* m= */ n as i32, /* n= */ m as i32,\n                            /* k= */ k as i32, /* alpha= */ 1., /* a= */ a,\n                            /* lda= */ lda, /* b= */ b, /* ldb= */ ldb,\n                            /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32,\n                        )\n                    }\n                }\n            }\n            DType::F64 => {\n                for step in 0..b {\n                    let lhs_p = &lhs[step * a_skip..];\n                    let rhs_p = &rhs[step * b_skip..];\n                    let dst_p = &mut dst[step * c_skip..];\n                    unsafe {\n                        let a = rhs_p.as_ptr() as *const f64;\n                        let b = lhs_p.as_ptr() as *const f64;\n                        let c = dst_p.as_mut_ptr() as *mut f64;\n                        let a = std::slice::from_raw_parts(a, a_skip);\n                        let b = std::slice::from_raw_parts(b, b_skip);\n                        let c = std::slice::from_raw_parts_mut(c, c_skip);\n                        crate::accelerate::dgemm(\n                            transa, transb, /* m= */ n as i32, /* n= */ m as i32,\n                            /* k= */ k as i32, /* alpha= */ 1., /* a= */ a,\n                            /* lda= */ lda, /* b= */ b, /* ldb= */ ldb,\n                            /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32,\n                        )\n                    }\n                }\n            }\n            dtype => Err(Error::UnsupportedDTypeForOp(dtype, \"matmul\").bt())?,\n        }\n        Ok(dst)\n    }\n\n    #[cfg(feature = \"mkl\")]\n    fn f<T: 'static + WithDType + num_traits::Num + Copy>(\n        &self,\n        lhs: &[T],\n        lhs_l: &Layout,\n        rhs: &[T],\n        rhs_l: &Layout,\n    ) -> Result<Vec<T>> {\n        let (b, m, n, k) = self.0;\n        let lhs = &lhs[lhs_l.start_offset()..];\n        let rhs = &rhs[rhs_l.start_offset()..];\n\n        let lhs_stride = lhs_l.stride();\n        let rhs_stride = rhs_l.stride();\n\n        let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?;\n        let c_skip: usize = m * n;\n\n        let rhs_m1 = rhs_stride[rhs_stride.len() - 1];\n        let rhs_m2 = rhs_stride[rhs_stride.len() - 2];\n        let lhs_m1 = lhs_stride[lhs_stride.len() - 1];\n        let lhs_m2 = lhs_stride[lhs_stride.len() - 2];\n\n        let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) {\n            (n as i32, b'N')\n        } else if rhs_m1 == k && rhs_m2 == 1 {\n            (k as i32, b'T')\n        } else {\n            Err(self.striding_error(lhs_l, rhs_l, \"non-contiguous rhs\"))?\n        };\n        // The b tensor has dims batching, m, k (lhs)\n        let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) {\n            (k as i32, b'N')\n        } else if lhs_m1 == m && lhs_m2 == 1 {\n            (m as i32, b'T')\n        } else {\n            Err(self.striding_error(lhs_l, rhs_l, \"non-contiguous lhs\"))?\n        };\n\n        let mut dst = vec![T::zero(); b * m * n];\n        match T::DTYPE {\n            DType::F16 => {\n                for step in 0..b {\n                    let lhs_p = &lhs[step * a_skip..];\n                    let rhs_p = &rhs[step * b_skip..];\n                    let dst_p = &mut dst[step * c_skip..];\n                    unsafe {\n                        let a = rhs_p.as_ptr() as *const f16;\n                        let b = lhs_p.as_ptr() as *const f16;\n                        let c = dst_p.as_mut_ptr() as *mut f16;\n                        let a = std::slice::from_raw_parts(a, a_skip);\n                        let b = std::slice::from_raw_parts(b, b_skip);\n                        let c = std::slice::from_raw_parts_mut(c, c_skip);\n                        crate::mkl::hgemm(\n                            transa,\n                            transb,\n                            /* m= */ n as i32,\n                            /* n= */ m as i32,\n                            /* k= */ k as i32,\n                            /* alpha= */ f16::ONE,\n                            /* a= */ a,\n                            /* lda= */ lda,\n                            /* b= */ b,\n                            /* ldb= */ ldb,\n                            /* beta= */ f16::ZERO,\n                            /* c= */ c,\n                            /* ldc= */ n as i32,\n                        )\n                    }\n                }\n            }\n            DType::F32 => {\n                for step in 0..b {\n                    let lhs_p = &lhs[step * a_skip..];\n                    let rhs_p = &rhs[step * b_skip..];\n                    let dst_p = &mut dst[step * c_skip..];\n                    unsafe {\n                        let a = rhs_p.as_ptr() as *const f32;\n                        let b = lhs_p.as_ptr() as *const f32;\n                        let c = dst_p.as_mut_ptr() as *mut f32;\n                        let a = std::slice::from_raw_parts(a, a_skip);\n                        let b = std::slice::from_raw_parts(b, b_skip);\n                        let c = std::slice::from_raw_parts_mut(c, c_skip);\n                        crate::mkl::sgemm(\n                            transa, transb, /* m= */ n as i32, /* n= */ m as i32,\n                            /* k= */ k as i32, /* alpha= */ 1., /* a= */ a,\n                            /* lda= */ lda, /* b= */ b, /* ldb= */ ldb,\n                            /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32,\n                        )\n                    }\n                }\n            }\n            DType::F64 => {\n                for step in 0..b {\n                    let lhs_p = &lhs[step * a_skip..];\n                    let rhs_p = &rhs[step * b_skip..];\n                    let dst_p = &mut dst[step * c_skip..];\n                    unsafe {\n                        let a = rhs_p.as_ptr() as *const f64;\n                        let b = lhs_p.as_ptr() as *const f64;\n                        let c = dst_p.as_mut_ptr() as *mut f64;\n                        let a = std::slice::from_raw_parts(a, a_skip);\n                        let b = std::slice::from_raw_parts(b, b_skip);\n                        let c = std::slice::from_raw_parts_mut(c, c_skip);\n                        crate::mkl::dgemm(\n                            transa, transb, /* m= */ n as i32, /* n= */ m as i32,\n                            /* k= */ k as i32, /* alpha= */ 1., /* a= */ a,\n                            /* lda= */ lda, /* b= */ b, /* ldb= */ ldb,\n                            /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32,\n                        )\n                    }\n                }\n            }\n            dtype => Err(Error::UnsupportedDTypeForOp(dtype, \"matmul\").bt())?,\n        }\n        Ok(dst)\n    }\n}\n\nfn elu<T: num_traits::Float>(v: T, alpha: T) -> T {\n    if v.is_sign_positive() {\n        v\n    } else {\n        (v.exp() - T::one()) * alpha\n    }\n}\n\nimpl CpuStorage {\n    pub fn as_slice<D: WithDType>(&self) -> Result<&[D]> {\n        D::cpu_storage_as_slice(self)\n    }\n\n    pub fn concat(storages: &[CpuStorage]) -> Result<CpuStorage> {\n        let storage0 = &storages[0];\n        let s = match storage0 {\n            Self::U8(_) => {\n                let storages = storages\n                    .iter()\n                    .map(|s| match s {\n                        Self::U8(s) => Ok(s.as_slice()),\n                        _ => crate::bail!(\"dtype mismatch\"),\n                    })\n                    .collect::<Result<Vec<_>>>()?\n                    .concat();\n                Self::U8(storages)\n            }\n            Self::U32(_) => {\n                let storages = storages\n                    .iter()\n                    .map(|s| match s {\n                        Self::U32(s) => Ok(s.as_slice()),\n                        _ => crate::bail!(\"dtype mismatch\"),\n                    })\n                    .collect::<Result<Vec<_>>>()?\n                    .concat();\n                Self::U32(storages)\n            }\n            Self::I64(_) => {\n                let storages = storages\n                    .iter()\n                    .map(|s| match s {\n                        Self::I64(s) => Ok(s.as_slice()),\n                        _ => crate::bail!(\"dtype mismatch\"),\n                    })\n                    .collect::<Result<Vec<_>>>()?\n                    .concat();\n                Self::I64(storages)\n            }\n            Self::BF16(_) => {\n                let storages = storages\n                    .iter()\n                    .map(|s| match s {\n                        Self::BF16(s) => Ok(s.as_slice()),\n                        _ => crate::bail!(\"dtype mismatch\"),\n                    })\n                    .collect::<Result<Vec<_>>>()?\n                    .concat();\n                Self::BF16(storages)\n            }\n            Self::F16(_) => {\n                let storages = storages\n                    .iter()\n                    .map(|s| match s {\n                        Self::F16(s) => Ok(s.as_slice()),\n                        _ => crate::bail!(\"dtype mismatch\"),\n                    })\n                    .collect::<Result<Vec<_>>>()?\n                    .concat();\n                Self::F16(storages)\n            }\n            Self::F32(_) => {\n                let storages = storages\n                    .iter()\n                    .map(|s| match s {\n                        Self::F32(s) => Ok(s.as_slice()),\n                        _ => crate::bail!(\"dtype mismatch\"),\n                    })\n                    .collect::<Result<Vec<_>>>()?\n                    .concat();\n                Self::F32(storages)\n            }\n            Self::F64(_) => {\n                let storages = storages\n                    .iter()\n                    .map(|s| match s {\n                        Self::F64(s) => Ok(s.as_slice()),\n                        _ => crate::bail!(\"dtype mismatch\"),\n                    })\n                    .collect::<Result<Vec<_>>>()?\n                    .concat();\n                Self::F64(storages)\n            }\n        };\n        Ok(s)\n    }\n}\n\nimpl BackendStorage for CpuStorage {\n    type Device = CpuDevice;\n\n    fn dtype(&self) -> DType {\n        match self {\n            Self::U8(_) => DType::U8,\n            Self::U32(_) => DType::U32,\n            Self::I64(_) => DType::I64,\n            Self::BF16(_) => DType::BF16,\n            Self::F16(_) => DType::F16,\n            Self::F32(_) => DType::F32,\n            Self::F64(_) => DType::F64,\n        }\n    }\n\n    fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result<Self> {\n        // TODO: find a way around the quadratic number of cases below.\n        match (self, dtype) {\n            (Self::U8(storage), DType::BF16) => {\n                let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32));\n                Ok(Self::BF16(data))\n            }\n            (Self::U32(storage), DType::BF16) => {\n                let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32));\n                Ok(Self::BF16(data))\n            }\n            (Self::I64(storage), DType::BF16) => {\n                let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32));\n                Ok(Self::BF16(data))\n            }\n            (Self::BF16(storage), DType::BF16) => {\n                let data = unary_map(storage, layout, |v| v);\n                Ok(Self::BF16(data))\n            }\n            (Self::F16(storage), DType::BF16) => {\n                let data = unary_map(storage, layout, |v| bf16::from_f32(v.to_f32()));\n                Ok(Self::BF16(data))\n            }\n            (Self::F32(storage), DType::BF16) => {\n                let data = unary_map(storage, layout, bf16::from_f32);\n                Ok(Self::BF16(data))\n            }\n            (Self::F64(storage), DType::BF16) => {\n                let data = unary_map(storage, layout, bf16::from_f64);\n                Ok(Self::BF16(data))\n            }\n            (Self::U8(storage), DType::F16) => {\n                let data = unary_map(storage, layout, |v| f16::from_f32(v as f32));\n                Ok(Self::F16(data))\n            }\n            (Self::U32(storage), DType::F16) => {\n                let data = unary_map(storage, layout, |v| f16::from_f32(v as f32));\n                Ok(Self::F16(data))\n            }\n            (Self::I64(storage), DType::F16) => {\n                let data = unary_map(storage, layout, |v| f16::from_f32(v as f32));\n                Ok(Self::F16(data))\n            }\n            (Self::BF16(storage), DType::F16) => {\n                let data = unary_map(storage, layout, |v| f16::from_f32(v.to_f32()));\n                Ok(Self::F16(data))\n            }\n            (Self::F16(storage), DType::F16) => {\n                let data = unary_map(storage, layout, |v| v);\n                Ok(Self::F16(data))\n            }\n            (Self::F32(storage), DType::F16) => {\n                let data = unary_map(storage, layout, f16::from_f32);\n                Ok(Self::F16(data))\n            }\n            (Self::F64(storage), DType::F16) => {\n                let data = unary_map(storage, layout, f16::from_f64);\n                Ok(Self::F16(data))\n            }\n            (Self::U8(storage), DType::F32) => {\n                let data = unary_map(storage, layout, |v| v as f32);\n                Ok(Self::F32(data))\n            }\n            (Self::U32(storage), DType::F32) => {\n                let data = unary_map(storage, layout, |v| v as f32);\n                Ok(Self::F32(data))\n            }\n            (Self::I64(storage), DType::F32) => {\n                let data = unary_map(storage, layout, |v| v as f32);\n                Ok(Self::F32(data))\n            }\n            (Self::BF16(storage), DType::F32) => {\n                let data = unary_map(storage, layout, |v| v.to_f32());\n                Ok(Self::F32(data))\n            }\n            (Self::F16(storage), DType::F32) => {\n                let data = unary_map(storage, layout, |v| v.to_f32());\n                Ok(Self::F32(data))\n            }\n            (Self::F32(storage), DType::F32) => {\n                let data = unary_map(storage, layout, |v| v);\n                Ok(Self::F32(data))\n            }\n            (Self::F64(storage), DType::F32) => {\n                let data = unary_map(storage, layout, |v| v as f32);\n                Ok(Self::F32(data))\n            }\n            (Self::U8(storage), DType::U8) => {\n                let data = unary_map(storage, layout, |v| v);\n                Ok(Self::U8(data))\n            }\n            (Self::BF16(storage), DType::U8) => {\n                let data = unary_map(storage, layout, |v| v.to_f32() as u8);\n                Ok(Self::U8(data))\n            }\n            (Self::F16(storage), DType::U8) => {\n                let data = unary_map(storage, layout, |v| v.to_f32() as u8);\n                Ok(Self::U8(data))\n            }\n            (Self::F32(storage), DType::U8) => {\n                let data = unary_map(storage, layout, |v| v as u8);\n                Ok(Self::U8(data))\n            }\n            (Self::F64(storage), DType::U8) => {\n                let data = unary_map(storage, layout, |v| v as u8);\n                Ok(Self::U8(data))\n            }\n            (Self::U32(storage), DType::U8) => {\n                let data = unary_map(storage, layout, |v| v as u8);\n                Ok(Self::U8(data))\n            }\n            (Self::I64(storage), DType::U8) => {\n                let data = unary_map(storage, layout, |v| v as u8);\n                Ok(Self::U8(data))\n            }\n            (Self::U8(storage), DType::U32) => {\n                let data = unary_map(storage, layout, |v| v as u32);\n                Ok(Self::U32(data))\n            }\n            (Self::U32(storage), DType::U32) => {\n                let data = unary_map(storage, layout, |v| v);\n                Ok(Self::U32(data))\n            }\n            (Self::I64(storage), DType::U32) => {\n                let data = unary_map(storage, layout, |v| v as u32);\n                Ok(Self::U32(data))\n            }\n            (Self::BF16(storage), DType::U32) => {\n                let data = unary_map(storage, layout, |v| v.to_f32() as u32);\n                Ok(Self::U32(data))\n            }\n            (Self::F16(storage), DType::U32) => {\n                let data = unary_map(storage, layout, |v| v.to_f32() as u32);\n                Ok(Self::U32(data))\n            }\n            (Self::F32(storage), DType::U32) => {\n                let data = unary_map(storage, layout, |v| v as u32);\n                Ok(Self::U32(data))\n            }\n            (Self::F64(storage), DType::U32) => {\n                let data = unary_map(storage, layout, |v| v as u32);\n                Ok(Self::U32(data))\n            }\n            (Self::U8(storage), DType::I64) => {\n                let data = unary_map(storage, layout, |v| v as i64);\n                Ok(Self::I64(data))\n            }\n            (Self::U32(storage), DType::I64) => {\n                let data = unary_map(storage, layout, |v| v as i64);\n                Ok(Self::I64(data))\n            }\n            (Self::I64(storage), DType::I64) => {\n                let data = unary_map(storage, layout, |v| v);\n                Ok(Self::I64(data))\n            }\n            (Self::BF16(storage), DType::I64) => {\n                let data = unary_map(storage, layout, |v| v.to_f32() as i64);\n                Ok(Self::I64(data))\n            }\n            (Self::F16(storage), DType::I64) => {\n                let data = unary_map(storage, layout, |v| v.to_f32() as i64);\n                Ok(Self::I64(data))\n            }\n            (Self::F32(storage), DType::I64) => {\n                let data = unary_map(storage, layout, |v| v as i64);\n                Ok(Self::I64(data))\n            }\n            (Self::F64(storage), DType::I64) => {\n                let data = unary_map(storage, layout, |v| v as i64);\n                Ok(Self::I64(data))\n            }\n            (Self::U8(storage), DType::F64) => {\n                let data = unary_map(storage, layout, |v| v as f64);\n                Ok(Self::F64(data))\n            }\n            (Self::U32(storage), DType::F64) => {\n                let data = unary_map(storage, layout, |v| v as f64);\n                Ok(Self::F64(data))\n            }\n            (Self::I64(storage), DType::F64) => {\n                let data = unary_map(storage, layout, |v| v as f64);\n                Ok(Self::F64(data))\n            }\n            (Self::BF16(storage), DType::F64) => {\n                let data = unary_map(storage, layout, |v| v.to_f64());\n                Ok(Self::F64(data))\n            }\n            (Self::F16(storage), DType::F64) => {\n                let data = unary_map(storage, layout, |v| v.to_f64());\n                Ok(Self::F64(data))\n            }\n            (Self::F32(storage), DType::F64) => {\n                let data = unary_map(storage, layout, |v| v as f64);\n                Ok(Self::F64(data))\n            }\n            (Self::F64(storage), DType::F64) => {\n                let data = unary_map(storage, layout, |v| v);\n                Ok(Self::F64(data))\n            }\n        }\n    }\n\n    fn reduce_op(&self, op: ReduceOp, layout: &Layout, reduce_dims: &[usize]) -> Result<Self> {\n        match op {\n            ReduceOp::Sum => {\n                let src_dims = layout.dims();\n                let mut dst_dims = src_dims.to_vec();\n                for &dim in reduce_dims.iter() {\n                    dst_dims[dim] = 1;\n                }\n                let dst_shape = Shape::from(dst_dims);\n                let mut reduce_dims = reduce_dims.to_vec();\n                // Sort the reduce_dims as they have to be processed from left to right when converting the\n                // indexes.\n                reduce_dims.sort();\n                let reduce_dims_and_stride: Vec<_> = reduce_dims\n                    .iter()\n                    .map(|&d| (src_dims[d], src_dims[d + 1..].iter().product::<usize>()))\n                    .collect();\n                ReduceSum {\n                    dst_shape: &dst_shape,\n                    reduce_dims: &reduce_dims,\n                    reduce_dims_and_stride,\n                }\n                .map(self, layout)\n            }\n            ReduceOp::Min | ReduceOp::ArgMin | ReduceOp::Max | ReduceOp::ArgMax => {\n                let reduce_dim_index = match reduce_dims {\n                    [reduce_dim_index] => *reduce_dim_index,\n                    _ => {\n                        let op = match op {\n                            ReduceOp::Min => \"min\",\n                            ReduceOp::ArgMin => \"argmin\",\n                            ReduceOp::Max => \"max\",\n                            ReduceOp::ArgMax => \"argmax\",\n                            _ => unreachable!(),\n                        };\n                        let dims = reduce_dims.to_vec();\n                        Err(Error::OnlySingleDimension { op, dims })?\n                    }\n                };\n                let (use_min, return_index) = match op {\n                    ReduceOp::Min => (true, false),\n                    ReduceOp::ArgMin => (true, true),\n                    ReduceOp::Max => (false, false),\n                    ReduceOp::ArgMax => (false, true),\n                    _ => unreachable!(),\n                };\n                ReduceIndex {\n                    reduce_dim_index,\n                    use_min,\n                    return_index,\n                }\n                .map(self, layout)\n            }\n        }\n    }\n\n    fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result<Self> {\n        Cmp(op).map(self, lhs_l, rhs, rhs_l)\n    }\n\n    fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result<Self> {\n        Affine(mul, add).map(self, layout)\n    }\n\n    fn avg_pool2d(\n        &self,\n        layout: &Layout,\n        kernel_size: (usize, usize),\n        stride: (usize, usize),\n    ) -> Result<Self> {\n        AvgPool2D(kernel_size, stride).map(self, layout)\n    }\n\n    fn max_pool2d(\n        &self,\n        layout: &Layout,\n        kernel_size: (usize, usize),\n        stride: (usize, usize),\n    ) -> Result<Self> {\n        MaxPool2D(kernel_size, stride).map(self, layout)\n    }\n\n    fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result<Self> {\n        UpsampleNearest1D(sz).map(self, layout)\n    }\n\n    fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result<Self> {\n        UpsampleNearest2D(h, w).map(self, layout)\n    }\n\n    fn powf(&self, layout: &Layout, e: f64) -> Result<Self> {\n        use num_traits::Float;\n        // TODO: Have some generic map for functions that apply on num_traits::Float elements.\n        match self {\n            Self::BF16(storage) => {\n                let data = unary_map(storage, layout, |v| v.powf(bf16::from_f64(e)));\n                Ok(Self::BF16(data))\n            }\n            Self::F16(storage) => {\n                let data = unary_map(storage, layout, |v| v.powf(f16::from_f64(e)));\n                Ok(Self::F16(data))\n            }\n            Self::F32(storage) => {\n                let data = unary_map(storage, layout, |v| v.powf(e as f32));\n                Ok(Self::F32(data))\n            }\n            Self::F64(storage) => {\n                let data = unary_map(storage, layout, |v| v.powf(e));\n                Ok(Self::F64(data))\n            }\n            Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, \"elu\").bt()),\n            Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, \"elu\").bt()),\n            Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, \"elu\").bt()),\n        }\n    }\n\n    fn elu(&self, layout: &Layout, alpha: f64) -> Result<Self> {\n        // TODO: Have some generic map for functions that apply on num_traits::Float elements.\n        match self {\n            Self::BF16(storage) => {\n                let data = unary_map(storage, layout, |v| elu(v, bf16::from_f64(alpha)));\n                Ok(Self::BF16(data))\n            }\n            Self::F16(storage) => {\n                let data = unary_map(storage, layout, |v| elu(v, f16::from_f64(alpha)));\n                Ok(Self::F16(data))\n            }\n            Self::F32(storage) => {\n                let data = unary_map(storage, layout, |v| elu(v, f32::from_f64(alpha)));\n                Ok(Self::F32(data))\n            }\n            Self::F64(storage) => {\n                let data = unary_map(storage, layout, |v| elu(v, alpha));\n                Ok(Self::F64(data))\n            }\n            Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, \"elu\").bt()),\n            Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, \"elu\").bt()),\n            Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, \"elu\").bt()),\n        }\n    }\n\n    fn unary_impl<B: UnaryOpT>(&self, layout: &Layout) -> Result<Self> {\n        match self {\n            Self::BF16(storage) => {\n                if B::BF16_VEC {\n                    let data = unary_map_vec(storage, layout, B::bf16, B::bf16_vec);\n                    Ok(Self::BF16(data))\n                } else {\n                    let data = unary_map(storage, layout, B::bf16);\n                    Ok(Self::BF16(data))\n                }\n            }\n            Self::F16(storage) => {\n                if B::F16_VEC {\n                    let data = unary_map_vec(storage, layout, B::f16, B::f16_vec);\n                    Ok(Self::F16(data))\n                } else {\n                    let data = unary_map(storage, layout, B::f16);\n                    Ok(Self::F16(data))\n                }\n            }\n            Self::F32(storage) => {\n                if B::F32_VEC {\n                    let data = unary_map_vec(storage, layout, B::f32, B::f32_vec);\n                    Ok(Self::F32(data))\n                } else {\n                    let data = unary_map(storage, layout, B::f32);\n                    Ok(Self::F32(data))\n                }\n            }\n            Self::F64(storage) => {\n                if B::F64_VEC {\n                    let data = unary_map_vec(storage, layout, B::f64, B::f64_vec);\n                    Ok(Self::F64(data))\n                } else {\n                    let data = unary_map(storage, layout, B::f64);\n                    Ok(Self::F64(data))\n                }\n            }\n            Self::U8(storage) => {\n                let data = unary_map(storage, layout, B::u8);\n                Ok(Self::U8(data))\n            }\n            Self::U32(storage) => {\n                let data = unary_map(storage, layout, B::u32);\n                Ok(Self::U32(data))\n            }\n            Self::I64(storage) => {\n                let data = unary_map(storage, layout, B::i64);\n                Ok(Self::I64(data))\n            }\n        }\n    }\n\n    fn binary_impl<B: BinaryOpT>(\n        &self,\n        rhs: &Self,\n        lhs_l: &Layout,\n        rhs_l: &Layout,\n    ) -> Result<Self> {\n        match (self, rhs) {\n            (Self::BF16(lhs), Self::BF16(rhs)) => {\n                let data = if B::BF16_VEC {\n                    binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::bf16, B::bf16_vec)\n                } else {\n                    binary_map(lhs_l, rhs_l, lhs, rhs, B::bf16)\n                };\n                Ok(Self::BF16(data))\n            }\n            (Self::F16(lhs), Self::F16(rhs)) => {\n                let data = if B::F16_VEC {\n                    binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f16, B::f16_vec)\n                } else {\n                    binary_map(lhs_l, rhs_l, lhs, rhs, B::f16)\n                };\n                Ok(Self::F16(data))\n            }\n            (Self::F32(lhs), Self::F32(rhs)) => {\n                let data = if B::F32_VEC {\n                    binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f32, B::f32_vec)\n                } else {\n                    binary_map(lhs_l, rhs_l, lhs, rhs, B::f32)\n                };\n                Ok(Self::F32(data))\n            }\n            (Self::F64(lhs), Self::F64(rhs)) => {\n                let data = if B::F64_VEC {\n                    binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f64, B::f64_vec)\n                } else {\n                    binary_map(lhs_l, rhs_l, lhs, rhs, B::f64)\n                };\n                Ok(Self::F64(data))\n            }\n            (Self::U32(lhs), Self::U32(rhs)) => {\n                let data = if B::U32_VEC {\n                    binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u32, B::u32_vec)\n                } else {\n                    binary_map(lhs_l, rhs_l, lhs, rhs, B::u32)\n                };\n                Ok(Self::U32(data))\n            }\n            (Self::I64(lhs), Self::I64(rhs)) => {\n                let data = if B::I64_VEC {\n                    binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::i64, B::i64_vec)\n                } else {\n                    binary_map(lhs_l, rhs_l, lhs, rhs, B::i64)\n                };\n                Ok(Self::I64(data))\n            }\n            (Self::U8(lhs), Self::U8(rhs)) => {\n                let data = if B::U8_VEC {\n                    binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u8, B::u8_vec)\n                } else {\n                    binary_map(lhs_l, rhs_l, lhs, rhs, B::u8)\n                };\n                Ok(Self::U8(data))\n            }\n            _ => {\n                // This should be covered by the dtype check above.\n                Err(Error::DTypeMismatchBinaryOp {\n                    lhs: self.dtype(),\n                    rhs: rhs.dtype(),\n                    op: B::NAME,\n                }\n                .bt())\n            }\n        }\n    }\n\n    fn copy2d(\n        &self,\n        dst: &mut Self,\n        d1: usize,\n        d2: usize,\n        src_s: usize,\n        dst_s: usize,\n        src_o: usize,\n        dst_o: usize,\n    ) -> Result<()> {\n        match (self, dst) {\n            (Self::U8(src), Self::U8(dst)) => copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o),\n            (Self::U32(src), Self::U32(dst)) => {\n                copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o)\n            }\n            (Self::I64(src), Self::I64(dst)) => {\n                copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o)\n            }\n            (Self::BF16(src), Self::BF16(dst)) => {\n                copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o)\n            }\n            (Self::F16(src), Self::F16(dst)) => {\n                copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o)\n            }\n            (Self::F32(src), Self::F32(dst)) => {\n                copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o)\n            }\n            (Self::F64(src), Self::F64(dst)) => {\n                copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o)\n            }\n            (_, dst) => {\n                return Err(Error::DTypeMismatchBinaryOp {\n                    lhs: self.dtype(),\n                    rhs: dst.dtype(),\n                    op: \"copy2d\",\n                }\n                .bt());\n            }\n        }\n        Ok(())\n    }\n\n    fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> {\n        match (self, dst) {\n            (Self::U8(src), Self::U8(dst)) => copy_strided_src_(src, dst, dst_offset, src_l),\n            (Self::U32(src), Self::U32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l),\n            (Self::I64(src), Self::I64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l),\n            (Self::BF16(src), Self::BF16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l),\n            (Self::F16(src), Self::F16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l),\n            (Self::F32(src), Self::F32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l),\n            (Self::F64(src), Self::F64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l),\n            (_, dst) => {\n                // This should be covered by the dtype check above.\n                return Err(Error::DTypeMismatchBinaryOp {\n                    lhs: self.dtype(),\n                    rhs: dst.dtype(),\n                    op: \"copy_strided\",\n                }\n                .bt());\n            }\n        }\n        Ok(())\n    }\n\n    fn where_cond(\n        &self,\n        layout: &Layout,\n        t: &Self,\n        t_l: &Layout,\n        f: &Self,\n        f_l: &Layout,\n    ) -> Result<Self> {\n        match self {\n            Self::U8(pred) => WCond(pred, layout).map(t, t_l, f, f_l),\n            Self::U32(pred) => WCond(pred, layout).map(t, t_l, f, f_l),\n            Self::I64(pred) => WCond(pred, layout).map(t, t_l, f, f_l),\n            _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), \"where-cond\")),\n        }\n    }\n\n    fn conv1d(\n        &self,\n        l: &Layout,\n        kernel: &Self,\n        kernel_l: &Layout,\n        params: &crate::conv::ParamsConv1D,\n    ) -> Result<Self> {\n        if !USE_IM2COL_CONV1D {\n            return Conv1D(params).map(self, l, kernel, kernel_l);\n        }\n        let op = Im2Col1D {\n            l_k: params.k_size,\n            padding: params.padding,\n            stride: params.stride,\n            dilation: params.dilation,\n        };\n        let col = op.map(self, l)?;\n        let b = params.b_size;\n        let n = params.c_out;\n        let l_out = params.l_out();\n        let k = op.l_k * params.c_in;\n        let m = l_out;\n        let col_l = Layout::contiguous((b, m, k));\n        let res = if kernel_l.is_contiguous() {\n            let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset())\n                .transpose(1, 2)?\n                .broadcast_as((b, k, n))?;\n            col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)?\n        } else {\n            // Make the kernel contiguous if not already the case.\n            let mut kernel_c = unsafe {\n                self.device()\n                    .alloc_uninit(kernel_l.shape(), kernel.dtype())?\n            };\n            kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?;\n            let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset())\n                .transpose(1, 2)?\n                .broadcast_as((b, k, n))?;\n            col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)?\n        };\n...\n", "needle_token_start": 2383, "needle_token_end": 2499, "code_context_ntokens": 16312, "output": ["```rust\nfn striding_error(&self, lhs_l: &Layout, rhs_l: &Layout, msg: &'static str) -> Error {\n    Error::MatMulUnexpectedStriding(Box::new(crate::error::MatMulUnexpectedStriding {\n        lhs_l: lhs_l.clone(),\n        rhs_l: rhs_l.clone(),\n        bmnk: self.0,\n        msg,\n    }))\n    .bt()\n}\n```"]}
{"repo": "huggingface/candle", "name": "erf_inv_impl", "language": "rust", "path": "candle-core/src/cpu/erf.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function calculates the inverse of the error function, which is useful in statistics and probability for deriving values from cumulative distribution functions of the normal distribution.\n2. **Input**: Three floating-point numbers representing intermediate calculations based on the probability value for which the inverse error function is needed.\n3. **Output**: A floating-point number that is the inverse error function result of the input probability.\n4. **Procedure**: The function first determines which segment of the input range the primary input falls into. Depending on this range, different polynomial approximations and mathematical transformations are applied to compute the result. The function uses a combination of logarithmic transformations, square roots, and polynomial evaluations to compute the inverse error function value accurately across different segments of the input range.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    -0.00222426529213447927281,\n    0.0185573306514231072324,\n    0.00950804701325919603619,\n    0.00187123492819559223345,\n    0.000157544617424960554631,\n    0.460469890584317994083e-5,\n    -0.230404776911882601748e-9,\n    0.266339227425782031962e-11,\n];\n\n/// Polynomial coefficients for a denominator of `erf_inv_impl`\n/// in the interval [0.75, 1] with x between 3 and 6.\nconst ERF_INV_IMPL_DD: &[f64] = &[\n    1.0,\n    1.3653349817554063097,\n    0.762059164553623404043,\n    0.220091105764131249824,\n    0.0341589143670947727934,\n    0.00263861676657015992959,\n    0.764675292302794483503e-4,\n];\n\n/// Polynomial coefficients for a numerator of `erf_inv_impl`\n/// in the interval [0.75, 1] with x between 6 and 18.\nconst ERF_INV_IMPL_EN: &[f64] = &[\n    -0.0167431005076633737133,\n    -0.00112951438745580278863,\n    0.00105628862152492910091,\n    0.000209386317487588078668,\n    0.149624783758342370182e-4,\n    0.449696789927706453732e-6,\n    0.462596163522878599135e-8,\n    -0.281128735628831791805e-13,\n    0.99055709973310326855e-16,\n];\n\n/// Polynomial coefficients for a denominator of `erf_inv_impl`\n/// in the interval [0.75, 1] with x between 6 and 18.\nconst ERF_INV_IMPL_ED: &[f64] = &[\n    1.0,\n    0.591429344886417493481,\n    0.138151865749083321638,\n    0.0160746087093676504695,\n    0.000964011807005165528527,\n    0.275335474764726041141e-4,\n    0.282243172016108031869e-6,\n];\n\n/// Polynomial coefficients for a numerator of `erf_inv_impl`\n/// in the interval [0.75, 1] with x between 18 and 44.\nconst ERF_INV_IMPL_FN: &[f64] = &[\n    -0.0024978212791898131227,\n    -0.779190719229053954292e-5,\n    0.254723037413027451751e-4,\n    0.162397777342510920873e-5,\n    0.396341011304801168516e-7,\n    0.411632831190944208473e-9,\n    0.145596286718675035587e-11,\n    -0.116765012397184275695e-17,\n];\n\n/// Polynomial coefficients for a denominator of `erf_inv_impl`\n/// in the interval [0.75, 1] with x between 18 and 44.\nconst ERF_INV_IMPL_FD: &[f64] = &[\n    1.0,\n    0.207123112214422517181,\n    0.0169410838120975906478,\n    0.000690538265622684595676,\n    0.145007359818232637924e-4,\n    0.144437756628144157666e-6,\n    0.509761276599778486139e-9,\n];\n\n/// Polynomial coefficients for a numerator of `erf_inv_impl`\n/// in the interval [0.75, 1] with x greater than 44.\nconst ERF_INV_IMPL_GN: &[f64] = &[\n    -0.000539042911019078575891,\n    -0.28398759004727721098e-6,\n    0.899465114892291446442e-6,\n    0.229345859265920864296e-7,\n    0.225561444863500149219e-9,\n    0.947846627503022684216e-12,\n    0.135880130108924861008e-14,\n    -0.348890393399948882918e-21,\n];\n\n/// Polynomial coefficients for a denominator of `erf_inv_impl`\n/// in the interval [0.75, 1] with x greater than 44.\nconst ERF_INV_IMPL_GD: &[f64] = &[\n    1.0,\n    0.0845746234001899436914,\n    0.00282092984726264681981,\n    0.468292921940894236786e-4,\n    0.399968812193862100054e-6,\n    0.161809290887904476097e-8,\n    0.231558608310259605225e-11,\n];\n\n/// `erf_impl` computes the error function at `z`.\n/// If `inv` is true, `1 - erf` is calculated as opposed to `erf`\nfn erf_impl(z: f64, inv: bool) -> f64 {\n    if z < 0.0 {\n        if !inv {\n            return -erf_impl(-z, false);\n        }\n        if z < -0.5 {\n            return 2.0 - erf_impl(-z, true);\n        }\n        return 1.0 + erf_impl(-z, false);\n    }\n\n    let result = if z < 0.5 {\n        if z < 1e-10 {\n            z * 1.125 + z * 0.003379167095512573896158903121545171688\n        } else {\n            z * 1.125\n                + z * evaluate::polynomial(z, ERF_IMPL_AN) / evaluate::polynomial(z, ERF_IMPL_AD)\n        }\n    } else if z < 110.0 {\n        let (r, b) = if z < 0.75 {\n            (\n                evaluate::polynomial(z - 0.5, ERF_IMPL_BN)\n                    / evaluate::polynomial(z - 0.5, ERF_IMPL_BD),\n                0.3440242112,\n            )\n        } else if z < 1.25 {\n            (\n                evaluate::polynomial(z - 0.75, ERF_IMPL_CN)\n                    / evaluate::polynomial(z - 0.75, ERF_IMPL_CD),\n                0.419990927,\n            )\n        } else if z < 2.25 {\n            (\n                evaluate::polynomial(z - 1.25, ERF_IMPL_DN)\n                    / evaluate::polynomial(z - 1.25, ERF_IMPL_DD),\n                0.4898625016,\n            )\n        } else if z < 3.5 {\n            (\n                evaluate::polynomial(z - 2.25, ERF_IMPL_EN)\n                    / evaluate::polynomial(z - 2.25, ERF_IMPL_ED),\n                0.5317370892,\n            )\n        } else if z < 5.25 {\n            (\n                evaluate::polynomial(z - 3.5, ERF_IMPL_FN)\n                    / evaluate::polynomial(z - 3.5, ERF_IMPL_FD),\n                0.5489973426,\n            )\n        } else if z < 8.0 {\n            (\n                evaluate::polynomial(z - 5.25, ERF_IMPL_GN)\n                    / evaluate::polynomial(z - 5.25, ERF_IMPL_GD),\n                0.5571740866,\n            )\n        } else if z < 11.5 {\n            (\n                evaluate::polynomial(z - 8.0, ERF_IMPL_HN)\n                    / evaluate::polynomial(z - 8.0, ERF_IMPL_HD),\n                0.5609807968,\n            )\n        } else if z < 17.0 {\n            (\n                evaluate::polynomial(z - 11.5, ERF_IMPL_IN)\n                    / evaluate::polynomial(z - 11.5, ERF_IMPL_ID),\n                0.5626493692,\n            )\n        } else if z < 24.0 {\n            (\n                evaluate::polynomial(z - 17.0, ERF_IMPL_JN)\n                    / evaluate::polynomial(z - 17.0, ERF_IMPL_JD),\n                0.5634598136,\n            )\n        } else if z < 38.0 {\n            (\n                evaluate::polynomial(z - 24.0, ERF_IMPL_KN)\n                    / evaluate::polynomial(z - 24.0, ERF_IMPL_KD),\n                0.5638477802,\n            )\n        } else if z < 60.0 {\n            (\n                evaluate::polynomial(z - 38.0, ERF_IMPL_LN)\n                    / evaluate::polynomial(z - 38.0, ERF_IMPL_LD),\n                0.5640528202,\n            )\n        } else if z < 85.0 {\n            (\n                evaluate::polynomial(z - 60.0, ERF_IMPL_MN)\n                    / evaluate::polynomial(z - 60.0, ERF_IMPL_MD),\n                0.5641309023,\n            )\n        } else {\n            (\n                evaluate::polynomial(z - 85.0, ERF_IMPL_NN)\n                    / evaluate::polynomial(z - 85.0, ERF_IMPL_ND),\n                0.5641584396,\n            )\n        };\n        let g = (-z * z).exp() / z;\n        g * b + g * r\n    } else {\n        0.0\n    };\n\n    if inv && z >= 0.5 {\n        result\n    } else if z >= 0.5 || inv {\n        1.0 - result\n    } else {\n        result\n    }\n}\n\n// `erf_inv_impl` computes the inverse error function where\n// `p`,`q`, and `s` are the first, second, and third intermediate\n// parameters respectively\n\nfn erf_inv_impl(p: f64, q: f64, s: f64) -> f64 {\n    let result = if p <= 0.5 {\n        let y = 0.0891314744949340820313;\n        let g = p * (p + 10.0);\n        let r = evaluate::polynomial(p, ERF_INV_IMPL_AN) / evaluate::polynomial(p, ERF_INV_IMPL_AD);\n        g * y + g * r\n    } else if q >= 0.25 {\n        let y = 2.249481201171875;\n        let g = (-2.0 * q.ln()).sqrt();\n        let xs = q - 0.25;\n        let r =\n            evaluate::polynomial(xs, ERF_INV_IMPL_BN) / evaluate::polynomial(xs, ERF_INV_IMPL_BD);\n        g / (y + r)\n    } else {\n        let x = (-q.ln()).sqrt();\n        if x < 3.0 {\n            let y = 0.807220458984375;\n            let xs = x - 1.125;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_CN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_CD);\n            y * x + r * x\n        } else if x < 6.0 {\n            let y = 0.93995571136474609375;\n            let xs = x - 3.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_DN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_DD);\n            y * x + r * x\n        } else if x < 18.0 {\n            let y = 0.98362827301025390625;\n            let xs = x - 6.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_EN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_ED);\n            y * x + r * x\n        } else if x < 44.0 {\n            let y = 0.99714565277099609375;\n            let xs = x - 18.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_FN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_FD);\n            y * x + r * x\n        } else {\n            let y = 0.99941349029541015625;\n            let xs = x - 44.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_GN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_GD);\n            y * x + r * x\n        }\n    };\n    s * result\n}\n\n// Path: candle-core/src/cpu/mod.rs\npub mod erf;\npub mod kernels;\n\ntrait Cpu<const ARR: usize> {\n    type Unit;\n    type Array;\n    const STEP: usize;\n    const EPR: usize;\n\n    fn n() -> usize;\n    unsafe fn zero() -> Self::Unit;\n    unsafe fn zero_array() -> Self::Array;\n    unsafe fn load(mem_addr: *const f32) -> Self::Unit;\n    unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit;\n    unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit;\n    unsafe fn vec_reduce(x: Self::Array, y: *mut f32);\n    unsafe fn from_f32(v: f32) -> Self::Unit;\n    unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit);\n}\n\ntrait CpuF16<const ARR: usize> {\n    type Unit;\n    type Array;\n    const STEP: usize;\n    const EPR: usize;\n\n    fn n() -> usize;\n    unsafe fn zero() -> Self::Unit;\n    unsafe fn zero_array() -> Self::Array;\n    unsafe fn load(mem_addr: *const f16) -> Self::Unit;\n    unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit;\n    unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit;\n    unsafe fn vec_reduce(x: Self::Array, y: *mut f32);\n    unsafe fn from_f32(v: f32) -> Self::Unit;\n    unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit);\n}\nuse half::f16;\n\n#[cfg(any(target_arch = \"x86\", target_arch = \"x86_64\"))]\n#[cfg(target_feature = \"avx\")]\npub mod avx;\n#[cfg(any(target_arch = \"x86\", target_arch = \"x86_64\"))]\n#[cfg(target_feature = \"avx\")]\npub use avx::{CurrentCpu, CurrentCpuF16};\n\n#[cfg(target_arch = \"wasm32\")]\n#[cfg(target_feature = \"simd128\")]\npub mod simd128;\n#[cfg(target_arch = \"wasm32\")]\n#[cfg(target_feature = \"simd128\")]\npub use simd128::CurrentCpu;\n\n#[cfg(any(target_arch = \"arm\", target_arch = \"aarch64\"))]\n#[cfg(target_feature = \"neon\")]\npub mod neon;\n#[cfg(any(target_arch = \"arm\", target_arch = \"aarch64\"))]\n#[cfg(target_feature = \"neon\")]\npub use neon::CurrentCpu;\n\n#[cfg(any(\n    target_feature = \"neon\",\n    target_feature = \"avx\",\n    target_feature = \"simd128\"\n))]\n#[inline(always)]\npub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) {\n    let np = k & !(CurrentCpu::STEP - 1);\n\n    let mut sum = CurrentCpu::zero_array();\n    let mut ax = CurrentCpu::zero_array();\n    let mut ay = CurrentCpu::zero_array();\n\n    for i in (0..np).step_by(CurrentCpu::STEP) {\n        for j in 0..CurrentCpu::n() {\n            ax[j] = CurrentCpu::load(a_row.add(i + j * CurrentCpu::EPR));\n            ay[j] = CurrentCpu::load(b_row.add(i + j * CurrentCpu::EPR));\n\n            sum[j] = CurrentCpu::vec_fma(sum[j], ax[j], ay[j]);\n        }\n    }\n\n    CurrentCpu::vec_reduce(sum, c);\n\n    // leftovers\n    for i in np..k {\n        *c += *a_row.add(i) * (*b_row.add(i));\n    }\n}\n\n#[cfg(not(any(\n    target_feature = \"neon\",\n    target_feature = \"avx\",\n    target_feature = \"simd128\"\n)))]\n#[inline(always)]\npub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) {\n    // leftovers\n    for i in 0..k {\n        *c += *a_row.add(i) * (*b_row.add(i));\n    }\n}\n\n#[cfg(any(\n    target_feature = \"neon\",\n    target_feature = \"avx\",\n    target_feature = \"simd128\"\n))]\n#[inline(always)]\npub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) {\n    let np = k & !(CurrentCpu::STEP - 1);\n\n    let mut sum = CurrentCpu::zero_array();\n    let mut x = CurrentCpu::zero_array();\n\n    for i in (0..np).step_by(CurrentCpu::STEP) {\n        for j in 0..CurrentCpu::n() {\n            x[j] = CurrentCpu::load(row.add(i + j * CurrentCpu::EPR));\n            sum[j] = CurrentCpu::vec_add(sum[j], x[j]);\n        }\n    }\n\n    CurrentCpu::vec_reduce(sum, b);\n\n    // leftovers\n    for i in np..k {\n        *b += *row.add(i)\n    }\n}\n\n#[cfg(not(any(\n    target_feature = \"neon\",\n    target_feature = \"avx\",\n    target_feature = \"simd128\"\n)))]\n#[inline(always)]\npub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) {\n    *b = 0f32;\n    for i in 0..k {\n        *b += *row.add(i)\n    }\n}\n\n#[cfg(target_feature = \"avx\")]\n#[inline(always)]\npub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) {\n    let mut sumf = 0.0f32;\n    let np = k & !(CurrentCpuF16::STEP - 1);\n\n    let mut sum = CurrentCpuF16::zero_array();\n    let mut ax = CurrentCpuF16::zero_array();\n    let mut ay = CurrentCpuF16::zero_array();\n\n    for i in (0..np).step_by(CurrentCpuF16::STEP) {\n        for j in 0..CurrentCpuF16::n() {\n            ax[j] = CurrentCpuF16::load(a_row.add(i + j * CurrentCpuF16::EPR));\n            ay[j] = CurrentCpuF16::load(b_row.add(i + j * CurrentCpuF16::EPR));\n\n            sum[j] = CurrentCpuF16::vec_fma(sum[j], ax[j], ay[j]);\n        }\n    }\n\n    CurrentCpuF16::vec_reduce(sum, &mut sumf);\n\n    // leftovers\n    for i in np..k {\n        sumf += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32();\n    }\n    *c = sumf;\n}\n\n#[cfg(not(target_feature = \"avx\"))]\n#[inline(always)]\npub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) {\n    // leftovers\n    let mut sum = 0.0;\n    for i in 0..k {\n        sum += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32();\n    }\n    *c = sum;\n}\n\n// Path: candle-core/src/lib.rs\n//! ML framework for Rust\n//!\n//! ```rust\n//! use candle_core::{Tensor, DType, Device};\n//! # use candle_core::Error;\n//! # fn main() -> Result<(), Error>{\n//!\n//! let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?;\n//! let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?;\n//!\n//! let c = a.matmul(&b)?;\n//! # Ok(())}\n//! ```\n//!\n//! ## Features\n//!\n//! - Simple syntax (looks and feels like PyTorch)\n//! - CPU and Cuda backends (and M1 support)\n//! - Enable serverless (CPU) small and fast deployments\n//! - Model training\n//! - Distributed computing (NCCL).\n//! - Models out of the box (Llama, Whisper, Falcon, ...)\n//!\n//! ## FAQ\n//!\n//! - Why Candle?\n//!\n//! Candle stems from the need to reduce binary size in order to *enable serverless*\n//! possible by making the whole engine smaller than PyTorch very large library volume\n//!\n//! And simply *removing Python* from production workloads.\n//! Python can really add overhead in more complex workflows and the [GIL](https://www.backblaze.com/blog/the-python-gil-past-present-and-future/) is a notorious source of headaches.\n//!\n//! Rust is cool, and a lot of the HF ecosystem already has Rust crates [safetensors](https://github.com/huggingface/safetensors) and [tokenizers](https://github.com/huggingface/tokenizers)\n\n#[cfg(feature = \"accelerate\")]\nmod accelerate;\npub mod backend;\npub mod backprop;\npub mod conv;\nmod convert;\npub mod cpu;\npub mod cpu_backend;\n#[cfg(feature = \"cuda\")]\npub mod cuda_backend;\nmod custom_op;\nmod device;\npub mod display;\nmod dtype;\nmod dummy_cuda_backend;\nmod dummy_metal_backend;\npub mod error;\nmod indexer;\npub mod layout;\n#[cfg(feature = \"metal\")]\npub mod metal_backend;\n#[cfg(feature = \"mkl\")]\nmod mkl;\npub mod npy;\npub mod op;\npub mod pickle;\npub mod quantized;\npub mod safetensors;\npub mod scalar;\npub mod shape;\nmod storage;\nmod strided_index;\nmod tensor;\nmod tensor_cat;\npub mod test_utils;\npub mod utils;\nmod variable;\n\n#[cfg(feature = \"cudnn\")]\npub use cuda_backend::cudnn;\n\npub use cpu_backend::CpuStorage;\npub use custom_op::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3};\npub use device::{Device, DeviceLocation, NdArray};\npub use dtype::{DType, DTypeParseError, FloatDType, IntDType, WithDType};\npub use error::{Error, Result};\npub use indexer::IndexOp;\npub use layout::Layout;\npub use shape::{Shape, D};\npub use storage::Storage;\npub use strided_index::{StridedBlocks, StridedIndex};\npub use tensor::{Tensor, TensorId};\npub use variable::Var;\n\n#[cfg(feature = \"cuda\")]\npub use cuda_backend::{CudaDevice, CudaStorage};\n\n#[cfg(not(feature = \"cuda\"))]\npub use dummy_cuda_backend::{CudaDevice, CudaStorage};\n\n#[cfg(feature = \"metal\")]\npub use metal_backend::{MetalDevice, MetalError, MetalStorage};\n\n#[cfg(not(feature = \"metal\"))]\npub use dummy_metal_backend::{MetalDevice, MetalError, MetalStorage};\n\n#[cfg(feature = \"mkl\")]\nextern crate intel_mkl_src;\n\n#[cfg(feature = \"accelerate\")]\nextern crate accelerate_src;\n\npub trait ToUsize2 {\n    fn to_usize2(self) -> (usize, usize);\n}\n\nimpl ToUsize2 for usize {\n    fn to_usize2(self) -> (usize, usize) {\n        (self, self)\n    }\n}\n\nimpl ToUsize2 for (usize, usize) {\n    fn to_usize2(self) -> (usize, usize) {\n        self\n    }\n}\n\n// A simple trait defining a module with forward method using a single argument.\npub trait Module {\n    fn forward(&self, xs: &Tensor) -> Result<Tensor>;\n}\n\nimpl<T: Fn(&Tensor) -> Result<Tensor>> Module for T {\n    fn forward(&self, xs: &Tensor) -> Result<Tensor> {\n        self(xs)\n    }\n}\n\nimpl<M: Module> Module for Option<&M> {\n    fn forward(&self, xs: &Tensor) -> Result<Tensor> {\n        match self {\n            None => Ok(xs.clone()),\n            Some(m) => m.forward(xs),\n        }\n    }\n}\n\n// A trait defining a module with forward method using a single tensor argument and a flag to\n// separate the training and evaluation behaviors.\npub trait ModuleT {\n    fn forward_t(&self, xs: &Tensor, train: bool) -> Result<Tensor>;\n}\n\nimpl<M: Module> ModuleT for M {\n    fn forward_t(&self, xs: &Tensor, _train: bool) -> Result<Tensor> {\n        self.forward(xs)\n    }\n}\n\n", "needle_token_start": 3648, "needle_token_end": 4477, "code_context_ntokens": 8216, "output": ["```rust\n/// `erf_inv_impl` computes the inverse error function where\n/// `p`,`q`, and `s` are the first, second, and third intermediate\n/// parameters respectively\nfn erf_inv_impl(p: f64, q: f64, s: f64) -> f64 {\n    let result = if p <= 0.5 {\n        let y = 0.0891314744949340820313;\n        let g = p * (p + 10.0);\n        let r = evaluate::polynomial(p, ERF_INV_IMPL_AN) / evaluate::polynomial(p, ERF_INV_IMPL_AD);\n        g * y + g * r\n    } else if q >= 0.25 {\n        let y = 2.249481201171875;\n        let g = (-2.0 * q.ln()).sqrt();\n        let xs = q - 0.25;\n        let r =\n            evaluate::polynomial(xs, ERF_INV_IMPL_BN) / evaluate::polynomial(xs, ERF_INV_IMPL_BD);\n        g / (y + r)\n    } else {\n        let x = (-q.ln()).sqrt();\n        if x < 3.0 {\n            let y = 0.807220458984375;\n            let xs = x - 1.125;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_CN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_CD);\n            y * x + r * x\n        } else if x < 6.0 {\n            let y = 0.93995571136474609375;\n            let xs = x - 3.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_DN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_DD);\n            y * x + r * x\n        } else if x < 18.0 {\n            let y = 0.98362827301025390625;\n            let xs = x - 6.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_EN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_ED);\n            y * x + r * x\n        } else if x < 44.0 {\n            let y = 0.99714565277099609375;\n            let xs = x - 18.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_FN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_FD);\n            y * x + r * x\n        } else {\n            let y = 0.99941349029541015625;\n            let xs = x - 44.0;\n            let r = evaluate::polynomial(xs, ERF_INV_IMPL_GN)\n                / evaluate::polynomial(xs, ERF_INV_IMPL_GD);\n            y * x + r * x\n        }\n    };\n    s * result\n}\n```"]}
{"repo": "huggingface/candle", "name": "index", "language": "rust", "path": "candle-core/src/indexer.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve a specific subset of elements from a tensor based on various indexing methods.\n2. **Input**: The function accepts a tensor and a list of indexers, which define how to slice or select elements from the tensor.\n3. **Output**: It returns a new tensor that contains the selected subset of elements from the original tensor.\n4. **Procedure**: The function iterates through each indexer in the list, applying the specified selection or slicing operation to the tensor. This involves narrowing down the tensor to specific dimensions or selecting specific indices, and potentially squeezing dimensions to reduce the rank of the tensor if necessary.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: candle-core/src/storage.rs\nuse crate::backend::BackendStorage;\nuse crate::op::{self, CmpOp, ReduceOp};\nuse crate::{CpuStorage, CudaStorage, DType, Device, Error, Layout, MetalStorage, Result, Shape};\nuse crate::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3};\n\n// We do not want to implement Clone on Storage as cloning may fail because of\n// out of memory. Instead try_clone should be used.\n#[derive(Debug)]\npub enum Storage {\n    Cpu(CpuStorage),\n    Cuda(CudaStorage),\n    Metal(MetalStorage),\n}\n\nimpl Storage {\n    pub fn try_clone(&self, layout: &Layout) -> Result<Self> {\n        match self {\n            Self::Cpu(storage) => Ok(Self::Cpu(storage.clone())),\n            Self::Cuda(storage) => {\n                let storage = storage.try_clone(layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.try_clone(layout)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub fn device(&self) -> Device {\n        match self {\n            Self::Cpu(_) => Device::Cpu,\n            Self::Cuda(storage) => Device::Cuda(storage.device().clone()),\n            Self::Metal(storage) => Device::Metal(storage.device().clone()),\n        }\n    }\n\n    pub fn dtype(&self) -> DType {\n        match self {\n            Self::Cpu(storage) => storage.dtype(),\n            Self::Cuda(storage) => storage.dtype(),\n            Self::Metal(storage) => storage.dtype(),\n        }\n    }\n\n    pub(crate) fn same_device(&self, rhs: &Self, op: &'static str) -> Result<()> {\n        let lhs_device = self.device();\n        let rhs_device = rhs.device();\n        let lhs = lhs_device.location();\n        let rhs = rhs_device.location();\n        let same_device = if self.device().is_metal() {\n            // On metal, we require the device to be exactly the same rather than\n            // having the same location. In cuda this is not necessary as all CudaDevice on the\n            // same GPU will use the same cuda stream.\n            lhs_device.same_device(&rhs_device)\n        } else {\n            lhs == rhs\n        };\n        if !same_device {\n            Err(Error::DeviceMismatchBinaryOp { lhs, rhs, op }.bt())\n        } else {\n            Ok(())\n        }\n    }\n\n    pub(crate) fn same_dtype(&self, rhs: &Self, op: &'static str) -> Result<()> {\n        let lhs = self.dtype();\n        let rhs = rhs.dtype();\n        if lhs != rhs {\n            Err(Error::DTypeMismatchBinaryOp { lhs, rhs, op }.bt())\n        } else {\n            Ok(())\n        }\n    }\n\n    pub(crate) fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn powf(&self, layout: &Layout, alpha: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn elu(&self, layout: &Layout, alpha: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn cmp(\n        &self,\n        op: CmpOp,\n        rhs: &Self,\n        lhs_layout: &Layout,\n        rhs_layout: &Layout,\n    ) -> Result<Self> {\n        self.same_device(rhs, \"cmp\")?;\n        self.same_dtype(rhs, \"cmp\")?;\n        match (self, rhs) {\n            (Storage::Cpu(lhs), Storage::Cpu(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cpu(storage))\n            }\n            (Self::Cuda(lhs), Self::Cuda(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            (Self::Metal(lhs), Self::Metal(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Metal(storage))\n            }\n            (lhs, rhs) => {\n                // Should not happen because of the same device check above but we're defensive\n                // anyway.\n                Err(Error::DeviceMismatchBinaryOp {\n                    lhs: lhs.device().location(),\n                    rhs: rhs.device().location(),\n                    op: \"cmp\",\n                }\n                .bt())\n            }\n        }\n    }\n\n    pub(crate) fn reduce_op(&self, op: ReduceOp, layout: &Layout, s: &[usize]) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn apply_op1(&self, l: &Layout, c: &dyn CustomOp1) -> Result<(Self, Shape)> {\n        match self {\n            Self::Cpu(storage) => {\n                let (storage, shape) = c.cpu_fwd(storage, l)?;\n                Ok((Self::Cpu(storage), shape))\n            }\n            Self::Cuda(storage) => {\n                let (storage, shape) = c.cuda_fwd(storage, l)?;\n                Ok((Self::Cuda(storage), shape))\n            }\n            Self::Metal(storage) => {\n                let (storage, shape) = c.metal_fwd(storage, l)?;\n                Ok((Self::Metal(storage), shape))\n            }\n        }\n    }\n\n    pub(crate) fn apply_op2(\n        &self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        c: &dyn CustomOp2,\n    ) -> Result<(Self, Shape)> {\n        self.same_device(t2, c.name())?;\n        match (self, t2) {\n            (Self::Cpu(s1), Self::Cpu(s2)) => {\n                let (s, shape) = c.cpu_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Cpu(s), shape))\n            }\n            (Self::Cuda(s1), Self::Cuda(s2)) => {\n                let (s, shape) = c.cuda_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Cuda(s), shape))\n            }\n            (Self::Metal(s1), Self::Metal(s2)) => {\n                let (s, shape) = c.metal_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Metal(s), shape))\n            }\n            _ => unreachable!(),\n        }\n    }\n\n    pub(crate) fn apply_op3(\n        &self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        t3: &Self,\n        l3: &Layout,\n        c: &dyn CustomOp3,\n    ) -> Result<(Self, Shape)> {\n        self.same_device(t2, c.name())?;\n        self.same_device(t3, c.name())?;\n        match (self, t2, t3) {\n            (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => {\n                let (s, shape) = c.cpu_fwd(s1, l1, s2, l2, s3, l3)?;\n                Ok((Self::Cpu(s), shape))\n            }\n            (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => {\n                let (s, shape) = c.cuda_fwd(s1, l1, s2, l2, s3, l3)?;\n                Ok((Self::Cuda(s), shape))\n            }\n            (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => {\n                let (s, shape) = c.metal_fwd(s1, l1, s2, l2, s3, l3)?;\n                Ok((Self::Metal(s), shape))\n            }\n            _ => unreachable!(),\n        }\n    }\n\n    pub(crate) fn inplace_op1(&mut self, l: &Layout, c: &dyn InplaceOp1) -> Result<()> {\n        match self {\n            Self::Cpu(storage) => c.cpu_fwd(storage, l),\n            Self::Cuda(storage) => c.cuda_fwd(storage, l),\n            Self::Metal(storage) => c.metal_fwd(storage, l),\n        }\n    }\n\n    pub(crate) fn inplace_op2(\n        &mut self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        c: &dyn InplaceOp2,\n    ) -> Result<()> {\n        self.same_device(t2, c.name())?;\n        match (self, t2) {\n            (Self::Cpu(s1), Self::Cpu(s2)) => c.cpu_fwd(s1, l1, s2, l2),\n            (Self::Cuda(s1), Self::Cuda(s2)) => c.cuda_fwd(s1, l1, s2, l2),\n            (Self::Metal(s1), Self::Metal(s2)) => c.metal_fwd(s1, l1, s2, l2),\n            _ => unreachable!(),\n        }\n    }\n\n    pub(crate) fn inplace_op3(\n        &mut self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        t3: &Self,\n        l3: &Layout,\n        c: &dyn InplaceOp3,\n    ) -> Result<()> {\n        self.same_device(t2, c.name())?;\n        self.same_device(t3, c.name())?;\n        match (self, t2, t3) {\n            (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => c.cpu_fwd(s1, l1, s2, l2, s3, l3),\n            (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => c.cuda_fwd(s1, l1, s2, l2, s3, l3),\n            (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => {\n                c.metal_fwd(s1, l1, s2, l2, s3, l3)\n            }\n            _ => unreachable!(),\n        }\n    }\n\n    pub(crate) fn unary_impl<B: op::UnaryOpT>(&self, layout: &Layout) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.unary_impl::<B>(layout)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.unary_impl::<B>(layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.unary_impl::<B>(layout)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn binary_impl<B: op::BinaryOpT>(\n        &self,\n        rhs: &Self,\n        lhs_layout: &Layout,\n        rhs_layout: &Layout,\n    ) -> Result<Self> {\n        self.same_device(rhs, B::NAME)?;\n        self.same_dtype(rhs, B::NAME)?;\n        match (self, rhs) {\n            (Storage::Cpu(lhs), Storage::Cpu(rhs)) => {\n                let storage = lhs.binary_impl::<B>(rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cpu(storage))\n            }\n            (Self::Cuda(lhs), Self::Cuda(rhs)) => {\n                let storage = lhs.binary_impl::<B>(rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            (Self::Metal(lhs), Self::Metal(rhs)) => {\n                let storage = lhs.binary_impl::<B>(rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Metal(storage))\n            }\n            (lhs, rhs) => {\n                // Should not happen because of the same device check above but we're defensive\n                // anyway.\n                Err(Error::DeviceMismatchBinaryOp {\n                    lhs: lhs.device().location(),\n                    rhs: rhs.device().location(),\n                    op: B::NAME,\n                }\n                .bt())\n            }\n        }\n    }\n\n    pub(crate) fn conv1d(\n        &self,\n        l: &Layout,\n        kernel: &Self,\n        kernel_l: &Layout,\n        params: &crate::conv::ParamsConv1D,\n    ) -> Result<Self> {\n        self.same_device(kernel, \"conv1d\")?;\n        self.same_dtype(kernel, \"conv1d\")?;\n        match (self, &kernel) {\n            (Storage::Cpu(inp), Storage::Cpu(kernel)) => {\n                let s = inp.conv1d(l, kernel, kernel_l, params)?;\n                Ok(Self::Cpu(s))\n            }\n            (Storage::Cuda(inp), Storage::Cuda(kernel)) => {\n                let s = inp.conv1d(l, kernel, kernel_l, params)?;\n                Ok(Self::Cuda(s))\n            }\n            (Storage::Metal(inp), Storage::Metal(kernel)) => {\n                let s = inp.conv1d(l, kernel, kernel_l, params)?;\n                Ok(Self::Metal(s))\n            }\n            (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp {\n                lhs: lhs.device().location(),\n                rhs: rhs.device().location(),\n                op: \"conv1d\",\n            }\n            .bt()),\n        }\n    }\n\n    pub(crate) fn conv_transpose1d(\n        &self,\n        l: &Layout,\n        kernel: &Self,\n        kernel_l: &Layout,\n        params: &crate::conv::ParamsConvTranspose1D,\n    ) -> Result<Self> {\n        self.same_device(kernel, \"conv-transpose1d\")?;\n        self.same_dtype(kernel, \"conv-transpose1d\")?;\n        match (self, &kernel) {\n            (Storage::Cpu(inp), Storage::Cpu(kernel)) => {\n                let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?;\n                Ok(Self::Cpu(s))\n            }\n            (Storage::Cuda(inp), Storage::Cuda(kernel)) => {\n                let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?;\n                Ok(Self::Cuda(s))\n            }\n            (Storage::Metal(inp), Storage::Metal(kernel)) => {\n                let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?;\n                Ok(Self::Metal(s))\n            }\n            (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp {\n                lhs: lhs.device().location(),\n                rhs: rhs.device().location(),\n                op: \"conv-transpose1d\",\n            }\n            .bt()),\n        }\n    }\n\n    pub(crate) fn conv2d(\n        &self,\n        l: &Layout,\n        kernel: &Self,\n        kernel_l: &Layout,\n        params: &crate::conv::ParamsConv2D,\n    ) -> Result<Self> {\n...\n// Path: candle-core/src/scalar.rs\nuse crate::{Result, Tensor, WithDType};\n\npub enum TensorScalar {\n    Tensor(Tensor),\n    Scalar(Tensor),\n}\n\npub trait TensorOrScalar {\n    fn to_tensor_scalar(self) -> Result<TensorScalar>;\n}\n\nimpl TensorOrScalar for &Tensor {\n    fn to_tensor_scalar(self) -> Result<TensorScalar> {\n        Ok(TensorScalar::Tensor(self.clone()))\n    }\n}\n\nimpl<T: WithDType> TensorOrScalar for T {\n    fn to_tensor_scalar(self) -> Result<TensorScalar> {\n        let scalar = Tensor::new(self, &crate::Device::Cpu)?;\n        Ok(TensorScalar::Scalar(scalar))\n    }\n}\n\n// Path: candle-core/src/indexer.rs\nuse crate::{Error, Tensor};\nuse std::ops::{\n    Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive,\n};\n\nimpl Tensor {\n    /// Intended to be use by the trait `.i()`\n    ///\n    /// ```\n    /// # use candle_core::{Tensor, DType, Device, IndexOp};\n    /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?;\n    ///\n    /// let c = a.i(0..1)?;\n    /// assert_eq!(c.shape().dims(), &[1, 3]);\n    ///\n    /// let c = a.i(0)?;\n    /// assert_eq!(c.shape().dims(), &[3]);\n    ///\n    /// let c = a.i((.., ..2) )?;\n    /// assert_eq!(c.shape().dims(), &[2, 2]);\n    ///\n    /// let c = a.i((.., ..=2))?;\n    /// assert_eq!(c.shape().dims(), &[2, 3]);\n    ///\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    \nfn index(&self, indexers: &[TensorIndexer]) -> Result<Self, Error> {\n        let mut x = self.clone();\n        let dims = self.shape().dims();\n        let mut current_dim = 0;\n        for (i, indexer) in indexers.iter().enumerate() {\n            x = match indexer {\n                TensorIndexer::Select(n) => x.narrow(current_dim, *n, 1)?.squeeze(current_dim)?,\n                TensorIndexer::Narrow(left_bound, right_bound) => {\n                    let start = match left_bound {\n                        Bound::Included(n) => *n,\n                        Bound::Excluded(n) => *n + 1,\n                        Bound::Unbounded => 0,\n                    };\n                    let stop = match right_bound {\n                        Bound::Included(n) => *n + 1,\n                        Bound::Excluded(n) => *n,\n                        Bound::Unbounded => dims[i],\n                    };\n                    let out = x.narrow(current_dim, start, stop.saturating_sub(start))?;\n                    current_dim += 1;\n                    out\n                }\n                TensorIndexer::IndexSelect(indexes) => {\n                    if indexes.rank() != 1 {\n                        crate::bail!(\"multi-dimensional tensor indexing is not supported\")\n                    }\n                    let out = x.index_select(&indexes.to_device(x.device())?, current_dim)?;\n                    current_dim += 1;\n                    out\n                }\n                TensorIndexer::Err(e) => crate::bail!(\"indexing error {e:?}\"),\n            };\n        }\n        Ok(x)\n    }\n}\n\n#[derive(Debug)]\n/// Generic structure used to index a slice of the tensor\npub enum TensorIndexer {\n    /// This selects the elements for which an index has some specific value.\n    Select(usize),\n    /// This is a regular slice, purely indexing a chunk of the tensor\n    Narrow(Bound<usize>, Bound<usize>),\n    /// Indexing via a 1d tensor\n    IndexSelect(Tensor),\n    Err(Error),\n}\n\nimpl From<usize> for TensorIndexer {\n    fn from(index: usize) -> Self {\n        TensorIndexer::Select(index)\n    }\n}\n\nimpl From<&[u32]> for TensorIndexer {\n    fn from(index: &[u32]) -> Self {\n        match Tensor::new(index, &crate::Device::Cpu) {\n            Ok(tensor) => TensorIndexer::IndexSelect(tensor),\n            Err(e) => TensorIndexer::Err(e),\n        }\n    }\n}\n\nimpl From<Vec<u32>> for TensorIndexer {\n    fn from(index: Vec<u32>) -> Self {\n        let len = index.len();\n        match Tensor::from_vec(index, len, &crate::Device::Cpu) {\n            Ok(tensor) => TensorIndexer::IndexSelect(tensor),\n            Err(e) => TensorIndexer::Err(e),\n        }\n    }\n}\n\nimpl From<&Tensor> for TensorIndexer {\n    fn from(tensor: &Tensor) -> Self {\n        TensorIndexer::IndexSelect(tensor.clone())\n    }\n}\n\ntrait RB: RangeBounds<usize> {}\nimpl RB for Range<usize> {}\nimpl RB for RangeFrom<usize> {}\nimpl RB for RangeFull {}\nimpl RB for RangeInclusive<usize> {}\nimpl RB for RangeTo<usize> {}\nimpl RB for RangeToInclusive<usize> {}\n\nimpl<T: RB> From<T> for TensorIndexer {\n    fn from(range: T) -> Self {\n        use std::ops::Bound::*;\n        let start = match range.start_bound() {\n            Included(idx) => Included(*idx),\n            Excluded(idx) => Excluded(*idx),\n            Unbounded => Unbounded,\n        };\n        let end = match range.end_bound() {\n            Included(idx) => Included(*idx),\n            Excluded(idx) => Excluded(*idx),\n            Unbounded => Unbounded,\n        };\n        TensorIndexer::Narrow(start, end)\n    }\n}\n\n/// Trait used to implement multiple signatures for ease of use of the slicing\n/// of a tensor\npub trait IndexOp<T> {\n    /// Returns a slicing iterator which are the chunks of data necessary to\n    /// reconstruct the desired tensor.\n    fn i(&self, index: T) -> Result<Tensor, Error>;\n}\n\nimpl<T> IndexOp<T> for Tensor\nwhere\n    T: Into<TensorIndexer>,\n{\n    fn i(&self, index: T) -> Result<Tensor, Error> {\n        self.index(&[index.into()])\n    }\n}\n\nmacro_rules! index_op_tuple {\n    ($($t:ident),+) => {\n        #[allow(non_snake_case)]\n        impl<$($t),*> IndexOp<($($t,)*)> for Tensor\n        where\n            $($t: Into<TensorIndexer>,)*\n        {\n            fn i(&self, ($($t,)*): ($($t,)*)) -> Result<Tensor, Error> {\n                self.index(&[$($t.into(),)*])\n            }\n        }\n    };\n}\nindex_op_tuple!(A);\nindex_op_tuple!(A, B);\nindex_op_tuple!(A, B, C);\nindex_op_tuple!(A, B, C, D);\nindex_op_tuple!(A, B, C, D, E);\nindex_op_tuple!(A, B, C, D, E, F);\nindex_op_tuple!(A, B, C, D, E, F, G);\n\n// Path: candle-core/src/backprop.rs\n/// Methods for backpropagation of gradients.\nuse crate::op::{BinaryOp, Op, ReduceOp, UnaryOp};\nuse crate::{Error, Result, Tensor, TensorId};\nuse std::collections::HashMap;\n\n// arg has been reduced to node via reduce_dims, expand it back to arg.\n// This has to handle keepdims.\nfn broadcast_back(arg: &Tensor, node: &Tensor, reduced_dims: &[usize]) -> Result<Tensor> {\n    if arg.rank() == node.rank() {\n        // keepdim = true\n        node.broadcast_as(arg.shape())\n    } else {\n        // keepdim = false\n        // first expand the reduced dims.\n        node.reshape(reduced_dims)?.broadcast_as(arg.shape())\n    }\n}\n\nthread_local! {\n    static CANDLE_GRAD_DO_NOT_DETACH: bool = {\n        match std::env::var(\"CANDLE_GRAD_DO_NOT_DETACH\") {\n            Ok(s) => {\n                !s.is_empty() && s != \"0\"\n            },\n            Err(_) => false,\n        }\n    }\n}\n\nimpl Tensor {\n    /// Return all the nodes that lead to this value in a topologically sorted vec, the first\n    /// elements having dependencies on the latter ones, e.g. the first element if any is the\n    /// argument.\n    /// This assumes that the op graph is a DAG.\n    fn sorted_nodes(&self) -> Vec<&Tensor> {\n        // The vec of sorted nodes is passed as an owned value rather than a mutable reference\n        // to get around some lifetime limitations.\n        fn walk<'a>(\n            node: &'a Tensor,\n            nodes: Vec<&'a Tensor>,\n            already_seen: &mut HashMap<TensorId, bool>,\n        ) -> (bool, Vec<&'a Tensor>) {\n            if let Some(&tg) = already_seen.get(&node.id()) {\n                return (tg, nodes);\n            }\n            let mut track_grad = false;\n            let mut nodes = if node.is_variable() {\n                // Do not call recursively on the \"leaf\" nodes.\n                track_grad = true;\n                nodes\n            } else if node.dtype().is_int() {\n                nodes\n            } else if let Some(op) = node.op() {\n                match op {\n                    Op::IndexAdd(t1, t2, t3, _)\n                    | Op::ScatterAdd(t1, t2, t3, _)\n                    | Op::CustomOp3(t1, t2, t3, _)\n                    | Op::WhereCond(t1, t2, t3) => {\n                        let (tg, nodes) = walk(t1, nodes, already_seen);\n                        track_grad |= tg;\n                        let (tg, nodes) = walk(t2, nodes, already_seen);\n                        track_grad |= tg;\n                        let (tg, nodes) = walk(t3, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }\n                    Op::Conv1D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::ConvTranspose1D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::Conv2D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::ConvTranspose2D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::CustomOp2(lhs, rhs, _)\n                    | Op::Binary(lhs, rhs, _)\n                    | Op::Gather(lhs, rhs, _)\n                    | Op::IndexSelect(lhs, rhs, _)\n                    | Op::Matmul(lhs, rhs)\n                    | Op::SliceScatter0(lhs, rhs, _) => {\n                        let (tg, nodes) = walk(lhs, nodes, already_seen);\n                        track_grad |= tg;\n                        let (tg, nodes) = walk(rhs, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }\n                    Op::Cat(args, _) => args.iter().fold(nodes, |nodes, arg| {\n                        let (tg, nodes) = walk(arg, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }),\n                    Op::Affine { arg, mul, .. } => {\n                        if *mul == 0. {\n                            nodes\n                        } else {\n                            let (tg, nodes) = walk(arg, nodes, already_seen);\n                            track_grad |= tg;\n                            nodes\n                        }\n                    }\n                    Op::Unary(_node, UnaryOp::Ceil)\n                    | Op::Unary(_node, UnaryOp::Floor)\n                    | Op::Unary(_node, UnaryOp::Round)\n                    | Op::Unary(_node, UnaryOp::Sign) => nodes,\n                    Op::Reshape(node)\n                    | Op::UpsampleNearest1D { arg: node, .. }\n                    | Op::UpsampleNearest2D { arg: node, .. }\n                    | Op::AvgPool2D { arg: node, .. }\n                    | Op::MaxPool2D { arg: node, .. }\n                    | Op::Copy(node)\n                    | Op::Broadcast(node)\n                    | Op::Cmp(node, _)\n                    | Op::Reduce(node, ReduceOp::Min | ReduceOp::Sum | ReduceOp::Max, _)\n                    | Op::ToDevice(node)\n                    | Op::Transpose(node, _, _)\n                    | Op::Permute(node, _)\n                    | Op::Narrow(node, _, _, _)\n                    | Op::Unary(node, _)\n                    | Op::Elu(node, _)\n                    | Op::Powf(node, _)\n                    | Op::CustomOp1(node, _) => {\n                        let (tg, nodes) = walk(node, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }\n                    Op::ToDType(node) => {\n                        if node.dtype().is_float() {\n                            let (tg, nodes) = walk(node, nodes, already_seen);\n                            track_grad |= tg;\n                            nodes\n                        } else {\n                            nodes\n                        }\n                    }\n                    Op::Reduce(_, ReduceOp::ArgMin | ReduceOp::ArgMax, _) => nodes,\n                }\n            } else {\n                nodes\n            };\n            already_seen.insert(node.id(), track_grad);\n            if track_grad {\n                nodes.push(node);\n            }\n            (track_grad, nodes)\n        }\n        let (_tg, mut nodes) = walk(self, vec![], &mut HashMap::new());\n        nodes.reverse();\n        nodes\n    }\n\n    pub fn backward(&self) -> Result<GradStore> {\n        let sorted_nodes = self.sorted_nodes();\n        let mut grads = GradStore::new();\n        grads.insert(self, self.ones_like()?.contiguous()?);\n        for node in sorted_nodes.iter() {\n            if node.is_variable() {\n                continue;\n            }\n            let grad = grads\n                .remove(node)\n                .expect(\"candle internal error - grad not populated\");\n            // https://github.com/huggingface/candle/issues/1241\n            // Ideally, we would make these operations in place where possible to ensure that we\n            // do not have to allocate too often. Here we just call `.detach` to avoid computing\n            // the backprop graph of the backprop itself. This would be an issue for second order\n            // derivatives but these are out of scope at the moment.\n            let do_not_detach = CANDLE_GRAD_DO_NOT_DETACH.with(|b| *b);\n            let grad = if do_not_detach { grad } else { grad.detach() };\n            if let Some(op) = node.op() {\n                match op {\n                    Op::Binary(lhs, rhs, BinaryOp::Add) => {\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&grad)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Sub) => {\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&grad)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.sub(&grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Mul) => {\n                        let lhs_grad = grad.mul(rhs)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n                        let rhs_grad = grad.mul(lhs)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Div) => {\n                        let lhs_grad = grad.div(rhs)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n                        let rhs_grad = grad.mul(lhs)?.div(&rhs.sqr()?)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.sub(&rhs_grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Minimum)\n                    | Op::Binary(lhs, rhs, BinaryOp::Maximum) => {\n                        let mask_lhs = node.eq(lhs)?.to_dtype(grad.dtype())?;\n                        let mask_rhs = node.eq(rhs)?.to_dtype(grad.dtype())?;\n\n                        // If both masks are 1 one the same point, we want to scale the\n                        // gradient by 0.5 rather than 1.\n                        let lhs_grad = mask_lhs.mul(&grad)?.div(&(&mask_rhs + 1.)?)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n\n                        let rhs_grad = mask_rhs.mul(&grad)?.div(&(&mask_lhs + 1.)?)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n                    }\n                    Op::WhereCond(pred, t, f) => {\n                        let zeros = grad.zeros_like()?;\n                        let t_sum_grad = grads.or_insert(t)?;\n                        let t_grad = pred.where_cond(&grad, &zeros)?;\n                        *t_sum_grad = t_sum_grad.add(&t_grad)?;\n                        let f_sum_grad = grads.or_insert(f)?;\n                        let f_grad = pred.where_cond(&zeros, &grad)?;\n                        *f_sum_grad = f_sum_grad.add(&f_grad)?;\n                    }\n                    Op::Conv1D {\n                        arg,\n                        kernel,\n                        padding,\n                        stride,\n                        dilation,\n                    } => {\n                        // The output height for conv_transpose1d is:\n                        // (l_in - 1) * stride - 2 * padding + dilation * (k_size - 1) + out_padding + 1\n                        let grad_l_in = grad.dim(2)?;\n                        let k_size = kernel.dim(2)?;\n                        let out_size =\n                            (grad_l_in - 1) * stride + dilation * (k_size - 1) + 1 - 2 * padding;\n                        let out_padding = arg.dim(2)? - out_size;\n                        let grad_arg = grad.conv_transpose1d(\n                            kernel,\n                            *padding,\n                            out_padding,\n                            *stride,\n                            *dilation,\n                            /* groups */ 1,\n                        )?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n\n                        let grad_kernel = arg\n                            .transpose(0, 1)?\n                            .conv1d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)?\n                            .transpose(0, 1)?;\n                        let sum_grad = grads.or_insert(kernel)?;\n                        let (_, _, k0) = kernel.dims3()?;\n                        let (_, _, g_k0) = grad_kernel.dims3()?;\n                        let grad_kernel = if g_k0 != k0 {\n                            grad_kernel.narrow(2, 0, k0)?\n                        } else {\n                            grad_kernel\n                        };\n                        *sum_grad = sum_grad.add(&grad_kernel)?;\n                    }\n                    Op::Conv2D {\n                        arg,\n                        kernel,\n                        padding,\n                        stride,\n                        dilation,\n                    } => {\n                        // The output height for conv_transpose2d is:\n                        // (i_h - 1) * stride - 2 * padding + dilation * (k_h - 1) + out_padding + 1\n                        let grad_h = grad.dim(2)?;\n                        let k_h = kernel.dim(2)?;\n                        let out_size =\n                            (grad_h - 1) * stride + dilation * (k_h - 1) + 1 - 2 * padding;\n                        let out_padding = arg.dim(2)? - out_size;\n                        let grad_arg = grad.conv_transpose2d(\n                            kernel,\n                            *padding,\n                            out_padding,\n                            *stride,\n                            *dilation,\n                        )?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n\n                        let grad_kernel = arg\n                            .transpose(0, 1)?\n                            .conv2d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)?\n                            .transpose(0, 1)?;\n                        let sum_grad = grads.or_insert(kernel)?;\n                        let (_, _, k0, k1) = kernel.dims4()?;\n                        let (_, _, g_k0, g_k1) = grad_kernel.dims4()?;\n                        let grad_kernel = if g_k0 != k0 || g_k1 != k1 {\n                            grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)?\n                        } else {\n                            grad_kernel\n                        };\n                        *sum_grad = sum_grad.add(&grad_kernel)?;\n                    }\n                    Op::ConvTranspose1D { .. } => Err(Error::BackwardNotSupported {\n                        op: \"conv-transpose1d\",\n                    })?,\n                    Op::ConvTranspose2D {\n                        arg,\n                        kernel,\n                        padding,\n                        stride,\n                        dilation,\n                        output_padding: _output_padding,\n                    } => {\n                        let grad_arg = grad.conv2d(kernel, *padding, *dilation, *stride, 1)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n\n                        let grad_kernel = grad\n                            .transpose(0, 1)?\n                            .conv2d(&arg.transpose(0, 1)?, *padding, *stride, *dilation, 1)?\n                            .transpose(0, 1)?;\n                        let sum_grad = grads.or_insert(kernel)?;\n                        let (_, _, k0, k1) = kernel.dims4()?;\n                        let (_, _, g_k0, g_k1) = grad_kernel.dims4()?;\n                        let grad_kernel = if g_k0 != k0 || g_k1 != k1 {\n                            grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)?\n                        } else {\n                            grad_kernel\n                        };\n                        *sum_grad = sum_grad.add(&grad_kernel)?;\n                    }\n                    Op::AvgPool2D {\n                        arg,\n                        kernel_size,\n                        stride,\n                    } => {\n                        if kernel_size != stride {\n                            crate::bail!(\"backward not supported for avgpool2d if ksize {kernel_size:?} != stride {stride:?}\")\n                        }\n                        let (_n, _c, h, w) = arg.dims4()?;\n                        let grad_arg = grad.upsample_nearest2d(h, w)?;\n                        let grad_arg =\n                            (grad_arg * (1f64 / (kernel_size.0 * kernel_size.1) as f64))?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n                    }\n                    Op::MaxPool2D {\n                        arg,\n                        kernel_size,\n                        stride,\n                    } => {\n                        if kernel_size != stride {\n                            crate::bail!(\"backward not supported for maxpool2d if ksize {kernel_size:?} != stride {stride:?}\")\n                        }\n                        let (_n, _c, h, w) = arg.dims4()?;\n                        // For computing the max-pool gradient, we compute a mask where a 1 means\n                        // that the element is the maximum, then we apply this mask to the\n                        // upsampled gradient (taking into account that multiple max may exist so\n                        // we scale the gradient for this case).\n                        let node_upsampled = node.upsample_nearest2d(h, w)?;\n                        let mask = arg.eq(&node_upsampled)?.to_dtype(arg.dtype())?;\n                        let avg = mask.avg_pool2d_with_stride(*kernel_size, *stride)?;\n                        let grad_arg = ((grad * avg)?.upsample_nearest2d(h, w)? * mask)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n                    }\n                    Op::UpsampleNearest1D { arg, target_size } => {\n                        let (_n, c, size) = arg.dims3()?;\n                        if target_size % size != 0 {\n                            crate::bail!(\"backward not supported for non integer upscaling factors\")\n                        }\n                        let scale = target_size / size;\n\n                        let kernel = Tensor::ones((c, 1, scale), arg.dtype(), arg.device())?;\n                        let conv_sum = grad.conv1d(&kernel, 0, scale, 1, c)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = conv_sum;\n                    }\n                    Op::UpsampleNearest2D {\n                        arg,\n                        target_h,\n                        target_w,\n                    } => {\n                        let (_n, c, h, w) = arg.dims4()?;\n                        if target_h % h != 0 || target_w % w != 0 {\n                            crate::bail!(\"backward not supported for non integer upscaling factors\")\n                        }\n                        let scale_h = target_h / h;\n                        let scale_w = target_w / w;\n\n                        if scale_h != scale_w {\n                            crate::bail!(\"backward not supported for non uniform upscaling factors\")\n                        };\n                        let kernel =\n                            Tensor::ones((c, 1, scale_h, scale_w), arg.dtype(), arg.device())?;\n                        let conv_sum = grad.conv2d(&kernel, 0, scale_h, 1, c)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = conv_sum;\n                    }\n                    Op::SliceScatter0(lhs, rhs, start_rhs) => {\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        let rhs_grad = grad.narrow(0, *start_rhs, rhs.dim(0)?)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        let lhs_grad = grad.slice_scatter0(&rhs.zeros_like()?, *start_rhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?\n                    }\n                    Op::Gather(arg, indexes, dim) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.scatter_add(indexes, &grad, *dim)?;\n                    }\n                    Op::ScatterAdd(init, indexes, src, dim) => {\n                        let init_sum_grad = grads.or_insert(init)?;\n                        *init_sum_grad = init_sum_grad.add(&grad)?;\n\n                        let src_grad = grad.gather(indexes, *dim)?;\n                        let src_sum_grad = grads.or_insert(src)?;\n                        *src_sum_grad = src_sum_grad.add(&src_grad)?;\n                    }\n                    Op::IndexAdd(init, indexes, src, dim) => {\n                        let init_sum_grad = grads.or_insert(init)?;\n                        *init_sum_grad = init_sum_grad.add(&grad)?;\n\n                        let src_grad = grad.index_select(indexes, *dim)?;\n                        let src_sum_grad = grads.or_insert(src)?;\n                        *src_sum_grad = src_sum_grad.add(&src_grad)?;\n                    }\n                    Op::IndexSelect(arg, indexes, dim) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.index_add(indexes, &grad, *dim)?;\n                    }\n                    Op::Matmul(lhs, rhs) => {\n                        // Skipping checks, the op went ok, we can skip\n                        // the matmul size checks for now.\n\n                        let lhs_grad = grad.matmul(&rhs.t()?)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n\n                        let rhs_grad = lhs.t()?.matmul(&grad)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n                    }\n                    Op::Cat(args, dim) => {\n                        let mut start_idx = 0;\n                        for arg in args {\n                            let len = arg.dims()[*dim];\n                            let arg_grad = grad.narrow(*dim, start_idx, len)?;\n                            let sum_grad = grads.or_insert(arg)?;\n                            *sum_grad = sum_grad.add(&arg_grad)?;\n                            start_idx += len;\n                        }\n                    }\n                    Op::Broadcast(arg) => {\n                        let arg_dims = arg.dims();\n                        let node_dims = node.dims();\n                        // The number of dims that have been inserted on the left.\n                        let left_dims = node_dims.len() - arg_dims.len();\n                        let mut sum_dims: Vec<usize> = (0..left_dims).collect();\n                        for (dim, (node_dim, arg_dim)) in node_dims[left_dims..]\n                            .iter()\n                            .zip(arg_dims.iter())\n                            .enumerate()\n                        {\n                            if node_dim != arg_dim {\n                                sum_dims.push(dim + left_dims)\n                            }\n                        }\n\n                        let mut arg_grad = grad.sum_keepdim(sum_dims.as_slice())?;\n                        for _i in 0..left_dims {\n                            arg_grad = arg_grad.squeeze(0)?\n                        }\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad.broadcast_as(sum_grad.dims())?)?;\n                    }\n                    Op::Reduce(arg, ReduceOp::Sum, reduced_dims) => {\n                        let grad = broadcast_back(arg, &grad, reduced_dims)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad)?;\n                    }\n                    Op::Reduce(arg, ReduceOp::Max, reduced_dims) => {\n                        let node = broadcast_back(arg, node, reduced_dims)?;\n                        let grad = broadcast_back(arg, &grad, reduced_dims)?;\n                        let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?;\n                    }\n                    Op::Reduce(arg, ReduceOp::Min, reduced_dims) => {\n                        let node = broadcast_back(arg, node, reduced_dims)?;\n                        let grad = broadcast_back(arg, &grad, reduced_dims)?;\n                        let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?;\n                    }\n                    Op::ToDType(arg) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad.to_dtype(arg.dtype())?)?\n                    }\n                    Op::Copy(arg) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad)?\n                    }\n                    Op::Affine { arg, mul, .. } => {\n                        let arg_grad = grad.affine(*mul, 0.)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Unary(arg, UnaryOp::Log) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&(grad / arg)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Sin) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&(&grad * arg.cos())?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Cos) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.sub(&(&grad * arg.sin())?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Tanh) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let minus_dtanh = (node.sqr()? - 1.)?;\n                        *sum_grad = sum_grad.sub(&(&grad * &minus_dtanh)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Abs) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let ones = arg.ones_like()?;\n                        let abs_grad = arg.ge(&arg.zeros_like()?)?.where_cond(&ones, &ones.neg()?);\n                        *sum_grad = sum_grad.add(&(&grad * abs_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Exp) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&(&grad * *node)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Neg) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.sub(&grad)?\n                    }\n                    Op::Unary(arg, UnaryOp::Recip) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let grad = (grad / arg.sqr()?)?;\n                        *sum_grad = sum_grad.sub(&grad)?\n                    }\n                    &Op::Narrow(ref arg, dim, start_idx, len) => {\n                        let arg_dims = arg.dims();\n                        let left_pad = if start_idx == 0 {\n                            None\n                        } else {\n                            let mut dims = arg_dims.to_vec();\n                            dims[dim] = start_idx;\n                            Some(Tensor::zeros(dims, grad.dtype(), grad.device())?)\n                        };\n                        let right_pad = arg_dims[dim] - start_idx - len;\n                        let right_pad = if right_pad == 0 {\n                            None\n                        } else {\n                            let mut dims = arg_dims.to_vec();\n                            dims[dim] = right_pad;\n                            Some(Tensor::zeros(dims, grad.dtype(), grad.device())?)\n                        };\n                        let arg_grad = match (left_pad, right_pad) {\n                            (None, None) => grad,\n                            (Some(l), None) => Tensor::cat(&[&l, &grad], dim)?,\n                            (None, Some(r)) => Tensor::cat(&[&grad, &r], dim)?,\n                            (Some(l), Some(r)) => Tensor::cat(&[&l, &grad, &r], dim)?,\n                        };\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Unary(_, UnaryOp::Floor)\n                    | Op::Unary(_, UnaryOp::Round)\n                    | Op::Reduce(_, ReduceOp::ArgMin, _)\n                    | Op::Reduce(_, ReduceOp::ArgMax, _)\n                    | Op::Unary(_, UnaryOp::Sign)\n                    | Op::Cmp(_, _) => {}\n                    Op::Reshape(arg) => {\n                        let arg_grad = grad.reshape(arg.dims())?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Unary(_, UnaryOp::Ceil) => Err(Error::BackwardNotSupported { op: \"ceil\" })?,\n                    Op::Unary(arg, UnaryOp::Gelu) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let cube = arg.powf(3.)?;\n                        let tanh = (0.0356774 * &cube + (0.797885 * arg)?)?.tanh()?;\n                        let gelu_grad = (((0.5 * &tanh)?\n                            + (0.0535161 * cube + (0.398942 * arg)?)? * (1. - tanh.powf(2.)?))?\n                            + 0.5)?;\n                        *sum_grad = sum_grad.add(&(&grad * gelu_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Erf) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        // d/dx erf(x) = 2/sqrt(pi) * e^(-x^2)\n                        let erf_grad =\n                            (2. / std::f64::consts::PI.sqrt()) * (arg.sqr()?.neg()?).exp()?;\n                        *sum_grad = sum_grad.add(&(&grad * erf_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::GeluErf) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        // d/dx gelu_erf(x) = 0.5 + 0.398942 e^(-x^2/2) x + 0.5 erf(x/sqrt(2))\n                        let neg_half_square = (arg.sqr()?.neg()? / 2.)?;\n                        let scaled_exp_arg = (0.398942 * neg_half_square.exp()? * arg)?;\n                        let arg_scaled_sqrt = (arg / 2f64.sqrt())?;\n                        let erf_scaled_sqrt = (0.5 * arg_scaled_sqrt.erf()?)?;\n                        let gelu_erf_grad = (0.5 + scaled_exp_arg + erf_scaled_sqrt)?;\n                        *sum_grad = sum_grad.add(&(&grad * gelu_erf_grad)?)?;\n                    }\n                    Op::Unary(arg, UnaryOp::Relu) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let relu_grad = arg.ge(&arg.zeros_like()?)?.to_dtype(arg.dtype())?;\n                        *sum_grad = sum_grad.add(&(&grad * relu_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Silu) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        // d/dx silu = sigmoid(x) * (1 + x * (1 - sigmoid(x)))\n                        let sigmoid_arg = (*node / arg)?;\n                        let silu_grad = (&sigmoid_arg * (1. + (arg * (1. - &sigmoid_arg)?)?)?)?;\n                        *sum_grad = sum_grad.add(&(&grad * silu_grad)?)?\n                    }\n                    Op::Elu(arg, alpha) => {\n...\n", "needle_token_start": 5473, "needle_token_end": 5918, "code_context_ntokens": 16297, "output": ["```rust\nfn index(&self, indexers: &[TensorIndexer]) -> Result<Self, Error> {\n    let mut x = self.clone();\n    let dims = self.shape().dims();\n    let mut current_dim = 0;\n    for (i, indexer) in indexers.iter().enumerate() {\n        x = match indexer {\n            TensorIndexer::Select(n) => x.narrow(current_dim, *n, 1)?.squeeze(current_dim)?,\n            TensorIndexer::Narrow(left_bound, right_bound) => {\n                let start = match left_bound {\n                    Bound::Included(n) => *n,\n                    Bound::Excluded(n) => *n + 1,\n                    Bound::Unbounded => 0,\n                };\n                let stop = match right_bound {\n                    Bound::Included(n) => *n + 1,\n                    Bound::Excluded(n) => *n,\n                    Bound::Unbounded => dims[i],\n                };\n                let out = x.narrow(current_dim, start, stop.saturating_sub(start))?;\n                current_dim += 1;\n                out\n            }\n            TensorIndexer::IndexSelect(indexes) => {\n                if indexes.rank() != 1 {\n                    crate::bail!(\"multi-dimensional tensor indexing is not supported\")\n                }\n                let out = x.index_select(&indexes.to_device(x.device())?, current_dim)?;\n                current_dim += 1;\n                out\n            }\n            TensorIndexer::Err(e) => crate::bail!(\"indexing error {e:?}\"),\n        };\n    }\n    Ok(x)\n}\n```"]}
{"repo": "huggingface/candle", "name": "read_to_vec", "language": "rust", "path": "candle-core/src/quantized/metal.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to convert a buffer of data into a vector of a specified type, ensuring type-safe access to the buffer's contents.\n2. **Input**: The function takes two parameters: a reference to a buffer containing data and an integer specifying the number of elements to read from the buffer.\n3. **Output**: It returns a vector containing elements of the specified type, extracted from the buffer.\n4. **Procedure**: The function first obtains a pointer to the buffer's contents, ensuring it is not null. It then constructs a slice from this pointer and the specified number of elements. Finally, it converts this slice into a vector, which is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: candle-core/src/quantized/cuda.rs\nuse super::{GgmlDType, QStorage};\nuse crate::quantized::k_quants::GgmlType;\nuse crate::{backend::BackendDevice, cuda_backend::WrapErr};\nuse crate::{CudaDevice, CudaStorage, Result};\n\nuse cudarc::driver::{CudaSlice, CudaView, DeviceSlice};\n\n#[derive(Clone, Debug)]\npub struct QCudaStorage {\n    data: CudaSlice<u8>,\n    dtype: GgmlDType,\n    device: CudaDevice,\n}\n\nstatic FORCE_DMMV: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);\n\npub fn set_force_dmmv(f: bool) {\n    FORCE_DMMV.store(f, std::sync::atomic::Ordering::Relaxed)\n}\n\npub const WARP_SIZE: usize = 32;\npub const MMQ_X_Q4_0_AMPERE: usize = 4;\npub const MMQ_Y_Q4_0_AMPERE: usize = 32;\npub const NWARPS_Q4_0_AMPERE: usize = 4;\npub const GGML_CUDA_MMV_X: usize = 32;\npub const GGML_CUDA_MMV_Y: usize = 1;\npub const CUDA_QUANTIZE_BLOCK_SIZE: usize = 256;\npub const CUDA_DEQUANTIZE_BLOCK_SIZE: usize = 256;\npub const MATRIX_ROW_PADDING: usize = 512;\n\nfn ceil_div(p: usize, q: usize) -> usize {\n    (p + q - 1) / q\n}\n\nfn pad(p: usize, q: usize) -> usize {\n    ceil_div(p, q) * q\n}\n\nfn quantize_q8_1(\n    src: &CudaView<f32>,\n    dst: &mut CudaSlice<u8>,\n    elem_count: usize,\n    dev: &CudaDevice,\n) -> Result<()> {\n    use cudarc::driver::LaunchAsync;\n\n    let kx = elem_count;\n    let kx_padded = pad(kx, MATRIX_ROW_PADDING);\n    let num_blocks = ceil_div(kx_padded, CUDA_QUANTIZE_BLOCK_SIZE);\n    let func = dev.get_or_load_func(\"quantize_q8_1\", candle_kernels::QUANTIZED)?;\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (num_blocks as u32, 1, 1),\n        block_dim: (CUDA_QUANTIZE_BLOCK_SIZE as u32, 1, 1),\n        shared_mem_bytes: 0,\n    };\n    let params = (src, dst, kx as i32, kx_padded as i32);\n    unsafe { func.launch(cfg, params) }.w()?;\n    Ok(())\n}\n\nfn dequantize(\n    data: &CudaSlice<u8>,\n    dtype: GgmlDType,\n    elem_count: usize,\n    dev: &CudaDevice,\n) -> Result<CudaStorage> {\n    use cudarc::driver::LaunchAsync;\n\n    let nb = (elem_count + 255) / 256;\n    let (kernel_name, is_k, block_dim, num_blocks) = match dtype {\n        GgmlDType::Q4_0 => (\"dequantize_block_q4_0\", false, 32, nb),\n        GgmlDType::Q4_1 => (\"dequantize_block_q4_1\", false, 32, nb),\n        GgmlDType::Q5_0 => (\n            \"dequantize_block_q5_0\",\n            false,\n            CUDA_DEQUANTIZE_BLOCK_SIZE,\n            ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE),\n        ),\n        GgmlDType::Q5_1 => (\n            \"dequantize_block_q5_1\",\n            false,\n            CUDA_DEQUANTIZE_BLOCK_SIZE,\n            ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE),\n        ),\n        GgmlDType::Q8_0 => (\"dequantize_block_q8_0\", false, 32, nb),\n        GgmlDType::Q2K => (\"dequantize_block_q2_K\", true, 64, nb),\n        GgmlDType::Q3K => (\"dequantize_block_q3_K\", true, 64, nb),\n        GgmlDType::Q4K => (\"dequantize_block_q4_K\", true, 32, nb),\n        GgmlDType::Q5K => (\"dequantize_block_q5_K\", true, 64, nb),\n        GgmlDType::Q6K => (\"dequantize_block_q6_K\", true, 64, nb),\n        GgmlDType::Q8K => (\"dequantize_block_q8_K\", true, 32, nb),\n        _ => crate::bail!(\"unsupported dtype for dequantize {dtype:?}\"),\n    };\n    let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?;\n    let dst = unsafe { dev.alloc::<f32>(elem_count).w()? };\n    // See e.g.\n    // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (num_blocks as u32, 1, 1),\n        block_dim: (block_dim as u32, 1, 1),\n        shared_mem_bytes: 0,\n    };\n\n    if is_k {\n        let params = (data, &dst);\n        unsafe { func.launch(cfg, params) }.w()?;\n    } else {\n        let nb32 = match dtype {\n            GgmlDType::Q5_0 | GgmlDType::Q5_1 => elem_count,\n            _ => elem_count / 32,\n        };\n        let params = (data, &dst, nb32 as i32);\n        unsafe { func.launch(cfg, params) }.w()?;\n    }\n    Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone()))\n}\n\nfn dequantize_mul_mat_vec(\n    data: &CudaSlice<u8>,\n    y: &CudaView<f32>,\n    dtype: GgmlDType,\n    ncols: usize,\n    nrows: usize,\n    dev: &CudaDevice,\n) -> Result<CudaStorage> {\n    use cudarc::driver::LaunchAsync;\n\n    let data_elems = data.len() / dtype.type_size() * dtype.block_size();\n    if data_elems < ncols * nrows {\n        crate::bail!(\"unexpected data size {}, ncols {ncols} {nrows}\", data_elems)\n    }\n    if y.len() != ncols {\n        crate::bail!(\"unexpected y size {}, ncols {ncols} {nrows}\", y.len())\n    }\n    let kernel_name = match dtype {\n        GgmlDType::Q4_0 => \"dequantize_mul_mat_vec_q4_0_cuda\",\n        GgmlDType::Q4_1 => \"dequantize_mul_mat_vec_q4_1_cuda\",\n        GgmlDType::Q5_0 => \"dequantize_mul_mat_vec_q5_0_cuda\",\n        GgmlDType::Q5_1 => \"dequantize_mul_mat_vec_q5_1_cuda\",\n        GgmlDType::Q8_0 => \"dequantize_mul_mat_vec_q8_0_cuda\",\n        GgmlDType::Q2K => \"dequantize_mul_mat_vec_q2_k\",\n        GgmlDType::Q3K => \"dequantize_mul_mat_vec_q3_k\",\n        GgmlDType::Q4K => \"dequantize_mul_mat_vec_q4_k\",\n        GgmlDType::Q5K => \"dequantize_mul_mat_vec_q5_k\",\n        GgmlDType::Q6K => \"dequantize_mul_mat_vec_q6_k\",\n        _ => crate::bail!(\"unsupported dtype for quantized matmul {dtype:?}\"),\n    };\n    let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?;\n    let dst = unsafe { dev.alloc::<f32>(nrows).w()? };\n    let block_num_y = ceil_div(nrows, GGML_CUDA_MMV_Y);\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (block_num_y as u32, 1, 1),\n        block_dim: (WARP_SIZE as u32, GGML_CUDA_MMV_Y as u32, 1),\n        shared_mem_bytes: 0,\n    };\n\n    let params = (data, y, &dst, ncols as i32, nrows as i32);\n    unsafe { func.launch(cfg, params) }.w()?;\n    Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone()))\n}\n\nfn mul_mat_vec_via_q8_1(\n    data: &CudaSlice<u8>,\n    y: &CudaView<f32>,\n    dtype: GgmlDType,\n    ncols: usize,\n    nrows: usize,\n    dev: &CudaDevice,\n) -> Result<CudaStorage> {\n    use cudarc::driver::LaunchAsync;\n\n    let data_elems = data.len() / dtype.type_size() * dtype.block_size();\n    if data_elems < ncols * nrows {\n        crate::bail!(\"unexpected data size {}, ncols {ncols} {nrows}\", data_elems)\n    }\n    if y.len() != ncols {\n        crate::bail!(\"unexpected y size {}, ncols {ncols} {nrows}\", y.len())\n    }\n    // Start by quantizing y\n    let ncols_padded = pad(ncols, MATRIX_ROW_PADDING);\n    let y_size_in_bytes = ncols_padded * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size();\n    let mut y_q8_1 = unsafe { dev.alloc::<u8>(y_size_in_bytes).w()? };\n    quantize_q8_1(y, &mut y_q8_1, ncols, dev)?;\n\n    let kernel_name = match dtype {\n        GgmlDType::Q4_0 => \"mul_mat_vec_q4_0_q8_1_cuda\",\n        GgmlDType::Q4_1 => \"mul_mat_vec_q4_1_q8_1_cuda\",\n        GgmlDType::Q5_0 => \"mul_mat_vec_q5_0_q8_1_cuda\",\n        GgmlDType::Q5_1 => \"mul_mat_vec_q5_1_q8_1_cuda\",\n        GgmlDType::Q8_0 => \"mul_mat_vec_q8_0_q8_1_cuda\",\n        GgmlDType::Q2K => \"mul_mat_vec_q2_K_q8_1_cuda\",\n        GgmlDType::Q3K => \"mul_mat_vec_q3_K_q8_1_cuda\",\n        GgmlDType::Q4K => \"mul_mat_vec_q4_K_q8_1_cuda\",\n        GgmlDType::Q5K => \"mul_mat_vec_q5_K_q8_1_cuda\",\n        GgmlDType::Q6K => \"mul_mat_vec_q6_K_q8_1_cuda\",\n        _ => crate::bail!(\"unsupported dtype for quantized matmul {dtype:?}\"),\n    };\n    let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?;\n    let dst = unsafe { dev.alloc::<f32>(nrows).w()? };\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (nrows as u32, 1, 1),\n        block_dim: (WARP_SIZE as u32, 4, 1),\n        shared_mem_bytes: 0,\n    };\n\n    let params = (\n        data,\n        &y_q8_1,\n        &dst,\n        /* ncols_x */ ncols as i32,\n        /* nrows_x */ nrows as i32,\n        /* nrows_y */ ncols as i32,\n        /* nrows_dst */ nrows as i32,\n    );\n    unsafe { func.launch(cfg, params) }.w()?;\n    Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone()))\n}\n\nimpl QCudaStorage {\n    pub fn zeros(device: &CudaDevice, el_count: usize, dtype: GgmlDType) -> Result<Self> {\n        let size_in_bytes = ceil_div(el_count, dtype.block_size()) * dtype.type_size();\n        let data = device.alloc_zeros::<u8>(size_in_bytes).w()?;\n        Ok(QCudaStorage {\n            data,\n            device: device.clone(),\n            dtype,\n        })\n    }\n\n    pub fn dtype(&self) -> GgmlDType {\n        self.dtype\n    }\n\n    pub fn device(&self) -> &CudaDevice {\n        &self.device\n    }\n\n    pub fn dequantize(&self, elem_count: usize) -> Result<CudaStorage> {\n        fn deq<T: GgmlType>(buffer: &[u8], n: usize, dst: &mut [f32]) -> Result<()> {\n            let slice = unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const T, n) };\n            let vec = slice.to_vec();\n            T::to_float(&vec, dst)\n        }\n\n        let fast_kernel = matches!(\n            self.dtype,\n            GgmlDType::Q4_0\n                | GgmlDType::Q4_1\n                | GgmlDType::Q5_0\n                | GgmlDType::Q5_1\n                | GgmlDType::Q8_0\n                | GgmlDType::Q2K\n                | GgmlDType::Q3K\n                | GgmlDType::Q4K\n                | GgmlDType::Q5K\n                | GgmlDType::Q6K\n                | GgmlDType::Q8K\n        );\n        if fast_kernel {\n            return dequantize(&self.data, self.dtype, elem_count, self.device());\n        }\n        // Run the dequantization on cpu.\n\n        let buffer = self.device.dtoh_sync_copy(&self.data).w()?;\n        let mut out = vec![0.0; elem_count];\n        let block_len = elem_count / self.dtype.block_size();\n        match self.dtype {\n            GgmlDType::F32 => deq::<f32>(&buffer, block_len, &mut out)?,\n            GgmlDType::F16 => deq::<half::f16>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q4_0 => deq::<crate::quantized::BlockQ4_0>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q4_1 => deq::<crate::quantized::BlockQ4_1>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q5_0 => deq::<crate::quantized::BlockQ5_0>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q5_1 => deq::<crate::quantized::BlockQ5_1>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q8_0 => deq::<crate::quantized::BlockQ8_0>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q8_1 => deq::<crate::quantized::BlockQ8_1>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q2K => deq::<crate::quantized::BlockQ2K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q3K => deq::<crate::quantized::BlockQ3K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q4K => deq::<crate::quantized::BlockQ4K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q5K => deq::<crate::quantized::BlockQ5K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q6K => deq::<crate::quantized::BlockQ6K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q8K => deq::<crate::quantized::BlockQ8K>(&buffer, block_len, &mut out)?,\n        }\n\n        self.device\n            .storage_from_cpu_storage(&crate::CpuStorage::F32(out))\n    }\n\n    pub fn quantize(&mut self, src: &CudaStorage) -> Result<()> {\n        // Run the quantization on cpu.\n...\n// Path: candle-core/src/quantized/metal.rs\nuse super::{GgmlDType, QStorage};\nuse crate::backend::BackendStorage;\nuse crate::{DType, MetalDevice, MetalStorage, Result, Shape};\nuse metal::Buffer;\nuse std::sync::Arc;\n\npub struct QMetalStorage {\n    dtype: GgmlDType,\n    device: MetalDevice,\n    buffer: Arc<Buffer>,\n}\n\nimpl QMetalStorage {\n    pub fn zeros(device: &MetalDevice, elem_count: usize, dtype: GgmlDType) -> Result<Self> {\n        let size = elem_count * dtype.type_size() / dtype.block_size();\n        let buffer = device.allocate_zeros(size)?;\n        Ok(Self {\n            buffer,\n            device: device.clone(),\n            dtype,\n        })\n    }\n\n    pub fn dtype(&self) -> GgmlDType {\n        self.dtype\n    }\n\n    pub fn device(&self) -> &MetalDevice {\n        &self.device\n    }\n\n    pub fn buffer(&self) -> &Buffer {\n        &self.buffer\n    }\n\n    pub fn dequantize(&self, elem_count: usize) -> Result<MetalStorage> {\n        use crate::quantized::k_quants::GgmlType;\n\n        let buffer = self.device.new_buffer_managed(self.buffer.length())?;\n        let command_buffer = self.device.command_buffer()?;\n        command_buffer.set_label(\"to_cpu\");\n        let blit = command_buffer.new_blit_command_encoder();\n        blit.set_label(\"blit_to_cpu\");\n        blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length());\n        blit.end_encoding();\n        self.device.wait_until_completed()?;\n        let mut out = vec![0.0; elem_count];\n        let block_len = elem_count / self.dtype.block_size();\n        match self.dtype {\n            GgmlDType::F32 => {\n                let vec: Vec<f32> = read_to_vec(&buffer, block_len);\n                f32::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::F16 => {\n                let vec: Vec<half::f16> = read_to_vec(&buffer, block_len);\n                half::f16::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q4_0 => {\n                let vec: Vec<crate::quantized::BlockQ4_0> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ4_0::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q4_1 => {\n                let vec: Vec<crate::quantized::BlockQ4_1> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ4_1::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q5_0 => {\n                let vec: Vec<crate::quantized::BlockQ5_0> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ5_0::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q5_1 => {\n                let vec: Vec<crate::quantized::BlockQ5_1> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ5_1::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q8_0 => {\n                let vec: Vec<crate::quantized::BlockQ8_0> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ8_0::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q8_1 => {\n                let vec: Vec<crate::quantized::BlockQ8_1> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ8_1::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q2K => {\n                let vec: Vec<crate::quantized::BlockQ2K> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ2K::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q3K => {\n                let vec: Vec<crate::quantized::BlockQ3K> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ3K::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q4K => {\n                let vec: Vec<crate::quantized::BlockQ4K> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ4K::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q5K => {\n                let vec: Vec<crate::quantized::BlockQ5K> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ5K::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q6K => {\n                let vec: Vec<crate::quantized::BlockQ6K> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ6K::to_float(&vec, &mut out)?;\n            }\n            GgmlDType::Q8K => {\n                let vec: Vec<crate::quantized::BlockQ8K> = read_to_vec(&buffer, block_len);\n                crate::quantized::BlockQ8K::to_float(&vec, &mut out)?;\n            }\n        }\n\n        let buffer = self.device.new_buffer_with_data(&out)?;\n        Ok(MetalStorage::new(\n            buffer,\n            self.device.clone(),\n            elem_count,\n            DType::F32,\n        ))\n    }\n\n    pub fn quantize(&mut self, src: &MetalStorage) -> Result<()> {\n        // Quantization only happens on CPU for now.\n        let src = src.to_cpu::<f32>()?;\n        let elem_count = src.len();\n        let src = crate::Storage::Cpu(crate::CpuStorage::F32(src));\n        let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?;\n        qcpu_storage.quantize(&src)?;\n        let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?;\n        self.buffer = buffer;\n        Ok(())\n    }\n\n    pub fn storage_size_in_bytes(&self) -> usize {\n        self.buffer.length() as usize\n    }\n\n    pub fn fwd(\n        &self,\n        self_shape: &Shape,\n        storage: &MetalStorage,\n        layout: &crate::Layout,\n    ) -> Result<(MetalStorage, Shape)> {\n        use crate::MetalError;\n\n        if !layout.is_contiguous() {\n            crate::bail!(\"input tensor is not contiguous {layout:?}\")\n        }\n        let src_shape = layout.shape();\n        // self is transposed so n is first then k.\n        if src_shape.rank() < 2 {\n            crate::bail!(\"input tensor has only one dimension {layout:?}\")\n        }\n        let (n, k) = self_shape.dims2()?;\n        let mut dst_shape = src_shape.dims().to_vec();\n\n        // We always use a single batch dimension and stack all the tensors in the batch on the\n        // second dimension as the implementation in candle-metal-kernels doesn't handle batch\n        // properly.\n        let (b, m) = match dst_shape.len() {\n            3 => (1, dst_shape[0] * dst_shape[1]),\n            2 => (1, dst_shape[0]),\n            n => crate::bail!(\"Invalid rank {n} for quantized matmul metal\"),\n        };\n        let last_k = dst_shape.pop().unwrap();\n        if last_k != k {\n            crate::bail!(\"input tensor {layout:?} incompatible with {:?}\", self_shape)\n        }\n        dst_shape.push(n);\n        let dst_shape = Shape::from(dst_shape);\n        let device = storage.device().clone();\n        let dst = device.new_buffer(dst_shape.elem_count(), DType::F32, \"qmatmul\")?;\n        let command_buffer = device.command_buffer()?;\n        candle_metal_kernels::call_quantized_matmul_t(\n            device.device(),\n            &command_buffer,\n            device.kernels(),\n            self.dtype.into(),\n            (b, m, n, k),\n            storage.buffer(),\n            layout.start_offset() * storage.dtype().size_in_bytes(),\n            &self.buffer,\n            &dst,\n        )\n        .map_err(MetalError::from)?;\n        let dst_storage = crate::MetalStorage::new(dst, device, dst_shape.elem_count(), DType::F32);\n        Ok((dst_storage, dst_shape))\n    }\n}\n\npub fn load_quantized<T: super::GgmlType + Send + Sync + 'static>(\n    device: &MetalDevice,\n    data: &[T],\n) -> Result<QStorage> {\n    let buffer = device.new_buffer_with_data(data)?;\n    let device = device.clone();\n    Ok(QStorage::Metal(QMetalStorage {\n        dtype: T::DTYPE,\n        device,\n        buffer,\n    }))\n}\n\n\nfn read_to_vec<T: Clone>(buffer: &Buffer, n: usize) -> Vec<T> {\n    let ptr = buffer.contents() as *const T;\n    assert!(!ptr.is_null());\n    let slice = unsafe { std::slice::from_raw_parts(ptr, n) };\n    slice.to_vec()\n}\n\nimpl From<GgmlDType> for candle_metal_kernels::GgmlDType {\n    fn from(value: GgmlDType) -> Self {\n        match value {\n            GgmlDType::Q4_0 => candle_metal_kernels::GgmlDType::Q4_0,\n            GgmlDType::Q4_1 => candle_metal_kernels::GgmlDType::Q4_1,\n            GgmlDType::Q5_0 => candle_metal_kernels::GgmlDType::Q5_0,\n            GgmlDType::Q5_1 => candle_metal_kernels::GgmlDType::Q5_1,\n            GgmlDType::Q8_0 => candle_metal_kernels::GgmlDType::Q8_0,\n            GgmlDType::Q8_1 => candle_metal_kernels::GgmlDType::Q8_1,\n            GgmlDType::Q2K => candle_metal_kernels::GgmlDType::Q2K,\n            GgmlDType::Q3K => candle_metal_kernels::GgmlDType::Q3K,\n            GgmlDType::Q4K => candle_metal_kernels::GgmlDType::Q4K,\n            GgmlDType::Q5K => candle_metal_kernels::GgmlDType::Q5K,\n            GgmlDType::Q6K => candle_metal_kernels::GgmlDType::Q6K,\n            GgmlDType::Q8K => candle_metal_kernels::GgmlDType::Q8K,\n            GgmlDType::F16 => candle_metal_kernels::GgmlDType::F16,\n            GgmlDType::F32 => candle_metal_kernels::GgmlDType::F32,\n        }\n    }\n}\n\n// Path: candle-core/src/quantized/gguf_file.rs\n//! Support for the GGUF file format.\n//!\n//! Spec: https://github.com/philpax/ggml/blob/gguf-spec/docs/gguf.md\n\nuse super::{GgmlDType, QTensor};\nuse crate::{Device, Result};\nuse byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};\nuse std::collections::HashMap;\n\npub const DEFAULT_ALIGNMENT: u64 = 32;\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\nenum Magic {\n    Gguf,\n}\n\nimpl TryFrom<u32> for Magic {\n    type Error = crate::Error;\n    fn try_from(value: u32) -> Result<Self> {\n        let magic = match value {\n            0x46554747 | 0x47475546 => Self::Gguf,\n            _ => crate::bail!(\"unknown magic 0x{value:08x}\"),\n        };\n        Ok(magic)\n    }\n}\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum VersionedMagic {\n    GgufV1,\n    GgufV2,\n    GgufV3,\n}\n\nimpl VersionedMagic {\n    fn read<R: std::io::Read>(reader: &mut R) -> Result<Self> {\n        let magic = reader.read_u32::<LittleEndian>()?;\n        let magic = Magic::try_from(magic)?;\n        let version = reader.read_u32::<LittleEndian>()?;\n        let versioned_magic = match (magic, version) {\n            (Magic::Gguf, 1) => Self::GgufV1,\n            (Magic::Gguf, 2) => Self::GgufV2,\n            (Magic::Gguf, 3) => Self::GgufV3,\n            _ => crate::bail!(\"gguf: unsupported magic/version {magic:?}/{version}\"),\n        };\n        Ok(versioned_magic)\n    }\n}\n\n#[derive(Debug)]\npub struct TensorInfo {\n    pub ggml_dtype: GgmlDType,\n    pub shape: crate::Shape,\n    pub offset: u64,\n}\n\nimpl TensorInfo {\n    pub fn read<R: std::io::Seek + std::io::Read>(\n        &self,\n        reader: &mut R,\n        tensor_data_offset: u64,\n        device: &Device,\n    ) -> Result<QTensor> {\n        let tensor_elems = self.shape.elem_count();\n        let block_size = self.ggml_dtype.block_size();\n        if tensor_elems % block_size != 0 {\n            crate::bail!(\n            \"the number of elements {tensor_elems} is not divisible by the block size {block_size}\"\n        )\n        }\n        let size_in_bytes = tensor_elems / block_size * self.ggml_dtype.type_size();\n        let mut raw_data = vec![0u8; size_in_bytes];\n        reader.seek(std::io::SeekFrom::Start(tensor_data_offset + self.offset))?;\n        reader.read_exact(&mut raw_data)?;\n        super::ggml_file::qtensor_from_ggml(\n            self.ggml_dtype,\n            &raw_data,\n            self.shape.dims().to_vec(),\n            device,\n        )\n    }\n}\n\n#[derive(Debug)]\npub struct Content {\n    pub magic: VersionedMagic,\n    pub metadata: HashMap<String, Value>,\n    pub tensor_infos: HashMap<String, TensorInfo>,\n    pub tensor_data_offset: u64,\n}\n\nfn read_string<R: std::io::Read>(reader: &mut R, magic: &VersionedMagic) -> Result<String> {\n    let len = match magic {\n        VersionedMagic::GgufV1 => reader.read_u32::<LittleEndian>()? as usize,\n        VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => {\n            reader.read_u64::<LittleEndian>()? as usize\n        }\n    };\n    let mut v = vec![0u8; len];\n    reader.read_exact(&mut v)?;\n    // GGUF strings are supposed to be non-null terminated but in practice this happens.\n    while let Some(0) = v.last() {\n        v.pop();\n    }\n    // GGUF strings are utf8 encoded but there are cases that don't seem to be valid.\n    Ok(String::from_utf8_lossy(&v).into_owned())\n}\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]\npub enum ValueType {\n    // The value is a 8-bit unsigned integer.\n    U8,\n    // The value is a 8-bit signed integer.\n    I8,\n    // The value is a 16-bit unsigned little-endian integer.\n    U16,\n    // The value is a 16-bit signed little-endian integer.\n    I16,\n    // The value is a 32-bit unsigned little-endian integer.\n    U32,\n    // The value is a 32-bit signed little-endian integer.\n    I32,\n    // The value is a 64-bit unsigned little-endian integer.\n    U64,\n    // The value is a 64-bit signed little-endian integer.\n    I64,\n    // The value is a 32-bit IEEE754 floating point number.\n    F32,\n    // The value is a 64-bit IEEE754 floating point number.\n    F64,\n    // The value is a boolean.\n    // 1-byte value where 0 is false and 1 is true.\n    // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy.\n    Bool,\n    // The value is a UTF-8 non-null-terminated string, with length prepended.\n    String,\n    // The value is an array of other values, with the length and type prepended.\n    ///\n    // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes.\n    Array,\n}\n\n#[derive(Debug, Clone)]\npub enum Value {\n    U8(u8),\n    I8(i8),\n    U16(u16),\n    I16(i16),\n    U32(u32),\n    I32(i32),\n    U64(u64),\n    I64(i64),\n    F32(f32),\n    F64(f64),\n    Bool(bool),\n    String(String),\n    Array(Vec<Value>),\n}\n\nimpl Value {\n    pub fn value_type(&self) -> ValueType {\n        match self {\n            Self::U8(_) => ValueType::U8,\n            Self::I8(_) => ValueType::I8,\n            Self::U16(_) => ValueType::U16,\n            Self::I16(_) => ValueType::I16,\n            Self::U32(_) => ValueType::U32,\n            Self::I32(_) => ValueType::I32,\n            Self::U64(_) => ValueType::U64,\n            Self::I64(_) => ValueType::I64,\n            Self::F32(_) => ValueType::F32,\n            Self::F64(_) => ValueType::F64,\n            Self::Bool(_) => ValueType::Bool,\n            Self::String(_) => ValueType::String,\n            Self::Array(_) => ValueType::Array,\n        }\n    }\n\n    pub fn to_u8(&self) -> Result<u8> {\n        match self {\n            Self::U8(v) => Ok(*v),\n            v => crate::bail!(\"not a u8 {v:?}\"),\n        }\n    }\n\n    pub fn to_i8(&self) -> Result<i8> {\n        match self {\n            Self::I8(v) => Ok(*v),\n            v => crate::bail!(\"not a i8 {v:?}\"),\n        }\n    }\n\n    pub fn to_u16(&self) -> Result<u16> {\n        match self {\n            Self::U16(v) => Ok(*v),\n            v => crate::bail!(\"not a u16 {v:?}\"),\n        }\n    }\n\n    pub fn to_i16(&self) -> Result<i16> {\n        match self {\n            Self::I16(v) => Ok(*v),\n            v => crate::bail!(\"not a i16 {v:?}\"),\n        }\n    }\n\n    pub fn to_u32(&self) -> Result<u32> {\n        match self {\n            Self::U32(v) => Ok(*v),\n            v => crate::bail!(\"not a u32 {v:?}\"),\n        }\n    }\n\n    pub fn to_i32(&self) -> Result<i32> {\n        match self {\n            Self::I32(v) => Ok(*v),\n            v => crate::bail!(\"not a i32 {v:?}\"),\n        }\n    }\n\n    pub fn to_u64(&self) -> Result<u64> {\n        match self {\n            Self::U64(v) => Ok(*v),\n            v => crate::bail!(\"not a u64 {v:?}\"),\n        }\n    }\n\n    pub fn to_i64(&self) -> Result<i64> {\n        match self {\n            Self::I64(v) => Ok(*v),\n            v => crate::bail!(\"not a i64 {v:?}\"),\n        }\n    }\n\n    pub fn to_f32(&self) -> Result<f32> {\n        match self {\n            Self::F32(v) => Ok(*v),\n            v => crate::bail!(\"not a f32 {v:?}\"),\n        }\n    }\n\n    pub fn to_f64(&self) -> Result<f64> {\n        match self {\n            Self::F64(v) => Ok(*v),\n            v => crate::bail!(\"not a f64 {v:?}\"),\n        }\n    }\n\n    pub fn to_bool(&self) -> Result<bool> {\n        match self {\n            Self::Bool(v) => Ok(*v),\n            v => crate::bail!(\"not a bool {v:?}\"),\n        }\n    }\n\n    pub fn to_vec(&self) -> Result<&Vec<Value>> {\n        match self {\n            Self::Array(v) => Ok(v),\n            v => crate::bail!(\"not a vec {v:?}\"),\n        }\n    }\n\n    pub fn to_string(&self) -> Result<&String> {\n        match self {\n            Self::String(v) => Ok(v),\n            v => crate::bail!(\"not a string {v:?}\"),\n        }\n    }\n\n    fn read<R: std::io::Read>(\n        reader: &mut R,\n        value_type: ValueType,\n        magic: &VersionedMagic,\n    ) -> Result<Self> {\n        let v = match value_type {\n            ValueType::U8 => Self::U8(reader.read_u8()?),\n            ValueType::I8 => Self::I8(reader.read_i8()?),\n            ValueType::U16 => Self::U16(reader.read_u16::<LittleEndian>()?),\n            ValueType::I16 => Self::I16(reader.read_i16::<LittleEndian>()?),\n            ValueType::U32 => Self::U32(reader.read_u32::<LittleEndian>()?),\n            ValueType::I32 => Self::I32(reader.read_i32::<LittleEndian>()?),\n            ValueType::U64 => Self::U64(reader.read_u64::<LittleEndian>()?),\n            ValueType::I64 => Self::I64(reader.read_i64::<LittleEndian>()?),\n            ValueType::F32 => Self::F32(reader.read_f32::<LittleEndian>()?),\n            ValueType::F64 => Self::F64(reader.read_f64::<LittleEndian>()?),\n            ValueType::Bool => match reader.read_u8()? {\n                0 => Self::Bool(false),\n                1 => Self::Bool(true),\n                b => crate::bail!(\"unexpected bool value {b}\"),\n            },\n            ValueType::String => Self::String(read_string(reader, magic)?),\n            ValueType::Array => {\n                let value_type = reader.read_u32::<LittleEndian>()?;\n                let value_type = ValueType::from_u32(value_type)?;\n                let len = match magic {\n                    VersionedMagic::GgufV1 => reader.read_u32::<LittleEndian>()? as usize,\n                    VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => {\n                        reader.read_u64::<LittleEndian>()? as usize\n                    }\n                };\n                let mut vs = Vec::with_capacity(len);\n                for _ in 0..len {\n                    vs.push(Value::read(reader, value_type, magic)?)\n                }\n                Self::Array(vs)\n            }\n        };\n        Ok(v)\n    }\n\n    fn write<W: std::io::Write>(&self, w: &mut W) -> Result<()> {\n        match self {\n            &Self::U8(v) => w.write_u8(v)?,\n            &Self::I8(v) => w.write_i8(v)?,\n            &Self::U16(v) => w.write_u16::<LittleEndian>(v)?,\n            &Self::I16(v) => w.write_i16::<LittleEndian>(v)?,\n            &Self::U32(v) => w.write_u32::<LittleEndian>(v)?,\n            &Self::I32(v) => w.write_i32::<LittleEndian>(v)?,\n            &Self::U64(v) => w.write_u64::<LittleEndian>(v)?,\n            &Self::I64(v) => w.write_i64::<LittleEndian>(v)?,\n            &Self::F32(v) => w.write_f32::<LittleEndian>(v)?,\n            &Self::F64(v) => w.write_f64::<LittleEndian>(v)?,\n            &Self::Bool(v) => w.write_u8(u8::from(v))?,\n            Self::String(v) => write_string(w, v.as_str())?,\n            Self::Array(v) => {\n                // The `Value` type does not enforce that all the values in an Array have the same\n                // type.\n                let value_type = if v.is_empty() {\n                    // Doesn't matter, the array is empty.\n                    ValueType::U32\n                } else {\n                    let value_type: std::collections::HashSet<_> =\n                        v.iter().map(|elem| elem.value_type()).collect();\n                    if value_type.len() != 1 {\n                        crate::bail!(\"multiple value-types in the same array {value_type:?}\")\n                    }\n                    value_type.into_iter().next().unwrap()\n                };\n                w.write_u32::<LittleEndian>(value_type.to_u32())?;\n                w.write_u64::<LittleEndian>(v.len() as u64)?;\n                for elem in v.iter() {\n                    elem.write(w)?\n                }\n            }\n        }\n        Ok(())\n    }\n}\n\nimpl ValueType {\n    fn from_u32(v: u32) -> Result<Self> {\n        let v = match v {\n            0 => Self::U8,\n            1 => Self::I8,\n            2 => Self::U16,\n            3 => Self::I16,\n            4 => Self::U32,\n            5 => Self::I32,\n            6 => Self::F32,\n            7 => Self::Bool,\n            8 => Self::String,\n            9 => Self::Array,\n            10 => Self::U64,\n            11 => Self::I64,\n            12 => Self::F64,\n            v => crate::bail!(\"unrecognized value-type {v:#08x}\"),\n        };\n        Ok(v)\n    }\n\n    fn to_u32(self) -> u32 {\n        match self {\n            Self::U8 => 0,\n            Self::I8 => 1,\n            Self::U16 => 2,\n            Self::I16 => 3,\n            Self::U32 => 4,\n            Self::I32 => 5,\n            Self::F32 => 6,\n            Self::Bool => 7,\n            Self::String => 8,\n            Self::Array => 9,\n            Self::U64 => 10,\n            Self::I64 => 11,\n            Self::F64 => 12,\n        }\n    }\n}\n\nimpl Content {\n    pub fn read<R: std::io::Seek + std::io::Read>(reader: &mut R) -> Result<Self> {\n        let magic = VersionedMagic::read(reader)?;\n\n        let tensor_count = match magic {\n            VersionedMagic::GgufV1 => reader.read_u32::<LittleEndian>()? as usize,\n            VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => {\n                reader.read_u64::<LittleEndian>()? as usize\n            }\n        };\n        let metadata_kv_count = match magic {\n            VersionedMagic::GgufV1 => reader.read_u32::<LittleEndian>()? as usize,\n            VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => {\n                reader.read_u64::<LittleEndian>()? as usize\n            }\n        };\n\n        let mut metadata = HashMap::new();\n        for _idx in 0..metadata_kv_count {\n            let key = read_string(reader, &magic)?;\n            let value_type = reader.read_u32::<LittleEndian>()?;\n            let value_type = ValueType::from_u32(value_type)?;\n            let value = Value::read(reader, value_type, &magic)?;\n            metadata.insert(key, value);\n        }\n        let mut tensor_infos = HashMap::new();\n        for _idx in 0..tensor_count {\n            let tensor_name = read_string(reader, &magic)?;\n            let n_dimensions = reader.read_u32::<LittleEndian>()?;\n\n            let mut dimensions: Vec<usize> = match magic {\n                VersionedMagic::GgufV1 => {\n                    let mut dimensions = vec![0; n_dimensions as usize];\n                    reader.read_u32_into::<LittleEndian>(&mut dimensions)?;\n                    dimensions.into_iter().map(|c| c as usize).collect()\n                }\n                VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => {\n                    let mut dimensions = vec![0; n_dimensions as usize];\n                    reader.read_u64_into::<LittleEndian>(&mut dimensions)?;\n                    dimensions.into_iter().map(|c| c as usize).collect()\n                }\n            };\n\n            dimensions.reverse();\n            let ggml_dtype = reader.read_u32::<LittleEndian>()?;\n            let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?;\n            let offset = reader.read_u64::<LittleEndian>()?;\n            tensor_infos.insert(\n                tensor_name,\n                TensorInfo {\n                    shape: crate::Shape::from(dimensions),\n                    offset,\n                    ggml_dtype,\n                },\n            );\n        }\n        let position = reader.stream_position()?;\n        let alignment = match metadata.get(\"general.alignment\") {\n            Some(Value::U8(v)) => *v as u64,\n            Some(Value::U16(v)) => *v as u64,\n            Some(Value::U32(v)) => *v as u64,\n            Some(Value::I8(v)) if *v >= 0 => *v as u64,\n            Some(Value::I16(v)) if *v >= 0 => *v as u64,\n            Some(Value::I32(v)) if *v >= 0 => *v as u64,\n            _ => DEFAULT_ALIGNMENT,\n        };\n        let tensor_data_offset = (position + alignment - 1) / alignment * alignment;\n        Ok(Self {\n            magic,\n            metadata,\n            tensor_infos,\n            tensor_data_offset,\n        })\n    }\n\n    pub fn tensor<R: std::io::Seek + std::io::Read>(\n        &self,\n        reader: &mut R,\n        name: &str,\n        device: &Device,\n    ) -> Result<QTensor> {\n        let tensor_info = match self.tensor_infos.get(name) {\n            Some(tensor_info) => tensor_info,\n            None => crate::bail!(\"cannot find tensor info for {name}\"),\n        };\n        tensor_info.read(reader, self.tensor_data_offset, device)\n    }\n}\n\nfn write_string<W: std::io::Write>(w: &mut W, str: &str) -> Result<()> {\n    let bytes = str.as_bytes();\n    w.write_u64::<LittleEndian>(bytes.len() as u64)?;\n    w.write_all(bytes)?;\n    Ok(())\n}\n\npub fn write<W: std::io::Seek + std::io::Write>(\n    w: &mut W,\n    metadata: &[(&str, &Value)],\n    tensors: &[(&str, &QTensor)],\n) -> Result<()> {\n    w.write_u32::<LittleEndian>(0x46554747)?;\n    w.write_u32::<LittleEndian>(2)?; // version 2.\n    w.write_u64::<LittleEndian>(tensors.len() as u64)?;\n    w.write_u64::<LittleEndian>(metadata.len() as u64)?;\n    for (name, value) in metadata.iter() {\n        write_string(w, name)?;\n        w.write_u32::<LittleEndian>(value.value_type().to_u32())?;\n        value.write(w)?;\n    }\n    let mut offset = 0usize;\n    let mut offsets = Vec::with_capacity(tensors.len());\n    for (name, tensor) in tensors.iter() {\n        write_string(w, name)?;\n        let dims = tensor.shape().dims();\n        w.write_u32::<LittleEndian>(dims.len() as u32)?;\n        for &dim in dims.iter().rev() {\n            w.write_u64::<LittleEndian>(dim as u64)?;\n        }\n        w.write_u32::<LittleEndian>(tensor.dtype().to_u32())?;\n        w.write_u64::<LittleEndian>(offset as u64)?;\n        offsets.push(offset);\n        let size_in_bytes = tensor.storage_size_in_bytes();\n        let padding = 31 - (31 + size_in_bytes) % 32;\n        offset += size_in_bytes + padding;\n    }\n    let pos = w.stream_position()? as usize;\n    let padding = 31 - (31 + pos) % 32;\n    w.write_all(&vec![0u8; padding])?;\n    let tensor_start_pos = w.stream_position()? as usize;\n    for (offset, (_name, tensor)) in offsets.iter().zip(tensors.iter()) {\n        let pos = w.stream_position()? as usize;\n        if tensor_start_pos + offset != pos {\n            crate::bail!(\n                \"internal error, unexpected current position {tensor_start_pos} {offset} {pos}\"\n            )\n        }\n        let data = tensor.data()?;\n        let size_in_bytes = data.len();\n        w.write_all(&data)?;\n        let padding = 31 - (31 + size_in_bytes) % 32;\n        w.write_all(&vec![0u8; padding])?;\n    }\n    Ok(())\n}\n\n// Path: candle-core/src/quantized/avx.rs\nuse super::k_quants::{\n    BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K,\n};\nuse crate::Result;\nuse byteorder::{ByteOrder, LittleEndian};\nuse half::f16;\n\n#[cfg(target_arch = \"x86\")]\nuse core::arch::x86::*;\n#[cfg(target_arch = \"x86_64\")]\nuse core::arch::x86_64::*;\n\n#[inline(always)]\npub(crate) unsafe fn sum_i16_pairs_float(x: __m256i) -> __m256 {\n    let ones = _mm256_set1_epi16(1);\n    let summed_pairs = _mm256_madd_epi16(ones, x);\n    _mm256_cvtepi32_ps(summed_pairs)\n}\n\n#[inline(always)]\npub(crate) unsafe fn mul_sum_us8_pairs_float(ax: __m256i, sy: __m256i) -> __m256 {\n    let dot = _mm256_maddubs_epi16(ax, sy);\n    sum_i16_pairs_float(dot)\n}\n\n#[inline(always)]\npub(crate) unsafe fn hsum_float_8(x: __m256) -> f32 {\n    let res = _mm256_extractf128_ps(x, 1);\n    let res = _mm_add_ps(res, _mm256_castps256_ps128(x));\n    let res = _mm_add_ps(res, _mm_movehl_ps(res, res));\n    let res = _mm_add_ss(res, _mm_movehdup_ps(res));\n    _mm_cvtss_f32(res)\n}\n\n#[inline(always)]\npub(crate) unsafe fn bytes_from_nibbles_32(rsi: *const u8) -> __m256i {\n    let tmp = _mm_loadu_si128(rsi as *const __m128i);\n    let bytes = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), _mm_srli_epi16(tmp, 4));\n    let low_mask = _mm256_set1_epi8(0xF);\n    _mm256_and_si256(low_mask, bytes)\n}\n\n#[inline(always)]\npub(crate) unsafe fn mul_sum_i8_pairs_float(x: __m256i, y: __m256i) -> __m256 {\n    let ax = _mm256_sign_epi8(x, x);\n    let sy = _mm256_sign_epi8(y, x);\n    mul_sum_us8_pairs_float(ax, sy)\n}\n\n#[inline(always)]\npub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> Result<f32> {\n    let qk = QK8_0;\n    if n % QK8_0 != 0 {\n        crate::bail!(\"vec_dot_q4_0_q8_0: {n} is not divisible by {qk}\")\n    }\n    unsafe {\n        let mut acc = _mm256_setzero_ps();\n        for (x, y) in xs.iter().zip(ys.iter()) {\n            let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d));\n            let bx = bytes_from_nibbles_32(x.qs.as_ptr());\n            let off = _mm256_set1_epi8(8);\n            let bx = _mm256_sub_epi8(bx, off);\n            let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i);\n            let q = mul_sum_i8_pairs_float(bx, by);\n            acc = _mm256_fmadd_ps(d, q, acc);\n        }\n        Ok(hsum_float_8(acc))\n    }\n}\n\n#[inline(always)]\npub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> Result<f32> {\n    let qk = QK8_0;\n    if n % QK8_0 != 0 {\n        crate::bail!(\"vec_dot_q8_0_q8_0: {n} is not divisible by {qk}\")\n    }\n    unsafe {\n        let mut acc = _mm256_setzero_ps();\n        for (x, y) in xs.iter().zip(ys.iter()) {\n            let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d));\n            let bx = _mm256_loadu_si256(x.qs.as_ptr() as *const __m256i);\n            let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i);\n            let q = mul_sum_i8_pairs_float(bx, by);\n            acc = _mm256_fmadd_ps(d, q, acc);\n        }\n        Ok(hsum_float_8(acc))\n    }\n}\n\n#[inline(always)]\nunsafe fn get_scale_shuffle(i: usize) -> __m128i {\n    const K_SHUFFLE: [u8; 128] = [\n        0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3,\n        3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7,\n        7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10,\n...\n", "needle_token_start": 7289, "needle_token_end": 7374, "code_context_ntokens": 16256, "output": ["```rust\nfn read_to_vec<T: Clone>(buffer: &Buffer, n: usize) -> Vec<T> {\n    let ptr = buffer.contents() as *const T;\n    assert!(!ptr.is_null());\n    let slice = unsafe { std::slice::from_raw_parts(ptr, n) };\n    slice.to_vec()\n}\n```"]}
{"repo": "huggingface/candle", "name": "unary_map", "language": "rust", "path": "candle-core/src/cpu_backend/utils.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function applies a transformation to each element of a slice based on a provided unary function, considering the specific memory layout of the data.\n2. **Input**: The function takes a slice of elements, a data layout descriptor, and a unary function that defines the transformation to be applied to each element.\n3. **Output**: It returns a new vector containing the transformed elements.\n4. **Procedure**: The function first determines the memory layout of the input slice, either processing a single contiguous block of data or multiple blocks. For each element in the specified blocks, the unary function is applied, and the result is stored in a new vector. The function handles different layouts by iterating over indices specified by the layout, applying the transformation function to each relevant element.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: candle-core/src/quantized/mod.rs\nuse crate::{CpuStorage, Device, Result, Shape, Storage, Tensor};\nuse k_quants::*;\nuse std::borrow::Cow;\n\n#[cfg(target_feature = \"avx\")]\npub mod avx;\nmod dummy_cuda;\nmod dummy_metal;\npub mod ggml_file;\npub mod gguf_file;\npub mod k_quants;\n#[cfg(feature = \"metal\")]\npub mod metal;\n#[cfg(not(feature = \"metal\"))]\nmod metal {\n    pub use super::dummy_metal::*;\n}\n#[cfg(feature = \"cuda\")]\npub mod cuda;\n#[cfg(not(feature = \"cuda\"))]\nmod cuda {\n    pub use super::dummy_cuda::*;\n}\n\n#[cfg(target_feature = \"neon\")]\npub mod neon;\n#[cfg(target_feature = \"simd128\")]\npub mod simd128;\npub mod utils;\nuse half::f16;\n\npub use k_quants::GgmlType;\n\npub struct QTensor {\n    storage: QStorage,\n    shape: Shape,\n}\n\nimpl Device {\n    fn qzeros(&self, elem_count: usize, dtype: GgmlDType) -> Result<QStorage> {\n        match self {\n            Device::Cpu => {\n                let storage = dtype.cpu_zeros(elem_count);\n                Ok(QStorage::Cpu(storage))\n            }\n            Device::Metal(metal) => {\n                let storage = metal::QMetalStorage::zeros(metal, elem_count, dtype)?;\n                Ok(QStorage::Metal(storage))\n            }\n            Device::Cuda(cuda) => {\n                let storage = cuda::QCudaStorage::zeros(cuda, elem_count, dtype)?;\n                Ok(QStorage::Cuda(storage))\n            }\n        }\n    }\n}\n\npub enum QStorage {\n    Cpu(Box<dyn QuantizedType>),\n    Metal(metal::QMetalStorage),\n    Cuda(cuda::QCudaStorage),\n}\n\nimpl QStorage {\n    fn block_size(&self) -> usize {\n        match self {\n            QStorage::Cpu(storage) => storage.block_size(),\n            QStorage::Metal(storage) => storage.dtype().block_size(),\n            QStorage::Cuda(storage) => storage.dtype().block_size(),\n        }\n    }\n\n    fn dtype(&self) -> GgmlDType {\n        match self {\n            QStorage::Cpu(storage) => storage.dtype(),\n            QStorage::Metal(storage) => storage.dtype(),\n            QStorage::Cuda(storage) => storage.dtype(),\n        }\n    }\n\n    fn device(&self) -> Device {\n        match self {\n...\n// Path: candle-core/src/quantized/dummy_cuda.rs\n#![allow(unused)]\nuse super::GgmlDType;\nuse crate::{CudaDevice, CudaStorage, Error, Result};\n\npub struct QCudaStorage {\n    dtype: GgmlDType,\n    device: CudaDevice,\n}\n\nimpl QCudaStorage {\n    pub fn zeros(_: &CudaDevice, _: usize, _: GgmlDType) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    pub fn dtype(&self) -> GgmlDType {\n        self.dtype\n    }\n\n    pub fn device(&self) -> &CudaDevice {\n        &self.device\n    }\n\n    pub fn dequantize(&self, _elem_count: usize) -> Result<CudaStorage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    pub fn quantize(&mut self, _src: &CudaStorage) -> Result<()> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    pub fn storage_size_in_bytes(&self) -> usize {\n        0\n    }\n\n    pub fn fwd(\n        &self,\n        _self_shape: &crate::Shape,\n        _storage: &CudaStorage,\n        _layout: &crate::Layout,\n    ) -> Result<(CudaStorage, crate::Shape)> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n}\n\npub fn load_quantized<T: super::GgmlType + Send + Sync + 'static>(\n    _device: &CudaDevice,\n    _data: &[T],\n) -> Result<super::QStorage> {\n    Err(Error::NotCompiledWithCudaSupport)\n}\n\n// Path: candle-core/src/device.rs\nuse crate::backend::BackendDevice;\nuse crate::cpu_backend::CpuDevice;\nuse crate::{CpuStorage, DType, Result, Shape, Storage, WithDType};\n\n/// A `DeviceLocation` represents a physical device whereas multiple `Device`\n/// can live on the same location (typically for cuda devices).\n#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]\npub enum DeviceLocation {\n    Cpu,\n    Cuda { gpu_id: usize },\n    Metal { gpu_id: usize },\n}\n\n#[derive(Debug, Clone)]\npub enum Device {\n    Cpu,\n    Cuda(crate::CudaDevice),\n    Metal(crate::MetalDevice),\n}\n\npub trait NdArray {\n    fn shape(&self) -> Result<Shape>;\n\n    fn to_cpu_storage(&self) -> CpuStorage;\n}\n\nimpl<S: WithDType> NdArray for S {\n    fn shape(&self) -> Result<Shape> {\n        Ok(Shape::from(()))\n    }\n\n    fn to_cpu_storage(&self) -> CpuStorage {\n        S::to_cpu_storage(&[*self])\n    }\n}\n\nimpl<S: WithDType, const N: usize> NdArray for &[S; N] {\n    fn shape(&self) -> Result<Shape> {\n        Ok(Shape::from(self.len()))\n    }\n\n    fn to_cpu_storage(&self) -> CpuStorage {\n        S::to_cpu_storage(self.as_slice())\n    }\n}\n\nimpl<S: WithDType> NdArray for &[S] {\n    fn shape(&self) -> Result<Shape> {\n        Ok(Shape::from(self.len()))\n    }\n\n    fn to_cpu_storage(&self) -> CpuStorage {\n        S::to_cpu_storage(self)\n    }\n}\n\nimpl<S: WithDType, const N: usize, const M: usize> NdArray for &[[S; N]; M] {\n    fn shape(&self) -> Result<Shape> {\n        Ok(Shape::from((M, N)))\n    }\n\n    fn to_cpu_storage(&self) -> CpuStorage {\n        S::to_cpu_storage_owned(self.concat())\n    }\n}\n\nimpl<S: WithDType, const N1: usize, const N2: usize, const N3: usize> NdArray\n    for &[[[S; N3]; N2]; N1]\n{\n    fn shape(&self) -> Result<Shape> {\n        Ok(Shape::from((N1, N2, N3)))\n    }\n\n    fn to_cpu_storage(&self) -> CpuStorage {\n        let mut vec = Vec::with_capacity(N1 * N2 * N3);\n        for i1 in 0..N1 {\n            for i2 in 0..N2 {\n                vec.extend(self[i1][i2])\n            }\n        }\n        S::to_cpu_storage_owned(vec)\n    }\n}\n\nimpl<S: WithDType, const N1: usize, const N2: usize, const N3: usize, const N4: usize> NdArray\n    for &[[[[S; N4]; N3]; N2]; N1]\n{\n    fn shape(&self) -> Result<Shape> {\n        Ok(Shape::from((N1, N2, N3, N4)))\n    }\n\n    fn to_cpu_storage(&self) -> CpuStorage {\n        let mut vec = Vec::with_capacity(N1 * N2 * N3 * N4);\n        for i1 in 0..N1 {\n            for i2 in 0..N2 {\n                for i3 in 0..N3 {\n                    vec.extend(self[i1][i2][i3])\n                }\n            }\n        }\n        S::to_cpu_storage_owned(vec)\n    }\n}\n\nimpl<S: NdArray> NdArray for Vec<S> {\n    fn shape(&self) -> Result<Shape> {\n        if self.is_empty() {\n            crate::bail!(\"empty array\")\n        }\n        let shape0 = self[0].shape()?;\n        let n = self.len();\n        for v in self.iter() {\n            let shape = v.shape()?;\n            if shape != shape0 {\n                crate::bail!(\"two elements have different shapes {shape:?} {shape0:?}\")\n            }\n        }\n        Ok(Shape::from([[n].as_slice(), shape0.dims()].concat()))\n    }\n\n    fn to_cpu_storage(&self) -> CpuStorage {\n        // This allocates intermediary memory and shouldn't be necessary.\n        let storages = self.iter().map(|v| v.to_cpu_storage()).collect::<Vec<_>>();\n        CpuStorage::concat(storages.as_slice()).unwrap()\n    }\n}\n\nimpl Device {\n    pub fn new_cuda(ordinal: usize) -> Result<Self> {\n        Ok(Self::Cuda(crate::CudaDevice::new(ordinal)?))\n    }\n\n    pub fn new_metal(ordinal: usize) -> Result<Self> {\n        Ok(Self::Metal(crate::MetalDevice::new(ordinal)?))\n    }\n\n    pub fn set_seed(&self, seed: u64) -> Result<()> {\n        match self {\n            Self::Cpu => CpuDevice.set_seed(seed),\n            Self::Cuda(c) => c.set_seed(seed),\n            Self::Metal(m) => m.set_seed(seed),\n        }\n    }\n\n    pub fn same_device(&self, rhs: &Self) -> bool {\n        match (self, rhs) {\n            (Self::Cpu, Self::Cpu) => true,\n            (Self::Cuda(lhs), Self::Cuda(rhs)) => lhs.same_device(rhs),\n            (Self::Metal(lhs), Self::Metal(rhs)) => lhs.same_device(rhs),\n            _ => false,\n        }\n    }\n\n    pub fn location(&self) -> DeviceLocation {\n        match self {\n            Self::Cpu => DeviceLocation::Cpu,\n            Self::Cuda(device) => device.location(),\n            Device::Metal(device) => device.location(),\n        }\n    }\n\n    pub fn is_cpu(&self) -> bool {\n        matches!(self, Self::Cpu)\n    }\n\n    pub fn is_cuda(&self) -> bool {\n        matches!(self, Self::Cuda(_))\n    }\n\n    pub fn is_metal(&self) -> bool {\n        matches!(self, Self::Metal(_))\n    }\n\n    pub fn cuda_if_available(ordinal: usize) -> Result<Self> {\n        if crate::utils::cuda_is_available() {\n            Self::new_cuda(ordinal)\n        } else {\n            Ok(Self::Cpu)\n        }\n    }\n\n    pub(crate) fn rand_uniform_f64(\n        &self,\n        lo: f64,\n        up: f64,\n        shape: &Shape,\n        dtype: DType,\n    ) -> Result<Storage> {\n        match self {\n            Device::Cpu => {\n                let storage = CpuDevice.rand_uniform(shape, dtype, lo, up)?;\n                Ok(Storage::Cpu(storage))\n            }\n            Device::Cuda(device) => {\n                // TODO: Remove the special case if we start supporting generating f16/bf16 directly.\n                if dtype == DType::F16 || dtype == DType::BF16 {\n                    let storage = device.rand_uniform(shape, DType::F32, lo, up)?;\n                    Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype)\n                } else {\n                    let storage = device.rand_uniform(shape, dtype, lo, up)?;\n                    Ok(Storage::Cuda(storage))\n                }\n            }\n            Device::Metal(device) => {\n                let storage = device.rand_uniform(shape, dtype, lo, up)?;\n                Ok(Storage::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn rand_uniform<T: crate::FloatDType>(\n        &self,\n        lo: T,\n        up: T,\n        shape: &Shape,\n    ) -> Result<Storage> {\n        self.rand_uniform_f64(lo.to_f64(), up.to_f64(), shape, T::DTYPE)\n    }\n\n    pub(crate) fn rand_normal_f64(\n        &self,\n        mean: f64,\n        std: f64,\n        shape: &Shape,\n        dtype: DType,\n    ) -> Result<Storage> {\n        match self {\n            Device::Cpu => {\n                let storage = CpuDevice.rand_normal(shape, dtype, mean, std)?;\n                Ok(Storage::Cpu(storage))\n            }\n            Device::Cuda(device) => {\n                // TODO: Remove the special case if we start supporting generating f16/bf16 directly.\n                if dtype == DType::F16 || dtype == DType::BF16 {\n                    let storage = device.rand_normal(shape, DType::F32, mean, std)?;\n                    Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype)\n                } else {\n                    let storage = device.rand_normal(shape, dtype, mean, std)?;\n                    Ok(Storage::Cuda(storage))\n                }\n            }\n            Device::Metal(device) => {\n                let storage = device.rand_normal(shape, dtype, mean, std)?;\n                Ok(Storage::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn rand_normal<T: crate::FloatDType>(\n        &self,\n        mean: T,\n        std: T,\n        shape: &Shape,\n    ) -> Result<Storage> {\n        self.rand_normal_f64(mean.to_f64(), std.to_f64(), shape, T::DTYPE)\n    }\n\n    pub(crate) fn ones(&self, shape: &Shape, dtype: DType) -> Result<Storage> {\n        match self {\n            Device::Cpu => {\n                let storage = CpuDevice.ones_impl(shape, dtype)?;\n                Ok(Storage::Cpu(storage))\n            }\n            Device::Cuda(device) => {\n                let storage = device.ones_impl(shape, dtype)?;\n                Ok(Storage::Cuda(storage))\n            }\n            Device::Metal(device) => {\n                let storage = device.ones_impl(shape, dtype)?;\n                Ok(Storage::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn zeros(&self, shape: &Shape, dtype: DType) -> Result<Storage> {\n        match self {\n            Device::Cpu => {\n                let storage = CpuDevice.zeros_impl(shape, dtype)?;\n                Ok(Storage::Cpu(storage))\n            }\n            Device::Cuda(device) => {\n                let storage = device.zeros_impl(shape, dtype)?;\n                Ok(Storage::Cuda(storage))\n            }\n            Device::Metal(device) => {\n                let storage = device.zeros_impl(shape, dtype)?;\n                Ok(Storage::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result<Storage> {\n        match self {\n            Device::Cpu => {\n                let storage = CpuDevice.alloc_uninit(shape, dtype)?;\n                Ok(Storage::Cpu(storage))\n            }\n            Device::Cuda(device) => {\n                let storage = device.alloc_uninit(shape, dtype)?;\n                Ok(Storage::Cuda(storage))\n            }\n            Device::Metal(device) => {\n                let storage = device.alloc_uninit(shape, dtype)?;\n                Ok(Storage::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn storage<A: NdArray>(&self, array: A) -> Result<Storage> {\n        match self {\n            Device::Cpu => Ok(Storage::Cpu(array.to_cpu_storage())),\n            Device::Cuda(device) => {\n                let storage = array.to_cpu_storage();\n                let storage = device.storage_from_cpu_storage_owned(storage)?;\n                Ok(Storage::Cuda(storage))\n            }\n            Device::Metal(device) => {\n                let storage = array.to_cpu_storage();\n                let storage = device.storage_from_cpu_storage_owned(storage)?;\n                Ok(Storage::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn storage_owned<S: WithDType>(&self, data: Vec<S>) -> Result<Storage> {\n        match self {\n            Device::Cpu => Ok(Storage::Cpu(S::to_cpu_storage_owned(data))),\n            Device::Cuda(device) => {\n                let storage = S::to_cpu_storage_owned(data);\n                let storage = device.storage_from_cpu_storage_owned(storage)?;\n                Ok(Storage::Cuda(storage))\n            }\n            Device::Metal(device) => {\n                let storage = S::to_cpu_storage_owned(data);\n                let storage = device.storage_from_cpu_storage_owned(storage)?;\n                Ok(Storage::Metal(storage))\n            }\n        }\n    }\n}\n\n// Path: candle-core/src/cpu_backend/utils.rs\n/// Helper functions to write CPU kernels.\nuse crate::backend::BackendStorage;\nuse crate::{Error, Layout, Result, WithDType};\n\ntype C = super::CpuStorage;\npub trait Map1 {\n    fn f<T: WithDType>(&self, vs: &[T], layout: &Layout) -> Result<Vec<T>>;\n\n    fn map(&self, vs: &C, layout: &Layout) -> Result<C> {\n        match vs {\n            C::U8(vs) => Ok(C::U8(self.f(vs, layout)?)),\n            C::U32(vs) => Ok(C::U32(self.f(vs, layout)?)),\n            C::I64(vs) => Ok(C::I64(self.f(vs, layout)?)),\n            C::BF16(vs) => Ok(C::BF16(self.f(vs, layout)?)),\n            C::F16(vs) => Ok(C::F16(self.f(vs, layout)?)),\n            C::F32(vs) => Ok(C::F32(self.f(vs, layout)?)),\n            C::F64(vs) => Ok(C::F64(self.f(vs, layout)?)),\n        }\n    }\n}\n\npub trait Map1Any {\n    fn f<T: WithDType, W: Fn(Vec<T>) -> C>(&self, vs: &[T], layout: &Layout, wrap: W) -> Result<C>;\n\n    fn map(&self, vs: &C, layout: &Layout) -> Result<C> {\n        match vs {\n            C::U8(vs) => Ok(self.f(vs, layout, C::U8)?),\n            C::U32(vs) => Ok(self.f(vs, layout, C::U32)?),\n            C::I64(vs) => Ok(self.f(vs, layout, C::I64)?),\n            C::BF16(vs) => Ok(self.f(vs, layout, C::BF16)?),\n            C::F16(vs) => Ok(self.f(vs, layout, C::F16)?),\n            C::F32(vs) => Ok(self.f(vs, layout, C::F32)?),\n            C::F64(vs) => Ok(self.f(vs, layout, C::F64)?),\n        }\n    }\n}\n\npub trait Map2 {\n    const OP: &'static str;\n    fn f<T: WithDType>(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result<Vec<T>>;\n\n    fn map(&self, v1: &C, l1: &Layout, v2: &C, l2: &Layout) -> Result<C> {\n        match (v1, v2) {\n            (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            (C::U32(v1), C::U32(v2)) => Ok(C::U32(self.f(v1, l1, v2, l2)?)),\n            (C::I64(v1), C::I64(v2)) => Ok(C::I64(self.f(v1, l1, v2, l2)?)),\n            (C::BF16(v1), C::BF16(v2)) => Ok(C::BF16(self.f(v1, l1, v2, l2)?)),\n            (C::F16(v1), C::F16(v2)) => Ok(C::F16(self.f(v1, l1, v2, l2)?)),\n            (C::F32(v1), C::F32(v2)) => Ok(C::F32(self.f(v1, l1, v2, l2)?)),\n            (C::F64(v1), C::F64(v2)) => Ok(C::F64(self.f(v1, l1, v2, l2)?)),\n            _ => Err(Error::DTypeMismatchBinaryOp {\n                lhs: v1.dtype(),\n                rhs: v2.dtype(),\n                op: Self::OP,\n            }\n            .bt()),\n        }\n    }\n}\n\npub trait Map2U8 {\n    const OP: &'static str;\n    fn f<T: WithDType>(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result<Vec<u8>>;\n\n    fn map(&self, v1: &C, l1: &Layout, v2: &C, l2: &Layout) -> Result<C> {\n        match (v1, v2) {\n            (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            (C::U32(v1), C::U32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            (C::I64(v1), C::I64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            (C::BF16(v1), C::BF16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            (C::F16(v1), C::F16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            (C::F32(v1), C::F32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            (C::F64(v1), C::F64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)),\n            _ => Err(Error::DTypeMismatchBinaryOp {\n                lhs: v1.dtype(),\n                rhs: v2.dtype(),\n                op: Self::OP,\n            }\n            .bt()),\n        }\n    }\n}\n\npub fn binary_map<T: Copy, U: Copy, F: FnMut(T, T) -> U>(\n    lhs_l: &Layout,\n    rhs_l: &Layout,\n    lhs: &[T],\n    rhs: &[T],\n    mut f: F,\n) -> Vec<U> {\n    match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) {\n        (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => lhs[o_l1..o_l2]\n            .iter()\n            .zip(rhs[o_r1..o_r2].iter())\n            .map(|(&l, &r)| f(l, r))\n            .collect(),\n        (Some((o_l1, o_l2)), None) => {\n            // TODO: Maybe we want to avoid going through the layout twice.\n            match rhs_l.offsets_b() {\n                Some(ob) => {\n                    let mut i_in_block = 0;\n                    let mut i_right_broadcast = 0;\n                    lhs[o_l1..o_l2]\n                        .iter()\n                        .map(|&l| {\n                            let r = unsafe { rhs.get_unchecked(i_in_block + ob.start) };\n                            i_right_broadcast += 1;\n                            if i_right_broadcast >= ob.right_broadcast {\n                                i_in_block += 1;\n                                i_right_broadcast = 0;\n                            }\n                            if i_in_block >= ob.len {\n                                i_in_block = 0\n                            }\n                            f(l, *r)\n                        })\n                        .collect()\n                }\n                None => lhs_l\n                    .strided_index()\n                    .zip(rhs_l.strided_index())\n                    .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i]))\n                    .collect(),\n            }\n        }\n        (None, Some((o_r1, o_r2))) => {\n            // TODO: Maybe we want to avoid going through the layout twice.\n            match lhs_l.offsets_b() {\n                Some(ob) => {\n                    let mut i_in_block = 0;\n                    let mut i_right_broadcast = 0;\n                    rhs[o_r1..o_r2]\n                        .iter()\n                        .map(|&r| {\n                            let l = unsafe { lhs.get_unchecked(i_in_block + ob.start) };\n                            i_right_broadcast += 1;\n                            if i_right_broadcast >= ob.right_broadcast {\n                                i_in_block += 1;\n                                i_right_broadcast = 0;\n                            }\n                            if i_in_block >= ob.len {\n                                i_in_block = 0\n                            }\n                            f(*l, r)\n                        })\n                        .collect()\n                }\n                None => lhs_l\n                    .strided_index()\n                    .zip(rhs_l.strided_index())\n                    .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i]))\n                    .collect(),\n            }\n        }\n        _ => lhs_l\n            .strided_index()\n            .zip(rhs_l.strided_index())\n            .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i]))\n            .collect(),\n    }\n}\n\n// Similar to binary_map but with vectorized variants.\npub fn binary_map_vec<T: Copy, F: FnMut(T, T) -> T, FV: FnMut(&[T], &[T], &mut [T])>(\n    lhs_l: &Layout,\n    rhs_l: &Layout,\n    lhs: &[T],\n    rhs: &[T],\n    mut f: F,\n    mut f_vec: FV,\n) -> Vec<T> {\n    let el_count = lhs_l.shape().elem_count();\n    match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) {\n        (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => {\n            let mut ys: Vec<T> = Vec::with_capacity(el_count);\n            let ys_to_set = ys.spare_capacity_mut();\n            let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) };\n            f_vec(&lhs[o_l1..o_l2], &rhs[o_r1..o_r2], ys_to_set);\n            // SAFETY: values are all set by f_vec.\n            unsafe { ys.set_len(el_count) };\n            ys\n        }\n        (Some((o_l1, o_l2)), None) => match rhs_l.offsets_b() {\n            Some(ob) if ob.right_broadcast == 1 => {\n                let rhs = &rhs[ob.start..ob.start + ob.len];\n                let mut ys: Vec<T> = Vec::with_capacity(el_count);\n                let ys_to_set = ys.spare_capacity_mut();\n                let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) };\n                let mut dst_i = 0;\n                for src_i in (o_l1..o_l2).step_by(ob.len) {\n                    f_vec(\n                        &lhs[src_i..src_i + ob.len],\n                        rhs,\n                        &mut ys_to_set[dst_i..dst_i + ob.len],\n                    );\n                    dst_i += ob.len;\n                }\n                // SAFETY: values are all set by f_vec.\n                unsafe { ys.set_len(el_count) };\n                ys\n            }\n            Some(ob) => {\n                let rhs = &rhs[ob.start..ob.start + ob.len];\n                let mut ys = lhs[o_l1..o_l2].to_vec();\n                for idx_l in 0..ob.left_broadcast {\n                    let start = idx_l * ob.len * ob.right_broadcast;\n                    for (i, &r) in rhs.iter().enumerate() {\n                        let start = start + i * ob.right_broadcast;\n                        for v in ys[start..start + ob.right_broadcast].iter_mut() {\n                            *v = f(*v, r)\n                        }\n                    }\n                }\n                ys\n            }\n            None => lhs_l\n                .strided_index()\n                .zip(rhs_l.strided_index())\n                .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i]))\n                .collect(),\n        },\n        (None, Some((o_r1, o_r2))) => match lhs_l.offsets_b() {\n            Some(ob) if ob.right_broadcast == 1 => {\n                let lhs = &lhs[ob.start..ob.start + ob.len];\n                let mut ys: Vec<T> = Vec::with_capacity(el_count);\n                let ys_to_set = ys.spare_capacity_mut();\n                let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) };\n                let mut dst_i = 0;\n                for src_i in (o_r1..o_r2).step_by(ob.len) {\n                    f_vec(\n                        lhs,\n                        &rhs[src_i..src_i + ob.len],\n                        &mut ys_to_set[dst_i..dst_i + ob.len],\n                    );\n                    dst_i += ob.len;\n                }\n                // SAFETY: values are all set by f_vec.\n                unsafe { ys.set_len(el_count) };\n                ys\n            }\n            Some(ob) => {\n                let lhs = &lhs[ob.start..ob.start + ob.len];\n                let mut ys = rhs[o_r1..o_r2].to_vec();\n                for idx_l in 0..ob.left_broadcast {\n                    let start = idx_l * ob.len * ob.right_broadcast;\n                    for (i, &l) in lhs.iter().enumerate() {\n                        let start = start + i * ob.right_broadcast;\n                        for v in ys[start..start + ob.right_broadcast].iter_mut() {\n                            *v = f(l, *v)\n                        }\n                    }\n                }\n                ys\n            }\n            None => lhs_l\n                .strided_index()\n                .zip(rhs_l.strided_index())\n                .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i]))\n                .collect(),\n        },\n        _ => lhs_l\n            .strided_index()\n            .zip(rhs_l.strided_index())\n            .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i]))\n            .collect(),\n    }\n}\n\n\npub fn unary_map<T: Copy, U: Copy, F: FnMut(T) -> U>(\n    vs: &[T],\n    layout: &Layout,\n    mut f: F,\n) -> Vec<U> {\n    match layout.strided_blocks() {\n        crate::StridedBlocks::SingleBlock { start_offset, len } => vs\n            [start_offset..start_offset + len]\n            .iter()\n            .map(|&v| f(v))\n            .collect(),\n        crate::StridedBlocks::MultipleBlocks {\n            block_start_index,\n            block_len,\n        } => {\n            let mut result = Vec::with_capacity(layout.shape().elem_count());\n            // Specialize the case where block_len is one to avoid the second loop.\n            if block_len == 1 {\n                for index in block_start_index {\n                    let v = unsafe { vs.get_unchecked(index) };\n                    result.push(f(*v))\n                }\n            } else {\n                for index in block_start_index {\n                    for offset in 0..block_len {\n                        let v = unsafe { vs.get_unchecked(index + offset) };\n                        result.push(f(*v))\n                    }\n                }\n            }\n            result\n        }\n    }\n}\n\npub fn unary_map_vec<T: Copy, U: Copy, F: FnMut(T) -> U, FV: FnMut(&[T], &mut [U])>(\n    vs: &[T],\n    layout: &Layout,\n    mut f: F,\n    mut f_vec: FV,\n) -> Vec<U> {\n    match layout.strided_blocks() {\n        crate::StridedBlocks::SingleBlock { start_offset, len } => {\n            let mut ys: Vec<U> = Vec::with_capacity(len);\n            let ys_to_set = ys.spare_capacity_mut();\n            let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) };\n            f_vec(&vs[start_offset..start_offset + len], ys_to_set);\n            // SAFETY: values are all set by f_vec.\n            unsafe { ys.set_len(len) };\n            ys\n        }\n        crate::StridedBlocks::MultipleBlocks {\n            block_start_index,\n            block_len,\n        } => {\n            let el_count = layout.shape().elem_count();\n            // Specialize the case where block_len is one to avoid the second loop.\n            if block_len == 1 {\n                let mut result = Vec::with_capacity(el_count);\n                for index in block_start_index {\n                    let v = unsafe { vs.get_unchecked(index) };\n                    result.push(f(*v))\n                }\n                result\n            } else {\n                let mut ys: Vec<U> = Vec::with_capacity(el_count);\n                let ys_to_set = ys.spare_capacity_mut();\n                let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) };\n                let mut dst_index = 0;\n                for src_index in block_start_index {\n                    let vs = &vs[src_index..src_index + block_len];\n                    let ys = &mut ys_to_set[dst_index..dst_index + block_len];\n                    f_vec(vs, ys);\n                    dst_index += block_len;\n                }\n                // SAFETY: values are all set by f_vec.\n                unsafe { ys.set_len(el_count) };\n                ys\n            }\n        }\n    }\n}\n\n// Path: candle-core/src/cpu_backend/mod.rs\nuse crate::backend::{BackendDevice, BackendStorage};\nuse crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT};\nuse crate::{DType, Error, IntDType, Layout, Result, Shape, WithDType};\nuse half::{bf16, f16};\nuse rayon::prelude::*;\n\nmod utils;\npub use utils::{\n    binary_map, binary_map_vec, unary_map, unary_map_vec, Map1, Map1Any, Map2, Map2U8,\n};\n\nconst USE_IM2COL_CONV1D: bool = true;\nconst USE_IM2COL_CONV1D_TR: bool = true;\nconst USE_IM2COL_CONV2D: bool = true;\n\n// TODO: Maybe we should not implement [Clone] here and instead have an explicit allocator +\n// intercept the oom errors to avoid panicking and provide a proper error.\n#[derive(Debug, Clone)]\npub enum CpuStorage {\n    U8(Vec<u8>),\n    U32(Vec<u32>),\n    I64(Vec<i64>),\n    BF16(Vec<bf16>),\n    F16(Vec<f16>),\n    F32(Vec<f32>),\n    F64(Vec<f64>),\n}\n\n#[derive(Debug, Clone)]\npub struct CpuDevice;\n\nstruct Cmp(CmpOp);\nimpl Map2U8 for Cmp {\n    const OP: &'static str = \"cmp\";\n    #[inline(always)]\n    fn f<T: WithDType>(\n        &self,\n        lhs: &[T],\n        lhs_l: &Layout,\n        rhs: &[T],\n        rhs_l: &Layout,\n    ) -> Result<Vec<u8>> {\n        let dst = match self.0 {\n            CmpOp::Eq => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x == y)),\n            CmpOp::Ne => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x != y)),\n            CmpOp::Lt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x < y)),\n            CmpOp::Le => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x <= y)),\n            CmpOp::Gt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x > y)),\n            CmpOp::Ge => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x >= y)),\n        };\n        Ok(dst)\n    }\n}\n\nstruct WCond<'a, T: IntDType>(&'a [T], &'a Layout);\n\nimpl<'a, I: IntDType> Map2 for WCond<'a, I> {\n    const OP: &'static str = \"where\";\n    #[inline(always)]\n    fn f<T: WithDType>(&self, t: &[T], t_l: &Layout, f: &[T], f_l: &Layout) -> Result<Vec<T>> {\n        let vs = match (\n            self.1.contiguous_offsets(),\n            t_l.contiguous_offsets(),\n            f_l.contiguous_offsets(),\n        ) {\n            (Some((o1, o2)), Some((o_t1, o_t2)), Some((o_f1, o_f2))) => {\n                let pred = &self.0[o1..o2];\n                let t = &t[o_t1..o_t2];\n                let f = &f[o_f1..o_f2];\n                pred.iter()\n                    .zip(t.iter().zip(f.iter()))\n                    .map(|(p, (&t, &f))| if p.is_true() { t } else { f })\n                    .collect::<Vec<_>>()\n            }\n            _ => self\n                .1\n                .strided_index()\n                .zip(t_l.strided_index().zip(f_l.strided_index()))\n                .map(|(i_p, (i_t, i_f))| {\n                    if self.0[i_p].is_true() {\n                        t[i_t]\n                    } else {\n                        f[i_f]\n                    }\n                })\n                .collect::<Vec<_>>(),\n        };\n        Ok(vs)\n    }\n}\n\nstruct ReduceIndex {\n    reduce_dim_index: usize,\n    use_min: bool,\n    return_index: bool,\n}\n\nimpl ReduceIndex {\n    // The value gets replaced if f(s[current_acc], s[i]) returns true.\n    #[inline(always)]\n    fn fold_impl<T, U, F, G>(&self, src: &[T], src_l: &Layout, f: F, g: G) -> Result<Vec<U>>\n    where\n        T: Clone + Copy,\n        U: Clone + Copy,\n        F: Fn(T, T) -> bool,\n        G: Fn(T, usize) -> U,\n    {\n        let reduce_dim_size = src_l.dims()[self.reduce_dim_index];\n        let reduce_dim_stride = src_l.stride()[self.reduce_dim_index];\n        let dst_len = src_l.shape().elem_count() / reduce_dim_size;\n        let mut dst: Vec<U> = Vec::with_capacity(dst_len);\n        let dst_to_set = dst.spare_capacity_mut();\n        let dst_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(dst_to_set) };\n        match src_l.contiguous_offsets() {\n            Some((o1, o2)) => {\n                let src = &src[o1..o2];\n                if reduce_dim_stride == 1 {\n                    for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() {\n                        let start_src_i = start_src_i * reduce_dim_size;\n                        let src = &src[start_src_i..start_src_i + reduce_dim_size];\n                        let mut acc = 0;\n                        let mut val = src[0];\n                        for (src_i, &s) in src.iter().enumerate() {\n                            if f(val, s) {\n                                acc = src_i;\n                                val = s\n                            }\n                        }\n                        *dst_v = g(val, acc)\n                    }\n                } else {\n                    for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() {\n                        let (p, q) = (\n                            start_src_i / reduce_dim_stride,\n                            start_src_i % reduce_dim_stride,\n                        );\n                        // start_src_i = p * reduce_dim_stride + q\n                        let start_src_i = p * reduce_dim_stride * reduce_dim_size + q;\n                        let src = &src[start_src_i..];\n                        let mut acc = 0;\n                        let mut val = src[0];\n                        for src_i in 0..reduce_dim_size {\n                            let s = src[src_i * reduce_dim_stride];\n                            if f(val, s) {\n                                acc = src_i;\n                                val = s\n                            }\n                        }\n                        *dst_v = g(val, acc)\n                    }\n                }\n            }\n            None => {\n                let l = src_l.narrow(self.reduce_dim_index, 0, 1)?;\n                for (unstr_index, src_index) in l.strided_index().enumerate() {\n                    let src = &src[src_index..];\n                    let mut acc = 0;\n                    let mut val = src[0];\n                    for src_i in 0..reduce_dim_size {\n                        let s = src[src_i * reduce_dim_stride];\n                        if f(val, s) {\n                            acc = src_i;\n                            val = s\n                        }\n                    }\n                    dst_to_set[unstr_index] = g(val, acc)\n                }\n            }\n        }\n        unsafe { dst.set_len(dst_len) };\n        Ok(dst)\n    }\n}\n\nimpl Map1Any for ReduceIndex {\n    #[inline(always)]\n    fn f<T: WithDType, W: Fn(Vec<T>) -> CpuStorage>(\n        &self,\n        src: &[T],\n        src_l: &Layout,\n        wrap: W,\n    ) -> Result<CpuStorage> {\n        if src_l.shape().elem_count() == 0 {\n            Err(Error::EmptyTensor { op: \"reduce\" }.bt())?\n        }\n        let dst = match (self.return_index, self.use_min) {\n            (false, true) => wrap(self.fold_impl(src, src_l, |x, y| x > y, |v, _i| v)?),\n            (false, false) => wrap(self.fold_impl(src, src_l, |x, y| x < y, |v, _i| v)?),\n            (true, true) => {\n                CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x > y, |_v, i| i as u32)?)\n            }\n            (true, false) => {\n                CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x < y, |_v, i| i as u32)?)\n            }\n        };\n        Ok(dst)\n    }\n}\n\nstruct ReduceSum<'a> {\n    dst_shape: &'a Shape,\n    reduce_dims: &'a [usize],\n    reduce_dims_and_stride: Vec<(usize, usize)>,\n}\n\nimpl<'a> ReduceSum<'a> {\n    #[inline(always)]\n    fn fold_impl<T>(&self, src: &[T], src_l: &Layout, start_elt: T) -> Result<Vec<T>>\n    where\n        T: WithDType,\n    {\n        let mut dst = vec![start_elt; self.dst_shape.elem_count()];\n        match src_l.contiguous_offsets() {\n            Some((o1, o2)) => {\n                let src = &src[o1..o2];\n                // Handle the case where we reduce over the last dimensions separately as it is\n                // fairly common and easy to optimize. This rely on the layout being contiguous!\n                // reduce_dims is sorted, check if it is ranging from a to n-1.\n                let reduce_over_last_dims = self\n                    .reduce_dims\n                    .iter()\n                    .rev()\n                    .enumerate()\n                    .all(|(i, &v)| v == src_l.shape().rank() - 1 - i);\n                if reduce_over_last_dims {\n                    let reduce_sz = self\n                        .reduce_dims_and_stride\n                        .iter()\n                        .map(|(u, _)| u)\n                        .product::<usize>();\n                    for (dst_i, dst_v) in dst.iter_mut().enumerate() {\n                        let src_i = dst_i * reduce_sz;\n                        unsafe {\n                            T::vec_reduce_sum(\n                                src[src_i..src_i + reduce_sz].as_ptr(),\n                                dst_v,\n                                reduce_sz,\n                            )\n                        };\n                    }\n                    return Ok(dst);\n                };\n                for (unstr_index, &src) in src.iter().enumerate() {\n                    let mut dst_index = unstr_index;\n                    // Set the reduce_dims indexes to 0.\n                    for &(dim, stride) in self.reduce_dims_and_stride.iter() {\n                        // The compiler is able to optimize the following in a single divmod op.\n                        let (pre, post) = (dst_index / stride, dst_index % stride);\n                        dst_index = (pre / dim) * stride + post;\n                    }\n                    dst[dst_index] += src;\n                }\n            }\n            None => {\n                for (unstr_index, src_index) in src_l.strided_index().enumerate() {\n                    let mut dst_index = unstr_index;\n                    // Set the reduce_dims indexes to 0.\n                    for &(dim, stride) in self.reduce_dims_and_stride.iter() {\n                        // The compiler is able to optimize the following in a single divmod op.\n                        let (pre, post) = (dst_index / stride, dst_index % stride);\n                        dst_index = (pre / dim) * stride + post;\n                    }\n                    dst[dst_index] += src[src_index];\n                }\n            }\n        }\n        Ok(dst)\n    }\n}\n\nimpl<'a> Map1 for ReduceSum<'a> {\n    #[inline(always)]\n    fn f<T: WithDType>(&self, src: &[T], src_l: &Layout) -> Result<Vec<T>> {\n        self.fold_impl(src, src_l, T::zero())\n    }\n}\n\nstruct Affine(f64, f64);\n\nimpl Map1 for Affine {\n    fn f<T: WithDType>(&self, vs: &[T], layout: &Layout) -> Result<Vec<T>> {\n        let mul = T::from_f64(self.0);\n        let add = T::from_f64(self.1);\n        Ok(unary_map(vs, layout, |v| v * mul + add))\n    }\n}\n\nstruct AvgPool2D((usize, usize), (usize, usize));\n\nimpl Map1 for AvgPool2D {\n    fn f<T: WithDType>(&self, src: &[T], layout: &Layout) -> Result<Vec<T>> {\n        // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html\n        let (k_h, k_w) = self.0;\n        let (s_h, s_w) = self.1;\n        let (b_sz, c, h, w) = layout.shape().dims4()?;\n        let stride = layout.stride();\n        let (stride_h, stride_w) = (stride[2], stride[3]);\n        let h_out = (h - k_h) / s_h + 1;\n        let w_out = (w - k_w) / s_w + 1;\n        let src_index = layout.start_offset();\n        let mut dst = vec![T::zero(); b_sz * c * h_out * w_out];\n        let scale = 1f64 / (k_h * k_w) as f64;\n        let scale = T::from_f64(scale);\n        for b_idx in 0..b_sz {\n            let dst = &mut dst[b_idx * c * h_out * w_out..];\n            let src_index = src_index + b_idx * stride[0];\n            for c_idx in 0..c {\n                let dst = &mut dst[c_idx * h_out * w_out..];\n                let src_index = src_index + c_idx * stride[1];\n                for h_idx in 0..h_out {\n                    for w_idx in 0..w_out {\n                        let mut sum = T::zero();\n                        for m in 0..k_h {\n                            for n in 0..k_w {\n                                let m = s_h * h_idx + m;\n                                let n = s_w * w_idx + n;\n                                sum += src[src_index + m * stride_h + n * stride_w]\n                            }\n                        }\n                        dst[h_idx * w_out + w_idx] = sum * scale;\n                    }\n                }\n            }\n        }\n        Ok(dst)\n    }\n}\n\nstruct MaxPool2D((usize, usize), (usize, usize));\n\nimpl Map1 for MaxPool2D {\n    fn f<T: WithDType>(&self, src: &[T], layout: &Layout) -> Result<Vec<T>> {\n        // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html\n        let (k_h, k_w) = self.0;\n        let (s_h, s_w) = self.1;\n        let (b_sz, c, h, w) = layout.shape().dims4()?;\n        let stride = layout.stride();\n        let (stride_h, stride_w) = (stride[2], stride[3]);\n        let h_out = (h - k_h) / s_h + 1;\n        let w_out = (w - k_w) / s_w + 1;\n        let src_index = layout.start_offset();\n        let mut dst = vec![T::zero(); b_sz * c * h_out * w_out];\n        for b_idx in 0..b_sz {\n            let dst = &mut dst[b_idx * c * h_out * w_out..];\n            let src_index = src_index + b_idx * stride[0];\n            for c_idx in 0..c {\n                let dst = &mut dst[c_idx * h_out * w_out..];\n                let src_index = src_index + c_idx * stride[1];\n                for h_idx in 0..h_out {\n                    for w_idx in 0..w_out {\n                        let mut largest =\n                            src[src_index + s_h * h_idx * stride_h + s_w * w_idx * stride_w];\n                        for m in 0..k_h {\n                            for n in 0..k_w {\n                                let m = s_h * h_idx + m;\n                                let n = s_w * w_idx + n;\n                                if largest < src[src_index + m * stride_h + n * stride_w] {\n                                    largest = src[src_index + m * stride_h + n * stride_w]\n                                }\n                            }\n                        }\n                        dst[h_idx * w_out + w_idx] = largest;\n                    }\n                }\n            }\n        }\n        Ok(dst)\n    }\n}\n\nstruct UpsampleNearest1D(usize);\n\nimpl Map1 for UpsampleNearest1D {\n    fn f<T: WithDType>(&self, src: &[T], layout: &Layout) -> Result<Vec<T>> {\n        // TODO: Specialized implementation for the case 2*sz?\n        let dst_sz = self.0;\n        let (b_sz, c, src_sz) = layout.shape().dims3()?;\n        let stride = layout.stride();\n        let stride_sz = stride[2];\n        let src_index = layout.start_offset();\n        let scale_sz = src_sz as f64 / dst_sz as f64;\n        let mut dst = vec![T::zero(); b_sz * c * dst_sz];\n        let src_idxs = (0..dst_sz)\n            .map(|idx| usize::min(src_sz - 1, (idx as f64 * scale_sz) as usize))\n            .collect::<Vec<_>>();\n        for b_idx in 0..b_sz {\n            let dst = &mut dst[b_idx * c * dst_sz..];\n            let src_index = src_index + b_idx * stride[0];\n            for c_idx in 0..c {\n                let dst = &mut dst[c_idx * dst_sz..];\n                let src_index = src_index + c_idx * stride[1];\n                for (idx, src_idx) in src_idxs.iter().enumerate() {\n                    dst[idx] = src[src_index + src_idx * stride_sz]\n                }\n            }\n        }\n        Ok(dst)\n    }\n}\n\nstruct UpsampleNearest2D(usize, usize);\n\nimpl Map1 for UpsampleNearest2D {\n    fn f<T: WithDType>(&self, src: &[T], layout: &Layout) -> Result<Vec<T>> {\n        // TODO: Specialized implementation for the case 2*h, 2*w?\n        let (dst_h, dst_w) = (self.0, self.1);\n        let (b_sz, c, src_h, src_w) = layout.shape().dims4()?;\n        let stride = layout.stride();\n        let (stride_h, stride_w) = (stride[2], stride[3]);\n        let src_index = layout.start_offset();\n        let scale_h = src_h as f64 / dst_h as f64;\n        let scale_w = src_w as f64 / dst_w as f64;\n        let mut dst = vec![T::zero(); b_sz * c * dst_h * dst_w];\n        let src_h_idxs = (0..dst_h)\n            .map(|h_idx| usize::min(src_h - 1, (h_idx as f64 * scale_h) as usize))\n            .collect::<Vec<_>>();\n        let src_w_idxs = (0..dst_w)\n            .map(|w_idx| usize::min(src_w - 1, (w_idx as f64 * scale_w) as usize))\n            .collect::<Vec<_>>();\n        for b_idx in 0..b_sz {\n            let dst = &mut dst[b_idx * c * dst_h * dst_w..];\n            let src_index = src_index + b_idx * stride[0];\n            for c_idx in 0..c {\n                let dst = &mut dst[c_idx * dst_h * dst_w..];\n                let src_index = src_index + c_idx * stride[1];\n                for (h_idx, src_h_idx) in src_h_idxs.iter().enumerate() {\n                    for (w_idx, src_w_idx) in src_w_idxs.iter().enumerate() {\n                        let src_index = src_index + src_h_idx * stride_h + src_w_idx * stride_w;\n                        dst[h_idx * dst_w + w_idx] = src[src_index]\n                    }\n                }\n            }\n        }\n        Ok(dst)\n    }\n}\n\nstruct Gather<'a, I: IntDType> {\n    ids: &'a [I],\n    ids_l: &'a Layout,\n    dim: usize,\n}\n\nimpl<'a, I: IntDType> Map1 for Gather<'a, I> {\n    fn f<T: WithDType>(&self, src: &[T], src_l: &Layout) -> Result<Vec<T>> {\n        let ids = match self.ids_l.contiguous_offsets() {\n            Some((a, b)) => &self.ids[a..b],\n            None => Err(Error::RequiresContiguous { op: \"gather\" }.bt())?,\n        };\n        let src = match src_l.contiguous_offsets() {\n            Some((a, b)) => &src[a..b],\n            None => Err(Error::RequiresContiguous { op: \"gather\" }.bt())?,\n        };\n        let dim = self.dim;\n        let ids_dims = self.ids_l.dims();\n        let src_dims = src_l.dims();\n        let dst_len: usize = ids_dims.iter().product();\n        let dst_left_len: usize = ids_dims[..dim].iter().product();\n        let dst_dim_len = ids_dims[dim];\n        let dst_right_len: usize = ids_dims[dim + 1..].iter().product();\n\n        let src_dim_len = src_dims[dim];\n        let src_right_len: usize = src_dims[dim + 1..].iter().product();\n\n        let mut dst = vec![T::zero(); dst_len];\n        for left_i in 0..dst_left_len {\n            let start_src_idx = left_i * src_right_len * src_dim_len;\n            let start_dst_idx = left_i * dst_right_len * dst_dim_len;\n            for i in 0..dst_dim_len {\n                let start_dst_idx = start_dst_idx + i * dst_right_len;\n                for right_i in 0..dst_right_len {\n                    let dst_idx = start_dst_idx + right_i;\n                    let index = ids[dst_idx].as_usize();\n                    if index >= src_dim_len {\n                        Err(Error::InvalidIndex {\n                            index,\n                            size: src_dim_len,\n...\n", "needle_token_start": 8776, "needle_token_end": 9109, "code_context_ntokens": 16290, "output": ["```rust\npub fn unary_map<T: Copy, U: Copy, F: FnMut(T) -> U>(\n    vs: &[T],\n    layout: &Layout,\n    mut f: F,\n) -> Vec<U> {\n    match layout.strided_blocks() {\n        crate::StridedBlocks::SingleBlock { start_offset, len } => vs\n            [start_offset..start_offset + len]\n            .iter()\n            .map(|&v| f(v))\n            .collect(),\n        crate::StridedBlocks::MultipleBlocks {\n            block_start_index,\n            block_len,\n        } => {\n            let mut result = Vec::with_capacity(layout.shape().elem_count());\n            // Specialize the case where block_len is one to avoid the second loop.\n            if block_len == 1 {\n                for index in block_start_index {\n                    let v = unsafe { vs.get_unchecked(index) };\n                    result.push(f(*v))\n                }\n            } else {\n                for index in block_start_index {\n                    for offset in 0..block_len {\n                        let v = unsafe { vs.get_unchecked(index + offset) };\n                        result.push(f(*v))\n                    }\n                }\n            }\n            result\n        }\n    }\n}\n```"]}
{"repo": "huggingface/candle", "name": "apply_op1_no_bwd", "language": "rust", "path": "candle-core/src/custom_op.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To apply a unary operation to a tensor without computing gradients for backpropagation, specifically designed for operations that do not support backward passes.\n2. **Input**: A tensor and a unary operation that conforms to a specific interface for unary operations.\n3. **Output**: A new tensor resulting from applying the unary operation to the input tensor.\n4. **Procedure**: The function retrieves the storage and layout of the input tensor, applies the unary operation, and constructs a new tensor from the resulting storage and shape. This new tensor is marked to not support backpropagation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: candle-core/src/npy.rs\n//! Numpy support for tensors.\n//!\n//! The spec for the npy format can be found in\n//! [npy-format](https://docs.scipy.org/doc/numpy-1.14.2/neps/npy-format.html).\n//! The functions from this module can be used to read tensors from npy/npz files\n//! or write tensors to these files. A npy file contains a single tensor (unnamed)\n//! whereas a npz file can contain multiple named tensors. npz files are also compressed.\n//!\n//! These two formats are easy to use in Python using the numpy library.\n//!\n//! ```python\n//! import numpy as np\n//! x = np.arange(10)\n//!\n//! # Write a npy file.\n//! np.save(\"test.npy\", x)\n//!\n//! # Read a value from the npy file.\n//! x = np.load(\"test.npy\")\n//!\n//! # Write multiple values to a npz file.\n//! values = { \"x\": x, \"x_plus_one\": x + 1 }\n//! np.savez(\"test.npz\", **values)\n//!\n//! # Load multiple values from a npz file.\n//! values = np.loadz(\"test.npz\")\n//! ```\nuse crate::{DType, Device, Error, Result, Shape, Tensor};\nuse byteorder::{LittleEndian, ReadBytesExt};\nuse half::{bf16, f16, slice::HalfFloatSliceExt};\nuse std::collections::HashMap;\nuse std::fs::File;\nuse std::io::{BufReader, Read, Write};\nuse std::path::Path;\n\nconst NPY_MAGIC_STRING: &[u8] = b\"\\x93NUMPY\";\nconst NPY_SUFFIX: &str = \".npy\";\n\nfn read_header<R: Read>(reader: &mut R) -> Result<String> {\n    let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()];\n    reader.read_exact(&mut magic_string)?;\n    if magic_string != NPY_MAGIC_STRING {\n        return Err(Error::Npy(\"magic string mismatch\".to_string()));\n    }\n    let mut version = [0u8; 2];\n    reader.read_exact(&mut version)?;\n    let header_len_len = match version[0] {\n        1 => 2,\n        2 => 4,\n        otherwise => return Err(Error::Npy(format!(\"unsupported version {otherwise}\"))),\n    };\n    let mut header_len = vec![0u8; header_len_len];\n    reader.read_exact(&mut header_len)?;\n    let header_len = header_len\n        .iter()\n        .rev()\n        .fold(0_usize, |acc, &v| 256 * acc + v as usize);\n    let mut header = vec![0u8; header_len];\n    reader.read_exact(&mut header)?;\n    Ok(String::from_utf8_lossy(&header).to_string())\n}\n\n#[derive(Debug, PartialEq)]\nstruct Header {\n    descr: DType,\n    fortran_order: bool,\n    shape: Vec<usize>,\n}\n\nimpl Header {\n    fn shape(&self) -> Shape {\n        Shape::from(self.shape.as_slice())\n    }\n\n    fn to_string(&self) -> Result<String> {\n        let fortran_order = if self.fortran_order { \"True\" } else { \"False\" };\n        let mut shape = self\n            .shape\n            .iter()\n            .map(|x| x.to_string())\n            .collect::<Vec<_>>()\n            .join(\",\");\n        let descr = match self.descr {\n            DType::BF16 => Err(Error::Npy(\"bf16 is not supported\".into()))?,\n            DType::F16 => \"f2\",\n            DType::F32 => \"f4\",\n            DType::F64 => \"f8\",\n            DType::I64 => \"i8\",\n            DType::U32 => \"u4\",\n            DType::U8 => \"u1\",\n        };\n        if !shape.is_empty() {\n            shape.push(',')\n        }\n        Ok(format!(\n            \"{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}\"\n        ))\n    }\n\n    // Hacky parser for the npy header, a typical example would be:\n    // {'descr': '<f8', 'fortran_order': False, 'shape': (128,), }\n    fn parse(header: &str) -> Result<Header> {\n        let header =\n            header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace());\n\n        let mut parts: Vec<String> = vec![];\n        let mut start_index = 0usize;\n        let mut cnt_parenthesis = 0i64;\n        for (index, c) in header.chars().enumerate() {\n            match c {\n                '(' => cnt_parenthesis += 1,\n                ')' => cnt_parenthesis -= 1,\n                ',' => {\n                    if cnt_parenthesis == 0 {\n                        parts.push(header[start_index..index].to_owned());\n                        start_index = index + 1;\n                    }\n                }\n                _ => {}\n            }\n        }\n        parts.push(header[start_index..].to_owned());\n        let mut part_map: HashMap<String, String> = HashMap::new();\n        for part in parts.iter() {\n            let part = part.trim();\n            if !part.is_empty() {\n                match part.split(':').collect::<Vec<_>>().as_slice() {\n                    [key, value] => {\n...\n// Path: candle-core/src/strided_index.rs\nuse crate::Layout;\n\n/// An iterator over offset position for items of an N-dimensional arrays stored in a\n/// flat buffer using some potential strides.\n#[derive(Debug)]\npub struct StridedIndex<'a> {\n    next_storage_index: Option<usize>,\n    multi_index: Vec<usize>,\n    dims: &'a [usize],\n    stride: &'a [usize],\n}\n\nimpl<'a> StridedIndex<'a> {\n    pub(crate) fn new(dims: &'a [usize], stride: &'a [usize], start_offset: usize) -> Self {\n        let elem_count: usize = dims.iter().product();\n        let next_storage_index = if elem_count == 0 {\n            None\n        } else {\n            // This applies to the scalar case.\n            Some(start_offset)\n        };\n        StridedIndex {\n            next_storage_index,\n            multi_index: vec![0; dims.len()],\n            dims,\n            stride,\n        }\n    }\n\n    pub(crate) fn from_layout(l: &'a Layout) -> Self {\n        Self::new(l.dims(), l.stride(), l.start_offset())\n    }\n}\n\nimpl<'a> Iterator for StridedIndex<'a> {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        let storage_index = match self.next_storage_index {\n            None => return None,\n            Some(storage_index) => storage_index,\n        };\n        let mut updated = false;\n        let mut next_storage_index = storage_index;\n        for ((multi_i, max_i), stride_i) in self\n            .multi_index\n            .iter_mut()\n            .zip(self.dims.iter())\n            .zip(self.stride.iter())\n            .rev()\n        {\n            let next_i = *multi_i + 1;\n            if next_i < *max_i {\n                *multi_i = next_i;\n                updated = true;\n                next_storage_index += stride_i;\n                break;\n            } else {\n                next_storage_index -= *multi_i * stride_i;\n                *multi_i = 0\n            }\n        }\n        self.next_storage_index = if updated {\n            Some(next_storage_index)\n        } else {\n            None\n        };\n        Some(storage_index)\n    }\n}\n\n#[derive(Debug)]\npub enum StridedBlocks<'a> {\n    SingleBlock {\n        start_offset: usize,\n        len: usize,\n    },\n    MultipleBlocks {\n        block_start_index: StridedIndex<'a>,\n        block_len: usize,\n    },\n}\n\n// Path: candle-core/src/layout.rs\nuse crate::{Error, Result, Shape};\n\n#[derive(Debug, PartialEq, Eq, Clone)]\npub struct Layout {\n    shape: Shape,\n    // The strides are given in number of elements and not in bytes.\n    stride: Vec<usize>,\n    start_offset: usize,\n}\n\nimpl Layout {\n    pub fn new(shape: Shape, stride: Vec<usize>, start_offset: usize) -> Self {\n        Self {\n            shape,\n            stride,\n            start_offset,\n        }\n    }\n\n    pub fn contiguous_with_offset<S: Into<Shape>>(shape: S, start_offset: usize) -> Self {\n        let shape = shape.into();\n        let stride = shape.stride_contiguous();\n        Self {\n            shape,\n            stride,\n            start_offset,\n        }\n    }\n\n    pub fn contiguous<S: Into<Shape>>(shape: S) -> Self {\n        Self::contiguous_with_offset(shape, 0)\n    }\n\n    pub fn dims(&self) -> &[usize] {\n        self.shape.dims()\n    }\n\n    pub fn shape(&self) -> &Shape {\n        &self.shape\n    }\n\n    pub fn stride(&self) -> &[usize] {\n        &self.stride\n    }\n\n    pub fn start_offset(&self) -> usize {\n        self.start_offset\n    }\n\n    /// Returns the appropriate start and stop offset if the data is stored in a C\n    /// contiguous (aka row major) way.\n    pub fn contiguous_offsets(&self) -> Option<(usize, usize)> {\n        if self.is_contiguous() {\n            let start_o = self.start_offset;\n            Some((start_o, start_o + self.shape.elem_count()))\n        } else {\n            None\n        }\n    }\n\n    /// Returns true if the data is stored in a C contiguous (aka row major) way.\n    /// Note that this does not implies that the start offset is 0 or that there are no extra\n    /// elements at the end of the storage.\n    pub fn is_contiguous(&self) -> bool {\n        self.shape.is_contiguous(&self.stride)\n    }\n\n    /// Returns true if the data is stored in a Fortran contiguous (aka column major) way.\n    pub fn is_fortran_contiguous(&self) -> bool {\n        self.shape.is_fortran_contiguous(&self.stride)\n    }\n\n    pub fn narrow(&self, dim: usize, start: usize, len: usize) -> Result<Self> {\n        let dims = self.shape().dims();\n        if dim >= dims.len() {\n            Err(Error::DimOutOfRange {\n                shape: self.shape().clone(),\n                dim: dim as i32,\n                op: \"narrow\",\n            }\n            .bt())?\n        }\n        if start + len > dims[dim] {\n            Err(Error::NarrowInvalidArgs {\n                shape: self.shape.clone(),\n                dim,\n                start,\n                len,\n                msg: \"start + len > dim_len\",\n            }\n            .bt())?\n        }\n        let mut dims = dims.to_vec();\n        dims[dim] = len;\n        Ok(Self {\n            shape: Shape::from(dims),\n            stride: self.stride.clone(),\n            start_offset: self.start_offset + self.stride[dim] * start,\n        })\n    }\n\n    pub fn transpose(&self, dim1: usize, dim2: usize) -> Result<Self> {\n        let rank = self.shape.rank();\n        if rank <= dim1 || rank <= dim2 {\n            Err(Error::UnexpectedNumberOfDims {\n                expected: usize::max(dim1, dim2),\n                got: rank,\n                shape: self.shape().clone(),\n            }\n            .bt())?\n        }\n        let mut stride = self.stride().to_vec();\n        let mut dims = self.shape().dims().to_vec();\n        dims.swap(dim1, dim2);\n        stride.swap(dim1, dim2);\n        Ok(Self {\n            shape: Shape::from(dims),\n            stride,\n            start_offset: self.start_offset,\n        })\n    }\n\n    pub fn permute(&self, idxs: &[usize]) -> Result<Self> {\n        let is_permutation =\n            idxs.len() == self.shape.rank() && (0..idxs.len()).all(|i| idxs.contains(&i));\n        if !is_permutation {\n            crate::bail!(\n                \"dimension mismatch in permute, tensor {:?}, dims: {:?}\",\n                self.dims(),\n                idxs\n            )\n        }\n        let stride = self.stride();\n        let dims = self.shape().dims();\n        let mut perm_stride = stride.to_vec();\n        let mut perm_dims = dims.to_vec();\n        for (i, &idx) in idxs.iter().enumerate() {\n            perm_stride[i] = stride[idx];\n            perm_dims[i] = dims[idx];\n        }\n        Ok(Self {\n            shape: Shape::from(perm_dims),\n            stride: perm_stride,\n            start_offset: self.start_offset,\n        })\n    }\n\n    pub fn broadcast_as<S: Into<Shape>>(&self, shape: S) -> Result<Self> {\n        let shape = shape.into();\n        if shape.rank() < self.shape().rank() {\n            return Err(Error::BroadcastIncompatibleShapes {\n                src_shape: self.shape().clone(),\n                dst_shape: shape,\n            }\n            .bt());\n        }\n        let added_dims = shape.rank() - self.shape().rank();\n        let mut stride = vec![0; added_dims];\n        for (&dst_dim, (&src_dim, &src_stride)) in shape.dims()[added_dims..]\n            .iter()\n            .zip(self.dims().iter().zip(self.stride()))\n        {\n            let s = if dst_dim == src_dim {\n                src_stride\n            } else if src_dim != 1 {\n                return Err(Error::BroadcastIncompatibleShapes {\n                    src_shape: self.shape().clone(),\n                    dst_shape: shape,\n                }\n                .bt());\n            } else {\n                0\n            };\n            stride.push(s)\n        }\n        Ok(Self {\n            shape,\n            stride,\n            start_offset: self.start_offset,\n        })\n    }\n\n    pub(crate) fn strided_index(&self) -> crate::StridedIndex {\n        crate::StridedIndex::from_layout(self)\n    }\n\n    pub(crate) fn strided_blocks(&self) -> crate::StridedBlocks {\n        let mut block_len = 1;\n        let mut contiguous_dims = 0; // These are counted from the right.\n        for (&stride, &dim) in self.stride().iter().zip(self.dims().iter()).rev() {\n            if stride != block_len {\n                break;\n            }\n            block_len *= dim;\n            contiguous_dims += 1;\n        }\n        let index_dims = self.dims().len() - contiguous_dims;\n        if index_dims == 0 {\n            crate::StridedBlocks::SingleBlock {\n                start_offset: self.start_offset,\n                len: block_len,\n            }\n        } else {\n            let block_start_index = crate::StridedIndex::new(\n                &self.dims()[..index_dims],\n                &self.stride[..index_dims],\n                self.start_offset,\n            );\n            crate::StridedBlocks::MultipleBlocks {\n                block_start_index,\n                block_len,\n            }\n        }\n    }\n\n    // Returns the contiguous offsets with broadcast if applicable.\n    pub(crate) fn offsets_b(&self) -> Option<ContiguousOffsetsWithBroadcast> {\n        let mut left_broadcast = 1;\n        let mut right_broadcast = 1;\n        let strides = self.stride();\n        let dims = self.dims();\n        let mut start_cont = 0;\n        let mut end_cont = dims.len();\n        for (&s, &d) in strides.iter().zip(dims.iter()) {\n            if s != 0 {\n                break;\n            }\n            start_cont += 1;\n            left_broadcast *= d;\n        }\n        if start_cont == dims.len() {\n            return Some(ContiguousOffsetsWithBroadcast {\n                start: self.start_offset,\n                len: 1,\n                left_broadcast,\n                right_broadcast: 1,\n            });\n        }\n        for (&s, &d) in strides.iter().zip(dims.iter()).rev() {\n            if s != 0 {\n                break;\n            }\n            end_cont -= 1;\n            right_broadcast *= d;\n        }\n        // Check that the inner dims are contiguous\n        let strides = &strides[start_cont..end_cont];\n        let dims = &dims[start_cont..end_cont];\n        let mut len = 1;\n        for (&stride, &dim) in strides.iter().zip(dims.iter()).rev() {\n            if stride != len {\n                return None;\n            }\n            len *= dim;\n        }\n        Some(ContiguousOffsetsWithBroadcast {\n            start: self.start_offset,\n            len,\n            left_broadcast,\n            right_broadcast,\n        })\n    }\n}\n\n#[derive(Debug, Clone, PartialEq, Eq)]\npub struct ContiguousOffsetsWithBroadcast {\n    pub start: usize,\n    pub len: usize,\n    pub left_broadcast: usize,\n    pub right_broadcast: usize,\n}\n\n// Path: candle-core/src/backend.rs\nuse crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT};\nuse crate::{CpuStorage, DType, Layout, Result, Shape};\n\npub trait BackendStorage: Sized {\n    type Device: BackendDevice;\n\n    fn try_clone(&self, _: &Layout) -> Result<Self>;\n\n    fn dtype(&self) -> DType;\n\n    fn device(&self) -> &Self::Device;\n\n    // Maybe this should return a Cow instead so that no copy is done on the cpu case.\n    fn to_cpu_storage(&self) -> Result<CpuStorage>;\n\n    fn affine(&self, _: &Layout, _: f64, _: f64) -> Result<Self>;\n\n    fn powf(&self, _: &Layout, _: f64) -> Result<Self>;\n\n    fn elu(&self, _: &Layout, _: f64) -> Result<Self>;\n\n    fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result<Self>;\n\n    fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result<Self>;\n\n    fn to_dtype(&self, _: &Layout, _: DType) -> Result<Self>;\n\n    fn unary_impl<B: UnaryOpT>(&self, _: &Layout) -> Result<Self>;\n\n    fn binary_impl<B: BinaryOpT>(&self, _: &Self, _: &Layout, _: &Layout) -> Result<Self>;\n\n    fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result<Self>;\n\n    fn conv1d(\n        &self,\n        _l: &Layout,\n        _kernel: &Self,\n        _kernel_l: &Layout,\n        _params: &crate::conv::ParamsConv1D,\n    ) -> Result<Self>;\n\n    fn conv_transpose1d(\n        &self,\n        _l: &Layout,\n        _kernel: &Self,\n        _kernel_l: &Layout,\n        _params: &crate::conv::ParamsConvTranspose1D,\n    ) -> Result<Self>;\n\n    fn conv2d(\n        &self,\n        _l: &Layout,\n        _kernel: &Self,\n        _kernel_l: &Layout,\n        _params: &crate::conv::ParamsConv2D,\n    ) -> Result<Self>;\n\n    fn conv_transpose2d(\n        &self,\n        _l: &Layout,\n        _kernel: &Self,\n        _kernel_l: &Layout,\n        _params: &crate::conv::ParamsConvTranspose2D,\n    ) -> Result<Self>;\n\n    fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result<Self>;\n    fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result<Self>;\n    fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result<Self>;\n    fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result<Self>;\n\n    fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result<Self>;\n    fn scatter_add(\n        &self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: usize,\n    ) -> Result<Self>;\n    fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result<Self>;\n    fn index_add(\n        &self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: usize,\n    ) -> Result<Self>;\n\n    fn matmul(\n        &self,\n        _: &Self,\n        _: (usize, usize, usize, usize),\n        _: &Layout,\n        _: &Layout,\n    ) -> Result<Self>;\n\n    fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()>;\n\n    #[allow(clippy::too_many_arguments)]\n    // Similar to cudaMemcpy2D, though values are in elements and not in bytes.\n    fn copy2d(\n        &self,\n        _: &mut Self,\n        _d1: usize,\n        _d2: usize,\n        _src_stride1: usize,\n        _dst_stride1: usize,\n        _src_offset: usize,\n        _dst_offset: usize,\n    ) -> Result<()>;\n}\n\npub trait BackendDevice: Sized + std::fmt::Debug + Clone {\n    type Storage: BackendStorage;\n\n    // TODO: Make the usize generic and part of a generic DeviceLocation.\n    fn new(_: usize) -> Result<Self>;\n\n    fn location(&self) -> crate::DeviceLocation;\n\n    fn same_device(&self, _: &Self) -> bool;\n\n    fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result<Self::Storage>;\n\n    fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result<Self::Storage>;\n\n    /// # Safety\n    /// This function is unsafe as it doesn't initialize the underlying data store.\n    /// The caller should ensure that the data is properly initialized as early as possible\n    /// after this call.\n    unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result<Self::Storage>;\n\n    fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result<Self::Storage>;\n\n    fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result<Self::Storage>;\n\n    fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result<Self::Storage>;\n\n    fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result<Self::Storage>;\n\n    fn set_seed(&self, _: u64) -> Result<()>;\n}\n\n// Path: candle-core/src/dummy_cuda_backend.rs\n#![allow(dead_code)]\nuse crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT};\nuse crate::{CpuStorage, DType, Error, Layout, Result, Shape};\n\n#[derive(Debug, Clone)]\npub struct CudaDevice;\n\n#[derive(Debug)]\npub struct CudaStorage;\n\nmacro_rules! fail {\n    () => {\n        unimplemented!(\"cuda support has not been enabled, add `cuda` feature to enable.\")\n    };\n}\n\nimpl crate::backend::BackendStorage for CudaStorage {\n    type Device = CudaDevice;\n\n    fn try_clone(&self, _: &Layout) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn dtype(&self) -> DType {\n        fail!()\n    }\n\n    fn device(&self) -> &Self::Device {\n        fail!()\n    }\n\n    fn to_cpu_storage(&self) -> Result<CpuStorage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn affine(&self, _: &Layout, _: f64, _: f64) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn powf(&self, _: &Layout, _: f64) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn elu(&self, _: &Layout, _: f64) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn to_dtype(&self, _: &Layout, _: DType) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn unary_impl<B: UnaryOpT>(&self, _: &Layout) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn binary_impl<B: BinaryOpT>(&self, _: &Self, _: &Layout, _: &Layout) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn conv1d(\n        &self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: &crate::conv::ParamsConv1D,\n    ) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn conv_transpose1d(\n        &self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: &crate::conv::ParamsConvTranspose1D,\n    ) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn conv2d(\n        &self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: &crate::conv::ParamsConv2D,\n    ) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn conv_transpose2d(\n        &self,\n        _l: &Layout,\n        _kernel: &Self,\n        _kernel_l: &Layout,\n        _params: &crate::conv::ParamsConvTranspose2D,\n    ) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n    fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn scatter_add(\n        &self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: usize,\n    ) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn index_add(\n        &self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: &Self,\n        _: &Layout,\n        _: usize,\n    ) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn matmul(\n        &self,\n        _: &Self,\n        _: (usize, usize, usize, usize),\n        _: &Layout,\n        _: &Layout,\n    ) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn copy2d(\n        &self,\n        _: &mut Self,\n        _: usize,\n        _: usize,\n        _: usize,\n        _: usize,\n        _: usize,\n        _: usize,\n    ) -> Result<()> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n}\n\nimpl crate::backend::BackendDevice for CudaDevice {\n    type Storage = CudaStorage;\n    fn new(_: usize) -> Result<Self> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn set_seed(&self, _: u64) -> Result<()> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn location(&self) -> crate::DeviceLocation {\n        fail!()\n    }\n\n    fn same_device(&self, _: &Self) -> bool {\n        fail!()\n    }\n\n    fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result<Self::Storage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result<Self::Storage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result<Self::Storage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result<Self::Storage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result<Self::Storage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result<Self::Storage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n\n    fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result<Self::Storage> {\n        Err(Error::NotCompiledWithCudaSupport)\n    }\n}\n\n// Path: candle-core/src/custom_op.rs\nuse crate::op::{BackpropOp, Op};\nuse crate::tensor::from_storage;\nuse crate::{CpuStorage, CudaStorage, Layout, MetalStorage, Result, Shape, Tensor};\nuse std::sync::Arc;\n\n/// Unary ops that can be defined in user-land.\npub trait CustomOp1 {\n    // Box<dyn> does not support const yet, so use a function to get the name.\n    fn name(&self) -> &'static str;\n\n    /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cpu_fwd(&self, storage: &CpuStorage, layout: &Layout) -> Result<(CpuStorage, Shape)>;\n\n    /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cuda_fwd(&self, _storage: &CudaStorage, _layout: &Layout) -> Result<(CudaStorage, Shape)> {\n        Err(crate::Error::Cuda(\n            format!(\"no cuda implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn metal_fwd(\n        &self,\n        _storage: &MetalStorage,\n        _layout: &Layout,\n    ) -> Result<(MetalStorage, Shape)> {\n        Err(crate::Error::Metal(\n            format!(\"no metal implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    /// This function takes as argument the argument `arg` used in the forward pass, the result\n    /// produced by the forward operation `res` and the gradient of the result `grad_res`.\n    /// The function should return the gradient of the argument.\n    fn bwd(&self, _arg: &Tensor, _res: &Tensor, _grad_res: &Tensor) -> Result<Option<Tensor>> {\n        Err(crate::Error::BackwardNotSupported { op: self.name() })\n    }\n}\n\npub trait CustomOp2 {\n    fn name(&self) -> &'static str;\n\n    /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cpu_fwd(\n        &self,\n        s1: &CpuStorage,\n        l1: &Layout,\n        s2: &CpuStorage,\n        l2: &Layout,\n    ) -> Result<(CpuStorage, Shape)>;\n\n    /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cuda_fwd(\n        &self,\n        _: &CudaStorage,\n        _: &Layout,\n        _: &CudaStorage,\n        _: &Layout,\n    ) -> Result<(CudaStorage, Shape)> {\n        Err(crate::Error::Cuda(\n            format!(\"no cuda implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn metal_fwd(\n        &self,\n        _: &MetalStorage,\n        _: &Layout,\n        _: &MetalStorage,\n        _: &Layout,\n    ) -> Result<(MetalStorage, Shape)> {\n        Err(crate::Error::Metal(\n            format!(\"no metal implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    fn bwd(\n        &self,\n        _arg1: &Tensor,\n        _arg2: &Tensor,\n        _res: &Tensor,\n        _grad_res: &Tensor,\n    ) -> Result<(Option<Tensor>, Option<Tensor>)> {\n        Err(crate::Error::BackwardNotSupported { op: self.name() })\n    }\n}\n\npub trait CustomOp3 {\n    fn name(&self) -> &'static str;\n\n    /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cpu_fwd(\n        &self,\n        s1: &CpuStorage,\n        l1: &Layout,\n        s2: &CpuStorage,\n        l2: &Layout,\n        s3: &CpuStorage,\n        l3: &Layout,\n    ) -> Result<(CpuStorage, Shape)>;\n\n    /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cuda_fwd(\n        &self,\n        _: &CudaStorage,\n        _: &Layout,\n        _: &CudaStorage,\n        _: &Layout,\n        _: &CudaStorage,\n        _: &Layout,\n    ) -> Result<(CudaStorage, Shape)> {\n        Err(crate::Error::Cuda(\n            format!(\"no cuda implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn metal_fwd(\n        &self,\n        _: &MetalStorage,\n        _: &Layout,\n        _: &MetalStorage,\n        _: &Layout,\n        _: &MetalStorage,\n        _: &Layout,\n    ) -> Result<(MetalStorage, Shape)> {\n        Err(crate::Error::Metal(\n            format!(\"no metal implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    fn bwd(\n        &self,\n        _arg1: &Tensor,\n        _arg2: &Tensor,\n        _arg3: &Tensor,\n        _res: &Tensor,\n        _grad_res: &Tensor,\n    ) -> Result<(Option<Tensor>, Option<Tensor>, Option<Tensor>)> {\n        Err(crate::Error::BackwardNotSupported { op: self.name() })\n    }\n}\n\nimpl Tensor {\n    /// Applies a unary custom op without backward support\n    \npub fn apply_op1_no_bwd<C: CustomOp1>(&self, c: &C) -> Result<Self> {\n        let (storage, shape) = self.storage().apply_op1(self.layout(), c)?;\n        Ok(from_storage(storage, shape, BackpropOp::none(), false))\n    }\n\n    /// Applies a binary custom op without backward support\n    pub fn apply_op2_no_bwd<C: CustomOp2>(&self, rhs: &Self, c: &C) -> Result<Self> {\n        let (storage, shape) =\n            self.storage()\n                .apply_op2(self.layout(), &rhs.storage(), rhs.layout(), c)?;\n        Ok(from_storage(storage, shape, BackpropOp::none(), false))\n    }\n\n    /// Applies a ternary custom op without backward support\n    pub fn apply_op3_no_bwd<C: CustomOp3>(&self, t2: &Self, t3: &Self, c: &C) -> Result<Self> {\n        let (storage, shape) = self.storage().apply_op3(\n            self.layout(),\n            &t2.storage(),\n            t2.layout(),\n            &t3.storage(),\n            t3.layout(),\n            c,\n        )?;\n        Ok(from_storage(storage, shape, BackpropOp::none(), false))\n    }\n\n    /// Applies a unary custom op.\n    pub fn apply_op1_arc(&self, c: Arc<Box<dyn CustomOp1 + Send + Sync>>) -> Result<Self> {\n        let (storage, shape) = self\n            .storage()\n            .apply_op1(self.layout(), c.as_ref().as_ref())?;\n        let op = BackpropOp::new1(self, |s| Op::CustomOp1(s, c.clone()));\n        Ok(from_storage(storage, shape, op, false))\n    }\n\n    pub fn apply_op1<C: 'static + CustomOp1 + Send + Sync>(&self, c: C) -> Result<Self> {\n        self.apply_op1_arc(Arc::new(Box::new(c)))\n    }\n\n    /// Applies a binary custom op.\n    pub fn apply_op2_arc(\n        &self,\n        rhs: &Self,\n        c: Arc<Box<dyn CustomOp2 + Send + Sync>>,\n    ) -> Result<Self> {\n        let (storage, shape) = self.storage().apply_op2(\n            self.layout(),\n            &rhs.storage(),\n            rhs.layout(),\n            c.as_ref().as_ref(),\n        )?;\n        let op = BackpropOp::new2(self, rhs, |t1, t2| Op::CustomOp2(t1, t2, c.clone()));\n        Ok(from_storage(storage, shape, op, false))\n    }\n\n    pub fn apply_op2<C: 'static + CustomOp2 + Send + Sync>(&self, r: &Self, c: C) -> Result<Self> {\n        self.apply_op2_arc(r, Arc::new(Box::new(c)))\n    }\n\n    /// Applies a ternary custom op.\n    pub fn apply_op3_arc(\n        &self,\n        t2: &Self,\n        t3: &Self,\n        c: Arc<Box<dyn CustomOp3 + Send + Sync>>,\n    ) -> Result<Self> {\n        let (storage, shape) = self.storage().apply_op3(\n            self.layout(),\n            &t2.storage(),\n            t2.layout(),\n            &t3.storage(),\n            t3.layout(),\n            c.as_ref().as_ref(),\n        )?;\n        let op = BackpropOp::new3(self, t2, t3, |t1, t2, t3| {\n            Op::CustomOp3(t1, t2, t3, c.clone())\n        });\n        Ok(from_storage(storage, shape, op, false))\n    }\n\n    pub fn apply_op3<C: 'static + CustomOp3 + Send + Sync>(\n        &self,\n        t2: &Self,\n        t3: &Self,\n        c: C,\n    ) -> Result<Self> {\n        self.apply_op3_arc(t2, t3, Arc::new(Box::new(c)))\n    }\n}\n\n// In place ops.\n\n/// Unary ops that can be defined in user-land.\n/// These ops work in place and as such back-prop is unsupported.\npub trait InplaceOp1 {\n    // Box<dyn> does not support const yet, so use a function to get the name.\n    fn name(&self) -> &'static str;\n\n    /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cpu_fwd(&self, storage: &mut CpuStorage, layout: &Layout) -> Result<()>;\n\n    /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cuda_fwd(&self, _storage: &mut CudaStorage, _layout: &Layout) -> Result<()> {\n        Err(crate::Error::Cuda(\n            format!(\"no cuda implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn metal_fwd(&self, _storage: &mut MetalStorage, _layout: &Layout) -> Result<()> {\n        Err(crate::Error::Metal(\n            format!(\"no metal implementation for {}\", self.name()).into(),\n        ))\n    }\n}\n\npub trait InplaceOp2 {\n    fn name(&self) -> &'static str;\n\n    /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cpu_fwd(&self, s1: &mut CpuStorage, l1: &Layout, s2: &CpuStorage, l2: &Layout)\n        -> Result<()>;\n\n    /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cuda_fwd(&self, _: &mut CudaStorage, _: &Layout, _: &CudaStorage, _: &Layout) -> Result<()> {\n        Err(crate::Error::Cuda(\n            format!(\"no cuda implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn metal_fwd(\n        &self,\n        _: &mut MetalStorage,\n        _: &Layout,\n        _: &MetalStorage,\n        _: &Layout,\n    ) -> Result<()> {\n        Err(crate::Error::Metal(\n            format!(\"no metal implementation for {}\", self.name()).into(),\n        ))\n    }\n}\n\npub trait InplaceOp3 {\n    fn name(&self) -> &'static str;\n\n    /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cpu_fwd(\n        &self,\n        s1: &mut CpuStorage,\n        l1: &Layout,\n        s2: &CpuStorage,\n        l2: &Layout,\n        s3: &CpuStorage,\n        l3: &Layout,\n    ) -> Result<()>;\n\n    /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn cuda_fwd(\n        &self,\n        _: &mut CudaStorage,\n        _: &Layout,\n        _: &CudaStorage,\n        _: &Layout,\n        _: &CudaStorage,\n        _: &Layout,\n    ) -> Result<()> {\n        Err(crate::Error::Cuda(\n            format!(\"no cuda implementation for {}\", self.name()).into(),\n        ))\n    }\n\n    /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides,\n    /// offsets etc so the associated layout should be used to access it.\n    fn metal_fwd(\n        &self,\n        _: &mut MetalStorage,\n        _: &Layout,\n        _: &MetalStorage,\n        _: &Layout,\n        _: &MetalStorage,\n        _: &Layout,\n    ) -> Result<()> {\n        Err(crate::Error::Metal(\n            format!(\"no metal implementation for {}\", self.name()).into(),\n        ))\n    }\n}\n\nimpl Tensor {\n    /// Applies a unary custom op in place.\n    pub fn inplace_op1<C: InplaceOp1>(&self, c: &C) -> Result<()> {\n        self.storage_mut().inplace_op1(self.layout(), c)\n    }\n\n    /// Applies a unary custom op in place (for the first tensor).\n    pub fn inplace_op2<C: InplaceOp2>(&self, rhs: &Self, c: &C) -> Result<()> {\n        self.storage_mut()\n            .inplace_op2(self.layout(), &rhs.storage(), rhs.layout(), c)\n    }\n\n    /// Applies a ternary custom op in place (for the first tensor).\n    pub fn inplace_op3<C: InplaceOp3>(&self, t2: &Self, t3: &Self, c: &C) -> Result<()> {\n        self.storage_mut().inplace_op3(\n            self.layout(),\n            &t2.storage(),\n            t2.layout(),\n            &t3.storage(),\n            t3.layout(),\n            c,\n        )\n    }\n}\n\n// Path: candle-core/src/storage.rs\nuse crate::backend::BackendStorage;\nuse crate::op::{self, CmpOp, ReduceOp};\nuse crate::{CpuStorage, CudaStorage, DType, Device, Error, Layout, MetalStorage, Result, Shape};\nuse crate::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3};\n\n// We do not want to implement Clone on Storage as cloning may fail because of\n// out of memory. Instead try_clone should be used.\n#[derive(Debug)]\npub enum Storage {\n    Cpu(CpuStorage),\n    Cuda(CudaStorage),\n    Metal(MetalStorage),\n}\n\nimpl Storage {\n    pub fn try_clone(&self, layout: &Layout) -> Result<Self> {\n        match self {\n            Self::Cpu(storage) => Ok(Self::Cpu(storage.clone())),\n            Self::Cuda(storage) => {\n                let storage = storage.try_clone(layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.try_clone(layout)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub fn device(&self) -> Device {\n        match self {\n            Self::Cpu(_) => Device::Cpu,\n            Self::Cuda(storage) => Device::Cuda(storage.device().clone()),\n            Self::Metal(storage) => Device::Metal(storage.device().clone()),\n        }\n    }\n\n    pub fn dtype(&self) -> DType {\n        match self {\n            Self::Cpu(storage) => storage.dtype(),\n            Self::Cuda(storage) => storage.dtype(),\n            Self::Metal(storage) => storage.dtype(),\n        }\n    }\n\n    pub(crate) fn same_device(&self, rhs: &Self, op: &'static str) -> Result<()> {\n        let lhs_device = self.device();\n        let rhs_device = rhs.device();\n        let lhs = lhs_device.location();\n        let rhs = rhs_device.location();\n        let same_device = if self.device().is_metal() {\n            // On metal, we require the device to be exactly the same rather than\n            // having the same location. In cuda this is not necessary as all CudaDevice on the\n            // same GPU will use the same cuda stream.\n            lhs_device.same_device(&rhs_device)\n        } else {\n            lhs == rhs\n        };\n        if !same_device {\n            Err(Error::DeviceMismatchBinaryOp { lhs, rhs, op }.bt())\n        } else {\n            Ok(())\n        }\n    }\n\n    pub(crate) fn same_dtype(&self, rhs: &Self, op: &'static str) -> Result<()> {\n        let lhs = self.dtype();\n        let rhs = rhs.dtype();\n        if lhs != rhs {\n            Err(Error::DTypeMismatchBinaryOp { lhs, rhs, op }.bt())\n        } else {\n            Ok(())\n        }\n    }\n\n    pub(crate) fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn powf(&self, layout: &Layout, alpha: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn elu(&self, layout: &Layout, alpha: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn cmp(\n        &self,\n        op: CmpOp,\n        rhs: &Self,\n        lhs_layout: &Layout,\n        rhs_layout: &Layout,\n    ) -> Result<Self> {\n        self.same_device(rhs, \"cmp\")?;\n        self.same_dtype(rhs, \"cmp\")?;\n        match (self, rhs) {\n            (Storage::Cpu(lhs), Storage::Cpu(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cpu(storage))\n            }\n            (Self::Cuda(lhs), Self::Cuda(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            (Self::Metal(lhs), Self::Metal(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Metal(storage))\n            }\n            (lhs, rhs) => {\n                // Should not happen because of the same device check above but we're defensive\n                // anyway.\n                Err(Error::DeviceMismatchBinaryOp {\n                    lhs: lhs.device().location(),\n                    rhs: rhs.device().location(),\n                    op: \"cmp\",\n                }\n                .bt())\n            }\n        }\n    }\n\n    pub(crate) fn reduce_op(&self, op: ReduceOp, layout: &Layout, s: &[usize]) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn apply_op1(&self, l: &Layout, c: &dyn CustomOp1) -> Result<(Self, Shape)> {\n        match self {\n            Self::Cpu(storage) => {\n                let (storage, shape) = c.cpu_fwd(storage, l)?;\n                Ok((Self::Cpu(storage), shape))\n            }\n            Self::Cuda(storage) => {\n                let (storage, shape) = c.cuda_fwd(storage, l)?;\n                Ok((Self::Cuda(storage), shape))\n            }\n            Self::Metal(storage) => {\n                let (storage, shape) = c.metal_fwd(storage, l)?;\n                Ok((Self::Metal(storage), shape))\n            }\n        }\n    }\n\n    pub(crate) fn apply_op2(\n        &self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        c: &dyn CustomOp2,\n    ) -> Result<(Self, Shape)> {\n        self.same_device(t2, c.name())?;\n        match (self, t2) {\n            (Self::Cpu(s1), Self::Cpu(s2)) => {\n                let (s, shape) = c.cpu_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Cpu(s), shape))\n            }\n            (Self::Cuda(s1), Self::Cuda(s2)) => {\n                let (s, shape) = c.cuda_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Cuda(s), shape))\n            }\n            (Self::Metal(s1), Self::Metal(s2)) => {\n                let (s, shape) = c.metal_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Metal(s), shape))\n            }\n            _ => unreachable!(),\n        }\n    }\n\n    pub(crate) fn apply_op3(\n        &self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        t3: &Self,\n        l3: &Layout,\n        c: &dyn CustomOp3,\n    ) -> Result<(Self, Shape)> {\n        self.same_device(t2, c.name())?;\n        self.same_device(t3, c.name())?;\n        match (self, t2, t3) {\n            (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => {\n                let (s, shape) = c.cpu_fwd(s1, l1, s2, l2, s3, l3)?;\n                Ok((Self::Cpu(s), shape))\n            }\n            (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => {\n                let (s, shape) = c.cuda_fwd(s1, l1, s2, l2, s3, l3)?;\n                Ok((Self::Cuda(s), shape))\n            }\n            (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => {\n                let (s, shape) = c.metal_fwd(s1, l1, s2, l2, s3, l3)?;\n                Ok((Self::Metal(s), shape))\n            }\n            _ => unreachable!(),\n        }\n    }\n\n    pub(crate) fn inplace_op1(&mut self, l: &Layout, c: &dyn InplaceOp1) -> Result<()> {\n        match self {\n            Self::Cpu(storage) => c.cpu_fwd(storage, l),\n            Self::Cuda(storage) => c.cuda_fwd(storage, l),\n            Self::Metal(storage) => c.metal_fwd(storage, l),\n        }\n    }\n\n    pub(crate) fn inplace_op2(\n        &mut self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        c: &dyn InplaceOp2,\n    ) -> Result<()> {\n        self.same_device(t2, c.name())?;\n        match (self, t2) {\n            (Self::Cpu(s1), Self::Cpu(s2)) => c.cpu_fwd(s1, l1, s2, l2),\n...\n", "needle_token_start": 10463, "needle_token_end": 10543, "code_context_ntokens": 16194, "output": ["```rust\npub fn apply_op1_no_bwd<C: CustomOp1>(&self, c: &C) -> Result<Self> {\n    let (storage, shape) = self.storage().apply_op1(self.layout(), c)?;\n    Ok(from_storage(storage, shape, BackpropOp::none(), false))\n}\n```"]}
{"repo": "huggingface/candle", "name": "write_npy", "language": "rust", "path": "candle-core/src/npy.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To save a multi-dimensional array to a file in a specific binary format commonly used for numerical data in Python environments.\n2. **Input**: A file path where the array should be saved.\n3. **Output**: A file is created or overwritten at the specified path, containing the array data in the specified format. The function returns a success or error state.\n4. **Procedure**: \n   - Open or create a file at the given path.\n   - Write a predefined magic string to identify the file format.\n   - Construct and write a header that includes data type, array order, and dimensions.\n   - Adjust the header to meet specific alignment requirements and append a newline character.\n   - Serialize the array data into the file following the header.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: candle-core/src/shape.rs\n//! The shape of a tensor is a tuple with the size of each of its dimensions.\n#![allow(clippy::redundant_closure_call)]\nuse crate::{Error, Result};\n\n#[derive(Clone, PartialEq, Eq)]\npub struct Shape(Vec<usize>);\n\npub const SCALAR: Shape = Shape(vec![]);\n\nimpl std::fmt::Debug for Shape {\n    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        write!(f, \"{:?}\", &self.dims())\n    }\n}\n\nimpl<const C: usize> From<&[usize; C]> for Shape {\n    fn from(dims: &[usize; C]) -> Self {\n        Self(dims.to_vec())\n    }\n}\n\nimpl From<&[usize]> for Shape {\n    fn from(dims: &[usize]) -> Self {\n        Self(dims.to_vec())\n    }\n}\n\nimpl From<&Shape> for Shape {\n    fn from(shape: &Shape) -> Self {\n        Self(shape.0.to_vec())\n    }\n}\n\nimpl From<()> for Shape {\n    fn from(_: ()) -> Self {\n        Self(vec![])\n    }\n}\n\nimpl From<usize> for Shape {\n    fn from(d1: usize) -> Self {\n        Self(vec![d1])\n    }\n}\n\nimpl From<(usize,)> for Shape {\n    fn from(d1: (usize,)) -> Self {\n        Self(vec![d1.0])\n    }\n}\n\nimpl From<(usize, usize)> for Shape {\n    fn from(d12: (usize, usize)) -> Self {\n        Self(vec![d12.0, d12.1])\n    }\n}\n\nimpl From<(usize, usize, usize)> for Shape {\n    fn from(d123: (usize, usize, usize)) -> Self {\n        Self(vec![d123.0, d123.1, d123.2])\n    }\n}\n\nimpl From<(usize, usize, usize, usize)> for Shape {\n    fn from(d1234: (usize, usize, usize, usize)) -> Self {\n        Self(vec![d1234.0, d1234.1, d1234.2, d1234.3])\n    }\n}\n\nimpl From<(usize, usize, usize, usize, usize)> for Shape {\n    fn from(d12345: (usize, usize, usize, usize, usize)) -> Self {\n        Self(vec![d12345.0, d12345.1, d12345.2, d12345.3, d12345.4])\n    }\n}\n\nimpl From<(usize, usize, usize, usize, usize, usize)> for Shape {\n    fn from(d123456: (usize, usize, usize, usize, usize, usize)) -> Self {\n        Self(vec![\n            d123456.0, d123456.1, d123456.2, d123456.3, d123456.4, d123456.5,\n        ])\n    }\n}\n\nimpl From<Vec<usize>> for Shape {\n    fn from(dims: Vec<usize>) -> Self {\n        Self(dims)\n    }\n}\n\nmacro_rules! extract_dims {\n    ($fn_name:ident, $cnt:tt, $dims:expr, $out_type:ty) => {\n        pub fn $fn_name(dims: &[usize]) -> Result<$out_type> {\n            if dims.len() != $cnt {\n                Err(Error::UnexpectedNumberOfDims {\n                    expected: $cnt,\n                    got: dims.len(),\n                    shape: Shape::from(dims),\n                }\n                .bt())\n            } else {\n                Ok($dims(dims))\n            }\n        }\n\n        impl Shape {\n            pub fn $fn_name(&self) -> Result<$out_type> {\n                $fn_name(self.0.as_slice())\n            }\n        }\n\n        impl crate::Tensor {\n            pub fn $fn_name(&self) -> Result<$out_type> {\n                self.shape().$fn_name()\n            }\n        }\n\n        impl std::convert::TryInto<$out_type> for Shape {\n            type Error = crate::Error;\n            fn try_into(self) -> std::result::Result<$out_type, Self::Error> {\n                self.$fn_name()\n            }\n        }\n    };\n}\n\nimpl Shape {\n    pub fn from_dims(dims: &[usize]) -> Self {\n        Self(dims.to_vec())\n    }\n\n    /// The rank is the number of dimensions, 0 for a scalar value, 1 for a vector, etc.\n    pub fn rank(&self) -> usize {\n        self.0.len()\n    }\n\n    pub fn into_dims(self) -> Vec<usize> {\n        self.0\n    }\n\n    /// The dimensions as a slice of `usize`.\n    pub fn dims(&self) -> &[usize] {\n        &self.0\n    }\n\n    /// The total number of elements, this is the product of all dimension sizes.\n    pub fn elem_count(&self) -> usize {\n        self.0.iter().product()\n    }\n\n    /// The strides given in number of elements for a contiguous n-dimensional\n    /// arrays using this shape.\n    pub(crate) fn stride_contiguous(&self) -> Vec<usize> {\n        let mut stride: Vec<_> = self\n            .0\n            .iter()\n            .rev()\n            .scan(1, |prod, u| {\n                let prod_pre_mult = *prod;\n                *prod *= u;\n                Some(prod_pre_mult)\n            })\n            .collect();\n        stride.reverse();\n        stride\n    }\n\n    /// Returns true if the strides are C contiguous (aka row major).\n    pub fn is_contiguous(&self, stride: &[usize]) -> bool {\n        if self.0.len() != stride.len() {\n            return false;\n        }\n        let mut acc = 1;\n        for (&stride, &dim) in stride.iter().zip(self.0.iter()).rev() {\n            if dim > 1 && stride != acc {\n                return false;\n            }\n            acc *= dim;\n        }\n        true\n    }\n\n    /// Returns true if the strides are Fortran contiguous (aka column major).\n    pub fn is_fortran_contiguous(&self, stride: &[usize]) -> bool {\n        if self.0.len() != stride.len() {\n            return false;\n        }\n        let mut acc = 1;\n        for (&stride, &dim) in stride.iter().zip(self.0.iter()) {\n            if dim > 1 && stride != acc {\n                return false;\n            }\n            acc *= dim;\n        }\n        true\n    }\n\n    /// Modifies the shape by adding a list of additional dimensions at the end of the existing\n    /// dimensions.\n    pub fn extend(mut self, additional_dims: &[usize]) -> Self {\n        self.0.extend(additional_dims);\n        self\n    }\n\n    /// Check whether the two shapes are compatible for broadcast, and if it is the case return the\n    /// broadcasted shape. This is to be used for binary pointwise ops.\n    pub fn broadcast_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result<Shape> {\n        let lhs = self;\n        let lhs_dims = lhs.dims();\n        let rhs_dims = rhs.dims();\n        let lhs_ndims = lhs_dims.len();\n        let rhs_ndims = rhs_dims.len();\n        let bcast_ndims = usize::max(lhs_ndims, rhs_ndims);\n        let mut bcast_dims = vec![0; bcast_ndims];\n        for (idx, bcast_value) in bcast_dims.iter_mut().enumerate() {\n            let rev_idx = bcast_ndims - idx;\n            let l_value = if lhs_ndims < rev_idx {\n                1\n            } else {\n                lhs_dims[lhs_ndims - rev_idx]\n            };\n            let r_value = if rhs_ndims < rev_idx {\n                1\n            } else {\n                rhs_dims[rhs_ndims - rev_idx]\n            };\n            *bcast_value = if l_value == r_value {\n                l_value\n            } else if l_value == 1 {\n                r_value\n            } else if r_value == 1 {\n                l_value\n            } else {\n                Err(Error::ShapeMismatchBinaryOp {\n                    lhs: lhs.clone(),\n                    rhs: rhs.clone(),\n                    op,\n                }\n                .bt())?\n            }\n        }\n        Ok(Shape::from(bcast_dims))\n    }\n\n    pub(crate) fn broadcast_shape_matmul(&self, rhs: &Self) -> Result<(Shape, Shape)> {\n        let lhs = self;\n        let lhs_dims = lhs.dims();\n        let rhs_dims = rhs.dims();\n        if lhs_dims.len() < 2 || rhs_dims.len() < 2 {\n            crate::bail!(\"only 2d matrixes are supported {lhs:?} {rhs:?}\")\n        }\n        let (m, lhs_k) = (lhs_dims[lhs_dims.len() - 2], lhs_dims[lhs_dims.len() - 1]);\n        let (rhs_k, n) = (rhs_dims[rhs_dims.len() - 2], rhs_dims[rhs_dims.len() - 1]);\n        if lhs_k != rhs_k {\n            crate::bail!(\"different inner dimensions in broadcast matmul {lhs:?} {rhs:?}\")\n        }\n\n        let lhs_b = Self::from(&lhs_dims[..lhs_dims.len() - 2]);\n        let rhs_b = Self::from(&rhs_dims[..rhs_dims.len() - 2]);\n        let bcast = lhs_b.broadcast_shape_binary_op(&rhs_b, \"broadcast_matmul\")?;\n        let bcast_dims = bcast.dims();\n\n        let bcast_lhs = [bcast_dims, &[m, lhs_k]].concat();\n        let bcast_rhs = [bcast_dims, &[rhs_k, n]].concat();\n        Ok((Shape::from(bcast_lhs), Shape::from(bcast_rhs)))\n    }\n}\n\npub trait Dim {\n    fn to_index(&self, shape: &Shape, op: &'static str) -> Result<usize>;\n    fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result<usize>;\n}\n\nimpl Dim for usize {\n    fn to_index(&self, shape: &Shape, op: &'static str) -> Result<usize> {\n        let dim = *self;\n        if dim >= shape.dims().len() {\n            Err(Error::DimOutOfRange {\n                shape: shape.clone(),\n                dim: dim as i32,\n                op,\n            }\n            .bt())?\n        } else {\n            Ok(dim)\n        }\n    }\n\n    fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result<usize> {\n        let dim = *self;\n        if dim > shape.dims().len() {\n            Err(Error::DimOutOfRange {\n                shape: shape.clone(),\n                dim: dim as i32,\n                op,\n            }\n            .bt())?\n        } else {\n            Ok(dim)\n        }\n    }\n}\n\n#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]\npub enum D {\n    Minus1,\n    Minus2,\n}\n\nimpl D {\n    fn out_of_range(&self, shape: &Shape, op: &'static str) -> Error {\n        let dim = match self {\n            Self::Minus1 => -1,\n            Self::Minus2 => -2,\n        };\n        Error::DimOutOfRange {\n            shape: shape.clone(),\n            dim,\n            op,\n        }\n        .bt()\n    }\n}\n\nimpl Dim for D {\n    fn to_index(&self, shape: &Shape, op: &'static str) -> Result<usize> {\n        let rank = shape.rank();\n        match self {\n            Self::Minus1 if rank >= 1 => Ok(rank - 1),\n            Self::Minus2 if rank >= 2 => Ok(rank - 2),\n            _ => Err(self.out_of_range(shape, op)),\n        }\n    }\n\n    fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result<usize> {\n        let rank = shape.rank();\n        match self {\n            Self::Minus1 => Ok(rank),\n            Self::Minus2 if rank >= 1 => Ok(rank - 1),\n            _ => Err(self.out_of_range(shape, op)),\n        }\n    }\n}\n\npub trait Dims: Sized {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>>;\n\n    fn to_indexes(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let dims = self.to_indexes_internal(shape, op)?;\n        for (i, &dim) in dims.iter().enumerate() {\n            if dims[..i].contains(&dim) {\n                Err(Error::DuplicateDimIndex {\n                    shape: shape.clone(),\n                    dims: dims.clone(),\n                    op,\n                }\n                .bt())?\n            }\n            if dim >= shape.rank() {\n                Err(Error::DimOutOfRange {\n                    shape: shape.clone(),\n                    dim: dim as i32,\n                    op,\n                }\n                .bt())?\n            }\n        }\n        Ok(dims)\n    }\n}\n\nimpl Dims for Vec<usize> {\n    fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result<Vec<usize>> {\n        Ok(self)\n    }\n}\n\nimpl<const N: usize> Dims for [usize; N] {\n    fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result<Vec<usize>> {\n        Ok(self.to_vec())\n    }\n}\n\nimpl Dims for &[usize] {\n    fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result<Vec<usize>> {\n        Ok(self.to_vec())\n    }\n}\n\nimpl Dims for () {\n    fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result<Vec<usize>> {\n        Ok(vec![])\n    }\n}\n\nimpl<D: Dim + Sized> Dims for D {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let dim = self.to_index(shape, op)?;\n        Ok(vec![dim])\n    }\n}\n\nimpl<D: Dim> Dims for (D,) {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let dim = self.0.to_index(shape, op)?;\n        Ok(vec![dim])\n    }\n}\n\nimpl<D1: Dim, D2: Dim> Dims for (D1, D2) {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let d0 = self.0.to_index(shape, op)?;\n        let d1 = self.1.to_index(shape, op)?;\n        Ok(vec![d0, d1])\n    }\n}\n\nimpl<D1: Dim, D2: Dim, D3: Dim> Dims for (D1, D2, D3) {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let d0 = self.0.to_index(shape, op)?;\n        let d1 = self.1.to_index(shape, op)?;\n        let d2 = self.2.to_index(shape, op)?;\n        Ok(vec![d0, d1, d2])\n    }\n}\n\nimpl<D1: Dim, D2: Dim, D3: Dim, D4: Dim> Dims for (D1, D2, D3, D4) {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let d0 = self.0.to_index(shape, op)?;\n        let d1 = self.1.to_index(shape, op)?;\n        let d2 = self.2.to_index(shape, op)?;\n        let d3 = self.3.to_index(shape, op)?;\n        Ok(vec![d0, d1, d2, d3])\n    }\n}\n\nimpl<D1: Dim, D2: Dim, D3: Dim, D4: Dim, D5: Dim> Dims for (D1, D2, D3, D4, D5) {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let d0 = self.0.to_index(shape, op)?;\n        let d1 = self.1.to_index(shape, op)?;\n        let d2 = self.2.to_index(shape, op)?;\n        let d3 = self.3.to_index(shape, op)?;\n        let d4 = self.4.to_index(shape, op)?;\n        Ok(vec![d0, d1, d2, d3, d4])\n    }\n}\n\nimpl<D1: Dim, D2: Dim, D3: Dim, D4: Dim, D5: Dim, D6: Dim> Dims for (D1, D2, D3, D4, D5, D6) {\n    fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result<Vec<usize>> {\n        let d0 = self.0.to_index(shape, op)?;\n        let d1 = self.1.to_index(shape, op)?;\n        let d2 = self.2.to_index(shape, op)?;\n        let d3 = self.3.to_index(shape, op)?;\n        let d4 = self.4.to_index(shape, op)?;\n        let d5 = self.5.to_index(shape, op)?;\n        Ok(vec![d0, d1, d2, d3, d4, d5])\n    }\n}\n\nextract_dims!(dims0, 0, |_: &[usize]| (), ());\nextract_dims!(dims1, 1, |d: &[usize]| d[0], usize);\nextract_dims!(dims2, 2, |d: &[usize]| (d[0], d[1]), (usize, usize));\nextract_dims!(\n    dims3,\n    3,\n    |d: &[usize]| (d[0], d[1], d[2]),\n    (usize, usize, usize)\n);\nextract_dims!(\n    dims4,\n    4,\n    |d: &[usize]| (d[0], d[1], d[2], d[3]),\n    (usize, usize, usize, usize)\n);\nextract_dims!(\n    dims5,\n    5,\n    |d: &[usize]| (d[0], d[1], d[2], d[3], d[4]),\n    (usize, usize, usize, usize, usize)\n);\n\npub trait ShapeWithOneHole {\n    fn into_shape(self, el_count: usize) -> Result<Shape>;\n}\n\nimpl<S: Into<Shape>> ShapeWithOneHole for S {\n    fn into_shape(self, _el_count: usize) -> Result<Shape> {\n        Ok(self.into())\n    }\n}\n\nimpl ShapeWithOneHole for ((),) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        Ok(el_count.into())\n    }\n}\n\nfn hole_size(el_count: usize, prod_d: usize, s: &dyn std::fmt::Debug) -> Result<usize> {\n    if prod_d == 0 {\n        crate::bail!(\"cannot reshape tensor of {el_count} elements to {s:?}\")\n    }\n    if el_count % prod_d != 0 {\n        crate::bail!(\"cannot reshape tensor with {el_count} elements to {s:?}\")\n    }\n    Ok(el_count / prod_d)\n}\n\nimpl ShapeWithOneHole for ((), usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let ((), d1) = self;\n        Ok((hole_size(el_count, d1, &self)?, d1).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, ()) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, ()) = self;\n        Ok((d1, hole_size(el_count, d1, &self)?).into())\n    }\n}\n\nimpl ShapeWithOneHole for ((), usize, usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let ((), d1, d2) = self;\n        Ok((hole_size(el_count, d1 * d2, &self)?, d1, d2).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, (), usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, (), d2) = self;\n        Ok((d1, hole_size(el_count, d1 * d2, &self)?, d2).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, usize, ()) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, d2, ()) = self;\n        Ok((d1, d2, hole_size(el_count, d1 * d2, &self)?).into())\n    }\n}\n\nimpl ShapeWithOneHole for ((), usize, usize, usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let ((), d1, d2, d3) = self;\n        let d = hole_size(el_count, d1 * d2 * d3, &self)?;\n        Ok((d, d1, d2, d3).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, (), usize, usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, (), d2, d3) = self;\n        let d = hole_size(el_count, d1 * d2 * d3, &self)?;\n        Ok((d1, d, d2, d3).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, usize, (), usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, d2, (), d3) = self;\n        let d = hole_size(el_count, d1 * d2 * d3, &self)?;\n        Ok((d1, d2, d, d3).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, usize, usize, ()) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, d2, d3, ()) = self;\n        let d = hole_size(el_count, d1 * d2 * d3, &self)?;\n        Ok((d1, d2, d3, d).into())\n    }\n}\n\nimpl ShapeWithOneHole for ((), usize, usize, usize, usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let ((), d1, d2, d3, d4) = self;\n        let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?;\n        Ok((d, d1, d2, d3, d4).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, (), usize, usize, usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, (), d2, d3, d4) = self;\n        let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?;\n        Ok((d1, d, d2, d3, d4).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, usize, (), usize, usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, d2, (), d3, d4) = self;\n        let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?;\n        Ok((d1, d2, d, d3, d4).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, usize, usize, (), usize) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, d2, d3, (), d4) = self;\n        let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?;\n        Ok((d1, d2, d3, d, d4).into())\n    }\n}\n\nimpl ShapeWithOneHole for (usize, usize, usize, usize, ()) {\n    fn into_shape(self, el_count: usize) -> Result<Shape> {\n        let (d1, d2, d3, d4, ()) = self;\n        let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?;\n        Ok((d1, d2, d3, d4, d).into())\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn stride() {\n        let shape = Shape::from(());\n        assert_eq!(shape.stride_contiguous(), Vec::<usize>::new());\n        let shape = Shape::from(42);\n        assert_eq!(shape.stride_contiguous(), [1]);\n        let shape = Shape::from((42, 1337));\n        assert_eq!(shape.stride_contiguous(), [1337, 1]);\n        let shape = Shape::from((299, 792, 458));\n        assert_eq!(shape.stride_contiguous(), [458 * 792, 458, 1]);\n    }\n}\n\n// Path: candle-core/src/npy.rs\n//! Numpy support for tensors.\n//!\n//! The spec for the npy format can be found in\n//! [npy-format](https://docs.scipy.org/doc/numpy-1.14.2/neps/npy-format.html).\n//! The functions from this module can be used to read tensors from npy/npz files\n//! or write tensors to these files. A npy file contains a single tensor (unnamed)\n//! whereas a npz file can contain multiple named tensors. npz files are also compressed.\n//!\n//! These two formats are easy to use in Python using the numpy library.\n//!\n//! ```python\n//! import numpy as np\n//! x = np.arange(10)\n//!\n//! # Write a npy file.\n//! np.save(\"test.npy\", x)\n//!\n//! # Read a value from the npy file.\n//! x = np.load(\"test.npy\")\n//!\n//! # Write multiple values to a npz file.\n//! values = { \"x\": x, \"x_plus_one\": x + 1 }\n//! np.savez(\"test.npz\", **values)\n//!\n//! # Load multiple values from a npz file.\n//! values = np.loadz(\"test.npz\")\n//! ```\nuse crate::{DType, Device, Error, Result, Shape, Tensor};\nuse byteorder::{LittleEndian, ReadBytesExt};\nuse half::{bf16, f16, slice::HalfFloatSliceExt};\nuse std::collections::HashMap;\nuse std::fs::File;\nuse std::io::{BufReader, Read, Write};\nuse std::path::Path;\n\nconst NPY_MAGIC_STRING: &[u8] = b\"\\x93NUMPY\";\nconst NPY_SUFFIX: &str = \".npy\";\n\nfn read_header<R: Read>(reader: &mut R) -> Result<String> {\n    let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()];\n    reader.read_exact(&mut magic_string)?;\n    if magic_string != NPY_MAGIC_STRING {\n        return Err(Error::Npy(\"magic string mismatch\".to_string()));\n    }\n    let mut version = [0u8; 2];\n    reader.read_exact(&mut version)?;\n    let header_len_len = match version[0] {\n        1 => 2,\n        2 => 4,\n        otherwise => return Err(Error::Npy(format!(\"unsupported version {otherwise}\"))),\n    };\n    let mut header_len = vec![0u8; header_len_len];\n    reader.read_exact(&mut header_len)?;\n    let header_len = header_len\n        .iter()\n        .rev()\n        .fold(0_usize, |acc, &v| 256 * acc + v as usize);\n    let mut header = vec![0u8; header_len];\n    reader.read_exact(&mut header)?;\n    Ok(String::from_utf8_lossy(&header).to_string())\n}\n\n#[derive(Debug, PartialEq)]\nstruct Header {\n    descr: DType,\n    fortran_order: bool,\n    shape: Vec<usize>,\n}\n\nimpl Header {\n    fn shape(&self) -> Shape {\n        Shape::from(self.shape.as_slice())\n    }\n\n    fn to_string(&self) -> Result<String> {\n        let fortran_order = if self.fortran_order { \"True\" } else { \"False\" };\n        let mut shape = self\n            .shape\n            .iter()\n            .map(|x| x.to_string())\n            .collect::<Vec<_>>()\n            .join(\",\");\n        let descr = match self.descr {\n            DType::BF16 => Err(Error::Npy(\"bf16 is not supported\".into()))?,\n            DType::F16 => \"f2\",\n            DType::F32 => \"f4\",\n            DType::F64 => \"f8\",\n            DType::I64 => \"i8\",\n            DType::U32 => \"u4\",\n            DType::U8 => \"u1\",\n        };\n        if !shape.is_empty() {\n            shape.push(',')\n        }\n        Ok(format!(\n            \"{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}\"\n        ))\n    }\n\n    // Hacky parser for the npy header, a typical example would be:\n    // {'descr': '<f8', 'fortran_order': False, 'shape': (128,), }\n    fn parse(header: &str) -> Result<Header> {\n        let header =\n            header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace());\n\n        let mut parts: Vec<String> = vec![];\n        let mut start_index = 0usize;\n        let mut cnt_parenthesis = 0i64;\n        for (index, c) in header.chars().enumerate() {\n            match c {\n                '(' => cnt_parenthesis += 1,\n                ')' => cnt_parenthesis -= 1,\n                ',' => {\n                    if cnt_parenthesis == 0 {\n                        parts.push(header[start_index..index].to_owned());\n                        start_index = index + 1;\n                    }\n                }\n                _ => {}\n            }\n        }\n        parts.push(header[start_index..].to_owned());\n        let mut part_map: HashMap<String, String> = HashMap::new();\n        for part in parts.iter() {\n            let part = part.trim();\n            if !part.is_empty() {\n                match part.split(':').collect::<Vec<_>>().as_slice() {\n                    [key, value] => {\n                        let key = key.trim_matches(|c: char| c == '\\'' || c.is_whitespace());\n                        let value = value.trim_matches(|c: char| c == '\\'' || c.is_whitespace());\n                        let _ = part_map.insert(key.to_owned(), value.to_owned());\n                    }\n                    _ => return Err(Error::Npy(format!(\"unable to parse header {header}\"))),\n                }\n            }\n        }\n        let fortran_order = match part_map.get(\"fortran_order\") {\n            None => false,\n            Some(fortran_order) => match fortran_order.as_ref() {\n                \"False\" => false,\n                \"True\" => true,\n                _ => return Err(Error::Npy(format!(\"unknown fortran_order {fortran_order}\"))),\n            },\n        };\n        let descr = match part_map.get(\"descr\") {\n            None => return Err(Error::Npy(\"no descr in header\".to_string())),\n            Some(descr) => {\n                if descr.is_empty() {\n                    return Err(Error::Npy(\"empty descr\".to_string()));\n                }\n                if descr.starts_with('>') {\n                    return Err(Error::Npy(format!(\"little-endian descr {descr}\")));\n                }\n                // the only supported types in tensor are:\n                //     float64, float32, float16,\n                //     complex64, complex128,\n                //     int64, int32, int16, int8,\n                //     uint8, and bool.\n                match descr.trim_matches(|c: char| c == '=' || c == '<' || c == '|') {\n                    \"e\" | \"f2\" => DType::F16,\n                    \"f\" | \"f4\" => DType::F32,\n                    \"d\" | \"f8\" => DType::F64,\n                    // \"i\" | \"i4\" => DType::S32,\n                    \"q\" | \"i8\" => DType::I64,\n                    // \"h\" | \"i2\" => DType::S16,\n                    // \"b\" | \"i1\" => DType::S8,\n                    \"B\" | \"u1\" => DType::U8,\n                    \"I\" | \"u4\" => DType::U32,\n                    \"?\" | \"b1\" => DType::U8,\n                    // \"F\" | \"F4\" => DType::C64,\n                    // \"D\" | \"F8\" => DType::C128,\n                    descr => return Err(Error::Npy(format!(\"unrecognized descr {descr}\"))),\n                }\n            }\n        };\n        let shape = match part_map.get(\"shape\") {\n            None => return Err(Error::Npy(\"no shape in header\".to_string())),\n            Some(shape) => {\n                let shape = shape.trim_matches(|c: char| c == '(' || c == ')' || c == ',');\n                if shape.is_empty() {\n                    vec![]\n                } else {\n                    shape\n                        .split(',')\n                        .map(|v| v.trim().parse::<usize>())\n                        .collect::<std::result::Result<Vec<_>, _>>()?\n                }\n            }\n        };\n        Ok(Header {\n            descr,\n            fortran_order,\n            shape,\n        })\n    }\n}\n\nimpl Tensor {\n    // TODO: Add the possibility to read directly to a device?\n    pub(crate) fn from_reader<R: std::io::Read>(\n        shape: Shape,\n        dtype: DType,\n        reader: &mut R,\n    ) -> Result<Self> {\n        let elem_count = shape.elem_count();\n        match dtype {\n            DType::BF16 => {\n                let mut data_t = vec![bf16::ZERO; elem_count];\n                reader.read_u16_into::<LittleEndian>(data_t.reinterpret_cast_mut())?;\n                Tensor::from_vec(data_t, shape, &Device::Cpu)\n            }\n            DType::F16 => {\n                let mut data_t = vec![f16::ZERO; elem_count];\n                reader.read_u16_into::<LittleEndian>(data_t.reinterpret_cast_mut())?;\n                Tensor::from_vec(data_t, shape, &Device::Cpu)\n            }\n            DType::F32 => {\n                let mut data_t = vec![0f32; elem_count];\n                reader.read_f32_into::<LittleEndian>(&mut data_t)?;\n                Tensor::from_vec(data_t, shape, &Device::Cpu)\n            }\n            DType::F64 => {\n                let mut data_t = vec![0f64; elem_count];\n                reader.read_f64_into::<LittleEndian>(&mut data_t)?;\n                Tensor::from_vec(data_t, shape, &Device::Cpu)\n            }\n            DType::U8 => {\n                let mut data_t = vec![0u8; elem_count];\n                reader.read_exact(&mut data_t)?;\n                Tensor::from_vec(data_t, shape, &Device::Cpu)\n            }\n            DType::U32 => {\n                let mut data_t = vec![0u32; elem_count];\n                reader.read_u32_into::<LittleEndian>(&mut data_t)?;\n                Tensor::from_vec(data_t, shape, &Device::Cpu)\n            }\n            DType::I64 => {\n                let mut data_t = vec![0i64; elem_count];\n                reader.read_i64_into::<LittleEndian>(&mut data_t)?;\n                Tensor::from_vec(data_t, shape, &Device::Cpu)\n            }\n        }\n    }\n\n    /// Reads a npy file and return the stored multi-dimensional array as a tensor.\n    pub fn read_npy<T: AsRef<Path>>(path: T) -> Result<Self> {\n        let mut reader = File::open(path.as_ref())?;\n        let header = read_header(&mut reader)?;\n        let header = Header::parse(&header)?;\n        if header.fortran_order {\n            return Err(Error::Npy(\"fortran order not supported\".to_string()));\n        }\n        Self::from_reader(header.shape(), header.descr, &mut reader)\n    }\n\n    /// Reads a npz file and returns the stored multi-dimensional arrays together with their names.\n    pub fn read_npz<T: AsRef<Path>>(path: T) -> Result<Vec<(String, Self)>> {\n        let zip_reader = BufReader::new(File::open(path.as_ref())?);\n        let mut zip = zip::ZipArchive::new(zip_reader)?;\n        let mut result = vec![];\n        for i in 0..zip.len() {\n            let mut reader = zip.by_index(i)?;\n            let name = {\n                let name = reader.name();\n                name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned()\n            };\n            let header = read_header(&mut reader)?;\n            let header = Header::parse(&header)?;\n            if header.fortran_order {\n                return Err(Error::Npy(\"fortran order not supported\".to_string()));\n            }\n            let s = Self::from_reader(header.shape(), header.descr, &mut reader)?;\n            result.push((name, s))\n        }\n        Ok(result)\n    }\n\n    /// Reads a npz file and returns the stored multi-dimensional arrays for some specified names.\n    pub fn read_npz_by_name<T: AsRef<Path>>(path: T, names: &[&str]) -> Result<Vec<Self>> {\n        let zip_reader = BufReader::new(File::open(path.as_ref())?);\n        let mut zip = zip::ZipArchive::new(zip_reader)?;\n        let mut result = vec![];\n        for name in names.iter() {\n            let mut reader = match zip.by_name(&format!(\"{name}{NPY_SUFFIX}\")) {\n                Ok(reader) => reader,\n                Err(_) => Err(Error::Npy(format!(\n                    \"no array for {name} in {:?}\",\n                    path.as_ref()\n                )))?,\n            };\n            let header = read_header(&mut reader)?;\n            let header = Header::parse(&header)?;\n            if header.fortran_order {\n                return Err(Error::Npy(\"fortran order not supported\".to_string()));\n            }\n            let s = Self::from_reader(header.shape(), header.descr, &mut reader)?;\n            result.push(s)\n        }\n        Ok(result)\n    }\n\n    fn write<T: Write>(&self, f: &mut T) -> Result<()> {\n        f.write_all(NPY_MAGIC_STRING)?;\n        f.write_all(&[1u8, 0u8])?;\n        let header = Header {\n            descr: self.dtype(),\n            fortran_order: false,\n            shape: self.dims().to_vec(),\n        };\n        let mut header = header.to_string()?;\n        let pad = 16 - (NPY_MAGIC_STRING.len() + 5 + header.len()) % 16;\n        for _ in 0..pad % 16 {\n            header.push(' ')\n        }\n        header.push('\\n');\n        f.write_all(&[(header.len() % 256) as u8, (header.len() / 256) as u8])?;\n        f.write_all(header.as_bytes())?;\n        self.write_bytes(f)\n    }\n\n    /// Writes a multi-dimensional array in the npy format.\n    \npub fn write_npy<T: AsRef<Path>>(&self, path: T) -> Result<()> {\n        let mut f = File::create(path.as_ref())?;\n        self.write(&mut f)\n    }\n\n    /// Writes multiple multi-dimensional arrays using the npz format.\n    pub fn write_npz<S: AsRef<str>, T: AsRef<Tensor>, P: AsRef<Path>>(\n        ts: &[(S, T)],\n        path: P,\n    ) -> Result<()> {\n        let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?);\n        let options =\n            zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored);\n\n        for (name, tensor) in ts.iter() {\n            zip.start_file(format!(\"{}.npy\", name.as_ref()), options)?;\n            tensor.as_ref().write(&mut zip)?\n        }\n        Ok(())\n    }\n}\n\n/// Lazy tensor loader.\npub struct NpzTensors {\n    index_per_name: HashMap<String, usize>,\n    path: std::path::PathBuf,\n    // We do not store a zip reader as it needs mutable access to extract data. Instead we\n    // re-create a zip reader for each tensor.\n}\n\nimpl NpzTensors {\n    pub fn new<T: AsRef<Path>>(path: T) -> Result<Self> {\n        let path = path.as_ref().to_owned();\n        let zip_reader = BufReader::new(File::open(&path)?);\n        let mut zip = zip::ZipArchive::new(zip_reader)?;\n        let mut index_per_name = HashMap::new();\n        for i in 0..zip.len() {\n            let file = zip.by_index(i)?;\n            let name = {\n                let name = file.name();\n                name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned()\n            };\n            index_per_name.insert(name, i);\n        }\n        Ok(Self {\n            index_per_name,\n            path,\n        })\n    }\n\n    pub fn names(&self) -> Vec<&String> {\n        self.index_per_name.keys().collect()\n    }\n\n    /// This only returns the shape and dtype for a named tensor. Compared to `get`, this avoids\n    /// reading the whole tensor data.\n    pub fn get_shape_and_dtype(&self, name: &str) -> Result<(Shape, DType)> {\n        let index = match self.index_per_name.get(name) {\n            None => crate::bail!(\"cannot find tensor {name}\"),\n            Some(index) => *index,\n        };\n        let zip_reader = BufReader::new(File::open(&self.path)?);\n        let mut zip = zip::ZipArchive::new(zip_reader)?;\n        let mut reader = zip.by_index(index)?;\n        let header = read_header(&mut reader)?;\n        let header = Header::parse(&header)?;\n        Ok((header.shape(), header.descr))\n    }\n\n    pub fn get(&self, name: &str) -> Result<Option<Tensor>> {\n        let index = match self.index_per_name.get(name) {\n            None => return Ok(None),\n            Some(index) => *index,\n        };\n        // We hope that the file has not changed since first reading it.\n        let zip_reader = BufReader::new(File::open(&self.path)?);\n        let mut zip = zip::ZipArchive::new(zip_reader)?;\n        let mut reader = zip.by_index(index)?;\n        let header = read_header(&mut reader)?;\n        let header = Header::parse(&header)?;\n        if header.fortran_order {\n            return Err(Error::Npy(\"fortran order not supported\".to_string()));\n        }\n        let tensor = Tensor::from_reader(header.shape(), header.descr, &mut reader)?;\n        Ok(Some(tensor))\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::Header;\n\n    #[test]\n    fn parse() {\n        let h = \"{'descr': '<f8', 'fortran_order': False, 'shape': (128,), }\";\n        assert_eq!(\n            Header::parse(h).unwrap(),\n            Header {\n                descr: crate::DType::F64,\n                fortran_order: false,\n                shape: vec![128]\n            }\n        );\n        let h = \"{'descr': '<f4', 'fortran_order': True, 'shape': (256,1,128), }\";\n        let h = Header::parse(h).unwrap();\n        assert_eq!(\n            h,\n            Header {\n                descr: crate::DType::F32,\n                fortran_order: true,\n                shape: vec![256, 1, 128]\n            }\n        );\n        assert_eq!(\n            h.to_string().unwrap(),\n            \"{'descr': '<f4', 'fortran_order': True, 'shape': (256,1,128,), }\"\n        );\n\n        let h = Header {\n            descr: crate::DType::U32,\n            fortran_order: false,\n            shape: vec![],\n        };\n        assert_eq!(\n            h.to_string().unwrap(),\n            \"{'descr': '<u4', 'fortran_order': False, 'shape': (), }\"\n        );\n    }\n}\n\n// Path: candle-core/src/strided_index.rs\nuse crate::Layout;\n\n/// An iterator over offset position for items of an N-dimensional arrays stored in a\n/// flat buffer using some potential strides.\n#[derive(Debug)]\npub struct StridedIndex<'a> {\n    next_storage_index: Option<usize>,\n    multi_index: Vec<usize>,\n    dims: &'a [usize],\n    stride: &'a [usize],\n}\n\nimpl<'a> StridedIndex<'a> {\n    pub(crate) fn new(dims: &'a [usize], stride: &'a [usize], start_offset: usize) -> Self {\n        let elem_count: usize = dims.iter().product();\n        let next_storage_index = if elem_count == 0 {\n            None\n        } else {\n            // This applies to the scalar case.\n            Some(start_offset)\n        };\n        StridedIndex {\n            next_storage_index,\n            multi_index: vec![0; dims.len()],\n            dims,\n            stride,\n        }\n    }\n\n    pub(crate) fn from_layout(l: &'a Layout) -> Self {\n        Self::new(l.dims(), l.stride(), l.start_offset())\n    }\n}\n\nimpl<'a> Iterator for StridedIndex<'a> {\n    type Item = usize;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        let storage_index = match self.next_storage_index {\n            None => return None,\n            Some(storage_index) => storage_index,\n        };\n        let mut updated = false;\n        let mut next_storage_index = storage_index;\n        for ((multi_i, max_i), stride_i) in self\n            .multi_index\n            .iter_mut()\n            .zip(self.dims.iter())\n            .zip(self.stride.iter())\n            .rev()\n        {\n            let next_i = *multi_i + 1;\n            if next_i < *max_i {\n                *multi_i = next_i;\n                updated = true;\n                next_storage_index += stride_i;\n                break;\n            } else {\n                next_storage_index -= *multi_i * stride_i;\n                *multi_i = 0\n            }\n        }\n        self.next_storage_index = if updated {\n            Some(next_storage_index)\n        } else {\n            None\n        };\n        Some(storage_index)\n    }\n}\n\n#[derive(Debug)]\npub enum StridedBlocks<'a> {\n    SingleBlock {\n        start_offset: usize,\n        len: usize,\n    },\n    MultipleBlocks {\n        block_start_index: StridedIndex<'a>,\n        block_len: usize,\n    },\n}\n\n// Path: candle-core/src/layout.rs\nuse crate::{Error, Result, Shape};\n\n#[derive(Debug, PartialEq, Eq, Clone)]\npub struct Layout {\n    shape: Shape,\n    // The strides are given in number of elements and not in bytes.\n    stride: Vec<usize>,\n    start_offset: usize,\n}\n\nimpl Layout {\n    pub fn new(shape: Shape, stride: Vec<usize>, start_offset: usize) -> Self {\n        Self {\n            shape,\n            stride,\n            start_offset,\n        }\n    }\n\n    pub fn contiguous_with_offset<S: Into<Shape>>(shape: S, start_offset: usize) -> Self {\n        let shape = shape.into();\n        let stride = shape.stride_contiguous();\n        Self {\n            shape,\n            stride,\n            start_offset,\n        }\n    }\n\n    pub fn contiguous<S: Into<Shape>>(shape: S) -> Self {\n        Self::contiguous_with_offset(shape, 0)\n    }\n\n    pub fn dims(&self) -> &[usize] {\n        self.shape.dims()\n    }\n\n    pub fn shape(&self) -> &Shape {\n        &self.shape\n    }\n\n    pub fn stride(&self) -> &[usize] {\n        &self.stride\n    }\n\n    pub fn start_offset(&self) -> usize {\n        self.start_offset\n    }\n\n    /// Returns the appropriate start and stop offset if the data is stored in a C\n    /// contiguous (aka row major) way.\n    pub fn contiguous_offsets(&self) -> Option<(usize, usize)> {\n        if self.is_contiguous() {\n            let start_o = self.start_offset;\n            Some((start_o, start_o + self.shape.elem_count()))\n        } else {\n            None\n        }\n    }\n\n    /// Returns true if the data is stored in a C contiguous (aka row major) way.\n    /// Note that this does not implies that the start offset is 0 or that there are no extra\n    /// elements at the end of the storage.\n    pub fn is_contiguous(&self) -> bool {\n        self.shape.is_contiguous(&self.stride)\n    }\n\n    /// Returns true if the data is stored in a Fortran contiguous (aka column major) way.\n    pub fn is_fortran_contiguous(&self) -> bool {\n        self.shape.is_fortran_contiguous(&self.stride)\n    }\n\n    pub fn narrow(&self, dim: usize, start: usize, len: usize) -> Result<Self> {\n        let dims = self.shape().dims();\n        if dim >= dims.len() {\n            Err(Error::DimOutOfRange {\n                shape: self.shape().clone(),\n                dim: dim as i32,\n                op: \"narrow\",\n            }\n            .bt())?\n        }\n        if start + len > dims[dim] {\n            Err(Error::NarrowInvalidArgs {\n                shape: self.shape.clone(),\n                dim,\n                start,\n                len,\n                msg: \"start + len > dim_len\",\n            }\n            .bt())?\n        }\n        let mut dims = dims.to_vec();\n        dims[dim] = len;\n        Ok(Self {\n            shape: Shape::from(dims),\n            stride: self.stride.clone(),\n            start_offset: self.start_offset + self.stride[dim] * start,\n        })\n    }\n\n    pub fn transpose(&self, dim1: usize, dim2: usize) -> Result<Self> {\n        let rank = self.shape.rank();\n        if rank <= dim1 || rank <= dim2 {\n            Err(Error::UnexpectedNumberOfDims {\n                expected: usize::max(dim1, dim2),\n                got: rank,\n                shape: self.shape().clone(),\n            }\n            .bt())?\n        }\n        let mut stride = self.stride().to_vec();\n        let mut dims = self.shape().dims().to_vec();\n        dims.swap(dim1, dim2);\n        stride.swap(dim1, dim2);\n        Ok(Self {\n            shape: Shape::from(dims),\n            stride,\n            start_offset: self.start_offset,\n        })\n    }\n\n    pub fn permute(&self, idxs: &[usize]) -> Result<Self> {\n        let is_permutation =\n            idxs.len() == self.shape.rank() && (0..idxs.len()).all(|i| idxs.contains(&i));\n        if !is_permutation {\n            crate::bail!(\n                \"dimension mismatch in permute, tensor {:?}, dims: {:?}\",\n                self.dims(),\n                idxs\n            )\n        }\n        let stride = self.stride();\n        let dims = self.shape().dims();\n        let mut perm_stride = stride.to_vec();\n        let mut perm_dims = dims.to_vec();\n        for (i, &idx) in idxs.iter().enumerate() {\n            perm_stride[i] = stride[idx];\n            perm_dims[i] = dims[idx];\n        }\n        Ok(Self {\n            shape: Shape::from(perm_dims),\n            stride: perm_stride,\n            start_offset: self.start_offset,\n        })\n    }\n\n    pub fn broadcast_as<S: Into<Shape>>(&self, shape: S) -> Result<Self> {\n        let shape = shape.into();\n        if shape.rank() < self.shape().rank() {\n            return Err(Error::BroadcastIncompatibleShapes {\n                src_shape: self.shape().clone(),\n                dst_shape: shape,\n            }\n            .bt());\n        }\n        let added_dims = shape.rank() - self.shape().rank();\n        let mut stride = vec![0; added_dims];\n        for (&dst_dim, (&src_dim, &src_stride)) in shape.dims()[added_dims..]\n            .iter()\n            .zip(self.dims().iter().zip(self.stride()))\n        {\n            let s = if dst_dim == src_dim {\n                src_stride\n            } else if src_dim != 1 {\n                return Err(Error::BroadcastIncompatibleShapes {\n                    src_shape: self.shape().clone(),\n                    dst_shape: shape,\n                }\n                .bt());\n            } else {\n                0\n            };\n            stride.push(s)\n        }\n        Ok(Self {\n            shape,\n            stride,\n            start_offset: self.start_offset,\n        })\n    }\n\n    pub(crate) fn strided_index(&self) -> crate::StridedIndex {\n        crate::StridedIndex::from_layout(self)\n    }\n\n    pub(crate) fn strided_blocks(&self) -> crate::StridedBlocks {\n        let mut block_len = 1;\n        let mut contiguous_dims = 0; // These are counted from the right.\n        for (&stride, &dim) in self.stride().iter().zip(self.dims().iter()).rev() {\n            if stride != block_len {\n                break;\n            }\n            block_len *= dim;\n            contiguous_dims += 1;\n        }\n        let index_dims = self.dims().len() - contiguous_dims;\n        if index_dims == 0 {\n            crate::StridedBlocks::SingleBlock {\n...\n", "needle_token_start": 11382, "needle_token_end": 11440, "code_context_ntokens": 15476, "output": ["```rust\n/// Writes a multi-dimensional array in the npy format.\npub fn write_npy<T: AsRef<Path>>(&self, path: T) -> Result<()> {\n    let mut f = File::create(path.as_ref())?;\n    self.write(&mut f)\n}\n```"]}
{"repo": "huggingface/candle", "name": "dequantize_mul_mat_vec", "language": "rust", "path": "candle-core/src/quantized/cuda.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to perform matrix-vector multiplication where the matrix is stored in a compressed quantized format, and then dequantize the result to a floating-point format.\n2. **Input**: It takes a quantized data slice, a vector in floating-point format, a data type specifying the quantization format, the number of columns and rows in the matrix, and a device context for CUDA operations.\n3. **Output**: It outputs a storage container holding the result of the dequantized matrix-vector multiplication in floating-point format.\n4. **Procedure**: The function first checks the validity of the input dimensions. It then determines the appropriate CUDA kernel based on the quantization data type. It configures the CUDA grid and block dimensions for efficient parallel computation and launches the CUDA kernel to perform the matrix-vector multiplication and dequantization in one step. The result is then wrapped in a storage container and returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: candle-core/src/quantized/k_quants.rs\nuse super::utils::{\n    get_scale_min_k4, group_for_dequantization, group_for_quantization, make_q3_quants,\n    make_qkx1_quants, make_qx_quants, nearest_int,\n};\nuse super::GgmlDType;\nuse crate::Result;\nuse byteorder::{ByteOrder, LittleEndian};\nuse half::f16;\nuse rayon::prelude::*;\n\n// Default to QK_K 256 rather than 64.\npub const QK_K: usize = 256;\npub const K_SCALE_SIZE: usize = 12;\n\npub const QK4_0: usize = 32;\npub const QK4_1: usize = 32;\npub const QK5_0: usize = 32;\npub const QK5_1: usize = 32;\npub const QK8_0: usize = 32;\npub const QK8_1: usize = 32;\n\npub trait GgmlType: Sized + Clone + Send + Sync {\n    const DTYPE: GgmlDType;\n    const BLCK_SIZE: usize;\n    type VecDotType: GgmlType;\n\n    // This is only safe for types that include immediate values such as float/int/...\n    fn zeros() -> Self {\n        unsafe { std::mem::MaybeUninit::zeroed().assume_init() }\n    }\n    fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()>;\n    fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()>;\n\n    /// Dot product used as a building block for quantized mat-mul.\n    /// n is the number of elements to be considered.\n    fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32>;\n\n    /// Generic implementation of the dot product without simd optimizations.\n    fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32>;\n}\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ4_0 {\n    pub(crate) d: f16,\n    pub(crate) qs: [u8; QK4_0 / 2],\n}\nconst _: () = assert!(std::mem::size_of::<BlockQ4_0>() == 18);\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ4_1 {\n    pub(crate) d: f16,\n    pub(crate) m: f16,\n    pub(crate) qs: [u8; QK4_1 / 2],\n}\nconst _: () = assert!(std::mem::size_of::<BlockQ4_1>() == 20);\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ5_0 {\n    pub(crate) d: f16,\n    pub(crate) qh: [u8; 4],\n    pub(crate) qs: [u8; QK5_0 / 2],\n}\nconst _: () = assert!(std::mem::size_of::<BlockQ5_0>() == 22);\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ5_1 {\n    pub(crate) d: f16,\n    pub(crate) m: f16,\n    pub(crate) qh: [u8; 4],\n    pub(crate) qs: [u8; QK5_1 / 2],\n}\nconst _: () = assert!(std::mem::size_of::<BlockQ5_1>() == 24);\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ8_0 {\n    pub(crate) d: f16,\n    pub(crate) qs: [i8; QK8_0],\n}\nconst _: () = assert!(std::mem::size_of::<BlockQ8_0>() == 34);\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ8_1 {\n    pub(crate) d: f16,\n    pub(crate) s: f16,\n    pub(crate) qs: [i8; QK8_1],\n}\nconst _: () = assert!(std::mem::size_of::<BlockQ8_1>() == 36);\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ2K {\n    pub(crate) scales: [u8; QK_K / 16],\n    pub(crate) qs: [u8; QK_K / 4],\n    pub(crate) d: f16,\n    pub(crate) dmin: f16,\n}\nconst _: () = assert!(QK_K / 16 + QK_K / 4 + 2 * 2 == std::mem::size_of::<BlockQ2K>());\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ3K {\n    pub(crate) hmask: [u8; QK_K / 8],\n    pub(crate) qs: [u8; QK_K / 4],\n    pub(crate) scales: [u8; 12],\n    pub(crate) d: f16,\n}\nconst _: () = assert!(QK_K / 8 + QK_K / 4 + 12 + 2 == std::mem::size_of::<BlockQ3K>());\n\n#[derive(Debug, Clone, PartialEq)]\n// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/k_quants.h#L82\n#[repr(C)]\npub struct BlockQ4K {\n    pub(crate) d: f16,\n    pub(crate) dmin: f16,\n    pub(crate) scales: [u8; K_SCALE_SIZE],\n    pub(crate) qs: [u8; QK_K / 2],\n}\nconst _: () = assert!(QK_K / 2 + K_SCALE_SIZE + 2 * 2 == std::mem::size_of::<BlockQ4K>());\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ5K {\n    pub(crate) d: f16,\n    pub(crate) dmin: f16,\n    pub(crate) scales: [u8; K_SCALE_SIZE],\n    pub(crate) qh: [u8; QK_K / 8],\n    pub(crate) qs: [u8; QK_K / 2],\n}\nconst _: () =\n    assert!(QK_K / 8 + QK_K / 2 + 2 * 2 + K_SCALE_SIZE == std::mem::size_of::<BlockQ5K>());\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ6K {\n    pub(crate) ql: [u8; QK_K / 2],\n    pub(crate) qh: [u8; QK_K / 4],\n    pub(crate) scales: [i8; QK_K / 16],\n    pub(crate) d: f16,\n}\nconst _: () = assert!(3 * QK_K / 4 + QK_K / 16 + 2 == std::mem::size_of::<BlockQ6K>());\n\n#[derive(Debug, Clone, PartialEq)]\n#[repr(C)]\npub struct BlockQ8K {\n    pub(crate) d: f32,\n    pub(crate) qs: [i8; QK_K],\n    pub(crate) bsums: [i16; QK_K / 16],\n}\nconst _: () = assert!(4 + QK_K + QK_K / 16 * 2 == std::mem::size_of::<BlockQ8K>());\n\nimpl GgmlType for BlockQ4_0 {\n    const DTYPE: GgmlDType = GgmlDType::Q4_0;\n    const BLCK_SIZE: usize = QK4_0;\n    type VecDotType = BlockQ8_0;\n\n    // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1525\n    fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> {\n        let k = ys.len();\n        let qk = Self::BLCK_SIZE;\n        if k % qk != 0 {\n            crate::bail!(\"dequantize_row_q4_0: {k} is not divisible by {qk}\")\n        }\n\n        let nb = k / qk;\n        for i in 0..nb {\n            let d = xs[i].d.to_f32();\n\n            for j in 0..(qk / 2) {\n                let x0 = (xs[i].qs[j] & 0x0F) as i16 - 8;\n                let x1 = (xs[i].qs[j] >> 4) as i16 - 8;\n\n                ys[i * qk + j] = (x0 as f32) * d;\n                ys[i * qk + j + qk / 2] = (x1 as f32) * d;\n            }\n        }\n        Ok(())\n    }\n\n    fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> {\n        // quantize_row_q4_0\n        let qk = Self::BLCK_SIZE;\n        let k = xs.len();\n        if k % qk != 0 {\n            crate::bail!(\"{k} is not divisible by {}\", qk);\n        };\n        let nb = k / qk;\n        if ys.len() != nb {\n            crate::bail!(\"size mismatch {} {} {}\", xs.len(), ys.len(), qk,)\n        }\n        for (i, ys) in ys.iter_mut().enumerate() {\n            let mut amax = 0f32;\n            let mut max = 0f32;\n\n            let xs = &xs[i * qk..(i + 1) * qk];\n            for &x in xs.iter() {\n                if amax < x.abs() {\n                    amax = x.abs();\n                    max = x;\n                }\n            }\n            let d = max / -8.0;\n            let id = if d != 0f32 { 1. / d } else { 0. };\n            ys.d = f16::from_f32(d);\n\n            for (j, q) in ys.qs.iter_mut().enumerate() {\n                let x0 = xs[j] * id;\n                let x1 = xs[qk / 2 + j] * id;\n                let xi0 = u8::min(15, (x0 + 8.5) as u8);\n                let xi1 = u8::min(15, (x1 + 8.5) as u8);\n                *q = xi0 | (xi1 << 4)\n            }\n        }\n        Ok(())\n    }\n\n    // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/ggml.c#L2361C10-L2361C122\n    #[allow(unreachable_code)]\n    fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        #[cfg(target_feature = \"avx\")]\n        return super::avx::vec_dot_q4_0_q8_0(n, xs, ys);\n\n        #[cfg(target_feature = \"neon\")]\n        return super::neon::vec_dot_q4_0_q8_0(n, xs, ys);\n\n        #[cfg(target_feature = \"simd128\")]\n        return super::simd128::vec_dot_q4_0_q8_0(n, xs, ys);\n\n        Self::vec_dot_unopt(n, xs, ys)\n    }\n\n    fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        let qk = QK8_0;\n        if n % QK8_0 != 0 {\n            crate::bail!(\"vec_dot_q4_0_q8_0: {n} is not divisible by {qk}\")\n        }\n        // Generic implementation.\n        let mut sumf = 0f32;\n        for (xs, ys) in xs.iter().zip(ys.iter()) {\n            let mut sum_i = 0;\n            for j in 0..qk / 2 {\n                let v0 = (xs.qs[j] & 0x0F) as i32 - 8;\n                let v1 = (xs.qs[j] >> 4) as i32 - 8;\n                sum_i += v0 * ys.qs[j] as i32 + v1 * ys.qs[j + qk / 2] as i32\n            }\n            sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d)\n        }\n        Ok(sumf)\n    }\n}\n\nimpl GgmlType for BlockQ4_1 {\n    const DTYPE: GgmlDType = GgmlDType::Q4_1;\n    const BLCK_SIZE: usize = QK4_1;\n    type VecDotType = BlockQ8_1;\n\n    fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        Self::vec_dot_unopt(n, xs, ys)\n    }\n\n    fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        // ggml_vec_dot_q4_1_q8_1\n        let qk = QK8_1;\n        if n % qk != 0 {\n            crate::bail!(\"vec_dot_q4_1_q8_1: {n} is not divisible by {qk}\")\n        }\n        let nb = n / qk;\n        if nb % 2 != 0 {\n            crate::bail!(\"vec_dot_q4_1_q8_1: {n}, nb is not divisible by 2\")\n        }\n\n        // Generic implementation.\n        let mut sumf = 0f32;\n\n        for (xs, ys) in xs.iter().zip(ys.iter()) {\n            let mut sumi = 0i32;\n\n            for j in 0..qk / 2 {\n                let v0 = xs.qs[j] as i32 & 0x0F;\n                let v1 = xs.qs[j] as i32 >> 4;\n                sumi += (v0 * ys.qs[j] as i32) + (v1 * ys.qs[j + qk / 2] as i32);\n            }\n\n            sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d)\n                + f16::to_f32(xs.m) * f16::to_f32(ys.s)\n        }\n        Ok(sumf)\n    }\n\n    fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> {\n        // quantize_row_q4_1\n        let qk = Self::BLCK_SIZE;\n        if ys.len() * qk != xs.len() {\n            crate::bail!(\"size mismatch {} {} {}\", xs.len(), ys.len(), qk,)\n        }\n        for (i, ys) in ys.iter_mut().enumerate() {\n            let xs = &xs[i * qk..(i + 1) * qk];\n\n            let mut min = f32::INFINITY;\n            let mut max = f32::NEG_INFINITY;\n            for &x in xs.iter() {\n                min = f32::min(x, min);\n                max = f32::max(x, max);\n            }\n            let d = (max - min) / ((1 << 4) - 1) as f32;\n            let id = if d != 0f32 { 1. / d } else { 0. };\n            ys.d = f16::from_f32(d);\n            ys.m = f16::from_f32(min);\n\n            for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() {\n                let x0 = (xs[j] - min) * id;\n                let x1 = (xs[qk / 2 + j] - min) * id;\n\n                let xi0 = u8::min(15, (x0 + 0.5) as u8);\n                let xi1 = u8::min(15, (x1 + 0.5) as u8);\n\n                *q = xi0 | (xi1 << 4);\n            }\n        }\n        Ok(())\n    }\n\n    // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1545\n    fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> {\n        let k = ys.len();\n        if k % QK4_1 != 0 {\n            crate::bail!(\"dequantize_row_q4_1: {k} is not divisible by {QK4_1}\");\n        }\n\n        let nb = k / QK4_1;\n        for i in 0..nb {\n            let d = xs[i].d.to_f32();\n            let m = xs[i].m.to_f32();\n\n            for j in 0..(QK4_1 / 2) {\n                let x0 = xs[i].qs[j] & 0x0F;\n                let x1 = xs[i].qs[j] >> 4;\n\n                ys[i * QK4_1 + j] = (x0 as f32) * d + m;\n                ys[i * QK4_1 + j + QK4_1 / 2] = (x1 as f32) * d + m;\n            }\n        }\n        Ok(())\n    }\n}\n\nimpl GgmlType for BlockQ5_0 {\n    const DTYPE: GgmlDType = GgmlDType::Q5_0;\n    const BLCK_SIZE: usize = QK5_0;\n    type VecDotType = BlockQ8_0;\n\n    fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        let qk = Self::BLCK_SIZE;\n        if n % Self::BLCK_SIZE != 0 {\n            crate::bail!(\"vec_dot_q5_0_q8_0: {n} is not divisible by {qk}\")\n        }\n        let nb = n / qk;\n        if nb % 2 != 0 {\n            crate::bail!(\"vec_dot_q5_0_q8_0: {n}, nb is not divisible by 2\")\n        }\n        Self::vec_dot_unopt(n, xs, ys)\n    }\n\n    fn vec_dot_unopt(_n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        // Generic implementation.\n        let mut sumf = 0f32;\n\n        for (xs, ys) in xs.iter().zip(ys.iter()) {\n            let qh = LittleEndian::read_u32(&xs.qh);\n            let mut sumi = 0i32;\n\n            for j in 0..Self::BLCK_SIZE / 2 {\n                let xh_0 = (((qh & (1u32 << j)) >> j) << 4) as u8;\n                let xh_1 = ((qh & (1u32 << (j + 16))) >> (j + 12)) as u8;\n\n                let x0 = ((xs.qs[j] & 0x0F) as i32 | xh_0 as i32) - 16;\n                let x1 = ((xs.qs[j] >> 4) as i32 | xh_1 as i32) - 16;\n\n                sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32);\n            }\n\n            sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d)\n        }\n        Ok(sumf)\n    }\n\n    fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> {\n        // quantize_row_q5_0\n        let k = xs.len();\n        if ys.len() * Self::BLCK_SIZE != k {\n            crate::bail!(\"size mismatch {k} {} {}\", ys.len(), Self::BLCK_SIZE)\n        }\n        for (i, ys) in ys.iter_mut().enumerate() {\n            let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE];\n\n            let mut amax = 0f32;\n            let mut max = 0f32;\n            for &x in xs.iter() {\n                if amax < x.abs() {\n                    amax = x.abs();\n                    max = x;\n                }\n            }\n            let d = max / -16.;\n            let id = if d != 0f32 { 1. / d } else { 0. };\n            ys.d = f16::from_f32(d);\n            let mut qh = 0u32;\n            for j in 0..Self::BLCK_SIZE / 2 {\n                let x0 = xs[j] * id;\n                let x1 = xs[j + Self::BLCK_SIZE / 2] * id;\n                let xi0 = ((x0 + 16.5) as i8).min(31) as u8;\n                let xi1 = ((x1 + 16.5) as i8).min(31) as u8;\n                ys.qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4);\n                qh |= ((xi0 as u32 & 0x10) >> 4) << j;\n                qh |= ((xi1 as u32 & 0x10) >> 4) << (j + Self::BLCK_SIZE / 2);\n            }\n            LittleEndian::write_u32(&mut ys.qh, qh)\n        }\n        Ok(())\n    }\n\n    // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1566\n    fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> {\n        let k = ys.len();\n        if k % QK5_0 != 0 {\n            crate::bail!(\"dequantize_row_q5_0: {k} is not divisible by {QK5_0}\");\n        }\n\n        let nb = k / QK5_0;\n        for i in 0..nb {\n            let d = xs[i].d.to_f32();\n            let qh: u32 = LittleEndian::read_u32(&xs[i].qh);\n\n            for j in 0..(QK5_0 / 2) {\n                let xh_0 = (((qh >> j) << 4) & 0x10) as u8;\n                let xh_1 = ((qh >> (j + 12)) & 0x10) as u8;\n\n                let x0 = ((xs[i].qs[j] & 0x0F) | xh_0) as i32 - 16;\n                let x1 = ((xs[i].qs[j] >> 4) | xh_1) as i32 - 16;\n\n                ys[i * QK5_0 + j] = (x0 as f32) * d;\n                ys[i * QK5_0 + j + QK5_0 / 2] = (x1 as f32) * d;\n            }\n        }\n        Ok(())\n    }\n}\n\nimpl GgmlType for BlockQ5_1 {\n    const DTYPE: GgmlDType = GgmlDType::Q5_1;\n    const BLCK_SIZE: usize = QK5_1;\n    type VecDotType = BlockQ8_1;\n\n    fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        Self::vec_dot_unopt(n, xs, ys)\n    }\n\n    fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result<f32> {\n        let qk = Self::BLCK_SIZE;\n        if n % Self::BLCK_SIZE != 0 {\n            crate::bail!(\"vec_dot_q5_1_q8_1: {n} is not divisible by {qk}\")\n        }\n        let nb = n / qk;\n        if nb % 2 != 0 {\n            crate::bail!(\"vec_dot_q5_1_q8_1: {n}, nb is not divisible by 2\")\n        }\n\n        // Generic implementation.\n        let mut sumf = 0f32;\n\n        for (xs, ys) in xs.iter().zip(ys.iter()) {\n            let qh = LittleEndian::read_u32(&xs.qh);\n            let mut sumi = 0i32;\n\n            for j in 0..Self::BLCK_SIZE / 2 {\n                let xh_0 = ((qh >> j) << 4) & 0x10;\n                let xh_1 = (qh >> (j + 12)) & 0x10;\n\n                let x0 = (xs.qs[j] as i32 & 0xF) | xh_0 as i32;\n                let x1 = (xs.qs[j] as i32 >> 4) | xh_1 as i32;\n\n                sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32);\n            }\n\n            sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d)\n                + f16::to_f32(xs.m) * f16::to_f32(ys.s)\n        }\n        Ok(sumf)\n    }\n\n    fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> {\n        // quantize_row_q5_1\n        let qk = Self::BLCK_SIZE;\n        if ys.len() * qk != xs.len() {\n            crate::bail!(\"size mismatch {} {} {}\", xs.len(), ys.len(), qk,)\n        }\n...\n// Path: candle-core/src/quantized/ggml_file.rs\n//! Support for the GGML file format.\n\nuse super::{k_quants, GgmlDType, QStorage};\nuse crate::{Device, Result};\nuse byteorder::{LittleEndian, ReadBytesExt};\nuse std::collections::HashMap;\n\n// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.h#L37\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\nenum Magic {\n    Ggjt,\n    Ggla,\n    Ggmf,\n    Ggml,\n    Ggsn,\n}\n\nimpl TryFrom<u32> for Magic {\n    type Error = crate::Error;\n    fn try_from(value: u32) -> Result<Self> {\n        let magic = match value {\n            0x67676a74 => Self::Ggjt,\n            0x67676c61 => Self::Ggla,\n            0x67676d66 => Self::Ggmf,\n            0x67676d6c => Self::Ggml,\n            0x6767736e => Self::Ggsn,\n            _ => crate::bail!(\"unknown magic {value:08x}\"),\n        };\n        Ok(magic)\n    }\n}\n\n#[derive(Debug, Clone, Copy, PartialEq, Eq)]\npub enum VersionedMagic {\n    GgmlUnversioned,\n    GgmfV1,\n    GgjtV1,\n    GgjtV2,\n    GgjtV3,\n}\n\nimpl VersionedMagic {\n    fn read<R: std::io::Read>(reader: &mut R) -> Result<Self> {\n        let magic = reader.read_u32::<LittleEndian>()?;\n        let magic = Magic::try_from(magic)?;\n        if magic == Magic::Ggml {\n            return Ok(Self::GgmlUnversioned);\n        }\n        let version = reader.read_u32::<LittleEndian>()?;\n        let versioned_magic = match (magic, version) {\n            (Magic::Ggmf, 1) => Self::GgmfV1,\n            (Magic::Ggjt, 1) => Self::GgjtV1,\n            (Magic::Ggjt, 2) => Self::GgjtV2,\n            (Magic::Ggjt, 3) => Self::GgjtV3,\n            _ => crate::bail!(\"ggml: unsupported magic/version {magic:?}/{version}\"),\n        };\n        Ok(versioned_magic)\n    }\n\n    fn align32(&self) -> bool {\n        match self {\n            Self::GgmlUnversioned | Self::GgmfV1 => false,\n            Self::GgjtV1 | Self::GgjtV2 | Self::GgjtV3 => true,\n        }\n    }\n}\n\n#[derive(Debug, Clone, PartialEq, Eq)]\npub struct HParams {\n    pub n_vocab: u32,\n    pub n_embd: u32,\n    pub n_mult: u32,\n    pub n_head: u32,\n    pub n_layer: u32,\n    pub n_rot: u32,\n    pub ftype: u32,\n}\n\nimpl HParams {\n    fn read<R: std::io::Read>(reader: &mut R) -> Result<Self> {\n        let n_vocab = reader.read_u32::<LittleEndian>()?;\n        let n_embd = reader.read_u32::<LittleEndian>()?;\n        let n_mult = reader.read_u32::<LittleEndian>()?;\n        let n_head = reader.read_u32::<LittleEndian>()?;\n        let n_layer = reader.read_u32::<LittleEndian>()?;\n        let n_rot = reader.read_u32::<LittleEndian>()?;\n        let ftype = reader.read_u32::<LittleEndian>()?;\n        Ok(Self {\n            n_vocab,\n            n_embd,\n            n_mult,\n            n_head,\n            n_layer,\n            n_rot,\n            ftype,\n        })\n    }\n}\n\n#[derive(Debug, Clone, PartialEq)]\npub struct Vocab {\n    pub token_score_pairs: Vec<(Vec<u8>, f32)>,\n}\n\nimpl Vocab {\n    fn read<R: std::io::Read>(reader: &mut R, n_vocab: usize) -> Result<Self> {\n        // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L556\n        let mut token_score_pairs = Vec::with_capacity(n_vocab);\n        for _index in 0..n_vocab {\n            let len = reader.read_u32::<LittleEndian>()? as usize;\n            let mut word = vec![0u8; len];\n            reader.read_exact(&mut word)?;\n            let score = reader.read_f32::<LittleEndian>()?;\n            token_score_pairs.push((word, score))\n        }\n        Ok(Self { token_score_pairs })\n    }\n}\n\nfn from_raw_data<T: super::GgmlType + Send + Sync + 'static>(\n    raw_data: &[u8],\n    size_in_bytes: usize,\n    dims: Vec<usize>,\n    device: &Device,\n) -> Result<super::QTensor> {\n    let raw_data_ptr = raw_data.as_ptr();\n    let n_blocks = size_in_bytes / std::mem::size_of::<T>();\n    let data = unsafe { std::slice::from_raw_parts(raw_data_ptr as *const T, n_blocks) };\n    let data: QStorage = match device {\n        Device::Cpu => QStorage::Cpu(Box::new(data.to_vec())),\n        Device::Metal(metal) => super::metal::load_quantized(metal, data)?,\n        Device::Cuda(cuda) => super::cuda::load_quantized(cuda, data)?,\n    };\n    super::QTensor::new(data, dims)\n}\n\n/// Creates a [Tensor] from a raw GGML tensor.\npub fn qtensor_from_ggml(\n    ggml_dtype: GgmlDType,\n    raw_data: &[u8],\n    dims: Vec<usize>,\n    device: &Device,\n) -> Result<super::QTensor> {\n    let tensor_elems = dims.iter().product::<usize>();\n    let block_size = ggml_dtype.block_size();\n    if tensor_elems % block_size != 0 {\n        crate::bail!(\n            \"the number of elements {tensor_elems} is not divisible by the block size {block_size}\"\n        )\n    }\n    let size_in_bytes = tensor_elems / block_size * ggml_dtype.type_size();\n\n    match ggml_dtype {\n        GgmlDType::F32 => from_raw_data::<f32>(raw_data, size_in_bytes, dims, device),\n        GgmlDType::F16 => from_raw_data::<half::f16>(raw_data, size_in_bytes, dims, device),\n        GgmlDType::Q4_0 => {\n            from_raw_data::<k_quants::BlockQ4_0>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q4_1 => {\n            from_raw_data::<k_quants::BlockQ4_1>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q5_0 => {\n            from_raw_data::<k_quants::BlockQ5_0>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q5_1 => {\n            from_raw_data::<k_quants::BlockQ5_1>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q8_0 => {\n            from_raw_data::<k_quants::BlockQ8_0>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q2K => {\n            from_raw_data::<k_quants::BlockQ2K>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q3K => {\n            from_raw_data::<k_quants::BlockQ3K>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q4K => {\n            from_raw_data::<k_quants::BlockQ4K>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q5K => {\n            from_raw_data::<k_quants::BlockQ5K>(raw_data, size_in_bytes, dims, device)\n        }\n        GgmlDType::Q6K => {\n            from_raw_data::<k_quants::BlockQ6K>(raw_data, size_in_bytes, dims, device)\n        }\n        _ => crate::bail!(\"quantized type {ggml_dtype:?} is not supported yet\"),\n    }\n}\n\nfn read_one_tensor<R: std::io::Seek + std::io::Read>(\n    reader: &mut R,\n    magic: VersionedMagic,\n    device: &Device,\n) -> Result<(String, super::QTensor)> {\n    let n_dims = reader.read_u32::<LittleEndian>()?;\n    let name_len = reader.read_u32::<LittleEndian>()?;\n    let ggml_dtype = reader.read_u32::<LittleEndian>()?;\n    let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?;\n    let mut dims = vec![0u32; n_dims as usize];\n    reader.read_u32_into::<LittleEndian>(&mut dims)?;\n    // The dimensions are stored in reverse order, see for example:\n    // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/convert.py#L969\n    dims.reverse();\n    let mut name = vec![0u8; name_len as usize];\n    reader.read_exact(&mut name)?;\n    let name = String::from_utf8_lossy(&name).into_owned();\n\n    if magic.align32() {\n        let pos = reader.stream_position()?;\n        reader.seek(std::io::SeekFrom::Current(((32 - pos % 32) % 32) as i64))?;\n    }\n    let dims = dims.iter().map(|&u| u as usize).collect::<Vec<_>>();\n    let tensor_elems = dims.iter().product::<usize>();\n    let size_in_bytes = tensor_elems * ggml_dtype.type_size() / ggml_dtype.block_size();\n    // TODO: Mmap version to avoid copying the data around?\n    let mut raw_data = vec![0u8; size_in_bytes];\n    reader.read_exact(&mut raw_data)?;\n    match qtensor_from_ggml(ggml_dtype, &raw_data, dims, device) {\n        Ok(tensor) => Ok((name, tensor)),\n        Err(e) => crate::bail!(\"Error creating tensor {name}: {e}\"),\n    }\n}\n\npub struct Content {\n    pub magic: VersionedMagic,\n    pub hparams: HParams,\n    pub vocab: Vocab,\n    pub tensors: HashMap<String, super::QTensor>,\n    pub device: Device,\n}\n\nimpl Content {\n    pub fn read<R: std::io::Seek + std::io::Read>(\n        reader: &mut R,\n        device: &Device,\n    ) -> Result<Content> {\n        // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L505\n        let last_position = reader.seek(std::io::SeekFrom::End(0))?;\n        reader.seek(std::io::SeekFrom::Start(0))?;\n        let magic = VersionedMagic::read(reader)?;\n        let hparams = HParams::read(reader)?;\n        let vocab = Vocab::read(reader, hparams.n_vocab as usize)?;\n        let mut tensors = HashMap::new();\n\n        while reader.stream_position()? != last_position {\n            let (name, tensor) = read_one_tensor(reader, magic, device)?;\n            tensors.insert(name, tensor);\n        }\n        let device = device.clone();\n        Ok(Self {\n            magic,\n            hparams,\n            vocab,\n            tensors,\n            device,\n        })\n    }\n\n    pub fn remove(&mut self, name: &str) -> Result<super::QTensor> {\n        match self.tensors.remove(name) {\n            None => crate::bail!(\"cannot find tensor with name '{name}'\"),\n            Some(tensor) => Ok(tensor),\n        }\n    }\n}\n\n// Path: candle-core/src/quantized/dummy_metal.rs\n#![allow(unused)]\nuse super::GgmlDType;\nuse crate::{Error, MetalDevice, MetalStorage, Result};\n\npub struct QMetalStorage {\n    dtype: GgmlDType,\n    device: MetalDevice,\n}\n\nimpl QMetalStorage {\n    pub fn zeros(_: &MetalDevice, _: usize, _: GgmlDType) -> Result<Self> {\n        Err(Error::NotCompiledWithMetalSupport)\n    }\n\n    pub fn dtype(&self) -> GgmlDType {\n        self.dtype\n    }\n\n    pub fn device(&self) -> &MetalDevice {\n        &self.device\n    }\n\n    pub fn dequantize(&self, _elem_count: usize) -> Result<MetalStorage> {\n        Err(Error::NotCompiledWithMetalSupport)\n    }\n\n    pub fn quantize(&mut self, _src: &MetalStorage) -> Result<()> {\n        Err(Error::NotCompiledWithMetalSupport)\n    }\n\n    pub fn storage_size_in_bytes(&self) -> usize {\n        0\n    }\n\n    pub fn fwd(\n        &self,\n        _self_shape: &crate::Shape,\n        _storage: &MetalStorage,\n        _layout: &crate::Layout,\n    ) -> Result<(MetalStorage, crate::Shape)> {\n        Err(Error::NotCompiledWithMetalSupport)\n    }\n}\n\npub fn load_quantized<T: super::GgmlType + Send + Sync + 'static>(\n    _device: &MetalDevice,\n    _data: &[T],\n) -> Result<super::QStorage> {\n    Err(Error::NotCompiledWithMetalSupport)\n}\n\n// Path: candle-core/src/quantized/cuda.rs\nuse super::{GgmlDType, QStorage};\nuse crate::quantized::k_quants::GgmlType;\nuse crate::{backend::BackendDevice, cuda_backend::WrapErr};\nuse crate::{CudaDevice, CudaStorage, Result};\n\nuse cudarc::driver::{CudaSlice, CudaView, DeviceSlice};\n\n#[derive(Clone, Debug)]\npub struct QCudaStorage {\n    data: CudaSlice<u8>,\n    dtype: GgmlDType,\n    device: CudaDevice,\n}\n\nstatic FORCE_DMMV: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);\n\npub fn set_force_dmmv(f: bool) {\n    FORCE_DMMV.store(f, std::sync::atomic::Ordering::Relaxed)\n}\n\npub const WARP_SIZE: usize = 32;\npub const MMQ_X_Q4_0_AMPERE: usize = 4;\npub const MMQ_Y_Q4_0_AMPERE: usize = 32;\npub const NWARPS_Q4_0_AMPERE: usize = 4;\npub const GGML_CUDA_MMV_X: usize = 32;\npub const GGML_CUDA_MMV_Y: usize = 1;\npub const CUDA_QUANTIZE_BLOCK_SIZE: usize = 256;\npub const CUDA_DEQUANTIZE_BLOCK_SIZE: usize = 256;\npub const MATRIX_ROW_PADDING: usize = 512;\n\nfn ceil_div(p: usize, q: usize) -> usize {\n    (p + q - 1) / q\n}\n\nfn pad(p: usize, q: usize) -> usize {\n    ceil_div(p, q) * q\n}\n\nfn quantize_q8_1(\n    src: &CudaView<f32>,\n    dst: &mut CudaSlice<u8>,\n    elem_count: usize,\n    dev: &CudaDevice,\n) -> Result<()> {\n    use cudarc::driver::LaunchAsync;\n\n    let kx = elem_count;\n    let kx_padded = pad(kx, MATRIX_ROW_PADDING);\n    let num_blocks = ceil_div(kx_padded, CUDA_QUANTIZE_BLOCK_SIZE);\n    let func = dev.get_or_load_func(\"quantize_q8_1\", candle_kernels::QUANTIZED)?;\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (num_blocks as u32, 1, 1),\n        block_dim: (CUDA_QUANTIZE_BLOCK_SIZE as u32, 1, 1),\n        shared_mem_bytes: 0,\n    };\n    let params = (src, dst, kx as i32, kx_padded as i32);\n    unsafe { func.launch(cfg, params) }.w()?;\n    Ok(())\n}\n\nfn dequantize(\n    data: &CudaSlice<u8>,\n    dtype: GgmlDType,\n    elem_count: usize,\n    dev: &CudaDevice,\n) -> Result<CudaStorage> {\n    use cudarc::driver::LaunchAsync;\n\n    let nb = (elem_count + 255) / 256;\n    let (kernel_name, is_k, block_dim, num_blocks) = match dtype {\n        GgmlDType::Q4_0 => (\"dequantize_block_q4_0\", false, 32, nb),\n        GgmlDType::Q4_1 => (\"dequantize_block_q4_1\", false, 32, nb),\n        GgmlDType::Q5_0 => (\n            \"dequantize_block_q5_0\",\n            false,\n            CUDA_DEQUANTIZE_BLOCK_SIZE,\n            ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE),\n        ),\n        GgmlDType::Q5_1 => (\n            \"dequantize_block_q5_1\",\n            false,\n            CUDA_DEQUANTIZE_BLOCK_SIZE,\n            ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE),\n        ),\n        GgmlDType::Q8_0 => (\"dequantize_block_q8_0\", false, 32, nb),\n        GgmlDType::Q2K => (\"dequantize_block_q2_K\", true, 64, nb),\n        GgmlDType::Q3K => (\"dequantize_block_q3_K\", true, 64, nb),\n        GgmlDType::Q4K => (\"dequantize_block_q4_K\", true, 32, nb),\n        GgmlDType::Q5K => (\"dequantize_block_q5_K\", true, 64, nb),\n        GgmlDType::Q6K => (\"dequantize_block_q6_K\", true, 64, nb),\n        GgmlDType::Q8K => (\"dequantize_block_q8_K\", true, 32, nb),\n        _ => crate::bail!(\"unsupported dtype for dequantize {dtype:?}\"),\n    };\n    let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?;\n    let dst = unsafe { dev.alloc::<f32>(elem_count).w()? };\n    // See e.g.\n    // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (num_blocks as u32, 1, 1),\n        block_dim: (block_dim as u32, 1, 1),\n        shared_mem_bytes: 0,\n    };\n\n    if is_k {\n        let params = (data, &dst);\n        unsafe { func.launch(cfg, params) }.w()?;\n    } else {\n        let nb32 = match dtype {\n            GgmlDType::Q5_0 | GgmlDType::Q5_1 => elem_count,\n            _ => elem_count / 32,\n        };\n        let params = (data, &dst, nb32 as i32);\n        unsafe { func.launch(cfg, params) }.w()?;\n    }\n    Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone()))\n}\n\n\nfn dequantize_mul_mat_vec(\n    data: &CudaSlice<u8>,\n    y: &CudaView<f32>,\n    dtype: GgmlDType,\n    ncols: usize,\n    nrows: usize,\n    dev: &CudaDevice,\n) -> Result<CudaStorage> {\n    use cudarc::driver::LaunchAsync;\n\n    let data_elems = data.len() / dtype.type_size() * dtype.block_size();\n    if data_elems < ncols * nrows {\n        crate::bail!(\"unexpected data size {}, ncols {ncols} {nrows}\", data_elems)\n    }\n    if y.len() != ncols {\n        crate::bail!(\"unexpected y size {}, ncols {ncols} {nrows}\", y.len())\n    }\n    let kernel_name = match dtype {\n        GgmlDType::Q4_0 => \"dequantize_mul_mat_vec_q4_0_cuda\",\n        GgmlDType::Q4_1 => \"dequantize_mul_mat_vec_q4_1_cuda\",\n        GgmlDType::Q5_0 => \"dequantize_mul_mat_vec_q5_0_cuda\",\n        GgmlDType::Q5_1 => \"dequantize_mul_mat_vec_q5_1_cuda\",\n        GgmlDType::Q8_0 => \"dequantize_mul_mat_vec_q8_0_cuda\",\n        GgmlDType::Q2K => \"dequantize_mul_mat_vec_q2_k\",\n        GgmlDType::Q3K => \"dequantize_mul_mat_vec_q3_k\",\n        GgmlDType::Q4K => \"dequantize_mul_mat_vec_q4_k\",\n        GgmlDType::Q5K => \"dequantize_mul_mat_vec_q5_k\",\n        GgmlDType::Q6K => \"dequantize_mul_mat_vec_q6_k\",\n        _ => crate::bail!(\"unsupported dtype for quantized matmul {dtype:?}\"),\n    };\n    let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?;\n    let dst = unsafe { dev.alloc::<f32>(nrows).w()? };\n    let block_num_y = ceil_div(nrows, GGML_CUDA_MMV_Y);\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (block_num_y as u32, 1, 1),\n        block_dim: (WARP_SIZE as u32, GGML_CUDA_MMV_Y as u32, 1),\n        shared_mem_bytes: 0,\n    };\n\n    let params = (data, y, &dst, ncols as i32, nrows as i32);\n    unsafe { func.launch(cfg, params) }.w()?;\n    Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone()))\n}\n\nfn mul_mat_vec_via_q8_1(\n    data: &CudaSlice<u8>,\n    y: &CudaView<f32>,\n    dtype: GgmlDType,\n    ncols: usize,\n    nrows: usize,\n    dev: &CudaDevice,\n) -> Result<CudaStorage> {\n    use cudarc::driver::LaunchAsync;\n\n    let data_elems = data.len() / dtype.type_size() * dtype.block_size();\n    if data_elems < ncols * nrows {\n        crate::bail!(\"unexpected data size {}, ncols {ncols} {nrows}\", data_elems)\n    }\n    if y.len() != ncols {\n        crate::bail!(\"unexpected y size {}, ncols {ncols} {nrows}\", y.len())\n    }\n    // Start by quantizing y\n    let ncols_padded = pad(ncols, MATRIX_ROW_PADDING);\n    let y_size_in_bytes = ncols_padded * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size();\n    let mut y_q8_1 = unsafe { dev.alloc::<u8>(y_size_in_bytes).w()? };\n    quantize_q8_1(y, &mut y_q8_1, ncols, dev)?;\n\n    let kernel_name = match dtype {\n        GgmlDType::Q4_0 => \"mul_mat_vec_q4_0_q8_1_cuda\",\n        GgmlDType::Q4_1 => \"mul_mat_vec_q4_1_q8_1_cuda\",\n        GgmlDType::Q5_0 => \"mul_mat_vec_q5_0_q8_1_cuda\",\n        GgmlDType::Q5_1 => \"mul_mat_vec_q5_1_q8_1_cuda\",\n        GgmlDType::Q8_0 => \"mul_mat_vec_q8_0_q8_1_cuda\",\n        GgmlDType::Q2K => \"mul_mat_vec_q2_K_q8_1_cuda\",\n        GgmlDType::Q3K => \"mul_mat_vec_q3_K_q8_1_cuda\",\n        GgmlDType::Q4K => \"mul_mat_vec_q4_K_q8_1_cuda\",\n        GgmlDType::Q5K => \"mul_mat_vec_q5_K_q8_1_cuda\",\n        GgmlDType::Q6K => \"mul_mat_vec_q6_K_q8_1_cuda\",\n        _ => crate::bail!(\"unsupported dtype for quantized matmul {dtype:?}\"),\n    };\n    let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?;\n    let dst = unsafe { dev.alloc::<f32>(nrows).w()? };\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (nrows as u32, 1, 1),\n        block_dim: (WARP_SIZE as u32, 4, 1),\n        shared_mem_bytes: 0,\n    };\n\n    let params = (\n        data,\n        &y_q8_1,\n        &dst,\n        /* ncols_x */ ncols as i32,\n        /* nrows_x */ nrows as i32,\n        /* nrows_y */ ncols as i32,\n        /* nrows_dst */ nrows as i32,\n    );\n    unsafe { func.launch(cfg, params) }.w()?;\n    Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone()))\n}\n\nimpl QCudaStorage {\n    pub fn zeros(device: &CudaDevice, el_count: usize, dtype: GgmlDType) -> Result<Self> {\n        let size_in_bytes = ceil_div(el_count, dtype.block_size()) * dtype.type_size();\n        let data = device.alloc_zeros::<u8>(size_in_bytes).w()?;\n        Ok(QCudaStorage {\n            data,\n            device: device.clone(),\n            dtype,\n        })\n    }\n\n    pub fn dtype(&self) -> GgmlDType {\n        self.dtype\n    }\n\n    pub fn device(&self) -> &CudaDevice {\n        &self.device\n    }\n\n    pub fn dequantize(&self, elem_count: usize) -> Result<CudaStorage> {\n        fn deq<T: GgmlType>(buffer: &[u8], n: usize, dst: &mut [f32]) -> Result<()> {\n            let slice = unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const T, n) };\n            let vec = slice.to_vec();\n            T::to_float(&vec, dst)\n        }\n\n        let fast_kernel = matches!(\n            self.dtype,\n            GgmlDType::Q4_0\n                | GgmlDType::Q4_1\n                | GgmlDType::Q5_0\n                | GgmlDType::Q5_1\n                | GgmlDType::Q8_0\n                | GgmlDType::Q2K\n                | GgmlDType::Q3K\n                | GgmlDType::Q4K\n                | GgmlDType::Q5K\n                | GgmlDType::Q6K\n                | GgmlDType::Q8K\n        );\n        if fast_kernel {\n            return dequantize(&self.data, self.dtype, elem_count, self.device());\n        }\n        // Run the dequantization on cpu.\n\n        let buffer = self.device.dtoh_sync_copy(&self.data).w()?;\n        let mut out = vec![0.0; elem_count];\n        let block_len = elem_count / self.dtype.block_size();\n        match self.dtype {\n            GgmlDType::F32 => deq::<f32>(&buffer, block_len, &mut out)?,\n            GgmlDType::F16 => deq::<half::f16>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q4_0 => deq::<crate::quantized::BlockQ4_0>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q4_1 => deq::<crate::quantized::BlockQ4_1>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q5_0 => deq::<crate::quantized::BlockQ5_0>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q5_1 => deq::<crate::quantized::BlockQ5_1>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q8_0 => deq::<crate::quantized::BlockQ8_0>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q8_1 => deq::<crate::quantized::BlockQ8_1>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q2K => deq::<crate::quantized::BlockQ2K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q3K => deq::<crate::quantized::BlockQ3K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q4K => deq::<crate::quantized::BlockQ4K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q5K => deq::<crate::quantized::BlockQ5K>(&buffer, block_len, &mut out)?,\n            GgmlDType::Q6K => deq::<crate::quantized::BlockQ6K>(&buffer, block_len, &mut out)?,\n...\n", "needle_token_start": 13431, "needle_token_end": 14217, "code_context_ntokens": 16273, "output": ["```rust\nfn dequantize_mul_mat_vec(\n    data: &CudaSlice<u8>,\n    y: &CudaView<f32>,\n    dtype: GgmlDType,\n    ncols: usize,\n    nrows: usize,\n    dev: &CudaDevice,\n) -> Result<CudaStorage> {\n    use cudarc::driver::LaunchAsync;\n\n    let data_elems = data.len() / dtype.type_size() * dtype.block_size();\n    if data_elems < ncols * nrows {\n        crate::bail!(\"unexpected data size {}, ncols {ncols} {nrows}\", data_elems)\n    }\n    if y.len() != ncols {\n        crate::bail!(\"unexpected y size {}, ncols {ncols} {nrows}\", y.len())\n    }\n    let kernel_name = match dtype {\n        GgmlDType::Q4_0 => \"dequantize_mul_mat_vec_q4_0_cuda\",\n        GgmlDType::Q4_1 => \"dequantize_mul_mat_vec_q4_1_cuda\",\n        GgmlDType::Q5_0 => \"dequantize_mul_mat_vec_q5_0_cuda\",\n        GgmlDType::Q5_1 => \"dequantize_mul_mat_vec_q5_1_cuda\",\n        GgmlDType::Q8_0 => \"dequantize_mul_mat_vec_q8_0_cuda\",\n        GgmlDType::Q2K => \"dequantize_mul_mat_vec_q2_k\",\n        GgmlDType::Q3K => \"dequantize_mul_mat_vec_q3_k\",\n        GgmlDType::Q4K => \"dequantize_mul_mat_vec_q4_k\",\n        GgmlDType::Q5K => \"dequantize_mul_mat_vec_q5_k\",\n        GgmlDType::Q6K => \"dequantize_mul_mat_vec_q6_k\",\n        _ => crate::bail!(\"unsupported dtype for quantized matmul {dtype:?}\"),\n    };\n    let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?;\n    let dst = unsafe { dev.alloc::<f32>(nrows).w()? };\n    let block_num_y = ceil_div(nrows, GGML_CUDA_MMV_Y);\n    let cfg = cudarc::driver::LaunchConfig {\n        grid_dim: (block_num_y as u32, 1, 1),\n        block_dim: (WARP_SIZE as u32, GGML_CUDA_MMV_Y as u32, 1),\n        shared_mem_bytes: 0,\n    };\n\n    let params = (data, y, &dst, ncols as i32, nrows as i32);\n    unsafe { func.launch(cfg, params) }.w()?;\n    Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone()))\n}\n```"]}
{"repo": "huggingface/candle", "name": "get_id", "language": "rust", "path": "candle-core/src/backprop.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the gradient tensor associated with a specific tensor identifier from a storage system used during back propagation in neural network computations.\n2. **Input**: A unique identifier for a tensor.\n3. **Output**: The gradient tensor corresponding to the provided identifier, if it exists; otherwise, `None`.\n4. **Procedure**: The function checks the internal storage (a hash map) using the provided identifier as the key. If the key exists in the map, the associated gradient tensor is returned. If the key does not exist, the function returns `None`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: candle-core/src/storage.rs\nuse crate::backend::BackendStorage;\nuse crate::op::{self, CmpOp, ReduceOp};\nuse crate::{CpuStorage, CudaStorage, DType, Device, Error, Layout, MetalStorage, Result, Shape};\nuse crate::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3};\n\n// We do not want to implement Clone on Storage as cloning may fail because of\n// out of memory. Instead try_clone should be used.\n#[derive(Debug)]\npub enum Storage {\n    Cpu(CpuStorage),\n    Cuda(CudaStorage),\n    Metal(MetalStorage),\n}\n\nimpl Storage {\n    pub fn try_clone(&self, layout: &Layout) -> Result<Self> {\n        match self {\n            Self::Cpu(storage) => Ok(Self::Cpu(storage.clone())),\n            Self::Cuda(storage) => {\n                let storage = storage.try_clone(layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.try_clone(layout)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub fn device(&self) -> Device {\n        match self {\n            Self::Cpu(_) => Device::Cpu,\n            Self::Cuda(storage) => Device::Cuda(storage.device().clone()),\n            Self::Metal(storage) => Device::Metal(storage.device().clone()),\n        }\n    }\n\n    pub fn dtype(&self) -> DType {\n        match self {\n            Self::Cpu(storage) => storage.dtype(),\n            Self::Cuda(storage) => storage.dtype(),\n            Self::Metal(storage) => storage.dtype(),\n        }\n    }\n\n    pub(crate) fn same_device(&self, rhs: &Self, op: &'static str) -> Result<()> {\n        let lhs_device = self.device();\n        let rhs_device = rhs.device();\n        let lhs = lhs_device.location();\n        let rhs = rhs_device.location();\n        let same_device = if self.device().is_metal() {\n            // On metal, we require the device to be exactly the same rather than\n            // having the same location. In cuda this is not necessary as all CudaDevice on the\n            // same GPU will use the same cuda stream.\n            lhs_device.same_device(&rhs_device)\n        } else {\n            lhs == rhs\n        };\n        if !same_device {\n            Err(Error::DeviceMismatchBinaryOp { lhs, rhs, op }.bt())\n        } else {\n            Ok(())\n        }\n    }\n\n    pub(crate) fn same_dtype(&self, rhs: &Self, op: &'static str) -> Result<()> {\n        let lhs = self.dtype();\n        let rhs = rhs.dtype();\n        if lhs != rhs {\n            Err(Error::DTypeMismatchBinaryOp { lhs, rhs, op }.bt())\n        } else {\n            Ok(())\n        }\n    }\n\n    pub(crate) fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.affine(layout, mul, add)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn powf(&self, layout: &Layout, alpha: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.powf(layout, alpha)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn elu(&self, layout: &Layout, alpha: f64) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.elu(layout, alpha)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn cmp(\n        &self,\n        op: CmpOp,\n        rhs: &Self,\n        lhs_layout: &Layout,\n        rhs_layout: &Layout,\n    ) -> Result<Self> {\n        self.same_device(rhs, \"cmp\")?;\n        self.same_dtype(rhs, \"cmp\")?;\n        match (self, rhs) {\n            (Storage::Cpu(lhs), Storage::Cpu(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cpu(storage))\n            }\n            (Self::Cuda(lhs), Self::Cuda(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Cuda(storage))\n            }\n            (Self::Metal(lhs), Self::Metal(rhs)) => {\n                let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?;\n                Ok(Self::Metal(storage))\n            }\n            (lhs, rhs) => {\n                // Should not happen because of the same device check above but we're defensive\n                // anyway.\n                Err(Error::DeviceMismatchBinaryOp {\n                    lhs: lhs.device().location(),\n                    rhs: rhs.device().location(),\n                    op: \"cmp\",\n                }\n                .bt())\n            }\n        }\n    }\n\n    pub(crate) fn reduce_op(&self, op: ReduceOp, layout: &Layout, s: &[usize]) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.reduce_op(op, layout, s)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result<Self> {\n        match self {\n            Storage::Cpu(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Cpu(storage))\n            }\n            Self::Cuda(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Cuda(storage))\n            }\n            Self::Metal(storage) => {\n                let storage = storage.to_dtype(layout, dtype)?;\n                Ok(Self::Metal(storage))\n            }\n        }\n    }\n\n    pub(crate) fn apply_op1(&self, l: &Layout, c: &dyn CustomOp1) -> Result<(Self, Shape)> {\n        match self {\n            Self::Cpu(storage) => {\n                let (storage, shape) = c.cpu_fwd(storage, l)?;\n                Ok((Self::Cpu(storage), shape))\n            }\n            Self::Cuda(storage) => {\n                let (storage, shape) = c.cuda_fwd(storage, l)?;\n                Ok((Self::Cuda(storage), shape))\n            }\n            Self::Metal(storage) => {\n                let (storage, shape) = c.metal_fwd(storage, l)?;\n                Ok((Self::Metal(storage), shape))\n            }\n        }\n    }\n\n    pub(crate) fn apply_op2(\n        &self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        c: &dyn CustomOp2,\n    ) -> Result<(Self, Shape)> {\n        self.same_device(t2, c.name())?;\n        match (self, t2) {\n            (Self::Cpu(s1), Self::Cpu(s2)) => {\n                let (s, shape) = c.cpu_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Cpu(s), shape))\n            }\n            (Self::Cuda(s1), Self::Cuda(s2)) => {\n                let (s, shape) = c.cuda_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Cuda(s), shape))\n            }\n            (Self::Metal(s1), Self::Metal(s2)) => {\n                let (s, shape) = c.metal_fwd(s1, l1, s2, l2)?;\n                Ok((Self::Metal(s), shape))\n            }\n            _ => unreachable!(),\n        }\n    }\n\n    pub(crate) fn apply_op3(\n        &self,\n        l1: &Layout,\n        t2: &Self,\n        l2: &Layout,\n        t3: &Self,\n        l3: &Layout,\n        c: &dyn CustomOp3,\n    ) -> Result<(Self, Shape)> {\n        self.same_device(t2, c.name())?;\n        self.same_device(t3, c.name())?;\n        match (self, t2, t3) {\n            (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => {\n                let (s, shape) = c.cpu_fwd(s1, l1, s2, l2, s3, l3)?;\n                Ok((Self::Cpu(s), shape))\n            }\n...\n// Path: candle-core/src/scalar.rs\nuse crate::{Result, Tensor, WithDType};\n\npub enum TensorScalar {\n    Tensor(Tensor),\n    Scalar(Tensor),\n}\n\npub trait TensorOrScalar {\n    fn to_tensor_scalar(self) -> Result<TensorScalar>;\n}\n\nimpl TensorOrScalar for &Tensor {\n    fn to_tensor_scalar(self) -> Result<TensorScalar> {\n        Ok(TensorScalar::Tensor(self.clone()))\n    }\n}\n\nimpl<T: WithDType> TensorOrScalar for T {\n    fn to_tensor_scalar(self) -> Result<TensorScalar> {\n        let scalar = Tensor::new(self, &crate::Device::Cpu)?;\n        Ok(TensorScalar::Scalar(scalar))\n    }\n}\n\n// Path: candle-core/src/indexer.rs\nuse crate::{Error, Tensor};\nuse std::ops::{\n    Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive,\n};\n\nimpl Tensor {\n    /// Intended to be use by the trait `.i()`\n    ///\n    /// ```\n    /// # use candle_core::{Tensor, DType, Device, IndexOp};\n    /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?;\n    ///\n    /// let c = a.i(0..1)?;\n    /// assert_eq!(c.shape().dims(), &[1, 3]);\n    ///\n    /// let c = a.i(0)?;\n    /// assert_eq!(c.shape().dims(), &[3]);\n    ///\n    /// let c = a.i((.., ..2) )?;\n    /// assert_eq!(c.shape().dims(), &[2, 2]);\n    ///\n    /// let c = a.i((.., ..=2))?;\n    /// assert_eq!(c.shape().dims(), &[2, 3]);\n    ///\n    /// # Ok::<(), candle_core::Error>(())\n    /// ```\n    fn index(&self, indexers: &[TensorIndexer]) -> Result<Self, Error> {\n        let mut x = self.clone();\n        let dims = self.shape().dims();\n        let mut current_dim = 0;\n        for (i, indexer) in indexers.iter().enumerate() {\n            x = match indexer {\n                TensorIndexer::Select(n) => x.narrow(current_dim, *n, 1)?.squeeze(current_dim)?,\n                TensorIndexer::Narrow(left_bound, right_bound) => {\n                    let start = match left_bound {\n                        Bound::Included(n) => *n,\n                        Bound::Excluded(n) => *n + 1,\n                        Bound::Unbounded => 0,\n                    };\n                    let stop = match right_bound {\n                        Bound::Included(n) => *n + 1,\n                        Bound::Excluded(n) => *n,\n                        Bound::Unbounded => dims[i],\n                    };\n                    let out = x.narrow(current_dim, start, stop.saturating_sub(start))?;\n                    current_dim += 1;\n                    out\n                }\n                TensorIndexer::IndexSelect(indexes) => {\n                    if indexes.rank() != 1 {\n                        crate::bail!(\"multi-dimensional tensor indexing is not supported\")\n                    }\n                    let out = x.index_select(&indexes.to_device(x.device())?, current_dim)?;\n                    current_dim += 1;\n                    out\n                }\n                TensorIndexer::Err(e) => crate::bail!(\"indexing error {e:?}\"),\n            };\n        }\n        Ok(x)\n    }\n}\n\n#[derive(Debug)]\n/// Generic structure used to index a slice of the tensor\npub enum TensorIndexer {\n    /// This selects the elements for which an index has some specific value.\n    Select(usize),\n    /// This is a regular slice, purely indexing a chunk of the tensor\n    Narrow(Bound<usize>, Bound<usize>),\n    /// Indexing via a 1d tensor\n    IndexSelect(Tensor),\n    Err(Error),\n}\n\nimpl From<usize> for TensorIndexer {\n    fn from(index: usize) -> Self {\n        TensorIndexer::Select(index)\n    }\n}\n\nimpl From<&[u32]> for TensorIndexer {\n    fn from(index: &[u32]) -> Self {\n        match Tensor::new(index, &crate::Device::Cpu) {\n            Ok(tensor) => TensorIndexer::IndexSelect(tensor),\n            Err(e) => TensorIndexer::Err(e),\n        }\n    }\n}\n\nimpl From<Vec<u32>> for TensorIndexer {\n    fn from(index: Vec<u32>) -> Self {\n        let len = index.len();\n        match Tensor::from_vec(index, len, &crate::Device::Cpu) {\n            Ok(tensor) => TensorIndexer::IndexSelect(tensor),\n            Err(e) => TensorIndexer::Err(e),\n        }\n    }\n}\n\nimpl From<&Tensor> for TensorIndexer {\n    fn from(tensor: &Tensor) -> Self {\n        TensorIndexer::IndexSelect(tensor.clone())\n    }\n}\n\ntrait RB: RangeBounds<usize> {}\nimpl RB for Range<usize> {}\nimpl RB for RangeFrom<usize> {}\nimpl RB for RangeFull {}\nimpl RB for RangeInclusive<usize> {}\nimpl RB for RangeTo<usize> {}\nimpl RB for RangeToInclusive<usize> {}\n\nimpl<T: RB> From<T> for TensorIndexer {\n    fn from(range: T) -> Self {\n        use std::ops::Bound::*;\n        let start = match range.start_bound() {\n            Included(idx) => Included(*idx),\n            Excluded(idx) => Excluded(*idx),\n            Unbounded => Unbounded,\n        };\n        let end = match range.end_bound() {\n            Included(idx) => Included(*idx),\n            Excluded(idx) => Excluded(*idx),\n            Unbounded => Unbounded,\n        };\n        TensorIndexer::Narrow(start, end)\n    }\n}\n\n/// Trait used to implement multiple signatures for ease of use of the slicing\n/// of a tensor\npub trait IndexOp<T> {\n    /// Returns a slicing iterator which are the chunks of data necessary to\n    /// reconstruct the desired tensor.\n    fn i(&self, index: T) -> Result<Tensor, Error>;\n}\n\nimpl<T> IndexOp<T> for Tensor\nwhere\n    T: Into<TensorIndexer>,\n{\n    fn i(&self, index: T) -> Result<Tensor, Error> {\n        self.index(&[index.into()])\n    }\n}\n\nmacro_rules! index_op_tuple {\n    ($($t:ident),+) => {\n        #[allow(non_snake_case)]\n        impl<$($t),*> IndexOp<($($t,)*)> for Tensor\n        where\n            $($t: Into<TensorIndexer>,)*\n        {\n            fn i(&self, ($($t,)*): ($($t,)*)) -> Result<Tensor, Error> {\n                self.index(&[$($t.into(),)*])\n            }\n        }\n    };\n}\nindex_op_tuple!(A);\nindex_op_tuple!(A, B);\nindex_op_tuple!(A, B, C);\nindex_op_tuple!(A, B, C, D);\nindex_op_tuple!(A, B, C, D, E);\nindex_op_tuple!(A, B, C, D, E, F);\nindex_op_tuple!(A, B, C, D, E, F, G);\n\n// Path: candle-core/src/backprop.rs\n/// Methods for backpropagation of gradients.\nuse crate::op::{BinaryOp, Op, ReduceOp, UnaryOp};\nuse crate::{Error, Result, Tensor, TensorId};\nuse std::collections::HashMap;\n\n// arg has been reduced to node via reduce_dims, expand it back to arg.\n// This has to handle keepdims.\nfn broadcast_back(arg: &Tensor, node: &Tensor, reduced_dims: &[usize]) -> Result<Tensor> {\n    if arg.rank() == node.rank() {\n        // keepdim = true\n        node.broadcast_as(arg.shape())\n    } else {\n        // keepdim = false\n        // first expand the reduced dims.\n        node.reshape(reduced_dims)?.broadcast_as(arg.shape())\n    }\n}\n\nthread_local! {\n    static CANDLE_GRAD_DO_NOT_DETACH: bool = {\n        match std::env::var(\"CANDLE_GRAD_DO_NOT_DETACH\") {\n            Ok(s) => {\n                !s.is_empty() && s != \"0\"\n            },\n            Err(_) => false,\n        }\n    }\n}\n\nimpl Tensor {\n    /// Return all the nodes that lead to this value in a topologically sorted vec, the first\n    /// elements having dependencies on the latter ones, e.g. the first element if any is the\n    /// argument.\n    /// This assumes that the op graph is a DAG.\n    fn sorted_nodes(&self) -> Vec<&Tensor> {\n        // The vec of sorted nodes is passed as an owned value rather than a mutable reference\n        // to get around some lifetime limitations.\n        fn walk<'a>(\n            node: &'a Tensor,\n            nodes: Vec<&'a Tensor>,\n            already_seen: &mut HashMap<TensorId, bool>,\n        ) -> (bool, Vec<&'a Tensor>) {\n            if let Some(&tg) = already_seen.get(&node.id()) {\n                return (tg, nodes);\n            }\n            let mut track_grad = false;\n            let mut nodes = if node.is_variable() {\n                // Do not call recursively on the \"leaf\" nodes.\n                track_grad = true;\n                nodes\n            } else if node.dtype().is_int() {\n                nodes\n            } else if let Some(op) = node.op() {\n                match op {\n                    Op::IndexAdd(t1, t2, t3, _)\n                    | Op::ScatterAdd(t1, t2, t3, _)\n                    | Op::CustomOp3(t1, t2, t3, _)\n                    | Op::WhereCond(t1, t2, t3) => {\n                        let (tg, nodes) = walk(t1, nodes, already_seen);\n                        track_grad |= tg;\n                        let (tg, nodes) = walk(t2, nodes, already_seen);\n                        track_grad |= tg;\n                        let (tg, nodes) = walk(t3, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }\n                    Op::Conv1D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::ConvTranspose1D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::Conv2D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::ConvTranspose2D {\n                        arg: lhs,\n                        kernel: rhs,\n                        ..\n                    }\n                    | Op::CustomOp2(lhs, rhs, _)\n                    | Op::Binary(lhs, rhs, _)\n                    | Op::Gather(lhs, rhs, _)\n                    | Op::IndexSelect(lhs, rhs, _)\n                    | Op::Matmul(lhs, rhs)\n                    | Op::SliceScatter0(lhs, rhs, _) => {\n                        let (tg, nodes) = walk(lhs, nodes, already_seen);\n                        track_grad |= tg;\n                        let (tg, nodes) = walk(rhs, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }\n                    Op::Cat(args, _) => args.iter().fold(nodes, |nodes, arg| {\n                        let (tg, nodes) = walk(arg, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }),\n                    Op::Affine { arg, mul, .. } => {\n                        if *mul == 0. {\n                            nodes\n                        } else {\n                            let (tg, nodes) = walk(arg, nodes, already_seen);\n                            track_grad |= tg;\n                            nodes\n                        }\n                    }\n                    Op::Unary(_node, UnaryOp::Ceil)\n                    | Op::Unary(_node, UnaryOp::Floor)\n                    | Op::Unary(_node, UnaryOp::Round)\n                    | Op::Unary(_node, UnaryOp::Sign) => nodes,\n                    Op::Reshape(node)\n                    | Op::UpsampleNearest1D { arg: node, .. }\n                    | Op::UpsampleNearest2D { arg: node, .. }\n                    | Op::AvgPool2D { arg: node, .. }\n                    | Op::MaxPool2D { arg: node, .. }\n                    | Op::Copy(node)\n                    | Op::Broadcast(node)\n                    | Op::Cmp(node, _)\n                    | Op::Reduce(node, ReduceOp::Min | ReduceOp::Sum | ReduceOp::Max, _)\n                    | Op::ToDevice(node)\n                    | Op::Transpose(node, _, _)\n                    | Op::Permute(node, _)\n                    | Op::Narrow(node, _, _, _)\n                    | Op::Unary(node, _)\n                    | Op::Elu(node, _)\n                    | Op::Powf(node, _)\n                    | Op::CustomOp1(node, _) => {\n                        let (tg, nodes) = walk(node, nodes, already_seen);\n                        track_grad |= tg;\n                        nodes\n                    }\n                    Op::ToDType(node) => {\n                        if node.dtype().is_float() {\n                            let (tg, nodes) = walk(node, nodes, already_seen);\n                            track_grad |= tg;\n                            nodes\n                        } else {\n                            nodes\n                        }\n                    }\n                    Op::Reduce(_, ReduceOp::ArgMin | ReduceOp::ArgMax, _) => nodes,\n                }\n            } else {\n                nodes\n            };\n            already_seen.insert(node.id(), track_grad);\n            if track_grad {\n                nodes.push(node);\n            }\n            (track_grad, nodes)\n        }\n        let (_tg, mut nodes) = walk(self, vec![], &mut HashMap::new());\n        nodes.reverse();\n        nodes\n    }\n\n    pub fn backward(&self) -> Result<GradStore> {\n        let sorted_nodes = self.sorted_nodes();\n        let mut grads = GradStore::new();\n        grads.insert(self, self.ones_like()?.contiguous()?);\n        for node in sorted_nodes.iter() {\n            if node.is_variable() {\n                continue;\n            }\n            let grad = grads\n                .remove(node)\n                .expect(\"candle internal error - grad not populated\");\n            // https://github.com/huggingface/candle/issues/1241\n            // Ideally, we would make these operations in place where possible to ensure that we\n            // do not have to allocate too often. Here we just call `.detach` to avoid computing\n            // the backprop graph of the backprop itself. This would be an issue for second order\n            // derivatives but these are out of scope at the moment.\n            let do_not_detach = CANDLE_GRAD_DO_NOT_DETACH.with(|b| *b);\n            let grad = if do_not_detach { grad } else { grad.detach() };\n            if let Some(op) = node.op() {\n                match op {\n                    Op::Binary(lhs, rhs, BinaryOp::Add) => {\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&grad)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Sub) => {\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&grad)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.sub(&grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Mul) => {\n                        let lhs_grad = grad.mul(rhs)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n                        let rhs_grad = grad.mul(lhs)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Div) => {\n                        let lhs_grad = grad.div(rhs)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n                        let rhs_grad = grad.mul(lhs)?.div(&rhs.sqr()?)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.sub(&rhs_grad)?;\n                    }\n                    Op::Binary(lhs, rhs, BinaryOp::Minimum)\n                    | Op::Binary(lhs, rhs, BinaryOp::Maximum) => {\n                        let mask_lhs = node.eq(lhs)?.to_dtype(grad.dtype())?;\n                        let mask_rhs = node.eq(rhs)?.to_dtype(grad.dtype())?;\n\n                        // If both masks are 1 one the same point, we want to scale the\n                        // gradient by 0.5 rather than 1.\n                        let lhs_grad = mask_lhs.mul(&grad)?.div(&(&mask_rhs + 1.)?)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n\n                        let rhs_grad = mask_rhs.mul(&grad)?.div(&(&mask_lhs + 1.)?)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n                    }\n                    Op::WhereCond(pred, t, f) => {\n                        let zeros = grad.zeros_like()?;\n                        let t_sum_grad = grads.or_insert(t)?;\n                        let t_grad = pred.where_cond(&grad, &zeros)?;\n                        *t_sum_grad = t_sum_grad.add(&t_grad)?;\n                        let f_sum_grad = grads.or_insert(f)?;\n                        let f_grad = pred.where_cond(&zeros, &grad)?;\n                        *f_sum_grad = f_sum_grad.add(&f_grad)?;\n                    }\n                    Op::Conv1D {\n                        arg,\n                        kernel,\n                        padding,\n                        stride,\n                        dilation,\n                    } => {\n                        // The output height for conv_transpose1d is:\n                        // (l_in - 1) * stride - 2 * padding + dilation * (k_size - 1) + out_padding + 1\n                        let grad_l_in = grad.dim(2)?;\n                        let k_size = kernel.dim(2)?;\n                        let out_size =\n                            (grad_l_in - 1) * stride + dilation * (k_size - 1) + 1 - 2 * padding;\n                        let out_padding = arg.dim(2)? - out_size;\n                        let grad_arg = grad.conv_transpose1d(\n                            kernel,\n                            *padding,\n                            out_padding,\n                            *stride,\n                            *dilation,\n                            /* groups */ 1,\n                        )?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n\n                        let grad_kernel = arg\n                            .transpose(0, 1)?\n                            .conv1d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)?\n                            .transpose(0, 1)?;\n                        let sum_grad = grads.or_insert(kernel)?;\n                        let (_, _, k0) = kernel.dims3()?;\n                        let (_, _, g_k0) = grad_kernel.dims3()?;\n                        let grad_kernel = if g_k0 != k0 {\n                            grad_kernel.narrow(2, 0, k0)?\n                        } else {\n                            grad_kernel\n                        };\n                        *sum_grad = sum_grad.add(&grad_kernel)?;\n                    }\n                    Op::Conv2D {\n                        arg,\n                        kernel,\n                        padding,\n                        stride,\n                        dilation,\n                    } => {\n                        // The output height for conv_transpose2d is:\n                        // (i_h - 1) * stride - 2 * padding + dilation * (k_h - 1) + out_padding + 1\n                        let grad_h = grad.dim(2)?;\n                        let k_h = kernel.dim(2)?;\n                        let out_size =\n                            (grad_h - 1) * stride + dilation * (k_h - 1) + 1 - 2 * padding;\n                        let out_padding = arg.dim(2)? - out_size;\n                        let grad_arg = grad.conv_transpose2d(\n                            kernel,\n                            *padding,\n                            out_padding,\n                            *stride,\n                            *dilation,\n                        )?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n\n                        let grad_kernel = arg\n                            .transpose(0, 1)?\n                            .conv2d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)?\n                            .transpose(0, 1)?;\n                        let sum_grad = grads.or_insert(kernel)?;\n                        let (_, _, k0, k1) = kernel.dims4()?;\n                        let (_, _, g_k0, g_k1) = grad_kernel.dims4()?;\n                        let grad_kernel = if g_k0 != k0 || g_k1 != k1 {\n                            grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)?\n                        } else {\n                            grad_kernel\n                        };\n                        *sum_grad = sum_grad.add(&grad_kernel)?;\n                    }\n                    Op::ConvTranspose1D { .. } => Err(Error::BackwardNotSupported {\n                        op: \"conv-transpose1d\",\n                    })?,\n                    Op::ConvTranspose2D {\n                        arg,\n                        kernel,\n                        padding,\n                        stride,\n                        dilation,\n                        output_padding: _output_padding,\n                    } => {\n                        let grad_arg = grad.conv2d(kernel, *padding, *dilation, *stride, 1)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n\n                        let grad_kernel = grad\n                            .transpose(0, 1)?\n                            .conv2d(&arg.transpose(0, 1)?, *padding, *stride, *dilation, 1)?\n                            .transpose(0, 1)?;\n                        let sum_grad = grads.or_insert(kernel)?;\n                        let (_, _, k0, k1) = kernel.dims4()?;\n                        let (_, _, g_k0, g_k1) = grad_kernel.dims4()?;\n                        let grad_kernel = if g_k0 != k0 || g_k1 != k1 {\n                            grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)?\n                        } else {\n                            grad_kernel\n                        };\n                        *sum_grad = sum_grad.add(&grad_kernel)?;\n                    }\n                    Op::AvgPool2D {\n                        arg,\n                        kernel_size,\n                        stride,\n                    } => {\n                        if kernel_size != stride {\n                            crate::bail!(\"backward not supported for avgpool2d if ksize {kernel_size:?} != stride {stride:?}\")\n                        }\n                        let (_n, _c, h, w) = arg.dims4()?;\n                        let grad_arg = grad.upsample_nearest2d(h, w)?;\n                        let grad_arg =\n                            (grad_arg * (1f64 / (kernel_size.0 * kernel_size.1) as f64))?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n                    }\n                    Op::MaxPool2D {\n                        arg,\n                        kernel_size,\n                        stride,\n                    } => {\n                        if kernel_size != stride {\n                            crate::bail!(\"backward not supported for maxpool2d if ksize {kernel_size:?} != stride {stride:?}\")\n                        }\n                        let (_n, _c, h, w) = arg.dims4()?;\n                        // For computing the max-pool gradient, we compute a mask where a 1 means\n                        // that the element is the maximum, then we apply this mask to the\n                        // upsampled gradient (taking into account that multiple max may exist so\n                        // we scale the gradient for this case).\n                        let node_upsampled = node.upsample_nearest2d(h, w)?;\n                        let mask = arg.eq(&node_upsampled)?.to_dtype(arg.dtype())?;\n                        let avg = mask.avg_pool2d_with_stride(*kernel_size, *stride)?;\n                        let grad_arg = ((grad * avg)?.upsample_nearest2d(h, w)? * mask)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad_arg)?;\n                    }\n                    Op::UpsampleNearest1D { arg, target_size } => {\n                        let (_n, c, size) = arg.dims3()?;\n                        if target_size % size != 0 {\n                            crate::bail!(\"backward not supported for non integer upscaling factors\")\n                        }\n                        let scale = target_size / size;\n\n                        let kernel = Tensor::ones((c, 1, scale), arg.dtype(), arg.device())?;\n                        let conv_sum = grad.conv1d(&kernel, 0, scale, 1, c)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = conv_sum;\n                    }\n                    Op::UpsampleNearest2D {\n                        arg,\n                        target_h,\n                        target_w,\n                    } => {\n                        let (_n, c, h, w) = arg.dims4()?;\n                        if target_h % h != 0 || target_w % w != 0 {\n                            crate::bail!(\"backward not supported for non integer upscaling factors\")\n                        }\n                        let scale_h = target_h / h;\n                        let scale_w = target_w / w;\n\n                        if scale_h != scale_w {\n                            crate::bail!(\"backward not supported for non uniform upscaling factors\")\n                        };\n                        let kernel =\n                            Tensor::ones((c, 1, scale_h, scale_w), arg.dtype(), arg.device())?;\n                        let conv_sum = grad.conv2d(&kernel, 0, scale_h, 1, c)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = conv_sum;\n                    }\n                    Op::SliceScatter0(lhs, rhs, start_rhs) => {\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        let rhs_grad = grad.narrow(0, *start_rhs, rhs.dim(0)?)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        let lhs_grad = grad.slice_scatter0(&rhs.zeros_like()?, *start_rhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?\n                    }\n                    Op::Gather(arg, indexes, dim) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.scatter_add(indexes, &grad, *dim)?;\n                    }\n                    Op::ScatterAdd(init, indexes, src, dim) => {\n                        let init_sum_grad = grads.or_insert(init)?;\n                        *init_sum_grad = init_sum_grad.add(&grad)?;\n\n                        let src_grad = grad.gather(indexes, *dim)?;\n                        let src_sum_grad = grads.or_insert(src)?;\n                        *src_sum_grad = src_sum_grad.add(&src_grad)?;\n                    }\n                    Op::IndexAdd(init, indexes, src, dim) => {\n                        let init_sum_grad = grads.or_insert(init)?;\n                        *init_sum_grad = init_sum_grad.add(&grad)?;\n\n                        let src_grad = grad.index_select(indexes, *dim)?;\n                        let src_sum_grad = grads.or_insert(src)?;\n                        *src_sum_grad = src_sum_grad.add(&src_grad)?;\n                    }\n                    Op::IndexSelect(arg, indexes, dim) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.index_add(indexes, &grad, *dim)?;\n                    }\n                    Op::Matmul(lhs, rhs) => {\n                        // Skipping checks, the op went ok, we can skip\n                        // the matmul size checks for now.\n\n                        let lhs_grad = grad.matmul(&rhs.t()?)?;\n                        let lhs_sum_grad = grads.or_insert(lhs)?;\n                        *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?;\n\n                        let rhs_grad = lhs.t()?.matmul(&grad)?;\n                        let rhs_sum_grad = grads.or_insert(rhs)?;\n                        *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?;\n                    }\n                    Op::Cat(args, dim) => {\n                        let mut start_idx = 0;\n                        for arg in args {\n                            let len = arg.dims()[*dim];\n                            let arg_grad = grad.narrow(*dim, start_idx, len)?;\n                            let sum_grad = grads.or_insert(arg)?;\n                            *sum_grad = sum_grad.add(&arg_grad)?;\n                            start_idx += len;\n                        }\n                    }\n                    Op::Broadcast(arg) => {\n                        let arg_dims = arg.dims();\n                        let node_dims = node.dims();\n                        // The number of dims that have been inserted on the left.\n                        let left_dims = node_dims.len() - arg_dims.len();\n                        let mut sum_dims: Vec<usize> = (0..left_dims).collect();\n                        for (dim, (node_dim, arg_dim)) in node_dims[left_dims..]\n                            .iter()\n                            .zip(arg_dims.iter())\n                            .enumerate()\n                        {\n                            if node_dim != arg_dim {\n                                sum_dims.push(dim + left_dims)\n                            }\n                        }\n\n                        let mut arg_grad = grad.sum_keepdim(sum_dims.as_slice())?;\n                        for _i in 0..left_dims {\n                            arg_grad = arg_grad.squeeze(0)?\n                        }\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad.broadcast_as(sum_grad.dims())?)?;\n                    }\n                    Op::Reduce(arg, ReduceOp::Sum, reduced_dims) => {\n                        let grad = broadcast_back(arg, &grad, reduced_dims)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad)?;\n                    }\n                    Op::Reduce(arg, ReduceOp::Max, reduced_dims) => {\n                        let node = broadcast_back(arg, node, reduced_dims)?;\n                        let grad = broadcast_back(arg, &grad, reduced_dims)?;\n                        let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?;\n                    }\n                    Op::Reduce(arg, ReduceOp::Min, reduced_dims) => {\n                        let node = broadcast_back(arg, node, reduced_dims)?;\n                        let grad = broadcast_back(arg, &grad, reduced_dims)?;\n                        let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?;\n                    }\n                    Op::ToDType(arg) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad.to_dtype(arg.dtype())?)?\n                    }\n                    Op::Copy(arg) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&grad)?\n                    }\n                    Op::Affine { arg, mul, .. } => {\n                        let arg_grad = grad.affine(*mul, 0.)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Unary(arg, UnaryOp::Log) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&(grad / arg)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Sin) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&(&grad * arg.cos())?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Cos) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.sub(&(&grad * arg.sin())?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Tanh) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let minus_dtanh = (node.sqr()? - 1.)?;\n                        *sum_grad = sum_grad.sub(&(&grad * &minus_dtanh)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Abs) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let ones = arg.ones_like()?;\n                        let abs_grad = arg.ge(&arg.zeros_like()?)?.where_cond(&ones, &ones.neg()?);\n                        *sum_grad = sum_grad.add(&(&grad * abs_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Exp) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&(&grad * *node)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Neg) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.sub(&grad)?\n                    }\n                    Op::Unary(arg, UnaryOp::Recip) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let grad = (grad / arg.sqr()?)?;\n                        *sum_grad = sum_grad.sub(&grad)?\n                    }\n                    &Op::Narrow(ref arg, dim, start_idx, len) => {\n                        let arg_dims = arg.dims();\n                        let left_pad = if start_idx == 0 {\n                            None\n                        } else {\n                            let mut dims = arg_dims.to_vec();\n                            dims[dim] = start_idx;\n                            Some(Tensor::zeros(dims, grad.dtype(), grad.device())?)\n                        };\n                        let right_pad = arg_dims[dim] - start_idx - len;\n                        let right_pad = if right_pad == 0 {\n                            None\n                        } else {\n                            let mut dims = arg_dims.to_vec();\n                            dims[dim] = right_pad;\n                            Some(Tensor::zeros(dims, grad.dtype(), grad.device())?)\n                        };\n                        let arg_grad = match (left_pad, right_pad) {\n                            (None, None) => grad,\n                            (Some(l), None) => Tensor::cat(&[&l, &grad], dim)?,\n                            (None, Some(r)) => Tensor::cat(&[&grad, &r], dim)?,\n                            (Some(l), Some(r)) => Tensor::cat(&[&l, &grad, &r], dim)?,\n                        };\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Unary(_, UnaryOp::Floor)\n                    | Op::Unary(_, UnaryOp::Round)\n                    | Op::Reduce(_, ReduceOp::ArgMin, _)\n                    | Op::Reduce(_, ReduceOp::ArgMax, _)\n                    | Op::Unary(_, UnaryOp::Sign)\n                    | Op::Cmp(_, _) => {}\n                    Op::Reshape(arg) => {\n                        let arg_grad = grad.reshape(arg.dims())?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Unary(_, UnaryOp::Ceil) => Err(Error::BackwardNotSupported { op: \"ceil\" })?,\n                    Op::Unary(arg, UnaryOp::Gelu) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let cube = arg.powf(3.)?;\n                        let tanh = (0.0356774 * &cube + (0.797885 * arg)?)?.tanh()?;\n                        let gelu_grad = (((0.5 * &tanh)?\n                            + (0.0535161 * cube + (0.398942 * arg)?)? * (1. - tanh.powf(2.)?))?\n                            + 0.5)?;\n                        *sum_grad = sum_grad.add(&(&grad * gelu_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Erf) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        // d/dx erf(x) = 2/sqrt(pi) * e^(-x^2)\n                        let erf_grad =\n                            (2. / std::f64::consts::PI.sqrt()) * (arg.sqr()?.neg()?).exp()?;\n                        *sum_grad = sum_grad.add(&(&grad * erf_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::GeluErf) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        // d/dx gelu_erf(x) = 0.5 + 0.398942 e^(-x^2/2) x + 0.5 erf(x/sqrt(2))\n                        let neg_half_square = (arg.sqr()?.neg()? / 2.)?;\n                        let scaled_exp_arg = (0.398942 * neg_half_square.exp()? * arg)?;\n                        let arg_scaled_sqrt = (arg / 2f64.sqrt())?;\n                        let erf_scaled_sqrt = (0.5 * arg_scaled_sqrt.erf()?)?;\n                        let gelu_erf_grad = (0.5 + scaled_exp_arg + erf_scaled_sqrt)?;\n                        *sum_grad = sum_grad.add(&(&grad * gelu_erf_grad)?)?;\n                    }\n                    Op::Unary(arg, UnaryOp::Relu) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let relu_grad = arg.ge(&arg.zeros_like()?)?.to_dtype(arg.dtype())?;\n                        *sum_grad = sum_grad.add(&(&grad * relu_grad)?)?\n                    }\n                    Op::Unary(arg, UnaryOp::Silu) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        // d/dx silu = sigmoid(x) * (1 + x * (1 - sigmoid(x)))\n                        let sigmoid_arg = (*node / arg)?;\n                        let silu_grad = (&sigmoid_arg * (1. + (arg * (1. - &sigmoid_arg)?)?)?)?;\n                        *sum_grad = sum_grad.add(&(&grad * silu_grad)?)?\n                    }\n                    Op::Elu(arg, alpha) => {\n                        // d/dx elu(x) = 1 for x > 0, alpha * e^x for x <= 0\n                        let sum_grad = grads.or_insert(arg)?;\n                        let zeros = arg.zeros_like()?;\n                        let positive_mask = arg.gt(&zeros)?.to_dtype(arg.dtype())?;\n                        let negative_mask = arg.le(&zeros)?.to_dtype(arg.dtype())?;\n                        let negative_exp_mask = ((negative_mask * arg.exp())? * *alpha)?;\n                        let combined_mask = (positive_mask + negative_exp_mask)?;\n                        *sum_grad = sum_grad.add(&(grad * combined_mask)?)?\n                    }\n                    Op::Powf(arg, e) => {\n                        let arg_grad = (&(grad * arg.powf(e - 1.)?)? * *e)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::CustomOp1(arg, c) => {\n                        if let Some(arg_grad) = c.bwd(arg, node, &grad)? {\n                            let sum_grad = grads.or_insert(arg)?;\n                            *sum_grad = sum_grad.add(&arg_grad)?\n                        }\n                    }\n                    Op::CustomOp2(arg1, arg2, c) => {\n                        let (arg_grad1, arg_grad2) = c.bwd(arg1, arg2, node, &grad)?;\n                        if let Some(arg_grad1) = arg_grad1 {\n                            let sum_grad = grads.or_insert(arg1)?;\n                            *sum_grad = sum_grad.add(&arg_grad1)?\n                        }\n                        if let Some(arg_grad2) = arg_grad2 {\n                            let sum_grad = grads.or_insert(arg2)?;\n                            *sum_grad = sum_grad.add(&arg_grad2)?\n                        }\n                    }\n                    Op::CustomOp3(arg1, arg2, arg3, c) => {\n                        let (arg_grad1, arg_grad2, arg_grad3) =\n                            c.bwd(arg1, arg2, arg3, node, &grad)?;\n                        if let Some(arg_grad1) = arg_grad1 {\n                            let sum_grad = grads.or_insert(arg1)?;\n                            *sum_grad = sum_grad.add(&arg_grad1)?\n                        }\n                        if let Some(arg_grad2) = arg_grad2 {\n                            let sum_grad = grads.or_insert(arg2)?;\n                            *sum_grad = sum_grad.add(&arg_grad2)?\n                        }\n                        if let Some(arg_grad3) = arg_grad3 {\n                            let sum_grad = grads.or_insert(arg3)?;\n                            *sum_grad = sum_grad.add(&arg_grad3)?\n                        }\n                    }\n                    Op::Unary(arg, UnaryOp::Sqr) => {\n                        let arg_grad = arg.mul(&grad)?.affine(2., 0.)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Unary(arg, UnaryOp::Sqrt) => {\n                        let arg_grad = grad.div(node)?.affine(0.5, 0.)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::ToDevice(arg) => {\n                        let sum_grad = grads.or_insert(arg)?;\n                        let arg_grad = grad.to_device(sum_grad.device())?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Transpose(arg, dim1, dim2) => {\n                        let arg_grad = grad.transpose(*dim1, *dim2)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                    Op::Permute(arg, dims) => {\n                        let mut inv_dims = vec![0; dims.len()];\n                        for (i, &dim_idx) in dims.iter().enumerate() {\n                            inv_dims[dim_idx] = i\n                        }\n                        let arg_grad = grad.permute(inv_dims)?;\n                        let sum_grad = grads.or_insert(arg)?;\n                        *sum_grad = sum_grad.add(&arg_grad)?\n                    }\n                };\n            }\n        }\n        Ok(grads)\n    }\n}\n\n/// A store for gradients, associating a tensor id to the corresponding gradient tensor, used for back propagation.\n#[derive(Debug)]\npub struct GradStore(HashMap<TensorId, Tensor>);\n\nimpl GradStore {\n    /// Create a new gradient store\n    fn new() -> Self {\n        GradStore(HashMap::new())\n    }\n\n    /// Get the gradient tensor corresponding to the given tensor id\n    \npub fn get_id(&self, id: TensorId) -> Option<&Tensor> {\n        self.0.get(&id)\n    }\n\n    /// Get the gradient tensor associated with the given tensor\n    pub fn get(&self, tensor: &Tensor) -> Option<&Tensor> {\n        self.0.get(&tensor.id())\n    }\n\n    /// Remove the gradient tensor associated with the given tensor, returning it if it exists\n    pub fn remove(&mut self, tensor: &Tensor) -> Option<Tensor> {\n        self.0.remove(&tensor.id())\n    }\n\n    /// Insert a gradient tensor associated with the given tensor, returning the previous gradient tensor if it existed\n    pub fn insert(&mut self, tensor: &Tensor, grad: Tensor) -> Option<Tensor> {\n        self.0.insert(tensor.id(), grad)\n    }\n\n    /// Get the gradient tensor associated with the given tensor, or, if it does not exist,\n    /// insert a tensor of zeroes, with the same shape and type as the given tensors and return it\n    fn or_insert(&mut self, tensor: &Tensor) -> Result<&mut Tensor> {\n        use std::collections::hash_map::Entry;\n        let grad = match self.0.entry(tensor.id()) {\n            Entry::Occupied(entry) => entry.into_mut(),\n            Entry::Vacant(entry) => {\n                let grad = tensor.zeros_like()?;\n                entry.insert(grad)\n            }\n        };\n        Ok(grad)\n    }\n}\n\n// Path: candle-core/src/tensor.rs\n//! Tensors are N-dimensional matrixes of elements using a single data type.\n#![allow(clippy::redundant_closure_call)]\nuse crate::backend::{BackendDevice, BackendStorage};\nuse crate::op::{BackpropOp, BinaryOp, CmpOp, Op, ReduceOp, UnaryOp};\nuse crate::scalar::TensorOrScalar;\nuse crate::shape::{Dim, Dims};\nuse crate::{bail, storage::Storage, DType, Device, Error, Layout, Result, Shape};\nuse std::sync::{Arc, RwLock};\n\n/// Unique identifier for tensors.\n#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]\npub struct TensorId(usize);\n\nimpl TensorId {\n    fn new() -> Self {\n        // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805\n        use std::sync::atomic;\n        static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1);\n        Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed))\n    }\n}\n\npub struct Tensor_ {\n    id: TensorId,\n    // As we provide inner mutability on the tensor content, the alternatives are:\n    // - Using a mutex, this would have the highest cost when retrieving the storage but would\n    //   prevent errors when concurrent access takes place. Mutex would also be subject to\n    //   deadlocks for example using the current code if the same tensor is used twice by a single\n    //   binary op.\n    // - Using a refcell unsafe cell would have some intermediary cost, borrow checking would be\n...\n", "needle_token_start": 15477, "needle_token_end": 15512, "code_context_ntokens": 16302, "output": ["```rust\npub fn get_id(&self, id: TensorId) -> Option<&Tensor> {\n    self.0.get(&id)\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "reject", "language": "rust", "path": "src/reject.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle situations where a request does not meet certain criteria within a web application's routing logic, effectively stopping further processing of the request and indicating that the resource was not found.\n2. **Input**: This function does not take any parameters.\n3. **Output**: Returns a standardized error object that encapsulates the status of a request being not found or unprocessable within the context of the application's routing logic.\n4. **Procedure**: When invoked, the function immediately returns an error object that represents a HTTP 404 Not Found error, signaling that the requested resource does not exist or cannot be handled by the current routing setup.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "//! doesn't fulfill its requirements, the other side can try to process\n//! the request.\n//!\n//! Many of the built-in [`filters`](../filters) will automatically reject\n//! the request with an appropriate rejection. However, you can also build\n//! new custom [`Filter`](../trait.Filter.html)s and still want other routes to be\n//! matchable in the case a predicate doesn't hold.\n//!\n//! As a request is processed by a Filter chain, the rejections are accumulated into\n//! a list contained by the [`Rejection`](struct.Rejection.html) type. Rejections from\n//! filters can be handled using [`Filter::recover`](../trait.Filter.html#method.recover).\n//! This is a convenient way to map rejections into a [`Reply`](../reply/trait.Reply.html).\n//!\n//! For a more complete example see the\n//! [Rejection Example](https://github.com/seanmonstar/warp/blob/master/examples/rejections.rs)\n//! from the repository.\n//!\n//! # Example\n//!\n//! ```\n//! use warp::{reply, Reply, Filter, reject, Rejection, http::StatusCode};\n//!\n//! #[derive(Debug)]\n//! struct InvalidParameter;\n//!\n//! impl reject::Reject for InvalidParameter {}\n//!\n//! // Custom rejection handler that maps rejections into responses.\n//! async fn handle_rejection(err: Rejection) -> Result<impl Reply, std::convert::Infallible> {\n//!     if err.is_not_found() {\n//!         Ok(reply::with_status(\"NOT_FOUND\", StatusCode::NOT_FOUND))\n//!     } else if let Some(e) = err.find::<InvalidParameter>() {\n//!         Ok(reply::with_status(\"BAD_REQUEST\", StatusCode::BAD_REQUEST))\n//!     } else {\n//!         eprintln!(\"unhandled rejection: {:?}\", err);\n//!         Ok(reply::with_status(\"INTERNAL_SERVER_ERROR\", StatusCode::INTERNAL_SERVER_ERROR))\n//!     }\n//! }\n//!\n//!\n//! // Filter on `/:id`, but reject with InvalidParameter if the `id` is `0`.\n//! // Recover from this rejection using a custom rejection handler.\n//! let route = warp::path::param()\n//!     .and_then(|id: u32| async move {\n//!         if id == 0 {\n//!             Err(warp::reject::custom(InvalidParameter))\n//!         } else {\n//!             Ok(\"id is valid\")\n//!         }\n//!     })\n//!     .recover(handle_rejection);\n//! ```\n\nuse std::any::Any;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::fmt;\n\nuse http::{\n    header::{HeaderValue, CONTENT_TYPE},\n    StatusCode,\n};\nuse hyper::Body;\n\npub(crate) use self::sealed::{CombineRejection, IsReject};\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\n\npub fn reject() -> Rejection {\n    not_found()\n}\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn not_found() -> Rejection {\n    Rejection {\n        reason: Reason::NotFound,\n    }\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_query() -> Rejection {\n    known(InvalidQuery { _p: () })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_header(name: &'static str) -> Rejection {\n    known(MissingHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_header(name: &'static str) -> Rejection {\n    known(InvalidHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_cookie(name: &'static str) -> Rejection {\n    known(MissingCookie { name })\n}\n\n// 405 Method Not Allowed\n#[inline]\npub(crate) fn method_not_allowed() -> Rejection {\n    known(MethodNotAllowed { _p: () })\n}\n\n// 411 Length Required\n#[inline]\npub(crate) fn length_required() -> Rejection {\n    known(LengthRequired { _p: () })\n}\n\n// 413 Payload Too Large\n#[inline]\npub(crate) fn payload_too_large() -> Rejection {\n    known(PayloadTooLarge { _p: () })\n}\n\n// 415 Unsupported Media Type\n//\n// Used by the body filters if the request payload content-type doesn't match\n// what can be deserialized.\n#[inline]\npub(crate) fn unsupported_media_type() -> Rejection {\n    known(UnsupportedMediaType { _p: () })\n}\n\n/// Rejects a request with a custom cause.\n///\n/// A [`recover`][] filter should convert this `Rejection` into a `Reply`,\n/// or else this will be returned as a `500 Internal Server Error`.\n///\n/// [`recover`]: ../trait.Filter.html#method.recover\npub fn custom<T: Reject>(err: T) -> Rejection {\n    Rejection::custom(Box::new(err))\n}\n\n/// Protect against re-rejecting a rejection.\n///\n/// ```compile_fail\n/// fn with(r: warp::Rejection) {\n///     let _wat = warp::reject::custom(r);\n/// }\n/// ```\nfn __reject_custom_compilefail() {}\n\n/// A marker trait to ensure proper types are used for custom rejections.\n///\n/// Can be converted into Rejection.\n///\n/// # Example\n///\n/// ```\n/// use warp::{Filter, reject::Reject};\n///\n/// #[derive(Debug)]\n/// struct RateLimited;\n///\n/// impl Reject for RateLimited {}\n///\n/// let route = warp::any().and_then(|| async {\n///     Err::<(), _>(warp::reject::custom(RateLimited))\n/// });\n/// ```\n// Require `Sized` for now to prevent passing a `Box<dyn Reject>`, since we\n// would be double-boxing it, and the downcasting wouldn't work as expected.\npub trait Reject: fmt::Debug + Sized + Send + Sync + 'static {}\n\ntrait Cause: fmt::Debug + Send + Sync + 'static {\n    fn as_any(&self) -> &dyn Any;\n}\n\nimpl<T> Cause for T\nwhere\n    T: fmt::Debug + Send + Sync + 'static,\n{\n    fn as_any(&self) -> &dyn Any {\n        self\n    }\n}\n\nimpl dyn Cause {\n    fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        self.as_any().downcast_ref::<T>()\n    }\n}\n\npub(crate) fn known<T: Into<Known>>(err: T) -> Rejection {\n    Rejection::known(err.into())\n}\n\n/// Rejection of a request by a [`Filter`](crate::Filter).\n///\n/// See the [`reject`](module@crate::reject) documentation for more.\npub struct Rejection {\n    reason: Reason,\n}\n\nenum Reason {\n    NotFound,\n    Other(Box<Rejections>),\n}\n\nenum Rejections {\n    Known(Known),\n    Custom(Box<dyn Cause>),\n    Combined(Box<Rejections>, Box<Rejections>),\n}\n\nmacro_rules! enum_known {\n     ($($(#[$attr:meta])* $var:ident($ty:path),)+) => (\n        pub(crate) enum Known {\n            $(\n            $(#[$attr])*\n            $var($ty),\n            )+\n        }\n\n        impl Known {\n            fn inner_as_any(&self) -> &dyn Any {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t,\n                    )+\n                }\n            }\n        }\n\n        impl fmt::Debug for Known {\n            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t.fmt(f),\n                    )+\n                }\n            }\n        }\n\n        impl fmt::Display for Known {\n            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t.fmt(f),\n                    )+\n                }\n            }\n        }\n\n        $(\n        #[doc(hidden)]\n        $(#[$attr])*\n        impl From<$ty> for Known {\n            fn from(ty: $ty) -> Known {\n                Known::$var(ty)\n            }\n        }\n        )+\n    );\n}\n\nenum_known! {\n    MethodNotAllowed(MethodNotAllowed),\n    InvalidHeader(InvalidHeader),\n    MissingHeader(MissingHeader),\n    MissingCookie(MissingCookie),\n    InvalidQuery(InvalidQuery),\n    LengthRequired(LengthRequired),\n    PayloadTooLarge(PayloadTooLarge),\n    UnsupportedMediaType(UnsupportedMediaType),\n    FileOpenError(crate::fs::FileOpenError),\n    FilePermissionError(crate::fs::FilePermissionError),\n    BodyReadError(crate::body::BodyReadError),\n    BodyDeserializeError(crate::body::BodyDeserializeError),\n    CorsForbidden(crate::cors::CorsForbidden),\n    #[cfg(feature = \"websocket\")]\n    MissingConnectionUpgrade(crate::ws::MissingConnectionUpgrade),\n    MissingExtension(crate::ext::MissingExtension),\n    BodyConsumedMultipleTimes(crate::body::BodyConsumedMultipleTimes),\n}\n\nimpl Rejection {\n    fn known(known: Known) -> Self {\n        Rejection {\n            reason: Reason::Other(Box::new(Rejections::Known(known))),\n        }\n    }\n\n    fn custom(other: Box<dyn Cause>) -> Self {\n        Rejection {\n            reason: Reason::Other(Box::new(Rejections::Custom(other))),\n        }\n    }\n\n    /// Searches this `Rejection` for a specific cause.\n    ///\n    /// A `Rejection` will accumulate causes over a `Filter` chain. This method\n    /// can search through them and return the first cause of this type.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// #[derive(Debug)]\n    /// struct Nope;\n    ///\n    /// impl warp::reject::Reject for Nope {}\n    ///\n    /// let reject = warp::reject::custom(Nope);\n    ///\n    /// if let Some(nope) = reject.find::<Nope>() {\n    ///    println!(\"found it: {:?}\", nope);\n    /// }\n    /// ```\n    pub fn find<T: 'static>(&self) -> Option<&T> {\n        if let Reason::Other(ref rejections) = self.reason {\n            return rejections.find();\n        }\n        None\n    }\n\n    /// Returns true if this Rejection was made via `warp::reject::not_found`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let rejection = warp::reject();\n    ///\n    /// assert!(rejection.is_not_found());\n    /// ```\n    pub fn is_not_found(&self) -> bool {\n        matches!(self.reason, Reason::NotFound)\n    }\n}\n\nimpl<T: Reject> From<T> for Rejection {\n    #[inline]\n    fn from(err: T) -> Rejection {\n        custom(err)\n    }\n}\n\nimpl From<Infallible> for Rejection {\n    #[inline]\n    fn from(infallible: Infallible) -> Rejection {\n        match infallible {}\n    }\n}\n\nimpl IsReject for Infallible {\n    fn status(&self) -> StatusCode {\n        match *self {}\n    }\n\n    fn into_response(&self) -> crate::reply::Response {\n        match *self {}\n    }\n}\n\nimpl IsReject for Rejection {\n    fn status(&self) -> StatusCode {\n        match self.reason {\n            Reason::NotFound => StatusCode::NOT_FOUND,\n            Reason::Other(ref other) => other.status(),\n        }\n    }\n\n    fn into_response(&self) -> crate::reply::Response {\n        match self.reason {\n            Reason::NotFound => {\n                let mut res = http::Response::default();\n                *res.status_mut() = StatusCode::NOT_FOUND;\n                res\n            }\n            Reason::Other(ref other) => other.into_response(),\n        }\n    }\n}\n\nimpl fmt::Debug for Rejection {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"Rejection\").field(&self.reason).finish()\n    }\n}\n\nimpl fmt::Debug for Reason {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match *self {\n            Reason::NotFound => f.write_str(\"NotFound\"),\n            Reason::Other(ref other) => match **other {\n                Rejections::Known(ref e) => fmt::Debug::fmt(e, f),\n                Rejections::Custom(ref e) => fmt::Debug::fmt(e, f),\n                Rejections::Combined(ref a, ref b) => {\n                    let mut list = f.debug_list();\n                    a.debug_list(&mut list);\n                    b.debug_list(&mut list);\n                    list.finish()\n                }\n            },\n        }\n    }\n}\n\n// ===== Rejections =====\n\nimpl Rejections {\n    fn status(&self) -> StatusCode {\n        match *self {\n            Rejections::Known(ref k) => match *k {\n                Known::MethodNotAllowed(_) => StatusCode::METHOD_NOT_ALLOWED,\n                Known::InvalidHeader(_)\n                | Known::MissingHeader(_)\n                | Known::MissingCookie(_)\n                | Known::InvalidQuery(_)\n                | Known::BodyReadError(_)\n                | Known::BodyDeserializeError(_) => StatusCode::BAD_REQUEST,\n                #[cfg(feature = \"websocket\")]\n                Known::MissingConnectionUpgrade(_) => StatusCode::BAD_REQUEST,\n                Known::LengthRequired(_) => StatusCode::LENGTH_REQUIRED,\n                Known::PayloadTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE,\n                Known::UnsupportedMediaType(_) => StatusCode::UNSUPPORTED_MEDIA_TYPE,\n                Known::FilePermissionError(_) | Known::CorsForbidden(_) => StatusCode::FORBIDDEN,\n                Known::FileOpenError(_)\n                | Known::MissingExtension(_)\n                | Known::BodyConsumedMultipleTimes(_) => StatusCode::INTERNAL_SERVER_ERROR,\n            },\n            Rejections::Custom(..) => StatusCode::INTERNAL_SERVER_ERROR,\n            Rejections::Combined(..) => self.preferred().status(),\n        }\n    }\n\n    fn into_response(&self) -> crate::reply::Response {\n        match *self {\n            Rejections::Known(ref e) => {\n                let mut res = http::Response::new(Body::from(e.to_string()));\n                *res.status_mut() = self.status();\n                res.headers_mut().insert(\n                    CONTENT_TYPE,\n                    HeaderValue::from_static(\"text/plain; charset=utf-8\"),\n                );\n                res\n            }\n            Rejections::Custom(ref e) => {\n                tracing::error!(\n                    \"unhandled custom rejection, returning 500 response: {:?}\",\n                    e\n                );\n                let body = format!(\"Unhandled rejection: {:?}\", e);\n                let mut res = http::Response::new(Body::from(body));\n                *res.status_mut() = self.status();\n                res.headers_mut().insert(\n                    CONTENT_TYPE,\n                    HeaderValue::from_static(\"text/plain; charset=utf-8\"),\n                );\n                res\n            }\n            Rejections::Combined(..) => self.preferred().into_response(),\n        }\n    }\n\n    fn find<T: 'static>(&self) -> Option<&T> {\n        match *self {\n            Rejections::Known(ref e) => e.inner_as_any().downcast_ref(),\n            Rejections::Custom(ref e) => e.downcast_ref(),\n            Rejections::Combined(ref a, ref b) => a.find().or_else(|| b.find()),\n        }\n    }\n\n    fn debug_list(&self, f: &mut fmt::DebugList<'_, '_>) {\n        match *self {\n            Rejections::Known(ref e) => {\n                f.entry(e);\n            }\n            Rejections::Custom(ref e) => {\n                f.entry(e);\n            }\n            Rejections::Combined(ref a, ref b) => {\n                a.debug_list(f);\n                b.debug_list(f);\n            }\n        }\n    }\n\n    fn preferred(&self) -> &Rejections {\n        match self {\n            Rejections::Known(_) | Rejections::Custom(_) => self,\n            Rejections::Combined(a, b) => {\n                let a = a.preferred();\n                let b = b.preferred();\n                // Now both a and b are known or custom, so it is safe\n                // to get status\n                // Compare status codes, with this priority:\n                // - NOT_FOUND is lowest\n                // - METHOD_NOT_ALLOWED is second\n                // - if one status code is greater than the other\n                // - otherwise, prefer A...\n                match (a.status(), b.status()) {\n                    (_, StatusCode::NOT_FOUND) => a,\n                    (StatusCode::NOT_FOUND, _) => b,\n                    (_, StatusCode::METHOD_NOT_ALLOWED) => a,\n                    (StatusCode::METHOD_NOT_ALLOWED, _) => b,\n                    (sa, sb) if sa < sb => b,\n                    _ => a,\n                }\n            }\n        }\n    }\n}\n\nunit_error! {\n    /// Invalid query\n    pub InvalidQuery: \"Invalid query string\"\n}\n\nunit_error! {\n    /// HTTP method not allowed\n    pub MethodNotAllowed: \"HTTP method not allowed\"\n}\n\nunit_error! {\n    /// A content-length header is required\n    pub LengthRequired: \"A content-length header is required\"\n}\n\nunit_error! {\n    /// The request payload is too large\n    pub PayloadTooLarge: \"The request payload is too large\"\n}\n\nunit_error! {\n    /// The request's content-type is not supported\n    pub UnsupportedMediaType: \"The request's content-type is not supported\"\n}\n\n/// Missing request header\n#[derive(Debug)]\npub struct MissingHeader {\n    name: &'static str,\n}\n\nimpl MissingHeader {\n    /// Retrieve the name of the header that was missing\n    pub fn name(&self) -> &str {\n        self.name\n    }\n}\n\nimpl fmt::Display for MissingHeader {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Missing request header {:?}\", self.name)\n    }\n}\n\nimpl StdError for MissingHeader {}\n\n/// Invalid request header\n#[derive(Debug)]\npub struct InvalidHeader {\n    name: &'static str,\n}\n\nimpl InvalidHeader {\n    /// Retrieve the name of the header that was invalid\n    pub fn name(&self) -> &str {\n        self.name\n    }\n}\n\nimpl fmt::Display for InvalidHeader {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Invalid request header {:?}\", self.name)\n    }\n}\n\nimpl StdError for InvalidHeader {}\n\n/// Missing cookie\n#[derive(Debug)]\npub struct MissingCookie {\n    name: &'static str,\n}\n\nimpl MissingCookie {\n    /// Retrieve the name of the cookie that was missing\n    pub fn name(&self) -> &str {\n        self.name\n    }\n}\n\nimpl fmt::Display for MissingCookie {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Missing request cookie {:?}\", self.name)\n    }\n}\n\nimpl StdError for MissingCookie {}\n\nmod sealed {\n    use super::{Reason, Rejection, Rejections};\n    use http::StatusCode;\n    use std::convert::Infallible;\n    use std::fmt;\n\n    // This sealed trait exists to allow Filters to return either `Rejection`\n    // or `!`. There are no other types that make sense, and so it is sealed.\n    pub trait IsReject: fmt::Debug + Send + Sync {\n        fn status(&self) -> StatusCode;\n        fn into_response(&self) -> crate::reply::Response;\n    }\n\n    fn _assert_object_safe() {\n        fn _assert(_: &dyn IsReject) {}\n    }\n\n    // This weird trait is to allow optimizations of propagating when a\n    // rejection can *never* happen (currently with the `Never` type,\n    // eventually to be replaced with `!`).\n    //\n    // Using this trait means the `Never` gets propagated to chained filters,\n    // allowing LLVM to eliminate more code paths. Without it, such as just\n    // requiring that `Rejection::from(Never)` were used in those filters,\n    // would mean that links later in the chain may assume a rejection *could*\n    // happen, and no longer eliminate those branches.\n    pub trait CombineRejection<E>: Send + Sized {\n        /// The type that should be returned when only 1 of the two\n        /// \"rejections\" occurs.\n        ///\n        /// # For example:\n        ///\n        /// `warp::any().and(warp::path(\"foo\"))` has the following steps:\n        ///\n        /// 1. Since this is `and`, only **one** of the rejections will occur,\n        ///    and as soon as it does, it will be returned.\n        /// 2. `warp::any()` rejects with `Never`. So, it will never return `Never`.\n        /// 3. `warp::path()` rejects with `Rejection`. It may return `Rejection`.\n        ///\n        /// Thus, if the above filter rejects, it will definitely be `Rejection`.\n        type One: IsReject + From<Self> + From<E> + Into<Rejection>;\n\n        /// The type that should be returned when both rejections occur,\n        /// and need to be combined.\n        type Combined: IsReject;\n\n        fn combine(self, other: E) -> Self::Combined;\n    }\n\n    impl CombineRejection<Rejection> for Rejection {\n        type One = Rejection;\n        type Combined = Rejection;\n\n        fn combine(self, other: Rejection) -> Self::Combined {\n            let reason = match (self.reason, other.reason) {\n                (Reason::Other(left), Reason::Other(right)) => {\n                    Reason::Other(Box::new(Rejections::Combined(left, right)))\n                }\n                (Reason::Other(other), Reason::NotFound)\n                | (Reason::NotFound, Reason::Other(other)) => {\n                    // ignore the NotFound\n                    Reason::Other(other)\n                }\n                (Reason::NotFound, Reason::NotFound) => Reason::NotFound,\n            };\n\n            Rejection { reason }\n        }\n    }\n\n    impl CombineRejection<Infallible> for Rejection {\n        type One = Rejection;\n        type Combined = Infallible;\n\n        fn combine(self, other: Infallible) -> Self::Combined {\n            match other {}\n        }\n    }\n\n    impl CombineRejection<Rejection> for Infallible {\n        type One = Rejection;\n        type Combined = Infallible;\n\n        fn combine(self, _: Rejection) -> Self::Combined {\n            match self {}\n        }\n    }\n\n    impl CombineRejection<Infallible> for Infallible {\n        type One = Infallible;\n        type Combined = Infallible;\n\n        fn combine(self, _: Infallible) -> Self::Combined {\n            match self {}\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[derive(Debug, PartialEq)]\n    struct Left;\n\n    #[derive(Debug, PartialEq)]\n    struct Right;\n\n    impl Reject for Left {}\n    impl Reject for Right {}\n\n    #[test]\n    fn rejection_status() {\n        assert_eq!(not_found().status(), StatusCode::NOT_FOUND);\n        assert_eq!(\n            method_not_allowed().status(),\n            StatusCode::METHOD_NOT_ALLOWED\n        );\n        assert_eq!(length_required().status(), StatusCode::LENGTH_REQUIRED);\n        assert_eq!(payload_too_large().status(), StatusCode::PAYLOAD_TOO_LARGE);\n        assert_eq!(\n            unsupported_media_type().status(),\n            StatusCode::UNSUPPORTED_MEDIA_TYPE\n        );\n        assert_eq!(custom(Left).status(), StatusCode::INTERNAL_SERVER_ERROR);\n    }\n\n    #[tokio::test]\n    async fn combine_rejection_causes_with_some_left_and_none_right() {\n        let left = custom(Left);\n        let right = not_found();\n        let reject = left.combine(right);\n        let resp = reject.into_response();\n\n        assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);\n        assert_eq!(\n            response_body_string(resp).await,\n            \"Unhandled rejection: Left\"\n        )\n    }\n\n    #[tokio::test]\n    async fn combine_rejection_causes_with_none_left_and_some_right() {\n        let left = not_found();\n        let right = custom(Right);\n        let reject = left.combine(right);\n        let resp = reject.into_response();\n\n        assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);\n        assert_eq!(\n            response_body_string(resp).await,\n            \"Unhandled rejection: Right\"\n        )\n    }\n\n    #[tokio::test]\n    async fn unhandled_customs() {\n        let reject = not_found().combine(custom(Right));\n\n        let resp = reject.into_response();\n        assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);\n        assert_eq!(\n            response_body_string(resp).await,\n            \"Unhandled rejection: Right\"\n        );\n\n        // There's no real way to determine which is worse, since both are a 500,\n        // so pick the first one.\n        let reject = custom(Left).combine(custom(Right));\n\n        let resp = reject.into_response();\n        assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);\n        assert_eq!(\n            response_body_string(resp).await,\n            \"Unhandled rejection: Left\"\n        );\n\n        // With many rejections, custom still is top priority.\n        let reject = not_found()\n            .combine(not_found())\n            .combine(not_found())\n            .combine(custom(Right))\n            .combine(not_found());\n\n        let resp = reject.into_response();\n        assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);\n        assert_eq!(\n            response_body_string(resp).await,\n            \"Unhandled rejection: Right\"\n        );\n    }\n\n    async fn response_body_string(resp: crate::reply::Response) -> String {\n        let (_, body) = resp.into_parts();\n        let body_bytes = hyper::body::to_bytes(body).await.expect(\"failed concat\");\n        String::from_utf8_lossy(&body_bytes).to_string()\n    }\n\n    #[test]\n    fn find_cause() {\n        let rej = custom(Left);\n\n        assert_eq!(rej.find::<Left>(), Some(&Left));\n\n        let rej = rej.combine(method_not_allowed());\n\n        assert_eq!(rej.find::<Left>(), Some(&Left));\n        assert!(rej.find::<MethodNotAllowed>().is_some(), \"MethodNotAllowed\");\n    }\n\n    #[test]\n    fn size_of_rejection() {\n        assert_eq!(\n            ::std::mem::size_of::<Rejection>(),\n            ::std::mem::size_of::<usize>(),\n        );\n    }\n\n    #[derive(Debug)]\n    struct X(#[allow(unused)] u32);\n    impl Reject for X {}\n\n    fn combine_n<F, R>(n: u32, new_reject: F) -> Rejection\n    where\n        F: Fn(u32) -> R,\n        R: Reject,\n    {\n        let mut rej = not_found();\n\n        for i in 0..n {\n            rej = rej.combine(custom(new_reject(i)));\n        }\n\n        rej\n    }\n\n    #[test]\n    fn test_debug() {\n        let rej = combine_n(3, X);\n\n        let s = format!(\"{:?}\", rej);\n        assert_eq!(s, \"Rejection([X(0), X(1), X(2)])\");\n    }\n\n    #[test]\n    fn convert_big_rejections_into_response() {\n        let mut rejections = Rejections::Custom(Box::new(std::io::Error::from_raw_os_error(100)));\n        for _ in 0..50 {\n            rejections = Rejections::Combined(\n                Box::new(Rejections::Known(Known::MethodNotAllowed(\n                    MethodNotAllowed { _p: () },\n                ))),\n                Box::new(rejections),\n            );\n        }\n        let reason = Reason::Other(Box::new(rejections));\n        let rejection = Rejection { reason };\n        assert_eq!(\n            StatusCode::INTERNAL_SERVER_ERROR,\n            rejection.into_response().status()\n        );\n    }\n}\n\n// Path: src/filter/and_then.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Func, Internal};\nuse crate::reject::CombineRejection;\n\n#[derive(Clone, Copy, Debug)]\npub struct AndThen<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F> FilterBase for AndThen<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract> + Clone + Send,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: CombineRejection<T::Error>,\n{\n    type Extract = (<F::Output as TryFuture>::Ok,);\n    type Error = <<F::Output as TryFuture>::Error as CombineRejection<T::Error>>::One;\n    type Future = AndThenFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        AndThenFuture {\n            state: State::First(self.filter.filter(Internal), self.callback.clone()),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct AndThenFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: CombineRejection<T::Error>,\n{\n    #[pin]\n    state: State<T::Future, F>,\n}\n\n#[pin_project(project = StateProj)]\nenum State<T, F>\nwhere\n    T: TryFuture,\n    F: Func<T::Ok>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: CombineRejection<T::Error>,\n{\n    First(#[pin] T, F),\n    Second(#[pin] F::Output),\n    Done,\n}\n\nimpl<T, F> Future for AndThenFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: CombineRejection<T::Error>,\n{\n    type Output = Result<\n        (<F::Output as TryFuture>::Ok,),\n        <<F::Output as TryFuture>::Error as CombineRejection<T::Error>>::One,\n    >;\n\n    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        self.project().state.poll(cx)\n    }\n}\n\nimpl<T, F> Future for State<T, F>\nwhere\n    T: TryFuture,\n    F: Func<T::Ok>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: CombineRejection<T::Error>,\n{\n    type Output = Result<\n        (<F::Output as TryFuture>::Ok,),\n        <<F::Output as TryFuture>::Error as CombineRejection<T::Error>>::One,\n    >;\n\n    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        loop {\n            match self.as_mut().project() {\n                StateProj::First(first, second) => {\n                    let ex1 = ready!(first.try_poll(cx))?;\n                    let fut2 = second.call(ex1);\n                    self.set(State::Second(fut2));\n                }\n                StateProj::Second(second) => {\n                    let ex2 = match ready!(second.try_poll(cx)) {\n                        Ok(item) => Ok((item,)),\n                        Err(err) => Err(From::from(err)),\n                    };\n                    self.set(State::Done);\n                    return Poll::Ready(ex2);\n                }\n                StateProj::Done => panic!(\"polled after complete\"),\n            }\n        }\n    }\n}\n\n// Path: src/filter/or_else.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Func, Internal};\nuse crate::reject::IsReject;\nuse crate::route;\n\n#[derive(Clone, Copy, Debug)]\npub struct OrElse<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F> FilterBase for OrElse<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error> + Clone + Send,\n    F::Output: TryFuture<Ok = T::Extract> + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    type Extract = <F::Output as TryFuture>::Ok;\n    type Error = <F::Output as TryFuture>::Error;\n    type Future = OrElseFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        let idx = route::with(|route| route.matched_path_index());\n        OrElseFuture {\n            state: State::First(self.filter.filter(Internal), self.callback.clone()),\n            original_path_index: PathIndex(idx),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct OrElseFuture<T: Filter, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture<Ok = T::Extract> + Send,\n{\n    #[pin]\n    state: State<T, F>,\n    original_path_index: PathIndex,\n}\n\n#[pin_project(project = StateProj)]\nenum State<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture<Ok = T::Extract> + Send,\n{\n    First(#[pin] T::Future, F),\n    Second(#[pin] F::Output),\n    Done,\n}\n\n#[derive(Copy, Clone)]\nstruct PathIndex(usize);\n\nimpl PathIndex {\n    fn reset_path(&self) {\n        route::with(|route| route.reset_matched_path_index(self.0));\n    }\n}\n\nimpl<T, F> Future for OrElseFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture<Ok = T::Extract> + Send,\n{\n    type Output = Result<<F::Output as TryFuture>::Ok, <F::Output as TryFuture>::Error>;\n\n    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        loop {\n            let pin = self.as_mut().project();\n            let (err, second) = match pin.state.project() {\n                StateProj::First(first, second) => match ready!(first.try_poll(cx)) {\n                    Ok(ex) => return Poll::Ready(Ok(ex)),\n                    Err(err) => (err, second),\n                },\n                StateProj::Second(second) => {\n                    let ex2 = ready!(second.try_poll(cx));\n                    self.set(OrElseFuture {\n                        state: State::Done,\n                        ..*self\n                    });\n                    return Poll::Ready(ex2);\n                }\n                StateProj::Done => panic!(\"polled after complete\"),\n            };\n\n            pin.original_path_index.reset_path();\n            let fut2 = second.call(err);\n            self.set(OrElseFuture {\n                state: State::Second(fut2),\n                ..*self\n            });\n        }\n    }\n}\n\n// Path: src/filter/map_err.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::TryFuture;\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Internal};\nuse crate::reject::IsReject;\n\n#[derive(Clone, Copy, Debug)]\npub struct MapErr<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F, E> FilterBase for MapErr<T, F>\nwhere\n    T: Filter,\n    F: Fn(T::Error) -> E + Clone + Send,\n    E: IsReject,\n{\n    type Extract = T::Extract;\n    type Error = E;\n    type Future = MapErrFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        MapErrFuture {\n            extract: self.filter.filter(Internal),\n            callback: self.callback.clone(),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct MapErrFuture<T: Filter, F> {\n    #[pin]\n    extract: T::Future,\n    callback: F,\n}\n\nimpl<T, F, E> Future for MapErrFuture<T, F>\nwhere\n    T: Filter,\n    F: Fn(T::Error) -> E,\n{\n    type Output = Result<T::Extract, E>;\n\n    #[inline]\n    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        self.as_mut()\n            .project()\n            .extract\n            .try_poll(cx)\n            .map_err(|err| (self.callback)(err))\n    }\n}\n\n// Path: src/filter/boxed.rs\nuse std::fmt;\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::sync::Arc;\n\nuse futures_util::TryFutureExt;\n\nuse super::{Filter, FilterBase, Internal, Tuple};\nuse crate::reject::Rejection;\n\n/// A type representing a boxed [`Filter`](crate::Filter) trait object.\n///\n/// The filter inside is a dynamic trait object. The purpose of this type is\n/// to ease returning `Filter`s from other functions.\n///\n/// To create one, call `Filter::boxed` on any filter.\n///\n/// # Examples\n///\n/// ```\n/// use warp::{Filter, filters::BoxedFilter, Reply};\n///\n/// pub fn assets_filter() -> BoxedFilter<(impl Reply,)> {\n///     warp::path(\"assets\")\n///         .and(warp::fs::dir(\"./assets\"))\n///         .boxed()\n/// }\n/// ```\n///\npub struct BoxedFilter<T: Tuple> {\n    filter: Arc<\n        dyn Filter<\n                Extract = T,\n                Error = Rejection,\n                Future = Pin<Box<dyn Future<Output = Result<T, Rejection>> + Send>>,\n            > + Send\n            + Sync,\n    >,\n}\n\nimpl<T: Tuple + Send> BoxedFilter<T> {\n    pub(super) fn new<F>(filter: F) -> BoxedFilter<T>\n    where\n        F: Filter<Extract = T> + Send + Sync + 'static,\n        F::Error: Into<Rejection>,\n    {\n        BoxedFilter {\n            filter: Arc::new(BoxingFilter {\n                filter: filter.map_err(super::Internal, Into::into),\n            }),\n        }\n    }\n}\n\nimpl<T: Tuple> Clone for BoxedFilter<T> {\n    fn clone(&self) -> BoxedFilter<T> {\n        BoxedFilter {\n            filter: self.filter.clone(),\n        }\n    }\n}\n\nimpl<T: Tuple> fmt::Debug for BoxedFilter<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"BoxedFilter\").finish()\n    }\n}\n\nfn _assert_send() {\n    fn _assert<T: Send>() {}\n    _assert::<BoxedFilter<()>>();\n}\n\nimpl<T: Tuple + Send> FilterBase for BoxedFilter<T> {\n    type Extract = T;\n    type Error = Rejection;\n    type Future = Pin<Box<dyn Future<Output = Result<T, Rejection>> + Send>>;\n\n    fn filter(&self, _: Internal) -> Self::Future {\n        self.filter.filter(Internal)\n    }\n}\n\nstruct BoxingFilter<F> {\n    filter: F,\n}\n\nimpl<F> FilterBase for BoxingFilter<F>\nwhere\n    F: Filter,\n    F::Future: Send + 'static,\n{\n    type Extract = F::Extract;\n    type Error = F::Error;\n    type Future = Pin<Box<dyn Future<Output = Result<Self::Extract, Self::Error>> + Send>>;\n\n    fn filter(&self, _: Internal) -> Self::Future {\n        Box::pin(self.filter.filter(Internal).into_future())\n    }\n}\n\n// Path: src/filter/untuple_one.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Internal, Tuple};\n\n#[derive(Clone, Copy, Debug)]\npub struct UntupleOne<F> {\n    pub(super) filter: F,\n}\n\nimpl<F, T> FilterBase for UntupleOne<F>\nwhere\n    F: Filter<Extract = (T,)>,\n    T: Tuple,\n{\n    type Extract = T;\n    type Error = F::Error;\n    type Future = UntupleOneFuture<F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        UntupleOneFuture {\n            extract: self.filter.filter(Internal),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct UntupleOneFuture<F: Filter> {\n    #[pin]\n    extract: F::Future,\n}\n\nimpl<F, T> Future for UntupleOneFuture<F>\nwhere\n    F: Filter<Extract = (T,)>,\n    T: Tuple,\n{\n    type Output = Result<T, F::Error>;\n\n    #[inline]\n    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        match ready!(self.project().extract.try_poll(cx)) {\n            Ok((t,)) => Poll::Ready(Ok(t)),\n            Err(err) => Poll::Ready(Err(err)),\n        }\n    }\n}\n\n// Path: src/filter/recover.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Func, Internal};\nuse crate::generic::Either;\nuse crate::reject::IsReject;\nuse crate::route;\n\n#[derive(Clone, Copy, Debug)]\npub struct Recover<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F> FilterBase for Recover<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error> + Clone + Send,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    type Extract = (Either<T::Extract, (<F::Output as TryFuture>::Ok,)>,);\n    type Error = <F::Output as TryFuture>::Error;\n    type Future = RecoverFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        let idx = route::with(|route| route.matched_path_index());\n        RecoverFuture {\n            state: State::First(self.filter.filter(Internal), self.callback.clone()),\n            original_path_index: PathIndex(idx),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct RecoverFuture<T: Filter, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    #[pin]\n    state: State<T, F>,\n    original_path_index: PathIndex,\n}\n\n#[pin_project(project = StateProj)]\nenum State<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    First(#[pin] T::Future, F),\n    Second(#[pin] F::Output),\n    Done,\n}\n\n#[derive(Copy, Clone)]\nstruct PathIndex(usize);\n\nimpl PathIndex {\n    fn reset_path(&self) {\n        route::with(|route| route.reset_matched_path_index(self.0));\n    }\n}\n\nimpl<T, F> Future for RecoverFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    type Output = Result<\n        (Either<T::Extract, (<F::Output as TryFuture>::Ok,)>,),\n        <F::Output as TryFuture>::Error,\n    >;\n\n    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        loop {\n            let pin = self.as_mut().project();\n            let (err, second) = match pin.state.project() {\n                StateProj::First(first, second) => match ready!(first.try_poll(cx)) {\n                    Ok(ex) => return Poll::Ready(Ok((Either::A(ex),))),\n                    Err(err) => (err, second),\n                },\n                StateProj::Second(second) => {\n                    let ex2 = match ready!(second.try_poll(cx)) {\n                        Ok(ex2) => Ok((Either::B((ex2,)),)),\n                        Err(e) => Err(e),\n                    };\n                    self.set(RecoverFuture {\n                        state: State::Done,\n                        ..*self\n                    });\n                    return Poll::Ready(ex2);\n                }\n                StateProj::Done => panic!(\"polled after complete\"),\n            };\n\n            pin.original_path_index.reset_path();\n            let fut2 = second.call(err);\n            self.set(RecoverFuture {\n                state: State::Second(fut2),\n                ..*self\n            });\n        }\n    }\n}\n\n// Path: src/filter/then.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Func, Internal};\n\n#[derive(Clone, Copy, Debug)]\npub struct Then<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F> FilterBase for Then<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract> + Clone + Send,\n    F::Output: Future + Send,\n{\n    type Extract = (<F::Output as Future>::Output,);\n    type Error = T::Error;\n    type Future = ThenFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        ThenFuture {\n            state: State::First(self.filter.filter(Internal), self.callback.clone()),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct ThenFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract>,\n    F::Output: Future + Send,\n{\n    #[pin]\n    state: State<T::Future, F>,\n}\n\n#[pin_project(project = StateProj)]\nenum State<T, F>\nwhere\n    T: TryFuture,\n    F: Func<T::Ok>,\n    F::Output: Future + Send,\n{\n    First(#[pin] T, F),\n    Second(#[pin] F::Output),\n    Done,\n}\n\nimpl<T, F> Future for ThenFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract>,\n    F::Output: Future + Send,\n{\n    type Output = Result<(<F::Output as Future>::Output,), T::Error>;\n\n    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        self.project().state.poll(cx)\n    }\n}\n\nimpl<T, F> Future for State<T, F>\nwhere\n    T: TryFuture,\n    F: Func<T::Ok>,\n    F::Output: Future + Send,\n{\n    type Output = Result<(<F::Output as Future>::Output,), T::Error>;\n\n    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        loop {\n            match self.as_mut().project() {\n                StateProj::First(first, second) => {\n                    let ex1 = ready!(first.try_poll(cx))?;\n                    let fut2 = second.call(ex1);\n                    self.set(State::Second(fut2));\n                }\n                StateProj::Second(second) => {\n                    let ex2 = (ready!(second.poll(cx)),);\n                    self.set(State::Done);\n                    return Poll::Ready(Ok(ex2));\n                }\n                StateProj::Done => panic!(\"polled after complete\"),\n            }\n        }\n    }\n}\n\n// Path: src/filter/mod.rs\nmod and;\nmod and_then;\nmod boxed;\nmod map;\nmod map_err;\nmod or;\nmod or_else;\nmod recover;\npub(crate) mod service;\nmod then;\nmod unify;\nmod untuple_one;\nmod wrap;\n\nuse std::future::Future;\n\nuse futures_util::{future, TryFuture, TryFutureExt};\n\npub(crate) use crate::generic::{one, Combine, Either, Func, One, Tuple};\nuse crate::reject::{CombineRejection, IsReject, Rejection};\nuse crate::route::{self, Route};\n\npub(crate) use self::and::And;\nuse self::and_then::AndThen;\npub use self::boxed::BoxedFilter;\npub(crate) use self::map::Map;\npub(crate) use self::map_err::MapErr;\npub(crate) use self::or::Or;\nuse self::or_else::OrElse;\nuse self::recover::Recover;\nuse self::then::Then;\nuse self::unify::Unify;\nuse self::untuple_one::UntupleOne;\npub use self::wrap::wrap_fn;\npub(crate) use self::wrap::{Wrap, WrapSealed};\n\n// A crate-private base trait, allowing the actual `filter` method to change\n// signatures without it being a breaking change.\npub trait FilterBase {\n    type Extract: Tuple; // + Send;\n    type Error: IsReject;\n    type Future: Future<Output = Result<Self::Extract, Self::Error>> + Send;\n\n    fn filter(&self, internal: Internal) -> Self::Future;\n\n    fn map_err<F, E>(self, _internal: Internal, fun: F) -> MapErr<Self, F>\n    where\n        Self: Sized,\n        F: Fn(Self::Error) -> E + Clone,\n        E: ::std::fmt::Debug + Send,\n    {\n        MapErr {\n            filter: self,\n            callback: fun,\n        }\n    }\n}\n\n// A crate-private argument to prevent users from calling methods on\n// the `FilterBase` trait.\n//\n// For instance, this innocent user code could otherwise call `filter`:\n//\n// ```\n// async fn with_filter<F: Filter>(f: F) -> Result<F::Extract, F::Error> {\n//     f.filter().await\n// }\n// ```\n#[allow(missing_debug_implementations)]\npub struct Internal;\n\n/// Composable request filters.\n///\n/// A `Filter` can optionally extract some data from a request, combine\n/// it with others, mutate it, and return back some value as a reply. The\n/// power of `Filter`s come from being able to isolate small subsets, and then\n/// chain and reuse them in various parts of your app.\n///\n/// # Extracting Tuples\n///\n/// You may notice that several of these filters extract some tuple, often\n/// times a tuple of just 1 item! Why?\n///\n/// If a filter extracts a `(String,)`, that simply means that it\n/// extracts a `String`. If you were to `map` the filter, the argument type\n/// would be exactly that, just a `String`.\n///\n/// What is it? It's just some type magic that allows for automatic combining\n/// and flattening of tuples. Without it, combining two filters together with\n/// `and`, where one extracted `()`, and another `String`, would mean the\n/// `map` would be given a single argument of `((), String,)`, which is just\n/// no fun.\npub trait Filter: FilterBase {\n    /// Composes a new `Filter` that requires both this and the other to filter a request.\n    ///\n    /// Additionally, this will join together the extracted values of both\n    /// filters, so that `map` and `and_then` receive them as separate arguments.\n    ///\n    /// If a `Filter` extracts nothing (so, `()`), combining with any other\n    /// filter will simply discard the `()`. If a `Filter` extracts one or\n    /// more items, combining will mean it extracts the values of itself\n    /// combined with the other.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Match `/hello/:name`...\n    /// warp::path(\"hello\")\n    ///     .and(warp::path::param::<String>());\n    /// ```\n    fn and<F>(self, other: F) -> And<Self, F>\n    where\n        Self: Sized,\n        <Self::Extract as Tuple>::HList: Combine<<F::Extract as Tuple>::HList>,\n        F: Filter + Clone,\n        F::Error: CombineRejection<Self::Error>,\n    {\n        And {\n            first: self,\n            second: other,\n        }\n    }\n\n    /// Composes a new `Filter` of either this or the other filter.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use std::net::SocketAddr;\n    /// use warp::Filter;\n    ///\n    /// // Match either `/:u32` or `/:socketaddr`\n    /// warp::path::param::<u32>()\n    ///     .or(warp::path::param::<SocketAddr>());\n    /// ```\n    fn or<F>(self, other: F) -> Or<Self, F>\n    where\n        Self: Filter<Error = Rejection> + Sized,\n        F: Filter,\n        F::Error: CombineRejection<Self::Error>,\n    {\n        Or {\n            first: self,\n            second: other,\n        }\n    }\n\n    /// Composes this `Filter` with a function receiving the extracted value.\n    ///\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Map `/:id`\n    /// warp::path::param().map(|id: u64| {\n    ///   format!(\"Hello #{}\", id)\n    /// });\n    /// ```\n    ///\n    /// # `Func`\n    ///\n    /// The generic `Func` trait is implemented for any function that receives\n    /// the same arguments as this `Filter` extracts. In practice, this\n    /// shouldn't ever bother you, and simply makes things feel more natural.\n    ///\n    /// For example, if three `Filter`s were combined together, suppose one\n    /// extracts nothing (so `()`), and the other two extract two integers,\n    /// a function that accepts exactly two integer arguments is allowed.\n    /// Specifically, any `Fn(u32, u32)`.\n    ///\n    /// Without `Product` and `Func`, this would be a lot messier. First of\n    /// all, the `()`s couldn't be discarded, and the tuples would be nested.\n    /// So, instead, you'd need to pass an `Fn(((), (u32, u32)))`. That's just\n    /// a single argument. Bleck!\n    ///\n    /// Even worse, the tuples would shuffle the types around depending on\n    /// the exact invocation of `and`s. So, `unit.and(int).and(int)` would\n    /// result in a different extracted type from `unit.and(int.and(int))`,\n    /// or from `int.and(unit).and(int)`. If you changed around the order\n    /// of filters, while still having them be semantically equivalent, you'd\n    /// need to update all your `map`s as well.\n    ///\n    /// `Product`, `HList`, and `Func` do all the heavy work so that none of\n    /// this is a bother to you. What's more, the types are enforced at\n    /// compile-time, and tuple flattening is optimized away to nothing by\n    /// LLVM.\n    fn map<F>(self, fun: F) -> Map<Self, F>\n    where\n        Self: Sized,\n        F: Func<Self::Extract> + Clone,\n    {\n        Map {\n            filter: self,\n            callback: fun,\n        }\n    }\n\n    /// Composes this `Filter` with an async function receiving\n    /// the extracted value.\n    ///\n    /// The function should return some `Future` type.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Map `/:id`\n    /// warp::path::param().then(|id: u64| async move {\n    ///   format!(\"Hello #{}\", id)\n    /// });\n    /// ```\n    fn then<F>(self, fun: F) -> Then<Self, F>\n    where\n        Self: Sized,\n        F: Func<Self::Extract> + Clone,\n        F::Output: Future + Send,\n    {\n        Then {\n            filter: self,\n            callback: fun,\n        }\n    }\n\n    /// Composes this `Filter` with a fallible async function receiving\n    /// the extracted value.\n    ///\n    /// The function should return some `TryFuture` type.\n    ///\n    /// The `Error` type of the return `Future` needs be a `Rejection`, which\n    /// means most futures will need to have their error mapped into one.\n    ///\n    /// Rejections are meant to say \"this filter didn't accept the request,\n    /// maybe another can\". So for application-level errors, consider using\n    /// [`Filter::then`] instead.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Validate after `/:id`\n    /// warp::path::param().and_then(|id: u64| async move {\n    ///     if id != 0 {\n    ///         Ok(format!(\"Hello #{}\", id))\n    ///     } else {\n    ///         Err(warp::reject::not_found())\n    ///     }\n    /// });\n    /// ```\n    fn and_then<F>(self, fun: F) -> AndThen<Self, F>\n    where\n        Self: Sized,\n        F: Func<Self::Extract> + Clone,\n...\n", "needle_token_start": 802, "needle_token_end": 818, "code_context_ntokens": 16216, "output": ["```rust\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn not_found() -> Rejection {\n    Rejection {\n        reason: Reason::NotFound,\n    }\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "map_err", "language": "rust", "path": "src/filter/mod.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform the error type of a future or result by applying a specified function to the error.\n2. **Input**: It takes two parameters: a future or result that might produce an error, and a function that defines how to transform the error type.\n3. **Output**: It produces a new future or result where the error has been transformed according to the provided function.\n4. **Procedure**: The function first checks if the input future or result has an error. If an error is present, it applies the transformation function to the error, creating a new error type. This transformed error is then used to construct the output future or result.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/filter/untuple_one.rs\n...\n// Path: src/filter/recover.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Func, Internal};\nuse crate::generic::Either;\nuse crate::reject::IsReject;\nuse crate::route;\n\n#[derive(Clone, Copy, Debug)]\npub struct Recover<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F> FilterBase for Recover<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error> + Clone + Send,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    type Extract = (Either<T::Extract, (<F::Output as TryFuture>::Ok,)>,);\n    type Error = <F::Output as TryFuture>::Error;\n    type Future = RecoverFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        let idx = route::with(|route| route.matched_path_index());\n        RecoverFuture {\n            state: State::First(self.filter.filter(Internal), self.callback.clone()),\n            original_path_index: PathIndex(idx),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct RecoverFuture<T: Filter, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    #[pin]\n    state: State<T, F>,\n    original_path_index: PathIndex,\n}\n\n#[pin_project(project = StateProj)]\nenum State<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    First(#[pin] T::Future, F),\n    Second(#[pin] F::Output),\n    Done,\n}\n\n#[derive(Copy, Clone)]\nstruct PathIndex(usize);\n\nimpl PathIndex {\n    fn reset_path(&self) {\n        route::with(|route| route.reset_matched_path_index(self.0));\n    }\n}\n\nimpl<T, F> Future for RecoverFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Error>,\n    F::Output: TryFuture + Send,\n    <F::Output as TryFuture>::Error: IsReject,\n{\n    type Output = Result<\n        (Either<T::Extract, (<F::Output as TryFuture>::Ok,)>,),\n        <F::Output as TryFuture>::Error,\n    >;\n\n    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        loop {\n            let pin = self.as_mut().project();\n            let (err, second) = match pin.state.project() {\n                StateProj::First(first, second) => match ready!(first.try_poll(cx)) {\n                    Ok(ex) => return Poll::Ready(Ok((Either::A(ex),))),\n                    Err(err) => (err, second),\n                },\n                StateProj::Second(second) => {\n                    let ex2 = match ready!(second.try_poll(cx)) {\n                        Ok(ex2) => Ok((Either::B((ex2,)),)),\n                        Err(e) => Err(e),\n                    };\n                    self.set(RecoverFuture {\n                        state: State::Done,\n                        ..*self\n                    });\n                    return Poll::Ready(ex2);\n                }\n                StateProj::Done => panic!(\"polled after complete\"),\n            };\n\n            pin.original_path_index.reset_path();\n            let fut2 = second.call(err);\n            self.set(RecoverFuture {\n                state: State::Second(fut2),\n                ..*self\n            });\n        }\n    }\n}\n\n// Path: src/filter/then.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Func, Internal};\n\n#[derive(Clone, Copy, Debug)]\npub struct Then<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F> FilterBase for Then<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract> + Clone + Send,\n    F::Output: Future + Send,\n{\n    type Extract = (<F::Output as Future>::Output,);\n    type Error = T::Error;\n    type Future = ThenFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        ThenFuture {\n            state: State::First(self.filter.filter(Internal), self.callback.clone()),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct ThenFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract>,\n    F::Output: Future + Send,\n{\n    #[pin]\n    state: State<T::Future, F>,\n}\n\n#[pin_project(project = StateProj)]\nenum State<T, F>\nwhere\n    T: TryFuture,\n    F: Func<T::Ok>,\n    F::Output: Future + Send,\n{\n    First(#[pin] T, F),\n    Second(#[pin] F::Output),\n    Done,\n}\n\nimpl<T, F> Future for ThenFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract>,\n    F::Output: Future + Send,\n{\n    type Output = Result<(<F::Output as Future>::Output,), T::Error>;\n\n    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        self.project().state.poll(cx)\n    }\n}\n\nimpl<T, F> Future for State<T, F>\nwhere\n    T: TryFuture,\n    F: Func<T::Ok>,\n    F::Output: Future + Send,\n{\n    type Output = Result<(<F::Output as Future>::Output,), T::Error>;\n\n    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        loop {\n            match self.as_mut().project() {\n                StateProj::First(first, second) => {\n                    let ex1 = ready!(first.try_poll(cx))?;\n                    let fut2 = second.call(ex1);\n                    self.set(State::Second(fut2));\n                }\n                StateProj::Second(second) => {\n                    let ex2 = (ready!(second.poll(cx)),);\n                    self.set(State::Done);\n                    return Poll::Ready(Ok(ex2));\n                }\n                StateProj::Done => panic!(\"polled after complete\"),\n            }\n        }\n    }\n}\n\n// Path: src/filter/mod.rs\nmod and;\nmod and_then;\nmod boxed;\nmod map;\nmod map_err;\nmod or;\nmod or_else;\nmod recover;\npub(crate) mod service;\nmod then;\nmod unify;\nmod untuple_one;\nmod wrap;\n\nuse std::future::Future;\n\nuse futures_util::{future, TryFuture, TryFutureExt};\n\npub(crate) use crate::generic::{one, Combine, Either, Func, One, Tuple};\nuse crate::reject::{CombineRejection, IsReject, Rejection};\nuse crate::route::{self, Route};\n\npub(crate) use self::and::And;\nuse self::and_then::AndThen;\npub use self::boxed::BoxedFilter;\npub(crate) use self::map::Map;\npub(crate) use self::map_err::MapErr;\npub(crate) use self::or::Or;\nuse self::or_else::OrElse;\nuse self::recover::Recover;\nuse self::then::Then;\nuse self::unify::Unify;\nuse self::untuple_one::UntupleOne;\npub use self::wrap::wrap_fn;\npub(crate) use self::wrap::{Wrap, WrapSealed};\n\n// A crate-private base trait, allowing the actual `filter` method to change\n// signatures without it being a breaking change.\npub trait FilterBase {\n    type Extract: Tuple; // + Send;\n    type Error: IsReject;\n    type Future: Future<Output = Result<Self::Extract, Self::Error>> + Send;\n\n    fn filter(&self, internal: Internal) -> Self::Future;\n\n    \nfn map_err<F, E>(self, _internal: Internal, fun: F) -> MapErr<Self, F>\n    where\n        Self: Sized,\n        F: Fn(Self::Error) -> E + Clone,\n        E: ::std::fmt::Debug + Send,\n    {\n        MapErr {\n            filter: self,\n            callback: fun,\n        }\n    }\n}\n\n// A crate-private argument to prevent users from calling methods on\n// the `FilterBase` trait.\n//\n// For instance, this innocent user code could otherwise call `filter`:\n//\n// ```\n// async fn with_filter<F: Filter>(f: F) -> Result<F::Extract, F::Error> {\n//     f.filter().await\n// }\n// ```\n#[allow(missing_debug_implementations)]\npub struct Internal;\n\n/// Composable request filters.\n///\n/// A `Filter` can optionally extract some data from a request, combine\n/// it with others, mutate it, and return back some value as a reply. The\n/// power of `Filter`s come from being able to isolate small subsets, and then\n/// chain and reuse them in various parts of your app.\n///\n/// # Extracting Tuples\n///\n/// You may notice that several of these filters extract some tuple, often\n/// times a tuple of just 1 item! Why?\n///\n/// If a filter extracts a `(String,)`, that simply means that it\n/// extracts a `String`. If you were to `map` the filter, the argument type\n/// would be exactly that, just a `String`.\n///\n/// What is it? It's just some type magic that allows for automatic combining\n/// and flattening of tuples. Without it, combining two filters together with\n/// `and`, where one extracted `()`, and another `String`, would mean the\n/// `map` would be given a single argument of `((), String,)`, which is just\n/// no fun.\npub trait Filter: FilterBase {\n    /// Composes a new `Filter` that requires both this and the other to filter a request.\n    ///\n    /// Additionally, this will join together the extracted values of both\n    /// filters, so that `map` and `and_then` receive them as separate arguments.\n    ///\n    /// If a `Filter` extracts nothing (so, `()`), combining with any other\n    /// filter will simply discard the `()`. If a `Filter` extracts one or\n    /// more items, combining will mean it extracts the values of itself\n    /// combined with the other.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Match `/hello/:name`...\n    /// warp::path(\"hello\")\n    ///     .and(warp::path::param::<String>());\n    /// ```\n    fn and<F>(self, other: F) -> And<Self, F>\n    where\n        Self: Sized,\n        <Self::Extract as Tuple>::HList: Combine<<F::Extract as Tuple>::HList>,\n        F: Filter + Clone,\n        F::Error: CombineRejection<Self::Error>,\n    {\n        And {\n            first: self,\n            second: other,\n        }\n    }\n\n    /// Composes a new `Filter` of either this or the other filter.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use std::net::SocketAddr;\n    /// use warp::Filter;\n    ///\n    /// // Match either `/:u32` or `/:socketaddr`\n    /// warp::path::param::<u32>()\n    ///     .or(warp::path::param::<SocketAddr>());\n    /// ```\n    fn or<F>(self, other: F) -> Or<Self, F>\n    where\n        Self: Filter<Error = Rejection> + Sized,\n        F: Filter,\n        F::Error: CombineRejection<Self::Error>,\n    {\n        Or {\n            first: self,\n            second: other,\n        }\n    }\n\n    /// Composes this `Filter` with a function receiving the extracted value.\n    ///\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Map `/:id`\n    /// warp::path::param().map(|id: u64| {\n    ///   format!(\"Hello #{}\", id)\n    /// });\n    /// ```\n    ///\n    /// # `Func`\n    ///\n    /// The generic `Func` trait is implemented for any function that receives\n    /// the same arguments as this `Filter` extracts. In practice, this\n    /// shouldn't ever bother you, and simply makes things feel more natural.\n    ///\n    /// For example, if three `Filter`s were combined together, suppose one\n    /// extracts nothing (so `()`), and the other two extract two integers,\n    /// a function that accepts exactly two integer arguments is allowed.\n    /// Specifically, any `Fn(u32, u32)`.\n    ///\n    /// Without `Product` and `Func`, this would be a lot messier. First of\n    /// all, the `()`s couldn't be discarded, and the tuples would be nested.\n    /// So, instead, you'd need to pass an `Fn(((), (u32, u32)))`. That's just\n    /// a single argument. Bleck!\n    ///\n    /// Even worse, the tuples would shuffle the types around depending on\n    /// the exact invocation of `and`s. So, `unit.and(int).and(int)` would\n    /// result in a different extracted type from `unit.and(int.and(int))`,\n    /// or from `int.and(unit).and(int)`. If you changed around the order\n    /// of filters, while still having them be semantically equivalent, you'd\n    /// need to update all your `map`s as well.\n    ///\n    /// `Product`, `HList`, and `Func` do all the heavy work so that none of\n    /// this is a bother to you. What's more, the types are enforced at\n    /// compile-time, and tuple flattening is optimized away to nothing by\n    /// LLVM.\n    fn map<F>(self, fun: F) -> Map<Self, F>\n    where\n        Self: Sized,\n        F: Func<Self::Extract> + Clone,\n    {\n        Map {\n            filter: self,\n            callback: fun,\n        }\n    }\n\n    /// Composes this `Filter` with an async function receiving\n    /// the extracted value.\n    ///\n    /// The function should return some `Future` type.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Map `/:id`\n    /// warp::path::param().then(|id: u64| async move {\n    ///   format!(\"Hello #{}\", id)\n    /// });\n    /// ```\n    fn then<F>(self, fun: F) -> Then<Self, F>\n    where\n        Self: Sized,\n        F: Func<Self::Extract> + Clone,\n        F::Output: Future + Send,\n    {\n        Then {\n            filter: self,\n            callback: fun,\n        }\n    }\n\n    /// Composes this `Filter` with a fallible async function receiving\n    /// the extracted value.\n    ///\n    /// The function should return some `TryFuture` type.\n    ///\n    /// The `Error` type of the return `Future` needs be a `Rejection`, which\n    /// means most futures will need to have their error mapped into one.\n    ///\n    /// Rejections are meant to say \"this filter didn't accept the request,\n    /// maybe another can\". So for application-level errors, consider using\n    /// [`Filter::then`] instead.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// // Validate after `/:id`\n    /// warp::path::param().and_then(|id: u64| async move {\n    ///     if id != 0 {\n    ///         Ok(format!(\"Hello #{}\", id))\n    ///     } else {\n    ///         Err(warp::reject::not_found())\n    ///     }\n    /// });\n    /// ```\n    fn and_then<F>(self, fun: F) -> AndThen<Self, F>\n    where\n        Self: Sized,\n        F: Func<Self::Extract> + Clone,\n        F::Output: TryFuture + Send,\n        <F::Output as TryFuture>::Error: CombineRejection<Self::Error>,\n    {\n        AndThen {\n            filter: self,\n            callback: fun,\n        }\n    }\n\n    /// Compose this `Filter` with a function receiving an error.\n    ///\n    /// The function should return some `TryFuture` type yielding the\n    /// same item and error types.\n    fn or_else<F>(self, fun: F) -> OrElse<Self, F>\n    where\n        Self: Filter<Error = Rejection> + Sized,\n        F: Func<Rejection>,\n        F::Output: TryFuture<Ok = Self::Extract> + Send,\n        <F::Output as TryFuture>::Error: IsReject,\n    {\n        OrElse {\n            filter: self,\n            callback: fun,\n        }\n    }\n\n    /// Compose this `Filter` with a function receiving an error and\n    /// returning a *new* type, instead of the *same* type.\n    ///\n    /// This is useful for \"customizing\" rejections into new response types.\n    /// See also the [rejections example][ex].\n    ///\n    /// [ex]: https://github.com/seanmonstar/warp/blob/master/examples/rejections.rs\n    fn recover<F>(self, fun: F) -> Recover<Self, F>\n    where\n        Self: Filter<Error = Rejection> + Sized,\n        F: Func<Rejection>,\n        F::Output: TryFuture + Send,\n        <F::Output as TryFuture>::Error: IsReject,\n    {\n        Recover {\n            filter: self,\n            callback: fun,\n        }\n    }\n\n    /// Unifies the extracted value of `Filter`s composed with `or`.\n    ///\n    /// When a `Filter` extracts some `Either<T, T>`, where both sides\n    /// are the same type, this combinator can be used to grab the\n    /// inner value, regardless of which side of `Either` it was. This\n    /// is useful for values that could be extracted from multiple parts\n    /// of a request, and the exact place isn't important.\n    ///\n    /// # Example\n    ///\n    /// ```rust\n    /// use std::net::SocketAddr;\n    /// use warp::Filter;\n    ///\n    /// let client_ip = warp::header(\"x-real-ip\")\n    ///     .or(warp::header(\"x-forwarded-for\"))\n    ///     .unify()\n    ///     .map(|ip: SocketAddr| {\n    ///         // Get the IP from either header,\n    ///         // and unify into the inner type.\n    ///     });\n    /// ```\n    fn unify<T>(self) -> Unify<Self>\n    where\n        Self: Filter<Extract = (Either<T, T>,)> + Sized,\n        T: Tuple,\n    {\n        Unify { filter: self }\n    }\n\n    /// Convenience method to remove one layer of tupling.\n    ///\n    /// This is useful for when things like `map` don't return a new value,\n    /// but just `()`, since warp will wrap it up into a `((),)`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// let route = warp::path::param()\n    ///     .map(|num: u64| {\n    ///         println!(\"just logging: {}\", num);\n    ///         // returning \"nothing\"\n    ///     })\n    ///     .untuple_one()\n    ///     .map(|| {\n    ///         println!(\"the ((),) was removed\");\n    ///         warp::reply()\n    ///     });\n    /// ```\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// let route = warp::any()\n    ///     .map(|| {\n    ///         // wanting to return a tuple\n    ///         (true, 33)\n    ///     })\n    ///     .untuple_one()\n    ///     .map(|is_enabled: bool, count: i32| {\n    ///         println!(\"untupled: ({}, {})\", is_enabled, count);\n    ///     });\n    /// ```\n    fn untuple_one<T>(self) -> UntupleOne<Self>\n    where\n        Self: Filter<Extract = (T,)> + Sized,\n        T: Tuple,\n    {\n        UntupleOne { filter: self }\n    }\n\n    /// Wraps the current filter with some wrapper.\n    ///\n    /// The wrapper may do some preparation work before starting this filter,\n    /// and may do post-processing after the filter completes.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// let route = warp::any()\n    ///     .map(warp::reply);\n    ///\n    /// // Wrap the route with a log wrapper.\n    /// let route = route.with(warp::log(\"example\"));\n    /// ```\n    fn with<W>(self, wrapper: W) -> W::Wrapped\n    where\n        Self: Sized,\n        W: Wrap<Self>,\n    {\n        wrapper.wrap(self)\n    }\n\n    /// Boxes this filter into a trait object, making it easier to name the type.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// use warp::Filter;\n    ///\n    /// fn impl_reply() -> warp::filters::BoxedFilter<(impl warp::Reply,)> {\n    ///     warp::any()\n    ///         .map(warp::reply)\n    ///         .boxed()\n    /// }\n    ///\n    /// fn named_i32() -> warp::filters::BoxedFilter<(i32,)> {\n    ///     warp::path::param::<i32>()\n    ///         .boxed()\n    /// }\n    ///\n    /// fn named_and() -> warp::filters::BoxedFilter<(i32, String)> {\n    ///     warp::path::param::<i32>()\n    ///         .and(warp::header::<String>(\"host\"))\n    ///         .boxed()\n    /// }\n    /// ```\n    fn boxed(self) -> BoxedFilter<Self::Extract>\n    where\n        Self: Sized + Send + Sync + 'static,\n        Self::Extract: Send,\n        Self::Error: Into<Rejection>,\n    {\n        BoxedFilter::new(self)\n    }\n}\n\nimpl<T: FilterBase> Filter for T {}\n\npub trait FilterClone: Filter + Clone {}\n\nimpl<T: Filter + Clone> FilterClone for T {}\n\nfn _assert_object_safe() {\n    fn _assert(_f: &dyn Filter<Extract = (), Error = (), Future = future::Ready<()>>) {}\n}\n\n// ===== FilterFn =====\n\npub(crate) fn filter_fn<F, U>(func: F) -> FilterFn<F>\nwhere\n    F: Fn(&mut Route) -> U,\n    U: TryFuture,\n    U::Ok: Tuple,\n    U::Error: IsReject,\n{\n    FilterFn { func }\n}\n\npub(crate) fn filter_fn_one<F, U>(\n    func: F,\n) -> impl Filter<Extract = (U::Ok,), Error = U::Error> + Copy\nwhere\n    F: Fn(&mut Route) -> U + Copy,\n    U: TryFuture + Send + 'static,\n    U::Ok: Send,\n    U::Error: IsReject,\n{\n    filter_fn(move |route| func(route).map_ok(|item| (item,)))\n}\n\n#[derive(Copy, Clone)]\n#[allow(missing_debug_implementations)]\npub(crate) struct FilterFn<F> {\n    // TODO: could include a `debug_str: &'static str` to be used in Debug impl\n    func: F,\n}\n\nimpl<F, U> FilterBase for FilterFn<F>\nwhere\n    F: Fn(&mut Route) -> U,\n    U: TryFuture + Send + 'static,\n    U::Ok: Tuple + Send,\n    U::Error: IsReject,\n{\n    type Extract = U::Ok;\n    type Error = U::Error;\n    type Future = future::IntoFuture<U>;\n\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        route::with(|route| (self.func)(route)).into_future()\n    }\n}\n\n// Path: src/filter/map.rs\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::{ready, TryFuture};\nuse pin_project::pin_project;\n\nuse super::{Filter, FilterBase, Func, Internal};\n\n#[derive(Clone, Copy, Debug)]\npub struct Map<T, F> {\n    pub(super) filter: T,\n    pub(super) callback: F,\n}\n\nimpl<T, F> FilterBase for Map<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract> + Clone + Send,\n{\n    type Extract = (F::Output,);\n    type Error = T::Error;\n    type Future = MapFuture<T, F>;\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        MapFuture {\n            extract: self.filter.filter(Internal),\n            callback: self.callback.clone(),\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\npub struct MapFuture<T: Filter, F> {\n    #[pin]\n    extract: T::Future,\n    callback: F,\n}\n\nimpl<T, F> Future for MapFuture<T, F>\nwhere\n    T: Filter,\n    F: Func<T::Extract>,\n{\n    type Output = Result<(F::Output,), T::Error>;\n\n    #[inline]\n    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        let pin = self.project();\n        match ready!(pin.extract.try_poll(cx)) {\n            Ok(ex) => {\n                let ex = (pin.callback.call(ex),);\n                Poll::Ready(Ok(ex))\n            }\n            Err(err) => Poll::Ready(Err(err)),\n        }\n    }\n}\n\n// Path: src/filters/reply.rs\n//! Reply Filters\n//!\n//! These \"filters\" behave a little differently than the rest. Instead of\n//! being used directly on requests, these filters \"wrap\" other filters.\n//!\n//!\n//! ## Wrapping a `Filter` (`with`)\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! let with_server = warp::reply::with::header(\"server\", \"warp\");\n//!\n//! let route = warp::any()\n//!     .map(warp::reply)\n//!     .with(with_server);\n//! ```\n//!\n//! Wrapping allows adding in conditional logic *before* the request enters\n//! the inner filter (though the `with::header` wrapper does not).\n\nuse std::convert::TryFrom;\nuse std::sync::Arc;\n\nuse http::header::{HeaderMap, HeaderName, HeaderValue};\n\nuse self::sealed::{WithDefaultHeader_, WithHeader_, WithHeaders_};\nuse crate::filter::{Filter, Map, WrapSealed};\nuse crate::reply::Reply;\n\n/// Wrap a [`Filter`] that adds a header to the reply.\n///\n/// # Note\n///\n/// This **only** adds a header if the underlying filter is successful, and\n/// returns a [`Reply`] If the underlying filter was rejected, the\n/// header is not added.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Always set `foo: bar` header.\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(warp::reply::with::header(\"foo\", \"bar\"));\n/// ```\npub fn header<K, V>(name: K, value: V) -> WithHeader\nwhere\n    HeaderName: TryFrom<K>,\n    <HeaderName as TryFrom<K>>::Error: Into<http::Error>,\n    HeaderValue: TryFrom<V>,\n    <HeaderValue as TryFrom<V>>::Error: Into<http::Error>,\n{\n    let (name, value) = assert_name_and_value(name, value);\n    WithHeader { name, value }\n}\n\n/// Wrap a [`Filter`] that adds multiple headers to the reply.\n///\n/// # Note\n///\n/// This **only** adds a header if the underlying filter is successful, and\n/// returns a [`Reply`] If the underlying filter was rejected, the\n/// header is not added.\n///\n/// # Example\n///\n/// ```\n/// use warp::http::header::{HeaderMap, HeaderValue};\n/// use warp::Filter;\n///\n/// let mut headers = HeaderMap::new();\n/// headers.insert(\"server\", HeaderValue::from_static(\"wee/0\"));\n/// headers.insert(\"foo\", HeaderValue::from_static(\"bar\"));\n///\n/// // Always set `server: wee/0` and `foo: bar` headers.\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(warp::reply::with::headers(headers));\n/// ```\npub fn headers(headers: HeaderMap) -> WithHeaders {\n    WithHeaders {\n        headers: Arc::new(headers),\n    }\n}\n\n// pub fn headers?\n\n/// Wrap a [`Filter`] that adds a header to the reply, if they\n/// aren't already set.\n///\n/// # Note\n///\n/// This **only** adds a header if the underlying filter is successful, and\n/// returns a [`Reply`] If the underlying filter was rejected, the\n/// header is not added.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Set `server: warp` if not already set.\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(warp::reply::with::default_header(\"server\", \"warp\"));\n/// ```\npub fn default_header<K, V>(name: K, value: V) -> WithDefaultHeader\nwhere\n    HeaderName: TryFrom<K>,\n    <HeaderName as TryFrom<K>>::Error: Into<http::Error>,\n    HeaderValue: TryFrom<V>,\n    <HeaderValue as TryFrom<V>>::Error: Into<http::Error>,\n{\n    let (name, value) = assert_name_and_value(name, value);\n    WithDefaultHeader { name, value }\n}\n\n/// Wrap a `Filter` to always set a header.\n#[derive(Clone, Debug)]\npub struct WithHeader {\n    name: HeaderName,\n    value: HeaderValue,\n}\n\nimpl<F, R> WrapSealed<F> for WithHeader\nwhere\n    F: Filter<Extract = (R,)>,\n    R: Reply,\n{\n    type Wrapped = Map<F, WithHeader_>;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        let with = WithHeader_ { with: self.clone() };\n        filter.map(with)\n    }\n}\n\n/// Wrap a `Filter` to always set multiple headers.\n#[derive(Clone, Debug)]\npub struct WithHeaders {\n    headers: Arc<HeaderMap>,\n}\n\nimpl<F, R> WrapSealed<F> for WithHeaders\nwhere\n    F: Filter<Extract = (R,)>,\n    R: Reply,\n{\n    type Wrapped = Map<F, WithHeaders_>;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        let with = WithHeaders_ { with: self.clone() };\n        filter.map(with)\n    }\n}\n\n/// Wrap a `Filter` to set a header if it is not already set.\n#[derive(Clone, Debug)]\npub struct WithDefaultHeader {\n    name: HeaderName,\n    value: HeaderValue,\n}\n\nimpl<F, R> WrapSealed<F> for WithDefaultHeader\nwhere\n    F: Filter<Extract = (R,)>,\n    R: Reply,\n{\n    type Wrapped = Map<F, WithDefaultHeader_>;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        let with = WithDefaultHeader_ { with: self.clone() };\n        filter.map(with)\n    }\n}\n\nfn assert_name_and_value<K, V>(name: K, value: V) -> (HeaderName, HeaderValue)\nwhere\n    HeaderName: TryFrom<K>,\n    <HeaderName as TryFrom<K>>::Error: Into<http::Error>,\n    HeaderValue: TryFrom<V>,\n    <HeaderValue as TryFrom<V>>::Error: Into<http::Error>,\n{\n    let name = <HeaderName as TryFrom<K>>::try_from(name)\n        .map_err(Into::into)\n        .unwrap_or_else(|_| panic!(\"invalid header name\"));\n\n    let value = <HeaderValue as TryFrom<V>>::try_from(value)\n        .map_err(Into::into)\n        .unwrap_or_else(|_| panic!(\"invalid header value\"));\n\n    (name, value)\n}\n\nmod sealed {\n    use super::{WithDefaultHeader, WithHeader, WithHeaders};\n    use crate::generic::{Func, One};\n    use crate::reply::{Reply, Reply_};\n\n    #[derive(Clone)]\n    #[allow(missing_debug_implementations)]\n    pub struct WithHeader_ {\n        pub(super) with: WithHeader,\n    }\n\n    impl<R: Reply> Func<One<R>> for WithHeader_ {\n        type Output = Reply_;\n\n        fn call(&self, args: One<R>) -> Self::Output {\n            let mut resp = args.0.into_response();\n            // Use \"insert\" to replace any set header...\n            resp.headers_mut()\n                .insert(&self.with.name, self.with.value.clone());\n            Reply_(resp)\n        }\n    }\n\n    #[derive(Clone)]\n    #[allow(missing_debug_implementations)]\n    pub struct WithHeaders_ {\n        pub(super) with: WithHeaders,\n    }\n\n    impl<R: Reply> Func<One<R>> for WithHeaders_ {\n        type Output = Reply_;\n\n        fn call(&self, args: One<R>) -> Self::Output {\n            let mut resp = args.0.into_response();\n            for (name, value) in &*self.with.headers {\n                resp.headers_mut().insert(name, value.clone());\n            }\n            Reply_(resp)\n        }\n    }\n\n    #[derive(Clone)]\n    #[allow(missing_debug_implementations)]\n    pub struct WithDefaultHeader_ {\n        pub(super) with: WithDefaultHeader,\n    }\n\n    impl<R: Reply> Func<One<R>> for WithDefaultHeader_ {\n        type Output = Reply_;\n\n        fn call(&self, args: One<R>) -> Self::Output {\n            let mut resp = args.0.into_response();\n            resp.headers_mut()\n                .entry(&self.with.name)\n                .or_insert_with(|| self.with.value.clone());\n\n            Reply_(resp)\n        }\n    }\n}\n\n// Path: src/reply.rs\n//! Reply to requests.\n//!\n//! A [`Reply`](./trait.Reply.html) is a type that can be converted into an HTTP\n//! response to be sent to the client. These are typically the successful\n//! counterpart to a [rejection](../reject).\n//!\n//! The functions in this module are helpers for quickly creating a reply.\n//! Besides them, you can return a type that implements [`Reply`](./trait.Reply.html). This\n//! could be any of the following:\n//!\n//! - [`http::Response<impl Into<hyper::Body>>`](https://docs.rs/http)\n//! - `String`\n//! - `&'static str`\n//! - `http::StatusCode`\n//!\n//! # Example\n//!\n//! ```\n//! use warp::{Filter, http::Response};\n//!\n//! // Returns an empty `200 OK` response.\n//! let empty_200 = warp::any().map(warp::reply);\n//!\n//! // Returns a `200 OK` response with custom header and body.\n//! let custom = warp::any().map(|| {\n//!     Response::builder()\n//!         .header(\"my-custom-header\", \"some-value\")\n//!         .body(\"and a custom body\")\n//! });\n//!\n//! // GET requests return the empty 200, POST return the custom.\n//! let routes = warp::get().and(empty_200)\n//!     .or(warp::post().and(custom));\n//! ```\n\nuse std::borrow::Cow;\nuse std::convert::TryFrom;\n\nuse crate::generic::{Either, One};\nuse http::header::{HeaderName, HeaderValue, CONTENT_TYPE};\nuse http::StatusCode;\nuse hyper::Body;\nuse serde::Serialize;\n\n// This re-export just looks weird in docs...\npub(crate) use self::sealed::Reply_;\nuse self::sealed::{BoxedReply, Internal};\n#[doc(hidden)]\npub use crate::filters::reply as with;\n\n/// Response type into which types implementing the `Reply` trait are convertable.\npub type Response = ::http::Response<Body>;\n\n/// Returns an empty `Reply` with status code `200 OK`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // GET /just-ok returns an empty `200 OK`.\n/// let route = warp::path(\"just-ok\")\n///     .map(|| {\n///         println!(\"got a /just-ok request!\");\n///         warp::reply()\n///     });\n/// ```\n#[inline]\npub fn reply() -> impl Reply {\n    StatusCode::OK\n}\n\n/// Convert the value into a `Reply` with the value encoded as JSON.\n///\n/// The passed value must implement [`Serialize`][ser]. Many\n/// collections do, and custom domain types can have `Serialize` derived.\n///\n/// [ser]: https://serde.rs\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // GET /ids returns a `200 OK` with a JSON array of ids:\n/// // `[1, 3, 7, 13]`\n/// let route = warp::path(\"ids\")\n///     .map(|| {\n///         let our_ids = vec![1, 3, 7, 13];\n///         warp::reply::json(&our_ids)\n///     });\n/// ```\n///\n/// # Note\n///\n/// If a type fails to be serialized into JSON, the error is logged at the\n/// `error` level, and the returned `impl Reply` will be an empty\n/// `500 Internal Server Error` response.\npub fn json<T>(val: &T) -> Json\nwhere\n    T: Serialize,\n{\n    Json {\n        inner: serde_json::to_vec(val).map_err(|err| {\n            tracing::error!(\"reply::json error: {}\", err);\n        }),\n    }\n}\n\n/// A JSON formatted reply.\n#[allow(missing_debug_implementations)]\npub struct Json {\n    inner: Result<Vec<u8>, ()>,\n}\n\nimpl Reply for Json {\n    #[inline]\n    fn into_response(self) -> Response {\n        match self.inner {\n            Ok(body) => {\n                let mut res = Response::new(body.into());\n                res.headers_mut()\n                    .insert(CONTENT_TYPE, HeaderValue::from_static(\"application/json\"));\n                res\n            }\n            Err(()) => StatusCode::INTERNAL_SERVER_ERROR.into_response(),\n        }\n    }\n}\n\n/// Reply with a body and `content-type` set to `text/html; charset=utf-8`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let body = r#\"\n/// <html>\n///     <head>\n///         <title>HTML with warp!</title>\n///     </head>\n///     <body>\n///         <h1>warp + HTML = &hearts;</h1>\n///     </body>\n/// </html>\n/// \"#;\n///\n/// let route = warp::any()\n///     .map(move || {\n///         warp::reply::html(body)\n///     });\n/// ```\npub fn html<T>(body: T) -> Html<T>\nwhere\n    Body: From<T>,\n    T: Send,\n{\n    Html { body }\n}\n\n/// An HTML reply.\n#[allow(missing_debug_implementations)]\npub struct Html<T> {\n    body: T,\n}\n\nimpl<T> Reply for Html<T>\nwhere\n    Body: From<T>,\n    T: Send,\n{\n    #[inline]\n    fn into_response(self) -> Response {\n        let mut res = Response::new(Body::from(self.body));\n        res.headers_mut().insert(\n            CONTENT_TYPE,\n            HeaderValue::from_static(\"text/html; charset=utf-8\"),\n        );\n        res\n    }\n}\n\n/// Types that can be converted into a `Response`.\n///\n/// This trait is implemented for the following:\n///\n/// - `http::StatusCode`\n/// - `http::Response<impl Into<hyper::Body>>`\n/// - `String`\n/// - `&'static str`\n///\n/// # Example\n///\n/// ```rust\n/// use warp::{Filter, http::Response};\n///\n/// struct Message {\n///     msg: String\n/// }\n///\n/// impl warp::Reply for Message {\n///     fn into_response(self) -> warp::reply::Response {\n///         Response::new(format!(\"message: {}\", self.msg).into())\n///     }\n/// }\n///\n/// fn handler() -> Message {\n///     Message { msg: \"Hello\".to_string() }\n/// }\n///\n/// let route = warp::any().map(handler);\n/// ```\npub trait Reply: BoxedReply + Send {\n    /// Converts the given value into a [`Response`].\n    ///\n    /// [`Response`]: type.Response.html\n    fn into_response(self) -> Response;\n\n    /*\n    TODO: Currently unsure about having trait methods here, as it\n    requires returning an exact type, which I'd rather not commit to.\n    Additionally, it doesn't work great with `Box<Reply>`.\n\n    A possible alternative is to have wrappers, like\n\n    - `WithStatus<R: Reply>(StatusCode, R)`\n\n\n    /// Change the status code of this `Reply`.\n    fn with_status(self, status: StatusCode) -> Reply_\n    where\n        Self: Sized,\n    {\n        let mut res = self.into_response();\n        *res.status_mut() = status;\n        Reply_(res)\n    }\n\n    /// Add a header to this `Reply`.\n    ///\n    /// # Example\n    ///\n    /// ```rust\n    /// use warp::Reply;\n    ///\n    /// let reply = warp::reply()\n    ///     .with_header(\"x-foo\", \"bar\");\n    /// ```\n    fn with_header<K, V>(self, name: K, value: V) -> Reply_\n    where\n        Self: Sized,\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        match <HeaderName as TryFrom<K>>::try_from(name) {\n            Ok(name) => match <HeaderValue as TryFrom<V>>::try_from(value) {\n                Ok(value) => {\n                    let mut res = self.into_response();\n                    res.headers_mut().append(name, value);\n                    Reply_(res)\n                },\n                Err(err) => {\n                    tracing::error!(\"with_header value error: {}\", err.into());\n                    Reply_(::reject::server_error()\n                        .into_response())\n                }\n            },\n            Err(err) => {\n                tracing::error!(\"with_header name error: {}\", err.into());\n                Reply_(::reject::server_error()\n                    .into_response())\n            }\n        }\n    }\n    */\n}\n\nimpl<T: Reply + ?Sized> Reply for Box<T> {\n    fn into_response(self) -> Response {\n        self.boxed_into_response(Internal)\n    }\n}\n\nfn _assert_object_safe() {\n    fn _assert(_: &dyn Reply) {}\n}\n\n/// Wrap an `impl Reply` to change its `StatusCode`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .map(|reply| {\n///         warp::reply::with_status(reply, warp::http::StatusCode::CREATED)\n///     });\n/// ```\npub fn with_status<T: Reply>(reply: T, status: StatusCode) -> WithStatus<T> {\n    WithStatus { reply, status }\n}\n\n/// Wrap an `impl Reply` to change its `StatusCode`.\n///\n/// Returned by `warp::reply::with_status`.\n#[derive(Debug)]\npub struct WithStatus<T> {\n    reply: T,\n    status: StatusCode,\n}\n\nimpl<T: Reply> Reply for WithStatus<T> {\n    fn into_response(self) -> Response {\n        let mut res = self.reply.into_response();\n        *res.status_mut() = self.status;\n        res\n    }\n}\n\n/// Wrap an `impl Reply` to add a header when rendering.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .map(|reply| {\n///         warp::reply::with_header(reply, \"server\", \"warp\")\n///     });\n/// ```\npub fn with_header<T: Reply, K, V>(reply: T, name: K, value: V) -> WithHeader<T>\nwhere\n    HeaderName: TryFrom<K>,\n    <HeaderName as TryFrom<K>>::Error: Into<http::Error>,\n    HeaderValue: TryFrom<V>,\n    <HeaderValue as TryFrom<V>>::Error: Into<http::Error>,\n{\n    let header = match <HeaderName as TryFrom<K>>::try_from(name) {\n        Ok(name) => match <HeaderValue as TryFrom<V>>::try_from(value) {\n            Ok(value) => Some((name, value)),\n            Err(err) => {\n                let err = err.into();\n                tracing::error!(\"with_header value error: {}\", err);\n                None\n            }\n        },\n        Err(err) => {\n            let err = err.into();\n            tracing::error!(\"with_header name error: {}\", err);\n            None\n        }\n    };\n\n    WithHeader { header, reply }\n}\n\n/// Wraps an `impl Reply` and adds a header when rendering.\n///\n/// Returned by `warp::reply::with_header`.\n#[derive(Debug)]\npub struct WithHeader<T> {\n    header: Option<(HeaderName, HeaderValue)>,\n    reply: T,\n}\n\nimpl<T: Reply> Reply for WithHeader<T> {\n    fn into_response(self) -> Response {\n        let mut res = self.reply.into_response();\n        if let Some((name, value)) = self.header {\n            res.headers_mut().insert(name, value);\n        }\n        res\n    }\n}\n\nimpl<T: Send> Reply for ::http::Response<T>\nwhere\n    Body: From<T>,\n{\n    #[inline]\n    fn into_response(self) -> Response {\n        self.map(Body::from)\n    }\n}\n\nimpl Reply for ::http::StatusCode {\n    #[inline]\n    fn into_response(self) -> Response {\n        let mut res = Response::default();\n        *res.status_mut() = self;\n        res\n    }\n}\n\nimpl Reply for ::http::Error {\n    #[inline]\n    fn into_response(self) -> Response {\n        tracing::error!(\"reply error: {:?}\", self);\n        StatusCode::INTERNAL_SERVER_ERROR.into_response()\n    }\n}\n\nimpl<T, E> Reply for Result<T, E>\nwhere\n    T: Reply,\n    E: Reply,\n{\n    #[inline]\n    fn into_response(self) -> Response {\n        match self {\n            Ok(t) => t.into_response(),\n            Err(e) => e.into_response(),\n        }\n    }\n}\n\nfn text_plain<T: Into<Body>>(body: T) -> Response {\n    let mut response = ::http::Response::new(body.into());\n    response.headers_mut().insert(\n        CONTENT_TYPE,\n        HeaderValue::from_static(\"text/plain; charset=utf-8\"),\n    );\n    response\n}\n\nimpl Reply for String {\n    #[inline]\n    fn into_response(self) -> Response {\n        text_plain(self)\n    }\n}\n\nimpl Reply for Vec<u8> {\n    #[inline]\n    fn into_response(self) -> Response {\n        ::http::Response::builder()\n            .header(\n                CONTENT_TYPE,\n                HeaderValue::from_static(\"application/octet-stream\"),\n            )\n            .body(Body::from(self))\n            .unwrap()\n    }\n}\n\nimpl Reply for &'static str {\n    #[inline]\n    fn into_response(self) -> Response {\n        text_plain(self)\n    }\n}\n\nimpl Reply for Cow<'static, str> {\n    #[inline]\n    fn into_response(self) -> Response {\n        match self {\n            Cow::Borrowed(s) => s.into_response(),\n            Cow::Owned(s) => s.into_response(),\n        }\n    }\n}\n\nimpl Reply for &'static [u8] {\n    #[inline]\n    fn into_response(self) -> Response {\n        ::http::Response::builder()\n            .header(\n                CONTENT_TYPE,\n                HeaderValue::from_static(\"application/octet-stream\"),\n            )\n            .body(Body::from(self))\n            .unwrap()\n    }\n}\n\nimpl<T, U> Reply for Either<T, U>\nwhere\n    T: Reply,\n    U: Reply,\n{\n    #[inline]\n    fn into_response(self) -> Response {\n        match self {\n            Either::A(a) => a.into_response(),\n            Either::B(b) => b.into_response(),\n        }\n    }\n}\n\nimpl<T> Reply for One<T>\nwhere\n    T: Reply,\n{\n    #[inline]\n    fn into_response(self) -> Response {\n        self.0.into_response()\n    }\n}\n\nimpl Reply for std::convert::Infallible {\n    #[inline(always)]\n    fn into_response(self) -> Response {\n        match self {}\n    }\n}\n\nmod sealed {\n    use super::{Reply, Response};\n\n    // An opaque type to return `impl Reply` from trait methods.\n    #[allow(missing_debug_implementations)]\n    pub struct Reply_(pub(crate) Response);\n\n    impl Reply for Reply_ {\n        #[inline]\n        fn into_response(self) -> Response {\n            self.0\n        }\n    }\n\n    #[allow(missing_debug_implementations)]\n    pub struct Internal;\n\n    // Implemented for all types that implement `Reply`.\n    //\n    // A user doesn't need to worry about this, it's just trait\n    // hackery to get `Box<dyn Reply>` working.\n    pub trait BoxedReply {\n        fn boxed_into_response(self: Box<Self>, internal: Internal) -> Response;\n    }\n\n    impl<T: Reply> BoxedReply for T {\n        fn boxed_into_response(self: Box<Self>, _: Internal) -> Response {\n            (*self).into_response()\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use std::collections::HashMap;\n\n    use super::*;\n\n    #[test]\n    fn json_serde_error() {\n        // a HashMap<Vec, _> cannot be serialized to JSON\n        let mut map = HashMap::new();\n        map.insert(vec![1, 2], 45);\n\n        let res = json(&map).into_response();\n        assert_eq!(res.status(), 500);\n    }\n\n    #[test]\n    fn response_builder_error() {\n        let res = ::http::Response::builder()\n            .status(1337)\n            .body(\"woops\")\n            .into_response();\n\n        assert_eq!(res.status(), 500);\n    }\n\n    #[test]\n    fn boxed_reply() {\n        let r: Box<dyn Reply> = Box::new(reply());\n        let resp = r.into_response();\n        assert_eq!(resp.status(), 200);\n    }\n}\n\n// Path: src/filter/service.rs\nuse std::convert::Infallible;\nuse std::future::Future;\nuse std::net::SocketAddr;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::future::TryFuture;\nuse hyper::service::Service;\nuse pin_project::pin_project;\n\nuse crate::reject::IsReject;\nuse crate::reply::{Reply, Response};\nuse crate::route::{self, Route};\nuse crate::{Filter, Request};\n\n/// Convert a `Filter` into a `Service`.\n///\n/// Filters are normally what APIs are built on in warp. However, it can be\n/// useful to convert a `Filter` into a [`Service`][Service], such as if\n/// further customizing a `hyper::Service`, or if wanting to make use of\n/// the greater [Tower][tower] set of middleware.\n///\n/// # Example\n///\n/// Running a `warp::Filter` on a regular `hyper::Server`:\n///\n/// ```\n/// # async fn run() -> Result<(), Box<dyn std::error::Error>> {\n/// use std::convert::Infallible;\n/// use warp::Filter;\n///\n/// // Our Filter...\n/// let route = warp::any().map(|| \"Hello From Warp!\");\n///\n/// // Convert it into a `Service`...\n/// let svc = warp::service(route);\n///\n/// // Typical hyper setup...\n/// let make_svc = hyper::service::make_service_fn(move |_| async move {\n///     Ok::<_, Infallible>(svc)\n/// });\n///\n/// hyper::Server::bind(&([127, 0, 0, 1], 3030).into())\n///     .serve(make_svc)\n///     .await?;\n/// # Ok(())\n/// # }\n/// ```\n///\n/// [Service]: https://docs.rs/hyper/0.13.*/hyper/service/trait.Service.html\n/// [tower]: https://docs.rs/tower\npub fn service<F>(filter: F) -> FilteredService<F>\nwhere\n    F: Filter,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    FilteredService { filter }\n}\n\n#[derive(Copy, Clone, Debug)]\npub struct FilteredService<F> {\n    filter: F,\n}\n\nimpl<F> FilteredService<F>\nwhere\n    F: Filter,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    #[inline]\n    pub(crate) fn call_with_addr(\n        &self,\n        req: Request,\n        remote_addr: Option<SocketAddr>,\n    ) -> FilteredFuture<F::Future> {\n        debug_assert!(!route::is_set(), \"nested route::set calls\");\n\n        let route = Route::new(req, remote_addr);\n        let fut = route::set(&route, || self.filter.filter(super::Internal));\n        FilteredFuture { future: fut, route }\n    }\n}\n\nimpl<F> Service<Request> for FilteredService<F>\nwhere\n    F: Filter,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    type Response = Response;\n    type Error = Infallible;\n    type Future = FilteredFuture<F::Future>;\n\n    fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {\n        Poll::Ready(Ok(()))\n    }\n\n    #[inline]\n    fn call(&mut self, req: Request) -> Self::Future {\n        self.call_with_addr(req, None)\n    }\n}\n\n#[pin_project]\n#[derive(Debug)]\npub struct FilteredFuture<F> {\n    #[pin]\n    future: F,\n    route: ::std::cell::RefCell<Route>,\n}\n\nimpl<F> Future for FilteredFuture<F>\nwhere\n    F: TryFuture,\n    F::Ok: Reply,\n    F::Error: IsReject,\n{\n    type Output = Result<Response, Infallible>;\n\n    #[inline]\n    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        debug_assert!(!route::is_set(), \"nested route::set calls\");\n\n        let pin = self.project();\n        let fut = pin.future;\n        match route::set(pin.route, || fut.try_poll(cx)) {\n            Poll::Ready(Ok(ok)) => Poll::Ready(Ok(ok.into_response())),\n            Poll::Pending => Poll::Pending,\n            Poll::Ready(Err(err)) => {\n                tracing::debug!(\"rejected: {:?}\", err);\n                Poll::Ready(Ok(err.into_response()))\n            }\n        }\n    }\n}\n\n// Path: src/filters/query.rs\n//! Query Filters\n\nuse futures_util::future;\nuse serde::de::DeserializeOwned;\n\nuse crate::filter::{filter_fn_one, Filter, One};\nuse crate::reject::{self, Rejection};\n\n/// Creates a `Filter` that decodes query parameters to the type `T`.\n///\n/// If cannot decode into a `T`, the request is rejected with a `400 Bad Request`.\n///\n/// # Example\n///\n/// ```\n/// use std::collections::HashMap;\n/// use warp::{\n///     http::Response,\n///     Filter,\n/// };\n///\n/// let route = warp::any()\n///     .and(warp::query::<HashMap<String, String>>())\n///     .map(|map: HashMap<String, String>| {\n///         let mut response: Vec<String> = Vec::new();\n///         for (key, value) in map.into_iter() {\n///             response.push(format!(\"{}={}\", key, value))\n///         }\n///         Response::builder().body(response.join(\";\"))\n///     });\n/// ```\n///\n/// You can define your custom query object and deserialize with [Serde][Serde]. Ensure to include\n/// the crate in your dependencies before usage.\n///\n/// ```\n/// use serde_derive::{Deserialize, Serialize};\n/// use std::collections::HashMap;\n/// use warp::{\n///     http::Response,\n///     Filter,\n/// };\n///\n/// #[derive(Serialize, Deserialize)]\n/// struct FooQuery {\n///     foo: Option<String>,\n///     bar: u8,\n/// }\n///\n/// let route = warp::any()\n///     .and(warp::query::<FooQuery>())\n///     .map(|q: FooQuery| {\n///         if let Some(foo) = q.foo {\n///             Response::builder().body(format!(\"foo={}\", foo))\n///         } else {\n///             Response::builder().body(format!(\"bar={}\", q.bar))\n///         }\n///     });\n/// ```\n///\n/// For more examples, please take a look at [examples/query_string.rs](https://github.com/seanmonstar/warp/blob/master/examples/query_string.rs).\n///\n/// [Serde]: https://docs.rs/serde\npub fn query<T: DeserializeOwned + Send + 'static>(\n) -> impl Filter<Extract = One<T>, Error = Rejection> + Copy {\n    filter_fn_one(|route| {\n        let query_string = route.query().unwrap_or_else(|| {\n            tracing::debug!(\"route was called without a query string, defaulting to empty\");\n            \"\"\n        });\n\n        let query_encoded = serde_urlencoded::from_str(query_string).map_err(|e| {\n            tracing::debug!(\"failed to decode query string '{}': {:?}\", query_string, e);\n            reject::invalid_query()\n        });\n        future::ready(query_encoded)\n    })\n}\n\n/// Creates a `Filter` that returns the raw query string as type String.\npub fn raw() -> impl Filter<Extract = One<String>, Error = Rejection> + Copy {\n    filter_fn_one(|route| {\n        let route = route\n            .query()\n            .map(|q| q.to_owned())\n            .map(Ok)\n            .unwrap_or_else(|| Err(reject::invalid_query()));\n        future::ready(route)\n    })\n}\n\n// Path: src/filters/any.rs\n//! A filter that matches any route.\nuse std::convert::Infallible;\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\n...\n", "needle_token_start": 2388, "needle_token_end": 2483, "code_context_ntokens": 16239, "output": ["```rust\nfn map_err<F, E>(self, _internal: Internal, fun: F) -> MapErr<Self, F>\n    where\n        Self: Sized,\n        F: Fn(Self::Error) -> E + Clone,\n        E: ::std::fmt::Debug + Send,\n    {\n        MapErr {\n            filter: self,\n            callback: fun,\n        }\n    }\n```"]}
{"repo": "seanmonstar/warp", "name": "poll_read", "language": "rust", "path": "src/transport.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to asynchronously read data from an input source into a buffer, adhering to the non-blocking I/O model.\n2. **Input**: It takes a mutable reference to itself, a context for handling asynchronous tasks, and a mutable buffer to store the read data.\n3. **Output**: Returns a `Poll` wrapped result which indicates whether the read operation is completed (`Ready`) or pending (`Pending`), and includes any I/O errors encountered.\n4. **Procedure**: The function first pins the inner data source to prevent it from moving in memory. It then delegates the read operation to the pinned data source's asynchronous read method, passing along the context and buffer. The result of this operation, indicating the progress and success of the read, is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/filters/sse.rs\n//! Server-Sent Events (SSE)\n//!\n//! # Example\n//!\n//! ```\n//!\n//! use std::time::Duration;\n//! use std::convert::Infallible;\n//! use warp::{Filter, sse::Event};\n//! use futures_util::{stream::iter, Stream};\n//!\n//! fn sse_events() -> impl Stream<Item = Result<Event, Infallible>> {\n//!     iter(vec![\n//!         Ok(Event::default().data(\"unnamed event\")),\n//!         Ok(\n//!             Event::default().event(\"chat\")\n//!             .data(\"chat message\")\n//!         ),\n//!         Ok(\n//!             Event::default().id(13.to_string())\n//!             .event(\"chat\")\n...\n// Path: src/filters/log.rs\n//! Logger Filters\n\nuse std::fmt;\nuse std::net::SocketAddr;\nuse std::time::{Duration, Instant};\n\nuse http::{header, StatusCode};\n\nuse crate::filter::{Filter, WrapSealed};\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::route::Route;\n\nuse self::internal::WithLog;\n\n/// Create a wrapping [`Filter`](crate::Filter) with the specified `name` as the `target`.\n///\n/// This uses the default access logging format, and log records produced\n/// will have their `target` set to `name`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // If using something like `pretty_env_logger`,\n/// // view logs by setting `RUST_LOG=example::api`.\n/// let log = warp::log(\"example::api\");\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(log);\n/// ```\npub fn log(name: &'static str) -> Log<impl Fn(Info<'_>) + Copy> {\n    let func = move |info: Info<'_>| {\n        // TODO?\n        // - response content length?\n        log::info!(\n            target: name,\n            \"{} \\\"{} {} {:?}\\\" {} \\\"{}\\\" \\\"{}\\\" {:?}\",\n            OptFmt(info.route.remote_addr()),\n            info.method(),\n            info.path(),\n            info.route.version(),\n            info.status().as_u16(),\n            OptFmt(info.referer()),\n            OptFmt(info.user_agent()),\n            info.elapsed(),\n        );\n    };\n    Log { func }\n}\n\n/// Create a wrapping [`Filter`](crate::Filter) that receives `warp::log::Info`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let log = warp::log::custom(|info| {\n///     // Use a log macro, or slog, or println, or whatever!\n///     eprintln!(\n///         \"{} {} {}\",\n///         info.method(),\n///         info.path(),\n///         info.status(),\n///     );\n/// });\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(log);\n/// ```\npub fn custom<F>(func: F) -> Log<F>\nwhere\n    F: Fn(Info<'_>),\n{\n    Log { func }\n}\n\n/// Decorates a [`Filter`] to log requests and responses.\n#[derive(Clone, Copy, Debug)]\npub struct Log<F> {\n    func: F,\n}\n\n/// Information about the request/response that can be used to prepare log lines.\n#[allow(missing_debug_implementations)]\npub struct Info<'a> {\n    route: &'a Route,\n    start: Instant,\n    status: StatusCode,\n}\n\nimpl<FN, F> WrapSealed<F> for Log<FN>\nwhere\n    FN: Fn(Info<'_>) + Clone + Send,\n    F: Filter + Clone + Send,\n    F::Extract: Reply,\n    F::Error: IsReject,\n{\n    type Wrapped = WithLog<FN, F>;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        WithLog {\n            filter,\n            log: self.clone(),\n        }\n    }\n}\n\nimpl<'a> Info<'a> {\n    /// View the remote `SocketAddr` of the request.\n    pub fn remote_addr(&self) -> Option<SocketAddr> {\n        self.route.remote_addr()\n    }\n\n    /// View the `http::Method` of the request.\n    pub fn method(&self) -> &http::Method {\n        self.route.method()\n    }\n\n    /// View the URI path of the request.\n    pub fn path(&self) -> &str {\n        self.route.full_path()\n    }\n\n    /// View the `http::Version` of the request.\n    pub fn version(&self) -> http::Version {\n        self.route.version()\n    }\n\n    /// View the `http::StatusCode` of the response.\n    pub fn status(&self) -> http::StatusCode {\n        self.status\n    }\n\n    /// View the referer of the request.\n    pub fn referer(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::REFERER)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// View the user agent of the request.\n    pub fn user_agent(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::USER_AGENT)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// View the `Duration` that elapsed for the request.\n    pub fn elapsed(&self) -> Duration {\n        tokio::time::Instant::now().into_std() - self.start\n    }\n\n    /// View the host of the request\n    pub fn host(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::HOST)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// Access the full headers of the request\n    pub fn request_headers(&self) -> &http::HeaderMap {\n        self.route.headers()\n    }\n}\n\nstruct OptFmt<T>(Option<T>);\n\nimpl<T: fmt::Display> fmt::Display for OptFmt<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        if let Some(ref t) = self.0 {\n            fmt::Display::fmt(t, f)\n        } else {\n            f.write_str(\"-\")\n        }\n    }\n}\n\nmod internal {\n    use std::future::Future;\n    use std::pin::Pin;\n    use std::task::{Context, Poll};\n    use std::time::Instant;\n\n    use futures_util::{ready, TryFuture};\n    use pin_project::pin_project;\n\n    use super::{Info, Log};\n    use crate::filter::{Filter, FilterBase, Internal};\n    use crate::reject::IsReject;\n    use crate::reply::{Reply, Response};\n    use crate::route;\n\n    #[allow(missing_debug_implementations)]\n    pub struct Logged(pub(super) Response);\n\n    impl Reply for Logged {\n        #[inline]\n        fn into_response(self) -> Response {\n            self.0\n        }\n    }\n\n    #[allow(missing_debug_implementations)]\n    #[derive(Clone, Copy)]\n    pub struct WithLog<FN, F> {\n        pub(super) filter: F,\n        pub(super) log: Log<FN>,\n    }\n\n    impl<FN, F> FilterBase for WithLog<FN, F>\n    where\n        FN: Fn(Info<'_>) + Clone + Send,\n        F: Filter + Clone + Send,\n        F::Extract: Reply,\n        F::Error: IsReject,\n    {\n        type Extract = (Logged,);\n        type Error = F::Error;\n        type Future = WithLogFuture<FN, F::Future>;\n\n        fn filter(&self, _: Internal) -> Self::Future {\n            let started = tokio::time::Instant::now().into_std();\n            WithLogFuture {\n                log: self.log.clone(),\n                future: self.filter.filter(Internal),\n                started,\n            }\n        }\n    }\n\n    #[allow(missing_debug_implementations)]\n    #[pin_project]\n    pub struct WithLogFuture<FN, F> {\n        log: Log<FN>,\n        #[pin]\n        future: F,\n        started: Instant,\n    }\n\n    impl<FN, F> Future for WithLogFuture<FN, F>\n    where\n        FN: Fn(Info<'_>),\n        F: TryFuture,\n        F::Ok: Reply,\n        F::Error: IsReject,\n    {\n        type Output = Result<(Logged,), F::Error>;\n\n        fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n            let pin = self.as_mut().project();\n            let (result, status) = match ready!(pin.future.try_poll(cx)) {\n                Ok(reply) => {\n                    let resp = reply.into_response();\n                    let status = resp.status();\n                    (Poll::Ready(Ok((Logged(resp),))), status)\n                }\n                Err(reject) => {\n                    let status = reject.status();\n                    (Poll::Ready(Err(reject)), status)\n                }\n            };\n\n            route::with(|route| {\n                (self.log.func)(Info {\n                    route,\n                    start: self.started,\n                    status,\n                });\n            });\n\n            result\n        }\n    }\n}\n\n// Path: src/filters/host.rs\n//! Host (\"authority\") filter\n//!\nuse crate::filter::{filter_fn_one, Filter, One};\nuse crate::reject::{self, Rejection};\nuse futures_util::future;\npub use http::uri::Authority;\nuse std::str::FromStr;\n\n/// Creates a `Filter` that requires a specific authority (target server's\n/// host and port) in the request.\n///\n/// Authority is specified either in the `Host` header or in the target URI.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let multihost =\n///     warp::host::exact(\"foo.com\").map(|| \"you've reached foo.com\")\n///     .or(warp::host::exact(\"bar.com\").map(|| \"you've reached bar.com\"));\n/// ```\npub fn exact(expected: &str) -> impl Filter<Extract = (), Error = Rejection> + Clone {\n    let expected = Authority::from_str(expected).expect(\"invalid host/authority\");\n    optional()\n        .and_then(move |option: Option<Authority>| match option {\n            Some(authority) if authority == expected => future::ok(()),\n            _ => future::err(reject::not_found()),\n        })\n        .untuple_one()\n}\n\n/// Creates a `Filter` that looks for an authority (target server's host\n/// and port) in the request.\n///\n/// Authority is specified either in the `Host` header or in the target URI.\n///\n/// If found, extracts the `Authority`, otherwise continues the request,\n/// extracting `None`.\n///\n/// Rejects with `400 Bad Request` if the `Host` header is malformed or if there\n/// is a mismatch between the `Host` header and the target URI.\n///\n/// # Example\n///\n/// ```\n/// use warp::{Filter, host::Authority};\n///\n/// let host = warp::host::optional()\n///     .map(|authority: Option<Authority>| {\n///         if let Some(a) = authority {\n///             format!(\"{} is currently not at home\", a.host())\n///         } else {\n///             \"please state who you're trying to reach\".to_owned()\n///         }\n///     });\n/// ```\npub fn optional() -> impl Filter<Extract = One<Option<Authority>>, Error = Rejection> + Copy {\n    filter_fn_one(move |route| {\n        // The authority can be sent by clients in various ways:\n        //\n        //  1) in the \"target URI\"\n        //    a) serialized in the start line (HTTP/1.1 proxy requests)\n        //    b) serialized in `:authority` pseudo-header (HTTP/2 generated - \"SHOULD\")\n        //  2) in the `Host` header (HTTP/1.1 origin requests, HTTP/2 converted)\n        //\n        // Hyper transparently handles 1a/1b, but not 2, so we must look at both.\n\n        let from_uri = route.uri().authority();\n\n        let name = \"host\";\n        let from_header = route.headers()\n            .get(name)\n            .map(|value|\n                // Header present, parse it\n                value.to_str().map_err(|_| reject::invalid_header(name))\n                    .and_then(|value| Authority::from_str(value).map_err(|_| reject::invalid_header(name)))\n            );\n\n        future::ready(match (from_uri, from_header) {\n            // no authority in the request (HTTP/1.0 or non-conforming)\n            (None, None) => Ok(None),\n\n            // authority specified in either or both matching\n            (Some(a), None) => Ok(Some(a.clone())),\n            (None, Some(Ok(a))) => Ok(Some(a)),\n            (Some(a), Some(Ok(b))) if *a == b => Ok(Some(b)),\n\n            // mismatch\n            (Some(_), Some(Ok(_))) => Err(reject::invalid_header(name)),\n\n            // parse error\n            (_, Some(Err(r))) => Err(r),\n        })\n    })\n}\n\n// Path: src/filters/mod.rs\n//! Built-in Filters\n//!\n//! This module mostly serves as documentation to group together the list of\n//! built-in filters. Most of these are available at more convenient paths.\n\npub mod addr;\npub mod any;\npub mod body;\n#[cfg(any(feature = \"compression-brotli\", feature = \"compression-gzip\"))]\npub mod compression;\npub mod cookie;\npub mod cors;\npub mod ext;\npub mod fs;\npub mod header;\npub mod host;\npub mod log;\npub mod method;\n#[cfg(feature = \"multipart\")]\npub mod multipart;\npub mod path;\npub mod query;\npub mod reply;\npub mod sse;\npub mod trace;\n#[cfg(feature = \"websocket\")]\npub mod ws;\n\npub use crate::filter::BoxedFilter;\n\n// Path: src/transport.rs\nuse std::io;\nuse std::net::SocketAddr;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse hyper::server::conn::AddrStream;\nuse tokio::io::{AsyncRead, AsyncWrite, ReadBuf};\n\npub trait Transport: AsyncRead + AsyncWrite {\n    fn remote_addr(&self) -> Option<SocketAddr>;\n}\n\nimpl Transport for AddrStream {\n    fn remote_addr(&self) -> Option<SocketAddr> {\n        Some(self.remote_addr())\n    }\n}\n\npub(crate) struct LiftIo<T>(pub(crate) T);\n\nimpl<T: AsyncRead + Unpin> AsyncRead for LiftIo<T> {\n    \nfn poll_read(\n        self: Pin<&mut Self>,\n        cx: &mut Context<'_>,\n        buf: &mut ReadBuf<'_>,\n    ) -> Poll<io::Result<()>> {\n        Pin::new(&mut self.get_mut().0).poll_read(cx, buf)\n    }\n}\n\nimpl<T: AsyncWrite + Unpin> AsyncWrite for LiftIo<T> {\n    fn poll_write(\n        self: Pin<&mut Self>,\n        cx: &mut Context<'_>,\n        buf: &[u8],\n    ) -> Poll<io::Result<usize>> {\n        Pin::new(&mut self.get_mut().0).poll_write(cx, buf)\n    }\n\n    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {\n        Pin::new(&mut self.get_mut().0).poll_flush(cx)\n    }\n\n    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {\n        Pin::new(&mut self.get_mut().0).poll_shutdown(cx)\n    }\n}\n\nimpl<T: AsyncRead + AsyncWrite + Unpin> Transport for LiftIo<T> {\n    fn remote_addr(&self) -> Option<SocketAddr> {\n        None\n    }\n}\n\n// Path: src/service.rs\n//! Convert `Filter`s into `Service`s\n\npub use crate::filter::service::service;\n\n// Path: src/server.rs\n#[cfg(feature = \"tls\")]\nuse crate::tls::TlsConfigBuilder;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"tls\")]\nuse std::path::Path;\n\nuse futures_util::{future, FutureExt, TryFuture, TryStream, TryStreamExt};\nuse hyper::server::conn::AddrIncoming;\nuse hyper::service::{make_service_fn, service_fn};\nuse hyper::Server as HyperServer;\nuse tokio::io::{AsyncRead, AsyncWrite};\nuse tracing::Instrument;\n\nuse crate::filter::Filter;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::transport::Transport;\n\n/// Create a `Server` with the provided `Filter`.\npub fn serve<F>(filter: F) -> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: IsReject,\n{\n    Server {\n        pipeline: false,\n        filter,\n    }\n}\n\n/// A Warp Server ready to filter requests.\n#[derive(Debug)]\npub struct Server<F> {\n    pipeline: bool,\n    filter: F,\n}\n\n/// A Warp Server ready to filter requests over TLS.\n///\n/// *This type requires the `\"tls\"` feature.*\n#[cfg(feature = \"tls\")]\npub struct TlsServer<F> {\n    server: Server<F>,\n    tls: TlsConfigBuilder,\n}\n\n// Getting all various generic bounds to make this a re-usable method is\n// very complicated, so instead this is just a macro.\nmacro_rules! into_service {\n    ($into:expr) => {{\n        let inner = crate::service($into);\n        make_service_fn(move |transport| {\n            let inner = inner.clone();\n            let remote_addr = Transport::remote_addr(transport);\n            future::ok::<_, Infallible>(service_fn(move |req| {\n                inner.call_with_addr(req, remote_addr)\n            }))\n        })\n    }};\n}\n\nmacro_rules! addr_incoming {\n    ($addr:expr) => {{\n        let mut incoming = AddrIncoming::bind($addr)?;\n        incoming.set_nodelay(true);\n        let addr = incoming.local_addr();\n        (addr, incoming)\n    }};\n}\n\nmacro_rules! bind_inner {\n    ($this:ident, $addr:expr) => {{\n        let service = into_service!($this.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let srv = HyperServer::builder(incoming)\n            .http1_pipeline_flush($this.pipeline)\n            .serve(service);\n        Ok::<_, hyper::Error>((addr, srv))\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let service = into_service!($this.server.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let tls = $this.tls.build()?;\n        let srv = HyperServer::builder(crate::tls::TlsAcceptor::new(tls, incoming))\n            .http1_pipeline_flush($this.server.pipeline)\n            .serve(service);\n        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((addr, srv))\n    }};\n}\n\nmacro_rules! bind {\n    ($this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!($this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!(tls: $this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n}\n\nmacro_rules! try_bind {\n    ($this:ident, $addr:expr) => {{\n        (|addr| bind_inner!($this, addr))($addr)\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        (|addr| bind_inner!(tls: $this, addr))($addr)\n    }};\n}\n\n// ===== impl Server =====\n\nimpl<F> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    /// Run this `Server` forever on the current thread.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub async fn run(self, addr: impl Into<SocketAddr>) {\n        let (addr, fut) = self.bind_ephemeral(addr);\n        let span = tracing::info_span!(\"Server::run\", ?addr);\n        tracing::info!(parent: &span, \"listening on http://{}\", addr);\n\n        fut.instrument(span).await;\n    }\n\n    /// Run this `Server` forever on the current thread with a specific stream\n    /// of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    pub async fn run_incoming<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        self.run_incoming2(incoming.map_ok(crate::transport::LiftIo).into_stream())\n            .instrument(tracing::info_span!(\"Server::run_incoming\"))\n            .await;\n    }\n\n    async fn run_incoming2<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: Transport + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let fut = self.serve_incoming2(incoming);\n\n        tracing::info!(\"listening with custom incoming\");\n\n        fut.await;\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind(self, addr: impl Into<SocketAddr> + 'static) -> impl Future<Output = ()> + 'static {\n        let (_, fut) = self.bind_ephemeral(addr);\n        fut\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on any runtime.\n    ///\n    /// In case we are unable to bind to the specified address, resolves to an\n    /// error and logs the reason.\n    pub async fn try_bind(self, addr: impl Into<SocketAddr>) {\n        let addr = addr.into();\n        let srv = match try_bind!(self, &addr) {\n            Ok((_, srv)) => srv,\n            Err(err) => {\n                tracing::error!(\"error binding to {}: {}\", addr, err);\n                return;\n            }\n        };\n\n        srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        })\n        .await;\n    }\n\n    /// Bind to a possibly ephemeral socket address.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        (addr, srv)\n    }\n\n    /// Tried to bind a possibly ephemeral socket address.\n    ///\n    /// Returns a `Result` which fails in case we are unable to bind with the\n    /// underlying error.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    pub fn try_bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use warp::Filter;\n    /// use futures_util::future::TryFutureExt;\n    /// use tokio::sync::oneshot;\n    ///\n    /// # fn main() {\n    /// let routes = warp::any()\n    ///     .map(|| \"Hello, World!\");\n    ///\n    /// let (tx, rx) = oneshot::channel();\n    ///\n    /// let (addr, server) = warp::serve(routes)\n    ///     .bind_with_graceful_shutdown(([127, 0, 0, 1], 3030), async {\n    ///          rx.await.ok();\n    ///     });\n    ///\n    /// // Spawn the server into a runtime\n    /// tokio::task::spawn(server);\n    ///\n    /// // Later, start the shutdown...\n    /// let _ = tx.send(());\n    /// # }\n    /// ```\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let fut = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n        (addr, fut)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    pub fn try_bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming<I>(self, incoming: I) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        self.serve_incoming2(incoming)\n            .instrument(tracing::info_span!(\"Server::serve_incoming\"))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections and a\n    /// signal to initiate graceful shutdown.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming_with_graceful_shutdown<I>(\n        self,\n        incoming: I,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        let service = into_service!(self.filter);\n        let pipeline = self.pipeline;\n\n        async move {\n            let srv =\n                HyperServer::builder(hyper::server::accept::from_stream(incoming.into_stream()))\n                    .http1_pipeline_flush(pipeline)\n                    .serve(service)\n                    .with_graceful_shutdown(signal)\n                    .await;\n\n            if let Err(err) = srv {\n                tracing::error!(\"server error: {}\", err);\n            }\n        }\n        .instrument(tracing::info_span!(\n            \"Server::serve_incoming_with_graceful_shutdown\"\n        ))\n    }\n\n    async fn serve_incoming2<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: Transport + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let service = into_service!(self.filter);\n\n        let srv = HyperServer::builder(hyper::server::accept::from_stream(incoming.into_stream()))\n            .http1_pipeline_flush(self.pipeline)\n            .serve(service)\n            .await;\n\n        if let Err(err) = srv {\n            tracing::error!(\"server error: {}\", err);\n        }\n    }\n\n    // Generally shouldn't be used, as it can slow down non-pipelined responses.\n    //\n    // It's only real use is to make silly pipeline benchmarks look better.\n    #[doc(hidden)]\n    pub fn unstable_pipeline(mut self) -> Self {\n        self.pipeline = true;\n        self\n    }\n\n    /// Configure a server to use TLS.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    #[cfg(feature = \"tls\")]\n    pub fn tls(self) -> TlsServer<F> {\n        TlsServer {\n            server: self,\n            tls: TlsConfigBuilder::new(),\n        }\n    }\n}\n\n// // ===== impl TlsServer =====\n\n#[cfg(feature = \"tls\")]\nimpl<F> TlsServer<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    // TLS config methods\n\n    /// Specify the file path to read the private key.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn key_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.key_path(path))\n    }\n\n    /// Specify the file path to read the certificate.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn cert_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.cert_path(path))\n    }\n\n    /// Specify the file path to read the trust anchor for optional client authentication.\n    ///\n    /// Anonymous and authenticated clients will be accepted. If no trust anchor is provided by any\n    /// of the `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_optional_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.client_auth_optional_path(path))\n    }\n\n    /// Specify the file path to read the trust anchor for required client authentication.\n    ///\n    /// Only authenticated clients will be accepted. If no trust anchor is provided by any of the\n    /// `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_required_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.client_auth_required_path(path))\n    }\n\n    /// Specify the in-memory contents of the private key.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn key(self, key: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.key(key.as_ref()))\n    }\n\n    /// Specify the in-memory contents of the certificate.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn cert(self, cert: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.cert(cert.as_ref()))\n    }\n\n    /// Specify the in-memory contents of the trust anchor for optional client authentication.\n    ///\n    /// Anonymous and authenticated clients will be accepted. If no trust anchor is provided by any\n    /// of the `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_optional(self, trust_anchor: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.client_auth_optional(trust_anchor.as_ref()))\n    }\n\n    /// Specify the in-memory contents of the trust anchor for required client authentication.\n    ///\n    /// Only authenticated clients will be accepted. If no trust anchor is provided by any of the\n    /// `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_required(self, trust_anchor: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.client_auth_required(trust_anchor.as_ref()))\n    }\n\n    /// Specify the DER-encoded OCSP response.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn ocsp_resp(self, resp: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.ocsp_resp(resp.as_ref()))\n    }\n\n    fn with_tls<Func>(self, func: Func) -> Self\n    where\n        Func: FnOnce(TlsConfigBuilder) -> TlsConfigBuilder,\n    {\n        let TlsServer { server, tls } = self;\n        let tls = func(tls);\n        TlsServer { server, tls }\n    }\n\n    // Server run methods\n\n    /// Run this `TlsServer` forever on the current thread.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub async fn run(self, addr: impl Into<SocketAddr>) {\n        let (addr, fut) = self.bind_ephemeral(addr);\n        let span = tracing::info_span!(\"TlsServer::run\", %addr);\n        tracing::info!(parent: &span, \"listening on https://{}\", addr);\n\n        fut.instrument(span).await;\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on a runtime.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub async fn bind(self, addr: impl Into<SocketAddr>) {\n        let (_, fut) = self.bind_ephemeral(addr);\n        fut.await;\n    }\n\n    /// Bind to a possibly ephemeral socket address.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(tls: self, addr);\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        (addr, srv)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(tls: self, addr);\n\n        let fut = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n        (addr, fut)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn try_bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(tls: self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n}\n\n#[cfg(feature = \"tls\")]\nimpl<F> ::std::fmt::Debug for TlsServer<F>\nwhere\n    F: ::std::fmt::Debug,\n{\n    fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {\n        f.debug_struct(\"TlsServer\")\n            .field(\"server\", &self.server)\n            .finish()\n    }\n}\n\n// Path: src/test.rs\n//! Test utilities to test your filters.\n//!\n//! [`Filter`](../trait.Filter.html)s can be easily tested without starting up an HTTP\n//! server, by making use of the [`RequestBuilder`](./struct.RequestBuilder.html) in this\n//! module.\n//!\n//! # Testing Filters\n//!\n//! It's easy to test filters, especially if smaller filters are used to build\n//! up your full set. Consider these example filters:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! fn sum() -> impl Filter<Extract = (u32,), Error = warp::Rejection> + Copy {\n//!     warp::path::param()\n//!         .and(warp::path::param())\n//!         .map(|x: u32, y: u32| {\n//!             x + y\n//!         })\n//! }\n//!\n//! fn math() -> impl Filter<Extract = (String,), Error = warp::Rejection> + Copy {\n//!     warp::post()\n//!         .and(sum())\n//!         .map(|z: u32| {\n//!             format!(\"Sum = {}\", z)\n//!         })\n//! }\n//! ```\n//!\n//! We can test some requests against the `sum` filter like this:\n//!\n//! ```\n//! # use warp::Filter;\n//! #[tokio::test]\n//! async fn test_sum() {\n//! #    let sum = || warp::any().map(|| 3);\n//!     let filter = sum();\n//!\n//!     // Execute `sum` and get the `Extract` back.\n//!     let value = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .filter(&filter)\n//!         .await\n//!         .unwrap();\n//!     assert_eq!(value, 3);\n//!\n//!     // Or simply test if a request matches (doesn't reject).\n//!     assert!(\n//!         warp::test::request()\n//!             .path(\"/1/-5\")\n//!             .matches(&filter)\n//!             .await\n//!     );\n//! }\n//! ```\n//!\n//! If the filter returns something that implements `Reply`, and thus can be\n//! turned into a response sent back to the client, we can test what exact\n//! response is returned. The `math` filter uses the `sum` filter, but returns\n//! a `String` that can be turned into a response.\n//!\n//! ```\n//! # use warp::Filter;\n//! #[test]\n//! fn test_math() {\n//! #    let math = || warp::any().map(warp::reply);\n//!     let filter = math();\n//!\n//!     let res = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 405, \"GET is not allowed\");\n//!\n//!     let res = warp::test::request()\n//!         .method(\"POST\")\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 200);\n//!     assert_eq!(res.body(), \"Sum is 3\");\n//! }\n//! ```\nuse std::convert::TryFrom;\nuse std::error::Error as StdError;\nuse std::fmt;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"websocket\")]\nuse std::pin::Pin;\n#[cfg(feature = \"websocket\")]\nuse std::task::Context;\n#[cfg(feature = \"websocket\")]\nuse std::task::{self, Poll};\n\nuse bytes::Bytes;\n#[cfg(feature = \"websocket\")]\nuse futures_channel::mpsc;\n#[cfg(feature = \"websocket\")]\nuse futures_util::StreamExt;\nuse futures_util::{future, FutureExt, TryFutureExt};\nuse http::{\n    header::{HeaderName, HeaderValue},\n    Response,\n};\nuse serde::Serialize;\n#[cfg(feature = \"websocket\")]\nuse tokio::sync::oneshot;\n\nuse crate::filter::Filter;\n#[cfg(feature = \"websocket\")]\nuse crate::filters::ws::Message;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::route::{self, Route};\nuse crate::Request;\n#[cfg(feature = \"websocket\")]\nuse crate::{Sink, Stream};\n\nuse self::inner::OneOrTuple;\n\n/// Starts a new test `RequestBuilder`.\npub fn request() -> RequestBuilder {\n    RequestBuilder {\n        remote_addr: None,\n        req: Request::default(),\n    }\n}\n\n/// Starts a new test `WsBuilder`.\n#[cfg(feature = \"websocket\")]\npub fn ws() -> WsBuilder {\n    WsBuilder { req: request() }\n}\n\n/// A request builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[must_use = \"RequestBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct RequestBuilder {\n    remote_addr: Option<SocketAddr>,\n    req: Request,\n}\n\n/// A Websocket builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[cfg(feature = \"websocket\")]\n#[must_use = \"WsBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct WsBuilder {\n    req: RequestBuilder,\n}\n\n/// A test client for Websocket filters.\n#[cfg(feature = \"websocket\")]\npub struct WsClient {\n    tx: mpsc::UnboundedSender<crate::ws::Message>,\n    rx: mpsc::UnboundedReceiver<Result<crate::ws::Message, crate::error::Error>>,\n}\n\n/// An error from Websocket filter tests.\n#[derive(Debug)]\npub struct WsError {\n    cause: Box<dyn StdError + Send + Sync>,\n}\n\nimpl RequestBuilder {\n    /// Sets the method of this builder.\n    ///\n    /// The default if not set is `GET`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .method(\"POST\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Method`.\n    pub fn method(mut self, method: &str) -> Self {\n        *self.req.method_mut() = method.parse().expect(\"valid method\");\n        self\n    }\n\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .path(\"/todos/33\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(mut self, p: &str) -> Self {\n        let uri = p.parse().expect(\"test request path invalid\");\n        *self.req.uri_mut() = uri;\n        self\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .header(\"accept\", \"application/json\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(mut self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        let name: HeaderName = TryFrom::try_from(key)\n            .map_err(|_| ())\n            .expect(\"invalid header name\");\n        let value = TryFrom::try_from(value)\n            .map_err(|_| ())\n            .expect(\"invalid header value\");\n        self.req.headers_mut().insert(name, value);\n        self\n    }\n\n    /// Set the remote address of this request\n    ///\n    /// Default is no remote address.\n    ///\n    /// # Example\n    /// ```\n    /// use std::net::{IpAddr, Ipv4Addr, SocketAddr};\n    ///\n    /// let req = warp::test::request()\n    ///     .remote_addr(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080));\n    /// ```\n    pub fn remote_addr(mut self, addr: SocketAddr) -> Self {\n        self.remote_addr = Some(addr);\n        self\n    }\n\n    /// Add a type to the request's `http::Extensions`.\n    pub fn extension<T>(mut self, ext: T) -> Self\n    where\n        T: Send + Sync + 'static,\n    {\n        self.req.extensions_mut().insert(ext);\n        self\n    }\n\n    /// Set the bytes of this request body.\n    ///\n    /// Default is an empty body.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .body(\"foo=bar&baz=quux\");\n    /// ```\n    pub fn body(mut self, body: impl AsRef<[u8]>) -> Self {\n        let body = body.as_ref().to_vec();\n        let len = body.len();\n        *self.req.body_mut() = body.into();\n        self.header(\"content-length\", len.to_string())\n    }\n\n    /// Set the bytes of this request body by serializing a value into JSON.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .json(&true);\n    /// ```\n    pub fn json(mut self, val: &impl Serialize) -> Self {\n        let vec = serde_json::to_vec(val).expect(\"json() must serialize to JSON\");\n        let len = vec.len();\n        *self.req.body_mut() = vec.into();\n        self.header(\"content-length\", len.to_string())\n            .header(\"content-type\", \"application/json\")\n    }\n\n    /// Tries to apply the `Filter` on this request.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let param = warp::path::param::<u32>();\n    ///\n    ///     let ex = warp::test::request()\n    ///         .path(\"/41\")\n    ///         .filter(&param)\n    ///         .await\n    ///         .unwrap();\n    ///\n    ///     assert_eq!(ex, 41);\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .path(\"/foo\")\n    ///             .filter(&param)\n    ///             .await\n    ///             .is_err()\n    ///     );\n    ///};\n    /// ```\n    pub async fn filter<F>(self, f: &F) -> Result<<F::Extract as OneOrTuple>::Output, F::Error>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: OneOrTuple + Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.map(|ex| ex.one_or_tuple())\n    }\n\n    /// Returns whether the `Filter` matches this request, or rejects it.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let get = warp::get();\n    ///     let post = warp::post();\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&get)\n    ///             .await\n    ///     );\n    ///\n    ///     assert!(\n    ///         !warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&post)\n    ///             .await\n    ///     );\n    ///};\n    /// ```\n    pub async fn matches<F>(self, f: &F) -> bool\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.is_ok()\n    }\n\n    /// Returns `Response` provided by applying the `Filter`.\n    ///\n    /// This requires that the supplied `Filter` return a [`Reply`].\n    pub async fn reply<F>(self, f: &F) -> Response<Bytes>\n    where\n        F: Filter + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        // TODO: de-duplicate this and apply_filter()\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(\n            route::set(&route, move || f.filter(crate::filter::Internal)).then(|result| {\n                let res = match result {\n                    Ok(rep) => rep.into_response(),\n                    Err(rej) => {\n                        tracing::debug!(\"rejected: {:?}\", rej);\n                        rej.into_response()\n                    }\n                };\n                let (parts, body) = res.into_parts();\n                hyper::body::to_bytes(body).map_ok(|chunk| Response::from_parts(parts, chunk))\n            }),\n        );\n\n        let fut = future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)));\n\n        fut.await.expect(\"reply shouldn't fail\")\n    }\n\n    fn apply_filter<F>(self, f: &F) -> impl Future<Output = Result<F::Extract, F::Error>>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(route::set(&route, move || {\n            f.filter(crate::filter::Internal)\n        }));\n        future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)))\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl WsBuilder {\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .path(\"/chat\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(self, p: &str) -> Self {\n        WsBuilder {\n            req: self.req.path(p),\n        }\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .header(\"foo\", \"bar\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        WsBuilder {\n            req: self.req.header(key, value),\n        }\n    }\n\n    /// Execute this Websocket request against the provided filter.\n    ///\n    /// If the handshake succeeds, returns a `WsClient`.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use futures_util::future;\n    /// use warp::Filter;\n    /// #[tokio::main]\n    /// # async fn main() {\n    ///\n    /// // Some route that accepts websockets (but drops them immediately).\n    /// let route = warp::ws()\n    ///     .map(|ws: warp::ws::Ws| {\n    ///         ws.on_upgrade(|_| future::ready(()))\n    ///     });\n    ///\n    /// let client = warp::test::ws()\n    ///     .handshake(route)\n    ///     .await\n    ///     .expect(\"handshake\");\n    /// # }\n    /// ```\n    pub async fn handshake<F>(self, f: F) -> Result<WsClient, WsError>\n    where\n        F: Filter + Clone + Send + Sync + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        let (upgraded_tx, upgraded_rx) = oneshot::channel();\n        let (wr_tx, wr_rx) = mpsc::unbounded();\n        let (rd_tx, rd_rx) = mpsc::unbounded();\n\n        tokio::spawn(async move {\n            use tokio_tungstenite::tungstenite::protocol;\n\n            let (addr, srv) = crate::serve(f).bind_ephemeral(([127, 0, 0, 1], 0));\n\n            let mut req = self\n                .req\n                .header(\"connection\", \"upgrade\")\n                .header(\"upgrade\", \"websocket\")\n                .header(\"sec-websocket-version\", \"13\")\n                .header(\"sec-websocket-key\", \"dGhlIHNhbXBsZSBub25jZQ==\")\n                .req;\n\n            let query_string = match req.uri().query() {\n                Some(q) => format!(\"?{}\", q),\n                None => String::from(\"\"),\n            };\n\n            let uri = format!(\"http://{}{}{}\", addr, req.uri().path(), query_string)\n                .parse()\n                .expect(\"addr + path is valid URI\");\n\n            *req.uri_mut() = uri;\n\n            // let mut rt = current_thread::Runtime::new().unwrap();\n            tokio::spawn(srv);\n\n            let upgrade = ::hyper::Client::builder()\n                .build(AddrConnect(addr))\n                .request(req)\n                .and_then(hyper::upgrade::on);\n\n            let upgraded = match upgrade.await {\n                Ok(up) => {\n                    let _ = upgraded_tx.send(Ok(()));\n                    up\n                }\n                Err(err) => {\n                    let _ = upgraded_tx.send(Err(err));\n                    return;\n                }\n            };\n            let ws = crate::ws::WebSocket::from_raw_socket(\n                upgraded,\n                protocol::Role::Client,\n                Default::default(),\n            )\n            .await;\n\n            let (tx, rx) = ws.split();\n            let write = wr_rx.map(Ok).forward(tx).map(|_| ());\n\n            let read = rx\n                .take_while(|result| match result {\n                    Err(_) => future::ready(false),\n                    Ok(m) => future::ready(!m.is_close()),\n                })\n                .for_each(move |item| {\n                    rd_tx.unbounded_send(item).expect(\"ws receive error\");\n                    future::ready(())\n                });\n\n            future::join(write, read).await;\n        });\n\n        match upgraded_rx.await {\n            Ok(Ok(())) => Ok(WsClient {\n                tx: wr_tx,\n                rx: rd_rx,\n            }),\n            Ok(Err(err)) => Err(WsError::new(err)),\n            Err(_canceled) => panic!(\"websocket handshake thread panicked\"),\n        }\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl WsClient {\n    /// Send a \"text\" websocket message to the server.\n    pub async fn send_text(&mut self, text: impl Into<String>) {\n        self.send(crate::ws::Message::text(text)).await;\n    }\n\n    /// Send a websocket message to the server.\n    pub async fn send(&mut self, msg: crate::ws::Message) {\n        self.tx.unbounded_send(msg).unwrap();\n    }\n\n    /// Receive a websocket message from the server.\n    pub async fn recv(&mut self) -> Result<crate::filters::ws::Message, WsError> {\n        self.rx\n            .next()\n            .await\n            .map(|result| result.map_err(WsError::new))\n            .unwrap_or_else(|| {\n                // websocket is closed\n                Err(WsError::new(\"closed\"))\n            })\n    }\n\n    /// Assert the server has closed the connection.\n    pub async fn recv_closed(&mut self) -> Result<(), WsError> {\n        self.rx\n            .next()\n            .await\n            .map(|result| match result {\n                Ok(msg) => Err(WsError::new(format!(\"received message: {:?}\", msg))),\n                Err(err) => Err(WsError::new(err)),\n...\n", "needle_token_start": 4010, "needle_token_end": 4088, "code_context_ntokens": 16247, "output": ["```rust\nfn poll_read(\n    self: Pin<&mut Self>,\n    cx: &mut Context<'_>,\n    buf: &mut ReadBuf<'_>,\n) -> Poll<io::Result<()>> {\n    Pin::new(&mut self.get_mut().0).poll_read(cx, buf)\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "wrap_fn", "language": "rust", "path": "src/filter/wrap.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a wrapper around a given filter function, allowing for the modification or enhancement of the filter's behavior by applying additional logic before and after the original filter.\n2. **Input**: It accepts a function that takes a filter as its argument and returns a modified version of that filter.\n3. **Output**: It returns a new type that encapsulates the provided function, enabling the modified filter to be used in a filtering pipeline.\n4. **Procedure**: The function takes a user-defined function as an argument. This user-defined function is expected to take a filter as input and return a new filter as output. The main function then wraps this user-defined function in a new type, which can be used to apply the user-defined transformations to a filter within a filtering context.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/filters/path.rs\n//! Path Filters\n//!\n//! The [`Filter`](crate::Filter)s here work on the \"path\" of requests.\n//!\n//! - [`path`](./fn.path.html) matches a specific segment, like `/foo`.\n//! - [`param`](./fn.param.html) tries to parse a segment into a type, like `/:u16`.\n//! - [`end`](./fn.end.html) matches when the path end is found.\n//! - [`path!`](../../macro.path.html) eases combining multiple `path` and `param` filters.\n//!\n//! # Routing\n//!\n//! Routing in warp is simple yet powerful.\n//!\n//! First up, matching a single segment:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! // GET /hi\n...\n// Path: src/filters/fs.rs\n//! File System Filters\n\nuse std::cmp;\nuse std::convert::Infallible;\nuse std::fs::Metadata;\nuse std::future::Future;\nuse std::io;\nuse std::path::{Path, PathBuf};\nuse std::pin::Pin;\nuse std::sync::Arc;\nuse std::task::Poll;\n\nuse bytes::{Bytes, BytesMut};\nuse futures_util::future::Either;\nuse futures_util::{future, ready, stream, FutureExt, Stream, StreamExt, TryFutureExt};\nuse headers::{\n    AcceptRanges, ContentLength, ContentRange, ContentType, HeaderMapExt, IfModifiedSince, IfRange,\n    IfUnmodifiedSince, LastModified, Range,\n};\nuse http::StatusCode;\nuse hyper::Body;\nuse percent_encoding::percent_decode_str;\nuse tokio::fs::File as TkFile;\nuse tokio::io::AsyncSeekExt;\nuse tokio_util::io::poll_read_buf;\n\nuse crate::filter::{Filter, FilterClone, One};\nuse crate::reject::{self, Rejection};\nuse crate::reply::{Reply, Response};\n\n/// Creates a `Filter` that serves a File at the `path`.\n///\n/// Does not filter out based on any information of the request. Always serves\n/// the file at the exact `path` provided. Thus, this can be used to serve a\n/// single file with `GET`s, but could also be used in combination with other\n/// filters, such as after validating in `POST` request, wanting to return a\n/// specific file as the body.\n///\n/// For serving a directory, see [dir].\n///\n/// # Example\n///\n/// ```\n/// // Always serves this file from the file system.\n/// let route = warp::fs::file(\"/www/static/app.js\");\n/// ```\npub fn file(path: impl Into<PathBuf>) -> impl FilterClone<Extract = One<File>, Error = Rejection> {\n    let path = Arc::new(path.into());\n    crate::any()\n        .map(move || {\n            tracing::trace!(\"file: {:?}\", path);\n            ArcPath(path.clone())\n        })\n        .and(conditionals())\n        .and_then(file_reply)\n}\n\n/// Creates a `Filter` that serves a directory at the base `path` joined\n/// by the request path.\n///\n/// This can be used to serve \"static files\" from a directory. By far the most\n/// common pattern of serving static files is for `GET` requests, so this\n/// filter automatically includes a `GET` check.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Matches requests that start with `/static`,\n/// // and then uses the rest of that path to lookup\n/// // and serve a file from `/www/static`.\n/// let route = warp::path(\"static\")\n///     .and(warp::fs::dir(\"/www/static\"));\n///\n/// // For example:\n/// // - `GET /static/app.js` would serve the file `/www/static/app.js`\n/// // - `GET /static/css/app.css` would serve the file `/www/static/css/app.css`\n/// ```\npub fn dir(path: impl Into<PathBuf>) -> impl FilterClone<Extract = One<File>, Error = Rejection> {\n    let base = Arc::new(path.into());\n    crate::get()\n        .or(crate::head())\n        .unify()\n        .and(path_from_tail(base))\n        .and(conditionals())\n        .and_then(file_reply)\n}\n\nfn path_from_tail(\n    base: Arc<PathBuf>,\n) -> impl FilterClone<Extract = One<ArcPath>, Error = Rejection> {\n    crate::path::tail().and_then(move |tail: crate::path::Tail| {\n        future::ready(sanitize_path(base.as_ref(), tail.as_str())).and_then(|mut buf| async {\n            let is_dir = tokio::fs::metadata(buf.clone())\n                .await\n                .map(|m| m.is_dir())\n                .unwrap_or(false);\n\n            if is_dir {\n                tracing::debug!(\"dir: appending index.html to directory path\");\n                buf.push(\"index.html\");\n            }\n            tracing::trace!(\"dir: {:?}\", buf);\n            Ok(ArcPath(Arc::new(buf)))\n        })\n    })\n}\n\nfn sanitize_path(base: impl AsRef<Path>, tail: &str) -> Result<PathBuf, Rejection> {\n    let mut buf = PathBuf::from(base.as_ref());\n    let p = match percent_decode_str(tail).decode_utf8() {\n        Ok(p) => p,\n        Err(err) => {\n            tracing::debug!(\"dir: failed to decode route={:?}: {:?}\", tail, err);\n            return Err(reject::not_found());\n        }\n    };\n    tracing::trace!(\"dir? base={:?}, route={:?}\", base.as_ref(), p);\n    for seg in p.split('/') {\n        if seg.starts_with(\"..\") {\n            tracing::warn!(\"dir: rejecting segment starting with '..'\");\n            return Err(reject::not_found());\n        } else if seg.contains('\\\\') {\n            tracing::warn!(\"dir: rejecting segment containing backslash (\\\\)\");\n            return Err(reject::not_found());\n        } else if cfg!(windows) && seg.contains(':') {\n            tracing::warn!(\"dir: rejecting segment containing colon (:)\");\n            return Err(reject::not_found());\n        } else {\n            buf.push(seg);\n        }\n    }\n    Ok(buf)\n}\n\n#[derive(Debug)]\nstruct Conditionals {\n    if_modified_since: Option<IfModifiedSince>,\n    if_unmodified_since: Option<IfUnmodifiedSince>,\n    if_range: Option<IfRange>,\n    range: Option<Range>,\n}\n\nenum Cond {\n    NoBody(Response),\n    WithBody(Option<Range>),\n}\n\nimpl Conditionals {\n    fn check(self, last_modified: Option<LastModified>) -> Cond {\n        if let Some(since) = self.if_unmodified_since {\n            let precondition = last_modified\n                .map(|time| since.precondition_passes(time.into()))\n                .unwrap_or(false);\n\n            tracing::trace!(\n                \"if-unmodified-since? {:?} vs {:?} = {}\",\n                since,\n                last_modified,\n                precondition\n            );\n            if !precondition {\n                let mut res = Response::new(Body::empty());\n                *res.status_mut() = StatusCode::PRECONDITION_FAILED;\n                return Cond::NoBody(res);\n            }\n        }\n\n        if let Some(since) = self.if_modified_since {\n            tracing::trace!(\n                \"if-modified-since? header = {:?}, file = {:?}\",\n                since,\n                last_modified\n            );\n            let unmodified = last_modified\n                .map(|time| !since.is_modified(time.into()))\n                // no last_modified means its always modified\n                .unwrap_or(false);\n            if unmodified {\n                let mut res = Response::new(Body::empty());\n                *res.status_mut() = StatusCode::NOT_MODIFIED;\n                return Cond::NoBody(res);\n            }\n        }\n\n        if let Some(if_range) = self.if_range {\n            tracing::trace!(\"if-range? {:?} vs {:?}\", if_range, last_modified);\n            let can_range = !if_range.is_modified(None, last_modified.as_ref());\n\n            if !can_range {\n                return Cond::WithBody(None);\n            }\n        }\n\n        Cond::WithBody(self.range)\n    }\n}\n\nfn conditionals() -> impl Filter<Extract = One<Conditionals>, Error = Infallible> + Copy {\n    crate::header::optional2()\n        .and(crate::header::optional2())\n        .and(crate::header::optional2())\n        .and(crate::header::optional2())\n        .map(\n            |if_modified_since, if_unmodified_since, if_range, range| Conditionals {\n                if_modified_since,\n                if_unmodified_since,\n                if_range,\n                range,\n            },\n        )\n}\n\n/// A file response.\n#[derive(Debug)]\npub struct File {\n    resp: Response,\n    path: ArcPath,\n}\n\nimpl File {\n    /// Extract the `&Path` of the file this `Response` delivers.\n    ///\n    /// # Example\n    ///\n    /// The example below changes the Content-Type response header for every file called `video.mp4`.\n    ///\n    /// ```\n    /// use warp::{Filter, reply::Reply};\n    ///\n    /// let route = warp::path(\"static\")\n    ///     .and(warp::fs::dir(\"/www/static\"))\n    ///     .map(|reply: warp::filters::fs::File| {\n    ///         if reply.path().ends_with(\"video.mp4\") {\n    ///             warp::reply::with_header(reply, \"Content-Type\", \"video/mp4\").into_response()\n    ///         } else {\n    ///             reply.into_response()\n    ///         }\n    ///     });\n    /// ```\n    pub fn path(&self) -> &Path {\n        self.path.as_ref()\n    }\n}\n\n// Silly wrapper since Arc<PathBuf> doesn't implement AsRef<Path> ;_;\n#[derive(Clone, Debug)]\nstruct ArcPath(Arc<PathBuf>);\n\nimpl AsRef<Path> for ArcPath {\n    fn as_ref(&self) -> &Path {\n        (*self.0).as_ref()\n    }\n}\n\nimpl Reply for File {\n    fn into_response(self) -> Response {\n        self.resp\n    }\n}\n\nfn file_reply(\n    path: ArcPath,\n    conditionals: Conditionals,\n) -> impl Future<Output = Result<File, Rejection>> + Send {\n    TkFile::open(path.clone()).then(move |res| match res {\n        Ok(f) => Either::Left(file_conditional(f, path, conditionals)),\n        Err(err) => {\n            let rej = match err.kind() {\n                io::ErrorKind::NotFound => {\n                    tracing::debug!(\"file not found: {:?}\", path.as_ref().display());\n                    reject::not_found()\n                }\n                io::ErrorKind::PermissionDenied => {\n                    tracing::warn!(\"file permission denied: {:?}\", path.as_ref().display());\n                    reject::known(FilePermissionError { _p: () })\n                }\n                _ => {\n                    tracing::error!(\n                        \"file open error (path={:?}): {} \",\n                        path.as_ref().display(),\n                        err\n                    );\n                    reject::known(FileOpenError { _p: () })\n                }\n            };\n            Either::Right(future::err(rej))\n        }\n    })\n}\n\nasync fn file_metadata(f: TkFile) -> Result<(TkFile, Metadata), Rejection> {\n    match f.metadata().await {\n        Ok(meta) => Ok((f, meta)),\n        Err(err) => {\n            tracing::debug!(\"file metadata error: {}\", err);\n            Err(reject::not_found())\n        }\n    }\n}\n\nfn file_conditional(\n    f: TkFile,\n    path: ArcPath,\n    conditionals: Conditionals,\n) -> impl Future<Output = Result<File, Rejection>> + Send {\n    file_metadata(f).map_ok(move |(file, meta)| {\n        let mut len = meta.len();\n        let modified = meta.modified().ok().map(LastModified::from);\n\n        let resp = match conditionals.check(modified) {\n            Cond::NoBody(resp) => resp,\n            Cond::WithBody(range) => {\n                bytes_range(range, len)\n                    .map(|(start, end)| {\n                        let sub_len = end - start;\n                        let buf_size = optimal_buf_size(&meta);\n                        let stream = file_stream(file, buf_size, (start, end));\n                        let body = Body::wrap_stream(stream);\n\n                        let mut resp = Response::new(body);\n\n                        if sub_len != len {\n                            *resp.status_mut() = StatusCode::PARTIAL_CONTENT;\n                            resp.headers_mut().typed_insert(\n                                ContentRange::bytes(start..end, len).expect(\"valid ContentRange\"),\n                            );\n\n                            len = sub_len;\n                        }\n\n                        let mime = mime_guess::from_path(path.as_ref()).first_or_octet_stream();\n\n                        resp.headers_mut().typed_insert(ContentLength(len));\n                        resp.headers_mut().typed_insert(ContentType::from(mime));\n                        resp.headers_mut().typed_insert(AcceptRanges::bytes());\n\n                        if let Some(last_modified) = modified {\n                            resp.headers_mut().typed_insert(last_modified);\n                        }\n\n                        resp\n                    })\n                    .unwrap_or_else(|BadRange| {\n                        // bad byte range\n                        let mut resp = Response::new(Body::empty());\n                        *resp.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE;\n                        resp.headers_mut()\n                            .typed_insert(ContentRange::unsatisfied_bytes(len));\n                        resp\n                    })\n            }\n        };\n\n        File { resp, path }\n    })\n}\n\nstruct BadRange;\n\nfn bytes_range(range: Option<Range>, max_len: u64) -> Result<(u64, u64), BadRange> {\n    use std::ops::Bound;\n\n    let range = if let Some(range) = range {\n        range\n    } else {\n        return Ok((0, max_len));\n    };\n\n    let ret = range\n        .iter()\n        .map(|(start, end)| {\n            let start = match start {\n                Bound::Unbounded => 0,\n                Bound::Included(s) => s,\n                Bound::Excluded(s) => s + 1,\n            };\n\n            let end = match end {\n                Bound::Unbounded => max_len,\n                Bound::Included(s) => {\n                    // For the special case where s == the file size\n                    if s == max_len {\n                        s\n                    } else {\n                        s + 1\n                    }\n                }\n                Bound::Excluded(s) => s,\n            };\n\n            if start < end && end <= max_len {\n                Ok((start, end))\n            } else {\n                tracing::trace!(\"unsatisfiable byte range: {}-{}/{}\", start, end, max_len);\n                Err(BadRange)\n            }\n        })\n        .next()\n        .unwrap_or(Ok((0, max_len)));\n    ret\n}\n\nfn file_stream(\n    mut file: TkFile,\n    buf_size: usize,\n    (start, end): (u64, u64),\n) -> impl Stream<Item = Result<Bytes, io::Error>> + Send {\n    use std::io::SeekFrom;\n\n    let seek = async move {\n        if start != 0 {\n            file.seek(SeekFrom::Start(start)).await?;\n        }\n        Ok(file)\n    };\n\n    seek.into_stream()\n        .map(move |result| {\n            let mut buf = BytesMut::new();\n            let mut len = end - start;\n            let mut f = match result {\n                Ok(f) => f,\n                Err(f) => return Either::Left(stream::once(future::err(f))),\n            };\n\n            Either::Right(stream::poll_fn(move |cx| {\n                if len == 0 {\n                    return Poll::Ready(None);\n                }\n                reserve_at_least(&mut buf, buf_size);\n\n                let n = match ready!(poll_read_buf(Pin::new(&mut f), cx, &mut buf)) {\n                    Ok(n) => n as u64,\n                    Err(err) => {\n                        tracing::debug!(\"file read error: {}\", err);\n                        return Poll::Ready(Some(Err(err)));\n                    }\n                };\n\n                if n == 0 {\n                    tracing::debug!(\"file read found EOF before expected length\");\n                    return Poll::Ready(None);\n                }\n\n                let mut chunk = buf.split().freeze();\n                if n > len {\n                    chunk = chunk.split_to(len as usize);\n                    len = 0;\n                } else {\n                    len -= n;\n                }\n\n                Poll::Ready(Some(Ok(chunk)))\n            }))\n        })\n        .flatten()\n}\n\nfn reserve_at_least(buf: &mut BytesMut, cap: usize) {\n    if buf.capacity() - buf.len() < cap {\n        buf.reserve(cap);\n    }\n}\n\nconst DEFAULT_READ_BUF_SIZE: usize = 8_192;\n\nfn optimal_buf_size(metadata: &Metadata) -> usize {\n    let block_size = get_block_size(metadata);\n\n    // If file length is smaller than block size, don't waste space\n    // reserving a bigger-than-needed buffer.\n    cmp::min(block_size as u64, metadata.len()) as usize\n}\n\n#[cfg(unix)]\nfn get_block_size(metadata: &Metadata) -> usize {\n    use std::os::unix::fs::MetadataExt;\n    //TODO: blksize() returns u64, should handle bad cast...\n    //(really, a block size bigger than 4gb?)\n\n    // Use device blocksize unless it's really small.\n    cmp::max(metadata.blksize() as usize, DEFAULT_READ_BUF_SIZE)\n}\n\n#[cfg(not(unix))]\nfn get_block_size(_metadata: &Metadata) -> usize {\n    DEFAULT_READ_BUF_SIZE\n}\n\n// ===== Rejections =====\n\nunit_error! {\n    pub(crate) FileOpenError: \"file open error\"\n}\n\nunit_error! {\n    pub(crate) FilePermissionError: \"file perimission error\"\n}\n\n#[cfg(test)]\nmod tests {\n    use super::sanitize_path;\n    use bytes::BytesMut;\n\n    #[test]\n    fn test_sanitize_path() {\n        let base = \"/var/www\";\n\n        fn p(s: &str) -> &::std::path::Path {\n            s.as_ref()\n        }\n\n        assert_eq!(\n            sanitize_path(base, \"/foo.html\").unwrap(),\n            p(\"/var/www/foo.html\")\n        );\n\n        // bad paths\n        sanitize_path(base, \"/../foo.html\").expect_err(\"dot dot\");\n\n        sanitize_path(base, \"/C:\\\\/foo.html\").expect_err(\"C:\\\\\");\n    }\n\n    #[test]\n    fn test_reserve_at_least() {\n        let mut buf = BytesMut::new();\n        let cap = 8_192;\n\n        assert_eq!(buf.len(), 0);\n        assert_eq!(buf.capacity(), 0);\n\n        super::reserve_at_least(&mut buf, cap);\n        assert_eq!(buf.len(), 0);\n        assert_eq!(buf.capacity(), cap);\n    }\n}\n\n// Path: src/filter/wrap.rs\nuse super::Filter;\n\npub trait WrapSealed<F: Filter> {\n    type Wrapped: Filter;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped;\n}\n\nimpl<'a, T, F> WrapSealed<F> for &'a T\nwhere\n    T: WrapSealed<F>,\n    F: Filter,\n{\n    type Wrapped = T::Wrapped;\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        (*self).wrap(filter)\n    }\n}\n\npub trait Wrap<F: Filter>: WrapSealed<F> {}\n\nimpl<T, F> Wrap<F> for T\nwhere\n    T: WrapSealed<F>,\n    F: Filter,\n{\n}\n\n/// Combines received filter with pre and after filters\n///\n/// # Example\n///\n/// ```\n/// use crate::warp::Filter;\n///\n/// let route = warp::any()\n///     .map(|| \"hello world\")\n///     .with(warp::wrap_fn(|filter| filter));\n/// ```\n///\n/// You can find the full example in the [usage example](https://github.com/seanmonstar/warp/blob/master/examples/wrapping.rs).\n\npub fn wrap_fn<F, T, U>(func: F) -> WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    WrapFn { func }\n}\n\n#[derive(Debug)]\npub struct WrapFn<F> {\n    func: F,\n}\n\nimpl<F, T, U> WrapSealed<T> for WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    type Wrapped = U;\n\n    fn wrap(&self, filter: T) -> Self::Wrapped {\n        (self.func)(filter)\n    }\n}\n\n// Path: src/filters/cors.rs\n//! CORS Filters\n\nuse std::collections::HashSet;\nuse std::convert::TryFrom;\nuse std::error::Error as StdError;\nuse std::fmt;\nuse std::sync::Arc;\n\nuse headers::{\n    AccessControlAllowHeaders, AccessControlAllowMethods, AccessControlExposeHeaders, HeaderMapExt,\n};\nuse http::header::{self, HeaderName, HeaderValue};\n\nuse crate::filter::{Filter, WrapSealed};\nuse crate::reject::{CombineRejection, Rejection};\nuse crate::reply::Reply;\n\nuse self::internal::{CorsFilter, IntoOrigin, Seconds};\n\n/// Create a wrapping [`Filter`](crate::Filter) that exposes [CORS][] behavior for a wrapped\n/// filter.\n///\n/// [CORS]: https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let cors = warp::cors()\n///     .allow_origin(\"https://hyper.rs\")\n///     .allow_methods(vec![\"GET\", \"POST\", \"DELETE\"]);\n///\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(cors);\n/// ```\n/// If you want to allow any route:\n/// ```\n/// use warp::Filter;\n/// let cors = warp::cors()\n///     .allow_any_origin();\n/// ```\n/// You can find more usage examples [here](https://github.com/seanmonstar/warp/blob/7fa54eaecd0fe12687137372791ff22fc7995766/tests/cors.rs).\npub fn cors() -> Builder {\n    Builder {\n        credentials: false,\n        allowed_headers: HashSet::new(),\n        exposed_headers: HashSet::new(),\n        max_age: None,\n        methods: HashSet::new(),\n        origins: None,\n    }\n}\n\n/// A wrapping [`Filter`](crate::Filter) constructed via `warp::cors()`.\n#[derive(Clone, Debug)]\npub struct Cors {\n    config: Arc<Configured>,\n}\n\n/// A constructed via `warp::cors()`.\n#[derive(Clone, Debug)]\npub struct Builder {\n    credentials: bool,\n    allowed_headers: HashSet<HeaderName>,\n    exposed_headers: HashSet<HeaderName>,\n    max_age: Option<u64>,\n    methods: HashSet<http::Method>,\n    origins: Option<HashSet<HeaderValue>>,\n}\n\nimpl Builder {\n    /// Sets whether to add the `Access-Control-Allow-Credentials` header.\n    pub fn allow_credentials(mut self, allow: bool) -> Self {\n        self.credentials = allow;\n        self\n    }\n\n    /// Adds a method to the existing list of allowed request methods.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::Method`.\n    pub fn allow_method<M>(mut self, method: M) -> Self\n    where\n        http::Method: TryFrom<M>,\n    {\n        let method = match TryFrom::try_from(method) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Method\"),\n        };\n        self.methods.insert(method);\n        self\n    }\n\n    /// Adds multiple methods to the existing list of allowed request methods.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::Method`.\n    pub fn allow_methods<I>(mut self, methods: I) -> Self\n    where\n        I: IntoIterator,\n        http::Method: TryFrom<I::Item>,\n    {\n        let iter = methods.into_iter().map(|m| match TryFrom::try_from(m) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Method\"),\n        });\n        self.methods.extend(iter);\n        self\n    }\n\n    /// Adds a header to the list of allowed request headers.\n    ///\n    /// **Note**: These should match the values the browser sends via `Access-Control-Request-Headers`, e.g. `content-type`.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::header::HeaderName`.\n    pub fn allow_header<H>(mut self, header: H) -> Self\n    where\n        HeaderName: TryFrom<H>,\n    {\n        let header = match TryFrom::try_from(header) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Header\"),\n        };\n        self.allowed_headers.insert(header);\n        self\n    }\n\n    /// Adds multiple headers to the list of allowed request headers.\n    ///\n    /// **Note**: These should match the values the browser sends via `Access-Control-Request-Headers`, e.g.`content-type`.\n    ///\n    /// # Panics\n    ///\n    /// Panics if any of the headers are not a valid `http::header::HeaderName`.\n    pub fn allow_headers<I>(mut self, headers: I) -> Self\n    where\n        I: IntoIterator,\n        HeaderName: TryFrom<I::Item>,\n    {\n        let iter = headers.into_iter().map(|h| match TryFrom::try_from(h) {\n            Ok(h) => h,\n            Err(_) => panic!(\"illegal Header\"),\n        });\n        self.allowed_headers.extend(iter);\n        self\n    }\n\n    /// Adds a header to the list of exposed headers.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::header::HeaderName`.\n    pub fn expose_header<H>(mut self, header: H) -> Self\n    where\n        HeaderName: TryFrom<H>,\n    {\n        let header = match TryFrom::try_from(header) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Header\"),\n        };\n        self.exposed_headers.insert(header);\n        self\n    }\n\n    /// Adds multiple headers to the list of exposed headers.\n    ///\n    /// # Panics\n    ///\n    /// Panics if any of the headers are not a valid `http::header::HeaderName`.\n    pub fn expose_headers<I>(mut self, headers: I) -> Self\n    where\n        I: IntoIterator,\n        HeaderName: TryFrom<I::Item>,\n    {\n        let iter = headers.into_iter().map(|h| match TryFrom::try_from(h) {\n            Ok(h) => h,\n            Err(_) => panic!(\"illegal Header\"),\n        });\n        self.exposed_headers.extend(iter);\n        self\n    }\n\n    /// Sets that *any* `Origin` header is allowed.\n    ///\n    /// # Warning\n    ///\n    /// This can allow websites you didn't intend to access this resource,\n    /// it is usually better to set an explicit list.\n    pub fn allow_any_origin(mut self) -> Self {\n        self.origins = None;\n        self\n    }\n\n    /// Add an origin to the existing list of allowed `Origin`s.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `Origin`.\n    pub fn allow_origin(self, origin: impl IntoOrigin) -> Self {\n        self.allow_origins(Some(origin))\n    }\n\n    /// Add multiple origins to the existing list of allowed `Origin`s.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `Origin`.\n    pub fn allow_origins<I>(mut self, origins: I) -> Self\n    where\n        I: IntoIterator,\n        I::Item: IntoOrigin,\n    {\n        let iter = origins\n            .into_iter()\n            .map(IntoOrigin::into_origin)\n            .map(|origin| {\n                origin\n                    .to_string()\n                    .parse()\n                    .expect(\"Origin is always a valid HeaderValue\")\n            });\n\n        self.origins.get_or_insert_with(HashSet::new).extend(iter);\n\n        self\n    }\n\n    /// Sets the `Access-Control-Max-Age` header.\n    ///\n    /// # Example\n    ///\n    ///\n    /// ```\n    /// use std::time::Duration;\n    /// use warp::Filter;\n    ///\n    /// let cors = warp::cors()\n    ///     .max_age(30) // 30u32 seconds\n    ///     .max_age(Duration::from_secs(30)); // or a Duration\n    /// ```\n    pub fn max_age(mut self, seconds: impl Seconds) -> Self {\n        self.max_age = Some(seconds.seconds());\n        self\n    }\n\n    /// Builds the `Cors` wrapper from the configured settings.\n    ///\n    /// This step isn't *required*, as the `Builder` itself can be passed\n    /// to `Filter::with`. This just allows constructing once, thus not needing\n    /// to pay the cost of \"building\" every time.\n    pub fn build(self) -> Cors {\n        let expose_headers_header = if self.exposed_headers.is_empty() {\n            None\n        } else {\n            Some(self.exposed_headers.iter().cloned().collect())\n        };\n        let allowed_headers_header = self.allowed_headers.iter().cloned().collect();\n        let methods_header = self.methods.iter().cloned().collect();\n\n        let config = Arc::new(Configured {\n            cors: self,\n            allowed_headers_header,\n            expose_headers_header,\n            methods_header,\n        });\n\n        Cors { config }\n    }\n}\n\nimpl<F> WrapSealed<F> for Builder\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: CombineRejection<Rejection>,\n    <F::Error as CombineRejection<Rejection>>::One: CombineRejection<Rejection>,\n{\n    type Wrapped = CorsFilter<F>;\n\n    fn wrap(&self, inner: F) -> Self::Wrapped {\n        let Cors { config } = self.clone().build();\n\n        CorsFilter { config, inner }\n    }\n}\n\nimpl<F> WrapSealed<F> for Cors\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: CombineRejection<Rejection>,\n    <F::Error as CombineRejection<Rejection>>::One: CombineRejection<Rejection>,\n{\n    type Wrapped = CorsFilter<F>;\n\n    fn wrap(&self, inner: F) -> Self::Wrapped {\n        let config = self.config.clone();\n\n        CorsFilter { config, inner }\n    }\n}\n\n/// An error used to reject requests that are forbidden by a `cors` filter.\npub struct CorsForbidden {\n    kind: Forbidden,\n}\n\n#[derive(Debug)]\nenum Forbidden {\n    OriginNotAllowed,\n    MethodNotAllowed,\n    HeaderNotAllowed,\n}\n\nimpl fmt::Debug for CorsForbidden {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"CorsForbidden\").field(&self.kind).finish()\n    }\n}\n\nimpl fmt::Display for CorsForbidden {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let detail = match self.kind {\n            Forbidden::OriginNotAllowed => \"origin not allowed\",\n            Forbidden::MethodNotAllowed => \"request-method not allowed\",\n            Forbidden::HeaderNotAllowed => \"header not allowed\",\n        };\n        write!(f, \"CORS request forbidden: {}\", detail)\n    }\n}\n\nimpl StdError for CorsForbidden {}\n\n#[derive(Clone, Debug)]\nstruct Configured {\n    cors: Builder,\n    allowed_headers_header: AccessControlAllowHeaders,\n    expose_headers_header: Option<AccessControlExposeHeaders>,\n    methods_header: AccessControlAllowMethods,\n}\n\nenum Validated {\n    Preflight(HeaderValue),\n    Simple(HeaderValue),\n    NotCors,\n}\n\nimpl Configured {\n    fn check_request(\n        &self,\n        method: &http::Method,\n        headers: &http::HeaderMap,\n    ) -> Result<Validated, Forbidden> {\n        match (headers.get(header::ORIGIN), method) {\n            (Some(origin), &http::Method::OPTIONS) => {\n                // OPTIONS requests are preflight CORS requests...\n\n                if !self.is_origin_allowed(origin) {\n                    return Err(Forbidden::OriginNotAllowed);\n                }\n\n                if let Some(req_method) = headers.get(header::ACCESS_CONTROL_REQUEST_METHOD) {\n                    if !self.is_method_allowed(req_method) {\n                        return Err(Forbidden::MethodNotAllowed);\n                    }\n                } else {\n                    tracing::trace!(\n                        \"preflight request missing access-control-request-method header\"\n                    );\n                    return Err(Forbidden::MethodNotAllowed);\n                }\n\n                if let Some(req_headers) = headers.get(header::ACCESS_CONTROL_REQUEST_HEADERS) {\n                    let headers = req_headers\n                        .to_str()\n                        .map_err(|_| Forbidden::HeaderNotAllowed)?;\n                    for header in headers.split(',') {\n                        if !self.is_header_allowed(header.trim()) {\n                            return Err(Forbidden::HeaderNotAllowed);\n                        }\n                    }\n                }\n\n                Ok(Validated::Preflight(origin.clone()))\n            }\n            (Some(origin), _) => {\n                // Any other method, simply check for a valid origin...\n\n                tracing::trace!(\"origin header: {:?}\", origin);\n                if self.is_origin_allowed(origin) {\n                    Ok(Validated::Simple(origin.clone()))\n                } else {\n                    Err(Forbidden::OriginNotAllowed)\n                }\n            }\n            (None, _) => {\n                // No `ORIGIN` header means this isn't CORS!\n                Ok(Validated::NotCors)\n            }\n        }\n    }\n\n    fn is_method_allowed(&self, header: &HeaderValue) -> bool {\n        http::Method::from_bytes(header.as_bytes())\n            .map(|method| self.cors.methods.contains(&method))\n            .unwrap_or(false)\n    }\n\n    fn is_header_allowed(&self, header: &str) -> bool {\n        HeaderName::from_bytes(header.as_bytes())\n            .map(|header| self.cors.allowed_headers.contains(&header))\n            .unwrap_or(false)\n    }\n\n    fn is_origin_allowed(&self, origin: &HeaderValue) -> bool {\n        if let Some(ref allowed) = self.cors.origins {\n            allowed.contains(origin)\n        } else {\n            true\n        }\n    }\n\n    fn append_preflight_headers(&self, headers: &mut http::HeaderMap) {\n        self.append_common_headers(headers);\n\n        headers.typed_insert(self.allowed_headers_header.clone());\n        headers.typed_insert(self.methods_header.clone());\n\n        if let Some(max_age) = self.cors.max_age {\n            headers.insert(header::ACCESS_CONTROL_MAX_AGE, max_age.into());\n        }\n    }\n\n    fn append_common_headers(&self, headers: &mut http::HeaderMap) {\n        if self.cors.credentials {\n            headers.insert(\n                header::ACCESS_CONTROL_ALLOW_CREDENTIALS,\n                HeaderValue::from_static(\"true\"),\n            );\n        }\n        if let Some(expose_headers_header) = &self.expose_headers_header {\n            headers.typed_insert(expose_headers_header.clone())\n        }\n    }\n}\n\nmod internal {\n    use std::future::Future;\n    use std::pin::Pin;\n    use std::sync::Arc;\n    use std::task::{Context, Poll};\n\n    use futures_util::{future, ready, TryFuture};\n    use headers::Origin;\n    use http::header;\n    use pin_project::pin_project;\n\n    use super::{Configured, CorsForbidden, Validated};\n    use crate::filter::{Filter, FilterBase, Internal, One};\n    use crate::generic::Either;\n    use crate::reject::{CombineRejection, Rejection};\n    use crate::route;\n\n    #[derive(Clone, Debug)]\n    pub struct CorsFilter<F> {\n        pub(super) config: Arc<Configured>,\n        pub(super) inner: F,\n    }\n\n    impl<F> FilterBase for CorsFilter<F>\n    where\n        F: Filter,\n        F::Extract: Send,\n        F::Future: Future,\n        F::Error: CombineRejection<Rejection>,\n    {\n        type Extract =\n            One<Either<One<Preflight>, One<Either<One<Wrapped<F::Extract>>, F::Extract>>>>;\n        type Error = <F::Error as CombineRejection<Rejection>>::One;\n        type Future = future::Either<\n            future::Ready<Result<Self::Extract, Self::Error>>,\n            WrappedFuture<F::Future>,\n        >;\n\n        fn filter(&self, _: Internal) -> Self::Future {\n            let validated =\n                route::with(|route| self.config.check_request(route.method(), route.headers()));\n\n            match validated {\n                Ok(Validated::Preflight(origin)) => {\n                    let preflight = Preflight {\n                        config: self.config.clone(),\n                        origin,\n                    };\n                    future::Either::Left(future::ok((Either::A((preflight,)),)))\n                }\n                Ok(Validated::Simple(origin)) => future::Either::Right(WrappedFuture {\n                    inner: self.inner.filter(Internal),\n                    wrapped: Some((self.config.clone(), origin)),\n                }),\n                Ok(Validated::NotCors) => future::Either::Right(WrappedFuture {\n                    inner: self.inner.filter(Internal),\n                    wrapped: None,\n                }),\n                Err(err) => {\n                    let rejection = crate::reject::known(CorsForbidden { kind: err });\n                    future::Either::Left(future::err(rejection.into()))\n                }\n            }\n        }\n    }\n\n    #[derive(Debug)]\n    pub struct Preflight {\n        config: Arc<Configured>,\n        origin: header::HeaderValue,\n    }\n\n    impl crate::reply::Reply for Preflight {\n        fn into_response(self) -> crate::reply::Response {\n            let mut res = crate::reply::Response::default();\n            self.config.append_preflight_headers(res.headers_mut());\n            res.headers_mut()\n                .insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, self.origin);\n            res\n        }\n    }\n\n    #[derive(Debug)]\n    pub struct Wrapped<R> {\n        config: Arc<Configured>,\n        inner: R,\n        origin: header::HeaderValue,\n    }\n\n    impl<R> crate::reply::Reply for Wrapped<R>\n    where\n        R: crate::reply::Reply,\n    {\n        fn into_response(self) -> crate::reply::Response {\n            let mut res = self.inner.into_response();\n            self.config.append_common_headers(res.headers_mut());\n            res.headers_mut()\n                .insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, self.origin);\n            res\n        }\n    }\n\n    #[pin_project]\n    #[derive(Debug)]\n    pub struct WrappedFuture<F> {\n        #[pin]\n        inner: F,\n        wrapped: Option<(Arc<Configured>, header::HeaderValue)>,\n    }\n\n    impl<F> Future for WrappedFuture<F>\n    where\n        F: TryFuture,\n        F::Error: CombineRejection<Rejection>,\n    {\n        type Output = Result<\n            One<Either<One<Preflight>, One<Either<One<Wrapped<F::Ok>>, F::Ok>>>>,\n            <F::Error as CombineRejection<Rejection>>::One,\n        >;\n\n        fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n            let pin = self.project();\n            match ready!(pin.inner.try_poll(cx)) {\n                Ok(inner) => {\n                    let item = if let Some((config, origin)) = pin.wrapped.take() {\n                        (Either::A((Wrapped {\n                            config,\n                            inner,\n                            origin,\n                        },)),)\n                    } else {\n                        (Either::B(inner),)\n                    };\n                    let item = (Either::B(item),);\n                    Poll::Ready(Ok(item))\n                }\n                Err(err) => Poll::Ready(Err(err.into())),\n            }\n        }\n    }\n\n    pub trait Seconds {\n        fn seconds(self) -> u64;\n    }\n\n    impl Seconds for u32 {\n        fn seconds(self) -> u64 {\n            self.into()\n        }\n    }\n\n    impl Seconds for ::std::time::Duration {\n        fn seconds(self) -> u64 {\n            self.as_secs()\n        }\n    }\n\n    pub trait IntoOrigin {\n        fn into_origin(self) -> Origin;\n    }\n\n    impl<'a> IntoOrigin for &'a str {\n        fn into_origin(self) -> Origin {\n            let mut parts = self.splitn(2, \"://\");\n            let scheme = parts.next().expect(\"missing scheme\");\n            let rest = parts.next().expect(\"missing scheme\");\n\n            Origin::try_from_parts(scheme, rest, None).expect(\"invalid Origin\")\n        }\n    }\n}\n\n// Path: src/filters/ext.rs\n//! Request Extensions\n\nuse std::convert::Infallible;\n\nuse futures_util::future;\n\nuse crate::filter::{filter_fn_one, Filter};\nuse crate::reject::{self, Rejection};\n\n/// Get a previously set extension of the current route.\n///\n/// If the extension doesn't exist, this rejects with a `MissingExtension`.\npub fn get<T: Clone + Send + Sync + 'static>(\n) -> impl Filter<Extract = (T,), Error = Rejection> + Copy {\n    filter_fn_one(|route| {\n        let route = route\n            .extensions()\n            .get::<T>()\n            .cloned()\n            .ok_or_else(|| reject::known(MissingExtension { _p: () }));\n        future::ready(route)\n    })\n}\n\n/// Get a previously set extension of the current route.\n///\n/// If the extension doesn't exist, it yields `None`.\npub fn optional<T: Clone + Send + Sync + 'static>(\n) -> impl Filter<Extract = (Option<T>,), Error = Infallible> + Copy {\n    filter_fn_one(|route| future::ok(route.extensions().get::<T>().cloned()))\n}\n\nunit_error! {\n    /// An error used to reject if `get` cannot find the extension.\n    pub MissingExtension: \"Missing request extension\"\n}\n\n// Path: src/reject.rs\n//! Rejections\n//!\n//! Part of the power of the [`Filter`](../trait.Filter.html) system is being able to\n//! reject a request from a filter chain. This allows for filters to be\n//! combined with `or`, so that if one side of the chain finds that a request\n//! doesn't fulfill its requirements, the other side can try to process\n//! the request.\n//!\n//! Many of the built-in [`filters`](../filters) will automatically reject\n//! the request with an appropriate rejection. However, you can also build\n//! new custom [`Filter`](../trait.Filter.html)s and still want other routes to be\n//! matchable in the case a predicate doesn't hold.\n//!\n//! As a request is processed by a Filter chain, the rejections are accumulated into\n//! a list contained by the [`Rejection`](struct.Rejection.html) type. Rejections from\n//! filters can be handled using [`Filter::recover`](../trait.Filter.html#method.recover).\n//! This is a convenient way to map rejections into a [`Reply`](../reply/trait.Reply.html).\n//!\n//! For a more complete example see the\n//! [Rejection Example](https://github.com/seanmonstar/warp/blob/master/examples/rejections.rs)\n//! from the repository.\n//!\n//! # Example\n//!\n//! ```\n//! use warp::{reply, Reply, Filter, reject, Rejection, http::StatusCode};\n//!\n//! #[derive(Debug)]\n//! struct InvalidParameter;\n//!\n//! impl reject::Reject for InvalidParameter {}\n//!\n//! // Custom rejection handler that maps rejections into responses.\n//! async fn handle_rejection(err: Rejection) -> Result<impl Reply, std::convert::Infallible> {\n//!     if err.is_not_found() {\n//!         Ok(reply::with_status(\"NOT_FOUND\", StatusCode::NOT_FOUND))\n//!     } else if let Some(e) = err.find::<InvalidParameter>() {\n//!         Ok(reply::with_status(\"BAD_REQUEST\", StatusCode::BAD_REQUEST))\n//!     } else {\n//!         eprintln!(\"unhandled rejection: {:?}\", err);\n//!         Ok(reply::with_status(\"INTERNAL_SERVER_ERROR\", StatusCode::INTERNAL_SERVER_ERROR))\n//!     }\n//! }\n//!\n//!\n//! // Filter on `/:id`, but reject with InvalidParameter if the `id` is `0`.\n//! // Recover from this rejection using a custom rejection handler.\n//! let route = warp::path::param()\n//!     .and_then(|id: u32| async move {\n//!         if id == 0 {\n//!             Err(warp::reject::custom(InvalidParameter))\n//!         } else {\n//!             Ok(\"id is valid\")\n//!         }\n//!     })\n//!     .recover(handle_rejection);\n//! ```\n\nuse std::any::Any;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::fmt;\n\nuse http::{\n    header::{HeaderValue, CONTENT_TYPE},\n    StatusCode,\n};\nuse hyper::Body;\n\npub(crate) use self::sealed::{CombineRejection, IsReject};\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn reject() -> Rejection {\n    not_found()\n}\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn not_found() -> Rejection {\n    Rejection {\n        reason: Reason::NotFound,\n    }\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_query() -> Rejection {\n    known(InvalidQuery { _p: () })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_header(name: &'static str) -> Rejection {\n    known(MissingHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_header(name: &'static str) -> Rejection {\n    known(InvalidHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_cookie(name: &'static str) -> Rejection {\n    known(MissingCookie { name })\n}\n\n// 405 Method Not Allowed\n#[inline]\npub(crate) fn method_not_allowed() -> Rejection {\n    known(MethodNotAllowed { _p: () })\n}\n\n// 411 Length Required\n#[inline]\npub(crate) fn length_required() -> Rejection {\n    known(LengthRequired { _p: () })\n}\n\n// 413 Payload Too Large\n#[inline]\npub(crate) fn payload_too_large() -> Rejection {\n    known(PayloadTooLarge { _p: () })\n}\n\n// 415 Unsupported Media Type\n//\n// Used by the body filters if the request payload content-type doesn't match\n// what can be deserialized.\n#[inline]\npub(crate) fn unsupported_media_type() -> Rejection {\n    known(UnsupportedMediaType { _p: () })\n}\n\n/// Rejects a request with a custom cause.\n///\n/// A [`recover`][] filter should convert this `Rejection` into a `Reply`,\n/// or else this will be returned as a `500 Internal Server Error`.\n///\n/// [`recover`]: ../trait.Filter.html#method.recover\npub fn custom<T: Reject>(err: T) -> Rejection {\n    Rejection::custom(Box::new(err))\n}\n\n/// Protect against re-rejecting a rejection.\n///\n/// ```compile_fail\n/// fn with(r: warp::Rejection) {\n///     let _wat = warp::reject::custom(r);\n/// }\n/// ```\nfn __reject_custom_compilefail() {}\n\n/// A marker trait to ensure proper types are used for custom rejections.\n///\n/// Can be converted into Rejection.\n///\n/// # Example\n///\n/// ```\n/// use warp::{Filter, reject::Reject};\n///\n/// #[derive(Debug)]\n/// struct RateLimited;\n///\n/// impl Reject for RateLimited {}\n///\n/// let route = warp::any().and_then(|| async {\n///     Err::<(), _>(warp::reject::custom(RateLimited))\n/// });\n/// ```\n// Require `Sized` for now to prevent passing a `Box<dyn Reject>`, since we\n// would be double-boxing it, and the downcasting wouldn't work as expected.\npub trait Reject: fmt::Debug + Sized + Send + Sync + 'static {}\n\ntrait Cause: fmt::Debug + Send + Sync + 'static {\n    fn as_any(&self) -> &dyn Any;\n}\n\nimpl<T> Cause for T\nwhere\n    T: fmt::Debug + Send + Sync + 'static,\n{\n    fn as_any(&self) -> &dyn Any {\n        self\n    }\n}\n\nimpl dyn Cause {\n    fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        self.as_any().downcast_ref::<T>()\n    }\n}\n\npub(crate) fn known<T: Into<Known>>(err: T) -> Rejection {\n    Rejection::known(err.into())\n}\n\n/// Rejection of a request by a [`Filter`](crate::Filter).\n///\n/// See the [`reject`](module@crate::reject) documentation for more.\npub struct Rejection {\n    reason: Reason,\n}\n\nenum Reason {\n    NotFound,\n    Other(Box<Rejections>),\n}\n\nenum Rejections {\n    Known(Known),\n    Custom(Box<dyn Cause>),\n    Combined(Box<Rejections>, Box<Rejections>),\n}\n\nmacro_rules! enum_known {\n     ($($(#[$attr:meta])* $var:ident($ty:path),)+) => (\n        pub(crate) enum Known {\n            $(\n            $(#[$attr])*\n            $var($ty),\n            )+\n        }\n\n        impl Known {\n            fn inner_as_any(&self) -> &dyn Any {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t,\n                    )+\n                }\n            }\n        }\n\n        impl fmt::Debug for Known {\n            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t.fmt(f),\n                    )+\n                }\n            }\n        }\n\n        impl fmt::Display for Known {\n            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t.fmt(f),\n                    )+\n                }\n            }\n        }\n\n        $(\n        #[doc(hidden)]\n        $(#[$attr])*\n        impl From<$ty> for Known {\n            fn from(ty: $ty) -> Known {\n                Known::$var(ty)\n            }\n        }\n        )+\n    );\n}\n\nenum_known! {\n    MethodNotAllowed(MethodNotAllowed),\n    InvalidHeader(InvalidHeader),\n    MissingHeader(MissingHeader),\n    MissingCookie(MissingCookie),\n    InvalidQuery(InvalidQuery),\n    LengthRequired(LengthRequired),\n    PayloadTooLarge(PayloadTooLarge),\n    UnsupportedMediaType(UnsupportedMediaType),\n    FileOpenError(crate::fs::FileOpenError),\n    FilePermissionError(crate::fs::FilePermissionError),\n    BodyReadError(crate::body::BodyReadError),\n    BodyDeserializeError(crate::body::BodyDeserializeError),\n    CorsForbidden(crate::cors::CorsForbidden),\n    #[cfg(feature = \"websocket\")]\n    MissingConnectionUpgrade(crate::ws::MissingConnectionUpgrade),\n    MissingExtension(crate::ext::MissingExtension),\n    BodyConsumedMultipleTimes(crate::body::BodyConsumedMultipleTimes),\n}\n\nimpl Rejection {\n    fn known(known: Known) -> Self {\n        Rejection {\n            reason: Reason::Other(Box::new(Rejections::Known(known))),\n        }\n    }\n\n    fn custom(other: Box<dyn Cause>) -> Self {\n        Rejection {\n            reason: Reason::Other(Box::new(Rejections::Custom(other))),\n        }\n    }\n\n    /// Searches this `Rejection` for a specific cause.\n    ///\n    /// A `Rejection` will accumulate causes over a `Filter` chain. This method\n    /// can search through them and return the first cause of this type.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// #[derive(Debug)]\n    /// struct Nope;\n    ///\n    /// impl warp::reject::Reject for Nope {}\n    ///\n    /// let reject = warp::reject::custom(Nope);\n    ///\n    /// if let Some(nope) = reject.find::<Nope>() {\n    ///    println!(\"found it: {:?}\", nope);\n    /// }\n    /// ```\n    pub fn find<T: 'static>(&self) -> Option<&T> {\n        if let Reason::Other(ref rejections) = self.reason {\n            return rejections.find();\n        }\n        None\n    }\n\n    /// Returns true if this Rejection was made via `warp::reject::not_found`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let rejection = warp::reject();\n    ///\n    /// assert!(rejection.is_not_found());\n    /// ```\n    pub fn is_not_found(&self) -> bool {\n        matches!(self.reason, Reason::NotFound)\n    }\n}\n\nimpl<T: Reject> From<T> for Rejection {\n    #[inline]\n    fn from(err: T) -> Rejection {\n        custom(err)\n    }\n}\n\nimpl From<Infallible> for Rejection {\n    #[inline]\n    fn from(infallible: Infallible) -> Rejection {\n        match infallible {}\n    }\n}\n\nimpl IsReject for Infallible {\n    fn status(&self) -> StatusCode {\n        match *self {}\n    }\n\n    fn into_response(&self) -> crate::reply::Response {\n        match *self {}\n    }\n}\n\nimpl IsReject for Rejection {\n    fn status(&self) -> StatusCode {\n        match self.reason {\n            Reason::NotFound => StatusCode::NOT_FOUND,\n            Reason::Other(ref other) => other.status(),\n        }\n    }\n\n    fn into_response(&self) -> crate::reply::Response {\n        match self.reason {\n            Reason::NotFound => {\n                let mut res = http::Response::default();\n                *res.status_mut() = StatusCode::NOT_FOUND;\n                res\n            }\n            Reason::Other(ref other) => other.into_response(),\n        }\n    }\n}\n\nimpl fmt::Debug for Rejection {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"Rejection\").field(&self.reason).finish()\n    }\n}\n\nimpl fmt::Debug for Reason {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match *self {\n            Reason::NotFound => f.write_str(\"NotFound\"),\n            Reason::Other(ref other) => match **other {\n                Rejections::Known(ref e) => fmt::Debug::fmt(e, f),\n                Rejections::Custom(ref e) => fmt::Debug::fmt(e, f),\n                Rejections::Combined(ref a, ref b) => {\n                    let mut list = f.debug_list();\n                    a.debug_list(&mut list);\n                    b.debug_list(&mut list);\n                    list.finish()\n                }\n            },\n        }\n    }\n}\n\n// ===== Rejections =====\n\nimpl Rejections {\n    fn status(&self) -> StatusCode {\n        match *self {\n            Rejections::Known(ref k) => match *k {\n                Known::MethodNotAllowed(_) => StatusCode::METHOD_NOT_ALLOWED,\n                Known::InvalidHeader(_)\n                | Known::MissingHeader(_)\n                | Known::MissingCookie(_)\n                | Known::InvalidQuery(_)\n                | Known::BodyReadError(_)\n                | Known::BodyDeserializeError(_) => StatusCode::BAD_REQUEST,\n                #[cfg(feature = \"websocket\")]\n                Known::MissingConnectionUpgrade(_) => StatusCode::BAD_REQUEST,\n                Known::LengthRequired(_) => StatusCode::LENGTH_REQUIRED,\n                Known::PayloadTooLarge(_) => StatusCode::PAYLOAD_TOO_LARGE,\n                Known::UnsupportedMediaType(_) => StatusCode::UNSUPPORTED_MEDIA_TYPE,\n                Known::FilePermissionError(_) | Known::CorsForbidden(_) => StatusCode::FORBIDDEN,\n                Known::FileOpenError(_)\n                | Known::MissingExtension(_)\n                | Known::BodyConsumedMultipleTimes(_) => StatusCode::INTERNAL_SERVER_ERROR,\n            },\n            Rejections::Custom(..) => StatusCode::INTERNAL_SERVER_ERROR,\n            Rejections::Combined(..) => self.preferred().status(),\n        }\n    }\n\n    fn into_response(&self) -> crate::reply::Response {\n        match *self {\n            Rejections::Known(ref e) => {\n                let mut res = http::Response::new(Body::from(e.to_string()));\n                *res.status_mut() = self.status();\n                res.headers_mut().insert(\n                    CONTENT_TYPE,\n                    HeaderValue::from_static(\"text/plain; charset=utf-8\"),\n                );\n                res\n            }\n            Rejections::Custom(ref e) => {\n                tracing::error!(\n                    \"unhandled custom rejection, returning 500 response: {:?}\",\n                    e\n                );\n                let body = format!(\"Unhandled rejection: {:?}\", e);\n                let mut res = http::Response::new(Body::from(body));\n                *res.status_mut() = self.status();\n                res.headers_mut().insert(\n                    CONTENT_TYPE,\n...\n", "needle_token_start": 5626, "needle_token_end": 5687, "code_context_ntokens": 16200, "output": ["```rust\n/// Combines received filter with pre and after filters\n///\n/// # Example\n///\n/// ```\n/// use crate::warp::Filter;\n///\n/// let route = warp::any()\n///     .map(|| \"hello world\")\n///     .with(warp::wrap_fn(|filter| filter));\n/// ```\n///\n/// You can find the full example in the [usage example](https://github.com/seanmonstar/warp/blob/master/examples/wrapping.rs).\n\npub fn wrap_fn<F, T, U>(func: F) -> WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    WrapFn { func }\n}\n\n#[derive(Debug)]\npub struct WrapFn<F> {\n    func: F,\n}\n\nimpl<F, T, U> WrapSealed<T> for WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    type Wrapped = U;\n\n    fn wrap(&self, filter: T) -> Self::Wrapped {\n        (self.func)(filter)\n    }\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "file_stream", "language": "rust", "path": "src/filters/fs.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to asynchronously stream a portion of a file from a specified start to an end position, handling file seeking and efficient data buffering.\n2. **Input**: It accepts a file handle, a buffer size for data chunks, and a tuple indicating the start and end byte positions within the file.\n3. **Output**: It produces a stream of byte chunks wrapped in a result type, which can either contain the data as bytes or an error if an issue occurs during file reading.\n4. **Procedure**: \n   - Initially, the function seeks to the specified start position in the file.\n   - It then reads chunks of the file asynchronously into a buffer until it reaches the specified end position or encounters an error.\n   - The data is continuously yielded as byte chunks until the end of the specified range is reached or the file ends, whichever comes first.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/filters/path.rs\n//! Path Filters\n//!\n//! The [`Filter`](crate::Filter)s here work on the \"path\" of requests.\n//!\n//! - [`path`](./fn.path.html) matches a specific segment, like `/foo`.\n//! - [`param`](./fn.param.html) tries to parse a segment into a type, like `/:u16`.\n//! - [`end`](./fn.end.html) matches when the path end is found.\n//! - [`path!`](../../macro.path.html) eases combining multiple `path` and `param` filters.\n//!\n//! # Routing\n//!\n//! Routing in warp is simple yet powerful.\n//!\n//! First up, matching a single segment:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! // GET /hi\n//! let hi = warp::path(\"hi\").map(|| {\n//!     \"Hello, World!\"\n//! });\n//! ```\n//!\n//! How about multiple segments? It's easiest with the `path!` macro:\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /hello/from/warp\n//! let hello_from_warp = warp::path!(\"hello\" / \"from\" / \"warp\").map(|| {\n//!     \"Hello from warp!\"\n//! });\n//! ```\n//!\n//! Neat! But how do I handle **parameters** in paths?\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /sum/:u32/:u32\n//! let sum = warp::path!(\"sum\" / u32 / u32).map(|a, b| {\n//!     format!(\"{} + {} = {}\", a, b, a + b)\n//! });\n//! ```\n//!\n//! In fact, any type that implements `FromStr` can be used, in any order:\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /:u16/times/:u16\n//! let times = warp::path!(u16 / \"times\" / u16).map(|a, b| {\n//!     format!(\"{} times {} = {}\", a, b, a * b)\n//! });\n//! ```\n//!\n//! Oh shoot, those math routes should be **mounted** at a different path,\n//! is that possible? Yep!\n//!\n//! ```\n//! # use warp::Filter;\n//! # let sum = warp::any().map(warp::reply);\n//! # let times = sum.clone();\n//! // GET /math/sum/:u32/:u32\n//! // GET /math/:u16/times/:u16\n//! let math = warp::path(\"math\");\n//! let math_sum = math.and(sum);\n//! let math_times = math.and(times);\n//! ```\n//!\n//! What! `and`? What's that do?\n//!\n//! It combines the filters in a sort of \"this and then that\" order. In fact,\n//! it's exactly what the `path!` macro has been doing internally.\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /bye/:string\n//! let bye = warp::path(\"bye\")\n//!     .and(warp::path::param())\n//!     .map(|name: String| {\n//!         format!(\"Good bye, {}!\", name)\n//!     });\n//! ```\n//!\n//! Ah, so, can filters do things besides `and`?\n//!\n//! Why, yes they can! They can also `or`! As you might expect, `or` creates a\n//! \"this or else that\" chain of filters. If the first doesn't succeed, then\n//! it tries the other.\n//!\n//! So, those `math` routes could have been **mounted** all as one, with `or`.\n//!\n//!\n//! ```\n//! # use warp::Filter;\n//! # let sum = warp::path(\"sum\");\n//! # let times = warp::path(\"times\");\n//! // GET /math/sum/:u32/:u32\n//! // GET /math/:u16/times/:u16\n//! let math = warp::path(\"math\")\n//!     .and(sum.or(times));\n//! ```\n//!\n//! It turns out, using `or` is how you combine everything together into a\n//! single API.\n//!\n//! ```\n//! # use warp::Filter;\n//! # let hi = warp::path(\"hi\");\n//! # let hello_from_warp = hi.clone();\n//! # let bye = hi.clone();\n//! # let math = hi.clone();\n//! // GET /hi\n//! // GET /hello/from/warp\n//! // GET /bye/:string\n//! // GET /math/sum/:u32/:u32\n//! // GET /math/:u16/times/:u16\n//! let routes = hi\n//!     .or(hello_from_warp)\n//!     .or(bye)\n//!     .or(math);\n//! ```\n//!\n//! Note that you will generally want path filters to come **before** other filters\n//! like `body` or `headers`. If a different type of filter comes first, a request\n//! with an invalid body for route `/right-path-wrong-body` may try matching against `/wrong-path`\n//! and return the error from `/wrong-path` instead of the correct body-related error.\n\nuse std::convert::Infallible;\nuse std::fmt;\nuse std::str::FromStr;\n\nuse futures_util::future;\nuse http::uri::PathAndQuery;\n\nuse self::internal::Opaque;\nuse crate::filter::{filter_fn, one, Filter, FilterBase, Internal, One, Tuple};\nuse crate::reject::{self, Rejection};\nuse crate::route::{self, Route};\n\n/// Create an exact match path segment [`Filter`](crate::Filter).\n///\n/// This will try to match exactly to the current request path segment.\n///\n/// # Note\n///\n/// - [`end()`](./fn.end.html) should be used to match the end of a path to avoid having\n///   filters for shorter paths like `/math` unintentionally match a longer\n///   path such as `/math/sum`\n/// - Path-related filters should generally come **before** other types of filters, such\n///   as those checking headers or body types. Including those other filters before\n///   the path checks may result in strange errors being returned because a given request\n///   does not match the parameters for a completely separate route.\n///\n/// # Panics\n///\n/// Exact path filters cannot be empty, or contain slashes.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Matches '/hello'\n/// let hello = warp::path(\"hello\")\n///     .map(|| \"Hello, World!\");\n/// ```\npub fn path<P>(p: P) -> Exact<Opaque<P>>\nwhere\n    P: AsRef<str>,\n{\n    let s = p.as_ref();\n    assert!(!s.is_empty(), \"exact path segments should not be empty\");\n    assert!(\n        !s.contains('/'),\n        \"exact path segments should not contain a slash: {:?}\",\n        s\n    );\n\n    Exact(Opaque(p))\n    /*\n    segment(move |seg| {\n        tracing::trace!(\"{:?}?: {:?}\", p, seg);\n        if seg == p {\n            Ok(())\n        } else {\n            Err(reject::not_found())\n        }\n    })\n    */\n}\n\n/// A [`Filter`](crate::Filter) matching an exact path segment.\n///\n/// Constructed from `path()` or `path!()`.\n#[allow(missing_debug_implementations)]\n#[derive(Clone, Copy)]\npub struct Exact<P>(P);\n\nimpl<P> FilterBase for Exact<P>\nwhere\n    P: AsRef<str>,\n{\n    type Extract = ();\n    type Error = Rejection;\n    type Future = future::Ready<Result<Self::Extract, Self::Error>>;\n\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        route::with(|route| {\n            let p = self.0.as_ref();\n            future::ready(with_segment(route, |seg| {\n                tracing::trace!(\"{:?}?: {:?}\", p, seg);\n\n                if seg == p {\n                    Ok(())\n                } else {\n                    Err(reject::not_found())\n                }\n            }))\n        })\n    }\n}\n\n/// Matches the end of a route.\n///\n/// Note that _not_ including `end()` may result in shorter paths like\n/// `/math` unintentionally matching `/math/sum`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Matches '/'\n/// let hello = warp::path::end()\n///     .map(|| \"Hello, World!\");\n/// ```\npub fn end() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    filter_fn(move |route| {\n        if route.path().is_empty() {\n            future::ok(())\n        } else {\n            future::err(reject::not_found())\n        }\n    })\n}\n\n/// Extract a parameter from a path segment.\n///\n/// This will try to parse a value from the current request path\n/// segment, and if successful, the value is returned as the `Filter`'s\n/// \"extracted\" value.\n///\n/// If the value could not be parsed, rejects with a `404 Not Found`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::path::param()\n///     .map(|id: u32| {\n///         format!(\"You asked for /{}\", id)\n///     });\n/// ```\npub fn param<T: FromStr + Send + 'static>(\n) -> impl Filter<Extract = One<T>, Error = Rejection> + Copy {\n    filter_segment(|seg| {\n        tracing::trace!(\"param?: {:?}\", seg);\n        if seg.is_empty() {\n            return Err(reject::not_found());\n        }\n        T::from_str(seg).map(one).map_err(|_| reject::not_found())\n    })\n}\n\n/// Extract the unmatched tail of the path.\n///\n/// This will return a `Tail`, which allows access to the rest of the path\n/// that previous filters have not already matched.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::path(\"foo\")\n///     .and(warp::path::tail())\n///     .map(|tail| {\n///         // GET /foo/bar/baz would return \"bar/baz\".\n///         format!(\"The tail after foo is {:?}\", tail)\n///     });\n/// ```\npub fn tail() -> impl Filter<Extract = One<Tail>, Error = Infallible> + Copy {\n    filter_fn(move |route| {\n        let path = path_and_query(route);\n        let idx = route.matched_path_index();\n\n        // Giving the user the full tail means we assume the full path\n        // has been matched now.\n        let end = path.path().len() - idx;\n        route.set_unmatched_path(end);\n\n        future::ok(one(Tail {\n            path,\n            start_index: idx,\n        }))\n    })\n}\n\n/// Represents the tail part of a request path, returned by the [`tail()`] filter.\npub struct Tail {\n    path: PathAndQuery,\n    start_index: usize,\n}\n\nimpl Tail {\n    /// Get the `&str` representation of the remaining path.\n    pub fn as_str(&self) -> &str {\n        &self.path.path()[self.start_index..]\n    }\n}\n\nimpl fmt::Debug for Tail {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Debug::fmt(self.as_str(), f)\n    }\n}\n\n/// Peek at the unmatched tail of the path, without affecting the matched path.\n///\n/// This will return a `Peek`, which allows access to the rest of the path\n/// that previous filters have not already matched. This differs from `tail`\n/// in that `peek` will **not** set the entire path as matched.\n///\n/// # Example\n///\n/// ```\n...\n// Path: src/filters/fs.rs\n//! File System Filters\n\nuse std::cmp;\nuse std::convert::Infallible;\nuse std::fs::Metadata;\nuse std::future::Future;\nuse std::io;\nuse std::path::{Path, PathBuf};\nuse std::pin::Pin;\nuse std::sync::Arc;\nuse std::task::Poll;\n\nuse bytes::{Bytes, BytesMut};\nuse futures_util::future::Either;\nuse futures_util::{future, ready, stream, FutureExt, Stream, StreamExt, TryFutureExt};\nuse headers::{\n    AcceptRanges, ContentLength, ContentRange, ContentType, HeaderMapExt, IfModifiedSince, IfRange,\n    IfUnmodifiedSince, LastModified, Range,\n};\nuse http::StatusCode;\nuse hyper::Body;\nuse percent_encoding::percent_decode_str;\nuse tokio::fs::File as TkFile;\nuse tokio::io::AsyncSeekExt;\nuse tokio_util::io::poll_read_buf;\n\nuse crate::filter::{Filter, FilterClone, One};\nuse crate::reject::{self, Rejection};\nuse crate::reply::{Reply, Response};\n\n/// Creates a `Filter` that serves a File at the `path`.\n///\n/// Does not filter out based on any information of the request. Always serves\n/// the file at the exact `path` provided. Thus, this can be used to serve a\n/// single file with `GET`s, but could also be used in combination with other\n/// filters, such as after validating in `POST` request, wanting to return a\n/// specific file as the body.\n///\n/// For serving a directory, see [dir].\n///\n/// # Example\n///\n/// ```\n/// // Always serves this file from the file system.\n/// let route = warp::fs::file(\"/www/static/app.js\");\n/// ```\npub fn file(path: impl Into<PathBuf>) -> impl FilterClone<Extract = One<File>, Error = Rejection> {\n    let path = Arc::new(path.into());\n    crate::any()\n        .map(move || {\n            tracing::trace!(\"file: {:?}\", path);\n            ArcPath(path.clone())\n        })\n        .and(conditionals())\n        .and_then(file_reply)\n}\n\n/// Creates a `Filter` that serves a directory at the base `path` joined\n/// by the request path.\n///\n/// This can be used to serve \"static files\" from a directory. By far the most\n/// common pattern of serving static files is for `GET` requests, so this\n/// filter automatically includes a `GET` check.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Matches requests that start with `/static`,\n/// // and then uses the rest of that path to lookup\n/// // and serve a file from `/www/static`.\n/// let route = warp::path(\"static\")\n///     .and(warp::fs::dir(\"/www/static\"));\n///\n/// // For example:\n/// // - `GET /static/app.js` would serve the file `/www/static/app.js`\n/// // - `GET /static/css/app.css` would serve the file `/www/static/css/app.css`\n/// ```\npub fn dir(path: impl Into<PathBuf>) -> impl FilterClone<Extract = One<File>, Error = Rejection> {\n    let base = Arc::new(path.into());\n    crate::get()\n        .or(crate::head())\n        .unify()\n        .and(path_from_tail(base))\n        .and(conditionals())\n        .and_then(file_reply)\n}\n\nfn path_from_tail(\n    base: Arc<PathBuf>,\n) -> impl FilterClone<Extract = One<ArcPath>, Error = Rejection> {\n    crate::path::tail().and_then(move |tail: crate::path::Tail| {\n        future::ready(sanitize_path(base.as_ref(), tail.as_str())).and_then(|mut buf| async {\n            let is_dir = tokio::fs::metadata(buf.clone())\n                .await\n                .map(|m| m.is_dir())\n                .unwrap_or(false);\n\n            if is_dir {\n                tracing::debug!(\"dir: appending index.html to directory path\");\n                buf.push(\"index.html\");\n            }\n            tracing::trace!(\"dir: {:?}\", buf);\n            Ok(ArcPath(Arc::new(buf)))\n        })\n    })\n}\n\nfn sanitize_path(base: impl AsRef<Path>, tail: &str) -> Result<PathBuf, Rejection> {\n    let mut buf = PathBuf::from(base.as_ref());\n    let p = match percent_decode_str(tail).decode_utf8() {\n        Ok(p) => p,\n        Err(err) => {\n            tracing::debug!(\"dir: failed to decode route={:?}: {:?}\", tail, err);\n            return Err(reject::not_found());\n        }\n    };\n    tracing::trace!(\"dir? base={:?}, route={:?}\", base.as_ref(), p);\n    for seg in p.split('/') {\n        if seg.starts_with(\"..\") {\n            tracing::warn!(\"dir: rejecting segment starting with '..'\");\n            return Err(reject::not_found());\n        } else if seg.contains('\\\\') {\n            tracing::warn!(\"dir: rejecting segment containing backslash (\\\\)\");\n            return Err(reject::not_found());\n        } else if cfg!(windows) && seg.contains(':') {\n            tracing::warn!(\"dir: rejecting segment containing colon (:)\");\n            return Err(reject::not_found());\n        } else {\n            buf.push(seg);\n        }\n    }\n    Ok(buf)\n}\n\n#[derive(Debug)]\nstruct Conditionals {\n    if_modified_since: Option<IfModifiedSince>,\n    if_unmodified_since: Option<IfUnmodifiedSince>,\n    if_range: Option<IfRange>,\n    range: Option<Range>,\n}\n\nenum Cond {\n    NoBody(Response),\n    WithBody(Option<Range>),\n}\n\nimpl Conditionals {\n    fn check(self, last_modified: Option<LastModified>) -> Cond {\n        if let Some(since) = self.if_unmodified_since {\n            let precondition = last_modified\n                .map(|time| since.precondition_passes(time.into()))\n                .unwrap_or(false);\n\n            tracing::trace!(\n                \"if-unmodified-since? {:?} vs {:?} = {}\",\n                since,\n                last_modified,\n                precondition\n            );\n            if !precondition {\n                let mut res = Response::new(Body::empty());\n                *res.status_mut() = StatusCode::PRECONDITION_FAILED;\n                return Cond::NoBody(res);\n            }\n        }\n\n        if let Some(since) = self.if_modified_since {\n            tracing::trace!(\n                \"if-modified-since? header = {:?}, file = {:?}\",\n                since,\n                last_modified\n            );\n            let unmodified = last_modified\n                .map(|time| !since.is_modified(time.into()))\n                // no last_modified means its always modified\n                .unwrap_or(false);\n            if unmodified {\n                let mut res = Response::new(Body::empty());\n                *res.status_mut() = StatusCode::NOT_MODIFIED;\n                return Cond::NoBody(res);\n            }\n        }\n\n        if let Some(if_range) = self.if_range {\n            tracing::trace!(\"if-range? {:?} vs {:?}\", if_range, last_modified);\n            let can_range = !if_range.is_modified(None, last_modified.as_ref());\n\n            if !can_range {\n                return Cond::WithBody(None);\n            }\n        }\n\n        Cond::WithBody(self.range)\n    }\n}\n\nfn conditionals() -> impl Filter<Extract = One<Conditionals>, Error = Infallible> + Copy {\n    crate::header::optional2()\n        .and(crate::header::optional2())\n        .and(crate::header::optional2())\n        .and(crate::header::optional2())\n        .map(\n            |if_modified_since, if_unmodified_since, if_range, range| Conditionals {\n                if_modified_since,\n                if_unmodified_since,\n                if_range,\n                range,\n            },\n        )\n}\n\n/// A file response.\n#[derive(Debug)]\npub struct File {\n    resp: Response,\n    path: ArcPath,\n}\n\nimpl File {\n    /// Extract the `&Path` of the file this `Response` delivers.\n    ///\n    /// # Example\n    ///\n    /// The example below changes the Content-Type response header for every file called `video.mp4`.\n    ///\n    /// ```\n    /// use warp::{Filter, reply::Reply};\n    ///\n    /// let route = warp::path(\"static\")\n    ///     .and(warp::fs::dir(\"/www/static\"))\n    ///     .map(|reply: warp::filters::fs::File| {\n    ///         if reply.path().ends_with(\"video.mp4\") {\n    ///             warp::reply::with_header(reply, \"Content-Type\", \"video/mp4\").into_response()\n    ///         } else {\n    ///             reply.into_response()\n    ///         }\n    ///     });\n    /// ```\n    pub fn path(&self) -> &Path {\n        self.path.as_ref()\n    }\n}\n\n// Silly wrapper since Arc<PathBuf> doesn't implement AsRef<Path> ;_;\n#[derive(Clone, Debug)]\nstruct ArcPath(Arc<PathBuf>);\n\nimpl AsRef<Path> for ArcPath {\n    fn as_ref(&self) -> &Path {\n        (*self.0).as_ref()\n    }\n}\n\nimpl Reply for File {\n    fn into_response(self) -> Response {\n        self.resp\n    }\n}\n\nfn file_reply(\n    path: ArcPath,\n    conditionals: Conditionals,\n) -> impl Future<Output = Result<File, Rejection>> + Send {\n    TkFile::open(path.clone()).then(move |res| match res {\n        Ok(f) => Either::Left(file_conditional(f, path, conditionals)),\n        Err(err) => {\n            let rej = match err.kind() {\n                io::ErrorKind::NotFound => {\n                    tracing::debug!(\"file not found: {:?}\", path.as_ref().display());\n                    reject::not_found()\n                }\n                io::ErrorKind::PermissionDenied => {\n                    tracing::warn!(\"file permission denied: {:?}\", path.as_ref().display());\n                    reject::known(FilePermissionError { _p: () })\n                }\n                _ => {\n                    tracing::error!(\n                        \"file open error (path={:?}): {} \",\n                        path.as_ref().display(),\n                        err\n                    );\n                    reject::known(FileOpenError { _p: () })\n                }\n            };\n            Either::Right(future::err(rej))\n        }\n    })\n}\n\nasync fn file_metadata(f: TkFile) -> Result<(TkFile, Metadata), Rejection> {\n    match f.metadata().await {\n        Ok(meta) => Ok((f, meta)),\n        Err(err) => {\n            tracing::debug!(\"file metadata error: {}\", err);\n            Err(reject::not_found())\n        }\n    }\n}\n\nfn file_conditional(\n    f: TkFile,\n    path: ArcPath,\n    conditionals: Conditionals,\n) -> impl Future<Output = Result<File, Rejection>> + Send {\n    file_metadata(f).map_ok(move |(file, meta)| {\n        let mut len = meta.len();\n        let modified = meta.modified().ok().map(LastModified::from);\n\n        let resp = match conditionals.check(modified) {\n            Cond::NoBody(resp) => resp,\n            Cond::WithBody(range) => {\n                bytes_range(range, len)\n                    .map(|(start, end)| {\n                        let sub_len = end - start;\n                        let buf_size = optimal_buf_size(&meta);\n                        let stream = file_stream(file, buf_size, (start, end));\n                        let body = Body::wrap_stream(stream);\n\n                        let mut resp = Response::new(body);\n\n                        if sub_len != len {\n                            *resp.status_mut() = StatusCode::PARTIAL_CONTENT;\n                            resp.headers_mut().typed_insert(\n                                ContentRange::bytes(start..end, len).expect(\"valid ContentRange\"),\n                            );\n\n                            len = sub_len;\n                        }\n\n                        let mime = mime_guess::from_path(path.as_ref()).first_or_octet_stream();\n\n                        resp.headers_mut().typed_insert(ContentLength(len));\n                        resp.headers_mut().typed_insert(ContentType::from(mime));\n                        resp.headers_mut().typed_insert(AcceptRanges::bytes());\n\n                        if let Some(last_modified) = modified {\n                            resp.headers_mut().typed_insert(last_modified);\n                        }\n\n                        resp\n                    })\n                    .unwrap_or_else(|BadRange| {\n                        // bad byte range\n                        let mut resp = Response::new(Body::empty());\n                        *resp.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE;\n                        resp.headers_mut()\n                            .typed_insert(ContentRange::unsatisfied_bytes(len));\n                        resp\n                    })\n            }\n        };\n\n        File { resp, path }\n    })\n}\n\nstruct BadRange;\n\nfn bytes_range(range: Option<Range>, max_len: u64) -> Result<(u64, u64), BadRange> {\n    use std::ops::Bound;\n\n    let range = if let Some(range) = range {\n        range\n    } else {\n        return Ok((0, max_len));\n    };\n\n    let ret = range\n        .iter()\n        .map(|(start, end)| {\n            let start = match start {\n                Bound::Unbounded => 0,\n                Bound::Included(s) => s,\n                Bound::Excluded(s) => s + 1,\n            };\n\n            let end = match end {\n                Bound::Unbounded => max_len,\n                Bound::Included(s) => {\n                    // For the special case where s == the file size\n                    if s == max_len {\n                        s\n                    } else {\n                        s + 1\n                    }\n                }\n                Bound::Excluded(s) => s,\n            };\n\n            if start < end && end <= max_len {\n                Ok((start, end))\n            } else {\n                tracing::trace!(\"unsatisfiable byte range: {}-{}/{}\", start, end, max_len);\n                Err(BadRange)\n            }\n        })\n        .next()\n        .unwrap_or(Ok((0, max_len)));\n    ret\n}\n\n\nfn file_stream(\n    mut file: TkFile,\n    buf_size: usize,\n    (start, end): (u64, u64),\n) -> impl Stream<Item = Result<Bytes, io::Error>> + Send {\n    use std::io::SeekFrom;\n\n    let seek = async move {\n        if start != 0 {\n            file.seek(SeekFrom::Start(start)).await?;\n        }\n        Ok(file)\n    };\n\n    seek.into_stream()\n        .map(move |result| {\n            let mut buf = BytesMut::new();\n            let mut len = end - start;\n            let mut f = match result {\n                Ok(f) => f,\n                Err(f) => return Either::Left(stream::once(future::err(f))),\n            };\n\n            Either::Right(stream::poll_fn(move |cx| {\n                if len == 0 {\n                    return Poll::Ready(None);\n                }\n                reserve_at_least(&mut buf, buf_size);\n\n                let n = match ready!(poll_read_buf(Pin::new(&mut f), cx, &mut buf)) {\n                    Ok(n) => n as u64,\n                    Err(err) => {\n                        tracing::debug!(\"file read error: {}\", err);\n                        return Poll::Ready(Some(Err(err)));\n                    }\n                };\n\n                if n == 0 {\n                    tracing::debug!(\"file read found EOF before expected length\");\n                    return Poll::Ready(None);\n                }\n\n                let mut chunk = buf.split().freeze();\n                if n > len {\n                    chunk = chunk.split_to(len as usize);\n                    len = 0;\n                } else {\n                    len -= n;\n                }\n\n                Poll::Ready(Some(Ok(chunk)))\n            }))\n        })\n        .flatten()\n}\n\nfn reserve_at_least(buf: &mut BytesMut, cap: usize) {\n    if buf.capacity() - buf.len() < cap {\n        buf.reserve(cap);\n    }\n}\n\nconst DEFAULT_READ_BUF_SIZE: usize = 8_192;\n\nfn optimal_buf_size(metadata: &Metadata) -> usize {\n    let block_size = get_block_size(metadata);\n\n    // If file length is smaller than block size, don't waste space\n    // reserving a bigger-than-needed buffer.\n    cmp::min(block_size as u64, metadata.len()) as usize\n}\n\n#[cfg(unix)]\nfn get_block_size(metadata: &Metadata) -> usize {\n    use std::os::unix::fs::MetadataExt;\n    //TODO: blksize() returns u64, should handle bad cast...\n    //(really, a block size bigger than 4gb?)\n\n    // Use device blocksize unless it's really small.\n    cmp::max(metadata.blksize() as usize, DEFAULT_READ_BUF_SIZE)\n}\n\n#[cfg(not(unix))]\nfn get_block_size(_metadata: &Metadata) -> usize {\n    DEFAULT_READ_BUF_SIZE\n}\n\n// ===== Rejections =====\n\nunit_error! {\n    pub(crate) FileOpenError: \"file open error\"\n}\n\nunit_error! {\n    pub(crate) FilePermissionError: \"file perimission error\"\n}\n\n#[cfg(test)]\nmod tests {\n    use super::sanitize_path;\n    use bytes::BytesMut;\n\n    #[test]\n    fn test_sanitize_path() {\n        let base = \"/var/www\";\n\n        fn p(s: &str) -> &::std::path::Path {\n            s.as_ref()\n        }\n\n        assert_eq!(\n            sanitize_path(base, \"/foo.html\").unwrap(),\n            p(\"/var/www/foo.html\")\n        );\n\n        // bad paths\n        sanitize_path(base, \"/../foo.html\").expect_err(\"dot dot\");\n\n        sanitize_path(base, \"/C:\\\\/foo.html\").expect_err(\"C:\\\\\");\n    }\n\n    #[test]\n    fn test_reserve_at_least() {\n        let mut buf = BytesMut::new();\n        let cap = 8_192;\n\n        assert_eq!(buf.len(), 0);\n        assert_eq!(buf.capacity(), 0);\n\n        super::reserve_at_least(&mut buf, cap);\n        assert_eq!(buf.len(), 0);\n        assert_eq!(buf.capacity(), cap);\n    }\n}\n\n// Path: src/filter/wrap.rs\nuse super::Filter;\n\npub trait WrapSealed<F: Filter> {\n    type Wrapped: Filter;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped;\n}\n\nimpl<'a, T, F> WrapSealed<F> for &'a T\nwhere\n    T: WrapSealed<F>,\n    F: Filter,\n{\n    type Wrapped = T::Wrapped;\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        (*self).wrap(filter)\n    }\n}\n\npub trait Wrap<F: Filter>: WrapSealed<F> {}\n\nimpl<T, F> Wrap<F> for T\nwhere\n    T: WrapSealed<F>,\n    F: Filter,\n{\n}\n\n/// Combines received filter with pre and after filters\n///\n/// # Example\n///\n/// ```\n/// use crate::warp::Filter;\n///\n/// let route = warp::any()\n///     .map(|| \"hello world\")\n///     .with(warp::wrap_fn(|filter| filter));\n/// ```\n///\n/// You can find the full example in the [usage example](https://github.com/seanmonstar/warp/blob/master/examples/wrapping.rs).\npub fn wrap_fn<F, T, U>(func: F) -> WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    WrapFn { func }\n}\n\n#[derive(Debug)]\npub struct WrapFn<F> {\n    func: F,\n}\n\nimpl<F, T, U> WrapSealed<T> for WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    type Wrapped = U;\n\n    fn wrap(&self, filter: T) -> Self::Wrapped {\n        (self.func)(filter)\n    }\n}\n\n// Path: src/filters/cors.rs\n//! CORS Filters\n\nuse std::collections::HashSet;\nuse std::convert::TryFrom;\nuse std::error::Error as StdError;\nuse std::fmt;\nuse std::sync::Arc;\n\nuse headers::{\n    AccessControlAllowHeaders, AccessControlAllowMethods, AccessControlExposeHeaders, HeaderMapExt,\n};\nuse http::header::{self, HeaderName, HeaderValue};\n\nuse crate::filter::{Filter, WrapSealed};\nuse crate::reject::{CombineRejection, Rejection};\nuse crate::reply::Reply;\n\nuse self::internal::{CorsFilter, IntoOrigin, Seconds};\n\n/// Create a wrapping [`Filter`](crate::Filter) that exposes [CORS][] behavior for a wrapped\n/// filter.\n///\n/// [CORS]: https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let cors = warp::cors()\n///     .allow_origin(\"https://hyper.rs\")\n///     .allow_methods(vec![\"GET\", \"POST\", \"DELETE\"]);\n///\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(cors);\n/// ```\n/// If you want to allow any route:\n/// ```\n/// use warp::Filter;\n/// let cors = warp::cors()\n///     .allow_any_origin();\n/// ```\n/// You can find more usage examples [here](https://github.com/seanmonstar/warp/blob/7fa54eaecd0fe12687137372791ff22fc7995766/tests/cors.rs).\npub fn cors() -> Builder {\n    Builder {\n        credentials: false,\n        allowed_headers: HashSet::new(),\n        exposed_headers: HashSet::new(),\n        max_age: None,\n        methods: HashSet::new(),\n        origins: None,\n    }\n}\n\n/// A wrapping [`Filter`](crate::Filter) constructed via `warp::cors()`.\n#[derive(Clone, Debug)]\npub struct Cors {\n    config: Arc<Configured>,\n}\n\n/// A constructed via `warp::cors()`.\n#[derive(Clone, Debug)]\npub struct Builder {\n    credentials: bool,\n    allowed_headers: HashSet<HeaderName>,\n    exposed_headers: HashSet<HeaderName>,\n    max_age: Option<u64>,\n    methods: HashSet<http::Method>,\n    origins: Option<HashSet<HeaderValue>>,\n}\n\nimpl Builder {\n    /// Sets whether to add the `Access-Control-Allow-Credentials` header.\n    pub fn allow_credentials(mut self, allow: bool) -> Self {\n        self.credentials = allow;\n        self\n    }\n\n    /// Adds a method to the existing list of allowed request methods.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::Method`.\n    pub fn allow_method<M>(mut self, method: M) -> Self\n    where\n        http::Method: TryFrom<M>,\n    {\n        let method = match TryFrom::try_from(method) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Method\"),\n        };\n        self.methods.insert(method);\n        self\n    }\n\n    /// Adds multiple methods to the existing list of allowed request methods.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::Method`.\n    pub fn allow_methods<I>(mut self, methods: I) -> Self\n    where\n        I: IntoIterator,\n        http::Method: TryFrom<I::Item>,\n    {\n        let iter = methods.into_iter().map(|m| match TryFrom::try_from(m) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Method\"),\n        });\n        self.methods.extend(iter);\n        self\n    }\n\n    /// Adds a header to the list of allowed request headers.\n    ///\n    /// **Note**: These should match the values the browser sends via `Access-Control-Request-Headers`, e.g. `content-type`.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::header::HeaderName`.\n    pub fn allow_header<H>(mut self, header: H) -> Self\n    where\n        HeaderName: TryFrom<H>,\n    {\n        let header = match TryFrom::try_from(header) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Header\"),\n        };\n        self.allowed_headers.insert(header);\n        self\n    }\n\n    /// Adds multiple headers to the list of allowed request headers.\n    ///\n    /// **Note**: These should match the values the browser sends via `Access-Control-Request-Headers`, e.g.`content-type`.\n    ///\n    /// # Panics\n    ///\n    /// Panics if any of the headers are not a valid `http::header::HeaderName`.\n    pub fn allow_headers<I>(mut self, headers: I) -> Self\n    where\n        I: IntoIterator,\n        HeaderName: TryFrom<I::Item>,\n    {\n        let iter = headers.into_iter().map(|h| match TryFrom::try_from(h) {\n            Ok(h) => h,\n            Err(_) => panic!(\"illegal Header\"),\n        });\n        self.allowed_headers.extend(iter);\n        self\n    }\n\n    /// Adds a header to the list of exposed headers.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::header::HeaderName`.\n    pub fn expose_header<H>(mut self, header: H) -> Self\n    where\n        HeaderName: TryFrom<H>,\n    {\n        let header = match TryFrom::try_from(header) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Header\"),\n        };\n        self.exposed_headers.insert(header);\n        self\n    }\n\n    /// Adds multiple headers to the list of exposed headers.\n    ///\n    /// # Panics\n    ///\n    /// Panics if any of the headers are not a valid `http::header::HeaderName`.\n    pub fn expose_headers<I>(mut self, headers: I) -> Self\n    where\n        I: IntoIterator,\n        HeaderName: TryFrom<I::Item>,\n    {\n        let iter = headers.into_iter().map(|h| match TryFrom::try_from(h) {\n            Ok(h) => h,\n            Err(_) => panic!(\"illegal Header\"),\n        });\n        self.exposed_headers.extend(iter);\n        self\n    }\n\n    /// Sets that *any* `Origin` header is allowed.\n    ///\n    /// # Warning\n    ///\n    /// This can allow websites you didn't intend to access this resource,\n    /// it is usually better to set an explicit list.\n    pub fn allow_any_origin(mut self) -> Self {\n        self.origins = None;\n        self\n    }\n\n    /// Add an origin to the existing list of allowed `Origin`s.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `Origin`.\n    pub fn allow_origin(self, origin: impl IntoOrigin) -> Self {\n        self.allow_origins(Some(origin))\n    }\n\n    /// Add multiple origins to the existing list of allowed `Origin`s.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `Origin`.\n    pub fn allow_origins<I>(mut self, origins: I) -> Self\n    where\n        I: IntoIterator,\n        I::Item: IntoOrigin,\n    {\n        let iter = origins\n            .into_iter()\n            .map(IntoOrigin::into_origin)\n            .map(|origin| {\n                origin\n                    .to_string()\n                    .parse()\n                    .expect(\"Origin is always a valid HeaderValue\")\n            });\n\n        self.origins.get_or_insert_with(HashSet::new).extend(iter);\n\n        self\n    }\n\n    /// Sets the `Access-Control-Max-Age` header.\n    ///\n    /// # Example\n    ///\n    ///\n    /// ```\n    /// use std::time::Duration;\n    /// use warp::Filter;\n    ///\n    /// let cors = warp::cors()\n    ///     .max_age(30) // 30u32 seconds\n    ///     .max_age(Duration::from_secs(30)); // or a Duration\n    /// ```\n    pub fn max_age(mut self, seconds: impl Seconds) -> Self {\n        self.max_age = Some(seconds.seconds());\n        self\n    }\n\n    /// Builds the `Cors` wrapper from the configured settings.\n    ///\n    /// This step isn't *required*, as the `Builder` itself can be passed\n    /// to `Filter::with`. This just allows constructing once, thus not needing\n    /// to pay the cost of \"building\" every time.\n    pub fn build(self) -> Cors {\n        let expose_headers_header = if self.exposed_headers.is_empty() {\n            None\n        } else {\n            Some(self.exposed_headers.iter().cloned().collect())\n        };\n        let allowed_headers_header = self.allowed_headers.iter().cloned().collect();\n        let methods_header = self.methods.iter().cloned().collect();\n\n        let config = Arc::new(Configured {\n            cors: self,\n            allowed_headers_header,\n            expose_headers_header,\n            methods_header,\n        });\n\n        Cors { config }\n    }\n}\n\nimpl<F> WrapSealed<F> for Builder\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: CombineRejection<Rejection>,\n    <F::Error as CombineRejection<Rejection>>::One: CombineRejection<Rejection>,\n{\n    type Wrapped = CorsFilter<F>;\n\n    fn wrap(&self, inner: F) -> Self::Wrapped {\n        let Cors { config } = self.clone().build();\n\n        CorsFilter { config, inner }\n    }\n}\n\nimpl<F> WrapSealed<F> for Cors\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: CombineRejection<Rejection>,\n    <F::Error as CombineRejection<Rejection>>::One: CombineRejection<Rejection>,\n{\n    type Wrapped = CorsFilter<F>;\n\n    fn wrap(&self, inner: F) -> Self::Wrapped {\n        let config = self.config.clone();\n\n        CorsFilter { config, inner }\n    }\n}\n\n/// An error used to reject requests that are forbidden by a `cors` filter.\npub struct CorsForbidden {\n    kind: Forbidden,\n}\n\n#[derive(Debug)]\nenum Forbidden {\n    OriginNotAllowed,\n    MethodNotAllowed,\n    HeaderNotAllowed,\n}\n\nimpl fmt::Debug for CorsForbidden {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"CorsForbidden\").field(&self.kind).finish()\n    }\n}\n\nimpl fmt::Display for CorsForbidden {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let detail = match self.kind {\n            Forbidden::OriginNotAllowed => \"origin not allowed\",\n            Forbidden::MethodNotAllowed => \"request-method not allowed\",\n            Forbidden::HeaderNotAllowed => \"header not allowed\",\n        };\n        write!(f, \"CORS request forbidden: {}\", detail)\n    }\n}\n\nimpl StdError for CorsForbidden {}\n\n#[derive(Clone, Debug)]\nstruct Configured {\n    cors: Builder,\n    allowed_headers_header: AccessControlAllowHeaders,\n    expose_headers_header: Option<AccessControlExposeHeaders>,\n    methods_header: AccessControlAllowMethods,\n}\n\nenum Validated {\n    Preflight(HeaderValue),\n    Simple(HeaderValue),\n    NotCors,\n}\n\nimpl Configured {\n    fn check_request(\n        &self,\n        method: &http::Method,\n        headers: &http::HeaderMap,\n    ) -> Result<Validated, Forbidden> {\n        match (headers.get(header::ORIGIN), method) {\n            (Some(origin), &http::Method::OPTIONS) => {\n                // OPTIONS requests are preflight CORS requests...\n\n                if !self.is_origin_allowed(origin) {\n                    return Err(Forbidden::OriginNotAllowed);\n                }\n\n                if let Some(req_method) = headers.get(header::ACCESS_CONTROL_REQUEST_METHOD) {\n                    if !self.is_method_allowed(req_method) {\n                        return Err(Forbidden::MethodNotAllowed);\n                    }\n                } else {\n                    tracing::trace!(\n                        \"preflight request missing access-control-request-method header\"\n                    );\n                    return Err(Forbidden::MethodNotAllowed);\n                }\n\n                if let Some(req_headers) = headers.get(header::ACCESS_CONTROL_REQUEST_HEADERS) {\n                    let headers = req_headers\n                        .to_str()\n                        .map_err(|_| Forbidden::HeaderNotAllowed)?;\n                    for header in headers.split(',') {\n                        if !self.is_header_allowed(header.trim()) {\n                            return Err(Forbidden::HeaderNotAllowed);\n                        }\n                    }\n                }\n\n                Ok(Validated::Preflight(origin.clone()))\n            }\n            (Some(origin), _) => {\n                // Any other method, simply check for a valid origin...\n\n                tracing::trace!(\"origin header: {:?}\", origin);\n                if self.is_origin_allowed(origin) {\n                    Ok(Validated::Simple(origin.clone()))\n                } else {\n                    Err(Forbidden::OriginNotAllowed)\n                }\n            }\n            (None, _) => {\n                // No `ORIGIN` header means this isn't CORS!\n                Ok(Validated::NotCors)\n            }\n        }\n    }\n\n    fn is_method_allowed(&self, header: &HeaderValue) -> bool {\n        http::Method::from_bytes(header.as_bytes())\n            .map(|method| self.cors.methods.contains(&method))\n            .unwrap_or(false)\n    }\n\n    fn is_header_allowed(&self, header: &str) -> bool {\n        HeaderName::from_bytes(header.as_bytes())\n            .map(|header| self.cors.allowed_headers.contains(&header))\n            .unwrap_or(false)\n    }\n\n    fn is_origin_allowed(&self, origin: &HeaderValue) -> bool {\n        if let Some(ref allowed) = self.cors.origins {\n            allowed.contains(origin)\n        } else {\n            true\n        }\n    }\n\n    fn append_preflight_headers(&self, headers: &mut http::HeaderMap) {\n        self.append_common_headers(headers);\n\n        headers.typed_insert(self.allowed_headers_header.clone());\n        headers.typed_insert(self.methods_header.clone());\n\n        if let Some(max_age) = self.cors.max_age {\n            headers.insert(header::ACCESS_CONTROL_MAX_AGE, max_age.into());\n        }\n    }\n\n    fn append_common_headers(&self, headers: &mut http::HeaderMap) {\n        if self.cors.credentials {\n            headers.insert(\n                header::ACCESS_CONTROL_ALLOW_CREDENTIALS,\n                HeaderValue::from_static(\"true\"),\n            );\n        }\n        if let Some(expose_headers_header) = &self.expose_headers_header {\n            headers.typed_insert(expose_headers_header.clone())\n        }\n    }\n}\n\nmod internal {\n    use std::future::Future;\n    use std::pin::Pin;\n    use std::sync::Arc;\n    use std::task::{Context, Poll};\n\n    use futures_util::{future, ready, TryFuture};\n    use headers::Origin;\n    use http::header;\n    use pin_project::pin_project;\n\n    use super::{Configured, CorsForbidden, Validated};\n    use crate::filter::{Filter, FilterBase, Internal, One};\n    use crate::generic::Either;\n    use crate::reject::{CombineRejection, Rejection};\n    use crate::route;\n\n    #[derive(Clone, Debug)]\n    pub struct CorsFilter<F> {\n        pub(super) config: Arc<Configured>,\n        pub(super) inner: F,\n    }\n\n    impl<F> FilterBase for CorsFilter<F>\n    where\n        F: Filter,\n        F::Extract: Send,\n        F::Future: Future,\n        F::Error: CombineRejection<Rejection>,\n    {\n        type Extract =\n            One<Either<One<Preflight>, One<Either<One<Wrapped<F::Extract>>, F::Extract>>>>;\n        type Error = <F::Error as CombineRejection<Rejection>>::One;\n        type Future = future::Either<\n            future::Ready<Result<Self::Extract, Self::Error>>,\n            WrappedFuture<F::Future>,\n        >;\n\n        fn filter(&self, _: Internal) -> Self::Future {\n            let validated =\n                route::with(|route| self.config.check_request(route.method(), route.headers()));\n\n            match validated {\n                Ok(Validated::Preflight(origin)) => {\n                    let preflight = Preflight {\n                        config: self.config.clone(),\n                        origin,\n                    };\n                    future::Either::Left(future::ok((Either::A((preflight,)),)))\n                }\n                Ok(Validated::Simple(origin)) => future::Either::Right(WrappedFuture {\n                    inner: self.inner.filter(Internal),\n                    wrapped: Some((self.config.clone(), origin)),\n                }),\n                Ok(Validated::NotCors) => future::Either::Right(WrappedFuture {\n                    inner: self.inner.filter(Internal),\n                    wrapped: None,\n                }),\n                Err(err) => {\n                    let rejection = crate::reject::known(CorsForbidden { kind: err });\n                    future::Either::Left(future::err(rejection.into()))\n                }\n            }\n        }\n    }\n\n    #[derive(Debug)]\n    pub struct Preflight {\n        config: Arc<Configured>,\n        origin: header::HeaderValue,\n    }\n\n    impl crate::reply::Reply for Preflight {\n        fn into_response(self) -> crate::reply::Response {\n            let mut res = crate::reply::Response::default();\n            self.config.append_preflight_headers(res.headers_mut());\n            res.headers_mut()\n                .insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, self.origin);\n            res\n        }\n    }\n\n    #[derive(Debug)]\n    pub struct Wrapped<R> {\n        config: Arc<Configured>,\n        inner: R,\n        origin: header::HeaderValue,\n    }\n\n    impl<R> crate::reply::Reply for Wrapped<R>\n    where\n        R: crate::reply::Reply,\n    {\n        fn into_response(self) -> crate::reply::Response {\n            let mut res = self.inner.into_response();\n            self.config.append_common_headers(res.headers_mut());\n            res.headers_mut()\n                .insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, self.origin);\n            res\n        }\n    }\n\n    #[pin_project]\n    #[derive(Debug)]\n    pub struct WrappedFuture<F> {\n        #[pin]\n        inner: F,\n        wrapped: Option<(Arc<Configured>, header::HeaderValue)>,\n    }\n\n    impl<F> Future for WrappedFuture<F>\n    where\n        F: TryFuture,\n        F::Error: CombineRejection<Rejection>,\n    {\n        type Output = Result<\n            One<Either<One<Preflight>, One<Either<One<Wrapped<F::Ok>>, F::Ok>>>>,\n            <F::Error as CombineRejection<Rejection>>::One,\n        >;\n\n        fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n            let pin = self.project();\n            match ready!(pin.inner.try_poll(cx)) {\n                Ok(inner) => {\n                    let item = if let Some((config, origin)) = pin.wrapped.take() {\n                        (Either::A((Wrapped {\n                            config,\n                            inner,\n                            origin,\n                        },)),)\n                    } else {\n                        (Either::B(inner),)\n                    };\n                    let item = (Either::B(item),);\n                    Poll::Ready(Ok(item))\n                }\n                Err(err) => Poll::Ready(Err(err.into())),\n            }\n        }\n    }\n\n    pub trait Seconds {\n        fn seconds(self) -> u64;\n    }\n\n    impl Seconds for u32 {\n        fn seconds(self) -> u64 {\n            self.into()\n        }\n    }\n\n    impl Seconds for ::std::time::Duration {\n        fn seconds(self) -> u64 {\n            self.as_secs()\n        }\n    }\n\n    pub trait IntoOrigin {\n        fn into_origin(self) -> Origin;\n    }\n\n    impl<'a> IntoOrigin for &'a str {\n        fn into_origin(self) -> Origin {\n            let mut parts = self.splitn(2, \"://\");\n            let scheme = parts.next().expect(\"missing scheme\");\n            let rest = parts.next().expect(\"missing scheme\");\n\n            Origin::try_from_parts(scheme, rest, None).expect(\"invalid Origin\")\n        }\n    }\n}\n\n// Path: src/filters/ext.rs\n//! Request Extensions\n\nuse std::convert::Infallible;\n\nuse futures_util::future;\n\nuse crate::filter::{filter_fn_one, Filter};\nuse crate::reject::{self, Rejection};\n\n/// Get a previously set extension of the current route.\n///\n/// If the extension doesn't exist, this rejects with a `MissingExtension`.\npub fn get<T: Clone + Send + Sync + 'static>(\n) -> impl Filter<Extract = (T,), Error = Rejection> + Copy {\n    filter_fn_one(|route| {\n        let route = route\n            .extensions()\n            .get::<T>()\n            .cloned()\n            .ok_or_else(|| reject::known(MissingExtension { _p: () }));\n        future::ready(route)\n    })\n}\n\n/// Get a previously set extension of the current route.\n///\n/// If the extension doesn't exist, it yields `None`.\npub fn optional<T: Clone + Send + Sync + 'static>(\n) -> impl Filter<Extract = (Option<T>,), Error = Infallible> + Copy {\n    filter_fn_one(|route| future::ok(route.extensions().get::<T>().cloned()))\n}\n\nunit_error! {\n    /// An error used to reject if `get` cannot find the extension.\n    pub MissingExtension: \"Missing request extension\"\n}\n\n// Path: src/reject.rs\n//! Rejections\n//!\n//! Part of the power of the [`Filter`](../trait.Filter.html) system is being able to\n//! reject a request from a filter chain. This allows for filters to be\n//! combined with `or`, so that if one side of the chain finds that a request\n//! doesn't fulfill its requirements, the other side can try to process\n//! the request.\n//!\n//! Many of the built-in [`filters`](../filters) will automatically reject\n//! the request with an appropriate rejection. However, you can also build\n//! new custom [`Filter`](../trait.Filter.html)s and still want other routes to be\n//! matchable in the case a predicate doesn't hold.\n//!\n//! As a request is processed by a Filter chain, the rejections are accumulated into\n//! a list contained by the [`Rejection`](struct.Rejection.html) type. Rejections from\n//! filters can be handled using [`Filter::recover`](../trait.Filter.html#method.recover).\n//! This is a convenient way to map rejections into a [`Reply`](../reply/trait.Reply.html).\n//!\n//! For a more complete example see the\n//! [Rejection Example](https://github.com/seanmonstar/warp/blob/master/examples/rejections.rs)\n//! from the repository.\n//!\n//! # Example\n//!\n//! ```\n//! use warp::{reply, Reply, Filter, reject, Rejection, http::StatusCode};\n//!\n//! #[derive(Debug)]\n//! struct InvalidParameter;\n//!\n//! impl reject::Reject for InvalidParameter {}\n//!\n//! // Custom rejection handler that maps rejections into responses.\n//! async fn handle_rejection(err: Rejection) -> Result<impl Reply, std::convert::Infallible> {\n//!     if err.is_not_found() {\n//!         Ok(reply::with_status(\"NOT_FOUND\", StatusCode::NOT_FOUND))\n//!     } else if let Some(e) = err.find::<InvalidParameter>() {\n//!         Ok(reply::with_status(\"BAD_REQUEST\", StatusCode::BAD_REQUEST))\n//!     } else {\n//!         eprintln!(\"unhandled rejection: {:?}\", err);\n//!         Ok(reply::with_status(\"INTERNAL_SERVER_ERROR\", StatusCode::INTERNAL_SERVER_ERROR))\n//!     }\n//! }\n//!\n//!\n//! // Filter on `/:id`, but reject with InvalidParameter if the `id` is `0`.\n//! // Recover from this rejection using a custom rejection handler.\n//! let route = warp::path::param()\n//!     .and_then(|id: u32| async move {\n//!         if id == 0 {\n//!             Err(warp::reject::custom(InvalidParameter))\n//!         } else {\n//!             Ok(\"id is valid\")\n//!         }\n//!     })\n//!     .recover(handle_rejection);\n//! ```\n\nuse std::any::Any;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::fmt;\n\nuse http::{\n    header::{HeaderValue, CONTENT_TYPE},\n    StatusCode,\n};\nuse hyper::Body;\n\npub(crate) use self::sealed::{CombineRejection, IsReject};\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn reject() -> Rejection {\n    not_found()\n}\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn not_found() -> Rejection {\n    Rejection {\n        reason: Reason::NotFound,\n    }\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_query() -> Rejection {\n    known(InvalidQuery { _p: () })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_header(name: &'static str) -> Rejection {\n    known(MissingHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_header(name: &'static str) -> Rejection {\n    known(InvalidHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_cookie(name: &'static str) -> Rejection {\n    known(MissingCookie { name })\n}\n\n// 405 Method Not Allowed\n#[inline]\npub(crate) fn method_not_allowed() -> Rejection {\n    known(MethodNotAllowed { _p: () })\n}\n\n// 411 Length Required\n#[inline]\npub(crate) fn length_required() -> Rejection {\n    known(LengthRequired { _p: () })\n}\n\n// 413 Payload Too Large\n#[inline]\npub(crate) fn payload_too_large() -> Rejection {\n    known(PayloadTooLarge { _p: () })\n}\n\n// 415 Unsupported Media Type\n//\n// Used by the body filters if the request payload content-type doesn't match\n// what can be deserialized.\n#[inline]\npub(crate) fn unsupported_media_type() -> Rejection {\n    known(UnsupportedMediaType { _p: () })\n}\n\n...\n", "needle_token_start": 7075, "needle_token_end": 7556, "code_context_ntokens": 16218, "output": ["```rust\nfn file_stream(\n    mut file: TkFile,\n    buf_size: usize,\n    (start, end): (u64, u64),\n) -> impl Stream<Item = Result<Bytes, io::Error>> + Send {\n    use std::io::SeekFrom;\n\n    let seek = async move {\n        if start != 0 {\n            file.seek(SeekFrom::Start(start)).await?;\n        }\n        Ok(file)\n    };\n\n    seek.into_stream()\n        .map(move |result| {\n            let mut buf = BytesMut::new();\n            let mut len = end - start;\n            let mut f = match result {\n                Ok(f) => f,\n                Err(f) => return Either::Left(stream::once(future::err(f))),\n            };\n\n            Either::Right(stream::poll_fn(move |cx| {\n                if len == 0 {\n                    return Poll::Ready(None);\n                }\n                reserve_at_least(&mut buf, buf_size);\n\n                let n = match ready!(poll_read_buf(Pin::new(&mut f), cx, &mut buf)) {\n                    Ok(n) => n as u64,\n                    Err(err) => {\n                        tracing::debug!(\"file read error: {}\", err);\n                        return Poll::Ready(Some(Err(err)));\n                    }\n                };\n\n                if n == 0 {\n                    tracing::debug!(\"file read found EOF before expected length\");\n                    return Poll::Ready(None);\n                }\n\n                let mut chunk = buf.split().freeze();\n                if n > len {\n                    chunk = chunk.split_to(len as usize);\n                    len = 0;\n                } else {\n                    len -= n;\n                }\n\n                Poll::Ready(Some(Ok(chunk)))\n            }))\n        })\n        .flatten()\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "send_text", "language": "rust", "path": "src/test.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To transmit a textual message over a WebSocket connection to the server.\n2. **Input**: A string or any type that can be converted into a string, representing the message to be sent.\n3. **Output**: There is no direct output from the function itself as it operates asynchronously, but it sends the message to the server.\n4. **Procedure**: The function takes the input message, converts it into a WebSocket text message format, and then sends it asynchronously through the WebSocket connection.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/server.rs\n#[cfg(feature = \"tls\")]\nuse crate::tls::TlsConfigBuilder;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"tls\")]\nuse std::path::Path;\n\nuse futures_util::{future, FutureExt, TryFuture, TryStream, TryStreamExt};\nuse hyper::server::conn::AddrIncoming;\nuse hyper::service::{make_service_fn, service_fn};\nuse hyper::Server as HyperServer;\nuse tokio::io::{AsyncRead, AsyncWrite};\nuse tracing::Instrument;\n\nuse crate::filter::Filter;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::transport::Transport;\n\n/// Create a `Server` with the provided `Filter`.\npub fn serve<F>(filter: F) -> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: IsReject,\n{\n    Server {\n        pipeline: false,\n        filter,\n    }\n}\n\n/// A Warp Server ready to filter requests.\n#[derive(Debug)]\npub struct Server<F> {\n    pipeline: bool,\n    filter: F,\n}\n\n/// A Warp Server ready to filter requests over TLS.\n///\n/// *This type requires the `\"tls\"` feature.*\n#[cfg(feature = \"tls\")]\npub struct TlsServer<F> {\n    server: Server<F>,\n    tls: TlsConfigBuilder,\n}\n\n// Getting all various generic bounds to make this a re-usable method is\n// very complicated, so instead this is just a macro.\nmacro_rules! into_service {\n    ($into:expr) => {{\n        let inner = crate::service($into);\n        make_service_fn(move |transport| {\n            let inner = inner.clone();\n            let remote_addr = Transport::remote_addr(transport);\n            future::ok::<_, Infallible>(service_fn(move |req| {\n                inner.call_with_addr(req, remote_addr)\n            }))\n        })\n    }};\n}\n\nmacro_rules! addr_incoming {\n    ($addr:expr) => {{\n        let mut incoming = AddrIncoming::bind($addr)?;\n        incoming.set_nodelay(true);\n        let addr = incoming.local_addr();\n        (addr, incoming)\n    }};\n}\n\nmacro_rules! bind_inner {\n    ($this:ident, $addr:expr) => {{\n        let service = into_service!($this.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let srv = HyperServer::builder(incoming)\n            .http1_pipeline_flush($this.pipeline)\n            .serve(service);\n        Ok::<_, hyper::Error>((addr, srv))\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let service = into_service!($this.server.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let tls = $this.tls.build()?;\n        let srv = HyperServer::builder(crate::tls::TlsAcceptor::new(tls, incoming))\n            .http1_pipeline_flush($this.server.pipeline)\n            .serve(service);\n        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((addr, srv))\n    }};\n}\n\nmacro_rules! bind {\n    ($this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!($this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!(tls: $this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n}\n\nmacro_rules! try_bind {\n    ($this:ident, $addr:expr) => {{\n        (|addr| bind_inner!($this, addr))($addr)\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        (|addr| bind_inner!(tls: $this, addr))($addr)\n    }};\n}\n\n// ===== impl Server =====\n\nimpl<F> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    /// Run this `Server` forever on the current thread.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub async fn run(self, addr: impl Into<SocketAddr>) {\n        let (addr, fut) = self.bind_ephemeral(addr);\n        let span = tracing::info_span!(\"Server::run\", ?addr);\n        tracing::info!(parent: &span, \"listening on http://{}\", addr);\n\n        fut.instrument(span).await;\n    }\n\n    /// Run this `Server` forever on the current thread with a specific stream\n    /// of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    pub async fn run_incoming<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        self.run_incoming2(incoming.map_ok(crate::transport::LiftIo).into_stream())\n            .instrument(tracing::info_span!(\"Server::run_incoming\"))\n            .await;\n    }\n\n    async fn run_incoming2<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: Transport + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let fut = self.serve_incoming2(incoming);\n\n        tracing::info!(\"listening with custom incoming\");\n\n        fut.await;\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind(self, addr: impl Into<SocketAddr> + 'static) -> impl Future<Output = ()> + 'static {\n        let (_, fut) = self.bind_ephemeral(addr);\n        fut\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on any runtime.\n    ///\n    /// In case we are unable to bind to the specified address, resolves to an\n    /// error and logs the reason.\n    pub async fn try_bind(self, addr: impl Into<SocketAddr>) {\n        let addr = addr.into();\n        let srv = match try_bind!(self, &addr) {\n            Ok((_, srv)) => srv,\n            Err(err) => {\n                tracing::error!(\"error binding to {}: {}\", addr, err);\n                return;\n            }\n        };\n\n        srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        })\n        .await;\n    }\n\n    /// Bind to a possibly ephemeral socket address.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        (addr, srv)\n    }\n\n    /// Tried to bind a possibly ephemeral socket address.\n    ///\n    /// Returns a `Result` which fails in case we are unable to bind with the\n    /// underlying error.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    pub fn try_bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use warp::Filter;\n    /// use futures_util::future::TryFutureExt;\n    /// use tokio::sync::oneshot;\n    ///\n    /// # fn main() {\n    /// let routes = warp::any()\n    ///     .map(|| \"Hello, World!\");\n    ///\n    /// let (tx, rx) = oneshot::channel();\n    ///\n    /// let (addr, server) = warp::serve(routes)\n    ///     .bind_with_graceful_shutdown(([127, 0, 0, 1], 3030), async {\n    ///          rx.await.ok();\n    ///     });\n    ///\n    /// // Spawn the server into a runtime\n    /// tokio::task::spawn(server);\n    ///\n    /// // Later, start the shutdown...\n    /// let _ = tx.send(());\n    /// # }\n    /// ```\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let fut = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n        (addr, fut)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    pub fn try_bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming<I>(self, incoming: I) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        self.serve_incoming2(incoming)\n            .instrument(tracing::info_span!(\"Server::serve_incoming\"))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections and a\n    /// signal to initiate graceful shutdown.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming_with_graceful_shutdown<I>(\n        self,\n        incoming: I,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        let service = into_service!(self.filter);\n        let pipeline = self.pipeline;\n\n        async move {\n            let srv =\n                HyperServer::builder(hyper::server::accept::from_stream(incoming.into_stream()))\n                    .http1_pipeline_flush(pipeline)\n                    .serve(service)\n                    .with_graceful_shutdown(signal)\n                    .await;\n\n            if let Err(err) = srv {\n                tracing::error!(\"server error: {}\", err);\n            }\n        }\n...\n// Path: src/test.rs\n//! Test utilities to test your filters.\n//!\n//! [`Filter`](../trait.Filter.html)s can be easily tested without starting up an HTTP\n//! server, by making use of the [`RequestBuilder`](./struct.RequestBuilder.html) in this\n//! module.\n//!\n//! # Testing Filters\n//!\n//! It's easy to test filters, especially if smaller filters are used to build\n//! up your full set. Consider these example filters:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! fn sum() -> impl Filter<Extract = (u32,), Error = warp::Rejection> + Copy {\n//!     warp::path::param()\n//!         .and(warp::path::param())\n//!         .map(|x: u32, y: u32| {\n//!             x + y\n//!         })\n//! }\n//!\n//! fn math() -> impl Filter<Extract = (String,), Error = warp::Rejection> + Copy {\n//!     warp::post()\n//!         .and(sum())\n//!         .map(|z: u32| {\n//!             format!(\"Sum = {}\", z)\n//!         })\n//! }\n//! ```\n//!\n//! We can test some requests against the `sum` filter like this:\n//!\n//! ```\n//! # use warp::Filter;\n//! #[tokio::test]\n//! async fn test_sum() {\n//! #    let sum = || warp::any().map(|| 3);\n//!     let filter = sum();\n//!\n//!     // Execute `sum` and get the `Extract` back.\n//!     let value = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .filter(&filter)\n//!         .await\n//!         .unwrap();\n//!     assert_eq!(value, 3);\n//!\n//!     // Or simply test if a request matches (doesn't reject).\n//!     assert!(\n//!         warp::test::request()\n//!             .path(\"/1/-5\")\n//!             .matches(&filter)\n//!             .await\n//!     );\n//! }\n//! ```\n//!\n//! If the filter returns something that implements `Reply`, and thus can be\n//! turned into a response sent back to the client, we can test what exact\n//! response is returned. The `math` filter uses the `sum` filter, but returns\n//! a `String` that can be turned into a response.\n//!\n//! ```\n//! # use warp::Filter;\n//! #[test]\n//! fn test_math() {\n//! #    let math = || warp::any().map(warp::reply);\n//!     let filter = math();\n//!\n//!     let res = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 405, \"GET is not allowed\");\n//!\n//!     let res = warp::test::request()\n//!         .method(\"POST\")\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 200);\n//!     assert_eq!(res.body(), \"Sum is 3\");\n//! }\n//! ```\nuse std::convert::TryFrom;\nuse std::error::Error as StdError;\nuse std::fmt;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"websocket\")]\nuse std::pin::Pin;\n#[cfg(feature = \"websocket\")]\nuse std::task::Context;\n#[cfg(feature = \"websocket\")]\nuse std::task::{self, Poll};\n\nuse bytes::Bytes;\n#[cfg(feature = \"websocket\")]\nuse futures_channel::mpsc;\n#[cfg(feature = \"websocket\")]\nuse futures_util::StreamExt;\nuse futures_util::{future, FutureExt, TryFutureExt};\nuse http::{\n    header::{HeaderName, HeaderValue},\n    Response,\n};\nuse serde::Serialize;\n#[cfg(feature = \"websocket\")]\nuse tokio::sync::oneshot;\n\nuse crate::filter::Filter;\n#[cfg(feature = \"websocket\")]\nuse crate::filters::ws::Message;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::route::{self, Route};\nuse crate::Request;\n#[cfg(feature = \"websocket\")]\nuse crate::{Sink, Stream};\n\nuse self::inner::OneOrTuple;\n\n/// Starts a new test `RequestBuilder`.\npub fn request() -> RequestBuilder {\n    RequestBuilder {\n        remote_addr: None,\n        req: Request::default(),\n    }\n}\n\n/// Starts a new test `WsBuilder`.\n#[cfg(feature = \"websocket\")]\npub fn ws() -> WsBuilder {\n    WsBuilder { req: request() }\n}\n\n/// A request builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[must_use = \"RequestBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct RequestBuilder {\n    remote_addr: Option<SocketAddr>,\n    req: Request,\n}\n\n/// A Websocket builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[cfg(feature = \"websocket\")]\n#[must_use = \"WsBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct WsBuilder {\n    req: RequestBuilder,\n}\n\n/// A test client for Websocket filters.\n#[cfg(feature = \"websocket\")]\npub struct WsClient {\n    tx: mpsc::UnboundedSender<crate::ws::Message>,\n    rx: mpsc::UnboundedReceiver<Result<crate::ws::Message, crate::error::Error>>,\n}\n\n/// An error from Websocket filter tests.\n#[derive(Debug)]\npub struct WsError {\n    cause: Box<dyn StdError + Send + Sync>,\n}\n\nimpl RequestBuilder {\n    /// Sets the method of this builder.\n    ///\n    /// The default if not set is `GET`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .method(\"POST\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Method`.\n    pub fn method(mut self, method: &str) -> Self {\n        *self.req.method_mut() = method.parse().expect(\"valid method\");\n        self\n    }\n\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .path(\"/todos/33\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(mut self, p: &str) -> Self {\n        let uri = p.parse().expect(\"test request path invalid\");\n        *self.req.uri_mut() = uri;\n        self\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .header(\"accept\", \"application/json\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(mut self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        let name: HeaderName = TryFrom::try_from(key)\n            .map_err(|_| ())\n            .expect(\"invalid header name\");\n        let value = TryFrom::try_from(value)\n            .map_err(|_| ())\n            .expect(\"invalid header value\");\n        self.req.headers_mut().insert(name, value);\n        self\n    }\n\n    /// Set the remote address of this request\n    ///\n    /// Default is no remote address.\n    ///\n    /// # Example\n    /// ```\n    /// use std::net::{IpAddr, Ipv4Addr, SocketAddr};\n    ///\n    /// let req = warp::test::request()\n    ///     .remote_addr(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080));\n    /// ```\n    pub fn remote_addr(mut self, addr: SocketAddr) -> Self {\n        self.remote_addr = Some(addr);\n        self\n    }\n\n    /// Add a type to the request's `http::Extensions`.\n    pub fn extension<T>(mut self, ext: T) -> Self\n    where\n        T: Send + Sync + 'static,\n    {\n        self.req.extensions_mut().insert(ext);\n        self\n    }\n\n    /// Set the bytes of this request body.\n    ///\n    /// Default is an empty body.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .body(\"foo=bar&baz=quux\");\n    /// ```\n    pub fn body(mut self, body: impl AsRef<[u8]>) -> Self {\n        let body = body.as_ref().to_vec();\n        let len = body.len();\n        *self.req.body_mut() = body.into();\n        self.header(\"content-length\", len.to_string())\n    }\n\n    /// Set the bytes of this request body by serializing a value into JSON.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .json(&true);\n    /// ```\n    pub fn json(mut self, val: &impl Serialize) -> Self {\n        let vec = serde_json::to_vec(val).expect(\"json() must serialize to JSON\");\n        let len = vec.len();\n        *self.req.body_mut() = vec.into();\n        self.header(\"content-length\", len.to_string())\n            .header(\"content-type\", \"application/json\")\n    }\n\n    /// Tries to apply the `Filter` on this request.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let param = warp::path::param::<u32>();\n    ///\n    ///     let ex = warp::test::request()\n    ///         .path(\"/41\")\n    ///         .filter(&param)\n    ///         .await\n    ///         .unwrap();\n    ///\n    ///     assert_eq!(ex, 41);\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .path(\"/foo\")\n    ///             .filter(&param)\n    ///             .await\n    ///             .is_err()\n    ///     );\n    ///};\n    /// ```\n    pub async fn filter<F>(self, f: &F) -> Result<<F::Extract as OneOrTuple>::Output, F::Error>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: OneOrTuple + Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.map(|ex| ex.one_or_tuple())\n    }\n\n    /// Returns whether the `Filter` matches this request, or rejects it.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let get = warp::get();\n    ///     let post = warp::post();\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&get)\n    ///             .await\n    ///     );\n    ///\n    ///     assert!(\n    ///         !warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&post)\n    ///             .await\n    ///     );\n    ///};\n    /// ```\n    pub async fn matches<F>(self, f: &F) -> bool\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.is_ok()\n    }\n\n    /// Returns `Response` provided by applying the `Filter`.\n    ///\n    /// This requires that the supplied `Filter` return a [`Reply`].\n    pub async fn reply<F>(self, f: &F) -> Response<Bytes>\n    where\n        F: Filter + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        // TODO: de-duplicate this and apply_filter()\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(\n            route::set(&route, move || f.filter(crate::filter::Internal)).then(|result| {\n                let res = match result {\n                    Ok(rep) => rep.into_response(),\n                    Err(rej) => {\n                        tracing::debug!(\"rejected: {:?}\", rej);\n                        rej.into_response()\n                    }\n                };\n                let (parts, body) = res.into_parts();\n                hyper::body::to_bytes(body).map_ok(|chunk| Response::from_parts(parts, chunk))\n            }),\n        );\n\n        let fut = future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)));\n\n        fut.await.expect(\"reply shouldn't fail\")\n    }\n\n    fn apply_filter<F>(self, f: &F) -> impl Future<Output = Result<F::Extract, F::Error>>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(route::set(&route, move || {\n            f.filter(crate::filter::Internal)\n        }));\n        future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)))\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl WsBuilder {\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .path(\"/chat\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(self, p: &str) -> Self {\n        WsBuilder {\n            req: self.req.path(p),\n        }\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .header(\"foo\", \"bar\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        WsBuilder {\n            req: self.req.header(key, value),\n        }\n    }\n\n    /// Execute this Websocket request against the provided filter.\n    ///\n    /// If the handshake succeeds, returns a `WsClient`.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use futures_util::future;\n    /// use warp::Filter;\n    /// #[tokio::main]\n    /// # async fn main() {\n    ///\n    /// // Some route that accepts websockets (but drops them immediately).\n    /// let route = warp::ws()\n    ///     .map(|ws: warp::ws::Ws| {\n    ///         ws.on_upgrade(|_| future::ready(()))\n    ///     });\n    ///\n    /// let client = warp::test::ws()\n    ///     .handshake(route)\n    ///     .await\n    ///     .expect(\"handshake\");\n    /// # }\n    /// ```\n    pub async fn handshake<F>(self, f: F) -> Result<WsClient, WsError>\n    where\n        F: Filter + Clone + Send + Sync + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        let (upgraded_tx, upgraded_rx) = oneshot::channel();\n        let (wr_tx, wr_rx) = mpsc::unbounded();\n        let (rd_tx, rd_rx) = mpsc::unbounded();\n\n        tokio::spawn(async move {\n            use tokio_tungstenite::tungstenite::protocol;\n\n            let (addr, srv) = crate::serve(f).bind_ephemeral(([127, 0, 0, 1], 0));\n\n            let mut req = self\n                .req\n                .header(\"connection\", \"upgrade\")\n                .header(\"upgrade\", \"websocket\")\n                .header(\"sec-websocket-version\", \"13\")\n                .header(\"sec-websocket-key\", \"dGhlIHNhbXBsZSBub25jZQ==\")\n                .req;\n\n            let query_string = match req.uri().query() {\n                Some(q) => format!(\"?{}\", q),\n                None => String::from(\"\"),\n            };\n\n            let uri = format!(\"http://{}{}{}\", addr, req.uri().path(), query_string)\n                .parse()\n                .expect(\"addr + path is valid URI\");\n\n            *req.uri_mut() = uri;\n\n            // let mut rt = current_thread::Runtime::new().unwrap();\n            tokio::spawn(srv);\n\n            let upgrade = ::hyper::Client::builder()\n                .build(AddrConnect(addr))\n                .request(req)\n                .and_then(hyper::upgrade::on);\n\n            let upgraded = match upgrade.await {\n                Ok(up) => {\n                    let _ = upgraded_tx.send(Ok(()));\n                    up\n                }\n                Err(err) => {\n                    let _ = upgraded_tx.send(Err(err));\n                    return;\n                }\n            };\n            let ws = crate::ws::WebSocket::from_raw_socket(\n                upgraded,\n                protocol::Role::Client,\n                Default::default(),\n            )\n            .await;\n\n            let (tx, rx) = ws.split();\n            let write = wr_rx.map(Ok).forward(tx).map(|_| ());\n\n            let read = rx\n                .take_while(|result| match result {\n                    Err(_) => future::ready(false),\n                    Ok(m) => future::ready(!m.is_close()),\n                })\n                .for_each(move |item| {\n                    rd_tx.unbounded_send(item).expect(\"ws receive error\");\n                    future::ready(())\n                });\n\n            future::join(write, read).await;\n        });\n\n        match upgraded_rx.await {\n            Ok(Ok(())) => Ok(WsClient {\n                tx: wr_tx,\n                rx: rd_rx,\n            }),\n            Ok(Err(err)) => Err(WsError::new(err)),\n            Err(_canceled) => panic!(\"websocket handshake thread panicked\"),\n        }\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl WsClient {\n    /// Send a \"text\" websocket message to the server.\n    \npub async fn send_text(&mut self, text: impl Into<String>) {\n        self.send(crate::ws::Message::text(text)).await;\n    }\n\n    /// Send a websocket message to the server.\n    pub async fn send(&mut self, msg: crate::ws::Message) {\n        self.tx.unbounded_send(msg).unwrap();\n    }\n\n    /// Receive a websocket message from the server.\n    pub async fn recv(&mut self) -> Result<crate::filters::ws::Message, WsError> {\n        self.rx\n            .next()\n            .await\n            .map(|result| result.map_err(WsError::new))\n            .unwrap_or_else(|| {\n                // websocket is closed\n                Err(WsError::new(\"closed\"))\n            })\n    }\n\n    /// Assert the server has closed the connection.\n    pub async fn recv_closed(&mut self) -> Result<(), WsError> {\n        self.rx\n            .next()\n            .await\n            .map(|result| match result {\n                Ok(msg) => Err(WsError::new(format!(\"received message: {:?}\", msg))),\n                Err(err) => Err(WsError::new(err)),\n            })\n            .unwrap_or_else(|| {\n                // closed successfully\n                Ok(())\n            })\n    }\n\n    fn pinned_tx(self: Pin<&mut Self>) -> Pin<&mut mpsc::UnboundedSender<crate::ws::Message>> {\n        let this = Pin::into_inner(self);\n        Pin::new(&mut this.tx)\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl fmt::Debug for WsClient {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"WsClient\").finish()\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl Sink<crate::ws::Message> for WsClient {\n    type Error = WsError;\n\n    fn poll_ready(\n        self: Pin<&mut Self>,\n        context: &mut Context<'_>,\n    ) -> Poll<Result<(), Self::Error>> {\n        self.pinned_tx().poll_ready(context).map_err(WsError::new)\n    }\n\n    fn start_send(self: Pin<&mut Self>, message: Message) -> Result<(), Self::Error> {\n        self.pinned_tx().start_send(message).map_err(WsError::new)\n    }\n\n    fn poll_flush(\n        self: Pin<&mut Self>,\n        context: &mut Context<'_>,\n    ) -> Poll<Result<(), Self::Error>> {\n        self.pinned_tx().poll_flush(context).map_err(WsError::new)\n    }\n\n    fn poll_close(\n        self: Pin<&mut Self>,\n        context: &mut Context<'_>,\n    ) -> Poll<Result<(), Self::Error>> {\n        self.pinned_tx().poll_close(context).map_err(WsError::new)\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl Stream for WsClient {\n    type Item = Result<crate::ws::Message, WsError>;\n\n    fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Option<Self::Item>> {\n        let this = Pin::into_inner(self);\n        let rx = Pin::new(&mut this.rx);\n        match rx.poll_next(context) {\n            Poll::Ready(Some(result)) => Poll::Ready(Some(result.map_err(WsError::new))),\n            Poll::Ready(None) => Poll::Ready(None),\n            Poll::Pending => Poll::Pending,\n        }\n    }\n}\n\n// ===== impl WsError =====\n\n#[cfg(feature = \"websocket\")]\nimpl WsError {\n    fn new<E: Into<Box<dyn StdError + Send + Sync>>>(cause: E) -> Self {\n        WsError {\n            cause: cause.into(),\n        }\n    }\n}\n\nimpl fmt::Display for WsError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"websocket error: {}\", self.cause)\n    }\n}\n\nimpl StdError for WsError {\n    fn description(&self) -> &str {\n        \"websocket error\"\n    }\n}\n\n// ===== impl AddrConnect =====\n\n#[cfg(feature = \"websocket\")]\n#[derive(Clone)]\nstruct AddrConnect(SocketAddr);\n\n#[cfg(feature = \"websocket\")]\nimpl tower_service::Service<::http::Uri> for AddrConnect {\n    type Response = ::tokio::net::TcpStream;\n    type Error = ::std::io::Error;\n    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;\n\n    fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll<Result<(), Self::Error>> {\n        Poll::Ready(Ok(()))\n    }\n\n    fn call(&mut self, _: ::http::Uri) -> Self::Future {\n        Box::pin(tokio::net::TcpStream::connect(self.0))\n    }\n}\n\nmod inner {\n    pub trait OneOrTuple {\n        type Output;\n\n        fn one_or_tuple(self) -> Self::Output;\n    }\n\n    impl OneOrTuple for () {\n        type Output = ();\n        fn one_or_tuple(self) -> Self::Output {}\n    }\n\n    macro_rules! one_or_tuple {\n        ($type1:ident) => {\n            impl<$type1> OneOrTuple for ($type1,) {\n                type Output = $type1;\n                fn one_or_tuple(self) -> Self::Output {\n                    self.0\n                }\n            }\n        };\n        ($type1:ident, $( $type:ident ),*) => {\n            one_or_tuple!($( $type ),*);\n\n            impl<$type1, $($type),*> OneOrTuple for ($type1, $($type),*) {\n                type Output = Self;\n                fn one_or_tuple(self) -> Self::Output {\n                    self\n                }\n            }\n        }\n    }\n\n    one_or_tuple! {\n        T1,\n        T2,\n        T3,\n        T4,\n        T5,\n        T6,\n        T7,\n        T8,\n        T9,\n        T10,\n        T11,\n        T12,\n        T13,\n        T14,\n        T15,\n        T16\n    }\n}\n\n// Path: src/lib.rs\n#![deny(missing_docs)]\n#![deny(missing_debug_implementations)]\n#![deny(rust_2018_idioms)]\n#![cfg_attr(test, deny(warnings))]\n\n//! # warp\n//!\n//! warp is a super-easy, composable, web server framework for warp speeds.\n//!\n//! Thanks to its [`Filter`][Filter] system, warp provides these out of the box:\n//!\n//! - Path routing and parameter extraction\n//! - Header requirements and extraction\n//! - Query string deserialization\n//! - JSON and Form bodies\n//! - Multipart form data\n//! - Static Files and Directories\n//! - Websockets\n//! - Access logging\n//! - Etc\n//!\n//! Since it builds on top of [hyper](https://hyper.rs), you automatically get:\n//!\n//! - HTTP/1\n//! - HTTP/2\n//! - Asynchronous\n//! - One of the fastest HTTP implementations\n//! - Tested and **correct**\n//!\n//! ## Filters\n//!\n//! The main concept in warp is the [`Filter`][Filter], which allows composition\n//! to describe various endpoints in your web service. Besides this powerful\n//! trait, warp comes with several built in [filters](filters/index.html), which\n//! can be combined for your specific needs.\n//!\n//! As a small example, consider an endpoint that has path and header requirements:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! let hi = warp::path(\"hello\")\n//!     .and(warp::path::param())\n//!     .and(warp::header(\"user-agent\"))\n//!     .map(|param: String, agent: String| {\n//!         format!(\"Hello {}, whose agent is {}\", param, agent)\n//!     });\n//! ```\n//!\n//! This example composes several [`Filter`s][Filter] together using `and`:\n//!\n//! - A path prefix of \"hello\"\n//! - A path parameter of a `String`\n//! - The `user-agent` header parsed as a `String`\n//!\n//! These specific filters will [`reject`][reject] requests that don't match\n//! their requirements.\n//!\n//! This ends up matching requests like:\n//!\n//! ```notrust\n//! GET /hello/sean HTTP/1.1\n//! Host: hyper.rs\n//! User-Agent: reqwest/v0.8.6\n//!\n//! ```\n//! And it returns a response similar to this:\n//!\n//! ```notrust\n//! HTTP/1.1 200 OK\n//! Content-Length: 41\n//! Date: ...\n//!\n//! Hello sean, whose agent is reqwest/v0.8.6\n//! ```\n//!\n//! Take a look at the full list of [`filters`](filters/index.html) to see what\n//! you can build.\n//!\n//! ## Testing\n//!\n//! Testing your web services easily is extremely important, and warp provides\n//! a [`test`](mod@self::test) module to help send mocked requests through your service.\n//!\n//! [Filter]: trait.Filter.html\n//! [reject]: reject/index.html\n\n#[macro_use]\nmod error;\nmod filter;\npub mod filters;\nmod generic;\npub mod redirect;\npub mod reject;\npub mod reply;\nmod route;\nmod server;\nmod service;\npub mod test;\n#[cfg(feature = \"tls\")]\nmod tls;\nmod transport;\n\npub use self::error::Error;\npub use self::filter::Filter;\n// This otherwise shows a big dump of re-exports in the doc homepage,\n// with zero context, so just hide it from the docs. Doc examples\n// on each can show that a convenient import exists.\n#[cfg(feature = \"compression\")]\n#[doc(hidden)]\npub use self::filters::compression;\n#[cfg(feature = \"multipart\")]\n#[doc(hidden)]\npub use self::filters::multipart;\n#[cfg(feature = \"websocket\")]\n#[doc(hidden)]\npub use self::filters::ws;\n#[doc(hidden)]\npub use self::filters::{\n    addr,\n    // any() function\n    any::any,\n    body,\n    cookie,\n    // cookie() function\n    cookie::cookie,\n    cors,\n    // cors() function\n    cors::cors,\n    ext,\n    fs,\n    header,\n    // header() function\n    header::header,\n    host,\n    log,\n    // log() function\n    log::log,\n    method::{delete, get, head, method, options, patch, post, put},\n    path,\n    // path() function and macro\n    path::path,\n    query,\n    // query() function\n    query::query,\n    sse,\n    trace,\n    // trace() function\n    trace::trace,\n};\n// ws() function\npub use self::filter::wrap_fn;\n#[cfg(feature = \"websocket\")]\n#[doc(hidden)]\npub use self::filters::ws::ws;\n#[doc(hidden)]\npub use self::redirect::redirect;\n#[doc(hidden)]\n#[allow(deprecated)]\npub use self::reject::{reject, Rejection};\n#[doc(hidden)]\npub use self::reply::{reply, Reply};\n#[cfg(feature = \"tls\")]\npub use self::server::TlsServer;\npub use self::server::{serve, Server};\npub use self::service::service;\n#[doc(hidden)]\npub use http;\n#[doc(hidden)]\npub use hyper;\n\n#[doc(hidden)]\npub use bytes::Buf;\n#[doc(hidden)]\npub use futures_util::{Future, Sink, Stream};\n#[doc(hidden)]\n\npub(crate) type Request = http::Request<hyper::Body>;\n\n", "needle_token_start": 8937, "needle_token_end": 8978, "code_context_ntokens": 12145, "output": ["```rust\n#[cfg(feature = \"websocket\")]\nimpl WsClient {\n    /// Send a \"text\" websocket message to the server.\n    pub async fn send_text(&mut self, text: impl Into<String>) {\n        self.send(crate::ws::Message::text(text)).await;\n    }\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "description", "language": "rust", "path": "src/test.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To provide a textual representation of the error encountered in a specific context related to web communication.\n2. **Input**: None, as it operates on the internal state of the error instance.\n3. **Output**: A string that describes the nature of the error.\n4. **Procedure**: This method accesses the internal error state and returns a predefined string that indicates a general error type related to web communication.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/server.rs\n#[cfg(feature = \"tls\")]\nuse crate::tls::TlsConfigBuilder;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"tls\")]\nuse std::path::Path;\n\nuse futures_util::{future, FutureExt, TryFuture, TryStream, TryStreamExt};\nuse hyper::server::conn::AddrIncoming;\nuse hyper::service::{make_service_fn, service_fn};\nuse hyper::Server as HyperServer;\nuse tokio::io::{AsyncRead, AsyncWrite};\nuse tracing::Instrument;\n\nuse crate::filter::Filter;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::transport::Transport;\n\n/// Create a `Server` with the provided `Filter`.\npub fn serve<F>(filter: F) -> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: IsReject,\n{\n    Server {\n        pipeline: false,\n        filter,\n    }\n}\n\n/// A Warp Server ready to filter requests.\n#[derive(Debug)]\npub struct Server<F> {\n    pipeline: bool,\n    filter: F,\n}\n\n/// A Warp Server ready to filter requests over TLS.\n///\n/// *This type requires the `\"tls\"` feature.*\n#[cfg(feature = \"tls\")]\npub struct TlsServer<F> {\n    server: Server<F>,\n    tls: TlsConfigBuilder,\n}\n\n// Getting all various generic bounds to make this a re-usable method is\n// very complicated, so instead this is just a macro.\nmacro_rules! into_service {\n    ($into:expr) => {{\n        let inner = crate::service($into);\n        make_service_fn(move |transport| {\n            let inner = inner.clone();\n            let remote_addr = Transport::remote_addr(transport);\n            future::ok::<_, Infallible>(service_fn(move |req| {\n                inner.call_with_addr(req, remote_addr)\n            }))\n        })\n    }};\n}\n\nmacro_rules! addr_incoming {\n    ($addr:expr) => {{\n        let mut incoming = AddrIncoming::bind($addr)?;\n        incoming.set_nodelay(true);\n        let addr = incoming.local_addr();\n        (addr, incoming)\n    }};\n}\n\nmacro_rules! bind_inner {\n    ($this:ident, $addr:expr) => {{\n        let service = into_service!($this.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let srv = HyperServer::builder(incoming)\n            .http1_pipeline_flush($this.pipeline)\n            .serve(service);\n        Ok::<_, hyper::Error>((addr, srv))\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let service = into_service!($this.server.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let tls = $this.tls.build()?;\n        let srv = HyperServer::builder(crate::tls::TlsAcceptor::new(tls, incoming))\n            .http1_pipeline_flush($this.server.pipeline)\n            .serve(service);\n        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((addr, srv))\n    }};\n}\n\nmacro_rules! bind {\n    ($this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!($this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!(tls: $this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n}\n\nmacro_rules! try_bind {\n    ($this:ident, $addr:expr) => {{\n        (|addr| bind_inner!($this, addr))($addr)\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        (|addr| bind_inner!(tls: $this, addr))($addr)\n    }};\n}\n\n// ===== impl Server =====\n\nimpl<F> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    /// Run this `Server` forever on the current thread.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub async fn run(self, addr: impl Into<SocketAddr>) {\n        let (addr, fut) = self.bind_ephemeral(addr);\n        let span = tracing::info_span!(\"Server::run\", ?addr);\n        tracing::info!(parent: &span, \"listening on http://{}\", addr);\n\n        fut.instrument(span).await;\n    }\n\n    /// Run this `Server` forever on the current thread with a specific stream\n    /// of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    pub async fn run_incoming<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        self.run_incoming2(incoming.map_ok(crate::transport::LiftIo).into_stream())\n            .instrument(tracing::info_span!(\"Server::run_incoming\"))\n            .await;\n    }\n\n    async fn run_incoming2<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: Transport + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let fut = self.serve_incoming2(incoming);\n\n        tracing::info!(\"listening with custom incoming\");\n\n        fut.await;\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind(self, addr: impl Into<SocketAddr> + 'static) -> impl Future<Output = ()> + 'static {\n        let (_, fut) = self.bind_ephemeral(addr);\n        fut\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on any runtime.\n    ///\n    /// In case we are unable to bind to the specified address, resolves to an\n    /// error and logs the reason.\n    pub async fn try_bind(self, addr: impl Into<SocketAddr>) {\n        let addr = addr.into();\n        let srv = match try_bind!(self, &addr) {\n            Ok((_, srv)) => srv,\n            Err(err) => {\n                tracing::error!(\"error binding to {}: {}\", addr, err);\n                return;\n            }\n        };\n\n        srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        })\n        .await;\n    }\n\n    /// Bind to a possibly ephemeral socket address.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        (addr, srv)\n    }\n\n    /// Tried to bind a possibly ephemeral socket address.\n    ///\n    /// Returns a `Result` which fails in case we are unable to bind with the\n    /// underlying error.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    pub fn try_bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use warp::Filter;\n    /// use futures_util::future::TryFutureExt;\n    /// use tokio::sync::oneshot;\n    ///\n    /// # fn main() {\n    /// let routes = warp::any()\n    ///     .map(|| \"Hello, World!\");\n    ///\n    /// let (tx, rx) = oneshot::channel();\n    ///\n    /// let (addr, server) = warp::serve(routes)\n    ///     .bind_with_graceful_shutdown(([127, 0, 0, 1], 3030), async {\n    ///          rx.await.ok();\n    ///     });\n    ///\n    /// // Spawn the server into a runtime\n    /// tokio::task::spawn(server);\n    ///\n    /// // Later, start the shutdown...\n    /// let _ = tx.send(());\n    /// # }\n    /// ```\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let fut = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n        (addr, fut)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    pub fn try_bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming<I>(self, incoming: I) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        self.serve_incoming2(incoming)\n            .instrument(tracing::info_span!(\"Server::serve_incoming\"))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections and a\n    /// signal to initiate graceful shutdown.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming_with_graceful_shutdown<I>(\n        self,\n        incoming: I,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        let service = into_service!(self.filter);\n        let pipeline = self.pipeline;\n\n        async move {\n            let srv =\n                HyperServer::builder(hyper::server::accept::from_stream(incoming.into_stream()))\n                    .http1_pipeline_flush(pipeline)\n                    .serve(service)\n                    .with_graceful_shutdown(signal)\n                    .await;\n\n            if let Err(err) = srv {\n                tracing::error!(\"server error: {}\", err);\n            }\n        }\n        .instrument(tracing::info_span!(\n            \"Server::serve_incoming_with_graceful_shutdown\"\n        ))\n    }\n\n    async fn serve_incoming2<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: Transport + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let service = into_service!(self.filter);\n\n        let srv = HyperServer::builder(hyper::server::accept::from_stream(incoming.into_stream()))\n            .http1_pipeline_flush(self.pipeline)\n            .serve(service)\n            .await;\n\n        if let Err(err) = srv {\n            tracing::error!(\"server error: {}\", err);\n        }\n    }\n\n    // Generally shouldn't be used, as it can slow down non-pipelined responses.\n    //\n    // It's only real use is to make silly pipeline benchmarks look better.\n    #[doc(hidden)]\n    pub fn unstable_pipeline(mut self) -> Self {\n        self.pipeline = true;\n        self\n    }\n\n    /// Configure a server to use TLS.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    #[cfg(feature = \"tls\")]\n    pub fn tls(self) -> TlsServer<F> {\n        TlsServer {\n            server: self,\n            tls: TlsConfigBuilder::new(),\n        }\n    }\n}\n\n// // ===== impl TlsServer =====\n\n#[cfg(feature = \"tls\")]\nimpl<F> TlsServer<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    // TLS config methods\n\n    /// Specify the file path to read the private key.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n...\n// Path: src/test.rs\n//! Test utilities to test your filters.\n//!\n//! [`Filter`](../trait.Filter.html)s can be easily tested without starting up an HTTP\n//! server, by making use of the [`RequestBuilder`](./struct.RequestBuilder.html) in this\n//! module.\n//!\n//! # Testing Filters\n//!\n//! It's easy to test filters, especially if smaller filters are used to build\n//! up your full set. Consider these example filters:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! fn sum() -> impl Filter<Extract = (u32,), Error = warp::Rejection> + Copy {\n//!     warp::path::param()\n//!         .and(warp::path::param())\n//!         .map(|x: u32, y: u32| {\n//!             x + y\n//!         })\n//! }\n//!\n//! fn math() -> impl Filter<Extract = (String,), Error = warp::Rejection> + Copy {\n//!     warp::post()\n//!         .and(sum())\n//!         .map(|z: u32| {\n//!             format!(\"Sum = {}\", z)\n//!         })\n//! }\n//! ```\n//!\n//! We can test some requests against the `sum` filter like this:\n//!\n//! ```\n//! # use warp::Filter;\n//! #[tokio::test]\n//! async fn test_sum() {\n//! #    let sum = || warp::any().map(|| 3);\n//!     let filter = sum();\n//!\n//!     // Execute `sum` and get the `Extract` back.\n//!     let value = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .filter(&filter)\n//!         .await\n//!         .unwrap();\n//!     assert_eq!(value, 3);\n//!\n//!     // Or simply test if a request matches (doesn't reject).\n//!     assert!(\n//!         warp::test::request()\n//!             .path(\"/1/-5\")\n//!             .matches(&filter)\n//!             .await\n//!     );\n//! }\n//! ```\n//!\n//! If the filter returns something that implements `Reply`, and thus can be\n//! turned into a response sent back to the client, we can test what exact\n//! response is returned. The `math` filter uses the `sum` filter, but returns\n//! a `String` that can be turned into a response.\n//!\n//! ```\n//! # use warp::Filter;\n//! #[test]\n//! fn test_math() {\n//! #    let math = || warp::any().map(warp::reply);\n//!     let filter = math();\n//!\n//!     let res = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 405, \"GET is not allowed\");\n//!\n//!     let res = warp::test::request()\n//!         .method(\"POST\")\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 200);\n//!     assert_eq!(res.body(), \"Sum is 3\");\n//! }\n//! ```\nuse std::convert::TryFrom;\nuse std::error::Error as StdError;\nuse std::fmt;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"websocket\")]\nuse std::pin::Pin;\n#[cfg(feature = \"websocket\")]\nuse std::task::Context;\n#[cfg(feature = \"websocket\")]\nuse std::task::{self, Poll};\n\nuse bytes::Bytes;\n#[cfg(feature = \"websocket\")]\nuse futures_channel::mpsc;\n#[cfg(feature = \"websocket\")]\nuse futures_util::StreamExt;\nuse futures_util::{future, FutureExt, TryFutureExt};\nuse http::{\n    header::{HeaderName, HeaderValue},\n    Response,\n};\nuse serde::Serialize;\n#[cfg(feature = \"websocket\")]\nuse tokio::sync::oneshot;\n\nuse crate::filter::Filter;\n#[cfg(feature = \"websocket\")]\nuse crate::filters::ws::Message;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::route::{self, Route};\nuse crate::Request;\n#[cfg(feature = \"websocket\")]\nuse crate::{Sink, Stream};\n\nuse self::inner::OneOrTuple;\n\n/// Starts a new test `RequestBuilder`.\npub fn request() -> RequestBuilder {\n    RequestBuilder {\n        remote_addr: None,\n        req: Request::default(),\n    }\n}\n\n/// Starts a new test `WsBuilder`.\n#[cfg(feature = \"websocket\")]\npub fn ws() -> WsBuilder {\n    WsBuilder { req: request() }\n}\n\n/// A request builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[must_use = \"RequestBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct RequestBuilder {\n    remote_addr: Option<SocketAddr>,\n    req: Request,\n}\n\n/// A Websocket builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[cfg(feature = \"websocket\")]\n#[must_use = \"WsBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct WsBuilder {\n    req: RequestBuilder,\n}\n\n/// A test client for Websocket filters.\n#[cfg(feature = \"websocket\")]\npub struct WsClient {\n    tx: mpsc::UnboundedSender<crate::ws::Message>,\n    rx: mpsc::UnboundedReceiver<Result<crate::ws::Message, crate::error::Error>>,\n}\n\n/// An error from Websocket filter tests.\n#[derive(Debug)]\npub struct WsError {\n    cause: Box<dyn StdError + Send + Sync>,\n}\n\nimpl RequestBuilder {\n    /// Sets the method of this builder.\n    ///\n    /// The default if not set is `GET`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .method(\"POST\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Method`.\n    pub fn method(mut self, method: &str) -> Self {\n        *self.req.method_mut() = method.parse().expect(\"valid method\");\n        self\n    }\n\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .path(\"/todos/33\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(mut self, p: &str) -> Self {\n        let uri = p.parse().expect(\"test request path invalid\");\n        *self.req.uri_mut() = uri;\n        self\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .header(\"accept\", \"application/json\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(mut self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        let name: HeaderName = TryFrom::try_from(key)\n            .map_err(|_| ())\n            .expect(\"invalid header name\");\n        let value = TryFrom::try_from(value)\n            .map_err(|_| ())\n            .expect(\"invalid header value\");\n        self.req.headers_mut().insert(name, value);\n        self\n    }\n\n    /// Set the remote address of this request\n    ///\n    /// Default is no remote address.\n    ///\n    /// # Example\n    /// ```\n    /// use std::net::{IpAddr, Ipv4Addr, SocketAddr};\n    ///\n    /// let req = warp::test::request()\n    ///     .remote_addr(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080));\n    /// ```\n    pub fn remote_addr(mut self, addr: SocketAddr) -> Self {\n        self.remote_addr = Some(addr);\n        self\n    }\n\n    /// Add a type to the request's `http::Extensions`.\n    pub fn extension<T>(mut self, ext: T) -> Self\n    where\n        T: Send + Sync + 'static,\n    {\n        self.req.extensions_mut().insert(ext);\n        self\n    }\n\n    /// Set the bytes of this request body.\n    ///\n    /// Default is an empty body.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .body(\"foo=bar&baz=quux\");\n    /// ```\n    pub fn body(mut self, body: impl AsRef<[u8]>) -> Self {\n        let body = body.as_ref().to_vec();\n        let len = body.len();\n        *self.req.body_mut() = body.into();\n        self.header(\"content-length\", len.to_string())\n    }\n\n    /// Set the bytes of this request body by serializing a value into JSON.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .json(&true);\n    /// ```\n    pub fn json(mut self, val: &impl Serialize) -> Self {\n        let vec = serde_json::to_vec(val).expect(\"json() must serialize to JSON\");\n        let len = vec.len();\n        *self.req.body_mut() = vec.into();\n        self.header(\"content-length\", len.to_string())\n            .header(\"content-type\", \"application/json\")\n    }\n\n    /// Tries to apply the `Filter` on this request.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let param = warp::path::param::<u32>();\n    ///\n    ///     let ex = warp::test::request()\n    ///         .path(\"/41\")\n    ///         .filter(&param)\n    ///         .await\n    ///         .unwrap();\n    ///\n    ///     assert_eq!(ex, 41);\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .path(\"/foo\")\n    ///             .filter(&param)\n    ///             .await\n    ///             .is_err()\n    ///     );\n    ///};\n    /// ```\n    pub async fn filter<F>(self, f: &F) -> Result<<F::Extract as OneOrTuple>::Output, F::Error>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: OneOrTuple + Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.map(|ex| ex.one_or_tuple())\n    }\n\n    /// Returns whether the `Filter` matches this request, or rejects it.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let get = warp::get();\n    ///     let post = warp::post();\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&get)\n    ///             .await\n    ///     );\n    ///\n    ///     assert!(\n    ///         !warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&post)\n    ///             .await\n    ///     );\n    ///};\n    /// ```\n    pub async fn matches<F>(self, f: &F) -> bool\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.is_ok()\n    }\n\n    /// Returns `Response` provided by applying the `Filter`.\n    ///\n    /// This requires that the supplied `Filter` return a [`Reply`].\n    pub async fn reply<F>(self, f: &F) -> Response<Bytes>\n    where\n        F: Filter + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        // TODO: de-duplicate this and apply_filter()\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(\n            route::set(&route, move || f.filter(crate::filter::Internal)).then(|result| {\n                let res = match result {\n                    Ok(rep) => rep.into_response(),\n                    Err(rej) => {\n                        tracing::debug!(\"rejected: {:?}\", rej);\n                        rej.into_response()\n                    }\n                };\n                let (parts, body) = res.into_parts();\n                hyper::body::to_bytes(body).map_ok(|chunk| Response::from_parts(parts, chunk))\n            }),\n        );\n\n        let fut = future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)));\n\n        fut.await.expect(\"reply shouldn't fail\")\n    }\n\n    fn apply_filter<F>(self, f: &F) -> impl Future<Output = Result<F::Extract, F::Error>>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(route::set(&route, move || {\n            f.filter(crate::filter::Internal)\n        }));\n        future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)))\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl WsBuilder {\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .path(\"/chat\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(self, p: &str) -> Self {\n        WsBuilder {\n            req: self.req.path(p),\n        }\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .header(\"foo\", \"bar\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        WsBuilder {\n            req: self.req.header(key, value),\n        }\n    }\n\n    /// Execute this Websocket request against the provided filter.\n    ///\n    /// If the handshake succeeds, returns a `WsClient`.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use futures_util::future;\n    /// use warp::Filter;\n    /// #[tokio::main]\n    /// # async fn main() {\n    ///\n    /// // Some route that accepts websockets (but drops them immediately).\n    /// let route = warp::ws()\n    ///     .map(|ws: warp::ws::Ws| {\n    ///         ws.on_upgrade(|_| future::ready(()))\n    ///     });\n    ///\n    /// let client = warp::test::ws()\n    ///     .handshake(route)\n    ///     .await\n    ///     .expect(\"handshake\");\n    /// # }\n    /// ```\n    pub async fn handshake<F>(self, f: F) -> Result<WsClient, WsError>\n    where\n        F: Filter + Clone + Send + Sync + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        let (upgraded_tx, upgraded_rx) = oneshot::channel();\n        let (wr_tx, wr_rx) = mpsc::unbounded();\n        let (rd_tx, rd_rx) = mpsc::unbounded();\n\n        tokio::spawn(async move {\n            use tokio_tungstenite::tungstenite::protocol;\n\n            let (addr, srv) = crate::serve(f).bind_ephemeral(([127, 0, 0, 1], 0));\n\n            let mut req = self\n                .req\n                .header(\"connection\", \"upgrade\")\n                .header(\"upgrade\", \"websocket\")\n                .header(\"sec-websocket-version\", \"13\")\n                .header(\"sec-websocket-key\", \"dGhlIHNhbXBsZSBub25jZQ==\")\n                .req;\n\n            let query_string = match req.uri().query() {\n                Some(q) => format!(\"?{}\", q),\n                None => String::from(\"\"),\n            };\n\n            let uri = format!(\"http://{}{}{}\", addr, req.uri().path(), query_string)\n                .parse()\n                .expect(\"addr + path is valid URI\");\n\n            *req.uri_mut() = uri;\n\n            // let mut rt = current_thread::Runtime::new().unwrap();\n            tokio::spawn(srv);\n\n            let upgrade = ::hyper::Client::builder()\n                .build(AddrConnect(addr))\n                .request(req)\n                .and_then(hyper::upgrade::on);\n\n            let upgraded = match upgrade.await {\n                Ok(up) => {\n                    let _ = upgraded_tx.send(Ok(()));\n                    up\n                }\n                Err(err) => {\n                    let _ = upgraded_tx.send(Err(err));\n                    return;\n                }\n            };\n            let ws = crate::ws::WebSocket::from_raw_socket(\n                upgraded,\n                protocol::Role::Client,\n                Default::default(),\n            )\n            .await;\n\n            let (tx, rx) = ws.split();\n            let write = wr_rx.map(Ok).forward(tx).map(|_| ());\n\n            let read = rx\n                .take_while(|result| match result {\n                    Err(_) => future::ready(false),\n                    Ok(m) => future::ready(!m.is_close()),\n                })\n                .for_each(move |item| {\n                    rd_tx.unbounded_send(item).expect(\"ws receive error\");\n                    future::ready(())\n                });\n\n            future::join(write, read).await;\n        });\n\n        match upgraded_rx.await {\n            Ok(Ok(())) => Ok(WsClient {\n                tx: wr_tx,\n                rx: rd_rx,\n            }),\n            Ok(Err(err)) => Err(WsError::new(err)),\n            Err(_canceled) => panic!(\"websocket handshake thread panicked\"),\n        }\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl WsClient {\n    /// Send a \"text\" websocket message to the server.\n    pub async fn send_text(&mut self, text: impl Into<String>) {\n        self.send(crate::ws::Message::text(text)).await;\n    }\n\n    /// Send a websocket message to the server.\n    pub async fn send(&mut self, msg: crate::ws::Message) {\n        self.tx.unbounded_send(msg).unwrap();\n    }\n\n    /// Receive a websocket message from the server.\n    pub async fn recv(&mut self) -> Result<crate::filters::ws::Message, WsError> {\n        self.rx\n            .next()\n            .await\n            .map(|result| result.map_err(WsError::new))\n            .unwrap_or_else(|| {\n                // websocket is closed\n                Err(WsError::new(\"closed\"))\n            })\n    }\n\n    /// Assert the server has closed the connection.\n    pub async fn recv_closed(&mut self) -> Result<(), WsError> {\n        self.rx\n            .next()\n            .await\n            .map(|result| match result {\n                Ok(msg) => Err(WsError::new(format!(\"received message: {:?}\", msg))),\n                Err(err) => Err(WsError::new(err)),\n            })\n            .unwrap_or_else(|| {\n                // closed successfully\n                Ok(())\n            })\n    }\n\n    fn pinned_tx(self: Pin<&mut Self>) -> Pin<&mut mpsc::UnboundedSender<crate::ws::Message>> {\n        let this = Pin::into_inner(self);\n        Pin::new(&mut this.tx)\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl fmt::Debug for WsClient {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"WsClient\").finish()\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl Sink<crate::ws::Message> for WsClient {\n    type Error = WsError;\n\n    fn poll_ready(\n        self: Pin<&mut Self>,\n        context: &mut Context<'_>,\n    ) -> Poll<Result<(), Self::Error>> {\n        self.pinned_tx().poll_ready(context).map_err(WsError::new)\n    }\n\n    fn start_send(self: Pin<&mut Self>, message: Message) -> Result<(), Self::Error> {\n        self.pinned_tx().start_send(message).map_err(WsError::new)\n    }\n\n    fn poll_flush(\n        self: Pin<&mut Self>,\n        context: &mut Context<'_>,\n    ) -> Poll<Result<(), Self::Error>> {\n        self.pinned_tx().poll_flush(context).map_err(WsError::new)\n    }\n\n    fn poll_close(\n        self: Pin<&mut Self>,\n        context: &mut Context<'_>,\n    ) -> Poll<Result<(), Self::Error>> {\n        self.pinned_tx().poll_close(context).map_err(WsError::new)\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl Stream for WsClient {\n    type Item = Result<crate::ws::Message, WsError>;\n\n    fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Option<Self::Item>> {\n        let this = Pin::into_inner(self);\n        let rx = Pin::new(&mut this.rx);\n        match rx.poll_next(context) {\n            Poll::Ready(Some(result)) => Poll::Ready(Some(result.map_err(WsError::new))),\n            Poll::Ready(None) => Poll::Ready(None),\n            Poll::Pending => Poll::Pending,\n        }\n    }\n}\n\n// ===== impl WsError =====\n\n#[cfg(feature = \"websocket\")]\nimpl WsError {\n    fn new<E: Into<Box<dyn StdError + Send + Sync>>>(cause: E) -> Self {\n        WsError {\n            cause: cause.into(),\n        }\n    }\n}\n\nimpl fmt::Display for WsError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"websocket error: {}\", self.cause)\n    }\n}\n\nimpl StdError for WsError {\n    \nfn description(&self) -> &str {\n        \"websocket error\"\n    }\n}\n\n// ===== impl AddrConnect =====\n\n#[cfg(feature = \"websocket\")]\n#[derive(Clone)]\nstruct AddrConnect(SocketAddr);\n\n#[cfg(feature = \"websocket\")]\nimpl tower_service::Service<::http::Uri> for AddrConnect {\n    type Response = ::tokio::net::TcpStream;\n    type Error = ::std::io::Error;\n    type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;\n\n    fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll<Result<(), Self::Error>> {\n        Poll::Ready(Ok(()))\n    }\n\n    fn call(&mut self, _: ::http::Uri) -> Self::Future {\n        Box::pin(tokio::net::TcpStream::connect(self.0))\n    }\n}\n\nmod inner {\n    pub trait OneOrTuple {\n        type Output;\n\n        fn one_or_tuple(self) -> Self::Output;\n    }\n\n    impl OneOrTuple for () {\n        type Output = ();\n        fn one_or_tuple(self) -> Self::Output {}\n    }\n\n    macro_rules! one_or_tuple {\n        ($type1:ident) => {\n            impl<$type1> OneOrTuple for ($type1,) {\n                type Output = $type1;\n                fn one_or_tuple(self) -> Self::Output {\n                    self.0\n                }\n            }\n        };\n        ($type1:ident, $( $type:ident ),*) => {\n            one_or_tuple!($( $type ),*);\n\n            impl<$type1, $($type),*> OneOrTuple for ($type1, $($type),*) {\n                type Output = Self;\n                fn one_or_tuple(self) -> Self::Output {\n                    self\n                }\n            }\n        }\n    }\n\n    one_or_tuple! {\n        T1,\n        T2,\n        T3,\n        T4,\n        T5,\n        T6,\n        T7,\n        T8,\n        T9,\n        T10,\n        T11,\n        T12,\n        T13,\n        T14,\n        T15,\n        T16\n    }\n}\n\n// Path: src/lib.rs\n#![deny(missing_docs)]\n#![deny(missing_debug_implementations)]\n#![deny(rust_2018_idioms)]\n#![cfg_attr(test, deny(warnings))]\n\n//! # warp\n//!\n//! warp is a super-easy, composable, web server framework for warp speeds.\n//!\n//! Thanks to its [`Filter`][Filter] system, warp provides these out of the box:\n//!\n//! - Path routing and parameter extraction\n//! - Header requirements and extraction\n//! - Query string deserialization\n//! - JSON and Form bodies\n//! - Multipart form data\n//! - Static Files and Directories\n//! - Websockets\n//! - Access logging\n//! - Etc\n//!\n//! Since it builds on top of [hyper](https://hyper.rs), you automatically get:\n//!\n//! - HTTP/1\n//! - HTTP/2\n//! - Asynchronous\n//! - One of the fastest HTTP implementations\n//! - Tested and **correct**\n//!\n//! ## Filters\n//!\n//! The main concept in warp is the [`Filter`][Filter], which allows composition\n//! to describe various endpoints in your web service. Besides this powerful\n//! trait, warp comes with several built in [filters](filters/index.html), which\n//! can be combined for your specific needs.\n//!\n//! As a small example, consider an endpoint that has path and header requirements:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! let hi = warp::path(\"hello\")\n//!     .and(warp::path::param())\n//!     .and(warp::header(\"user-agent\"))\n//!     .map(|param: String, agent: String| {\n//!         format!(\"Hello {}, whose agent is {}\", param, agent)\n//!     });\n//! ```\n//!\n//! This example composes several [`Filter`s][Filter] together using `and`:\n//!\n//! - A path prefix of \"hello\"\n//! - A path parameter of a `String`\n//! - The `user-agent` header parsed as a `String`\n//!\n//! These specific filters will [`reject`][reject] requests that don't match\n//! their requirements.\n//!\n//! This ends up matching requests like:\n//!\n//! ```notrust\n//! GET /hello/sean HTTP/1.1\n//! Host: hyper.rs\n//! User-Agent: reqwest/v0.8.6\n//!\n//! ```\n//! And it returns a response similar to this:\n//!\n//! ```notrust\n//! HTTP/1.1 200 OK\n//! Content-Length: 41\n//! Date: ...\n//!\n//! Hello sean, whose agent is reqwest/v0.8.6\n//! ```\n//!\n//! Take a look at the full list of [`filters`](filters/index.html) to see what\n//! you can build.\n//!\n//! ## Testing\n//!\n//! Testing your web services easily is extremely important, and warp provides\n//! a [`test`](mod@self::test) module to help send mocked requests through your service.\n//!\n//! [Filter]: trait.Filter.html\n//! [reject]: reject/index.html\n\n#[macro_use]\nmod error;\nmod filter;\npub mod filters;\nmod generic;\npub mod redirect;\npub mod reject;\npub mod reply;\nmod route;\nmod server;\nmod service;\npub mod test;\n#[cfg(feature = \"tls\")]\nmod tls;\nmod transport;\n\npub use self::error::Error;\npub use self::filter::Filter;\n// This otherwise shows a big dump of re-exports in the doc homepage,\n// with zero context, so just hide it from the docs. Doc examples\n// on each can show that a convenient import exists.\n#[cfg(feature = \"compression\")]\n#[doc(hidden)]\npub use self::filters::compression;\n#[cfg(feature = \"multipart\")]\n#[doc(hidden)]\npub use self::filters::multipart;\n#[cfg(feature = \"websocket\")]\n#[doc(hidden)]\npub use self::filters::ws;\n#[doc(hidden)]\npub use self::filters::{\n    addr,\n    // any() function\n    any::any,\n    body,\n    cookie,\n    // cookie() function\n    cookie::cookie,\n    cors,\n    // cors() function\n    cors::cors,\n    ext,\n    fs,\n    header,\n    // header() function\n    header::header,\n    host,\n    log,\n    // log() function\n    log::log,\n    method::{delete, get, head, method, options, patch, post, put},\n    path,\n    // path() function and macro\n    path::path,\n    query,\n    // query() function\n    query::query,\n    sse,\n    trace,\n    // trace() function\n    trace::trace,\n};\n// ws() function\npub use self::filter::wrap_fn;\n#[cfg(feature = \"websocket\")]\n#[doc(hidden)]\npub use self::filters::ws::ws;\n#[doc(hidden)]\npub use self::redirect::redirect;\n#[doc(hidden)]\n#[allow(deprecated)]\npub use self::reject::{reject, Rejection};\n#[doc(hidden)]\npub use self::reply::{reply, Reply};\n#[cfg(feature = \"tls\")]\npub use self::server::TlsServer;\npub use self::server::{serve, Server};\npub use self::service::service;\n#[doc(hidden)]\npub use http;\n#[doc(hidden)]\npub use hyper;\n\n#[doc(hidden)]\npub use bytes::Buf;\n#[doc(hidden)]\npub use futures_util::{Future, Sink, Stream};\n#[doc(hidden)]\n\npub(crate) type Request = http::Request<hyper::Body>;\n\n", "needle_token_start": 10568, "needle_token_end": 10587, "code_context_ntokens": 12659, "output": ["```rust\nimpl WsError {\n    fn description(&self) -> &str {\n        \"websocket error\"\n    }\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "convert_reply", "language": "rust", "path": "src/filters/trace.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to transform a generic reply into a specialized traced response format.\n2. **Input**: It accepts any object that implements a specific response interface.\n3. **Output**: It returns a tuple containing a traced version of the response.\n4. **Procedure**: The function takes the input, converts it into a response format, and then wraps this response in a traced structure before returning it as a tuple.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/reply.rs\n//! Reply to requests.\n//!\n//! A [`Reply`](./trait.Reply.html) is a type that can be converted into an HTTP\n//! response to be sent to the client. These are typically the successful\n//! counterpart to a [rejection](../reject).\n//!\n//! The functions in this module are helpers for quickly creating a reply.\n//! Besides them, you can return a type that implements [`Reply`](./trait.Reply.html). This\n//! could be any of the following:\n//!\n//! - [`http::Response<impl Into<hyper::Body>>`](https://docs.rs/http)\n//! - `String`\n//! - `&'static str`\n//! - `http::StatusCode`\n//!\n//! # Example\n//!\n//! ```\n//! use warp::{Filter, http::Response};\n//!\n//! // Returns an empty `200 OK` response.\n//! let empty_200 = warp::any().map(warp::reply);\n//!\n//! // Returns a `200 OK` response with custom header and body.\n//! let custom = warp::any().map(|| {\n//!     Response::builder()\n//!         .header(\"my-custom-header\", \"some-value\")\n//!         .body(\"and a custom body\")\n//! });\n//!\n//! // GET requests return the empty 200, POST return the custom.\n//! let routes = warp::get().and(empty_200)\n//!     .or(warp::post().and(custom));\n//! ```\n\nuse std::borrow::Cow;\nuse std::convert::TryFrom;\n\nuse crate::generic::{Either, One};\nuse http::header::{HeaderName, HeaderValue, CONTENT_TYPE};\nuse http::StatusCode;\nuse hyper::Body;\nuse serde::Serialize;\n\n// This re-export just looks weird in docs...\npub(crate) use self::sealed::Reply_;\nuse self::sealed::{BoxedReply, Internal};\n#[doc(hidden)]\npub use crate::filters::reply as with;\n\n/// Response type into which types implementing the `Reply` trait are convertable.\npub type Response = ::http::Response<Body>;\n\n/// Returns an empty `Reply` with status code `200 OK`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // GET /just-ok returns an empty `200 OK`.\n/// let route = warp::path(\"just-ok\")\n///     .map(|| {\n///         println!(\"got a /just-ok request!\");\n///         warp::reply()\n///     });\n/// ```\n#[inline]\npub fn reply() -> impl Reply {\n    StatusCode::OK\n}\n\n/// Convert the value into a `Reply` with the value encoded as JSON.\n///\n/// The passed value must implement [`Serialize`][ser]. Many\n/// collections do, and custom domain types can have `Serialize` derived.\n///\n/// [ser]: https://serde.rs\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // GET /ids returns a `200 OK` with a JSON array of ids:\n/// // `[1, 3, 7, 13]`\n/// let route = warp::path(\"ids\")\n///     .map(|| {\n///         let our_ids = vec![1, 3, 7, 13];\n///         warp::reply::json(&our_ids)\n///     });\n/// ```\n///\n/// # Note\n///\n/// If a type fails to be serialized into JSON, the error is logged at the\n/// `error` level, and the returned `impl Reply` will be an empty\n/// `500 Internal Server Error` response.\npub fn json<T>(val: &T) -> Json\nwhere\n    T: Serialize,\n{\n    Json {\n...\n// Path: src/filter/service.rs\nuse std::convert::Infallible;\nuse std::future::Future;\nuse std::net::SocketAddr;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse futures_util::future::TryFuture;\nuse hyper::service::Service;\nuse pin_project::pin_project;\n\nuse crate::reject::IsReject;\nuse crate::reply::{Reply, Response};\nuse crate::route::{self, Route};\nuse crate::{Filter, Request};\n\n/// Convert a `Filter` into a `Service`.\n///\n/// Filters are normally what APIs are built on in warp. However, it can be\n/// useful to convert a `Filter` into a [`Service`][Service], such as if\n/// further customizing a `hyper::Service`, or if wanting to make use of\n/// the greater [Tower][tower] set of middleware.\n///\n/// # Example\n///\n/// Running a `warp::Filter` on a regular `hyper::Server`:\n///\n/// ```\n/// # async fn run() -> Result<(), Box<dyn std::error::Error>> {\n/// use std::convert::Infallible;\n/// use warp::Filter;\n///\n/// // Our Filter...\n/// let route = warp::any().map(|| \"Hello From Warp!\");\n///\n/// // Convert it into a `Service`...\n/// let svc = warp::service(route);\n///\n/// // Typical hyper setup...\n/// let make_svc = hyper::service::make_service_fn(move |_| async move {\n///     Ok::<_, Infallible>(svc)\n/// });\n///\n/// hyper::Server::bind(&([127, 0, 0, 1], 3030).into())\n///     .serve(make_svc)\n///     .await?;\n/// # Ok(())\n/// # }\n/// ```\n///\n/// [Service]: https://docs.rs/hyper/0.13.*/hyper/service/trait.Service.html\n/// [tower]: https://docs.rs/tower\npub fn service<F>(filter: F) -> FilteredService<F>\nwhere\n    F: Filter,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    FilteredService { filter }\n}\n\n#[derive(Copy, Clone, Debug)]\npub struct FilteredService<F> {\n    filter: F,\n}\n\nimpl<F> FilteredService<F>\nwhere\n    F: Filter,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    #[inline]\n    pub(crate) fn call_with_addr(\n        &self,\n        req: Request,\n        remote_addr: Option<SocketAddr>,\n    ) -> FilteredFuture<F::Future> {\n        debug_assert!(!route::is_set(), \"nested route::set calls\");\n\n        let route = Route::new(req, remote_addr);\n        let fut = route::set(&route, || self.filter.filter(super::Internal));\n        FilteredFuture { future: fut, route }\n    }\n}\n\nimpl<F> Service<Request> for FilteredService<F>\nwhere\n    F: Filter,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    type Response = Response;\n    type Error = Infallible;\n    type Future = FilteredFuture<F::Future>;\n\n    fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {\n        Poll::Ready(Ok(()))\n    }\n\n    #[inline]\n    fn call(&mut self, req: Request) -> Self::Future {\n        self.call_with_addr(req, None)\n    }\n}\n\n#[pin_project]\n#[derive(Debug)]\npub struct FilteredFuture<F> {\n    #[pin]\n    future: F,\n    route: ::std::cell::RefCell<Route>,\n}\n\nimpl<F> Future for FilteredFuture<F>\nwhere\n    F: TryFuture,\n    F::Ok: Reply,\n    F::Error: IsReject,\n{\n    type Output = Result<Response, Infallible>;\n\n    #[inline]\n    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n        debug_assert!(!route::is_set(), \"nested route::set calls\");\n\n        let pin = self.project();\n        let fut = pin.future;\n        match route::set(pin.route, || fut.try_poll(cx)) {\n            Poll::Ready(Ok(ok)) => Poll::Ready(Ok(ok.into_response())),\n            Poll::Pending => Poll::Pending,\n            Poll::Ready(Err(err)) => {\n                tracing::debug!(\"rejected: {:?}\", err);\n                Poll::Ready(Ok(err.into_response()))\n            }\n        }\n    }\n}\n\n// Path: src/filters/query.rs\n//! Query Filters\n\nuse futures_util::future;\nuse serde::de::DeserializeOwned;\n\nuse crate::filter::{filter_fn_one, Filter, One};\nuse crate::reject::{self, Rejection};\n\n/// Creates a `Filter` that decodes query parameters to the type `T`.\n///\n/// If cannot decode into a `T`, the request is rejected with a `400 Bad Request`.\n///\n/// # Example\n///\n/// ```\n/// use std::collections::HashMap;\n/// use warp::{\n///     http::Response,\n///     Filter,\n/// };\n///\n/// let route = warp::any()\n///     .and(warp::query::<HashMap<String, String>>())\n///     .map(|map: HashMap<String, String>| {\n///         let mut response: Vec<String> = Vec::new();\n///         for (key, value) in map.into_iter() {\n///             response.push(format!(\"{}={}\", key, value))\n///         }\n///         Response::builder().body(response.join(\";\"))\n///     });\n/// ```\n///\n/// You can define your custom query object and deserialize with [Serde][Serde]. Ensure to include\n/// the crate in your dependencies before usage.\n///\n/// ```\n/// use serde_derive::{Deserialize, Serialize};\n/// use std::collections::HashMap;\n/// use warp::{\n///     http::Response,\n///     Filter,\n/// };\n///\n/// #[derive(Serialize, Deserialize)]\n/// struct FooQuery {\n///     foo: Option<String>,\n///     bar: u8,\n/// }\n///\n/// let route = warp::any()\n///     .and(warp::query::<FooQuery>())\n///     .map(|q: FooQuery| {\n///         if let Some(foo) = q.foo {\n///             Response::builder().body(format!(\"foo={}\", foo))\n///         } else {\n///             Response::builder().body(format!(\"bar={}\", q.bar))\n///         }\n///     });\n/// ```\n///\n/// For more examples, please take a look at [examples/query_string.rs](https://github.com/seanmonstar/warp/blob/master/examples/query_string.rs).\n///\n/// [Serde]: https://docs.rs/serde\npub fn query<T: DeserializeOwned + Send + 'static>(\n) -> impl Filter<Extract = One<T>, Error = Rejection> + Copy {\n    filter_fn_one(|route| {\n        let query_string = route.query().unwrap_or_else(|| {\n            tracing::debug!(\"route was called without a query string, defaulting to empty\");\n            \"\"\n        });\n\n        let query_encoded = serde_urlencoded::from_str(query_string).map_err(|e| {\n            tracing::debug!(\"failed to decode query string '{}': {:?}\", query_string, e);\n            reject::invalid_query()\n        });\n        future::ready(query_encoded)\n    })\n}\n\n/// Creates a `Filter` that returns the raw query string as type String.\npub fn raw() -> impl Filter<Extract = One<String>, Error = Rejection> + Copy {\n    filter_fn_one(|route| {\n        let route = route\n            .query()\n            .map(|q| q.to_owned())\n            .map(Ok)\n            .unwrap_or_else(|| Err(reject::invalid_query()));\n        future::ready(route)\n    })\n}\n\n// Path: src/filters/any.rs\n//! A filter that matches any route.\nuse std::convert::Infallible;\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse crate::filter::{Filter, FilterBase, Internal};\n\n/// A [`Filter`](crate::Filter) that matches any route.\n///\n/// This can be a useful building block to build new filters from,\n/// since [`Filter`] is otherwise a sealed trait.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::any()\n///     .map(|| {\n///         \"I always return this string!\"\n///     });\n/// ```\n///\n/// This could allow creating a single `impl Filter` returning a specific\n/// reply, that can then be used as the end of several different filter\n/// chains.\n///\n/// Another use case is turning some clone-able resource into a `Filter`,\n/// thus allowing to easily `and` it together with others.\n///\n/// ```\n/// use std::sync::Arc;\n/// use warp::Filter;\n///\n/// let state = Arc::new(vec![33, 41]);\n/// let with_state = warp::any().map(move || state.clone());\n///\n/// // Now we could `and` with any other filter:\n///\n/// let route = warp::path::param()\n///     .and(with_state)\n///     .map(|param_id: u32, db: Arc<Vec<u32>>| {\n///         db.contains(&param_id)\n///     });\n/// ```\npub fn any() -> impl Filter<Extract = (), Error = Infallible> + Copy {\n    Any\n}\n\n#[derive(Copy, Clone)]\n#[allow(missing_debug_implementations)]\nstruct Any;\n\nimpl FilterBase for Any {\n    type Extract = ();\n    type Error = Infallible;\n    type Future = AnyFut;\n\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        AnyFut\n    }\n}\n\n#[allow(missing_debug_implementations)]\nstruct AnyFut;\n\nimpl Future for AnyFut {\n    type Output = Result<(), Infallible>;\n\n    #[inline]\n    fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> {\n        Poll::Ready(Ok(()))\n    }\n}\n\n// Path: src/redirect.rs\n//! Redirect requests to a new location.\n//!\n//! The types in this module are helpers that implement [`Reply`], and easy\n//! to use in order to setup redirects.\n\nuse http::{header, StatusCode};\n\npub use self::sealed::AsLocation;\nuse crate::reply::{self, Reply};\n\n/// HTTP 301 Moved Permanently\n/// Description: The requested resource has been permanently moved to a new URL.\n/// Usage: It is used when a URL has permanently moved to a new location. Search engines will update their index to the new URL. Browsers and clients will automatically cache this redirect, so subsequent requests for the old URL will automatically go to the new URL without making a request to the old URL.\n/// Common Use Case: Changing domain names, restructuring website URLs.\n///\n/// # Example\n///\n/// ```\n/// use warp::{http::Uri, Filter};\n///\n/// let route = warp::path(\"v1\")\n///     .map(|| {\n///         warp::redirect(Uri::from_static(\"/v2\"))\n///     });\n/// ```\npub fn redirect(uri: impl AsLocation) -> impl Reply {\n    reply::with_header(\n        StatusCode::MOVED_PERMANENTLY,\n        header::LOCATION,\n        uri.header_value(),\n    )\n}\n\n/// HTTP 302 Found (or Temporary Redirect)\n/// Description: The requested resource can be found at a different URL temporarily.\n/// Usage: Historically, this status code was used for temporary redirects. However, its meaning was often misunderstood, and different clients treated it differently. As a result, it is recommended to use 307 (or 303) for temporary redirects instead.\n/// Common Use Case: Rarely used directly due to ambiguity; replaced by 307 or 303.\n///\n/// # Example\n///\n/// ```\n/// use warp::{http::Uri, Filter};\n///\n/// let route = warp::path(\"v1\")\n///     .map(|| {\n///         warp::redirect::found(Uri::from_static(\"/v2\"))\n///     });\n/// ```\npub fn found(uri: impl AsLocation) -> impl Reply {\n    reply::with_header(StatusCode::FOUND, header::LOCATION, uri.header_value())\n}\n\n/// HTTP 303 See Other\n/// Description: The response to the request can be found at a different URL, and the client should retrieve it using the GET method.\n/// Usage: It is typically used to redirect the client to another URL using a GET request after processing a POST request. It ensures that the client doesn't repeat the POST request if they refresh the page.\n/// Common Use Case: After form submissions or any non-idempotent request.\n///\n/// The HTTP method of the request to the new location will always be `GET`.\n///\n/// # Example\n///\n/// ```\n/// use warp::{http::Uri, Filter};\n///\n/// let route = warp::path(\"v1\")\n///     .map(|| {\n///         warp::redirect::see_other(Uri::from_static(\"/v2\"))\n///     });\n/// ```\npub fn see_other(uri: impl AsLocation) -> impl Reply {\n    reply::with_header(StatusCode::SEE_OTHER, header::LOCATION, uri.header_value())\n}\n\n/// HTTP 307 Temporary Redirect:\n/// Description: The requested resource can be found at a different URL temporarily.\n/// Usage: Similar to 302, but explicitly defined as a temporary redirect. The main difference between 307 and 302 is that 307 preserves the method of the original request when redirecting. If the original request was a POST, the subsequent request to the new URL will also be a POST.\n/// Common Use Case: Temporary redirects that should preserve the original request method.\n///\n/// This is similar to [`see_other`](fn@see_other) but the HTTP method and the body of the request\n/// to the new location will be the same as the method and body of the current request.\n///\n/// # Example\n///\n/// ```\n/// use warp::{http::Uri, Filter};\n///\n/// let route = warp::path(\"v1\")\n///     .map(|| {\n///         warp::redirect::temporary(Uri::from_static(\"/v2\"))\n///     });\n/// ```\npub fn temporary(uri: impl AsLocation) -> impl Reply {\n    reply::with_header(\n        StatusCode::TEMPORARY_REDIRECT,\n        header::LOCATION,\n        uri.header_value(),\n    )\n}\n\n/// HTTP 308 Permanent Redirect\n/// Description: The requested resource has been permanently moved to a new URL, and future requests should use the new URL.\n/// Usage: Similar to 301, but like 307, it preserves the original request method when redirecting. It indicates that the redirection is permanent, and browsers and clients will cache this redirect like they do for 301.\n// Common Use Case: Permanently moving resources to a new URL while maintaining the original request method.\n///\n/// This is similar to [`redirect`](fn@redirect) but the HTTP method of the request to the new\n/// location will be the same as the method of the current request.\n///\n/// # Example\n///\n/// ```\n/// use warp::{http::Uri, Filter};\n///\n/// let route = warp::path(\"v1\")\n///     .map(|| {\n///         warp::redirect::permanent(Uri::from_static(\"/v2\"))\n///     });\n/// ```\npub fn permanent(uri: impl AsLocation) -> impl Reply {\n    reply::with_header(\n        StatusCode::PERMANENT_REDIRECT,\n        header::LOCATION,\n        uri.header_value(),\n    )\n}\n\nmod sealed {\n    use bytes::Bytes;\n    use http::{header::HeaderValue, Uri};\n\n    /// Trait for redirect locations. Currently only a `Uri` can be used in\n    /// redirect.\n    /// This sealed trait exists to allow adding possibly new impls so other\n    /// arguments could be accepted, like maybe just `warp::redirect(\"/v2\")`.\n    pub trait AsLocation: Sealed {}\n    pub trait Sealed {\n        fn header_value(self) -> HeaderValue;\n    }\n\n    impl AsLocation for Uri {}\n\n    impl Sealed for Uri {\n        fn header_value(self) -> HeaderValue {\n            let bytes = Bytes::from(self.to_string());\n            HeaderValue::from_maybe_shared(bytes).expect(\"Uri is a valid HeaderValue\")\n        }\n    }\n}\n\n// Path: src/filters/addr.rs\n//! Socket Address filters.\n\nuse std::convert::Infallible;\nuse std::net::SocketAddr;\n\nuse crate::filter::{filter_fn_one, Filter};\n\n/// Creates a `Filter` to get the remote address of the connection.\n///\n/// If the underlying transport doesn't use socket addresses, this will yield\n/// `None`.\n///\n/// # Example\n///\n/// ```\n/// use std::net::SocketAddr;\n/// use warp::Filter;\n///\n/// let route = warp::addr::remote()\n///     .map(|addr: Option<SocketAddr>| {\n///         println!(\"remote address = {:?}\", addr);\n///     });\n/// ```\npub fn remote() -> impl Filter<Extract = (Option<SocketAddr>,), Error = Infallible> + Copy {\n    filter_fn_one(|route| futures_util::future::ok(route.remote_addr()))\n}\n\n// Path: src/filters/method.rs\n//! HTTP Method filters.\n//!\n//! The filters deal with the HTTP Method part of a request. Several here will\n//! match the request `Method`, and if not matched, will reject the request\n//! with a `405 Method Not Allowed`.\n//!\n//! There is also [`warp::method()`](method), which never rejects\n//! a request, and just extracts the method to be used in your filter chains.\nuse futures_util::future;\nuse http::Method;\n\nuse crate::filter::{filter_fn, filter_fn_one, Filter, One};\nuse crate::reject::Rejection;\nuse std::convert::Infallible;\n\n/// Create a `Filter` that requires the request method to be `GET`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let get_only = warp::get().map(warp::reply);\n/// ```\npub fn get() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    method_is(|| &Method::GET)\n}\n\n/// Create a `Filter` that requires the request method to be `POST`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let post_only = warp::post().map(warp::reply);\n/// ```\npub fn post() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    method_is(|| &Method::POST)\n}\n\n/// Create a `Filter` that requires the request method to be `PUT`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let put_only = warp::put().map(warp::reply);\n/// ```\npub fn put() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    method_is(|| &Method::PUT)\n}\n\n/// Create a `Filter` that requires the request method to be `DELETE`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let delete_only = warp::delete().map(warp::reply);\n/// ```\npub fn delete() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    method_is(|| &Method::DELETE)\n}\n\n/// Create a `Filter` that requires the request method to be `HEAD`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let head_only = warp::head().map(warp::reply);\n/// ```\npub fn head() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    method_is(|| &Method::HEAD)\n}\n\n/// Create a `Filter` that requires the request method to be `OPTIONS`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let options_only = warp::options().map(warp::reply);\n/// ```\npub fn options() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    method_is(|| &Method::OPTIONS)\n}\n\n/// Create a `Filter` that requires the request method to be `PATCH`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let patch_only = warp::patch().map(warp::reply);\n/// ```\npub fn patch() -> impl Filter<Extract = (), Error = Rejection> + Copy {\n    method_is(|| &Method::PATCH)\n}\n\n/// Extract the `Method` from the request.\n///\n/// This never rejects a request.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::method()\n///     .map(|method| {\n///         format!(\"You sent a {} request!\", method)\n///     });\n/// ```\npub fn method() -> impl Filter<Extract = One<Method>, Error = Infallible> + Copy {\n    filter_fn_one(|route| future::ok::<_, Infallible>(route.method().clone()))\n}\n\n// NOTE: This takes a static function instead of `&'static Method` directly\n// so that the `impl Filter` can be zero-sized. Moving it around should be\n// cheaper than holding a single static pointer (which would make it 1 word).\nfn method_is<F>(func: F) -> impl Filter<Extract = (), Error = Rejection> + Copy\nwhere\n    F: Fn() -> &'static Method + Copy,\n{\n    filter_fn(move |route| {\n        let method = func();\n        tracing::trace!(\"method::{:?}?: {:?}\", method, route.method());\n        if route.method() == method {\n            future::ok(())\n        } else {\n            future::err(crate::reject::method_not_allowed())\n        }\n    })\n}\n\n#[cfg(test)]\nmod tests {\n    #[test]\n    fn method_size_of() {\n        // See comment on `method_is` function.\n        assert_eq!(std::mem::size_of_val(&super::get()), 0,);\n    }\n}\n\n// Path: src/filters/multipart.rs\n//! Multipart body filters\n//!\n//! [`Filter`](crate::Filter)s that extract a multipart body for a route.\n\nuse std::error::Error as StdError;\nuse std::fmt::{Display, Formatter};\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\nuse std::{fmt, io};\n\nuse bytes::{Buf, Bytes};\nuse futures_util::{future, Stream};\nuse headers::ContentType;\nuse hyper::Body;\nuse mime::Mime;\nuse multer::{Field as PartInner, Multipart as FormDataInner};\n\nuse crate::filter::{Filter, FilterBase, Internal};\nuse crate::reject::{self, Rejection};\n\n// If not otherwise configured, default to 2MB.\nconst DEFAULT_FORM_DATA_MAX_LENGTH: u64 = 1024 * 1024 * 2;\n\n/// A [`Filter`](crate::Filter) to extract a `multipart/form-data` body from a request.\n///\n/// Create with the `warp::multipart::form()` function.\n#[derive(Debug, Clone)]\npub struct FormOptions {\n    max_length: Option<u64>,\n}\n\n/// A `Stream` of multipart/form-data `Part`s.\n///\n/// Extracted with a `warp::multipart::form` filter.\npub struct FormData {\n    inner: FormDataInner<'static>,\n}\n\n/// A single \"part\" of a multipart/form-data body.\n///\n/// Yielded from the `FormData` stream.\npub struct Part {\n    part: PartInner<'static>,\n}\n\n/// Create a [`Filter`](crate::Filter) to extract a `multipart/form-data` body from a request.\n///\n/// The extracted `FormData` type is a `Stream` of `Part`s, and each `Part`\n/// in turn is a `Stream` of bytes.\npub fn form() -> FormOptions {\n    FormOptions {\n        max_length: Some(DEFAULT_FORM_DATA_MAX_LENGTH),\n    }\n}\n\n// ===== impl Form =====\n\nimpl FormOptions {\n    /// Set the maximum byte length allowed for this body.\n    ///\n    /// `max_length(None)` means that maximum byte length is not checked.\n    /// Defaults to 2MB.\n    pub fn max_length(mut self, max: impl Into<Option<u64>>) -> Self {\n        self.max_length = max.into();\n        self\n    }\n}\n\ntype FormFut = Pin<Box<dyn Future<Output = Result<(FormData,), Rejection>> + Send>>;\n\nimpl FilterBase for FormOptions {\n    type Extract = (FormData,);\n    type Error = Rejection;\n    type Future = FormFut;\n\n    fn filter(&self, _: Internal) -> Self::Future {\n        let boundary = super::header::header2::<ContentType>().and_then(|ct| {\n            let mime = Mime::from(ct);\n            let mime = mime\n                .get_param(\"boundary\")\n                .map(|v| v.to_string())\n                .ok_or_else(|| reject::invalid_header(\"content-type\"));\n            future::ready(mime)\n        });\n\n        let filt = boundary\n            .and(super::body::body())\n            .map(|boundary: String, body| {\n                let body = BodyIoError(body);\n                FormData {\n                    inner: FormDataInner::new(body, &boundary),\n                }\n            });\n\n        if let Some(max_length) = self.max_length {\n            Box::pin(\n                super::body::content_length_limit(max_length)\n                    .and(filt)\n                    .filter(Internal),\n            )\n        } else {\n            Box::pin(filt.filter(Internal))\n        }\n    }\n}\n\n// ===== impl FormData =====\n\nimpl fmt::Debug for FormData {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"FormData\").finish()\n    }\n}\n\nimpl Stream for FormData {\n    type Item = Result<Part, crate::Error>;\n\n    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {\n        match self.inner.poll_next_field(cx) {\n            Poll::Pending => Poll::Pending,\n            Poll::Ready(Ok(Some(part))) => {\n                if part.name().is_some() || part.file_name().is_some() {\n                    Poll::Ready(Some(Ok(Part { part })))\n                } else {\n                    Poll::Ready(Some(Err(crate::Error::new(MultipartFieldMissingName))))\n                }\n            }\n            Poll::Ready(Ok(None)) => Poll::Ready(None),\n            Poll::Ready(Err(err)) => Poll::Ready(Some(Err(crate::Error::new(err)))),\n        }\n    }\n}\n\n// ===== impl Part =====\n\nimpl Part {\n    /// Get the name of this part.\n    pub fn name(&self) -> &str {\n        self.part\n            .name()\n            .unwrap_or_else(|| self.part.file_name().expect(\"checked for name previously\"))\n    }\n\n    /// Get the filename of this part, if present.\n    pub fn filename(&self) -> Option<&str> {\n        self.part.file_name()\n    }\n\n    /// Get the content-type of this part, if present.\n    pub fn content_type(&self) -> Option<&str> {\n        let content_type = self.part.content_type();\n        content_type.map(|t| t.as_ref())\n    }\n\n    /// Asynchronously get some of the data for this `Part`.\n    pub async fn data(&mut self) -> Option<Result<impl Buf, crate::Error>> {\n        future::poll_fn(|cx| self.poll_next(cx)).await\n    }\n\n    /// Convert this `Part` into a `Stream` of `Buf`s.\n    pub fn stream(self) -> impl Stream<Item = Result<impl Buf, crate::Error>> {\n        PartStream(self)\n    }\n\n    fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll<Option<Result<Bytes, crate::Error>>> {\n        match Pin::new(&mut self.part).poll_next(cx) {\n            Poll::Pending => Poll::Pending,\n            Poll::Ready(Some(Ok(bytes))) => Poll::Ready(Some(Ok(bytes))),\n            Poll::Ready(None) => Poll::Ready(None),\n            Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(crate::Error::new(err)))),\n        }\n    }\n}\n\nimpl fmt::Debug for Part {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let mut builder = f.debug_struct(\"Part\");\n        builder.field(\"name\", &self.name());\n\n        if let Some(ref filename) = self.part.file_name() {\n            builder.field(\"filename\", filename);\n        }\n\n        if let Some(ref mime) = self.part.content_type() {\n            builder.field(\"content_type\", mime);\n        }\n\n        builder.finish()\n    }\n}\n\nstruct PartStream(Part);\n\nimpl Stream for PartStream {\n    type Item = Result<Bytes, crate::Error>;\n\n    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {\n        self.0.poll_next(cx)\n    }\n}\n\nstruct BodyIoError(Body);\n\nimpl Stream for BodyIoError {\n    type Item = io::Result<Bytes>;\n\n    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {\n        match Pin::new(&mut self.0).poll_next(cx) {\n            Poll::Pending => Poll::Pending,\n            Poll::Ready(Some(Ok(bytes))) => Poll::Ready(Some(Ok(bytes))),\n            Poll::Ready(None) => Poll::Ready(None),\n            Poll::Ready(Some(Err(err))) => {\n                Poll::Ready(Some(Err(io::Error::new(io::ErrorKind::Other, err))))\n            }\n        }\n    }\n}\n\n/// An error used when a multipart field is missing a name.\n#[derive(Debug)]\nstruct MultipartFieldMissingName;\n\nimpl Display for MultipartFieldMissingName {\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        write!(f, \"Multipart field is missing a name\")\n    }\n}\n\nimpl StdError for MultipartFieldMissingName {}\n\n// Path: src/filters/cookie.rs\n//! Cookie Filters\n\nuse futures_util::future;\nuse headers::Cookie;\n\nuse super::header;\nuse crate::filter::{Filter, One};\nuse crate::reject::Rejection;\nuse std::convert::Infallible;\nuse std::str::FromStr;\n\n/// Creates a `Filter` that requires a cookie by name.\n///\n/// If found, extracts the value of the cookie, otherwise rejects.\npub fn cookie<T>(name: &'static str) -> impl Filter<Extract = One<T>, Error = Rejection> + Copy\nwhere\n    T: FromStr + Send + 'static,\n{\n    header::header2().and_then(move |cookie: Cookie| {\n        let cookie = cookie\n            .get(name)\n            .ok_or_else(|| crate::reject::missing_cookie(name))\n            .and_then(|s| T::from_str(s).map_err(|_| crate::reject::missing_cookie(name)));\n        future::ready(cookie)\n    })\n}\n\n/// Creates a `Filter` that looks for an optional cookie by name.\n///\n/// If found, extracts the value of the cookie, otherwise continues\n/// the request, extracting `None`.\npub fn optional<T>(\n    name: &'static str,\n) -> impl Filter<Extract = One<Option<T>>, Error = Infallible> + Copy\nwhere\n    T: FromStr + Send + 'static,\n{\n    header::optional2().map(move |opt: Option<Cookie>| {\n        let cookie = opt.and_then(|cookie| cookie.get(name).map(|x| T::from_str(x)));\n        match cookie {\n            Some(Ok(t)) => Some(t),\n            Some(Err(_)) => None,\n            None => None,\n        }\n    })\n}\n\n// Path: src/filters/trace.rs\n//! [`tracing`] filters.\n//!\n//! [`tracing`] is a framework for instrumenting Rust programs to\n//! collect scoped, structured, and async-aware diagnostics. This module\n//! provides a set of filters for instrumenting Warp applications with `tracing`\n//! spans. [`Spans`] can be used to associate individual events  with a request,\n//! and track contexts through the application.\n//!\n//! [`tracing`]: https://crates.io/crates/tracing\n//! [`Spans`]: https://docs.rs/tracing/latest/tracing/#spans\nuse tracing::Span;\n\nuse std::net::SocketAddr;\n\nuse http::header;\n\nuse crate::filter::{Filter, WrapSealed};\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::route::Route;\n\nuse self::internal::WithTrace;\n\n/// Create a wrapping filter that instruments every request with a `tracing`\n/// [`Span`] at the [`INFO`] level, containing a summary of the request.\n/// Additionally, if the [`DEBUG`] level is enabled, the span will contain an\n/// event recording the request's headers.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(warp::trace::request());\n/// ```\n///\n/// [`Span`]: https://docs.rs/tracing/latest/tracing/#spans\n/// [`INFO`]: https://docs.rs/tracing/0.1.16/tracing/struct.Level.html#associatedconstant.INFO\n/// [`DEBUG`]: https://docs.rs/tracing/0.1.16/tracing/struct.Level.html#associatedconstant.DEBUG\npub fn request() -> Trace<impl Fn(Info<'_>) -> Span + Clone> {\n    use tracing::field::{display, Empty};\n    trace(|info: Info<'_>| {\n        let span = tracing::info_span!(\n            \"request\",\n            remote.addr = Empty,\n            method = %info.method(),\n            path = %info.path(),\n            version = ?info.route.version(),\n            referer = Empty,\n        );\n\n        // Record optional fields.\n        if let Some(remote_addr) = info.remote_addr() {\n            span.record(\"remote.addr\", &display(remote_addr));\n        }\n\n        if let Some(referer) = info.referer() {\n            span.record(\"referer\", &display(referer));\n        }\n\n        tracing::debug!(parent: &span, \"received request\");\n\n        span\n    })\n}\n\n/// Create a wrapping filter that instruments every request with a custom\n/// `tracing` [`Span`] provided by a function.\n///\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(warp::trace(|info| {\n///         // Create a span using tracing macros\n///         tracing::info_span!(\n///             \"request\",\n///             method = %info.method(),\n///             path = %info.path(),\n///         )\n///     }));\n/// ```\n///\n/// [`Span`]: https://docs.rs/tracing/latest/tracing/#spans\npub fn trace<F>(func: F) -> Trace<F>\nwhere\n    F: Fn(Info<'_>) -> Span + Clone,\n{\n    Trace { func }\n}\n\n/// Create a wrapping filter that instruments every request with a `tracing`\n/// [`Span`] at the [`DEBUG`] level representing a named context.\n///\n/// This can be used to instrument multiple routes with their own sub-spans in a\n/// per-request trace.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let hello = warp::path(\"hello\")\n///     .map(warp::reply)\n///     .with(warp::trace::named(\"hello\"));\n///\n/// let goodbye = warp::path(\"goodbye\")\n///     .map(warp::reply)\n///     .with(warp::trace::named(\"goodbye\"));\n///\n/// let routes = hello.or(goodbye);\n/// ```\n///\n/// [`Span`]: https://docs.rs/tracing/latest/tracing/#spans\n/// [`DEBUG`]: https://docs.rs/tracing/0.1.16/tracing/struct.Level.html#associatedconstant.DEBUG\npub fn named(name: &'static str) -> Trace<impl Fn(Info<'_>) -> Span + Copy> {\n    trace(move |_| tracing::debug_span!(\"context\", \"{}\", name,))\n}\n\n/// Decorates a [`Filter`] to create a [`tracing`] [span] for\n/// requests and responses.\n///\n/// [`tracing`]: https://crates.io/crates/tracing\n/// [span]: https://docs.rs/tracing/latest/tracing/#spans\n#[derive(Clone, Copy, Debug)]\npub struct Trace<F> {\n    func: F,\n}\n\n/// Information about the request/response that can be used to prepare log lines.\n#[allow(missing_debug_implementations)]\npub struct Info<'a> {\n    route: &'a Route,\n}\n\nimpl<FN, F> WrapSealed<F> for Trace<FN>\nwhere\n    FN: Fn(Info<'_>) -> Span + Clone + Send,\n    F: Filter + Clone + Send,\n    F::Extract: Reply,\n    F::Error: IsReject,\n{\n    type Wrapped = WithTrace<FN, F>;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        WithTrace {\n            filter,\n            trace: self.clone(),\n        }\n    }\n}\n\nimpl<'a> Info<'a> {\n    /// View the remote `SocketAddr` of the request.\n    pub fn remote_addr(&self) -> Option<SocketAddr> {\n        self.route.remote_addr()\n    }\n\n    /// View the `http::Method` of the request.\n    pub fn method(&self) -> &http::Method {\n        self.route.method()\n    }\n\n    /// View the URI path of the request.\n    pub fn path(&self) -> &str {\n        self.route.full_path()\n    }\n\n    /// View the `http::Version` of the request.\n    pub fn version(&self) -> http::Version {\n        self.route.version()\n    }\n\n    /// View the referer of the request.\n    pub fn referer(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::REFERER)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// View the user agent of the request.\n    pub fn user_agent(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::USER_AGENT)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// View the host of the request\n    pub fn host(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::HOST)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// View the request headers.\n    pub fn request_headers(&self) -> &http::HeaderMap {\n        self.route.headers()\n    }\n}\n\nmod internal {\n    use futures_util::{future::Inspect, future::MapOk, FutureExt, TryFutureExt};\n\n    use super::{Info, Trace};\n    use crate::filter::{Filter, FilterBase, Internal};\n    use crate::reject::IsReject;\n    use crate::reply::Reply;\n    use crate::reply::Response;\n    use crate::route;\n\n    #[allow(missing_debug_implementations)]\n    pub struct Traced(pub(super) Response);\n\n    impl Reply for Traced {\n        #[inline]\n        fn into_response(self) -> Response {\n            self.0\n        }\n    }\n\n    #[allow(missing_debug_implementations)]\n    #[derive(Clone, Copy)]\n    pub struct WithTrace<FN, F> {\n        pub(super) filter: F,\n        pub(super) trace: Trace<FN>,\n    }\n\n    use tracing::instrument::{Instrument, Instrumented};\n    use tracing::Span;\n\n    fn finished_logger<E: IsReject>(reply: &Result<(Traced,), E>) {\n        let (status, error) = match reply {\n            Ok((Traced(resp),)) => (resp.status(), None),\n            Err(error) => (error.status(), Some(error)),\n        };\n\n        if status.is_success() {\n            tracing::info!(\n                target: \"warp::filters::trace\",\n                status = status.as_u16(),\n                \"finished processing with success\"\n            );\n        } else if status.is_server_error() {\n            tracing::error!(\n                target: \"warp::filters::trace\",\n                status = status.as_u16(),\n                error = ?error,\n                \"unable to process request (internal error)\"\n            );\n        } else if status.is_client_error() {\n            tracing::warn!(\n                target: \"warp::filters::trace\",\n                status = status.as_u16(),\n                error = ?error,\n                \"unable to serve request (client error)\"\n            );\n        } else {\n            // Either informational or redirect\n            tracing::info!(\n                target: \"warp::filters::trace\",\n                status = status.as_u16(),\n                error = ?error,\n                    \"finished processing with status\"\n            );\n        }\n    }\n\n    \nfn convert_reply<R: Reply>(reply: R) -> (Traced,) {\n        (Traced(reply.into_response()),)\n    }\n\n    impl<FN, F> FilterBase for WithTrace<FN, F>\n    where\n        FN: Fn(Info<'_>) -> Span + Clone + Send,\n        F: Filter + Clone + Send,\n        F::Extract: Reply,\n        F::Error: IsReject,\n    {\n        type Extract = (Traced,);\n        type Error = F::Error;\n        type Future = Instrumented<\n            Inspect<\n                MapOk<F::Future, fn(F::Extract) -> Self::Extract>,\n                fn(&Result<Self::Extract, F::Error>),\n            >,\n        >;\n\n        fn filter(&self, _: Internal) -> Self::Future {\n            let span = route::with(|route| (self.trace.func)(Info { route }));\n            let _entered = span.enter();\n\n            tracing::info!(target: \"warp::filters::trace\", \"processing request\");\n            self.filter\n                .filter(Internal)\n                .map_ok(convert_reply as fn(F::Extract) -> Self::Extract)\n                .inspect(finished_logger as fn(&Result<Self::Extract, F::Error>))\n                .instrument(span.clone())\n        }\n    }\n}\n\n// Path: src/filters/sse.rs\n//! Server-Sent Events (SSE)\n//!\n//! # Example\n//!\n//! ```\n//!\n//! use std::time::Duration;\n//! use std::convert::Infallible;\n//! use warp::{Filter, sse::Event};\n//! use futures_util::{stream::iter, Stream};\n//!\n//! fn sse_events() -> impl Stream<Item = Result<Event, Infallible>> {\n//!     iter(vec![\n//!         Ok(Event::default().data(\"unnamed event\")),\n//!         Ok(\n//!             Event::default().event(\"chat\")\n//!             .data(\"chat message\")\n//!         ),\n//!         Ok(\n//!             Event::default().id(13.to_string())\n//!             .event(\"chat\")\n//!             .data(\"other chat message\\nwith next line\")\n//!             .retry(Duration::from_millis(5000))\n//!         )\n//!     ])\n//! }\n//!\n//! let app = warp::path(\"push-notifications\")\n//!     .and(warp::get())\n//!     .map(|| {\n//!         warp::sse::reply(warp::sse::keep_alive().stream(sse_events()))\n//!     });\n//! ```\n//!\n//! Each field already is event which can be sent to client.\n//! The events with multiple fields can be created by combining fields using tuples.\n//!\n//! See also the [EventSource](https://developer.mozilla.org/en-US/docs/Web/API/EventSource) API,\n//! which specifies the expected behavior of Server Sent Events.\n//!\n\n#![allow(rustdoc::invalid_html_tags)]\n\nuse serde::Serialize;\nuse std::borrow::Cow;\nuse std::error::Error as StdError;\nuse std::fmt::{self, Write};\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::str::FromStr;\nuse std::task::{Context, Poll};\nuse std::time::Duration;\n\nuse futures_util::{future, Stream, TryStream, TryStreamExt};\nuse http::header::{HeaderValue, CACHE_CONTROL, CONTENT_TYPE};\nuse hyper::Body;\nuse pin_project::pin_project;\nuse serde_json::Error;\nuse tokio::time::{self, Sleep};\n\nuse self::sealed::SseError;\nuse super::header;\nuse crate::filter::One;\nuse crate::reply::Response;\nuse crate::{Filter, Rejection, Reply};\n\n// Server-sent event data type\n#[derive(Debug)]\nenum DataType {\n    Text(String),\n    Json(String),\n}\n\n/// Server-sent event\n#[derive(Default, Debug)]\npub struct Event {\n    id: Option<String>,\n    data: Option<DataType>,\n    event: Option<String>,\n    comment: Option<String>,\n    retry: Option<Duration>,\n}\n\nimpl Event {\n    /// Set Server-sent event data\n    /// data field(s) (\"data:<content>\")\n    pub fn data<T: Into<String>>(mut self, data: T) -> Event {\n        self.data = Some(DataType::Text(data.into()));\n        self\n    }\n\n    /// Set Server-sent event data\n    /// data field(s) (\"data:<content>\")\n    pub fn json_data<T: Serialize>(mut self, data: T) -> Result<Event, Error> {\n        self.data = Some(DataType::Json(serde_json::to_string(&data)?));\n        Ok(self)\n    }\n\n    /// Set Server-sent event comment\n    /// Comment field (\":<comment-text>\")\n    pub fn comment<T: Into<String>>(mut self, comment: T) -> Event {\n        self.comment = Some(comment.into());\n        self\n    }\n\n    /// Set Server-sent event event\n    /// Event name field (\"event:<event-name>\")\n    pub fn event<T: Into<String>>(mut self, event: T) -> Event {\n        self.event = Some(event.into());\n        self\n    }\n\n    /// Set Server-sent event retry\n    /// Retry timeout field (\"retry:<timeout>\")\n    pub fn retry(mut self, duration: Duration) -> Event {\n        self.retry = Some(duration);\n        self\n    }\n\n    /// Set Server-sent event id\n    /// Identifier field (\"id:<identifier>\")\n    pub fn id<T: Into<String>>(mut self, id: T) -> Event {\n        self.id = Some(id.into());\n        self\n    }\n}\n\nimpl fmt::Display for Event {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        if let Some(ref comment) = &self.comment {\n            \":\".fmt(f)?;\n            comment.fmt(f)?;\n            f.write_char('\\n')?;\n        }\n\n        if let Some(ref event) = &self.event {\n            \"event:\".fmt(f)?;\n            event.fmt(f)?;\n            f.write_char('\\n')?;\n        }\n\n        match self.data {\n            Some(DataType::Text(ref data)) => {\n                for line in data.split('\\n') {\n                    \"data:\".fmt(f)?;\n                    line.fmt(f)?;\n                    f.write_char('\\n')?;\n                }\n            }\n            Some(DataType::Json(ref data)) => {\n                \"data:\".fmt(f)?;\n                data.fmt(f)?;\n                f.write_char('\\n')?;\n            }\n            None => {}\n        }\n\n        if let Some(ref id) = &self.id {\n            \"id:\".fmt(f)?;\n            id.fmt(f)?;\n            f.write_char('\\n')?;\n        }\n\n        if let Some(ref duration) = &self.retry {\n            \"retry:\".fmt(f)?;\n\n            let secs = duration.as_secs();\n            let millis = duration.subsec_millis();\n\n            if secs > 0 {\n                // format seconds\n                secs.fmt(f)?;\n\n                // pad milliseconds\n                if millis < 10 {\n                    f.write_str(\"00\")?;\n                } else if millis < 100 {\n                    f.write_char('0')?;\n                }\n            }\n\n            // format milliseconds\n            millis.fmt(f)?;\n\n            f.write_char('\\n')?;\n        }\n\n        f.write_char('\\n')?;\n        Ok(())\n    }\n}\n\n/// Gets the optional last event id from request.\n/// Typically this identifier represented as number or string.\n///\n/// ```\n/// let app = warp::sse::last_event_id::<u32>();\n///\n/// // The identifier is present\n/// async {\n///     assert_eq!(\n///         warp::test::request()\n///            .header(\"Last-Event-ID\", \"12\")\n///            .filter(&app)\n///            .await\n///            .unwrap(),\n///         Some(12)\n///     );\n///\n///     // The identifier is missing\n///     assert_eq!(\n///        warp::test::request()\n///            .filter(&app)\n///            .await\n///            .unwrap(),\n///         None\n///     );\n///\n///     // The identifier is not a valid\n///     assert!(\n///        warp::test::request()\n///            .header(\"Last-Event-ID\", \"abc\")\n///            .filter(&app)\n///            .await\n///            .is_err(),\n///     );\n///};\n/// ```\npub fn last_event_id<T>() -> impl Filter<Extract = One<Option<T>>, Error = Rejection> + Copy\nwhere\n    T: FromStr + Send + Sync + 'static,\n{\n    header::optional(\"last-event-id\")\n}\n\n/// Server-sent events reply\n///\n/// This function converts stream of server events into a `Reply` with:\n///\n/// - Status of `200 OK`\n/// - Header `content-type: text/event-stream`\n/// - Header `cache-control: no-cache`.\n///\n/// # Example\n///\n/// ```\n///\n/// use std::time::Duration;\n/// use futures_util::Stream;\n/// use futures_util::stream::iter;\n/// use std::convert::Infallible;\n/// use warp::{Filter, sse::Event};\n/// use serde_derive::Serialize;\n///\n/// #[derive(Serialize)]\n/// struct Msg {\n///     from: u32,\n///     text: String,\n/// }\n///\n/// fn event_stream() -> impl Stream<Item = Result<Event, Infallible>> {\n///         iter(vec![\n///             // Unnamed event with data only\n///             Ok(Event::default().data(\"payload\")),\n///             // Named event with ID and retry timeout\n///             Ok(\n///                 Event::default().data(\"other message\\nwith next line\")\n///                 .event(\"chat\")\n///                 .id(1.to_string())\n///                 .retry(Duration::from_millis(15000))\n///             ),\n///             // Event with JSON data\n///             Ok(\n///                 Event::default().id(2.to_string())\n///                 .json_data(Msg {\n///                     from: 2,\n///                     text: \"hello\".into(),\n///                 }).unwrap(),\n///             )\n///         ])\n/// }\n///\n/// async {\n///     let app = warp::path(\"sse\").and(warp::get()).map(|| {\n///        warp::sse::reply(event_stream())\n///     });\n///\n///     let res = warp::test::request()\n///         .method(\"GET\")\n///         .header(\"Connection\", \"Keep-Alive\")\n///         .path(\"/sse\")\n///         .reply(&app)\n///         .await\n///         .into_body();\n///\n///     assert_eq!(\n///         res,\n///         r#\"data:payload\n///\n/// event:chat\n/// data:other message\n/// data:with next line\n/// id:1\n/// retry:15000\n///\n/// data:{\"from\":2,\"text\":\"hello\"}\n/// id:2\n///\n/// \"#\n///     );\n/// };\n/// ```\npub fn reply<S>(event_stream: S) -> impl Reply\nwhere\n    S: TryStream<Ok = Event> + Send + 'static,\n    S::Error: StdError + Send + Sync + 'static,\n{\n    SseReply { event_stream }\n}\n\n#[allow(missing_debug_implementations)]\nstruct SseReply<S> {\n    event_stream: S,\n}\n\nimpl<S> Reply for SseReply<S>\nwhere\n    S: TryStream<Ok = Event> + Send + 'static,\n    S::Error: StdError + Send + Sync + 'static,\n{\n    #[inline]\n    fn into_response(self) -> Response {\n        let body_stream = self\n            .event_stream\n            .map_err(|error| {\n                // FIXME: error logging\n                log::error!(\"sse stream error: {}\", error);\n                SseError\n            })\n            .into_stream()\n            .and_then(|event| future::ready(Ok(event.to_string())));\n\n        let mut res = Response::new(Body::wrap_stream(body_stream));\n        // Set appropriate content type\n        res.headers_mut()\n            .insert(CONTENT_TYPE, HeaderValue::from_static(\"text/event-stream\"));\n        // Disable response body caching\n        res.headers_mut()\n            .insert(CACHE_CONTROL, HeaderValue::from_static(\"no-cache\"));\n        res\n    }\n}\n\n/// Configure the interval between keep-alive messages, the content\n/// of each message, and the associated stream.\n#[derive(Debug)]\npub struct KeepAlive {\n    comment_text: Cow<'static, str>,\n    max_interval: Duration,\n}\n\nimpl KeepAlive {\n    /// Customize the interval between keep-alive messages.\n    ///\n    /// Default is 15 seconds.\n    pub fn interval(mut self, time: Duration) -> Self {\n        self.max_interval = time;\n        self\n    }\n\n    /// Customize the text of the keep-alive message.\n    ///\n    /// Default is an empty comment.\n    pub fn text(mut self, text: impl Into<Cow<'static, str>>) -> Self {\n        self.comment_text = text.into();\n        self\n    }\n\n    /// Wrap an event stream with keep-alive functionality.\n    ///\n    /// See [`keep_alive`] for more.\n    pub fn stream<S>(\n        self,\n        event_stream: S,\n    ) -> impl TryStream<Ok = Event, Error = impl StdError + Send + Sync + 'static> + Send + 'static\n    where\n        S: TryStream<Ok = Event> + Send + 'static,\n        S::Error: StdError + Send + Sync + 'static,\n    {\n        let alive_timer = time::sleep(self.max_interval);\n        SseKeepAlive {\n            event_stream,\n            comment_text: self.comment_text,\n            max_interval: self.max_interval,\n            alive_timer,\n        }\n    }\n}\n\n#[allow(missing_debug_implementations)]\n#[pin_project]\nstruct SseKeepAlive<S> {\n    #[pin]\n    event_stream: S,\n    comment_text: Cow<'static, str>,\n    max_interval: Duration,\n    #[pin]\n    alive_timer: Sleep,\n}\n\n/// Keeps event source connection alive when no events sent over a some time.\n///\n/// Some proxy servers may drop HTTP connection after a some timeout of inactivity.\n/// This function helps to prevent such behavior by sending comment events every\n/// `keep_interval` of inactivity.\n///\n/// By default the comment is `:` (an empty comment) and the time interval between\n/// events is 15 seconds. Both may be customized using the builder pattern\n/// as shown below.\n///\n/// ```\n/// use std::time::Duration;\n/// use std::convert::Infallible;\n/// use futures_util::StreamExt;\n/// use tokio::time::interval;\n/// use tokio_stream::wrappers::IntervalStream;\n/// use warp::{Filter, Stream, sse::Event};\n///\n/// // create server-sent event\n/// fn sse_counter(counter: u64) ->  Result<Event, Infallible> {\n...\n", "needle_token_start": 12139, "needle_token_end": 12181, "code_context_ntokens": 16212, "output": ["```rust\nfn convert_reply<R: Reply>(reply: R) -> (Traced,) {\n    (Traced(reply.into_response()),)\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "extension", "language": "rust", "path": "src/test.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To add custom data to a request's internal storage, allowing for enhanced flexibility and context-specific data handling during request processing.\n2. **Input**: A generic type `T` which must meet the constraints of being sendable between threads, synchronizable, and having a static lifetime.\n3. **Output**: The modified instance of the request builder, enabling method chaining.\n4. **Procedure**: The function takes a user-defined value and inserts it into the request's extension map. This map is designed to store arbitrary data types, facilitating data sharing across different components of the request handling process.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/filters/sse.rs\n//! Server-Sent Events (SSE)\n//!\n//! # Example\n//!\n//! ```\n//!\n//! use std::time::Duration;\n//! use std::convert::Infallible;\n//! use warp::{Filter, sse::Event};\n//! use futures_util::{stream::iter, Stream};\n//!\n//! fn sse_events() -> impl Stream<Item = Result<Event, Infallible>> {\n//!     iter(vec![\n//!         Ok(Event::default().data(\"unnamed event\")),\n//!         Ok(\n//!             Event::default().event(\"chat\")\n//!             .data(\"chat message\")\n//!         ),\n//!         Ok(\n//!             Event::default().id(13.to_string())\n//!             .event(\"chat\")\n//!             .data(\"other chat message\\nwith next line\")\n//!             .retry(Duration::from_millis(5000))\n//!         )\n//!     ])\n//! }\n//!\n//! let app = warp::path(\"push-notifications\")\n//!     .and(warp::get())\n//!     .map(|| {\n//!         warp::sse::reply(warp::sse::keep_alive().stream(sse_events()))\n//!     });\n//! ```\n//!\n//! Each field already is event which can be sent to client.\n//! The events with multiple fields can be created by combining fields using tuples.\n//!\n//! See also the [EventSource](https://developer.mozilla.org/en-US/docs/Web/API/EventSource) API,\n//! which specifies the expected behavior of Server Sent Events.\n//!\n\n#![allow(rustdoc::invalid_html_tags)]\n\nuse serde::Serialize;\nuse std::borrow::Cow;\nuse std::error::Error as StdError;\nuse std::fmt::{self, Write};\nuse std::future::Future;\nuse std::pin::Pin;\nuse std::str::FromStr;\nuse std::task::{Context, Poll};\nuse std::time::Duration;\n\nuse futures_util::{future, Stream, TryStream, TryStreamExt};\nuse http::header::{HeaderValue, CACHE_CONTROL, CONTENT_TYPE};\nuse hyper::Body;\nuse pin_project::pin_project;\nuse serde_json::Error;\nuse tokio::time::{self, Sleep};\n\nuse self::sealed::SseError;\nuse super::header;\nuse crate::filter::One;\nuse crate::reply::Response;\nuse crate::{Filter, Rejection, Reply};\n\n// Server-sent event data type\n#[derive(Debug)]\nenum DataType {\n    Text(String),\n    Json(String),\n}\n\n/// Server-sent event\n#[derive(Default, Debug)]\npub struct Event {\n    id: Option<String>,\n    data: Option<DataType>,\n    event: Option<String>,\n    comment: Option<String>,\n    retry: Option<Duration>,\n}\n\nimpl Event {\n    /// Set Server-sent event data\n    /// data field(s) (\"data:<content>\")\n    pub fn data<T: Into<String>>(mut self, data: T) -> Event {\n        self.data = Some(DataType::Text(data.into()));\n        self\n    }\n\n    /// Set Server-sent event data\n    /// data field(s) (\"data:<content>\")\n    pub fn json_data<T: Serialize>(mut self, data: T) -> Result<Event, Error> {\n        self.data = Some(DataType::Json(serde_json::to_string(&data)?));\n        Ok(self)\n    }\n\n    /// Set Server-sent event comment\n    /// Comment field (\":<comment-text>\")\n...\n// Path: src/filters/log.rs\n//! Logger Filters\n\nuse std::fmt;\nuse std::net::SocketAddr;\nuse std::time::{Duration, Instant};\n\nuse http::{header, StatusCode};\n\nuse crate::filter::{Filter, WrapSealed};\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::route::Route;\n\nuse self::internal::WithLog;\n\n/// Create a wrapping [`Filter`](crate::Filter) with the specified `name` as the `target`.\n///\n/// This uses the default access logging format, and log records produced\n/// will have their `target` set to `name`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // If using something like `pretty_env_logger`,\n/// // view logs by setting `RUST_LOG=example::api`.\n/// let log = warp::log(\"example::api\");\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(log);\n/// ```\npub fn log(name: &'static str) -> Log<impl Fn(Info<'_>) + Copy> {\n    let func = move |info: Info<'_>| {\n        // TODO?\n        // - response content length?\n        log::info!(\n            target: name,\n            \"{} \\\"{} {} {:?}\\\" {} \\\"{}\\\" \\\"{}\\\" {:?}\",\n            OptFmt(info.route.remote_addr()),\n            info.method(),\n            info.path(),\n            info.route.version(),\n            info.status().as_u16(),\n            OptFmt(info.referer()),\n            OptFmt(info.user_agent()),\n            info.elapsed(),\n        );\n    };\n    Log { func }\n}\n\n/// Create a wrapping [`Filter`](crate::Filter) that receives `warp::log::Info`.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let log = warp::log::custom(|info| {\n///     // Use a log macro, or slog, or println, or whatever!\n///     eprintln!(\n///         \"{} {} {}\",\n///         info.method(),\n///         info.path(),\n///         info.status(),\n///     );\n/// });\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(log);\n/// ```\npub fn custom<F>(func: F) -> Log<F>\nwhere\n    F: Fn(Info<'_>),\n{\n    Log { func }\n}\n\n/// Decorates a [`Filter`] to log requests and responses.\n#[derive(Clone, Copy, Debug)]\npub struct Log<F> {\n    func: F,\n}\n\n/// Information about the request/response that can be used to prepare log lines.\n#[allow(missing_debug_implementations)]\npub struct Info<'a> {\n    route: &'a Route,\n    start: Instant,\n    status: StatusCode,\n}\n\nimpl<FN, F> WrapSealed<F> for Log<FN>\nwhere\n    FN: Fn(Info<'_>) + Clone + Send,\n    F: Filter + Clone + Send,\n    F::Extract: Reply,\n    F::Error: IsReject,\n{\n    type Wrapped = WithLog<FN, F>;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        WithLog {\n            filter,\n            log: self.clone(),\n        }\n    }\n}\n\nimpl<'a> Info<'a> {\n    /// View the remote `SocketAddr` of the request.\n    pub fn remote_addr(&self) -> Option<SocketAddr> {\n        self.route.remote_addr()\n    }\n\n    /// View the `http::Method` of the request.\n    pub fn method(&self) -> &http::Method {\n        self.route.method()\n    }\n\n    /// View the URI path of the request.\n    pub fn path(&self) -> &str {\n        self.route.full_path()\n    }\n\n    /// View the `http::Version` of the request.\n    pub fn version(&self) -> http::Version {\n        self.route.version()\n    }\n\n    /// View the `http::StatusCode` of the response.\n    pub fn status(&self) -> http::StatusCode {\n        self.status\n    }\n\n    /// View the referer of the request.\n    pub fn referer(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::REFERER)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// View the user agent of the request.\n    pub fn user_agent(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::USER_AGENT)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// View the `Duration` that elapsed for the request.\n    pub fn elapsed(&self) -> Duration {\n        tokio::time::Instant::now().into_std() - self.start\n    }\n\n    /// View the host of the request\n    pub fn host(&self) -> Option<&str> {\n        self.route\n            .headers()\n            .get(header::HOST)\n            .and_then(|v| v.to_str().ok())\n    }\n\n    /// Access the full headers of the request\n    pub fn request_headers(&self) -> &http::HeaderMap {\n        self.route.headers()\n    }\n}\n\nstruct OptFmt<T>(Option<T>);\n\nimpl<T: fmt::Display> fmt::Display for OptFmt<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        if let Some(ref t) = self.0 {\n            fmt::Display::fmt(t, f)\n        } else {\n            f.write_str(\"-\")\n        }\n    }\n}\n\nmod internal {\n    use std::future::Future;\n    use std::pin::Pin;\n    use std::task::{Context, Poll};\n    use std::time::Instant;\n\n    use futures_util::{ready, TryFuture};\n    use pin_project::pin_project;\n\n    use super::{Info, Log};\n    use crate::filter::{Filter, FilterBase, Internal};\n    use crate::reject::IsReject;\n    use crate::reply::{Reply, Response};\n    use crate::route;\n\n    #[allow(missing_debug_implementations)]\n    pub struct Logged(pub(super) Response);\n\n    impl Reply for Logged {\n        #[inline]\n        fn into_response(self) -> Response {\n            self.0\n        }\n    }\n\n    #[allow(missing_debug_implementations)]\n    #[derive(Clone, Copy)]\n    pub struct WithLog<FN, F> {\n        pub(super) filter: F,\n        pub(super) log: Log<FN>,\n    }\n\n    impl<FN, F> FilterBase for WithLog<FN, F>\n    where\n        FN: Fn(Info<'_>) + Clone + Send,\n        F: Filter + Clone + Send,\n        F::Extract: Reply,\n        F::Error: IsReject,\n    {\n        type Extract = (Logged,);\n        type Error = F::Error;\n        type Future = WithLogFuture<FN, F::Future>;\n\n        fn filter(&self, _: Internal) -> Self::Future {\n            let started = tokio::time::Instant::now().into_std();\n            WithLogFuture {\n                log: self.log.clone(),\n                future: self.filter.filter(Internal),\n                started,\n            }\n        }\n    }\n\n    #[allow(missing_debug_implementations)]\n    #[pin_project]\n    pub struct WithLogFuture<FN, F> {\n        log: Log<FN>,\n        #[pin]\n        future: F,\n        started: Instant,\n    }\n\n    impl<FN, F> Future for WithLogFuture<FN, F>\n    where\n        FN: Fn(Info<'_>),\n        F: TryFuture,\n        F::Ok: Reply,\n        F::Error: IsReject,\n    {\n        type Output = Result<(Logged,), F::Error>;\n\n        fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n            let pin = self.as_mut().project();\n            let (result, status) = match ready!(pin.future.try_poll(cx)) {\n                Ok(reply) => {\n                    let resp = reply.into_response();\n                    let status = resp.status();\n                    (Poll::Ready(Ok((Logged(resp),))), status)\n                }\n                Err(reject) => {\n                    let status = reject.status();\n                    (Poll::Ready(Err(reject)), status)\n                }\n            };\n\n            route::with(|route| {\n                (self.log.func)(Info {\n                    route,\n                    start: self.started,\n                    status,\n                });\n            });\n\n            result\n        }\n    }\n}\n\n// Path: src/filters/host.rs\n//! Host (\"authority\") filter\n//!\nuse crate::filter::{filter_fn_one, Filter, One};\nuse crate::reject::{self, Rejection};\nuse futures_util::future;\npub use http::uri::Authority;\nuse std::str::FromStr;\n\n/// Creates a `Filter` that requires a specific authority (target server's\n/// host and port) in the request.\n///\n/// Authority is specified either in the `Host` header or in the target URI.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let multihost =\n///     warp::host::exact(\"foo.com\").map(|| \"you've reached foo.com\")\n///     .or(warp::host::exact(\"bar.com\").map(|| \"you've reached bar.com\"));\n/// ```\npub fn exact(expected: &str) -> impl Filter<Extract = (), Error = Rejection> + Clone {\n    let expected = Authority::from_str(expected).expect(\"invalid host/authority\");\n    optional()\n        .and_then(move |option: Option<Authority>| match option {\n            Some(authority) if authority == expected => future::ok(()),\n            _ => future::err(reject::not_found()),\n        })\n        .untuple_one()\n}\n\n/// Creates a `Filter` that looks for an authority (target server's host\n/// and port) in the request.\n///\n/// Authority is specified either in the `Host` header or in the target URI.\n///\n/// If found, extracts the `Authority`, otherwise continues the request,\n/// extracting `None`.\n///\n/// Rejects with `400 Bad Request` if the `Host` header is malformed or if there\n/// is a mismatch between the `Host` header and the target URI.\n///\n/// # Example\n///\n/// ```\n/// use warp::{Filter, host::Authority};\n///\n/// let host = warp::host::optional()\n///     .map(|authority: Option<Authority>| {\n///         if let Some(a) = authority {\n///             format!(\"{} is currently not at home\", a.host())\n///         } else {\n///             \"please state who you're trying to reach\".to_owned()\n///         }\n///     });\n/// ```\npub fn optional() -> impl Filter<Extract = One<Option<Authority>>, Error = Rejection> + Copy {\n    filter_fn_one(move |route| {\n        // The authority can be sent by clients in various ways:\n        //\n        //  1) in the \"target URI\"\n        //    a) serialized in the start line (HTTP/1.1 proxy requests)\n        //    b) serialized in `:authority` pseudo-header (HTTP/2 generated - \"SHOULD\")\n        //  2) in the `Host` header (HTTP/1.1 origin requests, HTTP/2 converted)\n        //\n        // Hyper transparently handles 1a/1b, but not 2, so we must look at both.\n\n        let from_uri = route.uri().authority();\n\n        let name = \"host\";\n        let from_header = route.headers()\n            .get(name)\n            .map(|value|\n                // Header present, parse it\n                value.to_str().map_err(|_| reject::invalid_header(name))\n                    .and_then(|value| Authority::from_str(value).map_err(|_| reject::invalid_header(name)))\n            );\n\n        future::ready(match (from_uri, from_header) {\n            // no authority in the request (HTTP/1.0 or non-conforming)\n            (None, None) => Ok(None),\n\n            // authority specified in either or both matching\n            (Some(a), None) => Ok(Some(a.clone())),\n            (None, Some(Ok(a))) => Ok(Some(a)),\n            (Some(a), Some(Ok(b))) if *a == b => Ok(Some(b)),\n\n            // mismatch\n            (Some(_), Some(Ok(_))) => Err(reject::invalid_header(name)),\n\n            // parse error\n            (_, Some(Err(r))) => Err(r),\n        })\n    })\n}\n\n// Path: src/filters/mod.rs\n//! Built-in Filters\n//!\n//! This module mostly serves as documentation to group together the list of\n//! built-in filters. Most of these are available at more convenient paths.\n\npub mod addr;\npub mod any;\npub mod body;\n#[cfg(any(feature = \"compression-brotli\", feature = \"compression-gzip\"))]\npub mod compression;\npub mod cookie;\npub mod cors;\npub mod ext;\npub mod fs;\npub mod header;\npub mod host;\npub mod log;\npub mod method;\n#[cfg(feature = \"multipart\")]\npub mod multipart;\npub mod path;\npub mod query;\npub mod reply;\npub mod sse;\npub mod trace;\n#[cfg(feature = \"websocket\")]\npub mod ws;\n\npub use crate::filter::BoxedFilter;\n\n// Path: src/transport.rs\nuse std::io;\nuse std::net::SocketAddr;\nuse std::pin::Pin;\nuse std::task::{Context, Poll};\n\nuse hyper::server::conn::AddrStream;\nuse tokio::io::{AsyncRead, AsyncWrite, ReadBuf};\n\npub trait Transport: AsyncRead + AsyncWrite {\n    fn remote_addr(&self) -> Option<SocketAddr>;\n}\n\nimpl Transport for AddrStream {\n    fn remote_addr(&self) -> Option<SocketAddr> {\n        Some(self.remote_addr())\n    }\n}\n\npub(crate) struct LiftIo<T>(pub(crate) T);\n\nimpl<T: AsyncRead + Unpin> AsyncRead for LiftIo<T> {\n    fn poll_read(\n        self: Pin<&mut Self>,\n        cx: &mut Context<'_>,\n        buf: &mut ReadBuf<'_>,\n    ) -> Poll<io::Result<()>> {\n        Pin::new(&mut self.get_mut().0).poll_read(cx, buf)\n    }\n}\n\nimpl<T: AsyncWrite + Unpin> AsyncWrite for LiftIo<T> {\n    fn poll_write(\n        self: Pin<&mut Self>,\n        cx: &mut Context<'_>,\n        buf: &[u8],\n    ) -> Poll<io::Result<usize>> {\n        Pin::new(&mut self.get_mut().0).poll_write(cx, buf)\n    }\n\n    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {\n        Pin::new(&mut self.get_mut().0).poll_flush(cx)\n    }\n\n    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {\n        Pin::new(&mut self.get_mut().0).poll_shutdown(cx)\n    }\n}\n\nimpl<T: AsyncRead + AsyncWrite + Unpin> Transport for LiftIo<T> {\n    fn remote_addr(&self) -> Option<SocketAddr> {\n        None\n    }\n}\n\n// Path: src/service.rs\n//! Convert `Filter`s into `Service`s\n\npub use crate::filter::service::service;\n\n// Path: src/server.rs\n#[cfg(feature = \"tls\")]\nuse crate::tls::TlsConfigBuilder;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"tls\")]\nuse std::path::Path;\n\nuse futures_util::{future, FutureExt, TryFuture, TryStream, TryStreamExt};\nuse hyper::server::conn::AddrIncoming;\nuse hyper::service::{make_service_fn, service_fn};\nuse hyper::Server as HyperServer;\nuse tokio::io::{AsyncRead, AsyncWrite};\nuse tracing::Instrument;\n\nuse crate::filter::Filter;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::transport::Transport;\n\n/// Create a `Server` with the provided `Filter`.\npub fn serve<F>(filter: F) -> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: IsReject,\n{\n    Server {\n        pipeline: false,\n        filter,\n    }\n}\n\n/// A Warp Server ready to filter requests.\n#[derive(Debug)]\npub struct Server<F> {\n    pipeline: bool,\n    filter: F,\n}\n\n/// A Warp Server ready to filter requests over TLS.\n///\n/// *This type requires the `\"tls\"` feature.*\n#[cfg(feature = \"tls\")]\npub struct TlsServer<F> {\n    server: Server<F>,\n    tls: TlsConfigBuilder,\n}\n\n// Getting all various generic bounds to make this a re-usable method is\n// very complicated, so instead this is just a macro.\nmacro_rules! into_service {\n    ($into:expr) => {{\n        let inner = crate::service($into);\n        make_service_fn(move |transport| {\n            let inner = inner.clone();\n            let remote_addr = Transport::remote_addr(transport);\n            future::ok::<_, Infallible>(service_fn(move |req| {\n                inner.call_with_addr(req, remote_addr)\n            }))\n        })\n    }};\n}\n\nmacro_rules! addr_incoming {\n    ($addr:expr) => {{\n        let mut incoming = AddrIncoming::bind($addr)?;\n        incoming.set_nodelay(true);\n        let addr = incoming.local_addr();\n        (addr, incoming)\n    }};\n}\n\nmacro_rules! bind_inner {\n    ($this:ident, $addr:expr) => {{\n        let service = into_service!($this.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let srv = HyperServer::builder(incoming)\n            .http1_pipeline_flush($this.pipeline)\n            .serve(service);\n        Ok::<_, hyper::Error>((addr, srv))\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let service = into_service!($this.server.filter);\n        let (addr, incoming) = addr_incoming!($addr);\n        let tls = $this.tls.build()?;\n        let srv = HyperServer::builder(crate::tls::TlsAcceptor::new(tls, incoming))\n            .http1_pipeline_flush($this.server.pipeline)\n            .serve(service);\n        Ok::<_, Box<dyn std::error::Error + Send + Sync>>((addr, srv))\n    }};\n}\n\nmacro_rules! bind {\n    ($this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!($this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        let addr = $addr.into();\n        (|addr| bind_inner!(tls: $this, addr))(&addr).unwrap_or_else(|e| {\n            panic!(\"error binding to {}: {}\", addr, e);\n        })\n    }};\n}\n\nmacro_rules! try_bind {\n    ($this:ident, $addr:expr) => {{\n        (|addr| bind_inner!($this, addr))($addr)\n    }};\n\n    (tls: $this:ident, $addr:expr) => {{\n        (|addr| bind_inner!(tls: $this, addr))($addr)\n    }};\n}\n\n// ===== impl Server =====\n\nimpl<F> Server<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    /// Run this `Server` forever on the current thread.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub async fn run(self, addr: impl Into<SocketAddr>) {\n        let (addr, fut) = self.bind_ephemeral(addr);\n        let span = tracing::info_span!(\"Server::run\", ?addr);\n        tracing::info!(parent: &span, \"listening on http://{}\", addr);\n\n        fut.instrument(span).await;\n    }\n\n    /// Run this `Server` forever on the current thread with a specific stream\n    /// of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    pub async fn run_incoming<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        self.run_incoming2(incoming.map_ok(crate::transport::LiftIo).into_stream())\n            .instrument(tracing::info_span!(\"Server::run_incoming\"))\n            .await;\n    }\n\n    async fn run_incoming2<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: Transport + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let fut = self.serve_incoming2(incoming);\n\n        tracing::info!(\"listening with custom incoming\");\n\n        fut.await;\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind(self, addr: impl Into<SocketAddr> + 'static) -> impl Future<Output = ()> + 'static {\n        let (_, fut) = self.bind_ephemeral(addr);\n        fut\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on any runtime.\n    ///\n    /// In case we are unable to bind to the specified address, resolves to an\n    /// error and logs the reason.\n    pub async fn try_bind(self, addr: impl Into<SocketAddr>) {\n        let addr = addr.into();\n        let srv = match try_bind!(self, &addr) {\n            Ok((_, srv)) => srv,\n            Err(err) => {\n                tracing::error!(\"error binding to {}: {}\", addr, err);\n                return;\n            }\n        };\n\n        srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        })\n        .await;\n    }\n\n    /// Bind to a possibly ephemeral socket address.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        (addr, srv)\n    }\n\n    /// Tried to bind a possibly ephemeral socket address.\n    ///\n    /// Returns a `Result` which fails in case we are unable to bind with the\n    /// underlying error.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    pub fn try_bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use warp::Filter;\n    /// use futures_util::future::TryFutureExt;\n    /// use tokio::sync::oneshot;\n    ///\n    /// # fn main() {\n    /// let routes = warp::any()\n    ///     .map(|| \"Hello, World!\");\n    ///\n    /// let (tx, rx) = oneshot::channel();\n    ///\n    /// let (addr, server) = warp::serve(routes)\n    ///     .bind_with_graceful_shutdown(([127, 0, 0, 1], 3030), async {\n    ///          rx.await.ok();\n    ///     });\n    ///\n    /// // Spawn the server into a runtime\n    /// tokio::task::spawn(server);\n    ///\n    /// // Later, start the shutdown...\n    /// let _ = tx.send(());\n    /// # }\n    /// ```\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(self, addr);\n        let fut = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n        (addr, fut)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    pub fn try_bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming<I>(self, incoming: I) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        self.serve_incoming2(incoming)\n            .instrument(tracing::info_span!(\"Server::serve_incoming\"))\n    }\n\n    /// Setup this `Server` with a specific stream of incoming connections and a\n    /// signal to initiate graceful shutdown.\n    ///\n    /// This can be used for Unix Domain Sockets, or TLS, etc.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// Returns a `Future` that can be executed on the current runtime.\n    pub fn serve_incoming_with_graceful_shutdown<I>(\n        self,\n        incoming: I,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> impl Future<Output = ()>\n    where\n        I: TryStream + Send,\n        I::Ok: AsyncRead + AsyncWrite + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let incoming = incoming.map_ok(crate::transport::LiftIo);\n        let service = into_service!(self.filter);\n        let pipeline = self.pipeline;\n\n        async move {\n            let srv =\n                HyperServer::builder(hyper::server::accept::from_stream(incoming.into_stream()))\n                    .http1_pipeline_flush(pipeline)\n                    .serve(service)\n                    .with_graceful_shutdown(signal)\n                    .await;\n\n            if let Err(err) = srv {\n                tracing::error!(\"server error: {}\", err);\n            }\n        }\n        .instrument(tracing::info_span!(\n            \"Server::serve_incoming_with_graceful_shutdown\"\n        ))\n    }\n\n    async fn serve_incoming2<I>(self, incoming: I)\n    where\n        I: TryStream + Send,\n        I::Ok: Transport + Send + 'static + Unpin,\n        I::Error: Into<Box<dyn StdError + Send + Sync>>,\n    {\n        let service = into_service!(self.filter);\n\n        let srv = HyperServer::builder(hyper::server::accept::from_stream(incoming.into_stream()))\n            .http1_pipeline_flush(self.pipeline)\n            .serve(service)\n            .await;\n\n        if let Err(err) = srv {\n            tracing::error!(\"server error: {}\", err);\n        }\n    }\n\n    // Generally shouldn't be used, as it can slow down non-pipelined responses.\n    //\n    // It's only real use is to make silly pipeline benchmarks look better.\n    #[doc(hidden)]\n    pub fn unstable_pipeline(mut self) -> Self {\n        self.pipeline = true;\n        self\n    }\n\n    /// Configure a server to use TLS.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    #[cfg(feature = \"tls\")]\n    pub fn tls(self) -> TlsServer<F> {\n        TlsServer {\n            server: self,\n            tls: TlsConfigBuilder::new(),\n        }\n    }\n}\n\n// // ===== impl TlsServer =====\n\n#[cfg(feature = \"tls\")]\nimpl<F> TlsServer<F>\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    <F::Future as TryFuture>::Ok: Reply,\n    <F::Future as TryFuture>::Error: IsReject,\n{\n    // TLS config methods\n\n    /// Specify the file path to read the private key.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn key_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.key_path(path))\n    }\n\n    /// Specify the file path to read the certificate.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn cert_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.cert_path(path))\n    }\n\n    /// Specify the file path to read the trust anchor for optional client authentication.\n    ///\n    /// Anonymous and authenticated clients will be accepted. If no trust anchor is provided by any\n    /// of the `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_optional_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.client_auth_optional_path(path))\n    }\n\n    /// Specify the file path to read the trust anchor for required client authentication.\n    ///\n    /// Only authenticated clients will be accepted. If no trust anchor is provided by any of the\n    /// `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_required_path(self, path: impl AsRef<Path>) -> Self {\n        self.with_tls(|tls| tls.client_auth_required_path(path))\n    }\n\n    /// Specify the in-memory contents of the private key.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn key(self, key: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.key(key.as_ref()))\n    }\n\n    /// Specify the in-memory contents of the certificate.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn cert(self, cert: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.cert(cert.as_ref()))\n    }\n\n    /// Specify the in-memory contents of the trust anchor for optional client authentication.\n    ///\n    /// Anonymous and authenticated clients will be accepted. If no trust anchor is provided by any\n    /// of the `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_optional(self, trust_anchor: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.client_auth_optional(trust_anchor.as_ref()))\n    }\n\n    /// Specify the in-memory contents of the trust anchor for required client authentication.\n    ///\n    /// Only authenticated clients will be accepted. If no trust anchor is provided by any of the\n    /// `client_auth_` methods, then client authentication is disabled by default.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn client_auth_required(self, trust_anchor: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.client_auth_required(trust_anchor.as_ref()))\n    }\n\n    /// Specify the DER-encoded OCSP response.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn ocsp_resp(self, resp: impl AsRef<[u8]>) -> Self {\n        self.with_tls(|tls| tls.ocsp_resp(resp.as_ref()))\n    }\n\n    fn with_tls<Func>(self, func: Func) -> Self\n    where\n        Func: FnOnce(TlsConfigBuilder) -> TlsConfigBuilder,\n    {\n        let TlsServer { server, tls } = self;\n        let tls = func(tls);\n        TlsServer { server, tls }\n    }\n\n    // Server run methods\n\n    /// Run this `TlsServer` forever on the current thread.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub async fn run(self, addr: impl Into<SocketAddr>) {\n        let (addr, fut) = self.bind_ephemeral(addr);\n        let span = tracing::info_span!(\"TlsServer::run\", %addr);\n        tracing::info!(parent: &span, \"listening on https://{}\", addr);\n\n        fut.instrument(span).await;\n    }\n\n    /// Bind to a socket address, returning a `Future` that can be\n    /// executed on a runtime.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub async fn bind(self, addr: impl Into<SocketAddr>) {\n        let (_, fut) = self.bind_ephemeral(addr);\n        fut.await;\n    }\n\n    /// Bind to a possibly ephemeral socket address.\n    ///\n    /// Returns the bound address and a `Future` that can be executed on\n    /// the current runtime.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_ephemeral(\n        self,\n        addr: impl Into<SocketAddr>,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(tls: self, addr);\n        let srv = srv.map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        (addr, srv)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    ///\n    /// # Panics\n    ///\n    /// Panics if we are unable to bind to the provided address.\n    pub fn bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> (SocketAddr, impl Future<Output = ()> + 'static) {\n        let (addr, srv) = bind!(tls: self, addr);\n\n        let fut = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n        (addr, fut)\n    }\n\n    /// Create a server with graceful shutdown signal.\n    ///\n    /// When the signal completes, the server will start the graceful shutdown\n    /// process.\n    ///\n    /// *This function requires the `\"tls\"` feature.*\n    pub fn try_bind_with_graceful_shutdown(\n        self,\n        addr: impl Into<SocketAddr> + 'static,\n        signal: impl Future<Output = ()> + Send + 'static,\n    ) -> Result<(SocketAddr, impl Future<Output = ()> + 'static), crate::Error> {\n        let addr = addr.into();\n        let (addr, srv) = try_bind!(tls: self, &addr).map_err(crate::Error::new)?;\n        let srv = srv.with_graceful_shutdown(signal).map(|result| {\n            if let Err(err) = result {\n                tracing::error!(\"server error: {}\", err)\n            }\n        });\n\n        Ok((addr, srv))\n    }\n}\n\n#[cfg(feature = \"tls\")]\nimpl<F> ::std::fmt::Debug for TlsServer<F>\nwhere\n    F: ::std::fmt::Debug,\n{\n    fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {\n        f.debug_struct(\"TlsServer\")\n            .field(\"server\", &self.server)\n            .finish()\n    }\n}\n\n// Path: src/test.rs\n//! Test utilities to test your filters.\n//!\n//! [`Filter`](../trait.Filter.html)s can be easily tested without starting up an HTTP\n//! server, by making use of the [`RequestBuilder`](./struct.RequestBuilder.html) in this\n//! module.\n//!\n//! # Testing Filters\n//!\n//! It's easy to test filters, especially if smaller filters are used to build\n//! up your full set. Consider these example filters:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! fn sum() -> impl Filter<Extract = (u32,), Error = warp::Rejection> + Copy {\n//!     warp::path::param()\n//!         .and(warp::path::param())\n//!         .map(|x: u32, y: u32| {\n//!             x + y\n//!         })\n//! }\n//!\n//! fn math() -> impl Filter<Extract = (String,), Error = warp::Rejection> + Copy {\n//!     warp::post()\n//!         .and(sum())\n//!         .map(|z: u32| {\n//!             format!(\"Sum = {}\", z)\n//!         })\n//! }\n//! ```\n//!\n//! We can test some requests against the `sum` filter like this:\n//!\n//! ```\n//! # use warp::Filter;\n//! #[tokio::test]\n//! async fn test_sum() {\n//! #    let sum = || warp::any().map(|| 3);\n//!     let filter = sum();\n//!\n//!     // Execute `sum` and get the `Extract` back.\n//!     let value = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .filter(&filter)\n//!         .await\n//!         .unwrap();\n//!     assert_eq!(value, 3);\n//!\n//!     // Or simply test if a request matches (doesn't reject).\n//!     assert!(\n//!         warp::test::request()\n//!             .path(\"/1/-5\")\n//!             .matches(&filter)\n//!             .await\n//!     );\n//! }\n//! ```\n//!\n//! If the filter returns something that implements `Reply`, and thus can be\n//! turned into a response sent back to the client, we can test what exact\n//! response is returned. The `math` filter uses the `sum` filter, but returns\n//! a `String` that can be turned into a response.\n//!\n//! ```\n//! # use warp::Filter;\n//! #[test]\n//! fn test_math() {\n//! #    let math = || warp::any().map(warp::reply);\n//!     let filter = math();\n//!\n//!     let res = warp::test::request()\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 405, \"GET is not allowed\");\n//!\n//!     let res = warp::test::request()\n//!         .method(\"POST\")\n//!         .path(\"/1/2\")\n//!         .reply(&filter);\n//!     assert_eq!(res.status(), 200);\n//!     assert_eq!(res.body(), \"Sum is 3\");\n//! }\n//! ```\nuse std::convert::TryFrom;\nuse std::error::Error as StdError;\nuse std::fmt;\nuse std::future::Future;\nuse std::net::SocketAddr;\n#[cfg(feature = \"websocket\")]\nuse std::pin::Pin;\n#[cfg(feature = \"websocket\")]\nuse std::task::Context;\n#[cfg(feature = \"websocket\")]\nuse std::task::{self, Poll};\n\nuse bytes::Bytes;\n#[cfg(feature = \"websocket\")]\nuse futures_channel::mpsc;\n#[cfg(feature = \"websocket\")]\nuse futures_util::StreamExt;\nuse futures_util::{future, FutureExt, TryFutureExt};\nuse http::{\n    header::{HeaderName, HeaderValue},\n    Response,\n};\nuse serde::Serialize;\n#[cfg(feature = \"websocket\")]\nuse tokio::sync::oneshot;\n\nuse crate::filter::Filter;\n#[cfg(feature = \"websocket\")]\nuse crate::filters::ws::Message;\nuse crate::reject::IsReject;\nuse crate::reply::Reply;\nuse crate::route::{self, Route};\nuse crate::Request;\n#[cfg(feature = \"websocket\")]\nuse crate::{Sink, Stream};\n\nuse self::inner::OneOrTuple;\n\n/// Starts a new test `RequestBuilder`.\npub fn request() -> RequestBuilder {\n    RequestBuilder {\n        remote_addr: None,\n        req: Request::default(),\n    }\n}\n\n/// Starts a new test `WsBuilder`.\n#[cfg(feature = \"websocket\")]\npub fn ws() -> WsBuilder {\n    WsBuilder { req: request() }\n}\n\n/// A request builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[must_use = \"RequestBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct RequestBuilder {\n    remote_addr: Option<SocketAddr>,\n    req: Request,\n}\n\n/// A Websocket builder for testing filters.\n///\n/// See [module documentation](crate::test) for an overview.\n#[cfg(feature = \"websocket\")]\n#[must_use = \"WsBuilder does nothing on its own\"]\n#[derive(Debug)]\npub struct WsBuilder {\n    req: RequestBuilder,\n}\n\n/// A test client for Websocket filters.\n#[cfg(feature = \"websocket\")]\npub struct WsClient {\n    tx: mpsc::UnboundedSender<crate::ws::Message>,\n    rx: mpsc::UnboundedReceiver<Result<crate::ws::Message, crate::error::Error>>,\n}\n\n/// An error from Websocket filter tests.\n#[derive(Debug)]\npub struct WsError {\n    cause: Box<dyn StdError + Send + Sync>,\n}\n\nimpl RequestBuilder {\n    /// Sets the method of this builder.\n    ///\n    /// The default if not set is `GET`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .method(\"POST\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Method`.\n    pub fn method(mut self, method: &str) -> Self {\n        *self.req.method_mut() = method.parse().expect(\"valid method\");\n        self\n    }\n\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .path(\"/todos/33\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(mut self, p: &str) -> Self {\n        let uri = p.parse().expect(\"test request path invalid\");\n        *self.req.uri_mut() = uri;\n        self\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .header(\"accept\", \"application/json\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(mut self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        let name: HeaderName = TryFrom::try_from(key)\n            .map_err(|_| ())\n            .expect(\"invalid header name\");\n        let value = TryFrom::try_from(value)\n            .map_err(|_| ())\n            .expect(\"invalid header value\");\n        self.req.headers_mut().insert(name, value);\n        self\n    }\n\n    /// Set the remote address of this request\n    ///\n    /// Default is no remote address.\n    ///\n    /// # Example\n    /// ```\n    /// use std::net::{IpAddr, Ipv4Addr, SocketAddr};\n    ///\n    /// let req = warp::test::request()\n    ///     .remote_addr(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080));\n    /// ```\n    pub fn remote_addr(mut self, addr: SocketAddr) -> Self {\n        self.remote_addr = Some(addr);\n        self\n    }\n\n    /// Add a type to the request's `http::Extensions`.\n    \npub fn extension<T>(mut self, ext: T) -> Self\n    where\n        T: Send + Sync + 'static,\n    {\n        self.req.extensions_mut().insert(ext);\n        self\n    }\n\n    /// Set the bytes of this request body.\n    ///\n    /// Default is an empty body.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .body(\"foo=bar&baz=quux\");\n    /// ```\n    pub fn body(mut self, body: impl AsRef<[u8]>) -> Self {\n        let body = body.as_ref().to_vec();\n        let len = body.len();\n        *self.req.body_mut() = body.into();\n        self.header(\"content-length\", len.to_string())\n    }\n\n    /// Set the bytes of this request body by serializing a value into JSON.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::request()\n    ///     .json(&true);\n    /// ```\n    pub fn json(mut self, val: &impl Serialize) -> Self {\n        let vec = serde_json::to_vec(val).expect(\"json() must serialize to JSON\");\n        let len = vec.len();\n        *self.req.body_mut() = vec.into();\n        self.header(\"content-length\", len.to_string())\n            .header(\"content-type\", \"application/json\")\n    }\n\n    /// Tries to apply the `Filter` on this request.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let param = warp::path::param::<u32>();\n    ///\n    ///     let ex = warp::test::request()\n    ///         .path(\"/41\")\n    ///         .filter(&param)\n    ///         .await\n    ///         .unwrap();\n    ///\n    ///     assert_eq!(ex, 41);\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .path(\"/foo\")\n    ///             .filter(&param)\n    ///             .await\n    ///             .is_err()\n    ///     );\n    ///};\n    /// ```\n    pub async fn filter<F>(self, f: &F) -> Result<<F::Extract as OneOrTuple>::Output, F::Error>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: OneOrTuple + Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.map(|ex| ex.one_or_tuple())\n    }\n\n    /// Returns whether the `Filter` matches this request, or rejects it.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// async {\n    ///     let get = warp::get();\n    ///     let post = warp::post();\n    ///\n    ///     assert!(\n    ///         warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&get)\n    ///             .await\n    ///     );\n    ///\n    ///     assert!(\n    ///         !warp::test::request()\n    ///             .method(\"GET\")\n    ///             .matches(&post)\n    ///             .await\n    ///     );\n    ///};\n    /// ```\n    pub async fn matches<F>(self, f: &F) -> bool\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        self.apply_filter(f).await.is_ok()\n    }\n\n    /// Returns `Response` provided by applying the `Filter`.\n    ///\n    /// This requires that the supplied `Filter` return a [`Reply`].\n    pub async fn reply<F>(self, f: &F) -> Response<Bytes>\n    where\n        F: Filter + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        // TODO: de-duplicate this and apply_filter()\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(\n            route::set(&route, move || f.filter(crate::filter::Internal)).then(|result| {\n                let res = match result {\n                    Ok(rep) => rep.into_response(),\n                    Err(rej) => {\n                        tracing::debug!(\"rejected: {:?}\", rej);\n                        rej.into_response()\n                    }\n                };\n                let (parts, body) = res.into_parts();\n                hyper::body::to_bytes(body).map_ok(|chunk| Response::from_parts(parts, chunk))\n            }),\n        );\n\n        let fut = future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)));\n\n        fut.await.expect(\"reply shouldn't fail\")\n    }\n\n    fn apply_filter<F>(self, f: &F) -> impl Future<Output = Result<F::Extract, F::Error>>\n    where\n        F: Filter,\n        F::Future: Send + 'static,\n        F::Extract: Send + 'static,\n        F::Error: Send + 'static,\n    {\n        assert!(!route::is_set(), \"nested test filter calls\");\n\n        let route = Route::new(self.req, self.remote_addr);\n        let mut fut = Box::pin(route::set(&route, move || {\n            f.filter(crate::filter::Internal)\n        }));\n        future::poll_fn(move |cx| route::set(&route, || fut.as_mut().poll(cx)))\n    }\n}\n\n#[cfg(feature = \"websocket\")]\nimpl WsBuilder {\n    /// Sets the request path of this builder.\n    ///\n    /// The default is not set is `/`.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .path(\"/chat\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed string is not able to be parsed as a valid\n    /// `Uri`.\n    pub fn path(self, p: &str) -> Self {\n        WsBuilder {\n            req: self.req.path(p),\n        }\n    }\n\n    /// Set a header for this request.\n    ///\n    /// # Example\n    ///\n    /// ```\n    /// let req = warp::test::ws()\n    ///     .header(\"foo\", \"bar\");\n    /// ```\n    ///\n    /// # Panic\n    ///\n    /// This panics if the passed strings are not able to be parsed as a valid\n    /// `HeaderName` and `HeaderValue`.\n    pub fn header<K, V>(self, key: K, value: V) -> Self\n    where\n        HeaderName: TryFrom<K>,\n        HeaderValue: TryFrom<V>,\n    {\n        WsBuilder {\n            req: self.req.header(key, value),\n        }\n    }\n\n    /// Execute this Websocket request against the provided filter.\n    ///\n    /// If the handshake succeeds, returns a `WsClient`.\n    ///\n    /// # Example\n    ///\n    /// ```no_run\n    /// use futures_util::future;\n    /// use warp::Filter;\n    /// #[tokio::main]\n    /// # async fn main() {\n    ///\n    /// // Some route that accepts websockets (but drops them immediately).\n    /// let route = warp::ws()\n    ///     .map(|ws: warp::ws::Ws| {\n    ///         ws.on_upgrade(|_| future::ready(()))\n    ///     });\n    ///\n    /// let client = warp::test::ws()\n    ///     .handshake(route)\n    ///     .await\n    ///     .expect(\"handshake\");\n    /// # }\n    /// ```\n    pub async fn handshake<F>(self, f: F) -> Result<WsClient, WsError>\n    where\n        F: Filter + Clone + Send + Sync + 'static,\n        F::Extract: Reply + Send,\n        F::Error: IsReject + Send,\n    {\n        let (upgraded_tx, upgraded_rx) = oneshot::channel();\n        let (wr_tx, wr_rx) = mpsc::unbounded();\n        let (rd_tx, rd_rx) = mpsc::unbounded();\n\n        tokio::spawn(async move {\n            use tokio_tungstenite::tungstenite::protocol;\n\n            let (addr, srv) = crate::serve(f).bind_ephemeral(([127, 0, 0, 1], 0));\n\n            let mut req = self\n                .req\n                .header(\"connection\", \"upgrade\")\n                .header(\"upgrade\", \"websocket\")\n                .header(\"sec-websocket-version\", \"13\")\n                .header(\"sec-websocket-key\", \"dGhlIHNhbXBsZSBub25jZQ==\")\n                .req;\n\n            let query_string = match req.uri().query() {\n                Some(q) => format!(\"?{}\", q),\n                None => String::from(\"\"),\n            };\n\n            let uri = format!(\"http://{}{}{}\", addr, req.uri().path(), query_string)\n                .parse()\n                .expect(\"addr + path is valid URI\");\n\n            *req.uri_mut() = uri;\n\n            // let mut rt = current_thread::Runtime::new().unwrap();\n            tokio::spawn(srv);\n\n...\n", "needle_token_start": 13777, "needle_token_end": 13830, "code_context_ntokens": 16237, "output": ["```rust\n/// Add a type to the request's `http::Extensions`.\npub fn extension<T>(mut self, ext: T) -> Self\nwhere\n    T: Send + Sync + 'static,\n{\n    self.req.extensions_mut().insert(ext);\n    self\n}\n```"]}
{"repo": "seanmonstar/warp", "name": "__reject_custom_compilefail", "language": "rust", "path": "src/reject.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To demonstrate a compilation failure scenario when attempting to re-reject an already rejected request, serving as a safeguard in the code documentation.\n2. **Input**: None, as this function is used purely for illustrative purposes in documentation.\n3. **Output**: None, since it does not execute any operational code but exists to trigger a compile-time error when used incorrectly.\n4. **Procedure**: The function is intentionally designed to be empty and is used in a code example that should fail to compile if someone tries to misuse the rejection mechanism in a specific way.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/filters/path.rs\n//! Path Filters\n//!\n//! The [`Filter`](crate::Filter)s here work on the \"path\" of requests.\n//!\n//! - [`path`](./fn.path.html) matches a specific segment, like `/foo`.\n//! - [`param`](./fn.param.html) tries to parse a segment into a type, like `/:u16`.\n//! - [`end`](./fn.end.html) matches when the path end is found.\n//! - [`path!`](../../macro.path.html) eases combining multiple `path` and `param` filters.\n//!\n//! # Routing\n//!\n//! Routing in warp is simple yet powerful.\n//!\n//! First up, matching a single segment:\n//!\n//! ```\n//! use warp::Filter;\n//!\n//! // GET /hi\n//! let hi = warp::path(\"hi\").map(|| {\n//!     \"Hello, World!\"\n//! });\n//! ```\n//!\n//! How about multiple segments? It's easiest with the `path!` macro:\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /hello/from/warp\n//! let hello_from_warp = warp::path!(\"hello\" / \"from\" / \"warp\").map(|| {\n//!     \"Hello from warp!\"\n//! });\n//! ```\n//!\n//! Neat! But how do I handle **parameters** in paths?\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /sum/:u32/:u32\n//! let sum = warp::path!(\"sum\" / u32 / u32).map(|a, b| {\n//!     format!(\"{} + {} = {}\", a, b, a + b)\n//! });\n//! ```\n//!\n//! In fact, any type that implements `FromStr` can be used, in any order:\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /:u16/times/:u16\n//! let times = warp::path!(u16 / \"times\" / u16).map(|a, b| {\n//!     format!(\"{} times {} = {}\", a, b, a * b)\n//! });\n//! ```\n//!\n//! Oh shoot, those math routes should be **mounted** at a different path,\n//! is that possible? Yep!\n//!\n//! ```\n//! # use warp::Filter;\n//! # let sum = warp::any().map(warp::reply);\n//! # let times = sum.clone();\n//! // GET /math/sum/:u32/:u32\n//! // GET /math/:u16/times/:u16\n//! let math = warp::path(\"math\");\n//! let math_sum = math.and(sum);\n//! let math_times = math.and(times);\n//! ```\n//!\n//! What! `and`? What's that do?\n//!\n//! It combines the filters in a sort of \"this and then that\" order. In fact,\n//! it's exactly what the `path!` macro has been doing internally.\n//!\n//! ```\n//! # use warp::Filter;\n//! // GET /bye/:string\n//! let bye = warp::path(\"bye\")\n//!     .and(warp::path::param())\n//!     .map(|name: String| {\n//!         format!(\"Good bye, {}!\", name)\n//!     });\n//! ```\n//!\n//! Ah, so, can filters do things besides `and`?\n//!\n//! Why, yes they can! They can also `or`! As you might expect, `or` creates a\n//! \"this or else that\" chain of filters. If the first doesn't succeed, then\n//! it tries the other.\n//!\n//! So, those `math` routes could have been **mounted** all as one, with `or`.\n//!\n//!\n//! ```\n//! # use warp::Filter;\n//! # let sum = warp::path(\"sum\");\n//! # let times = warp::path(\"times\");\n//! // GET /math/sum/:u32/:u32\n//! // GET /math/:u16/times/:u16\n//! let math = warp::path(\"math\")\n//!     .and(sum.or(times));\n//! ```\n//!\n//! It turns out, using `or` is how you combine everything together into a\n//! single API.\n//!\n//! ```\n//! # use warp::Filter;\n//! # let hi = warp::path(\"hi\");\n//! # let hello_from_warp = hi.clone();\n//! # let bye = hi.clone();\n//! # let math = hi.clone();\n//! // GET /hi\n//! // GET /hello/from/warp\n//! // GET /bye/:string\n//! // GET /math/sum/:u32/:u32\n//! // GET /math/:u16/times/:u16\n//! let routes = hi\n//!     .or(hello_from_warp)\n//!     .or(bye)\n//!     .or(math);\n//! ```\n//!\n//! Note that you will generally want path filters to come **before** other filters\n//! like `body` or `headers`. If a different type of filter comes first, a request\n//! with an invalid body for route `/right-path-wrong-body` may try matching against `/wrong-path`\n//! and return the error from `/wrong-path` instead of the correct body-related error.\n\nuse std::convert::Infallible;\nuse std::fmt;\nuse std::str::FromStr;\n\nuse futures_util::future;\nuse http::uri::PathAndQuery;\n\nuse self::internal::Opaque;\nuse crate::filter::{filter_fn, one, Filter, FilterBase, Internal, One, Tuple};\nuse crate::reject::{self, Rejection};\nuse crate::route::{self, Route};\n\n/// Create an exact match path segment [`Filter`](crate::Filter).\n///\n/// This will try to match exactly to the current request path segment.\n///\n/// # Note\n///\n/// - [`end()`](./fn.end.html) should be used to match the end of a path to avoid having\n///   filters for shorter paths like `/math` unintentionally match a longer\n///   path such as `/math/sum`\n/// - Path-related filters should generally come **before** other types of filters, such\n///   as those checking headers or body types. Including those other filters before\n///   the path checks may result in strange errors being returned because a given request\n///   does not match the parameters for a completely separate route.\n///\n/// # Panics\n///\n/// Exact path filters cannot be empty, or contain slashes.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Matches '/hello'\n/// let hello = warp::path(\"hello\")\n///     .map(|| \"Hello, World!\");\n/// ```\npub fn path<P>(p: P) -> Exact<Opaque<P>>\nwhere\n    P: AsRef<str>,\n{\n    let s = p.as_ref();\n    assert!(!s.is_empty(), \"exact path segments should not be empty\");\n    assert!(\n        !s.contains('/'),\n        \"exact path segments should not contain a slash: {:?}\",\n        s\n    );\n\n    Exact(Opaque(p))\n    /*\n    segment(move |seg| {\n        tracing::trace!(\"{:?}?: {:?}\", p, seg);\n        if seg == p {\n            Ok(())\n        } else {\n            Err(reject::not_found())\n        }\n    })\n    */\n}\n\n/// A [`Filter`](crate::Filter) matching an exact path segment.\n///\n/// Constructed from `path()` or `path!()`.\n#[allow(missing_debug_implementations)]\n#[derive(Clone, Copy)]\npub struct Exact<P>(P);\n\nimpl<P> FilterBase for Exact<P>\nwhere\n    P: AsRef<str>,\n{\n    type Extract = ();\n    type Error = Rejection;\n    type Future = future::Ready<Result<Self::Extract, Self::Error>>;\n\n    #[inline]\n    fn filter(&self, _: Internal) -> Self::Future {\n        route::with(|route| {\n            let p = self.0.as_ref();\n            future::ready(with_segment(route, |seg| {\n                tracing::trace!(\"{:?}?: {:?}\", p, seg);\n\n                if seg == p {\n                    Ok(())\n                } else {\n                    Err(reject::not_found())\n                }\n            }))\n...\n// Path: src/filters/fs.rs\n//! File System Filters\n\nuse std::cmp;\nuse std::convert::Infallible;\nuse std::fs::Metadata;\nuse std::future::Future;\nuse std::io;\nuse std::path::{Path, PathBuf};\nuse std::pin::Pin;\nuse std::sync::Arc;\nuse std::task::Poll;\n\nuse bytes::{Bytes, BytesMut};\nuse futures_util::future::Either;\nuse futures_util::{future, ready, stream, FutureExt, Stream, StreamExt, TryFutureExt};\nuse headers::{\n    AcceptRanges, ContentLength, ContentRange, ContentType, HeaderMapExt, IfModifiedSince, IfRange,\n    IfUnmodifiedSince, LastModified, Range,\n};\nuse http::StatusCode;\nuse hyper::Body;\nuse percent_encoding::percent_decode_str;\nuse tokio::fs::File as TkFile;\nuse tokio::io::AsyncSeekExt;\nuse tokio_util::io::poll_read_buf;\n\nuse crate::filter::{Filter, FilterClone, One};\nuse crate::reject::{self, Rejection};\nuse crate::reply::{Reply, Response};\n\n/// Creates a `Filter` that serves a File at the `path`.\n///\n/// Does not filter out based on any information of the request. Always serves\n/// the file at the exact `path` provided. Thus, this can be used to serve a\n/// single file with `GET`s, but could also be used in combination with other\n/// filters, such as after validating in `POST` request, wanting to return a\n/// specific file as the body.\n///\n/// For serving a directory, see [dir].\n///\n/// # Example\n///\n/// ```\n/// // Always serves this file from the file system.\n/// let route = warp::fs::file(\"/www/static/app.js\");\n/// ```\npub fn file(path: impl Into<PathBuf>) -> impl FilterClone<Extract = One<File>, Error = Rejection> {\n    let path = Arc::new(path.into());\n    crate::any()\n        .map(move || {\n            tracing::trace!(\"file: {:?}\", path);\n            ArcPath(path.clone())\n        })\n        .and(conditionals())\n        .and_then(file_reply)\n}\n\n/// Creates a `Filter` that serves a directory at the base `path` joined\n/// by the request path.\n///\n/// This can be used to serve \"static files\" from a directory. By far the most\n/// common pattern of serving static files is for `GET` requests, so this\n/// filter automatically includes a `GET` check.\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// // Matches requests that start with `/static`,\n/// // and then uses the rest of that path to lookup\n/// // and serve a file from `/www/static`.\n/// let route = warp::path(\"static\")\n///     .and(warp::fs::dir(\"/www/static\"));\n///\n/// // For example:\n/// // - `GET /static/app.js` would serve the file `/www/static/app.js`\n/// // - `GET /static/css/app.css` would serve the file `/www/static/css/app.css`\n/// ```\npub fn dir(path: impl Into<PathBuf>) -> impl FilterClone<Extract = One<File>, Error = Rejection> {\n    let base = Arc::new(path.into());\n    crate::get()\n        .or(crate::head())\n        .unify()\n        .and(path_from_tail(base))\n        .and(conditionals())\n        .and_then(file_reply)\n}\n\nfn path_from_tail(\n    base: Arc<PathBuf>,\n) -> impl FilterClone<Extract = One<ArcPath>, Error = Rejection> {\n    crate::path::tail().and_then(move |tail: crate::path::Tail| {\n        future::ready(sanitize_path(base.as_ref(), tail.as_str())).and_then(|mut buf| async {\n            let is_dir = tokio::fs::metadata(buf.clone())\n                .await\n                .map(|m| m.is_dir())\n                .unwrap_or(false);\n\n            if is_dir {\n                tracing::debug!(\"dir: appending index.html to directory path\");\n                buf.push(\"index.html\");\n            }\n            tracing::trace!(\"dir: {:?}\", buf);\n            Ok(ArcPath(Arc::new(buf)))\n        })\n    })\n}\n\nfn sanitize_path(base: impl AsRef<Path>, tail: &str) -> Result<PathBuf, Rejection> {\n    let mut buf = PathBuf::from(base.as_ref());\n    let p = match percent_decode_str(tail).decode_utf8() {\n        Ok(p) => p,\n        Err(err) => {\n            tracing::debug!(\"dir: failed to decode route={:?}: {:?}\", tail, err);\n            return Err(reject::not_found());\n        }\n    };\n    tracing::trace!(\"dir? base={:?}, route={:?}\", base.as_ref(), p);\n    for seg in p.split('/') {\n        if seg.starts_with(\"..\") {\n            tracing::warn!(\"dir: rejecting segment starting with '..'\");\n            return Err(reject::not_found());\n        } else if seg.contains('\\\\') {\n            tracing::warn!(\"dir: rejecting segment containing backslash (\\\\)\");\n            return Err(reject::not_found());\n        } else if cfg!(windows) && seg.contains(':') {\n            tracing::warn!(\"dir: rejecting segment containing colon (:)\");\n            return Err(reject::not_found());\n        } else {\n            buf.push(seg);\n        }\n    }\n    Ok(buf)\n}\n\n#[derive(Debug)]\nstruct Conditionals {\n    if_modified_since: Option<IfModifiedSince>,\n    if_unmodified_since: Option<IfUnmodifiedSince>,\n    if_range: Option<IfRange>,\n    range: Option<Range>,\n}\n\nenum Cond {\n    NoBody(Response),\n    WithBody(Option<Range>),\n}\n\nimpl Conditionals {\n    fn check(self, last_modified: Option<LastModified>) -> Cond {\n        if let Some(since) = self.if_unmodified_since {\n            let precondition = last_modified\n                .map(|time| since.precondition_passes(time.into()))\n                .unwrap_or(false);\n\n            tracing::trace!(\n                \"if-unmodified-since? {:?} vs {:?} = {}\",\n                since,\n                last_modified,\n                precondition\n            );\n            if !precondition {\n                let mut res = Response::new(Body::empty());\n                *res.status_mut() = StatusCode::PRECONDITION_FAILED;\n                return Cond::NoBody(res);\n            }\n        }\n\n        if let Some(since) = self.if_modified_since {\n            tracing::trace!(\n                \"if-modified-since? header = {:?}, file = {:?}\",\n                since,\n                last_modified\n            );\n            let unmodified = last_modified\n                .map(|time| !since.is_modified(time.into()))\n                // no last_modified means its always modified\n                .unwrap_or(false);\n            if unmodified {\n                let mut res = Response::new(Body::empty());\n                *res.status_mut() = StatusCode::NOT_MODIFIED;\n                return Cond::NoBody(res);\n            }\n        }\n\n        if let Some(if_range) = self.if_range {\n            tracing::trace!(\"if-range? {:?} vs {:?}\", if_range, last_modified);\n            let can_range = !if_range.is_modified(None, last_modified.as_ref());\n\n            if !can_range {\n                return Cond::WithBody(None);\n            }\n        }\n\n        Cond::WithBody(self.range)\n    }\n}\n\nfn conditionals() -> impl Filter<Extract = One<Conditionals>, Error = Infallible> + Copy {\n    crate::header::optional2()\n        .and(crate::header::optional2())\n        .and(crate::header::optional2())\n        .and(crate::header::optional2())\n        .map(\n            |if_modified_since, if_unmodified_since, if_range, range| Conditionals {\n                if_modified_since,\n                if_unmodified_since,\n                if_range,\n                range,\n            },\n        )\n}\n\n/// A file response.\n#[derive(Debug)]\npub struct File {\n    resp: Response,\n    path: ArcPath,\n}\n\nimpl File {\n    /// Extract the `&Path` of the file this `Response` delivers.\n    ///\n    /// # Example\n    ///\n    /// The example below changes the Content-Type response header for every file called `video.mp4`.\n    ///\n    /// ```\n    /// use warp::{Filter, reply::Reply};\n    ///\n    /// let route = warp::path(\"static\")\n    ///     .and(warp::fs::dir(\"/www/static\"))\n    ///     .map(|reply: warp::filters::fs::File| {\n    ///         if reply.path().ends_with(\"video.mp4\") {\n    ///             warp::reply::with_header(reply, \"Content-Type\", \"video/mp4\").into_response()\n    ///         } else {\n    ///             reply.into_response()\n    ///         }\n    ///     });\n    /// ```\n    pub fn path(&self) -> &Path {\n        self.path.as_ref()\n    }\n}\n\n// Silly wrapper since Arc<PathBuf> doesn't implement AsRef<Path> ;_;\n#[derive(Clone, Debug)]\nstruct ArcPath(Arc<PathBuf>);\n\nimpl AsRef<Path> for ArcPath {\n    fn as_ref(&self) -> &Path {\n        (*self.0).as_ref()\n    }\n}\n\nimpl Reply for File {\n    fn into_response(self) -> Response {\n        self.resp\n    }\n}\n\nfn file_reply(\n    path: ArcPath,\n    conditionals: Conditionals,\n) -> impl Future<Output = Result<File, Rejection>> + Send {\n    TkFile::open(path.clone()).then(move |res| match res {\n        Ok(f) => Either::Left(file_conditional(f, path, conditionals)),\n        Err(err) => {\n            let rej = match err.kind() {\n                io::ErrorKind::NotFound => {\n                    tracing::debug!(\"file not found: {:?}\", path.as_ref().display());\n                    reject::not_found()\n                }\n                io::ErrorKind::PermissionDenied => {\n                    tracing::warn!(\"file permission denied: {:?}\", path.as_ref().display());\n                    reject::known(FilePermissionError { _p: () })\n                }\n                _ => {\n                    tracing::error!(\n                        \"file open error (path={:?}): {} \",\n                        path.as_ref().display(),\n                        err\n                    );\n                    reject::known(FileOpenError { _p: () })\n                }\n            };\n            Either::Right(future::err(rej))\n        }\n    })\n}\n\nasync fn file_metadata(f: TkFile) -> Result<(TkFile, Metadata), Rejection> {\n    match f.metadata().await {\n        Ok(meta) => Ok((f, meta)),\n        Err(err) => {\n            tracing::debug!(\"file metadata error: {}\", err);\n            Err(reject::not_found())\n        }\n    }\n}\n\nfn file_conditional(\n    f: TkFile,\n    path: ArcPath,\n    conditionals: Conditionals,\n) -> impl Future<Output = Result<File, Rejection>> + Send {\n    file_metadata(f).map_ok(move |(file, meta)| {\n        let mut len = meta.len();\n        let modified = meta.modified().ok().map(LastModified::from);\n\n        let resp = match conditionals.check(modified) {\n            Cond::NoBody(resp) => resp,\n            Cond::WithBody(range) => {\n                bytes_range(range, len)\n                    .map(|(start, end)| {\n                        let sub_len = end - start;\n                        let buf_size = optimal_buf_size(&meta);\n                        let stream = file_stream(file, buf_size, (start, end));\n                        let body = Body::wrap_stream(stream);\n\n                        let mut resp = Response::new(body);\n\n                        if sub_len != len {\n                            *resp.status_mut() = StatusCode::PARTIAL_CONTENT;\n                            resp.headers_mut().typed_insert(\n                                ContentRange::bytes(start..end, len).expect(\"valid ContentRange\"),\n                            );\n\n                            len = sub_len;\n                        }\n\n                        let mime = mime_guess::from_path(path.as_ref()).first_or_octet_stream();\n\n                        resp.headers_mut().typed_insert(ContentLength(len));\n                        resp.headers_mut().typed_insert(ContentType::from(mime));\n                        resp.headers_mut().typed_insert(AcceptRanges::bytes());\n\n                        if let Some(last_modified) = modified {\n                            resp.headers_mut().typed_insert(last_modified);\n                        }\n\n                        resp\n                    })\n                    .unwrap_or_else(|BadRange| {\n                        // bad byte range\n                        let mut resp = Response::new(Body::empty());\n                        *resp.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE;\n                        resp.headers_mut()\n                            .typed_insert(ContentRange::unsatisfied_bytes(len));\n                        resp\n                    })\n            }\n        };\n\n        File { resp, path }\n    })\n}\n\nstruct BadRange;\n\nfn bytes_range(range: Option<Range>, max_len: u64) -> Result<(u64, u64), BadRange> {\n    use std::ops::Bound;\n\n    let range = if let Some(range) = range {\n        range\n    } else {\n        return Ok((0, max_len));\n    };\n\n    let ret = range\n        .iter()\n        .map(|(start, end)| {\n            let start = match start {\n                Bound::Unbounded => 0,\n                Bound::Included(s) => s,\n                Bound::Excluded(s) => s + 1,\n            };\n\n            let end = match end {\n                Bound::Unbounded => max_len,\n                Bound::Included(s) => {\n                    // For the special case where s == the file size\n                    if s == max_len {\n                        s\n                    } else {\n                        s + 1\n                    }\n                }\n                Bound::Excluded(s) => s,\n            };\n\n            if start < end && end <= max_len {\n                Ok((start, end))\n            } else {\n                tracing::trace!(\"unsatisfiable byte range: {}-{}/{}\", start, end, max_len);\n                Err(BadRange)\n            }\n        })\n        .next()\n        .unwrap_or(Ok((0, max_len)));\n    ret\n}\n\nfn file_stream(\n    mut file: TkFile,\n    buf_size: usize,\n    (start, end): (u64, u64),\n) -> impl Stream<Item = Result<Bytes, io::Error>> + Send {\n    use std::io::SeekFrom;\n\n    let seek = async move {\n        if start != 0 {\n            file.seek(SeekFrom::Start(start)).await?;\n        }\n        Ok(file)\n    };\n\n    seek.into_stream()\n        .map(move |result| {\n            let mut buf = BytesMut::new();\n            let mut len = end - start;\n            let mut f = match result {\n                Ok(f) => f,\n                Err(f) => return Either::Left(stream::once(future::err(f))),\n            };\n\n            Either::Right(stream::poll_fn(move |cx| {\n                if len == 0 {\n                    return Poll::Ready(None);\n                }\n                reserve_at_least(&mut buf, buf_size);\n\n                let n = match ready!(poll_read_buf(Pin::new(&mut f), cx, &mut buf)) {\n                    Ok(n) => n as u64,\n                    Err(err) => {\n                        tracing::debug!(\"file read error: {}\", err);\n                        return Poll::Ready(Some(Err(err)));\n                    }\n                };\n\n                if n == 0 {\n                    tracing::debug!(\"file read found EOF before expected length\");\n                    return Poll::Ready(None);\n                }\n\n                let mut chunk = buf.split().freeze();\n                if n > len {\n                    chunk = chunk.split_to(len as usize);\n                    len = 0;\n                } else {\n                    len -= n;\n                }\n\n                Poll::Ready(Some(Ok(chunk)))\n            }))\n        })\n        .flatten()\n}\n\nfn reserve_at_least(buf: &mut BytesMut, cap: usize) {\n    if buf.capacity() - buf.len() < cap {\n        buf.reserve(cap);\n    }\n}\n\nconst DEFAULT_READ_BUF_SIZE: usize = 8_192;\n\nfn optimal_buf_size(metadata: &Metadata) -> usize {\n    let block_size = get_block_size(metadata);\n\n    // If file length is smaller than block size, don't waste space\n    // reserving a bigger-than-needed buffer.\n    cmp::min(block_size as u64, metadata.len()) as usize\n}\n\n#[cfg(unix)]\nfn get_block_size(metadata: &Metadata) -> usize {\n    use std::os::unix::fs::MetadataExt;\n    //TODO: blksize() returns u64, should handle bad cast...\n    //(really, a block size bigger than 4gb?)\n\n    // Use device blocksize unless it's really small.\n    cmp::max(metadata.blksize() as usize, DEFAULT_READ_BUF_SIZE)\n}\n\n#[cfg(not(unix))]\nfn get_block_size(_metadata: &Metadata) -> usize {\n    DEFAULT_READ_BUF_SIZE\n}\n\n// ===== Rejections =====\n\nunit_error! {\n    pub(crate) FileOpenError: \"file open error\"\n}\n\nunit_error! {\n    pub(crate) FilePermissionError: \"file perimission error\"\n}\n\n#[cfg(test)]\nmod tests {\n    use super::sanitize_path;\n    use bytes::BytesMut;\n\n    #[test]\n    fn test_sanitize_path() {\n        let base = \"/var/www\";\n\n        fn p(s: &str) -> &::std::path::Path {\n            s.as_ref()\n        }\n\n        assert_eq!(\n            sanitize_path(base, \"/foo.html\").unwrap(),\n            p(\"/var/www/foo.html\")\n        );\n\n        // bad paths\n        sanitize_path(base, \"/../foo.html\").expect_err(\"dot dot\");\n\n        sanitize_path(base, \"/C:\\\\/foo.html\").expect_err(\"C:\\\\\");\n    }\n\n    #[test]\n    fn test_reserve_at_least() {\n        let mut buf = BytesMut::new();\n        let cap = 8_192;\n\n        assert_eq!(buf.len(), 0);\n        assert_eq!(buf.capacity(), 0);\n\n        super::reserve_at_least(&mut buf, cap);\n        assert_eq!(buf.len(), 0);\n        assert_eq!(buf.capacity(), cap);\n    }\n}\n\n// Path: src/filter/wrap.rs\nuse super::Filter;\n\npub trait WrapSealed<F: Filter> {\n    type Wrapped: Filter;\n\n    fn wrap(&self, filter: F) -> Self::Wrapped;\n}\n\nimpl<'a, T, F> WrapSealed<F> for &'a T\nwhere\n    T: WrapSealed<F>,\n    F: Filter,\n{\n    type Wrapped = T::Wrapped;\n    fn wrap(&self, filter: F) -> Self::Wrapped {\n        (*self).wrap(filter)\n    }\n}\n\npub trait Wrap<F: Filter>: WrapSealed<F> {}\n\nimpl<T, F> Wrap<F> for T\nwhere\n    T: WrapSealed<F>,\n    F: Filter,\n{\n}\n\n/// Combines received filter with pre and after filters\n///\n/// # Example\n///\n/// ```\n/// use crate::warp::Filter;\n///\n/// let route = warp::any()\n///     .map(|| \"hello world\")\n///     .with(warp::wrap_fn(|filter| filter));\n/// ```\n///\n/// You can find the full example in the [usage example](https://github.com/seanmonstar/warp/blob/master/examples/wrapping.rs).\npub fn wrap_fn<F, T, U>(func: F) -> WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    WrapFn { func }\n}\n\n#[derive(Debug)]\npub struct WrapFn<F> {\n    func: F,\n}\n\nimpl<F, T, U> WrapSealed<T> for WrapFn<F>\nwhere\n    F: Fn(T) -> U,\n    T: Filter,\n    U: Filter,\n{\n    type Wrapped = U;\n\n    fn wrap(&self, filter: T) -> Self::Wrapped {\n        (self.func)(filter)\n    }\n}\n\n// Path: src/filters/cors.rs\n//! CORS Filters\n\nuse std::collections::HashSet;\nuse std::convert::TryFrom;\nuse std::error::Error as StdError;\nuse std::fmt;\nuse std::sync::Arc;\n\nuse headers::{\n    AccessControlAllowHeaders, AccessControlAllowMethods, AccessControlExposeHeaders, HeaderMapExt,\n};\nuse http::header::{self, HeaderName, HeaderValue};\n\nuse crate::filter::{Filter, WrapSealed};\nuse crate::reject::{CombineRejection, Rejection};\nuse crate::reply::Reply;\n\nuse self::internal::{CorsFilter, IntoOrigin, Seconds};\n\n/// Create a wrapping [`Filter`](crate::Filter) that exposes [CORS][] behavior for a wrapped\n/// filter.\n///\n/// [CORS]: https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS\n///\n/// # Example\n///\n/// ```\n/// use warp::Filter;\n///\n/// let cors = warp::cors()\n///     .allow_origin(\"https://hyper.rs\")\n///     .allow_methods(vec![\"GET\", \"POST\", \"DELETE\"]);\n///\n/// let route = warp::any()\n///     .map(warp::reply)\n///     .with(cors);\n/// ```\n/// If you want to allow any route:\n/// ```\n/// use warp::Filter;\n/// let cors = warp::cors()\n///     .allow_any_origin();\n/// ```\n/// You can find more usage examples [here](https://github.com/seanmonstar/warp/blob/7fa54eaecd0fe12687137372791ff22fc7995766/tests/cors.rs).\npub fn cors() -> Builder {\n    Builder {\n        credentials: false,\n        allowed_headers: HashSet::new(),\n        exposed_headers: HashSet::new(),\n        max_age: None,\n        methods: HashSet::new(),\n        origins: None,\n    }\n}\n\n/// A wrapping [`Filter`](crate::Filter) constructed via `warp::cors()`.\n#[derive(Clone, Debug)]\npub struct Cors {\n    config: Arc<Configured>,\n}\n\n/// A constructed via `warp::cors()`.\n#[derive(Clone, Debug)]\npub struct Builder {\n    credentials: bool,\n    allowed_headers: HashSet<HeaderName>,\n    exposed_headers: HashSet<HeaderName>,\n    max_age: Option<u64>,\n    methods: HashSet<http::Method>,\n    origins: Option<HashSet<HeaderValue>>,\n}\n\nimpl Builder {\n    /// Sets whether to add the `Access-Control-Allow-Credentials` header.\n    pub fn allow_credentials(mut self, allow: bool) -> Self {\n        self.credentials = allow;\n        self\n    }\n\n    /// Adds a method to the existing list of allowed request methods.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::Method`.\n    pub fn allow_method<M>(mut self, method: M) -> Self\n    where\n        http::Method: TryFrom<M>,\n    {\n        let method = match TryFrom::try_from(method) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Method\"),\n        };\n        self.methods.insert(method);\n        self\n    }\n\n    /// Adds multiple methods to the existing list of allowed request methods.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::Method`.\n    pub fn allow_methods<I>(mut self, methods: I) -> Self\n    where\n        I: IntoIterator,\n        http::Method: TryFrom<I::Item>,\n    {\n        let iter = methods.into_iter().map(|m| match TryFrom::try_from(m) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Method\"),\n        });\n        self.methods.extend(iter);\n        self\n    }\n\n    /// Adds a header to the list of allowed request headers.\n    ///\n    /// **Note**: These should match the values the browser sends via `Access-Control-Request-Headers`, e.g. `content-type`.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::header::HeaderName`.\n    pub fn allow_header<H>(mut self, header: H) -> Self\n    where\n        HeaderName: TryFrom<H>,\n    {\n        let header = match TryFrom::try_from(header) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Header\"),\n        };\n        self.allowed_headers.insert(header);\n        self\n    }\n\n    /// Adds multiple headers to the list of allowed request headers.\n    ///\n    /// **Note**: These should match the values the browser sends via `Access-Control-Request-Headers`, e.g.`content-type`.\n    ///\n    /// # Panics\n    ///\n    /// Panics if any of the headers are not a valid `http::header::HeaderName`.\n    pub fn allow_headers<I>(mut self, headers: I) -> Self\n    where\n        I: IntoIterator,\n        HeaderName: TryFrom<I::Item>,\n    {\n        let iter = headers.into_iter().map(|h| match TryFrom::try_from(h) {\n            Ok(h) => h,\n            Err(_) => panic!(\"illegal Header\"),\n        });\n        self.allowed_headers.extend(iter);\n        self\n    }\n\n    /// Adds a header to the list of exposed headers.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `http::header::HeaderName`.\n    pub fn expose_header<H>(mut self, header: H) -> Self\n    where\n        HeaderName: TryFrom<H>,\n    {\n        let header = match TryFrom::try_from(header) {\n            Ok(m) => m,\n            Err(_) => panic!(\"illegal Header\"),\n        };\n        self.exposed_headers.insert(header);\n        self\n    }\n\n    /// Adds multiple headers to the list of exposed headers.\n    ///\n    /// # Panics\n    ///\n    /// Panics if any of the headers are not a valid `http::header::HeaderName`.\n    pub fn expose_headers<I>(mut self, headers: I) -> Self\n    where\n        I: IntoIterator,\n        HeaderName: TryFrom<I::Item>,\n    {\n        let iter = headers.into_iter().map(|h| match TryFrom::try_from(h) {\n            Ok(h) => h,\n            Err(_) => panic!(\"illegal Header\"),\n        });\n        self.exposed_headers.extend(iter);\n        self\n    }\n\n    /// Sets that *any* `Origin` header is allowed.\n    ///\n    /// # Warning\n    ///\n    /// This can allow websites you didn't intend to access this resource,\n    /// it is usually better to set an explicit list.\n    pub fn allow_any_origin(mut self) -> Self {\n        self.origins = None;\n        self\n    }\n\n    /// Add an origin to the existing list of allowed `Origin`s.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `Origin`.\n    pub fn allow_origin(self, origin: impl IntoOrigin) -> Self {\n        self.allow_origins(Some(origin))\n    }\n\n    /// Add multiple origins to the existing list of allowed `Origin`s.\n    ///\n    /// # Panics\n    ///\n    /// Panics if the provided argument is not a valid `Origin`.\n    pub fn allow_origins<I>(mut self, origins: I) -> Self\n    where\n        I: IntoIterator,\n        I::Item: IntoOrigin,\n    {\n        let iter = origins\n            .into_iter()\n            .map(IntoOrigin::into_origin)\n            .map(|origin| {\n                origin\n                    .to_string()\n                    .parse()\n                    .expect(\"Origin is always a valid HeaderValue\")\n            });\n\n        self.origins.get_or_insert_with(HashSet::new).extend(iter);\n\n        self\n    }\n\n    /// Sets the `Access-Control-Max-Age` header.\n    ///\n    /// # Example\n    ///\n    ///\n    /// ```\n    /// use std::time::Duration;\n    /// use warp::Filter;\n    ///\n    /// let cors = warp::cors()\n    ///     .max_age(30) // 30u32 seconds\n    ///     .max_age(Duration::from_secs(30)); // or a Duration\n    /// ```\n    pub fn max_age(mut self, seconds: impl Seconds) -> Self {\n        self.max_age = Some(seconds.seconds());\n        self\n    }\n\n    /// Builds the `Cors` wrapper from the configured settings.\n    ///\n    /// This step isn't *required*, as the `Builder` itself can be passed\n    /// to `Filter::with`. This just allows constructing once, thus not needing\n    /// to pay the cost of \"building\" every time.\n    pub fn build(self) -> Cors {\n        let expose_headers_header = if self.exposed_headers.is_empty() {\n            None\n        } else {\n            Some(self.exposed_headers.iter().cloned().collect())\n        };\n        let allowed_headers_header = self.allowed_headers.iter().cloned().collect();\n        let methods_header = self.methods.iter().cloned().collect();\n\n        let config = Arc::new(Configured {\n            cors: self,\n            allowed_headers_header,\n            expose_headers_header,\n            methods_header,\n        });\n\n        Cors { config }\n    }\n}\n\nimpl<F> WrapSealed<F> for Builder\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: CombineRejection<Rejection>,\n    <F::Error as CombineRejection<Rejection>>::One: CombineRejection<Rejection>,\n{\n    type Wrapped = CorsFilter<F>;\n\n    fn wrap(&self, inner: F) -> Self::Wrapped {\n        let Cors { config } = self.clone().build();\n\n        CorsFilter { config, inner }\n    }\n}\n\nimpl<F> WrapSealed<F> for Cors\nwhere\n    F: Filter + Clone + Send + Sync + 'static,\n    F::Extract: Reply,\n    F::Error: CombineRejection<Rejection>,\n    <F::Error as CombineRejection<Rejection>>::One: CombineRejection<Rejection>,\n{\n    type Wrapped = CorsFilter<F>;\n\n    fn wrap(&self, inner: F) -> Self::Wrapped {\n        let config = self.config.clone();\n\n        CorsFilter { config, inner }\n    }\n}\n\n/// An error used to reject requests that are forbidden by a `cors` filter.\npub struct CorsForbidden {\n    kind: Forbidden,\n}\n\n#[derive(Debug)]\nenum Forbidden {\n    OriginNotAllowed,\n    MethodNotAllowed,\n    HeaderNotAllowed,\n}\n\nimpl fmt::Debug for CorsForbidden {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"CorsForbidden\").field(&self.kind).finish()\n    }\n}\n\nimpl fmt::Display for CorsForbidden {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let detail = match self.kind {\n            Forbidden::OriginNotAllowed => \"origin not allowed\",\n            Forbidden::MethodNotAllowed => \"request-method not allowed\",\n            Forbidden::HeaderNotAllowed => \"header not allowed\",\n        };\n        write!(f, \"CORS request forbidden: {}\", detail)\n    }\n}\n\nimpl StdError for CorsForbidden {}\n\n#[derive(Clone, Debug)]\nstruct Configured {\n    cors: Builder,\n    allowed_headers_header: AccessControlAllowHeaders,\n    expose_headers_header: Option<AccessControlExposeHeaders>,\n    methods_header: AccessControlAllowMethods,\n}\n\nenum Validated {\n    Preflight(HeaderValue),\n    Simple(HeaderValue),\n    NotCors,\n}\n\nimpl Configured {\n    fn check_request(\n        &self,\n        method: &http::Method,\n        headers: &http::HeaderMap,\n    ) -> Result<Validated, Forbidden> {\n        match (headers.get(header::ORIGIN), method) {\n            (Some(origin), &http::Method::OPTIONS) => {\n                // OPTIONS requests are preflight CORS requests...\n\n                if !self.is_origin_allowed(origin) {\n                    return Err(Forbidden::OriginNotAllowed);\n                }\n\n                if let Some(req_method) = headers.get(header::ACCESS_CONTROL_REQUEST_METHOD) {\n                    if !self.is_method_allowed(req_method) {\n                        return Err(Forbidden::MethodNotAllowed);\n                    }\n                } else {\n                    tracing::trace!(\n                        \"preflight request missing access-control-request-method header\"\n                    );\n                    return Err(Forbidden::MethodNotAllowed);\n                }\n\n                if let Some(req_headers) = headers.get(header::ACCESS_CONTROL_REQUEST_HEADERS) {\n                    let headers = req_headers\n                        .to_str()\n                        .map_err(|_| Forbidden::HeaderNotAllowed)?;\n                    for header in headers.split(',') {\n                        if !self.is_header_allowed(header.trim()) {\n                            return Err(Forbidden::HeaderNotAllowed);\n                        }\n                    }\n                }\n\n                Ok(Validated::Preflight(origin.clone()))\n            }\n            (Some(origin), _) => {\n                // Any other method, simply check for a valid origin...\n\n                tracing::trace!(\"origin header: {:?}\", origin);\n                if self.is_origin_allowed(origin) {\n                    Ok(Validated::Simple(origin.clone()))\n                } else {\n                    Err(Forbidden::OriginNotAllowed)\n                }\n            }\n            (None, _) => {\n                // No `ORIGIN` header means this isn't CORS!\n                Ok(Validated::NotCors)\n            }\n        }\n    }\n\n    fn is_method_allowed(&self, header: &HeaderValue) -> bool {\n        http::Method::from_bytes(header.as_bytes())\n            .map(|method| self.cors.methods.contains(&method))\n            .unwrap_or(false)\n    }\n\n    fn is_header_allowed(&self, header: &str) -> bool {\n        HeaderName::from_bytes(header.as_bytes())\n            .map(|header| self.cors.allowed_headers.contains(&header))\n            .unwrap_or(false)\n    }\n\n    fn is_origin_allowed(&self, origin: &HeaderValue) -> bool {\n        if let Some(ref allowed) = self.cors.origins {\n            allowed.contains(origin)\n        } else {\n            true\n        }\n    }\n\n    fn append_preflight_headers(&self, headers: &mut http::HeaderMap) {\n        self.append_common_headers(headers);\n\n        headers.typed_insert(self.allowed_headers_header.clone());\n        headers.typed_insert(self.methods_header.clone());\n\n        if let Some(max_age) = self.cors.max_age {\n            headers.insert(header::ACCESS_CONTROL_MAX_AGE, max_age.into());\n        }\n    }\n\n    fn append_common_headers(&self, headers: &mut http::HeaderMap) {\n        if self.cors.credentials {\n            headers.insert(\n                header::ACCESS_CONTROL_ALLOW_CREDENTIALS,\n                HeaderValue::from_static(\"true\"),\n            );\n        }\n        if let Some(expose_headers_header) = &self.expose_headers_header {\n            headers.typed_insert(expose_headers_header.clone())\n        }\n    }\n}\n\nmod internal {\n    use std::future::Future;\n    use std::pin::Pin;\n    use std::sync::Arc;\n    use std::task::{Context, Poll};\n\n    use futures_util::{future, ready, TryFuture};\n    use headers::Origin;\n    use http::header;\n    use pin_project::pin_project;\n\n    use super::{Configured, CorsForbidden, Validated};\n    use crate::filter::{Filter, FilterBase, Internal, One};\n    use crate::generic::Either;\n    use crate::reject::{CombineRejection, Rejection};\n    use crate::route;\n\n    #[derive(Clone, Debug)]\n    pub struct CorsFilter<F> {\n        pub(super) config: Arc<Configured>,\n        pub(super) inner: F,\n    }\n\n    impl<F> FilterBase for CorsFilter<F>\n    where\n        F: Filter,\n        F::Extract: Send,\n        F::Future: Future,\n        F::Error: CombineRejection<Rejection>,\n    {\n        type Extract =\n            One<Either<One<Preflight>, One<Either<One<Wrapped<F::Extract>>, F::Extract>>>>;\n        type Error = <F::Error as CombineRejection<Rejection>>::One;\n        type Future = future::Either<\n            future::Ready<Result<Self::Extract, Self::Error>>,\n            WrappedFuture<F::Future>,\n        >;\n\n        fn filter(&self, _: Internal) -> Self::Future {\n            let validated =\n                route::with(|route| self.config.check_request(route.method(), route.headers()));\n\n            match validated {\n                Ok(Validated::Preflight(origin)) => {\n                    let preflight = Preflight {\n                        config: self.config.clone(),\n                        origin,\n                    };\n                    future::Either::Left(future::ok((Either::A((preflight,)),)))\n                }\n                Ok(Validated::Simple(origin)) => future::Either::Right(WrappedFuture {\n                    inner: self.inner.filter(Internal),\n                    wrapped: Some((self.config.clone(), origin)),\n                }),\n                Ok(Validated::NotCors) => future::Either::Right(WrappedFuture {\n                    inner: self.inner.filter(Internal),\n                    wrapped: None,\n                }),\n                Err(err) => {\n                    let rejection = crate::reject::known(CorsForbidden { kind: err });\n                    future::Either::Left(future::err(rejection.into()))\n                }\n            }\n        }\n    }\n\n    #[derive(Debug)]\n    pub struct Preflight {\n        config: Arc<Configured>,\n        origin: header::HeaderValue,\n    }\n\n    impl crate::reply::Reply for Preflight {\n        fn into_response(self) -> crate::reply::Response {\n            let mut res = crate::reply::Response::default();\n            self.config.append_preflight_headers(res.headers_mut());\n            res.headers_mut()\n                .insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, self.origin);\n            res\n        }\n    }\n\n    #[derive(Debug)]\n    pub struct Wrapped<R> {\n        config: Arc<Configured>,\n        inner: R,\n        origin: header::HeaderValue,\n    }\n\n    impl<R> crate::reply::Reply for Wrapped<R>\n    where\n        R: crate::reply::Reply,\n    {\n        fn into_response(self) -> crate::reply::Response {\n            let mut res = self.inner.into_response();\n            self.config.append_common_headers(res.headers_mut());\n            res.headers_mut()\n                .insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, self.origin);\n            res\n        }\n    }\n\n    #[pin_project]\n    #[derive(Debug)]\n    pub struct WrappedFuture<F> {\n        #[pin]\n        inner: F,\n        wrapped: Option<(Arc<Configured>, header::HeaderValue)>,\n    }\n\n    impl<F> Future for WrappedFuture<F>\n    where\n        F: TryFuture,\n        F::Error: CombineRejection<Rejection>,\n    {\n        type Output = Result<\n            One<Either<One<Preflight>, One<Either<One<Wrapped<F::Ok>>, F::Ok>>>>,\n            <F::Error as CombineRejection<Rejection>>::One,\n        >;\n\n        fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {\n            let pin = self.project();\n            match ready!(pin.inner.try_poll(cx)) {\n                Ok(inner) => {\n                    let item = if let Some((config, origin)) = pin.wrapped.take() {\n                        (Either::A((Wrapped {\n                            config,\n                            inner,\n                            origin,\n                        },)),)\n                    } else {\n                        (Either::B(inner),)\n                    };\n                    let item = (Either::B(item),);\n                    Poll::Ready(Ok(item))\n                }\n                Err(err) => Poll::Ready(Err(err.into())),\n            }\n        }\n    }\n\n    pub trait Seconds {\n        fn seconds(self) -> u64;\n    }\n\n    impl Seconds for u32 {\n        fn seconds(self) -> u64 {\n            self.into()\n        }\n    }\n\n    impl Seconds for ::std::time::Duration {\n        fn seconds(self) -> u64 {\n            self.as_secs()\n        }\n    }\n\n    pub trait IntoOrigin {\n        fn into_origin(self) -> Origin;\n    }\n\n    impl<'a> IntoOrigin for &'a str {\n        fn into_origin(self) -> Origin {\n            let mut parts = self.splitn(2, \"://\");\n            let scheme = parts.next().expect(\"missing scheme\");\n            let rest = parts.next().expect(\"missing scheme\");\n\n            Origin::try_from_parts(scheme, rest, None).expect(\"invalid Origin\")\n        }\n    }\n}\n\n// Path: src/filters/ext.rs\n//! Request Extensions\n\nuse std::convert::Infallible;\n\nuse futures_util::future;\n\nuse crate::filter::{filter_fn_one, Filter};\nuse crate::reject::{self, Rejection};\n\n/// Get a previously set extension of the current route.\n///\n/// If the extension doesn't exist, this rejects with a `MissingExtension`.\npub fn get<T: Clone + Send + Sync + 'static>(\n) -> impl Filter<Extract = (T,), Error = Rejection> + Copy {\n    filter_fn_one(|route| {\n        let route = route\n            .extensions()\n            .get::<T>()\n            .cloned()\n            .ok_or_else(|| reject::known(MissingExtension { _p: () }));\n        future::ready(route)\n    })\n}\n\n/// Get a previously set extension of the current route.\n///\n/// If the extension doesn't exist, it yields `None`.\npub fn optional<T: Clone + Send + Sync + 'static>(\n) -> impl Filter<Extract = (Option<T>,), Error = Infallible> + Copy {\n    filter_fn_one(|route| future::ok(route.extensions().get::<T>().cloned()))\n}\n\nunit_error! {\n    /// An error used to reject if `get` cannot find the extension.\n    pub MissingExtension: \"Missing request extension\"\n}\n\n// Path: src/reject.rs\n//! Rejections\n//!\n//! Part of the power of the [`Filter`](../trait.Filter.html) system is being able to\n//! reject a request from a filter chain. This allows for filters to be\n//! combined with `or`, so that if one side of the chain finds that a request\n//! doesn't fulfill its requirements, the other side can try to process\n//! the request.\n//!\n//! Many of the built-in [`filters`](../filters) will automatically reject\n//! the request with an appropriate rejection. However, you can also build\n//! new custom [`Filter`](../trait.Filter.html)s and still want other routes to be\n//! matchable in the case a predicate doesn't hold.\n//!\n//! As a request is processed by a Filter chain, the rejections are accumulated into\n//! a list contained by the [`Rejection`](struct.Rejection.html) type. Rejections from\n//! filters can be handled using [`Filter::recover`](../trait.Filter.html#method.recover).\n//! This is a convenient way to map rejections into a [`Reply`](../reply/trait.Reply.html).\n//!\n//! For a more complete example see the\n//! [Rejection Example](https://github.com/seanmonstar/warp/blob/master/examples/rejections.rs)\n//! from the repository.\n//!\n//! # Example\n//!\n//! ```\n//! use warp::{reply, Reply, Filter, reject, Rejection, http::StatusCode};\n//!\n//! #[derive(Debug)]\n//! struct InvalidParameter;\n//!\n//! impl reject::Reject for InvalidParameter {}\n//!\n//! // Custom rejection handler that maps rejections into responses.\n//! async fn handle_rejection(err: Rejection) -> Result<impl Reply, std::convert::Infallible> {\n//!     if err.is_not_found() {\n//!         Ok(reply::with_status(\"NOT_FOUND\", StatusCode::NOT_FOUND))\n//!     } else if let Some(e) = err.find::<InvalidParameter>() {\n//!         Ok(reply::with_status(\"BAD_REQUEST\", StatusCode::BAD_REQUEST))\n//!     } else {\n//!         eprintln!(\"unhandled rejection: {:?}\", err);\n//!         Ok(reply::with_status(\"INTERNAL_SERVER_ERROR\", StatusCode::INTERNAL_SERVER_ERROR))\n//!     }\n//! }\n//!\n//!\n//! // Filter on `/:id`, but reject with InvalidParameter if the `id` is `0`.\n//! // Recover from this rejection using a custom rejection handler.\n//! let route = warp::path::param()\n//!     .and_then(|id: u32| async move {\n//!         if id == 0 {\n//!             Err(warp::reject::custom(InvalidParameter))\n//!         } else {\n//!             Ok(\"id is valid\")\n//!         }\n//!     })\n//!     .recover(handle_rejection);\n//! ```\n\nuse std::any::Any;\nuse std::convert::Infallible;\nuse std::error::Error as StdError;\nuse std::fmt;\n\nuse http::{\n    header::{HeaderValue, CONTENT_TYPE},\n    StatusCode,\n};\nuse hyper::Body;\n\npub(crate) use self::sealed::{CombineRejection, IsReject};\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn reject() -> Rejection {\n    not_found()\n}\n\n/// Rejects a request with `404 Not Found`.\n#[inline]\npub fn not_found() -> Rejection {\n    Rejection {\n        reason: Reason::NotFound,\n    }\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_query() -> Rejection {\n    known(InvalidQuery { _p: () })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_header(name: &'static str) -> Rejection {\n    known(MissingHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn invalid_header(name: &'static str) -> Rejection {\n    known(InvalidHeader { name })\n}\n\n// 400 Bad Request\n#[inline]\npub(crate) fn missing_cookie(name: &'static str) -> Rejection {\n    known(MissingCookie { name })\n}\n\n// 405 Method Not Allowed\n#[inline]\npub(crate) fn method_not_allowed() -> Rejection {\n    known(MethodNotAllowed { _p: () })\n}\n\n// 411 Length Required\n#[inline]\npub(crate) fn length_required() -> Rejection {\n    known(LengthRequired { _p: () })\n}\n\n// 413 Payload Too Large\n#[inline]\npub(crate) fn payload_too_large() -> Rejection {\n    known(PayloadTooLarge { _p: () })\n}\n\n// 415 Unsupported Media Type\n//\n// Used by the body filters if the request payload content-type doesn't match\n// what can be deserialized.\n#[inline]\npub(crate) fn unsupported_media_type() -> Rejection {\n    known(UnsupportedMediaType { _p: () })\n}\n\n/// Rejects a request with a custom cause.\n///\n/// A [`recover`][] filter should convert this `Rejection` into a `Reply`,\n/// or else this will be returned as a `500 Internal Server Error`.\n///\n/// [`recover`]: ../trait.Filter.html#method.recover\npub fn custom<T: Reject>(err: T) -> Rejection {\n    Rejection::custom(Box::new(err))\n}\n\n/// Protect against re-rejecting a rejection.\n///\n/// ```compile_fail\n/// fn with(r: warp::Rejection) {\n///     let _wat = warp::reject::custom(r);\n/// }\n/// ```\n\nfn __reject_custom_compilefail() {}\n\n/// A marker trait to ensure proper types are used for custom rejections.\n///\n/// Can be converted into Rejection.\n///\n/// # Example\n///\n/// ```\n/// use warp::{Filter, reject::Reject};\n///\n/// #[derive(Debug)]\n/// struct RateLimited;\n///\n/// impl Reject for RateLimited {}\n///\n/// let route = warp::any().and_then(|| async {\n///     Err::<(), _>(warp::reject::custom(RateLimited))\n/// });\n/// ```\n// Require `Sized` for now to prevent passing a `Box<dyn Reject>`, since we\n// would be double-boxing it, and the downcasting wouldn't work as expected.\npub trait Reject: fmt::Debug + Sized + Send + Sync + 'static {}\n\ntrait Cause: fmt::Debug + Send + Sync + 'static {\n    fn as_any(&self) -> &dyn Any;\n}\n\nimpl<T> Cause for T\nwhere\n    T: fmt::Debug + Send + Sync + 'static,\n{\n    fn as_any(&self) -> &dyn Any {\n        self\n    }\n}\n\nimpl dyn Cause {\n    fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        self.as_any().downcast_ref::<T>()\n    }\n}\n\npub(crate) fn known<T: Into<Known>>(err: T) -> Rejection {\n    Rejection::known(err.into())\n}\n\n/// Rejection of a request by a [`Filter`](crate::Filter).\n///\n/// See the [`reject`](module@crate::reject) documentation for more.\npub struct Rejection {\n    reason: Reason,\n}\n\nenum Reason {\n    NotFound,\n    Other(Box<Rejections>),\n}\n\nenum Rejections {\n    Known(Known),\n    Custom(Box<dyn Cause>),\n    Combined(Box<Rejections>, Box<Rejections>),\n}\n\nmacro_rules! enum_known {\n     ($($(#[$attr:meta])* $var:ident($ty:path),)+) => (\n        pub(crate) enum Known {\n            $(\n            $(#[$attr])*\n            $var($ty),\n            )+\n        }\n\n        impl Known {\n            fn inner_as_any(&self) -> &dyn Any {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t,\n                    )+\n                }\n            }\n        }\n\n        impl fmt::Debug for Known {\n            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                match *self {\n                    $(\n                    $(#[$attr])*\n                    Known::$var(ref t) => t.fmt(f),\n                    )+\n                }\n            }\n        }\n\n        impl fmt::Display for Known {\n            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                match *self {\n                    $(\n                    $(#[$attr])*\n...\n", "needle_token_start": 15404, "needle_token_end": 15415, "code_context_ntokens": 16202, "output": ["```rust\nfn __reject_custom_compilefail() {}\n```"]}
{"repo": "serde-rs/serde", "name": "newtype_variant", "language": "rust", "path": "serde/src/de/mod.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To deserialize a variant that contains a single value, ensuring that the data format matches the expected type.\n2. **Input**: A seed that implements the `DeserializeSeed` trait, which provides the necessary logic to deserialize the expected type.\n3. **Output**: Returns the deserialized value wrapped in a `Result`, which can either be the expected type or an error if the data does not match the expected format.\n4. **Procedure**: The method uses the provided seed to attempt deserialization of the single value from the data source. If the data source contains a different type of variant (e.g., a unit or tuple variant), an error is generated indicating an invalid type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn unit_variant(self) -> Result<(), Self::Error>;\n\n    /// Called when deserializing a variant with a single value.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `VariantAccess::newtype_variant` instead.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// fn newtype_variant_seed<T>(self, _seed: T) -> Result<T::Value, Self::Error>\n    /// where\n    ///     T: DeserializeSeed<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"newtype variant\"))\n    /// }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value, Self::Error>\n    where\n        T: DeserializeSeed<'de>;\n\n    /// Called when deserializing a variant with a single value.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `VariantAccess` implementations should not override the default\n    /// behavior.\n    #[inline]\n    \nfn newtype_variant<T>(self) -> Result<T, Self::Error>\n    where\n        T: Deserialize<'de>,\n    {\n        self.newtype_variant_seed(PhantomData)\n    }\n\n    /// Called when deserializing a tuple-like variant.\n    ///\n    /// The `len` is the number of fields expected in the tuple variant.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// fn tuple_variant<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>\n    /// where\n    ///     V: Visitor<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"tuple variant\"))\n    /// }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn tuple_variant<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Called when deserializing a struct-like variant.\n    ///\n    /// The `fields` are the names of the fields of the struct variant.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// fn struct_variant<V>(\n    ///     self,\n    ///     _fields: &'static [&'static str],\n    ///     _visitor: V,\n    /// ) -> Result<V::Value, Self::Error>\n    /// where\n    ///     V: Visitor<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"struct variant\"))\n    /// }\n    /// # }\n    /// ```\n    fn struct_variant<V>(\n        self,\n        fields: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Converts an existing value into a `Deserializer` from which other values can\n/// be deserialized.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed from the resulting `Deserializer`. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example\n///\n/// ```edition2021\n/// use serde::de::{value, Deserialize, IntoDeserializer};\n/// use serde_derive::Deserialize;\n/// use std::str::FromStr;\n///\n/// #[derive(Deserialize)]\n/// enum Setting {\n///     On,\n///     Off,\n/// }\n///\n/// impl FromStr for Setting {\n///     type Err = value::Error;\n///\n///     fn from_str(s: &str) -> Result<Self, Self::Err> {\n///         Self::deserialize(s.into_deserializer())\n///     }\n/// }\n/// ```\npub trait IntoDeserializer<'de, E: Error = value::Error> {\n    /// The type of the deserializer being converted into.\n    type Deserializer: Deserializer<'de, Error = E>;\n\n    /// Convert this value into a deserializer.\n    fn into_deserializer(self) -> Self::Deserializer;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Used in error messages.\n///\n/// - expected `a`\n/// - expected `a` or `b`\n/// - expected one of `a`, `b`, `c`\n///\n/// The slice of names must not be empty.\nstruct OneOf {\n    names: &'static [&'static str],\n}\n\nimpl Display for OneOf {\n    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        match self.names.len() {\n            0 => panic!(), // special case elsewhere\n            1 => write!(formatter, \"`{}`\", self.names[0]),\n            2 => write!(formatter, \"`{}` or `{}`\", self.names[0], self.names[1]),\n            _ => {\n                tri!(formatter.write_str(\"one of \"));\n                for (i, alt) in self.names.iter().enumerate() {\n                    if i > 0 {\n                        tri!(formatter.write_str(\", \"));\n                    }\n                    tri!(write!(formatter, \"`{}`\", alt));\n                }\n                Ok(())\n            }\n        }\n    }\n}\n\nstruct WithDecimalPoint(f64);\n\nimpl Display for WithDecimalPoint {\n    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        struct LookForDecimalPoint<'f, 'a> {\n            formatter: &'f mut fmt::Formatter<'a>,\n            has_decimal_point: bool,\n        }\n\n        impl<'f, 'a> fmt::Write for LookForDecimalPoint<'f, 'a> {\n            fn write_str(&mut self, fragment: &str) -> fmt::Result {\n                self.has_decimal_point |= fragment.contains('.');\n                self.formatter.write_str(fragment)\n            }\n\n            fn write_char(&mut self, ch: char) -> fmt::Result {\n                self.has_decimal_point |= ch == '.';\n                self.formatter.write_char(ch)\n            }\n        }\n\n        let mut writer = LookForDecimalPoint {\n            formatter,\n            has_decimal_point: false,\n        };\n        tri!(write!(writer, \"{}\", self.0));\n        if !writer.has_decimal_point {\n            tri!(formatter.write_str(\".0\"));\n        }\n        Ok(())\n    }\n}\n\n// Path: serde/src/integer128.rs\n// No longer used. Old versions of serde used this macro for supporting targets\n// that did not yet have 128-bit integer support.\n#[macro_export]\n#[doc(hidden)]\nmacro_rules! serde_if_integer128 {\n    ($($tt:tt)*) => {\n        $($tt)*\n    };\n}\n\n// Path: serde/src/macros.rs\n// Super explicit first paragraph because this shows up at the top level and\n// trips up people who are just looking for basic Serialize / Deserialize\n// documentation.\n//\n/// Helper macro when implementing the `Deserializer` part of a new data format\n/// for Serde.\n///\n/// Some [`Deserializer`] implementations for self-describing formats do not\n/// care what hint the [`Visitor`] gives them, they just want to blindly call\n/// the [`Visitor`] method corresponding to the data they can tell is in the\n/// input. This requires repetitive implementations of all the [`Deserializer`]\n/// trait methods.\n///\n/// ```edition2021\n/// # use serde::forward_to_deserialize_any;\n/// # use serde::de::{value, Deserializer, Visitor};\n/// #\n/// # struct MyDeserializer;\n/// #\n/// # impl<'de> Deserializer<'de> for MyDeserializer {\n/// #     type Error = value::Error;\n/// #\n/// #     fn deserialize_any<V>(self, _: V) -> Result<V::Value, Self::Error>\n/// #     where\n/// #         V: Visitor<'de>,\n/// #     {\n/// #         unimplemented!()\n/// #     }\n/// #\n/// #[inline]\n/// fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n/// where\n///     V: Visitor<'de>,\n/// {\n///     self.deserialize_any(visitor)\n/// }\n/// #\n/// #     forward_to_deserialize_any! {\n/// #         i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string\n/// #         bytes byte_buf option unit unit_struct newtype_struct seq tuple\n/// #         tuple_struct map struct enum identifier ignored_any\n/// #     }\n/// # }\n/// ```\n///\n/// The `forward_to_deserialize_any!` macro implements these simple forwarding\n/// methods so that they forward directly to [`Deserializer::deserialize_any`].\n/// You can choose which methods to forward.\n///\n/// ```edition2021\n/// # use serde::forward_to_deserialize_any;\n/// # use serde::de::{value, Deserializer, Visitor};\n/// #\n/// # struct MyDeserializer;\n/// #\n/// impl<'de> Deserializer<'de> for MyDeserializer {\n/// #   type Error = value::Error;\n/// #\n///     fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n///     where\n///         V: Visitor<'de>,\n///     {\n///         /* ... */\n/// #       let _ = visitor;\n/// #       unimplemented!()\n///     }\n///\n///     forward_to_deserialize_any! {\n///         bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string\n///         bytes byte_buf option unit unit_struct newtype_struct seq tuple\n///         tuple_struct map struct enum identifier ignored_any\n///     }\n/// }\n/// ```\n///\n/// The macro assumes the convention that your `Deserializer` lifetime parameter\n/// is called `'de` and that the `Visitor` type parameters on each method are\n/// called `V`. A different type parameter and a different lifetime can be\n/// specified explicitly if necessary.\n///\n/// ```edition2021\n/// # use serde::forward_to_deserialize_any;\n/// # use serde::de::{value, Deserializer, Visitor};\n/// # use std::marker::PhantomData;\n/// #\n/// # struct MyDeserializer<V>(PhantomData<V>);\n/// #\n/// # impl<'q, V> Deserializer<'q> for MyDeserializer<V> {\n/// #     type Error = value::Error;\n/// #\n/// #     fn deserialize_any<W>(self, visitor: W) -> Result<W::Value, Self::Error>\n/// #     where\n/// #         W: Visitor<'q>,\n/// #     {\n/// #         unimplemented!()\n/// #     }\n/// #\n/// forward_to_deserialize_any! {\n///     <W: Visitor<'q>>\n///     bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string\n///     bytes byte_buf option unit unit_struct newtype_struct seq tuple\n///     tuple_struct map struct enum identifier ignored_any\n/// }\n/// # }\n/// ```\n///\n/// [`Deserializer`]: trait.Deserializer.html\n/// [`Visitor`]: de/trait.Visitor.html\n/// [`Deserializer::deserialize_any`]: trait.Deserializer.html#tymethod.deserialize_any\n#[macro_export(local_inner_macros)]\nmacro_rules! forward_to_deserialize_any {\n    (<$visitor:ident: Visitor<$lifetime:tt>> $($func:ident)*) => {\n        $(forward_to_deserialize_any_helper!{$func<$lifetime, $visitor>})*\n    };\n    // This case must be after the previous one.\n    ($($func:ident)*) => {\n        $(forward_to_deserialize_any_helper!{$func<'de, V>})*\n    };\n}\n\n#[doc(hidden)]\n#[macro_export]\nmacro_rules! forward_to_deserialize_any_method {\n    ($func:ident<$l:tt, $v:ident>($($arg:ident : $ty:ty),*)) => {\n        #[inline]\n        fn $func<$v>(self, $($arg: $ty,)* visitor: $v) -> $crate::__private::Result<$v::Value, Self::Error>\n        where\n            $v: $crate::de::Visitor<$l>,\n        {\n            $(\n                let _ = $arg;\n            )*\n            self.deserialize_any(visitor)\n        }\n    };\n}\n\n#[doc(hidden)]\n#[macro_export(local_inner_macros)]\nmacro_rules! forward_to_deserialize_any_helper {\n    (bool<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_bool<$l, $v>()}\n    };\n    (i8<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_i8<$l, $v>()}\n    };\n    (i16<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_i16<$l, $v>()}\n    };\n    (i32<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_i32<$l, $v>()}\n    };\n    (i64<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_i64<$l, $v>()}\n    };\n    (i128<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_i128<$l, $v>()}\n    };\n    (u8<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_u8<$l, $v>()}\n    };\n    (u16<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_u16<$l, $v>()}\n    };\n    (u32<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_u32<$l, $v>()}\n    };\n    (u64<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_u64<$l, $v>()}\n    };\n    (u128<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_u128<$l, $v>()}\n    };\n    (f32<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_f32<$l, $v>()}\n    };\n    (f64<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_f64<$l, $v>()}\n    };\n    (char<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_char<$l, $v>()}\n    };\n    (str<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_str<$l, $v>()}\n    };\n    (string<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_string<$l, $v>()}\n    };\n    (bytes<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_bytes<$l, $v>()}\n    };\n    (byte_buf<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_byte_buf<$l, $v>()}\n    };\n    (option<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_option<$l, $v>()}\n    };\n    (unit<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_unit<$l, $v>()}\n    };\n    (unit_struct<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_unit_struct<$l, $v>(name: &'static str)}\n    };\n    (newtype_struct<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_newtype_struct<$l, $v>(name: &'static str)}\n    };\n    (seq<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_seq<$l, $v>()}\n    };\n    (tuple<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_tuple<$l, $v>(len: usize)}\n    };\n    (tuple_struct<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_tuple_struct<$l, $v>(name: &'static str, len: usize)}\n    };\n    (map<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_map<$l, $v>()}\n    };\n    (struct<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_struct<$l, $v>(name: &'static str, fields: &'static [&'static str])}\n    };\n    (enum<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_enum<$l, $v>(name: &'static str, variants: &'static [&'static str])}\n    };\n    (identifier<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_identifier<$l, $v>()}\n    };\n    (ignored_any<$l:tt, $v:ident>) => {\n        forward_to_deserialize_any_method!{deserialize_ignored_any<$l, $v>()}\n    };\n}\n\n// Path: serde/src/lib.rs\n//! # Serde\n//!\n//! Serde is a framework for ***ser***ializing and ***de***serializing Rust data\n//! structures efficiently and generically.\n//!\n//! The Serde ecosystem consists of data structures that know how to serialize\n//! and deserialize themselves along with data formats that know how to\n//! serialize and deserialize other things. Serde provides the layer by which\n//! these two groups interact with each other, allowing any supported data\n//! structure to be serialized and deserialized using any supported data format.\n//!\n//! See the Serde website <https://serde.rs/> for additional documentation and\n//! usage examples.\n//!\n//! ## Design\n//!\n//! Where many other languages rely on runtime reflection for serializing data,\n//! Serde is instead built on Rust's powerful trait system. A data structure\n//! that knows how to serialize and deserialize itself is one that implements\n//! Serde's `Serialize` and `Deserialize` traits (or uses Serde's derive\n//! attribute to automatically generate implementations at compile time). This\n//! avoids any overhead of reflection or runtime type information. In fact in\n//! many situations the interaction between data structure and data format can\n//! be completely optimized away by the Rust compiler, leaving Serde\n//! serialization to perform the same speed as a handwritten serializer for the\n//! specific selection of data structure and data format.\n//!\n//! ## Data formats\n//!\n//! The following is a partial list of data formats that have been implemented\n//! for Serde by the community.\n//!\n//! - [JSON], the ubiquitous JavaScript Object Notation used by many HTTP APIs.\n//! - [Postcard], a no\\_std and embedded-systems friendly compact binary format.\n//! - [CBOR], a Concise Binary Object Representation designed for small message\n//!   size without the need for version negotiation.\n//! - [YAML], a self-proclaimed human-friendly configuration language that ain't\n//!   markup language.\n//! - [MessagePack], an efficient binary format that resembles a compact JSON.\n//! - [TOML], a minimal configuration format used by [Cargo].\n//! - [Pickle], a format common in the Python world.\n//! - [RON], a Rusty Object Notation.\n//! - [BSON], the data storage and network transfer format used by MongoDB.\n//! - [Avro], a binary format used within Apache Hadoop, with support for schema\n//!   definition.\n//! - [JSON5], a superset of JSON including some productions from ES5.\n//! - [URL] query strings, in the x-www-form-urlencoded format.\n//! - [Starlark], the format used for describing build targets by the Bazel and\n//!   Buck build systems. *(serialization only)*\n//! - [Envy], a way to deserialize environment variables into Rust structs.\n//!   *(deserialization only)*\n//! - [Envy Store], a way to deserialize [AWS Parameter Store] parameters into\n//!   Rust structs. *(deserialization only)*\n//! - [S-expressions], the textual representation of code and data used by the\n//!   Lisp language family.\n//! - [D-Bus]'s binary wire format.\n//! - [FlexBuffers], the schemaless cousin of Google's FlatBuffers zero-copy\n//!   serialization format.\n//! - [Bencode], a simple binary format used in the BitTorrent protocol.\n//! - [Token streams], for processing Rust procedural macro input.\n//!   *(deserialization only)*\n//! - [DynamoDB Items], the format used by [rusoto_dynamodb] to transfer data to\n//!   and from DynamoDB.\n//! - [Hjson], a syntax extension to JSON designed around human reading and\n//!   editing. *(deserialization only)*\n//! - [CSV], Comma-separated values is a tabular text file format.\n//!\n//! [JSON]: https://github.com/serde-rs/json\n//! [Postcard]: https://github.com/jamesmunns/postcard\n//! [CBOR]: https://github.com/enarx/ciborium\n//! [YAML]: https://github.com/dtolnay/serde-yaml\n//! [MessagePack]: https://github.com/3Hren/msgpack-rust\n//! [TOML]: https://docs.rs/toml\n//! [Pickle]: https://github.com/birkenfeld/serde-pickle\n//! [RON]: https://github.com/ron-rs/ron\n//! [BSON]: https://github.com/mongodb/bson-rust\n//! [Avro]: https://docs.rs/apache-avro\n//! [JSON5]: https://github.com/callum-oakley/json5-rs\n//! [URL]: https://docs.rs/serde_qs\n//! [Starlark]: https://github.com/dtolnay/serde-starlark\n//! [Envy]: https://github.com/softprops/envy\n//! [Envy Store]: https://github.com/softprops/envy-store\n//! [Cargo]: https://doc.rust-lang.org/cargo/reference/manifest.html\n//! [AWS Parameter Store]: https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html\n//! [S-expressions]: https://github.com/rotty/lexpr-rs\n//! [D-Bus]: https://docs.rs/zvariant\n//! [FlexBuffers]: https://github.com/google/flatbuffers/tree/master/rust/flexbuffers\n//! [Bencode]: https://github.com/P3KI/bendy\n//! [Token streams]: https://github.com/oxidecomputer/serde_tokenstream\n//! [DynamoDB Items]: https://docs.rs/serde_dynamo\n//! [rusoto_dynamodb]: https://docs.rs/rusoto_dynamodb\n//! [Hjson]: https://github.com/Canop/deser-hjson\n//! [CSV]: https://docs.rs/csv\n\n////////////////////////////////////////////////////////////////////////////////\n\n// Serde types in rustdoc of other crates get linked to here.\n#![doc(html_root_url = \"https://docs.rs/serde/1.0.197\")]\n// Support using Serde without the standard library!\n#![cfg_attr(not(feature = \"std\"), no_std)]\n// Show which crate feature enables conditionally compiled APIs in documentation.\n#![cfg_attr(doc_cfg, feature(doc_cfg))]\n// Unstable functionality only if the user asks for it. For tracking and\n// discussion of these features please refer to this issue:\n//\n//    https://github.com/serde-rs/serde/issues/812\n#![cfg_attr(feature = \"unstable\", feature(error_in_core, never_type))]\n#![allow(unknown_lints, bare_trait_objects, deprecated)]\n// Ignored clippy and clippy_pedantic lints\n#![allow(\n    // clippy bug: https://github.com/rust-lang/rust-clippy/issues/5704\n    clippy::unnested_or_patterns,\n    // clippy bug: https://github.com/rust-lang/rust-clippy/issues/7768\n    clippy::semicolon_if_nothing_returned,\n    // not available in our oldest supported compiler\n    clippy::empty_enum,\n    clippy::type_repetition_in_bounds, // https://github.com/rust-lang/rust-clippy/issues/8772\n    // integer and float ser/de requires these sorts of casts\n    clippy::cast_possible_truncation,\n    clippy::cast_possible_wrap,\n    clippy::cast_sign_loss,\n    // things are often more readable this way\n    clippy::cast_lossless,\n    clippy::module_name_repetitions,\n    clippy::single_match_else,\n    clippy::type_complexity,\n    clippy::use_self,\n    clippy::zero_prefixed_literal,\n    // correctly used\n    clippy::derive_partial_eq_without_eq,\n    clippy::enum_glob_use,\n    clippy::explicit_auto_deref,\n    clippy::incompatible_msrv,\n    clippy::let_underscore_untyped,\n    clippy::map_err_ignore,\n    clippy::new_without_default,\n    clippy::result_unit_err,\n    clippy::wildcard_imports,\n    // not practical\n    clippy::needless_pass_by_value,\n    clippy::similar_names,\n    clippy::too_many_lines,\n    // preference\n    clippy::doc_markdown,\n    clippy::unseparated_literal_suffix,\n    // false positive\n    clippy::needless_doctest_main,\n    // noisy\n    clippy::missing_errors_doc,\n    clippy::must_use_candidate,\n)]\n// Restrictions\n#![deny(clippy::question_mark_used)]\n// Rustc lints.\n#![deny(missing_docs, unused_imports)]\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(feature = \"alloc\")]\nextern crate alloc;\n\n/// A facade around all the types we need from the `std`, `core`, and `alloc`\n/// crates. This avoids elaborate import wrangling having to happen in every\n/// module.\nmod lib {\n    mod core {\n        #[cfg(not(feature = \"std\"))]\n        pub use core::*;\n        #[cfg(feature = \"std\")]\n        pub use std::*;\n    }\n\n    pub use self::core::{f32, f64};\n    pub use self::core::{i16, i32, i64, i8, isize};\n    pub use self::core::{iter, num, ptr, str};\n    pub use self::core::{u16, u32, u64, u8, usize};\n\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    pub use self::core::{cmp, mem, slice};\n\n    pub use self::core::cell::{Cell, RefCell};\n    pub use self::core::clone;\n    pub use self::core::cmp::Reverse;\n    pub use self::core::convert;\n    pub use self::core::default;\n    pub use self::core::fmt::{self, Debug, Display, Write as FmtWrite};\n    pub use self::core::marker::{self, PhantomData};\n    pub use self::core::num::Wrapping;\n    pub use self::core::ops::{Bound, Range, RangeFrom, RangeInclusive, RangeTo};\n    pub use self::core::option;\n    pub use self::core::result;\n    pub use self::core::time::Duration;\n\n    #[cfg(all(feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::borrow::{Cow, ToOwned};\n    #[cfg(feature = \"std\")]\n    pub use std::borrow::{Cow, ToOwned};\n\n    #[cfg(all(feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::string::{String, ToString};\n    #[cfg(feature = \"std\")]\n    pub use std::string::{String, ToString};\n\n    #[cfg(all(feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::vec::Vec;\n    #[cfg(feature = \"std\")]\n    pub use std::vec::Vec;\n\n    #[cfg(all(feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::boxed::Box;\n    #[cfg(feature = \"std\")]\n    pub use std::boxed::Box;\n\n    #[cfg(all(feature = \"rc\", feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::rc::{Rc, Weak as RcWeak};\n    #[cfg(all(feature = \"rc\", feature = \"std\"))]\n    pub use std::rc::{Rc, Weak as RcWeak};\n\n    #[cfg(all(feature = \"rc\", feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::sync::{Arc, Weak as ArcWeak};\n    #[cfg(all(feature = \"rc\", feature = \"std\"))]\n    pub use std::sync::{Arc, Weak as ArcWeak};\n\n    #[cfg(all(feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::collections::{BTreeMap, BTreeSet, BinaryHeap, LinkedList, VecDeque};\n    #[cfg(feature = \"std\")]\n    pub use std::collections::{BTreeMap, BTreeSet, BinaryHeap, LinkedList, VecDeque};\n\n    #[cfg(all(not(no_core_cstr), not(feature = \"std\")))]\n    pub use self::core::ffi::CStr;\n    #[cfg(feature = \"std\")]\n    pub use std::ffi::CStr;\n\n    #[cfg(all(not(no_core_cstr), feature = \"alloc\", not(feature = \"std\")))]\n    pub use alloc::ffi::CString;\n    #[cfg(feature = \"std\")]\n    pub use std::ffi::CString;\n\n    #[cfg(feature = \"std\")]\n    pub use std::{error, net};\n\n    #[cfg(feature = \"std\")]\n    pub use std::collections::{HashMap, HashSet};\n    #[cfg(feature = \"std\")]\n    pub use std::ffi::{OsStr, OsString};\n    #[cfg(feature = \"std\")]\n    pub use std::hash::{BuildHasher, Hash};\n    #[cfg(feature = \"std\")]\n    pub use std::io::Write;\n    #[cfg(feature = \"std\")]\n    pub use std::path::{Path, PathBuf};\n    #[cfg(feature = \"std\")]\n    pub use std::sync::{Mutex, RwLock};\n    #[cfg(feature = \"std\")]\n    pub use std::time::{SystemTime, UNIX_EPOCH};\n\n    #[cfg(all(feature = \"std\", no_target_has_atomic, not(no_std_atomic)))]\n    pub use std::sync::atomic::{\n        AtomicBool, AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicU16, AtomicU32, AtomicU8,\n        AtomicUsize, Ordering,\n    };\n    #[cfg(all(feature = \"std\", no_target_has_atomic, not(no_std_atomic64)))]\n    pub use std::sync::atomic::{AtomicI64, AtomicU64};\n\n    #[cfg(all(feature = \"std\", not(no_target_has_atomic)))]\n    pub use std::sync::atomic::Ordering;\n    #[cfg(all(feature = \"std\", not(no_target_has_atomic), target_has_atomic = \"8\"))]\n    pub use std::sync::atomic::{AtomicBool, AtomicI8, AtomicU8};\n    #[cfg(all(feature = \"std\", not(no_target_has_atomic), target_has_atomic = \"16\"))]\n    pub use std::sync::atomic::{AtomicI16, AtomicU16};\n    #[cfg(all(feature = \"std\", not(no_target_has_atomic), target_has_atomic = \"32\"))]\n    pub use std::sync::atomic::{AtomicI32, AtomicU32};\n    #[cfg(all(feature = \"std\", not(no_target_has_atomic), target_has_atomic = \"64\"))]\n    pub use std::sync::atomic::{AtomicI64, AtomicU64};\n    #[cfg(all(feature = \"std\", not(no_target_has_atomic), target_has_atomic = \"ptr\"))]\n    pub use std::sync::atomic::{AtomicIsize, AtomicUsize};\n}\n\n// None of this crate's error handling needs the `From::from` error conversion\n// performed implicitly by the `?` operator or the standard library's `try!`\n// macro. This simplified macro gives a 5.5% improvement in compile time\n// compared to standard `try!`, and 9% improvement compared to `?`.\nmacro_rules! tri {\n    ($expr:expr) => {\n        match $expr {\n            Ok(val) => val,\n            Err(err) => return Err(err),\n        }\n    };\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[macro_use]\nmod macros;\n\n#[macro_use]\nmod integer128;\n\npub mod de;\npub mod ser;\n\n#[doc(inline)]\npub use crate::de::{Deserialize, Deserializer};\n#[doc(inline)]\npub use crate::ser::{Serialize, Serializer};\n\n// Used by generated code and doc tests. Not public API.\n#[doc(hidden)]\n#[path = \"private/mod.rs\"]\npub mod __private;\n\n#[path = \"de/seed.rs\"]\nmod seed;\n\n#[cfg(not(any(feature = \"std\", feature = \"unstable\")))]\nmod std_error;\n\n// Re-export #[derive(Serialize, Deserialize)].\n//\n// The reason re-exporting is not enabled by default is that disabling it would\n// be annoying for crates that provide handwritten impls or data formats. They\n// would need to disable default features and then explicitly re-enable std.\n#[cfg(feature = \"serde_derive\")]\nextern crate serde_derive;\n\n/// Derive macro available if serde is built with `features = [\"derive\"]`.\n#[cfg(feature = \"serde_derive\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"derive\")))]\npub use serde_derive::{Deserialize, Serialize};\n\n#[cfg(all(not(no_serde_derive), any(feature = \"std\", feature = \"alloc\")))]\nmod actually_private {\n    pub struct T;\n}\n\n", "needle_token_start": 783, "needle_token_end": 836, "code_context_ntokens": 10702, "output": ["```rust\nfn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value, Self::Error>\nwhere\n    T: DeserializeSeed<'de>,\n{\n    // What the data actually contained; suppose it is a unit variant.\n    let unexp = Unexpected::UnitVariant;\n    Err(de::Error::invalid_type(unexp, &\"newtype variant\"))\n}\n```"]}
{"repo": "serde-rs/serde", "name": "from_bounds", "language": "rust", "path": "serde/src/de/size_hint.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to estimate the size of an iterator.\n2. **Input**: It takes a reference to an iterator.\n3. **Output**: It returns an optional value containing the estimated size of the iterator, or `None` if the size cannot be determined.\n4. **Procedure**: The function retrieves the size hint of the iterator and then processes this hint through another helper function to possibly refine or determine the size estimate.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: serde/src/de/size_hint.rs\nuse crate::lib::*;\n\n\npub fn from_bounds<I>(iter: &I) -> Option<usize>\nwhere\n    I: Iterator,\n{\n    helper(iter.size_hint())\n}\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\npub fn cautious<Element>(hint: Option<usize>) -> usize {\n    const MAX_PREALLOC_BYTES: usize = 1024 * 1024;\n\n    if mem::size_of::<Element>() == 0 {\n        0\n    } else {\n        cmp::min(\n            hint.unwrap_or(0),\n            MAX_PREALLOC_BYTES / mem::size_of::<Element>(),\n        )\n    }\n}\n\nfn helper(bounds: (usize, Option<usize>)) -> Option<usize> {\n    match bounds {\n        (lower, Some(upper)) if lower == upper => Some(upper),\n        _ => None,\n    }\n}\n\n// Path: serde/src/de/impls.rs\nuse crate::lib::*;\n\nuse crate::de::{\n    Deserialize, Deserializer, EnumAccess, Error, MapAccess, SeqAccess, Unexpected, VariantAccess,\n    Visitor,\n};\n\nuse crate::seed::InPlaceSeed;\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nuse crate::de::size_hint;\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct UnitVisitor;\n\nimpl<'de> Visitor<'de> for UnitVisitor {\n    type Value = ();\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"unit\")\n    }\n\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(())\n    }\n}\n\nimpl<'de> Deserialize<'de> for () {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_unit(UnitVisitor)\n    }\n}\n\n#[cfg(feature = \"unstable\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"unstable\")))]\nimpl<'de> Deserialize<'de> for ! {\n    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        Err(Error::custom(\"cannot deserialize `!`\"))\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct BoolVisitor;\n\nimpl<'de> Visitor<'de> for BoolVisitor {\n    type Value = bool;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a boolean\")\n    }\n\n    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n}\n\nimpl<'de> Deserialize<'de> for bool {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_bool(BoolVisitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! impl_deserialize_num {\n    ($primitive:ident, $nonzero:ident $(cfg($($cfg:tt)*))*, $deserialize:ident $($method:ident!($($val:ident : $visit:ident)*);)*) => {\n        impl_deserialize_num!($primitive, $deserialize $($method!($($val : $visit)*);)*);\n\n        $(#[cfg($($cfg)*)])*\n        impl<'de> Deserialize<'de> for num::$nonzero {\n            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n            where\n                D: Deserializer<'de>,\n            {\n                struct NonZeroVisitor;\n\n                impl<'de> Visitor<'de> for NonZeroVisitor {\n                    type Value = num::$nonzero;\n\n                    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                        formatter.write_str(concat!(\"a nonzero \", stringify!($primitive)))\n                    }\n\n                    $($($method!(nonzero $primitive $val : $visit);)*)*\n                }\n\n                deserializer.$deserialize(NonZeroVisitor)\n            }\n        }\n    };\n\n    ($primitive:ident, $deserialize:ident $($method:ident!($($val:ident : $visit:ident)*);)*) => {\n        impl<'de> Deserialize<'de> for $primitive {\n            #[inline]\n            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n            where\n                D: Deserializer<'de>,\n            {\n                struct PrimitiveVisitor;\n\n                impl<'de> Visitor<'de> for PrimitiveVisitor {\n                    type Value = $primitive;\n\n                    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                        formatter.write_str(stringify!($primitive))\n                    }\n\n                    $($($method!($val : $visit);)*)*\n                }\n\n                deserializer.$deserialize(PrimitiveVisitor)\n            }\n        }\n    };\n}\n\nmacro_rules! num_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            Ok(v)\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if let Some(nonzero) = Self::Value::new(v) {\n                Ok(nonzero)\n            } else {\n                Err(Error::invalid_value(Unexpected::Unsigned(0), &self))\n            }\n        }\n    };\n}\n\nmacro_rules! num_as_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            Ok(v as Self::Value)\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                Ok(nonzero)\n            } else {\n                Err(Error::invalid_value(Unexpected::Unsigned(0), &self))\n            }\n        }\n    };\n}\n\nmacro_rules! num_as_copysign_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            #[cfg(any(no_float_copysign, not(feature = \"std\")))]\n            {\n                Ok(v as Self::Value)\n            }\n\n            #[cfg(all(not(no_float_copysign), feature = \"std\"))]\n            {\n                // Preserve sign of NaN. The `as` produces a nondeterministic sign.\n                let sign = if v.is_sign_positive() { 1.0 } else { -1.0 };\n                Ok((v as Self::Value).copysign(sign))\n            }\n        }\n    };\n}\n\nmacro_rules! int_to_int {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if Self::Value::min_value() as i64 <= v as i64\n                && v as i64 <= Self::Value::max_value() as i64\n            {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if $primitive::min_value() as i64 <= v as i64\n                && v as i64 <= $primitive::max_value() as i64\n            {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    return Ok(nonzero);\n                }\n            }\n            Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n        }\n    };\n}\n\nmacro_rules! int_to_uint {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if 0 <= v && v as u64 <= Self::Value::max_value() as u64 {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if 0 < v && v as u64 <= $primitive::max_value() as u64 {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    return Ok(nonzero);\n                }\n            }\n            Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n        }\n    };\n}\n\nmacro_rules! uint_to_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as u64 <= Self::Value::max_value() as u64 {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(Unexpected::Unsigned(v as u64), &self))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as u64 <= $primitive::max_value() as u64 {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    return Ok(nonzero);\n                }\n            }\n            Err(Error::invalid_value(Unexpected::Unsigned(v as u64), &self))\n        }\n    };\n}\n\nimpl_deserialize_num! {\n    i8, NonZeroI8 cfg(not(no_num_nonzero_signed)), deserialize_i8\n    num_self!(i8:visit_i8);\n    int_to_int!(i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    i16, NonZeroI16 cfg(not(no_num_nonzero_signed)), deserialize_i16\n    num_self!(i16:visit_i16);\n    num_as_self!(i8:visit_i8);\n    int_to_int!(i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    i32, NonZeroI32 cfg(not(no_num_nonzero_signed)), deserialize_i32\n    num_self!(i32:visit_i32);\n    num_as_self!(i8:visit_i8 i16:visit_i16);\n    int_to_int!(i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    i64, NonZeroI64 cfg(not(no_num_nonzero_signed)), deserialize_i64\n    num_self!(i64:visit_i64);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    isize, NonZeroIsize cfg(not(no_num_nonzero_signed)), deserialize_i64\n    num_as_self!(i8:visit_i8 i16:visit_i16);\n    int_to_int!(i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u8, NonZeroU8, deserialize_u8\n    num_self!(u8:visit_u8);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u16, NonZeroU16, deserialize_u16\n    num_self!(u16:visit_u16);\n    num_as_self!(u8:visit_u8);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u32, NonZeroU32, deserialize_u32\n    num_self!(u32:visit_u32);\n    num_as_self!(u8:visit_u8 u16:visit_u16);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u64, NonZeroU64, deserialize_u64\n    num_self!(u64:visit_u64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n}\n\nimpl_deserialize_num! {\n    usize, NonZeroUsize, deserialize_u64\n    num_as_self!(u8:visit_u8 u16:visit_u16);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    f32, deserialize_f32\n    num_self!(f32:visit_f32);\n    num_as_copysign_self!(f64:visit_f64);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    f64, deserialize_f64\n    num_self!(f64:visit_f64);\n    num_as_copysign_self!(f32:visit_f32);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nmacro_rules! num_128 {\n    ($ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as i128 >= Self::Value::min_value() as i128\n                && v as u128 <= Self::Value::max_value() as u128\n            {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(\n                    Unexpected::Other(stringify!($ty)),\n                    &self,\n                ))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as i128 >= $primitive::min_value() as i128\n                && v as u128 <= $primitive::max_value() as u128\n            {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    Ok(nonzero)\n                } else {\n                    Err(Error::invalid_value(Unexpected::Unsigned(0), &self))\n                }\n            } else {\n                Err(Error::invalid_value(\n                    Unexpected::Other(stringify!($ty)),\n                    &self,\n                ))\n            }\n        }\n    };\n}\n\nimpl_deserialize_num! {\n    i128, NonZeroI128 cfg(not(no_num_nonzero_signed)), deserialize_i128\n    num_self!(i128:visit_i128);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n    num_128!(u128:visit_u128);\n}\n\nimpl_deserialize_num! {\n    u128, NonZeroU128, deserialize_u128\n    num_self!(u128:visit_u128);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_128!(i128:visit_i128);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct CharVisitor;\n\nimpl<'de> Visitor<'de> for CharVisitor {\n    type Value = char;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a character\")\n    }\n\n    #[inline]\n    fn visit_char<E>(self, v: char) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n\n    #[inline]\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut iter = v.chars();\n        match (iter.next(), iter.next()) {\n            (Some(c), None) => Ok(c),\n            _ => Err(Error::invalid_value(Unexpected::Str(v), &self)),\n        }\n    }\n}\n\nimpl<'de> Deserialize<'de> for char {\n    #[inline]\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_char(CharVisitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nstruct StringVisitor;\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nstruct StringInPlaceVisitor<'a>(&'a mut String);\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nimpl<'de> Visitor<'de> for StringVisitor {\n    type Value = String;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a string\")\n    }\n\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v.to_owned())\n    }\n\n    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n\n    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        match str::from_utf8(v) {\n            Ok(s) => Ok(s.to_owned()),\n            Err(_) => Err(Error::invalid_value(Unexpected::Bytes(v), &self)),\n        }\n    }\n\n    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        match String::from_utf8(v) {\n            Ok(s) => Ok(s),\n            Err(e) => Err(Error::invalid_value(\n                Unexpected::Bytes(&e.into_bytes()),\n                &self,\n            )),\n        }\n    }\n}\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nimpl<'a, 'de> Visitor<'de> for StringInPlaceVisitor<'a> {\n    type Value = ();\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a string\")\n    }\n\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.0.clear();\n        self.0.push_str(v);\n        Ok(())\n    }\n\n    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        *self.0 = v;\n        Ok(())\n    }\n\n    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        match str::from_utf8(v) {\n            Ok(s) => {\n                self.0.clear();\n                self.0.push_str(s);\n                Ok(())\n            }\n            Err(_) => Err(Error::invalid_value(Unexpected::Bytes(v), &self)),\n        }\n    }\n\n    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        match String::from_utf8(v) {\n            Ok(s) => {\n                *self.0 = s;\n                Ok(())\n            }\n            Err(e) => Err(Error::invalid_value(\n                Unexpected::Bytes(&e.into_bytes()),\n                &self,\n            )),\n        }\n    }\n}\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl<'de> Deserialize<'de> for String {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_string(StringVisitor)\n    }\n\n    fn deserialize_in_place<D>(deserializer: D, place: &mut Self) -> Result<(), D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_string(StringInPlaceVisitor(place))\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct StrVisitor;\n\nimpl<'a> Visitor<'a> for StrVisitor {\n    type Value = &'a str;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a borrowed string\")\n    }\n\n    fn visit_borrowed_str<E>(self, v: &'a str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v) // so easy\n    }\n\n    fn visit_borrowed_bytes<E>(self, v: &'a [u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        str::from_utf8(v).map_err(|_| Error::invalid_value(Unexpected::Bytes(v), &self))\n    }\n}\n\nimpl<'de: 'a, 'a> Deserialize<'de> for &'a str {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_str(StrVisitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct BytesVisitor;\n\nimpl<'a> Visitor<'a> for BytesVisitor {\n    type Value = &'a [u8];\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a borrowed byte array\")\n    }\n\n    fn visit_borrowed_bytes<E>(self, v: &'a [u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n\n    fn visit_borrowed_str<E>(self, v: &'a str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v.as_bytes())\n    }\n}\n\nimpl<'de: 'a, 'a> Deserialize<'de> for &'a [u8] {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_bytes(BytesVisitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(any(feature = \"std\", all(not(no_core_cstr), feature = \"alloc\")))]\nstruct CStringVisitor;\n\n#[cfg(any(feature = \"std\", all(not(no_core_cstr), feature = \"alloc\")))]\nimpl<'de> Visitor<'de> for CStringVisitor {\n    type Value = CString;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"byte array\")\n    }\n\n    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n    where\n        A: SeqAccess<'de>,\n    {\n        let capacity = size_hint::cautious::<u8>(seq.size_hint());\n        let mut values = Vec::<u8>::with_capacity(capacity);\n\n        while let Some(value) = tri!(seq.next_element()) {\n            values.push(value);\n        }\n\n        CString::new(values).map_err(Error::custom)\n    }\n\n    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        CString::new(v).map_err(Error::custom)\n    }\n\n    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        CString::new(v).map_err(Error::custom)\n    }\n\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        CString::new(v).map_err(Error::custom)\n    }\n\n    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        CString::new(v).map_err(Error::custom)\n    }\n}\n\n#[cfg(any(feature = \"std\", all(not(no_core_cstr), feature = \"alloc\")))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl<'de> Deserialize<'de> for CString {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_byte_buf(CStringVisitor)\n    }\n}\n\nmacro_rules! forwarded_impl {\n    (\n        $(#[$attr:meta])*\n        ($($id:ident),*), $ty:ty, $func:expr\n    ) => {\n        $(#[$attr])*\n        impl<'de $(, $id : Deserialize<'de>,)*> Deserialize<'de> for $ty {\n            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n            where\n                D: Deserializer<'de>,\n            {\n                Deserialize::deserialize(deserializer).map($func)\n            }\n        }\n    }\n}\n\nforwarded_impl! {\n    #[cfg(any(feature = \"std\", all(not(no_core_cstr), feature = \"alloc\")))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    (), Box<CStr>, CString::into_boxed_c_str\n}\n\nforwarded_impl! {\n    (T), Reverse<T>, Reverse\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct OptionVisitor<T> {\n    marker: PhantomData<T>,\n}\n\nimpl<'de, T> Visitor<'de> for OptionVisitor<T>\nwhere\n    T: Deserialize<'de>,\n{\n    type Value = Option<T>;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"option\")\n    }\n\n    #[inline]\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(None)\n    }\n\n    #[inline]\n    fn visit_none<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(None)\n    }\n\n    #[inline]\n    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        T::deserialize(deserializer).map(Some)\n    }\n\n    fn __private_visit_untagged_option<D>(self, deserializer: D) -> Result<Self::Value, ()>\n    where\n        D: Deserializer<'de>,\n    {\n        Ok(T::deserialize(deserializer).ok())\n    }\n}\n\nimpl<'de, T> Deserialize<'de> for Option<T>\nwhere\n    T: Deserialize<'de>,\n{\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_option(OptionVisitor {\n            marker: PhantomData,\n        })\n    }\n\n    // The Some variant's repr is opaque, so we can't play cute tricks with its\n    // tag to have deserialize_in_place build the content in place unconditionally.\n    //\n    // FIXME: investigate whether branching on the old value being Some to\n    // deserialize_in_place the value is profitable (probably data-dependent?)\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct PhantomDataVisitor<T: ?Sized> {\n    marker: PhantomData<T>,\n}\n\nimpl<'de, T> Visitor<'de> for PhantomDataVisitor<T>\nwhere\n    T: ?Sized,\n{\n    type Value = PhantomData<T>;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"unit\")\n    }\n\n    #[inline]\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(PhantomData)\n    }\n}\n\nimpl<'de, T> Deserialize<'de> for PhantomData<T>\nwhere\n    T: ?Sized,\n{\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let visitor = PhantomDataVisitor {\n            marker: PhantomData,\n        };\n        deserializer.deserialize_unit_struct(\"PhantomData\", visitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound1:ident $(+ $bound2:ident)*)*>,\n        $access:ident,\n        $clear:expr,\n        $with_capacity:expr,\n        $reserve:expr,\n        $insert:expr\n    ) => {\n        $(#[$attr])*\n        impl<'de, T $(, $typaram)*> Deserialize<'de> for $ty<T $(, $typaram)*>\n        where\n            T: Deserialize<'de> $(+ $tbound1 $(+ $tbound2)*)*,\n            $($typaram: $bound1 $(+ $bound2)*,)*\n        {\n            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n            where\n                D: Deserializer<'de>,\n            {\n                struct SeqVisitor<T $(, $typaram)*> {\n                    marker: PhantomData<$ty<T $(, $typaram)*>>,\n                }\n\n                impl<'de, T $(, $typaram)*> Visitor<'de> for SeqVisitor<T $(, $typaram)*>\n                where\n                    T: Deserialize<'de> $(+ $tbound1 $(+ $tbound2)*)*,\n                    $($typaram: $bound1 $(+ $bound2)*,)*\n                {\n                    type Value = $ty<T $(, $typaram)*>;\n\n                    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                        formatter.write_str(\"a sequence\")\n                    }\n\n                    #[inline]\n                    fn visit_seq<A>(self, mut $access: A) -> Result<Self::Value, A::Error>\n                    where\n                        A: SeqAccess<'de>,\n                    {\n                        let mut values = $with_capacity;\n\n                        while let Some(value) = tri!($access.next_element()) {\n                            $insert(&mut values, value);\n                        }\n\n                        Ok(values)\n                    }\n                }\n\n                let visitor = SeqVisitor { marker: PhantomData };\n                deserializer.deserialize_seq(visitor)\n            }\n\n            fn deserialize_in_place<D>(deserializer: D, place: &mut Self) -> Result<(), D::Error>\n            where\n                D: Deserializer<'de>,\n            {\n                struct SeqInPlaceVisitor<'a, T: 'a $(, $typaram: 'a)*>(&'a mut $ty<T $(, $typaram)*>);\n\n                impl<'a, 'de, T $(, $typaram)*> Visitor<'de> for SeqInPlaceVisitor<'a, T $(, $typaram)*>\n                where\n                    T: Deserialize<'de> $(+ $tbound1 $(+ $tbound2)*)*,\n                    $($typaram: $bound1 $(+ $bound2)*,)*\n                {\n                    type Value = ();\n\n                    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                        formatter.write_str(\"a sequence\")\n                    }\n\n                    #[inline]\n                    fn visit_seq<A>(mut self, mut $access: A) -> Result<Self::Value, A::Error>\n                    where\n                        A: SeqAccess<'de>,\n                    {\n                        $clear(&mut self.0);\n                        $reserve(&mut self.0, size_hint::cautious::<T>($access.size_hint()));\n\n                        // FIXME: try to overwrite old values here? (Vec, VecDeque, LinkedList)\n                        while let Some(value) = tri!($access.next_element()) {\n                            $insert(&mut self.0, value);\n                        }\n\n                        Ok(())\n                    }\n                }\n\n                deserializer.deserialize_seq(SeqInPlaceVisitor(place))\n            }\n        }\n    }\n}\n\n// Dummy impl of reserve\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nfn nop_reserve<T>(_seq: T, _n: usize) {}\n\nseq_impl!(\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BinaryHeap<T: Ord>,\n    seq,\n    BinaryHeap::clear,\n    BinaryHeap::with_capacity(size_hint::cautious::<T>(seq.size_hint())),\n    BinaryHeap::reserve,\n    BinaryHeap::push\n);\n\nseq_impl!(\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeSet<T: Eq + Ord>,\n    seq,\n    BTreeSet::clear,\n    BTreeSet::new(),\n    nop_reserve,\n    BTreeSet::insert\n);\n\nseq_impl!(\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    LinkedList<T>,\n    seq,\n    LinkedList::clear,\n    LinkedList::new(),\n    nop_reserve,\n    LinkedList::push_back\n);\n\nseq_impl!(\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashSet<T: Eq + Hash, S: BuildHasher + Default>,\n    seq,\n    HashSet::clear,\n    HashSet::with_capacity_and_hasher(size_hint::cautious::<T>(seq.size_hint()), S::default()),\n    HashSet::reserve,\n    HashSet::insert\n);\n\nseq_impl!(\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    VecDeque<T>,\n    seq,\n    VecDeque::clear,\n    VecDeque::with_capacity(size_hint::cautious::<T>(seq.size_hint())),\n    VecDeque::reserve,\n    VecDeque::push_back\n);\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl<'de, T> Deserialize<'de> for Vec<T>\nwhere\n    T: Deserialize<'de>,\n{\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct VecVisitor<T> {\n            marker: PhantomData<T>,\n        }\n\n        impl<'de, T> Visitor<'de> for VecVisitor<T>\n        where\n            T: Deserialize<'de>,\n        {\n            type Value = Vec<T>;\n\n            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                formatter.write_str(\"a sequence\")\n            }\n\n            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n            where\n                A: SeqAccess<'de>,\n            {\n                let capacity = size_hint::cautious::<T>(seq.size_hint());\n                let mut values = Vec::<T>::with_capacity(capacity);\n\n                while let Some(value) = tri!(seq.next_element()) {\n                    values.push(value);\n                }\n\n                Ok(values)\n            }\n        }\n\n        let visitor = VecVisitor {\n            marker: PhantomData,\n        };\n        deserializer.deserialize_seq(visitor)\n    }\n\n    fn deserialize_in_place<D>(deserializer: D, place: &mut Self) -> Result<(), D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct VecInPlaceVisitor<'a, T: 'a>(&'a mut Vec<T>);\n\n        impl<'a, 'de, T> Visitor<'de> for VecInPlaceVisitor<'a, T>\n        where\n            T: Deserialize<'de>,\n        {\n            type Value = ();\n\n            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                formatter.write_str(\"a sequence\")\n            }\n\n            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n            where\n                A: SeqAccess<'de>,\n            {\n                let hint = size_hint::cautious::<T>(seq.size_hint());\n                if let Some(additional) = hint.checked_sub(self.0.len()) {\n                    self.0.reserve(additional);\n                }\n\n                for i in 0..self.0.len() {\n                    let next = {\n                        let next_place = InPlaceSeed(&mut self.0[i]);\n                        tri!(seq.next_element_seed(next_place))\n                    };\n                    if next.is_none() {\n                        self.0.truncate(i);\n                        return Ok(());\n                    }\n                }\n\n                while let Some(value) = tri!(seq.next_element()) {\n                    self.0.push(value);\n                }\n\n                Ok(())\n            }\n        }\n\n        deserializer.deserialize_seq(VecInPlaceVisitor(place))\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct ArrayVisitor<A> {\n    marker: PhantomData<A>,\n}\nstruct ArrayInPlaceVisitor<'a, A: 'a>(&'a mut A);\n\nimpl<A> ArrayVisitor<A> {\n    fn new() -> Self {\n        ArrayVisitor {\n            marker: PhantomData,\n        }\n    }\n}\n\nimpl<'de, T> Visitor<'de> for ArrayVisitor<[T; 0]> {\n    type Value = [T; 0];\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"an empty array\")\n    }\n\n    #[inline]\n    fn visit_seq<A>(self, _: A) -> Result<Self::Value, A::Error>\n    where\n        A: SeqAccess<'de>,\n    {\n        Ok([])\n    }\n}\n\n// Does not require T: Deserialize<'de>.\nimpl<'de, T> Deserialize<'de> for [T; 0] {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_tuple(0, ArrayVisitor::<[T; 0]>::new())\n    }\n}\n\nmacro_rules! array_impls {\n    ($($len:expr => ($($n:tt)+))+) => {\n        $(\n            impl<'de, T> Visitor<'de> for ArrayVisitor<[T; $len]>\n            where\n                T: Deserialize<'de>,\n            {\n                type Value = [T; $len];\n\n                fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                    formatter.write_str(concat!(\"an array of length \", $len))\n                }\n\n                #[inline]\n                fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n                where\n                    A: SeqAccess<'de>,\n                {\n                    Ok([$(\n                        match tri!(seq.next_element()) {\n                            Some(val) => val,\n                            None => return Err(Error::invalid_length($n, &self)),\n                        }\n                    ),+])\n                }\n            }\n\n            impl<'a, 'de, T> Visitor<'de> for ArrayInPlaceVisitor<'a, [T; $len]>\n            where\n                T: Deserialize<'de>,\n            {\n                type Value = ();\n\n                fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                    formatter.write_str(concat!(\"an array of length \", $len))\n                }\n\n                #[inline]\n                fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n                where\n                    A: SeqAccess<'de>,\n                {\n                    let mut fail_idx = None;\n                    for (idx, dest) in self.0[..].iter_mut().enumerate() {\n                        if tri!(seq.next_element_seed(InPlaceSeed(dest))).is_none() {\n                            fail_idx = Some(idx);\n                            break;\n                        }\n                    }\n                    if let Some(idx) = fail_idx {\n                        return Err(Error::invalid_length(idx, &self));\n                    }\n                    Ok(())\n                }\n            }\n\n            impl<'de, T> Deserialize<'de> for [T; $len]\n            where\n                T: Deserialize<'de>,\n            {\n                fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n                where\n                    D: Deserializer<'de>,\n                {\n                    deserializer.deserialize_tuple($len, ArrayVisitor::<[T; $len]>::new())\n                }\n\n                fn deserialize_in_place<D>(deserializer: D, place: &mut Self) -> Result<(), D::Error>\n                where\n                    D: Deserializer<'de>,\n                {\n                    deserializer.deserialize_tuple($len, ArrayInPlaceVisitor(place))\n                }\n            }\n        )+\n    }\n}\n\narray_impls! {\n    1 => (0)\n    2 => (0 1)\n    3 => (0 1 2)\n    4 => (0 1 2 3)\n    5 => (0 1 2 3 4)\n    6 => (0 1 2 3 4 5)\n    7 => (0 1 2 3 4 5 6)\n    8 => (0 1 2 3 4 5 6 7)\n    9 => (0 1 2 3 4 5 6 7 8)\n    10 => (0 1 2 3 4 5 6 7 8 9)\n    11 => (0 1 2 3 4 5 6 7 8 9 10)\n    12 => (0 1 2 3 4 5 6 7 8 9 10 11)\n    13 => (0 1 2 3 4 5 6 7 8 9 10 11 12)\n    14 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13)\n    15 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14)\n    16 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)\n    17 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16)\n    18 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17)\n    19 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18)\n    20 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19)\n    21 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20)\n    22 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21)\n    23 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22)\n    24 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23)\n    25 => (0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24)\n...\n", "needle_token_start": 28, "needle_token_end": 71, "code_context_ntokens": 13706, "output": ["```rust\nuse crate::lib::*;\n\npub fn from_bounds<I>(iter: &I) -> Option<usize>\nwhere\n    I: Iterator,\n{\n    helper(iter.size_hint())\n}\n\nfn helper(bounds: (usize, Option<usize>)) -> Option<usize> {\n    match bounds {\n        (lower, Some(upper)) if lower == upper => Some(upper),\n        _ => None,\n    }\n}\n```"]}
{"repo": "serde-rs/serde", "name": "serialize_key", "language": "rust", "path": "serde/src/ser/impossible.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the serialization of keys in a map structure, but it is intentionally made to be non-functional, as indicated by its implementation which leads to a compile-time error.\n2. **Input**: The function takes a reference to a key of any type that implements a specific trait for serialization.\n3. **Output**: The function is supposed to return a result indicating success or failure of the serialization process; however, it always results in a compile-time error due to its design.\n4. **Procedure**: Upon invocation, the function immediately enters a match statement with an empty enum, which is logically impossible to execute, leading to a guaranteed compile-time error. This is used to enforce the non-usability of the function in contexts where it should not be used.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: serde/src/de/impls.rs\nuse crate::lib::*;\n\nuse crate::de::{\n    Deserialize, Deserializer, EnumAccess, Error, MapAccess, SeqAccess, Unexpected, VariantAccess,\n    Visitor,\n};\n\nuse crate::seed::InPlaceSeed;\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nuse crate::de::size_hint;\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct UnitVisitor;\n\nimpl<'de> Visitor<'de> for UnitVisitor {\n    type Value = ();\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"unit\")\n    }\n\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(())\n    }\n}\n\nimpl<'de> Deserialize<'de> for () {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_unit(UnitVisitor)\n    }\n}\n\n#[cfg(feature = \"unstable\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"unstable\")))]\nimpl<'de> Deserialize<'de> for ! {\n    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        Err(Error::custom(\"cannot deserialize `!`\"))\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct BoolVisitor;\n\nimpl<'de> Visitor<'de> for BoolVisitor {\n    type Value = bool;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a boolean\")\n    }\n\n    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n}\n\nimpl<'de> Deserialize<'de> for bool {\n...\n// Path: serde/src/de/ignored_any.rs\nuse crate::lib::*;\n\nuse crate::de::{\n    Deserialize, Deserializer, EnumAccess, Error, MapAccess, SeqAccess, VariantAccess, Visitor,\n};\n\n/// An efficient way of discarding data from a deserializer.\n///\n/// Think of this like `serde_json::Value` in that it can be deserialized from\n/// any type, except that it does not store any information about the data that\n/// gets deserialized.\n///\n/// ```edition2021\n/// use serde::de::{\n///     self, Deserialize, DeserializeSeed, Deserializer, IgnoredAny, SeqAccess, Visitor,\n/// };\n/// use std::fmt;\n/// use std::marker::PhantomData;\n///\n/// /// A seed that can be used to deserialize only the `n`th element of a sequence\n/// /// while efficiently discarding elements of any type before or after index `n`.\n/// ///\n/// /// For example to deserialize only the element at index 3:\n/// ///\n/// /// ```\n/// /// NthElement::new(3).deserialize(deserializer)\n/// /// ```\n/// pub struct NthElement<T> {\n///     n: usize,\n///     marker: PhantomData<T>,\n/// }\n///\n/// impl<T> NthElement<T> {\n///     pub fn new(n: usize) -> Self {\n///         NthElement {\n///             n: n,\n///             marker: PhantomData,\n///         }\n///     }\n/// }\n///\n/// impl<'de, T> Visitor<'de> for NthElement<T>\n/// where\n///     T: Deserialize<'de>,\n/// {\n///     type Value = T;\n///\n///     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///         write!(\n///             formatter,\n///             \"a sequence in which we care about element {}\",\n///             self.n\n///         )\n///     }\n///\n///     fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n///     where\n///         A: SeqAccess<'de>,\n///     {\n///         // Skip over the first `n` elements.\n///         for i in 0..self.n {\n///             // It is an error if the sequence ends before we get to element `n`.\n///             if seq.next_element::<IgnoredAny>()?.is_none() {\n///                 return Err(de::Error::invalid_length(i, &self));\n///             }\n///         }\n///\n///         // Deserialize the one we care about.\n///         let nth = match seq.next_element()? {\n///             Some(nth) => nth,\n///             None => {\n///                 return Err(de::Error::invalid_length(self.n, &self));\n///             }\n///         };\n///\n///         // Skip over any remaining elements in the sequence after `n`.\n///         while let Some(IgnoredAny) = seq.next_element()? {\n///             // ignore\n///         }\n///\n///         Ok(nth)\n///     }\n/// }\n///\n/// impl<'de, T> DeserializeSeed<'de> for NthElement<T>\n/// where\n///     T: Deserialize<'de>,\n/// {\n///     type Value = T;\n///\n///     fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n///     where\n///         D: Deserializer<'de>,\n///     {\n///         deserializer.deserialize_seq(self)\n///     }\n/// }\n///\n/// # fn example<'de, D>(deserializer: D) -> Result<(), D::Error>\n/// # where\n/// #     D: Deserializer<'de>,\n/// # {\n/// // Deserialize only the sequence element at index 3 from this deserializer.\n/// // The element at index 3 is required to be a string. Elements before and\n/// // after index 3 are allowed to be of any type.\n/// let s: String = NthElement::new(3).deserialize(deserializer)?;\n/// #     Ok(())\n/// # }\n/// ```\n#[derive(Copy, Clone, Debug, Default, PartialEq)]\npub struct IgnoredAny;\n\nimpl<'de> Visitor<'de> for IgnoredAny {\n    type Value = IgnoredAny;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"anything at all\")\n    }\n\n    #[inline]\n    fn visit_bool<E>(self, x: bool) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_i64<E>(self, x: i64) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_i128<E>(self, x: i128) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_u64<E>(self, x: u64) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_u128<E>(self, x: u128) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_f64<E>(self, x: f64) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let _ = s;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_none<E>(self) -> Result<Self::Value, E> {\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        IgnoredAny::deserialize(deserializer)\n    }\n\n    #[inline]\n    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        IgnoredAny::deserialize(deserializer)\n    }\n\n    #[inline]\n    fn visit_unit<E>(self) -> Result<Self::Value, E> {\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n    where\n        A: SeqAccess<'de>,\n    {\n        while let Some(IgnoredAny) = tri!(seq.next_element()) {\n            // Gobble\n        }\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>\n    where\n        A: MapAccess<'de>,\n    {\n        while let Some((IgnoredAny, IgnoredAny)) = tri!(map.next_entry()) {\n            // Gobble\n        }\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_bytes<E>(self, bytes: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let _ = bytes;\n        Ok(IgnoredAny)\n    }\n\n    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>\n    where\n        A: EnumAccess<'de>,\n    {\n        tri!(data.variant::<IgnoredAny>()).1.newtype_variant()\n    }\n}\n\nimpl<'de> Deserialize<'de> for IgnoredAny {\n    #[inline]\n    fn deserialize<D>(deserializer: D) -> Result<IgnoredAny, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_ignored_any(IgnoredAny)\n    }\n}\n\n// Path: serde/src/ser/impossible.rs\n//! This module contains `Impossible` serializer and its implementations.\n\nuse crate::lib::*;\n\nuse crate::ser::{\n    self, Serialize, SerializeMap, SerializeSeq, SerializeStruct, SerializeStructVariant,\n    SerializeTuple, SerializeTupleStruct, SerializeTupleVariant,\n};\n\n/// Helper type for implementing a `Serializer` that does not support\n/// serializing one of the compound types.\n///\n/// This type cannot be instantiated, but implements every one of the traits\n/// corresponding to the [`Serializer`] compound types: [`SerializeSeq`],\n/// [`SerializeTuple`], [`SerializeTupleStruct`], [`SerializeTupleVariant`],\n/// [`SerializeMap`], [`SerializeStruct`], and [`SerializeStructVariant`].\n///\n/// ```edition2021\n/// # use serde::ser::{Serializer, Impossible};\n/// # use serde::__private::doc::Error;\n/// #\n/// # struct MySerializer;\n/// #\n/// impl Serializer for MySerializer {\n///     type Ok = ();\n///     type Error = Error;\n///\n///     type SerializeSeq = Impossible<(), Error>;\n///     /* other associated types */\n///\n///     /// This data format does not support serializing sequences.\n///     fn serialize_seq(self,\n///                      len: Option<usize>)\n///                      -> Result<Self::SerializeSeq, Error> {\n///         // Given Impossible cannot be instantiated, the only\n///         // thing we can do here is to return an error.\n/// #         stringify! {\n///         Err(...)\n/// #         };\n/// #         unimplemented!()\n///     }\n///\n///     /* other Serializer methods */\n/// #     serde::__serialize_unimplemented! {\n/// #         bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str bytes none some\n/// #         unit unit_struct unit_variant newtype_struct newtype_variant\n/// #         tuple tuple_struct tuple_variant map struct struct_variant\n/// #     }\n/// }\n/// ```\n///\n/// [`Serializer`]: trait.Serializer.html\n/// [`SerializeSeq`]: trait.SerializeSeq.html\n/// [`SerializeTuple`]: trait.SerializeTuple.html\n/// [`SerializeTupleStruct`]: trait.SerializeTupleStruct.html\n/// [`SerializeTupleVariant`]: trait.SerializeTupleVariant.html\n/// [`SerializeMap`]: trait.SerializeMap.html\n/// [`SerializeStruct`]: trait.SerializeStruct.html\n/// [`SerializeStructVariant`]: trait.SerializeStructVariant.html\npub struct Impossible<Ok, Error> {\n    void: Void,\n    ok: PhantomData<Ok>,\n    error: PhantomData<Error>,\n}\n\nenum Void {}\n\nimpl<Ok, Error> SerializeSeq for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_element<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeTuple for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_element<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeTupleStruct for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeTupleVariant for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeMap for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    \nfn serialize_key<T>(&mut self, key: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = key;\n        match self.void {}\n    }\n\n    fn serialize_value<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeStruct for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = key;\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeStructVariant for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = key;\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\n// Path: serde/src/ser/fmt.rs\nuse crate::lib::*;\nuse crate::ser::{Error, Impossible, Serialize, Serializer};\n\nimpl Error for fmt::Error {\n    fn custom<T: Display>(_msg: T) -> Self {\n        fmt::Error\n    }\n}\n\nmacro_rules! fmt_primitives {\n    ($($f:ident: $t:ty,)*) => {\n        $(\n            fn $f(self, v: $t) -> fmt::Result {\n                Display::fmt(&v, self)\n            }\n        )*\n    };\n}\n\n/// ```edition2021\n/// use serde::ser::Serialize;\n/// use serde_derive::Serialize;\n/// use std::fmt::{self, Display};\n///\n/// #[derive(Serialize)]\n/// #[serde(rename_all = \"kebab-case\")]\n/// pub enum MessageType {\n///     StartRequest,\n///     EndRequest,\n/// }\n///\n/// impl Display for MessageType {\n///     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n///         self.serialize(f)\n///     }\n/// }\n/// ```\nimpl<'a, 'b> Serializer for &'a mut fmt::Formatter<'b> {\n    type Ok = ();\n    type Error = fmt::Error;\n    type SerializeSeq = Impossible<(), fmt::Error>;\n    type SerializeTuple = Impossible<(), fmt::Error>;\n    type SerializeTupleStruct = Impossible<(), fmt::Error>;\n    type SerializeTupleVariant = Impossible<(), fmt::Error>;\n    type SerializeMap = Impossible<(), fmt::Error>;\n    type SerializeStruct = Impossible<(), fmt::Error>;\n    type SerializeStructVariant = Impossible<(), fmt::Error>;\n\n    fmt_primitives! {\n        serialize_bool: bool,\n        serialize_i8: i8,\n        serialize_i16: i16,\n        serialize_i32: i32,\n        serialize_i64: i64,\n        serialize_i128: i128,\n        serialize_u8: u8,\n        serialize_u16: u16,\n        serialize_u32: u32,\n        serialize_u64: u64,\n        serialize_u128: u128,\n        serialize_f32: f32,\n        serialize_f64: f64,\n        serialize_char: char,\n        serialize_str: &str,\n        serialize_unit_struct: &'static str,\n    }\n\n    fn serialize_unit_variant(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        variant: &'static str,\n    ) -> fmt::Result {\n        Display::fmt(variant, self)\n    }\n\n    fn serialize_newtype_struct<T>(self, _name: &'static str, value: &T) -> fmt::Result\n    where\n        T: ?Sized + Serialize,\n    {\n        Serialize::serialize(value, self)\n    }\n\n    fn serialize_bytes(self, _v: &[u8]) -> fmt::Result {\n        Err(fmt::Error)\n    }\n\n    fn serialize_none(self) -> fmt::Result {\n        Err(fmt::Error)\n    }\n\n    fn serialize_some<T>(self, _value: &T) -> fmt::Result\n    where\n        T: ?Sized + Serialize,\n    {\n        Err(fmt::Error)\n    }\n\n    fn serialize_unit(self) -> fmt::Result {\n        Err(fmt::Error)\n    }\n\n    fn serialize_newtype_variant<T>(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        _variant: &'static str,\n        _value: &T,\n    ) -> fmt::Result\n    where\n        T: ?Sized + Serialize,\n    {\n        Err(fmt::Error)\n    }\n\n    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_tuple_struct(\n        self,\n        _name: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeTupleStruct, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_tuple_variant(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        _variant: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeTupleVariant, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_struct(\n        self,\n        _name: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeStruct, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_struct_variant(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        _variant: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeStructVariant, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn collect_str<T>(self, value: &T) -> fmt::Result\n    where\n        T: ?Sized + Display,\n    {\n        Display::fmt(value, self)\n    }\n}\n\n// Path: serde/src/ser/impls.rs\nuse crate::lib::*;\n\nuse crate::ser::{Error, Serialize, SerializeTuple, Serializer};\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! primitive_impl {\n    ($ty:ident, $method:ident $($cast:tt)*) => {\n        impl Serialize for $ty {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.$method(*self $($cast)*)\n            }\n        }\n    }\n}\n\nprimitive_impl!(bool, serialize_bool);\nprimitive_impl!(isize, serialize_i64 as i64);\nprimitive_impl!(i8, serialize_i8);\nprimitive_impl!(i16, serialize_i16);\nprimitive_impl!(i32, serialize_i32);\nprimitive_impl!(i64, serialize_i64);\nprimitive_impl!(i128, serialize_i128);\nprimitive_impl!(usize, serialize_u64 as u64);\nprimitive_impl!(u8, serialize_u8);\nprimitive_impl!(u16, serialize_u16);\nprimitive_impl!(u32, serialize_u32);\nprimitive_impl!(u64, serialize_u64);\nprimitive_impl!(u128, serialize_u128);\nprimitive_impl!(f32, serialize_f32);\nprimitive_impl!(f64, serialize_f64);\nprimitive_impl!(char, serialize_char);\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for str {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_str(self)\n    }\n}\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl Serialize for String {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_str(self)\n    }\n}\n\nimpl<'a> Serialize for fmt::Arguments<'a> {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.collect_str(self)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(any(feature = \"std\", not(no_core_cstr)))]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for CStr {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_bytes(self.to_bytes())\n    }\n}\n\n#[cfg(any(feature = \"std\", all(not(no_core_cstr), feature = \"alloc\")))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl Serialize for CString {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_bytes(self.to_bytes())\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for Option<T>\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Some(ref value) => serializer.serialize_some(value),\n            None => serializer.serialize_none(),\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for PhantomData<T>\nwhere\n    T: ?Sized,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_unit_struct(\"PhantomData\")\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// Does not require T: Serialize.\nimpl<T> Serialize for [T; 0] {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        tri!(serializer.serialize_tuple(0)).end()\n    }\n}\n\nmacro_rules! array_impls {\n    ($($len:tt)+) => {\n        $(\n            impl<T> Serialize for [T; $len]\n            where\n                T: Serialize,\n            {\n                #[inline]\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    let mut seq = tri!(serializer.serialize_tuple($len));\n                    for e in self {\n                        tri!(seq.serialize_element(e));\n                    }\n                    seq.end()\n                }\n            }\n        )+\n    }\n}\n\narray_impls! {\n    01 02 03 04 05 06 07 08 09 10\n    11 12 13 14 15 16 17 18 19 20\n    21 22 23 24 25 26 27 28 29 30\n    31 32\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for [T]\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.collect_seq(self)\n    }\n}\n\n#[cfg(not(no_relaxed_trait_bounds))]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize,\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\n#[cfg(no_relaxed_trait_bounds)]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize $(+ $tbound1 $(+ $tbound2)*)*,\n            $($typaram: $bound,)*\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BinaryHeap<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeSet<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashSet<T: Eq + Hash, H: BuildHasher>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    LinkedList<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    Vec<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    VecDeque<T>\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for Range<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"Range\", 2));\n        tri!(state.serialize_field(\"start\", &self.start));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeFrom<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeFrom\", 1));\n        tri!(state.serialize_field(\"start\", &self.start));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeInclusive<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeInclusive\", 2));\n        tri!(state.serialize_field(\"start\", &self.start()));\n        tri!(state.serialize_field(\"end\", &self.end()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeTo<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeTo\", 1));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for Bound<T>\nwhere\n    T: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Bound::Unbounded => serializer.serialize_unit_variant(\"Bound\", 0, \"Unbounded\"),\n            Bound::Included(ref value) => {\n                serializer.serialize_newtype_variant(\"Bound\", 1, \"Included\", value)\n            }\n            Bound::Excluded(ref value) => {\n                serializer.serialize_newtype_variant(\"Bound\", 2, \"Excluded\", value)\n            }\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for () {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_unit()\n    }\n}\n\n#[cfg(feature = \"unstable\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"unstable\")))]\nimpl Serialize for ! {\n    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        *self\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! tuple_impls {\n    ($($len:expr => ($($n:tt $name:ident)+))+) => {\n        $(\n            impl<$($name),+> Serialize for ($($name,)+)\n            where\n                $($name: Serialize,)+\n            {\n                #[inline]\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    let mut tuple = tri!(serializer.serialize_tuple($len));\n                    $(\n                        tri!(tuple.serialize_element(&self.$n));\n                    )+\n                    tuple.end()\n                }\n            }\n        )+\n    }\n}\n\ntuple_impls! {\n    1 => (0 T0)\n    2 => (0 T0 1 T1)\n    3 => (0 T0 1 T1 2 T2)\n    4 => (0 T0 1 T1 2 T2 3 T3)\n    5 => (0 T0 1 T1 2 T2 3 T3 4 T4)\n    6 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5)\n    7 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6)\n    8 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7)\n    9 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8)\n    10 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9)\n    11 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10)\n    12 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11)\n    13 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12)\n    14 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13)\n    15 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14)\n    16 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14 15 T15)\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(not(no_relaxed_trait_bounds))]\nmacro_rules! map_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <K $(: $kbound1:ident $(+ $kbound2:ident)*)*, V $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<K, V $(, $typaram)*> Serialize for $ty<K, V $(, $typaram)*>\n        where\n            K: Serialize,\n            V: Serialize,\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_map(self)\n            }\n        }\n    }\n}\n\n#[cfg(no_relaxed_trait_bounds)]\nmacro_rules! map_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <K $(: $kbound1:ident $(+ $kbound2:ident)*)*, V $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<K, V $(, $typaram)*> Serialize for $ty<K, V $(, $typaram)*>\n        where\n            K: Serialize $(+ $kbound1 $(+ $kbound2)*)*,\n            V: Serialize,\n            $($typaram: $bound,)*\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_map(self)\n            }\n        }\n    }\n}\n\nmap_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeMap<K: Ord, V>\n}\n\nmap_impl! {\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashMap<K: Eq + Hash, V, H: BuildHasher>\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! deref_impl {\n    (\n        $(#[$attr:meta])*\n        <$($desc:tt)+\n    ) => {\n        $(#[$attr])*\n        impl <$($desc)+ {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                (**self).serialize(serializer)\n            }\n        }\n    };\n}\n\nderef_impl! {\n    <'a, T> Serialize for &'a T where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    <'a, T> Serialize for &'a mut T where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    <T> Serialize for Box<T> where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    /// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n    ///\n    /// Serializing a data structure containing `Rc` will serialize a copy of\n    /// the contents of the `Rc` each time the `Rc` is referenced within the\n    /// data structure. Serialization will not attempt to deduplicate these\n    /// repeated data.\n    ///\n    /// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n    #[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n    #[cfg_attr(doc_cfg, doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))))]\n    <T> Serialize for Rc<T> where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    /// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n    ///\n    /// Serializing a data structure containing `Arc` will serialize a copy of\n    /// the contents of the `Arc` each time the `Arc` is referenced within the\n    /// data structure. Serialization will not attempt to deduplicate these\n    /// repeated data.\n    ///\n    /// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n    #[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n    #[cfg_attr(doc_cfg, doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))))]\n    <T> Serialize for Arc<T> where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    <'a, T> Serialize for Cow<'a, T> where T: ?Sized + Serialize + ToOwned\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n///\n/// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n#[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n#[cfg_attr(\n    doc_cfg,\n    doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\"))))\n)]\nimpl<T> Serialize for RcWeak<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.upgrade().serialize(serializer)\n    }\n}\n\n/// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n///\n/// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n#[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n#[cfg_attr(\n    doc_cfg,\n    doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\"))))\n)]\nimpl<T> Serialize for ArcWeak<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.upgrade().serialize(serializer)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! nonzero_integers {\n    ($($T:ident,)+) => {\n        $(\n            impl Serialize for num::$T {\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    self.get().serialize(serializer)\n                }\n            }\n        )+\n    }\n}\n\nnonzero_integers! {\n    NonZeroU8,\n    NonZeroU16,\n    NonZeroU32,\n    NonZeroU64,\n    NonZeroU128,\n    NonZeroUsize,\n}\n\n#[cfg(not(no_num_nonzero_signed))]\nnonzero_integers! {\n    NonZeroI8,\n    NonZeroI16,\n    NonZeroI32,\n    NonZeroI64,\n    NonZeroI128,\n    NonZeroIsize,\n}\n\nimpl<T> Serialize for Cell<T>\nwhere\n    T: Serialize + Copy,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.get().serialize(serializer)\n    }\n}\n\nimpl<T> Serialize for RefCell<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.try_borrow() {\n            Ok(value) => value.serialize(serializer),\n            Err(_) => Err(S::Error::custom(\"already mutably borrowed\")),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl<T> Serialize for Mutex<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.lock() {\n            Ok(locked) => locked.serialize(serializer),\n            Err(_) => Err(S::Error::custom(\"lock poison error while serializing\")),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl<T> Serialize for RwLock<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.read() {\n            Ok(locked) => locked.serialize(serializer),\n            Err(_) => Err(S::Error::custom(\"lock poison error while serializing\")),\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T, E> Serialize for Result<T, E>\nwhere\n    T: Serialize,\n    E: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Result::Ok(ref value) => serializer.serialize_newtype_variant(\"Result\", 0, \"Ok\", value),\n            Result::Err(ref value) => {\n                serializer.serialize_newtype_variant(\"Result\", 1, \"Err\", value)\n            }\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for Duration {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"Duration\", 2));\n        tri!(state.serialize_field(\"secs\", &self.as_secs()));\n        tri!(state.serialize_field(\"nanos\", &self.subsec_nanos()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for SystemTime {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let duration_since_epoch = match self.duration_since(UNIX_EPOCH) {\n            Ok(duration_since_epoch) => duration_since_epoch,\n            Err(_) => return Err(S::Error::custom(\"SystemTime must be later than UNIX_EPOCH\")),\n        };\n        let mut state = tri!(serializer.serialize_struct(\"SystemTime\", 2));\n        tri!(state.serialize_field(\"secs_since_epoch\", &duration_since_epoch.as_secs()));\n        tri!(state.serialize_field(\"nanos_since_epoch\", &duration_since_epoch.subsec_nanos()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Serialize a value that implements `Display` as a string, when that string is\n/// statically known to never have more than a constant `MAX_LEN` bytes.\n///\n/// Panics if the `Display` impl tries to write more than `MAX_LEN` bytes.\n#[cfg(feature = \"std\")]\nmacro_rules! serialize_display_bounded_length {\n    ($value:expr, $max:expr, $serializer:expr) => {{\n        let mut buffer = [0u8; $max];\n        let remaining_len = {\n            let mut remaining = &mut buffer[..];\n            write!(remaining, \"{}\", $value).unwrap();\n            remaining.len()\n        };\n        let written_len = buffer.len() - remaining_len;\n        let written = &buffer[..written_len];\n\n        // write! only provides fmt::Formatter to Display implementations, which\n        // has methods write_str and write_char but no method to write arbitrary\n        // bytes. Therefore `written` must be valid UTF-8.\n        let written_str = str::from_utf8(written).expect(\"must be valid UTF-8\");\n        $serializer.serialize_str(written_str)\n    }};\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::IpAddr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            match *self {\n                net::IpAddr::V4(ref a) => a.serialize(serializer),\n                net::IpAddr::V6(ref a) => a.serialize(serializer),\n            }\n        } else {\n            match *self {\n                net::IpAddr::V4(ref a) => {\n                    serializer.serialize_newtype_variant(\"IpAddr\", 0, \"V4\", a)\n                }\n                net::IpAddr::V6(ref a) => {\n                    serializer.serialize_newtype_variant(\"IpAddr\", 1, \"V6\", a)\n                }\n            }\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nconst DEC_DIGITS_LUT: &[u8] = b\"\\\n      0001020304050607080910111213141516171819\\\n      2021222324252627282930313233343536373839\\\n      4041424344454647484950515253545556575859\\\n      6061626364656667686970717273747576777879\\\n      8081828384858687888990919293949596979899\";\n\n#[cfg(feature = \"std\")]\n#[inline]\nfn format_u8(mut n: u8, out: &mut [u8]) -> usize {\n    if n >= 100 {\n        let d1 = ((n % 100) << 1) as usize;\n        n /= 100;\n        out[0] = b'0' + n;\n        out[1] = DEC_DIGITS_LUT[d1];\n        out[2] = DEC_DIGITS_LUT[d1 + 1];\n        3\n    } else if n >= 10 {\n        let d1 = (n << 1) as usize;\n        out[0] = DEC_DIGITS_LUT[d1];\n        out[1] = DEC_DIGITS_LUT[d1 + 1];\n        2\n    } else {\n        out[0] = b'0' + n;\n        1\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[test]\nfn test_format_u8() {\n    let mut i = 0u8;\n\n    loop {\n        let mut buf = [0u8; 3];\n        let written = format_u8(i, &mut buf);\n        assert_eq!(i.to_string().as_bytes(), &buf[..written]);\n\n        match i.checked_add(1) {\n            Some(next) => i = next,\n            None => break,\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::Ipv4Addr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 15;\n            debug_assert_eq!(MAX_LEN, \"101.102.103.104\".len());\n            let mut buf = [b'.'; MAX_LEN];\n            let mut written = format_u8(self.octets()[0], &mut buf);\n            for oct in &self.octets()[1..] {\n                // Skip over delimiters that we initialized buf with\n                written += format_u8(*oct, &mut buf[written + 1..]) + 1;\n            }\n            // Safety: We've only written ASCII bytes to the buffer, so it is valid UTF-8\n            let buf = unsafe { str::from_utf8_unchecked(&buf[..written]) };\n            serializer.serialize_str(buf)\n        } else {\n            self.octets().serialize(serializer)\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::Ipv6Addr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 39;\n            debug_assert_eq!(MAX_LEN, \"1001:1002:1003:1004:1005:1006:1007:1008\".len());\n            serialize_display_bounded_length!(self, MAX_LEN, serializer)\n        } else {\n            self.octets().serialize(serializer)\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::SocketAddr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            match *self {\n                net::SocketAddr::V4(ref addr) => addr.serialize(serializer),\n                net::SocketAddr::V6(ref addr) => addr.serialize(serializer),\n            }\n        } else {\n            match *self {\n                net::SocketAddr::V4(ref addr) => {\n                    serializer.serialize_newtype_variant(\"SocketAddr\", 0, \"V4\", addr)\n                }\n                net::SocketAddr::V6(ref addr) => {\n                    serializer.serialize_newtype_variant(\"SocketAddr\", 1, \"V6\", addr)\n                }\n            }\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::SocketAddrV4 {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 21;\n            debug_assert_eq!(MAX_LEN, \"101.102.103.104:65000\".len());\n            serialize_display_bounded_length!(self, MAX_LEN, serializer)\n        } else {\n            (self.ip(), self.port()).serialize(serializer)\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::SocketAddrV6 {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 58;\n            debug_assert_eq!(\n                MAX_LEN,\n                \"[1001:1002:1003:1004:1005:1006:1007:1008%4294967295]:65000\".len()\n            );\n            serialize_display_bounded_length!(self, MAX_LEN, serializer)\n        } else {\n            (self.ip(), self.port()).serialize(serializer)\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for Path {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.to_str() {\n            Some(s) => s.serialize(serializer),\n            None => Err(Error::custom(\"path contains invalid UTF-8 characters\")),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for PathBuf {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.as_path().serialize(serializer)\n    }\n}\n\n#[cfg(all(feature = \"std\", any(unix, windows)))]\n#[cfg_attr(doc_cfg, doc(cfg(all(feature = \"std\", any(unix, windows)))))]\nimpl Serialize for OsStr {\n    #[cfg(unix)]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use std::os::unix::ffi::OsStrExt;\n        serializer.serialize_newtype_variant(\"OsString\", 0, \"Unix\", self.as_bytes())\n    }\n\n    #[cfg(windows)]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use std::os::windows::ffi::OsStrExt;\n        let val = self.encode_wide().collect::<Vec<_>>();\n        serializer.serialize_newtype_variant(\"OsString\", 1, \"Windows\", &val)\n    }\n}\n\n#[cfg(all(feature = \"std\", any(unix, windows)))]\n...\n", "needle_token_start": 3993, "needle_token_end": 4049, "code_context_ntokens": 16161, "output": ["```rust\nfn serialize_key<T>(&mut self, key: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n{\n    let _ = key;\n    match self.void {}\n}\n```"]}
{"repo": "serde-rs/serde", "name": "__deserialize_content", "language": "rust", "path": "serde/src/de/mod.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To deserialize any type of content into a generic data structure without specifying the type of the content explicitly.\n2. **Input**: Takes a deserializer instance and a visitor that can handle the deserialization of any content.\n3. **Output**: Returns a generic data structure that represents the deserialized content or an error if the deserialization fails.\n4. **Procedure**: The function delegates the task of deserialization to a method that handles any type of data, using the provided visitor to interpret the incoming data into a generic format.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "///     }\n///\n///     fn visit_seq<A>(self, mut seq: A) -> Result<Vec<T>, A::Error>\n///     where\n///         A: SeqAccess<'de>,\n///     {\n///         // Create a single Vec to hold the flattened contents.\n///         let mut vec = Vec::new();\n///\n///         // Each iteration through this loop is one inner array.\n///         while let Some(()) = seq.next_element_seed(ExtendVec(&mut vec))? {\n///             // Nothing to do; inner array has been appended into `vec`.\n///         }\n///\n///         // Return the finished vec.\n///         Ok(vec)\n///     }\n/// }\n///\n/// # fn example<'de, D>(deserializer: D) -> Result<(), D::Error>\n/// # where\n/// #     D: Deserializer<'de>,\n/// # {\n/// let visitor = FlattenedVecVisitor(PhantomData);\n/// let flattened: Vec<u64> = deserializer.deserialize_seq(visitor)?;\n/// #     Ok(())\n/// # }\n/// ```\npub trait DeserializeSeed<'de>: Sized {\n    /// The type produced by using this seed.\n    type Value;\n\n    /// Equivalent to the more common `Deserialize::deserialize` method, except\n    /// with some initial piece of data (the seed) passed in.\n    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>;\n}\n\nimpl<'de, T> DeserializeSeed<'de> for PhantomData<T>\nwhere\n    T: Deserialize<'de>,\n{\n    type Value = T;\n\n    #[inline]\n    fn deserialize<D>(self, deserializer: D) -> Result<T, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        T::deserialize(deserializer)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data format** that can deserialize any data structure supported by\n/// Serde.\n///\n/// The role of this trait is to define the deserialization half of the [Serde\n/// data model], which is a way to categorize every Rust data type into one of\n/// 29 possible types. Each method of the `Deserializer` trait corresponds to one\n/// of the types of the data model.\n///\n/// Implementations of `Deserialize` map themselves into this data model by\n/// passing to the `Deserializer` a `Visitor` implementation that can receive\n/// these various types.\n///\n/// The types that make up the Serde data model are:\n///\n///  - **14 primitive types**\n///    - bool\n///    - i8, i16, i32, i64, i128\n///    - u8, u16, u32, u64, u128\n///    - f32, f64\n///    - char\n///  - **string**\n///    - UTF-8 bytes with a length and no null terminator.\n///    - When serializing, all strings are handled equally. When deserializing,\n///      there are three flavors of strings: transient, owned, and borrowed.\n///  - **byte array** - \\[u8\\]\n///    - Similar to strings, during deserialization byte arrays can be\n///      transient, owned, or borrowed.\n///  - **option**\n///    - Either none or some value.\n///  - **unit**\n///    - The type of `()` in Rust. It represents an anonymous value containing\n///      no data.\n///  - **unit_struct**\n///    - For example `struct Unit` or `PhantomData<T>`. It represents a named\n///      value containing no data.\n///  - **unit_variant**\n///    - For example the `E::A` and `E::B` in `enum E { A, B }`.\n///  - **newtype_struct**\n///    - For example `struct Millimeters(u8)`.\n///  - **newtype_variant**\n///    - For example the `E::N` in `enum E { N(u8) }`.\n///  - **seq**\n///    - A variably sized heterogeneous sequence of values, for example `Vec<T>`\n///      or `HashSet<T>`. When serializing, the length may or may not be known\n///      before iterating through all the data. When deserializing, the length\n///      is determined by looking at the serialized data.\n///  - **tuple**\n///    - A statically sized heterogeneous sequence of values for which the\n///      length will be known at deserialization time without looking at the\n///      serialized data, for example `(u8,)` or `(String, u64, Vec<T>)` or\n///      `[u64; 10]`.\n///  - **tuple_struct**\n///    - A named tuple, for example `struct Rgb(u8, u8, u8)`.\n///  - **tuple_variant**\n///    - For example the `E::T` in `enum E { T(u8, u8) }`.\n///  - **map**\n///    - A heterogeneous key-value pairing, for example `BTreeMap<K, V>`.\n///  - **struct**\n///    - A heterogeneous key-value pairing in which the keys are strings and\n///      will be known at deserialization time without looking at the serialized\n///      data, for example `struct S { r: u8, g: u8, b: u8 }`.\n///  - **struct_variant**\n///    - For example the `E::S` in `enum E { S { r: u8, g: u8, b: u8 } }`.\n///\n/// The `Deserializer` trait supports two entry point styles which enables\n/// different kinds of deserialization.\n///\n/// 1. The `deserialize_any` method. Self-describing data formats like JSON are\n///    able to look at the serialized data and tell what it represents. For\n///    example the JSON deserializer may see an opening curly brace (`{`) and\n///    know that it is seeing a map. If the data format supports\n///    `Deserializer::deserialize_any`, it will drive the Visitor using whatever\n///    type it sees in the input. JSON uses this approach when deserializing\n///    `serde_json::Value` which is an enum that can represent any JSON\n///    document. Without knowing what is in a JSON document, we can deserialize\n///    it to `serde_json::Value` by going through\n///    `Deserializer::deserialize_any`.\n///\n/// 2. The various `deserialize_*` methods. Non-self-describing formats like\n///    Postcard need to be told what is in the input in order to deserialize it.\n///    The `deserialize_*` methods are hints to the deserializer for how to\n///    interpret the next piece of input. Non-self-describing formats are not\n///    able to deserialize something like `serde_json::Value` which relies on\n///    `Deserializer::deserialize_any`.\n///\n/// When implementing `Deserialize`, you should avoid relying on\n/// `Deserializer::deserialize_any` unless you need to be told by the\n/// Deserializer what type is in the input. Know that relying on\n/// `Deserializer::deserialize_any` means your data type will be able to\n/// deserialize from self-describing formats only, ruling out Postcard and many\n/// others.\n///\n/// [Serde data model]: https://serde.rs/data-model.html\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed from the input when deserializing. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website contains example code for\n/// a basic JSON `Deserializer`.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait Deserializer<'de>: Sized {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// Require the `Deserializer` to figure out how to drive the visitor based\n    /// on what data type is in the input.\n    ///\n    /// When implementing `Deserialize`, you should avoid relying on\n    /// `Deserializer::deserialize_any` unless you need to be told by the\n    /// Deserializer what type is in the input. Know that relying on\n    /// `Deserializer::deserialize_any` means your data type will be able to\n    /// deserialize from self-describing formats only, ruling out Postcard and\n    /// many others.\n    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `bool` value.\n    fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i8` value.\n    fn deserialize_i8<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i16` value.\n    fn deserialize_i16<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i32` value.\n    fn deserialize_i32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i64` value.\n    fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i128` value.\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>,\n    {\n        let _ = visitor;\n        Err(Error::custom(\"i128 is not supported\"))\n    }\n\n    /// Hint that the `Deserialize` type is expecting a `u8` value.\n    fn deserialize_u8<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u16` value.\n    fn deserialize_u16<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u32` value.\n    fn deserialize_u32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u64` value.\n    fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `u128` value.\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>,\n    {\n        let _ = visitor;\n        Err(Error::custom(\"u128 is not supported\"))\n    }\n\n    /// Hint that the `Deserialize` type is expecting a `f32` value.\n    fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `f64` value.\n    fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `char` value.\n    fn deserialize_char<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a string value and does\n    /// not benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would benefit from taking ownership of `String` data,\n    /// indicate this to the `Deserializer` by using `deserialize_string`\n    /// instead.\n    fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a string value and would\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would not benefit from taking ownership of `String`\n    /// data, indicate that to the `Deserializer` by using `deserialize_str`\n    /// instead.\n    fn deserialize_string<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a byte array and does not\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would benefit from taking ownership of `Vec<u8>` data,\n    /// indicate this to the `Deserializer` by using `deserialize_byte_buf`\n    /// instead.\n    fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a byte array and would\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would not benefit from taking ownership of `Vec<u8>`\n    /// data, indicate that to the `Deserializer` by using `deserialize_bytes`\n    /// instead.\n    fn deserialize_byte_buf<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an optional value.\n    ///\n    /// This allows deserializers that encode an optional value as a nullable\n    /// value to convert the null value into `None` and a regular value into\n    /// `Some(value)`.\n    fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a unit value.\n    fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a unit struct with a\n    /// particular name.\n    fn deserialize_unit_struct<V>(\n        self,\n        name: &'static str,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a newtype struct with a\n    /// particular name.\n    fn deserialize_newtype_struct<V>(\n        self,\n        name: &'static str,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a sequence of values.\n    fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a sequence of values and\n    /// knows how many values there are without looking at the serialized data.\n    fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a tuple struct with a\n    /// particular name and number of fields.\n    fn deserialize_tuple_struct<V>(\n        self,\n        name: &'static str,\n        len: usize,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a map of key-value pairs.\n    fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a struct with a particular\n    /// name and fields.\n    fn deserialize_struct<V>(\n        self,\n        name: &'static str,\n        fields: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an enum value with a\n    /// particular name and possible variants.\n    fn deserialize_enum<V>(\n        self,\n        name: &'static str,\n        variants: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting the name of a struct\n    /// field or the discriminant of an enum variant.\n    fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type needs to deserialize a value whose type\n    /// doesn't matter because it is ignored.\n    ///\n    /// Deserializers for non-self-describing formats may not support this mode.\n    fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Determine whether `Deserialize` implementations should expect to\n    /// deserialize their human-readable form.\n    ///\n    /// Some types have a human-readable form that may be somewhat expensive to\n    /// construct, as well as a binary form that is compact and efficient.\n    /// Generally text-based formats like JSON and YAML will prefer to use the\n    /// human-readable one and binary formats like Postcard will prefer the\n    /// compact one.\n    ///\n    /// ```edition2021\n    /// # use std::ops::Add;\n    /// # use std::str::FromStr;\n    /// #\n    /// # struct Timestamp;\n    /// #\n    /// # impl Timestamp {\n    /// #     const EPOCH: Timestamp = Timestamp;\n    /// # }\n    /// #\n    /// # impl FromStr for Timestamp {\n    /// #     type Err = String;\n    /// #     fn from_str(_: &str) -> Result<Self, Self::Err> {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// # struct Duration;\n    /// #\n    /// # impl Duration {\n    /// #     fn seconds(_: u64) -> Self { unimplemented!() }\n    /// # }\n    /// #\n    /// # impl Add<Duration> for Timestamp {\n    /// #     type Output = Timestamp;\n    /// #     fn add(self, _: Duration) -> Self::Output {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::de::{self, Deserialize, Deserializer};\n    ///\n    /// impl<'de> Deserialize<'de> for Timestamp {\n    ///     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    ///     where\n    ///         D: Deserializer<'de>,\n    ///     {\n    ///         if deserializer.is_human_readable() {\n    ///             // Deserialize from a human-readable string like \"2015-05-15T17:01:00Z\".\n    ///             let s = String::deserialize(deserializer)?;\n    ///             Timestamp::from_str(&s).map_err(de::Error::custom)\n    ///         } else {\n    ///             // Deserialize from a compact binary representation, seconds since\n    ///             // the Unix epoch.\n    ///             let n = u64::deserialize(deserializer)?;\n    ///             Ok(Timestamp::EPOCH + Duration::seconds(n))\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default implementation of this method returns `true`. Data formats\n    /// may override this to `false` to request a compact form for types that\n    /// support one. Note that modifying this method to change a format from\n    /// human-readable to compact or vice versa should be regarded as a breaking\n    /// change, as a value serialized in human-readable mode is not required to\n    /// deserialize from the same data in compact mode.\n    #[inline]\n    fn is_human_readable(&self) -> bool {\n        true\n    }\n\n    // Not public API.\n    #[cfg(all(not(no_serde_derive), any(feature = \"std\", feature = \"alloc\")))]\n    #[doc(hidden)]\n    \nfn __deserialize_content<V>(\n        self,\n        _: crate::actually_private::T,\n        visitor: V,\n    ) -> Result<crate::__private::de::Content<'de>, Self::Error>\n    where\n        V: Visitor<'de, Value = crate::__private::de::Content<'de>>,\n    {\n        self.deserialize_any(visitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// This trait represents a visitor that walks through a deserializer.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the requirement for lifetime of data\n/// that may be borrowed by `Self::Value`. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example\n///\n/// ```edition2021\n/// # use serde::de::{self, Unexpected, Visitor};\n/// # use std::fmt;\n/// #\n/// /// A visitor that deserializes a long string - a string containing at least\n/// /// some minimum number of bytes.\n/// struct LongString {\n///     min: usize,\n/// }\n///\n/// impl<'de> Visitor<'de> for LongString {\n///     type Value = String;\n///\n///     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///         write!(formatter, \"a string containing at least {} bytes\", self.min)\n///     }\n///\n///     fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>\n///     where\n///         E: de::Error,\n///     {\n///         if s.len() >= self.min {\n///             Ok(s.to_owned())\n///         } else {\n///             Err(de::Error::invalid_value(Unexpected::Str(s), &self))\n///         }\n///     }\n/// }\n/// ```\npub trait Visitor<'de>: Sized {\n    /// The value produced by this visitor.\n    type Value;\n\n    /// Format a message stating what data this Visitor expects to receive.\n    ///\n    /// This is used in error messages. The message should complete the sentence\n    /// \"This Visitor expects to receive ...\", for example the message could be\n    /// \"an integer between 0 and 64\". The message should not be capitalized and\n    /// should not end with a period.\n    ///\n    /// ```edition2021\n    /// # use std::fmt;\n    /// #\n    /// # struct S {\n    /// #     max: usize,\n    /// # }\n    /// #\n    /// # impl<'de> serde::de::Visitor<'de> for S {\n    /// #     type Value = ();\n    /// #\n    /// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n    ///     write!(formatter, \"an integer between 0 and {}\", self.max)\n    /// }\n    /// # }\n    /// ```\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result;\n\n    /// The input contains a boolean.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Bool(v), &self))\n    }\n\n    /// The input contains an `i8`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i8<E>(self, v: i8) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i16`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i16<E>(self, v: i16) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i32`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i32<E>(self, v: i32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Signed(v), &self))\n    }\n\n    /// The input contains a `i128`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_i128<E>(self, v: i128) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut buf = [0u8; 58];\n        let mut writer = format::Buf::new(&mut buf);\n        fmt::Write::write_fmt(&mut writer, format_args!(\"integer `{}` as i128\", v)).unwrap();\n        Err(Error::invalid_type(\n            Unexpected::Other(writer.as_str()),\n            &self,\n        ))\n    }\n\n    /// The input contains a `u8`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u8<E>(self, v: u8) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u16`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u32`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u32<E>(self, v: u32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Unsigned(v), &self))\n    }\n\n    /// The input contains a `u128`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_u128<E>(self, v: u128) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut buf = [0u8; 57];\n        let mut writer = format::Buf::new(&mut buf);\n        fmt::Write::write_fmt(&mut writer, format_args!(\"integer `{}` as u128\", v)).unwrap();\n        Err(Error::invalid_type(\n            Unexpected::Other(writer.as_str()),\n            &self,\n        ))\n    }\n\n    /// The input contains an `f32`.\n    ///\n    /// The default implementation forwards to [`visit_f64`].\n    ///\n    /// [`visit_f64`]: #method.visit_f64\n    fn visit_f32<E>(self, v: f32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_f64(v as f64)\n    }\n\n    /// The input contains an `f64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Float(v), &self))\n    }\n\n    /// The input contains a `char`.\n    ///\n    /// The default implementation forwards to [`visit_str`] as a one-character\n    /// string.\n    ///\n    /// [`visit_str`]: #method.visit_str\n    #[inline]\n    fn visit_char<E>(self, v: char) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(v.encode_utf8(&mut [0u8; 4]))\n    }\n\n    /// The input contains a string. The lifetime of the string is ephemeral and\n    /// it may be destroyed after this method returns.\n    ///\n    /// This method allows the `Deserializer` to avoid a copy by retaining\n    /// ownership of any buffered data. `Deserialize` implementations that do\n    /// not benefit from taking ownership of `String` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_str` rather than\n    /// `Deserializer::deserialize_string`.\n    ///\n    /// It is never correct to implement `visit_string` without implementing\n    /// `visit_str`. Implement neither, both, or just `visit_str`.\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Str(v), &self))\n    }\n\n    /// The input contains a string that lives at least as long as the\n    /// `Deserializer`.\n    ///\n    /// This enables zero-copy deserialization of strings in some formats. For\n    /// example JSON input containing the JSON string `\"borrowed\"` can be\n    /// deserialized with zero copying into a `&'a str` as long as the input\n    /// data outlives `'a`.\n    ///\n    /// The default implementation forwards to `visit_str`.\n    #[inline]\n    fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(v)\n    }\n\n    /// The input contains a string and ownership of the string is being given\n    /// to the `Visitor`.\n    ///\n    /// This method allows the `Visitor` to avoid a copy by taking ownership of\n    /// a string created by the `Deserializer`. `Deserialize` implementations\n    /// that benefit from taking ownership of `String` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_string` rather\n    /// than `Deserializer::deserialize_str`, although not every deserializer\n    /// will honor such a request.\n    ///\n    /// It is never correct to implement `visit_string` without implementing\n    /// `visit_str`. Implement neither, both, or just `visit_str`.\n    ///\n    /// The default implementation forwards to `visit_str` and then drops the\n    /// `String`.\n    #[inline]\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(&v)\n    }\n\n    /// The input contains a byte array. The lifetime of the byte array is\n    /// ephemeral and it may be destroyed after this method returns.\n    ///\n    /// This method allows the `Deserializer` to avoid a copy by retaining\n    /// ownership of any buffered data. `Deserialize` implementations that do\n    /// not benefit from taking ownership of `Vec<u8>` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_bytes` rather\n    /// than `Deserializer::deserialize_byte_buf`.\n    ///\n    /// It is never correct to implement `visit_byte_buf` without implementing\n    /// `visit_bytes`. Implement neither, both, or just `visit_bytes`.\n    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Bytes(v), &self))\n    }\n\n    /// The input contains a byte array that lives at least as long as the\n    /// `Deserializer`.\n    ///\n    /// This enables zero-copy deserialization of bytes in some formats. For\n    /// example Postcard data containing bytes can be deserialized with zero\n    /// copying into a `&'a [u8]` as long as the input data outlives `'a`.\n    ///\n    /// The default implementation forwards to `visit_bytes`.\n    #[inline]\n    fn visit_borrowed_bytes<E>(self, v: &'de [u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_bytes(v)\n    }\n\n    /// The input contains a byte array and ownership of the byte array is being\n    /// given to the `Visitor`.\n    ///\n    /// This method allows the `Visitor` to avoid a copy by taking ownership of\n    /// a byte buffer created by the `Deserializer`. `Deserialize`\n    /// implementations that benefit from taking ownership of `Vec<u8>` data\n    /// should indicate that to the deserializer by using\n    /// `Deserializer::deserialize_byte_buf` rather than\n    /// `Deserializer::deserialize_bytes`, although not every deserializer will\n    /// honor such a request.\n    ///\n    /// It is never correct to implement `visit_byte_buf` without implementing\n    /// `visit_bytes`. Implement neither, both, or just `visit_bytes`.\n    ///\n    /// The default implementation forwards to `visit_bytes` and then drops the\n    /// `Vec<u8>`.\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_bytes(&v)\n    }\n\n    /// The input contains an optional that is absent.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_none<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Option, &self))\n    }\n\n    /// The input contains an optional that is present.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let _ = deserializer;\n        Err(Error::invalid_type(Unexpected::Option, &self))\n    }\n\n    /// The input contains a unit `()`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Unit, &self))\n    }\n\n    /// The input contains a newtype struct.\n    ///\n    /// The content of the newtype struct may be read from the given\n    /// `Deserializer`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let _ = deserializer;\n        Err(Error::invalid_type(Unexpected::NewtypeStruct, &self))\n    }\n\n    /// The input contains a sequence of elements.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>\n    where\n        A: SeqAccess<'de>,\n    {\n        let _ = seq;\n        Err(Error::invalid_type(Unexpected::Seq, &self))\n    }\n\n    /// The input contains a key-value map.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>\n    where\n        A: MapAccess<'de>,\n    {\n        let _ = map;\n        Err(Error::invalid_type(Unexpected::Map, &self))\n    }\n\n    /// The input contains an enum.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>\n    where\n        A: EnumAccess<'de>,\n    {\n        let _ = data;\n        Err(Error::invalid_type(Unexpected::Enum, &self))\n    }\n\n    // Used when deserializing a flattened Option field. Not public API.\n    #[doc(hidden)]\n    fn __private_visit_untagged_option<D>(self, _: D) -> Result<Self::Value, ()>\n    where\n        D: Deserializer<'de>,\n    {\n        Err(())\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Provides a `Visitor` access to each element of a sequence in the input.\n///\n/// This is a trait that a `Deserializer` passes to a `Visitor` implementation,\n/// which deserializes each item in a sequence.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by deserialized sequence elements. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SeqAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SeqAccess<'de> {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// This returns `Ok(Some(value))` for the next value in the sequence, or\n    /// `Ok(None)` if there are no more remaining items.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `SeqAccess::next_element` instead.\n    fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>\n    where\n        T: DeserializeSeed<'de>;\n\n    /// This returns `Ok(Some(value))` for the next value in the sequence, or\n    /// `Ok(None)` if there are no more remaining items.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `SeqAccess` implementations should not override the default behavior.\n    #[inline]\n    fn next_element<T>(&mut self) -> Result<Option<T>, Self::Error>\n    where\n        T: Deserialize<'de>,\n    {\n        self.next_element_seed(PhantomData)\n    }\n\n    /// Returns the number of elements remaining in the sequence, if known.\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        None\n    }\n}\n\nimpl<'de, 'a, A> SeqAccess<'de> for &'a mut A\nwhere\n    A: ?Sized + SeqAccess<'de>,\n{\n    type Error = A::Error;\n\n    #[inline]\n    fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>\n    where\n        T: DeserializeSeed<'de>,\n    {\n        (**self).next_element_seed(seed)\n    }\n\n    #[inline]\n    fn next_element<T>(&mut self) -> Result<Option<T>, Self::Error>\n    where\n        T: Deserialize<'de>,\n    {\n        (**self).next_element()\n    }\n\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        (**self).size_hint()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Provides a `Visitor` access to each entry of a map in the input.\n///\n/// This is a trait that a `Deserializer` passes to a `Visitor` implementation.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by deserialized map entries. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `MapAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait MapAccess<'de> {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// This returns `Ok(Some(key))` for the next key in the map, or `Ok(None)`\n    /// if there are no more remaining entries.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `MapAccess::next_key` or `MapAccess::next_entry` instead.\n    fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>\n    where\n        K: DeserializeSeed<'de>;\n\n    /// This returns a `Ok(value)` for the next value in the map.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `MapAccess::next_value` instead.\n    ///\n    /// # Panics\n    ///\n    /// Calling `next_value_seed` before `next_key_seed` is incorrect and is\n    /// allowed to panic or return bogus results.\n    fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>\n    where\n        V: DeserializeSeed<'de>;\n\n    /// This returns `Ok(Some((key, value)))` for the next (key-value) pair in\n    /// the map, or `Ok(None)` if there are no more remaining items.\n    ///\n    /// `MapAccess` implementations should override the default behavior if a\n    /// more efficient implementation is possible.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `MapAccess::next_entry` instead.\n    #[inline]\n    fn next_entry_seed<K, V>(\n        &mut self,\n        kseed: K,\n        vseed: V,\n    ) -> Result<Option<(K::Value, V::Value)>, Self::Error>\n    where\n        K: DeserializeSeed<'de>,\n        V: DeserializeSeed<'de>,\n    {\n        match tri!(self.next_key_seed(kseed)) {\n            Some(key) => {\n                let value = tri!(self.next_value_seed(vseed));\n                Ok(Some((key, value)))\n            }\n            None => Ok(None),\n        }\n    }\n\n    /// This returns `Ok(Some(key))` for the next key in the map, or `Ok(None)`\n    /// if there are no more remaining entries.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `MapAccess` implementations should not override the default behavior.\n    #[inline]\n    fn next_key<K>(&mut self) -> Result<Option<K>, Self::Error>\n    where\n        K: Deserialize<'de>,\n    {\n        self.next_key_seed(PhantomData)\n    }\n\n    /// This returns a `Ok(value)` for the next value in the map.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `MapAccess` implementations should not override the default behavior.\n    ///\n    /// # Panics\n    ///\n    /// Calling `next_value` before `next_key` is incorrect and is allowed to\n    /// panic or return bogus results.\n    #[inline]\n    fn next_value<V>(&mut self) -> Result<V, Self::Error>\n    where\n        V: Deserialize<'de>,\n    {\n        self.next_value_seed(PhantomData)\n    }\n\n    /// This returns `Ok(Some((key, value)))` for the next (key-value) pair in\n    /// the map, or `Ok(None)` if there are no more remaining items.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `MapAccess` implementations should not override the default behavior.\n    #[inline]\n    fn next_entry<K, V>(&mut self) -> Result<Option<(K, V)>, Self::Error>\n    where\n        K: Deserialize<'de>,\n        V: Deserialize<'de>,\n    {\n        self.next_entry_seed(PhantomData, PhantomData)\n    }\n\n    /// Returns the number of entries remaining in the map, if known.\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        None\n    }\n}\n\nimpl<'de, 'a, A> MapAccess<'de> for &'a mut A\nwhere\n    A: ?Sized + MapAccess<'de>,\n{\n    type Error = A::Error;\n\n    #[inline]\n    fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>\n    where\n        K: DeserializeSeed<'de>,\n    {\n        (**self).next_key_seed(seed)\n    }\n\n    #[inline]\n    fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>\n    where\n        V: DeserializeSeed<'de>,\n    {\n        (**self).next_value_seed(seed)\n    }\n\n    #[inline]\n    fn next_entry_seed<K, V>(\n        &mut self,\n        kseed: K,\n        vseed: V,\n    ) -> Result<Option<(K::Value, V::Value)>, Self::Error>\n    where\n        K: DeserializeSeed<'de>,\n        V: DeserializeSeed<'de>,\n    {\n        (**self).next_entry_seed(kseed, vseed)\n    }\n\n    #[inline]\n    fn next_entry<K, V>(&mut self) -> Result<Option<(K, V)>, Self::Error>\n    where\n        K: Deserialize<'de>,\n        V: Deserialize<'de>,\n    {\n        (**self).next_entry()\n    }\n\n    #[inline]\n    fn next_key<K>(&mut self) -> Result<Option<K>, Self::Error>\n    where\n        K: Deserialize<'de>,\n    {\n        (**self).next_key()\n    }\n\n    #[inline]\n    fn next_value<V>(&mut self) -> Result<V, Self::Error>\n    where\n        V: Deserialize<'de>,\n    {\n        (**self).next_value()\n    }\n\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        (**self).size_hint()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Provides a `Visitor` access to the data of an enum in the input.\n///\n/// `EnumAccess` is created by the `Deserializer` and passed to the\n/// `Visitor` in order to identify which variant of an enum to deserialize.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by the deserialized enum variant. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `EnumAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait EnumAccess<'de>: Sized {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n    /// The `Visitor` that will be used to deserialize the content of the enum\n    /// variant.\n    type Variant: VariantAccess<'de, Error = Self::Error>;\n\n    /// `variant` is called to identify which variant to deserialize.\n    ///\n    /// `Deserialize` implementations should typically use `EnumAccess::variant`\n    /// instead.\n    fn variant_seed<V>(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error>\n    where\n        V: DeserializeSeed<'de>;\n\n    /// `variant` is called to identify which variant to deserialize.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `EnumAccess` implementations should not override the default behavior.\n    #[inline]\n    fn variant<V>(self) -> Result<(V, Self::Variant), Self::Error>\n    where\n        V: Deserialize<'de>,\n    {\n        self.variant_seed(PhantomData)\n    }\n}\n\n/// `VariantAccess` is a visitor that is created by the `Deserializer` and\n/// passed to the `Deserialize` to deserialize the content of a particular enum\n/// variant.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by the deserialized enum variant. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `VariantAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait VariantAccess<'de>: Sized {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization. Must match the error type of our `EnumAccess`.\n    type Error: Error;\n\n    /// Called when deserializing a variant with no values.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// fn unit_variant(self) -> Result<(), Self::Error> {\n    ///     // What the data actually contained; suppose it is a tuple variant.\n    ///     let unexp = Unexpected::TupleVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"unit variant\"))\n    /// }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn unit_variant(self) -> Result<(), Self::Error>;\n\n    /// Called when deserializing a variant with a single value.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `VariantAccess::newtype_variant` instead.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// fn newtype_variant_seed<T>(self, _seed: T) -> Result<T::Value, Self::Error>\n    /// where\n    ///     T: DeserializeSeed<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"newtype variant\"))\n    /// }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value, Self::Error>\n    where\n        T: DeserializeSeed<'de>;\n\n    /// Called when deserializing a variant with a single value.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `VariantAccess` implementations should not override the default\n    /// behavior.\n    #[inline]\n    fn newtype_variant<T>(self) -> Result<T, Self::Error>\n    where\n        T: Deserialize<'de>,\n    {\n        self.newtype_variant_seed(PhantomData)\n    }\n\n    /// Called when deserializing a tuple-like variant.\n    ///\n    /// The `len` is the number of fields expected in the tuple variant.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// fn tuple_variant<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>\n    /// where\n    ///     V: Visitor<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"tuple variant\"))\n    /// }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn tuple_variant<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Called when deserializing a struct-like variant.\n    ///\n    /// The `fields` are the names of the fields of the struct variant.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// fn struct_variant<V>(\n    ///     self,\n    ///     _fields: &'static [&'static str],\n    ///     _visitor: V,\n    /// ) -> Result<V::Value, Self::Error>\n    /// where\n    ///     V: Visitor<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"struct variant\"))\n    /// }\n    /// # }\n    /// ```\n    fn struct_variant<V>(\n        self,\n        fields: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Converts an existing value into a `Deserializer` from which other values can\n/// be deserialized.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed from the resulting `Deserializer`. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example\n///\n/// ```edition2021\n/// use serde::de::{value, Deserialize, IntoDeserializer};\n/// use serde_derive::Deserialize;\n/// use std::str::FromStr;\n///\n/// #[derive(Deserialize)]\n/// enum Setting {\n///     On,\n///     Off,\n/// }\n///\n/// impl FromStr for Setting {\n///     type Err = value::Error;\n///\n///     fn from_str(s: &str) -> Result<Self, Self::Err> {\n///         Self::deserialize(s.into_deserializer())\n///     }\n/// }\n/// ```\npub trait IntoDeserializer<'de, E: Error = value::Error> {\n    /// The type of the deserializer being converted into.\n    type Deserializer: Deserializer<'de, Error = E>;\n\n    /// Convert this value into a deserializer.\n    fn into_deserializer(self) -> Self::Deserializer;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Used in error messages.\n///\n/// - expected `a`\n/// - expected `a` or `b`\n/// - expected one of `a`, `b`, `c`\n///\n/// The slice of names must not be empty.\nstruct OneOf {\n    names: &'static [&'static str],\n...\n", "needle_token_start": 5634, "needle_token_end": 5736, "code_context_ntokens": 16246, "output": ["```rust\nfn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\nwhere\n    V: Visitor<'de>,\n{\n    // Implementation of deserialize_any method\n    // This method is used to deserialize any type of content into a generic data structure\n    // It takes a deserializer instance and a visitor that can handle the deserialization of any content\n    // It returns a generic data structure that represents the deserialized content or an error if the deserialization fails\n    // The method delegates the task of deserialization to a method that handles any type of data, using the provided visitor to interpret the incoming data into a generic format\n}\n```"]}
{"repo": "serde-rs/serde", "name": "format_u8", "language": "rust", "path": "serde/src/ser/impls.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: Converts a single-byte unsigned integer into its decimal string representation and writes it into a provided byte array.\n2. **Input**: A single-byte unsigned integer and a mutable byte slice to store the resulting characters.\n3. **Output**: The number of bytes written to the output slice, representing the length of the string.\n4. **Procedure**: The function checks if the input number is greater than or equal to 100, between 10 and 99, or less than 10, and accordingly converts it into one, two, or three characters using precomputed lookup tables for decimal digits. These characters are then stored in the provided byte slice.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Some(ref value) => serializer.serialize_some(value),\n            None => serializer.serialize_none(),\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for PhantomData<T>\nwhere\n    T: ?Sized,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_unit_struct(\"PhantomData\")\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// Does not require T: Serialize.\nimpl<T> Serialize for [T; 0] {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        tri!(serializer.serialize_tuple(0)).end()\n    }\n}\n\nmacro_rules! array_impls {\n    ($($len:tt)+) => {\n        $(\n            impl<T> Serialize for [T; $len]\n            where\n                T: Serialize,\n            {\n                #[inline]\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    let mut seq = tri!(serializer.serialize_tuple($len));\n                    for e in self {\n                        tri!(seq.serialize_element(e));\n                    }\n                    seq.end()\n                }\n            }\n        )+\n    }\n}\n\narray_impls! {\n    01 02 03 04 05 06 07 08 09 10\n    11 12 13 14 15 16 17 18 19 20\n    21 22 23 24 25 26 27 28 29 30\n    31 32\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for [T]\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.collect_seq(self)\n    }\n}\n\n#[cfg(not(no_relaxed_trait_bounds))]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize,\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\n#[cfg(no_relaxed_trait_bounds)]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize $(+ $tbound1 $(+ $tbound2)*)*,\n            $($typaram: $bound,)*\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BinaryHeap<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeSet<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashSet<T: Eq + Hash, H: BuildHasher>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    LinkedList<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    Vec<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    VecDeque<T>\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for Range<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"Range\", 2));\n        tri!(state.serialize_field(\"start\", &self.start));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeFrom<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeFrom\", 1));\n        tri!(state.serialize_field(\"start\", &self.start));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeInclusive<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeInclusive\", 2));\n        tri!(state.serialize_field(\"start\", &self.start()));\n        tri!(state.serialize_field(\"end\", &self.end()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeTo<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeTo\", 1));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for Bound<T>\nwhere\n    T: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Bound::Unbounded => serializer.serialize_unit_variant(\"Bound\", 0, \"Unbounded\"),\n            Bound::Included(ref value) => {\n                serializer.serialize_newtype_variant(\"Bound\", 1, \"Included\", value)\n            }\n            Bound::Excluded(ref value) => {\n                serializer.serialize_newtype_variant(\"Bound\", 2, \"Excluded\", value)\n            }\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for () {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_unit()\n    }\n}\n\n#[cfg(feature = \"unstable\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"unstable\")))]\nimpl Serialize for ! {\n    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        *self\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! tuple_impls {\n    ($($len:expr => ($($n:tt $name:ident)+))+) => {\n        $(\n            impl<$($name),+> Serialize for ($($name,)+)\n            where\n                $($name: Serialize,)+\n            {\n                #[inline]\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    let mut tuple = tri!(serializer.serialize_tuple($len));\n                    $(\n                        tri!(tuple.serialize_element(&self.$n));\n                    )+\n                    tuple.end()\n                }\n            }\n        )+\n    }\n}\n\ntuple_impls! {\n    1 => (0 T0)\n    2 => (0 T0 1 T1)\n    3 => (0 T0 1 T1 2 T2)\n    4 => (0 T0 1 T1 2 T2 3 T3)\n    5 => (0 T0 1 T1 2 T2 3 T3 4 T4)\n    6 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5)\n    7 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6)\n    8 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7)\n    9 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8)\n    10 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9)\n    11 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10)\n    12 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11)\n    13 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12)\n    14 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13)\n    15 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14)\n    16 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14 15 T15)\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(not(no_relaxed_trait_bounds))]\nmacro_rules! map_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <K $(: $kbound1:ident $(+ $kbound2:ident)*)*, V $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<K, V $(, $typaram)*> Serialize for $ty<K, V $(, $typaram)*>\n        where\n            K: Serialize,\n            V: Serialize,\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_map(self)\n            }\n        }\n    }\n}\n\n#[cfg(no_relaxed_trait_bounds)]\nmacro_rules! map_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <K $(: $kbound1:ident $(+ $kbound2:ident)*)*, V $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<K, V $(, $typaram)*> Serialize for $ty<K, V $(, $typaram)*>\n        where\n            K: Serialize $(+ $kbound1 $(+ $kbound2)*)*,\n            V: Serialize,\n            $($typaram: $bound,)*\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_map(self)\n            }\n        }\n    }\n}\n\nmap_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeMap<K: Ord, V>\n}\n\nmap_impl! {\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashMap<K: Eq + Hash, V, H: BuildHasher>\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! deref_impl {\n    (\n        $(#[$attr:meta])*\n        <$($desc:tt)+\n    ) => {\n        $(#[$attr])*\n        impl <$($desc)+ {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                (**self).serialize(serializer)\n            }\n        }\n    };\n}\n\nderef_impl! {\n    <'a, T> Serialize for &'a T where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    <'a, T> Serialize for &'a mut T where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    <T> Serialize for Box<T> where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    /// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n    ///\n    /// Serializing a data structure containing `Rc` will serialize a copy of\n    /// the contents of the `Rc` each time the `Rc` is referenced within the\n    /// data structure. Serialization will not attempt to deduplicate these\n    /// repeated data.\n    ///\n    /// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n    #[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n    #[cfg_attr(doc_cfg, doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))))]\n    <T> Serialize for Rc<T> where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    /// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n    ///\n    /// Serializing a data structure containing `Arc` will serialize a copy of\n    /// the contents of the `Arc` each time the `Arc` is referenced within the\n    /// data structure. Serialization will not attempt to deduplicate these\n    /// repeated data.\n    ///\n    /// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n    #[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n    #[cfg_attr(doc_cfg, doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))))]\n    <T> Serialize for Arc<T> where T: ?Sized + Serialize\n}\n\nderef_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    <'a, T> Serialize for Cow<'a, T> where T: ?Sized + Serialize + ToOwned\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n///\n/// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n#[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n#[cfg_attr(\n    doc_cfg,\n    doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\"))))\n)]\nimpl<T> Serialize for RcWeak<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.upgrade().serialize(serializer)\n    }\n}\n\n/// This impl requires the [`\"rc\"`] Cargo feature of Serde.\n///\n/// [`\"rc\"`]: https://serde.rs/feature-flags.html#-features-rc\n#[cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\")))]\n#[cfg_attr(\n    doc_cfg,\n    doc(cfg(all(feature = \"rc\", any(feature = \"std\", feature = \"alloc\"))))\n)]\nimpl<T> Serialize for ArcWeak<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.upgrade().serialize(serializer)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! nonzero_integers {\n    ($($T:ident,)+) => {\n        $(\n            impl Serialize for num::$T {\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    self.get().serialize(serializer)\n                }\n            }\n        )+\n    }\n}\n\nnonzero_integers! {\n    NonZeroU8,\n    NonZeroU16,\n    NonZeroU32,\n    NonZeroU64,\n    NonZeroU128,\n    NonZeroUsize,\n}\n\n#[cfg(not(no_num_nonzero_signed))]\nnonzero_integers! {\n    NonZeroI8,\n    NonZeroI16,\n    NonZeroI32,\n    NonZeroI64,\n    NonZeroI128,\n    NonZeroIsize,\n}\n\nimpl<T> Serialize for Cell<T>\nwhere\n    T: Serialize + Copy,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.get().serialize(serializer)\n    }\n}\n\nimpl<T> Serialize for RefCell<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.try_borrow() {\n            Ok(value) => value.serialize(serializer),\n            Err(_) => Err(S::Error::custom(\"already mutably borrowed\")),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl<T> Serialize for Mutex<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.lock() {\n            Ok(locked) => locked.serialize(serializer),\n            Err(_) => Err(S::Error::custom(\"lock poison error while serializing\")),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl<T> Serialize for RwLock<T>\nwhere\n    T: ?Sized + Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.read() {\n            Ok(locked) => locked.serialize(serializer),\n            Err(_) => Err(S::Error::custom(\"lock poison error while serializing\")),\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T, E> Serialize for Result<T, E>\nwhere\n    T: Serialize,\n    E: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Result::Ok(ref value) => serializer.serialize_newtype_variant(\"Result\", 0, \"Ok\", value),\n            Result::Err(ref value) => {\n                serializer.serialize_newtype_variant(\"Result\", 1, \"Err\", value)\n            }\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for Duration {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"Duration\", 2));\n        tri!(state.serialize_field(\"secs\", &self.as_secs()));\n        tri!(state.serialize_field(\"nanos\", &self.subsec_nanos()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for SystemTime {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let duration_since_epoch = match self.duration_since(UNIX_EPOCH) {\n            Ok(duration_since_epoch) => duration_since_epoch,\n            Err(_) => return Err(S::Error::custom(\"SystemTime must be later than UNIX_EPOCH\")),\n        };\n        let mut state = tri!(serializer.serialize_struct(\"SystemTime\", 2));\n        tri!(state.serialize_field(\"secs_since_epoch\", &duration_since_epoch.as_secs()));\n        tri!(state.serialize_field(\"nanos_since_epoch\", &duration_since_epoch.subsec_nanos()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Serialize a value that implements `Display` as a string, when that string is\n/// statically known to never have more than a constant `MAX_LEN` bytes.\n///\n/// Panics if the `Display` impl tries to write more than `MAX_LEN` bytes.\n#[cfg(feature = \"std\")]\nmacro_rules! serialize_display_bounded_length {\n    ($value:expr, $max:expr, $serializer:expr) => {{\n        let mut buffer = [0u8; $max];\n        let remaining_len = {\n            let mut remaining = &mut buffer[..];\n            write!(remaining, \"{}\", $value).unwrap();\n            remaining.len()\n        };\n        let written_len = buffer.len() - remaining_len;\n        let written = &buffer[..written_len];\n\n        // write! only provides fmt::Formatter to Display implementations, which\n        // has methods write_str and write_char but no method to write arbitrary\n        // bytes. Therefore `written` must be valid UTF-8.\n        let written_str = str::from_utf8(written).expect(\"must be valid UTF-8\");\n        $serializer.serialize_str(written_str)\n    }};\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::IpAddr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            match *self {\n                net::IpAddr::V4(ref a) => a.serialize(serializer),\n                net::IpAddr::V6(ref a) => a.serialize(serializer),\n            }\n        } else {\n            match *self {\n                net::IpAddr::V4(ref a) => {\n                    serializer.serialize_newtype_variant(\"IpAddr\", 0, \"V4\", a)\n                }\n                net::IpAddr::V6(ref a) => {\n                    serializer.serialize_newtype_variant(\"IpAddr\", 1, \"V6\", a)\n                }\n            }\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nconst DEC_DIGITS_LUT: &[u8] = b\"\\\n      0001020304050607080910111213141516171819\\\n      2021222324252627282930313233343536373839\\\n      4041424344454647484950515253545556575859\\\n      6061626364656667686970717273747576777879\\\n      8081828384858687888990919293949596979899\";\n\n#[cfg(feature = \"std\")]\n#[inline]\n\nfn format_u8(mut n: u8, out: &mut [u8]) -> usize {\n    if n >= 100 {\n        let d1 = ((n % 100) << 1) as usize;\n        n /= 100;\n        out[0] = b'0' + n;\n        out[1] = DEC_DIGITS_LUT[d1];\n        out[2] = DEC_DIGITS_LUT[d1 + 1];\n        3\n    } else if n >= 10 {\n        let d1 = (n << 1) as usize;\n        out[0] = DEC_DIGITS_LUT[d1];\n        out[1] = DEC_DIGITS_LUT[d1 + 1];\n        2\n    } else {\n        out[0] = b'0' + n;\n        1\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[test]\nfn test_format_u8() {\n    let mut i = 0u8;\n\n    loop {\n        let mut buf = [0u8; 3];\n        let written = format_u8(i, &mut buf);\n        assert_eq!(i.to_string().as_bytes(), &buf[..written]);\n\n        match i.checked_add(1) {\n            Some(next) => i = next,\n            None => break,\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::Ipv4Addr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 15;\n            debug_assert_eq!(MAX_LEN, \"101.102.103.104\".len());\n            let mut buf = [b'.'; MAX_LEN];\n            let mut written = format_u8(self.octets()[0], &mut buf);\n            for oct in &self.octets()[1..] {\n                // Skip over delimiters that we initialized buf with\n                written += format_u8(*oct, &mut buf[written + 1..]) + 1;\n            }\n            // Safety: We've only written ASCII bytes to the buffer, so it is valid UTF-8\n            let buf = unsafe { str::from_utf8_unchecked(&buf[..written]) };\n            serializer.serialize_str(buf)\n        } else {\n            self.octets().serialize(serializer)\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::Ipv6Addr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 39;\n            debug_assert_eq!(MAX_LEN, \"1001:1002:1003:1004:1005:1006:1007:1008\".len());\n            serialize_display_bounded_length!(self, MAX_LEN, serializer)\n        } else {\n            self.octets().serialize(serializer)\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::SocketAddr {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            match *self {\n                net::SocketAddr::V4(ref addr) => addr.serialize(serializer),\n                net::SocketAddr::V6(ref addr) => addr.serialize(serializer),\n            }\n        } else {\n            match *self {\n                net::SocketAddr::V4(ref addr) => {\n                    serializer.serialize_newtype_variant(\"SocketAddr\", 0, \"V4\", addr)\n                }\n                net::SocketAddr::V6(ref addr) => {\n                    serializer.serialize_newtype_variant(\"SocketAddr\", 1, \"V6\", addr)\n                }\n            }\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::SocketAddrV4 {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 21;\n            debug_assert_eq!(MAX_LEN, \"101.102.103.104:65000\".len());\n            serialize_display_bounded_length!(self, MAX_LEN, serializer)\n        } else {\n            (self.ip(), self.port()).serialize(serializer)\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for net::SocketAddrV6 {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        if serializer.is_human_readable() {\n            const MAX_LEN: usize = 58;\n            debug_assert_eq!(\n                MAX_LEN,\n                \"[1001:1002:1003:1004:1005:1006:1007:1008%4294967295]:65000\".len()\n            );\n            serialize_display_bounded_length!(self, MAX_LEN, serializer)\n        } else {\n            (self.ip(), self.port()).serialize(serializer)\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for Path {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match self.to_str() {\n            Some(s) => s.serialize(serializer),\n            None => Err(Error::custom(\"path contains invalid UTF-8 characters\")),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for PathBuf {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.as_path().serialize(serializer)\n    }\n}\n\n#[cfg(all(feature = \"std\", any(unix, windows)))]\n#[cfg_attr(doc_cfg, doc(cfg(all(feature = \"std\", any(unix, windows)))))]\nimpl Serialize for OsStr {\n    #[cfg(unix)]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use std::os::unix::ffi::OsStrExt;\n        serializer.serialize_newtype_variant(\"OsString\", 0, \"Unix\", self.as_bytes())\n    }\n\n    #[cfg(windows)]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use std::os::windows::ffi::OsStrExt;\n        let val = self.encode_wide().collect::<Vec<_>>();\n        serializer.serialize_newtype_variant(\"OsString\", 1, \"Windows\", &val)\n    }\n}\n\n#[cfg(all(feature = \"std\", any(unix, windows)))]\n#[cfg_attr(doc_cfg, doc(cfg(all(feature = \"std\", any(unix, windows)))))]\nimpl Serialize for OsString {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.as_os_str().serialize(serializer)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for Wrapping<T>\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.0.serialize(serializer)\n    }\n}\n\nimpl<T> Serialize for Reverse<T>\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        self.0.serialize(serializer)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(all(feature = \"std\", not(no_std_atomic)))]\nmacro_rules! atomic_impl {\n    ($($ty:ident $size:expr)*) => {\n        $(\n            #[cfg(any(no_target_has_atomic, target_has_atomic = $size))]\n            #[cfg_attr(doc_cfg, doc(cfg(all(feature = \"std\", target_has_atomic = $size))))]\n            impl Serialize for $ty {\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    // Matches the atomic ordering used in libcore for the Debug impl\n                    self.load(Ordering::Relaxed).serialize(serializer)\n                }\n            }\n        )*\n    }\n}\n\n#[cfg(all(feature = \"std\", not(no_std_atomic)))]\natomic_impl! {\n    AtomicBool \"8\"\n    AtomicI8 \"8\"\n    AtomicI16 \"16\"\n    AtomicI32 \"32\"\n    AtomicIsize \"ptr\"\n    AtomicU8 \"8\"\n    AtomicU16 \"16\"\n    AtomicU32 \"32\"\n    AtomicUsize \"ptr\"\n}\n\n#[cfg(all(feature = \"std\", not(no_std_atomic64)))]\natomic_impl! {\n    AtomicI64 \"64\"\n    AtomicU64 \"64\"\n}\n\n// Path: serde/src/ser/mod.rs\n//! Generic data structure serialization framework.\n//!\n//! The two most important traits in this module are [`Serialize`] and\n//! [`Serializer`].\n//!\n//!  - **A type that implements `Serialize` is a data structure** that can be\n//!    serialized to any data format supported by Serde, and conversely\n//!  - **A type that implements `Serializer` is a data format** that can\n//!    serialize any data structure supported by Serde.\n//!\n//! # The Serialize trait\n//!\n//! Serde provides [`Serialize`] implementations for many Rust primitive and\n//! standard library types. The complete list is below. All of these can be\n//! serialized using Serde out of the box.\n//!\n//! Additionally, Serde provides a procedural macro called [`serde_derive`] to\n//! automatically generate [`Serialize`] implementations for structs and enums\n//! in your program. See the [derive section of the manual] for how to use this.\n//!\n//! In rare cases it may be necessary to implement [`Serialize`] manually for\n//! some type in your program. See the [Implementing `Serialize`] section of the\n//! manual for more about this.\n//!\n//! Third-party crates may provide [`Serialize`] implementations for types that\n//! they expose. For example the [`linked-hash-map`] crate provides a\n//! [`LinkedHashMap<K, V>`] type that is serializable by Serde because the crate\n//! provides an implementation of [`Serialize`] for it.\n//!\n//! # The Serializer trait\n//!\n//! [`Serializer`] implementations are provided by third-party crates, for\n//! example [`serde_json`], [`serde_yaml`] and [`postcard`].\n//!\n//! A partial list of well-maintained formats is given on the [Serde\n//! website][data formats].\n//!\n//! # Implementations of Serialize provided by Serde\n//!\n//!  - **Primitive types**:\n//!    - bool\n//!    - i8, i16, i32, i64, i128, isize\n//!    - u8, u16, u32, u64, u128, usize\n//!    - f32, f64\n//!    - char\n//!    - str\n//!    - &T and &mut T\n//!  - **Compound types**:\n//!    - \\[T\\]\n//!    - \\[T; 0\\] through \\[T; 32\\]\n//!    - tuples up to size 16\n//!  - **Common standard library types**:\n//!    - String\n//!    - Option\\<T\\>\n//!    - Result\\<T, E\\>\n//!    - PhantomData\\<T\\>\n//!  - **Wrapper types**:\n//!    - Box\\<T\\>\n//!    - Cow\\<'a, T\\>\n//!    - Cell\\<T\\>\n//!    - RefCell\\<T\\>\n//!    - Mutex\\<T\\>\n//!    - RwLock\\<T\\>\n//!    - Rc\\<T\\>&emsp;*(if* features = \\[\"rc\"\\] *is enabled)*\n//!    - Arc\\<T\\>&emsp;*(if* features = \\[\"rc\"\\] *is enabled)*\n//!  - **Collection types**:\n//!    - BTreeMap\\<K, V\\>\n//!    - BTreeSet\\<T\\>\n//!    - BinaryHeap\\<T\\>\n//!    - HashMap\\<K, V, H\\>\n//!    - HashSet\\<T, H\\>\n//!    - LinkedList\\<T\\>\n//!    - VecDeque\\<T\\>\n//!    - Vec\\<T\\>\n//!  - **FFI types**:\n//!    - CStr\n//!    - CString\n//!    - OsStr\n//!    - OsString\n//!  - **Miscellaneous standard library types**:\n//!    - Duration\n//!    - SystemTime\n//!    - Path\n//!    - PathBuf\n//!    - Range\\<T\\>\n//!    - RangeInclusive\\<T\\>\n//!    - Bound\\<T\\>\n//!    - num::NonZero*\n//!    - `!` *(unstable)*\n//!  - **Net types**:\n//!    - IpAddr\n//!    - Ipv4Addr\n//!    - Ipv6Addr\n//!    - SocketAddr\n//!    - SocketAddrV4\n//!    - SocketAddrV6\n//!\n//! [Implementing `Serialize`]: https://serde.rs/impl-serialize.html\n//! [`LinkedHashMap<K, V>`]: https://docs.rs/linked-hash-map/*/linked_hash_map/struct.LinkedHashMap.html\n//! [`Serialize`]: ../trait.Serialize.html\n//! [`Serializer`]: ../trait.Serializer.html\n//! [`postcard`]: https://github.com/jamesmunns/postcard\n//! [`linked-hash-map`]: https://crates.io/crates/linked-hash-map\n//! [`serde_derive`]: https://crates.io/crates/serde_derive\n//! [`serde_json`]: https://github.com/serde-rs/json\n//! [`serde_yaml`]: https://github.com/dtolnay/serde-yaml\n//! [derive section of the manual]: https://serde.rs/derive.html\n//! [data formats]: https://serde.rs/#data-formats\n\nuse crate::lib::*;\n\nmod fmt;\nmod impls;\nmod impossible;\n\npub use self::impossible::Impossible;\n\n#[cfg(not(any(feature = \"std\", feature = \"unstable\")))]\n#[doc(no_inline)]\npub use crate::std_error::Error as StdError;\n#[cfg(all(feature = \"unstable\", not(feature = \"std\")))]\n#[doc(no_inline)]\npub use core::error::Error as StdError;\n#[cfg(feature = \"std\")]\n#[doc(no_inline)]\npub use std::error::Error as StdError;\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! declare_error_trait {\n    (Error: Sized $(+ $($supertrait:ident)::+)*) => {\n        /// Trait used by `Serialize` implementations to generically construct\n        /// errors belonging to the `Serializer` against which they are\n        /// currently running.\n        ///\n        /// # Example implementation\n        ///\n        /// The [example data format] presented on the website shows an error\n        /// type appropriate for a basic JSON data format.\n        ///\n        /// [example data format]: https://serde.rs/data-format.html\n        pub trait Error: Sized $(+ $($supertrait)::+)* {\n            /// Used when a [`Serialize`] implementation encounters any error\n            /// while serializing a type.\n            ///\n            /// The message should not be capitalized and should not end with a\n            /// period.\n            ///\n            /// For example, a filesystem [`Path`] may refuse to serialize\n            /// itself if it contains invalid UTF-8 data.\n            ///\n            /// ```edition2021\n            /// # struct Path;\n            /// #\n            /// # impl Path {\n            /// #     fn to_str(&self) -> Option<&str> {\n            /// #         unimplemented!()\n            /// #     }\n            /// # }\n            /// #\n            /// use serde::ser::{self, Serialize, Serializer};\n            ///\n            /// impl Serialize for Path {\n            ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            ///     where\n            ///         S: Serializer,\n            ///     {\n            ///         match self.to_str() {\n            ///             Some(s) => serializer.serialize_str(s),\n            ///             None => Err(ser::Error::custom(\"path contains invalid UTF-8 characters\")),\n            ///         }\n            ///     }\n            /// }\n            /// ```\n            ///\n            /// [`Path`]: https://doc.rust-lang.org/std/path/struct.Path.html\n            /// [`Serialize`]: ../trait.Serialize.html\n            fn custom<T>(msg: T) -> Self\n            where\n                T: Display;\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\ndeclare_error_trait!(Error: Sized + StdError);\n\n#[cfg(not(feature = \"std\"))]\ndeclare_error_trait!(Error: Sized + Debug + Display);\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data structure** that can be serialized into any data format supported\n/// by Serde.\n///\n/// Serde provides `Serialize` implementations for many Rust primitive and\n/// standard library types. The complete list is [here][crate::ser]. All of\n/// these can be serialized using Serde out of the box.\n///\n/// Additionally, Serde provides a procedural macro called [`serde_derive`] to\n/// automatically generate `Serialize` implementations for structs and enums in\n/// your program. See the [derive section of the manual] for how to use this.\n///\n/// In rare cases it may be necessary to implement `Serialize` manually for some\n/// type in your program. See the [Implementing `Serialize`] section of the\n/// manual for more about this.\n///\n/// Third-party crates may provide `Serialize` implementations for types that\n/// they expose. For example the [`linked-hash-map`] crate provides a\n/// [`LinkedHashMap<K, V>`] type that is serializable by Serde because the crate\n/// provides an implementation of `Serialize` for it.\n///\n/// [Implementing `Serialize`]: https://serde.rs/impl-serialize.html\n/// [`LinkedHashMap<K, V>`]: https://docs.rs/linked-hash-map/*/linked_hash_map/struct.LinkedHashMap.html\n/// [`linked-hash-map`]: https://crates.io/crates/linked-hash-map\n/// [`serde_derive`]: https://crates.io/crates/serde_derive\n/// [derive section of the manual]: https://serde.rs/derive.html\npub trait Serialize {\n    /// Serialize this value into the given Serde serializer.\n    ///\n    /// See the [Implementing `Serialize`] section of the manual for more\n    /// information about how to implement this method.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeStruct, Serializer};\n    ///\n    /// struct Person {\n    ///     name: String,\n    ///     age: u8,\n    ///     phones: Vec<String>,\n    /// }\n    ///\n    /// // This is what #[derive(Serialize)] would generate.\n    /// impl Serialize for Person {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut s = serializer.serialize_struct(\"Person\", 3)?;\n    ///         s.serialize_field(\"name\", &self.name)?;\n    ///         s.serialize_field(\"age\", &self.age)?;\n    ///         s.serialize_field(\"phones\", &self.phones)?;\n    ///         s.end()\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// [Implementing `Serialize`]: https://serde.rs/impl-serialize.html\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data format** that can serialize any data structure supported by Serde.\n///\n/// The role of this trait is to define the serialization half of the [Serde\n/// data model], which is a way to categorize every Rust data structure into one\n/// of 29 possible types. Each method of the `Serializer` trait corresponds to\n/// one of the types of the data model.\n///\n/// Implementations of `Serialize` map themselves into this data model by\n/// invoking exactly one of the `Serializer` methods.\n///\n/// The types that make up the Serde data model are:\n///\n///  - **14 primitive types**\n///    - bool\n///    - i8, i16, i32, i64, i128\n///    - u8, u16, u32, u64, u128\n///    - f32, f64\n///    - char\n///  - **string**\n///    - UTF-8 bytes with a length and no null terminator.\n///    - When serializing, all strings are handled equally. When deserializing,\n///      there are three flavors of strings: transient, owned, and borrowed.\n///  - **byte array** - \\[u8\\]\n///    - Similar to strings, during deserialization byte arrays can be\n///      transient, owned, or borrowed.\n///  - **option**\n///    - Either none or some value.\n///  - **unit**\n///    - The type of `()` in Rust. It represents an anonymous value containing\n///      no data.\n///  - **unit_struct**\n///    - For example `struct Unit` or `PhantomData<T>`. It represents a named\n///      value containing no data.\n///  - **unit_variant**\n///    - For example the `E::A` and `E::B` in `enum E { A, B }`.\n///  - **newtype_struct**\n///    - For example `struct Millimeters(u8)`.\n///  - **newtype_variant**\n///    - For example the `E::N` in `enum E { N(u8) }`.\n///  - **seq**\n///    - A variably sized heterogeneous sequence of values, for example\n///      `Vec<T>` or `HashSet<T>`. When serializing, the length may or may not\n///      be known before iterating through all the data. When deserializing,\n///      the length is determined by looking at the serialized data.\n///  - **tuple**\n///    - A statically sized heterogeneous sequence of values for which the\n///      length will be known at deserialization time without looking at the\n///      serialized data, for example `(u8,)` or `(String, u64, Vec<T>)` or\n///      `[u64; 10]`.\n///  - **tuple_struct**\n///    - A named tuple, for example `struct Rgb(u8, u8, u8)`.\n///  - **tuple_variant**\n///    - For example the `E::T` in `enum E { T(u8, u8) }`.\n///  - **map**\n///    - A heterogeneous key-value pairing, for example `BTreeMap<K, V>`.\n///  - **struct**\n///    - A heterogeneous key-value pairing in which the keys are strings and\n///      will be known at deserialization time without looking at the\n///      serialized data, for example `struct S { r: u8, g: u8, b: u8 }`.\n///  - **struct_variant**\n///    - For example the `E::S` in `enum E { S { r: u8, g: u8, b: u8 } }`.\n///\n/// Many Serde serializers produce text or binary data as output, for example\n/// JSON or Postcard. This is not a requirement of the `Serializer` trait, and\n/// there are serializers that do not produce text or binary output. One example\n/// is the `serde_json::value::Serializer` (distinct from the main `serde_json`\n/// serializer) that produces a `serde_json::Value` data structure in memory as\n/// output.\n///\n/// [Serde data model]: https://serde.rs/data-model.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website contains example code for\n/// a basic JSON `Serializer`.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait Serializer: Sized {\n    /// The output type produced by this `Serializer` during successful\n    /// serialization. Most serializers that produce text or binary output\n    /// should set `Ok = ()` and serialize into an [`io::Write`] or buffer\n    /// contained within the `Serializer` instance. Serializers that build\n    /// in-memory data structures may be simplified by using `Ok` to propagate\n    /// the data structure around.\n    ///\n    /// [`io::Write`]: https://doc.rust-lang.org/std/io/trait.Write.html\n    type Ok;\n\n    /// The error type when some error occurs during serialization.\n    type Error: Error;\n\n    /// Type returned from [`serialize_seq`] for serializing the content of the\n    /// sequence.\n    ///\n    /// [`serialize_seq`]: #tymethod.serialize_seq\n    type SerializeSeq: SerializeSeq<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple`] for serializing the content of\n    /// the tuple.\n    ///\n    /// [`serialize_tuple`]: #tymethod.serialize_tuple\n    type SerializeTuple: SerializeTuple<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple_struct`] for serializing the\n    /// content of the tuple struct.\n    ///\n    /// [`serialize_tuple_struct`]: #tymethod.serialize_tuple_struct\n    type SerializeTupleStruct: SerializeTupleStruct<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple_variant`] for serializing the\n    /// content of the tuple variant.\n    ///\n    /// [`serialize_tuple_variant`]: #tymethod.serialize_tuple_variant\n    type SerializeTupleVariant: SerializeTupleVariant<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_map`] for serializing the content of the\n    /// map.\n    ///\n    /// [`serialize_map`]: #tymethod.serialize_map\n    type SerializeMap: SerializeMap<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_struct`] for serializing the content of\n    /// the struct.\n    ///\n    /// [`serialize_struct`]: #tymethod.serialize_struct\n    type SerializeStruct: SerializeStruct<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_struct_variant`] for serializing the\n    /// content of the struct variant.\n    ///\n    /// [`serialize_struct_variant`]: #tymethod.serialize_struct_variant\n    type SerializeStructVariant: SerializeStructVariant<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Serialize a `bool` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for bool {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_bool(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i8` value.\n    ///\n    /// If the format does not differentiate between `i8` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i8 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i8(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i16` value.\n    ///\n    /// If the format does not differentiate between `i16` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i16 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i16(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i32` value.\n    ///\n    /// If the format does not differentiate between `i32` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i32 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i32(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i64` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i64 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i64(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i128` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i128 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i128(*self)\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn serialize_i128(self, v: i128) -> Result<Self::Ok, Self::Error> {\n        let _ = v;\n        Err(Error::custom(\"i128 is not supported\"))\n    }\n\n    /// Serialize a `u8` value.\n    ///\n    /// If the format does not differentiate between `u8` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u8 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u8(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u16` value.\n    ///\n    /// If the format does not differentiate between `u16` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n...\n", "needle_token_start": 7159, "needle_token_end": 7389, "code_context_ntokens": 16250, "output": ["```rust\n#[cfg(feature = \"std\")]\n#[inline]\nfn format_u8(mut n: u8, out: &mut [u8]) -> usize {\n    if n >= 100 {\n        let d1 = ((n % 100) << 1) as usize;\n        n /= 100;\n        out[0] = b'0' + n;\n        out[1] = DEC_DIGITS_LUT[d1];\n        out[2] = DEC_DIGITS_LUT[d1 + 1];\n        3\n    } else if n >= 10 {\n        let d1 = (n << 1) as usize;\n        out[0] = DEC_DIGITS_LUT[d1];\n        out[1] = DEC_DIGITS_LUT[d1 + 1];\n        2\n    } else {\n        out[0] = b'0' + n;\n        1\n    }\n}\n```"]}
{"repo": "serde-rs/serde", "name": "serialize_i128", "language": "rust", "path": "serde/src/ser/mod.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To convert a 128-bit integer into a format suitable for storage or transmission.\n2. **Input**: A 128-bit integer.\n3. **Output**: A result that either contains the successfully serialized data or an error if the data type is not supported.\n4. **Procedure**: The function attempts to serialize the provided 128-bit integer. If the serialization format does not support this data type, it returns an error indicating that the data type is not supported.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: serde/src/ser/impls.rs\nuse crate::lib::*;\n\nuse crate::ser::{Error, Serialize, SerializeTuple, Serializer};\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! primitive_impl {\n    ($ty:ident, $method:ident $($cast:tt)*) => {\n        impl Serialize for $ty {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.$method(*self $($cast)*)\n            }\n        }\n    }\n}\n\nprimitive_impl!(bool, serialize_bool);\nprimitive_impl!(isize, serialize_i64 as i64);\nprimitive_impl!(i8, serialize_i8);\nprimitive_impl!(i16, serialize_i16);\nprimitive_impl!(i32, serialize_i32);\nprimitive_impl!(i64, serialize_i64);\nprimitive_impl!(i128, serialize_i128);\nprimitive_impl!(usize, serialize_u64 as u64);\nprimitive_impl!(u8, serialize_u8);\nprimitive_impl!(u16, serialize_u16);\nprimitive_impl!(u32, serialize_u32);\nprimitive_impl!(u64, serialize_u64);\nprimitive_impl!(u128, serialize_u128);\nprimitive_impl!(f32, serialize_f32);\nprimitive_impl!(f64, serialize_f64);\nprimitive_impl!(char, serialize_char);\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for str {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_str(self)\n    }\n}\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl Serialize for String {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_str(self)\n    }\n}\n\nimpl<'a> Serialize for fmt::Arguments<'a> {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.collect_str(self)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(any(feature = \"std\", not(no_core_cstr)))]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for CStr {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_bytes(self.to_bytes())\n    }\n}\n\n#[cfg(any(feature = \"std\", all(not(no_core_cstr), feature = \"alloc\")))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl Serialize for CString {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_bytes(self.to_bytes())\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for Option<T>\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Some(ref value) => serializer.serialize_some(value),\n            None => serializer.serialize_none(),\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for PhantomData<T>\nwhere\n    T: ?Sized,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_unit_struct(\"PhantomData\")\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// Does not require T: Serialize.\nimpl<T> Serialize for [T; 0] {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        tri!(serializer.serialize_tuple(0)).end()\n    }\n}\n\nmacro_rules! array_impls {\n    ($($len:tt)+) => {\n        $(\n            impl<T> Serialize for [T; $len]\n            where\n                T: Serialize,\n            {\n                #[inline]\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    let mut seq = tri!(serializer.serialize_tuple($len));\n                    for e in self {\n                        tri!(seq.serialize_element(e));\n                    }\n                    seq.end()\n                }\n            }\n        )+\n    }\n}\n\narray_impls! {\n    01 02 03 04 05 06 07 08 09 10\n    11 12 13 14 15 16 17 18 19 20\n    21 22 23 24 25 26 27 28 29 30\n    31 32\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for [T]\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.collect_seq(self)\n    }\n}\n\n#[cfg(not(no_relaxed_trait_bounds))]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize,\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\n#[cfg(no_relaxed_trait_bounds)]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize $(+ $tbound1 $(+ $tbound2)*)*,\n            $($typaram: $bound,)*\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BinaryHeap<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeSet<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashSet<T: Eq + Hash, H: BuildHasher>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    LinkedList<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    Vec<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    VecDeque<T>\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for Range<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"Range\", 2));\n        tri!(state.serialize_field(\"start\", &self.start));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeFrom<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeFrom\", 1));\n        tri!(state.serialize_field(\"start\", &self.start));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeInclusive<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeInclusive\", 2));\n        tri!(state.serialize_field(\"start\", &self.start()));\n        tri!(state.serialize_field(\"end\", &self.end()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeTo<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeTo\", 1));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n...\n// Path: serde/src/ser/mod.rs\n//! Generic data structure serialization framework.\n//!\n//! The two most important traits in this module are [`Serialize`] and\n//! [`Serializer`].\n//!\n//!  - **A type that implements `Serialize` is a data structure** that can be\n//!    serialized to any data format supported by Serde, and conversely\n//!  - **A type that implements `Serializer` is a data format** that can\n//!    serialize any data structure supported by Serde.\n//!\n//! # The Serialize trait\n//!\n//! Serde provides [`Serialize`] implementations for many Rust primitive and\n//! standard library types. The complete list is below. All of these can be\n//! serialized using Serde out of the box.\n//!\n//! Additionally, Serde provides a procedural macro called [`serde_derive`] to\n//! automatically generate [`Serialize`] implementations for structs and enums\n//! in your program. See the [derive section of the manual] for how to use this.\n//!\n//! In rare cases it may be necessary to implement [`Serialize`] manually for\n//! some type in your program. See the [Implementing `Serialize`] section of the\n//! manual for more about this.\n//!\n//! Third-party crates may provide [`Serialize`] implementations for types that\n//! they expose. For example the [`linked-hash-map`] crate provides a\n//! [`LinkedHashMap<K, V>`] type that is serializable by Serde because the crate\n//! provides an implementation of [`Serialize`] for it.\n//!\n//! # The Serializer trait\n//!\n//! [`Serializer`] implementations are provided by third-party crates, for\n//! example [`serde_json`], [`serde_yaml`] and [`postcard`].\n//!\n//! A partial list of well-maintained formats is given on the [Serde\n//! website][data formats].\n//!\n//! # Implementations of Serialize provided by Serde\n//!\n//!  - **Primitive types**:\n//!    - bool\n//!    - i8, i16, i32, i64, i128, isize\n//!    - u8, u16, u32, u64, u128, usize\n//!    - f32, f64\n//!    - char\n//!    - str\n//!    - &T and &mut T\n//!  - **Compound types**:\n//!    - \\[T\\]\n//!    - \\[T; 0\\] through \\[T; 32\\]\n//!    - tuples up to size 16\n//!  - **Common standard library types**:\n//!    - String\n//!    - Option\\<T\\>\n//!    - Result\\<T, E\\>\n//!    - PhantomData\\<T\\>\n//!  - **Wrapper types**:\n//!    - Box\\<T\\>\n//!    - Cow\\<'a, T\\>\n//!    - Cell\\<T\\>\n//!    - RefCell\\<T\\>\n//!    - Mutex\\<T\\>\n//!    - RwLock\\<T\\>\n//!    - Rc\\<T\\>&emsp;*(if* features = \\[\"rc\"\\] *is enabled)*\n//!    - Arc\\<T\\>&emsp;*(if* features = \\[\"rc\"\\] *is enabled)*\n//!  - **Collection types**:\n//!    - BTreeMap\\<K, V\\>\n//!    - BTreeSet\\<T\\>\n//!    - BinaryHeap\\<T\\>\n//!    - HashMap\\<K, V, H\\>\n//!    - HashSet\\<T, H\\>\n//!    - LinkedList\\<T\\>\n//!    - VecDeque\\<T\\>\n//!    - Vec\\<T\\>\n//!  - **FFI types**:\n//!    - CStr\n//!    - CString\n//!    - OsStr\n//!    - OsString\n//!  - **Miscellaneous standard library types**:\n//!    - Duration\n//!    - SystemTime\n//!    - Path\n//!    - PathBuf\n//!    - Range\\<T\\>\n//!    - RangeInclusive\\<T\\>\n//!    - Bound\\<T\\>\n//!    - num::NonZero*\n//!    - `!` *(unstable)*\n//!  - **Net types**:\n//!    - IpAddr\n//!    - Ipv4Addr\n//!    - Ipv6Addr\n//!    - SocketAddr\n//!    - SocketAddrV4\n//!    - SocketAddrV6\n//!\n//! [Implementing `Serialize`]: https://serde.rs/impl-serialize.html\n//! [`LinkedHashMap<K, V>`]: https://docs.rs/linked-hash-map/*/linked_hash_map/struct.LinkedHashMap.html\n//! [`Serialize`]: ../trait.Serialize.html\n//! [`Serializer`]: ../trait.Serializer.html\n//! [`postcard`]: https://github.com/jamesmunns/postcard\n//! [`linked-hash-map`]: https://crates.io/crates/linked-hash-map\n//! [`serde_derive`]: https://crates.io/crates/serde_derive\n//! [`serde_json`]: https://github.com/serde-rs/json\n//! [`serde_yaml`]: https://github.com/dtolnay/serde-yaml\n//! [derive section of the manual]: https://serde.rs/derive.html\n//! [data formats]: https://serde.rs/#data-formats\n\nuse crate::lib::*;\n\nmod fmt;\nmod impls;\nmod impossible;\n\npub use self::impossible::Impossible;\n\n#[cfg(not(any(feature = \"std\", feature = \"unstable\")))]\n#[doc(no_inline)]\npub use crate::std_error::Error as StdError;\n#[cfg(all(feature = \"unstable\", not(feature = \"std\")))]\n#[doc(no_inline)]\npub use core::error::Error as StdError;\n#[cfg(feature = \"std\")]\n#[doc(no_inline)]\npub use std::error::Error as StdError;\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! declare_error_trait {\n    (Error: Sized $(+ $($supertrait:ident)::+)*) => {\n        /// Trait used by `Serialize` implementations to generically construct\n        /// errors belonging to the `Serializer` against which they are\n        /// currently running.\n        ///\n        /// # Example implementation\n        ///\n        /// The [example data format] presented on the website shows an error\n        /// type appropriate for a basic JSON data format.\n        ///\n        /// [example data format]: https://serde.rs/data-format.html\n        pub trait Error: Sized $(+ $($supertrait)::+)* {\n            /// Used when a [`Serialize`] implementation encounters any error\n            /// while serializing a type.\n            ///\n            /// The message should not be capitalized and should not end with a\n            /// period.\n            ///\n            /// For example, a filesystem [`Path`] may refuse to serialize\n            /// itself if it contains invalid UTF-8 data.\n            ///\n            /// ```edition2021\n            /// # struct Path;\n            /// #\n            /// # impl Path {\n            /// #     fn to_str(&self) -> Option<&str> {\n            /// #         unimplemented!()\n            /// #     }\n            /// # }\n            /// #\n            /// use serde::ser::{self, Serialize, Serializer};\n            ///\n            /// impl Serialize for Path {\n            ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            ///     where\n            ///         S: Serializer,\n            ///     {\n            ///         match self.to_str() {\n            ///             Some(s) => serializer.serialize_str(s),\n            ///             None => Err(ser::Error::custom(\"path contains invalid UTF-8 characters\")),\n            ///         }\n            ///     }\n            /// }\n            /// ```\n            ///\n            /// [`Path`]: https://doc.rust-lang.org/std/path/struct.Path.html\n            /// [`Serialize`]: ../trait.Serialize.html\n            fn custom<T>(msg: T) -> Self\n            where\n                T: Display;\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\ndeclare_error_trait!(Error: Sized + StdError);\n\n#[cfg(not(feature = \"std\"))]\ndeclare_error_trait!(Error: Sized + Debug + Display);\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data structure** that can be serialized into any data format supported\n/// by Serde.\n///\n/// Serde provides `Serialize` implementations for many Rust primitive and\n/// standard library types. The complete list is [here][crate::ser]. All of\n/// these can be serialized using Serde out of the box.\n///\n/// Additionally, Serde provides a procedural macro called [`serde_derive`] to\n/// automatically generate `Serialize` implementations for structs and enums in\n/// your program. See the [derive section of the manual] for how to use this.\n///\n/// In rare cases it may be necessary to implement `Serialize` manually for some\n/// type in your program. See the [Implementing `Serialize`] section of the\n/// manual for more about this.\n///\n/// Third-party crates may provide `Serialize` implementations for types that\n/// they expose. For example the [`linked-hash-map`] crate provides a\n/// [`LinkedHashMap<K, V>`] type that is serializable by Serde because the crate\n/// provides an implementation of `Serialize` for it.\n///\n/// [Implementing `Serialize`]: https://serde.rs/impl-serialize.html\n/// [`LinkedHashMap<K, V>`]: https://docs.rs/linked-hash-map/*/linked_hash_map/struct.LinkedHashMap.html\n/// [`linked-hash-map`]: https://crates.io/crates/linked-hash-map\n/// [`serde_derive`]: https://crates.io/crates/serde_derive\n/// [derive section of the manual]: https://serde.rs/derive.html\npub trait Serialize {\n    /// Serialize this value into the given Serde serializer.\n    ///\n    /// See the [Implementing `Serialize`] section of the manual for more\n    /// information about how to implement this method.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeStruct, Serializer};\n    ///\n    /// struct Person {\n    ///     name: String,\n    ///     age: u8,\n    ///     phones: Vec<String>,\n    /// }\n    ///\n    /// // This is what #[derive(Serialize)] would generate.\n    /// impl Serialize for Person {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut s = serializer.serialize_struct(\"Person\", 3)?;\n    ///         s.serialize_field(\"name\", &self.name)?;\n    ///         s.serialize_field(\"age\", &self.age)?;\n    ///         s.serialize_field(\"phones\", &self.phones)?;\n    ///         s.end()\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// [Implementing `Serialize`]: https://serde.rs/impl-serialize.html\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data format** that can serialize any data structure supported by Serde.\n///\n/// The role of this trait is to define the serialization half of the [Serde\n/// data model], which is a way to categorize every Rust data structure into one\n/// of 29 possible types. Each method of the `Serializer` trait corresponds to\n/// one of the types of the data model.\n///\n/// Implementations of `Serialize` map themselves into this data model by\n/// invoking exactly one of the `Serializer` methods.\n///\n/// The types that make up the Serde data model are:\n///\n///  - **14 primitive types**\n///    - bool\n///    - i8, i16, i32, i64, i128\n///    - u8, u16, u32, u64, u128\n///    - f32, f64\n///    - char\n///  - **string**\n///    - UTF-8 bytes with a length and no null terminator.\n///    - When serializing, all strings are handled equally. When deserializing,\n///      there are three flavors of strings: transient, owned, and borrowed.\n///  - **byte array** - \\[u8\\]\n///    - Similar to strings, during deserialization byte arrays can be\n///      transient, owned, or borrowed.\n///  - **option**\n///    - Either none or some value.\n///  - **unit**\n///    - The type of `()` in Rust. It represents an anonymous value containing\n///      no data.\n///  - **unit_struct**\n///    - For example `struct Unit` or `PhantomData<T>`. It represents a named\n///      value containing no data.\n///  - **unit_variant**\n///    - For example the `E::A` and `E::B` in `enum E { A, B }`.\n///  - **newtype_struct**\n///    - For example `struct Millimeters(u8)`.\n///  - **newtype_variant**\n///    - For example the `E::N` in `enum E { N(u8) }`.\n///  - **seq**\n///    - A variably sized heterogeneous sequence of values, for example\n///      `Vec<T>` or `HashSet<T>`. When serializing, the length may or may not\n///      be known before iterating through all the data. When deserializing,\n///      the length is determined by looking at the serialized data.\n///  - **tuple**\n///    - A statically sized heterogeneous sequence of values for which the\n///      length will be known at deserialization time without looking at the\n///      serialized data, for example `(u8,)` or `(String, u64, Vec<T>)` or\n///      `[u64; 10]`.\n///  - **tuple_struct**\n///    - A named tuple, for example `struct Rgb(u8, u8, u8)`.\n///  - **tuple_variant**\n///    - For example the `E::T` in `enum E { T(u8, u8) }`.\n///  - **map**\n///    - A heterogeneous key-value pairing, for example `BTreeMap<K, V>`.\n///  - **struct**\n///    - A heterogeneous key-value pairing in which the keys are strings and\n///      will be known at deserialization time without looking at the\n///      serialized data, for example `struct S { r: u8, g: u8, b: u8 }`.\n///  - **struct_variant**\n///    - For example the `E::S` in `enum E { S { r: u8, g: u8, b: u8 } }`.\n///\n/// Many Serde serializers produce text or binary data as output, for example\n/// JSON or Postcard. This is not a requirement of the `Serializer` trait, and\n/// there are serializers that do not produce text or binary output. One example\n/// is the `serde_json::value::Serializer` (distinct from the main `serde_json`\n/// serializer) that produces a `serde_json::Value` data structure in memory as\n/// output.\n///\n/// [Serde data model]: https://serde.rs/data-model.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website contains example code for\n/// a basic JSON `Serializer`.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait Serializer: Sized {\n    /// The output type produced by this `Serializer` during successful\n    /// serialization. Most serializers that produce text or binary output\n    /// should set `Ok = ()` and serialize into an [`io::Write`] or buffer\n    /// contained within the `Serializer` instance. Serializers that build\n    /// in-memory data structures may be simplified by using `Ok` to propagate\n    /// the data structure around.\n    ///\n    /// [`io::Write`]: https://doc.rust-lang.org/std/io/trait.Write.html\n    type Ok;\n\n    /// The error type when some error occurs during serialization.\n    type Error: Error;\n\n    /// Type returned from [`serialize_seq`] for serializing the content of the\n    /// sequence.\n    ///\n    /// [`serialize_seq`]: #tymethod.serialize_seq\n    type SerializeSeq: SerializeSeq<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple`] for serializing the content of\n    /// the tuple.\n    ///\n    /// [`serialize_tuple`]: #tymethod.serialize_tuple\n    type SerializeTuple: SerializeTuple<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple_struct`] for serializing the\n    /// content of the tuple struct.\n    ///\n    /// [`serialize_tuple_struct`]: #tymethod.serialize_tuple_struct\n    type SerializeTupleStruct: SerializeTupleStruct<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple_variant`] for serializing the\n    /// content of the tuple variant.\n    ///\n    /// [`serialize_tuple_variant`]: #tymethod.serialize_tuple_variant\n    type SerializeTupleVariant: SerializeTupleVariant<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_map`] for serializing the content of the\n    /// map.\n    ///\n    /// [`serialize_map`]: #tymethod.serialize_map\n    type SerializeMap: SerializeMap<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_struct`] for serializing the content of\n    /// the struct.\n    ///\n    /// [`serialize_struct`]: #tymethod.serialize_struct\n    type SerializeStruct: SerializeStruct<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_struct_variant`] for serializing the\n    /// content of the struct variant.\n    ///\n    /// [`serialize_struct_variant`]: #tymethod.serialize_struct_variant\n    type SerializeStructVariant: SerializeStructVariant<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Serialize a `bool` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for bool {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_bool(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i8` value.\n    ///\n    /// If the format does not differentiate between `i8` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i8 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i8(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i16` value.\n    ///\n    /// If the format does not differentiate between `i16` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i16 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i16(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i32` value.\n    ///\n    /// If the format does not differentiate between `i32` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i32 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i32(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i64` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i64 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i64(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i128` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i128 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i128(*self)\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default behavior unconditionally returns an error.\n    \nfn serialize_i128(self, v: i128) -> Result<Self::Ok, Self::Error> {\n        let _ = v;\n        Err(Error::custom(\"i128 is not supported\"))\n    }\n\n    /// Serialize a `u8` value.\n    ///\n    /// If the format does not differentiate between `u8` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u8 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u8(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u16` value.\n    ///\n    /// If the format does not differentiate between `u16` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u16 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u16(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u32` value.\n    ///\n    /// If the format does not differentiate between `u32` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u32 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u32(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u64` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u64 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u64(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u64(self, v: u64) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u128` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u128 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u128(*self)\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn serialize_u128(self, v: u128) -> Result<Self::Ok, Self::Error> {\n        let _ = v;\n        Err(Error::custom(\"u128 is not supported\"))\n    }\n\n    /// Serialize an `f32` value.\n    ///\n    /// If the format does not differentiate between `f32` and `f64`, a\n    /// reasonable implementation would be to cast the value to `f64` and\n    /// forward to `serialize_f64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for f32 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_f32(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_f32(self, v: f32) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `f64` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for f64 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_f64(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_f64(self, v: f64) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a character.\n    ///\n    /// If the format does not support characters, it is reasonable to serialize\n    /// it as a single element `str` or a `u32`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for char {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_char(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_char(self, v: char) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `&str`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for str {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_str(self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_str(self, v: &str) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a chunk of raw byte data.\n    ///\n    /// Enables serializers to serialize byte slices more compactly or more\n    /// efficiently than other types of slices. If no efficient implementation\n    /// is available, a reasonable implementation would be to forward to\n    /// `serialize_seq`. If forwarded, the implementation looks usually just\n    /// like this:\n    ///\n    /// ```edition2021\n    /// # use serde::ser::{Serializer, SerializeSeq};\n    /// # use serde::__private::doc::Error;\n    /// #\n    /// # struct MySerializer;\n    /// #\n    /// # impl Serializer for MySerializer {\n    /// #     type Ok = ();\n    /// #     type Error = Error;\n    /// #\n    /// fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error> {\n    ///     let mut seq = self.serialize_seq(Some(v.len()))?;\n    ///     for b in v {\n    ///         seq.serialize_element(b)?;\n    ///     }\n    ///     seq.end()\n    /// }\n    /// #\n    /// #     serde::__serialize_unimplemented! {\n    /// #         bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str none some\n    /// #         unit unit_struct unit_variant newtype_struct newtype_variant\n    /// #         seq tuple tuple_struct tuple_variant map struct struct_variant\n    /// #     }\n    /// # }\n    /// ```\n    fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a [`None`] value.\n    ///\n    /// ```edition2021\n    /// # use serde::{Serialize, Serializer};\n    /// #\n    /// # enum Option<T> {\n    /// #     Some(T),\n    /// #     None,\n    /// # }\n    /// #\n    /// # use self::Option::{Some, None};\n    /// #\n    /// impl<T> Serialize for Option<T>\n    /// where\n    ///     T: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             Some(ref value) => serializer.serialize_some(value),\n    ///             None => serializer.serialize_none(),\n    ///         }\n    ///     }\n    /// }\n    /// #\n    /// # fn main() {}\n    /// ```\n    ///\n    /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None\n    fn serialize_none(self) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a [`Some(T)`] value.\n    ///\n    /// ```edition2021\n    /// # use serde::{Serialize, Serializer};\n    /// #\n    /// # enum Option<T> {\n    /// #     Some(T),\n    /// #     None,\n    /// # }\n    /// #\n    /// # use self::Option::{Some, None};\n    /// #\n    /// impl<T> Serialize for Option<T>\n    /// where\n    ///     T: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             Some(ref value) => serializer.serialize_some(value),\n    ///             None => serializer.serialize_none(),\n    ///         }\n    ///     }\n    /// }\n    /// #\n    /// # fn main() {}\n    /// ```\n    ///\n    /// [`Some(T)`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.Some\n    fn serialize_some<T>(self, value: &T) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Serialize a `()` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for () {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_unit()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_unit(self) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a unit struct like `struct Unit` or `PhantomData<T>`.\n    ///\n    /// A reasonable implementation would be to forward to `serialize_unit`.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// struct Nothing;\n    ///\n    /// impl Serialize for Nothing {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_unit_struct(\"Nothing\")\n    ///     }\n    /// }\n    /// ```\n    fn serialize_unit_struct(self, name: &'static str) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a unit variant like `E::A` in `enum E { A, B }`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, and the `variant` is the name of the\n    /// variant.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// enum E {\n    ///     A,\n    ///     B,\n    /// }\n    ///\n    /// impl Serialize for E {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             E::A => serializer.serialize_unit_variant(\"E\", 0, \"A\"),\n    ///             E::B => serializer.serialize_unit_variant(\"E\", 1, \"B\"),\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    fn serialize_unit_variant(\n        self,\n        name: &'static str,\n        variant_index: u32,\n        variant: &'static str,\n    ) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a newtype struct like `struct Millimeters(u8)`.\n    ///\n    /// Serializers are encouraged to treat newtype structs as insignificant\n    /// wrappers around the data they contain. A reasonable implementation would\n    /// be to forward to `value.serialize(self)`.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// struct Millimeters(u8);\n    ///\n    /// impl Serialize for Millimeters {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_newtype_struct(\"Millimeters\", &self.0)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_newtype_struct<T>(\n        self,\n        name: &'static str,\n        value: &T,\n    ) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Serialize a newtype variant like `E::N` in `enum E { N(u8) }`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, and the `variant` is the name of the\n    /// variant. The `value` is the data contained within this newtype variant.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// enum E {\n    ///     M(String),\n    ///     N(u8),\n    /// }\n    ///\n    /// impl Serialize for E {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             E::M(ref s) => serializer.serialize_newtype_variant(\"E\", 0, \"M\", s),\n    ///             E::N(n) => serializer.serialize_newtype_variant(\"E\", 1, \"N\", &n),\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    fn serialize_newtype_variant<T>(\n        self,\n        name: &'static str,\n        variant_index: u32,\n        variant: &'static str,\n        value: &T,\n    ) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Begin to serialize a variably sized sequence. This call must be\n    /// followed by zero or more calls to `serialize_element`, then a call to\n    /// `end`.\n    ///\n    /// The argument is the number of elements in the sequence, which may or may\n    /// not be computable before the sequence is iterated. Some serializers only\n    /// support sequences whose length is known up front.\n    ///\n    /// ```edition2021\n    /// # use std::marker::PhantomData;\n    /// #\n    /// # struct Vec<T>(PhantomData<T>);\n    /// #\n    /// # impl<T> Vec<T> {\n    /// #     fn len(&self) -> usize {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// # impl<'a, T> IntoIterator for &'a Vec<T> {\n    /// #     type Item = &'a T;\n    /// #     type IntoIter = Box<dyn Iterator<Item = &'a T>>;\n    /// #\n    /// #     fn into_iter(self) -> Self::IntoIter {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::ser::{Serialize, SerializeSeq, Serializer};\n    ///\n    /// impl<T> Serialize for Vec<T>\n    /// where\n    ///     T: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut seq = serializer.serialize_seq(Some(self.len()))?;\n    ///         for element in self {\n    ///             seq.serialize_element(element)?;\n    ///         }\n    ///         seq.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error>;\n\n    /// Begin to serialize a statically sized sequence whose length will be\n    /// known at deserialization time without looking at the serialized data.\n    /// This call must be followed by zero or more calls to `serialize_element`,\n    /// then a call to `end`.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTuple, Serializer};\n    ///\n    /// # mod fool {\n    /// #     trait Serialize {}\n    /// impl<A, B, C> Serialize for (A, B, C)\n    /// #     {}\n    /// # }\n    /// #\n    /// # struct Tuple3<A, B, C>(A, B, C);\n    /// #\n    /// # impl<A, B, C> Serialize for Tuple3<A, B, C>\n    /// where\n    ///     A: Serialize,\n    ///     B: Serialize,\n    ///     C: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut tup = serializer.serialize_tuple(3)?;\n    ///         tup.serialize_element(&self.0)?;\n    ///         tup.serialize_element(&self.1)?;\n    ///         tup.serialize_element(&self.2)?;\n    ///         tup.end()\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTuple, Serializer};\n    ///\n    /// const VRAM_SIZE: usize = 386;\n    /// struct Vram([u16; VRAM_SIZE]);\n    ///\n    /// impl Serialize for Vram {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut seq = serializer.serialize_tuple(VRAM_SIZE)?;\n    ///         for element in &self.0[..] {\n    ///             seq.serialize_element(element)?;\n    ///         }\n    ///         seq.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, Self::Error>;\n\n    /// Begin to serialize a tuple struct like `struct Rgb(u8, u8, u8)`. This\n    /// call must be followed by zero or more calls to `serialize_field`, then a\n    /// call to `end`.\n    ///\n    /// The `name` is the name of the tuple struct and the `len` is the number\n    /// of data fields that will be serialized.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTupleStruct, Serializer};\n    ///\n    /// struct Rgb(u8, u8, u8);\n    ///\n    /// impl Serialize for Rgb {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut ts = serializer.serialize_tuple_struct(\"Rgb\", 3)?;\n    ///         ts.serialize_field(&self.0)?;\n    ///         ts.serialize_field(&self.1)?;\n    ///         ts.serialize_field(&self.2)?;\n    ///         ts.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_tuple_struct(\n        self,\n        name: &'static str,\n        len: usize,\n    ) -> Result<Self::SerializeTupleStruct, Self::Error>;\n\n    /// Begin to serialize a tuple variant like `E::T` in `enum E { T(u8, u8)\n    /// }`. This call must be followed by zero or more calls to\n    /// `serialize_field`, then a call to `end`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, the `variant` is the name of the variant,\n    /// and the `len` is the number of data fields that will be serialized.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTupleVariant, Serializer};\n    ///\n    /// enum E {\n    ///     T(u8, u8),\n    ///     U(String, u32, u32),\n    /// }\n    ///\n    /// impl Serialize for E {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             E::T(ref a, ref b) => {\n    ///                 let mut tv = serializer.serialize_tuple_variant(\"E\", 0, \"T\", 2)?;\n    ///                 tv.serialize_field(a)?;\n    ///                 tv.serialize_field(b)?;\n    ///                 tv.end()\n    ///             }\n    ///             E::U(ref a, ref b, ref c) => {\n    ///                 let mut tv = serializer.serialize_tuple_variant(\"E\", 1, \"U\", 3)?;\n    ///                 tv.serialize_field(a)?;\n    ///                 tv.serialize_field(b)?;\n    ///                 tv.serialize_field(c)?;\n    ///                 tv.end()\n    ///             }\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    fn serialize_tuple_variant(\n        self,\n        name: &'static str,\n        variant_index: u32,\n        variant: &'static str,\n        len: usize,\n    ) -> Result<Self::SerializeTupleVariant, Self::Error>;\n\n    /// Begin to serialize a map. This call must be followed by zero or more\n    /// calls to `serialize_key` and `serialize_value`, then a call to `end`.\n    ///\n    /// The argument is the number of elements in the map, which may or may not\n    /// be computable before the map is iterated. Some serializers only support\n    /// maps whose length is known up front.\n    ///\n    /// ```edition2021\n    /// # use std::marker::PhantomData;\n    /// #\n    /// # struct HashMap<K, V>(PhantomData<K>, PhantomData<V>);\n    /// #\n    /// # impl<K, V> HashMap<K, V> {\n    /// #     fn len(&self) -> usize {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// # impl<'a, K, V> IntoIterator for &'a HashMap<K, V> {\n    /// #     type Item = (&'a K, &'a V);\n    /// #     type IntoIter = Box<dyn Iterator<Item = (&'a K, &'a V)>>;\n    /// #\n    /// #     fn into_iter(self) -> Self::IntoIter {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::ser::{Serialize, SerializeMap, Serializer};\n    ///\n    /// impl<K, V> Serialize for HashMap<K, V>\n    /// where\n    ///     K: Serialize,\n    ///     V: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut map = serializer.serialize_map(Some(self.len()))?;\n    ///         for (k, v) in self {\n    ///             map.serialize_entry(k, v)?;\n    ///         }\n    ///         map.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, Self::Error>;\n\n    /// Begin to serialize a struct like `struct Rgb { r: u8, g: u8, b: u8 }`.\n    /// This call must be followed by zero or more calls to `serialize_field`,\n    /// then a call to `end`.\n    ///\n    /// The `name` is the name of the struct and the `len` is the number of\n    /// data fields that will be serialized. `len` does not include fields\n    /// which are skipped with [`SerializeStruct::skip_field`].\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeStruct, Serializer};\n    ///\n    /// struct Rgb {\n    ///     r: u8,\n    ///     g: u8,\n    ///     b: u8,\n    /// }\n    ///\n    /// impl Serialize for Rgb {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut rgb = serializer.serialize_struct(\"Rgb\", 3)?;\n    ///         rgb.serialize_field(\"r\", &self.r)?;\n    ///         rgb.serialize_field(\"g\", &self.g)?;\n    ///         rgb.serialize_field(\"b\", &self.b)?;\n    ///         rgb.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_struct(\n        self,\n        name: &'static str,\n        len: usize,\n    ) -> Result<Self::SerializeStruct, Self::Error>;\n\n    /// Begin to serialize a struct variant like `E::S` in `enum E { S { r: u8,\n    /// g: u8, b: u8 } }`. This call must be followed by zero or more calls to\n    /// `serialize_field`, then a call to `end`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, the `variant` is the name of the variant,\n...\n", "needle_token_start": 8911, "needle_token_end": 8966, "code_context_ntokens": 16281, "output": ["```rust\nfn serialize_i128(self, v: i128) -> Result<Self::Ok, Self::Error> {\n    let _ = v;\n    Err(Error::custom(\"i128 is not supported\"))\n}\n```"]}
{"repo": "serde-rs/serde", "name": "serialize_unit_variant", "language": "rust", "path": "serde/src/ser/fmt.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the serialization of enum variants that do not have associated values, converting them into a format suitable for display or storage.\n2. **Input**: It accepts three parameters: the name of the enum, the index of the variant within the enum, and the string representation of the variant.\n3. **Output**: The output is a result indicating whether the serialization was successful, encapsulated in a format-specific result type.\n4. **Procedure**: The function takes the string representation of the variant and formats it using the display traits of the underlying formatter, effectively converting the variant into a string format suitable for output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: serde/src/de/impls.rs\nuse crate::lib::*;\n\nuse crate::de::{\n    Deserialize, Deserializer, EnumAccess, Error, MapAccess, SeqAccess, Unexpected, VariantAccess,\n    Visitor,\n};\n\nuse crate::seed::InPlaceSeed;\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nuse crate::de::size_hint;\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct UnitVisitor;\n\nimpl<'de> Visitor<'de> for UnitVisitor {\n    type Value = ();\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"unit\")\n    }\n\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(())\n    }\n}\n\nimpl<'de> Deserialize<'de> for () {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_unit(UnitVisitor)\n    }\n}\n\n#[cfg(feature = \"unstable\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"unstable\")))]\nimpl<'de> Deserialize<'de> for ! {\n    fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        Err(Error::custom(\"cannot deserialize `!`\"))\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct BoolVisitor;\n\nimpl<'de> Visitor<'de> for BoolVisitor {\n    type Value = bool;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a boolean\")\n    }\n\n    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n}\n\nimpl<'de> Deserialize<'de> for bool {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_bool(BoolVisitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! impl_deserialize_num {\n    ($primitive:ident, $nonzero:ident $(cfg($($cfg:tt)*))*, $deserialize:ident $($method:ident!($($val:ident : $visit:ident)*);)*) => {\n        impl_deserialize_num!($primitive, $deserialize $($method!($($val : $visit)*);)*);\n\n        $(#[cfg($($cfg)*)])*\n        impl<'de> Deserialize<'de> for num::$nonzero {\n            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n            where\n                D: Deserializer<'de>,\n            {\n                struct NonZeroVisitor;\n\n                impl<'de> Visitor<'de> for NonZeroVisitor {\n                    type Value = num::$nonzero;\n\n                    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                        formatter.write_str(concat!(\"a nonzero \", stringify!($primitive)))\n                    }\n\n                    $($($method!(nonzero $primitive $val : $visit);)*)*\n                }\n\n                deserializer.$deserialize(NonZeroVisitor)\n            }\n        }\n    };\n\n    ($primitive:ident, $deserialize:ident $($method:ident!($($val:ident : $visit:ident)*);)*) => {\n        impl<'de> Deserialize<'de> for $primitive {\n            #[inline]\n            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n            where\n                D: Deserializer<'de>,\n            {\n                struct PrimitiveVisitor;\n\n                impl<'de> Visitor<'de> for PrimitiveVisitor {\n                    type Value = $primitive;\n\n                    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n                        formatter.write_str(stringify!($primitive))\n                    }\n\n                    $($($method!($val : $visit);)*)*\n                }\n\n                deserializer.$deserialize(PrimitiveVisitor)\n            }\n        }\n    };\n}\n\nmacro_rules! num_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            Ok(v)\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if let Some(nonzero) = Self::Value::new(v) {\n                Ok(nonzero)\n            } else {\n                Err(Error::invalid_value(Unexpected::Unsigned(0), &self))\n            }\n        }\n    };\n}\n\nmacro_rules! num_as_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            Ok(v as Self::Value)\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                Ok(nonzero)\n            } else {\n                Err(Error::invalid_value(Unexpected::Unsigned(0), &self))\n            }\n        }\n    };\n}\n\nmacro_rules! num_as_copysign_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            #[cfg(any(no_float_copysign, not(feature = \"std\")))]\n            {\n                Ok(v as Self::Value)\n            }\n\n            #[cfg(all(not(no_float_copysign), feature = \"std\"))]\n            {\n                // Preserve sign of NaN. The `as` produces a nondeterministic sign.\n                let sign = if v.is_sign_positive() { 1.0 } else { -1.0 };\n                Ok((v as Self::Value).copysign(sign))\n            }\n        }\n    };\n}\n\nmacro_rules! int_to_int {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if Self::Value::min_value() as i64 <= v as i64\n                && v as i64 <= Self::Value::max_value() as i64\n            {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if $primitive::min_value() as i64 <= v as i64\n                && v as i64 <= $primitive::max_value() as i64\n            {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    return Ok(nonzero);\n                }\n            }\n            Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n        }\n    };\n}\n\nmacro_rules! int_to_uint {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if 0 <= v && v as u64 <= Self::Value::max_value() as u64 {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if 0 < v && v as u64 <= $primitive::max_value() as u64 {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    return Ok(nonzero);\n                }\n            }\n            Err(Error::invalid_value(Unexpected::Signed(v as i64), &self))\n        }\n    };\n}\n\nmacro_rules! uint_to_self {\n    ($ty:ident : $visit:ident) => {\n        #[inline]\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as u64 <= Self::Value::max_value() as u64 {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(Unexpected::Unsigned(v as u64), &self))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as u64 <= $primitive::max_value() as u64 {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    return Ok(nonzero);\n                }\n            }\n            Err(Error::invalid_value(Unexpected::Unsigned(v as u64), &self))\n        }\n    };\n}\n\nimpl_deserialize_num! {\n    i8, NonZeroI8 cfg(not(no_num_nonzero_signed)), deserialize_i8\n    num_self!(i8:visit_i8);\n    int_to_int!(i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    i16, NonZeroI16 cfg(not(no_num_nonzero_signed)), deserialize_i16\n    num_self!(i16:visit_i16);\n    num_as_self!(i8:visit_i8);\n    int_to_int!(i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    i32, NonZeroI32 cfg(not(no_num_nonzero_signed)), deserialize_i32\n    num_self!(i32:visit_i32);\n    num_as_self!(i8:visit_i8 i16:visit_i16);\n    int_to_int!(i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    i64, NonZeroI64 cfg(not(no_num_nonzero_signed)), deserialize_i64\n    num_self!(i64:visit_i64);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    isize, NonZeroIsize cfg(not(no_num_nonzero_signed)), deserialize_i64\n    num_as_self!(i8:visit_i8 i16:visit_i16);\n    int_to_int!(i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u8, NonZeroU8, deserialize_u8\n    num_self!(u8:visit_u8);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u16, NonZeroU16, deserialize_u16\n    num_self!(u16:visit_u16);\n    num_as_self!(u8:visit_u8);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u32, NonZeroU32, deserialize_u32\n    num_self!(u32:visit_u32);\n    num_as_self!(u8:visit_u8 u16:visit_u16);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    u64, NonZeroU64, deserialize_u64\n    num_self!(u64:visit_u64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n}\n\nimpl_deserialize_num! {\n    usize, NonZeroUsize, deserialize_u64\n    num_as_self!(u8:visit_u8 u16:visit_u16);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    uint_to_self!(u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    f32, deserialize_f32\n    num_self!(f32:visit_f32);\n    num_as_copysign_self!(f64:visit_f64);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nimpl_deserialize_num! {\n    f64, deserialize_f64\n    num_self!(f64:visit_f64);\n    num_as_copysign_self!(f32:visit_f32);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n}\n\nmacro_rules! num_128 {\n    ($ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as i128 >= Self::Value::min_value() as i128\n                && v as u128 <= Self::Value::max_value() as u128\n            {\n                Ok(v as Self::Value)\n            } else {\n                Err(Error::invalid_value(\n                    Unexpected::Other(stringify!($ty)),\n                    &self,\n                ))\n            }\n        }\n    };\n\n    (nonzero $primitive:ident $ty:ident : $visit:ident) => {\n        fn $visit<E>(self, v: $ty) -> Result<Self::Value, E>\n        where\n            E: Error,\n        {\n            if v as i128 >= $primitive::min_value() as i128\n                && v as u128 <= $primitive::max_value() as u128\n            {\n                if let Some(nonzero) = Self::Value::new(v as $primitive) {\n                    Ok(nonzero)\n                } else {\n                    Err(Error::invalid_value(Unexpected::Unsigned(0), &self))\n                }\n            } else {\n                Err(Error::invalid_value(\n                    Unexpected::Other(stringify!($ty)),\n                    &self,\n                ))\n            }\n        }\n    };\n}\n\nimpl_deserialize_num! {\n    i128, NonZeroI128 cfg(not(no_num_nonzero_signed)), deserialize_i128\n    num_self!(i128:visit_i128);\n    num_as_self!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n    num_128!(u128:visit_u128);\n}\n\nimpl_deserialize_num! {\n    u128, NonZeroU128, deserialize_u128\n    num_self!(u128:visit_u128);\n    num_as_self!(u8:visit_u8 u16:visit_u16 u32:visit_u32 u64:visit_u64);\n    int_to_uint!(i8:visit_i8 i16:visit_i16 i32:visit_i32 i64:visit_i64);\n    num_128!(i128:visit_i128);\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nstruct CharVisitor;\n\nimpl<'de> Visitor<'de> for CharVisitor {\n    type Value = char;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a character\")\n    }\n\n    #[inline]\n    fn visit_char<E>(self, v: char) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n\n    #[inline]\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut iter = v.chars();\n        match (iter.next(), iter.next()) {\n            (Some(c), None) => Ok(c),\n            _ => Err(Error::invalid_value(Unexpected::Str(v), &self)),\n        }\n    }\n}\n\nimpl<'de> Deserialize<'de> for char {\n    #[inline]\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_char(CharVisitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nstruct StringVisitor;\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nstruct StringInPlaceVisitor<'a>(&'a mut String);\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\nimpl<'de> Visitor<'de> for StringVisitor {\n    type Value = String;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"a string\")\n    }\n\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v.to_owned())\n    }\n\n    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Ok(v)\n    }\n\n    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        match str::from_utf8(v) {\n            Ok(s) => Ok(s.to_owned()),\n            Err(_) => Err(Error::invalid_value(Unexpected::Bytes(v), &self)),\n        }\n    }\n\n    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        match String::from_utf8(v) {\n            Ok(s) => Ok(s),\n            Err(e) => Err(Error::invalid_value(\n...\n// Path: serde/src/de/ignored_any.rs\nuse crate::lib::*;\n\nuse crate::de::{\n    Deserialize, Deserializer, EnumAccess, Error, MapAccess, SeqAccess, VariantAccess, Visitor,\n};\n\n/// An efficient way of discarding data from a deserializer.\n///\n/// Think of this like `serde_json::Value` in that it can be deserialized from\n/// any type, except that it does not store any information about the data that\n/// gets deserialized.\n///\n/// ```edition2021\n/// use serde::de::{\n///     self, Deserialize, DeserializeSeed, Deserializer, IgnoredAny, SeqAccess, Visitor,\n/// };\n/// use std::fmt;\n/// use std::marker::PhantomData;\n///\n/// /// A seed that can be used to deserialize only the `n`th element of a sequence\n/// /// while efficiently discarding elements of any type before or after index `n`.\n/// ///\n/// /// For example to deserialize only the element at index 3:\n/// ///\n/// /// ```\n/// /// NthElement::new(3).deserialize(deserializer)\n/// /// ```\n/// pub struct NthElement<T> {\n///     n: usize,\n///     marker: PhantomData<T>,\n/// }\n///\n/// impl<T> NthElement<T> {\n///     pub fn new(n: usize) -> Self {\n///         NthElement {\n///             n: n,\n///             marker: PhantomData,\n///         }\n///     }\n/// }\n///\n/// impl<'de, T> Visitor<'de> for NthElement<T>\n/// where\n///     T: Deserialize<'de>,\n/// {\n///     type Value = T;\n///\n///     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///         write!(\n///             formatter,\n///             \"a sequence in which we care about element {}\",\n///             self.n\n///         )\n///     }\n///\n///     fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n///     where\n///         A: SeqAccess<'de>,\n///     {\n///         // Skip over the first `n` elements.\n///         for i in 0..self.n {\n///             // It is an error if the sequence ends before we get to element `n`.\n///             if seq.next_element::<IgnoredAny>()?.is_none() {\n///                 return Err(de::Error::invalid_length(i, &self));\n///             }\n///         }\n///\n///         // Deserialize the one we care about.\n///         let nth = match seq.next_element()? {\n///             Some(nth) => nth,\n///             None => {\n///                 return Err(de::Error::invalid_length(self.n, &self));\n///             }\n///         };\n///\n///         // Skip over any remaining elements in the sequence after `n`.\n///         while let Some(IgnoredAny) = seq.next_element()? {\n///             // ignore\n///         }\n///\n///         Ok(nth)\n///     }\n/// }\n///\n/// impl<'de, T> DeserializeSeed<'de> for NthElement<T>\n/// where\n///     T: Deserialize<'de>,\n/// {\n///     type Value = T;\n///\n///     fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n///     where\n///         D: Deserializer<'de>,\n///     {\n///         deserializer.deserialize_seq(self)\n///     }\n/// }\n///\n/// # fn example<'de, D>(deserializer: D) -> Result<(), D::Error>\n/// # where\n/// #     D: Deserializer<'de>,\n/// # {\n/// // Deserialize only the sequence element at index 3 from this deserializer.\n/// // The element at index 3 is required to be a string. Elements before and\n/// // after index 3 are allowed to be of any type.\n/// let s: String = NthElement::new(3).deserialize(deserializer)?;\n/// #     Ok(())\n/// # }\n/// ```\n#[derive(Copy, Clone, Debug, Default, PartialEq)]\npub struct IgnoredAny;\n\nimpl<'de> Visitor<'de> for IgnoredAny {\n    type Value = IgnoredAny;\n\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(\"anything at all\")\n    }\n\n    #[inline]\n    fn visit_bool<E>(self, x: bool) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_i64<E>(self, x: i64) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_i128<E>(self, x: i128) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_u64<E>(self, x: u64) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_u128<E>(self, x: u128) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_f64<E>(self, x: f64) -> Result<Self::Value, E> {\n        let _ = x;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let _ = s;\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_none<E>(self) -> Result<Self::Value, E> {\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        IgnoredAny::deserialize(deserializer)\n    }\n\n    #[inline]\n    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        IgnoredAny::deserialize(deserializer)\n    }\n\n    #[inline]\n    fn visit_unit<E>(self) -> Result<Self::Value, E> {\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>\n    where\n        A: SeqAccess<'de>,\n    {\n        while let Some(IgnoredAny) = tri!(seq.next_element()) {\n            // Gobble\n        }\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>\n    where\n        A: MapAccess<'de>,\n    {\n        while let Some((IgnoredAny, IgnoredAny)) = tri!(map.next_entry()) {\n            // Gobble\n        }\n        Ok(IgnoredAny)\n    }\n\n    #[inline]\n    fn visit_bytes<E>(self, bytes: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let _ = bytes;\n        Ok(IgnoredAny)\n    }\n\n    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>\n    where\n        A: EnumAccess<'de>,\n    {\n        tri!(data.variant::<IgnoredAny>()).1.newtype_variant()\n    }\n}\n\nimpl<'de> Deserialize<'de> for IgnoredAny {\n    #[inline]\n    fn deserialize<D>(deserializer: D) -> Result<IgnoredAny, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        deserializer.deserialize_ignored_any(IgnoredAny)\n    }\n}\n\n// Path: serde/src/ser/impossible.rs\n//! This module contains `Impossible` serializer and its implementations.\n\nuse crate::lib::*;\n\nuse crate::ser::{\n    self, Serialize, SerializeMap, SerializeSeq, SerializeStruct, SerializeStructVariant,\n    SerializeTuple, SerializeTupleStruct, SerializeTupleVariant,\n};\n\n/// Helper type for implementing a `Serializer` that does not support\n/// serializing one of the compound types.\n///\n/// This type cannot be instantiated, but implements every one of the traits\n/// corresponding to the [`Serializer`] compound types: [`SerializeSeq`],\n/// [`SerializeTuple`], [`SerializeTupleStruct`], [`SerializeTupleVariant`],\n/// [`SerializeMap`], [`SerializeStruct`], and [`SerializeStructVariant`].\n///\n/// ```edition2021\n/// # use serde::ser::{Serializer, Impossible};\n/// # use serde::__private::doc::Error;\n/// #\n/// # struct MySerializer;\n/// #\n/// impl Serializer for MySerializer {\n///     type Ok = ();\n///     type Error = Error;\n///\n///     type SerializeSeq = Impossible<(), Error>;\n///     /* other associated types */\n///\n///     /// This data format does not support serializing sequences.\n///     fn serialize_seq(self,\n///                      len: Option<usize>)\n///                      -> Result<Self::SerializeSeq, Error> {\n///         // Given Impossible cannot be instantiated, the only\n///         // thing we can do here is to return an error.\n/// #         stringify! {\n///         Err(...)\n/// #         };\n/// #         unimplemented!()\n///     }\n///\n///     /* other Serializer methods */\n/// #     serde::__serialize_unimplemented! {\n/// #         bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str bytes none some\n/// #         unit unit_struct unit_variant newtype_struct newtype_variant\n/// #         tuple tuple_struct tuple_variant map struct struct_variant\n/// #     }\n/// }\n/// ```\n///\n/// [`Serializer`]: trait.Serializer.html\n/// [`SerializeSeq`]: trait.SerializeSeq.html\n/// [`SerializeTuple`]: trait.SerializeTuple.html\n/// [`SerializeTupleStruct`]: trait.SerializeTupleStruct.html\n/// [`SerializeTupleVariant`]: trait.SerializeTupleVariant.html\n/// [`SerializeMap`]: trait.SerializeMap.html\n/// [`SerializeStruct`]: trait.SerializeStruct.html\n/// [`SerializeStructVariant`]: trait.SerializeStructVariant.html\npub struct Impossible<Ok, Error> {\n    void: Void,\n    ok: PhantomData<Ok>,\n    error: PhantomData<Error>,\n}\n\nenum Void {}\n\nimpl<Ok, Error> SerializeSeq for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_element<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeTuple for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_element<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeTupleStruct for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeTupleVariant for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeMap for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_key<T>(&mut self, key: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = key;\n        match self.void {}\n    }\n\n    fn serialize_value<T>(&mut self, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeStruct for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = key;\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\nimpl<Ok, Error> SerializeStructVariant for Impossible<Ok, Error>\nwhere\n    Error: ser::Error,\n{\n    type Ok = Ok;\n    type Error = Error;\n\n    fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Error>\n    where\n        T: ?Sized + Serialize,\n    {\n        let _ = key;\n        let _ = value;\n        match self.void {}\n    }\n\n    fn end(self) -> Result<Ok, Error> {\n        match self.void {}\n    }\n}\n\n// Path: serde/src/ser/fmt.rs\nuse crate::lib::*;\nuse crate::ser::{Error, Impossible, Serialize, Serializer};\n\nimpl Error for fmt::Error {\n    fn custom<T: Display>(_msg: T) -> Self {\n        fmt::Error\n    }\n}\n\nmacro_rules! fmt_primitives {\n    ($($f:ident: $t:ty,)*) => {\n        $(\n            fn $f(self, v: $t) -> fmt::Result {\n                Display::fmt(&v, self)\n            }\n        )*\n    };\n}\n\n/// ```edition2021\n/// use serde::ser::Serialize;\n/// use serde_derive::Serialize;\n/// use std::fmt::{self, Display};\n///\n/// #[derive(Serialize)]\n/// #[serde(rename_all = \"kebab-case\")]\n/// pub enum MessageType {\n///     StartRequest,\n///     EndRequest,\n/// }\n///\n/// impl Display for MessageType {\n///     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {\n///         self.serialize(f)\n///     }\n/// }\n/// ```\nimpl<'a, 'b> Serializer for &'a mut fmt::Formatter<'b> {\n    type Ok = ();\n    type Error = fmt::Error;\n    type SerializeSeq = Impossible<(), fmt::Error>;\n    type SerializeTuple = Impossible<(), fmt::Error>;\n    type SerializeTupleStruct = Impossible<(), fmt::Error>;\n    type SerializeTupleVariant = Impossible<(), fmt::Error>;\n    type SerializeMap = Impossible<(), fmt::Error>;\n    type SerializeStruct = Impossible<(), fmt::Error>;\n    type SerializeStructVariant = Impossible<(), fmt::Error>;\n\n    fmt_primitives! {\n        serialize_bool: bool,\n        serialize_i8: i8,\n        serialize_i16: i16,\n        serialize_i32: i32,\n        serialize_i64: i64,\n        serialize_i128: i128,\n        serialize_u8: u8,\n        serialize_u16: u16,\n        serialize_u32: u32,\n        serialize_u64: u64,\n        serialize_u128: u128,\n        serialize_f32: f32,\n        serialize_f64: f64,\n        serialize_char: char,\n        serialize_str: &str,\n        serialize_unit_struct: &'static str,\n    }\n\n    \nfn serialize_unit_variant(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        variant: &'static str,\n    ) -> fmt::Result {\n        Display::fmt(variant, self)\n    }\n\n    fn serialize_newtype_struct<T>(self, _name: &'static str, value: &T) -> fmt::Result\n    where\n        T: ?Sized + Serialize,\n    {\n        Serialize::serialize(value, self)\n    }\n\n    fn serialize_bytes(self, _v: &[u8]) -> fmt::Result {\n        Err(fmt::Error)\n    }\n\n    fn serialize_none(self) -> fmt::Result {\n        Err(fmt::Error)\n    }\n\n    fn serialize_some<T>(self, _value: &T) -> fmt::Result\n    where\n        T: ?Sized + Serialize,\n    {\n        Err(fmt::Error)\n    }\n\n    fn serialize_unit(self) -> fmt::Result {\n        Err(fmt::Error)\n    }\n\n    fn serialize_newtype_variant<T>(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        _variant: &'static str,\n        _value: &T,\n    ) -> fmt::Result\n    where\n        T: ?Sized + Serialize,\n    {\n        Err(fmt::Error)\n    }\n\n    fn serialize_seq(self, _len: Option<usize>) -> Result<Self::SerializeSeq, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_tuple(self, _len: usize) -> Result<Self::SerializeTuple, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_tuple_struct(\n        self,\n        _name: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeTupleStruct, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_tuple_variant(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        _variant: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeTupleVariant, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_map(self, _len: Option<usize>) -> Result<Self::SerializeMap, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_struct(\n        self,\n        _name: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeStruct, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn serialize_struct_variant(\n        self,\n        _name: &'static str,\n        _variant_index: u32,\n        _variant: &'static str,\n        _len: usize,\n    ) -> Result<Self::SerializeStructVariant, fmt::Error> {\n        Err(fmt::Error)\n    }\n\n    fn collect_str<T>(self, value: &T) -> fmt::Result\n    where\n        T: ?Sized + Display,\n    {\n        Display::fmt(value, self)\n    }\n}\n\n// Path: serde/src/ser/impls.rs\nuse crate::lib::*;\n\nuse crate::ser::{Error, Serialize, SerializeTuple, Serializer};\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! primitive_impl {\n    ($ty:ident, $method:ident $($cast:tt)*) => {\n        impl Serialize for $ty {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.$method(*self $($cast)*)\n            }\n        }\n    }\n}\n\nprimitive_impl!(bool, serialize_bool);\nprimitive_impl!(isize, serialize_i64 as i64);\nprimitive_impl!(i8, serialize_i8);\nprimitive_impl!(i16, serialize_i16);\nprimitive_impl!(i32, serialize_i32);\nprimitive_impl!(i64, serialize_i64);\nprimitive_impl!(i128, serialize_i128);\nprimitive_impl!(usize, serialize_u64 as u64);\nprimitive_impl!(u8, serialize_u8);\nprimitive_impl!(u16, serialize_u16);\nprimitive_impl!(u32, serialize_u32);\nprimitive_impl!(u64, serialize_u64);\nprimitive_impl!(u128, serialize_u128);\nprimitive_impl!(f32, serialize_f32);\nprimitive_impl!(f64, serialize_f64);\nprimitive_impl!(char, serialize_char);\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for str {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_str(self)\n    }\n}\n\n#[cfg(any(feature = \"std\", feature = \"alloc\"))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl Serialize for String {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_str(self)\n    }\n}\n\nimpl<'a> Serialize for fmt::Arguments<'a> {\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.collect_str(self)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(any(feature = \"std\", not(no_core_cstr)))]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\nimpl Serialize for CStr {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_bytes(self.to_bytes())\n    }\n}\n\n#[cfg(any(feature = \"std\", all(not(no_core_cstr), feature = \"alloc\")))]\n#[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\nimpl Serialize for CString {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_bytes(self.to_bytes())\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for Option<T>\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Some(ref value) => serializer.serialize_some(value),\n            None => serializer.serialize_none(),\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for PhantomData<T>\nwhere\n    T: ?Sized,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_unit_struct(\"PhantomData\")\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n// Does not require T: Serialize.\nimpl<T> Serialize for [T; 0] {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        tri!(serializer.serialize_tuple(0)).end()\n    }\n}\n\nmacro_rules! array_impls {\n    ($($len:tt)+) => {\n        $(\n            impl<T> Serialize for [T; $len]\n            where\n                T: Serialize,\n            {\n                #[inline]\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    let mut seq = tri!(serializer.serialize_tuple($len));\n                    for e in self {\n                        tri!(seq.serialize_element(e));\n                    }\n                    seq.end()\n                }\n            }\n        )+\n    }\n}\n\narray_impls! {\n    01 02 03 04 05 06 07 08 09 10\n    11 12 13 14 15 16 17 18 19 20\n    21 22 23 24 25 26 27 28 29 30\n    31 32\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for [T]\nwhere\n    T: Serialize,\n{\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.collect_seq(self)\n    }\n}\n\n#[cfg(not(no_relaxed_trait_bounds))]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize,\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\n#[cfg(no_relaxed_trait_bounds)]\nmacro_rules! seq_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <T $(: $tbound1:ident $(+ $tbound2:ident)*)* $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<T $(, $typaram)*> Serialize for $ty<T $(, $typaram)*>\n        where\n            T: Serialize $(+ $tbound1 $(+ $tbound2)*)*,\n            $($typaram: $bound,)*\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_seq(self)\n            }\n        }\n    }\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BinaryHeap<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeSet<T: Ord>\n}\n\nseq_impl! {\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashSet<T: Eq + Hash, H: BuildHasher>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    LinkedList<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    Vec<T>\n}\n\nseq_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    VecDeque<T>\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for Range<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"Range\", 2));\n        tri!(state.serialize_field(\"start\", &self.start));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeFrom<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeFrom\", 1));\n        tri!(state.serialize_field(\"start\", &self.start));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeInclusive<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeInclusive\", 2));\n        tri!(state.serialize_field(\"start\", &self.start()));\n        tri!(state.serialize_field(\"end\", &self.end()));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<Idx> Serialize for RangeTo<Idx>\nwhere\n    Idx: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        use super::SerializeStruct;\n        let mut state = tri!(serializer.serialize_struct(\"RangeTo\", 1));\n        tri!(state.serialize_field(\"end\", &self.end));\n        state.end()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl<T> Serialize for Bound<T>\nwhere\n    T: Serialize,\n{\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        match *self {\n            Bound::Unbounded => serializer.serialize_unit_variant(\"Bound\", 0, \"Unbounded\"),\n            Bound::Included(ref value) => {\n                serializer.serialize_newtype_variant(\"Bound\", 1, \"Included\", value)\n            }\n            Bound::Excluded(ref value) => {\n                serializer.serialize_newtype_variant(\"Bound\", 2, \"Excluded\", value)\n            }\n        }\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nimpl Serialize for () {\n    #[inline]\n    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        serializer.serialize_unit()\n    }\n}\n\n#[cfg(feature = \"unstable\")]\n#[cfg_attr(doc_cfg, doc(cfg(feature = \"unstable\")))]\nimpl Serialize for ! {\n    fn serialize<S>(&self, _serializer: S) -> Result<S::Ok, S::Error>\n    where\n        S: Serializer,\n    {\n        *self\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! tuple_impls {\n    ($($len:expr => ($($n:tt $name:ident)+))+) => {\n        $(\n            impl<$($name),+> Serialize for ($($name,)+)\n            where\n                $($name: Serialize,)+\n            {\n                #[inline]\n                fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n                where\n                    S: Serializer,\n                {\n                    let mut tuple = tri!(serializer.serialize_tuple($len));\n                    $(\n                        tri!(tuple.serialize_element(&self.$n));\n                    )+\n                    tuple.end()\n                }\n            }\n        )+\n    }\n}\n\ntuple_impls! {\n    1 => (0 T0)\n    2 => (0 T0 1 T1)\n    3 => (0 T0 1 T1 2 T2)\n    4 => (0 T0 1 T1 2 T2 3 T3)\n    5 => (0 T0 1 T1 2 T2 3 T3 4 T4)\n    6 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5)\n    7 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6)\n    8 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7)\n    9 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8)\n    10 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9)\n    11 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10)\n    12 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11)\n    13 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12)\n    14 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13)\n    15 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14)\n    16 => (0 T0 1 T1 2 T2 3 T3 4 T4 5 T5 6 T6 7 T7 8 T8 9 T9 10 T10 11 T11 12 T12 13 T13 14 T14 15 T15)\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n#[cfg(not(no_relaxed_trait_bounds))]\nmacro_rules! map_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <K $(: $kbound1:ident $(+ $kbound2:ident)*)*, V $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<K, V $(, $typaram)*> Serialize for $ty<K, V $(, $typaram)*>\n        where\n            K: Serialize,\n            V: Serialize,\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_map(self)\n            }\n        }\n    }\n}\n\n#[cfg(no_relaxed_trait_bounds)]\nmacro_rules! map_impl {\n    (\n        $(#[$attr:meta])*\n        $ty:ident <K $(: $kbound1:ident $(+ $kbound2:ident)*)*, V $(, $typaram:ident : $bound:ident)*>\n    ) => {\n        $(#[$attr])*\n        impl<K, V $(, $typaram)*> Serialize for $ty<K, V $(, $typaram)*>\n        where\n            K: Serialize $(+ $kbound1 $(+ $kbound2)*)*,\n            V: Serialize,\n            $($typaram: $bound,)*\n        {\n            #[inline]\n            fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n            where\n                S: Serializer,\n            {\n                serializer.collect_map(self)\n            }\n        }\n    }\n}\n\nmap_impl! {\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    BTreeMap<K: Ord, V>\n}\n\nmap_impl! {\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(doc_cfg, doc(cfg(feature = \"std\")))]\n    HashMap<K: Eq + Hash, V, H: BuildHasher>\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\nmacro_rules! deref_impl {\n    (\n        $(#[$attr:meta])*\n        <$($desc:tt)+\n    ) => {\n        $(#[$attr])*\n        impl <$($desc)+ {\n            #[inline]\n...\n", "needle_token_start": 10479, "needle_token_end": 10541, "code_context_ntokens": 16157, "output": ["```rust\nfn serialize_unit_variant(\n    self,\n    _name: &'static str,\n    _variant_index: u32,\n    variant: &'static str,\n) -> fmt::Result {\n    Display::fmt(variant, self)\n}\n```"]}
{"repo": "serde-rs/serde", "name": "visit_i8", "language": "rust", "path": "serde/src/de/mod.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To process an 8-bit signed integer during deserialization and convert it to a different type as defined by the visitor's implementation.\n2. **Input**: An 8-bit signed integer (`i8`).\n3. **Output**: The result of the conversion, which could be any type as specified by the visitor's implementation, or an error if the conversion fails.\n4. **Procedure**: The function takes the 8-bit signed integer, casts it to a 64-bit signed integer (`i64`), and then delegates the processing to another function designed to handle 64-bit integers. This allows for a unified handling of numeric types by leveraging existing implementations for larger types.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            }\n\n            /// Raised when a `Deserialize` receives a value of the right type but that\n            /// is wrong for some other reason.\n            ///\n            /// The `unexp` argument provides information about what value was received.\n            /// This is the value that was present in the input file or other source\n            /// data of the Deserializer.\n            ///\n            /// The `exp` argument provides information about what value was being\n            /// expected. This is the type that is written in the program.\n            ///\n            /// For example if we try to deserialize a String out of some binary data\n            /// that is not valid UTF-8, the unexpected value is the bytes and the\n            /// expected value is a string.\n            #[cold]\n            fn invalid_value(unexp: Unexpected, exp: &Expected) -> Self {\n                Error::custom(format_args!(\"invalid value: {}, expected {}\", unexp, exp))\n            }\n\n            /// Raised when deserializing a sequence or map and the input data contains\n            /// too many or too few elements.\n            ///\n            /// The `len` argument is the number of elements encountered. The sequence\n            /// or map may have expected more arguments or fewer arguments.\n            ///\n            /// The `exp` argument provides information about what data was being\n            /// expected. For example `exp` might say that a tuple of size 6 was\n            /// expected.\n            #[cold]\n            fn invalid_length(len: usize, exp: &Expected) -> Self {\n                Error::custom(format_args!(\"invalid length {}, expected {}\", len, exp))\n            }\n\n            /// Raised when a `Deserialize` enum type received a variant with an\n            /// unrecognized name.\n            #[cold]\n            fn unknown_variant(variant: &str, expected: &'static [&'static str]) -> Self {\n                if expected.is_empty() {\n                    Error::custom(format_args!(\n                        \"unknown variant `{}`, there are no variants\",\n                        variant\n                    ))\n                } else {\n                    Error::custom(format_args!(\n                        \"unknown variant `{}`, expected {}\",\n                        variant,\n                        OneOf { names: expected }\n                    ))\n                }\n            }\n\n            /// Raised when a `Deserialize` struct type received a field with an\n            /// unrecognized name.\n            #[cold]\n            fn unknown_field(field: &str, expected: &'static [&'static str]) -> Self {\n                if expected.is_empty() {\n                    Error::custom(format_args!(\n                        \"unknown field `{}`, there are no fields\",\n                        field\n                    ))\n                } else {\n                    Error::custom(format_args!(\n                        \"unknown field `{}`, expected {}\",\n                        field,\n                        OneOf { names: expected }\n                    ))\n                }\n            }\n\n            /// Raised when a `Deserialize` struct type expected to receive a required\n            /// field with a particular name but that field was not present in the\n            /// input.\n            #[cold]\n            fn missing_field(field: &'static str) -> Self {\n                Error::custom(format_args!(\"missing field `{}`\", field))\n            }\n\n            /// Raised when a `Deserialize` struct type received more than one of the\n            /// same field.\n            #[cold]\n            fn duplicate_field(field: &'static str) -> Self {\n                Error::custom(format_args!(\"duplicate field `{}`\", field))\n            }\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\ndeclare_error_trait!(Error: Sized + StdError);\n\n#[cfg(not(feature = \"std\"))]\ndeclare_error_trait!(Error: Sized + Debug + Display);\n\n/// `Unexpected` represents an unexpected invocation of any one of the `Visitor`\n/// trait methods.\n///\n/// This is used as an argument to the `invalid_type`, `invalid_value`, and\n/// `invalid_length` methods of the `Error` trait to build error messages.\n///\n/// ```edition2021\n/// # use std::fmt;\n/// #\n/// # use serde::de::{self, Unexpected, Visitor};\n/// #\n/// # struct Example;\n/// #\n/// # impl<'de> Visitor<'de> for Example {\n/// #     type Value = ();\n/// #\n/// #     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n/// #         write!(formatter, \"definitely not a boolean\")\n/// #     }\n/// #\n/// fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n/// where\n///     E: de::Error,\n/// {\n///     Err(de::Error::invalid_type(Unexpected::Bool(v), &self))\n/// }\n/// # }\n/// ```\n#[derive(Copy, Clone, PartialEq, Debug)]\npub enum Unexpected<'a> {\n    /// The input contained a boolean value that was not expected.\n    Bool(bool),\n\n    /// The input contained an unsigned integer `u8`, `u16`, `u32` or `u64` that\n    /// was not expected.\n    Unsigned(u64),\n\n    /// The input contained a signed integer `i8`, `i16`, `i32` or `i64` that\n    /// was not expected.\n    Signed(i64),\n\n    /// The input contained a floating point `f32` or `f64` that was not\n    /// expected.\n    Float(f64),\n\n    /// The input contained a `char` that was not expected.\n    Char(char),\n\n    /// The input contained a `&str` or `String` that was not expected.\n    Str(&'a str),\n\n    /// The input contained a `&[u8]` or `Vec<u8>` that was not expected.\n    Bytes(&'a [u8]),\n\n    /// The input contained a unit `()` that was not expected.\n    Unit,\n\n    /// The input contained an `Option<T>` that was not expected.\n    Option,\n\n    /// The input contained a newtype struct that was not expected.\n    NewtypeStruct,\n\n    /// The input contained a sequence that was not expected.\n    Seq,\n\n    /// The input contained a map that was not expected.\n    Map,\n\n    /// The input contained an enum that was not expected.\n    Enum,\n\n    /// The input contained a unit variant that was not expected.\n    UnitVariant,\n\n    /// The input contained a newtype variant that was not expected.\n    NewtypeVariant,\n\n    /// The input contained a tuple variant that was not expected.\n    TupleVariant,\n\n    /// The input contained a struct variant that was not expected.\n    StructVariant,\n\n    /// A message stating what uncategorized thing the input contained that was\n    /// not expected.\n    ///\n    /// The message should be a noun or noun phrase, not capitalized and without\n    /// a period. An example message is \"unoriginal superhero\".\n    Other(&'a str),\n}\n\nimpl<'a> fmt::Display for Unexpected<'a> {\n    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        use self::Unexpected::*;\n        match *self {\n            Bool(b) => write!(formatter, \"boolean `{}`\", b),\n            Unsigned(i) => write!(formatter, \"integer `{}`\", i),\n            Signed(i) => write!(formatter, \"integer `{}`\", i),\n            Float(f) => write!(formatter, \"floating point `{}`\", WithDecimalPoint(f)),\n            Char(c) => write!(formatter, \"character `{}`\", c),\n            Str(s) => write!(formatter, \"string {:?}\", s),\n            Bytes(_) => formatter.write_str(\"byte array\"),\n            Unit => formatter.write_str(\"unit value\"),\n            Option => formatter.write_str(\"Option value\"),\n            NewtypeStruct => formatter.write_str(\"newtype struct\"),\n            Seq => formatter.write_str(\"sequence\"),\n            Map => formatter.write_str(\"map\"),\n            Enum => formatter.write_str(\"enum\"),\n            UnitVariant => formatter.write_str(\"unit variant\"),\n            NewtypeVariant => formatter.write_str(\"newtype variant\"),\n            TupleVariant => formatter.write_str(\"tuple variant\"),\n            StructVariant => formatter.write_str(\"struct variant\"),\n            Other(other) => formatter.write_str(other),\n        }\n    }\n}\n\n/// `Expected` represents an explanation of what data a `Visitor` was expecting\n/// to receive.\n///\n/// This is used as an argument to the `invalid_type`, `invalid_value`, and\n/// `invalid_length` methods of the `Error` trait to build error messages. The\n/// message should be a noun or noun phrase that completes the sentence \"This\n/// Visitor expects to receive ...\", for example the message could be \"an\n/// integer between 0 and 64\". The message should not be capitalized and should\n/// not end with a period.\n///\n/// Within the context of a `Visitor` implementation, the `Visitor` itself\n/// (`&self`) is an implementation of this trait.\n///\n/// ```edition2021\n/// # use serde::de::{self, Unexpected, Visitor};\n/// # use std::fmt;\n/// #\n/// # struct Example;\n/// #\n/// # impl<'de> Visitor<'de> for Example {\n/// #     type Value = ();\n/// #\n/// #     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n/// #         write!(formatter, \"definitely not a boolean\")\n/// #     }\n/// #\n/// fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n/// where\n///     E: de::Error,\n/// {\n///     Err(de::Error::invalid_type(Unexpected::Bool(v), &self))\n/// }\n/// # }\n/// ```\n///\n/// Outside of a `Visitor`, `&\"...\"` can be used.\n///\n/// ```edition2021\n/// # use serde::de::{self, Unexpected};\n/// #\n/// # fn example<E>() -> Result<(), E>\n/// # where\n/// #     E: de::Error,\n/// # {\n/// #     let v = true;\n/// return Err(de::Error::invalid_type(\n///     Unexpected::Bool(v),\n///     &\"a negative integer\",\n/// ));\n/// # }\n/// ```\npub trait Expected {\n    /// Format an explanation of what data was being expected. Same signature as\n    /// the `Display` and `Debug` traits.\n    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result;\n}\n\nimpl<'de, T> Expected for T\nwhere\n    T: Visitor<'de>,\n{\n    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        self.expecting(formatter)\n    }\n}\n\nimpl<'a> Expected for &'a str {\n    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        formatter.write_str(self)\n    }\n}\n\nimpl<'a> Display for Expected + 'a {\n    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n        Expected::fmt(self, formatter)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data structure** that can be deserialized from any data format supported\n/// by Serde.\n///\n/// Serde provides `Deserialize` implementations for many Rust primitive and\n/// standard library types. The complete list is [here][crate::de]. All of these\n/// can be deserialized using Serde out of the box.\n///\n/// Additionally, Serde provides a procedural macro called `serde_derive` to\n/// automatically generate `Deserialize` implementations for structs and enums\n/// in your program. See the [derive section of the manual][derive] for how to\n/// use this.\n///\n/// In rare cases it may be necessary to implement `Deserialize` manually for\n/// some type in your program. See the [Implementing\n/// `Deserialize`][impl-deserialize] section of the manual for more about this.\n///\n/// Third-party crates may provide `Deserialize` implementations for types that\n/// they expose. For example the `linked-hash-map` crate provides a\n/// `LinkedHashMap<K, V>` type that is deserializable by Serde because the crate\n/// provides an implementation of `Deserialize` for it.\n///\n/// [derive]: https://serde.rs/derive.html\n/// [impl-deserialize]: https://serde.rs/impl-deserialize.html\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by `Self` when deserialized. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\npub trait Deserialize<'de>: Sized {\n    /// Deserialize this value from the given Serde deserializer.\n    ///\n    /// See the [Implementing `Deserialize`][impl-deserialize] section of the\n    /// manual for more information about how to implement this method.\n    ///\n    /// [impl-deserialize]: https://serde.rs/impl-deserialize.html\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>;\n\n    /// Deserializes a value into `self` from the given Deserializer.\n    ///\n    /// The purpose of this method is to allow the deserializer to reuse\n    /// resources and avoid copies. As such, if this method returns an error,\n    /// `self` will be in an indeterminate state where some parts of the struct\n    /// have been overwritten. Although whatever state that is will be\n    /// memory-safe.\n    ///\n    /// This is generally useful when repeatedly deserializing values that\n    /// are processed one at a time, where the value of `self` doesn't matter\n    /// when the next deserialization occurs.\n    ///\n    /// If you manually implement this, your recursive deserializations should\n    /// use `deserialize_in_place`.\n    ///\n    /// This method is stable and an official public API, but hidden from the\n    /// documentation because it is almost never what newbies are looking for.\n    /// Showing it in rustdoc would cause it to be featured more prominently\n    /// than it deserves.\n    #[doc(hidden)]\n    fn deserialize_in_place<D>(deserializer: D, place: &mut Self) -> Result<(), D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        // Default implementation just delegates to `deserialize` impl.\n        *place = tri!(Deserialize::deserialize(deserializer));\n        Ok(())\n    }\n}\n\n/// A data structure that can be deserialized without borrowing any data from\n/// the deserializer.\n///\n/// This is primarily useful for trait bounds on functions. For example a\n/// `from_str` function may be able to deserialize a data structure that borrows\n/// from the input string, but a `from_reader` function may only deserialize\n/// owned data.\n///\n/// ```edition2021\n/// # use serde::de::{Deserialize, DeserializeOwned};\n/// # use std::io::{Read, Result};\n/// #\n/// # trait Ignore {\n/// fn from_str<'a, T>(s: &'a str) -> Result<T>\n/// where\n///     T: Deserialize<'a>;\n///\n/// fn from_reader<R, T>(rdr: R) -> Result<T>\n/// where\n///     R: Read,\n///     T: DeserializeOwned;\n/// # }\n/// ```\n///\n/// # Lifetime\n///\n/// The relationship between `Deserialize` and `DeserializeOwned` in trait\n/// bounds is explained in more detail on the page [Understanding deserializer\n/// lifetimes].\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\npub trait DeserializeOwned: for<'de> Deserialize<'de> {}\nimpl<T> DeserializeOwned for T where T: for<'de> Deserialize<'de> {}\n\n/// `DeserializeSeed` is the stateful form of the `Deserialize` trait. If you\n/// ever find yourself looking for a way to pass data into a `Deserialize` impl,\n/// this trait is the way to do it.\n///\n/// As one example of stateful deserialization consider deserializing a JSON\n/// array into an existing buffer. Using the `Deserialize` trait we could\n/// deserialize a JSON array into a `Vec<T>` but it would be a freshly allocated\n/// `Vec<T>`; there is no way for `Deserialize` to reuse a previously allocated\n/// buffer. Using `DeserializeSeed` instead makes this possible as in the\n/// example code below.\n///\n/// The canonical API for stateless deserialization looks like this:\n///\n/// ```edition2021\n/// # use serde::Deserialize;\n/// #\n/// # enum Error {}\n/// #\n/// fn func<'de, T: Deserialize<'de>>() -> Result<T, Error>\n/// # {\n/// #     unimplemented!()\n/// # }\n/// ```\n///\n/// Adjusting an API like this to support stateful deserialization is a matter\n/// of accepting a seed as input:\n///\n/// ```edition2021\n/// # use serde::de::DeserializeSeed;\n/// #\n/// # enum Error {}\n/// #\n/// fn func_seed<'de, T: DeserializeSeed<'de>>(seed: T) -> Result<T::Value, Error>\n/// # {\n/// #     let _ = seed;\n/// #     unimplemented!()\n/// # }\n/// ```\n///\n/// In practice the majority of deserialization is stateless. An API expecting a\n/// seed can be appeased by passing `std::marker::PhantomData` as a seed in the\n/// case of stateless deserialization.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by `Self::Value` when deserialized. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example\n///\n/// Suppose we have JSON that looks like `[[1, 2], [3, 4, 5], [6]]` and we need\n/// to deserialize it into a flat representation like `vec![1, 2, 3, 4, 5, 6]`.\n/// Allocating a brand new `Vec<T>` for each subarray would be slow. Instead we\n/// would like to allocate a single `Vec<T>` and then deserialize each subarray\n/// into it. This requires stateful deserialization using the `DeserializeSeed`\n/// trait.\n///\n/// ```edition2021\n/// use serde::de::{Deserialize, DeserializeSeed, Deserializer, SeqAccess, Visitor};\n/// use std::fmt;\n/// use std::marker::PhantomData;\n///\n/// // A DeserializeSeed implementation that uses stateful deserialization to\n/// // append array elements onto the end of an existing vector. The preexisting\n/// // state (\"seed\") in this case is the Vec<T>. The `deserialize` method of\n/// // `ExtendVec` will be traversing the inner arrays of the JSON input and\n/// // appending each integer into the existing Vec.\n/// struct ExtendVec<'a, T: 'a>(&'a mut Vec<T>);\n///\n/// impl<'de, 'a, T> DeserializeSeed<'de> for ExtendVec<'a, T>\n/// where\n///     T: Deserialize<'de>,\n/// {\n///     // The return type of the `deserialize` method. This implementation\n///     // appends onto an existing vector but does not create any new data\n///     // structure, so the return type is ().\n///     type Value = ();\n///\n///     fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n///     where\n///         D: Deserializer<'de>,\n///     {\n///         // Visitor implementation that will walk an inner array of the JSON\n///         // input.\n///         struct ExtendVecVisitor<'a, T: 'a>(&'a mut Vec<T>);\n///\n///         impl<'de, 'a, T> Visitor<'de> for ExtendVecVisitor<'a, T>\n///         where\n///             T: Deserialize<'de>,\n///         {\n///             type Value = ();\n///\n///             fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///                 write!(formatter, \"an array of integers\")\n///             }\n///\n///             fn visit_seq<A>(self, mut seq: A) -> Result<(), A::Error>\n///             where\n///                 A: SeqAccess<'de>,\n///             {\n///                 // Decrease the number of reallocations if there are many elements\n///                 if let Some(size_hint) = seq.size_hint() {\n///                     self.0.reserve(size_hint);\n///                 }\n///\n///                 // Visit each element in the inner array and push it onto\n///                 // the existing vector.\n///                 while let Some(elem) = seq.next_element()? {\n///                     self.0.push(elem);\n///                 }\n///                 Ok(())\n///             }\n///         }\n///\n///         deserializer.deserialize_seq(ExtendVecVisitor(self.0))\n///     }\n/// }\n///\n/// // Visitor implementation that will walk the outer array of the JSON input.\n/// struct FlattenedVecVisitor<T>(PhantomData<T>);\n///\n/// impl<'de, T> Visitor<'de> for FlattenedVecVisitor<T>\n/// where\n///     T: Deserialize<'de>,\n/// {\n///     // This Visitor constructs a single Vec<T> to hold the flattened\n///     // contents of the inner arrays.\n///     type Value = Vec<T>;\n///\n///     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///         write!(formatter, \"an array of arrays\")\n///     }\n///\n///     fn visit_seq<A>(self, mut seq: A) -> Result<Vec<T>, A::Error>\n///     where\n///         A: SeqAccess<'de>,\n///     {\n///         // Create a single Vec to hold the flattened contents.\n///         let mut vec = Vec::new();\n///\n///         // Each iteration through this loop is one inner array.\n///         while let Some(()) = seq.next_element_seed(ExtendVec(&mut vec))? {\n///             // Nothing to do; inner array has been appended into `vec`.\n///         }\n///\n///         // Return the finished vec.\n///         Ok(vec)\n///     }\n/// }\n///\n/// # fn example<'de, D>(deserializer: D) -> Result<(), D::Error>\n/// # where\n/// #     D: Deserializer<'de>,\n/// # {\n/// let visitor = FlattenedVecVisitor(PhantomData);\n/// let flattened: Vec<u64> = deserializer.deserialize_seq(visitor)?;\n/// #     Ok(())\n/// # }\n/// ```\npub trait DeserializeSeed<'de>: Sized {\n    /// The type produced by using this seed.\n    type Value;\n\n    /// Equivalent to the more common `Deserialize::deserialize` method, except\n    /// with some initial piece of data (the seed) passed in.\n    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>;\n}\n\nimpl<'de, T> DeserializeSeed<'de> for PhantomData<T>\nwhere\n    T: Deserialize<'de>,\n{\n    type Value = T;\n\n    #[inline]\n    fn deserialize<D>(self, deserializer: D) -> Result<T, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        T::deserialize(deserializer)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data format** that can deserialize any data structure supported by\n/// Serde.\n///\n/// The role of this trait is to define the deserialization half of the [Serde\n/// data model], which is a way to categorize every Rust data type into one of\n/// 29 possible types. Each method of the `Deserializer` trait corresponds to one\n/// of the types of the data model.\n///\n/// Implementations of `Deserialize` map themselves into this data model by\n/// passing to the `Deserializer` a `Visitor` implementation that can receive\n/// these various types.\n///\n/// The types that make up the Serde data model are:\n///\n///  - **14 primitive types**\n///    - bool\n///    - i8, i16, i32, i64, i128\n///    - u8, u16, u32, u64, u128\n///    - f32, f64\n///    - char\n///  - **string**\n///    - UTF-8 bytes with a length and no null terminator.\n///    - When serializing, all strings are handled equally. When deserializing,\n///      there are three flavors of strings: transient, owned, and borrowed.\n///  - **byte array** - \\[u8\\]\n///    - Similar to strings, during deserialization byte arrays can be\n///      transient, owned, or borrowed.\n///  - **option**\n///    - Either none or some value.\n///  - **unit**\n///    - The type of `()` in Rust. It represents an anonymous value containing\n///      no data.\n///  - **unit_struct**\n///    - For example `struct Unit` or `PhantomData<T>`. It represents a named\n///      value containing no data.\n///  - **unit_variant**\n///    - For example the `E::A` and `E::B` in `enum E { A, B }`.\n///  - **newtype_struct**\n///    - For example `struct Millimeters(u8)`.\n///  - **newtype_variant**\n///    - For example the `E::N` in `enum E { N(u8) }`.\n///  - **seq**\n///    - A variably sized heterogeneous sequence of values, for example `Vec<T>`\n///      or `HashSet<T>`. When serializing, the length may or may not be known\n///      before iterating through all the data. When deserializing, the length\n///      is determined by looking at the serialized data.\n///  - **tuple**\n///    - A statically sized heterogeneous sequence of values for which the\n///      length will be known at deserialization time without looking at the\n///      serialized data, for example `(u8,)` or `(String, u64, Vec<T>)` or\n///      `[u64; 10]`.\n///  - **tuple_struct**\n///    - A named tuple, for example `struct Rgb(u8, u8, u8)`.\n///  - **tuple_variant**\n///    - For example the `E::T` in `enum E { T(u8, u8) }`.\n///  - **map**\n///    - A heterogeneous key-value pairing, for example `BTreeMap<K, V>`.\n///  - **struct**\n///    - A heterogeneous key-value pairing in which the keys are strings and\n///      will be known at deserialization time without looking at the serialized\n///      data, for example `struct S { r: u8, g: u8, b: u8 }`.\n///  - **struct_variant**\n///    - For example the `E::S` in `enum E { S { r: u8, g: u8, b: u8 } }`.\n///\n/// The `Deserializer` trait supports two entry point styles which enables\n/// different kinds of deserialization.\n///\n/// 1. The `deserialize_any` method. Self-describing data formats like JSON are\n///    able to look at the serialized data and tell what it represents. For\n///    example the JSON deserializer may see an opening curly brace (`{`) and\n///    know that it is seeing a map. If the data format supports\n///    `Deserializer::deserialize_any`, it will drive the Visitor using whatever\n///    type it sees in the input. JSON uses this approach when deserializing\n///    `serde_json::Value` which is an enum that can represent any JSON\n///    document. Without knowing what is in a JSON document, we can deserialize\n///    it to `serde_json::Value` by going through\n///    `Deserializer::deserialize_any`.\n///\n/// 2. The various `deserialize_*` methods. Non-self-describing formats like\n///    Postcard need to be told what is in the input in order to deserialize it.\n///    The `deserialize_*` methods are hints to the deserializer for how to\n///    interpret the next piece of input. Non-self-describing formats are not\n///    able to deserialize something like `serde_json::Value` which relies on\n///    `Deserializer::deserialize_any`.\n///\n/// When implementing `Deserialize`, you should avoid relying on\n/// `Deserializer::deserialize_any` unless you need to be told by the\n/// Deserializer what type is in the input. Know that relying on\n/// `Deserializer::deserialize_any` means your data type will be able to\n/// deserialize from self-describing formats only, ruling out Postcard and many\n/// others.\n///\n/// [Serde data model]: https://serde.rs/data-model.html\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed from the input when deserializing. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website contains example code for\n/// a basic JSON `Deserializer`.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait Deserializer<'de>: Sized {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// Require the `Deserializer` to figure out how to drive the visitor based\n    /// on what data type is in the input.\n    ///\n    /// When implementing `Deserialize`, you should avoid relying on\n    /// `Deserializer::deserialize_any` unless you need to be told by the\n    /// Deserializer what type is in the input. Know that relying on\n    /// `Deserializer::deserialize_any` means your data type will be able to\n    /// deserialize from self-describing formats only, ruling out Postcard and\n    /// many others.\n    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `bool` value.\n    fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i8` value.\n    fn deserialize_i8<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i16` value.\n    fn deserialize_i16<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i32` value.\n    fn deserialize_i32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i64` value.\n    fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i128` value.\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>,\n    {\n        let _ = visitor;\n        Err(Error::custom(\"i128 is not supported\"))\n    }\n\n    /// Hint that the `Deserialize` type is expecting a `u8` value.\n    fn deserialize_u8<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u16` value.\n    fn deserialize_u16<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u32` value.\n    fn deserialize_u32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u64` value.\n    fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `u128` value.\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>,\n    {\n        let _ = visitor;\n        Err(Error::custom(\"u128 is not supported\"))\n    }\n\n    /// Hint that the `Deserialize` type is expecting a `f32` value.\n    fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `f64` value.\n    fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `char` value.\n    fn deserialize_char<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a string value and does\n    /// not benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would benefit from taking ownership of `String` data,\n    /// indicate this to the `Deserializer` by using `deserialize_string`\n    /// instead.\n    fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a string value and would\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would not benefit from taking ownership of `String`\n    /// data, indicate that to the `Deserializer` by using `deserialize_str`\n    /// instead.\n    fn deserialize_string<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a byte array and does not\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would benefit from taking ownership of `Vec<u8>` data,\n    /// indicate this to the `Deserializer` by using `deserialize_byte_buf`\n    /// instead.\n    fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a byte array and would\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would not benefit from taking ownership of `Vec<u8>`\n    /// data, indicate that to the `Deserializer` by using `deserialize_bytes`\n    /// instead.\n    fn deserialize_byte_buf<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an optional value.\n    ///\n    /// This allows deserializers that encode an optional value as a nullable\n    /// value to convert the null value into `None` and a regular value into\n    /// `Some(value)`.\n    fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a unit value.\n    fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a unit struct with a\n    /// particular name.\n    fn deserialize_unit_struct<V>(\n        self,\n        name: &'static str,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a newtype struct with a\n    /// particular name.\n    fn deserialize_newtype_struct<V>(\n        self,\n        name: &'static str,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a sequence of values.\n    fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a sequence of values and\n    /// knows how many values there are without looking at the serialized data.\n    fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a tuple struct with a\n    /// particular name and number of fields.\n    fn deserialize_tuple_struct<V>(\n        self,\n        name: &'static str,\n        len: usize,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a map of key-value pairs.\n    fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a struct with a particular\n    /// name and fields.\n    fn deserialize_struct<V>(\n        self,\n        name: &'static str,\n        fields: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an enum value with a\n    /// particular name and possible variants.\n    fn deserialize_enum<V>(\n        self,\n        name: &'static str,\n        variants: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting the name of a struct\n    /// field or the discriminant of an enum variant.\n    fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type needs to deserialize a value whose type\n    /// doesn't matter because it is ignored.\n    ///\n    /// Deserializers for non-self-describing formats may not support this mode.\n    fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Determine whether `Deserialize` implementations should expect to\n    /// deserialize their human-readable form.\n    ///\n    /// Some types have a human-readable form that may be somewhat expensive to\n    /// construct, as well as a binary form that is compact and efficient.\n    /// Generally text-based formats like JSON and YAML will prefer to use the\n    /// human-readable one and binary formats like Postcard will prefer the\n    /// compact one.\n    ///\n    /// ```edition2021\n    /// # use std::ops::Add;\n    /// # use std::str::FromStr;\n    /// #\n    /// # struct Timestamp;\n    /// #\n    /// # impl Timestamp {\n    /// #     const EPOCH: Timestamp = Timestamp;\n    /// # }\n    /// #\n    /// # impl FromStr for Timestamp {\n    /// #     type Err = String;\n    /// #     fn from_str(_: &str) -> Result<Self, Self::Err> {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// # struct Duration;\n    /// #\n    /// # impl Duration {\n    /// #     fn seconds(_: u64) -> Self { unimplemented!() }\n    /// # }\n    /// #\n    /// # impl Add<Duration> for Timestamp {\n    /// #     type Output = Timestamp;\n    /// #     fn add(self, _: Duration) -> Self::Output {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::de::{self, Deserialize, Deserializer};\n    ///\n    /// impl<'de> Deserialize<'de> for Timestamp {\n    ///     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    ///     where\n    ///         D: Deserializer<'de>,\n    ///     {\n    ///         if deserializer.is_human_readable() {\n    ///             // Deserialize from a human-readable string like \"2015-05-15T17:01:00Z\".\n    ///             let s = String::deserialize(deserializer)?;\n    ///             Timestamp::from_str(&s).map_err(de::Error::custom)\n    ///         } else {\n    ///             // Deserialize from a compact binary representation, seconds since\n    ///             // the Unix epoch.\n    ///             let n = u64::deserialize(deserializer)?;\n    ///             Ok(Timestamp::EPOCH + Duration::seconds(n))\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default implementation of this method returns `true`. Data formats\n    /// may override this to `false` to request a compact form for types that\n    /// support one. Note that modifying this method to change a format from\n    /// human-readable to compact or vice versa should be regarded as a breaking\n    /// change, as a value serialized in human-readable mode is not required to\n    /// deserialize from the same data in compact mode.\n    #[inline]\n    fn is_human_readable(&self) -> bool {\n        true\n    }\n\n    // Not public API.\n    #[cfg(all(not(no_serde_derive), any(feature = \"std\", feature = \"alloc\")))]\n    #[doc(hidden)]\n    fn __deserialize_content<V>(\n        self,\n        _: crate::actually_private::T,\n        visitor: V,\n    ) -> Result<crate::__private::de::Content<'de>, Self::Error>\n    where\n        V: Visitor<'de, Value = crate::__private::de::Content<'de>>,\n    {\n        self.deserialize_any(visitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// This trait represents a visitor that walks through a deserializer.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the requirement for lifetime of data\n/// that may be borrowed by `Self::Value`. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example\n///\n/// ```edition2021\n/// # use serde::de::{self, Unexpected, Visitor};\n/// # use std::fmt;\n/// #\n/// /// A visitor that deserializes a long string - a string containing at least\n/// /// some minimum number of bytes.\n/// struct LongString {\n///     min: usize,\n/// }\n///\n/// impl<'de> Visitor<'de> for LongString {\n///     type Value = String;\n///\n///     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///         write!(formatter, \"a string containing at least {} bytes\", self.min)\n///     }\n///\n///     fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>\n///     where\n///         E: de::Error,\n///     {\n///         if s.len() >= self.min {\n///             Ok(s.to_owned())\n///         } else {\n///             Err(de::Error::invalid_value(Unexpected::Str(s), &self))\n///         }\n///     }\n/// }\n/// ```\npub trait Visitor<'de>: Sized {\n    /// The value produced by this visitor.\n    type Value;\n\n    /// Format a message stating what data this Visitor expects to receive.\n    ///\n    /// This is used in error messages. The message should complete the sentence\n    /// \"This Visitor expects to receive ...\", for example the message could be\n    /// \"an integer between 0 and 64\". The message should not be capitalized and\n    /// should not end with a period.\n    ///\n    /// ```edition2021\n    /// # use std::fmt;\n    /// #\n    /// # struct S {\n    /// #     max: usize,\n    /// # }\n    /// #\n    /// # impl<'de> serde::de::Visitor<'de> for S {\n    /// #     type Value = ();\n    /// #\n    /// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n    ///     write!(formatter, \"an integer between 0 and {}\", self.max)\n    /// }\n    /// # }\n    /// ```\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result;\n\n    /// The input contains a boolean.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Bool(v), &self))\n    }\n\n    /// The input contains an `i8`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    \nfn visit_i8<E>(self, v: i8) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i16`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i16<E>(self, v: i16) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i32`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i32<E>(self, v: i32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Signed(v), &self))\n    }\n\n    /// The input contains a `i128`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_i128<E>(self, v: i128) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut buf = [0u8; 58];\n        let mut writer = format::Buf::new(&mut buf);\n        fmt::Write::write_fmt(&mut writer, format_args!(\"integer `{}` as i128\", v)).unwrap();\n        Err(Error::invalid_type(\n            Unexpected::Other(writer.as_str()),\n            &self,\n        ))\n    }\n\n    /// The input contains a `u8`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u8<E>(self, v: u8) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u16`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u32`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u32<E>(self, v: u32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Unsigned(v), &self))\n    }\n\n    /// The input contains a `u128`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_u128<E>(self, v: u128) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut buf = [0u8; 57];\n        let mut writer = format::Buf::new(&mut buf);\n        fmt::Write::write_fmt(&mut writer, format_args!(\"integer `{}` as u128\", v)).unwrap();\n        Err(Error::invalid_type(\n            Unexpected::Other(writer.as_str()),\n            &self,\n        ))\n    }\n\n    /// The input contains an `f32`.\n    ///\n    /// The default implementation forwards to [`visit_f64`].\n    ///\n    /// [`visit_f64`]: #method.visit_f64\n    fn visit_f32<E>(self, v: f32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_f64(v as f64)\n    }\n\n    /// The input contains an `f64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Float(v), &self))\n    }\n\n    /// The input contains a `char`.\n    ///\n    /// The default implementation forwards to [`visit_str`] as a one-character\n    /// string.\n    ///\n    /// [`visit_str`]: #method.visit_str\n    #[inline]\n    fn visit_char<E>(self, v: char) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(v.encode_utf8(&mut [0u8; 4]))\n    }\n\n    /// The input contains a string. The lifetime of the string is ephemeral and\n    /// it may be destroyed after this method returns.\n    ///\n    /// This method allows the `Deserializer` to avoid a copy by retaining\n    /// ownership of any buffered data. `Deserialize` implementations that do\n    /// not benefit from taking ownership of `String` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_str` rather than\n    /// `Deserializer::deserialize_string`.\n    ///\n    /// It is never correct to implement `visit_string` without implementing\n    /// `visit_str`. Implement neither, both, or just `visit_str`.\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Str(v), &self))\n    }\n\n    /// The input contains a string that lives at least as long as the\n    /// `Deserializer`.\n    ///\n    /// This enables zero-copy deserialization of strings in some formats. For\n    /// example JSON input containing the JSON string `\"borrowed\"` can be\n    /// deserialized with zero copying into a `&'a str` as long as the input\n    /// data outlives `'a`.\n    ///\n    /// The default implementation forwards to `visit_str`.\n    #[inline]\n    fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(v)\n    }\n\n    /// The input contains a string and ownership of the string is being given\n    /// to the `Visitor`.\n    ///\n    /// This method allows the `Visitor` to avoid a copy by taking ownership of\n    /// a string created by the `Deserializer`. `Deserialize` implementations\n    /// that benefit from taking ownership of `String` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_string` rather\n    /// than `Deserializer::deserialize_str`, although not every deserializer\n    /// will honor such a request.\n    ///\n    /// It is never correct to implement `visit_string` without implementing\n    /// `visit_str`. Implement neither, both, or just `visit_str`.\n    ///\n    /// The default implementation forwards to `visit_str` and then drops the\n    /// `String`.\n    #[inline]\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(&v)\n    }\n\n    /// The input contains a byte array. The lifetime of the byte array is\n    /// ephemeral and it may be destroyed after this method returns.\n    ///\n    /// This method allows the `Deserializer` to avoid a copy by retaining\n    /// ownership of any buffered data. `Deserialize` implementations that do\n    /// not benefit from taking ownership of `Vec<u8>` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_bytes` rather\n    /// than `Deserializer::deserialize_byte_buf`.\n    ///\n    /// It is never correct to implement `visit_byte_buf` without implementing\n    /// `visit_bytes`. Implement neither, both, or just `visit_bytes`.\n    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Bytes(v), &self))\n    }\n\n    /// The input contains a byte array that lives at least as long as the\n    /// `Deserializer`.\n    ///\n    /// This enables zero-copy deserialization of bytes in some formats. For\n    /// example Postcard data containing bytes can be deserialized with zero\n    /// copying into a `&'a [u8]` as long as the input data outlives `'a`.\n    ///\n    /// The default implementation forwards to `visit_bytes`.\n    #[inline]\n    fn visit_borrowed_bytes<E>(self, v: &'de [u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_bytes(v)\n    }\n\n    /// The input contains a byte array and ownership of the byte array is being\n    /// given to the `Visitor`.\n    ///\n    /// This method allows the `Visitor` to avoid a copy by taking ownership of\n    /// a byte buffer created by the `Deserializer`. `Deserialize`\n    /// implementations that benefit from taking ownership of `Vec<u8>` data\n    /// should indicate that to the deserializer by using\n    /// `Deserializer::deserialize_byte_buf` rather than\n    /// `Deserializer::deserialize_bytes`, although not every deserializer will\n    /// honor such a request.\n    ///\n    /// It is never correct to implement `visit_byte_buf` without implementing\n    /// `visit_bytes`. Implement neither, both, or just `visit_bytes`.\n    ///\n    /// The default implementation forwards to `visit_bytes` and then drops the\n    /// `Vec<u8>`.\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_bytes(&v)\n    }\n\n    /// The input contains an optional that is absent.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_none<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Option, &self))\n    }\n\n    /// The input contains an optional that is present.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let _ = deserializer;\n        Err(Error::invalid_type(Unexpected::Option, &self))\n    }\n\n    /// The input contains a unit `()`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Unit, &self))\n    }\n\n    /// The input contains a newtype struct.\n    ///\n    /// The content of the newtype struct may be read from the given\n    /// `Deserializer`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let _ = deserializer;\n        Err(Error::invalid_type(Unexpected::NewtypeStruct, &self))\n    }\n\n    /// The input contains a sequence of elements.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>\n    where\n        A: SeqAccess<'de>,\n    {\n        let _ = seq;\n        Err(Error::invalid_type(Unexpected::Seq, &self))\n    }\n\n    /// The input contains a key-value map.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>\n    where\n        A: MapAccess<'de>,\n    {\n        let _ = map;\n        Err(Error::invalid_type(Unexpected::Map, &self))\n    }\n\n    /// The input contains an enum.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>\n    where\n        A: EnumAccess<'de>,\n    {\n        let _ = data;\n        Err(Error::invalid_type(Unexpected::Enum, &self))\n    }\n\n    // Used when deserializing a flattened Option field. Not public API.\n    #[doc(hidden)]\n    fn __private_visit_untagged_option<D>(self, _: D) -> Result<Self::Value, ()>\n    where\n        D: Deserializer<'de>,\n    {\n        Err(())\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Provides a `Visitor` access to each element of a sequence in the input.\n///\n/// This is a trait that a `Deserializer` passes to a `Visitor` implementation,\n/// which deserializes each item in a sequence.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by deserialized sequence elements. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SeqAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SeqAccess<'de> {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// This returns `Ok(Some(value))` for the next value in the sequence, or\n    /// `Ok(None)` if there are no more remaining items.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `SeqAccess::next_element` instead.\n...\n", "needle_token_start": 12170, "needle_token_end": 12226, "code_context_ntokens": 16241, "output": ["```rust\n/// Hint that the `Deserialize` type is expecting an `i8` value.\nfn deserialize_i8<V>(self, visitor: V) -> Result<V::Value, Self::Error>\nwhere\n    V: Visitor<'de>,\n{\n    self.visit_i64(i64::from(self.visit_i8(v)?))\n}\n```"]}
{"repo": "serde-rs/serde", "name": "variant", "language": "rust", "path": "serde/src/de/mod.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine which specific variant of an enumeration should be deserialized from the input data.\n2. **Input**: It accepts a seed which is a hint or a helper that assists in the deserialization of the enumeration variant.\n3. **Output**: The function returns a tuple containing the deserialized value of the enumeration variant and an access point to further deserialize the content of that variant.\n4. **Procedure**: The function utilizes the provided seed to initiate the deserialization process. It then delegates the task to another method that handles the actual deserialization based on the seed, returning the appropriate variant and its content for further processing.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "///         impl<'de, 'a, T> Visitor<'de> for ExtendVecVisitor<'a, T>\n///         where\n///             T: Deserialize<'de>,\n///         {\n///             type Value = ();\n///\n///             fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///                 write!(formatter, \"an array of integers\")\n///             }\n///\n///             fn visit_seq<A>(self, mut seq: A) -> Result<(), A::Error>\n///             where\n///                 A: SeqAccess<'de>,\n///             {\n///                 // Decrease the number of reallocations if there are many elements\n///                 if let Some(size_hint) = seq.size_hint() {\n///                     self.0.reserve(size_hint);\n///                 }\n///\n///                 // Visit each element in the inner array and push it onto\n///                 // the existing vector.\n///                 while let Some(elem) = seq.next_element()? {\n///                     self.0.push(elem);\n///                 }\n///                 Ok(())\n///             }\n///         }\n///\n///         deserializer.deserialize_seq(ExtendVecVisitor(self.0))\n///     }\n/// }\n///\n/// // Visitor implementation that will walk the outer array of the JSON input.\n/// struct FlattenedVecVisitor<T>(PhantomData<T>);\n///\n/// impl<'de, T> Visitor<'de> for FlattenedVecVisitor<T>\n/// where\n///     T: Deserialize<'de>,\n/// {\n///     // This Visitor constructs a single Vec<T> to hold the flattened\n///     // contents of the inner arrays.\n///     type Value = Vec<T>;\n///\n///     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///         write!(formatter, \"an array of arrays\")\n///     }\n///\n///     fn visit_seq<A>(self, mut seq: A) -> Result<Vec<T>, A::Error>\n///     where\n///         A: SeqAccess<'de>,\n///     {\n///         // Create a single Vec to hold the flattened contents.\n///         let mut vec = Vec::new();\n///\n///         // Each iteration through this loop is one inner array.\n///         while let Some(()) = seq.next_element_seed(ExtendVec(&mut vec))? {\n///             // Nothing to do; inner array has been appended into `vec`.\n///         }\n///\n///         // Return the finished vec.\n///         Ok(vec)\n///     }\n/// }\n///\n/// # fn example<'de, D>(deserializer: D) -> Result<(), D::Error>\n/// # where\n/// #     D: Deserializer<'de>,\n/// # {\n/// let visitor = FlattenedVecVisitor(PhantomData);\n/// let flattened: Vec<u64> = deserializer.deserialize_seq(visitor)?;\n/// #     Ok(())\n/// # }\n/// ```\npub trait DeserializeSeed<'de>: Sized {\n    /// The type produced by using this seed.\n    type Value;\n\n    /// Equivalent to the more common `Deserialize::deserialize` method, except\n    /// with some initial piece of data (the seed) passed in.\n    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>;\n}\n\nimpl<'de, T> DeserializeSeed<'de> for PhantomData<T>\nwhere\n    T: Deserialize<'de>,\n{\n    type Value = T;\n\n    #[inline]\n    fn deserialize<D>(self, deserializer: D) -> Result<T, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        T::deserialize(deserializer)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// A **data format** that can deserialize any data structure supported by\n/// Serde.\n///\n/// The role of this trait is to define the deserialization half of the [Serde\n/// data model], which is a way to categorize every Rust data type into one of\n/// 29 possible types. Each method of the `Deserializer` trait corresponds to one\n/// of the types of the data model.\n///\n/// Implementations of `Deserialize` map themselves into this data model by\n/// passing to the `Deserializer` a `Visitor` implementation that can receive\n/// these various types.\n///\n/// The types that make up the Serde data model are:\n///\n///  - **14 primitive types**\n///    - bool\n///    - i8, i16, i32, i64, i128\n///    - u8, u16, u32, u64, u128\n///    - f32, f64\n///    - char\n///  - **string**\n///    - UTF-8 bytes with a length and no null terminator.\n///    - When serializing, all strings are handled equally. When deserializing,\n///      there are three flavors of strings: transient, owned, and borrowed.\n///  - **byte array** - \\[u8\\]\n///    - Similar to strings, during deserialization byte arrays can be\n///      transient, owned, or borrowed.\n///  - **option**\n///    - Either none or some value.\n///  - **unit**\n///    - The type of `()` in Rust. It represents an anonymous value containing\n///      no data.\n///  - **unit_struct**\n///    - For example `struct Unit` or `PhantomData<T>`. It represents a named\n///      value containing no data.\n///  - **unit_variant**\n///    - For example the `E::A` and `E::B` in `enum E { A, B }`.\n///  - **newtype_struct**\n///    - For example `struct Millimeters(u8)`.\n///  - **newtype_variant**\n///    - For example the `E::N` in `enum E { N(u8) }`.\n///  - **seq**\n///    - A variably sized heterogeneous sequence of values, for example `Vec<T>`\n///      or `HashSet<T>`. When serializing, the length may or may not be known\n///      before iterating through all the data. When deserializing, the length\n///      is determined by looking at the serialized data.\n///  - **tuple**\n///    - A statically sized heterogeneous sequence of values for which the\n///      length will be known at deserialization time without looking at the\n///      serialized data, for example `(u8,)` or `(String, u64, Vec<T>)` or\n///      `[u64; 10]`.\n///  - **tuple_struct**\n///    - A named tuple, for example `struct Rgb(u8, u8, u8)`.\n///  - **tuple_variant**\n///    - For example the `E::T` in `enum E { T(u8, u8) }`.\n///  - **map**\n///    - A heterogeneous key-value pairing, for example `BTreeMap<K, V>`.\n///  - **struct**\n///    - A heterogeneous key-value pairing in which the keys are strings and\n///      will be known at deserialization time without looking at the serialized\n///      data, for example `struct S { r: u8, g: u8, b: u8 }`.\n///  - **struct_variant**\n///    - For example the `E::S` in `enum E { S { r: u8, g: u8, b: u8 } }`.\n///\n/// The `Deserializer` trait supports two entry point styles which enables\n/// different kinds of deserialization.\n///\n/// 1. The `deserialize_any` method. Self-describing data formats like JSON are\n///    able to look at the serialized data and tell what it represents. For\n///    example the JSON deserializer may see an opening curly brace (`{`) and\n///    know that it is seeing a map. If the data format supports\n///    `Deserializer::deserialize_any`, it will drive the Visitor using whatever\n///    type it sees in the input. JSON uses this approach when deserializing\n///    `serde_json::Value` which is an enum that can represent any JSON\n///    document. Without knowing what is in a JSON document, we can deserialize\n///    it to `serde_json::Value` by going through\n///    `Deserializer::deserialize_any`.\n///\n/// 2. The various `deserialize_*` methods. Non-self-describing formats like\n///    Postcard need to be told what is in the input in order to deserialize it.\n///    The `deserialize_*` methods are hints to the deserializer for how to\n///    interpret the next piece of input. Non-self-describing formats are not\n///    able to deserialize something like `serde_json::Value` which relies on\n///    `Deserializer::deserialize_any`.\n///\n/// When implementing `Deserialize`, you should avoid relying on\n/// `Deserializer::deserialize_any` unless you need to be told by the\n/// Deserializer what type is in the input. Know that relying on\n/// `Deserializer::deserialize_any` means your data type will be able to\n/// deserialize from self-describing formats only, ruling out Postcard and many\n/// others.\n///\n/// [Serde data model]: https://serde.rs/data-model.html\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed from the input when deserializing. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website contains example code for\n/// a basic JSON `Deserializer`.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait Deserializer<'de>: Sized {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// Require the `Deserializer` to figure out how to drive the visitor based\n    /// on what data type is in the input.\n    ///\n    /// When implementing `Deserialize`, you should avoid relying on\n    /// `Deserializer::deserialize_any` unless you need to be told by the\n    /// Deserializer what type is in the input. Know that relying on\n    /// `Deserializer::deserialize_any` means your data type will be able to\n    /// deserialize from self-describing formats only, ruling out Postcard and\n    /// many others.\n    fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `bool` value.\n    fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i8` value.\n    fn deserialize_i8<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i16` value.\n    fn deserialize_i16<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i32` value.\n    fn deserialize_i32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i64` value.\n    fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `i128` value.\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>,\n    {\n        let _ = visitor;\n        Err(Error::custom(\"i128 is not supported\"))\n    }\n\n    /// Hint that the `Deserialize` type is expecting a `u8` value.\n    fn deserialize_u8<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u16` value.\n    fn deserialize_u16<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u32` value.\n    fn deserialize_u32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `u64` value.\n    fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an `u128` value.\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>,\n    {\n        let _ = visitor;\n        Err(Error::custom(\"u128 is not supported\"))\n    }\n\n    /// Hint that the `Deserialize` type is expecting a `f32` value.\n    fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `f64` value.\n    fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a `char` value.\n    fn deserialize_char<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a string value and does\n    /// not benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would benefit from taking ownership of `String` data,\n    /// indicate this to the `Deserializer` by using `deserialize_string`\n    /// instead.\n    fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a string value and would\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would not benefit from taking ownership of `String`\n    /// data, indicate that to the `Deserializer` by using `deserialize_str`\n    /// instead.\n    fn deserialize_string<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a byte array and does not\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would benefit from taking ownership of `Vec<u8>` data,\n    /// indicate this to the `Deserializer` by using `deserialize_byte_buf`\n    /// instead.\n    fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a byte array and would\n    /// benefit from taking ownership of buffered data owned by the\n    /// `Deserializer`.\n    ///\n    /// If the `Visitor` would not benefit from taking ownership of `Vec<u8>`\n    /// data, indicate that to the `Deserializer` by using `deserialize_bytes`\n    /// instead.\n    fn deserialize_byte_buf<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an optional value.\n    ///\n    /// This allows deserializers that encode an optional value as a nullable\n    /// value to convert the null value into `None` and a regular value into\n    /// `Some(value)`.\n    fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a unit value.\n    fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a unit struct with a\n    /// particular name.\n    fn deserialize_unit_struct<V>(\n        self,\n        name: &'static str,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a newtype struct with a\n    /// particular name.\n    fn deserialize_newtype_struct<V>(\n        self,\n        name: &'static str,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a sequence of values.\n    fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a sequence of values and\n    /// knows how many values there are without looking at the serialized data.\n    fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a tuple struct with a\n    /// particular name and number of fields.\n    fn deserialize_tuple_struct<V>(\n        self,\n        name: &'static str,\n        len: usize,\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a map of key-value pairs.\n    fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting a struct with a particular\n    /// name and fields.\n    fn deserialize_struct<V>(\n        self,\n        name: &'static str,\n        fields: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting an enum value with a\n    /// particular name and possible variants.\n    fn deserialize_enum<V>(\n        self,\n        name: &'static str,\n        variants: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type is expecting the name of a struct\n    /// field or the discriminant of an enum variant.\n    fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Hint that the `Deserialize` type needs to deserialize a value whose type\n    /// doesn't matter because it is ignored.\n    ///\n    /// Deserializers for non-self-describing formats may not support this mode.\n    fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Determine whether `Deserialize` implementations should expect to\n    /// deserialize their human-readable form.\n    ///\n    /// Some types have a human-readable form that may be somewhat expensive to\n    /// construct, as well as a binary form that is compact and efficient.\n    /// Generally text-based formats like JSON and YAML will prefer to use the\n    /// human-readable one and binary formats like Postcard will prefer the\n    /// compact one.\n    ///\n    /// ```edition2021\n    /// # use std::ops::Add;\n    /// # use std::str::FromStr;\n    /// #\n    /// # struct Timestamp;\n    /// #\n    /// # impl Timestamp {\n    /// #     const EPOCH: Timestamp = Timestamp;\n    /// # }\n    /// #\n    /// # impl FromStr for Timestamp {\n    /// #     type Err = String;\n    /// #     fn from_str(_: &str) -> Result<Self, Self::Err> {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// # struct Duration;\n    /// #\n    /// # impl Duration {\n    /// #     fn seconds(_: u64) -> Self { unimplemented!() }\n    /// # }\n    /// #\n    /// # impl Add<Duration> for Timestamp {\n    /// #     type Output = Timestamp;\n    /// #     fn add(self, _: Duration) -> Self::Output {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::de::{self, Deserialize, Deserializer};\n    ///\n    /// impl<'de> Deserialize<'de> for Timestamp {\n    ///     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    ///     where\n    ///         D: Deserializer<'de>,\n    ///     {\n    ///         if deserializer.is_human_readable() {\n    ///             // Deserialize from a human-readable string like \"2015-05-15T17:01:00Z\".\n    ///             let s = String::deserialize(deserializer)?;\n    ///             Timestamp::from_str(&s).map_err(de::Error::custom)\n    ///         } else {\n    ///             // Deserialize from a compact binary representation, seconds since\n    ///             // the Unix epoch.\n    ///             let n = u64::deserialize(deserializer)?;\n    ///             Ok(Timestamp::EPOCH + Duration::seconds(n))\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default implementation of this method returns `true`. Data formats\n    /// may override this to `false` to request a compact form for types that\n    /// support one. Note that modifying this method to change a format from\n    /// human-readable to compact or vice versa should be regarded as a breaking\n    /// change, as a value serialized in human-readable mode is not required to\n    /// deserialize from the same data in compact mode.\n    #[inline]\n    fn is_human_readable(&self) -> bool {\n        true\n    }\n\n    // Not public API.\n    #[cfg(all(not(no_serde_derive), any(feature = \"std\", feature = \"alloc\")))]\n    #[doc(hidden)]\n    fn __deserialize_content<V>(\n        self,\n        _: crate::actually_private::T,\n        visitor: V,\n    ) -> Result<crate::__private::de::Content<'de>, Self::Error>\n    where\n        V: Visitor<'de, Value = crate::__private::de::Content<'de>>,\n    {\n        self.deserialize_any(visitor)\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// This trait represents a visitor that walks through a deserializer.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the requirement for lifetime of data\n/// that may be borrowed by `Self::Value`. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example\n///\n/// ```edition2021\n/// # use serde::de::{self, Unexpected, Visitor};\n/// # use std::fmt;\n/// #\n/// /// A visitor that deserializes a long string - a string containing at least\n/// /// some minimum number of bytes.\n/// struct LongString {\n///     min: usize,\n/// }\n///\n/// impl<'de> Visitor<'de> for LongString {\n///     type Value = String;\n///\n///     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n///         write!(formatter, \"a string containing at least {} bytes\", self.min)\n///     }\n///\n///     fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>\n///     where\n///         E: de::Error,\n///     {\n///         if s.len() >= self.min {\n///             Ok(s.to_owned())\n///         } else {\n///             Err(de::Error::invalid_value(Unexpected::Str(s), &self))\n///         }\n///     }\n/// }\n/// ```\npub trait Visitor<'de>: Sized {\n    /// The value produced by this visitor.\n    type Value;\n\n    /// Format a message stating what data this Visitor expects to receive.\n    ///\n    /// This is used in error messages. The message should complete the sentence\n    /// \"This Visitor expects to receive ...\", for example the message could be\n    /// \"an integer between 0 and 64\". The message should not be capitalized and\n    /// should not end with a period.\n    ///\n    /// ```edition2021\n    /// # use std::fmt;\n    /// #\n    /// # struct S {\n    /// #     max: usize,\n    /// # }\n    /// #\n    /// # impl<'de> serde::de::Visitor<'de> for S {\n    /// #     type Value = ();\n    /// #\n    /// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n    ///     write!(formatter, \"an integer between 0 and {}\", self.max)\n    /// }\n    /// # }\n    /// ```\n    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result;\n\n    /// The input contains a boolean.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Bool(v), &self))\n    }\n\n    /// The input contains an `i8`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i8<E>(self, v: i8) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i16`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i16<E>(self, v: i16) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i32`.\n    ///\n    /// The default implementation forwards to [`visit_i64`].\n    ///\n    /// [`visit_i64`]: #method.visit_i64\n    fn visit_i32<E>(self, v: i32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_i64(v as i64)\n    }\n\n    /// The input contains an `i64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Signed(v), &self))\n    }\n\n    /// The input contains a `i128`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_i128<E>(self, v: i128) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut buf = [0u8; 58];\n        let mut writer = format::Buf::new(&mut buf);\n        fmt::Write::write_fmt(&mut writer, format_args!(\"integer `{}` as i128\", v)).unwrap();\n        Err(Error::invalid_type(\n            Unexpected::Other(writer.as_str()),\n            &self,\n        ))\n    }\n\n    /// The input contains a `u8`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u8<E>(self, v: u8) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u16`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u16<E>(self, v: u16) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u32`.\n    ///\n    /// The default implementation forwards to [`visit_u64`].\n    ///\n    /// [`visit_u64`]: #method.visit_u64\n    fn visit_u32<E>(self, v: u32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_u64(v as u64)\n    }\n\n    /// The input contains a `u64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Unsigned(v), &self))\n    }\n\n    /// The input contains a `u128`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_u128<E>(self, v: u128) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        let mut buf = [0u8; 57];\n        let mut writer = format::Buf::new(&mut buf);\n        fmt::Write::write_fmt(&mut writer, format_args!(\"integer `{}` as u128\", v)).unwrap();\n        Err(Error::invalid_type(\n            Unexpected::Other(writer.as_str()),\n            &self,\n        ))\n    }\n\n    /// The input contains an `f32`.\n    ///\n    /// The default implementation forwards to [`visit_f64`].\n    ///\n    /// [`visit_f64`]: #method.visit_f64\n    fn visit_f32<E>(self, v: f32) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_f64(v as f64)\n    }\n\n    /// The input contains an `f64`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Float(v), &self))\n    }\n\n    /// The input contains a `char`.\n    ///\n    /// The default implementation forwards to [`visit_str`] as a one-character\n    /// string.\n    ///\n    /// [`visit_str`]: #method.visit_str\n    #[inline]\n    fn visit_char<E>(self, v: char) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(v.encode_utf8(&mut [0u8; 4]))\n    }\n\n    /// The input contains a string. The lifetime of the string is ephemeral and\n    /// it may be destroyed after this method returns.\n    ///\n    /// This method allows the `Deserializer` to avoid a copy by retaining\n    /// ownership of any buffered data. `Deserialize` implementations that do\n    /// not benefit from taking ownership of `String` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_str` rather than\n    /// `Deserializer::deserialize_string`.\n    ///\n    /// It is never correct to implement `visit_string` without implementing\n    /// `visit_str`. Implement neither, both, or just `visit_str`.\n    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Str(v), &self))\n    }\n\n    /// The input contains a string that lives at least as long as the\n    /// `Deserializer`.\n    ///\n    /// This enables zero-copy deserialization of strings in some formats. For\n    /// example JSON input containing the JSON string `\"borrowed\"` can be\n    /// deserialized with zero copying into a `&'a str` as long as the input\n    /// data outlives `'a`.\n    ///\n    /// The default implementation forwards to `visit_str`.\n    #[inline]\n    fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(v)\n    }\n\n    /// The input contains a string and ownership of the string is being given\n    /// to the `Visitor`.\n    ///\n    /// This method allows the `Visitor` to avoid a copy by taking ownership of\n    /// a string created by the `Deserializer`. `Deserialize` implementations\n    /// that benefit from taking ownership of `String` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_string` rather\n    /// than `Deserializer::deserialize_str`, although not every deserializer\n    /// will honor such a request.\n    ///\n    /// It is never correct to implement `visit_string` without implementing\n    /// `visit_str`. Implement neither, both, or just `visit_str`.\n    ///\n    /// The default implementation forwards to `visit_str` and then drops the\n    /// `String`.\n    #[inline]\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_str(&v)\n    }\n\n    /// The input contains a byte array. The lifetime of the byte array is\n    /// ephemeral and it may be destroyed after this method returns.\n    ///\n    /// This method allows the `Deserializer` to avoid a copy by retaining\n    /// ownership of any buffered data. `Deserialize` implementations that do\n    /// not benefit from taking ownership of `Vec<u8>` data should indicate that\n    /// to the deserializer by using `Deserializer::deserialize_bytes` rather\n    /// than `Deserializer::deserialize_byte_buf`.\n    ///\n    /// It is never correct to implement `visit_byte_buf` without implementing\n    /// `visit_bytes`. Implement neither, both, or just `visit_bytes`.\n    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Bytes(v), &self))\n    }\n\n    /// The input contains a byte array that lives at least as long as the\n    /// `Deserializer`.\n    ///\n    /// This enables zero-copy deserialization of bytes in some formats. For\n    /// example Postcard data containing bytes can be deserialized with zero\n    /// copying into a `&'a [u8]` as long as the input data outlives `'a`.\n    ///\n    /// The default implementation forwards to `visit_bytes`.\n    #[inline]\n    fn visit_borrowed_bytes<E>(self, v: &'de [u8]) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_bytes(v)\n    }\n\n    /// The input contains a byte array and ownership of the byte array is being\n    /// given to the `Visitor`.\n    ///\n    /// This method allows the `Visitor` to avoid a copy by taking ownership of\n    /// a byte buffer created by the `Deserializer`. `Deserialize`\n    /// implementations that benefit from taking ownership of `Vec<u8>` data\n    /// should indicate that to the deserializer by using\n    /// `Deserializer::deserialize_byte_buf` rather than\n    /// `Deserializer::deserialize_bytes`, although not every deserializer will\n    /// honor such a request.\n    ///\n    /// It is never correct to implement `visit_byte_buf` without implementing\n    /// `visit_bytes`. Implement neither, both, or just `visit_bytes`.\n    ///\n    /// The default implementation forwards to `visit_bytes` and then drops the\n    /// `Vec<u8>`.\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    #[cfg_attr(doc_cfg, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        self.visit_bytes(&v)\n    }\n\n    /// The input contains an optional that is absent.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_none<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Option, &self))\n    }\n\n    /// The input contains an optional that is present.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let _ = deserializer;\n        Err(Error::invalid_type(Unexpected::Option, &self))\n    }\n\n    /// The input contains a unit `()`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_unit<E>(self) -> Result<Self::Value, E>\n    where\n        E: Error,\n    {\n        Err(Error::invalid_type(Unexpected::Unit, &self))\n    }\n\n    /// The input contains a newtype struct.\n    ///\n    /// The content of the newtype struct may be read from the given\n    /// `Deserializer`.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let _ = deserializer;\n        Err(Error::invalid_type(Unexpected::NewtypeStruct, &self))\n    }\n\n    /// The input contains a sequence of elements.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>\n    where\n        A: SeqAccess<'de>,\n    {\n        let _ = seq;\n        Err(Error::invalid_type(Unexpected::Seq, &self))\n    }\n\n    /// The input contains a key-value map.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>\n    where\n        A: MapAccess<'de>,\n    {\n        let _ = map;\n        Err(Error::invalid_type(Unexpected::Map, &self))\n    }\n\n    /// The input contains an enum.\n    ///\n    /// The default implementation fails with a type error.\n    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>\n    where\n        A: EnumAccess<'de>,\n    {\n        let _ = data;\n        Err(Error::invalid_type(Unexpected::Enum, &self))\n    }\n\n    // Used when deserializing a flattened Option field. Not public API.\n    #[doc(hidden)]\n    fn __private_visit_untagged_option<D>(self, _: D) -> Result<Self::Value, ()>\n    where\n        D: Deserializer<'de>,\n    {\n        Err(())\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Provides a `Visitor` access to each element of a sequence in the input.\n///\n/// This is a trait that a `Deserializer` passes to a `Visitor` implementation,\n/// which deserializes each item in a sequence.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by deserialized sequence elements. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SeqAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SeqAccess<'de> {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// This returns `Ok(Some(value))` for the next value in the sequence, or\n    /// `Ok(None)` if there are no more remaining items.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `SeqAccess::next_element` instead.\n    fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>\n    where\n        T: DeserializeSeed<'de>;\n\n    /// This returns `Ok(Some(value))` for the next value in the sequence, or\n    /// `Ok(None)` if there are no more remaining items.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `SeqAccess` implementations should not override the default behavior.\n    #[inline]\n    fn next_element<T>(&mut self) -> Result<Option<T>, Self::Error>\n    where\n        T: Deserialize<'de>,\n    {\n        self.next_element_seed(PhantomData)\n    }\n\n    /// Returns the number of elements remaining in the sequence, if known.\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        None\n    }\n}\n\nimpl<'de, 'a, A> SeqAccess<'de> for &'a mut A\nwhere\n    A: ?Sized + SeqAccess<'de>,\n{\n    type Error = A::Error;\n\n    #[inline]\n    fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>\n    where\n        T: DeserializeSeed<'de>,\n    {\n        (**self).next_element_seed(seed)\n    }\n\n    #[inline]\n    fn next_element<T>(&mut self) -> Result<Option<T>, Self::Error>\n    where\n        T: Deserialize<'de>,\n    {\n        (**self).next_element()\n    }\n\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        (**self).size_hint()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Provides a `Visitor` access to each entry of a map in the input.\n///\n/// This is a trait that a `Deserializer` passes to a `Visitor` implementation.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by deserialized map entries. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `MapAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait MapAccess<'de> {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n\n    /// This returns `Ok(Some(key))` for the next key in the map, or `Ok(None)`\n    /// if there are no more remaining entries.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `MapAccess::next_key` or `MapAccess::next_entry` instead.\n    fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>\n    where\n        K: DeserializeSeed<'de>;\n\n    /// This returns a `Ok(value)` for the next value in the map.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `MapAccess::next_value` instead.\n    ///\n    /// # Panics\n    ///\n    /// Calling `next_value_seed` before `next_key_seed` is incorrect and is\n    /// allowed to panic or return bogus results.\n    fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>\n    where\n        V: DeserializeSeed<'de>;\n\n    /// This returns `Ok(Some((key, value)))` for the next (key-value) pair in\n    /// the map, or `Ok(None)` if there are no more remaining items.\n    ///\n    /// `MapAccess` implementations should override the default behavior if a\n    /// more efficient implementation is possible.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `MapAccess::next_entry` instead.\n    #[inline]\n    fn next_entry_seed<K, V>(\n        &mut self,\n        kseed: K,\n        vseed: V,\n    ) -> Result<Option<(K::Value, V::Value)>, Self::Error>\n    where\n        K: DeserializeSeed<'de>,\n        V: DeserializeSeed<'de>,\n    {\n        match tri!(self.next_key_seed(kseed)) {\n            Some(key) => {\n                let value = tri!(self.next_value_seed(vseed));\n                Ok(Some((key, value)))\n            }\n            None => Ok(None),\n        }\n    }\n\n    /// This returns `Ok(Some(key))` for the next key in the map, or `Ok(None)`\n    /// if there are no more remaining entries.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `MapAccess` implementations should not override the default behavior.\n    #[inline]\n    fn next_key<K>(&mut self) -> Result<Option<K>, Self::Error>\n    where\n        K: Deserialize<'de>,\n    {\n        self.next_key_seed(PhantomData)\n    }\n\n    /// This returns a `Ok(value)` for the next value in the map.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `MapAccess` implementations should not override the default behavior.\n    ///\n    /// # Panics\n    ///\n    /// Calling `next_value` before `next_key` is incorrect and is allowed to\n    /// panic or return bogus results.\n    #[inline]\n    fn next_value<V>(&mut self) -> Result<V, Self::Error>\n    where\n        V: Deserialize<'de>,\n    {\n        self.next_value_seed(PhantomData)\n    }\n\n    /// This returns `Ok(Some((key, value)))` for the next (key-value) pair in\n    /// the map, or `Ok(None)` if there are no more remaining items.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `MapAccess` implementations should not override the default behavior.\n    #[inline]\n    fn next_entry<K, V>(&mut self) -> Result<Option<(K, V)>, Self::Error>\n    where\n        K: Deserialize<'de>,\n        V: Deserialize<'de>,\n    {\n        self.next_entry_seed(PhantomData, PhantomData)\n    }\n\n    /// Returns the number of entries remaining in the map, if known.\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        None\n    }\n}\n\nimpl<'de, 'a, A> MapAccess<'de> for &'a mut A\nwhere\n    A: ?Sized + MapAccess<'de>,\n{\n    type Error = A::Error;\n\n    #[inline]\n    fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>\n    where\n        K: DeserializeSeed<'de>,\n    {\n        (**self).next_key_seed(seed)\n    }\n\n    #[inline]\n    fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>\n    where\n        V: DeserializeSeed<'de>,\n    {\n        (**self).next_value_seed(seed)\n    }\n\n    #[inline]\n    fn next_entry_seed<K, V>(\n        &mut self,\n        kseed: K,\n        vseed: V,\n    ) -> Result<Option<(K::Value, V::Value)>, Self::Error>\n    where\n        K: DeserializeSeed<'de>,\n        V: DeserializeSeed<'de>,\n    {\n        (**self).next_entry_seed(kseed, vseed)\n    }\n\n    #[inline]\n    fn next_entry<K, V>(&mut self) -> Result<Option<(K, V)>, Self::Error>\n    where\n        K: Deserialize<'de>,\n        V: Deserialize<'de>,\n    {\n        (**self).next_entry()\n    }\n\n    #[inline]\n    fn next_key<K>(&mut self) -> Result<Option<K>, Self::Error>\n    where\n        K: Deserialize<'de>,\n    {\n        (**self).next_key()\n    }\n\n    #[inline]\n    fn next_value<V>(&mut self) -> Result<V, Self::Error>\n    where\n        V: Deserialize<'de>,\n    {\n        (**self).next_value()\n    }\n\n    #[inline]\n    fn size_hint(&self) -> Option<usize> {\n        (**self).size_hint()\n    }\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n/// Provides a `Visitor` access to the data of an enum in the input.\n///\n/// `EnumAccess` is created by the `Deserializer` and passed to the\n/// `Visitor` in order to identify which variant of an enum to deserialize.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by the deserialized enum variant. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `EnumAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait EnumAccess<'de>: Sized {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization.\n    type Error: Error;\n    /// The `Visitor` that will be used to deserialize the content of the enum\n    /// variant.\n    type Variant: VariantAccess<'de, Error = Self::Error>;\n\n    /// `variant` is called to identify which variant to deserialize.\n    ///\n    /// `Deserialize` implementations should typically use `EnumAccess::variant`\n    /// instead.\n    fn variant_seed<V>(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error>\n    where\n        V: DeserializeSeed<'de>;\n\n    /// `variant` is called to identify which variant to deserialize.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `EnumAccess` implementations should not override the default behavior.\n    #[inline]\n    \nfn variant<V>(self) -> Result<(V, Self::Variant), Self::Error>\n    where\n        V: Deserialize<'de>,\n    {\n        self.variant_seed(PhantomData)\n    }\n}\n\n/// `VariantAccess` is a visitor that is created by the `Deserializer` and\n/// passed to the `Deserialize` to deserialize the content of a particular enum\n/// variant.\n///\n/// # Lifetime\n///\n/// The `'de` lifetime of this trait is the lifetime of data that may be\n/// borrowed by the deserialized enum variant. See the page [Understanding\n/// deserializer lifetimes] for a more detailed explanation of these lifetimes.\n///\n/// [Understanding deserializer lifetimes]: https://serde.rs/lifetimes.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `VariantAccess` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait VariantAccess<'de>: Sized {\n    /// The error type that can be returned if some error occurs during\n    /// deserialization. Must match the error type of our `EnumAccess`.\n    type Error: Error;\n\n    /// Called when deserializing a variant with no values.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// fn unit_variant(self) -> Result<(), Self::Error> {\n    ///     // What the data actually contained; suppose it is a tuple variant.\n    ///     let unexp = Unexpected::TupleVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"unit variant\"))\n    /// }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn unit_variant(self) -> Result<(), Self::Error>;\n\n    /// Called when deserializing a variant with a single value.\n    ///\n    /// `Deserialize` implementations should typically use\n    /// `VariantAccess::newtype_variant` instead.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// fn newtype_variant_seed<T>(self, _seed: T) -> Result<T::Value, Self::Error>\n    /// where\n    ///     T: DeserializeSeed<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"newtype variant\"))\n    /// }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value, Self::Error>\n    where\n        T: DeserializeSeed<'de>;\n\n    /// Called when deserializing a variant with a single value.\n    ///\n    /// This method exists as a convenience for `Deserialize` implementations.\n    /// `VariantAccess` implementations should not override the default\n    /// behavior.\n    #[inline]\n    fn newtype_variant<T>(self) -> Result<T, Self::Error>\n    where\n        T: Deserialize<'de>,\n    {\n        self.newtype_variant_seed(PhantomData)\n    }\n\n    /// Called when deserializing a tuple-like variant.\n    ///\n    /// The `len` is the number of fields expected in the tuple variant.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// fn tuple_variant<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>\n    /// where\n    ///     V: Visitor<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"tuple variant\"))\n    /// }\n    /// #\n    /// #     fn struct_variant<V>(self, _: &[&str], _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// # }\n    /// ```\n    fn tuple_variant<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n\n    /// Called when deserializing a struct-like variant.\n    ///\n    /// The `fields` are the names of the fields of the struct variant.\n    ///\n    /// If the data contains a different type of variant, the following\n    /// `invalid_type` error should be constructed:\n    ///\n    /// ```edition2021\n    /// # use serde::de::{self, value, DeserializeSeed, Visitor, VariantAccess, Unexpected};\n    /// #\n    /// # struct X;\n    /// #\n    /// # impl<'de> VariantAccess<'de> for X {\n    /// #     type Error = value::Error;\n    /// #\n    /// #     fn unit_variant(self) -> Result<(), Self::Error> {\n    /// #         unimplemented!()\n    /// #     }\n    /// #\n    /// #     fn newtype_variant_seed<T>(self, _: T) -> Result<T::Value, Self::Error>\n    /// #     where\n    /// #         T: DeserializeSeed<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// #     fn tuple_variant<V>(self, _: usize, _: V) -> Result<V::Value, Self::Error>\n    /// #     where\n    /// #         V: Visitor<'de>,\n    /// #     { unimplemented!() }\n    /// #\n    /// fn struct_variant<V>(\n    ///     self,\n    ///     _fields: &'static [&'static str],\n    ///     _visitor: V,\n    /// ) -> Result<V::Value, Self::Error>\n    /// where\n    ///     V: Visitor<'de>,\n    /// {\n    ///     // What the data actually contained; suppose it is a unit variant.\n    ///     let unexp = Unexpected::UnitVariant;\n    ///     Err(de::Error::invalid_type(unexp, &\"struct variant\"))\n    /// }\n    /// # }\n    /// ```\n    fn struct_variant<V>(\n        self,\n        fields: &'static [&'static str],\n        visitor: V,\n    ) -> Result<V::Value, Self::Error>\n    where\n        V: Visitor<'de>;\n}\n\n////////////////////////////////////////////////////////////////////////////////\n\n...\n", "needle_token_start": 13785, "needle_token_end": 13838, "code_context_ntokens": 16253, "output": ["```rust\nfn variant_seed<V>(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error>\nwhere\n    V: DeserializeSeed<'de>;\n```"]}
{"repo": "serde-rs/serde", "name": "serialize_entry", "language": "rust", "path": "serde/src/ser/mod.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To efficiently serialize a map entry, which includes both a key and a value, into a format that can be stored or transmitted.\n2. **Input**: A key and a value, both of which must implement the serialization interface.\n3. **Output**: A result indicating either successful serialization or an error if serialization fails.\n4. **Procedure**: The method first attempts to serialize the key. If successful, it proceeds to serialize the value. This method is optimized for cases where both key and value are immediately available, allowing for potential performance improvements over serializing them separately.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "///  - **tuple_variant**\n///    - For example the `E::T` in `enum E { T(u8, u8) }`.\n///  - **map**\n///    - A heterogeneous key-value pairing, for example `BTreeMap<K, V>`.\n///  - **struct**\n///    - A heterogeneous key-value pairing in which the keys are strings and\n///      will be known at deserialization time without looking at the\n///      serialized data, for example `struct S { r: u8, g: u8, b: u8 }`.\n///  - **struct_variant**\n///    - For example the `E::S` in `enum E { S { r: u8, g: u8, b: u8 } }`.\n///\n/// Many Serde serializers produce text or binary data as output, for example\n/// JSON or Postcard. This is not a requirement of the `Serializer` trait, and\n/// there are serializers that do not produce text or binary output. One example\n/// is the `serde_json::value::Serializer` (distinct from the main `serde_json`\n/// serializer) that produces a `serde_json::Value` data structure in memory as\n/// output.\n///\n/// [Serde data model]: https://serde.rs/data-model.html\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website contains example code for\n/// a basic JSON `Serializer`.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait Serializer: Sized {\n    /// The output type produced by this `Serializer` during successful\n    /// serialization. Most serializers that produce text or binary output\n    /// should set `Ok = ()` and serialize into an [`io::Write`] or buffer\n    /// contained within the `Serializer` instance. Serializers that build\n    /// in-memory data structures may be simplified by using `Ok` to propagate\n    /// the data structure around.\n    ///\n    /// [`io::Write`]: https://doc.rust-lang.org/std/io/trait.Write.html\n    type Ok;\n\n    /// The error type when some error occurs during serialization.\n    type Error: Error;\n\n    /// Type returned from [`serialize_seq`] for serializing the content of the\n    /// sequence.\n    ///\n    /// [`serialize_seq`]: #tymethod.serialize_seq\n    type SerializeSeq: SerializeSeq<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple`] for serializing the content of\n    /// the tuple.\n    ///\n    /// [`serialize_tuple`]: #tymethod.serialize_tuple\n    type SerializeTuple: SerializeTuple<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple_struct`] for serializing the\n    /// content of the tuple struct.\n    ///\n    /// [`serialize_tuple_struct`]: #tymethod.serialize_tuple_struct\n    type SerializeTupleStruct: SerializeTupleStruct<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_tuple_variant`] for serializing the\n    /// content of the tuple variant.\n    ///\n    /// [`serialize_tuple_variant`]: #tymethod.serialize_tuple_variant\n    type SerializeTupleVariant: SerializeTupleVariant<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_map`] for serializing the content of the\n    /// map.\n    ///\n    /// [`serialize_map`]: #tymethod.serialize_map\n    type SerializeMap: SerializeMap<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_struct`] for serializing the content of\n    /// the struct.\n    ///\n    /// [`serialize_struct`]: #tymethod.serialize_struct\n    type SerializeStruct: SerializeStruct<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Type returned from [`serialize_struct_variant`] for serializing the\n    /// content of the struct variant.\n    ///\n    /// [`serialize_struct_variant`]: #tymethod.serialize_struct_variant\n    type SerializeStructVariant: SerializeStructVariant<Ok = Self::Ok, Error = Self::Error>;\n\n    /// Serialize a `bool` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for bool {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_bool(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i8` value.\n    ///\n    /// If the format does not differentiate between `i8` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i8 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i8(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i16` value.\n    ///\n    /// If the format does not differentiate between `i16` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i16 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i16(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i32` value.\n    ///\n    /// If the format does not differentiate between `i32` and `i64`, a\n    /// reasonable implementation would be to cast the value to `i64` and\n    /// forward to `serialize_i64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i32 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i32(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i64` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i64 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i64(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `i128` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for i128 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_i128(*self)\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn serialize_i128(self, v: i128) -> Result<Self::Ok, Self::Error> {\n        let _ = v;\n        Err(Error::custom(\"i128 is not supported\"))\n    }\n\n    /// Serialize a `u8` value.\n    ///\n    /// If the format does not differentiate between `u8` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u8 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u8(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u16` value.\n    ///\n    /// If the format does not differentiate between `u16` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u16 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u16(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u32` value.\n    ///\n    /// If the format does not differentiate between `u32` and `u64`, a\n    /// reasonable implementation would be to cast the value to `u64` and\n    /// forward to `serialize_u64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u32 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u32(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u64` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u64 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u64(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_u64(self, v: u64) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `u128` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for u128 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_u128(*self)\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default behavior unconditionally returns an error.\n    fn serialize_u128(self, v: u128) -> Result<Self::Ok, Self::Error> {\n        let _ = v;\n        Err(Error::custom(\"u128 is not supported\"))\n    }\n\n    /// Serialize an `f32` value.\n    ///\n    /// If the format does not differentiate between `f32` and `f64`, a\n    /// reasonable implementation would be to cast the value to `f64` and\n    /// forward to `serialize_f64`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for f32 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_f32(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_f32(self, v: f32) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize an `f64` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for f64 {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_f64(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_f64(self, v: f64) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a character.\n    ///\n    /// If the format does not support characters, it is reasonable to serialize\n    /// it as a single element `str` or a `u32`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for char {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_char(*self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_char(self, v: char) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a `&str`.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for str {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_str(self)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_str(self, v: &str) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a chunk of raw byte data.\n    ///\n    /// Enables serializers to serialize byte slices more compactly or more\n    /// efficiently than other types of slices. If no efficient implementation\n    /// is available, a reasonable implementation would be to forward to\n    /// `serialize_seq`. If forwarded, the implementation looks usually just\n    /// like this:\n    ///\n    /// ```edition2021\n    /// # use serde::ser::{Serializer, SerializeSeq};\n    /// # use serde::__private::doc::Error;\n    /// #\n    /// # struct MySerializer;\n    /// #\n    /// # impl Serializer for MySerializer {\n    /// #     type Ok = ();\n    /// #     type Error = Error;\n    /// #\n    /// fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error> {\n    ///     let mut seq = self.serialize_seq(Some(v.len()))?;\n    ///     for b in v {\n    ///         seq.serialize_element(b)?;\n    ///     }\n    ///     seq.end()\n    /// }\n    /// #\n    /// #     serde::__serialize_unimplemented! {\n    /// #         bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str none some\n    /// #         unit unit_struct unit_variant newtype_struct newtype_variant\n    /// #         seq tuple tuple_struct tuple_variant map struct struct_variant\n    /// #     }\n    /// # }\n    /// ```\n    fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a [`None`] value.\n    ///\n    /// ```edition2021\n    /// # use serde::{Serialize, Serializer};\n    /// #\n    /// # enum Option<T> {\n    /// #     Some(T),\n    /// #     None,\n    /// # }\n    /// #\n    /// # use self::Option::{Some, None};\n    /// #\n    /// impl<T> Serialize for Option<T>\n    /// where\n    ///     T: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             Some(ref value) => serializer.serialize_some(value),\n    ///             None => serializer.serialize_none(),\n    ///         }\n    ///     }\n    /// }\n    /// #\n    /// # fn main() {}\n    /// ```\n    ///\n    /// [`None`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.None\n    fn serialize_none(self) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a [`Some(T)`] value.\n    ///\n    /// ```edition2021\n    /// # use serde::{Serialize, Serializer};\n    /// #\n    /// # enum Option<T> {\n    /// #     Some(T),\n    /// #     None,\n    /// # }\n    /// #\n    /// # use self::Option::{Some, None};\n    /// #\n    /// impl<T> Serialize for Option<T>\n    /// where\n    ///     T: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             Some(ref value) => serializer.serialize_some(value),\n    ///             None => serializer.serialize_none(),\n    ///         }\n    ///     }\n    /// }\n    /// #\n    /// # fn main() {}\n    /// ```\n    ///\n    /// [`Some(T)`]: https://doc.rust-lang.org/std/option/enum.Option.html#variant.Some\n    fn serialize_some<T>(self, value: &T) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Serialize a `()` value.\n    ///\n    /// ```edition2021\n    /// # use serde::Serializer;\n    /// #\n    /// # serde::__private_serialize!();\n    /// #\n    /// impl Serialize for () {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_unit()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_unit(self) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a unit struct like `struct Unit` or `PhantomData<T>`.\n    ///\n    /// A reasonable implementation would be to forward to `serialize_unit`.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// struct Nothing;\n    ///\n    /// impl Serialize for Nothing {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_unit_struct(\"Nothing\")\n    ///     }\n    /// }\n    /// ```\n    fn serialize_unit_struct(self, name: &'static str) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a unit variant like `E::A` in `enum E { A, B }`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, and the `variant` is the name of the\n    /// variant.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// enum E {\n    ///     A,\n    ///     B,\n    /// }\n    ///\n    /// impl Serialize for E {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             E::A => serializer.serialize_unit_variant(\"E\", 0, \"A\"),\n    ///             E::B => serializer.serialize_unit_variant(\"E\", 1, \"B\"),\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    fn serialize_unit_variant(\n        self,\n        name: &'static str,\n        variant_index: u32,\n        variant: &'static str,\n    ) -> Result<Self::Ok, Self::Error>;\n\n    /// Serialize a newtype struct like `struct Millimeters(u8)`.\n    ///\n    /// Serializers are encouraged to treat newtype structs as insignificant\n    /// wrappers around the data they contain. A reasonable implementation would\n    /// be to forward to `value.serialize(self)`.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// struct Millimeters(u8);\n    ///\n    /// impl Serialize for Millimeters {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.serialize_newtype_struct(\"Millimeters\", &self.0)\n    ///     }\n    /// }\n    /// ```\n    fn serialize_newtype_struct<T>(\n        self,\n        name: &'static str,\n        value: &T,\n    ) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Serialize a newtype variant like `E::N` in `enum E { N(u8) }`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, and the `variant` is the name of the\n    /// variant. The `value` is the data contained within this newtype variant.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// enum E {\n    ///     M(String),\n    ///     N(u8),\n    /// }\n    ///\n    /// impl Serialize for E {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             E::M(ref s) => serializer.serialize_newtype_variant(\"E\", 0, \"M\", s),\n    ///             E::N(n) => serializer.serialize_newtype_variant(\"E\", 1, \"N\", &n),\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    fn serialize_newtype_variant<T>(\n        self,\n        name: &'static str,\n        variant_index: u32,\n        variant: &'static str,\n        value: &T,\n    ) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Begin to serialize a variably sized sequence. This call must be\n    /// followed by zero or more calls to `serialize_element`, then a call to\n    /// `end`.\n    ///\n    /// The argument is the number of elements in the sequence, which may or may\n    /// not be computable before the sequence is iterated. Some serializers only\n    /// support sequences whose length is known up front.\n    ///\n    /// ```edition2021\n    /// # use std::marker::PhantomData;\n    /// #\n    /// # struct Vec<T>(PhantomData<T>);\n    /// #\n    /// # impl<T> Vec<T> {\n    /// #     fn len(&self) -> usize {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// # impl<'a, T> IntoIterator for &'a Vec<T> {\n    /// #     type Item = &'a T;\n    /// #     type IntoIter = Box<dyn Iterator<Item = &'a T>>;\n    /// #\n    /// #     fn into_iter(self) -> Self::IntoIter {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::ser::{Serialize, SerializeSeq, Serializer};\n    ///\n    /// impl<T> Serialize for Vec<T>\n    /// where\n    ///     T: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut seq = serializer.serialize_seq(Some(self.len()))?;\n    ///         for element in self {\n    ///             seq.serialize_element(element)?;\n    ///         }\n    ///         seq.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error>;\n\n    /// Begin to serialize a statically sized sequence whose length will be\n    /// known at deserialization time without looking at the serialized data.\n    /// This call must be followed by zero or more calls to `serialize_element`,\n    /// then a call to `end`.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTuple, Serializer};\n    ///\n    /// # mod fool {\n    /// #     trait Serialize {}\n    /// impl<A, B, C> Serialize for (A, B, C)\n    /// #     {}\n    /// # }\n    /// #\n    /// # struct Tuple3<A, B, C>(A, B, C);\n    /// #\n    /// # impl<A, B, C> Serialize for Tuple3<A, B, C>\n    /// where\n    ///     A: Serialize,\n    ///     B: Serialize,\n    ///     C: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut tup = serializer.serialize_tuple(3)?;\n    ///         tup.serialize_element(&self.0)?;\n    ///         tup.serialize_element(&self.1)?;\n    ///         tup.serialize_element(&self.2)?;\n    ///         tup.end()\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTuple, Serializer};\n    ///\n    /// const VRAM_SIZE: usize = 386;\n    /// struct Vram([u16; VRAM_SIZE]);\n    ///\n    /// impl Serialize for Vram {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut seq = serializer.serialize_tuple(VRAM_SIZE)?;\n    ///         for element in &self.0[..] {\n    ///             seq.serialize_element(element)?;\n    ///         }\n    ///         seq.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, Self::Error>;\n\n    /// Begin to serialize a tuple struct like `struct Rgb(u8, u8, u8)`. This\n    /// call must be followed by zero or more calls to `serialize_field`, then a\n    /// call to `end`.\n    ///\n    /// The `name` is the name of the tuple struct and the `len` is the number\n    /// of data fields that will be serialized.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTupleStruct, Serializer};\n    ///\n    /// struct Rgb(u8, u8, u8);\n    ///\n    /// impl Serialize for Rgb {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut ts = serializer.serialize_tuple_struct(\"Rgb\", 3)?;\n    ///         ts.serialize_field(&self.0)?;\n    ///         ts.serialize_field(&self.1)?;\n    ///         ts.serialize_field(&self.2)?;\n    ///         ts.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_tuple_struct(\n        self,\n        name: &'static str,\n        len: usize,\n    ) -> Result<Self::SerializeTupleStruct, Self::Error>;\n\n    /// Begin to serialize a tuple variant like `E::T` in `enum E { T(u8, u8)\n    /// }`. This call must be followed by zero or more calls to\n    /// `serialize_field`, then a call to `end`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, the `variant` is the name of the variant,\n    /// and the `len` is the number of data fields that will be serialized.\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeTupleVariant, Serializer};\n    ///\n    /// enum E {\n    ///     T(u8, u8),\n    ///     U(String, u32, u32),\n    /// }\n    ///\n    /// impl Serialize for E {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             E::T(ref a, ref b) => {\n    ///                 let mut tv = serializer.serialize_tuple_variant(\"E\", 0, \"T\", 2)?;\n    ///                 tv.serialize_field(a)?;\n    ///                 tv.serialize_field(b)?;\n    ///                 tv.end()\n    ///             }\n    ///             E::U(ref a, ref b, ref c) => {\n    ///                 let mut tv = serializer.serialize_tuple_variant(\"E\", 1, \"U\", 3)?;\n    ///                 tv.serialize_field(a)?;\n    ///                 tv.serialize_field(b)?;\n    ///                 tv.serialize_field(c)?;\n    ///                 tv.end()\n    ///             }\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    fn serialize_tuple_variant(\n        self,\n        name: &'static str,\n        variant_index: u32,\n        variant: &'static str,\n        len: usize,\n    ) -> Result<Self::SerializeTupleVariant, Self::Error>;\n\n    /// Begin to serialize a map. This call must be followed by zero or more\n    /// calls to `serialize_key` and `serialize_value`, then a call to `end`.\n    ///\n    /// The argument is the number of elements in the map, which may or may not\n    /// be computable before the map is iterated. Some serializers only support\n    /// maps whose length is known up front.\n    ///\n    /// ```edition2021\n    /// # use std::marker::PhantomData;\n    /// #\n    /// # struct HashMap<K, V>(PhantomData<K>, PhantomData<V>);\n    /// #\n    /// # impl<K, V> HashMap<K, V> {\n    /// #     fn len(&self) -> usize {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// # impl<'a, K, V> IntoIterator for &'a HashMap<K, V> {\n    /// #     type Item = (&'a K, &'a V);\n    /// #     type IntoIter = Box<dyn Iterator<Item = (&'a K, &'a V)>>;\n    /// #\n    /// #     fn into_iter(self) -> Self::IntoIter {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::ser::{Serialize, SerializeMap, Serializer};\n    ///\n    /// impl<K, V> Serialize for HashMap<K, V>\n    /// where\n    ///     K: Serialize,\n    ///     V: Serialize,\n    /// {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut map = serializer.serialize_map(Some(self.len()))?;\n    ///         for (k, v) in self {\n    ///             map.serialize_entry(k, v)?;\n    ///         }\n    ///         map.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, Self::Error>;\n\n    /// Begin to serialize a struct like `struct Rgb { r: u8, g: u8, b: u8 }`.\n    /// This call must be followed by zero or more calls to `serialize_field`,\n    /// then a call to `end`.\n    ///\n    /// The `name` is the name of the struct and the `len` is the number of\n    /// data fields that will be serialized. `len` does not include fields\n    /// which are skipped with [`SerializeStruct::skip_field`].\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeStruct, Serializer};\n    ///\n    /// struct Rgb {\n    ///     r: u8,\n    ///     g: u8,\n    ///     b: u8,\n    /// }\n    ///\n    /// impl Serialize for Rgb {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         let mut rgb = serializer.serialize_struct(\"Rgb\", 3)?;\n    ///         rgb.serialize_field(\"r\", &self.r)?;\n    ///         rgb.serialize_field(\"g\", &self.g)?;\n    ///         rgb.serialize_field(\"b\", &self.b)?;\n    ///         rgb.end()\n    ///     }\n    /// }\n    /// ```\n    fn serialize_struct(\n        self,\n        name: &'static str,\n        len: usize,\n    ) -> Result<Self::SerializeStruct, Self::Error>;\n\n    /// Begin to serialize a struct variant like `E::S` in `enum E { S { r: u8,\n    /// g: u8, b: u8 } }`. This call must be followed by zero or more calls to\n    /// `serialize_field`, then a call to `end`.\n    ///\n    /// The `name` is the name of the enum, the `variant_index` is the index of\n    /// this variant within the enum, the `variant` is the name of the variant,\n    /// and the `len` is the number of data fields that will be serialized.\n    /// `len` does not include fields which are skipped with\n    /// [`SerializeStructVariant::skip_field`].\n    ///\n    /// ```edition2021\n    /// use serde::ser::{Serialize, SerializeStructVariant, Serializer};\n    ///\n    /// enum E {\n    ///     S { r: u8, g: u8, b: u8 },\n    /// }\n    ///\n    /// impl Serialize for E {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         match *self {\n    ///             E::S {\n    ///                 ref r,\n    ///                 ref g,\n    ///                 ref b,\n    ///             } => {\n    ///                 let mut sv = serializer.serialize_struct_variant(\"E\", 0, \"S\", 3)?;\n    ///                 sv.serialize_field(\"r\", r)?;\n    ///                 sv.serialize_field(\"g\", g)?;\n    ///                 sv.serialize_field(\"b\", b)?;\n    ///                 sv.end()\n    ///             }\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    fn serialize_struct_variant(\n        self,\n        name: &'static str,\n        variant_index: u32,\n        variant: &'static str,\n        len: usize,\n    ) -> Result<Self::SerializeStructVariant, Self::Error>;\n\n    /// Collect an iterator as a sequence.\n    ///\n    /// The default implementation serializes each item yielded by the iterator\n    /// using [`serialize_seq`]. Implementors should not need to override this\n    /// method.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// struct SecretlyOneHigher {\n    ///     data: Vec<i32>,\n    /// }\n    ///\n    /// impl Serialize for SecretlyOneHigher {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.collect_seq(self.data.iter().map(|x| x + 1))\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// [`serialize_seq`]: #tymethod.serialize_seq\n    fn collect_seq<I>(self, iter: I) -> Result<Self::Ok, Self::Error>\n    where\n        I: IntoIterator,\n        <I as IntoIterator>::Item: Serialize,\n    {\n        let mut iter = iter.into_iter();\n        let mut serializer = tri!(self.serialize_seq(iterator_len_hint(&iter)));\n        tri!(iter.try_for_each(|item| serializer.serialize_element(&item)));\n        serializer.end()\n    }\n\n    /// Collect an iterator as a map.\n    ///\n    /// The default implementation serializes each pair yielded by the iterator\n    /// using [`serialize_map`]. Implementors should not need to override this\n    /// method.\n    ///\n    /// ```edition2021\n    /// use serde::{Serialize, Serializer};\n    /// use std::collections::BTreeSet;\n    ///\n    /// struct MapToUnit {\n    ///     keys: BTreeSet<i32>,\n    /// }\n    ///\n    /// // Serializes as a map in which the values are all unit.\n    /// impl Serialize for MapToUnit {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.collect_map(self.keys.iter().map(|k| (k, ())))\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// [`serialize_map`]: #tymethod.serialize_map\n    fn collect_map<K, V, I>(self, iter: I) -> Result<Self::Ok, Self::Error>\n    where\n        K: Serialize,\n        V: Serialize,\n        I: IntoIterator<Item = (K, V)>,\n    {\n        let mut iter = iter.into_iter();\n        let mut serializer = tri!(self.serialize_map(iterator_len_hint(&iter)));\n        tri!(iter.try_for_each(|(key, value)| serializer.serialize_entry(&key, &value)));\n        serializer.end()\n    }\n\n    /// Serialize a string produced by an implementation of `Display`.\n    ///\n    /// The default implementation builds a heap-allocated [`String`] and\n    /// delegates to [`serialize_str`]. Serializers are encouraged to provide a\n    /// more efficient implementation if possible.\n    ///\n    /// ```edition2021\n    /// # struct DateTime;\n    /// #\n    /// # impl DateTime {\n    /// #     fn naive_local(&self) -> () { () }\n    /// #     fn offset(&self) -> () { () }\n    /// # }\n    /// #\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// impl Serialize for DateTime {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.collect_str(&format_args!(\"{:?}{:?}\", self.naive_local(), self.offset()))\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// [`String`]: https://doc.rust-lang.org/std/string/struct.String.html\n    /// [`serialize_str`]: #tymethod.serialize_str\n    #[cfg(any(feature = \"std\", feature = \"alloc\"))]\n    fn collect_str<T>(self, value: &T) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Display,\n    {\n        self.serialize_str(&value.to_string())\n    }\n\n    /// Serialize a string produced by an implementation of `Display`.\n    ///\n    /// Serializers that use `no_std` are required to provide an implementation\n    /// of this method. If no more sensible behavior is possible, the\n    /// implementation is expected to return an error.\n    ///\n    /// ```edition2021\n    /// # struct DateTime;\n    /// #\n    /// # impl DateTime {\n    /// #     fn naive_local(&self) -> () { () }\n    /// #     fn offset(&self) -> () { () }\n    /// # }\n    /// #\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// impl Serialize for DateTime {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         serializer.collect_str(&format_args!(\"{:?}{:?}\", self.naive_local(), self.offset()))\n    ///     }\n    /// }\n    /// ```\n    #[cfg(not(any(feature = \"std\", feature = \"alloc\")))]\n    fn collect_str<T>(self, value: &T) -> Result<Self::Ok, Self::Error>\n    where\n        T: ?Sized + Display;\n\n    /// Determine whether `Serialize` implementations should serialize in\n    /// human-readable form.\n    ///\n    /// Some types have a human-readable form that may be somewhat expensive to\n    /// construct, as well as a binary form that is compact and efficient.\n    /// Generally text-based formats like JSON and YAML will prefer to use the\n    /// human-readable one and binary formats like Postcard will prefer the\n    /// compact one.\n    ///\n    /// ```edition2021\n    /// # use std::fmt::{self, Display};\n    /// #\n    /// # struct Timestamp;\n    /// #\n    /// # impl Timestamp {\n    /// #     fn seconds_since_epoch(&self) -> u64 { unimplemented!() }\n    /// # }\n    /// #\n    /// # impl Display for Timestamp {\n    /// #     fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {\n    /// #         unimplemented!()\n    /// #     }\n    /// # }\n    /// #\n    /// use serde::{Serialize, Serializer};\n    ///\n    /// impl Serialize for Timestamp {\n    ///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n    ///     where\n    ///         S: Serializer,\n    ///     {\n    ///         if serializer.is_human_readable() {\n    ///             // Serialize to a human-readable string \"2015-05-15T17:01:00Z\".\n    ///             self.to_string().serialize(serializer)\n    ///         } else {\n    ///             // Serialize to a compact binary representation.\n    ///             self.seconds_since_epoch().serialize(serializer)\n    ///         }\n    ///     }\n    /// }\n    /// ```\n    ///\n    /// The default implementation of this method returns `true`. Data formats\n    /// may override this to `false` to request a compact form for types that\n    /// support one. Note that modifying this method to change a format from\n    /// human-readable to compact or vice versa should be regarded as a breaking\n    /// change, as a value serialized in human-readable mode is not required to\n    /// deserialize from the same data in compact mode.\n    #[inline]\n    fn is_human_readable(&self) -> bool {\n        true\n    }\n}\n\n/// Returned from `Serializer::serialize_seq`.\n///\n/// # Example use\n///\n/// ```edition2021\n/// # use std::marker::PhantomData;\n/// #\n/// # struct Vec<T>(PhantomData<T>);\n/// #\n/// # impl<T> Vec<T> {\n/// #     fn len(&self) -> usize {\n/// #         unimplemented!()\n/// #     }\n/// # }\n/// #\n/// # impl<'a, T> IntoIterator for &'a Vec<T> {\n/// #     type Item = &'a T;\n/// #     type IntoIter = Box<dyn Iterator<Item = &'a T>>;\n/// #     fn into_iter(self) -> Self::IntoIter {\n/// #         unimplemented!()\n/// #     }\n/// # }\n/// #\n/// use serde::ser::{Serialize, SerializeSeq, Serializer};\n///\n/// impl<T> Serialize for Vec<T>\n/// where\n///     T: Serialize,\n/// {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         let mut seq = serializer.serialize_seq(Some(self.len()))?;\n///         for element in self {\n///             seq.serialize_element(element)?;\n///         }\n///         seq.end()\n///     }\n/// }\n/// ```\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SerializeSeq` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SerializeSeq {\n    /// Must match the `Ok` type of our `Serializer`.\n    type Ok;\n\n    /// Must match the `Error` type of our `Serializer`.\n    type Error: Error;\n\n    /// Serialize a sequence element.\n    fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Finish serializing a sequence.\n    fn end(self) -> Result<Self::Ok, Self::Error>;\n}\n\n/// Returned from `Serializer::serialize_tuple`.\n///\n/// # Example use\n///\n/// ```edition2021\n/// use serde::ser::{Serialize, SerializeTuple, Serializer};\n///\n/// # mod fool {\n/// #     trait Serialize {}\n/// impl<A, B, C> Serialize for (A, B, C)\n/// #     {}\n/// # }\n/// #\n/// # struct Tuple3<A, B, C>(A, B, C);\n/// #\n/// # impl<A, B, C> Serialize for Tuple3<A, B, C>\n/// where\n///     A: Serialize,\n///     B: Serialize,\n///     C: Serialize,\n/// {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         let mut tup = serializer.serialize_tuple(3)?;\n///         tup.serialize_element(&self.0)?;\n///         tup.serialize_element(&self.1)?;\n///         tup.serialize_element(&self.2)?;\n///         tup.end()\n///     }\n/// }\n/// ```\n///\n/// ```edition2021\n/// # use std::marker::PhantomData;\n/// #\n/// # struct Array<T>(PhantomData<T>);\n/// #\n/// # impl<T> Array<T> {\n/// #     fn len(&self) -> usize {\n/// #         unimplemented!()\n/// #     }\n/// # }\n/// #\n/// # impl<'a, T> IntoIterator for &'a Array<T> {\n/// #     type Item = &'a T;\n/// #     type IntoIter = Box<dyn Iterator<Item = &'a T>>;\n/// #     fn into_iter(self) -> Self::IntoIter {\n/// #         unimplemented!()\n/// #     }\n/// # }\n/// #\n/// use serde::ser::{Serialize, SerializeTuple, Serializer};\n///\n/// # mod fool {\n/// #     trait Serialize {}\n/// impl<T> Serialize for [T; 16]\n/// #     {}\n/// # }\n/// #\n/// # impl<T> Serialize for Array<T>\n/// where\n///     T: Serialize,\n/// {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         let mut seq = serializer.serialize_tuple(16)?;\n///         for element in self {\n///             seq.serialize_element(element)?;\n///         }\n///         seq.end()\n///     }\n/// }\n/// ```\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SerializeTuple` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SerializeTuple {\n    /// Must match the `Ok` type of our `Serializer`.\n    type Ok;\n\n    /// Must match the `Error` type of our `Serializer`.\n    type Error: Error;\n\n    /// Serialize a tuple element.\n    fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Finish serializing a tuple.\n    fn end(self) -> Result<Self::Ok, Self::Error>;\n}\n\n/// Returned from `Serializer::serialize_tuple_struct`.\n///\n/// # Example use\n///\n/// ```edition2021\n/// use serde::ser::{Serialize, SerializeTupleStruct, Serializer};\n///\n/// struct Rgb(u8, u8, u8);\n///\n/// impl Serialize for Rgb {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         let mut ts = serializer.serialize_tuple_struct(\"Rgb\", 3)?;\n///         ts.serialize_field(&self.0)?;\n///         ts.serialize_field(&self.1)?;\n///         ts.serialize_field(&self.2)?;\n///         ts.end()\n///     }\n/// }\n/// ```\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SerializeTupleStruct` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SerializeTupleStruct {\n    /// Must match the `Ok` type of our `Serializer`.\n    type Ok;\n\n    /// Must match the `Error` type of our `Serializer`.\n    type Error: Error;\n\n    /// Serialize a tuple struct field.\n    fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Finish serializing a tuple struct.\n    fn end(self) -> Result<Self::Ok, Self::Error>;\n}\n\n/// Returned from `Serializer::serialize_tuple_variant`.\n///\n/// # Example use\n///\n/// ```edition2021\n/// use serde::ser::{Serialize, SerializeTupleVariant, Serializer};\n///\n/// enum E {\n///     T(u8, u8),\n///     U(String, u32, u32),\n/// }\n///\n/// impl Serialize for E {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         match *self {\n///             E::T(ref a, ref b) => {\n///                 let mut tv = serializer.serialize_tuple_variant(\"E\", 0, \"T\", 2)?;\n///                 tv.serialize_field(a)?;\n///                 tv.serialize_field(b)?;\n///                 tv.end()\n///             }\n///             E::U(ref a, ref b, ref c) => {\n///                 let mut tv = serializer.serialize_tuple_variant(\"E\", 1, \"U\", 3)?;\n///                 tv.serialize_field(a)?;\n///                 tv.serialize_field(b)?;\n///                 tv.serialize_field(c)?;\n///                 tv.end()\n///             }\n///         }\n///     }\n/// }\n/// ```\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SerializeTupleVariant` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SerializeTupleVariant {\n    /// Must match the `Ok` type of our `Serializer`.\n    type Ok;\n\n    /// Must match the `Error` type of our `Serializer`.\n    type Error: Error;\n\n    /// Serialize a tuple variant field.\n    fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Finish serializing a tuple variant.\n    fn end(self) -> Result<Self::Ok, Self::Error>;\n}\n\n/// Returned from `Serializer::serialize_map`.\n///\n/// # Example use\n///\n/// ```edition2021\n/// # use std::marker::PhantomData;\n/// #\n/// # struct HashMap<K, V>(PhantomData<K>, PhantomData<V>);\n/// #\n/// # impl<K, V> HashMap<K, V> {\n/// #     fn len(&self) -> usize {\n/// #         unimplemented!()\n/// #     }\n/// # }\n/// #\n/// # impl<'a, K, V> IntoIterator for &'a HashMap<K, V> {\n/// #     type Item = (&'a K, &'a V);\n/// #     type IntoIter = Box<dyn Iterator<Item = (&'a K, &'a V)>>;\n/// #\n/// #     fn into_iter(self) -> Self::IntoIter {\n/// #         unimplemented!()\n/// #     }\n/// # }\n/// #\n/// use serde::ser::{Serialize, SerializeMap, Serializer};\n///\n/// impl<K, V> Serialize for HashMap<K, V>\n/// where\n///     K: Serialize,\n///     V: Serialize,\n/// {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         let mut map = serializer.serialize_map(Some(self.len()))?;\n///         for (k, v) in self {\n///             map.serialize_entry(k, v)?;\n///         }\n///         map.end()\n///     }\n/// }\n/// ```\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SerializeMap` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SerializeMap {\n    /// Must match the `Ok` type of our `Serializer`.\n    type Ok;\n\n    /// Must match the `Error` type of our `Serializer`.\n    type Error: Error;\n\n    /// Serialize a map key.\n    ///\n    /// If possible, `Serialize` implementations are encouraged to use\n    /// `serialize_entry` instead as it may be implemented more efficiently in\n    /// some formats compared to a pair of calls to `serialize_key` and\n    /// `serialize_value`.\n    fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Serialize a map value.\n    ///\n    /// # Panics\n    ///\n    /// Calling `serialize_value` before `serialize_key` is incorrect and is\n    /// allowed to panic or produce bogus results.\n    fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Serialize a map entry consisting of a key and a value.\n    ///\n    /// Some [`Serialize`] types are not able to hold a key and value in memory\n    /// at the same time so `SerializeMap` implementations are required to\n    /// support [`serialize_key`] and [`serialize_value`] individually. The\n    /// `serialize_entry` method allows serializers to optimize for the case\n    /// where key and value are both available. [`Serialize`] implementations\n    /// are encouraged to use `serialize_entry` if possible.\n    ///\n    /// The default implementation delegates to [`serialize_key`] and\n    /// [`serialize_value`]. This is appropriate for serializers that do not\n    /// care about performance or are not able to optimize `serialize_entry` any\n    /// better than this.\n    ///\n    /// [`Serialize`]: ../trait.Serialize.html\n    /// [`serialize_key`]: #tymethod.serialize_key\n    /// [`serialize_value`]: #tymethod.serialize_value\n    \nfn serialize_entry<K, V>(&mut self, key: &K, value: &V) -> Result<(), Self::Error>\n    where\n        K: ?Sized + Serialize,\n        V: ?Sized + Serialize,\n    {\n        tri!(self.serialize_key(key));\n        self.serialize_value(value)\n    }\n\n    /// Finish serializing a map.\n    fn end(self) -> Result<Self::Ok, Self::Error>;\n}\n\n/// Returned from `Serializer::serialize_struct`.\n///\n/// # Example use\n///\n/// ```edition2021\n/// use serde::ser::{Serialize, SerializeStruct, Serializer};\n///\n/// struct Rgb {\n///     r: u8,\n///     g: u8,\n///     b: u8,\n/// }\n///\n/// impl Serialize for Rgb {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         let mut rgb = serializer.serialize_struct(\"Rgb\", 3)?;\n///         rgb.serialize_field(\"r\", &self.r)?;\n///         rgb.serialize_field(\"g\", &self.g)?;\n///         rgb.serialize_field(\"b\", &self.b)?;\n///         rgb.end()\n///     }\n/// }\n/// ```\n///\n/// # Example implementation\n///\n/// The [example data format] presented on the website demonstrates an\n/// implementation of `SerializeStruct` for a basic JSON data format.\n///\n/// [example data format]: https://serde.rs/data-format.html\npub trait SerializeStruct {\n    /// Must match the `Ok` type of our `Serializer`.\n    type Ok;\n\n    /// Must match the `Error` type of our `Serializer`.\n    type Error: Error;\n\n    /// Serialize a struct field.\n    fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>\n    where\n        T: ?Sized + Serialize;\n\n    /// Indicate that a struct field has been skipped.\n    ///\n    /// The default implementation does nothing.\n    #[inline]\n    fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> {\n        let _ = key;\n        Ok(())\n    }\n\n    /// Finish serializing a struct.\n    fn end(self) -> Result<Self::Ok, Self::Error>;\n}\n\n/// Returned from `Serializer::serialize_struct_variant`.\n///\n/// # Example use\n///\n/// ```edition2021\n/// use serde::ser::{Serialize, SerializeStructVariant, Serializer};\n///\n/// enum E {\n///     S { r: u8, g: u8, b: u8 },\n/// }\n///\n/// impl Serialize for E {\n///     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>\n///     where\n///         S: Serializer,\n///     {\n///         match *self {\n///             E::S {\n///                 ref r,\n///                 ref g,\n///                 ref b,\n///             } => {\n///                 let mut sv = serializer.serialize_struct_variant(\"E\", 0, \"S\", 3)?;\n///                 sv.serialize_field(\"r\", r)?;\n...\n", "needle_token_start": 15449, "needle_token_end": 15535, "code_context_ntokens": 16300, "output": ["```rust\nfn serialize_entry<K, V>(&mut self, key: &K, value: &V) -> Result<(), Self::Error>\n    where\n        K: ?Sized + Serialize,\n        V: ?Sized + Serialize,\n{\n    tri!(self.serialize_key(key));\n    self.serialize_value(value)\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "and", "language": "rust", "path": "tracing-core/src/collect.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To determine the common level of interest between two instances, particularly when their levels of interest differ.\n2. **Input**: Two instances representing levels of interest.\n3. **Output**: An instance representing the combined level of interest, which defaults to a middle level if the inputs differ.\n4. **Procedure**: The method checks if both input instances have the same level of interest. If they do, it returns the same level. If not, it defaults to returning the middle level of interest, indicating a need for further checks in varying conditions.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "///\n/// Collectors return an `Interest` from their [`register_callsite`] methods\n/// in order to determine whether that span should be enabled or disabled.\n///\n/// [`Collect`]: super::Collect\n/// [`register_callsite`]: super::Collect::register_callsite\n#[derive(Clone, Debug)]\npub struct Interest(InterestKind);\n\n#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]\nenum InterestKind {\n    Never = 0,\n    Sometimes = 1,\n    Always = 2,\n}\n\nimpl Interest {\n    /// Returns an `Interest` indicating that the collector is never interested\n    /// in being notified about a callsite.\n    ///\n    /// If all active collectors are `never()` interested in a callsite, it will\n    /// be completely disabled unless a new collector becomes active.\n    #[inline]\n    pub fn never() -> Self {\n        Interest(InterestKind::Never)\n    }\n\n    /// Returns an `Interest` indicating the collector is sometimes interested\n    /// in being notified about a callsite.\n    ///\n    /// If all active collectors are `sometimes` or `never` interested in a\n    /// callsite, the currently active collector will be asked to filter that\n    /// callsite every time it creates a span. This will be the case until a new\n    /// collector expresses that it is `always` interested in the callsite.\n    #[inline]\n    pub fn sometimes() -> Self {\n        Interest(InterestKind::Sometimes)\n    }\n\n    /// Returns an `Interest` indicating the collector is always interested in\n    /// being notified about a callsite.\n    ///\n    /// If any collector expresses that it is `always()` interested in a given\n    /// callsite, then the callsite will always be enabled.\n    #[inline]\n    pub fn always() -> Self {\n        Interest(InterestKind::Always)\n    }\n\n    /// Returns `true` if the collector is never interested in being notified\n    /// about this callsite.\n    #[inline]\n    pub fn is_never(&self) -> bool {\n        matches!(self.0, InterestKind::Never)\n    }\n\n    /// Returns `true` if the collector is sometimes interested in being notified\n    /// about this callsite.\n    #[inline]\n    pub fn is_sometimes(&self) -> bool {\n        matches!(self.0, InterestKind::Sometimes)\n    }\n\n    /// Returns `true` if the collector is always interested in being notified\n    /// about this callsite.\n    #[inline]\n    pub fn is_always(&self) -> bool {\n        matches!(self.0, InterestKind::Always)\n    }\n\n    /// Returns the common interest between these two Interests.\n    ///\n    /// If both interests are the same, this propagates that interest.\n    /// Otherwise, if they differ, the result must always be\n    /// `Interest::sometimes` --- if the two collectors differ in opinion, we\n    /// will have to ask the current collector what it thinks, no matter what.\n    // Only needed when combining interest from multiple collectors.\n    #[cfg(feature = \"std\")]\n    pub(cr\nate) fn and(self, rhs: Interest) -> Self {\n        if self.0 == rhs.0 {\n            self\n        } else {\n            Interest::sometimes()\n        }\n    }\n}\n\n/// A no-op [collector](Collect).\n///\n/// [`NoCollector`] implements the [`Collect`] trait by never being enabled,\n/// never being interested in any callsite, and drops all spans and events.\n#[derive(Debug, Default, Copy, Clone)]\npub struct NoCollector(());\n\nimpl NoCollector {\n    /// Returns a new `NoCollector` instance.\n    ///\n    /// This function is equivalent to calling `NoCollector::default()`, but\n    /// this is usable in `const fn` contexts.\n    pub const fn new() -> Self {\n        Self(())\n    }\n}\n\nimpl Collect for NoCollector {\n    #[inline]\n    fn register_callsite(&self, _: &'static Metadata<'static>) -> Interest {\n        Interest::never()\n    }\n\n    fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n        span::Id::from_u64(0xDEAD)\n    }\n\n    fn event(&self, _event: &Event<'_>) {}\n\n    fn record(&self, _span: &span::Id, _values: &span::Record<'_>) {}\n\n    fn record_follows_from(&self, _span: &span::Id, _follows: &span::Id) {}\n\n    #[inline]\n    fn enabled(&self, _metadata: &Metadata<'_>) -> bool {\n        false\n    }\n\n    fn current_span(&self) -> span::Current {\n        span::Current::none()\n    }\n\n    fn enter(&self, _span: &span::Id) {}\n    fn exit(&self, _span: &span::Id) {}\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<C> Collect for alloc::boxed::Box<C>\nwhere\n    C: Collect + ?Sized,\n{\n    #[inline]\n    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {\n        self.as_ref().register_callsite(metadata)\n    }\n\n    #[inline]\n    fn enabled(&self, metadata: &Metadata<'_>) -> bool {\n        self.as_ref().enabled(metadata)\n    }\n\n    #[inline]\n    fn max_level_hint(&self) -> Option<LevelFilter> {\n        self.as_ref().max_level_hint()\n    }\n\n    #[inline]\n    fn new_span(&self, span: &span::Attributes<'_>) -> span::Id {\n        self.as_ref().new_span(span)\n    }\n\n    #[inline]\n    fn record(&self, span: &span::Id, values: &span::Record<'_>) {\n        self.as_ref().record(span, values)\n    }\n\n    #[inline]\n    fn record_follows_from(&self, span: &span::Id, follows: &span::Id) {\n        self.as_ref().record_follows_from(span, follows)\n    }\n\n    #[inline]\n    fn event_enabled(&self, event: &Event<'_>) -> bool {\n        self.as_ref().event_enabled(event)\n    }\n\n    #[inline]\n    fn event(&self, event: &Event<'_>) {\n        self.as_ref().event(event)\n    }\n\n    #[inline]\n    fn enter(&self, span: &span::Id) {\n        self.as_ref().enter(span)\n    }\n\n    #[inline]\n    fn exit(&self, span: &span::Id) {\n        self.as_ref().exit(span)\n    }\n\n    #[inline]\n    fn clone_span(&self, id: &span::Id) -> span::Id {\n        self.as_ref().clone_span(id)\n    }\n\n    #[inline]\n    fn try_close(&self, id: span::Id) -> bool {\n        self.as_ref().try_close(id)\n    }\n\n    #[inline]\n    unsafe fn downcast_raw(&self, id: TypeId) -> Option<NonNull<()>> {\n        if id == TypeId::of::<Self>() {\n            return Some(NonNull::from(self).cast());\n        }\n\n        self.as_ref().downcast_raw(id)\n    }\n\n    fn current_span(&self) -> span::Current {\n        self.as_ref().current_span()\n    }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<C> Collect for alloc::sync::Arc<C>\nwhere\n    C: Collect + ?Sized,\n{\n    #[inline]\n    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {\n        self.as_ref().register_callsite(metadata)\n    }\n\n    #[inline]\n    fn enabled(&self, metadata: &Metadata<'_>) -> bool {\n        self.as_ref().enabled(metadata)\n    }\n\n    #[inline]\n    fn max_level_hint(&self) -> Option<LevelFilter> {\n        self.as_ref().max_level_hint()\n    }\n\n    #[inline]\n    fn new_span(&self, span: &span::Attributes<'_>) -> span::Id {\n        self.as_ref().new_span(span)\n    }\n\n    #[inline]\n    fn record(&self, span: &span::Id, values: &span::Record<'_>) {\n        self.as_ref().record(span, values)\n    }\n\n    #[inline]\n    fn record_follows_from(&self, span: &span::Id, follows: &span::Id) {\n        self.as_ref().record_follows_from(span, follows)\n    }\n\n    #[inline]\n    fn event_enabled(&self, event: &Event<'_>) -> bool {\n        self.as_ref().event_enabled(event)\n    }\n\n    #[inline]\n    fn event(&self, event: &Event<'_>) {\n        self.as_ref().event(event)\n    }\n\n    #[inline]\n    fn enter(&self, span: &span::Id) {\n        self.as_ref().enter(span)\n    }\n\n    #[inline]\n    fn exit(&self, span: &span::Id) {\n        self.as_ref().exit(span)\n    }\n\n    #[inline]\n    fn clone_span(&self, id: &span::Id) -> span::Id {\n        self.as_ref().clone_span(id)\n    }\n\n    #[inline]\n    fn try_close(&self, id: span::Id) -> bool {\n        self.as_ref().try_close(id)\n    }\n\n    #[inline]\n    unsafe fn downcast_raw(&self, id: TypeId) -> Option<NonNull<()>> {\n        if id == TypeId::of::<Self>() {\n            return Some(NonNull::from(self).cast());\n        }\n\n        self.as_ref().downcast_raw(id)\n    }\n\n    fn current_span(&self) -> span::Current {\n        self.as_ref().current_span()\n    }\n}\n\n// Path: tracing-core/src/callsite.rs\n//! Callsites represent the source locations from which spans or events\n//! originate.\n//!\n//! # What Are Callsites?\n//!\n//! Every span or event in `tracing` is associated with a [`Callsite`]. A\n//! callsite is a small `static` value that is responsible for the following:\n//!\n//! * Storing the span or event's [`Metadata`],\n//! * Uniquely [identifying](Identifier) the span or event definition,\n//! * Caching the collector's [`Interest`][^1] in that span or event, to avoid\n//!   re-evaluating filters,\n//! * Storing a [`Registration`] that allows the callsite to be part of a global\n//!   list of all callsites in the program.\n//!\n//! # Registering Callsites\n//!\n//! When a span or event is recorded for the first time, its callsite\n//! [`register`]s itself with the global callsite registry. Registering a\n//! callsite calls the [`Collect::register_callsite`][`register_callsite`]\n//! method with that callsite's [`Metadata`] on every currently active\n//! collector. This serves two primary purposes: informing collectors of the\n//! callsite's existence, and performing static filtering.\n//!\n//! ## Callsite Existence\n//!\n//! If a [`Collect`] implementation wishes to allocate storage for each\n//! unique span/event location in the program, or pre-compute some value\n//! that will be used to record that span or event in the future, it can\n//! do so in its [`register_callsite`] method.\n//!\n//! ## Performing Static Filtering\n//!\n//! The [`register_callsite`] method returns an [`Interest`] value,\n//! which indicates that the collector either [always] wishes to record\n//! that span or event, [sometimes] wishes to record it based on a\n//! dynamic filter evaluation, or [never] wishes to record it.\n//!\n//! When registering a new callsite, the [`Interest`]s returned by every\n//! currently active collector are combined, and the result is stored at\n//! each callsite. This way, when the span or event occurs in the\n//! future, the cached [`Interest`] value can be checked efficiently\n//! to determine if the span or event should be recorded, without\n//! needing to perform expensive filtering (i.e. calling the\n//! [`Collect::enabled`] method every time a span or event occurs).\n//!\n//! ### Rebuilding Cached Interest\n//!\n//! When a new [`Dispatch`] is created (i.e. a new collector becomes\n//! active), any previously cached [`Interest`] values are re-evaluated\n//! for all callsites in the program. This way, if the new collector\n//! will enable a callsite that was not previously enabled, the\n//! [`Interest`] in that callsite is updated. Similarly, when a\n//! collector is dropped, the interest cache is also re-evaluated, so\n//! that any callsites enabled only by that collector are disabled.\n//!\n//! In addition, the [`rebuild_interest_cache`] function in this module can be\n//! used to manually invalidate all cached interest and re-register those\n//! callsites. This function is useful in situations where a collector's\n//! interest can change, but it does so relatively infrequently. The collector\n//! may wish for its interest to be cached most of the time, and return\n//! [`Interest::always`][always] or [`Interest::never`][never] in its\n//! [`register_callsite`] method, so that its [`Collect::enabled`] method\n//! doesn't need to be evaluated every time a span or event is recorded.\n//! However, when the configuration changes, the collector can call\n//! [`rebuild_interest_cache`] to re-evaluate the entire interest cache with its\n//! new configuration. This is a relatively costly operation, but if the\n//! configuration changes infrequently, it may be more efficient than calling\n//! [`Collect::enabled`] frequently.\n//!\n//! [^1]: Returned by the [`Collect::register_callsite`][`register_callsite`]\n//!     method.\n//!\n//! [`Metadata`]: crate::metadata::Metadata\n//! [`Interest`]: crate::collect::Interest\n//! [`Collect`]: crate::collect::Collect\n//! [`register_callsite`]: crate::collect::Collect::register_callsite\n//! [`Collect::enabled`]: crate::collect::Collect::enabled\n//! [always]: crate::collect::Interest::always\n//! [sometimes]: crate::collect::Interest::sometimes\n//! [never]: crate::collect::Interest::never\n//! [`Dispatch`]: crate::dispatch::Dispatch\nuse crate::{\n    collect::Interest,\n    dispatch::{self, Dispatch},\n    metadata::{LevelFilter, Metadata},\n};\nuse core::{\n    fmt,\n    hash::{Hash, Hasher},\n    ptr,\n    sync::atomic::{AtomicPtr, Ordering},\n};\n\ntype Callsites = LinkedList;\n\n/// Trait implemented by callsites.\n///\n/// These functions are only intended to be called by the callsite registry, which\n/// correctly handles determining the common interest between all collectors.\n///\n/// See the [module-level documentation](crate::callsite) for details on\n/// callsites.\npub trait Callsite: Sync {\n    /// Sets the [`Interest`] for this callsite.\n    ///\n    /// See the [documentation on callsite interest caching][cache-docs] for\n    /// details.\n    ///\n    /// [`Interest`]: super::collect::Interest\n    /// [cache-docs]: crate::callsite#performing-static-filtering\n    fn set_interest(&self, interest: Interest);\n\n    /// Returns the [metadata] associated with the callsite.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    ///\n    /// **Note:** Implementations of this method should not produce [`Metadata`]\n    /// that share the same callsite [`Identifier`] but otherwise differ in any\n    /// way (e.g., have different `name`s).\n    ///\n    /// </pre></div>\n    ///\n    /// [metadata]: super::metadata::Metadata\n    fn metadata(&self) -> &Metadata<'_>;\n}\n\n/// Uniquely identifies a [`Callsite`]\n///\n/// Two `Identifier`s are equal if they both refer to the same callsite.\n///\n/// [`Callsite`]: super::callsite::Callsite\n#[derive(Clone)]\npub struct Identifier(\n    /// **Warning**: The fields on this type are currently `pub` because it must\n    /// be able to be constructed statically by macros. However, when `const\n    /// fn`s are available on stable Rust, this will no longer be necessary.\n    /// Thus, these fields are *not* considered stable public API, and they may\n    /// change warning. Do not rely on any fields on `Identifier`. When\n    /// constructing new `Identifier`s, use the `identify_callsite!` macro or\n    /// the `Callsite::id` function instead.\n    // TODO: When `Callsite::id` is a const fn, this need no longer be `pub`.\n    #[doc(hidden)]\n    pub &'static dyn Callsite,\n);\n\n/// A registration with the callsite registry.\n///\n/// Every [`Callsite`] implementation must provide a `&'static Registration`\n/// when calling [`register`] to add itself to the global callsite registry.\n///\n/// See [the documentation on callsite registration][registry-docs] for details\n/// on how callsites are registered.\n///\n/// [`Callsite`]: crate::callsite::Callsite\n/// [`register`]: crate::callsite::register\n/// [registry-docs]: crate::callsite#registering-callsites\npub struct Registration<T = &'static dyn Callsite> {\n    callsite: T,\n    next: AtomicPtr<Registration<T>>,\n}\n\npub(crate) use self::inner::register_dispatch;\npub use self::inner::{rebuild_interest_cache, register};\n\n#[cfg(feature = \"std\")]\nmod inner {\n    use super::*;\n    use once_cell::sync::Lazy;\n    use std::sync::RwLock;\n    use std::vec::Vec;\n\n    type Dispatchers = Vec<dispatch::Registrar>;\n\n    struct Registry {\n        callsites: Callsites,\n        dispatchers: RwLock<Dispatchers>,\n    }\n\n    static REGISTRY: Lazy<Registry> = Lazy::new(|| Registry {\n        callsites: LinkedList::new(),\n        dispatchers: RwLock::new(Vec::new()),\n    });\n\n    /// Clear and reregister interest on every [`Callsite`]\n    ///\n    /// This function is intended for runtime reconfiguration of filters on traces\n    /// when the filter recalculation is much less frequent than trace events are.\n    /// The alternative is to have the [`Collect`] that supports runtime\n    /// reconfiguration of filters always return [`Interest::sometimes()`] so that\n    /// [`enabled`] is evaluated for every event.\n    ///\n    /// This function will also re-compute the global maximum level as determined by\n    /// the [`max_level_hint`] method. If a [`Collect`]\n    /// implementation changes the value returned by its `max_level_hint`\n    /// implementation at runtime, then it **must** call this function after that\n    /// value changes, in order for the change to be reflected.\n    ///\n    /// See the [documentation on callsite interest caching][cache-docs] for\n    /// additional information on this function's usage.\n    ///\n    /// [`max_level_hint`]: crate::collect::Collect::max_level_hint\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [`enabled`]: crate::collect::Collect::enabled\n    /// [`Interest::sometimes()`]: crate::collect::Interest::sometimes\n    /// [`Collect`]: crate::collect::Collect\n    /// [cache-docs]: crate::callsite#rebuilding-cached-interest\n    pub fn rebuild_interest_cache() {\n        let mut dispatchers = REGISTRY.dispatchers.write().unwrap();\n        let callsites = &REGISTRY.callsites;\n        rebuild_interest(callsites, &mut dispatchers);\n    }\n\n    /// Register a new [`Callsite`] with the global registry.\n    ///\n    /// This should be called once per callsite after the callsite has been\n    /// constructed.\n    ///\n    /// See the [documentation on callsite registration][reg-docs] for details\n    /// on the global callsite registry.\n    ///\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [reg-docs]: crate::callsite#registering-callsites\n    pub fn register(registration: &'static Registration) {\n        let dispatchers = REGISTRY.dispatchers.read().unwrap();\n        rebuild_callsite_interest(&dispatchers, registration.callsite);\n        REGISTRY.callsites.push(registration);\n    }\n\n    pub(crate) fn register_dispatch(dispatch: &Dispatch) {\n        let mut dispatchers = REGISTRY.dispatchers.write().unwrap();\n        let callsites = &REGISTRY.callsites;\n\n        dispatch.collector().on_register_dispatch(dispatch);\n        dispatchers.push(dispatch.registrar());\n\n        rebuild_interest(callsites, &mut dispatchers);\n    }\n\n    fn rebuild_callsite_interest(\n        dispatchers: &[dispatch::Registrar],\n        callsite: &'static dyn Callsite,\n    ) {\n        let meta = callsite.metadata();\n\n        // Iterate over the collectors in the registry, and \u2014 if they are\n        // active \u2014 register the callsite with them.\n        let mut interests = dispatchers.iter().filter_map(|registrar| {\n            registrar\n                .upgrade()\n                .map(|dispatch| dispatch.register_callsite(meta))\n        });\n\n        // Use the first collector's `Interest` as the base value.\n        let interest = if let Some(interest) = interests.next() {\n            // Combine all remaining `Interest`s.\n            interests.fold(interest, Interest::and)\n        } else {\n            // If nobody was interested in this thing, just return `never`.\n            Interest::never()\n        };\n\n        callsite.set_interest(interest)\n    }\n\n    fn rebuild_interest(callsites: &Callsites, dispatchers: &mut Vec<dispatch::Registrar>) {\n        let mut max_level = LevelFilter::OFF;\n        dispatchers.retain(|registrar| {\n            if let Some(dispatch) = registrar.upgrade() {\n                // If the collector did not provide a max level hint, assume\n                // that it may enable every level.\n                let level_hint = dispatch.max_level_hint().unwrap_or(LevelFilter::TRACE);\n                if level_hint > max_level {\n                    max_level = level_hint;\n                }\n                true\n            } else {\n                false\n            }\n        });\n\n        callsites.for_each(|reg| rebuild_callsite_interest(dispatchers, reg.callsite));\n\n        LevelFilter::set_max(max_level);\n    }\n}\n\n#[cfg(not(feature = \"std\"))]\nmod inner {\n    use super::*;\n    static REGISTRY: Callsites = LinkedList::new();\n\n    /// Clear and reregister interest on every [`Callsite`]\n    ///\n    /// This function is intended for runtime reconfiguration of filters on traces\n    /// when the filter recalculation is much less frequent than trace events are.\n    /// The alternative is to have the [collector] that supports runtime\n    /// reconfiguration of filters always return [`Interest::sometimes()`] so that\n    /// [`enabled`] is evaluated for every event.\n    ///\n    /// This function will also re-compute the global maximum level as determined by\n    /// the [`max_level_hint`] method. If a [`Collect`]\n    /// implementation changes the value returned by its `max_level_hint`\n    /// implementation at runtime, then it **must** call this function after that\n    /// value changes, in order for the change to be reflected.\n    ///\n    /// See the [documentation on callsite interest caching][cache-docs] for\n    /// additional information on this function's usage.\n    ///\n    /// [`max_level_hint`]: crate::collector::Collector::max_level_hint\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [`enabled`]: crate::collector::Collector::enabled\n    /// [`Interest::sometimes()`]: crate::collect::Interest::sometimes\n    /// [collector]: crate::collect::Collect\n    /// [`Collect`]: crate::collect::Collect\n    /// [cache-docs]: crate::callsite#rebuilding-cached-interest\n    pub fn rebuild_interest_cache() {\n        register_dispatch(dispatch::get_global());\n    }\n\n    /// Register a new [`Callsite`] with the global registry.\n    ///\n    /// This should be called once per callsite after the callsite has been\n    /// constructed.\n    ///\n    /// See the [documentation on callsite registration][reg-docs] for details\n    /// on the global callsite registry.\n    ///\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [reg-docs]: crate::callsite#registering-callsites\n    pub fn register(registration: &'static Registration) {\n        rebuild_callsite_interest(dispatch::get_global(), registration.callsite);\n        REGISTRY.push(registration);\n    }\n\n    pub(crate) fn register_dispatch(dispatcher: &Dispatch) {\n        // If the collector did not provide a max level hint, assume\n        // that it may enable every level.\n        let level_hint = dispatcher.max_level_hint().unwrap_or(LevelFilter::TRACE);\n\n        REGISTRY.for_each(|reg| rebuild_callsite_interest(dispatcher, reg.callsite));\n\n        LevelFilter::set_max(level_hint);\n    }\n\n    fn rebuild_callsite_interest(dispatcher: &Dispatch, callsite: &'static dyn Callsite) {\n        let meta = callsite.metadata();\n\n        callsite.set_interest(dispatcher.register_callsite(meta))\n    }\n}\n\n// ===== impl Identifier =====\n\nimpl PartialEq for Identifier {\n    fn eq(&self, other: &Identifier) -> bool {\n        core::ptr::eq(\n            self.0 as *const _ as *const (),\n            other.0 as *const _ as *const (),\n        )\n    }\n}\n\nimpl Eq for Identifier {}\n\nimpl fmt::Debug for Identifier {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Identifier({:p})\", self.0)\n    }\n}\n\nimpl Hash for Identifier {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: Hasher,\n    {\n        (self.0 as *const dyn Callsite).hash(state)\n    }\n}\n\n// ===== impl Registration =====\n\nimpl<T> Registration<T> {\n    /// Construct a new `Registration` from some `&'static dyn Callsite`\n    pub const fn new(callsite: T) -> Self {\n        Self {\n            callsite,\n            next: AtomicPtr::new(ptr::null_mut()),\n        }\n    }\n}\n\nimpl fmt::Debug for Registration {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"Registration\")\n            .field(\"callsite\", &format_args!(\"{:p}\", self.callsite))\n            .field(\n                \"next\",\n                &format_args!(\"{:p}\", self.next.load(Ordering::Acquire)),\n            )\n            .finish()\n    }\n}\n\n// ===== impl LinkedList =====\n\n/// An intrusive atomic push-only linked list.\nstruct LinkedList<T = &'static dyn Callsite> {\n    head: AtomicPtr<Registration<T>>,\n}\n\nimpl<T> LinkedList<T> {\n    const fn new() -> Self {\n        LinkedList {\n            head: AtomicPtr::new(ptr::null_mut()),\n        }\n    }\n}\n\nimpl LinkedList {\n    fn for_each(&self, mut f: impl FnMut(&'static Registration)) {\n        let mut head = self.head.load(Ordering::Acquire);\n\n        while let Some(reg) = unsafe { head.as_ref() } {\n            f(reg);\n\n            head = reg.next.load(Ordering::Acquire);\n        }\n    }\n\n    fn push(&self, registration: &'static Registration) {\n        let mut head = self.head.load(Ordering::Acquire);\n\n        loop {\n            registration.next.store(head, Ordering::Release);\n\n            assert_ne!(\n                registration as *const _, head,\n                \"Attempted to register a `Callsite` that already exists! \\\n                This will cause an infinite loop when attempting to read from the \\\n                callsite cache. This is likely a bug! You should only need to call \\\n                `tracing-core::callsite::register` once per `Callsite`.\"\n            );\n\n            match self.head.compare_exchange(\n                head,\n                registration as *const _ as *mut _,\n                Ordering::AcqRel,\n                Ordering::Acquire,\n            ) {\n                Ok(_) => {\n                    break;\n                }\n                Err(current) => head = current,\n            }\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    struct TestCallsite;\n    static CS1: TestCallsite = TestCallsite;\n    static CS2: TestCallsite = TestCallsite;\n\n    impl Callsite for TestCallsite {\n        fn set_interest(&self, _interest: Interest) {}\n        fn metadata(&self) -> &Metadata<'_> {\n            unimplemented!(\"not needed for this test\")\n        }\n    }\n\n    #[test]\n    fn linked_list_push() {\n        static REG1: Registration = Registration::new(&CS1);\n        static REG2: Registration = Registration::new(&CS2);\n\n        let linked_list = LinkedList::new();\n\n        linked_list.push(&REG1);\n        linked_list.push(&REG2);\n\n        let mut i = 0;\n\n        linked_list.for_each(|reg| {\n            if i == 0 {\n                assert!(\n                    ptr::eq(reg, &REG2),\n                    \"Registration pointers need to match REG2\"\n                );\n            } else {\n                assert!(\n                    ptr::eq(reg, &REG1),\n                    \"Registration pointers need to match REG1\"\n                );\n            }\n\n            i += 1;\n        });\n    }\n\n    #[test]\n    fn linked_list_push_several() {\n        static REG1: Registration = Registration::new(&CS1);\n        static REG2: Registration = Registration::new(&CS2);\n        static REG3: Registration = Registration::new(&CS1);\n        static REG4: Registration = Registration::new(&CS2);\n\n        let linked_list = LinkedList::new();\n\n        fn expect(\n            callsites: &mut impl Iterator<Item = &'static Registration>,\n        ) -> impl FnMut(&'static Registration) + '_ {\n            move |reg: &'static Registration| {\n                let ptr = callsites\n                    .next()\n                    .expect(\"list contained more than the expected number of registrations!\");\n\n                assert!(\n                    ptr::eq(reg, ptr),\n                    \"Registration pointers need to match ({:?} != {:?})\",\n                    reg,\n                    ptr\n                );\n            }\n        }\n\n        linked_list.push(&REG1);\n        linked_list.push(&REG2);\n        let regs = [&REG2, &REG1];\n        let mut callsites = regs.iter().copied();\n        linked_list.for_each(expect(&mut callsites));\n        assert!(\n            callsites.next().is_none(),\n            \"some registrations were expected but not present: {:?}\",\n            callsites\n        );\n\n        linked_list.push(&REG3);\n        let regs = [&REG3, &REG2, &REG1];\n        let mut callsites = regs.iter().copied();\n        linked_list.for_each(expect(&mut callsites));\n        assert!(\n            callsites.next().is_none(),\n            \"some registrations were expected but not present: {:?}\",\n            callsites\n        );\n\n        linked_list.push(&REG4);\n        let regs = [&REG4, &REG3, &REG2, &REG1];\n        let mut callsites = regs.iter().copied();\n        linked_list.for_each(expect(&mut callsites));\n        assert!(\n            callsites.next().is_none(),\n            \"some registrations were expected but not present: {:?}\",\n            callsites\n        );\n    }\n\n    #[test]\n    #[should_panic]\n    fn linked_list_repeated() {\n        static REG1: Registration = Registration::new(&CS1);\n\n        let linked_list = LinkedList::new();\n\n        linked_list.push(&REG1);\n        // Pass in same reg and we should panic...\n        linked_list.push(&REG1);\n\n        linked_list.for_each(|_| {});\n    }\n\n    #[test]\n    fn linked_list_empty() {\n        let linked_list = LinkedList::new();\n\n        linked_list.for_each(|_| {\n            panic!(\"List should be empty\");\n        });\n    }\n}\n\n// Path: tracing-core/src/metadata.rs\n//! Metadata describing trace data.\nuse super::{callsite, field};\nuse core::{\n    cmp, fmt,\n    str::FromStr,\n    sync::atomic::{AtomicUsize, Ordering},\n};\n\n/// Metadata describing a [span] or [event].\n///\n/// All spans and events have the following metadata:\n/// - A [name], represented as a static string.\n/// - A [target], a string that categorizes part of the system where the span\n///   or event occurred. The `tracing` macros default to using the module\n///   path where the span or event originated as the target, but it may be\n///   overridden.\n/// - A [verbosity level]. This determines how verbose a given span or event\n///   is, and allows enabling or disabling more verbose diagnostics\n///   situationally. See the documentation for the [`Level`] type for details.\n/// - The names of the [fields] defined by the span or event.\n/// - Whether the metadata corresponds to a span or event.\n///\n/// In addition, the following optional metadata describing the source code\n/// location where the span or event originated _may_ be provided:\n/// - The [file name]\n/// - The [line number]\n/// - The [module path]\n///\n/// Metadata is used by [collector]s when filtering spans and events, and it\n/// may also be used as part of their data payload.\n///\n/// When created by the `event!` or `span!` macro, the metadata describing a\n/// particular event or span is constructed statically and exists as a single\n/// static instance. Thus, the overhead of creating the metadata is\n/// _significantly_ lower than that of creating the actual span. Therefore,\n/// filtering is based on metadata, rather than on the constructed span.\n///\n/// ## Equality\n///\n/// In well-behaved applications, two `Metadata` with equal\n/// [callsite identifiers] will be equal in all other ways (i.e., have the same\n/// `name`, `target`, etc.). Consequently, in release builds, [`Metadata::eq`]\n/// *only* checks that its arguments have equal callsites. However, the equality\n/// of `Metadata`'s other fields is checked in debug builds.\n///\n/// [span]: super::span\n/// [event]: super::event\n/// [name]: Self::name\n/// [target]: Self::target\n/// [fields]: Self::fields\n/// [verbosity level]: Self::level\n/// [file name]: Self::file\n/// [line number]: Self::line\n/// [module path]: Self::module_path\n/// [collector]: super::collect::Collect\n/// [callsite identifiers]: Self::callsite\npub struct Metadata<'a> {\n    /// The name of the span described by this metadata.\n    name: &'static str,\n\n    /// The part of the system that the span that this metadata describes\n    /// occurred in.\n    target: &'a str,\n\n    /// The level of verbosity of the described span.\n    level: Level,\n\n    /// The name of the Rust module where the span occurred, or `None` if this\n    /// could not be determined.\n    module_path: Option<&'a str>,\n\n    /// The name of the source code file where the span occurred, or `None` if\n    /// this could not be determined.\n    file: Option<&'a str>,\n\n    /// The line number in the source code file where the span occurred, or\n    /// `None` if this could not be determined.\n    line: Option<u32>,\n\n    /// The names of the key-value fields attached to the described span or\n    /// event.\n    fields: field::FieldSet,\n\n    /// The kind of the callsite.\n    kind: Kind,\n}\n\n/// Indicates whether the callsite is a span or event.\n#[derive(Clone, Eq, PartialEq)]\npub struct Kind(u8);\n\n/// Describes the level of verbosity of a span or event.\n///\n/// # Comparing Levels\n///\n/// `Level` implements the [`PartialOrd`] and [`Ord`] traits, allowing two\n/// `Level`s to be compared to determine which is considered more or less\n/// verbose. Levels which are more verbose are considered \"greater than\" levels\n/// which are less verbose, with [`Level::ERROR`] considered the lowest, and\n/// [`Level::TRACE`] considered the highest.\n///\n/// For example:\n/// ```\n/// use tracing_core::Level;\n///\n/// assert!(Level::TRACE > Level::DEBUG);\n/// assert!(Level::ERROR < Level::WARN);\n/// assert!(Level::INFO <= Level::DEBUG);\n/// assert_eq!(Level::TRACE, Level::TRACE);\n/// ```\n///\n/// # Filtering\n///\n/// `Level`s are typically used to implement filtering that determines which\n/// spans and events are enabled. Depending on the use case, more or less\n/// verbose diagnostics may be desired. For example, when running in\n/// development, [`DEBUG`]-level traces may be enabled by default. When running in\n/// production, only [`INFO`]-level and lower traces might be enabled. Libraries\n/// may include very verbose diagnostics at the [`DEBUG`] and/or [`TRACE`] levels.\n/// Applications using those libraries typically chose to ignore those traces. However, when\n/// debugging an issue involving said libraries, it may be useful to temporarily\n/// enable the more verbose traces.\n///\n/// The [`LevelFilter`] type is provided to enable filtering traces by\n/// verbosity. `Level`s can be compared against [`LevelFilter`]s, and\n/// [`LevelFilter`] has a variant for each `Level`, which compares analogously\n/// to that level. In addition, [`LevelFilter`] adds a [`LevelFilter::OFF`]\n/// variant, which is considered \"less verbose\" than every other `Level`. This is\n/// intended to allow filters to completely disable tracing in a particular context.\n///\n/// For example:\n/// ```\n/// use tracing_core::{Level, LevelFilter};\n///\n/// assert!(LevelFilter::OFF < Level::TRACE);\n/// assert!(LevelFilter::TRACE > Level::DEBUG);\n/// assert!(LevelFilter::ERROR < Level::WARN);\n/// assert!(LevelFilter::INFO <= Level::DEBUG);\n/// assert!(LevelFilter::INFO >= Level::INFO);\n/// ```\n///\n/// ## Examples\n///\n/// Below is a simple example of how a [collector] could implement filtering through\n/// a [`LevelFilter`]. When a span or event is recorded, the [`Collect::enabled`] method\n/// compares the span or event's `Level` against the configured [`LevelFilter`].\n/// The optional [`Collect::max_level_hint`] method can also be implemented to  allow spans\n/// and events above a maximum verbosity level to be skipped more efficiently,\n/// often improving performance in short-lived programs.\n///\n/// ```\n/// use tracing_core::{span, Event, Level, LevelFilter, Collect, Metadata};\n/// # use tracing_core::span::{Id, Record, Current};\n///\n/// #[derive(Debug)]\n/// pub struct MyCollector {\n///     /// The most verbose level that this collector will enable.\n///     max_level: LevelFilter,\n///\n///     // ...\n/// }\n///\n/// impl MyCollector {\n///     /// Returns a new `MyCollector` which will record spans and events up to\n///     /// `max_level`.\n///     pub fn with_max_level(max_level: LevelFilter) -> Self {\n///         Self {\n///             max_level,\n///             // ...\n///         }\n///     }\n/// }\n/// impl Collect for MyCollector {\n///     fn enabled(&self, meta: &Metadata<'_>) -> bool {\n///         // A span or event is enabled if it is at or below the configured\n///         // maximum level.\n///         meta.level() <= &self.max_level\n///     }\n///\n///     // This optional method returns the most verbose level that this\n///     // collector will enable. Although implementing this method is not\n///     // *required*, it permits additional optimizations when it is provided,\n///     // allowing spans and events above the max level to be skipped\n///     // more efficiently.\n///     fn max_level_hint(&self) -> Option<LevelFilter> {\n///         Some(self.max_level)\n///     }\n///\n///     // Implement the rest of the collector...\n///     fn new_span(&self, span: &span::Attributes<'_>) -> span::Id {\n///         // ...\n///         # drop(span); Id::from_u64(1)\n///     }\n\n///     fn event(&self, event: &Event<'_>) {\n///         // ...\n///         # drop(event);\n///     }\n///\n///     // ...\n///     # fn enter(&self, _: &Id) {}\n///     # fn exit(&self, _: &Id) {}\n///     # fn record(&self, _: &Id, _: &Record<'_>) {}\n///     # fn record_follows_from(&self, _: &Id, _: &Id) {}\n///     # fn current_span(&self) -> Current { Current::unknown() }\n/// }\n/// ```\n///\n/// It is worth noting that the `tracing-subscriber` crate provides [additional\n/// APIs][envfilter] for performing more sophisticated filtering, such as\n/// enabling different levels based on which module or crate a span or event is\n/// recorded in.\n///\n/// [`DEBUG`]: Level::DEBUG\n/// [`INFO`]: Level::INFO\n/// [`TRACE`]: Level::TRACE\n/// [`Collect::enabled`]: crate::collect::Collect::enabled\n/// [`Collect::max_level_hint`]: crate::collect::Collect::max_level_hint\n/// [collector]: crate::collect::Collect\n/// [envfilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html\n#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]\npub struct Level(LevelInner);\n\n/// A filter comparable to a verbosity [`Level`].\n///\n/// If a [`Level`] is considered less than or equal to a `LevelFilter`, it\n/// should be considered enabled; if greater than the `LevelFilter`, that level\n/// is disabled. See [`LevelFilter::current`] for more details.\n///\n/// Note that this is essentially identical to the `Level` type, but with the\n/// addition of an [`OFF`] level that completely disables all trace\n/// instrumentation.\n///\n/// See the documentation for the [`Level`] type to see how `Level`s\n/// and `LevelFilter`s interact.\n///\n/// [`OFF`]: LevelFilter::OFF\n#[repr(transparent)]\n#[derive(Copy, Clone, Eq, PartialEq, Hash)]\npub struct LevelFilter(Option<Level>);\n\n/// Indicates that a string could not be parsed to a valid level.\n#[derive(Clone, Debug)]\npub struct ParseLevelFilterError(());\n\nstatic MAX_LEVEL: AtomicUsize = AtomicUsize::new(LevelFilter::OFF_USIZE);\n\n// ===== impl Metadata =====\n\nimpl<'a> Metadata<'a> {\n    /// Construct new metadata for a span or event, with a name, target, level, field\n    /// names, and optional source code location.\n    pub const fn new(\n        name: &'static str,\n        target: &'a str,\n        level: Level,\n        file: Option<&'a str>,\n        line: Option<u32>,\n        module_path: Option<&'a str>,\n        fields: field::FieldSet,\n        kind: Kind,\n    ) -> Self {\n        Metadata {\n            name,\n            target,\n            level,\n            module_path,\n            file,\n            line,\n            fields,\n            kind,\n        }\n    }\n\n    /// Returns the names of the fields on the described span or event.\n    #[inline]\n    pub fn fields(&self) -> &field::FieldSet {\n        &self.fields\n    }\n\n    /// Returns the level of verbosity of the described span or event.\n    pub fn level(&self) -> &Level {\n        &self.level\n    }\n\n    /// Returns the name of the span.\n    pub fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Returns a string describing the part of the system where the span or\n    /// event that this metadata describes occurred.\n    ///\n    /// Typically, this is the module path, but alternate targets may be set\n    /// when spans or events are constructed.\n    pub fn target(&self) -> &'a str {\n        self.target\n    }\n\n    /// Returns the path to the Rust module where the span occurred, or\n    /// `None` if the module path is unknown.\n    pub fn module_path(&self) -> Option<&'a str> {\n        self.module_path\n    }\n\n    /// Returns the name of the source code file where the span\n    /// occurred, or `None` if the file is unknown\n    pub fn file(&self) -> Option<&'a str> {\n        self.file\n    }\n\n    /// Returns the line number in the source code file where the span\n    /// occurred, or `None` if the line number is unknown.\n    pub fn line(&self) -> Option<u32> {\n        self.line\n    }\n\n    /// Returns an opaque `Identifier` that uniquely identifies the callsite\n    /// this `Metadata` originated from.\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Returns true if the callsite kind is `Event`.\n    pub fn is_event(&self) -> bool {\n        self.kind.is_event()\n    }\n\n    /// Return true if the callsite kind is `Span`.\n    pub fn is_span(&self) -> bool {\n        self.kind.is_span()\n    }\n}\n\nimpl<'a> fmt::Debug for Metadata<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let mut meta = f.debug_struct(\"Metadata\");\n        meta.field(\"name\", &self.name)\n            .field(\"target\", &self.target)\n            .field(\"level\", &self.level);\n\n        if let Some(path) = self.module_path() {\n            meta.field(\"module_path\", &path);\n        }\n\n        match (self.file(), self.line()) {\n            (Some(file), Some(line)) => {\n                meta.field(\"location\", &format_args!(\"{}:{}\", file, line));\n            }\n            (Some(file), None) => {\n                meta.field(\"file\", &format_args!(\"{}\", file));\n            }\n\n            // Note: a line num with no file is a kind of weird case that _probably_ never occurs...\n            (None, Some(line)) => {\n                meta.field(\"line\", &line);\n            }\n            (None, None) => {}\n        };\n\n        meta.field(\"fields\", &format_args!(\"{}\", self.fields))\n            .field(\"callsite\", &self.callsite())\n            .field(\"kind\", &self.kind)\n            .finish()\n    }\n}\n\nimpl Kind {\n    const EVENT_BIT: u8 = 1 << 0;\n    const SPAN_BIT: u8 = 1 << 1;\n    const HINT_BIT: u8 = 1 << 2;\n\n    /// `Event` callsite\n    pub const EVENT: Kind = Kind(Self::EVENT_BIT);\n\n    /// `Span` callsite\n    pub const SPAN: Kind = Kind(Self::SPAN_BIT);\n\n    /// `enabled!` callsite. [`Collect`][`crate::collect::Collect`]s can assume\n    /// this `Kind` means they will never receive a\n    /// full event with this [`Metadata`].\n    pub const HINT: Kind = Kind(Self::HINT_BIT);\n\n    /// Return true if the callsite kind is `Span`\n    pub fn is_span(&self) -> bool {\n        self.0 & Self::SPAN_BIT == Self::SPAN_BIT\n    }\n\n    /// Return true if the callsite kind is `Event`\n    pub fn is_event(&self) -> bool {\n        self.0 & Self::EVENT_BIT == Self::EVENT_BIT\n    }\n\n    /// Return true if the callsite kind is `Hint`\n    pub fn is_hint(&self) -> bool {\n        self.0 & Self::HINT_BIT == Self::HINT_BIT\n    }\n\n    /// Sets that this `Kind` is a [hint](Self::HINT).\n    ///\n    /// This can be called on [`SPAN`](Self::SPAN) and [`EVENT`](Self::EVENT)\n    /// kinds to construct a hint callsite that also counts as a span or event.\n    pub const fn hint(self) -> Self {\n        Self(self.0 | Self::HINT_BIT)\n    }\n}\n\nimpl fmt::Debug for Kind {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.write_str(\"Kind(\")?;\n        let mut has_bits = false;\n        let mut write_bit = |name: &str| {\n            if has_bits {\n                f.write_str(\" | \")?;\n            }\n            f.write_str(name)?;\n            has_bits = true;\n            Ok(())\n        };\n\n        if self.is_event() {\n            write_bit(\"EVENT\")?;\n        }\n\n        if self.is_span() {\n            write_bit(\"SPAN\")?;\n        }\n\n        if self.is_hint() {\n            write_bit(\"HINT\")?;\n        }\n\n        // if none of the expected bits were set, something is messed up, so\n        // just print the bits for debugging purposes\n        if !has_bits {\n            write!(f, \"{:#b}\", self.0)?;\n        }\n\n        f.write_str(\")\")\n    }\n}\n\nimpl<'a> Eq for Metadata<'a> {}\n\nimpl<'a> PartialEq for Metadata<'a> {\n    #[inline]\n    fn eq(&self, other: &Self) -> bool {\n        if core::ptr::eq(&self, &other) {\n            true\n        } else if cfg!(not(debug_assertions)) {\n            // In a well-behaving application, two `Metadata` can be assumed to\n            // be totally equal so long as they share the same callsite.\n            self.callsite() == other.callsite()\n        } else {\n            // However, when debug-assertions are enabled, do not assume that\n            // the application is well-behaving; check every field of `Metadata`\n            // for equality.\n\n            // `Metadata` is destructured here to ensure a compile-error if the\n            // fields of `Metadata` change.\n            let Metadata {\n                name: lhs_name,\n                target: lhs_target,\n                level: lhs_level,\n                module_path: lhs_module_path,\n                file: lhs_file,\n                line: lhs_line,\n                fields: lhs_fields,\n                kind: lhs_kind,\n            } = self;\n\n            let Metadata {\n                name: rhs_name,\n                target: rhs_target,\n                level: rhs_level,\n                module_path: rhs_module_path,\n                file: rhs_file,\n                line: rhs_line,\n                fields: rhs_fields,\n                kind: rhs_kind,\n            } = &other;\n\n            // The initial comparison of callsites is purely an optimization;\n            // it can be removed without affecting the overall semantics of the\n            // expression.\n            self.callsite() == other.callsite()\n                && lhs_name == rhs_name\n                && lhs_target == rhs_target\n                && lhs_level == rhs_level\n                && lhs_module_path == rhs_module_path\n                && lhs_file == rhs_file\n                && lhs_line == rhs_line\n                && lhs_fields == rhs_fields\n                && lhs_kind == rhs_kind\n        }\n    }\n}\n\n// ===== impl Level =====\n\nimpl Level {\n    /// The \"error\" level.\n    ///\n    /// Designates very serious errors.\n    pub const ERROR: Level = Level(LevelInner::Error);\n    /// The \"warn\" level.\n    ///\n    /// Designates hazardous situations.\n    pub const WARN: Level = Level(LevelInner::Warn);\n    /// The \"info\" level.\n    ///\n    /// Designates useful information.\n    pub const INFO: Level = Level(LevelInner::Info);\n    /// The \"debug\" level.\n    ///\n    /// Designates lower priority information.\n    pub const DEBUG: Level = Level(LevelInner::Debug);\n    /// The \"trace\" level.\n    ///\n    /// Designates very low priority, often extremely verbose, information.\n    pub const TRACE: Level = Level(LevelInner::Trace);\n\n    /// Returns the string representation of the `Level`.\n    ///\n    /// This returns the same string as the `fmt::Display` implementation.\n    pub fn as_str(&self) -> &'static str {\n        match *self {\n            Level::TRACE => \"TRACE\",\n            Level::DEBUG => \"DEBUG\",\n            Level::INFO => \"INFO\",\n            Level::WARN => \"WARN\",\n            Level::ERROR => \"ERROR\",\n        }\n    }\n}\n\nimpl fmt::Display for Level {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match *self {\n            Level::TRACE => f.pad(\"TRACE\"),\n            Level::DEBUG => f.pad(\"DEBUG\"),\n            Level::INFO => f.pad(\"INFO\"),\n            Level::WARN => f.pad(\"WARN\"),\n            Level::ERROR => f.pad(\"ERROR\"),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl std::error::Error for ParseLevelError {}\n\nimpl FromStr for Level {\n    type Err = ParseLevelError;\n    fn from_str(s: &str) -> Result<Self, ParseLevelError> {\n        s.parse::<usize>()\n            .map_err(|_| ParseLevelError { _p: () })\n            .and_then(|num| match num {\n                1 => Ok(Level::ERROR),\n                2 => Ok(Level::WARN),\n                3 => Ok(Level::INFO),\n                4 => Ok(Level::DEBUG),\n                5 => Ok(Level::TRACE),\n                _ => Err(ParseLevelError { _p: () }),\n            })\n            .or_else(|_| match s {\n                s if s.eq_ignore_ascii_case(\"error\") => Ok(Level::ERROR),\n                s if s.eq_ignore_ascii_case(\"warn\") => Ok(Level::WARN),\n                s if s.eq_ignore_ascii_case(\"info\") => Ok(Level::INFO),\n                s if s.eq_ignore_ascii_case(\"debug\") => Ok(Level::DEBUG),\n                s if s.eq_ignore_ascii_case(\"trace\") => Ok(Level::TRACE),\n                _ => Err(ParseLevelError { _p: () }),\n            })\n    }\n}\n\n#[repr(usize)]\n#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]\nenum LevelInner {\n    /// The \"trace\" level.\n    ///\n    /// Designates very low priority, often extremely verbose, information.\n    Trace = 0,\n    /// The \"debug\" level.\n    ///\n    /// Designates lower priority information.\n    Debug = 1,\n    /// The \"info\" level.\n    ///\n    /// Designates useful information.\n    Info = 2,\n    /// The \"warn\" level.\n    ///\n    /// Designates hazardous situations.\n    Warn = 3,\n    /// The \"error\" level.\n    ///\n    /// Designates very serious errors.\n    Error = 4,\n}\n\n// === impl LevelFilter ===\n\nimpl From<Level> for LevelFilter {\n    #[inline]\n    fn from(level: Level) -> Self {\n        Self::from_level(level)\n    }\n}\n\nimpl From<Option<Level>> for LevelFilter {\n    #[inline]\n    fn from(level: Option<Level>) -> Self {\n        Self(level)\n    }\n}\n\nimpl From<LevelFilter> for Option<Level> {\n    #[inline]\n    fn from(filter: LevelFilter) -> Self {\n        filter.into_level()\n    }\n}\n\nimpl LevelFilter {\n    /// The \"off\" level.\n    ///\n    /// Designates that trace instrumentation should be completely disabled.\n    pub const OFF: LevelFilter = LevelFilter(None);\n    /// The \"error\" level.\n    ///\n    /// Designates very serious errors.\n    pub const ERROR: LevelFilter = LevelFilter::from_level(Level::ERROR);\n    /// The \"warn\" level.\n    ///\n    /// Designates hazardous situations.\n    pub const WARN: LevelFilter = LevelFilter::from_level(Level::WARN);\n    /// The \"info\" level.\n    ///\n    /// Designates useful information.\n    pub const INFO: LevelFilter = LevelFilter::from_level(Level::INFO);\n    /// The \"debug\" level.\n    ///\n    /// Designates lower priority information.\n    pub const DEBUG: LevelFilter = LevelFilter::from_level(Level::DEBUG);\n    /// The \"trace\" level.\n    ///\n    /// Designates very low priority, often extremely verbose, information.\n    pub const TRACE: LevelFilter = LevelFilter(Some(Level::TRACE));\n\n    /// Returns a `LevelFilter` that enables spans and events with verbosity up\n    /// to and including `level`.\n    pub const fn from_level(level: Level) -> Self {\n        Self(Some(level))\n    }\n\n    /// Returns the most verbose [`Level`] that this filter accepts, or `None`\n    /// if it is [`OFF`].\n    ///\n    /// [`Level`]: super::Level\n    /// [`OFF`]: LevelFilter::OFF\n    pub const fn into_level(self) -> Option<Level> {\n        self.0\n    }\n\n    // These consts are necessary because `as` casts are not allowed as\n    // match patterns.\n    const ERROR_USIZE: usize = LevelInner::Error as usize;\n    const WARN_USIZE: usize = LevelInner::Warn as usize;\n    const INFO_USIZE: usize = LevelInner::Info as usize;\n    const DEBUG_USIZE: usize = LevelInner::Debug as usize;\n    const TRACE_USIZE: usize = LevelInner::Trace as usize;\n    // Using the value of the last variant + 1 ensures that we match the value\n    // for `Option::None` as selected by the niche optimization for\n    // `LevelFilter`. If this is the case, converting a `usize` value into a\n    // `LevelFilter` (in `LevelFilter::current`) will be an identity conversion,\n    // rather than generating a lookup table.\n    const OFF_USIZE: usize = LevelInner::Error as usize + 1;\n\n    /// Returns a `LevelFilter` that matches the most verbose [`Level`] that any\n    /// currently active [collector] will enable.\n    ///\n    /// User code should treat this as a *hint*. If a given span or event has a\n    /// level *higher* than the returned `LevelFilter`, it will not be enabled.\n    /// However, if the level is less than or equal to this value, the span or\n    /// event is *not* guaranteed to be enabled; the collector will still\n    /// filter each callsite individually.\n    ///\n    /// Therefore, comparing a given span or event's level to the returned\n    /// `LevelFilter` **can** be used for determining if something is\n    /// *disabled*, but **should not** be used for determining if something is\n    /// *enabled*.\n    ///\n    /// [`Level`]: super::Level\n    /// [collector]: super::Collect\n    #[inline(always)]\n    pub fn current() -> Self {\n        match MAX_LEVEL.load(Ordering::Relaxed) {\n            Self::ERROR_USIZE => Self::ERROR,\n            Self::WARN_USIZE => Self::WARN,\n            Self::INFO_USIZE => Self::INFO,\n            Self::DEBUG_USIZE => Self::DEBUG,\n            Self::TRACE_USIZE => Self::TRACE,\n            Self::OFF_USIZE => Self::OFF,\n            #[cfg(debug_assertions)]\n            unknown => unreachable!(\n                \"/!\\\\ `LevelFilter` representation seems to have changed! /!\\\\ \\n\\\n                This is a bug (and it's pretty bad). Please contact the `tracing` \\\n                maintainers. Thank you and I'm sorry.\\n \\\n                The offending repr was: {:?}\",\n                unknown,\n            ),\n            #[cfg(not(debug_assertions))]\n            _ => unsafe {\n                // Using `unreachable_unchecked` here (rather than\n                // `unreachable!()`) is necessary to ensure that rustc generates\n                // an identity conversion from integer -> discriminant, rather\n                // than generating a lookup table. We want to ensure this\n                // function is a single `mov` instruction (on x86) if at all\n                // possible, because it is called *every* time a span/event\n                // callsite is hit; and it is (potentially) the only code in the\n                // hottest path for skipping a majority of callsites when level\n                // filtering is in use.\n                //\n                // safety: This branch is only truly unreachable if we guarantee\n                // that no values other than the possible enum discriminants\n                // will *ever* be present. The `AtomicUsize` is initialized to\n                // the `OFF` value. It is only set by the `set_max` function,\n                // which takes a `LevelFilter` as a parameter. This restricts\n                // the inputs to `set_max` to the set of valid discriminants.\n                // Therefore, **as long as `MAX_VALUE` is only ever set by\n                // `set_max`**, this is safe.\n                core::hint::unreachable_unchecked()\n            },\n        }\n    }\n\n    pub(crate) fn set_max(LevelFilter(level): LevelFilter) {\n        let val = match level {\n            Some(Level(level)) => level as usize,\n            None => Self::OFF_USIZE,\n        };\n\n        // using an AcqRel swap ensures an ordered relationship of writes to the\n        // max level.\n        MAX_LEVEL.swap(val, Ordering::AcqRel);\n    }\n}\n\nimpl fmt::Display for LevelFilter {\n...\n", "needle_token_start": 773, "needle_token_end": 825, "code_context_ntokens": 16179, "output": ["```rust\n#[cfg(feature = \"std\")]\npub(crate) fn and(self, rhs: Interest) -> Self {\n    if self.0 == rhs.0 {\n        self\n    } else {\n        Interest::sometimes()\n    }\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "set_global_default", "language": "rust", "path": "tracing-core/src/dispatch.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to establish a dispatch mechanism as the universal default for the entire duration of a program. It is intended to serve as a fallback when no thread-specific dispatch has been set.\n2. **Input**: The function takes a dispatch configuration as its argument.\n3. **Output**: It returns a success status if the dispatch was set successfully, or an error if a global default has already been established.\n4. **Procedure**: The function first checks if a global dispatch has been previously initialized. If not, it proceeds to set the provided dispatch as the global default. This involves potentially converting the dispatch to a globally accessible format and storing it in a global variable. The initialization status is then updated to reflect that a global dispatch has been set. If an attempt is made to set the global default after it has already been established, the function returns an error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "//!\n//! // `my_collector` is now the default\n//! ```\n//!\n//! <div class=\"example-wrap\" style=\"display:inline-block\">\n//! <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n//!\n//! **Note**: the thread-local scoped dispatcher ([`with_default`]) requires the\n//! Rust standard library. `no_std` users should use [`set_global_default`] instead.\n//!\n//! </pre></div>\n//!\n//! ## Accessing the Default Collector\n//!\n//! A thread's current default collector can be accessed using the\n//! [`get_default`] function, which executes a closure with a reference to the\n//! currently default `Dispatch`. This is used primarily by `tracing`\n//! instrumentation.\nuse crate::{\n    collect::{self, Collect, NoCollector},\n    span, Event, LevelFilter, Metadata,\n};\n\nuse core::{\n    any::Any,\n    fmt,\n    sync::atomic::{AtomicBool, AtomicUsize, Ordering},\n};\n\n#[cfg(feature = \"std\")]\nuse std::{\n    cell::{Cell, RefCell, RefMut},\n    error,\n};\n\n#[cfg(feature = \"alloc\")]\nuse alloc::sync::{Arc, Weak};\n\n#[cfg(feature = \"alloc\")]\nuse core::ops::Deref;\n\n/// `Dispatch` trace data to a [`Collect`].\n#[derive(Clone)]\npub struct Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Arc<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n/// `WeakDispatch` is a version of [`Dispatch`] that holds a non-owning reference\n/// to a [collector].\n///\n/// The collector may be accessed by calling [`WeakDispatch::upgrade`],\n/// which returns an `Option<Dispatch>`. If all [`Dispatch`] clones that point\n/// at the collector have been dropped, [`WeakDispatch::upgrade`] will return\n/// `None`. Otherwise, it will return `Some(Dispatch)`.\n///\n/// A `WeakDispatch` may be created from a [`Dispatch`] by calling the\n/// [`Dispatch::downgrade`] method. The primary use for creating a\n/// [`WeakDispatch`] is to allow a collector to hold a cyclical reference to\n/// itself without creating a memory leak. See [here] for details.\n///\n/// This type is analogous to the [`std::sync::Weak`] type, but for a\n/// [`Dispatch`] rather than an [`Arc`].\n///\n/// [collector]: Collect\n/// [`Arc`]: std::sync::Arc\n/// [here]: Collect#avoiding-memory-leaks\n#[derive(Clone)]\npub struct WeakDispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Weak<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n#[cfg(feature = \"alloc\")]\n#[derive(Clone)]\nenum Kind<T> {\n    Global(&'static (dyn Collect + Send + Sync)),\n    Scoped(T),\n}\n\n#[cfg(feature = \"std\")]\nthread_local! {\n    static CURRENT_STATE: State = const {\n        State {\n            default: RefCell::new(None),\n            can_enter: Cell::new(true),\n        }\n    };\n}\n\nstatic EXISTS: AtomicBool = AtomicBool::new(false);\nstatic GLOBAL_INIT: AtomicUsize = AtomicUsize::new(UNINITIALIZED);\n\n#[cfg(feature = \"std\")]\nstatic SCOPED_COUNT: AtomicUsize = AtomicUsize::new(0);\n\nconst UNINITIALIZED: usize = 0;\nconst INITIALIZING: usize = 1;\nconst INITIALIZED: usize = 2;\n\nstatic mut GLOBAL_DISPATCH: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NONE: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NO_COLLECTOR: NoCollector = NoCollector::new();\n\n/// The dispatch state of a thread.\n#[cfg(feature = \"std\")]\nstruct State {\n    /// This thread's current default dispatcher.\n    default: RefCell<Option<Dispatch>>,\n    /// Whether or not we can currently begin dispatching a trace event.\n    ///\n    /// This is set to `false` when functions such as `enter`, `exit`, `event`,\n    /// and `new_span` are called on this thread's default dispatcher, to\n    /// prevent further trace events triggered inside those functions from\n    /// creating an infinite recursion. When we finish handling a dispatch, this\n    /// is set back to `true`.\n    can_enter: Cell<bool>,\n}\n\n/// While this guard is active, additional calls to collector functions on\n/// the default dispatcher will not be able to access the dispatch context.\n/// Dropping the guard will allow the dispatch context to be re-entered.\n#[cfg(feature = \"std\")]\nstruct Entered<'a>(&'a State);\n\n/// A guard that resets the current default dispatcher to the prior\n/// default dispatcher when dropped.\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[derive(Debug)]\npub struct DefaultGuard(Option<Dispatch>);\n\n/// Sets this dispatch as the default for the duration of a closure.\n///\n/// The default dispatcher is used when creating a new [span] or\n/// [`Event`].\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: This function required the Rust standard library.\n/// <!-- hack: this whitespace makes rustdoc interpret the next line as markdown again -->\n///\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\npub fn with_default<T>(dispatcher: &Dispatch, f: impl FnOnce() -> T) -> T {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this (rather than simply resetting after calling\n    // `f`) ensures that we always reset to the prior dispatcher even if `f`\n    // panics.\n    let _guard = set_default(dispatcher);\n    f()\n}\n\n/// Sets the dispatch as the default dispatch for the duration of the lifetime\n/// of the returned DefaultGuard\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n///\n/// **Note**: This function required the Rust standard library.\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[must_use = \"Dropping the guard unregisters the dispatcher.\"]\npub fn set_default(dispatcher: &Dispatch) -> DefaultGuard {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this ensures that we always reset to the prior\n    // dispatcher even if the thread calling this function panics.\n    State::set_default(dispatcher.clone())\n}\n\n/// Sets this dispatch as the global default for the duration of the entire program.\n/// Will be used as a fallback if no thread-local dispatch has been set in a thread\n/// (using `with_default`.)\n///\n/// Can only be set once; subsequent attempts to set the global default will fail.\n/// Returns `Err` if the global default has already been set.\n///\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Warning</strong>: In general, libraries should <em>not</em> call\n/// <code>set_global_default()</code>! Doing so will cause conflicts when\n/// executables that depend on the library try to set the default collector later.\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\n\npub fn set_global_default(dispatcher: Dispatch) -> Result<(), SetGlobalDefaultError> {\n    // if `compare_exchange` returns Result::Ok(_), then `new` has been set and\n    // `current`\u2014now the prior value\u2014has been returned in the `Ok()` branch.\n    if GLOBAL_INIT\n        .compare_exchange(\n            UNINITIALIZED,\n            INITIALIZING,\n            Ordering::SeqCst,\n            Ordering::SeqCst,\n        )\n        .is_ok()\n    {\n        #[cfg(feature = \"alloc\")]\n        let collector = {\n            let collector = match dispatcher.collector {\n                Kind::Global(s) => s,\n                Kind::Scoped(s) => unsafe {\n                    // safety: this leaks the collector onto the heap. the\n                    // reference count will always be at least 1.\n                    &*Arc::into_raw(s)\n                },\n            };\n            Kind::Global(collector)\n        };\n\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = dispatcher.collector;\n\n        unsafe {\n            GLOBAL_DISPATCH = Dispatch { collector };\n        }\n        GLOBAL_INIT.store(INITIALIZED, Ordering::SeqCst);\n        EXISTS.store(true, Ordering::Release);\n        Ok(())\n    } else {\n        Err(SetGlobalDefaultError { _no_construct: () })\n    }\n}\n\n/// Returns true if a `tracing` dispatcher has ever been set.\n///\n/// This may be used to completely elide trace points if tracing is not in use\n/// at all or has yet to be initialized.\n#[doc(hidden)]\n#[inline(always)]\npub fn has_been_set() -> bool {\n    EXISTS.load(Ordering::Relaxed)\n}\n\n/// Returned if setting the global dispatcher fails.\npub struct SetGlobalDefaultError {\n    _no_construct: (),\n}\n\nimpl fmt::Debug for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"SetGlobalDefaultError\")\n            .field(&Self::MESSAGE)\n            .finish()\n    }\n}\n\nimpl fmt::Display for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(Self::MESSAGE)\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl error::Error for SetGlobalDefaultError {}\n\nimpl SetGlobalDefaultError {\n    const MESSAGE: &'static str = \"a global default trace dispatcher has already been set\";\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatch::Dispatch\n#[cfg(feature = \"std\")]\n#[inline(always)]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    if SCOPED_COUNT.load(Ordering::Acquire) == 0 {\n        // fast path if no scoped dispatcher has been set; just use the global\n        // default.\n        return f(get_global());\n    }\n\n    get_default_slow(f)\n}\n\n#[cfg(feature = \"std\")]\n#[inline(never)]\nfn get_default_slow<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    // While this guard is active, additional calls to collector functions on\n    // the default dispatcher will not be able to access the dispatch context.\n    // Dropping the guard will allow the dispatch context to be re-entered.\n    struct Entered<'a>(&'a Cell<bool>);\n    impl<'a> Drop for Entered<'a> {\n        #[inline]\n        fn drop(&mut self) {\n            self.0.set(true);\n        }\n    }\n\n    CURRENT_STATE\n        .try_with(|state| {\n            if state.can_enter.replace(false) {\n                let _guard = Entered(&state.can_enter);\n\n                let mut default = state.default.borrow_mut();\n                let default = default\n                    // if the local default for this thread has never been set,\n                    // populate it with the global default, so we don't have to\n                    // keep getting the global on every `get_default_slow` call.\n                    .get_or_insert_with(|| get_global().clone());\n\n                return f(&*default);\n            }\n\n            f(&Dispatch::none())\n        })\n        .unwrap_or_else(|_| f(&Dispatch::none()))\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(feature = \"std\")]\n#[doc(hidden)]\n#[inline(never)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    CURRENT_STATE\n        .try_with(|state| {\n            let entered = state.enter()?;\n            Some(f(&entered.current()))\n        })\n        .ok()?\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\n#[doc(hidden)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    Some(f(&get_global()))\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    f(get_global())\n}\n\n#[inline(always)]\npub(crate) fn get_global() -> &'static Dispatch {\n    if GLOBAL_INIT.load(Ordering::Acquire) != INITIALIZED {\n        return &NONE;\n    }\n    unsafe {\n        // This is safe given the invariant that setting the global dispatcher\n        // also sets `GLOBAL_INIT` to `INITIALIZED`.\n        &GLOBAL_DISPATCH\n    }\n}\n\n#[cfg(feature = \"std\")]\npub(crate) struct Registrar(Kind<Weak<dyn Collect + Send + Sync>>);\n\nimpl Dispatch {\n    /// Returns a new `Dispatch` that discards events and spans.\n    #[inline]\n    pub fn none() -> Self {\n        Dispatch {\n            #[cfg(feature = \"alloc\")]\n            collector: Kind::Global(&NO_COLLECTOR),\n            #[cfg(not(feature = \"alloc\"))]\n            collector: &NO_COLLECTOR,\n        }\n    }\n\n    /// Returns a `Dispatch` that forwards to the given [`Collect`].\n    ///\n    /// [`Collect`]: super::collect::Collect\n    #[cfg(feature = \"alloc\")]\n    #[cfg_attr(docsrs, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    pub fn new<C>(collector: C) -> Self\n    where\n        C: Collect + Send + Sync + 'static,\n    {\n        let me = Dispatch {\n            collector: Kind::Scoped(Arc::new(collector)),\n        };\n        crate::callsite::register_dispatch(&me);\n        me\n    }\n\n    /// Returns a `Dispatch` that forwards to the given static [collector].\n    ///\n    /// Unlike [`Dispatch::new`], this function is always available on all\n    /// platforms, even when the `std` or `alloc` features are disabled.\n    ///\n    /// In order to use `from_static`, the `Collector` itself must be stored in\n    /// a static. For example:\n    ///\n    /// ```rust\n    /// struct MyCollector {\n    ///    // ...\n    /// }\n    ///\n    /// # use tracing_core::{span::{Id, Attributes, Current, Record}, Event, Metadata};\n    /// impl tracing_core::Collect for MyCollector {\n    ///     // ...\n    /// #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n    /// #   fn record(&self, _: &Id, _: &Record) {}\n    /// #   fn event(&self, _: &Event) {}\n    /// #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n    /// #   fn enabled(&self, _: &Metadata) -> bool { false }\n    /// #   fn enter(&self, _: &Id) {}\n    /// #   fn exit(&self, _: &Id) {}\n    /// #   fn current_span(&self) -> Current { Current::unknown() }\n    /// }\n    ///\n    /// static COLLECTOR: MyCollector = MyCollector {\n    ///     // ...\n    /// };\n    ///\n    /// fn main() {\n    ///     use tracing_core::dispatch::{self, Dispatch};\n    ///\n    ///     let dispatch = Dispatch::from_static(&COLLECTOR);\n    ///\n    ///     dispatch::set_global_default(dispatch)\n    ///         .expect(\"no global default collector should have been set previously!\");\n    /// }\n    /// ```\n    ///\n    /// Constructing the collector in a static initializer may make some forms\n    /// of runtime configuration more challenging. If this is the case, users\n    /// with access to `liballoc` or the Rust standard library are encouraged to\n    /// use [`Dispatch::new`] rather than `from_static`. `no_std` users who\n    /// cannot allocate or do not have access to `liballoc` may want to consider\n    /// the [`once_cell`] crate, or another library which allows lazy\n    /// initialization of statics.\n    ///\n    /// [collector]: super::collect::Collect\n    /// [`once_cell`]: https://crates.io/crates/once_cell\n    pub fn from_static(collector: &'static (dyn Collect + Send + Sync)) -> Self {\n        #[cfg(feature = \"alloc\")]\n        let me = Self {\n            collector: Kind::Global(collector),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let me = Self { collector };\n        crate::callsite::register_dispatch(&me);\n        me\n    }\n\n    /// Creates a [`WeakDispatch`] from this `Dispatch`.\n    ///\n    /// A [`WeakDispatch`] is similar to a [`Dispatch`], but it does not prevent\n    /// the underlying [collector] from being dropped. Instead, it only permits\n    /// access while other references to the collector exist. This is equivalent\n    /// to the standard library's [`Arc::downgrade`] method, but for `Dispatch`\n    /// rather than `Arc`.\n    ///\n    /// The primary use for creating a [`WeakDispatch`] is to allow a collector\n    /// to hold a cyclical reference to itself without creating a memory leak.\n    /// See [here] for details.\n    ///\n    /// [collector]: Collect\n    /// [`Arc::downgrade`]: std::sync::Arc::downgrade\n    /// [here]: Collect#avoiding-memory-leaks\n    pub fn downgrade(&self) -> WeakDispatch {\n        #[cfg(feature = \"alloc\")]\n        let collector = match &self.collector {\n            Kind::Global(dispatch) => Kind::Global(*dispatch),\n            Kind::Scoped(dispatch) => Kind::Scoped(Arc::downgrade(dispatch)),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = self.collector;\n\n        WeakDispatch { collector }\n    }\n\n    #[cfg(feature = \"std\")]\n    pub(crate) fn registrar(&self) -> Registrar {\n        Registrar(match self.collector {\n            Kind::Scoped(ref s) => Kind::Scoped(Arc::downgrade(s)),\n            Kind::Global(s) => Kind::Global(s),\n        })\n    }\n\n    #[inline(always)]\n    #[cfg(feature = \"alloc\")]\n    pub(crate) fn collector(&self) -> &(dyn Collect + Send + Sync) {\n        match self.collector {\n            Kind::Scoped(ref s) => Arc::deref(s),\n            Kind::Global(s) => s,\n        }\n    }\n\n    #[inline(always)]\n    #[cfg(not(feature = \"alloc\"))]\n    pub(crate) fn collector(&self) -> &(dyn Collect + Send + Sync) {\n        self.collector\n    }\n\n    /// Registers a new callsite with this collector, returning whether or not\n    /// the collector is interested in being notified about the callsite.\n    ///\n    /// This calls the [`register_callsite`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`register_callsite`]: super::collect::Collect::register_callsite\n    #[inline]\n    pub fn register_callsite(&self, metadata: &'static Metadata<'static>) -> collect::Interest {\n        self.collector().register_callsite(metadata)\n    }\n\n    /// Returns the highest [verbosity level][level] that this [collector] will\n    /// enable, or `None`, if the collector does not implement level-based\n    /// filtering or chooses not to implement this method.\n    ///\n    /// This calls the [`max_level_hint`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [level]: super::Level\n    /// [collector]: super::collect::Collect\n    /// [`Collect`]: super::collect::Collect\n    /// [`register_callsite`]: super::collect::Collect::max_level_hint\n    // TODO(eliza): consider making this a public API?\n    #[inline]\n    pub(crate) fn max_level_hint(&self) -> Option<LevelFilter> {\n        self.collector().max_level_hint()\n    }\n\n    /// Record the construction of a new span, returning a new [ID] for the\n    /// span being constructed.\n    ///\n    /// This calls the [`new_span`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`new_span`]: super::collect::Collect::new_span\n    #[inline]\n    pub fn new_span(&self, span: &span::Attributes<'_>) -> span::Id {\n        self.collector().new_span(span)\n    }\n\n    /// Record a set of values on a span.\n    ///\n    /// This calls the [`record`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`record`]: super::collect::Collect::record\n    #[inline]\n    pub fn record(&self, span: &span::Id, values: &span::Record<'_>) {\n        self.collector().record(span, values)\n    }\n\n    /// Adds an indication that `span` follows from the span with the id\n    /// `follows`.\n    ///\n    /// This calls the [`record_follows_from`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`record_follows_from`]: super::collect::Collect::record_follows_from\n    #[inline]\n    pub fn record_follows_from(&self, span: &span::Id, follows: &span::Id) {\n        self.collector().record_follows_from(span, follows)\n    }\n\n    /// Returns true if a span with the specified [metadata] would be\n    /// recorded.\n    ///\n    /// This calls the [`enabled`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [metadata]: super::metadata::Metadata\n    /// [`Collect`]: super::collect::Collect\n    /// [`enabled`]: super::collect::Collect::enabled\n    #[inline]\n    pub fn enabled(&self, metadata: &Metadata<'_>) -> bool {\n        self.collector().enabled(metadata)\n    }\n\n    /// Records that an [`Event`] has occurred.\n    ///\n    /// This calls the [`event`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Event`]: super::event::Event\n    /// [`Collect`]: super::collect::Collect\n    /// [`event`]: super::collect::Collect::event\n    #[inline]\n    pub fn event(&self, event: &Event<'_>) {\n        let collector = self.collector();\n        if collector.event_enabled(event) {\n            collector.event(event);\n        }\n    }\n\n    /// Records that a span has been can_enter.\n    ///\n    /// This calls the [`enter`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`enter`]: super::collect::Collect::enter\n    pub fn enter(&self, span: &span::Id) {\n        self.collector().enter(span);\n    }\n\n    /// Records that a span has been exited.\n    ///\n    /// This calls the [`exit`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`exit`]: super::collect::Collect::exit\n    pub fn exit(&self, span: &span::Id) {\n        self.collector().exit(span);\n    }\n\n    /// Notifies the [collector] that a [span ID] has been cloned.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`clone_span`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [span ID]: super::span::Id\n    /// [collector]: super::collect::Collect\n    /// [`clone_span`]: super::collect::Collect::clone_span\n    /// [`new_span`]: super::collect::Collect::new_span\n    #[inline]\n    pub fn clone_span(&self, id: &span::Id) -> span::Id {\n        self.collector().clone_span(id)\n    }\n\n    /// Notifies the collector that a [span ID] has been dropped.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`drop_span`] function on the [`Collect`] that this\n    ///  `Dispatch` forwards to.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n    ///\n    /// **Deprecated**: The [`try_close`] method is functionally identical, but returns `true` if the span is now closed.\n    /// It should be used instead of this method.\n    ///\n    /// </pre></div>\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`drop_span`]: super::collect::Collect::drop_span\n    /// [`new_span`]: super::collect::Collect::new_span\n    /// [`try_close`]: Self::try_close\n    #[inline]\n    #[deprecated(since = \"0.1.2\", note = \"use `Dispatch::try_close` instead\")]\n    pub fn drop_span(&self, id: span::Id) {\n        #[allow(deprecated)]\n        self.collector().drop_span(id);\n    }\n\n    /// Notifies the collector that a [span ID] has been dropped, and returns\n    /// `true` if there are now 0 IDs referring to that span.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`try_close`] function on the [`Collect`] trait\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`try_close`]: super::collect::Collect::try_close\n    /// [`new_span`]: super::collect::Collect::new_span\n    pub fn try_close(&self, id: span::Id) -> bool {\n        self.collector().try_close(id)\n    }\n\n    /// Returns a type representing this collector's view of the current span.\n    ///\n    /// This calls the [`current`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`current`]: super::collect::Collect::current_span\n    #[inline]\n    pub fn current_span(&self) -> span::Current {\n        self.collector().current_span()\n    }\n\n    /// Returns `true` if this `Dispatch` forwards to a collector of type\n    /// `T`.\n    #[inline]\n    pub fn is<T: Any>(&self) -> bool {\n        <dyn Collect>::is::<T>(self.collector())\n    }\n\n    /// Returns some reference to the [`Collect`] this `Dispatch` forwards to\n    /// if it is of type `T`, or `None` if it isn't.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    #[inline]\n    pub fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        <dyn Collect>::downcast_ref(self.collector())\n    }\n}\n\nimpl Default for Dispatch {\n    /// Returns the current default dispatcher\n    fn default() -> Self {\n        get_default(|default| default.clone())\n    }\n}\n\nimpl fmt::Debug for Dispatch {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match &self.collector {\n            #[cfg(feature = \"alloc\")]\n            Kind::Global(collector) => f\n                .debug_tuple(\"Dispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(feature = \"alloc\")]\n            Kind::Scoped(collector) => f\n                .debug_tuple(\"Dispatch::Scoped\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(not(feature = \"alloc\"))]\n            collector => f\n                .debug_tuple(\"Dispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<C> From<C> for Dispatch\nwhere\n    C: Collect + Send + Sync + 'static,\n{\n    #[inline]\n    fn from(collector: C) -> Self {\n        Dispatch::new(collector)\n    }\n}\n\nimpl WeakDispatch {\n    /// Attempts to upgrade this `WeakDispatch` to a [`Dispatch`].\n    ///\n    /// Returns `None` if the referenced `Dispatch` has already been dropped.\n    ///\n    /// ## Examples\n    ///\n    /// ```\n    /// # use tracing_core::collect::NoCollector;\n    /// # use tracing_core::dispatch::Dispatch;\n    /// static COLLECTOR: NoCollector = NoCollector::new();\n    /// let strong = Dispatch::new(COLLECTOR);\n    /// let weak = strong.downgrade();\n    ///\n    /// // The strong here keeps it alive, so we can still access the object.\n    /// assert!(weak.upgrade().is_some());\n    ///\n    /// drop(strong); // But not any more.\n    /// assert!(weak.upgrade().is_none());\n    /// ```\n    pub fn upgrade(&self) -> Option<Dispatch> {\n        #[cfg(feature = \"alloc\")]\n        let collector = match &self.collector {\n            Kind::Global(dispatch) => Some(Kind::Global(*dispatch)),\n            Kind::Scoped(dispatch) => dispatch.upgrade().map(Kind::Scoped),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = Some(self.collector);\n\n        collector.map(|collector| Dispatch { collector })\n    }\n}\n\nimpl fmt::Debug for WeakDispatch {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match &self.collector {\n            #[cfg(feature = \"alloc\")]\n            Kind::Global(collector) => f\n                .debug_tuple(\"WeakDispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(feature = \"alloc\")]\n            Kind::Scoped(collector) => f\n                .debug_tuple(\"WeakDispatch::Scoped\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(not(feature = \"alloc\"))]\n            collector => f\n                .debug_tuple(\"WeakDispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl Registrar {\n    pub(crate) fn upgrade(&self) -> Option<Dispatch> {\n        match self.0 {\n            Kind::Global(s) => Some(Dispatch {\n                collector: Kind::Global(s),\n            }),\n            Kind::Scoped(ref s) => s.upgrade().map(|s| Dispatch {\n                collector: Kind::Scoped(s),\n            }),\n        }\n    }\n}\n\n// ===== impl State =====\n\n#[cfg(feature = \"std\")]\nimpl State {\n    /// Replaces the current default dispatcher on this thread with the provided\n    /// dispatcher.\n    ///\n    /// Dropping the returned `ResetGuard` will reset the default dispatcher to\n    /// the previous value.\n    #[inline]\n    fn set_default(new_dispatch: Dispatch) -> DefaultGuard {\n        let prior = CURRENT_STATE\n            .try_with(|state| {\n                state.can_enter.set(true);\n                state\n                    .default\n                    .replace(Some(new_dispatch))\n                    // if the scoped default was not set on this thread, set the\n                    // `prior` default to the global default to populate the\n                    // scoped default when unsetting *this* default\n                    .unwrap_or_else(|| get_global().clone())\n            })\n            .ok();\n        EXISTS.store(true, Ordering::Release);\n        SCOPED_COUNT.fetch_add(1, Ordering::Release);\n        DefaultGuard(prior)\n    }\n\n    #[inline]\n    fn enter(&self) -> Option<Entered<'_>> {\n        if self.can_enter.replace(false) {\n            Some(Entered(self))\n        } else {\n            None\n        }\n    }\n}\n\n// ===== impl Entered =====\n\n#[cfg(feature = \"std\")]\nimpl<'a> Entered<'a> {\n    #[inline]\n    fn current(&self) -> RefMut<'a, Dispatch> {\n        let default = self.0.default.borrow_mut();\n        RefMut::map(default, |default| {\n            default.get_or_insert_with(|| get_global().clone())\n        })\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<'a> Drop for Entered<'a> {\n    #[inline]\n    fn drop(&mut self) {\n        self.0.can_enter.set(true);\n    }\n}\n\n// ===== impl DefaultGuard =====\n\n#[cfg(feature = \"std\")]\nimpl Drop for DefaultGuard {\n    #[inline]\n    fn drop(&mut self) {\n        SCOPED_COUNT.fetch_sub(1, Ordering::Release);\n        if let Some(dispatch) = self.0.take() {\n            // Replace the dispatcher and then drop the old one outside\n            // of the thread-local context. Dropping the dispatch may\n            // lead to the drop of a collector which, in the process,\n            // could then also attempt to access the same thread local\n            // state -- causing a clash.\n            let prev = CURRENT_STATE.try_with(|state| state.default.replace(Some(dispatch)));\n            drop(prev)\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n\n    use super::*;\n    use crate::{\n        callsite::Callsite,\n        collect::Interest,\n        metadata::{Kind, Level, Metadata},\n    };\n\n    #[test]\n    fn dispatch_is() {\n        let dispatcher = Dispatch::from_static(&NO_COLLECTOR);\n        assert!(dispatcher.is::<NoCollector>());\n    }\n\n    #[test]\n    fn dispatch_downcasts() {\n        let dispatcher = Dispatch::from_static(&NO_COLLECTOR);\n        assert!(dispatcher.downcast_ref::<NoCollector>().is_some());\n    }\n\n    struct TestCallsite;\n    static TEST_CALLSITE: TestCallsite = TestCallsite;\n    static TEST_META: Metadata<'static> = metadata! {\n        name: \"test\",\n        target: module_path!(),\n        level: Level::DEBUG,\n        fields: &[],\n        callsite: &TEST_CALLSITE,\n        kind: Kind::EVENT\n    };\n\n    impl Callsite for TestCallsite {\n        fn set_interest(&self, _: Interest) {}\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META\n        }\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn events_dont_infinite_loop() {\n        // This test ensures that an event triggered within a collector\n        // won't cause an infinite loop of events.\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {\n                static EVENTS: AtomicUsize = AtomicUsize::new(0);\n                assert_eq!(\n                    EVENTS.fetch_add(1, Ordering::Relaxed),\n                    0,\n                    \"event method called twice!\"\n                );\n                Event::dispatch(&TEST_META, &TEST_META.fields().value_set(&[]))\n            }\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n\n        with_default(&Dispatch::new(TestCollector), || {\n            Event::dispatch(&TEST_META, &TEST_META.fields().value_set(&[]))\n        })\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn spans_dont_infinite_loop() {\n        // This test ensures that a span created within a collector\n        // won't cause an infinite loop of new spans.\n\n        fn mk_span() {\n            get_default(|current| {\n                current.new_span(&span::Attributes::new(\n                    &TEST_META,\n                    &TEST_META.fields().value_set(&[]),\n                ))\n            });\n        }\n\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                static NEW_SPANS: AtomicUsize = AtomicUsize::new(0);\n                assert_eq!(\n                    NEW_SPANS.fetch_add(1, Ordering::Relaxed),\n                    0,\n                    \"new_span method called twice!\"\n                );\n                mk_span();\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {}\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n\n        with_default(&Dispatch::new(TestCollector), mk_span)\n    }\n\n    #[test]\n    fn default_no_collector() {\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<NoCollector>());\n    }\n\n    #[cfg(feature = \"std\")]\n    #[test]\n    fn default_dispatch() {\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {}\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n        let guard = set_default(&Dispatch::new(TestCollector));\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<TestCollector>());\n\n        drop(guard);\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<NoCollector>());\n    }\n}\n\n// Path: tracing-core/src/collect.rs\n//! Collectors collect and record trace data.\nuse crate::{span, Dispatch, Event, LevelFilter, Metadata};\n\nuse core::any::{Any, TypeId};\nuse core::ptr::NonNull;\n\n/// Trait representing the functions required to collect trace data.\n///\n/// Crates that provide implementations of methods for collecting or recording\n/// trace data should implement the `Collect` interface. This trait is\n/// intended to represent fundamental primitives for collecting trace events and\n/// spans \u2014 other libraries may offer utility functions and types to make\n/// collector implementations more modular or improve the ergonomics of writing\n/// collectors.\n///\n/// A collector is responsible for the following:\n/// - Registering new spans as they are created, and providing them with span\n///   IDs. Implicitly, this means the collector may determine the strategy for\n///   determining span equality.\n/// - Recording the attachment of field values and follows-from annotations to\n///   spans.\n/// - Filtering spans and events, and determining when those filters must be\n///   invalidated.\n/// - Observing spans as they are entered, exited, and closed, and events as\n///   they occur.\n///\n/// When a span is entered or exited, the collector is provided only with the\n/// [ID] with which it tagged that span when it was created. This means\n/// that it is up to the collector to determine whether and how span _data_ \u2014\n/// the fields and metadata describing the span \u2014 should be stored. The\n/// [`new_span`] function is called when a new span is created, and at that\n/// point, the collector _may_ choose to store the associated data if it will\n/// be referenced again. However, if the data has already been recorded and will\n/// not be needed by the implementations of `enter` and `exit`, the collector\n/// may freely discard that data without allocating space to store it.\n///\n/// ## Overriding default impls\n///\n/// Some trait methods on `Collect` have default implementations, either in\n/// order to reduce the surface area of implementing `Collect`, or for\n/// backward-compatibility reasons. However, many collectors will likely want\n/// to override these default implementations.\n///\n/// The following methods are likely of interest:\n///\n/// - [`register_callsite`] is called once for each callsite from which a span\n///   event may originate, and returns an [`Interest`] value describing whether or\n///   not the collector wishes to see events or spans from that callsite. By\n///   default, it calls [`enabled`], and returns `Interest::always()` if\n///   `enabled` returns true, or `Interest::never()` if enabled returns false.\n///   However, if the collector's interest can change dynamically at runtime,\n///   it may want to override this function to return `Interest::sometimes()`.\n///   Additionally, collectors which wish to perform a behaviour once for each\n///   callsite, such as allocating storage for data related to that callsite,\n///   can perform it in `register_callsite`.\n///\n///   See also the [documentation on the callsite registry][cs-reg] for details\n///   on [`register_callsite`].\n///\n/// - [`event_enabled`] is called once before every call to the [`event`]\n///   method. This can be used to implement filtering on events once their field\n///   values are known, but before any processing is done in the `event` method.\n/// - [`clone_span`] is called every time a span ID is cloned, and [`try_close`]\n///   is called when a span ID is dropped. By default, these functions do\n///   nothing. However, they can be used to implement reference counting for\n///   spans, allowing collectors to free storage for span data and to determine\n///   when a span has _closed_ permanently (rather than being exited).\n///   Collectors which store per-span data or which need to track span closures\n///   should override these functions together.\n///\n/// [ID]: super::span::Id\n/// [`new_span`]: Collect::new_span\n/// [`register_callsite`]: Collect::register_callsite\n/// [`enabled`]: Collect::enabled\n/// [`clone_span`]: Collect::clone_span\n/// [`try_close`]: Collect::try_close\n/// [cs-reg]: crate::callsite#registering-callsites\n/// [`event`]: Collect::event\n/// [`event_enabled`]: Collect::event_enabled\npub trait Collect: 'static {\n    /// Invoked when this collector becomes a [`Dispatch`].\n    ///\n    /// ## Avoiding Memory Leaks\n    ///\n    /// Collectors should not store their own [`Dispatch`]. Because the\n    /// `Dispatch` owns the collector, storing the `Dispatch` within the\n    /// collector will create a reference count cycle, preventing the `Dispatch`\n    /// from ever being dropped.\n    ///\n    /// Instead, when it is necessary to store a cyclical reference to the\n    /// `Dispatch` within a collector, use [`Dispatch::downgrade`] to convert a\n    /// `Dispatch` into a [`WeakDispatch`]. This type is analogous to\n    /// [`std::sync::Weak`], and does not create a reference count cycle. A\n    /// [`WeakDispatch`] can be stored within a collector without causing a\n    /// memory leak, and can be [upgraded] into a `Dispatch` temporarily when\n    /// the `Dispatch` must be accessed by the collector.\n    ///\n    /// [`WeakDispatch`]: crate::dispatch::WeakDispatch\n    /// [upgraded]: crate::dispatch::WeakDispatch::upgrade\n    fn on_register_dispatch(&self, collector: &Dispatch) {\n        let _ = collector;\n    }\n\n    // === Span registry methods ==============================================\n\n    /// Registers a new [callsite] with this collector, returning whether or not\n    /// the collector is interested in being notified about the callsite.\n    ///\n    /// By default, this function assumes that the collector's [filter]\n    /// represents an unchanging view of its interest in the callsite. However,\n    /// if this is not the case, collectors may override this function to\n    /// indicate different interests, or to implement behaviour that should run\n    /// once for every callsite.\n    ///\n    /// This function is guaranteed to be called at least once per callsite on\n    /// every active collector. The collector may store the keys to fields it\n    /// cares about in order to reduce the cost of accessing fields by name,\n    /// preallocate storage for that callsite, or perform any other actions it\n    /// wishes to perform once for each callsite.\n    ///\n    /// The collector should then return an [`Interest`], indicating\n    /// whether it is interested in being notified about that callsite in the\n    /// future. This may be `Always` indicating that the collector always\n    /// wishes to be notified about the callsite, and its filter need not be\n    /// re-evaluated; `Sometimes`, indicating that the collector may sometimes\n    /// care about the callsite but not always (such as when sampling), or\n    /// `Never`, indicating that the collector never wishes to be notified about\n    /// that callsite. If all active collectors return `Never`, a callsite will\n    /// never be enabled unless a new collector expresses interest in it.\n    ///\n    /// `Collector`s which require their filters to be run every time an event\n    /// occurs or a span is entered/exited should return `Interest::sometimes`.\n    /// If a collector returns `Interest::sometimes`, then its' [`enabled`] method\n    /// will be called every time an event or span is created from that callsite.\n    ///\n    /// For example, suppose a sampling collector is implemented by\n    /// incrementing a counter every time `enabled` is called and only returning\n    /// `true` when the counter is divisible by a specified sampling rate. If\n    /// that collector returns `Interest::always` from `register_callsite`, then\n    /// the filter will not be re-evaluated once it has been applied to a given\n    /// set of metadata. Thus, the counter will not be incremented, and the span\n    /// or event that corresponds to the metadata will never be `enabled`.\n    ///\n    /// `Collector`s that need to change their filters occasionally should call\n    /// [`rebuild_interest_cache`] to re-evaluate `register_callsite` for all\n    /// callsites.\n    ///\n    /// Similarly, if a `Collector` has a filtering strategy that can be\n    /// changed dynamically at runtime, it would need to re-evaluate that filter\n    /// if the cached results have changed.\n    ///\n    /// A collector which manages fanout to multiple other collectors\n    /// should proxy this decision to all of its child collectors,\n    /// returning `Interest::never` only if _all_ such children return\n    /// `Interest::never`. If the set of collectors to which spans are\n    /// broadcast may change dynamically, the collector should also never\n    /// return `Interest::Never`, as a new collector may be added that _is_\n    /// interested.\n    ///\n    /// See the [documentation on the callsite registry][cs-reg] for more\n    /// details on how and when the `register_callsite` method is called.\n    ///\n    /// # Notes\n    ///\n    /// This function may be called again when a new collector is created or\n    /// when the registry is invalidated.\n    ///\n    /// If a collector returns `Interest::never` for a particular callsite, it\n    /// _may_ still see spans and events originating from that callsite, if\n    /// another collector expressed interest in it.\n    ///\n    /// [callsite]: crate::callsite\n    /// [filter]: Self::enabled\n    /// [metadata]: super::metadata::Metadata\n    /// [`enabled`]: Self::enabled\n    /// [`rebuild_interest_cache`]: super::callsite::rebuild_interest_cache\n    /// [cs-reg]: crate::callsite#registering-callsites\n    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {\n        if self.enabled(metadata) {\n            Interest::always()\n        } else {\n            Interest::never()\n        }\n    }\n\n    /// Returns true if a span or event with the specified [metadata] would be\n    /// recorded.\n    ///\n    /// By default, it is assumed that this filter needs only be evaluated once\n    /// for each callsite, so it is called by [`register_callsite`] when each\n    /// callsite is registered. The result is used to determine if the collector\n    /// is always [interested] or never interested in that callsite. This is intended\n    /// primarily as an optimization, so that expensive filters (such as those\n    /// involving string search, et cetera) need not be re-evaluated.\n    ///\n    /// However, if the collector's interest in a particular span or event may\n    /// change, or depends on contexts only determined dynamically at runtime,\n    /// then the `register_callsite` method should be overridden to return\n    /// [`Interest::sometimes`]. In that case, this function will be called every\n    /// time that span or event occurs.\n    ///\n    /// [metadata]: super::metadata::Metadata\n    /// [interested]: Interest\n    /// [`register_callsite`]: Self::register_callsite\n    fn enabled(&self, metadata: &Metadata<'_>) -> bool;\n\n    /// Returns the highest [verbosity level][level] that this `Collector` will\n    /// enable, or `None`, if the collector does not implement level-based\n    /// filtering or chooses not to implement this method.\n    ///\n    /// If this method returns a [`Level`][level], it will be used as a hint to\n    /// determine the most verbose level that will be enabled. This will allow\n    /// spans and events which are more verbose than that level to be skipped\n    /// more efficiently. collectors which perform filtering are strongly\n    /// encouraged to provide an implementation of this method.\n    ///\n    /// If the maximum level the collector will enable can change over the\n    /// course of its lifetime, it is free to return a different value from\n    /// multiple invocations of this method. However, note that changes in the\n    /// maximum level will **only** be reflected after the callsite [`Interest`]\n    /// cache is rebuilt, by calling the [`callsite::rebuild_interest_cache`][rebuild]\n    /// function. Therefore, if the collector will change the value returned by\n    /// this method, it is responsible for ensuring that\n    /// [`rebuild_interest_cache`][rebuild] is called after the value of the max\n    /// level changes.\n    ///\n    /// [level]: super::Level\n    /// [rebuild]: super::callsite::rebuild_interest_cache\n    fn max_level_hint(&self) -> Option<LevelFilter> {\n        None\n    }\n\n    /// Visit the construction of a new span, returning a new [span ID] for the\n    /// span being constructed.\n    ///\n    /// The provided [`Attributes`] contains any field values that were provided\n    /// when the span was created. The collector may pass a [visitor] to the\n    /// `Attributes`' [`record` method] to record these values.\n    ///\n    /// IDs are used to uniquely identify spans and events within the context of a\n    /// collector, so span equality will be based on the returned ID. Thus, if\n    /// the collector wishes for all spans with the same metadata to be\n    /// considered equal, it should return the same ID every time it is given a\n    /// particular set of metadata. Similarly, if it wishes for two separate\n    /// instances of a span with the same metadata to *not* be equal, it should\n    /// return a distinct ID every time this function is called, regardless of\n    /// the metadata.\n    ///\n    /// Note that the collector is free to assign span IDs based on whatever\n    /// scheme it sees fit. Any guarantees about uniqueness, ordering, or ID\n    /// reuse are left up to the collector implementation to determine.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Attributes`]: super::span::Attributes\n    /// [visitor]: super::field::Visit\n    /// [`record` method]: super::span::Attributes::record\n    fn new_span(&self, span: &span::Attributes<'_>) -> span::Id;\n\n    // === Notification methods ===============================================\n\n    /// Record a set of values on a span.\n    ///\n    /// This method will be invoked when value is recorded on a span.\n    /// Recording multiple values for the same field is possible,\n    /// but the actual behaviour is defined by the collector implementation.\n    ///\n    /// Keep in mind that a span might not provide a value\n    /// for each field it declares.\n    ///\n    /// The collector is expected to provide a [visitor] to the `Record`'s\n    /// [`record` method] in order to record the added values.\n    ///\n    /// # Example\n    ///  \"foo = 3\" will be recorded when [`record`] is called on the\n    /// `Attributes` passed to `new_span`.\n    /// Since values are not provided for the `bar` and `baz` fields,\n    /// the span's `Metadata` will indicate that it _has_ those fields,\n    /// but values for them won't be recorded at this time.\n    ///\n    /// ```rust,ignore\n    /// # use tracing::span;\n    ///\n    /// let mut span = span!(\"my_span\", foo = 3, bar, baz);\n    ///\n    /// // `Collector::record` will be called with a `Record`\n    /// // containing \"bar = false\"\n    /// span.record(\"bar\", &false);\n    ///\n    /// // `Collector::record` will be called with a `Record`\n    /// // containing \"baz = \"a string\"\"\n    /// span.record(\"baz\", &\"a string\");\n    /// ```\n    ///\n    /// [visitor]: super::field::Visit\n    /// [`record`]: super::span::Attributes::record\n    /// [`record` method]: super::span::Record::record\n    fn record(&self, span: &span::Id, values: &span::Record<'_>);\n\n    /// Adds an indication that `span` follows from the span with the id\n    /// `follows`.\n    ///\n    /// This relationship differs somewhat from the parent-child relationship: a\n    /// span may have any number of prior spans, rather than a single one; and\n    /// spans are not considered to be executing _inside_ of the spans they\n    /// follow from. This means that a span may close even if subsequent spans\n    /// that follow from it are still open, and time spent inside of a\n    /// subsequent span should not be included in the time its precedents were\n    /// executing. This is used to model causal relationships such as when a\n    /// single future spawns several related background tasks, et cetera.\n    ///\n    /// If the collector has spans corresponding to the given IDs, it should\n    /// record this relationship in whatever way it deems necessary. Otherwise,\n    /// if one or both of the given span IDs do not correspond to spans that the\n    /// collector knows about, or if a cyclical relationship would be created\n    /// (i.e., some span _a_ which proceeds some other span _b_ may not also\n    /// follow from _b_), it may silently do nothing.\n    fn record_follows_from(&self, span: &span::Id, follows: &span::Id);\n\n    /// Determine if an [`Event`] should be recorded.\n    ///\n    /// By default, this returns `true` and collectors can filter events in\n    /// [`event`][Self::event] without any penalty. However, when `event` is\n    /// more complicated, this can be used to determine if `event` should be\n    /// called at all, separating out the decision from the processing.\n    fn event_enabled(&self, event: &Event<'_>) -> bool {\n        let _ = event;\n        true\n    }\n\n    /// Records that an [`Event`] has occurred.\n    ///\n    /// This method will be invoked when an Event is constructed by\n    /// the `Event`'s [`dispatch` method]. For example, this happens internally\n    /// when an event macro from `tracing` is called.\n    ///\n    /// The key difference between this method and `record` is that `record` is\n    /// called when a value is recorded for a field defined by a span,\n    /// while `event` is called when a new event occurs.\n    ///\n    /// The provided `Event` struct contains any field values attached to the\n    /// event. The collector may pass a [visitor] to the `Event`'s\n    /// [`record` method] to record these values.\n    ///\n    /// [`Event`]: super::event::Event\n    /// [visitor]: super::field::Visit\n    /// [`record` method]: super::event::Event::record\n    /// [`dispatch` method]: super::event::Event::dispatch\n    fn event(&self, event: &Event<'_>);\n\n    /// Records that a span has been entered.\n    ///\n    /// When entering a span, this method is called to notify the collector\n    /// that the span has been entered. The collector is provided with the\n    /// [span ID] of the entered span, and should update any internal state\n    /// tracking the current span accordingly.\n    ///\n    /// [span ID]: super::span::Id\n    fn enter(&self, span: &span::Id);\n\n    /// Records that a span has been exited.\n    ///\n    /// When exiting a span, this method is called to notify the collector\n    /// that the span has been exited. The collector is provided with the\n    /// [span ID] of the exited span, and should update any internal state\n    /// tracking the current span accordingly.\n    ///\n    /// Exiting a span does not imply that the span will not be re-entered.\n    ///\n    /// [span ID]: super::span::Id\n    fn exit(&self, span: &span::Id);\n\n    /// Notifies the collector that a [span ID] has been cloned.\n    ///\n    /// This function is guaranteed to only be called with span IDs that were\n    /// returned by this collector's `new_span` function.\n    ///\n    /// Note that the default implementation of this function this is just the\n    /// identity function, passing through the identifier. However, it can be\n    /// used in conjunction with [`try_close`] to track the number of handles\n    /// capable of `enter`ing a span. When all the handles have been dropped\n    /// (i.e., `try_close` has been called one more time than `clone_span` for a\n    /// given ID), the collector may assume that the span will not be entered\n    /// again. It is then free to deallocate storage for data associated with\n    /// that span, write data from that span to IO, and so on.\n    ///\n    /// For more unsafe situations, however, if `id` is itself a pointer of some\n...\n", "needle_token_start": 2262, "needle_token_end": 2628, "code_context_ntokens": 16281, "output": ["```rust\n/// Sets this dispatch as the global default for the duration of the entire program.\n/// Will be used as a fallback if no thread-local dispatch has been set in a thread\n/// (using `with_default`.)\n///\n/// Can only be set once; subsequent attempts to set the global default will fail.\n/// Returns `Err` if the global default has already been set.\n///\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Warning</strong>: In general, libraries should <em>not</em> call\n/// <code>set_global_default()</code>! Doing so will cause conflicts when\n/// executables that depend on the library try to set the default collector later.\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\n\npub fn set_global_default(dispatcher: Dispatch) -> Result<(), SetGlobalDefaultError> {\n    // if `compare_exchange` returns Result::Ok(_), then `new` has been set and\n    // `current`\u2014now the prior value\u2014has been returned in the `Ok()` branch.\n    if GLOBAL_INIT\n        .compare_exchange(\n            UNINITIALIZED,\n            INITIALIZING,\n            Ordering::SeqCst,\n            Ordering::SeqCst,\n        )\n        .is_ok()\n    {\n        #[cfg(feature = \"alloc\")]\n        let collector = {\n            let collector = match dispatcher.collector {\n                Kind::Global(s) => s,\n                Kind::Scoped(s) => unsafe {\n                    // safety: this leaks the collector onto the heap. the\n                    // reference count will always be at least 1.\n                    &*Arc::into_raw(s)\n                },\n            };\n            Kind::Global(collector)\n        };\n\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = dispatcher.collector;\n\n        unsafe {\n            GLOBAL_DISPATCH = Dispatch { collector };\n        }\n        GLOBAL_INIT.store(INITIALIZED, Ordering::SeqCst);\n        EXISTS.store(true, Ordering::Release);\n        Ok(())\n    } else {\n        Err(SetGlobalDefaultError { _no_construct: () })\n    }\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "as_ref", "language": "rust", "path": "tracing-core/src/field.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide a reference to a string representation of an object, allowing for easier access and manipulation of its name property.\n2. **Input**: The function does not take any parameters.\n3. **Output**: Returns a reference to a string slice (`&str`) that represents the name of the object.\n4. **Procedure**: It retrieves the name of the object by accessing a specific field within the object's structure and returns a reference to this name string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "}\n\n/// A `Value` which serializes using `fmt::Display`.\n///\n/// Uses `record_debug` in the `Value` implementation to\n/// avoid an unnecessary evaluation.\n#[derive(Clone)]\npub struct DisplayValue<T: fmt::Display>(T);\n\n/// A `Value` which serializes as a string using `fmt::Debug`.\n#[derive(Clone)]\npub struct DebugValue<T: fmt::Debug>(T);\n\n/// Wraps a type implementing `fmt::Display` as a `Value` that can be\n/// recorded using its `Display` implementation.\npub fn display<T>(t: T) -> DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    DisplayValue(t)\n}\n\n/// Wraps a type implementing `fmt::Debug` as a `Value` that can be\n/// recorded using its `Debug` implementation.\npub fn debug<T>(t: T) -> DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    DebugValue(t)\n}\n\n// ===== impl Visit =====\n\nimpl<'a, 'b> Visit for fmt::DebugStruct<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.field(field.name(), value);\n    }\n}\n\nimpl<'a, 'b> Visit for fmt::DebugMap<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.entry(&format_args!(\"{}\", field), value);\n    }\n}\n\nimpl<F> Visit for F\nwhere\n    F: FnMut(&Field, &dyn fmt::Debug),\n{\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        (self)(field, value)\n    }\n}\n\n// ===== impl Value =====\n\nmacro_rules! impl_values {\n    ( $( $record:ident( $( $whatever:tt)+ ) ),+ ) => {\n        $(\n            impl_value!{ $record( $( $whatever )+ ) }\n        )+\n    }\n}\n\nmacro_rules! ty_to_nonzero {\n    (u8) => {\n        NonZeroU8\n    };\n    (u16) => {\n        NonZeroU16\n    };\n    (u32) => {\n        NonZeroU32\n    };\n    (u64) => {\n        NonZeroU64\n    };\n    (u128) => {\n        NonZeroU128\n    };\n    (usize) => {\n        NonZeroUsize\n    };\n    (i8) => {\n        NonZeroI8\n    };\n    (i16) => {\n        NonZeroI16\n    };\n    (i32) => {\n        NonZeroI32\n    };\n    (i64) => {\n        NonZeroI64\n    };\n    (i128) => {\n        NonZeroI128\n    };\n    (isize) => {\n        NonZeroIsize\n    };\n}\n\nmacro_rules! impl_one_value {\n    (f32, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f32, $op, $record);\n    };\n    (f64, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f64, $op, $record);\n    };\n    (bool, $op:expr, $record:ident) => {\n        impl_one_value!(normal, bool, $op, $record);\n    };\n    ($value_ty:tt, $op:expr, $record:ident) => {\n        impl_one_value!(normal, $value_ty, $op, $record);\n        impl_one_value!(nonzero, $value_ty, $op, $record);\n    };\n    (normal, $value_ty:tt, $op:expr, $record:ident) => {\n        impl $crate::sealed::Sealed for $value_ty {}\n        impl $crate::field::Value for $value_ty {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(*self))\n            }\n        }\n    };\n    (nonzero, $value_ty:tt, $op:expr, $record:ident) => {\n        // This `use num::*;` is reported as unused because it gets emitted\n        // for every single invocation of this macro, so there are multiple `use`s.\n        // All but the first are useless indeed.\n        // We need this import because we can't write a path where one part is\n        // the `ty_to_nonzero!($value_ty)` invocation.\n        #[allow(clippy::useless_attribute, unused)]\n        use num::*;\n        impl $crate::sealed::Sealed for ty_to_nonzero!($value_ty) {}\n        impl $crate::field::Value for ty_to_nonzero!($value_ty) {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(self.get()))\n            }\n        }\n    };\n}\n\nmacro_rules! impl_value {\n    ( $record:ident( $( $value_ty:tt ),+ ) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this, $record);\n        )+\n    };\n    ( $record:ident( $( $value_ty:tt ),+ as $as_ty:ty) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this as $as_ty, $record);\n        )+\n    };\n}\n\n// ===== impl Value =====\n\nimpl_values! {\n    record_u64(u64),\n    record_u64(usize, u32, u16, u8 as u64),\n    record_i64(i64),\n    record_i64(isize, i32, i16, i8 as i64),\n    record_u128(u128),\n    record_i128(i128),\n    record_bool(bool),\n    record_f64(f64, f32 as f64)\n}\n\nimpl<T: crate::sealed::Sealed> crate::sealed::Sealed for Wrapping<T> {}\nimpl<T: crate::field::Value> crate::field::Value for Wrapping<T> {\n    fn record(&self, key: &crate::field::Field, visitor: &mut dyn crate::field::Visit) {\n        self.0.record(key, visitor)\n    }\n}\n\nimpl crate::sealed::Sealed for str {}\n\nimpl Value for str {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_error(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (*self).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a mut T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a mut T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        // Don't use `(*self).record(key, visitor)`, otherwise would\n        // cause stack overflow due to `unconditional_recursion`.\n        T::record(self, key, visitor)\n    }\n}\n\nimpl<'a> crate::sealed::Sealed for fmt::Arguments<'a> {}\n\nimpl<'a> Value for fmt::Arguments<'a> {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<T: ?Sized> crate::sealed::Sealed for alloc::boxed::Box<T> where T: Value {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl<T: ?Sized> Value for alloc::boxed::Box<T>\nwhere\n    T: Value,\n{\n    #[inline]\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        self.as_ref().record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl crate::sealed::Sealed for alloc::string::String {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl Value for alloc::string::String {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self.as_str())\n    }\n}\n\nimpl fmt::Debug for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        // We are only going to be recording the field value, so we don't\n        // actually care about the field name here.\n        struct NullCallsite;\n        static NULL_CALLSITE: NullCallsite = NullCallsite;\n        impl crate::callsite::Callsite for NullCallsite {\n            fn set_interest(&self, _: crate::collect::Interest) {\n                unreachable!(\"you somehow managed to register the null callsite?\")\n            }\n\n            fn metadata(&self) -> &crate::Metadata<'_> {\n                unreachable!(\"you somehow managed to access the null callsite?\")\n            }\n        }\n\n        static FIELD: Field = Field {\n            i: 0,\n            fields: FieldSet::new(&[], crate::identify_callsite!(&NULL_CALLSITE)),\n        };\n\n        let mut res = Ok(());\n        self.record(&FIELD, &mut |_: &Field, val: &dyn fmt::Debug| {\n            res = write!(f, \"{:?}\", val);\n        });\n        res\n    }\n}\n\nimpl fmt::Display for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Debug::fmt(self, f)\n    }\n}\n\n// ===== impl DisplayValue =====\n\nimpl<T: fmt::Display> crate::sealed::Sealed for DisplayValue<T> {}\n\nimpl<T> Value for DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Debug for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Display::fmt(self, f)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Display for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\n// ===== impl DebugValue =====\n\nimpl<T: fmt::Debug> crate::sealed::Sealed for DebugValue<T> {}\n\nimpl<T: fmt::Debug> Value for DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, &self.0)\n    }\n}\n\nimpl<T: fmt::Debug> fmt::Debug for DebugValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\nimpl crate::sealed::Sealed for Empty {}\nimpl Value for Empty {\n    #[inline]\n    fn record(&self, _: &Field, _: &mut dyn Visit) {}\n}\n\n// ===== impl Field =====\n\nimpl Field {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this field.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Returns a string representing the name of the field.\n    pub fn name(&self) -> &'static str {\n        self.fields.names[self.i]\n    }\n}\n\nimpl fmt::Display for Field {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(self.name())\n    }\n}\n\nimpl AsRef<str> for Field {\n    \nfn as_ref(&self) -> &str {\n        self.name()\n    }\n}\n\nimpl PartialEq for Field {\n    fn eq(&self, other: &Self) -> bool {\n        self.callsite() == other.callsite() && self.i == other.i\n    }\n}\n\nimpl Eq for Field {}\n\nimpl Hash for Field {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: Hasher,\n    {\n        self.callsite().hash(state);\n        self.i.hash(state);\n    }\n}\n\nimpl Clone for Field {\n    fn clone(&self) -> Self {\n        Field {\n            i: self.i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        }\n    }\n}\n\n// ===== impl FieldSet =====\n\nimpl FieldSet {\n    /// Constructs a new `FieldSet` with the given array of field names and callsite.\n    pub const fn new(names: &'static [&'static str], callsite: callsite::Identifier) -> Self {\n        Self { names, callsite }\n    }\n\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this set of fields..\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub(crate) fn callsite(&self) -> callsite::Identifier {\n        callsite::Identifier(self.callsite.0)\n    }\n\n    /// Returns the [`Field`] named `name`, or `None` if no such field exists.\n    ///\n    /// [`Field`]: super::Field\n    pub fn field<Q: ?Sized>(&self, name: &Q) -> Option<Field>\n    where\n        Q: Borrow<str>,\n    {\n        let name = &name.borrow();\n        self.names.iter().position(|f| f == name).map(|i| Field {\n            i,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        })\n    }\n\n    /// Returns `true` if `self` contains the given `field`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: If <code>field</code> shares a name with a field\n    /// in this <code>FieldSet</code>, but was created by a <code>FieldSet</code>\n    /// with a different callsite, this <code>FieldSet</code> does <em>not</em>\n    /// contain it. This is so that if two separate span callsites define a field\n    /// named \"foo\", the <code>Field</code> corresponding to \"foo\" for each\n    /// of those callsites are not equivalent.\n    /// </pre></div>\n    pub fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite() && field.i <= self.len()\n    }\n\n    /// Returns an iterator over the `Field`s in this `FieldSet`.\n    #[inline]\n    pub fn iter(&self) -> Iter {\n        let idxs = 0..self.len();\n        Iter {\n            idxs,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        }\n    }\n\n    /// Returns a new `ValueSet` with entries for this `FieldSet`'s values.\n    #[doc(hidden)]\n    pub fn value_set<'v, V>(&'v self, values: &'v V) -> ValueSet<'v>\n    where\n        V: ValidLen<'v>,\n    {\n        ValueSet {\n            fields: self,\n            values: values.borrow(),\n        }\n    }\n\n    /// Returns the number of fields in this `FieldSet`.\n    #[inline]\n    pub fn len(&self) -> usize {\n        self.names.len()\n    }\n\n    /// Returns whether or not this `FieldSet` has fields.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.names.is_empty()\n    }\n}\n\nimpl<'a> IntoIterator for &'a FieldSet {\n    type IntoIter = Iter;\n    type Item = Field;\n    #[inline]\n    fn into_iter(self) -> Self::IntoIter {\n        self.iter()\n    }\n}\n\nimpl fmt::Debug for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"FieldSet\")\n            .field(\"names\", &self.names)\n            .field(\"callsite\", &self.callsite)\n            .finish()\n    }\n}\n\nimpl fmt::Display for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_set()\n            .entries(self.names.iter().map(display))\n            .finish()\n    }\n}\n\nimpl Eq for FieldSet {}\n\nimpl PartialEq for FieldSet {\n    fn eq(&self, other: &Self) -> bool {\n        if core::ptr::eq(&self, &other) {\n            true\n        } else if cfg!(not(debug_assertions)) {\n            // In a well-behaving application, two `FieldSet`s can be assumed to\n            // be totally equal so long as they share the same callsite.\n            self.callsite == other.callsite\n        } else {\n            // However, when debug-assertions are enabled, do NOT assume that\n            // the application is well-behaving; check every the field names of\n            // each `FieldSet` for equality.\n\n            // `FieldSet` is destructured here to ensure a compile-error if the\n            // fields of `FieldSet` change.\n            let Self {\n                names: lhs_names,\n                callsite: lhs_callsite,\n            } = self;\n\n            let Self {\n                names: rhs_names,\n                callsite: rhs_callsite,\n            } = &other;\n\n            // Check callsite equality first, as it is probably cheaper to do\n            // than str equality.\n            lhs_callsite == rhs_callsite && lhs_names == rhs_names\n        }\n    }\n}\n\n// ===== impl Iter =====\n\nimpl Iterator for Iter {\n    type Item = Field;\n    #[inline]\n    fn next(&mut self) -> Option<Field> {\n        let i = self.idxs.next()?;\n        Some(Field {\n            i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        })\n    }\n}\n\n// ===== impl ValueSet =====\n\nimpl<'a> ValueSet<'a> {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// defining the fields this `ValueSet` refers to.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Visits all the fields in this `ValueSet` with the provided [visitor].\n    ///\n    /// [visitor]: Visit\n    pub fn record(&self, visitor: &mut dyn Visit) {\n        let my_callsite = self.callsite();\n        for (field, value) in self.values {\n            if field.callsite() != my_callsite {\n                continue;\n            }\n            if let Some(value) = value {\n                value.record(field, visitor);\n            }\n        }\n    }\n\n    /// Returns the number of fields in this `ValueSet` that would be visited\n    /// by a given [visitor] to the [`ValueSet::record()`] method.\n    ///\n    /// [visitor]: Visit\n    /// [`ValueSet::record()`]: ValueSet::record()\n    pub fn len(&self) -> usize {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .filter(|(field, _)| field.callsite() == my_callsite)\n            .count()\n    }\n\n    /// Returns `true` if this `ValueSet` contains a value for the given `Field`.\n    pub(crate) fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite()\n            && self\n                .values\n                .iter()\n                .any(|(key, val)| *key == field && val.is_some())\n    }\n\n    /// Returns true if this `ValueSet` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .all(|(key, val)| val.is_none() || key.callsite() != my_callsite)\n    }\n\n    pub(crate) fn field_set(&self) -> &FieldSet {\n        self.fields\n    }\n}\n\nimpl<'a> fmt::Debug for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_struct(\"ValueSet\"), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .field(\"callsite\", &self.callsite())\n            .finish()\n    }\n}\n\nimpl<'a> fmt::Display for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_map(), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .finish()\n    }\n}\n\n// ===== impl ValidLen =====\n\nmod private {\n    use super::*;\n\n    /// Restrictions on `ValueSet` lengths were removed in #2508 but this type remains for backwards compatibility.\n    pub trait ValidLen<'a>: Borrow<[(&'a Field, Option<&'a (dyn Value + 'a)>)]> {}\n\n    impl<'a, const N: usize> ValidLen<'a> for [(&'a Field, Option<&'a (dyn Value + 'a)>); N] {}\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::metadata::{Kind, Level, Metadata};\n\n    // Make sure TEST_CALLSITE_* have non-zero size, so they can't be located at the same address.\n    struct TestCallsite1(u8);\n    static TEST_CALLSITE_1: TestCallsite1 = TestCallsite1(0);\n    static TEST_META_1: Metadata<'static> = metadata! {\n        name: \"field_test1\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_1,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite1 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_1\n        }\n    }\n\n    struct TestCallsite2(u8);\n    static TEST_CALLSITE_2: TestCallsite2 = TestCallsite2(0);\n    static TEST_META_2: Metadata<'static> = metadata! {\n        name: \"field_test2\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_2,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite2 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_2\n        }\n    }\n\n    #[test]\n    fn value_set_with_no_values_is_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), None),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn empty_value_set_is_empty() {\n        let fields = TEST_META_1.fields();\n        let valueset = fields.value_set(&[]);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn value_sets_with_fields_from_other_callsites_are_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = TEST_META_2.fields().value_set(values);\n        assert!(valueset.is_empty())\n    }\n\n    #[test]\n    fn sparse_value_sets_are_not_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(!valueset.is_empty());\n    }\n\n    #[test]\n    fn fields_from_other_callsets_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (\n                &TEST_META_2.fields().field(\"bar\").unwrap(),\n                Some(&57 as &dyn Value),\n            ),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.callsite(), TEST_META_1.callsite())\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    fn empty_fields_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&Empty as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&Empty as &dyn Value)),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.name(), \"bar\")\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_debug_fn() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = fields.value_set(values);\n        let mut result = String::new();\n        valueset.record(&mut |_: &Field, value: &dyn fmt::Debug| {\n            use core::fmt::Write;\n            write!(&mut result, \"{:?}\", value).unwrap();\n        });\n        assert_eq!(result, String::from(\"123\"));\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_error() {\n        let fields = TEST_META_1.fields();\n        let err: Box<dyn std::error::Error + Send + Sync + 'static> =\n            std::io::Error::new(std::io::ErrorKind::Other, \"lol\").into();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&err as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&Empty as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&Empty as &dyn Value)),\n        ];\n        let valueset = fields.value_set(values);\n        let mut result = String::new();\n        valueset.record(&mut |_: &Field, value: &dyn fmt::Debug| {\n            use core::fmt::Write;\n            write!(&mut result, \"{:?}\", value).unwrap();\n        });\n        assert_eq!(result, format!(\"{}\", err));\n    }\n}\n\n// Path: tracing-core/src/parent.rs\nuse crate::span::Id;\n\n#[derive(Debug)]\npub(crate) enum Parent {\n    /// The new span will be a root span.\n    Root,\n    /// The new span will be rooted in the current span.\n    Current,\n    /// The new span has an explicitly-specified parent.\n    Explicit(Id),\n}\n\n// Path: tracing-core/src/span.rs\n//! Spans represent periods of time in the execution of a program.\nuse core::num::NonZeroU64;\n\nuse crate::field::FieldSet;\nuse crate::parent::Parent;\nuse crate::{field, Metadata};\n\n/// Identifies a span within the context of a collector.\n///\n/// They are generated by [collector]s for each span as it is created, by\n/// the [`new_span`] trait method. See the documentation for that method for\n/// more information on span ID generation.\n///\n/// [collector]: super::collect::Collect\n/// [`new_span`]: super::collect::Collect::new_span\n#[derive(Clone, Debug, PartialEq, Eq, Hash)]\npub struct Id(NonZeroU64);\n\n/// Attributes provided to a collector describing a new span when it is\n/// created.\n#[derive(Debug)]\npub struct Attributes<'a> {\n    metadata: &'static Metadata<'static>,\n    values: &'a field::ValueSet<'a>,\n    parent: Parent,\n}\n\n/// A set of fields recorded by a span.\n#[derive(Debug)]\npub struct Record<'a> {\n    values: &'a field::ValueSet<'a>,\n}\n\n/// Indicates what [the collector considers] the \"current\" span.\n///\n/// As collectors may not track a notion of a current span, this has three\n/// possible states:\n/// - \"unknown\", indicating that the collector does not track a current span,\n/// - \"none\", indicating that the current context is known to not be in a span,\n/// - \"some\", with the current span's [`Id`] and [`Metadata`].\n///\n/// [the collector considers]: super::collect::Collect::current_span\n/// [`Metadata`]: super::metadata::Metadata\n#[derive(Debug)]\npub struct Current {\n    inner: CurrentInner,\n}\n\n#[derive(Debug)]\nenum CurrentInner {\n    Current {\n        id: Id,\n        metadata: &'static Metadata<'static>,\n    },\n    None,\n    Unknown,\n}\n\n// ===== impl Span =====\n\nimpl Id {\n    /// Constructs a new span ID from the given `u64`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: Span IDs must be greater than zero.</pre></div>\n    ///\n    /// # Panics\n    /// - If the provided `u64` is 0.\n    pub fn from_u64(u: u64) -> Self {\n        Id(NonZeroU64::new(u).expect(\"span IDs must be > 0\"))\n    }\n\n    /// Constructs a new span ID from the given `NonZeroU64`.\n    ///\n    /// Unlike [`Id::from_u64`](Self::from_u64), this will never panic.\n    #[inline]\n    pub const fn from_non_zero_u64(id: NonZeroU64) -> Self {\n        Id(id)\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `u64`.\n    pub fn into_u64(&self) -> u64 {\n        self.0.get()\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `NonZeroU64`.\n    #[inline]\n    pub const fn into_non_zero_u64(&self) -> NonZeroU64 {\n        self.0\n    }\n}\n\nimpl<'a> From<&'a Id> for Option<Id> {\n    fn from(id: &'a Id) -> Self {\n        Some(id.clone())\n    }\n}\n\n// ===== impl Attributes =====\n\nimpl<'a> Attributes<'a> {\n    /// Returns `Attributes` describing a new child span of the current span,\n    /// with the provided metadata and values.\n    pub fn new(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Current,\n        }\n    }\n\n    /// Returns `Attributes` describing a new span at the root of its own trace\n    /// tree, with the provided metadata and values.\n    pub fn new_root(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Root,\n        }\n    }\n\n    /// Returns `Attributes` describing a new child span of the specified\n    /// parent span, with the provided metadata and values.\n    pub fn child_of(\n        parent: Id,\n        metadata: &'static Metadata<'static>,\n        values: &'a field::ValueSet<'a>,\n    ) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Explicit(parent),\n        }\n    }\n\n    /// Returns a reference to the new span's metadata.\n    pub fn metadata(&self) -> &'static Metadata<'static> {\n        self.metadata\n    }\n\n    /// Returns a reference to a `ValueSet` containing any values the new span\n    /// was created with.\n    pub fn values(&self) -> &field::ValueSet<'a> {\n        self.values\n    }\n\n    /// Returns true if the new span should be a root.\n    pub fn is_root(&self) -> bool {\n        matches!(self.parent, Parent::Root)\n    }\n\n    /// Returns true if the new span's parent should be determined based on the\n    /// current context.\n    ///\n    /// If this is true and the current thread is currently inside a span, then\n    /// that span should be the new span's parent. Otherwise, if the current\n    /// thread is _not_ inside a span, then the new span will be the root of its\n    /// own trace tree.\n    pub fn is_contextual(&self) -> bool {\n        matches!(self.parent, Parent::Current)\n    }\n\n    /// Returns the new span's explicitly-specified parent, if there is one.\n    ///\n    /// Otherwise (if the new span is a root or is a child of the current span),\n    /// returns `None`.\n    pub fn parent(&self) -> Option<&Id> {\n        match self.parent {\n            Parent::Explicit(ref p) => Some(p),\n            _ => None,\n        }\n    }\n\n    /// Records all the fields in this set of `Attributes` with the provided\n    /// [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns `true` if this set of `Attributes` contains a value for the\n    /// given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this set of `Attributes` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n\n    /// Returns the set of all [fields] defined by this span's [`Metadata`].\n    ///\n    /// Note that the [`FieldSet`] returned by this method includes *all* the\n    /// fields declared by this span, not just those with values that are recorded\n    /// as part of this set of `Attributes`. Other fields with values not present in\n    /// this `Attributes`' value set may [record] values later.\n    ///\n    /// [fields]: crate::field\n    /// [record]: Attributes::record()\n    /// [`Metadata`]: crate::metadata::Metadata\n    /// [`FieldSet`]: crate::field::FieldSet\n    pub fn fields(&self) -> &FieldSet {\n        self.values.field_set()\n    }\n}\n\n// ===== impl Record =====\n\nimpl<'a> Record<'a> {\n    /// Constructs a new `Record` from a `ValueSet`.\n    pub fn new(values: &'a field::ValueSet<'a>) -> Self {\n        Self { values }\n    }\n\n    /// Records all the fields in this `Record` with the provided [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns the number of fields that would be visited from this `Record`\n    /// when [`Record::record()`] is called\n    ///\n    /// [`Record::record()`]: Record::record()\n    pub fn len(&self) -> usize {\n        self.values.len()\n    }\n\n    /// Returns `true` if this `Record` contains a value for the given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this `Record` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n}\n\n// ===== impl Current =====\n\nimpl Current {\n    /// Constructs a new `Current` that indicates the current context is a span\n    /// with the given `id` and `metadata`.\n    pub fn new(id: Id, metadata: &'static Metadata<'static>) -> Self {\n        Self {\n            inner: CurrentInner::Current { id, metadata },\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the current context is *not*\n    /// in a span.\n    pub fn none() -> Self {\n        Self {\n            inner: CurrentInner::None,\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the collector does not\n    /// track a current span.\n    pub fn unknown() -> Self {\n        Self {\n            inner: CurrentInner::Unknown,\n        }\n    }\n\n    /// Returns `true` if the collector that constructed this `Current` tracks a\n    /// current span.\n    ///\n    /// If this returns `true` and [`id`], [`metadata`], or [`into_inner`]\n    /// return `None`, that indicates that we are currently known to *not* be\n    /// inside a span. If this returns `false`, those methods will also return\n    /// `None`, but in this case, that is because the collector does not keep\n    /// track of the currently-entered span.\n    ///\n    /// [`id`]: Self::id\n    /// [`metadata`]: Self::metadata\n    /// [`into_inner`]: Self::into_inner\n    pub fn is_known(&self) -> bool {\n        !matches!(self.inner, CurrentInner::Unknown)\n    }\n\n    /// Consumes `self` and returns the span `Id` and `Metadata` of the current\n    /// span, if one exists and is known.\n    pub fn into_inner(self) -> Option<(Id, &'static Metadata<'static>)> {\n        match self.inner {\n            CurrentInner::Current { id, metadata } => Some((id, metadata)),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Id` of the current span, if one exists and is known.\n    pub fn id(&self) -> Option<&Id> {\n        match self.inner {\n            CurrentInner::Current { ref id, .. } => Some(id),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Metadata` of the current span, if one exists and is known.\n    pub fn metadata(&self) -> Option<&'static Metadata<'static>> {\n        match self.inner {\n            CurrentInner::Current { metadata, .. } => Some(metadata),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'a Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id()\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id().cloned()\n    }\n}\n\nimpl From<Current> for Option<Id> {\n    fn from(cur: Current) -> Self {\n        match cur.inner {\n            CurrentInner::Current { id, .. } => Some(id),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'static Metadata<'static>> {\n    fn from(cur: &'a Current) -> Self {\n        cur.metadata()\n    }\n}\n\n// Path: tracing-core/src/dispatch.rs\n//! Dispatches trace events to [`Collect`]s.\n//!\n//! The _dispatcher_ is the component of the tracing system which is responsible\n//! for forwarding trace data from the instrumentation points that generate it\n//! to the collector that collects it.\n//!\n//! # Using the Trace Dispatcher\n//!\n//! Every thread in a program using `tracing` has a _default collector_. When\n//! events occur, or spans are created, they are dispatched to the thread's\n//! current collector.\n//!\n//! ## Setting the Default Collector\n//!\n//! By default, the current collector is an empty implementation that does\n//! nothing. Trace data provided to this \"do nothing\" implementation is\n//! immediately discarded, and is not available for any purpose.\n//!\n//! To use another collector implementation, it must be set as the default.\n//! There are two methods for doing so: [`with_default`] and\n//! [`set_global_default`]. `with_default` sets the default collector for the\n//! duration of a scope, while `set_global_default` sets a default collector\n//! for the entire process.\n//!\n//! To use either of these functions, we must first wrap our collector in a\n//! [`Dispatch`], a cloneable, type-erased reference to a collector. For\n//! example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"alloc\")]\n//! use dispatch::Dispatch;\n//!\n//! # #[cfg(feature = \"alloc\")]\n//! let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"alloc\")]\n//! let my_dispatch = Dispatch::new(my_collector);\n//! ```\n//! Then, we can use [`with_default`] to set our `Dispatch` as the default for\n//! the duration of a block:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # let _my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(_my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::with_default(&my_dispatch, || {\n//!     // my_collector is the default\n//! });\n//!\n//! // no default collector again\n//! ```\n//! It's important to note that `with_default` will not propagate the current\n//! thread's default collector to any threads spawned within the `with_default`\n//! block. To propagate the default collector to new threads, either use\n//! `with_default` from the new thread, or use `set_global_default`.\n//!\n//! As an alternative to `with_default`, we can use [`set_global_default`] to\n//! set a `Dispatch` as the default for all threads, for the lifetime of the\n//! program. For example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"std\")]\n//! # let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::set_global_default(my_dispatch)\n//!     // `set_global_default` will return an error if the global default\n//!     // collector has already been set.\n//!     .expect(\"global default was already set!\");\n//!\n//! // `my_collector` is now the default\n//! ```\n//!\n//! <div class=\"example-wrap\" style=\"display:inline-block\">\n//! <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n//!\n//! **Note**: the thread-local scoped dispatcher ([`with_default`]) requires the\n//! Rust standard library. `no_std` users should use [`set_global_default`] instead.\n//!\n//! </pre></div>\n//!\n//! ## Accessing the Default Collector\n//!\n//! A thread's current default collector can be accessed using the\n//! [`get_default`] function, which executes a closure with a reference to the\n//! currently default `Dispatch`. This is used primarily by `tracing`\n//! instrumentation.\nuse crate::{\n    collect::{self, Collect, NoCollector},\n    span, Event, LevelFilter, Metadata,\n};\n\nuse core::{\n    any::Any,\n    fmt,\n    sync::atomic::{AtomicBool, AtomicUsize, Ordering},\n};\n\n#[cfg(feature = \"std\")]\nuse std::{\n    cell::{Cell, RefCell, RefMut},\n    error,\n};\n\n#[cfg(feature = \"alloc\")]\nuse alloc::sync::{Arc, Weak};\n\n#[cfg(feature = \"alloc\")]\nuse core::ops::Deref;\n\n/// `Dispatch` trace data to a [`Collect`].\n#[derive(Clone)]\npub struct Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Arc<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n/// `WeakDispatch` is a version of [`Dispatch`] that holds a non-owning reference\n/// to a [collector].\n///\n/// The collector may be accessed by calling [`WeakDispatch::upgrade`],\n/// which returns an `Option<Dispatch>`. If all [`Dispatch`] clones that point\n/// at the collector have been dropped, [`WeakDispatch::upgrade`] will return\n/// `None`. Otherwise, it will return `Some(Dispatch)`.\n///\n/// A `WeakDispatch` may be created from a [`Dispatch`] by calling the\n/// [`Dispatch::downgrade`] method. The primary use for creating a\n/// [`WeakDispatch`] is to allow a collector to hold a cyclical reference to\n/// itself without creating a memory leak. See [here] for details.\n///\n/// This type is analogous to the [`std::sync::Weak`] type, but for a\n/// [`Dispatch`] rather than an [`Arc`].\n///\n/// [collector]: Collect\n/// [`Arc`]: std::sync::Arc\n/// [here]: Collect#avoiding-memory-leaks\n#[derive(Clone)]\npub struct WeakDispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Weak<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n#[cfg(feature = \"alloc\")]\n#[derive(Clone)]\nenum Kind<T> {\n    Global(&'static (dyn Collect + Send + Sync)),\n    Scoped(T),\n}\n\n#[cfg(feature = \"std\")]\nthread_local! {\n    static CURRENT_STATE: State = const {\n        State {\n            default: RefCell::new(None),\n            can_enter: Cell::new(true),\n        }\n    };\n}\n\nstatic EXISTS: AtomicBool = AtomicBool::new(false);\nstatic GLOBAL_INIT: AtomicUsize = AtomicUsize::new(UNINITIALIZED);\n\n#[cfg(feature = \"std\")]\nstatic SCOPED_COUNT: AtomicUsize = AtomicUsize::new(0);\n\nconst UNINITIALIZED: usize = 0;\nconst INITIALIZING: usize = 1;\nconst INITIALIZED: usize = 2;\n\nstatic mut GLOBAL_DISPATCH: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NONE: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NO_COLLECTOR: NoCollector = NoCollector::new();\n\n/// The dispatch state of a thread.\n#[cfg(feature = \"std\")]\nstruct State {\n    /// This thread's current default dispatcher.\n    default: RefCell<Option<Dispatch>>,\n    /// Whether or not we can currently begin dispatching a trace event.\n    ///\n    /// This is set to `false` when functions such as `enter`, `exit`, `event`,\n    /// and `new_span` are called on this thread's default dispatcher, to\n    /// prevent further trace events triggered inside those functions from\n    /// creating an infinite recursion. When we finish handling a dispatch, this\n    /// is set back to `true`.\n    can_enter: Cell<bool>,\n}\n\n/// While this guard is active, additional calls to collector functions on\n/// the default dispatcher will not be able to access the dispatch context.\n/// Dropping the guard will allow the dispatch context to be re-entered.\n#[cfg(feature = \"std\")]\nstruct Entered<'a>(&'a State);\n\n/// A guard that resets the current default dispatcher to the prior\n/// default dispatcher when dropped.\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[derive(Debug)]\npub struct DefaultGuard(Option<Dispatch>);\n\n/// Sets this dispatch as the default for the duration of a closure.\n///\n/// The default dispatcher is used when creating a new [span] or\n/// [`Event`].\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: This function required the Rust standard library.\n/// <!-- hack: this whitespace makes rustdoc interpret the next line as markdown again -->\n///\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\npub fn with_default<T>(dispatcher: &Dispatch, f: impl FnOnce() -> T) -> T {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this (rather than simply resetting after calling\n    // `f`) ensures that we always reset to the prior dispatcher even if `f`\n    // panics.\n    let _guard = set_default(dispatcher);\n    f()\n}\n\n/// Sets the dispatch as the default dispatch for the duration of the lifetime\n/// of the returned DefaultGuard\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n///\n/// **Note**: This function required the Rust standard library.\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[must_use = \"Dropping the guard unregisters the dispatcher.\"]\npub fn set_default(dispatcher: &Dispatch) -> DefaultGuard {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this ensures that we always reset to the prior\n    // dispatcher even if the thread calling this function panics.\n    State::set_default(dispatcher.clone())\n}\n\n/// Sets this dispatch as the global default for the duration of the entire program.\n/// Will be used as a fallback if no thread-local dispatch has been set in a thread\n/// (using `with_default`.)\n///\n/// Can only be set once; subsequent attempts to set the global default will fail.\n/// Returns `Err` if the global default has already been set.\n///\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Warning</strong>: In general, libraries should <em>not</em> call\n/// <code>set_global_default()</code>! Doing so will cause conflicts when\n/// executables that depend on the library try to set the default collector later.\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\npub fn set_global_default(dispatcher: Dispatch) -> Result<(), SetGlobalDefaultError> {\n    // if `compare_exchange` returns Result::Ok(_), then `new` has been set and\n    // `current`\u2014now the prior value\u2014has been returned in the `Ok()` branch.\n    if GLOBAL_INIT\n        .compare_exchange(\n            UNINITIALIZED,\n            INITIALIZING,\n            Ordering::SeqCst,\n            Ordering::SeqCst,\n        )\n        .is_ok()\n    {\n        #[cfg(feature = \"alloc\")]\n        let collector = {\n            let collector = match dispatcher.collector {\n                Kind::Global(s) => s,\n                Kind::Scoped(s) => unsafe {\n                    // safety: this leaks the collector onto the heap. the\n                    // reference count will always be at least 1.\n                    &*Arc::into_raw(s)\n                },\n            };\n            Kind::Global(collector)\n        };\n\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = dispatcher.collector;\n\n        unsafe {\n            GLOBAL_DISPATCH = Dispatch { collector };\n        }\n        GLOBAL_INIT.store(INITIALIZED, Ordering::SeqCst);\n        EXISTS.store(true, Ordering::Release);\n        Ok(())\n    } else {\n        Err(SetGlobalDefaultError { _no_construct: () })\n    }\n}\n\n/// Returns true if a `tracing` dispatcher has ever been set.\n///\n/// This may be used to completely elide trace points if tracing is not in use\n/// at all or has yet to be initialized.\n#[doc(hidden)]\n#[inline(always)]\npub fn has_been_set() -> bool {\n    EXISTS.load(Ordering::Relaxed)\n}\n\n/// Returned if setting the global dispatcher fails.\npub struct SetGlobalDefaultError {\n    _no_construct: (),\n}\n\nimpl fmt::Debug for SetGlobalDefaultError {\n...\n", "needle_token_start": 4035, "needle_token_end": 4055, "code_context_ntokens": 16221, "output": ["```rust\nfn get_name(&self) -> &str {\n    &self.name\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "default", "language": "rust", "path": "tracing-core/src/dispatch.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To obtain the current default instance of the dispatcher.\n2. **Input**: None.\n3. **Output**: Returns an instance of the dispatcher.\n4. **Procedure**: Retrieves the default dispatcher by invoking a function that accesses a globally stored default instance and returns a clone of it.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    // `current`\u2014now the prior value\u2014has been returned in the `Ok()` branch.\n    if GLOBAL_INIT\n        .compare_exchange(\n            UNINITIALIZED,\n            INITIALIZING,\n            Ordering::SeqCst,\n            Ordering::SeqCst,\n        )\n        .is_ok()\n    {\n        #[cfg(feature = \"alloc\")]\n        let collector = {\n            let collector = match dispatcher.collector {\n                Kind::Global(s) => s,\n                Kind::Scoped(s) => unsafe {\n                    // safety: this leaks the collector onto the heap. the\n                    // reference count will always be at least 1.\n                    &*Arc::into_raw(s)\n                },\n            };\n            Kind::Global(collector)\n        };\n\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = dispatcher.collector;\n\n        unsafe {\n            GLOBAL_DISPATCH = Dispatch { collector };\n        }\n        GLOBAL_INIT.store(INITIALIZED, Ordering::SeqCst);\n        EXISTS.store(true, Ordering::Release);\n        Ok(())\n    } else {\n        Err(SetGlobalDefaultError { _no_construct: () })\n    }\n}\n\n/// Returns true if a `tracing` dispatcher has ever been set.\n///\n/// This may be used to completely elide trace points if tracing is not in use\n/// at all or has yet to be initialized.\n#[doc(hidden)]\n#[inline(always)]\npub fn has_been_set() -> bool {\n    EXISTS.load(Ordering::Relaxed)\n}\n\n/// Returned if setting the global dispatcher fails.\npub struct SetGlobalDefaultError {\n    _no_construct: (),\n}\n\nimpl fmt::Debug for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"SetGlobalDefaultError\")\n            .field(&Self::MESSAGE)\n            .finish()\n    }\n}\n\nimpl fmt::Display for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(Self::MESSAGE)\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl error::Error for SetGlobalDefaultError {}\n\nimpl SetGlobalDefaultError {\n    const MESSAGE: &'static str = \"a global default trace dispatcher has already been set\";\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatch::Dispatch\n#[cfg(feature = \"std\")]\n#[inline(always)]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    if SCOPED_COUNT.load(Ordering::Acquire) == 0 {\n        // fast path if no scoped dispatcher has been set; just use the global\n        // default.\n        return f(get_global());\n    }\n\n    get_default_slow(f)\n}\n\n#[cfg(feature = \"std\")]\n#[inline(never)]\nfn get_default_slow<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    // While this guard is active, additional calls to collector functions on\n    // the default dispatcher will not be able to access the dispatch context.\n    // Dropping the guard will allow the dispatch context to be re-entered.\n    struct Entered<'a>(&'a Cell<bool>);\n    impl<'a> Drop for Entered<'a> {\n        #[inline]\n        fn drop(&mut self) {\n            self.0.set(true);\n        }\n    }\n\n    CURRENT_STATE\n        .try_with(|state| {\n            if state.can_enter.replace(false) {\n                let _guard = Entered(&state.can_enter);\n\n                let mut default = state.default.borrow_mut();\n                let default = default\n                    // if the local default for this thread has never been set,\n                    // populate it with the global default, so we don't have to\n                    // keep getting the global on every `get_default_slow` call.\n                    .get_or_insert_with(|| get_global().clone());\n\n                return f(&*default);\n            }\n\n            f(&Dispatch::none())\n        })\n        .unwrap_or_else(|_| f(&Dispatch::none()))\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(feature = \"std\")]\n#[doc(hidden)]\n#[inline(never)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    CURRENT_STATE\n        .try_with(|state| {\n            let entered = state.enter()?;\n            Some(f(&entered.current()))\n        })\n        .ok()?\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\n#[doc(hidden)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    Some(f(&get_global()))\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    f(get_global())\n}\n\n#[inline(always)]\npub(crate) fn get_global() -> &'static Dispatch {\n    if GLOBAL_INIT.load(Ordering::Acquire) != INITIALIZED {\n        return &NONE;\n    }\n    unsafe {\n        // This is safe given the invariant that setting the global dispatcher\n        // also sets `GLOBAL_INIT` to `INITIALIZED`.\n        &GLOBAL_DISPATCH\n    }\n}\n\n#[cfg(feature = \"std\")]\npub(crate) struct Registrar(Kind<Weak<dyn Collect + Send + Sync>>);\n\nimpl Dispatch {\n    /// Returns a new `Dispatch` that discards events and spans.\n    #[inline]\n    pub fn none() -> Self {\n        Dispatch {\n            #[cfg(feature = \"alloc\")]\n            collector: Kind::Global(&NO_COLLECTOR),\n            #[cfg(not(feature = \"alloc\"))]\n            collector: &NO_COLLECTOR,\n        }\n    }\n\n    /// Returns a `Dispatch` that forwards to the given [`Collect`].\n    ///\n    /// [`Collect`]: super::collect::Collect\n    #[cfg(feature = \"alloc\")]\n    #[cfg_attr(docsrs, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    pub fn new<C>(collector: C) -> Self\n    where\n        C: Collect + Send + Sync + 'static,\n    {\n        let me = Dispatch {\n            collector: Kind::Scoped(Arc::new(collector)),\n        };\n        crate::callsite::register_dispatch(&me);\n        me\n    }\n\n    /// Returns a `Dispatch` that forwards to the given static [collector].\n    ///\n    /// Unlike [`Dispatch::new`], this function is always available on all\n    /// platforms, even when the `std` or `alloc` features are disabled.\n    ///\n    /// In order to use `from_static`, the `Collector` itself must be stored in\n    /// a static. For example:\n    ///\n    /// ```rust\n    /// struct MyCollector {\n    ///    // ...\n    /// }\n    ///\n    /// # use tracing_core::{span::{Id, Attributes, Current, Record}, Event, Metadata};\n    /// impl tracing_core::Collect for MyCollector {\n    ///     // ...\n    /// #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n    /// #   fn record(&self, _: &Id, _: &Record) {}\n    /// #   fn event(&self, _: &Event) {}\n    /// #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n    /// #   fn enabled(&self, _: &Metadata) -> bool { false }\n    /// #   fn enter(&self, _: &Id) {}\n    /// #   fn exit(&self, _: &Id) {}\n    /// #   fn current_span(&self) -> Current { Current::unknown() }\n    /// }\n    ///\n    /// static COLLECTOR: MyCollector = MyCollector {\n    ///     // ...\n    /// };\n    ///\n    /// fn main() {\n    ///     use tracing_core::dispatch::{self, Dispatch};\n    ///\n    ///     let dispatch = Dispatch::from_static(&COLLECTOR);\n    ///\n    ///     dispatch::set_global_default(dispatch)\n    ///         .expect(\"no global default collector should have been set previously!\");\n    /// }\n    /// ```\n    ///\n    /// Constructing the collector in a static initializer may make some forms\n    /// of runtime configuration more challenging. If this is the case, users\n    /// with access to `liballoc` or the Rust standard library are encouraged to\n    /// use [`Dispatch::new`] rather than `from_static`. `no_std` users who\n    /// cannot allocate or do not have access to `liballoc` may want to consider\n    /// the [`once_cell`] crate, or another library which allows lazy\n    /// initialization of statics.\n    ///\n    /// [collector]: super::collect::Collect\n    /// [`once_cell`]: https://crates.io/crates/once_cell\n    pub fn from_static(collector: &'static (dyn Collect + Send + Sync)) -> Self {\n        #[cfg(feature = \"alloc\")]\n        let me = Self {\n            collector: Kind::Global(collector),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let me = Self { collector };\n        crate::callsite::register_dispatch(&me);\n        me\n    }\n\n    /// Creates a [`WeakDispatch`] from this `Dispatch`.\n    ///\n    /// A [`WeakDispatch`] is similar to a [`Dispatch`], but it does not prevent\n    /// the underlying [collector] from being dropped. Instead, it only permits\n    /// access while other references to the collector exist. This is equivalent\n    /// to the standard library's [`Arc::downgrade`] method, but for `Dispatch`\n    /// rather than `Arc`.\n    ///\n    /// The primary use for creating a [`WeakDispatch`] is to allow a collector\n    /// to hold a cyclical reference to itself without creating a memory leak.\n    /// See [here] for details.\n    ///\n    /// [collector]: Collect\n    /// [`Arc::downgrade`]: std::sync::Arc::downgrade\n    /// [here]: Collect#avoiding-memory-leaks\n    pub fn downgrade(&self) -> WeakDispatch {\n        #[cfg(feature = \"alloc\")]\n        let collector = match &self.collector {\n            Kind::Global(dispatch) => Kind::Global(*dispatch),\n            Kind::Scoped(dispatch) => Kind::Scoped(Arc::downgrade(dispatch)),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = self.collector;\n\n        WeakDispatch { collector }\n    }\n\n    #[cfg(feature = \"std\")]\n    pub(crate) fn registrar(&self) -> Registrar {\n        Registrar(match self.collector {\n            Kind::Scoped(ref s) => Kind::Scoped(Arc::downgrade(s)),\n            Kind::Global(s) => Kind::Global(s),\n        })\n    }\n\n    #[inline(always)]\n    #[cfg(feature = \"alloc\")]\n    pub(crate) fn collector(&self) -> &(dyn Collect + Send + Sync) {\n        match self.collector {\n            Kind::Scoped(ref s) => Arc::deref(s),\n            Kind::Global(s) => s,\n        }\n    }\n\n    #[inline(always)]\n    #[cfg(not(feature = \"alloc\"))]\n    pub(crate) fn collector(&self) -> &(dyn Collect + Send + Sync) {\n        self.collector\n    }\n\n    /// Registers a new callsite with this collector, returning whether or not\n    /// the collector is interested in being notified about the callsite.\n    ///\n    /// This calls the [`register_callsite`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`register_callsite`]: super::collect::Collect::register_callsite\n    #[inline]\n    pub fn register_callsite(&self, metadata: &'static Metadata<'static>) -> collect::Interest {\n        self.collector().register_callsite(metadata)\n    }\n\n    /// Returns the highest [verbosity level][level] that this [collector] will\n    /// enable, or `None`, if the collector does not implement level-based\n    /// filtering or chooses not to implement this method.\n    ///\n    /// This calls the [`max_level_hint`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [level]: super::Level\n    /// [collector]: super::collect::Collect\n    /// [`Collect`]: super::collect::Collect\n    /// [`register_callsite`]: super::collect::Collect::max_level_hint\n    // TODO(eliza): consider making this a public API?\n    #[inline]\n    pub(crate) fn max_level_hint(&self) -> Option<LevelFilter> {\n        self.collector().max_level_hint()\n    }\n\n    /// Record the construction of a new span, returning a new [ID] for the\n    /// span being constructed.\n    ///\n    /// This calls the [`new_span`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`new_span`]: super::collect::Collect::new_span\n    #[inline]\n    pub fn new_span(&self, span: &span::Attributes<'_>) -> span::Id {\n        self.collector().new_span(span)\n    }\n\n    /// Record a set of values on a span.\n    ///\n    /// This calls the [`record`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`record`]: super::collect::Collect::record\n    #[inline]\n    pub fn record(&self, span: &span::Id, values: &span::Record<'_>) {\n        self.collector().record(span, values)\n    }\n\n    /// Adds an indication that `span` follows from the span with the id\n    /// `follows`.\n    ///\n    /// This calls the [`record_follows_from`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`record_follows_from`]: super::collect::Collect::record_follows_from\n    #[inline]\n    pub fn record_follows_from(&self, span: &span::Id, follows: &span::Id) {\n        self.collector().record_follows_from(span, follows)\n    }\n\n    /// Returns true if a span with the specified [metadata] would be\n    /// recorded.\n    ///\n    /// This calls the [`enabled`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [metadata]: super::metadata::Metadata\n    /// [`Collect`]: super::collect::Collect\n    /// [`enabled`]: super::collect::Collect::enabled\n    #[inline]\n    pub fn enabled(&self, metadata: &Metadata<'_>) -> bool {\n        self.collector().enabled(metadata)\n    }\n\n    /// Records that an [`Event`] has occurred.\n    ///\n    /// This calls the [`event`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Event`]: super::event::Event\n    /// [`Collect`]: super::collect::Collect\n    /// [`event`]: super::collect::Collect::event\n    #[inline]\n    pub fn event(&self, event: &Event<'_>) {\n        let collector = self.collector();\n        if collector.event_enabled(event) {\n            collector.event(event);\n        }\n    }\n\n    /// Records that a span has been can_enter.\n    ///\n    /// This calls the [`enter`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`enter`]: super::collect::Collect::enter\n    pub fn enter(&self, span: &span::Id) {\n        self.collector().enter(span);\n    }\n\n    /// Records that a span has been exited.\n    ///\n    /// This calls the [`exit`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`exit`]: super::collect::Collect::exit\n    pub fn exit(&self, span: &span::Id) {\n        self.collector().exit(span);\n    }\n\n    /// Notifies the [collector] that a [span ID] has been cloned.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`clone_span`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [span ID]: super::span::Id\n    /// [collector]: super::collect::Collect\n    /// [`clone_span`]: super::collect::Collect::clone_span\n    /// [`new_span`]: super::collect::Collect::new_span\n    #[inline]\n    pub fn clone_span(&self, id: &span::Id) -> span::Id {\n        self.collector().clone_span(id)\n    }\n\n    /// Notifies the collector that a [span ID] has been dropped.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`drop_span`] function on the [`Collect`] that this\n    ///  `Dispatch` forwards to.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n    ///\n    /// **Deprecated**: The [`try_close`] method is functionally identical, but returns `true` if the span is now closed.\n    /// It should be used instead of this method.\n    ///\n    /// </pre></div>\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`drop_span`]: super::collect::Collect::drop_span\n    /// [`new_span`]: super::collect::Collect::new_span\n    /// [`try_close`]: Self::try_close\n    #[inline]\n    #[deprecated(since = \"0.1.2\", note = \"use `Dispatch::try_close` instead\")]\n    pub fn drop_span(&self, id: span::Id) {\n        #[allow(deprecated)]\n        self.collector().drop_span(id);\n    }\n\n    /// Notifies the collector that a [span ID] has been dropped, and returns\n    /// `true` if there are now 0 IDs referring to that span.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`try_close`] function on the [`Collect`] trait\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`try_close`]: super::collect::Collect::try_close\n    /// [`new_span`]: super::collect::Collect::new_span\n    pub fn try_close(&self, id: span::Id) -> bool {\n        self.collector().try_close(id)\n    }\n\n    /// Returns a type representing this collector's view of the current span.\n    ///\n    /// This calls the [`current`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`current`]: super::collect::Collect::current_span\n    #[inline]\n    pub fn current_span(&self) -> span::Current {\n        self.collector().current_span()\n    }\n\n    /// Returns `true` if this `Dispatch` forwards to a collector of type\n    /// `T`.\n    #[inline]\n    pub fn is<T: Any>(&self) -> bool {\n        <dyn Collect>::is::<T>(self.collector())\n    }\n\n    /// Returns some reference to the [`Collect`] this `Dispatch` forwards to\n    /// if it is of type `T`, or `None` if it isn't.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    #[inline]\n    pub fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        <dyn Collect>::downcast_ref(self.collector())\n    }\n}\n\nimpl Default for Dispatch {\n    /// Returns the current default dispatcher\n    fn d\nefault() -> Self {\n        get_default(|default| default.clone())\n    }\n}\n\nimpl fmt::Debug for Dispatch {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match &self.collector {\n            #[cfg(feature = \"alloc\")]\n            Kind::Global(collector) => f\n                .debug_tuple(\"Dispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(feature = \"alloc\")]\n            Kind::Scoped(collector) => f\n                .debug_tuple(\"Dispatch::Scoped\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(not(feature = \"alloc\"))]\n            collector => f\n                .debug_tuple(\"Dispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<C> From<C> for Dispatch\nwhere\n    C: Collect + Send + Sync + 'static,\n{\n    #[inline]\n    fn from(collector: C) -> Self {\n        Dispatch::new(collector)\n    }\n}\n\nimpl WeakDispatch {\n    /// Attempts to upgrade this `WeakDispatch` to a [`Dispatch`].\n    ///\n    /// Returns `None` if the referenced `Dispatch` has already been dropped.\n    ///\n    /// ## Examples\n    ///\n    /// ```\n    /// # use tracing_core::collect::NoCollector;\n    /// # use tracing_core::dispatch::Dispatch;\n    /// static COLLECTOR: NoCollector = NoCollector::new();\n    /// let strong = Dispatch::new(COLLECTOR);\n    /// let weak = strong.downgrade();\n    ///\n    /// // The strong here keeps it alive, so we can still access the object.\n    /// assert!(weak.upgrade().is_some());\n    ///\n    /// drop(strong); // But not any more.\n    /// assert!(weak.upgrade().is_none());\n    /// ```\n    pub fn upgrade(&self) -> Option<Dispatch> {\n        #[cfg(feature = \"alloc\")]\n        let collector = match &self.collector {\n            Kind::Global(dispatch) => Some(Kind::Global(*dispatch)),\n            Kind::Scoped(dispatch) => dispatch.upgrade().map(Kind::Scoped),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = Some(self.collector);\n\n        collector.map(|collector| Dispatch { collector })\n    }\n}\n\nimpl fmt::Debug for WeakDispatch {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match &self.collector {\n            #[cfg(feature = \"alloc\")]\n            Kind::Global(collector) => f\n                .debug_tuple(\"WeakDispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(feature = \"alloc\")]\n            Kind::Scoped(collector) => f\n                .debug_tuple(\"WeakDispatch::Scoped\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(not(feature = \"alloc\"))]\n            collector => f\n                .debug_tuple(\"WeakDispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl Registrar {\n    pub(crate) fn upgrade(&self) -> Option<Dispatch> {\n        match self.0 {\n            Kind::Global(s) => Some(Dispatch {\n                collector: Kind::Global(s),\n            }),\n            Kind::Scoped(ref s) => s.upgrade().map(|s| Dispatch {\n                collector: Kind::Scoped(s),\n            }),\n        }\n    }\n}\n\n// ===== impl State =====\n\n#[cfg(feature = \"std\")]\nimpl State {\n    /// Replaces the current default dispatcher on this thread with the provided\n    /// dispatcher.\n    ///\n    /// Dropping the returned `ResetGuard` will reset the default dispatcher to\n    /// the previous value.\n    #[inline]\n    fn set_default(new_dispatch: Dispatch) -> DefaultGuard {\n        let prior = CURRENT_STATE\n            .try_with(|state| {\n                state.can_enter.set(true);\n                state\n                    .default\n                    .replace(Some(new_dispatch))\n                    // if the scoped default was not set on this thread, set the\n                    // `prior` default to the global default to populate the\n                    // scoped default when unsetting *this* default\n                    .unwrap_or_else(|| get_global().clone())\n            })\n            .ok();\n        EXISTS.store(true, Ordering::Release);\n        SCOPED_COUNT.fetch_add(1, Ordering::Release);\n        DefaultGuard(prior)\n    }\n\n    #[inline]\n    fn enter(&self) -> Option<Entered<'_>> {\n        if self.can_enter.replace(false) {\n            Some(Entered(self))\n        } else {\n            None\n        }\n    }\n}\n\n// ===== impl Entered =====\n\n#[cfg(feature = \"std\")]\nimpl<'a> Entered<'a> {\n    #[inline]\n    fn current(&self) -> RefMut<'a, Dispatch> {\n        let default = self.0.default.borrow_mut();\n        RefMut::map(default, |default| {\n            default.get_or_insert_with(|| get_global().clone())\n        })\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<'a> Drop for Entered<'a> {\n    #[inline]\n    fn drop(&mut self) {\n        self.0.can_enter.set(true);\n    }\n}\n\n// ===== impl DefaultGuard =====\n\n#[cfg(feature = \"std\")]\nimpl Drop for DefaultGuard {\n    #[inline]\n    fn drop(&mut self) {\n        SCOPED_COUNT.fetch_sub(1, Ordering::Release);\n        if let Some(dispatch) = self.0.take() {\n            // Replace the dispatcher and then drop the old one outside\n            // of the thread-local context. Dropping the dispatch may\n            // lead to the drop of a collector which, in the process,\n            // could then also attempt to access the same thread local\n            // state -- causing a clash.\n            let prev = CURRENT_STATE.try_with(|state| state.default.replace(Some(dispatch)));\n            drop(prev)\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n\n    use super::*;\n    use crate::{\n        callsite::Callsite,\n        collect::Interest,\n        metadata::{Kind, Level, Metadata},\n    };\n\n    #[test]\n    fn dispatch_is() {\n        let dispatcher = Dispatch::from_static(&NO_COLLECTOR);\n        assert!(dispatcher.is::<NoCollector>());\n    }\n\n    #[test]\n    fn dispatch_downcasts() {\n        let dispatcher = Dispatch::from_static(&NO_COLLECTOR);\n        assert!(dispatcher.downcast_ref::<NoCollector>().is_some());\n    }\n\n    struct TestCallsite;\n    static TEST_CALLSITE: TestCallsite = TestCallsite;\n    static TEST_META: Metadata<'static> = metadata! {\n        name: \"test\",\n        target: module_path!(),\n        level: Level::DEBUG,\n        fields: &[],\n        callsite: &TEST_CALLSITE,\n        kind: Kind::EVENT\n    };\n\n    impl Callsite for TestCallsite {\n        fn set_interest(&self, _: Interest) {}\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META\n        }\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn events_dont_infinite_loop() {\n        // This test ensures that an event triggered within a collector\n        // won't cause an infinite loop of events.\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {\n                static EVENTS: AtomicUsize = AtomicUsize::new(0);\n                assert_eq!(\n                    EVENTS.fetch_add(1, Ordering::Relaxed),\n                    0,\n                    \"event method called twice!\"\n                );\n                Event::dispatch(&TEST_META, &TEST_META.fields().value_set(&[]))\n            }\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n\n        with_default(&Dispatch::new(TestCollector), || {\n            Event::dispatch(&TEST_META, &TEST_META.fields().value_set(&[]))\n        })\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn spans_dont_infinite_loop() {\n        // This test ensures that a span created within a collector\n        // won't cause an infinite loop of new spans.\n\n        fn mk_span() {\n            get_default(|current| {\n                current.new_span(&span::Attributes::new(\n                    &TEST_META,\n                    &TEST_META.fields().value_set(&[]),\n                ))\n            });\n        }\n\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                static NEW_SPANS: AtomicUsize = AtomicUsize::new(0);\n                assert_eq!(\n                    NEW_SPANS.fetch_add(1, Ordering::Relaxed),\n                    0,\n                    \"new_span method called twice!\"\n                );\n                mk_span();\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {}\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n\n        with_default(&Dispatch::new(TestCollector), mk_span)\n    }\n\n    #[test]\n    fn default_no_collector() {\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<NoCollector>());\n    }\n\n    #[cfg(feature = \"std\")]\n    #[test]\n    fn default_dispatch() {\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {}\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n        let guard = set_default(&Dispatch::new(TestCollector));\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<TestCollector>());\n\n        drop(guard);\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<NoCollector>());\n    }\n}\n\n// Path: tracing-core/src/collect.rs\n//! Collectors collect and record trace data.\nuse crate::{span, Dispatch, Event, LevelFilter, Metadata};\n\nuse core::any::{Any, TypeId};\nuse core::ptr::NonNull;\n\n/// Trait representing the functions required to collect trace data.\n///\n/// Crates that provide implementations of methods for collecting or recording\n/// trace data should implement the `Collect` interface. This trait is\n/// intended to represent fundamental primitives for collecting trace events and\n/// spans \u2014 other libraries may offer utility functions and types to make\n/// collector implementations more modular or improve the ergonomics of writing\n/// collectors.\n///\n/// A collector is responsible for the following:\n/// - Registering new spans as they are created, and providing them with span\n///   IDs. Implicitly, this means the collector may determine the strategy for\n///   determining span equality.\n/// - Recording the attachment of field values and follows-from annotations to\n///   spans.\n/// - Filtering spans and events, and determining when those filters must be\n///   invalidated.\n/// - Observing spans as they are entered, exited, and closed, and events as\n///   they occur.\n///\n/// When a span is entered or exited, the collector is provided only with the\n/// [ID] with which it tagged that span when it was created. This means\n/// that it is up to the collector to determine whether and how span _data_ \u2014\n/// the fields and metadata describing the span \u2014 should be stored. The\n/// [`new_span`] function is called when a new span is created, and at that\n/// point, the collector _may_ choose to store the associated data if it will\n/// be referenced again. However, if the data has already been recorded and will\n/// not be needed by the implementations of `enter` and `exit`, the collector\n/// may freely discard that data without allocating space to store it.\n///\n/// ## Overriding default impls\n///\n/// Some trait methods on `Collect` have default implementations, either in\n/// order to reduce the surface area of implementing `Collect`, or for\n/// backward-compatibility reasons. However, many collectors will likely want\n/// to override these default implementations.\n///\n/// The following methods are likely of interest:\n///\n/// - [`register_callsite`] is called once for each callsite from which a span\n///   event may originate, and returns an [`Interest`] value describing whether or\n///   not the collector wishes to see events or spans from that callsite. By\n///   default, it calls [`enabled`], and returns `Interest::always()` if\n///   `enabled` returns true, or `Interest::never()` if enabled returns false.\n///   However, if the collector's interest can change dynamically at runtime,\n///   it may want to override this function to return `Interest::sometimes()`.\n///   Additionally, collectors which wish to perform a behaviour once for each\n///   callsite, such as allocating storage for data related to that callsite,\n///   can perform it in `register_callsite`.\n///\n///   See also the [documentation on the callsite registry][cs-reg] for details\n///   on [`register_callsite`].\n///\n/// - [`event_enabled`] is called once before every call to the [`event`]\n///   method. This can be used to implement filtering on events once their field\n///   values are known, but before any processing is done in the `event` method.\n/// - [`clone_span`] is called every time a span ID is cloned, and [`try_close`]\n///   is called when a span ID is dropped. By default, these functions do\n///   nothing. However, they can be used to implement reference counting for\n///   spans, allowing collectors to free storage for span data and to determine\n///   when a span has _closed_ permanently (rather than being exited).\n///   Collectors which store per-span data or which need to track span closures\n///   should override these functions together.\n///\n/// [ID]: super::span::Id\n/// [`new_span`]: Collect::new_span\n/// [`register_callsite`]: Collect::register_callsite\n/// [`enabled`]: Collect::enabled\n/// [`clone_span`]: Collect::clone_span\n/// [`try_close`]: Collect::try_close\n/// [cs-reg]: crate::callsite#registering-callsites\n/// [`event`]: Collect::event\n/// [`event_enabled`]: Collect::event_enabled\npub trait Collect: 'static {\n    /// Invoked when this collector becomes a [`Dispatch`].\n    ///\n    /// ## Avoiding Memory Leaks\n    ///\n    /// Collectors should not store their own [`Dispatch`]. Because the\n    /// `Dispatch` owns the collector, storing the `Dispatch` within the\n    /// collector will create a reference count cycle, preventing the `Dispatch`\n    /// from ever being dropped.\n    ///\n    /// Instead, when it is necessary to store a cyclical reference to the\n    /// `Dispatch` within a collector, use [`Dispatch::downgrade`] to convert a\n    /// `Dispatch` into a [`WeakDispatch`]. This type is analogous to\n    /// [`std::sync::Weak`], and does not create a reference count cycle. A\n    /// [`WeakDispatch`] can be stored within a collector without causing a\n    /// memory leak, and can be [upgraded] into a `Dispatch` temporarily when\n    /// the `Dispatch` must be accessed by the collector.\n    ///\n    /// [`WeakDispatch`]: crate::dispatch::WeakDispatch\n    /// [upgraded]: crate::dispatch::WeakDispatch::upgrade\n    fn on_register_dispatch(&self, collector: &Dispatch) {\n        let _ = collector;\n    }\n\n    // === Span registry methods ==============================================\n\n    /// Registers a new [callsite] with this collector, returning whether or not\n    /// the collector is interested in being notified about the callsite.\n    ///\n    /// By default, this function assumes that the collector's [filter]\n    /// represents an unchanging view of its interest in the callsite. However,\n    /// if this is not the case, collectors may override this function to\n    /// indicate different interests, or to implement behaviour that should run\n    /// once for every callsite.\n    ///\n    /// This function is guaranteed to be called at least once per callsite on\n    /// every active collector. The collector may store the keys to fields it\n    /// cares about in order to reduce the cost of accessing fields by name,\n    /// preallocate storage for that callsite, or perform any other actions it\n    /// wishes to perform once for each callsite.\n    ///\n    /// The collector should then return an [`Interest`], indicating\n    /// whether it is interested in being notified about that callsite in the\n    /// future. This may be `Always` indicating that the collector always\n    /// wishes to be notified about the callsite, and its filter need not be\n    /// re-evaluated; `Sometimes`, indicating that the collector may sometimes\n    /// care about the callsite but not always (such as when sampling), or\n    /// `Never`, indicating that the collector never wishes to be notified about\n    /// that callsite. If all active collectors return `Never`, a callsite will\n    /// never be enabled unless a new collector expresses interest in it.\n    ///\n    /// `Collector`s which require their filters to be run every time an event\n    /// occurs or a span is entered/exited should return `Interest::sometimes`.\n    /// If a collector returns `Interest::sometimes`, then its' [`enabled`] method\n    /// will be called every time an event or span is created from that callsite.\n    ///\n    /// For example, suppose a sampling collector is implemented by\n    /// incrementing a counter every time `enabled` is called and only returning\n    /// `true` when the counter is divisible by a specified sampling rate. If\n    /// that collector returns `Interest::always` from `register_callsite`, then\n    /// the filter will not be re-evaluated once it has been applied to a given\n    /// set of metadata. Thus, the counter will not be incremented, and the span\n    /// or event that corresponds to the metadata will never be `enabled`.\n    ///\n    /// `Collector`s that need to change their filters occasionally should call\n    /// [`rebuild_interest_cache`] to re-evaluate `register_callsite` for all\n    /// callsites.\n    ///\n    /// Similarly, if a `Collector` has a filtering strategy that can be\n    /// changed dynamically at runtime, it would need to re-evaluate that filter\n    /// if the cached results have changed.\n    ///\n    /// A collector which manages fanout to multiple other collectors\n    /// should proxy this decision to all of its child collectors,\n    /// returning `Interest::never` only if _all_ such children return\n    /// `Interest::never`. If the set of collectors to which spans are\n    /// broadcast may change dynamically, the collector should also never\n    /// return `Interest::Never`, as a new collector may be added that _is_\n    /// interested.\n    ///\n    /// See the [documentation on the callsite registry][cs-reg] for more\n    /// details on how and when the `register_callsite` method is called.\n    ///\n    /// # Notes\n    ///\n    /// This function may be called again when a new collector is created or\n    /// when the registry is invalidated.\n    ///\n    /// If a collector returns `Interest::never` for a particular callsite, it\n    /// _may_ still see spans and events originating from that callsite, if\n    /// another collector expressed interest in it.\n    ///\n    /// [callsite]: crate::callsite\n    /// [filter]: Self::enabled\n    /// [metadata]: super::metadata::Metadata\n    /// [`enabled`]: Self::enabled\n    /// [`rebuild_interest_cache`]: super::callsite::rebuild_interest_cache\n    /// [cs-reg]: crate::callsite#registering-callsites\n    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {\n        if self.enabled(metadata) {\n            Interest::always()\n        } else {\n            Interest::never()\n        }\n    }\n\n    /// Returns true if a span or event with the specified [metadata] would be\n    /// recorded.\n    ///\n    /// By default, it is assumed that this filter needs only be evaluated once\n    /// for each callsite, so it is called by [`register_callsite`] when each\n    /// callsite is registered. The result is used to determine if the collector\n    /// is always [interested] or never interested in that callsite. This is intended\n    /// primarily as an optimization, so that expensive filters (such as those\n    /// involving string search, et cetera) need not be re-evaluated.\n    ///\n    /// However, if the collector's interest in a particular span or event may\n    /// change, or depends on contexts only determined dynamically at runtime,\n    /// then the `register_callsite` method should be overridden to return\n    /// [`Interest::sometimes`]. In that case, this function will be called every\n    /// time that span or event occurs.\n    ///\n    /// [metadata]: super::metadata::Metadata\n    /// [interested]: Interest\n    /// [`register_callsite`]: Self::register_callsite\n    fn enabled(&self, metadata: &Metadata<'_>) -> bool;\n\n    /// Returns the highest [verbosity level][level] that this `Collector` will\n    /// enable, or `None`, if the collector does not implement level-based\n    /// filtering or chooses not to implement this method.\n    ///\n    /// If this method returns a [`Level`][level], it will be used as a hint to\n    /// determine the most verbose level that will be enabled. This will allow\n    /// spans and events which are more verbose than that level to be skipped\n    /// more efficiently. collectors which perform filtering are strongly\n    /// encouraged to provide an implementation of this method.\n    ///\n    /// If the maximum level the collector will enable can change over the\n    /// course of its lifetime, it is free to return a different value from\n    /// multiple invocations of this method. However, note that changes in the\n    /// maximum level will **only** be reflected after the callsite [`Interest`]\n    /// cache is rebuilt, by calling the [`callsite::rebuild_interest_cache`][rebuild]\n    /// function. Therefore, if the collector will change the value returned by\n    /// this method, it is responsible for ensuring that\n    /// [`rebuild_interest_cache`][rebuild] is called after the value of the max\n    /// level changes.\n    ///\n    /// [level]: super::Level\n    /// [rebuild]: super::callsite::rebuild_interest_cache\n    fn max_level_hint(&self) -> Option<LevelFilter> {\n        None\n    }\n\n    /// Visit the construction of a new span, returning a new [span ID] for the\n    /// span being constructed.\n    ///\n    /// The provided [`Attributes`] contains any field values that were provided\n    /// when the span was created. The collector may pass a [visitor] to the\n    /// `Attributes`' [`record` method] to record these values.\n    ///\n    /// IDs are used to uniquely identify spans and events within the context of a\n    /// collector, so span equality will be based on the returned ID. Thus, if\n    /// the collector wishes for all spans with the same metadata to be\n    /// considered equal, it should return the same ID every time it is given a\n    /// particular set of metadata. Similarly, if it wishes for two separate\n    /// instances of a span with the same metadata to *not* be equal, it should\n    /// return a distinct ID every time this function is called, regardless of\n    /// the metadata.\n    ///\n    /// Note that the collector is free to assign span IDs based on whatever\n    /// scheme it sees fit. Any guarantees about uniqueness, ordering, or ID\n    /// reuse are left up to the collector implementation to determine.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Attributes`]: super::span::Attributes\n    /// [visitor]: super::field::Visit\n    /// [`record` method]: super::span::Attributes::record\n    fn new_span(&self, span: &span::Attributes<'_>) -> span::Id;\n\n    // === Notification methods ===============================================\n\n    /// Record a set of values on a span.\n    ///\n    /// This method will be invoked when value is recorded on a span.\n    /// Recording multiple values for the same field is possible,\n    /// but the actual behaviour is defined by the collector implementation.\n    ///\n    /// Keep in mind that a span might not provide a value\n    /// for each field it declares.\n    ///\n    /// The collector is expected to provide a [visitor] to the `Record`'s\n    /// [`record` method] in order to record the added values.\n    ///\n    /// # Example\n    ///  \"foo = 3\" will be recorded when [`record`] is called on the\n    /// `Attributes` passed to `new_span`.\n    /// Since values are not provided for the `bar` and `baz` fields,\n    /// the span's `Metadata` will indicate that it _has_ those fields,\n    /// but values for them won't be recorded at this time.\n    ///\n    /// ```rust,ignore\n    /// # use tracing::span;\n    ///\n    /// let mut span = span!(\"my_span\", foo = 3, bar, baz);\n    ///\n    /// // `Collector::record` will be called with a `Record`\n    /// // containing \"bar = false\"\n    /// span.record(\"bar\", &false);\n    ///\n    /// // `Collector::record` will be called with a `Record`\n    /// // containing \"baz = \"a string\"\"\n    /// span.record(\"baz\", &\"a string\");\n    /// ```\n    ///\n    /// [visitor]: super::field::Visit\n    /// [`record`]: super::span::Attributes::record\n    /// [`record` method]: super::span::Record::record\n    fn record(&self, span: &span::Id, values: &span::Record<'_>);\n\n    /// Adds an indication that `span` follows from the span with the id\n    /// `follows`.\n    ///\n    /// This relationship differs somewhat from the parent-child relationship: a\n    /// span may have any number of prior spans, rather than a single one; and\n    /// spans are not considered to be executing _inside_ of the spans they\n    /// follow from. This means that a span may close even if subsequent spans\n    /// that follow from it are still open, and time spent inside of a\n    /// subsequent span should not be included in the time its precedents were\n    /// executing. This is used to model causal relationships such as when a\n    /// single future spawns several related background tasks, et cetera.\n    ///\n    /// If the collector has spans corresponding to the given IDs, it should\n    /// record this relationship in whatever way it deems necessary. Otherwise,\n    /// if one or both of the given span IDs do not correspond to spans that the\n    /// collector knows about, or if a cyclical relationship would be created\n    /// (i.e., some span _a_ which proceeds some other span _b_ may not also\n    /// follow from _b_), it may silently do nothing.\n    fn record_follows_from(&self, span: &span::Id, follows: &span::Id);\n\n    /// Determine if an [`Event`] should be recorded.\n    ///\n    /// By default, this returns `true` and collectors can filter events in\n    /// [`event`][Self::event] without any penalty. However, when `event` is\n    /// more complicated, this can be used to determine if `event` should be\n    /// called at all, separating out the decision from the processing.\n    fn event_enabled(&self, event: &Event<'_>) -> bool {\n        let _ = event;\n        true\n    }\n\n    /// Records that an [`Event`] has occurred.\n    ///\n    /// This method will be invoked when an Event is constructed by\n    /// the `Event`'s [`dispatch` method]. For example, this happens internally\n    /// when an event macro from `tracing` is called.\n    ///\n    /// The key difference between this method and `record` is that `record` is\n    /// called when a value is recorded for a field defined by a span,\n    /// while `event` is called when a new event occurs.\n    ///\n    /// The provided `Event` struct contains any field values attached to the\n    /// event. The collector may pass a [visitor] to the `Event`'s\n    /// [`record` method] to record these values.\n    ///\n    /// [`Event`]: super::event::Event\n    /// [visitor]: super::field::Visit\n    /// [`record` method]: super::event::Event::record\n    /// [`dispatch` method]: super::event::Event::dispatch\n    fn event(&self, event: &Event<'_>);\n\n    /// Records that a span has been entered.\n    ///\n    /// When entering a span, this method is called to notify the collector\n    /// that the span has been entered. The collector is provided with the\n    /// [span ID] of the entered span, and should update any internal state\n    /// tracking the current span accordingly.\n    ///\n    /// [span ID]: super::span::Id\n    fn enter(&self, span: &span::Id);\n\n    /// Records that a span has been exited.\n    ///\n    /// When exiting a span, this method is called to notify the collector\n    /// that the span has been exited. The collector is provided with the\n    /// [span ID] of the exited span, and should update any internal state\n    /// tracking the current span accordingly.\n    ///\n    /// Exiting a span does not imply that the span will not be re-entered.\n    ///\n    /// [span ID]: super::span::Id\n    fn exit(&self, span: &span::Id);\n\n    /// Notifies the collector that a [span ID] has been cloned.\n    ///\n    /// This function is guaranteed to only be called with span IDs that were\n    /// returned by this collector's `new_span` function.\n    ///\n    /// Note that the default implementation of this function this is just the\n    /// identity function, passing through the identifier. However, it can be\n    /// used in conjunction with [`try_close`] to track the number of handles\n    /// capable of `enter`ing a span. When all the handles have been dropped\n    /// (i.e., `try_close` has been called one more time than `clone_span` for a\n    /// given ID), the collector may assume that the span will not be entered\n    /// again. It is then free to deallocate storage for data associated with\n    /// that span, write data from that span to IO, and so on.\n    ///\n    /// For more unsafe situations, however, if `id` is itself a pointer of some\n    /// kind this can be used as a hook to \"clone\" the pointer, depending on\n    /// what that means for the specified pointer.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`try_close`]: Collect::try_close\n    fn clone_span(&self, id: &span::Id) -> span::Id {\n        id.clone()\n    }\n\n    /// **This method is deprecated.**\n    ///\n    /// Using `drop_span` may result in collectors composed using\n    /// `tracing-subscriber` crate's `Subscriber` trait from observing close events.\n    /// Use [`try_close`] instead.\n    ///\n    /// The default implementation of this function does nothing.\n    ///\n    /// [`try_close`]: Collect::try_close\n    #[deprecated(since = \"0.1.2\", note = \"use `Collector::try_close` instead\")]\n    fn drop_span(&self, _id: span::Id) {}\n\n    /// Notifies the collector that a [`span ID`] has been dropped, and returns\n    /// `true` if there are now 0 IDs that refer to that span.\n    ///\n    /// Higher-level libraries providing functionality for composing multiple\n    /// collector implementations may use this return value to notify any\n    /// \"layered\" collectors that this collector considers the span closed.\n    ///\n    /// The default implementation of this method calls the collector's\n    /// [`drop_span`] method and returns `false`. This means that, unless the\n    /// collector overrides the default implementation, close notifications\n    /// will never be sent to any layered collectors. In general, if the\n    /// collector tracks reference counts, this method should be implemented,\n    /// rather than `drop_span`.\n    ///\n    /// This function is guaranteed to only be called with span IDs that were\n    /// returned by this collector's `new_span` function.\n    ///\n    /// It's guaranteed that if this function has been called once more than the\n    /// number of times `clone_span` was called with the same `id`, then no more\n    /// handles that can enter the span with that `id` exist. This means that it\n    /// can be used in conjunction with [`clone_span`] to track the number of\n    /// handles capable of `enter`ing a span. When all the handles have been\n    /// dropped (i.e., `try_close` has been called one more time than\n    /// `clone_span` for a given ID), the collector may assume that the span\n    /// will not be entered again, and should return `true`. It is then free to\n    /// deallocate storage for data associated with that span, write data from\n    /// that span to IO, and so on.\n    ///\n    /// **Note**: since this function is called when spans are dropped,\n    /// implementations should ensure that they are unwind-safe. Panicking from\n    /// inside of a `try_close` function may cause a double panic, if the span\n    /// was dropped due to a thread unwinding.\n    ///\n    /// [`span ID`]: super::span::Id\n    /// [`clone_span`]: Collect::clone_span\n    /// [`drop_span`]: Collect::drop_span\n    fn try_close(&self, id: span::Id) -> bool {\n        #[allow(deprecated)]\n        self.drop_span(id);\n        false\n    }\n\n    /// Returns a type representing this collector's view of the current span.\n    ///\n    /// If collectors track a current span, they should return [`Current::new`]\n    /// if the thread from which this method is called is inside a span,\n    /// or [`Current::none`] if the thread is not inside a span.\n    ///\n    /// [`Current::new`]: super::span::Current::new\n    /// [`Current::none`]: super::span::Current::none\n    fn current_span(&self) -> span::Current;\n\n    // === Downcasting methods ================================================\n\n    /// If `self` is the same type as the provided `TypeId`, returns an untyped\n    /// [`NonNull`] pointer to that type. Otherwise, returns `None`.\n    ///\n    /// If you wish to downcast a `Collector`, it is strongly advised to use\n    /// the safe API provided by [`downcast_ref`] instead.\n    ///\n    /// This API is required for `downcast_raw` to be a trait method; a method\n    /// signature like [`downcast_ref`] (with a generic type parameter) is not\n    /// object-safe, and thus cannot be a trait method for `Collector`. This\n    /// means that if we only exposed `downcast_ref`, `Collector`\n    /// implementations could not override the downcasting behavior\n    ///\n    /// This method may be overridden by \"fan out\" or \"chained\" collector\n    /// implementations which consist of multiple composed types. Such\n    /// collectors might allow `downcast_raw` by returning references to those\n    /// component if they contain components with the given `TypeId`.\n    ///\n    /// # Safety\n    ///\n    /// The [`downcast_ref`] method expects that the pointer returned by\n    /// `downcast_raw` points to a valid instance of the type\n    /// with the provided `TypeId`. Failure to ensure this will result in\n    /// undefined behaviour, so implementing `downcast_raw` is unsafe.\n    ///\n    /// [`downcast_ref`]: #method.downcast_ref\n    /// [`NonNull`]: core::ptr::NonNull\n    unsafe fn downcast_raw(&self, id: TypeId) -> Option<NonNull<()>> {\n        if id == TypeId::of::<Self>() {\n            Some(NonNull::from(self).cast())\n        } else {\n            None\n        }\n    }\n}\n\nimpl dyn Collect {\n    /// Returns `true` if this `Collector` is the same type as `T`.\n    pub fn is<T: Any>(&self) -> bool {\n        self.downcast_ref::<T>().is_some()\n    }\n\n    /// Returns some reference to this `Collector` value if it is of type `T`,\n    /// or `None` if it isn't.\n    pub fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        unsafe {\n            let raw = self.downcast_raw(TypeId::of::<T>())?;\n            Some(&*(raw.cast().as_ptr()))\n        }\n    }\n}\n\nimpl dyn Collect + Send {\n    /// Returns `true` if this `Collector` is the same type as `T`.\n    pub fn is<T: Any>(&self) -> bool {\n        self.downcast_ref::<T>().is_some()\n    }\n\n    /// Returns some reference to this `Collector` value if it is of type `T`,\n    /// or `None` if it isn't.\n    pub fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        unsafe {\n            let raw = self.downcast_raw(TypeId::of::<T>())?;\n            Some(&*(raw.cast().as_ptr()))\n        }\n    }\n}\n\nimpl dyn Collect + Sync {\n    /// Returns `true` if this `Collector` is the same type as `T`.\n    pub fn is<T: Any>(&self) -> bool {\n        self.downcast_ref::<T>().is_some()\n    }\n\n    /// Returns some reference to this `Collector` value if it is of type `T`,\n    /// or `None` if it isn't.\n    pub fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        unsafe {\n            let raw = self.downcast_raw(TypeId::of::<T>())?;\n            Some(&*(raw.cast().as_ptr()))\n        }\n    }\n}\n\nimpl dyn Collect + Send + Sync {\n    /// Returns `true` if this `Collector` is the same type as `T`.\n    pub fn is<T: Any>(&self) -> bool {\n        self.downcast_ref::<T>().is_some()\n    }\n\n    /// Returns some reference to this `Collector` value if it is of type `T`,\n    /// or `None` if it isn't.\n    pub fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        unsafe {\n            let raw = self.downcast_raw(TypeId::of::<T>())?;\n            Some(&*(raw.cast().as_ptr()))\n        }\n    }\n}\n\n/// Indicates a [`Collect`]'s interest in a particular callsite.\n///\n/// Collectors return an `Interest` from their [`register_callsite`] methods\n/// in order to determine whether that span should be enabled or disabled.\n///\n/// [`Collect`]: super::Collect\n/// [`register_callsite`]: super::Collect::register_callsite\n#[derive(Clone, Debug)]\npub struct Interest(InterestKind);\n\n#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]\nenum InterestKind {\n    Never = 0,\n    Sometimes = 1,\n    Always = 2,\n}\n\nimpl Interest {\n    /// Returns an `Interest` indicating that the collector is never interested\n    /// in being notified about a callsite.\n    ///\n    /// If all active collectors are `never()` interested in a callsite, it will\n...\n", "needle_token_start": 5692, "needle_token_end": 5717, "code_context_ntokens": 16272, "output": ["```rust\n#[inline(always)]\npub(crate) fn get_global() -> &'static Dispatch {\n    if GLOBAL_INIT.load(Ordering::Acquire) != INITIALIZED {\n        return &NONE;\n    }\n    unsafe {\n        // This is safe given the invariant that setting the global dispatcher\n        // also sets `GLOBAL_INIT` to `INITIALIZED`.\n        &GLOBAL_DISPATCH\n    }\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "field", "language": "rust", "path": "tracing-core/src/field.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To retrieve a specific descriptor from a collection of descriptors based on its name, if it exists within the set.\n2. **Input**: A string reference representing the name of the descriptor sought.\n3. **Output**: An optional descriptor object that matches the input name, or `None` if no matching descriptor is found.\n4. **Procedure**: The function searches through an array of descriptor names within a set. If a name matches the input string, it constructs and returns a descriptor object with the corresponding index and set information. If no match is found, it returns `None`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "//! The set of field keys on a given span or event is defined on its [`Metadata`].\n//! When a span is created, it provides [`Attributes`] to the collector's\n//! [`new_span`] method, containing any fields whose values were provided when\n//! the span was created; and may call the collector's [`record`] method\n//! with additional [`Record`]s if values are added for more of its fields.\n//! Similarly, the [`Event`] type passed to the collector's [`event`] method\n//! will contain any fields attached to each event.\n//!\n//! `tracing` represents values as either one of a set of Rust primitives\n//! (`i64`, `u64`, `f64`, `i128`, `u128`, `bool`, and `&str`) or using a\n//! `fmt::Display` or `fmt::Debug` implementation. Collectors are provided\n//! these primitive value types as `dyn Value` trait objects.\n//!\n//! These trait objects can be formatted using `fmt::Debug`, but may also be\n//! recorded as typed data by calling the [`Value::record`] method on these\n//! trait objects with a _visitor_ implementing the [`Visit`] trait. This trait\n//! represents the behavior used to record values of various types. For example,\n//! we might record integers by incrementing counters for their field names,\n//! rather than printing them.\n//!\n//! [span]: super::span\n//! [`Event`]: super::event::Event\n//! [`Metadata`]: super::metadata::Metadata\n//! [`Attributes`]:  super::span::Attributes\n//! [`Record`]: super::span::Record\n//! [`new_span`]: super::collect::Collect::new_span\n//! [`record`]: super::collect::Collect::record\n//! [`event`]:  super::collect::Collect::event\nuse crate::callsite;\nuse core::{\n    borrow::Borrow,\n    fmt,\n    hash::{Hash, Hasher},\n    num,\n    ops::Range,\n};\n\nuse self::private::ValidLen;\n\n/// An opaque key allowing _O_(1) access to a field in a `Span`'s key-value\n/// data.\n///\n/// As keys are defined by the _metadata_ of a span, rather than by an\n/// individual instance of a span, a key may be used to access the same field\n/// across all instances of a given span with the same metadata. Thus, when a\n/// collector observes a new span, it need only access a field by name _once_,\n/// and use the key for that name for all other accesses.\n#[derive(Debug)]\npub struct Field {\n    i: usize,\n    fields: FieldSet,\n}\n\n/// An empty field.\n///\n/// This can be used to indicate that the value of a field is not currently\n/// present but will be recorded later.\n///\n/// When a field's value is `Empty`. it will not be recorded.\n#[derive(Debug, Eq, PartialEq)]\npub struct Empty;\n\n/// Describes the fields present on a span.\n///\n/// ## Equality\n///\n/// In well-behaved applications, two `FieldSet`s [initialized] with equal\n/// [callsite identifiers] will have identical fields. Consequently, in release\n/// builds, [`FieldSet::eq`] *only* checks that its arguments have equal\n/// callsites. However, the equality of field names is checked in debug builds.\n///\n/// [initialized]: Self::new\n/// [callsite identifiers]: callsite::Identifier\npub struct FieldSet {\n    /// The names of each field on the described span.\n    names: &'static [&'static str],\n    /// The callsite where the described span originates.\n    callsite: callsite::Identifier,\n}\n\n/// A set of fields and values for a span.\npub struct ValueSet<'a> {\n    values: &'a [(&'a Field, Option<&'a (dyn Value + 'a)>)],\n    fields: &'a FieldSet,\n}\n\n/// An iterator over a set of fields.\n#[derive(Debug)]\npub struct Iter {\n    idxs: Range<usize>,\n    fields: FieldSet,\n}\n\n/// Visits typed values.\n///\n/// An instance of `Visit` (\"a visitor\") represents the logic necessary to\n/// record field values of various types. When an implementor of [`Value`] is\n/// [recorded], it calls the appropriate method on the provided visitor to\n/// indicate the type that value should be recorded as.\n///\n/// When a [`Collect`] implementation [records an `Event`] or a\n/// [set of `Value`s added to a `Span`], it can pass an `&mut Visit` to the\n/// `record` method on the provided [`ValueSet`] or [`Event`]. This visitor\n/// will then be used to record all the field-value pairs present on that\n/// `Event` or `ValueSet`.\n///\n/// # Examples\n///\n/// A simple visitor that writes to a string might be implemented like so:\n/// ```\n/// # extern crate tracing_core as tracing;\n/// use std::fmt::{self, Write};\n/// use tracing::field::{Value, Visit, Field};\n/// pub struct StringVisitor<'a> {\n///     string: &'a mut String,\n/// }\n///\n/// impl<'a> Visit for StringVisitor<'a> {\n///     fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n///         write!(self.string, \"{} = {:?}; \", field.name(), value).unwrap();\n///     }\n/// }\n/// ```\n/// This visitor will format each recorded value using `fmt::Debug`, and\n/// append the field name and formatted value to the provided string,\n/// regardless of the type of the recorded value. When all the values have\n/// been recorded, the `StringVisitor` may be dropped, allowing the string\n/// to be printed or stored in some other data structure.\n///\n/// The `Visit` trait provides default implementations for `record_i64`,\n/// `record_u64`, `record_bool`, `record_str`, and `record_error`, which simply\n/// forward the recorded value to `record_debug`. Thus, `record_debug` is the\n/// only method which a `Visit` implementation *must* implement. However,\n/// visitors may override the default implementations of these functions in\n/// order to implement type-specific behavior.\n///\n/// Additionally, when a visitor receives a value of a type it does not care\n/// about, it is free to ignore those values completely. For example, a\n/// visitor which only records numeric data might look like this:\n///\n/// ```\n/// # extern crate tracing_core as tracing;\n/// # use std::fmt::{self, Write};\n/// # use tracing::field::{Value, Visit, Field};\n/// pub struct SumVisitor {\n///     sum: i64,\n/// }\n///\n/// impl Visit for SumVisitor {\n///     fn record_i64(&mut self, _field: &Field, value: i64) {\n///        self.sum += value;\n///     }\n///\n///     fn record_u64(&mut self, _field: &Field, value: u64) {\n///         self.sum += value as i64;\n///     }\n///\n///     fn record_debug(&mut self, _field: &Field, _value: &dyn fmt::Debug) {\n///         // Do nothing\n///     }\n/// }\n/// ```\n///\n/// This visitor (which is probably not particularly useful) keeps a running\n/// sum of all the numeric values it records, and ignores all other values. A\n/// more practical example of recording typed values is presented in\n/// `examples/counters.rs`, which demonstrates a very simple metrics system\n/// implemented using `tracing`.\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: The <code>record_error</code> trait method is only\n/// available when the Rust standard library is present, as it requires the\n/// <code>std::error::Error</code> trait.\n/// </pre></div>\n///\n/// [recorded]: Value::record\n/// [`Collect`]: super::collect::Collect\n/// [records an `Event`]: super::collect::Collect::event\n/// [set of `Value`s added to a `Span`]: super::collect::Collect::record\n/// [`Event`]: super::event::Event\npub trait Visit {\n    /// Visit a double-precision floating point value.\n    fn record_f64(&mut self, field: &Field, value: f64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 64-bit integer value.\n    fn record_i64(&mut self, field: &Field, value: i64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 64-bit integer value.\n    fn record_u64(&mut self, field: &Field, value: u64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 128-bit integer value.\n    fn record_i128(&mut self, field: &Field, value: i128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 128-bit integer value.\n    fn record_u128(&mut self, field: &Field, value: u128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a boolean value.\n    fn record_bool(&mut self, field: &Field, value: bool) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a string value.\n    fn record_str(&mut self, field: &Field, value: &str) {\n        self.record_debug(field, &value)\n    }\n\n    /// Records a type implementing `Error`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: This is only enabled when the Rust standard library is\n    /// present.\n    /// </pre>\n    /// </div>\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n    fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) {\n        self.record_debug(field, &DisplayValue(value))\n    }\n\n    /// Visit a value implementing `fmt::Debug`.\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug);\n}\n\n/// A field value of an erased type.\n///\n/// Implementors of `Value` may call the appropriate typed recording methods on\n/// the [visitor] passed to their `record` method in order to indicate how\n/// their data should be recorded.\n///\n/// [visitor]: Visit\npub trait Value: crate::sealed::Sealed {\n    /// Visits this value with the given `Visitor`.\n    fn record(&self, key: &Field, visitor: &mut dyn Visit);\n}\n\n/// A `Value` which serializes using `fmt::Display`.\n///\n/// Uses `record_debug` in the `Value` implementation to\n/// avoid an unnecessary evaluation.\n#[derive(Clone)]\npub struct DisplayValue<T: fmt::Display>(T);\n\n/// A `Value` which serializes as a string using `fmt::Debug`.\n#[derive(Clone)]\npub struct DebugValue<T: fmt::Debug>(T);\n\n/// Wraps a type implementing `fmt::Display` as a `Value` that can be\n/// recorded using its `Display` implementation.\npub fn display<T>(t: T) -> DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    DisplayValue(t)\n}\n\n/// Wraps a type implementing `fmt::Debug` as a `Value` that can be\n/// recorded using its `Debug` implementation.\npub fn debug<T>(t: T) -> DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    DebugValue(t)\n}\n\n// ===== impl Visit =====\n\nimpl<'a, 'b> Visit for fmt::DebugStruct<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.field(field.name(), value);\n    }\n}\n\nimpl<'a, 'b> Visit for fmt::DebugMap<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.entry(&format_args!(\"{}\", field), value);\n    }\n}\n\nimpl<F> Visit for F\nwhere\n    F: FnMut(&Field, &dyn fmt::Debug),\n{\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        (self)(field, value)\n    }\n}\n\n// ===== impl Value =====\n\nmacro_rules! impl_values {\n    ( $( $record:ident( $( $whatever:tt)+ ) ),+ ) => {\n        $(\n            impl_value!{ $record( $( $whatever )+ ) }\n        )+\n    }\n}\n\nmacro_rules! ty_to_nonzero {\n    (u8) => {\n        NonZeroU8\n    };\n    (u16) => {\n        NonZeroU16\n    };\n    (u32) => {\n        NonZeroU32\n    };\n    (u64) => {\n        NonZeroU64\n    };\n    (u128) => {\n        NonZeroU128\n    };\n    (usize) => {\n        NonZeroUsize\n    };\n    (i8) => {\n        NonZeroI8\n    };\n    (i16) => {\n        NonZeroI16\n    };\n    (i32) => {\n        NonZeroI32\n    };\n    (i64) => {\n        NonZeroI64\n    };\n    (i128) => {\n        NonZeroI128\n    };\n    (isize) => {\n        NonZeroIsize\n    };\n}\n\nmacro_rules! impl_one_value {\n    (f32, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f32, $op, $record);\n    };\n    (f64, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f64, $op, $record);\n    };\n    (bool, $op:expr, $record:ident) => {\n        impl_one_value!(normal, bool, $op, $record);\n    };\n    ($value_ty:tt, $op:expr, $record:ident) => {\n        impl_one_value!(normal, $value_ty, $op, $record);\n        impl_one_value!(nonzero, $value_ty, $op, $record);\n    };\n    (normal, $value_ty:tt, $op:expr, $record:ident) => {\n        impl $crate::sealed::Sealed for $value_ty {}\n        impl $crate::field::Value for $value_ty {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(*self))\n            }\n        }\n    };\n    (nonzero, $value_ty:tt, $op:expr, $record:ident) => {\n        // This `use num::*;` is reported as unused because it gets emitted\n        // for every single invocation of this macro, so there are multiple `use`s.\n        // All but the first are useless indeed.\n        // We need this import because we can't write a path where one part is\n        // the `ty_to_nonzero!($value_ty)` invocation.\n        #[allow(clippy::useless_attribute, unused)]\n        use num::*;\n        impl $crate::sealed::Sealed for ty_to_nonzero!($value_ty) {}\n        impl $crate::field::Value for ty_to_nonzero!($value_ty) {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(self.get()))\n            }\n        }\n    };\n}\n\nmacro_rules! impl_value {\n    ( $record:ident( $( $value_ty:tt ),+ ) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this, $record);\n        )+\n    };\n    ( $record:ident( $( $value_ty:tt ),+ as $as_ty:ty) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this as $as_ty, $record);\n        )+\n    };\n}\n\n// ===== impl Value =====\n\nimpl_values! {\n    record_u64(u64),\n    record_u64(usize, u32, u16, u8 as u64),\n    record_i64(i64),\n    record_i64(isize, i32, i16, i8 as i64),\n    record_u128(u128),\n    record_i128(i128),\n    record_bool(bool),\n    record_f64(f64, f32 as f64)\n}\n\nimpl<T: crate::sealed::Sealed> crate::sealed::Sealed for Wrapping<T> {}\nimpl<T: crate::field::Value> crate::field::Value for Wrapping<T> {\n    fn record(&self, key: &crate::field::Field, visitor: &mut dyn crate::field::Visit) {\n        self.0.record(key, visitor)\n    }\n}\n\nimpl crate::sealed::Sealed for str {}\n\nimpl Value for str {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_error(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (*self).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a mut T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a mut T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        // Don't use `(*self).record(key, visitor)`, otherwise would\n        // cause stack overflow due to `unconditional_recursion`.\n        T::record(self, key, visitor)\n    }\n}\n\nimpl<'a> crate::sealed::Sealed for fmt::Arguments<'a> {}\n\nimpl<'a> Value for fmt::Arguments<'a> {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<T: ?Sized> crate::sealed::Sealed for alloc::boxed::Box<T> where T: Value {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl<T: ?Sized> Value for alloc::boxed::Box<T>\nwhere\n    T: Value,\n{\n    #[inline]\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        self.as_ref().record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl crate::sealed::Sealed for alloc::string::String {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl Value for alloc::string::String {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self.as_str())\n    }\n}\n\nimpl fmt::Debug for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        // We are only going to be recording the field value, so we don't\n        // actually care about the field name here.\n        struct NullCallsite;\n        static NULL_CALLSITE: NullCallsite = NullCallsite;\n        impl crate::callsite::Callsite for NullCallsite {\n            fn set_interest(&self, _: crate::collect::Interest) {\n                unreachable!(\"you somehow managed to register the null callsite?\")\n            }\n\n            fn metadata(&self) -> &crate::Metadata<'_> {\n                unreachable!(\"you somehow managed to access the null callsite?\")\n            }\n        }\n\n        static FIELD: Field = Field {\n            i: 0,\n            fields: FieldSet::new(&[], crate::identify_callsite!(&NULL_CALLSITE)),\n        };\n\n        let mut res = Ok(());\n        self.record(&FIELD, &mut |_: &Field, val: &dyn fmt::Debug| {\n            res = write!(f, \"{:?}\", val);\n        });\n        res\n    }\n}\n\nimpl fmt::Display for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Debug::fmt(self, f)\n    }\n}\n\n// ===== impl DisplayValue =====\n\nimpl<T: fmt::Display> crate::sealed::Sealed for DisplayValue<T> {}\n\nimpl<T> Value for DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Debug for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Display::fmt(self, f)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Display for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\n// ===== impl DebugValue =====\n\nimpl<T: fmt::Debug> crate::sealed::Sealed for DebugValue<T> {}\n\nimpl<T: fmt::Debug> Value for DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, &self.0)\n    }\n}\n\nimpl<T: fmt::Debug> fmt::Debug for DebugValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\nimpl crate::sealed::Sealed for Empty {}\nimpl Value for Empty {\n    #[inline]\n    fn record(&self, _: &Field, _: &mut dyn Visit) {}\n}\n\n// ===== impl Field =====\n\nimpl Field {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this field.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Returns a string representing the name of the field.\n    pub fn name(&self) -> &'static str {\n        self.fields.names[self.i]\n    }\n}\n\nimpl fmt::Display for Field {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(self.name())\n    }\n}\n\nimpl AsRef<str> for Field {\n    fn as_ref(&self) -> &str {\n        self.name()\n    }\n}\n\nimpl PartialEq for Field {\n    fn eq(&self, other: &Self) -> bool {\n        self.callsite() == other.callsite() && self.i == other.i\n    }\n}\n\nimpl Eq for Field {}\n\nimpl Hash for Field {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: Hasher,\n    {\n        self.callsite().hash(state);\n        self.i.hash(state);\n    }\n}\n\nimpl Clone for Field {\n    fn clone(&self) -> Self {\n        Field {\n            i: self.i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        }\n    }\n}\n\n// ===== impl FieldSet =====\n\nimpl FieldSet {\n    /// Constructs a new `FieldSet` with the given array of field names and callsite.\n    pub const fn new(names: &'static [&'static str], callsite: callsite::Identifier) -> Self {\n        Self { names, callsite }\n    }\n\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this set of fields..\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub(crate) fn callsite(&self) -> callsite::Identifier {\n        callsite::Identifier(self.callsite.0)\n    }\n\n    /// Returns the [`Field`] named `name`, or `None` if no such field exists.\n    ///\n    /// [`Field`]: super::Field\n    \npub fn field<Q: ?Sized>(&self, name: &Q) -> Option<Field>\n    where\n        Q: Borrow<str>,\n    {\n        let name = &name.borrow();\n        self.names.iter().position(|f| f == name).map(|i| Field {\n            i,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        })\n    }\n\n    /// Returns `true` if `self` contains the given `field`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: If <code>field</code> shares a name with a field\n    /// in this <code>FieldSet</code>, but was created by a <code>FieldSet</code>\n    /// with a different callsite, this <code>FieldSet</code> does <em>not</em>\n    /// contain it. This is so that if two separate span callsites define a field\n    /// named \"foo\", the <code>Field</code> corresponding to \"foo\" for each\n    /// of those callsites are not equivalent.\n    /// </pre></div>\n    pub fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite() && field.i <= self.len()\n    }\n\n    /// Returns an iterator over the `Field`s in this `FieldSet`.\n    #[inline]\n    pub fn iter(&self) -> Iter {\n        let idxs = 0..self.len();\n        Iter {\n            idxs,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        }\n    }\n\n    /// Returns a new `ValueSet` with entries for this `FieldSet`'s values.\n    #[doc(hidden)]\n    pub fn value_set<'v, V>(&'v self, values: &'v V) -> ValueSet<'v>\n    where\n        V: ValidLen<'v>,\n    {\n        ValueSet {\n            fields: self,\n            values: values.borrow(),\n        }\n    }\n\n    /// Returns the number of fields in this `FieldSet`.\n    #[inline]\n    pub fn len(&self) -> usize {\n        self.names.len()\n    }\n\n    /// Returns whether or not this `FieldSet` has fields.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.names.is_empty()\n    }\n}\n\nimpl<'a> IntoIterator for &'a FieldSet {\n    type IntoIter = Iter;\n    type Item = Field;\n    #[inline]\n    fn into_iter(self) -> Self::IntoIter {\n        self.iter()\n    }\n}\n\nimpl fmt::Debug for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"FieldSet\")\n            .field(\"names\", &self.names)\n            .field(\"callsite\", &self.callsite)\n            .finish()\n    }\n}\n\nimpl fmt::Display for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_set()\n            .entries(self.names.iter().map(display))\n            .finish()\n    }\n}\n\nimpl Eq for FieldSet {}\n\nimpl PartialEq for FieldSet {\n    fn eq(&self, other: &Self) -> bool {\n        if core::ptr::eq(&self, &other) {\n            true\n        } else if cfg!(not(debug_assertions)) {\n            // In a well-behaving application, two `FieldSet`s can be assumed to\n            // be totally equal so long as they share the same callsite.\n            self.callsite == other.callsite\n        } else {\n            // However, when debug-assertions are enabled, do NOT assume that\n            // the application is well-behaving; check every the field names of\n            // each `FieldSet` for equality.\n\n            // `FieldSet` is destructured here to ensure a compile-error if the\n            // fields of `FieldSet` change.\n            let Self {\n                names: lhs_names,\n                callsite: lhs_callsite,\n            } = self;\n\n            let Self {\n                names: rhs_names,\n                callsite: rhs_callsite,\n            } = &other;\n\n            // Check callsite equality first, as it is probably cheaper to do\n            // than str equality.\n            lhs_callsite == rhs_callsite && lhs_names == rhs_names\n        }\n    }\n}\n\n// ===== impl Iter =====\n\nimpl Iterator for Iter {\n    type Item = Field;\n    #[inline]\n    fn next(&mut self) -> Option<Field> {\n        let i = self.idxs.next()?;\n        Some(Field {\n            i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        })\n    }\n}\n\n// ===== impl ValueSet =====\n\nimpl<'a> ValueSet<'a> {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// defining the fields this `ValueSet` refers to.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Visits all the fields in this `ValueSet` with the provided [visitor].\n    ///\n    /// [visitor]: Visit\n    pub fn record(&self, visitor: &mut dyn Visit) {\n        let my_callsite = self.callsite();\n        for (field, value) in self.values {\n            if field.callsite() != my_callsite {\n                continue;\n            }\n            if let Some(value) = value {\n                value.record(field, visitor);\n            }\n        }\n    }\n\n    /// Returns the number of fields in this `ValueSet` that would be visited\n    /// by a given [visitor] to the [`ValueSet::record()`] method.\n    ///\n    /// [visitor]: Visit\n    /// [`ValueSet::record()`]: ValueSet::record()\n    pub fn len(&self) -> usize {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .filter(|(field, _)| field.callsite() == my_callsite)\n            .count()\n    }\n\n    /// Returns `true` if this `ValueSet` contains a value for the given `Field`.\n    pub(crate) fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite()\n            && self\n                .values\n                .iter()\n                .any(|(key, val)| *key == field && val.is_some())\n    }\n\n    /// Returns true if this `ValueSet` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .all(|(key, val)| val.is_none() || key.callsite() != my_callsite)\n    }\n\n    pub(crate) fn field_set(&self) -> &FieldSet {\n        self.fields\n    }\n}\n\nimpl<'a> fmt::Debug for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_struct(\"ValueSet\"), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .field(\"callsite\", &self.callsite())\n            .finish()\n    }\n}\n\nimpl<'a> fmt::Display for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_map(), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .finish()\n    }\n}\n\n// ===== impl ValidLen =====\n\nmod private {\n    use super::*;\n\n    /// Restrictions on `ValueSet` lengths were removed in #2508 but this type remains for backwards compatibility.\n    pub trait ValidLen<'a>: Borrow<[(&'a Field, Option<&'a (dyn Value + 'a)>)]> {}\n\n    impl<'a, const N: usize> ValidLen<'a> for [(&'a Field, Option<&'a (dyn Value + 'a)>); N] {}\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::metadata::{Kind, Level, Metadata};\n\n    // Make sure TEST_CALLSITE_* have non-zero size, so they can't be located at the same address.\n    struct TestCallsite1(u8);\n    static TEST_CALLSITE_1: TestCallsite1 = TestCallsite1(0);\n    static TEST_META_1: Metadata<'static> = metadata! {\n        name: \"field_test1\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_1,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite1 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_1\n        }\n    }\n\n    struct TestCallsite2(u8);\n    static TEST_CALLSITE_2: TestCallsite2 = TestCallsite2(0);\n    static TEST_META_2: Metadata<'static> = metadata! {\n        name: \"field_test2\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_2,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite2 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_2\n        }\n    }\n\n    #[test]\n    fn value_set_with_no_values_is_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), None),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn empty_value_set_is_empty() {\n        let fields = TEST_META_1.fields();\n        let valueset = fields.value_set(&[]);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn value_sets_with_fields_from_other_callsites_are_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = TEST_META_2.fields().value_set(values);\n        assert!(valueset.is_empty())\n    }\n\n    #[test]\n    fn sparse_value_sets_are_not_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(!valueset.is_empty());\n    }\n\n    #[test]\n    fn fields_from_other_callsets_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (\n                &TEST_META_2.fields().field(\"bar\").unwrap(),\n                Some(&57 as &dyn Value),\n            ),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.callsite(), TEST_META_1.callsite())\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    fn empty_fields_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&Empty as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&Empty as &dyn Value)),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.name(), \"bar\")\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_debug_fn() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = fields.value_set(values);\n        let mut result = String::new();\n        valueset.record(&mut |_: &Field, value: &dyn fmt::Debug| {\n            use core::fmt::Write;\n            write!(&mut result, \"{:?}\", value).unwrap();\n        });\n        assert_eq!(result, String::from(\"123\"));\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_error() {\n        let fields = TEST_META_1.fields();\n        let err: Box<dyn std::error::Error + Send + Sync + 'static> =\n            std::io::Error::new(std::io::ErrorKind::Other, \"lol\").into();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&err as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&Empty as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&Empty as &dyn Value)),\n        ];\n        let valueset = fields.value_set(values);\n        let mut result = String::new();\n        valueset.record(&mut |_: &Field, value: &dyn fmt::Debug| {\n            use core::fmt::Write;\n            write!(&mut result, \"{:?}\", value).unwrap();\n        });\n        assert_eq!(result, format!(\"{}\", err));\n    }\n}\n\n// Path: tracing-core/src/parent.rs\nuse crate::span::Id;\n\n#[derive(Debug)]\npub(crate) enum Parent {\n    /// The new span will be a root span.\n    Root,\n    /// The new span will be rooted in the current span.\n    Current,\n    /// The new span has an explicitly-specified parent.\n    Explicit(Id),\n}\n\n// Path: tracing-core/src/span.rs\n//! Spans represent periods of time in the execution of a program.\nuse core::num::NonZeroU64;\n\nuse crate::field::FieldSet;\nuse crate::parent::Parent;\nuse crate::{field, Metadata};\n\n/// Identifies a span within the context of a collector.\n///\n/// They are generated by [collector]s for each span as it is created, by\n/// the [`new_span`] trait method. See the documentation for that method for\n/// more information on span ID generation.\n///\n/// [collector]: super::collect::Collect\n/// [`new_span`]: super::collect::Collect::new_span\n#[derive(Clone, Debug, PartialEq, Eq, Hash)]\npub struct Id(NonZeroU64);\n\n/// Attributes provided to a collector describing a new span when it is\n/// created.\n#[derive(Debug)]\npub struct Attributes<'a> {\n    metadata: &'static Metadata<'static>,\n    values: &'a field::ValueSet<'a>,\n    parent: Parent,\n}\n\n/// A set of fields recorded by a span.\n#[derive(Debug)]\npub struct Record<'a> {\n    values: &'a field::ValueSet<'a>,\n}\n\n/// Indicates what [the collector considers] the \"current\" span.\n///\n/// As collectors may not track a notion of a current span, this has three\n/// possible states:\n/// - \"unknown\", indicating that the collector does not track a current span,\n/// - \"none\", indicating that the current context is known to not be in a span,\n/// - \"some\", with the current span's [`Id`] and [`Metadata`].\n///\n/// [the collector considers]: super::collect::Collect::current_span\n/// [`Metadata`]: super::metadata::Metadata\n#[derive(Debug)]\npub struct Current {\n    inner: CurrentInner,\n}\n\n#[derive(Debug)]\nenum CurrentInner {\n    Current {\n        id: Id,\n        metadata: &'static Metadata<'static>,\n    },\n    None,\n    Unknown,\n}\n\n// ===== impl Span =====\n\nimpl Id {\n    /// Constructs a new span ID from the given `u64`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: Span IDs must be greater than zero.</pre></div>\n    ///\n    /// # Panics\n    /// - If the provided `u64` is 0.\n    pub fn from_u64(u: u64) -> Self {\n        Id(NonZeroU64::new(u).expect(\"span IDs must be > 0\"))\n    }\n\n    /// Constructs a new span ID from the given `NonZeroU64`.\n    ///\n    /// Unlike [`Id::from_u64`](Self::from_u64), this will never panic.\n    #[inline]\n    pub const fn from_non_zero_u64(id: NonZeroU64) -> Self {\n        Id(id)\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `u64`.\n    pub fn into_u64(&self) -> u64 {\n        self.0.get()\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `NonZeroU64`.\n    #[inline]\n    pub const fn into_non_zero_u64(&self) -> NonZeroU64 {\n        self.0\n    }\n}\n\nimpl<'a> From<&'a Id> for Option<Id> {\n    fn from(id: &'a Id) -> Self {\n        Some(id.clone())\n    }\n}\n\n// ===== impl Attributes =====\n\nimpl<'a> Attributes<'a> {\n    /// Returns `Attributes` describing a new child span of the current span,\n    /// with the provided metadata and values.\n    pub fn new(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Current,\n        }\n    }\n\n    /// Returns `Attributes` describing a new span at the root of its own trace\n    /// tree, with the provided metadata and values.\n    pub fn new_root(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Root,\n        }\n    }\n\n    /// Returns `Attributes` describing a new child span of the specified\n    /// parent span, with the provided metadata and values.\n    pub fn child_of(\n        parent: Id,\n        metadata: &'static Metadata<'static>,\n        values: &'a field::ValueSet<'a>,\n    ) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Explicit(parent),\n        }\n    }\n\n    /// Returns a reference to the new span's metadata.\n    pub fn metadata(&self) -> &'static Metadata<'static> {\n        self.metadata\n    }\n\n    /// Returns a reference to a `ValueSet` containing any values the new span\n    /// was created with.\n    pub fn values(&self) -> &field::ValueSet<'a> {\n        self.values\n    }\n\n    /// Returns true if the new span should be a root.\n    pub fn is_root(&self) -> bool {\n        matches!(self.parent, Parent::Root)\n    }\n\n    /// Returns true if the new span's parent should be determined based on the\n    /// current context.\n    ///\n    /// If this is true and the current thread is currently inside a span, then\n    /// that span should be the new span's parent. Otherwise, if the current\n    /// thread is _not_ inside a span, then the new span will be the root of its\n    /// own trace tree.\n    pub fn is_contextual(&self) -> bool {\n        matches!(self.parent, Parent::Current)\n    }\n\n    /// Returns the new span's explicitly-specified parent, if there is one.\n    ///\n    /// Otherwise (if the new span is a root or is a child of the current span),\n    /// returns `None`.\n    pub fn parent(&self) -> Option<&Id> {\n        match self.parent {\n            Parent::Explicit(ref p) => Some(p),\n            _ => None,\n        }\n    }\n\n    /// Records all the fields in this set of `Attributes` with the provided\n    /// [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns `true` if this set of `Attributes` contains a value for the\n    /// given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this set of `Attributes` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n\n    /// Returns the set of all [fields] defined by this span's [`Metadata`].\n    ///\n    /// Note that the [`FieldSet`] returned by this method includes *all* the\n    /// fields declared by this span, not just those with values that are recorded\n    /// as part of this set of `Attributes`. Other fields with values not present in\n    /// this `Attributes`' value set may [record] values later.\n    ///\n    /// [fields]: crate::field\n    /// [record]: Attributes::record()\n    /// [`Metadata`]: crate::metadata::Metadata\n    /// [`FieldSet`]: crate::field::FieldSet\n    pub fn fields(&self) -> &FieldSet {\n        self.values.field_set()\n    }\n}\n\n// ===== impl Record =====\n\nimpl<'a> Record<'a> {\n    /// Constructs a new `Record` from a `ValueSet`.\n    pub fn new(values: &'a field::ValueSet<'a>) -> Self {\n        Self { values }\n    }\n\n    /// Records all the fields in this `Record` with the provided [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns the number of fields that would be visited from this `Record`\n    /// when [`Record::record()`] is called\n    ///\n    /// [`Record::record()`]: Record::record()\n    pub fn len(&self) -> usize {\n        self.values.len()\n    }\n\n    /// Returns `true` if this `Record` contains a value for the given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this `Record` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n}\n\n// ===== impl Current =====\n\nimpl Current {\n    /// Constructs a new `Current` that indicates the current context is a span\n    /// with the given `id` and `metadata`.\n    pub fn new(id: Id, metadata: &'static Metadata<'static>) -> Self {\n        Self {\n            inner: CurrentInner::Current { id, metadata },\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the current context is *not*\n    /// in a span.\n    pub fn none() -> Self {\n        Self {\n            inner: CurrentInner::None,\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the collector does not\n    /// track a current span.\n    pub fn unknown() -> Self {\n        Self {\n            inner: CurrentInner::Unknown,\n        }\n    }\n\n    /// Returns `true` if the collector that constructed this `Current` tracks a\n    /// current span.\n    ///\n    /// If this returns `true` and [`id`], [`metadata`], or [`into_inner`]\n    /// return `None`, that indicates that we are currently known to *not* be\n    /// inside a span. If this returns `false`, those methods will also return\n    /// `None`, but in this case, that is because the collector does not keep\n    /// track of the currently-entered span.\n    ///\n    /// [`id`]: Self::id\n    /// [`metadata`]: Self::metadata\n    /// [`into_inner`]: Self::into_inner\n    pub fn is_known(&self) -> bool {\n        !matches!(self.inner, CurrentInner::Unknown)\n    }\n\n    /// Consumes `self` and returns the span `Id` and `Metadata` of the current\n    /// span, if one exists and is known.\n    pub fn into_inner(self) -> Option<(Id, &'static Metadata<'static>)> {\n        match self.inner {\n            CurrentInner::Current { id, metadata } => Some((id, metadata)),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Id` of the current span, if one exists and is known.\n    pub fn id(&self) -> Option<&Id> {\n        match self.inner {\n            CurrentInner::Current { ref id, .. } => Some(id),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Metadata` of the current span, if one exists and is known.\n    pub fn metadata(&self) -> Option<&'static Metadata<'static>> {\n        match self.inner {\n            CurrentInner::Current { metadata, .. } => Some(metadata),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'a Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id()\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id().cloned()\n    }\n}\n\nimpl From<Current> for Option<Id> {\n    fn from(cur: Current) -> Self {\n        match cur.inner {\n            CurrentInner::Current { id, .. } => Some(id),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'static Metadata<'static>> {\n    fn from(cur: &'a Current) -> Self {\n        cur.metadata()\n    }\n}\n\n// Path: tracing-core/src/dispatch.rs\n//! Dispatches trace events to [`Collect`]s.\n//!\n//! The _dispatcher_ is the component of the tracing system which is responsible\n//! for forwarding trace data from the instrumentation points that generate it\n//! to the collector that collects it.\n//!\n//! # Using the Trace Dispatcher\n//!\n//! Every thread in a program using `tracing` has a _default collector_. When\n//! events occur, or spans are created, they are dispatched to the thread's\n//! current collector.\n//!\n//! ## Setting the Default Collector\n//!\n//! By default, the current collector is an empty implementation that does\n//! nothing. Trace data provided to this \"do nothing\" implementation is\n//! immediately discarded, and is not available for any purpose.\n//!\n//! To use another collector implementation, it must be set as the default.\n//! There are two methods for doing so: [`with_default`] and\n//! [`set_global_default`]. `with_default` sets the default collector for the\n//! duration of a scope, while `set_global_default` sets a default collector\n//! for the entire process.\n//!\n//! To use either of these functions, we must first wrap our collector in a\n//! [`Dispatch`], a cloneable, type-erased reference to a collector. For\n//! example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"alloc\")]\n//! use dispatch::Dispatch;\n//!\n//! # #[cfg(feature = \"alloc\")]\n//! let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"alloc\")]\n//! let my_dispatch = Dispatch::new(my_collector);\n//! ```\n//! Then, we can use [`with_default`] to set our `Dispatch` as the default for\n//! the duration of a block:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # let _my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(_my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::with_default(&my_dispatch, || {\n//!     // my_collector is the default\n//! });\n//!\n//! // no default collector again\n//! ```\n//! It's important to note that `with_default` will not propagate the current\n//! thread's default collector to any threads spawned within the `with_default`\n//! block. To propagate the default collector to new threads, either use\n//! `with_default` from the new thread, or use `set_global_default`.\n//!\n//! As an alternative to `with_default`, we can use [`set_global_default`] to\n//! set a `Dispatch` as the default for all threads, for the lifetime of the\n//! program. For example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"std\")]\n//! # let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::set_global_default(my_dispatch)\n//!     // `set_global_default` will return an error if the global default\n//!     // collector has already been set.\n...\n", "needle_token_start": 7241, "needle_token_end": 7350, "code_context_ntokens": 16219, "output": ["```rust\npub fn field<Q: ?Sized>(&self, name: &Q) -> Option<Field>\n    where\n        Q: Borrow<str>,\n{\n    let name = &name.borrow();\n    self.names.iter().position(|f| f == name).map(|i| Field {\n        i,\n        fields: FieldSet {\n            names: self.names,\n            callsite: self.callsite(),\n        },\n    })\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "sparse_value_sets_are_not_empty", "language": "rust", "path": "tracing-core/src/field.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To verify that a collection of field-value pairs, even when some values are missing (i.e., `None`), is not considered empty if at least one value is present.\n2. **Input**: A set of field-value pairs where some fields are associated with a value and others are not.\n3. **Output**: A boolean result indicating that the collection is not empty.\n4. **Procedure**: The test initializes a set of field-value pairs with some fields having no associated value (`None`) and at least one field having a value. It then checks if the resulting collection of these pairs is considered non-empty, asserting that the presence of any value negates the empty status of the collection.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "/// to be printed or stored in some other data structure.\n///\n/// The `Visit` trait provides default implementations for `record_i64`,\n/// `record_u64`, `record_bool`, `record_str`, and `record_error`, which simply\n/// forward the recorded value to `record_debug`. Thus, `record_debug` is the\n/// only method which a `Visit` implementation *must* implement. However,\n/// visitors may override the default implementations of these functions in\n/// order to implement type-specific behavior.\n///\n/// Additionally, when a visitor receives a value of a type it does not care\n/// about, it is free to ignore those values completely. For example, a\n/// visitor which only records numeric data might look like this:\n///\n/// ```\n/// # extern crate tracing_core as tracing;\n/// # use std::fmt::{self, Write};\n/// # use tracing::field::{Value, Visit, Field};\n/// pub struct SumVisitor {\n///     sum: i64,\n/// }\n///\n/// impl Visit for SumVisitor {\n///     fn record_i64(&mut self, _field: &Field, value: i64) {\n///        self.sum += value;\n///     }\n///\n///     fn record_u64(&mut self, _field: &Field, value: u64) {\n///         self.sum += value as i64;\n///     }\n///\n///     fn record_debug(&mut self, _field: &Field, _value: &dyn fmt::Debug) {\n///         // Do nothing\n///     }\n/// }\n/// ```\n///\n/// This visitor (which is probably not particularly useful) keeps a running\n/// sum of all the numeric values it records, and ignores all other values. A\n/// more practical example of recording typed values is presented in\n/// `examples/counters.rs`, which demonstrates a very simple metrics system\n/// implemented using `tracing`.\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: The <code>record_error</code> trait method is only\n/// available when the Rust standard library is present, as it requires the\n/// <code>std::error::Error</code> trait.\n/// </pre></div>\n///\n/// [recorded]: Value::record\n/// [`Collect`]: super::collect::Collect\n/// [records an `Event`]: super::collect::Collect::event\n/// [set of `Value`s added to a `Span`]: super::collect::Collect::record\n/// [`Event`]: super::event::Event\npub trait Visit {\n    /// Visit a double-precision floating point value.\n    fn record_f64(&mut self, field: &Field, value: f64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 64-bit integer value.\n    fn record_i64(&mut self, field: &Field, value: i64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 64-bit integer value.\n    fn record_u64(&mut self, field: &Field, value: u64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 128-bit integer value.\n    fn record_i128(&mut self, field: &Field, value: i128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 128-bit integer value.\n    fn record_u128(&mut self, field: &Field, value: u128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a boolean value.\n    fn record_bool(&mut self, field: &Field, value: bool) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a string value.\n    fn record_str(&mut self, field: &Field, value: &str) {\n        self.record_debug(field, &value)\n    }\n\n    /// Records a type implementing `Error`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: This is only enabled when the Rust standard library is\n    /// present.\n    /// </pre>\n    /// </div>\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n    fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) {\n        self.record_debug(field, &DisplayValue(value))\n    }\n\n    /// Visit a value implementing `fmt::Debug`.\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug);\n}\n\n/// A field value of an erased type.\n///\n/// Implementors of `Value` may call the appropriate typed recording methods on\n/// the [visitor] passed to their `record` method in order to indicate how\n/// their data should be recorded.\n///\n/// [visitor]: Visit\npub trait Value: crate::sealed::Sealed {\n    /// Visits this value with the given `Visitor`.\n    fn record(&self, key: &Field, visitor: &mut dyn Visit);\n}\n\n/// A `Value` which serializes using `fmt::Display`.\n///\n/// Uses `record_debug` in the `Value` implementation to\n/// avoid an unnecessary evaluation.\n#[derive(Clone)]\npub struct DisplayValue<T: fmt::Display>(T);\n\n/// A `Value` which serializes as a string using `fmt::Debug`.\n#[derive(Clone)]\npub struct DebugValue<T: fmt::Debug>(T);\n\n/// Wraps a type implementing `fmt::Display` as a `Value` that can be\n/// recorded using its `Display` implementation.\npub fn display<T>(t: T) -> DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    DisplayValue(t)\n}\n\n/// Wraps a type implementing `fmt::Debug` as a `Value` that can be\n/// recorded using its `Debug` implementation.\npub fn debug<T>(t: T) -> DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    DebugValue(t)\n}\n\n// ===== impl Visit =====\n\nimpl<'a, 'b> Visit for fmt::DebugStruct<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.field(field.name(), value);\n    }\n}\n\nimpl<'a, 'b> Visit for fmt::DebugMap<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.entry(&format_args!(\"{}\", field), value);\n    }\n}\n\nimpl<F> Visit for F\nwhere\n    F: FnMut(&Field, &dyn fmt::Debug),\n{\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        (self)(field, value)\n    }\n}\n\n// ===== impl Value =====\n\nmacro_rules! impl_values {\n    ( $( $record:ident( $( $whatever:tt)+ ) ),+ ) => {\n        $(\n            impl_value!{ $record( $( $whatever )+ ) }\n        )+\n    }\n}\n\nmacro_rules! ty_to_nonzero {\n    (u8) => {\n        NonZeroU8\n    };\n    (u16) => {\n        NonZeroU16\n    };\n    (u32) => {\n        NonZeroU32\n    };\n    (u64) => {\n        NonZeroU64\n    };\n    (u128) => {\n        NonZeroU128\n    };\n    (usize) => {\n        NonZeroUsize\n    };\n    (i8) => {\n        NonZeroI8\n    };\n    (i16) => {\n        NonZeroI16\n    };\n    (i32) => {\n        NonZeroI32\n    };\n    (i64) => {\n        NonZeroI64\n    };\n    (i128) => {\n        NonZeroI128\n    };\n    (isize) => {\n        NonZeroIsize\n    };\n}\n\nmacro_rules! impl_one_value {\n    (f32, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f32, $op, $record);\n    };\n    (f64, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f64, $op, $record);\n    };\n    (bool, $op:expr, $record:ident) => {\n        impl_one_value!(normal, bool, $op, $record);\n    };\n    ($value_ty:tt, $op:expr, $record:ident) => {\n        impl_one_value!(normal, $value_ty, $op, $record);\n        impl_one_value!(nonzero, $value_ty, $op, $record);\n    };\n    (normal, $value_ty:tt, $op:expr, $record:ident) => {\n        impl $crate::sealed::Sealed for $value_ty {}\n        impl $crate::field::Value for $value_ty {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(*self))\n            }\n        }\n    };\n    (nonzero, $value_ty:tt, $op:expr, $record:ident) => {\n        // This `use num::*;` is reported as unused because it gets emitted\n        // for every single invocation of this macro, so there are multiple `use`s.\n        // All but the first are useless indeed.\n        // We need this import because we can't write a path where one part is\n        // the `ty_to_nonzero!($value_ty)` invocation.\n        #[allow(clippy::useless_attribute, unused)]\n        use num::*;\n        impl $crate::sealed::Sealed for ty_to_nonzero!($value_ty) {}\n        impl $crate::field::Value for ty_to_nonzero!($value_ty) {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(self.get()))\n            }\n        }\n    };\n}\n\nmacro_rules! impl_value {\n    ( $record:ident( $( $value_ty:tt ),+ ) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this, $record);\n        )+\n    };\n    ( $record:ident( $( $value_ty:tt ),+ as $as_ty:ty) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this as $as_ty, $record);\n        )+\n    };\n}\n\n// ===== impl Value =====\n\nimpl_values! {\n    record_u64(u64),\n    record_u64(usize, u32, u16, u8 as u64),\n    record_i64(i64),\n    record_i64(isize, i32, i16, i8 as i64),\n    record_u128(u128),\n    record_i128(i128),\n    record_bool(bool),\n    record_f64(f64, f32 as f64)\n}\n\nimpl<T: crate::sealed::Sealed> crate::sealed::Sealed for Wrapping<T> {}\nimpl<T: crate::field::Value> crate::field::Value for Wrapping<T> {\n    fn record(&self, key: &crate::field::Field, visitor: &mut dyn crate::field::Visit) {\n        self.0.record(key, visitor)\n    }\n}\n\nimpl crate::sealed::Sealed for str {}\n\nimpl Value for str {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_error(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (*self).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a mut T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a mut T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        // Don't use `(*self).record(key, visitor)`, otherwise would\n        // cause stack overflow due to `unconditional_recursion`.\n        T::record(self, key, visitor)\n    }\n}\n\nimpl<'a> crate::sealed::Sealed for fmt::Arguments<'a> {}\n\nimpl<'a> Value for fmt::Arguments<'a> {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<T: ?Sized> crate::sealed::Sealed for alloc::boxed::Box<T> where T: Value {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl<T: ?Sized> Value for alloc::boxed::Box<T>\nwhere\n    T: Value,\n{\n    #[inline]\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        self.as_ref().record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl crate::sealed::Sealed for alloc::string::String {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl Value for alloc::string::String {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self.as_str())\n    }\n}\n\nimpl fmt::Debug for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        // We are only going to be recording the field value, so we don't\n        // actually care about the field name here.\n        struct NullCallsite;\n        static NULL_CALLSITE: NullCallsite = NullCallsite;\n        impl crate::callsite::Callsite for NullCallsite {\n            fn set_interest(&self, _: crate::collect::Interest) {\n                unreachable!(\"you somehow managed to register the null callsite?\")\n            }\n\n            fn metadata(&self) -> &crate::Metadata<'_> {\n                unreachable!(\"you somehow managed to access the null callsite?\")\n            }\n        }\n\n        static FIELD: Field = Field {\n            i: 0,\n            fields: FieldSet::new(&[], crate::identify_callsite!(&NULL_CALLSITE)),\n        };\n\n        let mut res = Ok(());\n        self.record(&FIELD, &mut |_: &Field, val: &dyn fmt::Debug| {\n            res = write!(f, \"{:?}\", val);\n        });\n        res\n    }\n}\n\nimpl fmt::Display for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Debug::fmt(self, f)\n    }\n}\n\n// ===== impl DisplayValue =====\n\nimpl<T: fmt::Display> crate::sealed::Sealed for DisplayValue<T> {}\n\nimpl<T> Value for DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Debug for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Display::fmt(self, f)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Display for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\n// ===== impl DebugValue =====\n\nimpl<T: fmt::Debug> crate::sealed::Sealed for DebugValue<T> {}\n\nimpl<T: fmt::Debug> Value for DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, &self.0)\n    }\n}\n\nimpl<T: fmt::Debug> fmt::Debug for DebugValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\nimpl crate::sealed::Sealed for Empty {}\nimpl Value for Empty {\n    #[inline]\n    fn record(&self, _: &Field, _: &mut dyn Visit) {}\n}\n\n// ===== impl Field =====\n\nimpl Field {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this field.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Returns a string representing the name of the field.\n    pub fn name(&self) -> &'static str {\n        self.fields.names[self.i]\n    }\n}\n\nimpl fmt::Display for Field {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(self.name())\n    }\n}\n\nimpl AsRef<str> for Field {\n    fn as_ref(&self) -> &str {\n        self.name()\n    }\n}\n\nimpl PartialEq for Field {\n    fn eq(&self, other: &Self) -> bool {\n        self.callsite() == other.callsite() && self.i == other.i\n    }\n}\n\nimpl Eq for Field {}\n\nimpl Hash for Field {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: Hasher,\n    {\n        self.callsite().hash(state);\n        self.i.hash(state);\n    }\n}\n\nimpl Clone for Field {\n    fn clone(&self) -> Self {\n        Field {\n            i: self.i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        }\n    }\n}\n\n// ===== impl FieldSet =====\n\nimpl FieldSet {\n    /// Constructs a new `FieldSet` with the given array of field names and callsite.\n    pub const fn new(names: &'static [&'static str], callsite: callsite::Identifier) -> Self {\n        Self { names, callsite }\n    }\n\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this set of fields..\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub(crate) fn callsite(&self) -> callsite::Identifier {\n        callsite::Identifier(self.callsite.0)\n    }\n\n    /// Returns the [`Field`] named `name`, or `None` if no such field exists.\n    ///\n    /// [`Field`]: super::Field\n    pub fn field<Q: ?Sized>(&self, name: &Q) -> Option<Field>\n    where\n        Q: Borrow<str>,\n    {\n        let name = &name.borrow();\n        self.names.iter().position(|f| f == name).map(|i| Field {\n            i,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        })\n    }\n\n    /// Returns `true` if `self` contains the given `field`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: If <code>field</code> shares a name with a field\n    /// in this <code>FieldSet</code>, but was created by a <code>FieldSet</code>\n    /// with a different callsite, this <code>FieldSet</code> does <em>not</em>\n    /// contain it. This is so that if two separate span callsites define a field\n    /// named \"foo\", the <code>Field</code> corresponding to \"foo\" for each\n    /// of those callsites are not equivalent.\n    /// </pre></div>\n    pub fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite() && field.i <= self.len()\n    }\n\n    /// Returns an iterator over the `Field`s in this `FieldSet`.\n    #[inline]\n    pub fn iter(&self) -> Iter {\n        let idxs = 0..self.len();\n        Iter {\n            idxs,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        }\n    }\n\n    /// Returns a new `ValueSet` with entries for this `FieldSet`'s values.\n    #[doc(hidden)]\n    pub fn value_set<'v, V>(&'v self, values: &'v V) -> ValueSet<'v>\n    where\n        V: ValidLen<'v>,\n    {\n        ValueSet {\n            fields: self,\n            values: values.borrow(),\n        }\n    }\n\n    /// Returns the number of fields in this `FieldSet`.\n    #[inline]\n    pub fn len(&self) -> usize {\n        self.names.len()\n    }\n\n    /// Returns whether or not this `FieldSet` has fields.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.names.is_empty()\n    }\n}\n\nimpl<'a> IntoIterator for &'a FieldSet {\n    type IntoIter = Iter;\n    type Item = Field;\n    #[inline]\n    fn into_iter(self) -> Self::IntoIter {\n        self.iter()\n    }\n}\n\nimpl fmt::Debug for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"FieldSet\")\n            .field(\"names\", &self.names)\n            .field(\"callsite\", &self.callsite)\n            .finish()\n    }\n}\n\nimpl fmt::Display for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_set()\n            .entries(self.names.iter().map(display))\n            .finish()\n    }\n}\n\nimpl Eq for FieldSet {}\n\nimpl PartialEq for FieldSet {\n    fn eq(&self, other: &Self) -> bool {\n        if core::ptr::eq(&self, &other) {\n            true\n        } else if cfg!(not(debug_assertions)) {\n            // In a well-behaving application, two `FieldSet`s can be assumed to\n            // be totally equal so long as they share the same callsite.\n            self.callsite == other.callsite\n        } else {\n            // However, when debug-assertions are enabled, do NOT assume that\n            // the application is well-behaving; check every the field names of\n            // each `FieldSet` for equality.\n\n            // `FieldSet` is destructured here to ensure a compile-error if the\n            // fields of `FieldSet` change.\n            let Self {\n                names: lhs_names,\n                callsite: lhs_callsite,\n            } = self;\n\n            let Self {\n                names: rhs_names,\n                callsite: rhs_callsite,\n            } = &other;\n\n            // Check callsite equality first, as it is probably cheaper to do\n            // than str equality.\n            lhs_callsite == rhs_callsite && lhs_names == rhs_names\n        }\n    }\n}\n\n// ===== impl Iter =====\n\nimpl Iterator for Iter {\n    type Item = Field;\n    #[inline]\n    fn next(&mut self) -> Option<Field> {\n        let i = self.idxs.next()?;\n        Some(Field {\n            i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        })\n    }\n}\n\n// ===== impl ValueSet =====\n\nimpl<'a> ValueSet<'a> {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// defining the fields this `ValueSet` refers to.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Visits all the fields in this `ValueSet` with the provided [visitor].\n    ///\n    /// [visitor]: Visit\n    pub fn record(&self, visitor: &mut dyn Visit) {\n        let my_callsite = self.callsite();\n        for (field, value) in self.values {\n            if field.callsite() != my_callsite {\n                continue;\n            }\n            if let Some(value) = value {\n                value.record(field, visitor);\n            }\n        }\n    }\n\n    /// Returns the number of fields in this `ValueSet` that would be visited\n    /// by a given [visitor] to the [`ValueSet::record()`] method.\n    ///\n    /// [visitor]: Visit\n    /// [`ValueSet::record()`]: ValueSet::record()\n    pub fn len(&self) -> usize {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .filter(|(field, _)| field.callsite() == my_callsite)\n            .count()\n    }\n\n    /// Returns `true` if this `ValueSet` contains a value for the given `Field`.\n    pub(crate) fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite()\n            && self\n                .values\n                .iter()\n                .any(|(key, val)| *key == field && val.is_some())\n    }\n\n    /// Returns true if this `ValueSet` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .all(|(key, val)| val.is_none() || key.callsite() != my_callsite)\n    }\n\n    pub(crate) fn field_set(&self) -> &FieldSet {\n        self.fields\n    }\n}\n\nimpl<'a> fmt::Debug for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_struct(\"ValueSet\"), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .field(\"callsite\", &self.callsite())\n            .finish()\n    }\n}\n\nimpl<'a> fmt::Display for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_map(), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .finish()\n    }\n}\n\n// ===== impl ValidLen =====\n\nmod private {\n    use super::*;\n\n    /// Restrictions on `ValueSet` lengths were removed in #2508 but this type remains for backwards compatibility.\n    pub trait ValidLen<'a>: Borrow<[(&'a Field, Option<&'a (dyn Value + 'a)>)]> {}\n\n    impl<'a, const N: usize> ValidLen<'a> for [(&'a Field, Option<&'a (dyn Value + 'a)>); N] {}\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::metadata::{Kind, Level, Metadata};\n\n    // Make sure TEST_CALLSITE_* have non-zero size, so they can't be located at the same address.\n    struct TestCallsite1(u8);\n    static TEST_CALLSITE_1: TestCallsite1 = TestCallsite1(0);\n    static TEST_META_1: Metadata<'static> = metadata! {\n        name: \"field_test1\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_1,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite1 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_1\n        }\n    }\n\n    struct TestCallsite2(u8);\n    static TEST_CALLSITE_2: TestCallsite2 = TestCallsite2(0);\n    static TEST_META_2: Metadata<'static> = metadata! {\n        name: \"field_test2\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_2,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite2 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_2\n        }\n    }\n\n    #[test]\n    fn value_set_with_no_values_is_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), None),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn empty_value_set_is_empty() {\n        let fields = TEST_META_1.fields();\n        let valueset = fields.value_set(&[]);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn value_sets_with_fields_from_other_callsites_are_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = TEST_META_2.fields().value_set(values);\n        assert!(valueset.is_empty())\n    }\n\n    #[test]\n    \nfn sparse_value_sets_are_not_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(!valueset.is_empty());\n    }\n\n    #[test]\n    fn fields_from_other_callsets_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (\n                &TEST_META_2.fields().field(\"bar\").unwrap(),\n                Some(&57 as &dyn Value),\n            ),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.callsite(), TEST_META_1.callsite())\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    fn empty_fields_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&Empty as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&Empty as &dyn Value)),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.name(), \"bar\")\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_debug_fn() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = fields.value_set(values);\n        let mut result = String::new();\n        valueset.record(&mut |_: &Field, value: &dyn fmt::Debug| {\n            use core::fmt::Write;\n            write!(&mut result, \"{:?}\", value).unwrap();\n        });\n        assert_eq!(result, String::from(\"123\"));\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_error() {\n        let fields = TEST_META_1.fields();\n        let err: Box<dyn std::error::Error + Send + Sync + 'static> =\n            std::io::Error::new(std::io::ErrorKind::Other, \"lol\").into();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&err as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&Empty as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&Empty as &dyn Value)),\n        ];\n        let valueset = fields.value_set(values);\n        let mut result = String::new();\n        valueset.record(&mut |_: &Field, value: &dyn fmt::Debug| {\n            use core::fmt::Write;\n            write!(&mut result, \"{:?}\", value).unwrap();\n        });\n        assert_eq!(result, format!(\"{}\", err));\n    }\n}\n\n// Path: tracing-core/src/parent.rs\nuse crate::span::Id;\n\n#[derive(Debug)]\npub(crate) enum Parent {\n    /// The new span will be a root span.\n    Root,\n    /// The new span will be rooted in the current span.\n    Current,\n    /// The new span has an explicitly-specified parent.\n    Explicit(Id),\n}\n\n// Path: tracing-core/src/span.rs\n//! Spans represent periods of time in the execution of a program.\nuse core::num::NonZeroU64;\n\nuse crate::field::FieldSet;\nuse crate::parent::Parent;\nuse crate::{field, Metadata};\n\n/// Identifies a span within the context of a collector.\n///\n/// They are generated by [collector]s for each span as it is created, by\n/// the [`new_span`] trait method. See the documentation for that method for\n/// more information on span ID generation.\n///\n/// [collector]: super::collect::Collect\n/// [`new_span`]: super::collect::Collect::new_span\n#[derive(Clone, Debug, PartialEq, Eq, Hash)]\npub struct Id(NonZeroU64);\n\n/// Attributes provided to a collector describing a new span when it is\n/// created.\n#[derive(Debug)]\npub struct Attributes<'a> {\n    metadata: &'static Metadata<'static>,\n    values: &'a field::ValueSet<'a>,\n    parent: Parent,\n}\n\n/// A set of fields recorded by a span.\n#[derive(Debug)]\npub struct Record<'a> {\n    values: &'a field::ValueSet<'a>,\n}\n\n/// Indicates what [the collector considers] the \"current\" span.\n///\n/// As collectors may not track a notion of a current span, this has three\n/// possible states:\n/// - \"unknown\", indicating that the collector does not track a current span,\n/// - \"none\", indicating that the current context is known to not be in a span,\n/// - \"some\", with the current span's [`Id`] and [`Metadata`].\n///\n/// [the collector considers]: super::collect::Collect::current_span\n/// [`Metadata`]: super::metadata::Metadata\n#[derive(Debug)]\npub struct Current {\n    inner: CurrentInner,\n}\n\n#[derive(Debug)]\nenum CurrentInner {\n    Current {\n        id: Id,\n        metadata: &'static Metadata<'static>,\n    },\n    None,\n    Unknown,\n}\n\n// ===== impl Span =====\n\nimpl Id {\n    /// Constructs a new span ID from the given `u64`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: Span IDs must be greater than zero.</pre></div>\n    ///\n    /// # Panics\n    /// - If the provided `u64` is 0.\n    pub fn from_u64(u: u64) -> Self {\n        Id(NonZeroU64::new(u).expect(\"span IDs must be > 0\"))\n    }\n\n    /// Constructs a new span ID from the given `NonZeroU64`.\n    ///\n    /// Unlike [`Id::from_u64`](Self::from_u64), this will never panic.\n    #[inline]\n    pub const fn from_non_zero_u64(id: NonZeroU64) -> Self {\n        Id(id)\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `u64`.\n    pub fn into_u64(&self) -> u64 {\n        self.0.get()\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `NonZeroU64`.\n    #[inline]\n    pub const fn into_non_zero_u64(&self) -> NonZeroU64 {\n        self.0\n    }\n}\n\nimpl<'a> From<&'a Id> for Option<Id> {\n    fn from(id: &'a Id) -> Self {\n        Some(id.clone())\n    }\n}\n\n// ===== impl Attributes =====\n\nimpl<'a> Attributes<'a> {\n    /// Returns `Attributes` describing a new child span of the current span,\n    /// with the provided metadata and values.\n    pub fn new(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Current,\n        }\n    }\n\n    /// Returns `Attributes` describing a new span at the root of its own trace\n    /// tree, with the provided metadata and values.\n    pub fn new_root(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Root,\n        }\n    }\n\n    /// Returns `Attributes` describing a new child span of the specified\n    /// parent span, with the provided metadata and values.\n    pub fn child_of(\n        parent: Id,\n        metadata: &'static Metadata<'static>,\n        values: &'a field::ValueSet<'a>,\n    ) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Explicit(parent),\n        }\n    }\n\n    /// Returns a reference to the new span's metadata.\n    pub fn metadata(&self) -> &'static Metadata<'static> {\n        self.metadata\n    }\n\n    /// Returns a reference to a `ValueSet` containing any values the new span\n    /// was created with.\n    pub fn values(&self) -> &field::ValueSet<'a> {\n        self.values\n    }\n\n    /// Returns true if the new span should be a root.\n    pub fn is_root(&self) -> bool {\n        matches!(self.parent, Parent::Root)\n    }\n\n    /// Returns true if the new span's parent should be determined based on the\n    /// current context.\n    ///\n    /// If this is true and the current thread is currently inside a span, then\n    /// that span should be the new span's parent. Otherwise, if the current\n    /// thread is _not_ inside a span, then the new span will be the root of its\n    /// own trace tree.\n    pub fn is_contextual(&self) -> bool {\n        matches!(self.parent, Parent::Current)\n    }\n\n    /// Returns the new span's explicitly-specified parent, if there is one.\n    ///\n    /// Otherwise (if the new span is a root or is a child of the current span),\n    /// returns `None`.\n    pub fn parent(&self) -> Option<&Id> {\n        match self.parent {\n            Parent::Explicit(ref p) => Some(p),\n            _ => None,\n        }\n    }\n\n    /// Records all the fields in this set of `Attributes` with the provided\n    /// [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns `true` if this set of `Attributes` contains a value for the\n    /// given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this set of `Attributes` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n\n    /// Returns the set of all [fields] defined by this span's [`Metadata`].\n    ///\n    /// Note that the [`FieldSet`] returned by this method includes *all* the\n    /// fields declared by this span, not just those with values that are recorded\n    /// as part of this set of `Attributes`. Other fields with values not present in\n    /// this `Attributes`' value set may [record] values later.\n    ///\n    /// [fields]: crate::field\n    /// [record]: Attributes::record()\n    /// [`Metadata`]: crate::metadata::Metadata\n    /// [`FieldSet`]: crate::field::FieldSet\n    pub fn fields(&self) -> &FieldSet {\n        self.values.field_set()\n    }\n}\n\n// ===== impl Record =====\n\nimpl<'a> Record<'a> {\n    /// Constructs a new `Record` from a `ValueSet`.\n    pub fn new(values: &'a field::ValueSet<'a>) -> Self {\n        Self { values }\n    }\n\n    /// Records all the fields in this `Record` with the provided [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns the number of fields that would be visited from this `Record`\n    /// when [`Record::record()`] is called\n    ///\n    /// [`Record::record()`]: Record::record()\n    pub fn len(&self) -> usize {\n        self.values.len()\n    }\n\n    /// Returns `true` if this `Record` contains a value for the given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this `Record` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n}\n\n// ===== impl Current =====\n\nimpl Current {\n    /// Constructs a new `Current` that indicates the current context is a span\n    /// with the given `id` and `metadata`.\n    pub fn new(id: Id, metadata: &'static Metadata<'static>) -> Self {\n        Self {\n            inner: CurrentInner::Current { id, metadata },\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the current context is *not*\n    /// in a span.\n    pub fn none() -> Self {\n        Self {\n            inner: CurrentInner::None,\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the collector does not\n    /// track a current span.\n    pub fn unknown() -> Self {\n        Self {\n            inner: CurrentInner::Unknown,\n        }\n    }\n\n    /// Returns `true` if the collector that constructed this `Current` tracks a\n    /// current span.\n    ///\n    /// If this returns `true` and [`id`], [`metadata`], or [`into_inner`]\n    /// return `None`, that indicates that we are currently known to *not* be\n    /// inside a span. If this returns `false`, those methods will also return\n    /// `None`, but in this case, that is because the collector does not keep\n    /// track of the currently-entered span.\n    ///\n    /// [`id`]: Self::id\n    /// [`metadata`]: Self::metadata\n    /// [`into_inner`]: Self::into_inner\n    pub fn is_known(&self) -> bool {\n        !matches!(self.inner, CurrentInner::Unknown)\n    }\n\n    /// Consumes `self` and returns the span `Id` and `Metadata` of the current\n    /// span, if one exists and is known.\n    pub fn into_inner(self) -> Option<(Id, &'static Metadata<'static>)> {\n        match self.inner {\n            CurrentInner::Current { id, metadata } => Some((id, metadata)),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Id` of the current span, if one exists and is known.\n    pub fn id(&self) -> Option<&Id> {\n        match self.inner {\n            CurrentInner::Current { ref id, .. } => Some(id),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Metadata` of the current span, if one exists and is known.\n    pub fn metadata(&self) -> Option<&'static Metadata<'static>> {\n        match self.inner {\n            CurrentInner::Current { metadata, .. } => Some(metadata),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'a Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id()\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id().cloned()\n    }\n}\n\nimpl From<Current> for Option<Id> {\n    fn from(cur: Current) -> Self {\n        match cur.inner {\n            CurrentInner::Current { id, .. } => Some(id),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'static Metadata<'static>> {\n    fn from(cur: &'a Current) -> Self {\n        cur.metadata()\n    }\n}\n\n// Path: tracing-core/src/dispatch.rs\n//! Dispatches trace events to [`Collect`]s.\n//!\n//! The _dispatcher_ is the component of the tracing system which is responsible\n//! for forwarding trace data from the instrumentation points that generate it\n//! to the collector that collects it.\n//!\n//! # Using the Trace Dispatcher\n//!\n//! Every thread in a program using `tracing` has a _default collector_. When\n//! events occur, or spans are created, they are dispatched to the thread's\n//! current collector.\n//!\n//! ## Setting the Default Collector\n//!\n//! By default, the current collector is an empty implementation that does\n//! nothing. Trace data provided to this \"do nothing\" implementation is\n//! immediately discarded, and is not available for any purpose.\n//!\n//! To use another collector implementation, it must be set as the default.\n//! There are two methods for doing so: [`with_default`] and\n//! [`set_global_default`]. `with_default` sets the default collector for the\n//! duration of a scope, while `set_global_default` sets a default collector\n//! for the entire process.\n//!\n//! To use either of these functions, we must first wrap our collector in a\n//! [`Dispatch`], a cloneable, type-erased reference to a collector. For\n//! example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"alloc\")]\n//! use dispatch::Dispatch;\n//!\n//! # #[cfg(feature = \"alloc\")]\n//! let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"alloc\")]\n//! let my_dispatch = Dispatch::new(my_collector);\n//! ```\n//! Then, we can use [`with_default`] to set our `Dispatch` as the default for\n//! the duration of a block:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # let _my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(_my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::with_default(&my_dispatch, || {\n//!     // my_collector is the default\n//! });\n//!\n//! // no default collector again\n//! ```\n//! It's important to note that `with_default` will not propagate the current\n//! thread's default collector to any threads spawned within the `with_default`\n//! block. To propagate the default collector to new threads, either use\n//! `with_default` from the new thread, or use `set_global_default`.\n//!\n//! As an alternative to `with_default`, we can use [`set_global_default`] to\n//! set a `Dispatch` as the default for all threads, for the lifetime of the\n//! program. For example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"std\")]\n//! # let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::set_global_default(my_dispatch)\n//!     // `set_global_default` will return an error if the global default\n//!     // collector has already been set.\n//!     .expect(\"global default was already set!\");\n//!\n//! // `my_collector` is now the default\n//! ```\n//!\n//! <div class=\"example-wrap\" style=\"display:inline-block\">\n//! <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n//!\n//! **Note**: the thread-local scoped dispatcher ([`with_default`]) requires the\n//! Rust standard library. `no_std` users should use [`set_global_default`] instead.\n//!\n//! </pre></div>\n//!\n//! ## Accessing the Default Collector\n//!\n//! A thread's current default collector can be accessed using the\n//! [`get_default`] function, which executes a closure with a reference to the\n//! currently default `Dispatch`. This is used primarily by `tracing`\n//! instrumentation.\nuse crate::{\n    collect::{self, Collect, NoCollector},\n    span, Event, LevelFilter, Metadata,\n};\n\nuse core::{\n    any::Any,\n    fmt,\n    sync::atomic::{AtomicBool, AtomicUsize, Ordering},\n};\n\n#[cfg(feature = \"std\")]\nuse std::{\n    cell::{Cell, RefCell, RefMut},\n    error,\n};\n\n#[cfg(feature = \"alloc\")]\nuse alloc::sync::{Arc, Weak};\n\n#[cfg(feature = \"alloc\")]\nuse core::ops::Deref;\n\n/// `Dispatch` trace data to a [`Collect`].\n#[derive(Clone)]\npub struct Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Arc<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n/// `WeakDispatch` is a version of [`Dispatch`] that holds a non-owning reference\n/// to a [collector].\n///\n/// The collector may be accessed by calling [`WeakDispatch::upgrade`],\n/// which returns an `Option<Dispatch>`. If all [`Dispatch`] clones that point\n/// at the collector have been dropped, [`WeakDispatch::upgrade`] will return\n/// `None`. Otherwise, it will return `Some(Dispatch)`.\n///\n/// A `WeakDispatch` may be created from a [`Dispatch`] by calling the\n/// [`Dispatch::downgrade`] method. The primary use for creating a\n/// [`WeakDispatch`] is to allow a collector to hold a cyclical reference to\n/// itself without creating a memory leak. See [here] for details.\n///\n/// This type is analogous to the [`std::sync::Weak`] type, but for a\n/// [`Dispatch`] rather than an [`Arc`].\n///\n/// [collector]: Collect\n/// [`Arc`]: std::sync::Arc\n/// [here]: Collect#avoiding-memory-leaks\n#[derive(Clone)]\npub struct WeakDispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Weak<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n#[cfg(feature = \"alloc\")]\n#[derive(Clone)]\nenum Kind<T> {\n    Global(&'static (dyn Collect + Send + Sync)),\n    Scoped(T),\n}\n\n#[cfg(feature = \"std\")]\nthread_local! {\n    static CURRENT_STATE: State = const {\n        State {\n            default: RefCell::new(None),\n            can_enter: Cell::new(true),\n        }\n    };\n}\n\nstatic EXISTS: AtomicBool = AtomicBool::new(false);\nstatic GLOBAL_INIT: AtomicUsize = AtomicUsize::new(UNINITIALIZED);\n\n#[cfg(feature = \"std\")]\nstatic SCOPED_COUNT: AtomicUsize = AtomicUsize::new(0);\n\nconst UNINITIALIZED: usize = 0;\nconst INITIALIZING: usize = 1;\nconst INITIALIZED: usize = 2;\n\nstatic mut GLOBAL_DISPATCH: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NONE: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NO_COLLECTOR: NoCollector = NoCollector::new();\n\n/// The dispatch state of a thread.\n#[cfg(feature = \"std\")]\nstruct State {\n    /// This thread's current default dispatcher.\n    default: RefCell<Option<Dispatch>>,\n    /// Whether or not we can currently begin dispatching a trace event.\n    ///\n    /// This is set to `false` when functions such as `enter`, `exit`, `event`,\n    /// and `new_span` are called on this thread's default dispatcher, to\n    /// prevent further trace events triggered inside those functions from\n    /// creating an infinite recursion. When we finish handling a dispatch, this\n    /// is set back to `true`.\n    can_enter: Cell<bool>,\n}\n\n/// While this guard is active, additional calls to collector functions on\n/// the default dispatcher will not be able to access the dispatch context.\n/// Dropping the guard will allow the dispatch context to be re-entered.\n#[cfg(feature = \"std\")]\nstruct Entered<'a>(&'a State);\n\n/// A guard that resets the current default dispatcher to the prior\n...\n", "needle_token_start": 8857, "needle_token_end": 8975, "code_context_ntokens": 16234, "output": ["```rust\n#[test]\nfn sparse_value_sets_are_not_empty() {\n    let fields = TEST_META_1.fields();\n    let values = &[\n        (&fields.field(\"foo\").unwrap(), None),\n        (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n        (&fields.field(\"baz\").unwrap(), None),\n    ];\n    let valueset = fields.value_set(values);\n    assert!(!valueset.is_empty());\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "linked_list_push", "language": "rust", "path": "tracing-core/src/callsite.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To add a new element to the front of an intrusive atomic push-only linked list, ensuring thread safety and avoiding duplicates.\n2. **Input**: A reference to a statically allocated registration object.\n3. **Output**: None directly, but modifies the linked list by inserting the new registration at the beginning.\n4. **Procedure**: The function begins by loading the current head of the list. It then enters a loop where it sets the next pointer of the new registration to this head. A check ensures the new registration is not already in the list to prevent infinite loops. Using an atomic compare-and-exchange operation, it attempts to update the head of the list to the new registration. If the head has changed during this process (due to concurrent operations), it retries with the new head until successful.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: tracing-core/src/collect.rs\n//! Collectors collect and record trace data.\nuse crate::{span, Dispatch, Event, LevelFilter, Metadata};\n\nuse core::any::{Any, TypeId};\nuse core::ptr::NonNull;\n\n/// Trait representing the functions required to collect trace data.\n///\n/// Crates that provide implementations of methods for collecting or recording\n/// trace data should implement the `Collect` interface. This trait is\n/// intended to represent fundamental primitives for collecting trace events and\n/// spans \u2014 other libraries may offer utility functions and types to make\n/// collector implementations more modular or improve the ergonomics of writing\n/// collectors.\n///\n/// A collector is responsible for the following:\n/// - Registering new spans as they are created, and providing them with span\n///   IDs. Implicitly, this means the collector may determine the strategy for\n///   determining span equality.\n/// - Recording the attachment of field values and follows-from annotations to\n///   spans.\n/// - Filtering spans and events, and determining when those filters must be\n///   invalidated.\n/// - Observing spans as they are entered, exited, and closed, and events as\n///   they occur.\n///\n/// When a span is entered or exited, the collector is provided only with the\n/// [ID] with which it tagged that span when it was created. This means\n/// that it is up to the collector to determine whether and how span _data_ \u2014\n/// the fields and metadata describing the span \u2014 should be stored. The\n/// [`new_span`] function is called when a new span is created, and at that\n/// point, the collector _may_ choose to store the associated data if it will\n/// be referenced again. However, if the data has already been recorded and will\n/// not be needed by the implementations of `enter` and `exit`, the collector\n/// may freely discard that data without allocating space to store it.\n///\n/// ## Overriding default impls\n///\n/// Some trait methods on `Collect` have default implementations, either in\n/// order to reduce the surface area of implementing `Collect`, or for\n/// backward-compatibility reasons. However, many collectors will likely want\n/// to override these default implementations.\n///\n/// The following methods are likely of interest:\n///\n/// - [`register_callsite`] is called once for each callsite from which a span\n///   event may originate, and returns an [`Interest`] value describing whether or\n///   not the collector wishes to see events or spans from that callsite. By\n///   default, it calls [`enabled`], and returns `Interest::always()` if\n///   `enabled` returns true, or `Interest::never()` if enabled returns false.\n///   However, if the collector's interest can change dynamically at runtime,\n///   it may want to override this function to return `Interest::sometimes()`.\n///   Additionally, collectors which wish to perform a behaviour once for each\n///   callsite, such as allocating storage for data related to that callsite,\n///   can perform it in `register_callsite`.\n///\n///   See also the [documentation on the callsite registry][cs-reg] for details\n///   on [`register_callsite`].\n///\n/// - [`event_enabled`] is called once before every call to the [`event`]\n///   method. This can be used to implement filtering on events once their field\n///   values are known, but before any processing is done in the `event` method.\n/// - [`clone_span`] is called every time a span ID is cloned, and [`try_close`]\n///   is called when a span ID is dropped. By default, these functions do\n///   nothing. However, they can be used to implement reference counting for\n///   spans, allowing collectors to free storage for span data and to determine\n///   when a span has _closed_ permanently (rather than being exited).\n///   Collectors which store per-span data or which need to track span closures\n///   should override these functions together.\n///\n/// [ID]: super::span::Id\n/// [`new_span`]: Collect::new_span\n/// [`register_callsite`]: Collect::register_callsite\n/// [`enabled`]: Collect::enabled\n/// [`clone_span`]: Collect::clone_span\n/// [`try_close`]: Collect::try_close\n/// [cs-reg]: crate::callsite#registering-callsites\n/// [`event`]: Collect::event\n/// [`event_enabled`]: Collect::event_enabled\npub trait Collect: 'static {\n    /// Invoked when this collector becomes a [`Dispatch`].\n    ///\n    /// ## Avoiding Memory Leaks\n    ///\n    /// Collectors should not store their own [`Dispatch`]. Because the\n    /// `Dispatch` owns the collector, storing the `Dispatch` within the\n    /// collector will create a reference count cycle, preventing the `Dispatch`\n    /// from ever being dropped.\n    ///\n    /// Instead, when it is necessary to store a cyclical reference to the\n    /// `Dispatch` within a collector, use [`Dispatch::downgrade`] to convert a\n    /// `Dispatch` into a [`WeakDispatch`]. This type is analogous to\n    /// [`std::sync::Weak`], and does not create a reference count cycle. A\n    /// [`WeakDispatch`] can be stored within a collector without causing a\n    /// memory leak, and can be [upgraded] into a `Dispatch` temporarily when\n    /// the `Dispatch` must be accessed by the collector.\n    ///\n    /// [`WeakDispatch`]: crate::dispatch::WeakDispatch\n    /// [upgraded]: crate::dispatch::WeakDispatch::upgrade\n    fn on_register_dispatch(&self, collector: &Dispatch) {\n        let _ = collector;\n    }\n\n    // === Span registry methods ==============================================\n\n    /// Registers a new [callsite] with this collector, returning whether or not\n    /// the collector is interested in being notified about the callsite.\n    ///\n    /// By default, this function assumes that the collector's [filter]\n    /// represents an unchanging view of its interest in the callsite. However,\n    /// if this is not the case, collectors may override this function to\n    /// indicate different interests, or to implement behaviour that should run\n    /// once for every callsite.\n    ///\n    /// This function is guaranteed to be called at least once per callsite on\n    /// every active collector. The collector may store the keys to fields it\n    /// cares about in order to reduce the cost of accessing fields by name,\n    /// preallocate storage for that callsite, or perform any other actions it\n    /// wishes to perform once for each callsite.\n    ///\n    /// The collector should then return an [`Interest`], indicating\n    /// whether it is interested in being notified about that callsite in the\n    /// future. This may be `Always` indicating that the collector always\n    /// wishes to be notified about the callsite, and its filter need not be\n    /// re-evaluated; `Sometimes`, indicating that the collector may sometimes\n    /// care about the callsite but not always (such as when sampling), or\n    /// `Never`, indicating that the collector never wishes to be notified about\n    /// that callsite. If all active collectors return `Never`, a callsite will\n    /// never be enabled unless a new collector expresses interest in it.\n    ///\n    /// `Collector`s which require their filters to be run every time an event\n    /// occurs or a span is entered/exited should return `Interest::sometimes`.\n    /// If a collector returns `Interest::sometimes`, then its' [`enabled`] method\n    /// will be called every time an event or span is created from that callsite.\n    ///\n    /// For example, suppose a sampling collector is implemented by\n    /// incrementing a counter every time `enabled` is called and only returning\n    /// `true` when the counter is divisible by a specified sampling rate. If\n    /// that collector returns `Interest::always` from `register_callsite`, then\n    /// the filter will not be re-evaluated once it has been applied to a given\n    /// set of metadata. Thus, the counter will not be incremented, and the span\n    /// or event that corresponds to the metadata will never be `enabled`.\n    ///\n    /// `Collector`s that need to change their filters occasionally should call\n    /// [`rebuild_interest_cache`] to re-evaluate `register_callsite` for all\n    /// callsites.\n    ///\n    /// Similarly, if a `Collector` has a filtering strategy that can be\n    /// changed dynamically at runtime, it would need to re-evaluate that filter\n    /// if the cached results have changed.\n    ///\n    /// A collector which manages fanout to multiple other collectors\n    /// should proxy this decision to all of its child collectors,\n    /// returning `Interest::never` only if _all_ such children return\n    /// `Interest::never`. If the set of collectors to which spans are\n    /// broadcast may change dynamically, the collector should also never\n    /// return `Interest::Never`, as a new collector may be added that _is_\n    /// interested.\n    ///\n    /// See the [documentation on the callsite registry][cs-reg] for more\n    /// details on how and when the `register_callsite` method is called.\n    ///\n    /// # Notes\n    ///\n    /// This function may be called again when a new collector is created or\n    /// when the registry is invalidated.\n    ///\n    /// If a collector returns `Interest::never` for a particular callsite, it\n    /// _may_ still see spans and events originating from that callsite, if\n    /// another collector expressed interest in it.\n    ///\n    /// [callsite]: crate::callsite\n    /// [filter]: Self::enabled\n    /// [metadata]: super::metadata::Metadata\n    /// [`enabled`]: Self::enabled\n    /// [`rebuild_interest_cache`]: super::callsite::rebuild_interest_cache\n    /// [cs-reg]: crate::callsite#registering-callsites\n    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {\n        if self.enabled(metadata) {\n            Interest::always()\n        } else {\n            Interest::never()\n        }\n    }\n\n    /// Returns true if a span or event with the specified [metadata] would be\n    /// recorded.\n    ///\n    /// By default, it is assumed that this filter needs only be evaluated once\n    /// for each callsite, so it is called by [`register_callsite`] when each\n    /// callsite is registered. The result is used to determine if the collector\n    /// is always [interested] or never interested in that callsite. This is intended\n    /// primarily as an optimization, so that expensive filters (such as those\n    /// involving string search, et cetera) need not be re-evaluated.\n    ///\n    /// However, if the collector's interest in a particular span or event may\n    /// change, or depends on contexts only determined dynamically at runtime,\n    /// then the `register_callsite` method should be overridden to return\n    /// [`Interest::sometimes`]. In that case, this function will be called every\n    /// time that span or event occurs.\n    ///\n    /// [metadata]: super::metadata::Metadata\n    /// [interested]: Interest\n    /// [`register_callsite`]: Self::register_callsite\n    fn enabled(&self, metadata: &Metadata<'_>) -> bool;\n\n    /// Returns the highest [verbosity level][level] that this `Collector` will\n    /// enable, or `None`, if the collector does not implement level-based\n    /// filtering or chooses not to implement this method.\n    ///\n    /// If this method returns a [`Level`][level], it will be used as a hint to\n    /// determine the most verbose level that will be enabled. This will allow\n    /// spans and events which are more verbose than that level to be skipped\n    /// more efficiently. collectors which perform filtering are strongly\n    /// encouraged to provide an implementation of this method.\n    ///\n    /// If the maximum level the collector will enable can change over the\n    /// course of its lifetime, it is free to return a different value from\n    /// multiple invocations of this method. However, note that changes in the\n    /// maximum level will **only** be reflected after the callsite [`Interest`]\n    /// cache is rebuilt, by calling the [`callsite::rebuild_interest_cache`][rebuild]\n    /// function. Therefore, if the collector will change the value returned by\n    /// this method, it is responsible for ensuring that\n    /// [`rebuild_interest_cache`][rebuild] is called after the value of the max\n    /// level changes.\n    ///\n    /// [level]: super::Level\n    /// [rebuild]: super::callsite::rebuild_interest_cache\n    fn max_level_hint(&self) -> Option<LevelFilter> {\n        None\n    }\n\n    /// Visit the construction of a new span, returning a new [span ID] for the\n    /// span being constructed.\n    ///\n    /// The provided [`Attributes`] contains any field values that were provided\n    /// when the span was created. The collector may pass a [visitor] to the\n    /// `Attributes`' [`record` method] to record these values.\n    ///\n    /// IDs are used to uniquely identify spans and events within the context of a\n    /// collector, so span equality will be based on the returned ID. Thus, if\n    /// the collector wishes for all spans with the same metadata to be\n    /// considered equal, it should return the same ID every time it is given a\n    /// particular set of metadata. Similarly, if it wishes for two separate\n    /// instances of a span with the same metadata to *not* be equal, it should\n    /// return a distinct ID every time this function is called, regardless of\n    /// the metadata.\n    ///\n    /// Note that the collector is free to assign span IDs based on whatever\n    /// scheme it sees fit. Any guarantees about uniqueness, ordering, or ID\n    /// reuse are left up to the collector implementation to determine.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Attributes`]: super::span::Attributes\n    /// [visitor]: super::field::Visit\n    /// [`record` method]: super::span::Attributes::record\n    fn new_span(&self, span: &span::Attributes<'_>) -> span::Id;\n\n    // === Notification methods ===============================================\n\n    /// Record a set of values on a span.\n    ///\n    /// This method will be invoked when value is recorded on a span.\n    /// Recording multiple values for the same field is possible,\n    /// but the actual behaviour is defined by the collector implementation.\n    ///\n    /// Keep in mind that a span might not provide a value\n    /// for each field it declares.\n    ///\n    /// The collector is expected to provide a [visitor] to the `Record`'s\n    /// [`record` method] in order to record the added values.\n    ///\n    /// # Example\n    ///  \"foo = 3\" will be recorded when [`record`] is called on the\n    /// `Attributes` passed to `new_span`.\n    /// Since values are not provided for the `bar` and `baz` fields,\n    /// the span's `Metadata` will indicate that it _has_ those fields,\n    /// but values for them won't be recorded at this time.\n    ///\n    /// ```rust,ignore\n    /// # use tracing::span;\n    ///\n    /// let mut span = span!(\"my_span\", foo = 3, bar, baz);\n    ///\n    /// // `Collector::record` will be called with a `Record`\n    /// // containing \"bar = false\"\n    /// span.record(\"bar\", &false);\n    ///\n    /// // `Collector::record` will be called with a `Record`\n    /// // containing \"baz = \"a string\"\"\n    /// span.record(\"baz\", &\"a string\");\n    /// ```\n    ///\n    /// [visitor]: super::field::Visit\n    /// [`record`]: super::span::Attributes::record\n    /// [`record` method]: super::span::Record::record\n    fn record(&self, span: &span::Id, values: &span::Record<'_>);\n\n    /// Adds an indication that `span` follows from the span with the id\n    /// `follows`.\n    ///\n    /// This relationship differs somewhat from the parent-child relationship: a\n    /// span may have any number of prior spans, rather than a single one; and\n    /// spans are not considered to be executing _inside_ of the spans they\n    /// follow from. This means that a span may close even if subsequent spans\n    /// that follow from it are still open, and time spent inside of a\n    /// subsequent span should not be included in the time its precedents were\n    /// executing. This is used to model causal relationships such as when a\n    /// single future spawns several related background tasks, et cetera.\n    ///\n    /// If the collector has spans corresponding to the given IDs, it should\n    /// record this relationship in whatever way it deems necessary. Otherwise,\n    /// if one or both of the given span IDs do not correspond to spans that the\n    /// collector knows about, or if a cyclical relationship would be created\n    /// (i.e., some span _a_ which proceeds some other span _b_ may not also\n    /// follow from _b_), it may silently do nothing.\n    fn record_follows_from(&self, span: &span::Id, follows: &span::Id);\n\n    /// Determine if an [`Event`] should be recorded.\n    ///\n    /// By default, this returns `true` and collectors can filter events in\n    /// [`event`][Self::event] without any penalty. However, when `event` is\n    /// more complicated, this can be used to determine if `event` should be\n    /// called at all, separating out the decision from the processing.\n    fn event_enabled(&self, event: &Event<'_>) -> bool {\n        let _ = event;\n        true\n    }\n\n    /// Records that an [`Event`] has occurred.\n    ///\n    /// This method will be invoked when an Event is constructed by\n    /// the `Event`'s [`dispatch` method]. For example, this happens internally\n    /// when an event macro from `tracing` is called.\n    ///\n    /// The key difference between this method and `record` is that `record` is\n    /// called when a value is recorded for a field defined by a span,\n    /// while `event` is called when a new event occurs.\n    ///\n    /// The provided `Event` struct contains any field values attached to the\n    /// event. The collector may pass a [visitor] to the `Event`'s\n    /// [`record` method] to record these values.\n    ///\n    /// [`Event`]: super::event::Event\n    /// [visitor]: super::field::Visit\n    /// [`record` method]: super::event::Event::record\n    /// [`dispatch` method]: super::event::Event::dispatch\n    fn event(&self, event: &Event<'_>);\n\n    /// Records that a span has been entered.\n    ///\n    /// When entering a span, this method is called to notify the collector\n    /// that the span has been entered. The collector is provided with the\n    /// [span ID] of the entered span, and should update any internal state\n    /// tracking the current span accordingly.\n    ///\n    /// [span ID]: super::span::Id\n    fn enter(&self, span: &span::Id);\n\n    /// Records that a span has been exited.\n    ///\n    /// When exiting a span, this method is called to notify the collector\n    /// that the span has been exited. The collector is provided with the\n    /// [span ID] of the exited span, and should update any internal state\n    /// tracking the current span accordingly.\n    ///\n    /// Exiting a span does not imply that the span will not be re-entered.\n    ///\n    /// [span ID]: super::span::Id\n    fn exit(&self, span: &span::Id);\n\n    /// Notifies the collector that a [span ID] has been cloned.\n    ///\n    /// This function is guaranteed to only be called with span IDs that were\n    /// returned by this collector's `new_span` function.\n    ///\n    /// Note that the default implementation of this function this is just the\n    /// identity function, passing through the identifier. However, it can be\n    /// used in conjunction with [`try_close`] to track the number of handles\n    /// capable of `enter`ing a span. When all the handles have been dropped\n    /// (i.e., `try_close` has been called one more time than `clone_span` for a\n    /// given ID), the collector may assume that the span will not be entered\n    /// again. It is then free to deallocate storage for data associated with\n    /// that span, write data from that span to IO, and so on.\n    ///\n    /// For more unsafe situations, however, if `id` is itself a pointer of some\n    /// kind this can be used as a hook to \"clone\" the pointer, depending on\n    /// what that means for the specified pointer.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`try_close`]: Collect::try_close\n    fn clone_span(&self, id: &span::Id) -> span::Id {\n        id.clone()\n    }\n\n    /// **This method is deprecated.**\n    ///\n    /// Using `drop_span` may result in collectors composed using\n    /// `tracing-subscriber` crate's `Subscriber` trait from observing close events.\n    /// Use [`try_close`] instead.\n    ///\n    /// The default implementation of this function does nothing.\n    ///\n    /// [`try_close`]: Collect::try_close\n    #[deprecated(since = \"0.1.2\", note = \"use `Collector::try_close` instead\")]\n    fn drop_span(&self, _id: span::Id) {}\n\n    /// Notifies the collector that a [`span ID`] has been dropped, and returns\n    /// `true` if there are now 0 IDs that refer to that span.\n    ///\n    /// Higher-level libraries providing functionality for composing multiple\n...\n// Path: tracing-core/src/callsite.rs\n//! Callsites represent the source locations from which spans or events\n//! originate.\n//!\n//! # What Are Callsites?\n//!\n//! Every span or event in `tracing` is associated with a [`Callsite`]. A\n//! callsite is a small `static` value that is responsible for the following:\n//!\n//! * Storing the span or event's [`Metadata`],\n//! * Uniquely [identifying](Identifier) the span or event definition,\n//! * Caching the collector's [`Interest`][^1] in that span or event, to avoid\n//!   re-evaluating filters,\n//! * Storing a [`Registration`] that allows the callsite to be part of a global\n//!   list of all callsites in the program.\n//!\n//! # Registering Callsites\n//!\n//! When a span or event is recorded for the first time, its callsite\n//! [`register`]s itself with the global callsite registry. Registering a\n//! callsite calls the [`Collect::register_callsite`][`register_callsite`]\n//! method with that callsite's [`Metadata`] on every currently active\n//! collector. This serves two primary purposes: informing collectors of the\n//! callsite's existence, and performing static filtering.\n//!\n//! ## Callsite Existence\n//!\n//! If a [`Collect`] implementation wishes to allocate storage for each\n//! unique span/event location in the program, or pre-compute some value\n//! that will be used to record that span or event in the future, it can\n//! do so in its [`register_callsite`] method.\n//!\n//! ## Performing Static Filtering\n//!\n//! The [`register_callsite`] method returns an [`Interest`] value,\n//! which indicates that the collector either [always] wishes to record\n//! that span or event, [sometimes] wishes to record it based on a\n//! dynamic filter evaluation, or [never] wishes to record it.\n//!\n//! When registering a new callsite, the [`Interest`]s returned by every\n//! currently active collector are combined, and the result is stored at\n//! each callsite. This way, when the span or event occurs in the\n//! future, the cached [`Interest`] value can be checked efficiently\n//! to determine if the span or event should be recorded, without\n//! needing to perform expensive filtering (i.e. calling the\n//! [`Collect::enabled`] method every time a span or event occurs).\n//!\n//! ### Rebuilding Cached Interest\n//!\n//! When a new [`Dispatch`] is created (i.e. a new collector becomes\n//! active), any previously cached [`Interest`] values are re-evaluated\n//! for all callsites in the program. This way, if the new collector\n//! will enable a callsite that was not previously enabled, the\n//! [`Interest`] in that callsite is updated. Similarly, when a\n//! collector is dropped, the interest cache is also re-evaluated, so\n//! that any callsites enabled only by that collector are disabled.\n//!\n//! In addition, the [`rebuild_interest_cache`] function in this module can be\n//! used to manually invalidate all cached interest and re-register those\n//! callsites. This function is useful in situations where a collector's\n//! interest can change, but it does so relatively infrequently. The collector\n//! may wish for its interest to be cached most of the time, and return\n//! [`Interest::always`][always] or [`Interest::never`][never] in its\n//! [`register_callsite`] method, so that its [`Collect::enabled`] method\n//! doesn't need to be evaluated every time a span or event is recorded.\n//! However, when the configuration changes, the collector can call\n//! [`rebuild_interest_cache`] to re-evaluate the entire interest cache with its\n//! new configuration. This is a relatively costly operation, but if the\n//! configuration changes infrequently, it may be more efficient than calling\n//! [`Collect::enabled`] frequently.\n//!\n//! [^1]: Returned by the [`Collect::register_callsite`][`register_callsite`]\n//!     method.\n//!\n//! [`Metadata`]: crate::metadata::Metadata\n//! [`Interest`]: crate::collect::Interest\n//! [`Collect`]: crate::collect::Collect\n//! [`register_callsite`]: crate::collect::Collect::register_callsite\n//! [`Collect::enabled`]: crate::collect::Collect::enabled\n//! [always]: crate::collect::Interest::always\n//! [sometimes]: crate::collect::Interest::sometimes\n//! [never]: crate::collect::Interest::never\n//! [`Dispatch`]: crate::dispatch::Dispatch\nuse crate::{\n    collect::Interest,\n    dispatch::{self, Dispatch},\n    metadata::{LevelFilter, Metadata},\n};\nuse core::{\n    fmt,\n    hash::{Hash, Hasher},\n    ptr,\n    sync::atomic::{AtomicPtr, Ordering},\n};\n\ntype Callsites = LinkedList;\n\n/// Trait implemented by callsites.\n///\n/// These functions are only intended to be called by the callsite registry, which\n/// correctly handles determining the common interest between all collectors.\n///\n/// See the [module-level documentation](crate::callsite) for details on\n/// callsites.\npub trait Callsite: Sync {\n    /// Sets the [`Interest`] for this callsite.\n    ///\n    /// See the [documentation on callsite interest caching][cache-docs] for\n    /// details.\n    ///\n    /// [`Interest`]: super::collect::Interest\n    /// [cache-docs]: crate::callsite#performing-static-filtering\n    fn set_interest(&self, interest: Interest);\n\n    /// Returns the [metadata] associated with the callsite.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    ///\n    /// **Note:** Implementations of this method should not produce [`Metadata`]\n    /// that share the same callsite [`Identifier`] but otherwise differ in any\n    /// way (e.g., have different `name`s).\n    ///\n    /// </pre></div>\n    ///\n    /// [metadata]: super::metadata::Metadata\n    fn metadata(&self) -> &Metadata<'_>;\n}\n\n/// Uniquely identifies a [`Callsite`]\n///\n/// Two `Identifier`s are equal if they both refer to the same callsite.\n///\n/// [`Callsite`]: super::callsite::Callsite\n#[derive(Clone)]\npub struct Identifier(\n    /// **Warning**: The fields on this type are currently `pub` because it must\n    /// be able to be constructed statically by macros. However, when `const\n    /// fn`s are available on stable Rust, this will no longer be necessary.\n    /// Thus, these fields are *not* considered stable public API, and they may\n    /// change warning. Do not rely on any fields on `Identifier`. When\n    /// constructing new `Identifier`s, use the `identify_callsite!` macro or\n    /// the `Callsite::id` function instead.\n    // TODO: When `Callsite::id` is a const fn, this need no longer be `pub`.\n    #[doc(hidden)]\n    pub &'static dyn Callsite,\n);\n\n/// A registration with the callsite registry.\n///\n/// Every [`Callsite`] implementation must provide a `&'static Registration`\n/// when calling [`register`] to add itself to the global callsite registry.\n///\n/// See [the documentation on callsite registration][registry-docs] for details\n/// on how callsites are registered.\n///\n/// [`Callsite`]: crate::callsite::Callsite\n/// [`register`]: crate::callsite::register\n/// [registry-docs]: crate::callsite#registering-callsites\npub struct Registration<T = &'static dyn Callsite> {\n    callsite: T,\n    next: AtomicPtr<Registration<T>>,\n}\n\npub(crate) use self::inner::register_dispatch;\npub use self::inner::{rebuild_interest_cache, register};\n\n#[cfg(feature = \"std\")]\nmod inner {\n    use super::*;\n    use once_cell::sync::Lazy;\n    use std::sync::RwLock;\n    use std::vec::Vec;\n\n    type Dispatchers = Vec<dispatch::Registrar>;\n\n    struct Registry {\n        callsites: Callsites,\n        dispatchers: RwLock<Dispatchers>,\n    }\n\n    static REGISTRY: Lazy<Registry> = Lazy::new(|| Registry {\n        callsites: LinkedList::new(),\n        dispatchers: RwLock::new(Vec::new()),\n    });\n\n    /// Clear and reregister interest on every [`Callsite`]\n    ///\n    /// This function is intended for runtime reconfiguration of filters on traces\n    /// when the filter recalculation is much less frequent than trace events are.\n    /// The alternative is to have the [`Collect`] that supports runtime\n    /// reconfiguration of filters always return [`Interest::sometimes()`] so that\n    /// [`enabled`] is evaluated for every event.\n    ///\n    /// This function will also re-compute the global maximum level as determined by\n    /// the [`max_level_hint`] method. If a [`Collect`]\n    /// implementation changes the value returned by its `max_level_hint`\n    /// implementation at runtime, then it **must** call this function after that\n    /// value changes, in order for the change to be reflected.\n    ///\n    /// See the [documentation on callsite interest caching][cache-docs] for\n    /// additional information on this function's usage.\n    ///\n    /// [`max_level_hint`]: crate::collect::Collect::max_level_hint\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [`enabled`]: crate::collect::Collect::enabled\n    /// [`Interest::sometimes()`]: crate::collect::Interest::sometimes\n    /// [`Collect`]: crate::collect::Collect\n    /// [cache-docs]: crate::callsite#rebuilding-cached-interest\n    pub fn rebuild_interest_cache() {\n        let mut dispatchers = REGISTRY.dispatchers.write().unwrap();\n        let callsites = &REGISTRY.callsites;\n        rebuild_interest(callsites, &mut dispatchers);\n    }\n\n    /// Register a new [`Callsite`] with the global registry.\n    ///\n    /// This should be called once per callsite after the callsite has been\n    /// constructed.\n    ///\n    /// See the [documentation on callsite registration][reg-docs] for details\n    /// on the global callsite registry.\n    ///\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [reg-docs]: crate::callsite#registering-callsites\n    pub fn register(registration: &'static Registration) {\n        let dispatchers = REGISTRY.dispatchers.read().unwrap();\n        rebuild_callsite_interest(&dispatchers, registration.callsite);\n        REGISTRY.callsites.push(registration);\n    }\n\n    pub(crate) fn register_dispatch(dispatch: &Dispatch) {\n        let mut dispatchers = REGISTRY.dispatchers.write().unwrap();\n        let callsites = &REGISTRY.callsites;\n\n        dispatch.collector().on_register_dispatch(dispatch);\n        dispatchers.push(dispatch.registrar());\n\n        rebuild_interest(callsites, &mut dispatchers);\n    }\n\n    fn rebuild_callsite_interest(\n        dispatchers: &[dispatch::Registrar],\n        callsite: &'static dyn Callsite,\n    ) {\n        let meta = callsite.metadata();\n\n        // Iterate over the collectors in the registry, and \u2014 if they are\n        // active \u2014 register the callsite with them.\n        let mut interests = dispatchers.iter().filter_map(|registrar| {\n            registrar\n                .upgrade()\n                .map(|dispatch| dispatch.register_callsite(meta))\n        });\n\n        // Use the first collector's `Interest` as the base value.\n        let interest = if let Some(interest) = interests.next() {\n            // Combine all remaining `Interest`s.\n            interests.fold(interest, Interest::and)\n        } else {\n            // If nobody was interested in this thing, just return `never`.\n            Interest::never()\n        };\n\n        callsite.set_interest(interest)\n    }\n\n    fn rebuild_interest(callsites: &Callsites, dispatchers: &mut Vec<dispatch::Registrar>) {\n        let mut max_level = LevelFilter::OFF;\n        dispatchers.retain(|registrar| {\n            if let Some(dispatch) = registrar.upgrade() {\n                // If the collector did not provide a max level hint, assume\n                // that it may enable every level.\n                let level_hint = dispatch.max_level_hint().unwrap_or(LevelFilter::TRACE);\n                if level_hint > max_level {\n                    max_level = level_hint;\n                }\n                true\n            } else {\n                false\n            }\n        });\n\n        callsites.for_each(|reg| rebuild_callsite_interest(dispatchers, reg.callsite));\n\n        LevelFilter::set_max(max_level);\n    }\n}\n\n#[cfg(not(feature = \"std\"))]\nmod inner {\n    use super::*;\n    static REGISTRY: Callsites = LinkedList::new();\n\n    /// Clear and reregister interest on every [`Callsite`]\n    ///\n    /// This function is intended for runtime reconfiguration of filters on traces\n    /// when the filter recalculation is much less frequent than trace events are.\n    /// The alternative is to have the [collector] that supports runtime\n    /// reconfiguration of filters always return [`Interest::sometimes()`] so that\n    /// [`enabled`] is evaluated for every event.\n    ///\n    /// This function will also re-compute the global maximum level as determined by\n    /// the [`max_level_hint`] method. If a [`Collect`]\n    /// implementation changes the value returned by its `max_level_hint`\n    /// implementation at runtime, then it **must** call this function after that\n    /// value changes, in order for the change to be reflected.\n    ///\n    /// See the [documentation on callsite interest caching][cache-docs] for\n    /// additional information on this function's usage.\n    ///\n    /// [`max_level_hint`]: crate::collector::Collector::max_level_hint\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [`enabled`]: crate::collector::Collector::enabled\n    /// [`Interest::sometimes()`]: crate::collect::Interest::sometimes\n    /// [collector]: crate::collect::Collect\n    /// [`Collect`]: crate::collect::Collect\n    /// [cache-docs]: crate::callsite#rebuilding-cached-interest\n    pub fn rebuild_interest_cache() {\n        register_dispatch(dispatch::get_global());\n    }\n\n    /// Register a new [`Callsite`] with the global registry.\n    ///\n    /// This should be called once per callsite after the callsite has been\n    /// constructed.\n    ///\n    /// See the [documentation on callsite registration][reg-docs] for details\n    /// on the global callsite registry.\n    ///\n    /// [`Callsite`]: crate::callsite::Callsite\n    /// [reg-docs]: crate::callsite#registering-callsites\n    pub fn register(registration: &'static Registration) {\n        rebuild_callsite_interest(dispatch::get_global(), registration.callsite);\n        REGISTRY.push(registration);\n    }\n\n    pub(crate) fn register_dispatch(dispatcher: &Dispatch) {\n        // If the collector did not provide a max level hint, assume\n        // that it may enable every level.\n        let level_hint = dispatcher.max_level_hint().unwrap_or(LevelFilter::TRACE);\n\n        REGISTRY.for_each(|reg| rebuild_callsite_interest(dispatcher, reg.callsite));\n\n        LevelFilter::set_max(level_hint);\n    }\n\n    fn rebuild_callsite_interest(dispatcher: &Dispatch, callsite: &'static dyn Callsite) {\n        let meta = callsite.metadata();\n\n        callsite.set_interest(dispatcher.register_callsite(meta))\n    }\n}\n\n// ===== impl Identifier =====\n\nimpl PartialEq for Identifier {\n    fn eq(&self, other: &Identifier) -> bool {\n        core::ptr::eq(\n            self.0 as *const _ as *const (),\n            other.0 as *const _ as *const (),\n        )\n    }\n}\n\nimpl Eq for Identifier {}\n\nimpl fmt::Debug for Identifier {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"Identifier({:p})\", self.0)\n    }\n}\n\nimpl Hash for Identifier {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: Hasher,\n    {\n        (self.0 as *const dyn Callsite).hash(state)\n    }\n}\n\n// ===== impl Registration =====\n\nimpl<T> Registration<T> {\n    /// Construct a new `Registration` from some `&'static dyn Callsite`\n    pub const fn new(callsite: T) -> Self {\n        Self {\n            callsite,\n            next: AtomicPtr::new(ptr::null_mut()),\n        }\n    }\n}\n\nimpl fmt::Debug for Registration {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"Registration\")\n            .field(\"callsite\", &format_args!(\"{:p}\", self.callsite))\n            .field(\n                \"next\",\n                &format_args!(\"{:p}\", self.next.load(Ordering::Acquire)),\n            )\n            .finish()\n    }\n}\n\n// ===== impl LinkedList =====\n\n/// An intrusive atomic push-only linked list.\nstruct LinkedList<T = &'static dyn Callsite> {\n    head: AtomicPtr<Registration<T>>,\n}\n\nimpl<T> LinkedList<T> {\n    const fn new() -> Self {\n        LinkedList {\n            head: AtomicPtr::new(ptr::null_mut()),\n        }\n    }\n}\n\nimpl LinkedList {\n    fn for_each(&self, mut f: impl FnMut(&'static Registration)) {\n        let mut head = self.head.load(Ordering::Acquire);\n\n        while let Some(reg) = unsafe { head.as_ref() } {\n            f(reg);\n\n            head = reg.next.load(Ordering::Acquire);\n        }\n    }\n\n    fn push(&self, registration: &'static Registration) {\n        let mut head = self.head.load(Ordering::Acquire);\n\n        loop {\n            registration.next.store(head, Ordering::Release);\n\n            assert_ne!(\n                registration as *const _, head,\n                \"Attempted to register a `Callsite` that already exists! \\\n                This will cause an infinite loop when attempting to read from the \\\n                callsite cache. This is likely a bug! You should only need to call \\\n                `tracing-core::callsite::register` once per `Callsite`.\"\n            );\n\n            match self.head.compare_exchange(\n                head,\n                registration as *const _ as *mut _,\n                Ordering::AcqRel,\n                Ordering::Acquire,\n            ) {\n                Ok(_) => {\n                    break;\n                }\n                Err(current) => head = current,\n            }\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    struct TestCallsite;\n    static CS1: TestCallsite = TestCallsite;\n    static CS2: TestCallsite = TestCallsite;\n\n    impl Callsite for TestCallsite {\n        fn set_interest(&self, _interest: Interest) {}\n        fn metadata(&self) -> &Metadata<'_> {\n            unimplemented!(\"not needed for this test\")\n        }\n    }\n\n    #[test]\n    fn l\ninked_list_push() {\n        static REG1: Registration = Registration::new(&CS1);\n        static REG2: Registration = Registration::new(&CS2);\n\n        let linked_list = LinkedList::new();\n\n        linked_list.push(&REG1);\n        linked_list.push(&REG2);\n\n        let mut i = 0;\n\n        linked_list.for_each(|reg| {\n            if i == 0 {\n                assert!(\n                    ptr::eq(reg, &REG2),\n                    \"Registration pointers need to match REG2\"\n                );\n            } else {\n                assert!(\n                    ptr::eq(reg, &REG1),\n                    \"Registration pointers need to match REG1\"\n                );\n            }\n\n            i += 1;\n        });\n    }\n\n    #[test]\n    fn linked_list_push_several() {\n        static REG1: Registration = Registration::new(&CS1);\n        static REG2: Registration = Registration::new(&CS2);\n        static REG3: Registration = Registration::new(&CS1);\n        static REG4: Registration = Registration::new(&CS2);\n\n        let linked_list = LinkedList::new();\n\n        fn expect(\n            callsites: &mut impl Iterator<Item = &'static Registration>,\n        ) -> impl FnMut(&'static Registration) + '_ {\n            move |reg: &'static Registration| {\n                let ptr = callsites\n                    .next()\n                    .expect(\"list contained more than the expected number of registrations!\");\n\n                assert!(\n                    ptr::eq(reg, ptr),\n                    \"Registration pointers need to match ({:?} != {:?})\",\n                    reg,\n                    ptr\n                );\n            }\n        }\n\n        linked_list.push(&REG1);\n        linked_list.push(&REG2);\n        let regs = [&REG2, &REG1];\n        let mut callsites = regs.iter().copied();\n        linked_list.for_each(expect(&mut callsites));\n        assert!(\n            callsites.next().is_none(),\n            \"some registrations were expected but not present: {:?}\",\n            callsites\n        );\n\n        linked_list.push(&REG3);\n        let regs = [&REG3, &REG2, &REG1];\n        let mut callsites = regs.iter().copied();\n        linked_list.for_each(expect(&mut callsites));\n        assert!(\n            callsites.next().is_none(),\n            \"some registrations were expected but not present: {:?}\",\n            callsites\n        );\n\n        linked_list.push(&REG4);\n        let regs = [&REG4, &REG3, &REG2, &REG1];\n        let mut callsites = regs.iter().copied();\n        linked_list.for_each(expect(&mut callsites));\n        assert!(\n            callsites.next().is_none(),\n            \"some registrations were expected but not present: {:?}\",\n            callsites\n        );\n    }\n\n    #[test]\n    #[should_panic]\n    fn linked_list_repeated() {\n        static REG1: Registration = Registration::new(&CS1);\n\n        let linked_list = LinkedList::new();\n\n        linked_list.push(&REG1);\n        // Pass in same reg and we should panic...\n        linked_list.push(&REG1);\n\n        linked_list.for_each(|_| {});\n    }\n\n    #[test]\n    fn linked_list_empty() {\n        let linked_list = LinkedList::new();\n\n        linked_list.for_each(|_| {\n            panic!(\"List should be empty\");\n        });\n    }\n}\n\n// Path: tracing-core/src/metadata.rs\n//! Metadata describing trace data.\nuse super::{callsite, field};\nuse core::{\n    cmp, fmt,\n    str::FromStr,\n    sync::atomic::{AtomicUsize, Ordering},\n};\n\n/// Metadata describing a [span] or [event].\n///\n/// All spans and events have the following metadata:\n/// - A [name], represented as a static string.\n/// - A [target], a string that categorizes part of the system where the span\n///   or event occurred. The `tracing` macros default to using the module\n///   path where the span or event originated as the target, but it may be\n///   overridden.\n/// - A [verbosity level]. This determines how verbose a given span or event\n///   is, and allows enabling or disabling more verbose diagnostics\n///   situationally. See the documentation for the [`Level`] type for details.\n/// - The names of the [fields] defined by the span or event.\n/// - Whether the metadata corresponds to a span or event.\n///\n/// In addition, the following optional metadata describing the source code\n/// location where the span or event originated _may_ be provided:\n/// - The [file name]\n/// - The [line number]\n/// - The [module path]\n///\n/// Metadata is used by [collector]s when filtering spans and events, and it\n/// may also be used as part of their data payload.\n///\n/// When created by the `event!` or `span!` macro, the metadata describing a\n/// particular event or span is constructed statically and exists as a single\n/// static instance. Thus, the overhead of creating the metadata is\n/// _significantly_ lower than that of creating the actual span. Therefore,\n/// filtering is based on metadata, rather than on the constructed span.\n///\n/// ## Equality\n///\n/// In well-behaved applications, two `Metadata` with equal\n/// [callsite identifiers] will be equal in all other ways (i.e., have the same\n/// `name`, `target`, etc.). Consequently, in release builds, [`Metadata::eq`]\n/// *only* checks that its arguments have equal callsites. However, the equality\n/// of `Metadata`'s other fields is checked in debug builds.\n///\n/// [span]: super::span\n/// [event]: super::event\n/// [name]: Self::name\n/// [target]: Self::target\n/// [fields]: Self::fields\n/// [verbosity level]: Self::level\n/// [file name]: Self::file\n/// [line number]: Self::line\n/// [module path]: Self::module_path\n/// [collector]: super::collect::Collect\n/// [callsite identifiers]: Self::callsite\npub struct Metadata<'a> {\n    /// The name of the span described by this metadata.\n    name: &'static str,\n\n    /// The part of the system that the span that this metadata describes\n    /// occurred in.\n    target: &'a str,\n\n    /// The level of verbosity of the described span.\n    level: Level,\n\n    /// The name of the Rust module where the span occurred, or `None` if this\n    /// could not be determined.\n    module_path: Option<&'a str>,\n\n    /// The name of the source code file where the span occurred, or `None` if\n    /// this could not be determined.\n    file: Option<&'a str>,\n\n    /// The line number in the source code file where the span occurred, or\n    /// `None` if this could not be determined.\n    line: Option<u32>,\n\n    /// The names of the key-value fields attached to the described span or\n    /// event.\n    fields: field::FieldSet,\n\n    /// The kind of the callsite.\n    kind: Kind,\n}\n\n/// Indicates whether the callsite is a span or event.\n#[derive(Clone, Eq, PartialEq)]\npub struct Kind(u8);\n\n/// Describes the level of verbosity of a span or event.\n///\n/// # Comparing Levels\n///\n/// `Level` implements the [`PartialOrd`] and [`Ord`] traits, allowing two\n/// `Level`s to be compared to determine which is considered more or less\n/// verbose. Levels which are more verbose are considered \"greater than\" levels\n/// which are less verbose, with [`Level::ERROR`] considered the lowest, and\n/// [`Level::TRACE`] considered the highest.\n///\n/// For example:\n/// ```\n/// use tracing_core::Level;\n///\n/// assert!(Level::TRACE > Level::DEBUG);\n/// assert!(Level::ERROR < Level::WARN);\n/// assert!(Level::INFO <= Level::DEBUG);\n/// assert_eq!(Level::TRACE, Level::TRACE);\n/// ```\n///\n/// # Filtering\n///\n/// `Level`s are typically used to implement filtering that determines which\n/// spans and events are enabled. Depending on the use case, more or less\n/// verbose diagnostics may be desired. For example, when running in\n/// development, [`DEBUG`]-level traces may be enabled by default. When running in\n/// production, only [`INFO`]-level and lower traces might be enabled. Libraries\n/// may include very verbose diagnostics at the [`DEBUG`] and/or [`TRACE`] levels.\n/// Applications using those libraries typically chose to ignore those traces. However, when\n/// debugging an issue involving said libraries, it may be useful to temporarily\n/// enable the more verbose traces.\n///\n/// The [`LevelFilter`] type is provided to enable filtering traces by\n/// verbosity. `Level`s can be compared against [`LevelFilter`]s, and\n/// [`LevelFilter`] has a variant for each `Level`, which compares analogously\n/// to that level. In addition, [`LevelFilter`] adds a [`LevelFilter::OFF`]\n/// variant, which is considered \"less verbose\" than every other `Level`. This is\n/// intended to allow filters to completely disable tracing in a particular context.\n///\n/// For example:\n/// ```\n/// use tracing_core::{Level, LevelFilter};\n///\n/// assert!(LevelFilter::OFF < Level::TRACE);\n/// assert!(LevelFilter::TRACE > Level::DEBUG);\n/// assert!(LevelFilter::ERROR < Level::WARN);\n/// assert!(LevelFilter::INFO <= Level::DEBUG);\n/// assert!(LevelFilter::INFO >= Level::INFO);\n/// ```\n///\n/// ## Examples\n///\n/// Below is a simple example of how a [collector] could implement filtering through\n/// a [`LevelFilter`]. When a span or event is recorded, the [`Collect::enabled`] method\n/// compares the span or event's `Level` against the configured [`LevelFilter`].\n/// The optional [`Collect::max_level_hint`] method can also be implemented to  allow spans\n/// and events above a maximum verbosity level to be skipped more efficiently,\n/// often improving performance in short-lived programs.\n///\n/// ```\n/// use tracing_core::{span, Event, Level, LevelFilter, Collect, Metadata};\n/// # use tracing_core::span::{Id, Record, Current};\n///\n/// #[derive(Debug)]\n/// pub struct MyCollector {\n///     /// The most verbose level that this collector will enable.\n///     max_level: LevelFilter,\n///\n///     // ...\n/// }\n///\n/// impl MyCollector {\n///     /// Returns a new `MyCollector` which will record spans and events up to\n///     /// `max_level`.\n///     pub fn with_max_level(max_level: LevelFilter) -> Self {\n///         Self {\n///             max_level,\n///             // ...\n///         }\n///     }\n/// }\n/// impl Collect for MyCollector {\n///     fn enabled(&self, meta: &Metadata<'_>) -> bool {\n///         // A span or event is enabled if it is at or below the configured\n///         // maximum level.\n///         meta.level() <= &self.max_level\n///     }\n///\n///     // This optional method returns the most verbose level that this\n///     // collector will enable. Although implementing this method is not\n///     // *required*, it permits additional optimizations when it is provided,\n///     // allowing spans and events above the max level to be skipped\n///     // more efficiently.\n///     fn max_level_hint(&self) -> Option<LevelFilter> {\n///         Some(self.max_level)\n///     }\n///\n///     // Implement the rest of the collector...\n///     fn new_span(&self, span: &span::Attributes<'_>) -> span::Id {\n///         // ...\n///         # drop(span); Id::from_u64(1)\n///     }\n\n///     fn event(&self, event: &Event<'_>) {\n///         // ...\n///         # drop(event);\n///     }\n///\n///     // ...\n///     # fn enter(&self, _: &Id) {}\n///     # fn exit(&self, _: &Id) {}\n///     # fn record(&self, _: &Id, _: &Record<'_>) {}\n///     # fn record_follows_from(&self, _: &Id, _: &Id) {}\n///     # fn current_span(&self) -> Current { Current::unknown() }\n/// }\n/// ```\n///\n/// It is worth noting that the `tracing-subscriber` crate provides [additional\n/// APIs][envfilter] for performing more sophisticated filtering, such as\n/// enabling different levels based on which module or crate a span or event is\n/// recorded in.\n///\n/// [`DEBUG`]: Level::DEBUG\n/// [`INFO`]: Level::INFO\n/// [`TRACE`]: Level::TRACE\n/// [`Collect::enabled`]: crate::collect::Collect::enabled\n/// [`Collect::max_level_hint`]: crate::collect::Collect::max_level_hint\n/// [collector]: crate::collect::Collect\n/// [envfilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html\n#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]\npub struct Level(LevelInner);\n\n/// A filter comparable to a verbosity [`Level`].\n///\n/// If a [`Level`] is considered less than or equal to a `LevelFilter`, it\n/// should be considered enabled; if greater than the `LevelFilter`, that level\n/// is disabled. See [`LevelFilter::current`] for more details.\n///\n/// Note that this is essentially identical to the `Level` type, but with the\n/// addition of an [`OFF`] level that completely disables all trace\n/// instrumentation.\n///\n/// See the documentation for the [`Level`] type to see how `Level`s\n/// and `LevelFilter`s interact.\n///\n/// [`OFF`]: LevelFilter::OFF\n#[repr(transparent)]\n#[derive(Copy, Clone, Eq, PartialEq, Hash)]\npub struct LevelFilter(Option<Level>);\n\n/// Indicates that a string could not be parsed to a valid level.\n#[derive(Clone, Debug)]\npub struct ParseLevelFilterError(());\n\nstatic MAX_LEVEL: AtomicUsize = AtomicUsize::new(LevelFilter::OFF_USIZE);\n\n// ===== impl Metadata =====\n\nimpl<'a> Metadata<'a> {\n    /// Construct new metadata for a span or event, with a name, target, level, field\n    /// names, and optional source code location.\n    pub const fn new(\n        name: &'static str,\n        target: &'a str,\n        level: Level,\n        file: Option<&'a str>,\n        line: Option<u32>,\n        module_path: Option<&'a str>,\n        fields: field::FieldSet,\n        kind: Kind,\n    ) -> Self {\n        Metadata {\n            name,\n            target,\n            level,\n            module_path,\n            file,\n            line,\n            fields,\n            kind,\n        }\n    }\n\n    /// Returns the names of the fields on the described span or event.\n    #[inline]\n    pub fn fields(&self) -> &field::FieldSet {\n        &self.fields\n    }\n\n    /// Returns the level of verbosity of the described span or event.\n    pub fn level(&self) -> &Level {\n        &self.level\n    }\n\n    /// Returns the name of the span.\n    pub fn name(&self) -> &'static str {\n        self.name\n    }\n\n    /// Returns a string describing the part of the system where the span or\n    /// event that this metadata describes occurred.\n    ///\n    /// Typically, this is the module path, but alternate targets may be set\n    /// when spans or events are constructed.\n    pub fn target(&self) -> &'a str {\n        self.target\n    }\n\n    /// Returns the path to the Rust module where the span occurred, or\n    /// `None` if the module path is unknown.\n    pub fn module_path(&self) -> Option<&'a str> {\n        self.module_path\n    }\n\n    /// Returns the name of the source code file where the span\n    /// occurred, or `None` if the file is unknown\n    pub fn file(&self) -> Option<&'a str> {\n        self.file\n    }\n\n    /// Returns the line number in the source code file where the span\n    /// occurred, or `None` if the line number is unknown.\n    pub fn line(&self) -> Option<u32> {\n        self.line\n    }\n\n    /// Returns an opaque `Identifier` that uniquely identifies the callsite\n    /// this `Metadata` originated from.\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Returns true if the callsite kind is `Event`.\n    pub fn is_event(&self) -> bool {\n        self.kind.is_event()\n    }\n\n    /// Return true if the callsite kind is `Span`.\n    pub fn is_span(&self) -> bool {\n        self.kind.is_span()\n    }\n}\n\nimpl<'a> fmt::Debug for Metadata<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        let mut meta = f.debug_struct(\"Metadata\");\n        meta.field(\"name\", &self.name)\n            .field(\"target\", &self.target)\n            .field(\"level\", &self.level);\n\n        if let Some(path) = self.module_path() {\n            meta.field(\"module_path\", &path);\n        }\n\n        match (self.file(), self.line()) {\n            (Some(file), Some(line)) => {\n                meta.field(\"location\", &format_args!(\"{}:{}\", file, line));\n            }\n            (Some(file), None) => {\n                meta.field(\"file\", &format_args!(\"{}\", file));\n            }\n\n            // Note: a line num with no file is a kind of weird case that _probably_ never occurs...\n            (None, Some(line)) => {\n                meta.field(\"line\", &line);\n            }\n            (None, None) => {}\n        };\n\n        meta.field(\"fields\", &format_args!(\"{}\", self.fields))\n            .field(\"callsite\", &self.callsite())\n            .field(\"kind\", &self.kind)\n            .finish()\n    }\n}\n\nimpl Kind {\n    const EVENT_BIT: u8 = 1 << 0;\n    const SPAN_BIT: u8 = 1 << 1;\n    const HINT_BIT: u8 = 1 << 2;\n\n    /// `Event` callsite\n    pub const EVENT: Kind = Kind(Self::EVENT_BIT);\n\n    /// `Span` callsite\n    pub const SPAN: Kind = Kind(Self::SPAN_BIT);\n\n    /// `enabled!` callsite. [`Collect`][`crate::collect::Collect`]s can assume\n    /// this `Kind` means they will never receive a\n    /// full event with this [`Metadata`].\n    pub const HINT: Kind = Kind(Self::HINT_BIT);\n\n    /// Return true if the callsite kind is `Span`\n    pub fn is_span(&self) -> bool {\n        self.0 & Self::SPAN_BIT == Self::SPAN_BIT\n    }\n\n    /// Return true if the callsite kind is `Event`\n    pub fn is_event(&self) -> bool {\n        self.0 & Self::EVENT_BIT == Self::EVENT_BIT\n    }\n\n    /// Return true if the callsite kind is `Hint`\n    pub fn is_hint(&self) -> bool {\n        self.0 & Self::HINT_BIT == Self::HINT_BIT\n    }\n\n    /// Sets that this `Kind` is a [hint](Self::HINT).\n    ///\n    /// This can be called on [`SPAN`](Self::SPAN) and [`EVENT`](Self::EVENT)\n    /// kinds to construct a hint callsite that also counts as a span or event.\n    pub const fn hint(self) -> Self {\n        Self(self.0 | Self::HINT_BIT)\n    }\n}\n\nimpl fmt::Debug for Kind {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.write_str(\"Kind(\")?;\n        let mut has_bits = false;\n        let mut write_bit = |name: &str| {\n            if has_bits {\n                f.write_str(\" | \")?;\n            }\n            f.write_str(name)?;\n            has_bits = true;\n            Ok(())\n        };\n\n        if self.is_event() {\n            write_bit(\"EVENT\")?;\n        }\n\n        if self.is_span() {\n            write_bit(\"SPAN\")?;\n        }\n\n        if self.is_hint() {\n            write_bit(\"HINT\")?;\n        }\n\n        // if none of the expected bits were set, something is messed up, so\n        // just print the bits for debugging purposes\n        if !has_bits {\n            write!(f, \"{:#b}\", self.0)?;\n        }\n\n        f.write_str(\")\")\n    }\n}\n\nimpl<'a> Eq for Metadata<'a> {}\n\nimpl<'a> PartialEq for Metadata<'a> {\n    #[inline]\n    fn eq(&self, other: &Self) -> bool {\n        if core::ptr::eq(&self, &other) {\n            true\n        } else if cfg!(not(debug_assertions)) {\n            // In a well-behaving application, two `Metadata` can be assumed to\n            // be totally equal so long as they share the same callsite.\n            self.callsite() == other.callsite()\n        } else {\n            // However, when debug-assertions are enabled, do not assume that\n            // the application is well-behaving; check every field of `Metadata`\n            // for equality.\n\n            // `Metadata` is destructured here to ensure a compile-error if the\n            // fields of `Metadata` change.\n            let Metadata {\n                name: lhs_name,\n                target: lhs_target,\n                level: lhs_level,\n                module_path: lhs_module_path,\n                file: lhs_file,\n                line: lhs_line,\n                fields: lhs_fields,\n                kind: lhs_kind,\n            } = self;\n\n            let Metadata {\n                name: rhs_name,\n                target: rhs_target,\n                level: rhs_level,\n                module_path: rhs_module_path,\n                file: rhs_file,\n                line: rhs_line,\n                fields: rhs_fields,\n                kind: rhs_kind,\n            } = &other;\n\n            // The initial comparison of callsites is purely an optimization;\n            // it can be removed without affecting the overall semantics of the\n            // expression.\n...\n", "needle_token_start": 10474, "needle_token_end": 10679, "code_context_ntokens": 16247, "output": ["```rust\n// ===== impl LinkedList =====\n\n/// An intrusive atomic push-only linked list.\nstruct LinkedList<T = &'static dyn Callsite> {\n    head: AtomicPtr<Registration<T>>,\n}\n\nimpl<T> LinkedList<T> {\n    const fn new() -> Self {\n        LinkedList {\n            head: AtomicPtr::new(ptr::null_mut()),\n        }\n    }\n}\n\nimpl LinkedList {\n    fn for_each(&self, mut f: impl FnMut(&'static Registration)) {\n        let mut head = self.head.load(Ordering::Acquire);\n\n        while let Some(reg) = unsafe { head.as_ref() } {\n            f(reg);\n\n            head = reg.next.load(Ordering::Acquire);\n        }\n    }\n\n    fn push(&self, registration: &'static Registration) {\n        let mut head = self.head.load(Ordering::Acquire);\n\n        loop {\n            registration.next.store(head, Ordering::Release);\n\n            assert_ne!(\n                registration as *const _, head,\n                \"Attempted to register a `Callsite` that already exists! \\\n                This will cause an infinite loop when attempting to read from the \\\n                callsite cache. This is likely a bug! You should only need to call \\\n                `tracing-core::callsite::register` once per `Callsite`.\"\n            );\n\n            match self.head.compare_exchange(\n                head,\n                registration as *const _ as *mut _,\n                Ordering::AcqRel,\n                Ordering::Acquire,\n            ) {\n                Ok(_) => {\n                    break;\n                }\n                Err(current) => head = current,\n            }\n        }\n    }\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "into_u64", "language": "rust", "path": "tracing-core/src/span.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve the numerical identifier of a span in a tracing system, which is stored in a specialized non-zero format to ensure validity.\n2. **Input**: The function does not take any parameters; it operates on the internal state of the object it is called on.\n3. **Output**: Returns a positive integer that uniquely identifies a span within the context of a collector.\n4. **Procedure**: It accesses the encapsulated non-zero integer value and returns its underlying primitive integer representation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: tracing-core/src/field.rs\n//! Span and `Event` key-value data.\n//!\n//! Spans and events may be annotated with key-value data, known as _fields_.\n//! These fields consist of a mapping from a key (corresponding to a `&str` but\n//! represented internally as an array index) to a [`Value`].\n//!\n//! # `Value`s and `Collect`s\n//!\n//! Collectors consume `Value`s as fields attached to [span]s or [`Event`]s.\n//! The set of field keys on a given span or event is defined on its [`Metadata`].\n//! When a span is created, it provides [`Attributes`] to the collector's\n//! [`new_span`] method, containing any fields whose values were provided when\n//! the span was created; and may call the collector's [`record`] method\n//! with additional [`Record`]s if values are added for more of its fields.\n//! Similarly, the [`Event`] type passed to the collector's [`event`] method\n//! will contain any fields attached to each event.\n//!\n//! `tracing` represents values as either one of a set of Rust primitives\n//! (`i64`, `u64`, `f64`, `i128`, `u128`, `bool`, and `&str`) or using a\n//! `fmt::Display` or `fmt::Debug` implementation. Collectors are provided\n//! these primitive value types as `dyn Value` trait objects.\n//!\n//! These trait objects can be formatted using `fmt::Debug`, but may also be\n//! recorded as typed data by calling the [`Value::record`] method on these\n//! trait objects with a _visitor_ implementing the [`Visit`] trait. This trait\n//! represents the behavior used to record values of various types. For example,\n//! we might record integers by incrementing counters for their field names,\n//! rather than printing them.\n//!\n//! [span]: super::span\n//! [`Event`]: super::event::Event\n//! [`Metadata`]: super::metadata::Metadata\n//! [`Attributes`]:  super::span::Attributes\n//! [`Record`]: super::span::Record\n//! [`new_span`]: super::collect::Collect::new_span\n//! [`record`]: super::collect::Collect::record\n//! [`event`]:  super::collect::Collect::event\nuse crate::callsite;\nuse core::{\n    borrow::Borrow,\n    fmt,\n    hash::{Hash, Hasher},\n    num,\n    ops::Range,\n};\n\nuse self::private::ValidLen;\n\n/// An opaque key allowing _O_(1) access to a field in a `Span`'s key-value\n/// data.\n///\n/// As keys are defined by the _metadata_ of a span, rather than by an\n/// individual instance of a span, a key may be used to access the same field\n/// across all instances of a given span with the same metadata. Thus, when a\n/// collector observes a new span, it need only access a field by name _once_,\n/// and use the key for that name for all other accesses.\n#[derive(Debug)]\npub struct Field {\n    i: usize,\n    fields: FieldSet,\n}\n\n/// An empty field.\n///\n/// This can be used to indicate that the value of a field is not currently\n/// present but will be recorded later.\n///\n/// When a field's value is `Empty`. it will not be recorded.\n#[derive(Debug, Eq, PartialEq)]\npub struct Empty;\n\n/// Describes the fields present on a span.\n///\n/// ## Equality\n///\n/// In well-behaved applications, two `FieldSet`s [initialized] with equal\n/// [callsite identifiers] will have identical fields. Consequently, in release\n/// builds, [`FieldSet::eq`] *only* checks that its arguments have equal\n/// callsites. However, the equality of field names is checked in debug builds.\n///\n/// [initialized]: Self::new\n/// [callsite identifiers]: callsite::Identifier\npub struct FieldSet {\n    /// The names of each field on the described span.\n    names: &'static [&'static str],\n    /// The callsite where the described span originates.\n    callsite: callsite::Identifier,\n}\n\n/// A set of fields and values for a span.\npub struct ValueSet<'a> {\n    values: &'a [(&'a Field, Option<&'a (dyn Value + 'a)>)],\n    fields: &'a FieldSet,\n}\n\n/// An iterator over a set of fields.\n#[derive(Debug)]\npub struct Iter {\n    idxs: Range<usize>,\n    fields: FieldSet,\n}\n\n/// Visits typed values.\n///\n/// An instance of `Visit` (\"a visitor\") represents the logic necessary to\n/// record field values of various types. When an implementor of [`Value`] is\n/// [recorded], it calls the appropriate method on the provided visitor to\n/// indicate the type that value should be recorded as.\n///\n/// When a [`Collect`] implementation [records an `Event`] or a\n/// [set of `Value`s added to a `Span`], it can pass an `&mut Visit` to the\n/// `record` method on the provided [`ValueSet`] or [`Event`]. This visitor\n/// will then be used to record all the field-value pairs present on that\n/// `Event` or `ValueSet`.\n///\n/// # Examples\n///\n/// A simple visitor that writes to a string might be implemented like so:\n/// ```\n/// # extern crate tracing_core as tracing;\n/// use std::fmt::{self, Write};\n/// use tracing::field::{Value, Visit, Field};\n/// pub struct StringVisitor<'a> {\n///     string: &'a mut String,\n/// }\n///\n/// impl<'a> Visit for StringVisitor<'a> {\n///     fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n///         write!(self.string, \"{} = {:?}; \", field.name(), value).unwrap();\n///     }\n/// }\n/// ```\n/// This visitor will format each recorded value using `fmt::Debug`, and\n/// append the field name and formatted value to the provided string,\n/// regardless of the type of the recorded value. When all the values have\n/// been recorded, the `StringVisitor` may be dropped, allowing the string\n/// to be printed or stored in some other data structure.\n///\n/// The `Visit` trait provides default implementations for `record_i64`,\n/// `record_u64`, `record_bool`, `record_str`, and `record_error`, which simply\n/// forward the recorded value to `record_debug`. Thus, `record_debug` is the\n/// only method which a `Visit` implementation *must* implement. However,\n/// visitors may override the default implementations of these functions in\n/// order to implement type-specific behavior.\n///\n/// Additionally, when a visitor receives a value of a type it does not care\n/// about, it is free to ignore those values completely. For example, a\n/// visitor which only records numeric data might look like this:\n///\n/// ```\n/// # extern crate tracing_core as tracing;\n/// # use std::fmt::{self, Write};\n/// # use tracing::field::{Value, Visit, Field};\n/// pub struct SumVisitor {\n///     sum: i64,\n/// }\n///\n/// impl Visit for SumVisitor {\n///     fn record_i64(&mut self, _field: &Field, value: i64) {\n///        self.sum += value;\n///     }\n///\n///     fn record_u64(&mut self, _field: &Field, value: u64) {\n///         self.sum += value as i64;\n///     }\n///\n///     fn record_debug(&mut self, _field: &Field, _value: &dyn fmt::Debug) {\n///         // Do nothing\n///     }\n/// }\n/// ```\n///\n/// This visitor (which is probably not particularly useful) keeps a running\n/// sum of all the numeric values it records, and ignores all other values. A\n/// more practical example of recording typed values is presented in\n/// `examples/counters.rs`, which demonstrates a very simple metrics system\n/// implemented using `tracing`.\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: The <code>record_error</code> trait method is only\n/// available when the Rust standard library is present, as it requires the\n/// <code>std::error::Error</code> trait.\n/// </pre></div>\n///\n/// [recorded]: Value::record\n/// [`Collect`]: super::collect::Collect\n/// [records an `Event`]: super::collect::Collect::event\n/// [set of `Value`s added to a `Span`]: super::collect::Collect::record\n/// [`Event`]: super::event::Event\npub trait Visit {\n    /// Visit a double-precision floating point value.\n    fn record_f64(&mut self, field: &Field, value: f64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 64-bit integer value.\n    fn record_i64(&mut self, field: &Field, value: i64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 64-bit integer value.\n    fn record_u64(&mut self, field: &Field, value: u64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 128-bit integer value.\n    fn record_i128(&mut self, field: &Field, value: i128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 128-bit integer value.\n    fn record_u128(&mut self, field: &Field, value: u128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a boolean value.\n    fn record_bool(&mut self, field: &Field, value: bool) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a string value.\n    fn record_str(&mut self, field: &Field, value: &str) {\n        self.record_debug(field, &value)\n    }\n\n    /// Records a type implementing `Error`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: This is only enabled when the Rust standard library is\n    /// present.\n    /// </pre>\n    /// </div>\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n    fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) {\n        self.record_debug(field, &DisplayValue(value))\n    }\n\n    /// Visit a value implementing `fmt::Debug`.\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug);\n}\n\n/// A field value of an erased type.\n///\n/// Implementors of `Value` may call the appropriate typed recording methods on\n/// the [visitor] passed to their `record` method in order to indicate how\n/// their data should be recorded.\n///\n/// [visitor]: Visit\npub trait Value: crate::sealed::Sealed {\n    /// Visits this value with the given `Visitor`.\n    fn record(&self, key: &Field, visitor: &mut dyn Visit);\n}\n\n/// A `Value` which serializes using `fmt::Display`.\n///\n/// Uses `record_debug` in the `Value` implementation to\n/// avoid an unnecessary evaluation.\n#[derive(Clone)]\npub struct DisplayValue<T: fmt::Display>(T);\n\n/// A `Value` which serializes as a string using `fmt::Debug`.\n#[derive(Clone)]\npub struct DebugValue<T: fmt::Debug>(T);\n\n/// Wraps a type implementing `fmt::Display` as a `Value` that can be\n/// recorded using its `Display` implementation.\npub fn display<T>(t: T) -> DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    DisplayValue(t)\n}\n\n/// Wraps a type implementing `fmt::Debug` as a `Value` that can be\n/// recorded using its `Debug` implementation.\npub fn debug<T>(t: T) -> DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    DebugValue(t)\n}\n\n// ===== impl Visit =====\n\nimpl<'a, 'b> Visit for fmt::DebugStruct<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.field(field.name(), value);\n    }\n}\n\nimpl<'a, 'b> Visit for fmt::DebugMap<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.entry(&format_args!(\"{}\", field), value);\n    }\n}\n\nimpl<F> Visit for F\nwhere\n    F: FnMut(&Field, &dyn fmt::Debug),\n{\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        (self)(field, value)\n    }\n}\n\n// ===== impl Value =====\n\nmacro_rules! impl_values {\n    ( $( $record:ident( $( $whatever:tt)+ ) ),+ ) => {\n        $(\n            impl_value!{ $record( $( $whatever )+ ) }\n        )+\n    }\n}\n\nmacro_rules! ty_to_nonzero {\n    (u8) => {\n        NonZeroU8\n    };\n    (u16) => {\n        NonZeroU16\n    };\n    (u32) => {\n        NonZeroU32\n    };\n    (u64) => {\n        NonZeroU64\n    };\n    (u128) => {\n        NonZeroU128\n    };\n    (usize) => {\n        NonZeroUsize\n    };\n    (i8) => {\n        NonZeroI8\n    };\n    (i16) => {\n        NonZeroI16\n    };\n    (i32) => {\n        NonZeroI32\n    };\n    (i64) => {\n        NonZeroI64\n    };\n    (i128) => {\n        NonZeroI128\n    };\n    (isize) => {\n        NonZeroIsize\n    };\n}\n\nmacro_rules! impl_one_value {\n    (f32, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f32, $op, $record);\n    };\n    (f64, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f64, $op, $record);\n    };\n    (bool, $op:expr, $record:ident) => {\n        impl_one_value!(normal, bool, $op, $record);\n    };\n    ($value_ty:tt, $op:expr, $record:ident) => {\n        impl_one_value!(normal, $value_ty, $op, $record);\n        impl_one_value!(nonzero, $value_ty, $op, $record);\n    };\n    (normal, $value_ty:tt, $op:expr, $record:ident) => {\n        impl $crate::sealed::Sealed for $value_ty {}\n        impl $crate::field::Value for $value_ty {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(*self))\n            }\n        }\n    };\n    (nonzero, $value_ty:tt, $op:expr, $record:ident) => {\n        // This `use num::*;` is reported as unused because it gets emitted\n        // for every single invocation of this macro, so there are multiple `use`s.\n        // All but the first are useless indeed.\n        // We need this import because we can't write a path where one part is\n        // the `ty_to_nonzero!($value_ty)` invocation.\n        #[allow(clippy::useless_attribute, unused)]\n        use num::*;\n        impl $crate::sealed::Sealed for ty_to_nonzero!($value_ty) {}\n        impl $crate::field::Value for ty_to_nonzero!($value_ty) {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(self.get()))\n            }\n        }\n    };\n}\n\nmacro_rules! impl_value {\n    ( $record:ident( $( $value_ty:tt ),+ ) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this, $record);\n        )+\n    };\n    ( $record:ident( $( $value_ty:tt ),+ as $as_ty:ty) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this as $as_ty, $record);\n        )+\n    };\n}\n\n// ===== impl Value =====\n\nimpl_values! {\n    record_u64(u64),\n    record_u64(usize, u32, u16, u8 as u64),\n    record_i64(i64),\n    record_i64(isize, i32, i16, i8 as i64),\n    record_u128(u128),\n    record_i128(i128),\n    record_bool(bool),\n    record_f64(f64, f32 as f64)\n}\n\nimpl<T: crate::sealed::Sealed> crate::sealed::Sealed for Wrapping<T> {}\nimpl<T: crate::field::Value> crate::field::Value for Wrapping<T> {\n    fn record(&self, key: &crate::field::Field, visitor: &mut dyn crate::field::Visit) {\n        self.0.record(key, visitor)\n    }\n}\n\nimpl crate::sealed::Sealed for str {}\n\nimpl Value for str {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_error(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (*self).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a mut T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a mut T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        // Don't use `(*self).record(key, visitor)`, otherwise would\n        // cause stack overflow due to `unconditional_recursion`.\n        T::record(self, key, visitor)\n    }\n}\n\nimpl<'a> crate::sealed::Sealed for fmt::Arguments<'a> {}\n\nimpl<'a> Value for fmt::Arguments<'a> {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<T: ?Sized> crate::sealed::Sealed for alloc::boxed::Box<T> where T: Value {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl<T: ?Sized> Value for alloc::boxed::Box<T>\nwhere\n    T: Value,\n{\n    #[inline]\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        self.as_ref().record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl crate::sealed::Sealed for alloc::string::String {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl Value for alloc::string::String {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self.as_str())\n    }\n}\n\nimpl fmt::Debug for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        // We are only going to be recording the field value, so we don't\n        // actually care about the field name here.\n        struct NullCallsite;\n        static NULL_CALLSITE: NullCallsite = NullCallsite;\n        impl crate::callsite::Callsite for NullCallsite {\n            fn set_interest(&self, _: crate::collect::Interest) {\n                unreachable!(\"you somehow managed to register the null callsite?\")\n            }\n\n            fn metadata(&self) -> &crate::Metadata<'_> {\n                unreachable!(\"you somehow managed to access the null callsite?\")\n            }\n        }\n\n        static FIELD: Field = Field {\n            i: 0,\n            fields: FieldSet::new(&[], crate::identify_callsite!(&NULL_CALLSITE)),\n        };\n\n        let mut res = Ok(());\n        self.record(&FIELD, &mut |_: &Field, val: &dyn fmt::Debug| {\n            res = write!(f, \"{:?}\", val);\n        });\n        res\n    }\n}\n\nimpl fmt::Display for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Debug::fmt(self, f)\n    }\n}\n\n// ===== impl DisplayValue =====\n\nimpl<T: fmt::Display> crate::sealed::Sealed for DisplayValue<T> {}\n\nimpl<T> Value for DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Debug for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Display::fmt(self, f)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Display for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\n// ===== impl DebugValue =====\n\nimpl<T: fmt::Debug> crate::sealed::Sealed for DebugValue<T> {}\n\nimpl<T: fmt::Debug> Value for DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, &self.0)\n    }\n}\n\nimpl<T: fmt::Debug> fmt::Debug for DebugValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\nimpl crate::sealed::Sealed for Empty {}\nimpl Value for Empty {\n    #[inline]\n    fn record(&self, _: &Field, _: &mut dyn Visit) {}\n}\n\n// ===== impl Field =====\n\nimpl Field {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this field.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Returns a string representing the name of the field.\n    pub fn name(&self) -> &'static str {\n        self.fields.names[self.i]\n    }\n}\n\nimpl fmt::Display for Field {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(self.name())\n    }\n}\n\nimpl AsRef<str> for Field {\n    fn as_ref(&self) -> &str {\n        self.name()\n    }\n}\n\nimpl PartialEq for Field {\n    fn eq(&self, other: &Self) -> bool {\n        self.callsite() == other.callsite() && self.i == other.i\n    }\n}\n\nimpl Eq for Field {}\n\nimpl Hash for Field {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: Hasher,\n    {\n        self.callsite().hash(state);\n        self.i.hash(state);\n    }\n}\n\nimpl Clone for Field {\n    fn clone(&self) -> Self {\n        Field {\n            i: self.i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        }\n    }\n}\n\n// ===== impl FieldSet =====\n\nimpl FieldSet {\n    /// Constructs a new `FieldSet` with the given array of field names and callsite.\n    pub const fn new(names: &'static [&'static str], callsite: callsite::Identifier) -> Self {\n        Self { names, callsite }\n    }\n\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this set of fields..\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub(crate) fn callsite(&self) -> callsite::Identifier {\n        callsite::Identifier(self.callsite.0)\n    }\n\n    /// Returns the [`Field`] named `name`, or `None` if no such field exists.\n    ///\n    /// [`Field`]: super::Field\n    pub fn field<Q: ?Sized>(&self, name: &Q) -> Option<Field>\n    where\n        Q: Borrow<str>,\n    {\n        let name = &name.borrow();\n        self.names.iter().position(|f| f == name).map(|i| Field {\n            i,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        })\n    }\n\n    /// Returns `true` if `self` contains the given `field`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: If <code>field</code> shares a name with a field\n    /// in this <code>FieldSet</code>, but was created by a <code>FieldSet</code>\n    /// with a different callsite, this <code>FieldSet</code> does <em>not</em>\n    /// contain it. This is so that if two separate span callsites define a field\n    /// named \"foo\", the <code>Field</code> corresponding to \"foo\" for each\n    /// of those callsites are not equivalent.\n    /// </pre></div>\n    pub fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite() && field.i <= self.len()\n    }\n\n    /// Returns an iterator over the `Field`s in this `FieldSet`.\n    #[inline]\n    pub fn iter(&self) -> Iter {\n        let idxs = 0..self.len();\n        Iter {\n            idxs,\n            fields: FieldSet {\n                names: self.names,\n                callsite: self.callsite(),\n            },\n        }\n    }\n\n    /// Returns a new `ValueSet` with entries for this `FieldSet`'s values.\n    #[doc(hidden)]\n    pub fn value_set<'v, V>(&'v self, values: &'v V) -> ValueSet<'v>\n    where\n        V: ValidLen<'v>,\n    {\n        ValueSet {\n            fields: self,\n            values: values.borrow(),\n        }\n    }\n\n    /// Returns the number of fields in this `FieldSet`.\n    #[inline]\n    pub fn len(&self) -> usize {\n        self.names.len()\n    }\n\n    /// Returns whether or not this `FieldSet` has fields.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.names.is_empty()\n    }\n}\n\nimpl<'a> IntoIterator for &'a FieldSet {\n    type IntoIter = Iter;\n    type Item = Field;\n    #[inline]\n    fn into_iter(self) -> Self::IntoIter {\n        self.iter()\n    }\n}\n\nimpl fmt::Debug for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"FieldSet\")\n            .field(\"names\", &self.names)\n            .field(\"callsite\", &self.callsite)\n            .finish()\n    }\n}\n\nimpl fmt::Display for FieldSet {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_set()\n            .entries(self.names.iter().map(display))\n            .finish()\n    }\n}\n\nimpl Eq for FieldSet {}\n\nimpl PartialEq for FieldSet {\n    fn eq(&self, other: &Self) -> bool {\n        if core::ptr::eq(&self, &other) {\n            true\n        } else if cfg!(not(debug_assertions)) {\n            // In a well-behaving application, two `FieldSet`s can be assumed to\n            // be totally equal so long as they share the same callsite.\n            self.callsite == other.callsite\n        } else {\n            // However, when debug-assertions are enabled, do NOT assume that\n            // the application is well-behaving; check every the field names of\n            // each `FieldSet` for equality.\n\n            // `FieldSet` is destructured here to ensure a compile-error if the\n            // fields of `FieldSet` change.\n            let Self {\n                names: lhs_names,\n                callsite: lhs_callsite,\n            } = self;\n\n            let Self {\n                names: rhs_names,\n                callsite: rhs_callsite,\n            } = &other;\n\n            // Check callsite equality first, as it is probably cheaper to do\n            // than str equality.\n            lhs_callsite == rhs_callsite && lhs_names == rhs_names\n        }\n    }\n}\n\n// ===== impl Iter =====\n\nimpl Iterator for Iter {\n    type Item = Field;\n    #[inline]\n    fn next(&mut self) -> Option<Field> {\n        let i = self.idxs.next()?;\n        Some(Field {\n            i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n        })\n    }\n}\n\n// ===== impl ValueSet =====\n\nimpl<'a> ValueSet<'a> {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// defining the fields this `ValueSet` refers to.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Visits all the fields in this `ValueSet` with the provided [visitor].\n    ///\n    /// [visitor]: Visit\n    pub fn record(&self, visitor: &mut dyn Visit) {\n        let my_callsite = self.callsite();\n        for (field, value) in self.values {\n            if field.callsite() != my_callsite {\n                continue;\n            }\n            if let Some(value) = value {\n                value.record(field, visitor);\n            }\n        }\n    }\n\n    /// Returns the number of fields in this `ValueSet` that would be visited\n    /// by a given [visitor] to the [`ValueSet::record()`] method.\n    ///\n    /// [visitor]: Visit\n    /// [`ValueSet::record()`]: ValueSet::record()\n    pub fn len(&self) -> usize {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .filter(|(field, _)| field.callsite() == my_callsite)\n            .count()\n    }\n\n    /// Returns `true` if this `ValueSet` contains a value for the given `Field`.\n    pub(crate) fn contains(&self, field: &Field) -> bool {\n        field.callsite() == self.callsite()\n            && self\n                .values\n                .iter()\n                .any(|(key, val)| *key == field && val.is_some())\n    }\n\n    /// Returns true if this `ValueSet` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        let my_callsite = self.callsite();\n        self.values\n            .iter()\n            .all(|(key, val)| val.is_none() || key.callsite() != my_callsite)\n    }\n\n    pub(crate) fn field_set(&self) -> &FieldSet {\n        self.fields\n    }\n}\n\nimpl<'a> fmt::Debug for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_struct(\"ValueSet\"), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .field(\"callsite\", &self.callsite())\n            .finish()\n    }\n}\n\nimpl<'a> fmt::Display for ValueSet<'a> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.values\n            .iter()\n            .fold(&mut f.debug_map(), |dbg, (key, v)| {\n                if let Some(val) = v {\n                    val.record(key, dbg);\n                }\n                dbg\n            })\n            .finish()\n    }\n}\n\n// ===== impl ValidLen =====\n\nmod private {\n    use super::*;\n\n    /// Restrictions on `ValueSet` lengths were removed in #2508 but this type remains for backwards compatibility.\n    pub trait ValidLen<'a>: Borrow<[(&'a Field, Option<&'a (dyn Value + 'a)>)]> {}\n\n    impl<'a, const N: usize> ValidLen<'a> for [(&'a Field, Option<&'a (dyn Value + 'a)>); N] {}\n}\n\n#[cfg(test)]\nmod test {\n    use super::*;\n    use crate::metadata::{Kind, Level, Metadata};\n\n    // Make sure TEST_CALLSITE_* have non-zero size, so they can't be located at the same address.\n    struct TestCallsite1(u8);\n    static TEST_CALLSITE_1: TestCallsite1 = TestCallsite1(0);\n    static TEST_META_1: Metadata<'static> = metadata! {\n        name: \"field_test1\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_1,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite1 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_1\n        }\n    }\n\n    struct TestCallsite2(u8);\n    static TEST_CALLSITE_2: TestCallsite2 = TestCallsite2(0);\n    static TEST_META_2: Metadata<'static> = metadata! {\n        name: \"field_test2\",\n        target: module_path!(),\n        level: Level::INFO,\n        fields: &[\"foo\", \"bar\", \"baz\"],\n        callsite: &TEST_CALLSITE_2,\n        kind: Kind::SPAN,\n    };\n\n    impl crate::callsite::Callsite for TestCallsite2 {\n        fn set_interest(&self, _: crate::collect::Interest) {\n            unimplemented!()\n        }\n\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META_2\n        }\n    }\n\n    #[test]\n    fn value_set_with_no_values_is_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), None),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn empty_value_set_is_empty() {\n        let fields = TEST_META_1.fields();\n        let valueset = fields.value_set(&[]);\n        assert!(valueset.is_empty());\n    }\n\n    #[test]\n    fn value_sets_with_fields_from_other_callsites_are_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = TEST_META_2.fields().value_set(values);\n        assert!(valueset.is_empty())\n    }\n\n    #[test]\n    fn sparse_value_sets_are_not_empty() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n        let valueset = fields.value_set(values);\n        assert!(!valueset.is_empty());\n    }\n\n    #[test]\n    fn fields_from_other_callsets_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), None),\n            (\n                &TEST_META_2.fields().field(\"bar\").unwrap(),\n                Some(&57 as &dyn Value),\n            ),\n            (&fields.field(\"baz\").unwrap(), None),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.callsite(), TEST_META_1.callsite())\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    fn empty_fields_are_skipped() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&Empty as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&57 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&Empty as &dyn Value)),\n        ];\n\n        struct MyVisitor;\n        impl Visit for MyVisitor {\n            fn record_debug(&mut self, field: &Field, _: &dyn (core::fmt::Debug)) {\n                assert_eq!(field.name(), \"bar\")\n            }\n        }\n        let valueset = fields.value_set(values);\n        valueset.record(&mut MyVisitor);\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_debug_fn() {\n        let fields = TEST_META_1.fields();\n        let values = &[\n            (&fields.field(\"foo\").unwrap(), Some(&1 as &dyn Value)),\n            (&fields.field(\"bar\").unwrap(), Some(&2 as &dyn Value)),\n            (&fields.field(\"baz\").unwrap(), Some(&3 as &dyn Value)),\n        ];\n        let valueset = fields.value_set(values);\n        let mut result = String::new();\n        valueset.record(&mut |_: &Field, value: &dyn fmt::Debug| {\n            use core::fmt::Write;\n            write!(&mut result, \"{:?}\", value).unwrap();\n        });\n        assert_eq!(result, String::from(\"123\"));\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn record_error() {\n        let fields = TEST_META_1.fields();\n        let err: Box<dyn std::error::Error + Send + Sync + 'static> =\n...\n// Path: tracing-core/src/parent.rs\nuse crate::span::Id;\n\n#[derive(Debug)]\npub(crate) enum Parent {\n    /// The new span will be a root span.\n    Root,\n    /// The new span will be rooted in the current span.\n    Current,\n    /// The new span has an explicitly-specified parent.\n    Explicit(Id),\n}\n\n// Path: tracing-core/src/span.rs\n//! Spans represent periods of time in the execution of a program.\nuse core::num::NonZeroU64;\n\nuse crate::field::FieldSet;\nuse crate::parent::Parent;\nuse crate::{field, Metadata};\n\n/// Identifies a span within the context of a collector.\n///\n/// They are generated by [collector]s for each span as it is created, by\n/// the [`new_span`] trait method. See the documentation for that method for\n/// more information on span ID generation.\n///\n/// [collector]: super::collect::Collect\n/// [`new_span`]: super::collect::Collect::new_span\n#[derive(Clone, Debug, PartialEq, Eq, Hash)]\npub struct Id(NonZeroU64);\n\n/// Attributes provided to a collector describing a new span when it is\n/// created.\n#[derive(Debug)]\npub struct Attributes<'a> {\n    metadata: &'static Metadata<'static>,\n    values: &'a field::ValueSet<'a>,\n    parent: Parent,\n}\n\n/// A set of fields recorded by a span.\n#[derive(Debug)]\npub struct Record<'a> {\n    values: &'a field::ValueSet<'a>,\n}\n\n/// Indicates what [the collector considers] the \"current\" span.\n///\n/// As collectors may not track a notion of a current span, this has three\n/// possible states:\n/// - \"unknown\", indicating that the collector does not track a current span,\n/// - \"none\", indicating that the current context is known to not be in a span,\n/// - \"some\", with the current span's [`Id`] and [`Metadata`].\n///\n/// [the collector considers]: super::collect::Collect::current_span\n/// [`Metadata`]: super::metadata::Metadata\n#[derive(Debug)]\npub struct Current {\n    inner: CurrentInner,\n}\n\n#[derive(Debug)]\nenum CurrentInner {\n    Current {\n        id: Id,\n        metadata: &'static Metadata<'static>,\n    },\n    None,\n    Unknown,\n}\n\n// ===== impl Span =====\n\nimpl Id {\n    /// Constructs a new span ID from the given `u64`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: Span IDs must be greater than zero.</pre></div>\n    ///\n    /// # Panics\n    /// - If the provided `u64` is 0.\n    pub fn from_u64(u: u64) -> Self {\n        Id(NonZeroU64::new(u).expect(\"span IDs must be > 0\"))\n    }\n\n    /// Constructs a new span ID from the given `NonZeroU64`.\n    ///\n    /// Unlike [`Id::from_u64`](Self::from_u64), this will never panic.\n    #[inline]\n    pub const fn from_non_zero_u64(id: NonZeroU64) -> Self {\n        Id(id)\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `u64`.\n    \npub fn into_u64(&self) -> u64 {\n        self.0.get()\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `NonZeroU64`.\n    #[inline]\n    pub const fn into_non_zero_u64(&self) -> NonZeroU64 {\n        self.0\n    }\n}\n\nimpl<'a> From<&'a Id> for Option<Id> {\n    fn from(id: &'a Id) -> Self {\n        Some(id.clone())\n    }\n}\n\n// ===== impl Attributes =====\n\nimpl<'a> Attributes<'a> {\n    /// Returns `Attributes` describing a new child span of the current span,\n    /// with the provided metadata and values.\n    pub fn new(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Current,\n        }\n    }\n\n    /// Returns `Attributes` describing a new span at the root of its own trace\n    /// tree, with the provided metadata and values.\n    pub fn new_root(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Root,\n        }\n    }\n\n    /// Returns `Attributes` describing a new child span of the specified\n    /// parent span, with the provided metadata and values.\n    pub fn child_of(\n        parent: Id,\n        metadata: &'static Metadata<'static>,\n        values: &'a field::ValueSet<'a>,\n    ) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Explicit(parent),\n        }\n    }\n\n    /// Returns a reference to the new span's metadata.\n    pub fn metadata(&self) -> &'static Metadata<'static> {\n        self.metadata\n    }\n\n    /// Returns a reference to a `ValueSet` containing any values the new span\n    /// was created with.\n    pub fn values(&self) -> &field::ValueSet<'a> {\n        self.values\n    }\n\n    /// Returns true if the new span should be a root.\n    pub fn is_root(&self) -> bool {\n        matches!(self.parent, Parent::Root)\n    }\n\n    /// Returns true if the new span's parent should be determined based on the\n    /// current context.\n    ///\n    /// If this is true and the current thread is currently inside a span, then\n    /// that span should be the new span's parent. Otherwise, if the current\n    /// thread is _not_ inside a span, then the new span will be the root of its\n    /// own trace tree.\n    pub fn is_contextual(&self) -> bool {\n        matches!(self.parent, Parent::Current)\n    }\n\n    /// Returns the new span's explicitly-specified parent, if there is one.\n    ///\n    /// Otherwise (if the new span is a root or is a child of the current span),\n    /// returns `None`.\n    pub fn parent(&self) -> Option<&Id> {\n        match self.parent {\n            Parent::Explicit(ref p) => Some(p),\n            _ => None,\n        }\n    }\n\n    /// Records all the fields in this set of `Attributes` with the provided\n    /// [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns `true` if this set of `Attributes` contains a value for the\n    /// given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this set of `Attributes` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n\n    /// Returns the set of all [fields] defined by this span's [`Metadata`].\n    ///\n    /// Note that the [`FieldSet`] returned by this method includes *all* the\n    /// fields declared by this span, not just those with values that are recorded\n    /// as part of this set of `Attributes`. Other fields with values not present in\n    /// this `Attributes`' value set may [record] values later.\n    ///\n    /// [fields]: crate::field\n    /// [record]: Attributes::record()\n    /// [`Metadata`]: crate::metadata::Metadata\n    /// [`FieldSet`]: crate::field::FieldSet\n    pub fn fields(&self) -> &FieldSet {\n        self.values.field_set()\n    }\n}\n\n// ===== impl Record =====\n\nimpl<'a> Record<'a> {\n    /// Constructs a new `Record` from a `ValueSet`.\n    pub fn new(values: &'a field::ValueSet<'a>) -> Self {\n        Self { values }\n    }\n\n    /// Records all the fields in this `Record` with the provided [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns the number of fields that would be visited from this `Record`\n    /// when [`Record::record()`] is called\n    ///\n    /// [`Record::record()`]: Record::record()\n    pub fn len(&self) -> usize {\n        self.values.len()\n    }\n\n    /// Returns `true` if this `Record` contains a value for the given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this `Record` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n}\n\n// ===== impl Current =====\n\nimpl Current {\n    /// Constructs a new `Current` that indicates the current context is a span\n    /// with the given `id` and `metadata`.\n    pub fn new(id: Id, metadata: &'static Metadata<'static>) -> Self {\n        Self {\n            inner: CurrentInner::Current { id, metadata },\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the current context is *not*\n    /// in a span.\n    pub fn none() -> Self {\n        Self {\n            inner: CurrentInner::None,\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the collector does not\n    /// track a current span.\n    pub fn unknown() -> Self {\n        Self {\n            inner: CurrentInner::Unknown,\n        }\n    }\n\n    /// Returns `true` if the collector that constructed this `Current` tracks a\n    /// current span.\n    ///\n    /// If this returns `true` and [`id`], [`metadata`], or [`into_inner`]\n    /// return `None`, that indicates that we are currently known to *not* be\n    /// inside a span. If this returns `false`, those methods will also return\n    /// `None`, but in this case, that is because the collector does not keep\n    /// track of the currently-entered span.\n    ///\n    /// [`id`]: Self::id\n    /// [`metadata`]: Self::metadata\n    /// [`into_inner`]: Self::into_inner\n    pub fn is_known(&self) -> bool {\n        !matches!(self.inner, CurrentInner::Unknown)\n    }\n\n    /// Consumes `self` and returns the span `Id` and `Metadata` of the current\n    /// span, if one exists and is known.\n    pub fn into_inner(self) -> Option<(Id, &'static Metadata<'static>)> {\n        match self.inner {\n            CurrentInner::Current { id, metadata } => Some((id, metadata)),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Id` of the current span, if one exists and is known.\n    pub fn id(&self) -> Option<&Id> {\n        match self.inner {\n            CurrentInner::Current { ref id, .. } => Some(id),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Metadata` of the current span, if one exists and is known.\n    pub fn metadata(&self) -> Option<&'static Metadata<'static>> {\n        match self.inner {\n            CurrentInner::Current { metadata, .. } => Some(metadata),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'a Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id()\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id().cloned()\n    }\n}\n\nimpl From<Current> for Option<Id> {\n    fn from(cur: Current) -> Self {\n        match cur.inner {\n            CurrentInner::Current { id, .. } => Some(id),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'static Metadata<'static>> {\n    fn from(cur: &'a Current) -> Self {\n        cur.metadata()\n    }\n}\n\n// Path: tracing-core/src/dispatch.rs\n//! Dispatches trace events to [`Collect`]s.\n//!\n//! The _dispatcher_ is the component of the tracing system which is responsible\n//! for forwarding trace data from the instrumentation points that generate it\n//! to the collector that collects it.\n//!\n//! # Using the Trace Dispatcher\n//!\n//! Every thread in a program using `tracing` has a _default collector_. When\n//! events occur, or spans are created, they are dispatched to the thread's\n//! current collector.\n//!\n//! ## Setting the Default Collector\n//!\n//! By default, the current collector is an empty implementation that does\n//! nothing. Trace data provided to this \"do nothing\" implementation is\n//! immediately discarded, and is not available for any purpose.\n//!\n//! To use another collector implementation, it must be set as the default.\n//! There are two methods for doing so: [`with_default`] and\n//! [`set_global_default`]. `with_default` sets the default collector for the\n//! duration of a scope, while `set_global_default` sets a default collector\n//! for the entire process.\n//!\n//! To use either of these functions, we must first wrap our collector in a\n//! [`Dispatch`], a cloneable, type-erased reference to a collector. For\n//! example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"alloc\")]\n//! use dispatch::Dispatch;\n//!\n//! # #[cfg(feature = \"alloc\")]\n//! let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"alloc\")]\n//! let my_dispatch = Dispatch::new(my_collector);\n//! ```\n//! Then, we can use [`with_default`] to set our `Dispatch` as the default for\n//! the duration of a block:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # let _my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(_my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::with_default(&my_dispatch, || {\n//!     // my_collector is the default\n//! });\n//!\n//! // no default collector again\n//! ```\n//! It's important to note that `with_default` will not propagate the current\n//! thread's default collector to any threads spawned within the `with_default`\n//! block. To propagate the default collector to new threads, either use\n//! `with_default` from the new thread, or use `set_global_default`.\n//!\n//! As an alternative to `with_default`, we can use [`set_global_default`] to\n//! set a `Dispatch` as the default for all threads, for the lifetime of the\n//! program. For example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"std\")]\n//! # let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::set_global_default(my_dispatch)\n//!     // `set_global_default` will return an error if the global default\n//!     // collector has already been set.\n//!     .expect(\"global default was already set!\");\n//!\n//! // `my_collector` is now the default\n//! ```\n//!\n//! <div class=\"example-wrap\" style=\"display:inline-block\">\n//! <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n//!\n...\n", "needle_token_start": 12166, "needle_token_end": 12192, "code_context_ntokens": 16242, "output": ["```rust\nimpl Id {\n    /// Constructs a new span ID from the given `u64`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: Span IDs must be greater than zero.</pre></div>\n    ///\n    /// # Panics\n    /// - If the provided `u64` is 0.\n    pub fn from_u64(u: u64) -> Self {\n        Id(NonZeroU64::new(u).expect(\"span IDs must be > 0\"))\n    }\n\n    /// Constructs a new span ID from the given `NonZeroU64`.\n    ///\n    /// Unlike [`Id::from_u64`](Self::from_u64), this will never panic.\n    #[inline]\n    pub const fn from_non_zero_u64(id: NonZeroU64) -> Self {\n        Id(id)\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `u64`.\n    pub fn into_u64(&self) -> u64 {\n        self.0.get()\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `NonZeroU64`.\n    #[inline]\n    pub const fn into_non_zero_u64(&self) -> NonZeroU64 {\n        self.0\n    }\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "with_default", "language": "rust", "path": "tracing-core/src/dispatch.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function sets a specified dispatch as the default for the scope of a given closure, ensuring that any operations within that closure use this dispatch. This is particularly useful for temporarily changing the context in which events or spans are recorded or handled.\n2. **Input**: It takes two parameters: a reference to a dispatch configuration and a closure that encapsulates the operations to be performed under this temporary default setting.\n3. **Output**: The output is the result of the closure that is executed with the temporary default dispatch. The type of the result depends on the closure's return type.\n4. **Procedure**: The function first sets the provided dispatch as the current default. It then executes the provided closure. Regardless of whether the closure completes successfully or panics, the original default dispatch is restored once the closure's execution is finished or interrupted. This ensures that the default dispatch setting is only temporarily changed during the execution of the closure.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: tracing-core/src/field.rs\n//! Span and `Event` key-value data.\n//!\n//! Spans and events may be annotated with key-value data, known as _fields_.\n//! These fields consist of a mapping from a key (corresponding to a `&str` but\n//! represented internally as an array index) to a [`Value`].\n//!\n//! # `Value`s and `Collect`s\n//!\n//! Collectors consume `Value`s as fields attached to [span]s or [`Event`]s.\n//! The set of field keys on a given span or event is defined on its [`Metadata`].\n//! When a span is created, it provides [`Attributes`] to the collector's\n//! [`new_span`] method, containing any fields whose values were provided when\n//! the span was created; and may call the collector's [`record`] method\n//! with additional [`Record`]s if values are added for more of its fields.\n//! Similarly, the [`Event`] type passed to the collector's [`event`] method\n//! will contain any fields attached to each event.\n//!\n//! `tracing` represents values as either one of a set of Rust primitives\n//! (`i64`, `u64`, `f64`, `i128`, `u128`, `bool`, and `&str`) or using a\n//! `fmt::Display` or `fmt::Debug` implementation. Collectors are provided\n//! these primitive value types as `dyn Value` trait objects.\n//!\n//! These trait objects can be formatted using `fmt::Debug`, but may also be\n//! recorded as typed data by calling the [`Value::record`] method on these\n//! trait objects with a _visitor_ implementing the [`Visit`] trait. This trait\n//! represents the behavior used to record values of various types. For example,\n//! we might record integers by incrementing counters for their field names,\n//! rather than printing them.\n//!\n//! [span]: super::span\n//! [`Event`]: super::event::Event\n//! [`Metadata`]: super::metadata::Metadata\n//! [`Attributes`]:  super::span::Attributes\n//! [`Record`]: super::span::Record\n//! [`new_span`]: super::collect::Collect::new_span\n//! [`record`]: super::collect::Collect::record\n//! [`event`]:  super::collect::Collect::event\nuse crate::callsite;\nuse core::{\n    borrow::Borrow,\n    fmt,\n    hash::{Hash, Hasher},\n    num,\n    ops::Range,\n};\n\nuse self::private::ValidLen;\n\n/// An opaque key allowing _O_(1) access to a field in a `Span`'s key-value\n/// data.\n///\n/// As keys are defined by the _metadata_ of a span, rather than by an\n/// individual instance of a span, a key may be used to access the same field\n/// across all instances of a given span with the same metadata. Thus, when a\n/// collector observes a new span, it need only access a field by name _once_,\n/// and use the key for that name for all other accesses.\n#[derive(Debug)]\npub struct Field {\n    i: usize,\n    fields: FieldSet,\n}\n\n/// An empty field.\n///\n/// This can be used to indicate that the value of a field is not currently\n/// present but will be recorded later.\n///\n/// When a field's value is `Empty`. it will not be recorded.\n#[derive(Debug, Eq, PartialEq)]\npub struct Empty;\n\n/// Describes the fields present on a span.\n///\n/// ## Equality\n///\n/// In well-behaved applications, two `FieldSet`s [initialized] with equal\n/// [callsite identifiers] will have identical fields. Consequently, in release\n/// builds, [`FieldSet::eq`] *only* checks that its arguments have equal\n/// callsites. However, the equality of field names is checked in debug builds.\n///\n/// [initialized]: Self::new\n/// [callsite identifiers]: callsite::Identifier\npub struct FieldSet {\n    /// The names of each field on the described span.\n    names: &'static [&'static str],\n    /// The callsite where the described span originates.\n    callsite: callsite::Identifier,\n}\n\n/// A set of fields and values for a span.\npub struct ValueSet<'a> {\n    values: &'a [(&'a Field, Option<&'a (dyn Value + 'a)>)],\n    fields: &'a FieldSet,\n}\n\n/// An iterator over a set of fields.\n#[derive(Debug)]\npub struct Iter {\n    idxs: Range<usize>,\n    fields: FieldSet,\n}\n\n/// Visits typed values.\n///\n/// An instance of `Visit` (\"a visitor\") represents the logic necessary to\n/// record field values of various types. When an implementor of [`Value`] is\n/// [recorded], it calls the appropriate method on the provided visitor to\n/// indicate the type that value should be recorded as.\n///\n/// When a [`Collect`] implementation [records an `Event`] or a\n/// [set of `Value`s added to a `Span`], it can pass an `&mut Visit` to the\n/// `record` method on the provided [`ValueSet`] or [`Event`]. This visitor\n/// will then be used to record all the field-value pairs present on that\n/// `Event` or `ValueSet`.\n///\n/// # Examples\n///\n/// A simple visitor that writes to a string might be implemented like so:\n/// ```\n/// # extern crate tracing_core as tracing;\n/// use std::fmt::{self, Write};\n/// use tracing::field::{Value, Visit, Field};\n/// pub struct StringVisitor<'a> {\n///     string: &'a mut String,\n/// }\n///\n/// impl<'a> Visit for StringVisitor<'a> {\n///     fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n///         write!(self.string, \"{} = {:?}; \", field.name(), value).unwrap();\n///     }\n/// }\n/// ```\n/// This visitor will format each recorded value using `fmt::Debug`, and\n/// append the field name and formatted value to the provided string,\n/// regardless of the type of the recorded value. When all the values have\n/// been recorded, the `StringVisitor` may be dropped, allowing the string\n/// to be printed or stored in some other data structure.\n///\n/// The `Visit` trait provides default implementations for `record_i64`,\n/// `record_u64`, `record_bool`, `record_str`, and `record_error`, which simply\n/// forward the recorded value to `record_debug`. Thus, `record_debug` is the\n/// only method which a `Visit` implementation *must* implement. However,\n/// visitors may override the default implementations of these functions in\n/// order to implement type-specific behavior.\n///\n/// Additionally, when a visitor receives a value of a type it does not care\n/// about, it is free to ignore those values completely. For example, a\n/// visitor which only records numeric data might look like this:\n///\n/// ```\n/// # extern crate tracing_core as tracing;\n/// # use std::fmt::{self, Write};\n/// # use tracing::field::{Value, Visit, Field};\n/// pub struct SumVisitor {\n///     sum: i64,\n/// }\n///\n/// impl Visit for SumVisitor {\n///     fn record_i64(&mut self, _field: &Field, value: i64) {\n///        self.sum += value;\n///     }\n///\n///     fn record_u64(&mut self, _field: &Field, value: u64) {\n///         self.sum += value as i64;\n///     }\n///\n///     fn record_debug(&mut self, _field: &Field, _value: &dyn fmt::Debug) {\n///         // Do nothing\n///     }\n/// }\n/// ```\n///\n/// This visitor (which is probably not particularly useful) keeps a running\n/// sum of all the numeric values it records, and ignores all other values. A\n/// more practical example of recording typed values is presented in\n/// `examples/counters.rs`, which demonstrates a very simple metrics system\n/// implemented using `tracing`.\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: The <code>record_error</code> trait method is only\n/// available when the Rust standard library is present, as it requires the\n/// <code>std::error::Error</code> trait.\n/// </pre></div>\n///\n/// [recorded]: Value::record\n/// [`Collect`]: super::collect::Collect\n/// [records an `Event`]: super::collect::Collect::event\n/// [set of `Value`s added to a `Span`]: super::collect::Collect::record\n/// [`Event`]: super::event::Event\npub trait Visit {\n    /// Visit a double-precision floating point value.\n    fn record_f64(&mut self, field: &Field, value: f64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 64-bit integer value.\n    fn record_i64(&mut self, field: &Field, value: i64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 64-bit integer value.\n    fn record_u64(&mut self, field: &Field, value: u64) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a signed 128-bit integer value.\n    fn record_i128(&mut self, field: &Field, value: i128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit an unsigned 128-bit integer value.\n    fn record_u128(&mut self, field: &Field, value: u128) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a boolean value.\n    fn record_bool(&mut self, field: &Field, value: bool) {\n        self.record_debug(field, &value)\n    }\n\n    /// Visit a string value.\n    fn record_str(&mut self, field: &Field, value: &str) {\n        self.record_debug(field, &value)\n    }\n\n    /// Records a type implementing `Error`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: This is only enabled when the Rust standard library is\n    /// present.\n    /// </pre>\n    /// </div>\n    #[cfg(feature = \"std\")]\n    #[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n    fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) {\n        self.record_debug(field, &DisplayValue(value))\n    }\n\n    /// Visit a value implementing `fmt::Debug`.\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug);\n}\n\n/// A field value of an erased type.\n///\n/// Implementors of `Value` may call the appropriate typed recording methods on\n/// the [visitor] passed to their `record` method in order to indicate how\n/// their data should be recorded.\n///\n/// [visitor]: Visit\npub trait Value: crate::sealed::Sealed {\n    /// Visits this value with the given `Visitor`.\n    fn record(&self, key: &Field, visitor: &mut dyn Visit);\n}\n\n/// A `Value` which serializes using `fmt::Display`.\n///\n/// Uses `record_debug` in the `Value` implementation to\n/// avoid an unnecessary evaluation.\n#[derive(Clone)]\npub struct DisplayValue<T: fmt::Display>(T);\n\n/// A `Value` which serializes as a string using `fmt::Debug`.\n#[derive(Clone)]\npub struct DebugValue<T: fmt::Debug>(T);\n\n/// Wraps a type implementing `fmt::Display` as a `Value` that can be\n/// recorded using its `Display` implementation.\npub fn display<T>(t: T) -> DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    DisplayValue(t)\n}\n\n/// Wraps a type implementing `fmt::Debug` as a `Value` that can be\n/// recorded using its `Debug` implementation.\npub fn debug<T>(t: T) -> DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    DebugValue(t)\n}\n\n// ===== impl Visit =====\n\nimpl<'a, 'b> Visit for fmt::DebugStruct<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.field(field.name(), value);\n    }\n}\n\nimpl<'a, 'b> Visit for fmt::DebugMap<'a, 'b> {\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        self.entry(&format_args!(\"{}\", field), value);\n    }\n}\n\nimpl<F> Visit for F\nwhere\n    F: FnMut(&Field, &dyn fmt::Debug),\n{\n    fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {\n        (self)(field, value)\n    }\n}\n\n// ===== impl Value =====\n\nmacro_rules! impl_values {\n    ( $( $record:ident( $( $whatever:tt)+ ) ),+ ) => {\n        $(\n            impl_value!{ $record( $( $whatever )+ ) }\n        )+\n    }\n}\n\nmacro_rules! ty_to_nonzero {\n    (u8) => {\n        NonZeroU8\n    };\n    (u16) => {\n        NonZeroU16\n    };\n    (u32) => {\n        NonZeroU32\n    };\n    (u64) => {\n        NonZeroU64\n    };\n    (u128) => {\n        NonZeroU128\n    };\n    (usize) => {\n        NonZeroUsize\n    };\n    (i8) => {\n        NonZeroI8\n    };\n    (i16) => {\n        NonZeroI16\n    };\n    (i32) => {\n        NonZeroI32\n    };\n    (i64) => {\n        NonZeroI64\n    };\n    (i128) => {\n        NonZeroI128\n    };\n    (isize) => {\n        NonZeroIsize\n    };\n}\n\nmacro_rules! impl_one_value {\n    (f32, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f32, $op, $record);\n    };\n    (f64, $op:expr, $record:ident) => {\n        impl_one_value!(normal, f64, $op, $record);\n    };\n    (bool, $op:expr, $record:ident) => {\n        impl_one_value!(normal, bool, $op, $record);\n    };\n    ($value_ty:tt, $op:expr, $record:ident) => {\n        impl_one_value!(normal, $value_ty, $op, $record);\n        impl_one_value!(nonzero, $value_ty, $op, $record);\n    };\n    (normal, $value_ty:tt, $op:expr, $record:ident) => {\n        impl $crate::sealed::Sealed for $value_ty {}\n        impl $crate::field::Value for $value_ty {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(*self))\n            }\n        }\n    };\n    (nonzero, $value_ty:tt, $op:expr, $record:ident) => {\n        // This `use num::*;` is reported as unused because it gets emitted\n        // for every single invocation of this macro, so there are multiple `use`s.\n        // All but the first are useless indeed.\n        // We need this import because we can't write a path where one part is\n        // the `ty_to_nonzero!($value_ty)` invocation.\n        #[allow(clippy::useless_attribute, unused)]\n        use num::*;\n        impl $crate::sealed::Sealed for ty_to_nonzero!($value_ty) {}\n        impl $crate::field::Value for ty_to_nonzero!($value_ty) {\n            fn record(&self, key: &$crate::field::Field, visitor: &mut dyn $crate::field::Visit) {\n                // `op` is always a function; the closure is used because\n                // sometimes there isn't a real function corresponding to that\n                // operation. the clippy warning is not that useful here.\n                #[allow(clippy::redundant_closure_call)]\n                visitor.$record(key, $op(self.get()))\n            }\n        }\n    };\n}\n\nmacro_rules! impl_value {\n    ( $record:ident( $( $value_ty:tt ),+ ) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this, $record);\n        )+\n    };\n    ( $record:ident( $( $value_ty:tt ),+ as $as_ty:ty) ) => {\n        $(\n            impl_one_value!($value_ty, |this: $value_ty| this as $as_ty, $record);\n        )+\n    };\n}\n\n// ===== impl Value =====\n\nimpl_values! {\n    record_u64(u64),\n    record_u64(usize, u32, u16, u8 as u64),\n    record_i64(i64),\n    record_i64(isize, i32, i16, i8 as i64),\n    record_u128(u128),\n    record_i128(i128),\n    record_bool(bool),\n    record_f64(f64, f32 as f64)\n}\n\nimpl<T: crate::sealed::Sealed> crate::sealed::Sealed for Wrapping<T> {}\nimpl<T: crate::field::Value> crate::field::Value for Wrapping<T> {\n    fn record(&self, key: &crate::field::Field, visitor: &mut dyn crate::field::Visit) {\n        self.0.record(key, visitor)\n    }\n}\n\nimpl crate::sealed::Sealed for str {}\n\nimpl Value for str {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_error(key, self)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl crate::sealed::Sealed for dyn std::error::Error + Send + Sync + 'static {}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl Value for dyn std::error::Error + Send + Sync + 'static {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (self as &dyn std::error::Error).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        (*self).record(key, visitor)\n    }\n}\n\nimpl<'a, T: ?Sized> crate::sealed::Sealed for &'a mut T where T: Value + crate::sealed::Sealed + 'a {}\n\nimpl<'a, T: ?Sized> Value for &'a mut T\nwhere\n    T: Value + 'a,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        // Don't use `(*self).record(key, visitor)`, otherwise would\n        // cause stack overflow due to `unconditional_recursion`.\n        T::record(self, key, visitor)\n    }\n}\n\nimpl<'a> crate::sealed::Sealed for fmt::Arguments<'a> {}\n\nimpl<'a> Value for fmt::Arguments<'a> {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\nimpl<T: ?Sized> crate::sealed::Sealed for alloc::boxed::Box<T> where T: Value {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl<T: ?Sized> Value for alloc::boxed::Box<T>\nwhere\n    T: Value,\n{\n    #[inline]\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        self.as_ref().record(key, visitor)\n    }\n}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl crate::sealed::Sealed for alloc::string::String {}\n\n#[cfg(feature = \"alloc\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"alloc\")))]\nimpl Value for alloc::string::String {\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_str(key, self.as_str())\n    }\n}\n\nimpl fmt::Debug for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        // We are only going to be recording the field value, so we don't\n        // actually care about the field name here.\n        struct NullCallsite;\n        static NULL_CALLSITE: NullCallsite = NullCallsite;\n        impl crate::callsite::Callsite for NullCallsite {\n            fn set_interest(&self, _: crate::collect::Interest) {\n                unreachable!(\"you somehow managed to register the null callsite?\")\n            }\n\n            fn metadata(&self) -> &crate::Metadata<'_> {\n                unreachable!(\"you somehow managed to access the null callsite?\")\n            }\n        }\n\n        static FIELD: Field = Field {\n            i: 0,\n            fields: FieldSet::new(&[], crate::identify_callsite!(&NULL_CALLSITE)),\n        };\n\n        let mut res = Ok(());\n        self.record(&FIELD, &mut |_: &Field, val: &dyn fmt::Debug| {\n            res = write!(f, \"{:?}\", val);\n        });\n        res\n    }\n}\n\nimpl fmt::Display for dyn Value {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Debug::fmt(self, f)\n    }\n}\n\n// ===== impl DisplayValue =====\n\nimpl<T: fmt::Display> crate::sealed::Sealed for DisplayValue<T> {}\n\nimpl<T> Value for DisplayValue<T>\nwhere\n    T: fmt::Display,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, self)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Debug for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        fmt::Display::fmt(self, f)\n    }\n}\n\nimpl<T: fmt::Display> fmt::Display for DisplayValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\n// ===== impl DebugValue =====\n\nimpl<T: fmt::Debug> crate::sealed::Sealed for DebugValue<T> {}\n\nimpl<T: fmt::Debug> Value for DebugValue<T>\nwhere\n    T: fmt::Debug,\n{\n    fn record(&self, key: &Field, visitor: &mut dyn Visit) {\n        visitor.record_debug(key, &self.0)\n    }\n}\n\nimpl<T: fmt::Debug> fmt::Debug for DebugValue<T> {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        self.0.fmt(f)\n    }\n}\n\nimpl crate::sealed::Sealed for Empty {}\nimpl Value for Empty {\n    #[inline]\n    fn record(&self, _: &Field, _: &mut dyn Visit) {}\n}\n\n// ===== impl Field =====\n\nimpl Field {\n    /// Returns an [`Identifier`] that uniquely identifies the [`Callsite`]\n    /// which defines this field.\n    ///\n    /// [`Identifier`]: super::callsite::Identifier\n    /// [`Callsite`]: super::callsite::Callsite\n    #[inline]\n    pub fn callsite(&self) -> callsite::Identifier {\n        self.fields.callsite()\n    }\n\n    /// Returns a string representing the name of the field.\n    pub fn name(&self) -> &'static str {\n        self.fields.names[self.i]\n    }\n}\n\nimpl fmt::Display for Field {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(self.name())\n    }\n}\n\nimpl AsRef<str> for Field {\n    fn as_ref(&self) -> &str {\n        self.name()\n    }\n}\n\nimpl PartialEq for Field {\n    fn eq(&self, other: &Self) -> bool {\n        self.callsite() == other.callsite() && self.i == other.i\n    }\n}\n\nimpl Eq for Field {}\n\nimpl Hash for Field {\n    fn hash<H>(&self, state: &mut H)\n    where\n        H: Hasher,\n    {\n        self.callsite().hash(state);\n        self.i.hash(state);\n    }\n}\n\nimpl Clone for Field {\n    fn clone(&self) -> Self {\n        Field {\n            i: self.i,\n            fields: FieldSet {\n                names: self.fields.names,\n                callsite: self.fields.callsite(),\n            },\n...\n// Path: tracing-core/src/parent.rs\nuse crate::span::Id;\n\n#[derive(Debug)]\npub(crate) enum Parent {\n    /// The new span will be a root span.\n    Root,\n    /// The new span will be rooted in the current span.\n    Current,\n    /// The new span has an explicitly-specified parent.\n    Explicit(Id),\n}\n\n// Path: tracing-core/src/span.rs\n//! Spans represent periods of time in the execution of a program.\nuse core::num::NonZeroU64;\n\nuse crate::field::FieldSet;\nuse crate::parent::Parent;\nuse crate::{field, Metadata};\n\n/// Identifies a span within the context of a collector.\n///\n/// They are generated by [collector]s for each span as it is created, by\n/// the [`new_span`] trait method. See the documentation for that method for\n/// more information on span ID generation.\n///\n/// [collector]: super::collect::Collect\n/// [`new_span`]: super::collect::Collect::new_span\n#[derive(Clone, Debug, PartialEq, Eq, Hash)]\npub struct Id(NonZeroU64);\n\n/// Attributes provided to a collector describing a new span when it is\n/// created.\n#[derive(Debug)]\npub struct Attributes<'a> {\n    metadata: &'static Metadata<'static>,\n    values: &'a field::ValueSet<'a>,\n    parent: Parent,\n}\n\n/// A set of fields recorded by a span.\n#[derive(Debug)]\npub struct Record<'a> {\n    values: &'a field::ValueSet<'a>,\n}\n\n/// Indicates what [the collector considers] the \"current\" span.\n///\n/// As collectors may not track a notion of a current span, this has three\n/// possible states:\n/// - \"unknown\", indicating that the collector does not track a current span,\n/// - \"none\", indicating that the current context is known to not be in a span,\n/// - \"some\", with the current span's [`Id`] and [`Metadata`].\n///\n/// [the collector considers]: super::collect::Collect::current_span\n/// [`Metadata`]: super::metadata::Metadata\n#[derive(Debug)]\npub struct Current {\n    inner: CurrentInner,\n}\n\n#[derive(Debug)]\nenum CurrentInner {\n    Current {\n        id: Id,\n        metadata: &'static Metadata<'static>,\n    },\n    None,\n    Unknown,\n}\n\n// ===== impl Span =====\n\nimpl Id {\n    /// Constructs a new span ID from the given `u64`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: Span IDs must be greater than zero.</pre></div>\n    ///\n    /// # Panics\n    /// - If the provided `u64` is 0.\n    pub fn from_u64(u: u64) -> Self {\n        Id(NonZeroU64::new(u).expect(\"span IDs must be > 0\"))\n    }\n\n    /// Constructs a new span ID from the given `NonZeroU64`.\n    ///\n    /// Unlike [`Id::from_u64`](Self::from_u64), this will never panic.\n    #[inline]\n    pub const fn from_non_zero_u64(id: NonZeroU64) -> Self {\n        Id(id)\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `u64`.\n    pub fn into_u64(&self) -> u64 {\n        self.0.get()\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `NonZeroU64`.\n    #[inline]\n    pub const fn into_non_zero_u64(&self) -> NonZeroU64 {\n        self.0\n    }\n}\n\nimpl<'a> From<&'a Id> for Option<Id> {\n    fn from(id: &'a Id) -> Self {\n        Some(id.clone())\n    }\n}\n\n// ===== impl Attributes =====\n\nimpl<'a> Attributes<'a> {\n    /// Returns `Attributes` describing a new child span of the current span,\n    /// with the provided metadata and values.\n    pub fn new(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Current,\n        }\n    }\n\n    /// Returns `Attributes` describing a new span at the root of its own trace\n    /// tree, with the provided metadata and values.\n    pub fn new_root(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Root,\n        }\n    }\n\n    /// Returns `Attributes` describing a new child span of the specified\n    /// parent span, with the provided metadata and values.\n    pub fn child_of(\n        parent: Id,\n        metadata: &'static Metadata<'static>,\n        values: &'a field::ValueSet<'a>,\n    ) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Explicit(parent),\n        }\n    }\n\n    /// Returns a reference to the new span's metadata.\n    pub fn metadata(&self) -> &'static Metadata<'static> {\n        self.metadata\n    }\n\n    /// Returns a reference to a `ValueSet` containing any values the new span\n    /// was created with.\n    pub fn values(&self) -> &field::ValueSet<'a> {\n        self.values\n    }\n\n    /// Returns true if the new span should be a root.\n    pub fn is_root(&self) -> bool {\n        matches!(self.parent, Parent::Root)\n    }\n\n    /// Returns true if the new span's parent should be determined based on the\n    /// current context.\n    ///\n    /// If this is true and the current thread is currently inside a span, then\n    /// that span should be the new span's parent. Otherwise, if the current\n    /// thread is _not_ inside a span, then the new span will be the root of its\n    /// own trace tree.\n    pub fn is_contextual(&self) -> bool {\n        matches!(self.parent, Parent::Current)\n    }\n\n    /// Returns the new span's explicitly-specified parent, if there is one.\n    ///\n    /// Otherwise (if the new span is a root or is a child of the current span),\n    /// returns `None`.\n    pub fn parent(&self) -> Option<&Id> {\n        match self.parent {\n            Parent::Explicit(ref p) => Some(p),\n            _ => None,\n        }\n    }\n\n    /// Records all the fields in this set of `Attributes` with the provided\n    /// [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns `true` if this set of `Attributes` contains a value for the\n    /// given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this set of `Attributes` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n\n    /// Returns the set of all [fields] defined by this span's [`Metadata`].\n    ///\n    /// Note that the [`FieldSet`] returned by this method includes *all* the\n    /// fields declared by this span, not just those with values that are recorded\n    /// as part of this set of `Attributes`. Other fields with values not present in\n    /// this `Attributes`' value set may [record] values later.\n    ///\n    /// [fields]: crate::field\n    /// [record]: Attributes::record()\n    /// [`Metadata`]: crate::metadata::Metadata\n    /// [`FieldSet`]: crate::field::FieldSet\n    pub fn fields(&self) -> &FieldSet {\n        self.values.field_set()\n    }\n}\n\n// ===== impl Record =====\n\nimpl<'a> Record<'a> {\n    /// Constructs a new `Record` from a `ValueSet`.\n    pub fn new(values: &'a field::ValueSet<'a>) -> Self {\n        Self { values }\n    }\n\n    /// Records all the fields in this `Record` with the provided [Visitor].\n    ///\n    /// [visitor]: super::field::Visit\n    pub fn record(&self, visitor: &mut dyn field::Visit) {\n        self.values.record(visitor)\n    }\n\n    /// Returns the number of fields that would be visited from this `Record`\n    /// when [`Record::record()`] is called\n    ///\n    /// [`Record::record()`]: Record::record()\n    pub fn len(&self) -> usize {\n        self.values.len()\n    }\n\n    /// Returns `true` if this `Record` contains a value for the given `Field`.\n    pub fn contains(&self, field: &field::Field) -> bool {\n        self.values.contains(field)\n    }\n\n    /// Returns true if this `Record` contains _no_ values.\n    pub fn is_empty(&self) -> bool {\n        self.values.is_empty()\n    }\n}\n\n// ===== impl Current =====\n\nimpl Current {\n    /// Constructs a new `Current` that indicates the current context is a span\n    /// with the given `id` and `metadata`.\n    pub fn new(id: Id, metadata: &'static Metadata<'static>) -> Self {\n        Self {\n            inner: CurrentInner::Current { id, metadata },\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the current context is *not*\n    /// in a span.\n    pub fn none() -> Self {\n        Self {\n            inner: CurrentInner::None,\n        }\n    }\n\n    /// Constructs a new `Current` that indicates the collector does not\n    /// track a current span.\n    pub fn unknown() -> Self {\n        Self {\n            inner: CurrentInner::Unknown,\n        }\n    }\n\n    /// Returns `true` if the collector that constructed this `Current` tracks a\n    /// current span.\n    ///\n    /// If this returns `true` and [`id`], [`metadata`], or [`into_inner`]\n    /// return `None`, that indicates that we are currently known to *not* be\n    /// inside a span. If this returns `false`, those methods will also return\n    /// `None`, but in this case, that is because the collector does not keep\n    /// track of the currently-entered span.\n    ///\n    /// [`id`]: Self::id\n    /// [`metadata`]: Self::metadata\n    /// [`into_inner`]: Self::into_inner\n    pub fn is_known(&self) -> bool {\n        !matches!(self.inner, CurrentInner::Unknown)\n    }\n\n    /// Consumes `self` and returns the span `Id` and `Metadata` of the current\n    /// span, if one exists and is known.\n    pub fn into_inner(self) -> Option<(Id, &'static Metadata<'static>)> {\n        match self.inner {\n            CurrentInner::Current { id, metadata } => Some((id, metadata)),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Id` of the current span, if one exists and is known.\n    pub fn id(&self) -> Option<&Id> {\n        match self.inner {\n            CurrentInner::Current { ref id, .. } => Some(id),\n            _ => None,\n        }\n    }\n\n    /// Borrows the `Metadata` of the current span, if one exists and is known.\n    pub fn metadata(&self) -> Option<&'static Metadata<'static>> {\n        match self.inner {\n            CurrentInner::Current { metadata, .. } => Some(metadata),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'a Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id()\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<Id> {\n    fn from(cur: &'a Current) -> Self {\n        cur.id().cloned()\n    }\n}\n\nimpl From<Current> for Option<Id> {\n    fn from(cur: Current) -> Self {\n        match cur.inner {\n            CurrentInner::Current { id, .. } => Some(id),\n            _ => None,\n        }\n    }\n}\n\nimpl<'a> From<&'a Current> for Option<&'static Metadata<'static>> {\n    fn from(cur: &'a Current) -> Self {\n        cur.metadata()\n    }\n}\n\n// Path: tracing-core/src/dispatch.rs\n//! Dispatches trace events to [`Collect`]s.\n//!\n//! The _dispatcher_ is the component of the tracing system which is responsible\n//! for forwarding trace data from the instrumentation points that generate it\n//! to the collector that collects it.\n//!\n//! # Using the Trace Dispatcher\n//!\n//! Every thread in a program using `tracing` has a _default collector_. When\n//! events occur, or spans are created, they are dispatched to the thread's\n//! current collector.\n//!\n//! ## Setting the Default Collector\n//!\n//! By default, the current collector is an empty implementation that does\n//! nothing. Trace data provided to this \"do nothing\" implementation is\n//! immediately discarded, and is not available for any purpose.\n//!\n//! To use another collector implementation, it must be set as the default.\n//! There are two methods for doing so: [`with_default`] and\n//! [`set_global_default`]. `with_default` sets the default collector for the\n//! duration of a scope, while `set_global_default` sets a default collector\n//! for the entire process.\n//!\n//! To use either of these functions, we must first wrap our collector in a\n//! [`Dispatch`], a cloneable, type-erased reference to a collector. For\n//! example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"alloc\")]\n//! use dispatch::Dispatch;\n//!\n//! # #[cfg(feature = \"alloc\")]\n//! let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"alloc\")]\n//! let my_dispatch = Dispatch::new(my_collector);\n//! ```\n//! Then, we can use [`with_default`] to set our `Dispatch` as the default for\n//! the duration of a block:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # let _my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(_my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::with_default(&my_dispatch, || {\n//!     // my_collector is the default\n//! });\n//!\n//! // no default collector again\n//! ```\n//! It's important to note that `with_default` will not propagate the current\n//! thread's default collector to any threads spawned within the `with_default`\n//! block. To propagate the default collector to new threads, either use\n//! `with_default` from the new thread, or use `set_global_default`.\n//!\n//! As an alternative to `with_default`, we can use [`set_global_default`] to\n//! set a `Dispatch` as the default for all threads, for the lifetime of the\n//! program. For example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"std\")]\n//! # let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::set_global_default(my_dispatch)\n//!     // `set_global_default` will return an error if the global default\n//!     // collector has already been set.\n//!     .expect(\"global default was already set!\");\n//!\n//! // `my_collector` is now the default\n//! ```\n//!\n//! <div class=\"example-wrap\" style=\"display:inline-block\">\n//! <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n//!\n//! **Note**: the thread-local scoped dispatcher ([`with_default`]) requires the\n//! Rust standard library. `no_std` users should use [`set_global_default`] instead.\n//!\n//! </pre></div>\n//!\n//! ## Accessing the Default Collector\n//!\n//! A thread's current default collector can be accessed using the\n//! [`get_default`] function, which executes a closure with a reference to the\n//! currently default `Dispatch`. This is used primarily by `tracing`\n//! instrumentation.\nuse crate::{\n    collect::{self, Collect, NoCollector},\n    span, Event, LevelFilter, Metadata,\n};\n\nuse core::{\n    any::Any,\n    fmt,\n    sync::atomic::{AtomicBool, AtomicUsize, Ordering},\n};\n\n#[cfg(feature = \"std\")]\nuse std::{\n    cell::{Cell, RefCell, RefMut},\n    error,\n};\n\n#[cfg(feature = \"alloc\")]\nuse alloc::sync::{Arc, Weak};\n\n#[cfg(feature = \"alloc\")]\nuse core::ops::Deref;\n\n/// `Dispatch` trace data to a [`Collect`].\n#[derive(Clone)]\npub struct Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Arc<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n/// `WeakDispatch` is a version of [`Dispatch`] that holds a non-owning reference\n/// to a [collector].\n///\n/// The collector may be accessed by calling [`WeakDispatch::upgrade`],\n/// which returns an `Option<Dispatch>`. If all [`Dispatch`] clones that point\n/// at the collector have been dropped, [`WeakDispatch::upgrade`] will return\n/// `None`. Otherwise, it will return `Some(Dispatch)`.\n///\n/// A `WeakDispatch` may be created from a [`Dispatch`] by calling the\n/// [`Dispatch::downgrade`] method. The primary use for creating a\n/// [`WeakDispatch`] is to allow a collector to hold a cyclical reference to\n/// itself without creating a memory leak. See [here] for details.\n///\n/// This type is analogous to the [`std::sync::Weak`] type, but for a\n/// [`Dispatch`] rather than an [`Arc`].\n///\n/// [collector]: Collect\n/// [`Arc`]: std::sync::Arc\n/// [here]: Collect#avoiding-memory-leaks\n#[derive(Clone)]\npub struct WeakDispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Weak<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n#[cfg(feature = \"alloc\")]\n#[derive(Clone)]\nenum Kind<T> {\n    Global(&'static (dyn Collect + Send + Sync)),\n    Scoped(T),\n}\n\n#[cfg(feature = \"std\")]\nthread_local! {\n    static CURRENT_STATE: State = const {\n        State {\n            default: RefCell::new(None),\n            can_enter: Cell::new(true),\n        }\n    };\n}\n\nstatic EXISTS: AtomicBool = AtomicBool::new(false);\nstatic GLOBAL_INIT: AtomicUsize = AtomicUsize::new(UNINITIALIZED);\n\n#[cfg(feature = \"std\")]\nstatic SCOPED_COUNT: AtomicUsize = AtomicUsize::new(0);\n\nconst UNINITIALIZED: usize = 0;\nconst INITIALIZING: usize = 1;\nconst INITIALIZED: usize = 2;\n\nstatic mut GLOBAL_DISPATCH: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NONE: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NO_COLLECTOR: NoCollector = NoCollector::new();\n\n/// The dispatch state of a thread.\n#[cfg(feature = \"std\")]\nstruct State {\n    /// This thread's current default dispatcher.\n    default: RefCell<Option<Dispatch>>,\n    /// Whether or not we can currently begin dispatching a trace event.\n    ///\n    /// This is set to `false` when functions such as `enter`, `exit`, `event`,\n    /// and `new_span` are called on this thread's default dispatcher, to\n    /// prevent further trace events triggered inside those functions from\n    /// creating an infinite recursion. When we finish handling a dispatch, this\n    /// is set back to `true`.\n    can_enter: Cell<bool>,\n}\n\n/// While this guard is active, additional calls to collector functions on\n/// the default dispatcher will not be able to access the dispatch context.\n/// Dropping the guard will allow the dispatch context to be re-entered.\n#[cfg(feature = \"std\")]\nstruct Entered<'a>(&'a State);\n\n/// A guard that resets the current default dispatcher to the prior\n/// default dispatcher when dropped.\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[derive(Debug)]\npub struct DefaultGuard(Option<Dispatch>);\n\n/// Sets this dispatch as the default for the duration of a closure.\n///\n/// The default dispatcher is used when creating a new [span] or\n/// [`Event`].\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: This function required the Rust standard library.\n/// <!-- hack: this whitespace makes rustdoc interpret the next line as markdown again -->\n///\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n\npub fn with_default<T>(dispatcher: &Dispatch, f: impl FnOnce() -> T) -> T {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this (rather than simply resetting after calling\n    // `f`) ensures that we always reset to the prior dispatcher even if `f`\n    // panics.\n    let _guard = set_default(dispatcher);\n    f()\n}\n\n/// Sets the dispatch as the default dispatch for the duration of the lifetime\n/// of the returned DefaultGuard\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n///\n/// **Note**: This function required the Rust standard library.\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[must_use = \"Dropping the guard unregisters the dispatcher.\"]\npub fn set_default(dispatcher: &Dispatch) -> DefaultGuard {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this ensures that we always reset to the prior\n    // dispatcher even if the thread calling this function panics.\n    State::set_default(dispatcher.clone())\n}\n\n/// Sets this dispatch as the global default for the duration of the entire program.\n/// Will be used as a fallback if no thread-local dispatch has been set in a thread\n/// (using `with_default`.)\n///\n/// Can only be set once; subsequent attempts to set the global default will fail.\n/// Returns `Err` if the global default has already been set.\n///\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Warning</strong>: In general, libraries should <em>not</em> call\n/// <code>set_global_default()</code>! Doing so will cause conflicts when\n/// executables that depend on the library try to set the default collector later.\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\npub fn set_global_default(dispatcher: Dispatch) -> Result<(), SetGlobalDefaultError> {\n    // if `compare_exchange` returns Result::Ok(_), then `new` has been set and\n    // `current`\u2014now the prior value\u2014has been returned in the `Ok()` branch.\n    if GLOBAL_INIT\n        .compare_exchange(\n            UNINITIALIZED,\n            INITIALIZING,\n            Ordering::SeqCst,\n            Ordering::SeqCst,\n        )\n        .is_ok()\n    {\n        #[cfg(feature = \"alloc\")]\n        let collector = {\n            let collector = match dispatcher.collector {\n                Kind::Global(s) => s,\n                Kind::Scoped(s) => unsafe {\n                    // safety: this leaks the collector onto the heap. the\n                    // reference count will always be at least 1.\n                    &*Arc::into_raw(s)\n                },\n            };\n            Kind::Global(collector)\n        };\n\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = dispatcher.collector;\n\n        unsafe {\n            GLOBAL_DISPATCH = Dispatch { collector };\n        }\n        GLOBAL_INIT.store(INITIALIZED, Ordering::SeqCst);\n        EXISTS.store(true, Ordering::Release);\n        Ok(())\n    } else {\n        Err(SetGlobalDefaultError { _no_construct: () })\n    }\n}\n\n/// Returns true if a `tracing` dispatcher has ever been set.\n///\n/// This may be used to completely elide trace points if tracing is not in use\n/// at all or has yet to be initialized.\n#[doc(hidden)]\n#[inline(always)]\npub fn has_been_set() -> bool {\n    EXISTS.load(Ordering::Relaxed)\n}\n\n/// Returned if setting the global dispatcher fails.\npub struct SetGlobalDefaultError {\n    _no_construct: (),\n}\n\nimpl fmt::Debug for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"SetGlobalDefaultError\")\n            .field(&Self::MESSAGE)\n            .finish()\n    }\n}\n\nimpl fmt::Display for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(Self::MESSAGE)\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl error::Error for SetGlobalDefaultError {}\n\nimpl SetGlobalDefaultError {\n    const MESSAGE: &'static str = \"a global default trace dispatcher has already been set\";\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatch::Dispatch\n#[cfg(feature = \"std\")]\n#[inline(always)]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    if SCOPED_COUNT.load(Ordering::Acquire) == 0 {\n        // fast path if no scoped dispatcher has been set; just use the global\n        // default.\n        return f(get_global());\n    }\n\n    get_default_slow(f)\n}\n\n#[cfg(feature = \"std\")]\n#[inline(never)]\nfn get_default_slow<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    // While this guard is active, additional calls to collector functions on\n    // the default dispatcher will not be able to access the dispatch context.\n    // Dropping the guard will allow the dispatch context to be re-entered.\n    struct Entered<'a>(&'a Cell<bool>);\n    impl<'a> Drop for Entered<'a> {\n        #[inline]\n        fn drop(&mut self) {\n            self.0.set(true);\n        }\n    }\n\n    CURRENT_STATE\n        .try_with(|state| {\n            if state.can_enter.replace(false) {\n                let _guard = Entered(&state.can_enter);\n\n                let mut default = state.default.borrow_mut();\n                let default = default\n                    // if the local default for this thread has never been set,\n                    // populate it with the global default, so we don't have to\n                    // keep getting the global on every `get_default_slow` call.\n                    .get_or_insert_with(|| get_global().clone());\n\n                return f(&*default);\n            }\n\n            f(&Dispatch::none())\n        })\n        .unwrap_or_else(|_| f(&Dispatch::none()))\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(feature = \"std\")]\n#[doc(hidden)]\n#[inline(never)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    CURRENT_STATE\n        .try_with(|state| {\n            let entered = state.enter()?;\n            Some(f(&entered.current()))\n        })\n        .ok()?\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\n#[doc(hidden)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    Some(f(&get_global()))\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    f(get_global())\n}\n\n#[inline(always)]\npub(crate) fn get_global() -> &'static Dispatch {\n    if GLOBAL_INIT.load(Ordering::Acquire) != INITIALIZED {\n        return &NONE;\n    }\n    unsafe {\n        // This is safe given the invariant that setting the global dispatcher\n        // also sets `GLOBAL_INIT` to `INITIALIZED`.\n        &GLOBAL_DISPATCH\n    }\n}\n\n#[cfg(feature = \"std\")]\npub(crate) struct Registrar(Kind<Weak<dyn Collect + Send + Sync>>);\n\nimpl Dispatch {\n    /// Returns a new `Dispatch` that discards events and spans.\n    #[inline]\n    pub fn none() -> Self {\n        Dispatch {\n            #[cfg(feature = \"alloc\")]\n            collector: Kind::Global(&NO_COLLECTOR),\n            #[cfg(not(feature = \"alloc\"))]\n            collector: &NO_COLLECTOR,\n        }\n    }\n\n    /// Returns a `Dispatch` that forwards to the given [`Collect`].\n    ///\n    /// [`Collect`]: super::collect::Collect\n    #[cfg(feature = \"alloc\")]\n...\n", "needle_token_start": 13778, "needle_token_end": 13887, "code_context_ntokens": 16266, "output": ["```rust\n/// Sets this dispatch as the default for the duration of a closure.\n///\n/// The default dispatcher is used when creating a new [span] or\n/// [`Event`].\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: This function required the Rust standard library.\n/// <!-- hack: this whitespace makes rustdoc interpret the next line as markdown again -->\n///\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\npub fn with_default<T>(dispatcher: &Dispatch, f: impl FnOnce() -> T) -> T {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this (rather than simply resetting after calling\n    // `f`) ensures that we always reset to the prior dispatcher even if `f`\n    // panics.\n    let _guard = set_default(dispatcher);\n    f()\n}\n```"]}
{"repo": "tokio-rs/tracing", "name": "on_register_dispatch", "language": "rust", "path": "tracing-core/src/collect.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: This method is called when the collector is linked with a dispatch mechanism, primarily to handle initialization tasks or configurations related to the dispatch system.\n2. **Input**: It receives a reference to the dispatch mechanism that manages the collector.\n3. **Output**: There is no return value; the method performs operations internally and does not produce output.\n4. **Procedure**: The method typically contains a placeholder for implementation-specific logic that might involve setting up references or initializing state based on the dispatch mechanism. However, it should avoid storing a direct reference to the dispatch to prevent memory leaks.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: tracing-core/src/span.rs\n//! Spans represent periods of time in the execution of a program.\nuse core::num::NonZeroU64;\n\nuse crate::field::FieldSet;\nuse crate::parent::Parent;\nuse crate::{field, Metadata};\n\n/// Identifies a span within the context of a collector.\n///\n/// They are generated by [collector]s for each span as it is created, by\n/// the [`new_span`] trait method. See the documentation for that method for\n/// more information on span ID generation.\n///\n/// [collector]: super::collect::Collect\n/// [`new_span`]: super::collect::Collect::new_span\n#[derive(Clone, Debug, PartialEq, Eq, Hash)]\npub struct Id(NonZeroU64);\n\n/// Attributes provided to a collector describing a new span when it is\n/// created.\n#[derive(Debug)]\npub struct Attributes<'a> {\n    metadata: &'static Metadata<'static>,\n    values: &'a field::ValueSet<'a>,\n    parent: Parent,\n}\n\n/// A set of fields recorded by a span.\n#[derive(Debug)]\npub struct Record<'a> {\n    values: &'a field::ValueSet<'a>,\n}\n\n/// Indicates what [the collector considers] the \"current\" span.\n///\n/// As collectors may not track a notion of a current span, this has three\n/// possible states:\n/// - \"unknown\", indicating that the collector does not track a current span,\n/// - \"none\", indicating that the current context is known to not be in a span,\n/// - \"some\", with the current span's [`Id`] and [`Metadata`].\n///\n/// [the collector considers]: super::collect::Collect::current_span\n/// [`Metadata`]: super::metadata::Metadata\n#[derive(Debug)]\npub struct Current {\n    inner: CurrentInner,\n}\n\n#[derive(Debug)]\nenum CurrentInner {\n    Current {\n        id: Id,\n        metadata: &'static Metadata<'static>,\n    },\n    None,\n    Unknown,\n}\n\n// ===== impl Span =====\n\nimpl Id {\n    /// Constructs a new span ID from the given `u64`.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\">\n    /// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n    /// <strong>Note</strong>: Span IDs must be greater than zero.</pre></div>\n    ///\n    /// # Panics\n    /// - If the provided `u64` is 0.\n    pub fn from_u64(u: u64) -> Self {\n        Id(NonZeroU64::new(u).expect(\"span IDs must be > 0\"))\n    }\n\n    /// Constructs a new span ID from the given `NonZeroU64`.\n    ///\n    /// Unlike [`Id::from_u64`](Self::from_u64), this will never panic.\n    #[inline]\n    pub const fn from_non_zero_u64(id: NonZeroU64) -> Self {\n        Id(id)\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `u64`.\n    pub fn into_u64(&self) -> u64 {\n        self.0.get()\n    }\n\n    // Allow `into` by-ref since we don't want to impl Copy for Id\n    #[allow(clippy::wrong_self_convention)]\n    /// Returns the span's ID as a `NonZeroU64`.\n    #[inline]\n    pub const fn into_non_zero_u64(&self) -> NonZeroU64 {\n        self.0\n    }\n}\n\nimpl<'a> From<&'a Id> for Option<Id> {\n    fn from(id: &'a Id) -> Self {\n        Some(id.clone())\n    }\n}\n\n// ===== impl Attributes =====\n\nimpl<'a> Attributes<'a> {\n    /// Returns `Attributes` describing a new child span of the current span,\n    /// with the provided metadata and values.\n    pub fn new(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Current,\n        }\n    }\n\n    /// Returns `Attributes` describing a new span at the root of its own trace\n    /// tree, with the provided metadata and values.\n    pub fn new_root(metadata: &'static Metadata<'static>, values: &'a field::ValueSet<'a>) -> Self {\n        Attributes {\n            metadata,\n            values,\n            parent: Parent::Root,\n        }\n    }\n\n    /// Returns `Attributes` describing a new child span of the specified\n    /// parent span, with the provided metadata and values.\n...\n// Path: tracing-core/src/dispatch.rs\n//! Dispatches trace events to [`Collect`]s.\n//!\n//! The _dispatcher_ is the component of the tracing system which is responsible\n//! for forwarding trace data from the instrumentation points that generate it\n//! to the collector that collects it.\n//!\n//! # Using the Trace Dispatcher\n//!\n//! Every thread in a program using `tracing` has a _default collector_. When\n//! events occur, or spans are created, they are dispatched to the thread's\n//! current collector.\n//!\n//! ## Setting the Default Collector\n//!\n//! By default, the current collector is an empty implementation that does\n//! nothing. Trace data provided to this \"do nothing\" implementation is\n//! immediately discarded, and is not available for any purpose.\n//!\n//! To use another collector implementation, it must be set as the default.\n//! There are two methods for doing so: [`with_default`] and\n//! [`set_global_default`]. `with_default` sets the default collector for the\n//! duration of a scope, while `set_global_default` sets a default collector\n//! for the entire process.\n//!\n//! To use either of these functions, we must first wrap our collector in a\n//! [`Dispatch`], a cloneable, type-erased reference to a collector. For\n//! example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"alloc\")]\n//! use dispatch::Dispatch;\n//!\n//! # #[cfg(feature = \"alloc\")]\n//! let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"alloc\")]\n//! let my_dispatch = Dispatch::new(my_collector);\n//! ```\n//! Then, we can use [`with_default`] to set our `Dispatch` as the default for\n//! the duration of a block:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # let _my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(_my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::with_default(&my_dispatch, || {\n//!     // my_collector is the default\n//! });\n//!\n//! // no default collector again\n//! ```\n//! It's important to note that `with_default` will not propagate the current\n//! thread's default collector to any threads spawned within the `with_default`\n//! block. To propagate the default collector to new threads, either use\n//! `with_default` from the new thread, or use `set_global_default`.\n//!\n//! As an alternative to `with_default`, we can use [`set_global_default`] to\n//! set a `Dispatch` as the default for all threads, for the lifetime of the\n//! program. For example:\n//! ```rust\n//! # pub struct FooCollector;\n//! # use tracing_core::{\n//! #   dispatch, Event, Metadata,\n//! #   span::{Attributes, Current, Id, Record}\n//! # };\n//! # impl tracing_core::Collect for FooCollector {\n//! #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n//! #   fn record(&self, _: &Id, _: &Record) {}\n//! #   fn event(&self, _: &Event) {}\n//! #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n//! #   fn enabled(&self, _: &Metadata) -> bool { false }\n//! #   fn enter(&self, _: &Id) {}\n//! #   fn exit(&self, _: &Id) {}\n//! #   fn current_span(&self) -> Current { Current::unknown() }\n//! # }\n//! # impl FooCollector { fn new() -> Self { FooCollector } }\n//! # #[cfg(feature = \"std\")]\n//! # let my_collector = FooCollector::new();\n//! # #[cfg(feature = \"std\")]\n//! # let my_dispatch = dispatch::Dispatch::new(my_collector);\n//! // no default collector\n//!\n//! # #[cfg(feature = \"std\")]\n//! dispatch::set_global_default(my_dispatch)\n//!     // `set_global_default` will return an error if the global default\n//!     // collector has already been set.\n//!     .expect(\"global default was already set!\");\n//!\n//! // `my_collector` is now the default\n//! ```\n//!\n//! <div class=\"example-wrap\" style=\"display:inline-block\">\n//! <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n//!\n//! **Note**: the thread-local scoped dispatcher ([`with_default`]) requires the\n//! Rust standard library. `no_std` users should use [`set_global_default`] instead.\n//!\n//! </pre></div>\n//!\n//! ## Accessing the Default Collector\n//!\n//! A thread's current default collector can be accessed using the\n//! [`get_default`] function, which executes a closure with a reference to the\n//! currently default `Dispatch`. This is used primarily by `tracing`\n//! instrumentation.\nuse crate::{\n    collect::{self, Collect, NoCollector},\n    span, Event, LevelFilter, Metadata,\n};\n\nuse core::{\n    any::Any,\n    fmt,\n    sync::atomic::{AtomicBool, AtomicUsize, Ordering},\n};\n\n#[cfg(feature = \"std\")]\nuse std::{\n    cell::{Cell, RefCell, RefMut},\n    error,\n};\n\n#[cfg(feature = \"alloc\")]\nuse alloc::sync::{Arc, Weak};\n\n#[cfg(feature = \"alloc\")]\nuse core::ops::Deref;\n\n/// `Dispatch` trace data to a [`Collect`].\n#[derive(Clone)]\npub struct Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Arc<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n/// `WeakDispatch` is a version of [`Dispatch`] that holds a non-owning reference\n/// to a [collector].\n///\n/// The collector may be accessed by calling [`WeakDispatch::upgrade`],\n/// which returns an `Option<Dispatch>`. If all [`Dispatch`] clones that point\n/// at the collector have been dropped, [`WeakDispatch::upgrade`] will return\n/// `None`. Otherwise, it will return `Some(Dispatch)`.\n///\n/// A `WeakDispatch` may be created from a [`Dispatch`] by calling the\n/// [`Dispatch::downgrade`] method. The primary use for creating a\n/// [`WeakDispatch`] is to allow a collector to hold a cyclical reference to\n/// itself without creating a memory leak. See [here] for details.\n///\n/// This type is analogous to the [`std::sync::Weak`] type, but for a\n/// [`Dispatch`] rather than an [`Arc`].\n///\n/// [collector]: Collect\n/// [`Arc`]: std::sync::Arc\n/// [here]: Collect#avoiding-memory-leaks\n#[derive(Clone)]\npub struct WeakDispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind<Weak<dyn Collect + Send + Sync>>,\n\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &'static (dyn Collect + Send + Sync),\n}\n\n#[cfg(feature = \"alloc\")]\n#[derive(Clone)]\nenum Kind<T> {\n    Global(&'static (dyn Collect + Send + Sync)),\n    Scoped(T),\n}\n\n#[cfg(feature = \"std\")]\nthread_local! {\n    static CURRENT_STATE: State = const {\n        State {\n            default: RefCell::new(None),\n            can_enter: Cell::new(true),\n        }\n    };\n}\n\nstatic EXISTS: AtomicBool = AtomicBool::new(false);\nstatic GLOBAL_INIT: AtomicUsize = AtomicUsize::new(UNINITIALIZED);\n\n#[cfg(feature = \"std\")]\nstatic SCOPED_COUNT: AtomicUsize = AtomicUsize::new(0);\n\nconst UNINITIALIZED: usize = 0;\nconst INITIALIZING: usize = 1;\nconst INITIALIZED: usize = 2;\n\nstatic mut GLOBAL_DISPATCH: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NONE: Dispatch = Dispatch {\n    #[cfg(feature = \"alloc\")]\n    collector: Kind::Global(&NO_COLLECTOR),\n    #[cfg(not(feature = \"alloc\"))]\n    collector: &NO_COLLECTOR,\n};\nstatic NO_COLLECTOR: NoCollector = NoCollector::new();\n\n/// The dispatch state of a thread.\n#[cfg(feature = \"std\")]\nstruct State {\n    /// This thread's current default dispatcher.\n    default: RefCell<Option<Dispatch>>,\n    /// Whether or not we can currently begin dispatching a trace event.\n    ///\n    /// This is set to `false` when functions such as `enter`, `exit`, `event`,\n    /// and `new_span` are called on this thread's default dispatcher, to\n    /// prevent further trace events triggered inside those functions from\n    /// creating an infinite recursion. When we finish handling a dispatch, this\n    /// is set back to `true`.\n    can_enter: Cell<bool>,\n}\n\n/// While this guard is active, additional calls to collector functions on\n/// the default dispatcher will not be able to access the dispatch context.\n/// Dropping the guard will allow the dispatch context to be re-entered.\n#[cfg(feature = \"std\")]\nstruct Entered<'a>(&'a State);\n\n/// A guard that resets the current default dispatcher to the prior\n/// default dispatcher when dropped.\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[derive(Debug)]\npub struct DefaultGuard(Option<Dispatch>);\n\n/// Sets this dispatch as the default for the duration of a closure.\n///\n/// The default dispatcher is used when creating a new [span] or\n/// [`Event`].\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Note</strong>: This function required the Rust standard library.\n/// <!-- hack: this whitespace makes rustdoc interpret the next line as markdown again -->\n///\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\npub fn with_default<T>(dispatcher: &Dispatch, f: impl FnOnce() -> T) -> T {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this (rather than simply resetting after calling\n    // `f`) ensures that we always reset to the prior dispatcher even if `f`\n    // panics.\n    let _guard = set_default(dispatcher);\n    f()\n}\n\n/// Sets the dispatch as the default dispatch for the duration of the lifetime\n/// of the returned DefaultGuard\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\">\n/// <pre class=\"ignore\" style=\"white-space:normal;font:inherit;\">\n///\n/// **Note**: This function required the Rust standard library.\n/// `no_std` users should use [`set_global_default`] instead.\n///\n/// </pre></div>\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\n#[must_use = \"Dropping the guard unregisters the dispatcher.\"]\npub fn set_default(dispatcher: &Dispatch) -> DefaultGuard {\n    // When this guard is dropped, the default dispatcher will be reset to the\n    // prior default. Using this ensures that we always reset to the prior\n    // dispatcher even if the thread calling this function panics.\n    State::set_default(dispatcher.clone())\n}\n\n/// Sets this dispatch as the global default for the duration of the entire program.\n/// Will be used as a fallback if no thread-local dispatch has been set in a thread\n/// (using `with_default`.)\n///\n/// Can only be set once; subsequent attempts to set the global default will fail.\n/// Returns `Err` if the global default has already been set.\n///\n///\n/// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n/// <strong>Warning</strong>: In general, libraries should <em>not</em> call\n/// <code>set_global_default()</code>! Doing so will cause conflicts when\n/// executables that depend on the library try to set the default collector later.\n/// </pre></div>\n///\n/// [span]: super::span\n/// [`Event`]: super::event::Event\npub fn set_global_default(dispatcher: Dispatch) -> Result<(), SetGlobalDefaultError> {\n    // if `compare_exchange` returns Result::Ok(_), then `new` has been set and\n    // `current`\u2014now the prior value\u2014has been returned in the `Ok()` branch.\n    if GLOBAL_INIT\n        .compare_exchange(\n            UNINITIALIZED,\n            INITIALIZING,\n            Ordering::SeqCst,\n            Ordering::SeqCst,\n        )\n        .is_ok()\n    {\n        #[cfg(feature = \"alloc\")]\n        let collector = {\n            let collector = match dispatcher.collector {\n                Kind::Global(s) => s,\n                Kind::Scoped(s) => unsafe {\n                    // safety: this leaks the collector onto the heap. the\n                    // reference count will always be at least 1.\n                    &*Arc::into_raw(s)\n                },\n            };\n            Kind::Global(collector)\n        };\n\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = dispatcher.collector;\n\n        unsafe {\n            GLOBAL_DISPATCH = Dispatch { collector };\n        }\n        GLOBAL_INIT.store(INITIALIZED, Ordering::SeqCst);\n        EXISTS.store(true, Ordering::Release);\n        Ok(())\n    } else {\n        Err(SetGlobalDefaultError { _no_construct: () })\n    }\n}\n\n/// Returns true if a `tracing` dispatcher has ever been set.\n///\n/// This may be used to completely elide trace points if tracing is not in use\n/// at all or has yet to be initialized.\n#[doc(hidden)]\n#[inline(always)]\npub fn has_been_set() -> bool {\n    EXISTS.load(Ordering::Relaxed)\n}\n\n/// Returned if setting the global dispatcher fails.\npub struct SetGlobalDefaultError {\n    _no_construct: (),\n}\n\nimpl fmt::Debug for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_tuple(\"SetGlobalDefaultError\")\n            .field(&Self::MESSAGE)\n            .finish()\n    }\n}\n\nimpl fmt::Display for SetGlobalDefaultError {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.pad(Self::MESSAGE)\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[cfg_attr(docsrs, doc(cfg(feature = \"std\")))]\nimpl error::Error for SetGlobalDefaultError {}\n\nimpl SetGlobalDefaultError {\n    const MESSAGE: &'static str = \"a global default trace dispatcher has already been set\";\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatch::Dispatch\n#[cfg(feature = \"std\")]\n#[inline(always)]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    if SCOPED_COUNT.load(Ordering::Acquire) == 0 {\n        // fast path if no scoped dispatcher has been set; just use the global\n        // default.\n        return f(get_global());\n    }\n\n    get_default_slow(f)\n}\n\n#[cfg(feature = \"std\")]\n#[inline(never)]\nfn get_default_slow<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    // While this guard is active, additional calls to collector functions on\n    // the default dispatcher will not be able to access the dispatch context.\n    // Dropping the guard will allow the dispatch context to be re-entered.\n    struct Entered<'a>(&'a Cell<bool>);\n    impl<'a> Drop for Entered<'a> {\n        #[inline]\n        fn drop(&mut self) {\n            self.0.set(true);\n        }\n    }\n\n    CURRENT_STATE\n        .try_with(|state| {\n            if state.can_enter.replace(false) {\n                let _guard = Entered(&state.can_enter);\n\n                let mut default = state.default.borrow_mut();\n                let default = default\n                    // if the local default for this thread has never been set,\n                    // populate it with the global default, so we don't have to\n                    // keep getting the global on every `get_default_slow` call.\n                    .get_or_insert_with(|| get_global().clone());\n\n                return f(&*default);\n            }\n\n            f(&Dispatch::none())\n        })\n        .unwrap_or_else(|_| f(&Dispatch::none()))\n}\n\n/// Executes a closure with a reference to this thread's current [dispatcher].\n///\n/// Note that calls to `get_default` should not be nested; if this function is\n/// called while inside of another `get_default`, that closure will be provided\n/// with `Dispatch::none` rather than the previously set dispatcher.\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(feature = \"std\")]\n#[doc(hidden)]\n#[inline(never)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    CURRENT_STATE\n        .try_with(|state| {\n            let entered = state.enter()?;\n            Some(f(&entered.current()))\n        })\n        .ok()?\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\n#[doc(hidden)]\npub fn get_current<T>(f: impl FnOnce(&Dispatch) -> T) -> Option<T> {\n    Some(f(&get_global()))\n}\n\n/// Executes a closure with a reference to the current [dispatcher].\n///\n/// [dispatcher]: super::dispatcher::Dispatch\n#[cfg(not(feature = \"std\"))]\npub fn get_default<T, F>(mut f: F) -> T\nwhere\n    F: FnMut(&Dispatch) -> T,\n{\n    f(get_global())\n}\n\n#[inline(always)]\npub(crate) fn get_global() -> &'static Dispatch {\n    if GLOBAL_INIT.load(Ordering::Acquire) != INITIALIZED {\n        return &NONE;\n    }\n    unsafe {\n        // This is safe given the invariant that setting the global dispatcher\n        // also sets `GLOBAL_INIT` to `INITIALIZED`.\n        &GLOBAL_DISPATCH\n    }\n}\n\n#[cfg(feature = \"std\")]\npub(crate) struct Registrar(Kind<Weak<dyn Collect + Send + Sync>>);\n\nimpl Dispatch {\n    /// Returns a new `Dispatch` that discards events and spans.\n    #[inline]\n    pub fn none() -> Self {\n        Dispatch {\n            #[cfg(feature = \"alloc\")]\n            collector: Kind::Global(&NO_COLLECTOR),\n            #[cfg(not(feature = \"alloc\"))]\n            collector: &NO_COLLECTOR,\n        }\n    }\n\n    /// Returns a `Dispatch` that forwards to the given [`Collect`].\n    ///\n    /// [`Collect`]: super::collect::Collect\n    #[cfg(feature = \"alloc\")]\n    #[cfg_attr(docsrs, doc(cfg(any(feature = \"std\", feature = \"alloc\"))))]\n    pub fn new<C>(collector: C) -> Self\n    where\n        C: Collect + Send + Sync + 'static,\n    {\n        let me = Dispatch {\n            collector: Kind::Scoped(Arc::new(collector)),\n        };\n        crate::callsite::register_dispatch(&me);\n        me\n    }\n\n    /// Returns a `Dispatch` that forwards to the given static [collector].\n    ///\n    /// Unlike [`Dispatch::new`], this function is always available on all\n    /// platforms, even when the `std` or `alloc` features are disabled.\n    ///\n    /// In order to use `from_static`, the `Collector` itself must be stored in\n    /// a static. For example:\n    ///\n    /// ```rust\n    /// struct MyCollector {\n    ///    // ...\n    /// }\n    ///\n    /// # use tracing_core::{span::{Id, Attributes, Current, Record}, Event, Metadata};\n    /// impl tracing_core::Collect for MyCollector {\n    ///     // ...\n    /// #   fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) }\n    /// #   fn record(&self, _: &Id, _: &Record) {}\n    /// #   fn event(&self, _: &Event) {}\n    /// #   fn record_follows_from(&self, _: &Id, _: &Id) {}\n    /// #   fn enabled(&self, _: &Metadata) -> bool { false }\n    /// #   fn enter(&self, _: &Id) {}\n    /// #   fn exit(&self, _: &Id) {}\n    /// #   fn current_span(&self) -> Current { Current::unknown() }\n    /// }\n    ///\n    /// static COLLECTOR: MyCollector = MyCollector {\n    ///     // ...\n    /// };\n    ///\n    /// fn main() {\n    ///     use tracing_core::dispatch::{self, Dispatch};\n    ///\n    ///     let dispatch = Dispatch::from_static(&COLLECTOR);\n    ///\n    ///     dispatch::set_global_default(dispatch)\n    ///         .expect(\"no global default collector should have been set previously!\");\n    /// }\n    /// ```\n    ///\n    /// Constructing the collector in a static initializer may make some forms\n    /// of runtime configuration more challenging. If this is the case, users\n    /// with access to `liballoc` or the Rust standard library are encouraged to\n    /// use [`Dispatch::new`] rather than `from_static`. `no_std` users who\n    /// cannot allocate or do not have access to `liballoc` may want to consider\n    /// the [`once_cell`] crate, or another library which allows lazy\n    /// initialization of statics.\n    ///\n    /// [collector]: super::collect::Collect\n    /// [`once_cell`]: https://crates.io/crates/once_cell\n    pub fn from_static(collector: &'static (dyn Collect + Send + Sync)) -> Self {\n        #[cfg(feature = \"alloc\")]\n        let me = Self {\n            collector: Kind::Global(collector),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let me = Self { collector };\n        crate::callsite::register_dispatch(&me);\n        me\n    }\n\n    /// Creates a [`WeakDispatch`] from this `Dispatch`.\n    ///\n    /// A [`WeakDispatch`] is similar to a [`Dispatch`], but it does not prevent\n    /// the underlying [collector] from being dropped. Instead, it only permits\n    /// access while other references to the collector exist. This is equivalent\n    /// to the standard library's [`Arc::downgrade`] method, but for `Dispatch`\n    /// rather than `Arc`.\n    ///\n    /// The primary use for creating a [`WeakDispatch`] is to allow a collector\n    /// to hold a cyclical reference to itself without creating a memory leak.\n    /// See [here] for details.\n    ///\n    /// [collector]: Collect\n    /// [`Arc::downgrade`]: std::sync::Arc::downgrade\n    /// [here]: Collect#avoiding-memory-leaks\n    pub fn downgrade(&self) -> WeakDispatch {\n        #[cfg(feature = \"alloc\")]\n        let collector = match &self.collector {\n            Kind::Global(dispatch) => Kind::Global(*dispatch),\n            Kind::Scoped(dispatch) => Kind::Scoped(Arc::downgrade(dispatch)),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = self.collector;\n\n        WeakDispatch { collector }\n    }\n\n    #[cfg(feature = \"std\")]\n    pub(crate) fn registrar(&self) -> Registrar {\n        Registrar(match self.collector {\n            Kind::Scoped(ref s) => Kind::Scoped(Arc::downgrade(s)),\n            Kind::Global(s) => Kind::Global(s),\n        })\n    }\n\n    #[inline(always)]\n    #[cfg(feature = \"alloc\")]\n    pub(crate) fn collector(&self) -> &(dyn Collect + Send + Sync) {\n        match self.collector {\n            Kind::Scoped(ref s) => Arc::deref(s),\n            Kind::Global(s) => s,\n        }\n    }\n\n    #[inline(always)]\n    #[cfg(not(feature = \"alloc\"))]\n    pub(crate) fn collector(&self) -> &(dyn Collect + Send + Sync) {\n        self.collector\n    }\n\n    /// Registers a new callsite with this collector, returning whether or not\n    /// the collector is interested in being notified about the callsite.\n    ///\n    /// This calls the [`register_callsite`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`register_callsite`]: super::collect::Collect::register_callsite\n    #[inline]\n    pub fn register_callsite(&self, metadata: &'static Metadata<'static>) -> collect::Interest {\n        self.collector().register_callsite(metadata)\n    }\n\n    /// Returns the highest [verbosity level][level] that this [collector] will\n    /// enable, or `None`, if the collector does not implement level-based\n    /// filtering or chooses not to implement this method.\n    ///\n    /// This calls the [`max_level_hint`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [level]: super::Level\n    /// [collector]: super::collect::Collect\n    /// [`Collect`]: super::collect::Collect\n    /// [`register_callsite`]: super::collect::Collect::max_level_hint\n    // TODO(eliza): consider making this a public API?\n    #[inline]\n    pub(crate) fn max_level_hint(&self) -> Option<LevelFilter> {\n        self.collector().max_level_hint()\n    }\n\n    /// Record the construction of a new span, returning a new [ID] for the\n    /// span being constructed.\n    ///\n    /// This calls the [`new_span`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`new_span`]: super::collect::Collect::new_span\n    #[inline]\n    pub fn new_span(&self, span: &span::Attributes<'_>) -> span::Id {\n        self.collector().new_span(span)\n    }\n\n    /// Record a set of values on a span.\n    ///\n    /// This calls the [`record`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`record`]: super::collect::Collect::record\n    #[inline]\n    pub fn record(&self, span: &span::Id, values: &span::Record<'_>) {\n        self.collector().record(span, values)\n    }\n\n    /// Adds an indication that `span` follows from the span with the id\n    /// `follows`.\n    ///\n    /// This calls the [`record_follows_from`] function on the [`Collect`]\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`record_follows_from`]: super::collect::Collect::record_follows_from\n    #[inline]\n    pub fn record_follows_from(&self, span: &span::Id, follows: &span::Id) {\n        self.collector().record_follows_from(span, follows)\n    }\n\n    /// Returns true if a span with the specified [metadata] would be\n    /// recorded.\n    ///\n    /// This calls the [`enabled`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [metadata]: super::metadata::Metadata\n    /// [`Collect`]: super::collect::Collect\n    /// [`enabled`]: super::collect::Collect::enabled\n    #[inline]\n    pub fn enabled(&self, metadata: &Metadata<'_>) -> bool {\n        self.collector().enabled(metadata)\n    }\n\n    /// Records that an [`Event`] has occurred.\n    ///\n    /// This calls the [`event`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Event`]: super::event::Event\n    /// [`Collect`]: super::collect::Collect\n    /// [`event`]: super::collect::Collect::event\n    #[inline]\n    pub fn event(&self, event: &Event<'_>) {\n        let collector = self.collector();\n        if collector.event_enabled(event) {\n            collector.event(event);\n        }\n    }\n\n    /// Records that a span has been can_enter.\n    ///\n    /// This calls the [`enter`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`enter`]: super::collect::Collect::enter\n    pub fn enter(&self, span: &span::Id) {\n        self.collector().enter(span);\n    }\n\n    /// Records that a span has been exited.\n    ///\n    /// This calls the [`exit`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`exit`]: super::collect::Collect::exit\n    pub fn exit(&self, span: &span::Id) {\n        self.collector().exit(span);\n    }\n\n    /// Notifies the [collector] that a [span ID] has been cloned.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`clone_span`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [span ID]: super::span::Id\n    /// [collector]: super::collect::Collect\n    /// [`clone_span`]: super::collect::Collect::clone_span\n    /// [`new_span`]: super::collect::Collect::new_span\n    #[inline]\n    pub fn clone_span(&self, id: &span::Id) -> span::Id {\n        self.collector().clone_span(id)\n    }\n\n    /// Notifies the collector that a [span ID] has been dropped.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`drop_span`] function on the [`Collect`] that this\n    ///  `Dispatch` forwards to.\n    ///\n    /// <div class=\"example-wrap\" style=\"display:inline-block\"><pre class=\"compile_fail\" style=\"white-space:normal;font:inherit;\">\n    ///\n    /// **Deprecated**: The [`try_close`] method is functionally identical, but returns `true` if the span is now closed.\n    /// It should be used instead of this method.\n    ///\n    /// </pre></div>\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`drop_span`]: super::collect::Collect::drop_span\n    /// [`new_span`]: super::collect::Collect::new_span\n    /// [`try_close`]: Self::try_close\n    #[inline]\n    #[deprecated(since = \"0.1.2\", note = \"use `Dispatch::try_close` instead\")]\n    pub fn drop_span(&self, id: span::Id) {\n        #[allow(deprecated)]\n        self.collector().drop_span(id);\n    }\n\n    /// Notifies the collector that a [span ID] has been dropped, and returns\n    /// `true` if there are now 0 IDs referring to that span.\n    ///\n    /// This function must only be called with span IDs that were returned by\n    /// this `Dispatch`'s [`new_span`] function. The `tracing` crate upholds\n    /// this guarantee and any other libraries implementing instrumentation APIs\n    /// must as well.\n    ///\n    /// This calls the [`try_close`] function on the [`Collect`] trait\n    /// that this `Dispatch` forwards to.\n    ///\n    /// [span ID]: super::span::Id\n    /// [`Collect`]: super::collect::Collect\n    /// [`try_close`]: super::collect::Collect::try_close\n    /// [`new_span`]: super::collect::Collect::new_span\n    pub fn try_close(&self, id: span::Id) -> bool {\n        self.collector().try_close(id)\n    }\n\n    /// Returns a type representing this collector's view of the current span.\n    ///\n    /// This calls the [`current`] function on the [`Collect`] that this\n    /// `Dispatch` forwards to.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    /// [`current`]: super::collect::Collect::current_span\n    #[inline]\n    pub fn current_span(&self) -> span::Current {\n        self.collector().current_span()\n    }\n\n    /// Returns `true` if this `Dispatch` forwards to a collector of type\n    /// `T`.\n    #[inline]\n    pub fn is<T: Any>(&self) -> bool {\n        <dyn Collect>::is::<T>(self.collector())\n    }\n\n    /// Returns some reference to the [`Collect`] this `Dispatch` forwards to\n    /// if it is of type `T`, or `None` if it isn't.\n    ///\n    /// [`Collect`]: super::collect::Collect\n    #[inline]\n    pub fn downcast_ref<T: Any>(&self) -> Option<&T> {\n        <dyn Collect>::downcast_ref(self.collector())\n    }\n}\n\nimpl Default for Dispatch {\n    /// Returns the current default dispatcher\n    fn default() -> Self {\n        get_default(|default| default.clone())\n    }\n}\n\nimpl fmt::Debug for Dispatch {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match &self.collector {\n            #[cfg(feature = \"alloc\")]\n            Kind::Global(collector) => f\n                .debug_tuple(\"Dispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(feature = \"alloc\")]\n            Kind::Scoped(collector) => f\n                .debug_tuple(\"Dispatch::Scoped\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(not(feature = \"alloc\"))]\n            collector => f\n                .debug_tuple(\"Dispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<C> From<C> for Dispatch\nwhere\n    C: Collect + Send + Sync + 'static,\n{\n    #[inline]\n    fn from(collector: C) -> Self {\n        Dispatch::new(collector)\n    }\n}\n\nimpl WeakDispatch {\n    /// Attempts to upgrade this `WeakDispatch` to a [`Dispatch`].\n    ///\n    /// Returns `None` if the referenced `Dispatch` has already been dropped.\n    ///\n    /// ## Examples\n    ///\n    /// ```\n    /// # use tracing_core::collect::NoCollector;\n    /// # use tracing_core::dispatch::Dispatch;\n    /// static COLLECTOR: NoCollector = NoCollector::new();\n    /// let strong = Dispatch::new(COLLECTOR);\n    /// let weak = strong.downgrade();\n    ///\n    /// // The strong here keeps it alive, so we can still access the object.\n    /// assert!(weak.upgrade().is_some());\n    ///\n    /// drop(strong); // But not any more.\n    /// assert!(weak.upgrade().is_none());\n    /// ```\n    pub fn upgrade(&self) -> Option<Dispatch> {\n        #[cfg(feature = \"alloc\")]\n        let collector = match &self.collector {\n            Kind::Global(dispatch) => Some(Kind::Global(*dispatch)),\n            Kind::Scoped(dispatch) => dispatch.upgrade().map(Kind::Scoped),\n        };\n        #[cfg(not(feature = \"alloc\"))]\n        let collector = Some(self.collector);\n\n        collector.map(|collector| Dispatch { collector })\n    }\n}\n\nimpl fmt::Debug for WeakDispatch {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match &self.collector {\n            #[cfg(feature = \"alloc\")]\n            Kind::Global(collector) => f\n                .debug_tuple(\"WeakDispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(feature = \"alloc\")]\n            Kind::Scoped(collector) => f\n                .debug_tuple(\"WeakDispatch::Scoped\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n\n            #[cfg(not(feature = \"alloc\"))]\n            collector => f\n                .debug_tuple(\"WeakDispatch::Global\")\n                .field(&format_args!(\"{:p}\", collector))\n                .finish(),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl Registrar {\n    pub(crate) fn upgrade(&self) -> Option<Dispatch> {\n        match self.0 {\n            Kind::Global(s) => Some(Dispatch {\n                collector: Kind::Global(s),\n            }),\n            Kind::Scoped(ref s) => s.upgrade().map(|s| Dispatch {\n                collector: Kind::Scoped(s),\n            }),\n        }\n    }\n}\n\n// ===== impl State =====\n\n#[cfg(feature = \"std\")]\nimpl State {\n    /// Replaces the current default dispatcher on this thread with the provided\n    /// dispatcher.\n    ///\n    /// Dropping the returned `ResetGuard` will reset the default dispatcher to\n    /// the previous value.\n    #[inline]\n    fn set_default(new_dispatch: Dispatch) -> DefaultGuard {\n        let prior = CURRENT_STATE\n            .try_with(|state| {\n                state.can_enter.set(true);\n                state\n                    .default\n                    .replace(Some(new_dispatch))\n                    // if the scoped default was not set on this thread, set the\n                    // `prior` default to the global default to populate the\n                    // scoped default when unsetting *this* default\n                    .unwrap_or_else(|| get_global().clone())\n            })\n            .ok();\n        EXISTS.store(true, Ordering::Release);\n        SCOPED_COUNT.fetch_add(1, Ordering::Release);\n        DefaultGuard(prior)\n    }\n\n    #[inline]\n    fn enter(&self) -> Option<Entered<'_>> {\n        if self.can_enter.replace(false) {\n            Some(Entered(self))\n        } else {\n            None\n        }\n    }\n}\n\n// ===== impl Entered =====\n\n#[cfg(feature = \"std\")]\nimpl<'a> Entered<'a> {\n    #[inline]\n    fn current(&self) -> RefMut<'a, Dispatch> {\n        let default = self.0.default.borrow_mut();\n        RefMut::map(default, |default| {\n            default.get_or_insert_with(|| get_global().clone())\n        })\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<'a> Drop for Entered<'a> {\n    #[inline]\n    fn drop(&mut self) {\n        self.0.can_enter.set(true);\n    }\n}\n\n// ===== impl DefaultGuard =====\n\n#[cfg(feature = \"std\")]\nimpl Drop for DefaultGuard {\n    #[inline]\n    fn drop(&mut self) {\n        SCOPED_COUNT.fetch_sub(1, Ordering::Release);\n        if let Some(dispatch) = self.0.take() {\n            // Replace the dispatcher and then drop the old one outside\n            // of the thread-local context. Dropping the dispatch may\n            // lead to the drop of a collector which, in the process,\n            // could then also attempt to access the same thread local\n            // state -- causing a clash.\n            let prev = CURRENT_STATE.try_with(|state| state.default.replace(Some(dispatch)));\n            drop(prev)\n        }\n    }\n}\n\n#[cfg(test)]\nmod test {\n\n    use super::*;\n    use crate::{\n        callsite::Callsite,\n        collect::Interest,\n        metadata::{Kind, Level, Metadata},\n    };\n\n    #[test]\n    fn dispatch_is() {\n        let dispatcher = Dispatch::from_static(&NO_COLLECTOR);\n        assert!(dispatcher.is::<NoCollector>());\n    }\n\n    #[test]\n    fn dispatch_downcasts() {\n        let dispatcher = Dispatch::from_static(&NO_COLLECTOR);\n        assert!(dispatcher.downcast_ref::<NoCollector>().is_some());\n    }\n\n    struct TestCallsite;\n    static TEST_CALLSITE: TestCallsite = TestCallsite;\n    static TEST_META: Metadata<'static> = metadata! {\n        name: \"test\",\n        target: module_path!(),\n        level: Level::DEBUG,\n        fields: &[],\n        callsite: &TEST_CALLSITE,\n        kind: Kind::EVENT\n    };\n\n    impl Callsite for TestCallsite {\n        fn set_interest(&self, _: Interest) {}\n        fn metadata(&self) -> &Metadata<'_> {\n            &TEST_META\n        }\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn events_dont_infinite_loop() {\n        // This test ensures that an event triggered within a collector\n        // won't cause an infinite loop of events.\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {\n                static EVENTS: AtomicUsize = AtomicUsize::new(0);\n                assert_eq!(\n                    EVENTS.fetch_add(1, Ordering::Relaxed),\n                    0,\n                    \"event method called twice!\"\n                );\n                Event::dispatch(&TEST_META, &TEST_META.fields().value_set(&[]))\n            }\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n\n        with_default(&Dispatch::new(TestCollector), || {\n            Event::dispatch(&TEST_META, &TEST_META.fields().value_set(&[]))\n        })\n    }\n\n    #[test]\n    #[cfg(feature = \"std\")]\n    fn spans_dont_infinite_loop() {\n        // This test ensures that a span created within a collector\n        // won't cause an infinite loop of new spans.\n\n        fn mk_span() {\n            get_default(|current| {\n                current.new_span(&span::Attributes::new(\n                    &TEST_META,\n                    &TEST_META.fields().value_set(&[]),\n                ))\n            });\n        }\n\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                static NEW_SPANS: AtomicUsize = AtomicUsize::new(0);\n                assert_eq!(\n                    NEW_SPANS.fetch_add(1, Ordering::Relaxed),\n                    0,\n                    \"new_span method called twice!\"\n                );\n                mk_span();\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {}\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n\n        with_default(&Dispatch::new(TestCollector), mk_span)\n    }\n\n    #[test]\n    fn default_no_collector() {\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<NoCollector>());\n    }\n\n    #[cfg(feature = \"std\")]\n    #[test]\n    fn default_dispatch() {\n        struct TestCollector;\n        impl Collect for TestCollector {\n            fn enabled(&self, _: &Metadata<'_>) -> bool {\n                true\n            }\n\n            fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {\n                span::Id::from_u64(0xAAAA)\n            }\n\n            fn record(&self, _: &span::Id, _: &span::Record<'_>) {}\n\n            fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}\n\n            fn event(&self, _: &Event<'_>) {}\n\n            fn enter(&self, _: &span::Id) {}\n\n            fn exit(&self, _: &span::Id) {}\n\n            fn current_span(&self) -> span::Current {\n                span::Current::unknown()\n            }\n        }\n        let guard = set_default(&Dispatch::new(TestCollector));\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<TestCollector>());\n\n        drop(guard);\n        let default_dispatcher = Dispatch::default();\n        assert!(default_dispatcher.is::<NoCollector>());\n    }\n}\n\n// Path: tracing-core/src/collect.rs\n//! Collectors collect and record trace data.\nuse crate::{span, Dispatch, Event, LevelFilter, Metadata};\n\nuse core::any::{Any, TypeId};\nuse core::ptr::NonNull;\n\n/// Trait representing the functions required to collect trace data.\n///\n/// Crates that provide implementations of methods for collecting or recording\n/// trace data should implement the `Collect` interface. This trait is\n/// intended to represent fundamental primitives for collecting trace events and\n/// spans \u2014 other libraries may offer utility functions and types to make\n/// collector implementations more modular or improve the ergonomics of writing\n/// collectors.\n///\n/// A collector is responsible for the following:\n/// - Registering new spans as they are created, and providing them with span\n///   IDs. Implicitly, this means the collector may determine the strategy for\n///   determining span equality.\n/// - Recording the attachment of field values and follows-from annotations to\n///   spans.\n/// - Filtering spans and events, and determining when those filters must be\n///   invalidated.\n/// - Observing spans as they are entered, exited, and closed, and events as\n///   they occur.\n///\n/// When a span is entered or exited, the collector is provided only with the\n/// [ID] with which it tagged that span when it was created. This means\n/// that it is up to the collector to determine whether and how span _data_ \u2014\n/// the fields and metadata describing the span \u2014 should be stored. The\n/// [`new_span`] function is called when a new span is created, and at that\n/// point, the collector _may_ choose to store the associated data if it will\n/// be referenced again. However, if the data has already been recorded and will\n/// not be needed by the implementations of `enter` and `exit`, the collector\n/// may freely discard that data without allocating space to store it.\n///\n/// ## Overriding default impls\n///\n/// Some trait methods on `Collect` have default implementations, either in\n/// order to reduce the surface area of implementing `Collect`, or for\n/// backward-compatibility reasons. However, many collectors will likely want\n/// to override these default implementations.\n///\n/// The following methods are likely of interest:\n///\n/// - [`register_callsite`] is called once for each callsite from which a span\n///   event may originate, and returns an [`Interest`] value describing whether or\n///   not the collector wishes to see events or spans from that callsite. By\n///   default, it calls [`enabled`], and returns `Interest::always()` if\n///   `enabled` returns true, or `Interest::never()` if enabled returns false.\n///   However, if the collector's interest can change dynamically at runtime,\n///   it may want to override this function to return `Interest::sometimes()`.\n///   Additionally, collectors which wish to perform a behaviour once for each\n///   callsite, such as allocating storage for data related to that callsite,\n///   can perform it in `register_callsite`.\n///\n///   See also the [documentation on the callsite registry][cs-reg] for details\n///   on [`register_callsite`].\n///\n/// - [`event_enabled`] is called once before every call to the [`event`]\n///   method. This can be used to implement filtering on events once their field\n///   values are known, but before any processing is done in the `event` method.\n/// - [`clone_span`] is called every time a span ID is cloned, and [`try_close`]\n///   is called when a span ID is dropped. By default, these functions do\n///   nothing. However, they can be used to implement reference counting for\n///   spans, allowing collectors to free storage for span data and to determine\n///   when a span has _closed_ permanently (rather than being exited).\n///   Collectors which store per-span data or which need to track span closures\n///   should override these functions together.\n///\n/// [ID]: super::span::Id\n/// [`new_span`]: Collect::new_span\n/// [`register_callsite`]: Collect::register_callsite\n/// [`enabled`]: Collect::enabled\n/// [`clone_span`]: Collect::clone_span\n/// [`try_close`]: Collect::try_close\n/// [cs-reg]: crate::callsite#registering-callsites\n/// [`event`]: Collect::event\n/// [`event_enabled`]: Collect::event_enabled\npub trait Collect: 'static {\n    /// Invoked when this collector becomes a [`Dispatch`].\n    ///\n    /// ## Avoiding Memory Leaks\n    ///\n    /// Collectors should not store their own [`Dispatch`]. Because the\n    /// `Dispatch` owns the collector, storing the `Dispatch` within the\n    /// collector will create a reference count cycle, preventing the `Dispatch`\n    /// from ever being dropped.\n    ///\n    /// Instead, when it is necessary to store a cyclical reference to the\n    /// `Dispatch` within a collector, use [`Dispatch::downgrade`] to convert a\n    /// `Dispatch` into a [`WeakDispatch`]. This type is analogous to\n    /// [`std::sync::Weak`], and does not create a reference count cycle. A\n    /// [`WeakDispatch`] can be stored within a collector without causing a\n    /// memory leak, and can be [upgraded] into a `Dispatch` temporarily when\n    /// the `Dispatch` must be accessed by the collector.\n    ///\n    /// [`WeakDispatch`]: crate::dispatch::WeakDispatch\n    /// [upgraded]: crate::dispatch::WeakDispatch::upgrade\n    fn on_\nregister_dispatch(&self, collector: &Dispatch) {\n        let _ = collector;\n    }\n\n    // === Span registry methods ==============================================\n\n    /// Registers a new [callsite] with this collector, returning whether or not\n    /// the collector is interested in being notified about the callsite.\n    ///\n    /// By default, this function assumes that the collector's [filter]\n    /// represents an unchanging view of its interest in the callsite. However,\n    /// if this is not the case, collectors may override this function to\n    /// indicate different interests, or to implement behaviour that should run\n    /// once for every callsite.\n    ///\n    /// This function is guaranteed to be called at least once per callsite on\n    /// every active collector. The collector may store the keys to fields it\n    /// cares about in order to reduce the cost of accessing fields by name,\n    /// preallocate storage for that callsite, or perform any other actions it\n    /// wishes to perform once for each callsite.\n    ///\n    /// The collector should then return an [`Interest`], indicating\n    /// whether it is interested in being notified about that callsite in the\n    /// future. This may be `Always` indicating that the collector always\n    /// wishes to be notified about the callsite, and its filter need not be\n    /// re-evaluated; `Sometimes`, indicating that the collector may sometimes\n    /// care about the callsite but not always (such as when sampling), or\n    /// `Never`, indicating that the collector never wishes to be notified about\n    /// that callsite. If all active collectors return `Never`, a callsite will\n    /// never be enabled unless a new collector expresses interest in it.\n    ///\n    /// `Collector`s which require their filters to be run every time an event\n    /// occurs or a span is entered/exited should return `Interest::sometimes`.\n    /// If a collector returns `Interest::sometimes`, then its' [`enabled`] method\n    /// will be called every time an event or span is created from that callsite.\n    ///\n    /// For example, suppose a sampling collector is implemented by\n    /// incrementing a counter every time `enabled` is called and only returning\n    /// `true` when the counter is divisible by a specified sampling rate. If\n    /// that collector returns `Interest::always` from `register_callsite`, then\n    /// the filter will not be re-evaluated once it has been applied to a given\n    /// set of metadata. Thus, the counter will not be incremented, and the span\n    /// or event that corresponds to the metadata will never be `enabled`.\n    ///\n    /// `Collector`s that need to change their filters occasionally should call\n    /// [`rebuild_interest_cache`] to re-evaluate `register_callsite` for all\n    /// callsites.\n    ///\n    /// Similarly, if a `Collector` has a filtering strategy that can be\n    /// changed dynamically at runtime, it would need to re-evaluate that filter\n    /// if the cached results have changed.\n    ///\n    /// A collector which manages fanout to multiple other collectors\n    /// should proxy this decision to all of its child collectors,\n    /// returning `Interest::never` only if _all_ such children return\n    /// `Interest::never`. If the set of collectors to which spans are\n    /// broadcast may change dynamically, the collector should also never\n...\n", "needle_token_start": 15459, "needle_token_end": 15486, "code_context_ntokens": 16274, "output": ["```rust\n/// Invoked when this collector becomes a [`Dispatch`].\n///\n/// ## Avoiding Memory Leaks\n///\n/// Collectors should not store their own [`Dispatch`]. Because the\n/// `Dispatch` owns the collector, storing the `Dispatch` within the\n/// collector will create a reference count cycle, preventing the `Dispatch`\n/// from ever being dropped.\n///\n/// Instead, when it is necessary to store a cyclical reference to the\n/// `Dispatch` within a collector, use [`Dispatch::downgrade`] to convert a\n/// `Dispatch` into a [`WeakDispatch`]. This type is analogous to\n/// [`std::sync::Weak`], and does not create a reference count cycle. A\n/// [`WeakDispatch`] can be stored within a collector without causing a\n/// memory leak, and can be [upgraded] into a `Dispatch` temporarily when\n/// the `Dispatch` must be accessed by the collector.\n///\n/// [`WeakDispatch`]: crate::dispatch::WeakDispatch\n/// [upgraded]: crate::dispatch::WeakDispatch::upgrade\nfn on_register_dispatch(&self, collector: &Dispatch) {\n    let _ = collector;\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "pre_run_target_child", "language": "rust", "path": "libafl/src/executors/inprocess_fork/inner.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to prepare and configure the child process after a fork operation in a multi-process execution environment. It sets up shared memory, initializes timers, and runs necessary pre-execution hooks for observers and other components.\n2. **Input**: It takes references to the fuzzer state manager, the current state, the event manager, and the input to be processed.\n3. **Output**: Returns a result indicating success or an error if any step in the setup fails.\n4. **Procedure**: \n   - Initializes shared memory for the child process.\n   - Calls entry functions for target execution and pre-execution hooks.\n   - Executes observer pre-execution hooks.\n   - Depending on the operating system, sets up a timer using either `timer_settime` on Linux or `setitimer` on other Unix-like systems to manage execution time limits.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "where\n    OT: ObserversTuple<S> + Debug,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S> + Debug,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[cfg(target_os = \"linux\")]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"GenericInProcessForkExecutorInner\")\n            .field(\"observers\", &self.observers)\n            .field(\"shmem_provider\", &self.shmem_provider)\n            .field(\"itimerspec\", &self.itimerspec)\n            .finish_non_exhaustive()\n    }\n\n    #[cfg(not(target_os = \"linux\"))]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        #[cfg(not(target_os = \"linux\"))]\n        return f\n            .debug_struct(\"GenericInProcessForkExecutorInner\")\n            .field(\"observers\", &self.observers)\n            .field(\"shmem_provider\", &self.shmem_provider)\n            .field(\"itimerval\", &self.itimerval)\n            .finish_non_exhaustive();\n    }\n}\n\nimpl<HT, OT, S, SP, EM, Z> UsesState for GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>\nwhere\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type State = S;\n}\n\nimpl<EM, HT, OT, S, SP, Z> GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>\nwhere\n    OT: ObserversTuple<S> + Debug,\n    S: State + UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    Z: UsesState<State = S>,\n{\n    \npub(super) unsafe fn pre_run_target_child(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut <GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z> as UsesState>::State,\n        mgr: &mut EM,\n        input: &<GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z> as UsesInput>::Input,\n    ) -> Result<(), Error> {\n        self.shmem_provider.post_fork(true)?;\n\n        self.enter_target(fuzzer, state, mgr, input);\n        self.hooks.pre_exec_all(state, input);\n\n        self.observers\n            .pre_exec_child_all(state, input)\n            .expect(\"Failed to run post_exec on observers\");\n\n        #[cfg(target_os = \"linux\")]\n        {\n            let mut timerid: libc::timer_t = null_mut();\n            // creates a new per-process interval timer\n            // we can't do this from the parent, timerid is unique to each process.\n            libc::timer_create(libc::CLOCK_MONOTONIC, null_mut(), addr_of_mut!(timerid));\n\n            // log::info!(\"Set timer! {:#?} {timerid:#?}\", self.itimerspec);\n            let _: i32 = libc::timer_settime(timerid, 0, addr_of_mut!(self.itimerspec), null_mut());\n        }\n        #[cfg(not(target_os = \"linux\"))]\n        {\n            setitimer(ITIMER_REAL, &mut self.itimerval, null_mut());\n        }\n        // log::trace!(\"{v:#?} {}\", nix::errno::errno());\n\n        Ok(())\n    }\n\n    pub(super) unsafe fn post_run_target_child(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut <GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z> as UsesState>::State,\n        mgr: &mut EM,\n        input: &<GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z> as UsesInput>::Input,\n    ) {\n        self.observers\n            .post_exec_child_all(state, input, &ExitKind::Ok)\n            .expect(\"Failed to run post_exec on observers\");\n\n        self.hooks.post_exec_all(state, input);\n        self.leave_target(fuzzer, state, mgr, input);\n\n        libc::_exit(0);\n    }\n\n    pub(super) fn parent(&mut self, child: Pid) -> Result<ExitKind, Error> {\n        // log::trace!(\"from parent {} child is {}\", std::process::id(), child);\n        self.shmem_provider.post_fork(false)?;\n\n        let res = waitpid(child, None)?;\n        log::trace!(\"{res:#?}\");\n        match res {\n            WaitStatus::Signaled(_, signal, _) => match signal {\n                nix::sys::signal::Signal::SIGALRM | nix::sys::signal::Signal::SIGUSR2 => {\n                    Ok(ExitKind::Timeout)\n                }\n                _ => Ok(ExitKind::Crash),\n            },\n            WaitStatus::Exited(_, code) => {\n                if code > 128 && code < 160 {\n                    // Signal exit codes\n                    let signal = code - 128;\n                    if signal == Signal::SigAlarm as libc::c_int\n                        || signal == Signal::SigUser2 as libc::c_int\n                    {\n                        Ok(ExitKind::Timeout)\n                    } else {\n                        Ok(ExitKind::Crash)\n                    }\n                } else {\n                    Ok(ExitKind::Ok)\n                }\n            }\n            _ => Ok(ExitKind::Ok),\n        }\n    }\n}\n\nimpl<HT, OT, S, SP, EM, Z> GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[inline]\n    /// This function marks the boundary between the fuzzer and the target.\n    pub fn enter_target(\n        &mut self,\n        _fuzzer: &mut Z,\n        state: &mut <Self as UsesState>::State,\n        _event_mgr: &mut EM,\n        input: &<Self as UsesInput>::Input,\n    ) {\n        unsafe {\n            let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA);\n            write_volatile(\n                addr_of_mut!((*data).executor_ptr),\n                ptr::from_ref(self) as *const c_void,\n            );\n            write_volatile(\n                addr_of_mut!((*data).current_input_ptr),\n                ptr::from_ref(input) as *const c_void,\n            );\n            write_volatile(\n                addr_of_mut!((*data).state_ptr),\n                ptr::from_mut(state) as *mut c_void,\n            );\n            compiler_fence(Ordering::SeqCst);\n        }\n    }\n\n    #[inline]\n    /// This function marks the boundary between the fuzzer and the target.\n    pub fn leave_target(\n        &mut self,\n        _fuzzer: &mut Z,\n        _state: &mut <Self as UsesState>::State,\n        _event_mgr: &mut EM,\n        _input: &<Self as UsesInput>::Input,\n    ) {\n        // do nothing\n    }\n\n    /// Creates a new [`GenericInProcessForkExecutorInner`] with custom hooks\n    #[cfg(target_os = \"linux\")]\n    #[allow(clippy::too_many_arguments)]\n    pub fn with_hooks(\n        userhooks: HT,\n        observers: OT,\n        _fuzzer: &mut Z,\n        state: &mut S,\n        _event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        let default_hooks = InChildProcessHooks::new::<Self>()?;\n        let mut hooks = tuple_list!(default_hooks).merge(userhooks);\n        hooks.init_all::<Self>(state);\n\n        let milli_sec = timeout.as_millis();\n        let it_value = libc::timespec {\n            tv_sec: (milli_sec / 1000) as _,\n            tv_nsec: ((milli_sec % 1000) * 1000 * 1000) as _,\n        };\n        let it_interval = libc::timespec {\n            tv_sec: 0,\n            tv_nsec: 0,\n        };\n        let itimerspec = libc::itimerspec {\n            it_interval,\n            it_value,\n        };\n\n        Ok(Self {\n            shmem_provider,\n            observers,\n            hooks,\n            itimerspec,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Creates a new [`GenericInProcessForkExecutorInner`], non linux\n    #[cfg(not(target_os = \"linux\"))]\n    #[allow(clippy::too_many_arguments)]\n    pub fn with_hooks(\n        userhooks: HT,\n        observers: OT,\n        _fuzzer: &mut Z,\n        state: &mut S,\n        _event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        let default_hooks = InChildProcessHooks::new::<Self>()?;\n        let mut hooks = tuple_list!(default_hooks).merge(userhooks);\n        hooks.init_all::<Self>(state);\n\n        let milli_sec = timeout.as_millis();\n        let it_value = Timeval {\n            tv_sec: (milli_sec / 1000) as i64,\n            tv_usec: (milli_sec % 1000) as i64,\n        };\n        let it_interval = Timeval {\n            tv_sec: 0,\n            tv_usec: 0,\n        };\n        let itimerval = Itimerval {\n            it_interval,\n            it_value,\n        };\n\n        Ok(Self {\n            shmem_provider,\n            observers,\n            hooks,\n            itimerval,\n            phantom: PhantomData,\n        })\n    }\n}\n\nimpl<HT, OT, S, SP, EM, Z> UsesObservers for GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type Observers = OT;\n}\n\nimpl<HT, OT, S, SP, EM, Z> HasObservers for GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        &self.observers\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        &mut self.observers\n    }\n}\n\n// Path: libafl/src/events/simple.rs\n//! A very simple event manager, that just supports log outputs, but no multiprocessing\n\nuse alloc::{boxed::Box, vec::Vec};\n#[cfg(all(unix, not(miri), feature = \"std\"))]\nuse core::ptr::addr_of_mut;\nuse core::{fmt::Debug, marker::PhantomData};\n#[cfg(feature = \"std\")]\nuse core::{\n    sync::atomic::{compiler_fence, Ordering},\n    time::Duration,\n};\n\n#[cfg(all(feature = \"std\", any(windows, not(feature = \"fork\"))))]\nuse libafl_bolts::os::startable_self;\n#[cfg(all(unix, feature = \"std\", not(miri)))]\nuse libafl_bolts::os::unix_signals::setup_signal_handler;\n#[cfg(all(feature = \"std\", feature = \"fork\", unix))]\nuse libafl_bolts::os::{fork, ForkResult};\nuse libafl_bolts::ClientId;\n#[cfg(feature = \"std\")]\nuse libafl_bolts::{shmem::ShMemProvider, staterestore::StateRestorer};\n#[cfg(feature = \"std\")]\nuse serde::{de::DeserializeOwned, Serialize};\n\nuse super::{CustomBufEventResult, CustomBufHandlerFn, HasCustomBufHandlers, ProgressReporter};\n#[cfg(all(unix, feature = \"std\"))]\nuse crate::events::EVENTMGR_SIGHANDLER_STATE;\nuse crate::{\n    events::{\n        BrokerEventResult, Event, EventFirer, EventManager, EventManagerId, EventProcessor,\n        EventRestarter, HasEventManagerId,\n    },\n    inputs::UsesInput,\n    monitors::Monitor,\n    state::{HasExecutions, HasLastReportTime, HasMetadata, State, UsesState},\n    Error,\n};\n#[cfg(feature = \"std\")]\nuse crate::{\n    monitors::{ClientStats, SimplePrintingMonitor},\n    state::{HasCorpus, HasSolutions},\n};\n\n/// The llmp connection from the actual fuzzer to the process supervising it\nconst _ENV_FUZZER_SENDER: &str = \"_AFL_ENV_FUZZER_SENDER\";\nconst _ENV_FUZZER_RECEIVER: &str = \"_AFL_ENV_FUZZER_RECEIVER\";\n/// The llmp (2 way) connection from a fuzzer to the broker (broadcasting all other fuzzer messages)\nconst _ENV_FUZZER_BROKER_CLIENT_INITIAL: &str = \"_AFL_ENV_FUZZER_BROKER_CLIENT\";\n\n/// A simple, single-threaded event manager that just logs\npub struct SimpleEventManager<MT, S>\nwhere\n    S: UsesInput,\n{\n    /// The monitor\n    monitor: MT,\n    /// The events that happened since the last `handle_in_broker`\n    events: Vec<Event<S::Input>>,\n    /// The custom buf handler\n    custom_buf_handlers: Vec<Box<CustomBufHandlerFn<S>>>,\n    phantom: PhantomData<S>,\n}\n\nimpl<MT, S> Debug for SimpleEventManager<MT, S>\nwhere\n    MT: Debug,\n    S: UsesInput,\n{\n    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n        f.debug_struct(\"SimpleEventManager\")\n            //.field(\"custom_buf_handlers\", self.custom_buf_handlers)\n            .field(\"monitor\", &self.monitor)\n            .field(\"events\", &self.events)\n            .finish_non_exhaustive()\n    }\n}\n\nimpl<MT, S> UsesState for SimpleEventManager<MT, S>\nwhere\n    S: State,\n{\n    type State = S;\n}\n\nimpl<MT, S> EventFirer for SimpleEventManager<MT, S>\nwhere\n    MT: Monitor,\n    S: State,\n{\n    fn fire(\n        &mut self,\n        _state: &mut Self::State,\n        event: Event<<Self::State as UsesInput>::Input>,\n    ) -> Result<(), Error> {\n        match Self::handle_in_broker(&mut self.monitor, &event)? {\n            BrokerEventResult::Forward => self.events.push(event),\n            BrokerEventResult::Handled => (),\n        };\n        Ok(())\n    }\n}\n\nimpl<MT, S> EventRestarter for SimpleEventManager<MT, S>\nwhere\n    MT: Monitor,\n    S: State,\n{\n}\n\nimpl<E, MT, S, Z> EventProcessor<E, Z> for SimpleEventManager<MT, S>\nwhere\n    MT: Monitor,\n    S: State,\n{\n    fn process(\n        &mut self,\n        _fuzzer: &mut Z,\n        state: &mut S,\n        _executor: &mut E,\n    ) -> Result<usize, Error> {\n        let count = self.events.len();\n        while let Some(event) = self.events.pop() {\n            self.handle_in_client(state, event)?;\n        }\n        Ok(count)\n    }\n}\n\nimpl<E, MT, S, Z> EventManager<E, Z> for SimpleEventManager<MT, S>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions + HasLastReportTime + HasMetadata,\n{\n}\n\nimpl<MT, S> HasCustomBufHandlers for SimpleEventManager<MT, S>\nwhere\n    MT: Monitor, //CE: CustomEvent<I, OT>,\n    S: State,\n{\n    /// Adds a custom buffer handler that will run for each incoming `CustomBuf` event.\n    fn add_custom_buf_handler(\n        &mut self,\n        handler: Box<\n            dyn FnMut(&mut Self::State, &str, &[u8]) -> Result<CustomBufEventResult, Error>,\n        >,\n    ) {\n        self.custom_buf_handlers.push(handler);\n    }\n}\n\nimpl<MT, S> ProgressReporter for SimpleEventManager<MT, S>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions + HasMetadata + HasLastReportTime,\n{\n}\n\nimpl<MT, S> HasEventManagerId for SimpleEventManager<MT, S>\nwhere\n    MT: Monitor,\n    S: UsesInput,\n{\n    fn mgr_id(&self) -> EventManagerId {\n        EventManagerId(0)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<S> SimpleEventManager<SimplePrintingMonitor, S>\nwhere\n    S: UsesInput,\n{\n    /// Creates a [`SimpleEventManager`] that just prints to `stdout`.\n    #[must_use]\n    pub fn printing() -> Self {\n        Self::new(SimplePrintingMonitor::new())\n    }\n}\n\nimpl<MT, S> SimpleEventManager<MT, S>\nwhere\n    MT: Monitor, //TODO CE: CustomEvent,\n    S: UsesInput,\n{\n    /// Creates a new [`SimpleEventManager`].\n    pub fn new(monitor: MT) -> Self {\n        Self {\n            monitor,\n            events: vec![],\n            custom_buf_handlers: vec![],\n            phantom: PhantomData,\n        }\n    }\n\n    /// Handle arriving events in the broker\n    #[allow(clippy::unnecessary_wraps)]\n    fn handle_in_broker(\n        monitor: &mut MT,\n        event: &Event<S::Input>,\n    ) -> Result<BrokerEventResult, Error> {\n        match event {\n            Event::NewTestcase {\n                input: _,\n                client_config: _,\n                exit_kind: _,\n                corpus_size,\n                observers_buf: _,\n                time,\n                executions,\n                forward_id: _,\n            } => {\n                monitor.client_stats_insert(ClientId(0));\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_corpus_size(*corpus_size as u64);\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_executions(*executions, *time);\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::UpdateExecStats {\n                time,\n                executions,\n                phantom: _,\n            } => {\n                // TODO: The monitor buffer should be added on client add.\n                monitor.client_stats_insert(ClientId(0));\n                let client = monitor.client_stats_mut_for(ClientId(0));\n\n                client.update_executions(*executions, *time);\n\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::UpdateUserStats {\n                name,\n                value,\n                phantom: _,\n            } => {\n                monitor.client_stats_insert(ClientId(0));\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_user_stats(name.clone(), value.clone());\n                monitor.aggregate(name);\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            #[cfg(feature = \"introspection\")]\n            Event::UpdatePerfMonitor {\n                time,\n                executions,\n                introspection_monitor,\n                phantom: _,\n            } => {\n                // TODO: The monitor buffer should be added on client add.\n                monitor.client_stats_insert(ClientId(0));\n                let client = monitor.client_stats_mut_for(ClientId(0));\n                client.update_executions(*executions, *time);\n                client.update_introspection_monitor((**introspection_monitor).clone());\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::Objective {\n                objective_size,\n                executions,\n                time,\n            } => {\n                monitor.client_stats_insert(ClientId(0));\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_objective_size(*objective_size as u64);\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_executions(*executions, *time);\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::Log {\n                severity_level,\n                message,\n                phantom: _,\n            } => {\n                let (_, _) = (message, severity_level);\n                log::log!((*severity_level).into(), \"{message}\");\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::CustomBuf { .. } => Ok(BrokerEventResult::Forward),\n            //_ => Ok(BrokerEventResult::Forward),\n        }\n    }\n\n    // Handle arriving events in the client\n    #[allow(clippy::needless_pass_by_value, clippy::unused_self)]\n    fn handle_in_client(&mut self, state: &mut S, event: Event<S::Input>) -> Result<(), Error> {\n        if let Event::CustomBuf { tag, buf } = &event {\n            for handler in &mut self.custom_buf_handlers {\n                handler(state, tag, buf)?;\n            }\n            Ok(())\n        } else {\n            Err(Error::unknown(format!(\n                \"Received illegal message that message should not have arrived: {event:?}.\"\n            )))\n        }\n    }\n}\n\n/// Provides a `builder` which can be used to build a [`SimpleRestartingEventManager`], which is a combination of a\n/// `restarter` and `runner`, that can be used on systems both with and without `fork` support. The\n/// `restarter` will start a new process each time the child crashes or times out.\n#[cfg(feature = \"std\")]\n#[allow(clippy::default_trait_access)]\n#[derive(Debug)]\npub struct SimpleRestartingEventManager<MT, S, SP>\nwhere\n    S: UsesInput,\n    SP: ShMemProvider, //CE: CustomEvent<I, OT>,\n{\n    /// The actual simple event mgr\n    simple_event_mgr: SimpleEventManager<MT, S>,\n    /// [`StateRestorer`] for restarts\n    staterestorer: StateRestorer<SP>,\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> UsesState for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    S: State,\n    SP: ShMemProvider,\n{\n    type State = S;\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> EventFirer for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State,\n    SP: ShMemProvider,\n{\n    fn fire(\n        &mut self,\n        _state: &mut Self::State,\n        event: Event<<Self::State as UsesInput>::Input>,\n    ) -> Result<(), Error> {\n        self.simple_event_mgr.fire(_state, event)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> EventRestarter for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State,\n    SP: ShMemProvider,\n{\n    /// Reset the single page (we reuse it over and over from pos 0), then send the current state to the next runner.\n    fn on_restart(&mut self, state: &mut S) -> Result<(), Error> {\n        state.on_restart()?;\n\n        // First, reset the page to 0 so the next iteration can read read from the beginning of this page\n        self.staterestorer.reset();\n        self.staterestorer.save(&(\n            state,\n            self.simple_event_mgr.monitor.start_time(),\n            self.simple_event_mgr.monitor.client_stats(),\n        ))\n    }\n\n    fn send_exiting(&mut self) -> Result<(), Error> {\n        self.staterestorer.send_exiting();\n        Ok(())\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<E, MT, S, SP, Z> EventProcessor<E, Z> for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions,\n    SP: ShMemProvider,\n{\n    fn process(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        executor: &mut E,\n    ) -> Result<usize, Error> {\n        self.simple_event_mgr.process(fuzzer, state, executor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<E, MT, S, SP, Z> EventManager<E, Z> for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions + HasMetadata + HasLastReportTime + Serialize,\n    SP: ShMemProvider,\n{\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> HasCustomBufHandlers for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State,\n    SP: ShMemProvider,\n{\n    fn add_custom_buf_handler(\n        &mut self,\n        handler: Box<dyn FnMut(&mut S, &str, &[u8]) -> Result<CustomBufEventResult, Error>>,\n    ) {\n        self.simple_event_mgr.add_custom_buf_handler(handler);\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> ProgressReporter for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions + HasMetadata + HasLastReportTime,\n    SP: ShMemProvider,\n{\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> HasEventManagerId for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: UsesInput,\n    SP: ShMemProvider,\n{\n    fn mgr_id(&self) -> EventManagerId {\n        self.simple_event_mgr.mgr_id()\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[allow(clippy::type_complexity, clippy::too_many_lines)]\nimpl<MT, S, SP> SimpleRestartingEventManager<MT, S, SP>\nwhere\n    S: UsesInput,\n    SP: ShMemProvider,\n    MT: Monitor, //TODO CE: CustomEvent,\n{\n    /// Creates a new [`SimpleEventManager`].\n    fn launched(monitor: MT, staterestorer: StateRestorer<SP>) -> Self {\n        Self {\n            staterestorer,\n            simple_event_mgr: SimpleEventManager::new(monitor),\n        }\n    }\n\n    /// Internal function, returns true when shuttdown is requested by a `SIGINT` signal\n    #[inline]\n    #[allow(clippy::unused_self)]\n    fn is_shutting_down() -> bool {\n        #[cfg(unix)]\n        unsafe {\n            core::ptr::read_volatile(core::ptr::addr_of!(EVENTMGR_SIGHANDLER_STATE.shutting_down))\n        }\n\n        #[cfg(windows)]\n        false\n    }\n\n    /// Launch the simple restarting manager.\n    /// This [`EventManager`] is simple and single threaded,\n    /// but can still used shared maps to recover from crashes and timeouts.\n    #[allow(clippy::similar_names)]\n    pub fn launch(mut monitor: MT, shmem_provider: &mut SP) -> Result<(Option<S>, Self), Error>\n    where\n        S: DeserializeOwned + Serialize + HasCorpus + HasSolutions,\n        MT: Debug,\n    {\n        // We start ourself as child process to actually fuzz\n        let mut staterestorer = if std::env::var(_ENV_FUZZER_SENDER).is_err() {\n            // First, create a place to store state in, for restarts.\n            #[cfg(unix)]\n            let staterestorer: StateRestorer<SP> =\n                StateRestorer::new(shmem_provider.new_shmem(256 * 1024 * 1024)?);\n            #[cfg(not(unix))]\n            let staterestorer: StateRestorer<SP> =\n                StateRestorer::new(shmem_provider.new_shmem(256 * 1024 * 1024)?);\n\n            //let staterestorer = { LlmpSender::new(shmem_provider.clone(), 0, false)? };\n            staterestorer.write_to_env(_ENV_FUZZER_SENDER)?;\n\n            // We setup signal handlers to clean up shmem segments used by state restorer\n            #[cfg(all(unix, not(miri)))]\n            if let Err(_e) =\n                unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) }\n            {\n                // We can live without a proper ctrl+c signal handler. Print and ignore.\n                log::error!(\"Failed to setup signal handlers: {_e}\");\n            }\n\n            let mut ctr: u64 = 0;\n            // Client->parent loop\n            loop {\n                log::info!(\"Spawning next client (id {ctr})\");\n\n                // On Unix, we fork\n                #[cfg(all(unix, feature = \"fork\"))]\n                let child_status = {\n                    shmem_provider.pre_fork()?;\n                    match unsafe { fork() }? {\n                        ForkResult::Parent(handle) => {\n                            unsafe {\n                                // The parent will later exit through is_shutting down below\n                                // if the process exits gracefully, it cleans up the shmem.\n                                EVENTMGR_SIGHANDLER_STATE.set_exit_from_main();\n                            }\n                            shmem_provider.post_fork(false)?;\n                            handle.status()\n                        }\n                        ForkResult::Child => {\n                            shmem_provider.post_fork(true)?;\n                            break staterestorer;\n                        }\n                    }\n                };\n\n                // Same, as fork version, mark this main thread as the shmem allocator\n                // then it will not call exit or exitprocess in the sigint handler\n                // so that it exits after cleaning up the shmem segments\n                #[cfg(all(unix, not(feature = \"fork\")))]\n                unsafe {\n                    EVENTMGR_SIGHANDLER_STATE.set_exit_from_main();\n                }\n\n                // On Windows (or in any case without forks), we spawn ourself again\n                #[cfg(any(windows, not(feature = \"fork\")))]\n                let child_status = startable_self()?.status()?;\n                #[cfg(all(unix, not(feature = \"fork\")))]\n                let child_status = child_status.code().unwrap_or_default();\n\n                compiler_fence(Ordering::SeqCst);\n\n                if staterestorer.wants_to_exit() || Self::is_shutting_down() {\n                    return Err(Error::shutting_down());\n                }\n\n                #[allow(clippy::manual_assert)]\n                if !staterestorer.has_content() {\n                    #[cfg(unix)]\n                    if child_status == 137 {\n                        // Out of Memory, see https://tldp.org/LDP/abs/html/exitcodes.html\n                        // and https://github.com/AFLplusplus/LibAFL/issues/32 for discussion.\n                        panic!(\"Fuzzer-respawner: The fuzzed target crashed with an out of memory error! Fix your harness, or switch to another executor (for example, a forkserver).\");\n                    }\n\n                    // Storing state in the last round did not work\n                    panic!(\"Fuzzer-respawner: Storing state in crashed fuzzer instance did not work, no point to spawn the next client! This can happen if the child calls `exit()`, in that case make sure it uses `abort()`, if it got killed unrecoverable (OOM), or if there is a bug in the fuzzer itself. (Child exited with: {child_status})\");\n                }\n\n                ctr = ctr.wrapping_add(1);\n            }\n        } else {\n            // We are the newly started fuzzing instance (i.e. on Windows), first, connect to our own restore map.\n            // We get here *only on Windows*, if we were started by a restarting fuzzer.\n            // A staterestorer and a receiver for single communication\n            StateRestorer::from_env(shmem_provider, _ENV_FUZZER_SENDER)?\n        };\n\n        // If we're restarting, deserialize the old state.\n        let (state, mgr) = match staterestorer.restore::<(S, Duration, Vec<ClientStats>)>()? {\n            None => {\n                log::info!(\"First run. Let's set it all up\");\n                // Mgr to send and receive msgs from/to all other fuzzer instances\n                (\n                    None,\n                    SimpleRestartingEventManager::launched(monitor, staterestorer),\n                )\n            }\n            // Restoring from a previous run, deserialize state and corpus.\n            Some((state, start_time, clients_stats)) => {\n                log::info!(\"Subsequent run. Loaded previous state.\");\n                // We reset the staterestorer, the next staterestorer and receiver (after crash) will reuse the page from the initial message.\n                staterestorer.reset();\n\n                // reload the state of the monitor to display the correct stats after restarts\n                monitor.set_start_time(start_time);\n                *monitor.client_stats_mut() = clients_stats;\n\n                (\n                    Some(state),\n                    SimpleRestartingEventManager::launched(monitor, staterestorer),\n                )\n            }\n        };\n\n        /* TODO: Not sure if this is needed\n        // We commit an empty NO_RESTART message to this buf, against infinite loops,\n        // in case something crashes in the fuzzer.\n        staterestorer.send_buf(_LLMP_TAG_NO_RESTART, []);\n        */\n\n        Ok((state, mgr))\n    }\n}\n\n// Path: libafl/src/executors/inprocess_fork/stateful.rs\n//! The `StatefulGenericInProcessForkExecutor` to do forking before executing the harness in-processly. Harness can access internal state.\nuse core::{\n    fmt::{self, Debug, Formatter},\n    marker::PhantomData,\n    time::Duration,\n};\n\nuse libafl_bolts::{shmem::ShMemProvider, tuples::tuple_list};\nuse nix::unistd::{fork, ForkResult};\n\nuse super::super::hooks::ExecutorHooksTuple;\nuse crate::{\n    events::{EventFirer, EventRestarter},\n    executors::{\n        inprocess_fork::GenericInProcessForkExecutorInner, Executor, ExitKind, HasObservers,\n    },\n    feedbacks::Feedback,\n    fuzzer::HasObjective,\n    inputs::UsesInput,\n    observers::{ObserversTuple, UsesObservers},\n    state::{HasExecutions, HasSolutions, State, UsesState},\n    Error,\n};\n\n/// The `StatefulInProcessForkExecutor` with no user hooks\npub type StatefulInProcessForkExecutor<'a, H, OT, S, SP, ES, EM, Z> =\n    StatefulGenericInProcessForkExecutor<'a, H, (), OT, S, SP, ES, EM, Z>;\n\nimpl<'a, H, OT, S, SP, ES, EM, Z, OF> StatefulInProcessForkExecutor<'a, H, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: State + HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    #[allow(clippy::too_many_arguments)]\n    /// The constructor for `InProcessForkExecutor`\n    pub fn new(\n        harness_fn: &'a mut H,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        Self::with_hooks(\n            tuple_list!(),\n            harness_fn,\n            exposed_executor_state,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            timeout,\n            shmem_provider,\n        )\n    }\n}\n\n/// [`StatefulGenericInProcessForkExecutor`] is an executor that forks the current process before each execution. Harness can access some internal state.\npub struct StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    harness_fn: &'a mut H,\n    exposed_executor_state: ES,\n    inner: GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>,\n    phantom: PhantomData<ES>,\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> Debug\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S> + Debug,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[cfg(target_os = \"linux\")]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutionInner\", &self.inner)\n            .finish()\n    }\n\n    #[cfg(not(target_os = \"linux\"))]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        #[cfg(not(target_os = \"linux\"))]\n        return f\n            .debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutionInner\", &self.inner)\n            .finish();\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> UsesState\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type State = S;\n}\n\nimpl<'a, EM, H, HT, OT, S, SP, Z, ES, OF> Executor<EM, Z>\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: State + HasExecutions,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    Z: HasObjective<Objective = OF, State = S>,\n    OF: Feedback<S>,\n{\n    #[allow(unreachable_code)]\n    #[inline]\n    fn run_target(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        mgr: &mut EM,\n        input: &Self::Input,\n    ) -> Result<ExitKind, Error> {\n        *state.executions_mut() += 1;\n\n        unsafe {\n            self.inner.shmem_provider.pre_fork()?;\n            match fork() {\n                Ok(ForkResult::Child) => {\n                    // Child\n                    self.inner.pre_run_target_child(fuzzer, state, mgr, input)?;\n                    (self.harness_fn)(input, &mut self.exposed_executor_state);\n                    self.inner.post_run_target_child(fuzzer, state, mgr, input);\n                    Ok(ExitKind::Ok)\n                }\n                Ok(ForkResult::Parent { child }) => {\n                    // Parent\n                    self.inner.parent(child)\n                }\n                Err(e) => Err(Error::from(e)),\n            }\n        }\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z, OF>\n    StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    Z: UsesState<State = S>,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: State + HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    /// Creates a new [`StatefulGenericInProcessForkExecutor`] with custom hooks\n    #[allow(clippy::too_many_arguments)]\n    pub fn with_hooks(\n        userhooks: HT,\n        harness_fn: &'a mut H,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        Ok(Self {\n            harness_fn,\n            exposed_executor_state,\n            inner: GenericInProcessForkExecutorInner::with_hooks(\n                userhooks,\n                observers,\n                fuzzer,\n                state,\n                event_mgr,\n                timeout,\n                shmem_provider,\n            )?,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Retrieve the harness function.\n    #[inline]\n    pub fn harness(&self) -> &H {\n        self.harness_fn\n    }\n\n    /// Retrieve the harness function for a mutable reference.\n    #[inline]\n    pub fn harness_mut(&mut self) -> &mut H {\n        self.harness_fn\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> UsesObservers\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type Observers = OT;\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> HasObservers\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        self.inner.observers()\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        self.inner.observers_mut()\n    }\n}\n\n// Path: libafl/src/executors/inprocess_fork/mod.rs\n//! The `GenericInProcessForkExecutor` to do forking before executing the harness in-processly\nuse core::{\n    fmt::{self, Debug, Formatter},\n    time::Duration,\n};\n\nuse libafl_bolts::{\n    os::unix_signals::{ucontext_t, Signal},\n    shmem::ShMemProvider,\n    tuples::tuple_list,\n};\nuse libc::siginfo_t;\nuse nix::unistd::{fork, ForkResult};\n\nuse super::hooks::ExecutorHooksTuple;\nuse crate::{\n    events::{EventFirer, EventRestarter},\n    executors::{\n        hooks::inprocess_fork::InProcessForkExecutorGlobalData,\n        inprocess_fork::inner::GenericInProcessForkExecutorInner, Executor, ExitKind, HasObservers,\n    },\n    feedbacks::Feedback,\n    fuzzer::HasObjective,\n    inputs::UsesInput,\n    observers::{ObserversTuple, UsesObservers},\n    state::{HasExecutions, HasSolutions, State, UsesState},\n    Error,\n};\n\n/// The signature of the crash handler function\npub(crate) type ForkHandlerFuncPtr = unsafe fn(\n    Signal,\n    &mut siginfo_t,\n    Option<&mut ucontext_t>,\n    data: *mut InProcessForkExecutorGlobalData,\n);\n\n/// The inner structure of `InProcessForkExecutor`.\npub mod inner;\n/// A version of `InProcessForkExecutor` with a state accessible from the harness.\npub mod stateful;\n\n/// The `InProcessForkExecutor` with no user hooks\npub type InProcessForkExecutor<'a, H, OT, S, SP, EM, Z> =\n    GenericInProcessForkExecutor<'a, H, (), OT, S, SP, EM, Z>;\n\nimpl<'a, H, OT, S, SP, EM, Z, OF> InProcessForkExecutor<'a, H, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    #[allow(clippy::too_many_arguments)]\n    /// The constructor for `InProcessForkExecutor`\n    pub fn new(\n        harness_fn: &'a mut H,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        Self::with_hooks(\n            tuple_list!(),\n            harness_fn,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            timeout,\n            shmem_provider,\n        )\n    }\n}\n\n/// [`GenericInProcessForkExecutor`] is an executor that forks the current process before each execution.\npub struct GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    harness_fn: &'a mut H,\n    inner: GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>,\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> Debug\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S> + Debug,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[cfg(target_os = \"linux\")]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutorInner\", &self.inner)\n            .finish()\n    }\n\n    #[cfg(not(target_os = \"linux\"))]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        #[cfg(not(target_os = \"linux\"))]\n        return f\n            .debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutorInner\", &self.inner)\n            .finish();\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> UsesState\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type State = S;\n}\n\nimpl<'a, EM, H, HT, OT, S, SP, Z> Executor<EM, Z>\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: State + HasExecutions,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[allow(unreachable_code)]\n    #[inline]\n    fn run_target(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        mgr: &mut EM,\n        input: &Self::Input,\n    ) -> Result<ExitKind, Error> {\n        *state.executions_mut() += 1;\n\n        unsafe {\n            self.inner.shmem_provider.pre_fork()?;\n            match fork() {\n                Ok(ForkResult::Child) => {\n                    // Child\n                    self.inner.pre_run_target_child(fuzzer, state, mgr, input)?;\n                    (self.harness_fn)(input);\n                    self.inner.post_run_target_child(fuzzer, state, mgr, input);\n                    Ok(ExitKind::Ok)\n                }\n                Ok(ForkResult::Parent { child }) => {\n                    // Parent\n                    self.inner.parent(child)\n                }\n                Err(e) => Err(Error::from(e)),\n            }\n        }\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z, OF> GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: State + HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    /// Creates a new [`GenericInProcessForkExecutor`] with custom hooks\n    #[allow(clippy::too_many_arguments)]\n    pub fn with_hooks(\n        userhooks: HT,\n        harness_fn: &'a mut H,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error>\nwhere {\n        Ok(Self {\n            harness_fn,\n            inner: GenericInProcessForkExecutorInner::with_hooks(\n                userhooks,\n                observers,\n                fuzzer,\n                state,\n                event_mgr,\n                timeout,\n                shmem_provider,\n            )?,\n        })\n    }\n\n    /// Retrieve the harness function.\n    #[inline]\n    pub fn harness(&self) -> &H {\n        self.harness_fn\n    }\n\n    /// Retrieve the harness function for a mutable reference.\n    #[inline]\n    pub fn harness_mut(&mut self) -> &mut H {\n        self.harness_fn\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> UsesObservers\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type Observers = OT;\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> HasObservers\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        self.inner.observers()\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        self.inner.observers_mut()\n    }\n}\n\n/// signal hooks and `panic_hooks` for the child process\n\npub mod child_signal_handlers {\n    use alloc::boxed::Box;\n    use core::ptr::addr_of_mut;\n    use std::panic;\n\n    use libafl_bolts::os::unix_signals::{ucontext_t, Signal};\n    use libc::siginfo_t;\n\n    use crate::{\n        executors::{\n            hooks::inprocess_fork::{InProcessForkExecutorGlobalData, FORK_EXECUTOR_GLOBAL_DATA},\n            ExitKind, HasObservers,\n        },\n        inputs::UsesInput,\n        observers::ObserversTuple,\n    };\n\n    /// invokes the `post_exec_child` hook on all observer in case the child process panics\n    pub fn setup_child_panic_hook<E>()\n    where\n        E: HasObservers,\n    {\n        let old_hook = panic::take_hook();\n        panic::set_hook(Box::new(move |panic_info| unsafe {\n            old_hook(panic_info);\n            let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA);\n            if !data.is_null() && (*data).is_valid() {\n                let executor = (*data).executor_mut::<E>();\n                let observers = executor.observers_mut();\n                let state = (*data).state_mut::<E::State>();\n                // Invalidate data to not execute again the observer hooks in the crash handler\n                let input = (*data).take_current_input::<<E::State as UsesInput>::Input>();\n                observers\n                    .post_exec_child_all(state, input, &ExitKind::Crash)\n                    .expect(\"Failed to run post_exec on observers\");\n\n                // std::process::abort();\n                libc::_exit(128 + 6); // ABORT exit code\n            }\n        }));\n    }\n\n    /// invokes the `post_exec` hook on all observer in case the child process crashes\n    ///\n    /// # Safety\n    /// The function should only be called from a child crash handler.\n    /// It will dereference the `data` pointer and assume it's valid.\n    #[cfg(unix)]\n    #[allow(clippy::needless_pass_by_value)]\n    pub(crate) unsafe fn child_crash_handler<E>(\n        _signal: Signal,\n        _info: &mut siginfo_t,\n        _context: Option<&mut ucontext_t>,\n        data: &mut InProcessForkExecutorGlobalData,\n    ) where\n        E: HasObservers,\n    {\n        if data.is_valid() {\n            let executor = data.executor_mut::<E>();\n            let observers = executor.observers_mut();\n            let state = data.state_mut::<E::State>();\n            let input = data.take_current_input::<<E::State as UsesInput>::Input>();\n            observers\n                .post_exec_child_all(state, input, &ExitKind::Crash)\n                .expect(\"Failed to run post_exec on observers\");\n        }\n\n        libc::_exit(128 + (_signal as i32));\n    }\n\n    #[cfg(unix)]\n    #[allow(clippy::needless_pass_by_value)]\n    pub(crate) unsafe fn child_timeout_handler<E>(\n        _signal: Signal,\n        _info: &mut siginfo_t,\n        _context: Option<&mut ucontext_t>,\n        data: &mut InProcessForkExecutorGlobalData,\n    ) where\n        E: HasObservers,\n    {\n        if data.is_valid() {\n            let executor = data.executor_mut::<E>();\n            let observers = executor.observers_mut();\n            let state = data.state_mut::<E::State>();\n            let input = data.take_current_input::<<E::State as UsesInput>::Input>();\n            observers\n                .post_exec_child_all(state, input, &ExitKind::Timeout)\n                .expect(\"Failed to run post_exec on observers\");\n        }\n        libc::_exit(128 + (_signal as i32));\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use libafl_bolts::tuples::tuple_list;\n\n    use crate::{\n        executors::{inprocess_fork::GenericInProcessForkExecutorInner, Executor, ExitKind},\n        inputs::NopInput,\n    };\n\n    #[test]\n    #[cfg_attr(miri, ignore)]\n    #[cfg(all(feature = \"std\", feature = \"fork\", unix))]\n    fn test_inprocessfork_exec() {\n        use core::marker::PhantomData;\n\n        use libafl_bolts::shmem::{ShMemProvider, StdShMemProvider};\n...\n", "needle_token_start": 581, "needle_token_end": 1037, "code_context_ntokens": 16275, "output": ["```rust\npub(super) unsafe fn pre_run_target_child(\n    &mut self,\n    fuzzer: &mut Z,\n    state: &mut <GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z> as UsesState>::State,\n    mgr: &mut EM,\n    input: &<GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z> as UsesInput>::Input,\n) -> Result<(), Error> {\n    self.shmem_provider.post_fork(true)?;\n\n    self.enter_target(fuzzer, state, mgr, input);\n    self.hooks.pre_exec_all(state, input);\n\n    self.observers\n        .pre_exec_child_all(state, input)\n        .expect(\"Failed to run post_exec on observers\");\n\n    #[cfg(target_os = \"linux\")]\n    {\n        let mut timerid: libc::timer_t = null_mut();\n        // creates a new per-process interval timer\n        // we can't do this from the parent, timerid is unique to each process.\n        libc::timer_create(libc::CLOCK_MONOTONIC, null_mut(), addr_of_mut!(timerid));\n\n        // log::info!(\"Set timer! {:#?} {timerid:#?}\", self.itimerspec);\n        let _: i32 = libc::timer_settime(timerid, 0, addr_of_mut!(self.itimerspec), null_mut());\n    }\n    #[cfg(not(target_os = \"linux\"))]\n    {\n        setitimer(ITIMER_REAL, &mut self.itimerval, null_mut());\n    }\n    // log::trace!(\"{v:#?} {}\", nix::errno::errno());\n\n    Ok(())\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "launched", "language": "rust", "path": "libafl/src/events/simple.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function initializes a new instance of a manager that handles events and state restoration in a system that monitors and manages stateful operations.\n2. **Input**: It takes two parameters: a monitoring object that tracks system activities and a state restorer object that manages the saving and restoring of system state.\n3. **Output**: Returns a new instance of the event manager equipped with the provided monitoring and state restoration capabilities.\n4. **Procedure**: The function creates a new event manager by first initializing an internal simple event manager with the provided monitor. It then sets up the state restorer to manage the system state across different operational phases.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        monitor: &mut MT,\n        event: &Event<S::Input>,\n    ) -> Result<BrokerEventResult, Error> {\n        match event {\n            Event::NewTestcase {\n                input: _,\n                client_config: _,\n                exit_kind: _,\n                corpus_size,\n                observers_buf: _,\n                time,\n                executions,\n                forward_id: _,\n            } => {\n                monitor.client_stats_insert(ClientId(0));\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_corpus_size(*corpus_size as u64);\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_executions(*executions, *time);\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::UpdateExecStats {\n                time,\n                executions,\n                phantom: _,\n            } => {\n                // TODO: The monitor buffer should be added on client add.\n                monitor.client_stats_insert(ClientId(0));\n                let client = monitor.client_stats_mut_for(ClientId(0));\n\n                client.update_executions(*executions, *time);\n\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::UpdateUserStats {\n                name,\n                value,\n                phantom: _,\n            } => {\n                monitor.client_stats_insert(ClientId(0));\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_user_stats(name.clone(), value.clone());\n                monitor.aggregate(name);\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            #[cfg(feature = \"introspection\")]\n            Event::UpdatePerfMonitor {\n                time,\n                executions,\n                introspection_monitor,\n                phantom: _,\n            } => {\n                // TODO: The monitor buffer should be added on client add.\n                monitor.client_stats_insert(ClientId(0));\n                let client = monitor.client_stats_mut_for(ClientId(0));\n                client.update_executions(*executions, *time);\n                client.update_introspection_monitor((**introspection_monitor).clone());\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::Objective {\n                objective_size,\n                executions,\n                time,\n            } => {\n                monitor.client_stats_insert(ClientId(0));\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_objective_size(*objective_size as u64);\n                monitor\n                    .client_stats_mut_for(ClientId(0))\n                    .update_executions(*executions, *time);\n                monitor.display(event.name(), ClientId(0));\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::Log {\n                severity_level,\n                message,\n                phantom: _,\n            } => {\n                let (_, _) = (message, severity_level);\n                log::log!((*severity_level).into(), \"{message}\");\n                Ok(BrokerEventResult::Handled)\n            }\n            Event::CustomBuf { .. } => Ok(BrokerEventResult::Forward),\n            //_ => Ok(BrokerEventResult::Forward),\n        }\n    }\n\n    // Handle arriving events in the client\n    #[allow(clippy::needless_pass_by_value, clippy::unused_self)]\n    fn handle_in_client(&mut self, state: &mut S, event: Event<S::Input>) -> Result<(), Error> {\n        if let Event::CustomBuf { tag, buf } = &event {\n            for handler in &mut self.custom_buf_handlers {\n                handler(state, tag, buf)?;\n            }\n            Ok(())\n        } else {\n            Err(Error::unknown(format!(\n                \"Received illegal message that message should not have arrived: {event:?}.\"\n            )))\n        }\n    }\n}\n\n/// Provides a `builder` which can be used to build a [`SimpleRestartingEventManager`], which is a combination of a\n/// `restarter` and `runner`, that can be used on systems both with and without `fork` support. The\n/// `restarter` will start a new process each time the child crashes or times out.\n#[cfg(feature = \"std\")]\n#[allow(clippy::default_trait_access)]\n#[derive(Debug)]\npub struct SimpleRestartingEventManager<MT, S, SP>\nwhere\n    S: UsesInput,\n    SP: ShMemProvider, //CE: CustomEvent<I, OT>,\n{\n    /// The actual simple event mgr\n    simple_event_mgr: SimpleEventManager<MT, S>,\n    /// [`StateRestorer`] for restarts\n    staterestorer: StateRestorer<SP>,\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> UsesState for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    S: State,\n    SP: ShMemProvider,\n{\n    type State = S;\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> EventFirer for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State,\n    SP: ShMemProvider,\n{\n    fn fire(\n        &mut self,\n        _state: &mut Self::State,\n        event: Event<<Self::State as UsesInput>::Input>,\n    ) -> Result<(), Error> {\n        self.simple_event_mgr.fire(_state, event)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> EventRestarter for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State,\n    SP: ShMemProvider,\n{\n    /// Reset the single page (we reuse it over and over from pos 0), then send the current state to the next runner.\n    fn on_restart(&mut self, state: &mut S) -> Result<(), Error> {\n        state.on_restart()?;\n\n        // First, reset the page to 0 so the next iteration can read read from the beginning of this page\n        self.staterestorer.reset();\n        self.staterestorer.save(&(\n            state,\n            self.simple_event_mgr.monitor.start_time(),\n            self.simple_event_mgr.monitor.client_stats(),\n        ))\n    }\n\n    fn send_exiting(&mut self) -> Result<(), Error> {\n        self.staterestorer.send_exiting();\n        Ok(())\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<E, MT, S, SP, Z> EventProcessor<E, Z> for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions,\n    SP: ShMemProvider,\n{\n    fn process(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        executor: &mut E,\n    ) -> Result<usize, Error> {\n        self.simple_event_mgr.process(fuzzer, state, executor)\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<E, MT, S, SP, Z> EventManager<E, Z> for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions + HasMetadata + HasLastReportTime + Serialize,\n    SP: ShMemProvider,\n{\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> HasCustomBufHandlers for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State,\n    SP: ShMemProvider,\n{\n    fn add_custom_buf_handler(\n        &mut self,\n        handler: Box<dyn FnMut(&mut S, &str, &[u8]) -> Result<CustomBufEventResult, Error>>,\n    ) {\n        self.simple_event_mgr.add_custom_buf_handler(handler);\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> ProgressReporter for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: State + HasExecutions + HasMetadata + HasLastReportTime,\n    SP: ShMemProvider,\n{\n}\n\n#[cfg(feature = \"std\")]\nimpl<MT, S, SP> HasEventManagerId for SimpleRestartingEventManager<MT, S, SP>\nwhere\n    MT: Monitor,\n    S: UsesInput,\n    SP: ShMemProvider,\n{\n    fn mgr_id(&self) -> EventManagerId {\n        self.simple_event_mgr.mgr_id()\n    }\n}\n\n#[cfg(feature = \"std\")]\n#[allow(clippy::type_complexity, clippy::too_many_lines)]\nimpl<MT, S, SP> SimpleRestartingEventManager<MT, S, SP>\nwhere\n    S: UsesInput,\n    SP: ShMemProvider,\n    MT: Monitor, //TODO CE: CustomEvent,\n{\n    /// Creates a new [`SimpleEventManager`].\n    \nfn launched(monitor: MT, staterestorer: StateRestorer<SP>) -> Self {\n        Self {\n            staterestorer,\n            simple_event_mgr: SimpleEventManager::new(monitor),\n        }\n    }\n\n    /// Internal function, returns true when shuttdown is requested by a `SIGINT` signal\n    #[inline]\n    #[allow(clippy::unused_self)]\n    fn is_shutting_down() -> bool {\n        #[cfg(unix)]\n        unsafe {\n            core::ptr::read_volatile(core::ptr::addr_of!(EVENTMGR_SIGHANDLER_STATE.shutting_down))\n        }\n\n        #[cfg(windows)]\n        false\n    }\n\n    /// Launch the simple restarting manager.\n    /// This [`EventManager`] is simple and single threaded,\n    /// but can still used shared maps to recover from crashes and timeouts.\n    #[allow(clippy::similar_names)]\n    pub fn launch(mut monitor: MT, shmem_provider: &mut SP) -> Result<(Option<S>, Self), Error>\n    where\n        S: DeserializeOwned + Serialize + HasCorpus + HasSolutions,\n        MT: Debug,\n    {\n        // We start ourself as child process to actually fuzz\n        let mut staterestorer = if std::env::var(_ENV_FUZZER_SENDER).is_err() {\n            // First, create a place to store state in, for restarts.\n            #[cfg(unix)]\n            let staterestorer: StateRestorer<SP> =\n                StateRestorer::new(shmem_provider.new_shmem(256 * 1024 * 1024)?);\n            #[cfg(not(unix))]\n            let staterestorer: StateRestorer<SP> =\n                StateRestorer::new(shmem_provider.new_shmem(256 * 1024 * 1024)?);\n\n            //let staterestorer = { LlmpSender::new(shmem_provider.clone(), 0, false)? };\n            staterestorer.write_to_env(_ENV_FUZZER_SENDER)?;\n\n            // We setup signal handlers to clean up shmem segments used by state restorer\n            #[cfg(all(unix, not(miri)))]\n            if let Err(_e) =\n                unsafe { setup_signal_handler(addr_of_mut!(EVENTMGR_SIGHANDLER_STATE)) }\n            {\n                // We can live without a proper ctrl+c signal handler. Print and ignore.\n                log::error!(\"Failed to setup signal handlers: {_e}\");\n            }\n\n            let mut ctr: u64 = 0;\n            // Client->parent loop\n            loop {\n                log::info!(\"Spawning next client (id {ctr})\");\n\n                // On Unix, we fork\n                #[cfg(all(unix, feature = \"fork\"))]\n                let child_status = {\n                    shmem_provider.pre_fork()?;\n                    match unsafe { fork() }? {\n                        ForkResult::Parent(handle) => {\n                            unsafe {\n                                // The parent will later exit through is_shutting down below\n                                // if the process exits gracefully, it cleans up the shmem.\n                                EVENTMGR_SIGHANDLER_STATE.set_exit_from_main();\n                            }\n                            shmem_provider.post_fork(false)?;\n                            handle.status()\n                        }\n                        ForkResult::Child => {\n                            shmem_provider.post_fork(true)?;\n                            break staterestorer;\n                        }\n                    }\n                };\n\n                // Same, as fork version, mark this main thread as the shmem allocator\n                // then it will not call exit or exitprocess in the sigint handler\n                // so that it exits after cleaning up the shmem segments\n                #[cfg(all(unix, not(feature = \"fork\")))]\n                unsafe {\n                    EVENTMGR_SIGHANDLER_STATE.set_exit_from_main();\n                }\n\n                // On Windows (or in any case without forks), we spawn ourself again\n                #[cfg(any(windows, not(feature = \"fork\")))]\n                let child_status = startable_self()?.status()?;\n                #[cfg(all(unix, not(feature = \"fork\")))]\n                let child_status = child_status.code().unwrap_or_default();\n\n                compiler_fence(Ordering::SeqCst);\n\n                if staterestorer.wants_to_exit() || Self::is_shutting_down() {\n                    return Err(Error::shutting_down());\n                }\n\n                #[allow(clippy::manual_assert)]\n                if !staterestorer.has_content() {\n                    #[cfg(unix)]\n                    if child_status == 137 {\n                        // Out of Memory, see https://tldp.org/LDP/abs/html/exitcodes.html\n                        // and https://github.com/AFLplusplus/LibAFL/issues/32 for discussion.\n                        panic!(\"Fuzzer-respawner: The fuzzed target crashed with an out of memory error! Fix your harness, or switch to another executor (for example, a forkserver).\");\n                    }\n\n                    // Storing state in the last round did not work\n                    panic!(\"Fuzzer-respawner: Storing state in crashed fuzzer instance did not work, no point to spawn the next client! This can happen if the child calls `exit()`, in that case make sure it uses `abort()`, if it got killed unrecoverable (OOM), or if there is a bug in the fuzzer itself. (Child exited with: {child_status})\");\n                }\n\n                ctr = ctr.wrapping_add(1);\n            }\n        } else {\n            // We are the newly started fuzzing instance (i.e. on Windows), first, connect to our own restore map.\n            // We get here *only on Windows*, if we were started by a restarting fuzzer.\n            // A staterestorer and a receiver for single communication\n            StateRestorer::from_env(shmem_provider, _ENV_FUZZER_SENDER)?\n        };\n\n        // If we're restarting, deserialize the old state.\n        let (state, mgr) = match staterestorer.restore::<(S, Duration, Vec<ClientStats>)>()? {\n            None => {\n                log::info!(\"First run. Let's set it all up\");\n                // Mgr to send and receive msgs from/to all other fuzzer instances\n                (\n                    None,\n                    SimpleRestartingEventManager::launched(monitor, staterestorer),\n                )\n            }\n            // Restoring from a previous run, deserialize state and corpus.\n            Some((state, start_time, clients_stats)) => {\n                log::info!(\"Subsequent run. Loaded previous state.\");\n                // We reset the staterestorer, the next staterestorer and receiver (after crash) will reuse the page from the initial message.\n                staterestorer.reset();\n\n                // reload the state of the monitor to display the correct stats after restarts\n                monitor.set_start_time(start_time);\n                *monitor.client_stats_mut() = clients_stats;\n\n                (\n                    Some(state),\n                    SimpleRestartingEventManager::launched(monitor, staterestorer),\n                )\n            }\n        };\n\n        /* TODO: Not sure if this is needed\n        // We commit an empty NO_RESTART message to this buf, against infinite loops,\n        // in case something crashes in the fuzzer.\n        staterestorer.send_buf(_LLMP_TAG_NO_RESTART, []);\n        */\n\n        Ok((state, mgr))\n    }\n}\n\n// Path: libafl/src/executors/inprocess_fork/stateful.rs\n//! The `StatefulGenericInProcessForkExecutor` to do forking before executing the harness in-processly. Harness can access internal state.\nuse core::{\n    fmt::{self, Debug, Formatter},\n    marker::PhantomData,\n    time::Duration,\n};\n\nuse libafl_bolts::{shmem::ShMemProvider, tuples::tuple_list};\nuse nix::unistd::{fork, ForkResult};\n\nuse super::super::hooks::ExecutorHooksTuple;\nuse crate::{\n    events::{EventFirer, EventRestarter},\n    executors::{\n        inprocess_fork::GenericInProcessForkExecutorInner, Executor, ExitKind, HasObservers,\n    },\n    feedbacks::Feedback,\n    fuzzer::HasObjective,\n    inputs::UsesInput,\n    observers::{ObserversTuple, UsesObservers},\n    state::{HasExecutions, HasSolutions, State, UsesState},\n    Error,\n};\n\n/// The `StatefulInProcessForkExecutor` with no user hooks\npub type StatefulInProcessForkExecutor<'a, H, OT, S, SP, ES, EM, Z> =\n    StatefulGenericInProcessForkExecutor<'a, H, (), OT, S, SP, ES, EM, Z>;\n\nimpl<'a, H, OT, S, SP, ES, EM, Z, OF> StatefulInProcessForkExecutor<'a, H, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: State + HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    #[allow(clippy::too_many_arguments)]\n    /// The constructor for `InProcessForkExecutor`\n    pub fn new(\n        harness_fn: &'a mut H,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        Self::with_hooks(\n            tuple_list!(),\n            harness_fn,\n            exposed_executor_state,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            timeout,\n            shmem_provider,\n        )\n    }\n}\n\n/// [`StatefulGenericInProcessForkExecutor`] is an executor that forks the current process before each execution. Harness can access some internal state.\npub struct StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    harness_fn: &'a mut H,\n    exposed_executor_state: ES,\n    inner: GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>,\n    phantom: PhantomData<ES>,\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> Debug\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S> + Debug,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[cfg(target_os = \"linux\")]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutionInner\", &self.inner)\n            .finish()\n    }\n\n    #[cfg(not(target_os = \"linux\"))]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        #[cfg(not(target_os = \"linux\"))]\n        return f\n            .debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutionInner\", &self.inner)\n            .finish();\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> UsesState\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type State = S;\n}\n\nimpl<'a, EM, H, HT, OT, S, SP, Z, ES, OF> Executor<EM, Z>\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: State + HasExecutions,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    Z: HasObjective<Objective = OF, State = S>,\n    OF: Feedback<S>,\n{\n    #[allow(unreachable_code)]\n    #[inline]\n    fn run_target(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        mgr: &mut EM,\n        input: &Self::Input,\n    ) -> Result<ExitKind, Error> {\n        *state.executions_mut() += 1;\n\n        unsafe {\n            self.inner.shmem_provider.pre_fork()?;\n            match fork() {\n                Ok(ForkResult::Child) => {\n                    // Child\n                    self.inner.pre_run_target_child(fuzzer, state, mgr, input)?;\n                    (self.harness_fn)(input, &mut self.exposed_executor_state);\n                    self.inner.post_run_target_child(fuzzer, state, mgr, input);\n                    Ok(ExitKind::Ok)\n                }\n                Ok(ForkResult::Parent { child }) => {\n                    // Parent\n                    self.inner.parent(child)\n                }\n                Err(e) => Err(Error::from(e)),\n            }\n        }\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z, OF>\n    StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    Z: UsesState<State = S>,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: State + HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    /// Creates a new [`StatefulGenericInProcessForkExecutor`] with custom hooks\n    #[allow(clippy::too_many_arguments)]\n    pub fn with_hooks(\n        userhooks: HT,\n        harness_fn: &'a mut H,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        Ok(Self {\n            harness_fn,\n            exposed_executor_state,\n            inner: GenericInProcessForkExecutorInner::with_hooks(\n                userhooks,\n                observers,\n                fuzzer,\n                state,\n                event_mgr,\n                timeout,\n                shmem_provider,\n            )?,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Retrieve the harness function.\n    #[inline]\n    pub fn harness(&self) -> &H {\n        self.harness_fn\n    }\n\n    /// Retrieve the harness function for a mutable reference.\n    #[inline]\n    pub fn harness_mut(&mut self) -> &mut H {\n        self.harness_fn\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> UsesObservers\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type Observers = OT;\n}\n\nimpl<'a, H, HT, OT, S, SP, ES, EM, Z> HasObservers\n    for StatefulGenericInProcessForkExecutor<'a, H, HT, OT, S, SP, ES, EM, Z>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        self.inner.observers()\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        self.inner.observers_mut()\n    }\n}\n\n// Path: libafl/src/executors/inprocess_fork/mod.rs\n//! The `GenericInProcessForkExecutor` to do forking before executing the harness in-processly\nuse core::{\n    fmt::{self, Debug, Formatter},\n    time::Duration,\n};\n\nuse libafl_bolts::{\n    os::unix_signals::{ucontext_t, Signal},\n    shmem::ShMemProvider,\n    tuples::tuple_list,\n};\nuse libc::siginfo_t;\nuse nix::unistd::{fork, ForkResult};\n\nuse super::hooks::ExecutorHooksTuple;\nuse crate::{\n    events::{EventFirer, EventRestarter},\n    executors::{\n        hooks::inprocess_fork::InProcessForkExecutorGlobalData,\n        inprocess_fork::inner::GenericInProcessForkExecutorInner, Executor, ExitKind, HasObservers,\n    },\n    feedbacks::Feedback,\n    fuzzer::HasObjective,\n    inputs::UsesInput,\n    observers::{ObserversTuple, UsesObservers},\n    state::{HasExecutions, HasSolutions, State, UsesState},\n    Error,\n};\n\n/// The signature of the crash handler function\npub(crate) type ForkHandlerFuncPtr = unsafe fn(\n    Signal,\n    &mut siginfo_t,\n    Option<&mut ucontext_t>,\n    data: *mut InProcessForkExecutorGlobalData,\n);\n\n/// The inner structure of `InProcessForkExecutor`.\npub mod inner;\n/// A version of `InProcessForkExecutor` with a state accessible from the harness.\npub mod stateful;\n\n/// The `InProcessForkExecutor` with no user hooks\npub type InProcessForkExecutor<'a, H, OT, S, SP, EM, Z> =\n    GenericInProcessForkExecutor<'a, H, (), OT, S, SP, EM, Z>;\n\nimpl<'a, H, OT, S, SP, EM, Z, OF> InProcessForkExecutor<'a, H, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    #[allow(clippy::too_many_arguments)]\n    /// The constructor for `InProcessForkExecutor`\n    pub fn new(\n        harness_fn: &'a mut H,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error> {\n        Self::with_hooks(\n            tuple_list!(),\n            harness_fn,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            timeout,\n            shmem_provider,\n        )\n    }\n}\n\n/// [`GenericInProcessForkExecutor`] is an executor that forks the current process before each execution.\npub struct GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    harness_fn: &'a mut H,\n    inner: GenericInProcessForkExecutorInner<HT, OT, S, SP, EM, Z>,\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> Debug\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: UsesInput,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S> + Debug,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[cfg(target_os = \"linux\")]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutorInner\", &self.inner)\n            .finish()\n    }\n\n    #[cfg(not(target_os = \"linux\"))]\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        #[cfg(not(target_os = \"linux\"))]\n        return f\n            .debug_struct(\"GenericInProcessForkExecutor\")\n            .field(\"GenericInProcessForkExecutorInner\", &self.inner)\n            .finish();\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> UsesState\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type State = S;\n}\n\nimpl<'a, EM, H, HT, OT, S, SP, Z> Executor<EM, Z>\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S> + Debug,\n    S: State + HasExecutions,\n    SP: ShMemProvider,\n    HT: ExecutorHooksTuple<S>,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[allow(unreachable_code)]\n    #[inline]\n    fn run_target(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        mgr: &mut EM,\n        input: &Self::Input,\n    ) -> Result<ExitKind, Error> {\n        *state.executions_mut() += 1;\n\n        unsafe {\n            self.inner.shmem_provider.pre_fork()?;\n            match fork() {\n                Ok(ForkResult::Child) => {\n                    // Child\n                    self.inner.pre_run_target_child(fuzzer, state, mgr, input)?;\n                    (self.harness_fn)(input);\n                    self.inner.post_run_target_child(fuzzer, state, mgr, input);\n                    Ok(ExitKind::Ok)\n                }\n                Ok(ForkResult::Parent { child }) => {\n                    // Parent\n                    self.inner.parent(child)\n                }\n                Err(e) => Err(Error::from(e)),\n            }\n        }\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z, OF> GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: EventFirer<State = S> + EventRestarter<State = S>,\n    OF: Feedback<S>,\n    S: State + HasSolutions,\n    Z: HasObjective<Objective = OF, State = S>,\n{\n    /// Creates a new [`GenericInProcessForkExecutor`] with custom hooks\n    #[allow(clippy::too_many_arguments)]\n    pub fn with_hooks(\n        userhooks: HT,\n        harness_fn: &'a mut H,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n        shmem_provider: SP,\n    ) -> Result<Self, Error>\nwhere {\n        Ok(Self {\n            harness_fn,\n            inner: GenericInProcessForkExecutorInner::with_hooks(\n                userhooks,\n                observers,\n                fuzzer,\n                state,\n                event_mgr,\n                timeout,\n                shmem_provider,\n            )?,\n        })\n    }\n\n    /// Retrieve the harness function.\n    #[inline]\n    pub fn harness(&self) -> &H {\n        self.harness_fn\n    }\n\n    /// Retrieve the harness function for a mutable reference.\n    #[inline]\n    pub fn harness_mut(&mut self) -> &mut H {\n        self.harness_fn\n    }\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> UsesObservers\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    type Observers = OT;\n}\n\nimpl<'a, H, HT, OT, S, SP, EM, Z> HasObservers\n    for GenericInProcessForkExecutor<'a, H, HT, OT, S, SP, EM, Z>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HT: ExecutorHooksTuple<S>,\n    S: State,\n    OT: ObserversTuple<S>,\n    SP: ShMemProvider,\n    EM: UsesState<State = S>,\n    Z: UsesState<State = S>,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        self.inner.observers()\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        self.inner.observers_mut()\n    }\n}\n\n/// signal hooks and `panic_hooks` for the child process\n\npub mod child_signal_handlers {\n    use alloc::boxed::Box;\n    use core::ptr::addr_of_mut;\n    use std::panic;\n\n    use libafl_bolts::os::unix_signals::{ucontext_t, Signal};\n    use libc::siginfo_t;\n\n    use crate::{\n        executors::{\n            hooks::inprocess_fork::{InProcessForkExecutorGlobalData, FORK_EXECUTOR_GLOBAL_DATA},\n            ExitKind, HasObservers,\n        },\n        inputs::UsesInput,\n        observers::ObserversTuple,\n    };\n\n    /// invokes the `post_exec_child` hook on all observer in case the child process panics\n    pub fn setup_child_panic_hook<E>()\n    where\n        E: HasObservers,\n    {\n        let old_hook = panic::take_hook();\n        panic::set_hook(Box::new(move |panic_info| unsafe {\n            old_hook(panic_info);\n            let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA);\n            if !data.is_null() && (*data).is_valid() {\n                let executor = (*data).executor_mut::<E>();\n                let observers = executor.observers_mut();\n                let state = (*data).state_mut::<E::State>();\n                // Invalidate data to not execute again the observer hooks in the crash handler\n                let input = (*data).take_current_input::<<E::State as UsesInput>::Input>();\n                observers\n                    .post_exec_child_all(state, input, &ExitKind::Crash)\n                    .expect(\"Failed to run post_exec on observers\");\n\n                // std::process::abort();\n                libc::_exit(128 + 6); // ABORT exit code\n            }\n        }));\n    }\n\n    /// invokes the `post_exec` hook on all observer in case the child process crashes\n    ///\n    /// # Safety\n    /// The function should only be called from a child crash handler.\n    /// It will dereference the `data` pointer and assume it's valid.\n    #[cfg(unix)]\n    #[allow(clippy::needless_pass_by_value)]\n    pub(crate) unsafe fn child_crash_handler<E>(\n        _signal: Signal,\n        _info: &mut siginfo_t,\n        _context: Option<&mut ucontext_t>,\n        data: &mut InProcessForkExecutorGlobalData,\n    ) where\n        E: HasObservers,\n    {\n        if data.is_valid() {\n            let executor = data.executor_mut::<E>();\n            let observers = executor.observers_mut();\n            let state = data.state_mut::<E::State>();\n            let input = data.take_current_input::<<E::State as UsesInput>::Input>();\n            observers\n                .post_exec_child_all(state, input, &ExitKind::Crash)\n                .expect(\"Failed to run post_exec on observers\");\n        }\n\n        libc::_exit(128 + (_signal as i32));\n    }\n\n    #[cfg(unix)]\n    #[allow(clippy::needless_pass_by_value)]\n    pub(crate) unsafe fn child_timeout_handler<E>(\n        _signal: Signal,\n        _info: &mut siginfo_t,\n        _context: Option<&mut ucontext_t>,\n        data: &mut InProcessForkExecutorGlobalData,\n    ) where\n        E: HasObservers,\n    {\n        if data.is_valid() {\n            let executor = data.executor_mut::<E>();\n            let observers = executor.observers_mut();\n            let state = data.state_mut::<E::State>();\n            let input = data.take_current_input::<<E::State as UsesInput>::Input>();\n            observers\n                .post_exec_child_all(state, input, &ExitKind::Timeout)\n                .expect(\"Failed to run post_exec on observers\");\n        }\n        libc::_exit(128 + (_signal as i32));\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use libafl_bolts::tuples::tuple_list;\n\n    use crate::{\n        executors::{inprocess_fork::GenericInProcessForkExecutorInner, Executor, ExitKind},\n        inputs::NopInput,\n    };\n\n    #[test]\n    #[cfg_attr(miri, ignore)]\n    #[cfg(all(feature = \"std\", feature = \"fork\", unix))]\n    fn test_inprocessfork_exec() {\n        use core::marker::PhantomData;\n\n        use libafl_bolts::shmem::{ShMemProvider, StdShMemProvider};\n        #[cfg(target_os = \"linux\")]\n        use libc::{itimerspec, timespec};\n\n        #[cfg(not(target_os = \"linux\"))]\n        use crate::executors::hooks::timer::{Itimerval, Timeval};\n        use crate::{\n            events::SimpleEventManager,\n            executors::{\n                hooks::inprocess_fork::InChildProcessHooks,\n                inprocess_fork::GenericInProcessForkExecutor,\n            },\n            fuzzer::test::NopFuzzer,\n            state::NopState,\n        };\n\n        let provider = StdShMemProvider::new().unwrap();\n\n        #[cfg(target_os = \"linux\")]\n        let timespec = timespec {\n            tv_sec: 5,\n            tv_nsec: 0,\n        };\n        #[cfg(target_os = \"linux\")]\n        let itimerspec = itimerspec {\n            it_interval: timespec,\n            it_value: timespec,\n        };\n\n        #[cfg(not(target_os = \"linux\"))]\n        let timespec = Timeval {\n            tv_sec: 5,\n            tv_usec: 0,\n        };\n        #[cfg(not(target_os = \"linux\"))]\n        let itimerspec = Itimerval {\n            it_interval: timespec,\n            it_value: timespec,\n        };\n\n        let mut harness = |_buf: &NopInput| ExitKind::Ok;\n        let default = InChildProcessHooks::nop();\n        #[cfg(target_os = \"linux\")]\n        let mut in_process_fork_executor = GenericInProcessForkExecutor {\n            harness_fn: &mut harness,\n            inner: GenericInProcessForkExecutorInner {\n                hooks: tuple_list!(default),\n                shmem_provider: provider,\n                observers: tuple_list!(),\n                itimerspec,\n                phantom: PhantomData,\n            },\n        };\n        #[cfg(not(target_os = \"linux\"))]\n        let mut in_process_fork_executor = GenericInProcessForkExecutor {\n            harness_fn: &mut harness,\n            inner: GenericInProcessForkExecutorInner {\n                hooks: tuple_list!(default),\n                shmem_provider: provider,\n                observers: tuple_list!(),\n                itimerval: itimerspec,\n                phantom: PhantomData,\n            },\n        };\n        let input = NopInput {};\n        let mut fuzzer = NopFuzzer::new();\n        let mut state = NopState::new();\n        let mut mgr = SimpleEventManager::printing();\n        in_process_fork_executor\n            .run_target(&mut fuzzer, &mut state, &mut mgr, &input)\n            .unwrap();\n    }\n}\n\n// Path: libafl/src/executors/hooks/inprocess_fork.rs\n//! The hook for the `InProcessForkExecutor`\nuse alloc::vec::Vec;\nuse core::{\n    ffi::c_void,\n    marker::PhantomData,\n    ptr::{addr_of_mut, null},\n    sync::atomic::{compiler_fence, Ordering},\n};\nuse std::intrinsics::transmute;\n\n#[cfg(not(miri))]\nuse libafl_bolts::os::unix_signals::setup_signal_handler;\nuse libafl_bolts::os::unix_signals::{ucontext_t, Handler, Signal};\nuse libc::siginfo_t;\n\nuse crate::{\n    executors::{\n        common_signals,\n        hooks::ExecutorHook,\n        inprocess_fork::{child_signal_handlers, ForkHandlerFuncPtr},\n        HasObservers,\n    },\n    inputs::UsesInput,\n    Error,\n};\n\n/// The inmem fork executor's hooks.\n#[derive(Debug)]\npub struct InChildProcessHooks<S> {\n    /// On crash C function pointer\n    pub crash_handler: *const c_void,\n    /// On timeout C function pointer\n    pub timeout_handler: *const c_void,\n    phantom: PhantomData<S>,\n}\n\nimpl<S> ExecutorHook<S> for InChildProcessHooks<S>\nwhere\n    S: UsesInput,\n{\n    /// Init this hook\n    fn init<E: HasObservers>(&mut self, _state: &mut S) {}\n\n    /// Call before running a target.\n    fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) {\n        unsafe {\n            let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA);\n            (*data).crash_handler = self.crash_handler;\n            (*data).timeout_handler = self.timeout_handler;\n            compiler_fence(Ordering::SeqCst);\n        }\n    }\n\n    fn post_exec(&mut self, _state: &mut S, _input: &S::Input) {}\n}\n\nimpl<S> InChildProcessHooks<S> {\n    /// Create new [`InChildProcessHooks`].\n    pub fn new<E>() -> Result<Self, Error>\n    where\n        E: HasObservers,\n    {\n        #[cfg_attr(miri, allow(unused_variables))]\n        unsafe {\n            let data = addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA);\n            // child_signal_handlers::setup_child_panic_hook::<E, I, OT, S>();\n            #[cfg(not(miri))]\n            setup_signal_handler(data)?;\n            compiler_fence(Ordering::SeqCst);\n            Ok(Self {\n                crash_handler: child_signal_handlers::child_crash_handler::<E> as *const c_void,\n                timeout_handler: child_signal_handlers::child_timeout_handler::<E> as *const c_void,\n                phantom: PhantomData,\n            })\n        }\n    }\n\n    /// Replace the hooks with `nop` hooks, deactivating the hooks\n    #[must_use]\n    pub fn nop() -> Self {\n        Self {\n            crash_handler: null(),\n            timeout_handler: null(),\n            phantom: PhantomData,\n        }\n    }\n}\n\n/// The global state of the in-process-fork harness.\n\n#[derive(Debug)]\npub(crate) struct InProcessForkExecutorGlobalData {\n    /// Stores a pointer to the fork executor struct\n    pub executor_ptr: *const c_void,\n    /// Stores a pointer to the state\n    pub state_ptr: *const c_void,\n    /// Stores a pointer to the current input\n    pub current_input_ptr: *const c_void,\n    /// Stores a pointer to the `crash_handler` function\n    pub crash_handler: *const c_void,\n    /// Stores a pointer to the `timeout_handler` function\n    pub timeout_handler: *const c_void,\n}\n\nunsafe impl Sync for InProcessForkExecutorGlobalData {}\n\nunsafe impl Send for InProcessForkExecutorGlobalData {}\n\nimpl InProcessForkExecutorGlobalData {\n    pub(crate) fn executor_mut<'a, E>(&self) -> &'a mut E {\n        unsafe { (self.executor_ptr as *mut E).as_mut().unwrap() }\n    }\n\n    pub(crate) fn state_mut<'a, S>(&self) -> &'a mut S {\n        unsafe { (self.state_ptr as *mut S).as_mut().unwrap() }\n    }\n\n    /*fn current_input<'a, I>(&self) -> &'a I {\n        unsafe { (self.current_input_ptr as *const I).as_ref().unwrap() }\n    }*/\n\n    pub(crate) fn take_current_input<'a, I>(&mut self) -> &'a I {\n        let r = unsafe { (self.current_input_ptr as *const I).as_ref().unwrap() };\n        self.current_input_ptr = null();\n        r\n    }\n\n    pub(crate) fn is_valid(&self) -> bool {\n        !self.current_input_ptr.is_null()\n    }\n}\n\n/// a static variable storing the global state\n\npub(crate) static mut FORK_EXECUTOR_GLOBAL_DATA: InProcessForkExecutorGlobalData =\n    InProcessForkExecutorGlobalData {\n        executor_ptr: null(),\n        state_ptr: null(),\n        current_input_ptr: null(),\n        crash_handler: null(),\n        timeout_handler: null(),\n    };\n\nimpl Handler for InProcessForkExecutorGlobalData {\n    fn handle(&mut self, signal: Signal, info: &mut siginfo_t, context: Option<&mut ucontext_t>) {\n        match signal {\n            Signal::SigUser2 | Signal::SigAlarm => unsafe {\n                if !FORK_EXECUTOR_GLOBAL_DATA.timeout_handler.is_null() {\n                    let func: ForkHandlerFuncPtr =\n                        transmute(FORK_EXECUTOR_GLOBAL_DATA.timeout_handler);\n                    (func)(\n                        signal,\n                        info,\n                        context,\n                        addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA),\n                    );\n                }\n            },\n            _ => unsafe {\n                if !FORK_EXECUTOR_GLOBAL_DATA.crash_handler.is_null() {\n                    let func: ForkHandlerFuncPtr =\n                        transmute(FORK_EXECUTOR_GLOBAL_DATA.crash_handler);\n                    (func)(\n                        signal,\n                        info,\n                        context,\n                        addr_of_mut!(FORK_EXECUTOR_GLOBAL_DATA),\n                    );\n                }\n            },\n        }\n    }\n\n    fn signals(&self) -> Vec<Signal> {\n        common_signals()\n    }\n}\n\n// Path: libafl/src/executors/hooks/timer.rs\n//! The struct `TimerStruct` will absorb all the difference in timeout implementation in various system.\n#[cfg(any(windows, target_os = \"linux\"))]\nuse core::ptr::addr_of_mut;\nuse core::time::Duration;\n#[cfg(target_os = \"linux\")]\nuse core::{\n    mem::zeroed,\n    ptr::{addr_of, null_mut},\n};\n\n#[cfg(all(unix, not(target_os = \"linux\")))]\npub(crate) const ITIMER_REAL: core::ffi::c_int = 0;\n\n#[cfg(windows)]\nuse core::{\n    ffi::c_void,\n    ptr::write_volatile,\n    sync::atomic::{compiler_fence, Ordering},\n};\n\n#[cfg(target_os = \"linux\")]\nuse libafl_bolts::current_time;\n#[cfg(windows)]\nuse windows::Win32::{\n    Foundation::FILETIME,\n    System::Threading::{\n        CreateThreadpoolTimer, EnterCriticalSection, InitializeCriticalSection,\n        LeaveCriticalSection, SetThreadpoolTimer, CRITICAL_SECTION, PTP_CALLBACK_INSTANCE,\n        PTP_TIMER, TP_CALLBACK_ENVIRON_V3,\n    },\n};\n\n#[cfg(windows)]\nuse crate::executors::hooks::inprocess::GLOBAL_STATE;\n\n#[repr(C)]\n#[cfg(all(unix, not(target_os = \"linux\")))]\n#[derive(Copy, Clone)]\npub(crate) struct Timeval {\n    pub tv_sec: i64,\n    pub tv_usec: i64,\n}\n\n#[cfg(all(unix, not(target_os = \"linux\")))]\nimpl core::fmt::Debug for Timeval {\n    #[allow(clippy::cast_sign_loss)]\n    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {\n        write!(\n            f,\n            \"Timeval {{ tv_sec: {:?}, tv_usec: {:?} (tv: {:?}) }}\",\n            self.tv_sec,\n            self.tv_usec,\n            Duration::new(self.tv_sec as _, (self.tv_usec * 1000) as _)\n        )\n    }\n}\n\n#[repr(C)]\n#[cfg(all(unix, not(target_os = \"linux\")))]\n#[derive(Debug, Copy, Clone)]\npub(crate) struct Itimerval {\n    pub it_interval: Timeval,\n    pub it_value: Timeval,\n}\n\n#[cfg(all(feature = \"std\", unix, not(target_os = \"linux\")))]\nextern \"C\" {\n    pub(crate) fn setitimer(\n        which: libc::c_int,\n        new_value: *mut Itimerval,\n        old_value: *mut Itimerval,\n    ) -> libc::c_int;\n}\n\n/// The strcut about all the internals of the timer.\n/// This struct absorb all platform specific differences about timer.\n#[allow(missing_debug_implementations)]\npub struct TimerStruct {\n    // timeout time (windows)\n    #[cfg(windows)]\n    milli_sec: i64,\n    #[cfg(windows)]\n    ptp_timer: PTP_TIMER,\n    #[cfg(windows)]\n    critical: CRITICAL_SECTION,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) batch_mode: bool,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) exec_tmout: Duration,\n    #[cfg(all(unix, not(target_os = \"linux\")))]\n    itimerval: Itimerval,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) timerid: libc::timer_t,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) itimerspec: libc::itimerspec,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) executions: u32,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) avg_mul_k: u32,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) last_signal_time: Duration,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) avg_exec_time: Duration,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) start_time: Duration,\n    #[cfg(target_os = \"linux\")]\n    pub(crate) tmout_start_time: Duration,\n}\n\n#[cfg(all(feature = \"std\", windows))]\n#[allow(non_camel_case_types)]\ntype PTP_TIMER_CALLBACK = unsafe extern \"system\" fn(\n    param0: PTP_CALLBACK_INSTANCE,\n    param1: *mut c_void,\n    param2: PTP_TIMER,\n);\n\nimpl TimerStruct {\n    /// Timeout value in milli seconds\n    #[cfg(windows)]\n    #[must_use]\n    pub fn milli_sec(&self) -> i64 {\n        self.milli_sec\n    }\n\n    #[cfg(windows)]\n    /// Timeout value in milli seconds (mut ref)\n    pub fn milli_sec_mut(&mut self) -> &mut i64 {\n        &mut self.milli_sec\n    }\n\n    /// The timer object for windows\n    #[cfg(windows)]\n    #[must_use]\n    pub fn ptp_timer(&self) -> &PTP_TIMER {\n        &self.ptp_timer\n    }\n\n    #[cfg(windows)]\n    /// The timer object for windows\n    pub fn ptp_timer_mut(&mut self) -> &mut PTP_TIMER {\n        &mut self.ptp_timer\n    }\n\n    /// The critical section, we need to use critical section to access the globals\n    #[cfg(windows)]\n    #[must_use]\n    pub fn critical(&self) -> &CRITICAL_SECTION {\n        &self.critical\n    }\n\n    #[cfg(windows)]\n    /// The critical section (mut ref), we need to use critical section to access the globals\n    pub fn critical_mut(&mut self) -> &mut CRITICAL_SECTION {\n        &mut self.critical\n    }\n\n    /// Create a `TimerStruct` with the specified timeout\n    #[cfg(all(unix, not(target_os = \"linux\")))]\n    #[must_use]\n    pub fn new(exec_tmout: Duration) -> Self {\n        let milli_sec = exec_tmout.as_millis();\n        let it_value = Timeval {\n            tv_sec: (milli_sec / 1000) as i64,\n            tv_usec: (milli_sec % 1000) as i64,\n        };\n        let it_interval = Timeval {\n            tv_sec: 0,\n            tv_usec: 0,\n        };\n        let itimerval = Itimerval {\n            it_interval,\n            it_value,\n        };\n        Self { itimerval }\n    }\n\n    /// Constructor\n    /// # Safety\n    /// This function calls transmute to setup the timeout handler for windows\n    #[cfg(windows)]\n    #[must_use]\n    pub unsafe fn new(exec_tmout: Duration, timeout_handler: *const c_void) -> Self {\n        let milli_sec = exec_tmout.as_millis() as i64;\n\n        let timeout_handler: PTP_TIMER_CALLBACK = unsafe { std::mem::transmute(timeout_handler) };\n        let ptp_timer = unsafe {\n            CreateThreadpoolTimer(\n                Some(timeout_handler),\n                Some(addr_of_mut!(GLOBAL_STATE) as *mut c_void),\n                Some(&TP_CALLBACK_ENVIRON_V3::default()),\n            )\n        }\n        .expect(\"CreateThreadpoolTimer failed!\");\n\n        let mut critical = CRITICAL_SECTION::default();\n        unsafe {\n            InitializeCriticalSection(&mut critical);\n        }\n        Self {\n            milli_sec,\n            ptp_timer,\n            critical,\n        }\n    }\n\n    #[cfg(target_os = \"linux\")]\n    #[must_use]\n    #[allow(unused_unsafe)]\n    #[allow(unused_mut)]\n    /// Create a `TimerStruct` with the specified timeout\n    pub fn new(exec_tmout: Duration) -> Self {\n        let milli_sec = exec_tmout.as_millis();\n        let it_value = libc::timespec {\n            tv_sec: (milli_sec / 1000) as _,\n            tv_nsec: ((milli_sec % 1000) * 1000 * 1000) as _,\n        };\n        let it_interval = libc::timespec {\n            tv_sec: 0,\n            tv_nsec: 0,\n        };\n        let itimerspec = libc::itimerspec {\n            it_interval,\n            it_value,\n        };\n        let mut timerid: libc::timer_t = null_mut();\n        unsafe {\n            #[cfg(not(miri))]\n            // creates a new per-process interval timer\n            libc::timer_create(libc::CLOCK_MONOTONIC, null_mut(), addr_of_mut!(timerid));\n        }\n\n        Self {\n            batch_mode: false,\n            itimerspec,\n            timerid,\n            exec_tmout,\n            executions: 0,\n            avg_mul_k: 1,\n            last_signal_time: Duration::ZERO,\n            avg_exec_time: Duration::ZERO,\n            start_time: Duration::ZERO,\n            tmout_start_time: Duration::ZERO,\n        }\n    }\n\n    #[cfg(target_os = \"linux\")]\n    #[must_use]\n    /// Constructor but use batch mode\n...\n", "needle_token_start": 2420, "needle_token_end": 2478, "code_context_ntokens": 16284, "output": ["```rust\nfn launched(monitor: MT, staterestorer: StateRestorer<SP>) -> Self {\n    Self {\n        staterestorer,\n        simple_event_mgr: SimpleEventManager::new(monitor),\n    }\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "with_max_stack_pow", "language": "rust", "path": "libafl/src/mutators/scheduled.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a new instance of a mutator with a specified set of mutations and a configurable maximum limit for the number of iterations used in mutation stacking.\n2. **Input**: It accepts a tuple of mutations and a numerical value representing the maximum power for calculating the stack depth.\n3. **Output**: It returns a new instance of the mutator configured with the provided mutations and maximum stack depth limit.\n4. **Procedure**: The function initializes a new mutator instance by setting its name, mutations, and the maximum stack depth based on the input values. The name is formatted to include the names of the mutations, and the maximum stack depth is set to the provided numerical limit.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: libafl/src/mutators/grimoire.rs\n//! Grimoire is the rewritten grimoire mutator in rust.\n//! See the original repo [`Grimoire`](https://github.com/RUB-SysSec/grimoire) for more details.\n\nuse alloc::vec::Vec;\nuse core::cmp::{max, min};\n\nuse libafl_bolts::{rands::Rand, Named};\n\nuse crate::{\n    corpus::Corpus,\n    inputs::{GeneralizedInputMetadata, GeneralizedItem},\n    mutators::{token_mutations::Tokens, MutationResult, Mutator},\n    random_corpus_id,\n    state::{HasCorpus, HasMetadata, HasRand},\n    Error,\n};\n\nconst RECURSIVE_REPLACEMENT_DEPTH: [usize; 6] = [2, 4, 8, 16, 32, 64];\nconst MAX_RECURSIVE_REPLACEMENT_LEN: usize = 64 << 10;\nconst CHOOSE_SUBINPUT_PROB: u64 = 50;\n\nfn extend_with_random_generalized<S>(\n    state: &mut S,\n    items: &mut Vec<GeneralizedItem>,\n    gap_indices: &mut Vec<usize>,\n) -> Result<MutationResult, Error>\nwhere\n    S: HasMetadata + HasRand + HasCorpus,\n{\n    let idx = random_corpus_id!(state.corpus(), state.rand_mut());\n\n    if state.rand_mut().below(100) > CHOOSE_SUBINPUT_PROB {\n        if state.rand_mut().below(100) < 50 {\n            let rand1 = state.rand_mut().next() as usize;\n            let rand2 = state.rand_mut().next() as usize;\n\n            let other_testcase = state.corpus().get(idx)?.borrow();\n            if let Some(other) = other_testcase\n                .metadata_map()\n                .get::<GeneralizedInputMetadata>()\n            {\n                let gen = other.generalized();\n\n                for (i, _) in gen\n                    .iter()\n                    .enumerate()\n                    .filter(|&(_, x)| *x == GeneralizedItem::Gap)\n                {\n                    gap_indices.push(i);\n                }\n                let min_idx = gap_indices[rand1 % gap_indices.len()];\n                let max_idx = gap_indices[rand2 % gap_indices.len()];\n                let (mut min_idx, max_idx) = (min(min_idx, max_idx), max(min_idx, max_idx));\n\n                gap_indices.clear();\n\n                if items.last() == Some(&GeneralizedItem::Gap) {\n                    min_idx += 1;\n                }\n                items.extend_from_slice(&gen[min_idx..=max_idx]);\n\n                debug_assert!(items.first() == Some(&GeneralizedItem::Gap));\n                debug_assert!(items.last() == Some(&GeneralizedItem::Gap));\n\n                return Ok(MutationResult::Mutated);\n            }\n        }\n\n        let rand1 = state.rand_mut().next() as usize;\n\n        if let Some(meta) = state.metadata_map().get::<Tokens>() {\n            if !meta.tokens().is_empty() {\n                let tok = &meta.tokens()[rand1 % meta.tokens().len()];\n                if items.last() != Some(&GeneralizedItem::Gap) {\n                    items.push(GeneralizedItem::Gap);\n                }\n                items.push(GeneralizedItem::Bytes(tok.clone()));\n                items.push(GeneralizedItem::Gap);\n\n                debug_assert!(items.first() == Some(&GeneralizedItem::Gap));\n                debug_assert!(items.last() == Some(&GeneralizedItem::Gap));\n\n                return Ok(MutationResult::Mutated);\n            }\n        }\n    }\n\n    let other_testcase = state.corpus().get(idx)?.borrow();\n    if let Some(other) = other_testcase\n        .metadata_map()\n        .get::<GeneralizedInputMetadata>()\n    {\n        let gen = other.generalized();\n\n        if items.last() == Some(&GeneralizedItem::Gap) && gen.first() == Some(&GeneralizedItem::Gap)\n        {\n            items.extend_from_slice(&gen[1..]);\n        } else {\n            items.extend_from_slice(gen);\n        }\n\n        debug_assert!(items.first() == Some(&GeneralizedItem::Gap));\n        debug_assert!(items.last() == Some(&GeneralizedItem::Gap));\n\n        Ok(MutationResult::Mutated)\n    } else {\n        Ok(MutationResult::Skipped)\n    }\n}\n\n/// Extend the generalized input with another random one from the corpus\n#[derive(Debug, Default)]\npub struct GrimoireExtensionMutator {\n    gap_indices: Vec<usize>,\n}\n\nimpl<S> Mutator<GeneralizedInputMetadata, S> for GrimoireExtensionMutator\nwhere\n    S: HasMetadata + HasRand + HasCorpus,\n{\n    fn mutate(\n        &mut self,\n        state: &mut S,\n        generalised_meta: &mut GeneralizedInputMetadata,\n    ) -> Result<MutationResult, Error> {\n        extend_with_random_generalized(\n            state,\n            generalised_meta.generalized_mut(),\n            &mut self.gap_indices,\n        )\n    }\n}\n\nimpl Named for GrimoireExtensionMutator {\n    fn name(&self) -> &str {\n        \"GrimoireExtensionMutator\"\n    }\n}\n\nimpl GrimoireExtensionMutator {\n    /// Creates a new [`GrimoireExtensionMutator`].\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            gap_indices: vec![],\n        }\n    }\n}\n\n/// Extend the generalized input with another random one from the corpus\n#[derive(Debug, Default)]\npub struct GrimoireRecursiveReplacementMutator {\n    scratch: Vec<GeneralizedItem>,\n    gap_indices: Vec<usize>,\n}\n\nimpl<S> Mutator<GeneralizedInputMetadata, S> for GrimoireRecursiveReplacementMutator\nwhere\n    S: HasMetadata + HasRand + HasCorpus,\n{\n    fn mutate(\n        &mut self,\n        state: &mut S,\n        generalised_meta: &mut GeneralizedInputMetadata,\n    ) -> Result<MutationResult, Error> {\n        let mut mutated = MutationResult::Skipped;\n\n        let depth = *state.rand_mut().choose(&RECURSIVE_REPLACEMENT_DEPTH);\n        for _ in 0..depth {\n            if generalised_meta.generalized_len() >= MAX_RECURSIVE_REPLACEMENT_LEN {\n                break;\n            }\n\n            let gen = generalised_meta.generalized_mut();\n\n            for (i, _) in gen\n                .iter()\n                .enumerate()\n                .filter(|&(_, x)| *x == GeneralizedItem::Gap)\n...\n// Path: libafl/src/mutators/scheduled.rs\n//! The `ScheduledMutator` schedules multiple mutations internally.\n\nuse alloc::{string::String, vec::Vec};\nuse core::{\n    fmt::{self, Debug},\n    marker::PhantomData,\n};\n\nuse libafl_bolts::{\n    rands::Rand,\n    tuples::{tuple_list, tuple_list_type, Merge, NamedTuple},\n    AsMutSlice, AsSlice, Named,\n};\nuse serde::{Deserialize, Serialize};\n\nuse super::MutationId;\nuse crate::{\n    corpus::{Corpus, CorpusId},\n    mutators::{\n        mutations::{\n            BitFlipMutator, ByteAddMutator, ByteDecMutator, ByteFlipMutator, ByteIncMutator,\n            ByteInterestingMutator, ByteNegMutator, ByteRandMutator, BytesCopyMutator,\n            BytesDeleteMutator, BytesExpandMutator, BytesInsertCopyMutator, BytesInsertMutator,\n            BytesRandInsertMutator, BytesRandSetMutator, BytesSetMutator, BytesSwapMutator,\n            CrossoverInsertMutator, CrossoverReplaceMutator, DwordAddMutator,\n            DwordInterestingMutator, QwordAddMutator, WordAddMutator, WordInterestingMutator,\n        },\n        token_mutations::{TokenInsert, TokenReplace},\n        MutationResult, Mutator, MutatorsTuple,\n    },\n    state::{HasCorpus, HasMetadata, HasRand},\n    Error,\n};\n\n/// The metadata placed in a [`crate::corpus::Testcase`] by a [`LoggerScheduledMutator`].\n#[derive(Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct LogMutationMetadata {\n    /// A list of logs\n    pub list: Vec<String>,\n}\n\nlibafl_bolts::impl_serdeany!(LogMutationMetadata);\n\nimpl AsSlice for LogMutationMetadata {\n    type Entry = String;\n    #[must_use]\n    fn as_slice(&self) -> &[String] {\n        self.list.as_slice()\n    }\n}\nimpl AsMutSlice for LogMutationMetadata {\n    type Entry = String;\n    #[must_use]\n    fn as_mut_slice(&mut self) -> &mut [String] {\n        self.list.as_mut_slice()\n    }\n}\n\nimpl LogMutationMetadata {\n    /// Creates new [`struct@LogMutationMetadata`].\n    #[must_use]\n    pub fn new(list: Vec<String>) -> Self {\n        Self { list }\n    }\n}\n\n/// A [`Mutator`] that composes multiple mutations into one.\npub trait ComposedByMutations<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n{\n    /// Get the mutations\n    fn mutations(&self) -> &MT;\n\n    /// Get the mutations (mutable)\n    fn mutations_mut(&mut self) -> &mut MT;\n}\n\n/// A [`Mutator`] scheduling multiple [`Mutator`]s for an input.\npub trait ScheduledMutator<I, MT, S>: ComposedByMutations<I, MT, S> + Mutator<I, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n{\n    /// Compute the number of iterations used to apply stacked mutations\n    fn iterations(&self, state: &mut S, input: &I) -> u64;\n\n    /// Get the next mutation to apply\n    fn schedule(&self, state: &mut S, input: &I) -> MutationId;\n\n    /// New default implementation for mutate.\n    /// Implementations must forward `mutate()` to this method\n    fn scheduled_mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        let mut r = MutationResult::Skipped;\n        let num = self.iterations(state, input);\n        for _ in 0..num {\n            let idx = self.schedule(state, input);\n            let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?;\n            if outcome == MutationResult::Mutated {\n                r = MutationResult::Mutated;\n            }\n        }\n        Ok(r)\n    }\n}\n\n/// A [`Mutator`] that schedules one of the embedded mutations on each call.\npub struct StdScheduledMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand,\n{\n    name: String,\n    mutations: MT,\n    max_stack_pow: u64,\n    phantom: PhantomData<(I, S)>,\n}\n\nimpl<I, MT, S> Debug for StdScheduledMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand,\n{\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(\n            f,\n            \"StdScheduledMutator with {} mutations for Input type {}\",\n            self.mutations.len(),\n            core::any::type_name::<I>()\n        )\n    }\n}\n\nimpl<I, MT, S> Named for StdScheduledMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand,\n{\n    fn name(&self) -> &str {\n        &self.name\n    }\n}\n\nimpl<I, MT, S> Mutator<I, S> for StdScheduledMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand,\n{\n    #[inline]\n    fn mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        self.scheduled_mutate(state, input)\n    }\n}\n\nimpl<I, MT, S> ComposedByMutations<I, MT, S> for StdScheduledMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand,\n{\n    /// Get the mutations\n    #[inline]\n    fn mutations(&self) -> &MT {\n        &self.mutations\n    }\n\n    // Get the mutations (mutable)\n    #[inline]\n    fn mutations_mut(&mut self) -> &mut MT {\n        &mut self.mutations\n    }\n}\n\nimpl<I, MT, S> ScheduledMutator<I, MT, S> for StdScheduledMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand,\n{\n    /// Compute the number of iterations used to apply stacked mutations\n    fn iterations(&self, state: &mut S, _: &I) -> u64 {\n        1 << (1 + state.rand_mut().below(self.max_stack_pow))\n    }\n\n    /// Get the next mutation to apply\n    fn schedule(&self, state: &mut S, _: &I) -> MutationId {\n        debug_assert!(self.mutations.len() != 0);\n        state.rand_mut().below(self.mutations.len() as u64).into()\n    }\n}\n\nimpl<I, MT, S> StdScheduledMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand,\n{\n    /// Create a new [`StdScheduledMutator`] instance specifying mutations\n    pub fn new(mutations: MT) -> Self {\n        StdScheduledMutator {\n            name: format!(\"StdScheduledMutator[{}]\", mutations.names().join(\", \")),\n            mutations,\n            max_stack_pow: 7,\n            phantom: PhantomData,\n        }\n    }\n\n    /// Create a new [`StdScheduledMutator`] instance specifying mutations and the maximun number of iterations\n    \npub fn with_max_stack_pow(mutations: MT, max_stack_pow: u64) -> Self {\n        StdScheduledMutator {\n            name: format!(\"StdScheduledMutator[{}]\", mutations.names().join(\", \")),\n            mutations,\n            max_stack_pow,\n            phantom: PhantomData,\n        }\n    }\n}\n\n/// Tuple type of the mutations that compose the Havoc mutator without crossover mutations\npub type HavocMutationsNoCrossoverType = tuple_list_type!(\n    BitFlipMutator,\n    ByteFlipMutator,\n    ByteIncMutator,\n    ByteDecMutator,\n    ByteNegMutator,\n    ByteRandMutator,\n    ByteAddMutator,\n    WordAddMutator,\n    DwordAddMutator,\n    QwordAddMutator,\n    ByteInterestingMutator,\n    WordInterestingMutator,\n    DwordInterestingMutator,\n    BytesDeleteMutator,\n    BytesDeleteMutator,\n    BytesDeleteMutator,\n    BytesDeleteMutator,\n    BytesExpandMutator,\n    BytesInsertMutator,\n    BytesRandInsertMutator,\n    BytesSetMutator,\n    BytesRandSetMutator,\n    BytesCopyMutator,\n    BytesInsertCopyMutator,\n    BytesSwapMutator,\n);\n\n/// Tuple type of the mutations that compose the Havoc mutator's crossover mutations\npub type HavocCrossoverType<I> =\n    tuple_list_type!(CrossoverInsertMutator<I>, CrossoverReplaceMutator<I>);\n\n/// Tuple type of the mutations that compose the Havoc mutator\npub type HavocMutationsType<I> = tuple_list_type!(\n    BitFlipMutator,\n    ByteFlipMutator,\n    ByteIncMutator,\n    ByteDecMutator,\n    ByteNegMutator,\n    ByteRandMutator,\n    ByteAddMutator,\n    WordAddMutator,\n    DwordAddMutator,\n    QwordAddMutator,\n    ByteInterestingMutator,\n    WordInterestingMutator,\n    DwordInterestingMutator,\n    BytesDeleteMutator,\n    BytesDeleteMutator,\n    BytesDeleteMutator,\n    BytesDeleteMutator,\n    BytesExpandMutator,\n    BytesInsertMutator,\n    BytesRandInsertMutator,\n    BytesSetMutator,\n    BytesRandSetMutator,\n    BytesCopyMutator,\n    BytesInsertCopyMutator,\n    BytesSwapMutator,\n    CrossoverInsertMutator<I>,\n    CrossoverReplaceMutator<I>,\n);\n\n/// Get the mutations that compose the Havoc mutator (only applied to single inputs)\n#[must_use]\npub fn havoc_mutations_no_crossover() -> HavocMutationsNoCrossoverType {\n    tuple_list!(\n        BitFlipMutator::new(),\n        ByteFlipMutator::new(),\n        ByteIncMutator::new(),\n        ByteDecMutator::new(),\n        ByteNegMutator::new(),\n        ByteRandMutator::new(),\n        ByteAddMutator::new(),\n        WordAddMutator::new(),\n        DwordAddMutator::new(),\n        QwordAddMutator::new(),\n        ByteInterestingMutator::new(),\n        WordInterestingMutator::new(),\n        DwordInterestingMutator::new(),\n        BytesDeleteMutator::new(),\n        BytesDeleteMutator::new(),\n        BytesDeleteMutator::new(),\n        BytesDeleteMutator::new(),\n        BytesExpandMutator::new(),\n        BytesInsertMutator::new(),\n        BytesRandInsertMutator::new(),\n        BytesSetMutator::new(),\n        BytesRandSetMutator::new(),\n        BytesCopyMutator::new(),\n        BytesInsertCopyMutator::new(),\n        BytesSwapMutator::new(),\n    )\n}\n\n/// Get the mutations that compose the Havoc mutator's crossover strategy\n#[must_use]\npub fn havoc_crossover<I>() -> HavocCrossoverType<I> {\n    tuple_list!(\n        CrossoverInsertMutator::new(),\n        CrossoverReplaceMutator::new(),\n    )\n}\n\n/// Get the mutations that compose the Havoc mutator\n#[must_use]\npub fn havoc_mutations<I>() -> HavocMutationsType<I> {\n    havoc_mutations_no_crossover().merge(havoc_crossover())\n}\n\n/// Get the mutations that uses the Tokens metadata\n#[must_use]\npub fn tokens_mutations() -> tuple_list_type!(TokenInsert, TokenReplace) {\n    tuple_list!(TokenInsert::new(), TokenReplace::new())\n}\n\n/// A logging [`Mutator`] that wraps around a [`StdScheduledMutator`].\npub struct LoggerScheduledMutator<I, MT, S, SM>\nwhere\n    MT: MutatorsTuple<I, S> + NamedTuple,\n    S: HasRand + HasCorpus,\n    SM: ScheduledMutator<I, MT, S>,\n{\n    name: String,\n    scheduled: SM,\n    mutation_log: Vec<MutationId>,\n    phantom: PhantomData<(I, MT, S)>,\n}\n\nimpl<I, MT, S, SM> Debug for LoggerScheduledMutator<I, MT, S, SM>\nwhere\n    MT: MutatorsTuple<I, S> + NamedTuple,\n    S: HasRand + HasCorpus,\n    SM: ScheduledMutator<I, MT, S>,\n{\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(\n            f,\n            \"LoggerScheduledMutator with {} mutations for Input type {}\",\n            MT::LEN,\n            core::any::type_name::<I>()\n        )\n    }\n}\n\nimpl<I, MT, S, SM> Named for LoggerScheduledMutator<I, MT, S, SM>\nwhere\n    MT: MutatorsTuple<I, S> + NamedTuple,\n    S: HasRand + HasCorpus,\n    SM: ScheduledMutator<I, MT, S>,\n{\n    fn name(&self) -> &str {\n        &self.name\n    }\n}\n\nimpl<I, MT, S, SM> Mutator<I, S> for LoggerScheduledMutator<I, MT, S, SM>\nwhere\n    MT: MutatorsTuple<I, S> + NamedTuple,\n    S: HasRand + HasCorpus,\n    SM: ScheduledMutator<I, MT, S>,\n{\n    fn mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        self.scheduled_mutate(state, input)\n    }\n\n    fn post_exec(&mut self, state: &mut S, corpus_idx: Option<CorpusId>) -> Result<(), Error> {\n        if let Some(idx) = corpus_idx {\n            let mut testcase = (*state.corpus_mut().get(idx)?).borrow_mut();\n            let mut log = Vec::<String>::new();\n            while let Some(idx) = self.mutation_log.pop() {\n                let name = String::from(self.scheduled.mutations().name(idx.0).unwrap()); // TODO maybe return an Error on None\n                log.push(name);\n            }\n            let meta = LogMutationMetadata::new(log);\n            testcase.add_metadata(meta);\n        };\n        // Always reset the log for each run\n        self.mutation_log.clear();\n        Ok(())\n    }\n}\n\nimpl<I, MT, S, SM> ComposedByMutations<I, MT, S> for LoggerScheduledMutator<I, MT, S, SM>\nwhere\n    MT: MutatorsTuple<I, S> + NamedTuple,\n    S: HasRand + HasCorpus,\n    SM: ScheduledMutator<I, MT, S>,\n{\n    #[inline]\n    fn mutations(&self) -> &MT {\n        self.scheduled.mutations()\n    }\n\n    #[inline]\n    fn mutations_mut(&mut self) -> &mut MT {\n        self.scheduled.mutations_mut()\n    }\n}\n\nimpl<I, MT, S, SM> ScheduledMutator<I, MT, S> for LoggerScheduledMutator<I, MT, S, SM>\nwhere\n    MT: MutatorsTuple<I, S> + NamedTuple,\n    S: HasRand + HasCorpus,\n    SM: ScheduledMutator<I, MT, S>,\n{\n    /// Compute the number of iterations used to apply stacked mutations\n    fn iterations(&self, state: &mut S, _: &I) -> u64 {\n        1 << (1 + state.rand_mut().below(6))\n    }\n\n    /// Get the next mutation to apply\n    fn schedule(&self, state: &mut S, _: &I) -> MutationId {\n        debug_assert!(MT::LEN != 0);\n        state.rand_mut().below(MT::LEN as u64).into()\n    }\n\n    fn scheduled_mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        let mut r = MutationResult::Skipped;\n        let num = self.iterations(state, input);\n        self.mutation_log.clear();\n        for _ in 0..num {\n            let idx = self.schedule(state, input);\n            self.mutation_log.push(idx);\n            let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?;\n            if outcome == MutationResult::Mutated {\n                r = MutationResult::Mutated;\n            }\n        }\n        Ok(r)\n    }\n}\n\nimpl<I, MT, S, SM> LoggerScheduledMutator<I, MT, S, SM>\nwhere\n    MT: MutatorsTuple<I, S> + NamedTuple,\n    S: HasRand + HasCorpus,\n    SM: ScheduledMutator<I, MT, S>,\n{\n    /// Create a new [`LoggerScheduledMutator`] instance without mutations and corpus\n    /// This mutator logs all mutators.\n    pub fn new(scheduled: SM) -> Self {\n        Self {\n            name: format!(\"LoggerScheduledMutator[{}]\", scheduled.name()),\n            scheduled,\n            mutation_log: vec![],\n            phantom: PhantomData,\n        }\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use libafl_bolts::rands::{Rand, StdRand, XkcdRand};\n\n    use crate::{\n        corpus::{Corpus, InMemoryCorpus, Testcase},\n        feedbacks::ConstFeedback,\n        inputs::{BytesInput, HasBytesVec},\n        mutators::{\n            mutations::SpliceMutator,\n            scheduled::{havoc_mutations, StdScheduledMutator},\n            Mutator,\n        },\n        state::StdState,\n    };\n\n    #[test]\n    fn test_mut_scheduled() {\n        // With the current impl, seed of 1 will result in a split at pos 2.\n        let mut rand = XkcdRand::with_seed(5);\n        let mut corpus: InMemoryCorpus<BytesInput> = InMemoryCorpus::new();\n        corpus\n            .add(Testcase::new(vec![b'a', b'b', b'c'].into()))\n            .unwrap();\n        corpus\n            .add(Testcase::new(vec![b'd', b'e', b'f'].into()))\n            .unwrap();\n\n        let mut input = corpus.cloned_input_for_id(corpus.first().unwrap()).unwrap();\n\n        let mut feedback = ConstFeedback::new(false);\n        let mut objective = ConstFeedback::new(false);\n\n        let mut state = StdState::new(\n            rand,\n            corpus,\n            InMemoryCorpus::new(),\n            &mut feedback,\n            &mut objective,\n        )\n        .unwrap();\n\n        rand.set_seed(5);\n\n        let mut splice = SpliceMutator::new();\n        splice.mutate(&mut state, &mut input).unwrap();\n\n        log::trace!(\"{:?}\", input.bytes());\n\n        // The pre-seeded rand should have spliced at position 2.\n        // TODO: Maybe have a fixed rand for this purpose?\n        assert_eq!(input.bytes(), &[b'a', b'b', b'f']);\n    }\n\n    #[test]\n    fn test_havoc() {\n        // With the current impl, seed of 1 will result in a split at pos 2.\n        let rand = StdRand::with_seed(0x1337);\n        let mut corpus: InMemoryCorpus<BytesInput> = InMemoryCorpus::new();\n        corpus\n            .add(Testcase::new(vec![b'a', b'b', b'c'].into()))\n            .unwrap();\n        corpus\n            .add(Testcase::new(vec![b'd', b'e', b'f'].into()))\n            .unwrap();\n\n        let mut input = corpus.cloned_input_for_id(corpus.first().unwrap()).unwrap();\n        let input_prior = input.clone();\n\n        let mut feedback = ConstFeedback::new(false);\n        let mut objective = ConstFeedback::new(false);\n\n        let mut state = StdState::new(\n            rand,\n            corpus,\n            InMemoryCorpus::new(),\n            &mut feedback,\n            &mut objective,\n        )\n        .unwrap();\n\n        let mut havoc = StdScheduledMutator::new(havoc_mutations());\n\n        assert_eq!(input, input_prior);\n\n        let mut equal_in_a_row = 0;\n\n        for _ in 0..42 {\n            havoc.mutate(&mut state, &mut input).unwrap();\n\n            // Make sure we actually mutate something, at least sometimes\n            equal_in_a_row = if input == input_prior {\n                equal_in_a_row + 1\n            } else {\n                0\n            };\n            assert_ne!(equal_in_a_row, 5);\n        }\n    }\n}\n\n// Path: libafl/src/mutators/mopt_mutator.rs\n//! The `MOpt` mutator scheduler, see <https://github.com/puppet-meteor/MOpt-AFL> and <https://www.usenix.org/conference/usenixsecurity19/presentation/lyu>\nuse alloc::{\n    string::{String, ToString},\n    vec::Vec,\n};\nuse core::{\n    fmt::{self, Debug},\n    marker::PhantomData,\n};\n\nuse libafl_bolts::{\n    rands::{Rand, StdRand},\n    Named,\n};\nuse serde::{Deserialize, Serialize};\n\nuse super::MutationId;\nuse crate::{\n    corpus::{Corpus, CorpusId},\n    mutators::{ComposedByMutations, MutationResult, Mutator, MutatorsTuple, ScheduledMutator},\n    state::{HasCorpus, HasMetadata, HasRand, HasSolutions},\n    Error,\n};\n\n/// A Struct for managing MOpt-mutator parameters.\n/// There are 2 modes for `MOpt` scheduler, the core fuzzing mode and the pilot fuzzing mode.\n/// In short, in the pilot fuzzing mode, the fuzzer employs several `swarms` to compute the probability to choose the mutation operator.\n/// On the other hand, in the core fuzzing mode, the fuzzer chooses the best `swarms`, which was determined during the pilot fuzzing mode, to compute the probability to choose the operation operator.\n/// With the current implementation we are always in the pacemaker fuzzing mode.\n#[derive(Serialize, Deserialize, Clone)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct MOpt {\n    /// Random number generator\n    pub rand: StdRand,\n    /// The number of total findings (unique crashes and unique interesting paths). This is equivalent to `state.corpus().count() + state.solutions().count()`;\n    pub total_finds: usize,\n    /// The number of finds before until last swarm.\n    pub finds_until_last_swarm: usize,\n    /// These w_* and g_* values are the coefficients for updating variables according to the PSO algorithms\n    pub w_init: f64,\n    /// These w_* and g_* values are the coefficients for updating variables according to the PSO algorithms\n    pub w_end: f64,\n    /// These w_* and g_* values are the coefficients for updating variables according to the PSO algorithms\n    pub w_now: f64,\n    /// These w_* and g_* values are the coefficients for updating variables according to the PSO algorithms\n    pub g_now: f64,\n    /// These w_* and g_* values are the coefficients for updating variables according to the PSO algorithms\n    pub g_max: f64,\n    /// The number of mutation operators\n    pub operator_num: usize,\n    /// The number of swarms that we want to employ during the pilot fuzzing mode\n    pub swarm_num: usize,\n    /// We'll generate inputs for `period_pilot` times before we call `pso_update` in pilot fuzzing module\n    pub period_pilot: usize,\n    /// We'll generate inputs for `period_core` times before we call `pso_update` in core fuzzing module\n    pub period_core: usize,\n    /// The number of testcases generated during this pilot fuzzing mode\n    pub pilot_time: usize,\n    /// The number of testcases generated during this core fuzzing mode\n    pub core_time: usize,\n    /// The swarm identifier that we are currently using in the pilot fuzzing mode\n    pub swarm_now: usize,\n    /// A parameter for the PSO algorithm\n    x_now: Vec<Vec<f64>>,\n    /// A parameter for the PSO algorithm\n    l_best: Vec<Vec<f64>>,\n    /// A parameter for the PSO algorithm\n    eff_best: Vec<Vec<f64>>,\n    /// A parameter for the PSO algorithm\n    g_best: Vec<f64>,\n    /// A parameter for the PSO algorithm\n    v_now: Vec<Vec<f64>>,\n    /// The probability that we want to use to choose the mutation operator.\n    probability_now: Vec<Vec<f64>>,\n    /// The fitness for each swarm, we'll calculate the fitness in the pilot fuzzing mode and use the best one in the core fuzzing mode\n    pub swarm_fitness: Vec<f64>,\n    /// (Pilot Mode) Finds by each operators. This vector is used in `pso_update`\n    pub pilot_operator_finds: Vec<Vec<u64>>,\n    /// (Pilot Mode) Finds by each operator till now.\n    pub pilot_operator_finds_v2: Vec<Vec<u64>>,\n    /// (Pilot Mode) The number of mutation operator used. This vector is used in `pso_update`\n    pub pilot_operator_cycles: Vec<Vec<u64>>,\n    /// (Pilot Mode) The number of mutation operator used till now\n    pub pilot_operator_cycles_v2: Vec<Vec<u64>>,\n    /// (Pilot Mode) The number of mutation operator used till last execution\n    pub pilot_operator_cycles_v3: Vec<Vec<u64>>,\n    /// Vector used in `pso_update`\n    pub operator_finds_puppet: Vec<u64>,\n    /// (Core Mode) Finds by each operators. This vector is used in `pso_update`\n    pub core_operator_finds: Vec<u64>,\n    /// (Core Mode) Finds by each operator till now.\n    pub core_operator_finds_v2: Vec<u64>,\n    /// (Core Mode) The number of mutation operator used. This vector is used in `pso_update`\n    pub core_operator_cycles: Vec<u64>,\n    /// (Core Mode) The number of mutation operator used till now\n    pub core_operator_cycles_v2: Vec<u64>,\n    /// (Core Mode) The number of mutation operator used till last execution\n    pub core_operator_cycles_v3: Vec<u64>,\n}\n\nlibafl_bolts::impl_serdeany!(MOpt);\n\nimpl Debug for MOpt {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"MOpt\")\n            .field(\"\\ntotal_finds\", &self.total_finds)\n            .field(\"\\nfinds_until_last_swarm\", &self.finds_until_last_swarm)\n            .field(\"\\nw_init\", &self.w_init)\n            .field(\"\\nw_end\", &self.w_end)\n            .field(\"\\nw_now\", &self.g_now)\n            .field(\"\\ng_now\", &self.g_max)\n            .field(\"\\npilot_time\", &self.pilot_time)\n            .field(\"\\ncore_time\", &self.core_time)\n            .field(\"\\n\\nx_now\", &self.x_now)\n            .field(\"\\n\\nl_best\", &self.l_best)\n            .field(\"\\n\\neff_best\", &self.eff_best)\n            .field(\"\\n\\ng_best\", &self.g_best)\n            .field(\"\\n\\nv_now\", &self.v_now)\n            .field(\"\\n\\nprobability_now\", &self.probability_now)\n            .field(\"\\n\\nswarm_fitness\", &self.swarm_fitness)\n            .field(\"\\n\\npilot_operator_finds\", &self.pilot_operator_finds)\n            .field(\n                \"\\n\\npilot_operator_finds_this\",\n                &self.pilot_operator_finds_v2,\n            )\n            .field(\"\\n\\npilot_operator_cycles\", &self.pilot_operator_cycles)\n            .field(\n                \"\\n\\npilot_operator_cycles_v2\",\n                &self.pilot_operator_cycles_v2,\n            )\n            .field(\n                \"\\n\\npilot_operator_cycles_v3\",\n                &self.pilot_operator_cycles_v3,\n            )\n            .field(\"\\n\\noperator_finds_puppuet\", &self.operator_finds_puppet)\n            .field(\"\\n\\ncore_operator_finds\", &self.core_operator_finds)\n            .field(\"\\n\\ncore_operator_finds_v2\", &self.core_operator_finds_v2)\n            .field(\"\\n\\ncore_operator_cycles\", &self.core_operator_cycles)\n            .field(\"\\n\\ncore_operator_cycles_v2\", &self.core_operator_cycles_v2)\n            .field(\"\\n\\ncore_operator_cycles_v3\", &self.core_operator_cycles_v3)\n            .finish_non_exhaustive()\n    }\n}\n\nconst PERIOD_PILOT_COEF: f64 = 5000.0;\n\nimpl MOpt {\n    /// Creates a new [`struct@MOpt`] instance.\n    pub fn new(operator_num: usize, swarm_num: usize, rand_seed: u64) -> Result<Self, Error> {\n        let mut mopt = Self {\n            rand: StdRand::with_seed(rand_seed),\n            total_finds: 0,\n            finds_until_last_swarm: 0,\n            w_init: 0.9,\n            w_end: 0.3,\n            w_now: 0.0,\n            g_now: 0.0,\n            g_max: 5000.0,\n            operator_num,\n            swarm_num,\n            period_pilot: 50000,\n            period_core: 500000,\n            pilot_time: 0,\n            core_time: 0,\n            swarm_now: 0,\n            x_now: vec![vec![0.0; operator_num]; swarm_num],\n            l_best: vec![vec![0.0; operator_num]; swarm_num],\n            eff_best: vec![vec![0.0; operator_num]; swarm_num],\n            g_best: vec![0.0; operator_num],\n            v_now: vec![vec![0.0; operator_num]; swarm_num],\n            probability_now: vec![vec![0.0; operator_num]; swarm_num],\n            swarm_fitness: vec![0.0; swarm_num],\n            pilot_operator_finds: vec![vec![0; operator_num]; swarm_num],\n            pilot_operator_finds_v2: vec![vec![0; operator_num]; swarm_num],\n            pilot_operator_cycles: vec![vec![0; operator_num]; swarm_num],\n            pilot_operator_cycles_v2: vec![vec![0; operator_num]; swarm_num],\n            pilot_operator_cycles_v3: vec![vec![0; operator_num]; swarm_num],\n            operator_finds_puppet: vec![0; operator_num],\n            core_operator_finds: vec![0; operator_num],\n            core_operator_finds_v2: vec![0; operator_num],\n            core_operator_cycles: vec![0; operator_num],\n            core_operator_cycles_v2: vec![0; operator_num],\n            core_operator_cycles_v3: vec![0; operator_num],\n        };\n        mopt.pso_initialize()?;\n        Ok(mopt)\n    }\n\n    /// initialize pso\n    #[allow(clippy::cast_precision_loss)]\n    pub fn pso_initialize(&mut self) -> Result<(), Error> {\n        if self.g_now > self.g_max {\n            self.g_now = 0.0;\n        }\n        self.w_now =\n            (self.w_init - self.w_end) * (self.g_max - self.g_now) / self.g_max + self.w_end;\n\n        for swarm in 0..self.swarm_num {\n            let mut total_x_now = 0.0;\n            let mut x_sum = 0.0;\n            for i in 0..self.operator_num {\n                self.x_now[swarm][i] = (self.rand.below(7000) as f64) * 0.0001 + 0.1;\n                total_x_now += self.x_now[swarm][i];\n                self.v_now[swarm][i] = 0.1;\n                self.l_best[swarm][i] = 0.5;\n                self.g_best[i] = 0.5;\n            }\n\n            for i in 0..self.operator_num {\n                self.x_now[swarm][i] /= total_x_now;\n            }\n\n            for i in 0..self.operator_num {\n                self.v_now[swarm][i] = self.w_now * self.v_now[swarm][i]\n                    + (self.rand.below(1000) as f64)\n                        * 0.001\n                        * (self.l_best[swarm][i] - self.x_now[swarm][i])\n                    + (self.rand.below(1000) as f64)\n                        * 0.001\n                        * (self.g_best[i] - self.x_now[swarm][i]);\n                self.x_now[swarm][i] += self.v_now[swarm][i];\n\n                self.x_now[swarm][i] = self.x_now[swarm][i].clamp(V_MIN, V_MAX);\n\n                x_sum += self.x_now[swarm][i];\n            }\n\n            for i in 0..self.operator_num {\n                self.x_now[swarm][i] /= x_sum;\n                if i == 0 {\n                    self.probability_now[swarm][i] = self.x_now[swarm][i];\n                } else {\n                    self.probability_now[swarm][i] =\n                        self.probability_now[swarm][i - 1] + self.x_now[swarm][i];\n                }\n            }\n            if self.probability_now[swarm][self.operator_num - 1] < 0.99\n                || self.probability_now[swarm][self.operator_num - 1] > 1.01\n            {\n                return Err(Error::illegal_state(\n                    \"MOpt: Error in pso_update\".to_string(),\n                ));\n            }\n        }\n        Ok(())\n    }\n\n    /// Update the `PSO` algorithm parameters\n    /// See <https://github.com/puppet-meteor/MOpt-AFL/blob/master/MOpt/afl-fuzz.c#L10623>\n    #[allow(clippy::cast_precision_loss)]\n    pub fn pso_update(&mut self) -> Result<(), Error> {\n        self.g_now += 1.0;\n        if self.g_now > self.g_max {\n            self.g_now = 0.0;\n        }\n        self.w_now =\n            ((self.w_init - self.w_end) * (self.g_max - self.g_now) / self.g_max) + self.w_end;\n\n        let mut operator_finds_sum = 0;\n\n        for i in 0..self.operator_num {\n            self.operator_finds_puppet[i] = self.core_operator_finds[i];\n\n            for j in 0..self.swarm_num {\n                self.operator_finds_puppet[i] += self.pilot_operator_finds[j][i];\n            }\n            operator_finds_sum += self.operator_finds_puppet[i];\n        }\n\n        for i in 0..self.operator_num {\n            if self.operator_finds_puppet[i] > 0 {\n                self.g_best[i] =\n                    (self.operator_finds_puppet[i] as f64) / (operator_finds_sum as f64);\n            }\n        }\n\n        for swarm in 0..self.swarm_num {\n            let mut x_sum = 0.0;\n            for i in 0..self.operator_num {\n                self.probability_now[swarm][i] = 0.0;\n                self.v_now[swarm][i] = self.w_now * self.v_now[swarm][i]\n                    + (self.rand.below(1000) as f64)\n                        * 0.001\n                        * (self.l_best[swarm][i] - self.x_now[swarm][i])\n                    + (self.rand.below(1000) as f64)\n                        * 0.001\n                        * (self.g_best[i] - self.x_now[swarm][i]);\n                self.x_now[swarm][i] += self.v_now[swarm][i];\n\n                self.x_now[swarm][i] = self.x_now[swarm][i].clamp(V_MIN, V_MAX);\n\n                x_sum += self.x_now[swarm][i];\n            }\n\n            for i in 0..self.operator_num {\n                self.x_now[swarm][i] /= x_sum;\n                if i == 0 {\n                    self.probability_now[swarm][i] = self.x_now[swarm][i];\n                } else {\n                    self.probability_now[swarm][i] =\n                        self.probability_now[swarm][i - 1] + self.x_now[swarm][i];\n                }\n            }\n            if self.probability_now[swarm][self.operator_num - 1] < 0.99\n                || self.probability_now[swarm][self.operator_num - 1] > 1.01\n            {\n                return Err(Error::illegal_state(\n                    \"MOpt: Error in pso_update\".to_string(),\n                ));\n            }\n        }\n        self.swarm_now = 0;\n\n        // After pso_update, go back to pilot-fuzzing module\n        Ok(())\n    }\n\n    /// This function is used to decide the operator that we want to apply next\n    /// see <https://github.com/puppet-meteor/MOpt-AFL/blob/master/MOpt/afl-fuzz.c#L397>\n    #[allow(clippy::cast_precision_loss)]\n    pub fn select_algorithm(&mut self) -> Result<MutationId, Error> {\n        let mut res = 0;\n        let mut sentry = 0;\n\n        let operator_num = self.operator_num;\n\n        // Fetch a random sele value\n        let select_prob: f64 = self.probability_now[self.swarm_now][operator_num - 1]\n            * ((self.rand.below(10000) as f64) * 0.0001);\n\n        for i in 0..operator_num {\n            if i == 0 {\n                if select_prob < self.probability_now[self.swarm_now][i] {\n                    res = i;\n                    break;\n                }\n            } else if select_prob < self.probability_now[self.swarm_now][i] {\n                res = i;\n                sentry = 1;\n                break;\n            }\n        }\n\n        if (sentry == 1 && select_prob < self.probability_now[self.swarm_now][res - 1])\n            || (res + 1 < operator_num\n                && select_prob > self.probability_now[self.swarm_now][res + 1])\n        {\n            return Err(Error::illegal_state(\n                \"MOpt: Error in select_algorithm\".to_string(),\n            ));\n        }\n        Ok(res.into())\n    }\n}\n\nconst V_MAX: f64 = 1.0;\nconst V_MIN: f64 = 0.05;\n\n/// The `MOpt` mode to use\n#[derive(Serialize, Deserialize, Clone, Copy, Debug)]\npub enum MOptMode {\n    /// Pilot fuzzing mode\n    Pilotfuzzing,\n    /// Core fuzzing mode\n    Corefuzzing,\n}\n\n/// This is the main struct of `MOpt`, an `AFL` mutator.\n/// See the original `MOpt` implementation in <https://github.com/puppet-meteor/MOpt-AFL>\npub struct StdMOptMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand + HasMetadata + HasCorpus + HasSolutions,\n{\n    name: String,\n    mode: MOptMode,\n    finds_before: usize,\n    mutations: MT,\n    max_stack_pow: u64,\n    phantom: PhantomData<(I, S)>,\n}\n\nimpl<I, MT, S> Debug for StdMOptMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand + HasMetadata + HasCorpus + HasSolutions,\n{\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(\n            f,\n            \"StdMOptMutator with {} mutations for Input type {}\",\n            self.mutations.len(),\n            core::any::type_name::<I>()\n        )\n    }\n}\n\nimpl<I, MT, S> Mutator<I, S> for StdMOptMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand + HasMetadata + HasCorpus + HasSolutions,\n{\n    #[inline]\n    fn mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        self.finds_before = state.corpus().count() + state.solutions().count();\n        self.scheduled_mutate(state, input)\n    }\n\n    #[allow(clippy::cast_precision_loss)]\n    fn post_exec(&mut self, state: &mut S, _new_corpus_idx: Option<CorpusId>) -> Result<(), Error> {\n        let before = self.finds_before;\n        let after = state.corpus().count() + state.solutions().count();\n\n        let mopt = state.metadata_map_mut().get_mut::<MOpt>().unwrap();\n        let key_module = self.mode;\n        match key_module {\n            MOptMode::Corefuzzing => {\n                mopt.core_time += 1;\n\n                if after > before {\n                    let diff = after - before;\n                    mopt.total_finds += diff;\n                    for i in 0..mopt.operator_num {\n                        if mopt.core_operator_cycles_v2[i] > mopt.core_operator_cycles_v3[i] {\n                            mopt.core_operator_finds_v2[i] += diff as u64;\n                        }\n                    }\n                }\n\n                if mopt.core_time > mopt.period_core {\n                    mopt.core_time = 0;\n                    let total_finds = mopt.total_finds;\n                    mopt.finds_until_last_swarm = total_finds;\n                    for i in 0..mopt.operator_num {\n                        mopt.core_operator_finds[i] = mopt.core_operator_finds_v2[i];\n                        mopt.core_operator_cycles[i] = mopt.core_operator_cycles_v2[i];\n                    }\n                    mopt.pso_update()?;\n                    self.mode = MOptMode::Pilotfuzzing;\n                }\n            }\n            MOptMode::Pilotfuzzing => {\n                mopt.pilot_time += 1;\n                let swarm_now = mopt.swarm_now;\n\n                if after > before {\n                    let diff = after - before;\n                    mopt.total_finds += diff;\n                    for i in 0..mopt.operator_num {\n                        if mopt.pilot_operator_cycles_v2[swarm_now][i]\n                            > mopt.pilot_operator_cycles_v3[swarm_now][i]\n                        {\n                            mopt.pilot_operator_finds_v2[swarm_now][i] += diff as u64;\n                        }\n                    }\n                }\n\n                #[allow(clippy::cast_lossless)]\n                if mopt.pilot_time > mopt.period_pilot {\n                    let new_finds = mopt.total_finds - mopt.finds_until_last_swarm;\n                    let f = (new_finds as f64) / ((mopt.pilot_time as f64) / (PERIOD_PILOT_COEF));\n                    mopt.swarm_fitness[swarm_now] = f;\n                    mopt.pilot_time = 0;\n                    let total_finds = mopt.total_finds;\n                    mopt.finds_until_last_swarm = total_finds;\n\n                    for i in 0..mopt.operator_num {\n                        let mut eff = 0.0;\n                        if mopt.pilot_operator_cycles_v2[swarm_now][i]\n                            > mopt.pilot_operator_cycles[swarm_now][i]\n                        {\n                            eff = ((mopt.pilot_operator_finds_v2[swarm_now][i]\n                                - mopt.pilot_operator_finds[swarm_now][i])\n                                as f64)\n                                / ((mopt.pilot_operator_cycles_v2[swarm_now][i]\n                                    - mopt.pilot_operator_cycles[swarm_now][i])\n                                    as f64);\n                        }\n\n                        if mopt.eff_best[swarm_now][i] < eff {\n                            mopt.eff_best[swarm_now][i] = eff;\n                            mopt.l_best[swarm_now][i] = mopt.x_now[swarm_now][i];\n                        }\n\n                        mopt.pilot_operator_finds[swarm_now][i] =\n                            mopt.pilot_operator_finds_v2[swarm_now][i];\n                        mopt.pilot_operator_cycles[swarm_now][i] =\n                            mopt.pilot_operator_cycles_v2[swarm_now][i];\n                    }\n\n                    mopt.swarm_now += 1;\n\n                    if mopt.swarm_num == 1 {\n                        // If there's only 1 swarm, then no core_fuzzing mode.\n                        mopt.pso_update()?;\n                    } else if mopt.swarm_now == mopt.swarm_num {\n                        self.mode = MOptMode::Corefuzzing;\n\n                        for i in 0..mopt.operator_num {\n                            mopt.core_operator_cycles_v2[i] = mopt.core_operator_cycles[i];\n                            mopt.core_operator_cycles_v3[i] = mopt.core_operator_cycles[i];\n                            mopt.core_operator_finds_v2[i] = mopt.core_operator_finds[i];\n                        }\n\n                        let mut swarm_eff = 0.0;\n                        let mut best_swarm = 0;\n                        for i in 0..mopt.swarm_num {\n                            if mopt.swarm_fitness[i] > swarm_eff {\n                                swarm_eff = mopt.swarm_fitness[i];\n                                best_swarm = i;\n                            }\n                        }\n\n                        mopt.swarm_now = best_swarm;\n                    }\n                }\n            }\n        }\n        Ok(())\n    }\n}\n\nimpl<I, MT, S> StdMOptMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand + HasMetadata + HasCorpus + HasSolutions,\n{\n    /// Create a new [`StdMOptMutator`].\n    pub fn new(\n        state: &mut S,\n        mutations: MT,\n        max_stack_pow: u64,\n        swarm_num: usize,\n    ) -> Result<Self, Error> {\n        if !state.has_metadata::<MOpt>() {\n            let rand_seed = state.rand_mut().next();\n            state.add_metadata::<MOpt>(MOpt::new(mutations.len(), swarm_num, rand_seed)?);\n        }\n        Ok(Self {\n            name: format!(\"StdMOptMutator[{}]\", mutations.names().join(\",\")),\n            mode: MOptMode::Pilotfuzzing,\n            finds_before: 0,\n            mutations,\n            max_stack_pow,\n            phantom: PhantomData,\n        })\n    }\n    fn core_mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        let mut r = MutationResult::Skipped;\n        let mopt = state.metadata_map_mut().get_mut::<MOpt>().unwrap();\n        for i in 0..mopt.operator_num {\n            mopt.core_operator_cycles_v3[i] = mopt.core_operator_cycles_v2[i];\n        }\n\n        for _i in 0..self.iterations(state, input) {\n            let idx = self.schedule(state, input);\n            let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?;\n            if outcome == MutationResult::Mutated {\n                r = MutationResult::Mutated;\n            }\n\n            state\n                .metadata_map_mut()\n                .get_mut::<MOpt>()\n                .unwrap()\n                .core_operator_cycles_v2[idx.0] += 1;\n        }\n        Ok(r)\n    }\n\n    fn pilot_mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        let mut r = MutationResult::Skipped;\n        let swarm_now;\n        {\n            let mopt = state.metadata_map_mut().get_mut::<MOpt>().unwrap();\n            swarm_now = mopt.swarm_now;\n\n            for i in 0..mopt.operator_num {\n                mopt.pilot_operator_cycles_v3[swarm_now][i] =\n                    mopt.pilot_operator_cycles_v2[swarm_now][i];\n            }\n        }\n\n        for _i in 0..self.iterations(state, input) {\n            let idx = self.schedule(state, input);\n            let outcome = self.mutations_mut().get_and_mutate(idx, state, input)?;\n            if outcome == MutationResult::Mutated {\n                r = MutationResult::Mutated;\n            }\n\n            state\n                .metadata_map_mut()\n                .get_mut::<MOpt>()\n                .unwrap()\n                .pilot_operator_cycles_v2[swarm_now][idx.0] += 1;\n        }\n\n        Ok(r)\n    }\n}\n\nimpl<I, MT, S> ComposedByMutations<I, MT, S> for StdMOptMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand + HasMetadata + HasCorpus + HasSolutions,\n{\n    /// Get the mutations\n    #[inline]\n    fn mutations(&self) -> &MT {\n        &self.mutations\n    }\n\n    // Get the mutations (mutable)\n    #[inline]\n    fn mutations_mut(&mut self) -> &mut MT {\n        &mut self.mutations\n    }\n}\n\nimpl<I, MT, S> Named for StdMOptMutator<I, MT, S>\nwhere\n    MT: MutatorsTuple<I, S>,\n    S: HasRand + HasMetadata + HasCorpus + HasSolutions,\n{\n    fn name(&self) -> &str {\n...\n", "needle_token_start": 4003, "needle_token_end": 4095, "code_context_ntokens": 16248, "output": ["```rust\n/// Create a new [`StdScheduledMutator`] instance specifying mutations and the maximun number of iterations\npub fn with_max_stack_pow(mutations: MT, max_stack_pow: u64) -> Self {\n    StdScheduledMutator {\n        name: format!(\"StdScheduledMutator[{}]\", mutations.names().join(\", \")),\n        mutations,\n        max_stack_pow,\n        phantom: PhantomData,\n    }\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "objective_size", "language": "rust", "path": "libafl/src/monitors/mod.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To calculate the total number of objective elements across all client statistics.\n2. **Input**: None directly; it operates on the internal state of the monitor instance.\n3. **Output**: Returns the total count of objective elements as an unsigned 64-bit integer.\n4. **Procedure**: Iterates over each client's statistics within the monitor, summing up the objective sizes reported by each client to provide a cumulative total.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: libafl/src/monitors/multi.rs\n//! Monitor to display both cumulative and per-client monitor\n\nuse alloc::{string::String, vec::Vec};\nuse core::{\n    fmt::{Debug, Formatter, Write},\n    time::Duration,\n};\n\nuse libafl_bolts::{current_time, format_duration_hms, ClientId};\n\nuse super::Aggregator;\nuse crate::monitors::{ClientStats, Monitor};\n\n/// Tracking monitor during fuzzing and display both per-client and cumulative info.\n#[derive(Clone)]\npub struct MultiMonitor<F>\nwhere\n    F: FnMut(&str),\n{\n    print_fn: F,\n    start_time: Duration,\n    client_stats: Vec<ClientStats>,\n    aggregator: Aggregator,\n}\n\nimpl<F> Debug for MultiMonitor<F>\nwhere\n    F: FnMut(&str),\n{\n    fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result {\n        f.debug_struct(\"MultiMonitor\")\n            .field(\"start_time\", &self.start_time)\n            .field(\"client_stats\", &self.client_stats)\n...\n// Path: libafl/src/monitors/mod.rs\n//! Keep stats, and display them to the user. Usually used in a broker, or main node, of some sort.\n\npub mod multi;\npub use multi::MultiMonitor;\n\n#[cfg(all(feature = \"tui_monitor\", feature = \"std\"))]\n#[allow(missing_docs)]\npub mod tui;\n\n#[cfg(all(feature = \"prometheus_monitor\", feature = \"std\"))]\n#[allow(missing_docs)]\npub mod prometheus;\nuse alloc::string::ToString;\n\n#[cfg(all(feature = \"prometheus_monitor\", feature = \"std\"))]\npub use prometheus::PrometheusMonitor;\n#[cfg(feature = \"std\")]\npub mod disk;\nuse alloc::{fmt::Debug, string::String, vec::Vec};\nuse core::{fmt, fmt::Write, time::Duration};\n\n#[cfg(feature = \"std\")]\npub use disk::{OnDiskJSONMonitor, OnDiskTOMLMonitor};\nuse hashbrown::HashMap;\nuse libafl_bolts::{current_time, format_duration_hms, ClientId};\nuse serde::{Deserialize, Serialize};\n\n#[cfg(feature = \"afl_exec_sec\")]\nconst CLIENT_STATS_TIME_WINDOW_SECS: u64 = 5; // 5 seconds\n\n/// Definition of how we aggreate this across multiple clients\n#[derive(Serialize, Deserialize, Debug, Clone)]\npub enum AggregatorOps {\n    /// Do nothing\n    None,\n    /// Add stats up\n    Sum,\n    /// Average stats out\n    Avg,\n    /// Get the min\n    Min,\n    /// Get the max\n    Max,\n}\n\n/// The standard aggregator, plug this into the monitor to use\n#[derive(Serialize, Deserialize, Debug, Clone, Default)]\npub struct Aggregator {\n    // this struct could also have hashmap or vec for caching but for now i'll just keep it simple\n    // for example to calculate the sum you don't have to iterate over all clients (obviously)\n    aggregated: HashMap<String, UserStatsValue>,\n}\n\nimpl Aggregator {\n    /// constructor for this aggregator\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            aggregated: HashMap::new(),\n        }\n    }\n\n    /// takes the key and the ref to clients stats then aggregate them all.\n    fn aggregate(&mut self, name: &str, client_stats: &[ClientStats]) {\n        let mut gather = client_stats\n            .iter()\n            .filter_map(|client| client.user_monitor.get(name));\n\n        let gather_count = gather.clone().count();\n\n        let (mut init, op) = match gather.next() {\n            Some(x) => (x.value().clone(), x.aggregator_op().clone()),\n            _ => {\n                return;\n            }\n        };\n\n        for item in gather {\n            match op {\n                AggregatorOps::None => {\n                    // Nothing\n                    return;\n                }\n                AggregatorOps::Avg | AggregatorOps::Sum => {\n                    init = match init.stats_add(item.value()) {\n                        Some(x) => x,\n                        _ => {\n                            return;\n                        }\n                    };\n                }\n                AggregatorOps::Min => {\n                    init = match init.stats_min(item.value()) {\n                        Some(x) => x,\n                        _ => {\n                            return;\n                        }\n                    };\n                }\n                AggregatorOps::Max => {\n                    init = match init.stats_max(item.value()) {\n                        Some(x) => x,\n                        _ => {\n                            return;\n                        }\n                    };\n                }\n            }\n        }\n\n        if let AggregatorOps::Avg = op {\n            // if avg then divide last.\n            init = match init.stats_div(gather_count) {\n                Some(x) => x,\n                _ => {\n                    return;\n                }\n            }\n        }\n\n        self.aggregated.insert(name.to_string(), init);\n    }\n}\n\n/// user defined stats enum\n#[derive(Serialize, Deserialize, Debug, Clone)]\npub struct UserStats {\n    value: UserStatsValue,\n    aggregator_op: AggregatorOps,\n}\n\nimpl UserStats {\n    /// Get the `AggregatorOps`\n    #[must_use]\n    pub fn aggregator_op(&self) -> &AggregatorOps {\n        &self.aggregator_op\n    }\n    /// Get the actual value for the stats\n    #[must_use]\n    pub fn value(&self) -> &UserStatsValue {\n        &self.value\n    }\n    /// Constructor\n    #[must_use]\n    pub fn new(value: UserStatsValue, aggregator_op: AggregatorOps) -> Self {\n        Self {\n            value,\n            aggregator_op,\n        }\n    }\n}\n\n/// The actual value for the userstats\n#[derive(Serialize, Deserialize, Debug, Clone)]\npub enum UserStatsValue {\n    /// A numerical value\n    Number(u64),\n    /// A Float value\n    Float(f64),\n    /// A `String`\n    String(String),\n    /// A ratio of two values\n    Ratio(u64, u64),\n    /// Percent\n    Percent(f64),\n}\n\nimpl UserStatsValue {\n    /// Check if this guy is numeric\n    #[must_use]\n    pub fn is_numeric(&self) -> bool {\n        match &self {\n            Self::Number(_) | Self::Float(_) | Self::Ratio(_, _) | Self::Percent(_) => true,\n            Self::String(_) => false,\n        }\n    }\n\n    /// Divide by the number of elements\n    #[allow(clippy::cast_precision_loss)]\n    pub fn stats_div(&mut self, divisor: usize) -> Option<Self> {\n        match self {\n            Self::Number(x) => Some(Self::Float(*x as f64 / divisor as f64)),\n            Self::Float(x) => Some(Self::Float(*x / divisor as f64)),\n            Self::Percent(x) => Some(Self::Percent(*x / divisor as f64)),\n            Self::Ratio(x, y) => Some(Self::Percent((*x as f64 / divisor as f64) / *y as f64)),\n            Self::String(_) => None,\n        }\n    }\n\n    /// min user stats with the other\n    #[allow(clippy::cast_precision_loss)]\n    pub fn stats_max(&mut self, other: &Self) -> Option<Self> {\n        match (self, other) {\n            (Self::Number(x), Self::Number(y)) => {\n                if y > x {\n                    Some(Self::Number(*y))\n                } else {\n                    Some(Self::Number(*x))\n                }\n            }\n            (Self::Float(x), Self::Float(y)) => {\n                if y > x {\n                    Some(Self::Float(*y))\n                } else {\n                    Some(Self::Float(*x))\n                }\n            }\n            (Self::Ratio(x, a), Self::Ratio(y, b)) => {\n                let first = *x as f64 / *a as f64;\n                let second = *y as f64 / *b as f64;\n                if first > second {\n                    Some(Self::Percent(first))\n                } else {\n                    Some(Self::Percent(second))\n                }\n            }\n            (Self::Percent(x), Self::Percent(y)) => {\n                if y > x {\n                    Some(Self::Percent(*y))\n                } else {\n                    Some(Self::Percent(*x))\n                }\n            }\n            _ => None,\n        }\n    }\n\n    /// min user stats with the other\n    #[allow(clippy::cast_precision_loss)]\n    pub fn stats_min(&mut self, other: &Self) -> Option<Self> {\n        match (self, other) {\n            (Self::Number(x), Self::Number(y)) => {\n                if y > x {\n                    Some(Self::Number(*x))\n                } else {\n                    Some(Self::Number(*y))\n                }\n            }\n            (Self::Float(x), Self::Float(y)) => {\n                if y > x {\n                    Some(Self::Float(*x))\n                } else {\n                    Some(Self::Float(*y))\n                }\n            }\n            (Self::Ratio(x, a), Self::Ratio(y, b)) => {\n                let first = *x as f64 / *a as f64;\n                let second = *y as f64 / *b as f64;\n                if first > second {\n                    Some(Self::Percent(second))\n                } else {\n                    Some(Self::Percent(first))\n                }\n            }\n            (Self::Percent(x), Self::Percent(y)) => {\n                if y > x {\n                    Some(Self::Percent(*x))\n                } else {\n                    Some(Self::Percent(*y))\n                }\n            }\n            _ => None,\n        }\n    }\n\n    /// add user stats with the other\n    #[allow(clippy::cast_precision_loss)]\n    pub fn stats_add(&mut self, other: &Self) -> Option<Self> {\n        match (self, other) {\n            (Self::Number(x), Self::Number(y)) => Some(Self::Number(*x + *y)),\n            (Self::Float(x), Self::Float(y)) => Some(Self::Float(*x + *y)),\n            (Self::Percent(x), Self::Percent(y)) => Some(Self::Percent(*x + *y)),\n            (Self::Ratio(x, a), Self::Ratio(y, b)) => {\n                let first = *x as f64 / *a as f64;\n                let second = *y as f64 / *b as f64;\n                Some(Self::Percent(first + second))\n            }\n            (Self::Percent(x), Self::Ratio(y, b)) => {\n                let ratio = *y as f64 / *b as f64;\n                Some(Self::Percent(*x + ratio))\n            }\n            (Self::Ratio(x, a), Self::Percent(y)) => {\n                let ratio = *x as f64 / *a as f64;\n                Some(Self::Percent(ratio + *y))\n            }\n            _ => None,\n        }\n    }\n}\n\nimpl fmt::Display for UserStats {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"{}\", self.value())\n    }\n}\n\nimpl fmt::Display for UserStatsValue {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match &self {\n            UserStatsValue::Number(n) => write!(f, \"{n}\"),\n            UserStatsValue::Float(n) => write!(f, \"{}\", prettify_float(*n)),\n            UserStatsValue::Percent(n) => write!(f, \"{:.3}%\", n * 100.0),\n            UserStatsValue::String(s) => write!(f, \"{s}\"),\n            UserStatsValue::Ratio(a, b) => {\n                if *b == 0 {\n                    write!(f, \"{a}/{b}\")\n                } else {\n                    write!(f, \"{a}/{b} ({}%)\", a * 100 / b)\n                }\n            }\n        }\n    }\n}\n\n/// Prettifies float values for human-readable output\nfn prettify_float(value: f64) -> String {\n    let (value, suffix) = match value {\n        value if value >= 1000000.0 => (value / 1000000.0, \"M\"),\n        value if value >= 1000.0 => (value / 1000.0, \"k\"),\n        value => (value, \"\"),\n    };\n    match value {\n        value if value >= 1000.0 => {\n            format!(\"{value}{suffix}\")\n        }\n        value if value >= 100.0 => {\n            format!(\"{value:.1}{suffix}\")\n        }\n        value if value >= 10.0 => {\n            format!(\"{value:.2}{suffix}\")\n        }\n        value => {\n            format!(\"{value:.3}{suffix}\")\n        }\n    }\n}\n\n/// A simple struct to keep track of client monitor\n#[derive(Debug, Clone, Default, Serialize, Deserialize)]\npub struct ClientStats {\n    /// If this client is enabled. This is set to `true` the first time we see this client.\n    pub enabled: bool,\n    // monitor (maybe we need a separated struct?)\n    /// The corpus size for this client\n    pub corpus_size: u64,\n    /// The time for the last update of the corpus size\n    pub last_corpus_time: Duration,\n    /// The total executions for this client\n    pub executions: u64,\n    /// The number of executions of the previous state in case a client decrease the number of execution (e.g when restarting without saving the state)\n    pub prev_state_executions: u64,\n    /// The size of the objectives corpus for this client\n    pub objective_size: u64,\n    /// The time for the last update of the objective size\n    pub last_objective_time: Duration,\n    /// The last reported executions for this client\n    #[cfg(feature = \"afl_exec_sec\")]\n    pub last_window_executions: u64,\n    /// The last executions per sec\n    #[cfg(feature = \"afl_exec_sec\")]\n    pub last_execs_per_sec: f64,\n    /// The last time we got this information\n    pub last_window_time: Duration,\n    /// the start time of the client\n    pub start_time: Duration,\n    /// User-defined monitor\n    pub user_monitor: HashMap<String, UserStats>,\n    /// Client performance statistics\n    #[cfg(feature = \"introspection\")]\n    pub introspection_monitor: ClientPerfMonitor,\n}\n\nimpl ClientStats {\n    /// We got a new information about executions for this client, insert them.\n    #[cfg(feature = \"afl_exec_sec\")]\n    pub fn update_executions(&mut self, executions: u64, cur_time: Duration) {\n        let diff = cur_time\n            .checked_sub(self.last_window_time)\n            .map_or(0, |d| d.as_secs());\n        if diff > CLIENT_STATS_TIME_WINDOW_SECS {\n            let _: f64 = self.execs_per_sec(cur_time);\n            self.last_window_time = cur_time;\n            self.last_window_executions = self.executions;\n        }\n        if self.executions > self.prev_state_executions + executions {\n            // Something is strange here, sum the executions\n            self.prev_state_executions = self.executions;\n        }\n        self.executions = self.prev_state_executions + executions;\n    }\n\n    /// We got a new information about executions for this client, insert them.\n    #[cfg(not(feature = \"afl_exec_sec\"))]\n    pub fn update_executions(&mut self, executions: u64, _cur_time: Duration) {\n        if self.executions > self.prev_state_executions + executions {\n            // Something is strange here, sum the executions\n            self.prev_state_executions = self.executions;\n        }\n        self.executions = self.prev_state_executions + executions;\n    }\n\n    /// We got a new information about corpus size for this client, insert them.\n    pub fn update_corpus_size(&mut self, corpus_size: u64) {\n        self.corpus_size = corpus_size;\n        self.last_corpus_time = current_time();\n    }\n\n    /// We got a new information about objective corpus size for this client, insert them.\n    pub fn update_objective_size(&mut self, objective_size: u64) {\n        self.objective_size = objective_size;\n    }\n\n    /// Get the calculated executions per second for this client\n    #[allow(clippy::cast_precision_loss, clippy::cast_sign_loss)]\n    #[cfg(feature = \"afl_exec_sec\")]\n    pub fn execs_per_sec(&mut self, cur_time: Duration) -> f64 {\n        if self.executions == 0 {\n            return 0.0;\n        }\n\n        let elapsed = cur_time\n            .checked_sub(self.last_window_time)\n            .map_or(0.0, |d| d.as_secs_f64());\n        if elapsed as u64 == 0 {\n            return self.last_execs_per_sec;\n        }\n\n        let cur_avg = ((self.executions - self.last_window_executions) as f64) / elapsed;\n        if self.last_window_executions == 0 {\n            self.last_execs_per_sec = cur_avg;\n            return self.last_execs_per_sec;\n        }\n\n        // If there is a dramatic (5x+) jump in speed, reset the indicator more quickly\n        if cur_avg * 5.0 < self.last_execs_per_sec || cur_avg / 5.0 > self.last_execs_per_sec {\n            self.last_execs_per_sec = cur_avg;\n        }\n\n        self.last_execs_per_sec =\n            self.last_execs_per_sec * (1.0 - 1.0 / 16.0) + cur_avg * (1.0 / 16.0);\n        self.last_execs_per_sec\n    }\n\n    /// Get the calculated executions per second for this client\n    #[allow(clippy::cast_precision_loss, clippy::cast_sign_loss)]\n    #[cfg(not(feature = \"afl_exec_sec\"))]\n    pub fn execs_per_sec(&mut self, cur_time: Duration) -> f64 {\n        if self.executions == 0 {\n            return 0.0;\n        }\n\n        let elapsed = cur_time\n            .checked_sub(self.last_window_time)\n            .map_or(0.0, |d| d.as_secs_f64());\n        if elapsed as u64 == 0 {\n            return 0.0;\n        }\n\n        (self.executions as f64) / elapsed\n    }\n\n    /// Executions per second\n    fn execs_per_sec_pretty(&mut self, cur_time: Duration) -> String {\n        prettify_float(self.execs_per_sec(cur_time))\n    }\n\n    /// Update the user-defined stat with name and value\n    pub fn update_user_stats(&mut self, name: String, value: UserStats) -> Option<UserStats> {\n        self.user_monitor.insert(name, value)\n    }\n\n    #[must_use]\n    /// Get a user-defined stat using the name\n    pub fn get_user_stats(&self, name: &str) -> Option<&UserStats> {\n        self.user_monitor.get(name)\n    }\n\n    /// Update the current [`ClientPerfMonitor`] with the given [`ClientPerfMonitor`]\n    #[cfg(feature = \"introspection\")]\n    pub fn update_introspection_monitor(&mut self, introspection_monitor: ClientPerfMonitor) {\n        self.introspection_monitor = introspection_monitor;\n    }\n}\n\n/// The monitor trait keeps track of all the client's monitor, and offers methods to display them.\npub trait Monitor {\n    /// The client monitor (mutable)\n    fn client_stats_mut(&mut self) -> &mut Vec<ClientStats>;\n\n    /// The client monitor\n    fn client_stats(&self) -> &[ClientStats];\n\n    /// Creation time\n    fn start_time(&self) -> Duration;\n\n    /// Set creation time\n    fn set_start_time(&mut self, time: Duration);\n\n    /// Show the monitor to the user\n    fn display(&mut self, event_msg: &str, sender_id: ClientId);\n\n    /// Amount of elements in the corpus (combined for all children)\n    fn corpus_size(&self) -> u64 {\n        self.client_stats()\n            .iter()\n            .fold(0_u64, |acc, x| acc + x.corpus_size)\n    }\n\n    /// Count the number of enabled client stats\n    fn client_stats_count(&self) -> usize {\n        self.client_stats()\n            .iter()\n            .filter(|client| client.enabled)\n            .count()\n    }\n\n    /// Amount of elements in the objectives (combined for all children)\n    \nfn objective_size(&self) -> u64 {\n        self.client_stats()\n            .iter()\n            .fold(0_u64, |acc, x| acc + x.objective_size)\n    }\n\n    /// Total executions\n    #[inline]\n    fn total_execs(&self) -> u64 {\n        self.client_stats()\n            .iter()\n            .fold(0_u64, |acc, x| acc + x.executions)\n    }\n\n    /// Executions per second\n    #[allow(clippy::cast_sign_loss)]\n    #[inline]\n    fn execs_per_sec(&mut self) -> f64 {\n        let cur_time = current_time();\n        self.client_stats_mut()\n            .iter_mut()\n            .fold(0.0, |acc, x| acc + x.execs_per_sec(cur_time))\n    }\n\n    /// Executions per second\n    fn execs_per_sec_pretty(&mut self) -> String {\n        prettify_float(self.execs_per_sec())\n    }\n\n    /// The client monitor for a specific id, creating new if it doesn't exist\n    fn client_stats_insert(&mut self, client_id: ClientId) {\n        let total_client_stat_count = self.client_stats().len();\n        for _ in total_client_stat_count..=(client_id.0) as usize {\n            self.client_stats_mut().push(ClientStats {\n                enabled: false,\n                last_window_time: Duration::from_secs(0),\n                start_time: Duration::from_secs(0),\n                ..ClientStats::default()\n            });\n        }\n        let new_stat = self.client_stats_mut_for(client_id);\n        if !new_stat.enabled {\n            let timestamp = current_time();\n            // I have never seen this man in my life\n            new_stat.start_time = timestamp;\n            new_stat.last_window_time = timestamp;\n            new_stat.enabled = true;\n        }\n    }\n\n    /// Get mutable reference to client stats\n    fn client_stats_mut_for(&mut self, client_id: ClientId) -> &mut ClientStats {\n        &mut self.client_stats_mut()[client_id.0 as usize]\n    }\n\n    /// Get immutable reference to client stats\n    fn client_stats_for(&self, client_id: ClientId) -> &ClientStats {\n        &self.client_stats()[client_id.0 as usize]\n    }\n\n    /// Aggregate the results in case there're multiple clients\n    fn aggregate(&mut self, _name: &str) {}\n}\n\n/// Monitor that print exactly nothing.\n/// Not good for debuging, very good for speed.\n#[derive(Debug, Clone)]\npub struct NopMonitor {\n    start_time: Duration,\n    client_stats: Vec<ClientStats>,\n}\n\nimpl Monitor for NopMonitor {\n    /// The client monitor, mutable\n    fn client_stats_mut(&mut self) -> &mut Vec<ClientStats> {\n        &mut self.client_stats\n    }\n\n    /// The client monitor\n    fn client_stats(&self) -> &[ClientStats] {\n        &self.client_stats\n    }\n\n    /// Time this fuzzing run stated\n    fn start_time(&self) -> Duration {\n        self.start_time\n    }\n\n    /// Time this fuzzing run stated\n    fn set_start_time(&mut self, time: Duration) {\n        self.start_time = time;\n    }\n\n    #[inline]\n    fn display(&mut self, _event_msg: &str, _sender_id: ClientId) {}\n}\n\nimpl NopMonitor {\n    /// Create new [`NopMonitor`]\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            start_time: current_time(),\n            client_stats: vec![],\n        }\n    }\n}\n\nimpl Default for NopMonitor {\n    fn default() -> Self {\n        Self::new()\n    }\n}\n\n/// Tracking monitor during fuzzing that just prints to `stdout`.\n#[cfg(feature = \"std\")]\n#[derive(Debug, Clone)]\npub struct SimplePrintingMonitor {\n    start_time: Duration,\n    client_stats: Vec<ClientStats>,\n}\n\n#[cfg(feature = \"std\")]\nimpl Default for SimplePrintingMonitor {\n    fn default() -> Self {\n        Self {\n            start_time: current_time(),\n            client_stats: Vec::new(),\n        }\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl SimplePrintingMonitor {\n    /// Create a new [`SimplePrintingMonitor`]\n    #[must_use]\n    pub fn new() -> Self {\n        Self::default()\n    }\n}\n\n#[cfg(feature = \"std\")]\nimpl Monitor for SimplePrintingMonitor {\n    /// the client monitor, mutable\n    fn client_stats_mut(&mut self) -> &mut Vec<ClientStats> {\n        &mut self.client_stats\n    }\n\n    /// the client monitor\n    fn client_stats(&self) -> &[ClientStats] {\n        &self.client_stats\n    }\n\n    /// Time this fuzzing run stated\n    fn start_time(&self) -> Duration {\n        self.start_time\n    }\n\n    /// Time this fuzzing run stated\n    fn set_start_time(&mut self, time: Duration) {\n        self.start_time = time;\n    }\n\n    fn display(&mut self, event_msg: &str, sender_id: ClientId) {\n        let mut userstats = self.client_stats()[sender_id.0 as usize]\n            .user_monitor\n            .iter()\n            .map(|(key, value)| format!(\"{key}: {value}\"))\n            .collect::<Vec<_>>();\n        userstats.sort();\n        println!(\n            \"[{} #{}] run time: {}, clients: {}, corpus: {}, objectives: {}, executions: {}, exec/sec: {}, {}\",\n            event_msg,\n            sender_id.0,\n            format_duration_hms(&(current_time() - self.start_time)),\n            self.client_stats_count(),\n            self.corpus_size(),\n            self.objective_size(),\n            self.total_execs(),\n            self.execs_per_sec_pretty(),\n            userstats.join(\", \")\n        );\n\n        // Only print perf monitor if the feature is enabled\n        #[cfg(feature = \"introspection\")]\n        {\n            // Print the client performance monitor.\n            println!(\n                \"Client {:03}:\\n{}\",\n                sender_id.0, self.client_stats[sender_id.0 as usize].introspection_monitor\n            );\n            // Separate the spacing just a bit\n            println!();\n        }\n    }\n}\n\n/// Tracking monitor during fuzzing.\n#[derive(Clone)]\npub struct SimpleMonitor<F>\nwhere\n    F: FnMut(&str),\n{\n    print_fn: F,\n    start_time: Duration,\n    print_user_monitor: bool,\n    client_stats: Vec<ClientStats>,\n}\n\nimpl<F> Debug for SimpleMonitor<F>\nwhere\n    F: FnMut(&str),\n{\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"SimpleMonitor\")\n            .field(\"start_time\", &self.start_time)\n            .field(\"client_stats\", &self.client_stats)\n            .finish_non_exhaustive()\n    }\n}\n\nimpl<F> Monitor for SimpleMonitor<F>\nwhere\n    F: FnMut(&str),\n{\n    /// the client monitor, mutable\n    fn client_stats_mut(&mut self) -> &mut Vec<ClientStats> {\n        &mut self.client_stats\n    }\n\n    /// the client monitor\n    fn client_stats(&self) -> &[ClientStats] {\n        &self.client_stats\n    }\n\n    /// Time this fuzzing run stated\n    fn start_time(&self) -> Duration {\n        self.start_time\n    }\n\n    /// Set creation time\n    fn set_start_time(&mut self, time: Duration) {\n        self.start_time = time;\n    }\n\n    fn display(&mut self, event_msg: &str, sender_id: ClientId) {\n        let mut fmt = format!(\n            \"[{} #{}] run time: {}, clients: {}, corpus: {}, objectives: {}, executions: {}, exec/sec: {}\",\n            event_msg,\n            sender_id.0,\n            format_duration_hms(&(current_time() - self.start_time)),\n            self.client_stats_count(),\n            self.corpus_size(),\n            self.objective_size(),\n            self.total_execs(),\n            self.execs_per_sec_pretty()\n        );\n\n        if self.print_user_monitor {\n            self.client_stats_insert(sender_id);\n            let client = self.client_stats_mut_for(sender_id);\n            for (key, val) in &client.user_monitor {\n                write!(fmt, \", {key}: {val}\").unwrap();\n            }\n        }\n\n        (self.print_fn)(&fmt);\n\n        // Only print perf monitor if the feature is enabled\n        #[cfg(feature = \"introspection\")]\n        {\n            // Print the client performance monitor.\n            let fmt = format!(\n                \"Client {:03}:\\n{}\",\n                sender_id.0, self.client_stats[sender_id.0 as usize].introspection_monitor\n            );\n            (self.print_fn)(&fmt);\n\n            // Separate the spacing just a bit\n            (self.print_fn)(\"\");\n        }\n    }\n}\n\nimpl<F> SimpleMonitor<F>\nwhere\n    F: FnMut(&str),\n{\n    /// Creates the monitor, using the `current_time` as `start_time`.\n    pub fn new(print_fn: F) -> Self {\n        Self {\n            print_fn,\n            start_time: current_time(),\n            print_user_monitor: false,\n            client_stats: vec![],\n        }\n    }\n\n    /// Creates the monitor with a given `start_time`.\n    pub fn with_time(print_fn: F, start_time: Duration) -> Self {\n        Self {\n            print_fn,\n            start_time,\n            print_user_monitor: false,\n            client_stats: vec![],\n        }\n    }\n\n    /// Creates the monitor that also prints the user monitor\n    pub fn with_user_monitor(print_fn: F, print_user_monitor: bool) -> Self {\n        Self {\n            print_fn,\n            start_time: current_time(),\n            print_user_monitor,\n            client_stats: vec![],\n        }\n    }\n}\n\n/// Start the timer\n#[macro_export]\nmacro_rules! start_timer {\n    ($state:expr) => {{\n        // Start the timer\n        #[cfg(feature = \"introspection\")]\n        $state.introspection_monitor_mut().start_timer();\n    }};\n}\n\n/// Mark the elapsed time for the given feature\n#[macro_export]\nmacro_rules! mark_feature_time {\n    ($state:expr, $feature:expr) => {{\n        // Mark the elapsed time for the given feature\n        #[cfg(feature = \"introspection\")]\n        $state\n            .introspection_monitor_mut()\n            .mark_feature_time($feature);\n    }};\n}\n\n/// Mark the elapsed time for the given feature\n#[macro_export]\nmacro_rules! mark_feedback_time {\n    ($state:expr) => {{\n        // Mark the elapsed time for the given feature\n        #[cfg(feature = \"introspection\")]\n        $state.introspection_monitor_mut().mark_feedback_time();\n    }};\n}\n\n/// Client performance statistics\n#[derive(Serialize, Deserialize, Debug, Clone)]\npub struct ClientPerfMonitor {\n    /// Starting counter (in clock cycles from `read_time_counter`)\n    start_time: u64,\n\n    /// Current counter in the fuzzer (in clock cycles from `read_time_counter`\n    current_time: u64,\n\n    /// Clock cycles spent in the scheduler\n    scheduler: u64,\n\n    /// Clock cycles spent in the manager\n    manager: u64,\n\n    /// Current stage index to write the next stage benchmark time\n    curr_stage: u8,\n\n    /// Flag to dictate this stage is in use. Used during printing to not print the empty\n    /// stages if they are not in use.\n    stages_used: Vec<bool>,\n\n    /// Clock cycles spent in the the various features of each stage\n    stages: Vec<[u64; PerfFeature::Count as usize]>,\n\n    /// Clock cycles spent in each feedback mechanism of the fuzzer.\n    feedbacks: HashMap<String, u64>,\n\n    /// Current time set by `start_timer`\n    timer_start: Option<u64>,\n}\n\n#[derive(Debug, Clone, Serialize, Deserialize, Default)]\n/// Count the imported testcase from other nodes that came with observers\npub struct ScalabilityMonitor {\n    /// Imported testcase received with observer\n    pub testcase_with_observers: usize,\n    /// Imported testcase received without observer\n    pub testcase_without_observers: usize,\n}\n\nimpl ScalabilityMonitor {\n    /// Constructor\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            testcase_with_observers: 0,\n            testcase_without_observers: 0,\n        }\n    }\n}\n\n/// Various features that are measured for performance\n#[derive(Serialize, Deserialize, Debug, Clone)]\n#[repr(u8)]\npub enum PerfFeature {\n    /// Getting an input from the corpus\n    GetInputFromCorpus = 0,\n\n    /// Mutating the input\n    Mutate = 1,\n\n    /// Post-Exec Mutator callback\n    MutatePostExec = 2,\n\n    /// Actual time spent executing the target\n    TargetExecution = 3,\n\n    /// Time spent in `pre_exec`\n    PreExec = 4,\n\n    /// Time spent in `post_exec`\n    PostExec = 5,\n\n    /// Time spent in `observer` `pre_exec_all`\n    PreExecObservers = 6,\n\n    /// Time spent in `executor.observers_mut().post_exec_all`\n    PostExecObservers = 7,\n\n    /// Time spent getting the feedback from `is_interesting` from all feedbacks\n    GetFeedbackInterestingAll = 8,\n\n    /// Time spent getting the feedback from `is_interesting` from all objectives\n    GetObjectivesInterestingAll = 9,\n\n    /// Used as a counter to know how many elements are in [`PerfFeature`]. Must be the\n    /// last value in the enum.\n    Count, // !! No more values here since Count is last! !!\n           // !! No more values here since Count is last! !!\n}\n\n// TryFromPrimitive requires `std` so these are implemented manually\nimpl From<PerfFeature> for usize {\n    fn from(val: PerfFeature) -> usize {\n        match val {\n            PerfFeature::GetInputFromCorpus => PerfFeature::GetInputFromCorpus as usize,\n            PerfFeature::Mutate => PerfFeature::Mutate as usize,\n            PerfFeature::MutatePostExec => PerfFeature::MutatePostExec as usize,\n            PerfFeature::TargetExecution => PerfFeature::TargetExecution as usize,\n            PerfFeature::PreExec => PerfFeature::PreExec as usize,\n            PerfFeature::PostExec => PerfFeature::PostExec as usize,\n            PerfFeature::PreExecObservers => PerfFeature::PreExecObservers as usize,\n            PerfFeature::PostExecObservers => PerfFeature::PostExecObservers as usize,\n            PerfFeature::GetFeedbackInterestingAll => {\n                PerfFeature::GetFeedbackInterestingAll as usize\n            }\n            PerfFeature::GetObjectivesInterestingAll => {\n                PerfFeature::GetObjectivesInterestingAll as usize\n            }\n            PerfFeature::Count => PerfFeature::Count as usize,\n        }\n    }\n}\n\n// TryFromPrimitive requires `std` so these are implemented manually\nimpl From<usize> for PerfFeature {\n    fn from(val: usize) -> PerfFeature {\n        match val {\n            0 => PerfFeature::GetInputFromCorpus,\n            1 => PerfFeature::Mutate,\n            2 => PerfFeature::MutatePostExec,\n            3 => PerfFeature::TargetExecution,\n            4 => PerfFeature::PreExec,\n            5 => PerfFeature::PostExec,\n            6 => PerfFeature::PreExecObservers,\n            7 => PerfFeature::PostExecObservers,\n            8 => PerfFeature::GetFeedbackInterestingAll,\n            9 => PerfFeature::GetObjectivesInterestingAll,\n            _ => panic!(\"Unknown PerfFeature: {val}\"),\n        }\n    }\n}\n\n/// Number of features we can measure for performance\n#[cfg(feature = \"introspection\")]\npub const NUM_PERF_FEATURES: usize = PerfFeature::Count as usize;\n\n#[cfg(feature = \"introspection\")]\nimpl ClientPerfMonitor {\n    /// Create a blank [`ClientPerfMonitor`] with the `start_time` and `current_time` with\n    /// the current clock counter\n    #[must_use]\n    pub fn new() -> Self {\n        let start_time = libafl_bolts::cpu::read_time_counter();\n\n        Self {\n            start_time,\n            current_time: start_time,\n            scheduler: 0,\n            manager: 0,\n            curr_stage: 0,\n            stages: vec![],\n            stages_used: vec![],\n            feedbacks: HashMap::new(),\n            timer_start: None,\n        }\n    }\n\n    /// Set the current time with the given time\n    #[inline]\n    pub fn set_current_time(&mut self, time: u64) {\n        self.current_time = time;\n    }\n\n    /// Start a timer with the current time counter\n    #[inline]\n    pub fn start_timer(&mut self) {\n        self.timer_start = Some(libafl_bolts::cpu::read_time_counter());\n    }\n\n    /// Update the current [`ClientPerfMonitor`] with the given [`ClientPerfMonitor`]\n    pub fn update(&mut self, monitor: &ClientPerfMonitor) {\n        self.set_current_time(monitor.current_time);\n        self.update_scheduler(monitor.scheduler);\n        self.update_manager(monitor.manager);\n        self.update_stages(&monitor.stages);\n        self.update_feedbacks(&monitor.feedbacks);\n    }\n\n    /// Gets the elapsed time since the internal timer started. Resets the timer when\n    /// finished execution.\n    #[inline]\n    fn mark_time(&mut self) -> u64 {\n        match self.timer_start {\n            None => {\n                // Warning message if marking time without starting the timer first\n                log::warn!(\"Attempted to `mark_time` without starting timer first.\");\n\n                // Return 0 for no time marked\n                0\n            }\n            Some(timer_start) => {\n                // Calculate the elapsed time\n                let elapsed = libafl_bolts::cpu::read_time_counter() - timer_start;\n\n                // Reset the timer\n                self.timer_start = None;\n\n                // Return the elapsed time\n                elapsed\n            }\n        }\n    }\n\n    /// Update the time spent in the scheduler with the elapsed time that we have seen\n    #[inline]\n    pub fn mark_scheduler_time(&mut self) {\n        // Get the current elapsed time\n        let elapsed = self.mark_time();\n\n        // Add the time to the scheduler stat\n        self.update_scheduler(elapsed);\n    }\n\n    /// Update the time spent in the scheduler with the elapsed time that we have seen\n    #[inline]\n    pub fn mark_manager_time(&mut self) {\n        // Get the current elapsed time\n        let elapsed = self.mark_time();\n\n        // Add the time the manager stat\n        self.update_manager(elapsed);\n    }\n\n    /// Update the time spent in the given [`PerfFeature`] with the elapsed time that we have seen\n    #[inline]\n    pub fn mark_feature_time(&mut self, feature: PerfFeature) {\n        // Get the current elapsed time\n        let elapsed = self.mark_time();\n\n        // Add the time the the given feature\n        self.update_feature(feature, elapsed);\n    }\n\n    /// Add the given `time` to the `scheduler` monitor\n    #[inline]\n    pub fn update_scheduler(&mut self, time: u64) {\n        self.scheduler = self\n            .scheduler\n            .checked_add(time)\n            .expect(\"update_scheduler overflow\");\n    }\n\n    /// Add the given `time` to the `manager` monitor\n    #[inline]\n    pub fn update_manager(&mut self, time: u64) {\n        self.manager = self\n            .manager\n            .checked_add(time)\n            .expect(\"update_manager overflow\");\n    }\n\n    /// Update the total stage counter and increment the stage counter for the next stage\n    #[inline]\n    pub fn finish_stage(&mut self) {\n        // Increment the stage to the next index. The check is only done if this were to\n        // be used past the length of the `self.stages` buffer\n        self.curr_stage += 1;\n    }\n\n    /// Reset the stage index counter to zero\n    #[inline]\n    pub fn reset_stage_index(&mut self) {\n        self.curr_stage = 0;\n    }\n\n    /// Update the time spent in the feedback\n    pub fn update_feedback(&mut self, name: &str, time: u64) {\n        self.feedbacks.insert(\n            name.into(),\n            self.feedbacks\n                .get(name)\n                .unwrap_or(&0)\n                .checked_add(time)\n                .expect(\"update_feedback overflow\"),\n        );\n    }\n\n    /// Update the time spent in all the feedbacks\n    pub fn update_feedbacks(&mut self, feedbacks: &HashMap<String, u64>) {\n        for (key, value) in feedbacks {\n            self.update_feedback(key, *value);\n        }\n    }\n\n    /// Update the time spent in the stages\n    pub fn update_stages(&mut self, stages: &[[u64; PerfFeature::Count as usize]]) {\n        if self.stages.len() < stages.len() {\n            self.stages\n                .resize(stages.len(), [0; PerfFeature::Count as usize]);\n            self.stages_used.resize(stages.len(), false);\n        }\n        for (stage_index, features) in stages.iter().enumerate() {\n            for (feature_index, feature) in features.iter().enumerate() {\n                self.stages[stage_index][feature_index] = self.stages[stage_index][feature_index]\n                    .checked_add(*feature)\n                    .expect(\"Stage overflow\");\n            }\n        }\n    }\n\n    /// Update the given [`PerfFeature`] with the given `time`\n    pub fn update_feature(&mut self, feature: PerfFeature, time: u64) {\n        // Get the current stage index as `usize`\n        let stage_index: usize = self.curr_stage.into();\n\n        // Get the index of the given feature\n        let feature_index: usize = feature.into();\n\n        if stage_index >= self.stages.len() {\n            self.stages\n                .resize(stage_index + 1, [0; PerfFeature::Count as usize]);\n            self.stages_used.resize(stage_index + 1, false);\n        }\n\n        // Update the given feature\n        self.stages[stage_index][feature_index] = self.stages[stage_index][feature_index]\n            .checked_add(time)\n            .expect(\"Stage overflow\");\n\n        // Set that the current stage is being used\n        self.stages_used[stage_index] = true;\n    }\n\n    /// The elapsed cycles (or time)\n    #[must_use]\n    pub fn elapsed_cycles(&self) -> u64 {\n        self.current_time - self.start_time\n    }\n\n    /// The amount of cycles the `manager` did\n    #[must_use]\n    pub fn manager_cycles(&self) -> u64 {\n        self.manager\n    }\n\n    /// The amount of cycles the `scheduler` did\n    #[must_use]\n    pub fn scheduler_cycles(&self) -> u64 {\n        self.scheduler\n    }\n\n    /// Iterator over all used stages\n    pub fn used_stages(\n        &self,\n    ) -> impl Iterator<Item = (usize, &[u64; PerfFeature::Count as usize])> {\n        let used = self.stages_used.clone();\n        self.stages\n            .iter()\n            .enumerate()\n            .filter(move |(stage_index, _)| used[*stage_index])\n    }\n\n    /// A map of all `feedbacks`\n    #[must_use]\n    pub fn feedbacks(&self) -> &HashMap<String, u64> {\n        &self.feedbacks\n    }\n}\n\n#[cfg(feature = \"introspection\")]\nimpl fmt::Display for ClientPerfMonitor {\n    #[allow(clippy::cast_precision_loss)]\n    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {\n        // Calculate the elapsed time from the monitor\n        let elapsed: f64 = self.elapsed_cycles() as f64;\n\n        // Calculate the percentages for each benchmark\n        let scheduler_percent = self.scheduler as f64 / elapsed;\n        let manager_percent = self.manager as f64 / elapsed;\n\n        // Calculate the remaining percentage that has not been benchmarked\n        let mut other_percent = 1.0;\n        other_percent -= scheduler_percent;\n        other_percent -= manager_percent;\n\n        // Create the formatted string\n        writeln!(\n            f,\n            \"  {scheduler_percent:6.4}: Scheduler\\n  {manager_percent:6.4}: Manager\"\n        )?;\n\n        // Calculate each stage\n        // Make sure we only iterate over used stages\n        for (stage_index, features) in self.used_stages() {\n            // Write the stage header\n            writeln!(f, \"  Stage {stage_index}:\")?;\n\n            for (feature_index, feature) in features.iter().enumerate() {\n                // Calculate this current stage's percentage\n                let feature_percent = *feature as f64 / elapsed;\n\n                // Ignore this feature if it isn't used\n                if feature_percent == 0.0 {\n                    continue;\n                }\n\n                // Update the other percent by removing this current percent\n                other_percent -= feature_percent;\n\n                // Get the actual feature from the feature index for printing its name\n                let feature: PerfFeature = feature_index.into();\n\n                // Write the percentage for this feature\n                writeln!(f, \"    {feature_percent:6.4}: {feature:?}\")?;\n            }\n        }\n\n        writeln!(f, \"  Feedbacks:\")?;\n\n        for (feedback_name, feedback_time) in self.feedbacks() {\n            // Calculate this current stage's percentage\n            let feedback_percent = *feedback_time as f64 / elapsed;\n\n            // Ignore this feedback if it isn't used\n            if feedback_percent == 0.0 {\n                continue;\n            }\n\n            // Update the other percent by removing this current percent\n            other_percent -= feedback_percent;\n\n            // Write the percentage for this feedback\n            writeln!(f, \"    {feedback_percent:6.4}: {feedback_name}\")?;\n        }\n\n        write!(f, \"  {other_percent:6.4}: Not Measured\")?;\n\n        Ok(())\n    }\n}\n\n#[cfg(feature = \"introspection\")]\nimpl Default for ClientPerfMonitor {\n    #[must_use]\n    fn default() -> Self {\n        Self::new()\n    }\n}\n\n// Path: libafl/src/corpus/nop.rs\n//! The null corpus does not store any [`Testcase`]s.\nuse core::{cell::RefCell, marker::PhantomData};\n\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    corpus::{Corpus, CorpusId, Testcase},\n    inputs::{Input, UsesInput},\n    Error,\n};\n\n/// A corpus which does not store any [`Testcase`]s.\n#[derive(Default, Serialize, Deserialize, Clone, Debug)]\n#[serde(bound = \"I: serde::de::DeserializeOwned\")]\npub struct NopCorpus<I> {\n    empty: Option<CorpusId>,\n    phantom: PhantomData<I>,\n}\n\nimpl<I> UsesInput for NopCorpus<I>\nwhere\n    I: Input,\n{\n    type Input = I;\n}\n\nimpl<I> Corpus for NopCorpus<I>\nwhere\n    I: Input,\n{\n    /// Returns the number of elements\n    #[inline]\n    fn count(&self) -> usize {\n        0\n    }\n\n    /// Add an entry to the corpus and return its index\n    #[inline]\n    fn add(&mut self, _testcase: Testcase<I>) -> Result<CorpusId, Error> {\n        Err(Error::unsupported(\"Unsupported by NopCorpus\"))\n    }\n\n    /// Replaces the testcase at the given idx\n    #[inline]\n    fn replace(&mut self, _idx: CorpusId, _testcase: Testcase<I>) -> Result<Testcase<I>, Error> {\n        Err(Error::unsupported(\"Unsupported by NopCorpus\"))\n    }\n\n    /// Removes an entry from the corpus, returning it if it was present.\n    #[inline]\n    fn remove(&mut self, _idx: CorpusId) -> Result<Testcase<I>, Error> {\n        Err(Error::unsupported(\"Unsupported by NopCorpus\"))\n    }\n\n    /// Get by id\n    #[inline]\n    fn get(&self, _idx: CorpusId) -> Result<&RefCell<Testcase<I>>, Error> {\n        Err(Error::unsupported(\"Unsupported by NopCorpus\"))\n    }\n\n    /// Current testcase scheduled\n    #[inline]\n    fn current(&self) -> &Option<CorpusId> {\n        &self.empty\n    }\n\n    /// Current testcase scheduled (mutable)\n    #[inline]\n    fn current_mut(&mut self) -> &mut Option<CorpusId> {\n        &mut self.empty\n    }\n\n    #[inline]\n    fn next(&self, _idx: CorpusId) -> Option<CorpusId> {\n        None\n    }\n\n    #[inline]\n    fn prev(&self, _idx: CorpusId) -> Option<CorpusId> {\n        None\n    }\n\n    #[inline]\n    fn first(&self) -> Option<CorpusId> {\n        None\n    }\n\n    #[inline]\n    fn last(&self) -> Option<CorpusId> {\n        None\n    }\n\n    #[inline]\n    fn nth(&self, _nth: usize) -> CorpusId {\n        CorpusId::from(0_usize)\n    }\n\n    #[inline]\n    fn load_input_into(&self, _testcase: &mut Testcase<Self::Input>) -> Result<(), Error> {\n        Err(Error::unsupported(\"Unsupported by NopCorpus\"))\n    }\n\n    #[inline]\n    fn store_input_from(&self, _testcase: &Testcase<Self::Input>) -> Result<(), Error> {\n        Err(Error::unsupported(\"Unsupported by NopCorpus\"))\n    }\n}\n\nimpl<I> NopCorpus<I>\nwhere\n    I: Input,\n{\n    /// Creates a new [`NopCorpus`].\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            empty: None,\n            phantom: PhantomData {},\n        }\n    }\n}\n\n// Path: libafl/src/corpus/inmemory_ondisk.rs\n//! The [`InMemoryOnDiskCorpus`] stores [`Testcase`]s to disk.\n//! Additionally, _all_ of them are kept in memory.\n//! For a lower memory footprint, consider using [`crate::corpus::CachedOnDiskCorpus`]\n//! which only stores a certain number of [`Testcase`]s and removes additional ones in a FIFO manner.\n\nuse alloc::string::String;\nuse core::{cell::RefCell, time::Duration};\n#[cfg(feature = \"std\")]\nuse std::{fs, fs::File, io::Write};\nuse std::{\n    fs::OpenOptions,\n    path::{Path, PathBuf},\n};\n\n#[cfg(feature = \"gzip\")]\nuse libafl_bolts::compress::GzipCompressor;\nuse libafl_bolts::serdeany::SerdeAnyMap;\nuse serde::{Deserialize, Serialize};\n\nuse super::{\n    ondisk::{OnDiskMetadata, OnDiskMetadataFormat},\n    HasTestcase,\n};\nuse crate::{\n    corpus::{Corpus, CorpusId, InMemoryCorpus, Testcase},\n    inputs::{Input, UsesInput},\n    state::HasMetadata,\n    Error,\n};\n\n/// The [`Testcase`] metadata that'll be stored to disk\n#[cfg(feature = \"std\")]\n#[derive(Debug, Serialize)]\npub struct InMemoryOnDiskMetadata<'a> {\n    metadata: &'a SerdeAnyMap,\n    exec_time: &'a Option<Duration>,\n    executions: &'a usize,\n}\n\n/// A corpus able to store [`Testcase`]s to disk, while also keeping all of them in memory.\n///\n/// Metadata is written to a `.<filename>.metadata` file in the same folder by default.\n#[cfg(feature = \"std\")]\n#[derive(Default, Serialize, Deserialize, Clone, Debug)]\n#[serde(bound = \"I: serde::de::DeserializeOwned\")]\npub struct InMemoryOnDiskCorpus<I>\nwhere\n    I: Input,\n{\n    inner: InMemoryCorpus<I>,\n    dir_path: PathBuf,\n    meta_format: Option<OnDiskMetadataFormat>,\n    prefix: Option<String>,\n    locking: bool,\n}\n\nimpl<I> UsesInput for InMemoryOnDiskCorpus<I>\nwhere\n    I: Input,\n{\n    type Input = I;\n}\n\nimpl<I> Corpus for InMemoryOnDiskCorpus<I>\nwhere\n    I: Input,\n{\n    /// Returns the number of elements\n    #[inline]\n    fn count(&self) -> usize {\n        self.inner.count()\n    }\n\n    /// Add an entry to the corpus and return its index\n    #[inline]\n    fn add(&mut self, testcase: Testcase<I>) -> Result<CorpusId, Error> {\n        let idx = self.inner.add(testcase)?;\n        let testcase = &mut self.get(idx).unwrap().borrow_mut();\n        self.save_testcase(testcase, idx)?;\n        *testcase.input_mut() = None;\n        Ok(idx)\n    }\n\n    /// Replaces the testcase at the given idx\n    #[inline]\n    fn replace(&mut self, idx: CorpusId, testcase: Testcase<I>) -> Result<Testcase<I>, Error> {\n        let entry = self.inner.replace(idx, testcase)?;\n        self.remove_testcase(&entry)?;\n        let testcase = &mut self.get(idx).unwrap().borrow_mut();\n        self.save_testcase(testcase, idx)?;\n        *testcase.input_mut() = None;\n        Ok(entry)\n    }\n\n    /// Removes an entry from the corpus, returning it if it was present.\n    #[inline]\n    fn remove(&mut self, idx: CorpusId) -> Result<Testcase<I>, Error> {\n        let entry = self.inner.remove(idx)?;\n        self.remove_testcase(&entry)?;\n        Ok(entry)\n    }\n\n    /// Get by id\n    #[inline]\n    fn get(&self, idx: CorpusId) -> Result<&RefCell<Testcase<I>>, Error> {\n        self.inner.get(idx)\n    }\n\n    /// Current testcase scheduled\n    #[inline]\n    fn current(&self) -> &Option<CorpusId> {\n        self.inner.current()\n    }\n\n    /// Current testcase scheduled (mutable)\n    #[inline]\n    fn current_mut(&mut self) -> &mut Option<CorpusId> {\n        self.inner.current_mut()\n    }\n\n    #[inline]\n    fn next(&self, idx: CorpusId) -> Option<CorpusId> {\n        self.inner.next(idx)\n    }\n\n    #[inline]\n    fn prev(&self, idx: CorpusId) -> Option<CorpusId> {\n        self.inner.prev(idx)\n    }\n\n    #[inline]\n    fn first(&self) -> Option<CorpusId> {\n        self.inner.first()\n    }\n\n    #[inline]\n    fn last(&self) -> Option<CorpusId> {\n        self.inner.last()\n    }\n\n    #[inline]\n    fn nth(&self, nth: usize) -> CorpusId {\n        self.inner.nth(nth)\n    }\n\n    fn load_input_into(&self, testcase: &mut Testcase<Self::Input>) -> Result<(), Error> {\n        if testcase.input_mut().is_none() {\n            let Some(file_path) = testcase.file_path().as_ref() else {\n                return Err(Error::illegal_argument(\n                    \"No file path set for testcase. Could not load inputs.\",\n                ));\n            };\n            let input = I::from_file(file_path)?;\n            testcase.set_input(input);\n        }\n        Ok(())\n    }\n\n    fn store_input_from(&self, testcase: &Testcase<Self::Input>) -> Result<(), Error> {\n        // Store the input to disk\n        let Some(file_path) = testcase.file_path() else {\n            return Err(Error::illegal_argument(\n                \"No file path set for testcase. Could not store input to disk.\",\n            ));\n        };\n        let Some(input) = testcase.input() else {\n            return Err(Error::illegal_argument(\n                \"No input available for testcase. Could not store anything.\",\n            ));\n        };\n        input.to_file(file_path)\n    }\n}\n\nimpl<I> HasTestcase for InMemoryOnDiskCorpus<I>\nwhere\n    I: Input,\n{\n    fn testcase(\n        &self,\n        id: CorpusId,\n    ) -> Result<core::cell::Ref<Testcase<<Self as UsesInput>::Input>>, Error> {\n        Ok(self.get(id)?.borrow())\n    }\n\n    fn testcase_mut(\n        &self,\n        id: CorpusId,\n    ) -> Result<core::cell::RefMut<Testcase<<Self as UsesInput>::Input>>, Error> {\n        Ok(self.get(id)?.borrow_mut())\n    }\n}\n\nimpl<I> InMemoryOnDiskCorpus<I>\nwhere\n    I: Input,\n{\n    /// Creates an [`InMemoryOnDiskCorpus`].\n    ///\n    /// This corpus stores all testcases to disk, and keeps all of them in memory, as well.\n    ///\n    /// By default, it stores metadata for each [`Testcase`] as prettified json.\n    /// Metadata will be written to a file named `.<testcase>.metadata`\n    /// The metadata may include objective reason, specific information for a fuzz job, and more.\n    ///\n    /// If you don't want metadata, use [`InMemoryOnDiskCorpus::no_meta`].\n    /// To pick a different metadata format, use [`InMemoryOnDiskCorpus::with_meta_format`].\n    ///\n    /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.\n    pub fn new<P>(dir_path: P) -> Result<Self, Error>\n    where\n        P: AsRef<Path>,\n    {\n        Self::_new(\n            dir_path.as_ref(),\n            Some(OnDiskMetadataFormat::JsonPretty),\n            None,\n            true,\n        )\n    }\n\n    /// Creates the [`InMemoryOnDiskCorpus`] specifying the format in which `Metadata` will be saved to disk.\n    ///\n    /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.\n    pub fn with_meta_format<P>(\n        dir_path: P,\n        meta_format: Option<OnDiskMetadataFormat>,\n    ) -> Result<Self, Error>\n    where\n        P: AsRef<Path>,\n    {\n...\n", "needle_token_start": 5658, "needle_token_end": 5711, "code_context_ntokens": 16186, "output": ["```rust\n/// Objective size\nfn objective_size(&self) -> u64 {\n    self.client_stats()\n        .iter()\n        .fold(0_u64, |acc, x| acc + x.objective_size)\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "get_iters_by_name", "language": "rust", "path": "libafl/src/stages/tuneable.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the number of iterations configured for a specific mutational stage identified by its name.\n2. **Input**: The function takes a state object that holds metadata and a string representing the name of the mutational stage.\n3. **Output**: Returns either the number of iterations set for the specified stage as an optional unsigned 64-bit integer or an error if the stage is not found.\n4. **Procedure**: The function accesses the metadata map from the state object using the provided name. If the metadata for the specified name exists, it extracts and returns the number of iterations; otherwise, it returns an error indicating the stage is not in use.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: libafl/src/stages/tracing.rs\n//! The tracing stage can trace the target and enrich a testcase with metadata, for example for `CmpLog`.\n\nuse core::{fmt::Debug, marker::PhantomData};\n\nuse libafl_bolts::Named;\n\nuse crate::{\n    executors::{Executor, HasObservers, ShadowExecutor},\n    mark_feature_time,\n    observers::ObserversTuple,\n    stages::{RetryRestartHelper, Stage},\n    start_timer,\n    state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasNamedMetadata, State, UsesState},\n    Error,\n};\n#[cfg(feature = \"introspection\")]\nuse crate::{monitors::PerfFeature, state::HasClientPerfMonitor};\n\n/// A stage that runs a tracer executor\n#[derive(Clone, Debug)]\npub struct TracingStage<EM, TE, Z> {\n    tracer_executor: TE,\n    max_retries: usize,\n    #[allow(clippy::type_complexity)]\n    phantom: PhantomData<(EM, TE, Z)>,\n}\n\nimpl<EM, TE, Z> UsesState for TracingStage<EM, TE, Z>\nwhere\n    TE: UsesState,\n{\n    type State = TE::State;\n}\n\nimpl<EM, TE, Z> TracingStage<EM, TE, Z>\nwhere\n    TE: Executor<EM, Z> + HasObservers,\n    TE::State: HasExecutions + HasCorpus + HasNamedMetadata,\n    EM: UsesState<State = TE::State>,\n    Z: UsesState<State = TE::State>,\n{\n    /// Perform tracing on the given `CorpusId`. Useful for if wrapping [`TracingStage`] with your\n    /// own stage and you need to manage [`super::NestedStageRestartHelper`] differently; see\n    /// [`super::ConcolicTracingStage`]'s implementation as an example of usage.\n    pub fn trace(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut TE::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        start_timer!(state);\n        let input = state.current_input_cloned()?;\n\n        mark_feature_time!(state, PerfFeature::GetInputFromCorpus);\n\n        start_timer!(state);\n        self.tracer_executor\n            .observers_mut()\n            .pre_exec_all(state, &input)?;\n        mark_feature_time!(state, PerfFeature::PreExecObservers);\n\n        start_timer!(state);\n        let exit_kind = self\n            .tracer_executor\n            .run_target(fuzzer, state, manager, &input)?;\n        mark_feature_time!(state, PerfFeature::TargetExecution);\n\n        *state.executions_mut() += 1;\n\n        start_timer!(state);\n        self.tracer_executor\n            .observers_mut()\n            .post_exec_all(state, &input, &exit_kind)?;\n        mark_feature_time!(state, PerfFeature::PostExecObservers);\n\n        Ok(())\n    }\n}\n\nimpl<E, EM, TE, Z> Stage<E, EM, Z> for TracingStage<EM, TE, Z>\nwhere\n    E: UsesState<State = TE::State>,\n    TE: Executor<EM, Z> + HasObservers,\n    TE::State: HasExecutions + HasCorpus + HasNamedMetadata,\n    EM: UsesState<State = TE::State>,\n    Z: UsesState<State = TE::State>,\n{\n    #[inline]\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        _executor: &mut E,\n        state: &mut TE::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        self.trace(fuzzer, state, manager)\n    }\n\n    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {\n        RetryRestartHelper::restart_progress_should_run(state, self, self.max_retries)\n    }\n\n    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {\n        RetryRestartHelper::clear_restart_progress(state, self)\n    }\n}\n\nimpl<EM, TE, Z> Named for TracingStage<EM, TE, Z> {\n    fn name(&self) -> &str {\n        \"TracingStage\"\n    }\n}\n\nimpl<EM, TE, Z> TracingStage<EM, TE, Z> {\n    /// Creates a new default stage\n    pub fn new(tracer_executor: TE) -> Self {\n        Self {\n            tracer_executor,\n            max_retries: 10,\n            phantom: PhantomData,\n        }\n    }\n\n    /// Specify how many times that this stage will try again to trace the input before giving up\n    /// and not processing the input again. 0 retries means that the trace will be tried only once.\n    #[must_use]\n    pub fn with_retries(mut self, retries: usize) -> Self {\n        self.max_retries = retries;\n        self\n    }\n\n    /// Gets the underlying tracer executor\n    pub fn executor(&self) -> &TE {\n        &self.tracer_executor\n    }\n\n    /// Gets the underlying tracer executor (mut)\n    pub fn executor_mut(&mut self) -> &mut TE {\n        &mut self.tracer_executor\n    }\n}\n\n/// A stage that runs the shadow executor using also the shadow observers\n#[derive(Clone, Debug)]\npub struct ShadowTracingStage<E, EM, SOT, Z> {\n    max_retries: usize,\n    #[allow(clippy::type_complexity)]\n    phantom: PhantomData<(E, EM, SOT, Z)>,\n}\n\nimpl<E, EM, SOT, Z> UsesState for ShadowTracingStage<E, EM, SOT, Z>\nwhere\n    E: UsesState,\n{\n    type State = E::State;\n}\n\nimpl<E, EM, SOT, Z> Named for ShadowTracingStage<E, EM, SOT, Z>\nwhere\n    E: UsesState,\n{\n    fn name(&self) -> &str {\n        \"ShadowTracingStage\"\n    }\n}\n\nimpl<E, EM, SOT, Z> Stage<ShadowExecutor<E, SOT>, EM, Z> for ShadowTracingStage<E, EM, SOT, Z>\nwhere\n    E: Executor<EM, Z> + HasObservers,\n    EM: UsesState<State = E::State>,\n    SOT: ObserversTuple<E::State>,\n    Z: UsesState<State = E::State>,\n    E::State: State + HasExecutions + HasCorpus + HasNamedMetadata + Debug,\n{\n    #[inline]\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut ShadowExecutor<E, SOT>,\n        state: &mut E::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        start_timer!(state);\n        let input = state.current_input_cloned()?;\n\n        mark_feature_time!(state, PerfFeature::GetInputFromCorpus);\n\n        start_timer!(state);\n        executor\n            .shadow_observers_mut()\n            .pre_exec_all(state, &input)?;\n        executor.observers_mut().pre_exec_all(state, &input)?;\n        mark_feature_time!(state, PerfFeature::PreExecObservers);\n\n        start_timer!(state);\n        let exit_kind = executor.run_target(fuzzer, state, manager, &input)?;\n        mark_feature_time!(state, PerfFeature::TargetExecution);\n\n        *state.executions_mut() += 1;\n\n        start_timer!(state);\n        executor\n            .shadow_observers_mut()\n            .post_exec_all(state, &input, &exit_kind)?;\n        executor\n            .observers_mut()\n            .post_exec_all(state, &input, &exit_kind)?;\n        mark_feature_time!(state, PerfFeature::PostExecObservers);\n\n        Ok(())\n    }\n\n    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {\n        RetryRestartHelper::restart_progress_should_run(state, self, self.max_retries)\n    }\n\n    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {\n        RetryRestartHelper::clear_restart_progress(state, self)\n    }\n}\n\nimpl<E, EM, SOT, Z> ShadowTracingStage<E, EM, SOT, Z>\nwhere\n    E: Executor<EM, Z> + HasObservers,\n    E::State: State + HasExecutions + HasCorpus,\n    EM: UsesState<State = E::State>,\n    SOT: ObserversTuple<E::State>,\n    Z: UsesState<State = E::State>,\n{\n    /// Creates a new default stage\n    pub fn new(_executor: &mut ShadowExecutor<E, SOT>) -> Self {\n        Self {\n            max_retries: 10,\n            phantom: PhantomData,\n        }\n    }\n\n    /// Specify how many times that this stage will try again to trace the input before giving up\n    /// and not processing the input again. 0 retries means that the trace will be tried only once.\n    #[must_use]\n...\n// Path: libafl/src/stages/logics.rs\n//! Stage wrappers that add logics to stage list\n\nuse core::marker::PhantomData;\n\nuse crate::{\n    stages::{HasCurrentStage, HasNestedStageStatus, Stage, StagesTuple},\n    state::UsesState,\n    Error,\n};\n\n/// Progress for nested stages. This merely enters/exits the inner stage's scope.\n#[derive(Debug)]\npub struct NestedStageRestartHelper;\n\nimpl NestedStageRestartHelper {\n    fn restart_progress_should_run<S, ST>(state: &mut S, _stage: &ST) -> Result<bool, Error>\n    where\n        S: HasNestedStageStatus,\n    {\n        state.enter_inner_stage()?;\n        Ok(true)\n    }\n\n    fn clear_restart_progress<S, ST>(state: &mut S, _stage: &ST) -> Result<(), Error>\n    where\n        S: HasNestedStageStatus,\n    {\n        state.exit_inner_stage()?;\n        Ok(())\n    }\n}\n\n#[derive(Debug)]\n/// Perform the stage while the closure evaluates to true\npub struct WhileStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    closure: CB,\n    stages: ST,\n    phantom: PhantomData<(E, EM, Z)>,\n}\n\nimpl<CB, E, EM, ST, Z> UsesState for WhileStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    type State = E::State;\n}\n\nimpl<CB, E, EM, ST, Z> Stage<E, EM, Z> for WhileStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n    E::State: HasNestedStageStatus,\n{\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut E::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        while state.current_stage()?.is_some() || (self.closure)(fuzzer, executor, state, manager)?\n        {\n            self.stages.perform_all(fuzzer, executor, state, manager)?;\n        }\n\n        Ok(())\n    }\n\n    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {\n        NestedStageRestartHelper::restart_progress_should_run(state, self)\n    }\n\n    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {\n        NestedStageRestartHelper::clear_restart_progress(state, self)\n    }\n}\n\nimpl<CB, E, EM, ST, Z> WhileStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    /// Constructor\n    pub fn new(closure: CB, stages: ST) -> Self {\n        Self {\n            closure,\n            stages,\n            phantom: PhantomData,\n        }\n    }\n}\n\n/// A conditionally enabled stage.\n/// If the closure returns true, the wrapped stage will be executed, else it will be skipped.\n#[derive(Debug)]\npub struct IfStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    closure: CB,\n    if_stages: ST,\n    phantom: PhantomData<(E, EM, Z)>,\n}\n\nimpl<CB, E, EM, ST, Z> UsesState for IfStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    type State = E::State;\n}\n\nimpl<CB, E, EM, ST, Z> Stage<E, EM, Z> for IfStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n    E::State: HasNestedStageStatus,\n{\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut E::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        if state.current_stage()?.is_some() || (self.closure)(fuzzer, executor, state, manager)? {\n            self.if_stages\n                .perform_all(fuzzer, executor, state, manager)?;\n        }\n        Ok(())\n    }\n\n    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {\n        NestedStageRestartHelper::restart_progress_should_run(state, self)\n    }\n\n    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {\n        NestedStageRestartHelper::clear_restart_progress(state, self)\n    }\n}\n\nimpl<CB, E, EM, ST, Z> IfStage<CB, E, EM, ST, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    /// Constructor for this conditionally enabled stage.\n    /// If the closure returns true, the wrapped stage will be executed, else it will be skipped.\n    pub fn new(closure: CB, if_stages: ST) -> Self {\n        Self {\n            closure,\n            if_stages,\n            phantom: PhantomData,\n        }\n    }\n}\n\n/// Perform the stage if closure evaluates to true\n#[derive(Debug)]\npub struct IfElseStage<CB, E, EM, ST1, ST2, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST1: StagesTuple<E, EM, E::State, Z>,\n    ST2: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    closure: CB,\n    if_stages: ST1,\n    else_stages: ST2,\n    phantom: PhantomData<(E, EM, Z)>,\n}\n\nimpl<CB, E, EM, ST1, ST2, Z> UsesState for IfElseStage<CB, E, EM, ST1, ST2, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST1: StagesTuple<E, EM, E::State, Z>,\n    ST2: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    type State = E::State;\n}\n\nimpl<CB, E, EM, ST1, ST2, Z> Stage<E, EM, Z> for IfElseStage<CB, E, EM, ST1, ST2, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST1: StagesTuple<E, EM, E::State, Z>,\n    ST2: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n    E::State: HasNestedStageStatus,\n{\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut E::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        let current = state.current_stage()?;\n\n        let fresh = current.is_none();\n        let closure_return = fresh && (self.closure)(fuzzer, executor, state, manager)?;\n\n        if current == Some(0) || closure_return {\n            if fresh {\n                state.set_stage(0)?;\n            }\n            state.enter_inner_stage()?;\n            self.if_stages\n                .perform_all(fuzzer, executor, state, manager)?;\n        } else {\n            if fresh {\n                state.set_stage(1)?;\n            }\n            state.enter_inner_stage()?;\n            self.else_stages\n                .perform_all(fuzzer, executor, state, manager)?;\n        }\n\n        state.exit_inner_stage()?;\n        state.clear_stage()?;\n\n        Ok(())\n    }\n\n    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {\n        NestedStageRestartHelper::restart_progress_should_run(state, self)\n    }\n\n    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {\n        NestedStageRestartHelper::clear_restart_progress(state, self)\n    }\n}\n\nimpl<CB, E, EM, ST1, ST2, Z> IfElseStage<CB, E, EM, ST1, ST2, Z>\nwhere\n    CB: FnMut(&mut Z, &mut E, &mut E::State, &mut EM) -> Result<bool, Error>,\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST1: StagesTuple<E, EM, E::State, Z>,\n    ST2: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    /// Constructor\n    pub fn new(closure: CB, if_stages: ST1, else_stages: ST2) -> Self {\n        Self {\n            closure,\n            if_stages,\n            else_stages,\n            phantom: PhantomData,\n        }\n    }\n}\n\n/// A stage wrapper where the stages do not need to be initialized, but can be [`None`].\n#[derive(Debug)]\npub struct OptionalStage<E, EM, ST, Z>\nwhere\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    stages: Option<ST>,\n    phantom: PhantomData<(E, EM, Z)>,\n}\n\nimpl<E, EM, ST, Z> UsesState for OptionalStage<E, EM, ST, Z>\nwhere\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    type State = E::State;\n}\n\nimpl<E, EM, ST, Z> Stage<E, EM, Z> for OptionalStage<E, EM, ST, Z>\nwhere\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n    E::State: HasNestedStageStatus,\n{\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut E::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        if let Some(stages) = &mut self.stages {\n            stages.perform_all(fuzzer, executor, state, manager)\n        } else {\n            Ok(())\n        }\n    }\n\n    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {\n        NestedStageRestartHelper::restart_progress_should_run(state, self)\n    }\n\n    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {\n        NestedStageRestartHelper::clear_restart_progress(state, self)\n    }\n}\n\nimpl<E, EM, ST, Z> OptionalStage<E, EM, ST, Z>\nwhere\n    E: UsesState,\n    EM: UsesState<State = E::State>,\n    ST: StagesTuple<E, EM, E::State, Z>,\n    Z: UsesState<State = E::State>,\n{\n    /// Constructor for this conditionally enabled stage.\n    #[must_use]\n    pub fn new(stages: Option<ST>) -> Self {\n        Self {\n            stages,\n            phantom: PhantomData,\n        }\n    }\n\n    /// Constructor for this conditionally enabled stage with set stages.\n    #[must_use]\n    pub fn some(stages: ST) -> Self {\n        Self {\n            stages: Some(stages),\n            phantom: PhantomData,\n        }\n    }\n\n    /// Constructor for this conditionally enabled stage, without stages set.\n    #[must_use]\n    pub fn none() -> Self {\n        Self {\n            stages: None,\n            phantom: PhantomData,\n        }\n    }\n}\n\n// Path: libafl/src/stages/tuneable.rs\n//! A [`crate::stages::MutationalStage`] where the mutator iteration can be tuned at runtime\n\nuse alloc::string::{String, ToString};\nuse core::{marker::PhantomData, time::Duration};\n\nuse libafl_bolts::{current_time, impl_serdeany, rands::Rand};\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    mark_feature_time,\n    mutators::{MutationResult, Mutator},\n    stages::{\n        mutational::{MutatedTransform, MutatedTransformPost, DEFAULT_MUTATIONAL_MAX_ITERATIONS},\n        ExecutionCountRestartHelper, MutationalStage, Stage,\n    },\n    start_timer,\n    state::{\n        HasCorpus, HasCurrentTestcase, HasExecutions, HasMetadata, HasNamedMetadata, HasRand,\n        UsesState,\n    },\n    Error, Evaluator,\n};\n#[cfg(feature = \"introspection\")]\nuse crate::{monitors::PerfFeature, state::HasClientPerfMonitor};\n\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\n#[derive(Default, Clone, Copy, Eq, PartialEq, Debug, Serialize, Deserialize)]\nstruct TuneableMutationalStageMetadata {\n    iters: Option<u64>,\n    fuzz_time: Option<Duration>,\n}\n\nimpl_serdeany!(TuneableMutationalStageMetadata);\n\n/// The default name of the tunenable mutational stage.\npub const STD_TUNEABLE_MUTATIONAL_STAGE_NAME: &str = \"TuneableMutationalStage\";\n\n/// Set the number of iterations to be used by this mutational stage by name\npub fn set_iters_by_name<S>(state: &mut S, iters: u64, name: &str) -> Result<(), Error>\nwhere\n    S: HasNamedMetadata,\n{\n    let metadata = state\n        .named_metadata_map_mut()\n        .get_mut::<TuneableMutationalStageMetadata>(name)\n        .ok_or_else(|| Error::illegal_state(\"TuneableMutationalStage not in use\"));\n    metadata.map(|metadata| {\n        metadata.iters = Some(iters);\n    })\n}\n\n/// Set the number of iterations to be used by this mutational stage with a default name\npub fn set_iters_std<S>(state: &mut S, iters: u64) -> Result<(), Error>\nwhere\n    S: HasNamedMetadata,\n{\n    set_iters_by_name(state, iters, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n}\n\n/// Get the set iterations by name\n\npub fn get_iters_by_name<S>(state: &S, name: &str) -> Result<Option<u64>, Error>\nwhere\n    S: HasNamedMetadata,\n{\n    state\n        .named_metadata_map()\n        .get::<TuneableMutationalStageMetadata>(name)\n        .ok_or_else(|| Error::illegal_state(\"TuneableMutationalStage not in use\"))\n        .map(|metadata| metadata.iters)\n}\n\n/// Get the set iterations with a default name\npub fn get_iters_std<S>(state: &S) -> Result<Option<u64>, Error>\nwhere\n    S: HasNamedMetadata,\n{\n    get_iters_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n}\n\n/// Set the time for a single seed to be used by this mutational stage\npub fn set_seed_fuzz_time_by_name<S>(\n    state: &mut S,\n    fuzz_time: Duration,\n    name: &str,\n) -> Result<(), Error>\nwhere\n    S: HasNamedMetadata,\n{\n    let metadata = state\n        .named_metadata_map_mut()\n        .get_mut::<TuneableMutationalStageMetadata>(name)\n        .ok_or_else(|| Error::illegal_state(\"TuneableMutationalStage not in use\"));\n    metadata.map(|metadata| {\n        metadata.fuzz_time = Some(fuzz_time);\n    })\n}\n\n/// Set the time for a single seed to be used by this mutational stage with a default name\npub fn set_seed_fuzz_time_std<S>(state: &mut S, fuzz_time: Duration) -> Result<(), Error>\nwhere\n    S: HasNamedMetadata,\n{\n    set_seed_fuzz_time_by_name(state, fuzz_time, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n}\n\n/// Get the time for a single seed to be used by this mutational stage by name\npub fn get_seed_fuzz_time_by_name<S>(state: &S, name: &str) -> Result<Option<Duration>, Error>\nwhere\n    S: HasNamedMetadata,\n{\n    state\n        .named_metadata_map()\n        .get::<TuneableMutationalStageMetadata>(name)\n        .ok_or_else(|| Error::illegal_state(\"TuneableMutationalStage not in use\"))\n        .map(|metadata| metadata.fuzz_time)\n}\n\n/// Get the time for a single seed to be used by this mutational stage with a default name\npub fn get_seed_fuzz_time_std<S>(state: &S) -> Result<Option<Duration>, Error>\nwhere\n    S: HasNamedMetadata,\n{\n    get_seed_fuzz_time_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n}\n\n/// Reset this to a normal, randomized, stage by name\npub fn reset_by_name<S>(state: &mut S, name: &str) -> Result<(), Error>\nwhere\n    S: HasNamedMetadata,\n{\n    state\n        .named_metadata_map_mut()\n        .get_mut::<TuneableMutationalStageMetadata>(name)\n        .ok_or_else(|| Error::illegal_state(\"TuneableMutationalStage not in use\"))\n        .map(|metadata| {\n            metadata.iters = None;\n            metadata.fuzz_time = None;\n        })\n}\n\n/// Reset this to a normal, randomized, stage with a default name\npub fn reset_std<S>(state: &mut S) -> Result<(), Error>\nwhere\n    S: HasNamedMetadata,\n{\n    reset_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n}\n\n/// A [`crate::stages::MutationalStage`] where the mutator iteration can be tuned at runtime\n#[derive(Clone, Debug)]\npub struct TuneableMutationalStage<E, EM, I, M, Z> {\n    /// The mutator we use\n    mutator: M,\n    /// The name of this stage\n    name: String,\n    /// The progress helper we use to keep track of progress across restarts\n    restart_helper: ExecutionCountRestartHelper,\n    phantom: PhantomData<(E, EM, I, Z)>,\n}\n\nimpl<E, EM, I, M, Z> MutationalStage<E, EM, I, M, Z> for TuneableMutationalStage<E, EM, I, M, Z>\nwhere\n    E: UsesState<State = Z::State>,\n    EM: UsesState<State = Z::State>,\n    M: Mutator<I, Z::State>,\n    Z: Evaluator<E, EM>,\n    Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata + HasExecutions,\n    I: MutatedTransform<Z::Input, Z::State> + Clone,\n{\n    /// Runs this (mutational) stage for the given `testcase`\n    /// Exactly the same functionality as [`MutationalStage::perform_mutational`], but with added timeout support.\n    #[allow(clippy::cast_possible_wrap)] // more than i32 stages on 32 bit system - highly unlikely...\n    fn perform_mutational(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut Z::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        let fuzz_time = self.seed_fuzz_time(state)?;\n        let iters = self.fixed_iters(state)?;\n\n        start_timer!(state);\n        let mut testcase = state.current_testcase_mut()?;\n        let Ok(input) = I::try_transform_from(&mut testcase, state) else {\n            return Ok(());\n        };\n        drop(testcase);\n        mark_feature_time!(state, PerfFeature::GetInputFromCorpus);\n\n        match (fuzz_time, iters) {\n            (Some(fuzz_time), Some(iters)) => {\n                // perform n iterations or fuzz for provided time, whichever comes first\n                let start_time = current_time();\n                for _ in 1..=iters {\n                    if current_time() - start_time >= fuzz_time {\n                        break;\n                    }\n\n                    self.perform_mutation(fuzzer, executor, state, manager, &input)?;\n                }\n            }\n            (Some(fuzz_time), None) => {\n                // fuzz for provided time\n                let start_time = current_time();\n                for _ in 1.. {\n                    if current_time() - start_time >= fuzz_time {\n                        break;\n                    }\n\n                    self.perform_mutation(fuzzer, executor, state, manager, &input)?;\n                }\n            }\n            (None, Some(iters)) => {\n                // perform n iterations\n                for _ in 1..=iters {\n                    self.perform_mutation(fuzzer, executor, state, manager, &input)?;\n                }\n            }\n            (None, None) => {\n                // fall back to random\n                let iters = self\n                    .iterations(state)?\n                    .saturating_sub(self.execs_since_progress_start(state)?);\n                for _ in 1..=iters {\n                    self.perform_mutation(fuzzer, executor, state, manager, &input)?;\n                }\n            }\n        }\n        Ok(())\n    }\n\n    /// The mutator, added to this stage\n    #[inline]\n    fn mutator(&self) -> &M {\n        &self.mutator\n    }\n\n    /// The list of mutators, added to this stage (as mutable ref)\n    #[inline]\n    fn mutator_mut(&mut self) -> &mut M {\n        &mut self.mutator\n    }\n\n    /// Gets the number of iterations as a random number\n    #[allow(clippy::cast_possible_truncation)]\n    fn iterations(&self, state: &mut Z::State) -> Result<u64, Error> {\n        Ok(\n            // fall back to random\n            1 + state.rand_mut().below(DEFAULT_MUTATIONAL_MAX_ITERATIONS),\n        )\n    }\n\n    fn execs_since_progress_start(&mut self, state: &mut <Z>::State) -> Result<u64, Error> {\n        self.restart_helper.execs_since_progress_start(state)\n    }\n}\n\nimpl<E, EM, I, M, Z> UsesState for TuneableMutationalStage<E, EM, I, M, Z>\nwhere\n    E: UsesState<State = Z::State>,\n    EM: UsesState<State = Z::State>,\n    M: Mutator<I, Z::State>,\n    Z: Evaluator<E, EM>,\n    Z::State: HasCorpus + HasRand + HasExecutions,\n    I: MutatedTransform<Z::Input, Z::State> + Clone,\n{\n    type State = Z::State;\n}\n\nimpl<E, EM, I, M, Z> Stage<E, EM, Z> for TuneableMutationalStage<E, EM, I, M, Z>\nwhere\n    E: UsesState<State = Z::State>,\n    EM: UsesState<State = Z::State>,\n    M: Mutator<I, Z::State>,\n    Z: Evaluator<E, EM>,\n    Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata + HasExecutions,\n    I: MutatedTransform<Z::Input, Z::State> + Clone,\n{\n    #[inline]\n    #[allow(clippy::let_and_return)]\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut Z::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        let ret = self.perform_mutational(fuzzer, executor, state, manager);\n\n        #[cfg(feature = \"introspection\")]\n        state.introspection_monitor_mut().finish_stage();\n\n        ret\n    }\n\n    fn restart_progress_should_run(&mut self, state: &mut Self::State) -> Result<bool, Error> {\n        self.restart_helper.restart_progress_should_run(state)\n    }\n\n    fn clear_restart_progress(&mut self, state: &mut Self::State) -> Result<(), Error> {\n        self.restart_helper.clear_restart_progress(state)\n    }\n}\n\nimpl<E, EM, I, M, Z> TuneableMutationalStage<E, EM, I, M, Z>\nwhere\n    E: UsesState<State = Z::State>,\n    EM: UsesState<State = Z::State>,\n    M: Mutator<I, Z::State>,\n    Z: Evaluator<E, EM>,\n    Z::State: HasCorpus + HasRand + HasNamedMetadata + HasMetadata + HasExecutions,\n    I: MutatedTransform<Z::Input, Z::State> + Clone,\n{\n    /// Creates a new default tuneable mutational stage\n    #[must_use]\n    pub fn new(state: &mut Z::State, mutator: M) -> Self {\n        Self::transforming(state, mutator, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n    }\n\n    /// Crates a new tuneable mutational stage with the given name\n    pub fn with_name(state: &mut Z::State, mutator: M, name: &str) -> Self {\n        Self::transforming(state, mutator, name)\n    }\n\n    /// Set the number of iterations to be used by this [`TuneableMutationalStage`]\n    pub fn set_iters<S>(&self, state: &mut S, iters: u64) -> Result<(), Error>\n    where\n        S: HasNamedMetadata,\n    {\n        set_iters_by_name(state, iters, &self.name)\n    }\n\n    /// Set the number of iterations to be used by the std [`TuneableMutationalStage`]\n    pub fn set_iters_std(state: &mut Z::State, iters: u64) -> Result<(), Error> {\n        set_iters_by_name(state, iters, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n    }\n\n    /// Set the number of iterations to be used by the [`TuneableMutationalStage`] with the given name\n    pub fn set_iters_by_name<S>(state: &mut S, iters: u64, name: &str) -> Result<(), Error>\n    where\n        S: HasNamedMetadata,\n    {\n        set_iters_by_name(state, iters, name)\n    }\n\n    /// Get the set iterations for this [`TuneableMutationalStage`], if any\n    pub fn fixed_iters<S>(&self, state: &S) -> Result<Option<u64>, Error>\n    where\n        S: HasNamedMetadata,\n    {\n        get_iters_by_name(state, &self.name)\n    }\n\n    /// Get the set iterations for the std [`TuneableMutationalStage`], if any\n    pub fn iters_std(state: &Z::State) -> Result<Option<u64>, Error> {\n        get_iters_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n    }\n\n    /// Get the set iterations for the [`TuneableMutationalStage`] with the given name, if any\n    pub fn iters_by_name<S>(state: &S, name: &str) -> Result<Option<u64>, Error>\n    where\n        S: HasNamedMetadata,\n    {\n        get_iters_by_name(state, name)\n    }\n\n    /// Set the time to mutate a single input in this [`TuneableMutationalStage`]\n    pub fn set_seed_fuzz_time<S>(&self, state: &mut S, fuzz_time: Duration) -> Result<(), Error>\n    where\n        S: HasNamedMetadata,\n    {\n        set_seed_fuzz_time_by_name(state, fuzz_time, &self.name)\n    }\n\n    /// Set the time to mutate a single input in the std [`TuneableMutationalStage`]\n    pub fn set_seed_fuzz_time_std(state: &mut Z::State, fuzz_time: Duration) -> Result<(), Error> {\n        set_seed_fuzz_time_by_name(state, fuzz_time, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n    }\n\n    /// Set the time to mutate a single input in the [`TuneableMutationalStage`] with the given name\n    pub fn set_seed_fuzz_time_by_name<S>(\n        state: &mut S,\n        fuzz_time: Duration,\n        name: &str,\n    ) -> Result<(), Error>\n    where\n        S: HasNamedMetadata,\n    {\n        set_seed_fuzz_time_by_name(state, fuzz_time, name)\n    }\n\n    /// Set the time to mutate a single input in this [`TuneableMutationalStage`]\n    pub fn seed_fuzz_time<S>(&self, state: &S) -> Result<Option<Duration>, Error>\n    where\n        S: HasNamedMetadata,\n    {\n        get_seed_fuzz_time_by_name(state, &self.name)\n    }\n\n    /// Set the time to mutate a single input for the std [`TuneableMutationalStage`]\n    pub fn seed_fuzz_time_std(&self, state: &Z::State) -> Result<Option<Duration>, Error> {\n        get_seed_fuzz_time_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n    }\n\n    /// Set the time to mutate a single input for the [`TuneableMutationalStage`] with a given name\n    pub fn seed_fuzz_time_by_name<S>(\n        &self,\n        state: &S,\n        name: &str,\n    ) -> Result<Option<Duration>, Error>\n    where\n        S: HasNamedMetadata,\n    {\n        get_seed_fuzz_time_by_name(state, name)\n    }\n\n    /// Reset this to a normal, randomized, stage with\n    pub fn reset<S>(&self, state: &mut S) -> Result<(), Error>\n    where\n        S: HasNamedMetadata,\n    {\n        reset_by_name(state, &self.name)\n    }\n\n    /// Reset the std stage to a normal, randomized, stage\n    pub fn reset_std(state: &mut Z::State) -> Result<(), Error> {\n        reset_by_name(state, STD_TUNEABLE_MUTATIONAL_STAGE_NAME)\n    }\n\n    /// Reset this to a normal, randomized, stage by name\n    pub fn reset_by_name<S>(state: &mut S, name: &str) -> Result<(), Error>\n    where\n        S: HasNamedMetadata,\n    {\n        reset_by_name(state, name)\n    }\n\n    fn perform_mutation(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut Z::State,\n        manager: &mut EM,\n        input: &I,\n    ) -> Result<(), Error> {\n        let mut input = input.clone();\n\n        start_timer!(state);\n        let mutated = self.mutator_mut().mutate(state, &mut input)?;\n        mark_feature_time!(state, PerfFeature::Mutate);\n\n        if mutated == MutationResult::Skipped {\n            return Ok(());\n        }\n\n        // Time is measured directly the `evaluate_input` function\n        let (untransformed, post) = input.try_transform_into(state)?;\n        let (_, corpus_idx) = fuzzer.evaluate_input(state, executor, manager, untransformed)?;\n\n        start_timer!(state);\n        self.mutator_mut().post_exec(state, corpus_idx)?;\n        post.post_exec(state, corpus_idx)?;\n        mark_feature_time!(state, PerfFeature::MutatePostExec);\n\n        Ok(())\n    }\n}\n\nimpl<E, EM, I, M, Z> TuneableMutationalStage<E, EM, I, M, Z>\nwhere\n    E: UsesState<State = Z::State>,\n    EM: UsesState<State = Z::State>,\n    M: Mutator<I, Z::State>,\n    Z: Evaluator<E, EM>,\n    Z::State: HasCorpus + HasRand + HasNamedMetadata,\n{\n    /// Creates a new transforming mutational stage\n    #[must_use]\n    pub fn transforming(state: &mut Z::State, mutator: M, name: &str) -> Self {\n        let _ = state.named_metadata_or_insert_with(name, TuneableMutationalStageMetadata::default);\n        Self {\n            mutator,\n            name: name.to_string(),\n            restart_helper: ExecutionCountRestartHelper::default(),\n            phantom: PhantomData,\n        }\n    }\n}\n\n// Path: libafl/src/stages/power.rs\n//! The power schedules. This stage should be invoked after the calibration stage.\n\nuse core::{fmt::Debug, marker::PhantomData};\n\nuse crate::{\n    executors::{Executor, HasObservers},\n    fuzzer::Evaluator,\n    mutators::Mutator,\n    schedulers::{testcase_score::CorpusPowerTestcaseScore, TestcaseScore},\n    stages::{mutational::MutatedTransform, ExecutionCountRestartHelper, MutationalStage, Stage},\n    state::{HasCorpus, HasCurrentTestcase, HasExecutions, HasMetadata, HasRand, UsesState},\n    Error,\n};\n\n/// The mutational stage using power schedules\n#[derive(Clone, Debug)]\npub struct PowerMutationalStage<E, F, EM, I, M, Z> {\n    /// The mutators we use\n    mutator: M,\n    /// Helper for restarts\n    restart_helper: ExecutionCountRestartHelper,\n    #[allow(clippy::type_complexity)]\n    phantom: PhantomData<(E, F, EM, I, Z)>,\n}\n\nimpl<E, F, EM, I, M, Z> UsesState for PowerMutationalStage<E, F, EM, I, M, Z>\nwhere\n    E: UsesState,\n{\n    type State = E::State;\n}\n\nimpl<E, F, EM, I, M, Z> MutationalStage<E, EM, I, M, Z> for PowerMutationalStage<E, F, EM, I, M, Z>\nwhere\n    E: Executor<EM, Z> + HasObservers,\n    EM: UsesState<State = E::State>,\n    F: TestcaseScore<E::State>,\n    M: Mutator<I, E::State>,\n    E::State: HasCorpus + HasMetadata + HasRand + HasExecutions,\n    Z: Evaluator<E, EM, State = E::State>,\n    I: MutatedTransform<E::Input, E::State> + Clone,\n{\n    /// The mutator, added to this stage\n    #[inline]\n    fn mutator(&self) -> &M {\n        &self.mutator\n    }\n\n    /// The list of mutators, added to this stage (as mutable ref)\n    #[inline]\n    fn mutator_mut(&mut self) -> &mut M {\n        &mut self.mutator\n    }\n\n    /// Gets the number of iterations as a random number\n    #[allow(clippy::cast_sign_loss)]\n    fn iterations(&self, state: &mut E::State) -> Result<u64, Error> {\n        // Update handicap\n        let mut testcase = state.current_testcase_mut()?;\n        let score = F::compute(state, &mut testcase)? as u64;\n\n        Ok(score)\n    }\n\n    fn execs_since_progress_start(&mut self, state: &mut <Z>::State) -> Result<u64, Error> {\n        self.restart_helper.execs_since_progress_start(state)\n    }\n}\n\nimpl<E, F, EM, I, M, Z> Stage<E, EM, Z> for PowerMutationalStage<E, F, EM, I, M, Z>\nwhere\n    E: Executor<EM, Z> + HasObservers,\n    EM: UsesState<State = E::State>,\n    F: TestcaseScore<E::State>,\n    M: Mutator<I, E::State>,\n    E::State: HasCorpus + HasMetadata + HasRand + HasExecutions,\n    Z: Evaluator<E, EM, State = E::State>,\n    I: MutatedTransform<E::Input, E::State> + Clone,\n{\n    #[inline]\n    #[allow(clippy::let_and_return)]\n    fn perform(\n        &mut self,\n        fuzzer: &mut Z,\n        executor: &mut E,\n        state: &mut E::State,\n        manager: &mut EM,\n    ) -> Result<(), Error> {\n        let ret = self.perform_mutational(fuzzer, executor, state, manager);\n        ret\n    }\n\n    fn restart_progress_should_run(&mut self, _state: &mut Self::State) -> Result<bool, Error> {\n        Ok(true)\n        // self.restart_helper.restart_progress_should_run(state)\n    }\n\n    fn clear_restart_progress(&mut self, _state: &mut Self::State) -> Result<(), Error> {\n        Ok(())\n        // self.restart_helper.clear_restart_progress(state)\n    }\n}\n\nimpl<E, F, EM, M, Z> PowerMutationalStage<E, F, EM, E::Input, M, Z>\nwhere\n    E: Executor<EM, Z> + HasObservers,\n    EM: UsesState<State = E::State>,\n    F: TestcaseScore<E::State>,\n    M: Mutator<E::Input, E::State>,\n    E::State: HasCorpus + HasMetadata + HasRand,\n    Z: Evaluator<E, EM, State = E::State>,\n{\n    /// Creates a new [`PowerMutationalStage`]\n    pub fn new(mutator: M) -> Self {\n        Self::transforming(mutator)\n    }\n}\n\nimpl<E, F, EM, I, M, Z> PowerMutationalStage<E, F, EM, I, M, Z>\nwhere\n    E: Executor<EM, Z> + HasObservers,\n    EM: UsesState<State = E::State>,\n    F: TestcaseScore<E::State>,\n    M: Mutator<I, E::State>,\n    E::State: HasCorpus + HasMetadata + HasRand,\n    Z: Evaluator<E, EM, State = E::State>,\n{\n    /// Creates a new transforming [`PowerMutationalStage`]\n    pub fn transforming(mutator: M) -> Self {\n        Self {\n            mutator,\n            phantom: PhantomData,\n            restart_helper: ExecutionCountRestartHelper::default(),\n        }\n    }\n}\n\n/// The standard powerscheduling stage\npub type StdPowerMutationalStage<E, EM, I, M, Z> =\n    PowerMutationalStage<E, CorpusPowerTestcaseScore<<E as UsesState>::State>, EM, I, M, Z>;\n\n// Path: libafl/src/stages/push/mutational.rs\n//| The [`MutationalStage`] is the default stage used during fuzzing.\n//! For the current input, it will perform a range of random mutations, and then run them in the executor.\n\nuse alloc::rc::Rc;\nuse core::{\n    cell::{Cell, RefCell},\n    fmt::Debug,\n};\n\nuse libafl_bolts::rands::Rand;\n\nuse super::{PushStage, PushStageHelper, PushStageSharedState};\nuse crate::{\n    corpus::{Corpus, CorpusId},\n    events::{EventFirer, EventRestarter, HasEventManagerId, ProgressReporter},\n    executors::ExitKind,\n    inputs::UsesInput,\n    mark_feature_time,\n    mutators::Mutator,\n    observers::ObserversTuple,\n    schedulers::Scheduler,\n    start_timer,\n    state::{HasCorpus, HasExecutions, HasLastReportTime, HasMetadata, HasRand},\n    Error, EvaluatorObservers, ExecutionProcessor, HasScheduler,\n};\n#[cfg(feature = \"introspection\")]\nuse crate::{monitors::PerfFeature, state::HasClientPerfMonitor};\n\n/// The default maximum number of mutations to perform per input.\npub static DEFAULT_MUTATIONAL_MAX_ITERATIONS: u64 = 128;\n/// A Mutational push stage is the stage in a fuzzing run that mutates inputs.\n/// Mutational push stages will usually have a range of mutations that are\n/// being applied to the input one by one, between executions.\n/// The push version, in contrast to the normal stage, will return each testcase, instead of executing it.\n///\n/// Default value, how many iterations each stage gets, as an upper bound.\n/// It may randomly continue earlier.\n///\n/// The default mutational push stage\n#[derive(Clone, Debug)]\npub struct StdMutationalPushStage<CS, EM, M, OT, Z>\nwhere\n    CS: Scheduler,\n    EM: EventFirer<State = CS::State> + EventRestarter + HasEventManagerId,\n    M: Mutator<CS::Input, CS::State>,\n    OT: ObserversTuple<CS::State>,\n    CS::State: HasRand + HasCorpus + Clone + Debug,\n    Z: ExecutionProcessor<OT, State = CS::State>\n        + EvaluatorObservers<OT>\n        + HasScheduler<Scheduler = CS>,\n{\n    current_corpus_idx: Option<CorpusId>,\n    testcases_to_do: usize,\n    testcases_done: usize,\n\n    mutator: M,\n\n    psh: PushStageHelper<CS, EM, OT, Z>,\n}\n\nimpl<CS, EM, M, OT, Z> StdMutationalPushStage<CS, EM, M, OT, Z>\nwhere\n    CS: Scheduler,\n    EM: EventFirer<State = CS::State> + EventRestarter + HasEventManagerId,\n    M: Mutator<CS::Input, CS::State>,\n    OT: ObserversTuple<CS::State>,\n    CS::State: HasCorpus + HasRand + Clone + Debug,\n    Z: ExecutionProcessor<OT, State = CS::State>\n        + EvaluatorObservers<OT>\n        + HasScheduler<Scheduler = CS>,\n{\n    /// Gets the number of iterations as a random number\n    #[allow(clippy::unused_self, clippy::unnecessary_wraps)] // TODO: we should put this function into a trait later\n    fn iterations(&self, state: &mut CS::State, _corpus_idx: CorpusId) -> Result<usize, Error> {\n        Ok(1 + state.rand_mut().below(DEFAULT_MUTATIONAL_MAX_ITERATIONS) as usize)\n    }\n\n    /// Sets the current corpus index\n    pub fn set_current_corpus_idx(&mut self, current_corpus_idx: CorpusId) {\n        self.current_corpus_idx = Some(current_corpus_idx);\n    }\n}\n\nimpl<CS, EM, M, OT, Z> PushStage<CS, EM, OT, Z> for StdMutationalPushStage<CS, EM, M, OT, Z>\nwhere\n    CS: Scheduler,\n    EM: EventFirer<State = CS::State> + EventRestarter + HasEventManagerId + ProgressReporter,\n    M: Mutator<CS::Input, CS::State>,\n    OT: ObserversTuple<CS::State>,\n    CS::State:\n        HasCorpus + HasRand + HasExecutions + HasLastReportTime + HasMetadata + Clone + Debug,\n    Z: ExecutionProcessor<OT, State = CS::State>\n        + EvaluatorObservers<OT>\n        + HasScheduler<Scheduler = CS>,\n{\n    #[inline]\n    fn push_stage_helper(&self) -> &PushStageHelper<CS, EM, OT, Z> {\n        &self.psh\n    }\n\n    #[inline]\n    fn push_stage_helper_mut(&mut self) -> &mut PushStageHelper<CS, EM, OT, Z> {\n        &mut self.psh\n    }\n\n    /// Creates a new default mutational stage\n    fn init(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut CS::State,\n        _event_mgr: &mut EM,\n        _observers: &mut OT,\n    ) -> Result<(), Error> {\n        // Find a testcase to work on, unless someone already set it\n        self.current_corpus_idx = Some(if let Some(corpus_idx) = self.current_corpus_idx {\n            corpus_idx\n        } else {\n            fuzzer.scheduler_mut().next(state)?\n        });\n\n        self.testcases_to_do = self.iterations(state, self.current_corpus_idx.unwrap())?;\n        self.testcases_done = 0;\n        Ok(())\n    }\n\n    fn pre_exec(\n        &mut self,\n        _fuzzer: &mut Z,\n        state: &mut CS::State,\n        _event_mgr: &mut EM,\n        _observers: &mut OT,\n    ) -> Option<Result<<CS::State as UsesInput>::Input, Error>> {\n        if self.testcases_done >= self.testcases_to_do {\n            // finished with this cicle.\n            return None;\n        }\n\n        start_timer!(state);\n\n        let input = state\n            .corpus_mut()\n            .cloned_input_for_id(self.current_corpus_idx.unwrap());\n        let mut input = match input {\n            Err(e) => return Some(Err(e)),\n            Ok(input) => input,\n        };\n\n        mark_feature_time!(state, PerfFeature::GetInputFromCorpus);\n\n        start_timer!(state);\n        self.mutator.mutate(state, &mut input).unwrap();\n        mark_feature_time!(state, PerfFeature::Mutate);\n\n        self.push_stage_helper_mut()\n            .current_input\n            .replace(input.clone()); // TODO: Get rid of this\n\n        Some(Ok(input))\n    }\n\n    fn post_exec(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut CS::State,\n        event_mgr: &mut EM,\n        observers: &mut OT,\n        last_input: <CS::State as UsesInput>::Input,\n        exit_kind: ExitKind,\n    ) -> Result<(), Error> {\n        // todo: is_interesting, etc.\n\n        fuzzer.process_execution(state, event_mgr, last_input, observers, &exit_kind, true)?;\n\n        start_timer!(state);\n        self.mutator.post_exec(state, self.current_corpus_idx)?;\n        mark_feature_time!(state, PerfFeature::MutatePostExec);\n        self.testcases_done += 1;\n\n        Ok(())\n    }\n\n    #[inline]\n    fn deinit(\n        &mut self,\n        _fuzzer: &mut Z,\n        _state: &mut CS::State,\n        _event_mgr: &mut EM,\n        _observers: &mut OT,\n    ) -> Result<(), Error> {\n        self.current_corpus_idx = None;\n        Ok(())\n    }\n}\n\nimpl<CS, EM, M, OT, Z> Iterator for StdMutationalPushStage<CS, EM, M, OT, Z>\nwhere\n    CS: Scheduler,\n    EM: EventFirer + EventRestarter + HasEventManagerId + ProgressReporter<State = CS::State>,\n    M: Mutator<CS::Input, CS::State>,\n    OT: ObserversTuple<CS::State>,\n    CS::State:\n        HasCorpus + HasRand + HasExecutions + HasMetadata + HasLastReportTime + Clone + Debug,\n    Z: ExecutionProcessor<OT, State = CS::State>\n        + EvaluatorObservers<OT>\n        + HasScheduler<Scheduler = CS>,\n{\n    type Item = Result<<CS::State as UsesInput>::Input, Error>;\n\n    fn next(&mut self) -> Option<Result<<CS::State as UsesInput>::Input, Error>> {\n        self.next_std()\n    }\n}\n\nimpl<CS, EM, M, OT, Z> StdMutationalPushStage<CS, EM, M, OT, Z>\nwhere\n    CS: Scheduler,\n    EM: EventFirer<State = CS::State> + EventRestarter + HasEventManagerId,\n...\n", "needle_token_start": 7232, "needle_token_end": 7349, "code_context_ntokens": 16209, "output": ["```rust\n/// Get the set iterations by name\npub fn get_iters_by_name<S>(state: &S, name: &str) -> Result<Option<u64>, Error>\nwhere\n    S: HasNamedMetadata,\n{\n    state\n        .named_metadata_map()\n        .get::<TuneableMutationalStageMetadata>(name)\n        .ok_or_else(|| Error::illegal_state(\"TuneableMutationalStage not in use\"))\n        .map(|metadata| metadata.iters)\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "exposed_executor_state", "language": "rust", "path": "libafl/src/executors/inprocess/stateful.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide read-only access to a specific state used during the execution of a test harness, allowing for inspection or monitoring without modification.\n2. **Input**: This function does not take any parameters.\n3. **Output**: Returns a reference to the state that is used internally by the executor during test operations.\n4. **Procedure**: The function simply returns a reference to the internally maintained state, ensuring that external entities can observe but not alter the execution state.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: libafl/src/executors/hooks/inprocess.rs\n//! The hook for `InProcessExecutor`\n#[cfg(any(unix, feature = \"std\"))]\nuse core::ptr::addr_of_mut;\n#[cfg(any(unix, all(windows, feature = \"std\")))]\nuse core::sync::atomic::{compiler_fence, Ordering};\nuse core::{\n    ffi::c_void,\n    marker::PhantomData,\n    ptr::{self, null_mut},\n    time::Duration,\n};\n#[cfg(all(target_os = \"linux\", feature = \"std\"))]\nuse core::{mem::zeroed, ptr::addr_of};\n\n#[cfg(all(target_os = \"linux\", feature = \"std\"))]\nuse libafl_bolts::current_time;\n#[cfg(all(unix, feature = \"std\", not(miri)))]\nuse libafl_bolts::os::unix_signals::setup_signal_handler;\n#[cfg(all(windows, feature = \"std\"))]\nuse libafl_bolts::os::windows_exceptions::setup_exception_handler;\n#[cfg(all(windows, feature = \"std\"))]\nuse windows::Win32::System::Threading::{CRITICAL_SECTION, PTP_TIMER};\n\n#[cfg(feature = \"std\")]\nuse crate::executors::hooks::timer::TimerStruct;\n#[cfg(all(unix, feature = \"std\"))]\nuse crate::executors::hooks::unix::unix_signal_handler;\n#[cfg(windows)]\nuse crate::state::State;\nuse crate::{\n    events::{EventFirer, EventRestarter},\n    executors::{hooks::ExecutorHook, inprocess::HasInProcessHooks, Executor, HasObservers},\n    feedbacks::Feedback,\n    inputs::UsesInput,\n    state::{HasCorpus, HasExecutions, HasSolutions},\n    Error, HasObjective,\n};\n/// The inmem executor's handlers.\n#[allow(missing_debug_implementations)]\npub struct InProcessHooks<S>\nwhere\n    S: UsesInput,\n{\n    /// On crash C function pointer\n    #[cfg(feature = \"std\")]\n    pub crash_handler: *const c_void,\n    /// On timeout C function pointer\n    #[cfg(feature = \"std\")]\n    pub timeout_handler: *const c_void,\n    /// `TImer` struct\n    #[cfg(feature = \"std\")]\n    pub timer: TimerStruct,\n    phantom: PhantomData<S>,\n}\n\n/// Any hooks that is about timeout\npub trait HasTimeout {\n    /// Return ref to timer\n    #[cfg(feature = \"std\")]\n    fn timer(&self) -> &TimerStruct;\n    /// Return mut ref to timer\n    #[cfg(feature = \"std\")]\n    fn timer_mut(&mut self) -> &mut TimerStruct;\n    #[cfg(all(feature = \"std\", windows))]\n    /// The timer object\n    #[cfg(all(feature = \"std\", windows))]\n    fn ptp_timer(&self) -> &PTP_TIMER;\n    #[cfg(all(feature = \"std\", windows))]\n    /// The critical section\n    fn critical(&self) -> &CRITICAL_SECTION;\n    #[cfg(all(feature = \"std\", windows))]\n    /// The critical section (mut)\n    fn critical_mut(&mut self) -> &mut CRITICAL_SECTION;\n    #[cfg(all(feature = \"std\", windows))]\n    /// The timeout in milli sec\n    #[cfg(all(feature = \"std\", windows))]\n    fn milli_sec(&self) -> i64;\n    #[cfg(all(feature = \"std\", windows))]\n    /// The timeout in milli sec (mut ref)\n    fn millis_sec_mut(&mut self) -> &mut i64;\n    #[cfg(not(all(unix, feature = \"std\")))]\n    /// Handle timeout for batch mode timeout\n    fn handle_timeout(&mut self) -> bool;\n    #[cfg(all(unix, feature = \"std\"))]\n    /// Handle timeout for batch mode timeout\n    fn handle_timeout(&mut self, data: &mut InProcessExecutorHandlerData) -> bool;\n}\n\nimpl<S> HasTimeout for InProcessHooks<S>\nwhere\n    S: UsesInput,\n{\n    #[cfg(feature = \"std\")]\n    fn timer(&self) -> &TimerStruct {\n        &self.timer\n    }\n    #[cfg(feature = \"std\")]\n    fn timer_mut(&mut self) -> &mut TimerStruct {\n        &mut self.timer\n    }\n\n    #[cfg(all(feature = \"std\", windows))]\n    fn ptp_timer(&self) -> &PTP_TIMER {\n        self.timer().ptp_timer()\n    }\n\n    #[cfg(all(feature = \"std\", windows))]\n    fn critical(&self) -> &CRITICAL_SECTION {\n        self.timer().critical()\n    }\n\n    #[cfg(all(feature = \"std\", windows))]\n    fn critical_mut(&mut self) -> &mut CRITICAL_SECTION {\n        self.timer_mut().critical_mut()\n    }\n\n    #[cfg(all(feature = \"std\", windows))]\n    fn milli_sec(&self) -> i64 {\n        self.timer().milli_sec()\n    }\n\n    #[cfg(all(feature = \"std\", windows))]\n    fn millis_sec_mut(&mut self) -> &mut i64 {\n        self.timer_mut().milli_sec_mut()\n    }\n\n    #[cfg(not(all(unix, feature = \"std\")))]\n    fn handle_timeout(&mut self) -> bool {\n        false\n    }\n\n    #[cfg(all(unix, feature = \"std\"))]\n    #[allow(unused)]\n    fn handle_timeout(&mut self, data: &mut InProcessExecutorHandlerData) -> bool {\n        #[cfg(not(target_os = \"linux\"))]\n        {\n            false\n        }\n\n        #[cfg(target_os = \"linux\")]\n        {\n            if !self.timer().batch_mode {\n                return false;\n            }\n            //eprintln!(\"handle_timeout {:?} {}\", self.avg_exec_time, self.avg_mul_k);\n            let cur_time = current_time();\n            if !data.is_valid() {\n                // outside the target\n                unsafe {\n                    let disarmed: libc::itimerspec = zeroed();\n                    libc::timer_settime(\n                        self.timer_mut().timerid,\n                        0,\n                        addr_of!(disarmed),\n                        null_mut(),\n                    );\n                }\n                let elapsed = cur_time - self.timer().tmout_start_time;\n                // set timer the next exec\n                if self.timer().executions > 0 {\n                    self.timer_mut().avg_exec_time = elapsed / self.timer().executions;\n                    self.timer_mut().executions = 0;\n                }\n                self.timer_mut().avg_mul_k += 1;\n                self.timer_mut().last_signal_time = cur_time;\n                return true;\n            }\n\n            let elapsed_run = cur_time - self.timer_mut().start_time;\n            if elapsed_run < self.timer_mut().exec_tmout {\n                // fp, reset timeout\n                unsafe {\n                    libc::timer_settime(\n                        self.timer_mut().timerid,\n                        0,\n                        addr_of!(self.timer_mut().itimerspec),\n                        null_mut(),\n                    );\n                }\n                if self.timer().executions > 0 {\n                    let elapsed = cur_time - self.timer_mut().tmout_start_time;\n                    self.timer_mut().avg_exec_time = elapsed / self.timer().executions;\n                    self.timer_mut().executions = 0; // It will be 1 when the exec finish\n                }\n                self.timer_mut().tmout_start_time = current_time();\n                self.timer_mut().avg_mul_k += 1;\n                self.timer_mut().last_signal_time = cur_time;\n                true\n            } else {\n                false\n            }\n        }\n    }\n}\n\nimpl<S> ExecutorHook<S> for InProcessHooks<S>\nwhere\n    S: UsesInput,\n{\n    fn init<E: HasObservers>(&mut self, _state: &mut S) {}\n    /// Call before running a target.\n    #[allow(clippy::unused_self)]\n    #[allow(unused_variables)]\n    fn pre_exec(&mut self, state: &mut S, input: &S::Input) {\n        #[cfg(feature = \"std\")]\n        unsafe {\n            let data = addr_of_mut!(GLOBAL_STATE);\n            (*data).crash_handler = self.crash_handler;\n            (*data).timeout_handler = self.timeout_handler;\n        }\n\n        #[cfg(all(feature = \"std\", not(all(miri, target_vendor = \"apple\"))))]\n        self.timer_mut().set_timer();\n    }\n\n    /// Call after running a target.\n    #[allow(clippy::unused_self)]\n    fn post_exec(&mut self, _state: &mut S, _input: &S::Input) {\n        // timeout stuff\n        #[cfg(all(feature = \"std\", not(all(miri, target_vendor = \"apple\"))))]\n        self.timer_mut().unset_timer();\n    }\n}\n\nimpl<S> InProcessHooks<S>\nwhere\n    S: UsesInput,\n{\n    /// Create new [`InProcessHooks`].\n    #[cfg(unix)]\n    #[allow(unused_variables)]\n    pub fn new<E, EM, OF, Z>(exec_tmout: Duration) -> Result<Self, Error>\n    where\n        E: Executor<EM, Z> + HasObservers + HasInProcessHooks<E::State>,\n        EM: EventFirer<State = E::State> + EventRestarter<State = E::State>,\n        OF: Feedback<E::State>,\n        E::State: HasExecutions + HasSolutions + HasCorpus,\n        Z: HasObjective<Objective = OF, State = E::State>,\n    {\n        #[cfg_attr(miri, allow(unused_variables))]\n        unsafe {\n            let data = addr_of_mut!(GLOBAL_STATE);\n            #[cfg(feature = \"std\")]\n            unix_signal_handler::setup_panic_hook::<E, EM, OF, Z>();\n            #[cfg(all(not(miri), unix, feature = \"std\"))]\n            setup_signal_handler(data)?;\n            compiler_fence(Ordering::SeqCst);\n            Ok(Self {\n                #[cfg(feature = \"std\")]\n                crash_handler: unix_signal_handler::inproc_crash_handler::<E, EM, OF, Z>\n                    as *const c_void,\n                #[cfg(feature = \"std\")]\n                timeout_handler: unix_signal_handler::inproc_timeout_handler::<E, EM, OF, Z>\n...\n// Path: libafl/src/executors/inprocess/inner.rs\nuse core::{\n    ffi::c_void,\n    fmt::{self, Debug, Formatter},\n    marker::PhantomData,\n    ptr::{self, addr_of_mut, null, write_volatile},\n    sync::atomic::{compiler_fence, Ordering},\n    time::Duration,\n};\n\nuse libafl_bolts::tuples::{tuple_list, Merge};\n#[cfg(windows)]\nuse windows::Win32::System::Threading::SetThreadStackGuarantee;\n\n#[cfg(all(feature = \"std\", target_os = \"linux\"))]\nuse crate::executors::hooks::inprocess::HasTimeout;\n#[cfg(all(windows, feature = \"std\"))]\nuse crate::executors::hooks::inprocess::HasTimeout;\nuse crate::{\n    events::{EventFirer, EventRestarter},\n    executors::{\n        hooks::{\n            inprocess::{InProcessHooks, GLOBAL_STATE},\n            ExecutorHooksTuple,\n        },\n        inprocess::HasInProcessHooks,\n        Executor, HasObservers,\n    },\n    feedbacks::Feedback,\n    fuzzer::HasObjective,\n    inputs::UsesInput,\n    observers::{ObserversTuple, UsesObservers},\n    state::{HasCorpus, HasExecutions, HasSolutions, State, UsesState},\n    Error,\n};\n\n/// The internal state of `GenericInProcessExecutor`.\npub struct GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    /// The observers, observing each run\n    pub(super) observers: OT,\n    // Crash and timeout hah\n    pub(super) hooks: (InProcessHooks<S>, HT),\n    phantom: PhantomData<S>,\n}\n\nimpl<HT, OT, S> Debug for GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S> + Debug,\n    S: State,\n{\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"GenericInProcessExecutorState\")\n            .field(\"observers\", &self.observers)\n            .finish_non_exhaustive()\n    }\n}\n\nimpl<HT, OT, S> UsesState for GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    type State = S;\n}\n\nimpl<HT, OT, S> UsesObservers for GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    type Observers = OT;\n}\n\nimpl<HT, OT, S> HasObservers for GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        &self.observers\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        &mut self.observers\n    }\n}\n\nimpl<HT, OT, S> GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    /// This function marks the boundary between the fuzzer and the target\n    ///\n    /// # Safety\n    /// This function sets a bunch of raw pointers in global variables, reused in other parts of\n    /// the code.\n    #[inline]\n    pub unsafe fn enter_target<EM, Z>(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut <Self as UsesState>::State,\n        mgr: &mut EM,\n        input: &<Self as UsesInput>::Input,\n        executor_ptr: *const c_void,\n    ) {\n        unsafe {\n            let data = addr_of_mut!(GLOBAL_STATE);\n            write_volatile(\n                addr_of_mut!((*data).current_input_ptr),\n                ptr::from_ref(input) as *const c_void,\n            );\n            write_volatile(addr_of_mut!((*data).executor_ptr), executor_ptr);\n            // Direct raw pointers access /aliasing is pretty undefined behavior.\n            // Since the state and event may have moved in memory, refresh them right before the signal may happen\n            write_volatile(\n                addr_of_mut!((*data).state_ptr),\n                ptr::from_mut(state) as *mut c_void,\n            );\n            write_volatile(\n                addr_of_mut!((*data).event_mgr_ptr),\n                ptr::from_mut(mgr) as *mut c_void,\n            );\n            write_volatile(\n                addr_of_mut!((*data).fuzzer_ptr),\n                ptr::from_mut(fuzzer) as *mut c_void,\n            );\n            compiler_fence(Ordering::SeqCst);\n        }\n    }\n\n    /// This function marks the boundary between the fuzzer and the target\n    #[inline]\n    pub fn leave_target<EM, Z>(\n        &mut self,\n        _fuzzer: &mut Z,\n        _state: &mut <Self as UsesState>::State,\n        _mgr: &mut EM,\n        _input: &<Self as UsesInput>::Input,\n    ) {\n        unsafe {\n            let data = addr_of_mut!(GLOBAL_STATE);\n\n            write_volatile(addr_of_mut!((*data).current_input_ptr), null());\n            compiler_fence(Ordering::SeqCst);\n        }\n    }\n}\n\nimpl<HT, OT, S> GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: HasExecutions + HasSolutions + HasCorpus + State,\n{\n    /// Create a new in mem executor with the default timeout (5 sec)\n    pub fn generic<E, EM, OF, Z>(\n        user_hooks: HT,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n    ) -> Result<Self, Error>\n    where\n        E: Executor<EM, Z, State = S> + HasObservers + HasInProcessHooks<S>,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        Self::with_timeout_generic::<E, EM, OF, Z>(\n            user_hooks,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            Duration::from_millis(5000),\n        )\n    }\n\n    /// Create a new in mem executor with the default timeout and use batch mode(5 sec)\n    #[cfg(all(feature = \"std\", target_os = \"linux\"))]\n    pub fn batched_timeout_generic<E, EM, OF, Z>(\n        user_hooks: HT,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        exec_tmout: Duration,\n    ) -> Result<Self, Error>\n    where\n        E: Executor<EM, Z, State = S> + HasObservers + HasInProcessHooks<S>,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let mut me = Self::with_timeout_generic::<E, EM, OF, Z>(\n            user_hooks, observers, fuzzer, state, event_mgr, exec_tmout,\n        )?;\n        me.hooks_mut().0.timer_mut().batch_mode = true;\n        Ok(me)\n    }\n\n    /// Create a new in mem executor.\n    /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used,\n    /// depending on different corpus or state.\n    /// * `user_hooks` - the hooks run before and after the harness's execution\n    /// * `harness_fn` - the harness, executing the function\n    /// * `observers` - the observers observing the target during execution\n    /// This may return an error on unix, if signal handler setup fails\n    pub fn with_timeout_generic<E, EM, OF, Z>(\n        user_hooks: HT,\n        observers: OT,\n        _fuzzer: &mut Z,\n        state: &mut S,\n        _event_mgr: &mut EM,\n        timeout: Duration,\n    ) -> Result<Self, Error>\n    where\n        E: Executor<EM, Z, State = S> + HasObservers + HasInProcessHooks<S>,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let default = InProcessHooks::new::<E, EM, OF, Z>(timeout)?;\n        let mut hooks = tuple_list!(default).merge(user_hooks);\n        hooks.init_all::<Self>(state);\n\n        #[cfg(windows)]\n        // Some initialization necessary for windows.\n        unsafe {\n            /*\n                See https://github.com/AFLplusplus/LibAFL/pull/403\n                This one reserves certain amount of memory for the stack.\n                If stack overflow happens during fuzzing on windows, the program is transferred to our exception handler for windows.\n                However, if we run out of the stack memory again in this exception handler, we'll crash with STATUS_ACCESS_VIOLATION.\n                We need this API call because with the llmp_compression\n                feature enabled, the exception handler uses a lot of stack memory (in the compression lib code) on release build.\n                As far as I have observed, the compression uses around 0x10000 bytes, but for safety let's just reserve 0x20000 bytes for our exception handlers.\n                This number 0x20000 could vary depending on the compilers optimization for future compression library changes.\n            */\n            let mut stack_reserved = 0x20000;\n            SetThreadStackGuarantee(&mut stack_reserved)?;\n        }\n\n        #[cfg(all(feature = \"std\", windows))]\n        {\n            // set timeout for the handler\n            *hooks.0.millis_sec_mut() = timeout.as_millis() as i64;\n        }\n\n        Ok(Self {\n            observers,\n            hooks,\n            phantom: PhantomData,\n        })\n    }\n\n    /// The inprocess handlers\n    #[inline]\n    pub fn hooks(&self) -> &(InProcessHooks<S>, HT) {\n        &self.hooks\n    }\n\n    /// The inprocess handlers (mutable)\n    #[inline]\n    pub fn hooks_mut(&mut self) -> &mut (InProcessHooks<S>, HT) {\n        &mut self.hooks\n    }\n}\n\nimpl<HT, OT, S> HasInProcessHooks<S> for GenericInProcessExecutorInner<HT, OT, S>\nwhere\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State + HasExecutions + HasSolutions + HasCorpus,\n{\n    /// the timeout handler\n    #[inline]\n    fn inprocess_hooks(&self) -> &InProcessHooks<S> {\n        &self.hooks.0\n    }\n\n    /// the timeout handler\n    #[inline]\n    fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks<S> {\n        &mut self.hooks.0\n    }\n}\n\n// Path: libafl/src/executors/inprocess/stateful.rs\nuse alloc::boxed::Box;\nuse core::{\n    borrow::BorrowMut,\n    ffi::c_void,\n    fmt::{self, Debug, Formatter},\n    marker::PhantomData,\n    ptr,\n    time::Duration,\n};\n\nuse libafl_bolts::tuples::tuple_list;\n\nuse crate::{\n    events::{EventFirer, EventRestarter},\n    executors::{\n        hooks::{inprocess::InProcessHooks, ExecutorHooksTuple},\n        inprocess::{GenericInProcessExecutorInner, HasInProcessHooks},\n        Executor, ExitKind, HasObservers,\n    },\n    feedbacks::Feedback,\n    fuzzer::HasObjective,\n    inputs::UsesInput,\n    observers::{ObserversTuple, UsesObservers},\n    state::{HasCorpus, HasExecutions, HasSolutions, State, UsesState},\n    Error,\n};\n\n/// The process executor simply calls a target function, as mutable reference to a closure\n/// The internal state of the executor is made available to the harness.\npub type StatefulInProcessExecutor<'a, H, OT, S, ES> =\n    StatefulGenericInProcessExecutor<H, &'a mut H, (), OT, S, ES>;\n\n/// The process executor simply calls a target function, as boxed `FnMut` trait object\n/// The internal state of the executor is made available to the harness.\npub type OwnedInProcessExecutor<OT, S, ES> = StatefulGenericInProcessExecutor<\n    dyn FnMut(&<S as UsesInput>::Input, &mut ES) -> ExitKind,\n    Box<dyn FnMut(&<S as UsesInput>::Input, &mut ES) -> ExitKind>,\n    (),\n    OT,\n    S,\n    ES,\n>;\n\n/// The inmem executor simply calls a target function, then returns afterwards.\n/// The harness can access the internal state of the executor.\n#[allow(dead_code)]\npub struct StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    /// The harness function, being executed for each fuzzing loop execution\n    harness_fn: HB,\n    /// The state used as argument of the harness\n    pub exposed_executor_state: ES,\n    /// Inner state of the executor\n    pub inner: GenericInProcessExecutorInner<HT, OT, S>,\n    phantom: PhantomData<(ES, *const H)>,\n}\n\nimpl<H, HB, HT, OT, S, ES> Debug for StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S> + Debug,\n    S: State,\n{\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"StatefulGenericInProcessExecutor\")\n            .field(\"harness_fn\", &\"<fn>\")\n            .field(\"inner\", &self.inner)\n            .finish_non_exhaustive()\n    }\n}\n\nimpl<H, HB, HT, OT, S, ES> UsesState for StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    type State = S;\n}\n\nimpl<H, HB, HT, OT, S, ES> UsesObservers for StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    type Observers = OT;\n}\n\nimpl<EM, H, HB, HT, OT, S, Z, ES> Executor<EM, Z>\n    for StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    EM: UsesState<State = S>,\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State + HasExecutions,\n    Z: UsesState<State = S>,\n{\n    fn run_target(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        mgr: &mut EM,\n        input: &Self::Input,\n    ) -> Result<ExitKind, Error> {\n        *state.executions_mut() += 1;\n        unsafe {\n            let executor_ptr = ptr::from_ref(self) as *const c_void;\n            self.inner\n                .enter_target(fuzzer, state, mgr, input, executor_ptr);\n        }\n        self.inner.hooks.pre_exec_all(state, input);\n\n        let ret = (self.harness_fn.borrow_mut())(input, &mut self.exposed_executor_state);\n\n        self.inner.hooks.post_exec_all(state, input);\n        self.inner.leave_target(fuzzer, state, mgr, input);\n        Ok(ret)\n    }\n}\n\nimpl<H, HB, HT, OT, S, ES> HasObservers for StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        self.inner.observers()\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        self.inner.observers_mut()\n    }\n}\n\nimpl<'a, H, OT, S, ES> StatefulInProcessExecutor<'a, H, OT, S, ES>\nwhere\n    H: FnMut(&<S as UsesInput>::Input, &mut ES) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: HasExecutions + HasSolutions + HasCorpus + State,\n{\n    /// Create a new in mem executor with the default timeout (5 sec)\n    pub fn new<EM, OF, Z>(\n        harness_fn: &'a mut H,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S>,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        Self::with_timeout_generic(\n            tuple_list!(),\n            harness_fn,\n            exposed_executor_state,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            Duration::from_millis(5000),\n        )\n    }\n\n    /// Create a new in mem executor with the default timeout and use batch mode(5 sec)\n    #[cfg(all(feature = \"std\", target_os = \"linux\"))]\n    pub fn batched_timeout<EM, OF, Z>(\n        harness_fn: &'a mut H,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        exec_tmout: Duration,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S>,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::batched_timeout_generic::<Self, EM, OF, Z>(\n            tuple_list!(),\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            exec_tmout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            exposed_executor_state,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Create a new in mem executor.\n    /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used,\n    /// depending on different corpus or state.\n    /// * `user_hooks` - the hooks run before and after the harness's execution\n    /// * `harness_fn` - the harness, executing the function\n    /// * `observers` - the observers observing the target during execution\n    /// This may return an error on unix, if signal handler setup fails\n    pub fn with_timeout<EM, OF, Z>(\n        harness_fn: &'a mut H,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S>,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::with_timeout_generic::<Self, EM, OF, Z>(\n            tuple_list!(),\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            timeout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            exposed_executor_state,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n}\n\nimpl<H, HB, HT, OT, S, ES> StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    /// The executor state given to the harness\n    \npub fn exposed_executor_state(&self) -> &ES {\n        &self.exposed_executor_state\n    }\n\n    /// The mutable executor state given to the harness\n    pub fn exposed_executor_state_mut(&mut self) -> &mut ES {\n        &mut self.exposed_executor_state\n    }\n}\n\nimpl<H, HB, HT, OT, S, ES> StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&S::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State + HasExecutions + HasSolutions + HasCorpus,\n{\n    /// Create a new in mem executor with the default timeout (5 sec)\n    pub fn generic<EM, OF, Z>(\n        user_hooks: HT,\n        harness_fn: HB,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n    ) -> Result<Self, Error>\n    where\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        Self::with_timeout_generic(\n            user_hooks,\n            harness_fn,\n            exposed_executor_state,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            Duration::from_millis(5000),\n        )\n    }\n\n    /// Create a new in mem executor with the default timeout and use batch mode(5 sec)\n    #[cfg(all(feature = \"std\", target_os = \"linux\"))]\n    #[allow(clippy::too_many_arguments)]\n    pub fn batched_timeout_generic<EM, OF, Z>(\n        user_hooks: HT,\n        harness_fn: HB,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        exec_tmout: Duration,\n    ) -> Result<Self, Error>\n    where\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::batched_timeout_generic::<Self, EM, OF, Z>(\n            user_hooks, observers, fuzzer, state, event_mgr, exec_tmout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            exposed_executor_state,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Create a new in mem executor.\n    /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used,\n    /// depending on different corpus or state.\n    /// * `user_hooks` - the hooks run before and after the harness's execution\n    /// * `harness_fn` - the harness, executing the function\n    /// * `observers` - the observers observing the target during execution\n    /// This may return an error on unix, if signal handler setup fails\n    #[allow(clippy::too_many_arguments)]\n    pub fn with_timeout_generic<EM, OF, Z>(\n        user_hooks: HT,\n        harness_fn: HB,\n        exposed_executor_state: ES,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n    ) -> Result<Self, Error>\n    where\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::with_timeout_generic::<Self, EM, OF, Z>(\n            user_hooks, observers, fuzzer, state, event_mgr, timeout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            exposed_executor_state,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Retrieve the harness function.\n    #[inline]\n    pub fn harness(&self) -> &H {\n        self.harness_fn.borrow()\n    }\n\n    /// Retrieve the harness function for a mutable reference.\n    #[inline]\n    pub fn harness_mut(&mut self) -> &mut H {\n        self.harness_fn.borrow_mut()\n    }\n\n    /// The inprocess handlers\n    #[inline]\n    pub fn hooks(&self) -> &(InProcessHooks<S>, HT) {\n        self.inner.hooks()\n    }\n\n    /// The inprocess handlers (mutable)\n    #[inline]\n    pub fn hooks_mut(&mut self) -> &mut (InProcessHooks<S>, HT) {\n        self.inner.hooks_mut()\n    }\n}\n\nimpl<H, HB, HT, OT, S, ES> HasInProcessHooks<S>\n    for StatefulGenericInProcessExecutor<H, HB, HT, OT, S, ES>\nwhere\n    H: FnMut(&<S as UsesInput>::Input, &mut ES) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State + HasExecutions + HasSolutions + HasCorpus,\n{\n    /// the timeout handler\n    #[inline]\n    fn inprocess_hooks(&self) -> &InProcessHooks<S> {\n        self.inner.inprocess_hooks()\n    }\n\n    /// the timeout handler\n    #[inline]\n    fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks<S> {\n        self.inner.inprocess_hooks_mut()\n    }\n}\n\n// Path: libafl/src/executors/inprocess/mod.rs\n//! The [`InProcessExecutor`] is a libfuzzer-like executor, that will simply call a function.\n//! It should usually be paired with extra error-handling, such as a restarting event manager, to be effective.\n//!\n//! Needs the `fork` feature flag.\n#![allow(clippy::needless_pass_by_value)]\n\nuse alloc::boxed::Box;\n#[cfg(any(unix, feature = \"std\"))]\nuse core::ptr::addr_of_mut;\nuse core::{\n    borrow::BorrowMut,\n    ffi::c_void,\n    fmt::{self, Debug, Formatter},\n    marker::PhantomData,\n    ptr,\n    time::Duration,\n};\n\nuse libafl_bolts::tuples::tuple_list;\n\n#[cfg(any(unix, feature = \"std\"))]\nuse crate::executors::hooks::inprocess::GLOBAL_STATE;\nuse crate::{\n    corpus::{Corpus, Testcase},\n    events::{Event, EventFirer, EventRestarter},\n    executors::{\n        hooks::{inprocess::InProcessHooks, ExecutorHooksTuple},\n        inprocess::inner::GenericInProcessExecutorInner,\n        Executor, ExitKind, HasObservers,\n    },\n    feedbacks::Feedback,\n    fuzzer::HasObjective,\n    inputs::UsesInput,\n    observers::{ObserversTuple, UsesObservers},\n    state::{HasCorpus, HasExecutions, HasMetadata, HasSolutions, State, UsesState},\n    Error,\n};\n\n/// The inner structure of `InProcessExecutor`.\npub mod inner;\n/// A version of `InProcessExecutor` with a state accessible from the harness.\npub mod stateful;\n\n/// The process executor simply calls a target function, as mutable reference to a closure.\npub type InProcessExecutor<'a, H, OT, S> = GenericInProcessExecutor<H, &'a mut H, (), OT, S>;\n\n/// The inprocess executor that allows hooks\npub type HookableInProcessExecutor<'a, H, HT, OT, S> =\n    GenericInProcessExecutor<H, &'a mut H, HT, OT, S>;\n/// The process executor simply calls a target function, as boxed `FnMut` trait object\npub type OwnedInProcessExecutor<OT, S> = GenericInProcessExecutor<\n    dyn FnMut(&<S as UsesInput>::Input) -> ExitKind,\n    Box<dyn FnMut(&<S as UsesInput>::Input) -> ExitKind>,\n    (),\n    OT,\n    S,\n>;\n\n/// The inmem executor simply calls a target function, then returns afterwards.\n#[allow(dead_code)]\npub struct GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    harness_fn: HB,\n    inner: GenericInProcessExecutorInner<HT, OT, S>,\n    phantom: PhantomData<(*const H, HB)>,\n}\n\nimpl<H, HB, HT, OT, S> Debug for GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S> + Debug,\n    S: State,\n{\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        f.debug_struct(\"GenericInProcessExecutor\")\n            .field(\"inner\", &self.inner)\n            .field(\"harness_fn\", &\"<fn>\")\n            .finish_non_exhaustive()\n    }\n}\n\nimpl<H, HB, HT, OT, S> UsesState for GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    type State = S;\n}\n\nimpl<H, HB, HT, OT, S> UsesObservers for GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    type Observers = OT;\n}\n\nimpl<EM, H, HB, HT, OT, S, Z> Executor<EM, Z> for GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    EM: UsesState<State = S>,\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State + HasExecutions,\n    Z: UsesState<State = S>,\n{\n    fn run_target(\n        &mut self,\n        fuzzer: &mut Z,\n        state: &mut Self::State,\n        mgr: &mut EM,\n        input: &Self::Input,\n    ) -> Result<ExitKind, Error> {\n        *state.executions_mut() += 1;\n        unsafe {\n            let executor_ptr = ptr::from_ref(self) as *const c_void;\n            self.inner\n                .enter_target(fuzzer, state, mgr, input, executor_ptr);\n        }\n        self.inner.hooks.pre_exec_all(state, input);\n\n        let ret = (self.harness_fn.borrow_mut())(input);\n\n        self.inner.hooks.post_exec_all(state, input);\n        self.inner.leave_target(fuzzer, state, mgr, input);\n        Ok(ret)\n    }\n}\n\nimpl<H, HB, HT, OT, S> HasObservers for GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State,\n{\n    #[inline]\n    fn observers(&self) -> &OT {\n        self.inner.observers()\n    }\n\n    #[inline]\n    fn observers_mut(&mut self) -> &mut OT {\n        self.inner.observers_mut()\n    }\n}\n\nimpl<'a, H, OT, S> InProcessExecutor<'a, H, OT, S>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    OT: ObserversTuple<S>,\n    S: HasExecutions + HasSolutions + HasCorpus + State,\n{\n    /// Create a new in mem executor with the default timeout (5 sec)\n    pub fn new<EM, OF, Z>(\n        harness_fn: &'a mut H,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S> + HasObservers,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        Self::with_timeout_generic(\n            tuple_list!(),\n            harness_fn,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            Duration::from_millis(5000),\n        )\n    }\n\n    /// Create a new in mem executor with the default timeout and use batch mode(5 sec)\n    #[cfg(all(feature = \"std\", target_os = \"linux\"))]\n    pub fn batched_timeout<EM, OF, Z>(\n        harness_fn: &'a mut H,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        exec_tmout: Duration,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S>,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::batched_timeout_generic::<Self, EM, OF, Z>(\n            tuple_list!(),\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            exec_tmout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Create a new in mem executor.\n    /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used,\n    /// depending on different corpus or state.\n    /// * `user_hooks` - the hooks run before and after the harness's execution\n    /// * `harness_fn` - the harness, executing the function\n    /// * `observers` - the observers observing the target during execution\n    /// This may return an error on unix, if signal handler setup fails\n    pub fn with_timeout<EM, OF, Z>(\n        harness_fn: &'a mut H,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S> + HasObservers,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::with_timeout_generic::<Self, EM, OF, Z>(\n            tuple_list!(),\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            timeout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n}\n\nimpl<H, HB, HT, OT, S> GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    H: FnMut(&S::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State + HasExecutions + HasSolutions + HasCorpus,\n{\n    /// Create a new in mem executor with the default timeout (5 sec)\n    pub fn generic<EM, OF, Z>(\n        user_hooks: HT,\n        harness_fn: HB,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S> + HasObservers,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        Self::with_timeout_generic(\n            user_hooks,\n            harness_fn,\n            observers,\n            fuzzer,\n            state,\n            event_mgr,\n            Duration::from_millis(5000),\n        )\n    }\n\n    /// Create a new in mem executor with the default timeout and use batch mode(5 sec)\n    #[cfg(all(feature = \"std\", target_os = \"linux\"))]\n    pub fn batched_timeout_generic<EM, OF, Z>(\n        user_hooks: HT,\n        harness_fn: HB,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        exec_tmout: Duration,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S> + HasObservers,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::batched_timeout_generic::<Self, EM, OF, Z>(\n            user_hooks, observers, fuzzer, state, event_mgr, exec_tmout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Create a new in mem executor.\n    /// Caution: crash and restart in one of them will lead to odd behavior if multiple are used,\n    /// depending on different corpus or state.\n    /// * `user_hooks` - the hooks run before and after the harness's execution\n    /// * `harness_fn` - the harness, executing the function\n    /// * `observers` - the observers observing the target during execution\n    /// This may return an error on unix, if signal handler setup fails\n    pub fn with_timeout_generic<EM, OF, Z>(\n        user_hooks: HT,\n        harness_fn: HB,\n        observers: OT,\n        fuzzer: &mut Z,\n        state: &mut S,\n        event_mgr: &mut EM,\n        timeout: Duration,\n    ) -> Result<Self, Error>\n    where\n        Self: Executor<EM, Z, State = S> + HasObservers,\n        EM: EventFirer<State = S> + EventRestarter,\n        OF: Feedback<S>,\n        S: State,\n        Z: HasObjective<Objective = OF, State = S>,\n    {\n        let inner = GenericInProcessExecutorInner::with_timeout_generic::<Self, EM, OF, Z>(\n            user_hooks, observers, fuzzer, state, event_mgr, timeout,\n        )?;\n\n        Ok(Self {\n            harness_fn,\n            inner,\n            phantom: PhantomData,\n        })\n    }\n\n    /// Retrieve the harness function.\n    #[inline]\n    pub fn harness(&self) -> &H {\n        self.harness_fn.borrow()\n    }\n\n    /// Retrieve the harness function for a mutable reference.\n    #[inline]\n    pub fn harness_mut(&mut self) -> &mut H {\n        self.harness_fn.borrow_mut()\n    }\n\n    /// The inprocess handlers\n    #[inline]\n    pub fn hooks(&self) -> &(InProcessHooks<S>, HT) {\n        self.inner.hooks()\n    }\n\n    /// The inprocess handlers (mutable)\n    #[inline]\n    pub fn hooks_mut(&mut self) -> &mut (InProcessHooks<S>, HT) {\n        self.inner.hooks_mut()\n    }\n}\n\n/// The struct has [`InProcessHooks`].\npub trait HasInProcessHooks<S>\nwhere\n    S: UsesInput,\n{\n    /// Get the in-process handlers.\n    fn inprocess_hooks(&self) -> &InProcessHooks<S>;\n\n    /// Get the mut in-process handlers.\n    fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks<S>;\n}\n\nimpl<H, HB, HT, OT, S> HasInProcessHooks<S> for GenericInProcessExecutor<H, HB, HT, OT, S>\nwhere\n    H: FnMut(&<S as UsesInput>::Input) -> ExitKind + ?Sized,\n    HB: BorrowMut<H>,\n    HT: ExecutorHooksTuple<S>,\n    OT: ObserversTuple<S>,\n    S: State + HasExecutions + HasSolutions + HasCorpus,\n{\n    /// the timeout handler\n    #[inline]\n    fn inprocess_hooks(&self) -> &InProcessHooks<S> {\n        self.inner.inprocess_hooks()\n    }\n\n    /// the timeout handler\n    #[inline]\n    fn inprocess_hooks_mut(&mut self) -> &mut InProcessHooks<S> {\n        self.inner.inprocess_hooks_mut()\n    }\n}\n\n#[inline]\n#[allow(clippy::too_many_arguments)]\n/// Save state if it is an objective\npub fn run_observers_and_save_state<E, EM, OF, Z>(\n    executor: &mut E,\n    state: &mut E::State,\n    input: &<E::State as UsesInput>::Input,\n    fuzzer: &mut Z,\n    event_mgr: &mut EM,\n    exitkind: ExitKind,\n) where\n    E: HasObservers,\n    EM: EventFirer<State = E::State> + EventRestarter<State = E::State>,\n    OF: Feedback<E::State>,\n    E::State: HasExecutions + HasSolutions + HasCorpus,\n    Z: HasObjective<Objective = OF, State = E::State>,\n{\n    let observers = executor.observers_mut();\n\n    observers\n        .post_exec_all(state, input, &exitkind)\n        .expect(\"Observers post_exec_all failed\");\n\n    let interesting = fuzzer\n        .objective_mut()\n        .is_interesting(state, event_mgr, input, observers, &exitkind)\n        .expect(\"In run_observers_and_save_state objective failure.\");\n\n    if interesting {\n        let executions = *state.executions();\n        let mut new_testcase = Testcase::with_executions(input.clone(), executions);\n        new_testcase.add_metadata(exitkind);\n        new_testcase.set_parent_id_optional(*state.corpus().current());\n        fuzzer\n            .objective_mut()\n            .append_metadata(state, event_mgr, observers, &mut new_testcase)\n            .expect(\"Failed adding metadata\");\n        state\n            .solutions_mut()\n            .add(new_testcase)\n            .expect(\"In run_observers_and_save_state solutions failure.\");\n        event_mgr\n            .fire(\n                state,\n                Event::Objective {\n                    objective_size: state.solutions().count(),\n                    executions,\n                    time: libafl_bolts::current_time(),\n                },\n            )\n            .expect(\"Could not save state in run_observers_and_save_state\");\n    }\n\n    // Serialize the state and wait safely for the broker to read pending messages\n    event_mgr.on_restart(state).unwrap();\n\n    log::info!(\"Bye!\");\n}\n\n// TODO remove this after executor refactor and libafl qemu new executor\n/// Expose a version of the crash handler that can be called from e.g. an emulator\n///\n/// # Safety\n/// This will directly access `GLOBAL_STATE` and related data pointers\n#[cfg(any(unix, feature = \"std\"))]\npub unsafe fn generic_inproc_crash_handler<E, EM, OF, Z>()\nwhere\n    E: Executor<EM, Z> + HasObservers,\n    EM: EventFirer<State = E::State> + EventRestarter<State = E::State>,\n    OF: Feedback<E::State>,\n    E::State: HasExecutions + HasSolutions + HasCorpus,\n    Z: HasObjective<Objective = OF, State = E::State>,\n{\n    let data = addr_of_mut!(GLOBAL_STATE);\n    let in_handler = (*data).set_in_handler(true);\n\n    if (*data).is_valid() {\n        let executor = (*data).executor_mut::<E>();\n        let state = (*data).state_mut::<E::State>();\n        let event_mgr = (*data).event_mgr_mut::<EM>();\n        let fuzzer = (*data).fuzzer_mut::<Z>();\n        let input = (*data).take_current_input::<<E::State as UsesInput>::Input>();\n\n        run_observers_and_save_state::<E, EM, OF, Z>(\n            executor,\n            state,\n            input,\n            fuzzer,\n            event_mgr,\n            ExitKind::Crash,\n        );\n    }\n\n    (*data).set_in_handler(in_handler);\n}\n\n#[cfg(test)]\nmod tests {\n    use libafl_bolts::tuples::tuple_list;\n\n    use crate::{\n        corpus::InMemoryCorpus,\n        events::NopEventManager,\n        executors::{Executor, ExitKind, InProcessExecutor},\n        feedbacks::CrashFeedback,\n        inputs::{NopInput, UsesInput},\n        schedulers::RandScheduler,\n        state::StdState,\n        StdFuzzer,\n    };\n\n    impl UsesInput for () {\n        type Input = NopInput;\n    }\n\n    #[test]\n    #[allow(clippy::let_unit_value)]\n    fn test_inmem_exec() {\n        let mut harness = |_buf: &NopInput| ExitKind::Ok;\n        let rand = libafl_bolts::rands::XkcdRand::new();\n        let corpus = InMemoryCorpus::<NopInput>::new();\n        let solutions = InMemoryCorpus::new();\n...\n", "needle_token_start": 8951, "needle_token_end": 8981, "code_context_ntokens": 16287, "output": ["```rust\n/// The executor state given to the harness\npub fn exposed_executor_state(&self) -> &ES {\n    &self.exposed_executor_state\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "rtn_extend_encoding", "language": "rust", "path": "libafl/src/mutators/token_mutations.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to extend the encoding of input patterns based on certain conditions and comparisons. It aims to modify and potentially expand the input data by applying transformations and checking against patterns.\n2. **Input**: The function takes multiple byte arrays representing different patterns and buffers, indices indicating positions within these buffers, lengths of certain data segments, and a vector to store results.\n3. **Output**: Returns a boolean value indicating whether any modifications were successfully applied to the input data.\n4. **Procedure**: The function calculates the minimum length to consider based on input constraints and iterates over the input patterns and buffers up to this length. If discrepancies are found between the patterns and the corresponding buffer data, the function breaks out of the loop. If matching data is found, it copies and potentially modifies this data into a new buffer, which is then added to a result vector. The function concludes by returning true if any data was copied, otherwise false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    tokens_vec: Vec<Vec<u8>>,\n    tokens_set: HashSet<Vec<u8>>,\n}\n\nlibafl_bolts::impl_serdeany!(Tokens);\n\n/// The metadata used for token mutators\nimpl Tokens {\n    /// Creates a new tokens metadata (old-skool afl name: `dictornary`)\n    #[must_use]\n    pub fn new() -> Self {\n        Tokens::default()\n    }\n\n    /// Add tokens from a slice of Vecs of bytes\n    pub fn add_tokens<IT, V>(&mut self, tokens: IT) -> &mut Self\n    where\n        IT: IntoIterator<Item = V>,\n        V: AsRef<Vec<u8>>,\n    {\n        for token in tokens {\n            self.add_token(token.as_ref());\n        }\n        self\n    }\n\n    /// Build tokens from files\n    #[cfg(feature = \"std\")]\n    pub fn add_from_files<IT, P>(mut self, files: IT) -> Result<Self, Error>\n    where\n        IT: IntoIterator<Item = P>,\n        P: AsRef<Path>,\n    {\n        for file in files {\n            self.add_from_file(file)?;\n        }\n        Ok(self)\n    }\n\n    /// Parse autodict section\n    pub fn parse_autodict(&mut self, slice: &[u8], size: usize) {\n        let mut head = 0;\n        loop {\n            if head >= size {\n                // Sanity Check\n                assert!(head == size);\n                break;\n            }\n            let size = slice[head] as usize;\n            head += 1;\n            if size > 0 {\n                self.add_token(&slice[head..head + size].to_vec());\n                log::info!(\n                    \"Token size: {} content: {:x?}\",\n                    size,\n                    &slice[head..head + size].to_vec()\n                );\n                head += size;\n            }\n        }\n    }\n\n    /// Create a token section from a start and an end pointer\n    /// Reads from an autotokens section, returning the count of new entries read\n    ///\n    /// # Safety\n    /// The caller must ensure that the region between `token_start` and `token_stop`\n    /// is a valid region, containing autotokens in the expected format.\n    #[cfg(any(target_os = \"linux\", target_vendor = \"apple\"))]\n    pub unsafe fn from_mut_ptrs(\n        token_start: *const u8,\n        token_stop: *const u8,\n    ) -> Result<Self, Error> {\n        let mut ret = Self::default();\n        if token_start.is_null() || token_stop.is_null() {\n            return Ok(Self::new());\n        }\n        if token_stop < token_start {\n            return Err(Error::illegal_argument(format!(\n                \"Tried to create tokens from illegal section: stop < start ({token_stop:?} < {token_start:?})\"\n            )));\n        }\n        let section_size: usize = token_stop.offset_from(token_start).try_into().unwrap();\n        // log::info!(\"size: {}\", section_size);\n        let slice = from_raw_parts(token_start, section_size);\n\n        // Now we know the beginning and the end of the token section.. let's parse them into tokens\n        ret.parse_autodict(slice, section_size);\n\n        Ok(ret)\n    }\n\n    /// Creates a new instance from a file\n    #[cfg(feature = \"std\")]\n    pub fn from_file<P>(file: P) -> Result<Self, Error>\n    where\n        P: AsRef<Path>,\n    {\n        let mut ret = Self::new();\n        ret.add_from_file(file)?;\n        Ok(ret)\n    }\n\n    /// Adds a token to a dictionary, checking it is not a duplicate\n    /// Returns `false` if the token was already present and did not get added.\n    #[allow(clippy::ptr_arg)]\n    pub fn add_token(&mut self, token: &Vec<u8>) -> bool {\n        if !self.tokens_set.insert(token.clone()) {\n            return false;\n        }\n        self.tokens_vec.push(token.clone());\n        true\n    }\n\n    /// Reads a tokens file, returning the count of new entries read\n    #[cfg(feature = \"std\")]\n    pub fn add_from_file<P>(&mut self, file: P) -> Result<&mut Self, Error>\n    where\n        P: AsRef<Path>,\n    {\n        // log::info!(\"Loading tokens file {:?} ...\", file);\n\n        let file = File::open(file)?; // panic if not found\n        let reader = BufReader::new(file);\n\n        for line in reader.lines() {\n            let line = line.unwrap();\n            let line = line.trim_start().trim_end();\n\n            // we are only interested in '\"...\"', not prefixed 'foo = '\n            let start = line.chars().next();\n            if line.is_empty() || start == Some('#') {\n                continue;\n            }\n            let Some(pos_quote) = line.find('\\\"') else {\n                return Err(Error::illegal_argument(format!(\"Illegal line: {line}\")));\n            };\n            if line.chars().nth(line.len() - 1) != Some('\"') {\n                return Err(Error::illegal_argument(format!(\"Illegal line: {line}\")));\n            }\n\n            // extract item\n            let Some(item) = line.get(pos_quote + 1..line.len() - 1) else {\n                return Err(Error::illegal_argument(format!(\"Illegal line: {line}\")));\n            };\n            if item.is_empty() {\n                continue;\n            }\n\n            // decode\n            let token: Vec<u8> = match str_decode(item) {\n                Ok(val) => val,\n                Err(_) => {\n                    return Err(Error::illegal_argument(format!(\n                        \"Illegal line (hex decoding): {line}\"\n                    )))\n                }\n            };\n\n            // add\n            self.add_token(&token);\n        }\n\n        Ok(self)\n    }\n\n    /// Returns the amount of tokens in this Tokens instance\n    #[inline]\n    #[must_use]\n    pub fn len(&self) -> usize {\n        self.tokens_vec.len()\n    }\n\n    /// Returns if this tokens-instance is empty\n    #[inline]\n    #[must_use]\n    pub fn is_empty(&self) -> bool {\n        self.tokens_vec.is_empty()\n    }\n\n    /// Gets the tokens stored in this db\n    #[must_use]\n    pub fn tokens(&self) -> &[Vec<u8>] {\n        &self.tokens_vec\n    }\n\n    /// Returns an iterator over the tokens.\n    pub fn iter(&self) -> Iter<'_, Vec<u8>> {\n        <&Self as IntoIterator>::into_iter(self)\n    }\n}\n\nimpl AddAssign for Tokens {\n    fn add_assign(&mut self, other: Self) {\n        self.add_tokens(&other);\n    }\n}\n\nimpl AddAssign<&[Vec<u8>]> for Tokens {\n    fn add_assign(&mut self, other: &[Vec<u8>]) {\n        self.add_tokens(other);\n    }\n}\n\nimpl Add<&[Vec<u8>]> for Tokens {\n    type Output = Self;\n    fn add(self, other: &[Vec<u8>]) -> Self {\n        let mut ret = self;\n        ret.add_tokens(other);\n        ret\n    }\n}\n\nimpl Add for Tokens {\n    type Output = Self;\n\n    fn add(self, other: Self) -> Self {\n        self.add(other.tokens_vec.as_slice())\n    }\n}\n\nimpl<IT, V> From<IT> for Tokens\nwhere\n    IT: IntoIterator<Item = V>,\n    V: AsRef<Vec<u8>>,\n{\n    fn from(tokens: IT) -> Self {\n        let mut ret = Self::default();\n        ret.add_tokens(tokens);\n        ret\n    }\n}\n\nimpl AsSlice for Tokens {\n    type Entry = Vec<u8>;\n    fn as_slice(&self) -> &[Vec<u8>] {\n        self.tokens()\n    }\n}\n\nimpl Add for &Tokens {\n    type Output = Tokens;\n\n    fn add(self, other: Self) -> Tokens {\n        let mut ret: Tokens = self.clone();\n        ret.add_tokens(other);\n        ret\n    }\n}\n\nimpl<'it> IntoIterator for &'it Tokens {\n    type Item = <Iter<'it, Vec<u8>> as Iterator>::Item;\n    type IntoIter = Iter<'it, Vec<u8>>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        self.as_slice().iter()\n    }\n}\n\n/// Inserts a random token at a random position in the `Input`.\n#[derive(Debug, Default)]\npub struct TokenInsert;\n\nimpl<I, S> Mutator<I, S> for TokenInsert\nwhere\n    S: HasMetadata + HasRand + HasMaxSize,\n    I: HasBytesVec,\n{\n    fn mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        let max_size = state.max_size();\n        let tokens_len = {\n            let Some(meta) = state.metadata_map().get::<Tokens>() else {\n                return Ok(MutationResult::Skipped);\n            };\n            if meta.tokens().is_empty() {\n                return Ok(MutationResult::Skipped);\n            }\n            meta.tokens().len()\n        };\n        let token_idx = state.rand_mut().below(tokens_len as u64) as usize;\n\n        let size = input.bytes().len();\n        let off = state.rand_mut().below((size + 1) as u64) as usize;\n\n        let meta = state.metadata_map().get::<Tokens>().unwrap();\n        let token = &meta.tokens()[token_idx];\n        let mut len = token.len();\n\n        if size + len > max_size {\n            if max_size > size {\n                len = max_size - size;\n            } else {\n                return Ok(MutationResult::Skipped);\n            }\n        }\n\n        input.bytes_mut().resize(size + len, 0);\n        unsafe {\n            buffer_self_copy(input.bytes_mut(), off, off + len, size - off);\n            buffer_copy(input.bytes_mut(), token, 0, off, len);\n        }\n\n        Ok(MutationResult::Mutated)\n    }\n}\n\nimpl Named for TokenInsert {\n    fn name(&self) -> &str {\n        \"TokenInsert\"\n    }\n}\n\nimpl TokenInsert {\n    /// Create a `TokenInsert` `Mutation`.\n    #[must_use]\n    pub fn new() -> Self {\n        Self\n    }\n}\n\n/// A `TokenReplace` [`Mutator`] replaces a random part of the input with one of a range of tokens.\n/// From AFL terms, this is called as `Dictionary` mutation (which doesn't really make sense ;) ).\n#[derive(Debug, Default)]\npub struct TokenReplace;\n\nimpl<I, S> Mutator<I, S> for TokenReplace\nwhere\n    S: UsesInput + HasMetadata + HasRand + HasMaxSize,\n    I: HasBytesVec,\n{\n    fn mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        let size = input.bytes().len();\n        if size == 0 {\n            return Ok(MutationResult::Skipped);\n        }\n\n        let tokens_len = {\n            let Some(meta) = state.metadata_map().get::<Tokens>() else {\n                return Ok(MutationResult::Skipped);\n            };\n            if meta.tokens().is_empty() {\n                return Ok(MutationResult::Skipped);\n            }\n            meta.tokens().len()\n        };\n        let token_idx = state.rand_mut().below(tokens_len as u64) as usize;\n\n        let off = state.rand_mut().below(size as u64) as usize;\n\n        let meta = state.metadata_map().get::<Tokens>().unwrap();\n        let token = &meta.tokens()[token_idx];\n        let mut len = token.len();\n        if off + len > size {\n            len = size - off;\n        }\n\n        unsafe {\n            buffer_copy(input.bytes_mut(), token, 0, off, len);\n        }\n\n        Ok(MutationResult::Mutated)\n    }\n}\n\nimpl Named for TokenReplace {\n    fn name(&self) -> &str {\n        \"TokenReplace\"\n    }\n}\n\nimpl TokenReplace {\n    /// Creates a new `TokenReplace` struct.\n    #[must_use]\n    pub fn new() -> Self {\n        Self\n    }\n}\n\n/// A `I2SRandReplace` [`Mutator`] replaces a random matching input-2-state comparison operand with the other.\n/// It needs a valid [`CmpValuesMetadata`] in the state.\n#[derive(Debug, Default)]\npub struct I2SRandReplace;\n\nimpl<I, S> Mutator<I, S> for I2SRandReplace\nwhere\n    S: UsesInput + HasMetadata + HasRand + HasMaxSize,\n    I: HasBytesVec,\n{\n    #[allow(clippy::too_many_lines)]\n    fn mutate(&mut self, state: &mut S, input: &mut I) -> Result<MutationResult, Error> {\n        let size = input.bytes().len();\n        if size == 0 {\n            return Ok(MutationResult::Skipped);\n        }\n\n        let cmps_len = {\n            let Some(meta) = state.metadata_map().get::<CmpValuesMetadata>() else {\n                return Ok(MutationResult::Skipped);\n            };\n            log::trace!(\"meta: {:x?}\", meta);\n            if meta.list.is_empty() {\n                return Ok(MutationResult::Skipped);\n            }\n            meta.list.len()\n        };\n        let idx = state.rand_mut().below(cmps_len as u64) as usize;\n\n        let off = state.rand_mut().below(size as u64) as usize;\n        let len = input.bytes().len();\n        let bytes = input.bytes_mut();\n\n        let meta = state.metadata_map().get::<CmpValuesMetadata>().unwrap();\n        let cmp_values = &meta.list[idx];\n\n        let mut result = MutationResult::Skipped;\n        match cmp_values {\n            CmpValues::U8(v) => {\n                for byte in bytes.iter_mut().take(len).skip(off) {\n                    if *byte == v.0 {\n                        *byte = v.1;\n                        result = MutationResult::Mutated;\n                        break;\n                    } else if *byte == v.1 {\n                        *byte = v.0;\n                        result = MutationResult::Mutated;\n                        break;\n                    }\n                }\n            }\n            CmpValues::U16(v) => {\n                if len >= size_of::<u16>() {\n                    for i in off..len - (size_of::<u16>() - 1) {\n                        let val =\n                            u16::from_ne_bytes(bytes[i..i + size_of::<u16>()].try_into().unwrap());\n                        if val == v.0 {\n                            let new_bytes = v.1.to_ne_bytes();\n                            bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val.swap_bytes() == v.0 {\n                            let new_bytes = v.1.swap_bytes().to_ne_bytes();\n                            bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val == v.1 {\n                            let new_bytes = v.0.to_ne_bytes();\n                            bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val.swap_bytes() == v.1 {\n                            let new_bytes = v.0.swap_bytes().to_ne_bytes();\n                            bytes[i..i + size_of::<u16>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        }\n                    }\n                }\n            }\n            CmpValues::U32(v) => {\n                if len >= size_of::<u32>() {\n                    for i in off..len - (size_of::<u32>() - 1) {\n                        let val =\n                            u32::from_ne_bytes(bytes[i..i + size_of::<u32>()].try_into().unwrap());\n                        if val == v.0 {\n                            let new_bytes = v.1.to_ne_bytes();\n                            bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val.swap_bytes() == v.0 {\n                            let new_bytes = v.1.swap_bytes().to_ne_bytes();\n                            bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val == v.1 {\n                            let new_bytes = v.0.to_ne_bytes();\n                            bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val.swap_bytes() == v.1 {\n                            let new_bytes = v.0.swap_bytes().to_ne_bytes();\n                            bytes[i..i + size_of::<u32>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        }\n                    }\n                }\n            }\n            CmpValues::U64(v) => {\n                if len >= size_of::<u64>() {\n                    for i in off..len - (size_of::<u64>() - 1) {\n                        let val =\n                            u64::from_ne_bytes(bytes[i..i + size_of::<u64>()].try_into().unwrap());\n                        if val == v.0 {\n                            let new_bytes = v.1.to_ne_bytes();\n                            bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val.swap_bytes() == v.0 {\n                            let new_bytes = v.1.swap_bytes().to_ne_bytes();\n                            bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val == v.1 {\n                            let new_bytes = v.0.to_ne_bytes();\n                            bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        } else if val.swap_bytes() == v.1 {\n                            let new_bytes = v.0.swap_bytes().to_ne_bytes();\n                            bytes[i..i + size_of::<u64>()].copy_from_slice(&new_bytes);\n                            result = MutationResult::Mutated;\n                            break;\n                        }\n                    }\n                }\n            }\n            CmpValues::Bytes(v) => {\n                'outer: for i in off..len {\n                    let mut size = core::cmp::min(v.0.len(), len - i);\n                    while size != 0 {\n                        if v.0[0..size] == input.bytes()[i..i + size] {\n                            unsafe {\n                                buffer_copy(input.bytes_mut(), &v.1, 0, i, size);\n                            }\n                            result = MutationResult::Mutated;\n                            break 'outer;\n                        }\n                        size -= 1;\n                    }\n                    size = core::cmp::min(v.1.len(), len - i);\n                    while size != 0 {\n                        if v.1[0..size] == input.bytes()[i..i + size] {\n                            unsafe {\n                                buffer_copy(input.bytes_mut(), &v.0, 0, i, size);\n                            }\n                            result = MutationResult::Mutated;\n                            break 'outer;\n                        }\n                        size -= 1;\n                    }\n                }\n            }\n        }\n\n        Ok(result)\n    }\n}\n\nimpl Named for I2SRandReplace {\n    fn name(&self) -> &str {\n        \"I2SRandReplace\"\n    }\n}\n\nimpl I2SRandReplace {\n    /// Creates a new `I2SRandReplace` struct.\n    #[must_use]\n    pub fn new() -> Self {\n        Self\n    }\n}\n\nconst CMP_ATTTRIBUTE_IS_EQUAL: u8 = 1;\nconst CMP_ATTRIBUTE_IS_GREATER: u8 = 2;\nconst CMP_ATTRIBUTE_IS_LESSER: u8 = 4;\nconst CMP_ATTRIBUTE_IS_FP: u8 = 8;\nconst CMP_ATTRIBUTE_IS_FP_MOD: u8 = 16;\nconst CMP_ATTRIBUTE_IS_INT_MOD: u8 = 32;\nconst CMP_ATTRIBUTE_IS_TRANSFORM: u8 = 64;\n\n/// AFL++ redqueen mutation\n#[derive(Debug, Default)]\npub struct AFLppRedQueen {\n    enable_transform: bool,\n    enable_arith: bool,\n    text_type: TextType,\n    /// We use this variable to check if we scheduled a new `corpus_idx`\n    /// - and, hence, need to recalculate `text_type`\n    last_corpus_idx: Option<CorpusId>,\n}\n\nimpl AFLppRedQueen {\n    #[inline]\n    fn swapa(x: u8) -> u8 {\n        (x & 0xf8) + ((x & 7) ^ 0x07)\n    }\n\n    /// Cmplog Pattern Matching\n    #[allow(clippy::cast_sign_loss)]\n    #[allow(clippy::too_many_arguments)]\n    #[allow(clippy::too_many_lines)]\n    #[allow(clippy::cast_possible_wrap)]\n    #[allow(clippy::if_not_else)]\n    #[allow(clippy::cast_precision_loss)]\n    pub fn cmp_extend_encoding(\n        &self,\n        pattern: u64,\n        repl: u64,\n        another_pattern: u64,\n        changed_val: u64,\n        attr: u8,\n        another_buf: &[u8],\n        buf: &[u8],\n        buf_idx: usize,\n        taint_len: usize,\n        input_len: usize,\n        hshape: usize,\n        vec: &mut Vec<Vec<u8>>,\n    ) -> Result<bool, Error> {\n        // TODO: ascii2num (we need check q->is_ascii (in calibration stage(?)))\n\n        // try Transform\n        if self.enable_transform\n            && pattern != another_pattern\n            && repl == changed_val\n            && attr <= CMP_ATTTRIBUTE_IS_EQUAL\n        {\n            // Try to identify transform magic\n            let mut bytes: usize = match hshape {\n                0 => 0, // NEVER happen\n                1 => 1,\n                2 => 2,\n                3 | 4 => 4,\n                _ => 8,\n            };\n            // prevent overflow\n            bytes = core::cmp::min(bytes, input_len.wrapping_sub(buf_idx));\n\n            let (b_val, o_b_val, mask): (u64, u64, u64) = match bytes {\n                0 => {\n                    (0, 0, 0) // cannot happen\n                }\n                1 => (\n                    u64::from(buf[buf_idx]),\n                    u64::from(another_buf[buf_idx]),\n                    0xff,\n                ),\n                2 | 3 => (\n                    u64::from(u16::from_be_bytes(\n                        another_buf[buf_idx..buf_idx + 2].try_into().unwrap(),\n                    )),\n                    u64::from(u16::from_be_bytes(\n                        another_buf[buf_idx..buf_idx + 2].try_into().unwrap(),\n                    )),\n                    0xffff,\n                ),\n                4..=7 => (\n                    u64::from(u32::from_be_bytes(\n                        buf[buf_idx..buf_idx + 4].try_into().unwrap(),\n                    )),\n                    u64::from(u32::from_be_bytes(\n                        another_buf[buf_idx..buf_idx + 4].try_into().unwrap(),\n                    )),\n                    0xffff_ffff,\n                ),\n                _ => (\n                    u64::from_be_bytes(buf[buf_idx..buf_idx + 8].try_into().unwrap()),\n                    u64::from_be_bytes(another_buf[buf_idx..buf_idx + 8].try_into().unwrap()),\n                    0xffff_ffff_ffff_ffff,\n                ),\n            };\n\n            // Try arith\n            let diff = (pattern as i64).wrapping_sub(b_val as i64);\n            let new_diff = (another_pattern as i64).wrapping_sub(o_b_val as i64);\n\n            if diff == new_diff && diff != 0 {\n                let new_repl: u64 = (repl as i64).wrapping_sub(diff) as u64;\n\n                let ret = self.cmp_extend_encoding(\n                    pattern,\n                    new_repl,\n                    another_pattern,\n                    repl,\n                    CMP_ATTRIBUTE_IS_TRANSFORM,\n                    another_buf,\n                    buf,\n                    buf_idx,\n                    taint_len,\n                    input_len,\n                    hshape,\n                    vec,\n                )?;\n                if ret {\n                    return Ok(true);\n                }\n            }\n\n            // Try XOR\n\n            // Shadowing\n            let diff: i64 = (pattern ^ b_val) as i64;\n            let new_diff: i64 = (another_pattern ^ o_b_val) as i64;\n\n            if diff == new_diff && diff != 0 {\n                let new_repl: u64 = (repl as i64 ^ diff) as u64;\n                let ret = self.cmp_extend_encoding(\n                    pattern,\n                    new_repl,\n                    another_pattern,\n                    repl,\n                    CMP_ATTRIBUTE_IS_TRANSFORM,\n                    another_buf,\n                    buf,\n                    buf_idx,\n                    taint_len,\n                    input_len,\n                    hshape,\n                    vec,\n                )?;\n\n                if ret {\n                    return Ok(true);\n                }\n            }\n\n            // Try Lowercase\n            // Shadowing\n            let diff = (b_val | 0x2020_2020_2020_2020 & mask) == (pattern & mask);\n\n            let new_diff = (b_val | 0x2020_2020_2020_2020 & mask) == (another_pattern & mask);\n\n            if new_diff && diff {\n                let new_repl: u64 = repl & (0x5f5f_5f5f_5f5f_5f5f & mask);\n                let ret = self.cmp_extend_encoding(\n                    pattern,\n                    new_repl,\n                    another_pattern,\n                    repl,\n                    CMP_ATTRIBUTE_IS_TRANSFORM,\n                    another_buf,\n                    buf,\n                    buf_idx,\n                    taint_len,\n                    input_len,\n                    hshape,\n                    vec,\n                )?;\n\n                if ret {\n                    return Ok(true);\n                }\n            }\n\n            // Try Uppercase\n            // Shadowing\n            let diff = (b_val | 0x5f5f_5f5f_5f5f_5f5f & mask) == (pattern & mask);\n\n            let o_diff = (b_val | 0x5f5f_5f5f_5f5f_5f5f & mask) == (another_pattern & mask);\n\n            if o_diff && diff {\n                let new_repl: u64 = repl & (0x2020_2020_2020_2020 & mask);\n                let ret = self.cmp_extend_encoding(\n                    pattern,\n                    new_repl,\n                    another_pattern,\n                    repl,\n                    CMP_ATTRIBUTE_IS_TRANSFORM,\n                    another_buf,\n                    buf,\n                    buf_idx,\n                    taint_len,\n                    input_len,\n                    hshape,\n                    vec,\n                )?;\n\n                if ret {\n                    return Ok(true);\n                }\n            }\n        }\n\n        let its_len = core::cmp::min(input_len.wrapping_sub(buf_idx), taint_len);\n\n        // Try pattern matching\n        // println!(\"Pattern match\");\n        match hshape {\n            0 => (), // NEVER HAPPEN, Do nothing\n            1 => {\n                // 1 byte pattern match\n                let buf_8 = buf[buf_idx];\n                let another_buf_8 = another_buf[buf_idx];\n                if buf_8 == pattern as u8 && another_buf_8 == another_pattern as u8 {\n                    let mut cloned = buf.to_vec();\n                    cloned[buf_idx] = repl as u8;\n                    vec.push(cloned);\n                    return Ok(true);\n                }\n            }\n            2 | 3 => {\n                if its_len >= 2 {\n                    let buf_16 = u16::from_be_bytes(buf[buf_idx..buf_idx + 2].try_into()?);\n                    let another_buf_16 =\n                        u16::from_be_bytes(another_buf[buf_idx..buf_idx + 2].try_into()?);\n\n                    if buf_16 == pattern as u16 && another_buf_16 == another_pattern as u16 {\n                        let mut cloned = buf.to_vec();\n                        cloned[buf_idx + 1] = (repl & 0xff) as u8;\n                        cloned[buf_idx] = (repl >> 8 & 0xff) as u8;\n                        vec.push(cloned);\n                        return Ok(true);\n                    }\n                }\n            }\n            4..=7 => {\n                if its_len >= 4 {\n                    let buf_32 = u32::from_be_bytes(buf[buf_idx..buf_idx + 4].try_into()?);\n                    let another_buf_32 =\n                        u32::from_be_bytes(another_buf[buf_idx..buf_idx + 4].try_into()?);\n                    // println!(\"buf: {buf_32} {another_buf_32} {pattern} {another_pattern}\");\n                    if buf_32 == pattern as u32 && another_buf_32 == another_pattern as u32 {\n                        let mut cloned = buf.to_vec();\n                        cloned[buf_idx + 3] = (repl & 0xff) as u8;\n                        cloned[buf_idx + 2] = (repl >> 8 & 0xff) as u8;\n                        cloned[buf_idx + 1] = (repl >> 16 & 0xff) as u8;\n                        cloned[buf_idx] = (repl >> 24 & 0xff) as u8;\n                        vec.push(cloned);\n\n                        return Ok(true);\n                    }\n                }\n            }\n            _ => {\n                if its_len >= 8 {\n                    let buf_64 = u64::from_be_bytes(buf[buf_idx..buf_idx + 8].try_into()?);\n                    let another_buf_64 =\n                        u64::from_be_bytes(another_buf[buf_idx..buf_idx + 8].try_into()?);\n\n                    if buf_64 == pattern && another_buf_64 == another_pattern {\n                        let mut cloned = buf.to_vec();\n\n                        cloned[buf_idx + 7] = (repl & 0xff) as u8;\n                        cloned[buf_idx + 6] = (repl >> 8 & 0xff) as u8;\n                        cloned[buf_idx + 5] = (repl >> 16 & 0xff) as u8;\n                        cloned[buf_idx + 4] = (repl >> 24 & 0xff) as u8;\n                        cloned[buf_idx + 3] = (repl >> 32 & 0xff) as u8;\n                        cloned[buf_idx + 2] = (repl >> 32 & 0xff) as u8;\n                        cloned[buf_idx + 1] = (repl >> 40 & 0xff) as u8;\n                        cloned[buf_idx] = (repl >> 48 & 0xff) as u8;\n\n                        vec.push(cloned);\n                        return Ok(true);\n                    }\n                }\n            }\n        }\n\n        // Try arith\n        if self.enable_arith || attr != CMP_ATTRIBUTE_IS_TRANSFORM {\n            if (attr & (CMP_ATTRIBUTE_IS_GREATER | CMP_ATTRIBUTE_IS_LESSER)) == 0 || hshape < 4 {\n                return Ok(false);\n            }\n\n            // Transform >= to < and <= to >\n            let attr = if (attr & CMP_ATTTRIBUTE_IS_EQUAL) != 0\n                && (attr & (CMP_ATTRIBUTE_IS_GREATER | CMP_ATTRIBUTE_IS_LESSER)) != 0\n            {\n                if attr & CMP_ATTRIBUTE_IS_GREATER != 0 {\n                    attr + 2\n                } else {\n                    attr - 2\n                }\n            } else {\n                attr\n            };\n\n            // FP\n            if (CMP_ATTRIBUTE_IS_FP..CMP_ATTRIBUTE_IS_FP_MOD).contains(&attr) {\n                let repl_new: u64;\n\n                if attr & CMP_ATTRIBUTE_IS_GREATER != 0 {\n                    if hshape == 4 && its_len >= 4 {\n                        let mut g = repl as f32;\n                        g += 1.0;\n                        repl_new = u64::from(g as u32);\n                    } else if hshape == 8 && its_len >= 8 {\n                        let mut g = repl as f64;\n                        g += 1.0;\n                        repl_new = g as u64;\n                    } else {\n                        return Ok(false);\n                    }\n\n                    let ret = self.cmp_extend_encoding(\n                        pattern,\n                        repl,\n                        another_pattern,\n                        repl_new,\n                        CMP_ATTRIBUTE_IS_FP_MOD,\n                        another_buf,\n                        buf,\n                        buf_idx,\n                        taint_len,\n                        input_len,\n                        hshape,\n                        vec,\n                    )?;\n                    if ret {\n                        return Ok(true);\n                    }\n                } else {\n                    if hshape == 4 && its_len >= 4 {\n                        let mut g = repl as f32;\n                        g -= 1.0;\n                        repl_new = u64::from(g as u32);\n                    } else if hshape == 8 && its_len >= 8 {\n                        let mut g = repl as f64;\n                        g -= 1.0;\n                        repl_new = g as u64;\n                    } else {\n                        return Ok(false);\n                    }\n\n                    let ret = self.cmp_extend_encoding(\n                        pattern,\n                        repl,\n                        another_pattern,\n                        repl_new,\n                        CMP_ATTRIBUTE_IS_FP_MOD,\n                        another_buf,\n                        buf,\n                        buf_idx,\n                        taint_len,\n                        input_len,\n                        hshape,\n                        vec,\n                    )?;\n                    if ret {\n                        return Ok(true);\n                    }\n                }\n            } else if attr < CMP_ATTRIBUTE_IS_FP {\n                if attr & CMP_ATTRIBUTE_IS_GREATER != 0 {\n                    let repl_new = repl.wrapping_add(1);\n\n                    let ret = self.cmp_extend_encoding(\n                        pattern,\n                        repl,\n                        another_pattern,\n                        repl_new,\n                        CMP_ATTRIBUTE_IS_INT_MOD,\n                        another_buf,\n                        buf,\n                        buf_idx,\n                        taint_len,\n                        input_len,\n                        hshape,\n                        vec,\n                    )?;\n\n                    if ret {\n                        return Ok(true);\n                    }\n                } else {\n                    let repl_new = repl.wrapping_sub(1);\n\n                    let ret = self.cmp_extend_encoding(\n                        pattern,\n                        repl,\n                        another_pattern,\n                        repl_new,\n                        CMP_ATTRIBUTE_IS_INT_MOD,\n                        another_buf,\n                        buf,\n                        buf_idx,\n                        taint_len,\n                        input_len,\n                        hshape,\n                        vec,\n                    )?;\n\n                    if ret {\n                        return Ok(true);\n                    }\n                }\n            } else {\n                return Ok(false);\n            }\n        }\n\n        Ok(false)\n    }\n\n    /// rtn part from AFL++\n    #[allow(clippy::too_many_arguments)]\n    \npub fn rtn_extend_encoding(\n        &self,\n        pattern: &[u8],\n        repl: &[u8],\n        o_pattern: &[u8],\n        _changed_val: &[u8],\n        o_buf: &[u8],\n        buf: &[u8],\n        buf_idx: usize,\n        taint_len: usize,\n        input_len: usize,\n        hshape: usize,\n        vec: &mut Vec<Vec<u8>>,\n    ) -> bool {\n        let l0 = pattern.len();\n        let ol0 = o_pattern.len();\n        let lmax = core::cmp::max(l0, ol0);\n        let its_len = core::cmp::min(\n            core::cmp::min(input_len.wrapping_sub(buf_idx), taint_len),\n            core::cmp::min(lmax, hshape),\n        );\n\n        // TODO: Match before (This: https://github.com/AFLplusplus/AFLplusplus/blob/ea14f3fd40e32234989043a525e3853fcb33c1b6/src/afl-fuzz-redqueen.c#L2047)\n        let mut copy_len = 0;\n        for i in 0..its_len {\n            let b1 = i < pattern.len() && pattern[i] != buf[buf_idx + i];\n            let b2 = i < o_pattern.len() && o_pattern[i] != o_buf[buf_idx + i];\n\n            if b1 || b2 {\n                break;\n            }\n            copy_len += 1;\n        }\n\n        if copy_len > 0 {\n            unsafe {\n                for l in 1..=copy_len {\n                    let mut cloned = buf.to_vec();\n                    buffer_copy(&mut cloned, repl, 0, buf_idx, l);\n                    vec.push(cloned);\n                }\n                // vec.push(cloned);\n            }\n            true\n        } else {\n            false\n        }\n\n        // TODO: Transform (This: https://github.com/AFLplusplus/AFLplusplus/blob/stable/src/afl-fuzz-redqueen.c#L2089)\n        // It's hard to implement this naively\n        // because AFL++ redqueen does not check any pattern, but it calls its_fuzz() instead.\n        // we can't execute the harness inside a mutator\n\n        // Direct matching\n    }\n}\n\nimpl<I, S> MultiMutator<I, S> for AFLppRedQueen\nwhere\n    S: UsesInput + HasMetadata + HasRand + HasMaxSize + HasCorpus + HasCurrentCorpusIdx,\n    I: HasBytesVec + From<Vec<u8>>,\n{\n    #[allow(clippy::needless_range_loop)]\n    #[allow(clippy::too_many_lines)]\n    fn multi_mutate(\n        &mut self,\n        state: &mut S,\n        input: &I,\n        max_count: Option<usize>,\n    ) -> Result<Vec<I>, Error> {\n        // TODO\n        // handle 128-bits logs\n        let size = input.bytes().len();\n        if size == 0 {\n            return Ok(vec![]);\n        }\n\n        let (cmp_len, cmp_meta, taint_meta) = {\n            let (Some(cmp_meta), Some(taint_meta)) = (\n                state.metadata_map().get::<AFLppCmpValuesMetadata>(),\n                state.metadata_map().get::<TaintMetadata>(),\n            ) else {\n                return Ok(vec![]);\n            };\n\n            let cmp_len = cmp_meta.headers().len();\n            if cmp_len == 0 {\n                return Ok(vec![]);\n            }\n            (cmp_len, cmp_meta, taint_meta)\n        };\n\n        // These idxes must saved in this mutator itself!\n        let mut taint_idx = 0;\n        let orig_cmpvals = cmp_meta.orig_cmpvals();\n        let new_cmpvals = cmp_meta.new_cmpvals();\n        let headers = cmp_meta.headers();\n        let input_len = input.bytes().len();\n        let new_bytes = taint_meta.input_vec();\n        let orig_bytes = input.bytes();\n\n        let taint = taint_meta.ranges();\n        let mut ret = max_count.map_or_else(Vec::new, Vec::with_capacity);\n        let mut gathered_tokens = Tokens::new();\n        // println!(\"orig: {:#?} new: {:#?}\", orig_cmpvals, new_cmpvals);\n\n        // Compute when mutating it for the 1st time.\n        let current_corpus_idx = state.current_corpus_idx()?.ok_or_else(|| Error::key_not_found(\"No corpus-idx is currently being fuzzed, but called AFLppRedQueen::multi_mutated().\"))?;\n        if self.last_corpus_idx.is_none() || self.last_corpus_idx.unwrap() != current_corpus_idx {\n            self.text_type = check_if_text(orig_bytes, orig_bytes.len());\n            self.last_corpus_idx = Some(current_corpus_idx);\n        }\n        // println!(\"approximate size: {cmp_len} x {input_len}\");\n        for cmp_idx in 0..cmp_len {\n            let (w_idx, header) = headers[cmp_idx];\n\n            if orig_cmpvals.get(&w_idx).is_none() || new_cmpvals.get(&w_idx).is_none() {\n                // These two should have same boolean value\n\n                // so there's nothing interesting at cmp_idx, then just skip!\n                continue;\n            }\n\n            let orig_val = orig_cmpvals.get(&w_idx).unwrap();\n            let new_val = new_cmpvals.get(&w_idx).unwrap();\n\n            let logged = core::cmp::min(orig_val.len(), new_val.len());\n\n            for cmp_h_idx in 0..logged {\n                let mut skip_opt = false;\n                for prev_idx in 0..cmp_h_idx {\n                    if new_val[prev_idx] == new_val[cmp_h_idx] {\n                        skip_opt = true;\n                    }\n                }\n                // Opt not in the paper\n                if skip_opt {\n                    continue;\n                }\n\n                for cmp_buf_idx in 0..input_len {\n                    if let Some(max_count) = max_count {\n                        if ret.len() >= max_count {\n                            // TODO: does this bias towards earlier mutations?\n                            break;\n                        }\n                    }\n\n                    let taint_len = match taint.get(taint_idx) {\n                        Some(t) => {\n                            if cmp_buf_idx < t.start {\n                                input_len - cmp_buf_idx\n                            } else {\n                                // if cmp_buf_idx == t.end go to next range\n                                if cmp_buf_idx == t.end {\n                                    taint_idx += 1;\n                                }\n\n                                // Here cmp_buf_idx >= t.start\n                                t.end - cmp_buf_idx\n                            }\n                        }\n                        None => input_len - cmp_buf_idx,\n                    };\n\n                    let hshape = (header.shape() + 1) as usize;\n\n                    match (&orig_val[cmp_h_idx], &new_val[cmp_h_idx]) {\n                        (CmpValues::U8(_orig), CmpValues::U8(_new)) => {\n                            /* just don't do it for u8, not worth it. not even instrumented\n\n                            let (orig_v0, orig_v1, new_v0, new_v1) = (orig.0, orig.1, new.0, new.1);\n                            let attribute = header.attribute() as u8;\n\n                            let mut cmp_found = false;\n                            if new_v0 != orig_v0 && orig_v0 != orig_v1 {\n                                // Compare v0 against v1\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v0.into(),\n                                    orig_v1.into(),\n                                    new_v0.into(),\n                                    new_v1.into(),\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                );\n\n                                // Swapped\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v0.swap_bytes().into(),\n                                    orig_v1.swap_bytes().into(),\n                                    new_v0.swap_bytes().into(),\n                                    new_v1.swap_bytes().into(),\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                );\n                            }\n\n                            if new_v1 != orig_v1 && orig_v0 != orig_v1 {\n                                // Compare v1 against v0\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v1.into(),\n                                    orig_v0.into(),\n                                    new_v1.into(),\n                                    new_v0.into(),\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                );\n\n                                // Swapped\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v1.swap_bytes().into(),\n                                    orig_v0.swap_bytes().into(),\n                                    new_v1.swap_bytes().into(),\n                                    new_v0.swap_bytes().into(),\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                );\n                            }\n                            */\n\n                            /*\n                            U8 or U16 is not worth\n                            if !cmp_found && self.text_type.is_ascii_or_utf8() {\n                                if orig_v0 == new_v0 {\n                                    let v = orig_v0.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n\n                                if orig_v1 == new_v1 {\n                                    let v = orig_v1.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n                            }\n                            */\n                        }\n                        (CmpValues::U16(orig), CmpValues::U16(new)) => {\n                            let (orig_v0, orig_v1, new_v0, new_v1) = (orig.0, orig.1, new.0, new.1);\n                            let attribute: u8 = header.attribute() as u8;\n\n                            if new_v0 != orig_v0 && orig_v0 != orig_v1 {\n                                // Compare v0 against v1\n                                self.cmp_extend_encoding(\n                                    orig_v0.into(),\n                                    orig_v1.into(),\n                                    new_v0.into(),\n                                    new_v1.into(),\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n\n                                // Swapped\n                                // Compare v0 against v1\n                                self.cmp_extend_encoding(\n                                    orig_v0.swap_bytes().into(),\n                                    orig_v1.swap_bytes().into(),\n                                    new_v0.swap_bytes().into(),\n                                    new_v1.swap_bytes().into(),\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n                            }\n\n                            if new_v1 != orig_v1 && orig_v0 != orig_v1 {\n                                // Compare v1 against v0\n                                self.cmp_extend_encoding(\n                                    orig_v1.into(),\n                                    orig_v0.into(),\n                                    new_v1.into(),\n                                    new_v0.into(),\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n\n                                // Swapped\n                                self.cmp_extend_encoding(\n                                    orig_v1.swap_bytes().into(),\n                                    orig_v0.swap_bytes().into(),\n                                    new_v1.swap_bytes().into(),\n                                    new_v0.swap_bytes().into(),\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n                            }\n\n                            /*\n                            U8 or U16 is not worth\n                            if !cmp_found && self.text_type.is_ascii_or_utf8() {\n                                if orig_v0 == new_v0 {\n                                    let v = orig_v0.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n\n                                if orig_v1 == new_v1 {\n                                    let v = orig_v1.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n                            }\n                            */\n                        }\n                        (CmpValues::U32(orig), CmpValues::U32(new)) => {\n                            let (orig_v0, orig_v1, new_v0, new_v1) = (orig.0, orig.1, new.0, new.1);\n                            let attribute = header.attribute() as u8;\n\n                            let mut cmp_found = false;\n                            if new_v0 != orig_v0 && orig_v0 != orig_v1 {\n                                // Compare v0 against v1\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v0.into(),\n                                    orig_v1.into(),\n                                    new_v0.into(),\n                                    new_v1.into(),\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n\n                                // swapped\n                                // Compare v0 against v1\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v0.swap_bytes().into(),\n                                    orig_v1.swap_bytes().into(),\n                                    new_v0.swap_bytes().into(),\n                                    new_v1.swap_bytes().into(),\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n                            }\n\n                            if new_v1 != orig_v1 && orig_v0 != orig_v1 {\n                                // Compare v1 against v0\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v1.into(),\n                                    orig_v0.into(),\n                                    new_v1.into(),\n                                    new_v0.into(),\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n\n                                // Swapped\n                                // Compare v1 against v0\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v1.swap_bytes().into(),\n                                    orig_v0.swap_bytes().into(),\n                                    new_v1.swap_bytes().into(),\n                                    new_v0.swap_bytes().into(),\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n                            }\n\n                            if !cmp_found {\n                                if orig_v0 == new_v0\n                                    && check_if_text(orig_v0.to_ne_bytes().as_ref(), hshape).size()\n                                        == hshape\n                                {\n                                    let v = orig_v0.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n\n                                if orig_v1 == new_v1\n                                    && check_if_text(orig_v1.to_ne_bytes().as_ref(), hshape).size()\n                                        == hshape\n                                {\n                                    let v = orig_v1.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n                            }\n                        }\n                        (CmpValues::U64(orig), CmpValues::U64(new)) => {\n                            let (orig_v0, orig_v1, new_v0, new_v1) = (orig.0, orig.1, new.0, new.1);\n                            let attribute = header.attribute() as u8;\n\n                            let mut cmp_found = false;\n                            if new_v0 != orig_v0 && orig_v0 != orig_v1 {\n                                // Compare v0 against v1\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v0,\n                                    orig_v1,\n                                    new_v0,\n                                    new_v1,\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n\n                                // Swapped\n                                // Compare v0 against v1\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v0.swap_bytes(),\n                                    orig_v1.swap_bytes(),\n                                    new_v0.swap_bytes(),\n                                    new_v1.swap_bytes(),\n                                    attribute,\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n                            }\n\n                            if new_v1 != orig_v1 && orig_v0 != orig_v1 {\n                                // Compare v1 against v0\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v1,\n                                    orig_v0,\n                                    new_v1,\n                                    new_v0,\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n\n                                // Swapped\n                                // Compare v1 against v0\n                                cmp_found |= self.cmp_extend_encoding(\n                                    orig_v1.swap_bytes(),\n                                    orig_v0.swap_bytes(),\n                                    new_v1.swap_bytes(),\n                                    new_v0.swap_bytes(),\n                                    Self::swapa(attribute),\n                                    new_bytes,\n                                    orig_bytes,\n                                    cmp_buf_idx,\n                                    taint_len,\n                                    input_len,\n                                    hshape,\n                                    &mut ret,\n                                )?;\n                            }\n\n                            if !cmp_found {\n                                if orig_v0 == new_v0\n                                    && check_if_text(orig_v0.to_ne_bytes().as_ref(), hshape).size()\n                                        == hshape\n                                {\n                                    let v = orig_v0.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n\n                                if orig_v1 == new_v1\n                                    && check_if_text(orig_v1.to_ne_bytes().as_ref(), hshape).size()\n                                        == hshape\n                                {\n                                    let v = orig_v1.to_ne_bytes().to_vec();\n                                    Self::try_add_autotokens(&mut gathered_tokens, &v, hshape);\n                                }\n                            }\n                        }\n                        (CmpValues::Bytes(orig), CmpValues::Bytes(new)) => {\n                            let (orig_v0, orig_v1, new_v0, new_v1) =\n                                (&orig.0, &orig.1, &new.0, &new.1);\n                            // let attribute = header.attribute() as u8;\n                            let mut rtn_found = false;\n                            // Compare v0 against v1\n                            rtn_found |= self.rtn_extend_encoding(\n                                orig_v0,\n                                orig_v1,\n                                new_v0,\n...\n", "needle_token_start": 10204, "needle_token_end": 10848, "code_context_ntokens": 16221, "output": ["```rust\npub fn rtn_extend_encoding(\n    &self,\n    pattern: &[u8],\n    repl: &[u8],\n    o_pattern: &[u8],\n    _changed_val: &[u8],\n    o_buf: &[u8],\n    buf: &[u8],\n    buf_idx: usize,\n    taint_len: usize,\n    input_len: usize,\n    hshape: usize,\n    vec: &mut Vec<Vec<u8>>,\n) -> bool {\n    let l0 = pattern.len();\n    let ol0 = o_pattern.len();\n    let lmax = core::cmp::max(l0, ol0);\n    let its_len = core::cmp::min(\n        core::cmp::min(input_len.wrapping_sub(buf_idx), taint_len),\n        core::cmp::min(lmax, hshape),\n    );\n\n    // TODO: Match before (This: https://github.com/AFLplusplus/AFLplusplus/blob/ea14f3fd40e32234989043a525e3853fcb33c1b6/src/afl-fuzz-redqueen.c#L2047)\n    let mut copy_len = 0;\n    for i in 0..its_len {\n        let b1 = i < pattern.len() && pattern[i] != buf[buf_idx + i];\n        let b2 = i < o_pattern.len() && o_pattern[i] != o_buf[buf_idx + i];\n\n        if b1 || b2 {\n            break;\n        }\n        copy_len += 1;\n    }\n\n    if copy_len > 0 {\n        unsafe {\n            for l in 1..=copy_len {\n                let mut cloned = buf.to_vec();\n                buffer_copy(&mut cloned, repl, 0, buf_idx, l);\n                vec.push(cloned);\n            }\n            // vec.push(cloned);\n        }\n        true\n    } else {\n        false\n    }\n\n    // TODO: Transform (This: https://github.com/AFLplusplus/AFLplusplus/blob/stable/src/afl-fuzz-redqueen.c#L2089)\n    // It's hard to implement this naively\n    // because AFL++ redqueen does not check any pattern, but it calls its_fuzz() instead.\n    // we can't execute the harness inside a mutator\n\n    // Direct matching\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "with_history_map", "language": "rust", "path": "libafl/src/feedbacks/map.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To initialize a metadata structure with a pre-existing map of data, calculating the count of entries that differ from an initial value.\n2. **Input**: A vector containing data entries and a single data entry representing the initial value for comparison.\n3. **Output**: An instance of the metadata structure containing the provided map and the count of entries that are not equal to the initial value.\n4. **Procedure**: The function iterates over the provided vector, counting how many entries differ from the given initial value. This count and the original vector are then used to initialize and return the metadata structure.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: libafl/src/observers/map.rs\n//! The `MapObserver` provides access a map, usually injected into the target\n\nuse alloc::{\n    string::{String, ToString},\n    vec::Vec,\n};\nuse core::{\n    fmt::Debug,\n    hash::{BuildHasher, Hasher},\n    iter::Flatten,\n    marker::PhantomData,\n    mem::size_of,\n    slice::{self, Iter, IterMut},\n};\n\nuse ahash::RandomState;\nuse libafl_bolts::{\n    ownedref::{OwnedMutPtr, OwnedMutSlice},\n    AsIter, AsIterMut, AsMutSlice, AsSlice, HasLen, Named, Truncate,\n};\nuse meminterval::IntervalTree;\nuse num_traits::Bounded;\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    executors::ExitKind,\n    inputs::UsesInput,\n    observers::{DifferentialObserver, Observer, ObserversTuple},\n    Error,\n};\n\n/// Hitcounts class lookup\nstatic COUNT_CLASS_LOOKUP: [u8; 256] = [\n    0, 1, 2, 4, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,\n    32, 32, 32, 32, 32, 32, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n];\n\n/// Hitcounts class lookup for 16-byte values\nstatic mut COUNT_CLASS_LOOKUP_16: Vec<u16> = vec![];\n\n/// Initialize the 16-byte hitcounts map\n///\n/// # Safety\n///\n/// Calling this from multiple threads may be racey and hence leak 65k mem\nfn init_count_class_16() {\n    unsafe {\n        if !COUNT_CLASS_LOOKUP_16.is_empty() {\n            return;\n        }\n\n        COUNT_CLASS_LOOKUP_16 = vec![0; 65536];\n        for i in 0..256 {\n            for j in 0..256 {\n                COUNT_CLASS_LOOKUP_16[(i << 8) + j] =\n                    (u16::from(COUNT_CLASS_LOOKUP[i]) << 8) | u16::from(COUNT_CLASS_LOOKUP[j]);\n            }\n        }\n    }\n}\n\n/// Compute the hash of a slice\nfn hash_slice<T>(slice: &[T]) -> u64 {\n    let mut hasher = RandomState::with_seeds(0, 0, 0, 0).build_hasher();\n    let ptr = slice.as_ptr() as *const u8;\n    let map_size = slice.len() / size_of::<T>();\n    unsafe {\n        hasher.write(slice::from_raw_parts(ptr, map_size));\n    }\n    hasher.finish()\n}\n\n/// A [`MapObserver`] observes the static map, as oftentimes used for AFL-like coverage information\n///\n/// TODO: enforce `iter() -> AssociatedTypeIter` when generic associated types stabilize\npub trait MapObserver: HasLen + Named + Serialize + serde::de::DeserializeOwned\n// where\n//     for<'it> &'it Self: IntoIterator<Item = &'it Self::Entry>\n{\n    /// Type of each entry in this map\n    type Entry: Bounded + PartialEq + Default + Copy + Debug + 'static;\n\n    /// Get the value at `idx`\n    fn get(&self, idx: usize) -> &Self::Entry;\n\n    /// Get the value at `idx` (mutable)\n    fn get_mut(&mut self, idx: usize) -> &mut Self::Entry;\n\n    /// Get the number of usable entries in the map (all by default)\n    fn usable_count(&self) -> usize;\n\n    /// Count the set bytes in the map\n    fn count_bytes(&self) -> u64;\n\n    /// Compute the hash of the map\n    fn hash(&self) -> u64;\n\n    /// Get the initial value for `reset()`\n    fn initial(&self) -> Self::Entry;\n\n    /// Reset the map\n    fn reset_map(&mut self) -> Result<(), Error>;\n\n    /// Get these observer's contents as [`Vec`]\n    fn to_vec(&self) -> Vec<Self::Entry>;\n\n    /// Get the number of set entries with the specified indexes\n    fn how_many_set(&self, indexes: &[usize]) -> usize;\n}\n\n/// A Simple iterator calling `MapObserver::get`\n#[derive(Debug)]\npub struct MapObserverSimpleIterator<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    index: usize,\n    observer: *const O,\n    phantom: PhantomData<&'a u8>,\n}\n\nimpl<'a, O> Iterator for MapObserverSimpleIterator<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    type Item = &'a O::Entry;\n    fn next(&mut self) -> Option<Self::Item> {\n        unsafe {\n            if self.index >= self.observer.as_ref().unwrap().usable_count() {\n                None\n            } else {\n                let i = self.index;\n                self.index += 1;\n                Some(self.observer.as_ref().unwrap().get(i))\n            }\n        }\n    }\n}\n\n/// A Simple iterator calling `MapObserver::get_mut`\n#[derive(Debug)]\npub struct MapObserverSimpleIteratorMut<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    index: usize,\n    observer: *mut O,\n    phantom: PhantomData<&'a u8>,\n}\n\nimpl<'a, O> Iterator for MapObserverSimpleIteratorMut<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    type Item = &'a O::Entry;\n    fn next(&mut self) -> Option<Self::Item> {\n        unsafe {\n            if self.index >= self.observer.as_ref().unwrap().usable_count() {\n                None\n            } else {\n                let i = self.index;\n                self.index += 1;\n                Some(self.observer.as_mut().unwrap().get_mut(i))\n            }\n        }\n    }\n}\n\n/// The Map Observer retrieves the state of a map,\n/// that will get updated by the target.\n/// A well-known example is the AFL-Style coverage map.\n#[derive(Clone, Serialize, Deserialize, Debug)]\n#[serde(bound = \"T: serde::de::DeserializeOwned\")]\n#[allow(clippy::unsafe_derive_deserialize)]\npub struct StdMapObserver<'a, T, const DIFFERENTIAL: bool>\nwhere\n    T: Default + Copy + 'static + Serialize,\n{\n    map: OwnedMutSlice<'a, T>,\n    initial: T,\n    name: String,\n}\n\nimpl<'a, S, T> Observer<S> for StdMapObserver<'a, T, false>\nwhere\n    S: UsesInput,\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    #[inline]\n    fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> {\n        self.reset_map()\n    }\n}\n\nimpl<'a, S, T> Observer<S> for StdMapObserver<'a, T, true>\nwhere\n    S: UsesInput,\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> Named for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn name(&self) -> &str {\n        self.name.as_str()\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> HasLen for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn len(&self) -> usize {\n        self.map.as_slice().len()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> AsIter<'it> for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = Iter<'it, T>;\n\n    fn as_iter(&'it self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> AsIterMut<'it> for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = IterMut<'it, T>;\n\n    fn as_iter_mut(&'it mut self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator for &'it StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <Iter<'it, T> as Iterator>::Item;\n    type IntoIter = Iter<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator\n    for &'it mut StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <IterMut<'it, T> as Iterator>::Item;\n    type IntoIter = IterMut<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    /// Returns an iterator over the map.\n    pub fn iter(&self) -> Iter<'_, T> {\n        <&Self as IntoIterator>::into_iter(self)\n    }\n\n    /// Returns a mutable iterator over the map.\n    pub fn iter_mut(&mut self) -> IterMut<'_, T> {\n        <&mut Self as IntoIterator>::into_iter(self)\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> MapObserver for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Entry = T;\n\n    #[inline]\n    fn get(&self, pos: usize) -> &T {\n        &self.as_slice()[pos]\n    }\n\n    #[inline]\n    fn get_mut(&mut self, idx: usize) -> &mut T {\n        &mut self.as_mut_slice()[idx]\n    }\n\n    /// Count the set bytes in the map\n    fn count_bytes(&self) -> u64 {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for x in &map[0..cnt] {\n            if *x != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n\n    #[inline]\n    fn usable_count(&self) -> usize {\n        self.as_slice().len()\n    }\n\n    fn hash(&self) -> u64 {\n        hash_slice(self.as_slice())\n    }\n\n    #[inline]\n    fn initial(&self) -> T {\n        self.initial\n    }\n\n    fn to_vec(&self) -> Vec<T> {\n        self.as_slice().to_vec()\n    }\n\n    /// Reset the map\n    #[inline]\n    fn reset_map(&mut self) -> Result<(), Error> {\n        // Normal memset, see https://rust.godbolt.org/z/Trs5hv\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_mut_slice();\n        for x in &mut map[0..cnt] {\n            *x = initial;\n        }\n        Ok(())\n    }\n\n    fn how_many_set(&self, indexes: &[usize]) -> usize {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for i in indexes {\n            if *i < cnt && map[*i] != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> Truncate for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    fn truncate(&mut self, new_len: usize) {\n        self.map.truncate(new_len);\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> AsSlice for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n{\n    type Entry = T;\n    #[must_use]\n    #[inline]\n    fn as_slice(&self) -> &[T] {\n        self.map.as_slice()\n    }\n}\nimpl<'a, T, const DIFFERENTIAL: bool> AsMutSlice for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n{\n    type Entry = T;\n    #[must_use]\n    #[inline]\n    fn as_mut_slice(&mut self) -> &mut [T] {\n        self.map.as_mut_slice()\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    /// Creates a new [`MapObserver`]\n    ///\n    /// # Safety\n    /// Will get a pointer to the map and dereference it at any point in time.\n    /// The map must not move in memory!\n    #[must_use]\n    unsafe fn maybe_differential<S>(name: S, map: &'a mut [T]) -> Self\n    where\n        S: Into<String>,\n    {\n        let len = map.len();\n        let ptr = map.as_mut_ptr();\n        Self::maybe_differential_from_mut_ptr(name, ptr, len)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`]\n    #[must_use]\n    fn maybe_differential_from_mut_slice<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        StdMapObserver {\n            name: name.into(),\n            map,\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] with an owned map\n    #[must_use]\n    fn maybe_differential_owned<S>(name: S, map: Vec<T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self {\n            map: OwnedMutSlice::from(map),\n            name: name.into(),\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map.\n    ///\n    /// # Safety\n    /// Will dereference the owned slice with up to len elements.\n    #[must_use]\n    fn maybe_differential_from_ownedref<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self {\n            map,\n            name: name.into(),\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer\n    ///\n    /// # Safety\n    /// Will dereference the `map_ptr` with up to len elements.\n    unsafe fn maybe_differential_from_mut_ptr<S>(name: S, map_ptr: *mut T, len: usize) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_slice(\n            name,\n            OwnedMutSlice::from_raw_parts_mut(map_ptr, len),\n        )\n    }\n\n    /// Gets the initial value for this map, mutably\n    pub fn initial_mut(&mut self) -> &mut T {\n        &mut self.initial\n    }\n\n    /// Gets the backing for this map\n    pub fn map(&self) -> &OwnedMutSlice<'a, T> {\n        &self.map\n    }\n\n    /// Gets the backing for this map mutably\n    pub fn map_mut(&mut self) -> &mut OwnedMutSlice<'a, T> {\n        &mut self.map\n    }\n}\n\nimpl<'a, T> StdMapObserver<'a, T, false>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    /// Creates a new [`MapObserver`]\n    ///\n    /// # Safety\n    /// The observer will keep a pointer to the map.\n    /// Hence, the map may never move in memory.\n    #[must_use]\n    pub unsafe fn new<S>(name: S, map: &'a mut [T]) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`]\n    pub fn from_mut_slice<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_slice(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] with an owned map\n    #[must_use]\n    pub fn owned<S>(name: S, map: Vec<T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_owned(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map.\n    ///\n    /// # Note\n    /// Will dereference the owned slice with up to len elements.\n    #[must_use]\n    pub fn from_ownedref<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_ownedref(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer\n    ///\n    /// # Safety\n    /// Will dereference the `map_ptr` with up to len elements.\n    pub unsafe fn from_mut_ptr<S>(name: S, map_ptr: *mut T, len: usize) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_ptr(name, map_ptr, len)\n    }\n}\n\nimpl<'a, T> StdMapObserver<'a, T, true>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    /// Creates a new [`MapObserver`] in differential mode\n    ///\n    /// # Safety\n    /// Will get a pointer to the map and dereference it at any point in time.\n    /// The map must not move in memory!\n    #[must_use]\n    pub unsafe fn differential<S>(name: S, map: &'a mut [T]) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] with an owned map in differential mode\n    #[must_use]\n    pub fn differential_owned<S>(name: S, map: Vec<T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_owned(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map in differential mode.\n    ///\n    /// # Note\n    /// Will dereference the owned slice with up to len elements.\n    #[must_use]\n    pub fn differential_from_ownedref<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_ownedref(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer in differential mode\n    ///\n    /// # Safety\n    /// Will dereference the `map_ptr` with up to len elements.\n    pub unsafe fn differential_from_mut_ptr<S>(name: S, map_ptr: *mut T, len: usize) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_ptr(name, map_ptr, len)\n    }\n}\n\nimpl<'a, OTA, OTB, S, T> DifferentialObserver<OTA, OTB, S> for StdMapObserver<'a, T, true>\nwhere\n    OTA: ObserversTuple<S>,\n    OTB: ObserversTuple<S>,\n    S: UsesInput,\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n}\n\n/// Use a const size to speedup `Feedback::is_interesting` when the user can\n/// know the size of the map at compile time.\n#[derive(Serialize, Deserialize, Debug)]\n#[serde(bound = \"T: serde::de::DeserializeOwned\")]\n#[allow(clippy::unsafe_derive_deserialize)]\npub struct ConstMapObserver<'a, T, const N: usize>\nwhere\n    T: Default + Copy + 'static + Serialize,\n{\n    map: OwnedMutSlice<'a, T>,\n    initial: T,\n    name: String,\n}\n\nimpl<'a, S, T, const N: usize> Observer<S> for ConstMapObserver<'a, T, N>\nwhere\n    S: UsesInput,\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n    Self: MapObserver,\n{\n    #[inline]\n    fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> {\n        self.reset_map()\n    }\n}\n\nimpl<'a, T, const N: usize> Named for ConstMapObserver<'a, T, N>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn name(&self) -> &str {\n        self.name.as_str()\n    }\n}\n\nimpl<'a, T, const N: usize> HasLen for ConstMapObserver<'a, T, N>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn len(&self) -> usize {\n        N\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> AsIter<'it> for ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = Iter<'it, T>;\n\n    fn as_iter(&'it self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> AsIterMut<'it> for ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = IterMut<'it, T>;\n\n    fn as_iter_mut(&'it mut self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> IntoIterator for &'it ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <Iter<'it, T> as Iterator>::Item;\n    type IntoIter = Iter<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> IntoIterator for &'it mut ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <IterMut<'it, T> as Iterator>::Item;\n    type IntoIter = IterMut<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, T, const N: usize> ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    /// Returns an iterator over the map.\n    pub fn iter(&self) -> Iter<'_, T> {\n        <&Self as IntoIterator>::into_iter(self)\n    }\n\n    /// Returns a mutable iterator over the map.\n    pub fn iter_mut(&mut self) -> IterMut<'_, T> {\n        <&mut Self as IntoIterator>::into_iter(self)\n    }\n}\n\nimpl<'a, T, const N: usize> MapObserver for ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Entry = T;\n\n    #[inline]\n    fn initial(&self) -> T {\n        self.initial\n    }\n\n    #[inline]\n    fn get(&self, idx: usize) -> &T {\n        &self.as_slice()[idx]\n    }\n\n    #[inline]\n    fn get_mut(&mut self, idx: usize) -> &mut T {\n        &mut self.as_mut_slice()[idx]\n    }\n\n    /// Count the set bytes in the map\n...\n// Path: libafl/src/feedbacks/map.rs\n//! Map feedback, maximizing or minimizing maps, for example the afl-style map observer.\n\nuse alloc::{\n    string::{String, ToString},\n    vec::Vec,\n};\n#[rustversion::nightly]\nuse core::simd::prelude::SimdOrd;\nuse core::{\n    fmt::Debug,\n    marker::PhantomData,\n    ops::{BitAnd, BitOr},\n};\n\nuse libafl_bolts::{AsIter, AsMutSlice, AsSlice, HasRefCnt, Named};\nuse num_traits::PrimInt;\nuse serde::{de::DeserializeOwned, Deserialize, Serialize};\n\nuse crate::{\n    corpus::Testcase,\n    events::{Event, EventFirer},\n    executors::ExitKind,\n    feedbacks::{Feedback, HasObserverName},\n    inputs::UsesInput,\n    monitors::{AggregatorOps, UserStats, UserStatsValue},\n    observers::{MapObserver, Observer, ObserversTuple, UsesObserver},\n    state::{HasMetadata, HasNamedMetadata, State},\n    Error,\n};\n\n/// A [`MapFeedback`] that implements the AFL algorithm using an [`OrReducer`] combining the bits for the history map and the bit from ``HitcountsMapObserver``.\npub type AflMapFeedback<O, S, T> = MapFeedback<DifferentIsNovel, O, OrReducer, S, T>;\n\n/// A [`MapFeedback`] that strives to maximize the map contents.\npub type MaxMapFeedback<O, S, T> = MapFeedback<DifferentIsNovel, O, MaxReducer, S, T>;\n/// A [`MapFeedback`] that strives to minimize the map contents.\npub type MinMapFeedback<O, S, T> = MapFeedback<DifferentIsNovel, O, MinReducer, S, T>;\n\n/// A [`MapFeedback`] that always returns `true` for `is_interesting`. Useful for tracing all executions.\npub type AlwaysInterestingMapFeedback<O, S, T> = MapFeedback<AllIsNovel, O, NopReducer, S, T>;\n\n/// A [`MapFeedback`] that strives to maximize the map contents,\n/// but only, if a value is larger than `pow2` of the previous.\npub type MaxMapPow2Feedback<O, S, T> = MapFeedback<NextPow2IsNovel, O, MaxReducer, S, T>;\n/// A [`MapFeedback`] that strives to maximize the map contents,\n/// but only, if a value is larger than `pow2` of the previous.\npub type MaxMapOneOrFilledFeedback<O, S, T> = MapFeedback<OneOrFilledIsNovel, O, MaxReducer, S, T>;\n\n/// A `Reducer` function is used to aggregate values for the novelty search\npub trait Reducer<T>: 'static\nwhere\n    T: Default + Copy + 'static,\n{\n    /// Reduce two values to one value, with the current [`Reducer`].\n    fn reduce(first: T, second: T) -> T;\n}\n\n/// A [`OrReducer`] reduces the values returning the bitwise OR with the old value\n#[derive(Clone, Debug)]\npub struct OrReducer {}\n\nimpl<T> Reducer<T> for OrReducer\nwhere\n    T: BitOr<Output = T> + Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(history: T, new: T) -> T {\n        history | new\n    }\n}\n\n/// A [`AndReducer`] reduces the values returning the bitwise AND with the old value\n#[derive(Clone, Debug)]\npub struct AndReducer {}\n\nimpl<T> Reducer<T> for AndReducer\nwhere\n    T: BitAnd<Output = T> + Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(history: T, new: T) -> T {\n        history & new\n    }\n}\n\n/// A [`NopReducer`] does nothing, and just \"reduces\" to the second/`new` value.\n#[derive(Clone, Debug)]\npub struct NopReducer {}\n\nimpl<T> Reducer<T> for NopReducer\nwhere\n    T: Default + Copy + 'static,\n{\n    #[inline]\n    fn reduce(_history: T, new: T) -> T {\n        new\n    }\n}\n\n/// A [`MaxReducer`] reduces int values and returns their maximum.\n#[derive(Clone, Debug)]\npub struct MaxReducer {}\n\nimpl<T> Reducer<T> for MaxReducer\nwhere\n    T: Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(first: T, second: T) -> T {\n        if first > second {\n            first\n        } else {\n            second\n        }\n    }\n}\n\n/// A [`MinReducer`] reduces int values and returns their minimum.\n#[derive(Clone, Debug)]\npub struct MinReducer {}\n\nimpl<T> Reducer<T> for MinReducer\nwhere\n    T: Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(first: T, second: T) -> T {\n        if first < second {\n            first\n        } else {\n            second\n        }\n    }\n}\n\n/// A `IsNovel` function is used to discriminate if a reduced value is considered novel.\npub trait IsNovel<T>: 'static\nwhere\n    T: Default + Copy + 'static,\n{\n    /// If a new value in the [`MapFeedback`] was found,\n    /// this filter can decide if the result is considered novel or not.\n    fn is_novel(old: T, new: T) -> bool;\n}\n\n/// [`AllIsNovel`] consider everything a novelty. Here mostly just for debugging.\n#[derive(Clone, Debug)]\npub struct AllIsNovel {}\n\nimpl<T> IsNovel<T> for AllIsNovel\nwhere\n    T: Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(_old: T, _new: T) -> bool {\n        true\n    }\n}\n\n/// Calculate the next power of two\n/// See <https://stackoverflow.com/a/66253960/1345238>\n/// Will saturate at the max value.\n/// In case of negative values, returns 1.\n#[inline]\nfn saturating_next_power_of_two<T: PrimInt>(n: T) -> T {\n    if n <= T::one() {\n        T::one()\n    } else {\n        (T::max_value() >> (n - T::one()).leading_zeros().try_into().unwrap())\n            .saturating_add(T::one())\n    }\n}\n\n/// Consider as novelty if the reduced value is different from the old value.\n#[derive(Clone, Debug)]\npub struct DifferentIsNovel {}\nimpl<T> IsNovel<T> for DifferentIsNovel\nwhere\n    T: PartialEq + Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(old: T, new: T) -> bool {\n        old != new\n    }\n}\n\n/// Only consider as novel the values which are at least the next pow2 class of the old value\n#[derive(Clone, Debug)]\npub struct NextPow2IsNovel {}\nimpl<T> IsNovel<T> for NextPow2IsNovel\nwhere\n    T: PrimInt + Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(old: T, new: T) -> bool {\n        // We use a trait so we build our numbers from scratch here.\n        // This way it works with Nums of any size.\n        if new <= old {\n            false\n        } else {\n            let pow2 = saturating_next_power_of_two(old.saturating_add(T::one()));\n            new >= pow2\n        }\n    }\n}\n\n/// Only consider `T::one()` or `T::max_value()`, if they are bigger than the old value, as novel\n#[derive(Clone, Debug)]\npub struct OneOrFilledIsNovel {}\nimpl<T> IsNovel<T> for OneOrFilledIsNovel\nwhere\n    T: PrimInt + Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(old: T, new: T) -> bool {\n        (new == T::one() || new == T::max_value()) && new > old\n    }\n}\n\n/// A testcase metadata holding a list of indexes of a map\n#[derive(Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct MapIndexesMetadata {\n    /// The list of indexes.\n    pub list: Vec<usize>,\n    /// A refcount used to know when we can remove this metadata\n    pub tcref: isize,\n}\n\nlibafl_bolts::impl_serdeany!(MapIndexesMetadata);\n\nimpl AsSlice for MapIndexesMetadata {\n    type Entry = usize;\n    /// Convert to a slice\n    fn as_slice(&self) -> &[usize] {\n        self.list.as_slice()\n    }\n}\nimpl AsMutSlice for MapIndexesMetadata {\n    type Entry = usize;\n    /// Convert to a slice\n    fn as_mut_slice(&mut self) -> &mut [usize] {\n        self.list.as_mut_slice()\n    }\n}\n\nimpl HasRefCnt for MapIndexesMetadata {\n    fn refcnt(&self) -> isize {\n        self.tcref\n    }\n\n    fn refcnt_mut(&mut self) -> &mut isize {\n        &mut self.tcref\n    }\n}\n\nimpl MapIndexesMetadata {\n    /// Creates a new [`struct@MapIndexesMetadata`].\n    #[must_use]\n    pub fn new(list: Vec<usize>) -> Self {\n        Self { list, tcref: 0 }\n    }\n}\n\n/// A testcase metadata holding a list of indexes of a map\n#[derive(Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct MapNoveltiesMetadata {\n    /// A `list` of novelties.\n    pub list: Vec<usize>,\n}\n\nlibafl_bolts::impl_serdeany!(MapNoveltiesMetadata);\n\nimpl AsSlice for MapNoveltiesMetadata {\n    type Entry = usize;\n    /// Convert to a slice\n    #[must_use]\n    fn as_slice(&self) -> &[usize] {\n        self.list.as_slice()\n    }\n}\nimpl AsMutSlice for MapNoveltiesMetadata {\n    type Entry = usize;\n    /// Convert to a slice\n    #[must_use]\n    fn as_mut_slice(&mut self) -> &mut [usize] {\n        self.list.as_mut_slice()\n    }\n}\nimpl MapNoveltiesMetadata {\n    /// Creates a new [`struct@MapNoveltiesMetadata`]\n    #[must_use]\n    pub fn new(list: Vec<usize>) -> Self {\n        Self { list }\n    }\n}\n\n/// The state of [`MapFeedback`]\n#[derive(Default, Serialize, Deserialize, Clone, Debug)]\n#[serde(bound = \"T: DeserializeOwned\")]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct MapFeedbackMetadata<T>\nwhere\n    T: Default + Copy + 'static + Serialize,\n{\n    /// Contains information about untouched entries\n    pub history_map: Vec<T>,\n    /// Tells us how many non-initial entries there are in `history_map`\n    pub num_covered_map_indexes: usize,\n}\n\nlibafl_bolts::impl_serdeany!(\n    MapFeedbackMetadata<T: Debug + Default + Copy + 'static + Serialize + DeserializeOwned>,\n    <u8>,<u16>,<u32>,<u64>,<i8>,<i16>,<i32>,<i64>,<f32>,<f64>,<bool>,<char>,<usize>\n);\n\nimpl<T> MapFeedbackMetadata<T>\nwhere\n    T: Default + Copy + 'static + Serialize + DeserializeOwned + PartialEq,\n{\n    /// Create new `MapFeedbackMetadata`\n    #[must_use]\n    pub fn new(map_size: usize) -> Self {\n        Self {\n            history_map: vec![T::default(); map_size],\n            num_covered_map_indexes: 0,\n        }\n    }\n\n    /// Create new `MapFeedbackMetadata` using a name and a map.\n    /// The map can be shared.\n    /// `initial_elem_value` is used to calculate `Self.num_covered_map_indexes`\n    #[must_use]\n    \npub fn with_history_map(history_map: Vec<T>, initial_elem_value: T) -> Self {\n        let num_covered_map_indexes = history_map\n            .iter()\n            .fold(0, |acc, x| acc + usize::from(*x != initial_elem_value));\n        Self {\n            history_map,\n            num_covered_map_indexes,\n        }\n    }\n\n    /// Reset the map\n    pub fn reset(&mut self) -> Result<(), Error> {\n        let cnt = self.history_map.len();\n        for i in 0..cnt {\n            self.history_map[i] = T::default();\n        }\n        self.num_covered_map_indexes = 0;\n        Ok(())\n    }\n\n    /// Reset the map with any value\n    pub fn reset_with_value(&mut self, value: T) -> Result<(), Error> {\n        let cnt = self.history_map.len();\n        for i in 0..cnt {\n            self.history_map[i] = value;\n        }\n        // assume that resetting the map should indicate no coverage,\n        // regardless of value\n        self.num_covered_map_indexes = 0;\n        Ok(())\n    }\n}\n\n/// The most common AFL-like feedback type\n#[derive(Clone, Debug)]\npub struct MapFeedback<N, O, R, S, T> {\n    /// Indexes used in the last observation\n    indexes: bool,\n    /// New indexes observed in the last observation\n    novelties: Option<Vec<usize>>,\n    /// Name identifier of this instance\n    name: String,\n    /// Name identifier of the observer\n    observer_name: String,\n    /// Name of the feedback as shown in the `UserStats`\n    stats_name: String,\n    /// Phantom Data of Reducer\n    phantom: PhantomData<(N, O, R, S, T)>,\n}\n\nimpl<N, O, R, S, T> UsesObserver<S> for MapFeedback<N, O, R, S, T>\nwhere\n    S: UsesInput,\n    O: Observer<S>,\n{\n    type Observer = O;\n}\n\nimpl<N, O, R, S, T> Feedback<S> for MapFeedback<N, O, R, S, T>\nwhere\n    N: IsNovel<T>,\n    O: MapObserver<Entry = T> + for<'it> AsIter<'it, Item = T>,\n    R: Reducer<T>,\n    S: State + HasNamedMetadata,\n    T: Default + Copy + Serialize + for<'de> Deserialize<'de> + PartialEq + Debug + 'static,\n{\n    fn init_state(&mut self, state: &mut S) -> Result<(), Error> {\n        // Initialize `MapFeedbackMetadata` with an empty vector and add it to the state.\n        // The `MapFeedbackMetadata` would be resized on-demand in `is_interesting`\n        state.add_named_metadata(&self.name, MapFeedbackMetadata::<T>::default());\n        Ok(())\n    }\n\n    #[rustversion::nightly]\n    default fn is_interesting<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        input: &S::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>,\n    {\n        Ok(self.is_interesting_default(state, manager, input, observers, exit_kind))\n    }\n\n    #[rustversion::not(nightly)]\n    fn is_interesting<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        input: &<S as UsesInput>::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>,\n    {\n        Ok(self.is_interesting_default(state, manager, input, observers, exit_kind))\n    }\n\n    fn append_metadata<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        observers: &OT,\n        testcase: &mut Testcase<S::Input>,\n    ) -> Result<(), Error>\n    where\n        OT: ObserversTuple<S>,\n        EM: EventFirer<State = S>,\n    {\n        if let Some(novelties) = self.novelties.as_mut().map(core::mem::take) {\n            let meta = MapNoveltiesMetadata::new(novelties);\n            testcase.add_metadata(meta);\n        }\n        let observer = observers.match_name::<O>(&self.observer_name).unwrap();\n        let initial = observer.initial();\n        let map_state = state\n            .named_metadata_map_mut()\n            .get_mut::<MapFeedbackMetadata<T>>(&self.name)\n            .unwrap();\n        let len = observer.len();\n        if map_state.history_map.len() < len {\n            map_state.history_map.resize(len, observer.initial());\n        }\n\n        let history_map = map_state.history_map.as_mut_slice();\n        if self.indexes {\n            let mut indices = Vec::new();\n\n            for (i, value) in observer\n                .as_iter()\n                .copied()\n                .enumerate()\n                .filter(|(_, value)| *value != initial)\n            {\n                if history_map[i] == initial {\n                    map_state.num_covered_map_indexes += 1;\n                }\n                history_map[i] = R::reduce(history_map[i], value);\n                indices.push(i);\n            }\n            let meta = MapIndexesMetadata::new(indices);\n            testcase.add_metadata(meta);\n        } else {\n            for (i, value) in observer\n                .as_iter()\n                .copied()\n                .enumerate()\n                .filter(|(_, value)| *value != initial)\n            {\n                if history_map[i] == initial {\n                    map_state.num_covered_map_indexes += 1;\n                }\n                history_map[i] = R::reduce(history_map[i], value);\n            }\n        }\n\n        debug_assert!(\n            history_map\n                .iter()\n                .fold(0, |acc, x| acc + usize::from(*x != initial))\n                == map_state.num_covered_map_indexes,\n            \"history_map had {} filled, but map_state.num_covered_map_indexes was {}\",\n            history_map\n                .iter()\n                .fold(0, |acc, x| acc + usize::from(*x != initial)),\n            map_state.num_covered_map_indexes,\n        );\n\n        // at this point you are executing this code, the testcase is always interesting\n        let covered = map_state.num_covered_map_indexes;\n        let len = history_map.len();\n        // opt: if not tracking optimisations, we technically don't show the *current* history\n        // map but the *last* history map; this is better than walking over and allocating\n        // unnecessarily\n        manager.fire(\n            state,\n            Event::UpdateUserStats {\n                name: self.stats_name.to_string(),\n                value: UserStats::new(\n                    UserStatsValue::Ratio(covered as u64, len as u64),\n                    AggregatorOps::Avg,\n                ),\n                phantom: PhantomData,\n            },\n        )?;\n\n        Ok(())\n    }\n}\n\n/// Specialize for the common coverage map size, maximization of u8s\n#[rustversion::nightly]\nimpl<O, S> Feedback<S> for MapFeedback<DifferentIsNovel, O, MaxReducer, S, u8>\nwhere\n    O: MapObserver<Entry = u8> + AsSlice<Entry = u8>,\n    for<'it> O: AsIter<'it, Item = u8>,\n    S: State + HasNamedMetadata,\n{\n    #[allow(clippy::wrong_self_convention)]\n    #[allow(clippy::needless_range_loop)]\n    fn is_interesting<EM, OT>(\n        &mut self,\n        state: &mut S,\n        _manager: &mut EM,\n        _input: &S::Input,\n        observers: &OT,\n        _exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>,\n    {\n        // 128 bits vectors\n        type VectorType = core::simd::u8x16;\n\n        let mut interesting = false;\n        // TODO Replace with match_name_type when stable\n        let observer = observers.match_name::<O>(&self.observer_name).unwrap();\n\n        let map_state = state\n            .named_metadata_map_mut()\n            .get_mut::<MapFeedbackMetadata<u8>>(&self.name)\n            .unwrap();\n        let size = observer.usable_count();\n        let len = observer.len();\n        if map_state.history_map.len() < len {\n            map_state.history_map.resize(len, u8::default());\n        }\n\n        let map = observer.as_slice();\n        debug_assert!(map.len() >= size);\n\n        let history_map = map_state.history_map.as_slice();\n\n        // Non vector implementation for reference\n        /*for (i, history) in history_map.iter_mut().enumerate() {\n            let item = map[i];\n            let reduced = MaxReducer::reduce(*history, item);\n            if DifferentIsNovel::is_novel(*history, reduced) {\n                *history = reduced;\n                interesting = true;\n                if self.novelties.is_some() {\n                    self.novelties.as_mut().unwrap().push(i);\n                }\n            }\n        }*/\n\n        let steps = size / VectorType::LEN;\n        let left = size % VectorType::LEN;\n\n        if let Some(novelties) = self.novelties.as_mut() {\n            novelties.clear();\n            for step in 0..steps {\n                let i = step * VectorType::LEN;\n                let history = VectorType::from_slice(&history_map[i..]);\n                let items = VectorType::from_slice(&map[i..]);\n\n                if items.simd_max(history) != history {\n                    interesting = true;\n                    unsafe {\n                        for j in i..(i + VectorType::LEN) {\n                            let item = *map.get_unchecked(j);\n                            if item > *history_map.get_unchecked(j) {\n                                novelties.push(j);\n                            }\n                        }\n                    }\n                }\n            }\n\n            for j in (size - left)..size {\n                unsafe {\n                    let item = *map.get_unchecked(j);\n                    if item > *history_map.get_unchecked(j) {\n                        interesting = true;\n                        novelties.push(j);\n                    }\n                }\n            }\n        } else {\n            for step in 0..steps {\n                let i = step * VectorType::LEN;\n                let history = VectorType::from_slice(&history_map[i..]);\n                let items = VectorType::from_slice(&map[i..]);\n\n                if items.simd_max(history) != history {\n                    interesting = true;\n                    break;\n                }\n            }\n\n            if !interesting {\n                for j in (size - left)..size {\n                    unsafe {\n                        let item = *map.get_unchecked(j);\n                        if item > *history_map.get_unchecked(j) {\n                            interesting = true;\n                            break;\n                        }\n                    }\n                }\n            }\n        }\n\n        Ok(interesting)\n    }\n}\n\nimpl<N, O, R, S, T> Named for MapFeedback<N, O, R, S, T> {\n    #[inline]\n    fn name(&self) -> &str {\n        self.name.as_str()\n    }\n}\n\nimpl<N, O, R, S, T> HasObserverName for MapFeedback<N, O, R, S, T>\nwhere\n    T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug,\n    R: Reducer<T>,\n    N: IsNovel<T>,\n    O: MapObserver<Entry = T>,\n    for<'it> O: AsIter<'it, Item = T>,\n    S: HasNamedMetadata,\n{\n    #[inline]\n    fn observer_name(&self) -> &str {\n        self.observer_name.as_str()\n    }\n}\n\nfn create_stats_name(name: &str) -> String {\n    name.to_lowercase()\n}\n\nimpl<N, O, R, S, T> MapFeedback<N, O, R, S, T>\nwhere\n    T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug,\n    R: Reducer<T>,\n    O: MapObserver<Entry = T>,\n    for<'it> O: AsIter<'it, Item = T>,\n    N: IsNovel<T>,\n    S: UsesInput + HasNamedMetadata,\n{\n    /// Create new `MapFeedback`\n    #[must_use]\n    pub fn new(map_observer: &O) -> Self {\n        Self {\n            indexes: false,\n            novelties: None,\n            name: map_observer.name().to_string(),\n            observer_name: map_observer.name().to_string(),\n            stats_name: create_stats_name(map_observer.name()),\n            phantom: PhantomData,\n        }\n    }\n\n    /// Create new `MapFeedback` specifying if it must track indexes of used entries and/or novelties\n    #[must_use]\n    pub fn tracking(map_observer: &O, track_indexes: bool, track_novelties: bool) -> Self {\n        Self {\n            indexes: track_indexes,\n            novelties: if track_novelties { Some(vec![]) } else { None },\n            name: map_observer.name().to_string(),\n            observer_name: map_observer.name().to_string(),\n            stats_name: create_stats_name(map_observer.name()),\n            phantom: PhantomData,\n        }\n    }\n\n    /// Create new `MapFeedback`\n    #[must_use]\n    pub fn with_names(name: &'static str, observer_name: &'static str) -> Self {\n        Self {\n            indexes: false,\n            novelties: None,\n            name: name.to_string(),\n            observer_name: observer_name.to_string(),\n            stats_name: create_stats_name(name),\n            phantom: PhantomData,\n        }\n    }\n\n    /// Creating a new `MapFeedback` with a specific name. This is usefully whenever the same\n    /// feedback is needed twice, but with a different history. Using `new()` always results in the\n    /// same name and therefore also the same history.\n    #[must_use]\n    pub fn with_name(name: &'static str, map_observer: &O) -> Self {\n        Self {\n            indexes: false,\n            novelties: None,\n            name: name.to_string(),\n            observer_name: map_observer.name().to_string(),\n            stats_name: create_stats_name(name),\n            phantom: PhantomData,\n        }\n    }\n\n    /// Create new `MapFeedback` specifying if it must track indexes of used entries and/or novelties\n    #[must_use]\n    pub fn with_names_tracking(\n        name: &'static str,\n        observer_name: &'static str,\n        track_indexes: bool,\n...\n", "needle_token_start": 12136, "needle_token_end": 12239, "code_context_ntokens": 16253, "output": ["```rust\n/// Create new `MapFeedbackMetadata` using a name and a map.\n/// The map can be shared.\n/// `initial_elem_value` is used to calculate `Self.num_covered_map_indexes`\n#[must_use]\npub fn with_history_map(history_map: Vec<T>, initial_elem_value: T) -> Self {\n    let num_covered_map_indexes = history_map\n        .iter()\n        .fold(0, |acc, x| acc + usize::from(*x != initial_elem_value));\n    Self {\n        history_map,\n        num_covered_map_indexes,\n    }\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "store_probability", "language": "rust", "path": "libafl/src/schedulers/probabilistic_sampling.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to compute and store the probability score of a specific item in the corpus, updating the total probability of all items.\n2. **Input**: It takes a mutable reference to the state, which includes the corpus and metadata, and an identifier for the specific corpus item.\n3. **Output**: Returns a result indicating success or an error if the operation fails.\n4. **Procedure**: The function first computes the probability score for the specified corpus item using a scoring function. It then retrieves the metadata map from the state, updates the probability score for the item, and increments the total probability stored in the metadata.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: libafl/src/feedbacks/mod.rs\n//! The feedbacks reduce observer state after each run to a single `is_interesting`-value.\n//! If a testcase is interesting, it may be added to a Corpus.\n//!\n\n// TODO: make S of Feedback<S> an associated type when specialisation + AT is stable\n\npub mod map;\npub use map::*;\n\npub mod differential;\npub use differential::DiffFeedback;\n#[cfg(feature = \"std\")]\npub mod concolic;\n#[cfg(feature = \"std\")]\npub use concolic::ConcolicFeedback;\n\n#[cfg(feature = \"std\")]\npub mod new_hash_feedback;\n#[cfg(feature = \"std\")]\npub use new_hash_feedback::NewHashFeedback;\n#[cfg(feature = \"std\")]\npub use new_hash_feedback::NewHashFeedbackMetadata;\n\n#[cfg(feature = \"nautilus\")]\npub mod nautilus;\npub mod transferred;\n\n/// The module for list feedback\npub mod list;\nuse alloc::string::{String, ToString};\nuse core::{\n    fmt::{self, Debug, Formatter},\n    marker::PhantomData,\n};\n\nuse libafl_bolts::Named;\npub use list::*;\n#[cfg(feature = \"nautilus\")]\npub use nautilus::*;\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    corpus::Testcase,\n    events::EventFirer,\n    executors::ExitKind,\n    observers::{ObserversTuple, TimeObserver},\n    state::State,\n    Error,\n};\n\n/// Feedbacks evaluate the observers.\n/// Basically, they reduce the information provided by an observer to a value,\n/// indicating the \"interestingness\" of the last run.\npub trait Feedback<S>: Named\nwhere\n    S: State,\n{\n    /// Initializes the feedback state.\n    /// This method is called after that the `State` is created.\n    fn init_state(&mut self, _state: &mut S) -> Result<(), Error> {\n        Ok(())\n    }\n\n    /// `is_interesting ` return if an input is worth the addition to the corpus\n    #[allow(clippy::wrong_self_convention)]\n    fn is_interesting<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        input: &S::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>;\n\n    /// Returns if the result of a run is interesting and the value input should be stored in a corpus.\n    /// It also keeps track of introspection stats.\n    #[cfg(feature = \"introspection\")]\n    #[allow(clippy::too_many_arguments)]\n    #[allow(clippy::wrong_self_convention)]\n    fn is_interesting_introspection<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        input: &S::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>,\n    {\n        // Start a timer for this feedback\n        let start_time = libafl_bolts::cpu::read_time_counter();\n\n        // Execute this feedback\n        let ret = self.is_interesting(state, manager, input, observers, exit_kind);\n\n        // Get the elapsed time for checking this feedback\n        let elapsed = libafl_bolts::cpu::read_time_counter() - start_time;\n\n        // Add this stat to the feedback metrics\n        state\n            .introspection_monitor_mut()\n            .update_feedback(self.name(), elapsed);\n\n        ret\n    }\n\n    /// Append to the testcase the generated metadata in case of a new corpus item\n    #[inline]\n    #[allow(unused_variables)]\n    fn append_metadata<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        observers: &OT,\n        testcase: &mut Testcase<S::Input>,\n    ) -> Result<(), Error>\n    where\n        OT: ObserversTuple<S>,\n        EM: EventFirer<State = S>,\n    {\n        Ok(())\n    }\n\n    /// Discard the stored metadata in case that the testcase is not added to the corpus\n    #[inline]\n    fn discard_metadata(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> {\n        Ok(())\n    }\n}\n\n/// Has an associated observer name (mostly used to retrieve the observer with `MatchName` from an `ObserverTuple`)\npub trait HasObserverName {\n    /// The name associated with the observer\n    fn observer_name(&self) -> &str;\n}\n\n/// A combined feedback consisting of multiple [`Feedback`]s\n#[derive(Debug)]\npub struct CombinedFeedback<A, B, FL, S>\nwhere\n    A: Feedback<S>,\n    B: Feedback<S>,\n    FL: FeedbackLogic<A, B, S>,\n    S: State,\n{\n    /// First [`Feedback`]\n    pub first: A,\n    /// Second [`Feedback`]\n    pub second: B,\n    name: String,\n    phantom: PhantomData<(S, FL)>,\n}\n\nimpl<A, B, FL, S> Named for CombinedFeedback<A, B, FL, S>\nwhere\n    A: Feedback<S>,\n    B: Feedback<S>,\n    FL: FeedbackLogic<A, B, S>,\n    S: State,\n{\n    fn name(&self) -> &str {\n        self.name.as_ref()\n    }\n}\n\nimpl<A, B, FL, S> CombinedFeedback<A, B, FL, S>\nwhere\n    A: Feedback<S>,\n    B: Feedback<S>,\n    FL: FeedbackLogic<A, B, S>,\n    S: State,\n{\n    /// Create a new combined feedback\n    pub fn new(first: A, second: B) -> Self {\n        let name = format!(\"{} ({},{})\", FL::name(), first.name(), second.name());\n        Self {\n            first,\n            second,\n            name,\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<A, B, FL, S> Feedback<S> for CombinedFeedback<A, B, FL, S>\nwhere\n    A: Feedback<S>,\n    B: Feedback<S>,\n    FL: FeedbackLogic<A, B, S>,\n    S: State,\n{\n    fn init_state(&mut self, state: &mut S) -> Result<(), Error> {\n        self.first.init_state(state)?;\n        self.second.init_state(state)?;\n        Ok(())\n    }\n\n    #[allow(clippy::wrong_self_convention)]\n    fn is_interesting<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        input: &S::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>,\n    {\n        FL::is_pair_interesting(\n            &mut self.first,\n            &mut self.second,\n            state,\n            manager,\n            input,\n            observers,\n            exit_kind,\n        )\n    }\n\n    #[cfg(feature = \"introspection\")]\n    #[allow(clippy::wrong_self_convention)]\n    fn is_interesting_introspection<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        input: &S::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>,\n    {\n        FL::is_pair_interesting_introspection(\n            &mut self.first,\n            &mut self.second,\n            state,\n            manager,\n            input,\n            observers,\n            exit_kind,\n        )\n    }\n\n    #[inline]\n    fn append_metadata<EM, OT>(\n        &mut self,\n        state: &mut S,\n        manager: &mut EM,\n        observers: &OT,\n        testcase: &mut Testcase<S::Input>,\n    ) -> Result<(), Error>\n    where\n        OT: ObserversTuple<S>,\n        EM: EventFirer<State = S>,\n    {\n        self.first\n            .append_metadata(state, manager, observers, testcase)?;\n        self.second\n            .append_metadata(state, manager, observers, testcase)\n    }\n\n    #[inline]\n    fn discard_metadata(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> {\n        self.first.discard_metadata(state, input)?;\n        self.second.discard_metadata(state, input)\n    }\n}\n\n/// Logical combination of two feedbacks\npub trait FeedbackLogic<A, B, S>: 'static\nwhere\n    A: Feedback<S>,\n    B: Feedback<S>,\n    S: State,\n{\n    /// The name of this combination\n    fn name() -> &'static str;\n\n    /// If the feedback pair is interesting\n    fn is_pair_interesting<EM, OT>(\n        first: &mut A,\n        second: &mut B,\n        state: &mut S,\n        manager: &mut EM,\n        input: &S::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>;\n\n    /// If this pair is interesting (with introspection features enabled)\n    #[cfg(feature = \"introspection\")]\n    #[allow(clippy::too_many_arguments)]\n    fn is_pair_interesting_introspection<EM, OT>(\n        first: &mut A,\n        second: &mut B,\n        state: &mut S,\n        manager: &mut EM,\n        input: &S::Input,\n        observers: &OT,\n        exit_kind: &ExitKind,\n    ) -> Result<bool, Error>\n    where\n        EM: EventFirer<State = S>,\n        OT: ObserversTuple<S>;\n}\n\n/// Factory for feedbacks which should be sensitive to an existing context, e.g. observer(s) from a\n/// specific execution\npub trait FeedbackFactory<F, S, T>\nwhere\n    F: Feedback<S>,\n    S: State,\n{\n    /// Create the feedback from the provided context\n    fn create_feedback(&self, ctx: &T) -> F;\n}\n\nimpl<FE, FU, S, T> FeedbackFactory<FE, S, T> for FU\nwhere\n    FU: Fn(&T) -> FE,\n    FE: Feedback<S>,\n    S: State,\n{\n    fn create_feedback(&self, ctx: &T) -> FE {\n        self(ctx)\n    }\n}\n\n/// A feedback factory which merely invokes `::default()` for the feedback type provided\n#[derive(Default, Debug, Copy, Clone)]\npub struct DefaultFeedbackFactory<F>\nwhere\n    F: Default,\n{\n    phantom: PhantomData<F>,\n}\n\nimpl<F> DefaultFeedbackFactory<F>\nwhere\n    F: Default,\n{\n    /// Create the feedback factory\n    #[must_use]\n    pub fn new() -> Self {\n        Self::default()\n    }\n}\n\nimpl<F, S, T> FeedbackFactory<F, S, T> for DefaultFeedbackFactory<F>\nwhere\n    F: Feedback<S> + Default,\n    S: State,\n{\n    fn create_feedback(&self, _ctx: &T) -> F {\n        F::default()\n    }\n}\n\n/// Eager `OR` combination of two feedbacks\n#[derive(Debug, Clone)]\npub struct LogicEagerOr {}\n\n/// Fast `OR` combination of two feedbacks\n#[derive(Debug, Clone)]\npub struct LogicFastOr {}\n\n/// Eager `AND` combination of two feedbacks\n#[derive(Debug, Clone)]\npub struct LogicEagerAnd {}\n\n/// Fast `AND` combination of two feedbacks\n#[derive(Debug, Clone)]\npub struct LogicFastAnd {}\n\nimpl<A, B, S> FeedbackLogic<A, B, S> for LogicEagerOr\nwhere\n    A: Feedback<S>,\n    B: Feedback<S>,\n    S: State,\n{\n    fn name() -> &'static str {\n        \"Eager OR\"\n    }\n\n    fn is_pair_interesting<EM, OT>(\n        first: &mut A,\n        second: &mut B,\n        state: &mut S,\n...\n// Path: libafl/src/schedulers/queue.rs\n//! The queue corpus scheduler implements an AFL-like queue mechanism\n\nuse alloc::borrow::ToOwned;\nuse core::marker::PhantomData;\n\nuse crate::{\n    corpus::{Corpus, CorpusId, HasTestcase},\n    schedulers::{RemovableScheduler, Scheduler},\n    state::{HasCorpus, State, UsesState},\n    Error,\n};\n\n/// Walk the corpus in a queue-like fashion\n#[derive(Debug, Clone)]\npub struct QueueScheduler<S> {\n    phantom: PhantomData<S>,\n}\n\nimpl<S> UsesState for QueueScheduler<S>\nwhere\n    S: State,\n{\n    type State = S;\n}\n\nimpl<S> RemovableScheduler for QueueScheduler<S> where S: HasCorpus + HasTestcase + State {}\n\nimpl<S> Scheduler for QueueScheduler<S>\nwhere\n    S: HasCorpus + HasTestcase + State,\n{\n    fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> {\n        // Set parent id\n        let current_idx = *state.corpus().current();\n        state\n            .corpus()\n            .get(idx)?\n            .borrow_mut()\n            .set_parent_id_optional(current_idx);\n\n        Ok(())\n    }\n\n    /// Gets the next entry in the queue\n    fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {\n        if state.corpus().count() == 0 {\n            Err(Error::empty(\n                \"No entries in corpus. This often implies the target is not properly instrumented.\"\n                    .to_owned(),\n            ))\n        } else {\n            let id = state\n                .corpus()\n                .current()\n                .map(|id| state.corpus().next(id))\n                .flatten()\n                .unwrap_or_else(|| state.corpus().first().unwrap());\n            self.set_current_scheduled(state, Some(id))?;\n            Ok(id)\n        }\n    }\n}\n\nimpl<S> QueueScheduler<S> {\n    /// Creates a new `QueueScheduler`\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            phantom: PhantomData,\n        }\n    }\n}\n\nimpl<S> Default for QueueScheduler<S> {\n    fn default() -> Self {\n        Self::new()\n    }\n}\n\n#[cfg(test)]\n#[cfg(feature = \"std\")]\nmod tests {\n\n    use std::{fs, path::PathBuf};\n\n    use libafl_bolts::rands::StdRand;\n\n    use crate::{\n        corpus::{Corpus, OnDiskCorpus, Testcase},\n        feedbacks::ConstFeedback,\n        inputs::bytes::BytesInput,\n        schedulers::{QueueScheduler, Scheduler},\n        state::{HasCorpus, StdState},\n    };\n\n    #[test]\n    fn test_queuecorpus() {\n        let rand = StdRand::with_seed(4);\n        let mut scheduler = QueueScheduler::new();\n\n        let mut q =\n            OnDiskCorpus::<BytesInput>::new(PathBuf::from(\"target/.test/fancy/path\")).unwrap();\n        let t = Testcase::with_filename(BytesInput::new(vec![0_u8; 4]), \"fancyfile\".into());\n        q.add(t).unwrap();\n\n        let objective_q =\n            OnDiskCorpus::<BytesInput>::new(PathBuf::from(\"target/.test/fancy/objective/path\"))\n                .unwrap();\n\n        let mut feedback = ConstFeedback::new(false);\n        let mut objective = ConstFeedback::new(false);\n\n        let mut state = StdState::new(rand, q, objective_q, &mut feedback, &mut objective).unwrap();\n\n        let next_idx = scheduler.next(&mut state).unwrap();\n        let filename = state\n            .corpus()\n            .get(next_idx)\n            .unwrap()\n            .borrow()\n            .filename()\n            .as_ref()\n            .unwrap()\n            .clone();\n\n        assert_eq!(filename, \"fancyfile\");\n\n        fs::remove_dir_all(\"target/.test/fancy/path\").unwrap();\n    }\n}\n\n// Path: libafl/src/schedulers/weighted.rs\n//! The queue corpus scheduler with weighted queue item selection from aflpp (`https://github.com/AFLplusplus/AFLplusplus/blob/1d4f1e48797c064ee71441ba555b29fc3f467983/src/afl-fuzz-queue.c#L32`)\n//! This queue corpus scheduler needs calibration stage.\n\nuse alloc::string::{String, ToString};\nuse core::marker::PhantomData;\n\nuse hashbrown::HashMap;\nuse libafl_bolts::rands::Rand;\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    corpus::{Corpus, CorpusId, HasTestcase, Testcase},\n    inputs::UsesInput,\n    observers::{MapObserver, ObserversTuple},\n    random_corpus_id,\n    schedulers::{\n        powersched::{PowerSchedule, SchedulerMetadata},\n        testcase_score::{CorpusWeightTestcaseScore, TestcaseScore},\n        HasAFLRemovableScheduler, HasAFLSchedulerMetadata, RemovableScheduler, Scheduler,\n    },\n    state::{HasCorpus, HasMetadata, HasRand, State, UsesState},\n    Error,\n};\n\n/// The Metadata for `WeightedScheduler`\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\n#[derive(Serialize, Deserialize, Clone, Debug)]\npub struct WeightedScheduleMetadata {\n    /// The fuzzer execution spent in the current cycles\n    runs_in_current_cycle: usize,\n    /// Alias table for weighted queue entry selection\n    alias_table: HashMap<CorpusId, CorpusId>,\n    /// Probability for which queue entry is selected\n    alias_probability: HashMap<CorpusId, f64>,\n}\n\nimpl Default for WeightedScheduleMetadata {\n    fn default() -> Self {\n        Self::new()\n    }\n}\n\nimpl WeightedScheduleMetadata {\n    /// Constructor for `WeightedScheduleMetadata`\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            runs_in_current_cycle: 0,\n            alias_table: HashMap::default(),\n            alias_probability: HashMap::default(),\n        }\n    }\n\n    /// The getter for `runs_in_current_cycle`\n    #[must_use]\n    pub fn runs_in_current_cycle(&self) -> usize {\n        self.runs_in_current_cycle\n    }\n\n    /// The setter for `runs_in_current_cycle`\n    pub fn set_runs_current_cycle(&mut self, cycles: usize) {\n        self.runs_in_current_cycle = cycles;\n    }\n\n    /// The getter for `alias_table`\n    #[must_use]\n    pub fn alias_table(&self) -> &HashMap<CorpusId, CorpusId> {\n        &self.alias_table\n    }\n\n    /// The setter for `alias_table`\n    pub fn set_alias_table(&mut self, table: HashMap<CorpusId, CorpusId>) {\n        self.alias_table = table;\n    }\n\n    /// The getter for `alias_probability`\n    #[must_use]\n    pub fn alias_probability(&self) -> &HashMap<CorpusId, f64> {\n        &self.alias_probability\n    }\n\n    /// The setter for `alias_probability`\n    pub fn set_alias_probability(&mut self, probability: HashMap<CorpusId, f64>) {\n        self.alias_probability = probability;\n    }\n}\n\nlibafl_bolts::impl_serdeany!(WeightedScheduleMetadata);\n\n/// A corpus scheduler using power schedules with weighted queue item selection algo.\n#[derive(Clone, Debug)]\npub struct WeightedScheduler<F, O, S> {\n    strat: Option<PowerSchedule>,\n    map_observer_name: String,\n    last_hash: usize,\n    phantom: PhantomData<(F, O, S)>,\n}\n\nimpl<F, O, S> WeightedScheduler<F, O, S>\nwhere\n    F: TestcaseScore<S>,\n    O: MapObserver,\n    S: HasCorpus + HasMetadata + HasRand,\n{\n    /// Create a new [`WeightedScheduler`] without any power schedule\n    #[must_use]\n    pub fn new(state: &mut S, map_observer: &O) -> Self {\n        Self::with_schedule(state, map_observer, None)\n    }\n\n    /// Create a new [`WeightedScheduler`]\n    #[must_use]\n    pub fn with_schedule(state: &mut S, map_observer: &O, strat: Option<PowerSchedule>) -> Self {\n        let _ = state.metadata_or_insert_with(|| SchedulerMetadata::new(strat));\n        let _ = state.metadata_or_insert_with(WeightedScheduleMetadata::new);\n\n        Self {\n            strat,\n            map_observer_name: map_observer.name().to_string(),\n            last_hash: 0,\n            phantom: PhantomData,\n        }\n    }\n\n    #[must_use]\n    /// Getter for `strat`\n    pub fn strat(&self) -> &Option<PowerSchedule> {\n        &self.strat\n    }\n\n    /// Create a new alias table when the fuzzer finds a new corpus entry\n    #[allow(\n        clippy::unused_self,\n        clippy::similar_names,\n        clippy::cast_precision_loss,\n        clippy::cast_lossless\n    )]\n    pub fn create_alias_table(&self, state: &mut S) -> Result<(), Error> {\n        let n = state.corpus().count();\n\n        let mut alias_table: HashMap<CorpusId, CorpusId> = HashMap::default();\n        let mut alias_probability: HashMap<CorpusId, f64> = HashMap::default();\n        let mut weights: HashMap<CorpusId, f64> = HashMap::default();\n\n        let mut p_arr: HashMap<CorpusId, f64> = HashMap::default();\n        let mut s_arr: HashMap<usize, CorpusId> = HashMap::default();\n        let mut l_arr: HashMap<usize, CorpusId> = HashMap::default();\n\n        let mut sum: f64 = 0.0;\n\n        for i in state.corpus().ids() {\n            let mut testcase = state.corpus().get(i)?.borrow_mut();\n            let weight = F::compute(state, &mut *testcase)?;\n            weights.insert(i, weight);\n            sum += weight;\n        }\n\n        for (i, w) in &weights {\n            p_arr.insert(*i, w * (n as f64) / sum);\n        }\n\n        // # of items in queue S\n        let mut n_s = 0;\n\n        // # of items in queue L\n        let mut n_l = 0;\n        // Divide P into two queues, S and L\n        for s in state.corpus().ids().rev() {\n            if *p_arr.get(&s).unwrap() < 1.0 {\n                s_arr.insert(n_s, s);\n                n_s += 1;\n            } else {\n                l_arr.insert(n_l, s);\n                n_l += 1;\n            }\n        }\n\n        while n_s > 0 && n_l > 0 {\n            n_s -= 1;\n            n_l -= 1;\n            let a = *s_arr.get(&n_s).unwrap();\n            let g = *l_arr.get(&n_l).unwrap();\n\n            alias_probability.insert(a, *p_arr.get(&a).unwrap());\n            alias_table.insert(a, g);\n            *p_arr.get_mut(&g).unwrap() += p_arr.get(&a).unwrap() - 1.0;\n\n            if *p_arr.get(&g).unwrap() < 1.0 {\n                *s_arr.get_mut(&n_s).unwrap() = g;\n                n_s += 1;\n            } else {\n                *l_arr.get_mut(&n_l).unwrap() = g;\n                n_l += 1;\n            }\n        }\n\n        while n_l > 0 {\n            n_l -= 1;\n            alias_probability.insert(*l_arr.get(&n_l).unwrap(), 1.0);\n        }\n\n        while n_s > 0 {\n            n_s -= 1;\n            alias_probability.insert(*s_arr.get(&n_s).unwrap(), 1.0);\n        }\n\n        let wsmeta = state.metadata_mut::<WeightedScheduleMetadata>()?;\n\n        // Update metadata\n        wsmeta.set_alias_probability(alias_probability);\n        wsmeta.set_alias_table(alias_table);\n        Ok(())\n    }\n}\n\nimpl<F, O, S> UsesState for WeightedScheduler<F, O, S>\nwhere\n    S: State,\n{\n    type State = S;\n}\n\nimpl<F, O, S> HasAFLRemovableScheduler for WeightedScheduler<F, O, S>\nwhere\n    F: TestcaseScore<S>,\n    S: State + HasTestcase + HasMetadata + HasCorpus + HasRand,\n    O: MapObserver,\n{\n}\n\nimpl<F, O, S> RemovableScheduler for WeightedScheduler<F, O, S>\nwhere\n    F: TestcaseScore<S>,\n    O: MapObserver,\n    S: HasCorpus + HasMetadata + HasRand + HasTestcase + State,\n{\n    fn on_remove(\n        &mut self,\n        state: &mut Self::State,\n        idx: CorpusId,\n        prev: &Option<Testcase<<Self::State as UsesInput>::Input>>,\n    ) -> Result<(), Error> {\n        self.on_remove_metadata(state, idx, prev)\n    }\n\n    fn on_replace(\n        &mut self,\n        state: &mut Self::State,\n        idx: CorpusId,\n        prev: &Testcase<<Self::State as UsesInput>::Input>,\n    ) -> Result<(), Error> {\n        self.on_replace_metadata(state, idx, prev)\n    }\n}\n\nimpl<F, O, S> HasAFLSchedulerMetadata<O, S> for WeightedScheduler<F, O, S>\nwhere\n    F: TestcaseScore<S>,\n    S: HasCorpus + HasMetadata + HasTestcase + HasRand + State,\n    O: MapObserver,\n{\n    fn last_hash(&self) -> usize {\n        self.last_hash\n    }\n\n    fn set_last_hash(&mut self, hash: usize) {\n        self.last_hash = hash;\n    }\n\n    fn map_observer_name(&self) -> &str {\n        &self.map_observer_name\n    }\n}\n\nimpl<F, O, S> Scheduler for WeightedScheduler<F, O, S>\nwhere\n    F: TestcaseScore<S>,\n    O: MapObserver,\n    S: HasCorpus + HasMetadata + HasRand + HasTestcase + State,\n{\n    /// Called when a [`Testcase`] is added to the corpus\n    fn on_add(&mut self, state: &mut S, idx: CorpusId) -> Result<(), Error> {\n        self.on_add_metadata(state, idx)?;\n        self.create_alias_table(state)\n    }\n\n    fn on_evaluation<OT>(\n        &mut self,\n        state: &mut Self::State,\n        input: &<Self::State as UsesInput>::Input,\n        observers: &OT,\n    ) -> Result<(), Error>\n    where\n        OT: ObserversTuple<Self::State>,\n    {\n        self.on_evaluation_metadata(state, input, observers)\n    }\n\n    #[allow(clippy::similar_names, clippy::cast_precision_loss)]\n    fn next(&mut self, state: &mut S) -> Result<CorpusId, Error> {\n        let corpus_counts = state.corpus().count();\n        if corpus_counts == 0 {\n            Err(Error::empty(String::from(\n                \"No entries in corpus. This often implies the target is not properly instrumented.\",\n            )))\n        } else {\n            let s = random_corpus_id!(state.corpus(), state.rand_mut());\n\n            // Choose a random value between 0.000000000 and 1.000000000\n            let probability = state.rand_mut().between(0, 1000000000) as f64 / 1000000000_f64;\n\n            let wsmeta = state.metadata_mut::<WeightedScheduleMetadata>()?;\n\n            let current_cycles = wsmeta.runs_in_current_cycle();\n\n            // TODO deal with corpus_counts decreasing due to removals\n            if current_cycles >= corpus_counts {\n                wsmeta.set_runs_current_cycle(0);\n            } else {\n                wsmeta.set_runs_current_cycle(current_cycles + 1);\n            }\n\n            let idx = if probability < *wsmeta.alias_probability().get(&s).unwrap() {\n                s\n            } else {\n                *wsmeta.alias_table().get(&s).unwrap()\n            };\n\n            // Update depth\n            if current_cycles > corpus_counts {\n                let psmeta = state.metadata_mut::<SchedulerMetadata>()?;\n                psmeta.set_queue_cycles(psmeta.queue_cycles() + 1);\n            }\n\n            self.set_current_scheduled(state, Some(idx))?;\n            Ok(idx)\n        }\n    }\n\n    /// Set current fuzzed corpus id and `scheduled_count`\n    fn set_current_scheduled(\n        &mut self,\n        state: &mut Self::State,\n        next_idx: Option<CorpusId>,\n    ) -> Result<(), Error> {\n        self.on_next_metadata(state, next_idx)?;\n\n        *state.corpus_mut().current_mut() = next_idx;\n        Ok(())\n    }\n}\n\n/// The standard corpus weight, same as aflpp\npub type StdWeightedScheduler<O, S> = WeightedScheduler<CorpusWeightTestcaseScore<S>, O, S>;\n\n// Path: libafl/src/schedulers/tuneable.rs\n//! The queue corpus scheduler implements an AFL-like queue mechanism\n//! The [`TuneableScheduler`] extends the queue scheduler with a method to\n//! chose the next corpus entry manually\n\nuse alloc::borrow::ToOwned;\nuse core::marker::PhantomData;\n\nuse libafl_bolts::impl_serdeany;\nuse serde::{Deserialize, Serialize};\n\nuse super::RemovableScheduler;\nuse crate::{\n    corpus::{Corpus, CorpusId, HasTestcase},\n    schedulers::Scheduler,\n    state::{HasCorpus, HasMetadata, State, UsesState},\n    Error,\n};\n\n#[derive(Default, Clone, Copy, Eq, PartialEq, Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\nstruct TuneableSchedulerMetadata {\n    next: Option<CorpusId>,\n}\n\nimpl_serdeany!(TuneableSchedulerMetadata);\n\n/// Walk the corpus in a queue-like fashion\n/// With the specific `set_next` method, we can chose the next corpus entry manually\n#[derive(Debug, Clone)]\npub struct TuneableScheduler<S> {\n    phantom: PhantomData<S>,\n}\n\nimpl<S> TuneableScheduler<S>\nwhere\n    S: HasMetadata + HasCorpus,\n{\n    /// Creates a new `TuneableScheduler`\n    #[must_use]\n    pub fn new(state: &mut S) -> Self {\n        if !state.has_metadata::<TuneableSchedulerMetadata>() {\n            state.add_metadata(TuneableSchedulerMetadata::default());\n        }\n        Self {\n            phantom: PhantomData,\n        }\n    }\n\n    fn metadata_mut(state: &mut S) -> &mut TuneableSchedulerMetadata {\n        state\n            .metadata_map_mut()\n            .get_mut::<TuneableSchedulerMetadata>()\n            .unwrap()\n    }\n\n    fn metadata(state: &S) -> &TuneableSchedulerMetadata {\n        state\n            .metadata_map()\n            .get::<TuneableSchedulerMetadata>()\n            .unwrap()\n    }\n\n    /// Sets the next corpus id to be used\n    pub fn set_next(state: &mut S, next: CorpusId) {\n        Self::metadata_mut(state).next = Some(next);\n    }\n\n    /// Gets the next set corpus id\n    pub fn get_next(state: &S) -> Option<CorpusId> {\n        Self::metadata(state).next\n    }\n\n    /// Resets this to a queue scheduler\n    pub fn reset(state: &mut S) {\n        let metadata = Self::metadata_mut(state);\n        metadata.next = None;\n    }\n\n    /// Gets the current corpus entry id\n    pub fn get_current(state: &S) -> CorpusId {\n        state\n            .corpus()\n            .current()\n            .unwrap_or_else(|| state.corpus().first().expect(\"Empty corpus\"))\n    }\n}\n\nimpl<S> UsesState for TuneableScheduler<S>\nwhere\n    S: State,\n{\n    type State = S;\n}\n\nimpl<S> RemovableScheduler for TuneableScheduler<S> where\n    S: HasCorpus + HasMetadata + HasTestcase + State\n{\n}\n\nimpl<S> Scheduler for TuneableScheduler<S>\nwhere\n    S: HasCorpus + HasMetadata + HasTestcase + State,\n{\n    fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> {\n        // Set parent id\n        let current_idx = *state.corpus().current();\n        state\n            .corpus()\n            .get(idx)?\n            .borrow_mut()\n            .set_parent_id_optional(current_idx);\n\n        Ok(())\n    }\n\n    /// Gets the next entry in the queue\n    fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {\n        if state.corpus().count() == 0 {\n            return Err(Error::empty(\n                \"No entries in corpus. This often implies the target is not properly instrumented.\"\n                    .to_owned(),\n            ));\n        }\n        let id = if let Some(next) = Self::get_next(state) {\n            // next was set\n            next\n        } else if let Some(next) = state.corpus().next(Self::get_current(state)) {\n            next\n        } else {\n            state.corpus().first().unwrap()\n        };\n        self.set_current_scheduled(state, Some(id))?;\n        Ok(id)\n    }\n}\n\n// Path: libafl/src/schedulers/accounting.rs\n//! Coverage accounting corpus scheduler, more details at <https://www.ndss-symposium.org/wp-content/uploads/2020/02/24422-paper.pdf>\n\nuse alloc::vec::Vec;\nuse core::fmt::Debug;\n\nuse hashbrown::HashMap;\nuse libafl_bolts::{rands::Rand, AsMutSlice, AsSlice, HasLen, HasRefCnt};\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    corpus::{Corpus, CorpusId},\n    feedbacks::MapIndexesMetadata,\n    inputs::UsesInput,\n    observers::ObserversTuple,\n    schedulers::{\n        minimizer::{IsFavoredMetadata, MinimizerScheduler, DEFAULT_SKIP_NON_FAVORED_PROB},\n        LenTimeMulTestcaseScore, Scheduler,\n    },\n    state::{HasCorpus, HasMetadata, HasRand, UsesState},\n    Error,\n};\n\n/// A testcase metadata holding a list of indexes of a map\n#[derive(Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct AccountingIndexesMetadata {\n    /// The list of indexes.\n    pub list: Vec<usize>,\n    /// A refcount used to know when remove this meta\n    pub tcref: isize,\n}\n\nlibafl_bolts::impl_serdeany!(AccountingIndexesMetadata);\n\nimpl AsSlice for AccountingIndexesMetadata {\n    type Entry = usize;\n    /// Convert to a slice\n    fn as_slice(&self) -> &[usize] {\n        self.list.as_slice()\n    }\n}\nimpl AsMutSlice for AccountingIndexesMetadata {\n    type Entry = usize;\n\n    /// Convert to a slice\n    fn as_mut_slice(&mut self) -> &mut [usize] {\n        self.list.as_mut_slice()\n    }\n}\n\nimpl HasRefCnt for AccountingIndexesMetadata {\n    fn refcnt(&self) -> isize {\n        self.tcref\n    }\n\n    fn refcnt_mut(&mut self) -> &mut isize {\n        &mut self.tcref\n    }\n}\n\nimpl AccountingIndexesMetadata {\n    /// Creates a new [`struct@AccountingIndexesMetadata`].\n    #[must_use]\n    pub fn new(list: Vec<usize>) -> Self {\n        Self { list, tcref: 0 }\n    }\n\n    /// Creates a new [`struct@AccountingIndexesMetadata`] specifying the refcount.\n    #[must_use]\n    pub fn with_tcref(list: Vec<usize>, tcref: isize) -> Self {\n        Self { list, tcref }\n    }\n}\n\n/// A state metadata holding a map of favoreds testcases for each map entry\n#[derive(Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct TopAccountingMetadata {\n    /// map index -> corpus index\n    pub map: HashMap<usize, CorpusId>,\n    /// If changed sicne the previous add to the corpus\n    pub changed: bool,\n    /// The max accounting seen so far\n    pub max_accounting: Vec<u32>,\n}\n\nlibafl_bolts::impl_serdeany!(TopAccountingMetadata);\n\nimpl TopAccountingMetadata {\n    /// Creates a new [`struct@TopAccountingMetadata`]\n    #[must_use]\n    pub fn new(acc_len: usize) -> Self {\n        Self {\n            map: HashMap::default(),\n            changed: false,\n            max_accounting: vec![0; acc_len],\n        }\n    }\n}\n\n/// A minimizer scheduler using coverage accounting\n#[derive(Debug)]\npub struct CoverageAccountingScheduler<'a, CS>\nwhere\n    CS: UsesState,\n    CS::State: Debug,\n{\n    accounting_map: &'a [u32],\n    skip_non_favored_prob: u64,\n    inner: MinimizerScheduler<\n        CS,\n        LenTimeMulTestcaseScore<<CS as UsesState>::State>,\n        MapIndexesMetadata,\n    >,\n}\n\nimpl<'a, CS> UsesState for CoverageAccountingScheduler<'a, CS>\nwhere\n    CS: UsesState,\n    CS::State: Debug,\n{\n    type State = CS::State;\n}\n\nimpl<'a, CS> Scheduler for CoverageAccountingScheduler<'a, CS>\nwhere\n    CS: Scheduler,\n    CS::State: HasCorpus + HasMetadata + HasRand + Debug,\n    <CS::State as UsesInput>::Input: HasLen,\n{\n    fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> {\n        self.update_accounting_score(state, idx)?;\n        self.inner.on_add(state, idx)\n    }\n\n    fn on_evaluation<OT>(\n        &mut self,\n        state: &mut Self::State,\n        input: &<Self::State as UsesInput>::Input,\n        observers: &OT,\n    ) -> Result<(), Error>\n    where\n        OT: ObserversTuple<Self::State>,\n    {\n        self.inner.on_evaluation(state, input, observers)\n    }\n\n    fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {\n        if state\n            .metadata_map()\n            .get::<TopAccountingMetadata>()\n            .map_or(false, |x| x.changed)\n        {\n            self.accounting_cull(state)?;\n        } else {\n            self.inner.cull(state)?;\n        }\n        let mut idx = self.inner.base_mut().next(state)?;\n        while {\n            let has = !state\n                .corpus()\n                .get(idx)?\n                .borrow()\n                .has_metadata::<IsFavoredMetadata>();\n            has\n        } && state.rand_mut().below(100) < self.skip_non_favored_prob\n        {\n            idx = self.inner.base_mut().next(state)?;\n        }\n\n        // Don't add corpus.curret(). The inner scheduler will take care of it\n\n        Ok(idx)\n    }\n\n    /// Set current fuzzed corpus id and `scheduled_count`\n    fn set_current_scheduled(\n        &mut self,\n        _state: &mut Self::State,\n        _next_idx: Option<CorpusId>,\n    ) -> Result<(), Error> {\n        // We do nothing here, the inner scheduler will take care of it\n        Ok(())\n    }\n}\n\nimpl<'a, CS> CoverageAccountingScheduler<'a, CS>\nwhere\n    CS: Scheduler,\n    CS::State: HasCorpus + HasMetadata + HasRand + Debug,\n    <CS::State as UsesInput>::Input: HasLen,\n{\n    /// Update the `Corpus` score\n    #[allow(clippy::unused_self)]\n    #[allow(clippy::cast_possible_wrap)]\n    pub fn update_accounting_score(\n        &self,\n        state: &mut CS::State,\n        idx: CorpusId,\n    ) -> Result<(), Error> {\n        let mut indexes = vec![];\n        let mut new_favoreds = vec![];\n        {\n            for idx in 0..self.accounting_map.len() {\n                if self.accounting_map[idx] == 0 {\n                    continue;\n                }\n                indexes.push(idx);\n\n                let mut equal_score = false;\n                {\n                    let top_acc = state.metadata_map().get::<TopAccountingMetadata>().unwrap();\n\n                    if let Some(old_idx) = top_acc.map.get(&idx) {\n                        if top_acc.max_accounting[idx] > self.accounting_map[idx] {\n                            continue;\n                        }\n\n                        if top_acc.max_accounting[idx] == self.accounting_map[idx] {\n                            equal_score = true;\n                        }\n\n                        let mut old = state.corpus().get(*old_idx)?.borrow_mut();\n                        let must_remove = {\n                            let old_meta = old.metadata_map_mut().get_mut::<AccountingIndexesMetadata>().ok_or_else(|| {\n                                Error::key_not_found(format!(\n                                    \"AccountingIndexesMetadata, needed by CoverageAccountingScheduler, not found in testcase #{old_idx}\"\n                                ))\n                            })?;\n                            *old_meta.refcnt_mut() -= 1;\n                            old_meta.refcnt() <= 0\n                        };\n\n                        if must_remove {\n                            drop(old.metadata_map_mut().remove::<AccountingIndexesMetadata>());\n                        }\n                    }\n                }\n\n                let top_acc = state\n                    .metadata_map_mut()\n                    .get_mut::<TopAccountingMetadata>()\n                    .unwrap();\n\n                // if its accounting is equal to others', it's not favored\n                if equal_score {\n                    top_acc.map.remove(&idx);\n                } else if top_acc.max_accounting[idx] < self.accounting_map[idx] {\n                    new_favoreds.push(idx);\n\n                    top_acc.max_accounting[idx] = self.accounting_map[idx];\n                }\n            }\n        }\n\n        if new_favoreds.is_empty() {\n            return Ok(());\n        }\n\n        state\n            .corpus()\n            .get(idx)?\n            .borrow_mut()\n            .metadata_map_mut()\n            .insert(AccountingIndexesMetadata::with_tcref(\n                indexes,\n                new_favoreds.len() as isize,\n            ));\n\n        let top_acc = state\n            .metadata_map_mut()\n            .get_mut::<TopAccountingMetadata>()\n            .unwrap();\n        top_acc.changed = true;\n\n        for elem in new_favoreds {\n            top_acc.map.insert(elem, idx);\n        }\n\n        Ok(())\n    }\n\n    /// Cull the `Corpus`\n    #[allow(clippy::unused_self)]\n    pub fn accounting_cull(&self, state: &CS::State) -> Result<(), Error> {\n        let Some(top_rated) = state.metadata_map().get::<TopAccountingMetadata>() else {\n            return Ok(());\n        };\n\n        for (_key, idx) in &top_rated.map {\n            let mut entry = state.corpus().get(*idx)?.borrow_mut();\n            if entry.scheduled_count() > 0 {\n                continue;\n            }\n\n            entry.add_metadata(IsFavoredMetadata {});\n        }\n\n        Ok(())\n    }\n\n    /// Creates a new [`CoverageAccountingScheduler`] that wraps a `base` [`Scheduler`]\n    /// and has a default probability to skip non-faved Testcases of [`DEFAULT_SKIP_NON_FAVORED_PROB`].\n    pub fn new(state: &mut CS::State, base: CS, accounting_map: &'a [u32]) -> Self {\n        match state.metadata_map().get::<TopAccountingMetadata>() {\n            Some(meta) => {\n                if meta.max_accounting.len() != accounting_map.len() {\n                    state.add_metadata(TopAccountingMetadata::new(accounting_map.len()));\n                }\n            }\n            None => {\n                state.add_metadata(TopAccountingMetadata::new(accounting_map.len()));\n            }\n        }\n        Self {\n            accounting_map,\n            inner: MinimizerScheduler::new(base),\n            skip_non_favored_prob: DEFAULT_SKIP_NON_FAVORED_PROB,\n        }\n    }\n\n    /// Creates a new [`CoverageAccountingScheduler`] that wraps a `base` [`Scheduler`]\n    /// and has a non-default probability to skip non-faved Testcases using (`skip_non_favored_prob`).\n    pub fn with_skip_prob(\n        state: &mut CS::State,\n        base: CS,\n        skip_non_favored_prob: u64,\n        accounting_map: &'a [u32],\n    ) -> Self {\n        match state.metadata_map().get::<TopAccountingMetadata>() {\n            Some(meta) => {\n                if meta.max_accounting.len() != accounting_map.len() {\n                    state.add_metadata(TopAccountingMetadata::new(accounting_map.len()));\n                }\n            }\n            None => {\n                state.add_metadata(TopAccountingMetadata::new(accounting_map.len()));\n            }\n        }\n        Self {\n            accounting_map,\n            inner: MinimizerScheduler::with_skip_prob(base, skip_non_favored_prob),\n            skip_non_favored_prob,\n        }\n    }\n}\n\n// Path: libafl/src/schedulers/probabilistic_sampling.rs\n//! Probabilistic sampling scheduler is a corpus scheduler that feeds the fuzzer\n//! with sampled item from the corpus.\n\nuse alloc::string::String;\nuse core::marker::PhantomData;\n\nuse hashbrown::HashMap;\nuse libafl_bolts::rands::Rand;\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    corpus::{Corpus, CorpusId, HasTestcase, Testcase},\n    inputs::UsesInput,\n    schedulers::{RemovableScheduler, Scheduler, TestcaseScore},\n    state::{HasCorpus, HasMetadata, HasRand, State, UsesState},\n    Error,\n};\n\n/// Conduct reservoir sampling (probabilistic sampling) over all corpus elements.\n#[derive(Debug, Clone)]\npub struct ProbabilitySamplingScheduler<F, S>\nwhere\n    S: UsesInput,\n{\n    phantom: PhantomData<(F, S)>,\n}\n\n/// A state metadata holding a map of probability of corpus elements.\n#[derive(Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct ProbabilityMetadata {\n    /// corpus index -> probability\n    pub map: HashMap<CorpusId, f64>,\n    /// total probability of all items in the map\n    pub total_probability: f64,\n}\n\nlibafl_bolts::impl_serdeany!(ProbabilityMetadata);\n\nimpl ProbabilityMetadata {\n    /// Creates a new [`struct@ProbabilityMetadata`]\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            map: HashMap::default(),\n            total_probability: 0.0,\n        }\n    }\n}\n\nimpl Default for ProbabilityMetadata {\n    fn default() -> Self {\n        Self::new()\n    }\n}\n\nimpl<F, S> ProbabilitySamplingScheduler<F, S>\nwhere\n    F: TestcaseScore<S>,\n    S: HasCorpus + HasMetadata + HasRand,\n{\n    /// Creates a new [`struct@ProbabilitySamplingScheduler`]\n    #[must_use]\n    pub fn new() -> Self {\n        Self {\n            phantom: PhantomData,\n        }\n    }\n\n    /// Calculate the score and store in `ProbabilityMetadata`\n    #[allow(clippy::cast_precision_loss)]\n    #[allow(clippy::unused_self)]\n    \npub fn store_probability(&self, state: &mut S, idx: CorpusId) -> Result<(), Error> {\n        let prob = F::compute(state, &mut *state.corpus().get(idx)?.borrow_mut())?;\n        debug_assert!(\n            prob >= 0.0 && prob.is_finite(),\n            \"scheduler probability is {prob}; to work correctly it must be >= 0.0 and finite\"\n        );\n        let meta = state\n            .metadata_map_mut()\n            .get_mut::<ProbabilityMetadata>()\n            .unwrap();\n        meta.map.insert(idx, prob);\n        meta.total_probability += prob;\n        Ok(())\n    }\n}\n\nimpl<F, S> RemovableScheduler for ProbabilitySamplingScheduler<F, S>\nwhere\n    F: TestcaseScore<S>,\n    S: HasCorpus + HasMetadata + HasRand + HasTestcase + State,\n{\n    fn on_remove(\n        &mut self,\n        state: &mut Self::State,\n        idx: CorpusId,\n        _testcase: &Option<Testcase<<Self::State as UsesInput>::Input>>,\n    ) -> Result<(), Error> {\n        let meta = state\n            .metadata_map_mut()\n            .get_mut::<ProbabilityMetadata>()\n            .unwrap();\n        if let Some(prob) = meta.map.remove(&idx) {\n            meta.total_probability -= prob;\n        }\n        Ok(())\n    }\n\n    fn on_replace(\n        &mut self,\n        state: &mut Self::State,\n        idx: CorpusId,\n        _prev: &Testcase<<Self::State as UsesInput>::Input>,\n    ) -> Result<(), Error> {\n        let meta = state\n            .metadata_map_mut()\n            .get_mut::<ProbabilityMetadata>()\n            .unwrap();\n        if let Some(prob) = meta.map.remove(&idx) {\n            meta.total_probability -= prob;\n        }\n\n        self.store_probability(state, idx)\n    }\n}\n\nimpl<F, S> UsesState for ProbabilitySamplingScheduler<F, S>\nwhere\n    S: State + HasTestcase,\n{\n    type State = S;\n}\n\nimpl<F, S> Scheduler for ProbabilitySamplingScheduler<F, S>\nwhere\n    F: TestcaseScore<S>,\n    S: HasCorpus + HasMetadata + HasRand + HasTestcase + State,\n{\n    fn on_add(&mut self, state: &mut Self::State, idx: CorpusId) -> Result<(), Error> {\n        let current_idx = *state.corpus().current();\n        state\n            .corpus()\n            .get(idx)?\n            .borrow_mut()\n            .set_parent_id_optional(current_idx);\n\n        if state.metadata_map().get::<ProbabilityMetadata>().is_none() {\n            state.add_metadata(ProbabilityMetadata::new());\n        }\n        self.store_probability(state, idx)\n    }\n\n    /// Gets the next entry\n    #[allow(clippy::cast_precision_loss)]\n    fn next(&mut self, state: &mut Self::State) -> Result<CorpusId, Error> {\n        if state.corpus().count() == 0 {\n            Err(Error::empty(String::from(\n                \"No entries in corpus. This often implies the target is not properly instrumented.\",\n            )))\n        } else {\n            let rand_prob: f64 = (state.rand_mut().below(100) as f64) / 100.0;\n            let meta = state.metadata_map().get::<ProbabilityMetadata>().unwrap();\n            let threshold = meta.total_probability * rand_prob;\n            let mut k: f64 = 0.0;\n            let mut ret = *meta.map.keys().last().unwrap();\n            for (idx, prob) in &meta.map {\n                k += prob;\n                if k >= threshold {\n                    ret = *idx;\n                    break;\n                }\n            }\n            self.set_current_scheduled(state, Some(ret))?;\n            Ok(ret)\n        }\n    }\n}\n\nimpl<F, S> Default for ProbabilitySamplingScheduler<F, S>\nwhere\n    F: TestcaseScore<S>,\n    S: HasCorpus + HasMetadata + HasRand,\n{\n    fn default() -> Self {\n        Self::new()\n    }\n}\n\n#[cfg(test)]\n#[cfg(feature = \"std\")]\nmod tests {\n    use core::{borrow::BorrowMut, marker::PhantomData};\n\n    use libafl_bolts::rands::StdRand;\n\n    use crate::{\n        corpus::{Corpus, InMemoryCorpus, Testcase},\n        feedbacks::ConstFeedback,\n        inputs::{bytes::BytesInput, Input, UsesInput},\n        schedulers::{ProbabilitySamplingScheduler, Scheduler, TestcaseScore},\n        state::{HasCorpus, HasMetadata, StdState},\n        Error,\n    };\n\n    const FACTOR: f64 = 1337.0;\n\n    #[derive(Debug, Clone)]\n    pub struct UniformDistribution<I>\n    where\n        I: Input,\n    {\n        phantom: PhantomData<I>,\n    }\n\n    impl<S> TestcaseScore<S> for UniformDistribution<S::Input>\n    where\n        S: HasMetadata + HasCorpus,\n    {\n        fn compute(_state: &S, _: &mut Testcase<S::Input>) -> Result<f64, Error> {\n            Ok(FACTOR)\n        }\n    }\n\n    pub type UniformProbabilitySamplingScheduler<S> =\n        ProbabilitySamplingScheduler<UniformDistribution<<S as UsesInput>::Input>, S>;\n\n    #[test]\n    fn test_prob_sampling() {\n        // # Safety\n        // No concurrency per testcase\n        #[cfg(any(not(feature = \"serdeany_autoreg\"), miri))]\n        unsafe {\n            super::ProbabilityMetadata::register();\n        }\n\n        // the first 3 probabilities will be .69, .86, .44\n        let rand = StdRand::with_seed(12);\n\n        let mut scheduler = UniformProbabilitySamplingScheduler::new();\n\n        let mut feedback = ConstFeedback::new(false);\n        let mut objective = ConstFeedback::new(false);\n\n        let mut corpus = InMemoryCorpus::new();\n        let t1 = Testcase::with_filename(BytesInput::new(vec![0_u8; 4]), \"1\".into());\n        let t2 = Testcase::with_filename(BytesInput::new(vec![1_u8; 4]), \"2\".into());\n\n        let idx1 = corpus.add(t1).unwrap();\n        let idx2 = corpus.add(t2).unwrap();\n\n        let mut state = StdState::new(\n            rand,\n            corpus,\n            InMemoryCorpus::new(),\n            &mut feedback,\n            &mut objective,\n        )\n        .unwrap();\n        scheduler.on_add(state.borrow_mut(), idx1).unwrap();\n        scheduler.on_add(state.borrow_mut(), idx2).unwrap();\n        let next_idx1 = scheduler.next(&mut state).unwrap();\n        let next_idx2 = scheduler.next(&mut state).unwrap();\n        let next_idx3 = scheduler.next(&mut state).unwrap();\n        assert_eq!(next_idx1, next_idx2);\n        assert_ne!(next_idx1, next_idx3);\n    }\n}\n\n// Path: libafl/src/schedulers/mod.rs\n//! Schedule the access to the Corpus.\n\nuse alloc::{borrow::ToOwned, string::ToString};\nuse core::marker::PhantomData;\n\npub mod testcase_score;\npub use testcase_score::{LenTimeMulTestcaseScore, TestcaseScore};\n\npub mod queue;\npub use queue::QueueScheduler;\n\npub mod minimizer;\npub use minimizer::{\n    IndexesLenTimeMinimizerScheduler, LenTimeMinimizerScheduler, MinimizerScheduler,\n};\n\npub mod powersched;\npub use powersched::{PowerQueueScheduler, SchedulerMetadata};\n\npub mod probabilistic_sampling;\npub use probabilistic_sampling::ProbabilitySamplingScheduler;\n\npub mod accounting;\npub use accounting::CoverageAccountingScheduler;\n\npub mod weighted;\npub use weighted::{StdWeightedScheduler, WeightedScheduler};\n\npub mod tuneable;\nuse libafl_bolts::rands::Rand;\npub use tuneable::*;\n\nuse crate::{\n    corpus::{Corpus, CorpusId, HasTestcase, SchedulerTestcaseMetadata, Testcase},\n    inputs::UsesInput,\n    observers::{MapObserver, ObserversTuple},\n    random_corpus_id,\n    state::{HasCorpus, HasMetadata, HasRand, State, UsesState},\n    Error,\n};\n\n/// The scheduler also implements `on_remove` and `on_replace` if it implements this stage.\npub trait RemovableScheduler: Scheduler\nwhere\n    Self::State: HasCorpus,\n{\n    /// Removed the given entry from the corpus at the given index\n    fn on_remove(\n        &mut self,\n        _state: &mut Self::State,\n        _idx: CorpusId,\n        _testcase: &Option<Testcase<<Self::State as UsesInput>::Input>>,\n    ) -> Result<(), Error> {\n        Ok(())\n    }\n\n    /// Replaced the given testcase at the given idx\n    fn on_replace(\n        &mut self,\n        _state: &mut Self::State,\n        _idx: CorpusId,\n...\n", "needle_token_start": 13677, "needle_token_end": 13848, "code_context_ntokens": 16176, "output": ["```rust\n/// Calculate the score and store in `ProbabilityMetadata`\n#[allow(clippy::cast_precision_loss)]\n#[allow(clippy::unused_self)]\npub fn store_probability(&self, state: &mut S, idx: CorpusId) -> Result<(), Error> {\n    let prob = F::compute(state, &mut *state.corpus().get(idx)?.borrow_mut())?;\n    debug_assert!(\n        prob >= 0.0 && prob.is_finite(),\n        \"scheduler probability is {prob}; to work correctly it must be >= 0.0 and finite\"\n    );\n    let meta = state\n        .metadata_map_mut()\n        .get_mut::<ProbabilityMetadata>()\n        .unwrap();\n    meta.map.insert(idx, prob);\n    meta.total_probability += prob;\n    Ok(())\n}\n```"]}
{"repo": "AFLplusplus/LibAFL", "name": "saturating_next_power_of_two", "language": "rust", "path": "libafl/src/feedbacks/map.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function calculates the smallest power of two that is greater than or equal to a given integer, but it will not exceed the maximum value representable by the type of the integer. If the input is negative or zero, it returns one.\n2. **Input**: A single integer of a generic type that supports certain operations like bit shifting and finding the maximum value.\n3. **Output**: An integer of the same type as the input, representing the next power of two or one in special cases.\n4. **Procedure**: The function first checks if the input is less than or equal to one, returning one if true. Otherwise, it calculates the next power of two by shifting the maximum value of the integer type right by the number of leading zeros in the input minus one, then adds one to this value. It uses a saturating add operation to ensure it does not exceed the maximum value of the integer type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: libafl/src/observers/map.rs\n//! The `MapObserver` provides access a map, usually injected into the target\n\nuse alloc::{\n    string::{String, ToString},\n    vec::Vec,\n};\nuse core::{\n    fmt::Debug,\n    hash::{BuildHasher, Hasher},\n    iter::Flatten,\n    marker::PhantomData,\n    mem::size_of,\n    slice::{self, Iter, IterMut},\n};\n\nuse ahash::RandomState;\nuse libafl_bolts::{\n    ownedref::{OwnedMutPtr, OwnedMutSlice},\n    AsIter, AsIterMut, AsMutSlice, AsSlice, HasLen, Named, Truncate,\n};\nuse meminterval::IntervalTree;\nuse num_traits::Bounded;\nuse serde::{Deserialize, Serialize};\n\nuse crate::{\n    executors::ExitKind,\n    inputs::UsesInput,\n    observers::{DifferentialObserver, Observer, ObserversTuple},\n    Error,\n};\n\n/// Hitcounts class lookup\nstatic COUNT_CLASS_LOOKUP: [u8; 256] = [\n    0, 1, 2, 4, 8, 8, 8, 8, 16, 16, 16, 16, 16, 16, 16, 16, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,\n    32, 32, 32, 32, 32, 32, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,\n    64, 64, 64, 64, 64, 64, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n    128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,\n];\n\n/// Hitcounts class lookup for 16-byte values\nstatic mut COUNT_CLASS_LOOKUP_16: Vec<u16> = vec![];\n\n/// Initialize the 16-byte hitcounts map\n///\n/// # Safety\n///\n/// Calling this from multiple threads may be racey and hence leak 65k mem\nfn init_count_class_16() {\n    unsafe {\n        if !COUNT_CLASS_LOOKUP_16.is_empty() {\n            return;\n        }\n\n        COUNT_CLASS_LOOKUP_16 = vec![0; 65536];\n        for i in 0..256 {\n            for j in 0..256 {\n                COUNT_CLASS_LOOKUP_16[(i << 8) + j] =\n                    (u16::from(COUNT_CLASS_LOOKUP[i]) << 8) | u16::from(COUNT_CLASS_LOOKUP[j]);\n            }\n        }\n    }\n}\n\n/// Compute the hash of a slice\nfn hash_slice<T>(slice: &[T]) -> u64 {\n    let mut hasher = RandomState::with_seeds(0, 0, 0, 0).build_hasher();\n    let ptr = slice.as_ptr() as *const u8;\n    let map_size = slice.len() / size_of::<T>();\n    unsafe {\n        hasher.write(slice::from_raw_parts(ptr, map_size));\n    }\n    hasher.finish()\n}\n\n/// A [`MapObserver`] observes the static map, as oftentimes used for AFL-like coverage information\n///\n/// TODO: enforce `iter() -> AssociatedTypeIter` when generic associated types stabilize\npub trait MapObserver: HasLen + Named + Serialize + serde::de::DeserializeOwned\n// where\n//     for<'it> &'it Self: IntoIterator<Item = &'it Self::Entry>\n{\n    /// Type of each entry in this map\n    type Entry: Bounded + PartialEq + Default + Copy + Debug + 'static;\n\n    /// Get the value at `idx`\n    fn get(&self, idx: usize) -> &Self::Entry;\n\n    /// Get the value at `idx` (mutable)\n    fn get_mut(&mut self, idx: usize) -> &mut Self::Entry;\n\n    /// Get the number of usable entries in the map (all by default)\n    fn usable_count(&self) -> usize;\n\n    /// Count the set bytes in the map\n    fn count_bytes(&self) -> u64;\n\n    /// Compute the hash of the map\n    fn hash(&self) -> u64;\n\n    /// Get the initial value for `reset()`\n    fn initial(&self) -> Self::Entry;\n\n    /// Reset the map\n    fn reset_map(&mut self) -> Result<(), Error>;\n\n    /// Get these observer's contents as [`Vec`]\n    fn to_vec(&self) -> Vec<Self::Entry>;\n\n    /// Get the number of set entries with the specified indexes\n    fn how_many_set(&self, indexes: &[usize]) -> usize;\n}\n\n/// A Simple iterator calling `MapObserver::get`\n#[derive(Debug)]\npub struct MapObserverSimpleIterator<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    index: usize,\n    observer: *const O,\n    phantom: PhantomData<&'a u8>,\n}\n\nimpl<'a, O> Iterator for MapObserverSimpleIterator<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    type Item = &'a O::Entry;\n    fn next(&mut self) -> Option<Self::Item> {\n        unsafe {\n            if self.index >= self.observer.as_ref().unwrap().usable_count() {\n                None\n            } else {\n                let i = self.index;\n                self.index += 1;\n                Some(self.observer.as_ref().unwrap().get(i))\n            }\n        }\n    }\n}\n\n/// A Simple iterator calling `MapObserver::get_mut`\n#[derive(Debug)]\npub struct MapObserverSimpleIteratorMut<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    index: usize,\n    observer: *mut O,\n    phantom: PhantomData<&'a u8>,\n}\n\nimpl<'a, O> Iterator for MapObserverSimpleIteratorMut<'a, O>\nwhere\n    O: 'a + MapObserver,\n{\n    type Item = &'a O::Entry;\n    fn next(&mut self) -> Option<Self::Item> {\n        unsafe {\n            if self.index >= self.observer.as_ref().unwrap().usable_count() {\n                None\n            } else {\n                let i = self.index;\n                self.index += 1;\n                Some(self.observer.as_mut().unwrap().get_mut(i))\n            }\n        }\n    }\n}\n\n/// The Map Observer retrieves the state of a map,\n/// that will get updated by the target.\n/// A well-known example is the AFL-Style coverage map.\n#[derive(Clone, Serialize, Deserialize, Debug)]\n#[serde(bound = \"T: serde::de::DeserializeOwned\")]\n#[allow(clippy::unsafe_derive_deserialize)]\npub struct StdMapObserver<'a, T, const DIFFERENTIAL: bool>\nwhere\n    T: Default + Copy + 'static + Serialize,\n{\n    map: OwnedMutSlice<'a, T>,\n    initial: T,\n    name: String,\n}\n\nimpl<'a, S, T> Observer<S> for StdMapObserver<'a, T, false>\nwhere\n    S: UsesInput,\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    #[inline]\n    fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> {\n        self.reset_map()\n    }\n}\n\nimpl<'a, S, T> Observer<S> for StdMapObserver<'a, T, true>\nwhere\n    S: UsesInput,\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> Named for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn name(&self) -> &str {\n        self.name.as_str()\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> HasLen for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn len(&self) -> usize {\n        self.map.as_slice().len()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> AsIter<'it> for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = Iter<'it, T>;\n\n    fn as_iter(&'it self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> AsIterMut<'it> for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = IterMut<'it, T>;\n\n    fn as_iter_mut(&'it mut self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator for &'it StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <Iter<'it, T> as Iterator>::Item;\n    type IntoIter = Iter<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const DIFFERENTIAL: bool> IntoIterator\n    for &'it mut StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <IterMut<'it, T> as Iterator>::Item;\n    type IntoIter = IterMut<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    /// Returns an iterator over the map.\n    pub fn iter(&self) -> Iter<'_, T> {\n        <&Self as IntoIterator>::into_iter(self)\n    }\n\n    /// Returns a mutable iterator over the map.\n    pub fn iter_mut(&mut self) -> IterMut<'_, T> {\n        <&mut Self as IntoIterator>::into_iter(self)\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> MapObserver for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Entry = T;\n\n    #[inline]\n    fn get(&self, pos: usize) -> &T {\n        &self.as_slice()[pos]\n    }\n\n    #[inline]\n    fn get_mut(&mut self, idx: usize) -> &mut T {\n        &mut self.as_mut_slice()[idx]\n    }\n\n    /// Count the set bytes in the map\n    fn count_bytes(&self) -> u64 {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for x in &map[0..cnt] {\n            if *x != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n\n    #[inline]\n    fn usable_count(&self) -> usize {\n        self.as_slice().len()\n    }\n\n    fn hash(&self) -> u64 {\n        hash_slice(self.as_slice())\n    }\n\n    #[inline]\n    fn initial(&self) -> T {\n        self.initial\n    }\n\n    fn to_vec(&self) -> Vec<T> {\n        self.as_slice().to_vec()\n    }\n\n    /// Reset the map\n    #[inline]\n    fn reset_map(&mut self) -> Result<(), Error> {\n        // Normal memset, see https://rust.godbolt.org/z/Trs5hv\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_mut_slice();\n        for x in &mut map[0..cnt] {\n            *x = initial;\n        }\n        Ok(())\n    }\n\n    fn how_many_set(&self, indexes: &[usize]) -> usize {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for i in indexes {\n            if *i < cnt && map[*i] != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> Truncate for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    fn truncate(&mut self, new_len: usize) {\n        self.map.truncate(new_len);\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> AsSlice for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n{\n    type Entry = T;\n    #[must_use]\n    #[inline]\n    fn as_slice(&self) -> &[T] {\n        self.map.as_slice()\n    }\n}\nimpl<'a, T, const DIFFERENTIAL: bool> AsMutSlice for StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n{\n    type Entry = T;\n    #[must_use]\n    #[inline]\n    fn as_mut_slice(&mut self) -> &mut [T] {\n        self.map.as_mut_slice()\n    }\n}\n\nimpl<'a, T, const DIFFERENTIAL: bool> StdMapObserver<'a, T, DIFFERENTIAL>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    /// Creates a new [`MapObserver`]\n    ///\n    /// # Safety\n    /// Will get a pointer to the map and dereference it at any point in time.\n    /// The map must not move in memory!\n    #[must_use]\n    unsafe fn maybe_differential<S>(name: S, map: &'a mut [T]) -> Self\n    where\n        S: Into<String>,\n    {\n        let len = map.len();\n        let ptr = map.as_mut_ptr();\n        Self::maybe_differential_from_mut_ptr(name, ptr, len)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`]\n    #[must_use]\n    fn maybe_differential_from_mut_slice<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        StdMapObserver {\n            name: name.into(),\n            map,\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] with an owned map\n    #[must_use]\n    fn maybe_differential_owned<S>(name: S, map: Vec<T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self {\n            map: OwnedMutSlice::from(map),\n            name: name.into(),\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map.\n    ///\n    /// # Safety\n    /// Will dereference the owned slice with up to len elements.\n    #[must_use]\n    fn maybe_differential_from_ownedref<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self {\n            map,\n            name: name.into(),\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer\n    ///\n    /// # Safety\n    /// Will dereference the `map_ptr` with up to len elements.\n    unsafe fn maybe_differential_from_mut_ptr<S>(name: S, map_ptr: *mut T, len: usize) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_slice(\n            name,\n            OwnedMutSlice::from_raw_parts_mut(map_ptr, len),\n        )\n    }\n\n    /// Gets the initial value for this map, mutably\n    pub fn initial_mut(&mut self) -> &mut T {\n        &mut self.initial\n    }\n\n    /// Gets the backing for this map\n    pub fn map(&self) -> &OwnedMutSlice<'a, T> {\n        &self.map\n    }\n\n    /// Gets the backing for this map mutably\n    pub fn map_mut(&mut self) -> &mut OwnedMutSlice<'a, T> {\n        &mut self.map\n    }\n}\n\nimpl<'a, T> StdMapObserver<'a, T, false>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    /// Creates a new [`MapObserver`]\n    ///\n    /// # Safety\n    /// The observer will keep a pointer to the map.\n    /// Hence, the map may never move in memory.\n    #[must_use]\n    pub unsafe fn new<S>(name: S, map: &'a mut [T]) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`]\n    pub fn from_mut_slice<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_slice(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] with an owned map\n    #[must_use]\n    pub fn owned<S>(name: S, map: Vec<T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_owned(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map.\n    ///\n    /// # Note\n    /// Will dereference the owned slice with up to len elements.\n    #[must_use]\n    pub fn from_ownedref<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_ownedref(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer\n    ///\n    /// # Safety\n    /// Will dereference the `map_ptr` with up to len elements.\n    pub unsafe fn from_mut_ptr<S>(name: S, map_ptr: *mut T, len: usize) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_ptr(name, map_ptr, len)\n    }\n}\n\nimpl<'a, T> StdMapObserver<'a, T, true>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    /// Creates a new [`MapObserver`] in differential mode\n    ///\n    /// # Safety\n    /// Will get a pointer to the map and dereference it at any point in time.\n    /// The map must not move in memory!\n    #[must_use]\n    pub unsafe fn differential<S>(name: S, map: &'a mut [T]) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] with an owned map in differential mode\n    #[must_use]\n    pub fn differential_owned<S>(name: S, map: Vec<T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_owned(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`] map in differential mode.\n    ///\n    /// # Note\n    /// Will dereference the owned slice with up to len elements.\n    #[must_use]\n    pub fn differential_from_ownedref<S>(name: S, map: OwnedMutSlice<'a, T>) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_ownedref(name, map)\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer in differential mode\n    ///\n    /// # Safety\n    /// Will dereference the `map_ptr` with up to len elements.\n    pub unsafe fn differential_from_mut_ptr<S>(name: S, map_ptr: *mut T, len: usize) -> Self\n    where\n        S: Into<String>,\n    {\n        Self::maybe_differential_from_mut_ptr(name, map_ptr, len)\n    }\n}\n\nimpl<'a, OTA, OTB, S, T> DifferentialObserver<OTA, OTB, S> for StdMapObserver<'a, T, true>\nwhere\n    OTA: ObserversTuple<S>,\n    OTB: ObserversTuple<S>,\n    S: UsesInput,\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n}\n\n/// Use a const size to speedup `Feedback::is_interesting` when the user can\n/// know the size of the map at compile time.\n#[derive(Serialize, Deserialize, Debug)]\n#[serde(bound = \"T: serde::de::DeserializeOwned\")]\n#[allow(clippy::unsafe_derive_deserialize)]\npub struct ConstMapObserver<'a, T, const N: usize>\nwhere\n    T: Default + Copy + 'static + Serialize,\n{\n    map: OwnedMutSlice<'a, T>,\n    initial: T,\n    name: String,\n}\n\nimpl<'a, S, T, const N: usize> Observer<S> for ConstMapObserver<'a, T, N>\nwhere\n    S: UsesInput,\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n    Self: MapObserver,\n{\n    #[inline]\n    fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> {\n        self.reset_map()\n    }\n}\n\nimpl<'a, T, const N: usize> Named for ConstMapObserver<'a, T, N>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn name(&self) -> &str {\n        self.name.as_str()\n    }\n}\n\nimpl<'a, T, const N: usize> HasLen for ConstMapObserver<'a, T, N>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn len(&self) -> usize {\n        N\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> AsIter<'it> for ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = Iter<'it, T>;\n\n    fn as_iter(&'it self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> AsIterMut<'it> for ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = T;\n    type IntoIter = IterMut<'it, T>;\n\n    fn as_iter_mut(&'it mut self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> IntoIterator for &'it ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <Iter<'it, T> as Iterator>::Item;\n    type IntoIter = Iter<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T, const N: usize> IntoIterator for &'it mut ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Item = <IterMut<'it, T> as Iterator>::Item;\n    type IntoIter = IterMut<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, T, const N: usize> ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    /// Returns an iterator over the map.\n    pub fn iter(&self) -> Iter<'_, T> {\n        <&Self as IntoIterator>::into_iter(self)\n    }\n\n    /// Returns a mutable iterator over the map.\n    pub fn iter_mut(&mut self) -> IterMut<'_, T> {\n        <&mut Self as IntoIterator>::into_iter(self)\n    }\n}\n\nimpl<'a, T, const N: usize> MapObserver for ConstMapObserver<'a, T, N>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug,\n{\n    type Entry = T;\n\n    #[inline]\n    fn initial(&self) -> T {\n        self.initial\n    }\n\n    #[inline]\n    fn get(&self, idx: usize) -> &T {\n        &self.as_slice()[idx]\n    }\n\n    #[inline]\n    fn get_mut(&mut self, idx: usize) -> &mut T {\n        &mut self.as_mut_slice()[idx]\n    }\n\n    /// Count the set bytes in the map\n    fn count_bytes(&self) -> u64 {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for x in &map[0..cnt] {\n            if *x != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n\n    fn usable_count(&self) -> usize {\n        self.as_slice().len()\n    }\n\n    fn hash(&self) -> u64 {\n        hash_slice(self.as_slice())\n    }\n\n    /// Reset the map\n    #[inline]\n    fn reset_map(&mut self) -> Result<(), Error> {\n        // Normal memset, see https://rust.godbolt.org/z/Trs5hv\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_mut_slice();\n        for x in &mut map[0..cnt] {\n            *x = initial;\n        }\n        Ok(())\n    }\n\n    fn to_vec(&self) -> Vec<T> {\n        self.as_slice().to_vec()\n    }\n\n    /// Get the number of set entries with the specified indexes\n    fn how_many_set(&self, indexes: &[usize]) -> usize {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for i in indexes {\n            if *i < cnt && map[*i] != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n}\n\nimpl<'a, T, const N: usize> AsSlice for ConstMapObserver<'a, T, N>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n{\n    type Entry = T;\n    #[inline]\n    fn as_slice(&self) -> &[T] {\n        self.map.as_slice()\n    }\n}\nimpl<'a, T, const N: usize> AsMutSlice for ConstMapObserver<'a, T, N>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Debug,\n{\n    type Entry = T;\n    #[inline]\n    fn as_mut_slice(&mut self) -> &mut [T] {\n        self.map.as_mut_slice()\n    }\n}\n\nimpl<'a, T, const N: usize> ConstMapObserver<'a, T, N>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned,\n{\n    /// Creates a new [`MapObserver`]\n    ///\n    /// # Note\n    /// Will get a pointer to the map and dereference it at any point in time.\n    /// The map must not move in memory!\n    #[must_use]\n    pub fn new(name: &'static str, map: &'a mut [T]) -> Self {\n        assert!(map.len() >= N);\n        Self {\n            map: OwnedMutSlice::from(map),\n            name: name.to_string(),\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] with an owned map\n    #[must_use]\n    pub fn owned(name: &'static str, map: Vec<T>) -> Self {\n        assert!(map.len() >= N);\n        let initial = if map.is_empty() { T::default() } else { map[0] };\n        Self {\n            map: OwnedMutSlice::from(map),\n            name: name.to_string(),\n            initial,\n        }\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer\n    ///\n    /// # Safety\n    /// Will dereference the `map_ptr` with up to len elements.\n    pub unsafe fn from_mut_ptr(name: &'static str, map_ptr: *mut T) -> Self {\n        ConstMapObserver {\n            map: OwnedMutSlice::from_raw_parts_mut(map_ptr, N),\n            name: name.to_string(),\n            initial: T::default(),\n        }\n    }\n}\n\n/// Overlooking a variable bitmap\n#[derive(Serialize, Deserialize, Debug)]\n#[serde(bound = \"T: serde::de::DeserializeOwned\")]\n#[allow(clippy::unsafe_derive_deserialize)]\npub struct VariableMapObserver<'a, T>\nwhere\n    T: Default + Copy + 'static + Serialize + PartialEq + Bounded,\n{\n    map: OwnedMutSlice<'a, T>,\n    size: OwnedMutPtr<usize>,\n    initial: T,\n    name: String,\n}\n\nimpl<'a, S, T> Observer<S> for VariableMapObserver<'a, T>\nwhere\n    S: UsesInput,\n    T: Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + Bounded\n        + PartialEq,\n    Self: MapObserver,\n{\n    #[inline]\n    fn pre_exec(&mut self, _state: &mut S, _input: &S::Input) -> Result<(), Error> {\n        self.reset_map()\n    }\n}\n\nimpl<'a, T> Named for VariableMapObserver<'a, T>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + Bounded + PartialEq,\n{\n    #[inline]\n    fn name(&self) -> &str {\n        self.name.as_str()\n    }\n}\n\nimpl<'a, T> HasLen for VariableMapObserver<'a, T>\nwhere\n    T: Default + Copy + 'static + Serialize + serde::de::DeserializeOwned + PartialEq + Bounded,\n{\n    #[inline]\n    fn len(&self) -> usize {\n        *self.size.as_ref()\n    }\n}\n\nimpl<'a, 'it, T> AsIter<'it> for VariableMapObserver<'a, T>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    type Item = T;\n    type IntoIter = Iter<'it, T>;\n\n    fn as_iter(&'it self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T> AsIterMut<'it> for VariableMapObserver<'a, T>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    type Item = T;\n    type IntoIter = IterMut<'it, T>;\n\n    fn as_iter_mut(&'it mut self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, 'it, T> IntoIterator for &'it VariableMapObserver<'a, T>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    type Item = <Iter<'it, T> as Iterator>::Item;\n    type IntoIter = Iter<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_slice()[..cnt].iter()\n    }\n}\n\nimpl<'a, 'it, T> IntoIterator for &'it mut VariableMapObserver<'a, T>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    type Item = <IterMut<'it, T> as Iterator>::Item;\n    type IntoIter = IterMut<'it, T>;\n\n    fn into_iter(self) -> Self::IntoIter {\n        let cnt = self.usable_count();\n        self.as_mut_slice()[..cnt].iter_mut()\n    }\n}\n\nimpl<'a, T> VariableMapObserver<'a, T>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    /// Returns an iterator over the map.\n    pub fn iter(&self) -> Iter<'_, T> {\n        <&Self as IntoIterator>::into_iter(self)\n    }\n\n    /// Returns a mutable iterator over the map.\n    pub fn iter_mut(&mut self) -> IterMut<'_, T> {\n        <&mut Self as IntoIterator>::into_iter(self)\n    }\n}\n\nimpl<'a, T> MapObserver for VariableMapObserver<'a, T>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    type Entry = T;\n\n    #[inline]\n    fn initial(&self) -> T {\n        self.initial\n    }\n\n    #[inline]\n    fn usable_count(&self) -> usize {\n        *self.size.as_ref()\n    }\n\n    fn get(&self, idx: usize) -> &T {\n        &self.map.as_slice()[idx]\n    }\n\n    fn get_mut(&mut self, idx: usize) -> &mut T {\n        &mut self.map.as_mut_slice()[idx]\n    }\n\n    /// Count the set bytes in the map\n    fn count_bytes(&self) -> u64 {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for x in &map[0..cnt] {\n            if *x != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n    fn hash(&self) -> u64 {\n        hash_slice(self.as_slice())\n    }\n\n    /// Reset the map\n    #[inline]\n    fn reset_map(&mut self) -> Result<(), Error> {\n        // Normal memset, see https://rust.godbolt.org/z/Trs5hv\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_mut_slice();\n        for x in &mut map[0..cnt] {\n            *x = initial;\n        }\n        Ok(())\n    }\n\n    fn to_vec(&self) -> Vec<T> {\n        self.as_slice().to_vec()\n    }\n\n    fn how_many_set(&self, indexes: &[usize]) -> usize {\n        let initial = self.initial();\n        let cnt = self.usable_count();\n        let map = self.as_slice();\n        let mut res = 0;\n        for i in indexes {\n            if *i < cnt && map[*i] != initial {\n                res += 1;\n            }\n        }\n        res\n    }\n}\n\nimpl<'a, T> AsSlice for VariableMapObserver<'a, T>\nwhere\n    T: Bounded\n        + PartialEq\n        + Default\n        + Copy\n        + 'static\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    type Entry = T;\n    #[inline]\n    fn as_slice(&self) -> &[T] {\n        let cnt = self.usable_count();\n        &self.map.as_slice()[..cnt]\n    }\n}\nimpl<'a, T> AsMutSlice for VariableMapObserver<'a, T>\nwhere\n    T: 'static\n        + Default\n        + Copy\n        + Serialize\n        + serde::de::DeserializeOwned\n        + Debug\n        + PartialEq\n        + Bounded,\n{\n    type Entry = T;\n    #[inline]\n    fn as_mut_slice(&mut self) -> &mut [T] {\n        let cnt = self.usable_count();\n        &mut self.map.as_mut_slice()[..cnt]\n    }\n}\n\nimpl<'a, T> VariableMapObserver<'a, T>\nwhere\n    T: 'static + Default + Copy + Serialize + serde::de::DeserializeOwned + PartialEq + Bounded,\n{\n    /// Creates a new [`MapObserver`] from an [`OwnedMutSlice`]\n    ///\n    /// # Safety\n    /// The observer will dereference the owned slice, as well as the `map_ptr`.\n    /// Dereferences `map_ptr` with up to `max_len` elements of size.\n    pub unsafe fn from_mut_slice(\n        name: &'static str,\n        map_slice: OwnedMutSlice<'a, T>,\n        size: *mut usize,\n    ) -> Self {\n        VariableMapObserver {\n            name: name.into(),\n            map: map_slice,\n            size: OwnedMutPtr::Ptr(size),\n            initial: T::default(),\n        }\n    }\n\n    /// Creates a new [`MapObserver`] from a raw pointer\n    ///\n    /// # Safety\n    /// The observer will dereference the `size` ptr, as well as the `map_ptr`.\n    /// Dereferences `map_ptr` with up to `max_len` elements of size.\n    pub unsafe fn from_mut_ptr(\n        name: &'static str,\n        map_ptr: *mut T,\n        max_len: usize,\n        size: *mut usize,\n    ) -> Self {\n        Self::from_mut_slice(\n            name,\n            OwnedMutSlice::from_raw_parts_mut(map_ptr, max_len),\n            size,\n        )\n    }\n}\n\n/// Map observer with AFL-like hitcounts postprocessing\n///\n/// [`MapObserver`]s that are not slice-backed,\n/// such as [`MultiMapObserver`], can use [`HitcountsIterableMapObserver`] instead.\n#[derive(Serialize, Deserialize, Clone, Debug)]\n#[serde(bound = \"M: serde::de::DeserializeOwned\")]\npub struct HitcountsMapObserver<M>\nwhere\n    M: Serialize,\n{\n    base: M,\n}\n\nimpl<S, M> Observer<S> for HitcountsMapObserver<M>\nwhere\n    M: MapObserver<Entry = u8> + Observer<S> + AsMutSlice<Entry = u8>,\n    S: UsesInput,\n{\n    #[inline]\n    fn pre_exec(&mut self, state: &mut S, input: &S::Input) -> Result<(), Error> {\n        self.base.pre_exec(state, input)\n    }\n\n    #[inline]\n    #[allow(clippy::cast_ptr_alignment)]\n    fn post_exec(\n        &mut self,\n        state: &mut S,\n        input: &S::Input,\n        exit_kind: &ExitKind,\n    ) -> Result<(), Error> {\n        let map = self.as_mut_slice();\n        let mut len = map.len();\n        let align_offset = map.as_ptr().align_offset(size_of::<u16>());\n\n        // if len == 1, the next branch will already do this lookup\n        if len > 1 && align_offset != 0 {\n            debug_assert_eq!(\n                align_offset, 1,\n                \"Aligning u8 to u16 should always be offset of 1?\"\n            );\n            unsafe {\n                *map.get_unchecked_mut(0) =\n                    *COUNT_CLASS_LOOKUP.get_unchecked(*map.get_unchecked(0) as usize);\n            }\n            len -= 1;\n        }\n\n        // Fix the last element\n        if (len & 1) != 0 {\n            unsafe {\n                *map.get_unchecked_mut(len - 1) =\n                    *COUNT_CLASS_LOOKUP.get_unchecked(*map.get_unchecked(len - 1) as usize);\n            }\n        }\n\n        let cnt = len / 2;\n\n        let map16 = unsafe {\n            slice::from_raw_parts_mut(map.as_mut_ptr().add(align_offset) as *mut u16, cnt)\n        };\n        // 2022-07: Adding `enumerate` here increases execution speed/register allocation on x86_64.\n        #[allow(clippy::unused_enumerate_index)]\n        for (_i, item) in map16[0..cnt].iter_mut().enumerate() {\n            unsafe {\n                *item = *COUNT_CLASS_LOOKUP_16.get_unchecked(*item as usize);\n            }\n        }\n\n        self.base.post_exec(state, input, exit_kind)\n    }\n}\n\nimpl<M> Named for HitcountsMapObserver<M>\nwhere\n    M: Named + Serialize + serde::de::DeserializeOwned,\n{\n    #[inline]\n    fn name(&self) -> &str {\n        self.base.name()\n    }\n}\n\nimpl<M> HasLen for HitcountsMapObserver<M>\nwhere\n    M: MapObserver,\n{\n    #[inline]\n    fn len(&self) -> usize {\n        self.base.len()\n    }\n}\n\nimpl<M> MapObserver for HitcountsMapObserver<M>\nwhere\n    M: MapObserver<Entry = u8>,\n{\n    type Entry = u8;\n\n    #[inline]\n    fn initial(&self) -> u8 {\n        self.base.initial()\n    }\n\n    #[inline]\n    fn usable_count(&self) -> usize {\n        self.base.usable_count()\n    }\n\n    #[inline]\n    fn get(&self, idx: usize) -> &u8 {\n        self.base.get(idx)\n    }\n\n    #[inline]\n    fn get_mut(&mut self, idx: usize) -> &mut u8 {\n        self.base.get_mut(idx)\n    }\n\n    /// Count the set bytes in the map\n    fn count_bytes(&self) -> u64 {\n        self.base.count_bytes()\n    }\n\n    /// Reset the map\n    #[inline]\n    fn reset_map(&mut self) -> Result<(), Error> {\n        self.base.reset_map()\n    }\n\n    fn hash(&self) -> u64 {\n        self.base.hash()\n    }\n    fn to_vec(&self) -> Vec<u8> {\n        self.base.to_vec()\n    }\n\n    fn how_many_set(&self, indexes: &[usize]) -> usize {\n        self.base.how_many_set(indexes)\n    }\n}\n\n...\n// Path: libafl/src/feedbacks/map.rs\n//! Map feedback, maximizing or minimizing maps, for example the afl-style map observer.\n\nuse alloc::{\n    string::{String, ToString},\n    vec::Vec,\n};\n#[rustversion::nightly]\nuse core::simd::prelude::SimdOrd;\nuse core::{\n    fmt::Debug,\n    marker::PhantomData,\n    ops::{BitAnd, BitOr},\n};\n\nuse libafl_bolts::{AsIter, AsMutSlice, AsSlice, HasRefCnt, Named};\nuse num_traits::PrimInt;\nuse serde::{de::DeserializeOwned, Deserialize, Serialize};\n\nuse crate::{\n    corpus::Testcase,\n    events::{Event, EventFirer},\n    executors::ExitKind,\n    feedbacks::{Feedback, HasObserverName},\n    inputs::UsesInput,\n    monitors::{AggregatorOps, UserStats, UserStatsValue},\n    observers::{MapObserver, Observer, ObserversTuple, UsesObserver},\n    state::{HasMetadata, HasNamedMetadata, State},\n    Error,\n};\n\n/// A [`MapFeedback`] that implements the AFL algorithm using an [`OrReducer`] combining the bits for the history map and the bit from ``HitcountsMapObserver``.\npub type AflMapFeedback<O, S, T> = MapFeedback<DifferentIsNovel, O, OrReducer, S, T>;\n\n/// A [`MapFeedback`] that strives to maximize the map contents.\npub type MaxMapFeedback<O, S, T> = MapFeedback<DifferentIsNovel, O, MaxReducer, S, T>;\n/// A [`MapFeedback`] that strives to minimize the map contents.\npub type MinMapFeedback<O, S, T> = MapFeedback<DifferentIsNovel, O, MinReducer, S, T>;\n\n/// A [`MapFeedback`] that always returns `true` for `is_interesting`. Useful for tracing all executions.\npub type AlwaysInterestingMapFeedback<O, S, T> = MapFeedback<AllIsNovel, O, NopReducer, S, T>;\n\n/// A [`MapFeedback`] that strives to maximize the map contents,\n/// but only, if a value is larger than `pow2` of the previous.\npub type MaxMapPow2Feedback<O, S, T> = MapFeedback<NextPow2IsNovel, O, MaxReducer, S, T>;\n/// A [`MapFeedback`] that strives to maximize the map contents,\n/// but only, if a value is larger than `pow2` of the previous.\npub type MaxMapOneOrFilledFeedback<O, S, T> = MapFeedback<OneOrFilledIsNovel, O, MaxReducer, S, T>;\n\n/// A `Reducer` function is used to aggregate values for the novelty search\npub trait Reducer<T>: 'static\nwhere\n    T: Default + Copy + 'static,\n{\n    /// Reduce two values to one value, with the current [`Reducer`].\n    fn reduce(first: T, second: T) -> T;\n}\n\n/// A [`OrReducer`] reduces the values returning the bitwise OR with the old value\n#[derive(Clone, Debug)]\npub struct OrReducer {}\n\nimpl<T> Reducer<T> for OrReducer\nwhere\n    T: BitOr<Output = T> + Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(history: T, new: T) -> T {\n        history | new\n    }\n}\n\n/// A [`AndReducer`] reduces the values returning the bitwise AND with the old value\n#[derive(Clone, Debug)]\npub struct AndReducer {}\n\nimpl<T> Reducer<T> for AndReducer\nwhere\n    T: BitAnd<Output = T> + Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(history: T, new: T) -> T {\n        history & new\n    }\n}\n\n/// A [`NopReducer`] does nothing, and just \"reduces\" to the second/`new` value.\n#[derive(Clone, Debug)]\npub struct NopReducer {}\n\nimpl<T> Reducer<T> for NopReducer\nwhere\n    T: Default + Copy + 'static,\n{\n    #[inline]\n    fn reduce(_history: T, new: T) -> T {\n        new\n    }\n}\n\n/// A [`MaxReducer`] reduces int values and returns their maximum.\n#[derive(Clone, Debug)]\npub struct MaxReducer {}\n\nimpl<T> Reducer<T> for MaxReducer\nwhere\n    T: Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(first: T, second: T) -> T {\n        if first > second {\n            first\n        } else {\n            second\n        }\n    }\n}\n\n/// A [`MinReducer`] reduces int values and returns their minimum.\n#[derive(Clone, Debug)]\npub struct MinReducer {}\n\nimpl<T> Reducer<T> for MinReducer\nwhere\n    T: Default + Copy + 'static + PartialOrd,\n{\n    #[inline]\n    fn reduce(first: T, second: T) -> T {\n        if first < second {\n            first\n        } else {\n            second\n        }\n    }\n}\n\n/// A `IsNovel` function is used to discriminate if a reduced value is considered novel.\npub trait IsNovel<T>: 'static\nwhere\n    T: Default + Copy + 'static,\n{\n    /// If a new value in the [`MapFeedback`] was found,\n    /// this filter can decide if the result is considered novel or not.\n    fn is_novel(old: T, new: T) -> bool;\n}\n\n/// [`AllIsNovel`] consider everything a novelty. Here mostly just for debugging.\n#[derive(Clone, Debug)]\npub struct AllIsNovel {}\n\nimpl<T> IsNovel<T> for AllIsNovel\nwhere\n    T: Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(_old: T, _new: T) -> bool {\n        true\n    }\n}\n\n/// Calculate the next power of two\n/// See <https://stackoverflow.com/a/66253960/1345238>\n/// Will saturate at the max value.\n/// In case of negative values, returns 1.\n#[inline]\n\nfn saturating_next_power_of_two<T: PrimInt>(n: T) -> T {\n    if n <= T::one() {\n        T::one()\n    } else {\n        (T::max_value() >> (n - T::one()).leading_zeros().try_into().unwrap())\n            .saturating_add(T::one())\n    }\n}\n\n/// Consider as novelty if the reduced value is different from the old value.\n#[derive(Clone, Debug)]\npub struct DifferentIsNovel {}\nimpl<T> IsNovel<T> for DifferentIsNovel\nwhere\n    T: PartialEq + Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(old: T, new: T) -> bool {\n        old != new\n    }\n}\n\n/// Only consider as novel the values which are at least the next pow2 class of the old value\n#[derive(Clone, Debug)]\npub struct NextPow2IsNovel {}\nimpl<T> IsNovel<T> for NextPow2IsNovel\nwhere\n    T: PrimInt + Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(old: T, new: T) -> bool {\n        // We use a trait so we build our numbers from scratch here.\n        // This way it works with Nums of any size.\n        if new <= old {\n            false\n        } else {\n            let pow2 = saturating_next_power_of_two(old.saturating_add(T::one()));\n            new >= pow2\n        }\n    }\n}\n\n/// Only consider `T::one()` or `T::max_value()`, if they are bigger than the old value, as novel\n#[derive(Clone, Debug)]\npub struct OneOrFilledIsNovel {}\nimpl<T> IsNovel<T> for OneOrFilledIsNovel\nwhere\n    T: PrimInt + Default + Copy + 'static,\n{\n    #[inline]\n    fn is_novel(old: T, new: T) -> bool {\n        (new == T::one() || new == T::max_value()) && new > old\n    }\n}\n\n/// A testcase metadata holding a list of indexes of a map\n#[derive(Debug, Serialize, Deserialize)]\n#[cfg_attr(\n    any(not(feature = \"serdeany_autoreg\"), miri),\n    allow(clippy::unsafe_derive_deserialize)\n)] // for SerdeAny\npub struct MapIndexesMetadata {\n    /// The list of indexes.\n    pub list: Vec<usize>,\n    /// A refcount used to know when we can remove this metadata\n    pub tcref: isize,\n}\n\nlibafl_bolts::impl_serdeany!(MapIndexesMetadata);\n\nimpl AsSlice for MapIndexesMetadata {\n    type Entry = usize;\n    /// Convert to a slice\n    fn as_slice(&self) -> &[usize] {\n        self.list.as_slice()\n    }\n}\nimpl AsMutSlice for MapIndexesMetadata {\n    type Entry = usize;\n    /// Convert to a slice\n    fn as_mut_slice(&mut self) -> &mut [usize] {\n        self.list.as_mut_slice()\n    }\n}\n\nimpl HasRefCnt for MapIndexesMetadata {\n    fn refcnt(&self) -> isize {\n        self.tcref\n    }\n\n    fn refcnt_mut(&mut self) -> &mut isize {\n...\n", "needle_token_start": 15366, "needle_token_end": 15458, "code_context_ntokens": 16230, "output": ["```rust\n/// Calculate the next power of two\n/// See <https://stackoverflow.com/a/66253960/1345238>\n/// Will saturate at the max value.\n/// In case of negative values, returns 1.\n#[inline]\nfn saturating_next_power_of_two<T: PrimInt>(n: T) -> T {\n    if n <= T::one() {\n        T::one()\n    } else {\n        (T::max_value() >> (n - T::one()).leading_zeros().try_into().unwrap())\n            .saturating_add(T::one())\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "on_left_click", "language": "rust", "path": "alacritty/src/input/mod.rs", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function handles the actions triggered by a left mouse button click, including initiating or modifying text selection and moving the cursor in vi mode.\n2. **Input**: A point structure indicating the location of the mouse click.\n3. **Output**: There is no direct output; however, the function modifies the state of the application, such as updating selections and cursor positions.\n4. **Procedure**: \n   - The function first checks the current click state (single, double, or triple click) and modifies the selection based on this state and whether the control key is pressed.\n   - For a single click, it clears any existing selection and starts a new one based on the modifier key.\n   - For a double click, it starts a semantic selection (based on word boundaries).\n   - For a triple click, it starts a line-based selection.\n   - If in vi mode and not searching, it moves the vi mode cursor to the clicked position and marks the screen as needing a refresh.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "        let c = match state {\n            ElementState::Pressed => 'M',\n            ElementState::Released => 'm',\n        };\n\n        let msg = format!(\"\\x1b[<{};{};{}{}\", button, point.column + 1, point.line + 1, c);\n        self.ctx.write_to_pty(msg.into_bytes());\n    }\n\n    fn on_mouse_press(&mut self, button: MouseButton) {\n        // Handle mouse mode.\n        if !self.ctx.modifiers().state().shift_key() && self.ctx.mouse_mode() {\n            self.ctx.mouse_mut().click_state = ClickState::None;\n\n            let code = match button {\n                MouseButton::Left => 0,\n                MouseButton::Middle => 1,\n                MouseButton::Right => 2,\n                // Can't properly report more than three buttons..\n                MouseButton::Back | MouseButton::Forward | MouseButton::Other(_) => return,\n            };\n\n            self.mouse_report(code, ElementState::Pressed);\n        } else {\n            // Calculate time since the last click to handle double/triple clicks.\n            let now = Instant::now();\n            let elapsed = now - self.ctx.mouse().last_click_timestamp;\n            self.ctx.mouse_mut().last_click_timestamp = now;\n\n            // Update multi-click state.\n            self.ctx.mouse_mut().click_state = match self.ctx.mouse().click_state {\n                // Reset click state if button has changed.\n                _ if button != self.ctx.mouse().last_click_button => {\n                    self.ctx.mouse_mut().last_click_button = button;\n                    ClickState::Click\n                },\n                ClickState::Click if elapsed < CLICK_THRESHOLD => ClickState::DoubleClick,\n                ClickState::DoubleClick if elapsed < CLICK_THRESHOLD => ClickState::TripleClick,\n                _ => ClickState::Click,\n            };\n\n            // Load mouse point, treating message bar and padding as the closest cell.\n            let display_offset = self.ctx.terminal().grid().display_offset();\n            let point = self.ctx.mouse().point(&self.ctx.size_info(), display_offset);\n\n            if let MouseButton::Left = button {\n                self.on_left_click(point)\n            }\n        }\n    }\n\n    /// Handle left click selection and vi mode cursor movement.\n    \nfn on_left_click(&mut self, point: Point) {\n        let side = self.ctx.mouse().cell_side;\n\n        match self.ctx.mouse().click_state {\n            ClickState::Click => {\n                // Don't launch URLs if this click cleared the selection.\n                self.ctx.mouse_mut().block_hint_launcher = !self.ctx.selection_is_empty();\n\n                self.ctx.clear_selection();\n\n                // Start new empty selection.\n                if self.ctx.modifiers().state().control_key() {\n                    self.ctx.start_selection(SelectionType::Block, point, side);\n                } else {\n                    self.ctx.start_selection(SelectionType::Simple, point, side);\n                }\n            },\n            ClickState::DoubleClick => {\n                self.ctx.mouse_mut().block_hint_launcher = true;\n                self.ctx.start_selection(SelectionType::Semantic, point, side);\n            },\n            ClickState::TripleClick => {\n                self.ctx.mouse_mut().block_hint_launcher = true;\n                self.ctx.start_selection(SelectionType::Lines, point, side);\n            },\n            ClickState::None => (),\n        };\n\n        // Move vi mode cursor to mouse click position.\n        if self.ctx.terminal().mode().contains(TermMode::VI) && !self.ctx.search_active() {\n            self.ctx.terminal_mut().vi_mode_cursor.point = point;\n            self.ctx.mark_dirty();\n        }\n    }\n\n    fn on_mouse_release(&mut self, button: MouseButton) {\n        if !self.ctx.modifiers().state().shift_key() && self.ctx.mouse_mode() {\n            let code = match button {\n                MouseButton::Left => 0,\n                MouseButton::Middle => 1,\n                MouseButton::Right => 2,\n                // Can't properly report more than three buttons.\n                MouseButton::Back | MouseButton::Forward | MouseButton::Other(_) => return,\n            };\n            self.mouse_report(code, ElementState::Released);\n            return;\n        }\n\n        // Trigger hints highlighted by the mouse.\n        let hint = self.ctx.display().highlighted_hint.take();\n        if let Some(hint) = hint.as_ref().filter(|_| button == MouseButton::Left) {\n            self.ctx.trigger_hint(hint);\n        }\n        self.ctx.display().highlighted_hint = hint;\n\n        let timer_id = TimerId::new(Topic::SelectionScrolling, self.ctx.window().id());\n        self.ctx.scheduler_mut().unschedule(timer_id);\n\n        if let MouseButton::Left | MouseButton::Right = button {\n            // Copy selection on release, to prevent flooding the display server.\n            self.ctx.copy_selection(ClipboardType::Selection);\n        }\n    }\n\n    pub fn mouse_wheel_input(&mut self, delta: MouseScrollDelta, phase: TouchPhase) {\n        let multiplier = self.ctx.config().scrolling.multiplier;\n        match delta {\n            MouseScrollDelta::LineDelta(columns, lines) => {\n                let new_scroll_px_x = columns * self.ctx.size_info().cell_width();\n                let new_scroll_px_y = lines * self.ctx.size_info().cell_height();\n                self.scroll_terminal(\n                    new_scroll_px_x as f64,\n                    new_scroll_px_y as f64,\n                    multiplier as f64,\n                );\n            },\n            MouseScrollDelta::PixelDelta(mut lpos) => {\n                match phase {\n                    TouchPhase::Started => {\n                        // Reset offset to zero.\n                        self.ctx.mouse_mut().accumulated_scroll = Default::default();\n                    },\n                    TouchPhase::Moved => {\n                        // When the angle between (x, 0) and (x, y) is lower than ~25 degrees\n                        // (cosine is larger that 0.9) we consider this scrolling as horizontal.\n                        if lpos.x.abs() / lpos.x.hypot(lpos.y) > 0.9 {\n                            lpos.y = 0.;\n                        } else {\n                            lpos.x = 0.;\n                        }\n\n                        self.scroll_terminal(lpos.x, lpos.y, multiplier as f64);\n                    },\n                    _ => (),\n                }\n            },\n        }\n    }\n\n    fn scroll_terminal(&mut self, new_scroll_x_px: f64, new_scroll_y_px: f64, multiplier: f64) {\n        const MOUSE_WHEEL_UP: u8 = 64;\n        const MOUSE_WHEEL_DOWN: u8 = 65;\n        const MOUSE_WHEEL_LEFT: u8 = 66;\n        const MOUSE_WHEEL_RIGHT: u8 = 67;\n\n        let width = f64::from(self.ctx.size_info().cell_width());\n        let height = f64::from(self.ctx.size_info().cell_height());\n\n        if self.ctx.mouse_mode() {\n            self.ctx.mouse_mut().accumulated_scroll.x += new_scroll_x_px;\n            self.ctx.mouse_mut().accumulated_scroll.y += new_scroll_y_px;\n\n            let code = if new_scroll_y_px > 0. { MOUSE_WHEEL_UP } else { MOUSE_WHEEL_DOWN };\n            let lines = (self.ctx.mouse().accumulated_scroll.y / height).abs() as i32;\n\n            for _ in 0..lines {\n                self.mouse_report(code, ElementState::Pressed);\n            }\n\n            let code = if new_scroll_x_px > 0. { MOUSE_WHEEL_LEFT } else { MOUSE_WHEEL_RIGHT };\n            let columns = (self.ctx.mouse().accumulated_scroll.x / width).abs() as i32;\n\n            for _ in 0..columns {\n                self.mouse_report(code, ElementState::Pressed);\n            }\n        } else if self\n            .ctx\n            .terminal()\n            .mode()\n            .contains(TermMode::ALT_SCREEN | TermMode::ALTERNATE_SCROLL)\n            && !self.ctx.modifiers().state().shift_key()\n        {\n            self.ctx.mouse_mut().accumulated_scroll.x += new_scroll_x_px * multiplier;\n            self.ctx.mouse_mut().accumulated_scroll.y += new_scroll_y_px * multiplier;\n\n            // The chars here are the same as for the respective arrow keys.\n            let line_cmd = if new_scroll_y_px > 0. { b'A' } else { b'B' };\n            let column_cmd = if new_scroll_x_px > 0. { b'D' } else { b'C' };\n\n            let lines = (self.ctx.mouse().accumulated_scroll.y / height).abs() as usize;\n            let columns = (self.ctx.mouse().accumulated_scroll.x / width).abs() as usize;\n\n            let mut content = Vec::with_capacity(3 * (lines + columns));\n\n            for _ in 0..lines {\n                content.push(0x1b);\n                content.push(b'O');\n                content.push(line_cmd);\n            }\n\n            for _ in 0..columns {\n                content.push(0x1b);\n                content.push(b'O');\n                content.push(column_cmd);\n            }\n\n            self.ctx.write_to_pty(content);\n        } else {\n            self.ctx.mouse_mut().accumulated_scroll.y += new_scroll_y_px * multiplier;\n\n            let lines = (self.ctx.mouse().accumulated_scroll.y / height) as i32;\n\n            if lines != 0 {\n                self.ctx.scroll(Scroll::Delta(lines));\n            }\n        }\n\n        self.ctx.mouse_mut().accumulated_scroll.x %= width;\n        self.ctx.mouse_mut().accumulated_scroll.y %= height;\n    }\n\n    pub fn on_focus_change(&mut self, is_focused: bool) {\n        if self.ctx.terminal().mode().contains(TermMode::FOCUS_IN_OUT) {\n            let chr = if is_focused { \"I\" } else { \"O\" };\n\n            let msg = format!(\"\\x1b[{}\", chr);\n            self.ctx.write_to_pty(msg.into_bytes());\n        }\n    }\n\n    /// Handle touch input.\n    pub fn touch(&mut self, touch: TouchEvent) {\n        match touch.phase {\n            TouchPhase::Started => self.on_touch_start(touch),\n            TouchPhase::Moved => self.on_touch_motion(touch),\n            TouchPhase::Ended | TouchPhase::Cancelled => self.on_touch_end(touch),\n        }\n    }\n\n    /// Handle beginning of touch input.\n    pub fn on_touch_start(&mut self, touch: TouchEvent) {\n        let touch_purpose = self.ctx.touch_purpose();\n        *touch_purpose = match mem::take(touch_purpose) {\n            TouchPurpose::None => TouchPurpose::Tap(touch),\n            TouchPurpose::Tap(start) => TouchPurpose::Zoom(TouchZoom::new((start, touch))),\n            TouchPurpose::Zoom(zoom) => TouchPurpose::Invalid(zoom.slots()),\n            TouchPurpose::Scroll(event) | TouchPurpose::Select(event) => {\n                let mut set = HashSet::default();\n                set.insert(event.id);\n                TouchPurpose::Invalid(set)\n            },\n            TouchPurpose::Invalid(mut slots) => {\n                slots.insert(touch.id);\n                TouchPurpose::Invalid(slots)\n            },\n        };\n    }\n\n    /// Handle touch input movement.\n    pub fn on_touch_motion(&mut self, touch: TouchEvent) {\n        let touch_purpose = self.ctx.touch_purpose();\n        match touch_purpose {\n            TouchPurpose::None => (),\n            // Handle transition from tap to scroll/select.\n            TouchPurpose::Tap(start) => {\n                let delta_x = touch.location.x - start.location.x;\n                let delta_y = touch.location.y - start.location.y;\n                if delta_x.abs() > MAX_TAP_DISTANCE {\n                    // Update gesture state.\n                    let start_location = start.location;\n                    *touch_purpose = TouchPurpose::Select(*start);\n\n                    // Start simulated mouse input.\n                    self.mouse_moved(start_location);\n                    self.mouse_input(ElementState::Pressed, MouseButton::Left);\n\n                    // Apply motion since touch start.\n                    self.on_touch_motion(touch);\n                } else if delta_y.abs() > MAX_TAP_DISTANCE {\n                    // Update gesture state.\n                    *touch_purpose = TouchPurpose::Scroll(*start);\n\n                    // Apply motion since touch start.\n                    self.on_touch_motion(touch);\n                }\n            },\n            TouchPurpose::Zoom(zoom) => {\n                let font_delta = zoom.font_delta(touch);\n                self.ctx.change_font_size(font_delta);\n            },\n            TouchPurpose::Scroll(last_touch) => {\n                // Calculate delta and update last touch position.\n                let delta_y = touch.location.y - last_touch.location.y;\n                *touch_purpose = TouchPurpose::Scroll(touch);\n\n                // Use a fixed scroll factor for touchscreens, to accurately track finger motion.\n                self.scroll_terminal(0., delta_y, 1.0);\n            },\n            TouchPurpose::Select(_) => self.mouse_moved(touch.location),\n            TouchPurpose::Invalid(_) => (),\n        }\n    }\n\n    /// Handle end of touch input.\n    pub fn on_touch_end(&mut self, touch: TouchEvent) {\n        // Finalize the touch motion up to the release point.\n        self.on_touch_motion(touch);\n\n        let touch_purpose = self.ctx.touch_purpose();\n        match touch_purpose {\n            // Simulate LMB clicks.\n            TouchPurpose::Tap(start) => {\n                let start_location = start.location;\n                *touch_purpose = Default::default();\n\n                self.mouse_moved(start_location);\n                self.mouse_input(ElementState::Pressed, MouseButton::Left);\n                self.mouse_input(ElementState::Released, MouseButton::Left);\n            },\n            // Invalidate zoom once a finger was released.\n            TouchPurpose::Zoom(zoom) => {\n                let mut slots = zoom.slots();\n                slots.remove(&touch.id);\n                *touch_purpose = TouchPurpose::Invalid(slots);\n            },\n            // Reset touch state once all slots were released.\n            TouchPurpose::Invalid(slots) => {\n                slots.remove(&touch.id);\n                if slots.is_empty() {\n                    *touch_purpose = Default::default();\n                }\n            },\n            // Release simulated LMB.\n            TouchPurpose::Select(_) => {\n                *touch_purpose = Default::default();\n                self.mouse_input(ElementState::Released, MouseButton::Left);\n            },\n            // Reset touch state on scroll finish.\n            TouchPurpose::Scroll(_) => *touch_purpose = Default::default(),\n            TouchPurpose::None => (),\n        }\n    }\n\n    /// Reset mouse cursor based on modifier and terminal state.\n    #[inline]\n    pub fn reset_mouse_cursor(&mut self) {\n        let mouse_state = self.cursor_state();\n        self.ctx.window().set_mouse_cursor(mouse_state);\n    }\n\n    /// Modifier state change.\n    pub fn modifiers_input(&mut self, modifiers: Modifiers) {\n        *self.ctx.modifiers() = modifiers;\n\n        // Prompt hint highlight update.\n        self.ctx.mouse_mut().hint_highlight_dirty = true;\n\n        // Update mouse state and check for URL change.\n        let mouse_state = self.cursor_state();\n        self.ctx.window().set_mouse_cursor(mouse_state);\n    }\n\n    pub fn mouse_input(&mut self, state: ElementState, button: MouseButton) {\n        match button {\n            MouseButton::Left => self.ctx.mouse_mut().left_button_state = state,\n            MouseButton::Middle => self.ctx.mouse_mut().middle_button_state = state,\n            MouseButton::Right => self.ctx.mouse_mut().right_button_state = state,\n            _ => (),\n        }\n\n        // Skip normal mouse events if the message bar has been clicked.\n        if self.message_bar_cursor_state() == Some(CursorIcon::Pointer)\n            && state == ElementState::Pressed\n        {\n            let size = self.ctx.size_info();\n\n            let current_lines = self.ctx.message().map_or(0, |m| m.text(&size).len());\n\n            self.ctx.clear_selection();\n            self.ctx.pop_message();\n\n            // Reset cursor when message bar height changed or all messages are gone.\n            let new_lines = self.ctx.message().map_or(0, |m| m.text(&size).len());\n\n            let new_icon = match current_lines.cmp(&new_lines) {\n                Ordering::Less => CursorIcon::Default,\n                Ordering::Equal => CursorIcon::Pointer,\n                Ordering::Greater => {\n                    if self.ctx.mouse_mode() {\n                        CursorIcon::Default\n                    } else {\n                        CursorIcon::Text\n                    }\n                },\n            };\n\n            self.ctx.window().set_mouse_cursor(new_icon);\n        } else {\n            match state {\n                ElementState::Pressed => {\n                    // Process mouse press before bindings to update the `click_state`.\n                    self.on_mouse_press(button);\n                    self.process_mouse_bindings(button);\n                },\n                ElementState::Released => self.on_mouse_release(button),\n            }\n        }\n    }\n\n    /// Attempt to find a binding and execute its action.\n    ///\n    /// The provided mode, mods, and key must match what is allowed by a binding\n    /// for its action to be executed.\n    fn process_mouse_bindings(&mut self, button: MouseButton) {\n        let mode = BindingMode::new(self.ctx.terminal().mode(), self.ctx.search_active());\n        let mouse_mode = self.ctx.mouse_mode();\n        let mods = self.ctx.modifiers().state();\n        let mouse_bindings = self.ctx.config().mouse_bindings().to_owned();\n\n        // If mouse mode is active, also look for bindings without shift.\n        let mut check_fallback = mouse_mode && mods.contains(ModifiersState::SHIFT);\n\n        for binding in &mouse_bindings {\n            // Don't trigger normal bindings in mouse mode unless Shift is pressed.\n            if binding.is_triggered_by(mode, mods, &button) && (check_fallback || !mouse_mode) {\n                binding.action.execute(&mut self.ctx);\n                check_fallback = false;\n            }\n        }\n\n        if check_fallback {\n            let fallback_mods = mods & !ModifiersState::SHIFT;\n            for binding in &mouse_bindings {\n                if binding.is_triggered_by(mode, fallback_mods, &button) {\n                    binding.action.execute(&mut self.ctx);\n                }\n            }\n        }\n    }\n\n    /// Check mouse icon state in relation to the message bar.\n    fn message_bar_cursor_state(&self) -> Option<CursorIcon> {\n        // Since search is above the message bar, the button is offset by search's height.\n        let search_height = usize::from(self.ctx.search_active());\n\n        // Calculate Y position of the end of the last terminal line.\n        let size = self.ctx.size_info();\n        let terminal_end = size.padding_y() as usize\n            + size.cell_height() as usize * (size.screen_lines() + search_height);\n\n        let mouse = self.ctx.mouse();\n        let display_offset = self.ctx.terminal().grid().display_offset();\n        let point = self.ctx.mouse().point(&self.ctx.size_info(), display_offset);\n\n        if self.ctx.message().is_none() || (mouse.y <= terminal_end) {\n            None\n        } else if mouse.y <= terminal_end + size.cell_height() as usize\n            && point.column + message_bar::CLOSE_BUTTON_TEXT.len() >= size.columns()\n        {\n            Some(CursorIcon::Pointer)\n        } else {\n            Some(CursorIcon::Default)\n        }\n    }\n\n    /// Icon state of the cursor.\n    fn cursor_state(&mut self) -> CursorIcon {\n        let display_offset = self.ctx.terminal().grid().display_offset();\n        let point = self.ctx.mouse().point(&self.ctx.size_info(), display_offset);\n        let hyperlink = self.ctx.terminal().grid()[point].hyperlink();\n\n        // Function to check if mouse is on top of a hint.\n        let hint_highlighted = |hint: &HintMatch| hint.should_highlight(point, hyperlink.as_ref());\n\n        if let Some(mouse_state) = self.message_bar_cursor_state() {\n            mouse_state\n        } else if self.ctx.display().highlighted_hint.as_ref().map_or(false, hint_highlighted) {\n            CursorIcon::Pointer\n        } else if !self.ctx.modifiers().state().shift_key() && self.ctx.mouse_mode() {\n            CursorIcon::Default\n        } else {\n            CursorIcon::Text\n        }\n    }\n\n    /// Handle automatic scrolling when selecting above/below the window.\n    fn update_selection_scrolling(&mut self, mouse_y: i32) {\n        let scale_factor = self.ctx.window().scale_factor;\n        let size = self.ctx.size_info();\n        let window_id = self.ctx.window().id();\n        let scheduler = self.ctx.scheduler_mut();\n\n        // Scale constants by DPI.\n        let min_height = (MIN_SELECTION_SCROLLING_HEIGHT * scale_factor) as i32;\n        let step = (SELECTION_SCROLLING_STEP * scale_factor) as i32;\n\n        // Compute the height of the scrolling areas.\n        let end_top = max(min_height, size.padding_y() as i32);\n        let text_area_bottom = size.padding_y() + size.screen_lines() as f32 * size.cell_height();\n        let start_bottom = min(size.height() as i32 - min_height, text_area_bottom as i32);\n\n        // Get distance from closest window boundary.\n        let delta = if mouse_y < end_top {\n            end_top - mouse_y + step\n        } else if mouse_y >= start_bottom {\n            start_bottom - mouse_y - step\n        } else {\n            scheduler.unschedule(TimerId::new(Topic::SelectionScrolling, window_id));\n            return;\n        };\n\n        // Scale number of lines scrolled based on distance to boundary.\n        let event = Event::new(EventType::Scroll(Scroll::Delta(delta / step)), Some(window_id));\n\n        // Schedule event.\n        let timer_id = TimerId::new(Topic::SelectionScrolling, window_id);\n        scheduler.unschedule(timer_id);\n        scheduler.schedule(event, SELECTION_SCROLLING_INTERVAL, true, timer_id);\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    use winit::event::{DeviceId, Event as WinitEvent, WindowEvent};\n    use winit::keyboard::Key;\n    use winit::window::WindowId;\n\n    use alacritty_terminal::event::Event as TerminalEvent;\n\n    use crate::config::Binding;\n    use crate::message_bar::MessageBuffer;\n\n    const KEY: Key<&'static str> = Key::Character(\"0\");\n\n    struct MockEventProxy;\n    impl EventListener for MockEventProxy {}\n\n    struct ActionContext<'a, T> {\n        pub terminal: &'a mut Term<T>,\n        pub size_info: &'a SizeInfo,\n        pub mouse: &'a mut Mouse,\n        pub clipboard: &'a mut Clipboard,\n        pub message_buffer: &'a mut MessageBuffer,\n        pub modifiers: Modifiers,\n        config: &'a UiConfig,\n        inline_search_state: &'a mut InlineSearchState,\n    }\n\n    impl<'a, T: EventListener> super::ActionContext<T> for ActionContext<'a, T> {\n        fn search_next(\n            &mut self,\n            _origin: Point,\n            _direction: Direction,\n            _side: Side,\n        ) -> Option<Match> {\n            None\n        }\n\n        fn search_direction(&self) -> Direction {\n            Direction::Right\n        }\n\n        fn inline_search_state(&mut self) -> &mut InlineSearchState {\n            self.inline_search_state\n        }\n\n        fn search_active(&self) -> bool {\n            false\n        }\n\n        fn terminal(&self) -> &Term<T> {\n            self.terminal\n        }\n\n        fn terminal_mut(&mut self) -> &mut Term<T> {\n            self.terminal\n        }\n\n        fn size_info(&self) -> SizeInfo {\n            *self.size_info\n        }\n\n        fn selection_is_empty(&self) -> bool {\n            true\n        }\n\n        fn scroll(&mut self, scroll: Scroll) {\n            self.terminal.scroll_display(scroll);\n        }\n\n        fn mouse_mode(&self) -> bool {\n            false\n        }\n\n        #[inline]\n        fn mouse_mut(&mut self) -> &mut Mouse {\n            self.mouse\n        }\n\n        #[inline]\n        fn mouse(&self) -> &Mouse {\n            self.mouse\n        }\n\n        #[inline]\n        fn touch_purpose(&mut self) -> &mut TouchPurpose {\n            unimplemented!();\n        }\n\n        fn modifiers(&mut self) -> &mut Modifiers {\n            &mut self.modifiers\n        }\n\n        fn window(&mut self) -> &mut Window {\n            unimplemented!();\n        }\n\n        fn display(&mut self) -> &mut Display {\n            unimplemented!();\n        }\n\n        fn pop_message(&mut self) {\n            self.message_buffer.pop();\n        }\n\n        fn message(&self) -> Option<&Message> {\n            self.message_buffer.message()\n        }\n\n        fn config(&self) -> &UiConfig {\n            self.config\n        }\n\n        fn clipboard_mut(&mut self) -> &mut Clipboard {\n            self.clipboard\n        }\n\n        #[cfg(target_os = \"macos\")]\n        fn event_loop(&self) -> &EventLoopWindowTarget<Event> {\n            unimplemented!();\n        }\n\n        fn scheduler_mut(&mut self) -> &mut Scheduler {\n            unimplemented!();\n        }\n    }\n\n    macro_rules! test_clickstate {\n        {\n            name: $name:ident,\n            initial_state: $initial_state:expr,\n            initial_button: $initial_button:expr,\n            input: $input:expr,\n            end_state: $end_state:expr,\n            input_delay: $input_delay:expr,\n        } => {\n            #[test]\n            fn $name() {\n                let mut clipboard = Clipboard::new_nop();\n                let cfg = UiConfig::default();\n                let size = SizeInfo::new(\n                    21.0,\n                    51.0,\n                    3.0,\n                    3.0,\n                    0.,\n                    0.,\n                    false,\n                );\n\n                let mut terminal = Term::new(cfg.term_options(), &size, MockEventProxy);\n\n                let mut mouse = Mouse {\n                    click_state: $initial_state,\n                    last_click_button: $initial_button,\n                    last_click_timestamp: Instant::now() - $input_delay,\n                    ..Mouse::default()\n                };\n\n                let mut inline_search_state = InlineSearchState::default();\n                let mut message_buffer = MessageBuffer::default();\n\n                let context = ActionContext {\n                    terminal: &mut terminal,\n                    mouse: &mut mouse,\n                    size_info: &size,\n                    clipboard: &mut clipboard,\n                    modifiers: Default::default(),\n                    message_buffer: &mut message_buffer,\n                    inline_search_state: &mut inline_search_state,\n                    config: &cfg,\n                };\n\n                let mut processor = Processor::new(context);\n\n                let event: WinitEvent::<TerminalEvent> = $input;\n                if let WinitEvent::WindowEvent {\n                    event: WindowEvent::MouseInput {\n                        state,\n                        button,\n                        ..\n                    },\n                    ..\n                } = event\n                {\n                    processor.mouse_input(state, button);\n                };\n\n                assert_eq!(processor.ctx.mouse.click_state, $end_state);\n            }\n        }\n    }\n\n    macro_rules! test_process_binding {\n        {\n            name: $name:ident,\n            binding: $binding:expr,\n            triggers: $triggers:expr,\n            mode: $mode:expr,\n            mods: $mods:expr,\n        } => {\n            #[test]\n            fn $name() {\n                if $triggers {\n                    assert!($binding.is_triggered_by($mode, $mods, &KEY));\n                } else {\n                    assert!(!$binding.is_triggered_by($mode, $mods, &KEY));\n                }\n            }\n        }\n    }\n\n    test_clickstate! {\n        name: single_click,\n        initial_state: ClickState::None,\n        initial_button: MouseButton::Other(0),\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Left,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::Click,\n        input_delay: Duration::ZERO,\n    }\n\n    test_clickstate! {\n        name: single_right_click,\n        initial_state: ClickState::None,\n        initial_button: MouseButton::Other(0),\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Right,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::Click,\n        input_delay: Duration::ZERO,\n    }\n\n    test_clickstate! {\n        name: single_middle_click,\n        initial_state: ClickState::None,\n        initial_button: MouseButton::Other(0),\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Middle,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::Click,\n        input_delay: Duration::ZERO,\n    }\n\n    test_clickstate! {\n        name: double_click,\n        initial_state: ClickState::Click,\n        initial_button: MouseButton::Left,\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Left,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::DoubleClick,\n        input_delay: Duration::ZERO,\n    }\n\n    test_clickstate! {\n        name: double_click_failed,\n        initial_state: ClickState::Click,\n        initial_button: MouseButton::Left,\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Left,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::Click,\n        input_delay: CLICK_THRESHOLD,\n    }\n\n    test_clickstate! {\n        name: triple_click,\n        initial_state: ClickState::DoubleClick,\n        initial_button: MouseButton::Left,\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Left,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::TripleClick,\n        input_delay: Duration::ZERO,\n    }\n\n    test_clickstate! {\n        name: triple_click_failed,\n        initial_state: ClickState::DoubleClick,\n        initial_button: MouseButton::Left,\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Left,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::Click,\n        input_delay: CLICK_THRESHOLD,\n    }\n\n    test_clickstate! {\n        name: multi_click_separate_buttons,\n        initial_state: ClickState::DoubleClick,\n        initial_button: MouseButton::Left,\n        input: WinitEvent::WindowEvent {\n            event: WindowEvent::MouseInput {\n                state: ElementState::Pressed,\n                button: MouseButton::Right,\n                device_id: unsafe { DeviceId::dummy() },\n            },\n            window_id: unsafe { WindowId::dummy() },\n        },\n        end_state: ClickState::Click,\n        input_delay: Duration::ZERO,\n    }\n\n    test_process_binding! {\n        name: process_binding_nomode_shiftmod_require_shift,\n        binding: Binding { trigger: KEY, mods: ModifiersState::SHIFT, action: Action::from(\"\\x1b[1;2D\"), mode: BindingMode::empty(), notmode: BindingMode::empty() },\n        triggers: true,\n        mode: BindingMode::empty(),\n        mods: ModifiersState::SHIFT,\n    }\n\n    test_process_binding! {\n        name: process_binding_nomode_nomod_require_shift,\n        binding: Binding { trigger: KEY, mods: ModifiersState::SHIFT, action: Action::from(\"\\x1b[1;2D\"), mode: BindingMode::empty(), notmode: BindingMode::empty() },\n        triggers: false,\n        mode: BindingMode::empty(),\n        mods: ModifiersState::empty(),\n    }\n\n    test_process_binding! {\n        name: process_binding_nomode_controlmod,\n        binding: Binding { trigger: KEY, mods: ModifiersState::CONTROL, action: Action::from(\"\\x1b[1;5D\"), mode: BindingMode::empty(), notmode: BindingMode::empty() },\n        triggers: true,\n        mode: BindingMode::empty(),\n        mods: ModifiersState::CONTROL,\n    }\n\n    test_process_binding! {\n        name: process_binding_nomode_nomod_require_not_appcursor,\n        binding: Binding { trigger: KEY, mods: ModifiersState::empty(), action: Action::from(\"\\x1b[D\"), mode: BindingMode::empty(), notmode: BindingMode::APP_CURSOR },\n        triggers: true,\n        mode: BindingMode::empty(),\n        mods: ModifiersState::empty(),\n    }\n\n    test_process_binding! {\n        name: process_binding_appcursormode_nomod_require_appcursor,\n        binding: Binding { trigger: KEY, mods: ModifiersState::empty(), action: Action::from(\"\\x1bOD\"), mode: BindingMode::APP_CURSOR, notmode: BindingMode::empty() },\n        triggers: true,\n        mode: BindingMode::APP_CURSOR,\n        mods: ModifiersState::empty(),\n    }\n\n    test_process_binding! {\n        name: process_binding_nomode_nomod_require_appcursor,\n        binding: Binding { trigger: KEY, mods: ModifiersState::empty(), action: Action::from(\"\\x1bOD\"), mode: BindingMode::APP_CURSOR, notmode: BindingMode::empty() },\n        triggers: false,\n        mode: BindingMode::empty(),\n        mods: ModifiersState::empty(),\n    }\n\n    test_process_binding! {\n        name: process_binding_appcursormode_appkeypadmode_nomod_require_appcursor,\n        binding: Binding { trigger: KEY, mods: ModifiersState::empty(), action: Action::from(\"\\x1bOD\"), mode: BindingMode::APP_CURSOR, notmode: BindingMode::empty() },\n        triggers: true,\n        mode: BindingMode::APP_CURSOR | BindingMode::APP_KEYPAD,\n        mods: ModifiersState::empty(),\n    }\n\n    test_process_binding! {\n        name: process_binding_fail_with_extra_mods,\n        binding: Binding { trigger: KEY, mods: ModifiersState::SUPER, action: Action::from(\"arst\"), mode: BindingMode::empty(), notmode: BindingMode::empty() },\n        triggers: false,\n        mode: BindingMode::empty(),\n        mods: ModifiersState::ALT | ModifiersState::SUPER,\n    }\n}\n\n// Path: alacritty/src/display/damage.rs\nuse std::iter::Peekable;\nuse std::{cmp, mem};\n\nuse glutin::surface::Rect;\n\nuse alacritty_terminal::index::Point;\nuse alacritty_terminal::selection::SelectionRange;\nuse alacritty_terminal::term::{LineDamageBounds, TermDamageIterator};\n\nuse crate::display::SizeInfo;\n\n/// State of the damage tracking for the [`Display`].\n///\n/// [`Display`]: crate::display::Display\n#[derive(Debug)]\npub struct DamageTracker {\n    /// Position of the previously drawn Vi cursor.\n    pub old_vi_cursor: Option<Point<usize>>,\n    /// The location of the old selection.\n    pub old_selection: Option<SelectionRange>,\n    /// Highlight damage submitted for the compositor.\n    pub debug: bool,\n\n    /// The damage for the frames.\n    frames: [FrameDamage; 2],\n    screen_lines: usize,\n    columns: usize,\n}\n\nimpl DamageTracker {\n    pub fn new(screen_lines: usize, columns: usize) -> Self {\n        let mut tracker = Self {\n            columns,\n            screen_lines,\n            debug: false,\n            old_vi_cursor: None,\n            old_selection: None,\n            frames: Default::default(),\n        };\n        tracker.resize(screen_lines, columns);\n        tracker\n    }\n\n    #[inline]\n    #[must_use]\n    pub fn frame(&mut self) -> &mut FrameDamage {\n        &mut self.frames[0]\n    }\n\n    #[inline]\n    #[must_use]\n    pub fn next_frame(&mut self) -> &mut FrameDamage {\n        &mut self.frames[1]\n    }\n\n    /// Advance to the next frame resetting the state for the active frame.\n    #[inline]\n    pub fn swap_damage(&mut self) {\n        let screen_lines = self.screen_lines;\n        let columns = self.columns;\n        self.frame().reset(screen_lines, columns);\n        self.frames.swap(0, 1);\n    }\n\n    /// Resize the damage information in the tracker.\n    pub fn resize(&mut self, screen_lines: usize, columns: usize) {\n        self.screen_lines = screen_lines;\n        self.columns = columns;\n        for frame in &mut self.frames {\n            frame.reset(screen_lines, columns);\n        }\n        self.frame().full = true;\n    }\n\n    /// Damage vi cursor inside the viewport.\n    pub fn damage_vi_cursor(&mut self, mut vi_cursor: Option<Point<usize>>) {\n        mem::swap(&mut self.old_vi_cursor, &mut vi_cursor);\n\n        if self.frame().full {\n            return;\n        }\n\n        if let Some(vi_cursor) = self.old_vi_cursor {\n            self.frame().damage_point(vi_cursor);\n        }\n\n        if let Some(vi_cursor) = vi_cursor {\n            self.frame().damage_point(vi_cursor);\n        }\n    }\n\n    /// Get shaped frame damage for the active frame.\n    pub fn shape_frame_damage(&self, size_info: SizeInfo<u32>) -> Vec<Rect> {\n        if self.frames[0].full {\n            vec![Rect::new(0, 0, size_info.width() as i32, size_info.height() as i32)]\n        } else {\n            let lines_damage = RenderDamageIterator::new(\n                TermDamageIterator::new(&self.frames[0].lines, 0),\n                &size_info,\n            );\n            lines_damage.chain(self.frames[0].rects.iter().copied()).collect()\n        }\n    }\n\n    /// Add the current frame's selection damage.\n    pub fn damage_selection(\n        &mut self,\n        mut selection: Option<SelectionRange>,\n        display_offset: usize,\n    ) {\n        mem::swap(&mut self.old_selection, &mut selection);\n\n        if self.frame().full || selection == self.old_selection {\n            return;\n        }\n\n        for selection in self.old_selection.into_iter().chain(selection) {\n            let display_offset = display_offset as i32;\n            let last_visible_line = self.screen_lines as i32 - 1;\n            let columns = self.columns;\n\n            // Ignore invisible selection.\n            if selection.end.line.0 + display_offset < 0\n                || selection.start.line.0.abs() < display_offset - last_visible_line\n            {\n                continue;\n            };\n\n            let start = cmp::max(selection.start.line.0 + display_offset, 0) as usize;\n            let end = (selection.end.line.0 + display_offset).clamp(0, last_visible_line) as usize;\n            for line in start..=end {\n                self.frame().lines[line].expand(0, columns - 1);\n            }\n        }\n    }\n}\n\n/// Damage state for the rendering frame.\n#[derive(Debug, Default)]\npub struct FrameDamage {\n    /// The entire frame needs to be redrawn.\n    full: bool,\n    /// Terminal lines damaged in the given frame.\n    lines: Vec<LineDamageBounds>,\n    /// Rectangular regions damage in the given frame.\n    rects: Vec<Rect>,\n}\n\nimpl FrameDamage {\n    /// Damage line for the given frame.\n    #[inline]\n    pub fn damage_line(&mut self, damage: LineDamageBounds) {\n        self.lines[damage.line].expand(damage.left, damage.right);\n    }\n\n    #[inline]\n    pub fn damage_point(&mut self, point: Point<usize>) {\n        self.lines[point.line].expand(point.column.0, point.column.0);\n    }\n\n    /// Mark the frame as fully damaged.\n    #[inline]\n    pub fn mark_fully_damaged(&mut self) {\n        self.full = true;\n    }\n\n    /// Add viewport rectangle to damage.\n    ///\n    /// This allows covering elements outside of the terminal viewport, like message bar.\n    #[inline]\n    pub fn add_viewport_rect(\n        &mut self,\n        size_info: &SizeInfo,\n        x: i32,\n        y: i32,\n        width: i32,\n        height: i32,\n    ) {\n        let y = viewport_y_to_damage_y(size_info, y, height);\n        self.rects.push(Rect { x, y, width, height });\n    }\n\n    fn reset(&mut self, num_lines: usize, num_cols: usize) {\n        self.full = false;\n        self.rects.clear();\n        self.lines.clear();\n        self.lines.reserve(num_lines);\n        for line in 0..num_lines {\n            self.lines.push(LineDamageBounds::undamaged(line, num_cols));\n        }\n    }\n}\n\n/// Convert viewport `y` coordinate to [`Rect`] damage coordinate.\npub fn viewport_y_to_damage_y(size_info: &SizeInfo, y: i32, height: i32) -> i32 {\n    size_info.height() as i32 - y - height\n}\n\n/// Convert viewport `y` coordinate to [`Rect`] damage coordinate.\npub fn damage_y_to_viewport_y(size_info: &SizeInfo, rect: &Rect) -> i32 {\n    size_info.height() as i32 - rect.y - rect.height\n}\n\n/// Iterator which converts `alacritty_terminal` damage information into renderer damaged rects.\nstruct RenderDamageIterator<'a> {\n    damaged_lines: Peekable<TermDamageIterator<'a>>,\n    size_info: &'a SizeInfo<u32>,\n}\n\nimpl<'a> RenderDamageIterator<'a> {\n    pub fn new(damaged_lines: TermDamageIterator<'a>, size_info: &'a SizeInfo<u32>) -> Self {\n        Self { damaged_lines: damaged_lines.peekable(), size_info }\n    }\n\n    #[inline]\n    fn rect_for_line(&self, line_damage: LineDamageBounds) -> Rect {\n        let size_info = &self.size_info;\n        let y_top = size_info.height() - size_info.padding_y();\n        let x = size_info.padding_x() + line_damage.left as u32 * size_info.cell_width();\n        let y = y_top - (line_damage.line + 1) as u32 * size_info.cell_height();\n        let width = (line_damage.right - line_damage.left + 1) as u32 * size_info.cell_width();\n        Rect::new(x as i32, y as i32, width as i32, size_info.cell_height() as i32)\n    }\n\n    // Make sure to damage near cells to include wide chars.\n    #[inline]\n    fn overdamage(size_info: &SizeInfo<u32>, mut rect: Rect) -> Rect {\n        rect.x = (rect.x - size_info.cell_width() as i32).max(0);\n        rect.width = cmp::min(\n            size_info.width() as i32 - rect.x,\n            rect.width + 2 * size_info.cell_width() as i32,\n        );\n        rect.y = (rect.y - size_info.cell_height() as i32 / 2).max(0);\n        rect.height = cmp::min(\n            size_info.height() as i32 - rect.y,\n            rect.height + size_info.cell_height() as i32,\n        );\n\n        rect\n    }\n}\n\nimpl<'a> Iterator for RenderDamageIterator<'a> {\n    type Item = Rect;\n\n    fn next(&mut self) -> Option<Rect> {\n        let line = self.damaged_lines.next()?;\n        let size_info = &self.size_info;\n        let mut total_damage_rect = Self::overdamage(size_info, self.rect_for_line(line));\n\n        // Merge rectangles which overlap with each other.\n        while let Some(line) = self.damaged_lines.peek().copied() {\n            let next_rect = Self::overdamage(size_info, self.rect_for_line(line));\n            if !rects_overlap(total_damage_rect, next_rect) {\n                break;\n            }\n\n            total_damage_rect = merge_rects(total_damage_rect, next_rect);\n            let _ = self.damaged_lines.next();\n        }\n\n        Some(total_damage_rect)\n    }\n}\n\n/// Check if two given [`glutin::surface::Rect`] overlap.\nfn rects_overlap(lhs: Rect, rhs: Rect) -> bool {\n    !(\n        // `lhs` is left of `rhs`.\n        lhs.x + lhs.width < rhs.x\n        // `lhs` is right of `rhs`.\n        || rhs.x + rhs.width < lhs.x\n        // `lhs` is below `rhs`.\n        || lhs.y + lhs.height < rhs.y\n        // `lhs` is above `rhs`.\n        || rhs.y + rhs.height < lhs.y\n    )\n}\n\n/// Merge two [`glutin::surface::Rect`] by producing the smallest rectangle that contains both.\n#[inline]\nfn merge_rects(lhs: Rect, rhs: Rect) -> Rect {\n    let left_x = cmp::min(lhs.x, rhs.x);\n    let right_x = cmp::max(lhs.x + lhs.width, rhs.x + rhs.width);\n    let y_top = cmp::max(lhs.y + lhs.height, rhs.y + rhs.height);\n    let y_bottom = cmp::min(lhs.y, rhs.y);\n    Rect::new(left_x, y_bottom, right_x - left_x, y_top - y_bottom)\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn damage_rect_math() {\n        let rect_side = 10;\n        let cell_size = 4;\n        let bound = 100;\n\n        let size_info: SizeInfo<u32> = SizeInfo::new(\n            bound as f32,\n            bound as f32,\n            cell_size as f32,\n            cell_size as f32,\n            2.,\n            2.,\n            true,\n        )\n        .into();\n\n        // Test min clamping.\n        let rect = Rect::new(0, 0, rect_side, rect_side);\n        let rect = RenderDamageIterator::overdamage(&size_info, rect);\n        assert_eq!(Rect::new(0, 0, rect_side + 2 * cell_size, 10 + cell_size), rect);\n\n        // Test max clamping.\n        let rect = Rect::new(bound, bound, rect_side, rect_side);\n        let rect = RenderDamageIterator::overdamage(&size_info, rect);\n        assert_eq!(\n            Rect::new(bound - cell_size, bound - cell_size / 2, cell_size, cell_size / 2),\n            rect\n        );\n\n        // Test no clamping.\n        let rect = Rect::new(bound / 2, bound / 2, rect_side, rect_side);\n        let rect = RenderDamageIterator::overdamage(&size_info, rect);\n        assert_eq!(\n            Rect::new(\n                bound / 2 - cell_size,\n                bound / 2 - cell_size / 2,\n                rect_side + 2 * cell_size,\n                rect_side + cell_size\n            ),\n            rect\n        );\n    }\n\n    #[test]\n    fn add_viewport_damage() {\n        let mut frame_damage = FrameDamage::default();\n        let viewport_height = 100.;\n        let x = 0;\n        let y = 40;\n        let height = 5;\n        let width = 10;\n        let size_info = SizeInfo::new(viewport_height, viewport_height, 5., 5., 0., 0., true);\n        frame_damage.add_viewport_rect(&size_info, x, y, width, height);\n        assert_eq!(frame_damage.rects[0], Rect {\n            x,\n            y: viewport_height as i32 - y - height,\n            width,\n            height\n        });\n        assert_eq!(frame_damage.rects[0].y, viewport_y_to_damage_y(&size_info, y, height));\n        assert_eq!(damage_y_to_viewport_y(&size_info, &frame_damage.rects[0]), y);\n    }\n}\n\n// Path: alacritty/src/config/selection.rs\nuse alacritty_config_derive::ConfigDeserialize;\nuse alacritty_terminal::term::SEMANTIC_ESCAPE_CHARS;\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct Selection {\n    pub semantic_escape_chars: String,\n    pub save_to_clipboard: bool,\n}\n\nimpl Default for Selection {\n    fn default() -> Self {\n        Self {\n            semantic_escape_chars: SEMANTIC_ESCAPE_CHARS.to_owned(),\n            save_to_clipboard: Default::default(),\n        }\n    }\n}\n\n// Path: alacritty/src/daemon.rs\nuse std::ffi::OsStr;\n#[cfg(not(any(target_os = \"macos\", windows)))]\nuse std::fs;\nuse std::io;\n#[cfg(windows)]\nuse std::os::windows::process::CommandExt;\nuse std::process::{Command, Stdio};\n\n#[rustfmt::skip]\n#[cfg(not(windows))]\nuse {\n    std::error::Error,\n    std::os::unix::process::CommandExt,\n    std::os::unix::io::RawFd,\n    std::path::PathBuf,\n};\n\n#[cfg(not(windows))]\nuse libc::pid_t;\n#[cfg(windows)]\nuse windows_sys::Win32::System::Threading::{CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW};\n\n#[cfg(target_os = \"macos\")]\nuse crate::macos;\n\n/// Start a new process in the background.\n#[cfg(windows)]\npub fn spawn_daemon<I, S>(program: &str, args: I) -> io::Result<()>\nwhere\n    I: IntoIterator<Item = S> + Copy,\n    S: AsRef<OsStr>,\n{\n    // Setting all the I/O handles to null and setting the\n    // CREATE_NEW_PROCESS_GROUP and CREATE_NO_WINDOW has the effect\n    // that console applications will run without opening a new\n    // console window.\n    Command::new(program)\n        .args(args)\n        .stdin(Stdio::null())\n        .stdout(Stdio::null())\n        .stderr(Stdio::null())\n        .creation_flags(CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW)\n        .spawn()\n        .map(|_| ())\n}\n\n/// Start a new process in the background.\n#[cfg(not(windows))]\npub fn spawn_daemon<I, S>(\n    program: &str,\n    args: I,\n    master_fd: RawFd,\n    shell_pid: u32,\n) -> io::Result<()>\nwhere\n    I: IntoIterator<Item = S> + Copy,\n    S: AsRef<OsStr>,\n{\n    let mut command = Command::new(program);\n    command.args(args).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null());\n    if let Ok(cwd) = foreground_process_path(master_fd, shell_pid) {\n        command.current_dir(cwd);\n    }\n    unsafe {\n        command\n            .pre_exec(|| {\n                match libc::fork() {\n                    -1 => return Err(io::Error::last_os_error()),\n                    0 => (),\n                    _ => libc::_exit(0),\n                }\n\n                if libc::setsid() == -1 {\n                    return Err(io::Error::last_os_error());\n                }\n\n                Ok(())\n            })\n            .spawn()?\n            .wait()\n            .map(|_| ())\n    }\n}\n\n/// Get working directory of controlling process.\n#[cfg(not(windows))]\npub fn foreground_process_path(\n    master_fd: RawFd,\n    shell_pid: u32,\n) -> Result<PathBuf, Box<dyn Error>> {\n    let mut pid = unsafe { libc::tcgetpgrp(master_fd) };\n    if pid < 0 {\n        pid = shell_pid as pid_t;\n    }\n\n    #[cfg(not(any(target_os = \"macos\", target_os = \"freebsd\")))]\n    let link_path = format!(\"/proc/{}/cwd\", pid);\n    #[cfg(target_os = \"freebsd\")]\n    let link_path = format!(\"/compat/linux/proc/{}/cwd\", pid);\n\n    #[cfg(not(target_os = \"macos\"))]\n    let cwd = fs::read_link(link_path)?;\n\n    #[cfg(target_os = \"macos\")]\n    let cwd = macos::proc::cwd(pid)?;\n\n    Ok(cwd)\n}\n\n// Path: alacritty/src/renderer/text/builtin_font.rs\n//! Hand-rolled drawing of unicode characters that need to fully cover their character area.\n\nuse std::{cmp, mem, ops};\n\nuse crossfont::{BitmapBuffer, Metrics, RasterizedGlyph};\n\nuse crate::config::ui_config::Delta;\n\n// Colors which are used for filling shade variants.\nconst COLOR_FILL_ALPHA_STEP_1: Pixel = Pixel { _r: 192, _g: 192, _b: 192 };\nconst COLOR_FILL_ALPHA_STEP_2: Pixel = Pixel { _r: 128, _g: 128, _b: 128 };\nconst COLOR_FILL_ALPHA_STEP_3: Pixel = Pixel { _r: 64, _g: 64, _b: 64 };\n\n/// Default color used for filling.\nconst COLOR_FILL: Pixel = Pixel { _r: 255, _g: 255, _b: 255 };\n\nconst POWERLINE_TRIANGLE_LTR: char = '\\u{e0b0}';\nconst POWERLINE_ARROW_LTR: char = '\\u{e0b1}';\nconst POWERLINE_TRIANGLE_RTL: char = '\\u{e0b2}';\nconst POWERLINE_ARROW_RTL: char = '\\u{e0b3}';\n\n/// Returns the rasterized glyph if the character is part of the built-in font.\npub fn builtin_glyph(\n    character: char,\n    metrics: &Metrics,\n    offset: &Delta<i8>,\n    glyph_offset: &Delta<i8>,\n) -> Option<RasterizedGlyph> {\n    let mut glyph = match character {\n        // Box drawing characters and block elements.\n        '\\u{2500}'..='\\u{259f}' | '\\u{1fb00}'..='\\u{1fb3b}' => {\n            box_drawing(character, metrics, offset)\n        },\n        // Powerline symbols: '\ue0b0','\ue0b1','\ue0b2','\ue0b3'\n        POWERLINE_TRIANGLE_LTR..=POWERLINE_ARROW_RTL => {\n            powerline_drawing(character, metrics, offset)?\n        },\n        _ => return None,\n    };\n\n    // Since we want to ignore `glyph_offset` for the built-in font, subtract it to compensate its\n    // addition when loading glyphs in the renderer.\n    glyph.left -= glyph_offset.x as i32;\n    glyph.top -= glyph_offset.y as i32;\n\n    Some(glyph)\n}\n\nfn box_drawing(character: char, metrics: &Metrics, offset: &Delta<i8>) -> RasterizedGlyph {\n    // Ensure that width and height is at least one.\n    let height = (metrics.line_height as i32 + offset.y as i32).max(1) as usize;\n    let width = (metrics.average_advance as i32 + offset.x as i32).max(1) as usize;\n    let stroke_size = calculate_stroke_size(width);\n    let heavy_stroke_size = stroke_size * 2;\n\n    // Certain symbols require larger canvas than the cell itself, since for proper contiguous\n    // lines they require drawing on neighbour cells. So treat them specially early on and handle\n    // 'normal' characters later.\n    let mut canvas = match character {\n        // Diagonals: '\u2571', '\u2572', '\u2573'.\n        '\\u{2571}'..='\\u{2573}' => {\n            // Last coordinates.\n            let x_end = width as f32;\n            let mut y_end = height as f32;\n\n            let top = height as i32 + metrics.descent as i32 + stroke_size as i32;\n            let height = height + 2 * stroke_size;\n            let mut canvas = Canvas::new(width, height + 2 * stroke_size);\n\n            // The offset that we should take into account when drawing, since we've enlarged\n            // buffer vertically by twice of that amount.\n            let y_offset = stroke_size as f32;\n            y_end += y_offset;\n\n            let k = y_end / x_end;\n            let f_x = |x: f32, h: f32| -> f32 { -1. * k * x + h + y_offset };\n...\n", "needle_token_start": 602, "needle_token_end": 983, "code_context_ntokens": 16149, "output": ["```rust\nfn on_left_click(&mut self, point: Point) {\n    let side = self.ctx.mouse().cell_side;\n\n    match self.ctx.mouse().click_state {\n        ClickState::Click => {\n            // Don't launch URLs if this click cleared the selection.\n            self.ctx.mouse_mut().block_hint_launcher = !self.ctx.selection_is_empty();\n\n            self.ctx.clear_selection();\n\n            // Start new empty selection.\n            if self.ctx.modifiers().state().control_key() {\n                self.ctx.start_selection(SelectionType::Block, point, side);\n            } else {\n                self.ctx.start_selection(SelectionType::Simple, point, side);\n            }\n        },\n        ClickState::DoubleClick => {\n            self.ctx.mouse_mut().block_hint_launcher = true;\n            self.ctx.start_selection(SelectionType::Semantic, point, side);\n        },\n        ClickState::TripleClick => {\n            self.ctx.mouse_mut().block_hint_launcher = true;\n            self.ctx.start_selection(SelectionType::Lines, point, side);\n        },\n        ClickState::None => (),\n    };\n\n    // Move vi mode cursor to mouse click position.\n    if self.ctx.terminal().mode().contains(TermMode::VI) && !self.ctx.search_active() {\n        self.ctx.terminal_mut().vi_mode_cursor.point = point;\n        self.ctx.mark_dirty();\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "negative_url_parsing_regex_test", "language": "rust", "path": "alacritty/src/config/ui_config.rs", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To verify that a specific pattern does not incorrectly identify strings that resemble URLs but are not valid URLs.\n2. **Input**: A series of strings that are formatted similarly to URLs but contain errors or unconventional formats that should not be recognized as URLs.\n3. **Output**: The test checks if any of these strings are mistakenly identified as URLs. The expected output is that none of these strings match the URL pattern.\n4. **Procedure**: The test iterates through each string, applies a URL detection pattern, and checks if any matches are found. It asserts that there should be no matches for each string, confirming the pattern's accuracy in rejecting invalid URL formats.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "            where\n                M: MapAccess<'a>,\n            {\n                let mut content = Self::Value::default();\n\n                while let Some((key, value)) = map.next_entry::<String, toml::Value>()? {\n                    match key.as_str() {\n                        \"regex\" => match Option::<LazyRegex>::deserialize(value) {\n                            Ok(regex) => content.regex = regex,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: hint's regex: {}\", err\n                                );\n                            },\n                        },\n                        \"hyperlinks\" => match bool::deserialize(value) {\n                            Ok(hyperlink) => content.hyperlinks = hyperlink,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: hint's hyperlinks: {}\", err\n                                );\n                            },\n                        },\n                        \"command\" | \"action\" => (),\n                        key => warn!(target: LOG_TARGET_CONFIG, \"Unrecognized hint field: {key}\"),\n                    }\n                }\n\n                // Require at least one of hyperlinks or regex trigger hint matches.\n                if content.regex.is_none() && !content.hyperlinks {\n                    return Err(M::Error::custom(\n                        \"Config error: At least on of the hint's regex or hint's hyperlinks must \\\n                         be set\",\n                    ));\n                }\n\n                Ok(content)\n            }\n        }\n\n        deserializer.deserialize_any(HintContentVisitor)\n    }\n}\n\n/// Binding for triggering a keyboard hint.\n#[derive(Deserialize, Clone, Debug, PartialEq, Eq)]\n#[serde(deny_unknown_fields)]\npub struct HintBinding {\n    pub key: BindingKey,\n    #[serde(default)]\n    pub mods: ModsWrapper,\n    #[serde(default)]\n    pub mode: ModeWrapper,\n}\n\n/// Hint mouse highlighting.\n#[derive(ConfigDeserialize, Default, Copy, Clone, Debug, PartialEq, Eq)]\npub struct HintMouse {\n    /// Hint mouse highlighting availability.\n    pub enabled: bool,\n\n    /// Required mouse modifiers for hint highlighting.\n    pub mods: ModsWrapper,\n}\n\n/// Lazy regex with interior mutability.\n#[derive(Clone, Debug, PartialEq, Eq)]\npub struct LazyRegex(Rc<RefCell<LazyRegexVariant>>);\n\nimpl LazyRegex {\n    /// Execute a function with the compiled regex DFAs as parameter.\n    pub fn with_compiled<T, F>(&self, f: F) -> Option<T>\n    where\n        F: FnMut(&mut RegexSearch) -> T,\n    {\n        self.0.borrow_mut().compiled().map(f)\n    }\n}\n\nimpl<'de> Deserialize<'de> for LazyRegex {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let regex = LazyRegexVariant::Pattern(String::deserialize(deserializer)?);\n        Ok(Self(Rc::new(RefCell::new(regex))))\n    }\n}\n\n/// Regex which is compiled on demand, to avoid expensive computations at startup.\n#[derive(Clone, Debug)]\npub enum LazyRegexVariant {\n    Compiled(Box<RegexSearch>),\n    Pattern(String),\n    Uncompilable,\n}\n\nimpl LazyRegexVariant {\n    /// Get a reference to the compiled regex.\n    ///\n    /// If the regex is not already compiled, this will compile the DFAs and store them for future\n    /// access.\n    fn compiled(&mut self) -> Option<&mut RegexSearch> {\n        // Check if the regex has already been compiled.\n        let regex = match self {\n            Self::Compiled(regex_search) => return Some(regex_search),\n            Self::Uncompilable => return None,\n            Self::Pattern(regex) => regex,\n        };\n\n        // Compile the regex.\n        let regex_search = match RegexSearch::new(regex) {\n            Ok(regex_search) => regex_search,\n            Err(err) => {\n                error!(\"could not compile hint regex: {err}\");\n                *self = Self::Uncompilable;\n                return None;\n            },\n        };\n        *self = Self::Compiled(Box::new(regex_search));\n\n        // Return a reference to the compiled DFAs.\n        match self {\n            Self::Compiled(dfas) => Some(dfas),\n            _ => unreachable!(),\n        }\n    }\n}\n\nimpl PartialEq for LazyRegexVariant {\n    fn eq(&self, other: &Self) -> bool {\n        match (self, other) {\n            (Self::Pattern(regex), Self::Pattern(other_regex)) => regex == other_regex,\n            _ => false,\n        }\n    }\n}\nimpl Eq for LazyRegexVariant {}\n\n/// Wrapper around f32 that represents a percentage value between 0.0 and 1.0.\n#[derive(SerdeReplace, Deserialize, Clone, Copy, Debug, PartialEq)]\npub struct Percentage(f32);\n\nimpl Default for Percentage {\n    fn default() -> Self {\n        Percentage(1.0)\n    }\n}\n\nimpl Percentage {\n    pub fn new(value: f32) -> Self {\n        Percentage(value.clamp(0., 1.))\n    }\n\n    pub fn as_f32(self) -> f32 {\n        self.0\n    }\n}\n\n#[derive(Deserialize, Debug, Clone, PartialEq, Eq)]\n#[serde(untagged, deny_unknown_fields)]\npub enum Program {\n    Just(String),\n    WithArgs {\n        program: String,\n        #[serde(default)]\n        args: Vec<String>,\n    },\n}\n\nimpl Program {\n    pub fn program(&self) -> &str {\n        match self {\n            Program::Just(program) => program,\n            Program::WithArgs { program, .. } => program,\n        }\n    }\n\n    pub fn args(&self) -> &[String] {\n        match self {\n            Program::Just(_) => &[],\n            Program::WithArgs { args, .. } => args,\n        }\n    }\n}\n\nimpl From<Program> for Shell {\n    fn from(value: Program) -> Self {\n        match value {\n            Program::Just(program) => Shell::new(program, Vec::new()),\n            Program::WithArgs { program, args } => Shell::new(program, args),\n        }\n    }\n}\n\nimpl SerdeReplace for Program {\n    fn replace(&mut self, value: toml::Value) -> Result<(), Box<dyn Error>> {\n        *self = Self::deserialize(value)?;\n\n        Ok(())\n    }\n}\n\npub(crate) struct StringVisitor;\nimpl<'de> serde::de::Visitor<'de> for StringVisitor {\n    type Value = String;\n\n    fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {\n        formatter.write_str(\"a string\")\n    }\n\n    fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>\n    where\n        E: serde::de::Error,\n    {\n        Ok(s.to_lowercase())\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    use alacritty_terminal::term::test::mock_term;\n\n    use crate::display::hint::visible_regex_match_iter;\n\n    #[test]\n    fn positive_url_parsing_regex_test() {\n        for regular_url in [\n            \"ipfs:s0mEhAsh\",\n            \"ipns:an0TherHash1234\",\n            \"magnet:?xt=urn:btih:L0UDHA5H12\",\n            \"mailto:example@example.org\",\n            \"gemini://gemini.example.org/\",\n            \"gopher://gopher.example.org\",\n            \"https://www.example.org\",\n            \"http://example.org\",\n            \"news:some.news.portal\",\n            \"file:///C:/Windows/\",\n            \"file:/home/user/whatever\",\n            \"git://github.com/user/repo.git\",\n            \"ssh:git@github.com:user/repo.git\",\n            \"ftp://ftp.example.org\",\n        ] {\n            let term = mock_term(regular_url);\n            let mut regex = RegexSearch::new(URL_REGEX).unwrap();\n            let matches = visible_regex_match_iter(&term, &mut regex).collect::<Vec<_>>();\n            assert_eq!(\n                matches.len(),\n                1,\n                \"Should have exactly one match url {regular_url}, but instead got: {matches:?}\"\n            )\n        }\n    }\n\n    #[test]\n    fn n\negative_url_parsing_regex_test() {\n        for url_like in [\n            \"http::trace::on_request::log_parameters\",\n            \"http//www.example.org\",\n            \"/user:example.org\",\n            \"mailto: example@example.org\",\n            \"http://<script>alert('xss')</script>\",\n            \"mailto:\",\n        ] {\n            let term = mock_term(url_like);\n            let mut regex = RegexSearch::new(URL_REGEX).unwrap();\n            let matches = visible_regex_match_iter(&term, &mut regex).collect::<Vec<_>>();\n            assert!(\n                matches.is_empty(),\n                \"Should not match url in string {url_like}, but instead got: {matches:?}\"\n            )\n        }\n    }\n}\n\n// Path: alacritty/src/config/font.rs\nuse std::fmt;\n\nuse crossfont::Size as FontSize;\nuse serde::de::{self, Visitor};\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\nuse crate::config::ui_config::Delta;\n\n/// Font config.\n///\n/// Defaults are provided at the level of this struct per platform, but not per\n/// field in this struct. It might be nice in the future to have defaults for\n/// each value independently. Alternatively, maybe erroring when the user\n/// doesn't provide complete config is Ok.\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq, Eq)]\npub struct Font {\n    /// Extra spacing per character.\n    pub offset: Delta<i8>,\n\n    /// Glyph offset within character cell.\n    pub glyph_offset: Delta<i8>,\n\n    #[config(removed = \"set the AppleFontSmoothing user default instead\")]\n    pub use_thin_strokes: bool,\n\n    /// Normal font face.\n    normal: FontDescription,\n\n    /// Bold font face.\n    bold: SecondaryFontDescription,\n\n    /// Italic font face.\n    italic: SecondaryFontDescription,\n\n    /// Bold italic font face.\n    bold_italic: SecondaryFontDescription,\n\n    /// Font size in points.\n    size: Size,\n\n    /// Whether to use the built-in font for box drawing characters.\n    pub builtin_box_drawing: bool,\n}\n\nimpl Font {\n    /// Get a font clone with a size modification.\n    pub fn with_size(self, size: FontSize) -> Font {\n        Font { size: Size(size), ..self }\n    }\n\n    #[inline]\n    pub fn size(&self) -> FontSize {\n        self.size.0\n    }\n\n    /// Get normal font description.\n    pub fn normal(&self) -> &FontDescription {\n        &self.normal\n    }\n\n    /// Get bold font description.\n    pub fn bold(&self) -> FontDescription {\n        self.bold.desc(&self.normal)\n    }\n\n    /// Get italic font description.\n    pub fn italic(&self) -> FontDescription {\n        self.italic.desc(&self.normal)\n    }\n\n    /// Get bold italic font description.\n    pub fn bold_italic(&self) -> FontDescription {\n        self.bold_italic.desc(&self.normal)\n    }\n}\n\nimpl Default for Font {\n    fn default() -> Font {\n        Self {\n            builtin_box_drawing: true,\n            glyph_offset: Default::default(),\n            use_thin_strokes: Default::default(),\n            bold_italic: Default::default(),\n            italic: Default::default(),\n            offset: Default::default(),\n            normal: Default::default(),\n            bold: Default::default(),\n            size: Default::default(),\n        }\n    }\n}\n\n/// Description of the normal font.\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq, Eq)]\npub struct FontDescription {\n    pub family: String,\n    pub style: Option<String>,\n}\n\nimpl Default for FontDescription {\n    fn default() -> FontDescription {\n        FontDescription {\n            #[cfg(not(any(target_os = \"macos\", windows)))]\n            family: \"monospace\".into(),\n            #[cfg(target_os = \"macos\")]\n            family: \"Menlo\".into(),\n            #[cfg(windows)]\n            family: \"Consolas\".into(),\n            style: None,\n        }\n    }\n}\n\n/// Description of the italic and bold font.\n#[derive(ConfigDeserialize, Debug, Default, Clone, PartialEq, Eq)]\npub struct SecondaryFontDescription {\n    family: Option<String>,\n    style: Option<String>,\n}\n\nimpl SecondaryFontDescription {\n    pub fn desc(&self, fallback: &FontDescription) -> FontDescription {\n        FontDescription {\n            family: self.family.clone().unwrap_or_else(|| fallback.family.clone()),\n            style: self.style.clone(),\n        }\n    }\n}\n\n#[derive(SerdeReplace, Debug, Clone, PartialEq, Eq)]\nstruct Size(FontSize);\n\nimpl Default for Size {\n    fn default() -> Self {\n        Self(FontSize::new(11.25))\n    }\n}\n\nimpl<'de> Deserialize<'de> for Size {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct NumVisitor;\n        impl<'v> Visitor<'v> for NumVisitor {\n            type Value = Size;\n\n            fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                f.write_str(\"f64 or i64\")\n            }\n\n            fn visit_f64<E: de::Error>(self, value: f64) -> Result<Self::Value, E> {\n                Ok(Size(FontSize::new(value as f32)))\n            }\n\n            fn visit_i64<E: de::Error>(self, value: i64) -> Result<Self::Value, E> {\n                Ok(Size(FontSize::new(value as f32)))\n            }\n        }\n\n        deserializer.deserialize_any(NumVisitor)\n    }\n}\n\n// Path: alacritty/src/config/serde_utils.rs\n//! Serde helpers.\n\nuse toml::{Table, Value};\n\n/// Merge two serde structures.\n///\n/// This will take all values from `replacement` and use `base` whenever a value isn't present in\n/// `replacement`.\npub fn merge(base: Value, replacement: Value) -> Value {\n    match (base, replacement) {\n        (Value::Array(mut base), Value::Array(mut replacement)) => {\n            base.append(&mut replacement);\n            Value::Array(base)\n        },\n        (Value::Table(base), Value::Table(replacement)) => {\n            Value::Table(merge_tables(base, replacement))\n        },\n        (_, value) => value,\n    }\n}\n\n/// Merge two key/value tables.\nfn merge_tables(mut base: Table, replacement: Table) -> Table {\n    for (key, value) in replacement {\n        let value = match base.remove(&key) {\n            Some(base_value) => merge(base_value, value),\n            None => value,\n        };\n        base.insert(key, value);\n    }\n\n    base\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn merge_primitive() {\n        let base = Value::Table(Table::new());\n        let replacement = Value::Boolean(true);\n        assert_eq!(merge(base, replacement.clone()), replacement);\n\n        let base = Value::Boolean(false);\n        let replacement = Value::Boolean(true);\n        assert_eq!(merge(base, replacement.clone()), replacement);\n\n        let base = Value::Integer(0.into());\n        let replacement = Value::Integer(1.into());\n        assert_eq!(merge(base, replacement.clone()), replacement);\n\n        let base = Value::String(String::new());\n        let replacement = Value::String(String::from(\"test\"));\n        assert_eq!(merge(base, replacement.clone()), replacement);\n\n        let base = Value::Table(Table::new());\n        let replacement = Value::Table(Table::new());\n        assert_eq!(merge(base.clone(), replacement), base);\n    }\n\n    #[test]\n    fn merge_sequence() {\n        let base = Value::Array(vec![Value::Table(Table::new())]);\n        let replacement = Value::Array(vec![Value::Boolean(true)]);\n        let expected = Value::Array(vec![Value::Table(Table::new()), Value::Boolean(true)]);\n        assert_eq!(merge(base, replacement), expected);\n    }\n\n    #[test]\n    fn merge_tables() {\n        let mut base_table = Table::new();\n        base_table.insert(String::from(\"a\"), Value::Boolean(true));\n        base_table.insert(String::from(\"b\"), Value::Boolean(false));\n        let base = Value::Table(base_table);\n\n        let mut replacement_table = Table::new();\n        replacement_table.insert(String::from(\"a\"), Value::Boolean(true));\n        replacement_table.insert(String::from(\"c\"), Value::Boolean(false));\n        let replacement = Value::Table(replacement_table);\n\n        let merged = merge(base, replacement);\n\n        let mut expected_table = Table::new();\n        expected_table.insert(String::from(\"b\"), Value::Boolean(false));\n        expected_table.insert(String::from(\"a\"), Value::Boolean(true));\n        expected_table.insert(String::from(\"c\"), Value::Boolean(false));\n        let expected = Value::Table(expected_table);\n\n        assert_eq!(merged, expected);\n    }\n}\n\n// Path: alacritty/src/config/monitor.rs\nuse std::path::PathBuf;\nuse std::sync::mpsc::{self, RecvTimeoutError};\nuse std::time::{Duration, Instant};\n\nuse log::{debug, error};\nuse notify::{Config, EventKind, RecommendedWatcher, RecursiveMode, Watcher};\nuse winit::event_loop::EventLoopProxy;\n\nuse alacritty_terminal::thread;\n\nuse crate::event::{Event, EventType};\n\nconst DEBOUNCE_DELAY: Duration = Duration::from_millis(10);\n\n/// The fallback for `RecommendedWatcher` polling.\nconst FALLBACK_POLLING_TIMEOUT: Duration = Duration::from_secs(1);\n\npub fn watch(mut paths: Vec<PathBuf>, event_proxy: EventLoopProxy<Event>) {\n    // Don't monitor config if there is no path to watch.\n    if paths.is_empty() {\n        return;\n    }\n\n    // Exclude char devices like `/dev/null`, sockets, and so on, by checking that file type is a\n    // regular file.\n    paths.retain(|path| {\n        // Call `metadata` to resolve symbolic links.\n        path.metadata().map_or(false, |metadata| metadata.file_type().is_file())\n    });\n\n    // Canonicalize paths, keeping the base paths for symlinks.\n    for i in 0..paths.len() {\n        if let Ok(canonical_path) = paths[i].canonicalize() {\n            match paths[i].symlink_metadata() {\n                Ok(metadata) if metadata.file_type().is_symlink() => paths.push(canonical_path),\n                _ => paths[i] = canonical_path,\n            }\n        }\n    }\n\n    // The Duration argument is a debouncing period.\n    let (tx, rx) = mpsc::channel();\n    let mut watcher = match RecommendedWatcher::new(\n        tx,\n        Config::default().with_poll_interval(FALLBACK_POLLING_TIMEOUT),\n    ) {\n        Ok(watcher) => watcher,\n        Err(err) => {\n            error!(\"Unable to watch config file: {}\", err);\n            return;\n        },\n    };\n\n    thread::spawn_named(\"config watcher\", move || {\n        // Get all unique parent directories.\n        let mut parents = paths\n            .iter()\n            .map(|path| {\n                let mut path = path.clone();\n                path.pop();\n                path\n            })\n            .collect::<Vec<PathBuf>>();\n        parents.sort_unstable();\n        parents.dedup();\n\n        // Watch all configuration file directories.\n        for parent in &parents {\n            if let Err(err) = watcher.watch(parent, RecursiveMode::NonRecursive) {\n                debug!(\"Unable to watch config directory {:?}: {}\", parent, err);\n            }\n        }\n\n        // The current debouncing time.\n        let mut debouncing_deadline: Option<Instant> = None;\n\n        // The events accumulated during the debounce period.\n        let mut received_events = Vec::new();\n\n        loop {\n            // We use `recv_timeout` to debounce the events coming from the watcher and reduce\n            // the amount of config reloads.\n            let event = match debouncing_deadline.as_ref() {\n                Some(debouncing_deadline) => {\n                    rx.recv_timeout(debouncing_deadline.saturating_duration_since(Instant::now()))\n                },\n                None => {\n                    let event = rx.recv().map_err(Into::into);\n                    // Set the debouncing deadline after receiving the event.\n                    debouncing_deadline = Some(Instant::now() + DEBOUNCE_DELAY);\n                    event\n                },\n            };\n\n            match event {\n                Ok(Ok(event)) => match event.kind {\n                    EventKind::Any\n                    | EventKind::Create(_)\n                    | EventKind::Modify(_)\n                    | EventKind::Other => {\n                        received_events.push(event);\n                    },\n                    _ => (),\n                },\n                Err(RecvTimeoutError::Timeout) => {\n                    // Go back to polling the events.\n                    debouncing_deadline = None;\n\n                    if received_events\n                        .drain(..)\n                        .flat_map(|event| event.paths.into_iter())\n                        .any(|path| paths.contains(&path))\n                    {\n                        // Always reload the primary configuration file.\n                        let event = Event::new(EventType::ConfigReload(paths[0].clone()), None);\n                        let _ = event_proxy.send_event(event);\n                    }\n                },\n                Ok(Err(err)) => {\n                    debug!(\"Config watcher errors: {:?}\", err);\n                },\n                Err(err) => {\n                    debug!(\"Config watcher channel dropped unexpectedly: {}\", err);\n                    break;\n                },\n            };\n        }\n    });\n}\n\n// Path: alacritty/src/config/mod.rs\nuse std::fmt::{self, Display, Formatter};\nuse std::path::{Path, PathBuf};\nuse std::result::Result as StdResult;\nuse std::{env, fs, io};\n\nuse log::{debug, error, info, warn};\nuse serde::Deserialize;\nuse serde_yaml::Error as YamlError;\nuse toml::de::Error as TomlError;\nuse toml::ser::Error as TomlSeError;\nuse toml::{Table, Value};\n\npub mod bell;\npub mod color;\npub mod cursor;\npub mod debug;\npub mod font;\npub mod monitor;\npub mod scrolling;\npub mod selection;\npub mod serde_utils;\npub mod terminal;\npub mod ui_config;\npub mod window;\n\nmod bindings;\nmod mouse;\n\nuse crate::cli::Options;\n#[cfg(test)]\npub use crate::config::bindings::Binding;\npub use crate::config::bindings::{\n    Action, BindingKey, BindingMode, MouseAction, SearchAction, ViAction,\n};\npub use crate::config::ui_config::UiConfig;\nuse crate::logging::LOG_TARGET_CONFIG;\n\n/// Maximum number of depth for the configuration file imports.\npub const IMPORT_RECURSION_LIMIT: usize = 5;\n\n/// Result from config loading.\npub type Result<T> = std::result::Result<T, Error>;\n\n/// Errors occurring during config loading.\n#[derive(Debug)]\npub enum Error {\n    /// Config file not found.\n    NotFound,\n\n    /// Couldn't read $HOME environment variable.\n    ReadingEnvHome(env::VarError),\n\n    /// io error reading file.\n    Io(io::Error),\n\n    /// Invalid toml.\n    Toml(TomlError),\n\n    /// Failed toml serialization.\n    TomlSe(TomlSeError),\n\n    /// Invalid yaml.\n    Yaml(YamlError),\n}\n\nimpl std::error::Error for Error {\n    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {\n        match self {\n            Error::NotFound => None,\n            Error::ReadingEnvHome(err) => err.source(),\n            Error::Io(err) => err.source(),\n            Error::Toml(err) => err.source(),\n            Error::TomlSe(err) => err.source(),\n            Error::Yaml(err) => err.source(),\n        }\n    }\n}\n\nimpl Display for Error {\n    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {\n        match self {\n            Error::NotFound => write!(f, \"Unable to locate config file\"),\n            Error::ReadingEnvHome(err) => {\n                write!(f, \"Unable to read $HOME environment variable: {}\", err)\n            },\n            Error::Io(err) => write!(f, \"Error reading config file: {}\", err),\n            Error::Toml(err) => write!(f, \"Config error: {}\", err),\n            Error::TomlSe(err) => write!(f, \"Yaml conversion error: {}\", err),\n            Error::Yaml(err) => write!(f, \"Config error: {}\", err),\n        }\n    }\n}\n\nimpl From<env::VarError> for Error {\n    fn from(val: env::VarError) -> Self {\n        Error::ReadingEnvHome(val)\n    }\n}\n\nimpl From<io::Error> for Error {\n    fn from(val: io::Error) -> Self {\n        if val.kind() == io::ErrorKind::NotFound {\n            Error::NotFound\n        } else {\n            Error::Io(val)\n        }\n    }\n}\n\nimpl From<TomlError> for Error {\n    fn from(val: TomlError) -> Self {\n        Error::Toml(val)\n    }\n}\n\nimpl From<TomlSeError> for Error {\n    fn from(val: TomlSeError) -> Self {\n        Error::TomlSe(val)\n    }\n}\n\nimpl From<YamlError> for Error {\n    fn from(val: YamlError) -> Self {\n        Error::Yaml(val)\n    }\n}\n\n/// Load the configuration file.\npub fn load(options: &mut Options) -> UiConfig {\n    let config_path = options\n        .config_file\n        .clone()\n        .or_else(|| installed_config(\"toml\"))\n        .or_else(|| installed_config(\"yml\"));\n\n    // Load the config using the following fallback behavior:\n    //  - Config path + CLI overrides\n    //  - CLI overrides\n    //  - Default\n    let mut config = config_path\n        .as_ref()\n        .and_then(|config_path| load_from(config_path).ok())\n        .unwrap_or_else(|| {\n            let mut config = UiConfig::default();\n            match config_path {\n                Some(config_path) => config.config_paths.push(config_path),\n                None => info!(target: LOG_TARGET_CONFIG, \"No config file found; using default\"),\n            }\n            config\n        });\n\n    after_loading(&mut config, options);\n\n    config\n}\n\n/// Attempt to reload the configuration file.\npub fn reload(config_path: &Path, options: &mut Options) -> Result<UiConfig> {\n    debug!(\"Reloading configuration file: {:?}\", config_path);\n\n    // Load config, propagating errors.\n    let mut config = load_from(config_path)?;\n\n    after_loading(&mut config, options);\n\n    Ok(config)\n}\n\n/// Modifications after the `UiConfig` object is created.\nfn after_loading(config: &mut UiConfig, options: &mut Options) {\n    // Override config with CLI options.\n    options.override_config(config);\n\n    // Create key bindings for regex hints.\n    config.generate_hint_bindings();\n}\n\n/// Load configuration file and log errors.\nfn load_from(path: &Path) -> Result<UiConfig> {\n    match read_config(path) {\n        Ok(config) => Ok(config),\n        Err(err) => {\n            error!(target: LOG_TARGET_CONFIG, \"Unable to load config {:?}: {}\", path, err);\n            Err(err)\n        },\n    }\n}\n\n/// Deserialize configuration file from path.\nfn read_config(path: &Path) -> Result<UiConfig> {\n    let mut config_paths = Vec::new();\n    let config_value = parse_config(path, &mut config_paths, IMPORT_RECURSION_LIMIT)?;\n\n    // Deserialize to concrete type.\n    let mut config = UiConfig::deserialize(config_value)?;\n    config.config_paths = config_paths;\n\n    Ok(config)\n}\n\n/// Deserialize all configuration files as generic Value.\nfn parse_config(\n    path: &Path,\n    config_paths: &mut Vec<PathBuf>,\n    recursion_limit: usize,\n) -> Result<Value> {\n    config_paths.push(path.to_owned());\n\n    // Deserialize the configuration file.\n    let config = deserialize_config(path, false)?;\n\n    // Merge config with imports.\n    let imports = load_imports(&config, config_paths, recursion_limit);\n    Ok(serde_utils::merge(imports, config))\n}\n\n/// Deserialize a configuration file.\npub fn deserialize_config(path: &Path, warn_pruned: bool) -> Result<Value> {\n    let mut contents = fs::read_to_string(path)?;\n\n    // Remove UTF-8 BOM.\n    if contents.starts_with('\\u{FEFF}') {\n        contents = contents.split_off(3);\n    }\n\n    // Convert YAML to TOML as a transitionary fallback mechanism.\n    let extension = path.extension().unwrap_or_default();\n    if (extension == \"yaml\" || extension == \"yml\") && !contents.trim().is_empty() {\n        warn!(\n            \"YAML config {path:?} is deprecated, please migrate to TOML using `alacritty migrate`\"\n        );\n\n        let mut value: serde_yaml::Value = serde_yaml::from_str(&contents)?;\n        prune_yaml_nulls(&mut value, warn_pruned);\n        contents = toml::to_string(&value)?;\n    }\n\n    // Load configuration file as Value.\n    let config: Value = toml::from_str(&contents)?;\n\n    Ok(config)\n}\n\n/// Load all referenced configuration files.\nfn load_imports(config: &Value, config_paths: &mut Vec<PathBuf>, recursion_limit: usize) -> Value {\n    // Get paths for all imports.\n    let import_paths = match imports(config, recursion_limit) {\n        Ok(import_paths) => import_paths,\n        Err(err) => {\n            error!(target: LOG_TARGET_CONFIG, \"{err}\");\n            return Value::Table(Table::new());\n        },\n    };\n\n    // Parse configs for all imports recursively.\n    let mut merged = Value::Table(Table::new());\n    for import_path in import_paths {\n        let path = match import_path {\n            Ok(path) => path,\n            Err(err) => {\n                error!(target: LOG_TARGET_CONFIG, \"{err}\");\n                continue;\n            },\n        };\n\n        if !path.exists() {\n            info!(target: LOG_TARGET_CONFIG, \"Config import not found:\\n  {:?}\", path.display());\n            continue;\n        }\n\n        match parse_config(&path, config_paths, recursion_limit - 1) {\n            Ok(config) => merged = serde_utils::merge(merged, config),\n            Err(err) => {\n                error!(target: LOG_TARGET_CONFIG, \"Unable to import config {:?}: {}\", path, err)\n            },\n        }\n    }\n\n    merged\n}\n\n// TODO: Merge back with `load_imports` once `alacritty migrate` is dropped.\n//\n/// Get all import paths for a configuration.\npub fn imports(\n    config: &Value,\n    recursion_limit: usize,\n) -> StdResult<Vec<StdResult<PathBuf, String>>, String> {\n    let imports = match config.get(\"import\") {\n        Some(Value::Array(imports)) => imports,\n        Some(_) => return Err(\"Invalid import type: expected a sequence\".into()),\n        None => return Ok(Vec::new()),\n    };\n\n    // Limit recursion to prevent infinite loops.\n    if !imports.is_empty() && recursion_limit == 0 {\n        return Err(\"Exceeded maximum configuration import depth\".into());\n    }\n\n    let mut import_paths = Vec::new();\n\n    for import in imports {\n        let mut path = match import {\n            Value::String(path) => PathBuf::from(path),\n            _ => {\n                import_paths.push(Err(\"Invalid import element type: expected path string\".into()));\n                continue;\n            },\n        };\n\n        // Resolve paths relative to user's home directory.\n        if let (Ok(stripped), Some(home_dir)) = (path.strip_prefix(\"~/\"), home::home_dir()) {\n            path = home_dir.join(stripped);\n        }\n\n        import_paths.push(Ok(path));\n    }\n\n    Ok(import_paths)\n}\n\n/// Prune the nulls from the YAML to ensure TOML compatibility.\nfn prune_yaml_nulls(value: &mut serde_yaml::Value, warn_pruned: bool) {\n    fn walk(value: &mut serde_yaml::Value, warn_pruned: bool) -> bool {\n        match value {\n            serde_yaml::Value::Sequence(sequence) => {\n                sequence.retain_mut(|value| !walk(value, warn_pruned));\n                sequence.is_empty()\n            },\n            serde_yaml::Value::Mapping(mapping) => {\n                mapping.retain(|key, value| {\n                    let retain = !walk(value, warn_pruned);\n                    if let Some(key_name) = key.as_str().filter(|_| !retain && warn_pruned) {\n                        eprintln!(\"Removing null key \\\"{key_name}\\\" from the end config\");\n                    }\n                    retain\n                });\n                mapping.is_empty()\n            },\n            serde_yaml::Value::Null => true,\n            _ => false,\n        }\n    }\n\n    if walk(value, warn_pruned) {\n        // When the value itself is null return the mapping.\n        *value = serde_yaml::Value::Mapping(Default::default());\n    }\n}\n\n/// Get the location of the first found default config file paths\n/// according to the following order:\n///\n/// 1. $XDG_CONFIG_HOME/alacritty/alacritty.toml\n/// 2. $XDG_CONFIG_HOME/alacritty.toml\n/// 3. $HOME/.config/alacritty/alacritty.toml\n/// 4. $HOME/.alacritty.toml\n#[cfg(not(windows))]\npub fn installed_config(suffix: &str) -> Option<PathBuf> {\n    let file_name = format!(\"alacritty.{suffix}\");\n\n    // Try using XDG location by default.\n    xdg::BaseDirectories::with_prefix(\"alacritty\")\n        .ok()\n        .and_then(|xdg| xdg.find_config_file(&file_name))\n        .or_else(|| {\n            xdg::BaseDirectories::new()\n                .ok()\n                .and_then(|fallback| fallback.find_config_file(&file_name))\n        })\n        .or_else(|| {\n            if let Ok(home) = env::var(\"HOME\") {\n                // Fallback path: $HOME/.config/alacritty/alacritty.toml.\n                let fallback = PathBuf::from(&home).join(\".config/alacritty\").join(&file_name);\n                if fallback.exists() {\n                    return Some(fallback);\n                }\n                // Fallback path: $HOME/.alacritty.toml.\n                let hidden_name = format!(\".{file_name}\");\n                let fallback = PathBuf::from(&home).join(hidden_name);\n                if fallback.exists() {\n                    return Some(fallback);\n                }\n            }\n            None\n        })\n}\n\n#[cfg(windows)]\npub fn installed_config(suffix: &str) -> Option<PathBuf> {\n    let file_name = format!(\"alacritty.{suffix}\");\n    dirs::config_dir().map(|path| path.join(\"alacritty\").join(file_name)).filter(|new| new.exists())\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn empty_config() {\n        toml::from_str::<UiConfig>(\"\").unwrap();\n    }\n\n    fn yaml_to_toml(contents: &str) -> String {\n        let mut value: serde_yaml::Value = serde_yaml::from_str(contents).unwrap();\n        prune_yaml_nulls(&mut value, false);\n        toml::to_string(&value).unwrap()\n    }\n\n    #[test]\n    fn yaml_with_nulls() {\n        let contents = r#\"\n        window:\n            blinking: Always\n            cursor:\n            not_blinking: Always\n            some_array:\n              - { window: }\n              - { window: \"Hello\" }\n\n        \"#;\n        let toml = yaml_to_toml(contents);\n        assert_eq!(\n            toml.trim(),\n            r#\"[window]\nblinking = \"Always\"\nnot_blinking = \"Always\"\n\n[[window.some_array]]\nwindow = \"Hello\"\"#\n        );\n    }\n\n    #[test]\n    fn empty_yaml_to_toml() {\n        let contents = r#\"\n\n        \"#;\n        let toml = yaml_to_toml(contents);\n        assert!(toml.is_empty());\n    }\n}\n\n// Path: alacritty/src/migrate.rs\n//! Configuration file migration.\n\nuse std::fs;\nuse std::path::Path;\n\nuse toml::map::Entry;\nuse toml::{Table, Value};\n\nuse crate::cli::MigrateOptions;\nuse crate::config;\n\n/// Handle migration.\npub fn migrate(options: MigrateOptions) {\n    // Find configuration file path.\n    let config_path = options\n        .config_file\n        .clone()\n        .or_else(|| config::installed_config(\"toml\"))\n        .or_else(|| config::installed_config(\"yml\"));\n\n    // Abort if system has no installed configuration.\n    let config_path = match config_path {\n        Some(config_path) => config_path,\n        None => {\n            eprintln!(\"No configuration file found\");\n            std::process::exit(1);\n        },\n    };\n\n    // If we're doing a wet run, perform a dry run first for safety.\n    if !options.dry_run {\n        #[allow(clippy::redundant_clone)]\n        let mut options = options.clone();\n        options.silent = true;\n        options.dry_run = true;\n        if let Err(err) = migrate_config(&options, &config_path, config::IMPORT_RECURSION_LIMIT) {\n            eprintln!(\"Configuration file migration failed:\");\n            eprintln!(\"    {config_path:?}: {err}\");\n            std::process::exit(1);\n        }\n    }\n\n    // Migrate the root config.\n    match migrate_config(&options, &config_path, config::IMPORT_RECURSION_LIMIT) {\n        Ok(new_path) => {\n            if !options.silent {\n                println!(\"Successfully migrated {config_path:?} to {new_path:?}\");\n            }\n        },\n        Err(err) => {\n            eprintln!(\"Configuration file migration failed:\");\n            eprintln!(\"    {config_path:?}: {err}\");\n            std::process::exit(1);\n        },\n    }\n}\n\n/// Migrate a specific configuration file.\nfn migrate_config(\n    options: &MigrateOptions,\n    path: &Path,\n    recursion_limit: usize,\n) -> Result<String, String> {\n    // Ensure configuration file has an extension.\n    let path_str = path.to_string_lossy();\n    let (prefix, suffix) = match path_str.rsplit_once('.') {\n        Some((prefix, suffix)) => (prefix, suffix),\n        None => return Err(\"missing file extension\".to_string()),\n    };\n\n    // Abort if config is already toml.\n    if suffix == \"toml\" {\n        return Err(\"already in TOML format\".to_string());\n    }\n\n    // Try to parse the configuration file.\n    let mut config = match config::deserialize_config(path, !options.dry_run) {\n        Ok(config) => config,\n        Err(err) => return Err(format!(\"parsing error: {err}\")),\n    };\n\n    // Migrate config imports.\n    if !options.skip_imports {\n        migrate_imports(options, &mut config, recursion_limit)?;\n    }\n\n    // Migrate deprecated field names to their new location.\n    if !options.skip_renames {\n        migrate_renames(&mut config)?;\n    }\n\n    // Convert to TOML format.\n    let toml = toml::to_string(&config).map_err(|err| format!(\"conversion error: {err}\"))?;\n    let new_path = format!(\"{prefix}.toml\");\n\n    if options.dry_run && !options.silent {\n        // Output new content to STDOUT.\n        println!(\n            \"\\nv-----Start TOML for {path:?}-----v\\n\\n{toml}\\n^-----End TOML for {path:?}-----^\\n\"\n        );\n    } else if !options.dry_run {\n        // Write the new toml configuration.\n        fs::write(&new_path, toml).map_err(|err| format!(\"filesystem error: {err}\"))?;\n    }\n\n    Ok(new_path)\n}\n\n/// Migrate the imports of a config.\nfn migrate_imports(\n    options: &MigrateOptions,\n    config: &mut Value,\n    recursion_limit: usize,\n) -> Result<(), String> {\n    let imports = match config::imports(config, recursion_limit) {\n        Ok(imports) => imports,\n        Err(err) => return Err(format!(\"import error: {err}\")),\n    };\n\n    // Migrate the individual imports.\n    let mut new_imports = Vec::new();\n    for import in imports {\n        let import = match import {\n            Ok(import) => import,\n            Err(err) => return Err(format!(\"import error: {err}\")),\n        };\n\n        // Keep yaml import if path does not exist.\n        if !import.exists() {\n            if options.dry_run {\n                eprintln!(\"Keeping yaml config for nonexistent import: {import:?}\");\n            }\n            new_imports.push(Value::String(import.to_string_lossy().into()));\n            continue;\n        }\n\n        let new_path = migrate_config(options, &import, recursion_limit - 1)?;\n\n        // Print new import path.\n        if options.dry_run {\n            println!(\"Successfully migrated import {import:?} to {new_path:?}\");\n        }\n\n        new_imports.push(Value::String(new_path));\n    }\n\n    // Update the imports field.\n    if let Some(import) = config.get_mut(\"import\") {\n        *import = Value::Array(new_imports);\n    }\n\n    Ok(())\n}\n\n/// Migrate deprecated fields.\nfn migrate_renames(config: &mut Value) -> Result<(), String> {\n    let config_table = match config.as_table_mut() {\n        Some(config_table) => config_table,\n        None => return Ok(()),\n    };\n\n    // draw_bold_text_with_bright_colors -> colors.draw_bold_text_with_bright_colors\n    move_value(config_table, &[\"draw_bold_text_with_bright_colors\"], &[\n        \"colors\",\n        \"draw_bold_text_with_bright_colors\",\n    ])?;\n\n    // key_bindings -> keyboard.bindings\n    move_value(config_table, &[\"key_bindings\"], &[\"keyboard\", \"bindings\"])?;\n\n    // mouse_bindings -> mouse.bindings\n    move_value(config_table, &[\"mouse_bindings\"], &[\"mouse\", \"bindings\"])?;\n\n    Ok(())\n}\n\n/// Move a toml value from one map to another.\nfn move_value(config_table: &mut Table, origin: &[&str], target: &[&str]) -> Result<(), String> {\n    if let Some(value) = remove_node(config_table, origin)? {\n        if !insert_node_if_empty(config_table, target, value)? {\n            return Err(format!(\n                \"conflict: both `{}` and `{}` are set\",\n                origin.join(\".\"),\n                target.join(\".\")\n            ));\n        }\n    }\n\n    Ok(())\n}\n\n/// Remove a node from a tree of tables.\nfn remove_node(table: &mut Table, path: &[&str]) -> Result<Option<Value>, String> {\n    if path.len() == 1 {\n        Ok(table.remove(path[0]))\n    } else {\n        let next_table_value = match table.get_mut(path[0]) {\n            Some(next_table_value) => next_table_value,\n            None => return Ok(None),\n        };\n\n        let next_table = match next_table_value.as_table_mut() {\n            Some(next_table) => next_table,\n            None => return Err(format!(\"invalid `{}` table\", path[0])),\n        };\n\n        remove_node(next_table, &path[1..])\n    }\n}\n\n/// Try to insert a node into a tree of tables.\n///\n/// Returns `false` if the node already exists.\nfn insert_node_if_empty(table: &mut Table, path: &[&str], node: Value) -> Result<bool, String> {\n    if path.len() == 1 {\n        match table.entry(path[0]) {\n            Entry::Vacant(vacant_entry) => {\n                vacant_entry.insert(node);\n                Ok(true)\n            },\n            Entry::Occupied(_) => Ok(false),\n        }\n    } else {\n        let next_table_value = table.entry(path[0]).or_insert_with(|| Value::Table(Table::new()));\n\n        let next_table = match next_table_value.as_table_mut() {\n            Some(next_table) => next_table,\n            None => return Err(format!(\"invalid `{}` table\", path[0])),\n        };\n\n        insert_node_if_empty(next_table, &path[1..], node)\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn move_values() {\n        let input = r#\"\nroot_value = 3\n\n[table]\ntable_value = 5\n\n[preexisting]\nnot_moved = 9\n        \"#;\n\n        let mut value: Value = toml::from_str(input).unwrap();\n        let table = value.as_table_mut().unwrap();\n\n        move_value(table, &[\"root_value\"], &[\"new_table\", \"root_value\"]).unwrap();\n        move_value(table, &[\"table\", \"table_value\"], &[\"preexisting\", \"subtable\", \"new_name\"])\n            .unwrap();\n\n        let output = toml::to_string(table).unwrap();\n\n        assert_eq!(\n            output,\n            \"[new_table]\\nroot_value = 3\\n\\n[preexisting]\\nnot_moved = \\\n             9\\n\\n[preexisting.subtable]\\nnew_name = 5\\n\\n[table]\\n\"\n        );\n    }\n}\n\n// Path: alacritty/src/ipc.rs\n//! Alacritty socket IPC.\n\nuse std::ffi::OsStr;\nuse std::io::{BufRead, BufReader, Error as IoError, ErrorKind, Result as IoResult, Write};\nuse std::os::unix::net::{UnixListener, UnixStream};\nuse std::path::PathBuf;\nuse std::{env, fs, process};\n\nuse log::warn;\nuse winit::event_loop::EventLoopProxy;\nuse winit::window::WindowId;\n\nuse alacritty_terminal::thread;\n\nuse crate::cli::{Options, SocketMessage};\nuse crate::event::{Event, EventType};\n\n/// Environment variable name for the IPC socket path.\nconst ALACRITTY_SOCKET_ENV: &str = \"ALACRITTY_SOCKET\";\n\n/// Create an IPC socket.\npub fn spawn_ipc_socket(options: &Options, event_proxy: EventLoopProxy<Event>) -> Option<PathBuf> {\n    // Create the IPC socket and export its path as env variable if necessary.\n    let socket_path = options.socket.clone().unwrap_or_else(|| {\n        let mut path = socket_dir();\n        path.push(format!(\"{}-{}.sock\", socket_prefix(), process::id()));\n        path\n    });\n    env::set_var(ALACRITTY_SOCKET_ENV, socket_path.as_os_str());\n\n    let listener = match UnixListener::bind(&socket_path) {\n        Ok(listener) => listener,\n        Err(err) => {\n            warn!(\"Unable to create socket: {:?}\", err);\n            return None;\n        },\n    };\n\n    // Spawn a thread to listen on the IPC socket.\n    thread::spawn_named(\"socket listener\", move || {\n        let mut data = String::new();\n        for stream in listener.incoming().filter_map(Result::ok) {\n            data.clear();\n            let mut stream = BufReader::new(stream);\n\n            match stream.read_line(&mut data) {\n                Ok(0) | Err(_) => continue,\n                Ok(_) => (),\n            };\n\n            // Read pending events on socket.\n            let message: SocketMessage = match serde_json::from_str(&data) {\n                Ok(message) => message,\n                Err(err) => {\n                    warn!(\"Failed to convert data from socket: {}\", err);\n                    continue;\n                },\n            };\n\n            // Handle IPC events.\n            match message {\n                SocketMessage::CreateWindow(options) => {\n                    let event = Event::new(EventType::CreateWindow(options), None);\n                    let _ = event_proxy.send_event(event);\n                },\n                SocketMessage::Config(ipc_config) => {\n                    let window_id = ipc_config\n                        .window_id\n                        .and_then(|id| u64::try_from(id).ok())\n                        .map(WindowId::from);\n                    let event = Event::new(EventType::IpcConfig(ipc_config), window_id);\n                    let _ = event_proxy.send_event(event);\n                },\n            }\n        }\n    });\n\n    Some(socket_path)\n}\n\n/// Send a message to the active Alacritty socket.\npub fn send_message(socket: Option<PathBuf>, message: SocketMessage) -> IoResult<()> {\n    let mut socket = find_socket(socket)?;\n\n    let message = serde_json::to_string(&message)?;\n    socket.write_all(message[..].as_bytes())?;\n    let _ = socket.flush();\n\n    Ok(())\n}\n\n/// Directory for the IPC socket file.\n#[cfg(not(target_os = \"macos\"))]\nfn socket_dir() -> PathBuf {\n    xdg::BaseDirectories::with_prefix(\"alacritty\")\n        .ok()\n        .and_then(|xdg| xdg.get_runtime_directory().map(ToOwned::to_owned).ok())\n        .and_then(|path| fs::create_dir_all(&path).map(|_| path).ok())\n        .unwrap_or_else(env::temp_dir)\n}\n\n/// Directory for the IPC socket file.\n#[cfg(target_os = \"macos\")]\nfn socket_dir() -> PathBuf {\n    env::temp_dir()\n}\n\n/// Find the IPC socket path.\nfn find_socket(socket_path: Option<PathBuf>) -> IoResult<UnixStream> {\n    // Handle --socket CLI override.\n    if let Some(socket_path) = socket_path {\n        // Ensure we inform the user about an invalid path.\n        return UnixStream::connect(&socket_path).map_err(|err| {\n            let message = format!(\"invalid socket path {:?}\", socket_path);\n            IoError::new(err.kind(), message)\n        });\n    }\n\n    // Handle environment variable.\n    if let Ok(path) = env::var(ALACRITTY_SOCKET_ENV) {\n        let socket_path = PathBuf::from(path);\n        if let Ok(socket) = UnixStream::connect(socket_path) {\n            return Ok(socket);\n        }\n    }\n\n    // Search for sockets files.\n    for entry in fs::read_dir(socket_dir())?.filter_map(|entry| entry.ok()) {\n        let path = entry.path();\n\n        // Skip files that aren't Alacritty sockets.\n        let socket_prefix = socket_prefix();\n        if path\n            .file_name()\n            .and_then(OsStr::to_str)\n            .filter(|file| file.starts_with(&socket_prefix) && file.ends_with(\".sock\"))\n            .is_none()\n        {\n            continue;\n        }\n\n        // Attempt to connect to the socket.\n        match UnixStream::connect(&path) {\n            Ok(socket) => return Ok(socket),\n            // Delete orphan sockets.\n            Err(error) if error.kind() == ErrorKind::ConnectionRefused => {\n                let _ = fs::remove_file(&path);\n            },\n            // Ignore other errors like permission issues.\n            Err(_) => (),\n        }\n    }\n\n    Err(IoError::new(ErrorKind::NotFound, \"no socket found\"))\n}\n\n/// File prefix matching all available sockets.\n///\n/// This prefix will include display server information to allow for environments with multiple\n/// display servers running for the same user.\n#[cfg(not(target_os = \"macos\"))]\nfn socket_prefix() -> String {\n    let display = env::var(\"WAYLAND_DISPLAY\").or_else(|_| env::var(\"DISPLAY\")).unwrap_or_default();\n    format!(\"Alacritty-{}\", display.replace('/', \"-\"))\n}\n\n/// File prefix matching all available sockets.\n#[cfg(target_os = \"macos\")]\nfn socket_prefix() -> String {\n    String::from(\"Alacritty\")\n}\n\n// Path: alacritty/src/logging.rs\n//! Logging for Alacritty.\n//!\n//! The main executable is supposed to call `initialize()` exactly once during\n//! startup. All logging messages are written to stdout, given that their\n//! log-level is sufficient for the level configured in `cli::Options`.\n\nuse std::fs::{File, OpenOptions};\nuse std::io::{self, LineWriter, Stdout, Write};\nuse std::path::PathBuf;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::{Arc, Mutex, OnceLock};\nuse std::time::Instant;\nuse std::{env, process};\n\nuse log::{Level, LevelFilter};\nuse winit::event_loop::EventLoopProxy;\n\nuse crate::cli::Options;\nuse crate::event::{Event, EventType};\nuse crate::message_bar::{Message, MessageType};\n\n/// Logging target for IPC config error messages.\npub const LOG_TARGET_IPC_CONFIG: &str = \"alacritty_log_window_config\";\n\n/// Name for the environment variable containing the log file's path.\nconst ALACRITTY_LOG_ENV: &str = \"ALACRITTY_LOG\";\n\n/// Logging target for config error messages.\npub const LOG_TARGET_CONFIG: &str = \"alacritty_config_derive\";\n\n/// Name for the environment variable containing extra logging targets.\n///\n/// The targets are semicolon separated.\nconst ALACRITTY_EXTRA_LOG_TARGETS_ENV: &str = \"ALACRITTY_EXTRA_LOG_TARGETS\";\n\n/// User configurable extra log targets to include.\nfn extra_log_targets() -> &'static [String] {\n    static EXTRA_LOG_TARGETS: OnceLock<Vec<String>> = OnceLock::new();\n\n    EXTRA_LOG_TARGETS.get_or_init(|| {\n        env::var(ALACRITTY_EXTRA_LOG_TARGETS_ENV)\n            .map_or(Vec::new(), |targets| targets.split(';').map(ToString::to_string).collect())\n    })\n}\n\n/// List of targets which will be logged by Alacritty.\nconst ALLOWED_TARGETS: &[&str] = &[\n    LOG_TARGET_IPC_CONFIG,\n    LOG_TARGET_CONFIG,\n    \"alacritty_config_derive\",\n    \"alacritty_terminal\",\n    \"alacritty\",\n    \"crossfont\",\n];\n\n/// Initialize the logger to its defaults.\npub fn initialize(\n    options: &Options,\n    event_proxy: EventLoopProxy<Event>,\n) -> Result<Option<PathBuf>, log::SetLoggerError> {\n    log::set_max_level(options.log_level());\n\n    let logger = Logger::new(event_proxy);\n    let path = logger.file_path();\n    log::set_boxed_logger(Box::new(logger))?;\n\n    Ok(path)\n}\n\npub struct Logger {\n    logfile: Mutex<OnDemandLogFile>,\n    stdout: Mutex<LineWriter<Stdout>>,\n    event_proxy: Mutex<EventLoopProxy<Event>>,\n    start: Instant,\n}\n\nimpl Logger {\n    fn new(event_proxy: EventLoopProxy<Event>) -> Self {\n        let logfile = Mutex::new(OnDemandLogFile::new());\n        let stdout = Mutex::new(LineWriter::new(io::stdout()));\n\n        Logger { logfile, stdout, event_proxy: Mutex::new(event_proxy), start: Instant::now() }\n    }\n\n    fn file_path(&self) -> Option<PathBuf> {\n        if let Ok(logfile) = self.logfile.lock() {\n            Some(logfile.path().clone())\n        } else {\n            None\n        }\n    }\n\n    /// Log a record to the message bar.\n    fn message_bar_log(&self, record: &log::Record<'_>, logfile_path: &str) {\n        let message_type = match record.level() {\n            Level::Error => MessageType::Error,\n            Level::Warn => MessageType::Warning,\n            _ => return,\n        };\n\n        let event_proxy = match self.event_proxy.lock() {\n            Ok(event_proxy) => event_proxy,\n            Err(_) => return,\n        };\n\n        #[cfg(not(windows))]\n        let env_var = format!(\"${}\", ALACRITTY_LOG_ENV);\n        #[cfg(windows)]\n        let env_var = format!(\"%{}%\", ALACRITTY_LOG_ENV);\n\n        let message = format!(\n            \"[{}] See log at {} ({}):\\n{}\",\n            record.level(),\n            logfile_path,\n            env_var,\n            record.args(),\n        );\n\n        let mut message = Message::new(message, message_type);\n        message.set_target(record.target().to_owned());\n\n        let _ = event_proxy.send_event(Event::new(EventType::Message(message), None));\n    }\n}\n\nimpl log::Log for Logger {\n    fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {\n        metadata.level() <= log::max_level()\n    }\n\n    fn log(&self, record: &log::Record<'_>) {\n        // Get target crate.\n        let index = record.target().find(':').unwrap_or_else(|| record.target().len());\n        let target = &record.target()[..index];\n\n        // Only log our own crates, except when logging at Level::Trace.\n        if !self.enabled(record.metadata()) || !is_allowed_target(record.level(), target) {\n            return;\n        }\n\n        // Create log message for the given `record` and `target`.\n        let message = create_log_message(record, target, self.start);\n\n        if let Ok(mut logfile) = self.logfile.lock() {\n            // Write to logfile.\n            let _ = logfile.write_all(message.as_ref());\n\n            // Log relevant entries to message bar.\n            self.message_bar_log(record, &logfile.path.to_string_lossy());\n        }\n\n        // Write to stdout.\n...\n", "needle_token_start": 2325, "needle_token_end": 2526, "code_context_ntokens": 16120, "output": ["```rust\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    use alacritty_terminal::term::test::mock_term;\n\n    use crate::display::hint::visible_regex_match_iter;\n\n    #[test]\n    fn negative_url_parsing_regex_test() {\n        for url_like in [\n            \"http::trace::on_request::log_parameters\",\n            \"http//www.example.org\",\n            \"/user:example.org\",\n            \"mailto: example@example.org\",\n            \"http://<script>alert('xss')</script>\",\n            \"mailto:\",\n        ] {\n            let term = mock_term(url_like);\n            let mut regex = RegexSearch::new(URL_REGEX).unwrap();\n            let matches = visible_regex_match_iter(&term, &mut regex).collect::<Vec<_>>();\n            assert!(\n                matches.is_empty(),\n                \"Should not match url in string {url_like}, but instead got: {matches:?}\"\n            )\n        }\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "set_term_uniforms", "language": "rust", "path": "alacritty/src/renderer/text/glsl3.rs", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function configures shader uniforms related to the dimensions of a cell in a text rendering context.\n2. **Input**: It takes a structure containing properties about the size of the terminal, specifically the width and height of a cell.\n3. **Output**: There is no direct output; however, it updates the shader's uniform variables that affect how text is rendered on the screen.\n4. **Procedure**: The function retrieves the width and height from the input properties and sets these values as uniform variables in the OpenGL context to be used by the shader during rendering.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/renderer/text/atlas.rs\nuse std::borrow::Cow;\nuse std::ptr;\n\nuse crossfont::{BitmapBuffer, RasterizedGlyph};\n\nuse crate::gl;\nuse crate::gl::types::*;\n\nuse super::Glyph;\n\n/// Size of the Atlas.\npub const ATLAS_SIZE: i32 = 1024;\n\n/// Manages a single texture atlas.\n///\n/// The strategy for filling an atlas looks roughly like this:\n///\n/// ```text\n///                           (width, height)\n...\n// Path: alacritty/src/renderer/text/glsl3.rs\nuse std::mem::size_of;\nuse std::ptr;\n\nuse crossfont::RasterizedGlyph;\nuse log::info;\n\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::gl::types::*;\nuse crate::renderer::shader::{ShaderProgram, ShaderVersion};\nuse crate::renderer::{cstr, Error};\n\nuse super::atlas::{Atlas, ATLAS_SIZE};\nuse super::{\n    Glyph, LoadGlyph, LoaderApi, RenderingGlyphFlags, RenderingPass, TextRenderApi,\n    TextRenderBatch, TextRenderer, TextShader,\n};\n\n// Shader source.\npub static TEXT_SHADER_F: &str = include_str!(\"../../../res/glsl3/text.f.glsl\");\nstatic TEXT_SHADER_V: &str = include_str!(\"../../../res/glsl3/text.v.glsl\");\n\n/// Maximum items to be drawn in a batch.\nconst BATCH_MAX: usize = 0x1_0000;\n\n#[derive(Debug)]\npub struct Glsl3Renderer {\n    program: TextShaderProgram,\n    vao: GLuint,\n    ebo: GLuint,\n    vbo_instance: GLuint,\n    atlas: Vec<Atlas>,\n    current_atlas: usize,\n    active_tex: GLuint,\n    batch: Batch,\n}\n\nimpl Glsl3Renderer {\n    pub fn new() -> Result<Self, Error> {\n        info!(\"Using OpenGL 3.3 renderer\");\n\n        let program = TextShaderProgram::new(ShaderVersion::Glsl3)?;\n        let mut vao: GLuint = 0;\n        let mut ebo: GLuint = 0;\n        let mut vbo_instance: GLuint = 0;\n\n        unsafe {\n            gl::Enable(gl::BLEND);\n            gl::BlendFunc(gl::SRC1_COLOR, gl::ONE_MINUS_SRC1_COLOR);\n\n            // Disable depth mask, as the renderer never uses depth tests.\n            gl::DepthMask(gl::FALSE);\n\n            gl::GenVertexArrays(1, &mut vao);\n            gl::GenBuffers(1, &mut ebo);\n            gl::GenBuffers(1, &mut vbo_instance);\n            gl::BindVertexArray(vao);\n\n            // ---------------------\n            // Set up element buffer\n            // ---------------------\n            let indices: [u32; 6] = [0, 1, 3, 1, 2, 3];\n\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, ebo);\n            gl::BufferData(\n                gl::ELEMENT_ARRAY_BUFFER,\n                (6 * size_of::<u32>()) as isize,\n                indices.as_ptr() as *const _,\n                gl::STATIC_DRAW,\n            );\n\n            // ----------------------------\n            // Setup vertex instance buffer\n            // ----------------------------\n            gl::BindBuffer(gl::ARRAY_BUFFER, vbo_instance);\n            gl::BufferData(\n                gl::ARRAY_BUFFER,\n                (BATCH_MAX * size_of::<InstanceData>()) as isize,\n                ptr::null(),\n                gl::STREAM_DRAW,\n            );\n\n            let mut index = 0;\n            let mut size = 0;\n\n            macro_rules! add_attr {\n                ($count:expr, $gl_type:expr, $type:ty) => {\n                    gl::VertexAttribPointer(\n                        index,\n                        $count,\n                        $gl_type,\n                        gl::FALSE,\n                        size_of::<InstanceData>() as i32,\n                        size as *const _,\n                    );\n                    gl::EnableVertexAttribArray(index);\n                    gl::VertexAttribDivisor(index, 1);\n\n                    #[allow(unused_assignments)]\n                    {\n                        size += $count * size_of::<$type>();\n                        index += 1;\n                    }\n                };\n            }\n\n            // Coords.\n            add_attr!(2, gl::UNSIGNED_SHORT, u16);\n\n            // Glyph offset and size.\n            add_attr!(4, gl::SHORT, i16);\n\n            // UV offset.\n            add_attr!(4, gl::FLOAT, f32);\n\n            // Color and cell flags.\n            //\n            // These are packed together because of an OpenGL driver issue on macOS, which caused a\n            // `vec3(u8)` text color and a `u8` cell flags to increase the rendering time by a\n            // huge margin.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Background color.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Cleanup.\n            gl::BindVertexArray(0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n        }\n\n        Ok(Self {\n            program,\n            vao,\n            ebo,\n            vbo_instance,\n            atlas: vec![Atlas::new(ATLAS_SIZE, false)],\n            current_atlas: 0,\n            active_tex: 0,\n            batch: Batch::new(),\n        })\n    }\n}\n\nimpl<'a> TextRenderer<'a> for Glsl3Renderer {\n    type RenderApi = RenderApi<'a>;\n    type RenderBatch = Batch;\n    type Shader = TextShaderProgram;\n\n    fn with_api<'b: 'a, F, T>(&'b mut self, size_info: &'b SizeInfo, func: F) -> T\n    where\n        F: FnOnce(Self::RenderApi) -> T,\n    {\n        unsafe {\n            gl::UseProgram(self.program.id());\n            self.program.set_term_uniforms(size_info);\n\n            gl::BindVertexArray(self.vao);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, self.ebo);\n            gl::BindBuffer(gl::ARRAY_BUFFER, self.vbo_instance);\n            gl::ActiveTexture(gl::TEXTURE0);\n        }\n\n        let res = func(RenderApi {\n            active_tex: &mut self.active_tex,\n            batch: &mut self.batch,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n            program: &mut self.program,\n        });\n\n        unsafe {\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n            gl::BindVertexArray(0);\n\n            gl::UseProgram(0);\n        }\n\n        res\n    }\n\n    fn program(&self) -> &Self::Shader {\n        &self.program\n    }\n\n    fn loader_api(&mut self) -> LoaderApi<'_> {\n        LoaderApi {\n            active_tex: &mut self.active_tex,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n        }\n    }\n}\n\nimpl Drop for Glsl3Renderer {\n    fn drop(&mut self) {\n        unsafe {\n            gl::DeleteBuffers(1, &self.vbo_instance);\n            gl::DeleteBuffers(1, &self.ebo);\n            gl::DeleteVertexArrays(1, &self.vao);\n        }\n    }\n}\n\n#[derive(Debug)]\npub struct RenderApi<'a> {\n    active_tex: &'a mut GLuint,\n    batch: &'a mut Batch,\n    atlas: &'a mut Vec<Atlas>,\n    current_atlas: &'a mut usize,\n    program: &'a mut TextShaderProgram,\n}\n\nimpl<'a> TextRenderApi<Batch> for RenderApi<'a> {\n    fn batch(&mut self) -> &mut Batch {\n        self.batch\n    }\n\n    fn render_batch(&mut self) {\n        unsafe {\n            gl::BufferSubData(\n                gl::ARRAY_BUFFER,\n                0,\n                self.batch.size() as isize,\n                self.batch.instances.as_ptr() as *const _,\n            );\n        }\n\n        // Bind texture if necessary.\n        if *self.active_tex != self.batch.tex() {\n            unsafe {\n                gl::BindTexture(gl::TEXTURE_2D, self.batch.tex());\n            }\n            *self.active_tex = self.batch.tex();\n        }\n\n        unsafe {\n            self.program.set_rendering_pass(RenderingPass::Background);\n            gl::DrawElementsInstanced(\n                gl::TRIANGLES,\n                6,\n                gl::UNSIGNED_INT,\n                ptr::null(),\n                self.batch.len() as GLsizei,\n            );\n            self.program.set_rendering_pass(RenderingPass::SubpixelPass1);\n            gl::DrawElementsInstanced(\n                gl::TRIANGLES,\n                6,\n                gl::UNSIGNED_INT,\n                ptr::null(),\n                self.batch.len() as GLsizei,\n            );\n        }\n\n        self.batch.clear();\n    }\n}\n\nimpl<'a> LoadGlyph for RenderApi<'a> {\n    fn load_glyph(&mut self, rasterized: &RasterizedGlyph) -> Glyph {\n        Atlas::load_glyph(self.active_tex, self.atlas, self.current_atlas, rasterized)\n    }\n\n    fn clear(&mut self) {\n        Atlas::clear_atlas(self.atlas, self.current_atlas)\n    }\n}\n\nimpl<'a> Drop for RenderApi<'a> {\n    fn drop(&mut self) {\n        if !self.batch.is_empty() {\n            self.render_batch();\n        }\n    }\n}\n\n#[derive(Debug)]\n#[repr(C)]\nstruct InstanceData {\n    // Coords.\n    col: u16,\n    row: u16,\n\n    // Glyph offset.\n    left: i16,\n    top: i16,\n\n    // Glyph size.\n    width: i16,\n    height: i16,\n\n    // UV offset.\n    uv_left: f32,\n    uv_bot: f32,\n\n    // uv scale.\n    uv_width: f32,\n    uv_height: f32,\n\n    // Color.\n    r: u8,\n    g: u8,\n    b: u8,\n\n    // Cell flags like multicolor or fullwidth character.\n    cell_flags: RenderingGlyphFlags,\n\n    // Background color.\n    bg_r: u8,\n    bg_g: u8,\n    bg_b: u8,\n    bg_a: u8,\n}\n\n#[derive(Debug, Default)]\npub struct Batch {\n    tex: GLuint,\n    instances: Vec<InstanceData>,\n}\n\nimpl TextRenderBatch for Batch {\n    #[inline]\n    fn tex(&self) -> GLuint {\n        self.tex\n    }\n\n    #[inline]\n    fn full(&self) -> bool {\n        self.capacity() == self.len()\n    }\n\n    #[inline]\n    fn is_empty(&self) -> bool {\n        self.len() == 0\n    }\n\n    fn add_item(&mut self, cell: &RenderableCell, glyph: &Glyph, _: &SizeInfo) {\n        if self.is_empty() {\n            self.tex = glyph.tex_id;\n        }\n\n        let mut cell_flags = RenderingGlyphFlags::empty();\n        cell_flags.set(RenderingGlyphFlags::COLORED, glyph.multicolor);\n        cell_flags.set(RenderingGlyphFlags::WIDE_CHAR, cell.flags.contains(Flags::WIDE_CHAR));\n\n        self.instances.push(InstanceData {\n            col: cell.point.column.0 as u16,\n            row: cell.point.line as u16,\n\n            top: glyph.top,\n            left: glyph.left,\n            width: glyph.width,\n            height: glyph.height,\n\n            uv_bot: glyph.uv_bot,\n            uv_left: glyph.uv_left,\n            uv_width: glyph.uv_width,\n            uv_height: glyph.uv_height,\n\n            r: cell.fg.r,\n            g: cell.fg.g,\n            b: cell.fg.b,\n            cell_flags,\n\n            bg_r: cell.bg.r,\n            bg_g: cell.bg.g,\n            bg_b: cell.bg.b,\n            bg_a: (cell.bg_alpha * 255.0) as u8,\n        });\n    }\n}\n\nimpl Batch {\n    #[inline]\n    pub fn new() -> Self {\n        Self { tex: 0, instances: Vec::with_capacity(BATCH_MAX) }\n    }\n\n    #[inline]\n    pub fn len(&self) -> usize {\n        self.instances.len()\n    }\n\n    #[inline]\n    pub fn capacity(&self) -> usize {\n        BATCH_MAX\n    }\n\n    #[inline]\n    pub fn size(&self) -> usize {\n        self.len() * size_of::<InstanceData>()\n    }\n\n    pub fn clear(&mut self) {\n        self.tex = 0;\n        self.instances.clear();\n    }\n}\n\n/// Text drawing program.\n///\n/// Uniforms are prefixed with \"u\", and vertex attributes are prefixed with \"a\".\n#[derive(Debug)]\npub struct TextShaderProgram {\n    /// Shader program.\n    program: ShaderProgram,\n\n    /// Projection scale and offset uniform.\n    u_projection: GLint,\n\n    /// Cell dimensions (pixels).\n    u_cell_dim: GLint,\n\n    /// Background pass flag.\n    ///\n    /// Rendering is split into two passes; one for backgrounds, and one for text.\n    u_rendering_pass: GLint,\n}\n\nimpl TextShaderProgram {\n    pub fn new(shader_version: ShaderVersion) -> Result<TextShaderProgram, Error> {\n        let program = ShaderProgram::new(shader_version, None, TEXT_SHADER_V, TEXT_SHADER_F)?;\n        Ok(Self {\n            u_projection: program.get_uniform_location(cstr!(\"projection\"))?,\n            u_cell_dim: program.get_uniform_location(cstr!(\"cellDim\"))?,\n            u_rendering_pass: program.get_uniform_location(cstr!(\"renderingPass\"))?,\n            program,\n        })\n    }\n\n    \nfn set_term_uniforms(&self, props: &SizeInfo) {\n        unsafe {\n            gl::Uniform2f(self.u_cell_dim, props.cell_width(), props.cell_height());\n        }\n    }\n\n    fn set_rendering_pass(&self, rendering_pass: RenderingPass) {\n        let value = match rendering_pass {\n            RenderingPass::Background | RenderingPass::SubpixelPass1 => rendering_pass as i32,\n            _ => unreachable!(\"provided pass is not supported in GLSL3 renderer\"),\n        };\n\n        unsafe {\n            gl::Uniform1i(self.u_rendering_pass, value);\n        }\n    }\n}\n\nimpl TextShader for TextShaderProgram {\n    fn id(&self) -> GLuint {\n        self.program.id()\n    }\n\n    fn projection_uniform(&self) -> GLint {\n        self.u_projection\n    }\n}\n\n// Path: alacritty/src/renderer/text/gles2.rs\nuse std::mem::size_of;\nuse std::ptr;\n\nuse crossfont::RasterizedGlyph;\nuse log::info;\n\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::gl::types::*;\nuse crate::renderer::shader::{ShaderProgram, ShaderVersion};\nuse crate::renderer::{cstr, Error, GlExtensions};\n\nuse super::atlas::{Atlas, ATLAS_SIZE};\nuse super::{\n    glsl3, Glyph, LoadGlyph, LoaderApi, RenderingGlyphFlags, RenderingPass, TextRenderApi,\n    TextRenderBatch, TextRenderer, TextShader,\n};\n\n// Shader source.\nstatic TEXT_SHADER_F: &str = include_str!(\"../../../res/gles2/text.f.glsl\");\nstatic TEXT_SHADER_V: &str = include_str!(\"../../../res/gles2/text.v.glsl\");\n\n#[derive(Debug)]\npub struct Gles2Renderer {\n    program: TextShaderProgram,\n    vao: GLuint,\n    vbo: GLuint,\n    ebo: GLuint,\n    atlas: Vec<Atlas>,\n    batch: Batch,\n    current_atlas: usize,\n    active_tex: GLuint,\n    dual_source_blending: bool,\n}\n\nimpl Gles2Renderer {\n    pub fn new(allow_dsb: bool, is_gles_context: bool) -> Result<Self, Error> {\n        info!(\"Using OpenGL ES 2.0 renderer\");\n\n        let dual_source_blending = allow_dsb\n            && (GlExtensions::contains(\"GL_EXT_blend_func_extended\")\n                || GlExtensions::contains(\"GL_ARB_blend_func_extended\"));\n\n        if is_gles_context {\n            info!(\"Running on OpenGL ES context\");\n        }\n\n        if dual_source_blending {\n            info!(\"Using dual source blending\");\n        }\n\n        let program = TextShaderProgram::new(ShaderVersion::Gles2, dual_source_blending)?;\n        let mut vao: GLuint = 0;\n        let mut vbo: GLuint = 0;\n        let mut ebo: GLuint = 0;\n\n        let mut vertex_indices = Vec::with_capacity(BATCH_MAX / 4 * 6);\n        for index in 0..(BATCH_MAX / 4) as u16 {\n            let index = index * 4;\n            vertex_indices.push(index);\n            vertex_indices.push(index + 1);\n            vertex_indices.push(index + 3);\n\n            vertex_indices.push(index + 1);\n            vertex_indices.push(index + 2);\n            vertex_indices.push(index + 3);\n        }\n\n        unsafe {\n            gl::Enable(gl::BLEND);\n\n            gl::DepthMask(gl::FALSE);\n\n            gl::GenVertexArrays(1, &mut vao);\n            gl::GenBuffers(1, &mut ebo);\n            gl::GenBuffers(1, &mut vbo);\n            gl::BindVertexArray(vao);\n\n            // Elements buffer.\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, ebo);\n            gl::BufferData(\n                gl::ELEMENT_ARRAY_BUFFER,\n                (vertex_indices.capacity() * size_of::<u16>()) as isize,\n                vertex_indices.as_ptr() as *const _,\n                gl::STATIC_DRAW,\n            );\n\n            // Vertex buffer.\n            gl::BindBuffer(gl::ARRAY_BUFFER, vbo);\n            gl::BufferData(\n                gl::ARRAY_BUFFER,\n                (BATCH_MAX * size_of::<TextVertex>()) as isize,\n                ptr::null(),\n                gl::STREAM_DRAW,\n            );\n\n            let mut index = 0;\n            let mut size = 0;\n\n            macro_rules! add_attr {\n                ($count:expr, $gl_type:expr, $type:ty) => {\n                    gl::VertexAttribPointer(\n                        index,\n                        $count,\n                        $gl_type,\n                        gl::FALSE,\n                        size_of::<TextVertex>() as i32,\n                        size as *const _,\n                    );\n                    gl::EnableVertexAttribArray(index);\n\n                    #[allow(unused_assignments)]\n                    {\n                        size += $count * size_of::<$type>();\n                        index += 1;\n                    }\n                };\n            }\n\n            // Cell coords.\n            add_attr!(2, gl::SHORT, i16);\n\n            // Glyph coords.\n            add_attr!(2, gl::SHORT, i16);\n\n            // UV.\n            add_attr!(2, gl::FLOAT, u32);\n\n            // Color and bitmap color.\n            //\n            // These are packed together because of an OpenGL driver issue on macOS, which caused a\n            // `vec3(u8)` text color and a `u8` for glyph color to cause performance regressions.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Background color.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Cleanup.\n            gl::BindVertexArray(0);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n        }\n\n        Ok(Self {\n            program,\n            vao,\n            vbo,\n            ebo,\n            atlas: vec![Atlas::new(ATLAS_SIZE, is_gles_context)],\n            batch: Batch::new(),\n            current_atlas: 0,\n            active_tex: 0,\n            dual_source_blending,\n        })\n    }\n}\n\nimpl Drop for Gles2Renderer {\n    fn drop(&mut self) {\n        unsafe {\n            gl::DeleteBuffers(1, &self.vbo);\n            gl::DeleteBuffers(1, &self.ebo);\n            gl::DeleteVertexArrays(1, &self.vao);\n        }\n    }\n}\n\nimpl<'a> TextRenderer<'a> for Gles2Renderer {\n    type RenderApi = RenderApi<'a>;\n    type RenderBatch = Batch;\n    type Shader = TextShaderProgram;\n\n    fn program(&self) -> &Self::Shader {\n        &self.program\n    }\n\n    fn with_api<'b: 'a, F, T>(&'b mut self, _: &'b SizeInfo, func: F) -> T\n    where\n        F: FnOnce(Self::RenderApi) -> T,\n    {\n        unsafe {\n            gl::UseProgram(self.program.id());\n            gl::BindVertexArray(self.vao);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, self.ebo);\n            gl::BindBuffer(gl::ARRAY_BUFFER, self.vbo);\n            gl::ActiveTexture(gl::TEXTURE0);\n        }\n\n        let res = func(RenderApi {\n            active_tex: &mut self.active_tex,\n            batch: &mut self.batch,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n            program: &mut self.program,\n            dual_source_blending: self.dual_source_blending,\n        });\n\n        unsafe {\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n            gl::BindVertexArray(0);\n\n            gl::UseProgram(0);\n        }\n\n        res\n    }\n\n    fn loader_api(&mut self) -> LoaderApi<'_> {\n        LoaderApi {\n            active_tex: &mut self.active_tex,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n        }\n    }\n}\n\n/// Maximum items to be drawn in a batch.\n///\n/// We use the closest number to `u16::MAX` dividable by 4 (amount of vertices we push for a glyph),\n/// since it's the maximum possible index in `glDrawElements` in GLES2.\nconst BATCH_MAX: usize = (u16::MAX - u16::MAX % 4) as usize;\n\n#[derive(Debug)]\npub struct Batch {\n    tex: GLuint,\n    vertices: Vec<TextVertex>,\n}\n\nimpl Batch {\n    fn new() -> Self {\n        Self { tex: 0, vertices: Vec::with_capacity(BATCH_MAX) }\n    }\n\n    #[inline]\n    fn len(&self) -> usize {\n        self.vertices.len()\n    }\n\n    #[inline]\n    fn capacity(&self) -> usize {\n        BATCH_MAX\n    }\n\n    #[inline]\n    fn size(&self) -> usize {\n        self.len() * size_of::<TextVertex>()\n    }\n\n    #[inline]\n    fn clear(&mut self) {\n        self.vertices.clear();\n    }\n}\n\nimpl TextRenderBatch for Batch {\n    #[inline]\n    fn tex(&self) -> GLuint {\n        self.tex\n    }\n\n    #[inline]\n    fn full(&self) -> bool {\n        self.capacity() == self.len()\n    }\n\n    #[inline]\n    fn is_empty(&self) -> bool {\n        self.len() == 0\n    }\n\n    fn add_item(&mut self, cell: &RenderableCell, glyph: &Glyph, size_info: &SizeInfo) {\n        if self.is_empty() {\n            self.tex = glyph.tex_id;\n        }\n\n        // Calculate the cell position.\n        let x = cell.point.column.0 as i16 * size_info.cell_width() as i16;\n        let y = cell.point.line as i16 * size_info.cell_height() as i16;\n\n        // Calculate the glyph position.\n        let glyph_x = cell.point.column.0 as i16 * size_info.cell_width() as i16 + glyph.left;\n        let glyph_y = (cell.point.line + 1) as i16 * size_info.cell_height() as i16 - glyph.top;\n\n        let colored = if glyph.multicolor {\n            RenderingGlyphFlags::COLORED\n        } else {\n            RenderingGlyphFlags::empty()\n        };\n\n        let is_wide = if cell.flags.contains(Flags::WIDE_CHAR) { 2 } else { 1 };\n\n        let mut vertex = TextVertex {\n            x,\n            y: y + size_info.cell_height() as i16,\n\n            glyph_x,\n            glyph_y: glyph_y + glyph.height,\n\n            u: glyph.uv_left,\n            v: glyph.uv_bot + glyph.uv_height,\n            r: cell.fg.r,\n            g: cell.fg.g,\n            b: cell.fg.b,\n            colored,\n            bg_r: cell.bg.r,\n            bg_g: cell.bg.g,\n            bg_b: cell.bg.b,\n            bg_a: (cell.bg_alpha * 255.0) as u8,\n        };\n\n        self.vertices.push(vertex);\n\n        vertex.y = y;\n        vertex.glyph_y = glyph_y;\n        vertex.u = glyph.uv_left;\n        vertex.v = glyph.uv_bot;\n        self.vertices.push(vertex);\n\n        vertex.x = x + is_wide * size_info.cell_width() as i16;\n        vertex.glyph_x = glyph_x + glyph.width;\n        vertex.u = glyph.uv_left + glyph.uv_width;\n        vertex.v = glyph.uv_bot;\n        self.vertices.push(vertex);\n\n        vertex.x = x + is_wide * size_info.cell_width() as i16;\n        vertex.y = y + size_info.cell_height() as i16;\n        vertex.glyph_x = glyph_x + glyph.width;\n        vertex.glyph_y = glyph_y + glyph.height;\n        vertex.u = glyph.uv_left + glyph.uv_width;\n        vertex.v = glyph.uv_bot + glyph.uv_height;\n        self.vertices.push(vertex);\n    }\n}\n\n#[derive(Debug)]\npub struct RenderApi<'a> {\n    active_tex: &'a mut GLuint,\n    batch: &'a mut Batch,\n    atlas: &'a mut Vec<Atlas>,\n    current_atlas: &'a mut usize,\n    program: &'a mut TextShaderProgram,\n    dual_source_blending: bool,\n}\n\nimpl<'a> Drop for RenderApi<'a> {\n    fn drop(&mut self) {\n        if !self.batch.is_empty() {\n            self.render_batch();\n        }\n    }\n}\n\nimpl<'a> LoadGlyph for RenderApi<'a> {\n    fn load_glyph(&mut self, rasterized: &RasterizedGlyph) -> Glyph {\n        Atlas::load_glyph(self.active_tex, self.atlas, self.current_atlas, rasterized)\n    }\n\n    fn clear(&mut self) {\n        Atlas::clear_atlas(self.atlas, self.current_atlas)\n    }\n}\n\nimpl<'a> TextRenderApi<Batch> for RenderApi<'a> {\n    fn batch(&mut self) -> &mut Batch {\n        self.batch\n    }\n\n    fn render_batch(&mut self) {\n        unsafe {\n            gl::BufferSubData(\n                gl::ARRAY_BUFFER,\n                0,\n                self.batch.size() as isize,\n                self.batch.vertices.as_ptr() as *const _,\n            );\n        }\n\n        if *self.active_tex != self.batch.tex() {\n            unsafe {\n                gl::BindTexture(gl::TEXTURE_2D, self.batch.tex());\n            }\n            *self.active_tex = self.batch.tex();\n        }\n\n        unsafe {\n            let num_indices = (self.batch.len() / 4 * 6) as i32;\n\n            // The rendering is inspired by\n            // https://github.com/servo/webrender/blob/master/webrender/doc/text-rendering.md.\n\n            // Draw background.\n            self.program.set_rendering_pass(RenderingPass::Background);\n            gl::BlendFunc(gl::ONE, gl::ZERO);\n            gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n\n            self.program.set_rendering_pass(RenderingPass::SubpixelPass1);\n            if self.dual_source_blending {\n                // Text rendering pass.\n                gl::BlendFunc(gl::SRC1_COLOR, gl::ONE_MINUS_SRC1_COLOR);\n            } else {\n                // First text rendering pass.\n                gl::BlendFuncSeparate(gl::ZERO, gl::ONE_MINUS_SRC_COLOR, gl::ZERO, gl::ONE);\n                gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n\n                // Second text rendering pass.\n                self.program.set_rendering_pass(RenderingPass::SubpixelPass2);\n                gl::BlendFuncSeparate(gl::ONE_MINUS_DST_ALPHA, gl::ONE, gl::ZERO, gl::ONE);\n                gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n\n                // Third text rendering pass.\n                self.program.set_rendering_pass(RenderingPass::SubpixelPass3);\n                gl::BlendFuncSeparate(gl::ONE, gl::ONE, gl::ONE, gl::ONE_MINUS_SRC_ALPHA);\n            }\n\n            gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n        }\n\n        self.batch.clear();\n    }\n}\n\n#[repr(C)]\n#[derive(Debug, Copy, Clone)]\nstruct TextVertex {\n    // Cell coordinates.\n    x: i16,\n    y: i16,\n\n    // Glyph coordinates.\n    glyph_x: i16,\n    glyph_y: i16,\n\n    // Offsets into Atlas.\n    u: f32,\n    v: f32,\n\n    // Color.\n    r: u8,\n    g: u8,\n    b: u8,\n\n    // Whether the glyph is colored.\n    colored: RenderingGlyphFlags,\n\n    // Background color.\n    bg_r: u8,\n    bg_g: u8,\n    bg_b: u8,\n    bg_a: u8,\n}\n\n#[derive(Debug)]\npub struct TextShaderProgram {\n    /// Shader program.\n    program: ShaderProgram,\n\n    /// Projection scale and offset uniform.\n    u_projection: GLint,\n\n    /// Rendering pass.\n    ///\n    /// For dual source blending, there are 2 passes; one for background, another for text,\n    /// similar to the GLSL3 renderer.\n    ///\n    /// If GL_EXT_blend_func_extended is not available, the rendering is split into 4 passes.\n    /// One is used for the background and the rest to perform subpixel text rendering according to\n    /// <https://github.com/servo/webrender/blob/master/webrender/doc/text-rendering.md>.\n    ///\n    /// Rendering is split into three passes.\n    u_rendering_pass: GLint,\n}\n\nimpl TextShaderProgram {\n    pub fn new(shader_version: ShaderVersion, dual_source_blending: bool) -> Result<Self, Error> {\n        let fragment_shader =\n            if dual_source_blending { &glsl3::TEXT_SHADER_F } else { &TEXT_SHADER_F };\n\n        let program = ShaderProgram::new(shader_version, None, TEXT_SHADER_V, fragment_shader)?;\n\n        Ok(Self {\n            u_projection: program.get_uniform_location(cstr!(\"projection\"))?,\n            u_rendering_pass: program.get_uniform_location(cstr!(\"renderingPass\"))?,\n            program,\n        })\n    }\n\n    fn set_rendering_pass(&self, rendering_pass: RenderingPass) {\n        unsafe { gl::Uniform1i(self.u_rendering_pass, rendering_pass as i32) }\n    }\n}\n\nimpl TextShader for TextShaderProgram {\n    fn id(&self) -> GLuint {\n        self.program.id()\n    }\n\n    fn projection_uniform(&self) -> GLint {\n        self.u_projection\n    }\n}\n\n// Path: alacritty/src/renderer/text/mod.rs\nuse bitflags::bitflags;\nuse crossfont::{GlyphKey, RasterizedGlyph};\n\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::gl::types::*;\n\nmod atlas;\nmod builtin_font;\nmod gles2;\nmod glsl3;\npub mod glyph_cache;\n\nuse atlas::Atlas;\npub use gles2::Gles2Renderer;\npub use glsl3::Glsl3Renderer;\npub use glyph_cache::GlyphCache;\nuse glyph_cache::{Glyph, LoadGlyph};\n\n// NOTE: These flags must be in sync with their usage in the text.*.glsl shaders.\nbitflags! {\n    #[repr(C)]\n    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]\n    struct RenderingGlyphFlags: u8 {\n        const COLORED   = 0b0000_0001;\n        const WIDE_CHAR = 0b0000_0010;\n    }\n}\n\n/// Rendering passes, for both GLES2 and GLSL3 renderer.\n#[repr(u8)]\nenum RenderingPass {\n    /// Rendering pass used to render background color in text shaders.\n    Background = 0,\n\n    /// The first pass to render text with both GLES2 and GLSL3 renderers.\n    SubpixelPass1 = 1,\n\n    /// The second pass to render text with GLES2 renderer.\n    SubpixelPass2 = 2,\n\n    /// The third pass to render text with GLES2 renderer.\n    SubpixelPass3 = 3,\n}\n\npub trait TextRenderer<'a> {\n    type Shader: TextShader;\n    type RenderBatch: TextRenderBatch;\n    type RenderApi: TextRenderApi<Self::RenderBatch>;\n\n    /// Get loader API for the renderer.\n    fn loader_api(&mut self) -> LoaderApi<'_>;\n\n    /// Draw cells.\n    fn draw_cells<'b: 'a, I: Iterator<Item = RenderableCell>>(\n        &'b mut self,\n        size_info: &'b SizeInfo,\n        glyph_cache: &'a mut GlyphCache,\n        cells: I,\n    ) {\n        self.with_api(size_info, |mut api| {\n            for cell in cells {\n                api.draw_cell(cell, glyph_cache, size_info);\n            }\n        })\n    }\n\n    fn with_api<'b: 'a, F, T>(&'b mut self, size_info: &'b SizeInfo, func: F) -> T\n    where\n        F: FnOnce(Self::RenderApi) -> T;\n\n    fn program(&self) -> &Self::Shader;\n\n    /// Resize the text rendering.\n    fn resize(&self, size: &SizeInfo) {\n        unsafe {\n            let program = self.program();\n            gl::UseProgram(program.id());\n            update_projection(program.projection_uniform(), size);\n            gl::UseProgram(0);\n        }\n    }\n\n    /// Invoke renderer with the loader.\n    fn with_loader<F: FnOnce(LoaderApi<'_>) -> T, T>(&mut self, func: F) -> T {\n        unsafe {\n            gl::ActiveTexture(gl::TEXTURE0);\n        }\n\n        func(self.loader_api())\n    }\n}\n\npub trait TextRenderBatch {\n    /// Check if `Batch` is empty.\n    fn is_empty(&self) -> bool;\n\n    /// Check whether the `Batch` is full.\n    fn full(&self) -> bool;\n\n    /// Get texture `Batch` is using.\n    fn tex(&self) -> GLuint;\n\n    /// Add item to the batch.\n    fn add_item(&mut self, cell: &RenderableCell, glyph: &Glyph, size_info: &SizeInfo);\n}\n\npub trait TextRenderApi<T: TextRenderBatch>: LoadGlyph {\n    /// Get `Batch` the api is using.\n    fn batch(&mut self) -> &mut T;\n\n    /// Render the underlying data.\n    fn render_batch(&mut self);\n\n    /// Add item to the rendering queue.\n    #[inline]\n    fn add_render_item(&mut self, cell: &RenderableCell, glyph: &Glyph, size_info: &SizeInfo) {\n        // Flush batch if tex changing.\n        if !self.batch().is_empty() && self.batch().tex() != glyph.tex_id {\n            self.render_batch();\n        }\n\n        self.batch().add_item(cell, glyph, size_info);\n\n        // Render batch and clear if it's full.\n        if self.batch().full() {\n            self.render_batch();\n        }\n    }\n\n    /// Draw cell.\n    fn draw_cell(\n        &mut self,\n        mut cell: RenderableCell,\n        glyph_cache: &mut GlyphCache,\n        size_info: &SizeInfo,\n    ) {\n        // Get font key for cell.\n        let font_key = match cell.flags & Flags::BOLD_ITALIC {\n            Flags::BOLD_ITALIC => glyph_cache.bold_italic_key,\n            Flags::ITALIC => glyph_cache.italic_key,\n            Flags::BOLD => glyph_cache.bold_key,\n            _ => glyph_cache.font_key,\n        };\n\n        // Ignore hidden cells and render tabs as spaces to prevent font issues.\n        let hidden = cell.flags.contains(Flags::HIDDEN);\n        if cell.character == '\\t' || hidden {\n            cell.character = ' ';\n        }\n\n        let mut glyph_key =\n            GlyphKey { font_key, size: glyph_cache.font_size, character: cell.character };\n\n        // Add cell to batch.\n        let glyph = glyph_cache.get(glyph_key, self, true);\n        self.add_render_item(&cell, &glyph, size_info);\n\n        // Render visible zero-width characters.\n        if let Some(zerowidth) =\n            cell.extra.as_mut().and_then(|extra| extra.zerowidth.take().filter(|_| !hidden))\n        {\n            for character in zerowidth {\n                glyph_key.character = character;\n                let glyph = glyph_cache.get(glyph_key, self, false);\n                self.add_render_item(&cell, &glyph, size_info);\n            }\n        }\n    }\n}\n\npub trait TextShader {\n    fn id(&self) -> GLuint;\n\n    /// Id of the projection uniform.\n    fn projection_uniform(&self) -> GLint;\n}\n\n#[derive(Debug)]\npub struct LoaderApi<'a> {\n    active_tex: &'a mut GLuint,\n    atlas: &'a mut Vec<Atlas>,\n    current_atlas: &'a mut usize,\n}\n\nimpl<'a> LoadGlyph for LoaderApi<'a> {\n    fn load_glyph(&mut self, rasterized: &RasterizedGlyph) -> Glyph {\n        Atlas::load_glyph(self.active_tex, self.atlas, self.current_atlas, rasterized)\n    }\n\n    fn clear(&mut self) {\n        Atlas::clear_atlas(self.atlas, self.current_atlas)\n    }\n}\n\nfn update_projection(u_projection: GLint, size: &SizeInfo) {\n    let width = size.width();\n    let height = size.height();\n    let padding_x = size.padding_x();\n    let padding_y = size.padding_y();\n\n    // Bounds check.\n    if (width as u32) < (2 * padding_x as u32) || (height as u32) < (2 * padding_y as u32) {\n        return;\n    }\n\n    // Compute scale and offset factors, from pixel to ndc space. Y is inverted.\n    //   [0, width - 2 * padding_x] to [-1, 1]\n    //   [height - 2 * padding_y, 0] to [-1, 1]\n    let scale_x = 2. / (width - 2. * padding_x);\n    let scale_y = -2. / (height - 2. * padding_y);\n    let offset_x = -1.;\n    let offset_y = 1.;\n\n    unsafe {\n        gl::Uniform4f(u_projection, offset_x, offset_y, scale_x, scale_y);\n    }\n}\n\n// Path: alacritty/src/renderer/platform.rs\n//! The graphics platform that is used by the renderer.\n\nuse std::num::NonZeroU32;\n\nuse glutin::config::{ColorBufferType, Config, ConfigTemplateBuilder, GetGlConfig};\nuse glutin::context::{\n    ContextApi, ContextAttributesBuilder, GlProfile, NotCurrentContext, Version,\n};\nuse glutin::display::{Display, DisplayApiPreference, GetGlDisplay};\nuse glutin::error::Result as GlutinResult;\nuse glutin::prelude::*;\nuse glutin::surface::{Surface, SurfaceAttributesBuilder, WindowSurface};\nuse log::{debug, LevelFilter};\n\nuse raw_window_handle::{RawDisplayHandle, RawWindowHandle};\nuse winit::dpi::PhysicalSize;\n#[cfg(all(feature = \"x11\", not(any(target_os = \"macos\", windows))))]\nuse winit::platform::x11;\n\n/// Create the GL display.\npub fn create_gl_display(\n    raw_display_handle: RawDisplayHandle,\n    _raw_window_handle: Option<RawWindowHandle>,\n    _prefer_egl: bool,\n) -> GlutinResult<Display> {\n    #[cfg(target_os = \"macos\")]\n    let preference = DisplayApiPreference::Cgl;\n\n    #[cfg(windows)]\n    let preference = if _prefer_egl {\n        DisplayApiPreference::EglThenWgl(Some(_raw_window_handle.unwrap()))\n    } else {\n        DisplayApiPreference::WglThenEgl(Some(_raw_window_handle.unwrap()))\n    };\n\n    #[cfg(all(feature = \"x11\", not(any(target_os = \"macos\", windows))))]\n    let preference = if _prefer_egl {\n        DisplayApiPreference::EglThenGlx(Box::new(x11::register_xlib_error_hook))\n    } else {\n        DisplayApiPreference::GlxThenEgl(Box::new(x11::register_xlib_error_hook))\n    };\n\n    #[cfg(all(not(feature = \"x11\"), not(any(target_os = \"macos\", windows))))]\n    let preference = DisplayApiPreference::Egl;\n\n    let display = unsafe { Display::new(raw_display_handle, preference)? };\n    log::info!(\"Using {}\", { display.version_string() });\n    Ok(display)\n}\n\npub fn pick_gl_config(\n    gl_display: &Display,\n    raw_window_handle: Option<RawWindowHandle>,\n) -> Result<Config, String> {\n    let mut default_config = ConfigTemplateBuilder::new()\n        .with_depth_size(0)\n        .with_stencil_size(0)\n        .with_transparency(true);\n\n    if let Some(raw_window_handle) = raw_window_handle {\n        default_config = default_config.compatible_with_native_window(raw_window_handle);\n    }\n\n    let config_10bit = default_config\n        .clone()\n        .with_buffer_type(ColorBufferType::Rgb { r_size: 10, g_size: 10, b_size: 10 })\n        .with_alpha_size(2);\n\n    let configs = [\n        default_config.clone(),\n        config_10bit.clone(),\n        default_config.with_transparency(false),\n        config_10bit.with_transparency(false),\n    ];\n\n    for config in configs {\n        let gl_config = unsafe {\n            gl_display.find_configs(config.build()).ok().and_then(|mut configs| configs.next())\n        };\n\n        if let Some(gl_config) = gl_config {\n            debug!(\n                r#\"Picked GL Config:\n  buffer_type: {:?}\n  alpha_size: {}\n  num_samples: {}\n  hardware_accelerated: {:?}\n  supports_transparency: {:?}\n  config_api: {:?}\n  srgb_capable: {}\"#,\n                gl_config.color_buffer_type(),\n                gl_config.alpha_size(),\n                gl_config.num_samples(),\n                gl_config.hardware_accelerated(),\n                gl_config.supports_transparency(),\n                gl_config.api(),\n                gl_config.srgb_capable(),\n            );\n\n            return Ok(gl_config);\n        }\n    }\n\n    Err(String::from(\"failed to find suitable GL configuration.\"))\n}\n\npub fn create_gl_context(\n    gl_display: &Display,\n    gl_config: &Config,\n    raw_window_handle: Option<RawWindowHandle>,\n) -> GlutinResult<NotCurrentContext> {\n    let debug = log::max_level() >= LevelFilter::Debug;\n    let mut profiles = [\n        ContextAttributesBuilder::new()\n            .with_debug(debug)\n            .with_context_api(ContextApi::OpenGl(Some(Version::new(3, 3))))\n            .build(raw_window_handle),\n        // Try gles before OpenGL 2.1 as it tends to be more stable.\n        ContextAttributesBuilder::new()\n            .with_debug(debug)\n            .with_context_api(ContextApi::Gles(Some(Version::new(2, 0))))\n            .build(raw_window_handle),\n        ContextAttributesBuilder::new()\n            .with_debug(debug)\n            .with_profile(GlProfile::Compatibility)\n            .with_context_api(ContextApi::OpenGl(Some(Version::new(2, 1))))\n            .build(raw_window_handle),\n    ]\n    .into_iter();\n\n    // Try the optimal config first.\n    let mut picked_context =\n        unsafe { gl_display.create_context(gl_config, &profiles.next().unwrap()) };\n\n    // Try the fallback ones.\n    while let (Err(_), Some(profile)) = (picked_context.as_ref(), profiles.next()) {\n        picked_context = unsafe { gl_display.create_context(gl_config, &profile) };\n    }\n\n    picked_context\n}\n\npub fn create_gl_surface(\n    gl_context: &NotCurrentContext,\n    size: PhysicalSize<u32>,\n    raw_window_handle: RawWindowHandle,\n) -> GlutinResult<Surface<WindowSurface>> {\n    // Get the display and the config used to create that context.\n    let gl_display = gl_context.display();\n    let gl_config = gl_context.config();\n\n    let surface_attributes =\n        SurfaceAttributesBuilder::<WindowSurface>::new().with_srgb(Some(false)).build(\n            raw_window_handle,\n            NonZeroU32::new(size.width).unwrap(),\n            NonZeroU32::new(size.height).unwrap(),\n        );\n\n    // Create the GL surface to draw into.\n    unsafe { gl_display.create_window_surface(&gl_config, &surface_attributes) }\n}\n\n// Path: alacritty/src/renderer/mod.rs\nuse std::borrow::Cow;\nuse std::collections::HashSet;\nuse std::ffi::{CStr, CString};\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::OnceLock;\nuse std::{fmt, ptr};\n\nuse ahash::RandomState;\nuse crossfont::Metrics;\nuse glutin::context::{ContextApi, GlContext, PossiblyCurrentContext};\nuse glutin::display::{GetGlDisplay, GlDisplay};\nuse log::{debug, error, info, warn, LevelFilter};\nuse unicode_width::UnicodeWidthChar;\n\nuse alacritty_terminal::index::Point;\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::config::debug::RendererPreference;\nuse crate::display::color::Rgb;\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::renderer::rects::{RectRenderer, RenderRect};\nuse crate::renderer::shader::ShaderError;\n\npub mod platform;\npub mod rects;\nmod shader;\nmod text;\n\npub use text::{GlyphCache, LoaderApi};\n\nuse shader::ShaderVersion;\nuse text::{Gles2Renderer, Glsl3Renderer, TextRenderer};\n\nmacro_rules! cstr {\n    ($s:literal) => {\n        // This can be optimized into an no-op with pre-allocated NUL-terminated bytes.\n        unsafe { std::ffi::CStr::from_ptr(concat!($s, \"\\0\").as_ptr().cast()) }\n    };\n}\npub(crate) use cstr;\n\n/// Whether the OpenGL functions have been loaded.\npub static GL_FUNS_LOADED: AtomicBool = AtomicBool::new(false);\n\n#[derive(Debug)]\npub enum Error {\n    /// Shader error.\n    Shader(ShaderError),\n\n    /// Other error.\n    Other(String),\n}\n\nimpl std::error::Error for Error {\n    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {\n        match self {\n            Error::Shader(err) => err.source(),\n            Error::Other(_) => None,\n        }\n    }\n}\n\nimpl fmt::Display for Error {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            Error::Shader(err) => {\n                write!(f, \"There was an error initializing the shaders: {}\", err)\n            },\n            Error::Other(err) => {\n                write!(f, \"{}\", err)\n            },\n        }\n    }\n}\n\nimpl From<ShaderError> for Error {\n    fn from(val: ShaderError) -> Self {\n        Error::Shader(val)\n    }\n}\n\nimpl From<String> for Error {\n    fn from(val: String) -> Self {\n        Error::Other(val)\n    }\n}\n\n#[derive(Debug)]\nenum TextRendererProvider {\n    Gles2(Gles2Renderer),\n    Glsl3(Glsl3Renderer),\n}\n\n#[derive(Debug)]\npub struct Renderer {\n    text_renderer: TextRendererProvider,\n    rect_renderer: RectRenderer,\n}\n\n/// Wrapper around gl::GetString with error checking and reporting.\nfn gl_get_string(\n    string_id: gl::types::GLenum,\n    description: &str,\n) -> Result<Cow<'static, str>, Error> {\n    unsafe {\n        let string_ptr = gl::GetString(string_id);\n        match gl::GetError() {\n            gl::NO_ERROR if !string_ptr.is_null() => {\n                Ok(CStr::from_ptr(string_ptr as *const _).to_string_lossy())\n            },\n            gl::INVALID_ENUM => {\n                Err(format!(\"OpenGL error requesting {}: invalid enum\", description).into())\n            },\n            error_id => Err(format!(\"OpenGL error {} requesting {}\", error_id, description).into()),\n        }\n    }\n}\n\nimpl Renderer {\n    /// Create a new renderer.\n    ///\n    /// This will automatically pick between the GLES2 and GLSL3 renderer based on the GPU's\n    /// supported OpenGL version.\n    pub fn new(\n        context: &PossiblyCurrentContext,\n        renderer_preference: Option<RendererPreference>,\n    ) -> Result<Self, Error> {\n        // We need to load OpenGL functions once per instance, but only after we make our context\n        // current due to WGL limitations.\n        if !GL_FUNS_LOADED.swap(true, Ordering::Relaxed) {\n            let gl_display = context.display();\n            gl::load_with(|symbol| {\n                let symbol = CString::new(symbol).unwrap();\n                gl_display.get_proc_address(symbol.as_c_str()).cast()\n            });\n        }\n\n        let shader_version = gl_get_string(gl::SHADING_LANGUAGE_VERSION, \"shader version\")?;\n        let gl_version = gl_get_string(gl::VERSION, \"OpenGL version\")?;\n        let renderer = gl_get_string(gl::RENDERER, \"renderer version\")?;\n\n        info!(\"Running on {renderer}\");\n        info!(\"OpenGL version {gl_version}, shader_version {shader_version}\");\n\n        let is_gles_context = matches!(context.context_api(), ContextApi::Gles(_));\n\n        // Use the config option to enforce a particular renderer configuration.\n        let (use_glsl3, allow_dsb) = match renderer_preference {\n            Some(RendererPreference::Glsl3) => (true, true),\n            Some(RendererPreference::Gles2) => (false, true),\n            Some(RendererPreference::Gles2Pure) => (false, false),\n            None => (shader_version.as_ref() >= \"3.3\" && !is_gles_context, true),\n        };\n\n        let (text_renderer, rect_renderer) = if use_glsl3 {\n            let text_renderer = TextRendererProvider::Glsl3(Glsl3Renderer::new()?);\n            let rect_renderer = RectRenderer::new(ShaderVersion::Glsl3)?;\n            (text_renderer, rect_renderer)\n        } else {\n            let text_renderer =\n                TextRendererProvider::Gles2(Gles2Renderer::new(allow_dsb, is_gles_context)?);\n            let rect_renderer = RectRenderer::new(ShaderVersion::Gles2)?;\n            (text_renderer, rect_renderer)\n        };\n\n        // Enable debug logging for OpenGL as well.\n        if log::max_level() >= LevelFilter::Debug && GlExtensions::contains(\"GL_KHR_debug\") {\n            debug!(\"Enabled debug logging for OpenGL\");\n            unsafe {\n                gl::Enable(gl::DEBUG_OUTPUT);\n                gl::Enable(gl::DEBUG_OUTPUT_SYNCHRONOUS);\n                gl::DebugMessageCallback(Some(gl_debug_log), ptr::null_mut());\n            }\n        }\n\n        Ok(Self { text_renderer, rect_renderer })\n    }\n\n    pub fn draw_cells<I: Iterator<Item = RenderableCell>>(\n        &mut self,\n        size_info: &SizeInfo,\n        glyph_cache: &mut GlyphCache,\n        cells: I,\n    ) {\n        match &mut self.text_renderer {\n            TextRendererProvider::Gles2(renderer) => {\n                renderer.draw_cells(size_info, glyph_cache, cells)\n            },\n            TextRendererProvider::Glsl3(renderer) => {\n                renderer.draw_cells(size_info, glyph_cache, cells)\n            },\n        }\n    }\n\n    /// Draw a string in a variable location. Used for printing the render timer, warnings and\n    /// errors.\n    pub fn draw_string(\n        &mut self,\n        point: Point<usize>,\n        fg: Rgb,\n        bg: Rgb,\n        string_chars: impl Iterator<Item = char>,\n        size_info: &SizeInfo,\n        glyph_cache: &mut GlyphCache,\n    ) {\n        let mut skip_next = false;\n        let cells = string_chars.enumerate().filter_map(|(i, character)| {\n            if skip_next {\n                skip_next = false;\n                return None;\n            }\n\n            let mut flags = Flags::empty();\n            if character.width() == Some(2) {\n                flags.insert(Flags::WIDE_CHAR);\n                // Wide character is always followed by a spacer, so skip it.\n                skip_next = true;\n            }\n\n            Some(RenderableCell {\n                point: Point::new(point.line, point.column + i),\n                character,\n                extra: None,\n                flags: Flags::empty(),\n                bg_alpha: 1.0,\n                fg,\n                bg,\n                underline: fg,\n            })\n        });\n\n        self.draw_cells(size_info, glyph_cache, cells);\n    }\n\n    pub fn with_loader<F, T>(&mut self, func: F) -> T\n    where\n        F: FnOnce(LoaderApi<'_>) -> T,\n    {\n        match &mut self.text_renderer {\n            TextRendererProvider::Gles2(renderer) => renderer.with_loader(func),\n            TextRendererProvider::Glsl3(renderer) => renderer.with_loader(func),\n        }\n    }\n\n    /// Draw all rectangles simultaneously to prevent excessive program swaps.\n    pub fn draw_rects(&mut self, size_info: &SizeInfo, metrics: &Metrics, rects: Vec<RenderRect>) {\n        if rects.is_empty() {\n            return;\n        }\n\n        // Prepare rect rendering state.\n        unsafe {\n            // Remove padding from viewport.\n            gl::Viewport(0, 0, size_info.width() as i32, size_info.height() as i32);\n            gl::BlendFuncSeparate(gl::SRC_ALPHA, gl::ONE_MINUS_SRC_ALPHA, gl::SRC_ALPHA, gl::ONE);\n        }\n\n        self.rect_renderer.draw(size_info, metrics, rects);\n\n        // Activate regular state again.\n        unsafe {\n            // Reset blending strategy.\n            gl::BlendFunc(gl::SRC1_COLOR, gl::ONE_MINUS_SRC1_COLOR);\n\n            // Restore viewport with padding.\n            self.set_viewport(size_info);\n        }\n    }\n\n    /// Fill the window with `color` and `alpha`.\n    pub fn clear(&self, color: Rgb, alpha: f32) {\n        unsafe {\n            gl::ClearColor(\n                (f32::from(color.r) / 255.0).min(1.0) * alpha,\n                (f32::from(color.g) / 255.0).min(1.0) * alpha,\n                (f32::from(color.b) / 255.0).min(1.0) * alpha,\n                alpha,\n            );\n            gl::Clear(gl::COLOR_BUFFER_BIT);\n        }\n    }\n\n    pub fn finish(&self) {\n        unsafe {\n            gl::Finish();\n        }\n    }\n\n    /// Set the viewport for cell rendering.\n    #[inline]\n...\n", "needle_token_start": 3980, "needle_token_end": 4036, "code_context_ntokens": 16096, "output": ["```rust\nfn set_term_uniforms(&self, props: &SizeInfo) {\n    unsafe {\n        gl::Uniform2f(self.u_cell_dim, props.cell_width(), props.cell_height());\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "footer_bar_foreground", "language": "rust", "path": "alacritty/src/config/color.rs", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve the foreground color for a specific UI component's footer bar, defaulting to another primary color if the specific foreground color is not set.\n2. **Input**: There are no parameters required for this function; it operates on the instance's state.\n3. **Output**: Returns a color value representing the foreground color of the footer bar.\n4. **Procedure**: The function checks if a specific color is set for the footer bar's foreground. If it is set, this color is returned. If not, the function defaults to using the background color from another primary color set.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/config/color.rs\nuse serde::de::Error as SerdeError;\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::ConfigDeserialize;\n\nuse crate::display::color::{CellRgb, Rgb};\n\n#[derive(ConfigDeserialize, Clone, Debug, Default, PartialEq, Eq)]\npub struct Colors {\n    pub primary: PrimaryColors,\n    pub cursor: InvertedCellColors,\n    pub vi_mode_cursor: InvertedCellColors,\n    pub selection: InvertedCellColors,\n    pub normal: NormalColors,\n    pub bright: BrightColors,\n    pub dim: Option<DimColors>,\n    pub indexed_colors: Vec<IndexedColor>,\n    pub search: SearchColors,\n    pub line_indicator: LineIndicatorColors,\n    pub hints: HintColors,\n    pub transparent_background_colors: bool,\n    pub draw_bold_text_with_bright_colors: bool,\n    footer_bar: BarColors,\n}\n\nimpl Colors {\n    \npub fn footer_bar_foreground(&self) -> Rgb {\n        self.footer_bar.foreground.unwrap_or(self.primary.background)\n    }\n\n    pub fn footer_bar_background(&self) -> Rgb {\n        self.footer_bar.background.unwrap_or(self.primary.foreground)\n    }\n}\n\n#[derive(ConfigDeserialize, Copy, Clone, Default, Debug, PartialEq, Eq)]\npub struct LineIndicatorColors {\n    pub foreground: Option<Rgb>,\n    pub background: Option<Rgb>,\n}\n\n#[derive(ConfigDeserialize, Default, Copy, Clone, Debug, PartialEq, Eq)]\npub struct HintColors {\n    pub start: HintStartColors,\n    pub end: HintEndColors,\n}\n\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq)]\npub struct HintStartColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for HintStartColors {\n    fn default() -> Self {\n        Self {\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n            background: CellRgb::Rgb(Rgb::new(0xf4, 0xbf, 0x75)),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq)]\npub struct HintEndColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for HintEndColors {\n    fn default() -> Self {\n        Self {\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n            background: CellRgb::Rgb(Rgb::new(0xac, 0x42, 0x42)),\n        }\n    }\n}\n\n#[derive(Deserialize, Copy, Clone, Default, Debug, PartialEq, Eq)]\n#[serde(deny_unknown_fields)]\npub struct IndexedColor {\n    pub color: Rgb,\n\n    index: ColorIndex,\n}\n\nimpl IndexedColor {\n    #[inline]\n    pub fn index(&self) -> u8 {\n        self.index.0\n    }\n}\n\n#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)]\nstruct ColorIndex(u8);\n\nimpl<'de> Deserialize<'de> for ColorIndex {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let index = u8::deserialize(deserializer)?;\n\n        if index < 16 {\n            Err(SerdeError::custom(\n                \"Config error: indexed_color's index is {}, but a value bigger than 15 was \\\n                 expected; ignoring setting\",\n            ))\n        } else {\n            Ok(Self(index))\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub struct InvertedCellColors {\n    #[config(alias = \"text\")]\n    pub foreground: CellRgb,\n    #[config(alias = \"cursor\")]\n    pub background: CellRgb,\n}\n\nimpl Default for InvertedCellColors {\n    fn default() -> Self {\n        Self { foreground: CellRgb::CellBackground, background: CellRgb::CellForeground }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, Default, PartialEq, Eq)]\npub struct SearchColors {\n    pub focused_match: FocusedMatchColors,\n    pub matches: MatchColors,\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub struct FocusedMatchColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for FocusedMatchColors {\n    fn default() -> Self {\n        Self {\n            background: CellRgb::Rgb(Rgb::new(0xf4, 0xbf, 0x75)),\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub struct MatchColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for MatchColors {\n    fn default() -> Self {\n        Self {\n            background: CellRgb::Rgb(Rgb::new(0xac, 0x42, 0x42)),\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, Default, PartialEq, Eq)]\npub struct BarColors {\n    foreground: Option<Rgb>,\n    background: Option<Rgb>,\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct PrimaryColors {\n    pub foreground: Rgb,\n    pub background: Rgb,\n    pub bright_foreground: Option<Rgb>,\n    pub dim_foreground: Option<Rgb>,\n}\n\nimpl Default for PrimaryColors {\n    fn default() -> Self {\n        PrimaryColors {\n            background: Rgb::new(0x18, 0x18, 0x18),\n            foreground: Rgb::new(0xd8, 0xd8, 0xd8),\n            bright_foreground: Default::default(),\n            dim_foreground: Default::default(),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct NormalColors {\n    pub black: Rgb,\n    pub red: Rgb,\n    pub green: Rgb,\n    pub yellow: Rgb,\n    pub blue: Rgb,\n    pub magenta: Rgb,\n    pub cyan: Rgb,\n    pub white: Rgb,\n}\n\nimpl Default for NormalColors {\n    fn default() -> Self {\n        NormalColors {\n            black: Rgb::new(0x18, 0x18, 0x18),\n            red: Rgb::new(0xac, 0x42, 0x42),\n            green: Rgb::new(0x90, 0xa9, 0x59),\n            yellow: Rgb::new(0xf4, 0xbf, 0x75),\n            blue: Rgb::new(0x6a, 0x9f, 0xb5),\n            magenta: Rgb::new(0xaa, 0x75, 0x9f),\n            cyan: Rgb::new(0x75, 0xb5, 0xaa),\n            white: Rgb::new(0xd8, 0xd8, 0xd8),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct BrightColors {\n    pub black: Rgb,\n    pub red: Rgb,\n    pub green: Rgb,\n    pub yellow: Rgb,\n    pub blue: Rgb,\n    pub magenta: Rgb,\n    pub cyan: Rgb,\n    pub white: Rgb,\n}\n\nimpl Default for BrightColors {\n    fn default() -> Self {\n        // Generated with oklab by multiplying brightness by 1.12 and then adjusting numbers\n        // to make them look \"nicer\". Yellow color was generated the same way, however the first\n        // srgb representable color was picked.\n        BrightColors {\n            black: Rgb::new(0x6b, 0x6b, 0x6b),\n            red: Rgb::new(0xc5, 0x55, 0x55),\n            green: Rgb::new(0xaa, 0xc4, 0x74),\n            yellow: Rgb::new(0xfe, 0xca, 0x88),\n            blue: Rgb::new(0x82, 0xb8, 0xc8),\n            magenta: Rgb::new(0xc2, 0x8c, 0xb8),\n            cyan: Rgb::new(0x93, 0xd3, 0xc3),\n            white: Rgb::new(0xf8, 0xf8, 0xf8),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct DimColors {\n    pub black: Rgb,\n    pub red: Rgb,\n    pub green: Rgb,\n    pub yellow: Rgb,\n    pub blue: Rgb,\n    pub magenta: Rgb,\n    pub cyan: Rgb,\n    pub white: Rgb,\n}\n\nimpl Default for DimColors {\n    fn default() -> Self {\n        // Generated with builtin alacritty's color dimming function.\n        DimColors {\n            black: Rgb::new(0x0f, 0x0f, 0x0f),\n            red: Rgb::new(0x71, 0x2b, 0x2b),\n            green: Rgb::new(0x5f, 0x6f, 0x3a),\n            yellow: Rgb::new(0xa1, 0x7e, 0x4d),\n            blue: Rgb::new(0x45, 0x68, 0x77),\n            magenta: Rgb::new(0x70, 0x4d, 0x68),\n            cyan: Rgb::new(0x4d, 0x77, 0x70),\n            white: Rgb::new(0x8e, 0x8e, 0x8e),\n        }\n    }\n}\n\n// Path: alacritty/src/display/color.rs\nuse std::fmt::{self, Display, Formatter};\nuse std::ops::{Add, Deref, Index, IndexMut, Mul};\nuse std::str::FromStr;\n\nuse log::trace;\nuse serde::de::{Error as SerdeError, Visitor};\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::SerdeReplace;\nuse alacritty_terminal::term::color::COUNT;\nuse alacritty_terminal::vte::ansi::{NamedColor, Rgb as VteRgb};\n\nuse crate::config::color::Colors;\n\n/// Factor for automatic computation of dim colors.\npub const DIM_FACTOR: f32 = 0.66;\n\n#[derive(Copy, Clone)]\npub struct List([Rgb; COUNT]);\n\nimpl<'a> From<&'a Colors> for List {\n    fn from(colors: &Colors) -> List {\n        // Type inference fails without this annotation.\n        let mut list = List([Rgb::default(); COUNT]);\n\n        list.fill_named(colors);\n        list.fill_cube(colors);\n        list.fill_gray_ramp(colors);\n\n        list\n    }\n}\n\nimpl List {\n    pub fn fill_named(&mut self, colors: &Colors) {\n        // Normals.\n        self[NamedColor::Black] = colors.normal.black;\n        self[NamedColor::Red] = colors.normal.red;\n        self[NamedColor::Green] = colors.normal.green;\n        self[NamedColor::Yellow] = colors.normal.yellow;\n        self[NamedColor::Blue] = colors.normal.blue;\n        self[NamedColor::Magenta] = colors.normal.magenta;\n        self[NamedColor::Cyan] = colors.normal.cyan;\n        self[NamedColor::White] = colors.normal.white;\n\n        // Brights.\n        self[NamedColor::BrightBlack] = colors.bright.black;\n        self[NamedColor::BrightRed] = colors.bright.red;\n        self[NamedColor::BrightGreen] = colors.bright.green;\n        self[NamedColor::BrightYellow] = colors.bright.yellow;\n        self[NamedColor::BrightBlue] = colors.bright.blue;\n        self[NamedColor::BrightMagenta] = colors.bright.magenta;\n        self[NamedColor::BrightCyan] = colors.bright.cyan;\n        self[NamedColor::BrightWhite] = colors.bright.white;\n        self[NamedColor::BrightForeground] =\n            colors.primary.bright_foreground.unwrap_or(colors.primary.foreground);\n\n        // Foreground and background.\n        self[NamedColor::Foreground] = colors.primary.foreground;\n        self[NamedColor::Background] = colors.primary.background;\n\n        // Dims.\n        self[NamedColor::DimForeground] =\n            colors.primary.dim_foreground.unwrap_or(colors.primary.foreground * DIM_FACTOR);\n        match colors.dim {\n            Some(ref dim) => {\n                trace!(\"Using config-provided dim colors\");\n                self[NamedColor::DimBlack] = dim.black;\n                self[NamedColor::DimRed] = dim.red;\n                self[NamedColor::DimGreen] = dim.green;\n                self[NamedColor::DimYellow] = dim.yellow;\n                self[NamedColor::DimBlue] = dim.blue;\n                self[NamedColor::DimMagenta] = dim.magenta;\n                self[NamedColor::DimCyan] = dim.cyan;\n                self[NamedColor::DimWhite] = dim.white;\n            },\n            None => {\n                trace!(\"Deriving dim colors from normal colors\");\n                self[NamedColor::DimBlack] = colors.normal.black * DIM_FACTOR;\n                self[NamedColor::DimRed] = colors.normal.red * DIM_FACTOR;\n                self[NamedColor::DimGreen] = colors.normal.green * DIM_FACTOR;\n                self[NamedColor::DimYellow] = colors.normal.yellow * DIM_FACTOR;\n                self[NamedColor::DimBlue] = colors.normal.blue * DIM_FACTOR;\n                self[NamedColor::DimMagenta] = colors.normal.magenta * DIM_FACTOR;\n                self[NamedColor::DimCyan] = colors.normal.cyan * DIM_FACTOR;\n                self[NamedColor::DimWhite] = colors.normal.white * DIM_FACTOR;\n            },\n        }\n    }\n\n    pub fn fill_cube(&mut self, colors: &Colors) {\n        let mut index: usize = 16;\n        // Build colors.\n        for r in 0..6 {\n            for g in 0..6 {\n                for b in 0..6 {\n                    // Override colors 16..232 with the config (if present).\n                    if let Some(indexed_color) =\n                        colors.indexed_colors.iter().find(|ic| ic.index() == index as u8)\n                    {\n                        self[index] = indexed_color.color;\n                    } else {\n                        self[index] = Rgb::new(\n                            if r == 0 { 0 } else { r * 40 + 55 },\n                            if g == 0 { 0 } else { g * 40 + 55 },\n                            if b == 0 { 0 } else { b * 40 + 55 },\n                        );\n                    }\n                    index += 1;\n                }\n            }\n        }\n\n        debug_assert!(index == 232);\n    }\n\n    pub fn fill_gray_ramp(&mut self, colors: &Colors) {\n        let mut index: usize = 232;\n\n        for i in 0..24 {\n            // Index of the color is number of named colors + number of cube colors + i.\n            let color_index = 16 + 216 + i;\n\n            // Override colors 232..256 with the config (if present).\n            if let Some(indexed_color) =\n                colors.indexed_colors.iter().find(|ic| ic.index() == color_index)\n            {\n                self[index] = indexed_color.color;\n                index += 1;\n                continue;\n            }\n\n            let value = i * 10 + 8;\n            self[index] = Rgb::new(value, value, value);\n            index += 1;\n        }\n\n        debug_assert!(index == 256);\n    }\n}\n\nimpl Index<usize> for List {\n    type Output = Rgb;\n\n    #[inline]\n    fn index(&self, idx: usize) -> &Self::Output {\n        &self.0[idx]\n    }\n}\n\nimpl IndexMut<usize> for List {\n    #[inline]\n    fn index_mut(&mut self, idx: usize) -> &mut Self::Output {\n        &mut self.0[idx]\n    }\n}\n\nimpl Index<NamedColor> for List {\n    type Output = Rgb;\n\n    #[inline]\n    fn index(&self, idx: NamedColor) -> &Self::Output {\n        &self.0[idx as usize]\n    }\n}\n\nimpl IndexMut<NamedColor> for List {\n    #[inline]\n    fn index_mut(&mut self, idx: NamedColor) -> &mut Self::Output {\n        &mut self.0[idx as usize]\n    }\n}\n\n#[derive(SerdeReplace, Debug, Eq, PartialEq, Copy, Clone, Default)]\npub struct Rgb(pub VteRgb);\n\nimpl Rgb {\n    #[inline]\n    pub const fn new(r: u8, g: u8, b: u8) -> Self {\n        Self(VteRgb { r, g, b })\n    }\n\n    #[inline]\n    pub fn as_tuple(self) -> (u8, u8, u8) {\n        (self.0.r, self.0.g, self.0.b)\n    }\n}\n\nimpl From<VteRgb> for Rgb {\n    fn from(value: VteRgb) -> Self {\n        Self(value)\n    }\n}\n\nimpl Deref for Rgb {\n    type Target = VteRgb;\n\n    fn deref(&self) -> &Self::Target {\n        &self.0\n    }\n}\n\nimpl Mul<f32> for Rgb {\n    type Output = Rgb;\n\n    fn mul(self, rhs: f32) -> Self::Output {\n        Rgb(self.0 * rhs)\n    }\n}\n\nimpl Add<Rgb> for Rgb {\n    type Output = Rgb;\n\n    fn add(self, rhs: Rgb) -> Self::Output {\n        Rgb(self.0 + rhs.0)\n    }\n}\n\n/// Deserialize an Rgb from a hex string.\n///\n/// This is *not* the deserialize impl for Rgb since we want a symmetric\n/// serialize/deserialize impl for ref tests.\nimpl<'de> Deserialize<'de> for Rgb {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct RgbVisitor;\n\n        // Used for deserializing reftests.\n        #[derive(Deserialize)]\n        struct RgbDerivedDeser {\n            r: u8,\n            g: u8,\n            b: u8,\n        }\n\n        impl<'a> Visitor<'a> for RgbVisitor {\n            type Value = Rgb;\n\n            fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result {\n                f.write_str(\"hex color like #ff00ff\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Rgb, E>\n            where\n                E: serde::de::Error,\n            {\n                Rgb::from_str(value).map_err(|_| {\n                    E::custom(format!(\n                        \"failed to parse rgb color {value}; expected hex color like #ff00ff\"\n                    ))\n                })\n            }\n        }\n\n        // Return an error if the syntax is incorrect.\n        let value = toml::Value::deserialize(deserializer)?;\n\n        // Attempt to deserialize from struct form.\n        if let Ok(RgbDerivedDeser { r, g, b }) = RgbDerivedDeser::deserialize(value.clone()) {\n            return Ok(Rgb::new(r, g, b));\n        }\n\n        // Deserialize from hex notation (either 0xff00ff or #ff00ff).\n        value.deserialize_str(RgbVisitor).map_err(D::Error::custom)\n    }\n}\n\nimpl Display for Rgb {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"#{:02x}{:02x}{:02x}\", self.r, self.g, self.b)\n    }\n}\n\nimpl FromStr for Rgb {\n    type Err = ();\n\n    fn from_str(s: &str) -> Result<Rgb, ()> {\n        let chars = if s.starts_with(\"0x\") && s.len() == 8 {\n            &s[2..]\n        } else if s.starts_with('#') && s.len() == 7 {\n            &s[1..]\n        } else {\n            return Err(());\n        };\n\n        match u32::from_str_radix(chars, 16) {\n            Ok(mut color) => {\n                let b = (color & 0xff) as u8;\n                color >>= 8;\n                let g = (color & 0xff) as u8;\n                color >>= 8;\n                let r = color as u8;\n                Ok(Rgb::new(r, g, b))\n            },\n            Err(_) => Err(()),\n        }\n    }\n}\n\n/// RGB color optionally referencing the cell's foreground or background.\n#[derive(SerdeReplace, Copy, Clone, Debug, PartialEq, Eq)]\npub enum CellRgb {\n    CellForeground,\n    CellBackground,\n    Rgb(Rgb),\n}\n\nimpl CellRgb {\n    pub fn color(self, foreground: Rgb, background: Rgb) -> Rgb {\n        match self {\n            Self::CellForeground => foreground,\n            Self::CellBackground => background,\n            Self::Rgb(rgb) => rgb,\n        }\n    }\n}\n\nimpl Default for CellRgb {\n    fn default() -> Self {\n        Self::Rgb(Rgb::default())\n    }\n}\n\nimpl<'de> Deserialize<'de> for CellRgb {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        const EXPECTING: &str = \"CellForeground, CellBackground, or hex color like #ff00ff\";\n\n        struct CellRgbVisitor;\n        impl<'a> Visitor<'a> for CellRgbVisitor {\n            type Value = CellRgb;\n\n            fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result {\n                f.write_str(EXPECTING)\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<CellRgb, E>\n            where\n                E: serde::de::Error,\n            {\n                // Attempt to deserialize as enum constants.\n                match value {\n                    \"CellForeground\" => return Ok(CellRgb::CellForeground),\n                    \"CellBackground\" => return Ok(CellRgb::CellBackground),\n                    _ => (),\n                }\n\n                Rgb::from_str(value).map(CellRgb::Rgb).map_err(|_| {\n                    E::custom(format!(\"failed to parse color {value}; expected {EXPECTING}\"))\n                })\n            }\n        }\n\n        deserializer.deserialize_str(CellRgbVisitor).map_err(D::Error::custom)\n    }\n}\n\n// Path: alacritty/src/config/bell.rs\nuse std::time::Duration;\n\nuse alacritty_config_derive::ConfigDeserialize;\n\nuse crate::config::ui_config::Program;\nuse crate::display::color::Rgb;\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct BellConfig {\n    /// Visual bell animation function.\n    pub animation: BellAnimation,\n\n    /// Command to run on bell.\n    pub command: Option<Program>,\n\n    /// Visual bell flash color.\n    pub color: Rgb,\n\n    /// Visual bell duration in milliseconds.\n    duration: u16,\n}\n\nimpl Default for BellConfig {\n    fn default() -> Self {\n        Self {\n            color: Rgb::new(255, 255, 255),\n            animation: Default::default(),\n            command: Default::default(),\n            duration: Default::default(),\n        }\n    }\n}\n\nimpl BellConfig {\n    pub fn duration(&self) -> Duration {\n        Duration::from_millis(self.duration as u64)\n    }\n}\n\n/// `VisualBellAnimations` are modeled after a subset of CSS transitions and Robert\n/// Penner's Easing Functions.\n#[derive(ConfigDeserialize, Default, Clone, Copy, Debug, PartialEq, Eq)]\npub enum BellAnimation {\n    // CSS animation.\n    Ease,\n    // CSS animation.\n    EaseOut,\n    // Penner animation.\n    EaseOutSine,\n    // Penner animation.\n    EaseOutQuad,\n    // Penner animation.\n    EaseOutCubic,\n    // Penner animation.\n    EaseOutQuart,\n    // Penner animation.\n    EaseOutQuint,\n    // Penner animation.\n    EaseOutExpo,\n    // Penner animation.\n    EaseOutCirc,\n    // Penner animation.\n    #[default]\n    Linear,\n}\n\n// Path: alacritty/src/config/debug.rs\nuse log::LevelFilter;\n\nuse alacritty_config_derive::ConfigDeserialize;\n\n/// Debugging options.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]\npub struct Debug {\n    pub log_level: LevelFilter,\n\n    pub print_events: bool,\n\n    /// Keep the log file after quitting.\n    pub persistent_logging: bool,\n\n    /// Should show render timer.\n    pub render_timer: bool,\n\n    /// Highlight damage information produced by alacritty.\n    pub highlight_damage: bool,\n\n    /// The renderer alacritty should be using.\n    pub renderer: Option<RendererPreference>,\n\n    /// Use EGL as display API if the current platform allows it.\n    pub prefer_egl: bool,\n\n    /// Record ref test.\n    #[config(skip)]\n    pub ref_test: bool,\n}\n\nimpl Default for Debug {\n    fn default() -> Self {\n        Self {\n            log_level: LevelFilter::Warn,\n            print_events: Default::default(),\n            persistent_logging: Default::default(),\n            render_timer: Default::default(),\n            highlight_damage: Default::default(),\n            ref_test: Default::default(),\n            renderer: Default::default(),\n            prefer_egl: Default::default(),\n        }\n    }\n}\n\n/// The renderer configuration options.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]\npub enum RendererPreference {\n    /// OpenGL 3.3 renderer.\n    Glsl3,\n\n    /// GLES 2 renderer, with optional extensions like dual source blending.\n    Gles2,\n\n    /// Pure GLES 2 renderer.\n    Gles2Pure,\n}\n\n// Path: alacritty/src/config/window.rs\nuse std::fmt::{self, Formatter};\n\nuse log::{error, warn};\nuse serde::de::{self, MapAccess, Visitor};\nuse serde::{Deserialize, Deserializer, Serialize};\n\n#[cfg(target_os = \"macos\")]\nuse winit::platform::macos::OptionAsAlt as WinitOptionAsAlt;\nuse winit::window::{Fullscreen, Theme as WinitTheme};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\nuse crate::config::ui_config::{Delta, Percentage};\nuse crate::config::LOG_TARGET_CONFIG;\n\n/// Default Alacritty name, used for window title and class.\npub const DEFAULT_NAME: &str = \"Alacritty\";\n\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq)]\npub struct WindowConfig {\n    /// Initial position.\n    pub position: Option<Delta<i32>>,\n\n    /// Draw the window with title bar / borders.\n    pub decorations: Decorations,\n\n    /// Startup mode.\n    pub startup_mode: StartupMode,\n\n    /// XEmbed parent.\n    #[config(skip)]\n    pub embed: Option<u32>,\n\n    /// Spread out additional padding evenly.\n    pub dynamic_padding: bool,\n\n    /// Use dynamic title.\n    pub dynamic_title: bool,\n\n    /// Information to identify a particular window.\n    #[config(flatten)]\n    pub identity: Identity,\n\n    /// Background opacity from 0.0 to 1.0.\n    pub opacity: Percentage,\n\n    /// Request blur behind the window.\n    pub blur: bool,\n\n    /// Controls which `Option` key should be treated as `Alt`.\n    option_as_alt: OptionAsAlt,\n\n    /// Resize increments.\n    pub resize_increments: bool,\n\n    /// Pixel padding.\n    padding: Delta<u16>,\n\n    /// Initial dimensions.\n    dimensions: Dimensions,\n\n    /// System decorations theme variant.\n    decorations_theme_variant: Option<Theme>,\n}\n\nimpl Default for WindowConfig {\n    fn default() -> Self {\n        Self {\n            dynamic_title: true,\n            blur: Default::default(),\n            embed: Default::default(),\n            padding: Default::default(),\n            opacity: Default::default(),\n            position: Default::default(),\n            identity: Default::default(),\n            dimensions: Default::default(),\n            decorations: Default::default(),\n            startup_mode: Default::default(),\n            dynamic_padding: Default::default(),\n            resize_increments: Default::default(),\n            decorations_theme_variant: Default::default(),\n            option_as_alt: Default::default(),\n        }\n    }\n}\n\nimpl WindowConfig {\n    #[inline]\n    pub fn dimensions(&self) -> Option<Dimensions> {\n        let (lines, columns) = (self.dimensions.lines, self.dimensions.columns);\n        let (lines_is_non_zero, columns_is_non_zero) = (lines != 0, columns != 0);\n\n        if lines_is_non_zero && columns_is_non_zero {\n            // Return dimensions if both `lines` and `columns` are non-zero.\n            Some(self.dimensions)\n        } else if lines_is_non_zero || columns_is_non_zero {\n            // Warn if either `columns` or `lines` is non-zero.\n\n            let (zero_key, non_zero_key, non_zero_value) = if lines_is_non_zero {\n                (\"columns\", \"lines\", lines)\n            } else {\n                (\"lines\", \"columns\", columns)\n            };\n\n            warn!(\n                target: LOG_TARGET_CONFIG,\n                \"Both `lines` and `columns` must be non-zero for `window.dimensions` to take \\\n                 effect. Configured value of `{}` is 0 while that of `{}` is {}\",\n                zero_key,\n                non_zero_key,\n                non_zero_value,\n            );\n\n            None\n        } else {\n            None\n        }\n    }\n\n    #[inline]\n    pub fn padding(&self, scale_factor: f32) -> (f32, f32) {\n        let padding_x = (f32::from(self.padding.x) * scale_factor).floor();\n        let padding_y = (f32::from(self.padding.y) * scale_factor).floor();\n        (padding_x, padding_y)\n    }\n\n    #[inline]\n    pub fn fullscreen(&self) -> Option<Fullscreen> {\n        if self.startup_mode == StartupMode::Fullscreen {\n            Some(Fullscreen::Borderless(None))\n        } else {\n            None\n        }\n    }\n\n    #[inline]\n    pub fn maximized(&self) -> bool {\n        self.startup_mode == StartupMode::Maximized\n    }\n\n    #[cfg(target_os = \"macos\")]\n    pub fn option_as_alt(&self) -> WinitOptionAsAlt {\n        match self.option_as_alt {\n            OptionAsAlt::OnlyLeft => WinitOptionAsAlt::OnlyLeft,\n            OptionAsAlt::OnlyRight => WinitOptionAsAlt::OnlyRight,\n            OptionAsAlt::Both => WinitOptionAsAlt::Both,\n            OptionAsAlt::None => WinitOptionAsAlt::None,\n        }\n    }\n\n    pub fn theme(&self) -> Option<WinitTheme> {\n        self.decorations_theme_variant.map(WinitTheme::from)\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq, Eq)]\npub struct Identity {\n    /// Window title.\n    pub title: String,\n\n    /// Window class.\n    pub class: Class,\n}\n\nimpl Default for Identity {\n    fn default() -> Self {\n        Self { title: DEFAULT_NAME.into(), class: Default::default() }\n    }\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub enum StartupMode {\n    #[default]\n    Windowed,\n    Maximized,\n    Fullscreen,\n    SimpleFullscreen,\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub enum Decorations {\n    #[default]\n    Full,\n    Transparent,\n    Buttonless,\n    None,\n}\n\n/// Window Dimensions.\n///\n/// Newtype to avoid passing values incorrectly.\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub struct Dimensions {\n    /// Window width in character columns.\n    pub columns: usize,\n\n    /// Window Height in character lines.\n    pub lines: usize,\n}\n\n/// Window class hint.\n#[derive(SerdeReplace, Serialize, Debug, Clone, PartialEq, Eq)]\npub struct Class {\n    pub general: String,\n    pub instance: String,\n}\n\nimpl Class {\n    pub fn new(general: impl ToString, instance: impl ToString) -> Self {\n        Self { general: general.to_string(), instance: instance.to_string() }\n    }\n}\n\nimpl Default for Class {\n    fn default() -> Self {\n        Self::new(DEFAULT_NAME, DEFAULT_NAME)\n    }\n}\n\nimpl<'de> Deserialize<'de> for Class {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct ClassVisitor;\n        impl<'a> Visitor<'a> for ClassVisitor {\n            type Value = Class;\n\n            fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result {\n                f.write_str(\"a mapping\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>\n            where\n                E: de::Error,\n            {\n                Ok(Self::Value { instance: value.into(), ..Self::Value::default() })\n            }\n\n            fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>\n            where\n                M: MapAccess<'a>,\n            {\n                let mut class = Self::Value::default();\n\n                while let Some((key, value)) = map.next_entry::<String, toml::Value>()? {\n                    match key.as_str() {\n                        \"instance\" => match String::deserialize(value) {\n                            Ok(instance) => class.instance = instance,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: class.instance: {}\", err\n                                );\n                            },\n                        },\n                        \"general\" => match String::deserialize(value) {\n                            Ok(general) => class.general = general,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: class.instance: {}\", err\n                                );\n                            },\n                        },\n                        key => warn!(target: LOG_TARGET_CONFIG, \"Unrecognized class field: {key}\"),\n                    }\n                }\n\n                Ok(class)\n            }\n        }\n\n        deserializer.deserialize_any(ClassVisitor)\n    }\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Clone, Copy, PartialEq, Eq)]\npub enum OptionAsAlt {\n    /// The left `Option` key is treated as `Alt`.\n    OnlyLeft,\n\n    /// The right `Option` key is treated as `Alt`.\n    OnlyRight,\n\n    /// Both `Option` keys are treated as `Alt`.\n    Both,\n\n    /// No special handling is applied for `Option` key.\n    #[default]\n    None,\n}\n\n/// System decorations theme variant.\n#[derive(ConfigDeserialize, Debug, Clone, Copy, PartialEq, Eq)]\npub enum Theme {\n    Light,\n    Dark,\n}\n\nimpl From<Theme> for WinitTheme {\n    fn from(theme: Theme) -> Self {\n        match theme {\n            Theme::Light => WinitTheme::Light,\n            Theme::Dark => WinitTheme::Dark,\n        }\n    }\n}\n\n// Path: alacritty/src/cli.rs\nuse std::cmp::max;\nuse std::collections::HashMap;\nuse std::ops::{Deref, DerefMut};\nuse std::path::PathBuf;\nuse std::rc::Rc;\n\nuse alacritty_config::SerdeReplace;\nuse clap::{ArgAction, Args, Parser, Subcommand, ValueHint};\nuse log::{error, LevelFilter};\nuse serde::{Deserialize, Serialize};\nuse toml::Value;\n\nuse alacritty_terminal::tty::Options as PtyOptions;\n\nuse crate::config::ui_config::Program;\nuse crate::config::window::{Class, Identity};\nuse crate::config::UiConfig;\nuse crate::logging::LOG_TARGET_IPC_CONFIG;\n\n/// CLI options for the main Alacritty executable.\n#[derive(Parser, Default, Debug)]\n#[clap(author, about, version = env!(\"VERSION\"))]\npub struct Options {\n    /// Print all events to STDOUT.\n    #[clap(long)]\n    pub print_events: bool,\n\n    /// Generates ref test.\n    #[clap(long)]\n    pub ref_test: bool,\n\n    /// X11 window ID to embed Alacritty within (decimal or hexadecimal with \"0x\" prefix).\n    #[clap(long)]\n    pub embed: Option<String>,\n\n    /// Specify alternative configuration file [default:\n    /// $XDG_CONFIG_HOME/alacritty/alacritty.toml].\n    #[cfg(not(any(target_os = \"macos\", windows)))]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Specify alternative configuration file [default: %APPDATA%\\alacritty\\alacritty.toml].\n    #[cfg(windows)]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Specify alternative configuration file [default: $HOME/.config/alacritty/alacritty.toml].\n    #[cfg(target_os = \"macos\")]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Path for IPC socket creation.\n    #[cfg(unix)]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub socket: Option<PathBuf>,\n\n    /// Reduces the level of verbosity (the min level is -qq).\n    #[clap(short, conflicts_with(\"verbose\"), action = ArgAction::Count)]\n    quiet: u8,\n\n    /// Increases the level of verbosity (the max level is -vvv).\n    #[clap(short, conflicts_with(\"quiet\"), action = ArgAction::Count)]\n    verbose: u8,\n\n    /// CLI options for config overrides.\n...\n", "needle_token_start": 281, "needle_token_end": 320, "code_context_ntokens": 10800, "output": ["```rust\npub fn footer_bar_foreground(&self) -> Rgb {\n    self.footer_bar.foreground.unwrap_or(self.primary.background)\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "dynamic_title_ignoring_options_by_default", "language": "rust", "path": "alacritty/src/cli.rs", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To verify that the default configuration setting related to window titles remains unchanged when no specific command-line options are provided to modify it.\n2. **Input**: A default configuration object without any modifications.\n3. **Output**: A confirmation that the initial setting for window titles has not been altered after attempting to apply non-existent modifications.\n4. **Procedure**: \n   - Initialize a configuration object with default settings.\n   - Record the initial setting related to window titles.\n   - Attempt to apply modifications using an empty set of options.\n   - Check and assert that the window title setting remains the same as the initial value, confirming that no changes have been made.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/config/bell.rs\nuse std::time::Duration;\n\nuse alacritty_config_derive::ConfigDeserialize;\n\nuse crate::config::ui_config::Program;\nuse crate::display::color::Rgb;\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct BellConfig {\n    /// Visual bell animation function.\n    pub animation: BellAnimation,\n\n    /// Command to run on bell.\n...\n// Path: alacritty/src/config/debug.rs\nuse log::LevelFilter;\n\nuse alacritty_config_derive::ConfigDeserialize;\n\n/// Debugging options.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]\npub struct Debug {\n    pub log_level: LevelFilter,\n\n    pub print_events: bool,\n\n    /// Keep the log file after quitting.\n    pub persistent_logging: bool,\n\n    /// Should show render timer.\n    pub render_timer: bool,\n\n    /// Highlight damage information produced by alacritty.\n    pub highlight_damage: bool,\n\n    /// The renderer alacritty should be using.\n    pub renderer: Option<RendererPreference>,\n\n    /// Use EGL as display API if the current platform allows it.\n    pub prefer_egl: bool,\n\n    /// Record ref test.\n    #[config(skip)]\n    pub ref_test: bool,\n}\n\nimpl Default for Debug {\n    fn default() -> Self {\n        Self {\n            log_level: LevelFilter::Warn,\n            print_events: Default::default(),\n            persistent_logging: Default::default(),\n            render_timer: Default::default(),\n            highlight_damage: Default::default(),\n            ref_test: Default::default(),\n            renderer: Default::default(),\n            prefer_egl: Default::default(),\n        }\n    }\n}\n\n/// The renderer configuration options.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]\npub enum RendererPreference {\n    /// OpenGL 3.3 renderer.\n    Glsl3,\n\n    /// GLES 2 renderer, with optional extensions like dual source blending.\n    Gles2,\n\n    /// Pure GLES 2 renderer.\n    Gles2Pure,\n}\n\n// Path: alacritty/src/config/window.rs\nuse std::fmt::{self, Formatter};\n\nuse log::{error, warn};\nuse serde::de::{self, MapAccess, Visitor};\nuse serde::{Deserialize, Deserializer, Serialize};\n\n#[cfg(target_os = \"macos\")]\nuse winit::platform::macos::OptionAsAlt as WinitOptionAsAlt;\nuse winit::window::{Fullscreen, Theme as WinitTheme};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\nuse crate::config::ui_config::{Delta, Percentage};\nuse crate::config::LOG_TARGET_CONFIG;\n\n/// Default Alacritty name, used for window title and class.\npub const DEFAULT_NAME: &str = \"Alacritty\";\n\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq)]\npub struct WindowConfig {\n    /// Initial position.\n    pub position: Option<Delta<i32>>,\n\n    /// Draw the window with title bar / borders.\n    pub decorations: Decorations,\n\n    /// Startup mode.\n    pub startup_mode: StartupMode,\n\n    /// XEmbed parent.\n    #[config(skip)]\n    pub embed: Option<u32>,\n\n    /// Spread out additional padding evenly.\n    pub dynamic_padding: bool,\n\n    /// Use dynamic title.\n    pub dynamic_title: bool,\n\n    /// Information to identify a particular window.\n    #[config(flatten)]\n    pub identity: Identity,\n\n    /// Background opacity from 0.0 to 1.0.\n    pub opacity: Percentage,\n\n    /// Request blur behind the window.\n    pub blur: bool,\n\n    /// Controls which `Option` key should be treated as `Alt`.\n    option_as_alt: OptionAsAlt,\n\n    /// Resize increments.\n    pub resize_increments: bool,\n\n    /// Pixel padding.\n    padding: Delta<u16>,\n\n    /// Initial dimensions.\n    dimensions: Dimensions,\n\n    /// System decorations theme variant.\n    decorations_theme_variant: Option<Theme>,\n}\n\nimpl Default for WindowConfig {\n    fn default() -> Self {\n        Self {\n            dynamic_title: true,\n            blur: Default::default(),\n            embed: Default::default(),\n            padding: Default::default(),\n            opacity: Default::default(),\n            position: Default::default(),\n            identity: Default::default(),\n            dimensions: Default::default(),\n            decorations: Default::default(),\n            startup_mode: Default::default(),\n            dynamic_padding: Default::default(),\n            resize_increments: Default::default(),\n            decorations_theme_variant: Default::default(),\n            option_as_alt: Default::default(),\n        }\n    }\n}\n\nimpl WindowConfig {\n    #[inline]\n    pub fn dimensions(&self) -> Option<Dimensions> {\n        let (lines, columns) = (self.dimensions.lines, self.dimensions.columns);\n        let (lines_is_non_zero, columns_is_non_zero) = (lines != 0, columns != 0);\n\n        if lines_is_non_zero && columns_is_non_zero {\n            // Return dimensions if both `lines` and `columns` are non-zero.\n            Some(self.dimensions)\n        } else if lines_is_non_zero || columns_is_non_zero {\n            // Warn if either `columns` or `lines` is non-zero.\n\n            let (zero_key, non_zero_key, non_zero_value) = if lines_is_non_zero {\n                (\"columns\", \"lines\", lines)\n            } else {\n                (\"lines\", \"columns\", columns)\n            };\n\n            warn!(\n                target: LOG_TARGET_CONFIG,\n                \"Both `lines` and `columns` must be non-zero for `window.dimensions` to take \\\n                 effect. Configured value of `{}` is 0 while that of `{}` is {}\",\n                zero_key,\n                non_zero_key,\n                non_zero_value,\n            );\n\n            None\n        } else {\n            None\n        }\n    }\n\n    #[inline]\n    pub fn padding(&self, scale_factor: f32) -> (f32, f32) {\n        let padding_x = (f32::from(self.padding.x) * scale_factor).floor();\n        let padding_y = (f32::from(self.padding.y) * scale_factor).floor();\n        (padding_x, padding_y)\n    }\n\n    #[inline]\n    pub fn fullscreen(&self) -> Option<Fullscreen> {\n        if self.startup_mode == StartupMode::Fullscreen {\n            Some(Fullscreen::Borderless(None))\n        } else {\n            None\n        }\n    }\n\n    #[inline]\n    pub fn maximized(&self) -> bool {\n        self.startup_mode == StartupMode::Maximized\n    }\n\n    #[cfg(target_os = \"macos\")]\n    pub fn option_as_alt(&self) -> WinitOptionAsAlt {\n        match self.option_as_alt {\n            OptionAsAlt::OnlyLeft => WinitOptionAsAlt::OnlyLeft,\n            OptionAsAlt::OnlyRight => WinitOptionAsAlt::OnlyRight,\n            OptionAsAlt::Both => WinitOptionAsAlt::Both,\n            OptionAsAlt::None => WinitOptionAsAlt::None,\n        }\n    }\n\n    pub fn theme(&self) -> Option<WinitTheme> {\n        self.decorations_theme_variant.map(WinitTheme::from)\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq, Eq)]\npub struct Identity {\n    /// Window title.\n    pub title: String,\n\n    /// Window class.\n    pub class: Class,\n}\n\nimpl Default for Identity {\n    fn default() -> Self {\n        Self { title: DEFAULT_NAME.into(), class: Default::default() }\n    }\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub enum StartupMode {\n    #[default]\n    Windowed,\n    Maximized,\n    Fullscreen,\n    SimpleFullscreen,\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub enum Decorations {\n    #[default]\n    Full,\n    Transparent,\n    Buttonless,\n    None,\n}\n\n/// Window Dimensions.\n///\n/// Newtype to avoid passing values incorrectly.\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub struct Dimensions {\n    /// Window width in character columns.\n    pub columns: usize,\n\n    /// Window Height in character lines.\n    pub lines: usize,\n}\n\n/// Window class hint.\n#[derive(SerdeReplace, Serialize, Debug, Clone, PartialEq, Eq)]\npub struct Class {\n    pub general: String,\n    pub instance: String,\n}\n\nimpl Class {\n    pub fn new(general: impl ToString, instance: impl ToString) -> Self {\n        Self { general: general.to_string(), instance: instance.to_string() }\n    }\n}\n\nimpl Default for Class {\n    fn default() -> Self {\n        Self::new(DEFAULT_NAME, DEFAULT_NAME)\n    }\n}\n\nimpl<'de> Deserialize<'de> for Class {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct ClassVisitor;\n        impl<'a> Visitor<'a> for ClassVisitor {\n            type Value = Class;\n\n            fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result {\n                f.write_str(\"a mapping\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>\n            where\n                E: de::Error,\n            {\n                Ok(Self::Value { instance: value.into(), ..Self::Value::default() })\n            }\n\n            fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>\n            where\n                M: MapAccess<'a>,\n            {\n                let mut class = Self::Value::default();\n\n                while let Some((key, value)) = map.next_entry::<String, toml::Value>()? {\n                    match key.as_str() {\n                        \"instance\" => match String::deserialize(value) {\n                            Ok(instance) => class.instance = instance,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: class.instance: {}\", err\n                                );\n                            },\n                        },\n                        \"general\" => match String::deserialize(value) {\n                            Ok(general) => class.general = general,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: class.instance: {}\", err\n                                );\n                            },\n                        },\n                        key => warn!(target: LOG_TARGET_CONFIG, \"Unrecognized class field: {key}\"),\n                    }\n                }\n\n                Ok(class)\n            }\n        }\n\n        deserializer.deserialize_any(ClassVisitor)\n    }\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Clone, Copy, PartialEq, Eq)]\npub enum OptionAsAlt {\n    /// The left `Option` key is treated as `Alt`.\n    OnlyLeft,\n\n    /// The right `Option` key is treated as `Alt`.\n    OnlyRight,\n\n    /// Both `Option` keys are treated as `Alt`.\n    Both,\n\n    /// No special handling is applied for `Option` key.\n    #[default]\n    None,\n}\n\n/// System decorations theme variant.\n#[derive(ConfigDeserialize, Debug, Clone, Copy, PartialEq, Eq)]\npub enum Theme {\n    Light,\n    Dark,\n}\n\nimpl From<Theme> for WinitTheme {\n    fn from(theme: Theme) -> Self {\n        match theme {\n            Theme::Light => WinitTheme::Light,\n            Theme::Dark => WinitTheme::Dark,\n        }\n    }\n}\n\n// Path: alacritty/src/cli.rs\nuse std::cmp::max;\nuse std::collections::HashMap;\nuse std::ops::{Deref, DerefMut};\nuse std::path::PathBuf;\nuse std::rc::Rc;\n\nuse alacritty_config::SerdeReplace;\nuse clap::{ArgAction, Args, Parser, Subcommand, ValueHint};\nuse log::{error, LevelFilter};\nuse serde::{Deserialize, Serialize};\nuse toml::Value;\n\nuse alacritty_terminal::tty::Options as PtyOptions;\n\nuse crate::config::ui_config::Program;\nuse crate::config::window::{Class, Identity};\nuse crate::config::UiConfig;\nuse crate::logging::LOG_TARGET_IPC_CONFIG;\n\n/// CLI options for the main Alacritty executable.\n#[derive(Parser, Default, Debug)]\n#[clap(author, about, version = env!(\"VERSION\"))]\npub struct Options {\n    /// Print all events to STDOUT.\n    #[clap(long)]\n    pub print_events: bool,\n\n    /// Generates ref test.\n    #[clap(long)]\n    pub ref_test: bool,\n\n    /// X11 window ID to embed Alacritty within (decimal or hexadecimal with \"0x\" prefix).\n    #[clap(long)]\n    pub embed: Option<String>,\n\n    /// Specify alternative configuration file [default:\n    /// $XDG_CONFIG_HOME/alacritty/alacritty.toml].\n    #[cfg(not(any(target_os = \"macos\", windows)))]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Specify alternative configuration file [default: %APPDATA%\\alacritty\\alacritty.toml].\n    #[cfg(windows)]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Specify alternative configuration file [default: $HOME/.config/alacritty/alacritty.toml].\n    #[cfg(target_os = \"macos\")]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Path for IPC socket creation.\n    #[cfg(unix)]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub socket: Option<PathBuf>,\n\n    /// Reduces the level of verbosity (the min level is -qq).\n    #[clap(short, conflicts_with(\"verbose\"), action = ArgAction::Count)]\n    quiet: u8,\n\n    /// Increases the level of verbosity (the max level is -vvv).\n    #[clap(short, conflicts_with(\"quiet\"), action = ArgAction::Count)]\n    verbose: u8,\n\n    /// CLI options for config overrides.\n    #[clap(skip)]\n    pub config_options: ParsedOptions,\n\n    /// Options which can be passed via IPC.\n    #[clap(flatten)]\n    pub window_options: WindowOptions,\n\n    /// Subcommand passed to the CLI.\n    #[clap(subcommand)]\n    pub subcommands: Option<Subcommands>,\n}\n\nimpl Options {\n    pub fn new() -> Self {\n        let mut options = Self::parse();\n\n        // Parse CLI config overrides.\n        options.config_options = options.window_options.config_overrides();\n\n        options\n    }\n\n    /// Override configuration file with options from the CLI.\n    pub fn override_config(&mut self, config: &mut UiConfig) {\n        #[cfg(unix)]\n        {\n            config.ipc_socket |= self.socket.is_some();\n        }\n\n        config.window.dynamic_title &= self.window_options.window_identity.title.is_none();\n        config.window.embed = self.embed.as_ref().and_then(|embed| parse_hex_or_decimal(embed));\n        config.debug.print_events |= self.print_events;\n        config.debug.log_level = max(config.debug.log_level, self.log_level());\n        config.debug.ref_test |= self.ref_test;\n\n        if config.debug.print_events {\n            config.debug.log_level = max(config.debug.log_level, LevelFilter::Info);\n        }\n\n        // Replace CLI options.\n        self.config_options.override_config(config);\n    }\n\n    /// Logging filter level.\n    pub fn log_level(&self) -> LevelFilter {\n        match (self.quiet, self.verbose) {\n            // Force at least `Info` level for `--print-events`.\n            (_, 0) if self.print_events => LevelFilter::Info,\n\n            // Default.\n            (0, 0) => LevelFilter::Warn,\n\n            // Verbose.\n            (_, 1) => LevelFilter::Info,\n            (_, 2) => LevelFilter::Debug,\n            (0, _) => LevelFilter::Trace,\n\n            // Quiet.\n            (1, _) => LevelFilter::Error,\n            (..) => LevelFilter::Off,\n        }\n    }\n}\n\n/// Parse the class CLI parameter.\nfn parse_class(input: &str) -> Result<Class, String> {\n    let (general, instance) = match input.split_once(',') {\n        // Warn the user if they've passed too many values.\n        Some((_, instance)) if instance.contains(',') => {\n            return Err(String::from(\"Too many parameters\"))\n        },\n        Some((general, instance)) => (general, instance),\n        None => (input, input),\n    };\n\n    Ok(Class::new(general, instance))\n}\n\n/// Convert to hex if possible, else decimal\nfn parse_hex_or_decimal(input: &str) -> Option<u32> {\n    input\n        .strip_prefix(\"0x\")\n        .and_then(|value| u32::from_str_radix(value, 16).ok())\n        .or_else(|| input.parse().ok())\n}\n\n/// Terminal specific cli options which can be passed to new windows via IPC.\n#[derive(Serialize, Deserialize, Args, Default, Debug, Clone, PartialEq, Eq)]\npub struct TerminalOptions {\n    /// Start the shell in the specified working directory.\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub working_directory: Option<PathBuf>,\n\n    /// Remain open after child process exit.\n    #[clap(long)]\n    pub hold: bool,\n\n    /// Command and args to execute (must be last argument).\n    #[clap(short = 'e', long, allow_hyphen_values = true, num_args = 1..)]\n    command: Vec<String>,\n}\n\nimpl TerminalOptions {\n    /// Shell override passed through the CLI.\n    pub fn command(&self) -> Option<Program> {\n        let (program, args) = self.command.split_first()?;\n        Some(Program::WithArgs { program: program.clone(), args: args.to_vec() })\n    }\n\n    /// Override the [`PtyOptions`]'s fields with the [`TerminalOptions`].\n    pub fn override_pty_config(&self, pty_config: &mut PtyOptions) {\n        if let Some(working_directory) = &self.working_directory {\n            if working_directory.is_dir() {\n                pty_config.working_directory = Some(working_directory.to_owned());\n            } else {\n                error!(\"Invalid working directory: {:?}\", working_directory);\n            }\n        }\n\n        if let Some(command) = self.command() {\n            pty_config.shell = Some(command.into());\n        }\n\n        pty_config.hold |= self.hold;\n    }\n}\n\nimpl From<TerminalOptions> for PtyOptions {\n    fn from(mut options: TerminalOptions) -> Self {\n        PtyOptions {\n            working_directory: options.working_directory.take(),\n            shell: options.command().map(Into::into),\n            hold: options.hold,\n            env: HashMap::new(),\n        }\n    }\n}\n\n/// Window specific cli options which can be passed to new windows via IPC.\n#[derive(Serialize, Deserialize, Args, Default, Debug, Clone, PartialEq, Eq)]\npub struct WindowIdentity {\n    /// Defines the window title [default: Alacritty].\n    #[clap(short = 'T', short_alias('t'), long)]\n    pub title: Option<String>,\n\n    /// Defines window class/app_id on X11/Wayland [default: Alacritty].\n    #[clap(long, value_name = \"general> | <general>,<instance\", value_parser = parse_class)]\n    pub class: Option<Class>,\n}\n\nimpl WindowIdentity {\n    /// Override the [`WindowIdentity`]'s fields with the [`WindowOptions`].\n    pub fn override_identity_config(&self, identity: &mut Identity) {\n        if let Some(title) = &self.title {\n            identity.title = title.clone();\n        }\n        if let Some(class) = &self.class {\n            identity.class = class.clone();\n        }\n    }\n}\n\n/// Available CLI subcommands.\n#[derive(Subcommand, Debug)]\npub enum Subcommands {\n    #[cfg(unix)]\n    Msg(MessageOptions),\n    Migrate(MigrateOptions),\n}\n\n/// Send a message to the Alacritty socket.\n#[cfg(unix)]\n#[derive(Args, Debug)]\npub struct MessageOptions {\n    /// IPC socket connection path override.\n    #[clap(short, long, value_hint = ValueHint::FilePath)]\n    pub socket: Option<PathBuf>,\n\n    /// Message which should be sent.\n    #[clap(subcommand)]\n    pub message: SocketMessage,\n}\n\n/// Available socket messages.\n#[cfg(unix)]\n#[derive(Subcommand, Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]\npub enum SocketMessage {\n    /// Create a new window in the same Alacritty process.\n    CreateWindow(WindowOptions),\n\n    /// Update the Alacritty configuration.\n    Config(IpcConfig),\n}\n\n/// Migrate the configuration file.\n#[derive(Args, Clone, Debug)]\npub struct MigrateOptions {\n    /// Path to the configuration file.\n    #[clap(short, long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Only output TOML config to STDOUT.\n    #[clap(short, long)]\n    pub dry_run: bool,\n\n    /// Do not recurse over imports.\n    #[clap(short = 'i', long)]\n    pub skip_imports: bool,\n\n    /// Do not move renamed fields to their new location.\n    #[clap(long)]\n    pub skip_renames: bool,\n\n    #[clap(short, long)]\n    /// Do not output to STDOUT.\n    pub silent: bool,\n}\n\n/// Subset of options that we pass to 'create-window' IPC subcommand.\n#[derive(Serialize, Deserialize, Args, Default, Clone, Debug, PartialEq, Eq)]\npub struct WindowOptions {\n    /// Terminal options which can be passed via IPC.\n    #[clap(flatten)]\n    pub terminal_options: TerminalOptions,\n\n    #[clap(flatten)]\n    /// Window options which could be passed via IPC.\n    pub window_identity: WindowIdentity,\n\n    #[clap(skip)]\n    #[cfg(target_os = \"macos\")]\n    /// The window tabbing identifier to use when building a window.\n    pub window_tabbing_id: Option<String>,\n\n    /// Override configuration file options [example: 'cursor.style=\"Beam\"'].\n    #[clap(short = 'o', long, num_args = 1..)]\n    option: Vec<String>,\n}\n\nimpl WindowOptions {\n    /// Get the parsed set of CLI config overrides.\n    pub fn config_overrides(&self) -> ParsedOptions {\n        ParsedOptions::from_options(&self.option)\n    }\n}\n\n/// Parameters to the `config` IPC subcommand.\n#[cfg(unix)]\n#[derive(Args, Serialize, Deserialize, Default, Debug, Clone, PartialEq, Eq)]\npub struct IpcConfig {\n    /// Configuration file options [example: 'cursor.style=\"Beam\"'].\n    #[clap(required = true, value_name = \"CONFIG_OPTIONS\")]\n    pub options: Vec<String>,\n\n    /// Window ID for the new config.\n    ///\n    /// Use `-1` to apply this change to all windows.\n    #[clap(short, long, allow_hyphen_values = true, env = \"ALACRITTY_WINDOW_ID\")]\n    pub window_id: Option<i128>,\n\n    /// Clear all runtime configuration changes.\n    #[clap(short, long, conflicts_with = \"options\")]\n    pub reset: bool,\n}\n\n/// Parsed CLI config overrides.\n#[derive(Debug, Default)]\npub struct ParsedOptions {\n    config_options: Vec<(String, Value)>,\n}\n\nimpl ParsedOptions {\n    /// Parse CLI config overrides.\n    pub fn from_options(options: &[String]) -> Self {\n        let mut config_options = Vec::new();\n\n        for option in options {\n            let parsed = match toml::from_str(option) {\n                Ok(parsed) => parsed,\n                Err(err) => {\n                    eprintln!(\"Ignoring invalid CLI option '{option}': {err}\");\n                    continue;\n                },\n            };\n            config_options.push((option.clone(), parsed));\n        }\n\n        Self { config_options }\n    }\n\n    /// Apply CLI config overrides, removing broken ones.\n    pub fn override_config(&mut self, config: &mut UiConfig) {\n        let mut i = 0;\n        while i < self.config_options.len() {\n            let (option, parsed) = &self.config_options[i];\n            match config.replace(parsed.clone()) {\n                Err(err) => {\n                    error!(\n                        target: LOG_TARGET_IPC_CONFIG,\n                        \"Unable to override option '{}': {}\", option, err\n                    );\n                    self.config_options.swap_remove(i);\n                },\n                Ok(_) => i += 1,\n            }\n        }\n    }\n\n    /// Apply CLI config overrides to a CoW config.\n    pub fn override_config_rc(&mut self, config: Rc<UiConfig>) -> Rc<UiConfig> {\n        // Skip clone without write requirement.\n        if self.config_options.is_empty() {\n            return config;\n        }\n\n        // Override cloned config.\n        let mut config = (*config).clone();\n        self.override_config(&mut config);\n\n        Rc::new(config)\n    }\n}\n\nimpl Deref for ParsedOptions {\n    type Target = Vec<(String, Value)>;\n\n    fn deref(&self) -> &Self::Target {\n        &self.config_options\n    }\n}\n\nimpl DerefMut for ParsedOptions {\n    fn deref_mut(&mut self) -> &mut Self::Target {\n        &mut self.config_options\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[cfg(target_os = \"linux\")]\n    use std::fs::File;\n    #[cfg(target_os = \"linux\")]\n    use std::io::Read;\n\n    #[cfg(target_os = \"linux\")]\n    use clap::CommandFactory;\n    #[cfg(target_os = \"linux\")]\n    use clap_complete::Shell;\n    use toml::Table;\n\n    #[test]\n    \nfn dynamic_title_ignoring_options_by_default() {\n        let mut config = UiConfig::default();\n        let old_dynamic_title = config.window.dynamic_title;\n\n        Options::default().override_config(&mut config);\n\n        assert_eq!(old_dynamic_title, config.window.dynamic_title);\n    }\n\n    #[test]\n    fn dynamic_title_overridden_by_options() {\n        let mut config = UiConfig::default();\n\n        let title = Some(String::from(\"foo\"));\n        let window_identity = WindowIdentity { title, ..WindowIdentity::default() };\n        let new_window_options = WindowOptions { window_identity, ..WindowOptions::default() };\n        let mut options = Options { window_options: new_window_options, ..Options::default() };\n        options.override_config(&mut config);\n\n        assert!(!config.window.dynamic_title);\n    }\n\n    #[test]\n    fn dynamic_title_not_overridden_by_config() {\n        let mut config = UiConfig::default();\n\n        config.window.identity.title = \"foo\".to_owned();\n        Options::default().override_config(&mut config);\n\n        assert!(config.window.dynamic_title);\n    }\n\n    #[test]\n    fn valid_option_as_value() {\n        // Test with a single field.\n        let value: Value = toml::from_str(\"field=true\").unwrap();\n\n        let mut table = Table::new();\n        table.insert(String::from(\"field\"), Value::Boolean(true));\n\n        assert_eq!(value, Value::Table(table));\n\n        // Test with nested fields\n        let value: Value = toml::from_str(\"parent.field=true\").unwrap();\n\n        let mut parent_table = Table::new();\n        parent_table.insert(String::from(\"field\"), Value::Boolean(true));\n        let mut table = Table::new();\n        table.insert(String::from(\"parent\"), Value::Table(parent_table));\n\n        assert_eq!(value, Value::Table(table));\n    }\n\n    #[test]\n    fn invalid_option_as_value() {\n        let value = toml::from_str::<Value>(\"}\");\n        assert!(value.is_err());\n    }\n\n    #[test]\n    fn float_option_as_value() {\n        let value: Value = toml::from_str(\"float=3.4\").unwrap();\n\n        let mut expected = Table::new();\n        expected.insert(String::from(\"float\"), Value::Float(3.4));\n\n        assert_eq!(value, Value::Table(expected));\n    }\n\n    #[test]\n    fn parse_instance_class() {\n        let class = parse_class(\"one\").unwrap();\n        assert_eq!(class.general, \"one\");\n        assert_eq!(class.instance, \"one\");\n    }\n\n    #[test]\n    fn parse_general_class() {\n        let class = parse_class(\"one,two\").unwrap();\n        assert_eq!(class.general, \"one\");\n        assert_eq!(class.instance, \"two\");\n    }\n\n    #[test]\n    fn parse_invalid_class() {\n        let class = parse_class(\"one,two,three\");\n        assert!(class.is_err());\n    }\n\n    #[test]\n    fn valid_decimal() {\n        let value = parse_hex_or_decimal(\"10485773\");\n        assert_eq!(value, Some(10485773));\n    }\n\n    #[test]\n    fn valid_hex_to_decimal() {\n        let value = parse_hex_or_decimal(\"0xa0000d\");\n        assert_eq!(value, Some(10485773));\n    }\n\n    #[test]\n    fn invalid_hex_to_decimal() {\n        let value = parse_hex_or_decimal(\"0xa0xx0d\");\n        assert_eq!(value, None);\n    }\n\n    #[cfg(target_os = \"linux\")]\n    #[test]\n    fn completions() {\n        let mut clap = Options::command();\n\n        for (shell, file) in &[\n            (Shell::Bash, \"alacritty.bash\"),\n            (Shell::Fish, \"alacritty.fish\"),\n            (Shell::Zsh, \"_alacritty\"),\n        ] {\n            let mut generated = Vec::new();\n            clap_complete::generate(*shell, &mut clap, \"alacritty\", &mut generated);\n            let generated = String::from_utf8_lossy(&generated);\n\n            let mut completion = String::new();\n            let mut file = File::open(format!(\"../extra/completions/{}\", file)).unwrap();\n            file.read_to_string(&mut completion).unwrap();\n\n            assert_eq!(generated, completion);\n        }\n\n        // NOTE: Use this to generate new completions.\n        //\n        // let mut file = File::create(\"../extra/completions/alacritty.bash\").unwrap();\n        // clap_complete::generate(Shell::Bash, &mut clap, \"alacritty\", &mut file);\n        // let mut file = File::create(\"../extra/completions/alacritty.fish\").unwrap();\n        // clap_complete::generate(Shell::Fish, &mut clap, \"alacritty\", &mut file);\n        // let mut file = File::create(\"../extra/completions/_alacritty\").unwrap();\n        // clap_complete::generate(Shell::Zsh, &mut clap, \"alacritty\", &mut file);\n    }\n}\n\n// Path: alacritty/src/message_bar.rs\nuse std::collections::VecDeque;\n\nuse unicode_width::UnicodeWidthChar;\n\nuse alacritty_terminal::grid::Dimensions;\n\nuse crate::display::SizeInfo;\n\npub const CLOSE_BUTTON_TEXT: &str = \"[X]\";\nconst CLOSE_BUTTON_PADDING: usize = 1;\nconst MIN_FREE_LINES: usize = 3;\nconst TRUNCATED_MESSAGE: &str = \"[MESSAGE TRUNCATED]\";\n\n/// Message for display in the MessageBuffer.\n#[derive(Debug, Eq, PartialEq, Clone)]\npub struct Message {\n    text: String,\n    ty: MessageType,\n    target: Option<String>,\n}\n\n/// Purpose of the message.\n#[derive(Debug, Eq, PartialEq, Clone, Copy)]\npub enum MessageType {\n    /// A message represents an error.\n    Error,\n\n    /// A message represents a warning.\n    Warning,\n}\n\nimpl Message {\n    /// Create a new message.\n    pub fn new(text: String, ty: MessageType) -> Message {\n        Message { text, ty, target: None }\n    }\n\n    /// Formatted message text lines.\n    pub fn text(&self, size_info: &SizeInfo) -> Vec<String> {\n        let num_cols = size_info.columns();\n        let total_lines =\n            (size_info.height() - 2. * size_info.padding_y()) / size_info.cell_height();\n        let max_lines = (total_lines as usize).saturating_sub(MIN_FREE_LINES);\n        let button_len = CLOSE_BUTTON_TEXT.chars().count();\n\n        // Split line to fit the screen.\n        let mut lines = Vec::new();\n        let mut line = String::new();\n        let mut line_len = 0;\n        for c in self.text.trim().chars() {\n            if c == '\\n'\n                || line_len == num_cols\n                // Keep space in first line for button.\n                || (lines.is_empty()\n                    && num_cols >= button_len\n                    && line_len == num_cols.saturating_sub(button_len + CLOSE_BUTTON_PADDING))\n            {\n                let is_whitespace = c.is_whitespace();\n\n                // Attempt to wrap on word boundaries.\n                let mut new_line = String::new();\n                if let Some(index) = line.rfind(char::is_whitespace).filter(|_| !is_whitespace) {\n                    let split = line.split_off(index + 1);\n                    line.pop();\n                    new_line = split;\n                }\n\n                lines.push(Self::pad_text(line, num_cols));\n                line = new_line;\n                line_len = line.chars().count();\n\n                // Do not append whitespace at EOL.\n                if is_whitespace {\n                    continue;\n                }\n            }\n\n            line.push(c);\n\n            // Reserve extra column for fullwidth characters.\n            let width = c.width().unwrap_or(0);\n            if width == 2 {\n                line.push(' ');\n            }\n\n            line_len += width\n        }\n        lines.push(Self::pad_text(line, num_cols));\n\n        // Truncate output if it's too long.\n        if lines.len() > max_lines {\n            lines.truncate(max_lines);\n            if TRUNCATED_MESSAGE.len() <= num_cols {\n                if let Some(line) = lines.iter_mut().last() {\n                    *line = Self::pad_text(TRUNCATED_MESSAGE.into(), num_cols);\n                }\n            }\n        }\n\n        // Append close button to first line.\n        if button_len <= num_cols {\n            if let Some(line) = lines.get_mut(0) {\n                line.truncate(num_cols - button_len);\n                line.push_str(CLOSE_BUTTON_TEXT);\n            }\n        }\n\n        lines\n    }\n\n    /// Message type.\n    #[inline]\n    pub fn ty(&self) -> MessageType {\n        self.ty\n    }\n\n    /// Message target.\n    #[inline]\n    pub fn target(&self) -> Option<&String> {\n        self.target.as_ref()\n    }\n\n    /// Update the message target.\n    #[inline]\n    pub fn set_target(&mut self, target: String) {\n        self.target = Some(target);\n    }\n\n    /// Right-pad text to fit a specific number of columns.\n    #[inline]\n    fn pad_text(mut text: String, num_cols: usize) -> String {\n        let padding_len = num_cols.saturating_sub(text.chars().count());\n        text.extend(vec![' '; padding_len]);\n        text\n    }\n}\n\n/// Storage for message bar.\n#[derive(Debug, Default)]\npub struct MessageBuffer {\n    messages: VecDeque<Message>,\n}\n\nimpl MessageBuffer {\n    /// Check if there are any messages queued.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.messages.is_empty()\n    }\n\n    /// Current message.\n    #[inline]\n    pub fn message(&self) -> Option<&Message> {\n        self.messages.front()\n    }\n\n    /// Remove the currently visible message.\n    #[inline]\n    pub fn pop(&mut self) {\n        // Remove the message itself.\n        let msg = self.messages.pop_front();\n\n        // Remove all duplicates.\n        if let Some(msg) = msg {\n            self.messages = self.messages.drain(..).filter(|m| m != &msg).collect();\n        }\n    }\n\n    /// Remove all messages with a specific target.\n    #[inline]\n    pub fn remove_target(&mut self, target: &str) {\n        self.messages = self\n            .messages\n            .drain(..)\n            .filter(|m| m.target().map(String::as_str) != Some(target))\n            .collect();\n    }\n\n    /// Add a new message to the queue.\n    #[inline]\n    pub fn push(&mut self, message: Message) {\n        self.messages.push_back(message);\n    }\n\n    /// Check whether the message is already queued in the message bar.\n    #[inline]\n    pub fn is_queued(&self, message: &Message) -> bool {\n        self.messages.contains(message)\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    use crate::display::SizeInfo;\n\n    #[test]\n    fn appends_close_button() {\n        let input = \"a\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(7., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"a   [X]\")]);\n    }\n\n    #[test]\n    fn multiline_close_button_first_line() {\n        let input = \"fo\\nbar\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"fo [X]\"), String::from(\"bar   \")]);\n    }\n\n    #[test]\n    fn splits_on_newline() {\n        let input = \"a\\nb\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines.len(), 2);\n    }\n\n    #[test]\n    fn splits_on_length() {\n        let input = \"foobar1\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines.len(), 2);\n    }\n\n    #[test]\n    fn empty_with_shortterm() {\n        let input = \"foobar\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 0., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines.len(), 0);\n    }\n\n    #[test]\n    fn truncates_long_messages() {\n        let input = \"hahahahahahahahahahaha truncate this because it's too long for the term\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(22., (MIN_FREE_LINES + 2) as f32, 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![\n            String::from(\"hahahahahahahahaha [X]\"),\n            String::from(\"[MESSAGE TRUNCATED]   \")\n        ]);\n    }\n\n    #[test]\n    fn hide_button_when_too_narrow() {\n        let input = \"ha\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(2., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"ha\")]);\n    }\n\n    #[test]\n    fn hide_truncated_when_too_narrow() {\n        let input = \"hahahahahahahahaha\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(2., (MIN_FREE_LINES + 2) as f32, 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"ha\"), String::from(\"ha\")]);\n    }\n\n    #[test]\n    fn add_newline_for_button() {\n        let input = \"test\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(5., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"t [X]\"), String::from(\"est  \")]);\n    }\n\n    #[test]\n    fn remove_target() {\n        let mut message_buffer = MessageBuffer::default();\n        for i in 0..10 {\n            let mut msg = Message::new(i.to_string(), MessageType::Error);\n            if i % 2 == 0 && i < 5 {\n                msg.set_target(\"target\".into());\n            }\n            message_buffer.push(msg);\n        }\n\n        message_buffer.remove_target(\"target\");\n\n        // Count number of messages.\n        let mut num_messages = 0;\n        while message_buffer.message().is_some() {\n            num_messages += 1;\n            message_buffer.pop();\n        }\n\n        assert_eq!(num_messages, 7);\n    }\n\n    #[test]\n    fn pop() {\n        let mut message_buffer = MessageBuffer::default();\n        let one = Message::new(String::from(\"one\"), MessageType::Error);\n        message_buffer.push(one.clone());\n        let two = Message::new(String::from(\"two\"), MessageType::Warning);\n        message_buffer.push(two.clone());\n\n        assert_eq!(message_buffer.message(), Some(&one));\n\n        message_buffer.pop();\n\n        assert_eq!(message_buffer.message(), Some(&two));\n    }\n\n    #[test]\n    fn wrap_on_words() {\n        let input = \"a\\nbc defg\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(5., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![\n            String::from(\"a [X]\"),\n            String::from(\"bc   \"),\n            String::from(\"defg \")\n        ]);\n    }\n\n    #[test]\n    fn wrap_with_unicode() {\n        let input = \"ab\\nc \ud83d\udc69d fgh\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(7., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![\n            String::from(\"ab  [X]\"),\n            String::from(\"c \ud83d\udc69 d  \"),\n            String::from(\"fgh    \")\n        ]);\n    }\n\n    #[test]\n    fn strip_whitespace_at_linebreak() {\n        let input = \"\\n0 1 2 3\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(3., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"[X]\"), String::from(\"0 1\"), String::from(\"2 3\"),]);\n    }\n\n    #[test]\n    fn remove_duplicates() {\n        let mut message_buffer = MessageBuffer::default();\n        for _ in 0..10 {\n            let msg = Message::new(String::from(\"test\"), MessageType::Error);\n            message_buffer.push(msg);\n        }\n        message_buffer.push(Message::new(String::from(\"other\"), MessageType::Error));\n        message_buffer.push(Message::new(String::from(\"test\"), MessageType::Warning));\n        let _ = message_buffer.message();\n\n        message_buffer.pop();\n\n        // Count number of messages.\n        let mut num_messages = 0;\n        while message_buffer.message().is_some() {\n            num_messages += 1;\n            message_buffer.pop();\n        }\n\n        assert_eq!(num_messages, 2);\n    }\n}\n\n// Path: alacritty/src/config/scrolling.rs\nuse serde::de::Error as SerdeError;\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\n/// Maximum scrollback amount configurable.\npub const MAX_SCROLLBACK_LINES: u32 = 100_000;\n\n/// Struct for scrolling related settings.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq)]\npub struct Scrolling {\n    pub multiplier: u8,\n\n    history: ScrollingHistory,\n}\n\nimpl Default for Scrolling {\n    fn default() -> Self {\n        Self { multiplier: 3, history: Default::default() }\n    }\n}\n\nimpl Scrolling {\n    pub fn history(self) -> u32 {\n        self.history.0\n    }\n}\n\n#[derive(SerdeReplace, Copy, Clone, Debug, PartialEq, Eq)]\nstruct ScrollingHistory(u32);\n\nimpl Default for ScrollingHistory {\n    fn default() -> Self {\n        Self(10_000)\n    }\n}\n\nimpl<'de> Deserialize<'de> for ScrollingHistory {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let lines = u32::deserialize(deserializer)?;\n\n        if lines > MAX_SCROLLBACK_LINES {\n            Err(SerdeError::custom(format!(\n                \"exceeded maximum scrolling history ({lines}/{MAX_SCROLLBACK_LINES})\"\n            )))\n        } else {\n            Ok(Self(lines))\n        }\n    }\n}\n\n// Path: alacritty/src/display/hint.rs\nuse std::cmp::Reverse;\nuse std::collections::HashSet;\nuse std::iter;\n\nuse ahash::RandomState;\nuse winit::keyboard::ModifiersState;\n\nuse alacritty_terminal::grid::{BidirectionalIterator, Dimensions};\nuse alacritty_terminal::index::{Boundary, Column, Direction, Line, Point};\nuse alacritty_terminal::term::cell::Hyperlink;\nuse alacritty_terminal::term::search::{Match, RegexIter, RegexSearch};\nuse alacritty_terminal::term::{Term, TermMode};\n\nuse crate::config::ui_config::{Hint, HintAction};\nuse crate::config::UiConfig;\n\n/// Maximum number of linewraps followed outside of the viewport during search highlighting.\npub const MAX_SEARCH_LINES: usize = 100;\n\n/// Percentage of characters in the hints alphabet used for the last character.\nconst HINT_SPLIT_PERCENTAGE: f32 = 0.5;\n\n/// Keyboard regex hint state.\npub struct HintState {\n    /// Hint currently in use.\n    hint: Option<Hint>,\n\n    /// Alphabet for hint labels.\n    alphabet: String,\n\n    /// Visible matches.\n    matches: Vec<Match>,\n\n    /// Key label for each visible match.\n    labels: Vec<Vec<char>>,\n\n    /// Keys pressed for hint selection.\n    keys: Vec<char>,\n}\n\nimpl HintState {\n    /// Initialize an inactive hint state.\n    pub fn new<S: Into<String>>(alphabet: S) -> Self {\n        Self {\n            alphabet: alphabet.into(),\n            hint: Default::default(),\n            matches: Default::default(),\n            labels: Default::default(),\n            keys: Default::default(),\n        }\n    }\n\n    /// Check if a hint selection is in progress.\n    pub fn active(&self) -> bool {\n        self.hint.is_some()\n    }\n\n    /// Start the hint selection process.\n    pub fn start(&mut self, hint: Hint) {\n        self.hint = Some(hint);\n    }\n\n    /// Cancel the hint highlighting process.\n    fn stop(&mut self) {\n        self.matches.clear();\n        self.labels.clear();\n        self.keys.clear();\n        self.hint = None;\n    }\n\n    /// Update the visible hint matches and key labels.\n    pub fn update_matches<T>(&mut self, term: &Term<T>) {\n        let hint = match self.hint.as_mut() {\n            Some(hint) => hint,\n            None => return,\n        };\n\n        // Clear current matches.\n        self.matches.clear();\n\n        // Add escape sequence hyperlinks.\n        if hint.content.hyperlinks {\n            self.matches.extend(visible_unique_hyperlinks_iter(term));\n        }\n\n        // Add visible regex matches.\n        if let Some(regex) = hint.content.regex.as_ref() {\n            regex.with_compiled(|regex| {\n                let matches = visible_regex_match_iter(term, regex);\n\n                // Apply post-processing and search for sub-matches if necessary.\n                if hint.post_processing {\n                    let mut matches = matches.collect::<Vec<_>>();\n                    self.matches.extend(matches.drain(..).flat_map(|rm| {\n                        HintPostProcessor::new(term, regex, rm).collect::<Vec<_>>()\n                    }));\n                } else {\n                    self.matches.extend(matches);\n                }\n            });\n        }\n\n        // Cancel highlight with no visible matches.\n        if self.matches.is_empty() {\n            self.stop();\n            return;\n        }\n\n        // Sort and dedup ranges. Currently overlapped but not exactly same ranges are kept.\n        self.matches.sort_by_key(|bounds| (*bounds.start(), Reverse(*bounds.end())));\n        self.matches.dedup_by_key(|bounds| *bounds.start());\n\n        let mut generator = HintLabels::new(&self.alphabet, HINT_SPLIT_PERCENTAGE);\n        let match_count = self.matches.len();\n        let keys_len = self.keys.len();\n\n        // Get the label for each match.\n        self.labels.resize(match_count, Vec::new());\n        for i in (0..match_count).rev() {\n            let mut label = generator.next();\n            if label.len() >= keys_len && label[..keys_len] == self.keys[..] {\n                self.labels[i] = label.split_off(keys_len);\n            } else {\n                self.labels[i] = Vec::new();\n            }\n        }\n    }\n\n    /// Handle keyboard input during hint selection.\n    pub fn keyboard_input<T>(&mut self, term: &Term<T>, c: char) -> Option<HintMatch> {\n        match c {\n            // Use backspace to remove the last character pressed.\n            '\\x08' | '\\x1f' => {\n                self.keys.pop();\n            },\n            // Cancel hint highlighting on ESC/Ctrl+c.\n            '\\x1b' | '\\x03' => self.stop(),\n            _ => (),\n        }\n\n        // Update the visible matches.\n        self.update_matches(term);\n\n        let hint = self.hint.as_ref()?;\n\n        // Find the last label starting with the input character.\n        let mut labels = self.labels.iter().enumerate().rev();\n        let (index, label) = labels.find(|(_, label)| !label.is_empty() && label[0] == c)?;\n\n        // Check if the selected label is fully matched.\n        if label.len() == 1 {\n            let bounds = self.matches[index].clone();\n            let action = hint.action.clone();\n\n            // Exit hint mode unless it requires explicit dismissal.\n            if hint.persist {\n                self.keys.clear();\n            } else {\n                self.stop();\n            }\n\n            // Hyperlinks take precedence over regex matches.\n            let hyperlink = term.grid()[*bounds.start()].hyperlink();\n            Some(HintMatch { action, bounds, hyperlink })\n        } else {\n            // Store character to preserve the selection.\n            self.keys.push(c);\n\n            None\n        }\n    }\n\n    /// Hint key labels.\n    pub fn labels(&self) -> &Vec<Vec<char>> {\n        &self.labels\n    }\n\n    /// Visible hint regex matches.\n    pub fn matches(&self) -> &[Match] {\n        &self.matches\n    }\n\n    /// Update the alphabet used for hint labels.\n    pub fn update_alphabet(&mut self, alphabet: &str) {\n        if self.alphabet != alphabet {\n            self.alphabet = alphabet.to_owned();\n            self.keys.clear();\n        }\n    }\n}\n\n/// Hint match which was selected by the user.\n#[derive(PartialEq, Eq, Debug, Clone)]\npub struct HintMatch {\n    /// Action for handling the text.\n    action: HintAction,\n\n    /// Terminal range matching the hint.\n    bounds: Match,\n\n    hyperlink: Option<Hyperlink>,\n}\n\nimpl HintMatch {\n    #[inline]\n    pub fn should_highlight(&self, point: Point, pointed_hyperlink: Option<&Hyperlink>) -> bool {\n        self.hyperlink.as_ref() == pointed_hyperlink\n            && (self.hyperlink.is_some() || self.bounds.contains(&point))\n    }\n\n    #[inline]\n    pub fn action(&self) -> &HintAction {\n        &self.action\n    }\n\n    #[inline]\n    pub fn bounds(&self) -> &Match {\n        &self.bounds\n    }\n\n    pub fn hyperlink(&self) -> Option<&Hyperlink> {\n        self.hyperlink.as_ref()\n    }\n}\n\n/// Generator for creating new hint labels.\nstruct HintLabels {\n    /// Full character set available.\n    alphabet: Vec<char>,\n\n    /// Alphabet indices for the next label.\n    indices: Vec<usize>,\n\n    /// Point separating the alphabet's head and tail characters.\n    ///\n    /// To make identification of the tail character easy, part of the alphabet cannot be used for\n    /// any other position.\n    ///\n    /// All characters in the alphabet before this index will be used for the last character, while\n    /// the rest will be used for everything else.\n    split_point: usize,\n}\n\nimpl HintLabels {\n    /// Create a new label generator.\n    ///\n    /// The `split_ratio` should be a number between 0.0 and 1.0 representing the percentage of\n    /// elements in the alphabet which are reserved for the tail of the hint label.\n    fn new(alphabet: impl Into<String>, split_ratio: f32) -> Self {\n        let alphabet: Vec<char> = alphabet.into().chars().collect();\n        let split_point = ((alphabet.len() - 1) as f32 * split_ratio.min(1.)) as usize;\n\n        Self { indices: vec![0], split_point, alphabet }\n    }\n\n    /// Get the characters for the next label.\n    fn next(&mut self) -> Vec<char> {\n        let characters = self.indices.iter().rev().map(|index| self.alphabet[*index]).collect();\n        self.increment();\n        characters\n    }\n\n    /// Increment the character sequence.\n    fn increment(&mut self) {\n        // Increment the last character; if it's not at the split point we're done.\n        let tail = &mut self.indices[0];\n        if *tail < self.split_point {\n            *tail += 1;\n            return;\n        }\n        *tail = 0;\n\n        // Increment all other characters in reverse order.\n        let alphabet_len = self.alphabet.len();\n        for index in self.indices.iter_mut().skip(1) {\n            if *index + 1 == alphabet_len {\n                // Reset character and move to the next if it's already at the limit.\n                *index = self.split_point + 1;\n            } else {\n                // If the character can be incremented, we're done.\n                *index += 1;\n                return;\n            }\n        }\n\n        // Extend the sequence with another character when nothing could be incremented.\n        self.indices.push(self.split_point + 1);\n    }\n}\n\n/// Iterate over all visible regex matches.\npub fn visible_regex_match_iter<'a, T>(\n    term: &'a Term<T>,\n    regex: &'a mut RegexSearch,\n) -> impl Iterator<Item = Match> + 'a {\n    let viewport_start = Line(-(term.grid().display_offset() as i32));\n    let viewport_end = viewport_start + term.bottommost_line();\n    let mut start = term.line_search_left(Point::new(viewport_start, Column(0)));\n    let mut end = term.line_search_right(Point::new(viewport_end, Column(0)));\n    start.line = start.line.max(viewport_start - MAX_SEARCH_LINES);\n    end.line = end.line.min(viewport_end + MAX_SEARCH_LINES);\n\n    RegexIter::new(start, end, Direction::Right, term, regex)\n        .skip_while(move |rm| rm.end().line < viewport_start)\n        .take_while(move |rm| rm.start().line <= viewport_end)\n}\n\n/// Iterate over all visible hyperlinks, yanking only unique ones.\npub fn visible_unique_hyperlinks_iter<T>(term: &Term<T>) -> impl Iterator<Item = Match> + '_ {\n    let mut display_iter = term.grid().display_iter().peekable();\n\n...\n", "needle_token_start": 7214, "needle_token_end": 7296, "code_context_ntokens": 16092, "output": ["```rust\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn dynamic_title_ignoring_options_by_default() {\n        let mut config = UiConfig::default();\n        let old_dynamic_title = config.window.dynamic_title;\n\n        Options::default().override_config(&mut config);\n\n        assert_eq!(old_dynamic_title, config.window.dynamic_title);\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "keyboard_input", "language": "rust", "path": "alacritty/src/display/hint.rs", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To process user keystrokes during a hint selection session, updating the hint state and potentially finalizing a selection based on the input.\n2. **Input**: A terminal reference and a character representing the user's keystroke.\n3. **Output**: Optionally returns a structure representing a successfully selected hint match, including the action to be taken, the bounds of the match, and any associated hyperlink.\n4. **Procedure**: \n   - Handles special characters such as backspace and escape to modify or cancel the hint selection process.\n   - Updates the list of visible matches based on the current input state.\n   - Searches for a match starting with the input character among the updated labels.\n   - If a full match is found (label length is 1), constructs and returns the match details.\n   - If no full match is found, stores the character to refine future selections.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/cli.rs\nuse std::cmp::max;\nuse std::collections::HashMap;\nuse std::ops::{Deref, DerefMut};\nuse std::path::PathBuf;\nuse std::rc::Rc;\n\nuse alacritty_config::SerdeReplace;\nuse clap::{ArgAction, Args, Parser, Subcommand, ValueHint};\nuse log::{error, LevelFilter};\nuse serde::{Deserialize, Serialize};\nuse toml::Value;\n\nuse alacritty_terminal::tty::Options as PtyOptions;\n\nuse crate::config::ui_config::Program;\nuse crate::config::window::{Class, Identity};\nuse crate::config::UiConfig;\nuse crate::logging::LOG_TARGET_IPC_CONFIG;\n\n/// CLI options for the main Alacritty executable.\n#[derive(Parser, Default, Debug)]\n#[clap(author, about, version = env!(\"VERSION\"))]\npub struct Options {\n    /// Print all events to STDOUT.\n    #[clap(long)]\n    pub print_events: bool,\n\n    /// Generates ref test.\n    #[clap(long)]\n    pub ref_test: bool,\n\n    /// X11 window ID to embed Alacritty within (decimal or hexadecimal with \"0x\" prefix).\n    #[clap(long)]\n    pub embed: Option<String>,\n\n    /// Specify alternative configuration file [default:\n    /// $XDG_CONFIG_HOME/alacritty/alacritty.toml].\n    #[cfg(not(any(target_os = \"macos\", windows)))]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Specify alternative configuration file [default: %APPDATA%\\alacritty\\alacritty.toml].\n    #[cfg(windows)]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Specify alternative configuration file [default: $HOME/.config/alacritty/alacritty.toml].\n    #[cfg(target_os = \"macos\")]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Path for IPC socket creation.\n    #[cfg(unix)]\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub socket: Option<PathBuf>,\n\n    /// Reduces the level of verbosity (the min level is -qq).\n    #[clap(short, conflicts_with(\"verbose\"), action = ArgAction::Count)]\n    quiet: u8,\n\n    /// Increases the level of verbosity (the max level is -vvv).\n    #[clap(short, conflicts_with(\"quiet\"), action = ArgAction::Count)]\n    verbose: u8,\n\n    /// CLI options for config overrides.\n    #[clap(skip)]\n    pub config_options: ParsedOptions,\n\n    /// Options which can be passed via IPC.\n    #[clap(flatten)]\n    pub window_options: WindowOptions,\n\n    /// Subcommand passed to the CLI.\n    #[clap(subcommand)]\n    pub subcommands: Option<Subcommands>,\n}\n\nimpl Options {\n    pub fn new() -> Self {\n        let mut options = Self::parse();\n\n        // Parse CLI config overrides.\n        options.config_options = options.window_options.config_overrides();\n\n        options\n    }\n\n    /// Override configuration file with options from the CLI.\n    pub fn override_config(&mut self, config: &mut UiConfig) {\n        #[cfg(unix)]\n        {\n            config.ipc_socket |= self.socket.is_some();\n        }\n\n        config.window.dynamic_title &= self.window_options.window_identity.title.is_none();\n        config.window.embed = self.embed.as_ref().and_then(|embed| parse_hex_or_decimal(embed));\n        config.debug.print_events |= self.print_events;\n        config.debug.log_level = max(config.debug.log_level, self.log_level());\n        config.debug.ref_test |= self.ref_test;\n\n        if config.debug.print_events {\n            config.debug.log_level = max(config.debug.log_level, LevelFilter::Info);\n        }\n\n        // Replace CLI options.\n        self.config_options.override_config(config);\n    }\n\n    /// Logging filter level.\n    pub fn log_level(&self) -> LevelFilter {\n        match (self.quiet, self.verbose) {\n            // Force at least `Info` level for `--print-events`.\n            (_, 0) if self.print_events => LevelFilter::Info,\n\n            // Default.\n            (0, 0) => LevelFilter::Warn,\n\n            // Verbose.\n            (_, 1) => LevelFilter::Info,\n            (_, 2) => LevelFilter::Debug,\n            (0, _) => LevelFilter::Trace,\n\n            // Quiet.\n            (1, _) => LevelFilter::Error,\n            (..) => LevelFilter::Off,\n        }\n    }\n}\n\n/// Parse the class CLI parameter.\nfn parse_class(input: &str) -> Result<Class, String> {\n    let (general, instance) = match input.split_once(',') {\n        // Warn the user if they've passed too many values.\n        Some((_, instance)) if instance.contains(',') => {\n            return Err(String::from(\"Too many parameters\"))\n        },\n        Some((general, instance)) => (general, instance),\n        None => (input, input),\n    };\n\n    Ok(Class::new(general, instance))\n}\n\n/// Convert to hex if possible, else decimal\nfn parse_hex_or_decimal(input: &str) -> Option<u32> {\n    input\n        .strip_prefix(\"0x\")\n        .and_then(|value| u32::from_str_radix(value, 16).ok())\n        .or_else(|| input.parse().ok())\n}\n\n/// Terminal specific cli options which can be passed to new windows via IPC.\n#[derive(Serialize, Deserialize, Args, Default, Debug, Clone, PartialEq, Eq)]\npub struct TerminalOptions {\n    /// Start the shell in the specified working directory.\n    #[clap(long, value_hint = ValueHint::FilePath)]\n    pub working_directory: Option<PathBuf>,\n\n    /// Remain open after child process exit.\n    #[clap(long)]\n    pub hold: bool,\n\n    /// Command and args to execute (must be last argument).\n    #[clap(short = 'e', long, allow_hyphen_values = true, num_args = 1..)]\n    command: Vec<String>,\n}\n\nimpl TerminalOptions {\n    /// Shell override passed through the CLI.\n    pub fn command(&self) -> Option<Program> {\n        let (program, args) = self.command.split_first()?;\n        Some(Program::WithArgs { program: program.clone(), args: args.to_vec() })\n    }\n\n    /// Override the [`PtyOptions`]'s fields with the [`TerminalOptions`].\n    pub fn override_pty_config(&self, pty_config: &mut PtyOptions) {\n        if let Some(working_directory) = &self.working_directory {\n            if working_directory.is_dir() {\n                pty_config.working_directory = Some(working_directory.to_owned());\n            } else {\n                error!(\"Invalid working directory: {:?}\", working_directory);\n            }\n        }\n\n        if let Some(command) = self.command() {\n            pty_config.shell = Some(command.into());\n        }\n\n        pty_config.hold |= self.hold;\n    }\n}\n\nimpl From<TerminalOptions> for PtyOptions {\n    fn from(mut options: TerminalOptions) -> Self {\n        PtyOptions {\n            working_directory: options.working_directory.take(),\n            shell: options.command().map(Into::into),\n            hold: options.hold,\n            env: HashMap::new(),\n        }\n    }\n}\n\n/// Window specific cli options which can be passed to new windows via IPC.\n#[derive(Serialize, Deserialize, Args, Default, Debug, Clone, PartialEq, Eq)]\npub struct WindowIdentity {\n    /// Defines the window title [default: Alacritty].\n    #[clap(short = 'T', short_alias('t'), long)]\n    pub title: Option<String>,\n\n    /// Defines window class/app_id on X11/Wayland [default: Alacritty].\n    #[clap(long, value_name = \"general> | <general>,<instance\", value_parser = parse_class)]\n    pub class: Option<Class>,\n}\n\nimpl WindowIdentity {\n    /// Override the [`WindowIdentity`]'s fields with the [`WindowOptions`].\n    pub fn override_identity_config(&self, identity: &mut Identity) {\n        if let Some(title) = &self.title {\n            identity.title = title.clone();\n        }\n        if let Some(class) = &self.class {\n            identity.class = class.clone();\n        }\n    }\n}\n\n/// Available CLI subcommands.\n#[derive(Subcommand, Debug)]\npub enum Subcommands {\n    #[cfg(unix)]\n    Msg(MessageOptions),\n    Migrate(MigrateOptions),\n}\n\n/// Send a message to the Alacritty socket.\n#[cfg(unix)]\n#[derive(Args, Debug)]\npub struct MessageOptions {\n    /// IPC socket connection path override.\n    #[clap(short, long, value_hint = ValueHint::FilePath)]\n    pub socket: Option<PathBuf>,\n\n    /// Message which should be sent.\n    #[clap(subcommand)]\n    pub message: SocketMessage,\n}\n\n/// Available socket messages.\n#[cfg(unix)]\n#[derive(Subcommand, Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]\npub enum SocketMessage {\n    /// Create a new window in the same Alacritty process.\n    CreateWindow(WindowOptions),\n\n    /// Update the Alacritty configuration.\n    Config(IpcConfig),\n}\n\n/// Migrate the configuration file.\n#[derive(Args, Clone, Debug)]\npub struct MigrateOptions {\n    /// Path to the configuration file.\n    #[clap(short, long, value_hint = ValueHint::FilePath)]\n    pub config_file: Option<PathBuf>,\n\n    /// Only output TOML config to STDOUT.\n    #[clap(short, long)]\n    pub dry_run: bool,\n\n    /// Do not recurse over imports.\n    #[clap(short = 'i', long)]\n    pub skip_imports: bool,\n\n    /// Do not move renamed fields to their new location.\n    #[clap(long)]\n    pub skip_renames: bool,\n\n    #[clap(short, long)]\n    /// Do not output to STDOUT.\n    pub silent: bool,\n}\n\n/// Subset of options that we pass to 'create-window' IPC subcommand.\n#[derive(Serialize, Deserialize, Args, Default, Clone, Debug, PartialEq, Eq)]\npub struct WindowOptions {\n    /// Terminal options which can be passed via IPC.\n    #[clap(flatten)]\n    pub terminal_options: TerminalOptions,\n\n    #[clap(flatten)]\n    /// Window options which could be passed via IPC.\n    pub window_identity: WindowIdentity,\n\n    #[clap(skip)]\n    #[cfg(target_os = \"macos\")]\n    /// The window tabbing identifier to use when building a window.\n...\n// Path: alacritty/src/message_bar.rs\nuse std::collections::VecDeque;\n\nuse unicode_width::UnicodeWidthChar;\n\nuse alacritty_terminal::grid::Dimensions;\n\nuse crate::display::SizeInfo;\n\npub const CLOSE_BUTTON_TEXT: &str = \"[X]\";\nconst CLOSE_BUTTON_PADDING: usize = 1;\nconst MIN_FREE_LINES: usize = 3;\nconst TRUNCATED_MESSAGE: &str = \"[MESSAGE TRUNCATED]\";\n\n/// Message for display in the MessageBuffer.\n#[derive(Debug, Eq, PartialEq, Clone)]\npub struct Message {\n    text: String,\n    ty: MessageType,\n    target: Option<String>,\n}\n\n/// Purpose of the message.\n#[derive(Debug, Eq, PartialEq, Clone, Copy)]\npub enum MessageType {\n    /// A message represents an error.\n    Error,\n\n    /// A message represents a warning.\n    Warning,\n}\n\nimpl Message {\n    /// Create a new message.\n    pub fn new(text: String, ty: MessageType) -> Message {\n        Message { text, ty, target: None }\n    }\n\n    /// Formatted message text lines.\n    pub fn text(&self, size_info: &SizeInfo) -> Vec<String> {\n        let num_cols = size_info.columns();\n        let total_lines =\n            (size_info.height() - 2. * size_info.padding_y()) / size_info.cell_height();\n        let max_lines = (total_lines as usize).saturating_sub(MIN_FREE_LINES);\n        let button_len = CLOSE_BUTTON_TEXT.chars().count();\n\n        // Split line to fit the screen.\n        let mut lines = Vec::new();\n        let mut line = String::new();\n        let mut line_len = 0;\n        for c in self.text.trim().chars() {\n            if c == '\\n'\n                || line_len == num_cols\n                // Keep space in first line for button.\n                || (lines.is_empty()\n                    && num_cols >= button_len\n                    && line_len == num_cols.saturating_sub(button_len + CLOSE_BUTTON_PADDING))\n            {\n                let is_whitespace = c.is_whitespace();\n\n                // Attempt to wrap on word boundaries.\n                let mut new_line = String::new();\n                if let Some(index) = line.rfind(char::is_whitespace).filter(|_| !is_whitespace) {\n                    let split = line.split_off(index + 1);\n                    line.pop();\n                    new_line = split;\n                }\n\n                lines.push(Self::pad_text(line, num_cols));\n                line = new_line;\n                line_len = line.chars().count();\n\n                // Do not append whitespace at EOL.\n                if is_whitespace {\n                    continue;\n                }\n            }\n\n            line.push(c);\n\n            // Reserve extra column for fullwidth characters.\n            let width = c.width().unwrap_or(0);\n            if width == 2 {\n                line.push(' ');\n            }\n\n            line_len += width\n        }\n        lines.push(Self::pad_text(line, num_cols));\n\n        // Truncate output if it's too long.\n        if lines.len() > max_lines {\n            lines.truncate(max_lines);\n            if TRUNCATED_MESSAGE.len() <= num_cols {\n                if let Some(line) = lines.iter_mut().last() {\n                    *line = Self::pad_text(TRUNCATED_MESSAGE.into(), num_cols);\n                }\n            }\n        }\n\n        // Append close button to first line.\n        if button_len <= num_cols {\n            if let Some(line) = lines.get_mut(0) {\n                line.truncate(num_cols - button_len);\n                line.push_str(CLOSE_BUTTON_TEXT);\n            }\n        }\n\n        lines\n    }\n\n    /// Message type.\n    #[inline]\n    pub fn ty(&self) -> MessageType {\n        self.ty\n    }\n\n    /// Message target.\n    #[inline]\n    pub fn target(&self) -> Option<&String> {\n        self.target.as_ref()\n    }\n\n    /// Update the message target.\n    #[inline]\n    pub fn set_target(&mut self, target: String) {\n        self.target = Some(target);\n    }\n\n    /// Right-pad text to fit a specific number of columns.\n    #[inline]\n    fn pad_text(mut text: String, num_cols: usize) -> String {\n        let padding_len = num_cols.saturating_sub(text.chars().count());\n        text.extend(vec![' '; padding_len]);\n        text\n    }\n}\n\n/// Storage for message bar.\n#[derive(Debug, Default)]\npub struct MessageBuffer {\n    messages: VecDeque<Message>,\n}\n\nimpl MessageBuffer {\n    /// Check if there are any messages queued.\n    #[inline]\n    pub fn is_empty(&self) -> bool {\n        self.messages.is_empty()\n    }\n\n    /// Current message.\n    #[inline]\n    pub fn message(&self) -> Option<&Message> {\n        self.messages.front()\n    }\n\n    /// Remove the currently visible message.\n    #[inline]\n    pub fn pop(&mut self) {\n        // Remove the message itself.\n        let msg = self.messages.pop_front();\n\n        // Remove all duplicates.\n        if let Some(msg) = msg {\n            self.messages = self.messages.drain(..).filter(|m| m != &msg).collect();\n        }\n    }\n\n    /// Remove all messages with a specific target.\n    #[inline]\n    pub fn remove_target(&mut self, target: &str) {\n        self.messages = self\n            .messages\n            .drain(..)\n            .filter(|m| m.target().map(String::as_str) != Some(target))\n            .collect();\n    }\n\n    /// Add a new message to the queue.\n    #[inline]\n    pub fn push(&mut self, message: Message) {\n        self.messages.push_back(message);\n    }\n\n    /// Check whether the message is already queued in the message bar.\n    #[inline]\n    pub fn is_queued(&self, message: &Message) -> bool {\n        self.messages.contains(message)\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    use crate::display::SizeInfo;\n\n    #[test]\n    fn appends_close_button() {\n        let input = \"a\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(7., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"a   [X]\")]);\n    }\n\n    #[test]\n    fn multiline_close_button_first_line() {\n        let input = \"fo\\nbar\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"fo [X]\"), String::from(\"bar   \")]);\n    }\n\n    #[test]\n    fn splits_on_newline() {\n        let input = \"a\\nb\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines.len(), 2);\n    }\n\n    #[test]\n    fn splits_on_length() {\n        let input = \"foobar1\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines.len(), 2);\n    }\n\n    #[test]\n    fn empty_with_shortterm() {\n        let input = \"foobar\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(6., 0., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines.len(), 0);\n    }\n\n    #[test]\n    fn truncates_long_messages() {\n        let input = \"hahahahahahahahahahaha truncate this because it's too long for the term\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(22., (MIN_FREE_LINES + 2) as f32, 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![\n            String::from(\"hahahahahahahahaha [X]\"),\n            String::from(\"[MESSAGE TRUNCATED]   \")\n        ]);\n    }\n\n    #[test]\n    fn hide_button_when_too_narrow() {\n        let input = \"ha\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(2., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"ha\")]);\n    }\n\n    #[test]\n    fn hide_truncated_when_too_narrow() {\n        let input = \"hahahahahahahahaha\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(2., (MIN_FREE_LINES + 2) as f32, 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"ha\"), String::from(\"ha\")]);\n    }\n\n    #[test]\n    fn add_newline_for_button() {\n        let input = \"test\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(5., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"t [X]\"), String::from(\"est  \")]);\n    }\n\n    #[test]\n    fn remove_target() {\n        let mut message_buffer = MessageBuffer::default();\n        for i in 0..10 {\n            let mut msg = Message::new(i.to_string(), MessageType::Error);\n            if i % 2 == 0 && i < 5 {\n                msg.set_target(\"target\".into());\n            }\n            message_buffer.push(msg);\n        }\n\n        message_buffer.remove_target(\"target\");\n\n        // Count number of messages.\n        let mut num_messages = 0;\n        while message_buffer.message().is_some() {\n            num_messages += 1;\n            message_buffer.pop();\n        }\n\n        assert_eq!(num_messages, 7);\n    }\n\n    #[test]\n    fn pop() {\n        let mut message_buffer = MessageBuffer::default();\n        let one = Message::new(String::from(\"one\"), MessageType::Error);\n        message_buffer.push(one.clone());\n        let two = Message::new(String::from(\"two\"), MessageType::Warning);\n        message_buffer.push(two.clone());\n\n        assert_eq!(message_buffer.message(), Some(&one));\n\n        message_buffer.pop();\n\n        assert_eq!(message_buffer.message(), Some(&two));\n    }\n\n    #[test]\n    fn wrap_on_words() {\n        let input = \"a\\nbc defg\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(5., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![\n            String::from(\"a [X]\"),\n            String::from(\"bc   \"),\n            String::from(\"defg \")\n        ]);\n    }\n\n    #[test]\n    fn wrap_with_unicode() {\n        let input = \"ab\\nc \ud83d\udc69d fgh\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(7., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![\n            String::from(\"ab  [X]\"),\n            String::from(\"c \ud83d\udc69 d  \"),\n            String::from(\"fgh    \")\n        ]);\n    }\n\n    #[test]\n    fn strip_whitespace_at_linebreak() {\n        let input = \"\\n0 1 2 3\";\n        let mut message_buffer = MessageBuffer::default();\n        message_buffer.push(Message::new(input.into(), MessageType::Error));\n        let size = SizeInfo::new(3., 10., 1., 1., 0., 0., false);\n\n        let lines = message_buffer.message().unwrap().text(&size);\n\n        assert_eq!(lines, vec![String::from(\"[X]\"), String::from(\"0 1\"), String::from(\"2 3\"),]);\n    }\n\n    #[test]\n    fn remove_duplicates() {\n        let mut message_buffer = MessageBuffer::default();\n        for _ in 0..10 {\n            let msg = Message::new(String::from(\"test\"), MessageType::Error);\n            message_buffer.push(msg);\n        }\n        message_buffer.push(Message::new(String::from(\"other\"), MessageType::Error));\n        message_buffer.push(Message::new(String::from(\"test\"), MessageType::Warning));\n        let _ = message_buffer.message();\n\n        message_buffer.pop();\n\n        // Count number of messages.\n        let mut num_messages = 0;\n        while message_buffer.message().is_some() {\n            num_messages += 1;\n            message_buffer.pop();\n        }\n\n        assert_eq!(num_messages, 2);\n    }\n}\n\n// Path: alacritty/src/config/scrolling.rs\nuse serde::de::Error as SerdeError;\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\n/// Maximum scrollback amount configurable.\npub const MAX_SCROLLBACK_LINES: u32 = 100_000;\n\n/// Struct for scrolling related settings.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq)]\npub struct Scrolling {\n    pub multiplier: u8,\n\n    history: ScrollingHistory,\n}\n\nimpl Default for Scrolling {\n    fn default() -> Self {\n        Self { multiplier: 3, history: Default::default() }\n    }\n}\n\nimpl Scrolling {\n    pub fn history(self) -> u32 {\n        self.history.0\n    }\n}\n\n#[derive(SerdeReplace, Copy, Clone, Debug, PartialEq, Eq)]\nstruct ScrollingHistory(u32);\n\nimpl Default for ScrollingHistory {\n    fn default() -> Self {\n        Self(10_000)\n    }\n}\n\nimpl<'de> Deserialize<'de> for ScrollingHistory {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let lines = u32::deserialize(deserializer)?;\n\n        if lines > MAX_SCROLLBACK_LINES {\n            Err(SerdeError::custom(format!(\n                \"exceeded maximum scrolling history ({lines}/{MAX_SCROLLBACK_LINES})\"\n            )))\n        } else {\n            Ok(Self(lines))\n        }\n    }\n}\n\n// Path: alacritty/src/display/hint.rs\nuse std::cmp::Reverse;\nuse std::collections::HashSet;\nuse std::iter;\n\nuse ahash::RandomState;\nuse winit::keyboard::ModifiersState;\n\nuse alacritty_terminal::grid::{BidirectionalIterator, Dimensions};\nuse alacritty_terminal::index::{Boundary, Column, Direction, Line, Point};\nuse alacritty_terminal::term::cell::Hyperlink;\nuse alacritty_terminal::term::search::{Match, RegexIter, RegexSearch};\nuse alacritty_terminal::term::{Term, TermMode};\n\nuse crate::config::ui_config::{Hint, HintAction};\nuse crate::config::UiConfig;\n\n/// Maximum number of linewraps followed outside of the viewport during search highlighting.\npub const MAX_SEARCH_LINES: usize = 100;\n\n/// Percentage of characters in the hints alphabet used for the last character.\nconst HINT_SPLIT_PERCENTAGE: f32 = 0.5;\n\n/// Keyboard regex hint state.\npub struct HintState {\n    /// Hint currently in use.\n    hint: Option<Hint>,\n\n    /// Alphabet for hint labels.\n    alphabet: String,\n\n    /// Visible matches.\n    matches: Vec<Match>,\n\n    /// Key label for each visible match.\n    labels: Vec<Vec<char>>,\n\n    /// Keys pressed for hint selection.\n    keys: Vec<char>,\n}\n\nimpl HintState {\n    /// Initialize an inactive hint state.\n    pub fn new<S: Into<String>>(alphabet: S) -> Self {\n        Self {\n            alphabet: alphabet.into(),\n            hint: Default::default(),\n            matches: Default::default(),\n            labels: Default::default(),\n            keys: Default::default(),\n        }\n    }\n\n    /// Check if a hint selection is in progress.\n    pub fn active(&self) -> bool {\n        self.hint.is_some()\n    }\n\n    /// Start the hint selection process.\n    pub fn start(&mut self, hint: Hint) {\n        self.hint = Some(hint);\n    }\n\n    /// Cancel the hint highlighting process.\n    fn stop(&mut self) {\n        self.matches.clear();\n        self.labels.clear();\n        self.keys.clear();\n        self.hint = None;\n    }\n\n    /// Update the visible hint matches and key labels.\n    pub fn update_matches<T>(&mut self, term: &Term<T>) {\n        let hint = match self.hint.as_mut() {\n            Some(hint) => hint,\n            None => return,\n        };\n\n        // Clear current matches.\n        self.matches.clear();\n\n        // Add escape sequence hyperlinks.\n        if hint.content.hyperlinks {\n            self.matches.extend(visible_unique_hyperlinks_iter(term));\n        }\n\n        // Add visible regex matches.\n        if let Some(regex) = hint.content.regex.as_ref() {\n            regex.with_compiled(|regex| {\n                let matches = visible_regex_match_iter(term, regex);\n\n                // Apply post-processing and search for sub-matches if necessary.\n                if hint.post_processing {\n                    let mut matches = matches.collect::<Vec<_>>();\n                    self.matches.extend(matches.drain(..).flat_map(|rm| {\n                        HintPostProcessor::new(term, regex, rm).collect::<Vec<_>>()\n                    }));\n                } else {\n                    self.matches.extend(matches);\n                }\n            });\n        }\n\n        // Cancel highlight with no visible matches.\n        if self.matches.is_empty() {\n            self.stop();\n            return;\n        }\n\n        // Sort and dedup ranges. Currently overlapped but not exactly same ranges are kept.\n        self.matches.sort_by_key(|bounds| (*bounds.start(), Reverse(*bounds.end())));\n        self.matches.dedup_by_key(|bounds| *bounds.start());\n\n        let mut generator = HintLabels::new(&self.alphabet, HINT_SPLIT_PERCENTAGE);\n        let match_count = self.matches.len();\n        let keys_len = self.keys.len();\n\n        // Get the label for each match.\n        self.labels.resize(match_count, Vec::new());\n        for i in (0..match_count).rev() {\n            let mut label = generator.next();\n            if label.len() >= keys_len && label[..keys_len] == self.keys[..] {\n                self.labels[i] = label.split_off(keys_len);\n            } else {\n                self.labels[i] = Vec::new();\n            }\n        }\n    }\n\n    /// Handle keyboard input during hint selection.\n    \npub fn keyboard_input<T>(&mut self, term: &Term<T>, c: char) -> Option<HintMatch> {\n        match c {\n            // Use backspace to remove the last character pressed.\n            '\\x08' | '\\x1f' => {\n                self.keys.pop();\n            },\n            // Cancel hint highlighting on ESC/Ctrl+c.\n            '\\x1b' | '\\x03' => self.stop(),\n            _ => (),\n        }\n\n        // Update the visible matches.\n        self.update_matches(term);\n\n        let hint = self.hint.as_ref()?;\n\n        // Find the last label starting with the input character.\n        let mut labels = self.labels.iter().enumerate().rev();\n        let (index, label) = labels.find(|(_, label)| !label.is_empty() && label[0] == c)?;\n\n        // Check if the selected label is fully matched.\n        if label.len() == 1 {\n            let bounds = self.matches[index].clone();\n            let action = hint.action.clone();\n\n            // Exit hint mode unless it requires explicit dismissal.\n            if hint.persist {\n                self.keys.clear();\n            } else {\n                self.stop();\n            }\n\n            // Hyperlinks take precedence over regex matches.\n            let hyperlink = term.grid()[*bounds.start()].hyperlink();\n            Some(HintMatch { action, bounds, hyperlink })\n        } else {\n            // Store character to preserve the selection.\n            self.keys.push(c);\n\n            None\n        }\n    }\n\n    /// Hint key labels.\n    pub fn labels(&self) -> &Vec<Vec<char>> {\n        &self.labels\n    }\n\n    /// Visible hint regex matches.\n    pub fn matches(&self) -> &[Match] {\n        &self.matches\n    }\n\n    /// Update the alphabet used for hint labels.\n    pub fn update_alphabet(&mut self, alphabet: &str) {\n        if self.alphabet != alphabet {\n            self.alphabet = alphabet.to_owned();\n            self.keys.clear();\n        }\n    }\n}\n\n/// Hint match which was selected by the user.\n#[derive(PartialEq, Eq, Debug, Clone)]\npub struct HintMatch {\n    /// Action for handling the text.\n    action: HintAction,\n\n    /// Terminal range matching the hint.\n    bounds: Match,\n\n    hyperlink: Option<Hyperlink>,\n}\n\nimpl HintMatch {\n    #[inline]\n    pub fn should_highlight(&self, point: Point, pointed_hyperlink: Option<&Hyperlink>) -> bool {\n        self.hyperlink.as_ref() == pointed_hyperlink\n            && (self.hyperlink.is_some() || self.bounds.contains(&point))\n    }\n\n    #[inline]\n    pub fn action(&self) -> &HintAction {\n        &self.action\n    }\n\n    #[inline]\n    pub fn bounds(&self) -> &Match {\n        &self.bounds\n    }\n\n    pub fn hyperlink(&self) -> Option<&Hyperlink> {\n        self.hyperlink.as_ref()\n    }\n}\n\n/// Generator for creating new hint labels.\nstruct HintLabels {\n    /// Full character set available.\n    alphabet: Vec<char>,\n\n    /// Alphabet indices for the next label.\n    indices: Vec<usize>,\n\n    /// Point separating the alphabet's head and tail characters.\n    ///\n    /// To make identification of the tail character easy, part of the alphabet cannot be used for\n    /// any other position.\n    ///\n    /// All characters in the alphabet before this index will be used for the last character, while\n    /// the rest will be used for everything else.\n    split_point: usize,\n}\n\nimpl HintLabels {\n    /// Create a new label generator.\n    ///\n    /// The `split_ratio` should be a number between 0.0 and 1.0 representing the percentage of\n    /// elements in the alphabet which are reserved for the tail of the hint label.\n    fn new(alphabet: impl Into<String>, split_ratio: f32) -> Self {\n        let alphabet: Vec<char> = alphabet.into().chars().collect();\n        let split_point = ((alphabet.len() - 1) as f32 * split_ratio.min(1.)) as usize;\n\n        Self { indices: vec![0], split_point, alphabet }\n    }\n\n    /// Get the characters for the next label.\n    fn next(&mut self) -> Vec<char> {\n        let characters = self.indices.iter().rev().map(|index| self.alphabet[*index]).collect();\n        self.increment();\n        characters\n    }\n\n    /// Increment the character sequence.\n    fn increment(&mut self) {\n        // Increment the last character; if it's not at the split point we're done.\n        let tail = &mut self.indices[0];\n        if *tail < self.split_point {\n            *tail += 1;\n            return;\n        }\n        *tail = 0;\n\n        // Increment all other characters in reverse order.\n        let alphabet_len = self.alphabet.len();\n        for index in self.indices.iter_mut().skip(1) {\n            if *index + 1 == alphabet_len {\n                // Reset character and move to the next if it's already at the limit.\n                *index = self.split_point + 1;\n            } else {\n                // If the character can be incremented, we're done.\n                *index += 1;\n                return;\n            }\n        }\n\n        // Extend the sequence with another character when nothing could be incremented.\n        self.indices.push(self.split_point + 1);\n    }\n}\n\n/// Iterate over all visible regex matches.\npub fn visible_regex_match_iter<'a, T>(\n    term: &'a Term<T>,\n    regex: &'a mut RegexSearch,\n) -> impl Iterator<Item = Match> + 'a {\n    let viewport_start = Line(-(term.grid().display_offset() as i32));\n    let viewport_end = viewport_start + term.bottommost_line();\n    let mut start = term.line_search_left(Point::new(viewport_start, Column(0)));\n    let mut end = term.line_search_right(Point::new(viewport_end, Column(0)));\n    start.line = start.line.max(viewport_start - MAX_SEARCH_LINES);\n    end.line = end.line.min(viewport_end + MAX_SEARCH_LINES);\n\n    RegexIter::new(start, end, Direction::Right, term, regex)\n        .skip_while(move |rm| rm.end().line < viewport_start)\n        .take_while(move |rm| rm.start().line <= viewport_end)\n}\n\n/// Iterate over all visible hyperlinks, yanking only unique ones.\npub fn visible_unique_hyperlinks_iter<T>(term: &Term<T>) -> impl Iterator<Item = Match> + '_ {\n    let mut display_iter = term.grid().display_iter().peekable();\n\n    // Avoid creating hints for the same hyperlinks, but from a different places.\n    let mut unique_hyperlinks = HashSet::<Hyperlink, RandomState>::default();\n\n    iter::from_fn(move || {\n        // Find the start of the next unique hyperlink.\n        let (cell, hyperlink) = display_iter.find_map(|cell| {\n            let hyperlink = cell.hyperlink()?;\n            (!unique_hyperlinks.contains(&hyperlink)).then(|| {\n                unique_hyperlinks.insert(hyperlink.clone());\n                (cell, hyperlink)\n            })\n        })?;\n\n        let start = cell.point;\n        let mut end = start;\n\n        // Find the end bound of just found unique hyperlink.\n        while let Some(next_cell) = display_iter.peek() {\n            // Cell at display iter doesn't match, yield the hyperlink and start over with\n            // `find_map`.\n            if next_cell.hyperlink().as_ref() != Some(&hyperlink) {\n                break;\n            }\n\n            // Advance to the next cell.\n            end = next_cell.point;\n            let _ = display_iter.next();\n        }\n\n        Some(start..=end)\n    })\n}\n\n/// Retrieve the match, if the specified point is inside the content matching the regex.\nfn regex_match_at<T>(\n    term: &Term<T>,\n    point: Point,\n    regex: &mut RegexSearch,\n    post_processing: bool,\n) -> Option<Match> {\n    let regex_match = visible_regex_match_iter(term, regex).find(|rm| rm.contains(&point))?;\n\n    // Apply post-processing and search for sub-matches if necessary.\n    if post_processing {\n        HintPostProcessor::new(term, regex, regex_match).find(|rm| rm.contains(&point))\n    } else {\n        Some(regex_match)\n    }\n}\n\n/// Check if there is a hint highlighted at the specified point.\npub fn highlighted_at<T>(\n    term: &Term<T>,\n    config: &UiConfig,\n    point: Point,\n    mouse_mods: ModifiersState,\n) -> Option<HintMatch> {\n    let mouse_mode = term.mode().intersects(TermMode::MOUSE_MODE);\n\n    config.hints.enabled.iter().find_map(|hint| {\n        // Check if all required modifiers are pressed.\n        let highlight = hint.mouse.map_or(false, |mouse| {\n            mouse.enabled\n                && mouse_mods.contains(mouse.mods.0)\n                && (!mouse_mode || mouse_mods.contains(ModifiersState::SHIFT))\n        });\n        if !highlight {\n            return None;\n        }\n\n        if let Some((hyperlink, bounds)) =\n            hint.content.hyperlinks.then(|| hyperlink_at(term, point)).flatten()\n        {\n            return Some(HintMatch {\n                bounds,\n                action: hint.action.clone(),\n                hyperlink: Some(hyperlink),\n            });\n        }\n\n        let bounds = hint.content.regex.as_ref().and_then(|regex| {\n            regex.with_compiled(|regex| regex_match_at(term, point, regex, hint.post_processing))\n        });\n        if let Some(bounds) = bounds.flatten() {\n            return Some(HintMatch { bounds, action: hint.action.clone(), hyperlink: None });\n        }\n\n        None\n    })\n}\n\n/// Retrieve the hyperlink with its range, if there is one at the specified point.\n///\n/// This will only return contiguous cells, even if another hyperlink with the same ID exists.\nfn hyperlink_at<T>(term: &Term<T>, point: Point) -> Option<(Hyperlink, Match)> {\n    let hyperlink = term.grid()[point].hyperlink()?;\n\n    let grid = term.grid();\n\n    let mut match_end = point;\n    for cell in grid.iter_from(point) {\n        if cell.hyperlink().map_or(false, |link| link == hyperlink) {\n            match_end = cell.point;\n        } else {\n            break;\n        }\n    }\n\n    let mut match_start = point;\n    let mut iter = grid.iter_from(point);\n    while let Some(cell) = iter.prev() {\n        if cell.hyperlink().map_or(false, |link| link == hyperlink) {\n            match_start = cell.point;\n        } else {\n            break;\n        }\n    }\n\n    Some((hyperlink, match_start..=match_end))\n}\n\n/// Iterator over all post-processed matches inside an existing hint match.\nstruct HintPostProcessor<'a, T> {\n    /// Regex search DFAs.\n    regex: &'a mut RegexSearch,\n\n    /// Terminal reference.\n    term: &'a Term<T>,\n\n    /// Next hint match in the iterator.\n    next_match: Option<Match>,\n\n    /// Start point for the next search.\n    start: Point,\n\n    /// End point for the hint match iterator.\n    end: Point,\n}\n\nimpl<'a, T> HintPostProcessor<'a, T> {\n    /// Create a new iterator for an unprocessed match.\n    fn new(term: &'a Term<T>, regex: &'a mut RegexSearch, regex_match: Match) -> Self {\n        let mut post_processor = Self {\n            next_match: None,\n            start: *regex_match.start(),\n            end: *regex_match.end(),\n            term,\n            regex,\n        };\n\n        // Post-process the first hint match.\n        post_processor.next_processed_match(regex_match);\n\n        post_processor\n    }\n\n    /// Apply some hint post processing heuristics.\n    ///\n    /// This will check the end of the hint and make it shorter if certain characters are determined\n    /// to be unlikely to be intentionally part of the hint.\n    ///\n    /// This is most useful for identifying URLs appropriately.\n    fn hint_post_processing(&self, regex_match: &Match) -> Option<Match> {\n        let mut iter = self.term.grid().iter_from(*regex_match.start());\n\n        let mut c = iter.cell().c;\n\n        // Truncate uneven number of brackets.\n        let end = *regex_match.end();\n        let mut open_parents = 0;\n        let mut open_brackets = 0;\n        loop {\n            match c {\n                '(' => open_parents += 1,\n                '[' => open_brackets += 1,\n                ')' => {\n                    if open_parents == 0 {\n                        iter.prev();\n                        break;\n                    } else {\n                        open_parents -= 1;\n                    }\n                },\n                ']' => {\n                    if open_brackets == 0 {\n                        iter.prev();\n                        break;\n                    } else {\n                        open_brackets -= 1;\n                    }\n                },\n                _ => (),\n            }\n\n            if iter.point() == end {\n                break;\n            }\n\n            match iter.next() {\n                Some(indexed) => c = indexed.cell.c,\n                None => break,\n            }\n        }\n\n        // Truncate trailing characters which are likely to be delimiters.\n        let start = *regex_match.start();\n        while iter.point() != start {\n            if !matches!(c, '.' | ',' | ':' | ';' | '?' | '!' | '(' | '[' | '\\'') {\n                break;\n            }\n\n            match iter.prev() {\n                Some(indexed) => c = indexed.cell.c,\n                None => break,\n            }\n        }\n\n        if start > iter.point() {\n            None\n        } else {\n            Some(start..=iter.point())\n        }\n    }\n\n    /// Loop over submatches until a non-empty post-processed match is found.\n    fn next_processed_match(&mut self, mut regex_match: Match) {\n        self.next_match = loop {\n            if let Some(next_match) = self.hint_post_processing(&regex_match) {\n                self.start = next_match.end().add(self.term, Boundary::Grid, 1);\n                break Some(next_match);\n            }\n\n            self.start = regex_match.start().add(self.term, Boundary::Grid, 1);\n            if self.start > self.end {\n                return;\n            }\n\n            match self.term.regex_search_right(self.regex, self.start, self.end) {\n                Some(rm) => regex_match = rm,\n                None => return,\n            }\n        };\n    }\n}\n\nimpl<'a, T> Iterator for HintPostProcessor<'a, T> {\n    type Item = Match;\n\n    fn next(&mut self) -> Option<Self::Item> {\n        let next_match = self.next_match.take()?;\n\n        if self.start <= self.end {\n            if let Some(rm) = self.term.regex_search_right(self.regex, self.start, self.end) {\n                self.next_processed_match(rm);\n            }\n        }\n\n        Some(next_match)\n    }\n}\n\n#[cfg(test)]\nmod tests {\n    use alacritty_terminal::index::{Column, Line};\n    use alacritty_terminal::term::test::mock_term;\n    use alacritty_terminal::vte::ansi::Handler;\n\n    use super::*;\n\n    #[test]\n    fn hint_label_generation() {\n        let mut generator = HintLabels::new(\"0123\", 0.5);\n\n        assert_eq!(generator.next(), vec!['0']);\n        assert_eq!(generator.next(), vec!['1']);\n\n        assert_eq!(generator.next(), vec!['2', '0']);\n        assert_eq!(generator.next(), vec!['2', '1']);\n        assert_eq!(generator.next(), vec!['3', '0']);\n        assert_eq!(generator.next(), vec!['3', '1']);\n\n        assert_eq!(generator.next(), vec!['2', '2', '0']);\n        assert_eq!(generator.next(), vec!['2', '2', '1']);\n        assert_eq!(generator.next(), vec!['2', '3', '0']);\n        assert_eq!(generator.next(), vec!['2', '3', '1']);\n        assert_eq!(generator.next(), vec!['3', '2', '0']);\n        assert_eq!(generator.next(), vec!['3', '2', '1']);\n        assert_eq!(generator.next(), vec!['3', '3', '0']);\n        assert_eq!(generator.next(), vec!['3', '3', '1']);\n\n        assert_eq!(generator.next(), vec!['2', '2', '2', '0']);\n        assert_eq!(generator.next(), vec!['2', '2', '2', '1']);\n        assert_eq!(generator.next(), vec!['2', '2', '3', '0']);\n        assert_eq!(generator.next(), vec!['2', '2', '3', '1']);\n        assert_eq!(generator.next(), vec!['2', '3', '2', '0']);\n        assert_eq!(generator.next(), vec!['2', '3', '2', '1']);\n        assert_eq!(generator.next(), vec!['2', '3', '3', '0']);\n        assert_eq!(generator.next(), vec!['2', '3', '3', '1']);\n        assert_eq!(generator.next(), vec!['3', '2', '2', '0']);\n        assert_eq!(generator.next(), vec!['3', '2', '2', '1']);\n        assert_eq!(generator.next(), vec!['3', '2', '3', '0']);\n        assert_eq!(generator.next(), vec!['3', '2', '3', '1']);\n        assert_eq!(generator.next(), vec!['3', '3', '2', '0']);\n        assert_eq!(generator.next(), vec!['3', '3', '2', '1']);\n        assert_eq!(generator.next(), vec!['3', '3', '3', '0']);\n        assert_eq!(generator.next(), vec!['3', '3', '3', '1']);\n    }\n\n    #[test]\n    fn closed_bracket_does_not_result_in_infinite_iterator() {\n        let term = mock_term(\" ) \");\n\n        let mut search = RegexSearch::new(\"[^/ ]\").unwrap();\n\n        let count = HintPostProcessor::new(\n            &term,\n            &mut search,\n            Point::new(Line(0), Column(1))..=Point::new(Line(0), Column(1)),\n        )\n        .take(1)\n        .count();\n\n        assert_eq!(count, 0);\n    }\n\n    #[test]\n    fn collect_unique_hyperlinks() {\n        let mut term = mock_term(\"000\\r\\n111\");\n        term.goto(0, 0);\n\n        let hyperlink_foo = Hyperlink::new(Some(\"1\"), String::from(\"foo\"));\n        let hyperlink_bar = Hyperlink::new(Some(\"2\"), String::from(\"bar\"));\n\n        // Create 2 hyperlinks on the first line.\n        term.set_hyperlink(Some(hyperlink_foo.clone().into()));\n        term.input('b');\n        term.input('a');\n        term.set_hyperlink(Some(hyperlink_bar.clone().into()));\n        term.input('r');\n        term.set_hyperlink(Some(hyperlink_foo.clone().into()));\n        term.goto(1, 0);\n\n        // Ditto for the second line.\n        term.set_hyperlink(Some(hyperlink_foo.into()));\n        term.input('b');\n        term.input('a');\n        term.set_hyperlink(Some(hyperlink_bar.into()));\n        term.input('r');\n        term.set_hyperlink(None);\n\n        let mut unique_hyperlinks = visible_unique_hyperlinks_iter(&term);\n        assert_eq!(\n            Some(Match::new(Point::new(Line(0), Column(0)), Point::new(Line(0), Column(1)))),\n            unique_hyperlinks.next()\n        );\n        assert_eq!(\n            Some(Match::new(Point::new(Line(0), Column(2)), Point::new(Line(0), Column(2)))),\n            unique_hyperlinks.next()\n        );\n        assert_eq!(None, unique_hyperlinks.next());\n    }\n\n    #[test]\n    fn visible_regex_match_covers_entire_viewport() {\n        let content = \"I'm a match!\\r\\n\".repeat(4096);\n        // The Term returned from this call will have a viewport starting at 0 and ending at 4096.\n        // That's good enough for this test, since it only cares about visible content.\n        let term = mock_term(&content);\n        let mut regex = RegexSearch::new(\"match!\").unwrap();\n\n        // The iterator should match everything in the viewport.\n        assert_eq!(visible_regex_match_iter(&term, &mut regex).count(), 4096);\n    }\n}\n\n// Path: alacritty/src/input/keyboard.rs\nuse std::borrow::Cow;\nuse std::mem;\n\nuse winit::event::{ElementState, KeyEvent};\n#[cfg(target_os = \"macos\")]\nuse winit::keyboard::ModifiersKeyState;\nuse winit::keyboard::{Key, KeyLocation, ModifiersState, NamedKey};\n#[cfg(target_os = \"macos\")]\nuse winit::platform::macos::OptionAsAlt;\n\nuse alacritty_terminal::event::EventListener;\nuse alacritty_terminal::term::TermMode;\nuse winit::platform::modifier_supplement::KeyEventExtModifierSupplement;\n\nuse crate::config::{Action, BindingKey, BindingMode};\nuse crate::event::TYPING_SEARCH_DELAY;\nuse crate::input::{ActionContext, Execute, Processor};\nuse crate::scheduler::{TimerId, Topic};\n\nimpl<T: EventListener, A: ActionContext<T>> Processor<T, A> {\n    /// Process key input.\n    pub fn key_input(&mut self, key: KeyEvent) {\n        // IME input will be applied on commit and shouldn't trigger key bindings.\n        if self.ctx.display().ime.preedit().is_some() {\n            return;\n        }\n\n        let mode = *self.ctx.terminal().mode();\n        let mods = self.ctx.modifiers().state();\n\n        if key.state == ElementState::Released {\n            self.key_release(key, mode, mods);\n            return;\n        }\n\n        let text = key.text_with_all_modifiers().unwrap_or_default();\n\n        // All key bindings are disabled while a hint is being selected.\n        if self.ctx.display().hint_state.active() {\n            for character in text.chars() {\n                self.ctx.hint_input(character);\n            }\n            return;\n        }\n\n        // First key after inline search is captured.\n        let inline_state = self.ctx.inline_search_state();\n        if mem::take(&mut inline_state.char_pending) {\n            if let Some(c) = text.chars().next() {\n                inline_state.character = Some(c);\n\n                // Immediately move to the captured character.\n                self.ctx.inline_search_next();\n            }\n\n            // Ignore all other characters in `text`.\n            return;\n        }\n\n        // Reset search delay when the user is still typing.\n        self.reset_search_delay();\n\n        // Key bindings suppress the character input.\n        if self.process_key_bindings(&key) {\n            return;\n        }\n\n        if self.ctx.search_active() {\n            for character in text.chars() {\n                self.ctx.search_input(character);\n            }\n\n            return;\n        }\n\n        // Vi mode on its own doesn't have any input, the search input was done before.\n        if mode.contains(TermMode::VI) {\n            return;\n        }\n\n        // Mask `Alt` modifier from input when we won't send esc.\n        let mods = if self.alt_send_esc(&key, text) { mods } else { mods & !ModifiersState::ALT };\n\n        let build_key_sequence = Self::should_build_sequence(&key, text, mode, mods);\n\n        let bytes = if build_key_sequence {\n            build_sequence(key, mods, mode)\n        } else {\n            let mut bytes = Vec::with_capacity(text.len() + 1);\n            if mods.alt_key() {\n                bytes.push(b'\\x1b');\n            }\n\n            bytes.extend_from_slice(text.as_bytes());\n            bytes\n        };\n\n        // Write only if we have something to write.\n        if !bytes.is_empty() {\n            self.ctx.on_terminal_input_start();\n            self.ctx.write_to_pty(bytes);\n        }\n    }\n\n    fn alt_send_esc(&mut self, key: &KeyEvent, text: &str) -> bool {\n        #[cfg(not(target_os = \"macos\"))]\n        let alt_send_esc = self.ctx.modifiers().state().alt_key();\n\n        #[cfg(target_os = \"macos\")]\n        let alt_send_esc = {\n            let option_as_alt = self.ctx.config().window.option_as_alt();\n            self.ctx.modifiers().state().alt_key()\n                && (option_as_alt == OptionAsAlt::Both\n                    || (option_as_alt == OptionAsAlt::OnlyLeft\n                        && self.ctx.modifiers().lalt_state() == ModifiersKeyState::Pressed)\n                    || (option_as_alt == OptionAsAlt::OnlyRight\n                        && self.ctx.modifiers().ralt_state() == ModifiersKeyState::Pressed))\n        };\n\n        match key.logical_key {\n            Key::Named(named) => {\n                if named.to_text().is_some() {\n                    alt_send_esc\n                } else {\n                    // Treat `Alt` as modifier for named keys without text, like ArrowUp.\n                    self.ctx.modifiers().state().alt_key()\n                }\n            },\n            _ => alt_send_esc && text.chars().count() == 1,\n        }\n    }\n\n    /// Check whether we should try to build escape sequence for the [`KeyEvent`].\n    fn should_build_sequence(\n        key: &KeyEvent,\n        text: &str,\n        mode: TermMode,\n        mods: ModifiersState,\n    ) -> bool {\n        if mode.contains(TermMode::REPORT_ALL_KEYS_AS_ESC) {\n            return true;\n        }\n\n        let disambiguate = mode.contains(TermMode::DISAMBIGUATE_ESC_CODES)\n            && (key.logical_key == Key::Named(NamedKey::Escape)\n                || (!mods.is_empty() && mods != ModifiersState::SHIFT)\n                || key.location == KeyLocation::Numpad);\n\n        match key.logical_key {\n            _ if disambiguate => true,\n            // Exclude all the named keys unless they have textual representation.\n            Key::Named(named) => named.to_text().is_none(),\n            _ => text.is_empty(),\n        }\n    }\n\n    /// Attempt to find a binding and execute its action.\n    ///\n    /// The provided mode, mods, and key must match what is allowed by a binding\n    /// for its action to be executed.\n    fn process_key_bindings(&mut self, key: &KeyEvent) -> bool {\n        let mode = BindingMode::new(self.ctx.terminal().mode(), self.ctx.search_active());\n        let mods = self.ctx.modifiers().state();\n\n        // Don't suppress char if no bindings were triggered.\n        let mut suppress_chars = None;\n\n        // We don't want the key without modifier, because it means something else most of\n        // the time. However what we want is to manually lowercase the character to account\n        // for both small and capital letters on regular characters at the same time.\n...\n", "needle_token_start": 8660, "needle_token_end": 9050, "code_context_ntokens": 16097, "output": ["```rust\n/// Handle keyboard input during hint selection.\npub fn keyboard_input<T>(&mut self, term: &Term<T>, c: char) -> Option<HintMatch> {\n    match c {\n        // Use backspace to remove the last character pressed.\n        '\\x08' | '\\x1f' => {\n            self.keys.pop();\n        },\n        // Cancel hint highlighting on ESC/Ctrl+c.\n        '\\x1b' | '\\x03' => self.stop(),\n        _ => (),\n    }\n\n    // Update the visible matches.\n    self.update_matches(term);\n\n    let hint = self.hint.as_ref()?;\n\n    // Find the last label starting with the input character.\n    let mut labels = self.labels.iter().enumerate().rev();\n    let (index, label) = labels.find(|(_, label)| !label.is_empty() && label[0] == c)?;\n\n    // Check if the selected label is fully matched.\n    if label.len() == 1 {\n        let bounds = self.matches[index].clone();\n        let action = hint.action.clone();\n\n        // Exit hint mode unless it requires explicit dismissal.\n        if hint.persist {\n            self.keys.clear();\n        } else {\n            self.stop();\n        }\n\n        // Hyperlinks take precedence over regex matches.\n        let hyperlink = term.grid()[*bounds.start()].hyperlink();\n        Some(HintMatch { action, bounds, hyperlink })\n    } else {\n        // Store character to preserve the selection.\n        self.keys.push(c);\n\n        None\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "fill_cube", "language": "rust", "path": "alacritty/src/display/color.rs", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To populate a segment of a color list with a 6x6x6 color cube, which includes a range of colors generated by varying the red, green, and blue components. This function also allows for overriding these generated colors with custom colors specified in a configuration.\n2. **Input**: A reference to a structure containing color configurations, including any custom indexed colors.\n3. **Output**: The function does not return a value but modifies a part of the color list directly, updating it with either generated or configured colors.\n4. **Procedure**: The function iterates through all combinations of red, green, and blue intensities, each ranging from 0 to 5. For each combination, it calculates the color value and checks if there is a custom color specified for the current index in the configuration. If a custom color is found, it uses that color; otherwise, it computes the color based on the RGB intensities and updates the color list at the corresponding index.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/config/color.rs\nuse serde::de::Error as SerdeError;\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::ConfigDeserialize;\n\nuse crate::display::color::{CellRgb, Rgb};\n\n#[derive(ConfigDeserialize, Clone, Debug, Default, PartialEq, Eq)]\npub struct Colors {\n    pub primary: PrimaryColors,\n    pub cursor: InvertedCellColors,\n    pub vi_mode_cursor: InvertedCellColors,\n    pub selection: InvertedCellColors,\n    pub normal: NormalColors,\n    pub bright: BrightColors,\n    pub dim: Option<DimColors>,\n    pub indexed_colors: Vec<IndexedColor>,\n    pub search: SearchColors,\n    pub line_indicator: LineIndicatorColors,\n    pub hints: HintColors,\n    pub transparent_background_colors: bool,\n    pub draw_bold_text_with_bright_colors: bool,\n    footer_bar: BarColors,\n}\n\nimpl Colors {\n    pub fn footer_bar_foreground(&self) -> Rgb {\n        self.footer_bar.foreground.unwrap_or(self.primary.background)\n    }\n\n    pub fn footer_bar_background(&self) -> Rgb {\n        self.footer_bar.background.unwrap_or(self.primary.foreground)\n    }\n}\n\n#[derive(ConfigDeserialize, Copy, Clone, Default, Debug, PartialEq, Eq)]\npub struct LineIndicatorColors {\n    pub foreground: Option<Rgb>,\n    pub background: Option<Rgb>,\n}\n\n#[derive(ConfigDeserialize, Default, Copy, Clone, Debug, PartialEq, Eq)]\npub struct HintColors {\n    pub start: HintStartColors,\n    pub end: HintEndColors,\n}\n\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq)]\npub struct HintStartColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for HintStartColors {\n    fn default() -> Self {\n        Self {\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n            background: CellRgb::Rgb(Rgb::new(0xf4, 0xbf, 0x75)),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq)]\npub struct HintEndColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for HintEndColors {\n    fn default() -> Self {\n        Self {\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n            background: CellRgb::Rgb(Rgb::new(0xac, 0x42, 0x42)),\n        }\n    }\n}\n\n#[derive(Deserialize, Copy, Clone, Default, Debug, PartialEq, Eq)]\n#[serde(deny_unknown_fields)]\npub struct IndexedColor {\n    pub color: Rgb,\n\n    index: ColorIndex,\n}\n\nimpl IndexedColor {\n    #[inline]\n    pub fn index(&self) -> u8 {\n        self.index.0\n    }\n}\n\n#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)]\nstruct ColorIndex(u8);\n\nimpl<'de> Deserialize<'de> for ColorIndex {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let index = u8::deserialize(deserializer)?;\n\n        if index < 16 {\n            Err(SerdeError::custom(\n                \"Config error: indexed_color's index is {}, but a value bigger than 15 was \\\n                 expected; ignoring setting\",\n            ))\n        } else {\n            Ok(Self(index))\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub struct InvertedCellColors {\n    #[config(alias = \"text\")]\n    pub foreground: CellRgb,\n    #[config(alias = \"cursor\")]\n    pub background: CellRgb,\n}\n\nimpl Default for InvertedCellColors {\n    fn default() -> Self {\n        Self { foreground: CellRgb::CellBackground, background: CellRgb::CellForeground }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, Default, PartialEq, Eq)]\npub struct SearchColors {\n    pub focused_match: FocusedMatchColors,\n    pub matches: MatchColors,\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub struct FocusedMatchColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for FocusedMatchColors {\n    fn default() -> Self {\n        Self {\n            background: CellRgb::Rgb(Rgb::new(0xf4, 0xbf, 0x75)),\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub struct MatchColors {\n    pub foreground: CellRgb,\n    pub background: CellRgb,\n}\n\nimpl Default for MatchColors {\n    fn default() -> Self {\n        Self {\n            background: CellRgb::Rgb(Rgb::new(0xac, 0x42, 0x42)),\n            foreground: CellRgb::Rgb(Rgb::new(0x18, 0x18, 0x18)),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Copy, Clone, Default, PartialEq, Eq)]\npub struct BarColors {\n    foreground: Option<Rgb>,\n    background: Option<Rgb>,\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct PrimaryColors {\n    pub foreground: Rgb,\n    pub background: Rgb,\n    pub bright_foreground: Option<Rgb>,\n    pub dim_foreground: Option<Rgb>,\n}\n\nimpl Default for PrimaryColors {\n    fn default() -> Self {\n        PrimaryColors {\n            background: Rgb::new(0x18, 0x18, 0x18),\n            foreground: Rgb::new(0xd8, 0xd8, 0xd8),\n            bright_foreground: Default::default(),\n            dim_foreground: Default::default(),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct NormalColors {\n    pub black: Rgb,\n    pub red: Rgb,\n    pub green: Rgb,\n    pub yellow: Rgb,\n    pub blue: Rgb,\n    pub magenta: Rgb,\n    pub cyan: Rgb,\n    pub white: Rgb,\n}\n\nimpl Default for NormalColors {\n    fn default() -> Self {\n        NormalColors {\n            black: Rgb::new(0x18, 0x18, 0x18),\n            red: Rgb::new(0xac, 0x42, 0x42),\n            green: Rgb::new(0x90, 0xa9, 0x59),\n            yellow: Rgb::new(0xf4, 0xbf, 0x75),\n            blue: Rgb::new(0x6a, 0x9f, 0xb5),\n            magenta: Rgb::new(0xaa, 0x75, 0x9f),\n            cyan: Rgb::new(0x75, 0xb5, 0xaa),\n            white: Rgb::new(0xd8, 0xd8, 0xd8),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct BrightColors {\n    pub black: Rgb,\n    pub red: Rgb,\n    pub green: Rgb,\n    pub yellow: Rgb,\n    pub blue: Rgb,\n    pub magenta: Rgb,\n    pub cyan: Rgb,\n    pub white: Rgb,\n}\n\nimpl Default for BrightColors {\n    fn default() -> Self {\n        // Generated with oklab by multiplying brightness by 1.12 and then adjusting numbers\n        // to make them look \"nicer\". Yellow color was generated the same way, however the first\n        // srgb representable color was picked.\n        BrightColors {\n            black: Rgb::new(0x6b, 0x6b, 0x6b),\n            red: Rgb::new(0xc5, 0x55, 0x55),\n            green: Rgb::new(0xaa, 0xc4, 0x74),\n            yellow: Rgb::new(0xfe, 0xca, 0x88),\n            blue: Rgb::new(0x82, 0xb8, 0xc8),\n            magenta: Rgb::new(0xc2, 0x8c, 0xb8),\n            cyan: Rgb::new(0x93, 0xd3, 0xc3),\n            white: Rgb::new(0xf8, 0xf8, 0xf8),\n        }\n    }\n}\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct DimColors {\n    pub black: Rgb,\n    pub red: Rgb,\n    pub green: Rgb,\n    pub yellow: Rgb,\n    pub blue: Rgb,\n    pub magenta: Rgb,\n    pub cyan: Rgb,\n    pub white: Rgb,\n}\n\nimpl Default for DimColors {\n    fn default() -> Self {\n        // Generated with builtin alacritty's color dimming function.\n        DimColors {\n            black: Rgb::new(0x0f, 0x0f, 0x0f),\n            red: Rgb::new(0x71, 0x2b, 0x2b),\n            green: Rgb::new(0x5f, 0x6f, 0x3a),\n            yellow: Rgb::new(0xa1, 0x7e, 0x4d),\n            blue: Rgb::new(0x45, 0x68, 0x77),\n            magenta: Rgb::new(0x70, 0x4d, 0x68),\n            cyan: Rgb::new(0x4d, 0x77, 0x70),\n            white: Rgb::new(0x8e, 0x8e, 0x8e),\n        }\n    }\n}\n\n// Path: alacritty/src/display/color.rs\nuse std::fmt::{self, Display, Formatter};\nuse std::ops::{Add, Deref, Index, IndexMut, Mul};\nuse std::str::FromStr;\n\nuse log::trace;\nuse serde::de::{Error as SerdeError, Visitor};\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::SerdeReplace;\nuse alacritty_terminal::term::color::COUNT;\nuse alacritty_terminal::vte::ansi::{NamedColor, Rgb as VteRgb};\n\nuse crate::config::color::Colors;\n\n/// Factor for automatic computation of dim colors.\npub const DIM_FACTOR: f32 = 0.66;\n\n#[derive(Copy, Clone)]\npub struct List([Rgb; COUNT]);\n\nimpl<'a> From<&'a Colors> for List {\n    fn from(colors: &Colors) -> List {\n        // Type inference fails without this annotation.\n        let mut list = List([Rgb::default(); COUNT]);\n\n        list.fill_named(colors);\n        list.fill_cube(colors);\n        list.fill_gray_ramp(colors);\n\n        list\n    }\n}\n\nimpl List {\n    pub fn fill_named(&mut self, colors: &Colors) {\n        // Normals.\n        self[NamedColor::Black] = colors.normal.black;\n        self[NamedColor::Red] = colors.normal.red;\n        self[NamedColor::Green] = colors.normal.green;\n        self[NamedColor::Yellow] = colors.normal.yellow;\n        self[NamedColor::Blue] = colors.normal.blue;\n        self[NamedColor::Magenta] = colors.normal.magenta;\n        self[NamedColor::Cyan] = colors.normal.cyan;\n        self[NamedColor::White] = colors.normal.white;\n\n        // Brights.\n        self[NamedColor::BrightBlack] = colors.bright.black;\n        self[NamedColor::BrightRed] = colors.bright.red;\n        self[NamedColor::BrightGreen] = colors.bright.green;\n        self[NamedColor::BrightYellow] = colors.bright.yellow;\n        self[NamedColor::BrightBlue] = colors.bright.blue;\n        self[NamedColor::BrightMagenta] = colors.bright.magenta;\n        self[NamedColor::BrightCyan] = colors.bright.cyan;\n        self[NamedColor::BrightWhite] = colors.bright.white;\n        self[NamedColor::BrightForeground] =\n            colors.primary.bright_foreground.unwrap_or(colors.primary.foreground);\n\n        // Foreground and background.\n        self[NamedColor::Foreground] = colors.primary.foreground;\n        self[NamedColor::Background] = colors.primary.background;\n\n        // Dims.\n        self[NamedColor::DimForeground] =\n            colors.primary.dim_foreground.unwrap_or(colors.primary.foreground * DIM_FACTOR);\n        match colors.dim {\n            Some(ref dim) => {\n                trace!(\"Using config-provided dim colors\");\n                self[NamedColor::DimBlack] = dim.black;\n                self[NamedColor::DimRed] = dim.red;\n                self[NamedColor::DimGreen] = dim.green;\n                self[NamedColor::DimYellow] = dim.yellow;\n                self[NamedColor::DimBlue] = dim.blue;\n                self[NamedColor::DimMagenta] = dim.magenta;\n                self[NamedColor::DimCyan] = dim.cyan;\n                self[NamedColor::DimWhite] = dim.white;\n            },\n            None => {\n                trace!(\"Deriving dim colors from normal colors\");\n                self[NamedColor::DimBlack] = colors.normal.black * DIM_FACTOR;\n                self[NamedColor::DimRed] = colors.normal.red * DIM_FACTOR;\n                self[NamedColor::DimGreen] = colors.normal.green * DIM_FACTOR;\n                self[NamedColor::DimYellow] = colors.normal.yellow * DIM_FACTOR;\n                self[NamedColor::DimBlue] = colors.normal.blue * DIM_FACTOR;\n                self[NamedColor::DimMagenta] = colors.normal.magenta * DIM_FACTOR;\n                self[NamedColor::DimCyan] = colors.normal.cyan * DIM_FACTOR;\n                self[NamedColor::DimWhite] = colors.normal.white * DIM_FACTOR;\n            },\n        }\n    }\n\n    \npub fn fill_cube(&mut self, colors: &Colors) {\n        let mut index: usize = 16;\n        // Build colors.\n        for r in 0..6 {\n            for g in 0..6 {\n                for b in 0..6 {\n                    // Override colors 16..232 with the config (if present).\n                    if let Some(indexed_color) =\n                        colors.indexed_colors.iter().find(|ic| ic.index() == index as u8)\n                    {\n                        self[index] = indexed_color.color;\n                    } else {\n                        self[index] = Rgb::new(\n                            if r == 0 { 0 } else { r * 40 + 55 },\n                            if g == 0 { 0 } else { g * 40 + 55 },\n                            if b == 0 { 0 } else { b * 40 + 55 },\n                        );\n                    }\n                    index += 1;\n                }\n            }\n        }\n\n        debug_assert!(index == 232);\n    }\n\n    pub fn fill_gray_ramp(&mut self, colors: &Colors) {\n        let mut index: usize = 232;\n\n        for i in 0..24 {\n            // Index of the color is number of named colors + number of cube colors + i.\n            let color_index = 16 + 216 + i;\n\n            // Override colors 232..256 with the config (if present).\n            if let Some(indexed_color) =\n                colors.indexed_colors.iter().find(|ic| ic.index() == color_index)\n            {\n                self[index] = indexed_color.color;\n                index += 1;\n                continue;\n            }\n\n            let value = i * 10 + 8;\n            self[index] = Rgb::new(value, value, value);\n            index += 1;\n        }\n\n        debug_assert!(index == 256);\n    }\n}\n\nimpl Index<usize> for List {\n    type Output = Rgb;\n\n    #[inline]\n    fn index(&self, idx: usize) -> &Self::Output {\n        &self.0[idx]\n    }\n}\n\nimpl IndexMut<usize> for List {\n    #[inline]\n    fn index_mut(&mut self, idx: usize) -> &mut Self::Output {\n        &mut self.0[idx]\n    }\n}\n\nimpl Index<NamedColor> for List {\n    type Output = Rgb;\n\n    #[inline]\n    fn index(&self, idx: NamedColor) -> &Self::Output {\n        &self.0[idx as usize]\n    }\n}\n\nimpl IndexMut<NamedColor> for List {\n    #[inline]\n    fn index_mut(&mut self, idx: NamedColor) -> &mut Self::Output {\n        &mut self.0[idx as usize]\n    }\n}\n\n#[derive(SerdeReplace, Debug, Eq, PartialEq, Copy, Clone, Default)]\npub struct Rgb(pub VteRgb);\n\nimpl Rgb {\n    #[inline]\n    pub const fn new(r: u8, g: u8, b: u8) -> Self {\n        Self(VteRgb { r, g, b })\n    }\n\n    #[inline]\n    pub fn as_tuple(self) -> (u8, u8, u8) {\n        (self.0.r, self.0.g, self.0.b)\n    }\n}\n\nimpl From<VteRgb> for Rgb {\n    fn from(value: VteRgb) -> Self {\n        Self(value)\n    }\n}\n\nimpl Deref for Rgb {\n    type Target = VteRgb;\n\n    fn deref(&self) -> &Self::Target {\n        &self.0\n    }\n}\n\nimpl Mul<f32> for Rgb {\n    type Output = Rgb;\n\n    fn mul(self, rhs: f32) -> Self::Output {\n        Rgb(self.0 * rhs)\n    }\n}\n\nimpl Add<Rgb> for Rgb {\n    type Output = Rgb;\n\n    fn add(self, rhs: Rgb) -> Self::Output {\n        Rgb(self.0 + rhs.0)\n    }\n}\n\n/// Deserialize an Rgb from a hex string.\n///\n/// This is *not* the deserialize impl for Rgb since we want a symmetric\n/// serialize/deserialize impl for ref tests.\nimpl<'de> Deserialize<'de> for Rgb {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct RgbVisitor;\n\n        // Used for deserializing reftests.\n        #[derive(Deserialize)]\n        struct RgbDerivedDeser {\n            r: u8,\n            g: u8,\n            b: u8,\n        }\n\n        impl<'a> Visitor<'a> for RgbVisitor {\n            type Value = Rgb;\n\n            fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result {\n                f.write_str(\"hex color like #ff00ff\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Rgb, E>\n            where\n                E: serde::de::Error,\n            {\n                Rgb::from_str(value).map_err(|_| {\n                    E::custom(format!(\n                        \"failed to parse rgb color {value}; expected hex color like #ff00ff\"\n                    ))\n                })\n            }\n        }\n\n        // Return an error if the syntax is incorrect.\n        let value = toml::Value::deserialize(deserializer)?;\n\n        // Attempt to deserialize from struct form.\n        if let Ok(RgbDerivedDeser { r, g, b }) = RgbDerivedDeser::deserialize(value.clone()) {\n            return Ok(Rgb::new(r, g, b));\n        }\n\n        // Deserialize from hex notation (either 0xff00ff or #ff00ff).\n        value.deserialize_str(RgbVisitor).map_err(D::Error::custom)\n    }\n}\n\nimpl Display for Rgb {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        write!(f, \"#{:02x}{:02x}{:02x}\", self.r, self.g, self.b)\n    }\n}\n\nimpl FromStr for Rgb {\n    type Err = ();\n\n    fn from_str(s: &str) -> Result<Rgb, ()> {\n        let chars = if s.starts_with(\"0x\") && s.len() == 8 {\n            &s[2..]\n        } else if s.starts_with('#') && s.len() == 7 {\n            &s[1..]\n        } else {\n            return Err(());\n        };\n\n        match u32::from_str_radix(chars, 16) {\n            Ok(mut color) => {\n                let b = (color & 0xff) as u8;\n                color >>= 8;\n                let g = (color & 0xff) as u8;\n                color >>= 8;\n                let r = color as u8;\n                Ok(Rgb::new(r, g, b))\n            },\n            Err(_) => Err(()),\n        }\n    }\n}\n\n/// RGB color optionally referencing the cell's foreground or background.\n#[derive(SerdeReplace, Copy, Clone, Debug, PartialEq, Eq)]\npub enum CellRgb {\n    CellForeground,\n    CellBackground,\n    Rgb(Rgb),\n}\n\nimpl CellRgb {\n    pub fn color(self, foreground: Rgb, background: Rgb) -> Rgb {\n        match self {\n            Self::CellForeground => foreground,\n            Self::CellBackground => background,\n            Self::Rgb(rgb) => rgb,\n        }\n    }\n}\n\nimpl Default for CellRgb {\n    fn default() -> Self {\n        Self::Rgb(Rgb::default())\n    }\n}\n\nimpl<'de> Deserialize<'de> for CellRgb {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        const EXPECTING: &str = \"CellForeground, CellBackground, or hex color like #ff00ff\";\n\n        struct CellRgbVisitor;\n        impl<'a> Visitor<'a> for CellRgbVisitor {\n            type Value = CellRgb;\n\n            fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result {\n                f.write_str(EXPECTING)\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<CellRgb, E>\n            where\n                E: serde::de::Error,\n            {\n                // Attempt to deserialize as enum constants.\n                match value {\n                    \"CellForeground\" => return Ok(CellRgb::CellForeground),\n                    \"CellBackground\" => return Ok(CellRgb::CellBackground),\n                    _ => (),\n                }\n\n                Rgb::from_str(value).map(CellRgb::Rgb).map_err(|_| {\n                    E::custom(format!(\"failed to parse color {value}; expected {EXPECTING}\"))\n                })\n            }\n        }\n\n        deserializer.deserialize_str(CellRgbVisitor).map_err(D::Error::custom)\n    }\n}\n\n// Path: alacritty/src/config/bell.rs\nuse std::time::Duration;\n\nuse alacritty_config_derive::ConfigDeserialize;\n\nuse crate::config::ui_config::Program;\nuse crate::display::color::Rgb;\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct BellConfig {\n    /// Visual bell animation function.\n    pub animation: BellAnimation,\n\n    /// Command to run on bell.\n    pub command: Option<Program>,\n\n    /// Visual bell flash color.\n    pub color: Rgb,\n\n    /// Visual bell duration in milliseconds.\n    duration: u16,\n}\n\nimpl Default for BellConfig {\n    fn default() -> Self {\n        Self {\n            color: Rgb::new(255, 255, 255),\n            animation: Default::default(),\n            command: Default::default(),\n            duration: Default::default(),\n        }\n    }\n}\n\nimpl BellConfig {\n    pub fn duration(&self) -> Duration {\n        Duration::from_millis(self.duration as u64)\n    }\n}\n\n/// `VisualBellAnimations` are modeled after a subset of CSS transitions and Robert\n/// Penner's Easing Functions.\n#[derive(ConfigDeserialize, Default, Clone, Copy, Debug, PartialEq, Eq)]\npub enum BellAnimation {\n    // CSS animation.\n    Ease,\n    // CSS animation.\n    EaseOut,\n    // Penner animation.\n    EaseOutSine,\n    // Penner animation.\n    EaseOutQuad,\n    // Penner animation.\n    EaseOutCubic,\n    // Penner animation.\n    EaseOutQuart,\n    // Penner animation.\n    EaseOutQuint,\n    // Penner animation.\n    EaseOutExpo,\n    // Penner animation.\n    EaseOutCirc,\n    // Penner animation.\n    #[default]\n    Linear,\n}\n\n// Path: alacritty/src/config/debug.rs\nuse log::LevelFilter;\n\nuse alacritty_config_derive::ConfigDeserialize;\n\n/// Debugging options.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]\npub struct Debug {\n    pub log_level: LevelFilter,\n\n    pub print_events: bool,\n\n    /// Keep the log file after quitting.\n    pub persistent_logging: bool,\n\n    /// Should show render timer.\n    pub render_timer: bool,\n\n    /// Highlight damage information produced by alacritty.\n    pub highlight_damage: bool,\n\n    /// The renderer alacritty should be using.\n    pub renderer: Option<RendererPreference>,\n\n    /// Use EGL as display API if the current platform allows it.\n    pub prefer_egl: bool,\n\n    /// Record ref test.\n    #[config(skip)]\n    pub ref_test: bool,\n}\n\nimpl Default for Debug {\n    fn default() -> Self {\n        Self {\n            log_level: LevelFilter::Warn,\n            print_events: Default::default(),\n            persistent_logging: Default::default(),\n            render_timer: Default::default(),\n            highlight_damage: Default::default(),\n            ref_test: Default::default(),\n            renderer: Default::default(),\n            prefer_egl: Default::default(),\n        }\n    }\n}\n\n/// The renderer configuration options.\n#[derive(ConfigDeserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]\npub enum RendererPreference {\n    /// OpenGL 3.3 renderer.\n    Glsl3,\n\n    /// GLES 2 renderer, with optional extensions like dual source blending.\n    Gles2,\n\n    /// Pure GLES 2 renderer.\n    Gles2Pure,\n}\n\n// Path: alacritty/src/config/window.rs\nuse std::fmt::{self, Formatter};\n\nuse log::{error, warn};\nuse serde::de::{self, MapAccess, Visitor};\nuse serde::{Deserialize, Deserializer, Serialize};\n\n#[cfg(target_os = \"macos\")]\nuse winit::platform::macos::OptionAsAlt as WinitOptionAsAlt;\nuse winit::window::{Fullscreen, Theme as WinitTheme};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\nuse crate::config::ui_config::{Delta, Percentage};\nuse crate::config::LOG_TARGET_CONFIG;\n\n/// Default Alacritty name, used for window title and class.\npub const DEFAULT_NAME: &str = \"Alacritty\";\n\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq)]\npub struct WindowConfig {\n    /// Initial position.\n    pub position: Option<Delta<i32>>,\n\n    /// Draw the window with title bar / borders.\n    pub decorations: Decorations,\n\n    /// Startup mode.\n    pub startup_mode: StartupMode,\n\n    /// XEmbed parent.\n    #[config(skip)]\n    pub embed: Option<u32>,\n\n    /// Spread out additional padding evenly.\n    pub dynamic_padding: bool,\n\n    /// Use dynamic title.\n    pub dynamic_title: bool,\n\n    /// Information to identify a particular window.\n    #[config(flatten)]\n    pub identity: Identity,\n\n    /// Background opacity from 0.0 to 1.0.\n    pub opacity: Percentage,\n\n    /// Request blur behind the window.\n    pub blur: bool,\n\n    /// Controls which `Option` key should be treated as `Alt`.\n    option_as_alt: OptionAsAlt,\n\n    /// Resize increments.\n    pub resize_increments: bool,\n\n    /// Pixel padding.\n    padding: Delta<u16>,\n\n    /// Initial dimensions.\n    dimensions: Dimensions,\n\n    /// System decorations theme variant.\n    decorations_theme_variant: Option<Theme>,\n}\n\nimpl Default for WindowConfig {\n    fn default() -> Self {\n        Self {\n            dynamic_title: true,\n            blur: Default::default(),\n            embed: Default::default(),\n            padding: Default::default(),\n            opacity: Default::default(),\n            position: Default::default(),\n            identity: Default::default(),\n            dimensions: Default::default(),\n            decorations: Default::default(),\n            startup_mode: Default::default(),\n            dynamic_padding: Default::default(),\n            resize_increments: Default::default(),\n            decorations_theme_variant: Default::default(),\n            option_as_alt: Default::default(),\n        }\n    }\n}\n\nimpl WindowConfig {\n    #[inline]\n    pub fn dimensions(&self) -> Option<Dimensions> {\n        let (lines, columns) = (self.dimensions.lines, self.dimensions.columns);\n        let (lines_is_non_zero, columns_is_non_zero) = (lines != 0, columns != 0);\n\n        if lines_is_non_zero && columns_is_non_zero {\n            // Return dimensions if both `lines` and `columns` are non-zero.\n            Some(self.dimensions)\n        } else if lines_is_non_zero || columns_is_non_zero {\n            // Warn if either `columns` or `lines` is non-zero.\n\n            let (zero_key, non_zero_key, non_zero_value) = if lines_is_non_zero {\n                (\"columns\", \"lines\", lines)\n            } else {\n                (\"lines\", \"columns\", columns)\n            };\n\n            warn!(\n                target: LOG_TARGET_CONFIG,\n                \"Both `lines` and `columns` must be non-zero for `window.dimensions` to take \\\n                 effect. Configured value of `{}` is 0 while that of `{}` is {}\",\n                zero_key,\n                non_zero_key,\n                non_zero_value,\n            );\n\n            None\n        } else {\n            None\n        }\n    }\n\n    #[inline]\n    pub fn padding(&self, scale_factor: f32) -> (f32, f32) {\n        let padding_x = (f32::from(self.padding.x) * scale_factor).floor();\n        let padding_y = (f32::from(self.padding.y) * scale_factor).floor();\n        (padding_x, padding_y)\n    }\n\n    #[inline]\n    pub fn fullscreen(&self) -> Option<Fullscreen> {\n        if self.startup_mode == StartupMode::Fullscreen {\n            Some(Fullscreen::Borderless(None))\n        } else {\n            None\n        }\n    }\n\n    #[inline]\n    pub fn maximized(&self) -> bool {\n        self.startup_mode == StartupMode::Maximized\n    }\n\n    #[cfg(target_os = \"macos\")]\n    pub fn option_as_alt(&self) -> WinitOptionAsAlt {\n        match self.option_as_alt {\n            OptionAsAlt::OnlyLeft => WinitOptionAsAlt::OnlyLeft,\n            OptionAsAlt::OnlyRight => WinitOptionAsAlt::OnlyRight,\n            OptionAsAlt::Both => WinitOptionAsAlt::Both,\n            OptionAsAlt::None => WinitOptionAsAlt::None,\n        }\n    }\n\n    pub fn theme(&self) -> Option<WinitTheme> {\n        self.decorations_theme_variant.map(WinitTheme::from)\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq, Eq)]\npub struct Identity {\n    /// Window title.\n    pub title: String,\n\n    /// Window class.\n    pub class: Class,\n}\n\nimpl Default for Identity {\n    fn default() -> Self {\n        Self { title: DEFAULT_NAME.into(), class: Default::default() }\n    }\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub enum StartupMode {\n    #[default]\n    Windowed,\n    Maximized,\n    Fullscreen,\n    SimpleFullscreen,\n}\n\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub enum Decorations {\n    #[default]\n    Full,\n    Transparent,\n    Buttonless,\n    None,\n}\n\n/// Window Dimensions.\n///\n/// Newtype to avoid passing values incorrectly.\n#[derive(ConfigDeserialize, Default, Debug, Copy, Clone, PartialEq, Eq)]\npub struct Dimensions {\n    /// Window width in character columns.\n    pub columns: usize,\n\n    /// Window Height in character lines.\n    pub lines: usize,\n}\n\n/// Window class hint.\n#[derive(SerdeReplace, Serialize, Debug, Clone, PartialEq, Eq)]\npub struct Class {\n    pub general: String,\n    pub instance: String,\n}\n\nimpl Class {\n    pub fn new(general: impl ToString, instance: impl ToString) -> Self {\n        Self { general: general.to_string(), instance: instance.to_string() }\n    }\n}\n\nimpl Default for Class {\n    fn default() -> Self {\n        Self::new(DEFAULT_NAME, DEFAULT_NAME)\n    }\n}\n\nimpl<'de> Deserialize<'de> for Class {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        struct ClassVisitor;\n        impl<'a> Visitor<'a> for ClassVisitor {\n            type Value = Class;\n\n            fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result {\n                f.write_str(\"a mapping\")\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>\n            where\n                E: de::Error,\n            {\n                Ok(Self::Value { instance: value.into(), ..Self::Value::default() })\n            }\n\n            fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>\n            where\n                M: MapAccess<'a>,\n            {\n                let mut class = Self::Value::default();\n\n                while let Some((key, value)) = map.next_entry::<String, toml::Value>()? {\n                    match key.as_str() {\n                        \"instance\" => match String::deserialize(value) {\n                            Ok(instance) => class.instance = instance,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: class.instance: {}\", err\n                                );\n                            },\n                        },\n                        \"general\" => match String::deserialize(value) {\n                            Ok(general) => class.general = general,\n                            Err(err) => {\n                                error!(\n                                    target: LOG_TARGET_CONFIG,\n                                    \"Config error: class.instance: {}\", err\n                                );\n                            },\n                        },\n                        key => warn!(target: LOG_TARGET_CONFIG, \"Unrecognized class field: {key}\"),\n                    }\n                }\n\n                Ok(class)\n            }\n        }\n\n...\n", "needle_token_start": 4023, "needle_token_end": 4300, "code_context_ntokens": 9792, "output": ["```rust\npub fn fill_cube(&mut self, colors: &Colors) {\n    let mut index: usize = 16;\n    // Build colors.\n    for r in 0..6 {\n        for g in 0..6 {\n            for b in 0..6 {\n                // Override colors 16..232 with the config (if present).\n                if let Some(indexed_color) =\n                    colors.indexed_colors.iter().find(|ic| ic.index() == index as u8)\n                {\n                    self[index] = indexed_color.color;\n                } else {\n                    self[index] = Rgb::new(\n                        if r == 0 { 0 } else { r * 40 + 55 },\n                        if g == 0 { 0 } else { g * 40 + 55 },\n                        if b == 0 { 0 } else { b * 40 + 55 },\n                    );\n                }\n                index += 1;\n            }\n        }\n    }\n\n    debug_assert!(index == 232);\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "add_render_item", "language": "rust", "path": "alacritty/src/renderer/text/mod.rs", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to manage the addition of rendering items to a batch, ensuring efficient texture usage and batch processing for rendering text cells.\n2. **Input**: The function takes three parameters: a reference to a text cell structure, a glyph structure, and size information.\n3. **Output**: There is no direct output returned by the function; however, it indirectly affects the rendering process by managing the batch of render items.\n4. **Procedure**: The function first checks if the current batch is using a different texture from the glyph's texture. If so, it flushes the current batch to render its contents. Then, it adds the new item to the batch. If adding this item fills the batch, the function triggers another batch render to process these items and clear the batch for new entries.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/renderer/text/atlas.rs\nuse std::borrow::Cow;\nuse std::ptr;\n\nuse crossfont::{BitmapBuffer, RasterizedGlyph};\n\nuse crate::gl;\nuse crate::gl::types::*;\n\nuse super::Glyph;\n\n/// Size of the Atlas.\npub const ATLAS_SIZE: i32 = 1024;\n\n/// Manages a single texture atlas.\n///\n/// The strategy for filling an atlas looks roughly like this:\n///\n/// ```text\n///                           (width, height)\n///   \u250c\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2510\n///   \u2502 10  \u2502     \u2502     \u2502     \u2502     \u2502 <- Empty spaces; can be filled while\n///   \u2502     \u2502     \u2502     \u2502     \u2502     \u2502    glyph_height < height - row_baseline\n///   \u251c\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2524\n///   \u2502 5   \u2502 6   \u2502 7   \u2502 8   \u2502 9   \u2502\n///   \u2502     \u2502     \u2502     \u2502     \u2502     \u2502\n///   \u251c\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2524 <- Row height is tallest glyph in row; this is\n///   \u2502 1   \u2502 2   \u2502 3   \u2502 4         \u2502    used as the baseline for the following row.\n///   \u2502     \u2502     \u2502     \u2502           \u2502 <- Row considered full when next glyph doesn't\n///   \u2514\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518    fit in the row.\n/// (0, 0)  x->\n/// ```\n#[derive(Debug)]\npub struct Atlas {\n    /// Texture id for this atlas.\n    id: GLuint,\n\n    /// Width of atlas.\n    width: i32,\n\n    /// Height of atlas.\n    height: i32,\n\n    /// Left-most free pixel in a row.\n    ///\n    /// This is called the extent because it is the upper bound of used pixels\n    /// in a row.\n    row_extent: i32,\n\n    /// Baseline for glyphs in the current row.\n    row_baseline: i32,\n\n    /// Tallest glyph in current row.\n    ///\n    /// This is used as the advance when end of row is reached.\n    row_tallest: i32,\n\n    /// Gles context.\n    ///\n    /// This affects the texture loading.\n    is_gles_context: bool,\n}\n\n/// Error that can happen when inserting a texture to the Atlas.\npub enum AtlasInsertError {\n    /// Texture atlas is full.\n    Full,\n\n    /// The glyph cannot fit within a single texture.\n    GlyphTooLarge,\n}\n\nimpl Atlas {\n    pub fn new(size: i32, is_gles_context: bool) -> Self {\n        let mut id: GLuint = 0;\n        unsafe {\n            gl::PixelStorei(gl::UNPACK_ALIGNMENT, 1);\n            gl::GenTextures(1, &mut id);\n            gl::BindTexture(gl::TEXTURE_2D, id);\n            // Use RGBA texture for both normal and emoji glyphs, since it has no performance\n            // impact.\n            gl::TexImage2D(\n                gl::TEXTURE_2D,\n                0,\n                gl::RGBA as i32,\n                size,\n                size,\n                0,\n                gl::RGBA,\n                gl::UNSIGNED_BYTE,\n                ptr::null(),\n            );\n\n            gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_S, gl::CLAMP_TO_EDGE as i32);\n            gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_WRAP_T, gl::CLAMP_TO_EDGE as i32);\n            gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MIN_FILTER, gl::LINEAR as i32);\n            gl::TexParameteri(gl::TEXTURE_2D, gl::TEXTURE_MAG_FILTER, gl::LINEAR as i32);\n\n            gl::BindTexture(gl::TEXTURE_2D, 0);\n        }\n\n        Self {\n            id,\n            width: size,\n            height: size,\n            row_extent: 0,\n            row_baseline: 0,\n            row_tallest: 0,\n            is_gles_context,\n        }\n    }\n\n    pub fn clear(&mut self) {\n        self.row_extent = 0;\n        self.row_baseline = 0;\n        self.row_tallest = 0;\n    }\n\n    /// Insert a RasterizedGlyph into the texture atlas.\n    pub fn insert(\n        &mut self,\n        glyph: &RasterizedGlyph,\n        active_tex: &mut u32,\n    ) -> Result<Glyph, AtlasInsertError> {\n        if glyph.width > self.width || glyph.height > self.height {\n            return Err(AtlasInsertError::GlyphTooLarge);\n        }\n\n        // If there's not enough room in current row, go onto next one.\n        if !self.room_in_row(glyph) {\n            self.advance_row()?;\n        }\n\n        // If there's still not room, there's nothing that can be done here..\n        if !self.room_in_row(glyph) {\n            return Err(AtlasInsertError::Full);\n        }\n\n        // There appears to be room; load the glyph.\n        Ok(self.insert_inner(glyph, active_tex))\n    }\n\n    /// Insert the glyph without checking for room.\n    ///\n    /// Internal function for use once atlas has been checked for space. GL\n    /// errors could still occur at this point if we were checking for them;\n    /// hence, the Result.\n    fn insert_inner(&mut self, glyph: &RasterizedGlyph, active_tex: &mut u32) -> Glyph {\n        let offset_y = self.row_baseline;\n        let offset_x = self.row_extent;\n        let height = glyph.height;\n        let width = glyph.width;\n        let multicolor;\n\n        unsafe {\n            gl::BindTexture(gl::TEXTURE_2D, self.id);\n\n            // Load data into OpenGL.\n            let (format, buffer) = match &glyph.buffer {\n                BitmapBuffer::Rgb(buffer) => {\n                    multicolor = false;\n                    // Gles context doesn't allow uploading RGB data into RGBA texture, so need\n                    // explicit copy.\n                    if self.is_gles_context {\n                        let mut new_buffer = Vec::with_capacity(buffer.len() / 3 * 4);\n                        for rgb in buffer.chunks_exact(3) {\n                            new_buffer.push(rgb[0]);\n                            new_buffer.push(rgb[1]);\n                            new_buffer.push(rgb[2]);\n                            new_buffer.push(u8::MAX);\n                        }\n                        (gl::RGBA, Cow::Owned(new_buffer))\n                    } else {\n                        (gl::RGB, Cow::Borrowed(buffer))\n                    }\n                },\n                BitmapBuffer::Rgba(buffer) => {\n                    multicolor = true;\n                    (gl::RGBA, Cow::Borrowed(buffer))\n...\n// Path: alacritty/src/renderer/text/glsl3.rs\nuse std::mem::size_of;\nuse std::ptr;\n\nuse crossfont::RasterizedGlyph;\nuse log::info;\n\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::gl::types::*;\nuse crate::renderer::shader::{ShaderProgram, ShaderVersion};\nuse crate::renderer::{cstr, Error};\n\nuse super::atlas::{Atlas, ATLAS_SIZE};\nuse super::{\n    Glyph, LoadGlyph, LoaderApi, RenderingGlyphFlags, RenderingPass, TextRenderApi,\n    TextRenderBatch, TextRenderer, TextShader,\n};\n\n// Shader source.\npub static TEXT_SHADER_F: &str = include_str!(\"../../../res/glsl3/text.f.glsl\");\nstatic TEXT_SHADER_V: &str = include_str!(\"../../../res/glsl3/text.v.glsl\");\n\n/// Maximum items to be drawn in a batch.\nconst BATCH_MAX: usize = 0x1_0000;\n\n#[derive(Debug)]\npub struct Glsl3Renderer {\n    program: TextShaderProgram,\n    vao: GLuint,\n    ebo: GLuint,\n    vbo_instance: GLuint,\n    atlas: Vec<Atlas>,\n    current_atlas: usize,\n    active_tex: GLuint,\n    batch: Batch,\n}\n\nimpl Glsl3Renderer {\n    pub fn new() -> Result<Self, Error> {\n        info!(\"Using OpenGL 3.3 renderer\");\n\n        let program = TextShaderProgram::new(ShaderVersion::Glsl3)?;\n        let mut vao: GLuint = 0;\n        let mut ebo: GLuint = 0;\n        let mut vbo_instance: GLuint = 0;\n\n        unsafe {\n            gl::Enable(gl::BLEND);\n            gl::BlendFunc(gl::SRC1_COLOR, gl::ONE_MINUS_SRC1_COLOR);\n\n            // Disable depth mask, as the renderer never uses depth tests.\n            gl::DepthMask(gl::FALSE);\n\n            gl::GenVertexArrays(1, &mut vao);\n            gl::GenBuffers(1, &mut ebo);\n            gl::GenBuffers(1, &mut vbo_instance);\n            gl::BindVertexArray(vao);\n\n            // ---------------------\n            // Set up element buffer\n            // ---------------------\n            let indices: [u32; 6] = [0, 1, 3, 1, 2, 3];\n\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, ebo);\n            gl::BufferData(\n                gl::ELEMENT_ARRAY_BUFFER,\n                (6 * size_of::<u32>()) as isize,\n                indices.as_ptr() as *const _,\n                gl::STATIC_DRAW,\n            );\n\n            // ----------------------------\n            // Setup vertex instance buffer\n            // ----------------------------\n            gl::BindBuffer(gl::ARRAY_BUFFER, vbo_instance);\n            gl::BufferData(\n                gl::ARRAY_BUFFER,\n                (BATCH_MAX * size_of::<InstanceData>()) as isize,\n                ptr::null(),\n                gl::STREAM_DRAW,\n            );\n\n            let mut index = 0;\n            let mut size = 0;\n\n            macro_rules! add_attr {\n                ($count:expr, $gl_type:expr, $type:ty) => {\n                    gl::VertexAttribPointer(\n                        index,\n                        $count,\n                        $gl_type,\n                        gl::FALSE,\n                        size_of::<InstanceData>() as i32,\n                        size as *const _,\n                    );\n                    gl::EnableVertexAttribArray(index);\n                    gl::VertexAttribDivisor(index, 1);\n\n                    #[allow(unused_assignments)]\n                    {\n                        size += $count * size_of::<$type>();\n                        index += 1;\n                    }\n                };\n            }\n\n            // Coords.\n            add_attr!(2, gl::UNSIGNED_SHORT, u16);\n\n            // Glyph offset and size.\n            add_attr!(4, gl::SHORT, i16);\n\n            // UV offset.\n            add_attr!(4, gl::FLOAT, f32);\n\n            // Color and cell flags.\n            //\n            // These are packed together because of an OpenGL driver issue on macOS, which caused a\n            // `vec3(u8)` text color and a `u8` cell flags to increase the rendering time by a\n            // huge margin.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Background color.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Cleanup.\n            gl::BindVertexArray(0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n        }\n\n        Ok(Self {\n            program,\n            vao,\n            ebo,\n            vbo_instance,\n            atlas: vec![Atlas::new(ATLAS_SIZE, false)],\n            current_atlas: 0,\n            active_tex: 0,\n            batch: Batch::new(),\n        })\n    }\n}\n\nimpl<'a> TextRenderer<'a> for Glsl3Renderer {\n    type RenderApi = RenderApi<'a>;\n    type RenderBatch = Batch;\n    type Shader = TextShaderProgram;\n\n    fn with_api<'b: 'a, F, T>(&'b mut self, size_info: &'b SizeInfo, func: F) -> T\n    where\n        F: FnOnce(Self::RenderApi) -> T,\n    {\n        unsafe {\n            gl::UseProgram(self.program.id());\n            self.program.set_term_uniforms(size_info);\n\n            gl::BindVertexArray(self.vao);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, self.ebo);\n            gl::BindBuffer(gl::ARRAY_BUFFER, self.vbo_instance);\n            gl::ActiveTexture(gl::TEXTURE0);\n        }\n\n        let res = func(RenderApi {\n            active_tex: &mut self.active_tex,\n            batch: &mut self.batch,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n            program: &mut self.program,\n        });\n\n        unsafe {\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n            gl::BindVertexArray(0);\n\n            gl::UseProgram(0);\n        }\n\n        res\n    }\n\n    fn program(&self) -> &Self::Shader {\n        &self.program\n    }\n\n    fn loader_api(&mut self) -> LoaderApi<'_> {\n        LoaderApi {\n            active_tex: &mut self.active_tex,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n        }\n    }\n}\n\nimpl Drop for Glsl3Renderer {\n    fn drop(&mut self) {\n        unsafe {\n            gl::DeleteBuffers(1, &self.vbo_instance);\n            gl::DeleteBuffers(1, &self.ebo);\n            gl::DeleteVertexArrays(1, &self.vao);\n        }\n    }\n}\n\n#[derive(Debug)]\npub struct RenderApi<'a> {\n    active_tex: &'a mut GLuint,\n    batch: &'a mut Batch,\n    atlas: &'a mut Vec<Atlas>,\n    current_atlas: &'a mut usize,\n    program: &'a mut TextShaderProgram,\n}\n\nimpl<'a> TextRenderApi<Batch> for RenderApi<'a> {\n    fn batch(&mut self) -> &mut Batch {\n        self.batch\n    }\n\n    fn render_batch(&mut self) {\n        unsafe {\n            gl::BufferSubData(\n                gl::ARRAY_BUFFER,\n                0,\n                self.batch.size() as isize,\n                self.batch.instances.as_ptr() as *const _,\n            );\n        }\n\n        // Bind texture if necessary.\n        if *self.active_tex != self.batch.tex() {\n            unsafe {\n                gl::BindTexture(gl::TEXTURE_2D, self.batch.tex());\n            }\n            *self.active_tex = self.batch.tex();\n        }\n\n        unsafe {\n            self.program.set_rendering_pass(RenderingPass::Background);\n            gl::DrawElementsInstanced(\n                gl::TRIANGLES,\n                6,\n                gl::UNSIGNED_INT,\n                ptr::null(),\n                self.batch.len() as GLsizei,\n            );\n            self.program.set_rendering_pass(RenderingPass::SubpixelPass1);\n            gl::DrawElementsInstanced(\n                gl::TRIANGLES,\n                6,\n                gl::UNSIGNED_INT,\n                ptr::null(),\n                self.batch.len() as GLsizei,\n            );\n        }\n\n        self.batch.clear();\n    }\n}\n\nimpl<'a> LoadGlyph for RenderApi<'a> {\n    fn load_glyph(&mut self, rasterized: &RasterizedGlyph) -> Glyph {\n        Atlas::load_glyph(self.active_tex, self.atlas, self.current_atlas, rasterized)\n    }\n\n    fn clear(&mut self) {\n        Atlas::clear_atlas(self.atlas, self.current_atlas)\n    }\n}\n\nimpl<'a> Drop for RenderApi<'a> {\n    fn drop(&mut self) {\n        if !self.batch.is_empty() {\n            self.render_batch();\n        }\n    }\n}\n\n#[derive(Debug)]\n#[repr(C)]\nstruct InstanceData {\n    // Coords.\n    col: u16,\n    row: u16,\n\n    // Glyph offset.\n    left: i16,\n    top: i16,\n\n    // Glyph size.\n    width: i16,\n    height: i16,\n\n    // UV offset.\n    uv_left: f32,\n    uv_bot: f32,\n\n    // uv scale.\n    uv_width: f32,\n    uv_height: f32,\n\n    // Color.\n    r: u8,\n    g: u8,\n    b: u8,\n\n    // Cell flags like multicolor or fullwidth character.\n    cell_flags: RenderingGlyphFlags,\n\n    // Background color.\n    bg_r: u8,\n    bg_g: u8,\n    bg_b: u8,\n    bg_a: u8,\n}\n\n#[derive(Debug, Default)]\npub struct Batch {\n    tex: GLuint,\n    instances: Vec<InstanceData>,\n}\n\nimpl TextRenderBatch for Batch {\n    #[inline]\n    fn tex(&self) -> GLuint {\n        self.tex\n    }\n\n    #[inline]\n    fn full(&self) -> bool {\n        self.capacity() == self.len()\n    }\n\n    #[inline]\n    fn is_empty(&self) -> bool {\n        self.len() == 0\n    }\n\n    fn add_item(&mut self, cell: &RenderableCell, glyph: &Glyph, _: &SizeInfo) {\n        if self.is_empty() {\n            self.tex = glyph.tex_id;\n        }\n\n        let mut cell_flags = RenderingGlyphFlags::empty();\n        cell_flags.set(RenderingGlyphFlags::COLORED, glyph.multicolor);\n        cell_flags.set(RenderingGlyphFlags::WIDE_CHAR, cell.flags.contains(Flags::WIDE_CHAR));\n\n        self.instances.push(InstanceData {\n            col: cell.point.column.0 as u16,\n            row: cell.point.line as u16,\n\n            top: glyph.top,\n            left: glyph.left,\n            width: glyph.width,\n            height: glyph.height,\n\n            uv_bot: glyph.uv_bot,\n            uv_left: glyph.uv_left,\n            uv_width: glyph.uv_width,\n            uv_height: glyph.uv_height,\n\n            r: cell.fg.r,\n            g: cell.fg.g,\n            b: cell.fg.b,\n            cell_flags,\n\n            bg_r: cell.bg.r,\n            bg_g: cell.bg.g,\n            bg_b: cell.bg.b,\n            bg_a: (cell.bg_alpha * 255.0) as u8,\n        });\n    }\n}\n\nimpl Batch {\n    #[inline]\n    pub fn new() -> Self {\n        Self { tex: 0, instances: Vec::with_capacity(BATCH_MAX) }\n    }\n\n    #[inline]\n    pub fn len(&self) -> usize {\n        self.instances.len()\n    }\n\n    #[inline]\n    pub fn capacity(&self) -> usize {\n        BATCH_MAX\n    }\n\n    #[inline]\n    pub fn size(&self) -> usize {\n        self.len() * size_of::<InstanceData>()\n    }\n\n    pub fn clear(&mut self) {\n        self.tex = 0;\n        self.instances.clear();\n    }\n}\n\n/// Text drawing program.\n///\n/// Uniforms are prefixed with \"u\", and vertex attributes are prefixed with \"a\".\n#[derive(Debug)]\npub struct TextShaderProgram {\n    /// Shader program.\n    program: ShaderProgram,\n\n    /// Projection scale and offset uniform.\n    u_projection: GLint,\n\n    /// Cell dimensions (pixels).\n    u_cell_dim: GLint,\n\n    /// Background pass flag.\n    ///\n    /// Rendering is split into two passes; one for backgrounds, and one for text.\n    u_rendering_pass: GLint,\n}\n\nimpl TextShaderProgram {\n    pub fn new(shader_version: ShaderVersion) -> Result<TextShaderProgram, Error> {\n        let program = ShaderProgram::new(shader_version, None, TEXT_SHADER_V, TEXT_SHADER_F)?;\n        Ok(Self {\n            u_projection: program.get_uniform_location(cstr!(\"projection\"))?,\n            u_cell_dim: program.get_uniform_location(cstr!(\"cellDim\"))?,\n            u_rendering_pass: program.get_uniform_location(cstr!(\"renderingPass\"))?,\n            program,\n        })\n    }\n\n    fn set_term_uniforms(&self, props: &SizeInfo) {\n        unsafe {\n            gl::Uniform2f(self.u_cell_dim, props.cell_width(), props.cell_height());\n        }\n    }\n\n    fn set_rendering_pass(&self, rendering_pass: RenderingPass) {\n        let value = match rendering_pass {\n            RenderingPass::Background | RenderingPass::SubpixelPass1 => rendering_pass as i32,\n            _ => unreachable!(\"provided pass is not supported in GLSL3 renderer\"),\n        };\n\n        unsafe {\n            gl::Uniform1i(self.u_rendering_pass, value);\n        }\n    }\n}\n\nimpl TextShader for TextShaderProgram {\n    fn id(&self) -> GLuint {\n        self.program.id()\n    }\n\n    fn projection_uniform(&self) -> GLint {\n        self.u_projection\n    }\n}\n\n// Path: alacritty/src/renderer/text/gles2.rs\nuse std::mem::size_of;\nuse std::ptr;\n\nuse crossfont::RasterizedGlyph;\nuse log::info;\n\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::gl::types::*;\nuse crate::renderer::shader::{ShaderProgram, ShaderVersion};\nuse crate::renderer::{cstr, Error, GlExtensions};\n\nuse super::atlas::{Atlas, ATLAS_SIZE};\nuse super::{\n    glsl3, Glyph, LoadGlyph, LoaderApi, RenderingGlyphFlags, RenderingPass, TextRenderApi,\n    TextRenderBatch, TextRenderer, TextShader,\n};\n\n// Shader source.\nstatic TEXT_SHADER_F: &str = include_str!(\"../../../res/gles2/text.f.glsl\");\nstatic TEXT_SHADER_V: &str = include_str!(\"../../../res/gles2/text.v.glsl\");\n\n#[derive(Debug)]\npub struct Gles2Renderer {\n    program: TextShaderProgram,\n    vao: GLuint,\n    vbo: GLuint,\n    ebo: GLuint,\n    atlas: Vec<Atlas>,\n    batch: Batch,\n    current_atlas: usize,\n    active_tex: GLuint,\n    dual_source_blending: bool,\n}\n\nimpl Gles2Renderer {\n    pub fn new(allow_dsb: bool, is_gles_context: bool) -> Result<Self, Error> {\n        info!(\"Using OpenGL ES 2.0 renderer\");\n\n        let dual_source_blending = allow_dsb\n            && (GlExtensions::contains(\"GL_EXT_blend_func_extended\")\n                || GlExtensions::contains(\"GL_ARB_blend_func_extended\"));\n\n        if is_gles_context {\n            info!(\"Running on OpenGL ES context\");\n        }\n\n        if dual_source_blending {\n            info!(\"Using dual source blending\");\n        }\n\n        let program = TextShaderProgram::new(ShaderVersion::Gles2, dual_source_blending)?;\n        let mut vao: GLuint = 0;\n        let mut vbo: GLuint = 0;\n        let mut ebo: GLuint = 0;\n\n        let mut vertex_indices = Vec::with_capacity(BATCH_MAX / 4 * 6);\n        for index in 0..(BATCH_MAX / 4) as u16 {\n            let index = index * 4;\n            vertex_indices.push(index);\n            vertex_indices.push(index + 1);\n            vertex_indices.push(index + 3);\n\n            vertex_indices.push(index + 1);\n            vertex_indices.push(index + 2);\n            vertex_indices.push(index + 3);\n        }\n\n        unsafe {\n            gl::Enable(gl::BLEND);\n\n            gl::DepthMask(gl::FALSE);\n\n            gl::GenVertexArrays(1, &mut vao);\n            gl::GenBuffers(1, &mut ebo);\n            gl::GenBuffers(1, &mut vbo);\n            gl::BindVertexArray(vao);\n\n            // Elements buffer.\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, ebo);\n            gl::BufferData(\n                gl::ELEMENT_ARRAY_BUFFER,\n                (vertex_indices.capacity() * size_of::<u16>()) as isize,\n                vertex_indices.as_ptr() as *const _,\n                gl::STATIC_DRAW,\n            );\n\n            // Vertex buffer.\n            gl::BindBuffer(gl::ARRAY_BUFFER, vbo);\n            gl::BufferData(\n                gl::ARRAY_BUFFER,\n                (BATCH_MAX * size_of::<TextVertex>()) as isize,\n                ptr::null(),\n                gl::STREAM_DRAW,\n            );\n\n            let mut index = 0;\n            let mut size = 0;\n\n            macro_rules! add_attr {\n                ($count:expr, $gl_type:expr, $type:ty) => {\n                    gl::VertexAttribPointer(\n                        index,\n                        $count,\n                        $gl_type,\n                        gl::FALSE,\n                        size_of::<TextVertex>() as i32,\n                        size as *const _,\n                    );\n                    gl::EnableVertexAttribArray(index);\n\n                    #[allow(unused_assignments)]\n                    {\n                        size += $count * size_of::<$type>();\n                        index += 1;\n                    }\n                };\n            }\n\n            // Cell coords.\n            add_attr!(2, gl::SHORT, i16);\n\n            // Glyph coords.\n            add_attr!(2, gl::SHORT, i16);\n\n            // UV.\n            add_attr!(2, gl::FLOAT, u32);\n\n            // Color and bitmap color.\n            //\n            // These are packed together because of an OpenGL driver issue on macOS, which caused a\n            // `vec3(u8)` text color and a `u8` for glyph color to cause performance regressions.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Background color.\n            add_attr!(4, gl::UNSIGNED_BYTE, u8);\n\n            // Cleanup.\n            gl::BindVertexArray(0);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n        }\n\n        Ok(Self {\n            program,\n            vao,\n            vbo,\n            ebo,\n            atlas: vec![Atlas::new(ATLAS_SIZE, is_gles_context)],\n            batch: Batch::new(),\n            current_atlas: 0,\n            active_tex: 0,\n            dual_source_blending,\n        })\n    }\n}\n\nimpl Drop for Gles2Renderer {\n    fn drop(&mut self) {\n        unsafe {\n            gl::DeleteBuffers(1, &self.vbo);\n            gl::DeleteBuffers(1, &self.ebo);\n            gl::DeleteVertexArrays(1, &self.vao);\n        }\n    }\n}\n\nimpl<'a> TextRenderer<'a> for Gles2Renderer {\n    type RenderApi = RenderApi<'a>;\n    type RenderBatch = Batch;\n    type Shader = TextShaderProgram;\n\n    fn program(&self) -> &Self::Shader {\n        &self.program\n    }\n\n    fn with_api<'b: 'a, F, T>(&'b mut self, _: &'b SizeInfo, func: F) -> T\n    where\n        F: FnOnce(Self::RenderApi) -> T,\n    {\n        unsafe {\n            gl::UseProgram(self.program.id());\n            gl::BindVertexArray(self.vao);\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, self.ebo);\n            gl::BindBuffer(gl::ARRAY_BUFFER, self.vbo);\n            gl::ActiveTexture(gl::TEXTURE0);\n        }\n\n        let res = func(RenderApi {\n            active_tex: &mut self.active_tex,\n            batch: &mut self.batch,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n            program: &mut self.program,\n            dual_source_blending: self.dual_source_blending,\n        });\n\n        unsafe {\n            gl::BindBuffer(gl::ELEMENT_ARRAY_BUFFER, 0);\n            gl::BindBuffer(gl::ARRAY_BUFFER, 0);\n            gl::BindVertexArray(0);\n\n            gl::UseProgram(0);\n        }\n\n        res\n    }\n\n    fn loader_api(&mut self) -> LoaderApi<'_> {\n        LoaderApi {\n            active_tex: &mut self.active_tex,\n            atlas: &mut self.atlas,\n            current_atlas: &mut self.current_atlas,\n        }\n    }\n}\n\n/// Maximum items to be drawn in a batch.\n///\n/// We use the closest number to `u16::MAX` dividable by 4 (amount of vertices we push for a glyph),\n/// since it's the maximum possible index in `glDrawElements` in GLES2.\nconst BATCH_MAX: usize = (u16::MAX - u16::MAX % 4) as usize;\n\n#[derive(Debug)]\npub struct Batch {\n    tex: GLuint,\n    vertices: Vec<TextVertex>,\n}\n\nimpl Batch {\n    fn new() -> Self {\n        Self { tex: 0, vertices: Vec::with_capacity(BATCH_MAX) }\n    }\n\n    #[inline]\n    fn len(&self) -> usize {\n        self.vertices.len()\n    }\n\n    #[inline]\n    fn capacity(&self) -> usize {\n        BATCH_MAX\n    }\n\n    #[inline]\n    fn size(&self) -> usize {\n        self.len() * size_of::<TextVertex>()\n    }\n\n    #[inline]\n    fn clear(&mut self) {\n        self.vertices.clear();\n    }\n}\n\nimpl TextRenderBatch for Batch {\n    #[inline]\n    fn tex(&self) -> GLuint {\n        self.tex\n    }\n\n    #[inline]\n    fn full(&self) -> bool {\n        self.capacity() == self.len()\n    }\n\n    #[inline]\n    fn is_empty(&self) -> bool {\n        self.len() == 0\n    }\n\n    fn add_item(&mut self, cell: &RenderableCell, glyph: &Glyph, size_info: &SizeInfo) {\n        if self.is_empty() {\n            self.tex = glyph.tex_id;\n        }\n\n        // Calculate the cell position.\n        let x = cell.point.column.0 as i16 * size_info.cell_width() as i16;\n        let y = cell.point.line as i16 * size_info.cell_height() as i16;\n\n        // Calculate the glyph position.\n        let glyph_x = cell.point.column.0 as i16 * size_info.cell_width() as i16 + glyph.left;\n        let glyph_y = (cell.point.line + 1) as i16 * size_info.cell_height() as i16 - glyph.top;\n\n        let colored = if glyph.multicolor {\n            RenderingGlyphFlags::COLORED\n        } else {\n            RenderingGlyphFlags::empty()\n        };\n\n        let is_wide = if cell.flags.contains(Flags::WIDE_CHAR) { 2 } else { 1 };\n\n        let mut vertex = TextVertex {\n            x,\n            y: y + size_info.cell_height() as i16,\n\n            glyph_x,\n            glyph_y: glyph_y + glyph.height,\n\n            u: glyph.uv_left,\n            v: glyph.uv_bot + glyph.uv_height,\n            r: cell.fg.r,\n            g: cell.fg.g,\n            b: cell.fg.b,\n            colored,\n            bg_r: cell.bg.r,\n            bg_g: cell.bg.g,\n            bg_b: cell.bg.b,\n            bg_a: (cell.bg_alpha * 255.0) as u8,\n        };\n\n        self.vertices.push(vertex);\n\n        vertex.y = y;\n        vertex.glyph_y = glyph_y;\n        vertex.u = glyph.uv_left;\n        vertex.v = glyph.uv_bot;\n        self.vertices.push(vertex);\n\n        vertex.x = x + is_wide * size_info.cell_width() as i16;\n        vertex.glyph_x = glyph_x + glyph.width;\n        vertex.u = glyph.uv_left + glyph.uv_width;\n        vertex.v = glyph.uv_bot;\n        self.vertices.push(vertex);\n\n        vertex.x = x + is_wide * size_info.cell_width() as i16;\n        vertex.y = y + size_info.cell_height() as i16;\n        vertex.glyph_x = glyph_x + glyph.width;\n        vertex.glyph_y = glyph_y + glyph.height;\n        vertex.u = glyph.uv_left + glyph.uv_width;\n        vertex.v = glyph.uv_bot + glyph.uv_height;\n        self.vertices.push(vertex);\n    }\n}\n\n#[derive(Debug)]\npub struct RenderApi<'a> {\n    active_tex: &'a mut GLuint,\n    batch: &'a mut Batch,\n    atlas: &'a mut Vec<Atlas>,\n    current_atlas: &'a mut usize,\n    program: &'a mut TextShaderProgram,\n    dual_source_blending: bool,\n}\n\nimpl<'a> Drop for RenderApi<'a> {\n    fn drop(&mut self) {\n        if !self.batch.is_empty() {\n            self.render_batch();\n        }\n    }\n}\n\nimpl<'a> LoadGlyph for RenderApi<'a> {\n    fn load_glyph(&mut self, rasterized: &RasterizedGlyph) -> Glyph {\n        Atlas::load_glyph(self.active_tex, self.atlas, self.current_atlas, rasterized)\n    }\n\n    fn clear(&mut self) {\n        Atlas::clear_atlas(self.atlas, self.current_atlas)\n    }\n}\n\nimpl<'a> TextRenderApi<Batch> for RenderApi<'a> {\n    fn batch(&mut self) -> &mut Batch {\n        self.batch\n    }\n\n    fn render_batch(&mut self) {\n        unsafe {\n            gl::BufferSubData(\n                gl::ARRAY_BUFFER,\n                0,\n                self.batch.size() as isize,\n                self.batch.vertices.as_ptr() as *const _,\n            );\n        }\n\n        if *self.active_tex != self.batch.tex() {\n            unsafe {\n                gl::BindTexture(gl::TEXTURE_2D, self.batch.tex());\n            }\n            *self.active_tex = self.batch.tex();\n        }\n\n        unsafe {\n            let num_indices = (self.batch.len() / 4 * 6) as i32;\n\n            // The rendering is inspired by\n            // https://github.com/servo/webrender/blob/master/webrender/doc/text-rendering.md.\n\n            // Draw background.\n            self.program.set_rendering_pass(RenderingPass::Background);\n            gl::BlendFunc(gl::ONE, gl::ZERO);\n            gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n\n            self.program.set_rendering_pass(RenderingPass::SubpixelPass1);\n            if self.dual_source_blending {\n                // Text rendering pass.\n                gl::BlendFunc(gl::SRC1_COLOR, gl::ONE_MINUS_SRC1_COLOR);\n            } else {\n                // First text rendering pass.\n                gl::BlendFuncSeparate(gl::ZERO, gl::ONE_MINUS_SRC_COLOR, gl::ZERO, gl::ONE);\n                gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n\n                // Second text rendering pass.\n                self.program.set_rendering_pass(RenderingPass::SubpixelPass2);\n                gl::BlendFuncSeparate(gl::ONE_MINUS_DST_ALPHA, gl::ONE, gl::ZERO, gl::ONE);\n                gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n\n                // Third text rendering pass.\n                self.program.set_rendering_pass(RenderingPass::SubpixelPass3);\n                gl::BlendFuncSeparate(gl::ONE, gl::ONE, gl::ONE, gl::ONE_MINUS_SRC_ALPHA);\n            }\n\n            gl::DrawElements(gl::TRIANGLES, num_indices, gl::UNSIGNED_SHORT, ptr::null());\n        }\n\n        self.batch.clear();\n    }\n}\n\n#[repr(C)]\n#[derive(Debug, Copy, Clone)]\nstruct TextVertex {\n    // Cell coordinates.\n    x: i16,\n    y: i16,\n\n    // Glyph coordinates.\n    glyph_x: i16,\n    glyph_y: i16,\n\n    // Offsets into Atlas.\n    u: f32,\n    v: f32,\n\n    // Color.\n    r: u8,\n    g: u8,\n    b: u8,\n\n    // Whether the glyph is colored.\n    colored: RenderingGlyphFlags,\n\n    // Background color.\n    bg_r: u8,\n    bg_g: u8,\n    bg_b: u8,\n    bg_a: u8,\n}\n\n#[derive(Debug)]\npub struct TextShaderProgram {\n    /// Shader program.\n    program: ShaderProgram,\n\n    /// Projection scale and offset uniform.\n    u_projection: GLint,\n\n    /// Rendering pass.\n    ///\n    /// For dual source blending, there are 2 passes; one for background, another for text,\n    /// similar to the GLSL3 renderer.\n    ///\n    /// If GL_EXT_blend_func_extended is not available, the rendering is split into 4 passes.\n    /// One is used for the background and the rest to perform subpixel text rendering according to\n    /// <https://github.com/servo/webrender/blob/master/webrender/doc/text-rendering.md>.\n    ///\n    /// Rendering is split into three passes.\n    u_rendering_pass: GLint,\n}\n\nimpl TextShaderProgram {\n    pub fn new(shader_version: ShaderVersion, dual_source_blending: bool) -> Result<Self, Error> {\n        let fragment_shader =\n            if dual_source_blending { &glsl3::TEXT_SHADER_F } else { &TEXT_SHADER_F };\n\n        let program = ShaderProgram::new(shader_version, None, TEXT_SHADER_V, fragment_shader)?;\n\n        Ok(Self {\n            u_projection: program.get_uniform_location(cstr!(\"projection\"))?,\n            u_rendering_pass: program.get_uniform_location(cstr!(\"renderingPass\"))?,\n            program,\n        })\n    }\n\n    fn set_rendering_pass(&self, rendering_pass: RenderingPass) {\n        unsafe { gl::Uniform1i(self.u_rendering_pass, rendering_pass as i32) }\n    }\n}\n\nimpl TextShader for TextShaderProgram {\n    fn id(&self) -> GLuint {\n        self.program.id()\n    }\n\n    fn projection_uniform(&self) -> GLint {\n        self.u_projection\n    }\n}\n\n// Path: alacritty/src/renderer/text/mod.rs\nuse bitflags::bitflags;\nuse crossfont::{GlyphKey, RasterizedGlyph};\n\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::gl::types::*;\n\nmod atlas;\nmod builtin_font;\nmod gles2;\nmod glsl3;\npub mod glyph_cache;\n\nuse atlas::Atlas;\npub use gles2::Gles2Renderer;\npub use glsl3::Glsl3Renderer;\npub use glyph_cache::GlyphCache;\nuse glyph_cache::{Glyph, LoadGlyph};\n\n// NOTE: These flags must be in sync with their usage in the text.*.glsl shaders.\nbitflags! {\n    #[repr(C)]\n    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]\n    struct RenderingGlyphFlags: u8 {\n        const COLORED   = 0b0000_0001;\n        const WIDE_CHAR = 0b0000_0010;\n    }\n}\n\n/// Rendering passes, for both GLES2 and GLSL3 renderer.\n#[repr(u8)]\nenum RenderingPass {\n    /// Rendering pass used to render background color in text shaders.\n    Background = 0,\n\n    /// The first pass to render text with both GLES2 and GLSL3 renderers.\n    SubpixelPass1 = 1,\n\n    /// The second pass to render text with GLES2 renderer.\n    SubpixelPass2 = 2,\n\n    /// The third pass to render text with GLES2 renderer.\n    SubpixelPass3 = 3,\n}\n\npub trait TextRenderer<'a> {\n    type Shader: TextShader;\n    type RenderBatch: TextRenderBatch;\n    type RenderApi: TextRenderApi<Self::RenderBatch>;\n\n    /// Get loader API for the renderer.\n    fn loader_api(&mut self) -> LoaderApi<'_>;\n\n    /// Draw cells.\n    fn draw_cells<'b: 'a, I: Iterator<Item = RenderableCell>>(\n        &'b mut self,\n        size_info: &'b SizeInfo,\n        glyph_cache: &'a mut GlyphCache,\n        cells: I,\n    ) {\n        self.with_api(size_info, |mut api| {\n            for cell in cells {\n                api.draw_cell(cell, glyph_cache, size_info);\n            }\n        })\n    }\n\n    fn with_api<'b: 'a, F, T>(&'b mut self, size_info: &'b SizeInfo, func: F) -> T\n    where\n        F: FnOnce(Self::RenderApi) -> T;\n\n    fn program(&self) -> &Self::Shader;\n\n    /// Resize the text rendering.\n    fn resize(&self, size: &SizeInfo) {\n        unsafe {\n            let program = self.program();\n            gl::UseProgram(program.id());\n            update_projection(program.projection_uniform(), size);\n            gl::UseProgram(0);\n        }\n    }\n\n    /// Invoke renderer with the loader.\n    fn with_loader<F: FnOnce(LoaderApi<'_>) -> T, T>(&mut self, func: F) -> T {\n        unsafe {\n            gl::ActiveTexture(gl::TEXTURE0);\n        }\n\n        func(self.loader_api())\n    }\n}\n\npub trait TextRenderBatch {\n    /// Check if `Batch` is empty.\n    fn is_empty(&self) -> bool;\n\n    /// Check whether the `Batch` is full.\n    fn full(&self) -> bool;\n\n    /// Get texture `Batch` is using.\n    fn tex(&self) -> GLuint;\n\n    /// Add item to the batch.\n    fn add_item(&mut self, cell: &RenderableCell, glyph: &Glyph, size_info: &SizeInfo);\n}\n\npub trait TextRenderApi<T: TextRenderBatch>: LoadGlyph {\n    /// Get `Batch` the api is using.\n    fn batch(&mut self) -> &mut T;\n\n    /// Render the underlying data.\n    fn render_batch(&mut self);\n\n    /// Add item to the rendering queue.\n    #[inline]\n    \nfn add_render_item(&mut self, cell: &RenderableCell, glyph: &Glyph, size_info: &SizeInfo) {\n        // Flush batch if tex changing.\n        if !self.batch().is_empty() && self.batch().tex() != glyph.tex_id {\n            self.render_batch();\n        }\n\n        self.batch().add_item(cell, glyph, size_info);\n\n        // Render batch and clear if it's full.\n        if self.batch().full() {\n            self.render_batch();\n        }\n    }\n\n    /// Draw cell.\n    fn draw_cell(\n        &mut self,\n        mut cell: RenderableCell,\n        glyph_cache: &mut GlyphCache,\n        size_info: &SizeInfo,\n    ) {\n        // Get font key for cell.\n        let font_key = match cell.flags & Flags::BOLD_ITALIC {\n            Flags::BOLD_ITALIC => glyph_cache.bold_italic_key,\n            Flags::ITALIC => glyph_cache.italic_key,\n            Flags::BOLD => glyph_cache.bold_key,\n            _ => glyph_cache.font_key,\n        };\n\n        // Ignore hidden cells and render tabs as spaces to prevent font issues.\n        let hidden = cell.flags.contains(Flags::HIDDEN);\n        if cell.character == '\\t' || hidden {\n            cell.character = ' ';\n        }\n\n        let mut glyph_key =\n            GlyphKey { font_key, size: glyph_cache.font_size, character: cell.character };\n\n        // Add cell to batch.\n        let glyph = glyph_cache.get(glyph_key, self, true);\n        self.add_render_item(&cell, &glyph, size_info);\n\n        // Render visible zero-width characters.\n        if let Some(zerowidth) =\n            cell.extra.as_mut().and_then(|extra| extra.zerowidth.take().filter(|_| !hidden))\n        {\n            for character in zerowidth {\n                glyph_key.character = character;\n                let glyph = glyph_cache.get(glyph_key, self, false);\n                self.add_render_item(&cell, &glyph, size_info);\n            }\n        }\n    }\n}\n\npub trait TextShader {\n    fn id(&self) -> GLuint;\n\n    /// Id of the projection uniform.\n    fn projection_uniform(&self) -> GLint;\n}\n\n#[derive(Debug)]\npub struct LoaderApi<'a> {\n    active_tex: &'a mut GLuint,\n    atlas: &'a mut Vec<Atlas>,\n    current_atlas: &'a mut usize,\n}\n\nimpl<'a> LoadGlyph for LoaderApi<'a> {\n    fn load_glyph(&mut self, rasterized: &RasterizedGlyph) -> Glyph {\n        Atlas::load_glyph(self.active_tex, self.atlas, self.current_atlas, rasterized)\n    }\n\n    fn clear(&mut self) {\n        Atlas::clear_atlas(self.atlas, self.current_atlas)\n    }\n}\n\nfn update_projection(u_projection: GLint, size: &SizeInfo) {\n    let width = size.width();\n    let height = size.height();\n    let padding_x = size.padding_x();\n    let padding_y = size.padding_y();\n\n    // Bounds check.\n    if (width as u32) < (2 * padding_x as u32) || (height as u32) < (2 * padding_y as u32) {\n        return;\n    }\n\n    // Compute scale and offset factors, from pixel to ndc space. Y is inverted.\n    //   [0, width - 2 * padding_x] to [-1, 1]\n    //   [height - 2 * padding_y, 0] to [-1, 1]\n    let scale_x = 2. / (width - 2. * padding_x);\n    let scale_y = -2. / (height - 2. * padding_y);\n    let offset_x = -1.;\n    let offset_y = 1.;\n\n    unsafe {\n        gl::Uniform4f(u_projection, offset_x, offset_y, scale_x, scale_y);\n    }\n}\n\n// Path: alacritty/src/renderer/platform.rs\n//! The graphics platform that is used by the renderer.\n\nuse std::num::NonZeroU32;\n\nuse glutin::config::{ColorBufferType, Config, ConfigTemplateBuilder, GetGlConfig};\nuse glutin::context::{\n    ContextApi, ContextAttributesBuilder, GlProfile, NotCurrentContext, Version,\n};\nuse glutin::display::{Display, DisplayApiPreference, GetGlDisplay};\nuse glutin::error::Result as GlutinResult;\nuse glutin::prelude::*;\nuse glutin::surface::{Surface, SurfaceAttributesBuilder, WindowSurface};\nuse log::{debug, LevelFilter};\n\nuse raw_window_handle::{RawDisplayHandle, RawWindowHandle};\nuse winit::dpi::PhysicalSize;\n#[cfg(all(feature = \"x11\", not(any(target_os = \"macos\", windows))))]\nuse winit::platform::x11;\n\n/// Create the GL display.\npub fn create_gl_display(\n    raw_display_handle: RawDisplayHandle,\n    _raw_window_handle: Option<RawWindowHandle>,\n    _prefer_egl: bool,\n) -> GlutinResult<Display> {\n    #[cfg(target_os = \"macos\")]\n    let preference = DisplayApiPreference::Cgl;\n\n    #[cfg(windows)]\n    let preference = if _prefer_egl {\n        DisplayApiPreference::EglThenWgl(Some(_raw_window_handle.unwrap()))\n    } else {\n        DisplayApiPreference::WglThenEgl(Some(_raw_window_handle.unwrap()))\n    };\n\n    #[cfg(all(feature = \"x11\", not(any(target_os = \"macos\", windows))))]\n    let preference = if _prefer_egl {\n        DisplayApiPreference::EglThenGlx(Box::new(x11::register_xlib_error_hook))\n    } else {\n        DisplayApiPreference::GlxThenEgl(Box::new(x11::register_xlib_error_hook))\n    };\n\n    #[cfg(all(not(feature = \"x11\"), not(any(target_os = \"macos\", windows))))]\n    let preference = DisplayApiPreference::Egl;\n\n    let display = unsafe { Display::new(raw_display_handle, preference)? };\n    log::info!(\"Using {}\", { display.version_string() });\n    Ok(display)\n}\n\npub fn pick_gl_config(\n    gl_display: &Display,\n    raw_window_handle: Option<RawWindowHandle>,\n) -> Result<Config, String> {\n    let mut default_config = ConfigTemplateBuilder::new()\n        .with_depth_size(0)\n        .with_stencil_size(0)\n        .with_transparency(true);\n\n    if let Some(raw_window_handle) = raw_window_handle {\n        default_config = default_config.compatible_with_native_window(raw_window_handle);\n    }\n\n    let config_10bit = default_config\n        .clone()\n        .with_buffer_type(ColorBufferType::Rgb { r_size: 10, g_size: 10, b_size: 10 })\n        .with_alpha_size(2);\n\n    let configs = [\n        default_config.clone(),\n        config_10bit.clone(),\n        default_config.with_transparency(false),\n        config_10bit.with_transparency(false),\n    ];\n\n    for config in configs {\n        let gl_config = unsafe {\n            gl_display.find_configs(config.build()).ok().and_then(|mut configs| configs.next())\n        };\n\n        if let Some(gl_config) = gl_config {\n            debug!(\n                r#\"Picked GL Config:\n  buffer_type: {:?}\n  alpha_size: {}\n  num_samples: {}\n  hardware_accelerated: {:?}\n  supports_transparency: {:?}\n  config_api: {:?}\n  srgb_capable: {}\"#,\n                gl_config.color_buffer_type(),\n                gl_config.alpha_size(),\n                gl_config.num_samples(),\n                gl_config.hardware_accelerated(),\n                gl_config.supports_transparency(),\n                gl_config.api(),\n                gl_config.srgb_capable(),\n            );\n\n            return Ok(gl_config);\n        }\n    }\n\n    Err(String::from(\"failed to find suitable GL configuration.\"))\n}\n\npub fn create_gl_context(\n    gl_display: &Display,\n    gl_config: &Config,\n    raw_window_handle: Option<RawWindowHandle>,\n) -> GlutinResult<NotCurrentContext> {\n    let debug = log::max_level() >= LevelFilter::Debug;\n    let mut profiles = [\n        ContextAttributesBuilder::new()\n            .with_debug(debug)\n            .with_context_api(ContextApi::OpenGl(Some(Version::new(3, 3))))\n            .build(raw_window_handle),\n        // Try gles before OpenGL 2.1 as it tends to be more stable.\n        ContextAttributesBuilder::new()\n            .with_debug(debug)\n            .with_context_api(ContextApi::Gles(Some(Version::new(2, 0))))\n            .build(raw_window_handle),\n        ContextAttributesBuilder::new()\n            .with_debug(debug)\n            .with_profile(GlProfile::Compatibility)\n            .with_context_api(ContextApi::OpenGl(Some(Version::new(2, 1))))\n            .build(raw_window_handle),\n    ]\n    .into_iter();\n\n    // Try the optimal config first.\n    let mut picked_context =\n        unsafe { gl_display.create_context(gl_config, &profiles.next().unwrap()) };\n\n    // Try the fallback ones.\n    while let (Err(_), Some(profile)) = (picked_context.as_ref(), profiles.next()) {\n        picked_context = unsafe { gl_display.create_context(gl_config, &profile) };\n    }\n\n    picked_context\n}\n\npub fn create_gl_surface(\n    gl_context: &NotCurrentContext,\n    size: PhysicalSize<u32>,\n    raw_window_handle: RawWindowHandle,\n) -> GlutinResult<Surface<WindowSurface>> {\n    // Get the display and the config used to create that context.\n    let gl_display = gl_context.display();\n    let gl_config = gl_context.config();\n\n    let surface_attributes =\n        SurfaceAttributesBuilder::<WindowSurface>::new().with_srgb(Some(false)).build(\n            raw_window_handle,\n            NonZeroU32::new(size.width).unwrap(),\n            NonZeroU32::new(size.height).unwrap(),\n        );\n\n    // Create the GL surface to draw into.\n    unsafe { gl_display.create_window_surface(&gl_config, &surface_attributes) }\n}\n\n// Path: alacritty/src/renderer/mod.rs\nuse std::borrow::Cow;\nuse std::collections::HashSet;\nuse std::ffi::{CStr, CString};\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::OnceLock;\nuse std::{fmt, ptr};\n\nuse ahash::RandomState;\nuse crossfont::Metrics;\nuse glutin::context::{ContextApi, GlContext, PossiblyCurrentContext};\nuse glutin::display::{GetGlDisplay, GlDisplay};\nuse log::{debug, error, info, warn, LevelFilter};\nuse unicode_width::UnicodeWidthChar;\n\nuse alacritty_terminal::index::Point;\nuse alacritty_terminal::term::cell::Flags;\n\nuse crate::config::debug::RendererPreference;\nuse crate::display::color::Rgb;\nuse crate::display::content::RenderableCell;\nuse crate::display::SizeInfo;\nuse crate::gl;\nuse crate::renderer::rects::{RectRenderer, RenderRect};\nuse crate::renderer::shader::ShaderError;\n\npub mod platform;\npub mod rects;\nmod shader;\nmod text;\n\npub use text::{GlyphCache, LoaderApi};\n\nuse shader::ShaderVersion;\nuse text::{Gles2Renderer, Glsl3Renderer, TextRenderer};\n\nmacro_rules! cstr {\n    ($s:literal) => {\n        // This can be optimized into an no-op with pre-allocated NUL-terminated bytes.\n        unsafe { std::ffi::CStr::from_ptr(concat!($s, \"\\0\").as_ptr().cast()) }\n    };\n}\npub(crate) use cstr;\n\n/// Whether the OpenGL functions have been loaded.\npub static GL_FUNS_LOADED: AtomicBool = AtomicBool::new(false);\n\n#[derive(Debug)]\npub enum Error {\n    /// Shader error.\n    Shader(ShaderError),\n\n    /// Other error.\n    Other(String),\n}\n\nimpl std::error::Error for Error {\n    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {\n        match self {\n            Error::Shader(err) => err.source(),\n            Error::Other(_) => None,\n        }\n    }\n}\n\nimpl fmt::Display for Error {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            Error::Shader(err) => {\n                write!(f, \"There was an error initializing the shaders: {}\", err)\n            },\n            Error::Other(err) => {\n                write!(f, \"{}\", err)\n            },\n        }\n    }\n}\n\nimpl From<ShaderError> for Error {\n    fn from(val: ShaderError) -> Self {\n        Error::Shader(val)\n    }\n}\n\nimpl From<String> for Error {\n    fn from(val: String) -> Self {\n        Error::Other(val)\n    }\n}\n\n#[derive(Debug)]\nenum TextRendererProvider {\n    Gles2(Gles2Renderer),\n    Glsl3(Glsl3Renderer),\n}\n\n#[derive(Debug)]\npub struct Renderer {\n    text_renderer: TextRendererProvider,\n    rect_renderer: RectRenderer,\n}\n\n/// Wrapper around gl::GetString with error checking and reporting.\nfn gl_get_string(\n    string_id: gl::types::GLenum,\n    description: &str,\n) -> Result<Cow<'static, str>, Error> {\n    unsafe {\n        let string_ptr = gl::GetString(string_id);\n        match gl::GetError() {\n            gl::NO_ERROR if !string_ptr.is_null() => {\n                Ok(CStr::from_ptr(string_ptr as *const _).to_string_lossy())\n            },\n            gl::INVALID_ENUM => {\n                Err(format!(\"OpenGL error requesting {}: invalid enum\", description).into())\n            },\n            error_id => Err(format!(\"OpenGL error {} requesting {}\", error_id, description).into()),\n        }\n    }\n}\n\nimpl Renderer {\n    /// Create a new renderer.\n    ///\n    /// This will automatically pick between the GLES2 and GLSL3 renderer based on the GPU's\n    /// supported OpenGL version.\n    pub fn new(\n        context: &PossiblyCurrentContext,\n        renderer_preference: Option<RendererPreference>,\n    ) -> Result<Self, Error> {\n        // We need to load OpenGL functions once per instance, but only after we make our context\n        // current due to WGL limitations.\n        if !GL_FUNS_LOADED.swap(true, Ordering::Relaxed) {\n            let gl_display = context.display();\n            gl::load_with(|symbol| {\n                let symbol = CString::new(symbol).unwrap();\n...\n", "needle_token_start": 12008, "needle_token_end": 12147, "code_context_ntokens": 16119, "output": ["```rust\nfn add_render_item(&mut self, cell: &RenderableCell, glyph: &Glyph, size_info: &SizeInfo) {\n    // Flush batch if tex changing.\n    if !self.batch().is_empty() && self.batch().tex() != glyph.tex_id {\n        self.render_batch();\n    }\n\n    self.batch().add_item(cell, glyph, size_info);\n\n    // Render batch and clear if it's full.\n    if self.batch().full() {\n        self.render_batch();\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "term_options", "language": "rust", "path": "alacritty/src/config/ui_config.rs", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a configuration object for a terminal based on the settings specified in a user interface configuration object. This configuration object controls various terminal behaviors such as cursor styles, scroll history, and special terminal features.\n2. **Input**: The function takes no explicit parameters as it is a method of a class. It utilizes the instance's current state to generate the output.\n3. **Output**: Returns a configuration object for the terminal, which includes settings for escape characters, scroll history, cursor styles, and support for specific terminal features.\n4. **Procedure**: The function extracts relevant settings from the instance (like escape characters, scroll history, and cursor styles) and uses these to populate a new terminal configuration object. It sets some terminal features to fixed values (e.g., support for a specific keyboard protocol).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/config/bindings.rs\n#![allow(clippy::enum_glob_use)]\n\nuse std::fmt::{self, Debug, Display};\n\nuse bitflags::bitflags;\nuse serde::de::{self, Error as SerdeError, MapAccess, Unexpected, Visitor};\nuse serde::{Deserialize, Deserializer};\nuse toml::Value as SerdeValue;\nuse winit::event::MouseButton;\nuse winit::keyboard::{\n    Key, KeyCode, KeyLocation as WinitKeyLocation, ModifiersState, NamedKey, PhysicalKey,\n};\nuse winit::platform::scancode::PhysicalKeyExtScancode;\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\nuse alacritty_terminal::term::TermMode;\nuse alacritty_terminal::vi_mode::ViMotion;\n\nuse crate::config::ui_config::{Hint, Program, StringVisitor};\n\n/// Describes a state and action to take in that state.\n///\n/// This is the shared component of `MouseBinding` and `KeyBinding`.\n#[derive(Debug, Clone, PartialEq, Eq)]\npub struct Binding<T> {\n    /// Modifier keys required to activate binding.\n    pub mods: ModifiersState,\n\n    /// String to send to PTY if mods and mode match.\n    pub action: Action,\n\n    /// Binding mode required to activate binding.\n    pub mode: BindingMode,\n\n    /// Excluded binding modes where the binding won't be activated.\n    pub notmode: BindingMode,\n\n    /// This property is used as part of the trigger detection code.\n    ///\n    /// For example, this might be a key like \"G\", or a mouse button.\n    pub trigger: T,\n}\n\n/// Bindings that are triggered by a keyboard key.\npub type KeyBinding = Binding<BindingKey>;\n\n/// Bindings that are triggered by a mouse button.\npub type MouseBinding = Binding<MouseButton>;\n\nimpl<T: Eq> Binding<T> {\n    #[inline]\n    pub fn is_triggered_by(&self, mode: BindingMode, mods: ModifiersState, input: &T) -> bool {\n        // Check input first since bindings are stored in one big list. This is\n        // the most likely item to fail so prioritizing it here allows more\n        // checks to be short circuited.\n        self.trigger == *input\n            && self.mods == mods\n            && mode.contains(self.mode)\n            && !mode.intersects(self.notmode)\n    }\n\n    #[inline]\n    pub fn triggers_match(&self, binding: &Binding<T>) -> bool {\n        // Check the binding's key and modifiers.\n        if self.trigger != binding.trigger || self.mods != binding.mods {\n            return false;\n        }\n\n        let selfmode = if self.mode.is_empty() { BindingMode::all() } else { self.mode };\n        let bindingmode = if binding.mode.is_empty() { BindingMode::all() } else { binding.mode };\n\n        if !selfmode.intersects(bindingmode) {\n            return false;\n        }\n\n        // The bindings are never active at the same time when the required modes of one binding\n        // are part of the forbidden bindings of the other.\n        if self.mode.intersects(binding.notmode) || binding.mode.intersects(self.notmode) {\n            return false;\n        }\n\n        true\n    }\n}\n\n#[derive(ConfigDeserialize, Debug, Clone, PartialEq, Eq)]\npub enum Action {\n    /// Write an escape sequence.\n    #[config(skip)]\n    Esc(String),\n\n    /// Run given command.\n    #[config(skip)]\n    Command(Program),\n\n    /// Regex keyboard hints.\n    #[config(skip)]\n    Hint(Hint),\n\n    /// Move vi mode cursor.\n    #[config(skip)]\n    ViMotion(ViMotion),\n\n    /// Perform vi mode action.\n    #[config(skip)]\n    Vi(ViAction),\n\n    /// Perform search mode action.\n    #[config(skip)]\n    Search(SearchAction),\n\n    /// Perform mouse binding exclusive action.\n    #[config(skip)]\n    Mouse(MouseAction),\n\n    /// Paste contents of system clipboard.\n    Paste,\n\n    /// Store current selection into clipboard.\n    Copy,\n\n    /// Store current selection into selection buffer.\n    CopySelection,\n\n    /// Paste contents of selection buffer.\n    PasteSelection,\n\n    /// Increase font size.\n    IncreaseFontSize,\n\n    /// Decrease font size.\n    DecreaseFontSize,\n\n    /// Reset font size to the config value.\n    ResetFontSize,\n\n    /// Scroll exactly one page up.\n    ScrollPageUp,\n\n    /// Scroll exactly one page down.\n    ScrollPageDown,\n\n    /// Scroll half a page up.\n    ScrollHalfPageUp,\n\n    /// Scroll half a page down.\n    ScrollHalfPageDown,\n\n    /// Scroll one line up.\n    ScrollLineUp,\n\n    /// Scroll one line down.\n    ScrollLineDown,\n\n    /// Scroll all the way to the top.\n    ScrollToTop,\n\n    /// Scroll all the way to the bottom.\n    ScrollToBottom,\n\n    /// Clear the display buffer(s) to remove history.\n    ClearHistory,\n\n    /// Hide the Alacritty window.\n    Hide,\n\n    /// Hide all windows other than Alacritty on macOS.\n    HideOtherApplications,\n\n    /// Minimize the Alacritty window.\n    Minimize,\n\n    /// Quit Alacritty.\n    Quit,\n\n    /// Clear warning and error notices.\n    ClearLogNotice,\n\n    /// Spawn a new instance of Alacritty.\n    SpawnNewInstance,\n\n    /// Select next tab.\n    SelectNextTab,\n\n    /// Select previous tab.\n    SelectPreviousTab,\n\n    /// Select the first tab.\n    SelectTab1,\n\n    /// Select the second tab.\n    SelectTab2,\n\n    /// Select the third tab.\n    SelectTab3,\n\n    /// Select the fourth tab.\n    SelectTab4,\n\n    /// Select the fifth tab.\n    SelectTab5,\n\n    /// Select the sixth tab.\n    SelectTab6,\n\n    /// Select the seventh tab.\n    SelectTab7,\n\n    /// Select the eighth tab.\n    SelectTab8,\n\n    /// Select the ninth tab.\n    SelectTab9,\n\n    /// Select the last tab.\n    SelectLastTab,\n\n    /// Create a new Alacritty window.\n    CreateNewWindow,\n\n    /// Create new window in a tab.\n    CreateNewTab,\n\n    /// Toggle fullscreen.\n    ToggleFullscreen,\n\n    /// Toggle maximized.\n    ToggleMaximized,\n\n    /// Toggle simple fullscreen on macOS.\n    ToggleSimpleFullscreen,\n\n    /// Clear active selection.\n    ClearSelection,\n\n    /// Toggle vi mode.\n    ToggleViMode,\n\n    /// Allow receiving char input.\n    ReceiveChar,\n\n    /// Start a forward buffer search.\n    SearchForward,\n\n    /// Start a backward buffer search.\n    SearchBackward,\n\n    /// No action.\n    None,\n}\n\nimpl From<&'static str> for Action {\n    fn from(s: &'static str) -> Action {\n        Action::Esc(s.into())\n    }\n}\n\nimpl From<ViAction> for Action {\n    fn from(action: ViAction) -> Self {\n        Self::Vi(action)\n    }\n}\n\nimpl From<ViMotion> for Action {\n    fn from(motion: ViMotion) -> Self {\n        Self::ViMotion(motion)\n    }\n}\n\nimpl From<SearchAction> for Action {\n    fn from(action: SearchAction) -> Self {\n        Self::Search(action)\n    }\n}\n\nimpl From<MouseAction> for Action {\n    fn from(action: MouseAction) -> Self {\n        Self::Mouse(action)\n    }\n}\n\n/// Display trait used for error logging.\nimpl Display for Action {\n    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n        match self {\n            Action::ViMotion(motion) => motion.fmt(f),\n            Action::Vi(action) => action.fmt(f),\n            Action::Mouse(action) => action.fmt(f),\n            _ => write!(f, \"{:?}\", self),\n        }\n    }\n}\n\n/// Vi mode specific actions.\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub enum ViAction {\n    /// Toggle normal vi selection.\n    ToggleNormalSelection,\n    /// Toggle line vi selection.\n    ToggleLineSelection,\n    /// Toggle block vi selection.\n    ToggleBlockSelection,\n    /// Toggle semantic vi selection.\n    ToggleSemanticSelection,\n    /// Jump to the beginning of the next match.\n    SearchNext,\n    /// Jump to the beginning of the previous match.\n    SearchPrevious,\n    /// Jump to the next start of a match to the left of the origin.\n    SearchStart,\n    /// Jump to the next end of a match to the right of the origin.\n    SearchEnd,\n    /// Launch the URL below the vi mode cursor.\n    Open,\n    /// Centers the screen around the vi mode cursor.\n    CenterAroundViCursor,\n    /// Search forward within the current line.\n    InlineSearchForward,\n    /// Search backward within the current line.\n    InlineSearchBackward,\n    /// Search forward within the current line, stopping just short of the character.\n    InlineSearchForwardShort,\n    /// Search backward within the current line, stopping just short of the character.\n    InlineSearchBackwardShort,\n    /// Jump to the next inline search match.\n    InlineSearchNext,\n    /// Jump to the previous inline search match.\n    InlineSearchPrevious,\n}\n\n/// Search mode specific actions.\n#[allow(clippy::enum_variant_names)]\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub enum SearchAction {\n    /// Move the focus to the next search match.\n    SearchFocusNext,\n    /// Move the focus to the previous search match.\n    SearchFocusPrevious,\n    /// Confirm the active search.\n    SearchConfirm,\n    /// Cancel the active search.\n    SearchCancel,\n    /// Reset the search regex.\n    SearchClear,\n    /// Delete the last word in the search regex.\n    SearchDeleteWord,\n    /// Go to the previous regex in the search history.\n    SearchHistoryPrevious,\n    /// Go to the next regex in the search history.\n    SearchHistoryNext,\n}\n\n/// Mouse binding specific actions.\n#[derive(ConfigDeserialize, Debug, Copy, Clone, PartialEq, Eq)]\npub enum MouseAction {\n    /// Expand the selection to the current mouse cursor position.\n    ExpandSelection,\n}\n\nmacro_rules! bindings {\n    (\n        $ty:ident;\n        $(\n            $key:tt$(::$button:ident)?\n            $(=>$location:expr)?\n            $(,$mods:expr)*\n            $(,+$mode:expr)*\n            $(,~$notmode:expr)*\n            ;$action:expr\n        );*\n        $(;)*\n    ) => {{\n        let mut v = Vec::new();\n\n        $(\n            let mut _mods = ModifiersState::empty();\n            $(_mods = $mods;)*\n            let mut _mode = BindingMode::empty();\n            $(_mode.insert($mode);)*\n            let mut _notmode = BindingMode::empty();\n            $(_notmode.insert($notmode);)*\n\n            v.push($ty {\n                trigger: trigger!($ty, $key$(::$button)?, $($location)?),\n                mods: _mods,\n                mode: _mode,\n                notmode: _notmode,\n                action: $action.into(),\n            });\n        )*\n\n        v\n    }};\n}\n\nmacro_rules! trigger {\n    (KeyBinding, $key:literal, $location:expr) => {{\n        BindingKey::Keycode { key: Key::Character($key.into()), location: $location }\n    }};\n    (KeyBinding, $key:literal,) => {{\n        BindingKey::Keycode { key: Key::Character($key.into()), location: KeyLocation::Any }\n    }};\n    (KeyBinding, $key:ident, $location:expr) => {{\n        BindingKey::Keycode { key: Key::Named(NamedKey::$key), location: $location }\n    }};\n    (KeyBinding, $key:ident,) => {{\n        BindingKey::Keycode { key: Key::Named(NamedKey::$key), location: KeyLocation::Any }\n    }};\n    (MouseBinding, $base:ident::$button:ident,) => {{\n        $base::$button\n    }};\n}\n\npub fn default_mouse_bindings() -> Vec<MouseBinding> {\n    bindings!(\n        MouseBinding;\n        MouseButton::Right;                            MouseAction::ExpandSelection;\n        MouseButton::Right,   ModifiersState::CONTROL; MouseAction::ExpandSelection;\n        MouseButton::Middle, ~BindingMode::VI;         Action::PasteSelection;\n    )\n}\n\n// NOTE: key sequences which are not present here, like F5-F20, PageUp/PageDown codes are\n// built on the fly in input/keyboard.rs.\npub fn default_key_bindings() -> Vec<KeyBinding> {\n    let mut bindings = bindings!(\n        KeyBinding;\n        Copy; Action::Copy;\n        Copy,  +BindingMode::VI; Action::ClearSelection;\n        Paste, ~BindingMode::VI; Action::Paste;\n        Paste, +BindingMode::VI, +BindingMode::SEARCH; Action::Paste;\n        \"l\",       ModifiersState::CONTROL; Action::ClearLogNotice;\n        \"l\",       ModifiersState::CONTROL; Action::ReceiveChar;\n        Home,      ModifiersState::SHIFT, ~BindingMode::ALT_SCREEN; Action::ScrollToTop;\n        End,       ModifiersState::SHIFT, ~BindingMode::ALT_SCREEN; Action::ScrollToBottom;\n        PageUp,    ModifiersState::SHIFT, ~BindingMode::ALT_SCREEN; Action::ScrollPageUp;\n        PageDown,  ModifiersState::SHIFT, ~BindingMode::ALT_SCREEN; Action::ScrollPageDown;\n        // App cursor mode.\n        Home,       +BindingMode::APP_CURSOR, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x1bOH\".into());\n        End,        +BindingMode::APP_CURSOR, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x1bOF\".into());\n        ArrowUp,    +BindingMode::APP_CURSOR, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x1bOA\".into());\n        ArrowDown,  +BindingMode::APP_CURSOR, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x1bOB\".into());\n        ArrowRight, +BindingMode::APP_CURSOR, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x1bOC\".into());\n        ArrowLeft,  +BindingMode::APP_CURSOR, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x1bOD\".into());\n        // Legacy keys handling which can't be automatically encoded.\n        F1,         ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC, ~BindingMode::DISAMBIGUATE_ESC_CODES; Action::Esc(\"\\x1bOP\".into());\n        F2,         ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC, ~BindingMode::DISAMBIGUATE_ESC_CODES; Action::Esc(\"\\x1bOQ\".into());\n        F3,         ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC, ~BindingMode::DISAMBIGUATE_ESC_CODES; Action::Esc(\"\\x1bOR\".into());\n        F4,         ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC, ~BindingMode::DISAMBIGUATE_ESC_CODES; Action::Esc(\"\\x1bOS\".into());\n        Tab,       ModifiersState::SHIFT,   ~BindingMode::VI,   ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC; Action::Esc(\"\\x1b[Z\".into());\n        Tab,       ModifiersState::SHIFT | ModifiersState::ALT, ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC; Action::Esc(\"\\x1b\\x1b[Z\".into());\n        Backspace, ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC; Action::Esc(\"\\x7f\".into());\n        Backspace, ModifiersState::ALT,     ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC; Action::Esc(\"\\x1b\\x7f\".into());\n        Backspace, ModifiersState::SHIFT,   ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC; Action::Esc(\"\\x7f\".into());\n        Enter => KeyLocation::Numpad, ~BindingMode::VI, ~BindingMode::SEARCH, ~BindingMode::REPORT_ALL_KEYS_AS_ESC, ~BindingMode::DISAMBIGUATE_ESC_CODES; Action::Esc(\"\\n\".into());\n        // Vi mode.\n        Space, ModifiersState::SHIFT | ModifiersState::CONTROL, ~BindingMode::SEARCH; Action::ToggleViMode;\n        Space, ModifiersState::SHIFT | ModifiersState::CONTROL, +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollToBottom;\n        Escape,                             +BindingMode::VI, ~BindingMode::SEARCH; Action::ClearSelection;\n        \"i\",                                +BindingMode::VI, ~BindingMode::SEARCH; Action::ToggleViMode;\n        \"i\",                                +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollToBottom;\n        \"c\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; Action::ToggleViMode;\n        \"y\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollLineUp;\n        \"e\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollLineDown;\n        \"g\",                                +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollToTop;\n        \"g\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollToBottom;\n        \"b\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollPageUp;\n        \"f\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollPageDown;\n        \"u\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollHalfPageUp;\n        \"d\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; Action::ScrollHalfPageDown;\n        \"y\",                                +BindingMode::VI, ~BindingMode::SEARCH; Action::Copy;\n        \"y\",                                +BindingMode::VI, ~BindingMode::SEARCH; Action::ClearSelection;\n        \"/\",                                +BindingMode::VI, ~BindingMode::SEARCH; Action::SearchForward;\n        \"?\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; Action::SearchBackward;\n        \"v\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViAction::ToggleNormalSelection;\n        \"v\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViAction::ToggleLineSelection;\n        \"v\",      ModifiersState::CONTROL,  +BindingMode::VI, ~BindingMode::SEARCH; ViAction::ToggleBlockSelection;\n        \"v\",      ModifiersState::ALT,      +BindingMode::VI, ~BindingMode::SEARCH; ViAction::ToggleSemanticSelection;\n        \"n\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViAction::SearchNext;\n        \"n\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViAction::SearchPrevious;\n        Enter,                              +BindingMode::VI, ~BindingMode::SEARCH; ViAction::Open;\n        \"z\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViAction::CenterAroundViCursor;\n        \"f\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViAction::InlineSearchForward;\n        \"f\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViAction::InlineSearchBackward;\n        \"t\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViAction::InlineSearchForwardShort;\n        \"t\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViAction::InlineSearchBackwardShort;\n        \";\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViAction::InlineSearchNext;\n        \",\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViAction::InlineSearchPrevious;\n        \"k\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Up;\n        \"j\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Down;\n        \"h\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Left;\n        \"l\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Right;\n        ArrowUp,                            +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Up;\n        ArrowDown,                          +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Down;\n        ArrowLeft,                          +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Left;\n        ArrowRight,                         +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Right;\n        \"0\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::First;\n        \"$\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Last;\n        Home,                               +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::First;\n        End,                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Last;\n        \"^\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::FirstOccupied;\n        \"h\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::High;\n        \"m\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Middle;\n        \"l\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Low;\n        \"b\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::SemanticLeft;\n        \"w\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::SemanticRight;\n        \"e\",                                +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::SemanticRightEnd;\n        \"b\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::WordLeft;\n        \"w\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::WordRight;\n        \"e\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::WordRightEnd;\n        \"%\",      ModifiersState::SHIFT,    +BindingMode::VI, ~BindingMode::SEARCH; ViMotion::Bracket;\n        Enter,                              +BindingMode::VI, +BindingMode::SEARCH; SearchAction::SearchConfirm;\n        // Plain search.\n        Escape,                             +BindingMode::SEARCH; SearchAction::SearchCancel;\n        \"c\",      ModifiersState::CONTROL,  +BindingMode::SEARCH; SearchAction::SearchCancel;\n        \"u\",      ModifiersState::CONTROL,  +BindingMode::SEARCH; SearchAction::SearchClear;\n        \"w\",      ModifiersState::CONTROL,  +BindingMode::SEARCH; SearchAction::SearchDeleteWord;\n        \"p\",      ModifiersState::CONTROL,  +BindingMode::SEARCH; SearchAction::SearchHistoryPrevious;\n        \"n\",      ModifiersState::CONTROL,  +BindingMode::SEARCH; SearchAction::SearchHistoryNext;\n        ArrowUp,                            +BindingMode::SEARCH; SearchAction::SearchHistoryPrevious;\n        ArrowDown,                          +BindingMode::SEARCH; SearchAction::SearchHistoryNext;\n        Enter,                              +BindingMode::SEARCH, ~BindingMode::VI; SearchAction::SearchFocusNext;\n        Enter, ModifiersState::SHIFT,       +BindingMode::SEARCH, ~BindingMode::VI; SearchAction::SearchFocusPrevious;\n    );\n\n    bindings.extend(platform_key_bindings());\n\n    bindings\n}\n\n#[cfg(not(any(target_os = \"macos\", test)))]\nfn common_keybindings() -> Vec<KeyBinding> {\n    bindings!(\n        KeyBinding;\n        \"v\",    ModifiersState::CONTROL | ModifiersState::SHIFT, ~BindingMode::VI;                       Action::Paste;\n        \"v\",    ModifiersState::CONTROL | ModifiersState::SHIFT, +BindingMode::VI, +BindingMode::SEARCH; Action::Paste;\n        \"f\",    ModifiersState::CONTROL | ModifiersState::SHIFT, ~BindingMode::SEARCH;                   Action::SearchForward;\n        \"b\",    ModifiersState::CONTROL | ModifiersState::SHIFT, ~BindingMode::SEARCH;                   Action::SearchBackward;\n        Insert, ModifiersState::SHIFT,                           ~BindingMode::VI;                       Action::PasteSelection;\n        \"c\",    ModifiersState::CONTROL | ModifiersState::SHIFT;                                         Action::Copy;\n        \"c\",    ModifiersState::CONTROL | ModifiersState::SHIFT, +BindingMode::VI, ~BindingMode::SEARCH; Action::ClearSelection;\n        \"0\",    ModifiersState::CONTROL;                                                                 Action::ResetFontSize;\n        \"=\",    ModifiersState::CONTROL;                                                                 Action::IncreaseFontSize;\n        \"+\",    ModifiersState::CONTROL;                                                                 Action::IncreaseFontSize;\n        \"-\",    ModifiersState::CONTROL;                                                                 Action::DecreaseFontSize;\n        \"+\" => KeyLocation::Numpad, ModifiersState::CONTROL;                                             Action::IncreaseFontSize;\n        \"-\" => KeyLocation::Numpad, ModifiersState::CONTROL;                                             Action::DecreaseFontSize;\n    )\n}\n\n#[cfg(not(any(target_os = \"macos\", target_os = \"windows\", test)))]\npub fn platform_key_bindings() -> Vec<KeyBinding> {\n    common_keybindings()\n}\n\n#[cfg(all(target_os = \"windows\", not(test)))]\npub fn platform_key_bindings() -> Vec<KeyBinding> {\n    let mut bindings = bindings!(\n        KeyBinding;\n        Enter, ModifiersState::ALT; Action::ToggleFullscreen;\n    );\n    bindings.extend(common_keybindings());\n    bindings\n}\n\n#[cfg(all(target_os = \"macos\", not(test)))]\npub fn platform_key_bindings() -> Vec<KeyBinding> {\n    bindings!(\n        KeyBinding;\n        Insert, ModifiersState::SHIFT, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x1b[2;2~\".into());\n        // Tabbing api.\n        \"t\",    ModifiersState::SUPER;                                         Action::CreateNewTab;\n        \"]\",    ModifiersState::SUPER | ModifiersState::SHIFT;                 Action::SelectNextTab;\n        \"[\",    ModifiersState::SUPER | ModifiersState::SHIFT;                 Action::SelectPreviousTab;\n        Tab,    ModifiersState::SUPER;                                         Action::SelectNextTab;\n        Tab,    ModifiersState::SUPER | ModifiersState::SHIFT;                 Action::SelectPreviousTab;\n        \"1\",    ModifiersState::SUPER;                                         Action::SelectTab1;\n        \"2\",    ModifiersState::SUPER;                                         Action::SelectTab2;\n        \"3\",    ModifiersState::SUPER;                                         Action::SelectTab3;\n        \"4\",    ModifiersState::SUPER;                                         Action::SelectTab4;\n        \"5\",    ModifiersState::SUPER;                                         Action::SelectTab5;\n        \"6\",    ModifiersState::SUPER;                                         Action::SelectTab6;\n        \"7\",    ModifiersState::SUPER;                                         Action::SelectTab7;\n        \"8\",    ModifiersState::SUPER;                                         Action::SelectTab8;\n        \"9\",    ModifiersState::SUPER;                                         Action::SelectLastTab;\n        \"0\",    ModifiersState::SUPER;                                         Action::ResetFontSize;\n        \"=\",    ModifiersState::SUPER;                                         Action::IncreaseFontSize;\n        \"+\",    ModifiersState::SUPER;                                         Action::IncreaseFontSize;\n        \"-\",    ModifiersState::SUPER;                                         Action::DecreaseFontSize;\n        \"k\",    ModifiersState::SUPER, ~BindingMode::VI, ~BindingMode::SEARCH; Action::Esc(\"\\x0c\".into());\n        \"k\",    ModifiersState::SUPER, ~BindingMode::VI, ~BindingMode::SEARCH; Action::ClearHistory;\n        \"v\",    ModifiersState::SUPER, ~BindingMode::VI;                       Action::Paste;\n        \"v\",    ModifiersState::SUPER, +BindingMode::VI, +BindingMode::SEARCH; Action::Paste;\n        \"n\",    ModifiersState::SUPER;                                         Action::CreateNewWindow;\n        \"f\",    ModifiersState::CONTROL | ModifiersState::SUPER;               Action::ToggleFullscreen;\n        \"c\",    ModifiersState::SUPER;                                         Action::Copy;\n        \"c\",    ModifiersState::SUPER, +BindingMode::VI, ~BindingMode::SEARCH; Action::ClearSelection;\n        \"h\",    ModifiersState::SUPER;                                         Action::Hide;\n        \"h\",    ModifiersState::SUPER   | ModifiersState::ALT;                 Action::HideOtherApplications;\n        \"m\",    ModifiersState::SUPER;                                         Action::Minimize;\n        \"q\",    ModifiersState::SUPER;                                         Action::Quit;\n        \"w\",    ModifiersState::SUPER;                                         Action::Quit;\n        \"f\",    ModifiersState::SUPER, ~BindingMode::SEARCH;                   Action::SearchForward;\n        \"b\",    ModifiersState::SUPER, ~BindingMode::SEARCH;                   Action::SearchBackward;\n        \"+\" => KeyLocation::Numpad, ModifiersState::SUPER;                     Action::IncreaseFontSize;\n        \"-\" => KeyLocation::Numpad, ModifiersState::SUPER;                     Action::DecreaseFontSize;\n    )\n}\n\n// Don't return any bindings for tests since they are commented-out by default.\n#[cfg(test)]\npub fn platform_key_bindings() -> Vec<KeyBinding> {\n    vec![]\n}\n\n#[derive(Clone, Debug, PartialEq, Eq)]\npub enum BindingKey {\n    Scancode(PhysicalKey),\n    Keycode { key: Key, location: KeyLocation },\n}\n\n/// Key location for matching bindings.\n#[derive(Debug, Clone, Copy, Eq)]\npub enum KeyLocation {\n    /// The key is in its standard position.\n    Standard,\n    /// The key is on the numeric pad.\n    Numpad,\n    /// The key could be anywhere on the keyboard.\n    Any,\n}\n\nimpl From<WinitKeyLocation> for KeyLocation {\n    fn from(value: WinitKeyLocation) -> Self {\n        match value {\n            WinitKeyLocation::Standard => KeyLocation::Standard,\n            WinitKeyLocation::Left => KeyLocation::Any,\n            WinitKeyLocation::Right => KeyLocation::Any,\n            WinitKeyLocation::Numpad => KeyLocation::Numpad,\n        }\n    }\n}\n\nimpl PartialEq for KeyLocation {\n    fn eq(&self, other: &Self) -> bool {\n        matches!(\n            (self, other),\n            (_, KeyLocation::Any)\n                | (KeyLocation::Any, _)\n                | (KeyLocation::Standard, KeyLocation::Standard)\n                | (KeyLocation::Numpad, KeyLocation::Numpad)\n        )\n    }\n}\n\nimpl<'a> Deserialize<'a> for BindingKey {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'a>,\n    {\n        let value = SerdeValue::deserialize(deserializer)?;\n        match u32::deserialize(value.clone()) {\n            Ok(scancode) => Ok(BindingKey::Scancode(PhysicalKey::from_scancode(scancode))),\n            Err(_) => {\n                let keycode = String::deserialize(value.clone()).map_err(D::Error::custom)?;\n                let (key, location) = if keycode.chars().count() == 1 {\n                    (Key::Character(keycode.to_lowercase().into()), KeyLocation::Any)\n                } else {\n                    // Translate legacy winit codes into their modern counterparts.\n                    match keycode.as_str() {\n                        \"Back\" => (Key::Named(NamedKey::Backspace), KeyLocation::Any),\n                        \"Up\" => (Key::Named(NamedKey::ArrowUp), KeyLocation::Any),\n                        \"Down\" => (Key::Named(NamedKey::ArrowDown), KeyLocation::Any),\n                        \"Left\" => (Key::Named(NamedKey::ArrowLeft), KeyLocation::Any),\n                        \"Right\" => (Key::Named(NamedKey::ArrowRight), KeyLocation::Any),\n                        \"At\" => (Key::Character(\"@\".into()), KeyLocation::Any),\n                        \"Colon\" => (Key::Character(\":\".into()), KeyLocation::Any),\n                        \"Period\" => (Key::Character(\".\".into()), KeyLocation::Any),\n                        \"LBracket\" => (Key::Character(\"[\".into()), KeyLocation::Any),\n                        \"RBracket\" => (Key::Character(\"]\".into()), KeyLocation::Any),\n                        \"Semicolon\" => (Key::Character(\";\".into()), KeyLocation::Any),\n                        \"Backslash\" => (Key::Character(\"\\\\\".into()), KeyLocation::Any),\n\n                        // The keys which has alternative on numeric pad.\n                        \"Enter\" => (Key::Named(NamedKey::Enter), KeyLocation::Standard),\n                        \"Return\" => (Key::Named(NamedKey::Enter), KeyLocation::Standard),\n                        \"Plus\" => (Key::Character(\"+\".into()), KeyLocation::Standard),\n                        \"Comma\" => (Key::Character(\",\".into()), KeyLocation::Standard),\n                        \"Slash\" => (Key::Character(\"/\".into()), KeyLocation::Standard),\n                        \"Equals\" => (Key::Character(\"=\".into()), KeyLocation::Standard),\n                        \"Minus\" => (Key::Character(\"-\".into()), KeyLocation::Standard),\n                        \"Asterisk\" => (Key::Character(\"*\".into()), KeyLocation::Standard),\n                        \"Key1\" => (Key::Character(\"1\".into()), KeyLocation::Standard),\n                        \"Key2\" => (Key::Character(\"2\".into()), KeyLocation::Standard),\n                        \"Key3\" => (Key::Character(\"3\".into()), KeyLocation::Standard),\n                        \"Key4\" => (Key::Character(\"4\".into()), KeyLocation::Standard),\n                        \"Key5\" => (Key::Character(\"5\".into()), KeyLocation::Standard),\n                        \"Key6\" => (Key::Character(\"6\".into()), KeyLocation::Standard),\n                        \"Key7\" => (Key::Character(\"7\".into()), KeyLocation::Standard),\n                        \"Key8\" => (Key::Character(\"8\".into()), KeyLocation::Standard),\n                        \"Key9\" => (Key::Character(\"9\".into()), KeyLocation::Standard),\n                        \"Key0\" => (Key::Character(\"0\".into()), KeyLocation::Standard),\n\n                        // Special case numpad.\n                        \"NumpadEnter\" => (Key::Named(NamedKey::Enter), KeyLocation::Numpad),\n                        \"NumpadAdd\" => (Key::Character(\"+\".into()), KeyLocation::Numpad),\n                        \"NumpadComma\" => (Key::Character(\",\".into()), KeyLocation::Numpad),\n                        \"NumpadDecimal\" => (Key::Character(\".\".into()), KeyLocation::Numpad),\n                        \"NumpadDivide\" => (Key::Character(\"/\".into()), KeyLocation::Numpad),\n                        \"NumpadEquals\" => (Key::Character(\"=\".into()), KeyLocation::Numpad),\n                        \"NumpadSubtract\" => (Key::Character(\"-\".into()), KeyLocation::Numpad),\n                        \"NumpadMultiply\" => (Key::Character(\"*\".into()), KeyLocation::Numpad),\n                        \"Numpad1\" => (Key::Character(\"1\".into()), KeyLocation::Numpad),\n                        \"Numpad2\" => (Key::Character(\"2\".into()), KeyLocation::Numpad),\n                        \"Numpad3\" => (Key::Character(\"3\".into()), KeyLocation::Numpad),\n                        \"Numpad4\" => (Key::Character(\"4\".into()), KeyLocation::Numpad),\n                        \"Numpad5\" => (Key::Character(\"5\".into()), KeyLocation::Numpad),\n                        \"Numpad6\" => (Key::Character(\"6\".into()), KeyLocation::Numpad),\n                        \"Numpad7\" => (Key::Character(\"7\".into()), KeyLocation::Numpad),\n                        \"Numpad8\" => (Key::Character(\"8\".into()), KeyLocation::Numpad),\n                        \"Numpad9\" => (Key::Character(\"9\".into()), KeyLocation::Numpad),\n                        \"Numpad0\" => (Key::Character(\"0\".into()), KeyLocation::Numpad),\n                        _ if keycode.starts_with(\"Dead\") => {\n                            (Key::deserialize(value).map_err(D::Error::custom)?, KeyLocation::Any)\n                        },\n                        _ => (\n                            Key::Named(NamedKey::deserialize(value).map_err(D::Error::custom)?),\n                            KeyLocation::Any,\n                        ),\n                    }\n                };\n\n                Ok(BindingKey::Keycode { key, location })\n            },\n        }\n    }\n}\n\n#[derive(Copy, Clone, Debug, PartialEq, Eq)]\npub struct ModeWrapper {\n    pub mode: BindingMode,\n    pub not_mode: BindingMode,\n}\n\nbitflags! {\n    /// Modes available for key bindings.\n    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]\n    pub struct BindingMode: u8 {\n        const APP_CURSOR             = 0b0000_0001;\n        const APP_KEYPAD             = 0b0000_0010;\n        const ALT_SCREEN             = 0b0000_0100;\n        const VI                     = 0b0000_1000;\n        const SEARCH                 = 0b0001_0000;\n        const DISAMBIGUATE_ESC_CODES = 0b0010_0000;\n        const REPORT_ALL_KEYS_AS_ESC = 0b0100_0000;\n    }\n}\n\nimpl BindingMode {\n    pub fn new(mode: &TermMode, search: bool) -> BindingMode {\n        let mut binding_mode = BindingMode::empty();\n        binding_mode.set(BindingMode::APP_CURSOR, mode.contains(TermMode::APP_CURSOR));\n        binding_mode.set(BindingMode::APP_KEYPAD, mode.contains(TermMode::APP_KEYPAD));\n        binding_mode.set(BindingMode::ALT_SCREEN, mode.contains(TermMode::ALT_SCREEN));\n        binding_mode.set(BindingMode::VI, mode.contains(TermMode::VI));\n        binding_mode.set(BindingMode::SEARCH, search);\n        binding_mode.set(\n            BindingMode::DISAMBIGUATE_ESC_CODES,\n            mode.contains(TermMode::DISAMBIGUATE_ESC_CODES),\n        );\n        binding_mode.set(\n            BindingMode::REPORT_ALL_KEYS_AS_ESC,\n            mode.contains(TermMode::REPORT_ALL_KEYS_AS_ESC),\n        );\n        binding_mode\n    }\n}\n\nimpl Default for ModeWrapper {\n    fn default() -> Self {\n        Self { mode: BindingMode::empty(), not_mode: BindingMode::empty() }\n    }\n}\n\nimpl<'a> Deserialize<'a> for ModeWrapper {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'a>,\n    {\n        struct ModeVisitor;\n\n        impl<'a> Visitor<'a> for ModeVisitor {\n            type Value = ModeWrapper;\n\n            fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                f.write_str(\n                    \"a combination of AppCursor | AppKeypad | Alt | Vi, possibly with negation (~)\",\n                )\n            }\n\n            fn visit_str<E>(self, value: &str) -> Result<ModeWrapper, E>\n            where\n                E: de::Error,\n            {\n                let mut res =\n                    ModeWrapper { mode: BindingMode::empty(), not_mode: BindingMode::empty() };\n\n                for modifier in value.split('|') {\n                    match modifier.trim().to_lowercase().as_str() {\n                        \"appcursor\" => res.mode |= BindingMode::APP_CURSOR,\n                        \"~appcursor\" => res.not_mode |= BindingMode::APP_CURSOR,\n                        \"appkeypad\" => res.mode |= BindingMode::APP_KEYPAD,\n                        \"~appkeypad\" => res.not_mode |= BindingMode::APP_KEYPAD,\n                        \"alt\" => res.mode |= BindingMode::ALT_SCREEN,\n                        \"~alt\" => res.not_mode |= BindingMode::ALT_SCREEN,\n                        \"vi\" => res.mode |= BindingMode::VI,\n                        \"~vi\" => res.not_mode |= BindingMode::VI,\n                        \"search\" => res.mode |= BindingMode::SEARCH,\n                        \"~search\" => res.not_mode |= BindingMode::SEARCH,\n                        _ => return Err(E::invalid_value(Unexpected::Str(modifier), &self)),\n                    }\n                }\n\n                Ok(res)\n            }\n        }\n        deserializer.deserialize_str(ModeVisitor)\n    }\n}\n\nstruct MouseButtonWrapper(MouseButton);\n\nimpl MouseButtonWrapper {\n    fn into_inner(self) -> MouseButton {\n        self.0\n    }\n}\n\nimpl<'a> Deserialize<'a> for MouseButtonWrapper {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'a>,\n    {\n        struct MouseButtonVisitor;\n\n        impl<'a> Visitor<'a> for MouseButtonVisitor {\n            type Value = MouseButtonWrapper;\n\n            fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {\n                f.write_str(\"Left, Right, Middle, Back, Forward, or a number from 0 to 65536\")\n            }\n\n            fn visit_i64<E>(self, value: i64) -> Result<MouseButtonWrapper, E>\n            where\n                E: de::Error,\n            {\n                match value {\n                    0..=65536 => Ok(MouseButtonWrapper(MouseButton::Other(value as u16))),\n                    _ => Err(E::invalid_value(Unexpected::Signed(value), &self)),\n                }\n            }\n\n            fn visit_u64<E>(self, value: u64) -> Result<MouseButtonWrapper, E>\n            where\n                E: de::Error,\n            {\n                match value {\n                    0..=65536 => Ok(MouseButtonWrapper(MouseButton::Other(value as u16))),\n...\n// Path: alacritty/src/config/mouse.rs\nuse serde::{Deserialize, Deserializer};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\n\nuse crate::config::bindings::{self, MouseBinding};\nuse crate::config::ui_config;\n\n#[derive(ConfigDeserialize, Default, Clone, Debug, PartialEq, Eq)]\npub struct Mouse {\n    pub hide_when_typing: bool,\n    pub bindings: MouseBindings,\n}\n\n#[derive(SerdeReplace, Clone, Debug, PartialEq, Eq)]\npub struct MouseBindings(pub Vec<MouseBinding>);\n\nimpl Default for MouseBindings {\n    fn default() -> Self {\n        Self(bindings::default_mouse_bindings())\n    }\n}\n\nimpl<'de> Deserialize<'de> for MouseBindings {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        Ok(Self(ui_config::deserialize_bindings(deserializer, Self::default().0)?))\n    }\n}\n\n// Path: alacritty/src/config/terminal.rs\nuse serde::{de, Deserialize, Deserializer};\nuse toml::Value;\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\nuse alacritty_terminal::term::Osc52;\n\nuse crate::config::ui_config::StringVisitor;\n\n#[derive(ConfigDeserialize, Default, Copy, Clone, Debug, PartialEq)]\npub struct Terminal {\n    /// OSC52 support mode.\n    pub osc52: SerdeOsc52,\n}\n\n#[derive(SerdeReplace, Default, Copy, Clone, Debug, PartialEq)]\npub struct SerdeOsc52(pub Osc52);\n\nimpl<'de> Deserialize<'de> for SerdeOsc52 {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let value = deserializer.deserialize_str(StringVisitor)?;\n        Osc52::deserialize(Value::String(value)).map(SerdeOsc52).map_err(de::Error::custom)\n    }\n}\n\n// Path: alacritty/src/config/ui_config.rs\nuse std::cell::RefCell;\nuse std::collections::HashMap;\nuse std::error::Error;\nuse std::fmt::{self, Formatter};\nuse std::path::PathBuf;\nuse std::rc::Rc;\n\nuse alacritty_config::SerdeReplace;\nuse alacritty_terminal::term::Config as TermConfig;\nuse alacritty_terminal::tty::{Options as PtyOptions, Shell};\nuse log::{error, warn};\nuse serde::de::{Error as SerdeError, MapAccess, Visitor};\nuse serde::{Deserialize, Deserializer};\nuse unicode_width::UnicodeWidthChar;\nuse winit::keyboard::{Key, ModifiersState};\n\nuse alacritty_config_derive::{ConfigDeserialize, SerdeReplace};\nuse alacritty_terminal::term::search::RegexSearch;\n\nuse crate::config::bell::BellConfig;\nuse crate::config::bindings::{\n    self, Action, Binding, BindingKey, KeyBinding, KeyLocation, ModeWrapper, ModsWrapper,\n    MouseBinding,\n};\nuse crate::config::color::Colors;\nuse crate::config::cursor::Cursor;\nuse crate::config::debug::Debug;\nuse crate::config::font::Font;\nuse crate::config::mouse::{Mouse, MouseBindings};\nuse crate::config::scrolling::Scrolling;\nuse crate::config::selection::Selection;\nuse crate::config::terminal::Terminal;\nuse crate::config::window::WindowConfig;\nuse crate::config::LOG_TARGET_CONFIG;\n\n/// Regex used for the default URL hint.\n#[rustfmt::skip]\nconst URL_REGEX: &str = \"(ipfs:|ipns:|magnet:|mailto:|gemini://|gopher://|https://|http://|news:|file:|git://|ssh:|ftp://)\\\n                         [^\\u{0000}-\\u{001F}\\u{007F}-\\u{009F}<>\\\"\\\\s{-}\\\\^\u27e8\u27e9`]+\";\n\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq)]\npub struct UiConfig {\n    /// Extra environment variables.\n    pub env: HashMap<String, String>,\n\n    /// How much scrolling history to keep.\n    pub scrolling: Scrolling,\n\n    /// Cursor configuration.\n    pub cursor: Cursor,\n\n    /// Selection configuration.\n    pub selection: Selection,\n\n    /// Font configuration.\n    pub font: Font,\n\n    /// Window configuration.\n    pub window: WindowConfig,\n\n    /// Mouse configuration.\n    pub mouse: Mouse,\n\n    /// Debug options.\n    pub debug: Debug,\n\n    /// Send escape sequences using the alt key.\n    #[config(removed = \"It's now always set to 'true'. If you're on macOS use \\\n                        'window.option_as_alt' to alter behavior of Option\")]\n    pub alt_send_esc: Option<bool>,\n\n    /// Live config reload.\n    pub live_config_reload: bool,\n\n    /// Bell configuration.\n    pub bell: BellConfig,\n\n    /// RGB values for colors.\n    pub colors: Colors,\n\n    /// Path where config was loaded from.\n    #[config(skip)]\n    pub config_paths: Vec<PathBuf>,\n\n    /// Regex hints for interacting with terminal content.\n    pub hints: Hints,\n\n    /// Offer IPC through a unix socket.\n    #[cfg(unix)]\n    pub ipc_socket: bool,\n\n    /// Config for the alacritty_terminal itself.\n    pub terminal: Terminal,\n\n    /// Path to a shell program to run on startup.\n    pub shell: Option<Program>,\n\n    /// Shell startup directory.\n    pub working_directory: Option<PathBuf>,\n\n    /// Keyboard configuration.\n    keyboard: Keyboard,\n\n    /// Should draw bold text with brighter colors instead of bold font.\n    #[config(deprecated = \"use colors.draw_bold_text_with_bright_colors instead\")]\n    draw_bold_text_with_bright_colors: bool,\n\n    /// Keybindings.\n    #[config(deprecated = \"use keyboard.bindings instead\")]\n    key_bindings: Option<KeyBindings>,\n\n    /// Bindings for the mouse.\n    #[config(deprecated = \"use mouse.bindings instead\")]\n    mouse_bindings: Option<MouseBindings>,\n\n    /// Configuration file imports.\n    ///\n    /// This is never read since the field is directly accessed through the config's\n    /// [`toml::Value`], but still present to prevent unused field warnings.\n    import: Vec<String>,\n}\n\nimpl Default for UiConfig {\n    fn default() -> Self {\n        Self {\n            live_config_reload: true,\n            #[cfg(unix)]\n            ipc_socket: true,\n            draw_bold_text_with_bright_colors: Default::default(),\n            working_directory: Default::default(),\n            mouse_bindings: Default::default(),\n            config_paths: Default::default(),\n            key_bindings: Default::default(),\n            alt_send_esc: Default::default(),\n            scrolling: Default::default(),\n            selection: Default::default(),\n            keyboard: Default::default(),\n            terminal: Default::default(),\n            import: Default::default(),\n            cursor: Default::default(),\n            window: Default::default(),\n            colors: Default::default(),\n            shell: Default::default(),\n            mouse: Default::default(),\n            debug: Default::default(),\n            hints: Default::default(),\n            font: Default::default(),\n            bell: Default::default(),\n            env: Default::default(),\n        }\n    }\n}\n\nimpl UiConfig {\n    /// Derive [`TermConfig`] from the config.\n    pub \nfn term_options(&self) -> TermConfig {\n        TermConfig {\n            semantic_escape_chars: self.selection.semantic_escape_chars.clone(),\n            scrolling_history: self.scrolling.history() as usize,\n            vi_mode_cursor_style: self.cursor.vi_mode_style(),\n            default_cursor_style: self.cursor.style(),\n            osc52: self.terminal.osc52.0,\n            kitty_keyboard: true,\n        }\n    }\n\n    /// Derive [`PtyOptions`] from the config.\n    pub fn pty_config(&self) -> PtyOptions {\n        let shell = self.shell.clone().map(Into::into);\n        PtyOptions {\n            shell,\n            working_directory: self.working_directory.clone(),\n            hold: false,\n            env: HashMap::new(),\n        }\n    }\n\n    /// Generate key bindings for all keyboard hints.\n    pub fn generate_hint_bindings(&mut self) {\n        // Check which key bindings is most likely to be the user's configuration.\n        //\n        // Both will be non-empty due to the presence of the default keybindings.\n        let key_bindings = if let Some(key_bindings) = self.key_bindings.as_mut() {\n            &mut key_bindings.0\n        } else {\n            &mut self.keyboard.bindings.0\n        };\n\n        for hint in &self.hints.enabled {\n            let binding = match &hint.binding {\n                Some(binding) => binding,\n                None => continue,\n            };\n\n            let binding = KeyBinding {\n                trigger: binding.key.clone(),\n                mods: binding.mods.0,\n                mode: binding.mode.mode,\n                notmode: binding.mode.not_mode,\n                action: Action::Hint(hint.clone()),\n            };\n\n            key_bindings.push(binding);\n        }\n    }\n\n    #[inline]\n    pub fn window_opacity(&self) -> f32 {\n        self.window.opacity.as_f32()\n    }\n\n    #[inline]\n    pub fn key_bindings(&self) -> &[KeyBinding] {\n        if let Some(key_bindings) = self.key_bindings.as_ref() {\n            &key_bindings.0\n        } else {\n            &self.keyboard.bindings.0\n        }\n    }\n\n    #[inline]\n    pub fn mouse_bindings(&self) -> &[MouseBinding] {\n        if let Some(mouse_bindings) = self.mouse_bindings.as_ref() {\n            &mouse_bindings.0\n        } else {\n            &self.mouse.bindings.0\n        }\n    }\n\n    #[inline]\n    pub fn draw_bold_text_with_bright_colors(&self) -> bool {\n        self.colors.draw_bold_text_with_bright_colors || self.draw_bold_text_with_bright_colors\n    }\n}\n\n/// Keyboard configuration.\n#[derive(ConfigDeserialize, Default, Clone, Debug, PartialEq)]\nstruct Keyboard {\n    /// Keybindings.\n    bindings: KeyBindings,\n}\n\n#[derive(SerdeReplace, Clone, Debug, PartialEq, Eq)]\nstruct KeyBindings(Vec<KeyBinding>);\n\nimpl Default for KeyBindings {\n    fn default() -> Self {\n        Self(bindings::default_key_bindings())\n    }\n}\n\nimpl<'de> Deserialize<'de> for KeyBindings {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        Ok(Self(deserialize_bindings(deserializer, Self::default().0)?))\n    }\n}\n\npub fn deserialize_bindings<'a, D, T>(\n    deserializer: D,\n    mut default: Vec<Binding<T>>,\n) -> Result<Vec<Binding<T>>, D::Error>\nwhere\n    D: Deserializer<'a>,\n    T: Clone + Eq,\n    Binding<T>: Deserialize<'a>,\n{\n    let values = Vec::<toml::Value>::deserialize(deserializer)?;\n\n    // Skip all invalid values.\n    let mut bindings = Vec::with_capacity(values.len());\n    for value in values {\n        match Binding::<T>::deserialize(value) {\n            Ok(binding) => bindings.push(binding),\n            Err(err) => {\n                error!(target: LOG_TARGET_CONFIG, \"Config error: {}; ignoring binding\", err);\n            },\n        }\n    }\n\n    // Remove matching default bindings.\n    for binding in bindings.iter() {\n        default.retain(|b| !b.triggers_match(binding));\n    }\n\n    bindings.extend(default);\n\n    Ok(bindings)\n}\n\n/// A delta for a point in a 2 dimensional plane.\n#[derive(ConfigDeserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]\npub struct Delta<T: Default> {\n    /// Horizontal change.\n    pub x: T,\n    /// Vertical change.\n    pub y: T,\n}\n\n/// Regex terminal hints.\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub struct Hints {\n    /// Characters for the hint labels.\n    alphabet: HintsAlphabet,\n\n    /// All configured terminal hints.\n    pub enabled: Vec<Hint>,\n}\n\nimpl Default for Hints {\n    fn default() -> Self {\n        // Add URL hint by default when no other hint is present.\n        let pattern = LazyRegexVariant::Pattern(String::from(URL_REGEX));\n        let regex = LazyRegex(Rc::new(RefCell::new(pattern)));\n        let content = HintContent::new(Some(regex), true);\n\n        #[cfg(not(any(target_os = \"macos\", windows)))]\n        let action = HintAction::Command(Program::Just(String::from(\"xdg-open\")));\n        #[cfg(target_os = \"macos\")]\n        let action = HintAction::Command(Program::Just(String::from(\"open\")));\n        #[cfg(windows)]\n        let action = HintAction::Command(Program::WithArgs {\n            program: String::from(\"cmd\"),\n            args: vec![\"/c\".to_string(), \"start\".to_string(), \"\".to_string()],\n        });\n\n        Self {\n            enabled: vec![Hint {\n                content,\n                action,\n                persist: false,\n                post_processing: true,\n                mouse: Some(HintMouse { enabled: true, mods: Default::default() }),\n                binding: Some(HintBinding {\n                    key: BindingKey::Keycode {\n                        key: Key::Character(\"u\".into()),\n                        location: KeyLocation::Standard,\n                    },\n                    mods: ModsWrapper(ModifiersState::SHIFT | ModifiersState::CONTROL),\n                    mode: Default::default(),\n                }),\n            }],\n            alphabet: Default::default(),\n        }\n    }\n}\n\nimpl Hints {\n    /// Characters for the hint labels.\n    pub fn alphabet(&self) -> &str {\n        &self.alphabet.0\n    }\n}\n\n#[derive(SerdeReplace, Clone, Debug, PartialEq, Eq)]\nstruct HintsAlphabet(String);\n\nimpl Default for HintsAlphabet {\n    fn default() -> Self {\n        Self(String::from(\"jfkdls;ahgurieowpq\"))\n    }\n}\n\nimpl<'de> Deserialize<'de> for HintsAlphabet {\n    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>\n    where\n        D: Deserializer<'de>,\n    {\n        let value = String::deserialize(deserializer)?;\n\n        let mut character_count = 0;\n        for character in value.chars() {\n            if character.width() != Some(1) {\n                return Err(D::Error::custom(\"characters must be of width 1\"));\n            }\n            character_count += 1;\n        }\n\n        if character_count < 2 {\n            return Err(D::Error::custom(\"must include at last 2 characters\"));\n        }\n\n        Ok(Self(value))\n    }\n}\n\n/// Built-in actions for hint mode.\n#[derive(ConfigDeserialize, Clone, Debug, PartialEq, Eq)]\npub enum HintInternalAction {\n    /// Copy the text to the clipboard.\n    Copy,\n    /// Write the text to the PTY/search.\n    Paste,\n    /// Select the text matching the hint.\n    Select,\n    /// Move the vi mode cursor to the beginning of the hint.\n    MoveViModeCursor,\n}\n\n/// Actions for hint bindings.\n#[derive(Deserialize, Clone, Debug, PartialEq, Eq)]\npub enum HintAction {\n    /// Built-in hint action.\n    #[serde(rename = \"action\")]\n    Action(HintInternalAction),\n\n    /// Command the text will be piped to.\n    #[serde(rename = \"command\")]\n    Command(Program),\n}\n\n/// Hint configuration.\n#[derive(Deserialize, Clone, Debug, PartialEq, Eq)]\npub struct Hint {\n    /// Regex for finding matches.\n    #[serde(flatten)]\n    pub content: HintContent,\n\n    /// Action executed when this hint is triggered.\n    #[serde(flatten)]\n    pub action: HintAction,\n\n    /// Hint text post processing.\n    #[serde(default)]\n    pub post_processing: bool,\n\n    /// Persist hints after selection.\n    #[serde(default)]\n    pub persist: bool,\n\n    /// Hint mouse highlighting.\n    pub mouse: Option<HintMouse>,\n\n...\n", "needle_token_start": 13716, "needle_token_end": 13843, "code_context_ntokens": 16205, "output": ["```rust\nfn term_options(&self) -> TermConfig {\n    TermConfig {\n        semantic_escape_chars: self.selection.semantic_escape_chars.clone(),\n        scrolling_history: self.scrolling.history() as usize,\n        vi_mode_cursor_style: self.cursor.vi_mode_style(),\n        default_cursor_style: self.cursor.style(),\n        osc52: self.terminal.osc52.0,\n        kitty_keyboard: true,\n    }\n}\n```"]}
{"repo": "alacritty/alacritty", "name": "merge_rects", "language": "rust", "path": "alacritty/src/display/damage.rs", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To create a single rectangle that encompasses two overlapping or adjacent rectangles, effectively combining their spatial coverage.\n2. **Input**: Two rectangles, each defined by its x and y coordinates, width, and height.\n3. **Output**: A new rectangle that is the smallest possible rectangle capable of containing both input rectangles.\n4. **Procedure**: The function calculates the minimum x-coordinate and the maximum x-coordinate plus width from both rectangles to determine the new rectangle's x-coordinate and width. Similarly, it calculates the minimum y-coordinate and the maximum y-coordinate plus height to set the new rectangle's y-coordinate and height.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: alacritty/src/input/mod.rs\n//! Handle input from winit.\n//!\n//! Certain key combinations should send some escape sequence back to the PTY.\n//! In order to figure that out, state about which modifier keys are pressed\n//! needs to be tracked. Additionally, we need a bit of a state machine to\n//! determine what to do when a non-modifier key is pressed.\n\nuse std::borrow::Cow;\nuse std::cmp::{max, min, Ordering};\nuse std::collections::HashSet;\nuse std::ffi::OsStr;\nuse std::fmt::Debug;\nuse std::marker::PhantomData;\nuse std::mem;\nuse std::time::{Duration, Instant};\n\nuse log::debug;\nuse winit::dpi::PhysicalPosition;\nuse winit::event::{\n    ElementState, Modifiers, MouseButton, MouseScrollDelta, Touch as TouchEvent, TouchPhase,\n};\n#[cfg(target_os = \"macos\")]\nuse winit::event_loop::EventLoopWindowTarget;\nuse winit::keyboard::ModifiersState;\n#[cfg(target_os = \"macos\")]\nuse winit::platform::macos::EventLoopWindowTargetExtMacOS;\nuse winit::window::CursorIcon;\n\nuse alacritty_terminal::event::EventListener;\nuse alacritty_terminal::grid::{Dimensions, Scroll};\nuse alacritty_terminal::index::{Boundary, Column, Direction, Point, Side};\nuse alacritty_terminal::selection::SelectionType;\nuse alacritty_terminal::term::search::Match;\nuse alacritty_terminal::term::{ClipboardType, Term, TermMode};\nuse alacritty_terminal::vi_mode::ViMotion;\nuse alacritty_terminal::vte::ansi::{ClearMode, Handler};\n\nuse crate::clipboard::Clipboard;\nuse crate::config::{Action, BindingMode, MouseAction, SearchAction, UiConfig, ViAction};\nuse crate::display::hint::HintMatch;\nuse crate::display::window::Window;\nuse crate::display::{Display, SizeInfo};\nuse crate::event::{\n    ClickState, Event, EventType, InlineSearchState, Mouse, TouchPurpose, TouchZoom,\n};\nuse crate::message_bar::{self, Message};\nuse crate::scheduler::{Scheduler, TimerId, Topic};\n\npub mod keyboard;\n\n/// Font size change interval in px.\npub const FONT_SIZE_STEP: f32 = 1.;\n\n/// Interval for mouse scrolling during selection outside of the boundaries.\nconst SELECTION_SCROLLING_INTERVAL: Duration = Duration::from_millis(15);\n\n/// Minimum number of pixels at the bottom/top where selection scrolling is performed.\nconst MIN_SELECTION_SCROLLING_HEIGHT: f64 = 5.;\n\n/// Number of pixels for increasing the selection scrolling speed factor by one.\nconst SELECTION_SCROLLING_STEP: f64 = 20.;\n\n/// Distance before a touch input is considered a drag.\nconst MAX_TAP_DISTANCE: f64 = 20.;\n\n/// Threshold used for double_click/triple_click.\nconst CLICK_THRESHOLD: Duration = Duration::from_millis(400);\n\n/// Processes input from winit.\n///\n/// An escape sequence may be emitted in case specific keys or key combinations\n/// are activated.\npub struct Processor<T: EventListener, A: ActionContext<T>> {\n    pub ctx: A,\n    _phantom: PhantomData<T>,\n}\n\npub trait ActionContext<T: EventListener> {\n    fn write_to_pty<B: Into<Cow<'static, [u8]>>>(&self, _data: B) {}\n    fn mark_dirty(&mut self) {}\n    fn size_info(&self) -> SizeInfo;\n    fn copy_selection(&mut self, _ty: ClipboardType) {}\n    fn start_selection(&mut self, _ty: SelectionType, _point: Point, _side: Side) {}\n    fn toggle_selection(&mut self, _ty: SelectionType, _point: Point, _side: Side) {}\n    fn update_selection(&mut self, _point: Point, _side: Side) {}\n    fn clear_selection(&mut self) {}\n    fn selection_is_empty(&self) -> bool;\n    fn mouse_mut(&mut self) -> &mut Mouse;\n    fn mouse(&self) -> &Mouse;\n    fn touch_purpose(&mut self) -> &mut TouchPurpose;\n    fn modifiers(&mut self) -> &mut Modifiers;\n    fn scroll(&mut self, _scroll: Scroll) {}\n    fn window(&mut self) -> &mut Window;\n    fn display(&mut self) -> &mut Display;\n    fn terminal(&self) -> &Term<T>;\n    fn terminal_mut(&mut self) -> &mut Term<T>;\n    fn spawn_new_instance(&mut self) {}\n    #[cfg(target_os = \"macos\")]\n    fn create_new_window(&mut self, _tabbing_id: Option<String>) {}\n    #[cfg(not(target_os = \"macos\"))]\n    fn create_new_window(&mut self) {}\n    fn change_font_size(&mut self, _delta: f32) {}\n    fn reset_font_size(&mut self) {}\n    fn pop_message(&mut self) {}\n    fn message(&self) -> Option<&Message>;\n    fn config(&self) -> &UiConfig;\n    #[cfg(target_os = \"macos\")]\n    fn event_loop(&self) -> &EventLoopWindowTarget<Event>;\n    fn mouse_mode(&self) -> bool;\n    fn clipboard_mut(&mut self) -> &mut Clipboard;\n    fn scheduler_mut(&mut self) -> &mut Scheduler;\n    fn start_search(&mut self, _direction: Direction) {}\n    fn confirm_search(&mut self) {}\n    fn cancel_search(&mut self) {}\n    fn search_input(&mut self, _c: char) {}\n    fn search_pop_word(&mut self) {}\n    fn search_history_previous(&mut self) {}\n    fn search_history_next(&mut self) {}\n    fn search_next(&mut self, origin: Point, direction: Direction, side: Side) -> Option<Match>;\n    fn advance_search_origin(&mut self, _direction: Direction) {}\n    fn search_direction(&self) -> Direction;\n    fn search_active(&self) -> bool;\n    fn on_typing_start(&mut self) {}\n    fn toggle_vi_mode(&mut self) {}\n    fn inline_search_state(&mut self) -> &mut InlineSearchState;\n    fn start_inline_search(&mut self, _direction: Direction, _stop_short: bool) {}\n    fn inline_search_next(&mut self) {}\n    fn inline_search_previous(&mut self) {}\n    fn hint_input(&mut self, _character: char) {}\n    fn trigger_hint(&mut self, _hint: &HintMatch) {}\n    fn expand_selection(&mut self) {}\n    fn on_terminal_input_start(&mut self) {}\n    fn paste(&mut self, _text: &str, _bracketed: bool) {}\n    fn spawn_daemon<I, S>(&self, _program: &str, _args: I)\n    where\n        I: IntoIterator<Item = S> + Debug + Copy,\n        S: AsRef<OsStr>,\n    {\n    }\n}\n\nimpl Action {\n    fn toggle_selection<T, A>(ctx: &mut A, ty: SelectionType)\n    where\n        A: ActionContext<T>,\n        T: EventListener,\n    {\n        ctx.toggle_selection(ty, ctx.terminal().vi_mode_cursor.point, Side::Left);\n\n        // Make sure initial selection is not empty.\n        if let Some(selection) = &mut ctx.terminal_mut().selection {\n            selection.include_all();\n        }\n    }\n}\n\ntrait Execute<T: EventListener> {\n    fn execute<A: ActionContext<T>>(&self, ctx: &mut A);\n}\n\nimpl<T: EventListener> Execute<T> for Action {\n    #[inline]\n    fn execute<A: ActionContext<T>>(&self, ctx: &mut A) {\n        match self {\n            Action::Esc(s) => ctx.paste(s, false),\n            Action::Command(program) => ctx.spawn_daemon(program.program(), program.args()),\n            Action::Hint(hint) => {\n                ctx.display().hint_state.start(hint.clone());\n                ctx.mark_dirty();\n            },\n            Action::ToggleViMode => {\n                ctx.on_typing_start();\n                ctx.toggle_vi_mode()\n            },\n            action @ (Action::ViMotion(_) | Action::Vi(_))\n                if !ctx.terminal().mode().contains(TermMode::VI) =>\n            {\n                debug!(\"Ignoring {action:?}: Vi mode inactive\");\n            },\n            Action::ViMotion(motion) => {\n                ctx.on_typing_start();\n                ctx.terminal_mut().vi_motion(*motion);\n                ctx.mark_dirty();\n            },\n            Action::Vi(ViAction::ToggleNormalSelection) => {\n                Self::toggle_selection(ctx, SelectionType::Simple);\n            },\n            Action::Vi(ViAction::ToggleLineSelection) => {\n                Self::toggle_selection(ctx, SelectionType::Lines);\n            },\n            Action::Vi(ViAction::ToggleBlockSelection) => {\n                Self::toggle_selection(ctx, SelectionType::Block);\n            },\n            Action::Vi(ViAction::ToggleSemanticSelection) => {\n                Self::toggle_selection(ctx, SelectionType::Semantic);\n            },\n            Action::Vi(ViAction::Open) => {\n                let hint = ctx.display().vi_highlighted_hint.take();\n                if let Some(hint) = &hint {\n                    ctx.mouse_mut().block_hint_launcher = false;\n                    ctx.trigger_hint(hint);\n                }\n                ctx.display().vi_highlighted_hint = hint;\n            },\n            Action::Vi(ViAction::SearchNext) => {\n                ctx.on_typing_start();\n\n                let terminal = ctx.terminal();\n                let direction = ctx.search_direction();\n                let vi_point = terminal.vi_mode_cursor.point;\n                let origin = match direction {\n                    Direction::Right => vi_point.add(terminal, Boundary::None, 1),\n                    Direction::Left => vi_point.sub(terminal, Boundary::None, 1),\n                };\n\n                if let Some(regex_match) = ctx.search_next(origin, direction, Side::Left) {\n                    ctx.terminal_mut().vi_goto_point(*regex_match.start());\n                    ctx.mark_dirty();\n                }\n            },\n            Action::Vi(ViAction::SearchPrevious) => {\n                ctx.on_typing_start();\n\n                let terminal = ctx.terminal();\n                let direction = ctx.search_direction().opposite();\n                let vi_point = terminal.vi_mode_cursor.point;\n                let origin = match direction {\n                    Direction::Right => vi_point.add(terminal, Boundary::None, 1),\n                    Direction::Left => vi_point.sub(terminal, Boundary::None, 1),\n                };\n\n                if let Some(regex_match) = ctx.search_next(origin, direction, Side::Left) {\n                    ctx.terminal_mut().vi_goto_point(*regex_match.start());\n                    ctx.mark_dirty();\n                }\n            },\n            Action::Vi(ViAction::SearchStart) => {\n                let terminal = ctx.terminal();\n                let origin = terminal.vi_mode_cursor.point.sub(terminal, Boundary::None, 1);\n\n                if let Some(regex_match) = ctx.search_next(origin, Direction::Left, Side::Left) {\n                    ctx.terminal_mut().vi_goto_point(*regex_match.start());\n                    ctx.mark_dirty();\n                }\n            },\n            Action::Vi(ViAction::SearchEnd) => {\n                let terminal = ctx.terminal();\n                let origin = terminal.vi_mode_cursor.point.add(terminal, Boundary::None, 1);\n\n                if let Some(regex_match) = ctx.search_next(origin, Direction::Right, Side::Right) {\n                    ctx.terminal_mut().vi_goto_point(*regex_match.end());\n                    ctx.mark_dirty();\n                }\n            },\n            Action::Vi(ViAction::CenterAroundViCursor) => {\n                let term = ctx.terminal();\n                let display_offset = term.grid().display_offset() as i32;\n                let target = -display_offset + term.screen_lines() as i32 / 2 - 1;\n                let line = term.vi_mode_cursor.point.line;\n                let scroll_lines = target - line.0;\n\n                ctx.scroll(Scroll::Delta(scroll_lines));\n            },\n            Action::Vi(ViAction::InlineSearchForward) => {\n                ctx.start_inline_search(Direction::Right, false)\n            },\n            Action::Vi(ViAction::InlineSearchBackward) => {\n                ctx.start_inline_search(Direction::Left, false)\n            },\n            Action::Vi(ViAction::InlineSearchForwardShort) => {\n                ctx.start_inline_search(Direction::Right, true)\n            },\n            Action::Vi(ViAction::InlineSearchBackwardShort) => {\n                ctx.start_inline_search(Direction::Left, true)\n            },\n            Action::Vi(ViAction::InlineSearchNext) => ctx.inline_search_next(),\n            Action::Vi(ViAction::InlineSearchPrevious) => ctx.inline_search_previous(),\n            action @ Action::Search(_) if !ctx.search_active() => {\n                debug!(\"Ignoring {action:?}: Search mode inactive\");\n            },\n            Action::Search(SearchAction::SearchFocusNext) => {\n                ctx.advance_search_origin(ctx.search_direction());\n            },\n            Action::Search(SearchAction::SearchFocusPrevious) => {\n                let direction = ctx.search_direction().opposite();\n                ctx.advance_search_origin(direction);\n            },\n            Action::Search(SearchAction::SearchConfirm) => ctx.confirm_search(),\n            Action::Search(SearchAction::SearchCancel) => ctx.cancel_search(),\n            Action::Search(SearchAction::SearchClear) => {\n                let direction = ctx.search_direction();\n                ctx.cancel_search();\n                ctx.start_search(direction);\n            },\n            Action::Search(SearchAction::SearchDeleteWord) => ctx.search_pop_word(),\n            Action::Search(SearchAction::SearchHistoryPrevious) => ctx.search_history_previous(),\n            Action::Search(SearchAction::SearchHistoryNext) => ctx.search_history_next(),\n            Action::Mouse(MouseAction::ExpandSelection) => ctx.expand_selection(),\n            Action::SearchForward => ctx.start_search(Direction::Right),\n            Action::SearchBackward => ctx.start_search(Direction::Left),\n            Action::Copy => ctx.copy_selection(ClipboardType::Clipboard),\n            #[cfg(not(any(target_os = \"macos\", windows)))]\n            Action::CopySelection => ctx.copy_selection(ClipboardType::Selection),\n            Action::ClearSelection => ctx.clear_selection(),\n            Action::Paste => {\n                let text = ctx.clipboard_mut().load(ClipboardType::Clipboard);\n                ctx.paste(&text, true);\n            },\n            Action::PasteSelection => {\n                let text = ctx.clipboard_mut().load(ClipboardType::Selection);\n                ctx.paste(&text, true);\n            },\n            Action::ToggleFullscreen => ctx.window().toggle_fullscreen(),\n            Action::ToggleMaximized => ctx.window().toggle_maximized(),\n            #[cfg(target_os = \"macos\")]\n            Action::ToggleSimpleFullscreen => ctx.window().toggle_simple_fullscreen(),\n            #[cfg(target_os = \"macos\")]\n            Action::Hide => ctx.event_loop().hide_application(),\n            #[cfg(target_os = \"macos\")]\n            Action::HideOtherApplications => ctx.event_loop().hide_other_applications(),\n            #[cfg(not(target_os = \"macos\"))]\n            Action::Hide => ctx.window().set_visible(false),\n            Action::Minimize => ctx.window().set_minimized(true),\n            Action::Quit => ctx.terminal_mut().exit(),\n            Action::IncreaseFontSize => ctx.change_font_size(FONT_SIZE_STEP),\n            Action::DecreaseFontSize => ctx.change_font_size(-FONT_SIZE_STEP),\n            Action::ResetFontSize => ctx.reset_font_size(),\n            Action::ScrollPageUp\n            | Action::ScrollPageDown\n            | Action::ScrollHalfPageUp\n            | Action::ScrollHalfPageDown => {\n                // Move vi mode cursor.\n                let term = ctx.terminal_mut();\n                let (scroll, amount) = match self {\n                    Action::ScrollPageUp => (Scroll::PageUp, term.screen_lines() as i32),\n                    Action::ScrollPageDown => (Scroll::PageDown, -(term.screen_lines() as i32)),\n                    Action::ScrollHalfPageUp => {\n                        let amount = term.screen_lines() as i32 / 2;\n                        (Scroll::Delta(amount), amount)\n                    },\n                    Action::ScrollHalfPageDown => {\n                        let amount = -(term.screen_lines() as i32 / 2);\n                        (Scroll::Delta(amount), amount)\n                    },\n                    _ => unreachable!(),\n                };\n\n                let old_vi_cursor = term.vi_mode_cursor;\n                term.vi_mode_cursor = term.vi_mode_cursor.scroll(term, amount);\n                if old_vi_cursor != term.vi_mode_cursor {\n                    ctx.mark_dirty();\n                }\n\n                ctx.scroll(scroll);\n            },\n            Action::ScrollLineUp => ctx.scroll(Scroll::Delta(1)),\n            Action::ScrollLineDown => ctx.scroll(Scroll::Delta(-1)),\n            Action::ScrollToTop => {\n                ctx.scroll(Scroll::Top);\n\n                // Move vi mode cursor.\n                let topmost_line = ctx.terminal().topmost_line();\n                ctx.terminal_mut().vi_mode_cursor.point.line = topmost_line;\n                ctx.terminal_mut().vi_motion(ViMotion::FirstOccupied);\n                ctx.mark_dirty();\n            },\n            Action::ScrollToBottom => {\n                ctx.scroll(Scroll::Bottom);\n\n                // Move vi mode cursor.\n                let term = ctx.terminal_mut();\n                term.vi_mode_cursor.point.line = term.bottommost_line();\n\n                // Move to beginning twice, to always jump across linewraps.\n                term.vi_motion(ViMotion::FirstOccupied);\n                term.vi_motion(ViMotion::FirstOccupied);\n                ctx.mark_dirty();\n            },\n            Action::ClearHistory => ctx.terminal_mut().clear_screen(ClearMode::Saved),\n            Action::ClearLogNotice => ctx.pop_message(),\n            #[cfg(not(target_os = \"macos\"))]\n            Action::CreateNewWindow => ctx.create_new_window(),\n            Action::SpawnNewInstance => ctx.spawn_new_instance(),\n            #[cfg(target_os = \"macos\")]\n            Action::CreateNewWindow => ctx.create_new_window(None),\n            #[cfg(target_os = \"macos\")]\n            Action::CreateNewTab => {\n                let tabbing_id = Some(ctx.window().tabbing_id());\n                ctx.create_new_window(tabbing_id);\n            },\n            #[cfg(target_os = \"macos\")]\n            Action::SelectNextTab => ctx.window().select_next_tab(),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectPreviousTab => ctx.window().select_previous_tab(),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab1 => ctx.window().select_tab_at_index(0),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab2 => ctx.window().select_tab_at_index(1),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab3 => ctx.window().select_tab_at_index(2),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab4 => ctx.window().select_tab_at_index(3),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab5 => ctx.window().select_tab_at_index(4),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab6 => ctx.window().select_tab_at_index(5),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab7 => ctx.window().select_tab_at_index(6),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab8 => ctx.window().select_tab_at_index(7),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectTab9 => ctx.window().select_tab_at_index(8),\n            #[cfg(target_os = \"macos\")]\n            Action::SelectLastTab => ctx.window().select_last_tab(),\n            _ => (),\n        }\n    }\n}\n\nimpl<T: EventListener, A: ActionContext<T>> Processor<T, A> {\n    pub fn new(ctx: A) -> Self {\n        Self { ctx, _phantom: Default::default() }\n    }\n\n    #[inline]\n    pub fn mouse_moved(&mut self, position: PhysicalPosition<f64>) {\n        let size_info = self.ctx.size_info();\n\n        let (x, y) = position.into();\n\n        let lmb_pressed = self.ctx.mouse().left_button_state == ElementState::Pressed;\n        let rmb_pressed = self.ctx.mouse().right_button_state == ElementState::Pressed;\n        if !self.ctx.selection_is_empty() && (lmb_pressed || rmb_pressed) {\n            self.update_selection_scrolling(y);\n        }\n\n        let display_offset = self.ctx.terminal().grid().display_offset();\n        let old_point = self.ctx.mouse().point(&size_info, display_offset);\n\n        let x = x.clamp(0, size_info.width() as i32 - 1) as usize;\n        let y = y.clamp(0, size_info.height() as i32 - 1) as usize;\n        self.ctx.mouse_mut().x = x;\n        self.ctx.mouse_mut().y = y;\n\n        let inside_text_area = size_info.contains_point(x, y);\n        let cell_side = self.cell_side(x);\n\n        let point = self.ctx.mouse().point(&size_info, display_offset);\n        let cell_changed = old_point != point;\n\n        // If the mouse hasn't changed cells, do nothing.\n        if !cell_changed\n            && self.ctx.mouse().cell_side == cell_side\n            && self.ctx.mouse().inside_text_area == inside_text_area\n        {\n            return;\n        }\n\n        self.ctx.mouse_mut().inside_text_area = inside_text_area;\n        self.ctx.mouse_mut().cell_side = cell_side;\n\n        // Update mouse state and check for URL change.\n        let mouse_state = self.cursor_state();\n        self.ctx.window().set_mouse_cursor(mouse_state);\n\n        // Prompt hint highlight update.\n        self.ctx.mouse_mut().hint_highlight_dirty = true;\n\n        // Don't launch URLs if mouse has moved.\n        self.ctx.mouse_mut().block_hint_launcher = true;\n\n        if (lmb_pressed || rmb_pressed)\n            && (self.ctx.modifiers().state().shift_key() || !self.ctx.mouse_mode())\n        {\n            self.ctx.update_selection(point, cell_side);\n        } else if cell_changed\n            && self.ctx.terminal().mode().intersects(TermMode::MOUSE_MOTION | TermMode::MOUSE_DRAG)\n        {\n            if lmb_pressed {\n                self.mouse_report(32, ElementState::Pressed);\n            } else if self.ctx.mouse().middle_button_state == ElementState::Pressed {\n                self.mouse_report(33, ElementState::Pressed);\n            } else if self.ctx.mouse().right_button_state == ElementState::Pressed {\n                self.mouse_report(34, ElementState::Pressed);\n            } else if self.ctx.terminal().mode().contains(TermMode::MOUSE_MOTION) {\n                self.mouse_report(35, ElementState::Pressed);\n            }\n        }\n    }\n\n    /// Check which side of a cell an X coordinate lies on.\n    fn cell_side(&self, x: usize) -> Side {\n        let size_info = self.ctx.size_info();\n\n        let cell_x =\n            x.saturating_sub(size_info.padding_x() as usize) % size_info.cell_width() as usize;\n        let half_cell_width = (size_info.cell_width() / 2.0) as usize;\n\n        let additional_padding =\n            (size_info.width() - size_info.padding_x() * 2.) % size_info.cell_width();\n        let end_of_grid = size_info.width() - size_info.padding_x() - additional_padding;\n\n        if cell_x > half_cell_width\n            // Edge case when mouse leaves the window.\n            || x as f32 >= end_of_grid\n        {\n            Side::Right\n        } else {\n            Side::Left\n        }\n    }\n\n    fn mouse_report(&mut self, button: u8, state: ElementState) {\n        let display_offset = self.ctx.terminal().grid().display_offset();\n        let point = self.ctx.mouse().point(&self.ctx.size_info(), display_offset);\n\n        // Assure the mouse point is not in the scrollback.\n        if point.line < 0 {\n            return;\n        }\n\n        // Calculate modifiers value.\n        let mut mods = 0;\n        let modifiers = self.ctx.modifiers().state();\n        if modifiers.shift_key() {\n            mods += 4;\n        }\n        if modifiers.alt_key() {\n            mods += 8;\n        }\n        if modifiers.control_key() {\n            mods += 16;\n        }\n\n        // Report mouse events.\n        if self.ctx.terminal().mode().contains(TermMode::SGR_MOUSE) {\n            self.sgr_mouse_report(point, button + mods, state);\n        } else if let ElementState::Released = state {\n            self.normal_mouse_report(point, 3 + mods);\n        } else {\n            self.normal_mouse_report(point, button + mods);\n        }\n    }\n\n    fn normal_mouse_report(&mut self, point: Point, button: u8) {\n        let Point { line, column } = point;\n        let utf8 = self.ctx.terminal().mode().contains(TermMode::UTF8_MOUSE);\n\n        let max_point = if utf8 { 2015 } else { 223 };\n\n        if line >= max_point || column >= max_point {\n            return;\n        }\n\n        let mut msg = vec![b'\\x1b', b'[', b'M', 32 + button];\n\n        let mouse_pos_encode = |pos: usize| -> Vec<u8> {\n            let pos = 32 + 1 + pos;\n            let first = 0xC0 + pos / 64;\n            let second = 0x80 + (pos & 63);\n            vec![first as u8, second as u8]\n        };\n\n        if utf8 && column >= Column(95) {\n            msg.append(&mut mouse_pos_encode(column.0));\n        } else {\n            msg.push(32 + 1 + column.0 as u8);\n        }\n\n        if utf8 && line >= 95 {\n            msg.append(&mut mouse_pos_encode(line.0 as usize));\n        } else {\n            msg.push(32 + 1 + line.0 as u8);\n        }\n\n        self.ctx.write_to_pty(msg);\n    }\n\n    fn sgr_mouse_report(&mut self, point: Point, button: u8, state: ElementState) {\n        let c = match state {\n            ElementState::Pressed => 'M',\n            ElementState::Released => 'm',\n        };\n\n        let msg = format!(\"\\x1b[<{};{};{}{}\", button, point.column + 1, point.line + 1, c);\n        self.ctx.write_to_pty(msg.into_bytes());\n    }\n\n    fn on_mouse_press(&mut self, button: MouseButton) {\n        // Handle mouse mode.\n        if !self.ctx.modifiers().state().shift_key() && self.ctx.mouse_mode() {\n            self.ctx.mouse_mut().click_state = ClickState::None;\n\n            let code = match button {\n                MouseButton::Left => 0,\n                MouseButton::Middle => 1,\n                MouseButton::Right => 2,\n                // Can't properly report more than three buttons..\n                MouseButton::Back | MouseButton::Forward | MouseButton::Other(_) => return,\n            };\n\n            self.mouse_report(code, ElementState::Pressed);\n        } else {\n            // Calculate time since the last click to handle double/triple clicks.\n            let now = Instant::now();\n            let elapsed = now - self.ctx.mouse().last_click_timestamp;\n            self.ctx.mouse_mut().last_click_timestamp = now;\n\n            // Update multi-click state.\n            self.ctx.mouse_mut().click_state = match self.ctx.mouse().click_state {\n                // Reset click state if button has changed.\n                _ if button != self.ctx.mouse().last_click_button => {\n                    self.ctx.mouse_mut().last_click_button = button;\n                    ClickState::Click\n                },\n                ClickState::Click if elapsed < CLICK_THRESHOLD => ClickState::DoubleClick,\n                ClickState::DoubleClick if elapsed < CLICK_THRESHOLD => ClickState::TripleClick,\n                _ => ClickState::Click,\n            };\n\n            // Load mouse point, treating message bar and padding as the closest cell.\n            let display_offset = self.ctx.terminal().grid().display_offset();\n            let point = self.ctx.mouse().point(&self.ctx.size_info(), display_offset);\n\n            if let MouseButton::Left = button {\n                self.on_left_click(point)\n            }\n        }\n    }\n\n    /// Handle left click selection and vi mode cursor movement.\n    fn on_left_click(&mut self, point: Point) {\n        let side = self.ctx.mouse().cell_side;\n\n        match self.ctx.mouse().click_state {\n            ClickState::Click => {\n                // Don't launch URLs if this click cleared the selection.\n                self.ctx.mouse_mut().block_hint_launcher = !self.ctx.selection_is_empty();\n\n                self.ctx.clear_selection();\n\n                // Start new empty selection.\n                if self.ctx.modifiers().state().control_key() {\n                    self.ctx.start_selection(SelectionType::Block, point, side);\n                } else {\n                    self.ctx.start_selection(SelectionType::Simple, point, side);\n                }\n            },\n            ClickState::DoubleClick => {\n                self.ctx.mouse_mut().block_hint_launcher = true;\n                self.ctx.start_selection(SelectionType::Semantic, point, side);\n            },\n            ClickState::TripleClick => {\n                self.ctx.mouse_mut().block_hint_launcher = true;\n                self.ctx.start_selection(SelectionType::Lines, point, side);\n            },\n            ClickState::None => (),\n        };\n\n        // Move vi mode cursor to mouse click position.\n        if self.ctx.terminal().mode().contains(TermMode::VI) && !self.ctx.search_active() {\n            self.ctx.terminal_mut().vi_mode_cursor.point = point;\n            self.ctx.mark_dirty();\n        }\n    }\n\n    fn on_mouse_release(&mut self, button: MouseButton) {\n        if !self.ctx.modifiers().state().shift_key() && self.ctx.mouse_mode() {\n            let code = match button {\n                MouseButton::Left => 0,\n                MouseButton::Middle => 1,\n                MouseButton::Right => 2,\n                // Can't properly report more than three buttons.\n                MouseButton::Back | MouseButton::Forward | MouseButton::Other(_) => return,\n            };\n            self.mouse_report(code, ElementState::Released);\n            return;\n        }\n\n        // Trigger hints highlighted by the mouse.\n        let hint = self.ctx.display().highlighted_hint.take();\n        if let Some(hint) = hint.as_ref().filter(|_| button == MouseButton::Left) {\n            self.ctx.trigger_hint(hint);\n        }\n        self.ctx.display().highlighted_hint = hint;\n\n        let timer_id = TimerId::new(Topic::SelectionScrolling, self.ctx.window().id());\n        self.ctx.scheduler_mut().unschedule(timer_id);\n\n        if let MouseButton::Left | MouseButton::Right = button {\n            // Copy selection on release, to prevent flooding the display server.\n            self.ctx.copy_selection(ClipboardType::Selection);\n        }\n    }\n\n    pub fn mouse_wheel_input(&mut self, delta: MouseScrollDelta, phase: TouchPhase) {\n        let multiplier = self.ctx.config().scrolling.multiplier;\n        match delta {\n            MouseScrollDelta::LineDelta(columns, lines) => {\n                let new_scroll_px_x = columns * self.ctx.size_info().cell_width();\n                let new_scroll_px_y = lines * self.ctx.size_info().cell_height();\n                self.scroll_terminal(\n                    new_scroll_px_x as f64,\n                    new_scroll_px_y as f64,\n                    multiplier as f64,\n                );\n            },\n            MouseScrollDelta::PixelDelta(mut lpos) => {\n                match phase {\n                    TouchPhase::Started => {\n                        // Reset offset to zero.\n                        self.ctx.mouse_mut().accumulated_scroll = Default::default();\n                    },\n                    TouchPhase::Moved => {\n                        // When the angle between (x, 0) and (x, y) is lower than ~25 degrees\n                        // (cosine is larger that 0.9) we consider this scrolling as horizontal.\n                        if lpos.x.abs() / lpos.x.hypot(lpos.y) > 0.9 {\n                            lpos.y = 0.;\n                        } else {\n                            lpos.x = 0.;\n                        }\n\n                        self.scroll_terminal(lpos.x, lpos.y, multiplier as f64);\n                    },\n                    _ => (),\n                }\n            },\n        }\n    }\n\n    fn scroll_terminal(&mut self, new_scroll_x_px: f64, new_scroll_y_px: f64, multiplier: f64) {\n        const MOUSE_WHEEL_UP: u8 = 64;\n        const MOUSE_WHEEL_DOWN: u8 = 65;\n        const MOUSE_WHEEL_LEFT: u8 = 66;\n        const MOUSE_WHEEL_RIGHT: u8 = 67;\n\n        let width = f64::from(self.ctx.size_info().cell_width());\n        let height = f64::from(self.ctx.size_info().cell_height());\n\n        if self.ctx.mouse_mode() {\n            self.ctx.mouse_mut().accumulated_scroll.x += new_scroll_x_px;\n            self.ctx.mouse_mut().accumulated_scroll.y += new_scroll_y_px;\n\n            let code = if new_scroll_y_px > 0. { MOUSE_WHEEL_UP } else { MOUSE_WHEEL_DOWN };\n            let lines = (self.ctx.mouse().accumulated_scroll.y / height).abs() as i32;\n\n            for _ in 0..lines {\n                self.mouse_report(code, ElementState::Pressed);\n            }\n\n            let code = if new_scroll_x_px > 0. { MOUSE_WHEEL_LEFT } else { MOUSE_WHEEL_RIGHT };\n            let columns = (self.ctx.mouse().accumulated_scroll.x / width).abs() as i32;\n\n            for _ in 0..columns {\n                self.mouse_report(code, ElementState::Pressed);\n            }\n        } else if self\n            .ctx\n            .terminal()\n            .mode()\n            .contains(TermMode::ALT_SCREEN | TermMode::ALTERNATE_SCROLL)\n            && !self.ctx.modifiers().state().shift_key()\n        {\n            self.ctx.mouse_mut().accumulated_scroll.x += new_scroll_x_px * multiplier;\n            self.ctx.mouse_mut().accumulated_scroll.y += new_scroll_y_px * multiplier;\n\n            // The chars here are the same as for the respective arrow keys.\n            let line_cmd = if new_scroll_y_px > 0. { b'A' } else { b'B' };\n            let column_cmd = if new_scroll_x_px > 0. { b'D' } else { b'C' };\n\n            let lines = (self.ctx.mouse().accumulated_scroll.y / height).abs() as usize;\n            let columns = (self.ctx.mouse().accumulated_scroll.x / width).abs() as usize;\n\n            let mut content = Vec::with_capacity(3 * (lines + columns));\n\n            for _ in 0..lines {\n                content.push(0x1b);\n                content.push(b'O');\n                content.push(line_cmd);\n            }\n\n            for _ in 0..columns {\n                content.push(0x1b);\n                content.push(b'O');\n                content.push(column_cmd);\n            }\n\n            self.ctx.write_to_pty(content);\n        } else {\n            self.ctx.mouse_mut().accumulated_scroll.y += new_scroll_y_px * multiplier;\n\n            let lines = (self.ctx.mouse().accumulated_scroll.y / height) as i32;\n\n            if lines != 0 {\n                self.ctx.scroll(Scroll::Delta(lines));\n            }\n        }\n\n        self.ctx.mouse_mut().accumulated_scroll.x %= width;\n        self.ctx.mouse_mut().accumulated_scroll.y %= height;\n    }\n\n    pub fn on_focus_change(&mut self, is_focused: bool) {\n        if self.ctx.terminal().mode().contains(TermMode::FOCUS_IN_OUT) {\n            let chr = if is_focused { \"I\" } else { \"O\" };\n\n            let msg = format!(\"\\x1b[{}\", chr);\n            self.ctx.write_to_pty(msg.into_bytes());\n        }\n    }\n\n    /// Handle touch input.\n    pub fn touch(&mut self, touch: TouchEvent) {\n        match touch.phase {\n            TouchPhase::Started => self.on_touch_start(touch),\n            TouchPhase::Moved => self.on_touch_motion(touch),\n            TouchPhase::Ended | TouchPhase::Cancelled => self.on_touch_end(touch),\n        }\n    }\n\n    /// Handle beginning of touch input.\n    pub fn on_touch_start(&mut self, touch: TouchEvent) {\n        let touch_purpose = self.ctx.touch_purpose();\n        *touch_purpose = match mem::take(touch_purpose) {\n            TouchPurpose::None => TouchPurpose::Tap(touch),\n            TouchPurpose::Tap(start) => TouchPurpose::Zoom(TouchZoom::new((start, touch))),\n            TouchPurpose::Zoom(zoom) => TouchPurpose::Invalid(zoom.slots()),\n            TouchPurpose::Scroll(event) | TouchPurpose::Select(event) => {\n                let mut set = HashSet::default();\n                set.insert(event.id);\n                TouchPurpose::Invalid(set)\n            },\n            TouchPurpose::Invalid(mut slots) => {\n                slots.insert(touch.id);\n                TouchPurpose::Invalid(slots)\n            },\n        };\n    }\n\n    /// Handle touch input movement.\n    pub fn on_touch_motion(&mut self, touch: TouchEvent) {\n        let touch_purpose = self.ctx.touch_purpose();\n        match touch_purpose {\n            TouchPurpose::None => (),\n            // Handle transition from tap to scroll/select.\n            TouchPurpose::Tap(start) => {\n                let delta_x = touch.location.x - start.location.x;\n                let delta_y = touch.location.y - start.location.y;\n                if delta_x.abs() > MAX_TAP_DISTANCE {\n                    // Update gesture state.\n                    let start_location = start.location;\n                    *touch_purpose = TouchPurpose::Select(*start);\n\n                    // Start simulated mouse input.\n                    self.mouse_moved(start_location);\n                    self.mouse_input(ElementState::Pressed, MouseButton::Left);\n\n                    // Apply motion since touch start.\n                    self.on_touch_motion(touch);\n                } else if delta_y.abs() > MAX_TAP_DISTANCE {\n                    // Update gesture state.\n                    *touch_purpose = TouchPurpose::Scroll(*start);\n\n                    // Apply motion since touch start.\n                    self.on_touch_motion(touch);\n                }\n            },\n            TouchPurpose::Zoom(zoom) => {\n                let font_delta = zoom.font_delta(touch);\n                self.ctx.change_font_size(font_delta);\n            },\n            TouchPurpose::Scroll(last_touch) => {\n                // Calculate delta and update last touch position.\n                let delta_y = touch.location.y - last_touch.location.y;\n                *touch_purpose = TouchPurpose::Scroll(touch);\n\n                // Use a fixed scroll factor for touchscreens, to accurately track finger motion.\n                self.scroll_terminal(0., delta_y, 1.0);\n            },\n            TouchPurpose::Select(_) => self.mouse_moved(touch.location),\n            TouchPurpose::Invalid(_) => (),\n        }\n    }\n\n    /// Handle end of touch input.\n    pub fn on_touch_end(&mut self, touch: TouchEvent) {\n        // Finalize the touch motion up to the release point.\n        self.on_touch_motion(touch);\n\n        let touch_purpose = self.ctx.touch_purpose();\n        match touch_purpose {\n            // Simulate LMB clicks.\n            TouchPurpose::Tap(start) => {\n                let start_location = start.location;\n                *touch_purpose = Default::default();\n\n                self.mouse_moved(start_location);\n                self.mouse_input(ElementState::Pressed, MouseButton::Left);\n                self.mouse_input(ElementState::Released, MouseButton::Left);\n            },\n            // Invalidate zoom once a finger was released.\n            TouchPurpose::Zoom(zoom) => {\n                let mut slots = zoom.slots();\n                slots.remove(&touch.id);\n                *touch_purpose = TouchPurpose::Invalid(slots);\n            },\n            // Reset touch state once all slots were released.\n            TouchPurpose::Invalid(slots) => {\n                slots.remove(&touch.id);\n                if slots.is_empty() {\n                    *touch_purpose = Default::default();\n                }\n            },\n            // Release simulated LMB.\n            TouchPurpose::Select(_) => {\n                *touch_purpose = Default::default();\n                self.mouse_input(ElementState::Released, MouseButton::Left);\n            },\n            // Reset touch state on scroll finish.\n            TouchPurpose::Scroll(_) => *touch_purpose = Default::default(),\n            TouchPurpose::None => (),\n        }\n    }\n\n    /// Reset mouse cursor based on modifier and terminal state.\n    #[inline]\n    pub fn reset_mouse_cursor(&mut self) {\n        let mouse_state = self.cursor_state();\n        self.ctx.window().set_mouse_cursor(mouse_state);\n    }\n\n    /// Modifier state change.\n    pub fn modifiers_input(&mut self, modifiers: Modifiers) {\n        *self.ctx.modifiers() = modifiers;\n\n        // Prompt hint highlight update.\n        self.ctx.mouse_mut().hint_highlight_dirty = true;\n\n        // Update mouse state and check for URL change.\n        let mouse_state = self.cursor_state();\n        self.ctx.window().set_mouse_cursor(mouse_state);\n    }\n\n    pub fn mouse_input(&mut self, state: ElementState, button: MouseButton) {\n        match button {\n            MouseButton::Left => self.ctx.mouse_mut().left_button_state = state,\n            MouseButton::Middle => self.ctx.mouse_mut().middle_button_state = state,\n            MouseButton::Right => self.ctx.mouse_mut().right_button_state = state,\n            _ => (),\n        }\n\n        // Skip normal mouse events if the message bar has been clicked.\n        if self.message_bar_cursor_state() == Some(CursorIcon::Pointer)\n            && state == ElementState::Pressed\n        {\n            let size = self.ctx.size_info();\n\n            let current_lines = self.ctx.message().map_or(0, |m| m.text(&size).len());\n\n            self.ctx.clear_selection();\n            self.ctx.pop_message();\n\n            // Reset cursor when message bar height changed or all messages are gone.\n            let new_lines = self.ctx.message().map_or(0, |m| m.text(&size).len());\n\n            let new_icon = match current_lines.cmp(&new_lines) {\n                Ordering::Less => CursorIcon::Default,\n                Ordering::Equal => CursorIcon::Pointer,\n                Ordering::Greater => {\n                    if self.ctx.mouse_mode() {\n                        CursorIcon::Default\n                    } else {\n                        CursorIcon::Text\n                    }\n                },\n            };\n\n            self.ctx.window().set_mouse_cursor(new_icon);\n        } else {\n            match state {\n                ElementState::Pressed => {\n                    // Process mouse press before bindings to update the `click_state`.\n                    self.on_mouse_press(button);\n                    self.process_mouse_bindings(button);\n                },\n                ElementState::Released => self.on_mouse_release(button),\n            }\n        }\n    }\n\n    /// Attempt to find a binding and execute its action.\n    ///\n    /// The provided mode, mods, and key must match what is allowed by a binding\n    /// for its action to be executed.\n    fn process_mouse_bindings(&mut self, button: MouseButton) {\n        let mode = BindingMode::new(self.ctx.terminal().mode(), self.ctx.search_active());\n        let mouse_mode = self.ctx.mouse_mode();\n        let mods = self.ctx.modifiers().state();\n        let mouse_bindings = self.ctx.config().mouse_bindings().to_owned();\n\n        // If mouse mode is active, also look for bindings without shift.\n        let mut check_fallback = mouse_mode && mods.contains(ModifiersState::SHIFT);\n\n        for binding in &mouse_bindings {\n            // Don't trigger normal bindings in mouse mode unless Shift is pressed.\n...\n// Path: alacritty/src/display/damage.rs\nuse std::iter::Peekable;\nuse std::{cmp, mem};\n\nuse glutin::surface::Rect;\n\nuse alacritty_terminal::index::Point;\nuse alacritty_terminal::selection::SelectionRange;\nuse alacritty_terminal::term::{LineDamageBounds, TermDamageIterator};\n\nuse crate::display::SizeInfo;\n\n/// State of the damage tracking for the [`Display`].\n///\n/// [`Display`]: crate::display::Display\n#[derive(Debug)]\npub struct DamageTracker {\n    /// Position of the previously drawn Vi cursor.\n    pub old_vi_cursor: Option<Point<usize>>,\n    /// The location of the old selection.\n    pub old_selection: Option<SelectionRange>,\n    /// Highlight damage submitted for the compositor.\n    pub debug: bool,\n\n    /// The damage for the frames.\n    frames: [FrameDamage; 2],\n    screen_lines: usize,\n    columns: usize,\n}\n\nimpl DamageTracker {\n    pub fn new(screen_lines: usize, columns: usize) -> Self {\n        let mut tracker = Self {\n            columns,\n            screen_lines,\n            debug: false,\n            old_vi_cursor: None,\n            old_selection: None,\n            frames: Default::default(),\n        };\n        tracker.resize(screen_lines, columns);\n        tracker\n    }\n\n    #[inline]\n    #[must_use]\n    pub fn frame(&mut self) -> &mut FrameDamage {\n        &mut self.frames[0]\n    }\n\n    #[inline]\n    #[must_use]\n    pub fn next_frame(&mut self) -> &mut FrameDamage {\n        &mut self.frames[1]\n    }\n\n    /// Advance to the next frame resetting the state for the active frame.\n    #[inline]\n    pub fn swap_damage(&mut self) {\n        let screen_lines = self.screen_lines;\n        let columns = self.columns;\n        self.frame().reset(screen_lines, columns);\n        self.frames.swap(0, 1);\n    }\n\n    /// Resize the damage information in the tracker.\n    pub fn resize(&mut self, screen_lines: usize, columns: usize) {\n        self.screen_lines = screen_lines;\n        self.columns = columns;\n        for frame in &mut self.frames {\n            frame.reset(screen_lines, columns);\n        }\n        self.frame().full = true;\n    }\n\n    /// Damage vi cursor inside the viewport.\n    pub fn damage_vi_cursor(&mut self, mut vi_cursor: Option<Point<usize>>) {\n        mem::swap(&mut self.old_vi_cursor, &mut vi_cursor);\n\n        if self.frame().full {\n            return;\n        }\n\n        if let Some(vi_cursor) = self.old_vi_cursor {\n            self.frame().damage_point(vi_cursor);\n        }\n\n        if let Some(vi_cursor) = vi_cursor {\n            self.frame().damage_point(vi_cursor);\n        }\n    }\n\n    /// Get shaped frame damage for the active frame.\n    pub fn shape_frame_damage(&self, size_info: SizeInfo<u32>) -> Vec<Rect> {\n        if self.frames[0].full {\n            vec![Rect::new(0, 0, size_info.width() as i32, size_info.height() as i32)]\n        } else {\n            let lines_damage = RenderDamageIterator::new(\n                TermDamageIterator::new(&self.frames[0].lines, 0),\n                &size_info,\n            );\n            lines_damage.chain(self.frames[0].rects.iter().copied()).collect()\n        }\n    }\n\n    /// Add the current frame's selection damage.\n    pub fn damage_selection(\n        &mut self,\n        mut selection: Option<SelectionRange>,\n        display_offset: usize,\n    ) {\n        mem::swap(&mut self.old_selection, &mut selection);\n\n        if self.frame().full || selection == self.old_selection {\n            return;\n        }\n\n        for selection in self.old_selection.into_iter().chain(selection) {\n            let display_offset = display_offset as i32;\n            let last_visible_line = self.screen_lines as i32 - 1;\n            let columns = self.columns;\n\n            // Ignore invisible selection.\n            if selection.end.line.0 + display_offset < 0\n                || selection.start.line.0.abs() < display_offset - last_visible_line\n            {\n                continue;\n            };\n\n            let start = cmp::max(selection.start.line.0 + display_offset, 0) as usize;\n            let end = (selection.end.line.0 + display_offset).clamp(0, last_visible_line) as usize;\n            for line in start..=end {\n                self.frame().lines[line].expand(0, columns - 1);\n            }\n        }\n    }\n}\n\n/// Damage state for the rendering frame.\n#[derive(Debug, Default)]\npub struct FrameDamage {\n    /// The entire frame needs to be redrawn.\n    full: bool,\n    /// Terminal lines damaged in the given frame.\n    lines: Vec<LineDamageBounds>,\n    /// Rectangular regions damage in the given frame.\n    rects: Vec<Rect>,\n}\n\nimpl FrameDamage {\n    /// Damage line for the given frame.\n    #[inline]\n    pub fn damage_line(&mut self, damage: LineDamageBounds) {\n        self.lines[damage.line].expand(damage.left, damage.right);\n    }\n\n    #[inline]\n    pub fn damage_point(&mut self, point: Point<usize>) {\n        self.lines[point.line].expand(point.column.0, point.column.0);\n    }\n\n    /// Mark the frame as fully damaged.\n    #[inline]\n    pub fn mark_fully_damaged(&mut self) {\n        self.full = true;\n    }\n\n    /// Add viewport rectangle to damage.\n    ///\n    /// This allows covering elements outside of the terminal viewport, like message bar.\n    #[inline]\n    pub fn add_viewport_rect(\n        &mut self,\n        size_info: &SizeInfo,\n        x: i32,\n        y: i32,\n        width: i32,\n        height: i32,\n    ) {\n        let y = viewport_y_to_damage_y(size_info, y, height);\n        self.rects.push(Rect { x, y, width, height });\n    }\n\n    fn reset(&mut self, num_lines: usize, num_cols: usize) {\n        self.full = false;\n        self.rects.clear();\n        self.lines.clear();\n        self.lines.reserve(num_lines);\n        for line in 0..num_lines {\n            self.lines.push(LineDamageBounds::undamaged(line, num_cols));\n        }\n    }\n}\n\n/// Convert viewport `y` coordinate to [`Rect`] damage coordinate.\npub fn viewport_y_to_damage_y(size_info: &SizeInfo, y: i32, height: i32) -> i32 {\n    size_info.height() as i32 - y - height\n}\n\n/// Convert viewport `y` coordinate to [`Rect`] damage coordinate.\npub fn damage_y_to_viewport_y(size_info: &SizeInfo, rect: &Rect) -> i32 {\n    size_info.height() as i32 - rect.y - rect.height\n}\n\n/// Iterator which converts `alacritty_terminal` damage information into renderer damaged rects.\nstruct RenderDamageIterator<'a> {\n    damaged_lines: Peekable<TermDamageIterator<'a>>,\n    size_info: &'a SizeInfo<u32>,\n}\n\nimpl<'a> RenderDamageIterator<'a> {\n    pub fn new(damaged_lines: TermDamageIterator<'a>, size_info: &'a SizeInfo<u32>) -> Self {\n        Self { damaged_lines: damaged_lines.peekable(), size_info }\n    }\n\n    #[inline]\n    fn rect_for_line(&self, line_damage: LineDamageBounds) -> Rect {\n        let size_info = &self.size_info;\n        let y_top = size_info.height() - size_info.padding_y();\n        let x = size_info.padding_x() + line_damage.left as u32 * size_info.cell_width();\n        let y = y_top - (line_damage.line + 1) as u32 * size_info.cell_height();\n        let width = (line_damage.right - line_damage.left + 1) as u32 * size_info.cell_width();\n        Rect::new(x as i32, y as i32, width as i32, size_info.cell_height() as i32)\n    }\n\n    // Make sure to damage near cells to include wide chars.\n    #[inline]\n    fn overdamage(size_info: &SizeInfo<u32>, mut rect: Rect) -> Rect {\n        rect.x = (rect.x - size_info.cell_width() as i32).max(0);\n        rect.width = cmp::min(\n            size_info.width() as i32 - rect.x,\n            rect.width + 2 * size_info.cell_width() as i32,\n        );\n        rect.y = (rect.y - size_info.cell_height() as i32 / 2).max(0);\n        rect.height = cmp::min(\n            size_info.height() as i32 - rect.y,\n            rect.height + size_info.cell_height() as i32,\n        );\n\n        rect\n    }\n}\n\nimpl<'a> Iterator for RenderDamageIterator<'a> {\n    type Item = Rect;\n\n    fn next(&mut self) -> Option<Rect> {\n        let line = self.damaged_lines.next()?;\n        let size_info = &self.size_info;\n        let mut total_damage_rect = Self::overdamage(size_info, self.rect_for_line(line));\n\n        // Merge rectangles which overlap with each other.\n        while let Some(line) = self.damaged_lines.peek().copied() {\n            let next_rect = Self::overdamage(size_info, self.rect_for_line(line));\n            if !rects_overlap(total_damage_rect, next_rect) {\n                break;\n            }\n\n            total_damage_rect = merge_rects(total_damage_rect, next_rect);\n            let _ = self.damaged_lines.next();\n        }\n\n        Some(total_damage_rect)\n    }\n}\n\n/// Check if two given [`glutin::surface::Rect`] overlap.\nfn rects_overlap(lhs: Rect, rhs: Rect) -> bool {\n    !(\n        // `lhs` is left of `rhs`.\n        lhs.x + lhs.width < rhs.x\n        // `lhs` is right of `rhs`.\n        || rhs.x + rhs.width < lhs.x\n        // `lhs` is below `rhs`.\n        || lhs.y + lhs.height < rhs.y\n        // `lhs` is above `rhs`.\n        || rhs.y + rhs.height < lhs.y\n    )\n}\n\n/// Merge two [`glutin::surface::Rect`] by producing the smallest rectangle that contains both.\n#[inline]\n\nfn merge_rects(lhs: Rect, rhs: Rect) -> Rect {\n    let left_x = cmp::min(lhs.x, rhs.x);\n    let right_x = cmp::max(lhs.x + lhs.width, rhs.x + rhs.width);\n    let y_top = cmp::max(lhs.y + lhs.height, rhs.y + rhs.height);\n    let y_bottom = cmp::min(lhs.y, rhs.y);\n    Rect::new(left_x, y_bottom, right_x - left_x, y_top - y_bottom)\n}\n\n#[cfg(test)]\nmod tests {\n    use super::*;\n\n    #[test]\n    fn damage_rect_math() {\n        let rect_side = 10;\n        let cell_size = 4;\n        let bound = 100;\n\n        let size_info: SizeInfo<u32> = SizeInfo::new(\n            bound as f32,\n            bound as f32,\n            cell_size as f32,\n            cell_size as f32,\n            2.,\n            2.,\n            true,\n        )\n        .into();\n\n        // Test min clamping.\n        let rect = Rect::new(0, 0, rect_side, rect_side);\n        let rect = RenderDamageIterator::overdamage(&size_info, rect);\n        assert_eq!(Rect::new(0, 0, rect_side + 2 * cell_size, 10 + cell_size), rect);\n\n        // Test max clamping.\n        let rect = Rect::new(bound, bound, rect_side, rect_side);\n        let rect = RenderDamageIterator::overdamage(&size_info, rect);\n        assert_eq!(\n            Rect::new(bound - cell_size, bound - cell_size / 2, cell_size, cell_size / 2),\n            rect\n        );\n\n        // Test no clamping.\n        let rect = Rect::new(bound / 2, bound / 2, rect_side, rect_side);\n        let rect = RenderDamageIterator::overdamage(&size_info, rect);\n        assert_eq!(\n            Rect::new(\n                bound / 2 - cell_size,\n                bound / 2 - cell_size / 2,\n                rect_side + 2 * cell_size,\n                rect_side + cell_size\n            ),\n            rect\n        );\n    }\n\n    #[test]\n    fn add_viewport_damage() {\n        let mut frame_damage = FrameDamage::default();\n        let viewport_height = 100.;\n        let x = 0;\n        let y = 40;\n        let height = 5;\n        let width = 10;\n        let size_info = SizeInfo::new(viewport_height, viewport_height, 5., 5., 0., 0., true);\n        frame_damage.add_viewport_rect(&size_info, x, y, width, height);\n        assert_eq!(frame_damage.rects[0], Rect {\n            x,\n            y: viewport_height as i32 - y - height,\n            width,\n            height\n        });\n        assert_eq!(frame_damage.rects[0].y, viewport_y_to_damage_y(&size_info, y, height));\n        assert_eq!(damage_y_to_viewport_y(&size_info, &frame_damage.rects[0]), y);\n    }\n}\n\n...\n", "needle_token_start": 15320, "needle_token_end": 15472, "code_context_ntokens": 16204, "output": ["```rust\n/// Merge two [`glutin::surface::Rect`] by producing the smallest rectangle that contains both.\n#[inline]\nfn merge_rects(lhs: Rect, rhs: Rect) -> Rect {\n    let left_x = cmp::min(lhs.x, rhs.x);\n    let right_x = cmp::max(lhs.x + lhs.width, rhs.x + rhs.width);\n    let y_top = cmp::max(lhs.y + lhs.height, rhs.y + rhs.height);\n    let y_bottom = cmp::min(lhs.y, rhs.y);\n    Rect::new(left_x, y_bottom, right_x - left_x, y_top - y_bottom)\n}\n```"]}
{"repo": "junegunn/fzf", "name": "parseLabelPosition", "language": "go", "path": "src/options.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to interpret and set the positioning attributes for a label based on a given string argument.\n2. **Input**: It takes a pointer to a structure holding label settings and a string argument specifying the desired position attributes.\n3. **Output**: There is no direct output; however, the function modifies the label settings in the provided structure based on the string argument.\n4. **Procedure**: The function initializes the label's column index to zero and its vertical position to not bottom. It then processes each token in the string argument (split by a regular expression and converted to lowercase). Depending on the token, it adjusts the label's column index or vertical position (either top or bottom).\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "    --walker-skip=DIRS     Comma-separated list of directory names to skip\n                           (default: .git,node_modules)\n\n  Shell integration\n    --bash                 Print script to set up Bash shell integration\n    --zsh                  Print script to set up Zsh shell integration\n    --fish                 Print script to set up Fish shell integration\n\n  Environment variables\n    FZF_DEFAULT_COMMAND    Default command to use when input is tty\n    FZF_DEFAULT_OPTS       Default options (e.g. '--layout=reverse --info=inline')\n    FZF_DEFAULT_OPTS_FILE  Location of the file to read default options from\n    FZF_API_KEY            X-API-Key header for HTTP server (--listen)\n\n`\n\nconst defaultInfoPrefix = \" < \"\n\n// Case denotes case-sensitivity of search\ntype Case int\n\n// Case-sensitivities\nconst (\n\tCaseSmart Case = iota\n\tCaseIgnore\n\tCaseRespect\n)\n\n// Sort criteria\ntype criterion int\n\nconst (\n\tbyScore criterion = iota\n\tbyChunk\n\tbyLength\n\tbyBegin\n\tbyEnd\n)\n\ntype heightSpec struct {\n\tsize    float64\n\tpercent bool\n\tauto    bool\n\tinverse bool\n}\n\ntype sizeSpec struct {\n\tsize    float64\n\tpercent bool\n}\n\nfunc defaultMargin() [4]sizeSpec {\n\treturn [4]sizeSpec{}\n}\n\ntype trackOption int\n\nconst (\n\ttrackDisabled trackOption = iota\n\ttrackEnabled\n\ttrackCurrent\n)\n\ntype windowPosition int\n\nconst (\n\tposUp windowPosition = iota\n\tposDown\n\tposLeft\n\tposRight\n)\n\ntype layoutType int\n\nconst (\n\tlayoutDefault layoutType = iota\n\tlayoutReverse\n\tlayoutReverseList\n)\n\ntype infoStyle int\n\nconst (\n\tinfoDefault infoStyle = iota\n\tinfoRight\n\tinfoInline\n\tinfoInlineRight\n\tinfoHidden\n)\n\ntype labelOpts struct {\n\tlabel  string\n\tcolumn int\n\tbottom bool\n}\n\ntype previewOpts struct {\n\tcommand     string\n\tposition    windowPosition\n\tsize        sizeSpec\n\tscroll      string\n\thidden      bool\n\twrap        bool\n\tcycle       bool\n\tfollow      bool\n\tborder      tui.BorderShape\n\theaderLines int\n\tthreshold   int\n\talternative *previewOpts\n}\n\nfunc (o *previewOpts) Visible() bool {\n\treturn o.size.size > 0 || o.alternative != nil && o.alternative.size.size > 0\n}\n\nfunc (o *previewOpts) Toggle() {\n\to.hidden = !o.hidden\n}\n\n\nfunc parseLabelPosition(opts *labelOpts, arg string) {\n\topts.column = 0\n\topts.bottom = false\n\tfor _, token := range splitRegexp.Split(strings.ToLower(arg), -1) {\n\t\tswitch token {\n\t\tcase \"center\":\n\t\t\topts.column = 0\n\t\tcase \"bottom\":\n\t\t\topts.bottom = true\n\t\tcase \"top\":\n\t\t\topts.bottom = false\n\t\tdefault:\n\t\t\topts.column = atoi(token)\n\t\t}\n\t}\n}\n\nfunc (a previewOpts) aboveOrBelow() bool {\n\treturn a.size.size > 0 && (a.position == posUp || a.position == posDown)\n}\n\nfunc (a previewOpts) sameLayout(b previewOpts) bool {\n\treturn a.size == b.size && a.position == b.position && a.border == b.border && a.hidden == b.hidden && a.threshold == b.threshold &&\n\t\t(a.alternative != nil && b.alternative != nil && a.alternative.sameLayout(*b.alternative) ||\n\t\t\ta.alternative == nil && b.alternative == nil)\n}\n\nfunc (a previewOpts) sameContentLayout(b previewOpts) bool {\n\treturn a.wrap == b.wrap && a.headerLines == b.headerLines\n}\n\nfunc firstLine(s string) string {\n\treturn strings.SplitN(s, \"\\n\", 2)[0]\n}\n\ntype walkerOpts struct {\n\tfile   bool\n\tdir    bool\n\thidden bool\n\tfollow bool\n}\n\n// Options stores the values of command-line options\ntype Options struct {\n\tBash         bool\n\tZsh          bool\n\tFish         bool\n\tFuzzy        bool\n\tFuzzyAlgo    algo.Algo\n\tScheme       string\n\tExtended     bool\n\tPhony        bool\n\tCase         Case\n\tNormalize    bool\n\tNth          []Range\n\tWithNth      []Range\n\tDelimiter    Delimiter\n\tSort         int\n\tTrack        trackOption\n\tTac          bool\n\tCriteria     []criterion\n\tMulti        int\n\tAnsi         bool\n\tMouse        bool\n\tTheme        *tui.ColorTheme\n\tBlack        bool\n\tBold         bool\n\tHeight       heightSpec\n\tMinHeight    int\n\tLayout       layoutType\n\tCycle        bool\n\tKeepRight    bool\n\tHscroll      bool\n\tHscrollOff   int\n\tScrollOff    int\n\tFileWord     bool\n\tInfoStyle    infoStyle\n\tInfoPrefix   string\n\tSeparator    *string\n\tJumpLabels   string\n\tPrompt       string\n\tPointer      string\n\tMarker       string\n\tQuery        string\n\tSelect1      bool\n\tExit0        bool\n\tFilter       *string\n\tToggleSort   bool\n\tExpect       map[tui.Event]string\n\tKeymap       map[tui.Event][]*action\n\tPreview      previewOpts\n\tPrintQuery   bool\n\tReadZero     bool\n\tPrinter      func(string)\n\tPrintSep     string\n\tSync         bool\n\tHistory      *History\n\tHeader       []string\n\tHeaderLines  int\n\tHeaderFirst  bool\n\tEllipsis     string\n\tScrollbar    *string\n\tMargin       [4]sizeSpec\n\tPadding      [4]sizeSpec\n\tBorderShape  tui.BorderShape\n\tBorderLabel  labelOpts\n\tPreviewLabel labelOpts\n\tUnicode      bool\n\tAmbidouble   bool\n\tTabstop      int\n\tListenAddr   *listenAddress\n\tUnsafe       bool\n\tClearOnExit  bool\n\tWalkerOpts   walkerOpts\n\tWalkerRoot   string\n\tWalkerSkip   []string\n\tVersion      bool\n\tCPUProfile   string\n\tMEMProfile   string\n\tBlockProfile string\n\tMutexProfile string\n}\n\nfunc filterNonEmpty(input []string) []string {\n\toutput := make([]string, 0, len(input))\n\tfor _, str := range input {\n\t\tif len(str) > 0 {\n\t\t\toutput = append(output, str)\n\t\t}\n\t}\n\treturn output\n}\n\nfunc defaultPreviewOpts(command string) previewOpts {\n\treturn previewOpts{command, posRight, sizeSpec{50, true}, \"\", false, false, false, false, tui.DefaultBorderShape, 0, 0, nil}\n}\n\nfunc defaultOptions() *Options {\n\treturn &Options{\n\t\tBash:         false,\n\t\tZsh:          false,\n\t\tFish:         false,\n\t\tFuzzy:        true,\n\t\tFuzzyAlgo:    algo.FuzzyMatchV2,\n\t\tScheme:       \"default\",\n\t\tExtended:     true,\n\t\tPhony:        false,\n\t\tCase:         CaseSmart,\n\t\tNormalize:    true,\n\t\tNth:          make([]Range, 0),\n\t\tWithNth:      make([]Range, 0),\n\t\tDelimiter:    Delimiter{},\n\t\tSort:         1000,\n\t\tTrack:        trackDisabled,\n\t\tTac:          false,\n\t\tCriteria:     []criterion{byScore, byLength},\n\t\tMulti:        0,\n\t\tAnsi:         false,\n\t\tMouse:        true,\n\t\tTheme:        tui.EmptyTheme(),\n\t\tBlack:        false,\n\t\tBold:         true,\n\t\tMinHeight:    10,\n\t\tLayout:       layoutDefault,\n\t\tCycle:        false,\n\t\tKeepRight:    false,\n\t\tHscroll:      true,\n\t\tHscrollOff:   10,\n\t\tScrollOff:    0,\n\t\tFileWord:     false,\n\t\tInfoStyle:    infoDefault,\n\t\tSeparator:    nil,\n\t\tJumpLabels:   defaultJumpLabels,\n\t\tPrompt:       \"> \",\n\t\tPointer:      \">\",\n\t\tMarker:       \">\",\n\t\tQuery:        \"\",\n\t\tSelect1:      false,\n\t\tExit0:        false,\n\t\tFilter:       nil,\n\t\tToggleSort:   false,\n\t\tExpect:       make(map[tui.Event]string),\n\t\tKeymap:       make(map[tui.Event][]*action),\n\t\tPreview:      defaultPreviewOpts(\"\"),\n\t\tPrintQuery:   false,\n\t\tReadZero:     false,\n\t\tPrinter:      func(str string) { fmt.Println(str) },\n\t\tPrintSep:     \"\\n\",\n\t\tSync:         false,\n\t\tHistory:      nil,\n\t\tHeader:       make([]string, 0),\n\t\tHeaderLines:  0,\n\t\tHeaderFirst:  false,\n\t\tEllipsis:     \"..\",\n\t\tScrollbar:    nil,\n\t\tMargin:       defaultMargin(),\n\t\tPadding:      defaultMargin(),\n\t\tUnicode:      true,\n\t\tAmbidouble:   os.Getenv(\"RUNEWIDTH_EASTASIAN\") == \"1\",\n\t\tTabstop:      8,\n\t\tBorderLabel:  labelOpts{},\n\t\tPreviewLabel: labelOpts{},\n\t\tUnsafe:       false,\n\t\tClearOnExit:  true,\n\t\tWalkerOpts:   walkerOpts{file: true, hidden: true, follow: true},\n\t\tWalkerRoot:   \".\",\n\t\tWalkerSkip:   []string{\".git\", \"node_modules\"},\n\t\tVersion:      false}\n}\n\nfunc help(code int) {\n\tos.Stdout.WriteString(usage)\n\tutil.Exit(code)\n}\n\nvar errorContext = \"\"\n\nfunc errorExit(msg string) {\n\tos.Stderr.WriteString(errorContext + msg + \"\\n\")\n\tutil.Exit(exitError)\n}\n\nfunc optString(arg string, prefixes ...string) (bool, string) {\n\tfor _, prefix := range prefixes {\n\t\tif strings.HasPrefix(arg, prefix) {\n\t\t\treturn true, arg[len(prefix):]\n\t\t}\n\t}\n\treturn false, \"\"\n}\n\nfunc nextString(args []string, i *int, message string) string {\n\tif len(args) > *i+1 {\n\t\t*i++\n\t} else {\n\t\terrorExit(message)\n\t}\n\treturn args[*i]\n}\n\nfunc optionalNextString(args []string, i *int) (bool, string) {\n\tif len(args) > *i+1 && !strings.HasPrefix(args[*i+1], \"-\") && !strings.HasPrefix(args[*i+1], \"+\") {\n\t\t*i++\n\t\treturn true, args[*i]\n\t}\n\treturn false, \"\"\n}\n\nfunc atoi(str string) int {\n\tnum, err := strconv.Atoi(str)\n\tif err != nil {\n\t\terrorExit(\"not a valid integer: \" + str)\n\t}\n\treturn num\n}\n\nfunc atof(str string) float64 {\n\tnum, err := strconv.ParseFloat(str, 64)\n\tif err != nil {\n\t\terrorExit(\"not a valid number: \" + str)\n\t}\n\treturn num\n}\n\nfunc nextInt(args []string, i *int, message string) int {\n\tif len(args) > *i+1 {\n\t\t*i++\n\t} else {\n\t\terrorExit(message)\n\t}\n\treturn atoi(args[*i])\n}\n\nfunc optionalNumeric(args []string, i *int, defaultValue int) int {\n\tif len(args) > *i+1 {\n\t\tif strings.IndexAny(args[*i+1], \"0123456789\") == 0 {\n\t\t\t*i++\n\t\t\treturn atoi(args[*i])\n\t\t}\n\t}\n\treturn defaultValue\n}\n\nfunc splitNth(str string) []Range {\n\tif match, _ := regexp.MatchString(\"^[0-9,-.]+$\", str); !match {\n\t\terrorExit(\"invalid format: \" + str)\n\t}\n\n\ttokens := strings.Split(str, \",\")\n\tranges := make([]Range, len(tokens))\n\tfor idx, s := range tokens {\n\t\tr, ok := ParseRange(&s)\n\t\tif !ok {\n\t\t\terrorExit(\"invalid format: \" + str)\n\t\t}\n\t\tranges[idx] = r\n\t}\n\treturn ranges\n}\n\nfunc delimiterRegexp(str string) Delimiter {\n\t// Special handling of \\t\n\tstr = strings.Replace(str, \"\\\\t\", \"\\t\", -1)\n\n\t// 1. Pattern does not contain any special character\n\tif regexp.QuoteMeta(str) == str {\n\t\treturn Delimiter{str: &str}\n\t}\n\n\trx, e := regexp.Compile(str)\n\t// 2. Pattern is not a valid regular expression\n\tif e != nil {\n\t\treturn Delimiter{str: &str}\n\t}\n\n\t// 3. Pattern as regular expression. Slow.\n\treturn Delimiter{regex: rx}\n}\n\nfunc isAlphabet(char uint8) bool {\n\treturn char >= 'a' && char <= 'z'\n}\n\nfunc isNumeric(char uint8) bool {\n\treturn char >= '0' && char <= '9'\n}\n\nfunc parseAlgo(str string) algo.Algo {\n\tswitch str {\n\tcase \"v1\":\n\t\treturn algo.FuzzyMatchV1\n\tcase \"v2\":\n\t\treturn algo.FuzzyMatchV2\n\tdefault:\n\t\terrorExit(\"invalid algorithm (expected: v1 or v2)\")\n\t}\n\treturn algo.FuzzyMatchV2\n}\n\nfunc processScheme(opts *Options) {\n\tif !algo.Init(opts.Scheme) {\n\t\terrorExit(\"invalid scoring scheme (expected: default|path|history)\")\n\t}\n\tif opts.Scheme == \"history\" {\n\t\topts.Criteria = []criterion{byScore}\n\t}\n}\n\nfunc parseBorder(str string, optional bool) tui.BorderShape {\n\tswitch str {\n\tcase \"rounded\":\n\t\treturn tui.BorderRounded\n\tcase \"sharp\":\n\t\treturn tui.BorderSharp\n\tcase \"bold\":\n\t\treturn tui.BorderBold\n\tcase \"block\":\n\t\treturn tui.BorderBlock\n\tcase \"thinblock\":\n\t\treturn tui.BorderThinBlock\n\tcase \"double\":\n\t\treturn tui.BorderDouble\n\tcase \"horizontal\":\n\t\treturn tui.BorderHorizontal\n\tcase \"vertical\":\n\t\treturn tui.BorderVertical\n\tcase \"top\":\n\t\treturn tui.BorderTop\n\tcase \"bottom\":\n\t\treturn tui.BorderBottom\n\tcase \"left\":\n\t\treturn tui.BorderLeft\n\tcase \"right\":\n\t\treturn tui.BorderRight\n\tcase \"none\":\n\t\treturn tui.BorderNone\n\tdefault:\n\t\tif optional && str == \"\" {\n\t\t\treturn tui.DefaultBorderShape\n\t\t}\n\t\terrorExit(\"invalid border style (expected: rounded|sharp|bold|block|thinblock|double|horizontal|vertical|top|bottom|left|right|none)\")\n\t}\n\treturn tui.BorderNone\n}\n\nfunc parseKeyChords(str string, message string) map[tui.Event]string {\n\treturn parseKeyChordsImpl(str, message, errorExit)\n}\n\nfunc parseKeyChordsImpl(str string, message string, exit func(string)) map[tui.Event]string {\n\tif len(str) == 0 {\n\t\texit(message)\n\t\treturn nil\n\t}\n\n\tstr = regexp.MustCompile(\"(?i)(alt-),\").ReplaceAllString(str, \"$1\"+string([]rune{escapedComma}))\n\ttokens := strings.Split(str, \",\")\n\tif str == \",\" || strings.HasPrefix(str, \",,\") || strings.HasSuffix(str, \",,\") || strings.Contains(str, \",,,\") {\n\t\ttokens = append(tokens, \",\")\n\t}\n\n\tchords := make(map[tui.Event]string)\n\tfor _, key := range tokens {\n\t\tif len(key) == 0 {\n\t\t\tcontinue // ignore\n\t\t}\n\t\tkey = strings.ReplaceAll(key, string([]rune{escapedComma}), \",\")\n\t\tlkey := strings.ToLower(key)\n\t\tadd := func(e tui.EventType) {\n\t\t\tchords[e.AsEvent()] = key\n\t\t}\n\t\tswitch lkey {\n\t\tcase \"up\":\n\t\t\tadd(tui.Up)\n\t\tcase \"down\":\n\t\t\tadd(tui.Down)\n\t\tcase \"left\":\n\t\t\tadd(tui.Left)\n\t\tcase \"right\":\n\t\t\tadd(tui.Right)\n\t\tcase \"enter\", \"return\":\n\t\t\tadd(tui.CtrlM)\n\t\tcase \"space\":\n\t\t\tchords[tui.Key(' ')] = key\n\t\tcase \"backspace\", \"bspace\", \"bs\":\n\t\t\tadd(tui.Backspace)\n\t\tcase \"ctrl-space\":\n\t\t\tadd(tui.CtrlSpace)\n\t\tcase \"ctrl-delete\":\n\t\t\tadd(tui.CtrlDelete)\n\t\tcase \"ctrl-^\", \"ctrl-6\":\n\t\t\tadd(tui.CtrlCaret)\n\t\tcase \"ctrl-/\", \"ctrl-_\":\n\t\t\tadd(tui.CtrlSlash)\n\t\tcase \"ctrl-\\\\\":\n\t\t\tadd(tui.CtrlBackSlash)\n\t\tcase \"ctrl-]\":\n\t\t\tadd(tui.CtrlRightBracket)\n\t\tcase \"change\":\n\t\t\tadd(tui.Change)\n\t\tcase \"backward-eof\":\n\t\t\tadd(tui.BackwardEOF)\n\t\tcase \"start\":\n\t\t\tadd(tui.Start)\n\t\tcase \"load\":\n\t\t\tadd(tui.Load)\n\t\tcase \"focus\":\n\t\t\tadd(tui.Focus)\n\t\tcase \"result\":\n\t\t\tadd(tui.Result)\n\t\tcase \"resize\":\n\t\t\tadd(tui.Resize)\n\t\tcase \"one\":\n\t\t\tadd(tui.One)\n\t\tcase \"zero\":\n\t\t\tadd(tui.Zero)\n\t\tcase \"jump\":\n\t\t\tadd(tui.Jump)\n\t\tcase \"jump-cancel\":\n\t\t\tadd(tui.JumpCancel)\n\t\tcase \"alt-enter\", \"alt-return\":\n\t\t\tchords[tui.CtrlAltKey('m')] = key\n\t\tcase \"alt-space\":\n\t\t\tchords[tui.AltKey(' ')] = key\n\t\tcase \"alt-bs\", \"alt-bspace\", \"alt-backspace\":\n\t\t\tadd(tui.AltBackspace)\n\t\tcase \"alt-up\":\n\t\t\tadd(tui.AltUp)\n\t\tcase \"alt-down\":\n\t\t\tadd(tui.AltDown)\n\t\tcase \"alt-left\":\n\t\t\tadd(tui.AltLeft)\n\t\tcase \"alt-right\":\n\t\t\tadd(tui.AltRight)\n\t\tcase \"tab\":\n\t\t\tadd(tui.Tab)\n\t\tcase \"btab\", \"shift-tab\":\n\t\t\tadd(tui.ShiftTab)\n\t\tcase \"esc\":\n\t\t\tadd(tui.Esc)\n\t\tcase \"delete\", \"del\":\n\t\t\tadd(tui.Delete)\n\t\tcase \"home\":\n\t\t\tadd(tui.Home)\n\t\tcase \"end\":\n\t\t\tadd(tui.End)\n\t\tcase \"insert\":\n\t\t\tadd(tui.Insert)\n\t\tcase \"pgup\", \"page-up\":\n\t\t\tadd(tui.PageUp)\n\t\tcase \"pgdn\", \"page-down\":\n\t\t\tadd(tui.PageDown)\n\t\tcase \"alt-shift-up\", \"shift-alt-up\":\n\t\t\tadd(tui.AltShiftUp)\n\t\tcase \"alt-shift-down\", \"shift-alt-down\":\n\t\t\tadd(tui.AltShiftDown)\n\t\tcase \"alt-shift-left\", \"shift-alt-left\":\n\t\t\tadd(tui.AltShiftLeft)\n\t\tcase \"alt-shift-right\", \"shift-alt-right\":\n\t\t\tadd(tui.AltShiftRight)\n\t\tcase \"shift-up\":\n\t\t\tadd(tui.ShiftUp)\n\t\tcase \"shift-down\":\n\t\t\tadd(tui.ShiftDown)\n\t\tcase \"shift-left\":\n\t\t\tadd(tui.ShiftLeft)\n\t\tcase \"shift-right\":\n\t\t\tadd(tui.ShiftRight)\n\t\tcase \"shift-delete\":\n\t\t\tadd(tui.ShiftDelete)\n\t\tcase \"left-click\":\n\t\t\tadd(tui.LeftClick)\n\t\tcase \"right-click\":\n\t\t\tadd(tui.RightClick)\n\t\tcase \"shift-left-click\":\n\t\t\tadd(tui.SLeftClick)\n\t\tcase \"shift-right-click\":\n\t\t\tadd(tui.SRightClick)\n\t\tcase \"double-click\":\n\t\t\tadd(tui.DoubleClick)\n\t\tcase \"scroll-up\":\n\t\t\tadd(tui.ScrollUp)\n\t\tcase \"scroll-down\":\n\t\t\tadd(tui.ScrollDown)\n\t\tcase \"shift-scroll-up\":\n\t\t\tadd(tui.SScrollUp)\n\t\tcase \"shift-scroll-down\":\n\t\t\tadd(tui.SScrollDown)\n\t\tcase \"preview-scroll-up\":\n\t\t\tadd(tui.PreviewScrollUp)\n\t\tcase \"preview-scroll-down\":\n\t\t\tadd(tui.PreviewScrollDown)\n\t\tcase \"f10\":\n\t\t\tadd(tui.F10)\n\t\tcase \"f11\":\n\t\t\tadd(tui.F11)\n\t\tcase \"f12\":\n\t\t\tadd(tui.F12)\n\t\tdefault:\n\t\t\trunes := []rune(key)\n\t\t\tif len(key) == 10 && strings.HasPrefix(lkey, \"ctrl-alt-\") && isAlphabet(lkey[9]) {\n\t\t\t\tchords[tui.CtrlAltKey(rune(key[9]))] = key\n\t\t\t} else if len(key) == 6 && strings.HasPrefix(lkey, \"ctrl-\") && isAlphabet(lkey[5]) {\n\t\t\t\tadd(tui.EventType(tui.CtrlA.Int() + int(lkey[5]) - 'a'))\n\t\t\t} else if len(runes) == 5 && strings.HasPrefix(lkey, \"alt-\") {\n\t\t\t\tr := runes[4]\n\t\t\t\tswitch r {\n\t\t\t\tcase escapedColon:\n\t\t\t\t\tr = ':'\n\t\t\t\tcase escapedComma:\n\t\t\t\t\tr = ','\n\t\t\t\tcase escapedPlus:\n\t\t\t\t\tr = '+'\n\t\t\t\t}\n\t\t\t\tchords[tui.AltKey(r)] = key\n\t\t\t} else if len(key) == 2 && strings.HasPrefix(lkey, \"f\") && key[1] >= '1' && key[1] <= '9' {\n\t\t\t\tadd(tui.EventType(tui.F1.Int() + int(key[1]) - '1'))\n\t\t\t} else if len(runes) == 1 {\n\t\t\t\tchords[tui.Key(runes[0])] = key\n\t\t\t} else {\n\t\t\t\texit(\"unsupported key: \" + key)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n\treturn chords\n}\n\nfunc parseTiebreak(str string) []criterion {\n\tcriteria := []criterion{byScore}\n\thasIndex := false\n\thasChunk := false\n\thasLength := false\n\thasBegin := false\n\thasEnd := false\n\tcheck := func(notExpected *bool, name string) {\n\t\tif *notExpected {\n\t\t\terrorExit(\"duplicate sort criteria: \" + name)\n\t\t}\n\t\tif hasIndex {\n\t\t\terrorExit(\"index should be the last criterion\")\n\t\t}\n\t\t*notExpected = true\n\t}\n\tfor _, str := range strings.Split(strings.ToLower(str), \",\") {\n\t\tswitch str {\n\t\tcase \"index\":\n\t\t\tcheck(&hasIndex, \"index\")\n\t\tcase \"chunk\":\n\t\t\tcheck(&hasChunk, \"chunk\")\n\t\t\tcriteria = append(criteria, byChunk)\n\t\tcase \"length\":\n\t\t\tcheck(&hasLength, \"length\")\n\t\t\tcriteria = append(criteria, byLength)\n\t\tcase \"begin\":\n\t\t\tcheck(&hasBegin, \"begin\")\n\t\t\tcriteria = append(criteria, byBegin)\n\t\tcase \"end\":\n\t\t\tcheck(&hasEnd, \"end\")\n\t\t\tcriteria = append(criteria, byEnd)\n\t\tdefault:\n\t\t\terrorExit(\"invalid sort criterion: \" + str)\n\t\t}\n\t}\n\tif len(criteria) > 4 {\n\t\terrorExit(\"at most 3 tiebreaks are allowed: \" + str)\n\t}\n\treturn criteria\n}\n\nfunc dupeTheme(theme *tui.ColorTheme) *tui.ColorTheme {\n\tdupe := *theme\n\treturn &dupe\n}\n\nfunc parseTheme(defaultTheme *tui.ColorTheme, str string) *tui.ColorTheme {\n\ttheme := dupeTheme(defaultTheme)\n\trrggbb := regexp.MustCompile(\"^#[0-9a-fA-F]{6}$\")\n\tfor _, str := range strings.Split(strings.ToLower(str), \",\") {\n\t\tswitch str {\n\t\tcase \"dark\":\n\t\t\ttheme = dupeTheme(tui.Dark256)\n\t\tcase \"light\":\n\t\t\ttheme = dupeTheme(tui.Light256)\n\t\tcase \"16\":\n\t\t\ttheme = dupeTheme(tui.Default16)\n\t\tcase \"bw\", \"no\":\n\t\t\ttheme = tui.NoColorTheme()\n\t\tdefault:\n\t\t\tfail := func() {\n\t\t\t\terrorExit(\"invalid color specification: \" + str)\n\t\t\t}\n\t\t\t// Color is disabled\n\t\t\tif theme == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tcomponents := strings.Split(str, \":\")\n\t\t\tif len(components) < 2 {\n\t\t\t\tfail()\n\t\t\t}\n\n\t\t\tmergeAttr := func(cattr *tui.ColorAttr) {\n\t\t\t\tfor _, component := range components[1:] {\n\t\t\t\t\tswitch component {\n\t\t\t\t\tcase \"regular\":\n\t\t\t\t\t\tcattr.Attr = tui.AttrRegular\n\t\t\t\t\tcase \"bold\", \"strong\":\n\t\t\t\t\t\tcattr.Attr |= tui.Bold\n\t\t\t\t\tcase \"dim\":\n\t\t\t\t\t\tcattr.Attr |= tui.Dim\n\t\t\t\t\tcase \"italic\":\n\t\t\t\t\t\tcattr.Attr |= tui.Italic\n\t\t\t\t\tcase \"underline\":\n\t\t\t\t\t\tcattr.Attr |= tui.Underline\n\t\t\t\t\tcase \"blink\":\n\t\t\t\t\t\tcattr.Attr |= tui.Blink\n\t\t\t\t\tcase \"reverse\":\n\t\t\t\t\t\tcattr.Attr |= tui.Reverse\n\t\t\t\t\tcase \"strikethrough\":\n\t\t\t\t\t\tcattr.Attr |= tui.StrikeThrough\n\t\t\t\t\tcase \"black\":\n\t\t\t\t\t\tcattr.Color = tui.Color(0)\n\t\t\t\t\tcase \"red\":\n\t\t\t\t\t\tcattr.Color = tui.Color(1)\n\t\t\t\t\tcase \"green\":\n\t\t\t\t\t\tcattr.Color = tui.Color(2)\n\t\t\t\t\tcase \"yellow\":\n\t\t\t\t\t\tcattr.Color = tui.Color(3)\n\t\t\t\t\tcase \"blue\":\n\t\t\t\t\t\tcattr.Color = tui.Color(4)\n\t\t\t\t\tcase \"magenta\":\n\t\t\t\t\t\tcattr.Color = tui.Color(5)\n\t\t\t\t\tcase \"cyan\":\n\t\t\t\t\t\tcattr.Color = tui.Color(6)\n\t\t\t\t\tcase \"white\":\n\t\t\t\t\t\tcattr.Color = tui.Color(7)\n\t\t\t\t\tcase \"bright-black\", \"gray\", \"grey\":\n\t\t\t\t\t\tcattr.Color = tui.Color(8)\n\t\t\t\t\tcase \"bright-red\":\n\t\t\t\t\t\tcattr.Color = tui.Color(9)\n\t\t\t\t\tcase \"bright-green\":\n\t\t\t\t\t\tcattr.Color = tui.Color(10)\n\t\t\t\t\tcase \"bright-yellow\":\n\t\t\t\t\t\tcattr.Color = tui.Color(11)\n\t\t\t\t\tcase \"bright-blue\":\n\t\t\t\t\t\tcattr.Color = tui.Color(12)\n\t\t\t\t\tcase \"bright-magenta\":\n\t\t\t\t\t\tcattr.Color = tui.Color(13)\n\t\t\t\t\tcase \"bright-cyan\":\n\t\t\t\t\t\tcattr.Color = tui.Color(14)\n\t\t\t\t\tcase \"bright-white\":\n\t\t\t\t\t\tcattr.Color = tui.Color(15)\n\t\t\t\t\tcase \"\":\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tif rrggbb.MatchString(component) {\n\t\t\t\t\t\t\tcattr.Color = tui.HexToColor(component)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tansi32, err := strconv.Atoi(component)\n\t\t\t\t\t\t\tif err != nil || ansi32 < -1 || ansi32 > 255 {\n\t\t\t\t\t\t\t\tfail()\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcattr.Color = tui.Color(ansi32)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tswitch components[0] {\n\t\t\tcase \"query\", \"input\":\n\t\t\t\tmergeAttr(&theme.Input)\n\t\t\tcase \"disabled\":\n\t\t\t\tmergeAttr(&theme.Disabled)\n\t\t\tcase \"fg\":\n\t\t\t\tmergeAttr(&theme.Fg)\n\t\t\tcase \"bg\":\n\t\t\t\tmergeAttr(&theme.Bg)\n\t\t\tcase \"preview-fg\":\n\t\t\t\tmergeAttr(&theme.PreviewFg)\n\t\t\tcase \"preview-bg\":\n\t\t\t\tmergeAttr(&theme.PreviewBg)\n\t\t\tcase \"fg+\":\n\t\t\t\tmergeAttr(&theme.Current)\n\t\t\tcase \"bg+\":\n\t\t\t\tmergeAttr(&theme.DarkBg)\n\t\t\tcase \"gutter\":\n\t\t\t\tmergeAttr(&theme.Gutter)\n\t\t\tcase \"hl\":\n\t\t\t\tmergeAttr(&theme.Match)\n\t\t\tcase \"hl+\":\n\t\t\t\tmergeAttr(&theme.CurrentMatch)\n\t\t\tcase \"border\":\n\t\t\t\tmergeAttr(&theme.Border)\n\t\t\tcase \"preview-border\":\n\t\t\t\tmergeAttr(&theme.PreviewBorder)\n\t\t\tcase \"separator\":\n\t\t\t\tmergeAttr(&theme.Separator)\n\t\t\tcase \"scrollbar\":\n\t\t\t\tmergeAttr(&theme.Scrollbar)\n\t\t\tcase \"preview-scrollbar\":\n\t\t\t\tmergeAttr(&theme.PreviewScrollbar)\n\t\t\tcase \"label\":\n\t\t\t\tmergeAttr(&theme.BorderLabel)\n\t\t\tcase \"preview-label\":\n\t\t\t\tmergeAttr(&theme.PreviewLabel)\n\t\t\tcase \"prompt\":\n\t\t\t\tmergeAttr(&theme.Prompt)\n\t\t\tcase \"spinner\":\n\t\t\t\tmergeAttr(&theme.Spinner)\n\t\t\tcase \"info\":\n\t\t\t\tmergeAttr(&theme.Info)\n\t\t\tcase \"pointer\":\n\t\t\t\tmergeAttr(&theme.Cursor)\n\t\t\tcase \"marker\":\n\t\t\t\tmergeAttr(&theme.Selected)\n\t\t\tcase \"header\":\n\t\t\t\tmergeAttr(&theme.Header)\n\t\t\tdefault:\n\t\t\t\tfail()\n\t\t\t}\n\t\t}\n\t}\n\treturn theme\n}\n\nfunc parseWalkerOpts(str string) walkerOpts {\n\topts := walkerOpts{}\n\tfor _, str := range strings.Split(strings.ToLower(str), \",\") {\n\t\tswitch str {\n\t\tcase \"file\":\n\t\t\topts.file = true\n\t\tcase \"dir\":\n\t\t\topts.dir = true\n\t\tcase \"hidden\":\n\t\t\topts.hidden = true\n\t\tcase \"follow\":\n\t\t\topts.follow = true\n\t\tcase \"\":\n\t\t\t// Ignored\n\t\tdefault:\n\t\t\terrorExit(\"invalid walker option: \" + str)\n\t\t}\n\t}\n\tif !opts.file && !opts.dir {\n\t\terrorExit(\"at least one of 'file' or 'dir' should be specified\")\n\t}\n\treturn opts\n}\n\nvar (\n\texecuteRegexp    *regexp.Regexp\n\tsplitRegexp      *regexp.Regexp\n\tactionNameRegexp *regexp.Regexp\n)\n\nfunc firstKey(keymap map[tui.Event]string) tui.Event {\n\tfor k := range keymap {\n\t\treturn k\n\t}\n\treturn tui.EventType(0).AsEvent()\n}\n\nconst (\n\tescapedColon = 0\n\tescapedComma = 1\n\tescapedPlus  = 2\n)\n\nfunc init() {\n\texecuteRegexp = regexp.MustCompile(\n\t\t`(?si)[:+](become|execute(?:-multi|-silent)?|reload(?:-sync)?|preview|(?:change|transform)-(?:header|query|prompt|border-label|preview-label)|transform|change-preview-window|change-preview|(?:re|un)bind|pos|put)`)\n\tsplitRegexp = regexp.MustCompile(\"[,:]+\")\n\tactionNameRegexp = regexp.MustCompile(\"(?i)^[a-z-]+\")\n}\n\nfunc maskActionContents(action string) string {\n\tmasked := \"\"\nLoop:\n\tfor len(action) > 0 {\n\t\tloc := executeRegexp.FindStringIndex(action)\n\t\tif loc == nil {\n\t\t\tmasked += action\n\t\t\tbreak\n\t\t}\n\t\tmasked += action[:loc[1]]\n\t\taction = action[loc[1]:]\n\t\tif len(action) == 0 {\n\t\t\tbreak\n\t\t}\n\t\tcs := string(action[0])\n\t\tce := \")\"\n\t\tswitch action[0] {\n\t\tcase ':':\n\t\t\tmasked += strings.Repeat(\" \", len(action))\n\t\t\tbreak Loop\n\t\tcase '(':\n\t\t\tce = \")\"\n\t\tcase '{':\n\t\t\tce = \"}\"\n\t\tcase '[':\n\t\t\tce = \"]\"\n\t\tcase '<':\n\t\t\tce = \">\"\n\t\tcase '~', '!', '@', '#', '$', '%', '^', '&', '*', ';', '/', '|':\n\t\t\tce = string(cs)\n\t\tdefault:\n\t\t\tcontinue\n\t\t}\n\t\tcs = regexp.QuoteMeta(cs)\n\t\tce = regexp.QuoteMeta(ce)\n\n\t\t// @$ or @+\n\t\tloc = regexp.MustCompile(fmt.Sprintf(`(?s)^%s.*?(%s[+,]|%s$)`, cs, ce, ce)).FindStringIndex(action)\n\t\tif loc == nil {\n\t\t\tmasked += action\n\t\t\tbreak\n\t\t}\n\t\t// Keep + or , at the end\n\t\tlastChar := action[loc[1]-1]\n\t\tif lastChar == '+' || lastChar == ',' {\n\t\t\tloc[1]--\n\t\t}\n\t\tmasked += strings.Repeat(\" \", loc[1])\n\t\taction = action[loc[1]:]\n\t}\n\tmasked = strings.Replace(masked, \"::\", string([]rune{escapedColon, ':'}), -1)\n\tmasked = strings.Replace(masked, \",:\", string([]rune{escapedComma, ':'}), -1)\n\tmasked = strings.Replace(masked, \"+:\", string([]rune{escapedPlus, ':'}), -1)\n\treturn masked\n}\n\nfunc parseSingleActionList(str string, exit func(string)) []*action {\n\t// We prepend a colon to satisfy executeRegexp and remove it later\n\tmasked := maskActionContents(\":\" + str)[1:]\n\treturn parseActionList(masked, str, []*action{}, false, exit)\n}\n\nfunc parseActionList(masked string, original string, prevActions []*action, putAllowed bool, exit func(string)) []*action {\n\tmaskedStrings := strings.Split(masked, \"+\")\n\toriginalStrings := make([]string, len(maskedStrings))\n\tidx := 0\n\tfor i, maskedString := range maskedStrings {\n\t\toriginalStrings[i] = original[idx : idx+len(maskedString)]\n\t\tidx += len(maskedString) + 1\n\t}\n\tactions := make([]*action, 0, len(maskedStrings))\n\tappendAction := func(types ...actionType) {\n\t\tactions = append(actions, toActions(types...)...)\n\t}\n\tprevSpec := \"\"\n\tfor specIndex, spec := range originalStrings {\n\t\tspec = prevSpec + spec\n\t\tspecLower := strings.ToLower(spec)\n\t\tswitch specLower {\n\t\tcase \"ignore\":\n\t\t\tappendAction(actIgnore)\n\t\tcase \"beginning-of-line\":\n\t\t\tappendAction(actBeginningOfLine)\n\t\tcase \"abort\":\n\t\t\tappendAction(actAbort)\n\t\tcase \"accept\":\n\t\t\tappendAction(actAccept)\n\t\tcase \"accept-non-empty\":\n\t\t\tappendAction(actAcceptNonEmpty)\n\t\tcase \"accept-or-print-query\":\n\t\t\tappendAction(actAcceptOrPrintQuery)\n\t\tcase \"print-query\":\n\t\t\tappendAction(actPrintQuery)\n\t\tcase \"refresh-preview\":\n\t\t\tappendAction(actRefreshPreview)\n\t\tcase \"replace-query\":\n\t\t\tappendAction(actReplaceQuery)\n\t\tcase \"backward-char\":\n\t\t\tappendAction(actBackwardChar)\n\t\tcase \"backward-delete-char\":\n\t\t\tappendAction(actBackwardDeleteChar)\n\t\tcase \"backward-delete-char/eof\":\n\t\t\tappendAction(actBackwardDeleteCharEof)\n\t\tcase \"backward-word\":\n\t\t\tappendAction(actBackwardWord)\n\t\tcase \"clear-screen\":\n\t\t\tappendAction(actClearScreen)\n\t\tcase \"delete-char\":\n\t\t\tappendAction(actDeleteChar)\n\t\tcase \"delete-char/eof\":\n\t\t\tappendAction(actDeleteCharEof)\n\t\tcase \"deselect\":\n\t\t\tappendAction(actDeselect)\n\t\tcase \"end-of-line\":\n\t\t\tappendAction(actEndOfLine)\n\t\tcase \"cancel\":\n\t\t\tappendAction(actCancel)\n\t\tcase \"clear-query\":\n\t\t\tappendAction(actClearQuery)\n\t\tcase \"clear-selection\":\n\t\t\tappendAction(actClearSelection)\n\t\tcase \"forward-char\":\n\t\t\tappendAction(actForwardChar)\n\t\tcase \"forward-word\":\n\t\t\tappendAction(actForwardWord)\n\t\tcase \"jump\":\n\t\t\tappendAction(actJump)\n\t\tcase \"jump-accept\":\n\t\t\tappendAction(actJumpAccept)\n\t\tcase \"kill-line\":\n\t\t\tappendAction(actKillLine)\n\t\tcase \"kill-word\":\n\t\t\tappendAction(actKillWord)\n\t\tcase \"unix-line-discard\", \"line-discard\":\n\t\t\tappendAction(actUnixLineDiscard)\n\t\tcase \"unix-word-rubout\", \"word-rubout\":\n\t\t\tappendAction(actUnixWordRubout)\n\t\tcase \"yank\":\n\t\t\tappendAction(actYank)\n\t\tcase \"backward-kill-word\":\n\t\t\tappendAction(actBackwardKillWord)\n\t\tcase \"toggle-down\":\n\t\t\tappendAction(actToggle, actDown)\n\t\tcase \"toggle-up\":\n\t\t\tappendAction(actToggle, actUp)\n\t\tcase \"toggle-in\":\n\t\t\tappendAction(actToggleIn)\n\t\tcase \"toggle-out\":\n\t\t\tappendAction(actToggleOut)\n\t\tcase \"toggle-all\":\n\t\t\tappendAction(actToggleAll)\n\t\tcase \"toggle-search\":\n\t\t\tappendAction(actToggleSearch)\n\t\tcase \"toggle-track\":\n\t\t\tappendAction(actToggleTrack)\n\t\tcase \"toggle-track-current\":\n\t\t\tappendAction(actToggleTrackCurrent)\n\t\tcase \"toggle-header\":\n\t\t\tappendAction(actToggleHeader)\n\t\tcase \"show-header\":\n\t\t\tappendAction(actShowHeader)\n\t\tcase \"hide-header\":\n\t\t\tappendAction(actHideHeader)\n\t\tcase \"track\", \"track-current\":\n\t\t\tappendAction(actTrackCurrent)\n\t\tcase \"untrack-current\":\n\t\t\tappendAction(actUntrackCurrent)\n\t\tcase \"select\":\n\t\t\tappendAction(actSelect)\n\t\tcase \"select-all\":\n\t\t\tappendAction(actSelectAll)\n\t\tcase \"deselect-all\":\n\t\t\tappendAction(actDeselectAll)\n\t\tcase \"close\":\n\t\t\tappendAction(actClose)\n\t\tcase \"toggle\":\n\t\t\tappendAction(actToggle)\n\t\tcase \"down\":\n\t\t\tappendAction(actDown)\n\t\tcase \"up\":\n\t\t\tappendAction(actUp)\n\t\tcase \"first\", \"top\":\n\t\t\tappendAction(actFirst)\n\t\tcase \"last\":\n\t\t\tappendAction(actLast)\n\t\tcase \"page-up\":\n\t\t\tappendAction(actPageUp)\n\t\tcase \"page-down\":\n\t\t\tappendAction(actPageDown)\n\t\tcase \"half-page-up\":\n\t\t\tappendAction(actHalfPageUp)\n\t\tcase \"half-page-down\":\n\t\t\tappendAction(actHalfPageDown)\n\t\tcase \"prev-history\", \"previous-history\":\n\t\t\tappendAction(actPrevHistory)\n\t\tcase \"next-history\":\n\t\t\tappendAction(actNextHistory)\n\t\tcase \"prev-selected\":\n\t\t\tappendAction(actPrevSelected)\n\t\tcase \"next-selected\":\n\t\t\tappendAction(actNextSelected)\n\t\tcase \"show-preview\":\n\t\t\tappendAction(actShowPreview)\n\t\tcase \"hide-preview\":\n\t\t\tappendAction(actHidePreview)\n\t\tcase \"toggle-preview\":\n\t\t\tappendAction(actTogglePreview)\n\t\tcase \"toggle-preview-wrap\":\n\t\t\tappendAction(actTogglePreviewWrap)\n\t\tcase \"toggle-sort\":\n\t\t\tappendAction(actToggleSort)\n\t\tcase \"offset-up\":\n\t\t\tappendAction(actOffsetUp)\n\t\tcase \"offset-down\":\n\t\t\tappendAction(actOffsetDown)\n\t\tcase \"preview-top\":\n\t\t\tappendAction(actPreviewTop)\n\t\tcase \"preview-bottom\":\n\t\t\tappendAction(actPreviewBottom)\n\t\tcase \"preview-up\":\n\t\t\tappendAction(actPreviewUp)\n\t\tcase \"preview-down\":\n\t\t\tappendAction(actPreviewDown)\n\t\tcase \"preview-page-up\":\n\t\t\tappendAction(actPreviewPageUp)\n\t\tcase \"preview-page-down\":\n\t\t\tappendAction(actPreviewPageDown)\n\t\tcase \"preview-half-page-up\":\n\t\t\tappendAction(actPreviewHalfPageUp)\n\t\tcase \"preview-half-page-down\":\n\t\t\tappendAction(actPreviewHalfPageDown)\n\t\tcase \"enable-search\":\n\t\t\tappendAction(actEnableSearch)\n\t\tcase \"disable-search\":\n\t\t\tappendAction(actDisableSearch)\n\t\tcase \"put\":\n\t\t\tif putAllowed {\n\t\t\t\tappendAction(actChar)\n\t\t\t} else {\n\t\t\t\texit(\"unable to put non-printable character\")\n\t\t\t}\n\t\tdefault:\n\t\t\tt := isExecuteAction(specLower)\n\t\t\tif t == actIgnore {\n\t\t\t\tif specIndex == 0 && specLower == \"\" {\n\t\t\t\t\tactions = append(prevActions, actions...)\n\t\t\t\t} else {\n\t\t\t\t\texit(\"unknown action: \" + spec)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\toffset := len(actionNameRegexp.FindString(spec))\n\t\t\t\tvar actionArg string\n\t\t\t\tif spec[offset] == ':' {\n\t\t\t\t\tif specIndex == len(originalStrings)-1 {\n\t\t\t\t\t\tactionArg = spec[offset+1:]\n\t\t\t\t\t\tactions = append(actions, &action{t: t, a: actionArg})\n\t\t\t\t\t} else {\n\t\t\t\t\t\tprevSpec = spec + \"+\"\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tactionArg = spec[offset+1 : len(spec)-1]\n\t\t\t\t\tactions = append(actions, &action{t: t, a: actionArg})\n\t\t\t\t}\n\t\t\t\tswitch t {\n\t\t\t\tcase actBecome:\n\t\t\t\t\tif util.IsWindows() {\n\t\t\t\t\t\texit(\"become action is not supported on Windows\")\n\t\t\t\t\t}\n\t\t\t\tcase actUnbind, actRebind:\n\t\t\t\t\tparseKeyChordsImpl(actionArg, spec[0:offset]+\" target required\", exit)\n\t\t\t\tcase actChangePreviewWindow:\n\t\t\t\t\topts := previewOpts{}\n\t\t\t\t\tfor _, arg := range strings.Split(actionArg, \"|\") {\n\t\t\t\t\t\t// Make sure that each expression is valid\n\t\t\t\t\t\tparsePreviewWindowImpl(&opts, arg, exit)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tprevSpec = \"\"\n\t}\n\treturn actions\n}\n\nfunc parseKeymap(keymap map[tui.Event][]*action, str string, exit func(string)) {\n\tmasked := maskActionContents(str)\n\tidx := 0\n\tfor _, pairStr := range strings.Split(masked, \",\") {\n\t\torigPairStr := str[idx : idx+len(pairStr)]\n\t\tidx += len(pairStr) + 1\n\n\t\tpair := strings.SplitN(pairStr, \":\", 2)\n\t\tif len(pair) < 2 {\n\t\t\texit(\"bind action not specified: \" + origPairStr)\n\t\t}\n\t\tvar key tui.Event\n\t\tif len(pair[0]) == 1 && pair[0][0] == escapedColon {\n\t\t\tkey = tui.Key(':')\n\t\t} else if len(pair[0]) == 1 && pair[0][0] == escapedComma {\n\t\t\tkey = tui.Key(',')\n\t\t} else if len(pair[0]) == 1 && pair[0][0] == escapedPlus {\n\t\t\tkey = tui.Key('+')\n\t\t} else {\n\t\t\tkeys := parseKeyChordsImpl(pair[0], \"key name required\", exit)\n\t\t\tkey = firstKey(keys)\n\t\t}\n\t\tputAllowed := key.Type == tui.Rune && unicode.IsGraphic(key.Char)\n\t\tkeymap[key] = parseActionList(pair[1], origPairStr[len(pair[0])+1:], keymap[key], putAllowed, exit)\n\t}\n}\n\nfunc isExecuteAction(str string) actionType {\n\tmasked := maskActionContents(\":\" + str)[1:]\n\tif masked == str {\n\t\t// Not masked\n\t\treturn actIgnore\n\t}\n\n\tprefix := actionNameRegexp.FindString(str)\n\tswitch prefix {\n\tcase \"become\":\n\t\treturn actBecome\n\tcase \"reload\":\n\t\treturn actReload\n\tcase \"reload-sync\":\n\t\treturn actReloadSync\n\tcase \"unbind\":\n\t\treturn actUnbind\n\tcase \"rebind\":\n\t\treturn actRebind\n\tcase \"preview\":\n\t\treturn actPreview\n\tcase \"change-border-label\":\n\t\treturn actChangeBorderLabel\n\tcase \"change-header\":\n\t\treturn actChangeHeader\n\tcase \"change-preview-label\":\n\t\treturn actChangePreviewLabel\n\tcase \"change-preview-window\":\n\t\treturn actChangePreviewWindow\n\tcase \"change-preview\":\n\t\treturn actChangePreview\n\tcase \"change-prompt\":\n\t\treturn actChangePrompt\n\tcase \"change-query\":\n\t\treturn actChangeQuery\n\tcase \"pos\":\n\t\treturn actPosition\n\tcase \"execute\":\n\t\treturn actExecute\n\tcase \"execute-silent\":\n\t\treturn actExecuteSilent\n\tcase \"execute-multi\":\n\t\treturn actExecuteMulti\n\tcase \"put\":\n\t\treturn actPut\n\tcase \"transform\":\n\t\treturn actTransform\n\tcase \"transform-border-label\":\n\t\treturn actTransformBorderLabel\n\tcase \"transform-preview-label\":\n\t\treturn actTransformPreviewLabel\n\tcase \"transform-header\":\n\t\treturn actTransformHeader\n\tcase \"transform-prompt\":\n\t\treturn actTransformPrompt\n\tcase \"transform-query\":\n\t\treturn actTransformQuery\n\t}\n\treturn actIgnore\n}\n\nfunc parseToggleSort(keymap map[tui.Event][]*action, str string) {\n\tkeys := parseKeyChords(str, \"key name required\")\n\tif len(keys) != 1 {\n\t\terrorExit(\"multiple keys specified\")\n\t}\n\tkeymap[firstKey(keys)] = toActions(actToggleSort)\n}\n\nfunc strLines(str string) []string {\n\treturn strings.Split(strings.TrimSuffix(str, \"\\n\"), \"\\n\")\n}\n\nfunc parseSize(str string, maxPercent float64, label string) sizeSpec {\n\tvar val float64\n\tpercent := strings.HasSuffix(str, \"%\")\n\tif percent {\n\t\tval = atof(str[:len(str)-1])\n\t\tif val < 0 {\n\t\t\terrorExit(label + \" must be non-negative\")\n\t\t}\n\t\tif val > maxPercent {\n\t\t\terrorExit(fmt.Sprintf(\"%s too large (max: %d%%)\", label, int(maxPercent)))\n\t\t}\n\t} else {\n\t\tif strings.Contains(str, \".\") {\n\t\t\terrorExit(label + \" (without %) must be a non-negative integer\")\n\t\t}\n\n\t\tval = float64(atoi(str))\n\t\tif val < 0 {\n\t\t\terrorExit(label + \" must be non-negative\")\n\t\t}\n\t}\n\treturn sizeSpec{val, percent}\n}\n\nfunc parseHeight(str string) heightSpec {\n\theightSpec := heightSpec{}\n\tif strings.HasPrefix(str, \"~\") {\n\t\theightSpec.auto = true\n\t\tstr = str[1:]\n\t}\n\tif strings.HasPrefix(str, \"-\") {\n\t\tif heightSpec.auto {\n\t\t\terrorExit(\"negative(-) height is not compatible with adaptive(~) height\")\n\t\t}\n\t\theightSpec.inverse = true\n\t\tstr = str[1:]\n\t}\n\n\tsize := parseSize(str, 100, \"height\")\n\theightSpec.size = size.size\n\theightSpec.percent = size.percent\n\treturn heightSpec\n}\n\nfunc parseLayout(str string) layoutType {\n\tswitch str {\n\tcase \"default\":\n\t\treturn layoutDefault\n\tcase \"reverse\":\n\t\treturn layoutReverse\n\tcase \"reverse-list\":\n\t\treturn layoutReverseList\n\tdefault:\n\t\terrorExit(\"invalid layout (expected: default / reverse / reverse-list)\")\n\t}\n\treturn layoutDefault\n}\n\nfunc parseInfoStyle(str string) (infoStyle, string) {\n\tswitch str {\n\tcase \"default\":\n\t\treturn infoDefault, \"\"\n\tcase \"right\":\n\t\treturn infoRight, \"\"\n\tcase \"inline\":\n\t\treturn infoInline, defaultInfoPrefix\n\tcase \"inline-right\":\n\t\treturn infoInlineRight, \"\"\n\tcase \"hidden\":\n\t\treturn infoHidden, \"\"\n\tdefault:\n\t\ttype infoSpec struct {\n\t\t\tname  string\n\t\t\tstyle infoStyle\n\t\t}\n\t\tfor _, spec := range []infoSpec{\n\t\t\t{\"inline\", infoInline},\n\t\t\t{\"inline-right\", infoInlineRight}} {\n\t\t\tif strings.HasPrefix(str, spec.name+\":\") {\n\t\t\t\treturn spec.style, strings.ReplaceAll(str[len(spec.name)+1:], \"\\n\", \" \")\n\t\t\t}\n\t\t}\n\t\terrorExit(\"invalid info style (expected: default|right|hidden|inline[-right][:PREFIX])\")\n\t}\n\treturn infoDefault, \"\"\n}\n\nfunc parsePreviewWindow(opts *previewOpts, input string) {\n\tparsePreviewWindowImpl(opts, input, errorExit)\n}\n\nfunc parsePreviewWindowImpl(opts *previewOpts, input string, exit func(string)) {\n\ttokenRegex := regexp.MustCompile(`[:,]*(<([1-9][0-9]*)\\(([^)<]+)\\)|[^,:]+)`)\n\tsizeRegex := regexp.MustCompile(\"^[0-9]+%?$\")\n\toffsetRegex := regexp.MustCompile(`^(\\+{-?[0-9]+})?([+-][0-9]+)*(-?/[1-9][0-9]*)?$`)\n\theaderRegex := regexp.MustCompile(\"^~(0|[1-9][0-9]*)$\")\n\ttokens := tokenRegex.FindAllStringSubmatch(input, -1)\n\tvar alternative string\n\tfor _, match := range tokens {\n\t\tif len(match[2]) > 0 {\n\t\t\topts.threshold = atoi(match[2])\n\t\t\talternative = match[3]\n\t\t\tcontinue\n\t\t}\n\t\ttoken := match[1]\n\t\tswitch token {\n\t\tcase \"\":\n\t\tcase \"default\":\n\t\t\t*opts = defaultPreviewOpts(opts.command)\n\t\tcase \"hidden\":\n\t\t\topts.hidden = true\n\t\tcase \"nohidden\":\n\t\t\topts.hidden = false\n\t\tcase \"wrap\":\n\t\t\topts.wrap = true\n\t\tcase \"nowrap\":\n\t\t\topts.wrap = false\n\t\tcase \"cycle\":\n\t\t\topts.cycle = true\n\t\tcase \"nocycle\":\n\t\t\topts.cycle = false\n\t\tcase \"up\", \"top\":\n\t\t\topts.position = posUp\n\t\tcase \"down\", \"bottom\":\n\t\t\topts.position = posDown\n\t\tcase \"left\":\n\t\t\topts.position = posLeft\n\t\tcase \"right\":\n\t\t\topts.position = posRight\n\t\tcase \"rounded\", \"border\", \"border-rounded\":\n\t\t\topts.border = tui.BorderRounded\n\t\tcase \"sharp\", \"border-sharp\":\n\t\t\topts.border = tui.BorderSharp\n\t\tcase \"border-bold\":\n\t\t\topts.border = tui.BorderBold\n\t\tcase \"border-block\":\n\t\t\topts.border = tui.BorderBlock\n\t\tcase \"border-thinblock\":\n\t\t\topts.border = tui.BorderThinBlock\n\t\tcase \"border-double\":\n\t\t\topts.border = tui.BorderDouble\n\t\tcase \"noborder\", \"border-none\":\n\t\t\topts.border = tui.BorderNone\n\t\tcase \"border-horizontal\":\n\t\t\topts.border = tui.BorderHorizontal\n\t\tcase \"border-vertical\":\n\t\t\topts.border = tui.BorderVertical\n\t\tcase \"border-up\", \"border-top\":\n\t\t\topts.border = tui.BorderTop\n\t\tcase \"border-down\", \"border-bottom\":\n\t\t\topts.border = tui.BorderBottom\n\t\tcase \"border-left\":\n\t\t\topts.border = tui.BorderLeft\n\t\tcase \"border-right\":\n\t\t\topts.border = tui.BorderRight\n\t\tcase \"follow\":\n\t\t\topts.follow = true\n\t\tcase \"nofollow\":\n\t\t\topts.follow = false\n\t\tdefault:\n\t\t\tif headerRegex.MatchString(token) {\n\t\t\t\topts.headerLines = atoi(token[1:])\n\t\t\t} else if sizeRegex.MatchString(token) {\n\t\t\t\topts.size = parseSize(token, 99, \"window size\")\n\t\t\t} else if offsetRegex.MatchString(token) {\n\t\t\t\topts.scroll = token\n\t\t\t} else {\n\t\t\t\texit(\"invalid preview window option: \" + token)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\tif len(alternative) > 0 {\n\t\talternativeOpts := *opts\n\t\topts.alternative = &alternativeOpts\n\t\topts.alternative.hidden = false\n\t\topts.alternative.alternative = nil\n\t\tparsePreviewWindowImpl(opts.alternative, alternative, exit)\n\t}\n}\n\nfunc parseMargin(opt string, margin string) [4]sizeSpec {\n\tmargins := strings.Split(margin, \",\")\n\tchecked := func(str string) sizeSpec {\n\t\treturn parseSize(str, 49, opt)\n\t}\n\tswitch len(margins) {\n\tcase 1:\n\t\tm := checked(margins[0])\n\t\treturn [4]sizeSpec{m, m, m, m}\n\tcase 2:\n...\n", "needle_token_start": 671, "needle_token_end": 807, "code_context_ntokens": 15032, "output": ["```go\nfunc parseLabelPosition(opts *labelOpts, arg string) {\n\topts.column = 0\n\topts.bottom = false\n\tfor _, token := range splitRegexp.Split(strings.ToLower(arg), -1) {\n\t\tswitch token {\n\t\tcase \"center\":\n\t\t\topts.column = 0\n\t\tcase \"bottom\":\n\t\t\topts.bottom = true\n\t\tcase \"top\":\n\t\t\topts.bottom = false\n\t\tdefault:\n\t\t\topts.column = atoi(token)\n\t\t}\n\t}\n}\n```"]}
{"repo": "junegunn/fzf", "name": "clearPatternCache", "language": "go", "path": "src/pattern.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to reset and initialize the storage used for caching search patterns, ensuring that any previously stored patterns are cleared and the cache is ready for new entries.\n2. **Input**: This function does not take any inputs.\n3. **Output**: There is no direct output from this function; however, it affects the state of the internal cache by resetting it.\n4. **Procedure**: The function initializes the internal cache as an empty map, effectively removing all previously stored data and preparing it for new caching operations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/result.go\npackage fzf\n\nimport (\n\t\"math\"\n\t\"sort\"\n\t\"unicode\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// Offset holds two 32-bit integers denoting the offsets of a matched substring\ntype Offset [2]int32\n\ntype colorOffset struct {\n\toffset [2]int32\n\tcolor  tui.ColorPair\n}\n\ntype Result struct {\n\titem   *Item\n\tpoints [4]uint16\n}\n\nfunc buildResult(item *Item, offsets []Offset, score int) Result {\n\tif len(offsets) > 1 {\n\t\tsort.Sort(ByOrder(offsets))\n\t}\n\n\tresult := Result{item: item}\n\tnumChars := item.text.Length()\n\tminBegin := math.MaxUint16\n\tminEnd := math.MaxUint16\n\tmaxEnd := 0\n\tvalidOffsetFound := false\n\tfor _, offset := range offsets {\n\t\tb, e := int(offset[0]), int(offset[1])\n\t\tif b < e {\n\t\t\tminBegin = util.Min(b, minBegin)\n\t\t\tminEnd = util.Min(e, minEnd)\n\t\t\tmaxEnd = util.Max(e, maxEnd)\n\t\t\tvalidOffsetFound = true\n\t\t}\n\t}\n\n\tfor idx, criterion := range sortCriteria {\n\t\tval := uint16(math.MaxUint16)\n\t\tswitch criterion {\n\t\tcase byScore:\n\t\t\t// Higher is better\n\t\t\tval = math.MaxUint16 - util.AsUint16(score)\n\t\tcase byChunk:\n\t\t\tif validOffsetFound {\n\t\t\t\tb := minBegin\n\t\t\t\te := maxEnd\n\t\t\t\tfor ; b >= 1; b-- {\n\t\t\t\t\tif unicode.IsSpace(item.text.Get(b - 1)) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor ; e < numChars; e++ {\n\t\t\t\t\tif unicode.IsSpace(item.text.Get(e)) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tval = util.AsUint16(e - b)\n\t\t\t}\n\t\tcase byLength:\n\t\t\tval = item.TrimLength()\n\t\tcase byBegin, byEnd:\n\t\t\tif validOffsetFound {\n\t\t\t\twhitePrefixLen := 0\n\t\t\t\tfor idx := 0; idx < numChars; idx++ {\n\t\t\t\t\tr := item.text.Get(idx)\n\t\t\t\t\twhitePrefixLen = idx\n\t\t\t\t\tif idx == minBegin || !unicode.IsSpace(r) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif criterion == byBegin {\n\t\t\t\t\tval = util.AsUint16(minEnd - whitePrefixLen)\n\t\t\t\t} else {\n\t\t\t\t\tval = util.AsUint16(math.MaxUint16 - math.MaxUint16*(maxEnd-whitePrefixLen)/int(item.TrimLength()+1))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tresult.points[3-idx] = val\n\t}\n\n\treturn result\n}\n\n// Sort criteria to use. Never changes once fzf is started.\nvar sortCriteria []criterion\n\n// Index returns ordinal index of the Item\nfunc (result *Result) Index() int32 {\n\treturn result.item.Index()\n}\n\nfunc minRank() Result {\n\treturn Result{item: &minItem, points: [4]uint16{math.MaxUint16, 0, 0, 0}}\n}\n\nfunc (result *Result) colorOffsets(matchOffsets []Offset, theme *tui.ColorTheme, colBase tui.ColorPair, colMatch tui.ColorPair, current bool) []colorOffset {\n\titemColors := result.item.Colors()\n\n\t// No ANSI codes\n\tif len(itemColors) == 0 {\n\t\tvar offsets []colorOffset\n\t\tfor _, off := range matchOffsets {\n\t\t\toffsets = append(offsets, colorOffset{offset: [2]int32{off[0], off[1]}, color: colMatch})\n\t\t}\n\t\treturn offsets\n\t}\n\n\t// Find max column\n\tvar maxCol int32\n\tfor _, off := range matchOffsets {\n\t\tif off[1] > maxCol {\n\t\t\tmaxCol = off[1]\n\t\t}\n\t}\n\tfor _, ansi := range itemColors {\n\t\tif ansi.offset[1] > maxCol {\n\t\t\tmaxCol = ansi.offset[1]\n\t\t}\n\t}\n\n\tcols := make([]int, maxCol)\n\tfor colorIndex, ansi := range itemColors {\n\t\tfor i := ansi.offset[0]; i < ansi.offset[1]; i++ {\n\t\t\tcols[i] = colorIndex + 1 // 1-based index of itemColors\n\t\t}\n\t}\n\n\tfor _, off := range matchOffsets {\n\t\tfor i := off[0]; i < off[1]; i++ {\n\t\t\t// Negative of 1-based index of itemColors\n\t\t\t// - The extra -1 means highlighted\n\t\t\tif cols[i] >= 0 {\n\t\t\t\tcols[i] = cols[i]*-1 - 1\n\t\t\t}\n\t\t}\n\t}\n\n\t// sort.Sort(ByOrder(offsets))\n\n\t// Merge offsets\n\t// ------------  ----  --  ----\n\t//   ++++++++      ++++++++++\n\t// --++++++++--  --++++++++++---\n\tcurr := 0\n\tstart := 0\n\tansiToColorPair := func(ansi ansiOffset, base tui.ColorPair) tui.ColorPair {\n\t\tfg := ansi.color.fg\n\t\tbg := ansi.color.bg\n\t\tif fg == -1 {\n\t\t\tif current {\n\t\t\t\tfg = theme.Current.Color\n\t\t\t} else {\n\t\t\t\tfg = theme.Fg.Color\n\t\t\t}\n\t\t}\n\t\tif bg == -1 {\n\t\t\tif current {\n\t\t\t\tbg = theme.DarkBg.Color\n\t\t\t} else {\n\t\t\t\tbg = theme.Bg.Color\n\t\t\t}\n\t\t}\n...\n// Path: src/pattern.go\npackage fzf\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/junegunn/fzf/src/algo\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// fuzzy\n// 'exact\n// ^prefix-exact\n// suffix-exact$\n// !inverse-exact\n// !'inverse-fuzzy\n// !^inverse-prefix-exact\n// !inverse-suffix-exact$\n\ntype termType int\n\nconst (\n\ttermFuzzy termType = iota\n\ttermExact\n\ttermPrefix\n\ttermSuffix\n\ttermEqual\n)\n\ntype term struct {\n\ttyp           termType\n\tinv           bool\n\ttext          []rune\n\tcaseSensitive bool\n\tnormalize     bool\n}\n\n// String returns the string representation of a term.\nfunc (t term) String() string {\n\treturn fmt.Sprintf(\"term{typ: %d, inv: %v, text: []rune(%q), caseSensitive: %v}\", t.typ, t.inv, string(t.text), t.caseSensitive)\n}\n\ntype termSet []term\n\n// Pattern represents search pattern\ntype Pattern struct {\n\tfuzzy         bool\n\tfuzzyAlgo     algo.Algo\n\textended      bool\n\tcaseSensitive bool\n\tnormalize     bool\n\tforward       bool\n\twithPos       bool\n\ttext          []rune\n\ttermSets      []termSet\n\tsortable      bool\n\tcacheable     bool\n\tcacheKey      string\n\tdelimiter     Delimiter\n\tnth           []Range\n\tprocFun       map[termType]algo.Algo\n}\n\nvar (\n\t_patternCache map[string]*Pattern\n\t_splitRegex   *regexp.Regexp\n\t_cache        ChunkCache\n)\n\nfunc init() {\n\t_splitRegex = regexp.MustCompile(\" +\")\n\tclearPatternCache()\n\tclearChunkCache()\n}\n\n\nfunc clearPatternCache() {\n\t// We can uniquely identify the pattern for a given string since\n\t// search mode and caseMode do not change while the program is running\n\t_patternCache = make(map[string]*Pattern)\n}\n\nfunc clearChunkCache() {\n\t_cache = NewChunkCache()\n}\n\n// BuildPattern builds Pattern object from the given arguments\nfunc BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, normalize bool, forward bool,\n\twithPos bool, cacheable bool, nth []Range, delimiter Delimiter, runes []rune) *Pattern {\n\n\tvar asString string\n\tif extended {\n\t\tasString = strings.TrimLeft(string(runes), \" \")\n\t\tfor strings.HasSuffix(asString, \" \") && !strings.HasSuffix(asString, \"\\\\ \") {\n\t\t\tasString = asString[:len(asString)-1]\n\t\t}\n\t} else {\n\t\tasString = string(runes)\n\t}\n\n\tcached, found := _patternCache[asString]\n\tif found {\n\t\treturn cached\n\t}\n\n\tcaseSensitive := true\n\tsortable := true\n\ttermSets := []termSet{}\n\n\tif extended {\n\t\ttermSets = parseTerms(fuzzy, caseMode, normalize, asString)\n\t\t// We should not sort the result if there are only inverse search terms\n\t\tsortable = false\n\tLoop:\n\t\tfor _, termSet := range termSets {\n\t\t\tfor idx, term := range termSet {\n\t\t\t\tif !term.inv {\n\t\t\t\t\tsortable = true\n\t\t\t\t}\n\t\t\t\t// If the query contains inverse search terms or OR operators,\n\t\t\t\t// we cannot cache the search scope\n\t\t\t\tif !cacheable || idx > 0 || term.inv || fuzzy && term.typ != termFuzzy || !fuzzy && term.typ != termExact {\n\t\t\t\t\tcacheable = false\n\t\t\t\t\tif sortable {\n\t\t\t\t\t\t// Can't break until we see at least one non-inverse term\n\t\t\t\t\t\tbreak Loop\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlowerString := strings.ToLower(asString)\n\t\tnormalize = normalize &&\n\t\t\tlowerString == string(algo.NormalizeRunes([]rune(lowerString)))\n\t\tcaseSensitive = caseMode == CaseRespect ||\n\t\t\tcaseMode == CaseSmart && lowerString != asString\n\t\tif !caseSensitive {\n\t\t\tasString = lowerString\n\t\t}\n\t}\n\n\tptr := &Pattern{\n\t\tfuzzy:         fuzzy,\n\t\tfuzzyAlgo:     fuzzyAlgo,\n\t\textended:      extended,\n\t\tcaseSensitive: caseSensitive,\n\t\tnormalize:     normalize,\n\t\tforward:       forward,\n\t\twithPos:       withPos,\n\t\ttext:          []rune(asString),\n\t\ttermSets:      termSets,\n\t\tsortable:      sortable,\n\t\tcacheable:     cacheable,\n\t\tnth:           nth,\n\t\tdelimiter:     delimiter,\n\t\tprocFun:       make(map[termType]algo.Algo)}\n\n\tptr.cacheKey = ptr.buildCacheKey()\n\tptr.procFun[termFuzzy] = fuzzyAlgo\n\tptr.procFun[termEqual] = algo.EqualMatch\n\tptr.procFun[termExact] = algo.ExactMatchNaive\n\tptr.procFun[termPrefix] = algo.PrefixMatch\n\tptr.procFun[termSuffix] = algo.SuffixMatch\n\n\t_patternCache[asString] = ptr\n\treturn ptr\n}\n\nfunc parseTerms(fuzzy bool, caseMode Case, normalize bool, str string) []termSet {\n\tstr = strings.Replace(str, \"\\\\ \", \"\\t\", -1)\n\ttokens := _splitRegex.Split(str, -1)\n\tsets := []termSet{}\n\tset := termSet{}\n\tswitchSet := false\n\tafterBar := false\n\tfor _, token := range tokens {\n\t\ttyp, inv, text := termFuzzy, false, strings.Replace(token, \"\\t\", \" \", -1)\n\t\tlowerText := strings.ToLower(text)\n\t\tcaseSensitive := caseMode == CaseRespect ||\n\t\t\tcaseMode == CaseSmart && text != lowerText\n\t\tnormalizeTerm := normalize &&\n\t\t\tlowerText == string(algo.NormalizeRunes([]rune(lowerText)))\n\t\tif !caseSensitive {\n\t\t\ttext = lowerText\n\t\t}\n\t\tif !fuzzy {\n\t\t\ttyp = termExact\n\t\t}\n\n\t\tif len(set) > 0 && !afterBar && text == \"|\" {\n\t\t\tswitchSet = false\n\t\t\tafterBar = true\n\t\t\tcontinue\n\t\t}\n\t\tafterBar = false\n\n\t\tif strings.HasPrefix(text, \"!\") {\n\t\t\tinv = true\n\t\t\ttyp = termExact\n\t\t\ttext = text[1:]\n\t\t}\n\n\t\tif text != \"$\" && strings.HasSuffix(text, \"$\") {\n\t\t\ttyp = termSuffix\n\t\t\ttext = text[:len(text)-1]\n\t\t}\n\n\t\tif strings.HasPrefix(text, \"'\") {\n\t\t\t// Flip exactness\n\t\t\tif fuzzy && !inv {\n\t\t\t\ttyp = termExact\n\t\t\t} else {\n\t\t\t\ttyp = termFuzzy\n\t\t\t}\n\t\t\ttext = text[1:]\n\t\t} else if strings.HasPrefix(text, \"^\") {\n\t\t\tif typ == termSuffix {\n\t\t\t\ttyp = termEqual\n\t\t\t} else {\n\t\t\t\ttyp = termPrefix\n\t\t\t}\n\t\t\ttext = text[1:]\n\t\t}\n\n\t\tif len(text) > 0 {\n\t\t\tif switchSet {\n\t\t\t\tsets = append(sets, set)\n\t\t\t\tset = termSet{}\n\t\t\t}\n\t\t\ttextRunes := []rune(text)\n\t\t\tif normalizeTerm {\n\t\t\t\ttextRunes = algo.NormalizeRunes(textRunes)\n\t\t\t}\n\t\t\tset = append(set, term{\n\t\t\t\ttyp:           typ,\n\t\t\t\tinv:           inv,\n\t\t\t\ttext:          textRunes,\n\t\t\t\tcaseSensitive: caseSensitive,\n\t\t\t\tnormalize:     normalizeTerm})\n\t\t\tswitchSet = true\n\t\t}\n\t}\n\tif len(set) > 0 {\n\t\tsets = append(sets, set)\n\t}\n\treturn sets\n}\n\n// IsEmpty returns true if the pattern is effectively empty\nfunc (p *Pattern) IsEmpty() bool {\n\tif !p.extended {\n\t\treturn len(p.text) == 0\n\t}\n\treturn len(p.termSets) == 0\n}\n\n// AsString returns the search query in string type\nfunc (p *Pattern) AsString() string {\n\treturn string(p.text)\n}\n\nfunc (p *Pattern) buildCacheKey() string {\n\tif !p.extended {\n\t\treturn p.AsString()\n\t}\n\tcacheableTerms := []string{}\n\tfor _, termSet := range p.termSets {\n\t\tif len(termSet) == 1 && !termSet[0].inv && (p.fuzzy || termSet[0].typ == termExact) {\n\t\t\tcacheableTerms = append(cacheableTerms, string(termSet[0].text))\n\t\t}\n\t}\n\treturn strings.Join(cacheableTerms, \"\\t\")\n}\n\n// CacheKey is used to build string to be used as the key of result cache\nfunc (p *Pattern) CacheKey() string {\n\treturn p.cacheKey\n}\n\n// Match returns the list of matches Items in the given Chunk\nfunc (p *Pattern) Match(chunk *Chunk, slab *util.Slab) []Result {\n\t// ChunkCache: Exact match\n\tcacheKey := p.CacheKey()\n\tif p.cacheable {\n\t\tif cached := _cache.Lookup(chunk, cacheKey); cached != nil {\n\t\t\treturn cached\n\t\t}\n\t}\n\n\t// Prefix/suffix cache\n\tspace := _cache.Search(chunk, cacheKey)\n\n\tmatches := p.matchChunk(chunk, space, slab)\n\n\tif p.cacheable {\n\t\t_cache.Add(chunk, cacheKey, matches)\n\t}\n\treturn matches\n}\n\nfunc (p *Pattern) matchChunk(chunk *Chunk, space []Result, slab *util.Slab) []Result {\n\tmatches := []Result{}\n\n\tif space == nil {\n\t\tfor idx := 0; idx < chunk.count; idx++ {\n\t\t\tif match, _, _ := p.MatchItem(&chunk.items[idx], p.withPos, slab); match != nil {\n\t\t\t\tmatches = append(matches, *match)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tfor _, result := range space {\n\t\t\tif match, _, _ := p.MatchItem(result.item, p.withPos, slab); match != nil {\n\t\t\t\tmatches = append(matches, *match)\n\t\t\t}\n\t\t}\n\t}\n\treturn matches\n}\n\n// MatchItem returns true if the Item is a match\nfunc (p *Pattern) MatchItem(item *Item, withPos bool, slab *util.Slab) (*Result, []Offset, *[]int) {\n\tif p.extended {\n\t\tif offsets, bonus, pos := p.extendedMatch(item, withPos, slab); len(offsets) == len(p.termSets) {\n\t\t\tresult := buildResult(item, offsets, bonus)\n\t\t\treturn &result, offsets, pos\n\t\t}\n\t\treturn nil, nil, nil\n\t}\n\toffset, bonus, pos := p.basicMatch(item, withPos, slab)\n\tif sidx := offset[0]; sidx >= 0 {\n\t\toffsets := []Offset{offset}\n\t\tresult := buildResult(item, offsets, bonus)\n\t\treturn &result, offsets, pos\n\t}\n\treturn nil, nil, nil\n}\n\nfunc (p *Pattern) basicMatch(item *Item, withPos bool, slab *util.Slab) (Offset, int, *[]int) {\n\tvar input []Token\n\tif len(p.nth) == 0 {\n\t\tinput = []Token{{text: &item.text, prefixLength: 0}}\n\t} else {\n\t\tinput = p.transformInput(item)\n\t}\n\tif p.fuzzy {\n\t\treturn p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)\n\t}\n\treturn p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)\n}\n\nfunc (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Offset, int, *[]int) {\n\tvar input []Token\n\tif len(p.nth) == 0 {\n\t\tinput = []Token{{text: &item.text, prefixLength: 0}}\n\t} else {\n\t\tinput = p.transformInput(item)\n\t}\n\toffsets := []Offset{}\n\tvar totalScore int\n\tvar allPos *[]int\n\tif withPos {\n\t\tallPos = &[]int{}\n\t}\n\tfor _, termSet := range p.termSets {\n\t\tvar offset Offset\n\t\tvar currentScore int\n\t\tmatched := false\n\t\tfor _, term := range termSet {\n\t\t\tpfun := p.procFun[term.typ]\n\t\t\toff, score, pos := p.iter(pfun, input, term.caseSensitive, term.normalize, p.forward, term.text, withPos, slab)\n\t\t\tif sidx := off[0]; sidx >= 0 {\n\t\t\t\tif term.inv {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\toffset, currentScore = off, score\n\t\t\t\tmatched = true\n\t\t\t\tif withPos {\n\t\t\t\t\tif pos != nil {\n\t\t\t\t\t\t*allPos = append(*allPos, *pos...)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfor idx := off[0]; idx < off[1]; idx++ {\n\t\t\t\t\t\t\t*allPos = append(*allPos, int(idx))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t} else if term.inv {\n\t\t\t\toffset, currentScore = Offset{0, 0}, 0\n\t\t\t\tmatched = true\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif matched {\n\t\t\toffsets = append(offsets, offset)\n\t\t\ttotalScore += currentScore\n\t\t}\n\t}\n\treturn offsets, totalScore, allPos\n}\n\nfunc (p *Pattern) transformInput(item *Item) []Token {\n\tif item.transformed != nil {\n\t\treturn *item.transformed\n\t}\n\n\ttokens := Tokenize(item.text.ToString(), p.delimiter)\n\tret := Transform(tokens, p.nth)\n\titem.transformed = &ret\n\treturn ret\n}\n\nfunc (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, normalize bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, *[]int) {\n\tfor _, part := range tokens {\n\t\tif res, pos := pfun(caseSensitive, normalize, forward, part.text, pattern, withPos, slab); res.Start >= 0 {\n\t\t\tsidx := int32(res.Start) + part.prefixLength\n\t\t\teidx := int32(res.End) + part.prefixLength\n\t\t\tif pos != nil {\n\t\t\t\tfor idx := range *pos {\n\t\t\t\t\t(*pos)[idx] += int(part.prefixLength)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn Offset{sidx, eidx}, res.Score, pos\n\t\t}\n\t}\n\treturn Offset{-1, -1}, 0, nil\n}\n\n// Path: src/merger.go\npackage fzf\n\nimport \"fmt\"\n\n// EmptyMerger is a Merger with no data\nfunc EmptyMerger(revision int) *Merger {\n\treturn NewMerger(nil, [][]Result{}, false, false, revision)\n}\n\n// Merger holds a set of locally sorted lists of items and provides the view of\n// a single, globally-sorted list\ntype Merger struct {\n\tpattern  *Pattern\n\tlists    [][]Result\n\tmerged   []Result\n\tchunks   *[]*Chunk\n\tcursors  []int\n\tsorted   bool\n\ttac      bool\n\tfinal    bool\n\tcount    int\n\tpass     bool\n\trevision int\n}\n\n// PassMerger returns a new Merger that simply returns the items in the\n// original order\nfunc PassMerger(chunks *[]*Chunk, tac bool, revision int) *Merger {\n\tmg := Merger{\n\t\tpattern:  nil,\n\t\tchunks:   chunks,\n\t\ttac:      tac,\n\t\tcount:    0,\n\t\tpass:     true,\n\t\trevision: revision}\n\n\tfor _, chunk := range *mg.chunks {\n\t\tmg.count += chunk.count\n\t}\n\treturn &mg\n}\n\n// NewMerger returns a new Merger\nfunc NewMerger(pattern *Pattern, lists [][]Result, sorted bool, tac bool, revision int) *Merger {\n\tmg := Merger{\n\t\tpattern:  pattern,\n\t\tlists:    lists,\n\t\tmerged:   []Result{},\n\t\tchunks:   nil,\n\t\tcursors:  make([]int, len(lists)),\n\t\tsorted:   sorted,\n\t\ttac:      tac,\n\t\tfinal:    false,\n\t\tcount:    0,\n\t\trevision: revision}\n\n\tfor _, list := range mg.lists {\n\t\tmg.count += len(list)\n\t}\n\treturn &mg\n}\n\n// Revision returns revision number\nfunc (mg *Merger) Revision() int {\n\treturn mg.revision\n}\n\n// Length returns the number of items\nfunc (mg *Merger) Length() int {\n\treturn mg.count\n}\n\nfunc (mg *Merger) First() Result {\n\tif mg.tac && !mg.sorted {\n\t\treturn mg.Get(mg.count - 1)\n\t}\n\treturn mg.Get(0)\n}\n\n// FindIndex returns the index of the item with the given item index\nfunc (mg *Merger) FindIndex(itemIndex int32) int {\n\tindex := -1\n\tif mg.pass {\n\t\tindex = int(itemIndex)\n\t\tif mg.tac {\n\t\t\tindex = mg.count - index - 1\n\t\t}\n\t} else {\n\t\tfor i := 0; i < mg.count; i++ {\n\t\t\tif mg.Get(i).item.Index() == itemIndex {\n\t\t\t\tindex = i\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn index\n}\n\n// Get returns the pointer to the Result object indexed by the given integer\nfunc (mg *Merger) Get(idx int) Result {\n\tif mg.chunks != nil {\n\t\tif mg.tac {\n\t\t\tidx = mg.count - idx - 1\n\t\t}\n\t\tchunk := (*mg.chunks)[idx/chunkSize]\n\t\treturn Result{item: &chunk.items[idx%chunkSize]}\n\t}\n\n\tif mg.sorted {\n\t\treturn mg.mergedGet(idx)\n\t}\n\n\tif mg.tac {\n\t\tidx = mg.count - idx - 1\n\t}\n\tfor _, list := range mg.lists {\n\t\tnumItems := len(list)\n\t\tif idx < numItems {\n\t\t\treturn list[idx]\n\t\t}\n\t\tidx -= numItems\n\t}\n\tpanic(fmt.Sprintf(\"Index out of bounds (unsorted, %d/%d)\", idx, mg.count))\n}\n\nfunc (mg *Merger) cacheable() bool {\n\treturn mg.count < mergerCacheMax\n}\n\nfunc (mg *Merger) mergedGet(idx int) Result {\n\tfor i := len(mg.merged); i <= idx; i++ {\n\t\tminRank := minRank()\n\t\tminIdx := -1\n\t\tfor listIdx, list := range mg.lists {\n\t\t\tcursor := mg.cursors[listIdx]\n\t\t\tif cursor < 0 || cursor == len(list) {\n\t\t\t\tmg.cursors[listIdx] = -1\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif cursor >= 0 {\n\t\t\t\trank := list[cursor]\n\t\t\t\tif minIdx < 0 || compareRanks(rank, minRank, mg.tac) {\n\t\t\t\t\tminRank = rank\n\t\t\t\t\tminIdx = listIdx\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif minIdx >= 0 {\n\t\t\tchosen := mg.lists[minIdx]\n\t\t\tmg.merged = append(mg.merged, chosen[mg.cursors[minIdx]])\n\t\t\tmg.cursors[minIdx]++\n\t\t} else {\n\t\t\tpanic(fmt.Sprintf(\"Index out of bounds (sorted, %d/%d)\", i, mg.count))\n\t\t}\n\t}\n\treturn mg.merged[idx]\n}\n\n// Path: src/reader.go\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/charlievieth/fastwalk\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// Reader reads from command or standard input\ntype Reader struct {\n\tpusher   func([]byte) bool\n\teventBox *util.EventBox\n\tdelimNil bool\n\tevent    int32\n\tfinChan  chan bool\n\tmutex    sync.Mutex\n\texec     *exec.Cmd\n\tcommand  *string\n\tkilled   bool\n\twait     bool\n}\n\n// NewReader returns new Reader object\nfunc NewReader(pusher func([]byte) bool, eventBox *util.EventBox, delimNil bool, wait bool) *Reader {\n\treturn &Reader{pusher, eventBox, delimNil, int32(EvtReady), make(chan bool, 1), sync.Mutex{}, nil, nil, false, wait}\n}\n\nfunc (r *Reader) startEventPoller() {\n\tgo func() {\n\t\tptr := &r.event\n\t\tpollInterval := readerPollIntervalMin\n\t\tfor {\n\t\t\tif atomic.CompareAndSwapInt32(ptr, int32(EvtReadNew), int32(EvtReady)) {\n\t\t\t\tr.eventBox.Set(EvtReadNew, (*string)(nil))\n\t\t\t\tpollInterval = readerPollIntervalMin\n\t\t\t} else if atomic.LoadInt32(ptr) == int32(EvtReadFin) {\n\t\t\t\tif r.wait {\n\t\t\t\t\tr.finChan <- true\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tpollInterval += readerPollIntervalStep\n\t\t\t\tif pollInterval > readerPollIntervalMax {\n\t\t\t\t\tpollInterval = readerPollIntervalMax\n\t\t\t\t}\n\t\t\t}\n\t\t\ttime.Sleep(pollInterval)\n\t\t}\n\t}()\n}\n\nfunc (r *Reader) fin(success bool) {\n\tatomic.StoreInt32(&r.event, int32(EvtReadFin))\n\tif r.wait {\n\t\t<-r.finChan\n\t}\n\n\tr.mutex.Lock()\n\tret := r.command\n\tif success || r.killed {\n\t\tret = nil\n\t}\n\tr.mutex.Unlock()\n\n\tr.eventBox.Set(EvtReadFin, ret)\n}\n\nfunc (r *Reader) terminate() {\n\tr.mutex.Lock()\n\tr.killed = true\n\tif r.exec != nil && r.exec.Process != nil {\n\t\tutil.KillCommand(r.exec)\n\t} else {\n\t\tos.Stdin.Close()\n\t}\n\tr.mutex.Unlock()\n}\n\nfunc (r *Reader) restart(command string, environ []string) {\n\tr.event = int32(EvtReady)\n\tr.startEventPoller()\n\tsuccess := r.readFromCommand(command, environ)\n\tr.fin(success)\n}\n\n// ReadSource reads data from the default command or from standard input\nfunc (r *Reader) ReadSource(root string, opts walkerOpts, ignores []string) {\n\tr.startEventPoller()\n\tvar success bool\n\tif util.IsTty() {\n\t\tcmd := os.Getenv(\"FZF_DEFAULT_COMMAND\")\n\t\tif len(cmd) == 0 {\n\t\t\tsuccess = r.readFiles(root, opts, ignores)\n\t\t} else {\n\t\t\t// We can't export FZF_* environment variables to the default command\n\t\t\tsuccess = r.readFromCommand(cmd, nil)\n\t\t}\n\t} else {\n\t\tsuccess = r.readFromStdin()\n\t}\n\tr.fin(success)\n}\n\nfunc (r *Reader) feed(src io.Reader) {\n\t/*\n\t\treaderSlabSize, ae := strconv.Atoi(os.Getenv(\"SLAB_KB\"))\n\t\tif ae != nil {\n\t\t\treaderSlabSize = 128 * 1024\n\t\t} else {\n\t\t\treaderSlabSize *= 1024\n\t\t}\n\t\treaderBufferSize, be := strconv.Atoi(os.Getenv(\"BUF_KB\"))\n\t\tif be != nil {\n\t\t\treaderBufferSize = 64 * 1024\n\t\t} else {\n\t\t\treaderBufferSize *= 1024\n\t\t}\n\t*/\n\n\tdelim := byte('\\n')\n\ttrimCR := util.IsWindows()\n\tif r.delimNil {\n\t\tdelim = '\\000'\n\t\ttrimCR = false\n\t}\n\n\tslab := make([]byte, readerSlabSize)\n\tleftover := []byte{}\n\tvar err error\n\tfor {\n\t\tn := 0\n\t\tscope := slab[:util.Min(len(slab), readerBufferSize)]\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tn, err = src.Read(scope)\n\t\t\tif n > 0 || err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// We're not making any progress after 100 tries. Stop.\n\t\tif n == 0 && err == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tbuf := slab[:n]\n\t\tslab = slab[n:]\n\n\t\tfor len(buf) > 0 {\n\t\t\tif i := bytes.IndexByte(buf, delim); i >= 0 {\n\t\t\t\t// Found the delimiter\n\t\t\t\tslice := buf[:i+1]\n\t\t\t\tbuf = buf[i+1:]\n\t\t\t\tif trimCR && len(slice) >= 2 && slice[len(slice)-2] == byte('\\r') {\n\t\t\t\t\tslice = slice[:len(slice)-2]\n\t\t\t\t} else {\n\t\t\t\t\tslice = slice[:len(slice)-1]\n\t\t\t\t}\n\t\t\t\tif len(leftover) > 0 {\n\t\t\t\t\tslice = append(leftover, slice...)\n\t\t\t\t\tleftover = []byte{}\n\t\t\t\t}\n\t\t\t\tif (err == nil || len(slice) > 0) && r.pusher(slice) {\n\t\t\t\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Could not find the delimiter in the buffer\n\t\t\t\t//   NOTE: We can further optimize this by keeping track of the cursor\n\t\t\t\t//   position in the slab so that a straddling item that doesn't go\n\t\t\t\t//   beyond the boundary of a slab doesn't need to be copied to\n\t\t\t\t//   another buffer. However, the performance gain is negligible in\n\t\t\t\t//   practice (< 0.1%) and is not\n\t\t\t\t//   worth the added complexity.\n\t\t\t\tleftover = append(leftover, buf...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif err == io.EOF {\n\t\t\tleftover = append(leftover, buf...)\n\t\t\tbreak\n\t\t}\n\n\t\tif len(slab) == 0 {\n\t\t\tslab = make([]byte, readerSlabSize)\n\t\t}\n\t}\n\tif len(leftover) > 0 && r.pusher(leftover) {\n\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t}\n}\n\nfunc (r *Reader) readFromStdin() bool {\n\tr.feed(os.Stdin)\n\treturn true\n}\n\nfunc (r *Reader) readFiles(root string, opts walkerOpts, ignores []string) bool {\n\tr.killed = false\n\tconf := fastwalk.Config{Follow: opts.follow}\n\tfn := func(path string, de os.DirEntry, err error) error {\n\t\tif err != nil {\n\t\t\treturn nil\n\t\t}\n\t\tpath = filepath.Clean(path)\n\t\tif path != \".\" {\n\t\t\tisDir := de.IsDir()\n\t\t\tif isDir {\n\t\t\t\tbase := filepath.Base(path)\n\t\t\t\tif !opts.hidden && base[0] == '.' {\n\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t}\n\t\t\t\tfor _, ignore := range ignores {\n\t\t\t\t\tif ignore == base {\n\t\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif ((opts.file && !isDir) || (opts.dir && isDir)) && r.pusher([]byte(path)) {\n\t\t\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t\t\t}\n\t\t}\n\t\tr.mutex.Lock()\n\t\tdefer r.mutex.Unlock()\n\t\tif r.killed {\n\t\t\treturn context.Canceled\n\t\t}\n\t\treturn nil\n\t}\n\treturn fastwalk.Walk(&conf, root, fn) == nil\n}\n\nfunc (r *Reader) readFromCommand(command string, environ []string) bool {\n\tr.mutex.Lock()\n\tr.killed = false\n\tr.command = &command\n\tr.exec = util.ExecCommand(command, true)\n\tif environ != nil {\n\t\tr.exec.Env = environ\n\t}\n\tout, err := r.exec.StdoutPipe()\n\tif err != nil {\n\t\tr.mutex.Unlock()\n\t\treturn false\n\t}\n\terr = r.exec.Start()\n\tr.mutex.Unlock()\n\tif err != nil {\n\t\treturn false\n\t}\n\tr.feed(out)\n\treturn r.exec.Wait() == nil\n}\n\n// Path: src/terminal.go\npackage fzf\n\nimport (\n\t\"bufio\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/rivo/uniseg\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// import \"github.com/pkg/profile\"\n\n/*\nPlaceholder regex is used to extract placeholders from fzf's template\nstrings. Acts as input validation for parsePlaceholder function.\nDescribes the syntax, but it is fairly lenient.\n\nThe following pseudo regex has been reverse engineered from the\nimplementation. It is overly strict, but better describes what's possible.\nAs such it is not useful for validation, but rather to generate test\ncases for example.\n\n\t\\\\?(?:                                      # escaped type\n\t    {\\+?s?f?RANGE(?:,RANGE)*}               # token type\n\t    |{q}                                    # query type\n\t    |{\\+?n?f?}                              # item type (notice no mandatory element inside brackets)\n\t)\n\tRANGE = (?:\n\t    (?:-?[0-9]+)?\\.\\.(?:-?[0-9]+)?          # ellipsis syntax for token range (x..y)\n\t    |-?[0-9]+                               # shorthand syntax (x..x)\n\t)\n*/\nvar placeholder *regexp.Regexp\nvar whiteSuffix *regexp.Regexp\nvar offsetComponentRegex *regexp.Regexp\nvar offsetTrimCharsRegex *regexp.Regexp\nvar activeTempFiles []string\nvar passThroughRegex *regexp.Regexp\nvar actionTypeRegex *regexp.Regexp\n\nconst clearCode string = \"\\x1b[2J\"\n\n// Number of maximum focus events to process synchronously\nconst maxFocusEvents = 10000\n\nfunc init() {\n\tplaceholder = regexp.MustCompile(`\\\\?(?:{[+sf]*[0-9,-.]*}|{q}|{fzf:(?:query|action|prompt)}|{\\+?f?nf?})`)\n\twhiteSuffix = regexp.MustCompile(`\\s*$`)\n\toffsetComponentRegex = regexp.MustCompile(`([+-][0-9]+)|(-?/[1-9][0-9]*)`)\n\toffsetTrimCharsRegex = regexp.MustCompile(`[^0-9/+-]`)\n\tactiveTempFiles = []string{}\n\n\t// Parts of the preview output that should be passed through to the terminal\n\t// * https://github.com/tmux/tmux/wiki/FAQ#what-is-the-passthrough-escape-sequence-and-how-do-i-use-it\n\t// * https://sw.kovidgoyal.net/kitty/graphics-protocol\n\t// * https://en.wikipedia.org/wiki/Sixel\n\t// * https://iterm2.com/documentation-images.html\n\tpassThroughRegex = regexp.MustCompile(`\\x1bPtmux;\\x1b\\x1b.*?[^\\x1b]\\x1b\\\\|\\x1b(_G|P[0-9;]*q).*?\\x1b\\\\\\r?|\\x1b]1337;.*?(\\a|\\x1b\\\\)`)\n}\n\ntype jumpMode int\n\nconst (\n\tjumpDisabled jumpMode = iota\n\tjumpEnabled\n\tjumpAcceptEnabled\n)\n\ntype resumableState int\n\nconst (\n\tdisabledState resumableState = iota\n\tpausedState\n\tenabledState\n)\n\nfunc (s resumableState) Enabled() bool {\n\treturn s == enabledState\n}\n\nfunc (s *resumableState) Force(flag bool) {\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = disabledState\n\t}\n}\n\nfunc (s *resumableState) Set(flag bool) {\n\tif *s == disabledState {\n\t\treturn\n\t}\n\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = pausedState\n\t}\n}\n\ntype previewer struct {\n\tversion    int64\n\tlines      []string\n\toffset     int\n\tscrollable bool\n\tfinal      bool\n\tfollowing  resumableState\n\tspinner    string\n\tbar        []bool\n}\n\ntype previewed struct {\n\tversion   int64\n\tnumLines  int\n\toffset    int\n\tfilled    bool\n\timage     bool\n\twipe      bool\n\twireframe bool\n}\n\ntype eachLine struct {\n\tline string\n\terr  error\n}\n\ntype itemLine struct {\n\toffset   int\n\tcurrent  bool\n\tselected bool\n\tlabel    string\n\tqueryLen int\n\twidth    int\n\tbar      bool\n\tresult   Result\n}\n\ntype fitpad struct {\n\tfit int\n\tpad int\n}\n\nvar emptyLine = itemLine{}\n\ntype labelPrinter func(tui.Window, int)\n\ntype StatusItem struct {\n\tIndex int    `json:\"index\"`\n\tText  string `json:\"text\"`\n}\n\ntype Status struct {\n\tReading    bool         `json:\"reading\"`\n\tProgress   int          `json:\"progress\"`\n\tQuery      string       `json:\"query\"`\n\tPosition   int          `json:\"position\"`\n\tSort       bool         `json:\"sort\"`\n\tTotalCount int          `json:\"totalCount\"`\n\tMatchCount int          `json:\"matchCount\"`\n\tCurrent    *StatusItem  `json:\"current\"`\n\tMatches    []StatusItem `json:\"matches\"`\n\tSelected   []StatusItem `json:\"selected\"`\n}\n\n// Terminal represents terminal input/output\ntype Terminal struct {\n\tinitDelay          time.Duration\n\tinfoStyle          infoStyle\n\tinfoPrefix         string\n\tseparator          labelPrinter\n\tseparatorLen       int\n\tspinner            []string\n\tpromptString       string\n\tprompt             func()\n\tpromptLen          int\n\tborderLabel        labelPrinter\n\tborderLabelLen     int\n\tborderLabelOpts    labelOpts\n\tpreviewLabel       labelPrinter\n\tpreviewLabelLen    int\n\tpreviewLabelOpts   labelOpts\n\tpointer            string\n\tpointerLen         int\n\tpointerEmpty       string\n\tmarker             string\n\tmarkerLen          int\n\tmarkerEmpty        string\n\tqueryLen           [2]int\n\tlayout             layoutType\n\tfullscreen         bool\n\tkeepRight          bool\n\thscroll            bool\n\thscrollOff         int\n\tscrollOff          int\n\twordRubout         string\n\twordNext           string\n\tcx                 int\n\tcy                 int\n\toffset             int\n\txoffset            int\n\tyanked             []rune\n\tinput              []rune\n\tmulti              int\n\tsort               bool\n\ttoggleSort         bool\n\ttrack              trackOption\n\tdelimiter          Delimiter\n\texpect             map[tui.Event]string\n\tkeymap             map[tui.Event][]*action\n\tkeymapOrg          map[tui.Event][]*action\n\tpressed            string\n\tprintQuery         bool\n\thistory            *History\n\tcycle              bool\n\theaderVisible      bool\n\theaderFirst        bool\n\theaderLines        int\n\theader             []string\n\theader0            []string\n\tellipsis           string\n\tscrollbar          string\n\tpreviewScrollbar   string\n\tansi               bool\n\ttabstop            int\n\tmargin             [4]sizeSpec\n\tpadding            [4]sizeSpec\n\tunicode            bool\n\tlistenAddr         *listenAddress\n\tlistenPort         *int\n\tlistenUnsafe       bool\n\tborderShape        tui.BorderShape\n\tcleanExit          bool\n\tpaused             bool\n\tborder             tui.Window\n\twindow             tui.Window\n\tpborder            tui.Window\n\tpwindow            tui.Window\n\tborderWidth        int\n\tcount              int\n\tprogress           int\n\thasResultActions   bool\n\thasFocusActions    bool\n\thasLoadActions     bool\n\thasResizeActions   bool\n\ttriggerLoad        bool\n\treading            bool\n\trunning            bool\n\tfailed             *string\n\tjumping            jumpMode\n\tjumpLabels         string\n\tprinter            func(string)\n\tprintsep           string\n\tmerger             *Merger\n\tselected           map[int32]selectedItem\n\tversion            int64\n\trevision           int\n\treqBox             *util.EventBox\n\tinitialPreviewOpts previewOpts\n\tpreviewOpts        previewOpts\n\tactivePreviewOpts  *previewOpts\n\tpreviewer          previewer\n\tpreviewed          previewed\n\tpreviewBox         *util.EventBox\n\teventBox           *util.EventBox\n\tmutex              sync.Mutex\n\tinitFunc           func()\n\tprevLines          []itemLine\n\tsuppress           bool\n\tsigstop            bool\n\tstartChan          chan fitpad\n\tkillChan           chan int\n\tserverInputChan    chan []*action\n\tserverOutputChan   chan string\n\teventChan          chan tui.Event\n\tslab               *util.Slab\n\ttheme              *tui.ColorTheme\n\ttui                tui.Renderer\n\texecuting          *util.AtomicBool\n\ttermSize           tui.TermSize\n\tlastAction         actionType\n\tlastKey            string\n\tlastFocus          int32\n\tareaLines          int\n\tareaColumns        int\n\tforcePreview       bool\n}\n\ntype selectedItem struct {\n\tat   time.Time\n\titem *Item\n}\n\ntype byTimeOrder []selectedItem\n\nfunc (a byTimeOrder) Len() int {\n\treturn len(a)\n}\n\nfunc (a byTimeOrder) Swap(i, j int) {\n\ta[i], a[j] = a[j], a[i]\n}\n\nfunc (a byTimeOrder) Less(i, j int) bool {\n\treturn a[i].at.Before(a[j].at)\n}\n\nconst (\n\treqPrompt util.EventType = iota\n\treqInfo\n\treqHeader\n\treqList\n\treqJump\n\treqRefresh\n\treqReinit\n\treqFullRedraw\n\treqResize\n\treqRedrawBorderLabel\n\treqRedrawPreviewLabel\n\treqClose\n\treqPrintQuery\n\treqPreviewEnqueue\n\treqPreviewDisplay\n\treqPreviewRefresh\n\treqPreviewDelayed\n\treqQuit\n)\n\ntype action struct {\n\tt actionType\n\ta string\n}\n\n//go:generate stringer -type=actionType\ntype actionType int\n\nconst (\n\tactIgnore actionType = iota\n\tactStart\n\tactClick\n\tactInvalid\n\tactChar\n\tactMouse\n\tactBeginningOfLine\n\tactAbort\n\tactAccept\n\tactAcceptNonEmpty\n\tactAcceptOrPrintQuery\n\tactBackwardChar\n\tactBackwardDeleteChar\n\tactBackwardDeleteCharEof\n\tactBackwardWord\n\tactCancel\n\tactChangeBorderLabel\n\tactChangeHeader\n\tactChangePreviewLabel\n\tactChangePrompt\n\tactChangeQuery\n\tactClearScreen\n\tactClearQuery\n\tactClearSelection\n\tactClose\n\tactDeleteChar\n\tactDeleteCharEof\n\tactEndOfLine\n\tactForwardChar\n\tactForwardWord\n\tactKillLine\n\tactKillWord\n\tactUnixLineDiscard\n\tactUnixWordRubout\n\tactYank\n\tactBackwardKillWord\n\tactSelectAll\n\tactDeselectAll\n\tactToggle\n\tactToggleSearch\n\tactToggleAll\n\tactToggleDown\n\tactToggleUp\n\tactToggleIn\n\tactToggleOut\n\tactToggleTrack\n\tactToggleTrackCurrent\n\tactToggleHeader\n\tactTrackCurrent\n\tactUntrackCurrent\n\tactDown\n\tactUp\n\tactPageUp\n\tactPageDown\n\tactPosition\n\tactHalfPageUp\n\tactHalfPageDown\n\tactOffsetUp\n\tactOffsetDown\n\tactJump\n\tactJumpAccept // XXX Deprecated in favor of jump:accept binding\n\tactPrintQuery\n\tactRefreshPreview\n\tactReplaceQuery\n\tactToggleSort\n\tactShowPreview\n\tactHidePreview\n\tactTogglePreview\n\tactTogglePreviewWrap\n\tactTransform\n\tactTransformBorderLabel\n\tactTransformHeader\n\tactTransformPreviewLabel\n\tactTransformPrompt\n\tactTransformQuery\n\tactPreview\n\tactChangePreview\n\tactChangePreviewWindow\n\tactPreviewTop\n\tactPreviewBottom\n\tactPreviewUp\n\tactPreviewDown\n\tactPreviewPageUp\n\tactPreviewPageDown\n\tactPreviewHalfPageUp\n\tactPreviewHalfPageDown\n\tactPrevHistory\n\tactPrevSelected\n\tactPut\n\tactNextHistory\n\tactNextSelected\n\tactExecute\n\tactExecuteSilent\n\tactExecuteMulti // Deprecated\n\tactSigStop\n\tactFirst\n\tactLast\n\tactReload\n\tactReloadSync\n\tactDisableSearch\n\tactEnableSearch\n\tactSelect\n\tactDeselect\n\tactUnbind\n\tactRebind\n\tactBecome\n\tactResponse\n\tactShowHeader\n\tactHideHeader\n)\n\nfunc (a actionType) Name() string {\n\treturn util.ToKebabCase(a.String()[3:])\n}\n\nfunc processExecution(action actionType) bool {\n\tswitch action {\n\tcase actTransform,\n\t\tactTransformBorderLabel,\n\t\tactTransformHeader,\n\t\tactTransformPreviewLabel,\n\t\tactTransformPrompt,\n\t\tactTransformQuery,\n\t\tactPreview,\n\t\tactChangePreview,\n\t\tactRefreshPreview,\n\t\tactExecute,\n\t\tactExecuteSilent,\n\t\tactExecuteMulti,\n\t\tactReload,\n\t\tactReloadSync,\n\t\tactBecome:\n\t\treturn true\n\t}\n\treturn false\n}\n\ntype placeholderFlags struct {\n\tplus          bool\n\tpreserveSpace bool\n\tnumber        bool\n\tforceUpdate   bool\n\tfile          bool\n}\n\ntype searchRequest struct {\n\tsort    bool\n\tsync    bool\n\tcommand *string\n\tenviron []string\n\tchanged bool\n}\n\ntype previewRequest struct {\n\ttemplate     string\n\tpwindow      tui.Window\n\tpwindowSize  tui.TermSize\n\tscrollOffset int\n\tlist         []*Item\n}\n\ntype previewResult struct {\n\tversion int64\n\tlines   []string\n\toffset  int\n\tspinner string\n}\n\nfunc toActions(types ...actionType) []*action {\n\tactions := make([]*action, len(types))\n\tfor idx, t := range types {\n\t\tactions[idx] = &action{t: t, a: \"\"}\n\t}\n\treturn actions\n}\n\nfunc defaultKeymap() map[tui.Event][]*action {\n\tkeymap := make(map[tui.Event][]*action)\n\tadd := func(e tui.EventType, a actionType) {\n\t\tkeymap[e.AsEvent()] = toActions(a)\n\t}\n\taddEvent := func(e tui.Event, a actionType) {\n\t\tkeymap[e] = toActions(a)\n\t}\n\n\tadd(tui.Invalid, actInvalid)\n\tadd(tui.CtrlA, actBeginningOfLine)\n\tadd(tui.CtrlB, actBackwardChar)\n\tadd(tui.CtrlC, actAbort)\n\tadd(tui.CtrlG, actAbort)\n\tadd(tui.CtrlQ, actAbort)\n\tadd(tui.Esc, actAbort)\n\tadd(tui.CtrlD, actDeleteCharEof)\n\tadd(tui.CtrlE, actEndOfLine)\n\tadd(tui.CtrlF, actForwardChar)\n\tadd(tui.CtrlH, actBackwardDeleteChar)\n\tadd(tui.Backspace, actBackwardDeleteChar)\n\tadd(tui.Tab, actToggleDown)\n\tadd(tui.ShiftTab, actToggleUp)\n\tadd(tui.CtrlJ, actDown)\n\tadd(tui.CtrlK, actUp)\n\tadd(tui.CtrlL, actClearScreen)\n\tadd(tui.CtrlM, actAccept)\n\tadd(tui.CtrlN, actDown)\n\tadd(tui.CtrlP, actUp)\n\tadd(tui.CtrlU, actUnixLineDiscard)\n\tadd(tui.CtrlW, actUnixWordRubout)\n\tadd(tui.CtrlY, actYank)\n\tif !util.IsWindows() {\n\t\tadd(tui.CtrlZ, actSigStop)\n\t}\n\n\taddEvent(tui.AltKey('b'), actBackwardWord)\n\tadd(tui.ShiftLeft, actBackwardWord)\n\taddEvent(tui.AltKey('f'), actForwardWord)\n\tadd(tui.ShiftRight, actForwardWord)\n\taddEvent(tui.AltKey('d'), actKillWord)\n\tadd(tui.AltBackspace, actBackwardKillWord)\n\n\tadd(tui.Up, actUp)\n\tadd(tui.Down, actDown)\n\tadd(tui.Left, actBackwardChar)\n\tadd(tui.Right, actForwardChar)\n\n\tadd(tui.Home, actBeginningOfLine)\n\tadd(tui.End, actEndOfLine)\n\tadd(tui.Delete, actDeleteChar)\n\tadd(tui.PageUp, actPageUp)\n\tadd(tui.PageDown, actPageDown)\n\n\tadd(tui.ShiftUp, actPreviewUp)\n\tadd(tui.ShiftDown, actPreviewDown)\n\n\tadd(tui.Mouse, actMouse)\n\tadd(tui.LeftClick, actClick)\n\tadd(tui.RightClick, actToggle)\n\tadd(tui.SLeftClick, actToggle)\n\tadd(tui.SRightClick, actToggle)\n\n\tadd(tui.ScrollUp, actUp)\n\tadd(tui.ScrollDown, actDown)\n\tkeymap[tui.SScrollUp.AsEvent()] = toActions(actToggle, actUp)\n\tkeymap[tui.SScrollDown.AsEvent()] = toActions(actToggle, actDown)\n\n\tadd(tui.PreviewScrollUp, actPreviewUp)\n\tadd(tui.PreviewScrollDown, actPreviewDown)\n\treturn keymap\n}\n\nfunc trimQuery(query string) []rune {\n\treturn []rune(strings.Replace(query, \"\\t\", \" \", -1))\n}\n\nfunc mayTriggerPreview(opts *Options) bool {\n\tif opts.ListenAddr != nil {\n\t\treturn true\n\t}\n\tfor _, actions := range opts.Keymap {\n\t\tfor _, action := range actions {\n\t\t\tswitch action.t {\n\t\t\tcase actPreview, actChangePreview, actTransform:\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc makeSpinner(unicode bool) []string {\n\tif unicode {\n\t\treturn []string{`\u280b`, `\u2819`, `\u2839`, `\u2838`, `\u283c`, `\u2834`, `\u2826`, `\u2827`, `\u2807`, `\u280f`}\n\t}\n\treturn []string{`-`, `\\`, `|`, `/`, `-`, `\\`, `|`, `/`}\n}\n\nfunc evaluateHeight(opts *Options, termHeight int) int {\n\tsize := opts.Height.size\n\tif opts.Height.percent {\n\t\tif opts.Height.inverse {\n\t\t\tsize = 100 - size\n\t\t}\n\t\treturn util.Max(int(size*float64(termHeight)/100.0), opts.MinHeight)\n\t}\n\tif opts.Height.inverse {\n\t\tsize = float64(termHeight) - size\n\t}\n\treturn int(size)\n}\n\n// NewTerminal returns new Terminal object\nfunc NewTerminal(opts *Options, eventBox *util.EventBox) *Terminal {\n...\n", "needle_token_start": 2218, "needle_token_end": 2271, "code_context_ntokens": 14951, "output": ["```go\nfunc clearPatternCache() {\n\t// We can uniquely identify the pattern for a given string since\n\t// search mode and caseMode do not change while the program is running\n\t_patternCache = make(map[string]*Pattern)\n}\n```"]}
{"repo": "junegunn/fzf", "name": "newRange", "language": "go", "path": "src/tokenizer.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a structured representation of a range with special handling for specific values to denote unbounded ranges.\n2. **Input**: Two integers representing the beginning and the end of a range.\n3. **Output**: A structured representation of a range where specific integer values are translated to a predefined constant to indicate special meanings like \"all remaining elements\".\n4. **Procedure**: The function checks if the beginning value is 1 or the end value is -1 and replaces them with a predefined constant to signify an unbounded start or end respectively. It then returns a structured representation of this range.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/terminal_test.go\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"os\"\n\t\"regexp\"\n\t\"strings\"\n\t\"testing\"\n\t\"text/template\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\nfunc replacePlaceholderTest(template string, stripAnsi bool, delimiter Delimiter, printsep string, forcePlus bool, query string, allItems []*Item) string {\n\treturn replacePlaceholder(replacePlaceholderParams{\n\t\ttemplate:   template,\n\t\tstripAnsi:  stripAnsi,\n\t\tdelimiter:  delimiter,\n\t\tprintsep:   printsep,\n\t\tforcePlus:  forcePlus,\n\t\tquery:      query,\n\t\tallItems:   allItems,\n\t\tlastAction: actBackwardDeleteCharEof,\n\t\tprompt:     \"prompt\",\n\t})\n}\n\nfunc TestReplacePlaceholder(t *testing.T) {\n\titem1 := newItem(\"  foo'bar \\x1b[31mbaz\\x1b[m\")\n\titems1 := []*Item{item1, item1}\n\titems2 := []*Item{\n\t\tnewItem(\"foo'bar \\x1b[31mbaz\\x1b[m\"),\n\t\tnewItem(\"foo'bar \\x1b[31mbaz\\x1b[m\"),\n\t\tnewItem(\"FOO'BAR \\x1b[31mBAZ\\x1b[m\")}\n\n\tdelim := \"'\"\n\tvar regex *regexp.Regexp\n\n\tvar result string\n\tcheck := func(expected string) {\n\t\tif result != expected {\n\t\t\tt.Errorf(\"expected: %s, actual: %s\", expected, result)\n\t\t}\n\t}\n\t// helper function that converts template format into string and carries out the check()\n\tcheckFormat := func(format string) {\n\t\ttype quotes struct{ O, I, S string } // outer, inner quotes, print separator\n\t\tunixStyle := quotes{`'`, `'\\''`, \"\\n\"}\n\t\twindowsStyle := quotes{`^\"`, `'`, \"\\n\"}\n\t\tvar effectiveStyle quotes\n\n\t\tif util.IsWindows() {\n\t\t\teffectiveStyle = windowsStyle\n\t\t} else {\n\t\t\teffectiveStyle = unixStyle\n\t\t}\n\n\t\texpected := templateToString(format, effectiveStyle)\n\t\tcheck(expected)\n\t}\n\tprintsep := \"\\n\"\n\n\t/*\n\t\tTest multiple placeholders and the function parameters.\n\t*/\n\n\t// {}, preserve ansi\n\tresult = replacePlaceholderTest(\"echo {}\", false, Delimiter{}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}  foo{{.I}}bar \\x1b[31mbaz\\x1b[m{{.O}}\")\n\n\t// {}, strip ansi\n\tresult = replacePlaceholderTest(\"echo {}\", true, Delimiter{}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}  foo{{.I}}bar baz{{.O}}\")\n\n\t// {}, with multiple items\n\tresult = replacePlaceholderTest(\"echo {}\", true, Delimiter{}, printsep, false, \"query\", items2)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar baz{{.O}}\")\n\n\t// {..}, strip leading whitespaces, preserve ansi\n\tresult = replacePlaceholderTest(\"echo {..}\", false, Delimiter{}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar \\x1b[31mbaz\\x1b[m{{.O}}\")\n\n\t// {..}, strip leading whitespaces, strip ansi\n\tresult = replacePlaceholderTest(\"echo {..}\", true, Delimiter{}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar baz{{.O}}\")\n\n\t// {q}\n\tresult = replacePlaceholderTest(\"echo {} {q}\", true, Delimiter{}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}  foo{{.I}}bar baz{{.O}} {{.O}}query{{.O}}\")\n\n\t// {q}, multiple items\n\tresult = replacePlaceholderTest(\"echo {+}{q}{+}\", true, Delimiter{}, printsep, false, \"query 'string'\", items2)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar baz{{.O}} {{.O}}FOO{{.I}}BAR BAZ{{.O}}{{.O}}query {{.I}}string{{.I}}{{.O}}{{.O}}foo{{.I}}bar baz{{.O}} {{.O}}FOO{{.I}}BAR BAZ{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {}{q}{}\", true, Delimiter{}, printsep, false, \"query 'string'\", items2)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar baz{{.O}}{{.O}}query {{.I}}string{{.I}}{{.O}}{{.O}}foo{{.I}}bar baz{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {1}/{2}/{2,1}/{-1}/{-2}/{}/{..}/{n.t}/\\\\{}/\\\\{1}/\\\\{q}/{3}\", true, Delimiter{}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar{{.O}}/{{.O}}baz{{.O}}/{{.O}}bazfoo{{.I}}bar{{.O}}/{{.O}}baz{{.O}}/{{.O}}foo{{.I}}bar{{.O}}/{{.O}}  foo{{.I}}bar baz{{.O}}/{{.O}}foo{{.I}}bar baz{{.O}}/{n.t}/{}/{1}/{q}/{{.O}}{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {1}/{2}/{-1}/{-2}/{..}/{n.t}/\\\\{}/\\\\{1}/\\\\{q}/{3}\", true, Delimiter{}, printsep, false, \"query\", items2)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar{{.O}}/{{.O}}baz{{.O}}/{{.O}}baz{{.O}}/{{.O}}foo{{.I}}bar{{.O}}/{{.O}}foo{{.I}}bar baz{{.O}}/{n.t}/{}/{1}/{q}/{{.O}}{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {+1}/{+2}/{+-1}/{+-2}/{+..}/{n.t}/\\\\{}/\\\\{1}/\\\\{q}/{+3}\", true, Delimiter{}, printsep, false, \"query\", items2)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar{{.O}} {{.O}}FOO{{.I}}BAR{{.O}}/{{.O}}baz{{.O}} {{.O}}BAZ{{.O}}/{{.O}}baz{{.O}} {{.O}}BAZ{{.O}}/{{.O}}foo{{.I}}bar{{.O}} {{.O}}FOO{{.I}}BAR{{.O}}/{{.O}}foo{{.I}}bar baz{{.O}} {{.O}}FOO{{.I}}BAR BAZ{{.O}}/{n.t}/{}/{1}/{q}/{{.O}}{{.O}} {{.O}}{{.O}}\")\n\n\t// forcePlus\n\tresult = replacePlaceholderTest(\"echo {1}/{2}/{-1}/{-2}/{..}/{n.t}/\\\\{}/\\\\{1}/\\\\{q}/{3}\", true, Delimiter{}, printsep, true, \"query\", items2)\n\tcheckFormat(\"echo {{.O}}foo{{.I}}bar{{.O}} {{.O}}FOO{{.I}}BAR{{.O}}/{{.O}}baz{{.O}} {{.O}}BAZ{{.O}}/{{.O}}baz{{.O}} {{.O}}BAZ{{.O}}/{{.O}}foo{{.I}}bar{{.O}} {{.O}}FOO{{.I}}BAR{{.O}}/{{.O}}foo{{.I}}bar baz{{.O}} {{.O}}FOO{{.I}}BAR BAZ{{.O}}/{n.t}/{}/{1}/{q}/{{.O}}{{.O}} {{.O}}{{.O}}\")\n\n\t// Whitespace preserving flag with \"'\" delimiter\n\tresult = replacePlaceholderTest(\"echo {s1}\", true, Delimiter{str: &delim}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}  foo{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {s2}\", true, Delimiter{str: &delim}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}bar baz{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {s}\", true, Delimiter{str: &delim}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}  foo{{.I}}bar baz{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {s..}\", true, Delimiter{str: &delim}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}  foo{{.I}}bar baz{{.O}}\")\n\n\t// Whitespace preserving flag with regex delimiter\n\tregex = regexp.MustCompile(`\\w+`)\n\n\tresult = replacePlaceholderTest(\"echo {s1}\", true, Delimiter{regex: regex}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}  {{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {s2}\", true, Delimiter{regex: regex}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}}{{.I}}{{.O}}\")\n\n\tresult = replacePlaceholderTest(\"echo {s3}\", true, Delimiter{regex: regex}, printsep, false, \"query\", items1)\n\tcheckFormat(\"echo {{.O}} {{.O}}\")\n\n\t// No match\n\tresult = replacePlaceholderTest(\"echo {}/{+}\", true, Delimiter{}, printsep, false, \"query\", []*Item{nil, nil})\n\tcheck(\"echo /\")\n\n\t// No match, but with selections\n\tresult = replacePlaceholderTest(\"echo {}/{+}\", true, Delimiter{}, printsep, false, \"query\", []*Item{nil, item1})\n\tcheckFormat(\"echo /{{.O}}  foo{{.I}}bar baz{{.O}}\")\n\n\t// String delimiter\n...\n// Path: src/terminal_unix.go\n//go:build !windows\n\npackage fzf\n\nimport (\n\t\"os\"\n\t\"os/signal\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\nvar escaper *strings.Replacer\n\nfunc init() {\n\ttokens := strings.Split(os.Getenv(\"SHELL\"), \"/\")\n\tif tokens[len(tokens)-1] == \"fish\" {\n\t\t// https://fishshell.com/docs/current/language.html#quotes\n\t\t// > The only meaningful escape sequences in single quotes are \\', which\n\t\t// > escapes a single quote and \\\\, which escapes the backslash symbol.\n\t\tescaper = strings.NewReplacer(\"\\\\\", \"\\\\\\\\\", \"'\", \"\\\\'\")\n\t} else {\n\t\tescaper = strings.NewReplacer(\"'\", \"'\\\\''\")\n\t}\n}\n\nfunc notifyOnResize(resizeChan chan<- os.Signal) {\n\tsignal.Notify(resizeChan, syscall.SIGWINCH)\n}\n\nfunc notifyStop(p *os.Process) {\n\tpid := p.Pid\n\tpgid, err := unix.Getpgid(pid)\n\tif err == nil {\n\t\tpid = pgid * -1\n\t}\n\tunix.Kill(pid, syscall.SIGSTOP)\n}\n\nfunc notifyOnCont(resizeChan chan<- os.Signal) {\n\tsignal.Notify(resizeChan, syscall.SIGCONT)\n}\n\nfunc quoteEntry(entry string) string {\n\treturn \"'\" + escaper.Replace(entry) + \"'\"\n}\n\n// Path: src/terminal_windows.go\n//go:build windows\n\npackage fzf\n\nimport (\n\t\"os\"\n\t\"regexp\"\n\t\"strings\"\n)\n\nfunc notifyOnResize(resizeChan chan<- os.Signal) {\n\t// TODO\n}\n\nfunc notifyStop(p *os.Process) {\n\t// NOOP\n}\n\nfunc notifyOnCont(resizeChan chan<- os.Signal) {\n\t// NOOP\n}\n\nfunc quoteEntry(entry string) string {\n\tshell := os.Getenv(\"SHELL\")\n\tif len(shell) == 0 {\n\t\tshell = \"cmd\"\n\t}\n\n\tif strings.Contains(shell, \"cmd\") {\n\t\t// backslash escaping is done here for applications\n\t\t// (see ripgrep test case in terminal_test.go#TestWindowsCommands)\n\t\tescaped := strings.Replace(entry, `\\`, `\\\\`, -1)\n\t\tescaped = `\"` + strings.Replace(escaped, `\"`, `\\\"`, -1) + `\"`\n\t\t// caret is the escape character for cmd shell\n\t\tr, _ := regexp.Compile(`[&|<>()@^%!\"]`)\n\t\treturn r.ReplaceAllStringFunc(escaped, func(match string) string {\n\t\t\treturn \"^\" + match\n\t\t})\n\t} else if strings.Contains(shell, \"pwsh\") || strings.Contains(shell, \"powershell\") {\n\t\tescaped := strings.Replace(entry, `\"`, `\\\"`, -1)\n\t\treturn \"'\" + strings.Replace(escaped, \"'\", \"''\", -1) + \"'\"\n\t} else {\n\t\treturn \"'\" + strings.Replace(entry, \"'\", \"'\\\\''\", -1) + \"'\"\n\t}\n}\n\n// Path: src/tokenizer.go\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\nconst rangeEllipsis = 0\n\n// Range represents nth-expression\ntype Range struct {\n\tbegin int\n\tend   int\n}\n\n// Token contains the tokenized part of the strings and its prefix length\ntype Token struct {\n\ttext         *util.Chars\n\tprefixLength int32\n}\n\n// String returns the string representation of a Token.\nfunc (t Token) String() string {\n\treturn fmt.Sprintf(\"Token{text: %s, prefixLength: %d}\", t.text, t.prefixLength)\n}\n\n// Delimiter for tokenizing the input\ntype Delimiter struct {\n\tregex *regexp.Regexp\n\tstr   *string\n}\n\n// String returns the string representation of a Delimiter.\nfunc (d Delimiter) String() string {\n\treturn fmt.Sprintf(\"Delimiter{regex: %v, str: &%q}\", d.regex, *d.str)\n}\n\n\nfunc newRange(begin int, end int) Range {\n\tif begin == 1 {\n\t\tbegin = rangeEllipsis\n\t}\n\tif end == -1 {\n\t\tend = rangeEllipsis\n\t}\n\treturn Range{begin, end}\n}\n\n// ParseRange parses nth-expression and returns the corresponding Range object\nfunc ParseRange(str *string) (Range, bool) {\n\tif (*str) == \"..\" {\n\t\treturn newRange(rangeEllipsis, rangeEllipsis), true\n\t} else if strings.HasPrefix(*str, \"..\") {\n\t\tend, err := strconv.Atoi((*str)[2:])\n\t\tif err != nil || end == 0 {\n\t\t\treturn Range{}, false\n\t\t}\n\t\treturn newRange(rangeEllipsis, end), true\n\t} else if strings.HasSuffix(*str, \"..\") {\n\t\tbegin, err := strconv.Atoi((*str)[:len(*str)-2])\n\t\tif err != nil || begin == 0 {\n\t\t\treturn Range{}, false\n\t\t}\n\t\treturn newRange(begin, rangeEllipsis), true\n\t} else if strings.Contains(*str, \"..\") {\n\t\tns := strings.Split(*str, \"..\")\n\t\tif len(ns) != 2 {\n\t\t\treturn Range{}, false\n\t\t}\n\t\tbegin, err1 := strconv.Atoi(ns[0])\n\t\tend, err2 := strconv.Atoi(ns[1])\n\t\tif err1 != nil || err2 != nil || begin == 0 || end == 0 {\n\t\t\treturn Range{}, false\n\t\t}\n\t\treturn newRange(begin, end), true\n\t}\n\n\tn, err := strconv.Atoi(*str)\n\tif err != nil || n == 0 {\n\t\treturn Range{}, false\n\t}\n\treturn newRange(n, n), true\n}\n\nfunc withPrefixLengths(tokens []string, begin int) []Token {\n\tret := make([]Token, len(tokens))\n\n\tprefixLength := begin\n\tfor idx := range tokens {\n\t\tchars := util.ToChars(sbytes(tokens[idx]))\n\t\tret[idx] = Token{&chars, int32(prefixLength)}\n\t\tprefixLength += chars.Length()\n\t}\n\treturn ret\n}\n\nconst (\n\tawkNil = iota\n\tawkBlack\n\tawkWhite\n)\n\nfunc awkTokenizer(input string) ([]string, int) {\n\t// 9, 32\n\tret := []string{}\n\tprefixLength := 0\n\tstate := awkNil\n\tbegin := 0\n\tend := 0\n\tfor idx := 0; idx < len(input); idx++ {\n\t\tr := input[idx]\n\t\twhite := r == 9 || r == 32\n\t\tswitch state {\n\t\tcase awkNil:\n\t\t\tif white {\n\t\t\t\tprefixLength++\n\t\t\t} else {\n\t\t\t\tstate, begin, end = awkBlack, idx, idx+1\n\t\t\t}\n\t\tcase awkBlack:\n\t\t\tend = idx + 1\n\t\t\tif white {\n\t\t\t\tstate = awkWhite\n\t\t\t}\n\t\tcase awkWhite:\n\t\t\tif white {\n\t\t\t\tend = idx + 1\n\t\t\t} else {\n\t\t\t\tret = append(ret, input[begin:end])\n\t\t\t\tstate, begin, end = awkBlack, idx, idx+1\n\t\t\t}\n\t\t}\n\t}\n\tif begin < end {\n\t\tret = append(ret, input[begin:end])\n\t}\n\treturn ret, prefixLength\n}\n\n// Tokenize tokenizes the given string with the delimiter\nfunc Tokenize(text string, delimiter Delimiter) []Token {\n\tif delimiter.str == nil && delimiter.regex == nil {\n\t\t// AWK-style (\\S+\\s*)\n\t\ttokens, prefixLength := awkTokenizer(text)\n\t\treturn withPrefixLengths(tokens, prefixLength)\n\t}\n\n\tif delimiter.str != nil {\n\t\treturn withPrefixLengths(strings.SplitAfter(text, *delimiter.str), 0)\n\t}\n\n\t// FIXME performance\n\tvar tokens []string\n\tif delimiter.regex != nil {\n\t\tlocs := delimiter.regex.FindAllStringIndex(text, -1)\n\t\tbegin := 0\n\t\tfor _, loc := range locs {\n\t\t\ttokens = append(tokens, text[begin:loc[1]])\n\t\t\tbegin = loc[1]\n\t\t}\n\t\tif begin < len(text) {\n\t\t\ttokens = append(tokens, text[begin:])\n\t\t}\n\t}\n\treturn withPrefixLengths(tokens, 0)\n}\n\nfunc joinTokens(tokens []Token) string {\n\tvar output bytes.Buffer\n\tfor _, token := range tokens {\n\t\toutput.WriteString(token.text.ToString())\n\t}\n\treturn output.String()\n}\n\n// Transform is used to transform the input when --with-nth option is given\nfunc Transform(tokens []Token, withNth []Range) []Token {\n\ttransTokens := make([]Token, len(withNth))\n\tnumTokens := len(tokens)\n\tfor idx, r := range withNth {\n\t\tparts := []*util.Chars{}\n\t\tminIdx := 0\n\t\tif r.begin == r.end {\n\t\t\tidx := r.begin\n\t\t\tif idx == rangeEllipsis {\n\t\t\t\tchars := util.ToChars(sbytes(joinTokens(tokens)))\n\t\t\t\tparts = append(parts, &chars)\n\t\t\t} else {\n\t\t\t\tif idx < 0 {\n\t\t\t\t\tidx += numTokens + 1\n\t\t\t\t}\n\t\t\t\tif idx >= 1 && idx <= numTokens {\n\t\t\t\t\tminIdx = idx - 1\n\t\t\t\t\tparts = append(parts, tokens[idx-1].text)\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tvar begin, end int\n\t\t\tif r.begin == rangeEllipsis { // ..N\n\t\t\t\tbegin, end = 1, r.end\n\t\t\t\tif end < 0 {\n\t\t\t\t\tend += numTokens + 1\n\t\t\t\t}\n\t\t\t} else if r.end == rangeEllipsis { // N..\n\t\t\t\tbegin, end = r.begin, numTokens\n\t\t\t\tif begin < 0 {\n\t\t\t\t\tbegin += numTokens + 1\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tbegin, end = r.begin, r.end\n\t\t\t\tif begin < 0 {\n\t\t\t\t\tbegin += numTokens + 1\n\t\t\t\t}\n\t\t\t\tif end < 0 {\n\t\t\t\t\tend += numTokens + 1\n\t\t\t\t}\n\t\t\t}\n\t\t\tminIdx = util.Max(0, begin-1)\n\t\t\tfor idx := begin; idx <= end; idx++ {\n\t\t\t\tif idx >= 1 && idx <= numTokens {\n\t\t\t\t\tparts = append(parts, tokens[idx-1].text)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Merge multiple parts\n\t\tvar merged util.Chars\n\t\tswitch len(parts) {\n\t\tcase 0:\n\t\t\tmerged = util.ToChars([]byte{})\n\t\tcase 1:\n\t\t\tmerged = *parts[0]\n\t\tdefault:\n\t\t\tvar output bytes.Buffer\n\t\t\tfor _, part := range parts {\n\t\t\t\toutput.WriteString(part.ToString())\n\t\t\t}\n\t\t\tmerged = util.ToChars(output.Bytes())\n\t\t}\n\n\t\tvar prefixLength int32\n\t\tif minIdx < numTokens {\n\t\t\tprefixLength = tokens[minIdx].prefixLength\n\t\t} else {\n\t\t\tprefixLength = 0\n\t\t}\n\t\ttransTokens[idx] = Token{&merged, prefixLength}\n\t}\n\treturn transTokens\n}\n\n// Path: src/tokenizer_test.go\npackage fzf\n\nimport (\n\t\"testing\"\n)\n\nfunc TestParseRange(t *testing.T) {\n\t{\n\t\ti := \"..\"\n\t\tr, _ := ParseRange(&i)\n\t\tif r.begin != rangeEllipsis || r.end != rangeEllipsis {\n\t\t\tt.Errorf(\"%v\", r)\n\t\t}\n\t}\n\t{\n\t\ti := \"3..\"\n\t\tr, _ := ParseRange(&i)\n\t\tif r.begin != 3 || r.end != rangeEllipsis {\n\t\t\tt.Errorf(\"%v\", r)\n\t\t}\n\t}\n\t{\n\t\ti := \"3..5\"\n\t\tr, _ := ParseRange(&i)\n\t\tif r.begin != 3 || r.end != 5 {\n\t\t\tt.Errorf(\"%v\", r)\n\t\t}\n\t}\n\t{\n\t\ti := \"-3..-5\"\n\t\tr, _ := ParseRange(&i)\n\t\tif r.begin != -3 || r.end != -5 {\n\t\t\tt.Errorf(\"%v\", r)\n\t\t}\n\t}\n\t{\n\t\ti := \"3\"\n\t\tr, _ := ParseRange(&i)\n\t\tif r.begin != 3 || r.end != 3 {\n\t\t\tt.Errorf(\"%v\", r)\n\t\t}\n\t}\n}\n\nfunc TestTokenize(t *testing.T) {\n\t// AWK-style\n\tinput := \"  abc:  def:  ghi  \"\n\ttokens := Tokenize(input, Delimiter{})\n\tif tokens[0].text.ToString() != \"abc:  \" || tokens[0].prefixLength != 2 {\n\t\tt.Errorf(\"%s\", tokens)\n\t}\n\n\t// With delimiter\n\ttokens = Tokenize(input, delimiterRegexp(\":\"))\n\tif tokens[0].text.ToString() != \"  abc:\" || tokens[0].prefixLength != 0 {\n\t\tt.Error(tokens[0].text.ToString(), tokens[0].prefixLength)\n\t}\n\n\t// With delimiter regex\n\ttokens = Tokenize(input, delimiterRegexp(\"\\\\s+\"))\n\tif tokens[0].text.ToString() != \"  \" || tokens[0].prefixLength != 0 ||\n\t\ttokens[1].text.ToString() != \"abc:  \" || tokens[1].prefixLength != 2 ||\n\t\ttokens[2].text.ToString() != \"def:  \" || tokens[2].prefixLength != 8 ||\n\t\ttokens[3].text.ToString() != \"ghi  \" || tokens[3].prefixLength != 14 {\n\t\tt.Errorf(\"%s\", tokens)\n\t}\n}\n\nfunc TestTransform(t *testing.T) {\n\tinput := \"  abc:  def:  ghi:  jkl\"\n\t{\n\t\ttokens := Tokenize(input, Delimiter{})\n\t\t{\n\t\t\tranges := splitNth(\"1,2,3\")\n\t\t\ttx := Transform(tokens, ranges)\n\t\t\tif joinTokens(tx) != \"abc:  def:  ghi:  \" {\n\t\t\t\tt.Errorf(\"%s\", tx)\n\t\t\t}\n\t\t}\n\t\t{\n\t\t\tranges := splitNth(\"1..2,3,2..,1\")\n\t\t\ttx := Transform(tokens, ranges)\n\t\t\tif string(joinTokens(tx)) != \"abc:  def:  ghi:  def:  ghi:  jklabc:  \" ||\n\t\t\t\tlen(tx) != 4 ||\n\t\t\t\ttx[0].text.ToString() != \"abc:  def:  \" || tx[0].prefixLength != 2 ||\n\t\t\t\ttx[1].text.ToString() != \"ghi:  \" || tx[1].prefixLength != 14 ||\n\t\t\t\ttx[2].text.ToString() != \"def:  ghi:  jkl\" || tx[2].prefixLength != 8 ||\n\t\t\t\ttx[3].text.ToString() != \"abc:  \" || tx[3].prefixLength != 2 {\n\t\t\t\tt.Errorf(\"%s\", tx)\n\t\t\t}\n\t\t}\n\t}\n\t{\n\t\ttokens := Tokenize(input, delimiterRegexp(\":\"))\n\t\t{\n\t\t\tranges := splitNth(\"1..2,3,2..,1\")\n\t\t\ttx := Transform(tokens, ranges)\n\t\t\tif joinTokens(tx) != \"  abc:  def:  ghi:  def:  ghi:  jkl  abc:\" ||\n\t\t\t\tlen(tx) != 4 ||\n\t\t\t\ttx[0].text.ToString() != \"  abc:  def:\" || tx[0].prefixLength != 0 ||\n\t\t\t\ttx[1].text.ToString() != \"  ghi:\" || tx[1].prefixLength != 12 ||\n\t\t\t\ttx[2].text.ToString() != \"  def:  ghi:  jkl\" || tx[2].prefixLength != 6 ||\n\t\t\t\ttx[3].text.ToString() != \"  abc:\" || tx[3].prefixLength != 0 {\n\t\t\t\tt.Errorf(\"%s\", tx)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestTransformIndexOutOfBounds(t *testing.T) {\n\tTransform([]Token{}, splitNth(\"1\"))\n}\n\n// Path: src/tui/dummy.go\n//go:build !tcell && !windows\n\npackage tui\n\ntype Attr int32\n\nfunc HasFullscreenRenderer() bool {\n\treturn false\n}\n\nvar DefaultBorderShape BorderShape = BorderRounded\n\nfunc (a Attr) Merge(b Attr) Attr {\n\treturn a | b\n}\n\nconst (\n\tAttrUndefined = Attr(0)\n\tAttrRegular   = Attr(1 << 8)\n\tAttrClear     = Attr(1 << 9)\n\n\tBold          = Attr(1)\n\tDim           = Attr(1 << 1)\n\tItalic        = Attr(1 << 2)\n\tUnderline     = Attr(1 << 3)\n\tBlink         = Attr(1 << 4)\n\tBlink2        = Attr(1 << 5)\n\tReverse       = Attr(1 << 6)\n\tStrikeThrough = Attr(1 << 7)\n)\n\nfunc (r *FullscreenRenderer) Init()                              {}\nfunc (r *FullscreenRenderer) Resize(maxHeightFunc func(int) int) {}\nfunc (r *FullscreenRenderer) Pause(bool)                         {}\nfunc (r *FullscreenRenderer) Resume(bool, bool)                  {}\nfunc (r *FullscreenRenderer) PassThrough(string)                 {}\nfunc (r *FullscreenRenderer) Clear()                             {}\nfunc (r *FullscreenRenderer) NeedScrollbarRedraw() bool          { return false }\nfunc (r *FullscreenRenderer) ShouldEmitResizeEvent() bool        { return false }\nfunc (r *FullscreenRenderer) Refresh()                           {}\nfunc (r *FullscreenRenderer) Close()                             {}\nfunc (r *FullscreenRenderer) Size() TermSize                     { return TermSize{} }\n\nfunc (r *FullscreenRenderer) GetChar() Event { return Event{} }\nfunc (r *FullscreenRenderer) Top() int       { return 0 }\nfunc (r *FullscreenRenderer) MaxX() int      { return 0 }\nfunc (r *FullscreenRenderer) MaxY() int      { return 0 }\n\nfunc (r *FullscreenRenderer) RefreshWindows(windows []Window) {}\n\nfunc (r *FullscreenRenderer) NewWindow(top int, left int, width int, height int, preview bool, borderStyle BorderStyle) Window {\n\treturn nil\n}\n\n// Path: src/tui/eventtype_string.go\n// Code generated by \"stringer -type=EventType\"; DO NOT EDIT.\n\npackage tui\n\nimport \"strconv\"\n\nfunc _() {\n\t// An \"invalid array index\" compiler error signifies that the constant values have changed.\n\t// Re-run the stringer command to generate them again.\n\tvar x [1]struct{}\n\t_ = x[Rune-0]\n\t_ = x[CtrlA-1]\n\t_ = x[CtrlB-2]\n\t_ = x[CtrlC-3]\n\t_ = x[CtrlD-4]\n\t_ = x[CtrlE-5]\n\t_ = x[CtrlF-6]\n\t_ = x[CtrlG-7]\n\t_ = x[CtrlH-8]\n\t_ = x[Tab-9]\n\t_ = x[CtrlJ-10]\n\t_ = x[CtrlK-11]\n\t_ = x[CtrlL-12]\n\t_ = x[CtrlM-13]\n\t_ = x[CtrlN-14]\n\t_ = x[CtrlO-15]\n\t_ = x[CtrlP-16]\n\t_ = x[CtrlQ-17]\n\t_ = x[CtrlR-18]\n\t_ = x[CtrlS-19]\n\t_ = x[CtrlT-20]\n\t_ = x[CtrlU-21]\n\t_ = x[CtrlV-22]\n\t_ = x[CtrlW-23]\n\t_ = x[CtrlX-24]\n\t_ = x[CtrlY-25]\n\t_ = x[CtrlZ-26]\n\t_ = x[Esc-27]\n\t_ = x[CtrlSpace-28]\n\t_ = x[CtrlDelete-29]\n\t_ = x[CtrlBackSlash-30]\n\t_ = x[CtrlRightBracket-31]\n\t_ = x[CtrlCaret-32]\n\t_ = x[CtrlSlash-33]\n\t_ = x[ShiftTab-34]\n\t_ = x[Backspace-35]\n\t_ = x[Delete-36]\n\t_ = x[PageUp-37]\n\t_ = x[PageDown-38]\n\t_ = x[Up-39]\n\t_ = x[Down-40]\n\t_ = x[Left-41]\n\t_ = x[Right-42]\n\t_ = x[Home-43]\n\t_ = x[End-44]\n\t_ = x[Insert-45]\n\t_ = x[ShiftUp-46]\n\t_ = x[ShiftDown-47]\n\t_ = x[ShiftLeft-48]\n\t_ = x[ShiftRight-49]\n\t_ = x[ShiftDelete-50]\n\t_ = x[F1-51]\n\t_ = x[F2-52]\n\t_ = x[F3-53]\n\t_ = x[F4-54]\n\t_ = x[F5-55]\n\t_ = x[F6-56]\n\t_ = x[F7-57]\n\t_ = x[F8-58]\n\t_ = x[F9-59]\n\t_ = x[F10-60]\n\t_ = x[F11-61]\n\t_ = x[F12-62]\n\t_ = x[AltBackspace-63]\n\t_ = x[AltUp-64]\n\t_ = x[AltDown-65]\n\t_ = x[AltLeft-66]\n\t_ = x[AltRight-67]\n\t_ = x[AltShiftUp-68]\n\t_ = x[AltShiftDown-69]\n\t_ = x[AltShiftLeft-70]\n\t_ = x[AltShiftRight-71]\n\t_ = x[Alt-72]\n\t_ = x[CtrlAlt-73]\n\t_ = x[Invalid-74]\n\t_ = x[Mouse-75]\n\t_ = x[DoubleClick-76]\n\t_ = x[LeftClick-77]\n\t_ = x[RightClick-78]\n\t_ = x[SLeftClick-79]\n\t_ = x[SRightClick-80]\n\t_ = x[ScrollUp-81]\n\t_ = x[ScrollDown-82]\n\t_ = x[SScrollUp-83]\n\t_ = x[SScrollDown-84]\n\t_ = x[PreviewScrollUp-85]\n\t_ = x[PreviewScrollDown-86]\n\t_ = x[Resize-87]\n\t_ = x[Change-88]\n\t_ = x[BackwardEOF-89]\n\t_ = x[Start-90]\n\t_ = x[Load-91]\n\t_ = x[Focus-92]\n\t_ = x[One-93]\n\t_ = x[Zero-94]\n\t_ = x[Result-95]\n\t_ = x[Jump-96]\n\t_ = x[JumpCancel-97]\n}\n\nconst _EventType_name = \"RuneCtrlACtrlBCtrlCCtrlDCtrlECtrlFCtrlGCtrlHTabCtrlJCtrlKCtrlLCtrlMCtrlNCtrlOCtrlPCtrlQCtrlRCtrlSCtrlTCtrlUCtrlVCtrlWCtrlXCtrlYCtrlZEscCtrlSpaceCtrlDeleteCtrlBackSlashCtrlRightBracketCtrlCaretCtrlSlashShiftTabBackspaceDeletePageUpPageDownUpDownLeftRightHomeEndInsertShiftUpShiftDownShiftLeftShiftRightShiftDeleteF1F2F3F4F5F6F7F8F9F10F11F12AltBackspaceAltUpAltDownAltLeftAltRightAltShiftUpAltShiftDownAltShiftLeftAltShiftRightAltCtrlAltInvalidMouseDoubleClickLeftClickRightClickSLeftClickSRightClickScrollUpScrollDownSScrollUpSScrollDownPreviewScrollUpPreviewScrollDownResizeChangeBackwardEOFStartLoadFocusOneZeroResultJumpJumpCancel\"\n\nvar _EventType_index = [...]uint16{0, 4, 9, 14, 19, 24, 29, 34, 39, 44, 47, 52, 57, 62, 67, 72, 77, 82, 87, 92, 97, 102, 107, 112, 117, 122, 127, 132, 135, 144, 154, 167, 183, 192, 201, 209, 218, 224, 230, 238, 240, 244, 248, 253, 257, 260, 266, 273, 282, 291, 301, 312, 314, 316, 318, 320, 322, 324, 326, 328, 330, 333, 336, 339, 351, 356, 363, 370, 378, 388, 400, 412, 425, 428, 435, 442, 447, 458, 467, 477, 487, 498, 506, 516, 525, 536, 551, 568, 574, 580, 591, 596, 600, 605, 608, 612, 618, 622, 632}\n\nfunc (i EventType) String() string {\n\tif i < 0 || i >= EventType(len(_EventType_index)-1) {\n\t\treturn \"EventType(\" + strconv.FormatInt(int64(i), 10) + \")\"\n\t}\n\treturn _EventType_name[_EventType_index[i]:_EventType_index[i+1]]\n}\n\n// Path: src/tui/light.go\npackage tui\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\t\"unicode/utf8\"\n\n\t\"github.com/rivo/uniseg\"\n\n\t\"golang.org/x/term\"\n)\n\nconst (\n\tdefaultWidth  = 80\n\tdefaultHeight = 24\n\n\tdefaultEscDelay = 100\n\tescPollInterval = 5\n\toffsetPollTries = 10\n\tmaxInputBuffer  = 1024 * 1024\n)\n\nconst consoleDevice string = \"/dev/tty\"\n\nvar offsetRegexp *regexp.Regexp = regexp.MustCompile(\"(.*)\\x1b\\\\[([0-9]+);([0-9]+)R\")\nvar offsetRegexpBegin *regexp.Regexp = regexp.MustCompile(\"^\\x1b\\\\[[0-9]+;[0-9]+R\")\n\nfunc (r *LightRenderer) PassThrough(str string) {\n\tr.queued.WriteString(\"\\x1b7\" + str + \"\\x1b8\")\n}\n\nfunc (r *LightRenderer) stderr(str string) {\n\tr.stderrInternal(str, true, \"\")\n}\n\nconst CR string = \"\\x1b[2m\u240d\"\nconst LF string = \"\\x1b[2m\u240a\"\n\nfunc (r *LightRenderer) stderrInternal(str string, allowNLCR bool, resetCode string) {\n\tbytes := []byte(str)\n\trunes := []rune{}\n\tfor len(bytes) > 0 {\n\t\tr, sz := utf8.DecodeRune(bytes)\n\t\tnlcr := r == '\\n' || r == '\\r'\n\t\tif r >= 32 || r == '\\x1b' || nlcr {\n\t\t\tif nlcr && !allowNLCR {\n\t\t\t\tif r == '\\r' {\n\t\t\t\t\trunes = append(runes, []rune(CR+resetCode)...)\n\t\t\t\t} else {\n\t\t\t\t\trunes = append(runes, []rune(LF+resetCode)...)\n\t\t\t\t}\n\t\t\t} else if r != utf8.RuneError {\n\t\t\t\trunes = append(runes, r)\n\t\t\t}\n\t\t}\n\t\tbytes = bytes[sz:]\n\t}\n\tr.queued.WriteString(string(runes))\n}\n\nfunc (r *LightRenderer) csi(code string) string {\n\tfullcode := \"\\x1b[\" + code\n\tr.stderr(fullcode)\n\treturn fullcode\n}\n\nfunc (r *LightRenderer) flush() {\n\tif r.queued.Len() > 0 {\n\t\tfmt.Fprint(os.Stderr, \"\\x1b[?7l\\x1b[?25l\"+r.queued.String()+\"\\x1b[?25h\\x1b[?7h\")\n\t\tr.queued.Reset()\n\t}\n}\n\n// Light renderer\ntype LightRenderer struct {\n\ttheme         *ColorTheme\n\tmouse         bool\n\tforceBlack    bool\n\tclearOnExit   bool\n\tprevDownTime  time.Time\n\tclicks        [][2]int\n\tttyin         *os.File\n\tbuffer        []byte\n\torigState     *term.State\n\twidth         int\n\theight        int\n\tyoffset       int\n\ttabstop       int\n\tescDelay      int\n\tfullscreen    bool\n\tupOneLine     bool\n\tqueued        strings.Builder\n\ty             int\n\tx             int\n\tmaxHeightFunc func(int) int\n\n\t// Windows only\n\tttyinChannel    chan byte\n\tinHandle        uintptr\n\toutHandle       uintptr\n\torigStateInput  uint32\n\torigStateOutput uint32\n}\n\ntype LightWindow struct {\n\trenderer *LightRenderer\n\tcolored  bool\n\tpreview  bool\n\tborder   BorderStyle\n\ttop      int\n\tleft     int\n\twidth    int\n\theight   int\n\tposx     int\n\tposy     int\n\ttabstop  int\n\tfg       Color\n\tbg       Color\n}\n\nfunc NewLightRenderer(theme *ColorTheme, forceBlack bool, mouse bool, tabstop int, clearOnExit bool, fullscreen bool, maxHeightFunc func(int) int) Renderer {\n\tr := LightRenderer{\n\t\ttheme:         theme,\n\t\tforceBlack:    forceBlack,\n\t\tmouse:         mouse,\n\t\tclearOnExit:   clearOnExit,\n\t\tttyin:         openTtyIn(),\n\t\tyoffset:       0,\n\t\ttabstop:       tabstop,\n\t\tfullscreen:    fullscreen,\n\t\tupOneLine:     false,\n\t\tmaxHeightFunc: maxHeightFunc}\n\treturn &r\n}\n\nfunc repeat(r rune, times int) string {\n\tif times > 0 {\n\t\treturn strings.Repeat(string(r), times)\n\t}\n\treturn \"\"\n}\n\nfunc atoi(s string, defaultValue int) int {\n\tvalue, err := strconv.Atoi(s)\n\tif err != nil {\n\t\treturn defaultValue\n\t}\n\treturn value\n}\n\nfunc (r *LightRenderer) Init() {\n\tr.escDelay = atoi(os.Getenv(\"ESCDELAY\"), defaultEscDelay)\n\n\tif err := r.initPlatform(); err != nil {\n\t\terrorExit(err.Error())\n\t}\n\tr.updateTerminalSize()\n\tinitTheme(r.theme, r.defaultTheme(), r.forceBlack)\n\n\tif r.fullscreen {\n\t\tr.smcup()\n\t} else {\n\t\t// We assume that --no-clear is used for repetitive relaunching of fzf.\n\t\t// So we do not clear the lower bottom of the screen.\n\t\tif r.clearOnExit {\n\t\t\tr.csi(\"J\")\n\t\t}\n\t\ty, x := r.findOffset()\n\t\tr.mouse = r.mouse && y >= 0\n\t\t// When --no-clear is used for repetitive relaunching, there is a small\n\t\t// time frame between fzf processes where the user keystrokes are not\n\t\t// captured by either of fzf process which can cause x offset to be\n\t\t// increased and we're left with unwanted extra new line.\n\t\tif x > 0 && r.clearOnExit {\n\t\t\tr.upOneLine = true\n\t\t\tr.makeSpace()\n\t\t}\n\t\tfor i := 1; i < r.MaxY(); i++ {\n\t\t\tr.makeSpace()\n\t\t}\n\t}\n\n\tr.enableMouse()\n\tr.csi(fmt.Sprintf(\"%dA\", r.MaxY()-1))\n\tr.csi(\"G\")\n\tr.csi(\"K\")\n\tif !r.clearOnExit && !r.fullscreen {\n\t\tr.csi(\"s\")\n\t}\n\tif !r.fullscreen && r.mouse {\n\t\tr.yoffset, _ = r.findOffset()\n\t}\n}\n\nfunc (r *LightRenderer) Resize(maxHeightFunc func(int) int) {\n\tr.maxHeightFunc = maxHeightFunc\n}\n\nfunc (r *LightRenderer) makeSpace() {\n\tr.stderr(\"\\n\")\n\tr.csi(\"G\")\n}\n\nfunc (r *LightRenderer) move(y int, x int) {\n\t// w.csi(\"u\")\n\tif r.y < y {\n\t\tr.csi(fmt.Sprintf(\"%dB\", y-r.y))\n\t} else if r.y > y {\n\t\tr.csi(fmt.Sprintf(\"%dA\", r.y-y))\n\t}\n\tr.stderr(\"\\r\")\n\tif x > 0 {\n\t\tr.csi(fmt.Sprintf(\"%dC\", x))\n\t}\n\tr.y = y\n\tr.x = x\n}\n\nfunc (r *LightRenderer) origin() {\n\tr.move(0, 0)\n}\n\nfunc getEnv(name string, defaultValue int) int {\n\tenv := os.Getenv(name)\n\tif len(env) == 0 {\n\t\treturn defaultValue\n\t}\n\treturn atoi(env, defaultValue)\n}\n\nfunc (r *LightRenderer) getBytes() []byte {\n\treturn r.getBytesInternal(r.buffer, false)\n}\n\nfunc (r *LightRenderer) getBytesInternal(buffer []byte, nonblock bool) []byte {\n\tc, ok := r.getch(nonblock)\n\tif !nonblock && !ok {\n\t\tr.Close()\n\t\terrorExit(\"Failed to read \" + consoleDevice)\n\t}\n\n\tretries := 0\n\tif c == Esc.Int() || nonblock {\n\t\tretries = r.escDelay / escPollInterval\n\t}\n\tbuffer = append(buffer, byte(c))\n\n\tpc := c\n\tfor {\n\t\tc, ok = r.getch(true)\n\t\tif !ok {\n\t\t\tif retries > 0 {\n\t\t\t\tretries--\n\t\t\t\ttime.Sleep(escPollInterval * time.Millisecond)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbreak\n\t\t} else if c == Esc.Int() && pc != c {\n\t\t\tretries = r.escDelay / escPollInterval\n\t\t} else {\n\t\t\tretries = 0\n\t\t}\n\t\tbuffer = append(buffer, byte(c))\n\t\tpc = c\n\n\t\t// This should never happen under normal conditions,\n\t\t// so terminate fzf immediately.\n\t\tif len(buffer) > maxInputBuffer {\n\t\t\tr.Close()\n\t\t\tpanic(fmt.Sprintf(\"Input buffer overflow (%d): %v\", len(buffer), buffer))\n\t\t}\n\t}\n\n\treturn buffer\n}\n\nfunc (r *LightRenderer) GetChar() Event {\n\tif len(r.buffer) == 0 {\n\t\tr.buffer = r.getBytes()\n\t}\n\tif len(r.buffer) == 0 {\n\t\tpanic(\"Empty buffer\")\n\t}\n\n\tsz := 1\n\tdefer func() {\n\t\tr.buffer = r.buffer[sz:]\n\t}()\n\n\tswitch r.buffer[0] {\n\tcase CtrlC.Byte():\n\t\treturn Event{CtrlC, 0, nil}\n\tcase CtrlG.Byte():\n\t\treturn Event{CtrlG, 0, nil}\n\tcase CtrlQ.Byte():\n\t\treturn Event{CtrlQ, 0, nil}\n\tcase 127:\n\t\treturn Event{Backspace, 0, nil}\n\tcase 0:\n\t\treturn Event{CtrlSpace, 0, nil}\n\tcase 28:\n\t\treturn Event{CtrlBackSlash, 0, nil}\n\tcase 29:\n\t\treturn Event{CtrlRightBracket, 0, nil}\n\tcase 30:\n\t\treturn Event{CtrlCaret, 0, nil}\n\tcase 31:\n\t\treturn Event{CtrlSlash, 0, nil}\n\tcase Esc.Byte():\n\t\tev := r.escSequence(&sz)\n\t\t// Second chance\n\t\tif ev.Type == Invalid {\n\t\t\tr.buffer = r.getBytes()\n\t\t\tev = r.escSequence(&sz)\n\t\t}\n\t\treturn ev\n\t}\n\n\t// CTRL-A ~ CTRL-Z\n\tif r.buffer[0] <= CtrlZ.Byte() {\n\t\treturn Event{EventType(r.buffer[0]), 0, nil}\n\t}\n\tchar, rsz := utf8.DecodeRune(r.buffer)\n\tif char == utf8.RuneError {\n\t\treturn Event{Esc, 0, nil}\n\t}\n\tsz = rsz\n\treturn Event{Rune, char, nil}\n}\n\nfunc (r *LightRenderer) escSequence(sz *int) Event {\n\tif len(r.buffer) < 2 {\n\t\treturn Event{Esc, 0, nil}\n\t}\n\n\tloc := offsetRegexpBegin.FindIndex(r.buffer)\n\tif loc != nil && loc[0] == 0 {\n\t\t*sz = loc[1]\n\t\treturn Event{Invalid, 0, nil}\n\t}\n\n\t*sz = 2\n\tif r.buffer[1] >= 1 && r.buffer[1] <= 'z'-'a'+1 {\n\t\treturn CtrlAltKey(rune(r.buffer[1] + 'a' - 1))\n\t}\n\talt := false\n\tif len(r.buffer) > 2 && r.buffer[1] == Esc.Byte() {\n\t\tr.buffer = r.buffer[1:]\n\t\talt = true\n\t}\n\tswitch r.buffer[1] {\n\tcase Esc.Byte():\n\t\treturn Event{Esc, 0, nil}\n\tcase 127:\n\t\treturn Event{AltBackspace, 0, nil}\n\tcase '[', 'O':\n\t\tif len(r.buffer) < 3 {\n\t\t\treturn Event{Invalid, 0, nil}\n\t\t}\n\t\t*sz = 3\n\t\tswitch r.buffer[2] {\n\t\tcase 'D':\n\t\t\tif alt {\n\t\t\t\treturn Event{AltLeft, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Left, 0, nil}\n\t\tcase 'C':\n\t\t\tif alt {\n\t\t\t\t// Ugh..\n\t\t\t\treturn Event{AltRight, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Right, 0, nil}\n\t\tcase 'B':\n\t\t\tif alt {\n\t\t\t\treturn Event{AltDown, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Down, 0, nil}\n\t\tcase 'A':\n\t\t\tif alt {\n\t\t\t\treturn Event{AltUp, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Up, 0, nil}\n\t\tcase 'Z':\n\t\t\treturn Event{ShiftTab, 0, nil}\n\t\tcase 'H':\n\t\t\treturn Event{Home, 0, nil}\n\t\tcase 'F':\n\t\t\treturn Event{End, 0, nil}\n\t\tcase '<':\n\t\t\treturn r.mouseSequence(sz)\n\t\tcase 'P':\n\t\t\treturn Event{F1, 0, nil}\n\t\tcase 'Q':\n\t\t\treturn Event{F2, 0, nil}\n\t\tcase 'R':\n\t\t\treturn Event{F3, 0, nil}\n\t\tcase 'S':\n\t\t\treturn Event{F4, 0, nil}\n\t\tcase '1', '2', '3', '4', '5', '6', '7', '8':\n\t\t\tif len(r.buffer) < 4 {\n\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\t}\n\t\t\t*sz = 4\n\t\t\tswitch r.buffer[2] {\n\t\t\tcase '2':\n\t\t\t\tif r.buffer[3] == '~' {\n\t\t\t\t\treturn Event{Insert, 0, nil}\n\t\t\t\t}\n\t\t\t\tif len(r.buffer) > 4 && r.buffer[4] == '~' {\n\t\t\t\t\t*sz = 5\n\t\t\t\t\tswitch r.buffer[3] {\n\t\t\t\t\tcase '0':\n\t\t\t\t\t\treturn Event{F9, 0, nil}\n\t\t\t\t\tcase '1':\n\t\t\t\t\t\treturn Event{F10, 0, nil}\n\t\t\t\t\tcase '3':\n\t\t\t\t\t\treturn Event{F11, 0, nil}\n\t\t\t\t\tcase '4':\n\t\t\t\t\t\treturn Event{F12, 0, nil}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Bracketed paste mode: \\e[200~ ... \\e[201~\n\t\t\t\tif len(r.buffer) > 5 && r.buffer[3] == '0' && (r.buffer[4] == '0' || r.buffer[4] == '1') && r.buffer[5] == '~' {\n\t\t\t\t\t// Immediately discard the sequence from the buffer and reread input\n\t\t\t\t\tr.buffer = r.buffer[6:]\n\t\t\t\t\t*sz = 0\n\t\t\t\t\treturn r.GetChar()\n\t\t\t\t}\n\t\t\t\treturn Event{Invalid, 0, nil} // INS\n\t\t\tcase '3':\n\t\t\t\tif r.buffer[3] == '~' {\n\t\t\t\t\treturn Event{Delete, 0, nil}\n\t\t\t\t}\n\t\t\t\tif len(r.buffer) == 6 && r.buffer[5] == '~' {\n\t\t\t\t\t*sz = 6\n\t\t\t\t\tswitch r.buffer[4] {\n\t\t\t\t\tcase '5':\n\t\t\t\t\t\treturn Event{CtrlDelete, 0, nil}\n\t\t\t\t\tcase '2':\n\t\t\t\t\t\treturn Event{ShiftDelete, 0, nil}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\tcase '4':\n\t\t\t\treturn Event{End, 0, nil}\n\t\t\tcase '5':\n\t\t\t\treturn Event{PageUp, 0, nil}\n\t\t\tcase '6':\n\t\t\t\treturn Event{PageDown, 0, nil}\n\t\t\tcase '7':\n\t\t\t\treturn Event{Home, 0, nil}\n\t\t\tcase '8':\n\t\t\t\treturn Event{End, 0, nil}\n\t\t\tcase '1':\n\t\t\t\tswitch r.buffer[3] {\n\t\t\t\tcase '~':\n\t\t\t\t\treturn Event{Home, 0, nil}\n\t\t\t\tcase '1', '2', '3', '4', '5', '7', '8', '9':\n\t\t\t\t\tif len(r.buffer) == 5 && r.buffer[4] == '~' {\n\t\t\t\t\t\t*sz = 5\n\t\t\t\t\t\tswitch r.buffer[3] {\n\t\t\t\t\t\tcase '1':\n\t\t\t\t\t\t\treturn Event{F1, 0, nil}\n\t\t\t\t\t\tcase '2':\n\t\t\t\t\t\t\treturn Event{F2, 0, nil}\n\t\t\t\t\t\tcase '3':\n\t\t\t\t\t\t\treturn Event{F3, 0, nil}\n\t\t\t\t\t\tcase '4':\n\t\t\t\t\t\t\treturn Event{F4, 0, nil}\n\t\t\t\t\t\tcase '5':\n\t\t\t\t\t\t\treturn Event{F5, 0, nil}\n\t\t\t\t\t\tcase '7':\n\t\t\t\t\t\t\treturn Event{F6, 0, nil}\n\t\t\t\t\t\tcase '8':\n\t\t\t\t\t\t\treturn Event{F7, 0, nil}\n\t\t\t\t\t\tcase '9':\n\t\t\t\t\t\t\treturn Event{F8, 0, nil}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\t\tcase ';':\n\t\t\t\t\tif len(r.buffer) < 6 {\n\t\t\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\t\t\t}\n\t\t\t\t\t*sz = 6\n\t\t\t\t\tswitch r.buffer[4] {\n\t\t\t\t\tcase '1', '2', '3', '4', '5':\n\t\t\t\t\t\t//                   Kitty      iTerm2     WezTerm\n\t\t\t\t\t\t// SHIFT-ARROW       \"\\e[1;2D\"\n\t\t\t\t\t\t// ALT-SHIFT-ARROW   \"\\e[1;4D\"  \"\\e[1;10D\" \"\\e[1;4D\"\n\t\t\t\t\t\t// CTRL-SHIFT-ARROW  \"\\e[1;6D\"             N/A\n\t\t\t\t\t\t// CMD-SHIFT-ARROW   \"\\e[1;10D\" N/A        N/A (\"\\e[1;2D\")\n\t\t\t\t\t\talt := r.buffer[4] == '3'\n\t\t\t\t\t\tchar := r.buffer[5]\n\t\t\t\t\t\taltShift := false\n\t\t\t\t\t\tif r.buffer[4] == '1' && r.buffer[5] == '0' {\n\t\t\t\t\t\t\taltShift = true\n\t\t\t\t\t\t\tif len(r.buffer) < 7 {\n\t\t\t\t\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t*sz = 7\n\t\t\t\t\t\t\tchar = r.buffer[6]\n...\n", "needle_token_start": 3818, "needle_token_end": 3883, "code_context_ntokens": 15244, "output": ["```go\nfunc newRange(begin int, end int) Range {\n\tif begin == 1 {\n\t\tbegin = rangeEllipsis\n\t}\n\tif end == -1 {\n\t\tend = rangeEllipsis\n\t}\n\treturn Range{begin, end}\n}\n```"]}
{"repo": "junegunn/fzf", "name": "EmptyMerger", "language": "go", "path": "src/merger.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a merger object that contains no data, typically used in scenarios where a default or placeholder merger is needed.\n2. **Input**: The function takes a single integer parameter that represents the revision number of the merger.\n3. **Output**: It returns a pointer to a merger object that has been initialized without any data.\n4. **Procedure**: The function initializes a merger object by calling another function with `nil` data and empty results, along with flags set to `false`, and the provided revision number.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/result.go\npackage fzf\n\nimport (\n\t\"math\"\n\t\"sort\"\n\t\"unicode\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// Offset holds two 32-bit integers denoting the offsets of a matched substring\ntype Offset [2]int32\n\ntype colorOffset struct {\n\toffset [2]int32\n\tcolor  tui.ColorPair\n}\n\ntype Result struct {\n\titem   *Item\n\tpoints [4]uint16\n}\n\nfunc buildResult(item *Item, offsets []Offset, score int) Result {\n\tif len(offsets) > 1 {\n\t\tsort.Sort(ByOrder(offsets))\n\t}\n\n\tresult := Result{item: item}\n\tnumChars := item.text.Length()\n\tminBegin := math.MaxUint16\n\tminEnd := math.MaxUint16\n\tmaxEnd := 0\n\tvalidOffsetFound := false\n\tfor _, offset := range offsets {\n\t\tb, e := int(offset[0]), int(offset[1])\n\t\tif b < e {\n\t\t\tminBegin = util.Min(b, minBegin)\n\t\t\tminEnd = util.Min(e, minEnd)\n\t\t\tmaxEnd = util.Max(e, maxEnd)\n\t\t\tvalidOffsetFound = true\n\t\t}\n\t}\n\n\tfor idx, criterion := range sortCriteria {\n\t\tval := uint16(math.MaxUint16)\n\t\tswitch criterion {\n\t\tcase byScore:\n\t\t\t// Higher is better\n\t\t\tval = math.MaxUint16 - util.AsUint16(score)\n\t\tcase byChunk:\n\t\t\tif validOffsetFound {\n\t\t\t\tb := minBegin\n\t\t\t\te := maxEnd\n\t\t\t\tfor ; b >= 1; b-- {\n\t\t\t\t\tif unicode.IsSpace(item.text.Get(b - 1)) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor ; e < numChars; e++ {\n\t\t\t\t\tif unicode.IsSpace(item.text.Get(e)) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tval = util.AsUint16(e - b)\n\t\t\t}\n\t\tcase byLength:\n\t\t\tval = item.TrimLength()\n\t\tcase byBegin, byEnd:\n\t\t\tif validOffsetFound {\n\t\t\t\twhitePrefixLen := 0\n\t\t\t\tfor idx := 0; idx < numChars; idx++ {\n\t\t\t\t\tr := item.text.Get(idx)\n\t\t\t\t\twhitePrefixLen = idx\n\t\t\t\t\tif idx == minBegin || !unicode.IsSpace(r) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif criterion == byBegin {\n\t\t\t\t\tval = util.AsUint16(minEnd - whitePrefixLen)\n\t\t\t\t} else {\n\t\t\t\t\tval = util.AsUint16(math.MaxUint16 - math.MaxUint16*(maxEnd-whitePrefixLen)/int(item.TrimLength()+1))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tresult.points[3-idx] = val\n\t}\n\n\treturn result\n}\n\n// Sort criteria to use. Never changes once fzf is started.\nvar sortCriteria []criterion\n\n// Index returns ordinal index of the Item\nfunc (result *Result) Index() int32 {\n\treturn result.item.Index()\n}\n\nfunc minRank() Result {\n\treturn Result{item: &minItem, points: [4]uint16{math.MaxUint16, 0, 0, 0}}\n}\n\n...\n// Path: src/pattern.go\npackage fzf\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/junegunn/fzf/src/algo\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// fuzzy\n// 'exact\n// ^prefix-exact\n// suffix-exact$\n// !inverse-exact\n// !'inverse-fuzzy\n// !^inverse-prefix-exact\n// !inverse-suffix-exact$\n\ntype termType int\n\nconst (\n\ttermFuzzy termType = iota\n\ttermExact\n\ttermPrefix\n\ttermSuffix\n\ttermEqual\n)\n\ntype term struct {\n\ttyp           termType\n\tinv           bool\n\ttext          []rune\n\tcaseSensitive bool\n\tnormalize     bool\n}\n\n// String returns the string representation of a term.\nfunc (t term) String() string {\n\treturn fmt.Sprintf(\"term{typ: %d, inv: %v, text: []rune(%q), caseSensitive: %v}\", t.typ, t.inv, string(t.text), t.caseSensitive)\n}\n\ntype termSet []term\n\n// Pattern represents search pattern\ntype Pattern struct {\n\tfuzzy         bool\n\tfuzzyAlgo     algo.Algo\n\textended      bool\n\tcaseSensitive bool\n\tnormalize     bool\n\tforward       bool\n\twithPos       bool\n\ttext          []rune\n\ttermSets      []termSet\n\tsortable      bool\n\tcacheable     bool\n\tcacheKey      string\n\tdelimiter     Delimiter\n\tnth           []Range\n\tprocFun       map[termType]algo.Algo\n}\n\nvar (\n\t_patternCache map[string]*Pattern\n\t_splitRegex   *regexp.Regexp\n\t_cache        ChunkCache\n)\n\nfunc init() {\n\t_splitRegex = regexp.MustCompile(\" +\")\n\tclearPatternCache()\n\tclearChunkCache()\n}\n\nfunc clearPatternCache() {\n\t// We can uniquely identify the pattern for a given string since\n\t// search mode and caseMode do not change while the program is running\n\t_patternCache = make(map[string]*Pattern)\n}\n\nfunc clearChunkCache() {\n\t_cache = NewChunkCache()\n}\n\n// BuildPattern builds Pattern object from the given arguments\nfunc BuildPattern(fuzzy bool, fuzzyAlgo algo.Algo, extended bool, caseMode Case, normalize bool, forward bool,\n\twithPos bool, cacheable bool, nth []Range, delimiter Delimiter, runes []rune) *Pattern {\n\n\tvar asString string\n\tif extended {\n\t\tasString = strings.TrimLeft(string(runes), \" \")\n\t\tfor strings.HasSuffix(asString, \" \") && !strings.HasSuffix(asString, \"\\\\ \") {\n\t\t\tasString = asString[:len(asString)-1]\n\t\t}\n\t} else {\n\t\tasString = string(runes)\n\t}\n\n\tcached, found := _patternCache[asString]\n\tif found {\n\t\treturn cached\n\t}\n\n\tcaseSensitive := true\n\tsortable := true\n\ttermSets := []termSet{}\n\n\tif extended {\n\t\ttermSets = parseTerms(fuzzy, caseMode, normalize, asString)\n\t\t// We should not sort the result if there are only inverse search terms\n\t\tsortable = false\n\tLoop:\n\t\tfor _, termSet := range termSets {\n\t\t\tfor idx, term := range termSet {\n\t\t\t\tif !term.inv {\n\t\t\t\t\tsortable = true\n\t\t\t\t}\n\t\t\t\t// If the query contains inverse search terms or OR operators,\n\t\t\t\t// we cannot cache the search scope\n\t\t\t\tif !cacheable || idx > 0 || term.inv || fuzzy && term.typ != termFuzzy || !fuzzy && term.typ != termExact {\n\t\t\t\t\tcacheable = false\n\t\t\t\t\tif sortable {\n\t\t\t\t\t\t// Can't break until we see at least one non-inverse term\n\t\t\t\t\t\tbreak Loop\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\tlowerString := strings.ToLower(asString)\n\t\tnormalize = normalize &&\n\t\t\tlowerString == string(algo.NormalizeRunes([]rune(lowerString)))\n\t\tcaseSensitive = caseMode == CaseRespect ||\n\t\t\tcaseMode == CaseSmart && lowerString != asString\n\t\tif !caseSensitive {\n\t\t\tasString = lowerString\n\t\t}\n\t}\n\n\tptr := &Pattern{\n\t\tfuzzy:         fuzzy,\n\t\tfuzzyAlgo:     fuzzyAlgo,\n\t\textended:      extended,\n\t\tcaseSensitive: caseSensitive,\n\t\tnormalize:     normalize,\n\t\tforward:       forward,\n\t\twithPos:       withPos,\n\t\ttext:          []rune(asString),\n\t\ttermSets:      termSets,\n\t\tsortable:      sortable,\n\t\tcacheable:     cacheable,\n\t\tnth:           nth,\n\t\tdelimiter:     delimiter,\n\t\tprocFun:       make(map[termType]algo.Algo)}\n\n\tptr.cacheKey = ptr.buildCacheKey()\n\tptr.procFun[termFuzzy] = fuzzyAlgo\n\tptr.procFun[termEqual] = algo.EqualMatch\n\tptr.procFun[termExact] = algo.ExactMatchNaive\n\tptr.procFun[termPrefix] = algo.PrefixMatch\n\tptr.procFun[termSuffix] = algo.SuffixMatch\n\n\t_patternCache[asString] = ptr\n\treturn ptr\n}\n\nfunc parseTerms(fuzzy bool, caseMode Case, normalize bool, str string) []termSet {\n\tstr = strings.Replace(str, \"\\\\ \", \"\\t\", -1)\n\ttokens := _splitRegex.Split(str, -1)\n\tsets := []termSet{}\n\tset := termSet{}\n\tswitchSet := false\n\tafterBar := false\n\tfor _, token := range tokens {\n\t\ttyp, inv, text := termFuzzy, false, strings.Replace(token, \"\\t\", \" \", -1)\n\t\tlowerText := strings.ToLower(text)\n\t\tcaseSensitive := caseMode == CaseRespect ||\n\t\t\tcaseMode == CaseSmart && text != lowerText\n\t\tnormalizeTerm := normalize &&\n\t\t\tlowerText == string(algo.NormalizeRunes([]rune(lowerText)))\n\t\tif !caseSensitive {\n\t\t\ttext = lowerText\n\t\t}\n\t\tif !fuzzy {\n\t\t\ttyp = termExact\n\t\t}\n\n\t\tif len(set) > 0 && !afterBar && text == \"|\" {\n\t\t\tswitchSet = false\n\t\t\tafterBar = true\n\t\t\tcontinue\n\t\t}\n\t\tafterBar = false\n\n\t\tif strings.HasPrefix(text, \"!\") {\n\t\t\tinv = true\n\t\t\ttyp = termExact\n\t\t\ttext = text[1:]\n\t\t}\n\n\t\tif text != \"$\" && strings.HasSuffix(text, \"$\") {\n\t\t\ttyp = termSuffix\n\t\t\ttext = text[:len(text)-1]\n\t\t}\n\n\t\tif strings.HasPrefix(text, \"'\") {\n\t\t\t// Flip exactness\n\t\t\tif fuzzy && !inv {\n\t\t\t\ttyp = termExact\n\t\t\t} else {\n\t\t\t\ttyp = termFuzzy\n\t\t\t}\n\t\t\ttext = text[1:]\n\t\t} else if strings.HasPrefix(text, \"^\") {\n\t\t\tif typ == termSuffix {\n\t\t\t\ttyp = termEqual\n\t\t\t} else {\n\t\t\t\ttyp = termPrefix\n\t\t\t}\n\t\t\ttext = text[1:]\n\t\t}\n\n\t\tif len(text) > 0 {\n\t\t\tif switchSet {\n\t\t\t\tsets = append(sets, set)\n\t\t\t\tset = termSet{}\n\t\t\t}\n\t\t\ttextRunes := []rune(text)\n\t\t\tif normalizeTerm {\n\t\t\t\ttextRunes = algo.NormalizeRunes(textRunes)\n\t\t\t}\n\t\t\tset = append(set, term{\n\t\t\t\ttyp:           typ,\n\t\t\t\tinv:           inv,\n\t\t\t\ttext:          textRunes,\n\t\t\t\tcaseSensitive: caseSensitive,\n\t\t\t\tnormalize:     normalizeTerm})\n\t\t\tswitchSet = true\n\t\t}\n\t}\n\tif len(set) > 0 {\n\t\tsets = append(sets, set)\n\t}\n\treturn sets\n}\n\n// IsEmpty returns true if the pattern is effectively empty\nfunc (p *Pattern) IsEmpty() bool {\n\tif !p.extended {\n\t\treturn len(p.text) == 0\n\t}\n\treturn len(p.termSets) == 0\n}\n\n// AsString returns the search query in string type\nfunc (p *Pattern) AsString() string {\n\treturn string(p.text)\n}\n\nfunc (p *Pattern) buildCacheKey() string {\n\tif !p.extended {\n\t\treturn p.AsString()\n\t}\n\tcacheableTerms := []string{}\n\tfor _, termSet := range p.termSets {\n\t\tif len(termSet) == 1 && !termSet[0].inv && (p.fuzzy || termSet[0].typ == termExact) {\n\t\t\tcacheableTerms = append(cacheableTerms, string(termSet[0].text))\n\t\t}\n\t}\n\treturn strings.Join(cacheableTerms, \"\\t\")\n}\n\n// CacheKey is used to build string to be used as the key of result cache\nfunc (p *Pattern) CacheKey() string {\n\treturn p.cacheKey\n}\n\n// Match returns the list of matches Items in the given Chunk\nfunc (p *Pattern) Match(chunk *Chunk, slab *util.Slab) []Result {\n\t// ChunkCache: Exact match\n\tcacheKey := p.CacheKey()\n\tif p.cacheable {\n\t\tif cached := _cache.Lookup(chunk, cacheKey); cached != nil {\n\t\t\treturn cached\n\t\t}\n\t}\n\n\t// Prefix/suffix cache\n\tspace := _cache.Search(chunk, cacheKey)\n\n\tmatches := p.matchChunk(chunk, space, slab)\n\n\tif p.cacheable {\n\t\t_cache.Add(chunk, cacheKey, matches)\n\t}\n\treturn matches\n}\n\nfunc (p *Pattern) matchChunk(chunk *Chunk, space []Result, slab *util.Slab) []Result {\n\tmatches := []Result{}\n\n\tif space == nil {\n\t\tfor idx := 0; idx < chunk.count; idx++ {\n\t\t\tif match, _, _ := p.MatchItem(&chunk.items[idx], p.withPos, slab); match != nil {\n\t\t\t\tmatches = append(matches, *match)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tfor _, result := range space {\n\t\t\tif match, _, _ := p.MatchItem(result.item, p.withPos, slab); match != nil {\n\t\t\t\tmatches = append(matches, *match)\n\t\t\t}\n\t\t}\n\t}\n\treturn matches\n}\n\n// MatchItem returns true if the Item is a match\nfunc (p *Pattern) MatchItem(item *Item, withPos bool, slab *util.Slab) (*Result, []Offset, *[]int) {\n\tif p.extended {\n\t\tif offsets, bonus, pos := p.extendedMatch(item, withPos, slab); len(offsets) == len(p.termSets) {\n\t\t\tresult := buildResult(item, offsets, bonus)\n\t\t\treturn &result, offsets, pos\n\t\t}\n\t\treturn nil, nil, nil\n\t}\n\toffset, bonus, pos := p.basicMatch(item, withPos, slab)\n\tif sidx := offset[0]; sidx >= 0 {\n\t\toffsets := []Offset{offset}\n\t\tresult := buildResult(item, offsets, bonus)\n\t\treturn &result, offsets, pos\n\t}\n\treturn nil, nil, nil\n}\n\nfunc (p *Pattern) basicMatch(item *Item, withPos bool, slab *util.Slab) (Offset, int, *[]int) {\n\tvar input []Token\n\tif len(p.nth) == 0 {\n\t\tinput = []Token{{text: &item.text, prefixLength: 0}}\n\t} else {\n\t\tinput = p.transformInput(item)\n\t}\n\tif p.fuzzy {\n\t\treturn p.iter(p.fuzzyAlgo, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)\n\t}\n\treturn p.iter(algo.ExactMatchNaive, input, p.caseSensitive, p.normalize, p.forward, p.text, withPos, slab)\n}\n\nfunc (p *Pattern) extendedMatch(item *Item, withPos bool, slab *util.Slab) ([]Offset, int, *[]int) {\n\tvar input []Token\n\tif len(p.nth) == 0 {\n\t\tinput = []Token{{text: &item.text, prefixLength: 0}}\n\t} else {\n\t\tinput = p.transformInput(item)\n\t}\n\toffsets := []Offset{}\n\tvar totalScore int\n\tvar allPos *[]int\n\tif withPos {\n\t\tallPos = &[]int{}\n\t}\n\tfor _, termSet := range p.termSets {\n\t\tvar offset Offset\n\t\tvar currentScore int\n\t\tmatched := false\n\t\tfor _, term := range termSet {\n\t\t\tpfun := p.procFun[term.typ]\n\t\t\toff, score, pos := p.iter(pfun, input, term.caseSensitive, term.normalize, p.forward, term.text, withPos, slab)\n\t\t\tif sidx := off[0]; sidx >= 0 {\n\t\t\t\tif term.inv {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\toffset, currentScore = off, score\n\t\t\t\tmatched = true\n\t\t\t\tif withPos {\n\t\t\t\t\tif pos != nil {\n\t\t\t\t\t\t*allPos = append(*allPos, *pos...)\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfor idx := off[0]; idx < off[1]; idx++ {\n\t\t\t\t\t\t\t*allPos = append(*allPos, int(idx))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t} else if term.inv {\n\t\t\t\toffset, currentScore = Offset{0, 0}, 0\n\t\t\t\tmatched = true\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif matched {\n\t\t\toffsets = append(offsets, offset)\n\t\t\ttotalScore += currentScore\n\t\t}\n\t}\n\treturn offsets, totalScore, allPos\n}\n\nfunc (p *Pattern) transformInput(item *Item) []Token {\n\tif item.transformed != nil {\n\t\treturn *item.transformed\n\t}\n\n\ttokens := Tokenize(item.text.ToString(), p.delimiter)\n\tret := Transform(tokens, p.nth)\n\titem.transformed = &ret\n\treturn ret\n}\n\nfunc (p *Pattern) iter(pfun algo.Algo, tokens []Token, caseSensitive bool, normalize bool, forward bool, pattern []rune, withPos bool, slab *util.Slab) (Offset, int, *[]int) {\n\tfor _, part := range tokens {\n\t\tif res, pos := pfun(caseSensitive, normalize, forward, part.text, pattern, withPos, slab); res.Start >= 0 {\n\t\t\tsidx := int32(res.Start) + part.prefixLength\n\t\t\teidx := int32(res.End) + part.prefixLength\n\t\t\tif pos != nil {\n\t\t\t\tfor idx := range *pos {\n\t\t\t\t\t(*pos)[idx] += int(part.prefixLength)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn Offset{sidx, eidx}, res.Score, pos\n\t\t}\n\t}\n\treturn Offset{-1, -1}, 0, nil\n}\n\n// Path: src/merger.go\npackage fzf\n\nimport \"fmt\"\n\n// EmptyMerger is a Merger with no data\n\nfunc EmptyMerger(revision int) *Merger {\n\treturn NewMerger(nil, [][]Result{}, false, false, revision)\n}\n\n// Merger holds a set of locally sorted lists of items and provides the view of\n// a single, globally-sorted list\ntype Merger struct {\n\tpattern  *Pattern\n\tlists    [][]Result\n\tmerged   []Result\n\tchunks   *[]*Chunk\n\tcursors  []int\n\tsorted   bool\n\ttac      bool\n\tfinal    bool\n\tcount    int\n\tpass     bool\n\trevision int\n}\n\n// PassMerger returns a new Merger that simply returns the items in the\n// original order\nfunc PassMerger(chunks *[]*Chunk, tac bool, revision int) *Merger {\n\tmg := Merger{\n\t\tpattern:  nil,\n\t\tchunks:   chunks,\n\t\ttac:      tac,\n\t\tcount:    0,\n\t\tpass:     true,\n\t\trevision: revision}\n\n\tfor _, chunk := range *mg.chunks {\n\t\tmg.count += chunk.count\n\t}\n\treturn &mg\n}\n\n// NewMerger returns a new Merger\nfunc NewMerger(pattern *Pattern, lists [][]Result, sorted bool, tac bool, revision int) *Merger {\n\tmg := Merger{\n\t\tpattern:  pattern,\n\t\tlists:    lists,\n\t\tmerged:   []Result{},\n\t\tchunks:   nil,\n\t\tcursors:  make([]int, len(lists)),\n\t\tsorted:   sorted,\n\t\ttac:      tac,\n\t\tfinal:    false,\n\t\tcount:    0,\n\t\trevision: revision}\n\n\tfor _, list := range mg.lists {\n\t\tmg.count += len(list)\n\t}\n\treturn &mg\n}\n\n// Revision returns revision number\nfunc (mg *Merger) Revision() int {\n\treturn mg.revision\n}\n\n// Length returns the number of items\nfunc (mg *Merger) Length() int {\n\treturn mg.count\n}\n\nfunc (mg *Merger) First() Result {\n\tif mg.tac && !mg.sorted {\n\t\treturn mg.Get(mg.count - 1)\n\t}\n\treturn mg.Get(0)\n}\n\n// FindIndex returns the index of the item with the given item index\nfunc (mg *Merger) FindIndex(itemIndex int32) int {\n\tindex := -1\n\tif mg.pass {\n\t\tindex = int(itemIndex)\n\t\tif mg.tac {\n\t\t\tindex = mg.count - index - 1\n\t\t}\n\t} else {\n\t\tfor i := 0; i < mg.count; i++ {\n\t\t\tif mg.Get(i).item.Index() == itemIndex {\n\t\t\t\tindex = i\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn index\n}\n\n// Get returns the pointer to the Result object indexed by the given integer\nfunc (mg *Merger) Get(idx int) Result {\n\tif mg.chunks != nil {\n\t\tif mg.tac {\n\t\t\tidx = mg.count - idx - 1\n\t\t}\n\t\tchunk := (*mg.chunks)[idx/chunkSize]\n\t\treturn Result{item: &chunk.items[idx%chunkSize]}\n\t}\n\n\tif mg.sorted {\n\t\treturn mg.mergedGet(idx)\n\t}\n\n\tif mg.tac {\n\t\tidx = mg.count - idx - 1\n\t}\n\tfor _, list := range mg.lists {\n\t\tnumItems := len(list)\n\t\tif idx < numItems {\n\t\t\treturn list[idx]\n\t\t}\n\t\tidx -= numItems\n\t}\n\tpanic(fmt.Sprintf(\"Index out of bounds (unsorted, %d/%d)\", idx, mg.count))\n}\n\nfunc (mg *Merger) cacheable() bool {\n\treturn mg.count < mergerCacheMax\n}\n\nfunc (mg *Merger) mergedGet(idx int) Result {\n\tfor i := len(mg.merged); i <= idx; i++ {\n\t\tminRank := minRank()\n\t\tminIdx := -1\n\t\tfor listIdx, list := range mg.lists {\n\t\t\tcursor := mg.cursors[listIdx]\n\t\t\tif cursor < 0 || cursor == len(list) {\n\t\t\t\tmg.cursors[listIdx] = -1\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif cursor >= 0 {\n\t\t\t\trank := list[cursor]\n\t\t\t\tif minIdx < 0 || compareRanks(rank, minRank, mg.tac) {\n\t\t\t\t\tminRank = rank\n\t\t\t\t\tminIdx = listIdx\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif minIdx >= 0 {\n\t\t\tchosen := mg.lists[minIdx]\n\t\t\tmg.merged = append(mg.merged, chosen[mg.cursors[minIdx]])\n\t\t\tmg.cursors[minIdx]++\n\t\t} else {\n\t\t\tpanic(fmt.Sprintf(\"Index out of bounds (sorted, %d/%d)\", i, mg.count))\n\t\t}\n\t}\n\treturn mg.merged[idx]\n}\n\n// Path: src/reader.go\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/charlievieth/fastwalk\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// Reader reads from command or standard input\ntype Reader struct {\n\tpusher   func([]byte) bool\n\teventBox *util.EventBox\n\tdelimNil bool\n\tevent    int32\n\tfinChan  chan bool\n\tmutex    sync.Mutex\n\texec     *exec.Cmd\n\tcommand  *string\n\tkilled   bool\n\twait     bool\n}\n\n// NewReader returns new Reader object\nfunc NewReader(pusher func([]byte) bool, eventBox *util.EventBox, delimNil bool, wait bool) *Reader {\n\treturn &Reader{pusher, eventBox, delimNil, int32(EvtReady), make(chan bool, 1), sync.Mutex{}, nil, nil, false, wait}\n}\n\nfunc (r *Reader) startEventPoller() {\n\tgo func() {\n\t\tptr := &r.event\n\t\tpollInterval := readerPollIntervalMin\n\t\tfor {\n\t\t\tif atomic.CompareAndSwapInt32(ptr, int32(EvtReadNew), int32(EvtReady)) {\n\t\t\t\tr.eventBox.Set(EvtReadNew, (*string)(nil))\n\t\t\t\tpollInterval = readerPollIntervalMin\n\t\t\t} else if atomic.LoadInt32(ptr) == int32(EvtReadFin) {\n\t\t\t\tif r.wait {\n\t\t\t\t\tr.finChan <- true\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tpollInterval += readerPollIntervalStep\n\t\t\t\tif pollInterval > readerPollIntervalMax {\n\t\t\t\t\tpollInterval = readerPollIntervalMax\n\t\t\t\t}\n\t\t\t}\n\t\t\ttime.Sleep(pollInterval)\n\t\t}\n\t}()\n}\n\nfunc (r *Reader) fin(success bool) {\n\tatomic.StoreInt32(&r.event, int32(EvtReadFin))\n\tif r.wait {\n\t\t<-r.finChan\n\t}\n\n\tr.mutex.Lock()\n\tret := r.command\n\tif success || r.killed {\n\t\tret = nil\n\t}\n\tr.mutex.Unlock()\n\n\tr.eventBox.Set(EvtReadFin, ret)\n}\n\nfunc (r *Reader) terminate() {\n\tr.mutex.Lock()\n\tr.killed = true\n\tif r.exec != nil && r.exec.Process != nil {\n\t\tutil.KillCommand(r.exec)\n\t} else {\n\t\tos.Stdin.Close()\n\t}\n\tr.mutex.Unlock()\n}\n\nfunc (r *Reader) restart(command string, environ []string) {\n\tr.event = int32(EvtReady)\n\tr.startEventPoller()\n\tsuccess := r.readFromCommand(command, environ)\n\tr.fin(success)\n}\n\n// ReadSource reads data from the default command or from standard input\nfunc (r *Reader) ReadSource(root string, opts walkerOpts, ignores []string) {\n\tr.startEventPoller()\n\tvar success bool\n\tif util.IsTty() {\n\t\tcmd := os.Getenv(\"FZF_DEFAULT_COMMAND\")\n\t\tif len(cmd) == 0 {\n\t\t\tsuccess = r.readFiles(root, opts, ignores)\n\t\t} else {\n\t\t\t// We can't export FZF_* environment variables to the default command\n\t\t\tsuccess = r.readFromCommand(cmd, nil)\n\t\t}\n\t} else {\n\t\tsuccess = r.readFromStdin()\n\t}\n\tr.fin(success)\n}\n\nfunc (r *Reader) feed(src io.Reader) {\n\t/*\n\t\treaderSlabSize, ae := strconv.Atoi(os.Getenv(\"SLAB_KB\"))\n\t\tif ae != nil {\n\t\t\treaderSlabSize = 128 * 1024\n\t\t} else {\n\t\t\treaderSlabSize *= 1024\n\t\t}\n\t\treaderBufferSize, be := strconv.Atoi(os.Getenv(\"BUF_KB\"))\n\t\tif be != nil {\n\t\t\treaderBufferSize = 64 * 1024\n\t\t} else {\n\t\t\treaderBufferSize *= 1024\n\t\t}\n\t*/\n\n\tdelim := byte('\\n')\n\ttrimCR := util.IsWindows()\n\tif r.delimNil {\n\t\tdelim = '\\000'\n\t\ttrimCR = false\n\t}\n\n\tslab := make([]byte, readerSlabSize)\n\tleftover := []byte{}\n\tvar err error\n\tfor {\n\t\tn := 0\n\t\tscope := slab[:util.Min(len(slab), readerBufferSize)]\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tn, err = src.Read(scope)\n\t\t\tif n > 0 || err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// We're not making any progress after 100 tries. Stop.\n\t\tif n == 0 && err == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tbuf := slab[:n]\n\t\tslab = slab[n:]\n\n\t\tfor len(buf) > 0 {\n\t\t\tif i := bytes.IndexByte(buf, delim); i >= 0 {\n\t\t\t\t// Found the delimiter\n\t\t\t\tslice := buf[:i+1]\n\t\t\t\tbuf = buf[i+1:]\n\t\t\t\tif trimCR && len(slice) >= 2 && slice[len(slice)-2] == byte('\\r') {\n\t\t\t\t\tslice = slice[:len(slice)-2]\n\t\t\t\t} else {\n\t\t\t\t\tslice = slice[:len(slice)-1]\n\t\t\t\t}\n\t\t\t\tif len(leftover) > 0 {\n\t\t\t\t\tslice = append(leftover, slice...)\n\t\t\t\t\tleftover = []byte{}\n\t\t\t\t}\n\t\t\t\tif (err == nil || len(slice) > 0) && r.pusher(slice) {\n\t\t\t\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Could not find the delimiter in the buffer\n\t\t\t\t//   NOTE: We can further optimize this by keeping track of the cursor\n\t\t\t\t//   position in the slab so that a straddling item that doesn't go\n\t\t\t\t//   beyond the boundary of a slab doesn't need to be copied to\n\t\t\t\t//   another buffer. However, the performance gain is negligible in\n\t\t\t\t//   practice (< 0.1%) and is not\n\t\t\t\t//   worth the added complexity.\n\t\t\t\tleftover = append(leftover, buf...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif err == io.EOF {\n\t\t\tleftover = append(leftover, buf...)\n\t\t\tbreak\n\t\t}\n\n\t\tif len(slab) == 0 {\n\t\t\tslab = make([]byte, readerSlabSize)\n\t\t}\n\t}\n\tif len(leftover) > 0 && r.pusher(leftover) {\n\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t}\n}\n\nfunc (r *Reader) readFromStdin() bool {\n\tr.feed(os.Stdin)\n\treturn true\n}\n\nfunc (r *Reader) readFiles(root string, opts walkerOpts, ignores []string) bool {\n\tr.killed = false\n\tconf := fastwalk.Config{Follow: opts.follow}\n\tfn := func(path string, de os.DirEntry, err error) error {\n\t\tif err != nil {\n\t\t\treturn nil\n\t\t}\n\t\tpath = filepath.Clean(path)\n\t\tif path != \".\" {\n\t\t\tisDir := de.IsDir()\n\t\t\tif isDir {\n\t\t\t\tbase := filepath.Base(path)\n\t\t\t\tif !opts.hidden && base[0] == '.' {\n\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t}\n\t\t\t\tfor _, ignore := range ignores {\n\t\t\t\t\tif ignore == base {\n\t\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif ((opts.file && !isDir) || (opts.dir && isDir)) && r.pusher([]byte(path)) {\n\t\t\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t\t\t}\n\t\t}\n\t\tr.mutex.Lock()\n\t\tdefer r.mutex.Unlock()\n\t\tif r.killed {\n\t\t\treturn context.Canceled\n\t\t}\n\t\treturn nil\n\t}\n\treturn fastwalk.Walk(&conf, root, fn) == nil\n}\n\nfunc (r *Reader) readFromCommand(command string, environ []string) bool {\n\tr.mutex.Lock()\n\tr.killed = false\n\tr.command = &command\n\tr.exec = util.ExecCommand(command, true)\n\tif environ != nil {\n\t\tr.exec.Env = environ\n\t}\n\tout, err := r.exec.StdoutPipe()\n\tif err != nil {\n\t\tr.mutex.Unlock()\n\t\treturn false\n\t}\n\terr = r.exec.Start()\n\tr.mutex.Unlock()\n\tif err != nil {\n\t\treturn false\n\t}\n\tr.feed(out)\n\treturn r.exec.Wait() == nil\n}\n\n// Path: src/terminal.go\npackage fzf\n\nimport (\n\t\"bufio\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/rivo/uniseg\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// import \"github.com/pkg/profile\"\n\n/*\nPlaceholder regex is used to extract placeholders from fzf's template\nstrings. Acts as input validation for parsePlaceholder function.\nDescribes the syntax, but it is fairly lenient.\n\nThe following pseudo regex has been reverse engineered from the\nimplementation. It is overly strict, but better describes what's possible.\nAs such it is not useful for validation, but rather to generate test\ncases for example.\n\n\t\\\\?(?:                                      # escaped type\n\t    {\\+?s?f?RANGE(?:,RANGE)*}               # token type\n\t    |{q}                                    # query type\n\t    |{\\+?n?f?}                              # item type (notice no mandatory element inside brackets)\n\t)\n\tRANGE = (?:\n\t    (?:-?[0-9]+)?\\.\\.(?:-?[0-9]+)?          # ellipsis syntax for token range (x..y)\n\t    |-?[0-9]+                               # shorthand syntax (x..x)\n\t)\n*/\nvar placeholder *regexp.Regexp\nvar whiteSuffix *regexp.Regexp\nvar offsetComponentRegex *regexp.Regexp\nvar offsetTrimCharsRegex *regexp.Regexp\nvar activeTempFiles []string\nvar passThroughRegex *regexp.Regexp\nvar actionTypeRegex *regexp.Regexp\n\nconst clearCode string = \"\\x1b[2J\"\n\n// Number of maximum focus events to process synchronously\nconst maxFocusEvents = 10000\n\nfunc init() {\n\tplaceholder = regexp.MustCompile(`\\\\?(?:{[+sf]*[0-9,-.]*}|{q}|{fzf:(?:query|action|prompt)}|{\\+?f?nf?})`)\n\twhiteSuffix = regexp.MustCompile(`\\s*$`)\n\toffsetComponentRegex = regexp.MustCompile(`([+-][0-9]+)|(-?/[1-9][0-9]*)`)\n\toffsetTrimCharsRegex = regexp.MustCompile(`[^0-9/+-]`)\n\tactiveTempFiles = []string{}\n\n\t// Parts of the preview output that should be passed through to the terminal\n\t// * https://github.com/tmux/tmux/wiki/FAQ#what-is-the-passthrough-escape-sequence-and-how-do-i-use-it\n\t// * https://sw.kovidgoyal.net/kitty/graphics-protocol\n\t// * https://en.wikipedia.org/wiki/Sixel\n\t// * https://iterm2.com/documentation-images.html\n\tpassThroughRegex = regexp.MustCompile(`\\x1bPtmux;\\x1b\\x1b.*?[^\\x1b]\\x1b\\\\|\\x1b(_G|P[0-9;]*q).*?\\x1b\\\\\\r?|\\x1b]1337;.*?(\\a|\\x1b\\\\)`)\n}\n\ntype jumpMode int\n\nconst (\n\tjumpDisabled jumpMode = iota\n\tjumpEnabled\n\tjumpAcceptEnabled\n)\n\ntype resumableState int\n\nconst (\n\tdisabledState resumableState = iota\n\tpausedState\n\tenabledState\n)\n\nfunc (s resumableState) Enabled() bool {\n\treturn s == enabledState\n}\n\nfunc (s *resumableState) Force(flag bool) {\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = disabledState\n\t}\n}\n\nfunc (s *resumableState) Set(flag bool) {\n\tif *s == disabledState {\n\t\treturn\n\t}\n\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = pausedState\n\t}\n}\n\ntype previewer struct {\n\tversion    int64\n\tlines      []string\n\toffset     int\n\tscrollable bool\n\tfinal      bool\n\tfollowing  resumableState\n\tspinner    string\n\tbar        []bool\n}\n\ntype previewed struct {\n\tversion   int64\n\tnumLines  int\n\toffset    int\n\tfilled    bool\n\timage     bool\n\twipe      bool\n\twireframe bool\n}\n\ntype eachLine struct {\n\tline string\n\terr  error\n}\n\ntype itemLine struct {\n\toffset   int\n\tcurrent  bool\n\tselected bool\n\tlabel    string\n\tqueryLen int\n\twidth    int\n\tbar      bool\n\tresult   Result\n}\n\ntype fitpad struct {\n\tfit int\n\tpad int\n}\n\nvar emptyLine = itemLine{}\n\ntype labelPrinter func(tui.Window, int)\n\ntype StatusItem struct {\n\tIndex int    `json:\"index\"`\n\tText  string `json:\"text\"`\n}\n\ntype Status struct {\n\tReading    bool         `json:\"reading\"`\n\tProgress   int          `json:\"progress\"`\n\tQuery      string       `json:\"query\"`\n\tPosition   int          `json:\"position\"`\n\tSort       bool         `json:\"sort\"`\n\tTotalCount int          `json:\"totalCount\"`\n\tMatchCount int          `json:\"matchCount\"`\n\tCurrent    *StatusItem  `json:\"current\"`\n\tMatches    []StatusItem `json:\"matches\"`\n\tSelected   []StatusItem `json:\"selected\"`\n}\n\n// Terminal represents terminal input/output\ntype Terminal struct {\n\tinitDelay          time.Duration\n\tinfoStyle          infoStyle\n\tinfoPrefix         string\n\tseparator          labelPrinter\n\tseparatorLen       int\n\tspinner            []string\n\tpromptString       string\n\tprompt             func()\n\tpromptLen          int\n\tborderLabel        labelPrinter\n\tborderLabelLen     int\n\tborderLabelOpts    labelOpts\n\tpreviewLabel       labelPrinter\n\tpreviewLabelLen    int\n\tpreviewLabelOpts   labelOpts\n\tpointer            string\n\tpointerLen         int\n\tpointerEmpty       string\n\tmarker             string\n\tmarkerLen          int\n\tmarkerEmpty        string\n\tqueryLen           [2]int\n\tlayout             layoutType\n\tfullscreen         bool\n\tkeepRight          bool\n\thscroll            bool\n\thscrollOff         int\n\tscrollOff          int\n\twordRubout         string\n\twordNext           string\n\tcx                 int\n\tcy                 int\n\toffset             int\n\txoffset            int\n\tyanked             []rune\n\tinput              []rune\n\tmulti              int\n\tsort               bool\n\ttoggleSort         bool\n\ttrack              trackOption\n\tdelimiter          Delimiter\n\texpect             map[tui.Event]string\n\tkeymap             map[tui.Event][]*action\n\tkeymapOrg          map[tui.Event][]*action\n\tpressed            string\n\tprintQuery         bool\n\thistory            *History\n\tcycle              bool\n\theaderVisible      bool\n\theaderFirst        bool\n\theaderLines        int\n\theader             []string\n\theader0            []string\n\tellipsis           string\n\tscrollbar          string\n\tpreviewScrollbar   string\n\tansi               bool\n\ttabstop            int\n\tmargin             [4]sizeSpec\n\tpadding            [4]sizeSpec\n\tunicode            bool\n\tlistenAddr         *listenAddress\n\tlistenPort         *int\n\tlistenUnsafe       bool\n\tborderShape        tui.BorderShape\n\tcleanExit          bool\n\tpaused             bool\n\tborder             tui.Window\n\twindow             tui.Window\n\tpborder            tui.Window\n\tpwindow            tui.Window\n\tborderWidth        int\n\tcount              int\n\tprogress           int\n\thasResultActions   bool\n\thasFocusActions    bool\n\thasLoadActions     bool\n\thasResizeActions   bool\n\ttriggerLoad        bool\n\treading            bool\n\trunning            bool\n\tfailed             *string\n\tjumping            jumpMode\n\tjumpLabels         string\n\tprinter            func(string)\n\tprintsep           string\n\tmerger             *Merger\n\tselected           map[int32]selectedItem\n\tversion            int64\n\trevision           int\n\treqBox             *util.EventBox\n\tinitialPreviewOpts previewOpts\n\tpreviewOpts        previewOpts\n\tactivePreviewOpts  *previewOpts\n\tpreviewer          previewer\n\tpreviewed          previewed\n\tpreviewBox         *util.EventBox\n\teventBox           *util.EventBox\n\tmutex              sync.Mutex\n\tinitFunc           func()\n\tprevLines          []itemLine\n\tsuppress           bool\n\tsigstop            bool\n\tstartChan          chan fitpad\n\tkillChan           chan int\n\tserverInputChan    chan []*action\n\tserverOutputChan   chan string\n\teventChan          chan tui.Event\n\tslab               *util.Slab\n\ttheme              *tui.ColorTheme\n\ttui                tui.Renderer\n\texecuting          *util.AtomicBool\n\ttermSize           tui.TermSize\n\tlastAction         actionType\n\tlastKey            string\n\tlastFocus          int32\n\tareaLines          int\n\tareaColumns        int\n\tforcePreview       bool\n}\n\ntype selectedItem struct {\n\tat   time.Time\n\titem *Item\n}\n\ntype byTimeOrder []selectedItem\n\nfunc (a byTimeOrder) Len() int {\n\treturn len(a)\n}\n\nfunc (a byTimeOrder) Swap(i, j int) {\n\ta[i], a[j] = a[j], a[i]\n}\n\nfunc (a byTimeOrder) Less(i, j int) bool {\n\treturn a[i].at.Before(a[j].at)\n}\n\nconst (\n\treqPrompt util.EventType = iota\n\treqInfo\n\treqHeader\n\treqList\n\treqJump\n\treqRefresh\n\treqReinit\n\treqFullRedraw\n\treqResize\n\treqRedrawBorderLabel\n\treqRedrawPreviewLabel\n\treqClose\n\treqPrintQuery\n\treqPreviewEnqueue\n\treqPreviewDisplay\n\treqPreviewRefresh\n\treqPreviewDelayed\n\treqQuit\n)\n\ntype action struct {\n\tt actionType\n\ta string\n}\n\n//go:generate stringer -type=actionType\ntype actionType int\n\nconst (\n\tactIgnore actionType = iota\n\tactStart\n\tactClick\n\tactInvalid\n\tactChar\n\tactMouse\n\tactBeginningOfLine\n\tactAbort\n\tactAccept\n\tactAcceptNonEmpty\n\tactAcceptOrPrintQuery\n\tactBackwardChar\n\tactBackwardDeleteChar\n\tactBackwardDeleteCharEof\n\tactBackwardWord\n\tactCancel\n\tactChangeBorderLabel\n\tactChangeHeader\n\tactChangePreviewLabel\n\tactChangePrompt\n\tactChangeQuery\n\tactClearScreen\n\tactClearQuery\n\tactClearSelection\n\tactClose\n\tactDeleteChar\n\tactDeleteCharEof\n\tactEndOfLine\n\tactForwardChar\n\tactForwardWord\n\tactKillLine\n\tactKillWord\n\tactUnixLineDiscard\n\tactUnixWordRubout\n\tactYank\n\tactBackwardKillWord\n\tactSelectAll\n\tactDeselectAll\n\tactToggle\n\tactToggleSearch\n\tactToggleAll\n\tactToggleDown\n\tactToggleUp\n\tactToggleIn\n\tactToggleOut\n\tactToggleTrack\n\tactToggleTrackCurrent\n\tactToggleHeader\n\tactTrackCurrent\n\tactUntrackCurrent\n\tactDown\n\tactUp\n\tactPageUp\n\tactPageDown\n\tactPosition\n\tactHalfPageUp\n\tactHalfPageDown\n\tactOffsetUp\n\tactOffsetDown\n\tactJump\n\tactJumpAccept // XXX Deprecated in favor of jump:accept binding\n\tactPrintQuery\n\tactRefreshPreview\n\tactReplaceQuery\n\tactToggleSort\n\tactShowPreview\n\tactHidePreview\n\tactTogglePreview\n\tactTogglePreviewWrap\n\tactTransform\n\tactTransformBorderLabel\n\tactTransformHeader\n\tactTransformPreviewLabel\n\tactTransformPrompt\n\tactTransformQuery\n\tactPreview\n\tactChangePreview\n\tactChangePreviewWindow\n\tactPreviewTop\n\tactPreviewBottom\n\tactPreviewUp\n\tactPreviewDown\n\tactPreviewPageUp\n\tactPreviewPageDown\n\tactPreviewHalfPageUp\n\tactPreviewHalfPageDown\n\tactPrevHistory\n\tactPrevSelected\n\tactPut\n\tactNextHistory\n\tactNextSelected\n\tactExecute\n\tactExecuteSilent\n\tactExecuteMulti // Deprecated\n\tactSigStop\n\tactFirst\n\tactLast\n\tactReload\n\tactReloadSync\n\tactDisableSearch\n\tactEnableSearch\n\tactSelect\n\tactDeselect\n\tactUnbind\n\tactRebind\n\tactBecome\n\tactResponse\n\tactShowHeader\n\tactHideHeader\n)\n\nfunc (a actionType) Name() string {\n\treturn util.ToKebabCase(a.String()[3:])\n}\n\nfunc processExecution(action actionType) bool {\n\tswitch action {\n\tcase actTransform,\n\t\tactTransformBorderLabel,\n\t\tactTransformHeader,\n\t\tactTransformPreviewLabel,\n\t\tactTransformPrompt,\n\t\tactTransformQuery,\n\t\tactPreview,\n\t\tactChangePreview,\n\t\tactRefreshPreview,\n\t\tactExecute,\n\t\tactExecuteSilent,\n\t\tactExecuteMulti,\n\t\tactReload,\n\t\tactReloadSync,\n\t\tactBecome:\n\t\treturn true\n\t}\n\treturn false\n}\n\ntype placeholderFlags struct {\n\tplus          bool\n\tpreserveSpace bool\n\tnumber        bool\n\tforceUpdate   bool\n\tfile          bool\n}\n\ntype searchRequest struct {\n\tsort    bool\n\tsync    bool\n\tcommand *string\n\tenviron []string\n\tchanged bool\n}\n\ntype previewRequest struct {\n\ttemplate     string\n\tpwindow      tui.Window\n\tpwindowSize  tui.TermSize\n\tscrollOffset int\n\tlist         []*Item\n}\n\ntype previewResult struct {\n\tversion int64\n\tlines   []string\n\toffset  int\n\tspinner string\n}\n\nfunc toActions(types ...actionType) []*action {\n\tactions := make([]*action, len(types))\n\tfor idx, t := range types {\n\t\tactions[idx] = &action{t: t, a: \"\"}\n\t}\n\treturn actions\n}\n\nfunc defaultKeymap() map[tui.Event][]*action {\n\tkeymap := make(map[tui.Event][]*action)\n\tadd := func(e tui.EventType, a actionType) {\n\t\tkeymap[e.AsEvent()] = toActions(a)\n\t}\n\taddEvent := func(e tui.Event, a actionType) {\n\t\tkeymap[e] = toActions(a)\n\t}\n\n\tadd(tui.Invalid, actInvalid)\n\tadd(tui.CtrlA, actBeginningOfLine)\n\tadd(tui.CtrlB, actBackwardChar)\n\tadd(tui.CtrlC, actAbort)\n\tadd(tui.CtrlG, actAbort)\n\tadd(tui.CtrlQ, actAbort)\n\tadd(tui.Esc, actAbort)\n\tadd(tui.CtrlD, actDeleteCharEof)\n\tadd(tui.CtrlE, actEndOfLine)\n\tadd(tui.CtrlF, actForwardChar)\n\tadd(tui.CtrlH, actBackwardDeleteChar)\n\tadd(tui.Backspace, actBackwardDeleteChar)\n\tadd(tui.Tab, actToggleDown)\n\tadd(tui.ShiftTab, actToggleUp)\n\tadd(tui.CtrlJ, actDown)\n\tadd(tui.CtrlK, actUp)\n\tadd(tui.CtrlL, actClearScreen)\n\tadd(tui.CtrlM, actAccept)\n\tadd(tui.CtrlN, actDown)\n\tadd(tui.CtrlP, actUp)\n\tadd(tui.CtrlU, actUnixLineDiscard)\n\tadd(tui.CtrlW, actUnixWordRubout)\n\tadd(tui.CtrlY, actYank)\n\tif !util.IsWindows() {\n\t\tadd(tui.CtrlZ, actSigStop)\n\t}\n\n\taddEvent(tui.AltKey('b'), actBackwardWord)\n\tadd(tui.ShiftLeft, actBackwardWord)\n\taddEvent(tui.AltKey('f'), actForwardWord)\n\tadd(tui.ShiftRight, actForwardWord)\n\taddEvent(tui.AltKey('d'), actKillWord)\n\tadd(tui.AltBackspace, actBackwardKillWord)\n\n\tadd(tui.Up, actUp)\n\tadd(tui.Down, actDown)\n\tadd(tui.Left, actBackwardChar)\n\tadd(tui.Right, actForwardChar)\n\n\tadd(tui.Home, actBeginningOfLine)\n\tadd(tui.End, actEndOfLine)\n\tadd(tui.Delete, actDeleteChar)\n\tadd(tui.PageUp, actPageUp)\n\tadd(tui.PageDown, actPageDown)\n\n\tadd(tui.ShiftUp, actPreviewUp)\n\tadd(tui.ShiftDown, actPreviewDown)\n\n\tadd(tui.Mouse, actMouse)\n\tadd(tui.LeftClick, actClick)\n\tadd(tui.RightClick, actToggle)\n\tadd(tui.SLeftClick, actToggle)\n\tadd(tui.SRightClick, actToggle)\n\n\tadd(tui.ScrollUp, actUp)\n\tadd(tui.ScrollDown, actDown)\n\tkeymap[tui.SScrollUp.AsEvent()] = toActions(actToggle, actUp)\n\tkeymap[tui.SScrollDown.AsEvent()] = toActions(actToggle, actDown)\n\n\tadd(tui.PreviewScrollUp, actPreviewUp)\n\tadd(tui.PreviewScrollDown, actPreviewDown)\n\treturn keymap\n}\n\nfunc trimQuery(query string) []rune {\n\treturn []rune(strings.Replace(query, \"\\t\", \" \", -1))\n}\n\nfunc mayTriggerPreview(opts *Options) bool {\n\tif opts.ListenAddr != nil {\n\t\treturn true\n\t}\n\tfor _, actions := range opts.Keymap {\n\t\tfor _, action := range actions {\n\t\t\tswitch action.t {\n\t\t\tcase actPreview, actChangePreview, actTransform:\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc makeSpinner(unicode bool) []string {\n\tif unicode {\n\t\treturn []string{`\u280b`, `\u2819`, `\u2839`, `\u2838`, `\u283c`, `\u2834`, `\u2826`, `\u2827`, `\u2807`, `\u280f`}\n\t}\n\treturn []string{`-`, `\\`, `|`, `/`, `-`, `\\`, `|`, `/`}\n}\n\nfunc evaluateHeight(opts *Options, termHeight int) int {\n\tsize := opts.Height.size\n\tif opts.Height.percent {\n\t\tif opts.Height.inverse {\n\t\t\tsize = 100 - size\n\t\t}\n\t\treturn util.Max(int(size*float64(termHeight)/100.0), opts.MinHeight)\n\t}\n\tif opts.Height.inverse {\n\t\tsize = float64(termHeight) - size\n\t}\n\treturn int(size)\n}\n\n// NewTerminal returns new Terminal object\nfunc NewTerminal(opts *Options, eventBox *util.EventBox) *Terminal {\n\tinput := trimQuery(opts.Query)\n\tvar delay time.Duration\n\tif opts.Tac {\n\t\tdelay = initialDelayTac\n\t} else {\n\t\tdelay = initialDelay\n\t}\n\tvar previewBox *util.EventBox\n\t// We need to start the previewer even when --preview option is not specified\n\t// * if HTTP server is enabled\n\t// * if 'preview' or 'change-preview' action is bound to a key\n\t// * if 'transform' action is bound to a key\n\tif len(opts.Preview.command) > 0 || mayTriggerPreview(opts) {\n\t\tpreviewBox = util.NewEventBox()\n\t}\n\tvar renderer tui.Renderer\n\tfullscreen := !opts.Height.auto && (opts.Height.size == 0 || opts.Height.percent && opts.Height.size == 100)\n\tif fullscreen {\n\t\tif tui.HasFullscreenRenderer() {\n\t\t\trenderer = tui.NewFullscreenRenderer(opts.Theme, opts.Black, opts.Mouse)\n\t\t} else {\n\t\t\trenderer = tui.NewLightRenderer(opts.Theme, opts.Black, opts.Mouse, opts.Tabstop, opts.ClearOnExit,\n\t\t\t\ttrue, func(h int) int { return h })\n\t\t}\n\t} else {\n\t\tmaxHeightFunc := func(termHeight int) int {\n\t\t\t// Minimum height required to render fzf excluding margin and padding\n\t\t\teffectiveMinHeight := minHeight\n\t\t\tif previewBox != nil && opts.Preview.aboveOrBelow() {\n\t\t\t\teffectiveMinHeight += 1 + borderLines(opts.Preview.border)\n\t\t\t}\n\t\t\tif noSeparatorLine(opts.InfoStyle, opts.Separator == nil || uniseg.StringWidth(*opts.Separator) > 0) {\n\t\t\t\teffectiveMinHeight--\n\t\t\t}\n\t\t\teffectiveMinHeight += borderLines(opts.BorderShape)\n\t\t\treturn util.Min(termHeight, util.Max(evaluateHeight(opts, termHeight), effectiveMinHeight))\n\t\t}\n\t\trenderer = tui.NewLightRenderer(opts.Theme, opts.Black, opts.Mouse, opts.Tabstop, opts.ClearOnExit, false, maxHeightFunc)\n\t}\n\twordRubout := \"[^\\\\pL\\\\pN][\\\\pL\\\\pN]\"\n\twordNext := \"[\\\\pL\\\\pN][^\\\\pL\\\\pN]|(.$)\"\n\tif opts.FileWord {\n\t\tsep := regexp.QuoteMeta(string(os.PathSeparator))\n\t\twordRubout = fmt.Sprintf(\"%s[^%s]\", sep, sep)\n\t\twordNext = fmt.Sprintf(\"[^%s]%s|(.$)\", sep, sep)\n\t}\n\tkeymapCopy := make(map[tui.Event][]*action)\n...\n", "needle_token_start": 5245, "needle_token_end": 5281, "code_context_ntokens": 14969, "output": ["```go\n// EmptyMerger is a Merger with no data\nfunc EmptyMerger(revision int) *Merger {\n\treturn NewMerger(nil, [][]Result{}, false, false, revision)\n}\n```"]}
{"repo": "junegunn/fzf", "name": "processExecution", "language": "go", "path": "src/terminal.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: Determines whether a given action requires processing that involves transformation, execution, or reloading of data.\n2. **Input**: An enumerated type representing various user interface actions.\n3. **Output**: A boolean value indicating whether the specified action involves complex operations such as data transformation, command execution, or content reloading.\n4. **Procedure**: The function checks if the provided action is one of several specific actions related to data transformation, preview management, command execution, or data reloading. If the action matches any in the predefined list, it returns true, indicating that additional processing is required; otherwise, it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/merger.go\npackage fzf\n\nimport \"fmt\"\n\n// EmptyMerger is a Merger with no data\nfunc EmptyMerger(revision int) *Merger {\n\treturn NewMerger(nil, [][]Result{}, false, false, revision)\n}\n\n// Merger holds a set of locally sorted lists of items and provides the view of\n// a single, globally-sorted list\ntype Merger struct {\n\tpattern  *Pattern\n\tlists    [][]Result\n\tmerged   []Result\n\tchunks   *[]*Chunk\n\tcursors  []int\n\tsorted   bool\n\ttac      bool\n\tfinal    bool\n\tcount    int\n\tpass     bool\n\trevision int\n}\n\n// PassMerger returns a new Merger that simply returns the items in the\n// original order\nfunc PassMerger(chunks *[]*Chunk, tac bool, revision int) *Merger {\n\tmg := Merger{\n\t\tpattern:  nil,\n\t\tchunks:   chunks,\n\t\ttac:      tac,\n\t\tcount:    0,\n\t\tpass:     true,\n\t\trevision: revision}\n\n\tfor _, chunk := range *mg.chunks {\n\t\tmg.count += chunk.count\n\t}\n\treturn &mg\n}\n\n// NewMerger returns a new Merger\nfunc NewMerger(pattern *Pattern, lists [][]Result, sorted bool, tac bool, revision int) *Merger {\n\tmg := Merger{\n\t\tpattern:  pattern,\n\t\tlists:    lists,\n\t\tmerged:   []Result{},\n\t\tchunks:   nil,\n\t\tcursors:  make([]int, len(lists)),\n\t\tsorted:   sorted,\n\t\ttac:      tac,\n\t\tfinal:    false,\n\t\tcount:    0,\n\t\trevision: revision}\n\n\tfor _, list := range mg.lists {\n\t\tmg.count += len(list)\n\t}\n\treturn &mg\n}\n\n// Revision returns revision number\nfunc (mg *Merger) Revision() int {\n\treturn mg.revision\n}\n\n// Length returns the number of items\nfunc (mg *Merger) Length() int {\n\treturn mg.count\n}\n\nfunc (mg *Merger) First() Result {\n\tif mg.tac && !mg.sorted {\n\t\treturn mg.Get(mg.count - 1)\n\t}\n\treturn mg.Get(0)\n}\n\n// FindIndex returns the index of the item with the given item index\nfunc (mg *Merger) FindIndex(itemIndex int32) int {\n\tindex := -1\n\tif mg.pass {\n\t\tindex = int(itemIndex)\n...\n// Path: src/reader.go\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/charlievieth/fastwalk\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// Reader reads from command or standard input\ntype Reader struct {\n\tpusher   func([]byte) bool\n\teventBox *util.EventBox\n\tdelimNil bool\n\tevent    int32\n\tfinChan  chan bool\n\tmutex    sync.Mutex\n\texec     *exec.Cmd\n\tcommand  *string\n\tkilled   bool\n\twait     bool\n}\n\n// NewReader returns new Reader object\nfunc NewReader(pusher func([]byte) bool, eventBox *util.EventBox, delimNil bool, wait bool) *Reader {\n\treturn &Reader{pusher, eventBox, delimNil, int32(EvtReady), make(chan bool, 1), sync.Mutex{}, nil, nil, false, wait}\n}\n\nfunc (r *Reader) startEventPoller() {\n\tgo func() {\n\t\tptr := &r.event\n\t\tpollInterval := readerPollIntervalMin\n\t\tfor {\n\t\t\tif atomic.CompareAndSwapInt32(ptr, int32(EvtReadNew), int32(EvtReady)) {\n\t\t\t\tr.eventBox.Set(EvtReadNew, (*string)(nil))\n\t\t\t\tpollInterval = readerPollIntervalMin\n\t\t\t} else if atomic.LoadInt32(ptr) == int32(EvtReadFin) {\n\t\t\t\tif r.wait {\n\t\t\t\t\tr.finChan <- true\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tpollInterval += readerPollIntervalStep\n\t\t\t\tif pollInterval > readerPollIntervalMax {\n\t\t\t\t\tpollInterval = readerPollIntervalMax\n\t\t\t\t}\n\t\t\t}\n\t\t\ttime.Sleep(pollInterval)\n\t\t}\n\t}()\n}\n\nfunc (r *Reader) fin(success bool) {\n\tatomic.StoreInt32(&r.event, int32(EvtReadFin))\n\tif r.wait {\n\t\t<-r.finChan\n\t}\n\n\tr.mutex.Lock()\n\tret := r.command\n\tif success || r.killed {\n\t\tret = nil\n\t}\n\tr.mutex.Unlock()\n\n\tr.eventBox.Set(EvtReadFin, ret)\n}\n\nfunc (r *Reader) terminate() {\n\tr.mutex.Lock()\n\tr.killed = true\n\tif r.exec != nil && r.exec.Process != nil {\n\t\tutil.KillCommand(r.exec)\n\t} else {\n\t\tos.Stdin.Close()\n\t}\n\tr.mutex.Unlock()\n}\n\nfunc (r *Reader) restart(command string, environ []string) {\n\tr.event = int32(EvtReady)\n\tr.startEventPoller()\n\tsuccess := r.readFromCommand(command, environ)\n\tr.fin(success)\n}\n\n// ReadSource reads data from the default command or from standard input\nfunc (r *Reader) ReadSource(root string, opts walkerOpts, ignores []string) {\n\tr.startEventPoller()\n\tvar success bool\n\tif util.IsTty() {\n\t\tcmd := os.Getenv(\"FZF_DEFAULT_COMMAND\")\n\t\tif len(cmd) == 0 {\n\t\t\tsuccess = r.readFiles(root, opts, ignores)\n\t\t} else {\n\t\t\t// We can't export FZF_* environment variables to the default command\n\t\t\tsuccess = r.readFromCommand(cmd, nil)\n\t\t}\n\t} else {\n\t\tsuccess = r.readFromStdin()\n\t}\n\tr.fin(success)\n}\n\nfunc (r *Reader) feed(src io.Reader) {\n\t/*\n\t\treaderSlabSize, ae := strconv.Atoi(os.Getenv(\"SLAB_KB\"))\n\t\tif ae != nil {\n\t\t\treaderSlabSize = 128 * 1024\n\t\t} else {\n\t\t\treaderSlabSize *= 1024\n\t\t}\n\t\treaderBufferSize, be := strconv.Atoi(os.Getenv(\"BUF_KB\"))\n\t\tif be != nil {\n\t\t\treaderBufferSize = 64 * 1024\n\t\t} else {\n\t\t\treaderBufferSize *= 1024\n\t\t}\n\t*/\n\n\tdelim := byte('\\n')\n\ttrimCR := util.IsWindows()\n\tif r.delimNil {\n\t\tdelim = '\\000'\n\t\ttrimCR = false\n\t}\n\n\tslab := make([]byte, readerSlabSize)\n\tleftover := []byte{}\n\tvar err error\n\tfor {\n\t\tn := 0\n\t\tscope := slab[:util.Min(len(slab), readerBufferSize)]\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tn, err = src.Read(scope)\n\t\t\tif n > 0 || err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// We're not making any progress after 100 tries. Stop.\n\t\tif n == 0 && err == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tbuf := slab[:n]\n\t\tslab = slab[n:]\n\n\t\tfor len(buf) > 0 {\n\t\t\tif i := bytes.IndexByte(buf, delim); i >= 0 {\n\t\t\t\t// Found the delimiter\n\t\t\t\tslice := buf[:i+1]\n\t\t\t\tbuf = buf[i+1:]\n\t\t\t\tif trimCR && len(slice) >= 2 && slice[len(slice)-2] == byte('\\r') {\n\t\t\t\t\tslice = slice[:len(slice)-2]\n\t\t\t\t} else {\n\t\t\t\t\tslice = slice[:len(slice)-1]\n\t\t\t\t}\n\t\t\t\tif len(leftover) > 0 {\n\t\t\t\t\tslice = append(leftover, slice...)\n\t\t\t\t\tleftover = []byte{}\n\t\t\t\t}\n\t\t\t\tif (err == nil || len(slice) > 0) && r.pusher(slice) {\n\t\t\t\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Could not find the delimiter in the buffer\n\t\t\t\t//   NOTE: We can further optimize this by keeping track of the cursor\n\t\t\t\t//   position in the slab so that a straddling item that doesn't go\n\t\t\t\t//   beyond the boundary of a slab doesn't need to be copied to\n\t\t\t\t//   another buffer. However, the performance gain is negligible in\n\t\t\t\t//   practice (< 0.1%) and is not\n\t\t\t\t//   worth the added complexity.\n\t\t\t\tleftover = append(leftover, buf...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif err == io.EOF {\n\t\t\tleftover = append(leftover, buf...)\n\t\t\tbreak\n\t\t}\n\n\t\tif len(slab) == 0 {\n\t\t\tslab = make([]byte, readerSlabSize)\n\t\t}\n\t}\n\tif len(leftover) > 0 && r.pusher(leftover) {\n\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t}\n}\n\nfunc (r *Reader) readFromStdin() bool {\n\tr.feed(os.Stdin)\n\treturn true\n}\n\nfunc (r *Reader) readFiles(root string, opts walkerOpts, ignores []string) bool {\n\tr.killed = false\n\tconf := fastwalk.Config{Follow: opts.follow}\n\tfn := func(path string, de os.DirEntry, err error) error {\n\t\tif err != nil {\n\t\t\treturn nil\n\t\t}\n\t\tpath = filepath.Clean(path)\n\t\tif path != \".\" {\n\t\t\tisDir := de.IsDir()\n\t\t\tif isDir {\n\t\t\t\tbase := filepath.Base(path)\n\t\t\t\tif !opts.hidden && base[0] == '.' {\n\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t}\n\t\t\t\tfor _, ignore := range ignores {\n\t\t\t\t\tif ignore == base {\n\t\t\t\t\t\treturn filepath.SkipDir\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif ((opts.file && !isDir) || (opts.dir && isDir)) && r.pusher([]byte(path)) {\n\t\t\t\tatomic.StoreInt32(&r.event, int32(EvtReadNew))\n\t\t\t}\n\t\t}\n\t\tr.mutex.Lock()\n\t\tdefer r.mutex.Unlock()\n\t\tif r.killed {\n\t\t\treturn context.Canceled\n\t\t}\n\t\treturn nil\n\t}\n\treturn fastwalk.Walk(&conf, root, fn) == nil\n}\n\nfunc (r *Reader) readFromCommand(command string, environ []string) bool {\n\tr.mutex.Lock()\n\tr.killed = false\n\tr.command = &command\n\tr.exec = util.ExecCommand(command, true)\n\tif environ != nil {\n\t\tr.exec.Env = environ\n\t}\n\tout, err := r.exec.StdoutPipe()\n\tif err != nil {\n\t\tr.mutex.Unlock()\n\t\treturn false\n\t}\n\terr = r.exec.Start()\n\tr.mutex.Unlock()\n\tif err != nil {\n\t\treturn false\n\t}\n\tr.feed(out)\n\treturn r.exec.Wait() == nil\n}\n\n// Path: src/terminal.go\npackage fzf\n\nimport (\n\t\"bufio\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/rivo/uniseg\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// import \"github.com/pkg/profile\"\n\n/*\nPlaceholder regex is used to extract placeholders from fzf's template\nstrings. Acts as input validation for parsePlaceholder function.\nDescribes the syntax, but it is fairly lenient.\n\nThe following pseudo regex has been reverse engineered from the\nimplementation. It is overly strict, but better describes what's possible.\nAs such it is not useful for validation, but rather to generate test\ncases for example.\n\n\t\\\\?(?:                                      # escaped type\n\t    {\\+?s?f?RANGE(?:,RANGE)*}               # token type\n\t    |{q}                                    # query type\n\t    |{\\+?n?f?}                              # item type (notice no mandatory element inside brackets)\n\t)\n\tRANGE = (?:\n\t    (?:-?[0-9]+)?\\.\\.(?:-?[0-9]+)?          # ellipsis syntax for token range (x..y)\n\t    |-?[0-9]+                               # shorthand syntax (x..x)\n\t)\n*/\nvar placeholder *regexp.Regexp\nvar whiteSuffix *regexp.Regexp\nvar offsetComponentRegex *regexp.Regexp\nvar offsetTrimCharsRegex *regexp.Regexp\nvar activeTempFiles []string\nvar passThroughRegex *regexp.Regexp\nvar actionTypeRegex *regexp.Regexp\n\nconst clearCode string = \"\\x1b[2J\"\n\n// Number of maximum focus events to process synchronously\nconst maxFocusEvents = 10000\n\nfunc init() {\n\tplaceholder = regexp.MustCompile(`\\\\?(?:{[+sf]*[0-9,-.]*}|{q}|{fzf:(?:query|action|prompt)}|{\\+?f?nf?})`)\n\twhiteSuffix = regexp.MustCompile(`\\s*$`)\n\toffsetComponentRegex = regexp.MustCompile(`([+-][0-9]+)|(-?/[1-9][0-9]*)`)\n\toffsetTrimCharsRegex = regexp.MustCompile(`[^0-9/+-]`)\n\tactiveTempFiles = []string{}\n\n\t// Parts of the preview output that should be passed through to the terminal\n\t// * https://github.com/tmux/tmux/wiki/FAQ#what-is-the-passthrough-escape-sequence-and-how-do-i-use-it\n\t// * https://sw.kovidgoyal.net/kitty/graphics-protocol\n\t// * https://en.wikipedia.org/wiki/Sixel\n\t// * https://iterm2.com/documentation-images.html\n\tpassThroughRegex = regexp.MustCompile(`\\x1bPtmux;\\x1b\\x1b.*?[^\\x1b]\\x1b\\\\|\\x1b(_G|P[0-9;]*q).*?\\x1b\\\\\\r?|\\x1b]1337;.*?(\\a|\\x1b\\\\)`)\n}\n\ntype jumpMode int\n\nconst (\n\tjumpDisabled jumpMode = iota\n\tjumpEnabled\n\tjumpAcceptEnabled\n)\n\ntype resumableState int\n\nconst (\n\tdisabledState resumableState = iota\n\tpausedState\n\tenabledState\n)\n\nfunc (s resumableState) Enabled() bool {\n\treturn s == enabledState\n}\n\nfunc (s *resumableState) Force(flag bool) {\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = disabledState\n\t}\n}\n\nfunc (s *resumableState) Set(flag bool) {\n\tif *s == disabledState {\n\t\treturn\n\t}\n\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = pausedState\n\t}\n}\n\ntype previewer struct {\n\tversion    int64\n\tlines      []string\n\toffset     int\n\tscrollable bool\n\tfinal      bool\n\tfollowing  resumableState\n\tspinner    string\n\tbar        []bool\n}\n\ntype previewed struct {\n\tversion   int64\n\tnumLines  int\n\toffset    int\n\tfilled    bool\n\timage     bool\n\twipe      bool\n\twireframe bool\n}\n\ntype eachLine struct {\n\tline string\n\terr  error\n}\n\ntype itemLine struct {\n\toffset   int\n\tcurrent  bool\n\tselected bool\n\tlabel    string\n\tqueryLen int\n\twidth    int\n\tbar      bool\n\tresult   Result\n}\n\ntype fitpad struct {\n\tfit int\n\tpad int\n}\n\nvar emptyLine = itemLine{}\n\ntype labelPrinter func(tui.Window, int)\n\ntype StatusItem struct {\n\tIndex int    `json:\"index\"`\n\tText  string `json:\"text\"`\n}\n\ntype Status struct {\n\tReading    bool         `json:\"reading\"`\n\tProgress   int          `json:\"progress\"`\n\tQuery      string       `json:\"query\"`\n\tPosition   int          `json:\"position\"`\n\tSort       bool         `json:\"sort\"`\n\tTotalCount int          `json:\"totalCount\"`\n\tMatchCount int          `json:\"matchCount\"`\n\tCurrent    *StatusItem  `json:\"current\"`\n\tMatches    []StatusItem `json:\"matches\"`\n\tSelected   []StatusItem `json:\"selected\"`\n}\n\n// Terminal represents terminal input/output\ntype Terminal struct {\n\tinitDelay          time.Duration\n\tinfoStyle          infoStyle\n\tinfoPrefix         string\n\tseparator          labelPrinter\n\tseparatorLen       int\n\tspinner            []string\n\tpromptString       string\n\tprompt             func()\n\tpromptLen          int\n\tborderLabel        labelPrinter\n\tborderLabelLen     int\n\tborderLabelOpts    labelOpts\n\tpreviewLabel       labelPrinter\n\tpreviewLabelLen    int\n\tpreviewLabelOpts   labelOpts\n\tpointer            string\n\tpointerLen         int\n\tpointerEmpty       string\n\tmarker             string\n\tmarkerLen          int\n\tmarkerEmpty        string\n\tqueryLen           [2]int\n\tlayout             layoutType\n\tfullscreen         bool\n\tkeepRight          bool\n\thscroll            bool\n\thscrollOff         int\n\tscrollOff          int\n\twordRubout         string\n\twordNext           string\n\tcx                 int\n\tcy                 int\n\toffset             int\n\txoffset            int\n\tyanked             []rune\n\tinput              []rune\n\tmulti              int\n\tsort               bool\n\ttoggleSort         bool\n\ttrack              trackOption\n\tdelimiter          Delimiter\n\texpect             map[tui.Event]string\n\tkeymap             map[tui.Event][]*action\n\tkeymapOrg          map[tui.Event][]*action\n\tpressed            string\n\tprintQuery         bool\n\thistory            *History\n\tcycle              bool\n\theaderVisible      bool\n\theaderFirst        bool\n\theaderLines        int\n\theader             []string\n\theader0            []string\n\tellipsis           string\n\tscrollbar          string\n\tpreviewScrollbar   string\n\tansi               bool\n\ttabstop            int\n\tmargin             [4]sizeSpec\n\tpadding            [4]sizeSpec\n\tunicode            bool\n\tlistenAddr         *listenAddress\n\tlistenPort         *int\n\tlistenUnsafe       bool\n\tborderShape        tui.BorderShape\n\tcleanExit          bool\n\tpaused             bool\n\tborder             tui.Window\n\twindow             tui.Window\n\tpborder            tui.Window\n\tpwindow            tui.Window\n\tborderWidth        int\n\tcount              int\n\tprogress           int\n\thasResultActions   bool\n\thasFocusActions    bool\n\thasLoadActions     bool\n\thasResizeActions   bool\n\ttriggerLoad        bool\n\treading            bool\n\trunning            bool\n\tfailed             *string\n\tjumping            jumpMode\n\tjumpLabels         string\n\tprinter            func(string)\n\tprintsep           string\n\tmerger             *Merger\n\tselected           map[int32]selectedItem\n\tversion            int64\n\trevision           int\n\treqBox             *util.EventBox\n\tinitialPreviewOpts previewOpts\n\tpreviewOpts        previewOpts\n\tactivePreviewOpts  *previewOpts\n\tpreviewer          previewer\n\tpreviewed          previewed\n\tpreviewBox         *util.EventBox\n\teventBox           *util.EventBox\n\tmutex              sync.Mutex\n\tinitFunc           func()\n\tprevLines          []itemLine\n\tsuppress           bool\n\tsigstop            bool\n\tstartChan          chan fitpad\n\tkillChan           chan int\n\tserverInputChan    chan []*action\n\tserverOutputChan   chan string\n\teventChan          chan tui.Event\n\tslab               *util.Slab\n\ttheme              *tui.ColorTheme\n\ttui                tui.Renderer\n\texecuting          *util.AtomicBool\n\ttermSize           tui.TermSize\n\tlastAction         actionType\n\tlastKey            string\n\tlastFocus          int32\n\tareaLines          int\n\tareaColumns        int\n\tforcePreview       bool\n}\n\ntype selectedItem struct {\n\tat   time.Time\n\titem *Item\n}\n\ntype byTimeOrder []selectedItem\n\nfunc (a byTimeOrder) Len() int {\n\treturn len(a)\n}\n\nfunc (a byTimeOrder) Swap(i, j int) {\n\ta[i], a[j] = a[j], a[i]\n}\n\nfunc (a byTimeOrder) Less(i, j int) bool {\n\treturn a[i].at.Before(a[j].at)\n}\n\nconst (\n\treqPrompt util.EventType = iota\n\treqInfo\n\treqHeader\n\treqList\n\treqJump\n\treqRefresh\n\treqReinit\n\treqFullRedraw\n\treqResize\n\treqRedrawBorderLabel\n\treqRedrawPreviewLabel\n\treqClose\n\treqPrintQuery\n\treqPreviewEnqueue\n\treqPreviewDisplay\n\treqPreviewRefresh\n\treqPreviewDelayed\n\treqQuit\n)\n\ntype action struct {\n\tt actionType\n\ta string\n}\n\n//go:generate stringer -type=actionType\ntype actionType int\n\nconst (\n\tactIgnore actionType = iota\n\tactStart\n\tactClick\n\tactInvalid\n\tactChar\n\tactMouse\n\tactBeginningOfLine\n\tactAbort\n\tactAccept\n\tactAcceptNonEmpty\n\tactAcceptOrPrintQuery\n\tactBackwardChar\n\tactBackwardDeleteChar\n\tactBackwardDeleteCharEof\n\tactBackwardWord\n\tactCancel\n\tactChangeBorderLabel\n\tactChangeHeader\n\tactChangePreviewLabel\n\tactChangePrompt\n\tactChangeQuery\n\tactClearScreen\n\tactClearQuery\n\tactClearSelection\n\tactClose\n\tactDeleteChar\n\tactDeleteCharEof\n\tactEndOfLine\n\tactForwardChar\n\tactForwardWord\n\tactKillLine\n\tactKillWord\n\tactUnixLineDiscard\n\tactUnixWordRubout\n\tactYank\n\tactBackwardKillWord\n\tactSelectAll\n\tactDeselectAll\n\tactToggle\n\tactToggleSearch\n\tactToggleAll\n\tactToggleDown\n\tactToggleUp\n\tactToggleIn\n\tactToggleOut\n\tactToggleTrack\n\tactToggleTrackCurrent\n\tactToggleHeader\n\tactTrackCurrent\n\tactUntrackCurrent\n\tactDown\n\tactUp\n\tactPageUp\n\tactPageDown\n\tactPosition\n\tactHalfPageUp\n\tactHalfPageDown\n\tactOffsetUp\n\tactOffsetDown\n\tactJump\n\tactJumpAccept // XXX Deprecated in favor of jump:accept binding\n\tactPrintQuery\n\tactRefreshPreview\n\tactReplaceQuery\n\tactToggleSort\n\tactShowPreview\n\tactHidePreview\n\tactTogglePreview\n\tactTogglePreviewWrap\n\tactTransform\n\tactTransformBorderLabel\n\tactTransformHeader\n\tactTransformPreviewLabel\n\tactTransformPrompt\n\tactTransformQuery\n\tactPreview\n\tactChangePreview\n\tactChangePreviewWindow\n\tactPreviewTop\n\tactPreviewBottom\n\tactPreviewUp\n\tactPreviewDown\n\tactPreviewPageUp\n\tactPreviewPageDown\n\tactPreviewHalfPageUp\n\tactPreviewHalfPageDown\n\tactPrevHistory\n\tactPrevSelected\n\tactPut\n\tactNextHistory\n\tactNextSelected\n\tactExecute\n\tactExecuteSilent\n\tactExecuteMulti // Deprecated\n\tactSigStop\n\tactFirst\n\tactLast\n\tactReload\n\tactReloadSync\n\tactDisableSearch\n\tactEnableSearch\n\tactSelect\n\tactDeselect\n\tactUnbind\n\tactRebind\n\tactBecome\n\tactResponse\n\tactShowHeader\n\tactHideHeader\n)\n\nfunc (a actionType) Name() string {\n\treturn util.ToKebabCase(a.String()[3:])\n}\n\n\nfunc processExecution(action actionType) bool {\n\tswitch action {\n\tcase actTransform,\n\t\tactTransformBorderLabel,\n\t\tactTransformHeader,\n\t\tactTransformPreviewLabel,\n\t\tactTransformPrompt,\n\t\tactTransformQuery,\n\t\tactPreview,\n\t\tactChangePreview,\n\t\tactRefreshPreview,\n\t\tactExecute,\n\t\tactExecuteSilent,\n\t\tactExecuteMulti,\n\t\tactReload,\n\t\tactReloadSync,\n\t\tactBecome:\n\t\treturn true\n\t}\n\treturn false\n}\n\ntype placeholderFlags struct {\n\tplus          bool\n\tpreserveSpace bool\n\tnumber        bool\n\tforceUpdate   bool\n\tfile          bool\n}\n\ntype searchRequest struct {\n\tsort    bool\n\tsync    bool\n\tcommand *string\n\tenviron []string\n\tchanged bool\n}\n\ntype previewRequest struct {\n\ttemplate     string\n\tpwindow      tui.Window\n\tpwindowSize  tui.TermSize\n\tscrollOffset int\n\tlist         []*Item\n}\n\ntype previewResult struct {\n\tversion int64\n\tlines   []string\n\toffset  int\n\tspinner string\n}\n\nfunc toActions(types ...actionType) []*action {\n\tactions := make([]*action, len(types))\n\tfor idx, t := range types {\n\t\tactions[idx] = &action{t: t, a: \"\"}\n\t}\n\treturn actions\n}\n\nfunc defaultKeymap() map[tui.Event][]*action {\n\tkeymap := make(map[tui.Event][]*action)\n\tadd := func(e tui.EventType, a actionType) {\n\t\tkeymap[e.AsEvent()] = toActions(a)\n\t}\n\taddEvent := func(e tui.Event, a actionType) {\n\t\tkeymap[e] = toActions(a)\n\t}\n\n\tadd(tui.Invalid, actInvalid)\n\tadd(tui.CtrlA, actBeginningOfLine)\n\tadd(tui.CtrlB, actBackwardChar)\n\tadd(tui.CtrlC, actAbort)\n\tadd(tui.CtrlG, actAbort)\n\tadd(tui.CtrlQ, actAbort)\n\tadd(tui.Esc, actAbort)\n\tadd(tui.CtrlD, actDeleteCharEof)\n\tadd(tui.CtrlE, actEndOfLine)\n\tadd(tui.CtrlF, actForwardChar)\n\tadd(tui.CtrlH, actBackwardDeleteChar)\n\tadd(tui.Backspace, actBackwardDeleteChar)\n\tadd(tui.Tab, actToggleDown)\n\tadd(tui.ShiftTab, actToggleUp)\n\tadd(tui.CtrlJ, actDown)\n\tadd(tui.CtrlK, actUp)\n\tadd(tui.CtrlL, actClearScreen)\n\tadd(tui.CtrlM, actAccept)\n\tadd(tui.CtrlN, actDown)\n\tadd(tui.CtrlP, actUp)\n\tadd(tui.CtrlU, actUnixLineDiscard)\n\tadd(tui.CtrlW, actUnixWordRubout)\n\tadd(tui.CtrlY, actYank)\n\tif !util.IsWindows() {\n\t\tadd(tui.CtrlZ, actSigStop)\n\t}\n\n\taddEvent(tui.AltKey('b'), actBackwardWord)\n\tadd(tui.ShiftLeft, actBackwardWord)\n\taddEvent(tui.AltKey('f'), actForwardWord)\n\tadd(tui.ShiftRight, actForwardWord)\n\taddEvent(tui.AltKey('d'), actKillWord)\n\tadd(tui.AltBackspace, actBackwardKillWord)\n\n\tadd(tui.Up, actUp)\n\tadd(tui.Down, actDown)\n\tadd(tui.Left, actBackwardChar)\n\tadd(tui.Right, actForwardChar)\n\n\tadd(tui.Home, actBeginningOfLine)\n\tadd(tui.End, actEndOfLine)\n\tadd(tui.Delete, actDeleteChar)\n\tadd(tui.PageUp, actPageUp)\n\tadd(tui.PageDown, actPageDown)\n\n\tadd(tui.ShiftUp, actPreviewUp)\n\tadd(tui.ShiftDown, actPreviewDown)\n\n\tadd(tui.Mouse, actMouse)\n\tadd(tui.LeftClick, actClick)\n\tadd(tui.RightClick, actToggle)\n\tadd(tui.SLeftClick, actToggle)\n\tadd(tui.SRightClick, actToggle)\n\n\tadd(tui.ScrollUp, actUp)\n\tadd(tui.ScrollDown, actDown)\n\tkeymap[tui.SScrollUp.AsEvent()] = toActions(actToggle, actUp)\n\tkeymap[tui.SScrollDown.AsEvent()] = toActions(actToggle, actDown)\n\n\tadd(tui.PreviewScrollUp, actPreviewUp)\n\tadd(tui.PreviewScrollDown, actPreviewDown)\n\treturn keymap\n}\n\nfunc trimQuery(query string) []rune {\n\treturn []rune(strings.Replace(query, \"\\t\", \" \", -1))\n}\n\nfunc mayTriggerPreview(opts *Options) bool {\n\tif opts.ListenAddr != nil {\n\t\treturn true\n\t}\n\tfor _, actions := range opts.Keymap {\n\t\tfor _, action := range actions {\n\t\t\tswitch action.t {\n\t\t\tcase actPreview, actChangePreview, actTransform:\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc makeSpinner(unicode bool) []string {\n\tif unicode {\n\t\treturn []string{`\u280b`, `\u2819`, `\u2839`, `\u2838`, `\u283c`, `\u2834`, `\u2826`, `\u2827`, `\u2807`, `\u280f`}\n\t}\n\treturn []string{`-`, `\\`, `|`, `/`, `-`, `\\`, `|`, `/`}\n}\n\nfunc evaluateHeight(opts *Options, termHeight int) int {\n\tsize := opts.Height.size\n\tif opts.Height.percent {\n\t\tif opts.Height.inverse {\n\t\t\tsize = 100 - size\n\t\t}\n\t\treturn util.Max(int(size*float64(termHeight)/100.0), opts.MinHeight)\n\t}\n\tif opts.Height.inverse {\n\t\tsize = float64(termHeight) - size\n\t}\n\treturn int(size)\n}\n\n// NewTerminal returns new Terminal object\nfunc NewTerminal(opts *Options, eventBox *util.EventBox) *Terminal {\n\tinput := trimQuery(opts.Query)\n\tvar delay time.Duration\n\tif opts.Tac {\n\t\tdelay = initialDelayTac\n\t} else {\n\t\tdelay = initialDelay\n\t}\n\tvar previewBox *util.EventBox\n\t// We need to start the previewer even when --preview option is not specified\n\t// * if HTTP server is enabled\n\t// * if 'preview' or 'change-preview' action is bound to a key\n\t// * if 'transform' action is bound to a key\n\tif len(opts.Preview.command) > 0 || mayTriggerPreview(opts) {\n\t\tpreviewBox = util.NewEventBox()\n\t}\n\tvar renderer tui.Renderer\n\tfullscreen := !opts.Height.auto && (opts.Height.size == 0 || opts.Height.percent && opts.Height.size == 100)\n\tif fullscreen {\n\t\tif tui.HasFullscreenRenderer() {\n\t\t\trenderer = tui.NewFullscreenRenderer(opts.Theme, opts.Black, opts.Mouse)\n\t\t} else {\n\t\t\trenderer = tui.NewLightRenderer(opts.Theme, opts.Black, opts.Mouse, opts.Tabstop, opts.ClearOnExit,\n\t\t\t\ttrue, func(h int) int { return h })\n\t\t}\n\t} else {\n\t\tmaxHeightFunc := func(termHeight int) int {\n\t\t\t// Minimum height required to render fzf excluding margin and padding\n\t\t\teffectiveMinHeight := minHeight\n\t\t\tif previewBox != nil && opts.Preview.aboveOrBelow() {\n\t\t\t\teffectiveMinHeight += 1 + borderLines(opts.Preview.border)\n\t\t\t}\n\t\t\tif noSeparatorLine(opts.InfoStyle, opts.Separator == nil || uniseg.StringWidth(*opts.Separator) > 0) {\n\t\t\t\teffectiveMinHeight--\n\t\t\t}\n\t\t\teffectiveMinHeight += borderLines(opts.BorderShape)\n\t\t\treturn util.Min(termHeight, util.Max(evaluateHeight(opts, termHeight), effectiveMinHeight))\n\t\t}\n\t\trenderer = tui.NewLightRenderer(opts.Theme, opts.Black, opts.Mouse, opts.Tabstop, opts.ClearOnExit, false, maxHeightFunc)\n\t}\n\twordRubout := \"[^\\\\pL\\\\pN][\\\\pL\\\\pN]\"\n\twordNext := \"[\\\\pL\\\\pN][^\\\\pL\\\\pN]|(.$)\"\n\tif opts.FileWord {\n\t\tsep := regexp.QuoteMeta(string(os.PathSeparator))\n\t\twordRubout = fmt.Sprintf(\"%s[^%s]\", sep, sep)\n\t\twordNext = fmt.Sprintf(\"[^%s]%s|(.$)\", sep, sep)\n\t}\n\tkeymapCopy := make(map[tui.Event][]*action)\n\tfor key, action := range opts.Keymap {\n\t\tkeymapCopy[key] = action\n\t}\n\tt := Terminal{\n\t\tinitDelay:          delay,\n\t\tinfoStyle:          opts.InfoStyle,\n\t\tinfoPrefix:         opts.InfoPrefix,\n\t\tseparator:          nil,\n\t\tspinner:            makeSpinner(opts.Unicode),\n\t\tpromptString:       opts.Prompt,\n\t\tqueryLen:           [2]int{0, 0},\n\t\tlayout:             opts.Layout,\n\t\tfullscreen:         fullscreen,\n\t\tkeepRight:          opts.KeepRight,\n\t\thscroll:            opts.Hscroll,\n\t\thscrollOff:         opts.HscrollOff,\n\t\tscrollOff:          opts.ScrollOff,\n\t\twordRubout:         wordRubout,\n\t\twordNext:           wordNext,\n\t\tcx:                 len(input),\n\t\tcy:                 0,\n\t\toffset:             0,\n\t\txoffset:            0,\n\t\tyanked:             []rune{},\n\t\tinput:              input,\n\t\tmulti:              opts.Multi,\n\t\tsort:               opts.Sort > 0,\n\t\ttoggleSort:         opts.ToggleSort,\n\t\ttrack:              opts.Track,\n\t\tdelimiter:          opts.Delimiter,\n\t\texpect:             opts.Expect,\n\t\tkeymap:             opts.Keymap,\n\t\tkeymapOrg:          keymapCopy,\n\t\tpressed:            \"\",\n\t\tprintQuery:         opts.PrintQuery,\n\t\thistory:            opts.History,\n\t\tmargin:             opts.Margin,\n\t\tpadding:            opts.Padding,\n\t\tunicode:            opts.Unicode,\n\t\tlistenAddr:         opts.ListenAddr,\n\t\tlistenUnsafe:       opts.Unsafe,\n\t\tborderShape:        opts.BorderShape,\n\t\tborderWidth:        1,\n\t\tborderLabel:        nil,\n\t\tborderLabelOpts:    opts.BorderLabel,\n\t\tpreviewLabel:       nil,\n\t\tpreviewLabelOpts:   opts.PreviewLabel,\n\t\tcleanExit:          opts.ClearOnExit,\n\t\tpaused:             opts.Phony,\n\t\tcycle:              opts.Cycle,\n\t\theaderVisible:      true,\n\t\theaderFirst:        opts.HeaderFirst,\n\t\theaderLines:        opts.HeaderLines,\n\t\theader:             []string{},\n\t\theader0:            opts.Header,\n\t\tellipsis:           opts.Ellipsis,\n\t\tansi:               opts.Ansi,\n\t\ttabstop:            opts.Tabstop,\n\t\thasResultActions:   false,\n\t\thasFocusActions:    false,\n\t\thasLoadActions:     false,\n\t\ttriggerLoad:        false,\n\t\treading:            true,\n\t\trunning:            true,\n\t\tfailed:             nil,\n\t\tjumping:            jumpDisabled,\n\t\tjumpLabels:         opts.JumpLabels,\n\t\tprinter:            opts.Printer,\n\t\tprintsep:           opts.PrintSep,\n\t\tmerger:             EmptyMerger(0),\n\t\tselected:           make(map[int32]selectedItem),\n\t\treqBox:             util.NewEventBox(),\n\t\tinitialPreviewOpts: opts.Preview,\n\t\tpreviewOpts:        opts.Preview,\n\t\tpreviewer:          previewer{0, []string{}, 0, false, true, disabledState, \"\", []bool{}},\n\t\tpreviewed:          previewed{0, 0, 0, false, false, false, false},\n\t\tpreviewBox:         previewBox,\n\t\teventBox:           eventBox,\n\t\tmutex:              sync.Mutex{},\n\t\tsuppress:           true,\n\t\tsigstop:            false,\n\t\tslab:               util.MakeSlab(slab16Size, slab32Size),\n\t\ttheme:              opts.Theme,\n\t\tstartChan:          make(chan fitpad, 1),\n\t\tkillChan:           make(chan int),\n\t\tserverInputChan:    make(chan []*action, 100),\n\t\tserverOutputChan:   make(chan string),\n\t\teventChan:          make(chan tui.Event, 6), // (load + result + zero|one) | (focus) | (resize) | (GetChar)\n\t\ttui:                renderer,\n\t\tinitFunc:           func() { renderer.Init() },\n\t\texecuting:          util.NewAtomicBool(false),\n\t\tlastAction:         actStart,\n\t\tlastFocus:          minItem.Index()}\n\tt.prompt, t.promptLen = t.parsePrompt(opts.Prompt)\n\tt.pointer, t.pointerLen = t.processTabs([]rune(opts.Pointer), 0)\n\tt.marker, t.markerLen = t.processTabs([]rune(opts.Marker), 0)\n\t// Pre-calculated empty pointer and marker signs\n\tt.pointerEmpty = strings.Repeat(\" \", t.pointerLen)\n\tt.markerEmpty = strings.Repeat(\" \", t.markerLen)\n\tt.borderLabel, t.borderLabelLen = t.ansiLabelPrinter(opts.BorderLabel.label, &tui.ColBorderLabel, false)\n\tt.previewLabel, t.previewLabelLen = t.ansiLabelPrinter(opts.PreviewLabel.label, &tui.ColPreviewLabel, false)\n\tif opts.Separator == nil || len(*opts.Separator) > 0 {\n\t\tbar := \"\u2500\"\n\t\tif opts.Separator != nil {\n\t\t\tbar = *opts.Separator\n\t\t} else if !t.unicode {\n\t\t\tbar = \"-\"\n\t\t}\n\t\tt.separator, t.separatorLen = t.ansiLabelPrinter(bar, &tui.ColSeparator, true)\n\t}\n\tif t.unicode {\n\t\tt.borderWidth = uniseg.StringWidth(\"\u2502\")\n\t}\n\tif opts.Scrollbar == nil {\n\t\tif t.unicode && t.borderWidth == 1 {\n\t\t\tt.scrollbar = \"\u2502\"\n\t\t} else {\n\t\t\tt.scrollbar = \"|\"\n\t\t}\n\t\tt.previewScrollbar = t.scrollbar\n\t} else {\n\t\trunes := []rune(*opts.Scrollbar)\n\t\tif len(runes) > 0 {\n\t\t\tt.scrollbar = string(runes[0])\n\t\t\tt.previewScrollbar = t.scrollbar\n\t\t\tif len(runes) > 1 {\n\t\t\t\tt.previewScrollbar = string(runes[1])\n\t\t\t}\n\t\t}\n\t}\n\n\tvar resizeActions []*action\n\tresizeActions, t.hasResizeActions = t.keymap[tui.Resize.AsEvent()]\n\tif t.tui.ShouldEmitResizeEvent() {\n\t\tt.keymap[tui.Resize.AsEvent()] = append(toActions(actClearScreen), resizeActions...)\n\t}\n\t_, t.hasResultActions = t.keymap[tui.Result.AsEvent()]\n\t_, t.hasFocusActions = t.keymap[tui.Focus.AsEvent()]\n\t_, t.hasLoadActions = t.keymap[tui.Load.AsEvent()]\n\n\tif t.listenAddr != nil {\n\t\tport, err := startHttpServer(*t.listenAddr, t.serverInputChan, t.serverOutputChan)\n\t\tif err != nil {\n\t\t\terrorExit(err.Error())\n\t\t}\n\t\tt.listenPort = &port\n\t}\n\n\treturn &t\n}\n\nfunc (t *Terminal) environ() []string {\n\tenv := os.Environ()\n\tif t.listenPort != nil {\n\t\tenv = append(env, fmt.Sprintf(\"FZF_PORT=%d\", *t.listenPort))\n\t}\n\tenv = append(env, \"FZF_QUERY=\"+string(t.input))\n\tenv = append(env, \"FZF_ACTION=\"+t.lastAction.Name())\n\tenv = append(env, \"FZF_KEY=\"+t.lastKey)\n\tenv = append(env, \"FZF_PROMPT=\"+string(t.promptString))\n\tenv = append(env, \"FZF_PREVIEW_LABEL=\"+t.previewLabelOpts.label)\n\tenv = append(env, \"FZF_BORDER_LABEL=\"+t.borderLabelOpts.label)\n\tenv = append(env, fmt.Sprintf(\"FZF_TOTAL_COUNT=%d\", t.count))\n\tenv = append(env, fmt.Sprintf(\"FZF_MATCH_COUNT=%d\", t.merger.Length()))\n\tenv = append(env, fmt.Sprintf(\"FZF_SELECT_COUNT=%d\", len(t.selected)))\n\tenv = append(env, fmt.Sprintf(\"FZF_LINES=%d\", t.areaLines))\n\tenv = append(env, fmt.Sprintf(\"FZF_COLUMNS=%d\", t.areaColumns))\n\treturn env\n}\n\nfunc borderLines(shape tui.BorderShape) int {\n\tswitch shape {\n\tcase tui.BorderHorizontal, tui.BorderRounded, tui.BorderSharp, tui.BorderBold, tui.BorderBlock, tui.BorderThinBlock, tui.BorderDouble:\n\t\treturn 2\n\tcase tui.BorderTop, tui.BorderBottom:\n\t\treturn 1\n\t}\n\treturn 0\n}\n\nfunc (t *Terminal) visibleHeaderLines() int {\n\tif !t.headerVisible {\n\t\treturn 0\n\t}\n\treturn len(t.header0) + t.headerLines\n}\n\n// Extra number of lines needed to display fzf\nfunc (t *Terminal) extraLines() int {\n\textra := t.visibleHeaderLines() + 1\n\tif !t.noSeparatorLine() {\n\t\textra++\n\t}\n\treturn extra\n}\n\nfunc (t *Terminal) MaxFitAndPad() (int, int) {\n\t_, screenHeight, marginInt, paddingInt := t.adjustMarginAndPadding()\n\tpadHeight := marginInt[0] + marginInt[2] + paddingInt[0] + paddingInt[2]\n\tfit := screenHeight - padHeight - t.extraLines()\n\treturn fit, padHeight\n}\n\nfunc (t *Terminal) ansiLabelPrinter(str string, color *tui.ColorPair, fill bool) (labelPrinter, int) {\n\t// Nothing to do\n\tif len(str) == 0 {\n\t\treturn nil, 0\n\t}\n\n\t// Extract ANSI color codes\n\tstr = firstLine(str)\n\ttext, colors, _ := extractColor(str, nil, nil)\n\trunes := []rune(text)\n\n\t// Simpler printer for strings without ANSI colors or tab characters\n\tif colors == nil && !strings.ContainsRune(str, '\\t') {\n\t\tlength := util.StringWidth(str)\n\t\tif length == 0 {\n\t\t\treturn nil, 0\n\t\t}\n\t\tprintFn := func(window tui.Window, limit int) {\n\t\t\tif length > limit {\n\t\t\t\ttrimmedRunes, _ := t.trimRight(runes, limit)\n\t\t\t\twindow.CPrint(*color, string(trimmedRunes))\n\t\t\t} else if fill {\n\t\t\t\twindow.CPrint(*color, util.RepeatToFill(str, length, limit))\n\t\t\t} else {\n\t\t\t\twindow.CPrint(*color, str)\n\t\t\t}\n\t\t}\n\t\treturn printFn, length\n\t}\n\n\t// Printer that correctly handles ANSI color codes and tab characters\n\titem := &Item{text: util.RunesToChars(runes), colors: colors}\n\tlength := t.displayWidth(runes)\n\tif length == 0 {\n\t\treturn nil, 0\n\t}\n\tresult := Result{item: item}\n\tvar offsets []colorOffset\n\tprintFn := func(window tui.Window, limit int) {\n\t\tif offsets == nil {\n\t\t\t// tui.Col* are not initialized until renderer.Init()\n\t\t\toffsets = result.colorOffsets(nil, t.theme, *color, *color, false)\n\t\t}\n\t\tfor limit > 0 {\n\t\t\tif length > limit {\n\t\t\t\ttrimmedRunes, _ := t.trimRight(runes, limit)\n\t\t\t\tt.printColoredString(window, trimmedRunes, offsets, *color)\n\t\t\t\tbreak\n\t\t\t} else if fill {\n\t\t\t\tt.printColoredString(window, runes, offsets, *color)\n\t\t\t\tlimit -= length\n\t\t\t} else {\n\t\t\t\tt.printColoredString(window, runes, offsets, *color)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn printFn, length\n}\n\nfunc (t *Terminal) parsePrompt(prompt string) (func(), int) {\n\tvar state *ansiState\n\tprompt = firstLine(prompt)\n\ttrimmed, colors, _ := extractColor(prompt, state, nil)\n\titem := &Item{text: util.ToChars([]byte(trimmed)), colors: colors}\n\n\t// \"Prompt>  \"\n\t//  -------    // Do not apply ANSI attributes to the trailing whitespaces\n\t//             // unless the part has a non-default ANSI state\n\tloc := whiteSuffix.FindStringIndex(trimmed)\n\tif loc != nil {\n\t\tblankState := ansiOffset{[2]int32{int32(loc[0]), int32(loc[1])}, ansiState{-1, -1, tui.AttrClear, -1}}\n\t\tif item.colors != nil {\n\t\t\tlastColor := (*item.colors)[len(*item.colors)-1]\n\t\t\tif lastColor.offset[1] < int32(loc[1]) {\n\t\t\t\tblankState.offset[0] = lastColor.offset[1]\n\t\t\t\tcolors := append(*item.colors, blankState)\n\t\t\t\titem.colors = &colors\n\t\t\t}\n\t\t} else {\n\t\t\tcolors := []ansiOffset{blankState}\n\t\t\titem.colors = &colors\n\t\t}\n\t}\n\toutput := func() {\n\t\tt.printHighlighted(\n\t\t\tResult{item: item}, tui.ColPrompt, tui.ColPrompt, false, false)\n\t}\n\t_, promptLen := t.processTabs([]rune(trimmed), 0)\n\n\treturn output, promptLen\n}\n\nfunc noSeparatorLine(style infoStyle, separator bool) bool {\n\tswitch style {\n\tcase infoInline:\n\t\treturn true\n\tcase infoHidden, infoInlineRight:\n\t\treturn !separator\n\t}\n\treturn false\n}\n\nfunc (t *Terminal) noSeparatorLine() bool {\n\treturn noSeparatorLine(t.infoStyle, t.separatorLen > 0)\n}\n\nfunc getScrollbar(total int, height int, offset int) (int, int) {\n\tif total == 0 || total <= height {\n\t\treturn 0, 0\n\t}\n\tbarLength := util.Max(1, height*height/total)\n\tvar barStart int\n\tif total == height {\n\t\tbarStart = 0\n\t} else {\n\t\tbarStart = (height - barLength) * offset / (total - height)\n\t}\n\treturn barLength, barStart\n}\n\nfunc (t *Terminal) getScrollbar() (int, int) {\n\treturn getScrollbar(t.merger.Length(), t.maxItems(), t.offset)\n}\n\n// Input returns current query string\nfunc (t *Terminal) Input() (bool, []rune) {\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\treturn t.paused, copySlice(t.input)\n}\n\n// UpdateCount updates the count information\nfunc (t *Terminal) UpdateCount(cnt int, final bool, failedCommand *string) {\n\tt.mutex.Lock()\n\tt.count = cnt\n\tif t.hasLoadActions && t.reading && final {\n\t\tt.triggerLoad = true\n\t}\n\tt.reading = !final\n\tt.failed = failedCommand\n\tt.mutex.Unlock()\n\tt.reqBox.Set(reqInfo, nil)\n\tif final {\n\t\tt.reqBox.Set(reqRefresh, nil)\n\t}\n}\n\nfunc (t *Terminal) changeHeader(header string) bool {\n\tlines := strings.Split(strings.TrimSuffix(header, \"\\n\"), \"\\n\")\n\tneedFullRedraw := len(t.header0) != len(lines)\n\tt.header0 = lines\n\treturn needFullRedraw\n}\n\n// UpdateHeader updates the header\nfunc (t *Terminal) UpdateHeader(header []string) {\n\tt.mutex.Lock()\n\tt.header = header\n\tt.mutex.Unlock()\n\tt.reqBox.Set(reqHeader, nil)\n}\n\n// UpdateProgress updates the search progress\nfunc (t *Terminal) UpdateProgress(progress float32) {\n\tt.mutex.Lock()\n\tnewProgress := int(progress * 100)\n\tchanged := t.progress != newProgress\n\tt.progress = newProgress\n\tt.mutex.Unlock()\n\n\tif changed {\n\t\tt.reqBox.Set(reqInfo, nil)\n\t}\n}\n\n// UpdateList updates Merger to display the list\nfunc (t *Terminal) UpdateList(merger *Merger, triggerResultEvent bool) {\n\tt.mutex.Lock()\n\tprevIndex := minItem.Index()\n\treset := t.revision != merger.Revision()\n\tif !reset && t.track != trackDisabled {\n\t\tif t.merger.Length() > 0 {\n\t\t\tprevIndex = t.currentIndex()\n\t\t} else if merger.Length() > 0 {\n\t\t\tprevIndex = merger.First().item.Index()\n\t\t}\n\t}\n\tt.progress = 100\n\tt.merger = merger\n\tif reset {\n\t\tt.selected = make(map[int32]selectedItem)\n\t\tt.revision = merger.Revision()\n\t\tt.version++\n\t}\n\tif t.triggerLoad {\n\t\tt.triggerLoad = false\n\t\tt.eventChan <- tui.Load.AsEvent()\n\t}\n\tif prevIndex >= 0 {\n\t\tpos := t.cy - t.offset\n\t\tcount := t.merger.Length()\n\t\ti := t.merger.FindIndex(prevIndex)\n\t\tif i >= 0 {\n\t\t\tt.cy = i\n\t\t\tt.offset = t.cy - pos\n\t\t} else if t.track == trackCurrent {\n\t\t\tt.track = trackDisabled\n\t\t\tt.cy = pos\n\t\t\tt.offset = 0\n\t\t} else if t.cy > count {\n\t\t\t// Try to keep the vertical position when the list shrinks\n\t\t\tt.cy = count - util.Min(count, t.maxItems()) + pos\n\t\t}\n\t}\n\tif !t.reading {\n\t\tswitch t.merger.Length() {\n\t\tcase 0:\n\t\t\tzero := tui.Zero.AsEvent()\n\t\t\tif _, prs := t.keymap[zero]; prs {\n\t\t\t\tt.eventChan <- zero\n\t\t\t}\n\t\tcase 1:\n\t\t\tone := tui.One.AsEvent()\n\t\t\tif _, prs := t.keymap[one]; prs {\n\t\t\t\tt.eventChan <- one\n\t\t\t}\n\t\t}\n\t\tif triggerResultEvent && t.hasResultActions {\n\t\t\tt.eventChan <- tui.Result.AsEvent()\n\t\t}\n\t}\n\tt.mutex.Unlock()\n\tt.reqBox.Set(reqInfo, nil)\n\tt.reqBox.Set(reqList, nil)\n}\n\nfunc (t *Terminal) output() bool {\n\tif t.printQuery {\n\t\tt.printer(string(t.input))\n\t}\n\tif len(t.expect) > 0 {\n\t\tt.printer(t.pressed)\n\t}\n\tfound := len(t.selected) > 0\n\tif !found {\n\t\tcurrent := t.currentItem()\n\t\tif current != nil {\n\t\t\tt.printer(current.AsString(t.ansi))\n\t\t\tfound = true\n\t\t}\n\t} else {\n\t\tfor _, sel := range t.sortSelected() {\n\t\t\tt.printer(sel.item.AsString(t.ansi))\n\t\t}\n\t}\n\treturn found\n}\n\nfunc (t *Terminal) sortSelected() []selectedItem {\n\tsels := make([]selectedItem, 0, len(t.selected))\n\tfor _, sel := range t.selected {\n\t\tsels = append(sels, sel)\n\t}\n\tsort.Sort(byTimeOrder(sels))\n\treturn sels\n}\n\nfunc (t *Terminal) displayWidth(runes []rune) int {\n\twidth, _ := util.RunesWidth(runes, 0, t.tabstop, math.MaxInt32)\n\treturn width\n}\n\nconst (\n\tminWidth  = 4\n\tminHeight = 3\n)\n\nfunc calculateSize(base int, size sizeSpec, occupied int, minSize int, pad int) int {\n\tmax := base - occupied\n\tif max < minSize {\n\t\tmax = minSize\n\t}\n\tif size.percent {\n\t\treturn util.Constrain(int(float64(base)*0.01*size.size), minSize, max)\n\t}\n\treturn util.Constrain(int(size.size)+pad, minSize, max)\n}\n\nfunc (t *Terminal) adjustMarginAndPadding() (int, int, [4]int, [4]int) {\n\tscreenWidth := t.tui.MaxX()\n\tscreenHeight := t.tui.MaxY()\n\tmarginInt := [4]int{}  // TRBL\n\tpaddingInt := [4]int{} // TRBL\n\tsizeSpecToInt := func(index int, spec sizeSpec) int {\n\t\tif spec.percent {\n\t\t\tvar max float64\n\t\t\tif index%2 == 0 {\n\t\t\t\tmax = float64(screenHeight)\n\t\t\t} else {\n\t\t\t\tmax = float64(screenWidth)\n\t\t\t}\n\t\t\treturn int(max * spec.size * 0.01)\n\t\t}\n\t\treturn int(spec.size)\n\t}\n\tfor idx, sizeSpec := range t.padding {\n\t\tpaddingInt[idx] = sizeSpecToInt(idx, sizeSpec)\n\t}\n\n\tbw := t.borderWidth\n\textraMargin := [4]int{} // TRBL\n\tfor idx, sizeSpec := range t.margin {\n\t\tswitch t.borderShape {\n\t\tcase tui.BorderHorizontal:\n\t\t\textraMargin[idx] += 1 - idx%2\n\t\tcase tui.BorderVertical:\n\t\t\textraMargin[idx] += (1 + bw) * (idx % 2)\n\t\tcase tui.BorderTop:\n\t\t\tif idx == 0 {\n\t\t\t\textraMargin[idx]++\n\t\t\t}\n\t\tcase tui.BorderRight:\n\t\t\tif idx == 1 {\n\t\t\t\textraMargin[idx] += 1 + bw\n\t\t\t}\n\t\tcase tui.BorderBottom:\n\t\t\tif idx == 2 {\n\t\t\t\textraMargin[idx]++\n\t\t\t}\n...\n", "needle_token_start": 6616, "needle_token_end": 6758, "code_context_ntokens": 15031, "output": ["```go\nfunc processExecution(action actionType) bool {\n\tswitch action {\n\tcase actTransform,\n\t\tactTransformBorderLabel,\n\t\tactTransformHeader,\n\t\tactTransformPreviewLabel,\n\t\tactTransformPrompt,\n\t\tactTransformQuery,\n\t\tactPreview,\n\t\tactChangePreview,\n\t\tactRefreshPreview,\n\t\tactExecute,\n\t\tactExecuteSilent,\n\t\tactExecuteMulti,\n\t\tactReload,\n\t\tactReloadSync,\n\t\tactBecome:\n\t\treturn true\n\t}\n\treturn false\n}\n```"]}
{"repo": "junegunn/fzf", "name": "asTcellColor", "language": "go", "path": "src/tui/tcell.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function converts a custom color representation into a format compatible with a specific terminal UI library.\n2. **Input**: A custom color type that may include standard colors or 24-bit RGB values.\n3. **Output**: A color object from the terminal UI library that represents the input color, formatted for use in terminal displays.\n4. **Procedure**: The function checks if the input color is a default value and returns the default color of the library if true. If not, it calculates a numerical value by adding a validity flag to the input color's numeric value. If the input color is a 24-bit RGB color, an additional RGB flag is included in the calculation. The resulting value is then converted to the library's color type.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/tui/light.go\npackage tui\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\t\"unicode/utf8\"\n\n\t\"github.com/rivo/uniseg\"\n\n\t\"golang.org/x/term\"\n)\n\nconst (\n\tdefaultWidth  = 80\n\tdefaultHeight = 24\n\n\tdefaultEscDelay = 100\n\tescPollInterval = 5\n\toffsetPollTries = 10\n\tmaxInputBuffer  = 1024 * 1024\n)\n\nconst consoleDevice string = \"/dev/tty\"\n\nvar offsetRegexp *regexp.Regexp = regexp.MustCompile(\"(.*)\\x1b\\\\[([0-9]+);([0-9]+)R\")\nvar offsetRegexpBegin *regexp.Regexp = regexp.MustCompile(\"^\\x1b\\\\[[0-9]+;[0-9]+R\")\n\nfunc (r *LightRenderer) PassThrough(str string) {\n\tr.queued.WriteString(\"\\x1b7\" + str + \"\\x1b8\")\n}\n\nfunc (r *LightRenderer) stderr(str string) {\n\tr.stderrInternal(str, true, \"\")\n}\n\nconst CR string = \"\\x1b[2m\u240d\"\nconst LF string = \"\\x1b[2m\u240a\"\n\nfunc (r *LightRenderer) stderrInternal(str string, allowNLCR bool, resetCode string) {\n\tbytes := []byte(str)\n\trunes := []rune{}\n\tfor len(bytes) > 0 {\n\t\tr, sz := utf8.DecodeRune(bytes)\n\t\tnlcr := r == '\\n' || r == '\\r'\n\t\tif r >= 32 || r == '\\x1b' || nlcr {\n\t\t\tif nlcr && !allowNLCR {\n\t\t\t\tif r == '\\r' {\n\t\t\t\t\trunes = append(runes, []rune(CR+resetCode)...)\n\t\t\t\t} else {\n\t\t\t\t\trunes = append(runes, []rune(LF+resetCode)...)\n\t\t\t\t}\n\t\t\t} else if r != utf8.RuneError {\n\t\t\t\trunes = append(runes, r)\n\t\t\t}\n\t\t}\n\t\tbytes = bytes[sz:]\n\t}\n\tr.queued.WriteString(string(runes))\n}\n\nfunc (r *LightRenderer) csi(code string) string {\n\tfullcode := \"\\x1b[\" + code\n\tr.stderr(fullcode)\n\treturn fullcode\n}\n\nfunc (r *LightRenderer) flush() {\n\tif r.queued.Len() > 0 {\n\t\tfmt.Fprint(os.Stderr, \"\\x1b[?7l\\x1b[?25l\"+r.queued.String()+\"\\x1b[?25h\\x1b[?7h\")\n\t\tr.queued.Reset()\n\t}\n}\n\n// Light renderer\ntype LightRenderer struct {\n\ttheme         *ColorTheme\n\tmouse         bool\n\tforceBlack    bool\n\tclearOnExit   bool\n\tprevDownTime  time.Time\n\tclicks        [][2]int\n\tttyin         *os.File\n\tbuffer        []byte\n\torigState     *term.State\n\twidth         int\n\theight        int\n\tyoffset       int\n\ttabstop       int\n\tescDelay      int\n\tfullscreen    bool\n\tupOneLine     bool\n\tqueued        strings.Builder\n\ty             int\n\tx             int\n\tmaxHeightFunc func(int) int\n\n\t// Windows only\n\tttyinChannel    chan byte\n\tinHandle        uintptr\n\toutHandle       uintptr\n\torigStateInput  uint32\n\torigStateOutput uint32\n}\n\ntype LightWindow struct {\n\trenderer *LightRenderer\n\tcolored  bool\n\tpreview  bool\n\tborder   BorderStyle\n\ttop      int\n\tleft     int\n\twidth    int\n\theight   int\n\tposx     int\n\tposy     int\n\ttabstop  int\n\tfg       Color\n\tbg       Color\n}\n\nfunc NewLightRenderer(theme *ColorTheme, forceBlack bool, mouse bool, tabstop int, clearOnExit bool, fullscreen bool, maxHeightFunc func(int) int) Renderer {\n\tr := LightRenderer{\n\t\ttheme:         theme,\n\t\tforceBlack:    forceBlack,\n\t\tmouse:         mouse,\n\t\tclearOnExit:   clearOnExit,\n\t\tttyin:         openTtyIn(),\n\t\tyoffset:       0,\n\t\ttabstop:       tabstop,\n\t\tfullscreen:    fullscreen,\n\t\tupOneLine:     false,\n\t\tmaxHeightFunc: maxHeightFunc}\n\treturn &r\n}\n\nfunc repeat(r rune, times int) string {\n\tif times > 0 {\n\t\treturn strings.Repeat(string(r), times)\n\t}\n\treturn \"\"\n}\n\nfunc atoi(s string, defaultValue int) int {\n\tvalue, err := strconv.Atoi(s)\n\tif err != nil {\n\t\treturn defaultValue\n\t}\n\treturn value\n}\n\nfunc (r *LightRenderer) Init() {\n\tr.escDelay = atoi(os.Getenv(\"ESCDELAY\"), defaultEscDelay)\n\n\tif err := r.initPlatform(); err != nil {\n\t\terrorExit(err.Error())\n\t}\n\tr.updateTerminalSize()\n\tinitTheme(r.theme, r.defaultTheme(), r.forceBlack)\n\n\tif r.fullscreen {\n\t\tr.smcup()\n\t} else {\n\t\t// We assume that --no-clear is used for repetitive relaunching of fzf.\n\t\t// So we do not clear the lower bottom of the screen.\n\t\tif r.clearOnExit {\n\t\t\tr.csi(\"J\")\n\t\t}\n\t\ty, x := r.findOffset()\n\t\tr.mouse = r.mouse && y >= 0\n\t\t// When --no-clear is used for repetitive relaunching, there is a small\n\t\t// time frame between fzf processes where the user keystrokes are not\n\t\t// captured by either of fzf process which can cause x offset to be\n\t\t// increased and we're left with unwanted extra new line.\n\t\tif x > 0 && r.clearOnExit {\n\t\t\tr.upOneLine = true\n\t\t\tr.makeSpace()\n\t\t}\n\t\tfor i := 1; i < r.MaxY(); i++ {\n\t\t\tr.makeSpace()\n\t\t}\n\t}\n\n\tr.enableMouse()\n\tr.csi(fmt.Sprintf(\"%dA\", r.MaxY()-1))\n\tr.csi(\"G\")\n\tr.csi(\"K\")\n\tif !r.clearOnExit && !r.fullscreen {\n\t\tr.csi(\"s\")\n\t}\n\tif !r.fullscreen && r.mouse {\n\t\tr.yoffset, _ = r.findOffset()\n\t}\n}\n\nfunc (r *LightRenderer) Resize(maxHeightFunc func(int) int) {\n\tr.maxHeightFunc = maxHeightFunc\n}\n\nfunc (r *LightRenderer) makeSpace() {\n\tr.stderr(\"\\n\")\n\tr.csi(\"G\")\n}\n\nfunc (r *LightRenderer) move(y int, x int) {\n\t// w.csi(\"u\")\n\tif r.y < y {\n\t\tr.csi(fmt.Sprintf(\"%dB\", y-r.y))\n\t} else if r.y > y {\n\t\tr.csi(fmt.Sprintf(\"%dA\", r.y-y))\n\t}\n\tr.stderr(\"\\r\")\n\tif x > 0 {\n\t\tr.csi(fmt.Sprintf(\"%dC\", x))\n\t}\n\tr.y = y\n\tr.x = x\n}\n\nfunc (r *LightRenderer) origin() {\n\tr.move(0, 0)\n}\n\nfunc getEnv(name string, defaultValue int) int {\n\tenv := os.Getenv(name)\n\tif len(env) == 0 {\n\t\treturn defaultValue\n\t}\n\treturn atoi(env, defaultValue)\n}\n\nfunc (r *LightRenderer) getBytes() []byte {\n\treturn r.getBytesInternal(r.buffer, false)\n}\n\nfunc (r *LightRenderer) getBytesInternal(buffer []byte, nonblock bool) []byte {\n\tc, ok := r.getch(nonblock)\n\tif !nonblock && !ok {\n\t\tr.Close()\n\t\terrorExit(\"Failed to read \" + consoleDevice)\n\t}\n\n\tretries := 0\n\tif c == Esc.Int() || nonblock {\n\t\tretries = r.escDelay / escPollInterval\n\t}\n\tbuffer = append(buffer, byte(c))\n\n\tpc := c\n\tfor {\n\t\tc, ok = r.getch(true)\n\t\tif !ok {\n\t\t\tif retries > 0 {\n\t\t\t\tretries--\n\t\t\t\ttime.Sleep(escPollInterval * time.Millisecond)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbreak\n\t\t} else if c == Esc.Int() && pc != c {\n\t\t\tretries = r.escDelay / escPollInterval\n\t\t} else {\n\t\t\tretries = 0\n\t\t}\n\t\tbuffer = append(buffer, byte(c))\n\t\tpc = c\n\n\t\t// This should never happen under normal conditions,\n\t\t// so terminate fzf immediately.\n\t\tif len(buffer) > maxInputBuffer {\n\t\t\tr.Close()\n\t\t\tpanic(fmt.Sprintf(\"Input buffer overflow (%d): %v\", len(buffer), buffer))\n\t\t}\n\t}\n\n\treturn buffer\n}\n\nfunc (r *LightRenderer) GetChar() Event {\n\tif len(r.buffer) == 0 {\n\t\tr.buffer = r.getBytes()\n\t}\n\tif len(r.buffer) == 0 {\n\t\tpanic(\"Empty buffer\")\n\t}\n\n\tsz := 1\n\tdefer func() {\n\t\tr.buffer = r.buffer[sz:]\n\t}()\n\n\tswitch r.buffer[0] {\n\tcase CtrlC.Byte():\n\t\treturn Event{CtrlC, 0, nil}\n\tcase CtrlG.Byte():\n\t\treturn Event{CtrlG, 0, nil}\n\tcase CtrlQ.Byte():\n\t\treturn Event{CtrlQ, 0, nil}\n\tcase 127:\n\t\treturn Event{Backspace, 0, nil}\n\tcase 0:\n\t\treturn Event{CtrlSpace, 0, nil}\n\tcase 28:\n\t\treturn Event{CtrlBackSlash, 0, nil}\n\tcase 29:\n\t\treturn Event{CtrlRightBracket, 0, nil}\n\tcase 30:\n\t\treturn Event{CtrlCaret, 0, nil}\n\tcase 31:\n\t\treturn Event{CtrlSlash, 0, nil}\n\tcase Esc.Byte():\n\t\tev := r.escSequence(&sz)\n\t\t// Second chance\n\t\tif ev.Type == Invalid {\n\t\t\tr.buffer = r.getBytes()\n\t\t\tev = r.escSequence(&sz)\n\t\t}\n\t\treturn ev\n\t}\n\n\t// CTRL-A ~ CTRL-Z\n\tif r.buffer[0] <= CtrlZ.Byte() {\n\t\treturn Event{EventType(r.buffer[0]), 0, nil}\n\t}\n\tchar, rsz := utf8.DecodeRune(r.buffer)\n\tif char == utf8.RuneError {\n\t\treturn Event{Esc, 0, nil}\n\t}\n\tsz = rsz\n\treturn Event{Rune, char, nil}\n}\n\nfunc (r *LightRenderer) escSequence(sz *int) Event {\n\tif len(r.buffer) < 2 {\n\t\treturn Event{Esc, 0, nil}\n\t}\n\n\tloc := offsetRegexpBegin.FindIndex(r.buffer)\n\tif loc != nil && loc[0] == 0 {\n\t\t*sz = loc[1]\n\t\treturn Event{Invalid, 0, nil}\n\t}\n\n\t*sz = 2\n\tif r.buffer[1] >= 1 && r.buffer[1] <= 'z'-'a'+1 {\n\t\treturn CtrlAltKey(rune(r.buffer[1] + 'a' - 1))\n\t}\n\talt := false\n\tif len(r.buffer) > 2 && r.buffer[1] == Esc.Byte() {\n\t\tr.buffer = r.buffer[1:]\n\t\talt = true\n\t}\n\tswitch r.buffer[1] {\n\tcase Esc.Byte():\n\t\treturn Event{Esc, 0, nil}\n\tcase 127:\n\t\treturn Event{AltBackspace, 0, nil}\n\tcase '[', 'O':\n\t\tif len(r.buffer) < 3 {\n\t\t\treturn Event{Invalid, 0, nil}\n\t\t}\n\t\t*sz = 3\n\t\tswitch r.buffer[2] {\n\t\tcase 'D':\n\t\t\tif alt {\n\t\t\t\treturn Event{AltLeft, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Left, 0, nil}\n\t\tcase 'C':\n\t\t\tif alt {\n\t\t\t\t// Ugh..\n\t\t\t\treturn Event{AltRight, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Right, 0, nil}\n\t\tcase 'B':\n\t\t\tif alt {\n\t\t\t\treturn Event{AltDown, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Down, 0, nil}\n\t\tcase 'A':\n\t\t\tif alt {\n\t\t\t\treturn Event{AltUp, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Up, 0, nil}\n\t\tcase 'Z':\n\t\t\treturn Event{ShiftTab, 0, nil}\n\t\tcase 'H':\n\t\t\treturn Event{Home, 0, nil}\n\t\tcase 'F':\n\t\t\treturn Event{End, 0, nil}\n\t\tcase '<':\n\t\t\treturn r.mouseSequence(sz)\n\t\tcase 'P':\n\t\t\treturn Event{F1, 0, nil}\n\t\tcase 'Q':\n\t\t\treturn Event{F2, 0, nil}\n\t\tcase 'R':\n\t\t\treturn Event{F3, 0, nil}\n\t\tcase 'S':\n\t\t\treturn Event{F4, 0, nil}\n\t\tcase '1', '2', '3', '4', '5', '6', '7', '8':\n\t\t\tif len(r.buffer) < 4 {\n\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\t}\n\t\t\t*sz = 4\n\t\t\tswitch r.buffer[2] {\n\t\t\tcase '2':\n\t\t\t\tif r.buffer[3] == '~' {\n\t\t\t\t\treturn Event{Insert, 0, nil}\n\t\t\t\t}\n\t\t\t\tif len(r.buffer) > 4 && r.buffer[4] == '~' {\n\t\t\t\t\t*sz = 5\n\t\t\t\t\tswitch r.buffer[3] {\n\t\t\t\t\tcase '0':\n\t\t\t\t\t\treturn Event{F9, 0, nil}\n\t\t\t\t\tcase '1':\n\t\t\t\t\t\treturn Event{F10, 0, nil}\n\t\t\t\t\tcase '3':\n\t\t\t\t\t\treturn Event{F11, 0, nil}\n\t\t\t\t\tcase '4':\n\t\t\t\t\t\treturn Event{F12, 0, nil}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Bracketed paste mode: \\e[200~ ... \\e[201~\n\t\t\t\tif len(r.buffer) > 5 && r.buffer[3] == '0' && (r.buffer[4] == '0' || r.buffer[4] == '1') && r.buffer[5] == '~' {\n\t\t\t\t\t// Immediately discard the sequence from the buffer and reread input\n\t\t\t\t\tr.buffer = r.buffer[6:]\n\t\t\t\t\t*sz = 0\n\t\t\t\t\treturn r.GetChar()\n\t\t\t\t}\n\t\t\t\treturn Event{Invalid, 0, nil} // INS\n\t\t\tcase '3':\n\t\t\t\tif r.buffer[3] == '~' {\n\t\t\t\t\treturn Event{Delete, 0, nil}\n\t\t\t\t}\n\t\t\t\tif len(r.buffer) == 6 && r.buffer[5] == '~' {\n\t\t\t\t\t*sz = 6\n\t\t\t\t\tswitch r.buffer[4] {\n\t\t\t\t\tcase '5':\n\t\t\t\t\t\treturn Event{CtrlDelete, 0, nil}\n\t\t\t\t\tcase '2':\n\t\t\t\t\t\treturn Event{ShiftDelete, 0, nil}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\tcase '4':\n\t\t\t\treturn Event{End, 0, nil}\n\t\t\tcase '5':\n\t\t\t\treturn Event{PageUp, 0, nil}\n\t\t\tcase '6':\n\t\t\t\treturn Event{PageDown, 0, nil}\n\t\t\tcase '7':\n\t\t\t\treturn Event{Home, 0, nil}\n\t\t\tcase '8':\n\t\t\t\treturn Event{End, 0, nil}\n\t\t\tcase '1':\n\t\t\t\tswitch r.buffer[3] {\n\t\t\t\tcase '~':\n\t\t\t\t\treturn Event{Home, 0, nil}\n\t\t\t\tcase '1', '2', '3', '4', '5', '7', '8', '9':\n\t\t\t\t\tif len(r.buffer) == 5 && r.buffer[4] == '~' {\n\t\t\t\t\t\t*sz = 5\n\t\t\t\t\t\tswitch r.buffer[3] {\n\t\t\t\t\t\tcase '1':\n\t\t\t\t\t\t\treturn Event{F1, 0, nil}\n\t\t\t\t\t\tcase '2':\n\t\t\t\t\t\t\treturn Event{F2, 0, nil}\n\t\t\t\t\t\tcase '3':\n\t\t\t\t\t\t\treturn Event{F3, 0, nil}\n\t\t\t\t\t\tcase '4':\n\t\t\t\t\t\t\treturn Event{F4, 0, nil}\n\t\t\t\t\t\tcase '5':\n\t\t\t\t\t\t\treturn Event{F5, 0, nil}\n\t\t\t\t\t\tcase '7':\n\t\t\t\t\t\t\treturn Event{F6, 0, nil}\n\t\t\t\t\t\tcase '8':\n\t\t\t\t\t\t\treturn Event{F7, 0, nil}\n\t\t\t\t\t\tcase '9':\n\t\t\t\t\t\t\treturn Event{F8, 0, nil}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\t\tcase ';':\n\t\t\t\t\tif len(r.buffer) < 6 {\n\t\t\t\t\t\treturn Event{Invalid, 0, nil}\n\t\t\t\t\t}\n\t\t\t\t\t*sz = 6\n\t\t\t\t\tswitch r.buffer[4] {\n\t\t\t\t\tcase '1', '2', '3', '4', '5':\n\t\t\t\t\t\t//                   Kitty      iTerm2     WezTerm\n\t\t\t\t\t\t// SHIFT-ARROW       \"\\e[1;2D\"\n\t\t\t\t\t\t// ALT-SHIFT-ARROW   \"\\e[1;4D\"  \"\\e[1;10D\" \"\\e[1;4D\"\n\t\t\t\t\t\t// CTRL-SHIFT-ARROW  \"\\e[1;6D\"             N/A\n\t\t\t\t\t\t// CMD-SHIFT-ARROW   \"\\e[1;10D\" N/A        N/A (\"\\e[1;2D\")\n\t\t\t\t\t\talt := r.buffer[4] == '3'\n\t\t\t\t\t\tchar := r.buffer[5]\n\t\t\t\t\t\taltShift := false\n...\n// Path: src/tui/light_unix.go\n//go:build !windows\n\npackage tui\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"syscall\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n\t\"golang.org/x/sys/unix\"\n\t\"golang.org/x/term\"\n)\n\nfunc IsLightRendererSupported() bool {\n\treturn true\n}\n\nfunc (r *LightRenderer) defaultTheme() *ColorTheme {\n\tif strings.Contains(os.Getenv(\"TERM\"), \"256\") {\n\t\treturn Dark256\n\t}\n\tcolors, err := exec.Command(\"tput\", \"colors\").Output()\n\tif err == nil && atoi(strings.TrimSpace(string(colors)), 16) > 16 {\n\t\treturn Dark256\n\t}\n\treturn Default16\n}\n\nfunc (r *LightRenderer) fd() int {\n\treturn int(r.ttyin.Fd())\n}\n\nfunc (r *LightRenderer) initPlatform() error {\n\tfd := r.fd()\n\torigState, err := term.GetState(fd)\n\tif err != nil {\n\t\treturn err\n\t}\n\tr.origState = origState\n\tterm.MakeRaw(fd)\n\treturn nil\n}\n\nfunc (r *LightRenderer) closePlatform() {\n\t// NOOP\n}\n\nfunc openTtyIn() *os.File {\n\tin, err := os.OpenFile(consoleDevice, syscall.O_RDONLY, 0)\n\tif err != nil {\n\t\ttty := ttyname()\n\t\tif len(tty) > 0 {\n\t\t\tif in, err := os.OpenFile(tty, syscall.O_RDONLY, 0); err == nil {\n\t\t\t\treturn in\n\t\t\t}\n\t\t}\n\t\tfmt.Fprintln(os.Stderr, \"Failed to open \"+consoleDevice)\n\t\tutil.Exit(2)\n\t}\n\treturn in\n}\n\nfunc (r *LightRenderer) setupTerminal() {\n\tterm.MakeRaw(r.fd())\n}\n\nfunc (r *LightRenderer) restoreTerminal() {\n\tterm.Restore(r.fd(), r.origState)\n}\n\nfunc (r *LightRenderer) updateTerminalSize() {\n\twidth, height, err := term.GetSize(r.fd())\n\n\tif err == nil {\n\t\tr.width = width\n\t\tr.height = r.maxHeightFunc(height)\n\t} else {\n\t\tr.width = getEnv(\"COLUMNS\", defaultWidth)\n\t\tr.height = r.maxHeightFunc(getEnv(\"LINES\", defaultHeight))\n\t}\n}\n\nfunc (r *LightRenderer) findOffset() (row int, col int) {\n\tr.csi(\"6n\")\n\tr.flush()\n\tbytes := []byte{}\n\tfor tries := 0; tries < offsetPollTries; tries++ {\n\t\tbytes = r.getBytesInternal(bytes, tries > 0)\n\t\toffsets := offsetRegexp.FindSubmatch(bytes)\n\t\tif len(offsets) > 3 {\n\t\t\t// Add anything we skipped over to the input buffer\n\t\t\tr.buffer = append(r.buffer, offsets[1]...)\n\t\t\treturn atoi(string(offsets[2]), 0) - 1, atoi(string(offsets[3]), 0) - 1\n\t\t}\n\t}\n\treturn -1, -1\n}\n\nfunc (r *LightRenderer) getch(nonblock bool) (int, bool) {\n\tb := make([]byte, 1)\n\tfd := r.fd()\n\tutil.SetNonblock(r.ttyin, nonblock)\n\t_, err := util.Read(fd, b)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\treturn int(b[0]), true\n}\n\nfunc (r *LightRenderer) Size() TermSize {\n\tws, err := unix.IoctlGetWinsize(int(r.ttyin.Fd()), unix.TIOCGWINSZ)\n\tif err != nil {\n\t\treturn TermSize{}\n\t}\n\treturn TermSize{int(ws.Row), int(ws.Col), int(ws.Xpixel), int(ws.Ypixel)}\n}\n\n// Path: src/tui/light_windows.go\n//go:build windows\n\npackage tui\n\nimport (\n\t\"os\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n\t\"golang.org/x/sys/windows\"\n)\n\nconst (\n\ttimeoutInterval = 10\n)\n\nvar (\n\tconsoleFlagsInput  = uint32(windows.ENABLE_VIRTUAL_TERMINAL_INPUT | windows.ENABLE_PROCESSED_INPUT | windows.ENABLE_EXTENDED_FLAGS)\n\tconsoleFlagsOutput = uint32(windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING | windows.ENABLE_PROCESSED_OUTPUT | windows.DISABLE_NEWLINE_AUTO_RETURN)\n)\n\n// IsLightRendererSupported checks to see if the Light renderer is supported\nfunc IsLightRendererSupported() bool {\n\tvar oldState uint32\n\t// enable vt100 emulation (https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences)\n\tif windows.GetConsoleMode(windows.Stderr, &oldState) != nil {\n\t\treturn false\n\t}\n\t// attempt to set mode to determine if we support VT 100 codes. This will work on newer Windows 10\n\t// version:\n\tcanSetVt100 := windows.SetConsoleMode(windows.Stderr, oldState|windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING) == nil\n\tvar checkState uint32\n\tif windows.GetConsoleMode(windows.Stderr, &checkState) != nil ||\n\t\t(checkState&windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING) != windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING {\n\t\treturn false\n\t}\n\twindows.SetConsoleMode(windows.Stderr, oldState)\n\treturn canSetVt100\n}\n\nfunc (r *LightRenderer) defaultTheme() *ColorTheme {\n\t// the getenv check is borrowed from here: https://github.com/gdamore/tcell/commit/0c473b86d82f68226a142e96cc5a34c5a29b3690#diff-b008fcd5e6934bf31bc3d33bf49f47d8R178:\n\tif !IsLightRendererSupported() || os.Getenv(\"ConEmuPID\") != \"\" || os.Getenv(\"TCELL_TRUECOLOR\") == \"disable\" {\n\t\treturn Default16\n\t}\n\treturn Dark256\n}\n\nfunc (r *LightRenderer) initPlatform() error {\n\t//outHandle := windows.Stdout\n\toutHandle, _ := syscall.Open(\"CONOUT$\", syscall.O_RDWR, 0)\n\t// enable vt100 emulation (https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences)\n\tif err := windows.GetConsoleMode(windows.Handle(outHandle), &r.origStateOutput); err != nil {\n\t\treturn err\n\t}\n\tr.outHandle = uintptr(outHandle)\n\tinHandle, _ := syscall.Open(\"CONIN$\", syscall.O_RDWR, 0)\n\tif err := windows.GetConsoleMode(windows.Handle(inHandle), &r.origStateInput); err != nil {\n\t\treturn err\n\t}\n\tr.inHandle = uintptr(inHandle)\n\n\tr.setupTerminal()\n\n\t// channel for non-blocking reads. Buffer to make sure\n\t// we get the ESC sets:\n\tr.ttyinChannel = make(chan byte, 1024)\n\n\t// the following allows for non-blocking IO.\n\t// syscall.SetNonblock() is a NOOP under Windows.\n\tgo func() {\n\t\tfd := int(r.inHandle)\n\t\tb := make([]byte, 1)\n\t\tfor {\n\t\t\t// HACK: if run from PSReadline, something resets ConsoleMode to remove ENABLE_VIRTUAL_TERMINAL_INPUT.\n\t\t\t_ = windows.SetConsoleMode(windows.Handle(r.inHandle), consoleFlagsInput)\n\n\t\t\t_, err := util.Read(fd, b)\n\t\t\tif err == nil {\n\t\t\t\tr.ttyinChannel <- b[0]\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn nil\n}\n\nfunc (r *LightRenderer) closePlatform() {\n\twindows.SetConsoleMode(windows.Handle(r.outHandle), r.origStateOutput)\n\twindows.SetConsoleMode(windows.Handle(r.inHandle), r.origStateInput)\n}\n\nfunc openTtyIn() *os.File {\n\t// not used\n\treturn nil\n}\n\nfunc (r *LightRenderer) setupTerminal() error {\n\tif err := windows.SetConsoleMode(windows.Handle(r.outHandle), consoleFlagsOutput); err != nil {\n\t\treturn err\n\t}\n\treturn windows.SetConsoleMode(windows.Handle(r.inHandle), consoleFlagsInput)\n}\n\nfunc (r *LightRenderer) restoreTerminal() error {\n\tif err := windows.SetConsoleMode(windows.Handle(r.inHandle), r.origStateInput); err != nil {\n\t\treturn err\n\t}\n\treturn windows.SetConsoleMode(windows.Handle(r.outHandle), r.origStateOutput)\n}\n\nfunc (r *LightRenderer) Size() TermSize {\n\tvar w, h int\n\tvar bufferInfo windows.ConsoleScreenBufferInfo\n\tif err := windows.GetConsoleScreenBufferInfo(windows.Handle(r.outHandle), &bufferInfo); err != nil {\n\t\tw = getEnv(\"COLUMNS\", defaultWidth)\n\t\th = r.maxHeightFunc(getEnv(\"LINES\", defaultHeight))\n\n\t} else {\n\t\tw = int(bufferInfo.Window.Right - bufferInfo.Window.Left)\n\t\th = r.maxHeightFunc(int(bufferInfo.Window.Bottom - bufferInfo.Window.Top))\n\t}\n\treturn TermSize{h, w, 0, 0}\n}\n\nfunc (r *LightRenderer) updateTerminalSize() {\n\tsize := r.Size()\n\tr.width = size.Columns\n\tr.height = size.Lines\n}\n\nfunc (r *LightRenderer) findOffset() (row int, col int) {\n\tvar bufferInfo windows.ConsoleScreenBufferInfo\n\tif err := windows.GetConsoleScreenBufferInfo(windows.Handle(r.outHandle), &bufferInfo); err != nil {\n\t\treturn -1, -1\n\t}\n\treturn int(bufferInfo.CursorPosition.X), int(bufferInfo.CursorPosition.Y)\n}\n\nfunc (r *LightRenderer) getch(nonblock bool) (int, bool) {\n\tif nonblock {\n\t\tselect {\n\t\tcase bc := <-r.ttyinChannel:\n\t\t\treturn int(bc), true\n\t\tcase <-time.After(timeoutInterval * time.Millisecond):\n\t\t\treturn 0, false\n\t\t}\n\t} else {\n\t\tbc := <-r.ttyinChannel\n\t\treturn int(bc), true\n\t}\n}\n\n// Path: src/tui/tcell.go\n//go:build tcell || windows\n\npackage tui\n\nimport (\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/gdamore/tcell/v2\"\n\t\"github.com/gdamore/tcell/v2/encoding\"\n\t\"github.com/junegunn/fzf/src/util\"\n\n\t\"github.com/rivo/uniseg\"\n)\n\nfunc HasFullscreenRenderer() bool {\n\treturn true\n}\n\nvar DefaultBorderShape BorderShape = BorderSharp\n\n\nfunc asTcellColor(color Color) tcell.Color {\n\tif color == colDefault {\n\t\treturn tcell.ColorDefault\n\t}\n\n\tvalue := uint64(tcell.ColorValid) + uint64(color)\n\tif color.is24() {\n\t\tvalue = value | uint64(tcell.ColorIsRGB)\n\t}\n\treturn tcell.Color(value)\n}\n\nfunc (p ColorPair) style() tcell.Style {\n\tstyle := tcell.StyleDefault\n\treturn style.Foreground(asTcellColor(p.Fg())).Background(asTcellColor(p.Bg()))\n}\n\ntype Attr int32\n\ntype TcellWindow struct {\n\tcolor       bool\n\tpreview     bool\n\ttop         int\n\tleft        int\n\twidth       int\n\theight      int\n\tnormal      ColorPair\n\tlastX       int\n\tlastY       int\n\tmoveCursor  bool\n\tborderStyle BorderStyle\n}\n\nfunc (w *TcellWindow) Top() int {\n\treturn w.top\n}\n\nfunc (w *TcellWindow) Left() int {\n\treturn w.left\n}\n\nfunc (w *TcellWindow) Width() int {\n\treturn w.width\n}\n\nfunc (w *TcellWindow) Height() int {\n\treturn w.height\n}\n\nfunc (w *TcellWindow) Refresh() {\n\tif w.moveCursor {\n\t\t_screen.ShowCursor(w.left+w.lastX, w.top+w.lastY)\n\t\tw.moveCursor = false\n\t}\n\tw.lastX = 0\n\tw.lastY = 0\n}\n\nfunc (w *TcellWindow) FinishFill() {\n\t// NO-OP\n}\n\nconst (\n\tBold          Attr = Attr(tcell.AttrBold)\n\tDim                = Attr(tcell.AttrDim)\n\tBlink              = Attr(tcell.AttrBlink)\n\tReverse            = Attr(tcell.AttrReverse)\n\tUnderline          = Attr(tcell.AttrUnderline)\n\tStrikeThrough      = Attr(tcell.AttrStrikeThrough)\n\tItalic             = Attr(tcell.AttrItalic)\n)\n\nconst (\n\tAttrUndefined = Attr(0)\n\tAttrRegular   = Attr(1 << 7)\n\tAttrClear     = Attr(1 << 8)\n)\n\nfunc (r *FullscreenRenderer) PassThrough(str string) {\n\t// No-op\n\t// https://github.com/gdamore/tcell/pull/650#issuecomment-1806442846\n}\n\nfunc (r *FullscreenRenderer) Resize(maxHeightFunc func(int) int) {}\n\nfunc (r *FullscreenRenderer) defaultTheme() *ColorTheme {\n\tif _screen.Colors() >= 256 {\n\t\treturn Dark256\n\t}\n\treturn Default16\n}\n\nvar (\n\t_colorToAttribute = []tcell.Color{\n\t\ttcell.ColorBlack,\n\t\ttcell.ColorRed,\n\t\ttcell.ColorGreen,\n\t\ttcell.ColorYellow,\n\t\ttcell.ColorBlue,\n\t\ttcell.ColorDarkMagenta,\n\t\ttcell.ColorLightCyan,\n\t\ttcell.ColorWhite,\n\t}\n)\n\nfunc (c Color) Style() tcell.Color {\n\tif c <= colDefault {\n\t\treturn tcell.ColorDefault\n\t} else if c >= colBlack && c <= colWhite {\n\t\treturn _colorToAttribute[int(c)]\n\t} else {\n\t\treturn tcell.Color(c)\n\t}\n}\n\nfunc (a Attr) Merge(b Attr) Attr {\n\treturn a | b\n}\n\n// handle the following as private members of FullscreenRenderer instance\n// they are declared here to prevent introducing tcell library in non-windows builds\nvar (\n\t_screen          tcell.Screen\n\t_prevMouseButton tcell.ButtonMask\n\t_initialResize   bool = true\n)\n\nfunc (r *FullscreenRenderer) initScreen() {\n\ts, e := tcell.NewScreen()\n\tif e != nil {\n\t\terrorExit(e.Error())\n\t}\n\tif e = s.Init(); e != nil {\n\t\terrorExit(e.Error())\n\t}\n\tif r.mouse {\n\t\ts.EnableMouse()\n\t} else {\n\t\ts.DisableMouse()\n\t}\n\t_screen = s\n}\n\nfunc (r *FullscreenRenderer) Init() {\n\tif os.Getenv(\"TERM\") == \"cygwin\" {\n\t\tos.Setenv(\"TERM\", \"\")\n\t}\n\tencoding.Register()\n\n\tr.initScreen()\n\tinitTheme(r.theme, r.defaultTheme(), r.forceBlack)\n}\n\nfunc (r *FullscreenRenderer) Top() int {\n\treturn 0\n}\n\nfunc (r *FullscreenRenderer) MaxX() int {\n\tncols, _ := _screen.Size()\n\treturn int(ncols)\n}\n\nfunc (r *FullscreenRenderer) MaxY() int {\n\t_, nlines := _screen.Size()\n\treturn int(nlines)\n}\n\nfunc (w *TcellWindow) X() int {\n\treturn w.lastX\n}\n\nfunc (w *TcellWindow) Y() int {\n\treturn w.lastY\n}\n\nfunc (r *FullscreenRenderer) Clear() {\n\t_screen.Sync()\n\t_screen.Clear()\n}\n\nfunc (r *FullscreenRenderer) NeedScrollbarRedraw() bool {\n\treturn true\n}\n\nfunc (r *FullscreenRenderer) ShouldEmitResizeEvent() bool {\n\treturn true\n}\n\nfunc (r *FullscreenRenderer) Refresh() {\n\t// noop\n}\n\n// TODO: Pixel width and height not implemented\nfunc (r *FullscreenRenderer) Size() TermSize {\n\tcols, lines := _screen.Size()\n\treturn TermSize{lines, cols, 0, 0}\n}\n\nfunc (r *FullscreenRenderer) GetChar() Event {\n\tev := _screen.PollEvent()\n\tswitch ev := ev.(type) {\n\tcase *tcell.EventResize:\n\t\t// Ignore the first resize event\n\t\t// https://github.com/gdamore/tcell/blob/v2.7.0/TUTORIAL.md?plain=1#L18\n\t\tif _initialResize {\n\t\t\t_initialResize = false\n\t\t\treturn Event{Invalid, 0, nil}\n\t\t}\n\t\treturn Event{Resize, 0, nil}\n\n\t// process mouse events:\n\tcase *tcell.EventMouse:\n\t\t// mouse down events have zeroed buttons, so we can't use them\n\t\t// mouse up event consists of two events, 1. (main) event with modifier and other metadata, 2. event with zeroed buttons\n\t\t// so mouse click is three consecutive events, but the first and last are indistinguishable from movement events (with released buttons)\n\t\t// dragging has same structure, it only repeats the middle (main) event appropriately\n\t\tx, y := ev.Position()\n\t\tmod := ev.Modifiers() != 0\n\n\t\t// since we dont have mouse down events (unlike LightRenderer), we need to track state in prevButton\n\t\tprevButton, button := _prevMouseButton, ev.Buttons()\n\t\t_prevMouseButton = button\n\t\tdrag := prevButton == button\n\n\t\tswitch {\n\t\tcase button&tcell.WheelDown != 0:\n\t\t\treturn Event{Mouse, 0, &MouseEvent{y, x, -1, false, false, false, mod}}\n\t\tcase button&tcell.WheelUp != 0:\n\t\t\treturn Event{Mouse, 0, &MouseEvent{y, x, +1, false, false, false, mod}}\n\t\tcase button&tcell.Button1 != 0:\n\t\t\tdouble := false\n\t\t\tif !drag {\n\t\t\t\t// all potential double click events put their coordinates in the clicks array\n\t\t\t\t// double click event has two conditions, temporal and spatial, the first is checked here\n\t\t\t\tnow := time.Now()\n\t\t\t\tif now.Sub(r.prevDownTime) < doubleClickDuration {\n\t\t\t\t\tr.clicks = append(r.clicks, [2]int{x, y})\n\t\t\t\t} else {\n\t\t\t\t\tr.clicks = [][2]int{{x, y}}\n\t\t\t\t}\n\t\t\t\tr.prevDownTime = now\n\n\t\t\t\t// detect double clicks (also check for spatial condition)\n\t\t\t\tn := len(r.clicks)\n\t\t\t\tdouble = n > 1 && r.clicks[n-2][0] == r.clicks[n-1][0] && r.clicks[n-2][1] == r.clicks[n-1][1]\n\t\t\t\tif double {\n\t\t\t\t\t// make sure two consecutive double clicks require four clicks\n\t\t\t\t\tr.clicks = [][2]int{}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// fire single or double click event\n\t\t\treturn Event{Mouse, 0, &MouseEvent{y, x, 0, true, !double, double, mod}}\n\t\tcase button&tcell.Button2 != 0:\n\t\t\treturn Event{Mouse, 0, &MouseEvent{y, x, 0, false, true, false, mod}}\n\t\tdefault:\n\t\t\t// double and single taps on Windows don't quite work due to\n\t\t\t// the console acting on the events and not allowing us\n\t\t\t// to consume them.\n\t\t\tleft := button&tcell.Button1 != 0\n\t\t\tdown := left || button&tcell.Button3 != 0\n\t\t\tdouble := false\n\n\t\t\treturn Event{Mouse, 0, &MouseEvent{y, x, 0, left, down, double, mod}}\n\t\t}\n\n\t\t// process keyboard:\n\tcase *tcell.EventKey:\n\t\tmods := ev.Modifiers()\n\t\tnone := mods == tcell.ModNone\n\t\talt := (mods & tcell.ModAlt) > 0\n\t\tctrl := (mods & tcell.ModCtrl) > 0\n\t\tshift := (mods & tcell.ModShift) > 0\n\t\tctrlAlt := ctrl && alt\n\t\taltShift := alt && shift\n\n\t\tkeyfn := func(r rune) Event {\n\t\t\tif alt {\n\t\t\t\treturn CtrlAltKey(r)\n\t\t\t}\n\t\t\treturn EventType(CtrlA.Int() - 'a' + int(r)).AsEvent()\n\t\t}\n\t\tswitch ev.Key() {\n\t\t// section 1: Ctrl+(Alt)+[a-z]\n\t\tcase tcell.KeyCtrlA:\n\t\t\treturn keyfn('a')\n\t\tcase tcell.KeyCtrlB:\n\t\t\treturn keyfn('b')\n\t\tcase tcell.KeyCtrlC:\n\t\t\treturn keyfn('c')\n\t\tcase tcell.KeyCtrlD:\n\t\t\treturn keyfn('d')\n\t\tcase tcell.KeyCtrlE:\n\t\t\treturn keyfn('e')\n\t\tcase tcell.KeyCtrlF:\n\t\t\treturn keyfn('f')\n\t\tcase tcell.KeyCtrlG:\n\t\t\treturn keyfn('g')\n\t\tcase tcell.KeyCtrlH:\n\t\t\tswitch ev.Rune() {\n\t\t\tcase 0:\n\t\t\t\tif ctrl {\n\t\t\t\t\treturn Event{Backspace, 0, nil}\n\t\t\t\t}\n\t\t\tcase rune(tcell.KeyCtrlH):\n\t\t\t\tswitch {\n\t\t\t\tcase ctrl:\n\t\t\t\t\treturn keyfn('h')\n\t\t\t\tcase alt:\n\t\t\t\t\treturn Event{AltBackspace, 0, nil}\n\t\t\t\tcase none, shift:\n\t\t\t\t\treturn Event{Backspace, 0, nil}\n\t\t\t\t}\n\t\t\t}\n\t\tcase tcell.KeyCtrlI:\n\t\t\treturn keyfn('i')\n\t\tcase tcell.KeyCtrlJ:\n\t\t\treturn keyfn('j')\n\t\tcase tcell.KeyCtrlK:\n\t\t\treturn keyfn('k')\n\t\tcase tcell.KeyCtrlL:\n\t\t\treturn keyfn('l')\n\t\tcase tcell.KeyCtrlM:\n\t\t\treturn keyfn('m')\n\t\tcase tcell.KeyCtrlN:\n\t\t\treturn keyfn('n')\n\t\tcase tcell.KeyCtrlO:\n\t\t\treturn keyfn('o')\n\t\tcase tcell.KeyCtrlP:\n\t\t\treturn keyfn('p')\n\t\tcase tcell.KeyCtrlQ:\n\t\t\treturn keyfn('q')\n\t\tcase tcell.KeyCtrlR:\n\t\t\treturn keyfn('r')\n\t\tcase tcell.KeyCtrlS:\n\t\t\treturn keyfn('s')\n\t\tcase tcell.KeyCtrlT:\n\t\t\treturn keyfn('t')\n\t\tcase tcell.KeyCtrlU:\n\t\t\treturn keyfn('u')\n\t\tcase tcell.KeyCtrlV:\n\t\t\treturn keyfn('v')\n\t\tcase tcell.KeyCtrlW:\n\t\t\treturn keyfn('w')\n\t\tcase tcell.KeyCtrlX:\n\t\t\treturn keyfn('x')\n\t\tcase tcell.KeyCtrlY:\n\t\t\treturn keyfn('y')\n\t\tcase tcell.KeyCtrlZ:\n\t\t\treturn keyfn('z')\n\t\t// section 2: Ctrl+[ \\]_]\n\t\tcase tcell.KeyCtrlSpace:\n\t\t\treturn Event{CtrlSpace, 0, nil}\n\t\tcase tcell.KeyCtrlBackslash:\n\t\t\treturn Event{CtrlBackSlash, 0, nil}\n\t\tcase tcell.KeyCtrlRightSq:\n\t\t\treturn Event{CtrlRightBracket, 0, nil}\n\t\tcase tcell.KeyCtrlCarat:\n\t\t\treturn Event{CtrlCaret, 0, nil}\n\t\tcase tcell.KeyCtrlUnderscore:\n\t\t\treturn Event{CtrlSlash, 0, nil}\n\t\t// section 3: (Alt)+Backspace2\n\t\tcase tcell.KeyBackspace2:\n\t\t\tif alt {\n\t\t\t\treturn Event{AltBackspace, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Backspace, 0, nil}\n\n\t\t// section 4: (Alt+Shift)+Key(Up|Down|Left|Right)\n\t\tcase tcell.KeyUp:\n\t\t\tif altShift {\n\t\t\t\treturn Event{AltShiftUp, 0, nil}\n\t\t\t}\n\t\t\tif shift {\n\t\t\t\treturn Event{ShiftUp, 0, nil}\n\t\t\t}\n\t\t\tif alt {\n\t\t\t\treturn Event{AltUp, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Up, 0, nil}\n\t\tcase tcell.KeyDown:\n\t\t\tif altShift {\n\t\t\t\treturn Event{AltShiftDown, 0, nil}\n\t\t\t}\n\t\t\tif shift {\n\t\t\t\treturn Event{ShiftDown, 0, nil}\n\t\t\t}\n\t\t\tif alt {\n\t\t\t\treturn Event{AltDown, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Down, 0, nil}\n\t\tcase tcell.KeyLeft:\n\t\t\tif altShift {\n\t\t\t\treturn Event{AltShiftLeft, 0, nil}\n\t\t\t}\n\t\t\tif shift {\n\t\t\t\treturn Event{ShiftLeft, 0, nil}\n\t\t\t}\n\t\t\tif alt {\n\t\t\t\treturn Event{AltLeft, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Left, 0, nil}\n\t\tcase tcell.KeyRight:\n\t\t\tif altShift {\n\t\t\t\treturn Event{AltShiftRight, 0, nil}\n\t\t\t}\n\t\t\tif shift {\n\t\t\t\treturn Event{ShiftRight, 0, nil}\n\t\t\t}\n\t\t\tif alt {\n\t\t\t\treturn Event{AltRight, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Right, 0, nil}\n\n\t\t// section 5: (Insert|Home|Delete|End|PgUp|PgDn|BackTab|F1-F12)\n\t\tcase tcell.KeyInsert:\n\t\t\treturn Event{Insert, 0, nil}\n\t\tcase tcell.KeyHome:\n\t\t\treturn Event{Home, 0, nil}\n\t\tcase tcell.KeyDelete:\n\t\t\tif ctrl {\n\t\t\t\treturn Event{CtrlDelete, 0, nil}\n\t\t\t}\n\t\t\tif shift {\n\t\t\t\treturn Event{ShiftDelete, 0, nil}\n\t\t\t}\n\t\t\treturn Event{Delete, 0, nil}\n\t\tcase tcell.KeyEnd:\n\t\t\treturn Event{End, 0, nil}\n\t\tcase tcell.KeyPgUp:\n\t\t\treturn Event{PageUp, 0, nil}\n\t\tcase tcell.KeyPgDn:\n\t\t\treturn Event{PageDown, 0, nil}\n\t\tcase tcell.KeyBacktab:\n\t\t\treturn Event{ShiftTab, 0, nil}\n\t\tcase tcell.KeyF1:\n\t\t\treturn Event{F1, 0, nil}\n\t\tcase tcell.KeyF2:\n\t\t\treturn Event{F2, 0, nil}\n\t\tcase tcell.KeyF3:\n\t\t\treturn Event{F3, 0, nil}\n\t\tcase tcell.KeyF4:\n\t\t\treturn Event{F4, 0, nil}\n\t\tcase tcell.KeyF5:\n\t\t\treturn Event{F5, 0, nil}\n\t\tcase tcell.KeyF6:\n\t\t\treturn Event{F6, 0, nil}\n\t\tcase tcell.KeyF7:\n\t\t\treturn Event{F7, 0, nil}\n\t\tcase tcell.KeyF8:\n\t\t\treturn Event{F8, 0, nil}\n\t\tcase tcell.KeyF9:\n\t\t\treturn Event{F9, 0, nil}\n\t\tcase tcell.KeyF10:\n\t\t\treturn Event{F10, 0, nil}\n\t\tcase tcell.KeyF11:\n\t\t\treturn Event{F11, 0, nil}\n\t\tcase tcell.KeyF12:\n\t\t\treturn Event{F12, 0, nil}\n\n\t\t// section 6: (Ctrl+Alt)+'rune'\n\t\tcase tcell.KeyRune:\n\t\t\tr := ev.Rune()\n\n\t\t\tswitch {\n\t\t\t// translate native key events to ascii control characters\n\t\t\tcase r == ' ' && ctrl:\n\t\t\t\treturn Event{CtrlSpace, 0, nil}\n\t\t\t// handle AltGr characters\n\t\t\tcase ctrlAlt:\n\t\t\t\treturn Event{Rune, r, nil} // dropping modifiers\n\t\t\t// simple characters (possibly with modifier)\n\t\t\tcase alt:\n\t\t\t\treturn AltKey(r)\n\t\t\tdefault:\n\t\t\t\treturn Event{Rune, r, nil}\n\t\t\t}\n\n\t\t// section 7: Esc\n\t\tcase tcell.KeyEsc:\n\t\t\treturn Event{Esc, 0, nil}\n\t\t}\n\t}\n\n\t// section 8: Invalid\n\treturn Event{Invalid, 0, nil}\n}\n\nfunc (r *FullscreenRenderer) Pause(clear bool) {\n\tif clear {\n\t\t_screen.Fini()\n\t}\n}\n\nfunc (r *FullscreenRenderer) Resume(clear bool, sigcont bool) {\n\tif clear {\n\t\tr.initScreen()\n\t}\n}\n\nfunc (r *FullscreenRenderer) Close() {\n\t_screen.Fini()\n}\n\nfunc (r *FullscreenRenderer) RefreshWindows(windows []Window) {\n\t// TODO\n\tfor _, w := range windows {\n\t\tw.Refresh()\n\t}\n\t_screen.Show()\n}\n\nfunc (r *FullscreenRenderer) NewWindow(top int, left int, width int, height int, preview bool, borderStyle BorderStyle) Window {\n\tnormal := ColNormal\n\tif preview {\n\t\tnormal = ColPreview\n\t}\n\tw := &TcellWindow{\n\t\tcolor:       r.theme.Colored,\n\t\tpreview:     preview,\n\t\ttop:         top,\n\t\tleft:        left,\n\t\twidth:       width,\n\t\theight:      height,\n\t\tnormal:      normal,\n\t\tborderStyle: borderStyle}\n\tw.Erase()\n\treturn w\n}\n\nfunc (w *TcellWindow) Close() {\n\t// TODO\n}\n\nfunc fill(x, y, w, h int, n ColorPair, r rune) {\n\tfor ly := 0; ly <= h; ly++ {\n\t\tfor lx := 0; lx <= w; lx++ {\n\t\t\t_screen.SetContent(x+lx, y+ly, r, nil, n.style())\n\t\t}\n\t}\n}\n\nfunc (w *TcellWindow) Erase() {\n\tfill(w.left-1, w.top, w.width+1, w.height-1, w.normal, ' ')\n\tw.drawBorder(false)\n}\n\nfunc (w *TcellWindow) EraseMaybe() bool {\n\tw.Erase()\n\treturn true\n}\n\nfunc (w *TcellWindow) Enclose(y int, x int) bool {\n\treturn x >= w.left && x < (w.left+w.width) &&\n\t\ty >= w.top && y < (w.top+w.height)\n}\n\nfunc (w *TcellWindow) Move(y int, x int) {\n\tw.lastX = x\n\tw.lastY = y\n\tw.moveCursor = true\n}\n\nfunc (w *TcellWindow) MoveAndClear(y int, x int) {\n\tw.Move(y, x)\n\tfor i := w.lastX; i < w.width; i++ {\n\t\t_screen.SetContent(i+w.left, w.lastY+w.top, rune(' '), nil, w.normal.style())\n\t}\n\tw.lastX = x\n}\n\nfunc (w *TcellWindow) Print(text string) {\n\tw.printString(text, w.normal)\n}\n\nfunc (w *TcellWindow) printString(text string, pair ColorPair) {\n\tlx := 0\n\ta := pair.Attr()\n\n\tstyle := pair.style()\n\tif a&AttrClear == 0 {\n\t\tstyle = style.\n\t\t\tReverse(a&Attr(tcell.AttrReverse) != 0).\n\t\t\tUnderline(a&Attr(tcell.AttrUnderline) != 0).\n\t\t\tStrikeThrough(a&Attr(tcell.AttrStrikeThrough) != 0).\n\t\t\tItalic(a&Attr(tcell.AttrItalic) != 0).\n\t\t\tBlink(a&Attr(tcell.AttrBlink) != 0).\n\t\t\tDim(a&Attr(tcell.AttrDim) != 0)\n\t}\n\n\tgr := uniseg.NewGraphemes(text)\n\tfor gr.Next() {\n\t\tst := style\n\t\trs := gr.Runes()\n\n\t\tif len(rs) == 1 {\n\t\t\tr := rs[0]\n\t\t\tif r == '\\r' {\n\t\t\t\tst = style.Dim(true)\n\t\t\t\trs[0] = '\u240d'\n\t\t\t} else if r == '\\n' {\n\t\t\t\tst = style.Dim(true)\n\t\t\t\trs[0] = '\u240a'\n\t\t\t} else if r < rune(' ') { // ignore control characters\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tvar xPos = w.left + w.lastX + lx\n\t\tvar yPos = w.top + w.lastY\n\t\tif xPos < (w.left+w.width) && yPos < (w.top+w.height) {\n\t\t\t_screen.SetContent(xPos, yPos, rs[0], rs[1:], st)\n\t\t}\n\t\tlx += util.StringWidth(string(rs))\n\t}\n\tw.lastX += lx\n}\n\nfunc (w *TcellWindow) CPrint(pair ColorPair, text string) {\n\tw.printString(text, pair)\n}\n\nfunc (w *TcellWindow) fillString(text string, pair ColorPair) FillReturn {\n\tlx := 0\n\ta := pair.Attr()\n\n\tvar style tcell.Style\n\tif w.color {\n\t\tstyle = pair.style()\n\t} else {\n\t\tstyle = w.normal.style()\n\t}\n\tstyle = style.\n\t\tBlink(a&Attr(tcell.AttrBlink) != 0).\n\t\tBold(a&Attr(tcell.AttrBold) != 0).\n\t\tDim(a&Attr(tcell.AttrDim) != 0).\n\t\tReverse(a&Attr(tcell.AttrReverse) != 0).\n\t\tUnderline(a&Attr(tcell.AttrUnderline) != 0).\n\t\tStrikeThrough(a&Attr(tcell.AttrStrikeThrough) != 0).\n\t\tItalic(a&Attr(tcell.AttrItalic) != 0)\n\n\tgr := uniseg.NewGraphemes(text)\nLoop:\n\tfor gr.Next() {\n\t\tst := style\n\t\trs := gr.Runes()\n\t\tif len(rs) == 1 {\n\t\t\tr := rs[0]\n\t\t\tswitch r {\n\t\t\tcase '\\r':\n\t\t\t\tst = style.Dim(true)\n\t\t\t\trs[0] = '\u240d'\n\t\t\tcase '\\n':\n\t\t\t\tw.lastY++\n\t\t\t\tw.lastX = 0\n\t\t\t\tlx = 0\n\t\t\t\tcontinue Loop\n\t\t\t}\n\t\t}\n\n\t\t// word wrap:\n\t\txPos := w.left + w.lastX + lx\n\t\tif xPos >= (w.left + w.width) {\n\t\t\tw.lastY++\n\t\t\tw.lastX = 0\n\t\t\tlx = 0\n\t\t\txPos = w.left\n\t\t}\n\n\t\tyPos := w.top + w.lastY\n\t\tif yPos >= (w.top + w.height) {\n\t\t\treturn FillSuspend\n\t\t}\n\n\t\t_screen.SetContent(xPos, yPos, rs[0], rs[1:], st)\n\t\tlx += util.StringWidth(string(rs))\n\t}\n\tw.lastX += lx\n\tif w.lastX == w.width {\n\t\tw.lastY++\n\t\tw.lastX = 0\n\t\treturn FillNextLine\n\t}\n\n\treturn FillContinue\n}\n\n...\n", "needle_token_start": 8259, "needle_token_end": 8358, "code_context_ntokens": 15092, "output": ["```go\nfunc asTcellColor(color Color) tcell.Color {\n\tif color == colDefault {\n\t\treturn tcell.ColorDefault\n\t}\n\n\tvalue := uint64(tcell.ColorValid) + uint64(color)\n\tif color.is24() {\n\t\tvalue = value | uint64(tcell.ColorIsRGB)\n\t}\n\treturn tcell.Color(value)\n}\n```"]}
{"repo": "junegunn/fzf", "name": "validateSign", "language": "go", "path": "src/options.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To ensure that a given string is not empty and its display width does not exceed a specified limit.\n2. **Input**: A string to be validated and a string representing the name of the option being validated.\n3. **Output**: Returns an error if the string is empty or if its display width exceeds the limit; otherwise, returns nil indicating no error.\n4. **Procedure**: The function first checks if the input string is empty. If it is, it returns an error stating that the option cannot be empty. Next, it checks the display width of the string using a function that calculates the width considering Unicode characters. If the width is more than 2, it returns an error stating that the display width should be up to 2. If neither condition is violated, it returns nil.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\treturn actions\n}\n\nfunc parseKeymap(keymap map[tui.Event][]*action, str string, exit func(string)) {\n\tmasked := maskActionContents(str)\n\tidx := 0\n\tfor _, pairStr := range strings.Split(masked, \",\") {\n\t\torigPairStr := str[idx : idx+len(pairStr)]\n\t\tidx += len(pairStr) + 1\n\n\t\tpair := strings.SplitN(pairStr, \":\", 2)\n\t\tif len(pair) < 2 {\n\t\t\texit(\"bind action not specified: \" + origPairStr)\n\t\t}\n\t\tvar key tui.Event\n\t\tif len(pair[0]) == 1 && pair[0][0] == escapedColon {\n\t\t\tkey = tui.Key(':')\n\t\t} else if len(pair[0]) == 1 && pair[0][0] == escapedComma {\n\t\t\tkey = tui.Key(',')\n\t\t} else if len(pair[0]) == 1 && pair[0][0] == escapedPlus {\n\t\t\tkey = tui.Key('+')\n\t\t} else {\n\t\t\tkeys := parseKeyChordsImpl(pair[0], \"key name required\", exit)\n\t\t\tkey = firstKey(keys)\n\t\t}\n\t\tputAllowed := key.Type == tui.Rune && unicode.IsGraphic(key.Char)\n\t\tkeymap[key] = parseActionList(pair[1], origPairStr[len(pair[0])+1:], keymap[key], putAllowed, exit)\n\t}\n}\n\nfunc isExecuteAction(str string) actionType {\n\tmasked := maskActionContents(\":\" + str)[1:]\n\tif masked == str {\n\t\t// Not masked\n\t\treturn actIgnore\n\t}\n\n\tprefix := actionNameRegexp.FindString(str)\n\tswitch prefix {\n\tcase \"become\":\n\t\treturn actBecome\n\tcase \"reload\":\n\t\treturn actReload\n\tcase \"reload-sync\":\n\t\treturn actReloadSync\n\tcase \"unbind\":\n\t\treturn actUnbind\n\tcase \"rebind\":\n\t\treturn actRebind\n\tcase \"preview\":\n\t\treturn actPreview\n\tcase \"change-border-label\":\n\t\treturn actChangeBorderLabel\n\tcase \"change-header\":\n\t\treturn actChangeHeader\n\tcase \"change-preview-label\":\n\t\treturn actChangePreviewLabel\n\tcase \"change-preview-window\":\n\t\treturn actChangePreviewWindow\n\tcase \"change-preview\":\n\t\treturn actChangePreview\n\tcase \"change-prompt\":\n\t\treturn actChangePrompt\n\tcase \"change-query\":\n\t\treturn actChangeQuery\n\tcase \"pos\":\n\t\treturn actPosition\n\tcase \"execute\":\n\t\treturn actExecute\n\tcase \"execute-silent\":\n\t\treturn actExecuteSilent\n\tcase \"execute-multi\":\n\t\treturn actExecuteMulti\n\tcase \"put\":\n\t\treturn actPut\n\tcase \"transform\":\n\t\treturn actTransform\n\tcase \"transform-border-label\":\n\t\treturn actTransformBorderLabel\n\tcase \"transform-preview-label\":\n\t\treturn actTransformPreviewLabel\n\tcase \"transform-header\":\n\t\treturn actTransformHeader\n\tcase \"transform-prompt\":\n\t\treturn actTransformPrompt\n\tcase \"transform-query\":\n\t\treturn actTransformQuery\n\t}\n\treturn actIgnore\n}\n\nfunc parseToggleSort(keymap map[tui.Event][]*action, str string) {\n\tkeys := parseKeyChords(str, \"key name required\")\n\tif len(keys) != 1 {\n\t\terrorExit(\"multiple keys specified\")\n\t}\n\tkeymap[firstKey(keys)] = toActions(actToggleSort)\n}\n\nfunc strLines(str string) []string {\n\treturn strings.Split(strings.TrimSuffix(str, \"\\n\"), \"\\n\")\n}\n\nfunc parseSize(str string, maxPercent float64, label string) sizeSpec {\n\tvar val float64\n\tpercent := strings.HasSuffix(str, \"%\")\n\tif percent {\n\t\tval = atof(str[:len(str)-1])\n\t\tif val < 0 {\n\t\t\terrorExit(label + \" must be non-negative\")\n\t\t}\n\t\tif val > maxPercent {\n\t\t\terrorExit(fmt.Sprintf(\"%s too large (max: %d%%)\", label, int(maxPercent)))\n\t\t}\n\t} else {\n\t\tif strings.Contains(str, \".\") {\n\t\t\terrorExit(label + \" (without %) must be a non-negative integer\")\n\t\t}\n\n\t\tval = float64(atoi(str))\n\t\tif val < 0 {\n\t\t\terrorExit(label + \" must be non-negative\")\n\t\t}\n\t}\n\treturn sizeSpec{val, percent}\n}\n\nfunc parseHeight(str string) heightSpec {\n\theightSpec := heightSpec{}\n\tif strings.HasPrefix(str, \"~\") {\n\t\theightSpec.auto = true\n\t\tstr = str[1:]\n\t}\n\tif strings.HasPrefix(str, \"-\") {\n\t\tif heightSpec.auto {\n\t\t\terrorExit(\"negative(-) height is not compatible with adaptive(~) height\")\n\t\t}\n\t\theightSpec.inverse = true\n\t\tstr = str[1:]\n\t}\n\n\tsize := parseSize(str, 100, \"height\")\n\theightSpec.size = size.size\n\theightSpec.percent = size.percent\n\treturn heightSpec\n}\n\nfunc parseLayout(str string) layoutType {\n\tswitch str {\n\tcase \"default\":\n\t\treturn layoutDefault\n\tcase \"reverse\":\n\t\treturn layoutReverse\n\tcase \"reverse-list\":\n\t\treturn layoutReverseList\n\tdefault:\n\t\terrorExit(\"invalid layout (expected: default / reverse / reverse-list)\")\n\t}\n\treturn layoutDefault\n}\n\nfunc parseInfoStyle(str string) (infoStyle, string) {\n\tswitch str {\n\tcase \"default\":\n\t\treturn infoDefault, \"\"\n\tcase \"right\":\n\t\treturn infoRight, \"\"\n\tcase \"inline\":\n\t\treturn infoInline, defaultInfoPrefix\n\tcase \"inline-right\":\n\t\treturn infoInlineRight, \"\"\n\tcase \"hidden\":\n\t\treturn infoHidden, \"\"\n\tdefault:\n\t\ttype infoSpec struct {\n\t\t\tname  string\n\t\t\tstyle infoStyle\n\t\t}\n\t\tfor _, spec := range []infoSpec{\n\t\t\t{\"inline\", infoInline},\n\t\t\t{\"inline-right\", infoInlineRight}} {\n\t\t\tif strings.HasPrefix(str, spec.name+\":\") {\n\t\t\t\treturn spec.style, strings.ReplaceAll(str[len(spec.name)+1:], \"\\n\", \" \")\n\t\t\t}\n\t\t}\n\t\terrorExit(\"invalid info style (expected: default|right|hidden|inline[-right][:PREFIX])\")\n\t}\n\treturn infoDefault, \"\"\n}\n\nfunc parsePreviewWindow(opts *previewOpts, input string) {\n\tparsePreviewWindowImpl(opts, input, errorExit)\n}\n\nfunc parsePreviewWindowImpl(opts *previewOpts, input string, exit func(string)) {\n\ttokenRegex := regexp.MustCompile(`[:,]*(<([1-9][0-9]*)\\(([^)<]+)\\)|[^,:]+)`)\n\tsizeRegex := regexp.MustCompile(\"^[0-9]+%?$\")\n\toffsetRegex := regexp.MustCompile(`^(\\+{-?[0-9]+})?([+-][0-9]+)*(-?/[1-9][0-9]*)?$`)\n\theaderRegex := regexp.MustCompile(\"^~(0|[1-9][0-9]*)$\")\n\ttokens := tokenRegex.FindAllStringSubmatch(input, -1)\n\tvar alternative string\n\tfor _, match := range tokens {\n\t\tif len(match[2]) > 0 {\n\t\t\topts.threshold = atoi(match[2])\n\t\t\talternative = match[3]\n\t\t\tcontinue\n\t\t}\n\t\ttoken := match[1]\n\t\tswitch token {\n\t\tcase \"\":\n\t\tcase \"default\":\n\t\t\t*opts = defaultPreviewOpts(opts.command)\n\t\tcase \"hidden\":\n\t\t\topts.hidden = true\n\t\tcase \"nohidden\":\n\t\t\topts.hidden = false\n\t\tcase \"wrap\":\n\t\t\topts.wrap = true\n\t\tcase \"nowrap\":\n\t\t\topts.wrap = false\n\t\tcase \"cycle\":\n\t\t\topts.cycle = true\n\t\tcase \"nocycle\":\n\t\t\topts.cycle = false\n\t\tcase \"up\", \"top\":\n\t\t\topts.position = posUp\n\t\tcase \"down\", \"bottom\":\n\t\t\topts.position = posDown\n\t\tcase \"left\":\n\t\t\topts.position = posLeft\n\t\tcase \"right\":\n\t\t\topts.position = posRight\n\t\tcase \"rounded\", \"border\", \"border-rounded\":\n\t\t\topts.border = tui.BorderRounded\n\t\tcase \"sharp\", \"border-sharp\":\n\t\t\topts.border = tui.BorderSharp\n\t\tcase \"border-bold\":\n\t\t\topts.border = tui.BorderBold\n\t\tcase \"border-block\":\n\t\t\topts.border = tui.BorderBlock\n\t\tcase \"border-thinblock\":\n\t\t\topts.border = tui.BorderThinBlock\n\t\tcase \"border-double\":\n\t\t\topts.border = tui.BorderDouble\n\t\tcase \"noborder\", \"border-none\":\n\t\t\topts.border = tui.BorderNone\n\t\tcase \"border-horizontal\":\n\t\t\topts.border = tui.BorderHorizontal\n\t\tcase \"border-vertical\":\n\t\t\topts.border = tui.BorderVertical\n\t\tcase \"border-up\", \"border-top\":\n\t\t\topts.border = tui.BorderTop\n\t\tcase \"border-down\", \"border-bottom\":\n\t\t\topts.border = tui.BorderBottom\n\t\tcase \"border-left\":\n\t\t\topts.border = tui.BorderLeft\n\t\tcase \"border-right\":\n\t\t\topts.border = tui.BorderRight\n\t\tcase \"follow\":\n\t\t\topts.follow = true\n\t\tcase \"nofollow\":\n\t\t\topts.follow = false\n\t\tdefault:\n\t\t\tif headerRegex.MatchString(token) {\n\t\t\t\topts.headerLines = atoi(token[1:])\n\t\t\t} else if sizeRegex.MatchString(token) {\n\t\t\t\topts.size = parseSize(token, 99, \"window size\")\n\t\t\t} else if offsetRegex.MatchString(token) {\n\t\t\t\topts.scroll = token\n\t\t\t} else {\n\t\t\t\texit(\"invalid preview window option: \" + token)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\tif len(alternative) > 0 {\n\t\talternativeOpts := *opts\n\t\topts.alternative = &alternativeOpts\n\t\topts.alternative.hidden = false\n\t\topts.alternative.alternative = nil\n\t\tparsePreviewWindowImpl(opts.alternative, alternative, exit)\n\t}\n}\n\nfunc parseMargin(opt string, margin string) [4]sizeSpec {\n\tmargins := strings.Split(margin, \",\")\n\tchecked := func(str string) sizeSpec {\n\t\treturn parseSize(str, 49, opt)\n\t}\n\tswitch len(margins) {\n\tcase 1:\n\t\tm := checked(margins[0])\n\t\treturn [4]sizeSpec{m, m, m, m}\n\tcase 2:\n\t\ttb := checked(margins[0])\n\t\trl := checked(margins[1])\n\t\treturn [4]sizeSpec{tb, rl, tb, rl}\n\tcase 3:\n\t\tt := checked(margins[0])\n\t\trl := checked(margins[1])\n\t\tb := checked(margins[2])\n\t\treturn [4]sizeSpec{t, rl, b, rl}\n\tcase 4:\n\t\treturn [4]sizeSpec{\n\t\t\tchecked(margins[0]), checked(margins[1]),\n\t\t\tchecked(margins[2]), checked(margins[3])}\n\tdefault:\n\t\terrorExit(\"invalid \" + opt + \": \" + margin)\n\t}\n\treturn defaultMargin()\n}\n\nfunc parseOptions(opts *Options, allArgs []string) {\n\tvar historyMax int\n\tif opts.History == nil {\n\t\thistoryMax = defaultHistoryMax\n\t} else {\n\t\thistoryMax = opts.History.maxSize\n\t}\n\tsetHistory := func(path string) {\n\t\th, e := NewHistory(path, historyMax)\n\t\tif e != nil {\n\t\t\terrorExit(e.Error())\n\t\t}\n\t\topts.History = h\n\t}\n\tsetHistoryMax := func(max int) {\n\t\thistoryMax = max\n\t\tif historyMax < 1 {\n\t\t\terrorExit(\"history max must be a positive integer\")\n\t\t}\n\t\tif opts.History != nil {\n\t\t\topts.History.maxSize = historyMax\n\t\t}\n\t}\n\tvalidateJumpLabels := false\n\tfor i := 0; i < len(allArgs); i++ {\n\t\targ := allArgs[i]\n\t\tswitch arg {\n\t\tcase \"--bash\":\n\t\t\topts.Bash = true\n\t\t\tif opts.Zsh || opts.Fish {\n\t\t\t\terrorExit(\"cannot specify --bash with --zsh or --fish\")\n\t\t\t}\n\t\tcase \"--zsh\":\n\t\t\topts.Zsh = true\n\t\t\tif opts.Bash || opts.Fish {\n\t\t\t\terrorExit(\"cannot specify --zsh with --bash or --fish\")\n\t\t\t}\n\t\tcase \"--fish\":\n\t\t\topts.Fish = true\n\t\t\tif opts.Bash || opts.Zsh {\n\t\t\t\terrorExit(\"cannot specify --fish with --bash or --zsh\")\n\t\t\t}\n\t\tcase \"-h\", \"--help\":\n\t\t\thelp(exitOk)\n\t\tcase \"-x\", \"--extended\":\n\t\t\topts.Extended = true\n\t\tcase \"-e\", \"--exact\":\n\t\t\topts.Fuzzy = false\n\t\tcase \"--extended-exact\":\n\t\t\t// Note that we now don't have --no-extended-exact\n\t\t\topts.Fuzzy = false\n\t\t\topts.Extended = true\n\t\tcase \"+x\", \"--no-extended\":\n\t\t\topts.Extended = false\n\t\tcase \"+e\", \"--no-exact\":\n\t\t\topts.Fuzzy = true\n\t\tcase \"-q\", \"--query\":\n\t\t\topts.Query = nextString(allArgs, &i, \"query string required\")\n\t\tcase \"-f\", \"--filter\":\n\t\t\tfilter := nextString(allArgs, &i, \"query string required\")\n\t\t\topts.Filter = &filter\n\t\tcase \"--literal\":\n\t\t\topts.Normalize = false\n\t\tcase \"--no-literal\":\n\t\t\topts.Normalize = true\n\t\tcase \"--algo\":\n\t\t\topts.FuzzyAlgo = parseAlgo(nextString(allArgs, &i, \"algorithm required (v1|v2)\"))\n\t\tcase \"--scheme\":\n\t\t\topts.Scheme = strings.ToLower(nextString(allArgs, &i, \"scoring scheme required (default|path|history)\"))\n\t\tcase \"--expect\":\n\t\t\tfor k, v := range parseKeyChords(nextString(allArgs, &i, \"key names required\"), \"key names required\") {\n\t\t\t\topts.Expect[k] = v\n\t\t\t}\n\t\tcase \"--no-expect\":\n\t\t\topts.Expect = make(map[tui.Event]string)\n\t\tcase \"--enabled\", \"--no-phony\":\n\t\t\topts.Phony = false\n\t\tcase \"--disabled\", \"--phony\":\n\t\t\topts.Phony = true\n\t\tcase \"--tiebreak\":\n\t\t\topts.Criteria = parseTiebreak(nextString(allArgs, &i, \"sort criterion required\"))\n\t\tcase \"--bind\":\n\t\t\tparseKeymap(opts.Keymap, nextString(allArgs, &i, \"bind expression required\"), errorExit)\n\t\tcase \"--color\":\n\t\t\t_, spec := optionalNextString(allArgs, &i)\n\t\t\tif len(spec) == 0 {\n\t\t\t\topts.Theme = tui.EmptyTheme()\n\t\t\t} else {\n\t\t\t\topts.Theme = parseTheme(opts.Theme, spec)\n\t\t\t}\n\t\tcase \"--toggle-sort\":\n\t\t\tparseToggleSort(opts.Keymap, nextString(allArgs, &i, \"key name required\"))\n\t\tcase \"-d\", \"--delimiter\":\n\t\t\topts.Delimiter = delimiterRegexp(nextString(allArgs, &i, \"delimiter required\"))\n\t\tcase \"-n\", \"--nth\":\n\t\t\topts.Nth = splitNth(nextString(allArgs, &i, \"nth expression required\"))\n\t\tcase \"--with-nth\":\n\t\t\topts.WithNth = splitNth(nextString(allArgs, &i, \"nth expression required\"))\n\t\tcase \"-s\", \"--sort\":\n\t\t\topts.Sort = optionalNumeric(allArgs, &i, 1)\n\t\tcase \"+s\", \"--no-sort\":\n\t\t\topts.Sort = 0\n\t\tcase \"--track\":\n\t\t\topts.Track = trackEnabled\n\t\tcase \"--no-track\":\n\t\t\topts.Track = trackDisabled\n\t\tcase \"--tac\":\n\t\t\topts.Tac = true\n\t\tcase \"--no-tac\":\n\t\t\topts.Tac = false\n\t\tcase \"-i\":\n\t\t\topts.Case = CaseIgnore\n\t\tcase \"+i\":\n\t\t\topts.Case = CaseRespect\n\t\tcase \"-m\", \"--multi\":\n\t\t\topts.Multi = optionalNumeric(allArgs, &i, maxMulti)\n\t\tcase \"+m\", \"--no-multi\":\n\t\t\topts.Multi = 0\n\t\tcase \"--ansi\":\n\t\t\topts.Ansi = true\n\t\tcase \"--no-ansi\":\n\t\t\topts.Ansi = false\n\t\tcase \"--no-mouse\":\n\t\t\topts.Mouse = false\n\t\tcase \"+c\", \"--no-color\":\n\t\t\topts.Theme = tui.NoColorTheme()\n\t\tcase \"+2\", \"--no-256\":\n\t\t\topts.Theme = tui.Default16\n\t\tcase \"--black\":\n\t\t\topts.Black = true\n\t\tcase \"--no-black\":\n\t\t\topts.Black = false\n\t\tcase \"--bold\":\n\t\t\topts.Bold = true\n\t\tcase \"--no-bold\":\n\t\t\topts.Bold = false\n\t\tcase \"--layout\":\n\t\t\topts.Layout = parseLayout(\n\t\t\t\tnextString(allArgs, &i, \"layout required (default / reverse / reverse-list)\"))\n\t\tcase \"--reverse\":\n\t\t\topts.Layout = layoutReverse\n\t\tcase \"--no-reverse\":\n\t\t\topts.Layout = layoutDefault\n\t\tcase \"--cycle\":\n\t\t\topts.Cycle = true\n\t\tcase \"--no-cycle\":\n\t\t\topts.Cycle = false\n\t\tcase \"--keep-right\":\n\t\t\topts.KeepRight = true\n\t\tcase \"--no-keep-right\":\n\t\t\topts.KeepRight = false\n\t\tcase \"--hscroll\":\n\t\t\topts.Hscroll = true\n\t\tcase \"--no-hscroll\":\n\t\t\topts.Hscroll = false\n\t\tcase \"--hscroll-off\":\n\t\t\topts.HscrollOff = nextInt(allArgs, &i, \"hscroll offset required\")\n\t\tcase \"--scroll-off\":\n\t\t\topts.ScrollOff = nextInt(allArgs, &i, \"scroll offset required\")\n\t\tcase \"--filepath-word\":\n\t\t\topts.FileWord = true\n\t\tcase \"--no-filepath-word\":\n\t\t\topts.FileWord = false\n\t\tcase \"--info\":\n\t\t\topts.InfoStyle, opts.InfoPrefix = parseInfoStyle(\n\t\t\t\tnextString(allArgs, &i, \"info style required\"))\n\t\tcase \"--no-info\":\n\t\t\topts.InfoStyle = infoHidden\n\t\tcase \"--inline-info\":\n\t\t\topts.InfoStyle = infoInline\n\t\t\topts.InfoPrefix = defaultInfoPrefix\n\t\tcase \"--no-inline-info\":\n\t\t\topts.InfoStyle = infoDefault\n\t\tcase \"--separator\":\n\t\t\tseparator := nextString(allArgs, &i, \"separator character required\")\n\t\t\topts.Separator = &separator\n\t\tcase \"--no-separator\":\n\t\t\tnosep := \"\"\n\t\t\topts.Separator = &nosep\n\t\tcase \"--scrollbar\":\n\t\t\tgiven, bar := optionalNextString(allArgs, &i)\n\t\t\tif given {\n\t\t\t\topts.Scrollbar = &bar\n\t\t\t} else {\n\t\t\t\topts.Scrollbar = nil\n\t\t\t}\n\t\tcase \"--no-scrollbar\":\n\t\t\tnoBar := \"\"\n\t\t\topts.Scrollbar = &noBar\n\t\tcase \"--jump-labels\":\n\t\t\topts.JumpLabels = nextString(allArgs, &i, \"label characters required\")\n\t\t\tvalidateJumpLabels = true\n\t\tcase \"-1\", \"--select-1\":\n\t\t\topts.Select1 = true\n\t\tcase \"+1\", \"--no-select-1\":\n\t\t\topts.Select1 = false\n\t\tcase \"-0\", \"--exit-0\":\n\t\t\topts.Exit0 = true\n\t\tcase \"+0\", \"--no-exit-0\":\n\t\t\topts.Exit0 = false\n\t\tcase \"--read0\":\n\t\t\topts.ReadZero = true\n\t\tcase \"--no-read0\":\n\t\t\topts.ReadZero = false\n\t\tcase \"--print0\":\n\t\t\topts.Printer = func(str string) { fmt.Print(str, \"\\x00\") }\n\t\t\topts.PrintSep = \"\\x00\"\n\t\tcase \"--no-print0\":\n\t\t\topts.Printer = func(str string) { fmt.Println(str) }\n\t\t\topts.PrintSep = \"\\n\"\n\t\tcase \"--print-query\":\n\t\t\topts.PrintQuery = true\n\t\tcase \"--no-print-query\":\n\t\t\topts.PrintQuery = false\n\t\tcase \"--prompt\":\n\t\t\topts.Prompt = nextString(allArgs, &i, \"prompt string required\")\n\t\tcase \"--pointer\":\n\t\t\topts.Pointer = firstLine(nextString(allArgs, &i, \"pointer sign string required\"))\n\t\tcase \"--marker\":\n\t\t\topts.Marker = firstLine(nextString(allArgs, &i, \"selected sign string required\"))\n\t\tcase \"--sync\":\n\t\t\topts.Sync = true\n\t\tcase \"--no-sync\":\n\t\t\topts.Sync = false\n\t\tcase \"--async\":\n\t\t\topts.Sync = false\n\t\tcase \"--no-history\":\n\t\t\topts.History = nil\n\t\tcase \"--history\":\n\t\t\tsetHistory(nextString(allArgs, &i, \"history file path required\"))\n\t\tcase \"--history-size\":\n\t\t\tsetHistoryMax(nextInt(allArgs, &i, \"history max size required\"))\n\t\tcase \"--no-header\":\n\t\t\topts.Header = []string{}\n\t\tcase \"--no-header-lines\":\n\t\t\topts.HeaderLines = 0\n\t\tcase \"--header\":\n\t\t\topts.Header = strLines(nextString(allArgs, &i, \"header string required\"))\n\t\tcase \"--header-lines\":\n\t\t\topts.HeaderLines = atoi(\n\t\t\t\tnextString(allArgs, &i, \"number of header lines required\"))\n\t\tcase \"--header-first\":\n\t\t\topts.HeaderFirst = true\n\t\tcase \"--no-header-first\":\n\t\t\topts.HeaderFirst = false\n\t\tcase \"--ellipsis\":\n\t\t\topts.Ellipsis = nextString(allArgs, &i, \"ellipsis string required\")\n\t\tcase \"--preview\":\n\t\t\topts.Preview.command = nextString(allArgs, &i, \"preview command required\")\n\t\tcase \"--no-preview\":\n\t\t\topts.Preview.command = \"\"\n\t\tcase \"--preview-window\":\n\t\t\tparsePreviewWindow(&opts.Preview,\n\t\t\t\tnextString(allArgs, &i, \"preview window layout required: [up|down|left|right][,SIZE[%]][,border-BORDER_OPT][,wrap][,cycle][,hidden][,+SCROLL[OFFSETS][/DENOM]][,~HEADER_LINES][,default]\"))\n\t\tcase \"--height\":\n\t\t\topts.Height = parseHeight(nextString(allArgs, &i, \"height required: [~]HEIGHT[%]\"))\n\t\tcase \"--min-height\":\n\t\t\topts.MinHeight = nextInt(allArgs, &i, \"height required: HEIGHT\")\n\t\tcase \"--no-height\":\n\t\t\topts.Height = heightSpec{}\n\t\tcase \"--no-margin\":\n\t\t\topts.Margin = defaultMargin()\n\t\tcase \"--no-padding\":\n\t\t\topts.Padding = defaultMargin()\n\t\tcase \"--no-border\":\n\t\t\topts.BorderShape = tui.BorderNone\n\t\tcase \"--border\":\n\t\t\thasArg, arg := optionalNextString(allArgs, &i)\n\t\t\topts.BorderShape = parseBorder(arg, !hasArg)\n\t\tcase \"--no-border-label\":\n\t\t\topts.BorderLabel.label = \"\"\n\t\tcase \"--border-label\":\n\t\t\topts.BorderLabel.label = nextString(allArgs, &i, \"label required\")\n\t\tcase \"--border-label-pos\":\n\t\t\tpos := nextString(allArgs, &i, \"label position required (positive or negative integer or 'center')\")\n\t\t\tparseLabelPosition(&opts.BorderLabel, pos)\n\t\tcase \"--no-preview-label\":\n\t\t\topts.PreviewLabel.label = \"\"\n\t\tcase \"--preview-label\":\n\t\t\topts.PreviewLabel.label = nextString(allArgs, &i, \"preview label required\")\n\t\tcase \"--preview-label-pos\":\n\t\t\tpos := nextString(allArgs, &i, \"preview label position required (positive or negative integer or 'center')\")\n\t\t\tparseLabelPosition(&opts.PreviewLabel, pos)\n\t\tcase \"--no-unicode\":\n\t\t\topts.Unicode = false\n\t\tcase \"--unicode\":\n\t\t\topts.Unicode = true\n\t\tcase \"--ambidouble\":\n\t\t\topts.Ambidouble = true\n\t\tcase \"--no-ambidouble\":\n\t\t\topts.Ambidouble = false\n\t\tcase \"--margin\":\n\t\t\topts.Margin = parseMargin(\n\t\t\t\t\"margin\",\n\t\t\t\tnextString(allArgs, &i, \"margin required (TRBL / TB,RL / T,RL,B / T,R,B,L)\"))\n\t\tcase \"--padding\":\n\t\t\topts.Padding = parseMargin(\n\t\t\t\t\"padding\",\n\t\t\t\tnextString(allArgs, &i, \"padding required (TRBL / TB,RL / T,RL,B / T,R,B,L)\"))\n\t\tcase \"--tabstop\":\n\t\t\topts.Tabstop = nextInt(allArgs, &i, \"tab stop required\")\n\t\tcase \"--listen\", \"--listen-unsafe\":\n\t\t\tgiven, str := optionalNextString(allArgs, &i)\n\t\t\taddr := defaultListenAddr\n\t\t\tif given {\n\t\t\t\tvar err error\n\t\t\t\taddr, err = parseListenAddress(str)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrorExit(err.Error())\n\t\t\t\t}\n\t\t\t}\n\t\t\topts.ListenAddr = &addr\n\t\t\topts.Unsafe = arg == \"--listen-unsafe\"\n\t\tcase \"--no-listen\", \"--no-listen-unsafe\":\n\t\t\topts.ListenAddr = nil\n\t\t\topts.Unsafe = false\n\t\tcase \"--clear\":\n\t\t\topts.ClearOnExit = true\n\t\tcase \"--no-clear\":\n\t\t\topts.ClearOnExit = false\n\t\tcase \"--walker\":\n\t\t\topts.WalkerOpts = parseWalkerOpts(nextString(allArgs, &i, \"walker options required [file][,dir][,follow][,hidden]\"))\n\t\tcase \"--walker-root\":\n\t\t\topts.WalkerRoot = nextString(allArgs, &i, \"directory required\")\n\t\tcase \"--walker-skip\":\n\t\t\topts.WalkerSkip = filterNonEmpty(strings.Split(nextString(allArgs, &i, \"directory names to ignore required\"), \",\"))\n\t\tcase \"--version\":\n\t\t\topts.Version = true\n\t\tcase \"--profile-cpu\":\n\t\t\topts.CPUProfile = nextString(allArgs, &i, \"file path required: cpu\")\n\t\tcase \"--profile-mem\":\n\t\t\topts.MEMProfile = nextString(allArgs, &i, \"file path required: mem\")\n\t\tcase \"--profile-block\":\n\t\t\topts.BlockProfile = nextString(allArgs, &i, \"file path required: block\")\n\t\tcase \"--profile-mutex\":\n\t\t\topts.MutexProfile = nextString(allArgs, &i, \"file path required: mutex\")\n\t\tcase \"--\":\n\t\t\t// Ignored\n\t\tdefault:\n\t\t\tif match, value := optString(arg, \"--algo=\"); match {\n\t\t\t\topts.FuzzyAlgo = parseAlgo(value)\n\t\t\t} else if match, value := optString(arg, \"--scheme=\"); match {\n\t\t\t\topts.Scheme = strings.ToLower(value)\n\t\t\t} else if match, value := optString(arg, \"-q\", \"--query=\"); match {\n\t\t\t\topts.Query = value\n\t\t\t} else if match, value := optString(arg, \"-f\", \"--filter=\"); match {\n\t\t\t\topts.Filter = &value\n\t\t\t} else if match, value := optString(arg, \"-d\", \"--delimiter=\"); match {\n\t\t\t\topts.Delimiter = delimiterRegexp(value)\n\t\t\t} else if match, value := optString(arg, \"--border=\"); match {\n\t\t\t\topts.BorderShape = parseBorder(value, false)\n\t\t\t} else if match, value := optString(arg, \"--border-label=\"); match {\n\t\t\t\topts.BorderLabel.label = value\n\t\t\t} else if match, value := optString(arg, \"--border-label-pos=\"); match {\n\t\t\t\tparseLabelPosition(&opts.BorderLabel, value)\n\t\t\t} else if match, value := optString(arg, \"--preview-label=\"); match {\n\t\t\t\topts.PreviewLabel.label = value\n\t\t\t} else if match, value := optString(arg, \"--preview-label-pos=\"); match {\n\t\t\t\tparseLabelPosition(&opts.PreviewLabel, value)\n\t\t\t} else if match, value := optString(arg, \"--prompt=\"); match {\n\t\t\t\topts.Prompt = value\n\t\t\t} else if match, value := optString(arg, \"--pointer=\"); match {\n\t\t\t\topts.Pointer = firstLine(value)\n\t\t\t} else if match, value := optString(arg, \"--marker=\"); match {\n\t\t\t\topts.Marker = firstLine(value)\n\t\t\t} else if match, value := optString(arg, \"-n\", \"--nth=\"); match {\n\t\t\t\topts.Nth = splitNth(value)\n\t\t\t} else if match, value := optString(arg, \"--with-nth=\"); match {\n\t\t\t\topts.WithNth = splitNth(value)\n\t\t\t} else if match, _ := optString(arg, \"-s\", \"--sort=\"); match {\n\t\t\t\topts.Sort = 1 // Don't care\n\t\t\t} else if match, value := optString(arg, \"-m\", \"--multi=\"); match {\n\t\t\t\topts.Multi = atoi(value)\n\t\t\t} else if match, value := optString(arg, \"--height=\"); match {\n\t\t\t\topts.Height = parseHeight(value)\n\t\t\t} else if match, value := optString(arg, \"--min-height=\"); match {\n\t\t\t\topts.MinHeight = atoi(value)\n\t\t\t} else if match, value := optString(arg, \"--layout=\"); match {\n\t\t\t\topts.Layout = parseLayout(value)\n\t\t\t} else if match, value := optString(arg, \"--info=\"); match {\n\t\t\t\topts.InfoStyle, opts.InfoPrefix = parseInfoStyle(value)\n\t\t\t} else if match, value := optString(arg, \"--separator=\"); match {\n\t\t\t\topts.Separator = &value\n\t\t\t} else if match, value := optString(arg, \"--scrollbar=\"); match {\n\t\t\t\topts.Scrollbar = &value\n\t\t\t} else if match, value := optString(arg, \"--toggle-sort=\"); match {\n\t\t\t\tparseToggleSort(opts.Keymap, value)\n\t\t\t} else if match, value := optString(arg, \"--expect=\"); match {\n\t\t\t\tfor k, v := range parseKeyChords(value, \"key names required\") {\n\t\t\t\t\topts.Expect[k] = v\n\t\t\t\t}\n\t\t\t} else if match, value := optString(arg, \"--tiebreak=\"); match {\n\t\t\t\topts.Criteria = parseTiebreak(value)\n\t\t\t} else if match, value := optString(arg, \"--color=\"); match {\n\t\t\t\topts.Theme = parseTheme(opts.Theme, value)\n\t\t\t} else if match, value := optString(arg, \"--bind=\"); match {\n\t\t\t\tparseKeymap(opts.Keymap, value, errorExit)\n\t\t\t} else if match, value := optString(arg, \"--history=\"); match {\n\t\t\t\tsetHistory(value)\n\t\t\t} else if match, value := optString(arg, \"--history-size=\"); match {\n\t\t\t\tsetHistoryMax(atoi(value))\n\t\t\t} else if match, value := optString(arg, \"--header=\"); match {\n\t\t\t\topts.Header = strLines(value)\n\t\t\t} else if match, value := optString(arg, \"--header-lines=\"); match {\n\t\t\t\topts.HeaderLines = atoi(value)\n\t\t\t} else if match, value := optString(arg, \"--ellipsis=\"); match {\n\t\t\t\topts.Ellipsis = value\n\t\t\t} else if match, value := optString(arg, \"--preview=\"); match {\n\t\t\t\topts.Preview.command = value\n\t\t\t} else if match, value := optString(arg, \"--preview-window=\"); match {\n\t\t\t\tparsePreviewWindow(&opts.Preview, value)\n\t\t\t} else if match, value := optString(arg, \"--margin=\"); match {\n\t\t\t\topts.Margin = parseMargin(\"margin\", value)\n\t\t\t} else if match, value := optString(arg, \"--padding=\"); match {\n\t\t\t\topts.Padding = parseMargin(\"padding\", value)\n\t\t\t} else if match, value := optString(arg, \"--tabstop=\"); match {\n\t\t\t\topts.Tabstop = atoi(value)\n\t\t\t} else if match, value := optString(arg, \"--listen=\"); match {\n\t\t\t\taddr, err := parseListenAddress(value)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrorExit(err.Error())\n\t\t\t\t}\n\t\t\t\topts.ListenAddr = &addr\n\t\t\t\topts.Unsafe = false\n\t\t\t} else if match, value := optString(arg, \"--listen-unsafe=\"); match {\n\t\t\t\taddr, err := parseListenAddress(value)\n\t\t\t\tif err != nil {\n\t\t\t\t\terrorExit(err.Error())\n\t\t\t\t}\n\t\t\t\topts.ListenAddr = &addr\n\t\t\t\topts.Unsafe = true\n\t\t\t} else if match, value := optString(arg, \"--walker=\"); match {\n\t\t\t\topts.WalkerOpts = parseWalkerOpts(value)\n\t\t\t} else if match, value := optString(arg, \"--walker-root=\"); match {\n\t\t\t\topts.WalkerRoot = value\n\t\t\t} else if match, value := optString(arg, \"--walker-skip=\"); match {\n\t\t\t\topts.WalkerSkip = filterNonEmpty(strings.Split(value, \",\"))\n\t\t\t} else if match, value := optString(arg, \"--hscroll-off=\"); match {\n\t\t\t\topts.HscrollOff = atoi(value)\n\t\t\t} else if match, value := optString(arg, \"--scroll-off=\"); match {\n\t\t\t\topts.ScrollOff = atoi(value)\n\t\t\t} else if match, value := optString(arg, \"--jump-labels=\"); match {\n\t\t\t\topts.JumpLabels = value\n\t\t\t\tvalidateJumpLabels = true\n\t\t\t} else {\n\t\t\t\terrorExit(\"unknown option: \" + arg)\n\t\t\t}\n\t\t}\n\t}\n\n\tif opts.HeaderLines < 0 {\n\t\terrorExit(\"header lines must be a non-negative integer\")\n\t}\n\n\tif opts.HscrollOff < 0 {\n\t\terrorExit(\"hscroll offset must be a non-negative integer\")\n\t}\n\n\tif opts.ScrollOff < 0 {\n\t\terrorExit(\"scroll offset must be a non-negative integer\")\n\t}\n\n\tif opts.Tabstop < 1 {\n\t\terrorExit(\"tab stop must be a positive integer\")\n\t}\n\n\tif len(opts.JumpLabels) == 0 {\n\t\terrorExit(\"empty jump labels\")\n\t}\n\n\tif validateJumpLabels {\n\t\tfor _, r := range opts.JumpLabels {\n\t\t\tif r < 32 || r > 126 {\n\t\t\t\terrorExit(\"non-ascii jump labels are not allowed\")\n\t\t\t}\n\t\t}\n\t}\n}\n\n\nfunc validateSign(sign string, signOptName string) error {\n\tif sign == \"\" {\n\t\treturn fmt.Errorf(\"%v cannot be empty\", signOptName)\n\t}\n\tif uniseg.StringWidth(sign) > 2 {\n\t\treturn fmt.Errorf(\"%v display width should be up to 2\", signOptName)\n\t}\n\treturn nil\n}\n\nfunc postProcessOptions(opts *Options) {\n\tif opts.Ambidouble {\n\t\tuniseg.EastAsianAmbiguousWidth = 2\n\t}\n\n\tif err := validateSign(opts.Pointer, \"pointer\"); err != nil {\n\t\terrorExit(err.Error())\n\t}\n\n\tif err := validateSign(opts.Marker, \"marker\"); err != nil {\n\t\terrorExit(err.Error())\n\t}\n\n\tif !opts.Version && !tui.IsLightRendererSupported() && opts.Height.size > 0 {\n\t\terrorExit(\"--height option is currently not supported on this platform\")\n\t}\n\n\tif opts.Scrollbar != nil {\n\t\trunes := []rune(*opts.Scrollbar)\n\t\tif len(runes) > 2 {\n\t\t\terrorExit(\"--scrollbar should be given one or two characters\")\n\t\t}\n\t\tfor _, r := range runes {\n\t\t\tif uniseg.StringWidth(string(r)) != 1 {\n\t\t\t\terrorExit(\"scrollbar display width should be 1\")\n\t\t\t}\n\t\t}\n\t}\n\n\t// Default actions for CTRL-N / CTRL-P when --history is set\n\tif opts.History != nil {\n\t\tif _, prs := opts.Keymap[tui.CtrlP.AsEvent()]; !prs {\n\t\t\topts.Keymap[tui.CtrlP.AsEvent()] = toActions(actPrevHistory)\n\t\t}\n\t\tif _, prs := opts.Keymap[tui.CtrlN.AsEvent()]; !prs {\n\t\t\topts.Keymap[tui.CtrlN.AsEvent()] = toActions(actNextHistory)\n\t\t}\n\t}\n\n\t// Extend the default key map\n\tkeymap := defaultKeymap()\n\tfor key, actions := range opts.Keymap {\n\t\treordered := []*action{}\n\t\tfor _, act := range actions {\n\t\t\tswitch act.t {\n\t\t\tcase actToggleSort:\n\t\t\t\t// To display \"+S\"/\"-S\" on info line\n\t\t\t\topts.ToggleSort = true\n\t\t\tcase actTogglePreview, actShowPreview, actHidePreview, actChangePreviewWindow:\n\t\t\t\treordered = append(reordered, act)\n\t\t\t}\n\t\t}\n\n\t\t// Re-organize actions so that we put actions that change the preview window first in the list.\n\t\t//  *  change-preview-window(up,+10)+preview(sleep 3; cat {})+change-preview-window(up,+20)\n\t\t//  -> change-preview-window(up,+10)+change-preview-window(up,+20)+preview(sleep 3; cat {})\n\t\tif len(reordered) > 0 {\n\t\t\tfor _, act := range actions {\n\t\t\t\tswitch act.t {\n\t\t\t\tcase actTogglePreview, actShowPreview, actHidePreview, actChangePreviewWindow:\n\t\t\t\tdefault:\n\t\t\t\t\treordered = append(reordered, act)\n\t\t\t\t}\n\t\t\t}\n\t\t\tactions = reordered\n\t\t}\n\t\tkeymap[key] = actions\n\t}\n\topts.Keymap = keymap\n\n\t// If 'double-click' is left unbound, bind it to the action bound to 'enter'\n\tif _, prs := opts.Keymap[tui.DoubleClick.AsEvent()]; !prs {\n\t\topts.Keymap[tui.DoubleClick.AsEvent()] = opts.Keymap[tui.CtrlM.AsEvent()]\n\t}\n\n\tif opts.Height.auto {\n\t\tfor _, s := range []sizeSpec{opts.Margin[0], opts.Margin[2]} {\n\t\t\tif s.percent {\n\t\t\t\terrorExit(\"adaptive height is not compatible with top/bottom percent margin\")\n\t\t\t}\n\t\t}\n\t\tfor _, s := range []sizeSpec{opts.Padding[0], opts.Padding[2]} {\n\t\t\tif s.percent {\n\t\t\t\terrorExit(\"adaptive height is not compatible with top/bottom percent padding\")\n\t\t\t}\n\t\t}\n\t}\n\n\t// If we're not using extended search mode, --nth option becomes irrelevant\n\t// if it contains the whole range\n\tif !opts.Extended || len(opts.Nth) == 1 {\n\t\tfor _, r := range opts.Nth {\n\t\t\tif r.begin == rangeEllipsis && r.end == rangeEllipsis {\n\t\t\t\topts.Nth = make([]Range, 0)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif opts.Bold {\n\t\ttheme := opts.Theme\n\t\tboldify := func(c tui.ColorAttr) tui.ColorAttr {\n\t\t\tdup := c\n\t\t\tif (c.Attr & tui.AttrRegular) == 0 {\n\t\t\t\tdup.Attr |= tui.Bold\n\t\t\t}\n\t\t\treturn dup\n\t\t}\n\t\ttheme.Current = boldify(theme.Current)\n\t\ttheme.CurrentMatch = boldify(theme.CurrentMatch)\n\t\ttheme.Prompt = boldify(theme.Prompt)\n\t\ttheme.Input = boldify(theme.Input)\n\t\ttheme.Cursor = boldify(theme.Cursor)\n\t\ttheme.Spinner = boldify(theme.Spinner)\n\t}\n\n\tprocessScheme(opts)\n}\n\nfunc expectsArbitraryString(opt string) bool {\n\tswitch opt {\n\tcase \"-q\", \"--query\", \"-f\", \"--filter\", \"--header\", \"--prompt\":\n\t\treturn true\n\t}\n\treturn false\n}\n\n// ParseOptions parses command-line options\nfunc ParseOptions() *Options {\n\topts := defaultOptions()\n\n\tfor idx, arg := range os.Args[1:] {\n\t\tif arg == \"--version\" && (idx == 0 || idx > 0 && !expectsArbitraryString(os.Args[idx])) {\n\t\t\topts.Version = true\n\t\t\treturn opts\n\t\t}\n\t}\n\n\t// 1. Options from $FZF_DEFAULT_OPTS_FILE\n\tif path := os.Getenv(\"FZF_DEFAULT_OPTS_FILE\"); path != \"\" {\n\t\tbytes, err := os.ReadFile(path)\n\t\tif err != nil {\n\t\t\terrorContext = \"$FZF_DEFAULT_OPTS_FILE: \"\n\t\t\terrorExit(err.Error())\n\t\t}\n\n\t\twords, parseErr := shellwords.Parse(string(bytes))\n\t\tif parseErr != nil {\n\t\t\terrorContext = path + \": \"\n\t\t\terrorExit(parseErr.Error())\n\t\t}\n\t\tif len(words) > 0 {\n\t\t\tparseOptions(opts, words)\n\t\t}\n\t}\n\n\t// 2. Options from $FZF_DEFAULT_OPTS string\n\twords, parseErr := shellwords.Parse(os.Getenv(\"FZF_DEFAULT_OPTS\"))\n\terrorContext = \"$FZF_DEFAULT_OPTS: \"\n\tif parseErr != nil {\n\t\terrorExit(parseErr.Error())\n\t}\n\tif len(words) > 0 {\n\t\tparseOptions(opts, words)\n\t}\n\n\t// 3. Options from command-line arguments\n\terrorContext = \"\"\n\tparseOptions(opts, os.Args[1:])\n\n\tif err := opts.initProfiling(); err != nil {\n\t\terrorExit(\"failed to start pprof profiles: \" + err.Error())\n\t}\n\n\tpostProcessOptions(opts)\n\n\treturn opts\n}\n\n// Path: src/options_no_pprof.go\n//go:build !pprof\n// +build !pprof\n\npackage fzf\n\nfunc (o *Options) initProfiling() error {\n\tif o.CPUProfile != \"\" || o.MEMProfile != \"\" || o.BlockProfile != \"\" || o.MutexProfile != \"\" {\n\t\terrorExit(\"error: profiling not supported: FZF must be built with '-tags=pprof' to enable profiling\")\n\t}\n\treturn nil\n}\n\n// Path: src/options_pprof.go\n//go:build pprof\n// +build pprof\n\npackage fzf\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\t\"runtime/pprof\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\nfunc (o *Options) initProfiling() error {\n\tif o.CPUProfile != \"\" {\n\t\tf, err := os.Create(o.CPUProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create CPU profile: %w\", err)\n\t\t}\n\n\t\tif err := pprof.StartCPUProfile(f); err != nil {\n\t\t\treturn fmt.Errorf(\"could not start CPU profile: %w\", err)\n\t\t}\n\n\t\tutil.AtExit(func() {\n\t\t\tpprof.StopCPUProfile()\n\t\t\tif err := f.Close(); err != nil {\n\t\t\t\tfmt.Fprintln(os.Stderr, \"Error: closing cpu profile:\", err)\n\t\t\t}\n\t\t})\n\t}\n\n\tstopProfile := func(name string, f *os.File) {\n\t\tif err := pprof.Lookup(name).WriteTo(f, 0); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"Error: could not write %s profile: %v\\n\", name, err)\n\t\t}\n\t\tif err := f.Close(); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"Error: closing %s profile: %v\\n\", name, err)\n\t\t}\n\t}\n\n\tif o.MEMProfile != \"\" {\n\t\tf, err := os.Create(o.MEMProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create MEM profile: %w\", err)\n\t\t}\n\t\tutil.AtExit(func() {\n\t\t\truntime.GC()\n\t\t\tstopProfile(\"allocs\", f)\n\t\t})\n\t}\n\n\tif o.BlockProfile != \"\" {\n\t\truntime.SetBlockProfileRate(1)\n\t\tf, err := os.Create(o.BlockProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create BLOCK profile: %w\", err)\n\t\t}\n\t\tutil.AtExit(func() { stopProfile(\"block\", f) })\n\t}\n\n\tif o.MutexProfile != \"\" {\n\t\truntime.SetMutexProfileFraction(1)\n\t\tf, err := os.Create(o.MutexProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create MUTEX profile: %w\", err)\n\t\t}\n\t\tutil.AtExit(func() { stopProfile(\"mutex\", f) })\n\t}\n\n\treturn nil\n}\n\n// Path: src/options_pprof_test.go\n//go:build pprof\n// +build pprof\n\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"flag\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// runInitProfileTests is an internal flag used TestInitProfiling\nvar runInitProfileTests = flag.Bool(\"test-init-profile\", false, \"run init profile tests\")\n\nfunc TestInitProfiling(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skip(\"short test\")\n\t}\n\n\t// Run this test in a separate process since it interferes with\n\t// profiling and modifies the global atexit state. Without this\n\t// running `go test -bench . -cpuprofile cpu.out` will fail.\n\tif !*runInitProfileTests {\n\t\tt.Parallel()\n\n\t\t// Make sure we are not the child process.\n\t\tif os.Getenv(\"_FZF_CHILD_PROC\") != \"\" {\n\t\t\tt.Fatal(\"already running as child process!\")\n\t\t}\n\n\t\tcmd := exec.Command(os.Args[0],\n\t\t\t\"-test.timeout\", \"30s\",\n\t\t\t\"-test.run\", \"^\"+t.Name()+\"$\",\n\t\t\t\"-test-init-profile\",\n\t\t)\n\t\tcmd.Env = append(os.Environ(), \"_FZF_CHILD_PROC=1\")\n\n\t\tout, err := cmd.CombinedOutput()\n\t\tout = bytes.TrimSpace(out)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Child test process failed: %v:\\n%s\", err, out)\n\t\t}\n\t\t// Make sure the test actually ran\n\t\tif bytes.Contains(out, []byte(\"no tests to run\")) {\n\t\t\tt.Fatalf(\"Failed to run test %q:\\n%s\", t.Name(), out)\n\t\t}\n\t\treturn\n\t}\n\n\t// Child process\n\n\ttempdir := t.TempDir()\n\tt.Cleanup(util.RunAtExitFuncs)\n\n\to := Options{\n\t\tCPUProfile:   filepath.Join(tempdir, \"cpu.prof\"),\n\t\tMEMProfile:   filepath.Join(tempdir, \"mem.prof\"),\n\t\tBlockProfile: filepath.Join(tempdir, \"block.prof\"),\n\t\tMutexProfile: filepath.Join(tempdir, \"mutex.prof\"),\n\t}\n\tif err := o.initProfiling(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tprofiles := []string{\n\t\to.CPUProfile,\n\t\to.MEMProfile,\n\t\to.BlockProfile,\n\t\to.MutexProfile,\n\t}\n\tfor _, name := range profiles {\n\t\tif _, err := os.Stat(name); err != nil {\n\t\t\tt.Errorf(\"Failed to create profile %s: %v\", filepath.Base(name), err)\n\t\t}\n\t}\n\n\tutil.RunAtExitFuncs()\n\n\tfor _, name := range profiles {\n\t\tif _, err := os.Stat(name); err != nil {\n\t\t\tt.Errorf(\"Failed to write profile %s: %v\", filepath.Base(name), err)\n\t\t}\n\t}\n}\n\n// Path: src/options_test.go\npackage fzf\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n)\n\nfunc TestDelimiterRegex(t *testing.T) {\n\t// Valid regex\n\tdelim := delimiterRegexp(\".\")\n\tif delim.regex == nil || delim.str != nil {\n\t\tt.Error(delim)\n\t}\n\t// Broken regex -> string\n\tdelim = delimiterRegexp(\"[0-9\")\n\tif delim.regex != nil || *delim.str != \"[0-9\" {\n\t\tt.Error(delim)\n\t}\n\t// Valid regex\n\tdelim = delimiterRegexp(\"[0-9]\")\n\tif delim.regex.String() != \"[0-9]\" || delim.str != nil {\n\t\tt.Error(delim)\n\t}\n\t// Tab character\n\tdelim = delimiterRegexp(\"\\t\")\n\tif delim.regex != nil || *delim.str != \"\\t\" {\n\t\tt.Error(delim)\n\t}\n\t// Tab expression\n\tdelim = delimiterRegexp(\"\\\\t\")\n\tif delim.regex != nil || *delim.str != \"\\t\" {\n\t\tt.Error(delim)\n\t}\n\t// Tabs -> regex\n\tdelim = delimiterRegexp(\"\\t+\")\n\tif delim.regex == nil || delim.str != nil {\n\t\tt.Error(delim)\n\t}\n}\n\nfunc TestDelimiterRegexString(t *testing.T) {\n\tdelim := delimiterRegexp(\"*\")\n\ttokens := Tokenize(\"-*--*---**---\", delim)\n\tif delim.regex != nil ||\n\t\ttokens[0].text.ToString() != \"-*\" ||\n\t\ttokens[1].text.ToString() != \"--*\" ||\n\t\ttokens[2].text.ToString() != \"---*\" ||\n\t\ttokens[3].text.ToString() != \"*\" ||\n\t\ttokens[4].text.ToString() != \"---\" {\n\t\tt.Errorf(\"%s %v %d\", delim, tokens, len(tokens))\n\t}\n}\n\nfunc TestDelimiterRegexRegex(t *testing.T) {\n\tdelim := delimiterRegexp(\"--\\\\*\")\n\ttokens := Tokenize(\"-*--*---**---\", delim)\n\tif delim.str != nil ||\n\t\ttokens[0].text.ToString() != \"-*--*\" ||\n\t\ttokens[1].text.ToString() != \"---*\" ||\n\t\ttokens[2].text.ToString() != \"*---\" {\n\t\tt.Errorf(\"%s %d\", tokens, len(tokens))\n\t}\n}\n\nfunc TestDelimiterRegexRegexCaret(t *testing.T) {\n\tdelim := delimiterRegexp(`(^\\s*|\\s+)`)\n\ttokens := Tokenize(\"foo  bar baz\", delim)\n\tif delim.str != nil ||\n\t\tlen(tokens) != 4 ||\n\t\ttokens[0].text.ToString() != \"\" ||\n\t\ttokens[1].text.ToString() != \"foo  \" ||\n\t\ttokens[2].text.ToString() != \"bar \" ||\n\t\ttokens[3].text.ToString() != \"baz\" {\n\t\tt.Errorf(\"%s %d\", tokens, len(tokens))\n\t}\n}\n\nfunc TestSplitNth(t *testing.T) {\n\t{\n\t\tranges := splitNth(\"..\")\n\t\tif len(ranges) != 1 ||\n\t\t\tranges[0].begin != rangeEllipsis ||\n\t\t\tranges[0].end != rangeEllipsis {\n\t\t\tt.Errorf(\"%v\", ranges)\n\t\t}\n\t}\n\t{\n\t\tranges := splitNth(\"..3,1..,2..3,4..-1,-3..-2,..,2,-2,2..-2,1..-1\")\n\t\tif len(ranges) != 10 ||\n\t\t\tranges[0].begin != rangeEllipsis || ranges[0].end != 3 ||\n\t\t\tranges[1].begin != rangeEllipsis || ranges[1].end != rangeEllipsis ||\n\t\t\tranges[2].begin != 2 || ranges[2].end != 3 ||\n\t\t\tranges[3].begin != 4 || ranges[3].end != rangeEllipsis ||\n\t\t\tranges[4].begin != -3 || ranges[4].end != -2 ||\n\t\t\tranges[5].begin != rangeEllipsis || ranges[5].end != rangeEllipsis ||\n\t\t\tranges[6].begin != 2 || ranges[6].end != 2 ||\n\t\t\tranges[7].begin != -2 || ranges[7].end != -2 ||\n\t\t\tranges[8].begin != 2 || ranges[8].end != -2 ||\n\t\t\tranges[9].begin != rangeEllipsis || ranges[9].end != rangeEllipsis {\n\t\t\tt.Errorf(\"%v\", ranges)\n\t\t}\n\t}\n}\n\nfunc TestIrrelevantNth(t *testing.T) {\n\t{\n\t\topts := defaultOptions()\n\t\twords := []string{\"--nth\", \"..\", \"-x\"}\n\t\tparseOptions(opts, words)\n\t\tpostProcessOptions(opts)\n\t\tif len(opts.Nth) != 0 {\n\t\t\tt.Errorf(\"nth should be empty: %v\", opts.Nth)\n\t\t}\n\t}\n\tfor _, words := range [][]string{{\"--nth\", \"..,3\", \"+x\"}, {\"--nth\", \"3,1..\", \"+x\"}, {\"--nth\", \"..-1,1\", \"+x\"}} {\n\t\t{\n\t\t\topts := defaultOptions()\n\t\t\tparseOptions(opts, words)\n\t\t\tpostProcessOptions(opts)\n\t\t\tif len(opts.Nth) != 0 {\n\t\t\t\tt.Errorf(\"nth should be empty: %v\", opts.Nth)\n\t\t\t}\n\t\t}\n\t\t{\n\t\t\topts := defaultOptions()\n\t\t\twords = append(words, \"-x\")\n\t\t\tparseOptions(opts, words)\n\t\t\tpostProcessOptions(opts)\n...\n", "needle_token_start": 9837, "needle_token_end": 9927, "code_context_ntokens": 15167, "output": ["```go\nfunc validateSign(sign string, signOptName string) error {\n\tif sign == \"\" {\n\t\treturn fmt.Errorf(\"%v cannot be empty\", signOptName)\n\t}\n\tif uniseg.StringWidth(sign) > 2 {\n\t\treturn fmt.Errorf(\"%v display width should be up to 2\", signOptName)\n\t}\n\treturn nil\n}\n```"]}
{"repo": "junegunn/fzf", "name": "calculateScore", "language": "go", "path": "src/algo/algo.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to compute a scoring metric that quantifies the similarity between a text and a pattern, taking into account character matching, consecutive character sequences, and gaps. It also optionally records the positions of matched characters in the text.\n2. **Input**: The function accepts several parameters including case sensitivity flag, normalization flag, the text as a character array, the pattern as an array of runes, start and end indices for the text segment to be scored, and a flag indicating whether to record the positions of matches.\n3. **Output**: Returns a tuple containing the total score and an optional array of positions where characters from the pattern match the text.\n4. **Procedure**: The function iterates over the specified segment of the text, comparing each character to the corresponding character in the pattern. It adjusts the score based on whether characters match, whether they form a consecutive sequence, or if there is a gap. The scoring is influenced by predefined bonuses for consecutive matches and penalties for gaps. Additionally, the function can track the positions of matching characters if required.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/actiontype_string.go\n// Code generated by \"stringer -type=actionType\"; DO NOT EDIT.\n\npackage fzf\n\nimport \"strconv\"\n\nfunc _() {\n\t// An \"invalid array index\" compiler error signifies that the constant values have changed.\n\t// Re-run the stringer command to generate them again.\n\tvar x [1]struct{}\n\t_ = x[actIgnore-0]\n\t_ = x[actStart-1]\n\t_ = x[actClick-2]\n\t_ = x[actInvalid-3]\n\t_ = x[actChar-4]\n\t_ = x[actMouse-5]\n\t_ = x[actBeginningOfLine-6]\n\t_ = x[actAbort-7]\n\t_ = x[actAccept-8]\n\t_ = x[actAcceptNonEmpty-9]\n\t_ = x[actAcceptOrPrintQuery-10]\n\t_ = x[actBackwardChar-11]\n\t_ = x[actBackwardDeleteChar-12]\n\t_ = x[actBackwardDeleteCharEof-13]\n\t_ = x[actBackwardWord-14]\n\t_ = x[actCancel-15]\n\t_ = x[actChangeBorderLabel-16]\n\t_ = x[actChangeHeader-17]\n\t_ = x[actChangePreviewLabel-18]\n\t_ = x[actChangePrompt-19]\n\t_ = x[actChangeQuery-20]\n\t_ = x[actClearScreen-21]\n\t_ = x[actClearQuery-22]\n\t_ = x[actClearSelection-23]\n\t_ = x[actClose-24]\n\t_ = x[actDeleteChar-25]\n\t_ = x[actDeleteCharEof-26]\n\t_ = x[actEndOfLine-27]\n\t_ = x[actForwardChar-28]\n\t_ = x[actForwardWord-29]\n\t_ = x[actKillLine-30]\n\t_ = x[actKillWord-31]\n\t_ = x[actUnixLineDiscard-32]\n\t_ = x[actUnixWordRubout-33]\n\t_ = x[actYank-34]\n\t_ = x[actBackwardKillWord-35]\n\t_ = x[actSelectAll-36]\n\t_ = x[actDeselectAll-37]\n\t_ = x[actToggle-38]\n\t_ = x[actToggleSearch-39]\n\t_ = x[actToggleAll-40]\n\t_ = x[actToggleDown-41]\n\t_ = x[actToggleUp-42]\n\t_ = x[actToggleIn-43]\n\t_ = x[actToggleOut-44]\n\t_ = x[actToggleTrack-45]\n\t_ = x[actToggleTrackCurrent-46]\n\t_ = x[actToggleHeader-47]\n\t_ = x[actTrackCurrent-48]\n\t_ = x[actUntrackCurrent-49]\n\t_ = x[actDown-50]\n\t_ = x[actUp-51]\n\t_ = x[actPageUp-52]\n\t_ = x[actPageDown-53]\n\t_ = x[actPosition-54]\n\t_ = x[actHalfPageUp-55]\n\t_ = x[actHalfPageDown-56]\n\t_ = x[actOffsetUp-57]\n\t_ = x[actOffsetDown-58]\n\t_ = x[actJump-59]\n\t_ = x[actJumpAccept-60]\n\t_ = x[actPrintQuery-61]\n\t_ = x[actRefreshPreview-62]\n\t_ = x[actReplaceQuery-63]\n\t_ = x[actToggleSort-64]\n\t_ = x[actShowPreview-65]\n\t_ = x[actHidePreview-66]\n\t_ = x[actTogglePreview-67]\n\t_ = x[actTogglePreviewWrap-68]\n\t_ = x[actTransform-69]\n\t_ = x[actTransformBorderLabel-70]\n\t_ = x[actTransformHeader-71]\n\t_ = x[actTransformPreviewLabel-72]\n\t_ = x[actTransformPrompt-73]\n\t_ = x[actTransformQuery-74]\n\t_ = x[actPreview-75]\n\t_ = x[actChangePreview-76]\n\t_ = x[actChangePreviewWindow-77]\n\t_ = x[actPreviewTop-78]\n\t_ = x[actPreviewBottom-79]\n\t_ = x[actPreviewUp-80]\n\t_ = x[actPreviewDown-81]\n\t_ = x[actPreviewPageUp-82]\n\t_ = x[actPreviewPageDown-83]\n\t_ = x[actPreviewHalfPageUp-84]\n\t_ = x[actPreviewHalfPageDown-85]\n\t_ = x[actPrevHistory-86]\n\t_ = x[actPrevSelected-87]\n\t_ = x[actPut-88]\n\t_ = x[actNextHistory-89]\n\t_ = x[actNextSelected-90]\n\t_ = x[actExecute-91]\n\t_ = x[actExecuteSilent-92]\n\t_ = x[actExecuteMulti-93]\n\t_ = x[actSigStop-94]\n\t_ = x[actFirst-95]\n\t_ = x[actLast-96]\n\t_ = x[actReload-97]\n\t_ = x[actReloadSync-98]\n\t_ = x[actDisableSearch-99]\n\t_ = x[actEnableSearch-100]\n\t_ = x[actSelect-101]\n\t_ = x[actDeselect-102]\n\t_ = x[actUnbind-103]\n\t_ = x[actRebind-104]\n\t_ = x[actBecome-105]\n\t_ = x[actResponse-106]\n\t_ = x[actShowHeader-107]\n\t_ = x[actHideHeader-108]\n}\n\nconst _actionType_name = \"actIgnoreactStartactClickactInvalidactCharactMouseactBeginningOfLineactAbortactAcceptactAcceptNonEmptyactAcceptOrPrintQueryactBackwardCharactBackwardDeleteCharactBackwardDeleteCharEofactBackwardWordactCancelactChangeBorderLabelactChangeHeaderactChangePreviewLabelactChangePromptactChangeQueryactClearScreenactClearQueryactClearSelectionactCloseactDeleteCharactDeleteCharEofactEndOfLineactForwardCharactForwardWordactKillLineactKillWordactUnixLineDiscardactUnixWordRuboutactYankactBackwardKillWordactSelectAllactDeselectAllactToggleactToggleSearchactToggleAllactToggleDownactToggleUpactToggleInactToggleOutactToggleTrackactToggleTrackCurrentactToggleHeaderactTrackCurrentactUntrackCurrentactDownactUpactPageUpactPageDownactPositionactHalfPageUpactHalfPageDownactOffsetUpactOffsetDownactJumpactJumpAcceptactPrintQueryactRefreshPreviewactReplaceQueryactToggleSortactShowPreviewactHidePreviewactTogglePreviewactTogglePreviewWrapactTransformactTransformBorderLabelactTransformHeaderactTransformPreviewLabelactTransformPromptactTransformQueryactPreviewactChangePreviewactChangePreviewWindowactPreviewTopactPreviewBottomactPreviewUpactPreviewDownactPreviewPageUpactPreviewPageDownactPreviewHalfPageUpactPreviewHalfPageDownactPrevHistoryactPrevSelectedactPutactNextHistoryactNextSelectedactExecuteactExecuteSilentactExecuteMultiactSigStopactFirstactLastactReloadactReloadSyncactDisableSearchactEnableSearchactSelectactDeselectactUnbindactRebindactBecomeactResponseactShowHeaderactHideHeader\"\n\nvar _actionType_index = [...]uint16{0, 9, 17, 25, 35, 42, 50, 68, 76, 85, 102, 123, 138, 159, 183, 198, 207, 227, 242, 263, 278, 292, 306, 319, 336, 344, 357, 373, 385, 399, 413, 424, 435, 453, 470, 477, 496, 508, 522, 531, 546, 558, 571, 582, 593, 605, 619, 640, 655, 670, 687, 694, 699, 708, 719, 730, 743, 758, 769, 782, 789, 802, 815, 832, 847, 860, 874, 888, 904, 924, 936, 959, 977, 1001, 1019, 1036, 1046, 1062, 1084, 1097, 1113, 1125, 1139, 1155, 1173, 1193, 1215, 1229, 1244, 1250, 1264, 1279, 1289, 1305, 1320, 1330, 1338, 1345, 1354, 1367, 1383, 1398, 1407, 1418, 1427, 1436, 1445, 1456, 1469, 1482}\n\nfunc (i actionType) String() string {\n\tif i < 0 || i >= actionType(len(_actionType_index)-1) {\n\t\treturn \"actionType(\" + strconv.FormatInt(int64(i), 10) + \")\"\n\t}\n\treturn _actionType_name[_actionType_index[i]:_actionType_index[i+1]]\n}\n\n// Path: src/util/util.go\npackage util\n\nimport (\n\t\"math\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/rivo/uniseg\"\n)\n\n// StringWidth returns string width where each CR/LF character takes 1 column\nfunc StringWidth(s string) int {\n\treturn uniseg.StringWidth(s) + strings.Count(s, \"\\n\") + strings.Count(s, \"\\r\")\n}\n\n// RunesWidth returns runes width\nfunc RunesWidth(runes []rune, prefixWidth int, tabstop int, limit int) (int, int) {\n\twidth := 0\n\tgr := uniseg.NewGraphemes(string(runes))\n\tidx := 0\n\tfor gr.Next() {\n\t\trs := gr.Runes()\n\t\tvar w int\n\t\tif len(rs) == 1 && rs[0] == '\\t' {\n\t\t\tw = tabstop - (prefixWidth+width)%tabstop\n\t\t} else {\n\t\t\tw = StringWidth(string(rs))\n\t\t}\n\t\twidth += w\n\t\tif width > limit {\n\t\t\treturn width, idx\n\t\t}\n\t\tidx += len(rs)\n\t}\n\treturn width, -1\n}\n\n// Truncate returns the truncated runes and its width\nfunc Truncate(input string, limit int) ([]rune, int) {\n\trunes := []rune{}\n\twidth := 0\n\tgr := uniseg.NewGraphemes(input)\n\tfor gr.Next() {\n\t\trs := gr.Runes()\n\t\tw := StringWidth(string(rs))\n\t\tif width+w > limit {\n\t\t\treturn runes, width\n\t\t}\n\t\twidth += w\n\t\trunes = append(runes, rs...)\n\t}\n\treturn runes, width\n}\n\n// Max returns the largest integer\nfunc Max(first int, second int) int {\n\tif first >= second {\n\t\treturn first\n\t}\n\treturn second\n}\n\n// Max16 returns the largest integer\nfunc Max16(first int16, second int16) int16 {\n\tif first >= second {\n\t\treturn first\n\t}\n\treturn second\n}\n\n// Max32 returns the largest 32-bit integer\nfunc Max32(first int32, second int32) int32 {\n\tif first > second {\n\t\treturn first\n\t}\n\treturn second\n}\n\n// Min returns the smallest integer\nfunc Min(first int, second int) int {\n\tif first <= second {\n\t\treturn first\n\t}\n\treturn second\n}\n\n// Min32 returns the smallest 32-bit integer\nfunc Min32(first int32, second int32) int32 {\n\tif first <= second {\n\t\treturn first\n\t}\n\treturn second\n}\n\n// Constrain32 limits the given 32-bit integer with the upper and lower bounds\nfunc Constrain32(val int32, min int32, max int32) int32 {\n\tif val < min {\n\t\treturn min\n\t}\n\tif val > max {\n\t\treturn max\n\t}\n\treturn val\n}\n\n// Constrain limits the given integer with the upper and lower bounds\nfunc Constrain(val int, min int, max int) int {\n\tif val < min {\n\t\treturn min\n\t}\n\tif val > max {\n\t\treturn max\n\t}\n\treturn val\n}\n\nfunc AsUint16(val int) uint16 {\n\tif val > math.MaxUint16 {\n\t\treturn math.MaxUint16\n\t} else if val < 0 {\n\t\treturn 0\n\t}\n\treturn uint16(val)\n}\n\n// DurWithin limits the given time.Duration with the upper and lower bounds\nfunc DurWithin(\n\tval time.Duration, min time.Duration, max time.Duration) time.Duration {\n\tif val < min {\n\t\treturn min\n\t}\n\tif val > max {\n\t\treturn max\n\t}\n\treturn val\n}\n\n// IsTty returns true if stdin is a terminal\nfunc IsTty() bool {\n\treturn isatty.IsTerminal(os.Stdin.Fd())\n}\n\n// ToTty returns true if stdout is a terminal\nfunc ToTty() bool {\n\treturn isatty.IsTerminal(os.Stdout.Fd())\n}\n\n// Once returns a function that returns the specified boolean value only once\nfunc Once(nextResponse bool) func() bool {\n\tstate := nextResponse\n\treturn func() bool {\n\t\tprevState := state\n\t\tstate = false\n\t\treturn prevState\n\t}\n}\n\n// RepeatToFill repeats the given string to fill the given width\nfunc RepeatToFill(str string, length int, limit int) string {\n\ttimes := limit / length\n\trest := limit % length\n\toutput := strings.Repeat(str, times)\n\tif rest > 0 {\n\t\tfor _, r := range str {\n\t\t\trest -= uniseg.StringWidth(string(r))\n\t\t\tif rest < 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\toutput += string(r)\n\t\t\tif rest == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn output\n}\n\n// ToKebabCase converts the given CamelCase string to kebab-case\nfunc ToKebabCase(s string) string {\n\tname := \"\"\n\tfor i, r := range s {\n\t\tif i > 0 && r >= 'A' && r <= 'Z' {\n\t\t\tname += \"-\"\n\t\t}\n\t\tname += string(r)\n\t}\n\treturn strings.ToLower(name)\n}\n\n// Path: src/util/slab.go\npackage util\n\ntype Slab struct {\n\tI16 []int16\n\tI32 []int32\n}\n\nfunc MakeSlab(size16 int, size32 int) *Slab {\n\treturn &Slab{\n\t\tI16: make([]int16, size16),\n\t\tI32: make([]int32, size32)}\n}\n\n// Path: src/algo/algo.go\npackage algo\n\n/*\n\nAlgorithm\n---------\n\nFuzzyMatchV1 finds the first \"fuzzy\" occurrence of the pattern within the given\ntext in O(n) time where n is the length of the text. Once the position of the\nlast character is located, it traverses backwards to see if there's a shorter\nsubstring that matches the pattern.\n\n    a_____b___abc__  To find \"abc\"\n    *-----*-----*>   1. Forward scan\n             <***    2. Backward scan\n\nThe algorithm is simple and fast, but as it only sees the first occurrence,\nit is not guaranteed to find the occurrence with the highest score.\n\n    a_____b__c__abc\n    *-----*--*  ***\n\nFuzzyMatchV2 implements a modified version of Smith-Waterman algorithm to find\nthe optimal solution (highest score) according to the scoring criteria. Unlike\nthe original algorithm, omission or mismatch of a character in the pattern is\nnot allowed.\n\nPerformance\n-----------\n\nThe new V2 algorithm is slower than V1 as it examines all occurrences of the\npattern instead of stopping immediately after finding the first one. The time\ncomplexity of the algorithm is O(nm) if a match is found and O(n) otherwise\nwhere n is the length of the item and m is the length of the pattern. Thus, the\nperformance overhead may not be noticeable for a query with high selectivity.\nHowever, if the performance is more important than the quality of the result,\nyou can still choose v1 algorithm with --algo=v1.\n\nScoring criteria\n----------------\n\n- We prefer matches at special positions, such as the start of a word, or\n  uppercase character in camelCase words.\n\n- That is, we prefer an occurrence of the pattern with more characters\n  matching at special positions, even if the total match length is longer.\n    e.g. \"fuzzyfinder\" vs. \"fuzzy-finder\" on \"ff\"\n                            ````````````\n- Also, if the first character in the pattern appears at one of the special\n  positions, the bonus point for the position is multiplied by a constant\n  as it is extremely likely that the first character in the typed pattern\n  has more significance than the rest.\n    e.g. \"fo-bar\" vs. \"foob-r\" on \"br\"\n          ``````\n- But since fzf is still a fuzzy finder, not an acronym finder, we should also\n  consider the total length of the matched substring. This is why we have the\n  gap penalty. The gap penalty increases as the length of the gap (distance\n  between the matching characters) increases, so the effect of the bonus is\n  eventually cancelled at some point.\n    e.g. \"fuzzyfinder\" vs. \"fuzzy-blurry-finder\" on \"ff\"\n          ```````````\n- Consequently, it is crucial to find the right balance between the bonus\n  and the gap penalty. The parameters were chosen that the bonus is cancelled\n  when the gap size increases beyond 8 characters.\n\n- The bonus mechanism can have the undesirable side effect where consecutive\n  matches are ranked lower than the ones with gaps.\n    e.g. \"foobar\" vs. \"foo-bar\" on \"foob\"\n                       ```````\n- To correct this anomaly, we also give extra bonus point to each character\n  in a consecutive matching chunk.\n    e.g. \"foobar\" vs. \"foo-bar\" on \"foob\"\n          ``````\n- The amount of consecutive bonus is primarily determined by the bonus of the\n  first character in the chunk.\n    e.g. \"foobar\" vs. \"out-of-bound\" on \"oob\"\n                       ````````````\n*/\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"unicode\"\n\t\"unicode/utf8\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\nvar DEBUG bool\n\nvar delimiterChars = \"/,:;|\"\n\nconst whiteChars = \" \\t\\n\\v\\f\\r\\x85\\xA0\"\n\nfunc indexAt(index int, max int, forward bool) int {\n\tif forward {\n\t\treturn index\n\t}\n\treturn max - index - 1\n}\n\n// Result contains the results of running a match function.\ntype Result struct {\n\t// TODO int32 should suffice\n\tStart int\n\tEnd   int\n\tScore int\n}\n\nconst (\n\tscoreMatch        = 16\n\tscoreGapStart     = -3\n\tscoreGapExtension = -1\n\n\t// We prefer matches at the beginning of a word, but the bonus should not be\n\t// too great to prevent the longer acronym matches from always winning over\n\t// shorter fuzzy matches. The bonus point here was specifically chosen that\n\t// the bonus is cancelled when the gap between the acronyms grows over\n\t// 8 characters, which is approximately the average length of the words found\n\t// in web2 dictionary and my file system.\n\tbonusBoundary = scoreMatch / 2\n\n\t// Although bonus point for non-word characters is non-contextual, we need it\n\t// for computing bonus points for consecutive chunks starting with a non-word\n\t// character.\n\tbonusNonWord = scoreMatch / 2\n\n\t// Edge-triggered bonus for matches in camelCase words.\n\t// Compared to word-boundary case, they don't accompany single-character gaps\n\t// (e.g. FooBar vs. foo-bar), so we deduct bonus point accordingly.\n\tbonusCamel123 = bonusBoundary + scoreGapExtension\n\n\t// Minimum bonus point given to characters in consecutive chunks.\n\t// Note that bonus points for consecutive matches shouldn't have needed if we\n\t// used fixed match score as in the original algorithm.\n\tbonusConsecutive = -(scoreGapStart + scoreGapExtension)\n\n\t// The first character in the typed pattern usually has more significance\n\t// than the rest so it's important that it appears at special positions where\n\t// bonus points are given, e.g. \"to-go\" vs. \"ongoing\" on \"og\" or on \"ogo\".\n\t// The amount of the extra bonus should be limited so that the gap penalty is\n\t// still respected.\n\tbonusFirstCharMultiplier = 2\n)\n\nvar (\n\t// Extra bonus for word boundary after whitespace character or beginning of the string\n\tbonusBoundaryWhite int16 = bonusBoundary + 2\n\n\t// Extra bonus for word boundary after slash, colon, semi-colon, and comma\n\tbonusBoundaryDelimiter int16 = bonusBoundary + 1\n\n\tinitialCharClass charClass = charWhite\n\n\t// A minor optimization that can give 15%+ performance boost\n\tasciiCharClasses [unicode.MaxASCII + 1]charClass\n\n\t// A minor optimization that can give yet another 5% performance boost\n\tbonusMatrix [charNumber + 1][charNumber + 1]int16\n)\n\ntype charClass int\n\nconst (\n\tcharWhite charClass = iota\n\tcharNonWord\n\tcharDelimiter\n\tcharLower\n\tcharUpper\n\tcharLetter\n\tcharNumber\n)\n\nfunc Init(scheme string) bool {\n\tswitch scheme {\n\tcase \"default\":\n\t\tbonusBoundaryWhite = bonusBoundary + 2\n\t\tbonusBoundaryDelimiter = bonusBoundary + 1\n\tcase \"path\":\n\t\tbonusBoundaryWhite = bonusBoundary\n\t\tbonusBoundaryDelimiter = bonusBoundary + 1\n\t\tif os.PathSeparator == '/' {\n\t\t\tdelimiterChars = \"/\"\n\t\t} else {\n\t\t\tdelimiterChars = string([]rune{os.PathSeparator, '/'})\n\t\t}\n\t\tinitialCharClass = charDelimiter\n\tcase \"history\":\n\t\tbonusBoundaryWhite = bonusBoundary\n\t\tbonusBoundaryDelimiter = bonusBoundary\n\tdefault:\n\t\treturn false\n\t}\n\tfor i := 0; i <= unicode.MaxASCII; i++ {\n\t\tchar := rune(i)\n\t\tc := charNonWord\n\t\tif char >= 'a' && char <= 'z' {\n\t\t\tc = charLower\n\t\t} else if char >= 'A' && char <= 'Z' {\n\t\t\tc = charUpper\n\t\t} else if char >= '0' && char <= '9' {\n\t\t\tc = charNumber\n\t\t} else if strings.ContainsRune(whiteChars, char) {\n\t\t\tc = charWhite\n\t\t} else if strings.ContainsRune(delimiterChars, char) {\n\t\t\tc = charDelimiter\n\t\t}\n\t\tasciiCharClasses[i] = c\n\t}\n\tfor i := 0; i <= int(charNumber); i++ {\n\t\tfor j := 0; j <= int(charNumber); j++ {\n\t\t\tbonusMatrix[i][j] = bonusFor(charClass(i), charClass(j))\n\t\t}\n\t}\n\treturn true\n}\n\nfunc posArray(withPos bool, len int) *[]int {\n\tif withPos {\n\t\tpos := make([]int, 0, len)\n\t\treturn &pos\n\t}\n\treturn nil\n}\n\nfunc alloc16(offset int, slab *util.Slab, size int) (int, []int16) {\n\tif slab != nil && cap(slab.I16) > offset+size {\n\t\tslice := slab.I16[offset : offset+size]\n\t\treturn offset + size, slice\n\t}\n\treturn offset, make([]int16, size)\n}\n\nfunc alloc32(offset int, slab *util.Slab, size int) (int, []int32) {\n\tif slab != nil && cap(slab.I32) > offset+size {\n\t\tslice := slab.I32[offset : offset+size]\n\t\treturn offset + size, slice\n\t}\n\treturn offset, make([]int32, size)\n}\n\nfunc charClassOfNonAscii(char rune) charClass {\n\tif unicode.IsLower(char) {\n\t\treturn charLower\n\t} else if unicode.IsUpper(char) {\n\t\treturn charUpper\n\t} else if unicode.IsNumber(char) {\n\t\treturn charNumber\n\t} else if unicode.IsLetter(char) {\n\t\treturn charLetter\n\t} else if unicode.IsSpace(char) {\n\t\treturn charWhite\n\t} else if strings.ContainsRune(delimiterChars, char) {\n\t\treturn charDelimiter\n\t}\n\treturn charNonWord\n}\n\nfunc charClassOf(char rune) charClass {\n\tif char <= unicode.MaxASCII {\n\t\treturn asciiCharClasses[char]\n\t}\n\treturn charClassOfNonAscii(char)\n}\n\nfunc bonusFor(prevClass charClass, class charClass) int16 {\n\tif class > charNonWord {\n\t\tswitch prevClass {\n\t\tcase charWhite:\n\t\t\t// Word boundary after whitespace\n\t\t\treturn bonusBoundaryWhite\n\t\tcase charDelimiter:\n\t\t\t// Word boundary after a delimiter character\n\t\t\treturn bonusBoundaryDelimiter\n\t\tcase charNonWord:\n\t\t\t// Word boundary\n\t\t\treturn bonusBoundary\n\t\t}\n\t}\n\n\tif prevClass == charLower && class == charUpper ||\n\t\tprevClass != charNumber && class == charNumber {\n\t\t// camelCase letter123\n\t\treturn bonusCamel123\n\t}\n\n\tswitch class {\n\tcase charNonWord, charDelimiter:\n\t\treturn bonusNonWord\n\tcase charWhite:\n\t\treturn bonusBoundaryWhite\n\t}\n\treturn 0\n}\n\nfunc bonusAt(input *util.Chars, idx int) int16 {\n\tif idx == 0 {\n\t\treturn bonusBoundaryWhite\n\t}\n\treturn bonusMatrix[charClassOf(input.Get(idx-1))][charClassOf(input.Get(idx))]\n}\n\nfunc normalizeRune(r rune) rune {\n\tif r < 0x00C0 || r > 0x2184 {\n\t\treturn r\n\t}\n\n\tn := normalized[r]\n\tif n > 0 {\n\t\treturn n\n\t}\n\treturn r\n}\n\n// Algo functions make two assumptions\n// 1. \"pattern\" is given in lowercase if \"caseSensitive\" is false\n// 2. \"pattern\" is already normalized if \"normalize\" is true\ntype Algo func(caseSensitive bool, normalize bool, forward bool, input *util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int)\n\nfunc trySkip(input *util.Chars, caseSensitive bool, b byte, from int) int {\n\tbyteArray := input.Bytes()[from:]\n\tidx := bytes.IndexByte(byteArray, b)\n\tif idx == 0 {\n\t\t// Can't skip any further\n\t\treturn from\n\t}\n\t// We may need to search for the uppercase letter again. We don't have to\n\t// consider normalization as we can be sure that this is an ASCII string.\n\tif !caseSensitive && b >= 'a' && b <= 'z' {\n\t\tif idx > 0 {\n\t\t\tbyteArray = byteArray[:idx]\n\t\t}\n\t\tuidx := bytes.IndexByte(byteArray, b-32)\n\t\tif uidx >= 0 {\n\t\t\tidx = uidx\n\t\t}\n\t}\n\tif idx < 0 {\n\t\treturn -1\n\t}\n\treturn from + idx\n}\n\nfunc isAscii(runes []rune) bool {\n\tfor _, r := range runes {\n\t\tif r >= utf8.RuneSelf {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc asciiFuzzyIndex(input *util.Chars, pattern []rune, caseSensitive bool) (int, int) {\n\t// Can't determine\n\tif !input.IsBytes() {\n\t\treturn 0, input.Length()\n\t}\n\n\t// Not possible\n\tif !isAscii(pattern) {\n\t\treturn -1, -1\n\t}\n\n\tfirstIdx, idx, lastIdx := 0, 0, 0\n\tvar b byte\n\tfor pidx := 0; pidx < len(pattern); pidx++ {\n\t\tb = byte(pattern[pidx])\n\t\tidx = trySkip(input, caseSensitive, b, idx)\n\t\tif idx < 0 {\n\t\t\treturn -1, -1\n\t\t}\n\t\tif pidx == 0 && idx > 0 {\n\t\t\t// Step back to find the right bonus point\n\t\t\tfirstIdx = idx - 1\n\t\t}\n\t\tlastIdx = idx\n\t\tidx++\n\t}\n\n\t// Find the last appearance of the last character of the pattern to limit the search scope\n\tbu := b\n\tif !caseSensitive && b >= 'a' && b <= 'z' {\n\t\tbu = b - 32\n\t}\n\tscope := input.Bytes()[lastIdx:]\n\tfor offset := len(scope) - 1; offset > 0; offset-- {\n\t\tif scope[offset] == b || scope[offset] == bu {\n\t\t\treturn firstIdx, lastIdx + offset + 1\n\t\t}\n\t}\n\treturn firstIdx, lastIdx + 1\n}\n\nfunc debugV2(T []rune, pattern []rune, F []int32, lastIdx int, H []int16, C []int16) {\n\twidth := lastIdx - int(F[0]) + 1\n\n\tfor i, f := range F {\n\t\tI := i * width\n\t\tif i == 0 {\n\t\t\tfmt.Print(\"  \")\n\t\t\tfor j := int(f); j <= lastIdx; j++ {\n\t\t\t\tfmt.Printf(\" \" + string(T[j]) + \" \")\n\t\t\t}\n\t\t\tfmt.Println()\n\t\t}\n\t\tfmt.Print(string(pattern[i]) + \" \")\n\t\tfor idx := int(F[0]); idx < int(f); idx++ {\n\t\t\tfmt.Print(\" 0 \")\n\t\t}\n\t\tfor idx := int(f); idx <= lastIdx; idx++ {\n\t\t\tfmt.Printf(\"%2d \", H[i*width+idx-int(F[0])])\n\t\t}\n\t\tfmt.Println()\n\n\t\tfmt.Print(\"  \")\n\t\tfor idx, p := range C[I : I+width] {\n\t\t\tif idx+int(F[0]) < int(F[i]) {\n\t\t\t\tp = 0\n\t\t\t}\n\t\t\tif p > 0 {\n\t\t\t\tfmt.Printf(\"%2d \", p)\n\t\t\t} else {\n\t\t\t\tfmt.Print(\"   \")\n\t\t\t}\n\t\t}\n\t\tfmt.Println()\n\t}\n}\n\nfunc FuzzyMatchV2(caseSensitive bool, normalize bool, forward bool, input *util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {\n\t// Assume that pattern is given in lowercase if case-insensitive.\n\t// First check if there's a match and calculate bonus for each position.\n\t// If the input string is too long, consider finding the matching chars in\n\t// this phase as well (non-optimal alignment).\n\tM := len(pattern)\n\tif M == 0 {\n\t\treturn Result{0, 0, 0}, posArray(withPos, M)\n\t}\n\tN := input.Length()\n\tif M > N {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\n\t// Since O(nm) algorithm can be prohibitively expensive for large input,\n\t// we fall back to the greedy algorithm.\n\tif slab != nil && N*M > cap(slab.I16) {\n\t\treturn FuzzyMatchV1(caseSensitive, normalize, forward, input, pattern, withPos, slab)\n\t}\n\n\t// Phase 1. Optimized search for ASCII string\n\tminIdx, maxIdx := asciiFuzzyIndex(input, pattern, caseSensitive)\n\tif minIdx < 0 {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\t// fmt.Println(N, maxIdx, idx, maxIdx-idx, input.ToString())\n\tN = maxIdx - minIdx\n\n\t// Reuse pre-allocated integer slice to avoid unnecessary sweeping of garbages\n\toffset16 := 0\n\toffset32 := 0\n\toffset16, H0 := alloc16(offset16, slab, N)\n\toffset16, C0 := alloc16(offset16, slab, N)\n\t// Bonus point for each position\n\toffset16, B := alloc16(offset16, slab, N)\n\t// The first occurrence of each character in the pattern\n\toffset32, F := alloc32(offset32, slab, M)\n\t// Rune array\n\t_, T := alloc32(offset32, slab, N)\n\tinput.CopyRunes(T, minIdx)\n\n\t// Phase 2. Calculate bonus for each point\n\tmaxScore, maxScorePos := int16(0), 0\n\tpidx, lastIdx := 0, 0\n\tpchar0, pchar, prevH0, prevClass, inGap := pattern[0], pattern[0], int16(0), initialCharClass, false\n\tfor off, char := range T {\n\t\tvar class charClass\n\t\tif char <= unicode.MaxASCII {\n\t\t\tclass = asciiCharClasses[char]\n\t\t\tif !caseSensitive && class == charUpper {\n\t\t\t\tchar += 32\n\t\t\t\tT[off] = char\n\t\t\t}\n\t\t} else {\n\t\t\tclass = charClassOfNonAscii(char)\n\t\t\tif !caseSensitive && class == charUpper {\n\t\t\t\tchar = unicode.To(unicode.LowerCase, char)\n\t\t\t}\n\t\t\tif normalize {\n\t\t\t\tchar = normalizeRune(char)\n\t\t\t}\n\t\t\tT[off] = char\n\t\t}\n\n\t\tbonus := bonusMatrix[prevClass][class]\n\t\tB[off] = bonus\n\t\tprevClass = class\n\n\t\tif char == pchar {\n\t\t\tif pidx < M {\n\t\t\t\tF[pidx] = int32(off)\n\t\t\t\tpidx++\n\t\t\t\tpchar = pattern[util.Min(pidx, M-1)]\n\t\t\t}\n\t\t\tlastIdx = off\n\t\t}\n\n\t\tif char == pchar0 {\n\t\t\tscore := scoreMatch + bonus*bonusFirstCharMultiplier\n\t\t\tH0[off] = score\n\t\t\tC0[off] = 1\n\t\t\tif M == 1 && (forward && score > maxScore || !forward && score >= maxScore) {\n\t\t\t\tmaxScore, maxScorePos = score, off\n\t\t\t\tif forward && bonus >= bonusBoundary {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tinGap = false\n\t\t} else {\n\t\t\tif inGap {\n\t\t\t\tH0[off] = util.Max16(prevH0+scoreGapExtension, 0)\n\t\t\t} else {\n\t\t\t\tH0[off] = util.Max16(prevH0+scoreGapStart, 0)\n\t\t\t}\n\t\t\tC0[off] = 0\n\t\t\tinGap = true\n\t\t}\n\t\tprevH0 = H0[off]\n\t}\n\tif pidx != M {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\tif M == 1 {\n\t\tresult := Result{minIdx + maxScorePos, minIdx + maxScorePos + 1, int(maxScore)}\n\t\tif !withPos {\n\t\t\treturn result, nil\n\t\t}\n\t\tpos := []int{minIdx + maxScorePos}\n\t\treturn result, &pos\n\t}\n\n\t// Phase 3. Fill in score matrix (H)\n\t// Unlike the original algorithm, we do not allow omission.\n\tf0 := int(F[0])\n\twidth := lastIdx - f0 + 1\n\toffset16, H := alloc16(offset16, slab, width*M)\n\tcopy(H, H0[f0:lastIdx+1])\n\n\t// Possible length of consecutive chunk at each position.\n\t_, C := alloc16(offset16, slab, width*M)\n\tcopy(C, C0[f0:lastIdx+1])\n\n\tFsub := F[1:]\n\tPsub := pattern[1:][:len(Fsub)]\n\tfor off, f := range Fsub {\n\t\tf := int(f)\n\t\tpchar := Psub[off]\n\t\tpidx := off + 1\n\t\trow := pidx * width\n\t\tinGap := false\n\t\tTsub := T[f : lastIdx+1]\n\t\tBsub := B[f:][:len(Tsub)]\n\t\tCsub := C[row+f-f0:][:len(Tsub)]\n\t\tCdiag := C[row+f-f0-1-width:][:len(Tsub)]\n\t\tHsub := H[row+f-f0:][:len(Tsub)]\n\t\tHdiag := H[row+f-f0-1-width:][:len(Tsub)]\n\t\tHleft := H[row+f-f0-1:][:len(Tsub)]\n\t\tHleft[0] = 0\n\t\tfor off, char := range Tsub {\n\t\t\tcol := off + f\n\t\t\tvar s1, s2, consecutive int16\n\n\t\t\tif inGap {\n\t\t\t\ts2 = Hleft[off] + scoreGapExtension\n\t\t\t} else {\n\t\t\t\ts2 = Hleft[off] + scoreGapStart\n\t\t\t}\n\n\t\t\tif pchar == char {\n\t\t\t\ts1 = Hdiag[off] + scoreMatch\n\t\t\t\tb := Bsub[off]\n\t\t\t\tconsecutive = Cdiag[off] + 1\n\t\t\t\tif consecutive > 1 {\n\t\t\t\t\tfb := B[col-int(consecutive)+1]\n\t\t\t\t\t// Break consecutive chunk\n\t\t\t\t\tif b >= bonusBoundary && b > fb {\n\t\t\t\t\t\tconsecutive = 1\n\t\t\t\t\t} else {\n\t\t\t\t\t\tb = util.Max16(b, util.Max16(bonusConsecutive, fb))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif s1+b < s2 {\n\t\t\t\t\ts1 += Bsub[off]\n\t\t\t\t\tconsecutive = 0\n\t\t\t\t} else {\n\t\t\t\t\ts1 += b\n\t\t\t\t}\n\t\t\t}\n\t\t\tCsub[off] = consecutive\n\n\t\t\tinGap = s1 < s2\n\t\t\tscore := util.Max16(util.Max16(s1, s2), 0)\n\t\t\tif pidx == M-1 && (forward && score > maxScore || !forward && score >= maxScore) {\n\t\t\t\tmaxScore, maxScorePos = score, col\n\t\t\t}\n\t\t\tHsub[off] = score\n\t\t}\n\t}\n\n\tif DEBUG {\n\t\tdebugV2(T, pattern, F, lastIdx, H, C)\n\t}\n\n\t// Phase 4. (Optional) Backtrace to find character positions\n\tpos := posArray(withPos, M)\n\tj := f0\n\tif withPos {\n\t\ti := M - 1\n\t\tj = maxScorePos\n\t\tpreferMatch := true\n\t\tfor {\n\t\t\tI := i * width\n\t\t\tj0 := j - f0\n\t\t\ts := H[I+j0]\n\n\t\t\tvar s1, s2 int16\n\t\t\tif i > 0 && j >= int(F[i]) {\n\t\t\t\ts1 = H[I-width+j0-1]\n\t\t\t}\n\t\t\tif j > int(F[i]) {\n\t\t\t\ts2 = H[I+j0-1]\n\t\t\t}\n\n\t\t\tif s > s1 && (s > s2 || s == s2 && preferMatch) {\n\t\t\t\t*pos = append(*pos, j+minIdx)\n\t\t\t\tif i == 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\ti--\n\t\t\t}\n\t\t\tpreferMatch = C[I+j0] > 1 || I+width+j0+1 < len(C) && C[I+width+j0+1] > 0\n\t\t\tj--\n\t\t}\n\t}\n\t// Start offset we return here is only relevant when begin tiebreak is used.\n\t// However finding the accurate offset requires backtracking, and we don't\n\t// want to pay extra cost for the option that has lost its importance.\n\treturn Result{minIdx + j, minIdx + maxScorePos + 1, int(maxScore)}, pos\n}\n\n// Implement the same sorting criteria as V2\n\nfunc calculateScore(caseSensitive bool, normalize bool, text *util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {\n\tpidx, score, inGap, consecutive, firstBonus := 0, 0, false, 0, int16(0)\n\tpos := posArray(withPos, len(pattern))\n\tprevClass := initialCharClass\n\tif sidx > 0 {\n\t\tprevClass = charClassOf(text.Get(sidx - 1))\n\t}\n\tfor idx := sidx; idx < eidx; idx++ {\n\t\tchar := text.Get(idx)\n\t\tclass := charClassOf(char)\n\t\tif !caseSensitive {\n\t\t\tif char >= 'A' && char <= 'Z' {\n\t\t\t\tchar += 32\n\t\t\t} else if char > unicode.MaxASCII {\n\t\t\t\tchar = unicode.To(unicode.LowerCase, char)\n\t\t\t}\n\t\t}\n\t\t// pattern is already normalized\n\t\tif normalize {\n\t\t\tchar = normalizeRune(char)\n\t\t}\n\t\tif char == pattern[pidx] {\n\t\t\tif withPos {\n\t\t\t\t*pos = append(*pos, idx)\n\t\t\t}\n\t\t\tscore += scoreMatch\n\t\t\tbonus := bonusMatrix[prevClass][class]\n\t\t\tif consecutive == 0 {\n\t\t\t\tfirstBonus = bonus\n\t\t\t} else {\n\t\t\t\t// Break consecutive chunk\n\t\t\t\tif bonus >= bonusBoundary && bonus > firstBonus {\n\t\t\t\t\tfirstBonus = bonus\n\t\t\t\t}\n\t\t\t\tbonus = util.Max16(util.Max16(bonus, firstBonus), bonusConsecutive)\n\t\t\t}\n\t\t\tif pidx == 0 {\n\t\t\t\tscore += int(bonus * bonusFirstCharMultiplier)\n\t\t\t} else {\n\t\t\t\tscore += int(bonus)\n\t\t\t}\n\t\t\tinGap = false\n\t\t\tconsecutive++\n\t\t\tpidx++\n\t\t} else {\n\t\t\tif inGap {\n\t\t\t\tscore += scoreGapExtension\n\t\t\t} else {\n\t\t\t\tscore += scoreGapStart\n\t\t\t}\n\t\t\tinGap = true\n\t\t\tconsecutive = 0\n\t\t\tfirstBonus = 0\n\t\t}\n\t\tprevClass = class\n\t}\n\treturn score, pos\n}\n\n// FuzzyMatchV1 performs fuzzy-match\nfunc FuzzyMatchV1(caseSensitive bool, normalize bool, forward bool, text *util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {\n\tif len(pattern) == 0 {\n\t\treturn Result{0, 0, 0}, nil\n\t}\n\tidx, _ := asciiFuzzyIndex(text, pattern, caseSensitive)\n\tif idx < 0 {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\n\tpidx := 0\n\tsidx := -1\n\teidx := -1\n\n\tlenRunes := text.Length()\n\tlenPattern := len(pattern)\n\n\tfor index := 0; index < lenRunes; index++ {\n\t\tchar := text.Get(indexAt(index, lenRunes, forward))\n\t\t// This is considerably faster than blindly applying strings.ToLower to the\n\t\t// whole string\n\t\tif !caseSensitive {\n\t\t\t// Partially inlining `unicode.ToLower`. Ugly, but makes a noticeable\n\t\t\t// difference in CPU cost. (Measured on Go 1.4.1. Also note that the Go\n\t\t\t// compiler as of now does not inline non-leaf functions.)\n\t\t\tif char >= 'A' && char <= 'Z' {\n\t\t\t\tchar += 32\n\t\t\t} else if char > unicode.MaxASCII {\n\t\t\t\tchar = unicode.To(unicode.LowerCase, char)\n\t\t\t}\n\t\t}\n\t\tif normalize {\n\t\t\tchar = normalizeRune(char)\n\t\t}\n\t\tpchar := pattern[indexAt(pidx, lenPattern, forward)]\n\t\tif char == pchar {\n\t\t\tif sidx < 0 {\n\t\t\t\tsidx = index\n\t\t\t}\n\t\t\tif pidx++; pidx == lenPattern {\n\t\t\t\teidx = index + 1\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif sidx >= 0 && eidx >= 0 {\n\t\tpidx--\n\t\tfor index := eidx - 1; index >= sidx; index-- {\n\t\t\ttidx := indexAt(index, lenRunes, forward)\n\t\t\tchar := text.Get(tidx)\n\t\t\tif !caseSensitive {\n\t\t\t\tif char >= 'A' && char <= 'Z' {\n\t\t\t\t\tchar += 32\n\t\t\t\t} else if char > unicode.MaxASCII {\n\t\t\t\t\tchar = unicode.To(unicode.LowerCase, char)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpidx_ := indexAt(pidx, lenPattern, forward)\n\t\t\tpchar := pattern[pidx_]\n\t\t\tif char == pchar {\n\t\t\t\tif pidx--; pidx < 0 {\n\t\t\t\t\tsidx = index\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif !forward {\n\t\t\tsidx, eidx = lenRunes-eidx, lenRunes-sidx\n\t\t}\n\n\t\tscore, pos := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, withPos)\n\t\treturn Result{sidx, eidx, score}, pos\n\t}\n\treturn Result{-1, -1, 0}, nil\n}\n\n// ExactMatchNaive is a basic string searching algorithm that handles case\n// sensitivity. Although naive, it still performs better than the combination\n// of strings.ToLower + strings.Index for typical fzf use cases where input\n// strings and patterns are not very long.\n//\n// Since 0.15.0, this function searches for the match with the highest\n// bonus point, instead of stopping immediately after finding the first match.\n// The solution is much cheaper since there is only one possible alignment of\n// the pattern.\nfunc ExactMatchNaive(caseSensitive bool, normalize bool, forward bool, text *util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {\n\tif len(pattern) == 0 {\n\t\treturn Result{0, 0, 0}, nil\n\t}\n\n\tlenRunes := text.Length()\n\tlenPattern := len(pattern)\n\n\tif lenRunes < lenPattern {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\n\tidx, _ := asciiFuzzyIndex(text, pattern, caseSensitive)\n\tif idx < 0 {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\n\t// For simplicity, only look at the bonus at the first character position\n\tpidx := 0\n\tbestPos, bonus, bestBonus := -1, int16(0), int16(-1)\n\tfor index := 0; index < lenRunes; index++ {\n\t\tindex_ := indexAt(index, lenRunes, forward)\n\t\tchar := text.Get(index_)\n\t\tif !caseSensitive {\n\t\t\tif char >= 'A' && char <= 'Z' {\n\t\t\t\tchar += 32\n\t\t\t} else if char > unicode.MaxASCII {\n\t\t\t\tchar = unicode.To(unicode.LowerCase, char)\n\t\t\t}\n\t\t}\n\t\tif normalize {\n\t\t\tchar = normalizeRune(char)\n\t\t}\n\t\tpidx_ := indexAt(pidx, lenPattern, forward)\n\t\tpchar := pattern[pidx_]\n\t\tif pchar == char {\n\t\t\tif pidx_ == 0 {\n\t\t\t\tbonus = bonusAt(text, index_)\n\t\t\t}\n\t\t\tpidx++\n\t\t\tif pidx == lenPattern {\n\t\t\t\tif bonus > bestBonus {\n\t\t\t\t\tbestPos, bestBonus = index, bonus\n\t\t\t\t}\n\t\t\t\tif bonus >= bonusBoundary {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tindex -= pidx - 1\n\t\t\t\tpidx, bonus = 0, 0\n\t\t\t}\n\t\t} else {\n\t\t\tindex -= pidx\n\t\t\tpidx, bonus = 0, 0\n\t\t}\n\t}\n\tif bestPos >= 0 {\n\t\tvar sidx, eidx int\n\t\tif forward {\n\t\t\tsidx = bestPos - lenPattern + 1\n\t\t\teidx = bestPos + 1\n\t\t} else {\n\t\t\tsidx = lenRunes - (bestPos + 1)\n\t\t\teidx = lenRunes - (bestPos - lenPattern + 1)\n\t\t}\n\t\tscore, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)\n\t\treturn Result{sidx, eidx, score}, nil\n\t}\n\treturn Result{-1, -1, 0}, nil\n}\n\n// PrefixMatch performs prefix-match\nfunc PrefixMatch(caseSensitive bool, normalize bool, forward bool, text *util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {\n\tif len(pattern) == 0 {\n\t\treturn Result{0, 0, 0}, nil\n\t}\n\n\ttrimmedLen := 0\n\tif !unicode.IsSpace(pattern[0]) {\n\t\ttrimmedLen = text.LeadingWhitespaces()\n\t}\n\n\tif text.Length()-trimmedLen < len(pattern) {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\n\tfor index, r := range pattern {\n\t\tchar := text.Get(trimmedLen + index)\n\t\tif !caseSensitive {\n\t\t\tchar = unicode.ToLower(char)\n\t\t}\n\t\tif normalize {\n\t\t\tchar = normalizeRune(char)\n\t\t}\n\t\tif char != r {\n\t\t\treturn Result{-1, -1, 0}, nil\n\t\t}\n\t}\n\tlenPattern := len(pattern)\n\tscore, _ := calculateScore(caseSensitive, normalize, text, pattern, trimmedLen, trimmedLen+lenPattern, false)\n\treturn Result{trimmedLen, trimmedLen + lenPattern, score}, nil\n}\n\n// SuffixMatch performs suffix-match\nfunc SuffixMatch(caseSensitive bool, normalize bool, forward bool, text *util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {\n\tlenRunes := text.Length()\n\ttrimmedLen := lenRunes\n\tif len(pattern) == 0 || !unicode.IsSpace(pattern[len(pattern)-1]) {\n\t\ttrimmedLen -= text.TrailingWhitespaces()\n\t}\n\tif len(pattern) == 0 {\n\t\treturn Result{trimmedLen, trimmedLen, 0}, nil\n\t}\n\tdiff := trimmedLen - len(pattern)\n\tif diff < 0 {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\n\tfor index, r := range pattern {\n\t\tchar := text.Get(index + diff)\n\t\tif !caseSensitive {\n\t\t\tchar = unicode.ToLower(char)\n\t\t}\n\t\tif normalize {\n\t\t\tchar = normalizeRune(char)\n\t\t}\n\t\tif char != r {\n\t\t\treturn Result{-1, -1, 0}, nil\n\t\t}\n\t}\n\tlenPattern := len(pattern)\n\tsidx := trimmedLen - lenPattern\n\teidx := trimmedLen\n\tscore, _ := calculateScore(caseSensitive, normalize, text, pattern, sidx, eidx, false)\n\treturn Result{sidx, eidx, score}, nil\n}\n\n// EqualMatch performs equal-match\nfunc EqualMatch(caseSensitive bool, normalize bool, forward bool, text *util.Chars, pattern []rune, withPos bool, slab *util.Slab) (Result, *[]int) {\n\tlenPattern := len(pattern)\n\tif lenPattern == 0 {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\n\t// Strip leading whitespaces\n\ttrimmedLen := 0\n\tif !unicode.IsSpace(pattern[0]) {\n\t\ttrimmedLen = text.LeadingWhitespaces()\n\t}\n\n\t// Strip trailing whitespaces\n\ttrimmedEndLen := 0\n\tif !unicode.IsSpace(pattern[lenPattern-1]) {\n\t\ttrimmedEndLen = text.TrailingWhitespaces()\n\t}\n\n\tif text.Length()-trimmedLen-trimmedEndLen != lenPattern {\n\t\treturn Result{-1, -1, 0}, nil\n\t}\n\tmatch := true\n\tif normalize {\n\t\trunes := text.ToRunes()\n\t\tfor idx, pchar := range pattern {\n\t\t\tchar := runes[trimmedLen+idx]\n\t\t\tif !caseSensitive {\n\t\t\t\tchar = unicode.To(unicode.LowerCase, char)\n\t\t\t}\n\t\t\tif normalizeRune(pchar) != normalizeRune(char) {\n\t\t\t\tmatch = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t} else {\n\t\trunes := text.ToRunes()\n\t\trunesStr := string(runes[trimmedLen : len(runes)-trimmedEndLen])\n\t\tif !caseSensitive {\n\t\t\trunesStr = strings.ToLower(runesStr)\n\t\t}\n\t\tmatch = runesStr == string(pattern)\n\t}\n\tif match {\n\t\treturn Result{trimmedLen, trimmedLen + lenPattern, (scoreMatch+int(bonusBoundaryWhite))*lenPattern +\n\t\t\t(bonusFirstCharMultiplier-1)*int(bonusBoundaryWhite)}, nil\n\t}\n\treturn Result{-1, -1, 0}, nil\n}\n\n// Path: src/algo/algo_test.go\npackage algo\n\nimport (\n\t\"math\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\nfunc init() {\n\tInit(\"default\")\n}\n\nfunc assertMatch(t *testing.T, fun Algo, caseSensitive, forward bool, input, pattern string, sidx int, eidx int, score int) {\n\tassertMatch2(t, fun, caseSensitive, false, forward, input, pattern, sidx, eidx, score)\n}\n\nfunc assertMatch2(t *testing.T, fun Algo, caseSensitive, normalize, forward bool, input, pattern string, sidx int, eidx int, score int) {\n\tif !caseSensitive {\n\t\tpattern = strings.ToLower(pattern)\n\t}\n\tchars := util.ToChars([]byte(input))\n\tres, pos := fun(caseSensitive, normalize, forward, &chars, []rune(pattern), true, nil)\n\tvar start, end int\n\tif pos == nil || len(*pos) == 0 {\n\t\tstart = res.Start\n\t\tend = res.End\n\t} else {\n\t\tsort.Ints(*pos)\n\t\tstart = (*pos)[0]\n\t\tend = (*pos)[len(*pos)-1] + 1\n\t}\n\tif start != sidx {\n\t\tt.Errorf(\"Invalid start index: %d (expected: %d, %s / %s)\", start, sidx, input, pattern)\n\t}\n\tif end != eidx {\n\t\tt.Errorf(\"Invalid end index: %d (expected: %d, %s / %s)\", end, eidx, input, pattern)\n\t}\n\tif res.Score != score {\n...\n", "needle_token_start": 11119, "needle_token_end": 11758, "code_context_ntokens": 15235, "output": ["```go\nfunc calculateScore(caseSensitive bool, normalize bool, text *util.Chars, pattern []rune, sidx int, eidx int, withPos bool) (int, *[]int) {\n\tpidx, score, inGap, consecutive, firstBonus := 0, 0, false, 0, int16(0)\n\tpos := posArray(withPos, len(pattern))\n\tprevClass := initialCharClass\n\tif sidx > 0 {\n\t\tprevClass = charClassOf(text.Get(sidx - 1))\n\t}\n\tfor idx := sidx; idx < eidx; idx++ {\n\t\tchar := text.Get(idx)\n\t\tclass := charClassOf(char)\n\t\tif !caseSensitive {\n\t\t\tif char >= 'A' && char <= 'Z' {\n\t\t\t\tchar += 32\n\t\t\t} else if char > unicode.MaxASCII {\n\t\t\t\tchar = unicode.To(unicode.LowerCase, char)\n\t\t\t}\n\t\t}\n\t\t// pattern is already normalized\n\t\tif normalize {\n\t\t\tchar = normalizeRune(char)\n\t\t}\n\t\tif char == pattern[pidx] {\n\t\t\tif withPos {\n\t\t\t\t*pos = append(*pos, idx)\n\t\t\t}\n\t\t\tscore += scoreMatch\n\t\t\tbonus := bonusMatrix[prevClass][class]\n\t\t\tif consecutive == 0 {\n\t\t\t\tfirstBonus = bonus\n\t\t\t} else {\n\t\t\t\t// Break consecutive chunk\n\t\t\t\tif bonus >= bonusBoundary && bonus > firstBonus {\n\t\t\t\t\tfirstBonus = bonus\n\t\t\t\t}\n\t\t\t\tbonus = util.Max16(util.Max16(bonus, firstBonus), bonusConsecutive)\n\t\t\t}\n\t\t\tif pidx == 0 {\n\t\t\t\tscore += int(bonus * bonusFirstCharMultiplier)\n\t\t\t} else {\n\t\t\t\tscore += int(bonus)\n\t\t\t}\n\t\t\tinGap = false\n\t\t\tconsecutive++\n\t\t\tpidx++\n\t\t} else {\n\t\t\tif inGap {\n\t\t\t\tscore += scoreGapExtension\n\t\t\t} else {\n\t\t\t\tscore += scoreGapStart\n\t\t\t}\n\t\t\tinGap = true\n\t\t\tconsecutive = 0\n\t\t\tfirstBonus = 0\n\t\t}\n\t\tprevClass = class\n\t}\n\treturn score, pos\n}\n```"]}
{"repo": "junegunn/fzf", "name": "calculateSize", "language": "go", "path": "src/terminal.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function calculates the size of a UI component based on available space, a size specification, and minimum size constraints.\n2. **Input**: It takes five parameters: the total available space, a size specification struct (which includes a size value and a flag indicating if the size is a percentage), the space already occupied by other elements, the minimum allowable size for the component, and additional padding to be added to the size.\n3. **Output**: Returns the calculated size of the component as an integer, which respects the minimum size and does not exceed the available space.\n4. **Procedure**: \n   - First, the function calculates the remaining space by subtracting the occupied space from the total available space.\n   - It ensures that the remaining space does not fall below the specified minimum size.\n   - If the size specification is a percentage, it calculates the size as a percentage of the total available space and then constrains this value between the minimum size and the remaining space.\n   - If the size specification is an absolute value, it adds the specified padding to this value and then constrains the result between the minimum size and the remaining space.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/reader.go\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/charlievieth/fastwalk\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// Reader reads from command or standard input\ntype Reader struct {\n\tpusher   func([]byte) bool\n\teventBox *util.EventBox\n\tdelimNil bool\n\tevent    int32\n\tfinChan  chan bool\n\tmutex    sync.Mutex\n\texec     *exec.Cmd\n\tcommand  *string\n\tkilled   bool\n\twait     bool\n}\n\n// NewReader returns new Reader object\nfunc NewReader(pusher func([]byte) bool, eventBox *util.EventBox, delimNil bool, wait bool) *Reader {\n\treturn &Reader{pusher, eventBox, delimNil, int32(EvtReady), make(chan bool, 1), sync.Mutex{}, nil, nil, false, wait}\n}\n\nfunc (r *Reader) startEventPoller() {\n\tgo func() {\n\t\tptr := &r.event\n\t\tpollInterval := readerPollIntervalMin\n\t\tfor {\n\t\t\tif atomic.CompareAndSwapInt32(ptr, int32(EvtReadNew), int32(EvtReady)) {\n\t\t\t\tr.eventBox.Set(EvtReadNew, (*string)(nil))\n\t\t\t\tpollInterval = readerPollIntervalMin\n\t\t\t} else if atomic.LoadInt32(ptr) == int32(EvtReadFin) {\n\t\t\t\tif r.wait {\n\t\t\t\t\tr.finChan <- true\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t} else {\n\t\t\t\tpollInterval += readerPollIntervalStep\n\t\t\t\tif pollInterval > readerPollIntervalMax {\n\t\t\t\t\tpollInterval = readerPollIntervalMax\n\t\t\t\t}\n\t\t\t}\n\t\t\ttime.Sleep(pollInterval)\n\t\t}\n\t}()\n}\n\nfunc (r *Reader) fin(success bool) {\n\tatomic.StoreInt32(&r.event, int32(EvtReadFin))\n\tif r.wait {\n\t\t<-r.finChan\n\t}\n\n\tr.mutex.Lock()\n\tret := r.command\n\tif success || r.killed {\n\t\tret = nil\n\t}\n\tr.mutex.Unlock()\n\n\tr.eventBox.Set(EvtReadFin, ret)\n}\n\nfunc (r *Reader) terminate() {\n\tr.mutex.Lock()\n\tr.killed = true\n\tif r.exec != nil && r.exec.Process != nil {\n\t\tutil.KillCommand(r.exec)\n\t} else {\n\t\tos.Stdin.Close()\n\t}\n\tr.mutex.Unlock()\n}\n\nfunc (r *Reader) restart(command string, environ []string) {\n\tr.event = int32(EvtReady)\n\tr.startEventPoller()\n\tsuccess := r.readFromCommand(command, environ)\n\tr.fin(success)\n}\n\n// ReadSource reads data from the default command or from standard input\nfunc (r *Reader) ReadSource(root string, opts walkerOpts, ignores []string) {\n\tr.startEventPoller()\n\tvar success bool\n\tif util.IsTty() {\n\t\tcmd := os.Getenv(\"FZF_DEFAULT_COMMAND\")\n\t\tif len(cmd) == 0 {\n\t\t\tsuccess = r.readFiles(root, opts, ignores)\n\t\t} else {\n\t\t\t// We can't export FZF_* environment variables to the default command\n\t\t\tsuccess = r.readFromCommand(cmd, nil)\n\t\t}\n\t} else {\n\t\tsuccess = r.readFromStdin()\n\t}\n\tr.fin(success)\n}\n\nfunc (r *Reader) feed(src io.Reader) {\n\t/*\n\t\treaderSlabSize, ae := strconv.Atoi(os.Getenv(\"SLAB_KB\"))\n\t\tif ae != nil {\n\t\t\treaderSlabSize = 128 * 1024\n\t\t} else {\n\t\t\treaderSlabSize *= 1024\n\t\t}\n\t\treaderBufferSize, be := strconv.Atoi(os.Getenv(\"BUF_KB\"))\n\t\tif be != nil {\n\t\t\treaderBufferSize = 64 * 1024\n\t\t} else {\n\t\t\treaderBufferSize *= 1024\n\t\t}\n\t*/\n\n\tdelim := byte('\\n')\n\ttrimCR := util.IsWindows()\n\tif r.delimNil {\n\t\tdelim = '\\000'\n\t\ttrimCR = false\n\t}\n\n\tslab := make([]byte, readerSlabSize)\n\tleftover := []byte{}\n\tvar err error\n\tfor {\n\t\tn := 0\n\t\tscope := slab[:util.Min(len(slab), readerBufferSize)]\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tn, err = src.Read(scope)\n\t\t\tif n > 0 || err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// We're not making any progress after 100 tries. Stop.\n\t\tif n == 0 && err == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tbuf := slab[:n]\n\t\tslab = slab[n:]\n\n\t\tfor len(buf) > 0 {\n\t\t\tif i := bytes.IndexByte(buf, delim); i >= 0 {\n\t\t\t\t// Found the delimiter\n\t\t\t\tslice := buf[:i+1]\n\t\t\t\tbuf = buf[i+1:]\n...\n// Path: src/terminal.go\npackage fzf\n\nimport (\n\t\"bufio\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/rivo/uniseg\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// import \"github.com/pkg/profile\"\n\n/*\nPlaceholder regex is used to extract placeholders from fzf's template\nstrings. Acts as input validation for parsePlaceholder function.\nDescribes the syntax, but it is fairly lenient.\n\nThe following pseudo regex has been reverse engineered from the\nimplementation. It is overly strict, but better describes what's possible.\nAs such it is not useful for validation, but rather to generate test\ncases for example.\n\n\t\\\\?(?:                                      # escaped type\n\t    {\\+?s?f?RANGE(?:,RANGE)*}               # token type\n\t    |{q}                                    # query type\n\t    |{\\+?n?f?}                              # item type (notice no mandatory element inside brackets)\n\t)\n\tRANGE = (?:\n\t    (?:-?[0-9]+)?\\.\\.(?:-?[0-9]+)?          # ellipsis syntax for token range (x..y)\n\t    |-?[0-9]+                               # shorthand syntax (x..x)\n\t)\n*/\nvar placeholder *regexp.Regexp\nvar whiteSuffix *regexp.Regexp\nvar offsetComponentRegex *regexp.Regexp\nvar offsetTrimCharsRegex *regexp.Regexp\nvar activeTempFiles []string\nvar passThroughRegex *regexp.Regexp\nvar actionTypeRegex *regexp.Regexp\n\nconst clearCode string = \"\\x1b[2J\"\n\n// Number of maximum focus events to process synchronously\nconst maxFocusEvents = 10000\n\nfunc init() {\n\tplaceholder = regexp.MustCompile(`\\\\?(?:{[+sf]*[0-9,-.]*}|{q}|{fzf:(?:query|action|prompt)}|{\\+?f?nf?})`)\n\twhiteSuffix = regexp.MustCompile(`\\s*$`)\n\toffsetComponentRegex = regexp.MustCompile(`([+-][0-9]+)|(-?/[1-9][0-9]*)`)\n\toffsetTrimCharsRegex = regexp.MustCompile(`[^0-9/+-]`)\n\tactiveTempFiles = []string{}\n\n\t// Parts of the preview output that should be passed through to the terminal\n\t// * https://github.com/tmux/tmux/wiki/FAQ#what-is-the-passthrough-escape-sequence-and-how-do-i-use-it\n\t// * https://sw.kovidgoyal.net/kitty/graphics-protocol\n\t// * https://en.wikipedia.org/wiki/Sixel\n\t// * https://iterm2.com/documentation-images.html\n\tpassThroughRegex = regexp.MustCompile(`\\x1bPtmux;\\x1b\\x1b.*?[^\\x1b]\\x1b\\\\|\\x1b(_G|P[0-9;]*q).*?\\x1b\\\\\\r?|\\x1b]1337;.*?(\\a|\\x1b\\\\)`)\n}\n\ntype jumpMode int\n\nconst (\n\tjumpDisabled jumpMode = iota\n\tjumpEnabled\n\tjumpAcceptEnabled\n)\n\ntype resumableState int\n\nconst (\n\tdisabledState resumableState = iota\n\tpausedState\n\tenabledState\n)\n\nfunc (s resumableState) Enabled() bool {\n\treturn s == enabledState\n}\n\nfunc (s *resumableState) Force(flag bool) {\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = disabledState\n\t}\n}\n\nfunc (s *resumableState) Set(flag bool) {\n\tif *s == disabledState {\n\t\treturn\n\t}\n\n\tif flag {\n\t\t*s = enabledState\n\t} else {\n\t\t*s = pausedState\n\t}\n}\n\ntype previewer struct {\n\tversion    int64\n\tlines      []string\n\toffset     int\n\tscrollable bool\n\tfinal      bool\n\tfollowing  resumableState\n\tspinner    string\n\tbar        []bool\n}\n\ntype previewed struct {\n\tversion   int64\n\tnumLines  int\n\toffset    int\n\tfilled    bool\n\timage     bool\n\twipe      bool\n\twireframe bool\n}\n\ntype eachLine struct {\n\tline string\n\terr  error\n}\n\ntype itemLine struct {\n\toffset   int\n\tcurrent  bool\n\tselected bool\n\tlabel    string\n\tqueryLen int\n\twidth    int\n\tbar      bool\n\tresult   Result\n}\n\ntype fitpad struct {\n\tfit int\n\tpad int\n}\n\nvar emptyLine = itemLine{}\n\ntype labelPrinter func(tui.Window, int)\n\ntype StatusItem struct {\n\tIndex int    `json:\"index\"`\n\tText  string `json:\"text\"`\n}\n\ntype Status struct {\n\tReading    bool         `json:\"reading\"`\n\tProgress   int          `json:\"progress\"`\n\tQuery      string       `json:\"query\"`\n\tPosition   int          `json:\"position\"`\n\tSort       bool         `json:\"sort\"`\n\tTotalCount int          `json:\"totalCount\"`\n\tMatchCount int          `json:\"matchCount\"`\n\tCurrent    *StatusItem  `json:\"current\"`\n\tMatches    []StatusItem `json:\"matches\"`\n\tSelected   []StatusItem `json:\"selected\"`\n}\n\n// Terminal represents terminal input/output\ntype Terminal struct {\n\tinitDelay          time.Duration\n\tinfoStyle          infoStyle\n\tinfoPrefix         string\n\tseparator          labelPrinter\n\tseparatorLen       int\n\tspinner            []string\n\tpromptString       string\n\tprompt             func()\n\tpromptLen          int\n\tborderLabel        labelPrinter\n\tborderLabelLen     int\n\tborderLabelOpts    labelOpts\n\tpreviewLabel       labelPrinter\n\tpreviewLabelLen    int\n\tpreviewLabelOpts   labelOpts\n\tpointer            string\n\tpointerLen         int\n\tpointerEmpty       string\n\tmarker             string\n\tmarkerLen          int\n\tmarkerEmpty        string\n\tqueryLen           [2]int\n\tlayout             layoutType\n\tfullscreen         bool\n\tkeepRight          bool\n\thscroll            bool\n\thscrollOff         int\n\tscrollOff          int\n\twordRubout         string\n\twordNext           string\n\tcx                 int\n\tcy                 int\n\toffset             int\n\txoffset            int\n\tyanked             []rune\n\tinput              []rune\n\tmulti              int\n\tsort               bool\n\ttoggleSort         bool\n\ttrack              trackOption\n\tdelimiter          Delimiter\n\texpect             map[tui.Event]string\n\tkeymap             map[tui.Event][]*action\n\tkeymapOrg          map[tui.Event][]*action\n\tpressed            string\n\tprintQuery         bool\n\thistory            *History\n\tcycle              bool\n\theaderVisible      bool\n\theaderFirst        bool\n\theaderLines        int\n\theader             []string\n\theader0            []string\n\tellipsis           string\n\tscrollbar          string\n\tpreviewScrollbar   string\n\tansi               bool\n\ttabstop            int\n\tmargin             [4]sizeSpec\n\tpadding            [4]sizeSpec\n\tunicode            bool\n\tlistenAddr         *listenAddress\n\tlistenPort         *int\n\tlistenUnsafe       bool\n\tborderShape        tui.BorderShape\n\tcleanExit          bool\n\tpaused             bool\n\tborder             tui.Window\n\twindow             tui.Window\n\tpborder            tui.Window\n\tpwindow            tui.Window\n\tborderWidth        int\n\tcount              int\n\tprogress           int\n\thasResultActions   bool\n\thasFocusActions    bool\n\thasLoadActions     bool\n\thasResizeActions   bool\n\ttriggerLoad        bool\n\treading            bool\n\trunning            bool\n\tfailed             *string\n\tjumping            jumpMode\n\tjumpLabels         string\n\tprinter            func(string)\n\tprintsep           string\n\tmerger             *Merger\n\tselected           map[int32]selectedItem\n\tversion            int64\n\trevision           int\n\treqBox             *util.EventBox\n\tinitialPreviewOpts previewOpts\n\tpreviewOpts        previewOpts\n\tactivePreviewOpts  *previewOpts\n\tpreviewer          previewer\n\tpreviewed          previewed\n\tpreviewBox         *util.EventBox\n\teventBox           *util.EventBox\n\tmutex              sync.Mutex\n\tinitFunc           func()\n\tprevLines          []itemLine\n\tsuppress           bool\n\tsigstop            bool\n\tstartChan          chan fitpad\n\tkillChan           chan int\n\tserverInputChan    chan []*action\n\tserverOutputChan   chan string\n\teventChan          chan tui.Event\n\tslab               *util.Slab\n\ttheme              *tui.ColorTheme\n\ttui                tui.Renderer\n\texecuting          *util.AtomicBool\n\ttermSize           tui.TermSize\n\tlastAction         actionType\n\tlastKey            string\n\tlastFocus          int32\n\tareaLines          int\n\tareaColumns        int\n\tforcePreview       bool\n}\n\ntype selectedItem struct {\n\tat   time.Time\n\titem *Item\n}\n\ntype byTimeOrder []selectedItem\n\nfunc (a byTimeOrder) Len() int {\n\treturn len(a)\n}\n\nfunc (a byTimeOrder) Swap(i, j int) {\n\ta[i], a[j] = a[j], a[i]\n}\n\nfunc (a byTimeOrder) Less(i, j int) bool {\n\treturn a[i].at.Before(a[j].at)\n}\n\nconst (\n\treqPrompt util.EventType = iota\n\treqInfo\n\treqHeader\n\treqList\n\treqJump\n\treqRefresh\n\treqReinit\n\treqFullRedraw\n\treqResize\n\treqRedrawBorderLabel\n\treqRedrawPreviewLabel\n\treqClose\n\treqPrintQuery\n\treqPreviewEnqueue\n\treqPreviewDisplay\n\treqPreviewRefresh\n\treqPreviewDelayed\n\treqQuit\n)\n\ntype action struct {\n\tt actionType\n\ta string\n}\n\n//go:generate stringer -type=actionType\ntype actionType int\n\nconst (\n\tactIgnore actionType = iota\n\tactStart\n\tactClick\n\tactInvalid\n\tactChar\n\tactMouse\n\tactBeginningOfLine\n\tactAbort\n\tactAccept\n\tactAcceptNonEmpty\n\tactAcceptOrPrintQuery\n\tactBackwardChar\n\tactBackwardDeleteChar\n\tactBackwardDeleteCharEof\n\tactBackwardWord\n\tactCancel\n\tactChangeBorderLabel\n\tactChangeHeader\n\tactChangePreviewLabel\n\tactChangePrompt\n\tactChangeQuery\n\tactClearScreen\n\tactClearQuery\n\tactClearSelection\n\tactClose\n\tactDeleteChar\n\tactDeleteCharEof\n\tactEndOfLine\n\tactForwardChar\n\tactForwardWord\n\tactKillLine\n\tactKillWord\n\tactUnixLineDiscard\n\tactUnixWordRubout\n\tactYank\n\tactBackwardKillWord\n\tactSelectAll\n\tactDeselectAll\n\tactToggle\n\tactToggleSearch\n\tactToggleAll\n\tactToggleDown\n\tactToggleUp\n\tactToggleIn\n\tactToggleOut\n\tactToggleTrack\n\tactToggleTrackCurrent\n\tactToggleHeader\n\tactTrackCurrent\n\tactUntrackCurrent\n\tactDown\n\tactUp\n\tactPageUp\n\tactPageDown\n\tactPosition\n\tactHalfPageUp\n\tactHalfPageDown\n\tactOffsetUp\n\tactOffsetDown\n\tactJump\n\tactJumpAccept // XXX Deprecated in favor of jump:accept binding\n\tactPrintQuery\n\tactRefreshPreview\n\tactReplaceQuery\n\tactToggleSort\n\tactShowPreview\n\tactHidePreview\n\tactTogglePreview\n\tactTogglePreviewWrap\n\tactTransform\n\tactTransformBorderLabel\n\tactTransformHeader\n\tactTransformPreviewLabel\n\tactTransformPrompt\n\tactTransformQuery\n\tactPreview\n\tactChangePreview\n\tactChangePreviewWindow\n\tactPreviewTop\n\tactPreviewBottom\n\tactPreviewUp\n\tactPreviewDown\n\tactPreviewPageUp\n\tactPreviewPageDown\n\tactPreviewHalfPageUp\n\tactPreviewHalfPageDown\n\tactPrevHistory\n\tactPrevSelected\n\tactPut\n\tactNextHistory\n\tactNextSelected\n\tactExecute\n\tactExecuteSilent\n\tactExecuteMulti // Deprecated\n\tactSigStop\n\tactFirst\n\tactLast\n\tactReload\n\tactReloadSync\n\tactDisableSearch\n\tactEnableSearch\n\tactSelect\n\tactDeselect\n\tactUnbind\n\tactRebind\n\tactBecome\n\tactResponse\n\tactShowHeader\n\tactHideHeader\n)\n\nfunc (a actionType) Name() string {\n\treturn util.ToKebabCase(a.String()[3:])\n}\n\nfunc processExecution(action actionType) bool {\n\tswitch action {\n\tcase actTransform,\n\t\tactTransformBorderLabel,\n\t\tactTransformHeader,\n\t\tactTransformPreviewLabel,\n\t\tactTransformPrompt,\n\t\tactTransformQuery,\n\t\tactPreview,\n\t\tactChangePreview,\n\t\tactRefreshPreview,\n\t\tactExecute,\n\t\tactExecuteSilent,\n\t\tactExecuteMulti,\n\t\tactReload,\n\t\tactReloadSync,\n\t\tactBecome:\n\t\treturn true\n\t}\n\treturn false\n}\n\ntype placeholderFlags struct {\n\tplus          bool\n\tpreserveSpace bool\n\tnumber        bool\n\tforceUpdate   bool\n\tfile          bool\n}\n\ntype searchRequest struct {\n\tsort    bool\n\tsync    bool\n\tcommand *string\n\tenviron []string\n\tchanged bool\n}\n\ntype previewRequest struct {\n\ttemplate     string\n\tpwindow      tui.Window\n\tpwindowSize  tui.TermSize\n\tscrollOffset int\n\tlist         []*Item\n}\n\ntype previewResult struct {\n\tversion int64\n\tlines   []string\n\toffset  int\n\tspinner string\n}\n\nfunc toActions(types ...actionType) []*action {\n\tactions := make([]*action, len(types))\n\tfor idx, t := range types {\n\t\tactions[idx] = &action{t: t, a: \"\"}\n\t}\n\treturn actions\n}\n\nfunc defaultKeymap() map[tui.Event][]*action {\n\tkeymap := make(map[tui.Event][]*action)\n\tadd := func(e tui.EventType, a actionType) {\n\t\tkeymap[e.AsEvent()] = toActions(a)\n\t}\n\taddEvent := func(e tui.Event, a actionType) {\n\t\tkeymap[e] = toActions(a)\n\t}\n\n\tadd(tui.Invalid, actInvalid)\n\tadd(tui.CtrlA, actBeginningOfLine)\n\tadd(tui.CtrlB, actBackwardChar)\n\tadd(tui.CtrlC, actAbort)\n\tadd(tui.CtrlG, actAbort)\n\tadd(tui.CtrlQ, actAbort)\n\tadd(tui.Esc, actAbort)\n\tadd(tui.CtrlD, actDeleteCharEof)\n\tadd(tui.CtrlE, actEndOfLine)\n\tadd(tui.CtrlF, actForwardChar)\n\tadd(tui.CtrlH, actBackwardDeleteChar)\n\tadd(tui.Backspace, actBackwardDeleteChar)\n\tadd(tui.Tab, actToggleDown)\n\tadd(tui.ShiftTab, actToggleUp)\n\tadd(tui.CtrlJ, actDown)\n\tadd(tui.CtrlK, actUp)\n\tadd(tui.CtrlL, actClearScreen)\n\tadd(tui.CtrlM, actAccept)\n\tadd(tui.CtrlN, actDown)\n\tadd(tui.CtrlP, actUp)\n\tadd(tui.CtrlU, actUnixLineDiscard)\n\tadd(tui.CtrlW, actUnixWordRubout)\n\tadd(tui.CtrlY, actYank)\n\tif !util.IsWindows() {\n\t\tadd(tui.CtrlZ, actSigStop)\n\t}\n\n\taddEvent(tui.AltKey('b'), actBackwardWord)\n\tadd(tui.ShiftLeft, actBackwardWord)\n\taddEvent(tui.AltKey('f'), actForwardWord)\n\tadd(tui.ShiftRight, actForwardWord)\n\taddEvent(tui.AltKey('d'), actKillWord)\n\tadd(tui.AltBackspace, actBackwardKillWord)\n\n\tadd(tui.Up, actUp)\n\tadd(tui.Down, actDown)\n\tadd(tui.Left, actBackwardChar)\n\tadd(tui.Right, actForwardChar)\n\n\tadd(tui.Home, actBeginningOfLine)\n\tadd(tui.End, actEndOfLine)\n\tadd(tui.Delete, actDeleteChar)\n\tadd(tui.PageUp, actPageUp)\n\tadd(tui.PageDown, actPageDown)\n\n\tadd(tui.ShiftUp, actPreviewUp)\n\tadd(tui.ShiftDown, actPreviewDown)\n\n\tadd(tui.Mouse, actMouse)\n\tadd(tui.LeftClick, actClick)\n\tadd(tui.RightClick, actToggle)\n\tadd(tui.SLeftClick, actToggle)\n\tadd(tui.SRightClick, actToggle)\n\n\tadd(tui.ScrollUp, actUp)\n\tadd(tui.ScrollDown, actDown)\n\tkeymap[tui.SScrollUp.AsEvent()] = toActions(actToggle, actUp)\n\tkeymap[tui.SScrollDown.AsEvent()] = toActions(actToggle, actDown)\n\n\tadd(tui.PreviewScrollUp, actPreviewUp)\n\tadd(tui.PreviewScrollDown, actPreviewDown)\n\treturn keymap\n}\n\nfunc trimQuery(query string) []rune {\n\treturn []rune(strings.Replace(query, \"\\t\", \" \", -1))\n}\n\nfunc mayTriggerPreview(opts *Options) bool {\n\tif opts.ListenAddr != nil {\n\t\treturn true\n\t}\n\tfor _, actions := range opts.Keymap {\n\t\tfor _, action := range actions {\n\t\t\tswitch action.t {\n\t\t\tcase actPreview, actChangePreview, actTransform:\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc makeSpinner(unicode bool) []string {\n\tif unicode {\n\t\treturn []string{`\u280b`, `\u2819`, `\u2839`, `\u2838`, `\u283c`, `\u2834`, `\u2826`, `\u2827`, `\u2807`, `\u280f`}\n\t}\n\treturn []string{`-`, `\\`, `|`, `/`, `-`, `\\`, `|`, `/`}\n}\n\nfunc evaluateHeight(opts *Options, termHeight int) int {\n\tsize := opts.Height.size\n\tif opts.Height.percent {\n\t\tif opts.Height.inverse {\n\t\t\tsize = 100 - size\n\t\t}\n\t\treturn util.Max(int(size*float64(termHeight)/100.0), opts.MinHeight)\n\t}\n\tif opts.Height.inverse {\n\t\tsize = float64(termHeight) - size\n\t}\n\treturn int(size)\n}\n\n// NewTerminal returns new Terminal object\nfunc NewTerminal(opts *Options, eventBox *util.EventBox) *Terminal {\n\tinput := trimQuery(opts.Query)\n\tvar delay time.Duration\n\tif opts.Tac {\n\t\tdelay = initialDelayTac\n\t} else {\n\t\tdelay = initialDelay\n\t}\n\tvar previewBox *util.EventBox\n\t// We need to start the previewer even when --preview option is not specified\n\t// * if HTTP server is enabled\n\t// * if 'preview' or 'change-preview' action is bound to a key\n\t// * if 'transform' action is bound to a key\n\tif len(opts.Preview.command) > 0 || mayTriggerPreview(opts) {\n\t\tpreviewBox = util.NewEventBox()\n\t}\n\tvar renderer tui.Renderer\n\tfullscreen := !opts.Height.auto && (opts.Height.size == 0 || opts.Height.percent && opts.Height.size == 100)\n\tif fullscreen {\n\t\tif tui.HasFullscreenRenderer() {\n\t\t\trenderer = tui.NewFullscreenRenderer(opts.Theme, opts.Black, opts.Mouse)\n\t\t} else {\n\t\t\trenderer = tui.NewLightRenderer(opts.Theme, opts.Black, opts.Mouse, opts.Tabstop, opts.ClearOnExit,\n\t\t\t\ttrue, func(h int) int { return h })\n\t\t}\n\t} else {\n\t\tmaxHeightFunc := func(termHeight int) int {\n\t\t\t// Minimum height required to render fzf excluding margin and padding\n\t\t\teffectiveMinHeight := minHeight\n\t\t\tif previewBox != nil && opts.Preview.aboveOrBelow() {\n\t\t\t\teffectiveMinHeight += 1 + borderLines(opts.Preview.border)\n\t\t\t}\n\t\t\tif noSeparatorLine(opts.InfoStyle, opts.Separator == nil || uniseg.StringWidth(*opts.Separator) > 0) {\n\t\t\t\teffectiveMinHeight--\n\t\t\t}\n\t\t\teffectiveMinHeight += borderLines(opts.BorderShape)\n\t\t\treturn util.Min(termHeight, util.Max(evaluateHeight(opts, termHeight), effectiveMinHeight))\n\t\t}\n\t\trenderer = tui.NewLightRenderer(opts.Theme, opts.Black, opts.Mouse, opts.Tabstop, opts.ClearOnExit, false, maxHeightFunc)\n\t}\n\twordRubout := \"[^\\\\pL\\\\pN][\\\\pL\\\\pN]\"\n\twordNext := \"[\\\\pL\\\\pN][^\\\\pL\\\\pN]|(.$)\"\n\tif opts.FileWord {\n\t\tsep := regexp.QuoteMeta(string(os.PathSeparator))\n\t\twordRubout = fmt.Sprintf(\"%s[^%s]\", sep, sep)\n\t\twordNext = fmt.Sprintf(\"[^%s]%s|(.$)\", sep, sep)\n\t}\n\tkeymapCopy := make(map[tui.Event][]*action)\n\tfor key, action := range opts.Keymap {\n\t\tkeymapCopy[key] = action\n\t}\n\tt := Terminal{\n\t\tinitDelay:          delay,\n\t\tinfoStyle:          opts.InfoStyle,\n\t\tinfoPrefix:         opts.InfoPrefix,\n\t\tseparator:          nil,\n\t\tspinner:            makeSpinner(opts.Unicode),\n\t\tpromptString:       opts.Prompt,\n\t\tqueryLen:           [2]int{0, 0},\n\t\tlayout:             opts.Layout,\n\t\tfullscreen:         fullscreen,\n\t\tkeepRight:          opts.KeepRight,\n\t\thscroll:            opts.Hscroll,\n\t\thscrollOff:         opts.HscrollOff,\n\t\tscrollOff:          opts.ScrollOff,\n\t\twordRubout:         wordRubout,\n\t\twordNext:           wordNext,\n\t\tcx:                 len(input),\n\t\tcy:                 0,\n\t\toffset:             0,\n\t\txoffset:            0,\n\t\tyanked:             []rune{},\n\t\tinput:              input,\n\t\tmulti:              opts.Multi,\n\t\tsort:               opts.Sort > 0,\n\t\ttoggleSort:         opts.ToggleSort,\n\t\ttrack:              opts.Track,\n\t\tdelimiter:          opts.Delimiter,\n\t\texpect:             opts.Expect,\n\t\tkeymap:             opts.Keymap,\n\t\tkeymapOrg:          keymapCopy,\n\t\tpressed:            \"\",\n\t\tprintQuery:         opts.PrintQuery,\n\t\thistory:            opts.History,\n\t\tmargin:             opts.Margin,\n\t\tpadding:            opts.Padding,\n\t\tunicode:            opts.Unicode,\n\t\tlistenAddr:         opts.ListenAddr,\n\t\tlistenUnsafe:       opts.Unsafe,\n\t\tborderShape:        opts.BorderShape,\n\t\tborderWidth:        1,\n\t\tborderLabel:        nil,\n\t\tborderLabelOpts:    opts.BorderLabel,\n\t\tpreviewLabel:       nil,\n\t\tpreviewLabelOpts:   opts.PreviewLabel,\n\t\tcleanExit:          opts.ClearOnExit,\n\t\tpaused:             opts.Phony,\n\t\tcycle:              opts.Cycle,\n\t\theaderVisible:      true,\n\t\theaderFirst:        opts.HeaderFirst,\n\t\theaderLines:        opts.HeaderLines,\n\t\theader:             []string{},\n\t\theader0:            opts.Header,\n\t\tellipsis:           opts.Ellipsis,\n\t\tansi:               opts.Ansi,\n\t\ttabstop:            opts.Tabstop,\n\t\thasResultActions:   false,\n\t\thasFocusActions:    false,\n\t\thasLoadActions:     false,\n\t\ttriggerLoad:        false,\n\t\treading:            true,\n\t\trunning:            true,\n\t\tfailed:             nil,\n\t\tjumping:            jumpDisabled,\n\t\tjumpLabels:         opts.JumpLabels,\n\t\tprinter:            opts.Printer,\n\t\tprintsep:           opts.PrintSep,\n\t\tmerger:             EmptyMerger(0),\n\t\tselected:           make(map[int32]selectedItem),\n\t\treqBox:             util.NewEventBox(),\n\t\tinitialPreviewOpts: opts.Preview,\n\t\tpreviewOpts:        opts.Preview,\n\t\tpreviewer:          previewer{0, []string{}, 0, false, true, disabledState, \"\", []bool{}},\n\t\tpreviewed:          previewed{0, 0, 0, false, false, false, false},\n\t\tpreviewBox:         previewBox,\n\t\teventBox:           eventBox,\n\t\tmutex:              sync.Mutex{},\n\t\tsuppress:           true,\n\t\tsigstop:            false,\n\t\tslab:               util.MakeSlab(slab16Size, slab32Size),\n\t\ttheme:              opts.Theme,\n\t\tstartChan:          make(chan fitpad, 1),\n\t\tkillChan:           make(chan int),\n\t\tserverInputChan:    make(chan []*action, 100),\n\t\tserverOutputChan:   make(chan string),\n\t\teventChan:          make(chan tui.Event, 6), // (load + result + zero|one) | (focus) | (resize) | (GetChar)\n\t\ttui:                renderer,\n\t\tinitFunc:           func() { renderer.Init() },\n\t\texecuting:          util.NewAtomicBool(false),\n\t\tlastAction:         actStart,\n\t\tlastFocus:          minItem.Index()}\n\tt.prompt, t.promptLen = t.parsePrompt(opts.Prompt)\n\tt.pointer, t.pointerLen = t.processTabs([]rune(opts.Pointer), 0)\n\tt.marker, t.markerLen = t.processTabs([]rune(opts.Marker), 0)\n\t// Pre-calculated empty pointer and marker signs\n\tt.pointerEmpty = strings.Repeat(\" \", t.pointerLen)\n\tt.markerEmpty = strings.Repeat(\" \", t.markerLen)\n\tt.borderLabel, t.borderLabelLen = t.ansiLabelPrinter(opts.BorderLabel.label, &tui.ColBorderLabel, false)\n\tt.previewLabel, t.previewLabelLen = t.ansiLabelPrinter(opts.PreviewLabel.label, &tui.ColPreviewLabel, false)\n\tif opts.Separator == nil || len(*opts.Separator) > 0 {\n\t\tbar := \"\u2500\"\n\t\tif opts.Separator != nil {\n\t\t\tbar = *opts.Separator\n\t\t} else if !t.unicode {\n\t\t\tbar = \"-\"\n\t\t}\n\t\tt.separator, t.separatorLen = t.ansiLabelPrinter(bar, &tui.ColSeparator, true)\n\t}\n\tif t.unicode {\n\t\tt.borderWidth = uniseg.StringWidth(\"\u2502\")\n\t}\n\tif opts.Scrollbar == nil {\n\t\tif t.unicode && t.borderWidth == 1 {\n\t\t\tt.scrollbar = \"\u2502\"\n\t\t} else {\n\t\t\tt.scrollbar = \"|\"\n\t\t}\n\t\tt.previewScrollbar = t.scrollbar\n\t} else {\n\t\trunes := []rune(*opts.Scrollbar)\n\t\tif len(runes) > 0 {\n\t\t\tt.scrollbar = string(runes[0])\n\t\t\tt.previewScrollbar = t.scrollbar\n\t\t\tif len(runes) > 1 {\n\t\t\t\tt.previewScrollbar = string(runes[1])\n\t\t\t}\n\t\t}\n\t}\n\n\tvar resizeActions []*action\n\tresizeActions, t.hasResizeActions = t.keymap[tui.Resize.AsEvent()]\n\tif t.tui.ShouldEmitResizeEvent() {\n\t\tt.keymap[tui.Resize.AsEvent()] = append(toActions(actClearScreen), resizeActions...)\n\t}\n\t_, t.hasResultActions = t.keymap[tui.Result.AsEvent()]\n\t_, t.hasFocusActions = t.keymap[tui.Focus.AsEvent()]\n\t_, t.hasLoadActions = t.keymap[tui.Load.AsEvent()]\n\n\tif t.listenAddr != nil {\n\t\tport, err := startHttpServer(*t.listenAddr, t.serverInputChan, t.serverOutputChan)\n\t\tif err != nil {\n\t\t\terrorExit(err.Error())\n\t\t}\n\t\tt.listenPort = &port\n\t}\n\n\treturn &t\n}\n\nfunc (t *Terminal) environ() []string {\n\tenv := os.Environ()\n\tif t.listenPort != nil {\n\t\tenv = append(env, fmt.Sprintf(\"FZF_PORT=%d\", *t.listenPort))\n\t}\n\tenv = append(env, \"FZF_QUERY=\"+string(t.input))\n\tenv = append(env, \"FZF_ACTION=\"+t.lastAction.Name())\n\tenv = append(env, \"FZF_KEY=\"+t.lastKey)\n\tenv = append(env, \"FZF_PROMPT=\"+string(t.promptString))\n\tenv = append(env, \"FZF_PREVIEW_LABEL=\"+t.previewLabelOpts.label)\n\tenv = append(env, \"FZF_BORDER_LABEL=\"+t.borderLabelOpts.label)\n\tenv = append(env, fmt.Sprintf(\"FZF_TOTAL_COUNT=%d\", t.count))\n\tenv = append(env, fmt.Sprintf(\"FZF_MATCH_COUNT=%d\", t.merger.Length()))\n\tenv = append(env, fmt.Sprintf(\"FZF_SELECT_COUNT=%d\", len(t.selected)))\n\tenv = append(env, fmt.Sprintf(\"FZF_LINES=%d\", t.areaLines))\n\tenv = append(env, fmt.Sprintf(\"FZF_COLUMNS=%d\", t.areaColumns))\n\treturn env\n}\n\nfunc borderLines(shape tui.BorderShape) int {\n\tswitch shape {\n\tcase tui.BorderHorizontal, tui.BorderRounded, tui.BorderSharp, tui.BorderBold, tui.BorderBlock, tui.BorderThinBlock, tui.BorderDouble:\n\t\treturn 2\n\tcase tui.BorderTop, tui.BorderBottom:\n\t\treturn 1\n\t}\n\treturn 0\n}\n\nfunc (t *Terminal) visibleHeaderLines() int {\n\tif !t.headerVisible {\n\t\treturn 0\n\t}\n\treturn len(t.header0) + t.headerLines\n}\n\n// Extra number of lines needed to display fzf\nfunc (t *Terminal) extraLines() int {\n\textra := t.visibleHeaderLines() + 1\n\tif !t.noSeparatorLine() {\n\t\textra++\n\t}\n\treturn extra\n}\n\nfunc (t *Terminal) MaxFitAndPad() (int, int) {\n\t_, screenHeight, marginInt, paddingInt := t.adjustMarginAndPadding()\n\tpadHeight := marginInt[0] + marginInt[2] + paddingInt[0] + paddingInt[2]\n\tfit := screenHeight - padHeight - t.extraLines()\n\treturn fit, padHeight\n}\n\nfunc (t *Terminal) ansiLabelPrinter(str string, color *tui.ColorPair, fill bool) (labelPrinter, int) {\n\t// Nothing to do\n\tif len(str) == 0 {\n\t\treturn nil, 0\n\t}\n\n\t// Extract ANSI color codes\n\tstr = firstLine(str)\n\ttext, colors, _ := extractColor(str, nil, nil)\n\trunes := []rune(text)\n\n\t// Simpler printer for strings without ANSI colors or tab characters\n\tif colors == nil && !strings.ContainsRune(str, '\\t') {\n\t\tlength := util.StringWidth(str)\n\t\tif length == 0 {\n\t\t\treturn nil, 0\n\t\t}\n\t\tprintFn := func(window tui.Window, limit int) {\n\t\t\tif length > limit {\n\t\t\t\ttrimmedRunes, _ := t.trimRight(runes, limit)\n\t\t\t\twindow.CPrint(*color, string(trimmedRunes))\n\t\t\t} else if fill {\n\t\t\t\twindow.CPrint(*color, util.RepeatToFill(str, length, limit))\n\t\t\t} else {\n\t\t\t\twindow.CPrint(*color, str)\n\t\t\t}\n\t\t}\n\t\treturn printFn, length\n\t}\n\n\t// Printer that correctly handles ANSI color codes and tab characters\n\titem := &Item{text: util.RunesToChars(runes), colors: colors}\n\tlength := t.displayWidth(runes)\n\tif length == 0 {\n\t\treturn nil, 0\n\t}\n\tresult := Result{item: item}\n\tvar offsets []colorOffset\n\tprintFn := func(window tui.Window, limit int) {\n\t\tif offsets == nil {\n\t\t\t// tui.Col* are not initialized until renderer.Init()\n\t\t\toffsets = result.colorOffsets(nil, t.theme, *color, *color, false)\n\t\t}\n\t\tfor limit > 0 {\n\t\t\tif length > limit {\n\t\t\t\ttrimmedRunes, _ := t.trimRight(runes, limit)\n\t\t\t\tt.printColoredString(window, trimmedRunes, offsets, *color)\n\t\t\t\tbreak\n\t\t\t} else if fill {\n\t\t\t\tt.printColoredString(window, runes, offsets, *color)\n\t\t\t\tlimit -= length\n\t\t\t} else {\n\t\t\t\tt.printColoredString(window, runes, offsets, *color)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn printFn, length\n}\n\nfunc (t *Terminal) parsePrompt(prompt string) (func(), int) {\n\tvar state *ansiState\n\tprompt = firstLine(prompt)\n\ttrimmed, colors, _ := extractColor(prompt, state, nil)\n\titem := &Item{text: util.ToChars([]byte(trimmed)), colors: colors}\n\n\t// \"Prompt>  \"\n\t//  -------    // Do not apply ANSI attributes to the trailing whitespaces\n\t//             // unless the part has a non-default ANSI state\n\tloc := whiteSuffix.FindStringIndex(trimmed)\n\tif loc != nil {\n\t\tblankState := ansiOffset{[2]int32{int32(loc[0]), int32(loc[1])}, ansiState{-1, -1, tui.AttrClear, -1}}\n\t\tif item.colors != nil {\n\t\t\tlastColor := (*item.colors)[len(*item.colors)-1]\n\t\t\tif lastColor.offset[1] < int32(loc[1]) {\n\t\t\t\tblankState.offset[0] = lastColor.offset[1]\n\t\t\t\tcolors := append(*item.colors, blankState)\n\t\t\t\titem.colors = &colors\n\t\t\t}\n\t\t} else {\n\t\t\tcolors := []ansiOffset{blankState}\n\t\t\titem.colors = &colors\n\t\t}\n\t}\n\toutput := func() {\n\t\tt.printHighlighted(\n\t\t\tResult{item: item}, tui.ColPrompt, tui.ColPrompt, false, false)\n\t}\n\t_, promptLen := t.processTabs([]rune(trimmed), 0)\n\n\treturn output, promptLen\n}\n\nfunc noSeparatorLine(style infoStyle, separator bool) bool {\n\tswitch style {\n\tcase infoInline:\n\t\treturn true\n\tcase infoHidden, infoInlineRight:\n\t\treturn !separator\n\t}\n\treturn false\n}\n\nfunc (t *Terminal) noSeparatorLine() bool {\n\treturn noSeparatorLine(t.infoStyle, t.separatorLen > 0)\n}\n\nfunc getScrollbar(total int, height int, offset int) (int, int) {\n\tif total == 0 || total <= height {\n\t\treturn 0, 0\n\t}\n\tbarLength := util.Max(1, height*height/total)\n\tvar barStart int\n\tif total == height {\n\t\tbarStart = 0\n\t} else {\n\t\tbarStart = (height - barLength) * offset / (total - height)\n\t}\n\treturn barLength, barStart\n}\n\nfunc (t *Terminal) getScrollbar() (int, int) {\n\treturn getScrollbar(t.merger.Length(), t.maxItems(), t.offset)\n}\n\n// Input returns current query string\nfunc (t *Terminal) Input() (bool, []rune) {\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\treturn t.paused, copySlice(t.input)\n}\n\n// UpdateCount updates the count information\nfunc (t *Terminal) UpdateCount(cnt int, final bool, failedCommand *string) {\n\tt.mutex.Lock()\n\tt.count = cnt\n\tif t.hasLoadActions && t.reading && final {\n\t\tt.triggerLoad = true\n\t}\n\tt.reading = !final\n\tt.failed = failedCommand\n\tt.mutex.Unlock()\n\tt.reqBox.Set(reqInfo, nil)\n\tif final {\n\t\tt.reqBox.Set(reqRefresh, nil)\n\t}\n}\n\nfunc (t *Terminal) changeHeader(header string) bool {\n\tlines := strings.Split(strings.TrimSuffix(header, \"\\n\"), \"\\n\")\n\tneedFullRedraw := len(t.header0) != len(lines)\n\tt.header0 = lines\n\treturn needFullRedraw\n}\n\n// UpdateHeader updates the header\nfunc (t *Terminal) UpdateHeader(header []string) {\n\tt.mutex.Lock()\n\tt.header = header\n\tt.mutex.Unlock()\n\tt.reqBox.Set(reqHeader, nil)\n}\n\n// UpdateProgress updates the search progress\nfunc (t *Terminal) UpdateProgress(progress float32) {\n\tt.mutex.Lock()\n\tnewProgress := int(progress * 100)\n\tchanged := t.progress != newProgress\n\tt.progress = newProgress\n\tt.mutex.Unlock()\n\n\tif changed {\n\t\tt.reqBox.Set(reqInfo, nil)\n\t}\n}\n\n// UpdateList updates Merger to display the list\nfunc (t *Terminal) UpdateList(merger *Merger, triggerResultEvent bool) {\n\tt.mutex.Lock()\n\tprevIndex := minItem.Index()\n\treset := t.revision != merger.Revision()\n\tif !reset && t.track != trackDisabled {\n\t\tif t.merger.Length() > 0 {\n\t\t\tprevIndex = t.currentIndex()\n\t\t} else if merger.Length() > 0 {\n\t\t\tprevIndex = merger.First().item.Index()\n\t\t}\n\t}\n\tt.progress = 100\n\tt.merger = merger\n\tif reset {\n\t\tt.selected = make(map[int32]selectedItem)\n\t\tt.revision = merger.Revision()\n\t\tt.version++\n\t}\n\tif t.triggerLoad {\n\t\tt.triggerLoad = false\n\t\tt.eventChan <- tui.Load.AsEvent()\n\t}\n\tif prevIndex >= 0 {\n\t\tpos := t.cy - t.offset\n\t\tcount := t.merger.Length()\n\t\ti := t.merger.FindIndex(prevIndex)\n\t\tif i >= 0 {\n\t\t\tt.cy = i\n\t\t\tt.offset = t.cy - pos\n\t\t} else if t.track == trackCurrent {\n\t\t\tt.track = trackDisabled\n\t\t\tt.cy = pos\n\t\t\tt.offset = 0\n\t\t} else if t.cy > count {\n\t\t\t// Try to keep the vertical position when the list shrinks\n\t\t\tt.cy = count - util.Min(count, t.maxItems()) + pos\n\t\t}\n\t}\n\tif !t.reading {\n\t\tswitch t.merger.Length() {\n\t\tcase 0:\n\t\t\tzero := tui.Zero.AsEvent()\n\t\t\tif _, prs := t.keymap[zero]; prs {\n\t\t\t\tt.eventChan <- zero\n\t\t\t}\n\t\tcase 1:\n\t\t\tone := tui.One.AsEvent()\n\t\t\tif _, prs := t.keymap[one]; prs {\n\t\t\t\tt.eventChan <- one\n\t\t\t}\n\t\t}\n\t\tif triggerResultEvent && t.hasResultActions {\n\t\t\tt.eventChan <- tui.Result.AsEvent()\n\t\t}\n\t}\n\tt.mutex.Unlock()\n\tt.reqBox.Set(reqInfo, nil)\n\tt.reqBox.Set(reqList, nil)\n}\n\nfunc (t *Terminal) output() bool {\n\tif t.printQuery {\n\t\tt.printer(string(t.input))\n\t}\n\tif len(t.expect) > 0 {\n\t\tt.printer(t.pressed)\n\t}\n\tfound := len(t.selected) > 0\n\tif !found {\n\t\tcurrent := t.currentItem()\n\t\tif current != nil {\n\t\t\tt.printer(current.AsString(t.ansi))\n\t\t\tfound = true\n\t\t}\n\t} else {\n\t\tfor _, sel := range t.sortSelected() {\n\t\t\tt.printer(sel.item.AsString(t.ansi))\n\t\t}\n\t}\n\treturn found\n}\n\nfunc (t *Terminal) sortSelected() []selectedItem {\n\tsels := make([]selectedItem, 0, len(t.selected))\n\tfor _, sel := range t.selected {\n\t\tsels = append(sels, sel)\n\t}\n\tsort.Sort(byTimeOrder(sels))\n\treturn sels\n}\n\nfunc (t *Terminal) displayWidth(runes []rune) int {\n\twidth, _ := util.RunesWidth(runes, 0, t.tabstop, math.MaxInt32)\n\treturn width\n}\n\nconst (\n\tminWidth  = 4\n\tminHeight = 3\n)\n\nfunc calculateSize(base in\nt, size sizeSpec, occupied int, minSize int, pad int) int {\n\tmax := base - occupied\n\tif max < minSize {\n\t\tmax = minSize\n\t}\n\tif size.percent {\n\t\treturn util.Constrain(int(float64(base)*0.01*size.size), minSize, max)\n\t}\n\treturn util.Constrain(int(size.size)+pad, minSize, max)\n}\n\nfunc (t *Terminal) adjustMarginAndPadding() (int, int, [4]int, [4]int) {\n\tscreenWidth := t.tui.MaxX()\n\tscreenHeight := t.tui.MaxY()\n\tmarginInt := [4]int{}  // TRBL\n\tpaddingInt := [4]int{} // TRBL\n\tsizeSpecToInt := func(index int, spec sizeSpec) int {\n\t\tif spec.percent {\n\t\t\tvar max float64\n\t\t\tif index%2 == 0 {\n\t\t\t\tmax = float64(screenHeight)\n\t\t\t} else {\n\t\t\t\tmax = float64(screenWidth)\n\t\t\t}\n\t\t\treturn int(max * spec.size * 0.01)\n\t\t}\n\t\treturn int(spec.size)\n\t}\n\tfor idx, sizeSpec := range t.padding {\n\t\tpaddingInt[idx] = sizeSpecToInt(idx, sizeSpec)\n\t}\n\n\tbw := t.borderWidth\n\textraMargin := [4]int{} // TRBL\n\tfor idx, sizeSpec := range t.margin {\n\t\tswitch t.borderShape {\n\t\tcase tui.BorderHorizontal:\n\t\t\textraMargin[idx] += 1 - idx%2\n\t\tcase tui.BorderVertical:\n\t\t\textraMargin[idx] += (1 + bw) * (idx % 2)\n\t\tcase tui.BorderTop:\n\t\t\tif idx == 0 {\n\t\t\t\textraMargin[idx]++\n\t\t\t}\n\t\tcase tui.BorderRight:\n\t\t\tif idx == 1 {\n\t\t\t\textraMargin[idx] += 1 + bw\n\t\t\t}\n\t\tcase tui.BorderBottom:\n\t\t\tif idx == 2 {\n\t\t\t\textraMargin[idx]++\n\t\t\t}\n\t\tcase tui.BorderLeft:\n\t\t\tif idx == 3 {\n\t\t\t\textraMargin[idx] += 1 + bw\n\t\t\t}\n\t\tcase tui.BorderRounded, tui.BorderSharp, tui.BorderBold, tui.BorderBlock, tui.BorderThinBlock, tui.BorderDouble:\n\t\t\textraMargin[idx] += 1 + bw*(idx%2)\n\t\t}\n\t\tmarginInt[idx] = sizeSpecToInt(idx, sizeSpec) + extraMargin[idx]\n\t}\n\n\tadjust := func(idx1 int, idx2 int, max int, min int) {\n\t\tif min > max {\n\t\t\tmin = max\n\t\t}\n\t\tmargin := marginInt[idx1] + marginInt[idx2] + paddingInt[idx1] + paddingInt[idx2]\n\t\tif max-margin < min {\n\t\t\tdesired := max - min\n\t\t\tpaddingInt[idx1] = desired * paddingInt[idx1] / margin\n\t\t\tpaddingInt[idx2] = desired * paddingInt[idx2] / margin\n\t\t\tmarginInt[idx1] = util.Max(extraMargin[idx1], desired*marginInt[idx1]/margin)\n\t\t\tmarginInt[idx2] = util.Max(extraMargin[idx2], desired*marginInt[idx2]/margin)\n\t\t}\n\t}\n\n\tminAreaWidth := minWidth\n\tminAreaHeight := minHeight\n\tif t.noSeparatorLine() {\n\t\tminAreaHeight -= 1\n\t}\n\tif t.needPreviewWindow() {\n\t\tminPreviewHeight := 1 + borderLines(t.previewOpts.border)\n\t\tminPreviewWidth := 5\n\t\tswitch t.previewOpts.position {\n\t\tcase posUp, posDown:\n\t\t\tminAreaHeight += minPreviewHeight\n\t\t\tminAreaWidth = util.Max(minPreviewWidth, minAreaWidth)\n\t\tcase posLeft, posRight:\n\t\t\tminAreaWidth += minPreviewWidth\n\t\t\tminAreaHeight = util.Max(minPreviewHeight, minAreaHeight)\n\t\t}\n\t}\n\tadjust(1, 3, screenWidth, minAreaWidth)\n\tadjust(0, 2, screenHeight, minAreaHeight)\n\n\treturn screenWidth, screenHeight, marginInt, paddingInt\n}\n\nfunc (t *Terminal) resizeWindows(forcePreview bool) {\n\tt.forcePreview = forcePreview\n\tscreenWidth, screenHeight, marginInt, paddingInt := t.adjustMarginAndPadding()\n\twidth := screenWidth - marginInt[1] - marginInt[3]\n\theight := screenHeight - marginInt[0] - marginInt[2]\n\n\tt.prevLines = make([]itemLine, screenHeight)\n\tif t.border != nil {\n\t\tt.border.Close()\n\t}\n\tif t.window != nil {\n\t\tt.window.Close()\n\t\tt.window = nil\n\t}\n\tif t.pborder != nil {\n\t\tt.pborder.Close()\n\t\tt.pborder = nil\n\t}\n\tif t.pwindow != nil {\n\t\tt.pwindow.Close()\n\t\tt.pwindow = nil\n\t}\n\t// Reset preview version so that full redraw occurs\n\tt.previewed.version = 0\n\n\tbw := t.borderWidth\n\tswitch t.borderShape {\n\tcase tui.BorderHorizontal:\n\t\tt.border = t.tui.NewWindow(\n\t\t\tmarginInt[0]-1, marginInt[3], width, height+2,\n\t\t\tfalse, tui.MakeBorderStyle(tui.BorderHorizontal, t.unicode))\n\tcase tui.BorderVertical:\n\t\tt.border = t.tui.NewWindow(\n\t\t\tmarginInt[0], marginInt[3]-(1+bw), width+(1+bw)*2, height,\n\t\t\tfalse, tui.MakeBorderStyle(tui.BorderVertical, t.unicode))\n\tcase tui.BorderTop:\n\t\tt.border = t.tui.NewWindow(\n\t\t\tmarginInt[0]-1, marginInt[3], width, height+1,\n\t\t\tfalse, tui.MakeBorderStyle(tui.BorderTop, t.unicode))\n\tcase tui.BorderBottom:\n\t\tt.border = t.tui.NewWindow(\n\t\t\tmarginInt[0], marginInt[3], width, height+1,\n\t\t\tfalse, tui.MakeBorderStyle(tui.BorderBottom, t.unicode))\n\tcase tui.BorderLeft:\n\t\tt.border = t.tui.NewWindow(\n\t\t\tmarginInt[0], marginInt[3]-(1+bw), width+(1+bw), height,\n\t\t\tfalse, tui.MakeBorderStyle(tui.BorderLeft, t.unicode))\n\tcase tui.BorderRight:\n\t\tt.border = t.tui.NewWindow(\n\t\t\tmarginInt[0], marginInt[3], width+(1+bw), height,\n\t\t\tfalse, tui.MakeBorderStyle(tui.BorderRight, t.unicode))\n\tcase tui.BorderRounded, tui.BorderSharp, tui.BorderBold, tui.BorderBlock, tui.BorderThinBlock, tui.BorderDouble:\n\t\tt.border = t.tui.NewWindow(\n\t\t\tmarginInt[0]-1, marginInt[3]-(1+bw), width+(1+bw)*2, height+2,\n\t\t\tfalse, tui.MakeBorderStyle(t.borderShape, t.unicode))\n\t}\n\n\t// Add padding to margin\n\tfor idx, val := range paddingInt {\n\t\tmarginInt[idx] += val\n\t}\n\twidth -= paddingInt[1] + paddingInt[3]\n\theight -= paddingInt[0] + paddingInt[2]\n\n\tt.areaLines = height\n\tt.areaColumns = width\n\n\t// Set up preview window\n\tnoBorder := tui.MakeBorderStyle(tui.BorderNone, t.unicode)\n\tif forcePreview || t.needPreviewWindow() {\n\t\tvar resizePreviewWindows func(previewOpts *previewOpts)\n\t\tresizePreviewWindows = func(previewOpts *previewOpts) {\n\t\t\tt.activePreviewOpts = previewOpts\n\t\t\tif previewOpts.size.size == 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t\thasThreshold := previewOpts.threshold > 0 && previewOpts.alternative != nil\n\t\t\tcreatePreviewWindow := func(y int, x int, w int, h int) {\n\t\t\t\tpwidth := w\n\t\t\t\tpheight := h\n\t\t\t\tvar previewBorder tui.BorderStyle\n\t\t\t\tif previewOpts.border == tui.BorderNone {\n\t\t\t\t\tpreviewBorder = tui.MakeTransparentBorder()\n\t\t\t\t} else {\n\t\t\t\t\tpreviewBorder = tui.MakeBorderStyle(previewOpts.border, t.unicode)\n\t\t\t\t}\n...\n", "needle_token_start": 12695, "needle_token_end": 12813, "code_context_ntokens": 15041, "output": ["```go\nfunc calculateSize(base int, size sizeSpec, occupied int, minSize int, pad int) int {\n\tmax := base - occupied\n\tif max < minSize {\n\t\tmax = minSize\n\t}\n\tif size.percent {\n\t\treturn util.Constrain(int(float64(base)*0.01*size.size), minSize, max)\n\t}\n\treturn util.Constrain(int(size.size)+pad, minSize, max)\n}\n```"]}
{"repo": "junegunn/fzf", "name": "optsFor", "language": "go", "path": "src/options_test.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to configure and finalize settings based on a list of command-line arguments, ensuring that the application behaves according to user-specified or default configurations.\n2. **Input**: It accepts an array of strings, which typically represent command-line arguments that modify the behavior of an application.\n3. **Output**: Returns a fully configured options object that encapsulates various settings derived from the input arguments.\n4. **Procedure**: The function begins by initializing a default options object. It then parses the input strings to modify this object according to the specified arguments. After parsing, a final processing step is applied to ensure that all options are correctly set up and consistent, resulting in the finalized configuration object being returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/options.go\npackage fzf\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"unicode\"\n\n\t\"github.com/junegunn/fzf/src/algo\"\n\t\"github.com/junegunn/fzf/src/tui\"\n\t\"github.com/junegunn/fzf/src/util\"\n\n\t\"github.com/mattn/go-shellwords\"\n\t\"github.com/rivo/uniseg\"\n)\n\nconst usage = `usage: fzf [options]\n\n  Search\n    -x, --extended         Extended-search mode\n                           (enabled by default; +x or --no-extended to disable)\n    -e, --exact            Enable Exact-match\n    -i                     Case-insensitive match (default: smart-case match)\n    +i                     Case-sensitive match\n    --scheme=SCHEME        Scoring scheme [default|path|history]\n    --literal              Do not normalize latin script letters before matching\n    -n, --nth=N[,..]       Comma-separated list of field index expressions\n                           for limiting search scope. Each can be a non-zero\n                           integer or a range expression ([BEGIN]..[END]).\n    --with-nth=N[,..]      Transform the presentation of each line using\n                           field index expressions\n    -d, --delimiter=STR    Field delimiter regex (default: AWK-style)\n    +s, --no-sort          Do not sort the result\n    --track                Track the current selection when the result is updated\n    --tac                  Reverse the order of the input\n    --disabled             Do not perform search\n    --tiebreak=CRI[,..]    Comma-separated list of sort criteria to apply\n                           when the scores are tied [length|chunk|begin|end|index]\n                           (default: length)\n\n  Interface\n    -m, --multi[=MAX]      Enable multi-select with tab/shift-tab\n    --no-mouse             Disable mouse\n    --bind=KEYBINDS        Custom key bindings. Refer to the man page.\n    --cycle                Enable cyclic scroll\n    --keep-right           Keep the right end of the line visible on overflow\n    --scroll-off=LINES     Number of screen lines to keep above or below when\n                           scrolling to the top or to the bottom (default: 0)\n    --no-hscroll           Disable horizontal scroll\n    --hscroll-off=COLS     Number of screen columns to keep to the right of the\n                           highlighted substring (default: 10)\n    --filepath-word        Make word-wise movements respect path separators\n    --jump-labels=CHARS    Label characters for jump mode\n\n  Layout\n    --height=[~]HEIGHT[%]  Display fzf window below the cursor with the given\n                           height instead of using fullscreen.\n                           A negative value is calculated as the terminal height\n                           minus the given value.\n                           If prefixed with '~', fzf will determine the height\n                           according to the input size.\n    --min-height=HEIGHT    Minimum height when --height is given in percent\n                           (default: 10)\n    --layout=LAYOUT        Choose layout: [default|reverse|reverse-list]\n    --border[=STYLE]       Draw border around the finder\n                           [rounded|sharp|bold|block|thinblock|double|horizontal|vertical|\n                            top|bottom|left|right|none] (default: rounded)\n    --border-label=LABEL   Label to print on the border\n    --border-label-pos=COL Position of the border label\n                           [POSITIVE_INTEGER: columns from left|\n                            NEGATIVE_INTEGER: columns from right][:bottom]\n                           (default: 0 or center)\n    --margin=MARGIN        Screen margin (TRBL | TB,RL | T,RL,B | T,R,B,L)\n    --padding=PADDING      Padding inside border (TRBL | TB,RL | T,RL,B | T,R,B,L)\n    --info=STYLE           Finder info style\n                           [default|right|hidden|inline[-right][:PREFIX]]\n    --separator=STR        String to form horizontal separator on info line\n    --no-separator         Hide info line separator\n    --scrollbar[=C1[C2]]   Scrollbar character(s) (each for main and preview window)\n    --no-scrollbar         Hide scrollbar\n    --prompt=STR           Input prompt (default: '> ')\n    --pointer=STR          Pointer to the current line (default: '>')\n    --marker=STR           Multi-select marker (default: '>')\n    --header=STR           String to print as header\n    --header-lines=N       The first N lines of the input are treated as header\n    --header-first         Print header before the prompt line\n    --ellipsis=STR         Ellipsis to show when line is truncated (default: '..')\n\n  Display\n    --ansi                 Enable processing of ANSI color codes\n    --tabstop=SPACES       Number of spaces for a tab character (default: 8)\n    --color=COLSPEC        Base scheme (dark|light|16|bw) and/or custom colors\n    --no-bold              Do not use bold text\n\n  History\n    --history=FILE         History file\n    --history-size=N       Maximum number of history entries (default: 1000)\n\n  Preview\n    --preview=COMMAND      Command to preview highlighted line ({})\n    --preview-window=OPT   Preview window layout (default: right:50%)\n                           [up|down|left|right][,SIZE[%]]\n                           [,[no]wrap][,[no]cycle][,[no]follow][,[no]hidden]\n                           [,border-BORDER_OPT]\n                           [,+SCROLL[OFFSETS][/DENOM]][,~HEADER_LINES]\n                           [,default][,<SIZE_THRESHOLD(ALTERNATIVE_LAYOUT)]\n    --preview-label=LABEL\n    --preview-label-pos=N  Same as --border-label and --border-label-pos,\n                           but for preview window\n\n  Scripting\n    -q, --query=STR        Start the finder with the given query\n    -1, --select-1         Automatically select the only match\n    -0, --exit-0           Exit immediately when there's no match\n    -f, --filter=STR       Filter mode. Do not start interactive finder.\n    --print-query          Print query as the first line\n    --expect=KEYS          Comma-separated list of keys to complete fzf\n    --read0                Read input delimited by ASCII NUL characters\n    --print0               Print output delimited by ASCII NUL characters\n    --sync                 Synchronous search for multi-staged filtering\n    --listen[=[ADDR:]PORT] Start HTTP server to receive actions (POST /)\n                           (To allow remote process execution, use --listen-unsafe)\n    --version              Display version information and exit\n\n  Directory traversal      (Only used when $FZF_DEFAULT_COMMAND is not set)\n    --walker=OPTS          [file][,dir][,follow][,hidden] (default: file,follow,hidden)\n    --walker-root=DIR      Root directory from which to start walker (default: .)\n    --walker-skip=DIRS     Comma-separated list of directory names to skip\n                           (default: .git,node_modules)\n\n  Shell integration\n    --bash                 Print script to set up Bash shell integration\n    --zsh                  Print script to set up Zsh shell integration\n    --fish                 Print script to set up Fish shell integration\n\n  Environment variables\n    FZF_DEFAULT_COMMAND    Default command to use when input is tty\n    FZF_DEFAULT_OPTS       Default options (e.g. '--layout=reverse --info=inline')\n    FZF_DEFAULT_OPTS_FILE  Location of the file to read default options from\n    FZF_API_KEY            X-API-Key header for HTTP server (--listen)\n\n`\n\nconst defaultInfoPrefix = \" < \"\n\n// Case denotes case-sensitivity of search\ntype Case int\n\n// Case-sensitivities\nconst (\n\tCaseSmart Case = iota\n\tCaseIgnore\n\tCaseRespect\n)\n\n// Sort criteria\ntype criterion int\n\nconst (\n\tbyScore criterion = iota\n\tbyChunk\n\tbyLength\n\tbyBegin\n\tbyEnd\n)\n\ntype heightSpec struct {\n\tsize    float64\n\tpercent bool\n\tauto    bool\n\tinverse bool\n}\n\ntype sizeSpec struct {\n\tsize    float64\n\tpercent bool\n}\n\nfunc defaultMargin() [4]sizeSpec {\n\treturn [4]sizeSpec{}\n}\n\ntype trackOption int\n\nconst (\n\ttrackDisabled trackOption = iota\n\ttrackEnabled\n\ttrackCurrent\n)\n\ntype windowPosition int\n\nconst (\n\tposUp windowPosition = iota\n\tposDown\n\tposLeft\n\tposRight\n)\n\ntype layoutType int\n\nconst (\n\tlayoutDefault layoutType = iota\n\tlayoutReverse\n\tlayoutReverseList\n)\n\ntype infoStyle int\n\nconst (\n\tinfoDefault infoStyle = iota\n\tinfoRight\n\tinfoInline\n\tinfoInlineRight\n\tinfoHidden\n)\n\ntype labelOpts struct {\n\tlabel  string\n\tcolumn int\n\tbottom bool\n}\n\ntype previewOpts struct {\n\tcommand     string\n\tposition    windowPosition\n\tsize        sizeSpec\n\tscroll      string\n\thidden      bool\n\twrap        bool\n\tcycle       bool\n\tfollow      bool\n\tborder      tui.BorderShape\n\theaderLines int\n\tthreshold   int\n\talternative *previewOpts\n}\n\nfunc (o *previewOpts) Visible() bool {\n\treturn o.size.size > 0 || o.alternative != nil && o.alternative.size.size > 0\n}\n\nfunc (o *previewOpts) Toggle() {\n\to.hidden = !o.hidden\n}\n\nfunc parseLabelPosition(opts *labelOpts, arg string) {\n\topts.column = 0\n\topts.bottom = false\n\tfor _, token := range splitRegexp.Split(strings.ToLower(arg), -1) {\n\t\tswitch token {\n\t\tcase \"center\":\n\t\t\topts.column = 0\n\t\tcase \"bottom\":\n\t\t\topts.bottom = true\n\t\tcase \"top\":\n\t\t\topts.bottom = false\n\t\tdefault:\n\t\t\topts.column = atoi(token)\n\t\t}\n\t}\n}\n\nfunc (a previewOpts) aboveOrBelow() bool {\n\treturn a.size.size > 0 && (a.position == posUp || a.position == posDown)\n}\n\nfunc (a previewOpts) sameLayout(b previewOpts) bool {\n\treturn a.size == b.size && a.position == b.position && a.border == b.border && a.hidden == b.hidden && a.threshold == b.threshold &&\n\t\t(a.alternative != nil && b.alternative != nil && a.alternative.sameLayout(*b.alternative) ||\n\t\t\ta.alternative == nil && b.alternative == nil)\n}\n\nfunc (a previewOpts) sameContentLayout(b previewOpts) bool {\n\treturn a.wrap == b.wrap && a.headerLines == b.headerLines\n}\n\nfunc firstLine(s string) string {\n\treturn strings.SplitN(s, \"\\n\", 2)[0]\n}\n\ntype walkerOpts struct {\n\tfile   bool\n\tdir    bool\n\thidden bool\n\tfollow bool\n}\n\n// Options stores the values of command-line options\ntype Options struct {\n\tBash         bool\n\tZsh          bool\n\tFish         bool\n\tFuzzy        bool\n\tFuzzyAlgo    algo.Algo\n\tScheme       string\n\tExtended     bool\n\tPhony        bool\n\tCase         Case\n\tNormalize    bool\n\tNth          []Range\n\tWithNth      []Range\n\tDelimiter    Delimiter\n\tSort         int\n\tTrack        trackOption\n\tTac          bool\n\tCriteria     []criterion\n\tMulti        int\n\tAnsi         bool\n\tMouse        bool\n\tTheme        *tui.ColorTheme\n\tBlack        bool\n\tBold         bool\n\tHeight       heightSpec\n\tMinHeight    int\n\tLayout       layoutType\n\tCycle        bool\n\tKeepRight    bool\n\tHscroll      bool\n\tHscrollOff   int\n\tScrollOff    int\n\tFileWord     bool\n\tInfoStyle    infoStyle\n\tInfoPrefix   string\n\tSeparator    *string\n\tJumpLabels   string\n\tPrompt       string\n\tPointer      string\n\tMarker       string\n\tQuery        string\n\tSelect1      bool\n\tExit0        bool\n\tFilter       *string\n\tToggleSort   bool\n\tExpect       map[tui.Event]string\n\tKeymap       map[tui.Event][]*action\n\tPreview      previewOpts\n\tPrintQuery   bool\n\tReadZero     bool\n\tPrinter      func(string)\n\tPrintSep     string\n\tSync         bool\n\tHistory      *History\n\tHeader       []string\n\tHeaderLines  int\n\tHeaderFirst  bool\n\tEllipsis     string\n\tScrollbar    *string\n\tMargin       [4]sizeSpec\n\tPadding      [4]sizeSpec\n\tBorderShape  tui.BorderShape\n\tBorderLabel  labelOpts\n\tPreviewLabel labelOpts\n\tUnicode      bool\n\tAmbidouble   bool\n\tTabstop      int\n\tListenAddr   *listenAddress\n\tUnsafe       bool\n\tClearOnExit  bool\n\tWalkerOpts   walkerOpts\n\tWalkerRoot   string\n\tWalkerSkip   []string\n\tVersion      bool\n\tCPUProfile   string\n\tMEMProfile   string\n\tBlockProfile string\n\tMutexProfile string\n}\n\nfunc filterNonEmpty(input []string) []string {\n\toutput := make([]string, 0, len(input))\n\tfor _, str := range input {\n\t\tif len(str) > 0 {\n\t\t\toutput = append(output, str)\n\t\t}\n\t}\n\treturn output\n}\n\nfunc defaultPreviewOpts(command string) previewOpts {\n\treturn previewOpts{command, posRight, sizeSpec{50, true}, \"\", false, false, false, false, tui.DefaultBorderShape, 0, 0, nil}\n}\n\nfunc defaultOptions() *Options {\n\treturn &Options{\n\t\tBash:         false,\n\t\tZsh:          false,\n\t\tFish:         false,\n\t\tFuzzy:        true,\n\t\tFuzzyAlgo:    algo.FuzzyMatchV2,\n\t\tScheme:       \"default\",\n\t\tExtended:     true,\n\t\tPhony:        false,\n\t\tCase:         CaseSmart,\n\t\tNormalize:    true,\n\t\tNth:          make([]Range, 0),\n\t\tWithNth:      make([]Range, 0),\n\t\tDelimiter:    Delimiter{},\n\t\tSort:         1000,\n\t\tTrack:        trackDisabled,\n\t\tTac:          false,\n\t\tCriteria:     []criterion{byScore, byLength},\n\t\tMulti:        0,\n\t\tAnsi:         false,\n\t\tMouse:        true,\n\t\tTheme:        tui.EmptyTheme(),\n\t\tBlack:        false,\n\t\tBold:         true,\n\t\tMinHeight:    10,\n\t\tLayout:       layoutDefault,\n\t\tCycle:        false,\n\t\tKeepRight:    false,\n\t\tHscroll:      true,\n\t\tHscrollOff:   10,\n\t\tScrollOff:    0,\n\t\tFileWord:     false,\n\t\tInfoStyle:    infoDefault,\n\t\tSeparator:    nil,\n\t\tJumpLabels:   defaultJumpLabels,\n\t\tPrompt:       \"> \",\n\t\tPointer:      \">\",\n\t\tMarker:       \">\",\n\t\tQuery:        \"\",\n\t\tSelect1:      false,\n\t\tExit0:        false,\n\t\tFilter:       nil,\n\t\tToggleSort:   false,\n\t\tExpect:       make(map[tui.Event]string),\n\t\tKeymap:       make(map[tui.Event][]*action),\n\t\tPreview:      defaultPreviewOpts(\"\"),\n\t\tPrintQuery:   false,\n\t\tReadZero:     false,\n\t\tPrinter:      func(str string) { fmt.Println(str) },\n\t\tPrintSep:     \"\\n\",\n\t\tSync:         false,\n\t\tHistory:      nil,\n\t\tHeader:       make([]string, 0),\n\t\tHeaderLines:  0,\n\t\tHeaderFirst:  false,\n\t\tEllipsis:     \"..\",\n\t\tScrollbar:    nil,\n\t\tMargin:       defaultMargin(),\n\t\tPadding:      defaultMargin(),\n\t\tUnicode:      true,\n\t\tAmbidouble:   os.Getenv(\"RUNEWIDTH_EASTASIAN\") == \"1\",\n\t\tTabstop:      8,\n\t\tBorderLabel:  labelOpts{},\n\t\tPreviewLabel: labelOpts{},\n\t\tUnsafe:       false,\n\t\tClearOnExit:  true,\n\t\tWalkerOpts:   walkerOpts{file: true, hidden: true, follow: true},\n\t\tWalkerRoot:   \".\",\n\t\tWalkerSkip:   []string{\".git\", \"node_modules\"},\n\t\tVersion:      false}\n}\n\nfunc help(code int) {\n\tos.Stdout.WriteString(usage)\n\tutil.Exit(code)\n}\n\nvar errorContext = \"\"\n\nfunc errorExit(msg string) {\n\tos.Stderr.WriteString(errorContext + msg + \"\\n\")\n\tutil.Exit(exitError)\n}\n\nfunc optString(arg string, prefixes ...string) (bool, string) {\n\tfor _, prefix := range prefixes {\n\t\tif strings.HasPrefix(arg, prefix) {\n\t\t\treturn true, arg[len(prefix):]\n\t\t}\n\t}\n\treturn false, \"\"\n}\n\nfunc nextString(args []string, i *int, message string) string {\n\tif len(args) > *i+1 {\n\t\t*i++\n\t} else {\n\t\terrorExit(message)\n\t}\n\treturn args[*i]\n}\n\nfunc optionalNextString(args []string, i *int) (bool, string) {\n\tif len(args) > *i+1 && !strings.HasPrefix(args[*i+1], \"-\") && !strings.HasPrefix(args[*i+1], \"+\") {\n\t\t*i++\n\t\treturn true, args[*i]\n\t}\n\treturn false, \"\"\n}\n\nfunc atoi(str string) int {\n\tnum, err := strconv.Atoi(str)\n\tif err != nil {\n\t\terrorExit(\"not a valid integer: \" + str)\n\t}\n\treturn num\n}\n\nfunc atof(str string) float64 {\n\tnum, err := strconv.ParseFloat(str, 64)\n\tif err != nil {\n\t\terrorExit(\"not a valid number: \" + str)\n\t}\n\treturn num\n}\n\nfunc nextInt(args []string, i *int, message string) int {\n\tif len(args) > *i+1 {\n\t\t*i++\n\t} else {\n\t\terrorExit(message)\n\t}\n\treturn atoi(args[*i])\n}\n\nfunc optionalNumeric(args []string, i *int, defaultValue int) int {\n\tif len(args) > *i+1 {\n\t\tif strings.IndexAny(args[*i+1], \"0123456789\") == 0 {\n\t\t\t*i++\n\t\t\treturn atoi(args[*i])\n\t\t}\n\t}\n\treturn defaultValue\n}\n\nfunc splitNth(str string) []Range {\n\tif match, _ := regexp.MatchString(\"^[0-9,-.]+$\", str); !match {\n\t\terrorExit(\"invalid format: \" + str)\n\t}\n\n\ttokens := strings.Split(str, \",\")\n\tranges := make([]Range, len(tokens))\n\tfor idx, s := range tokens {\n\t\tr, ok := ParseRange(&s)\n\t\tif !ok {\n\t\t\terrorExit(\"invalid format: \" + str)\n\t\t}\n\t\tranges[idx] = r\n\t}\n\treturn ranges\n}\n\nfunc delimiterRegexp(str string) Delimiter {\n\t// Special handling of \\t\n\tstr = strings.Replace(str, \"\\\\t\", \"\\t\", -1)\n\n\t// 1. Pattern does not contain any special character\n\tif regexp.QuoteMeta(str) == str {\n\t\treturn Delimiter{str: &str}\n\t}\n\n\trx, e := regexp.Compile(str)\n\t// 2. Pattern is not a valid regular expression\n\tif e != nil {\n\t\treturn Delimiter{str: &str}\n\t}\n\n\t// 3. Pattern as regular expression. Slow.\n\treturn Delimiter{regex: rx}\n}\n\nfunc isAlphabet(char uint8) bool {\n\treturn char >= 'a' && char <= 'z'\n}\n\nfunc isNumeric(char uint8) bool {\n\treturn char >= '0' && char <= '9'\n}\n\nfunc parseAlgo(str string) algo.Algo {\n\tswitch str {\n\tcase \"v1\":\n\t\treturn algo.FuzzyMatchV1\n\tcase \"v2\":\n\t\treturn algo.FuzzyMatchV2\n\tdefault:\n\t\terrorExit(\"invalid algorithm (expected: v1 or v2)\")\n\t}\n\treturn algo.FuzzyMatchV2\n}\n\nfunc processScheme(opts *Options) {\n\tif !algo.Init(opts.Scheme) {\n\t\terrorExit(\"invalid scoring scheme (expected: default|path|history)\")\n\t}\n\tif opts.Scheme == \"history\" {\n\t\topts.Criteria = []criterion{byScore}\n\t}\n}\n\nfunc parseBorder(str string, optional bool) tui.BorderShape {\n\tswitch str {\n\tcase \"rounded\":\n\t\treturn tui.BorderRounded\n\tcase \"sharp\":\n\t\treturn tui.BorderSharp\n\tcase \"bold\":\n\t\treturn tui.BorderBold\n\tcase \"block\":\n\t\treturn tui.BorderBlock\n\tcase \"thinblock\":\n\t\treturn tui.BorderThinBlock\n\tcase \"double\":\n\t\treturn tui.BorderDouble\n\tcase \"horizontal\":\n\t\treturn tui.BorderHorizontal\n\tcase \"vertical\":\n\t\treturn tui.BorderVertical\n\tcase \"top\":\n\t\treturn tui.BorderTop\n\tcase \"bottom\":\n\t\treturn tui.BorderBottom\n\tcase \"left\":\n\t\treturn tui.BorderLeft\n\tcase \"right\":\n\t\treturn tui.BorderRight\n\tcase \"none\":\n\t\treturn tui.BorderNone\n\tdefault:\n\t\tif optional && str == \"\" {\n\t\t\treturn tui.DefaultBorderShape\n\t\t}\n\t\terrorExit(\"invalid border style (expected: rounded|sharp|bold|block|thinblock|double|horizontal|vertical|top|bottom|left|right|none)\")\n\t}\n\treturn tui.BorderNone\n}\n\nfunc parseKeyChords(str string, message string) map[tui.Event]string {\n\treturn parseKeyChordsImpl(str, message, errorExit)\n}\n\nfunc parseKeyChordsImpl(str string, message string, exit func(string)) map[tui.Event]string {\n\tif len(str) == 0 {\n\t\texit(message)\n\t\treturn nil\n\t}\n\n\tstr = regexp.MustCompile(\"(?i)(alt-),\").ReplaceAllString(str, \"$1\"+string([]rune{escapedComma}))\n\ttokens := strings.Split(str, \",\")\n\tif str == \",\" || strings.HasPrefix(str, \",,\") || strings.HasSuffix(str, \",,\") || strings.Contains(str, \",,,\") {\n\t\ttokens = append(tokens, \",\")\n\t}\n\n\tchords := make(map[tui.Event]string)\n\tfor _, key := range tokens {\n\t\tif len(key) == 0 {\n\t\t\tcontinue // ignore\n\t\t}\n\t\tkey = strings.ReplaceAll(key, string([]rune{escapedComma}), \",\")\n\t\tlkey := strings.ToLower(key)\n\t\tadd := func(e tui.EventType) {\n\t\t\tchords[e.AsEvent()] = key\n\t\t}\n\t\tswitch lkey {\n\t\tcase \"up\":\n\t\t\tadd(tui.Up)\n\t\tcase \"down\":\n\t\t\tadd(tui.Down)\n\t\tcase \"left\":\n\t\t\tadd(tui.Left)\n\t\tcase \"right\":\n\t\t\tadd(tui.Right)\n\t\tcase \"enter\", \"return\":\n\t\t\tadd(tui.CtrlM)\n\t\tcase \"space\":\n\t\t\tchords[tui.Key(' ')] = key\n\t\tcase \"backspace\", \"bspace\", \"bs\":\n\t\t\tadd(tui.Backspace)\n\t\tcase \"ctrl-space\":\n\t\t\tadd(tui.CtrlSpace)\n\t\tcase \"ctrl-delete\":\n\t\t\tadd(tui.CtrlDelete)\n\t\tcase \"ctrl-^\", \"ctrl-6\":\n\t\t\tadd(tui.CtrlCaret)\n\t\tcase \"ctrl-/\", \"ctrl-_\":\n\t\t\tadd(tui.CtrlSlash)\n\t\tcase \"ctrl-\\\\\":\n\t\t\tadd(tui.CtrlBackSlash)\n\t\tcase \"ctrl-]\":\n\t\t\tadd(tui.CtrlRightBracket)\n\t\tcase \"change\":\n\t\t\tadd(tui.Change)\n\t\tcase \"backward-eof\":\n\t\t\tadd(tui.BackwardEOF)\n\t\tcase \"start\":\n\t\t\tadd(tui.Start)\n\t\tcase \"load\":\n\t\t\tadd(tui.Load)\n\t\tcase \"focus\":\n\t\t\tadd(tui.Focus)\n\t\tcase \"result\":\n\t\t\tadd(tui.Result)\n\t\tcase \"resize\":\n\t\t\tadd(tui.Resize)\n\t\tcase \"one\":\n\t\t\tadd(tui.One)\n\t\tcase \"zero\":\n\t\t\tadd(tui.Zero)\n\t\tcase \"jump\":\n\t\t\tadd(tui.Jump)\n\t\tcase \"jump-cancel\":\n\t\t\tadd(tui.JumpCancel)\n\t\tcase \"alt-enter\", \"alt-return\":\n\t\t\tchords[tui.CtrlAltKey('m')] = key\n\t\tcase \"alt-space\":\n\t\t\tchords[tui.AltKey(' ')] = key\n\t\tcase \"alt-bs\", \"alt-bspace\", \"alt-backspace\":\n\t\t\tadd(tui.AltBackspace)\n\t\tcase \"alt-up\":\n\t\t\tadd(tui.AltUp)\n\t\tcase \"alt-down\":\n\t\t\tadd(tui.AltDown)\n\t\tcase \"alt-left\":\n\t\t\tadd(tui.AltLeft)\n\t\tcase \"alt-right\":\n\t\t\tadd(tui.AltRight)\n\t\tcase \"tab\":\n\t\t\tadd(tui.Tab)\n\t\tcase \"btab\", \"shift-tab\":\n\t\t\tadd(tui.ShiftTab)\n\t\tcase \"esc\":\n\t\t\tadd(tui.Esc)\n\t\tcase \"delete\", \"del\":\n\t\t\tadd(tui.Delete)\n\t\tcase \"home\":\n\t\t\tadd(tui.Home)\n\t\tcase \"end\":\n\t\t\tadd(tui.End)\n\t\tcase \"insert\":\n\t\t\tadd(tui.Insert)\n\t\tcase \"pgup\", \"page-up\":\n\t\t\tadd(tui.PageUp)\n\t\tcase \"pgdn\", \"page-down\":\n\t\t\tadd(tui.PageDown)\n\t\tcase \"alt-shift-up\", \"shift-alt-up\":\n\t\t\tadd(tui.AltShiftUp)\n\t\tcase \"alt-shift-down\", \"shift-alt-down\":\n\t\t\tadd(tui.AltShiftDown)\n\t\tcase \"alt-shift-left\", \"shift-alt-left\":\n\t\t\tadd(tui.AltShiftLeft)\n\t\tcase \"alt-shift-right\", \"shift-alt-right\":\n\t\t\tadd(tui.AltShiftRight)\n\t\tcase \"shift-up\":\n\t\t\tadd(tui.ShiftUp)\n\t\tcase \"shift-down\":\n\t\t\tadd(tui.ShiftDown)\n\t\tcase \"shift-left\":\n\t\t\tadd(tui.ShiftLeft)\n\t\tcase \"shift-right\":\n\t\t\tadd(tui.ShiftRight)\n\t\tcase \"shift-delete\":\n\t\t\tadd(tui.ShiftDelete)\n\t\tcase \"left-click\":\n\t\t\tadd(tui.LeftClick)\n\t\tcase \"right-click\":\n\t\t\tadd(tui.RightClick)\n\t\tcase \"shift-left-click\":\n\t\t\tadd(tui.SLeftClick)\n\t\tcase \"shift-right-click\":\n\t\t\tadd(tui.SRightClick)\n\t\tcase \"double-click\":\n\t\t\tadd(tui.DoubleClick)\n\t\tcase \"scroll-up\":\n\t\t\tadd(tui.ScrollUp)\n\t\tcase \"scroll-down\":\n\t\t\tadd(tui.ScrollDown)\n\t\tcase \"shift-scroll-up\":\n\t\t\tadd(tui.SScrollUp)\n\t\tcase \"shift-scroll-down\":\n\t\t\tadd(tui.SScrollDown)\n\t\tcase \"preview-scroll-up\":\n\t\t\tadd(tui.PreviewScrollUp)\n\t\tcase \"preview-scroll-down\":\n\t\t\tadd(tui.PreviewScrollDown)\n\t\tcase \"f10\":\n\t\t\tadd(tui.F10)\n\t\tcase \"f11\":\n\t\t\tadd(tui.F11)\n\t\tcase \"f12\":\n\t\t\tadd(tui.F12)\n\t\tdefault:\n\t\t\trunes := []rune(key)\n\t\t\tif len(key) == 10 && strings.HasPrefix(lkey, \"ctrl-alt-\") && isAlphabet(lkey[9]) {\n\t\t\t\tchords[tui.CtrlAltKey(rune(key[9]))] = key\n\t\t\t} else if len(key) == 6 && strings.HasPrefix(lkey, \"ctrl-\") && isAlphabet(lkey[5]) {\n\t\t\t\tadd(tui.EventType(tui.CtrlA.Int() + int(lkey[5]) - 'a'))\n...\n// Path: src/options_no_pprof.go\n//go:build !pprof\n// +build !pprof\n\npackage fzf\n\nfunc (o *Options) initProfiling() error {\n\tif o.CPUProfile != \"\" || o.MEMProfile != \"\" || o.BlockProfile != \"\" || o.MutexProfile != \"\" {\n\t\terrorExit(\"error: profiling not supported: FZF must be built with '-tags=pprof' to enable profiling\")\n\t}\n\treturn nil\n}\n\n// Path: src/options_pprof.go\n//go:build pprof\n// +build pprof\n\npackage fzf\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\t\"runtime/pprof\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\nfunc (o *Options) initProfiling() error {\n\tif o.CPUProfile != \"\" {\n\t\tf, err := os.Create(o.CPUProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create CPU profile: %w\", err)\n\t\t}\n\n\t\tif err := pprof.StartCPUProfile(f); err != nil {\n\t\t\treturn fmt.Errorf(\"could not start CPU profile: %w\", err)\n\t\t}\n\n\t\tutil.AtExit(func() {\n\t\t\tpprof.StopCPUProfile()\n\t\t\tif err := f.Close(); err != nil {\n\t\t\t\tfmt.Fprintln(os.Stderr, \"Error: closing cpu profile:\", err)\n\t\t\t}\n\t\t})\n\t}\n\n\tstopProfile := func(name string, f *os.File) {\n\t\tif err := pprof.Lookup(name).WriteTo(f, 0); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"Error: could not write %s profile: %v\\n\", name, err)\n\t\t}\n\t\tif err := f.Close(); err != nil {\n\t\t\tfmt.Fprintf(os.Stderr, \"Error: closing %s profile: %v\\n\", name, err)\n\t\t}\n\t}\n\n\tif o.MEMProfile != \"\" {\n\t\tf, err := os.Create(o.MEMProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create MEM profile: %w\", err)\n\t\t}\n\t\tutil.AtExit(func() {\n\t\t\truntime.GC()\n\t\t\tstopProfile(\"allocs\", f)\n\t\t})\n\t}\n\n\tif o.BlockProfile != \"\" {\n\t\truntime.SetBlockProfileRate(1)\n\t\tf, err := os.Create(o.BlockProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create BLOCK profile: %w\", err)\n\t\t}\n\t\tutil.AtExit(func() { stopProfile(\"block\", f) })\n\t}\n\n\tif o.MutexProfile != \"\" {\n\t\truntime.SetMutexProfileFraction(1)\n\t\tf, err := os.Create(o.MutexProfile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not create MUTEX profile: %w\", err)\n\t\t}\n\t\tutil.AtExit(func() { stopProfile(\"mutex\", f) })\n\t}\n\n\treturn nil\n}\n\n// Path: src/options_pprof_test.go\n//go:build pprof\n// +build pprof\n\npackage fzf\n\nimport (\n\t\"bytes\"\n\t\"flag\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/junegunn/fzf/src/util\"\n)\n\n// runInitProfileTests is an internal flag used TestInitProfiling\nvar runInitProfileTests = flag.Bool(\"test-init-profile\", false, \"run init profile tests\")\n\nfunc TestInitProfiling(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skip(\"short test\")\n\t}\n\n\t// Run this test in a separate process since it interferes with\n\t// profiling and modifies the global atexit state. Without this\n\t// running `go test -bench . -cpuprofile cpu.out` will fail.\n\tif !*runInitProfileTests {\n\t\tt.Parallel()\n\n\t\t// Make sure we are not the child process.\n\t\tif os.Getenv(\"_FZF_CHILD_PROC\") != \"\" {\n\t\t\tt.Fatal(\"already running as child process!\")\n\t\t}\n\n\t\tcmd := exec.Command(os.Args[0],\n\t\t\t\"-test.timeout\", \"30s\",\n\t\t\t\"-test.run\", \"^\"+t.Name()+\"$\",\n\t\t\t\"-test-init-profile\",\n\t\t)\n\t\tcmd.Env = append(os.Environ(), \"_FZF_CHILD_PROC=1\")\n\n\t\tout, err := cmd.CombinedOutput()\n\t\tout = bytes.TrimSpace(out)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Child test process failed: %v:\\n%s\", err, out)\n\t\t}\n\t\t// Make sure the test actually ran\n\t\tif bytes.Contains(out, []byte(\"no tests to run\")) {\n\t\t\tt.Fatalf(\"Failed to run test %q:\\n%s\", t.Name(), out)\n\t\t}\n\t\treturn\n\t}\n\n\t// Child process\n\n\ttempdir := t.TempDir()\n\tt.Cleanup(util.RunAtExitFuncs)\n\n\to := Options{\n\t\tCPUProfile:   filepath.Join(tempdir, \"cpu.prof\"),\n\t\tMEMProfile:   filepath.Join(tempdir, \"mem.prof\"),\n\t\tBlockProfile: filepath.Join(tempdir, \"block.prof\"),\n\t\tMutexProfile: filepath.Join(tempdir, \"mutex.prof\"),\n\t}\n\tif err := o.initProfiling(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tprofiles := []string{\n\t\to.CPUProfile,\n\t\to.MEMProfile,\n\t\to.BlockProfile,\n\t\to.MutexProfile,\n\t}\n\tfor _, name := range profiles {\n\t\tif _, err := os.Stat(name); err != nil {\n\t\t\tt.Errorf(\"Failed to create profile %s: %v\", filepath.Base(name), err)\n\t\t}\n\t}\n\n\tutil.RunAtExitFuncs()\n\n\tfor _, name := range profiles {\n\t\tif _, err := os.Stat(name); err != nil {\n\t\t\tt.Errorf(\"Failed to write profile %s: %v\", filepath.Base(name), err)\n\t\t}\n\t}\n}\n\n// Path: src/options_test.go\npackage fzf\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/junegunn/fzf/src/tui\"\n)\n\nfunc TestDelimiterRegex(t *testing.T) {\n\t// Valid regex\n\tdelim := delimiterRegexp(\".\")\n\tif delim.regex == nil || delim.str != nil {\n\t\tt.Error(delim)\n\t}\n\t// Broken regex -> string\n\tdelim = delimiterRegexp(\"[0-9\")\n\tif delim.regex != nil || *delim.str != \"[0-9\" {\n\t\tt.Error(delim)\n\t}\n\t// Valid regex\n\tdelim = delimiterRegexp(\"[0-9]\")\n\tif delim.regex.String() != \"[0-9]\" || delim.str != nil {\n\t\tt.Error(delim)\n\t}\n\t// Tab character\n\tdelim = delimiterRegexp(\"\\t\")\n\tif delim.regex != nil || *delim.str != \"\\t\" {\n\t\tt.Error(delim)\n\t}\n\t// Tab expression\n\tdelim = delimiterRegexp(\"\\\\t\")\n\tif delim.regex != nil || *delim.str != \"\\t\" {\n\t\tt.Error(delim)\n\t}\n\t// Tabs -> regex\n\tdelim = delimiterRegexp(\"\\t+\")\n\tif delim.regex == nil || delim.str != nil {\n\t\tt.Error(delim)\n\t}\n}\n\nfunc TestDelimiterRegexString(t *testing.T) {\n\tdelim := delimiterRegexp(\"*\")\n\ttokens := Tokenize(\"-*--*---**---\", delim)\n\tif delim.regex != nil ||\n\t\ttokens[0].text.ToString() != \"-*\" ||\n\t\ttokens[1].text.ToString() != \"--*\" ||\n\t\ttokens[2].text.ToString() != \"---*\" ||\n\t\ttokens[3].text.ToString() != \"*\" ||\n\t\ttokens[4].text.ToString() != \"---\" {\n\t\tt.Errorf(\"%s %v %d\", delim, tokens, len(tokens))\n\t}\n}\n\nfunc TestDelimiterRegexRegex(t *testing.T) {\n\tdelim := delimiterRegexp(\"--\\\\*\")\n\ttokens := Tokenize(\"-*--*---**---\", delim)\n\tif delim.str != nil ||\n\t\ttokens[0].text.ToString() != \"-*--*\" ||\n\t\ttokens[1].text.ToString() != \"---*\" ||\n\t\ttokens[2].text.ToString() != \"*---\" {\n\t\tt.Errorf(\"%s %d\", tokens, len(tokens))\n\t}\n}\n\nfunc TestDelimiterRegexRegexCaret(t *testing.T) {\n\tdelim := delimiterRegexp(`(^\\s*|\\s+)`)\n\ttokens := Tokenize(\"foo  bar baz\", delim)\n\tif delim.str != nil ||\n\t\tlen(tokens) != 4 ||\n\t\ttokens[0].text.ToString() != \"\" ||\n\t\ttokens[1].text.ToString() != \"foo  \" ||\n\t\ttokens[2].text.ToString() != \"bar \" ||\n\t\ttokens[3].text.ToString() != \"baz\" {\n\t\tt.Errorf(\"%s %d\", tokens, len(tokens))\n\t}\n}\n\nfunc TestSplitNth(t *testing.T) {\n\t{\n\t\tranges := splitNth(\"..\")\n\t\tif len(ranges) != 1 ||\n\t\t\tranges[0].begin != rangeEllipsis ||\n\t\t\tranges[0].end != rangeEllipsis {\n\t\t\tt.Errorf(\"%v\", ranges)\n\t\t}\n\t}\n\t{\n\t\tranges := splitNth(\"..3,1..,2..3,4..-1,-3..-2,..,2,-2,2..-2,1..-1\")\n\t\tif len(ranges) != 10 ||\n\t\t\tranges[0].begin != rangeEllipsis || ranges[0].end != 3 ||\n\t\t\tranges[1].begin != rangeEllipsis || ranges[1].end != rangeEllipsis ||\n\t\t\tranges[2].begin != 2 || ranges[2].end != 3 ||\n\t\t\tranges[3].begin != 4 || ranges[3].end != rangeEllipsis ||\n\t\t\tranges[4].begin != -3 || ranges[4].end != -2 ||\n\t\t\tranges[5].begin != rangeEllipsis || ranges[5].end != rangeEllipsis ||\n\t\t\tranges[6].begin != 2 || ranges[6].end != 2 ||\n\t\t\tranges[7].begin != -2 || ranges[7].end != -2 ||\n\t\t\tranges[8].begin != 2 || ranges[8].end != -2 ||\n\t\t\tranges[9].begin != rangeEllipsis || ranges[9].end != rangeEllipsis {\n\t\t\tt.Errorf(\"%v\", ranges)\n\t\t}\n\t}\n}\n\nfunc TestIrrelevantNth(t *testing.T) {\n\t{\n\t\topts := defaultOptions()\n\t\twords := []string{\"--nth\", \"..\", \"-x\"}\n\t\tparseOptions(opts, words)\n\t\tpostProcessOptions(opts)\n\t\tif len(opts.Nth) != 0 {\n\t\t\tt.Errorf(\"nth should be empty: %v\", opts.Nth)\n\t\t}\n\t}\n\tfor _, words := range [][]string{{\"--nth\", \"..,3\", \"+x\"}, {\"--nth\", \"3,1..\", \"+x\"}, {\"--nth\", \"..-1,1\", \"+x\"}} {\n\t\t{\n\t\t\topts := defaultOptions()\n\t\t\tparseOptions(opts, words)\n\t\t\tpostProcessOptions(opts)\n\t\t\tif len(opts.Nth) != 0 {\n\t\t\t\tt.Errorf(\"nth should be empty: %v\", opts.Nth)\n\t\t\t}\n\t\t}\n\t\t{\n\t\t\topts := defaultOptions()\n\t\t\twords = append(words, \"-x\")\n\t\t\tparseOptions(opts, words)\n\t\t\tpostProcessOptions(opts)\n\t\t\tif len(opts.Nth) != 2 {\n\t\t\t\tt.Errorf(\"nth should not be empty: %v\", opts.Nth)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestParseKeys(t *testing.T) {\n\tpairs := parseKeyChords(\"ctrl-z,alt-z,f2,@,Alt-a,!,ctrl-G,J,g,ctrl-alt-a,ALT-enter,alt-SPACE\", \"\")\n\tcheckEvent := func(e tui.Event, s string) {\n\t\tif pairs[e] != s {\n\t\t\tt.Errorf(\"%s != %s\", pairs[e], s)\n\t\t}\n\t}\n\tcheck := func(et tui.EventType, s string) {\n\t\tcheckEvent(et.AsEvent(), s)\n\t}\n\tif len(pairs) != 12 {\n\t\tt.Error(12)\n\t}\n\tcheck(tui.CtrlZ, \"ctrl-z\")\n\tcheck(tui.F2, \"f2\")\n\tcheck(tui.CtrlG, \"ctrl-G\")\n\tcheckEvent(tui.AltKey('z'), \"alt-z\")\n\tcheckEvent(tui.Key('@'), \"@\")\n\tcheckEvent(tui.AltKey('a'), \"Alt-a\")\n\tcheckEvent(tui.Key('!'), \"!\")\n\tcheckEvent(tui.Key('J'), \"J\")\n\tcheckEvent(tui.Key('g'), \"g\")\n\tcheckEvent(tui.CtrlAltKey('a'), \"ctrl-alt-a\")\n\tcheckEvent(tui.CtrlAltKey('m'), \"ALT-enter\")\n\tcheckEvent(tui.AltKey(' '), \"alt-SPACE\")\n\n\t// Synonyms\n\tpairs = parseKeyChords(\"enter,Return,space,tab,btab,esc,up,down,left,right\", \"\")\n\tif len(pairs) != 9 {\n\t\tt.Error(9)\n\t}\n\tcheck(tui.CtrlM, \"Return\")\n\tcheckEvent(tui.Key(' '), \"space\")\n\tcheck(tui.Tab, \"tab\")\n\tcheck(tui.ShiftTab, \"btab\")\n\tcheck(tui.Esc, \"esc\")\n\tcheck(tui.Up, \"up\")\n\tcheck(tui.Down, \"down\")\n\tcheck(tui.Left, \"left\")\n\tcheck(tui.Right, \"right\")\n\n\tpairs = parseKeyChords(\"Tab,Ctrl-I,PgUp,page-up,pgdn,Page-Down,Home,End,Alt-BS,Alt-BSpace,shift-left,shift-right,btab,shift-tab,return,Enter,bspace\", \"\")\n\tif len(pairs) != 11 {\n\t\tt.Error(11)\n\t}\n\tcheck(tui.Tab, \"Ctrl-I\")\n\tcheck(tui.PageUp, \"page-up\")\n\tcheck(tui.PageDown, \"Page-Down\")\n\tcheck(tui.Home, \"Home\")\n\tcheck(tui.End, \"End\")\n\tcheck(tui.AltBackspace, \"Alt-BSpace\")\n\tcheck(tui.ShiftLeft, \"shift-left\")\n\tcheck(tui.ShiftRight, \"shift-right\")\n\tcheck(tui.ShiftTab, \"shift-tab\")\n\tcheck(tui.CtrlM, \"Enter\")\n\tcheck(tui.Backspace, \"bspace\")\n}\n\nfunc TestParseKeysWithComma(t *testing.T) {\n\tcheckN := func(a int, b int) {\n\t\tif a != b {\n\t\t\tt.Errorf(\"%d != %d\", a, b)\n\t\t}\n\t}\n\tcheck := func(pairs map[tui.Event]string, e tui.Event, s string) {\n\t\tif pairs[e] != s {\n\t\t\tt.Errorf(\"%s != %s\", pairs[e], s)\n\t\t}\n\t}\n\n\tpairs := parseKeyChords(\",\", \"\")\n\tcheckN(len(pairs), 1)\n\tcheck(pairs, tui.Key(','), \",\")\n\n\tpairs = parseKeyChords(\",,a,b\", \"\")\n\tcheckN(len(pairs), 3)\n\tcheck(pairs, tui.Key('a'), \"a\")\n\tcheck(pairs, tui.Key('b'), \"b\")\n\tcheck(pairs, tui.Key(','), \",\")\n\n\tpairs = parseKeyChords(\"a,b,,\", \"\")\n\tcheckN(len(pairs), 3)\n\tcheck(pairs, tui.Key('a'), \"a\")\n\tcheck(pairs, tui.Key('b'), \"b\")\n\tcheck(pairs, tui.Key(','), \",\")\n\n\tpairs = parseKeyChords(\"a,,,b\", \"\")\n\tcheckN(len(pairs), 3)\n\tcheck(pairs, tui.Key('a'), \"a\")\n\tcheck(pairs, tui.Key('b'), \"b\")\n\tcheck(pairs, tui.Key(','), \",\")\n\n\tpairs = parseKeyChords(\"a,,,b,c\", \"\")\n\tcheckN(len(pairs), 4)\n\tcheck(pairs, tui.Key('a'), \"a\")\n\tcheck(pairs, tui.Key('b'), \"b\")\n\tcheck(pairs, tui.Key('c'), \"c\")\n\tcheck(pairs, tui.Key(','), \",\")\n\n\tpairs = parseKeyChords(\",,,\", \"\")\n\tcheckN(len(pairs), 1)\n\tcheck(pairs, tui.Key(','), \",\")\n\n\tpairs = parseKeyChords(\",ALT-,,\", \"\")\n\tcheckN(len(pairs), 1)\n\tcheck(pairs, tui.AltKey(','), \"ALT-,\")\n}\n\nfunc TestBind(t *testing.T) {\n\tkeymap := defaultKeymap()\n\tcheck := func(event tui.Event, arg1 string, types ...actionType) {\n\t\tif len(keymap[event]) != len(types) {\n\t\t\tt.Errorf(\"invalid number of actions for %v (%d != %d)\",\n\t\t\t\tevent, len(types), len(keymap[event]))\n\t\t\treturn\n\t\t}\n\t\tfor idx, action := range keymap[event] {\n\t\t\tif types[idx] != action.t {\n\t\t\t\tt.Errorf(\"invalid action type (%d != %d)\", types[idx], action.t)\n\t\t\t}\n\t\t}\n\t\tif len(arg1) > 0 && keymap[event][0].a != arg1 {\n\t\t\tt.Errorf(\"invalid action argument: (%s != %s)\", arg1, keymap[event][0].a)\n\t\t}\n\t}\n\tcheck(tui.CtrlA.AsEvent(), \"\", actBeginningOfLine)\n\terrorString := \"\"\n\terrorFn := func(e string) {\n\t\terrorString = e\n\t}\n\tparseKeymap(keymap,\n\t\t\"ctrl-a:kill-line,ctrl-b:toggle-sort+up+down,c:page-up,alt-z:page-down,\"+\n\t\t\t\"f1:execute(ls {+})+abort+execute(echo \\n{+})+select-all,f2:execute/echo {}, {}, {}/,f3:execute[echo '({})'],f4:execute;less {};,\"+\n\t\t\t\"alt-a:execute-Multi@echo (,),[,],/,:,;,%,{}@,alt-b:execute;echo (,),[,],/,:,@,%,{};,\"+\n\t\t\t\"x:Execute(foo+bar),X:execute/bar+baz/\"+\n\t\t\t\",f1:+first,f1:+top\"+\n\t\t\t\",,:abort,::accept,+:execute:++\\nfoobar,Y:execute(baz)+up\", errorFn)\n\tcheck(tui.CtrlA.AsEvent(), \"\", actKillLine)\n\tcheck(tui.CtrlB.AsEvent(), \"\", actToggleSort, actUp, actDown)\n\tcheck(tui.Key('c'), \"\", actPageUp)\n\tcheck(tui.Key(','), \"\", actAbort)\n\tcheck(tui.Key(':'), \"\", actAccept)\n\tcheck(tui.AltKey('z'), \"\", actPageDown)\n\tcheck(tui.F1.AsEvent(), \"ls {+}\", actExecute, actAbort, actExecute, actSelectAll, actFirst, actFirst)\n\tcheck(tui.F2.AsEvent(), \"echo {}, {}, {}\", actExecute)\n\tcheck(tui.F3.AsEvent(), \"echo '({})'\", actExecute)\n\tcheck(tui.F4.AsEvent(), \"less {}\", actExecute)\n\tcheck(tui.Key('x'), \"foo+bar\", actExecute)\n\tcheck(tui.Key('X'), \"bar+baz\", actExecute)\n\tcheck(tui.AltKey('a'), \"echo (,),[,],/,:,;,%,{}\", actExecuteMulti)\n\tcheck(tui.AltKey('b'), \"echo (,),[,],/,:,@,%,{}\", actExecute)\n\tcheck(tui.Key('+'), \"++\\nfoobar,Y:execute(baz)+up\", actExecute)\n\n\tfor idx, char := range []rune{'~', '!', '@', '#', '$', '%', '^', '&', '*', '|', ';', '/'} {\n\t\tparseKeymap(keymap, fmt.Sprintf(\"%d:execute%cfoobar%c\", idx%10, char, char), errorFn)\n\t\tcheck(tui.Key([]rune(fmt.Sprintf(\"%d\", idx%10))[0]), \"foobar\", actExecute)\n\t}\n\n\tparseKeymap(keymap, \"f1:abort\", errorFn)\n\tcheck(tui.F1.AsEvent(), \"\", actAbort)\n\tif len(errorString) > 0 {\n\t\tt.Errorf(\"error parsing keymap: %s\", errorString)\n\t}\n}\n\nfunc TestColorSpec(t *testing.T) {\n\ttheme := tui.Dark256\n\tdark := parseTheme(theme, \"dark\")\n\tif *dark != *theme {\n\t\tt.Errorf(\"colors should be equivalent\")\n\t}\n\tif dark == theme {\n\t\tt.Errorf(\"point should not be equivalent\")\n\t}\n\n\tlight := parseTheme(theme, \"dark,light\")\n\tif *light == *theme {\n\t\tt.Errorf(\"should not be equivalent\")\n\t}\n\tif *light != *tui.Light256 {\n\t\tt.Errorf(\"colors should be equivalent\")\n\t}\n\tif light == theme {\n\t\tt.Errorf(\"point should not be equivalent\")\n\t}\n\n\tcustomized := parseTheme(theme, \"fg:231,bg:232\")\n\tif customized.Fg.Color != 231 || customized.Bg.Color != 232 {\n\t\tt.Errorf(\"color not customized\")\n\t}\n\tif *tui.Dark256 == *customized {\n\t\tt.Errorf(\"colors should not be equivalent\")\n\t}\n\tcustomized.Fg = tui.Dark256.Fg\n\tcustomized.Bg = tui.Dark256.Bg\n\tif *tui.Dark256 != *customized {\n\t\tt.Errorf(\"colors should now be equivalent: %v, %v\", tui.Dark256, customized)\n\t}\n\n\tcustomized = parseTheme(theme, \"fg:231,dark,bg:232\")\n\tif customized.Fg != tui.Dark256.Fg || customized.Bg == tui.Dark256.Bg {\n\t\tt.Errorf(\"color not customized\")\n\t}\n}\n\nfunc TestDefaultCtrlNP(t *testing.T) {\n\tcheck := func(words []string, et tui.EventType, expected actionType) {\n\t\te := et.AsEvent()\n\t\topts := defaultOptions()\n\t\tparseOptions(opts, words)\n\t\tpostProcessOptions(opts)\n\t\tif opts.Keymap[e][0].t != expected {\n\t\t\tt.Error()\n\t\t}\n\t}\n\tcheck([]string{}, tui.CtrlN, actDown)\n\tcheck([]string{}, tui.CtrlP, actUp)\n\n\tcheck([]string{\"--bind=ctrl-n:accept\"}, tui.CtrlN, actAccept)\n\tcheck([]string{\"--bind=ctrl-p:accept\"}, tui.CtrlP, actAccept)\n\n\tf, _ := os.CreateTemp(\"\", \"fzf-history\")\n\tf.Close()\n\thist := \"--history=\" + f.Name()\n\tcheck([]string{hist}, tui.CtrlN, actNextHistory)\n\tcheck([]string{hist}, tui.CtrlP, actPrevHistory)\n\n\tcheck([]string{hist, \"--bind=ctrl-n:accept\"}, tui.CtrlN, actAccept)\n\tcheck([]string{hist, \"--bind=ctrl-n:accept\"}, tui.CtrlP, actPrevHistory)\n\n\tcheck([]string{hist, \"--bind=ctrl-p:accept\"}, tui.CtrlN, actNextHistory)\n\tcheck([]string{hist, \"--bind=ctrl-p:accept\"}, tui.CtrlP, actAccept)\n}\n\n\nfunc optsFor(words ...string) *Options {\n\topts := defaultOptions()\n\tparseOptions(opts, words)\n\tpostProcessOptions(opts)\n\treturn opts\n}\n\nfunc TestToggle(t *testing.T) {\n\topts := optsFor()\n\tif opts.ToggleSort {\n\t\tt.Error()\n\t}\n\n\topts = optsFor(\"--bind=a:toggle-sort\")\n\tif !opts.ToggleSort {\n\t\tt.Error()\n\t}\n\n\topts = optsFor(\"--bind=a:toggle-sort\", \"--bind=a:up\")\n\tif opts.ToggleSort {\n\t\tt.Error()\n\t}\n}\n\nfunc TestPreviewOpts(t *testing.T) {\n\topts := optsFor()\n\tif !(opts.Preview.command == \"\" &&\n\t\topts.Preview.hidden == false &&\n\t\topts.Preview.wrap == false &&\n\t\topts.Preview.position == posRight &&\n\t\topts.Preview.size.percent == true &&\n\t\topts.Preview.size.size == 50) {\n\t\tt.Error()\n\t}\n\topts = optsFor(\"--preview\", \"cat {}\", \"--preview-window=left:15,hidden,wrap:+{1}-/2\")\n\tif !(opts.Preview.command == \"cat {}\" &&\n\t\topts.Preview.hidden == true &&\n\t\topts.Preview.wrap == true &&\n\t\topts.Preview.position == posLeft &&\n\t\topts.Preview.scroll == \"+{1}-/2\" &&\n\t\topts.Preview.size.percent == false &&\n\t\topts.Preview.size.size == 15) {\n\t\tt.Error(opts.Preview)\n\t}\n\topts = optsFor(\"--preview-window=up,15,wrap,hidden,+{1}+3-1-2/2\", \"--preview-window=down\", \"--preview-window=cycle\")\n\tif !(opts.Preview.command == \"\" &&\n\t\topts.Preview.hidden == true &&\n\t\topts.Preview.wrap == true &&\n\t\topts.Preview.cycle == true &&\n\t\topts.Preview.position == posDown &&\n\t\topts.Preview.scroll == \"+{1}+3-1-2/2\" &&\n\t\topts.Preview.size.percent == false &&\n\t\topts.Preview.size.size == 15) {\n\t\tt.Error(opts.Preview.size.size)\n\t}\n\topts = optsFor(\"--preview-window=up:15:wrap:hidden\")\n\tif !(opts.Preview.command == \"\" &&\n\t\topts.Preview.hidden == true &&\n\t\topts.Preview.wrap == true &&\n\t\topts.Preview.position == posUp &&\n\t\topts.Preview.size.percent == false &&\n\t\topts.Preview.size.size == 15) {\n\t\tt.Error(opts.Preview)\n\t}\n\topts = optsFor(\"--preview=foo\", \"--preview-window=up\", \"--preview-window=default:70%\")\n\tif !(opts.Preview.command == \"foo\" &&\n...\n", "needle_token_start": 14456, "needle_token_end": 14497, "code_context_ntokens": 15240, "output": ["```go\nfunc parseOptions(opts *Options, args []string) {\n\tfor i := 0; i < len(args); {\n\t\targ := args[i]\n\t\tif arg == \"--help\" || arg == \"-h\" {\n\t\t\thelp(0)\n\t\t}\n\t\tif arg == \"--version\" {\n\t\t\tfmt.Println(version)\n\t\t\tos.Exit(0)\n\t\t}\n\t\tif arg == \"--debug\" {\n\t\t\topts.Debug = true\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-mouse\" {\n\t\t\topts.Mouse = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-bold\" {\n\t\t\topts.Bold = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-hscroll\" {\n\t\t\topts.Hscroll = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-separator\" {\n\t\t\topts.Separator = nil\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-scrollbar\" {\n\t\t\topts.Scrollbar = nil\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-sort\" {\n\t\t\topts.Sort = 0\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-track\" {\n\t\t\topts.Track = trackDisabled\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-tac\" {\n\t\t\topts.Tac = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-ansi\" {\n\t\t\topts.Ansi = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-cycle\" {\n\t\t\topts.Cycle = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-keep-right\" {\n\t\t\topts.KeepRight = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-print-query\" {\n\t\t\topts.PrintQuery = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-read0\" {\n\t\t\topts.ReadZero = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-sync\" {\n\t\t\topts.Sync = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-version\" {\n\t\t\topts.Version = false\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--no-pprof\" {\n\t\t\topts.CPUProfile = \"\"\n\t\t\topts.MEMProfile = \"\"\n\t\t\topts.BlockProfile = \"\"\n\t\t\topts.MutexProfile = \"\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--pprof\" {\n\t\t\topts.CPUProfile = \"cpu.prof\"\n\t\t\topts.MEMProfile = \"mem.prof\"\n\t\t\topts.BlockProfile = \"block.prof\"\n\t\t\topts.MutexProfile = \"mutex.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile\" {\n\t\t\topts.CPUProfile = \"cpu.prof\"\n\t\t\topts.MEMProfile = \"mem.prof\"\n\t\t\topts.BlockProfile = \"block.prof\"\n\t\t\topts.MutexProfile = \"mutex.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-cpu\" {\n\t\t\topts.CPUProfile = \"cpu.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-mem\" {\n\t\t\topts.MEMProfile = \"mem.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-block\" {\n\t\t\topts.BlockProfile = \"block.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-mutex\" {\n\t\t\topts.MutexProfile = \"mutex.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-pprof\" {\n\t\t\topts.CPUProfile = \"cpu.prof\"\n\t\t\topts.MEMProfile = \"mem.prof\"\n\t\t\topts.BlockProfile = \"block.prof\"\n\t\t\topts.MutexProfile = \"mutex.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-pprof-cpu\" {\n\t\t\topts.CPUProfile = \"cpu.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-pprof-mem\" {\n\t\t\topts.MEMProfile = \"mem.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-pprof-block\" {\n\t\t\topts.BlockProfile = \"block.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-pprof-mutex\" {\n\t\t\topts.MutexProfile = \"mutex.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-pprof-cpu-mem\" {\n\t\t\topts.CPUProfile = \"cpu.prof\"\n\t\t\topts.MEMProfile = \"mem.prof\"\n\t\t\ti++\n\t\t\tcontinue\n\t\t}\n\t\tif arg == \"--profile-pprof-cpu-block\" {\n\t\t\topts.CPUProfile = \"cpu.prof\"\n\t\t\topts.BlockProfile = \"block.prof\"\n\t\t\ti++\n"]}
{"repo": "fatedier/frp", "name": "DialHookCustomTLSHeadByte", "language": "go", "path": "pkg/util/net/dial.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to conditionally prepend a specific byte to the data stream of a network connection, typically used to indicate a custom TLS handshake or protocol modification.\n2. **Input**: The function takes three parameters: a boolean to enable TLS, a boolean to disable the custom TLS head byte, and a network connection.\n3. **Output**: It returns a modified context and network connection, along with any errors that occur during the writing process.\n4. **Procedure**: The function checks if TLS is enabled and the custom TLS head byte is not disabled. If both conditions are met, it writes a predefined byte to the beginning of the network connection's data stream. It then returns the context and connection for further use, handling any errors that might arise during the write operation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/util/metric/date_counter_test.go\npackage metric\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestDateCounter(t *testing.T) {\n\tassert := assert.New(t)\n\n...\n// Path: pkg/util/net/websocket.go\npackage net\n\nimport (\n\t\"errors\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"golang.org/x/net/websocket\"\n)\n\nvar ErrWebsocketListenerClosed = errors.New(\"websocket listener closed\")\n\nconst (\n\tFrpWebsocketPath = \"/~!frp\"\n)\n\ntype WebsocketListener struct {\n\tln       net.Listener\n\tacceptCh chan net.Conn\n\n\tserver *http.Server\n}\n\n// NewWebsocketListener to handle websocket connections\n// ln: tcp listener for websocket connections\nfunc NewWebsocketListener(ln net.Listener) (wl *WebsocketListener) {\n\twl = &WebsocketListener{\n\t\tacceptCh: make(chan net.Conn),\n\t}\n\n\tmuxer := http.NewServeMux()\n\tmuxer.Handle(FrpWebsocketPath, websocket.Handler(func(c *websocket.Conn) {\n\t\tnotifyCh := make(chan struct{})\n\t\tconn := WrapCloseNotifyConn(c, func() {\n\t\t\tclose(notifyCh)\n\t\t})\n\t\twl.acceptCh <- conn\n\t\t<-notifyCh\n\t}))\n\n\twl.server = &http.Server{\n\t\tAddr:              ln.Addr().String(),\n\t\tHandler:           muxer,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\n\tgo func() {\n\t\t_ = wl.server.Serve(ln)\n\t}()\n\treturn\n}\n\nfunc (p *WebsocketListener) Accept() (net.Conn, error) {\n\tc, ok := <-p.acceptCh\n\tif !ok {\n\t\treturn nil, ErrWebsocketListenerClosed\n\t}\n\treturn c, nil\n}\n\nfunc (p *WebsocketListener) Close() error {\n\treturn p.server.Close()\n}\n\nfunc (p *WebsocketListener) Addr() net.Addr {\n\treturn p.ln.Addr()\n}\n\n// Path: pkg/util/net/dial.go\npackage net\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"net/url\"\n\n\tlibnet \"github.com/fatedier/golib/net\"\n\t\"golang.org/x/net/websocket\"\n)\n\n\nfunc DialHookCustomTLSHeadByte(enableTLS bool, disableCustomTLSHeadByte bool) libnet.AfterHookFunc {\n\treturn func(ctx context.Context, c net.Conn, addr string) (context.Context, net.Conn, error) {\n\t\tif enableTLS && !disableCustomTLSHeadByte {\n\t\t\t_, err := c.Write([]byte{byte(FRPTLSHeadByte)})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\t\treturn ctx, c, nil\n\t}\n}\n\nfunc DialHookWebsocket(protocol string, host string) libnet.AfterHookFunc {\n\treturn func(ctx context.Context, c net.Conn, addr string) (context.Context, net.Conn, error) {\n\t\tif protocol != \"wss\" {\n\t\t\tprotocol = \"ws\"\n\t\t}\n\t\tif host == \"\" {\n\t\t\thost = addr\n\t\t}\n\t\taddr = protocol + \"://\" + host + FrpWebsocketPath\n\t\turi, err := url.Parse(addr)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\torigin := \"http://\" + uri.Host\n\t\tcfg, err := websocket.NewConfig(addr, origin)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\n\t\tconn, err := websocket.NewClient(cfg, c)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\treturn ctx, conn, nil\n\t}\n}\n\n// Path: pkg/util/net/dns.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage net\n\nimport (\n\t\"context\"\n\t\"net\"\n)\n\nfunc SetDefaultDNSAddress(dnsAddress string) {\n\tif _, _, err := net.SplitHostPort(dnsAddress); err != nil {\n\t\tdnsAddress = net.JoinHostPort(dnsAddress, \"53\")\n\t}\n\t// Change default dns server\n\tnet.DefaultResolver = &net.Resolver{\n\t\tPreferGo: true,\n\t\tDial: func(ctx context.Context, network, _ string) (net.Conn, error) {\n\t\t\treturn net.Dial(network, dnsAddress)\n\t\t},\n\t}\n}\n\n// Path: pkg/util/net/listener.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage net\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/fatedier/golib/errors\"\n)\n\n// InternalListener is a listener that can be used to accept connections from\n// other goroutines.\ntype InternalListener struct {\n\tacceptCh chan net.Conn\n\tclosed   bool\n\tmu       sync.Mutex\n}\n\nfunc NewInternalListener() *InternalListener {\n\treturn &InternalListener{\n\t\tacceptCh: make(chan net.Conn, 128),\n\t}\n}\n\nfunc (l *InternalListener) Accept() (net.Conn, error) {\n\tconn, ok := <-l.acceptCh\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"listener closed\")\n\t}\n\treturn conn, nil\n}\n\nfunc (l *InternalListener) PutConn(conn net.Conn) error {\n\terr := errors.PanicToError(func() {\n\t\tselect {\n\t\tcase l.acceptCh <- conn:\n\t\tdefault:\n\t\t\tconn.Close()\n\t\t}\n\t})\n\tif err != nil {\n\t\treturn fmt.Errorf(\"put conn error: listener is closed\")\n\t}\n\treturn nil\n}\n\nfunc (l *InternalListener) Close() error {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\tif !l.closed {\n\t\tclose(l.acceptCh)\n\t\tl.closed = true\n\t}\n\treturn nil\n}\n\nfunc (l *InternalListener) Addr() net.Addr {\n\treturn &InternalAddr{}\n}\n\ntype InternalAddr struct{}\n\nfunc (ia *InternalAddr) Network() string {\n\treturn \"internal\"\n}\n\nfunc (ia *InternalAddr) String() string {\n\treturn \"internal\"\n}\n\n// Path: pkg/util/net/kcp.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage net\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\n\tkcp \"github.com/xtaci/kcp-go/v5\"\n)\n\ntype KCPListener struct {\n\tlistener  net.Listener\n\tacceptCh  chan net.Conn\n\tcloseFlag bool\n}\n\nfunc ListenKcp(address string) (l *KCPListener, err error) {\n\tlistener, err := kcp.ListenWithOptions(address, nil, 10, 3)\n\tif err != nil {\n\t\treturn l, err\n\t}\n\t_ = listener.SetReadBuffer(4194304)\n\t_ = listener.SetWriteBuffer(4194304)\n\n\tl = &KCPListener{\n\t\tlistener:  listener,\n\t\tacceptCh:  make(chan net.Conn),\n\t\tcloseFlag: false,\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := listener.AcceptKCP()\n\t\t\tif err != nil {\n\t\t\t\tif l.closeFlag {\n\t\t\t\t\tclose(l.acceptCh)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tconn.SetStreamMode(true)\n\t\t\tconn.SetWriteDelay(true)\n\t\t\tconn.SetNoDelay(1, 20, 2, 1)\n\t\t\tconn.SetMtu(1350)\n\t\t\tconn.SetWindowSize(1024, 1024)\n\t\t\tconn.SetACKNoDelay(false)\n\n\t\t\tl.acceptCh <- conn\n\t\t}\n\t}()\n\treturn l, err\n}\n\nfunc (l *KCPListener) Accept() (net.Conn, error) {\n\tconn, ok := <-l.acceptCh\n\tif !ok {\n\t\treturn conn, fmt.Errorf(\"channel for kcp listener closed\")\n\t}\n\treturn conn, nil\n}\n\nfunc (l *KCPListener) Close() error {\n\tif !l.closeFlag {\n\t\tl.closeFlag = true\n\t\tl.listener.Close()\n\t}\n\treturn nil\n}\n\nfunc (l *KCPListener) Addr() net.Addr {\n\treturn l.listener.Addr()\n}\n\nfunc NewKCPConnFromUDP(conn *net.UDPConn, connected bool, raddr string) (net.Conn, error) {\n\tudpAddr, err := net.ResolveUDPAddr(\"udp\", raddr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar pConn net.PacketConn = conn\n\tif connected {\n\t\tpConn = &ConnectedUDPConn{conn}\n\t}\n\tkcpConn, err := kcp.NewConn3(1, udpAddr, nil, 10, 3, pConn)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tkcpConn.SetStreamMode(true)\n\tkcpConn.SetWriteDelay(true)\n\tkcpConn.SetNoDelay(1, 20, 2, 1)\n\tkcpConn.SetMtu(1350)\n\tkcpConn.SetWindowSize(1024, 1024)\n\tkcpConn.SetACKNoDelay(false)\n\treturn kcpConn, nil\n}\n\n// Path: pkg/util/net/udp.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage net\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"strconv\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/pool\"\n)\n\ntype UDPPacket struct {\n\tBuf        []byte\n\tLocalAddr  net.Addr\n\tRemoteAddr net.Addr\n}\n\ntype FakeUDPConn struct {\n\tl *UDPListener\n\n\tlocalAddr  net.Addr\n\tremoteAddr net.Addr\n\tpackets    chan []byte\n\tcloseFlag  bool\n\n\tlastActive time.Time\n\tmu         sync.RWMutex\n}\n\nfunc NewFakeUDPConn(l *UDPListener, laddr, raddr net.Addr) *FakeUDPConn {\n\tfc := &FakeUDPConn{\n\t\tl:          l,\n\t\tlocalAddr:  laddr,\n\t\tremoteAddr: raddr,\n\t\tpackets:    make(chan []byte, 20),\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(5 * time.Second)\n\t\t\tfc.mu.RLock()\n\t\t\tif time.Since(fc.lastActive) > 10*time.Second {\n\t\t\t\tfc.mu.RUnlock()\n\t\t\t\tfc.Close()\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tfc.mu.RUnlock()\n\t\t}\n\t}()\n\treturn fc\n}\n\nfunc (c *FakeUDPConn) putPacket(content []byte) {\n\tdefer func() {\n\t\t_ = recover()\n\t}()\n\n\tselect {\n\tcase c.packets <- content:\n\tdefault:\n\t}\n}\n\nfunc (c *FakeUDPConn) Read(b []byte) (n int, err error) {\n\tcontent, ok := <-c.packets\n\tif !ok {\n\t\treturn 0, io.EOF\n\t}\n\tc.mu.Lock()\n\tc.lastActive = time.Now()\n\tc.mu.Unlock()\n\n\tif len(b) < len(content) {\n\t\tn = len(b)\n\t} else {\n\t\tn = len(content)\n\t}\n\tcopy(b, content)\n\treturn n, nil\n}\n\nfunc (c *FakeUDPConn) Write(b []byte) (n int, err error) {\n\tc.mu.RLock()\n\tif c.closeFlag {\n\t\tc.mu.RUnlock()\n\t\treturn 0, io.ErrClosedPipe\n\t}\n\tc.mu.RUnlock()\n\n\tpacket := &UDPPacket{\n\t\tBuf:        b,\n\t\tLocalAddr:  c.localAddr,\n\t\tRemoteAddr: c.remoteAddr,\n\t}\n\t_ = c.l.writeUDPPacket(packet)\n\n\tc.mu.Lock()\n\tc.lastActive = time.Now()\n\tc.mu.Unlock()\n\treturn len(b), nil\n}\n\nfunc (c *FakeUDPConn) Close() error {\n\tc.mu.Lock()\n\tdefer c.mu.Unlock()\n\tif !c.closeFlag {\n\t\tc.closeFlag = true\n\t\tclose(c.packets)\n\t}\n\treturn nil\n}\n\nfunc (c *FakeUDPConn) IsClosed() bool {\n\tc.mu.RLock()\n\tdefer c.mu.RUnlock()\n\treturn c.closeFlag\n}\n\nfunc (c *FakeUDPConn) LocalAddr() net.Addr {\n\treturn c.localAddr\n}\n\nfunc (c *FakeUDPConn) RemoteAddr() net.Addr {\n\treturn c.remoteAddr\n}\n\nfunc (c *FakeUDPConn) SetDeadline(_ time.Time) error {\n\treturn nil\n}\n\nfunc (c *FakeUDPConn) SetReadDeadline(_ time.Time) error {\n\treturn nil\n}\n\nfunc (c *FakeUDPConn) SetWriteDeadline(_ time.Time) error {\n\treturn nil\n}\n\ntype UDPListener struct {\n\taddr      net.Addr\n\tacceptCh  chan net.Conn\n\twriteCh   chan *UDPPacket\n\treadConn  net.Conn\n\tcloseFlag bool\n\n\tfakeConns map[string]*FakeUDPConn\n}\n\nfunc ListenUDP(bindAddr string, bindPort int) (l *UDPListener, err error) {\n\tudpAddr, err := net.ResolveUDPAddr(\"udp\", net.JoinHostPort(bindAddr, strconv.Itoa(bindPort)))\n\tif err != nil {\n\t\treturn l, err\n\t}\n\treadConn, err := net.ListenUDP(\"udp\", udpAddr)\n\n\tl = &UDPListener{\n\t\taddr:      udpAddr,\n\t\tacceptCh:  make(chan net.Conn),\n\t\twriteCh:   make(chan *UDPPacket, 1000),\n\t\tfakeConns: make(map[string]*FakeUDPConn),\n\t}\n\n\t// for reading\n\tgo func() {\n\t\tfor {\n\t\t\tbuf := pool.GetBuf(1450)\n\t\t\tn, remoteAddr, err := readConn.ReadFromUDP(buf)\n\t\t\tif err != nil {\n\t\t\t\tclose(l.acceptCh)\n\t\t\t\tclose(l.writeCh)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfakeConn, exist := l.fakeConns[remoteAddr.String()]\n\t\t\tif !exist || fakeConn.IsClosed() {\n\t\t\t\tfakeConn = NewFakeUDPConn(l, l.Addr(), remoteAddr)\n\t\t\t\tl.fakeConns[remoteAddr.String()] = fakeConn\n\t\t\t}\n\t\t\tfakeConn.putPacket(buf[:n])\n\n\t\t\tl.acceptCh <- fakeConn\n\t\t}\n\t}()\n\n\t// for writing\n\tgo func() {\n\t\tfor {\n\t\t\tpacket, ok := <-l.writeCh\n\t\t\tif !ok {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif addr, ok := packet.RemoteAddr.(*net.UDPAddr); ok {\n\t\t\t\t_, _ = readConn.WriteToUDP(packet.Buf, addr)\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn\n}\n\nfunc (l *UDPListener) writeUDPPacket(packet *UDPPacket) (err error) {\n\tdefer func() {\n\t\tif errRet := recover(); errRet != nil {\n\t\t\terr = fmt.Errorf(\"udp write closed listener\")\n\t\t}\n\t}()\n\tl.writeCh <- packet\n\treturn\n}\n\nfunc (l *UDPListener) WriteMsg(buf []byte, remoteAddr *net.UDPAddr) (err error) {\n\t// only set remote addr here\n\tpacket := &UDPPacket{\n\t\tBuf:        buf,\n\t\tRemoteAddr: remoteAddr,\n\t}\n\terr = l.writeUDPPacket(packet)\n\treturn\n}\n\nfunc (l *UDPListener) Accept() (net.Conn, error) {\n\tconn, ok := <-l.acceptCh\n\tif !ok {\n\t\treturn conn, fmt.Errorf(\"channel for udp listener closed\")\n\t}\n\treturn conn, nil\n}\n\nfunc (l *UDPListener) Close() error {\n\tif !l.closeFlag {\n\t\tl.closeFlag = true\n\t\tif l.readConn != nil {\n\t\t\tl.readConn.Close()\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (l *UDPListener) Addr() net.Addr {\n\treturn l.addr\n}\n\n// ConnectedUDPConn is a wrapper for net.UDPConn which converts WriteTo syscalls\n// to Write syscalls that are 4 times faster on some OS'es. This should only be\n// used for connections that were produced by a net.Dial* call.\ntype ConnectedUDPConn struct{ *net.UDPConn }\n\n// WriteTo redirects all writes to the Write syscall, which is 4 times faster.\nfunc (c *ConnectedUDPConn) WriteTo(b []byte, _ net.Addr) (int, error) { return c.Write(b) }\n\n// Path: pkg/util/system/system.go\n// Copyright 2024 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !android\n\npackage system\n\n// EnableCompatibilityMode enables compatibility mode for different system.\n// For example, on Android, the inability to obtain the correct time zone will result in incorrect log time output.\nfunc EnableCompatibilityMode() {\n}\n\n// Path: pkg/util/system/system_android.go\n// Copyright 2024 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage system\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"time\"\n)\n\nfunc EnableCompatibilityMode() {\n\tfixTimezone()\n\tfixDNSResolver()\n}\n\n// fixTimezone is used to try our best to fix timezone issue on some Android devices.\nfunc fixTimezone() {\n\tout, err := exec.Command(\"/system/bin/getprop\", \"persist.sys.timezone\").Output()\n\tif err != nil {\n\t\treturn\n\t}\n\tloc, err := time.LoadLocation(strings.TrimSpace(string(out)))\n\tif err != nil {\n\t\treturn\n\t}\n\ttime.Local = loc\n}\n\n// fixDNSResolver will first attempt to resolve google.com to check if the current DNS is available.\n// If it is not available, it will default to using 8.8.8.8 as the DNS server.\n// This is a workaround for the issue that golang can't get the default DNS servers on Android.\nfunc fixDNSResolver() {\n\t// First, we attempt to resolve a domain. If resolution is successful, no modifications are necessary.\n\t// In real-world scenarios, users may have already configured /etc/resolv.conf, or compiled directly\n\t// in the Android environment instead of using cross-platform compilation, so this issue does not arise.\n\tif net.DefaultResolver != nil {\n\t\ttimeoutCtx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\tdefer cancel()\n\t\t_, err := net.DefaultResolver.LookupHost(timeoutCtx, \"google.com\")\n\t\tif err == nil {\n\t\t\treturn\n\t\t}\n\t}\n\t// If the resolution fails, use 8.8.8.8 as the DNS server.\n\t// Note: If there are other methods to obtain the default DNS servers, the default DNS servers should be used preferentially.\n\tnet.DefaultResolver = &net.Resolver{\n\t\tPreferGo: true,\n\t\tDial: func(ctx context.Context, network, _ string) (net.Conn, error) {\n\t\t\treturn net.Dial(network, \"8.8.8.8:53\")\n\t\t},\n\t}\n}\n\n// Path: pkg/util/vhost/vhost.go\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/errors\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n\t\"github.com/fatedier/frp/pkg/util/xlog\"\n)\n\ntype RouteInfo string\n\nconst (\n\tRouteInfoKey RouteInfo = \"routeInfo\"\n)\n\ntype RequestRouteInfo struct {\n\tURL        string\n\tHost       string\n\tHTTPUser   string\n\tRemoteAddr string\n\tURLHost    string\n\tEndpoint   string\n}\n\ntype (\n\tmuxFunc         func(net.Conn) (net.Conn, map[string]string, error)\n\tauthFunc        func(conn net.Conn, username, password string, reqInfoMap map[string]string) (bool, error)\n\thostRewriteFunc func(net.Conn, string) (net.Conn, error)\n\tsuccessHookFunc func(net.Conn, map[string]string) error\n\tfailHookFunc    func(net.Conn)\n)\n\n// Muxer is a functional component used for https and tcpmux proxies.\n// It accepts connections and extracts vhost information from the beginning of the connection data.\n// It then routes the connection to its appropriate listener.\ntype Muxer struct {\n\tlistener net.Listener\n\ttimeout  time.Duration\n\n\tvhostFunc      muxFunc\n\tcheckAuth      authFunc\n\tsuccessHook    successHookFunc\n\tfailHook       failHookFunc\n\trewriteHost    hostRewriteFunc\n\tregistryRouter *Routers\n}\n\nfunc NewMuxer(\n\tlistener net.Listener,\n\tvhostFunc muxFunc,\n\ttimeout time.Duration,\n) (mux *Muxer, err error) {\n\tmux = &Muxer{\n\t\tlistener:       listener,\n\t\ttimeout:        timeout,\n\t\tvhostFunc:      vhostFunc,\n\t\tregistryRouter: NewRouters(),\n\t}\n\tgo mux.run()\n\treturn mux, nil\n}\n\nfunc (v *Muxer) SetCheckAuthFunc(f authFunc) *Muxer {\n\tv.checkAuth = f\n\treturn v\n}\n\nfunc (v *Muxer) SetSuccessHookFunc(f successHookFunc) *Muxer {\n\tv.successHook = f\n\treturn v\n}\n\nfunc (v *Muxer) SetFailHookFunc(f failHookFunc) *Muxer {\n\tv.failHook = f\n\treturn v\n}\n\nfunc (v *Muxer) SetRewriteHostFunc(f hostRewriteFunc) *Muxer {\n\tv.rewriteHost = f\n\treturn v\n}\n\ntype ChooseEndpointFunc func() (string, error)\n\ntype CreateConnFunc func(remoteAddr string) (net.Conn, error)\n\ntype CreateConnByEndpointFunc func(endpoint, remoteAddr string) (net.Conn, error)\n\n// RouteConfig is the params used to match HTTP requests\ntype RouteConfig struct {\n\tDomain          string\n\tLocation        string\n\tRewriteHost     string\n\tUsername        string\n\tPassword        string\n\tHeaders         map[string]string\n\tRouteByHTTPUser string\n\n\tCreateConnFn           CreateConnFunc\n\tChooseEndpointFn       ChooseEndpointFunc\n\tCreateConnByEndpointFn CreateConnByEndpointFunc\n}\n\n// listen for a new domain name, if rewriteHost is not empty and rewriteHost func is not nil,\n// then rewrite the host header to rewriteHost\nfunc (v *Muxer) Listen(ctx context.Context, cfg *RouteConfig) (l *Listener, err error) {\n\tl = &Listener{\n\t\tname:            cfg.Domain,\n\t\tlocation:        cfg.Location,\n\t\trouteByHTTPUser: cfg.RouteByHTTPUser,\n\t\trewriteHost:     cfg.RewriteHost,\n\t\tusername:        cfg.Username,\n\t\tpassword:        cfg.Password,\n\t\tmux:             v,\n\t\taccept:          make(chan net.Conn),\n\t\tctx:             ctx,\n\t}\n\terr = v.registryRouter.Add(cfg.Domain, cfg.Location, cfg.RouteByHTTPUser, l)\n\tif err != nil {\n\t\treturn\n\t}\n\treturn l, nil\n}\n\nfunc (v *Muxer) getListener(name, path, httpUser string) (*Listener, bool) {\n\tfindRouter := func(inName, inPath, inHTTPUser string) (*Listener, bool) {\n\t\tvr, ok := v.registryRouter.Get(inName, inPath, inHTTPUser)\n\t\tif ok {\n\t\t\treturn vr.payload.(*Listener), true\n\t\t}\n\t\t// Try to check if there is one proxy that doesn't specify routerByHTTPUser, it means match all.\n\t\tvr, ok = v.registryRouter.Get(inName, inPath, \"\")\n\t\tif ok {\n\t\t\treturn vr.payload.(*Listener), true\n\t\t}\n\t\treturn nil, false\n\t}\n\n\t// first we check the full hostname\n\t// if not exist, then check the wildcard_domain such as *.example.com\n\tl, ok := findRouter(name, path, httpUser)\n\tif ok {\n\t\treturn l, true\n\t}\n\n\tdomainSplit := strings.Split(name, \".\")\n\tfor {\n\t\tif len(domainSplit) < 3 {\n\t\t\tbreak\n\t\t}\n\n\t\tdomainSplit[0] = \"*\"\n\t\tname = strings.Join(domainSplit, \".\")\n\n\t\tl, ok = findRouter(name, path, httpUser)\n\t\tif ok {\n\t\t\treturn l, true\n\t\t}\n\t\tdomainSplit = domainSplit[1:]\n\t}\n\t// Finally, try to check if there is one proxy that domain is \"*\" means match all domains.\n\tl, ok = findRouter(\"*\", path, httpUser)\n\tif ok {\n\t\treturn l, true\n\t}\n\treturn nil, false\n}\n\nfunc (v *Muxer) run() {\n\tfor {\n\t\tconn, err := v.listener.Accept()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tgo v.handle(conn)\n\t}\n}\n\nfunc (v *Muxer) handle(c net.Conn) {\n\tif err := c.SetDeadline(time.Now().Add(v.timeout)); err != nil {\n\t\t_ = c.Close()\n\t\treturn\n\t}\n\n\tsConn, reqInfoMap, err := v.vhostFunc(c)\n\tif err != nil {\n\t\tlog.Debugf(\"get hostname from http/https request error: %v\", err)\n\t\t_ = c.Close()\n\t\treturn\n\t}\n\n\tname := strings.ToLower(reqInfoMap[\"Host\"])\n\tpath := strings.ToLower(reqInfoMap[\"Path\"])\n\thttpUser := reqInfoMap[\"HTTPUser\"]\n\tl, ok := v.getListener(name, path, httpUser)\n\tif !ok {\n\t\tlog.Debugf(\"http request for host [%s] path [%s] httpUser [%s] not found\", name, path, httpUser)\n\t\tv.failHook(sConn)\n\t\treturn\n\t}\n\n\txl := xlog.FromContextSafe(l.ctx)\n\tif v.successHook != nil {\n\t\tif err := v.successHook(c, reqInfoMap); err != nil {\n\t\t\txl.Infof(\"success func failure on vhost connection: %v\", err)\n\t\t\t_ = c.Close()\n\t\t\treturn\n\t\t}\n\t}\n\n\t// if checkAuth func is exist and username/password is set\n\t// then verify user access\n\tif l.mux.checkAuth != nil && l.username != \"\" {\n\t\tok, err := l.mux.checkAuth(c, l.username, l.password, reqInfoMap)\n\t\tif !ok || err != nil {\n\t\t\txl.Debugf(\"auth failed for user: %s\", l.username)\n\t\t\t_ = c.Close()\n\t\t\treturn\n\t\t}\n\t}\n\n\tif err = sConn.SetDeadline(time.Time{}); err != nil {\n\t\t_ = c.Close()\n\t\treturn\n\t}\n\tc = sConn\n\n\txl.Debugf(\"new request host [%s] path [%s] httpUser [%s]\", name, path, httpUser)\n\terr = errors.PanicToError(func() {\n\t\tl.accept <- c\n\t})\n\tif err != nil {\n\t\txl.Warnf(\"listener is already closed, ignore this request\")\n\t}\n}\n\ntype Listener struct {\n\tname            string\n\tlocation        string\n\trouteByHTTPUser string\n\trewriteHost     string\n\tusername        string\n\tpassword        string\n\tmux             *Muxer // for closing Muxer\n\taccept          chan net.Conn\n\tctx             context.Context\n}\n\nfunc (l *Listener) Accept() (net.Conn, error) {\n\txl := xlog.FromContextSafe(l.ctx)\n\tconn, ok := <-l.accept\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"Listener closed\")\n\t}\n\n\t// if rewriteHost func is exist\n\t// rewrite http requests with a modified host header\n\t// if l.rewriteHost is empty, nothing to do\n\tif l.mux.rewriteHost != nil {\n\t\tsConn, err := l.mux.rewriteHost(conn, l.rewriteHost)\n\t\tif err != nil {\n\t\t\txl.Warnf(\"host header rewrite failed: %v\", err)\n\t\t\treturn nil, fmt.Errorf(\"host header rewrite failed\")\n\t\t}\n\t\txl.Debugf(\"rewrite host to [%s] success\", l.rewriteHost)\n\t\tconn = sConn\n\t}\n\treturn netpkg.NewContextConn(l.ctx, conn), nil\n}\n\nfunc (l *Listener) Close() error {\n\tl.mux.registryRouter.Del(l.name, l.location, l.routeByHTTPUser)\n\tclose(l.accept)\n\treturn nil\n}\n\nfunc (l *Listener) Name() string {\n\treturn l.name\n}\n\nfunc (l *Listener) Addr() net.Addr {\n\treturn (*net.TCPAddr)(nil)\n}\n\n// Path: pkg/util/tcpmux/httpconnect.go\n// Copyright 2020 guylewin, guy@lewin.co.il\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tcpmux\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n\n\tlibnet \"github.com/fatedier/golib/net\"\n\n\thttppkg \"github.com/fatedier/frp/pkg/util/http\"\n\t\"github.com/fatedier/frp/pkg/util/vhost\"\n)\n\ntype HTTPConnectTCPMuxer struct {\n\t*vhost.Muxer\n\n\t// If passthrough is set to true, the CONNECT request will be forwarded to the backend service.\n\t// Otherwise, it will return an OK response to the client and forward the remaining content to the backend service.\n\tpassthrough bool\n}\n\nfunc NewHTTPConnectTCPMuxer(listener net.Listener, passthrough bool, timeout time.Duration) (*HTTPConnectTCPMuxer, error) {\n\tret := &HTTPConnectTCPMuxer{passthrough: passthrough}\n\tmux, err := vhost.NewMuxer(listener, ret.getHostFromHTTPConnect, timeout)\n\tmux.SetCheckAuthFunc(ret.auth).\n\t\tSetSuccessHookFunc(ret.sendConnectResponse).\n\t\tSetFailHookFunc(vhostFailed)\n\tret.Muxer = mux\n\treturn ret, err\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) readHTTPConnectRequest(rd io.Reader) (host, httpUser, httpPwd string, err error) {\n\tbufioReader := bufio.NewReader(rd)\n\n\treq, err := http.ReadRequest(bufioReader)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tif req.Method != \"CONNECT\" {\n\t\terr = fmt.Errorf(\"connections to tcp vhost must be of method CONNECT\")\n\t\treturn\n\t}\n\n\thost, _ = httppkg.CanonicalHost(req.Host)\n\tproxyAuth := req.Header.Get(\"Proxy-Authorization\")\n\tif proxyAuth != \"\" {\n\t\thttpUser, httpPwd, _ = httppkg.ParseBasicAuth(proxyAuth)\n\t}\n\treturn\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) sendConnectResponse(c net.Conn, _ map[string]string) error {\n\tif muxer.passthrough {\n\t\treturn nil\n\t}\n\tres := httppkg.OkResponse()\n\tif res.Body != nil {\n\t\tdefer res.Body.Close()\n\t}\n\treturn res.Write(c)\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) auth(c net.Conn, username, password string, reqInfo map[string]string) (bool, error) {\n\treqUsername := reqInfo[\"HTTPUser\"]\n\treqPassword := reqInfo[\"HTTPPwd\"]\n\tif username == reqUsername && password == reqPassword {\n\t\treturn true, nil\n\t}\n\n\tresp := httppkg.ProxyUnauthorizedResponse()\n\tif resp.Body != nil {\n\t\tdefer resp.Body.Close()\n\t}\n\t_ = resp.Write(c)\n\treturn false, nil\n}\n\nfunc vhostFailed(c net.Conn) {\n\tres := vhost.NotFoundResponse()\n\tif res.Body != nil {\n\t\tdefer res.Body.Close()\n\t}\n\t_ = res.Write(c)\n\t_ = c.Close()\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) getHostFromHTTPConnect(c net.Conn) (net.Conn, map[string]string, error) {\n\treqInfoMap := make(map[string]string, 0)\n\tsc, rd := libnet.NewSharedConn(c)\n\n\thost, httpUser, httpPwd, err := muxer.readHTTPConnectRequest(rd)\n\tif err != nil {\n\t\treturn nil, reqInfoMap, err\n\t}\n\n\treqInfoMap[\"Host\"] = host\n\treqInfoMap[\"Scheme\"] = \"tcp\"\n\treqInfoMap[\"HTTPUser\"] = httpUser\n\treqInfoMap[\"HTTPPwd\"] = httpPwd\n\n\toutConn := c\n\tif muxer.passthrough {\n\t\toutConn = sc\n\t}\n\treturn outConn, reqInfoMap, nil\n}\n\n// Path: pkg/util/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRandId(t *testing.T) {\n\tassert := assert.New(t)\n\tid, err := RandID()\n\tassert.NoError(err)\n\tt.Log(id)\n\tassert.Equal(16, len(id))\n}\n\nfunc TestGetAuthKey(t *testing.T) {\n\tassert := assert.New(t)\n\tkey := GetAuthKey(\"1234\", 1488720000)\n\tassert.Equal(\"6df41a43725f0c770fd56379e12acf8c\", key)\n}\n\nfunc TestParseRangeNumbers(t *testing.T) {\n\tassert := assert.New(t)\n\tnumbers, err := ParseRangeNumbers(\"2-5\")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{2, 3, 4, 5}, numbers)\n\t}\n\n\tnumbers, err = ParseRangeNumbers(\"1\")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{1}, numbers)\n\t}\n\n\tnumbers, err = ParseRangeNumbers(\"3-5,8\")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{3, 4, 5, 8}, numbers)\n\t}\n\n\tnumbers, err = ParseRangeNumbers(\" 3-5,8, 10-12 \")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{3, 4, 5, 8, 10, 11, 12}, numbers)\n\t}\n\n\t_, err = ParseRangeNumbers(\"3-a\")\n\tassert.Error(err)\n}\n\n// Path: pkg/util/version/version.go\n// Copyright 2016 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage version\n\nvar version = \"0.57.0\"\n\nfunc Full() string {\n\treturn version\n}\n\n// Path: pkg/util/vhost/https.go\n// Copyright 2016 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"crypto/tls\"\n\t\"io\"\n\t\"net\"\n\t\"time\"\n\n\tlibnet \"github.com/fatedier/golib/net\"\n)\n\ntype HTTPSMuxer struct {\n\t*Muxer\n}\n\nfunc NewHTTPSMuxer(listener net.Listener, timeout time.Duration) (*HTTPSMuxer, error) {\n\tmux, err := NewMuxer(listener, GetHTTPSHostname, timeout)\n\tmux.SetFailHookFunc(vhostFailed)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &HTTPSMuxer{mux}, err\n}\n\nfunc GetHTTPSHostname(c net.Conn) (_ net.Conn, _ map[string]string, err error) {\n\treqInfoMap := make(map[string]string, 0)\n\tsc, rd := libnet.NewSharedConn(c)\n\n\tclientHello, err := readClientHello(rd)\n\tif err != nil {\n\t\treturn nil, reqInfoMap, err\n\t}\n\n\treqInfoMap[\"Host\"] = clientHello.ServerName\n\treqInfoMap[\"Scheme\"] = \"https\"\n\treturn sc, reqInfoMap, nil\n}\n\nfunc readClientHello(reader io.Reader) (*tls.ClientHelloInfo, error) {\n\tvar hello *tls.ClientHelloInfo\n\n\t// Note that Handshake always fails because the readOnlyConn is not a real connection.\n\t// As long as the Client Hello is successfully read, the failure should only happen after GetConfigForClient is called,\n\t// so we only care about the error if hello was never set.\n\terr := tls.Server(readOnlyConn{reader: reader}, &tls.Config{\n\t\tGetConfigForClient: func(argHello *tls.ClientHelloInfo) (*tls.Config, error) {\n\t\t\thello = &tls.ClientHelloInfo{}\n\t\t\t*hello = *argHello\n\t\t\treturn nil, nil\n\t\t},\n\t}).Handshake()\n\n\tif hello == nil {\n\t\treturn nil, err\n\t}\n\treturn hello, nil\n}\n\nfunc vhostFailed(c net.Conn) {\n\t// Alert with alertUnrecognizedName\n\t_ = tls.Server(c, &tls.Config{}).Handshake()\n\tc.Close()\n}\n\ntype readOnlyConn struct {\n\treader io.Reader\n}\n\nfunc (conn readOnlyConn) Read(p []byte) (int, error)         { return conn.reader.Read(p) }\nfunc (conn readOnlyConn) Write(_ []byte) (int, error)        { return 0, io.ErrClosedPipe }\nfunc (conn readOnlyConn) Close() error                       { return nil }\nfunc (conn readOnlyConn) LocalAddr() net.Addr                { return nil }\nfunc (conn readOnlyConn) RemoteAddr() net.Addr               { return nil }\nfunc (conn readOnlyConn) SetDeadline(_ time.Time) error      { return nil }\nfunc (conn readOnlyConn) SetReadDeadline(_ time.Time) error  { return nil }\nfunc (conn readOnlyConn) SetWriteDeadline(_ time.Time) error { return nil }\n\n// Path: pkg/util/vhost/https_test.go\npackage vhost\n\nimport (\n\t\"crypto/tls\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestGetHTTPSHostname(t *testing.T) {\n\trequire := require.New(t)\n\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:\")\n\trequire.NoError(err)\n\tdefer l.Close()\n\n\tvar conn net.Conn\n\tgo func() {\n\t\tconn, _ = l.Accept()\n\t\trequire.NotNil(conn)\n\t}()\n\n\tgo func() {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\ttls.Dial(\"tcp\", l.Addr().String(), &tls.Config{\n\t\t\tInsecureSkipVerify: true,\n\t\t\tServerName:         \"example.com\",\n\t\t})\n\t}()\n\n\ttime.Sleep(200 * time.Millisecond)\n\t_, infos, err := GetHTTPSHostname(conn)\n\trequire.NoError(err)\n\trequire.Equal(\"example.com\", infos[\"Host\"])\n\trequire.Equal(\"https\", infos[\"Scheme\"])\n}\n\n// Path: pkg/util/vhost/resource.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\t\"github.com/fatedier/frp/pkg/util/version\"\n)\n\nvar NotFoundPagePath = \"\"\n\nconst (\n\tNotFound = `<!DOCTYPE html>\n<html>\n<head>\n<title>Not Found</title>\n<style>\n    body {\n        width: 35em;\n        margin: 0 auto;\n        font-family: Tahoma, Verdana, Arial, sans-serif;\n    }\n</style>\n</head>\n<body>\n<h1>The page you requested was not found.</h1>\n<p>Sorry, the page you are looking for is currently unavailable.<br/>\nPlease try again later.</p>\n<p>The server is powered by <a href=\"https://github.com/fatedier/frp\">frp</a>.</p>\n<p><em>Faithfully yours, frp.</em></p>\n</body>\n</html>\n`\n)\n\nfunc getNotFoundPageContent() []byte {\n\tvar (\n\t\tbuf []byte\n\t\terr error\n\t)\n\tif NotFoundPagePath != \"\" {\n\t\tbuf, err = os.ReadFile(NotFoundPagePath)\n\t\tif err != nil {\n\t\t\tlog.Warnf(\"read custom 404 page error: %v\", err)\n\t\t\tbuf = []byte(NotFound)\n\t\t}\n\t} else {\n\t\tbuf = []byte(NotFound)\n\t}\n\treturn buf\n}\n\nfunc NotFoundResponse() *http.Response {\n\theader := make(http.Header)\n\theader.Set(\"server\", \"frp/\"+version.Full())\n\theader.Set(\"Content-Type\", \"text/html\")\n\n\tcontent := getNotFoundPageContent()\n\tres := &http.Response{\n\t\tStatus:        \"Not Found\",\n\t\tStatusCode:    404,\n\t\tProto:         \"HTTP/1.1\",\n\t\tProtoMajor:    1,\n\t\tProtoMinor:    1,\n\t\tHeader:        header,\n\t\tBody:          io.NopCloser(bytes.NewReader(content)),\n\t\tContentLength: int64(len(content)),\n\t}\n\treturn res\n}\n\n// Path: pkg/util/wait/backoff.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage wait\n\nimport (\n\t\"math/rand/v2\"\n\t\"time\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype BackoffFunc func(previousDuration time.Duration, previousConditionError bool) time.Duration\n\nfunc (f BackoffFunc) Backoff(previousDuration time.Duration, previousConditionError bool) time.Duration {\n\treturn f(previousDuration, previousConditionError)\n}\n\ntype BackoffManager interface {\n\tBackoff(previousDuration time.Duration, previousConditionError bool) time.Duration\n}\n\ntype FastBackoffOptions struct {\n\tDuration           time.Duration\n\tFactor             float64\n\tJitter             float64\n\tMaxDuration        time.Duration\n\tInitDurationIfFail time.Duration\n\n\t// If FastRetryCount > 0, then within the FastRetryWindow time window,\n\t// the retry will be performed with a delay of FastRetryDelay for the first FastRetryCount calls.\n\tFastRetryCount  int\n\tFastRetryDelay  time.Duration\n\tFastRetryJitter float64\n\tFastRetryWindow time.Duration\n}\n\ntype fastBackoffImpl struct {\n\toptions FastBackoffOptions\n\n\tlastCalledTime      time.Time\n\tconsecutiveErrCount int\n\n\tfastRetryCutoffTime     time.Time\n\tcountsInFastRetryWindow int\n}\n\nfunc NewFastBackoffManager(options FastBackoffOptions) BackoffManager {\n\treturn &fastBackoffImpl{\n\t\toptions:                 options,\n\t\tcountsInFastRetryWindow: 1,\n\t}\n}\n\nfunc (f *fastBackoffImpl) Backoff(previousDuration time.Duration, previousConditionError bool) time.Duration {\n\tif f.lastCalledTime.IsZero() {\n\t\tf.lastCalledTime = time.Now()\n\t\treturn f.options.Duration\n\t}\n\tnow := time.Now()\n\tf.lastCalledTime = now\n\n\tif previousConditionError {\n\t\tf.consecutiveErrCount++\n\t} else {\n\t\tf.consecutiveErrCount = 0\n\t}\n\n\tif f.options.FastRetryCount > 0 && previousConditionError {\n\t\tf.countsInFastRetryWindow++\n\t\tif f.countsInFastRetryWindow <= f.options.FastRetryCount {\n\t\t\treturn Jitter(f.options.FastRetryDelay, f.options.FastRetryJitter)\n\t\t}\n\t\tif now.After(f.fastRetryCutoffTime) {\n\t\t\t// reset\n\t\t\tf.fastRetryCutoffTime = now.Add(f.options.FastRetryWindow)\n\t\t\tf.countsInFastRetryWindow = 0\n\t\t}\n\t}\n\n\tif previousConditionError {\n\t\tvar duration time.Duration\n\t\tif f.consecutiveErrCount == 1 {\n\t\t\tduration = util.EmptyOr(f.options.InitDurationIfFail, previousDuration)\n\t\t} else {\n\t\t\tduration = previousDuration\n\t\t}\n\n\t\tduration = util.EmptyOr(duration, time.Second)\n\t\tif f.options.Factor != 0 {\n\t\t\tduration = time.Duration(float64(duration) * f.options.Factor)\n\t\t}\n\t\tif f.options.Jitter > 0 {\n\t\t\tduration = Jitter(duration, f.options.Jitter)\n\t\t}\n\t\tif f.options.MaxDuration > 0 && duration > f.options.MaxDuration {\n\t\t\tduration = f.options.MaxDuration\n\t\t}\n\t\treturn duration\n\t}\n\treturn f.options.Duration\n}\n\nfunc BackoffUntil(f func() (bool, error), backoff BackoffManager, sliding bool, stopCh <-chan struct{}) {\n\tvar delay time.Duration\n\tpreviousError := false\n\n\tticker := time.NewTicker(backoff.Backoff(delay, previousError))\n\tdefer ticker.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-stopCh:\n\t\t\treturn\n\t\tdefault:\n\t\t}\n\n\t\tif !sliding {\n\t\t\tdelay = backoff.Backoff(delay, previousError)\n\t\t}\n\n\t\tif done, err := f(); done {\n\t\t\treturn\n\t\t} else if err != nil {\n\t\t\tpreviousError = true\n\t\t} else {\n\t\t\tpreviousError = false\n\t\t}\n\n\t\tif sliding {\n\t\t\tdelay = backoff.Backoff(delay, previousError)\n\t\t}\n\n\t\tticker.Reset(delay)\n\t\tselect {\n\t\tcase <-stopCh:\n\t\t\treturn\n\t\tcase <-ticker.C:\n\t\t}\n\t}\n}\n\n// Jitter returns a time.Duration between duration and duration + maxFactor *\n// duration.\n//\n// This allows clients to avoid converging on periodic behavior. If maxFactor\n// is 0.0, a suggested default value will be chosen.\nfunc Jitter(duration time.Duration, maxFactor float64) time.Duration {\n\tif maxFactor <= 0.0 {\n\t\tmaxFactor = 1.0\n\t}\n\twait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration))\n\treturn wait\n}\n\nfunc Until(f func(), period time.Duration, stopCh <-chan struct{}) {\n\tff := func() (bool, error) {\n\t\tf()\n\t\treturn false, nil\n\t}\n\tBackoffUntil(ff, BackoffFunc(func(time.Duration, bool) time.Duration {\n\t\treturn period\n\t}), true, stopCh)\n}\n\n", "needle_token_start": 674, "needle_token_end": 819, "code_context_ntokens": 14936, "output": ["```go\nfunc DialHookCustomTLSHeadByte(enableTLS bool, disableCustomTLSHeadByte bool) libnet.AfterHookFunc {\n\treturn func(ctx context.Context, c net.Conn, addr string) (context.Context, net.Conn, error) {\n\t\tif enableTLS && !disableCustomTLSHeadByte {\n\t\t\t_, err := c.Write([]byte{byte(FRPTLSHeadByte)})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\t\treturn ctx, c, nil\n\t}\n}\n```"]}
{"repo": "fatedier/frp", "name": "waitDetectMessage", "language": "go", "path": "pkg/nathole/nathole.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to wait for a specific UDP message that matches a given session identifier (SID) and to handle the response based on the role (sender or receiver) in the communication process.\n2. **Input**: It takes a network connection, a session identifier, a cryptographic key, a timeout duration, and the role of the participant (sender or receiver).\n3. **Output**: Returns the remote network address from which the message was received if the message matches the expected criteria, or an error if the message does not match or if a timeout occurs.\n4. **Procedure**: The function continuously reads messages from the network within the specified timeout. For each message, it checks if the message's SID matches the expected SID. If the SID matches and the participant is a receiver, it sends a response back. If the participant is a sender, it only waits for response messages. The process repeats until a matching message is received or the timeout expires.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t}\n\tif len(addrs) < 2 {\n\t\treturn nil, fmt.Errorf(\"discover error: not enough addresses\")\n\t}\n\n\tlocalIPs, _ := ListLocalIPsForNatHole(10)\n\tnatFeature, err := ClassifyNATFeature(addrs, localIPs)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"classify nat feature error: %v\", err)\n\t}\n\n\tladdr, err := net.ResolveUDPAddr(\"udp4\", localAddr.String())\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"resolve local udp addr error: %v\", err)\n\t}\n\tlistenConn, err := net.ListenUDP(\"udp4\", laddr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"listen local udp addr error: %v\", err)\n\t}\n\n\tassistedAddrs := make([]string, 0, len(localIPs))\n\tfor _, ip := range localIPs {\n\t\tassistedAddrs = append(assistedAddrs, net.JoinHostPort(ip, strconv.Itoa(laddr.Port)))\n\t}\n\treturn &PrepareResult{\n\t\tAddrs:         addrs,\n\t\tAssistedAddrs: assistedAddrs,\n\t\tListenConn:    listenConn,\n\t\tNatType:       natFeature.NatType,\n\t\tBehavior:      natFeature.Behavior,\n\t}, nil\n}\n\n// ExchangeInfo is used to exchange information between client and visitor.\n// 1. Send input message to server by msgTransporter.\n// 2. Server will gather information from client and visitor and analyze it. Then send back a NatHoleResp message to them to tell them how to do next.\n// 3. Receive NatHoleResp message from server.\nfunc ExchangeInfo(\n\tctx context.Context, transporter transport.MessageTransporter,\n\tlaneKey string, m msg.Message, timeout time.Duration,\n) (*msg.NatHoleResp, error) {\n\ttimeoutCtx, cancel := context.WithTimeout(ctx, timeout)\n\tdefer cancel()\n\n\tvar natHoleRespMsg *msg.NatHoleResp\n\tm, err := transporter.Do(timeoutCtx, m, laneKey, msg.TypeNameNatHoleResp)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"get natHoleRespMsg error: %v\", err)\n\t}\n\tmm, ok := m.(*msg.NatHoleResp)\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"get natHoleRespMsg error: invalid message type\")\n\t}\n\tnatHoleRespMsg = mm\n\n\tif natHoleRespMsg.Error != \"\" {\n\t\treturn nil, fmt.Errorf(\"natHoleRespMsg get error info: %s\", natHoleRespMsg.Error)\n\t}\n\tif len(natHoleRespMsg.CandidateAddrs) == 0 {\n\t\treturn nil, fmt.Errorf(\"natHoleRespMsg get empty candidate addresses\")\n\t}\n\treturn natHoleRespMsg, nil\n}\n\n// MakeHole is used to make a NAT hole between client and visitor.\nfunc MakeHole(ctx context.Context, listenConn *net.UDPConn, m *msg.NatHoleResp, key []byte) (*net.UDPConn, *net.UDPAddr, error) {\n\txl := xlog.FromContextSafe(ctx)\n\ttransactionID := NewTransactionID()\n\tsendToRangePortsFunc := func(conn *net.UDPConn, addr string) error {\n\t\treturn sendSidMessage(ctx, conn, m.Sid, transactionID, addr, key, m.DetectBehavior.TTL)\n\t}\n\n\tlistenConns := []*net.UDPConn{listenConn}\n\tvar detectAddrs []string\n\tif m.DetectBehavior.Role == DetectRoleSender {\n\t\t// sender\n\t\tif m.DetectBehavior.SendDelayMs > 0 {\n\t\t\ttime.Sleep(time.Duration(m.DetectBehavior.SendDelayMs) * time.Millisecond)\n\t\t}\n\t\tdetectAddrs = m.AssistedAddrs\n\t\tdetectAddrs = append(detectAddrs, m.CandidateAddrs...)\n\t} else {\n\t\t// receiver\n\t\tif len(m.DetectBehavior.CandidatePorts) == 0 {\n\t\t\tdetectAddrs = m.CandidateAddrs\n\t\t}\n\n\t\tif m.DetectBehavior.ListenRandomPorts > 0 {\n\t\t\tfor i := 0; i < m.DetectBehavior.ListenRandomPorts; i++ {\n\t\t\t\ttmpConn, err := net.ListenUDP(\"udp4\", nil)\n\t\t\t\tif err != nil {\n\t\t\t\t\txl.Warnf(\"listen random udp addr error: %v\", err)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tlistenConns = append(listenConns, tmpConn)\n\t\t\t}\n\t\t}\n\t}\n\n\tdetectAddrs = slices.Compact(detectAddrs)\n\tfor _, detectAddr := range detectAddrs {\n\t\tfor _, conn := range listenConns {\n\t\t\tif err := sendSidMessage(ctx, conn, m.Sid, transactionID, detectAddr, key, m.DetectBehavior.TTL); err != nil {\n\t\t\t\txl.Tracef(\"send sid message from %s to %s error: %v\", conn.LocalAddr(), detectAddr, err)\n\t\t\t}\n\t\t}\n\t}\n\tif len(m.DetectBehavior.CandidatePorts) > 0 {\n\t\tfor _, conn := range listenConns {\n\t\t\tsendSidMessageToRangePorts(ctx, conn, m.CandidateAddrs, m.DetectBehavior.CandidatePorts, sendToRangePortsFunc)\n\t\t}\n\t}\n\tif m.DetectBehavior.SendRandomPorts > 0 {\n\t\tctx, cancel := context.WithCancel(ctx)\n\t\tdefer cancel()\n\t\tfor i := range listenConns {\n\t\t\tgo sendSidMessageToRandomPorts(ctx, listenConns[i], m.CandidateAddrs, m.DetectBehavior.SendRandomPorts, sendToRangePortsFunc)\n\t\t}\n\t}\n\n\ttimeout := 5 * time.Second\n\tif m.DetectBehavior.ReadTimeoutMs > 0 {\n\t\ttimeout = time.Duration(m.DetectBehavior.ReadTimeoutMs) * time.Millisecond\n\t}\n\n\tif len(listenConns) == 1 {\n\t\traddr, err := waitDetectMessage(ctx, listenConns[0], m.Sid, key, timeout, m.DetectBehavior.Role)\n\t\tif err != nil {\n\t\t\treturn nil, nil, fmt.Errorf(\"wait detect message error: %v\", err)\n\t\t}\n\t\treturn listenConns[0], raddr, nil\n\t}\n\n\ttype result struct {\n\t\tlConn *net.UDPConn\n\t\traddr *net.UDPAddr\n\t}\n\tresultCh := make(chan result)\n\tfor _, conn := range listenConns {\n\t\tgo func(lConn *net.UDPConn) {\n\t\t\taddr, err := waitDetectMessage(ctx, lConn, m.Sid, key, timeout, m.DetectBehavior.Role)\n\t\t\tif err != nil {\n\t\t\t\tlConn.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tselect {\n\t\t\tcase resultCh <- result{lConn: lConn, raddr: addr}:\n\t\t\tdefault:\n\t\t\t\tlConn.Close()\n\t\t\t}\n\t\t}(conn)\n\t}\n\n\tselect {\n\tcase result := <-resultCh:\n\t\treturn result.lConn, result.raddr, nil\n\tcase <-time.After(timeout):\n\t\treturn nil, nil, fmt.Errorf(\"wait detect message timeout\")\n\tcase <-ctx.Done():\n\t\treturn nil, nil, fmt.Errorf(\"wait detect message canceled\")\n\t}\n}\n\n\nfunc waitDetectMessage(\n\tctx context.Context, conn *net.UDPConn, sid string, key []byte,\n\ttimeout time.Duration, role string,\n) (*net.UDPAddr, error) {\n\txl := xlog.FromContextSafe(ctx)\n\tfor {\n\t\tbuf := pool.GetBuf(1024)\n\t\t_ = conn.SetReadDeadline(time.Now().Add(timeout))\n\t\tn, raddr, err := conn.ReadFromUDP(buf)\n\t\t_ = conn.SetReadDeadline(time.Time{})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\txl.Debugf(\"get udp message local %s, from %s\", conn.LocalAddr(), raddr)\n\t\tvar m msg.NatHoleSid\n\t\tif err := DecodeMessageInto(buf[:n], key, &m); err != nil {\n\t\t\txl.Warnf(\"decode sid message error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tpool.PutBuf(buf)\n\n\t\tif m.Sid != sid {\n\t\t\txl.Warnf(\"get sid message with wrong sid: %s, expect: %s\", m.Sid, sid)\n\t\t\tcontinue\n\t\t}\n\n\t\tif !m.Response {\n\t\t\t// only wait for response messages if we are a sender\n\t\t\tif role == DetectRoleSender {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tm.Response = true\n\t\t\tbuf2, err := EncodeMessage(&m, key)\n\t\t\tif err != nil {\n\t\t\t\txl.Warnf(\"encode sid message error: %v\", err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_, _ = conn.WriteToUDP(buf2, raddr)\n\t\t}\n\t\treturn raddr, nil\n\t}\n}\n\nfunc sendSidMessage(\n\tctx context.Context, conn *net.UDPConn,\n\tsid string, transactionID string, addr string, key []byte, ttl int,\n) error {\n\txl := xlog.FromContextSafe(ctx)\n\tttlStr := \"\"\n\tif ttl > 0 {\n\t\tttlStr = fmt.Sprintf(\" with ttl %d\", ttl)\n\t}\n\txl.Tracef(\"send sid message from %s to %s%s\", conn.LocalAddr(), addr, ttlStr)\n\traddr, err := net.ResolveUDPAddr(\"udp4\", addr)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif transactionID == \"\" {\n\t\ttransactionID = NewTransactionID()\n\t}\n\tm := &msg.NatHoleSid{\n\t\tTransactionID: transactionID,\n\t\tSid:           sid,\n\t\tResponse:      false,\n\t\tNonce:         strings.Repeat(\"0\", rand.IntN(20)),\n\t}\n\tbuf, err := EncodeMessage(m, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif ttl > 0 {\n\t\tuConn := ipv4.NewConn(conn)\n\t\toriginal, err := uConn.TTL()\n\t\tif err != nil {\n\t\t\txl.Tracef(\"get ttl error %v\", err)\n\t\t\treturn err\n\t\t}\n\t\txl.Tracef(\"original ttl %d\", original)\n\n\t\terr = uConn.SetTTL(ttl)\n\t\tif err != nil {\n\t\t\txl.Tracef(\"set ttl error %v\", err)\n\t\t} else {\n\t\t\tdefer func() {\n\t\t\t\t_ = uConn.SetTTL(original)\n\t\t\t}()\n\t\t}\n\t}\n\n\tif _, err := conn.WriteToUDP(buf, raddr); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc sendSidMessageToRangePorts(\n\tctx context.Context, conn *net.UDPConn, addrs []string, ports []msg.PortsRange,\n\tsendFunc func(*net.UDPConn, string) error,\n) {\n\txl := xlog.FromContextSafe(ctx)\n\tfor _, ip := range slices.Compact(parseIPs(addrs)) {\n\t\tfor _, portsRange := range ports {\n\t\t\tfor i := portsRange.From; i <= portsRange.To; i++ {\n\t\t\t\tdetectAddr := net.JoinHostPort(ip, strconv.Itoa(i))\n\t\t\t\tif err := sendFunc(conn, detectAddr); err != nil {\n\t\t\t\t\txl.Tracef(\"send sid message from %s to %s error: %v\", conn.LocalAddr(), detectAddr, err)\n\t\t\t\t}\n\t\t\t\ttime.Sleep(2 * time.Millisecond)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc sendSidMessageToRandomPorts(\n\tctx context.Context, conn *net.UDPConn, addrs []string, count int,\n\tsendFunc func(*net.UDPConn, string) error,\n) {\n\txl := xlog.FromContextSafe(ctx)\n\tused := sets.New[int]()\n\tgetUnusedPort := func() int {\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tport := rand.IntN(65535-1024) + 1024\n\t\t\tif !used.Has(port) {\n\t\t\t\tused.Insert(port)\n\t\t\t\treturn port\n\t\t\t}\n\t\t}\n\t\treturn 0\n\t}\n\n\tfor i := 0; i < count; i++ {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\tdefault:\n\t\t}\n\n\t\tport := getUnusedPort()\n\t\tif port == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tfor _, ip := range slices.Compact(parseIPs(addrs)) {\n\t\t\tdetectAddr := net.JoinHostPort(ip, strconv.Itoa(port))\n\t\t\tif err := sendFunc(conn, detectAddr); err != nil {\n\t\t\t\txl.Tracef(\"send sid message from %s to %s error: %v\", conn.LocalAddr(), detectAddr, err)\n\t\t\t}\n\t\t\ttime.Sleep(time.Millisecond * 15)\n\t\t}\n\t}\n}\n\nfunc parseIPs(addrs []string) []string {\n\tvar ips []string\n\tfor _, addr := range addrs {\n\t\tif ip, _, err := net.SplitHostPort(addr); err == nil {\n\t\t\tips = append(ips, ip)\n\t\t}\n\t}\n\treturn ips\n}\n\n// Path: pkg/nathole/utils.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nathole\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"net\"\n\t\"strconv\"\n\n\t\"github.com/fatedier/golib/crypto\"\n\t\"github.com/pion/stun/v2\"\n\n\t\"github.com/fatedier/frp/pkg/msg\"\n)\n\nfunc EncodeMessage(m msg.Message, key []byte) ([]byte, error) {\n\tbuffer := bytes.NewBuffer(nil)\n\tif err := msg.WriteMsg(buffer, m); err != nil {\n\t\treturn nil, err\n\t}\n\n\tbuf, err := crypto.Encode(buffer.Bytes(), key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn buf, nil\n}\n\nfunc DecodeMessageInto(data, key []byte, m msg.Message) error {\n\tbuf, err := crypto.Decode(data, key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn msg.ReadMsgInto(bytes.NewReader(buf), m)\n}\n\ntype ChangedAddress struct {\n\tIP   net.IP\n\tPort int\n}\n\nfunc (s *ChangedAddress) GetFrom(m *stun.Message) error {\n\ta := (*stun.MappedAddress)(s)\n\treturn a.GetFromAs(m, stun.AttrChangedAddress)\n}\n\nfunc (s *ChangedAddress) String() string {\n\treturn net.JoinHostPort(s.IP.String(), strconv.Itoa(s.Port))\n}\n\nfunc ListAllLocalIPs() ([]net.IP, error) {\n\taddrs, err := net.InterfaceAddrs()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tips := make([]net.IP, 0, len(addrs))\n\tfor _, addr := range addrs {\n\t\tip, _, err := net.ParseCIDR(addr.String())\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tips = append(ips, ip)\n\t}\n\treturn ips, nil\n}\n\nfunc ListLocalIPsForNatHole(max int) ([]string, error) {\n\tif max <= 0 {\n\t\treturn nil, fmt.Errorf(\"max must be greater than 0\")\n\t}\n\n\tips, err := ListAllLocalIPs()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfiltered := make([]string, 0, max)\n\tfor _, ip := range ips {\n\t\tif len(filtered) >= max {\n\t\t\tbreak\n\t\t}\n\n\t\t// ignore ipv6 address\n\t\tif ip.To4() == nil {\n\t\t\tcontinue\n\t\t}\n\t\t// ignore localhost IP\n\t\tif ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {\n\t\t\tcontinue\n\t\t}\n\n\t\tfiltered = append(filtered, ip.String())\n\t}\n\treturn filtered, nil\n}\n\n// Path: pkg/plugin/client/http2https.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"crypto/tls\"\n\t\"io\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\n\t\"github.com/fatedier/golib/pool\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTP2HTTPS, NewHTTP2HTTPSPlugin)\n}\n\ntype HTTP2HTTPSPlugin struct {\n\topts *v1.HTTP2HTTPSPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTP2HTTPSPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTP2HTTPSPluginOptions)\n\n\tlistener := NewProxyListener()\n\n\tp := &HTTP2HTTPSPlugin{\n\t\topts: opts,\n\t\tl:    listener,\n\t}\n\n\ttr := &http.Transport{\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\n\trp := &httputil.ReverseProxy{\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.Out.Header[\"X-Forwarded-Host\"] = r.In.Header[\"X-Forwarded-Host\"]\n\t\t\tr.Out.Header[\"X-Forwarded-Proto\"] = r.In.Header[\"X-Forwarded-Proto\"]\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"https\"\n\t\t\treq.URL.Host = p.opts.LocalAddr\n\t\t\tif p.opts.HostHeaderRewrite != \"\" {\n\t\t\t\treq.Host = p.opts.HostHeaderRewrite\n\t\t\t}\n\t\t\tfor k, v := range p.opts.RequestHeaders.Set {\n\t\t\t\treq.Header.Set(k, v)\n\t\t\t}\n\t\t},\n\t\tTransport:  tr,\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n\t\tErrorLog:   stdlog.New(log.NewWriteLogger(log.WarnLevel, 2), \"\", 0),\n\t}\n\n\tp.s = &http.Server{\n\t\tHandler:           rp,\n\t\tReadHeaderTimeout: 0,\n\t}\n\n\tgo func() {\n\t\t_ = p.s.Serve(listener)\n\t}()\n\n\treturn p, nil\n}\n\nfunc (p *HTTP2HTTPSPlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, _ *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\t_ = p.l.PutConn(wrapConn)\n}\n\nfunc (p *HTTP2HTTPSPlugin) Name() string {\n\treturn v1.PluginHTTP2HTTPS\n}\n\nfunc (p *HTTP2HTTPSPlugin) Close() error {\n\treturn p.s.Close()\n}\n\n// Path: pkg/plugin/client/http_proxy.go\n// Copyright 2017 frp team\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"bufio\"\n\t\"encoding/base64\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\tlibio \"github.com/fatedier/golib/io\"\n\tlibnet \"github.com/fatedier/golib/net\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTPProxy, NewHTTPProxyPlugin)\n}\n\ntype HTTPProxy struct {\n\topts *v1.HTTPProxyPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTPProxyPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTPProxyPluginOptions)\n\tlistener := NewProxyListener()\n\n\thp := &HTTPProxy{\n\t\tl:    listener,\n\t\topts: opts,\n\t}\n\n\thp.s = &http.Server{\n\t\tHandler:           hp,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\n\tgo func() {\n\t\t_ = hp.s.Serve(listener)\n\t}()\n\treturn hp, nil\n}\n\nfunc (hp *HTTPProxy) Name() string {\n\treturn v1.PluginHTTPProxy\n}\n\nfunc (hp *HTTPProxy) Handle(conn io.ReadWriteCloser, realConn net.Conn, _ *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\n\tsc, rd := libnet.NewSharedConn(wrapConn)\n\tfirstBytes := make([]byte, 7)\n\t_, err := rd.Read(firstBytes)\n\tif err != nil {\n\t\twrapConn.Close()\n\t\treturn\n\t}\n\n\tif strings.ToUpper(string(firstBytes)) == \"CONNECT\" {\n\t\tbufRd := bufio.NewReader(sc)\n\t\trequest, err := http.ReadRequest(bufRd)\n\t\tif err != nil {\n\t\t\twrapConn.Close()\n\t\t\treturn\n\t\t}\n\t\thp.handleConnectReq(request, libio.WrapReadWriteCloser(bufRd, wrapConn, wrapConn.Close))\n\t\treturn\n\t}\n\n\t_ = hp.l.PutConn(sc)\n}\n\nfunc (hp *HTTPProxy) Close() error {\n\thp.s.Close()\n\thp.l.Close()\n\treturn nil\n}\n\nfunc (hp *HTTPProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {\n\tif ok := hp.Auth(req); !ok {\n\t\trw.Header().Set(\"Proxy-Authenticate\", \"Basic\")\n\t\trw.WriteHeader(http.StatusProxyAuthRequired)\n\t\treturn\n\t}\n\n\tif req.Method == http.MethodConnect {\n\t\t// deprecated\n\t\t// Connect request is handled in Handle function.\n\t\thp.ConnectHandler(rw, req)\n\t} else {\n\t\thp.HTTPHandler(rw, req)\n\t}\n}\n\nfunc (hp *HTTPProxy) HTTPHandler(rw http.ResponseWriter, req *http.Request) {\n\tremoveProxyHeaders(req)\n\n\tresp, err := http.DefaultTransport.RoundTrip(req)\n\tif err != nil {\n\t\thttp.Error(rw, err.Error(), http.StatusInternalServerError)\n\t\treturn\n\t}\n\tdefer resp.Body.Close()\n\n\tcopyHeaders(rw.Header(), resp.Header)\n\trw.WriteHeader(resp.StatusCode)\n\n\t_, err = io.Copy(rw, resp.Body)\n\tif err != nil && err != io.EOF {\n\t\treturn\n\t}\n}\n\n// deprecated\n// Hijack needs to SetReadDeadline on the Conn of the request, but if we use stream compression here,\n// we may always get i/o timeout error.\nfunc (hp *HTTPProxy) ConnectHandler(rw http.ResponseWriter, req *http.Request) {\n\thj, ok := rw.(http.Hijacker)\n\tif !ok {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tclient, _, err := hj.Hijack()\n\tif err != nil {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tremote, err := net.Dial(\"tcp\", req.URL.Host)\n\tif err != nil {\n\t\thttp.Error(rw, \"Failed\", http.StatusBadRequest)\n\t\tclient.Close()\n\t\treturn\n\t}\n\t_, _ = client.Write([]byte(\"HTTP/1.1 200 OK\\r\\n\\r\\n\"))\n\n\tgo libio.Join(remote, client)\n}\n\nfunc (hp *HTTPProxy) Auth(req *http.Request) bool {\n\tif hp.opts.HTTPUser == \"\" && hp.opts.HTTPPassword == \"\" {\n\t\treturn true\n\t}\n\n\ts := strings.SplitN(req.Header.Get(\"Proxy-Authorization\"), \" \", 2)\n\tif len(s) != 2 {\n\t\treturn false\n\t}\n\n\tb, err := base64.StdEncoding.DecodeString(s[1])\n\tif err != nil {\n\t\treturn false\n\t}\n\n\tpair := strings.SplitN(string(b), \":\", 2)\n\tif len(pair) != 2 {\n\t\treturn false\n\t}\n\n\tif !util.ConstantTimeEqString(pair[0], hp.opts.HTTPUser) ||\n\t\t!util.ConstantTimeEqString(pair[1], hp.opts.HTTPPassword) {\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc (hp *HTTPProxy) handleConnectReq(req *http.Request, rwc io.ReadWriteCloser) {\n\tdefer rwc.Close()\n\tif ok := hp.Auth(req); !ok {\n\t\tres := getBadResponse()\n\t\t_ = res.Write(rwc)\n\t\tif res.Body != nil {\n\t\t\tres.Body.Close()\n\t\t}\n\t\treturn\n\t}\n\n\tremote, err := net.Dial(\"tcp\", req.URL.Host)\n\tif err != nil {\n\t\tres := &http.Response{\n\t\t\tStatusCode: 400,\n\t\t\tProto:      \"HTTP/1.1\",\n\t\t\tProtoMajor: 1,\n\t\t\tProtoMinor: 1,\n\t\t}\n\t\t_ = res.Write(rwc)\n\t\treturn\n\t}\n\t_, _ = rwc.Write([]byte(\"HTTP/1.1 200 OK\\r\\n\\r\\n\"))\n\n\tlibio.Join(remote, rwc)\n}\n\nfunc copyHeaders(dst, src http.Header) {\n\tfor key, values := range src {\n\t\tfor _, value := range values {\n\t\t\tdst.Add(key, value)\n\t\t}\n\t}\n}\n\nfunc removeProxyHeaders(req *http.Request) {\n\treq.RequestURI = \"\"\n\treq.Header.Del(\"Proxy-Connection\")\n\treq.Header.Del(\"Connection\")\n\treq.Header.Del(\"Proxy-Authenticate\")\n\treq.Header.Del(\"Proxy-Authorization\")\n\treq.Header.Del(\"TE\")\n\treq.Header.Del(\"Trailers\")\n\treq.Header.Del(\"Transfer-Encoding\")\n\treq.Header.Del(\"Upgrade\")\n}\n\nfunc getBadResponse() *http.Response {\n\theader := make(map[string][]string)\n\theader[\"Proxy-Authenticate\"] = []string{\"Basic\"}\n\theader[\"Connection\"] = []string{\"close\"}\n\tres := &http.Response{\n\t\tStatus:     \"407 Not authorized\",\n\t\tStatusCode: 407,\n\t\tProto:      \"HTTP/1.1\",\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 1,\n\t\tHeader:     header,\n\t}\n\treturn res\n}\n\n// Path: pkg/plugin/client/https2http.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/pool\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/transport\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTPS2HTTP, NewHTTPS2HTTPPlugin)\n}\n\ntype HTTPS2HTTPPlugin struct {\n\topts *v1.HTTPS2HTTPPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTPS2HTTPPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTPS2HTTPPluginOptions)\n\tlistener := NewProxyListener()\n\n\tp := &HTTPS2HTTPPlugin{\n\t\topts: opts,\n\t\tl:    listener,\n\t}\n\n\trp := &httputil.ReverseProxy{\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.SetXForwarded()\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"http\"\n\t\t\treq.URL.Host = p.opts.LocalAddr\n\t\t\tif p.opts.HostHeaderRewrite != \"\" {\n\t\t\t\treq.Host = p.opts.HostHeaderRewrite\n\t\t\t}\n\t\t\tfor k, v := range p.opts.RequestHeaders.Set {\n\t\t\t\treq.Header.Set(k, v)\n\t\t\t}\n\t\t},\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n\t\tErrorLog:   stdlog.New(log.NewWriteLogger(log.WarnLevel, 2), \"\", 0),\n\t}\n\n\tp.s = &http.Server{\n\t\tHandler:           rp,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\n\tvar (\n\t\ttlsConfig *tls.Config\n\t\terr       error\n\t)\n\tif opts.CrtPath != \"\" || opts.KeyPath != \"\" {\n\t\ttlsConfig, err = p.genTLSConfig()\n\t} else {\n\t\ttlsConfig, err = transport.NewServerTLSConfig(\"\", \"\", \"\")\n\t\ttlsConfig.InsecureSkipVerify = true\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"gen TLS config error: %v\", err)\n\t}\n\tln := tls.NewListener(listener, tlsConfig)\n\n\tgo func() {\n\t\t_ = p.s.Serve(ln)\n\t}()\n\treturn p, nil\n}\n\nfunc (p *HTTPS2HTTPPlugin) genTLSConfig() (*tls.Config, error) {\n\tcert, err := tls.LoadX509KeyPair(p.opts.CrtPath, p.opts.KeyPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfig := &tls.Config{Certificates: []tls.Certificate{cert}}\n\treturn config, nil\n}\n\nfunc (p *HTTPS2HTTPPlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, extra *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\tif extra.SrcAddr != nil {\n\t\twrapConn.SetRemoteAddr(extra.SrcAddr)\n\t}\n\t_ = p.l.PutConn(wrapConn)\n}\n\nfunc (p *HTTPS2HTTPPlugin) Name() string {\n\treturn v1.PluginHTTPS2HTTP\n}\n\nfunc (p *HTTPS2HTTPPlugin) Close() error {\n\treturn p.s.Close()\n}\n\n// Path: pkg/plugin/client/https2https.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/pool\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/transport\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTPS2HTTPS, NewHTTPS2HTTPSPlugin)\n}\n\ntype HTTPS2HTTPSPlugin struct {\n\topts *v1.HTTPS2HTTPSPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTPS2HTTPSPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTPS2HTTPSPluginOptions)\n\n\tlistener := NewProxyListener()\n\n\tp := &HTTPS2HTTPSPlugin{\n\t\topts: opts,\n\t\tl:    listener,\n\t}\n\n\ttr := &http.Transport{\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\n\trp := &httputil.ReverseProxy{\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.SetXForwarded()\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"https\"\n\t\t\treq.URL.Host = p.opts.LocalAddr\n\t\t\tif p.opts.HostHeaderRewrite != \"\" {\n\t\t\t\treq.Host = p.opts.HostHeaderRewrite\n\t\t\t}\n\t\t\tfor k, v := range p.opts.RequestHeaders.Set {\n\t\t\t\treq.Header.Set(k, v)\n\t\t\t}\n\t\t},\n\t\tTransport:  tr,\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n\t\tErrorLog:   stdlog.New(log.NewWriteLogger(log.WarnLevel, 2), \"\", 0),\n\t}\n\n\tp.s = &http.Server{\n\t\tHandler:           rp,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\n\tvar (\n\t\ttlsConfig *tls.Config\n\t\terr       error\n\t)\n\tif opts.CrtPath != \"\" || opts.KeyPath != \"\" {\n\t\ttlsConfig, err = p.genTLSConfig()\n\t} else {\n\t\ttlsConfig, err = transport.NewServerTLSConfig(\"\", \"\", \"\")\n\t\ttlsConfig.InsecureSkipVerify = true\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"gen TLS config error: %v\", err)\n\t}\n\tln := tls.NewListener(listener, tlsConfig)\n\n\tgo func() {\n\t\t_ = p.s.Serve(ln)\n\t}()\n\treturn p, nil\n}\n\nfunc (p *HTTPS2HTTPSPlugin) genTLSConfig() (*tls.Config, error) {\n\tcert, err := tls.LoadX509KeyPair(p.opts.CrtPath, p.opts.KeyPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfig := &tls.Config{Certificates: []tls.Certificate{cert}}\n\treturn config, nil\n}\n\nfunc (p *HTTPS2HTTPSPlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, extra *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\tif extra.SrcAddr != nil {\n\t\twrapConn.SetRemoteAddr(extra.SrcAddr)\n\t}\n\t_ = p.l.PutConn(wrapConn)\n}\n\nfunc (p *HTTPS2HTTPSPlugin) Name() string {\n\treturn v1.PluginHTTPS2HTTPS\n}\n\nfunc (p *HTTPS2HTTPSPlugin) Close() error {\n\treturn p.s.Close()\n}\n\n// Path: pkg/plugin/client/plugin.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/fatedier/golib/errors\"\n\tpp \"github.com/pires/go-proxyproto\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\n// Creators is used for create plugins to handle connections.\nvar creators = make(map[string]CreatorFn)\n\n// params has prefix \"plugin_\"\ntype CreatorFn func(options v1.ClientPluginOptions) (Plugin, error)\n\nfunc Register(name string, fn CreatorFn) {\n\tif _, exist := creators[name]; exist {\n\t\tpanic(fmt.Sprintf(\"plugin [%s] is already registered\", name))\n\t}\n\tcreators[name] = fn\n}\n\nfunc Create(name string, options v1.ClientPluginOptions) (p Plugin, err error) {\n\tif fn, ok := creators[name]; ok {\n\t\tp, err = fn(options)\n\t} else {\n\t\terr = fmt.Errorf(\"plugin [%s] is not registered\", name)\n\t}\n\treturn\n}\n\ntype ExtraInfo struct {\n\tProxyProtocolHeader *pp.Header\n\tSrcAddr             net.Addr\n\tDstAddr             net.Addr\n}\n\ntype Plugin interface {\n\tName() string\n\n\tHandle(conn io.ReadWriteCloser, realConn net.Conn, extra *ExtraInfo)\n\tClose() error\n}\n\ntype Listener struct {\n\tconns  chan net.Conn\n\tclosed bool\n\tmu     sync.Mutex\n}\n\nfunc NewProxyListener() *Listener {\n\treturn &Listener{\n\t\tconns: make(chan net.Conn, 64),\n\t}\n}\n\nfunc (l *Listener) Accept() (net.Conn, error) {\n\tconn, ok := <-l.conns\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"listener closed\")\n\t}\n\treturn conn, nil\n}\n\nfunc (l *Listener) PutConn(conn net.Conn) error {\n\terr := errors.PanicToError(func() {\n\t\tl.conns <- conn\n\t})\n\treturn err\n}\n\nfunc (l *Listener) Close() error {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\tif !l.closed {\n\t\tclose(l.conns)\n\t\tl.closed = true\n\t}\n\treturn nil\n}\n\nfunc (l *Listener) Addr() net.Addr {\n\treturn (*net.TCPAddr)(nil)\n}\n\n// Path: pkg/plugin/client/socks5.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"io\"\n\t\"log\"\n\t\"net\"\n\n\tgosocks5 \"github.com/armon/go-socks5\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginSocks5, NewSocks5Plugin)\n}\n\ntype Socks5Plugin struct {\n\tServer *gosocks5.Server\n}\n\nfunc NewSocks5Plugin(options v1.ClientPluginOptions) (p Plugin, err error) {\n\topts := options.(*v1.Socks5PluginOptions)\n\n\tcfg := &gosocks5.Config{\n\t\tLogger: log.New(io.Discard, \"\", log.LstdFlags),\n\t}\n\tif opts.Username != \"\" || opts.Password != \"\" {\n\t\tcfg.Credentials = gosocks5.StaticCredentials(map[string]string{opts.Username: opts.Password})\n\t}\n\tsp := &Socks5Plugin{}\n\tsp.Server, err = gosocks5.New(cfg)\n\tp = sp\n\treturn\n}\n\nfunc (sp *Socks5Plugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, _ *ExtraInfo) {\n\tdefer conn.Close()\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\t_ = sp.Server.ServeConn(wrapConn)\n}\n\nfunc (sp *Socks5Plugin) Name() string {\n\treturn v1.PluginSocks5\n}\n\nfunc (sp *Socks5Plugin) Close() error {\n\treturn nil\n}\n\n// Path: pkg/util/vhost/router.go\npackage vhost\n\nimport (\n\t\"cmp\"\n\t\"errors\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n)\n\nvar ErrRouterConfigConflict = errors.New(\"router config conflict\")\n\ntype routerByHTTPUser map[string][]*Router\n\ntype Routers struct {\n\tindexByDomain map[string]routerByHTTPUser\n\n\tmutex sync.RWMutex\n}\n\ntype Router struct {\n\tdomain   string\n\tlocation string\n\thttpUser string\n\n\t// store any object here\n\tpayload interface{}\n}\n\nfunc NewRouters() *Routers {\n\treturn &Routers{\n\t\tindexByDomain: make(map[string]routerByHTTPUser),\n\t}\n}\n\nfunc (r *Routers) Add(domain, location, httpUser string, payload interface{}) error {\n\tdomain = strings.ToLower(domain)\n\n\tr.mutex.Lock()\n\tdefer r.mutex.Unlock()\n\n\tif _, exist := r.exist(domain, location, httpUser); exist {\n\t\treturn ErrRouterConfigConflict\n\t}\n\n\troutersByHTTPUser, found := r.indexByDomain[domain]\n\tif !found {\n\t\troutersByHTTPUser = make(map[string][]*Router)\n\t}\n\tvrs, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\tvrs = make([]*Router, 0, 1)\n\t}\n\n\tvr := &Router{\n\t\tdomain:   domain,\n\t\tlocation: location,\n\t\thttpUser: httpUser,\n\t\tpayload:  payload,\n\t}\n\tvrs = append(vrs, vr)\n\n\tslices.SortFunc(vrs, func(a, b *Router) int {\n\t\treturn -cmp.Compare(a.location, b.location)\n\t})\n\n\troutersByHTTPUser[httpUser] = vrs\n\tr.indexByDomain[domain] = routersByHTTPUser\n\treturn nil\n}\n\nfunc (r *Routers) Del(domain, location, httpUser string) {\n\tdomain = strings.ToLower(domain)\n\n\tr.mutex.Lock()\n\tdefer r.mutex.Unlock()\n\n\troutersByHTTPUser, found := r.indexByDomain[domain]\n\tif !found {\n\t\treturn\n\t}\n\n\tvrs, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\treturn\n\t}\n\tnewVrs := make([]*Router, 0)\n\tfor _, vr := range vrs {\n\t\tif vr.location != location {\n\t\t\tnewVrs = append(newVrs, vr)\n\t\t}\n\t}\n\troutersByHTTPUser[httpUser] = newVrs\n}\n\nfunc (r *Routers) Get(host, path, httpUser string) (vr *Router, exist bool) {\n\thost = strings.ToLower(host)\n\n\tr.mutex.RLock()\n\tdefer r.mutex.RUnlock()\n\n\troutersByHTTPUser, found := r.indexByDomain[host]\n\tif !found {\n\t\treturn\n\t}\n\n\tvrs, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\treturn\n\t}\n\n\tfor _, vr = range vrs {\n\t\tif strings.HasPrefix(path, vr.location) {\n\t\t\treturn vr, true\n\t\t}\n\t}\n\treturn\n}\n\nfunc (r *Routers) exist(host, path, httpUser string) (route *Router, exist bool) {\n\troutersByHTTPUser, found := r.indexByDomain[host]\n\tif !found {\n\t\treturn\n\t}\n\trouters, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\treturn\n\t}\n\n\tfor _, route = range routers {\n\t\tif path == route.location {\n\t\t\treturn route, true\n\t\t}\n\t}\n\treturn\n}\n\n// Path: pkg/plugin/client/static_file.go\n// Copyright 2018 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/gorilla/mux\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginStaticFile, NewStaticFilePlugin)\n}\n\ntype StaticFilePlugin struct {\n\topts *v1.StaticFilePluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewStaticFilePlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.StaticFilePluginOptions)\n\n\tlistener := NewProxyListener()\n\n\tsp := &StaticFilePlugin{\n\t\topts: opts,\n\n\t\tl: listener,\n\t}\n\tvar prefix string\n\tif opts.StripPrefix != \"\" {\n\t\tprefix = \"/\" + opts.StripPrefix + \"/\"\n\t} else {\n\t\tprefix = \"/\"\n\t}\n\n\trouter := mux.NewRouter()\n\trouter.Use(netpkg.NewHTTPAuthMiddleware(opts.HTTPUser, opts.HTTPPassword).SetAuthFailDelay(200 * time.Millisecond).Middleware)\n\trouter.PathPrefix(prefix).Handler(netpkg.MakeHTTPGzipHandler(http.StripPrefix(prefix, http.FileServer(http.Dir(opts.LocalPath))))).Methods(\"GET\")\n\tsp.s = &http.Server{\n\t\tHandler:           router,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\tgo func() {\n\t\t_ = sp.s.Serve(listener)\n\t}()\n\treturn sp, nil\n}\n\nfunc (sp *StaticFilePlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, _ *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\t_ = sp.l.PutConn(wrapConn)\n}\n\nfunc (sp *StaticFilePlugin) Name() string {\n\treturn v1.PluginStaticFile\n}\n\nfunc (sp *StaticFilePlugin) Close() error {\n\tsp.s.Close()\n\tsp.l.Close()\n\treturn nil\n}\n\n// Path: pkg/plugin/client/unix_domain_socket.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"io\"\n\t\"net\"\n\n\tlibio \"github.com/fatedier/golib/io\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc init() {\n\tRegister(v1.PluginUnixDomainSocket, NewUnixDomainSocketPlugin)\n}\n\ntype UnixDomainSocketPlugin struct {\n\tUnixAddr *net.UnixAddr\n}\n\nfunc NewUnixDomainSocketPlugin(options v1.ClientPluginOptions) (p Plugin, err error) {\n\topts := options.(*v1.UnixDomainSocketPluginOptions)\n\n\tunixAddr, errRet := net.ResolveUnixAddr(\"unix\", opts.UnixPath)\n\tif errRet != nil {\n\t\terr = errRet\n\t\treturn\n\t}\n\n\tp = &UnixDomainSocketPlugin{\n\t\tUnixAddr: unixAddr,\n\t}\n\treturn\n}\n\nfunc (uds *UnixDomainSocketPlugin) Handle(conn io.ReadWriteCloser, _ net.Conn, extra *ExtraInfo) {\n\tlocalConn, err := net.DialUnix(\"unix\", nil, uds.UnixAddr)\n\tif err != nil {\n\t\treturn\n\t}\n\tif extra.ProxyProtocolHeader != nil {\n\t\tif _, err := extra.ProxyProtocolHeader.WriteTo(localConn); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\tlibio.Join(localConn, conn)\n}\n\nfunc (uds *UnixDomainSocketPlugin) Name() string {\n\treturn v1.PluginUnixDomainSocket\n}\n\nfunc (uds *UnixDomainSocketPlugin) Close() error {\n\treturn nil\n}\n\n// Path: pkg/plugin/server/manager.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n\t\"github.com/fatedier/frp/pkg/util/xlog\"\n)\n\ntype Manager struct {\n\tloginPlugins       []Plugin\n\tnewProxyPlugins    []Plugin\n\tcloseProxyPlugins  []Plugin\n\tpingPlugins        []Plugin\n\tnewWorkConnPlugins []Plugin\n\tnewUserConnPlugins []Plugin\n}\n\nfunc NewManager() *Manager {\n\treturn &Manager{\n\t\tloginPlugins:       make([]Plugin, 0),\n\t\tnewProxyPlugins:    make([]Plugin, 0),\n\t\tcloseProxyPlugins:  make([]Plugin, 0),\n\t\tpingPlugins:        make([]Plugin, 0),\n\t\tnewWorkConnPlugins: make([]Plugin, 0),\n\t\tnewUserConnPlugins: make([]Plugin, 0),\n\t}\n}\n\nfunc (m *Manager) Register(p Plugin) {\n\tif p.IsSupport(OpLogin) {\n\t\tm.loginPlugins = append(m.loginPlugins, p)\n\t}\n\tif p.IsSupport(OpNewProxy) {\n\t\tm.newProxyPlugins = append(m.newProxyPlugins, p)\n\t}\n\tif p.IsSupport(OpCloseProxy) {\n\t\tm.closeProxyPlugins = append(m.closeProxyPlugins, p)\n\t}\n\tif p.IsSupport(OpPing) {\n\t\tm.pingPlugins = append(m.pingPlugins, p)\n\t}\n\tif p.IsSupport(OpNewWorkConn) {\n\t\tm.newWorkConnPlugins = append(m.newWorkConnPlugins, p)\n\t}\n...\n", "needle_token_start": 2066, "needle_token_end": 2543, "code_context_ntokens": 15203, "output": ["```go\nfunc waitDetectMessage(\n\tctx context.Context, conn *net.UDPConn, sid string, key []byte,\n\ttimeout time.Duration, role string,\n) (*net.UDPAddr, error) {\n\txl := xlog.FromContextSafe(ctx)\n\tfor {\n\t\tbuf := pool.GetBuf(1024)\n\t\t_ = conn.SetReadDeadline(time.Now().Add(timeout))\n\t\tn, raddr, err := conn.ReadFromUDP(buf)\n\t\t_ = conn.SetReadDeadline(time.Time{})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\txl.Debugf(\"get udp message local %s, from %s\", conn.LocalAddr(), raddr)\n\t\tvar m msg.NatHoleSid\n\t\tif err := DecodeMessageInto(buf[:n], key, &m); err != nil {\n\t\t\txl.Warnf(\"decode sid message error: %v\", err)\n\t\t\tcontinue\n\t\t}\n\t\tpool.PutBuf(buf)\n\n\t\tif m.Sid != sid {\n\t\t\txl.Warnf(\"get sid message with wrong sid: %s, expect: %s\", m.Sid, sid)\n\t\t\tcontinue\n\t\t}\n\n\t\tif !m.Response {\n\t\t\t// only wait for response messages if we are a sender\n\t\t\tif role == DetectRoleSender {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tm.Response = true\n\t\t\tbuf2, err := EncodeMessage(&m, key)\n\t\t\tif err != nil {\n\t\t\t\txl.Warnf(\"encode sid message error: %v\", err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_, _ = conn.WriteToUDP(buf2, raddr)\n\t\t}\n\t\treturn raddr, nil\n\t}\n}\n```"]}
{"repo": "fatedier/frp", "name": "ValidateAllClientConfig", "language": "go", "path": "pkg/config/v1/validation/client.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate the configuration settings for a client, including its common configurations, proxy configurations, and visitor configurations, ensuring they adhere to specified rules and constraints.\n2. **Input**: The function takes three parameters: a structure representing common client settings, a list of proxy configuration structures, and a list of visitor configuration structures.\n3. **Output**: It returns a collection of warnings and an error. The warnings indicate minor issues that do not prevent the system from operating, while the error indicates a critical issue that must be resolved.\n4. **Procedure**: \n   - First, the function validates the common client settings and accumulates any warnings or errors.\n   - If an error is found in the common settings, it returns immediately with the accumulated warnings and the error.\n   - Next, it iterates over each proxy configuration, validating each one. If an error is found in any proxy configuration, it stops further processing and returns.\n   - It then validates each visitor configuration. If an error is found, it stops and returns the warnings and the error.\n   - If all configurations are valid, it returns the accumulated warnings and no error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/config/v1/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"github.com/samber/lo\"\n\n\t\"github.com/fatedier/frp/pkg/config/types\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype ServerConfig struct {\n\tAPIMetadata\n\n\tAuth AuthServerConfig `json:\"auth,omitempty\"`\n\t// BindAddr specifies the address that the server binds to. By default,\n\t// this value is \"0.0.0.0\".\n\tBindAddr string `json:\"bindAddr,omitempty\"`\n\t// BindPort specifies the port that the server listens on. By default, this\n\t// value is 7000.\n\tBindPort int `json:\"bindPort,omitempty\"`\n\t// KCPBindPort specifies the KCP port that the server listens on. If this\n\t// value is 0, the server will not listen for KCP connections.\n\tKCPBindPort int `json:\"kcpBindPort,omitempty\"`\n\t// QUICBindPort specifies the QUIC port that the server listens on.\n\t// Set this value to 0 will disable this feature.\n\tQUICBindPort int `json:\"quicBindPort,omitempty\"`\n\t// ProxyBindAddr specifies the address that the proxy binds to. This value\n\t// may be the same as BindAddr.\n\tProxyBindAddr string `json:\"proxyBindAddr,omitempty\"`\n\t// VhostHTTPPort specifies the port that the server listens for HTTP Vhost\n\t// requests. If this value is 0, the server will not listen for HTTP\n\t// requests.\n\tVhostHTTPPort int `json:\"vhostHTTPPort,omitempty\"`\n\t// VhostHTTPTimeout specifies the response header timeout for the Vhost\n\t// HTTP server, in seconds. By default, this value is 60.\n\tVhostHTTPTimeout int64 `json:\"vhostHTTPTimeout,omitempty\"`\n\t// VhostHTTPSPort specifies the port that the server listens for HTTPS\n\t// Vhost requests. If this value is 0, the server will not listen for HTTPS\n\t// requests.\n\tVhostHTTPSPort int `json:\"vhostHTTPSPort,omitempty\"`\n\t// TCPMuxHTTPConnectPort specifies the port that the server listens for TCP\n\t// HTTP CONNECT requests. If the value is 0, the server will not multiplex TCP\n\t// requests on one single port. If it's not - it will listen on this value for\n\t// HTTP CONNECT requests.\n\tTCPMuxHTTPConnectPort int `json:\"tcpmuxHTTPConnectPort,omitempty\"`\n\t// If TCPMuxPassthrough is true, frps won't do any update on traffic.\n\tTCPMuxPassthrough bool `json:\"tcpmuxPassthrough,omitempty\"`\n\t// SubDomainHost specifies the domain that will be attached to sub-domains\n\t// requested by the client when using Vhost proxying. For example, if this\n\t// value is set to \"frps.com\" and the client requested the subdomain\n\t// \"test\", the resulting URL would be \"test.frps.com\".\n\tSubDomainHost string `json:\"subDomainHost,omitempty\"`\n\t// Custom404Page specifies a path to a custom 404 page to display. If this\n\t// value is \"\", a default page will be displayed.\n\tCustom404Page string `json:\"custom404Page,omitempty\"`\n\n\tSSHTunnelGateway SSHTunnelGateway `json:\"sshTunnelGateway,omitempty\"`\n\n\tWebServer WebServerConfig `json:\"webServer,omitempty\"`\n\t// EnablePrometheus will export prometheus metrics on webserver address\n\t// in /metrics api.\n\tEnablePrometheus bool `json:\"enablePrometheus,omitempty\"`\n\n\tLog LogConfig `json:\"log,omitempty\"`\n\n\tTransport ServerTransportConfig `json:\"transport,omitempty\"`\n\n\t// DetailedErrorsToClient defines whether to send the specific error (with\n\t// debug info) to frpc. By default, this value is true.\n\tDetailedErrorsToClient *bool `json:\"detailedErrorsToClient,omitempty\"`\n\t// MaxPortsPerClient specifies the maximum number of ports a single client\n\t// may proxy to. If this value is 0, no limit will be applied.\n\tMaxPortsPerClient int64 `json:\"maxPortsPerClient,omitempty\"`\n\t// UserConnTimeout specifies the maximum time to wait for a work\n\t// connection. By default, this value is 10.\n\tUserConnTimeout int64 `json:\"userConnTimeout,omitempty\"`\n\t// UDPPacketSize specifies the UDP packet size\n\t// By default, this value is 1500\n\tUDPPacketSize int64 `json:\"udpPacketSize,omitempty\"`\n\t// NatHoleAnalysisDataReserveHours specifies the hours to reserve nat hole analysis data.\n\tNatHoleAnalysisDataReserveHours int64 `json:\"natholeAnalysisDataReserveHours,omitempty\"`\n\n\tAllowPorts []types.PortsRange `json:\"allowPorts,omitempty\"`\n\n\tHTTPPlugins []HTTPPluginOptions `json:\"httpPlugins,omitempty\"`\n}\n\nfunc (c *ServerConfig) Complete() {\n\tc.Auth.Complete()\n\tc.Log.Complete()\n\tc.Transport.Complete()\n\tc.WebServer.Complete()\n\tc.SSHTunnelGateway.Complete()\n\n\tc.BindAddr = util.EmptyOr(c.BindAddr, \"0.0.0.0\")\n\tc.BindPort = util.EmptyOr(c.BindPort, 7000)\n\tif c.ProxyBindAddr == \"\" {\n\t\tc.ProxyBindAddr = c.BindAddr\n\t}\n\n\tif c.WebServer.Port > 0 {\n\t\tc.WebServer.Addr = util.EmptyOr(c.WebServer.Addr, \"0.0.0.0\")\n\t}\n\n\tc.VhostHTTPTimeout = util.EmptyOr(c.VhostHTTPTimeout, 60)\n\tc.DetailedErrorsToClient = util.EmptyOr(c.DetailedErrorsToClient, lo.ToPtr(true))\n\tc.UserConnTimeout = util.EmptyOr(c.UserConnTimeout, 10)\n\tc.UDPPacketSize = util.EmptyOr(c.UDPPacketSize, 1500)\n\tc.NatHoleAnalysisDataReserveHours = util.EmptyOr(c.NatHoleAnalysisDataReserveHours, 7*24)\n}\n\ntype AuthServerConfig struct {\n\tMethod           AuthMethod           `json:\"method,omitempty\"`\n\tAdditionalScopes []AuthScope          `json:\"additionalScopes,omitempty\"`\n\tToken            string               `json:\"token,omitempty\"`\n\tOIDC             AuthOIDCServerConfig `json:\"oidc,omitempty\"`\n}\n\nfunc (c *AuthServerConfig) Complete() {\n\tc.Method = util.EmptyOr(c.Method, \"token\")\n}\n\ntype AuthOIDCServerConfig struct {\n\t// Issuer specifies the issuer to verify OIDC tokens with. This issuer\n\t// will be used to load public keys to verify signature and will be compared\n\t// with the issuer claim in the OIDC token.\n\tIssuer string `json:\"issuer,omitempty\"`\n\t// Audience specifies the audience OIDC tokens should contain when validated.\n\t// If this value is empty, audience (\"client ID\") verification will be skipped.\n\tAudience string `json:\"audience,omitempty\"`\n\t// SkipExpiryCheck specifies whether to skip checking if the OIDC token is\n\t// expired.\n\tSkipExpiryCheck bool `json:\"skipExpiryCheck,omitempty\"`\n\t// SkipIssuerCheck specifies whether to skip checking if the OIDC token's\n\t// issuer claim matches the issuer specified in OidcIssuer.\n\tSkipIssuerCheck bool `json:\"skipIssuerCheck,omitempty\"`\n}\n\ntype ServerTransportConfig struct {\n\t// TCPMux toggles TCP stream multiplexing. This allows multiple requests\n\t// from a client to share a single TCP connection. By default, this value\n\t// is true.\n\t// $HideFromDoc\n\tTCPMux *bool `json:\"tcpMux,omitempty\"`\n\t// TCPMuxKeepaliveInterval specifies the keep alive interval for TCP stream multiplier.\n\t// If TCPMux is true, heartbeat of application layer is unnecessary because it can only rely on heartbeat in TCPMux.\n\tTCPMuxKeepaliveInterval int64 `json:\"tcpMuxKeepaliveInterval,omitempty\"`\n\t// TCPKeepAlive specifies the interval between keep-alive probes for an active network connection between frpc and frps.\n...\n// Path: pkg/config/v1/server_test.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"testing\"\n\n\t\"github.com/samber/lo\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestServerConfigComplete(t *testing.T) {\n\trequire := require.New(t)\n\tc := &ServerConfig{}\n\tc.Complete()\n\n\trequire.EqualValues(\"token\", c.Auth.Method)\n\trequire.Equal(true, lo.FromPtr(c.Transport.TCPMux))\n\trequire.Equal(true, lo.FromPtr(c.DetailedErrorsToClient))\n}\n\n// Path: pkg/config/v1/validation/client.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n\n\t\"github.com/samber/lo\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateClientCommonConfig(c *v1.ClientCommonConfig) (Warning, error) {\n\tvar (\n\t\twarnings Warning\n\t\terrs     error\n\t)\n\tif !slices.Contains(SupportedAuthMethods, c.Auth.Method) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth method, optional values are %v\", SupportedAuthMethods))\n\t}\n\tif !lo.Every(SupportedAuthAdditionalScopes, c.Auth.AdditionalScopes) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth additional scopes, optional values are %v\", SupportedAuthAdditionalScopes))\n\t}\n\n\tif err := validateLogConfig(&c.Log); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\tif err := validateWebServerConfig(&c.WebServer); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\tif c.Transport.HeartbeatTimeout > 0 && c.Transport.HeartbeatInterval > 0 {\n\t\tif c.Transport.HeartbeatTimeout < c.Transport.HeartbeatInterval {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"invalid transport.heartbeatTimeout, heartbeat timeout should not less than heartbeat interval\"))\n\t\t}\n\t}\n\n\tif !lo.FromPtr(c.Transport.TLS.Enable) {\n\t\tcheckTLSConfig := func(name string, value string) Warning {\n\t\t\tif value != \"\" {\n\t\t\t\treturn fmt.Errorf(\"%s is invalid when transport.tls.enable is false\", name)\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\n\t\twarnings = AppendError(warnings, checkTLSConfig(\"transport.tls.certFile\", c.Transport.TLS.CertFile))\n\t\twarnings = AppendError(warnings, checkTLSConfig(\"transport.tls.keyFile\", c.Transport.TLS.KeyFile))\n\t\twarnings = AppendError(warnings, checkTLSConfig(\"transport.tls.trustedCaFile\", c.Transport.TLS.TrustedCaFile))\n\t}\n\n\tif !slices.Contains(SupportedTransportProtocols, c.Transport.Protocol) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid transport.protocol, optional values are %v\", SupportedTransportProtocols))\n\t}\n\n\tfor _, f := range c.IncludeConfigFiles {\n\t\tabsDir, err := filepath.Abs(filepath.Dir(f))\n\t\tif err != nil {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"include: parse directory of %s failed: %v\", f, err))\n\t\t\tcontinue\n\t\t}\n\t\tif _, err := os.Stat(absDir); os.IsNotExist(err) {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"include: directory of %s not exist\", f))\n\t\t}\n\t}\n\treturn warnings, errs\n}\n\n\nfunc ValidateAllClientConfig(c *v1.ClientCommonConfig, proxyCfgs []v1.ProxyConfigurer, visitorCfgs []v1.VisitorConfigurer) (Warning, error) {\n\tvar warnings Warning\n\tif c != nil {\n\t\twarning, err := ValidateClientCommonConfig(c)\n\t\twarnings = AppendError(warnings, warning)\n\t\tif err != nil {\n\t\t\treturn warnings, err\n\t\t}\n\t}\n\n\tfor _, c := range proxyCfgs {\n\t\tif err := ValidateProxyConfigurerForClient(c); err != nil {\n\t\t\treturn warnings, fmt.Errorf(\"proxy %s: %v\", c.GetBaseConfig().Name, err)\n\t\t}\n\t}\n\n\tfor _, c := range visitorCfgs {\n\t\tif err := ValidateVisitorConfigurer(c); err != nil {\n\t\t\treturn warnings, fmt.Errorf(\"visitor %s: %v\", c.GetBaseConfig().Name, err)\n\t\t}\n\t}\n\treturn warnings, nil\n}\n\n// Path: pkg/config/v1/validation/plugin.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"errors\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateClientPluginOptions(c v1.ClientPluginOptions) error {\n\tswitch v := c.(type) {\n\tcase *v1.HTTP2HTTPSPluginOptions:\n\t\treturn validateHTTP2HTTPSPluginOptions(v)\n\tcase *v1.HTTPS2HTTPPluginOptions:\n\t\treturn validateHTTPS2HTTPPluginOptions(v)\n\tcase *v1.HTTPS2HTTPSPluginOptions:\n\t\treturn validateHTTPS2HTTPSPluginOptions(v)\n\tcase *v1.StaticFilePluginOptions:\n\t\treturn validateStaticFilePluginOptions(v)\n\tcase *v1.UnixDomainSocketPluginOptions:\n\t\treturn validateUnixDomainSocketPluginOptions(v)\n\t}\n\treturn nil\n}\n\nfunc validateHTTP2HTTPSPluginOptions(c *v1.HTTP2HTTPSPluginOptions) error {\n\tif c.LocalAddr == \"\" {\n\t\treturn errors.New(\"localAddr is required\")\n\t}\n\treturn nil\n}\n\nfunc validateHTTPS2HTTPPluginOptions(c *v1.HTTPS2HTTPPluginOptions) error {\n\tif c.LocalAddr == \"\" {\n\t\treturn errors.New(\"localAddr is required\")\n\t}\n\treturn nil\n}\n\nfunc validateHTTPS2HTTPSPluginOptions(c *v1.HTTPS2HTTPSPluginOptions) error {\n\tif c.LocalAddr == \"\" {\n\t\treturn errors.New(\"localAddr is required\")\n\t}\n\treturn nil\n}\n\nfunc validateStaticFilePluginOptions(c *v1.StaticFilePluginOptions) error {\n\tif c.LocalPath == \"\" {\n\t\treturn errors.New(\"localPath is required\")\n\t}\n\treturn nil\n}\n\nfunc validateUnixDomainSocketPluginOptions(c *v1.UnixDomainSocketPluginOptions) error {\n\tif c.UnixPath == \"\" {\n\t\treturn errors.New(\"unixPath is required\")\n\t}\n\treturn nil\n}\n\n// Path: pkg/config/v1/validation/proxy.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"slices\"\n\t\"strings\"\n\n\t\"k8s.io/apimachinery/pkg/util/validation\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc validateProxyBaseConfigForClient(c *v1.ProxyBaseConfig) error {\n\tif c.Name == \"\" {\n\t\treturn errors.New(\"name should not be empty\")\n\t}\n\n\tif err := ValidateAnnotations(c.Annotations); err != nil {\n\t\treturn err\n\t}\n\tif !slices.Contains([]string{\"\", \"v1\", \"v2\"}, c.Transport.ProxyProtocolVersion) {\n\t\treturn fmt.Errorf(\"not support proxy protocol version: %s\", c.Transport.ProxyProtocolVersion)\n\t}\n\tif !slices.Contains([]string{\"client\", \"server\"}, c.Transport.BandwidthLimitMode) {\n\t\treturn fmt.Errorf(\"bandwidth limit mode should be client or server\")\n\t}\n\n\tif c.Plugin.Type == \"\" {\n\t\tif err := ValidatePort(c.LocalPort, \"localPort\"); err != nil {\n\t\t\treturn fmt.Errorf(\"localPort: %v\", err)\n\t\t}\n\t}\n\n\tif !slices.Contains([]string{\"\", \"tcp\", \"http\"}, c.HealthCheck.Type) {\n\t\treturn fmt.Errorf(\"not support health check type: %s\", c.HealthCheck.Type)\n\t}\n\tif c.HealthCheck.Type != \"\" {\n\t\tif c.HealthCheck.Type == \"http\" &&\n\t\t\tc.HealthCheck.Path == \"\" {\n\t\t\treturn fmt.Errorf(\"health check path should not be empty\")\n\t\t}\n\t}\n\n\tif c.Plugin.Type != \"\" {\n\t\tif err := ValidateClientPluginOptions(c.Plugin.ClientPluginOptions); err != nil {\n\t\t\treturn fmt.Errorf(\"plugin %s: %v\", c.Plugin.Type, err)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc validateProxyBaseConfigForServer(c *v1.ProxyBaseConfig) error {\n\tif err := ValidateAnnotations(c.Annotations); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc validateDomainConfigForClient(c *v1.DomainConfig) error {\n\tif c.SubDomain == \"\" && len(c.CustomDomains) == 0 {\n\t\treturn errors.New(\"subdomain and custom domains should not be both empty\")\n\t}\n\treturn nil\n}\n\nfunc validateDomainConfigForServer(c *v1.DomainConfig, s *v1.ServerConfig) error {\n\tfor _, domain := range c.CustomDomains {\n\t\tif s.SubDomainHost != \"\" && len(strings.Split(s.SubDomainHost, \".\")) < len(strings.Split(domain, \".\")) {\n\t\t\tif strings.Contains(domain, s.SubDomainHost) {\n\t\t\t\treturn fmt.Errorf(\"custom domain [%s] should not belong to subdomain host [%s]\", domain, s.SubDomainHost)\n\t\t\t}\n\t\t}\n\t}\n\n\tif c.SubDomain != \"\" {\n\t\tif s.SubDomainHost == \"\" {\n\t\t\treturn errors.New(\"subdomain is not supported because this feature is not enabled in server\")\n\t\t}\n\n\t\tif strings.Contains(c.SubDomain, \".\") || strings.Contains(c.SubDomain, \"*\") {\n\t\t\treturn errors.New(\"'.' and '*' are not supported in subdomain\")\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc ValidateProxyConfigurerForClient(c v1.ProxyConfigurer) error {\n\tbase := c.GetBaseConfig()\n\tif err := validateProxyBaseConfigForClient(base); err != nil {\n\t\treturn err\n\t}\n\n\tswitch v := c.(type) {\n\tcase *v1.TCPProxyConfig:\n\t\treturn validateTCPProxyConfigForClient(v)\n\tcase *v1.UDPProxyConfig:\n\t\treturn validateUDPProxyConfigForClient(v)\n\tcase *v1.TCPMuxProxyConfig:\n\t\treturn validateTCPMuxProxyConfigForClient(v)\n\tcase *v1.HTTPProxyConfig:\n\t\treturn validateHTTPProxyConfigForClient(v)\n\tcase *v1.HTTPSProxyConfig:\n\t\treturn validateHTTPSProxyConfigForClient(v)\n\tcase *v1.STCPProxyConfig:\n\t\treturn validateSTCPProxyConfigForClient(v)\n\tcase *v1.XTCPProxyConfig:\n\t\treturn validateXTCPProxyConfigForClient(v)\n\tcase *v1.SUDPProxyConfig:\n\t\treturn validateSUDPProxyConfigForClient(v)\n\t}\n\treturn errors.New(\"unknown proxy config type\")\n}\n\nfunc validateTCPProxyConfigForClient(c *v1.TCPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateUDPProxyConfigForClient(c *v1.UDPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateTCPMuxProxyConfigForClient(c *v1.TCPMuxProxyConfig) error {\n\tif err := validateDomainConfigForClient(&c.DomainConfig); err != nil {\n\t\treturn err\n\t}\n\n\tif !slices.Contains([]string{string(v1.TCPMultiplexerHTTPConnect)}, c.Multiplexer) {\n\t\treturn fmt.Errorf(\"not support multiplexer: %s\", c.Multiplexer)\n\t}\n\treturn nil\n}\n\nfunc validateHTTPProxyConfigForClient(c *v1.HTTPProxyConfig) error {\n\treturn validateDomainConfigForClient(&c.DomainConfig)\n}\n\nfunc validateHTTPSProxyConfigForClient(c *v1.HTTPSProxyConfig) error {\n\treturn validateDomainConfigForClient(&c.DomainConfig)\n}\n\nfunc validateSTCPProxyConfigForClient(c *v1.STCPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateXTCPProxyConfigForClient(c *v1.XTCPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateSUDPProxyConfigForClient(c *v1.SUDPProxyConfig) error {\n\treturn nil\n}\n\nfunc ValidateProxyConfigurerForServer(c v1.ProxyConfigurer, s *v1.ServerConfig) error {\n\tbase := c.GetBaseConfig()\n\tif err := validateProxyBaseConfigForServer(base); err != nil {\n\t\treturn err\n\t}\n\n\tswitch v := c.(type) {\n\tcase *v1.TCPProxyConfig:\n\t\treturn validateTCPProxyConfigForServer(v, s)\n\tcase *v1.UDPProxyConfig:\n\t\treturn validateUDPProxyConfigForServer(v, s)\n\tcase *v1.TCPMuxProxyConfig:\n\t\treturn validateTCPMuxProxyConfigForServer(v, s)\n\tcase *v1.HTTPProxyConfig:\n\t\treturn validateHTTPProxyConfigForServer(v, s)\n\tcase *v1.HTTPSProxyConfig:\n\t\treturn validateHTTPSProxyConfigForServer(v, s)\n\tcase *v1.STCPProxyConfig:\n\t\treturn validateSTCPProxyConfigForServer(v, s)\n\tcase *v1.XTCPProxyConfig:\n\t\treturn validateXTCPProxyConfigForServer(v, s)\n\tcase *v1.SUDPProxyConfig:\n\t\treturn validateSUDPProxyConfigForServer(v, s)\n\tdefault:\n\t\treturn errors.New(\"unknown proxy config type\")\n\t}\n}\n\nfunc validateTCPProxyConfigForServer(c *v1.TCPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateUDPProxyConfigForServer(c *v1.UDPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateTCPMuxProxyConfigForServer(c *v1.TCPMuxProxyConfig, s *v1.ServerConfig) error {\n\tif c.Multiplexer == string(v1.TCPMultiplexerHTTPConnect) &&\n\t\ts.TCPMuxHTTPConnectPort == 0 {\n\t\treturn fmt.Errorf(\"tcpmux with multiplexer httpconnect not supported because this feature is not enabled in server\")\n\t}\n\n\treturn validateDomainConfigForServer(&c.DomainConfig, s)\n}\n\nfunc validateHTTPProxyConfigForServer(c *v1.HTTPProxyConfig, s *v1.ServerConfig) error {\n\tif s.VhostHTTPPort == 0 {\n\t\treturn fmt.Errorf(\"type [http] not supported when vhost http port is not set\")\n\t}\n\n\treturn validateDomainConfigForServer(&c.DomainConfig, s)\n}\n\nfunc validateHTTPSProxyConfigForServer(c *v1.HTTPSProxyConfig, s *v1.ServerConfig) error {\n\tif s.VhostHTTPSPort == 0 {\n\t\treturn fmt.Errorf(\"type [https] not supported when vhost https port is not set\")\n\t}\n\n\treturn validateDomainConfigForServer(&c.DomainConfig, s)\n}\n\nfunc validateSTCPProxyConfigForServer(c *v1.STCPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateXTCPProxyConfigForServer(c *v1.XTCPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateSUDPProxyConfigForServer(c *v1.SUDPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\n// ValidateAnnotations validates that a set of annotations are correctly defined.\nfunc ValidateAnnotations(annotations map[string]string) error {\n\tif len(annotations) == 0 {\n\t\treturn nil\n\t}\n\n\tvar errs error\n\tfor k := range annotations {\n\t\tfor _, msg := range validation.IsQualifiedName(strings.ToLower(k)) {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"annotation key %s is invalid: %s\", k, msg))\n\t\t}\n\t}\n\tif err := ValidateAnnotationsSize(annotations); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\treturn errs\n}\n\nconst TotalAnnotationSizeLimitB int = 256 * (1 << 10) // 256 kB\n\nfunc ValidateAnnotationsSize(annotations map[string]string) error {\n\tvar totalSize int64\n\tfor k, v := range annotations {\n\t\ttotalSize += (int64)(len(k)) + (int64)(len(v))\n\t}\n\tif totalSize > (int64)(TotalAnnotationSizeLimitB) {\n\t\treturn fmt.Errorf(\"annotations size %d is larger than limit %d\", totalSize, TotalAnnotationSizeLimitB)\n\t}\n\treturn nil\n}\n\n// Path: pkg/config/v1/validation/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"fmt\"\n\t\"slices\"\n\n\t\"github.com/samber/lo\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateServerConfig(c *v1.ServerConfig) (Warning, error) {\n\tvar (\n\t\twarnings Warning\n\t\terrs     error\n\t)\n\tif !slices.Contains(SupportedAuthMethods, c.Auth.Method) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth method, optional values are %v\", SupportedAuthMethods))\n\t}\n\tif !lo.Every(SupportedAuthAdditionalScopes, c.Auth.AdditionalScopes) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth additional scopes, optional values are %v\", SupportedAuthAdditionalScopes))\n\t}\n\n\tif err := validateLogConfig(&c.Log); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\tif err := validateWebServerConfig(&c.WebServer); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\terrs = AppendError(errs, ValidatePort(c.BindPort, \"bindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.KCPBindPort, \"kcpBindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.QUICBindPort, \"quicBindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.VhostHTTPPort, \"vhostHTTPPort\"))\n\terrs = AppendError(errs, ValidatePort(c.VhostHTTPSPort, \"vhostHTTPSPort\"))\n\terrs = AppendError(errs, ValidatePort(c.TCPMuxHTTPConnectPort, \"tcpMuxHTTPConnectPort\"))\n\n\tfor _, p := range c.HTTPPlugins {\n\t\tif !lo.Every(SupportedHTTPPluginOps, p.Ops) {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"invalid http plugin ops, optional values are %v\", SupportedHTTPPluginOps))\n\t\t}\n\t}\n\treturn warnings, errs\n}\n\n// Path: pkg/config/v1/validation/visitor.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"slices\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateVisitorConfigurer(c v1.VisitorConfigurer) error {\n\tbase := c.GetBaseConfig()\n\tif err := validateVisitorBaseConfig(base); err != nil {\n\t\treturn err\n\t}\n\n\tswitch v := c.(type) {\n\tcase *v1.STCPVisitorConfig:\n\tcase *v1.SUDPVisitorConfig:\n\tcase *v1.XTCPVisitorConfig:\n\t\treturn validateXTCPVisitorConfig(v)\n\tdefault:\n\t\treturn errors.New(\"unknown visitor config type\")\n\t}\n\treturn nil\n}\n\nfunc validateVisitorBaseConfig(c *v1.VisitorBaseConfig) error {\n\tif c.Name == \"\" {\n\t\treturn errors.New(\"name is required\")\n\t}\n\n\tif c.ServerName == \"\" {\n\t\treturn errors.New(\"server name is required\")\n\t}\n\n\tif c.BindPort == 0 {\n\t\treturn errors.New(\"bind port is required\")\n\t}\n\treturn nil\n}\n\nfunc validateXTCPVisitorConfig(c *v1.XTCPVisitorConfig) error {\n\tif !slices.Contains([]string{\"kcp\", \"quic\"}, c.Protocol) {\n\t\treturn fmt.Errorf(\"protocol should be kcp or quic\")\n\t}\n\treturn nil\n}\n\n// Path: pkg/config/v1/visitor.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/samber/lo\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype VisitorTransport struct {\n\tUseEncryption  bool `json:\"useEncryption,omitempty\"`\n\tUseCompression bool `json:\"useCompression,omitempty\"`\n}\n\ntype VisitorBaseConfig struct {\n\tName      string           `json:\"name\"`\n\tType      string           `json:\"type\"`\n\tTransport VisitorTransport `json:\"transport,omitempty\"`\n\tSecretKey string           `json:\"secretKey,omitempty\"`\n\t// if the server user is not set, it defaults to the current user\n\tServerUser string `json:\"serverUser,omitempty\"`\n\tServerName string `json:\"serverName,omitempty\"`\n\tBindAddr   string `json:\"bindAddr,omitempty\"`\n\t// BindPort is the port that visitor listens on.\n\t// It can be less than 0, it means don't bind to the port and only receive connections redirected from\n\t// other visitors. (This is not supported for SUDP now)\n\tBindPort int `json:\"bindPort,omitempty\"`\n}\n\nfunc (c *VisitorBaseConfig) GetBaseConfig() *VisitorBaseConfig {\n\treturn c\n}\n\nfunc (c *VisitorBaseConfig) Complete(g *ClientCommonConfig) {\n\tif c.BindAddr == \"\" {\n\t\tc.BindAddr = \"127.0.0.1\"\n\t}\n\n\tnamePrefix := \"\"\n\tif g.User != \"\" {\n\t\tnamePrefix = g.User + \".\"\n\t}\n\tc.Name = namePrefix + c.Name\n\n\tif c.ServerUser != \"\" {\n\t\tc.ServerName = c.ServerUser + \".\" + c.ServerName\n\t} else {\n\t\tc.ServerName = namePrefix + c.ServerName\n\t}\n}\n\ntype VisitorConfigurer interface {\n\tComplete(*ClientCommonConfig)\n\tGetBaseConfig() *VisitorBaseConfig\n}\n\ntype VisitorType string\n\nconst (\n\tVisitorTypeSTCP VisitorType = \"stcp\"\n\tVisitorTypeXTCP VisitorType = \"xtcp\"\n\tVisitorTypeSUDP VisitorType = \"sudp\"\n)\n\nvar visitorConfigTypeMap = map[VisitorType]reflect.Type{\n\tVisitorTypeSTCP: reflect.TypeOf(STCPVisitorConfig{}),\n\tVisitorTypeXTCP: reflect.TypeOf(XTCPVisitorConfig{}),\n\tVisitorTypeSUDP: reflect.TypeOf(SUDPVisitorConfig{}),\n}\n\ntype TypedVisitorConfig struct {\n\tType string `json:\"type\"`\n\tVisitorConfigurer\n}\n\nfunc (c *TypedVisitorConfig) UnmarshalJSON(b []byte) error {\n\tif len(b) == 4 && string(b) == \"null\" {\n\t\treturn errors.New(\"type is required\")\n\t}\n\n\ttypeStruct := struct {\n\t\tType string `json:\"type\"`\n\t}{}\n\tif err := json.Unmarshal(b, &typeStruct); err != nil {\n\t\treturn err\n\t}\n\n\tc.Type = typeStruct.Type\n\tconfigurer := NewVisitorConfigurerByType(VisitorType(typeStruct.Type))\n\tif configurer == nil {\n\t\treturn fmt.Errorf(\"unknown visitor type: %s\", typeStruct.Type)\n\t}\n\tdecoder := json.NewDecoder(bytes.NewBuffer(b))\n\tif DisallowUnknownFields {\n\t\tdecoder.DisallowUnknownFields()\n\t}\n\tif err := decoder.Decode(configurer); err != nil {\n\t\treturn fmt.Errorf(\"unmarshal VisitorConfig error: %v\", err)\n\t}\n\tc.VisitorConfigurer = configurer\n\treturn nil\n}\n\nfunc NewVisitorConfigurerByType(t VisitorType) VisitorConfigurer {\n\tv, ok := visitorConfigTypeMap[t]\n\tif !ok {\n\t\treturn nil\n\t}\n\tvc := reflect.New(v).Interface().(VisitorConfigurer)\n\tvc.GetBaseConfig().Type = string(t)\n\treturn vc\n}\n\nvar _ VisitorConfigurer = &STCPVisitorConfig{}\n\ntype STCPVisitorConfig struct {\n\tVisitorBaseConfig\n}\n\nvar _ VisitorConfigurer = &SUDPVisitorConfig{}\n\ntype SUDPVisitorConfig struct {\n\tVisitorBaseConfig\n}\n\nvar _ VisitorConfigurer = &XTCPVisitorConfig{}\n\ntype XTCPVisitorConfig struct {\n\tVisitorBaseConfig\n\n\tProtocol          string `json:\"protocol,omitempty\"`\n\tKeepTunnelOpen    bool   `json:\"keepTunnelOpen,omitempty\"`\n\tMaxRetriesAnHour  int    `json:\"maxRetriesAnHour,omitempty\"`\n\tMinRetryInterval  int    `json:\"minRetryInterval,omitempty\"`\n\tFallbackTo        string `json:\"fallbackTo,omitempty\"`\n\tFallbackTimeoutMs int    `json:\"fallbackTimeoutMs,omitempty\"`\n}\n\nfunc (c *XTCPVisitorConfig) Complete(g *ClientCommonConfig) {\n\tc.VisitorBaseConfig.Complete(g)\n\n\tc.Protocol = util.EmptyOr(c.Protocol, \"quic\")\n\tc.MaxRetriesAnHour = util.EmptyOr(c.MaxRetriesAnHour, 8)\n\tc.MinRetryInterval = util.EmptyOr(c.MinRetryInterval, 90)\n\tc.FallbackTimeoutMs = util.EmptyOr(c.FallbackTimeoutMs, 1000)\n\n\tif c.FallbackTo != \"\" {\n\t\tc.FallbackTo = lo.Ternary(g.User == \"\", \"\", g.User+\".\") + c.FallbackTo\n\t}\n}\n\n// Path: pkg/metrics/metrics.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"github.com/fatedier/frp/pkg/metrics/aggregate\"\n)\n\nvar (\n\tEnableMem        = aggregate.EnableMem\n\tEnablePrometheus = aggregate.EnablePrometheus\n)\n\n// Path: pkg/util/metric/metrics.go\n// Copyright 2020 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metric\n\n// GaugeMetric represents a single numerical value that can arbitrarily go up\n// and down.\ntype GaugeMetric interface {\n\tInc()\n\tDec()\n\tSet(float64)\n}\n\n// CounterMetric represents a single numerical value that only ever\n// goes up.\ntype CounterMetric interface {\n\tInc()\n}\n\n// HistogramMetric counts individual observations.\ntype HistogramMetric interface {\n\tObserve(float64)\n}\n\n// Path: pkg/metrics/aggregate/server.go\n// Copyright 2020 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage aggregate\n\nimport (\n\t\"github.com/fatedier/frp/pkg/metrics/mem\"\n\t\"github.com/fatedier/frp/pkg/metrics/prometheus\"\n\t\"github.com/fatedier/frp/server/metrics\"\n)\n\n// EnableMem start to mark metrics to memory monitor system.\nfunc EnableMem() {\n\tsm.Add(mem.ServerMetrics)\n}\n\n// EnablePrometheus start to mark metrics to prometheus.\nfunc EnablePrometheus() {\n\tsm.Add(prometheus.ServerMetrics)\n}\n\nvar sm = &serverMetrics{}\n\nfunc init() {\n\tmetrics.Register(sm)\n}\n\ntype serverMetrics struct {\n\tms []metrics.ServerMetrics\n}\n\nfunc (m *serverMetrics) Add(sm metrics.ServerMetrics) {\n\tm.ms = append(m.ms, sm)\n}\n\nfunc (m *serverMetrics) NewClient() {\n\tfor _, v := range m.ms {\n\t\tv.NewClient()\n\t}\n}\n\nfunc (m *serverMetrics) CloseClient() {\n\tfor _, v := range m.ms {\n\t\tv.CloseClient()\n\t}\n}\n\nfunc (m *serverMetrics) NewProxy(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.NewProxy(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) CloseProxy(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.CloseProxy(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) OpenConnection(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.OpenConnection(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) CloseConnection(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.CloseConnection(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficIn(name string, proxyType string, trafficBytes int64) {\n\tfor _, v := range m.ms {\n\t\tv.AddTrafficIn(name, proxyType, trafficBytes)\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficOut(name string, proxyType string, trafficBytes int64) {\n\tfor _, v := range m.ms {\n\t\tv.AddTrafficOut(name, proxyType, trafficBytes)\n\t}\n}\n\n// Path: pkg/util/limit/writer.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage limit\n\nimport (\n\t\"context\"\n\t\"io\"\n\n\t\"golang.org/x/time/rate\"\n)\n\ntype Writer struct {\n\tw       io.Writer\n\tlimiter *rate.Limiter\n}\n\nfunc NewWriter(w io.Writer, limiter *rate.Limiter) *Writer {\n\treturn &Writer{\n\t\tw:       w,\n\t\tlimiter: limiter,\n\t}\n}\n\nfunc (w *Writer) Write(p []byte) (n int, err error) {\n\tvar nn int\n\tb := w.limiter.Burst()\n\tfor {\n\t\tend := len(p)\n\t\tif end == 0 {\n\t\t\tbreak\n\t\t}\n\t\tif b < len(p) {\n\t\t\tend = b\n\t\t}\n\t\terr = w.limiter.WaitN(context.Background(), end)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tnn, err = w.w.Write(p[:end])\n\t\tn += nn\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tp = p[end:]\n\t}\n\treturn\n}\n\n// Path: pkg/util/log/log.go\n// Copyright 2016 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\n\t\"github.com/fatedier/golib/log\"\n)\n\nvar (\n\tTraceLevel = log.TraceLevel\n\tDebugLevel = log.DebugLevel\n\tInfoLevel  = log.InfoLevel\n\tWarnLevel  = log.WarnLevel\n\tErrorLevel = log.ErrorLevel\n)\n\nvar Logger *log.Logger\n\nfunc init() {\n\tLogger = log.New(\n\t\tlog.WithCaller(true),\n\t\tlog.AddCallerSkip(1),\n\t\tlog.WithLevel(log.InfoLevel),\n\t)\n}\n\nfunc InitLogger(logPath string, levelStr string, maxDays int, disableLogColor bool) {\n\toptions := []log.Option{}\n\tif logPath == \"console\" {\n\t\tif !disableLogColor {\n\t\t\toptions = append(options,\n\t\t\t\tlog.WithOutput(log.NewConsoleWriter(log.ConsoleConfig{\n\t\t\t\t\tColorful: true,\n\t\t\t\t}, os.Stdout)),\n\t\t\t)\n\t\t}\n\t} else {\n\t\twriter := log.NewRotateFileWriter(log.RotateFileConfig{\n\t\t\tFileName: logPath,\n\t\t\tMode:     log.RotateFileModeDaily,\n\t\t\tMaxDays:  maxDays,\n\t\t})\n\t\twriter.Init()\n\t\toptions = append(options, log.WithOutput(writer))\n\t}\n\n\tlevel, err := log.ParseLevel(levelStr)\n\tif err != nil {\n\t\tlevel = log.InfoLevel\n\t}\n\toptions = append(options, log.WithLevel(level))\n\tLogger = Logger.WithOptions(options...)\n}\n\nfunc Errorf(format string, v ...interface{}) {\n\tLogger.Errorf(format, v...)\n}\n\nfunc Warnf(format string, v ...interface{}) {\n\tLogger.Warnf(format, v...)\n}\n\nfunc Infof(format string, v ...interface{}) {\n\tLogger.Infof(format, v...)\n}\n\nfunc Debugf(format string, v ...interface{}) {\n\tLogger.Debugf(format, v...)\n}\n\nfunc Tracef(format string, v ...interface{}) {\n\tLogger.Tracef(format, v...)\n}\n\nfunc Logf(level log.Level, offset int, format string, v ...interface{}) {\n\tLogger.Logf(level, offset, format, v...)\n}\n\ntype WriteLogger struct {\n\tlevel  log.Level\n\toffset int\n}\n\nfunc NewWriteLogger(level log.Level, offset int) *WriteLogger {\n\treturn &WriteLogger{\n\t\tlevel:  level,\n\t\toffset: offset,\n\t}\n}\n\nfunc (w *WriteLogger) Write(p []byte) (n int, err error) {\n\tLogger.Log(w.level, w.offset, string(bytes.TrimRight(p, \"\\n\")))\n\treturn len(p), nil\n}\n\n// Path: pkg/util/xlog/xlog.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage xlog\n\nimport (\n\t\"cmp\"\n\t\"slices\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n)\n\ntype LogPrefix struct {\n\t// Name is the name of the prefix, it won't be displayed in log but used to identify the prefix.\n\tName string\n\t// Value is the value of the prefix, it will be displayed in log.\n\tValue string\n\t// The prefix with higher priority will be displayed first, default is 10.\n\tPriority int\n}\n\n// Logger is not thread safety for operations on prefix\ntype Logger struct {\n\tprefixes []LogPrefix\n\n\tprefixString string\n}\n\nfunc New() *Logger {\n\treturn &Logger{\n\t\tprefixes: make([]LogPrefix, 0),\n\t}\n}\n\nfunc (l *Logger) ResetPrefixes() (old []LogPrefix) {\n\told = l.prefixes\n\tl.prefixes = make([]LogPrefix, 0)\n\tl.prefixString = \"\"\n\treturn\n}\n\nfunc (l *Logger) AppendPrefix(prefix string) *Logger {\n\treturn l.AddPrefix(LogPrefix{\n\t\tName:     prefix,\n\t\tValue:    prefix,\n\t\tPriority: 10,\n\t})\n}\n\nfunc (l *Logger) AddPrefix(prefix LogPrefix) *Logger {\n\tfound := false\n\tif prefix.Priority <= 0 {\n\t\tprefix.Priority = 10\n\t}\n\tfor _, p := range l.prefixes {\n\t\tif p.Name == prefix.Name {\n\t\t\tfound = true\n\t\t\tp.Value = prefix.Value\n\t\t\tp.Priority = prefix.Priority\n\t\t}\n\t}\n\tif !found {\n\t\tl.prefixes = append(l.prefixes, prefix)\n\t}\n\tl.renderPrefixString()\n\treturn l\n}\n\nfunc (l *Logger) renderPrefixString() {\n\tslices.SortStableFunc(l.prefixes, func(a, b LogPrefix) int {\n\t\treturn cmp.Compare(a.Priority, b.Priority)\n\t})\n\tl.prefixString = \"\"\n\tfor _, v := range l.prefixes {\n\t\tl.prefixString += \"[\" + v.Value + \"] \"\n\t}\n}\n\nfunc (l *Logger) Spawn() *Logger {\n\tnl := New()\n\tnl.prefixes = append(nl.prefixes, l.prefixes...)\n\tnl.renderPrefixString()\n\treturn nl\n}\n\nfunc (l *Logger) Errorf(format string, v ...interface{}) {\n\tlog.Logger.Errorf(l.prefixString+format, v...)\n}\n\nfunc (l *Logger) Warnf(format string, v ...interface{}) {\n\tlog.Logger.Warnf(l.prefixString+format, v...)\n}\n\nfunc (l *Logger) Infof(format string, v ...interface{}) {\n\tlog.Logger.Infof(l.prefixString+format, v...)\n}\n\nfunc (l *Logger) Debugf(format string, v ...interface{}) {\n\tlog.Logger.Debugf(l.prefixString+format, v...)\n}\n\nfunc (l *Logger) Tracef(format string, v ...interface{}) {\n\tlog.Logger.Tracef(l.prefixString+format, v...)\n}\n\n// Path: pkg/ssh/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ssh\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\tlibio \"github.com/fatedier/golib/io\"\n\t\"github.com/spf13/cobra\"\n\tflag \"github.com/spf13/pflag\"\n\t\"golang.org/x/crypto/ssh\"\n\n\t\"github.com/fatedier/frp/client/proxy\"\n\t\"github.com/fatedier/frp/pkg/config\"\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/msg\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n\t\"github.com/fatedier/frp/pkg/util/xlog\"\n\t\"github.com/fatedier/frp/pkg/virtual\"\n)\n\nconst (\n\t// https://datatracker.ietf.org/doc/html/rfc4254#page-16\n\tChannelTypeServerOpenChannel = \"forwarded-tcpip\"\n\tRequestTypeForward           = \"tcpip-forward\"\n)\n\ntype tcpipForward struct {\n\tHost string\n\tPort uint32\n}\n\n// https://datatracker.ietf.org/doc/html/rfc4254#page-16\ntype forwardedTCPPayload struct {\n\tAddr string\n\tPort uint32\n\n\tOriginAddr string\n\tOriginPort uint32\n}\n\ntype TunnelServer struct {\n\tunderlyingConn net.Conn\n\tsshConn        *ssh.ServerConn\n\tsc             *ssh.ServerConfig\n\tfirstChannel   ssh.Channel\n\n\tvc                 *virtual.Client\n\tpeerServerListener *netpkg.InternalListener\n\tdoneCh             chan struct{}\n\tcloseDoneChOnce    sync.Once\n}\n\nfunc NewTunnelServer(conn net.Conn, sc *ssh.ServerConfig, peerServerListener *netpkg.InternalListener) (*TunnelServer, error) {\n\ts := &TunnelServer{\n\t\tunderlyingConn:     conn,\n\t\tsc:                 sc,\n\t\tpeerServerListener: peerServerListener,\n\t\tdoneCh:             make(chan struct{}),\n\t}\n\treturn s, nil\n}\n\nfunc (s *TunnelServer) Run() error {\n\tsshConn, channels, requests, err := ssh.NewServerConn(s.underlyingConn, s.sc)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ts.sshConn = sshConn\n\n\taddr, extraPayload, err := s.waitForwardAddrAndExtraPayload(channels, requests, 3*time.Second)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tclientCfg, pc, helpMessage, err := s.parseClientAndProxyConfigurer(addr, extraPayload)\n\tif err != nil {\n\t\tif errors.Is(err, flag.ErrHelp) {\n\t\t\ts.writeToClient(helpMessage)\n\t\t\treturn nil\n\t\t}\n...\n", "needle_token_start": 3740, "needle_token_end": 3997, "code_context_ntokens": 15372, "output": ["```go\nfunc ValidateAllClientConfig(c *v1.ClientCommonConfig, proxyCfgs []v1.ProxyConfigurer, visitorCfgs []v1.VisitorConfigurer) (Warning, error) {\n\tvar warnings Warning\n\tif c != nil {\n\t\twarning, err := ValidateClientCommonConfig(c)\n\t\twarnings = AppendError(warnings, warning)\n\t\tif err != nil {\n\t\t\treturn warnings, err\n\t\t}\n\t}\n\n\tfor _, c := range proxyCfgs {\n\t\tif err := ValidateProxyConfigurerForClient(c); err != nil {\n\t\t\treturn warnings, fmt.Errorf(\"proxy %s: %v\", c.GetBaseConfig().Name, err)\n\t\t}\n\t}\n\n\tfor _, c := range visitorCfgs {\n\t\tif err := ValidateVisitorConfigurer(c); err != nil {\n\t\t\treturn warnings, fmt.Errorf(\"visitor %s: %v\", c.GetBaseConfig().Name, err)\n\t\t}\n\t}\n\treturn warnings, nil\n}\n```"]}
{"repo": "fatedier/frp", "name": "getNotFoundPageContent", "language": "go", "path": "pkg/util/vhost/resource.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To provide the content of a 404 error page, either from a predefined HTML template or from a custom file if specified.\n2. **Input**: None.\n3. **Output**: Returns a byte array containing the HTML content of the 404 error page.\n4. **Procedure**: The function first checks if a custom file path for the 404 page is specified. If it is, the function attempts to read from this file. If the file read is successful, the content of the file is used. If there is an error in reading the file, or if no custom path is specified, a predefined HTML template is used as the error page content.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/util/vhost/vhost.go\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/errors\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n\t\"github.com/fatedier/frp/pkg/util/xlog\"\n)\n\ntype RouteInfo string\n\nconst (\n\tRouteInfoKey RouteInfo = \"routeInfo\"\n)\n\ntype RequestRouteInfo struct {\n\tURL        string\n\tHost       string\n\tHTTPUser   string\n\tRemoteAddr string\n\tURLHost    string\n\tEndpoint   string\n}\n\ntype (\n\tmuxFunc         func(net.Conn) (net.Conn, map[string]string, error)\n\tauthFunc        func(conn net.Conn, username, password string, reqInfoMap map[string]string) (bool, error)\n\thostRewriteFunc func(net.Conn, string) (net.Conn, error)\n\tsuccessHookFunc func(net.Conn, map[string]string) error\n\tfailHookFunc    func(net.Conn)\n)\n\n// Muxer is a functional component used for https and tcpmux proxies.\n// It accepts connections and extracts vhost information from the beginning of the connection data.\n// It then routes the connection to its appropriate listener.\ntype Muxer struct {\n\tlistener net.Listener\n\ttimeout  time.Duration\n\n\tvhostFunc      muxFunc\n\tcheckAuth      authFunc\n\tsuccessHook    successHookFunc\n\tfailHook       failHookFunc\n\trewriteHost    hostRewriteFunc\n\tregistryRouter *Routers\n}\n\nfunc NewMuxer(\n\tlistener net.Listener,\n\tvhostFunc muxFunc,\n\ttimeout time.Duration,\n) (mux *Muxer, err error) {\n\tmux = &Muxer{\n\t\tlistener:       listener,\n\t\ttimeout:        timeout,\n\t\tvhostFunc:      vhostFunc,\n\t\tregistryRouter: NewRouters(),\n\t}\n\tgo mux.run()\n\treturn mux, nil\n}\n\nfunc (v *Muxer) SetCheckAuthFunc(f authFunc) *Muxer {\n\tv.checkAuth = f\n\treturn v\n}\n\nfunc (v *Muxer) SetSuccessHookFunc(f successHookFunc) *Muxer {\n\tv.successHook = f\n\treturn v\n}\n\nfunc (v *Muxer) SetFailHookFunc(f failHookFunc) *Muxer {\n\tv.failHook = f\n\treturn v\n}\n\nfunc (v *Muxer) SetRewriteHostFunc(f hostRewriteFunc) *Muxer {\n\tv.rewriteHost = f\n\treturn v\n}\n\ntype ChooseEndpointFunc func() (string, error)\n\ntype CreateConnFunc func(remoteAddr string) (net.Conn, error)\n\ntype CreateConnByEndpointFunc func(endpoint, remoteAddr string) (net.Conn, error)\n\n// RouteConfig is the params used to match HTTP requests\ntype RouteConfig struct {\n\tDomain          string\n\tLocation        string\n\tRewriteHost     string\n\tUsername        string\n\tPassword        string\n\tHeaders         map[string]string\n\tRouteByHTTPUser string\n\n\tCreateConnFn           CreateConnFunc\n\tChooseEndpointFn       ChooseEndpointFunc\n\tCreateConnByEndpointFn CreateConnByEndpointFunc\n}\n\n// listen for a new domain name, if rewriteHost is not empty and rewriteHost func is not nil,\n// then rewrite the host header to rewriteHost\nfunc (v *Muxer) Listen(ctx context.Context, cfg *RouteConfig) (l *Listener, err error) {\n\tl = &Listener{\n\t\tname:            cfg.Domain,\n\t\tlocation:        cfg.Location,\n\t\trouteByHTTPUser: cfg.RouteByHTTPUser,\n\t\trewriteHost:     cfg.RewriteHost,\n\t\tusername:        cfg.Username,\n\t\tpassword:        cfg.Password,\n\t\tmux:             v,\n\t\taccept:          make(chan net.Conn),\n\t\tctx:             ctx,\n\t}\n\terr = v.registryRouter.Add(cfg.Domain, cfg.Location, cfg.RouteByHTTPUser, l)\n\tif err != nil {\n\t\treturn\n\t}\n\treturn l, nil\n}\n\nfunc (v *Muxer) getListener(name, path, httpUser string) (*Listener, bool) {\n\tfindRouter := func(inName, inPath, inHTTPUser string) (*Listener, bool) {\n\t\tvr, ok := v.registryRouter.Get(inName, inPath, inHTTPUser)\n\t\tif ok {\n\t\t\treturn vr.payload.(*Listener), true\n\t\t}\n\t\t// Try to check if there is one proxy that doesn't specify routerByHTTPUser, it means match all.\n\t\tvr, ok = v.registryRouter.Get(inName, inPath, \"\")\n\t\tif ok {\n\t\t\treturn vr.payload.(*Listener), true\n\t\t}\n\t\treturn nil, false\n\t}\n\n\t// first we check the full hostname\n\t// if not exist, then check the wildcard_domain such as *.example.com\n\tl, ok := findRouter(name, path, httpUser)\n\tif ok {\n\t\treturn l, true\n\t}\n\n...\n// Path: pkg/util/tcpmux/httpconnect.go\n// Copyright 2020 guylewin, guy@lewin.co.il\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage tcpmux\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n\n\tlibnet \"github.com/fatedier/golib/net\"\n\n\thttppkg \"github.com/fatedier/frp/pkg/util/http\"\n\t\"github.com/fatedier/frp/pkg/util/vhost\"\n)\n\ntype HTTPConnectTCPMuxer struct {\n\t*vhost.Muxer\n\n\t// If passthrough is set to true, the CONNECT request will be forwarded to the backend service.\n\t// Otherwise, it will return an OK response to the client and forward the remaining content to the backend service.\n\tpassthrough bool\n}\n\nfunc NewHTTPConnectTCPMuxer(listener net.Listener, passthrough bool, timeout time.Duration) (*HTTPConnectTCPMuxer, error) {\n\tret := &HTTPConnectTCPMuxer{passthrough: passthrough}\n\tmux, err := vhost.NewMuxer(listener, ret.getHostFromHTTPConnect, timeout)\n\tmux.SetCheckAuthFunc(ret.auth).\n\t\tSetSuccessHookFunc(ret.sendConnectResponse).\n\t\tSetFailHookFunc(vhostFailed)\n\tret.Muxer = mux\n\treturn ret, err\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) readHTTPConnectRequest(rd io.Reader) (host, httpUser, httpPwd string, err error) {\n\tbufioReader := bufio.NewReader(rd)\n\n\treq, err := http.ReadRequest(bufioReader)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tif req.Method != \"CONNECT\" {\n\t\terr = fmt.Errorf(\"connections to tcp vhost must be of method CONNECT\")\n\t\treturn\n\t}\n\n\thost, _ = httppkg.CanonicalHost(req.Host)\n\tproxyAuth := req.Header.Get(\"Proxy-Authorization\")\n\tif proxyAuth != \"\" {\n\t\thttpUser, httpPwd, _ = httppkg.ParseBasicAuth(proxyAuth)\n\t}\n\treturn\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) sendConnectResponse(c net.Conn, _ map[string]string) error {\n\tif muxer.passthrough {\n\t\treturn nil\n\t}\n\tres := httppkg.OkResponse()\n\tif res.Body != nil {\n\t\tdefer res.Body.Close()\n\t}\n\treturn res.Write(c)\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) auth(c net.Conn, username, password string, reqInfo map[string]string) (bool, error) {\n\treqUsername := reqInfo[\"HTTPUser\"]\n\treqPassword := reqInfo[\"HTTPPwd\"]\n\tif username == reqUsername && password == reqPassword {\n\t\treturn true, nil\n\t}\n\n\tresp := httppkg.ProxyUnauthorizedResponse()\n\tif resp.Body != nil {\n\t\tdefer resp.Body.Close()\n\t}\n\t_ = resp.Write(c)\n\treturn false, nil\n}\n\nfunc vhostFailed(c net.Conn) {\n\tres := vhost.NotFoundResponse()\n\tif res.Body != nil {\n\t\tdefer res.Body.Close()\n\t}\n\t_ = res.Write(c)\n\t_ = c.Close()\n}\n\nfunc (muxer *HTTPConnectTCPMuxer) getHostFromHTTPConnect(c net.Conn) (net.Conn, map[string]string, error) {\n\treqInfoMap := make(map[string]string, 0)\n\tsc, rd := libnet.NewSharedConn(c)\n\n\thost, httpUser, httpPwd, err := muxer.readHTTPConnectRequest(rd)\n\tif err != nil {\n\t\treturn nil, reqInfoMap, err\n\t}\n\n\treqInfoMap[\"Host\"] = host\n\treqInfoMap[\"Scheme\"] = \"tcp\"\n\treqInfoMap[\"HTTPUser\"] = httpUser\n\treqInfoMap[\"HTTPPwd\"] = httpPwd\n\n\toutConn := c\n\tif muxer.passthrough {\n\t\toutConn = sc\n\t}\n\treturn outConn, reqInfoMap, nil\n}\n\n// Path: pkg/util/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRandId(t *testing.T) {\n\tassert := assert.New(t)\n\tid, err := RandID()\n\tassert.NoError(err)\n\tt.Log(id)\n\tassert.Equal(16, len(id))\n}\n\nfunc TestGetAuthKey(t *testing.T) {\n\tassert := assert.New(t)\n\tkey := GetAuthKey(\"1234\", 1488720000)\n\tassert.Equal(\"6df41a43725f0c770fd56379e12acf8c\", key)\n}\n\nfunc TestParseRangeNumbers(t *testing.T) {\n\tassert := assert.New(t)\n\tnumbers, err := ParseRangeNumbers(\"2-5\")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{2, 3, 4, 5}, numbers)\n\t}\n\n\tnumbers, err = ParseRangeNumbers(\"1\")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{1}, numbers)\n\t}\n\n\tnumbers, err = ParseRangeNumbers(\"3-5,8\")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{3, 4, 5, 8}, numbers)\n\t}\n\n\tnumbers, err = ParseRangeNumbers(\" 3-5,8, 10-12 \")\n\tif assert.NoError(err) {\n\t\tassert.Equal([]int64{3, 4, 5, 8, 10, 11, 12}, numbers)\n\t}\n\n\t_, err = ParseRangeNumbers(\"3-a\")\n\tassert.Error(err)\n}\n\n// Path: pkg/util/version/version.go\n// Copyright 2016 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage version\n\nvar version = \"0.57.0\"\n\nfunc Full() string {\n\treturn version\n}\n\n// Path: pkg/util/vhost/https.go\n// Copyright 2016 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"crypto/tls\"\n\t\"io\"\n\t\"net\"\n\t\"time\"\n\n\tlibnet \"github.com/fatedier/golib/net\"\n)\n\ntype HTTPSMuxer struct {\n\t*Muxer\n}\n\nfunc NewHTTPSMuxer(listener net.Listener, timeout time.Duration) (*HTTPSMuxer, error) {\n\tmux, err := NewMuxer(listener, GetHTTPSHostname, timeout)\n\tmux.SetFailHookFunc(vhostFailed)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &HTTPSMuxer{mux}, err\n}\n\nfunc GetHTTPSHostname(c net.Conn) (_ net.Conn, _ map[string]string, err error) {\n\treqInfoMap := make(map[string]string, 0)\n\tsc, rd := libnet.NewSharedConn(c)\n\n\tclientHello, err := readClientHello(rd)\n\tif err != nil {\n\t\treturn nil, reqInfoMap, err\n\t}\n\n\treqInfoMap[\"Host\"] = clientHello.ServerName\n\treqInfoMap[\"Scheme\"] = \"https\"\n\treturn sc, reqInfoMap, nil\n}\n\nfunc readClientHello(reader io.Reader) (*tls.ClientHelloInfo, error) {\n\tvar hello *tls.ClientHelloInfo\n\n\t// Note that Handshake always fails because the readOnlyConn is not a real connection.\n\t// As long as the Client Hello is successfully read, the failure should only happen after GetConfigForClient is called,\n\t// so we only care about the error if hello was never set.\n\terr := tls.Server(readOnlyConn{reader: reader}, &tls.Config{\n\t\tGetConfigForClient: func(argHello *tls.ClientHelloInfo) (*tls.Config, error) {\n\t\t\thello = &tls.ClientHelloInfo{}\n\t\t\t*hello = *argHello\n\t\t\treturn nil, nil\n\t\t},\n\t}).Handshake()\n\n\tif hello == nil {\n\t\treturn nil, err\n\t}\n\treturn hello, nil\n}\n\nfunc vhostFailed(c net.Conn) {\n\t// Alert with alertUnrecognizedName\n\t_ = tls.Server(c, &tls.Config{}).Handshake()\n\tc.Close()\n}\n\ntype readOnlyConn struct {\n\treader io.Reader\n}\n\nfunc (conn readOnlyConn) Read(p []byte) (int, error)         { return conn.reader.Read(p) }\nfunc (conn readOnlyConn) Write(_ []byte) (int, error)        { return 0, io.ErrClosedPipe }\nfunc (conn readOnlyConn) Close() error                       { return nil }\nfunc (conn readOnlyConn) LocalAddr() net.Addr                { return nil }\nfunc (conn readOnlyConn) RemoteAddr() net.Addr               { return nil }\nfunc (conn readOnlyConn) SetDeadline(_ time.Time) error      { return nil }\nfunc (conn readOnlyConn) SetReadDeadline(_ time.Time) error  { return nil }\nfunc (conn readOnlyConn) SetWriteDeadline(_ time.Time) error { return nil }\n\n// Path: pkg/util/vhost/https_test.go\npackage vhost\n\nimport (\n\t\"crypto/tls\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestGetHTTPSHostname(t *testing.T) {\n\trequire := require.New(t)\n\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:\")\n\trequire.NoError(err)\n\tdefer l.Close()\n\n\tvar conn net.Conn\n\tgo func() {\n\t\tconn, _ = l.Accept()\n\t\trequire.NotNil(conn)\n\t}()\n\n\tgo func() {\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\ttls.Dial(\"tcp\", l.Addr().String(), &tls.Config{\n\t\t\tInsecureSkipVerify: true,\n\t\t\tServerName:         \"example.com\",\n\t\t})\n\t}()\n\n\ttime.Sleep(200 * time.Millisecond)\n\t_, infos, err := GetHTTPSHostname(conn)\n\trequire.NoError(err)\n\trequire.Equal(\"example.com\", infos[\"Host\"])\n\trequire.Equal(\"https\", infos[\"Scheme\"])\n}\n\n// Path: pkg/util/vhost/resource.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\t\"github.com/fatedier/frp/pkg/util/version\"\n)\n\nvar NotFoundPagePath = \"\"\n\nconst (\n\tNotFound = `<!DOCTYPE html>\n<html>\n<head>\n<title>Not Found</title>\n<style>\n    body {\n        width: 35em;\n        margin: 0 auto;\n        font-family: Tahoma, Verdana, Arial, sans-serif;\n    }\n</style>\n</head>\n<body>\n<h1>The page you requested was not found.</h1>\n<p>Sorry, the page you are looking for is currently unavailable.<br/>\nPlease try again later.</p>\n<p>The server is powered by <a href=\"https://github.com/fatedier/frp\">frp</a>.</p>\n<p><em>Faithfully yours, frp.</em></p>\n</body>\n</html>\n`\n)\n\n\nfunc getNotFoundPageContent() []byte {\n\tvar (\n\t\tbuf []byte\n\t\terr error\n\t)\n\tif NotFoundPagePath != \"\" {\n\t\tbuf, err = os.ReadFile(NotFoundPagePath)\n\t\tif err != nil {\n\t\t\tlog.Warnf(\"read custom 404 page error: %v\", err)\n\t\t\tbuf = []byte(NotFound)\n\t\t}\n\t} else {\n\t\tbuf = []byte(NotFound)\n\t}\n\treturn buf\n}\n\nfunc NotFoundResponse() *http.Response {\n\theader := make(http.Header)\n\theader.Set(\"server\", \"frp/\"+version.Full())\n\theader.Set(\"Content-Type\", \"text/html\")\n\n\tcontent := getNotFoundPageContent()\n\tres := &http.Response{\n\t\tStatus:        \"Not Found\",\n\t\tStatusCode:    404,\n\t\tProto:         \"HTTP/1.1\",\n\t\tProtoMajor:    1,\n\t\tProtoMinor:    1,\n\t\tHeader:        header,\n\t\tBody:          io.NopCloser(bytes.NewReader(content)),\n\t\tContentLength: int64(len(content)),\n\t}\n\treturn res\n}\n\n// Path: pkg/util/wait/backoff.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage wait\n\nimport (\n\t\"math/rand/v2\"\n\t\"time\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype BackoffFunc func(previousDuration time.Duration, previousConditionError bool) time.Duration\n\nfunc (f BackoffFunc) Backoff(previousDuration time.Duration, previousConditionError bool) time.Duration {\n\treturn f(previousDuration, previousConditionError)\n}\n\ntype BackoffManager interface {\n\tBackoff(previousDuration time.Duration, previousConditionError bool) time.Duration\n}\n\ntype FastBackoffOptions struct {\n\tDuration           time.Duration\n\tFactor             float64\n\tJitter             float64\n\tMaxDuration        time.Duration\n\tInitDurationIfFail time.Duration\n\n\t// If FastRetryCount > 0, then within the FastRetryWindow time window,\n\t// the retry will be performed with a delay of FastRetryDelay for the first FastRetryCount calls.\n\tFastRetryCount  int\n\tFastRetryDelay  time.Duration\n\tFastRetryJitter float64\n\tFastRetryWindow time.Duration\n}\n\ntype fastBackoffImpl struct {\n\toptions FastBackoffOptions\n\n\tlastCalledTime      time.Time\n\tconsecutiveErrCount int\n\n\tfastRetryCutoffTime     time.Time\n\tcountsInFastRetryWindow int\n}\n\nfunc NewFastBackoffManager(options FastBackoffOptions) BackoffManager {\n\treturn &fastBackoffImpl{\n\t\toptions:                 options,\n\t\tcountsInFastRetryWindow: 1,\n\t}\n}\n\nfunc (f *fastBackoffImpl) Backoff(previousDuration time.Duration, previousConditionError bool) time.Duration {\n\tif f.lastCalledTime.IsZero() {\n\t\tf.lastCalledTime = time.Now()\n\t\treturn f.options.Duration\n\t}\n\tnow := time.Now()\n\tf.lastCalledTime = now\n\n\tif previousConditionError {\n\t\tf.consecutiveErrCount++\n\t} else {\n\t\tf.consecutiveErrCount = 0\n\t}\n\n\tif f.options.FastRetryCount > 0 && previousConditionError {\n\t\tf.countsInFastRetryWindow++\n\t\tif f.countsInFastRetryWindow <= f.options.FastRetryCount {\n\t\t\treturn Jitter(f.options.FastRetryDelay, f.options.FastRetryJitter)\n\t\t}\n\t\tif now.After(f.fastRetryCutoffTime) {\n\t\t\t// reset\n\t\t\tf.fastRetryCutoffTime = now.Add(f.options.FastRetryWindow)\n\t\t\tf.countsInFastRetryWindow = 0\n\t\t}\n\t}\n\n\tif previousConditionError {\n\t\tvar duration time.Duration\n\t\tif f.consecutiveErrCount == 1 {\n\t\t\tduration = util.EmptyOr(f.options.InitDurationIfFail, previousDuration)\n\t\t} else {\n\t\t\tduration = previousDuration\n\t\t}\n\n\t\tduration = util.EmptyOr(duration, time.Second)\n\t\tif f.options.Factor != 0 {\n\t\t\tduration = time.Duration(float64(duration) * f.options.Factor)\n\t\t}\n\t\tif f.options.Jitter > 0 {\n\t\t\tduration = Jitter(duration, f.options.Jitter)\n\t\t}\n\t\tif f.options.MaxDuration > 0 && duration > f.options.MaxDuration {\n\t\t\tduration = f.options.MaxDuration\n\t\t}\n\t\treturn duration\n\t}\n\treturn f.options.Duration\n}\n\nfunc BackoffUntil(f func() (bool, error), backoff BackoffManager, sliding bool, stopCh <-chan struct{}) {\n\tvar delay time.Duration\n\tpreviousError := false\n\n\tticker := time.NewTicker(backoff.Backoff(delay, previousError))\n\tdefer ticker.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-stopCh:\n\t\t\treturn\n\t\tdefault:\n\t\t}\n\n\t\tif !sliding {\n\t\t\tdelay = backoff.Backoff(delay, previousError)\n\t\t}\n\n\t\tif done, err := f(); done {\n\t\t\treturn\n\t\t} else if err != nil {\n\t\t\tpreviousError = true\n\t\t} else {\n\t\t\tpreviousError = false\n\t\t}\n\n\t\tif sliding {\n\t\t\tdelay = backoff.Backoff(delay, previousError)\n\t\t}\n\n\t\tticker.Reset(delay)\n\t\tselect {\n\t\tcase <-stopCh:\n\t\t\treturn\n\t\tcase <-ticker.C:\n\t\t}\n\t}\n}\n\n// Jitter returns a time.Duration between duration and duration + maxFactor *\n// duration.\n//\n// This allows clients to avoid converging on periodic behavior. If maxFactor\n// is 0.0, a suggested default value will be chosen.\nfunc Jitter(duration time.Duration, maxFactor float64) time.Duration {\n\tif maxFactor <= 0.0 {\n\t\tmaxFactor = 1.0\n\t}\n\twait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration))\n\treturn wait\n}\n\nfunc Until(f func(), period time.Duration, stopCh <-chan struct{}) {\n\tff := func() (bool, error) {\n\t\tf()\n\t\treturn false, nil\n\t}\n\tBackoffUntil(ff, BackoffFunc(func(time.Duration, bool) time.Duration {\n\t\treturn period\n\t}), true, stopCh)\n}\n\n", "needle_token_start": 5306, "needle_token_end": 5430, "code_context_ntokens": 7218, "output": ["```go\npackage vhost\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\t\"github.com/fatedier/frp/pkg/util/version\"\n)\n\nvar NotFoundPagePath = \"\"\n\nconst (\n\tNotFound = `<!DOCTYPE html>\n<html>\n<head>\n<title>Not Found</title>\n<style>\n    body {\n        width: 35em;\n        margin: 0 auto;\n        font-family: Tahoma, Verdana, Arial, sans-serif;\n    }\n</style>\n</head>\n<body>\n<h1>The page you requested was not found.</h1>\n<p>Sorry, the page you are looking for is currently unavailable.<br/>\nPlease try again later.</p>\n<p>The server is powered by <a href=\"https://github.com/fatedier/frp\">frp</a>.</p>\n<p><em>Faithfully yours, frp.</em></p>\n</body>\n</html>\n`\n)\n\nfunc getNotFoundPageContent() []byte {\n\tvar (\n\t\tbuf []byte\n\t\terr error\n\t)\n\tif NotFoundPagePath != \"\" {\n\t\tbuf, err = os.ReadFile(NotFoundPagePath)\n\t\tif err != nil {\n\t\t\tlog.Warnf(\"read custom 404 page error: %v\", err)\n\t\t\tbuf = []byte(NotFound)\n\t\t}\n\t} else {\n\t\tbuf = []byte(NotFound)\n\t}\n\treturn buf\n}\n\nfunc NotFoundResponse() *http.Response {\n\theader := make(http.Header)\n\theader.Set(\"server\", \"frp/\"+version.Full())\n\theader.Set(\"Content-Type\", \"text/html\")\n\n\tcontent := getNotFoundPageContent()\n\tres := &http.Response{\n\t\tStatus:        \"Not Found\",\n\t\tStatusCode:    404,\n\t\tProto:         \"HTTP/1.1\",\n\t\tProtoMajor:    1,\n\t\tProtoMinor:    1,\n\t\tHeader:        header,\n\t\tBody:          io.NopCloser(bytes.NewReader(content)),\n\t\tContentLength: int64(len(content)),\n\t}\n\treturn res\n}\n```"]}
{"repo": "fatedier/frp", "name": "parseBasicAuth", "language": "go", "path": "pkg/util/vhost/http.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to decode an authorization header value that uses Basic Authentication, extracting the username and password from it.\n2. **Input**: A single string representing the authorization header value.\n3. **Output**: Returns three values: the username (string), the password (string), and a boolean indicating whether the decoding was successful.\n4. **Procedure**: \n   - The function first checks if the provided header value starts with the expected prefix for Basic Authentication, ignoring case differences.\n   - If the prefix is correct, the remainder of the string (after the prefix) is decoded from Base64 format.\n   - The decoded string is then split at the first colon to separate the username and password.\n   - If all steps are successful, the username and password are returned along with a boolean set to true. If any step fails, it returns default values with the boolean set to false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/virtual/client.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage virtual\n\nimport (\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/fatedier/frp/client\"\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/msg\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\ntype ClientOptions struct {\n\tCommon           *v1.ClientCommonConfig\n\tSpec             *msg.ClientSpec\n\tHandleWorkConnCb func(*v1.ProxyBaseConfig, net.Conn, *msg.StartWorkConn) bool\n}\n\ntype Client struct {\n\tl   *netpkg.InternalListener\n\tsvr *client.Service\n}\n\nfunc NewClient(options ClientOptions) (*Client, error) {\n\tif options.Common != nil {\n\t\toptions.Common.Complete()\n\t}\n\n\tln := netpkg.NewInternalListener()\n\n\tserviceOptions := client.ServiceOptions{\n\t\tCommon:     options.Common,\n\t\tClientSpec: options.Spec,\n\t\tConnectorCreator: func(context.Context, *v1.ClientCommonConfig) client.Connector {\n\t\t\treturn &pipeConnector{\n\t\t\t\tpeerListener: ln,\n\t\t\t}\n\t\t},\n\t\tHandleWorkConnCb: options.HandleWorkConnCb,\n\t}\n\tsvr, err := client.NewService(serviceOptions)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Client{\n...\n// Path: pkg/util/net/http.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage net\n\nimport (\n\t\"compress/gzip\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype HTTPAuthMiddleware struct {\n\tuser          string\n\tpasswd        string\n\tauthFailDelay time.Duration\n}\n\nfunc NewHTTPAuthMiddleware(user, passwd string) *HTTPAuthMiddleware {\n\treturn &HTTPAuthMiddleware{\n\t\tuser:   user,\n\t\tpasswd: passwd,\n\t}\n}\n\nfunc (authMid *HTTPAuthMiddleware) SetAuthFailDelay(delay time.Duration) *HTTPAuthMiddleware {\n\tauthMid.authFailDelay = delay\n\treturn authMid\n}\n\nfunc (authMid *HTTPAuthMiddleware) Middleware(next http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\treqUser, reqPasswd, hasAuth := r.BasicAuth()\n\t\tif (authMid.user == \"\" && authMid.passwd == \"\") ||\n\t\t\t(hasAuth && util.ConstantTimeEqString(reqUser, authMid.user) &&\n\t\t\t\tutil.ConstantTimeEqString(reqPasswd, authMid.passwd)) {\n\t\t\tnext.ServeHTTP(w, r)\n\t\t} else {\n\t\t\tif authMid.authFailDelay > 0 {\n\t\t\t\ttime.Sleep(authMid.authFailDelay)\n\t\t\t}\n\t\t\tw.Header().Set(\"WWW-Authenticate\", `Basic realm=\"Restricted\"`)\n\t\t\thttp.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)\n\t\t}\n\t})\n}\n\ntype HTTPGzipWrapper struct {\n\th http.Handler\n}\n\nfunc (gw *HTTPGzipWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tif !strings.Contains(r.Header.Get(\"Accept-Encoding\"), \"gzip\") {\n\t\tgw.h.ServeHTTP(w, r)\n\t\treturn\n\t}\n\tw.Header().Set(\"Content-Encoding\", \"gzip\")\n\tgz := gzip.NewWriter(w)\n\tdefer gz.Close()\n\tgzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}\n\tgw.h.ServeHTTP(gzr, r)\n}\n\nfunc MakeHTTPGzipHandler(h http.Handler) http.Handler {\n\treturn &HTTPGzipWrapper{\n\t\th: h,\n\t}\n}\n\ntype gzipResponseWriter struct {\n\tio.Writer\n\thttp.ResponseWriter\n}\n\nfunc (w gzipResponseWriter) Write(b []byte) (int, error) {\n\treturn w.Writer.Write(b)\n}\n\n// Path: pkg/util/http/http.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage http\n\nimport (\n\t\"encoding/base64\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n)\n\nfunc OkResponse() *http.Response {\n\theader := make(http.Header)\n\n\tres := &http.Response{\n\t\tStatus:     \"OK\",\n\t\tStatusCode: 200,\n\t\tProto:      \"HTTP/1.1\",\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 1,\n\t\tHeader:     header,\n\t}\n\treturn res\n}\n\nfunc ProxyUnauthorizedResponse() *http.Response {\n\theader := make(http.Header)\n\theader.Set(\"Proxy-Authenticate\", `Basic realm=\"Restricted\"`)\n\tres := &http.Response{\n\t\tStatus:     \"Proxy Authentication Required\",\n\t\tStatusCode: 407,\n\t\tProto:      \"HTTP/1.1\",\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 1,\n\t\tHeader:     header,\n\t}\n\treturn res\n}\n\n// canonicalHost strips port from host if present and returns the canonicalized\n// host name.\nfunc CanonicalHost(host string) (string, error) {\n\tvar err error\n\thost = strings.ToLower(host)\n\tif hasPort(host) {\n\t\thost, _, err = net.SplitHostPort(host)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t}\n\t// Strip trailing dot from fully qualified domain names.\n\thost = strings.TrimSuffix(host, \".\")\n\treturn host, nil\n}\n\n// hasPort reports whether host contains a port number. host may be a host\n// name, an IPv4 or an IPv6 address.\nfunc hasPort(host string) bool {\n\tcolons := strings.Count(host, \":\")\n\tif colons == 0 {\n\t\treturn false\n\t}\n\tif colons == 1 {\n\t\treturn true\n\t}\n\treturn host[0] == '[' && strings.Contains(host, \"]:\")\n}\n\nfunc ParseBasicAuth(auth string) (username, password string, ok bool) {\n\tconst prefix = \"Basic \"\n\t// Case insensitive prefix match. See Issue 22736.\n\tif len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {\n\t\treturn\n\t}\n\tc, err := base64.StdEncoding.DecodeString(auth[len(prefix):])\n\tif err != nil {\n\t\treturn\n\t}\n\tcs := string(c)\n\ts := strings.IndexByte(cs, ':')\n\tif s < 0 {\n\t\treturn\n\t}\n\treturn cs[:s], cs[s+1:], true\n}\n\nfunc BasicAuth(username, passwd string) string {\n\tauth := username + \":\" + passwd\n\treturn \"Basic \" + base64.StdEncoding.EncodeToString([]byte(auth))\n}\n\n// Path: pkg/sdk/client/client.go\npackage client\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/fatedier/frp/client\"\n\thttppkg \"github.com/fatedier/frp/pkg/util/http\"\n)\n\ntype Client struct {\n\taddress  string\n\tauthUser string\n\tauthPwd  string\n}\n\nfunc New(host string, port int) *Client {\n\treturn &Client{\n\t\taddress: net.JoinHostPort(host, strconv.Itoa(port)),\n\t}\n}\n\nfunc (c *Client) SetAuth(user, pwd string) {\n\tc.authUser = user\n\tc.authPwd = pwd\n}\n\nfunc (c *Client) GetProxyStatus(name string) (*client.ProxyStatusResp, error) {\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/status\", nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcontent, err := c.do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tallStatus := make(client.StatusResp)\n\tif err = json.Unmarshal([]byte(content), &allStatus); err != nil {\n\t\treturn nil, fmt.Errorf(\"unmarshal http response error: %s\", strings.TrimSpace(content))\n\t}\n\tfor _, pss := range allStatus {\n\t\tfor _, ps := range pss {\n\t\t\tif ps.Name == name {\n\t\t\t\treturn &ps, nil\n\t\t\t}\n\t\t}\n\t}\n\treturn nil, fmt.Errorf(\"no proxy status found\")\n}\n\nfunc (c *Client) GetAllProxyStatus() (client.StatusResp, error) {\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/status\", nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcontent, err := c.do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tallStatus := make(client.StatusResp)\n\tif err = json.Unmarshal([]byte(content), &allStatus); err != nil {\n\t\treturn nil, fmt.Errorf(\"unmarshal http response error: %s\", strings.TrimSpace(content))\n\t}\n\treturn allStatus, nil\n}\n\nfunc (c *Client) Reload(strictMode bool) error {\n\tv := url.Values{}\n\tif strictMode {\n\t\tv.Set(\"strictConfig\", \"true\")\n\t}\n\tqueryStr := \"\"\n\tif len(v) > 0 {\n\t\tqueryStr = \"?\" + v.Encode()\n\t}\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/reload\"+queryStr, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = c.do(req)\n\treturn err\n}\n\nfunc (c *Client) Stop() error {\n\treq, err := http.NewRequest(\"POST\", \"http://\"+c.address+\"/api/stop\", nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = c.do(req)\n\treturn err\n}\n\nfunc (c *Client) GetConfig() (string, error) {\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/config\", nil)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn c.do(req)\n}\n\nfunc (c *Client) UpdateConfig(content string) error {\n\treq, err := http.NewRequest(\"PUT\", \"http://\"+c.address+\"/api/config\", strings.NewReader(content))\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = c.do(req)\n\treturn err\n}\n\nfunc (c *Client) setAuthHeader(req *http.Request) {\n\tif c.authUser != \"\" || c.authPwd != \"\" {\n\t\treq.Header.Set(\"Authorization\", httppkg.BasicAuth(c.authUser, c.authPwd))\n\t}\n}\n\nfunc (c *Client) do(req *http.Request) (string, error) {\n\tc.setAuthHeader(req)\n\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != 200 {\n\t\treturn \"\", fmt.Errorf(\"api status code [%d]\", resp.StatusCode)\n\t}\n\tbuf, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn string(buf), nil\n}\n\n// Path: pkg/util/vhost/http.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"context\"\n\t\"encoding/base64\"\n\t\"errors\"\n\t\"fmt\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\tlibio \"github.com/fatedier/golib/io\"\n\t\"github.com/fatedier/golib/pool\"\n\n\thttppkg \"github.com/fatedier/frp/pkg/util/http\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n)\n\nvar ErrNoRouteFound = errors.New(\"no route found\")\n\ntype HTTPReverseProxyOptions struct {\n\tResponseHeaderTimeoutS int64\n}\n\ntype HTTPReverseProxy struct {\n\tproxy       *httputil.ReverseProxy\n\tvhostRouter *Routers\n\n\tresponseHeaderTimeout time.Duration\n}\n\nfunc NewHTTPReverseProxy(option HTTPReverseProxyOptions, vhostRouter *Routers) *HTTPReverseProxy {\n\tif option.ResponseHeaderTimeoutS <= 0 {\n\t\toption.ResponseHeaderTimeoutS = 60\n\t}\n\trp := &HTTPReverseProxy{\n\t\tresponseHeaderTimeout: time.Duration(option.ResponseHeaderTimeoutS) * time.Second,\n\t\tvhostRouter:           vhostRouter,\n\t}\n\tproxy := &httputil.ReverseProxy{\n\t\t// Modify incoming requests by route policies.\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.SetXForwarded()\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"http\"\n\t\t\treqRouteInfo := req.Context().Value(RouteInfoKey).(*RequestRouteInfo)\n\t\t\toldHost, _ := httppkg.CanonicalHost(reqRouteInfo.Host)\n\n\t\t\trc := rp.GetRouteConfig(oldHost, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n\t\t\tif rc != nil {\n\t\t\t\tif rc.RewriteHost != \"\" {\n\t\t\t\t\treq.Host = rc.RewriteHost\n\t\t\t\t}\n\n\t\t\t\tvar endpoint string\n\t\t\t\tif rc.ChooseEndpointFn != nil {\n\t\t\t\t\t// ignore error here, it will use CreateConnFn instead later\n\t\t\t\t\tendpoint, _ = rc.ChooseEndpointFn()\n\t\t\t\t\treqRouteInfo.Endpoint = endpoint\n\t\t\t\t\tlog.Tracef(\"choose endpoint name [%s] for http request host [%s] path [%s] httpuser [%s]\",\n\t\t\t\t\t\tendpoint, oldHost, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n\t\t\t\t}\n\t\t\t\t// Set {domain}.{location}.{routeByHTTPUser}.{endpoint} as URL host here to let http transport reuse connections.\n\t\t\t\treq.URL.Host = rc.Domain + \".\" +\n\t\t\t\t\tbase64.StdEncoding.EncodeToString([]byte(rc.Location)) + \".\" +\n\t\t\t\t\tbase64.StdEncoding.EncodeToString([]byte(rc.RouteByHTTPUser)) + \".\" +\n\t\t\t\t\tbase64.StdEncoding.EncodeToString([]byte(endpoint))\n\n\t\t\t\tfor k, v := range rc.Headers {\n\t\t\t\t\treq.Header.Set(k, v)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treq.URL.Host = req.Host\n\t\t\t}\n\t\t},\n\t\t// Create a connection to one proxy routed by route policy.\n\t\tTransport: &http.Transport{\n\t\t\tResponseHeaderTimeout: rp.responseHeaderTimeout,\n\t\t\tIdleConnTimeout:       60 * time.Second,\n\t\t\tMaxIdleConnsPerHost:   5,\n\t\t\tDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {\n\t\t\t\treturn rp.CreateConnection(ctx.Value(RouteInfoKey).(*RequestRouteInfo), true)\n\t\t\t},\n\t\t\tProxy: func(req *http.Request) (*url.URL, error) {\n\t\t\t\t// Use proxy mode if there is host in HTTP first request line.\n\t\t\t\t// GET http://example.com/ HTTP/1.1\n\t\t\t\t// Host: example.com\n\t\t\t\t//\n\t\t\t\t// Normal:\n\t\t\t\t// GET / HTTP/1.1\n\t\t\t\t// Host: example.com\n\t\t\t\turlHost := req.Context().Value(RouteInfoKey).(*RequestRouteInfo).URLHost\n\t\t\t\tif urlHost != \"\" {\n\t\t\t\t\treturn req.URL, nil\n\t\t\t\t}\n\t\t\t\treturn nil, nil\n\t\t\t},\n\t\t},\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n\t\tErrorLog:   stdlog.New(log.NewWriteLogger(log.WarnLevel, 2), \"\", 0),\n\t\tErrorHandler: func(rw http.ResponseWriter, req *http.Request, err error) {\n\t\t\tlog.Logf(log.WarnLevel, 1, \"do http proxy request [host: %s] error: %v\", req.Host, err)\n\t\t\tif err != nil {\n\t\t\t\tif e, ok := err.(net.Error); ok && e.Timeout() {\n\t\t\t\t\trw.WriteHeader(http.StatusGatewayTimeout)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\trw.WriteHeader(http.StatusNotFound)\n\t\t\t_, _ = rw.Write(getNotFoundPageContent())\n\t\t},\n\t}\n\trp.proxy = proxy\n\treturn rp\n}\n\n// Register register the route config to reverse proxy\n// reverse proxy will use CreateConnFn from routeCfg to create a connection to the remote service\nfunc (rp *HTTPReverseProxy) Register(routeCfg RouteConfig) error {\n\terr := rp.vhostRouter.Add(routeCfg.Domain, routeCfg.Location, routeCfg.RouteByHTTPUser, &routeCfg)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// UnRegister unregister route config by domain and location\nfunc (rp *HTTPReverseProxy) UnRegister(routeCfg RouteConfig) {\n\trp.vhostRouter.Del(routeCfg.Domain, routeCfg.Location, routeCfg.RouteByHTTPUser)\n}\n\nfunc (rp *HTTPReverseProxy) GetRouteConfig(domain, location, routeByHTTPUser string) *RouteConfig {\n\tvr, ok := rp.getVhost(domain, location, routeByHTTPUser)\n\tif ok {\n\t\tlog.Debugf(\"get new HTTP request host [%s] path [%s] httpuser [%s]\", domain, location, routeByHTTPUser)\n\t\treturn vr.payload.(*RouteConfig)\n\t}\n\treturn nil\n}\n\nfunc (rp *HTTPReverseProxy) GetHeaders(domain, location, routeByHTTPUser string) (headers map[string]string) {\n\tvr, ok := rp.getVhost(domain, location, routeByHTTPUser)\n\tif ok {\n\t\theaders = vr.payload.(*RouteConfig).Headers\n\t}\n\treturn\n}\n\n// CreateConnection create a new connection by route config\nfunc (rp *HTTPReverseProxy) CreateConnection(reqRouteInfo *RequestRouteInfo, byEndpoint bool) (net.Conn, error) {\n\thost, _ := httppkg.CanonicalHost(reqRouteInfo.Host)\n\tvr, ok := rp.getVhost(host, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n\tif ok {\n\t\tif byEndpoint {\n\t\t\tfn := vr.payload.(*RouteConfig).CreateConnByEndpointFn\n\t\t\tif fn != nil {\n\t\t\t\treturn fn(reqRouteInfo.Endpoint, reqRouteInfo.RemoteAddr)\n\t\t\t}\n\t\t}\n\t\tfn := vr.payload.(*RouteConfig).CreateConnFn\n\t\tif fn != nil {\n\t\t\treturn fn(reqRouteInfo.RemoteAddr)\n\t\t}\n\t}\n\treturn nil, fmt.Errorf(\"%v: %s %s %s\", ErrNoRouteFound, host, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n}\n\nfunc (rp *HTTPReverseProxy) CheckAuth(domain, location, routeByHTTPUser, user, passwd string) bool {\n\tvr, ok := rp.getVhost(domain, location, routeByHTTPUser)\n\tif ok {\n\t\tcheckUser := vr.payload.(*RouteConfig).Username\n\t\tcheckPasswd := vr.payload.(*RouteConfig).Password\n\t\tif (checkUser != \"\" || checkPasswd != \"\") && (checkUser != user || checkPasswd != passwd) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// getVhost tries to get vhost router by route policy.\nfunc (rp *HTTPReverseProxy) getVhost(domain, location, routeByHTTPUser string) (*Router, bool) {\n\tfindRouter := func(inDomain, inLocation, inRouteByHTTPUser string) (*Router, bool) {\n\t\tvr, ok := rp.vhostRouter.Get(inDomain, inLocation, inRouteByHTTPUser)\n\t\tif ok {\n\t\t\treturn vr, ok\n\t\t}\n\t\t// Try to check if there is one proxy that doesn't specify routerByHTTPUser, it means match all.\n\t\tvr, ok = rp.vhostRouter.Get(inDomain, inLocation, \"\")\n\t\tif ok {\n\t\t\treturn vr, ok\n\t\t}\n\t\treturn nil, false\n\t}\n\n\t// First we check the full hostname\n\t// if not exist, then check the wildcard_domain such as *.example.com\n\tvr, ok := findRouter(domain, location, routeByHTTPUser)\n\tif ok {\n\t\treturn vr, ok\n\t}\n\n\t// e.g. domain = test.example.com, try to match wildcard domains.\n\t// *.example.com\n\t// *.com\n\tdomainSplit := strings.Split(domain, \".\")\n\tfor {\n\t\tif len(domainSplit) < 3 {\n\t\t\tbreak\n\t\t}\n\n\t\tdomainSplit[0] = \"*\"\n\t\tdomain = strings.Join(domainSplit, \".\")\n\t\tvr, ok = findRouter(domain, location, routeByHTTPUser)\n\t\tif ok {\n\t\t\treturn vr, true\n\t\t}\n\t\tdomainSplit = domainSplit[1:]\n\t}\n\n\t// Finally, try to check if there is one proxy that domain is \"*\" means match all domains.\n\tvr, ok = findRouter(\"*\", location, routeByHTTPUser)\n\tif ok {\n\t\treturn vr, true\n\t}\n\treturn nil, false\n}\n\nfunc (rp *HTTPReverseProxy) connectHandler(rw http.ResponseWriter, req *http.Request) {\n\thj, ok := rw.(http.Hijacker)\n\tif !ok {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tclient, _, err := hj.Hijack()\n\tif err != nil {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tremote, err := rp.CreateConnection(req.Context().Value(RouteInfoKey).(*RequestRouteInfo), false)\n\tif err != nil {\n\t\t_ = NotFoundResponse().Write(client)\n\t\tclient.Close()\n\t\treturn\n\t}\n\t_ = req.Write(remote)\n\tgo libio.Join(remote, client)\n}\n\n\nfunc parseBasicAuth(auth string) (username, password string, ok bool) {\n\tconst prefix = \"Basic \"\n\t// Case insensitive prefix match. See Issue 22736.\n\tif len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {\n\t\treturn\n\t}\n\tc, err := base64.StdEncoding.DecodeString(auth[len(prefix):])\n\tif err != nil {\n\t\treturn\n\t}\n\tcs := string(c)\n\ts := strings.IndexByte(cs, ':')\n\tif s < 0 {\n\t\treturn\n\t}\n\treturn cs[:s], cs[s+1:], true\n}\n\nfunc (rp *HTTPReverseProxy) injectRequestInfoToCtx(req *http.Request) *http.Request {\n\tuser := \"\"\n\t// If url host isn't empty, it's a proxy request. Get http user from Proxy-Authorization header.\n\tif req.URL.Host != \"\" {\n\t\tproxyAuth := req.Header.Get(\"Proxy-Authorization\")\n\t\tif proxyAuth != \"\" {\n\t\t\tuser, _, _ = parseBasicAuth(proxyAuth)\n\t\t}\n\t}\n\tif user == \"\" {\n\t\tuser, _, _ = req.BasicAuth()\n\t}\n\n\treqRouteInfo := &RequestRouteInfo{\n\t\tURL:        req.URL.Path,\n\t\tHost:       req.Host,\n\t\tHTTPUser:   user,\n\t\tRemoteAddr: req.RemoteAddr,\n\t\tURLHost:    req.URL.Host,\n\t}\n\tnewctx := req.Context()\n\tnewctx = context.WithValue(newctx, RouteInfoKey, reqRouteInfo)\n\treturn req.Clone(newctx)\n}\n\nfunc (rp *HTTPReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {\n\tdomain, _ := httppkg.CanonicalHost(req.Host)\n\tlocation := req.URL.Path\n\tuser, passwd, _ := req.BasicAuth()\n\tif !rp.CheckAuth(domain, location, user, user, passwd) {\n\t\trw.Header().Set(\"WWW-Authenticate\", `Basic realm=\"Restricted\"`)\n\t\thttp.Error(rw, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)\n\t\treturn\n\t}\n\n\tnewreq := rp.injectRequestInfoToCtx(req)\n\tif req.Method == http.MethodConnect {\n\t\trp.connectHandler(rw, newreq)\n\t} else {\n\t\trp.proxy.ServeHTTP(rw, newreq)\n\t}\n}\n\n// Path: pkg/plugin/server/http.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"strings\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\ntype httpPlugin struct {\n\toptions v1.HTTPPluginOptions\n\n\turl    string\n\tclient *http.Client\n}\n\nfunc NewHTTPPluginOptions(options v1.HTTPPluginOptions) Plugin {\n\turl := fmt.Sprintf(\"%s%s\", options.Addr, options.Path)\n\n\tvar client *http.Client\n\tif strings.HasPrefix(url, \"https://\") {\n\t\ttr := &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: !options.TLSVerify},\n\t\t}\n\t\tclient = &http.Client{Transport: tr}\n\t} else {\n\t\tclient = &http.Client{}\n\t}\n\n\tif !strings.HasPrefix(url, \"https://\") && !strings.HasPrefix(url, \"http://\") {\n\t\turl = \"http://\" + url\n\t}\n\treturn &httpPlugin{\n\t\toptions: options,\n\t\turl:     url,\n\t\tclient:  client,\n\t}\n}\n\nfunc (p *httpPlugin) Name() string {\n\treturn p.options.Name\n}\n\nfunc (p *httpPlugin) IsSupport(op string) bool {\n\tfor _, v := range p.options.Ops {\n\t\tif v == op {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (p *httpPlugin) Handle(ctx context.Context, op string, content interface{}) (*Response, interface{}, error) {\n\tr := &Request{\n\t\tVersion: APIVersion,\n\t\tOp:      op,\n\t\tContent: content,\n\t}\n\tvar res Response\n\tres.Content = reflect.New(reflect.TypeOf(content)).Interface()\n\tif err := p.do(ctx, r, &res); err != nil {\n\t\treturn nil, nil, err\n\t}\n\treturn &res, res.Content, nil\n}\n\nfunc (p *httpPlugin) do(ctx context.Context, r *Request, res *Response) error {\n\tbuf, err := json.Marshal(r)\n\tif err != nil {\n\t\treturn err\n\t}\n\tv := url.Values{}\n\tv.Set(\"version\", r.Version)\n\tv.Set(\"op\", r.Op)\n\treq, err := http.NewRequest(\"POST\", p.url+\"?\"+v.Encode(), bytes.NewReader(buf))\n\tif err != nil {\n\t\treturn err\n\t}\n\treq = req.WithContext(ctx)\n\treq.Header.Set(\"X-Frp-Reqid\", GetReqidFromContext(ctx))\n\treq.Header.Set(\"Content-Type\", \"application/json\")\n\tresp, err := p.client.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn fmt.Errorf(\"do http request error code: %d\", resp.StatusCode)\n\t}\n\tbuf, err = io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn json.Unmarshal(buf, res)\n}\n\n// Path: pkg/util/http/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage http\n\nimport (\n\t\"crypto/tls\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/pprof\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/gorilla/mux\"\n\n\t\"github.com/fatedier/frp/assets\"\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nvar (\n\tdefaultReadTimeout  = 60 * time.Second\n\tdefaultWriteTimeout = 60 * time.Second\n)\n\ntype Server struct {\n\taddr   string\n\tln     net.Listener\n\ttlsCfg *tls.Config\n\n\trouter *mux.Router\n\ths     *http.Server\n\n\tauthMiddleware mux.MiddlewareFunc\n}\n\nfunc NewServer(cfg v1.WebServerConfig) (*Server, error) {\n\tassets.Load(cfg.AssetsDir)\n\n\taddr := net.JoinHostPort(cfg.Addr, strconv.Itoa(cfg.Port))\n\tif addr == \":\" {\n\t\taddr = \":http\"\n\t}\n\n\tln, err := net.Listen(\"tcp\", addr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trouter := mux.NewRouter()\n\ths := &http.Server{\n\t\tAddr:         addr,\n\t\tHandler:      router,\n\t\tReadTimeout:  defaultReadTimeout,\n\t\tWriteTimeout: defaultWriteTimeout,\n\t}\n\ts := &Server{\n\t\taddr:   addr,\n\t\tln:     ln,\n\t\ths:     hs,\n\t\trouter: router,\n\t}\n\tif cfg.PprofEnable {\n\t\ts.registerPprofHandlers()\n\t}\n\tif cfg.TLS != nil {\n\t\tcert, err := tls.LoadX509KeyPair(cfg.TLS.CertFile, cfg.TLS.KeyFile)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ts.tlsCfg = &tls.Config{\n\t\t\tCertificates: []tls.Certificate{cert},\n\t\t}\n\t}\n\ts.authMiddleware = netpkg.NewHTTPAuthMiddleware(cfg.User, cfg.Password).SetAuthFailDelay(200 * time.Millisecond).Middleware\n\treturn s, nil\n}\n\nfunc (s *Server) Address() string {\n\treturn s.addr\n}\n\nfunc (s *Server) Run() error {\n\tln := s.ln\n\tif s.tlsCfg != nil {\n\t\tln = tls.NewListener(ln, s.tlsCfg)\n\t}\n\treturn s.hs.Serve(ln)\n}\n\nfunc (s *Server) Close() error {\n\treturn s.hs.Close()\n}\n\ntype RouterRegisterHelper struct {\n\tRouter         *mux.Router\n\tAssetsFS       http.FileSystem\n\tAuthMiddleware mux.MiddlewareFunc\n}\n\nfunc (s *Server) RouteRegister(register func(helper *RouterRegisterHelper)) {\n\tregister(&RouterRegisterHelper{\n\t\tRouter:         s.router,\n\t\tAssetsFS:       assets.FileSystem,\n\t\tAuthMiddleware: s.authMiddleware,\n\t})\n}\n\nfunc (s *Server) registerPprofHandlers() {\n\ts.router.HandleFunc(\"/debug/pprof/cmdline\", pprof.Cmdline)\n\ts.router.HandleFunc(\"/debug/pprof/profile\", pprof.Profile)\n\ts.router.HandleFunc(\"/debug/pprof/symbol\", pprof.Symbol)\n\ts.router.HandleFunc(\"/debug/pprof/trace\", pprof.Trace)\n\ts.router.PathPrefix(\"/debug/pprof/\").HandlerFunc(pprof.Index)\n}\n\n// Path: pkg/util/metric/counter.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metric\n\nimport (\n\t\"sync/atomic\"\n)\n\ntype Counter interface {\n\tCount() int32\n\tInc(int32)\n\tDec(int32)\n\tSnapshot() Counter\n\tClear()\n}\n\nfunc NewCounter() Counter {\n\treturn &StandardCounter{\n\t\tcount: 0,\n\t}\n}\n\ntype StandardCounter struct {\n\tcount int32\n}\n\nfunc (c *StandardCounter) Count() int32 {\n\treturn atomic.LoadInt32(&c.count)\n}\n\nfunc (c *StandardCounter) Inc(count int32) {\n\tatomic.AddInt32(&c.count, count)\n}\n\nfunc (c *StandardCounter) Dec(count int32) {\n\tatomic.AddInt32(&c.count, -count)\n}\n\nfunc (c *StandardCounter) Snapshot() Counter {\n\ttmp := &StandardCounter{\n\t\tcount: atomic.LoadInt32(&c.count),\n\t}\n\treturn tmp\n}\n\nfunc (c *StandardCounter) Clear() {\n\tatomic.StoreInt32(&c.count, 0)\n}\n\n// Path: pkg/metrics/mem/server.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mem\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\t\"github.com/fatedier/frp/pkg/util/metric\"\n\tserver \"github.com/fatedier/frp/server/metrics\"\n)\n\nvar (\n\tsm = newServerMetrics()\n\n\tServerMetrics  server.ServerMetrics\n\tStatsCollector Collector\n)\n\nfunc init() {\n\tServerMetrics = sm\n\tStatsCollector = sm\n\tsm.run()\n}\n\ntype serverMetrics struct {\n\tinfo *ServerStatistics\n\tmu   sync.Mutex\n}\n\nfunc newServerMetrics() *serverMetrics {\n\treturn &serverMetrics{\n\t\tinfo: &ServerStatistics{\n\t\t\tTotalTrafficIn:  metric.NewDateCounter(ReserveDays),\n\t\t\tTotalTrafficOut: metric.NewDateCounter(ReserveDays),\n\t\t\tCurConns:        metric.NewCounter(),\n\n\t\t\tClientCounts:    metric.NewCounter(),\n\t\t\tProxyTypeCounts: make(map[string]metric.Counter),\n\n\t\t\tProxyStatistics: make(map[string]*ProxyStatistics),\n\t\t},\n\t}\n}\n\nfunc (m *serverMetrics) run() {\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(12 * time.Hour)\n\t\t\tstart := time.Now()\n\t\t\tcount, total := m.clearUselessInfo(time.Duration(7*24) * time.Hour)\n\t\t\tlog.Debugf(\"clear useless proxy statistics data count %d/%d, cost %v\", count, total, time.Since(start))\n\t\t}\n\t}()\n}\n\nfunc (m *serverMetrics) clearUselessInfo(continuousOfflineDuration time.Duration) (int, int) {\n\tcount := 0\n\ttotal := 0\n\t// To check if there are any proxies that have been closed for more than continuousOfflineDuration and remove them.\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\ttotal = len(m.info.ProxyStatistics)\n\tfor name, data := range m.info.ProxyStatistics {\n\t\tif !data.LastCloseTime.IsZero() &&\n\t\t\tdata.LastStartTime.Before(data.LastCloseTime) &&\n\t\t\ttime.Since(data.LastCloseTime) > continuousOfflineDuration {\n\t\t\tdelete(m.info.ProxyStatistics, name)\n\t\t\tcount++\n\t\t\tlog.Tracef(\"clear proxy [%s]'s statistics data, lastCloseTime: [%s]\", name, data.LastCloseTime.String())\n\t\t}\n\t}\n\treturn count, total\n}\n\nfunc (m *serverMetrics) ClearOfflineProxies() (int, int) {\n\treturn m.clearUselessInfo(0)\n}\n\nfunc (m *serverMetrics) NewClient() {\n\tm.info.ClientCounts.Inc(1)\n}\n\nfunc (m *serverMetrics) CloseClient() {\n\tm.info.ClientCounts.Dec(1)\n}\n\nfunc (m *serverMetrics) NewProxy(name string, proxyType string) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tcounter, ok := m.info.ProxyTypeCounts[proxyType]\n\tif !ok {\n\t\tcounter = metric.NewCounter()\n\t}\n\tcounter.Inc(1)\n\tm.info.ProxyTypeCounts[proxyType] = counter\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif !(ok && proxyStats.ProxyType == proxyType) {\n\t\tproxyStats = &ProxyStatistics{\n\t\t\tName:       name,\n\t\t\tProxyType:  proxyType,\n\t\t\tCurConns:   metric.NewCounter(),\n\t\t\tTrafficIn:  metric.NewDateCounter(ReserveDays),\n\t\t\tTrafficOut: metric.NewDateCounter(ReserveDays),\n\t\t}\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n\tproxyStats.LastStartTime = time.Now()\n}\n\nfunc (m *serverMetrics) CloseProxy(name string, proxyType string) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tif counter, ok := m.info.ProxyTypeCounts[proxyType]; ok {\n\t\tcounter.Dec(1)\n\t}\n\tif proxyStats, ok := m.info.ProxyStatistics[name]; ok {\n\t\tproxyStats.LastCloseTime = time.Now()\n\t}\n}\n\nfunc (m *serverMetrics) OpenConnection(name string, _ string) {\n\tm.info.CurConns.Inc(1)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.CurConns.Inc(1)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\nfunc (m *serverMetrics) CloseConnection(name string, _ string) {\n\tm.info.CurConns.Dec(1)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.CurConns.Dec(1)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficIn(name string, _ string, trafficBytes int64) {\n\tm.info.TotalTrafficIn.Inc(trafficBytes)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.TrafficIn.Inc(trafficBytes)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficOut(name string, _ string, trafficBytes int64) {\n\tm.info.TotalTrafficOut.Inc(trafficBytes)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.TrafficOut.Inc(trafficBytes)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\n// Get stats data api.\n\nfunc (m *serverMetrics) GetServer() *ServerStats {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\ts := &ServerStats{\n\t\tTotalTrafficIn:  m.info.TotalTrafficIn.TodayCount(),\n\t\tTotalTrafficOut: m.info.TotalTrafficOut.TodayCount(),\n\t\tCurConns:        int64(m.info.CurConns.Count()),\n\t\tClientCounts:    int64(m.info.ClientCounts.Count()),\n\t\tProxyTypeCounts: make(map[string]int64),\n\t}\n\tfor k, v := range m.info.ProxyTypeCounts {\n\t\ts.ProxyTypeCounts[k] = int64(v.Count())\n\t}\n\treturn s\n}\n\nfunc (m *serverMetrics) GetProxiesByType(proxyType string) []*ProxyStats {\n\tres := make([]*ProxyStats, 0)\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tfor name, proxyStats := range m.info.ProxyStatistics {\n\t\tif proxyStats.ProxyType != proxyType {\n\t\t\tcontinue\n\t\t}\n\n\t\tps := &ProxyStats{\n\t\t\tName:            name,\n\t\t\tType:            proxyStats.ProxyType,\n\t\t\tTodayTrafficIn:  proxyStats.TrafficIn.TodayCount(),\n\t\t\tTodayTrafficOut: proxyStats.TrafficOut.TodayCount(),\n\t\t\tCurConns:        int64(proxyStats.CurConns.Count()),\n\t\t}\n\t\tif !proxyStats.LastStartTime.IsZero() {\n\t\t\tps.LastStartTime = proxyStats.LastStartTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tif !proxyStats.LastCloseTime.IsZero() {\n\t\t\tps.LastCloseTime = proxyStats.LastCloseTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tres = append(res, ps)\n\t}\n\treturn res\n}\n\nfunc (m *serverMetrics) GetProxiesByTypeAndName(proxyType string, proxyName string) (res *ProxyStats) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tfor name, proxyStats := range m.info.ProxyStatistics {\n\t\tif proxyStats.ProxyType != proxyType {\n\t\t\tcontinue\n\t\t}\n\n\t\tif name != proxyName {\n\t\t\tcontinue\n\t\t}\n\n\t\tres = &ProxyStats{\n\t\t\tName:            name,\n\t\t\tType:            proxyStats.ProxyType,\n\t\t\tTodayTrafficIn:  proxyStats.TrafficIn.TodayCount(),\n\t\t\tTodayTrafficOut: proxyStats.TrafficOut.TodayCount(),\n\t\t\tCurConns:        int64(proxyStats.CurConns.Count()),\n\t\t}\n\t\tif !proxyStats.LastStartTime.IsZero() {\n\t\t\tres.LastStartTime = proxyStats.LastStartTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tif !proxyStats.LastCloseTime.IsZero() {\n\t\t\tres.LastCloseTime = proxyStats.LastCloseTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tbreak\n\t}\n\treturn\n}\n\nfunc (m *serverMetrics) GetProxyTraffic(name string) (res *ProxyTrafficInfo) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tres = &ProxyTrafficInfo{\n\t\t\tName: name,\n\t\t}\n\t\tres.TrafficIn = proxyStats.TrafficIn.GetLastDaysCount(ReserveDays)\n\t\tres.TrafficOut = proxyStats.TrafficOut.GetLastDaysCount(ReserveDays)\n\t}\n\treturn\n}\n\n// Path: pkg/msg/ctl.go\n// Copyright 2018 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage msg\n\nimport (\n\t\"io\"\n\n\tjsonMsg \"github.com/fatedier/golib/msg/json\"\n)\n\ntype Message = jsonMsg.Message\n\nvar msgCtl *jsonMsg.MsgCtl\n\nfunc init() {\n\tmsgCtl = jsonMsg.NewMsgCtl()\n\tfor typeByte, msg := range msgTypeMap {\n\t\tmsgCtl.RegisterMsg(typeByte, msg)\n\t}\n}\n\nfunc ReadMsg(c io.Reader) (msg Message, err error) {\n\treturn msgCtl.ReadMsg(c)\n}\n\nfunc ReadMsgInto(c io.Reader, msg Message) (err error) {\n\treturn msgCtl.ReadMsgInto(c, msg)\n}\n\nfunc WriteMsg(c io.Writer, msg interface{}) (err error) {\n\treturn msgCtl.WriteMsg(c, msg)\n}\n\n// Path: pkg/msg/handler.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage msg\n\nimport (\n\t\"io\"\n\t\"reflect\"\n)\n\nfunc AsyncHandler(f func(Message)) func(Message) {\n\treturn func(m Message) {\n\t\tgo f(m)\n\t}\n}\n\n// Dispatcher is used to send messages to net.Conn or register handlers for messages read from net.Conn.\ntype Dispatcher struct {\n\trw io.ReadWriter\n\n\tsendCh         chan Message\n\tdoneCh         chan struct{}\n\tmsgHandlers    map[reflect.Type]func(Message)\n\tdefaultHandler func(Message)\n}\n\nfunc NewDispatcher(rw io.ReadWriter) *Dispatcher {\n\treturn &Dispatcher{\n\t\trw:          rw,\n\t\tsendCh:      make(chan Message, 100),\n\t\tdoneCh:      make(chan struct{}),\n\t\tmsgHandlers: make(map[reflect.Type]func(Message)),\n\t}\n}\n\n// Run will block until io.EOF or some error occurs.\nfunc (d *Dispatcher) Run() {\n\tgo d.sendLoop()\n\tgo d.readLoop()\n}\n\nfunc (d *Dispatcher) sendLoop() {\n\tfor {\n\t\tselect {\n\t\tcase <-d.doneCh:\n\t\t\treturn\n\t\tcase m := <-d.sendCh:\n\t\t\t_ = WriteMsg(d.rw, m)\n\t\t}\n\t}\n}\n\nfunc (d *Dispatcher) readLoop() {\n\tfor {\n\t\tm, err := ReadMsg(d.rw)\n\t\tif err != nil {\n\t\t\tclose(d.doneCh)\n\t\t\treturn\n\t\t}\n\n\t\tif handler, ok := d.msgHandlers[reflect.TypeOf(m)]; ok {\n\t\t\thandler(m)\n\t\t} else if d.defaultHandler != nil {\n\t\t\td.defaultHandler(m)\n\t\t}\n\t}\n}\n\nfunc (d *Dispatcher) Send(m Message) error {\n\tselect {\n\tcase <-d.doneCh:\n\t\treturn io.EOF\n\tcase d.sendCh <- m:\n\t\treturn nil\n\t}\n}\n\nfunc (d *Dispatcher) SendChannel() chan Message {\n\treturn d.sendCh\n}\n\nfunc (d *Dispatcher) RegisterHandler(msg Message, handler func(Message)) {\n\td.msgHandlers[reflect.TypeOf(msg)] = handler\n}\n\nfunc (d *Dispatcher) RegisterDefaultHandler(handler func(Message)) {\n\td.defaultHandler = handler\n}\n\nfunc (d *Dispatcher) Done() chan struct{} {\n\treturn d.doneCh\n}\n\n// Path: pkg/nathole/analysis.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage nathole\n\nimport (\n\t\"cmp\"\n\t\"slices\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/samber/lo\"\n)\n\nvar (\n\t// mode 0, both EasyNAT, PublicNetwork is always receiver\n\t// sender | receiver, ttl 7\n\t// receiver, ttl 7 | sender\n\t// sender | receiver, ttl 4\n\t// receiver, ttl 4 | sender\n\t// sender | receiver\n\t// receiver | sender\n\t// sender, sendDelayMs 5000 | receiver\n\t// sender, sendDelayMs 10000 | receiver\n\t// receiver | sender, sendDelayMs 5000\n\t// receiver | sender, sendDelayMs 10000\n\tmode0Behaviors = []lo.Tuple2[RecommandBehavior, RecommandBehavior]{\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 7}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleReceiver, TTL: 7}, RecommandBehavior{Role: DetectRoleSender}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 4}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleReceiver, TTL: 4}, RecommandBehavior{Role: DetectRoleSender}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleReceiver}, RecommandBehavior{Role: DetectRoleSender}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 5000}, RecommandBehavior{Role: DetectRoleReceiver}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 10000}, RecommandBehavior{Role: DetectRoleReceiver}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleReceiver}, RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 5000}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleReceiver}, RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 10000}),\n\t}\n\n\t// mode 1, HardNAT is sender, EasyNAT is receiver, port changes is regular\n\t// sender | receiver, ttl 7, portsRangeNumber max 10\n\t// sender, sendDelayMs 2000 | receiver, ttl 7, portsRangeNumber max 10\n\t// sender | receiver, ttl 4, portsRangeNumber max 10\n\t// sender, sendDelayMs 2000 | receiver, ttl 4, portsRangeNumber max 10\n\t// sender | receiver, portsRangeNumber max 10\n\t// sender, sendDelayMs 2000 | receiver, portsRangeNumber max 10\n\tmode1Behaviors = []lo.Tuple2[RecommandBehavior, RecommandBehavior]{\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 7, PortsRangeNumber: 10}),\n\t\tlo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 2000}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 7, PortsRangeNumber: 10}),\n...\n", "needle_token_start": 6777, "needle_token_end": 6952, "code_context_ntokens": 15264, "output": ["```go\nfunc parseBasicAuth(auth string) (username, password string, ok bool) {\n\tconst prefix = \"Basic \"\n\t// Case insensitive prefix match. See Issue 22736.\n\tif len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {\n\t\treturn\n\t}\n\tc, err := base64.StdEncoding.DecodeString(auth[len(prefix):])\n\tif err != nil {\n\t\treturn\n\t}\n\tcs := string(c)\n\ts := strings.IndexByte(cs, ':')\n\tif s < 0 {\n\t\treturn\n\t}\n\treturn cs[:s], cs[s+1:], true\n}\n```"]}
{"repo": "fatedier/frp", "name": "TestLoadServerConfigStrictMode", "language": "go", "path": "pkg/config/load_test.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To verify that the configuration loading function behaves correctly under strict and non-strict parsing conditions, particularly focusing on how it handles errors when configuration content contains typos or incorrect fields.\n2. **Input**: The function takes configuration data in different formats (TOML, YAML, JSON) with intentional errors introduced (e.g., a typo in a key name).\n3. **Output**: The function outputs either an error message when parsing in strict mode if there are typos or incorrect fields, or no error and incomplete configuration data when parsing in non-strict mode.\n4. **Procedure**: The function iterates over a set of predefined configuration contents in different formats. For each format, it introduces a typo into the configuration content and attempts to load this broken configuration in both strict and non-strict modes. It checks for errors in strict mode and the absence of errors but with incomplete data in non-strict mode.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/config/legacy/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"strings\"\n\n\t\"gopkg.in/ini.v1\"\n\n\tlegacyauth \"github.com/fatedier/frp/pkg/auth/legacy\"\n)\n\ntype HTTPPluginOptions struct {\n\tName      string   `ini:\"name\"`\n\tAddr      string   `ini:\"addr\"`\n\tPath      string   `ini:\"path\"`\n\tOps       []string `ini:\"ops\"`\n\tTLSVerify bool     `ini:\"tlsVerify\"`\n}\n\n// ServerCommonConf contains information for a server service. It is\n// recommended to use GetDefaultServerConf instead of creating this object\n// directly, so that all unspecified fields have reasonable default values.\ntype ServerCommonConf struct {\n\tlegacyauth.ServerConfig `ini:\",extends\"`\n\n\t// BindAddr specifies the address that the server binds to. By default,\n\t// this value is \"0.0.0.0\".\n\tBindAddr string `ini:\"bind_addr\" json:\"bind_addr\"`\n\t// BindPort specifies the port that the server listens on. By default, this\n\t// value is 7000.\n\tBindPort int `ini:\"bind_port\" json:\"bind_port\"`\n\t// KCPBindPort specifies the KCP port that the server listens on. If this\n\t// value is 0, the server will not listen for KCP connections. By default,\n\t// this value is 0.\n\tKCPBindPort int `ini:\"kcp_bind_port\" json:\"kcp_bind_port\"`\n\t// QUICBindPort specifies the QUIC port that the server listens on.\n\t// Set this value to 0 will disable this feature.\n\t// By default, the value is 0.\n\tQUICBindPort int `ini:\"quic_bind_port\" json:\"quic_bind_port\"`\n\t// QUIC protocol options\n\tQUICKeepalivePeriod    int `ini:\"quic_keepalive_period\" json:\"quic_keepalive_period\"`\n\tQUICMaxIdleTimeout     int `ini:\"quic_max_idle_timeout\" json:\"quic_max_idle_timeout\"`\n...\n// Path: pkg/config/legacy/utils.go\n// Copyright 2020 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"strings\"\n)\n\nfunc GetMapWithoutPrefix(set map[string]string, prefix string) map[string]string {\n\tm := make(map[string]string)\n\n\tfor key, value := range set {\n\t\tif strings.HasPrefix(key, prefix) {\n\t\t\tm[strings.TrimPrefix(key, prefix)] = value\n\t\t}\n\t}\n\n\tif len(m) == 0 {\n\t\treturn nil\n\t}\n\n\treturn m\n}\n\nfunc GetMapByPrefix(set map[string]string, prefix string) map[string]string {\n\tm := make(map[string]string)\n\n\tfor key, value := range set {\n\t\tif strings.HasPrefix(key, prefix) {\n\t\t\tm[key] = value\n\t\t}\n\t}\n\n\tif len(m) == 0 {\n\t\treturn nil\n\t}\n\n\treturn m\n}\n\n// Path: pkg/config/template.go\n// Copyright 2024 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype NumberPair struct {\n\tFirst  int64\n\tSecond int64\n}\n\nfunc parseNumberRangePair(firstRangeStr, secondRangeStr string) ([]NumberPair, error) {\n\tfirstRangeNumbers, err := util.ParseRangeNumbers(firstRangeStr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsecondRangeNumbers, err := util.ParseRangeNumbers(secondRangeStr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(firstRangeNumbers) != len(secondRangeNumbers) {\n\t\treturn nil, fmt.Errorf(\"first and second range numbers are not in pairs\")\n\t}\n\tpairs := make([]NumberPair, 0, len(firstRangeNumbers))\n\tfor i := 0; i < len(firstRangeNumbers); i++ {\n\t\tpairs = append(pairs, NumberPair{\n\t\t\tFirst:  firstRangeNumbers[i],\n\t\t\tSecond: secondRangeNumbers[i],\n\t\t})\n\t}\n\treturn pairs, nil\n}\n\nfunc parseNumberRange(firstRangeStr string) ([]int64, error) {\n\treturn util.ParseRangeNumbers(firstRangeStr)\n}\n\n// Path: pkg/config/legacy/value.go\n// Copyright 2020 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"strings\"\n\t\"text/template\"\n)\n\nvar glbEnvs map[string]string\n\nfunc init() {\n\tglbEnvs = make(map[string]string)\n\tenvs := os.Environ()\n\tfor _, env := range envs {\n\t\tpair := strings.SplitN(env, \"=\", 2)\n\t\tif len(pair) != 2 {\n\t\t\tcontinue\n\t\t}\n\t\tglbEnvs[pair[0]] = pair[1]\n\t}\n}\n\ntype Values struct {\n\tEnvs map[string]string // environment vars\n}\n\nfunc GetValues() *Values {\n\treturn &Values{\n\t\tEnvs: glbEnvs,\n\t}\n}\n\nfunc RenderContent(in []byte) (out []byte, err error) {\n\ttmpl, errRet := template.New(\"frp\").Parse(string(in))\n\tif errRet != nil {\n\t\terr = errRet\n\t\treturn\n\t}\n\n\tbuffer := bytes.NewBufferString(\"\")\n\tv := GetValues()\n\terr = tmpl.Execute(buffer, v)\n\tif err != nil {\n\t\treturn\n\t}\n\tout = buffer.Bytes()\n\treturn\n}\n\nfunc GetRenderedConfFromFile(path string) (out []byte, err error) {\n\tvar b []byte\n\tb, err = os.ReadFile(path)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tout, err = RenderContent(b)\n\treturn\n}\n\n// Path: pkg/config/legacy/visitor.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"gopkg.in/ini.v1\"\n)\n\ntype VisitorType string\n\nconst (\n\tVisitorTypeSTCP VisitorType = \"stcp\"\n\tVisitorTypeXTCP VisitorType = \"xtcp\"\n\tVisitorTypeSUDP VisitorType = \"sudp\"\n)\n\n// Visitor\nvar (\n\tvisitorConfTypeMap = map[VisitorType]reflect.Type{\n\t\tVisitorTypeSTCP: reflect.TypeOf(STCPVisitorConf{}),\n\t\tVisitorTypeXTCP: reflect.TypeOf(XTCPVisitorConf{}),\n\t\tVisitorTypeSUDP: reflect.TypeOf(SUDPVisitorConf{}),\n\t}\n)\n\ntype VisitorConf interface {\n\t// GetBaseConfig returns the base config of visitor.\n\tGetBaseConfig() *BaseVisitorConf\n\t// UnmarshalFromIni unmarshals config from ini.\n\tUnmarshalFromIni(prefix string, name string, section *ini.Section) error\n}\n\n// DefaultVisitorConf creates a empty VisitorConf object by visitorType.\n// If visitorType doesn't exist, return nil.\nfunc DefaultVisitorConf(visitorType VisitorType) VisitorConf {\n\tv, ok := visitorConfTypeMap[visitorType]\n\tif !ok {\n\t\treturn nil\n\t}\n\treturn reflect.New(v).Interface().(VisitorConf)\n}\n\ntype BaseVisitorConf struct {\n\tProxyName      string `ini:\"name\" json:\"name\"`\n\tProxyType      string `ini:\"type\" json:\"type\"`\n\tUseEncryption  bool   `ini:\"use_encryption\" json:\"use_encryption\"`\n\tUseCompression bool   `ini:\"use_compression\" json:\"use_compression\"`\n\tRole           string `ini:\"role\" json:\"role\"`\n\tSk             string `ini:\"sk\" json:\"sk\"`\n\t// if the server user is not set, it defaults to the current user\n\tServerUser string `ini:\"server_user\" json:\"server_user\"`\n\tServerName string `ini:\"server_name\" json:\"server_name\"`\n\tBindAddr   string `ini:\"bind_addr\" json:\"bind_addr\"`\n\t// BindPort is the port that visitor listens on.\n\t// It can be less than 0, it means don't bind to the port and only receive connections redirected from\n\t// other visitors. (This is not supported for SUDP now)\n\tBindPort int `ini:\"bind_port\" json:\"bind_port\"`\n}\n\n// Base\nfunc (cfg *BaseVisitorConf) GetBaseConfig() *BaseVisitorConf {\n\treturn cfg\n}\n\nfunc (cfg *BaseVisitorConf) unmarshalFromIni(_ string, name string, _ *ini.Section) error {\n\t// Custom decoration after basic unmarshal:\n\tcfg.ProxyName = name\n\n\t// bind_addr\n\tif cfg.BindAddr == \"\" {\n\t\tcfg.BindAddr = \"127.0.0.1\"\n\t}\n\treturn nil\n}\n\nfunc preVisitorUnmarshalFromIni(cfg VisitorConf, prefix string, name string, section *ini.Section) error {\n\terr := section.MapTo(cfg)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = cfg.GetBaseConfig().unmarshalFromIni(prefix, name, section)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\ntype SUDPVisitorConf struct {\n\tBaseVisitorConf `ini:\",extends\"`\n}\n\nfunc (cfg *SUDPVisitorConf) UnmarshalFromIni(prefix string, name string, section *ini.Section) (err error) {\n\terr = preVisitorUnmarshalFromIni(cfg, prefix, name, section)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// Add custom logic unmarshal, if exists\n\n\treturn\n}\n\ntype STCPVisitorConf struct {\n\tBaseVisitorConf `ini:\",extends\"`\n}\n\nfunc (cfg *STCPVisitorConf) UnmarshalFromIni(prefix string, name string, section *ini.Section) (err error) {\n\terr = preVisitorUnmarshalFromIni(cfg, prefix, name, section)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// Add custom logic unmarshal, if exists\n\n\treturn\n}\n\ntype XTCPVisitorConf struct {\n\tBaseVisitorConf `ini:\",extends\"`\n\n\tProtocol          string `ini:\"protocol\" json:\"protocol,omitempty\"`\n\tKeepTunnelOpen    bool   `ini:\"keep_tunnel_open\" json:\"keep_tunnel_open,omitempty\"`\n\tMaxRetriesAnHour  int    `ini:\"max_retries_an_hour\" json:\"max_retries_an_hour,omitempty\"`\n\tMinRetryInterval  int    `ini:\"min_retry_interval\" json:\"min_retry_interval,omitempty\"`\n\tFallbackTo        string `ini:\"fallback_to\" json:\"fallback_to,omitempty\"`\n\tFallbackTimeoutMs int    `ini:\"fallback_timeout_ms\" json:\"fallback_timeout_ms,omitempty\"`\n}\n\nfunc (cfg *XTCPVisitorConf) UnmarshalFromIni(prefix string, name string, section *ini.Section) (err error) {\n\terr = preVisitorUnmarshalFromIni(cfg, prefix, name, section)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// Add custom logic unmarshal, if exists\n\tif cfg.Protocol == \"\" {\n\t\tcfg.Protocol = \"quic\"\n\t}\n\tif cfg.MaxRetriesAnHour <= 0 {\n\t\tcfg.MaxRetriesAnHour = 8\n\t}\n\tif cfg.MinRetryInterval <= 0 {\n\t\tcfg.MinRetryInterval = 90\n\t}\n\tif cfg.FallbackTimeoutMs <= 0 {\n\t\tcfg.FallbackTimeoutMs = 1000\n\t}\n\treturn\n}\n\n// Visitor loaded from ini\nfunc NewVisitorConfFromIni(prefix string, name string, section *ini.Section) (VisitorConf, error) {\n\t// section.Key: if key not exists, section will set it with default value.\n\tvisitorType := VisitorType(section.Key(\"type\").String())\n\n\tif visitorType == \"\" {\n\t\treturn nil, fmt.Errorf(\"type shouldn't be empty\")\n\t}\n\n\tconf := DefaultVisitorConf(visitorType)\n\tif conf == nil {\n\t\treturn nil, fmt.Errorf(\"type [%s] error\", visitorType)\n\t}\n\n\tif err := conf.UnmarshalFromIni(prefix, name, section); err != nil {\n\t\treturn nil, fmt.Errorf(\"type [%s] error\", visitorType)\n\t}\n\treturn conf, nil\n}\n\n// Path: pkg/config/load.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"html/template\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\ttoml \"github.com/pelletier/go-toml/v2\"\n\t\"github.com/samber/lo\"\n\t\"gopkg.in/ini.v1\"\n\t\"k8s.io/apimachinery/pkg/util/sets\"\n\t\"k8s.io/apimachinery/pkg/util/yaml\"\n\n\t\"github.com/fatedier/frp/pkg/config/legacy\"\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/config/v1/validation\"\n\t\"github.com/fatedier/frp/pkg/msg\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\nvar glbEnvs map[string]string\n\nfunc init() {\n\tglbEnvs = make(map[string]string)\n\tenvs := os.Environ()\n\tfor _, env := range envs {\n\t\tpair := strings.SplitN(env, \"=\", 2)\n\t\tif len(pair) != 2 {\n\t\t\tcontinue\n\t\t}\n\t\tglbEnvs[pair[0]] = pair[1]\n\t}\n}\n\ntype Values struct {\n\tEnvs map[string]string // environment vars\n}\n\nfunc GetValues() *Values {\n\treturn &Values{\n\t\tEnvs: glbEnvs,\n\t}\n}\n\nfunc DetectLegacyINIFormat(content []byte) bool {\n\tf, err := ini.Load(content)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif _, err := f.GetSection(\"common\"); err == nil {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc DetectLegacyINIFormatFromFile(path string) bool {\n\tb, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn false\n\t}\n\treturn DetectLegacyINIFormat(b)\n}\n\nfunc RenderWithTemplate(in []byte, values *Values) ([]byte, error) {\n\ttmpl, err := template.New(\"frp\").Funcs(template.FuncMap{\n\t\t\"parseNumberRange\":     parseNumberRange,\n\t\t\"parseNumberRangePair\": parseNumberRangePair,\n\t}).Parse(string(in))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbuffer := bytes.NewBufferString(\"\")\n\tif err := tmpl.Execute(buffer, values); err != nil {\n\t\treturn nil, err\n\t}\n\treturn buffer.Bytes(), nil\n}\n\nfunc LoadFileContentWithTemplate(path string, values *Values) ([]byte, error) {\n\tb, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn RenderWithTemplate(b, values)\n}\n\nfunc LoadConfigureFromFile(path string, c any, strict bool) error {\n\tcontent, err := LoadFileContentWithTemplate(path, GetValues())\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn LoadConfigure(content, c, strict)\n}\n\n// LoadConfigure loads configuration from bytes and unmarshal into c.\n// Now it supports json, yaml and toml format.\nfunc LoadConfigure(b []byte, c any, strict bool) error {\n\tv1.DisallowUnknownFieldsMu.Lock()\n\tdefer v1.DisallowUnknownFieldsMu.Unlock()\n\tv1.DisallowUnknownFields = strict\n\n\tvar tomlObj interface{}\n\t// Try to unmarshal as TOML first; swallow errors from that (assume it's not valid TOML).\n\tif err := toml.Unmarshal(b, &tomlObj); err == nil {\n\t\tb, err = json.Marshal(&tomlObj)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// If the buffer smells like JSON (first non-whitespace character is '{'), unmarshal as JSON directly.\n\tif yaml.IsJSONBuffer(b) {\n\t\tdecoder := json.NewDecoder(bytes.NewBuffer(b))\n\t\tif strict {\n\t\t\tdecoder.DisallowUnknownFields()\n\t\t}\n\t\treturn decoder.Decode(c)\n\t}\n\t// It wasn't JSON. Unmarshal as YAML.\n\tif strict {\n\t\treturn yaml.UnmarshalStrict(b, c)\n\t}\n\treturn yaml.Unmarshal(b, c)\n}\n\nfunc NewProxyConfigurerFromMsg(m *msg.NewProxy, serverCfg *v1.ServerConfig) (v1.ProxyConfigurer, error) {\n\tm.ProxyType = util.EmptyOr(m.ProxyType, string(v1.ProxyTypeTCP))\n\n\tconfigurer := v1.NewProxyConfigurerByType(v1.ProxyType(m.ProxyType))\n\tif configurer == nil {\n\t\treturn nil, fmt.Errorf(\"unknown proxy type: %s\", m.ProxyType)\n\t}\n\n\tconfigurer.UnmarshalFromMsg(m)\n\tconfigurer.Complete(\"\")\n\n\tif err := validation.ValidateProxyConfigurerForServer(configurer, serverCfg); err != nil {\n\t\treturn nil, err\n\t}\n\treturn configurer, nil\n}\n\nfunc LoadServerConfig(path string, strict bool) (*v1.ServerConfig, bool, error) {\n\tvar (\n\t\tsvrCfg         *v1.ServerConfig\n\t\tisLegacyFormat bool\n\t)\n\t// detect legacy ini format\n\tif DetectLegacyINIFormatFromFile(path) {\n\t\tcontent, err := legacy.GetRenderedConfFromFile(path)\n\t\tif err != nil {\n\t\t\treturn nil, true, err\n\t\t}\n\t\tlegacyCfg, err := legacy.UnmarshalServerConfFromIni(content)\n\t\tif err != nil {\n\t\t\treturn nil, true, err\n\t\t}\n\t\tsvrCfg = legacy.Convert_ServerCommonConf_To_v1(&legacyCfg)\n\t\tisLegacyFormat = true\n\t} else {\n\t\tsvrCfg = &v1.ServerConfig{}\n\t\tif err := LoadConfigureFromFile(path, svrCfg, strict); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t}\n\tif svrCfg != nil {\n\t\tsvrCfg.Complete()\n\t}\n\treturn svrCfg, isLegacyFormat, nil\n}\n\nfunc LoadClientConfig(path string, strict bool) (\n\t*v1.ClientCommonConfig,\n\t[]v1.ProxyConfigurer,\n\t[]v1.VisitorConfigurer,\n\tbool, error,\n) {\n\tvar (\n\t\tcliCfg         *v1.ClientCommonConfig\n\t\tproxyCfgs      = make([]v1.ProxyConfigurer, 0)\n\t\tvisitorCfgs    = make([]v1.VisitorConfigurer, 0)\n\t\tisLegacyFormat bool\n\t)\n\n\tif DetectLegacyINIFormatFromFile(path) {\n\t\tlegacyCommon, legacyProxyCfgs, legacyVisitorCfgs, err := legacy.ParseClientConfig(path)\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, true, err\n\t\t}\n\t\tcliCfg = legacy.Convert_ClientCommonConf_To_v1(&legacyCommon)\n\t\tfor _, c := range legacyProxyCfgs {\n\t\t\tproxyCfgs = append(proxyCfgs, legacy.Convert_ProxyConf_To_v1(c))\n\t\t}\n\t\tfor _, c := range legacyVisitorCfgs {\n\t\t\tvisitorCfgs = append(visitorCfgs, legacy.Convert_VisitorConf_To_v1(c))\n\t\t}\n\t\tisLegacyFormat = true\n\t} else {\n\t\tallCfg := v1.ClientConfig{}\n\t\tif err := LoadConfigureFromFile(path, &allCfg, strict); err != nil {\n\t\t\treturn nil, nil, nil, false, err\n\t\t}\n\t\tcliCfg = &allCfg.ClientCommonConfig\n\t\tfor _, c := range allCfg.Proxies {\n\t\t\tproxyCfgs = append(proxyCfgs, c.ProxyConfigurer)\n\t\t}\n\t\tfor _, c := range allCfg.Visitors {\n\t\t\tvisitorCfgs = append(visitorCfgs, c.VisitorConfigurer)\n\t\t}\n\t}\n\n\t// Load additional config from includes.\n\t// legacy ini format already handle this in ParseClientConfig.\n\tif len(cliCfg.IncludeConfigFiles) > 0 && !isLegacyFormat {\n\t\textProxyCfgs, extVisitorCfgs, err := LoadAdditionalClientConfigs(cliCfg.IncludeConfigFiles, isLegacyFormat, strict)\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, isLegacyFormat, err\n\t\t}\n\t\tproxyCfgs = append(proxyCfgs, extProxyCfgs...)\n\t\tvisitorCfgs = append(visitorCfgs, extVisitorCfgs...)\n\t}\n\n\t// Filter by start\n\tif len(cliCfg.Start) > 0 {\n\t\tstartSet := sets.New(cliCfg.Start...)\n\t\tproxyCfgs = lo.Filter(proxyCfgs, func(c v1.ProxyConfigurer, _ int) bool {\n\t\t\treturn startSet.Has(c.GetBaseConfig().Name)\n\t\t})\n\t\tvisitorCfgs = lo.Filter(visitorCfgs, func(c v1.VisitorConfigurer, _ int) bool {\n\t\t\treturn startSet.Has(c.GetBaseConfig().Name)\n\t\t})\n\t}\n\n\tif cliCfg != nil {\n\t\tcliCfg.Complete()\n\t}\n\tfor _, c := range proxyCfgs {\n\t\tc.Complete(cliCfg.User)\n\t}\n\tfor _, c := range visitorCfgs {\n\t\tc.Complete(cliCfg)\n\t}\n\treturn cliCfg, proxyCfgs, visitorCfgs, isLegacyFormat, nil\n}\n\nfunc LoadAdditionalClientConfigs(paths []string, isLegacyFormat bool, strict bool) ([]v1.ProxyConfigurer, []v1.VisitorConfigurer, error) {\n\tproxyCfgs := make([]v1.ProxyConfigurer, 0)\n\tvisitorCfgs := make([]v1.VisitorConfigurer, 0)\n\tfor _, path := range paths {\n\t\tabsDir, err := filepath.Abs(filepath.Dir(path))\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tif _, err := os.Stat(absDir); os.IsNotExist(err) {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tfiles, err := os.ReadDir(absDir)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tfor _, fi := range files {\n\t\t\tif fi.IsDir() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tabsFile := filepath.Join(absDir, fi.Name())\n\t\t\tif matched, _ := filepath.Match(filepath.Join(absDir, filepath.Base(path)), absFile); matched {\n\t\t\t\t// support yaml/json/toml\n\t\t\t\tcfg := v1.ClientConfig{}\n\t\t\t\tif err := LoadConfigureFromFile(absFile, &cfg, strict); err != nil {\n\t\t\t\t\treturn nil, nil, fmt.Errorf(\"load additional config from %s error: %v\", absFile, err)\n\t\t\t\t}\n\t\t\t\tfor _, c := range cfg.Proxies {\n\t\t\t\t\tproxyCfgs = append(proxyCfgs, c.ProxyConfigurer)\n\t\t\t\t}\n\t\t\t\tfor _, c := range cfg.Visitors {\n\t\t\t\t\tvisitorCfgs = append(visitorCfgs, c.VisitorConfigurer)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn proxyCfgs, visitorCfgs, nil\n}\n\n// Path: pkg/config/load_test.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nconst tomlServerContent = `\nbindAddr = \"127.0.0.1\"\nkcpBindPort = 7000\nquicBindPort = 7001\ntcpmuxHTTPConnectPort = 7005\ncustom404Page = \"/abc.html\"\ntransport.tcpKeepalive = 10\n`\n\nconst yamlServerContent = `\nbindAddr: 127.0.0.1\nkcpBindPort: 7000\nquicBindPort: 7001\ntcpmuxHTTPConnectPort: 7005\ncustom404Page: /abc.html\ntransport:\n  tcpKeepalive: 10\n`\n\nconst jsonServerContent = `\n{\n  \"bindAddr\": \"127.0.0.1\",\n  \"kcpBindPort\": 7000,\n  \"quicBindPort\": 7001,\n  \"tcpmuxHTTPConnectPort\": 7005,\n  \"custom404Page\": \"/abc.html\",\n  \"transport\": {\n    \"tcpKeepalive\": 10\n  }\n}\n`\n\nfunc TestLoadServerConfig(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\tcontent string\n\t}{\n\t\t{\"toml\", tomlServerContent},\n\t\t{\"yaml\", yamlServerContent},\n\t\t{\"json\", jsonServerContent},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\trequire := require.New(t)\n\t\t\tsvrCfg := v1.ServerConfig{}\n\t\t\terr := LoadConfigure([]byte(test.content), &svrCfg, true)\n\t\t\trequire.NoError(err)\n\t\t\trequire.EqualValues(\"127.0.0.1\", svrCfg.BindAddr)\n\t\t\trequire.EqualValues(7000, svrCfg.KCPBindPort)\n\t\t\trequire.EqualValues(7001, svrCfg.QUICBindPort)\n\t\t\trequire.EqualValues(7005, svrCfg.TCPMuxHTTPConnectPort)\n\t\t\trequire.EqualValues(\"/abc.html\", svrCfg.Custom404Page)\n\t\t\trequire.EqualValues(10, svrCfg.Transport.TCPKeepAlive)\n\t\t})\n\t}\n}\n\n// Test that loading in strict mode fails when the config is invalid.\n\nfunc TestLoadServerConfigStrictMode(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\tcontent string\n\t}{\n\t\t{\"toml\", tomlServerContent},\n\t\t{\"yaml\", yamlServerContent},\n\t\t{\"json\", jsonServerContent},\n\t}\n\n\tfor _, strict := range []bool{false, true} {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(fmt.Sprintf(\"%s-strict-%t\", test.name, strict), func(t *testing.T) {\n\t\t\t\trequire := require.New(t)\n\t\t\t\t// Break the content with an innocent typo\n\t\t\t\tbrokenContent := strings.Replace(test.content, \"bindAddr\", \"bindAdur\", 1)\n\t\t\t\tsvrCfg := v1.ServerConfig{}\n\t\t\t\terr := LoadConfigure([]byte(brokenContent), &svrCfg, strict)\n\t\t\t\tif strict {\n\t\t\t\t\trequire.ErrorContains(err, \"bindAdur\")\n\t\t\t\t} else {\n\t\t\t\t\trequire.NoError(err)\n\t\t\t\t\t// BindAddr didn't get parsed because of the typo.\n\t\t\t\t\trequire.EqualValues(\"\", svrCfg.BindAddr)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestCustomStructStrictMode(t *testing.T) {\n\trequire := require.New(t)\n\n\tproxyStr := `\nserverPort = 7000\n\n[[proxies]]\nname = \"test\"\ntype = \"tcp\"\nremotePort = 6000\n`\n\tclientCfg := v1.ClientConfig{}\n\terr := LoadConfigure([]byte(proxyStr), &clientCfg, true)\n\trequire.NoError(err)\n\n\tproxyStr += `unknown = \"unknown\"`\n\terr = LoadConfigure([]byte(proxyStr), &clientCfg, true)\n\trequire.Error(err)\n\n\tvisitorStr := `\nserverPort = 7000\n\n[[visitors]]\nname = \"test\"\ntype = \"stcp\"\nbindPort = 6000\nserverName = \"server\"\n`\n\terr = LoadConfigure([]byte(visitorStr), &clientCfg, true)\n\trequire.NoError(err)\n\n\tvisitorStr += `unknown = \"unknown\"`\n\terr = LoadConfigure([]byte(visitorStr), &clientCfg, true)\n\trequire.Error(err)\n\n\tpluginStr := `\nserverPort = 7000\n\n[[proxies]]\nname = \"test\"\ntype = \"tcp\"\nremotePort = 6000\n[proxies.plugin]\ntype = \"unix_domain_socket\"\nunixPath = \"/tmp/uds.sock\"\n`\n\terr = LoadConfigure([]byte(pluginStr), &clientCfg, true)\n\trequire.NoError(err)\n\tpluginStr += `unknown = \"unknown\"`\n\terr = LoadConfigure([]byte(pluginStr), &clientCfg, true)\n\trequire.Error(err)\n}\n\n// Path: pkg/config/types/types_test.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage types\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\ntype Wrap struct {\n\tB   BandwidthQuantity `json:\"b\"`\n\tInt int               `json:\"int\"`\n}\n\nfunc TestBandwidthQuantity(t *testing.T) {\n\trequire := require.New(t)\n\n\tvar w Wrap\n\terr := json.Unmarshal([]byte(`{\"b\":\"1KB\",\"int\":5}`), &w)\n\trequire.NoError(err)\n\trequire.EqualValues(1*KB, w.B.Bytes())\n\n\tbuf, err := json.Marshal(&w)\n\trequire.NoError(err)\n\trequire.Equal(`{\"b\":\"1KB\",\"int\":5}`, string(buf))\n}\n\nfunc TestPortsRangeSlice2String(t *testing.T) {\n\trequire := require.New(t)\n\n\tports := []PortsRange{\n\t\t{\n\t\t\tStart: 1000,\n\t\t\tEnd:   2000,\n\t\t},\n\t\t{\n\t\t\tSingle: 3000,\n\t\t},\n\t}\n\tstr := PortsRangeSlice(ports).String()\n\trequire.Equal(\"1000-2000,3000\", str)\n}\n\nfunc TestNewPortsRangeSliceFromString(t *testing.T) {\n\trequire := require.New(t)\n\n\tports, err := NewPortsRangeSliceFromString(\"1000-2000,3000\")\n\trequire.NoError(err)\n\trequire.Equal([]PortsRange{\n\t\t{\n\t\t\tStart: 1000,\n\t\t\tEnd:   2000,\n\t\t},\n\t\t{\n\t\t\tSingle: 3000,\n\t\t},\n\t}, ports)\n}\n\n// Path: pkg/config/v1/api.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\ntype APIMetadata struct {\n\tVersion string `json:\"version\"`\n}\n\n// Path: pkg/config/v1/client.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"os\"\n\n\t\"github.com/samber/lo\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype ClientConfig struct {\n\tClientCommonConfig\n\n\tProxies  []TypedProxyConfig   `json:\"proxies,omitempty\"`\n\tVisitors []TypedVisitorConfig `json:\"visitors,omitempty\"`\n}\n\ntype ClientCommonConfig struct {\n\tAPIMetadata\n\n\tAuth AuthClientConfig `json:\"auth,omitempty\"`\n\t// User specifies a prefix for proxy names to distinguish them from other\n\t// clients. If this value is not \"\", proxy names will automatically be\n\t// changed to \"{user}.{proxy_name}\".\n\tUser string `json:\"user,omitempty\"`\n\n\t// ServerAddr specifies the address of the server to connect to. By\n\t// default, this value is \"0.0.0.0\".\n\tServerAddr string `json:\"serverAddr,omitempty\"`\n\t// ServerPort specifies the port to connect to the server on. By default,\n\t// this value is 7000.\n\tServerPort int `json:\"serverPort,omitempty\"`\n\t// STUN server to help penetrate NAT hole.\n\tNatHoleSTUNServer string `json:\"natHoleStunServer,omitempty\"`\n\t// DNSServer specifies a DNS server address for FRPC to use. If this value\n\t// is \"\", the default DNS will be used.\n\tDNSServer string `json:\"dnsServer,omitempty\"`\n\t// LoginFailExit controls whether or not the client should exit after a\n\t// failed login attempt. If false, the client will retry until a login\n\t// attempt succeeds. By default, this value is true.\n\tLoginFailExit *bool `json:\"loginFailExit,omitempty\"`\n\t// Start specifies a set of enabled proxies by name. If this set is empty,\n\t// all supplied proxies are enabled. By default, this value is an empty\n\t// set.\n\tStart []string `json:\"start,omitempty\"`\n\n\tLog       LogConfig             `json:\"log,omitempty\"`\n\tWebServer WebServerConfig       `json:\"webServer,omitempty\"`\n\tTransport ClientTransportConfig `json:\"transport,omitempty\"`\n\n\t// UDPPacketSize specifies the udp packet size\n\t// By default, this value is 1500\n\tUDPPacketSize int64 `json:\"udpPacketSize,omitempty\"`\n\t// Client metadata info\n\tMetadatas map[string]string `json:\"metadatas,omitempty\"`\n\n\t// Include other config files for proxies.\n\tIncludeConfigFiles []string `json:\"includes,omitempty\"`\n}\n\nfunc (c *ClientCommonConfig) Complete() {\n\tc.ServerAddr = util.EmptyOr(c.ServerAddr, \"0.0.0.0\")\n\tc.ServerPort = util.EmptyOr(c.ServerPort, 7000)\n\tc.LoginFailExit = util.EmptyOr(c.LoginFailExit, lo.ToPtr(true))\n\tc.NatHoleSTUNServer = util.EmptyOr(c.NatHoleSTUNServer, \"stun.easyvoip.com:3478\")\n\n\tc.Auth.Complete()\n\tc.Log.Complete()\n\tc.Transport.Complete()\n\tc.WebServer.Complete()\n\n\tc.UDPPacketSize = util.EmptyOr(c.UDPPacketSize, 1500)\n}\n\ntype ClientTransportConfig struct {\n\t// Protocol specifies the protocol to use when interacting with the server.\n\t// Valid values are \"tcp\", \"kcp\", \"quic\", \"websocket\" and \"wss\". By default, this value\n\t// is \"tcp\".\n\tProtocol string `json:\"protocol,omitempty\"`\n\t// The maximum amount of time a dial to server will wait for a connect to complete.\n\tDialServerTimeout int64 `json:\"dialServerTimeout,omitempty\"`\n\t// DialServerKeepAlive specifies the interval between keep-alive probes for an active network connection between frpc and frps.\n\t// If negative, keep-alive probes are disabled.\n\tDialServerKeepAlive int64 `json:\"dialServerKeepalive,omitempty\"`\n\t// ConnectServerLocalIP specifies the address of the client bind when it connect to server.\n\t// Note: This value only use in TCP/Websocket protocol. Not support in KCP protocol.\n\tConnectServerLocalIP string `json:\"connectServerLocalIP,omitempty\"`\n\t// ProxyURL specifies a proxy address to connect to the server through. If\n\t// this value is \"\", the server will be connected to directly. By default,\n\t// this value is read from the \"http_proxy\" environment variable.\n\tProxyURL string `json:\"proxyURL,omitempty\"`\n\t// PoolCount specifies the number of connections the client will make to\n\t// the server in advance.\n\tPoolCount int `json:\"poolCount,omitempty\"`\n\t// TCPMux toggles TCP stream multiplexing. This allows multiple requests\n\t// from a client to share a single TCP connection. If this value is true,\n\t// the server must have TCP multiplexing enabled as well. By default, this\n\t// value is true.\n\tTCPMux *bool `json:\"tcpMux,omitempty\"`\n\t// TCPMuxKeepaliveInterval specifies the keep alive interval for TCP stream multiplier.\n\t// If TCPMux is true, heartbeat of application layer is unnecessary because it can only rely on heartbeat in TCPMux.\n\tTCPMuxKeepaliveInterval int64 `json:\"tcpMuxKeepaliveInterval,omitempty\"`\n\t// QUIC protocol options.\n\tQUIC QUICOptions `json:\"quic,omitempty\"`\n\t// HeartBeatInterval specifies at what interval heartbeats are sent to the\n\t// server, in seconds. It is not recommended to change this value. By\n\t// default, this value is 30. Set negative value to disable it.\n\tHeartbeatInterval int64 `json:\"heartbeatInterval,omitempty\"`\n\t// HeartBeatTimeout specifies the maximum allowed heartbeat response delay\n\t// before the connection is terminated, in seconds. It is not recommended\n\t// to change this value. By default, this value is 90. Set negative value to disable it.\n\tHeartbeatTimeout int64 `json:\"heartbeatTimeout,omitempty\"`\n\t// TLS specifies TLS settings for the connection to the server.\n\tTLS TLSClientConfig `json:\"tls,omitempty\"`\n}\n\nfunc (c *ClientTransportConfig) Complete() {\n\tc.Protocol = util.EmptyOr(c.Protocol, \"tcp\")\n\tc.DialServerTimeout = util.EmptyOr(c.DialServerTimeout, 10)\n\tc.DialServerKeepAlive = util.EmptyOr(c.DialServerKeepAlive, 7200)\n\tc.ProxyURL = util.EmptyOr(c.ProxyURL, os.Getenv(\"http_proxy\"))\n\tc.PoolCount = util.EmptyOr(c.PoolCount, 1)\n\tc.TCPMux = util.EmptyOr(c.TCPMux, lo.ToPtr(true))\n\tc.TCPMuxKeepaliveInterval = util.EmptyOr(c.TCPMuxKeepaliveInterval, 60)\n\tc.HeartbeatInterval = util.EmptyOr(c.HeartbeatInterval, 30)\n\tc.HeartbeatTimeout = util.EmptyOr(c.HeartbeatTimeout, 90)\n\tc.QUIC.Complete()\n\tc.TLS.Complete()\n}\n\ntype TLSClientConfig struct {\n\t// TLSEnable specifies whether or not TLS should be used when communicating\n\t// with the server. If \"tls.certFile\" and \"tls.keyFile\" are valid,\n\t// client will load the supplied tls configuration.\n\t// Since v0.50.0, the default value has been changed to true, and tls is enabled by default.\n\tEnable *bool `json:\"enable,omitempty\"`\n\t// If DisableCustomTLSFirstByte is set to false, frpc will establish a connection with frps using the\n\t// first custom byte when tls is enabled.\n\t// Since v0.50.0, the default value has been changed to true, and the first custom byte is disabled by default.\n\tDisableCustomTLSFirstByte *bool `json:\"disableCustomTLSFirstByte,omitempty\"`\n\n\tTLSConfig\n}\n\nfunc (c *TLSClientConfig) Complete() {\n\tc.Enable = util.EmptyOr(c.Enable, lo.ToPtr(true))\n\tc.DisableCustomTLSFirstByte = util.EmptyOr(c.DisableCustomTLSFirstByte, lo.ToPtr(true))\n}\n\ntype AuthClientConfig struct {\n\t// Method specifies what authentication method to use to\n\t// authenticate frpc with frps. If \"token\" is specified - token will be\n\t// read into login message. If \"oidc\" is specified - OIDC (Open ID Connect)\n\t// token will be issued using OIDC settings. By default, this value is \"token\".\n\tMethod AuthMethod `json:\"method,omitempty\"`\n\t// Specify whether to include auth info in additional scope.\n\t// Current supported scopes are: \"HeartBeats\", \"NewWorkConns\".\n\tAdditionalScopes []AuthScope `json:\"additionalScopes,omitempty\"`\n\t// Token specifies the authorization token used to create keys to be sent\n\t// to the server. The server must have a matching token for authorization\n\t// to succeed.  By default, this value is \"\".\n\tToken string               `json:\"token,omitempty\"`\n\tOIDC  AuthOIDCClientConfig `json:\"oidc,omitempty\"`\n}\n\nfunc (c *AuthClientConfig) Complete() {\n\tc.Method = util.EmptyOr(c.Method, \"token\")\n}\n\ntype AuthOIDCClientConfig struct {\n\t// ClientID specifies the client ID to use to get a token in OIDC authentication.\n\tClientID string `json:\"clientID,omitempty\"`\n\t// ClientSecret specifies the client secret to use to get a token in OIDC\n\t// authentication.\n\tClientSecret string `json:\"clientSecret,omitempty\"`\n\t// Audience specifies the audience of the token in OIDC authentication.\n\tAudience string `json:\"audience,omitempty\"`\n\t// Scope specifies the scope of the token in OIDC authentication.\n\tScope string `json:\"scope,omitempty\"`\n\t// TokenEndpointURL specifies the URL which implements OIDC Token Endpoint.\n\t// It will be used to get an OIDC token.\n\tTokenEndpointURL string `json:\"tokenEndpointURL,omitempty\"`\n\t// AdditionalEndpointParams specifies additional parameters to be sent\n\t// this field will be transfer to map[string][]string in OIDC token generator.\n\tAdditionalEndpointParams map[string]string `json:\"additionalEndpointParams,omitempty\"`\n}\n\n// Path: pkg/config/v1/client_test.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"testing\"\n\n\t\"github.com/samber/lo\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestClientConfigComplete(t *testing.T) {\n\trequire := require.New(t)\n\tc := &ClientConfig{}\n\tc.Complete()\n\n\trequire.EqualValues(\"token\", c.Auth.Method)\n\trequire.Equal(true, lo.FromPtr(c.Transport.TCPMux))\n\trequire.Equal(true, lo.FromPtr(c.LoginFailExit))\n\trequire.Equal(true, lo.FromPtr(c.Transport.TLS.Enable))\n\trequire.Equal(true, lo.FromPtr(c.Transport.TLS.DisableCustomTLSFirstByte))\n\trequire.NotEmpty(c.NatHoleSTUNServer)\n}\n\n// Path: pkg/config/v1/plugin.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"reflect\"\n)\n\ntype ClientPluginOptions interface{}\n\ntype TypedClientPluginOptions struct {\n\tType string `json:\"type\"`\n\tClientPluginOptions\n}\n\nfunc (c *TypedClientPluginOptions) UnmarshalJSON(b []byte) error {\n\tif len(b) == 4 && string(b) == \"null\" {\n\t\treturn nil\n\t}\n\n\ttypeStruct := struct {\n\t\tType string `json:\"type\"`\n\t}{}\n\tif err := json.Unmarshal(b, &typeStruct); err != nil {\n\t\treturn err\n\t}\n\n\tc.Type = typeStruct.Type\n\tif c.Type == \"\" {\n\t\treturn nil\n\t}\n\n\tv, ok := clientPluginOptionsTypeMap[typeStruct.Type]\n\tif !ok {\n\t\treturn fmt.Errorf(\"unknown plugin type: %s\", typeStruct.Type)\n\t}\n\toptions := reflect.New(v).Interface().(ClientPluginOptions)\n\n\tdecoder := json.NewDecoder(bytes.NewBuffer(b))\n\tif DisallowUnknownFields {\n\t\tdecoder.DisallowUnknownFields()\n\t}\n\n\tif err := decoder.Decode(options); err != nil {\n\t\treturn fmt.Errorf(\"unmarshal ClientPluginOptions error: %v\", err)\n\t}\n\tc.ClientPluginOptions = options\n\treturn nil\n}\n\nconst (\n\tPluginHTTP2HTTPS       = \"http2https\"\n\tPluginHTTPProxy        = \"http_proxy\"\n\tPluginHTTPS2HTTP       = \"https2http\"\n\tPluginHTTPS2HTTPS      = \"https2https\"\n\tPluginSocks5           = \"socks5\"\n\tPluginStaticFile       = \"static_file\"\n\tPluginUnixDomainSocket = \"unix_domain_socket\"\n)\n\nvar clientPluginOptionsTypeMap = map[string]reflect.Type{\n\tPluginHTTP2HTTPS:       reflect.TypeOf(HTTP2HTTPSPluginOptions{}),\n\tPluginHTTPProxy:        reflect.TypeOf(HTTPProxyPluginOptions{}),\n\tPluginHTTPS2HTTP:       reflect.TypeOf(HTTPS2HTTPPluginOptions{}),\n\tPluginHTTPS2HTTPS:      reflect.TypeOf(HTTPS2HTTPSPluginOptions{}),\n\tPluginSocks5:           reflect.TypeOf(Socks5PluginOptions{}),\n\tPluginStaticFile:       reflect.TypeOf(StaticFilePluginOptions{}),\n\tPluginUnixDomainSocket: reflect.TypeOf(UnixDomainSocketPluginOptions{}),\n}\n\ntype HTTP2HTTPSPluginOptions struct {\n\tType              string           `json:\"type,omitempty\"`\n\tLocalAddr         string           `json:\"localAddr,omitempty\"`\n\tHostHeaderRewrite string           `json:\"hostHeaderRewrite,omitempty\"`\n\tRequestHeaders    HeaderOperations `json:\"requestHeaders,omitempty\"`\n}\n\ntype HTTPProxyPluginOptions struct {\n\tType         string `json:\"type,omitempty\"`\n\tHTTPUser     string `json:\"httpUser,omitempty\"`\n\tHTTPPassword string `json:\"httpPassword,omitempty\"`\n}\n\ntype HTTPS2HTTPPluginOptions struct {\n\tType              string           `json:\"type,omitempty\"`\n\tLocalAddr         string           `json:\"localAddr,omitempty\"`\n\tHostHeaderRewrite string           `json:\"hostHeaderRewrite,omitempty\"`\n\tRequestHeaders    HeaderOperations `json:\"requestHeaders,omitempty\"`\n\tCrtPath           string           `json:\"crtPath,omitempty\"`\n\tKeyPath           string           `json:\"keyPath,omitempty\"`\n}\n\ntype HTTPS2HTTPSPluginOptions struct {\n\tType              string           `json:\"type,omitempty\"`\n\tLocalAddr         string           `json:\"localAddr,omitempty\"`\n\tHostHeaderRewrite string           `json:\"hostHeaderRewrite,omitempty\"`\n\tRequestHeaders    HeaderOperations `json:\"requestHeaders,omitempty\"`\n\tCrtPath           string           `json:\"crtPath,omitempty\"`\n\tKeyPath           string           `json:\"keyPath,omitempty\"`\n}\n\ntype Socks5PluginOptions struct {\n\tType     string `json:\"type,omitempty\"`\n\tUsername string `json:\"username,omitempty\"`\n\tPassword string `json:\"password,omitempty\"`\n}\n\ntype StaticFilePluginOptions struct {\n\tType         string `json:\"type,omitempty\"`\n\tLocalPath    string `json:\"localPath,omitempty\"`\n\tStripPrefix  string `json:\"stripPrefix,omitempty\"`\n\tHTTPUser     string `json:\"httpUser,omitempty\"`\n\tHTTPPassword string `json:\"httpPassword,omitempty\"`\n}\n\ntype UnixDomainSocketPluginOptions struct {\n\tType     string `json:\"type,omitempty\"`\n\tUnixPath string `json:\"unixPath,omitempty\"`\n}\n\n// Path: pkg/config/v1/proxy.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/samber/lo\"\n\n\t\"github.com/fatedier/frp/pkg/config/types\"\n\t\"github.com/fatedier/frp/pkg/msg\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype ProxyTransport struct {\n\t// UseEncryption controls whether or not communication with the server will\n\t// be encrypted. Encryption is done using the tokens supplied in the server\n\t// and client configuration.\n\tUseEncryption bool `json:\"useEncryption,omitempty\"`\n\t// UseCompression controls whether or not communication with the server\n\t// will be compressed.\n\tUseCompression bool `json:\"useCompression,omitempty\"`\n\t// BandwidthLimit limit the bandwidth\n\t// 0 means no limit\n\tBandwidthLimit types.BandwidthQuantity `json:\"bandwidthLimit,omitempty\"`\n\t// BandwidthLimitMode specifies whether to limit the bandwidth on the\n\t// client or server side. Valid values include \"client\" and \"server\".\n\t// By default, this value is \"client\".\n\tBandwidthLimitMode string `json:\"bandwidthLimitMode,omitempty\"`\n\t// ProxyProtocolVersion specifies which protocol version to use. Valid\n\t// values include \"v1\", \"v2\", and \"\". If the value is \"\", a protocol\n\t// version will be automatically selected. By default, this value is \"\".\n\tProxyProtocolVersion string `json:\"proxyProtocolVersion,omitempty\"`\n}\n\ntype LoadBalancerConfig struct {\n\t// Group specifies which group the is a part of. The server will use\n\t// this information to load balance proxies in the same group. If the value\n\t// is \"\", this will not be in a group.\n\tGroup string `json:\"group\"`\n\t// GroupKey specifies a group key, which should be the same among proxies\n\t// of the same group.\n\tGroupKey string `json:\"groupKey,omitempty\"`\n}\n\ntype ProxyBackend struct {\n\t// LocalIP specifies the IP address or host name of the backend.\n\tLocalIP string `json:\"localIP,omitempty\"`\n\t// LocalPort specifies the port of the backend.\n\tLocalPort int `json:\"localPort,omitempty\"`\n\n\t// Plugin specifies what plugin should be used for handling connections. If this value\n...\n", "needle_token_start": 8270, "needle_token_end": 8611, "code_context_ntokens": 15393, "output": ["```go\nfunc TestLoadServerConfigStrictMode(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\tcontent string\n\t}{\n\t\t{\"toml\", tomlServerContent},\n\t\t{\"yaml\", yamlServerContent},\n\t\t{\"json\", jsonServerContent},\n\t}\n\n\tfor _, strict := range []bool{false, true} {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(fmt.Sprintf(\"%s-strict-%t\", test.name, strict), func(t *testing.T) {\n\t\t\t\trequire := require.New(t)\n\t\t\t\t// Break the content with an innocent typo\n\t\t\t\tbrokenContent := strings.Replace(test.content, \"bindAddr\", \"bindAdur\", 1)\n\t\t\t\tsvrCfg := v1.ServerConfig{}\n\t\t\t\terr := LoadConfigure([]byte(brokenContent), &svrCfg, strict)\n\t\t\t\tif strict {\n\t\t\t\t\trequire.ErrorContains(err, \"bindAdur\")\n\t\t\t\t} else {\n\t\t\t\t\trequire.NoError(err)\n\t\t\t\t\t// BindAddr didn't get parsed because of the typo.\n\t\t\t\t\trequire.EqualValues(\"\", svrCfg.BindAddr)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n```"]}
{"repo": "fatedier/frp", "name": "ValidateServerConfig", "language": "go", "path": "pkg/config/v1/validation/server.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To ensure that the server configuration adheres to specific rules and supported values, preventing invalid or unsupported settings from being used.\n2. **Input**: A structure representing server configuration details.\n3. **Output**: A list of warnings for minor issues and an error object that aggregates all critical issues found during validation.\n4. **Procedure**: The function checks various aspects of the server configuration such as authentication methods, log configuration, web server settings, and port assignments. It validates each component against predefined lists of supported values and configurations. If any setting does not meet the criteria, an error is recorded. Additionally, it checks for the presence and validity of optional features like HTTP plugins.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/config/v1/proxy.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/samber/lo\"\n\n\t\"github.com/fatedier/frp/pkg/config/types\"\n\t\"github.com/fatedier/frp/pkg/msg\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype ProxyTransport struct {\n\t// UseEncryption controls whether or not communication with the server will\n\t// be encrypted. Encryption is done using the tokens supplied in the server\n\t// and client configuration.\n\tUseEncryption bool `json:\"useEncryption,omitempty\"`\n\t// UseCompression controls whether or not communication with the server\n\t// will be compressed.\n\tUseCompression bool `json:\"useCompression,omitempty\"`\n\t// BandwidthLimit limit the bandwidth\n\t// 0 means no limit\n\tBandwidthLimit types.BandwidthQuantity `json:\"bandwidthLimit,omitempty\"`\n\t// BandwidthLimitMode specifies whether to limit the bandwidth on the\n\t// client or server side. Valid values include \"client\" and \"server\".\n\t// By default, this value is \"client\".\n\tBandwidthLimitMode string `json:\"bandwidthLimitMode,omitempty\"`\n\t// ProxyProtocolVersion specifies which protocol version to use. Valid\n\t// values include \"v1\", \"v2\", and \"\". If the value is \"\", a protocol\n\t// version will be automatically selected. By default, this value is \"\".\n\tProxyProtocolVersion string `json:\"proxyProtocolVersion,omitempty\"`\n}\n\ntype LoadBalancerConfig struct {\n\t// Group specifies which group the is a part of. The server will use\n\t// this information to load balance proxies in the same group. If the value\n\t// is \"\", this will not be in a group.\n\tGroup string `json:\"group\"`\n\t// GroupKey specifies a group key, which should be the same among proxies\n\t// of the same group.\n\tGroupKey string `json:\"groupKey,omitempty\"`\n}\n\ntype ProxyBackend struct {\n\t// LocalIP specifies the IP address or host name of the backend.\n\tLocalIP string `json:\"localIP,omitempty\"`\n\t// LocalPort specifies the port of the backend.\n\tLocalPort int `json:\"localPort,omitempty\"`\n\n\t// Plugin specifies what plugin should be used for handling connections. If this value\n\t// is set, the LocalIP and LocalPort values will be ignored.\n\tPlugin TypedClientPluginOptions `json:\"plugin,omitempty\"`\n}\n\n// HealthCheckConfig configures health checking. This can be useful for load\n// balancing purposes to detect and remove proxies to failing services.\ntype HealthCheckConfig struct {\n...\n// Path: pkg/config/v1/proxy_test.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestUnmarshalTypedProxyConfig(t *testing.T) {\n\trequire := require.New(t)\n\tproxyConfigs := struct {\n\t\tProxies []TypedProxyConfig `json:\"proxies,omitempty\"`\n\t}{}\n\n\tstrs := `{\n\t\t\"proxies\": [\n\t\t\t{\n\t\t\t\t\"type\": \"tcp\",\n\t\t\t\t\"localPort\": 22,\n\t\t\t\t\"remotePort\": 6000\n\t\t\t},\n\t\t\t{\n\t\t\t\t\"type\": \"http\",\n\t\t\t\t\"localPort\": 80,\n\t\t\t\t\"customDomains\": [\"www.example.com\"]\n\t\t\t}\n\t\t]\n\t}`\n\terr := json.Unmarshal([]byte(strs), &proxyConfigs)\n\trequire.NoError(err)\n\n\trequire.IsType(&TCPProxyConfig{}, proxyConfigs.Proxies[0].ProxyConfigurer)\n\trequire.IsType(&HTTPProxyConfig{}, proxyConfigs.Proxies[1].ProxyConfigurer)\n}\n\n// Path: pkg/config/v1/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"github.com/samber/lo\"\n\n\t\"github.com/fatedier/frp/pkg/config/types\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype ServerConfig struct {\n\tAPIMetadata\n\n\tAuth AuthServerConfig `json:\"auth,omitempty\"`\n\t// BindAddr specifies the address that the server binds to. By default,\n\t// this value is \"0.0.0.0\".\n\tBindAddr string `json:\"bindAddr,omitempty\"`\n\t// BindPort specifies the port that the server listens on. By default, this\n\t// value is 7000.\n\tBindPort int `json:\"bindPort,omitempty\"`\n\t// KCPBindPort specifies the KCP port that the server listens on. If this\n\t// value is 0, the server will not listen for KCP connections.\n\tKCPBindPort int `json:\"kcpBindPort,omitempty\"`\n\t// QUICBindPort specifies the QUIC port that the server listens on.\n\t// Set this value to 0 will disable this feature.\n\tQUICBindPort int `json:\"quicBindPort,omitempty\"`\n\t// ProxyBindAddr specifies the address that the proxy binds to. This value\n\t// may be the same as BindAddr.\n\tProxyBindAddr string `json:\"proxyBindAddr,omitempty\"`\n\t// VhostHTTPPort specifies the port that the server listens for HTTP Vhost\n\t// requests. If this value is 0, the server will not listen for HTTP\n\t// requests.\n\tVhostHTTPPort int `json:\"vhostHTTPPort,omitempty\"`\n\t// VhostHTTPTimeout specifies the response header timeout for the Vhost\n\t// HTTP server, in seconds. By default, this value is 60.\n\tVhostHTTPTimeout int64 `json:\"vhostHTTPTimeout,omitempty\"`\n\t// VhostHTTPSPort specifies the port that the server listens for HTTPS\n\t// Vhost requests. If this value is 0, the server will not listen for HTTPS\n\t// requests.\n\tVhostHTTPSPort int `json:\"vhostHTTPSPort,omitempty\"`\n\t// TCPMuxHTTPConnectPort specifies the port that the server listens for TCP\n\t// HTTP CONNECT requests. If the value is 0, the server will not multiplex TCP\n\t// requests on one single port. If it's not - it will listen on this value for\n\t// HTTP CONNECT requests.\n\tTCPMuxHTTPConnectPort int `json:\"tcpmuxHTTPConnectPort,omitempty\"`\n\t// If TCPMuxPassthrough is true, frps won't do any update on traffic.\n\tTCPMuxPassthrough bool `json:\"tcpmuxPassthrough,omitempty\"`\n\t// SubDomainHost specifies the domain that will be attached to sub-domains\n\t// requested by the client when using Vhost proxying. For example, if this\n\t// value is set to \"frps.com\" and the client requested the subdomain\n\t// \"test\", the resulting URL would be \"test.frps.com\".\n\tSubDomainHost string `json:\"subDomainHost,omitempty\"`\n\t// Custom404Page specifies a path to a custom 404 page to display. If this\n\t// value is \"\", a default page will be displayed.\n\tCustom404Page string `json:\"custom404Page,omitempty\"`\n\n\tSSHTunnelGateway SSHTunnelGateway `json:\"sshTunnelGateway,omitempty\"`\n\n\tWebServer WebServerConfig `json:\"webServer,omitempty\"`\n\t// EnablePrometheus will export prometheus metrics on webserver address\n\t// in /metrics api.\n\tEnablePrometheus bool `json:\"enablePrometheus,omitempty\"`\n\n\tLog LogConfig `json:\"log,omitempty\"`\n\n\tTransport ServerTransportConfig `json:\"transport,omitempty\"`\n\n\t// DetailedErrorsToClient defines whether to send the specific error (with\n\t// debug info) to frpc. By default, this value is true.\n\tDetailedErrorsToClient *bool `json:\"detailedErrorsToClient,omitempty\"`\n\t// MaxPortsPerClient specifies the maximum number of ports a single client\n\t// may proxy to. If this value is 0, no limit will be applied.\n\tMaxPortsPerClient int64 `json:\"maxPortsPerClient,omitempty\"`\n\t// UserConnTimeout specifies the maximum time to wait for a work\n\t// connection. By default, this value is 10.\n\tUserConnTimeout int64 `json:\"userConnTimeout,omitempty\"`\n\t// UDPPacketSize specifies the UDP packet size\n\t// By default, this value is 1500\n\tUDPPacketSize int64 `json:\"udpPacketSize,omitempty\"`\n\t// NatHoleAnalysisDataReserveHours specifies the hours to reserve nat hole analysis data.\n\tNatHoleAnalysisDataReserveHours int64 `json:\"natholeAnalysisDataReserveHours,omitempty\"`\n\n\tAllowPorts []types.PortsRange `json:\"allowPorts,omitempty\"`\n\n\tHTTPPlugins []HTTPPluginOptions `json:\"httpPlugins,omitempty\"`\n}\n\nfunc (c *ServerConfig) Complete() {\n\tc.Auth.Complete()\n\tc.Log.Complete()\n\tc.Transport.Complete()\n\tc.WebServer.Complete()\n\tc.SSHTunnelGateway.Complete()\n\n\tc.BindAddr = util.EmptyOr(c.BindAddr, \"0.0.0.0\")\n\tc.BindPort = util.EmptyOr(c.BindPort, 7000)\n\tif c.ProxyBindAddr == \"\" {\n\t\tc.ProxyBindAddr = c.BindAddr\n\t}\n\n\tif c.WebServer.Port > 0 {\n\t\tc.WebServer.Addr = util.EmptyOr(c.WebServer.Addr, \"0.0.0.0\")\n\t}\n\n\tc.VhostHTTPTimeout = util.EmptyOr(c.VhostHTTPTimeout, 60)\n\tc.DetailedErrorsToClient = util.EmptyOr(c.DetailedErrorsToClient, lo.ToPtr(true))\n\tc.UserConnTimeout = util.EmptyOr(c.UserConnTimeout, 10)\n\tc.UDPPacketSize = util.EmptyOr(c.UDPPacketSize, 1500)\n\tc.NatHoleAnalysisDataReserveHours = util.EmptyOr(c.NatHoleAnalysisDataReserveHours, 7*24)\n}\n\ntype AuthServerConfig struct {\n\tMethod           AuthMethod           `json:\"method,omitempty\"`\n\tAdditionalScopes []AuthScope          `json:\"additionalScopes,omitempty\"`\n\tToken            string               `json:\"token,omitempty\"`\n\tOIDC             AuthOIDCServerConfig `json:\"oidc,omitempty\"`\n}\n\nfunc (c *AuthServerConfig) Complete() {\n\tc.Method = util.EmptyOr(c.Method, \"token\")\n}\n\ntype AuthOIDCServerConfig struct {\n\t// Issuer specifies the issuer to verify OIDC tokens with. This issuer\n\t// will be used to load public keys to verify signature and will be compared\n\t// with the issuer claim in the OIDC token.\n\tIssuer string `json:\"issuer,omitempty\"`\n\t// Audience specifies the audience OIDC tokens should contain when validated.\n\t// If this value is empty, audience (\"client ID\") verification will be skipped.\n\tAudience string `json:\"audience,omitempty\"`\n\t// SkipExpiryCheck specifies whether to skip checking if the OIDC token is\n\t// expired.\n\tSkipExpiryCheck bool `json:\"skipExpiryCheck,omitempty\"`\n\t// SkipIssuerCheck specifies whether to skip checking if the OIDC token's\n\t// issuer claim matches the issuer specified in OidcIssuer.\n\tSkipIssuerCheck bool `json:\"skipIssuerCheck,omitempty\"`\n}\n\ntype ServerTransportConfig struct {\n\t// TCPMux toggles TCP stream multiplexing. This allows multiple requests\n\t// from a client to share a single TCP connection. By default, this value\n\t// is true.\n\t// $HideFromDoc\n\tTCPMux *bool `json:\"tcpMux,omitempty\"`\n\t// TCPMuxKeepaliveInterval specifies the keep alive interval for TCP stream multiplier.\n\t// If TCPMux is true, heartbeat of application layer is unnecessary because it can only rely on heartbeat in TCPMux.\n\tTCPMuxKeepaliveInterval int64 `json:\"tcpMuxKeepaliveInterval,omitempty\"`\n\t// TCPKeepAlive specifies the interval between keep-alive probes for an active network connection between frpc and frps.\n\t// If negative, keep-alive probes are disabled.\n\tTCPKeepAlive int64 `json:\"tcpKeepalive,omitempty\"`\n\t// MaxPoolCount specifies the maximum pool size for each proxy. By default,\n\t// this value is 5.\n\tMaxPoolCount int64 `json:\"maxPoolCount,omitempty\"`\n\t// HeartBeatTimeout specifies the maximum time to wait for a heartbeat\n\t// before terminating the connection. It is not recommended to change this\n\t// value. By default, this value is 90. Set negative value to disable it.\n\tHeartbeatTimeout int64 `json:\"heartbeatTimeout,omitempty\"`\n\t// QUIC options.\n\tQUIC QUICOptions `json:\"quic,omitempty\"`\n\t// TLS specifies TLS settings for the connection from the client.\n\tTLS TLSServerConfig `json:\"tls,omitempty\"`\n}\n\nfunc (c *ServerTransportConfig) Complete() {\n\tc.TCPMux = util.EmptyOr(c.TCPMux, lo.ToPtr(true))\n\tc.TCPMuxKeepaliveInterval = util.EmptyOr(c.TCPMuxKeepaliveInterval, 60)\n\tc.TCPKeepAlive = util.EmptyOr(c.TCPKeepAlive, 7200)\n\tc.MaxPoolCount = util.EmptyOr(c.MaxPoolCount, 5)\n\tc.HeartbeatTimeout = util.EmptyOr(c.HeartbeatTimeout, 90)\n\tc.QUIC.Complete()\n\tif c.TLS.TrustedCaFile != \"\" {\n\t\tc.TLS.Force = true\n\t}\n}\n\ntype TLSServerConfig struct {\n\t// Force specifies whether to only accept TLS-encrypted connections.\n\tForce bool `json:\"force,omitempty\"`\n\n\tTLSConfig\n}\n\ntype SSHTunnelGateway struct {\n\tBindPort              int    `json:\"bindPort,omitempty\"`\n\tPrivateKeyFile        string `json:\"privateKeyFile,omitempty\"`\n\tAutoGenPrivateKeyPath string `json:\"autoGenPrivateKeyPath,omitempty\"`\n\tAuthorizedKeysFile    string `json:\"authorizedKeysFile,omitempty\"`\n}\n\nfunc (c *SSHTunnelGateway) Complete() {\n\tc.AutoGenPrivateKeyPath = util.EmptyOr(c.AutoGenPrivateKeyPath, \"./.autogen_ssh_key\")\n}\n\n// Path: pkg/config/v1/server_test.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"testing\"\n\n\t\"github.com/samber/lo\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestServerConfigComplete(t *testing.T) {\n\trequire := require.New(t)\n\tc := &ServerConfig{}\n\tc.Complete()\n\n\trequire.EqualValues(\"token\", c.Auth.Method)\n\trequire.Equal(true, lo.FromPtr(c.Transport.TCPMux))\n\trequire.Equal(true, lo.FromPtr(c.DetailedErrorsToClient))\n}\n\n// Path: pkg/config/v1/validation/client.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n\n\t\"github.com/samber/lo\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateClientCommonConfig(c *v1.ClientCommonConfig) (Warning, error) {\n\tvar (\n\t\twarnings Warning\n\t\terrs     error\n\t)\n\tif !slices.Contains(SupportedAuthMethods, c.Auth.Method) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth method, optional values are %v\", SupportedAuthMethods))\n\t}\n\tif !lo.Every(SupportedAuthAdditionalScopes, c.Auth.AdditionalScopes) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth additional scopes, optional values are %v\", SupportedAuthAdditionalScopes))\n\t}\n\n\tif err := validateLogConfig(&c.Log); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\tif err := validateWebServerConfig(&c.WebServer); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\tif c.Transport.HeartbeatTimeout > 0 && c.Transport.HeartbeatInterval > 0 {\n\t\tif c.Transport.HeartbeatTimeout < c.Transport.HeartbeatInterval {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"invalid transport.heartbeatTimeout, heartbeat timeout should not less than heartbeat interval\"))\n\t\t}\n\t}\n\n\tif !lo.FromPtr(c.Transport.TLS.Enable) {\n\t\tcheckTLSConfig := func(name string, value string) Warning {\n\t\t\tif value != \"\" {\n\t\t\t\treturn fmt.Errorf(\"%s is invalid when transport.tls.enable is false\", name)\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\n\t\twarnings = AppendError(warnings, checkTLSConfig(\"transport.tls.certFile\", c.Transport.TLS.CertFile))\n\t\twarnings = AppendError(warnings, checkTLSConfig(\"transport.tls.keyFile\", c.Transport.TLS.KeyFile))\n\t\twarnings = AppendError(warnings, checkTLSConfig(\"transport.tls.trustedCaFile\", c.Transport.TLS.TrustedCaFile))\n\t}\n\n\tif !slices.Contains(SupportedTransportProtocols, c.Transport.Protocol) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid transport.protocol, optional values are %v\", SupportedTransportProtocols))\n\t}\n\n\tfor _, f := range c.IncludeConfigFiles {\n\t\tabsDir, err := filepath.Abs(filepath.Dir(f))\n\t\tif err != nil {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"include: parse directory of %s failed: %v\", f, err))\n\t\t\tcontinue\n\t\t}\n\t\tif _, err := os.Stat(absDir); os.IsNotExist(err) {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"include: directory of %s not exist\", f))\n\t\t}\n\t}\n\treturn warnings, errs\n}\n\nfunc ValidateAllClientConfig(c *v1.ClientCommonConfig, proxyCfgs []v1.ProxyConfigurer, visitorCfgs []v1.VisitorConfigurer) (Warning, error) {\n\tvar warnings Warning\n\tif c != nil {\n\t\twarning, err := ValidateClientCommonConfig(c)\n\t\twarnings = AppendError(warnings, warning)\n\t\tif err != nil {\n\t\t\treturn warnings, err\n\t\t}\n\t}\n\n\tfor _, c := range proxyCfgs {\n\t\tif err := ValidateProxyConfigurerForClient(c); err != nil {\n\t\t\treturn warnings, fmt.Errorf(\"proxy %s: %v\", c.GetBaseConfig().Name, err)\n\t\t}\n\t}\n\n\tfor _, c := range visitorCfgs {\n\t\tif err := ValidateVisitorConfigurer(c); err != nil {\n\t\t\treturn warnings, fmt.Errorf(\"visitor %s: %v\", c.GetBaseConfig().Name, err)\n\t\t}\n\t}\n\treturn warnings, nil\n}\n\n// Path: pkg/config/v1/validation/plugin.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"errors\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateClientPluginOptions(c v1.ClientPluginOptions) error {\n\tswitch v := c.(type) {\n\tcase *v1.HTTP2HTTPSPluginOptions:\n\t\treturn validateHTTP2HTTPSPluginOptions(v)\n\tcase *v1.HTTPS2HTTPPluginOptions:\n\t\treturn validateHTTPS2HTTPPluginOptions(v)\n\tcase *v1.HTTPS2HTTPSPluginOptions:\n\t\treturn validateHTTPS2HTTPSPluginOptions(v)\n\tcase *v1.StaticFilePluginOptions:\n\t\treturn validateStaticFilePluginOptions(v)\n\tcase *v1.UnixDomainSocketPluginOptions:\n\t\treturn validateUnixDomainSocketPluginOptions(v)\n\t}\n\treturn nil\n}\n\nfunc validateHTTP2HTTPSPluginOptions(c *v1.HTTP2HTTPSPluginOptions) error {\n\tif c.LocalAddr == \"\" {\n\t\treturn errors.New(\"localAddr is required\")\n\t}\n\treturn nil\n}\n\nfunc validateHTTPS2HTTPPluginOptions(c *v1.HTTPS2HTTPPluginOptions) error {\n\tif c.LocalAddr == \"\" {\n\t\treturn errors.New(\"localAddr is required\")\n\t}\n\treturn nil\n}\n\nfunc validateHTTPS2HTTPSPluginOptions(c *v1.HTTPS2HTTPSPluginOptions) error {\n\tif c.LocalAddr == \"\" {\n\t\treturn errors.New(\"localAddr is required\")\n\t}\n\treturn nil\n}\n\nfunc validateStaticFilePluginOptions(c *v1.StaticFilePluginOptions) error {\n\tif c.LocalPath == \"\" {\n\t\treturn errors.New(\"localPath is required\")\n\t}\n\treturn nil\n}\n\nfunc validateUnixDomainSocketPluginOptions(c *v1.UnixDomainSocketPluginOptions) error {\n\tif c.UnixPath == \"\" {\n\t\treturn errors.New(\"unixPath is required\")\n\t}\n\treturn nil\n}\n\n// Path: pkg/config/v1/validation/proxy.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"slices\"\n\t\"strings\"\n\n\t\"k8s.io/apimachinery/pkg/util/validation\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc validateProxyBaseConfigForClient(c *v1.ProxyBaseConfig) error {\n\tif c.Name == \"\" {\n\t\treturn errors.New(\"name should not be empty\")\n\t}\n\n\tif err := ValidateAnnotations(c.Annotations); err != nil {\n\t\treturn err\n\t}\n\tif !slices.Contains([]string{\"\", \"v1\", \"v2\"}, c.Transport.ProxyProtocolVersion) {\n\t\treturn fmt.Errorf(\"not support proxy protocol version: %s\", c.Transport.ProxyProtocolVersion)\n\t}\n\tif !slices.Contains([]string{\"client\", \"server\"}, c.Transport.BandwidthLimitMode) {\n\t\treturn fmt.Errorf(\"bandwidth limit mode should be client or server\")\n\t}\n\n\tif c.Plugin.Type == \"\" {\n\t\tif err := ValidatePort(c.LocalPort, \"localPort\"); err != nil {\n\t\t\treturn fmt.Errorf(\"localPort: %v\", err)\n\t\t}\n\t}\n\n\tif !slices.Contains([]string{\"\", \"tcp\", \"http\"}, c.HealthCheck.Type) {\n\t\treturn fmt.Errorf(\"not support health check type: %s\", c.HealthCheck.Type)\n\t}\n\tif c.HealthCheck.Type != \"\" {\n\t\tif c.HealthCheck.Type == \"http\" &&\n\t\t\tc.HealthCheck.Path == \"\" {\n\t\t\treturn fmt.Errorf(\"health check path should not be empty\")\n\t\t}\n\t}\n\n\tif c.Plugin.Type != \"\" {\n\t\tif err := ValidateClientPluginOptions(c.Plugin.ClientPluginOptions); err != nil {\n\t\t\treturn fmt.Errorf(\"plugin %s: %v\", c.Plugin.Type, err)\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc validateProxyBaseConfigForServer(c *v1.ProxyBaseConfig) error {\n\tif err := ValidateAnnotations(c.Annotations); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc validateDomainConfigForClient(c *v1.DomainConfig) error {\n\tif c.SubDomain == \"\" && len(c.CustomDomains) == 0 {\n\t\treturn errors.New(\"subdomain and custom domains should not be both empty\")\n\t}\n\treturn nil\n}\n\nfunc validateDomainConfigForServer(c *v1.DomainConfig, s *v1.ServerConfig) error {\n\tfor _, domain := range c.CustomDomains {\n\t\tif s.SubDomainHost != \"\" && len(strings.Split(s.SubDomainHost, \".\")) < len(strings.Split(domain, \".\")) {\n\t\t\tif strings.Contains(domain, s.SubDomainHost) {\n\t\t\t\treturn fmt.Errorf(\"custom domain [%s] should not belong to subdomain host [%s]\", domain, s.SubDomainHost)\n\t\t\t}\n\t\t}\n\t}\n\n\tif c.SubDomain != \"\" {\n\t\tif s.SubDomainHost == \"\" {\n\t\t\treturn errors.New(\"subdomain is not supported because this feature is not enabled in server\")\n\t\t}\n\n\t\tif strings.Contains(c.SubDomain, \".\") || strings.Contains(c.SubDomain, \"*\") {\n\t\t\treturn errors.New(\"'.' and '*' are not supported in subdomain\")\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc ValidateProxyConfigurerForClient(c v1.ProxyConfigurer) error {\n\tbase := c.GetBaseConfig()\n\tif err := validateProxyBaseConfigForClient(base); err != nil {\n\t\treturn err\n\t}\n\n\tswitch v := c.(type) {\n\tcase *v1.TCPProxyConfig:\n\t\treturn validateTCPProxyConfigForClient(v)\n\tcase *v1.UDPProxyConfig:\n\t\treturn validateUDPProxyConfigForClient(v)\n\tcase *v1.TCPMuxProxyConfig:\n\t\treturn validateTCPMuxProxyConfigForClient(v)\n\tcase *v1.HTTPProxyConfig:\n\t\treturn validateHTTPProxyConfigForClient(v)\n\tcase *v1.HTTPSProxyConfig:\n\t\treturn validateHTTPSProxyConfigForClient(v)\n\tcase *v1.STCPProxyConfig:\n\t\treturn validateSTCPProxyConfigForClient(v)\n\tcase *v1.XTCPProxyConfig:\n\t\treturn validateXTCPProxyConfigForClient(v)\n\tcase *v1.SUDPProxyConfig:\n\t\treturn validateSUDPProxyConfigForClient(v)\n\t}\n\treturn errors.New(\"unknown proxy config type\")\n}\n\nfunc validateTCPProxyConfigForClient(c *v1.TCPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateUDPProxyConfigForClient(c *v1.UDPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateTCPMuxProxyConfigForClient(c *v1.TCPMuxProxyConfig) error {\n\tif err := validateDomainConfigForClient(&c.DomainConfig); err != nil {\n\t\treturn err\n\t}\n\n\tif !slices.Contains([]string{string(v1.TCPMultiplexerHTTPConnect)}, c.Multiplexer) {\n\t\treturn fmt.Errorf(\"not support multiplexer: %s\", c.Multiplexer)\n\t}\n\treturn nil\n}\n\nfunc validateHTTPProxyConfigForClient(c *v1.HTTPProxyConfig) error {\n\treturn validateDomainConfigForClient(&c.DomainConfig)\n}\n\nfunc validateHTTPSProxyConfigForClient(c *v1.HTTPSProxyConfig) error {\n\treturn validateDomainConfigForClient(&c.DomainConfig)\n}\n\nfunc validateSTCPProxyConfigForClient(c *v1.STCPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateXTCPProxyConfigForClient(c *v1.XTCPProxyConfig) error {\n\treturn nil\n}\n\nfunc validateSUDPProxyConfigForClient(c *v1.SUDPProxyConfig) error {\n\treturn nil\n}\n\nfunc ValidateProxyConfigurerForServer(c v1.ProxyConfigurer, s *v1.ServerConfig) error {\n\tbase := c.GetBaseConfig()\n\tif err := validateProxyBaseConfigForServer(base); err != nil {\n\t\treturn err\n\t}\n\n\tswitch v := c.(type) {\n\tcase *v1.TCPProxyConfig:\n\t\treturn validateTCPProxyConfigForServer(v, s)\n\tcase *v1.UDPProxyConfig:\n\t\treturn validateUDPProxyConfigForServer(v, s)\n\tcase *v1.TCPMuxProxyConfig:\n\t\treturn validateTCPMuxProxyConfigForServer(v, s)\n\tcase *v1.HTTPProxyConfig:\n\t\treturn validateHTTPProxyConfigForServer(v, s)\n\tcase *v1.HTTPSProxyConfig:\n\t\treturn validateHTTPSProxyConfigForServer(v, s)\n\tcase *v1.STCPProxyConfig:\n\t\treturn validateSTCPProxyConfigForServer(v, s)\n\tcase *v1.XTCPProxyConfig:\n\t\treturn validateXTCPProxyConfigForServer(v, s)\n\tcase *v1.SUDPProxyConfig:\n\t\treturn validateSUDPProxyConfigForServer(v, s)\n\tdefault:\n\t\treturn errors.New(\"unknown proxy config type\")\n\t}\n}\n\nfunc validateTCPProxyConfigForServer(c *v1.TCPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateUDPProxyConfigForServer(c *v1.UDPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateTCPMuxProxyConfigForServer(c *v1.TCPMuxProxyConfig, s *v1.ServerConfig) error {\n\tif c.Multiplexer == string(v1.TCPMultiplexerHTTPConnect) &&\n\t\ts.TCPMuxHTTPConnectPort == 0 {\n\t\treturn fmt.Errorf(\"tcpmux with multiplexer httpconnect not supported because this feature is not enabled in server\")\n\t}\n\n\treturn validateDomainConfigForServer(&c.DomainConfig, s)\n}\n\nfunc validateHTTPProxyConfigForServer(c *v1.HTTPProxyConfig, s *v1.ServerConfig) error {\n\tif s.VhostHTTPPort == 0 {\n\t\treturn fmt.Errorf(\"type [http] not supported when vhost http port is not set\")\n\t}\n\n\treturn validateDomainConfigForServer(&c.DomainConfig, s)\n}\n\nfunc validateHTTPSProxyConfigForServer(c *v1.HTTPSProxyConfig, s *v1.ServerConfig) error {\n\tif s.VhostHTTPSPort == 0 {\n\t\treturn fmt.Errorf(\"type [https] not supported when vhost https port is not set\")\n\t}\n\n\treturn validateDomainConfigForServer(&c.DomainConfig, s)\n}\n\nfunc validateSTCPProxyConfigForServer(c *v1.STCPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateXTCPProxyConfigForServer(c *v1.XTCPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\nfunc validateSUDPProxyConfigForServer(c *v1.SUDPProxyConfig, s *v1.ServerConfig) error {\n\treturn nil\n}\n\n// ValidateAnnotations validates that a set of annotations are correctly defined.\nfunc ValidateAnnotations(annotations map[string]string) error {\n\tif len(annotations) == 0 {\n\t\treturn nil\n\t}\n\n\tvar errs error\n\tfor k := range annotations {\n\t\tfor _, msg := range validation.IsQualifiedName(strings.ToLower(k)) {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"annotation key %s is invalid: %s\", k, msg))\n\t\t}\n\t}\n\tif err := ValidateAnnotationsSize(annotations); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\treturn errs\n}\n\nconst TotalAnnotationSizeLimitB int = 256 * (1 << 10) // 256 kB\n\nfunc ValidateAnnotationsSize(annotations map[string]string) error {\n\tvar totalSize int64\n\tfor k, v := range annotations {\n\t\ttotalSize += (int64)(len(k)) + (int64)(len(v))\n\t}\n\tif totalSize > (int64)(TotalAnnotationSizeLimitB) {\n\t\treturn fmt.Errorf(\"annotations size %d is larger than limit %d\", totalSize, TotalAnnotationSizeLimitB)\n\t}\n\treturn nil\n}\n\n// Path: pkg/config/v1/validation/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"fmt\"\n\t\"slices\"\n\n\t\"github.com/samber/lo\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\n\nfunc ValidateServerConfig(c *v1.ServerConfig) (Warning, error) {\n\tvar (\n\t\twarnings Warning\n\t\terrs     error\n\t)\n\tif !slices.Contains(SupportedAuthMethods, c.Auth.Method) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth method, optional values are %v\", SupportedAuthMethods))\n\t}\n\tif !lo.Every(SupportedAuthAdditionalScopes, c.Auth.AdditionalScopes) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth additional scopes, optional values are %v\", SupportedAuthAdditionalScopes))\n\t}\n\n\tif err := validateLogConfig(&c.Log); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\tif err := validateWebServerConfig(&c.WebServer); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\terrs = AppendError(errs, ValidatePort(c.BindPort, \"bindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.KCPBindPort, \"kcpBindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.QUICBindPort, \"quicBindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.VhostHTTPPort, \"vhostHTTPPort\"))\n\terrs = AppendError(errs, ValidatePort(c.VhostHTTPSPort, \"vhostHTTPSPort\"))\n\terrs = AppendError(errs, ValidatePort(c.TCPMuxHTTPConnectPort, \"tcpMuxHTTPConnectPort\"))\n\n\tfor _, p := range c.HTTPPlugins {\n\t\tif !lo.Every(SupportedHTTPPluginOps, p.Ops) {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"invalid http plugin ops, optional values are %v\", SupportedHTTPPluginOps))\n\t\t}\n\t}\n\treturn warnings, errs\n}\n\n// Path: pkg/config/v1/validation/visitor.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage validation\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"slices\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateVisitorConfigurer(c v1.VisitorConfigurer) error {\n\tbase := c.GetBaseConfig()\n\tif err := validateVisitorBaseConfig(base); err != nil {\n\t\treturn err\n\t}\n\n\tswitch v := c.(type) {\n\tcase *v1.STCPVisitorConfig:\n\tcase *v1.SUDPVisitorConfig:\n\tcase *v1.XTCPVisitorConfig:\n\t\treturn validateXTCPVisitorConfig(v)\n\tdefault:\n\t\treturn errors.New(\"unknown visitor config type\")\n\t}\n\treturn nil\n}\n\nfunc validateVisitorBaseConfig(c *v1.VisitorBaseConfig) error {\n\tif c.Name == \"\" {\n\t\treturn errors.New(\"name is required\")\n\t}\n\n\tif c.ServerName == \"\" {\n\t\treturn errors.New(\"server name is required\")\n\t}\n\n\tif c.BindPort == 0 {\n\t\treturn errors.New(\"bind port is required\")\n\t}\n\treturn nil\n}\n\nfunc validateXTCPVisitorConfig(c *v1.XTCPVisitorConfig) error {\n\tif !slices.Contains([]string{\"kcp\", \"quic\"}, c.Protocol) {\n\t\treturn fmt.Errorf(\"protocol should be kcp or quic\")\n\t}\n\treturn nil\n}\n\n// Path: pkg/config/v1/visitor.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage v1\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"github.com/samber/lo\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype VisitorTransport struct {\n\tUseEncryption  bool `json:\"useEncryption,omitempty\"`\n\tUseCompression bool `json:\"useCompression,omitempty\"`\n}\n\ntype VisitorBaseConfig struct {\n\tName      string           `json:\"name\"`\n\tType      string           `json:\"type\"`\n\tTransport VisitorTransport `json:\"transport,omitempty\"`\n\tSecretKey string           `json:\"secretKey,omitempty\"`\n\t// if the server user is not set, it defaults to the current user\n\tServerUser string `json:\"serverUser,omitempty\"`\n\tServerName string `json:\"serverName,omitempty\"`\n\tBindAddr   string `json:\"bindAddr,omitempty\"`\n\t// BindPort is the port that visitor listens on.\n\t// It can be less than 0, it means don't bind to the port and only receive connections redirected from\n\t// other visitors. (This is not supported for SUDP now)\n\tBindPort int `json:\"bindPort,omitempty\"`\n}\n\nfunc (c *VisitorBaseConfig) GetBaseConfig() *VisitorBaseConfig {\n\treturn c\n}\n\nfunc (c *VisitorBaseConfig) Complete(g *ClientCommonConfig) {\n\tif c.BindAddr == \"\" {\n\t\tc.BindAddr = \"127.0.0.1\"\n\t}\n\n\tnamePrefix := \"\"\n\tif g.User != \"\" {\n\t\tnamePrefix = g.User + \".\"\n\t}\n\tc.Name = namePrefix + c.Name\n\n\tif c.ServerUser != \"\" {\n\t\tc.ServerName = c.ServerUser + \".\" + c.ServerName\n\t} else {\n\t\tc.ServerName = namePrefix + c.ServerName\n\t}\n}\n\ntype VisitorConfigurer interface {\n\tComplete(*ClientCommonConfig)\n\tGetBaseConfig() *VisitorBaseConfig\n}\n\ntype VisitorType string\n\nconst (\n\tVisitorTypeSTCP VisitorType = \"stcp\"\n\tVisitorTypeXTCP VisitorType = \"xtcp\"\n\tVisitorTypeSUDP VisitorType = \"sudp\"\n)\n\nvar visitorConfigTypeMap = map[VisitorType]reflect.Type{\n\tVisitorTypeSTCP: reflect.TypeOf(STCPVisitorConfig{}),\n\tVisitorTypeXTCP: reflect.TypeOf(XTCPVisitorConfig{}),\n\tVisitorTypeSUDP: reflect.TypeOf(SUDPVisitorConfig{}),\n}\n\ntype TypedVisitorConfig struct {\n\tType string `json:\"type\"`\n\tVisitorConfigurer\n}\n\nfunc (c *TypedVisitorConfig) UnmarshalJSON(b []byte) error {\n\tif len(b) == 4 && string(b) == \"null\" {\n\t\treturn errors.New(\"type is required\")\n\t}\n\n\ttypeStruct := struct {\n\t\tType string `json:\"type\"`\n\t}{}\n\tif err := json.Unmarshal(b, &typeStruct); err != nil {\n\t\treturn err\n\t}\n\n\tc.Type = typeStruct.Type\n\tconfigurer := NewVisitorConfigurerByType(VisitorType(typeStruct.Type))\n\tif configurer == nil {\n\t\treturn fmt.Errorf(\"unknown visitor type: %s\", typeStruct.Type)\n\t}\n\tdecoder := json.NewDecoder(bytes.NewBuffer(b))\n\tif DisallowUnknownFields {\n\t\tdecoder.DisallowUnknownFields()\n\t}\n\tif err := decoder.Decode(configurer); err != nil {\n\t\treturn fmt.Errorf(\"unmarshal VisitorConfig error: %v\", err)\n\t}\n\tc.VisitorConfigurer = configurer\n\treturn nil\n}\n\nfunc NewVisitorConfigurerByType(t VisitorType) VisitorConfigurer {\n\tv, ok := visitorConfigTypeMap[t]\n\tif !ok {\n\t\treturn nil\n\t}\n\tvc := reflect.New(v).Interface().(VisitorConfigurer)\n\tvc.GetBaseConfig().Type = string(t)\n\treturn vc\n}\n\nvar _ VisitorConfigurer = &STCPVisitorConfig{}\n\ntype STCPVisitorConfig struct {\n\tVisitorBaseConfig\n}\n\nvar _ VisitorConfigurer = &SUDPVisitorConfig{}\n\ntype SUDPVisitorConfig struct {\n\tVisitorBaseConfig\n}\n\nvar _ VisitorConfigurer = &XTCPVisitorConfig{}\n\ntype XTCPVisitorConfig struct {\n\tVisitorBaseConfig\n\n\tProtocol          string `json:\"protocol,omitempty\"`\n\tKeepTunnelOpen    bool   `json:\"keepTunnelOpen,omitempty\"`\n\tMaxRetriesAnHour  int    `json:\"maxRetriesAnHour,omitempty\"`\n\tMinRetryInterval  int    `json:\"minRetryInterval,omitempty\"`\n\tFallbackTo        string `json:\"fallbackTo,omitempty\"`\n\tFallbackTimeoutMs int    `json:\"fallbackTimeoutMs,omitempty\"`\n}\n\nfunc (c *XTCPVisitorConfig) Complete(g *ClientCommonConfig) {\n\tc.VisitorBaseConfig.Complete(g)\n\n\tc.Protocol = util.EmptyOr(c.Protocol, \"quic\")\n\tc.MaxRetriesAnHour = util.EmptyOr(c.MaxRetriesAnHour, 8)\n\tc.MinRetryInterval = util.EmptyOr(c.MinRetryInterval, 90)\n\tc.FallbackTimeoutMs = util.EmptyOr(c.FallbackTimeoutMs, 1000)\n\n\tif c.FallbackTo != \"\" {\n\t\tc.FallbackTo = lo.Ternary(g.User == \"\", \"\", g.User+\".\") + c.FallbackTo\n\t}\n}\n\n// Path: pkg/metrics/metrics.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"github.com/fatedier/frp/pkg/metrics/aggregate\"\n)\n\nvar (\n\tEnableMem        = aggregate.EnableMem\n\tEnablePrometheus = aggregate.EnablePrometheus\n)\n\n// Path: pkg/util/metric/metrics.go\n// Copyright 2020 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metric\n\n// GaugeMetric represents a single numerical value that can arbitrarily go up\n// and down.\ntype GaugeMetric interface {\n\tInc()\n\tDec()\n\tSet(float64)\n}\n\n// CounterMetric represents a single numerical value that only ever\n// goes up.\ntype CounterMetric interface {\n\tInc()\n}\n\n// HistogramMetric counts individual observations.\ntype HistogramMetric interface {\n\tObserve(float64)\n}\n\n// Path: pkg/metrics/aggregate/server.go\n// Copyright 2020 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage aggregate\n\nimport (\n\t\"github.com/fatedier/frp/pkg/metrics/mem\"\n\t\"github.com/fatedier/frp/pkg/metrics/prometheus\"\n\t\"github.com/fatedier/frp/server/metrics\"\n)\n\n// EnableMem start to mark metrics to memory monitor system.\nfunc EnableMem() {\n\tsm.Add(mem.ServerMetrics)\n}\n\n// EnablePrometheus start to mark metrics to prometheus.\nfunc EnablePrometheus() {\n\tsm.Add(prometheus.ServerMetrics)\n}\n\nvar sm = &serverMetrics{}\n\nfunc init() {\n\tmetrics.Register(sm)\n}\n\ntype serverMetrics struct {\n\tms []metrics.ServerMetrics\n}\n\nfunc (m *serverMetrics) Add(sm metrics.ServerMetrics) {\n\tm.ms = append(m.ms, sm)\n}\n\nfunc (m *serverMetrics) NewClient() {\n\tfor _, v := range m.ms {\n\t\tv.NewClient()\n\t}\n}\n\nfunc (m *serverMetrics) CloseClient() {\n\tfor _, v := range m.ms {\n\t\tv.CloseClient()\n\t}\n}\n\nfunc (m *serverMetrics) NewProxy(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.NewProxy(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) CloseProxy(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.CloseProxy(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) OpenConnection(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.OpenConnection(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) CloseConnection(name string, proxyType string) {\n\tfor _, v := range m.ms {\n\t\tv.CloseConnection(name, proxyType)\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficIn(name string, proxyType string, trafficBytes int64) {\n\tfor _, v := range m.ms {\n\t\tv.AddTrafficIn(name, proxyType, trafficBytes)\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficOut(name string, proxyType string, trafficBytes int64) {\n\tfor _, v := range m.ms {\n\t\tv.AddTrafficOut(name, proxyType, trafficBytes)\n\t}\n}\n\n// Path: pkg/util/limit/writer.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage limit\n\nimport (\n\t\"context\"\n\t\"io\"\n\n\t\"golang.org/x/time/rate\"\n)\n\ntype Writer struct {\n\tw       io.Writer\n\tlimiter *rate.Limiter\n}\n\nfunc NewWriter(w io.Writer, limiter *rate.Limiter) *Writer {\n\treturn &Writer{\n\t\tw:       w,\n\t\tlimiter: limiter,\n\t}\n}\n\nfunc (w *Writer) Write(p []byte) (n int, err error) {\n\tvar nn int\n\tb := w.limiter.Burst()\n\tfor {\n\t\tend := len(p)\n\t\tif end == 0 {\n\t\t\tbreak\n\t\t}\n\t\tif b < len(p) {\n\t\t\tend = b\n\t\t}\n\t\terr = w.limiter.WaitN(context.Background(), end)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tnn, err = w.w.Write(p[:end])\n\t\tn += nn\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tp = p[end:]\n\t}\n\treturn\n}\n\n// Path: pkg/util/log/log.go\n// Copyright 2016 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage log\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\n\t\"github.com/fatedier/golib/log\"\n)\n\nvar (\n\tTraceLevel = log.TraceLevel\n\tDebugLevel = log.DebugLevel\n\tInfoLevel  = log.InfoLevel\n\tWarnLevel  = log.WarnLevel\n\tErrorLevel = log.ErrorLevel\n)\n\nvar Logger *log.Logger\n\nfunc init() {\n\tLogger = log.New(\n\t\tlog.WithCaller(true),\n\t\tlog.AddCallerSkip(1),\n\t\tlog.WithLevel(log.InfoLevel),\n\t)\n}\n\nfunc InitLogger(logPath string, levelStr string, maxDays int, disableLogColor bool) {\n\toptions := []log.Option{}\n\tif logPath == \"console\" {\n\t\tif !disableLogColor {\n\t\t\toptions = append(options,\n\t\t\t\tlog.WithOutput(log.NewConsoleWriter(log.ConsoleConfig{\n\t\t\t\t\tColorful: true,\n\t\t\t\t}, os.Stdout)),\n\t\t\t)\n\t\t}\n\t} else {\n\t\twriter := log.NewRotateFileWriter(log.RotateFileConfig{\n\t\t\tFileName: logPath,\n\t\t\tMode:     log.RotateFileModeDaily,\n\t\t\tMaxDays:  maxDays,\n\t\t})\n\t\twriter.Init()\n\t\toptions = append(options, log.WithOutput(writer))\n\t}\n\n\tlevel, err := log.ParseLevel(levelStr)\n\tif err != nil {\n\t\tlevel = log.InfoLevel\n\t}\n\toptions = append(options, log.WithLevel(level))\n\tLogger = Logger.WithOptions(options...)\n}\n\nfunc Errorf(format string, v ...interface{}) {\n\tLogger.Errorf(format, v...)\n}\n\nfunc Warnf(format string, v ...interface{}) {\n\tLogger.Warnf(format, v...)\n}\n\nfunc Infof(format string, v ...interface{}) {\n\tLogger.Infof(format, v...)\n}\n\nfunc Debugf(format string, v ...interface{}) {\n\tLogger.Debugf(format, v...)\n}\n\nfunc Tracef(format string, v ...interface{}) {\n\tLogger.Tracef(format, v...)\n}\n\nfunc Logf(level log.Level, offset int, format string, v ...interface{}) {\n\tLogger.Logf(level, offset, format, v...)\n}\n\ntype WriteLogger struct {\n\tlevel  log.Level\n\toffset int\n}\n\nfunc NewWriteLogger(level log.Level, offset int) *WriteLogger {\n\treturn &WriteLogger{\n\t\tlevel:  level,\n\t\toffset: offset,\n\t}\n}\n\nfunc (w *WriteLogger) Write(p []byte) (n int, err error) {\n\tLogger.Log(w.level, w.offset, string(bytes.TrimRight(p, \"\\n\")))\n\treturn len(p), nil\n}\n\n// Path: pkg/util/xlog/xlog.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n...\n", "needle_token_start": 9776, "needle_token_end": 10274, "code_context_ntokens": 15411, "output": ["```go\npackage validation\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"slices\"\n\n\t\"github.com/samber/lo\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc ValidateServerConfig(c *v1.ServerConfig) (Warning, error) {\n\tvar (\n\t\twarnings Warning\n\t\terrs     error\n\t)\n\tif !slices.Contains(SupportedAuthMethods, c.Auth.Method) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth method, optional values are %v\", SupportedAuthMethods))\n\t}\n\tif !lo.Every(SupportedAuthAdditionalScopes, c.Auth.AdditionalScopes) {\n\t\terrs = AppendError(errs, fmt.Errorf(\"invalid auth additional scopes, optional values are %v\", SupportedAuthAdditionalScopes))\n\t}\n\n\tif err := validateLogConfig(&c.Log); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\tif err := validateWebServerConfig(&c.WebServer); err != nil {\n\t\terrs = AppendError(errs, err)\n\t}\n\n\terrs = AppendError(errs, ValidatePort(c.BindPort, \"bindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.KCPBindPort, \"kcpBindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.QUICBindPort, \"quicBindPort\"))\n\terrs = AppendError(errs, ValidatePort(c.VhostHTTPPort, \"vhostHTTPPort\"))\n\terrs = AppendError(errs, ValidatePort(c.VhostHTTPSPort, \"vhostHTTPSPort\"))\n\terrs = AppendError(errs, ValidatePort(c.TCPMuxHTTPConnectPort, \"tcpMuxHTTPConnectPort\"))\n\n\tfor _, p := range c.HTTPPlugins {\n\t\tif !lo.Every(SupportedHTTPPluginOps, p.Ops) {\n\t\t\terrs = AppendError(errs, fmt.Errorf(\"invalid http plugin ops, optional values are %v\", SupportedHTTPPluginOps))\n\t\t}\n\t}\n\treturn warnings, errs\n}\n```"]}
{"repo": "fatedier/frp", "name": "LoadServerConfig", "language": "go", "path": "pkg/config/load.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To load and parse the server configuration from a specified file, handling both legacy and current formats, and ensuring the configuration adheres to the expected structure.\n2. **Input**: A file path as a string and a boolean indicating whether to enforce strict parsing rules.\n3. **Output**: Returns a server configuration object, a boolean indicating if the legacy format was used, and an error object if an issue occurred during the process.\n4. **Procedure**: \n   - First, the function checks if the file is in a legacy format.\n   - If it is in the legacy format, the file is read and parsed according to the legacy specifications, then converted to the current configuration format.\n   - If it is not in the legacy format, the file is read and parsed using a template rendering method that injects runtime values into the configuration.\n   - The parsed configuration is then validated and completed, ensuring all necessary fields are properly set.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/config/legacy/proxy.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"gopkg.in/ini.v1\"\n\n\t\"github.com/fatedier/frp/pkg/config/types\"\n)\n\ntype ProxyType string\n\nconst (\n\tProxyTypeTCP    ProxyType = \"tcp\"\n\tProxyTypeUDP    ProxyType = \"udp\"\n\tProxyTypeTCPMUX ProxyType = \"tcpmux\"\n\tProxyTypeHTTP   ProxyType = \"http\"\n\tProxyTypeHTTPS  ProxyType = \"https\"\n\tProxyTypeSTCP   ProxyType = \"stcp\"\n\tProxyTypeXTCP   ProxyType = \"xtcp\"\n\tProxyTypeSUDP   ProxyType = \"sudp\"\n)\n\n// Proxy\nvar (\n\tproxyConfTypeMap = map[ProxyType]reflect.Type{\n\t\tProxyTypeTCP:    reflect.TypeOf(TCPProxyConf{}),\n\t\tProxyTypeUDP:    reflect.TypeOf(UDPProxyConf{}),\n\t\tProxyTypeTCPMUX: reflect.TypeOf(TCPMuxProxyConf{}),\n\t\tProxyTypeHTTP:   reflect.TypeOf(HTTPProxyConf{}),\n\t\tProxyTypeHTTPS:  reflect.TypeOf(HTTPSProxyConf{}),\n\t\tProxyTypeSTCP:   reflect.TypeOf(STCPProxyConf{}),\n\t\tProxyTypeXTCP:   reflect.TypeOf(XTCPProxyConf{}),\n\t\tProxyTypeSUDP:   reflect.TypeOf(SUDPProxyConf{}),\n\t}\n)\n\ntype ProxyConf interface {\n\t// GetBaseConfig returns the BaseProxyConf for this config.\n\tGetBaseConfig() *BaseProxyConf\n\t// UnmarshalFromIni unmarshals a ini.Section into this config. This function\n\t// will be called on the frpc side.\n\tUnmarshalFromIni(string, string, *ini.Section) error\n}\n\nfunc NewConfByType(proxyType ProxyType) ProxyConf {\n\tv, ok := proxyConfTypeMap[proxyType]\n\tif !ok {\n\t\treturn nil\n\t}\n\tcfg := reflect.New(v).Interface().(ProxyConf)\n\treturn cfg\n}\n\n// Proxy Conf Loader\n// DefaultProxyConf creates a empty ProxyConf object by proxyType.\n// If proxyType doesn't exist, return nil.\nfunc DefaultProxyConf(proxyType ProxyType) ProxyConf {\n\treturn NewConfByType(proxyType)\n}\n\n// Proxy loaded from ini\nfunc NewProxyConfFromIni(prefix, name string, section *ini.Section) (ProxyConf, error) {\n\t// section.Key: if key not exists, section will set it with default value.\n\tproxyType := ProxyType(section.Key(\"type\").String())\n\tif proxyType == \"\" {\n\t\tproxyType = ProxyTypeTCP\n\t}\n\n\tconf := DefaultProxyConf(proxyType)\n\tif conf == nil {\n\t\treturn nil, fmt.Errorf(\"invalid type [%s]\", proxyType)\n\t}\n\n\tif err := conf.UnmarshalFromIni(prefix, name, section); err != nil {\n\t\treturn nil, err\n\t}\n\treturn conf, nil\n}\n\n// LocalSvrConf configures what location the client will to, or what\n// plugin will be used.\ntype LocalSvrConf struct {\n\t// LocalIP specifies the IP address or host name to to.\n\tLocalIP string `ini:\"local_ip\" json:\"local_ip\"`\n\t// LocalPort specifies the port to to.\n\tLocalPort int `ini:\"local_port\" json:\"local_port\"`\n\n\t// Plugin specifies what plugin should be used for ng. If this value\n\t// is set, the LocalIp and LocalPort values will be ignored. By default,\n\t// this value is \"\".\n\tPlugin string `ini:\"plugin\" json:\"plugin\"`\n\t// PluginParams specify parameters to be passed to the plugin, if one is\n\t// being used. By default, this value is an empty map.\n\tPluginParams map[string]string `ini:\"-\"`\n}\n\n// HealthCheckConf configures health checking. This can be useful for load\n// balancing purposes to detect and remove proxies to failing services.\ntype HealthCheckConf struct {\n\t// HealthCheckType specifies what protocol to use for health checking.\n\t// Valid values include \"tcp\", \"http\", and \"\". If this value is \"\", health\n\t// checking will not be performed. By default, this value is \"\".\n\t//\n\t// If the type is \"tcp\", a connection will be attempted to the target\n\t// server. If a connection cannot be established, the health check fails.\n\t//\n\t// If the type is \"http\", a GET request will be made to the endpoint\n\t// specified by HealthCheckURL. If the response is not a 200, the health\n\t// check fails.\n\tHealthCheckType string `ini:\"health_check_type\" json:\"health_check_type\"` // tcp | http\n\t// HealthCheckTimeoutS specifies the number of seconds to wait for a health\n\t// check attempt to connect. If the timeout is reached, this counts as a\n\t// health check failure. By default, this value is 3.\n\tHealthCheckTimeoutS int `ini:\"health_check_timeout_s\" json:\"health_check_timeout_s\"`\n\t// HealthCheckMaxFailed specifies the number of allowed failures before the\n\t// is stopped. By default, this value is 1.\n\tHealthCheckMaxFailed int `ini:\"health_check_max_failed\" json:\"health_check_max_failed\"`\n\t// HealthCheckIntervalS specifies the time in seconds between health\n\t// checks. By default, this value is 10.\n\tHealthCheckIntervalS int `ini:\"health_check_interval_s\" json:\"health_check_interval_s\"`\n\t// HealthCheckURL specifies the address to send health checks to if the\n\t// health check type is \"http\".\n\tHealthCheckURL string `ini:\"health_check_url\" json:\"health_check_url\"`\n\t// HealthCheckAddr specifies the address to connect to if the health check\n\t// type is \"tcp\".\n\tHealthCheckAddr string `ini:\"-\"`\n}\n\n// BaseProxyConf provides configuration info that is common to all types.\ntype BaseProxyConf struct {\n\t// ProxyName is the name of this\n\tProxyName string `ini:\"name\" json:\"name\"`\n\t// ProxyType specifies the type of this  Valid values include \"tcp\",\n\t// \"udp\", \"http\", \"https\", \"stcp\", and \"xtcp\". By default, this value is\n\t// \"tcp\".\n\tProxyType string `ini:\"type\" json:\"type\"`\n\n\t// UseEncryption controls whether or not communication with the server will\n\t// be encrypted. Encryption is done using the tokens supplied in the server\n\t// and client configuration. By default, this value is false.\n\tUseEncryption bool `ini:\"use_encryption\" json:\"use_encryption\"`\n\t// UseCompression controls whether or not communication with the server\n\t// will be compressed. By default, this value is false.\n\tUseCompression bool `ini:\"use_compression\" json:\"use_compression\"`\n...\n// Path: pkg/config/legacy/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"strings\"\n\n\t\"gopkg.in/ini.v1\"\n\n\tlegacyauth \"github.com/fatedier/frp/pkg/auth/legacy\"\n)\n\ntype HTTPPluginOptions struct {\n\tName      string   `ini:\"name\"`\n\tAddr      string   `ini:\"addr\"`\n\tPath      string   `ini:\"path\"`\n\tOps       []string `ini:\"ops\"`\n\tTLSVerify bool     `ini:\"tlsVerify\"`\n}\n\n// ServerCommonConf contains information for a server service. It is\n// recommended to use GetDefaultServerConf instead of creating this object\n// directly, so that all unspecified fields have reasonable default values.\ntype ServerCommonConf struct {\n\tlegacyauth.ServerConfig `ini:\",extends\"`\n\n\t// BindAddr specifies the address that the server binds to. By default,\n\t// this value is \"0.0.0.0\".\n\tBindAddr string `ini:\"bind_addr\" json:\"bind_addr\"`\n\t// BindPort specifies the port that the server listens on. By default, this\n\t// value is 7000.\n\tBindPort int `ini:\"bind_port\" json:\"bind_port\"`\n\t// KCPBindPort specifies the KCP port that the server listens on. If this\n\t// value is 0, the server will not listen for KCP connections. By default,\n\t// this value is 0.\n\tKCPBindPort int `ini:\"kcp_bind_port\" json:\"kcp_bind_port\"`\n\t// QUICBindPort specifies the QUIC port that the server listens on.\n\t// Set this value to 0 will disable this feature.\n\t// By default, the value is 0.\n\tQUICBindPort int `ini:\"quic_bind_port\" json:\"quic_bind_port\"`\n\t// QUIC protocol options\n\tQUICKeepalivePeriod    int `ini:\"quic_keepalive_period\" json:\"quic_keepalive_period\"`\n\tQUICMaxIdleTimeout     int `ini:\"quic_max_idle_timeout\" json:\"quic_max_idle_timeout\"`\n\tQUICMaxIncomingStreams int `ini:\"quic_max_incoming_streams\" json:\"quic_max_incoming_streams\"`\n\t// ProxyBindAddr specifies the address that the proxy binds to. This value\n\t// may be the same as BindAddr.\n\tProxyBindAddr string `ini:\"proxy_bind_addr\" json:\"proxy_bind_addr\"`\n\t// VhostHTTPPort specifies the port that the server listens for HTTP Vhost\n\t// requests. If this value is 0, the server will not listen for HTTP\n\t// requests. By default, this value is 0.\n\tVhostHTTPPort int `ini:\"vhost_http_port\" json:\"vhost_http_port\"`\n\t// VhostHTTPSPort specifies the port that the server listens for HTTPS\n\t// Vhost requests. If this value is 0, the server will not listen for HTTPS\n\t// requests. By default, this value is 0.\n\tVhostHTTPSPort int `ini:\"vhost_https_port\" json:\"vhost_https_port\"`\n\t// TCPMuxHTTPConnectPort specifies the port that the server listens for TCP\n\t// HTTP CONNECT requests. If the value is 0, the server will not multiplex TCP\n\t// requests on one single port. If it's not - it will listen on this value for\n\t// HTTP CONNECT requests. By default, this value is 0.\n\tTCPMuxHTTPConnectPort int `ini:\"tcpmux_httpconnect_port\" json:\"tcpmux_httpconnect_port\"`\n\t// If TCPMuxPassthrough is true, frps won't do any update on traffic.\n\tTCPMuxPassthrough bool `ini:\"tcpmux_passthrough\" json:\"tcpmux_passthrough\"`\n\t// VhostHTTPTimeout specifies the response header timeout for the Vhost\n\t// HTTP server, in seconds. By default, this value is 60.\n\tVhostHTTPTimeout int64 `ini:\"vhost_http_timeout\" json:\"vhost_http_timeout\"`\n\t// DashboardAddr specifies the address that the dashboard binds to. By\n\t// default, this value is \"0.0.0.0\".\n\tDashboardAddr string `ini:\"dashboard_addr\" json:\"dashboard_addr\"`\n\t// DashboardPort specifies the port that the dashboard listens on. If this\n\t// value is 0, the dashboard will not be started. By default, this value is\n\t// 0.\n\tDashboardPort int `ini:\"dashboard_port\" json:\"dashboard_port\"`\n\t// DashboardTLSCertFile specifies the path of the cert file that the server will\n\t// load. If \"dashboard_tls_cert_file\", \"dashboard_tls_key_file\" are valid, the server will use this\n\t// supplied tls configuration.\n\tDashboardTLSCertFile string `ini:\"dashboard_tls_cert_file\" json:\"dashboard_tls_cert_file\"`\n\t// DashboardTLSKeyFile specifies the path of the secret key that the server will\n\t// load. If \"dashboard_tls_cert_file\", \"dashboard_tls_key_file\" are valid, the server will use this\n\t// supplied tls configuration.\n\tDashboardTLSKeyFile string `ini:\"dashboard_tls_key_file\" json:\"dashboard_tls_key_file\"`\n\t// DashboardTLSMode specifies the mode of the dashboard between HTTP or HTTPS modes. By\n\t// default, this value is false, which is HTTP mode.\n\tDashboardTLSMode bool `ini:\"dashboard_tls_mode\" json:\"dashboard_tls_mode\"`\n\t// DashboardUser specifies the username that the dashboard will use for\n\t// login.\n\tDashboardUser string `ini:\"dashboard_user\" json:\"dashboard_user\"`\n\t// DashboardPwd specifies the password that the dashboard will use for\n\t// login.\n\tDashboardPwd string `ini:\"dashboard_pwd\" json:\"dashboard_pwd\"`\n\t// EnablePrometheus will export prometheus metrics on {dashboard_addr}:{dashboard_port}\n\t// in /metrics api.\n\tEnablePrometheus bool `ini:\"enable_prometheus\" json:\"enable_prometheus\"`\n\t// AssetsDir specifies the local directory that the dashboard will load\n\t// resources from. If this value is \"\", assets will be loaded from the\n\t// bundled executable using statik. By default, this value is \"\".\n\tAssetsDir string `ini:\"assets_dir\" json:\"assets_dir\"`\n\t// LogFile specifies a file where logs will be written to. This value will\n\t// only be used if LogWay is set appropriately. By default, this value is\n\t// \"console\".\n\tLogFile string `ini:\"log_file\" json:\"log_file\"`\n\t// LogWay specifies the way logging is managed. Valid values are \"console\"\n\t// or \"file\". If \"console\" is used, logs will be printed to stdout. If\n\t// \"file\" is used, logs will be printed to LogFile. By default, this value\n\t// is \"console\".\n\tLogWay string `ini:\"log_way\" json:\"log_way\"`\n\t// LogLevel specifies the minimum log level. Valid values are \"trace\",\n\t// \"debug\", \"info\", \"warn\", and \"error\". By default, this value is \"info\".\n\tLogLevel string `ini:\"log_level\" json:\"log_level\"`\n\t// LogMaxDays specifies the maximum number of days to store log information\n\t// before deletion. This is only used if LogWay == \"file\". By default, this\n\t// value is 0.\n\tLogMaxDays int64 `ini:\"log_max_days\" json:\"log_max_days\"`\n\t// DisableLogColor disables log colors when LogWay == \"console\" when set to\n\t// true. By default, this value is false.\n\tDisableLogColor bool `ini:\"disable_log_color\" json:\"disable_log_color\"`\n\t// DetailedErrorsToClient defines whether to send the specific error (with\n\t// debug info) to frpc. By default, this value is true.\n\tDetailedErrorsToClient bool `ini:\"detailed_errors_to_client\" json:\"detailed_errors_to_client\"`\n\n\t// SubDomainHost specifies the domain that will be attached to sub-domains\n\t// requested by the client when using Vhost proxying. For example, if this\n\t// value is set to \"frps.com\" and the client requested the subdomain\n\t// \"test\", the resulting URL would be \"test.frps.com\". By default, this\n\t// value is \"\".\n\tSubDomainHost string `ini:\"subdomain_host\" json:\"subdomain_host\"`\n\t// TCPMux toggles TCP stream multiplexing. This allows multiple requests\n\t// from a client to share a single TCP connection. By default, this value\n\t// is true.\n\tTCPMux bool `ini:\"tcp_mux\" json:\"tcp_mux\"`\n\t// TCPMuxKeepaliveInterval specifies the keep alive interval for TCP stream multiplier.\n\t// If TCPMux is true, heartbeat of application layer is unnecessary because it can only rely on heartbeat in TCPMux.\n\tTCPMuxKeepaliveInterval int64 `ini:\"tcp_mux_keepalive_interval\" json:\"tcp_mux_keepalive_interval\"`\n\t// TCPKeepAlive specifies the interval between keep-alive probes for an active network connection between frpc and frps.\n\t// If negative, keep-alive probes are disabled.\n\tTCPKeepAlive int64 `ini:\"tcp_keepalive\" json:\"tcp_keepalive\"`\n\t// Custom404Page specifies a path to a custom 404 page to display. If this\n\t// value is \"\", a default page will be displayed. By default, this value is\n\t// \"\".\n\tCustom404Page string `ini:\"custom_404_page\" json:\"custom_404_page\"`\n\n\t// AllowPorts specifies a set of ports that clients are able to proxy to.\n\t// If the length of this value is 0, all ports are allowed. By default,\n\t// this value is an empty set.\n\tAllowPorts map[int]struct{} `ini:\"-\" json:\"-\"`\n\t// Original string.\n\tAllowPortsStr string `ini:\"-\" json:\"-\"`\n\t// MaxPoolCount specifies the maximum pool size for each proxy. By default,\n\t// this value is 5.\n\tMaxPoolCount int64 `ini:\"max_pool_count\" json:\"max_pool_count\"`\n\t// MaxPortsPerClient specifies the maximum number of ports a single client\n\t// may proxy to. If this value is 0, no limit will be applied. By default,\n\t// this value is 0.\n\tMaxPortsPerClient int64 `ini:\"max_ports_per_client\" json:\"max_ports_per_client\"`\n\t// TLSOnly specifies whether to only accept TLS-encrypted connections.\n\t// By default, the value is false.\n\tTLSOnly bool `ini:\"tls_only\" json:\"tls_only\"`\n\t// TLSCertFile specifies the path of the cert file that the server will\n\t// load. If \"tls_cert_file\", \"tls_key_file\" are valid, the server will use this\n\t// supplied tls configuration. Otherwise, the server will use the tls\n\t// configuration generated by itself.\n\tTLSCertFile string `ini:\"tls_cert_file\" json:\"tls_cert_file\"`\n\t// TLSKeyFile specifies the path of the secret key that the server will\n\t// load. If \"tls_cert_file\", \"tls_key_file\" are valid, the server will use this\n\t// supplied tls configuration. Otherwise, the server will use the tls\n\t// configuration generated by itself.\n\tTLSKeyFile string `ini:\"tls_key_file\" json:\"tls_key_file\"`\n\t// TLSTrustedCaFile specifies the paths of the client cert files that the\n\t// server will load. It only works when \"tls_only\" is true. If\n\t// \"tls_trusted_ca_file\" is valid, the server will verify each client's\n\t// certificate.\n\tTLSTrustedCaFile string `ini:\"tls_trusted_ca_file\" json:\"tls_trusted_ca_file\"`\n\t// HeartBeatTimeout specifies the maximum time to wait for a heartbeat\n\t// before terminating the connection. It is not recommended to change this\n\t// value. By default, this value is 90. Set negative value to disable it.\n\tHeartbeatTimeout int64 `ini:\"heartbeat_timeout\" json:\"heartbeat_timeout\"`\n\t// UserConnTimeout specifies the maximum time to wait for a work\n\t// connection. By default, this value is 10.\n\tUserConnTimeout int64 `ini:\"user_conn_timeout\" json:\"user_conn_timeout\"`\n\t// HTTPPlugins specify the server plugins support HTTP protocol.\n\tHTTPPlugins map[string]HTTPPluginOptions `ini:\"-\" json:\"http_plugins\"`\n\t// UDPPacketSize specifies the UDP packet size\n\t// By default, this value is 1500\n\tUDPPacketSize int64 `ini:\"udp_packet_size\" json:\"udp_packet_size\"`\n\t// Enable golang pprof handlers in dashboard listener.\n\t// Dashboard port must be set first.\n\tPprofEnable bool `ini:\"pprof_enable\" json:\"pprof_enable\"`\n\t// NatHoleAnalysisDataReserveHours specifies the hours to reserve nat hole analysis data.\n\tNatHoleAnalysisDataReserveHours int64 `ini:\"nat_hole_analysis_data_reserve_hours\" json:\"nat_hole_analysis_data_reserve_hours\"`\n}\n\n// GetDefaultServerConf returns a server configuration with reasonable\n// defaults.\nfunc GetDefaultServerConf() ServerCommonConf {\n\treturn ServerCommonConf{\n\t\tServerConfig:                    legacyauth.GetDefaultServerConf(),\n\t\tBindAddr:                        \"0.0.0.0\",\n\t\tBindPort:                        7000,\n\t\tQUICKeepalivePeriod:             10,\n\t\tQUICMaxIdleTimeout:              30,\n\t\tQUICMaxIncomingStreams:          100000,\n\t\tVhostHTTPTimeout:                60,\n\t\tDashboardAddr:                   \"0.0.0.0\",\n\t\tLogFile:                         \"console\",\n\t\tLogWay:                          \"console\",\n\t\tLogLevel:                        \"info\",\n\t\tLogMaxDays:                      3,\n\t\tDetailedErrorsToClient:          true,\n\t\tTCPMux:                          true,\n\t\tTCPMuxKeepaliveInterval:         60,\n\t\tTCPKeepAlive:                    7200,\n\t\tAllowPorts:                      make(map[int]struct{}),\n\t\tMaxPoolCount:                    5,\n\t\tMaxPortsPerClient:               0,\n\t\tHeartbeatTimeout:                90,\n\t\tUserConnTimeout:                 10,\n\t\tHTTPPlugins:                     make(map[string]HTTPPluginOptions),\n\t\tUDPPacketSize:                   1500,\n\t\tNatHoleAnalysisDataReserveHours: 7 * 24,\n\t}\n}\n\nfunc UnmarshalServerConfFromIni(source interface{}) (ServerCommonConf, error) {\n\tf, err := ini.LoadSources(ini.LoadOptions{\n\t\tInsensitive:         false,\n\t\tInsensitiveSections: false,\n\t\tInsensitiveKeys:     false,\n\t\tIgnoreInlineComment: true,\n\t\tAllowBooleanKeys:    true,\n\t}, source)\n\tif err != nil {\n\t\treturn ServerCommonConf{}, err\n\t}\n\n\ts, err := f.GetSection(\"common\")\n\tif err != nil {\n\t\treturn ServerCommonConf{}, err\n\t}\n\n\tcommon := GetDefaultServerConf()\n\terr = s.MapTo(&common)\n\tif err != nil {\n\t\treturn ServerCommonConf{}, err\n\t}\n\n\t// allow_ports\n\tallowPortStr := s.Key(\"allow_ports\").String()\n\tif allowPortStr != \"\" {\n\t\tcommon.AllowPortsStr = allowPortStr\n\t}\n\n\t// plugin.xxx\n\tpluginOpts := make(map[string]HTTPPluginOptions)\n\tfor _, section := range f.Sections() {\n\t\tname := section.Name()\n\t\tif !strings.HasPrefix(name, \"plugin.\") {\n\t\t\tcontinue\n\t\t}\n\n\t\topt, err := loadHTTPPluginOpt(section)\n\t\tif err != nil {\n\t\t\treturn ServerCommonConf{}, err\n\t\t}\n\n\t\tpluginOpts[opt.Name] = *opt\n\t}\n\tcommon.HTTPPlugins = pluginOpts\n\n\treturn common, nil\n}\n\nfunc loadHTTPPluginOpt(section *ini.Section) (*HTTPPluginOptions, error) {\n\tname := strings.TrimSpace(strings.TrimPrefix(section.Name(), \"plugin.\"))\n\n\topt := &HTTPPluginOptions{}\n\terr := section.MapTo(opt)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\topt.Name = name\n\n\treturn opt, nil\n}\n\n// Path: pkg/config/legacy/utils.go\n// Copyright 2020 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"strings\"\n)\n\nfunc GetMapWithoutPrefix(set map[string]string, prefix string) map[string]string {\n\tm := make(map[string]string)\n\n\tfor key, value := range set {\n\t\tif strings.HasPrefix(key, prefix) {\n\t\t\tm[strings.TrimPrefix(key, prefix)] = value\n\t\t}\n\t}\n\n\tif len(m) == 0 {\n\t\treturn nil\n\t}\n\n\treturn m\n}\n\nfunc GetMapByPrefix(set map[string]string, prefix string) map[string]string {\n\tm := make(map[string]string)\n\n\tfor key, value := range set {\n\t\tif strings.HasPrefix(key, prefix) {\n\t\t\tm[key] = value\n\t\t}\n\t}\n\n\tif len(m) == 0 {\n\t\treturn nil\n\t}\n\n\treturn m\n}\n\n// Path: pkg/config/template.go\n// Copyright 2024 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype NumberPair struct {\n\tFirst  int64\n\tSecond int64\n}\n\nfunc parseNumberRangePair(firstRangeStr, secondRangeStr string) ([]NumberPair, error) {\n\tfirstRangeNumbers, err := util.ParseRangeNumbers(firstRangeStr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsecondRangeNumbers, err := util.ParseRangeNumbers(secondRangeStr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif len(firstRangeNumbers) != len(secondRangeNumbers) {\n\t\treturn nil, fmt.Errorf(\"first and second range numbers are not in pairs\")\n\t}\n\tpairs := make([]NumberPair, 0, len(firstRangeNumbers))\n\tfor i := 0; i < len(firstRangeNumbers); i++ {\n\t\tpairs = append(pairs, NumberPair{\n\t\t\tFirst:  firstRangeNumbers[i],\n\t\t\tSecond: secondRangeNumbers[i],\n\t\t})\n\t}\n\treturn pairs, nil\n}\n\nfunc parseNumberRange(firstRangeStr string) ([]int64, error) {\n\treturn util.ParseRangeNumbers(firstRangeStr)\n}\n\n// Path: pkg/config/legacy/value.go\n// Copyright 2020 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"strings\"\n\t\"text/template\"\n)\n\nvar glbEnvs map[string]string\n\nfunc init() {\n\tglbEnvs = make(map[string]string)\n\tenvs := os.Environ()\n\tfor _, env := range envs {\n\t\tpair := strings.SplitN(env, \"=\", 2)\n\t\tif len(pair) != 2 {\n\t\t\tcontinue\n\t\t}\n\t\tglbEnvs[pair[0]] = pair[1]\n\t}\n}\n\ntype Values struct {\n\tEnvs map[string]string // environment vars\n}\n\nfunc GetValues() *Values {\n\treturn &Values{\n\t\tEnvs: glbEnvs,\n\t}\n}\n\nfunc RenderContent(in []byte) (out []byte, err error) {\n\ttmpl, errRet := template.New(\"frp\").Parse(string(in))\n\tif errRet != nil {\n\t\terr = errRet\n\t\treturn\n\t}\n\n\tbuffer := bytes.NewBufferString(\"\")\n\tv := GetValues()\n\terr = tmpl.Execute(buffer, v)\n\tif err != nil {\n\t\treturn\n\t}\n\tout = buffer.Bytes()\n\treturn\n}\n\nfunc GetRenderedConfFromFile(path string) (out []byte, err error) {\n\tvar b []byte\n\tb, err = os.ReadFile(path)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tout, err = RenderContent(b)\n\treturn\n}\n\n// Path: pkg/config/legacy/visitor.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage legacy\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\n\t\"gopkg.in/ini.v1\"\n)\n\ntype VisitorType string\n\nconst (\n\tVisitorTypeSTCP VisitorType = \"stcp\"\n\tVisitorTypeXTCP VisitorType = \"xtcp\"\n\tVisitorTypeSUDP VisitorType = \"sudp\"\n)\n\n// Visitor\nvar (\n\tvisitorConfTypeMap = map[VisitorType]reflect.Type{\n\t\tVisitorTypeSTCP: reflect.TypeOf(STCPVisitorConf{}),\n\t\tVisitorTypeXTCP: reflect.TypeOf(XTCPVisitorConf{}),\n\t\tVisitorTypeSUDP: reflect.TypeOf(SUDPVisitorConf{}),\n\t}\n)\n\ntype VisitorConf interface {\n\t// GetBaseConfig returns the base config of visitor.\n\tGetBaseConfig() *BaseVisitorConf\n\t// UnmarshalFromIni unmarshals config from ini.\n\tUnmarshalFromIni(prefix string, name string, section *ini.Section) error\n}\n\n// DefaultVisitorConf creates a empty VisitorConf object by visitorType.\n// If visitorType doesn't exist, return nil.\nfunc DefaultVisitorConf(visitorType VisitorType) VisitorConf {\n\tv, ok := visitorConfTypeMap[visitorType]\n\tif !ok {\n\t\treturn nil\n\t}\n\treturn reflect.New(v).Interface().(VisitorConf)\n}\n\ntype BaseVisitorConf struct {\n\tProxyName      string `ini:\"name\" json:\"name\"`\n\tProxyType      string `ini:\"type\" json:\"type\"`\n\tUseEncryption  bool   `ini:\"use_encryption\" json:\"use_encryption\"`\n\tUseCompression bool   `ini:\"use_compression\" json:\"use_compression\"`\n\tRole           string `ini:\"role\" json:\"role\"`\n\tSk             string `ini:\"sk\" json:\"sk\"`\n\t// if the server user is not set, it defaults to the current user\n\tServerUser string `ini:\"server_user\" json:\"server_user\"`\n\tServerName string `ini:\"server_name\" json:\"server_name\"`\n\tBindAddr   string `ini:\"bind_addr\" json:\"bind_addr\"`\n\t// BindPort is the port that visitor listens on.\n\t// It can be less than 0, it means don't bind to the port and only receive connections redirected from\n\t// other visitors. (This is not supported for SUDP now)\n\tBindPort int `ini:\"bind_port\" json:\"bind_port\"`\n}\n\n// Base\nfunc (cfg *BaseVisitorConf) GetBaseConfig() *BaseVisitorConf {\n\treturn cfg\n}\n\nfunc (cfg *BaseVisitorConf) unmarshalFromIni(_ string, name string, _ *ini.Section) error {\n\t// Custom decoration after basic unmarshal:\n\tcfg.ProxyName = name\n\n\t// bind_addr\n\tif cfg.BindAddr == \"\" {\n\t\tcfg.BindAddr = \"127.0.0.1\"\n\t}\n\treturn nil\n}\n\nfunc preVisitorUnmarshalFromIni(cfg VisitorConf, prefix string, name string, section *ini.Section) error {\n\terr := section.MapTo(cfg)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = cfg.GetBaseConfig().unmarshalFromIni(prefix, name, section)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\ntype SUDPVisitorConf struct {\n\tBaseVisitorConf `ini:\",extends\"`\n}\n\nfunc (cfg *SUDPVisitorConf) UnmarshalFromIni(prefix string, name string, section *ini.Section) (err error) {\n\terr = preVisitorUnmarshalFromIni(cfg, prefix, name, section)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// Add custom logic unmarshal, if exists\n\n\treturn\n}\n\ntype STCPVisitorConf struct {\n\tBaseVisitorConf `ini:\",extends\"`\n}\n\nfunc (cfg *STCPVisitorConf) UnmarshalFromIni(prefix string, name string, section *ini.Section) (err error) {\n\terr = preVisitorUnmarshalFromIni(cfg, prefix, name, section)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// Add custom logic unmarshal, if exists\n\n\treturn\n}\n\ntype XTCPVisitorConf struct {\n\tBaseVisitorConf `ini:\",extends\"`\n\n\tProtocol          string `ini:\"protocol\" json:\"protocol,omitempty\"`\n\tKeepTunnelOpen    bool   `ini:\"keep_tunnel_open\" json:\"keep_tunnel_open,omitempty\"`\n\tMaxRetriesAnHour  int    `ini:\"max_retries_an_hour\" json:\"max_retries_an_hour,omitempty\"`\n\tMinRetryInterval  int    `ini:\"min_retry_interval\" json:\"min_retry_interval,omitempty\"`\n\tFallbackTo        string `ini:\"fallback_to\" json:\"fallback_to,omitempty\"`\n\tFallbackTimeoutMs int    `ini:\"fallback_timeout_ms\" json:\"fallback_timeout_ms,omitempty\"`\n}\n\nfunc (cfg *XTCPVisitorConf) UnmarshalFromIni(prefix string, name string, section *ini.Section) (err error) {\n\terr = preVisitorUnmarshalFromIni(cfg, prefix, name, section)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// Add custom logic unmarshal, if exists\n\tif cfg.Protocol == \"\" {\n\t\tcfg.Protocol = \"quic\"\n\t}\n\tif cfg.MaxRetriesAnHour <= 0 {\n\t\tcfg.MaxRetriesAnHour = 8\n\t}\n\tif cfg.MinRetryInterval <= 0 {\n\t\tcfg.MinRetryInterval = 90\n\t}\n\tif cfg.FallbackTimeoutMs <= 0 {\n\t\tcfg.FallbackTimeoutMs = 1000\n\t}\n\treturn\n}\n\n// Visitor loaded from ini\nfunc NewVisitorConfFromIni(prefix string, name string, section *ini.Section) (VisitorConf, error) {\n\t// section.Key: if key not exists, section will set it with default value.\n\tvisitorType := VisitorType(section.Key(\"type\").String())\n\n\tif visitorType == \"\" {\n\t\treturn nil, fmt.Errorf(\"type shouldn't be empty\")\n\t}\n\n\tconf := DefaultVisitorConf(visitorType)\n\tif conf == nil {\n\t\treturn nil, fmt.Errorf(\"type [%s] error\", visitorType)\n\t}\n\n\tif err := conf.UnmarshalFromIni(prefix, name, section); err != nil {\n\t\treturn nil, fmt.Errorf(\"type [%s] error\", visitorType)\n\t}\n\treturn conf, nil\n}\n\n// Path: pkg/config/load.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"html/template\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\ttoml \"github.com/pelletier/go-toml/v2\"\n\t\"github.com/samber/lo\"\n\t\"gopkg.in/ini.v1\"\n\t\"k8s.io/apimachinery/pkg/util/sets\"\n\t\"k8s.io/apimachinery/pkg/util/yaml\"\n\n\t\"github.com/fatedier/frp/pkg/config/legacy\"\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/config/v1/validation\"\n\t\"github.com/fatedier/frp/pkg/msg\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\nvar glbEnvs map[string]string\n\nfunc init() {\n\tglbEnvs = make(map[string]string)\n\tenvs := os.Environ()\n\tfor _, env := range envs {\n\t\tpair := strings.SplitN(env, \"=\", 2)\n\t\tif len(pair) != 2 {\n\t\t\tcontinue\n\t\t}\n\t\tglbEnvs[pair[0]] = pair[1]\n\t}\n}\n\ntype Values struct {\n\tEnvs map[string]string // environment vars\n}\n\nfunc GetValues() *Values {\n\treturn &Values{\n\t\tEnvs: glbEnvs,\n\t}\n}\n\nfunc DetectLegacyINIFormat(content []byte) bool {\n\tf, err := ini.Load(content)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif _, err := f.GetSection(\"common\"); err == nil {\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc DetectLegacyINIFormatFromFile(path string) bool {\n\tb, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn false\n\t}\n\treturn DetectLegacyINIFormat(b)\n}\n\nfunc RenderWithTemplate(in []byte, values *Values) ([]byte, error) {\n\ttmpl, err := template.New(\"frp\").Funcs(template.FuncMap{\n\t\t\"parseNumberRange\":     parseNumberRange,\n\t\t\"parseNumberRangePair\": parseNumberRangePair,\n\t}).Parse(string(in))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tbuffer := bytes.NewBufferString(\"\")\n\tif err := tmpl.Execute(buffer, values); err != nil {\n\t\treturn nil, err\n\t}\n\treturn buffer.Bytes(), nil\n}\n\nfunc LoadFileContentWithTemplate(path string, values *Values) ([]byte, error) {\n\tb, err := os.ReadFile(path)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn RenderWithTemplate(b, values)\n}\n\nfunc LoadConfigureFromFile(path string, c any, strict bool) error {\n\tcontent, err := LoadFileContentWithTemplate(path, GetValues())\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn LoadConfigure(content, c, strict)\n}\n\n// LoadConfigure loads configuration from bytes and unmarshal into c.\n// Now it supports json, yaml and toml format.\nfunc LoadConfigure(b []byte, c any, strict bool) error {\n\tv1.DisallowUnknownFieldsMu.Lock()\n\tdefer v1.DisallowUnknownFieldsMu.Unlock()\n\tv1.DisallowUnknownFields = strict\n\n\tvar tomlObj interface{}\n\t// Try to unmarshal as TOML first; swallow errors from that (assume it's not valid TOML).\n\tif err := toml.Unmarshal(b, &tomlObj); err == nil {\n\t\tb, err = json.Marshal(&tomlObj)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// If the buffer smells like JSON (first non-whitespace character is '{'), unmarshal as JSON directly.\n\tif yaml.IsJSONBuffer(b) {\n\t\tdecoder := json.NewDecoder(bytes.NewBuffer(b))\n\t\tif strict {\n\t\t\tdecoder.DisallowUnknownFields()\n\t\t}\n\t\treturn decoder.Decode(c)\n\t}\n\t// It wasn't JSON. Unmarshal as YAML.\n\tif strict {\n\t\treturn yaml.UnmarshalStrict(b, c)\n\t}\n\treturn yaml.Unmarshal(b, c)\n}\n\nfunc NewProxyConfigurerFromMsg(m *msg.NewProxy, serverCfg *v1.ServerConfig) (v1.ProxyConfigurer, error) {\n\tm.ProxyType = util.EmptyOr(m.ProxyType, string(v1.ProxyTypeTCP))\n\n\tconfigurer := v1.NewProxyConfigurerByType(v1.ProxyType(m.ProxyType))\n\tif configurer == nil {\n\t\treturn nil, fmt.Errorf(\"unknown proxy type: %s\", m.ProxyType)\n\t}\n\n\tconfigurer.UnmarshalFromMsg(m)\n\tconfigurer.Complete(\"\")\n\n\tif err := validation.ValidateProxyConfigurerForServer(configurer, serverCfg); err != nil {\n\t\treturn nil, err\n\t}\n\treturn configurer, nil\n}\n\n\nfunc LoadServerConfig(path string, strict bool) (*v1.ServerConfig, bool, error) {\n\tvar (\n\t\tsvrCfg         *v1.ServerConfig\n\t\tisLegacyFormat bool\n\t)\n\t// detect legacy ini format\n\tif DetectLegacyINIFormatFromFile(path) {\n\t\tcontent, err := legacy.GetRenderedConfFromFile(path)\n\t\tif err != nil {\n\t\t\treturn nil, true, err\n\t\t}\n\t\tlegacyCfg, err := legacy.UnmarshalServerConfFromIni(content)\n\t\tif err != nil {\n\t\t\treturn nil, true, err\n\t\t}\n\t\tsvrCfg = legacy.Convert_ServerCommonConf_To_v1(&legacyCfg)\n\t\tisLegacyFormat = true\n\t} else {\n\t\tsvrCfg = &v1.ServerConfig{}\n\t\tif err := LoadConfigureFromFile(path, svrCfg, strict); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t}\n\tif svrCfg != nil {\n\t\tsvrCfg.Complete()\n\t}\n\treturn svrCfg, isLegacyFormat, nil\n}\n\nfunc LoadClientConfig(path string, strict bool) (\n\t*v1.ClientCommonConfig,\n\t[]v1.ProxyConfigurer,\n\t[]v1.VisitorConfigurer,\n\tbool, error,\n) {\n\tvar (\n\t\tcliCfg         *v1.ClientCommonConfig\n\t\tproxyCfgs      = make([]v1.ProxyConfigurer, 0)\n\t\tvisitorCfgs    = make([]v1.VisitorConfigurer, 0)\n\t\tisLegacyFormat bool\n\t)\n\n\tif DetectLegacyINIFormatFromFile(path) {\n\t\tlegacyCommon, legacyProxyCfgs, legacyVisitorCfgs, err := legacy.ParseClientConfig(path)\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, true, err\n\t\t}\n\t\tcliCfg = legacy.Convert_ClientCommonConf_To_v1(&legacyCommon)\n\t\tfor _, c := range legacyProxyCfgs {\n\t\t\tproxyCfgs = append(proxyCfgs, legacy.Convert_ProxyConf_To_v1(c))\n\t\t}\n\t\tfor _, c := range legacyVisitorCfgs {\n\t\t\tvisitorCfgs = append(visitorCfgs, legacy.Convert_VisitorConf_To_v1(c))\n\t\t}\n\t\tisLegacyFormat = true\n\t} else {\n\t\tallCfg := v1.ClientConfig{}\n\t\tif err := LoadConfigureFromFile(path, &allCfg, strict); err != nil {\n\t\t\treturn nil, nil, nil, false, err\n\t\t}\n\t\tcliCfg = &allCfg.ClientCommonConfig\n\t\tfor _, c := range allCfg.Proxies {\n\t\t\tproxyCfgs = append(proxyCfgs, c.ProxyConfigurer)\n\t\t}\n\t\tfor _, c := range allCfg.Visitors {\n\t\t\tvisitorCfgs = append(visitorCfgs, c.VisitorConfigurer)\n\t\t}\n\t}\n\n\t// Load additional config from includes.\n\t// legacy ini format already handle this in ParseClientConfig.\n\tif len(cliCfg.IncludeConfigFiles) > 0 && !isLegacyFormat {\n\t\textProxyCfgs, extVisitorCfgs, err := LoadAdditionalClientConfigs(cliCfg.IncludeConfigFiles, isLegacyFormat, strict)\n\t\tif err != nil {\n\t\t\treturn nil, nil, nil, isLegacyFormat, err\n\t\t}\n\t\tproxyCfgs = append(proxyCfgs, extProxyCfgs...)\n\t\tvisitorCfgs = append(visitorCfgs, extVisitorCfgs...)\n\t}\n\n\t// Filter by start\n\tif len(cliCfg.Start) > 0 {\n\t\tstartSet := sets.New(cliCfg.Start...)\n\t\tproxyCfgs = lo.Filter(proxyCfgs, func(c v1.ProxyConfigurer, _ int) bool {\n\t\t\treturn startSet.Has(c.GetBaseConfig().Name)\n\t\t})\n\t\tvisitorCfgs = lo.Filter(visitorCfgs, func(c v1.VisitorConfigurer, _ int) bool {\n\t\t\treturn startSet.Has(c.GetBaseConfig().Name)\n\t\t})\n\t}\n\n\tif cliCfg != nil {\n\t\tcliCfg.Complete()\n\t}\n\tfor _, c := range proxyCfgs {\n\t\tc.Complete(cliCfg.User)\n\t}\n\tfor _, c := range visitorCfgs {\n\t\tc.Complete(cliCfg)\n\t}\n\treturn cliCfg, proxyCfgs, visitorCfgs, isLegacyFormat, nil\n}\n\nfunc LoadAdditionalClientConfigs(paths []string, isLegacyFormat bool, strict bool) ([]v1.ProxyConfigurer, []v1.VisitorConfigurer, error) {\n\tproxyCfgs := make([]v1.ProxyConfigurer, 0)\n\tvisitorCfgs := make([]v1.VisitorConfigurer, 0)\n\tfor _, path := range paths {\n\t\tabsDir, err := filepath.Abs(filepath.Dir(path))\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tif _, err := os.Stat(absDir); os.IsNotExist(err) {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tfiles, err := os.ReadDir(absDir)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tfor _, fi := range files {\n\t\t\tif fi.IsDir() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tabsFile := filepath.Join(absDir, fi.Name())\n\t\t\tif matched, _ := filepath.Match(filepath.Join(absDir, filepath.Base(path)), absFile); matched {\n\t\t\t\t// support yaml/json/toml\n\t\t\t\tcfg := v1.ClientConfig{}\n\t\t\t\tif err := LoadConfigureFromFile(absFile, &cfg, strict); err != nil {\n\t\t\t\t\treturn nil, nil, fmt.Errorf(\"load additional config from %s error: %v\", absFile, err)\n\t\t\t\t}\n\t\t\t\tfor _, c := range cfg.Proxies {\n\t\t\t\t\tproxyCfgs = append(proxyCfgs, c.ProxyConfigurer)\n\t\t\t\t}\n\t\t\t\tfor _, c := range cfg.Visitors {\n\t\t\t\t\tvisitorCfgs = append(visitorCfgs, c.VisitorConfigurer)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn proxyCfgs, visitorCfgs, nil\n}\n\n// Path: pkg/config/load_test.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage config\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nconst tomlServerContent = `\nbindAddr = \"127.0.0.1\"\nkcpBindPort = 7000\nquicBindPort = 7001\ntcpmuxHTTPConnectPort = 7005\ncustom404Page = \"/abc.html\"\ntransport.tcpKeepalive = 10\n`\n\nconst yamlServerContent = `\nbindAddr: 127.0.0.1\nkcpBindPort: 7000\nquicBindPort: 7001\ntcpmuxHTTPConnectPort: 7005\ncustom404Page: /abc.html\ntransport:\n  tcpKeepalive: 10\n`\n\nconst jsonServerContent = `\n{\n  \"bindAddr\": \"127.0.0.1\",\n  \"kcpBindPort\": 7000,\n  \"quicBindPort\": 7001,\n  \"tcpmuxHTTPConnectPort\": 7005,\n  \"custom404Page\": \"/abc.html\",\n  \"transport\": {\n    \"tcpKeepalive\": 10\n  }\n}\n`\n\nfunc TestLoadServerConfig(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\tcontent string\n\t}{\n\t\t{\"toml\", tomlServerContent},\n\t\t{\"yaml\", yamlServerContent},\n\t\t{\"json\", jsonServerContent},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\trequire := require.New(t)\n\t\t\tsvrCfg := v1.ServerConfig{}\n\t\t\terr := LoadConfigure([]byte(test.content), &svrCfg, true)\n\t\t\trequire.NoError(err)\n\t\t\trequire.EqualValues(\"127.0.0.1\", svrCfg.BindAddr)\n\t\t\trequire.EqualValues(7000, svrCfg.KCPBindPort)\n\t\t\trequire.EqualValues(7001, svrCfg.QUICBindPort)\n\t\t\trequire.EqualValues(7005, svrCfg.TCPMuxHTTPConnectPort)\n\t\t\trequire.EqualValues(\"/abc.html\", svrCfg.Custom404Page)\n\t\t\trequire.EqualValues(10, svrCfg.Transport.TCPKeepAlive)\n\t\t})\n\t}\n}\n\n// Test that loading in strict mode fails when the config is invalid.\nfunc TestLoadServerConfigStrictMode(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\tcontent string\n\t}{\n\t\t{\"toml\", tomlServerContent},\n\t\t{\"yaml\", yamlServerContent},\n\t\t{\"json\", jsonServerContent},\n\t}\n\n\tfor _, strict := range []bool{false, true} {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(fmt.Sprintf(\"%s-strict-%t\", test.name, strict), func(t *testing.T) {\n\t\t\t\trequire := require.New(t)\n\t\t\t\t// Break the content with an innocent typo\n\t\t\t\tbrokenContent := strings.Replace(test.content, \"bindAddr\", \"bindAdur\", 1)\n\t\t\t\tsvrCfg := v1.ServerConfig{}\n\t\t\t\terr := LoadConfigure([]byte(brokenContent), &svrCfg, strict)\n\t\t\t\tif strict {\n\t\t\t\t\trequire.ErrorContains(err, \"bindAdur\")\n\t\t\t\t} else {\n\t\t\t\t\trequire.NoError(err)\n\t\t\t\t\t// BindAddr didn't get parsed because of the typo.\n\t\t\t\t\trequire.EqualValues(\"\", svrCfg.BindAddr)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestCustomStructStrictMode(t *testing.T) {\n\trequire := require.New(t)\n\n\tproxyStr := `\nserverPort = 7000\n\n[[proxies]]\nname = \"test\"\ntype = \"tcp\"\nremotePort = 6000\n`\n\tclientCfg := v1.ClientConfig{}\n\terr := LoadConfigure([]byte(proxyStr), &clientCfg, true)\n\trequire.NoError(err)\n\n\tproxyStr += `unknown = \"unknown\"`\n\terr = LoadConfigure([]byte(proxyStr), &clientCfg, true)\n\trequire.Error(err)\n\n\tvisitorStr := `\nserverPort = 7000\n\n[[visitors]]\nname = \"test\"\ntype = \"stcp\"\nbindPort = 6000\nserverName = \"server\"\n`\n\terr = LoadConfigure([]byte(visitorStr), &clientCfg, true)\n\trequire.NoError(err)\n\n\tvisitorStr += `unknown = \"unknown\"`\n\terr = LoadConfigure([]byte(visitorStr), &clientCfg, true)\n\trequire.Error(err)\n\n\tpluginStr := `\nserverPort = 7000\n\n[[proxies]]\nname = \"test\"\ntype = \"tcp\"\nremotePort = 6000\n[proxies.plugin]\ntype = \"unix_domain_socket\"\nunixPath = \"/tmp/uds.sock\"\n`\n\terr = LoadConfigure([]byte(pluginStr), &clientCfg, true)\n\trequire.NoError(err)\n\tpluginStr += `unknown = \"unknown\"`\n\terr = LoadConfigure([]byte(pluginStr), &clientCfg, true)\n\trequire.Error(err)\n}\n\n// Path: pkg/config/types/types_test.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage types\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\ntype Wrap struct {\n\tB   BandwidthQuantity `json:\"b\"`\n\tInt int               `json:\"int\"`\n}\n\nfunc TestBandwidthQuantity(t *testing.T) {\n\trequire := require.New(t)\n\n\tvar w Wrap\n\terr := json.Unmarshal([]byte(`{\"b\":\"1KB\",\"int\":5}`), &w)\n\trequire.NoError(err)\n\trequire.EqualValues(1*KB, w.B.Bytes())\n\n\tbuf, err := json.Marshal(&w)\n\trequire.NoError(err)\n\trequire.Equal(`{\"b\":\"1KB\",\"int\":5}`, string(buf))\n}\n\nfunc TestPortsRangeSlice2String(t *testing.T) {\n\trequire := require.New(t)\n\n\tports := []PortsRange{\n\t\t{\n\t\t\tStart: 1000,\n\t\t\tEnd:   2000,\n\t\t},\n\t\t{\n\t\t\tSingle: 3000,\n\t\t},\n\t}\n\tstr := PortsRangeSlice(ports).String()\n\trequire.Equal(\"1000-2000,3000\", str)\n}\n\nfunc TestNewPortsRangeSliceFromString(t *testing.T) {\n\trequire := require.New(t)\n\n\tports, err := NewPortsRangeSliceFromString(\"1000-2000,3000\")\n\trequire.NoError(err)\n\trequire.Equal([]PortsRange{\n\t\t{\n...\n", "needle_token_start": 11384, "needle_token_end": 11681, "code_context_ntokens": 15379, "output": ["```go\nfunc LoadServerConfig(path string, strict bool) (*v1.ServerConfig, bool, error) {\n\tvar (\n\t\tsvrCfg         *v1.ServerConfig\n\t\tisLegacyFormat bool\n\t)\n\t// detect legacy ini format\n\tif DetectLegacyINIFormatFromFile(path) {\n\t\tcontent, err := legacy.GetRenderedConfFromFile(path)\n\t\tif err != nil {\n\t\t\treturn nil, true, err\n\t\t}\n\t\tlegacyCfg, err := legacy.UnmarshalServerConfFromIni(content)\n\t\tif err != nil {\n\t\t\treturn nil, true, err\n\t\t}\n\t\tsvrCfg = legacy.Convert_ServerCommonConf_To_v1(&legacyCfg)\n\t\tisLegacyFormat = true\n\t} else {\n\t\tsvrCfg = &v1.ServerConfig{}\n\t\tif err := LoadConfigureFromFile(path, svrCfg, strict); err != nil {\n\t\t\treturn nil, false, err\n\t\t}\n\t}\n\tif svrCfg != nil {\n\t\tsvrCfg.Complete()\n\t}\n\treturn svrCfg, isLegacyFormat, nil\n}\n```"]}
{"repo": "fatedier/frp", "name": "NewReqidContext", "language": "go", "path": "pkg/plugin/server/tracer.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To create a new context that carries a unique identifier which can be used to trace and differentiate individual operations or requests within a system.\n2. **Input**: A base context and a string representing a unique identifier.\n3. **Output**: A new context derived from the base context, which includes the unique identifier embedded within it.\n4. **Procedure**: The function takes an existing context and a unique identifier as inputs. It then embeds the unique identifier into the context using a predefined key, and returns this modified context.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/plugin/client/http2https.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"crypto/tls\"\n\t\"io\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\n\t\"github.com/fatedier/golib/pool\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTP2HTTPS, NewHTTP2HTTPSPlugin)\n}\n\ntype HTTP2HTTPSPlugin struct {\n\topts *v1.HTTP2HTTPSPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTP2HTTPSPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTP2HTTPSPluginOptions)\n\n\tlistener := NewProxyListener()\n\n\tp := &HTTP2HTTPSPlugin{\n\t\topts: opts,\n\t\tl:    listener,\n\t}\n\n\ttr := &http.Transport{\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\n\trp := &httputil.ReverseProxy{\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.Out.Header[\"X-Forwarded-Host\"] = r.In.Header[\"X-Forwarded-Host\"]\n\t\t\tr.Out.Header[\"X-Forwarded-Proto\"] = r.In.Header[\"X-Forwarded-Proto\"]\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"https\"\n\t\t\treq.URL.Host = p.opts.LocalAddr\n\t\t\tif p.opts.HostHeaderRewrite != \"\" {\n\t\t\t\treq.Host = p.opts.HostHeaderRewrite\n\t\t\t}\n\t\t\tfor k, v := range p.opts.RequestHeaders.Set {\n\t\t\t\treq.Header.Set(k, v)\n\t\t\t}\n\t\t},\n\t\tTransport:  tr,\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n...\n// Path: pkg/plugin/client/http_proxy.go\n// Copyright 2017 frp team\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"bufio\"\n\t\"encoding/base64\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\tlibio \"github.com/fatedier/golib/io\"\n\tlibnet \"github.com/fatedier/golib/net\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTPProxy, NewHTTPProxyPlugin)\n}\n\ntype HTTPProxy struct {\n\topts *v1.HTTPProxyPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTPProxyPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTPProxyPluginOptions)\n\tlistener := NewProxyListener()\n\n\thp := &HTTPProxy{\n\t\tl:    listener,\n\t\topts: opts,\n\t}\n\n\thp.s = &http.Server{\n\t\tHandler:           hp,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\n\tgo func() {\n\t\t_ = hp.s.Serve(listener)\n\t}()\n\treturn hp, nil\n}\n\nfunc (hp *HTTPProxy) Name() string {\n\treturn v1.PluginHTTPProxy\n}\n\nfunc (hp *HTTPProxy) Handle(conn io.ReadWriteCloser, realConn net.Conn, _ *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\n\tsc, rd := libnet.NewSharedConn(wrapConn)\n\tfirstBytes := make([]byte, 7)\n\t_, err := rd.Read(firstBytes)\n\tif err != nil {\n\t\twrapConn.Close()\n\t\treturn\n\t}\n\n\tif strings.ToUpper(string(firstBytes)) == \"CONNECT\" {\n\t\tbufRd := bufio.NewReader(sc)\n\t\trequest, err := http.ReadRequest(bufRd)\n\t\tif err != nil {\n\t\t\twrapConn.Close()\n\t\t\treturn\n\t\t}\n\t\thp.handleConnectReq(request, libio.WrapReadWriteCloser(bufRd, wrapConn, wrapConn.Close))\n\t\treturn\n\t}\n\n\t_ = hp.l.PutConn(sc)\n}\n\nfunc (hp *HTTPProxy) Close() error {\n\thp.s.Close()\n\thp.l.Close()\n\treturn nil\n}\n\nfunc (hp *HTTPProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {\n\tif ok := hp.Auth(req); !ok {\n\t\trw.Header().Set(\"Proxy-Authenticate\", \"Basic\")\n\t\trw.WriteHeader(http.StatusProxyAuthRequired)\n\t\treturn\n\t}\n\n\tif req.Method == http.MethodConnect {\n\t\t// deprecated\n\t\t// Connect request is handled in Handle function.\n\t\thp.ConnectHandler(rw, req)\n\t} else {\n\t\thp.HTTPHandler(rw, req)\n\t}\n}\n\nfunc (hp *HTTPProxy) HTTPHandler(rw http.ResponseWriter, req *http.Request) {\n\tremoveProxyHeaders(req)\n\n\tresp, err := http.DefaultTransport.RoundTrip(req)\n\tif err != nil {\n\t\thttp.Error(rw, err.Error(), http.StatusInternalServerError)\n\t\treturn\n\t}\n\tdefer resp.Body.Close()\n\n\tcopyHeaders(rw.Header(), resp.Header)\n\trw.WriteHeader(resp.StatusCode)\n\n\t_, err = io.Copy(rw, resp.Body)\n\tif err != nil && err != io.EOF {\n\t\treturn\n\t}\n}\n\n// deprecated\n// Hijack needs to SetReadDeadline on the Conn of the request, but if we use stream compression here,\n// we may always get i/o timeout error.\nfunc (hp *HTTPProxy) ConnectHandler(rw http.ResponseWriter, req *http.Request) {\n\thj, ok := rw.(http.Hijacker)\n\tif !ok {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tclient, _, err := hj.Hijack()\n\tif err != nil {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tremote, err := net.Dial(\"tcp\", req.URL.Host)\n\tif err != nil {\n\t\thttp.Error(rw, \"Failed\", http.StatusBadRequest)\n\t\tclient.Close()\n\t\treturn\n\t}\n\t_, _ = client.Write([]byte(\"HTTP/1.1 200 OK\\r\\n\\r\\n\"))\n\n\tgo libio.Join(remote, client)\n}\n\nfunc (hp *HTTPProxy) Auth(req *http.Request) bool {\n\tif hp.opts.HTTPUser == \"\" && hp.opts.HTTPPassword == \"\" {\n\t\treturn true\n\t}\n\n\ts := strings.SplitN(req.Header.Get(\"Proxy-Authorization\"), \" \", 2)\n\tif len(s) != 2 {\n\t\treturn false\n\t}\n\n\tb, err := base64.StdEncoding.DecodeString(s[1])\n\tif err != nil {\n\t\treturn false\n\t}\n\n\tpair := strings.SplitN(string(b), \":\", 2)\n\tif len(pair) != 2 {\n\t\treturn false\n\t}\n\n\tif !util.ConstantTimeEqString(pair[0], hp.opts.HTTPUser) ||\n\t\t!util.ConstantTimeEqString(pair[1], hp.opts.HTTPPassword) {\n\t\ttime.Sleep(200 * time.Millisecond)\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc (hp *HTTPProxy) handleConnectReq(req *http.Request, rwc io.ReadWriteCloser) {\n\tdefer rwc.Close()\n\tif ok := hp.Auth(req); !ok {\n\t\tres := getBadResponse()\n\t\t_ = res.Write(rwc)\n\t\tif res.Body != nil {\n\t\t\tres.Body.Close()\n\t\t}\n\t\treturn\n\t}\n\n\tremote, err := net.Dial(\"tcp\", req.URL.Host)\n\tif err != nil {\n\t\tres := &http.Response{\n\t\t\tStatusCode: 400,\n\t\t\tProto:      \"HTTP/1.1\",\n\t\t\tProtoMajor: 1,\n\t\t\tProtoMinor: 1,\n\t\t}\n\t\t_ = res.Write(rwc)\n\t\treturn\n\t}\n\t_, _ = rwc.Write([]byte(\"HTTP/1.1 200 OK\\r\\n\\r\\n\"))\n\n\tlibio.Join(remote, rwc)\n}\n\nfunc copyHeaders(dst, src http.Header) {\n\tfor key, values := range src {\n\t\tfor _, value := range values {\n\t\t\tdst.Add(key, value)\n\t\t}\n\t}\n}\n\nfunc removeProxyHeaders(req *http.Request) {\n\treq.RequestURI = \"\"\n\treq.Header.Del(\"Proxy-Connection\")\n\treq.Header.Del(\"Connection\")\n\treq.Header.Del(\"Proxy-Authenticate\")\n\treq.Header.Del(\"Proxy-Authorization\")\n\treq.Header.Del(\"TE\")\n\treq.Header.Del(\"Trailers\")\n\treq.Header.Del(\"Transfer-Encoding\")\n\treq.Header.Del(\"Upgrade\")\n}\n\nfunc getBadResponse() *http.Response {\n\theader := make(map[string][]string)\n\theader[\"Proxy-Authenticate\"] = []string{\"Basic\"}\n\theader[\"Connection\"] = []string{\"close\"}\n\tres := &http.Response{\n\t\tStatus:     \"407 Not authorized\",\n\t\tStatusCode: 407,\n\t\tProto:      \"HTTP/1.1\",\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 1,\n\t\tHeader:     header,\n\t}\n\treturn res\n}\n\n// Path: pkg/plugin/client/https2http.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/pool\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/transport\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTPS2HTTP, NewHTTPS2HTTPPlugin)\n}\n\ntype HTTPS2HTTPPlugin struct {\n\topts *v1.HTTPS2HTTPPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTPS2HTTPPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTPS2HTTPPluginOptions)\n\tlistener := NewProxyListener()\n\n\tp := &HTTPS2HTTPPlugin{\n\t\topts: opts,\n\t\tl:    listener,\n\t}\n\n\trp := &httputil.ReverseProxy{\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.SetXForwarded()\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"http\"\n\t\t\treq.URL.Host = p.opts.LocalAddr\n\t\t\tif p.opts.HostHeaderRewrite != \"\" {\n\t\t\t\treq.Host = p.opts.HostHeaderRewrite\n\t\t\t}\n\t\t\tfor k, v := range p.opts.RequestHeaders.Set {\n\t\t\t\treq.Header.Set(k, v)\n\t\t\t}\n\t\t},\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n\t\tErrorLog:   stdlog.New(log.NewWriteLogger(log.WarnLevel, 2), \"\", 0),\n\t}\n\n\tp.s = &http.Server{\n\t\tHandler:           rp,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\n\tvar (\n\t\ttlsConfig *tls.Config\n\t\terr       error\n\t)\n\tif opts.CrtPath != \"\" || opts.KeyPath != \"\" {\n\t\ttlsConfig, err = p.genTLSConfig()\n\t} else {\n\t\ttlsConfig, err = transport.NewServerTLSConfig(\"\", \"\", \"\")\n\t\ttlsConfig.InsecureSkipVerify = true\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"gen TLS config error: %v\", err)\n\t}\n\tln := tls.NewListener(listener, tlsConfig)\n\n\tgo func() {\n\t\t_ = p.s.Serve(ln)\n\t}()\n\treturn p, nil\n}\n\nfunc (p *HTTPS2HTTPPlugin) genTLSConfig() (*tls.Config, error) {\n\tcert, err := tls.LoadX509KeyPair(p.opts.CrtPath, p.opts.KeyPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfig := &tls.Config{Certificates: []tls.Certificate{cert}}\n\treturn config, nil\n}\n\nfunc (p *HTTPS2HTTPPlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, extra *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\tif extra.SrcAddr != nil {\n\t\twrapConn.SetRemoteAddr(extra.SrcAddr)\n\t}\n\t_ = p.l.PutConn(wrapConn)\n}\n\nfunc (p *HTTPS2HTTPPlugin) Name() string {\n\treturn v1.PluginHTTPS2HTTP\n}\n\nfunc (p *HTTPS2HTTPPlugin) Close() error {\n\treturn p.s.Close()\n}\n\n// Path: pkg/plugin/client/https2https.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/pool\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/transport\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginHTTPS2HTTPS, NewHTTPS2HTTPSPlugin)\n}\n\ntype HTTPS2HTTPSPlugin struct {\n\topts *v1.HTTPS2HTTPSPluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewHTTPS2HTTPSPlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.HTTPS2HTTPSPluginOptions)\n\n\tlistener := NewProxyListener()\n\n\tp := &HTTPS2HTTPSPlugin{\n\t\topts: opts,\n\t\tl:    listener,\n\t}\n\n\ttr := &http.Transport{\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\n\trp := &httputil.ReverseProxy{\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.SetXForwarded()\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"https\"\n\t\t\treq.URL.Host = p.opts.LocalAddr\n\t\t\tif p.opts.HostHeaderRewrite != \"\" {\n\t\t\t\treq.Host = p.opts.HostHeaderRewrite\n\t\t\t}\n\t\t\tfor k, v := range p.opts.RequestHeaders.Set {\n\t\t\t\treq.Header.Set(k, v)\n\t\t\t}\n\t\t},\n\t\tTransport:  tr,\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n\t\tErrorLog:   stdlog.New(log.NewWriteLogger(log.WarnLevel, 2), \"\", 0),\n\t}\n\n\tp.s = &http.Server{\n\t\tHandler:           rp,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\n\tvar (\n\t\ttlsConfig *tls.Config\n\t\terr       error\n\t)\n\tif opts.CrtPath != \"\" || opts.KeyPath != \"\" {\n\t\ttlsConfig, err = p.genTLSConfig()\n\t} else {\n\t\ttlsConfig, err = transport.NewServerTLSConfig(\"\", \"\", \"\")\n\t\ttlsConfig.InsecureSkipVerify = true\n\t}\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"gen TLS config error: %v\", err)\n\t}\n\tln := tls.NewListener(listener, tlsConfig)\n\n\tgo func() {\n\t\t_ = p.s.Serve(ln)\n\t}()\n\treturn p, nil\n}\n\nfunc (p *HTTPS2HTTPSPlugin) genTLSConfig() (*tls.Config, error) {\n\tcert, err := tls.LoadX509KeyPair(p.opts.CrtPath, p.opts.KeyPath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconfig := &tls.Config{Certificates: []tls.Certificate{cert}}\n\treturn config, nil\n}\n\nfunc (p *HTTPS2HTTPSPlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, extra *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\tif extra.SrcAddr != nil {\n\t\twrapConn.SetRemoteAddr(extra.SrcAddr)\n\t}\n\t_ = p.l.PutConn(wrapConn)\n}\n\nfunc (p *HTTPS2HTTPSPlugin) Name() string {\n\treturn v1.PluginHTTPS2HTTPS\n}\n\nfunc (p *HTTPS2HTTPSPlugin) Close() error {\n\treturn p.s.Close()\n}\n\n// Path: pkg/plugin/client/plugin.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"sync\"\n\n\t\"github.com/fatedier/golib/errors\"\n\tpp \"github.com/pires/go-proxyproto\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\n// Creators is used for create plugins to handle connections.\nvar creators = make(map[string]CreatorFn)\n\n// params has prefix \"plugin_\"\ntype CreatorFn func(options v1.ClientPluginOptions) (Plugin, error)\n\nfunc Register(name string, fn CreatorFn) {\n\tif _, exist := creators[name]; exist {\n\t\tpanic(fmt.Sprintf(\"plugin [%s] is already registered\", name))\n\t}\n\tcreators[name] = fn\n}\n\nfunc Create(name string, options v1.ClientPluginOptions) (p Plugin, err error) {\n\tif fn, ok := creators[name]; ok {\n\t\tp, err = fn(options)\n\t} else {\n\t\terr = fmt.Errorf(\"plugin [%s] is not registered\", name)\n\t}\n\treturn\n}\n\ntype ExtraInfo struct {\n\tProxyProtocolHeader *pp.Header\n\tSrcAddr             net.Addr\n\tDstAddr             net.Addr\n}\n\ntype Plugin interface {\n\tName() string\n\n\tHandle(conn io.ReadWriteCloser, realConn net.Conn, extra *ExtraInfo)\n\tClose() error\n}\n\ntype Listener struct {\n\tconns  chan net.Conn\n\tclosed bool\n\tmu     sync.Mutex\n}\n\nfunc NewProxyListener() *Listener {\n\treturn &Listener{\n\t\tconns: make(chan net.Conn, 64),\n\t}\n}\n\nfunc (l *Listener) Accept() (net.Conn, error) {\n\tconn, ok := <-l.conns\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"listener closed\")\n\t}\n\treturn conn, nil\n}\n\nfunc (l *Listener) PutConn(conn net.Conn) error {\n\terr := errors.PanicToError(func() {\n\t\tl.conns <- conn\n\t})\n\treturn err\n}\n\nfunc (l *Listener) Close() error {\n\tl.mu.Lock()\n\tdefer l.mu.Unlock()\n\tif !l.closed {\n\t\tclose(l.conns)\n\t\tl.closed = true\n\t}\n\treturn nil\n}\n\nfunc (l *Listener) Addr() net.Addr {\n\treturn (*net.TCPAddr)(nil)\n}\n\n// Path: pkg/plugin/client/socks5.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"io\"\n\t\"log\"\n\t\"net\"\n\n\tgosocks5 \"github.com/armon/go-socks5\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginSocks5, NewSocks5Plugin)\n}\n\ntype Socks5Plugin struct {\n\tServer *gosocks5.Server\n}\n\nfunc NewSocks5Plugin(options v1.ClientPluginOptions) (p Plugin, err error) {\n\topts := options.(*v1.Socks5PluginOptions)\n\n\tcfg := &gosocks5.Config{\n\t\tLogger: log.New(io.Discard, \"\", log.LstdFlags),\n\t}\n\tif opts.Username != \"\" || opts.Password != \"\" {\n\t\tcfg.Credentials = gosocks5.StaticCredentials(map[string]string{opts.Username: opts.Password})\n\t}\n\tsp := &Socks5Plugin{}\n\tsp.Server, err = gosocks5.New(cfg)\n\tp = sp\n\treturn\n}\n\nfunc (sp *Socks5Plugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, _ *ExtraInfo) {\n\tdefer conn.Close()\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\t_ = sp.Server.ServeConn(wrapConn)\n}\n\nfunc (sp *Socks5Plugin) Name() string {\n\treturn v1.PluginSocks5\n}\n\nfunc (sp *Socks5Plugin) Close() error {\n\treturn nil\n}\n\n// Path: pkg/util/vhost/router.go\npackage vhost\n\nimport (\n\t\"cmp\"\n\t\"errors\"\n\t\"slices\"\n\t\"strings\"\n\t\"sync\"\n)\n\nvar ErrRouterConfigConflict = errors.New(\"router config conflict\")\n\ntype routerByHTTPUser map[string][]*Router\n\ntype Routers struct {\n\tindexByDomain map[string]routerByHTTPUser\n\n\tmutex sync.RWMutex\n}\n\ntype Router struct {\n\tdomain   string\n\tlocation string\n\thttpUser string\n\n\t// store any object here\n\tpayload interface{}\n}\n\nfunc NewRouters() *Routers {\n\treturn &Routers{\n\t\tindexByDomain: make(map[string]routerByHTTPUser),\n\t}\n}\n\nfunc (r *Routers) Add(domain, location, httpUser string, payload interface{}) error {\n\tdomain = strings.ToLower(domain)\n\n\tr.mutex.Lock()\n\tdefer r.mutex.Unlock()\n\n\tif _, exist := r.exist(domain, location, httpUser); exist {\n\t\treturn ErrRouterConfigConflict\n\t}\n\n\troutersByHTTPUser, found := r.indexByDomain[domain]\n\tif !found {\n\t\troutersByHTTPUser = make(map[string][]*Router)\n\t}\n\tvrs, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\tvrs = make([]*Router, 0, 1)\n\t}\n\n\tvr := &Router{\n\t\tdomain:   domain,\n\t\tlocation: location,\n\t\thttpUser: httpUser,\n\t\tpayload:  payload,\n\t}\n\tvrs = append(vrs, vr)\n\n\tslices.SortFunc(vrs, func(a, b *Router) int {\n\t\treturn -cmp.Compare(a.location, b.location)\n\t})\n\n\troutersByHTTPUser[httpUser] = vrs\n\tr.indexByDomain[domain] = routersByHTTPUser\n\treturn nil\n}\n\nfunc (r *Routers) Del(domain, location, httpUser string) {\n\tdomain = strings.ToLower(domain)\n\n\tr.mutex.Lock()\n\tdefer r.mutex.Unlock()\n\n\troutersByHTTPUser, found := r.indexByDomain[domain]\n\tif !found {\n\t\treturn\n\t}\n\n\tvrs, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\treturn\n\t}\n\tnewVrs := make([]*Router, 0)\n\tfor _, vr := range vrs {\n\t\tif vr.location != location {\n\t\t\tnewVrs = append(newVrs, vr)\n\t\t}\n\t}\n\troutersByHTTPUser[httpUser] = newVrs\n}\n\nfunc (r *Routers) Get(host, path, httpUser string) (vr *Router, exist bool) {\n\thost = strings.ToLower(host)\n\n\tr.mutex.RLock()\n\tdefer r.mutex.RUnlock()\n\n\troutersByHTTPUser, found := r.indexByDomain[host]\n\tif !found {\n\t\treturn\n\t}\n\n\tvrs, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\treturn\n\t}\n\n\tfor _, vr = range vrs {\n\t\tif strings.HasPrefix(path, vr.location) {\n\t\t\treturn vr, true\n\t\t}\n\t}\n\treturn\n}\n\nfunc (r *Routers) exist(host, path, httpUser string) (route *Router, exist bool) {\n\troutersByHTTPUser, found := r.indexByDomain[host]\n\tif !found {\n\t\treturn\n\t}\n\trouters, found := routersByHTTPUser[httpUser]\n\tif !found {\n\t\treturn\n\t}\n\n\tfor _, route = range routers {\n\t\tif path == route.location {\n\t\t\treturn route, true\n\t\t}\n\t}\n\treturn\n}\n\n// Path: pkg/plugin/client/static_file.go\n// Copyright 2018 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/gorilla/mux\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nfunc init() {\n\tRegister(v1.PluginStaticFile, NewStaticFilePlugin)\n}\n\ntype StaticFilePlugin struct {\n\topts *v1.StaticFilePluginOptions\n\n\tl *Listener\n\ts *http.Server\n}\n\nfunc NewStaticFilePlugin(options v1.ClientPluginOptions) (Plugin, error) {\n\topts := options.(*v1.StaticFilePluginOptions)\n\n\tlistener := NewProxyListener()\n\n\tsp := &StaticFilePlugin{\n\t\topts: opts,\n\n\t\tl: listener,\n\t}\n\tvar prefix string\n\tif opts.StripPrefix != \"\" {\n\t\tprefix = \"/\" + opts.StripPrefix + \"/\"\n\t} else {\n\t\tprefix = \"/\"\n\t}\n\n\trouter := mux.NewRouter()\n\trouter.Use(netpkg.NewHTTPAuthMiddleware(opts.HTTPUser, opts.HTTPPassword).SetAuthFailDelay(200 * time.Millisecond).Middleware)\n\trouter.PathPrefix(prefix).Handler(netpkg.MakeHTTPGzipHandler(http.StripPrefix(prefix, http.FileServer(http.Dir(opts.LocalPath))))).Methods(\"GET\")\n\tsp.s = &http.Server{\n\t\tHandler:           router,\n\t\tReadHeaderTimeout: 60 * time.Second,\n\t}\n\tgo func() {\n\t\t_ = sp.s.Serve(listener)\n\t}()\n\treturn sp, nil\n}\n\nfunc (sp *StaticFilePlugin) Handle(conn io.ReadWriteCloser, realConn net.Conn, _ *ExtraInfo) {\n\twrapConn := netpkg.WrapReadWriteCloserToConn(conn, realConn)\n\t_ = sp.l.PutConn(wrapConn)\n}\n\nfunc (sp *StaticFilePlugin) Name() string {\n\treturn v1.PluginStaticFile\n}\n\nfunc (sp *StaticFilePlugin) Close() error {\n\tsp.s.Close()\n\tsp.l.Close()\n\treturn nil\n}\n\n// Path: pkg/plugin/client/unix_domain_socket.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !frps\n\npackage plugin\n\nimport (\n\t\"io\"\n\t\"net\"\n\n\tlibio \"github.com/fatedier/golib/io\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\nfunc init() {\n\tRegister(v1.PluginUnixDomainSocket, NewUnixDomainSocketPlugin)\n}\n\ntype UnixDomainSocketPlugin struct {\n\tUnixAddr *net.UnixAddr\n}\n\nfunc NewUnixDomainSocketPlugin(options v1.ClientPluginOptions) (p Plugin, err error) {\n\topts := options.(*v1.UnixDomainSocketPluginOptions)\n\n\tunixAddr, errRet := net.ResolveUnixAddr(\"unix\", opts.UnixPath)\n\tif errRet != nil {\n\t\terr = errRet\n\t\treturn\n\t}\n\n\tp = &UnixDomainSocketPlugin{\n\t\tUnixAddr: unixAddr,\n\t}\n\treturn\n}\n\nfunc (uds *UnixDomainSocketPlugin) Handle(conn io.ReadWriteCloser, _ net.Conn, extra *ExtraInfo) {\n\tlocalConn, err := net.DialUnix(\"unix\", nil, uds.UnixAddr)\n\tif err != nil {\n\t\treturn\n\t}\n\tif extra.ProxyProtocolHeader != nil {\n\t\tif _, err := extra.ProxyProtocolHeader.WriteTo(localConn); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\tlibio.Join(localConn, conn)\n}\n\nfunc (uds *UnixDomainSocketPlugin) Name() string {\n\treturn v1.PluginUnixDomainSocket\n}\n\nfunc (uds *UnixDomainSocketPlugin) Close() error {\n\treturn nil\n}\n\n// Path: pkg/plugin/server/manager.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n\t\"github.com/fatedier/frp/pkg/util/xlog\"\n)\n\ntype Manager struct {\n\tloginPlugins       []Plugin\n\tnewProxyPlugins    []Plugin\n\tcloseProxyPlugins  []Plugin\n\tpingPlugins        []Plugin\n\tnewWorkConnPlugins []Plugin\n\tnewUserConnPlugins []Plugin\n}\n\nfunc NewManager() *Manager {\n\treturn &Manager{\n\t\tloginPlugins:       make([]Plugin, 0),\n\t\tnewProxyPlugins:    make([]Plugin, 0),\n\t\tcloseProxyPlugins:  make([]Plugin, 0),\n\t\tpingPlugins:        make([]Plugin, 0),\n\t\tnewWorkConnPlugins: make([]Plugin, 0),\n\t\tnewUserConnPlugins: make([]Plugin, 0),\n\t}\n}\n\nfunc (m *Manager) Register(p Plugin) {\n\tif p.IsSupport(OpLogin) {\n\t\tm.loginPlugins = append(m.loginPlugins, p)\n\t}\n\tif p.IsSupport(OpNewProxy) {\n\t\tm.newProxyPlugins = append(m.newProxyPlugins, p)\n\t}\n\tif p.IsSupport(OpCloseProxy) {\n\t\tm.closeProxyPlugins = append(m.closeProxyPlugins, p)\n\t}\n\tif p.IsSupport(OpPing) {\n\t\tm.pingPlugins = append(m.pingPlugins, p)\n\t}\n\tif p.IsSupport(OpNewWorkConn) {\n\t\tm.newWorkConnPlugins = append(m.newWorkConnPlugins, p)\n\t}\n\tif p.IsSupport(OpNewUserConn) {\n\t\tm.newUserConnPlugins = append(m.newUserConnPlugins, p)\n\t}\n}\n\nfunc (m *Manager) Login(content *LoginContent) (*LoginContent, error) {\n\tif len(m.loginPlugins) == 0 {\n\t\treturn content, nil\n\t}\n\n\tvar (\n\t\tres = &Response{\n\t\t\tReject:   false,\n\t\t\tUnchange: true,\n\t\t}\n\t\tretContent interface{}\n\t\terr        error\n\t)\n\treqid, _ := util.RandID()\n\txl := xlog.New().AppendPrefix(\"reqid: \" + reqid)\n\tctx := xlog.NewContext(context.Background(), xl)\n\tctx = NewReqidContext(ctx, reqid)\n\n\tfor _, p := range m.loginPlugins {\n\t\tres, retContent, err = p.Handle(ctx, OpLogin, *content)\n\t\tif err != nil {\n\t\t\txl.Warnf(\"send Login request to plugin [%s] error: %v\", p.Name(), err)\n\t\t\treturn nil, errors.New(\"send Login request to plugin error\")\n\t\t}\n\t\tif res.Reject {\n\t\t\treturn nil, fmt.Errorf(\"%s\", res.RejectReason)\n\t\t}\n\t\tif !res.Unchange {\n\t\t\tcontent = retContent.(*LoginContent)\n\t\t}\n\t}\n\treturn content, nil\n}\n\nfunc (m *Manager) NewProxy(content *NewProxyContent) (*NewProxyContent, error) {\n\tif len(m.newProxyPlugins) == 0 {\n\t\treturn content, nil\n\t}\n\n\tvar (\n\t\tres = &Response{\n\t\t\tReject:   false,\n\t\t\tUnchange: true,\n\t\t}\n\t\tretContent interface{}\n\t\terr        error\n\t)\n\treqid, _ := util.RandID()\n\txl := xlog.New().AppendPrefix(\"reqid: \" + reqid)\n\tctx := xlog.NewContext(context.Background(), xl)\n\tctx = NewReqidContext(ctx, reqid)\n\n\tfor _, p := range m.newProxyPlugins {\n\t\tres, retContent, err = p.Handle(ctx, OpNewProxy, *content)\n\t\tif err != nil {\n\t\t\txl.Warnf(\"send NewProxy request to plugin [%s] error: %v\", p.Name(), err)\n\t\t\treturn nil, errors.New(\"send NewProxy request to plugin error\")\n\t\t}\n\t\tif res.Reject {\n\t\t\treturn nil, fmt.Errorf(\"%s\", res.RejectReason)\n\t\t}\n\t\tif !res.Unchange {\n\t\t\tcontent = retContent.(*NewProxyContent)\n\t\t}\n\t}\n\treturn content, nil\n}\n\nfunc (m *Manager) CloseProxy(content *CloseProxyContent) error {\n\tif len(m.closeProxyPlugins) == 0 {\n\t\treturn nil\n\t}\n\n\terrs := make([]string, 0)\n\treqid, _ := util.RandID()\n\txl := xlog.New().AppendPrefix(\"reqid: \" + reqid)\n\tctx := xlog.NewContext(context.Background(), xl)\n\tctx = NewReqidContext(ctx, reqid)\n\n\tfor _, p := range m.closeProxyPlugins {\n\t\t_, _, err := p.Handle(ctx, OpCloseProxy, *content)\n\t\tif err != nil {\n\t\t\txl.Warnf(\"send CloseProxy request to plugin [%s] error: %v\", p.Name(), err)\n\t\t\terrs = append(errs, fmt.Sprintf(\"[%s]: %v\", p.Name(), err))\n\t\t}\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn fmt.Errorf(\"send CloseProxy request to plugin errors: %s\", strings.Join(errs, \"; \"))\n\t}\n\treturn nil\n}\n\nfunc (m *Manager) Ping(content *PingContent) (*PingContent, error) {\n\tif len(m.pingPlugins) == 0 {\n\t\treturn content, nil\n\t}\n\n\tvar (\n\t\tres = &Response{\n\t\t\tReject:   false,\n\t\t\tUnchange: true,\n\t\t}\n\t\tretContent interface{}\n\t\terr        error\n\t)\n\treqid, _ := util.RandID()\n\txl := xlog.New().AppendPrefix(\"reqid: \" + reqid)\n\tctx := xlog.NewContext(context.Background(), xl)\n\tctx = NewReqidContext(ctx, reqid)\n\n\tfor _, p := range m.pingPlugins {\n\t\tres, retContent, err = p.Handle(ctx, OpPing, *content)\n\t\tif err != nil {\n\t\t\txl.Warnf(\"send Ping request to plugin [%s] error: %v\", p.Name(), err)\n\t\t\treturn nil, errors.New(\"send Ping request to plugin error\")\n\t\t}\n\t\tif res.Reject {\n\t\t\treturn nil, fmt.Errorf(\"%s\", res.RejectReason)\n\t\t}\n\t\tif !res.Unchange {\n\t\t\tcontent = retContent.(*PingContent)\n\t\t}\n\t}\n\treturn content, nil\n}\n\nfunc (m *Manager) NewWorkConn(content *NewWorkConnContent) (*NewWorkConnContent, error) {\n\tif len(m.newWorkConnPlugins) == 0 {\n\t\treturn content, nil\n\t}\n\n\tvar (\n\t\tres = &Response{\n\t\t\tReject:   false,\n\t\t\tUnchange: true,\n\t\t}\n\t\tretContent interface{}\n\t\terr        error\n\t)\n\treqid, _ := util.RandID()\n\txl := xlog.New().AppendPrefix(\"reqid: \" + reqid)\n\tctx := xlog.NewContext(context.Background(), xl)\n\tctx = NewReqidContext(ctx, reqid)\n\n\tfor _, p := range m.newWorkConnPlugins {\n\t\tres, retContent, err = p.Handle(ctx, OpNewWorkConn, *content)\n\t\tif err != nil {\n\t\t\txl.Warnf(\"send NewWorkConn request to plugin [%s] error: %v\", p.Name(), err)\n\t\t\treturn nil, errors.New(\"send NewWorkConn request to plugin error\")\n\t\t}\n\t\tif res.Reject {\n\t\t\treturn nil, fmt.Errorf(\"%s\", res.RejectReason)\n\t\t}\n\t\tif !res.Unchange {\n\t\t\tcontent = retContent.(*NewWorkConnContent)\n\t\t}\n\t}\n\treturn content, nil\n}\n\nfunc (m *Manager) NewUserConn(content *NewUserConnContent) (*NewUserConnContent, error) {\n\tif len(m.newUserConnPlugins) == 0 {\n\t\treturn content, nil\n\t}\n\n\tvar (\n\t\tres = &Response{\n\t\t\tReject:   false,\n\t\t\tUnchange: true,\n\t\t}\n\t\tretContent interface{}\n\t\terr        error\n\t)\n\treqid, _ := util.RandID()\n\txl := xlog.New().AppendPrefix(\"reqid: \" + reqid)\n\tctx := xlog.NewContext(context.Background(), xl)\n\tctx = NewReqidContext(ctx, reqid)\n\n\tfor _, p := range m.newUserConnPlugins {\n\t\tres, retContent, err = p.Handle(ctx, OpNewUserConn, *content)\n\t\tif err != nil {\n\t\t\txl.Infof(\"send NewUserConn request to plugin [%s] error: %v\", p.Name(), err)\n\t\t\treturn nil, errors.New(\"send NewUserConn request to plugin error\")\n\t\t}\n\t\tif res.Reject {\n\t\t\treturn nil, fmt.Errorf(\"%s\", res.RejectReason)\n\t\t}\n\t\tif !res.Unchange {\n\t\t\tcontent = retContent.(*NewUserConnContent)\n\t\t}\n\t}\n\treturn content, nil\n}\n\n// Path: pkg/plugin/server/plugin.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"context\"\n)\n\nconst (\n\tAPIVersion = \"0.1.0\"\n\n\tOpLogin       = \"Login\"\n\tOpNewProxy    = \"NewProxy\"\n\tOpCloseProxy  = \"CloseProxy\"\n\tOpPing        = \"Ping\"\n\tOpNewWorkConn = \"NewWorkConn\"\n\tOpNewUserConn = \"NewUserConn\"\n)\n\ntype Plugin interface {\n\tName() string\n\tIsSupport(op string) bool\n\tHandle(ctx context.Context, op string, content interface{}) (res *Response, retContent interface{}, err error)\n}\n\n// Path: pkg/plugin/server/tracer.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"context\"\n)\n\ntype key int\n\nconst (\n\treqidKey key = 0\n)\n\n\nfunc NewReqidContext(ctx context.Context, reqid string) context.Context {\n\treturn context.WithValue(ctx, reqidKey, reqid)\n}\n\nfunc GetReqidFromContext(ctx context.Context) string {\n\tret, _ := ctx.Value(reqidKey).(string)\n\treturn ret\n}\n\n// Path: pkg/proto/udp/udp.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage udp\n\nimport (\n\t\"encoding/base64\"\n\t\"net\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/fatedier/golib/errors\"\n\t\"github.com/fatedier/golib/pool\"\n\n\t\"github.com/fatedier/frp/pkg/msg\"\n)\n\nfunc NewUDPPacket(buf []byte, laddr, raddr *net.UDPAddr) *msg.UDPPacket {\n\treturn &msg.UDPPacket{\n\t\tContent:    base64.StdEncoding.EncodeToString(buf),\n\t\tLocalAddr:  laddr,\n\t\tRemoteAddr: raddr,\n\t}\n}\n\nfunc GetContent(m *msg.UDPPacket) (buf []byte, err error) {\n\tbuf, err = base64.StdEncoding.DecodeString(m.Content)\n\treturn\n}\n\nfunc ForwardUserConn(udpConn *net.UDPConn, readCh <-chan *msg.UDPPacket, sendCh chan<- *msg.UDPPacket, bufSize int) {\n\t// read\n\tgo func() {\n\t\tfor udpMsg := range readCh {\n\t\t\tbuf, err := GetContent(udpMsg)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_, _ = udpConn.WriteToUDP(buf, udpMsg.RemoteAddr)\n\t\t}\n\t}()\n\n\t// write\n\tbuf := pool.GetBuf(bufSize)\n\tdefer pool.PutBuf(buf)\n\tfor {\n\t\tn, remoteAddr, err := udpConn.ReadFromUDP(buf)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\t// buf[:n] will be encoded to string, so the bytes can be reused\n\t\tudpMsg := NewUDPPacket(buf[:n], nil, remoteAddr)\n\n\t\tselect {\n\t\tcase sendCh <- udpMsg:\n\t\tdefault:\n\t\t}\n\t}\n}\n\nfunc Forwarder(dstAddr *net.UDPAddr, readCh <-chan *msg.UDPPacket, sendCh chan<- msg.Message, bufSize int) {\n\tvar mu sync.RWMutex\n\tudpConnMap := make(map[string]*net.UDPConn)\n\n\t// read from dstAddr and write to sendCh\n\twriterFn := func(raddr *net.UDPAddr, udpConn *net.UDPConn) {\n\t\taddr := raddr.String()\n\t\tdefer func() {\n\t\t\tmu.Lock()\n\t\t\tdelete(udpConnMap, addr)\n\t\t\tmu.Unlock()\n\t\t\tudpConn.Close()\n\t\t}()\n\n\t\tbuf := pool.GetBuf(bufSize)\n\t\tfor {\n\t\t\t_ = udpConn.SetReadDeadline(time.Now().Add(30 * time.Second))\n\t\t\tn, _, err := udpConn.ReadFromUDP(buf)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tudpMsg := NewUDPPacket(buf[:n], nil, raddr)\n\t\t\tif err = errors.PanicToError(func() {\n\t\t\t\tselect {\n\t\t\t\tcase sendCh <- udpMsg:\n\t\t\t\tdefault:\n\t\t\t\t}\n\t\t\t}); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\t// read from readCh\n\tgo func() {\n\t\tfor udpMsg := range readCh {\n\t\t\tbuf, err := GetContent(udpMsg)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmu.Lock()\n\t\t\tudpConn, ok := udpConnMap[udpMsg.RemoteAddr.String()]\n\t\t\tif !ok {\n\t\t\t\tudpConn, err = net.DialUDP(\"udp\", nil, dstAddr)\n\t\t\t\tif err != nil {\n\t\t\t\t\tmu.Unlock()\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tudpConnMap[udpMsg.RemoteAddr.String()] = udpConn\n\t\t\t}\n\t\t\tmu.Unlock()\n\n\t\t\t_, err = udpConn.Write(buf)\n\t\t\tif err != nil {\n\t\t\t\tudpConn.Close()\n\t\t\t}\n\n\t\t\tif !ok {\n\t\t\t\tgo writerFn(udpMsg.RemoteAddr, udpConn)\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// Path: pkg/proto/udp/udp_test.go\npackage udp\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestUdpPacket(t *testing.T) {\n\tassert := assert.New(t)\n\n\tbuf := []byte(\"hello world\")\n\tudpMsg := NewUDPPacket(buf, nil, nil)\n\n\tnewBuf, err := GetContent(udpMsg)\n\tassert.NoError(err)\n\tassert.EqualValues(buf, newBuf)\n}\n\n// Path: pkg/ssh/gateway.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage ssh\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"golang.org/x/crypto/ssh\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/transport\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\ntype Gateway struct {\n\tbindPort int\n\tln       net.Listener\n\n\tpeerServerListener *netpkg.InternalListener\n\n\tsshConfig *ssh.ServerConfig\n}\n\nfunc NewGateway(\n\tcfg v1.SSHTunnelGateway, bindAddr string,\n\tpeerServerListener *netpkg.InternalListener,\n) (*Gateway, error) {\n\tsshConfig := &ssh.ServerConfig{}\n\n\t// privateKey\n\tvar (\n\t\tprivateKeyBytes []byte\n\t\terr             error\n\t)\n\tif cfg.PrivateKeyFile != \"\" {\n\t\tprivateKeyBytes, err = os.ReadFile(cfg.PrivateKeyFile)\n\t} else {\n\t\tif cfg.AutoGenPrivateKeyPath != \"\" {\n\t\t\tprivateKeyBytes, _ = os.ReadFile(cfg.AutoGenPrivateKeyPath)\n\t\t}\n\t\tif len(privateKeyBytes) == 0 {\n\t\t\tprivateKeyBytes, err = transport.NewRandomPrivateKey()\n\t\t\tif err == nil && cfg.AutoGenPrivateKeyPath != \"\" {\n\t\t\t\terr = os.WriteFile(cfg.AutoGenPrivateKeyPath, privateKeyBytes, 0o600)\n\t\t\t}\n\t\t}\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tprivateKey, err := ssh.ParsePrivateKey(privateKeyBytes)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n...\n", "needle_token_start": 12881, "needle_token_end": 12920, "code_context_ntokens": 15173, "output": ["```go\nfunc NewReqidContext(ctx context.Context, reqid string) context.Context {\n\treturn context.WithValue(ctx, reqidKey, reqid)\n}\n```"]}
{"repo": "fatedier/frp", "name": "ReadMsg", "language": "go", "path": "pkg/msg/ctl.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To decode a message from a data stream using a predefined set of message types.\n2. **Input**: An `io.Reader` interface which provides a stream of bytes.\n3. **Output**: Returns a decoded message object and an error if there is any issue during the reading or decoding process.\n4. **Procedure**: The function utilizes a control structure initialized with various message types to read and decode a message from the provided byte stream. It handles the decoding based on the registered message types and returns the appropriate message object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/transport/tls.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage transport\n\nimport (\n\t\"crypto/rand\"\n\t\"crypto/rsa\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/pem\"\n\t\"math/big\"\n\t\"os\"\n)\n\nfunc newCustomTLSKeyPair(certfile, keyfile string) (*tls.Certificate, error) {\n\ttlsCert, err := tls.LoadX509KeyPair(certfile, keyfile)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &tlsCert, nil\n}\n\nfunc newRandomTLSKeyPair() *tls.Certificate {\n\tkey, err := rsa.GenerateKey(rand.Reader, 2048)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttemplate := x509.Certificate{SerialNumber: big.NewInt(1)}\n\tcertDER, err := x509.CreateCertificate(\n\t\trand.Reader,\n\t\t&template,\n\t\t&template,\n\t\t&key.PublicKey,\n\t\tkey)\n\tif err != nil {\n\t\tpanic(err)\n...\n// Path: pkg/util/xlog/ctx.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage xlog\n\nimport (\n\t\"context\"\n)\n\ntype key int\n\nconst (\n\txlogKey key = 0\n)\n\nfunc NewContext(ctx context.Context, xl *Logger) context.Context {\n\treturn context.WithValue(ctx, xlogKey, xl)\n}\n\nfunc FromContext(ctx context.Context) (xl *Logger, ok bool) {\n\txl, ok = ctx.Value(xlogKey).(*Logger)\n\treturn\n}\n\nfunc FromContextSafe(ctx context.Context) *Logger {\n\txl, ok := ctx.Value(xlogKey).(*Logger)\n\tif !ok {\n\t\txl = New()\n\t}\n\treturn xl\n}\n\n// Path: pkg/virtual/client.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage virtual\n\nimport (\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/fatedier/frp/client\"\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\t\"github.com/fatedier/frp/pkg/msg\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\ntype ClientOptions struct {\n\tCommon           *v1.ClientCommonConfig\n\tSpec             *msg.ClientSpec\n\tHandleWorkConnCb func(*v1.ProxyBaseConfig, net.Conn, *msg.StartWorkConn) bool\n}\n\ntype Client struct {\n\tl   *netpkg.InternalListener\n\tsvr *client.Service\n}\n\nfunc NewClient(options ClientOptions) (*Client, error) {\n\tif options.Common != nil {\n\t\toptions.Common.Complete()\n\t}\n\n\tln := netpkg.NewInternalListener()\n\n\tserviceOptions := client.ServiceOptions{\n\t\tCommon:     options.Common,\n\t\tClientSpec: options.Spec,\n\t\tConnectorCreator: func(context.Context, *v1.ClientCommonConfig) client.Connector {\n\t\t\treturn &pipeConnector{\n\t\t\t\tpeerListener: ln,\n\t\t\t}\n\t\t},\n\t\tHandleWorkConnCb: options.HandleWorkConnCb,\n\t}\n\tsvr, err := client.NewService(serviceOptions)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Client{\n\t\tl:   ln,\n\t\tsvr: svr,\n\t}, nil\n}\n\nfunc (c *Client) PeerListener() net.Listener {\n\treturn c.l\n}\n\nfunc (c *Client) UpdateProxyConfigurer(proxyCfgs []v1.ProxyConfigurer) {\n\t_ = c.svr.UpdateAllConfigurer(proxyCfgs, nil)\n}\n\nfunc (c *Client) Run(ctx context.Context) error {\n\treturn c.svr.Run(ctx)\n}\n\nfunc (c *Client) Service() *client.Service {\n\treturn c.svr\n}\n\nfunc (c *Client) Close() {\n\tc.svr.Close()\n\tc.l.Close()\n}\n\ntype pipeConnector struct {\n\tpeerListener *netpkg.InternalListener\n}\n\nfunc (pc *pipeConnector) Open() error {\n\treturn nil\n}\n\nfunc (pc *pipeConnector) Connect() (net.Conn, error) {\n\tc1, c2 := net.Pipe()\n\tif err := pc.peerListener.PutConn(c1); err != nil {\n\t\tc1.Close()\n\t\tc2.Close()\n\t\treturn nil, err\n\t}\n\treturn c2, nil\n}\n\nfunc (pc *pipeConnector) Close() error {\n\tpc.peerListener.Close()\n\treturn nil\n}\n\n// Path: pkg/util/net/http.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage net\n\nimport (\n\t\"compress/gzip\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/fatedier/frp/pkg/util/util\"\n)\n\ntype HTTPAuthMiddleware struct {\n\tuser          string\n\tpasswd        string\n\tauthFailDelay time.Duration\n}\n\nfunc NewHTTPAuthMiddleware(user, passwd string) *HTTPAuthMiddleware {\n\treturn &HTTPAuthMiddleware{\n\t\tuser:   user,\n\t\tpasswd: passwd,\n\t}\n}\n\nfunc (authMid *HTTPAuthMiddleware) SetAuthFailDelay(delay time.Duration) *HTTPAuthMiddleware {\n\tauthMid.authFailDelay = delay\n\treturn authMid\n}\n\nfunc (authMid *HTTPAuthMiddleware) Middleware(next http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\treqUser, reqPasswd, hasAuth := r.BasicAuth()\n\t\tif (authMid.user == \"\" && authMid.passwd == \"\") ||\n\t\t\t(hasAuth && util.ConstantTimeEqString(reqUser, authMid.user) &&\n\t\t\t\tutil.ConstantTimeEqString(reqPasswd, authMid.passwd)) {\n\t\t\tnext.ServeHTTP(w, r)\n\t\t} else {\n\t\t\tif authMid.authFailDelay > 0 {\n\t\t\t\ttime.Sleep(authMid.authFailDelay)\n\t\t\t}\n\t\t\tw.Header().Set(\"WWW-Authenticate\", `Basic realm=\"Restricted\"`)\n\t\t\thttp.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)\n\t\t}\n\t})\n}\n\ntype HTTPGzipWrapper struct {\n\th http.Handler\n}\n\nfunc (gw *HTTPGzipWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tif !strings.Contains(r.Header.Get(\"Accept-Encoding\"), \"gzip\") {\n\t\tgw.h.ServeHTTP(w, r)\n\t\treturn\n\t}\n\tw.Header().Set(\"Content-Encoding\", \"gzip\")\n\tgz := gzip.NewWriter(w)\n\tdefer gz.Close()\n\tgzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}\n\tgw.h.ServeHTTP(gzr, r)\n}\n\nfunc MakeHTTPGzipHandler(h http.Handler) http.Handler {\n\treturn &HTTPGzipWrapper{\n\t\th: h,\n\t}\n}\n\ntype gzipResponseWriter struct {\n\tio.Writer\n\thttp.ResponseWriter\n}\n\nfunc (w gzipResponseWriter) Write(b []byte) (int, error) {\n\treturn w.Writer.Write(b)\n}\n\n// Path: pkg/util/http/http.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage http\n\nimport (\n\t\"encoding/base64\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n)\n\nfunc OkResponse() *http.Response {\n\theader := make(http.Header)\n\n\tres := &http.Response{\n\t\tStatus:     \"OK\",\n\t\tStatusCode: 200,\n\t\tProto:      \"HTTP/1.1\",\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 1,\n\t\tHeader:     header,\n\t}\n\treturn res\n}\n\nfunc ProxyUnauthorizedResponse() *http.Response {\n\theader := make(http.Header)\n\theader.Set(\"Proxy-Authenticate\", `Basic realm=\"Restricted\"`)\n\tres := &http.Response{\n\t\tStatus:     \"Proxy Authentication Required\",\n\t\tStatusCode: 407,\n\t\tProto:      \"HTTP/1.1\",\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 1,\n\t\tHeader:     header,\n\t}\n\treturn res\n}\n\n// canonicalHost strips port from host if present and returns the canonicalized\n// host name.\nfunc CanonicalHost(host string) (string, error) {\n\tvar err error\n\thost = strings.ToLower(host)\n\tif hasPort(host) {\n\t\thost, _, err = net.SplitHostPort(host)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t}\n\t// Strip trailing dot from fully qualified domain names.\n\thost = strings.TrimSuffix(host, \".\")\n\treturn host, nil\n}\n\n// hasPort reports whether host contains a port number. host may be a host\n// name, an IPv4 or an IPv6 address.\nfunc hasPort(host string) bool {\n\tcolons := strings.Count(host, \":\")\n\tif colons == 0 {\n\t\treturn false\n\t}\n\tif colons == 1 {\n\t\treturn true\n\t}\n\treturn host[0] == '[' && strings.Contains(host, \"]:\")\n}\n\nfunc ParseBasicAuth(auth string) (username, password string, ok bool) {\n\tconst prefix = \"Basic \"\n\t// Case insensitive prefix match. See Issue 22736.\n\tif len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {\n\t\treturn\n\t}\n\tc, err := base64.StdEncoding.DecodeString(auth[len(prefix):])\n\tif err != nil {\n\t\treturn\n\t}\n\tcs := string(c)\n\ts := strings.IndexByte(cs, ':')\n\tif s < 0 {\n\t\treturn\n\t}\n\treturn cs[:s], cs[s+1:], true\n}\n\nfunc BasicAuth(username, passwd string) string {\n\tauth := username + \":\" + passwd\n\treturn \"Basic \" + base64.StdEncoding.EncodeToString([]byte(auth))\n}\n\n// Path: pkg/sdk/client/client.go\npackage client\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/fatedier/frp/client\"\n\thttppkg \"github.com/fatedier/frp/pkg/util/http\"\n)\n\ntype Client struct {\n\taddress  string\n\tauthUser string\n\tauthPwd  string\n}\n\nfunc New(host string, port int) *Client {\n\treturn &Client{\n\t\taddress: net.JoinHostPort(host, strconv.Itoa(port)),\n\t}\n}\n\nfunc (c *Client) SetAuth(user, pwd string) {\n\tc.authUser = user\n\tc.authPwd = pwd\n}\n\nfunc (c *Client) GetProxyStatus(name string) (*client.ProxyStatusResp, error) {\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/status\", nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcontent, err := c.do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tallStatus := make(client.StatusResp)\n\tif err = json.Unmarshal([]byte(content), &allStatus); err != nil {\n\t\treturn nil, fmt.Errorf(\"unmarshal http response error: %s\", strings.TrimSpace(content))\n\t}\n\tfor _, pss := range allStatus {\n\t\tfor _, ps := range pss {\n\t\t\tif ps.Name == name {\n\t\t\t\treturn &ps, nil\n\t\t\t}\n\t\t}\n\t}\n\treturn nil, fmt.Errorf(\"no proxy status found\")\n}\n\nfunc (c *Client) GetAllProxyStatus() (client.StatusResp, error) {\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/status\", nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcontent, err := c.do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tallStatus := make(client.StatusResp)\n\tif err = json.Unmarshal([]byte(content), &allStatus); err != nil {\n\t\treturn nil, fmt.Errorf(\"unmarshal http response error: %s\", strings.TrimSpace(content))\n\t}\n\treturn allStatus, nil\n}\n\nfunc (c *Client) Reload(strictMode bool) error {\n\tv := url.Values{}\n\tif strictMode {\n\t\tv.Set(\"strictConfig\", \"true\")\n\t}\n\tqueryStr := \"\"\n\tif len(v) > 0 {\n\t\tqueryStr = \"?\" + v.Encode()\n\t}\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/reload\"+queryStr, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = c.do(req)\n\treturn err\n}\n\nfunc (c *Client) Stop() error {\n\treq, err := http.NewRequest(\"POST\", \"http://\"+c.address+\"/api/stop\", nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = c.do(req)\n\treturn err\n}\n\nfunc (c *Client) GetConfig() (string, error) {\n\treq, err := http.NewRequest(\"GET\", \"http://\"+c.address+\"/api/config\", nil)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn c.do(req)\n}\n\nfunc (c *Client) UpdateConfig(content string) error {\n\treq, err := http.NewRequest(\"PUT\", \"http://\"+c.address+\"/api/config\", strings.NewReader(content))\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = c.do(req)\n\treturn err\n}\n\nfunc (c *Client) setAuthHeader(req *http.Request) {\n\tif c.authUser != \"\" || c.authPwd != \"\" {\n\t\treq.Header.Set(\"Authorization\", httppkg.BasicAuth(c.authUser, c.authPwd))\n\t}\n}\n\nfunc (c *Client) do(req *http.Request) (string, error) {\n\tc.setAuthHeader(req)\n\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != 200 {\n\t\treturn \"\", fmt.Errorf(\"api status code [%d]\", resp.StatusCode)\n\t}\n\tbuf, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn string(buf), nil\n}\n\n// Path: pkg/util/vhost/http.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage vhost\n\nimport (\n\t\"context\"\n\t\"encoding/base64\"\n\t\"errors\"\n\t\"fmt\"\n\tstdlog \"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\tlibio \"github.com/fatedier/golib/io\"\n\t\"github.com/fatedier/golib/pool\"\n\n\thttppkg \"github.com/fatedier/frp/pkg/util/http\"\n\t\"github.com/fatedier/frp/pkg/util/log\"\n)\n\nvar ErrNoRouteFound = errors.New(\"no route found\")\n\ntype HTTPReverseProxyOptions struct {\n\tResponseHeaderTimeoutS int64\n}\n\ntype HTTPReverseProxy struct {\n\tproxy       *httputil.ReverseProxy\n\tvhostRouter *Routers\n\n\tresponseHeaderTimeout time.Duration\n}\n\nfunc NewHTTPReverseProxy(option HTTPReverseProxyOptions, vhostRouter *Routers) *HTTPReverseProxy {\n\tif option.ResponseHeaderTimeoutS <= 0 {\n\t\toption.ResponseHeaderTimeoutS = 60\n\t}\n\trp := &HTTPReverseProxy{\n\t\tresponseHeaderTimeout: time.Duration(option.ResponseHeaderTimeoutS) * time.Second,\n\t\tvhostRouter:           vhostRouter,\n\t}\n\tproxy := &httputil.ReverseProxy{\n\t\t// Modify incoming requests by route policies.\n\t\tRewrite: func(r *httputil.ProxyRequest) {\n\t\t\tr.Out.Header[\"X-Forwarded-For\"] = r.In.Header[\"X-Forwarded-For\"]\n\t\t\tr.SetXForwarded()\n\t\t\treq := r.Out\n\t\t\treq.URL.Scheme = \"http\"\n\t\t\treqRouteInfo := req.Context().Value(RouteInfoKey).(*RequestRouteInfo)\n\t\t\toldHost, _ := httppkg.CanonicalHost(reqRouteInfo.Host)\n\n\t\t\trc := rp.GetRouteConfig(oldHost, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n\t\t\tif rc != nil {\n\t\t\t\tif rc.RewriteHost != \"\" {\n\t\t\t\t\treq.Host = rc.RewriteHost\n\t\t\t\t}\n\n\t\t\t\tvar endpoint string\n\t\t\t\tif rc.ChooseEndpointFn != nil {\n\t\t\t\t\t// ignore error here, it will use CreateConnFn instead later\n\t\t\t\t\tendpoint, _ = rc.ChooseEndpointFn()\n\t\t\t\t\treqRouteInfo.Endpoint = endpoint\n\t\t\t\t\tlog.Tracef(\"choose endpoint name [%s] for http request host [%s] path [%s] httpuser [%s]\",\n\t\t\t\t\t\tendpoint, oldHost, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n\t\t\t\t}\n\t\t\t\t// Set {domain}.{location}.{routeByHTTPUser}.{endpoint} as URL host here to let http transport reuse connections.\n\t\t\t\treq.URL.Host = rc.Domain + \".\" +\n\t\t\t\t\tbase64.StdEncoding.EncodeToString([]byte(rc.Location)) + \".\" +\n\t\t\t\t\tbase64.StdEncoding.EncodeToString([]byte(rc.RouteByHTTPUser)) + \".\" +\n\t\t\t\t\tbase64.StdEncoding.EncodeToString([]byte(endpoint))\n\n\t\t\t\tfor k, v := range rc.Headers {\n\t\t\t\t\treq.Header.Set(k, v)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treq.URL.Host = req.Host\n\t\t\t}\n\t\t},\n\t\t// Create a connection to one proxy routed by route policy.\n\t\tTransport: &http.Transport{\n\t\t\tResponseHeaderTimeout: rp.responseHeaderTimeout,\n\t\t\tIdleConnTimeout:       60 * time.Second,\n\t\t\tMaxIdleConnsPerHost:   5,\n\t\t\tDialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {\n\t\t\t\treturn rp.CreateConnection(ctx.Value(RouteInfoKey).(*RequestRouteInfo), true)\n\t\t\t},\n\t\t\tProxy: func(req *http.Request) (*url.URL, error) {\n\t\t\t\t// Use proxy mode if there is host in HTTP first request line.\n\t\t\t\t// GET http://example.com/ HTTP/1.1\n\t\t\t\t// Host: example.com\n\t\t\t\t//\n\t\t\t\t// Normal:\n\t\t\t\t// GET / HTTP/1.1\n\t\t\t\t// Host: example.com\n\t\t\t\turlHost := req.Context().Value(RouteInfoKey).(*RequestRouteInfo).URLHost\n\t\t\t\tif urlHost != \"\" {\n\t\t\t\t\treturn req.URL, nil\n\t\t\t\t}\n\t\t\t\treturn nil, nil\n\t\t\t},\n\t\t},\n\t\tBufferPool: pool.NewBuffer(32 * 1024),\n\t\tErrorLog:   stdlog.New(log.NewWriteLogger(log.WarnLevel, 2), \"\", 0),\n\t\tErrorHandler: func(rw http.ResponseWriter, req *http.Request, err error) {\n\t\t\tlog.Logf(log.WarnLevel, 1, \"do http proxy request [host: %s] error: %v\", req.Host, err)\n\t\t\tif err != nil {\n\t\t\t\tif e, ok := err.(net.Error); ok && e.Timeout() {\n\t\t\t\t\trw.WriteHeader(http.StatusGatewayTimeout)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\trw.WriteHeader(http.StatusNotFound)\n\t\t\t_, _ = rw.Write(getNotFoundPageContent())\n\t\t},\n\t}\n\trp.proxy = proxy\n\treturn rp\n}\n\n// Register register the route config to reverse proxy\n// reverse proxy will use CreateConnFn from routeCfg to create a connection to the remote service\nfunc (rp *HTTPReverseProxy) Register(routeCfg RouteConfig) error {\n\terr := rp.vhostRouter.Add(routeCfg.Domain, routeCfg.Location, routeCfg.RouteByHTTPUser, &routeCfg)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// UnRegister unregister route config by domain and location\nfunc (rp *HTTPReverseProxy) UnRegister(routeCfg RouteConfig) {\n\trp.vhostRouter.Del(routeCfg.Domain, routeCfg.Location, routeCfg.RouteByHTTPUser)\n}\n\nfunc (rp *HTTPReverseProxy) GetRouteConfig(domain, location, routeByHTTPUser string) *RouteConfig {\n\tvr, ok := rp.getVhost(domain, location, routeByHTTPUser)\n\tif ok {\n\t\tlog.Debugf(\"get new HTTP request host [%s] path [%s] httpuser [%s]\", domain, location, routeByHTTPUser)\n\t\treturn vr.payload.(*RouteConfig)\n\t}\n\treturn nil\n}\n\nfunc (rp *HTTPReverseProxy) GetHeaders(domain, location, routeByHTTPUser string) (headers map[string]string) {\n\tvr, ok := rp.getVhost(domain, location, routeByHTTPUser)\n\tif ok {\n\t\theaders = vr.payload.(*RouteConfig).Headers\n\t}\n\treturn\n}\n\n// CreateConnection create a new connection by route config\nfunc (rp *HTTPReverseProxy) CreateConnection(reqRouteInfo *RequestRouteInfo, byEndpoint bool) (net.Conn, error) {\n\thost, _ := httppkg.CanonicalHost(reqRouteInfo.Host)\n\tvr, ok := rp.getVhost(host, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n\tif ok {\n\t\tif byEndpoint {\n\t\t\tfn := vr.payload.(*RouteConfig).CreateConnByEndpointFn\n\t\t\tif fn != nil {\n\t\t\t\treturn fn(reqRouteInfo.Endpoint, reqRouteInfo.RemoteAddr)\n\t\t\t}\n\t\t}\n\t\tfn := vr.payload.(*RouteConfig).CreateConnFn\n\t\tif fn != nil {\n\t\t\treturn fn(reqRouteInfo.RemoteAddr)\n\t\t}\n\t}\n\treturn nil, fmt.Errorf(\"%v: %s %s %s\", ErrNoRouteFound, host, reqRouteInfo.URL, reqRouteInfo.HTTPUser)\n}\n\nfunc (rp *HTTPReverseProxy) CheckAuth(domain, location, routeByHTTPUser, user, passwd string) bool {\n\tvr, ok := rp.getVhost(domain, location, routeByHTTPUser)\n\tif ok {\n\t\tcheckUser := vr.payload.(*RouteConfig).Username\n\t\tcheckPasswd := vr.payload.(*RouteConfig).Password\n\t\tif (checkUser != \"\" || checkPasswd != \"\") && (checkUser != user || checkPasswd != passwd) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// getVhost tries to get vhost router by route policy.\nfunc (rp *HTTPReverseProxy) getVhost(domain, location, routeByHTTPUser string) (*Router, bool) {\n\tfindRouter := func(inDomain, inLocation, inRouteByHTTPUser string) (*Router, bool) {\n\t\tvr, ok := rp.vhostRouter.Get(inDomain, inLocation, inRouteByHTTPUser)\n\t\tif ok {\n\t\t\treturn vr, ok\n\t\t}\n\t\t// Try to check if there is one proxy that doesn't specify routerByHTTPUser, it means match all.\n\t\tvr, ok = rp.vhostRouter.Get(inDomain, inLocation, \"\")\n\t\tif ok {\n\t\t\treturn vr, ok\n\t\t}\n\t\treturn nil, false\n\t}\n\n\t// First we check the full hostname\n\t// if not exist, then check the wildcard_domain such as *.example.com\n\tvr, ok := findRouter(domain, location, routeByHTTPUser)\n\tif ok {\n\t\treturn vr, ok\n\t}\n\n\t// e.g. domain = test.example.com, try to match wildcard domains.\n\t// *.example.com\n\t// *.com\n\tdomainSplit := strings.Split(domain, \".\")\n\tfor {\n\t\tif len(domainSplit) < 3 {\n\t\t\tbreak\n\t\t}\n\n\t\tdomainSplit[0] = \"*\"\n\t\tdomain = strings.Join(domainSplit, \".\")\n\t\tvr, ok = findRouter(domain, location, routeByHTTPUser)\n\t\tif ok {\n\t\t\treturn vr, true\n\t\t}\n\t\tdomainSplit = domainSplit[1:]\n\t}\n\n\t// Finally, try to check if there is one proxy that domain is \"*\" means match all domains.\n\tvr, ok = findRouter(\"*\", location, routeByHTTPUser)\n\tif ok {\n\t\treturn vr, true\n\t}\n\treturn nil, false\n}\n\nfunc (rp *HTTPReverseProxy) connectHandler(rw http.ResponseWriter, req *http.Request) {\n\thj, ok := rw.(http.Hijacker)\n\tif !ok {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tclient, _, err := hj.Hijack()\n\tif err != nil {\n\t\trw.WriteHeader(http.StatusInternalServerError)\n\t\treturn\n\t}\n\n\tremote, err := rp.CreateConnection(req.Context().Value(RouteInfoKey).(*RequestRouteInfo), false)\n\tif err != nil {\n\t\t_ = NotFoundResponse().Write(client)\n\t\tclient.Close()\n\t\treturn\n\t}\n\t_ = req.Write(remote)\n\tgo libio.Join(remote, client)\n}\n\nfunc parseBasicAuth(auth string) (username, password string, ok bool) {\n\tconst prefix = \"Basic \"\n\t// Case insensitive prefix match. See Issue 22736.\n\tif len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {\n\t\treturn\n\t}\n\tc, err := base64.StdEncoding.DecodeString(auth[len(prefix):])\n\tif err != nil {\n\t\treturn\n\t}\n\tcs := string(c)\n\ts := strings.IndexByte(cs, ':')\n\tif s < 0 {\n\t\treturn\n\t}\n\treturn cs[:s], cs[s+1:], true\n}\n\nfunc (rp *HTTPReverseProxy) injectRequestInfoToCtx(req *http.Request) *http.Request {\n\tuser := \"\"\n\t// If url host isn't empty, it's a proxy request. Get http user from Proxy-Authorization header.\n\tif req.URL.Host != \"\" {\n\t\tproxyAuth := req.Header.Get(\"Proxy-Authorization\")\n\t\tif proxyAuth != \"\" {\n\t\t\tuser, _, _ = parseBasicAuth(proxyAuth)\n\t\t}\n\t}\n\tif user == \"\" {\n\t\tuser, _, _ = req.BasicAuth()\n\t}\n\n\treqRouteInfo := &RequestRouteInfo{\n\t\tURL:        req.URL.Path,\n\t\tHost:       req.Host,\n\t\tHTTPUser:   user,\n\t\tRemoteAddr: req.RemoteAddr,\n\t\tURLHost:    req.URL.Host,\n\t}\n\tnewctx := req.Context()\n\tnewctx = context.WithValue(newctx, RouteInfoKey, reqRouteInfo)\n\treturn req.Clone(newctx)\n}\n\nfunc (rp *HTTPReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {\n\tdomain, _ := httppkg.CanonicalHost(req.Host)\n\tlocation := req.URL.Path\n\tuser, passwd, _ := req.BasicAuth()\n\tif !rp.CheckAuth(domain, location, user, user, passwd) {\n\t\trw.Header().Set(\"WWW-Authenticate\", `Basic realm=\"Restricted\"`)\n\t\thttp.Error(rw, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)\n\t\treturn\n\t}\n\n\tnewreq := rp.injectRequestInfoToCtx(req)\n\tif req.Method == http.MethodConnect {\n\t\trp.connectHandler(rw, newreq)\n\t} else {\n\t\trp.proxy.ServeHTTP(rw, newreq)\n\t}\n}\n\n// Path: pkg/plugin/server/http.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage plugin\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"reflect\"\n\t\"strings\"\n\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n)\n\ntype httpPlugin struct {\n\toptions v1.HTTPPluginOptions\n\n\turl    string\n\tclient *http.Client\n}\n\nfunc NewHTTPPluginOptions(options v1.HTTPPluginOptions) Plugin {\n\turl := fmt.Sprintf(\"%s%s\", options.Addr, options.Path)\n\n\tvar client *http.Client\n\tif strings.HasPrefix(url, \"https://\") {\n\t\ttr := &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: !options.TLSVerify},\n\t\t}\n\t\tclient = &http.Client{Transport: tr}\n\t} else {\n\t\tclient = &http.Client{}\n\t}\n\n\tif !strings.HasPrefix(url, \"https://\") && !strings.HasPrefix(url, \"http://\") {\n\t\turl = \"http://\" + url\n\t}\n\treturn &httpPlugin{\n\t\toptions: options,\n\t\turl:     url,\n\t\tclient:  client,\n\t}\n}\n\nfunc (p *httpPlugin) Name() string {\n\treturn p.options.Name\n}\n\nfunc (p *httpPlugin) IsSupport(op string) bool {\n\tfor _, v := range p.options.Ops {\n\t\tif v == op {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (p *httpPlugin) Handle(ctx context.Context, op string, content interface{}) (*Response, interface{}, error) {\n\tr := &Request{\n\t\tVersion: APIVersion,\n\t\tOp:      op,\n\t\tContent: content,\n\t}\n\tvar res Response\n\tres.Content = reflect.New(reflect.TypeOf(content)).Interface()\n\tif err := p.do(ctx, r, &res); err != nil {\n\t\treturn nil, nil, err\n\t}\n\treturn &res, res.Content, nil\n}\n\nfunc (p *httpPlugin) do(ctx context.Context, r *Request, res *Response) error {\n\tbuf, err := json.Marshal(r)\n\tif err != nil {\n\t\treturn err\n\t}\n\tv := url.Values{}\n\tv.Set(\"version\", r.Version)\n\tv.Set(\"op\", r.Op)\n\treq, err := http.NewRequest(\"POST\", p.url+\"?\"+v.Encode(), bytes.NewReader(buf))\n\tif err != nil {\n\t\treturn err\n\t}\n\treq = req.WithContext(ctx)\n\treq.Header.Set(\"X-Frp-Reqid\", GetReqidFromContext(ctx))\n\treq.Header.Set(\"Content-Type\", \"application/json\")\n\tresp, err := p.client.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn fmt.Errorf(\"do http request error code: %d\", resp.StatusCode)\n\t}\n\tbuf, err = io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn json.Unmarshal(buf, res)\n}\n\n// Path: pkg/util/http/server.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage http\n\nimport (\n\t\"crypto/tls\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/pprof\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/gorilla/mux\"\n\n\t\"github.com/fatedier/frp/assets\"\n\tv1 \"github.com/fatedier/frp/pkg/config/v1\"\n\tnetpkg \"github.com/fatedier/frp/pkg/util/net\"\n)\n\nvar (\n\tdefaultReadTimeout  = 60 * time.Second\n\tdefaultWriteTimeout = 60 * time.Second\n)\n\ntype Server struct {\n\taddr   string\n\tln     net.Listener\n\ttlsCfg *tls.Config\n\n\trouter *mux.Router\n\ths     *http.Server\n\n\tauthMiddleware mux.MiddlewareFunc\n}\n\nfunc NewServer(cfg v1.WebServerConfig) (*Server, error) {\n\tassets.Load(cfg.AssetsDir)\n\n\taddr := net.JoinHostPort(cfg.Addr, strconv.Itoa(cfg.Port))\n\tif addr == \":\" {\n\t\taddr = \":http\"\n\t}\n\n\tln, err := net.Listen(\"tcp\", addr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trouter := mux.NewRouter()\n\ths := &http.Server{\n\t\tAddr:         addr,\n\t\tHandler:      router,\n\t\tReadTimeout:  defaultReadTimeout,\n\t\tWriteTimeout: defaultWriteTimeout,\n\t}\n\ts := &Server{\n\t\taddr:   addr,\n\t\tln:     ln,\n\t\ths:     hs,\n\t\trouter: router,\n\t}\n\tif cfg.PprofEnable {\n\t\ts.registerPprofHandlers()\n\t}\n\tif cfg.TLS != nil {\n\t\tcert, err := tls.LoadX509KeyPair(cfg.TLS.CertFile, cfg.TLS.KeyFile)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ts.tlsCfg = &tls.Config{\n\t\t\tCertificates: []tls.Certificate{cert},\n\t\t}\n\t}\n\ts.authMiddleware = netpkg.NewHTTPAuthMiddleware(cfg.User, cfg.Password).SetAuthFailDelay(200 * time.Millisecond).Middleware\n\treturn s, nil\n}\n\nfunc (s *Server) Address() string {\n\treturn s.addr\n}\n\nfunc (s *Server) Run() error {\n\tln := s.ln\n\tif s.tlsCfg != nil {\n\t\tln = tls.NewListener(ln, s.tlsCfg)\n\t}\n\treturn s.hs.Serve(ln)\n}\n\nfunc (s *Server) Close() error {\n\treturn s.hs.Close()\n}\n\ntype RouterRegisterHelper struct {\n\tRouter         *mux.Router\n\tAssetsFS       http.FileSystem\n\tAuthMiddleware mux.MiddlewareFunc\n}\n\nfunc (s *Server) RouteRegister(register func(helper *RouterRegisterHelper)) {\n\tregister(&RouterRegisterHelper{\n\t\tRouter:         s.router,\n\t\tAssetsFS:       assets.FileSystem,\n\t\tAuthMiddleware: s.authMiddleware,\n\t})\n}\n\nfunc (s *Server) registerPprofHandlers() {\n\ts.router.HandleFunc(\"/debug/pprof/cmdline\", pprof.Cmdline)\n\ts.router.HandleFunc(\"/debug/pprof/profile\", pprof.Profile)\n\ts.router.HandleFunc(\"/debug/pprof/symbol\", pprof.Symbol)\n\ts.router.HandleFunc(\"/debug/pprof/trace\", pprof.Trace)\n\ts.router.PathPrefix(\"/debug/pprof/\").HandlerFunc(pprof.Index)\n}\n\n// Path: pkg/util/metric/counter.go\n// Copyright 2017 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metric\n\nimport (\n\t\"sync/atomic\"\n)\n\ntype Counter interface {\n\tCount() int32\n\tInc(int32)\n\tDec(int32)\n\tSnapshot() Counter\n\tClear()\n}\n\nfunc NewCounter() Counter {\n\treturn &StandardCounter{\n\t\tcount: 0,\n\t}\n}\n\ntype StandardCounter struct {\n\tcount int32\n}\n\nfunc (c *StandardCounter) Count() int32 {\n\treturn atomic.LoadInt32(&c.count)\n}\n\nfunc (c *StandardCounter) Inc(count int32) {\n\tatomic.AddInt32(&c.count, count)\n}\n\nfunc (c *StandardCounter) Dec(count int32) {\n\tatomic.AddInt32(&c.count, -count)\n}\n\nfunc (c *StandardCounter) Snapshot() Counter {\n\ttmp := &StandardCounter{\n\t\tcount: atomic.LoadInt32(&c.count),\n\t}\n\treturn tmp\n}\n\nfunc (c *StandardCounter) Clear() {\n\tatomic.StoreInt32(&c.count, 0)\n}\n\n// Path: pkg/metrics/mem/server.go\n// Copyright 2019 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mem\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/fatedier/frp/pkg/util/log\"\n\t\"github.com/fatedier/frp/pkg/util/metric\"\n\tserver \"github.com/fatedier/frp/server/metrics\"\n)\n\nvar (\n\tsm = newServerMetrics()\n\n\tServerMetrics  server.ServerMetrics\n\tStatsCollector Collector\n)\n\nfunc init() {\n\tServerMetrics = sm\n\tStatsCollector = sm\n\tsm.run()\n}\n\ntype serverMetrics struct {\n\tinfo *ServerStatistics\n\tmu   sync.Mutex\n}\n\nfunc newServerMetrics() *serverMetrics {\n\treturn &serverMetrics{\n\t\tinfo: &ServerStatistics{\n\t\t\tTotalTrafficIn:  metric.NewDateCounter(ReserveDays),\n\t\t\tTotalTrafficOut: metric.NewDateCounter(ReserveDays),\n\t\t\tCurConns:        metric.NewCounter(),\n\n\t\t\tClientCounts:    metric.NewCounter(),\n\t\t\tProxyTypeCounts: make(map[string]metric.Counter),\n\n\t\t\tProxyStatistics: make(map[string]*ProxyStatistics),\n\t\t},\n\t}\n}\n\nfunc (m *serverMetrics) run() {\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(12 * time.Hour)\n\t\t\tstart := time.Now()\n\t\t\tcount, total := m.clearUselessInfo(time.Duration(7*24) * time.Hour)\n\t\t\tlog.Debugf(\"clear useless proxy statistics data count %d/%d, cost %v\", count, total, time.Since(start))\n\t\t}\n\t}()\n}\n\nfunc (m *serverMetrics) clearUselessInfo(continuousOfflineDuration time.Duration) (int, int) {\n\tcount := 0\n\ttotal := 0\n\t// To check if there are any proxies that have been closed for more than continuousOfflineDuration and remove them.\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\ttotal = len(m.info.ProxyStatistics)\n\tfor name, data := range m.info.ProxyStatistics {\n\t\tif !data.LastCloseTime.IsZero() &&\n\t\t\tdata.LastStartTime.Before(data.LastCloseTime) &&\n\t\t\ttime.Since(data.LastCloseTime) > continuousOfflineDuration {\n\t\t\tdelete(m.info.ProxyStatistics, name)\n\t\t\tcount++\n\t\t\tlog.Tracef(\"clear proxy [%s]'s statistics data, lastCloseTime: [%s]\", name, data.LastCloseTime.String())\n\t\t}\n\t}\n\treturn count, total\n}\n\nfunc (m *serverMetrics) ClearOfflineProxies() (int, int) {\n\treturn m.clearUselessInfo(0)\n}\n\nfunc (m *serverMetrics) NewClient() {\n\tm.info.ClientCounts.Inc(1)\n}\n\nfunc (m *serverMetrics) CloseClient() {\n\tm.info.ClientCounts.Dec(1)\n}\n\nfunc (m *serverMetrics) NewProxy(name string, proxyType string) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tcounter, ok := m.info.ProxyTypeCounts[proxyType]\n\tif !ok {\n\t\tcounter = metric.NewCounter()\n\t}\n\tcounter.Inc(1)\n\tm.info.ProxyTypeCounts[proxyType] = counter\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif !(ok && proxyStats.ProxyType == proxyType) {\n\t\tproxyStats = &ProxyStatistics{\n\t\t\tName:       name,\n\t\t\tProxyType:  proxyType,\n\t\t\tCurConns:   metric.NewCounter(),\n\t\t\tTrafficIn:  metric.NewDateCounter(ReserveDays),\n\t\t\tTrafficOut: metric.NewDateCounter(ReserveDays),\n\t\t}\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n\tproxyStats.LastStartTime = time.Now()\n}\n\nfunc (m *serverMetrics) CloseProxy(name string, proxyType string) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tif counter, ok := m.info.ProxyTypeCounts[proxyType]; ok {\n\t\tcounter.Dec(1)\n\t}\n\tif proxyStats, ok := m.info.ProxyStatistics[name]; ok {\n\t\tproxyStats.LastCloseTime = time.Now()\n\t}\n}\n\nfunc (m *serverMetrics) OpenConnection(name string, _ string) {\n\tm.info.CurConns.Inc(1)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.CurConns.Inc(1)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\nfunc (m *serverMetrics) CloseConnection(name string, _ string) {\n\tm.info.CurConns.Dec(1)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.CurConns.Dec(1)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficIn(name string, _ string, trafficBytes int64) {\n\tm.info.TotalTrafficIn.Inc(trafficBytes)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.TrafficIn.Inc(trafficBytes)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\nfunc (m *serverMetrics) AddTrafficOut(name string, _ string, trafficBytes int64) {\n\tm.info.TotalTrafficOut.Inc(trafficBytes)\n\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tproxyStats.TrafficOut.Inc(trafficBytes)\n\t\tm.info.ProxyStatistics[name] = proxyStats\n\t}\n}\n\n// Get stats data api.\n\nfunc (m *serverMetrics) GetServer() *ServerStats {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\ts := &ServerStats{\n\t\tTotalTrafficIn:  m.info.TotalTrafficIn.TodayCount(),\n\t\tTotalTrafficOut: m.info.TotalTrafficOut.TodayCount(),\n\t\tCurConns:        int64(m.info.CurConns.Count()),\n\t\tClientCounts:    int64(m.info.ClientCounts.Count()),\n\t\tProxyTypeCounts: make(map[string]int64),\n\t}\n\tfor k, v := range m.info.ProxyTypeCounts {\n\t\ts.ProxyTypeCounts[k] = int64(v.Count())\n\t}\n\treturn s\n}\n\nfunc (m *serverMetrics) GetProxiesByType(proxyType string) []*ProxyStats {\n\tres := make([]*ProxyStats, 0)\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tfor name, proxyStats := range m.info.ProxyStatistics {\n\t\tif proxyStats.ProxyType != proxyType {\n\t\t\tcontinue\n\t\t}\n\n\t\tps := &ProxyStats{\n\t\t\tName:            name,\n\t\t\tType:            proxyStats.ProxyType,\n\t\t\tTodayTrafficIn:  proxyStats.TrafficIn.TodayCount(),\n\t\t\tTodayTrafficOut: proxyStats.TrafficOut.TodayCount(),\n\t\t\tCurConns:        int64(proxyStats.CurConns.Count()),\n\t\t}\n\t\tif !proxyStats.LastStartTime.IsZero() {\n\t\t\tps.LastStartTime = proxyStats.LastStartTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tif !proxyStats.LastCloseTime.IsZero() {\n\t\t\tps.LastCloseTime = proxyStats.LastCloseTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tres = append(res, ps)\n\t}\n\treturn res\n}\n\nfunc (m *serverMetrics) GetProxiesByTypeAndName(proxyType string, proxyName string) (res *ProxyStats) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tfor name, proxyStats := range m.info.ProxyStatistics {\n\t\tif proxyStats.ProxyType != proxyType {\n\t\t\tcontinue\n\t\t}\n\n\t\tif name != proxyName {\n\t\t\tcontinue\n\t\t}\n\n\t\tres = &ProxyStats{\n\t\t\tName:            name,\n\t\t\tType:            proxyStats.ProxyType,\n\t\t\tTodayTrafficIn:  proxyStats.TrafficIn.TodayCount(),\n\t\t\tTodayTrafficOut: proxyStats.TrafficOut.TodayCount(),\n\t\t\tCurConns:        int64(proxyStats.CurConns.Count()),\n\t\t}\n\t\tif !proxyStats.LastStartTime.IsZero() {\n\t\t\tres.LastStartTime = proxyStats.LastStartTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tif !proxyStats.LastCloseTime.IsZero() {\n\t\t\tres.LastCloseTime = proxyStats.LastCloseTime.Format(\"01-02 15:04:05\")\n\t\t}\n\t\tbreak\n\t}\n\treturn\n}\n\nfunc (m *serverMetrics) GetProxyTraffic(name string) (res *ProxyTrafficInfo) {\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\n\tproxyStats, ok := m.info.ProxyStatistics[name]\n\tif ok {\n\t\tres = &ProxyTrafficInfo{\n\t\t\tName: name,\n\t\t}\n\t\tres.TrafficIn = proxyStats.TrafficIn.GetLastDaysCount(ReserveDays)\n\t\tres.TrafficOut = proxyStats.TrafficOut.GetLastDaysCount(ReserveDays)\n\t}\n\treturn\n}\n\n// Path: pkg/msg/ctl.go\n// Copyright 2018 fatedier, fatedier@gmail.com\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage msg\n\nimport (\n\t\"io\"\n\n\tjsonMsg \"github.com/fatedier/golib/msg/json\"\n)\n\ntype Message = jsonMsg.Message\n\nvar msgCtl *jsonMsg.MsgCtl\n\nfunc init() {\n\tmsgCtl = jsonMsg.NewMsgCtl()\n\tfor typeByte, msg := range msgTypeMap {\n\t\tmsgCtl.RegisterMsg(typeByte, msg)\n\t}\n}\n\n\nfunc ReadMsg(c io.Reader) (msg Message, err error) {\n\treturn msgCtl.ReadMsg(c)\n}\n\nfunc ReadMsgInto(c io.Reader, msg Message) (err error) {\n\treturn msgCtl.ReadMsgInto(c, msg)\n}\n\nfunc WriteMsg(c io.Writer, msg interface{}) (err error) {\n\treturn msgCtl.WriteMsg(c, msg)\n}\n\n// Path: pkg/msg/handler.go\n// Copyright 2023 The frp Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage msg\n\nimport (\n\t\"io\"\n\t\"reflect\"\n)\n\nfunc AsyncHandler(f func(Message)) func(Message) {\n\treturn func(m Message) {\n\t\tgo f(m)\n\t}\n}\n\n// Dispatcher is used to send messages to net.Conn or register handlers for messages read from net.Conn.\ntype Dispatcher struct {\n\trw io.ReadWriter\n\n\tsendCh         chan Message\n\tdoneCh         chan struct{}\n\tmsgHandlers    map[reflect.Type]func(Message)\n\tdefaultHandler func(Message)\n}\n\nfunc NewDispatcher(rw io.ReadWriter) *Dispatcher {\n\treturn &Dispatcher{\n\t\trw:          rw,\n\t\tsendCh:      make(chan Message, 100),\n\t\tdoneCh:      make(chan struct{}),\n\t\tmsgHandlers: make(map[reflect.Type]func(Message)),\n\t}\n}\n\n// Run will block until io.EOF or some error occurs.\nfunc (d *Dispatcher) Run() {\n\tgo d.sendLoop()\n\tgo d.readLoop()\n}\n\nfunc (d *Dispatcher) sendLoop() {\n\tfor {\n\t\tselect {\n\t\tcase <-d.doneCh:\n\t\t\treturn\n\t\tcase m := <-d.sendCh:\n\t\t\t_ = WriteMsg(d.rw, m)\n\t\t}\n\t}\n}\n\nfunc (d *Dispatcher) readLoop() {\n\tfor {\n\t\tm, err := ReadMsg(d.rw)\n\t\tif err != nil {\n\t\t\tclose(d.doneCh)\n\t\t\treturn\n\t\t}\n\n\t\tif handler, ok := d.msgHandlers[reflect.TypeOf(m)]; ok {\n\t\t\thandler(m)\n\t\t} else if d.defaultHandler != nil {\n\t\t\td.defaultHandler(m)\n\t\t}\n\t}\n}\n\nfunc (d *Dispatcher) Send(m Message) error {\n\tselect {\n...\n", "needle_token_start": 14463, "needle_token_end": 14494, "code_context_ntokens": 15241, "output": ["```go\nfunc ReadMsg(c io.Reader) (msg Message, err error) {\n\treturn msgCtl.ReadMsg(c)\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "BenchmarkHeaderREMatcher", "language": "go", "path": "modules/caddyhttp/matchers_test.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To measure the performance of a function that matches HTTP headers against a specified regular expression pattern.\n2. **Input**: A simulated HTTP request with a specific header and a regular expression pattern to match against this header.\n3. **Output**: There is no direct output from the benchmark itself, as it primarily measures the time and efficiency of the matching operation.\n4. **Procedure**: \n   - A regular expression pattern is defined and associated with a header field.\n   - The pattern is provisioned and validated for correctness.\n   - A simulated HTTP request is created with the specified header.\n   - A context and replacer are set up for the request to handle dynamic replacements based on the regular expression match.\n   - The matching function is executed repeatedly in a loop to measure its performance under continuous operation.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\t\tinput:  http.Header{},\n\t\t\thost:   \"localhost\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHeaderRE{\"host\": &MatchRegexp{Pattern: \"^local$\", Name: \"name\"}},\n\t\t\tinput:  http.Header{},\n\t\t\thost:   \"localhost\",\n\t\t\texpect: false,\n\t\t},\n\t} {\n\t\t// compile the regexp and validate its name\n\t\terr := tc.match.Provision(caddy.Context{})\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Test %d %v: Provisioning: %v\", i, tc.match, err)\n\t\t\tcontinue\n\t\t}\n\t\terr = tc.match.Validate()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Test %d %v: Validating: %v\", i, tc.match, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// set up the fake request and its Replacer\n\t\treq := &http.Request{Header: tc.input, URL: new(url.URL), Host: tc.host}\n\t\trepl := caddy.NewReplacer()\n\t\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\t\treq = req.WithContext(ctx)\n\t\taddHTTPVarsToReplacer(repl, req, httptest.NewRecorder())\n\n\t\tactual := tc.match.Match(req)\n\t\tif actual != tc.expect {\n\t\t\tt.Errorf(\"Test %d [%v]: Expected %t, got %t for input '%s'\",\n\t\t\t\ti, tc.match, tc.expect, actual, tc.input)\n\t\t\tcontinue\n\t\t}\n\n\t\tfor key, expectVal := range tc.expectRepl {\n\t\t\tplaceholder := fmt.Sprintf(\"{http.regexp.%s}\", key)\n\t\t\tactualVal := repl.ReplaceAll(placeholder, \"<empty>\")\n\t\t\tif actualVal != expectVal {\n\t\t\t\tt.Errorf(\"Test %d [%v]: Expected placeholder {http.regexp.%s} to be '%s' but got '%s'\",\n\t\t\t\t\ti, tc.match, key, expectVal, actualVal)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n}\n\n\nfunc BenchmarkHeaderREMatcher(b *testing.B) {\n\ti := 0\n\tmatch := MatchHeaderRE{\"Field\": &MatchRegexp{Pattern: \"^foo(.*)$\", Name: \"name\"}}\n\tinput := http.Header{\"Field\": []string{\"foobar\"}}\n\tvar host string\n\terr := match.Provision(caddy.Context{})\n\tif err != nil {\n\t\tb.Errorf(\"Test %d %v: Provisioning: %v\", i, match, err)\n\t}\n\terr = match.Validate()\n\tif err != nil {\n\t\tb.Errorf(\"Test %d %v: Validating: %v\", i, match, err)\n\t}\n\n\t// set up the fake request and its Replacer\n\treq := &http.Request{Header: input, URL: new(url.URL), Host: host}\n\trepl := caddy.NewReplacer()\n\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\treq = req.WithContext(ctx)\n\taddHTTPVarsToReplacer(repl, req, httptest.NewRecorder())\n\tfor run := 0; run < b.N; run++ {\n\t\tmatch.Match(req)\n\t}\n}\n\nfunc TestVarREMatcher(t *testing.T) {\n\tfor i, tc := range []struct {\n\t\tdesc       string\n\t\tmatch      MatchVarsRE\n\t\tinput      VarsMiddleware\n\t\texpect     bool\n\t\texpectRepl map[string]string\n\t}{\n\t\t{\n\t\t\tdesc:   \"match static value within var set by the VarsMiddleware succeeds\",\n\t\t\tmatch:  MatchVarsRE{\"Var1\": &MatchRegexp{Pattern: \"foo\"}},\n\t\t\tinput:  VarsMiddleware{\"Var1\": \"here is foo val\"},\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tdesc:   \"value set by VarsMiddleware not satisfying regexp matcher fails to match\",\n\t\t\tmatch:  MatchVarsRE{\"Var1\": &MatchRegexp{Pattern: \"$foo^\"}},\n\t\t\tinput:  VarsMiddleware{\"Var1\": \"foobar\"},\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tdesc:       \"successfully matched value is captured and its placeholder is added to replacer\",\n\t\t\tmatch:      MatchVarsRE{\"Var1\": &MatchRegexp{Pattern: \"^foo(.*)$\", Name: \"name\"}},\n\t\t\tinput:      VarsMiddleware{\"Var1\": \"foobar\"},\n\t\t\texpect:     true,\n\t\t\texpectRepl: map[string]string{\"name.1\": \"bar\"},\n\t\t},\n\t\t{\n\t\t\tdesc:   \"matching against a value of standard variables succeeds\",\n\t\t\tmatch:  MatchVarsRE{\"{http.request.method}\": &MatchRegexp{Pattern: \"^G.[tT]$\"}},\n\t\t\tinput:  VarsMiddleware{},\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tdesc:   \"matching against value of var set by the VarsMiddleware and referenced by its placeholder succeeds\",\n\t\t\tmatch:  MatchVarsRE{\"{http.vars.Var1}\": &MatchRegexp{Pattern: \"[vV]ar[0-9]\"}},\n\t\t\tinput:  VarsMiddleware{\"Var1\": \"var1Value\"},\n\t\t\texpect: true,\n\t\t},\n\t} {\n\t\ti := i   // capture range value\n\t\ttc := tc // capture range value\n\t\tt.Run(tc.desc, func(t *testing.T) {\n\t\t\tt.Parallel()\n\t\t\t// compile the regexp and validate its name\n\t\t\terr := tc.match.Provision(caddy.Context{})\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Test %d %v: Provisioning: %v\", i, tc.match, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\terr = tc.match.Validate()\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"Test %d %v: Validating: %v\", i, tc.match, err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// set up the fake request and its Replacer\n\t\t\treq := &http.Request{URL: new(url.URL), Method: http.MethodGet}\n\t\t\trepl := caddy.NewReplacer()\n\t\t\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\t\t\tctx = context.WithValue(ctx, VarsCtxKey, make(map[string]any))\n\t\t\treq = req.WithContext(ctx)\n\n\t\t\taddHTTPVarsToReplacer(repl, req, httptest.NewRecorder())\n\n\t\t\ttc.input.ServeHTTP(httptest.NewRecorder(), req, emptyHandler)\n\n\t\t\tactual := tc.match.Match(req)\n\t\t\tif actual != tc.expect {\n\t\t\t\tt.Errorf(\"Test %d [%v]: Expected %t, got %t for input '%s'\",\n\t\t\t\t\ti, tc.match, tc.expect, actual, tc.input)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor key, expectVal := range tc.expectRepl {\n\t\t\t\tplaceholder := fmt.Sprintf(\"{http.regexp.%s}\", key)\n\t\t\t\tactualVal := repl.ReplaceAll(placeholder, \"<empty>\")\n\t\t\t\tif actualVal != expectVal {\n\t\t\t\t\tt.Errorf(\"Test %d [%v]: Expected placeholder {http.regexp.%s} to be '%s' but got '%s'\",\n\t\t\t\t\t\ti, tc.match, key, expectVal, actualVal)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNotMatcher(t *testing.T) {\n\tfor i, tc := range []struct {\n\t\thost, path string\n\t\tmatch      MatchNot\n\t\texpect     bool\n\t}{\n\t\t{\n\t\t\thost: \"example.com\", path: \"/\",\n\t\t\tmatch:  MatchNot{},\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\thost: \"example.com\", path: \"/foo\",\n\t\t\tmatch: MatchNot{\n\t\t\t\tMatcherSets: []MatcherSet{\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchPath{\"/foo\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\thost: \"example.com\", path: \"/bar\",\n\t\t\tmatch: MatchNot{\n\t\t\t\tMatcherSets: []MatcherSet{\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchPath{\"/foo\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\thost: \"example.com\", path: \"/bar\",\n\t\t\tmatch: MatchNot{\n\t\t\t\tMatcherSets: []MatcherSet{\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchPath{\"/foo\"},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchHost{\"example.com\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\thost: \"example.com\", path: \"/bar\",\n\t\t\tmatch: MatchNot{\n\t\t\t\tMatcherSets: []MatcherSet{\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchPath{\"/bar\"},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchHost{\"example.com\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\thost: \"example.com\", path: \"/foo\",\n\t\t\tmatch: MatchNot{\n\t\t\t\tMatcherSets: []MatcherSet{\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchPath{\"/bar\"},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchHost{\"sub.example.com\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\thost: \"example.com\", path: \"/foo\",\n\t\t\tmatch: MatchNot{\n\t\t\t\tMatcherSets: []MatcherSet{\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchPath{\"/foo\"},\n\t\t\t\t\t\tMatchHost{\"example.com\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\thost: \"example.com\", path: \"/foo\",\n\t\t\tmatch: MatchNot{\n\t\t\t\tMatcherSets: []MatcherSet{\n\t\t\t\t\t{\n\t\t\t\t\t\tMatchPath{\"/bar\"},\n\t\t\t\t\t\tMatchHost{\"example.com\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: true,\n\t\t},\n\t} {\n\t\treq := &http.Request{Host: tc.host, URL: &url.URL{Path: tc.path}}\n\t\trepl := caddy.NewReplacer()\n\t\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\t\treq = req.WithContext(ctx)\n\n\t\tactual := tc.match.Match(req)\n\t\tif actual != tc.expect {\n\t\t\tt.Errorf(\"Test %d %+v: Expected %t, got %t for: host=%s path=%s'\", i, tc.match, tc.expect, actual, tc.host, tc.path)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc BenchmarkLargeHostMatcher(b *testing.B) {\n\t// this benchmark simulates a large host matcher (thousands of entries) where each\n\t// value is an exact hostname (not a placeholder or wildcard) - compare the results\n\t// of this with and without the binary search (comment out the various fast path\n\t// sections in Match) to conduct experiments\n\n\tconst n = 10000\n\tlastHost := fmt.Sprintf(\"%d.example.com\", n-1)\n\treq := &http.Request{Host: lastHost}\n\trepl := caddy.NewReplacer()\n\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\treq = req.WithContext(ctx)\n\n\tmatcher := make(MatchHost, n)\n\tfor i := 0; i < n; i++ {\n\t\tmatcher[i] = fmt.Sprintf(\"%d.example.com\", i)\n\t}\n\terr := matcher.Provision(caddy.Context{})\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tmatcher.Match(req)\n\t}\n}\n\nfunc BenchmarkHostMatcherWithoutPlaceholder(b *testing.B) {\n\treq := &http.Request{Host: \"localhost\"}\n\trepl := caddy.NewReplacer()\n\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\treq = req.WithContext(ctx)\n\n\tmatch := MatchHost{\"localhost\"}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tmatch.Match(req)\n\t}\n}\n\nfunc BenchmarkHostMatcherWithPlaceholder(b *testing.B) {\n\terr := os.Setenv(\"GO_BENCHMARK_DOMAIN\", \"localhost\")\n\tif err != nil {\n\t\tb.Errorf(\"error while setting up environment: %v\", err)\n\t}\n\n\treq := &http.Request{Host: \"localhost\"}\n\trepl := caddy.NewReplacer()\n\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\treq = req.WithContext(ctx)\n\tmatch := MatchHost{\"{env.GO_BENCHMARK_DOMAIN}\"}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tmatch.Match(req)\n\t}\n}\n\n// Path: modules/caddyhttp/reverseproxy/metrics.go\npackage reverseproxy\n\nimport (\n\t\"runtime/debug\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n\t\"go.uber.org/zap\"\n)\n\nvar reverseProxyMetrics = struct {\n\tinit             sync.Once\n\tupstreamsHealthy *prometheus.GaugeVec\n\tlogger           *zap.Logger\n}{}\n\nfunc initReverseProxyMetrics(handler *Handler) {\n\tconst ns, sub = \"caddy\", \"reverse_proxy\"\n\n\tupstreamsLabels := []string{\"upstream\"}\n\treverseProxyMetrics.upstreamsHealthy = promauto.NewGaugeVec(prometheus.GaugeOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"upstreams_healthy\",\n\t\tHelp:      \"Health status of reverse proxy upstreams.\",\n\t}, upstreamsLabels)\n\n\treverseProxyMetrics.logger = handler.logger.Named(\"reverse_proxy.metrics\")\n}\n\ntype metricsUpstreamsHealthyUpdater struct {\n\thandler *Handler\n}\n\nfunc newMetricsUpstreamsHealthyUpdater(handler *Handler) *metricsUpstreamsHealthyUpdater {\n\treverseProxyMetrics.init.Do(func() {\n\t\tinitReverseProxyMetrics(handler)\n\t})\n\n\treverseProxyMetrics.upstreamsHealthy.Reset()\n\n\treturn &metricsUpstreamsHealthyUpdater{handler}\n}\n\nfunc (m *metricsUpstreamsHealthyUpdater) Init() {\n\tgo func() {\n\t\tdefer func() {\n\t\t\tif err := recover(); err != nil {\n\t\t\t\treverseProxyMetrics.logger.Error(\"upstreams healthy metrics updater panicked\",\n\t\t\t\t\tzap.Any(\"error\", err),\n\t\t\t\t\tzap.ByteString(\"stack\", debug.Stack()))\n\t\t\t}\n\t\t}()\n\n\t\tm.update()\n\n\t\tticker := time.NewTicker(10 * time.Second)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ticker.C:\n\t\t\t\tm.update()\n\t\t\tcase <-m.handler.ctx.Done():\n\t\t\t\tticker.Stop()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n}\n\nfunc (m *metricsUpstreamsHealthyUpdater) update() {\n\tfor _, upstream := range m.handler.Upstreams {\n\t\tlabels := prometheus.Labels{\"upstream\": upstream.Dial}\n\n\t\tgaugeValue := 0.0\n\t\tif upstream.Healthy() {\n\t\t\tgaugeValue = 1.0\n\t\t}\n\n\t\treverseProxyMetrics.upstreamsHealthy.With(labels).Set(gaugeValue)\n\t}\n}\n\n// Path: modules/metrics/metrics.go\n// Copyright 2020 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage metrics\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(Metrics{})\n\thttpcaddyfile.RegisterHandlerDirective(\"metrics\", parseCaddyfile)\n}\n\n// Metrics is a module that serves a /metrics endpoint so that any gathered\n// metrics can be exposed for scraping. This module is configurable by end-users\n// unlike AdminMetrics.\ntype Metrics struct {\n\tmetricsHandler http.Handler\n\n\t// Disable OpenMetrics negotiation, enabled by default. May be necessary if\n\t// the produced metrics cannot be parsed by the service scraping metrics.\n\tDisableOpenMetrics bool `json:\"disable_openmetrics,omitempty\"`\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (Metrics) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.handlers.metrics\",\n\t\tNew: func() caddy.Module { return new(Metrics) },\n\t}\n}\n\ntype zapLogger struct {\n\tzl *zap.Logger\n}\n\nfunc (l *zapLogger) Println(v ...any) {\n\tl.zl.Sugar().Error(v...)\n}\n\n// Provision sets up m.\nfunc (m *Metrics) Provision(ctx caddy.Context) error {\n\tlog := ctx.Logger()\n\tm.metricsHandler = createMetricsHandler(&zapLogger{log}, !m.DisableOpenMetrics)\n\treturn nil\n}\n\nfunc parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {\n\tvar m Metrics\n\terr := m.UnmarshalCaddyfile(h.Dispenser)\n\treturn m, err\n}\n\n// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:\n//\n//\tmetrics [<matcher>] {\n//\t    disable_openmetrics\n//\t}\nfunc (m *Metrics) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume directive name\n\targs := d.RemainingArgs()\n\tif len(args) > 0 {\n\t\treturn d.ArgErr()\n\t}\n\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"disable_openmetrics\":\n\t\t\tm.DisableOpenMetrics = true\n\t\tdefault:\n\t\t\treturn d.Errf(\"unrecognized subdirective %q\", d.Val())\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {\n\tm.metricsHandler.ServeHTTP(w, r)\n\treturn nil\n}\n\n// Interface guards\nvar (\n\t_ caddy.Provisioner           = (*Metrics)(nil)\n\t_ caddyhttp.MiddlewareHandler = (*Metrics)(nil)\n\t_ caddyfile.Unmarshaler       = (*Metrics)(nil)\n)\n\nfunc createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool) http.Handler {\n\treturn promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer,\n\t\tpromhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{\n\t\t\t// will only log errors if logger is non-nil\n\t\t\tErrorLog: logger,\n\n\t\t\t// Allow OpenMetrics format to be negotiated - largely compatible,\n\t\t\t// except quantile/le label values always have a decimal.\n\t\t\tEnableOpenMetrics: enableOpenMetrics,\n\t\t}),\n\t)\n}\n\n// Path: modules/caddyhttp/metrics.go\npackage caddyhttp\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net/http\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promauto\"\n\n\t\"github.com/caddyserver/caddy/v2/internal/metrics\"\n)\n\n// Metrics configures metrics observations.\n// EXPERIMENTAL and subject to change or removal.\ntype Metrics struct{}\n\nvar httpMetrics = struct {\n\tinit             sync.Once\n\trequestInFlight  *prometheus.GaugeVec\n\trequestCount     *prometheus.CounterVec\n\trequestErrors    *prometheus.CounterVec\n\trequestDuration  *prometheus.HistogramVec\n\trequestSize      *prometheus.HistogramVec\n\tresponseSize     *prometheus.HistogramVec\n\tresponseDuration *prometheus.HistogramVec\n}{\n\tinit: sync.Once{},\n}\n\nfunc initHTTPMetrics() {\n\tconst ns, sub = \"caddy\", \"http\"\n\n\tbasicLabels := []string{\"server\", \"handler\"}\n\thttpMetrics.requestInFlight = promauto.NewGaugeVec(prometheus.GaugeOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"requests_in_flight\",\n\t\tHelp:      \"Number of requests currently handled by this server.\",\n\t}, basicLabels)\n\thttpMetrics.requestErrors = promauto.NewCounterVec(prometheus.CounterOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"request_errors_total\",\n\t\tHelp:      \"Number of requests resulting in middleware errors.\",\n\t}, basicLabels)\n\thttpMetrics.requestCount = promauto.NewCounterVec(prometheus.CounterOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"requests_total\",\n\t\tHelp:      \"Counter of HTTP(S) requests made.\",\n\t}, basicLabels)\n\n\t// TODO: allow these to be customized in the config\n\tdurationBuckets := prometheus.DefBuckets\n\tsizeBuckets := prometheus.ExponentialBuckets(256, 4, 8)\n\n\thttpLabels := []string{\"server\", \"handler\", \"code\", \"method\"}\n\thttpMetrics.requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"request_duration_seconds\",\n\t\tHelp:      \"Histogram of round-trip request durations.\",\n\t\tBuckets:   durationBuckets,\n\t}, httpLabels)\n\thttpMetrics.requestSize = promauto.NewHistogramVec(prometheus.HistogramOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"request_size_bytes\",\n\t\tHelp:      \"Total size of the request. Includes body\",\n\t\tBuckets:   sizeBuckets,\n\t}, httpLabels)\n\thttpMetrics.responseSize = promauto.NewHistogramVec(prometheus.HistogramOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"response_size_bytes\",\n\t\tHelp:      \"Size of the returned response.\",\n\t\tBuckets:   sizeBuckets,\n\t}, httpLabels)\n\thttpMetrics.responseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{\n\t\tNamespace: ns,\n\t\tSubsystem: sub,\n\t\tName:      \"response_duration_seconds\",\n\t\tHelp:      \"Histogram of times to first byte in response bodies.\",\n\t\tBuckets:   durationBuckets,\n\t}, httpLabels)\n}\n\n// serverNameFromContext extracts the current server name from the context.\n// Returns \"UNKNOWN\" if none is available (should probably never happen).\nfunc serverNameFromContext(ctx context.Context) string {\n\tsrv, ok := ctx.Value(ServerCtxKey).(*Server)\n\tif !ok || srv == nil || srv.name == \"\" {\n\t\treturn \"UNKNOWN\"\n\t}\n\treturn srv.name\n}\n\ntype metricsInstrumentedHandler struct {\n\thandler string\n\tmh      MiddlewareHandler\n}\n\nfunc newMetricsInstrumentedHandler(handler string, mh MiddlewareHandler) *metricsInstrumentedHandler {\n\thttpMetrics.init.Do(func() {\n\t\tinitHTTPMetrics()\n\t})\n\n\treturn &metricsInstrumentedHandler{handler, mh}\n}\n\nfunc (h *metricsInstrumentedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {\n\tserver := serverNameFromContext(r.Context())\n\tlabels := prometheus.Labels{\"server\": server, \"handler\": h.handler}\n\tmethod := metrics.SanitizeMethod(r.Method)\n\t// the \"code\" value is set later, but initialized here to eliminate the possibility\n\t// of a panic\n\tstatusLabels := prometheus.Labels{\"server\": server, \"handler\": h.handler, \"method\": method, \"code\": \"\"}\n\n\tinFlight := httpMetrics.requestInFlight.With(labels)\n\tinFlight.Inc()\n\tdefer inFlight.Dec()\n\n\tstart := time.Now()\n\n\t// This is a _bit_ of a hack - it depends on the ShouldBufferFunc always\n\t// being called when the headers are written.\n\t// Effectively the same behaviour as promhttp.InstrumentHandlerTimeToWriteHeader.\n\twriteHeaderRecorder := ShouldBufferFunc(func(status int, header http.Header) bool {\n\t\tstatusLabels[\"code\"] = metrics.SanitizeCode(status)\n\t\tttfb := time.Since(start).Seconds()\n\t\thttpMetrics.responseDuration.With(statusLabels).Observe(ttfb)\n\t\treturn false\n\t})\n\twrec := NewResponseRecorder(w, nil, writeHeaderRecorder)\n\terr := h.mh.ServeHTTP(wrec, r, next)\n\tdur := time.Since(start).Seconds()\n\thttpMetrics.requestCount.With(labels).Inc()\n\n\tobserveRequest := func(status int) {\n\t\t// If the code hasn't been set yet, and we didn't encounter an error, we're\n\t\t// probably falling through with an empty handler.\n\t\tif statusLabels[\"code\"] == \"\" {\n\t\t\t// we still sanitize it, even though it's likely to be 0. A 200 is\n\t\t\t// returned on fallthrough so we want to reflect that.\n\t\t\tstatusLabels[\"code\"] = metrics.SanitizeCode(status)\n\t\t}\n\n\t\thttpMetrics.requestDuration.With(statusLabels).Observe(dur)\n\t\thttpMetrics.requestSize.With(statusLabels).Observe(float64(computeApproximateRequestSize(r)))\n\t\thttpMetrics.responseSize.With(statusLabels).Observe(float64(wrec.Size()))\n\t}\n\n\tif err != nil {\n\t\tvar handlerErr HandlerError\n\t\tif errors.As(err, &handlerErr) {\n\t\t\tobserveRequest(handlerErr.StatusCode)\n\t\t}\n\n\t\thttpMetrics.requestErrors.With(labels).Inc()\n\n\t\treturn err\n\t}\n\n\tobserveRequest(wrec.Status())\n\n\treturn nil\n}\n\n// taken from https://github.com/prometheus/client_golang/blob/6007b2b5cae01203111de55f753e76d8dac1f529/prometheus/promhttp/instrument_server.go#L298\nfunc computeApproximateRequestSize(r *http.Request) int {\n\ts := 0\n\tif r.URL != nil {\n\t\ts += len(r.URL.String())\n\t}\n\n\ts += len(r.Method)\n\ts += len(r.Proto)\n\tfor name, values := range r.Header {\n\t\ts += len(name)\n\t\tfor _, value := range values {\n\t\t\ts += len(value)\n\t\t}\n\t}\n\ts += len(r.Host)\n\n\t// N.B. r.Form and r.MultipartForm are assumed to be included in r.URL.\n\n\tif r.ContentLength != -1 {\n\t\ts += int(r.ContentLength)\n\t}\n\treturn s\n}\n\n// Path: modules/caddyhttp/metrics_test.go\npackage caddyhttp\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/testutil\"\n)\n\nfunc TestServerNameFromContext(t *testing.T) {\n\tctx := context.Background()\n\texpected := \"UNKNOWN\"\n\tif actual := serverNameFromContext(ctx); actual != expected {\n\t\tt.Errorf(\"Not equal: expected %q, but got %q\", expected, actual)\n\t}\n\n\tin := \"foo\"\n\tctx = context.WithValue(ctx, ServerCtxKey, &Server{name: in})\n\tif actual := serverNameFromContext(ctx); actual != in {\n\t\tt.Errorf(\"Not equal: expected %q, but got %q\", in, actual)\n\t}\n}\n\nfunc TestMetricsInstrumentedHandler(t *testing.T) {\n\thandlerErr := errors.New(\"oh noes\")\n\tresponse := []byte(\"hello world!\")\n\th := HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {\n\t\tif actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 1.0 {\n\t\t\tt.Errorf(\"Not same: expected %#v, but got %#v\", 1.0, actual)\n\t\t}\n\t\tif handlerErr == nil {\n\t\t\tw.Write(response)\n\t\t}\n\t\treturn handlerErr\n\t})\n\n\tmh := middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {\n\t\treturn h.ServeHTTP(w, r)\n\t})\n\n\tih := newMetricsInstrumentedHandler(\"bar\", mh)\n\n\tr := httptest.NewRequest(\"GET\", \"/\", nil)\n\tw := httptest.NewRecorder()\n\n\tif actual := ih.ServeHTTP(w, r, h); actual != handlerErr {\n\t\tt.Errorf(\"Not same: expected %#v, but got %#v\", handlerErr, actual)\n\t}\n\tif actual := testutil.ToFloat64(httpMetrics.requestInFlight); actual != 0.0 {\n\t\tt.Errorf(\"Not same: expected %#v, but got %#v\", 0.0, actual)\n\t}\n\n\thandlerErr = nil\n\tif err := ih.ServeHTTP(w, r, h); err != nil {\n\t\tt.Errorf(\"Received unexpected error: %v\", err)\n\t}\n\n\t// an empty handler - no errors, no header written\n\tmh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {\n\t\treturn nil\n\t})\n\tih = newMetricsInstrumentedHandler(\"empty\", mh)\n\tr = httptest.NewRequest(\"GET\", \"/\", nil)\n\tw = httptest.NewRecorder()\n\n\tif err := ih.ServeHTTP(w, r, h); err != nil {\n\t\tt.Errorf(\"Received unexpected error: %v\", err)\n\t}\n\tif actual := w.Result().StatusCode; actual != 200 {\n\t\tt.Errorf(\"Not same: expected status code %#v, but got %#v\", 200, actual)\n\t}\n\tif actual := w.Result().Header; len(actual) != 0 {\n\t\tt.Errorf(\"Not empty: expected headers to be empty, but got %#v\", actual)\n\t}\n\n\t// handler returning an error with an HTTP status\n\tmh = middlewareHandlerFunc(func(w http.ResponseWriter, r *http.Request, h Handler) error {\n\t\treturn Error(http.StatusTooManyRequests, nil)\n\t})\n\n\tih = newMetricsInstrumentedHandler(\"foo\", mh)\n\n\tr = httptest.NewRequest(\"GET\", \"/\", nil)\n\tw = httptest.NewRecorder()\n\n\tif err := ih.ServeHTTP(w, r, nil); err == nil {\n\t\tt.Errorf(\"expected error to be propagated\")\n\t}\n\n\texpected := `\n\t# HELP caddy_http_request_duration_seconds Histogram of round-trip request durations.\n\t# TYPE caddy_http_request_duration_seconds histogram\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"0.005\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"0.01\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"0.025\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"0.05\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"0.1\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"0.25\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"0.5\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"1\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"2.5\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"5\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"10\"} 1\n\tcaddy_http_request_duration_seconds_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"+Inf\"} 1\n\tcaddy_http_request_duration_seconds_count{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\"} 1\n\t# HELP caddy_http_request_size_bytes Total size of the request. Includes body\n\t# TYPE caddy_http_request_size_bytes histogram\n\tcaddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"256\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"1024\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"4096\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"16384\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"65536\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"262144\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"1.048576e+06\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"4.194304e+06\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"+Inf\"} 1\n    caddy_http_request_size_bytes_sum{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\"} 23\n    caddy_http_request_size_bytes_count{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"256\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"1024\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"4096\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"16384\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"65536\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"262144\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"1.048576e+06\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"4.194304e+06\"} 1\n    caddy_http_request_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"+Inf\"} 1\n    caddy_http_request_size_bytes_sum{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\"} 23\n    caddy_http_request_size_bytes_count{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"256\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"1024\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"4096\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"16384\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"65536\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"262144\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"1.048576e+06\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"4.194304e+06\"} 1\n\tcaddy_http_request_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"+Inf\"} 1\n\tcaddy_http_request_size_bytes_sum{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\"} 23\n\tcaddy_http_request_size_bytes_count{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\"} 1\n\t# HELP caddy_http_response_size_bytes Size of the returned response.\n\t# TYPE caddy_http_response_size_bytes histogram\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"256\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"1024\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"4096\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"16384\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"65536\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"262144\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"1.048576e+06\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"4.194304e+06\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\",le=\"+Inf\"} 1\n\tcaddy_http_response_size_bytes_sum{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\"} 12\n\tcaddy_http_response_size_bytes_count{code=\"200\",handler=\"bar\",method=\"GET\",server=\"UNKNOWN\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"256\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"1024\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"4096\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"16384\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"65536\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"262144\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"1.048576e+06\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"4.194304e+06\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\",le=\"+Inf\"} 1\n\tcaddy_http_response_size_bytes_sum{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\"} 0\n\tcaddy_http_response_size_bytes_count{code=\"200\",handler=\"empty\",method=\"GET\",server=\"UNKNOWN\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"256\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"1024\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"4096\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"16384\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"65536\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"262144\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"1.048576e+06\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"4.194304e+06\"} 1\n\tcaddy_http_response_size_bytes_bucket{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\",le=\"+Inf\"} 1\n\tcaddy_http_response_size_bytes_sum{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\"} 0\n\tcaddy_http_response_size_bytes_count{code=\"429\",handler=\"foo\",method=\"GET\",server=\"UNKNOWN\"} 1\n\t# HELP caddy_http_request_errors_total Number of requests resulting in middleware errors.\n\t# TYPE caddy_http_request_errors_total counter\n\tcaddy_http_request_errors_total{handler=\"bar\",server=\"UNKNOWN\"} 1\n\tcaddy_http_request_errors_total{handler=\"foo\",server=\"UNKNOWN\"} 1\n\t`\n\tif err := testutil.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(expected),\n\t\t\"caddy_http_request_size_bytes\",\n\t\t\"caddy_http_response_size_bytes\",\n\t\t// caddy_http_request_duration_seconds_sum will vary based on how long the test took to run,\n\t\t// so we check just the _bucket and _count metrics\n\t\t\"caddy_http_request_duration_seconds_bucket\",\n\t\t\"caddy_http_request_duration_seconds_count\",\n\t\t\"caddy_http_request_errors_total\",\n\t); err != nil {\n\t\tt.Errorf(\"received unexpected error: %s\", err)\n\t}\n}\n\ntype middlewareHandlerFunc func(http.ResponseWriter, *http.Request, Handler) error\n\nfunc (f middlewareHandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, h Handler) error {\n\treturn f(w, r, h)\n}\n\n// Path: modules/caddyhttp/proxyprotocol/listenerwrapper.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage proxyprotocol\n\nimport (\n\t\"net\"\n\t\"net/netip\"\n\t\"time\"\n\n\tgoproxy \"github.com/pires/go-proxyproto\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\n// ListenerWrapper provides PROXY protocol support to Caddy by implementing\n// the caddy.ListenerWrapper interface. It must be loaded before the `tls` listener.\n//\n// Credit goes to https://github.com/mastercactapus/caddy2-proxyprotocol for having\n// initially implemented this as a plugin.\ntype ListenerWrapper struct {\n\t// Timeout specifies an optional maximum time for\n\t// the PROXY header to be received.\n\t// If zero, timeout is disabled. Default is 5s.\n\tTimeout caddy.Duration `json:\"timeout,omitempty\"`\n\n\t// Allow is an optional list of CIDR ranges to\n\t// allow/require PROXY headers from.\n\tAllow []string `json:\"allow,omitempty\"`\n\tallow []netip.Prefix\n\n\t// Denby is an optional list of CIDR ranges to\n\t// deny PROXY headers from.\n\tDeny []string `json:\"deny,omitempty\"`\n\tdeny []netip.Prefix\n\n\t// Accepted values are: ignore, use, reject, require, skip\n\t// default: ignore\n\t// Policy definitions are here: https://pkg.go.dev/github.com/pires/go-proxyproto@v0.7.0#Policy\n\tFallbackPolicy Policy `json:\"fallback_policy,omitempty\"`\n\n\tpolicy goproxy.PolicyFunc\n}\n\n// Provision sets up the listener wrapper.\nfunc (pp *ListenerWrapper) Provision(ctx caddy.Context) error {\n\tfor _, cidr := range pp.Allow {\n\t\tipnet, err := netip.ParsePrefix(cidr)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tpp.allow = append(pp.allow, ipnet)\n\t}\n\tfor _, cidr := range pp.Deny {\n\t\tipnet, err := netip.ParsePrefix(cidr)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tpp.deny = append(pp.deny, ipnet)\n\t}\n\tpp.policy = func(upstream net.Addr) (goproxy.Policy, error) {\n\t\t// trust unix sockets\n\t\tif network := upstream.Network(); caddy.IsUnixNetwork(network) {\n\t\t\treturn goproxy.USE, nil\n\t\t}\n\t\tret := pp.FallbackPolicy\n\t\thost, _, err := net.SplitHostPort(upstream.String())\n\t\tif err != nil {\n\t\t\treturn goproxy.REJECT, err\n\t\t}\n\n\t\tip, err := netip.ParseAddr(host)\n\t\tif err != nil {\n\t\t\treturn goproxy.REJECT, err\n\t\t}\n\t\tfor _, ipnet := range pp.deny {\n\t\t\tif ipnet.Contains(ip) {\n\t\t\t\treturn goproxy.REJECT, nil\n\t\t\t}\n\t\t}\n\t\tfor _, ipnet := range pp.allow {\n\t\t\tif ipnet.Contains(ip) {\n\t\t\t\tret = PolicyUSE\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\treturn policyToGoProxyPolicy[ret], nil\n\t}\n\treturn nil\n}\n\n// WrapListener adds PROXY protocol support to the listener.\nfunc (pp *ListenerWrapper) WrapListener(l net.Listener) net.Listener {\n\tpl := &goproxy.Listener{\n\t\tListener:          l,\n\t\tReadHeaderTimeout: time.Duration(pp.Timeout),\n\t}\n\tpl.Policy = pp.policy\n\treturn pl\n}\n\n// Path: modules/caddyhttp/proxyprotocol/module.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage proxyprotocol\n\nimport (\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(ListenerWrapper{})\n}\n\nfunc (ListenerWrapper) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"caddy.listeners.proxy_protocol\",\n\t\tNew: func() caddy.Module { return new(ListenerWrapper) },\n\t}\n}\n\n// UnmarshalCaddyfile sets up the listener Listenerwrapper from Caddyfile tokens. Syntax:\n//\n//\tproxy_protocol {\n//\t\ttimeout <duration>\n//\t\tallow <IPs...>\n//\t\tdeny <IPs...>\n//\t\tfallback_policy <policy>\n//\t}\nfunc (w *ListenerWrapper) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume wrapper name\n\n\t// No same-line options are supported\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"timeout\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tdur, err := caddy.ParseDuration(d.Val())\n\t\t\tif err != nil {\n\t\t\t\treturn d.Errf(\"parsing proxy_protocol timeout duration: %v\", err)\n\t\t\t}\n\t\t\tw.Timeout = caddy.Duration(dur)\n\n\t\tcase \"allow\":\n\t\t\tw.Allow = append(w.Allow, d.RemainingArgs()...)\n\t\tcase \"deny\":\n\t\t\tw.Deny = append(w.Deny, d.RemainingArgs()...)\n\t\tcase \"fallback_policy\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tp, err := parsePolicy(d.Val())\n\t\t\tif err != nil {\n\t\t\t\treturn d.WrapErr(err)\n\t\t\t}\n\t\t\tw.FallbackPolicy = p\n\t\tdefault:\n\t\t\treturn d.ArgErr()\n\t\t}\n\t}\n\treturn nil\n}\n\n// Interface guards\nvar (\n\t_ caddy.Provisioner     = (*ListenerWrapper)(nil)\n\t_ caddy.Module          = (*ListenerWrapper)(nil)\n\t_ caddy.ListenerWrapper = (*ListenerWrapper)(nil)\n\t_ caddyfile.Unmarshaler = (*ListenerWrapper)(nil)\n)\n\n// Path: modules/caddyhttp/proxyprotocol/policy.go\npackage proxyprotocol\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\n\tgoproxy \"github.com/pires/go-proxyproto\"\n)\n\ntype Policy int\n\n// as defined in: https://pkg.go.dev/github.com/pires/go-proxyproto@v0.7.0#Policy\nconst (\n\t// IGNORE address from PROXY header, but accept connection\n\tPolicyIGNORE Policy = iota\n\t// USE address from PROXY header\n\tPolicyUSE\n\t// REJECT connection when PROXY header is sent\n\t// Note: even though the first read on the connection returns an error if\n\t// a PROXY header is present, subsequent reads do not. It is the task of\n\t// the code using the connection to handle that case properly.\n\tPolicyREJECT\n\t// REQUIRE connection to send PROXY header, reject if not present\n\t// Note: even though the first read on the connection returns an error if\n\t// a PROXY header is not present, subsequent reads do not. It is the task\n\t// of the code using the connection to handle that case properly.\n\tPolicyREQUIRE\n...\n", "needle_token_start": 613, "needle_token_end": 925, "code_context_ntokens": 15424, "output": ["```go\nfunc BenchmarkHeaderREMatcher(b *testing.B) {\n\ti := 0\n\tmatch := MatchHeaderRE{\"Field\": &MatchRegexp{Pattern: \"^foo(.*)$\", Name: \"name\"}}\n\tinput := http.Header{\"Field\": []string{\"foobar\"}}\n\tvar host string\n\terr := match.Provision(caddy.Context{})\n\tif err != nil {\n\t\tb.Errorf(\"Test %d %v: Provisioning: %v\", i, match, err)\n\t}\n\terr = match.Validate()\n\tif err != nil {\n\t\tb.Errorf(\"Test %d %v: Validating: %v\", i, match, err)\n\t}\n\n\t// set up the fake request and its Replacer\n\treq := &http.Request{Header: input, URL: new(url.URL), Host: host}\n\trepl := caddy.NewReplacer()\n\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\treq = req.WithContext(ctx)\n\taddHTTPVarsToReplacer(repl, req, httptest.NewRecorder())\n\tfor run := 0; run < b.N; run++ {\n\t\tmatch.Match(req)\n\t}\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "hasVaryValue", "language": "go", "path": "modules/caddyhttp/encode/encode.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function checks if a specific header value is present in the 'Vary' header of an HTTP response.\n2. **Input**: It takes an HTTP header and a target string as inputs.\n3. **Output**: It returns a boolean value indicating whether the target string is present in the 'Vary' header.\n4. **Procedure**: The function retrieves the values of the 'Vary' header, splits each value by commas to handle multiple entries, and checks each entry against the target string using a case-insensitive comparison. If the target string is found, it returns true; otherwise, it continues checking until all entries are exhausted and returns false if the target is not found.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\treturn fmt.Errorf(\"encoder already added: %s\", ae)\n\t}\n\tif enc.writerPools == nil {\n\t\tenc.writerPools = make(map[string]*sync.Pool)\n\t}\n\tenc.writerPools[ae] = &sync.Pool{\n\t\tNew: func() any {\n\t\t\treturn e.NewEncoder()\n\t\t},\n\t}\n\treturn nil\n}\n\n// openResponseWriter creates a new response writer that may (or may not)\n// encode the response with encodingName. The returned response writer MUST\n// be closed after the handler completes.\nfunc (enc *Encode) openResponseWriter(encodingName string, w http.ResponseWriter) *responseWriter {\n\tvar rw responseWriter\n\treturn enc.initResponseWriter(&rw, encodingName, w)\n}\n\n// initResponseWriter initializes the responseWriter instance\n// allocated in openResponseWriter, enabling mid-stack inlining.\nfunc (enc *Encode) initResponseWriter(rw *responseWriter, encodingName string, wrappedRW http.ResponseWriter) *responseWriter {\n\tif rww, ok := wrappedRW.(*caddyhttp.ResponseWriterWrapper); ok {\n\t\trw.ResponseWriter = rww\n\t} else {\n\t\trw.ResponseWriter = &caddyhttp.ResponseWriterWrapper{ResponseWriter: wrappedRW}\n\t}\n\trw.encodingName = encodingName\n\trw.config = enc\n\n\treturn rw\n}\n\n// responseWriter writes to an underlying response writer\n// using the encoding represented by encodingName and\n// configured by config.\ntype responseWriter struct {\n\thttp.ResponseWriter\n\tencodingName string\n\tw            Encoder\n\tconfig       *Encode\n\tstatusCode   int\n\twroteHeader  bool\n}\n\n// WriteHeader stores the status to write when the time comes\n// to actually write the header.\nfunc (rw *responseWriter) WriteHeader(status int) {\n\trw.statusCode = status\n\n\t// See #5849 and RFC 9110 section 15.4.5 (https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5) - 304\n\t// Not Modified must have certain headers set as if it was a 200 response, and according to the issue\n\t// we would miss the Vary header in this case when compression was also enabled; note that we set this\n\t// header in the responseWriter.init() method but that is only called if we are writing a response body\n\tif status == http.StatusNotModified && !hasVaryValue(rw.Header(), \"Accept-Encoding\") {\n\t\trw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\t}\n\n\t// write status immediately when status code is informational\n\t// see: https://caddy.community/t/disappear-103-early-hints-response-with-encode-enable-caddy-v2-7-6/23081/5\n\tif 100 <= status && status <= 199 {\n\t\trw.ResponseWriter.WriteHeader(status)\n\t}\n}\n\n// Match determines, if encoding should be done based on the ResponseMatcher.\nfunc (enc *Encode) Match(rw *responseWriter) bool {\n\treturn enc.Matcher.Match(rw.statusCode, rw.Header())\n}\n\n// FlushError is an alternative Flush returning an error. It delays the actual Flush of the underlying\n// ResponseWriterWrapper until headers were written.\nfunc (rw *responseWriter) FlushError() error {\n\tif !rw.wroteHeader {\n\t\t// flushing the underlying ResponseWriter will write header and status code,\n\t\t// but we need to delay that until we can determine if we must encode and\n\t\t// therefore add the Content-Encoding header; this happens in the first call\n\t\t// to rw.Write (see bug in #4314)\n\t\treturn nil\n\t}\n\t//nolint:bodyclose\n\treturn http.NewResponseController(rw.ResponseWriter).Flush()\n}\n\n// Write writes to the response. If the response qualifies,\n// it is encoded using the encoder, which is initialized\n// if not done so already.\nfunc (rw *responseWriter) Write(p []byte) (int, error) {\n\t// ignore zero data writes, probably head request\n\tif len(p) == 0 {\n\t\treturn 0, nil\n\t}\n\n\t// sniff content-type and determine content-length\n\tif !rw.wroteHeader && rw.config.MinLength > 0 {\n\t\tvar gtMinLength bool\n\t\tif len(p) > rw.config.MinLength {\n\t\t\tgtMinLength = true\n\t\t} else if cl, err := strconv.Atoi(rw.Header().Get(\"Content-Length\")); err == nil && cl > rw.config.MinLength {\n\t\t\tgtMinLength = true\n\t\t}\n\n\t\tif gtMinLength {\n\t\t\tif rw.Header().Get(\"Content-Type\") == \"\" {\n\t\t\t\trw.Header().Set(\"Content-Type\", http.DetectContentType(p))\n\t\t\t}\n\t\t\trw.init()\n\t\t}\n\t}\n\n\t// before we write to the response, we need to make\n\t// sure the header is written exactly once; we do\n\t// that by checking if a status code has been set,\n\t// and if so, that means we haven't written the\n\t// header OR the default status code will be written\n\t// by the standard library\n\tif !rw.wroteHeader {\n\t\tif rw.statusCode != 0 {\n\t\t\trw.ResponseWriter.WriteHeader(rw.statusCode)\n\t\t}\n\t\trw.wroteHeader = true\n\t}\n\n\tif rw.w != nil {\n\t\treturn rw.w.Write(p)\n\t} else {\n\t\treturn rw.ResponseWriter.Write(p)\n\t}\n}\n\n// Close writes any remaining buffered response and\n// deallocates any active resources.\nfunc (rw *responseWriter) Close() error {\n\t// didn't write, probably head request\n\tif !rw.wroteHeader {\n\t\tcl, err := strconv.Atoi(rw.Header().Get(\"Content-Length\"))\n\t\tif err == nil && cl > rw.config.MinLength {\n\t\t\trw.init()\n\t\t}\n\n\t\t// issue #5059, don't write status code if not set explicitly.\n\t\tif rw.statusCode != 0 {\n\t\t\trw.ResponseWriter.WriteHeader(rw.statusCode)\n\t\t}\n\t\trw.wroteHeader = true\n\t}\n\n\tvar err error\n\tif rw.w != nil {\n\t\terr = rw.w.Close()\n\t\trw.w.Reset(nil)\n\t\trw.config.writerPools[rw.encodingName].Put(rw.w)\n\t\trw.w = nil\n\t}\n\treturn err\n}\n\n// Unwrap returns the underlying ResponseWriter.\nfunc (rw *responseWriter) Unwrap() http.ResponseWriter {\n\treturn rw.ResponseWriter\n}\n\n// init should be called before we write a response, if rw.buf has contents.\nfunc (rw *responseWriter) init() {\n\thdr := rw.Header()\n\tif hdr.Get(\"Content-Encoding\") == \"\" && isEncodeAllowed(hdr) &&\n\t\trw.config.Match(rw) {\n\t\trw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)\n\t\trw.w.Reset(rw.ResponseWriter)\n\t\thdr.Del(\"Content-Length\") // https://github.com/golang/go/issues/14975\n\t\thdr.Set(\"Content-Encoding\", rw.encodingName)\n\t\tif !hasVaryValue(hdr, \"Accept-Encoding\") {\n\t\t\thdr.Add(\"Vary\", \"Accept-Encoding\")\n\t\t}\n\t\thdr.Del(\"Accept-Ranges\") // we don't know ranges for dynamically-encoded content\n\n\t\t// strong ETags need to be distinct depending on the encoding (\"selected representation\")\n\t\t// see RFC 9110 section 8.8.3.3:\n\t\t// https://www.rfc-editor.org/rfc/rfc9110.html#name-example-entity-tags-varying\n\t\t// I don't know a great way to do this... how about appending? That's a neat trick!\n\t\t// (We have to strip the value we append from If-None-Match headers before\n\t\t// sending subsequent requests back upstream, however, since upstream handlers\n\t\t// don't know about our appending to their Etag since they've already done their work)\n\t\tif etag := hdr.Get(\"Etag\"); etag != \"\" && !strings.HasPrefix(etag, \"W/\") {\n\t\t\tetag = fmt.Sprintf(`%s-%s\"`, strings.TrimSuffix(etag, `\"`), rw.encodingName)\n\t\t\thdr.Set(\"Etag\", etag)\n\t\t}\n\t}\n}\n\n\nfunc hasVaryValue(hdr http.Header, target string) bool {\n\tfor _, vary := range hdr.Values(\"Vary\") {\n\t\tvals := strings.Split(vary, \",\")\n\t\tfor _, val := range vals {\n\t\t\tif strings.EqualFold(strings.TrimSpace(val), target) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// AcceptedEncodings returns the list of encodings that the\n// client supports, in descending order of preference.\n// The client preference via q-factor and the server\n// preference via Prefer setting are taken into account. If\n// the Sec-WebSocket-Key header is present then non-identity\n// encodings are not considered. See\n// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.\nfunc AcceptedEncodings(r *http.Request, preferredOrder []string) []string {\n\tacceptEncHeader := r.Header.Get(\"Accept-Encoding\")\n\twebsocketKey := r.Header.Get(\"Sec-WebSocket-Key\")\n\tif acceptEncHeader == \"\" {\n\t\treturn []string{}\n\t}\n\n\tprefs := []encodingPreference{}\n\n\tfor _, accepted := range strings.Split(acceptEncHeader, \",\") {\n\t\tparts := strings.Split(accepted, \";\")\n\t\tencName := strings.ToLower(strings.TrimSpace(parts[0]))\n\n\t\t// determine q-factor\n\t\tqFactor := 1.0\n\t\tif len(parts) > 1 {\n\t\t\tqFactorStr := strings.ToLower(strings.TrimSpace(parts[1]))\n\t\t\tif strings.HasPrefix(qFactorStr, \"q=\") {\n\t\t\t\tif qFactorFloat, err := strconv.ParseFloat(qFactorStr[2:], 32); err == nil {\n\t\t\t\t\tif qFactorFloat >= 0 && qFactorFloat <= 1 {\n\t\t\t\t\t\tqFactor = qFactorFloat\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// encodings with q-factor of 0 are not accepted;\n\t\t// use a small threshold to account for float precision\n\t\tif qFactor < 0.00001 {\n\t\t\tcontinue\n\t\t}\n\n\t\t// don't encode WebSocket handshakes\n\t\tif websocketKey != \"\" && encName != \"identity\" {\n\t\t\tcontinue\n\t\t}\n\n\t\t// set server preference\n\t\tprefOrder := -1\n\t\tfor i, p := range preferredOrder {\n\t\t\tif encName == p {\n\t\t\t\tprefOrder = len(preferredOrder) - i\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tprefs = append(prefs, encodingPreference{\n\t\t\tencoding:    encName,\n\t\t\tq:           qFactor,\n\t\t\tpreferOrder: prefOrder,\n\t\t})\n\t}\n\n\t// sort preferences by descending q-factor first, then by preferOrder\n\tsort.Slice(prefs, func(i, j int) bool {\n\t\tif math.Abs(prefs[i].q-prefs[j].q) < 0.00001 {\n\t\t\treturn prefs[i].preferOrder > prefs[j].preferOrder\n\t\t}\n\t\treturn prefs[i].q > prefs[j].q\n\t})\n\n\tprefEncNames := make([]string, len(prefs))\n\tfor i := range prefs {\n\t\tprefEncNames[i] = prefs[i].encoding\n\t}\n\n\treturn prefEncNames\n}\n\n// encodingPreference pairs an encoding with its q-factor.\ntype encodingPreference struct {\n\tencoding    string\n\tq           float64\n\tpreferOrder int\n}\n\n// Encoder is a type which can encode a stream of data.\ntype Encoder interface {\n\tio.WriteCloser\n\tReset(io.Writer)\n}\n\n// Encoding is a type which can create encoders of its kind\n// and return the name used in the Accept-Encoding header.\ntype Encoding interface {\n\tAcceptEncoding() string\n\tNewEncoder() Encoder\n}\n\n// Precompressed is a type which returns filename suffix of precompressed\n// file and Accept-Encoding header to use when serving this file.\ntype Precompressed interface {\n\tAcceptEncoding() string\n\tSuffix() string\n}\n\n// defaultMinLength is the minimum length at which to compress content.\nconst defaultMinLength = 512\n\n// Interface guards\nvar (\n\t_ caddy.Provisioner           = (*Encode)(nil)\n\t_ caddy.Validator             = (*Encode)(nil)\n\t_ caddyhttp.MiddlewareHandler = (*Encode)(nil)\n)\n\n// Path: modules/caddyhttp/fileserver/caddyfile.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fileserver\n\nimport (\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite\"\n)\n\nfunc init() {\n\thttpcaddyfile.RegisterHandlerDirective(\"file_server\", parseCaddyfile)\n\thttpcaddyfile.RegisterDirective(\"try_files\", parseTryFiles)\n}\n\n// parseCaddyfile parses the file_server directive.\n// See UnmarshalCaddyfile for the syntax.\nfunc parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {\n\tfsrv := new(FileServer)\n\terr := fsrv.UnmarshalCaddyfile(h.Dispenser)\n\tif err != nil {\n\t\treturn fsrv, err\n\t}\n\terr = fsrv.FinalizeUnmarshalCaddyfile(h)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn fsrv, err\n}\n\n// UnmarshalCaddyfile parses the file_server directive. It enables\n// the static file server and configures it with this syntax:\n//\n//\tfile_server [<matcher>] [browse] {\n//\t    fs            <filesystem>\n//\t    root          <path>\n//\t    hide          <files...>\n//\t    index         <files...>\n//\t    browse        [<template_file>]\n//\t    precompressed <formats...>\n//\t    status        <status>\n//\t    disable_canonical_uris\n//\t}\n//\n// The FinalizeUnmarshalCaddyfile method should be called after this\n// to finalize setup of hidden Caddyfiles.\nfunc (fsrv *FileServer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume directive name\n\n\targs := d.RemainingArgs()\n\tswitch len(args) {\n\tcase 0:\n\tcase 1:\n\t\tif args[0] != \"browse\" {\n\t\t\treturn d.ArgErr()\n\t\t}\n\t\tfsrv.Browse = new(Browse)\n\tdefault:\n\t\treturn d.ArgErr()\n\t}\n\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"fs\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tif fsrv.FileSystem != \"\" {\n\t\t\t\treturn d.Err(\"file system already specified\")\n\t\t\t}\n\t\t\tfsrv.FileSystem = d.Val()\n\n\t\tcase \"hide\":\n\t\t\tfsrv.Hide = d.RemainingArgs()\n\t\t\tif len(fsrv.Hide) == 0 {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\n\t\tcase \"index\":\n\t\t\tfsrv.IndexNames = d.RemainingArgs()\n\t\t\tif len(fsrv.IndexNames) == 0 {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\n\t\tcase \"root\":\n\t\t\tif !d.Args(&fsrv.Root) {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\n\t\tcase \"browse\":\n\t\t\tif fsrv.Browse != nil {\n\t\t\t\treturn d.Err(\"browsing is already configured\")\n\t\t\t}\n\t\t\tfsrv.Browse = new(Browse)\n\t\t\td.Args(&fsrv.Browse.TemplateFile)\n\t\t\tfor nesting := d.Nesting(); d.NextBlock(nesting); {\n\t\t\t\tswitch d.Val() {\n\t\t\t\tcase \"reveal_symlinks\":\n\t\t\t\t\tif fsrv.Browse.RevealSymlinks {\n\t\t\t\t\t\treturn d.Err(\"Symlinks path reveal is already enabled\")\n\t\t\t\t\t}\n\t\t\t\t\tfsrv.Browse.RevealSymlinks = true\n\t\t\t\tdefault:\n\t\t\t\t\treturn d.Errf(\"unknown subdirective '%s'\", d.Val())\n\t\t\t\t}\n\t\t\t}\n\n\t\tcase \"precompressed\":\n\t\t\tvar order []string\n\t\t\tfor d.NextArg() {\n\t\t\t\tmodID := \"http.precompressed.\" + d.Val()\n\t\t\t\tmod, err := caddy.GetModule(modID)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn d.Errf(\"getting module named '%s': %v\", modID, err)\n\t\t\t\t}\n\t\t\t\tinst := mod.New()\n\t\t\t\tprecompress, ok := inst.(encode.Precompressed)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn d.Errf(\"module %s is not a precompressor; is %T\", modID, inst)\n\t\t\t\t}\n\t\t\t\tif fsrv.PrecompressedRaw == nil {\n\t\t\t\t\tfsrv.PrecompressedRaw = make(caddy.ModuleMap)\n\t\t\t\t}\n\t\t\t\tfsrv.PrecompressedRaw[d.Val()] = caddyconfig.JSON(precompress, nil)\n\t\t\t\torder = append(order, d.Val())\n\t\t\t}\n\t\t\tfsrv.PrecompressedOrder = order\n\n\t\tcase \"status\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tfsrv.StatusCode = caddyhttp.WeakString(d.Val())\n\n\t\tcase \"disable_canonical_uris\":\n\t\t\tif d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tfalseBool := false\n\t\t\tfsrv.CanonicalURIs = &falseBool\n\n\t\tcase \"pass_thru\":\n\t\t\tif d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tfsrv.PassThru = true\n\n\t\tcase \"etag_file_extensions\":\n\t\t\tetagFileExtensions := d.RemainingArgs()\n\t\t\tif len(etagFileExtensions) == 0 {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tfsrv.EtagFileExtensions = etagFileExtensions\n\n\t\tdefault:\n\t\t\treturn d.Errf(\"unknown subdirective '%s'\", d.Val())\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// FinalizeUnmarshalCaddyfile finalizes the Caddyfile parsing which\n// requires having an httpcaddyfile.Helper to function, to setup hidden Caddyfiles.\nfunc (fsrv *FileServer) FinalizeUnmarshalCaddyfile(h httpcaddyfile.Helper) error {\n\t// Hide the Caddyfile (and any imported Caddyfiles).\n\t// This needs to be done in here instead of UnmarshalCaddyfile\n\t// because UnmarshalCaddyfile only has access to the dispenser\n\t// and not the helper, and only the helper has access to the\n\t// Caddyfiles function.\n\tif configFiles := h.Caddyfiles(); len(configFiles) > 0 {\n\t\tfor _, file := range configFiles {\n\t\t\tfile = filepath.Clean(file)\n\t\t\tif !fileHidden(file, fsrv.Hide) {\n\t\t\t\t// if there's no path separator, the file server module will hide all\n\t\t\t\t// files by that name, rather than a specific one; but we want to hide\n\t\t\t\t// only this specific file, so ensure there's always a path separator\n\t\t\t\tif !strings.Contains(file, separator) {\n\t\t\t\t\tfile = \".\" + separator + file\n\t\t\t\t}\n\t\t\t\tfsrv.Hide = append(fsrv.Hide, file)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// parseTryFiles parses the try_files directive. It combines a file matcher\n// with a rewrite directive, so this is not a standard handler directive.\n// A try_files directive has this syntax (notice no matcher tokens accepted):\n//\n//\ttry_files <files...> {\n//\t\tpolicy first_exist|smallest_size|largest_size|most_recently_modified\n//\t}\n//\n// and is basically shorthand for:\n//\n//\t@try_files file {\n//\t\ttry_files <files...>\n//\t\tpolicy first_exist|smallest_size|largest_size|most_recently_modified\n//\t}\n//\trewrite @try_files {http.matchers.file.relative}\n//\n// This directive rewrites request paths only, preserving any other part\n// of the URI, unless the part is explicitly given in the file list. For\n// example, if any of the files in the list have a query string:\n//\n//\ttry_files {path} index.php?{query}&p={path}\n//\n// then the query string will not be treated as part of the file name; and\n// if that file matches, the given query string will replace any query string\n// that already exists on the request URI.\nfunc parseTryFiles(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) {\n\tif !h.Next() {\n\t\treturn nil, h.ArgErr()\n\t}\n\n\ttryFiles := h.RemainingArgs()\n\tif len(tryFiles) == 0 {\n\t\treturn nil, h.ArgErr()\n\t}\n\n\t// parse out the optional try policy\n\tvar tryPolicy string\n\tfor h.NextBlock(0) {\n\t\tswitch h.Val() {\n\t\tcase \"policy\":\n\t\t\tif tryPolicy != \"\" {\n\t\t\t\treturn nil, h.Err(\"try policy already configured\")\n\t\t\t}\n\t\t\tif !h.NextArg() {\n\t\t\t\treturn nil, h.ArgErr()\n\t\t\t}\n\t\t\ttryPolicy = h.Val()\n\n\t\t\tswitch tryPolicy {\n\t\t\tcase tryPolicyFirstExist, tryPolicyLargestSize, tryPolicySmallestSize, tryPolicyMostRecentlyMod:\n\t\t\tdefault:\n\t\t\t\treturn nil, h.Errf(\"unrecognized try policy: %s\", tryPolicy)\n\t\t\t}\n\t\t}\n\t}\n\n\t// makeRoute returns a route that tries the files listed in try\n\t// and then rewrites to the matched file; userQueryString is\n\t// appended to the rewrite rule.\n\tmakeRoute := func(try []string, userQueryString string) []httpcaddyfile.ConfigValue {\n\t\thandler := rewrite.Rewrite{\n\t\t\tURI: \"{http.matchers.file.relative}\" + userQueryString,\n\t\t}\n\t\tmatcherSet := caddy.ModuleMap{\n\t\t\t\"file\": h.JSON(MatchFile{TryFiles: try, TryPolicy: tryPolicy}),\n\t\t}\n\t\treturn h.NewRoute(matcherSet, handler)\n\t}\n\n\tvar result []httpcaddyfile.ConfigValue\n\n\t// if there are query strings in the list, we have to split into\n\t// a separate route for each item with a query string, because\n\t// the rewrite is different for that item\n\ttry := make([]string, 0, len(tryFiles))\n\tfor _, item := range tryFiles {\n\t\tif idx := strings.Index(item, \"?\"); idx >= 0 {\n\t\t\tif len(try) > 0 {\n\t\t\t\tresult = append(result, makeRoute(try, \"\")...)\n\t\t\t\ttry = []string{}\n\t\t\t}\n\t\t\tresult = append(result, makeRoute([]string{item[:idx]}, item[idx:])...)\n\t\t\tcontinue\n\t\t}\n\t\t// accumulate consecutive non-query-string parameters\n\t\ttry = append(try, item)\n\t}\n\tif len(try) > 0 {\n\t\tresult = append(result, makeRoute(try, \"\")...)\n\t}\n\n\t// ensure that multiple routes (possible if rewrite targets\n\t// have query strings, for example) are grouped together\n\t// so only the first matching rewrite is performed (#2891)\n\th.GroupRoutes(result)\n\n\treturn result, nil\n}\n\nvar _ caddyfile.Unmarshaler = (*FileServer)(nil)\n\n// Path: modules/caddyhttp/encode/caddyfile.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage encode\n\nimport (\n\t\"strconv\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc init() {\n\thttpcaddyfile.RegisterHandlerDirective(\"encode\", parseCaddyfile)\n}\n\nfunc parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {\n\tenc := new(Encode)\n\terr := enc.UnmarshalCaddyfile(h.Dispenser)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn enc, nil\n}\n\n// UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:\n//\n//\tencode [<matcher>] <formats...> {\n//\t    gzip           [<level>]\n//\t    zstd\n//\t    minimum_length <length>\n//\t    # response matcher block\n//\t    match {\n//\t        status <code...>\n//\t        header <field> [<value>]\n//\t    }\n//\t    # or response matcher single line syntax\n//\t    match [header <field> [<value>]] | [status <code...>]\n//\t}\n//\n// Specifying the formats on the first line will use those formats' defaults.\nfunc (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume directive name\n\n\tprefer := []string{}\n\tfor _, arg := range d.RemainingArgs() {\n\t\tmod, err := caddy.GetModule(\"http.encoders.\" + arg)\n\t\tif err != nil {\n\t\t\treturn d.Errf(\"finding encoder module '%s': %v\", mod, err)\n\t\t}\n\t\tencoding, ok := mod.New().(Encoding)\n\t\tif !ok {\n\t\t\treturn d.Errf(\"module %s is not an HTTP encoding\", mod)\n\t\t}\n\t\tif enc.EncodingsRaw == nil {\n\t\t\tenc.EncodingsRaw = make(caddy.ModuleMap)\n\t\t}\n\t\tenc.EncodingsRaw[arg] = caddyconfig.JSON(encoding, nil)\n\t\tprefer = append(prefer, arg)\n\t}\n\n\tresponseMatchers := make(map[string]caddyhttp.ResponseMatcher)\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"minimum_length\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tminLength, err := strconv.Atoi(d.Val())\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tenc.MinLength = minLength\n\t\tcase \"match\":\n\t\t\terr := caddyhttp.ParseNamedResponseMatcher(d.NewFromNextSegment(), responseMatchers)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tmatcher := responseMatchers[\"match\"]\n\t\t\tenc.Matcher = &matcher\n\t\tdefault:\n\t\t\tname := d.Val()\n\t\t\tmodID := \"http.encoders.\" + name\n\t\t\tunm, err := caddyfile.UnmarshalModule(d, modID)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tencoding, ok := unm.(Encoding)\n\t\t\tif !ok {\n\t\t\t\treturn d.Errf(\"module %s is not an HTTP encoding; is %T\", modID, unm)\n\t\t\t}\n\t\t\tif enc.EncodingsRaw == nil {\n\t\t\t\tenc.EncodingsRaw = make(caddy.ModuleMap)\n\t\t\t}\n\t\t\tenc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)\n\t\t\tprefer = append(prefer, name)\n\t\t}\n\t}\n\n\t// use the order in which the encoders were defined.\n\tenc.Prefer = prefer\n\n\treturn nil\n}\n\n// Interface guard\nvar _ caddyfile.Unmarshaler = (*Encode)(nil)\n\n// Path: modules/caddyevents/eventsconfig/caddyfile.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Package eventsconfig is for configuring caddyevents.App with the\n// Caddyfile. This code can't be in the caddyevents package because\n// the httpcaddyfile package imports caddyhttp, which imports\n// caddyevents: hence, it creates an import cycle.\npackage eventsconfig\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyevents\"\n)\n\nfunc init() {\n\thttpcaddyfile.RegisterGlobalOption(\"events\", parseApp)\n}\n\n// parseApp configures the \"events\" global option from Caddyfile to set up the events app.\n// Syntax:\n//\n//\tevents {\n//\t\ton <event> <handler_module...>\n//\t}\n//\n// If <event> is *, then it will bind to all events.\nfunc parseApp(d *caddyfile.Dispenser, _ any) (any, error) {\n\td.Next() // consume option name\n\tapp := new(caddyevents.App)\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"on\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn nil, d.ArgErr()\n\t\t\t}\n\t\t\teventName := d.Val()\n\t\t\tif eventName == \"*\" {\n\t\t\t\teventName = \"\"\n\t\t\t}\n\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn nil, d.ArgErr()\n\t\t\t}\n\t\t\thandlerName := d.Val()\n\t\t\tmodID := \"events.handlers.\" + handlerName\n\t\t\tunm, err := caddyfile.UnmarshalModule(d, modID)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tapp.Subscriptions = append(app.Subscriptions, &caddyevents.Subscription{\n\t\t\t\tEvents: []string{eventName},\n\t\t\t\tHandlersRaw: []json.RawMessage{\n\t\t\t\t\tcaddyconfig.JSONModuleObject(unm, \"handler\", handlerName, nil),\n\t\t\t\t},\n\t\t\t})\n\n\t\tdefault:\n\t\t\treturn nil, d.ArgErr()\n\t\t}\n\t}\n\n\treturn httpcaddyfile.App{\n\t\tName:  \"events\",\n\t\tValue: caddyconfig.JSON(app, nil),\n\t}, nil\n}\n\n// Path: modules/caddyhttp/caddyhttp.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(tlsPlaceholderWrapper{})\n}\n\n// RequestMatcher is a type that can match to a request.\n// A route matcher MUST NOT modify the request, with the\n// only exception being its context.\ntype RequestMatcher interface {\n\tMatch(*http.Request) bool\n}\n\n// Handler is like http.Handler except ServeHTTP may return an error.\n//\n// If any handler encounters an error, it should be returned for proper\n// handling. Return values should be propagated down the middleware chain\n// by returning it unchanged. Returned errors should not be re-wrapped\n// if they are already HandlerError values.\ntype Handler interface {\n\tServeHTTP(http.ResponseWriter, *http.Request) error\n}\n\n// HandlerFunc is a convenience type like http.HandlerFunc.\ntype HandlerFunc func(http.ResponseWriter, *http.Request) error\n\n// ServeHTTP implements the Handler interface.\nfunc (f HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) error {\n\treturn f(w, r)\n}\n\n// Middleware chains one Handler to the next by being passed\n// the next Handler in the chain.\ntype Middleware func(Handler) Handler\n\n// MiddlewareHandler is like Handler except it takes as a third\n// argument the next handler in the chain. The next handler will\n// never be nil, but may be a no-op handler if this is the last\n// handler in the chain. Handlers which act as middleware should\n// call the next handler's ServeHTTP method so as to propagate\n// the request down the chain properly. Handlers which act as\n// responders (content origins) need not invoke the next handler,\n// since the last handler in the chain should be the first to\n// write the response.\ntype MiddlewareHandler interface {\n\tServeHTTP(http.ResponseWriter, *http.Request, Handler) error\n}\n\n// emptyHandler is used as a no-op handler.\nvar emptyHandler Handler = HandlerFunc(func(http.ResponseWriter, *http.Request) error { return nil })\n\n// An implicit suffix middleware that, if reached, sets the StatusCode to the\n// error stored in the ErrorCtxKey. This is to prevent situations where the\n// Error chain does not actually handle the error (for instance, it matches only\n// on some errors). See #3053\nvar errorEmptyHandler Handler = HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {\n\thttpError := r.Context().Value(ErrorCtxKey)\n\tif handlerError, ok := httpError.(HandlerError); ok {\n\t\tw.WriteHeader(handlerError.StatusCode)\n\t} else {\n\t\tw.WriteHeader(http.StatusInternalServerError)\n\t}\n\treturn nil\n})\n\n// ResponseHandler pairs a response matcher with custom handling\n// logic. Either the status code can be changed to something else\n// while using the original response body, or, if a status code\n// is not set, it can execute a custom route list; this is useful\n// for executing handler routes based on the properties of an HTTP\n// response that has not been written out to the client yet.\n//\n// To use this type, provision it at module load time, then when\n// ready to use, match the response against its matcher; if it\n// matches (or doesn't have a matcher), change the status code on\n// the response if configured; otherwise invoke the routes by\n// calling `rh.Routes.Compile(next).ServeHTTP(rw, req)` (or similar).\ntype ResponseHandler struct {\n\t// The response matcher for this handler. If empty/nil,\n\t// it always matches.\n\tMatch *ResponseMatcher `json:\"match,omitempty\"`\n\n\t// To write the original response body but with a different\n\t// status code, set this field to the desired status code.\n\t// If set, this takes priority over routes.\n\tStatusCode WeakString `json:\"status_code,omitempty\"`\n\n\t// The list of HTTP routes to execute if no status code is\n\t// specified. If evaluated, the original response body\n\t// will not be written.\n\tRoutes RouteList `json:\"routes,omitempty\"`\n}\n\n// Provision sets up the routse in rh.\nfunc (rh *ResponseHandler) Provision(ctx caddy.Context) error {\n\tif rh.Routes != nil {\n\t\terr := rh.Routes.Provision(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// WeakString is a type that unmarshals any JSON value\n// as a string literal, with the following exceptions:\n//\n// 1. actual string values are decoded as strings; and\n// 2. null is decoded as empty string;\n//\n// and provides methods for getting the value as various\n// primitive types. However, using this type removes any\n// type safety as far as deserializing JSON is concerned.\ntype WeakString string\n\n// UnmarshalJSON satisfies json.Unmarshaler according to\n// this type's documentation.\nfunc (ws *WeakString) UnmarshalJSON(b []byte) error {\n\tif len(b) == 0 {\n\t\treturn io.EOF\n\t}\n\tif b[0] == byte('\"') && b[len(b)-1] == byte('\"') {\n\t\tvar s string\n\t\terr := json.Unmarshal(b, &s)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t*ws = WeakString(s)\n\t\treturn nil\n\t}\n\tif bytes.Equal(b, []byte(\"null\")) {\n\t\treturn nil\n\t}\n\t*ws = WeakString(b)\n\treturn nil\n}\n\n// MarshalJSON marshals was a boolean if true or false,\n// a number if an integer, or a string otherwise.\nfunc (ws WeakString) MarshalJSON() ([]byte, error) {\n\tif ws == \"true\" {\n\t\treturn []byte(\"true\"), nil\n\t}\n\tif ws == \"false\" {\n\t\treturn []byte(\"false\"), nil\n\t}\n\tif num, err := strconv.Atoi(string(ws)); err == nil {\n\t\treturn json.Marshal(num)\n\t}\n\treturn json.Marshal(string(ws))\n}\n\n// Int returns ws as an integer. If ws is not an\n// integer, 0 is returned.\nfunc (ws WeakString) Int() int {\n\tnum, _ := strconv.Atoi(string(ws))\n\treturn num\n}\n\n// Float64 returns ws as a float64. If ws is not a\n// float value, the zero value is returned.\nfunc (ws WeakString) Float64() float64 {\n\tnum, _ := strconv.ParseFloat(string(ws), 64)\n\treturn num\n}\n\n// Bool returns ws as a boolean. If ws is not a\n// boolean, false is returned.\nfunc (ws WeakString) Bool() bool {\n\treturn string(ws) == \"true\"\n}\n\n// String returns ws as a string.\nfunc (ws WeakString) String() string {\n\treturn string(ws)\n}\n\n// StatusCodeMatches returns true if a real HTTP status code matches\n// the configured status code, which may be either a real HTTP status\n// code or an integer representing a class of codes (e.g. 4 for all\n// 4xx statuses).\nfunc StatusCodeMatches(actual, configured int) bool {\n\tif actual == configured {\n\t\treturn true\n\t}\n\tif configured < 100 &&\n\t\tactual >= configured*100 &&\n\t\tactual < (configured+1)*100 {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// SanitizedPathJoin performs filepath.Join(root, reqPath) that\n// is safe against directory traversal attacks. It uses logic\n// similar to that in the Go standard library, specifically\n// in the implementation of http.Dir. The root is assumed to\n// be a trusted path, but reqPath is not; and the output will\n// never be outside of root. The resulting path can be used\n// with the local file system. If root is empty, the current\n// directory is assumed. If the cleaned request path is deemed\n// not local according to lexical processing (i.e. ignoring links),\n// it will be rejected as unsafe and only the root will be returned.\nfunc SanitizedPathJoin(root, reqPath string) string {\n\tif root == \"\" {\n\t\troot = \".\"\n\t}\n\n\trelPath := path.Clean(\"/\" + reqPath)[1:] // clean path and trim the leading /\n\tif !filepath.IsLocal(relPath) {\n\t\t// path is unsafe (see https://github.com/golang/go/issues/56336#issuecomment-1416214885)\n\t\treturn root\n\t}\n\n\tpath := filepath.Join(root, filepath.FromSlash(relPath))\n\n\t// filepath.Join also cleans the path, and cleaning strips\n\t// the trailing slash, so we need to re-add it afterwards.\n\t// if the length is 1, then it's a path to the root,\n\t// and that should return \".\", so we don't append the separator.\n\tif strings.HasSuffix(reqPath, \"/\") && len(reqPath) > 1 {\n\t\tpath += separator\n\t}\n\n\treturn path\n}\n\n// CleanPath cleans path p according to path.Clean(), but only\n// merges repeated slashes if collapseSlashes is true, and always\n// preserves trailing slashes.\nfunc CleanPath(p string, collapseSlashes bool) string {\n\tif collapseSlashes {\n\t\treturn cleanPath(p)\n\t}\n\n\t// insert an invalid/impossible URI character into each two consecutive\n\t// slashes to expand empty path segments; then clean the path as usual,\n\t// and then remove the remaining temporary characters.\n\tconst tmpCh = 0xff\n\tvar sb strings.Builder\n\tfor i, ch := range p {\n\t\tif ch == '/' && i > 0 && p[i-1] == '/' {\n\t\t\tsb.WriteByte(tmpCh)\n\t\t}\n\t\tsb.WriteRune(ch)\n\t}\n\thalfCleaned := cleanPath(sb.String())\n\thalfCleaned = strings.ReplaceAll(halfCleaned, string([]byte{tmpCh}), \"\")\n\n\treturn halfCleaned\n}\n\n// cleanPath does path.Clean(p) but preserves any trailing slash.\nfunc cleanPath(p string) string {\n\tcleaned := path.Clean(p)\n\tif cleaned != \"/\" && strings.HasSuffix(p, \"/\") {\n\t\tcleaned = cleaned + \"/\"\n\t}\n\treturn cleaned\n}\n\n// tlsPlaceholderWrapper is a no-op listener wrapper that marks\n// where the TLS listener should be in a chain of listener wrappers.\n// It should only be used if another listener wrapper must be placed\n// in front of the TLS handshake.\ntype tlsPlaceholderWrapper struct{}\n\nfunc (tlsPlaceholderWrapper) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"caddy.listeners.tls\",\n\t\tNew: func() caddy.Module { return new(tlsPlaceholderWrapper) },\n\t}\n}\n\nfunc (tlsPlaceholderWrapper) WrapListener(ln net.Listener) net.Listener { return ln }\n\nfunc (tlsPlaceholderWrapper) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return nil }\n\nconst (\n\t// DefaultHTTPPort is the default port for HTTP.\n\tDefaultHTTPPort = 80\n\n\t// DefaultHTTPSPort is the default port for HTTPS.\n\tDefaultHTTPSPort = 443\n)\n\nconst separator = string(filepath.Separator)\n\n// Interface guard\nvar (\n\t_ caddy.ListenerWrapper = (*tlsPlaceholderWrapper)(nil)\n\t_ caddyfile.Unmarshaler = (*tlsPlaceholderWrapper)(nil)\n)\n\n// Path: modules/caddyhttp/headers/headers.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage headers\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(Handler{})\n}\n\n// Handler is a middleware which modifies request and response headers.\n//\n// Changes to headers are applied immediately, except for the response\n// headers when Deferred is true or when Required is set. In those cases,\n// the changes are applied when the headers are written to the response.\n// Note that deferred changes do not take effect if an error occurs later\n// in the middleware chain.\n//\n// Properties in this module accept placeholders.\n//\n// Response header operations can be conditioned upon response status code\n// and/or other header values.\ntype Handler struct {\n\tRequest  *HeaderOps     `json:\"request,omitempty\"`\n\tResponse *RespHeaderOps `json:\"response,omitempty\"`\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (Handler) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.handlers.headers\",\n\t\tNew: func() caddy.Module { return new(Handler) },\n\t}\n}\n\n// Provision sets up h's configuration.\nfunc (h *Handler) Provision(ctx caddy.Context) error {\n\tif h.Request != nil {\n\t\terr := h.Request.Provision(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif h.Response != nil {\n\t\terr := h.Response.Provision(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Validate ensures h's configuration is valid.\nfunc (h Handler) Validate() error {\n\tif h.Request != nil {\n\t\terr := h.Request.validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif h.Response != nil {\n\t\terr := h.Response.validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\n\tif h.Request != nil {\n\t\th.Request.ApplyToRequest(r)\n\t}\n\n\tif h.Response != nil {\n\t\tif h.Response.Deferred || h.Response.Require != nil {\n\t\t\tw = &responseWriterWrapper{\n\t\t\t\tResponseWriterWrapper: &caddyhttp.ResponseWriterWrapper{ResponseWriter: w},\n\t\t\t\treplacer:              repl,\n\t\t\t\trequire:               h.Response.Require,\n\t\t\t\theaderOps:             h.Response.HeaderOps,\n\t\t\t}\n\t\t} else {\n\t\t\th.Response.ApplyTo(w.Header(), repl)\n\t\t}\n\t}\n\n\treturn next.ServeHTTP(w, r)\n}\n\n// HeaderOps defines manipulations for HTTP headers.\ntype HeaderOps struct {\n\t// Adds HTTP headers; does not replace any existing header fields.\n\tAdd http.Header `json:\"add,omitempty\"`\n\n\t// Sets HTTP headers; replaces existing header fields.\n\tSet http.Header `json:\"set,omitempty\"`\n\n\t// Names of HTTP header fields to delete. Basic wildcards are supported:\n\t//\n\t// - Start with `*` for all field names with the given suffix;\n\t// - End with `*` for all field names with the given prefix;\n\t// - Start and end with `*` for all field names containing a substring.\n\tDelete []string `json:\"delete,omitempty\"`\n\n\t// Performs in-situ substring replacements of HTTP headers.\n\t// Keys are the field names on which to perform the associated replacements.\n\t// If the field name is `*`, the replacements are performed on all header fields.\n\tReplace map[string][]Replacement `json:\"replace,omitempty\"`\n}\n\n// Provision sets up the header operations.\nfunc (ops *HeaderOps) Provision(_ caddy.Context) error {\n\tfor fieldName, replacements := range ops.Replace {\n\t\tfor i, r := range replacements {\n\t\t\tif r.SearchRegexp != \"\" {\n\t\t\t\tre, err := regexp.Compile(r.SearchRegexp)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn fmt.Errorf(\"replacement %d for header field '%s': %v\", i, fieldName, err)\n\t\t\t\t}\n\t\t\t\treplacements[i].re = re\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (ops HeaderOps) validate() error {\n\tfor fieldName, replacements := range ops.Replace {\n\t\tfor _, r := range replacements {\n\t\t\tif r.Search != \"\" && r.SearchRegexp != \"\" {\n\t\t\t\treturn fmt.Errorf(\"cannot specify both a substring search and a regular expression search for field '%s'\", fieldName)\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// Replacement describes a string replacement,\n// either a simple and fast substring search\n// or a slower but more powerful regex search.\ntype Replacement struct {\n\t// The substring to search for.\n\tSearch string `json:\"search,omitempty\"`\n\n\t// The regular expression to search with.\n\tSearchRegexp string `json:\"search_regexp,omitempty\"`\n\n\t// The string with which to replace matches.\n\tReplace string `json:\"replace,omitempty\"`\n\n\tre *regexp.Regexp\n}\n\n// RespHeaderOps defines manipulations for response headers.\ntype RespHeaderOps struct {\n\t*HeaderOps\n\n\t// If set, header operations will be deferred until\n\t// they are written out and only performed if the\n\t// response matches these criteria.\n\tRequire *caddyhttp.ResponseMatcher `json:\"require,omitempty\"`\n\n\t// If true, header operations will be deferred until\n\t// they are written out. Superceded if Require is set.\n\t// Usually you will need to set this to true if any\n\t// fields are being deleted.\n\tDeferred bool `json:\"deferred,omitempty\"`\n}\n\n// ApplyTo applies ops to hdr using repl.\nfunc (ops HeaderOps) ApplyTo(hdr http.Header, repl *caddy.Replacer) {\n\t// before manipulating headers in other ways, check if there\n\t// is configuration to delete all headers, and do that first\n\t// because if a header is to be added, we don't want to delete\n\t// it also\n\tfor _, fieldName := range ops.Delete {\n\t\tfieldName = repl.ReplaceKnown(fieldName, \"\")\n\t\tif fieldName == \"*\" {\n\t\t\tfor existingField := range hdr {\n\t\t\t\tdelete(hdr, existingField)\n\t\t\t}\n\t\t}\n\t}\n\n\t// add\n\tfor fieldName, vals := range ops.Add {\n\t\tfieldName = repl.ReplaceKnown(fieldName, \"\")\n\t\tfor _, v := range vals {\n\t\t\thdr.Add(fieldName, repl.ReplaceKnown(v, \"\"))\n\t\t}\n\t}\n\n\t// set\n\tfor fieldName, vals := range ops.Set {\n\t\tfieldName = repl.ReplaceKnown(fieldName, \"\")\n\t\tvar newVals []string\n\t\tfor i := range vals {\n\t\t\t// append to new slice so we don't overwrite\n\t\t\t// the original values in ops.Set\n\t\t\tnewVals = append(newVals, repl.ReplaceKnown(vals[i], \"\"))\n\t\t}\n\t\thdr.Set(fieldName, strings.Join(newVals, \",\"))\n\t}\n\n\t// delete\n\tfor _, fieldName := range ops.Delete {\n\t\tfieldName = strings.ToLower(repl.ReplaceKnown(fieldName, \"\"))\n\t\tif fieldName == \"*\" {\n\t\t\tcontinue // handled above\n\t\t}\n\t\tswitch {\n\t\tcase strings.HasPrefix(fieldName, \"*\") && strings.HasSuffix(fieldName, \"*\"):\n\t\t\tfor existingField := range hdr {\n\t\t\t\tif strings.Contains(strings.ToLower(existingField), fieldName[1:len(fieldName)-1]) {\n\t\t\t\t\tdelete(hdr, existingField)\n\t\t\t\t}\n\t\t\t}\n\t\tcase strings.HasPrefix(fieldName, \"*\"):\n\t\t\tfor existingField := range hdr {\n\t\t\t\tif strings.HasSuffix(strings.ToLower(existingField), fieldName[1:]) {\n\t\t\t\t\tdelete(hdr, existingField)\n\t\t\t\t}\n\t\t\t}\n\t\tcase strings.HasSuffix(fieldName, \"*\"):\n\t\t\tfor existingField := range hdr {\n\t\t\t\tif strings.HasPrefix(strings.ToLower(existingField), fieldName[:len(fieldName)-1]) {\n\t\t\t\t\tdelete(hdr, existingField)\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\thdr.Del(fieldName)\n\t\t}\n\t}\n\n\t// replace\n\tfor fieldName, replacements := range ops.Replace {\n\t\tfieldName = http.CanonicalHeaderKey(repl.ReplaceKnown(fieldName, \"\"))\n\n\t\t// all fields...\n\t\tif fieldName == \"*\" {\n\t\t\tfor _, r := range replacements {\n\t\t\t\tsearch := repl.ReplaceKnown(r.Search, \"\")\n\t\t\t\treplace := repl.ReplaceKnown(r.Replace, \"\")\n\t\t\t\tfor fieldName, vals := range hdr {\n\t\t\t\t\tfor i := range vals {\n\t\t\t\t\t\tif r.re != nil {\n\t\t\t\t\t\t\thdr[fieldName][i] = r.re.ReplaceAllString(hdr[fieldName][i], replace)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\thdr[fieldName][i] = strings.ReplaceAll(hdr[fieldName][i], search, replace)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// ...or only with the named field\n\t\tfor _, r := range replacements {\n\t\t\tsearch := repl.ReplaceKnown(r.Search, \"\")\n\t\t\treplace := repl.ReplaceKnown(r.Replace, \"\")\n\t\t\tfor hdrFieldName, vals := range hdr {\n\t\t\t\t// see issue #4330 for why we don't simply use hdr[fieldName]\n\t\t\t\tif http.CanonicalHeaderKey(hdrFieldName) != fieldName {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tfor i := range vals {\n\t\t\t\t\tif r.re != nil {\n\t\t\t\t\t\thdr[hdrFieldName][i] = r.re.ReplaceAllString(hdr[hdrFieldName][i], replace)\n\t\t\t\t\t} else {\n...\n", "needle_token_start": 2245, "needle_token_end": 2351, "code_context_ntokens": 15346, "output": ["```go\nfunc hasVaryValue(hdr http.Header, target string) bool {\n\tfor _, vary := range hdr.Values(\"Vary\") {\n\t\tvals := strings.Split(vary, \",\")\n\t\tfor _, val := range vals {\n\t\t\tif strings.EqualFold(strings.TrimSpace(val), target) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "CELMatcherImpl", "language": "go", "path": "modules/caddyhttp/celmatcher.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a new library for expression matching, allowing the integration of custom matchers into expressions. It facilitates the use of specific data types and matching logic within the Common Expression Language (CEL) environment.\n2. **Input**: The function receives a macro name, a function name, a list of data types for the matcher, and a factory for creating matcher instances.\n3. **Output**: It returns a new CEL library configured with the specified macro and function, or an error if the provided data types or names are unsupported or cause conflicts.\n4. **Procedure**: \n   - The function first defines a request type.\n   - Based on the number and types of the input data types, it selects and configures a macro for expression parsing.\n   - It sets up an environment with the defined macro and a function that binds runtime logic for the matcher.\n   - It configures program options with custom decorators for additional runtime behavior.\n   - Finally, it constructs and returns a new CEL library using the environment and program options configured.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": ")\n\nfunc init() {\n\tcaddy.RegisterModule(MatchExpression{})\n}\n\n// MatchExpression matches requests by evaluating a\n// [CEL](https://github.com/google/cel-spec) expression.\n// This enables complex logic to be expressed using a comfortable,\n// familiar syntax. Please refer to\n// [the standard definitions of CEL functions and operators](https://github.com/google/cel-spec/blob/master/doc/langdef.md#standard-definitions).\n//\n// This matcher's JSON interface is actually a string, not a struct.\n// The generated docs are not correct because this type has custom\n// marshaling logic.\n//\n// COMPATIBILITY NOTE: This module is still experimental and is not\n// subject to Caddy's compatibility guarantee.\ntype MatchExpression struct {\n\t// The CEL expression to evaluate. Any Caddy placeholders\n\t// will be expanded and situated into proper CEL function\n\t// calls before evaluating.\n\tExpr string `json:\"expr,omitempty\"`\n\n\t// Name is an optional name for this matcher.\n\t// This is used to populate the name for regexp\n\t// matchers that appear in the expression.\n\tName string `json:\"name,omitempty\"`\n\n\texpandedExpr string\n\tprg          cel.Program\n\tta           types.Adapter\n\n\tlog *zap.Logger\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchExpression) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.expression\",\n\t\tNew: func() caddy.Module { return new(MatchExpression) },\n\t}\n}\n\n// MarshalJSON marshals m's expression.\nfunc (m MatchExpression) MarshalJSON() ([]byte, error) {\n\t// if the name is empty, then we can marshal just the expression string\n\tif m.Name == \"\" {\n\t\treturn json.Marshal(m.Expr)\n\t}\n\t// otherwise, we need to marshal the full object, using an\n\t// anonymous struct to avoid infinite recursion\n\treturn json.Marshal(struct {\n\t\tExpr string `json:\"expr\"`\n\t\tName string `json:\"name\"`\n\t}{\n\t\tExpr: m.Expr,\n\t\tName: m.Name,\n\t})\n}\n\n// UnmarshalJSON unmarshals m's expression.\nfunc (m *MatchExpression) UnmarshalJSON(data []byte) error {\n\t// if the data is a string, then it's just the expression\n\tif data[0] == '\"' {\n\t\treturn json.Unmarshal(data, &m.Expr)\n\t}\n\t// otherwise, it's a full object, so unmarshal it,\n\t// using an temp map to avoid infinite recursion\n\tvar tmpJson map[string]any\n\terr := json.Unmarshal(data, &tmpJson)\n\t*m = MatchExpression{\n\t\tExpr: tmpJson[\"expr\"].(string),\n\t\tName: tmpJson[\"name\"].(string),\n\t}\n\treturn err\n}\n\n// Provision sets ups m.\nfunc (m *MatchExpression) Provision(ctx caddy.Context) error {\n\tm.log = ctx.Logger()\n\n\t// replace placeholders with a function call - this is just some\n\t// light (and possibly na\u00efve) syntactic sugar\n\tm.expandedExpr = placeholderRegexp.ReplaceAllString(m.Expr, placeholderExpansion)\n\n\t// our type adapter expands CEL's standard type support\n\tm.ta = celTypeAdapter{}\n\n\t// initialize the CEL libraries from the Matcher implementations which\n\t// have been configured to support CEL.\n\tmatcherLibProducers := []CELLibraryProducer{}\n\tfor _, info := range caddy.GetModules(\"http.matchers\") {\n\t\tp, ok := info.New().(CELLibraryProducer)\n\t\tif ok {\n\t\t\tmatcherLibProducers = append(matcherLibProducers, p)\n\t\t}\n\t}\n\n\t// add the matcher name to the context so that the matcher name\n\t// can be used by regexp matchers being provisioned\n\tctx = ctx.WithValue(MatcherNameCtxKey, m.Name)\n\n\t// Assemble the compilation and program options from the different library\n\t// producers into a single cel.Library implementation.\n\tmatcherEnvOpts := []cel.EnvOption{}\n\tmatcherProgramOpts := []cel.ProgramOption{}\n\tfor _, producer := range matcherLibProducers {\n\t\tl, err := producer.CELLibrary(ctx)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"error initializing CEL library for %T: %v\", producer, err)\n\t\t}\n\t\tmatcherEnvOpts = append(matcherEnvOpts, l.CompileOptions()...)\n\t\tmatcherProgramOpts = append(matcherProgramOpts, l.ProgramOptions()...)\n\t}\n\tmatcherLib := cel.Lib(NewMatcherCELLibrary(matcherEnvOpts, matcherProgramOpts))\n\n\t// create the CEL environment\n\tenv, err := cel.NewEnv(\n\t\tcel.Function(placeholderFuncName, cel.SingletonBinaryBinding(m.caddyPlaceholderFunc), cel.Overload(\n\t\t\tplaceholderFuncName+\"_httpRequest_string\",\n\t\t\t[]*cel.Type{httpRequestObjectType, cel.StringType},\n\t\t\tcel.AnyType,\n\t\t)),\n\t\tcel.Variable(\"request\", httpRequestObjectType),\n\t\tcel.CustomTypeAdapter(m.ta),\n\t\text.Strings(),\n\t\tmatcherLib,\n\t)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"setting up CEL environment: %v\", err)\n\t}\n\n\t// parse and type-check the expression\n\tchecked, issues := env.Compile(m.expandedExpr)\n\tif issues.Err() != nil {\n\t\treturn fmt.Errorf(\"compiling CEL program: %s\", issues.Err())\n\t}\n\n\t// request matching is a boolean operation, so we don't really know\n\t// what to do if the expression returns a non-boolean type\n\tif checked.OutputType() != cel.BoolType {\n\t\treturn fmt.Errorf(\"CEL request matcher expects return type of bool, not %s\", checked.OutputType())\n\t}\n\n\t// compile the \"program\"\n\tm.prg, err = env.Program(checked, cel.EvalOptions(cel.OptOptimize))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"compiling CEL program: %s\", err)\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m.\nfunc (m MatchExpression) Match(r *http.Request) bool {\n\tcelReq := celHTTPRequest{r}\n\tout, _, err := m.prg.Eval(celReq)\n\tif err != nil {\n\t\tm.log.Error(\"evaluating expression\", zap.Error(err))\n\t\tSetVar(r.Context(), MatcherErrorVarKey, err)\n\t\treturn false\n\t}\n\tif outBool, ok := out.Value().(bool); ok {\n\t\treturn outBool\n\t}\n\treturn false\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchExpression) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume matcher name\n\n\t// if there's multiple args, then we need to keep the raw\n\t// tokens because the user may have used quotes within their\n\t// CEL expression (e.g. strings) and we should retain that\n\tif d.CountRemainingArgs() > 1 {\n\t\tm.Expr = strings.Join(d.RemainingArgsRaw(), \" \")\n\t\treturn nil\n\t}\n\n\t// there should at least be one arg\n\tif !d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\n\t// if there's only one token, then we can safely grab the\n\t// cleaned token (no quotes) and use that as the expression\n\t// because there's no valid CEL expression that is only a\n\t// quoted string; commonly quotes are used in Caddyfile to\n\t// define the expression\n\tm.Expr = d.Val()\n\n\t// use the named matcher's name, to fill regexp\n\t// matchers names by default\n\tm.Name = d.GetContextString(caddyfile.MatcherNameCtxKey)\n\n\treturn nil\n}\n\n// caddyPlaceholderFunc implements the custom CEL function that accesses the\n// Replacer on a request and gets values from it.\nfunc (m MatchExpression) caddyPlaceholderFunc(lhs, rhs ref.Val) ref.Val {\n\tcelReq, ok := lhs.(celHTTPRequest)\n\tif !ok {\n\t\treturn types.NewErr(\n\t\t\t\"invalid request of type '%v' to %s(request, placeholderVarName)\",\n\t\t\tlhs.Type(),\n\t\t\tplaceholderFuncName,\n\t\t)\n\t}\n\tphStr, ok := rhs.(types.String)\n\tif !ok {\n\t\treturn types.NewErr(\n\t\t\t\"invalid placeholder variable name of type '%v' to %s(request, placeholderVarName)\",\n\t\t\trhs.Type(),\n\t\t\tplaceholderFuncName,\n\t\t)\n\t}\n\n\trepl := celReq.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\tval, _ := repl.Get(string(phStr))\n\n\treturn m.ta.NativeToValue(val)\n}\n\n// httpRequestCELType is the type representation of a native HTTP request.\nvar httpRequestCELType = cel.ObjectType(\"http.Request\", traits.ReceiverType)\n\n// celHTTPRequest wraps an http.Request with ref.Val interface methods.\n//\n// This type also implements the interpreter.Activation interface which\n// drops allocation costs for CEL expression evaluations by roughly half.\ntype celHTTPRequest struct{ *http.Request }\n\nfunc (cr celHTTPRequest) ResolveName(name string) (any, bool) {\n\tif name == \"request\" {\n\t\treturn cr, true\n\t}\n\treturn nil, false\n}\n\nfunc (cr celHTTPRequest) Parent() interpreter.Activation {\n\treturn nil\n}\n\nfunc (cr celHTTPRequest) ConvertToNative(typeDesc reflect.Type) (any, error) {\n\treturn cr.Request, nil\n}\n\nfunc (celHTTPRequest) ConvertToType(typeVal ref.Type) ref.Val {\n\tpanic(\"not implemented\")\n}\n\nfunc (cr celHTTPRequest) Equal(other ref.Val) ref.Val {\n\tif o, ok := other.Value().(celHTTPRequest); ok {\n\t\treturn types.Bool(o.Request == cr.Request)\n\t}\n\treturn types.ValOrErr(other, \"%v is not comparable type\", other)\n}\nfunc (celHTTPRequest) Type() ref.Type { return httpRequestCELType }\nfunc (cr celHTTPRequest) Value() any  { return cr }\n\nvar pkixNameCELType = cel.ObjectType(\"pkix.Name\", traits.ReceiverType)\n\n// celPkixName wraps an pkix.Name with\n// methods to satisfy the ref.Val interface.\ntype celPkixName struct{ *pkix.Name }\n\nfunc (pn celPkixName) ConvertToNative(typeDesc reflect.Type) (any, error) {\n\treturn pn.Name, nil\n}\n\nfunc (pn celPkixName) ConvertToType(typeVal ref.Type) ref.Val {\n\tif typeVal.TypeName() == \"string\" {\n\t\treturn types.String(pn.Name.String())\n\t}\n\tpanic(\"not implemented\")\n}\n\nfunc (pn celPkixName) Equal(other ref.Val) ref.Val {\n\tif o, ok := other.Value().(string); ok {\n\t\treturn types.Bool(pn.Name.String() == o)\n\t}\n\treturn types.ValOrErr(other, \"%v is not comparable type\", other)\n}\nfunc (celPkixName) Type() ref.Type { return pkixNameCELType }\nfunc (pn celPkixName) Value() any  { return pn }\n\n// celTypeAdapter can adapt our custom types to a CEL value.\ntype celTypeAdapter struct{}\n\nfunc (celTypeAdapter) NativeToValue(value any) ref.Val {\n\tswitch v := value.(type) {\n\tcase celHTTPRequest:\n\t\treturn v\n\tcase pkix.Name:\n\t\treturn celPkixName{&v}\n\tcase time.Time:\n\t\treturn types.Timestamp{Time: v}\n\tcase error:\n\t\ttypes.NewErr(v.Error())\n\t}\n\treturn types.DefaultTypeAdapter.NativeToValue(value)\n}\n\n// CELLibraryProducer provide CEL libraries that expose a Matcher\n// implementation as a first class function within the CEL expression\n// matcher.\ntype CELLibraryProducer interface {\n\t// CELLibrary creates a cel.Library which makes it possible to use the\n\t// target object within CEL expression matchers.\n\tCELLibrary(caddy.Context) (cel.Library, error)\n}\n\n// CELMatcherImpl creates a new cel.Library based on the following pieces of\n// data:\n//\n//   - macroName: the function name to be used within CEL. This will be a macro\n//     and not a function proper.\n//   - funcName: the function overload name generated by the CEL macro used to\n//     represent the matcher.\n//   - matcherDataTypes: the argument types to the macro.\n//   - fac: a matcherFactory implementation which converts from CEL constant\n//     values to a Matcher instance.\n//\n// Note, macro names and function names must not collide with other macros or\n// functions exposed within CEL expressions, or an error will be produced\n// during the expression matcher plan time.\n//\n// The existing CELMatcherImpl support methods are configured to support a\n// limited set of function signatures. For strong type validation you may need\n// to provide a custom macro which does a more detailed analysis of the CEL\n// literal provided to the macro as an argument.\nf\nunc CELMatcherImpl(macroName, funcName string, matcherDataTypes []*cel.Type, fac CELMatcherFactory) (cel.Library, error) {\n\trequestType := cel.ObjectType(\"http.Request\")\n\tvar macro parser.Macro\n\tswitch len(matcherDataTypes) {\n\tcase 1:\n\t\tmatcherDataType := matcherDataTypes[0]\n\t\tswitch matcherDataType.String() {\n\t\tcase \"list(string)\":\n\t\t\tmacro = parser.NewGlobalVarArgMacro(macroName, celMatcherStringListMacroExpander(funcName))\n\t\tcase cel.StringType.String():\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 1, celMatcherStringMacroExpander(funcName))\n\t\tcase CELTypeJSON.String():\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 1, celMatcherJSONMacroExpander(funcName))\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"unsupported matcher data type: %s\", matcherDataType)\n\t\t}\n\tcase 2:\n\t\tif matcherDataTypes[0] == cel.StringType && matcherDataTypes[1] == cel.StringType {\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 2, celMatcherStringListMacroExpander(funcName))\n\t\t\tmatcherDataTypes = []*cel.Type{cel.ListType(cel.StringType)}\n\t\t} else {\n\t\t\treturn nil, fmt.Errorf(\"unsupported matcher data type: %s, %s\", matcherDataTypes[0], matcherDataTypes[1])\n\t\t}\n\tcase 3:\n\t\tif matcherDataTypes[0] == cel.StringType && matcherDataTypes[1] == cel.StringType && matcherDataTypes[2] == cel.StringType {\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 3, celMatcherStringListMacroExpander(funcName))\n\t\t\tmatcherDataTypes = []*cel.Type{cel.ListType(cel.StringType)}\n\t\t} else {\n\t\t\treturn nil, fmt.Errorf(\"unsupported matcher data type: %s, %s, %s\", matcherDataTypes[0], matcherDataTypes[1], matcherDataTypes[2])\n\t\t}\n\t}\n\tenvOptions := []cel.EnvOption{\n\t\tcel.Macros(macro),\n\t\tcel.Function(funcName,\n\t\t\tcel.Overload(funcName, append([]*cel.Type{requestType}, matcherDataTypes...), cel.BoolType),\n\t\t\tcel.SingletonBinaryBinding(CELMatcherRuntimeFunction(funcName, fac))),\n\t}\n\tprogramOptions := []cel.ProgramOption{\n\t\tcel.CustomDecorator(CELMatcherDecorator(funcName, fac)),\n\t}\n\treturn NewMatcherCELLibrary(envOptions, programOptions), nil\n}\n\n// CELMatcherFactory converts a constant CEL value into a RequestMatcher.\ntype CELMatcherFactory func(data ref.Val) (RequestMatcher, error)\n\n// matcherCELLibrary is a simplistic configurable cel.Library implementation.\ntype matcherCELLibrary struct {\n\tenvOptions     []cel.EnvOption\n\tprogramOptions []cel.ProgramOption\n}\n\n// NewMatcherCELLibrary creates a matcherLibrary from option setes.\nfunc NewMatcherCELLibrary(envOptions []cel.EnvOption, programOptions []cel.ProgramOption) cel.Library {\n\treturn &matcherCELLibrary{\n\t\tenvOptions:     envOptions,\n\t\tprogramOptions: programOptions,\n\t}\n}\n\nfunc (lib *matcherCELLibrary) CompileOptions() []cel.EnvOption {\n\treturn lib.envOptions\n}\n\nfunc (lib *matcherCELLibrary) ProgramOptions() []cel.ProgramOption {\n\treturn lib.programOptions\n}\n\n// CELMatcherDecorator matches a call overload generated by a CEL macro\n// that takes a single argument, and optimizes the implementation to precompile\n// the matcher and return a function that references the precompiled and\n// provisioned matcher.\nfunc CELMatcherDecorator(funcName string, fac CELMatcherFactory) interpreter.InterpretableDecorator {\n\treturn func(i interpreter.Interpretable) (interpreter.Interpretable, error) {\n\t\tcall, ok := i.(interpreter.InterpretableCall)\n\t\tif !ok {\n\t\t\treturn i, nil\n\t\t}\n\t\tif call.OverloadID() != funcName {\n\t\t\treturn i, nil\n\t\t}\n\t\tcallArgs := call.Args()\n\t\treqAttr, ok := callArgs[0].(interpreter.InterpretableAttribute)\n\t\tif !ok {\n\t\t\treturn nil, errors.New(\"missing 'request' argument\")\n\t\t}\n\t\tnsAttr, ok := reqAttr.Attr().(interpreter.NamespacedAttribute)\n\t\tif !ok {\n\t\t\treturn nil, errors.New(\"missing 'request' argument\")\n\t\t}\n\t\tvarNames := nsAttr.CandidateVariableNames()\n\t\tif len(varNames) != 1 || len(varNames) == 1 && varNames[0] != \"request\" {\n\t\t\treturn nil, errors.New(\"missing 'request' argument\")\n\t\t}\n\t\tmatcherData, ok := callArgs[1].(interpreter.InterpretableConst)\n\t\tif !ok {\n\t\t\t// If the matcher arguments are not constant, then this means\n\t\t\t// they contain a Caddy placeholder reference and the evaluation\n\t\t\t// and matcher provisioning should be handled at dynamically.\n\t\t\treturn i, nil\n\t\t}\n\t\tmatcher, err := fac(matcherData.Value())\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn interpreter.NewCall(\n\t\t\ti.ID(), funcName, funcName+\"_opt\",\n\t\t\t[]interpreter.Interpretable{reqAttr},\n\t\t\tfunc(args ...ref.Val) ref.Val {\n\t\t\t\t// The request value, guaranteed to be of type celHTTPRequest\n\t\t\t\tcelReq := args[0]\n\t\t\t\t// If needed this call could be changed to convert the value\n\t\t\t\t// to a *http.Request using CEL's ConvertToNative method.\n\t\t\t\thttpReq := celReq.Value().(celHTTPRequest)\n\t\t\t\treturn types.Bool(matcher.Match(httpReq.Request))\n\t\t\t},\n\t\t), nil\n\t}\n}\n\n// CELMatcherRuntimeFunction creates a function binding for when the input to the matcher\n// is dynamically resolved rather than a set of static constant values.\nfunc CELMatcherRuntimeFunction(funcName string, fac CELMatcherFactory) functions.BinaryOp {\n\treturn func(celReq, matcherData ref.Val) ref.Val {\n\t\tmatcher, err := fac(matcherData)\n\t\tif err != nil {\n\t\t\treturn types.NewErr(err.Error())\n\t\t}\n\t\thttpReq := celReq.Value().(celHTTPRequest)\n\t\treturn types.Bool(matcher.Match(httpReq.Request))\n\t}\n}\n\n// celMatcherStringListMacroExpander validates that the macro is called\n// with a variable number of string arguments (at least one).\n//\n// The arguments are collected into a single list argument the following\n// function call returned: <funcName>(request, [args])\nfunc celMatcherStringListMacroExpander(funcName string) cel.MacroFactory {\n\treturn func(eh cel.MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *common.Error) {\n\t\tmatchArgs := []ast.Expr{}\n\t\tif len(args) == 0 {\n\t\t\treturn nil, eh.NewError(0, \"matcher requires at least one argument\")\n\t\t}\n\t\tfor _, arg := range args {\n\t\t\tif isCELStringExpr(arg) {\n\t\t\t\tmatchArgs = append(matchArgs, arg)\n\t\t\t} else {\n\t\t\t\treturn nil, eh.NewError(arg.ID(), \"matcher arguments must be string constants\")\n\t\t\t}\n\t\t}\n\t\treturn eh.NewCall(funcName, eh.NewIdent(\"request\"), eh.NewList(matchArgs...)), nil\n\t}\n}\n\n// celMatcherStringMacroExpander validates that the macro is called a single\n// string argument.\n//\n// The following function call is returned: <funcName>(request, arg)\nfunc celMatcherStringMacroExpander(funcName string) parser.MacroExpander {\n\treturn func(eh cel.MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *common.Error) {\n\t\tif len(args) != 1 {\n\t\t\treturn nil, eh.NewError(0, \"matcher requires one argument\")\n\t\t}\n\t\tif isCELStringExpr(args[0]) {\n\t\t\treturn eh.NewCall(funcName, eh.NewIdent(\"request\"), args[0]), nil\n\t\t}\n\t\treturn nil, eh.NewError(args[0].ID(), \"matcher argument must be a string literal\")\n\t}\n}\n\n// celMatcherJSONMacroExpander validates that the macro is called a single\n// map literal argument.\n//\n// The following function call is returned: <funcName>(request, arg)\nfunc celMatcherJSONMacroExpander(funcName string) parser.MacroExpander {\n\treturn func(eh cel.MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *common.Error) {\n\t\tif len(args) != 1 {\n\t\t\treturn nil, eh.NewError(0, \"matcher requires a map literal argument\")\n\t\t}\n\t\targ := args[0]\n\n\t\tswitch arg.Kind() {\n\t\tcase ast.StructKind:\n\t\t\treturn nil, eh.NewError(arg.ID(),\n\t\t\t\tfmt.Sprintf(\"matcher input must be a map literal, not a %s\", arg.AsStruct().TypeName()))\n\t\tcase ast.MapKind:\n\t\t\tmapExpr := arg.AsMap()\n\t\t\tfor _, entry := range mapExpr.Entries() {\n\t\t\t\tisStringPlaceholder := isCELStringExpr(entry.AsMapEntry().Key())\n\t\t\t\tif !isStringPlaceholder {\n\t\t\t\t\treturn nil, eh.NewError(entry.ID(), \"matcher map keys must be string literals\")\n\t\t\t\t}\n\t\t\t\tisStringListPlaceholder := isCELStringExpr(entry.AsMapEntry().Value()) ||\n\t\t\t\t\tisCELStringListLiteral(entry.AsMapEntry().Value())\n\t\t\t\tif !isStringListPlaceholder {\n\t\t\t\t\treturn nil, eh.NewError(entry.AsMapEntry().Value().ID(), \"matcher map values must be string or list literals\")\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn eh.NewCall(funcName, eh.NewIdent(\"request\"), arg), nil\n\t\tcase ast.UnspecifiedExprKind, ast.CallKind, ast.ComprehensionKind, ast.IdentKind, ast.ListKind, ast.LiteralKind, ast.SelectKind:\n\t\t\t// appeasing the linter :)\n\t\t}\n\n\t\treturn nil, eh.NewError(arg.ID(), \"matcher requires a map literal argument\")\n\t}\n}\n\n// CELValueToMapStrList converts a CEL value to a map[string][]string\n//\n// Earlier validation stages should guarantee that the value has this type\n// at compile time, and that the runtime value type is map[string]any.\n// The reason for the slight difference in value type is that CEL allows for\n// map literals containing heterogeneous values, in this case string and list\n// of string.\nfunc CELValueToMapStrList(data ref.Val) (map[string][]string, error) {\n\tmapStrType := reflect.TypeOf(map[string]any{})\n\tmapStrRaw, err := data.ConvertToNative(mapStrType)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tmapStrIface := mapStrRaw.(map[string]any)\n\tmapStrListStr := make(map[string][]string, len(mapStrIface))\n\tfor k, v := range mapStrIface {\n\t\tswitch val := v.(type) {\n\t\tcase string:\n\t\t\tmapStrListStr[k] = []string{val}\n\t\tcase types.String:\n\t\t\tmapStrListStr[k] = []string{string(val)}\n\t\tcase []string:\n\t\t\tmapStrListStr[k] = val\n\t\tcase []ref.Val:\n\t\t\tconvVals := make([]string, len(val))\n\t\t\tfor i, elem := range val {\n\t\t\t\tstrVal, ok := elem.(types.String)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil, fmt.Errorf(\"unsupported value type in header match: %T\", val)\n\t\t\t\t}\n\t\t\t\tconvVals[i] = string(strVal)\n\t\t\t}\n\t\t\tmapStrListStr[k] = convVals\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"unsupported value type in header match: %T\", val)\n\t\t}\n\t}\n\treturn mapStrListStr, nil\n}\n\n// isCELStringExpr indicates whether the expression is a supported string expression\nfunc isCELStringExpr(e ast.Expr) bool {\n\treturn isCELStringLiteral(e) || isCELCaddyPlaceholderCall(e) || isCELConcatCall(e)\n}\n\n// isCELStringLiteral returns whether the expression is a CEL string literal.\nfunc isCELStringLiteral(e ast.Expr) bool {\n\tswitch e.Kind() {\n\tcase ast.LiteralKind:\n\t\tconstant := e.AsLiteral()\n\t\tswitch constant.Type() {\n\t\tcase types.StringType:\n\t\t\treturn true\n\t\t}\n\tcase ast.UnspecifiedExprKind, ast.CallKind, ast.ComprehensionKind, ast.IdentKind, ast.ListKind, ast.MapKind, ast.SelectKind, ast.StructKind:\n\t\t// appeasing the linter :)\n\t}\n\treturn false\n}\n\n// isCELCaddyPlaceholderCall returns whether the expression is a caddy placeholder call.\nfunc isCELCaddyPlaceholderCall(e ast.Expr) bool {\n\tswitch e.Kind() {\n\tcase ast.CallKind:\n\t\tcall := e.AsCall()\n\t\tif call.FunctionName() == \"caddyPlaceholder\" {\n\t\t\treturn true\n\t\t}\n\tcase ast.UnspecifiedExprKind, ast.ComprehensionKind, ast.IdentKind, ast.ListKind, ast.LiteralKind, ast.MapKind, ast.SelectKind, ast.StructKind:\n\t\t// appeasing the linter :)\n\t}\n\treturn false\n}\n\n// isCELConcatCall tests whether the expression is a concat function (+) with string, placeholder, or\n// other concat call arguments.\nfunc isCELConcatCall(e ast.Expr) bool {\n\tswitch e.Kind() {\n\tcase ast.CallKind:\n\t\tcall := e.AsCall()\n\t\tif call.Target().Kind() != ast.UnspecifiedExprKind {\n\t\t\treturn false\n\t\t}\n\t\tif call.FunctionName() != operators.Add {\n\t\t\treturn false\n\t\t}\n\t\tfor _, arg := range call.Args() {\n\t\t\tif !isCELStringExpr(arg) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\tcase ast.UnspecifiedExprKind, ast.ComprehensionKind, ast.IdentKind, ast.ListKind, ast.LiteralKind, ast.MapKind, ast.SelectKind, ast.StructKind:\n\t\t// appeasing the linter :)\n\t}\n\treturn false\n}\n\n// isCELStringListLiteral returns whether the expression resolves to a list literal\n// containing only string constants or a placeholder call.\nfunc isCELStringListLiteral(e ast.Expr) bool {\n\tswitch e.Kind() {\n\tcase ast.ListKind:\n\t\tlist := e.AsList()\n\t\tfor _, elem := range list.Elements() {\n\t\t\tif !isCELStringExpr(elem) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\tcase ast.UnspecifiedExprKind, ast.CallKind, ast.ComprehensionKind, ast.IdentKind, ast.LiteralKind, ast.MapKind, ast.SelectKind, ast.StructKind:\n\t\t// appeasing the linter :)\n\t}\n\treturn false\n}\n\n// Variables used for replacing Caddy placeholders in CEL\n// expressions with a proper CEL function call; this is\n// just for syntactic sugar.\nvar (\n\tplaceholderRegexp    = regexp.MustCompile(`{([a-zA-Z][\\w.-]+)}`)\n\tplaceholderExpansion = `caddyPlaceholder(request, \"${1}\")`\n\n\tCELTypeJSON = cel.MapType(cel.StringType, cel.DynType)\n)\n\nvar httpRequestObjectType = cel.ObjectType(\"http.Request\")\n\n// The name of the CEL function which accesses Replacer values.\nconst placeholderFuncName = \"caddyPlaceholder\"\n\nconst MatcherNameCtxKey = \"matcher_name\"\n\n// Interface guards\nvar (\n\t_ caddy.Provisioner     = (*MatchExpression)(nil)\n\t_ RequestMatcher        = (*MatchExpression)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchExpression)(nil)\n\t_ json.Marshaler        = (*MatchExpression)(nil)\n\t_ json.Unmarshaler      = (*MatchExpression)(nil)\n)\n\n// Path: modules/caddyhttp/celmatcher_test.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/pem\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\nvar (\n\tclientCert = []byte(`-----BEGIN CERTIFICATE-----\nMIIB9jCCAV+gAwIBAgIBAjANBgkqhkiG9w0BAQsFADAYMRYwFAYDVQQDDA1DYWRk\neSBUZXN0IENBMB4XDTE4MDcyNDIxMzUwNVoXDTI4MDcyMTIxMzUwNVowHTEbMBkG\nA1UEAwwSY2xpZW50LmxvY2FsZG9tYWluMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB\niQKBgQDFDEpzF0ew68teT3xDzcUxVFaTII+jXH1ftHXxxP4BEYBU4q90qzeKFneF\nz83I0nC0WAQ45ZwHfhLMYHFzHPdxr6+jkvKPASf0J2v2HDJuTM1bHBbik5Ls5eq+\nfVZDP8o/VHKSBKxNs8Goc2NTsr5b07QTIpkRStQK+RJALk4x9QIDAQABo0swSTAJ\nBgNVHRMEAjAAMAsGA1UdDwQEAwIHgDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A\nAAEwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQELBQADgYEANSjz2Sk+\neqp31wM9il1n+guTNyxJd+FzVAH+hCZE5K+tCgVDdVFUlDEHHbS/wqb2PSIoouLV\n3Q9fgDkiUod+uIK0IynzIKvw+Cjg+3nx6NQ0IM0zo8c7v398RzB4apbXKZyeeqUH\n9fNwfEi+OoXR6s+upSKobCmLGLGi9Na5s5g=\n-----END CERTIFICATE-----`)\n\n\tmatcherTests = []struct {\n\t\tname              string\n\t\texpression        *MatchExpression\n\t\turlTarget         string\n\t\thttpMethod        string\n\t\thttpHeader        *http.Header\n\t\twantErr           bool\n\t\twantResult        bool\n\t\tclientCertificate []byte\n\t}{\n\t\t{\n\t\t\tname: \"boolean matches succeed for placeholder http.request.tls.client.subject\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: \"{http.request.tls.client.subject} == 'CN=client.localdomain'\",\n\t\t\t},\n\t\t\tclientCertificate: clientCert,\n\t\t\turlTarget:         \"https://example.com/foo\",\n\t\t\twantResult:        true,\n\t\t},\n\t\t{\n\t\t\tname: \"header matches (MatchHeader)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `header({'Field': 'foo'})`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\thttpHeader: &http.Header{\"Field\": []string{\"foo\", \"bar\"}},\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"header error (MatchHeader)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `header('foo')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\thttpHeader: &http.Header{\"Field\": []string{\"foo\", \"bar\"}},\n\t\t\twantErr:    true,\n\t\t},\n\t\t{\n\t\t\tname: \"header_regexp matches (MatchHeaderRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `header_regexp('Field', 'fo{2}')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\thttpHeader: &http.Header{\"Field\": []string{\"foo\", \"bar\"}},\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"header_regexp matches with name (MatchHeaderRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `header_regexp('foo', 'Field', 'fo{2}')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\thttpHeader: &http.Header{\"Field\": []string{\"foo\", \"bar\"}},\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"header_regexp does not match (MatchHeaderRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `header_regexp('foo', 'Nope', 'fo{2}')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\thttpHeader: &http.Header{\"Field\": []string{\"foo\", \"bar\"}},\n\t\t\twantResult: false,\n\t\t},\n\t\t{\n\t\t\tname: \"header_regexp error (MatchHeaderRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `header_regexp('foo')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\thttpHeader: &http.Header{\"Field\": []string{\"foo\", \"bar\"}},\n\t\t\twantErr:    true,\n\t\t},\n\t\t{\n\t\t\tname: \"host matches localhost (MatchHost)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `host('localhost')`,\n\t\t\t},\n\t\t\turlTarget:  \"http://localhost\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"host matches (MatchHost)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `host('*.example.com')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://foo.example.com\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"host does not match (MatchHost)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `host('example.net', '*.example.com')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://foo.example.org\",\n\t\t\twantResult: false,\n\t\t},\n\t\t{\n\t\t\tname: \"host error (MatchHost)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `host(80)`,\n\t\t\t},\n\t\t\turlTarget: \"http://localhost:80\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"method does not match (MatchMethod)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `method('PUT')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://foo.example.com\",\n\t\t\thttpMethod: \"GET\",\n\t\t\twantResult: false,\n\t\t},\n\t\t{\n\t\t\tname: \"method matches (MatchMethod)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `method('DELETE', 'PUT', 'POST')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://foo.example.com\",\n\t\t\thttpMethod: \"PUT\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"method error not enough arguments (MatchMethod)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `method()`,\n\t\t\t},\n\t\t\turlTarget:  \"https://foo.example.com\",\n\t\t\thttpMethod: \"PUT\",\n\t\t\twantErr:    true,\n\t\t},\n\t\t{\n\t\t\tname: \"path matches substring (MatchPath)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path('*substring*')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/substring/bar.txt\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"path does not match (MatchPath)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path('/foo')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/bar\",\n\t\t\twantResult: false,\n\t\t},\n\t\t{\n\t\t\tname: \"path matches end url fragment (MatchPath)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path('/foo')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/FOO\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"path matches end fragment with substring prefix (MatchPath)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path('/foo*')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/FOOOOO\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"path matches one of multiple (MatchPath)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path('/foo', '/foo/*', '/bar', '/bar/*', '/baz', '/baz*')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"path_regexp with empty regex matches empty path (MatchPathRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path_regexp('')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"path_regexp with slash regex matches empty path (MatchPathRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path_regexp('/')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"path_regexp matches end url fragment (MatchPathRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path_regexp('^/foo')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"path_regexp does not match fragment at end (MatchPathRE)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `path_regexp('bar_at_start', '^/bar')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/bar\",\n\t\t\twantResult: false,\n\t\t},\n\t\t{\n\t\t\tname: \"protocol matches (MatchProtocol)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `protocol('HTTPs')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"protocol does not match (MatchProtocol)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `protocol('grpc')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com\",\n\t\t\twantResult: false,\n\t\t},\n\t\t{\n\t\t\tname: \"protocol invocation error no args (MatchProtocol)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `protocol()`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"protocol invocation error too many args (MatchProtocol)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `protocol('grpc', 'https')`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"protocol invocation error wrong arg type (MatchProtocol)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `protocol(true)`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"query does not match against a specific value (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query({\"debug\": \"1\"})`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\twantResult: false,\n\t\t},\n\t\t{\n\t\t\tname: \"query matches against a specific value (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query({\"debug\": \"1\"})`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/?debug=1\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"query matches against multiple values (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query({\"debug\": [\"0\", \"1\", {http.request.uri.query.debug}+\"1\"]})`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/?debug=1\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"query matches against a wildcard (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query({\"debug\": [\"*\"]})`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/?debug=something\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"query matches against a placeholder value (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query({\"debug\": {http.request.uri.query.debug}})`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo/?debug=1\",\n\t\t\twantResult: true,\n\t\t},\n\t\t{\n\t\t\tname: \"query error bad map key type (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query({1: \"1\"})`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com/foo\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"query error typed struct instead of map (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query(Message{field: \"1\"})`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com/foo\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"query error bad map value type (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query({\"debug\": 1})`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com/foo/?debug=1\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"query error no args (MatchQuery)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `query()`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com/foo/?debug=1\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"remote_ip error no args (MatchRemoteIP)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `remote_ip()`,\n\t\t\t},\n\t\t\turlTarget: \"https://example.com/foo\",\n\t\t\twantErr:   true,\n\t\t},\n\t\t{\n\t\t\tname: \"remote_ip single IP match (MatchRemoteIP)\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: `remote_ip('192.0.2.1')`,\n\t\t\t},\n\t\t\turlTarget:  \"https://example.com/foo\",\n\t\t\twantResult: true,\n\t\t},\n\t}\n)\n\nfunc TestMatchExpressionMatch(t *testing.T) {\n\tfor _, tst := range matcherTests {\n\t\ttc := tst\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tcaddyCtx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})\n\t\t\tdefer cancel()\n\t\t\terr := tc.expression.Provision(caddyCtx)\n\t\t\tif err != nil {\n\t\t\t\tif !tc.wantErr {\n\t\t\t\t\tt.Errorf(\"MatchExpression.Provision() error = %v, wantErr %v\", err, tc.wantErr)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\treq := httptest.NewRequest(tc.httpMethod, tc.urlTarget, nil)\n\t\t\tif tc.httpHeader != nil {\n\t\t\t\treq.Header = *tc.httpHeader\n\t\t\t}\n\t\t\trepl := caddy.NewReplacer()\n\t\t\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\t\t\treq = req.WithContext(ctx)\n\t\t\taddHTTPVarsToReplacer(repl, req, httptest.NewRecorder())\n\n\t\t\tif tc.clientCertificate != nil {\n\t\t\t\tblock, _ := pem.Decode(clientCert)\n\t\t\t\tif block == nil {\n\t\t\t\t\tt.Fatalf(\"failed to decode PEM certificate\")\n\t\t\t\t}\n\n\t\t\t\tcert, err := x509.ParseCertificate(block.Bytes)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"failed to decode PEM certificate: %v\", err)\n\t\t\t\t}\n\n\t\t\t\treq.TLS = &tls.ConnectionState{\n\t\t\t\t\tPeerCertificates: []*x509.Certificate{cert},\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif tc.expression.Match(req) != tc.wantResult {\n\t\t\t\tt.Errorf(\"MatchExpression.Match() expected to return '%t', for expression : '%s'\", tc.wantResult, tc.expression.Expr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc BenchmarkMatchExpressionMatch(b *testing.B) {\n\tfor _, tst := range matcherTests {\n\t\ttc := tst\n\t\tif tc.wantErr {\n\t\t\tcontinue\n\t\t}\n\t\tb.Run(tst.name, func(b *testing.B) {\n\t\t\ttc.expression.Provision(caddy.Context{})\n\t\t\treq := httptest.NewRequest(tc.httpMethod, tc.urlTarget, nil)\n\t\t\tif tc.httpHeader != nil {\n\t\t\t\treq.Header = *tc.httpHeader\n\t\t\t}\n\t\t\trepl := caddy.NewReplacer()\n\t\t\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\t\t\treq = req.WithContext(ctx)\n\t\t\taddHTTPVarsToReplacer(repl, req, httptest.NewRecorder())\n\t\t\tif tc.clientCertificate != nil {\n\t\t\t\tblock, _ := pem.Decode(clientCert)\n\t\t\t\tif block == nil {\n\t\t\t\t\tb.Fatalf(\"failed to decode PEM certificate\")\n\t\t\t\t}\n\n\t\t\t\tcert, err := x509.ParseCertificate(block.Bytes)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatalf(\"failed to decode PEM certificate: %v\", err)\n\t\t\t\t}\n\n\t\t\t\treq.TLS = &tls.ConnectionState{\n\t\t\t\t\tPeerCertificates: []*x509.Certificate{cert},\n\t\t\t\t}\n\t\t\t}\n\t\t\tb.ResetTimer()\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\ttc.expression.Match(req)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMatchExpressionProvision(t *testing.T) {\n\ttests := []struct {\n\t\tname       string\n\t\texpression *MatchExpression\n\t\twantErr    bool\n\t}{\n\t\t{\n\t\t\tname: \"boolean matches succeed\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: \"{http.request.uri.query} != ''\",\n\t\t\t},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"reject expressions with non-boolean results\",\n\t\t\texpression: &MatchExpression{\n\t\t\t\tExpr: \"{http.request.uri.query}\",\n\t\t\t},\n\t\t\twantErr: true,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})\n\t\t\tdefer cancel()\n\t\t\tif err := tt.expression.Provision(ctx); (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"MatchExpression.Provision() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: modules/caddyhttp/encode/brotli/brotli_precompressed.go\npackage caddybrotli\n\nimport (\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(BrotliPrecompressed{})\n}\n\n// BrotliPrecompressed provides the file extension for files precompressed with brotli encoding.\ntype BrotliPrecompressed struct{}\n\n// CaddyModule returns the Caddy module information.\nfunc (BrotliPrecompressed) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.precompressed.br\",\n\t\tNew: func() caddy.Module { return new(BrotliPrecompressed) },\n\t}\n}\n\n// AcceptEncoding returns the name of the encoding as\n// used in the Accept-Encoding request headers.\nfunc (BrotliPrecompressed) AcceptEncoding() string { return \"br\" }\n\n// Suffix returns the filename suffix of precompressed files.\nfunc (BrotliPrecompressed) Suffix() string { return \".br\" }\n\n// Interface guards\nvar _ encode.Precompressed = (*BrotliPrecompressed)(nil)\n\n// Path: modules/caddyhttp/encode/encode_test.go\npackage encode\n\nimport (\n\t\"net/http\"\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc BenchmarkOpenResponseWriter(b *testing.B) {\n\tenc := new(Encode)\n\tfor n := 0; n < b.N; n++ {\n\t\tenc.openResponseWriter(\"test\", nil)\n\t}\n}\n\nfunc TestPreferOrder(t *testing.T) {\n\ttestCases := []struct {\n\t\tname     string\n\t\taccept   string\n\t\tprefer   []string\n\t\texpected []string\n\t}{\n\t\t{\n\t\t\tname:     \"PreferOrder(): 4 accept, 3 prefer\",\n\t\t\taccept:   \"deflate, gzip, br, zstd\",\n\t\t\tprefer:   []string{\"zstd\", \"br\", \"gzip\"},\n\t\t\texpected: []string{\"zstd\", \"br\", \"gzip\", \"deflate\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): 2 accept, 3 prefer\",\n\t\t\taccept:   \"deflate, zstd\",\n\t\t\tprefer:   []string{\"zstd\", \"br\", \"gzip\"},\n\t\t\texpected: []string{\"zstd\", \"deflate\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): 2 accept (1 empty), 3 prefer\",\n\t\t\taccept:   \"gzip,,zstd\",\n\t\t\tprefer:   []string{\"zstd\", \"br\", \"gzip\"},\n\t\t\texpected: []string{\"zstd\", \"gzip\", \"\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): 1 accept, 2 prefer\",\n\t\t\taccept:   \"gzip\",\n\t\t\tprefer:   []string{\"zstd\", \"gzip\"},\n\t\t\texpected: []string{\"gzip\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): 4 accept (1 duplicate), 1 prefer\",\n\t\t\taccept:   \"deflate, gzip, br, br\",\n\t\t\tprefer:   []string{\"br\"},\n\t\t\texpected: []string{\"br\", \"br\", \"deflate\", \"gzip\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): empty accept, 0 prefer\",\n\t\t\taccept:   \"\",\n\t\t\tprefer:   []string{},\n\t\t\texpected: []string{},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): empty accept, 1 prefer\",\n\t\t\taccept:   \"\",\n\t\t\tprefer:   []string{\"gzip\"},\n\t\t\texpected: []string{},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): with q-factor\",\n\t\t\taccept:   \"deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd\",\n\t\t\tprefer:   []string{\"gzip\"},\n\t\t\texpected: []string{\"zstd\", \"deflate\", \"gzip\", \"br\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): with q-factor, no prefer\",\n\t\t\taccept:   \"deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd\",\n\t\t\tprefer:   []string{},\n\t\t\texpected: []string{\"zstd\", \"deflate\", \"gzip\", \"br\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): q-factor=0 filtered out\",\n\t\t\taccept:   \"deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0\",\n\t\t\tprefer:   []string{\"gzip\"},\n\t\t\texpected: []string{\"br\", \"gzip\", \"deflate\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): q-factor=0 filtered out, no prefer\",\n\t\t\taccept:   \"deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0\",\n\t\t\tprefer:   []string{},\n\t\t\texpected: []string{\"br\", \"gzip\", \"deflate\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): with invalid q-factor\",\n\t\t\taccept:   \"br, deflate, gzip;q=2, zstd;q=0.1\",\n\t\t\tprefer:   []string{\"zstd\", \"gzip\"},\n\t\t\texpected: []string{\"gzip\", \"br\", \"deflate\", \"zstd\"},\n\t\t},\n\t\t{\n\t\t\tname:     \"PreferOrder(): with invalid q-factor, no prefer\",\n\t\t\taccept:   \"br, deflate, gzip;q=2, zstd;q=0.1\",\n...\n", "needle_token_start": 3479, "needle_token_end": 4205, "code_context_ntokens": 15325, "output": ["```go\nfunc CELMatcherImpl(macroName, funcName string, matcherDataTypes []*cel.Type, fac CELMatcherFactory) (cel.Library, error) {\n\trequestType := cel.ObjectType(\"http.Request\")\n\tvar macro parser.Macro\n\tswitch len(matcherDataTypes) {\n\tcase 1:\n\t\tmatcherDataType := matcherDataTypes[0]\n\t\tswitch matcherDataType.String() {\n\t\tcase \"list(string)\":\n\t\t\tmacro = parser.NewGlobalVarArgMacro(macroName, celMatcherStringListMacroExpander(funcName))\n\t\tcase cel.StringType.String():\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 1, celMatcherStringMacroExpander(funcName))\n\t\tcase CELTypeJSON.String():\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 1, celMatcherJSONMacroExpander(funcName))\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"unsupported matcher data type: %s\", matcherDataType)\n\t\t}\n\tcase 2:\n\t\tif matcherDataTypes[0] == cel.StringType && matcherDataTypes[1] == cel.StringType {\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 2, celMatcherStringListMacroExpander(funcName))\n\t\t\tmatcherDataTypes = []*cel.Type{cel.ListType(cel.StringType)}\n\t\t} else {\n\t\t\treturn nil, fmt.Errorf(\"unsupported matcher data type: %s, %s\", matcherDataTypes[0], matcherDataTypes[1])\n\t\t}\n\tcase 3:\n\t\tif matcherDataTypes[0] == cel.StringType && matcherDataTypes[1] == cel.StringType && matcherDataTypes[2] == cel.StringType {\n\t\t\tmacro = parser.NewGlobalMacro(macroName, 3, celMatcherStringListMacroExpander(funcName))\n\t\t\tmatcherDataTypes = []*cel.Type{cel.ListType(cel.StringType)}\n\t\t} else {\n\t\t\treturn nil, fmt.Errorf(\"unsupported matcher data type: %s, %s, %s\", matcherDataTypes[0], matcherDataTypes[1], matcherDataTypes[2])\n\t\t}\n\t}\n\tenvOptions := []cel.EnvOption{\n\t\tcel.Macros(macro),\n\t\tcel.Function(funcName,\n\t\t\tcel.Overload(funcName, append([]*cel.Type{requestType}, matcherDataTypes...), cel.BoolType),\n\t\t\tcel.SingletonBinaryBinding(CELMatcherRuntimeFunction(funcName, fac))),\n\t}\n\tprogramOptions := []cel.ProgramOption{\n\t\tcel.CustomDecorator(CELMatcherDecorator(funcName, fac)),\n\t}\n\treturn NewMatcherCELLibrary(envOptions, programOptions), nil\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "fileHidden", "language": "go", "path": "modules/caddyhttp/fileserver/staticfiles.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To determine if a specified file path should be considered hidden based on a list of patterns or specific file names.\n2. **Input**: A string representing a file path and a list of strings representing patterns or file names to hide.\n3. **Output**: A boolean value indicating whether the file path matches any of the hide patterns or specific file names.\n4. **Procedure**: \n   - If the list of hide patterns is empty, the function returns false.\n   - Converts the file path to an absolute path for accurate comparison.\n   - If a pattern does not contain a path separator, the function checks each component of the file path against the pattern.\n   - If a pattern contains a path separator, the function checks if the file path starts with the pattern, ensuring that it matches directories correctly.\n   - Additionally, the function performs a glob pattern matching to cover general cases where direct substring matching is insufficient.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\tif !strings.Contains(h, \"{\") && strings.Contains(h, separator) {\n\t\t\tif abs, err := filepath.Abs(h); err == nil {\n\t\t\t\tfsrv.Hide[i] = abs\n\t\t\t}\n\t\t}\n\t}\n\n\t// support precompressed sidecar files\n\tmods, err := ctx.LoadModule(fsrv, \"PrecompressedRaw\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"loading encoder modules: %v\", err)\n\t}\n\tfor modName, modIface := range mods.(map[string]any) {\n\t\tp, ok := modIface.(encode.Precompressed)\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"module %s is not precompressor\", modName)\n\t\t}\n\t\tae := p.AcceptEncoding()\n\t\tif ae == \"\" {\n\t\t\treturn fmt.Errorf(\"precompressor does not specify an Accept-Encoding value\")\n\t\t}\n\t\tsuffix := p.Suffix()\n\t\tif suffix == \"\" {\n\t\t\treturn fmt.Errorf(\"precompressor does not specify a Suffix value\")\n\t\t}\n\t\tif _, ok := fsrv.precompressors[ae]; ok {\n\t\t\treturn fmt.Errorf(\"precompressor already added: %s\", ae)\n\t\t}\n\t\tif fsrv.precompressors == nil {\n\t\t\tfsrv.precompressors = make(map[string]encode.Precompressed)\n\t\t}\n\t\tfsrv.precompressors[ae] = p\n\t}\n\n\treturn nil\n}\n\nfunc (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\n\tif runtime.GOOS == \"windows\" {\n\t\t// reject paths with Alternate Data Streams (ADS)\n\t\tif strings.Contains(r.URL.Path, \":\") {\n\t\t\treturn caddyhttp.Error(http.StatusBadRequest, fmt.Errorf(\"illegal ADS path\"))\n\t\t}\n\t\t// reject paths with \"8.3\" short names\n\t\ttrimmedPath := strings.TrimRight(r.URL.Path, \". \") // Windows ignores trailing dots and spaces, sigh\n\t\tif len(path.Base(trimmedPath)) <= 12 && strings.Contains(trimmedPath, \"~\") {\n\t\t\treturn caddyhttp.Error(http.StatusBadRequest, fmt.Errorf(\"illegal short name\"))\n\t\t}\n\t\t// both of those could bypass file hiding or possibly leak information even if the file is not hidden\n\t}\n\n\tfilesToHide := fsrv.transformHidePaths(repl)\n\n\troot := repl.ReplaceAll(fsrv.Root, \".\")\n\tfsName := repl.ReplaceAll(fsrv.FileSystem, \"\")\n\n\tfileSystem, ok := fsrv.fsmap.Get(fsName)\n\tif !ok {\n\t\treturn caddyhttp.Error(http.StatusNotFound, fmt.Errorf(\"filesystem not found\"))\n\t}\n\n\t// remove any trailing `/` as it breaks fs.ValidPath() in the stdlib\n\tfilename := strings.TrimSuffix(caddyhttp.SanitizedPathJoin(root, r.URL.Path), \"/\")\n\n\tfsrv.logger.Debug(\"sanitized path join\",\n\t\tzap.String(\"site_root\", root),\n\t\tzap.String(\"fs\", fsName),\n\t\tzap.String(\"request_path\", r.URL.Path),\n\t\tzap.String(\"result\", filename))\n\n\t// get information about the file\n\tinfo, err := fs.Stat(fileSystem, filename)\n\tif err != nil {\n\t\terr = fsrv.mapDirOpenError(fileSystem, err, filename)\n\t\tif errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrInvalid) {\n\t\t\treturn fsrv.notFound(w, r, next)\n\t\t} else if errors.Is(err, fs.ErrPermission) {\n\t\t\treturn caddyhttp.Error(http.StatusForbidden, err)\n\t\t}\n\t\treturn caddyhttp.Error(http.StatusInternalServerError, err)\n\t}\n\n\t// if the request mapped to a directory, see if\n\t// there is an index file we can serve\n\tvar implicitIndexFile bool\n\tif info.IsDir() && len(fsrv.IndexNames) > 0 {\n\t\tfor _, indexPage := range fsrv.IndexNames {\n\t\t\tindexPage := repl.ReplaceAll(indexPage, \"\")\n\t\t\tindexPath := caddyhttp.SanitizedPathJoin(filename, indexPage)\n\t\t\tif fileHidden(indexPath, filesToHide) {\n\t\t\t\t// pretend this file doesn't exist\n\t\t\t\tfsrv.logger.Debug(\"hiding index file\",\n\t\t\t\t\tzap.String(\"filename\", indexPath),\n\t\t\t\t\tzap.Strings(\"files_to_hide\", filesToHide))\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tindexInfo, err := fs.Stat(fileSystem, indexPath)\n\t\t\tif err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// don't rewrite the request path to append\n\t\t\t// the index file, because we might need to\n\t\t\t// do a canonical-URL redirect below based\n\t\t\t// on the URL as-is\n\n\t\t\t// we've chosen to use this index file,\n\t\t\t// so replace the last file info and path\n\t\t\t// with that of the index file\n\t\t\tinfo = indexInfo\n\t\t\tfilename = indexPath\n\t\t\timplicitIndexFile = true\n\t\t\tfsrv.logger.Debug(\"located index file\", zap.String(\"filename\", filename))\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// if still referencing a directory, delegate\n\t// to browse or return an error\n\tif info.IsDir() {\n\t\tfsrv.logger.Debug(\"no index file in directory\",\n\t\t\tzap.String(\"path\", filename),\n\t\t\tzap.Strings(\"index_filenames\", fsrv.IndexNames))\n\t\tif fsrv.Browse != nil && !fileHidden(filename, filesToHide) {\n\t\t\treturn fsrv.serveBrowse(fileSystem, root, filename, w, r, next)\n\t\t}\n\t\treturn fsrv.notFound(w, r, next)\n\t}\n\n\t// one last check to ensure the file isn't hidden (we might\n\t// have changed the filename from when we last checked)\n\tif fileHidden(filename, filesToHide) {\n\t\tfsrv.logger.Debug(\"hiding file\",\n\t\t\tzap.String(\"filename\", filename),\n\t\t\tzap.Strings(\"files_to_hide\", filesToHide))\n\t\treturn fsrv.notFound(w, r, next)\n\t}\n\n\t// if URL canonicalization is enabled, we need to enforce trailing\n\t// slash convention: if a directory, trailing slash; if a file, no\n\t// trailing slash - not enforcing this can break relative hrefs\n\t// in HTML (see https://github.com/caddyserver/caddy/issues/2741)\n\tif fsrv.CanonicalURIs == nil || *fsrv.CanonicalURIs {\n\t\t// Only redirect if the last element of the path (the filename) was not\n\t\t// rewritten; if the admin wanted to rewrite to the canonical path, they\n\t\t// would have, and we have to be very careful not to introduce unwanted\n\t\t// redirects and especially redirect loops!\n\t\t// See https://github.com/caddyserver/caddy/issues/4205.\n\t\torigReq := r.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request)\n\t\tif path.Base(origReq.URL.Path) == path.Base(r.URL.Path) {\n\t\t\tif implicitIndexFile && !strings.HasSuffix(origReq.URL.Path, \"/\") {\n\t\t\t\tto := origReq.URL.Path + \"/\"\n\t\t\t\tfsrv.logger.Debug(\"redirecting to canonical URI (adding trailing slash for directory)\",\n\t\t\t\t\tzap.String(\"from_path\", origReq.URL.Path),\n\t\t\t\t\tzap.String(\"to_path\", to))\n\t\t\t\treturn redirect(w, r, to)\n\t\t\t} else if !implicitIndexFile && strings.HasSuffix(origReq.URL.Path, \"/\") {\n\t\t\t\tto := origReq.URL.Path[:len(origReq.URL.Path)-1]\n\t\t\t\tfsrv.logger.Debug(\"redirecting to canonical URI (removing trailing slash for file)\",\n\t\t\t\t\tzap.String(\"from_path\", origReq.URL.Path),\n\t\t\t\t\tzap.String(\"to_path\", to))\n\t\t\t\treturn redirect(w, r, to)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar file fs.File\n\n\t// etag is usually unset, but if the user knows what they're doing, let them override it\n\tetag := w.Header().Get(\"Etag\")\n\n\t// static file responses are often compressed, either on-the-fly\n\t// or with precompressed sidecar files; in any case, the headers\n\t// should contain \"Vary: Accept-Encoding\" even when not compressed\n\t// so caches can craft a reliable key (according to REDbot results)\n\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t// check for precompressed files\n\tfor _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) {\n\t\tprecompress, ok := fsrv.precompressors[ae]\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\tcompressedFilename := filename + precompress.Suffix()\n\t\tcompressedInfo, err := fs.Stat(fileSystem, compressedFilename)\n\t\tif err != nil || compressedInfo.IsDir() {\n\t\t\tfsrv.logger.Debug(\"precompressed file not accessible\", zap.String(\"filename\", compressedFilename), zap.Error(err))\n\t\t\tcontinue\n\t\t}\n\t\tfsrv.logger.Debug(\"opening compressed sidecar file\", zap.String(\"filename\", compressedFilename), zap.Error(err))\n\t\tfile, err = fsrv.openFile(fileSystem, compressedFilename, w)\n\t\tif err != nil {\n\t\t\tfsrv.logger.Warn(\"opening precompressed file failed\", zap.String(\"filename\", compressedFilename), zap.Error(err))\n\t\t\tif caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfile = nil\n\t\t\tcontinue\n\t\t}\n\t\tdefer file.Close()\n\t\tw.Header().Set(\"Content-Encoding\", ae)\n\t\tw.Header().Del(\"Accept-Ranges\")\n\n\t\t// try to get the etag from pre computed files if an etag suffix list was provided\n\t\tif etag == \"\" && fsrv.EtagFileExtensions != nil {\n\t\t\tetag, err = fsrv.getEtagFromFile(fileSystem, compressedFilename)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\t// don't assign info = compressedInfo because sidecars are kind\n\t\t// of transparent; however we do need to set the Etag:\n\t\t// https://caddy.community/t/gzipped-sidecar-file-wrong-same-etag/16793\n\t\tif etag == \"\" {\n\t\t\tetag = calculateEtag(compressedInfo)\n\t\t}\n\n\t\tbreak\n\t}\n\n\t// no precompressed file found, use the actual file\n\tif file == nil {\n\t\tfsrv.logger.Debug(\"opening file\", zap.String(\"filename\", filename))\n\n\t\t// open the file\n\t\tfile, err = fsrv.openFile(fileSystem, filename, w)\n\t\tif err != nil {\n\t\t\tif herr, ok := err.(caddyhttp.HandlerError); ok &&\n\t\t\t\therr.StatusCode == http.StatusNotFound {\n\t\t\t\treturn fsrv.notFound(w, r, next)\n\t\t\t}\n\t\t\treturn err // error is already structured\n\t\t}\n\t\tdefer file.Close()\n\t\t// try to get the etag from pre computed files if an etag suffix list was provided\n\t\tif etag == \"\" && fsrv.EtagFileExtensions != nil {\n\t\t\tetag, err = fsrv.getEtagFromFile(fileSystem, filename)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif etag == \"\" {\n\t\t\tetag = calculateEtag(info)\n\t\t}\n\t}\n\n\t// at this point, we're serving a file; Go std lib supports only\n\t// GET and HEAD, which is sensible for a static file server - reject\n\t// any other methods (see issue #5166)\n\tif r.Method != http.MethodGet && r.Method != http.MethodHead {\n\t\t// if we're in an error context, then it doesn't make sense\n\t\t// to repeat the error; just continue because we're probably\n\t\t// trying to write an error page response (see issue #5703)\n\t\tif _, ok := r.Context().Value(caddyhttp.ErrorCtxKey).(error); !ok {\n\t\t\tw.Header().Add(\"Allow\", \"GET, HEAD\")\n\t\t\treturn caddyhttp.Error(http.StatusMethodNotAllowed, nil)\n\t\t}\n\t}\n\n\t// set the Etag - note that a conditional If-None-Match request is handled\n\t// by http.ServeContent below, which checks against this Etag value\n\tif etag != \"\" {\n\t\tw.Header().Set(\"Etag\", etag)\n\t}\n\n\tif w.Header().Get(\"Content-Type\") == \"\" {\n\t\tmtyp := mime.TypeByExtension(filepath.Ext(filename))\n\t\tif mtyp == \"\" {\n\t\t\t// do not allow Go to sniff the content-type; see https://www.youtube.com/watch?v=8t8JYpt0egE\n\t\t\tw.Header()[\"Content-Type\"] = nil\n\t\t} else {\n\t\t\tw.Header().Set(\"Content-Type\", mtyp)\n\t\t}\n\t}\n\n\tvar statusCodeOverride int\n\n\t// if this handler exists in an error context (i.e. is part of a\n\t// handler chain that is supposed to handle a previous error),\n\t// we should set status code to the one from the error instead\n\t// of letting http.ServeContent set the default (usually 200)\n\tif reqErr, ok := r.Context().Value(caddyhttp.ErrorCtxKey).(error); ok {\n\t\tstatusCodeOverride = http.StatusInternalServerError\n\t\tif handlerErr, ok := reqErr.(caddyhttp.HandlerError); ok {\n\t\t\tif handlerErr.StatusCode > 0 {\n\t\t\t\tstatusCodeOverride = handlerErr.StatusCode\n\t\t\t}\n\t\t}\n\t}\n\n\t// if a status code override is configured, run the replacer on it\n\tif codeStr := fsrv.StatusCode.String(); codeStr != \"\" {\n\t\tstatusCodeOverride, err = strconv.Atoi(repl.ReplaceAll(codeStr, \"\"))\n\t\tif err != nil {\n\t\t\treturn caddyhttp.Error(http.StatusInternalServerError, err)\n\t\t}\n\t}\n\n\t// if we do have an override from the previous two parts, then\n\t// we wrap the response writer to intercept the WriteHeader call\n\tif statusCodeOverride > 0 {\n\t\tw = statusOverrideResponseWriter{ResponseWriter: w, code: statusCodeOverride}\n\t}\n\n\t// let the standard library do what it does best; note, however,\n\t// that errors generated by ServeContent are written immediately\n\t// to the response, so we cannot handle them (but errors there\n\t// are rare)\n\thttp.ServeContent(w, r, info.Name(), info.ModTime(), file.(io.ReadSeeker))\n\n\treturn nil\n}\n\n// openFile opens the file at the given filename. If there was an error,\n// the response is configured to inform the client how to best handle it\n// and a well-described handler error is returned (do not wrap the\n// returned error value).\nfunc (fsrv *FileServer) openFile(fileSystem fs.FS, filename string, w http.ResponseWriter) (fs.File, error) {\n\tfile, err := fileSystem.Open(filename)\n\tif err != nil {\n\t\terr = fsrv.mapDirOpenError(fileSystem, err, filename)\n\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\tfsrv.logger.Debug(\"file not found\", zap.String(\"filename\", filename), zap.Error(err))\n\t\t\treturn nil, caddyhttp.Error(http.StatusNotFound, err)\n\t\t} else if errors.Is(err, fs.ErrPermission) {\n\t\t\tfsrv.logger.Debug(\"permission denied\", zap.String(\"filename\", filename), zap.Error(err))\n\t\t\treturn nil, caddyhttp.Error(http.StatusForbidden, err)\n\t\t}\n\t\t// maybe the server is under load and ran out of file descriptors?\n\t\t// have client wait arbitrary seconds to help prevent a stampede\n\t\t//nolint:gosec\n\t\tbackoff := weakrand.Intn(maxBackoff-minBackoff) + minBackoff\n\t\tw.Header().Set(\"Retry-After\", strconv.Itoa(backoff))\n\t\tfsrv.logger.Debug(\"retry after backoff\", zap.String(\"filename\", filename), zap.Int(\"backoff\", backoff), zap.Error(err))\n\t\treturn nil, caddyhttp.Error(http.StatusServiceUnavailable, err)\n\t}\n\treturn file, nil\n}\n\n// mapDirOpenError maps the provided non-nil error from opening name\n// to a possibly better non-nil error. In particular, it turns OS-specific errors\n// about opening files in non-directories into os.ErrNotExist. See golang/go#18984.\n// Adapted from the Go standard library; originally written by Nathaniel Caza.\n// https://go-review.googlesource.com/c/go/+/36635/\n// https://go-review.googlesource.com/c/go/+/36804/\nfunc (fsrv *FileServer) mapDirOpenError(fileSystem fs.FS, originalErr error, name string) error {\n\tif errors.Is(originalErr, fs.ErrNotExist) || errors.Is(originalErr, fs.ErrPermission) {\n\t\treturn originalErr\n\t}\n\n\tparts := strings.Split(name, separator)\n\tfor i := range parts {\n\t\tif parts[i] == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\tfi, err := fs.Stat(fileSystem, strings.Join(parts[:i+1], separator))\n\t\tif err != nil {\n\t\t\treturn originalErr\n\t\t}\n\t\tif !fi.IsDir() {\n\t\t\treturn fs.ErrNotExist\n\t\t}\n\t}\n\n\treturn originalErr\n}\n\n// transformHidePaths performs replacements for all the elements of fsrv.Hide and\n// makes them absolute paths (if they contain a path separator), then returns a\n// new list of the transformed values.\nfunc (fsrv *FileServer) transformHidePaths(repl *caddy.Replacer) []string {\n\thide := make([]string, len(fsrv.Hide))\n\tfor i := range fsrv.Hide {\n\t\thide[i] = repl.ReplaceAll(fsrv.Hide[i], \"\")\n\t\tif strings.Contains(hide[i], separator) {\n\t\t\tabs, err := filepath.Abs(hide[i])\n\t\t\tif err == nil {\n\t\t\t\thide[i] = abs\n\t\t\t}\n\t\t}\n\t}\n\treturn hide\n}\n\n// fileHidden returns true if filename is hidden according to the hide list.\n// filename must be a relative or absolute file system path, not a request\n// URI path. It is expected that all the paths in the hide list are absolute\n// paths or are singular filenames (without a path separator).\n\nfunc fileHidden(filename string, hide []string) bool {\n\tif len(hide) == 0 {\n\t\treturn false\n\t}\n\n\t// all path comparisons use the complete absolute path if possible\n\tfilenameAbs, err := filepath.Abs(filename)\n\tif err == nil {\n\t\tfilename = filenameAbs\n\t}\n\n\tvar components []string\n\n\tfor _, h := range hide {\n\t\tif !strings.Contains(h, separator) {\n\t\t\t// if there is no separator in h, then we assume the user\n\t\t\t// wants to hide any files or folders that match that\n\t\t\t// name; thus we have to compare against each component\n\t\t\t// of the filename, e.g. hiding \"bar\" would hide \"/bar\"\n\t\t\t// as well as \"/foo/bar/baz\" but not \"/barstool\".\n\t\t\tif len(components) == 0 {\n\t\t\t\tcomponents = strings.Split(filename, separator)\n\t\t\t}\n\t\t\tfor _, c := range components {\n\t\t\t\tif hidden, _ := filepath.Match(h, c); hidden {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t} else if strings.HasPrefix(filename, h) {\n\t\t\t// if there is a separator in h, and filename is exactly\n\t\t\t// prefixed with h, then we can do a prefix match so that\n\t\t\t// \"/foo\" matches \"/foo/bar\" but not \"/foobar\".\n\t\t\twithoutPrefix := strings.TrimPrefix(filename, h)\n\t\t\tif strings.HasPrefix(withoutPrefix, separator) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\n\t\t// in the general case, a glob match will suffice\n\t\tif hidden, _ := filepath.Match(h, filename); hidden {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\n// notFound returns a 404 error or, if pass-thru is enabled,\n// it calls the next handler in the chain.\nfunc (fsrv *FileServer) notFound(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {\n\tif fsrv.PassThru {\n\t\treturn next.ServeHTTP(w, r)\n\t}\n\treturn caddyhttp.Error(http.StatusNotFound, nil)\n}\n\n// calculateEtag computes an entity tag using a strong validator\n// without consuming the contents of the file. It requires the\n// file info contain the correct size and modification time.\n// It strives to implement the semantics regarding ETags as defined\n// by RFC 9110 section 8.8.3 and 8.8.1. See\n// https://www.rfc-editor.org/rfc/rfc9110.html#section-8.8.3.\n//\n// As our implementation uses file modification timestamp and size,\n// note the following from RFC 9110 section 8.8.1: \"A representation's\n// modification time, if defined with only one-second resolution,\n// might be a weak validator if it is possible for the representation to\n// be modified twice during a single second and retrieved between those\n// modifications.\" The ext4 file system, which underpins the vast majority\n// of Caddy deployments, stores mod times with millisecond precision,\n// which we consider precise enough to qualify as a strong validator.\nfunc calculateEtag(d os.FileInfo) string {\n\tmtime := d.ModTime()\n\tif mtimeUnix := mtime.Unix(); mtimeUnix == 0 || mtimeUnix == 1 {\n\t\treturn \"\" // not useful anyway; see issue #5548\n\t}\n\tvar sb strings.Builder\n\tsb.WriteRune('\"')\n\tsb.WriteString(strconv.FormatInt(mtime.UnixNano(), 36))\n\tsb.WriteString(strconv.FormatInt(d.Size(), 36))\n\tsb.WriteRune('\"')\n\treturn sb.String()\n}\n\n// Finds the first corresponding etag file for a given file in the file system and return its content\nfunc (fsrv *FileServer) getEtagFromFile(fileSystem fs.FS, filename string) (string, error) {\n\tfor _, suffix := range fsrv.EtagFileExtensions {\n\t\tetagFilename := filename + suffix\n\t\tetag, err := fs.ReadFile(fileSystem, etagFilename)\n\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\tcontinue\n\t\t}\n\t\tif err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"cannot read etag from file %s: %v\", etagFilename, err)\n\t\t}\n\t\treturn string(etag), nil\n\t}\n\treturn \"\", nil\n}\n\n// redirect performs a redirect to a given path. The 'toPath' parameter\n// MUST be solely a path, and MUST NOT include a query.\nfunc redirect(w http.ResponseWriter, r *http.Request, toPath string) error {\n\tfor strings.HasPrefix(toPath, \"//\") {\n\t\t// prevent path-based open redirects\n\t\ttoPath = strings.TrimPrefix(toPath, \"/\")\n\t}\n\t// preserve the query string if present\n\tif r.URL.RawQuery != \"\" {\n\t\ttoPath += \"?\" + r.URL.RawQuery\n\t}\n\thttp.Redirect(w, r, toPath, http.StatusPermanentRedirect)\n\treturn nil\n}\n\n// statusOverrideResponseWriter intercepts WriteHeader calls\n// to instead write the HTTP status code we want instead\n// of the one http.ServeContent will use by default (usually 200)\ntype statusOverrideResponseWriter struct {\n\thttp.ResponseWriter\n\tcode int\n}\n\n// WriteHeader intercepts calls by the stdlib to WriteHeader\n// to instead write the HTTP status code we want.\nfunc (wr statusOverrideResponseWriter) WriteHeader(int) {\n\twr.ResponseWriter.WriteHeader(wr.code)\n}\n\n// Unwrap returns the underlying ResponseWriter, necessary for\n// http.ResponseController to work correctly.\nfunc (wr statusOverrideResponseWriter) Unwrap() http.ResponseWriter {\n\treturn wr.ResponseWriter\n}\n\nvar defaultIndexNames = []string{\"index.html\", \"index.txt\"}\n\nconst (\n\tminBackoff, maxBackoff = 2, 5\n\tseparator              = string(filepath.Separator)\n)\n\n// Interface guards\nvar (\n\t_ caddy.Provisioner           = (*FileServer)(nil)\n\t_ caddyhttp.MiddlewareHandler = (*FileServer)(nil)\n)\n\n// Path: modules/caddyhttp/fileserver/staticfiles_test.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fileserver\n\nimport (\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestFileHidden(t *testing.T) {\n\tfor i, tc := range []struct {\n\t\tinputHide []string\n\t\tinputPath string\n\t\texpect    bool\n\t}{\n\t\t{\n\t\t\tinputHide: nil,\n\t\t\tinputPath: \"\",\n\t\t\texpect:    false,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\".gitignore\"},\n\t\t\tinputPath: \"/.gitignore\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\".git\"},\n\t\t\tinputPath: \"/.gitignore\",\n\t\t\texpect:    false,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/.git\"},\n\t\t\tinputPath: \"/.gitignore\",\n\t\t\texpect:    false,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\".git\"},\n\t\t\tinputPath: \"/.git\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\".git\"},\n\t\t\tinputPath: \"/.git/foo\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\".git\"},\n\t\t\tinputPath: \"/foo/.git/bar\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/prefix\"},\n\t\t\tinputPath: \"/prefix/foo\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/foo/*/bar\"},\n\t\t\tinputPath: \"/foo/asdf/bar\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"*.txt\"},\n\t\t\tinputPath: \"/foo/bar.txt\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/foo/bar/*.txt\"},\n\t\t\tinputPath: \"/foo/bar/baz.txt\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/foo/bar/*.txt\"},\n\t\t\tinputPath: \"/foo/bar.txt\",\n\t\t\texpect:    false,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/foo/bar/*.txt\"},\n\t\t\tinputPath: \"/foo/bar/index.html\",\n\t\t\texpect:    false,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/foo\"},\n\t\t\tinputPath: \"/foo\",\n\t\t\texpect:    true,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"/foo\"},\n\t\t\tinputPath: \"/foobar\",\n\t\t\texpect:    false,\n\t\t},\n\t\t{\n\t\t\tinputHide: []string{\"first\", \"second\"},\n\t\t\tinputPath: \"/second\",\n\t\t\texpect:    true,\n\t\t},\n\t} {\n\t\tif runtime.GOOS == \"windows\" {\n\t\t\tif strings.HasPrefix(tc.inputPath, \"/\") {\n\t\t\t\ttc.inputPath, _ = filepath.Abs(tc.inputPath)\n\t\t\t}\n\t\t\ttc.inputPath = filepath.FromSlash(tc.inputPath)\n\t\t\tfor i := range tc.inputHide {\n\t\t\t\tif strings.HasPrefix(tc.inputHide[i], \"/\") {\n\t\t\t\t\ttc.inputHide[i], _ = filepath.Abs(tc.inputHide[i])\n\t\t\t\t}\n\t\t\t\ttc.inputHide[i] = filepath.FromSlash(tc.inputHide[i])\n\t\t\t}\n\t\t}\n\n\t\tactual := fileHidden(tc.inputPath, tc.inputHide)\n\t\tif actual != tc.expect {\n\t\t\tt.Errorf(\"Test %d: Does %v hide %s? Got %t but expected %t\",\n\t\t\t\ti, tc.inputHide, tc.inputPath, actual, tc.expect)\n\t\t}\n\t}\n}\n\n// Path: modules/caddyhttp/headers/headers_test.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage headers\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc TestHandler(t *testing.T) {\n\tfor i, tc := range []struct {\n\t\thandler            Handler\n\t\treqHeader          http.Header\n\t\trespHeader         http.Header\n\t\trespStatusCode     int\n\t\texpectedReqHeader  http.Header\n\t\texpectedRespHeader http.Header\n\t}{\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tRequest: &HeaderOps{\n\t\t\t\t\tAdd: http.Header{\n\t\t\t\t\t\t\"Expose-Secrets\": []string{\"always\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\treqHeader: http.Header{\n\t\t\t\t\"Expose-Secrets\": []string{\"i'm serious\"},\n\t\t\t},\n\t\t\texpectedReqHeader: http.Header{\n\t\t\t\t\"Expose-Secrets\": []string{\"i'm serious\", \"always\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tRequest: &HeaderOps{\n\t\t\t\t\tSet: http.Header{\n\t\t\t\t\t\t\"Who-Wins\": []string{\"batman\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\treqHeader: http.Header{\n\t\t\t\t\"Who-Wins\": []string{\"joker\"},\n\t\t\t},\n\t\t\texpectedReqHeader: http.Header{\n\t\t\t\t\"Who-Wins\": []string{\"batman\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tRequest: &HeaderOps{\n\t\t\t\t\tDelete: []string{\"Kick-Me\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\treqHeader: http.Header{\n\t\t\t\t\"Kick-Me\": []string{\"if you can\"},\n\t\t\t\t\"Keep-Me\": []string{\"i swear i'm innocent\"},\n\t\t\t},\n\t\t\texpectedReqHeader: http.Header{\n\t\t\t\t\"Keep-Me\": []string{\"i swear i'm innocent\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tRequest: &HeaderOps{\n\t\t\t\t\tDelete: []string{\n\t\t\t\t\t\t\"*-suffix\",\n\t\t\t\t\t\t\"prefix-*\",\n\t\t\t\t\t\t\"*_*\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\treqHeader: http.Header{\n\t\t\t\t\"Header-Suffix\": []string{\"lalala\"},\n\t\t\t\t\"Prefix-Test\":   []string{\"asdf\"},\n\t\t\t\t\"Host_Header\":   []string{\"silly django... sigh\"}, // see issue #4830\n\t\t\t\t\"Keep-Me\":       []string{\"foofoofoo\"},\n\t\t\t},\n\t\t\texpectedReqHeader: http.Header{\n\t\t\t\t\"Keep-Me\": []string{\"foofoofoo\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tRequest: &HeaderOps{\n\t\t\t\t\tReplace: map[string][]Replacement{\n\t\t\t\t\t\t\"Best-Server\": {\n\t\t\t\t\t\t\tReplacement{\n\t\t\t\t\t\t\t\tSearch:  \"NGINX\",\n\t\t\t\t\t\t\t\tReplace: \"the Caddy web server\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t\tReplacement{\n\t\t\t\t\t\t\t\tSearchRegexp: `Apache(\\d+)`,\n\t\t\t\t\t\t\t\tReplace:      \"Caddy\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\treqHeader: http.Header{\n\t\t\t\t\"Best-Server\": []string{\"it's NGINX, undoubtedly\", \"I love Apache2\"},\n\t\t\t},\n\t\t\texpectedReqHeader: http.Header{\n\t\t\t\t\"Best-Server\": []string{\"it's the Caddy web server, undoubtedly\", \"I love Caddy\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tResponse: &RespHeaderOps{\n\t\t\t\t\tRequire: &caddyhttp.ResponseMatcher{\n\t\t\t\t\t\tHeaders: http.Header{\n\t\t\t\t\t\t\t\"Cache-Control\": nil,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tHeaderOps: &HeaderOps{\n\t\t\t\t\t\tAdd: http.Header{\n\t\t\t\t\t\t\t\"Cache-Control\": []string{\"no-cache\"},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\trespHeader: http.Header{},\n\t\t\texpectedRespHeader: http.Header{\n\t\t\t\t\"Cache-Control\": []string{\"no-cache\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tResponse: &RespHeaderOps{\n\t\t\t\t\tRequire: &caddyhttp.ResponseMatcher{\n\t\t\t\t\t\tHeaders: http.Header{\n\t\t\t\t\t\t\t\"Cache-Control\": []string{\"no-cache\"},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\tHeaderOps: &HeaderOps{\n\t\t\t\t\t\tDelete: []string{\"Cache-Control\"},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\trespHeader: http.Header{\n\t\t\t\t\"Cache-Control\": []string{\"no-cache\"},\n\t\t\t},\n\t\t\texpectedRespHeader: http.Header{},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tResponse: &RespHeaderOps{\n\t\t\t\t\tRequire: &caddyhttp.ResponseMatcher{\n\t\t\t\t\t\tStatusCode: []int{5},\n\t\t\t\t\t},\n\t\t\t\t\tHeaderOps: &HeaderOps{\n\t\t\t\t\t\tAdd: http.Header{\n\t\t\t\t\t\t\t\"Fail-5xx\": []string{\"true\"},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\trespStatusCode: 503,\n\t\t\trespHeader:     http.Header{},\n\t\t\texpectedRespHeader: http.Header{\n\t\t\t\t\"Fail-5xx\": []string{\"true\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\thandler: Handler{\n\t\t\t\tRequest: &HeaderOps{\n\t\t\t\t\tReplace: map[string][]Replacement{\n\t\t\t\t\t\t\"Case-Insensitive\": {\n\t\t\t\t\t\t\tReplacement{\n\t\t\t\t\t\t\t\tSearch:  \"issue4330\",\n\t\t\t\t\t\t\t\tReplace: \"issue #4330\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\treqHeader: http.Header{\n\t\t\t\t\"case-insensitive\": []string{\"issue4330\"},\n\t\t\t\t\"Other-Header\":     []string{\"issue4330\"},\n\t\t\t},\n\t\t\texpectedReqHeader: http.Header{\n\t\t\t\t\"case-insensitive\": []string{\"issue #4330\"},\n\t\t\t\t\"Other-Header\":     []string{\"issue4330\"},\n\t\t\t},\n\t\t},\n\t} {\n\t\trr := httptest.NewRecorder()\n\n\t\treq := &http.Request{Header: tc.reqHeader}\n\t\trepl := caddy.NewReplacer()\n\t\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\t\treq = req.WithContext(ctx)\n\n\t\ttc.handler.Provision(caddy.Context{})\n\n\t\tnext := nextHandler(func(w http.ResponseWriter, r *http.Request) error {\n\t\t\tfor k, hdrs := range tc.respHeader {\n\t\t\t\tfor _, v := range hdrs {\n\t\t\t\t\tw.Header().Add(k, v)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tstatus := 200\n\t\t\tif tc.respStatusCode != 0 {\n\t\t\t\tstatus = tc.respStatusCode\n\t\t\t}\n\t\t\tw.WriteHeader(status)\n\n\t\t\tif tc.expectedReqHeader != nil && !reflect.DeepEqual(r.Header, tc.expectedReqHeader) {\n\t\t\t\treturn fmt.Errorf(\"expected request header %v, got %v\", tc.expectedReqHeader, r.Header)\n\t\t\t}\n\n\t\t\treturn nil\n\t\t})\n\n\t\tif err := tc.handler.ServeHTTP(rr, req, next); err != nil {\n\t\t\tt.Errorf(\"Test %d: %v\", i, err)\n\t\t\tcontinue\n\t\t}\n\n\t\tactual := rr.Header()\n\t\tif tc.expectedRespHeader != nil && !reflect.DeepEqual(actual, tc.expectedRespHeader) {\n\t\t\tt.Errorf(\"Test %d: expected response header %v, got %v\", i, tc.expectedRespHeader, actual)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\ntype nextHandler func(http.ResponseWriter, *http.Request) error\n\nfunc (f nextHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) error {\n\treturn f(w, r)\n}\n\n// Path: modules/caddyhttp/http2listener.go\npackage caddyhttp\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\tweakrand \"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"golang.org/x/net/http2\"\n)\n\n// http2Listener wraps the listener to solve the following problems:\n// 1. server h2 natively without using h2c hack when listener handles tls connection but\n// don't return *tls.Conn\n// 2. graceful shutdown. the shutdown logic is copied from stdlib http.Server, it's an extra maintenance burden but\n// whatever, the shutdown logic maybe extracted to be used with h2c graceful shutdown. http2.Server supports graceful shutdown\n// sending GO_AWAY frame to connected clients, but doesn't track connection status. It requires explicit call of http2.ConfigureServer\ntype http2Listener struct {\n\tcnt uint64\n\tnet.Listener\n\tserver   *http.Server\n\th2server *http2.Server\n}\n\ntype connectionStateConn interface {\n\tnet.Conn\n\tConnectionState() tls.ConnectionState\n}\n\nfunc (h *http2Listener) Accept() (net.Conn, error) {\n\tfor {\n\t\tconn, err := h.Listener.Accept()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif csc, ok := conn.(connectionStateConn); ok {\n\t\t\t// *tls.Conn will return empty string because it's only populated after handshake is complete\n\t\t\tif csc.ConnectionState().NegotiatedProtocol == http2.NextProtoTLS {\n\t\t\t\tgo h.serveHttp2(csc)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\treturn conn, nil\n\t}\n}\n\nfunc (h *http2Listener) serveHttp2(csc connectionStateConn) {\n\tatomic.AddUint64(&h.cnt, 1)\n\th.runHook(csc, http.StateNew)\n\tdefer func() {\n\t\tcsc.Close()\n\t\tatomic.AddUint64(&h.cnt, ^uint64(0))\n\t\th.runHook(csc, http.StateClosed)\n\t}()\n\th.h2server.ServeConn(csc, &http2.ServeConnOpts{\n\t\tContext:    h.server.ConnContext(context.Background(), csc),\n\t\tBaseConfig: h.server,\n\t\tHandler:    h.server.Handler,\n\t})\n}\n\nconst shutdownPollIntervalMax = 500 * time.Millisecond\n\nfunc (h *http2Listener) Shutdown(ctx context.Context) error {\n\tpollIntervalBase := time.Millisecond\n\tnextPollInterval := func() time.Duration {\n\t\t// Add 10% jitter.\n\t\t//nolint:gosec\n\t\tinterval := pollIntervalBase + time.Duration(weakrand.Intn(int(pollIntervalBase/10)))\n\t\t// Double and clamp for next time.\n\t\tpollIntervalBase *= 2\n\t\tif pollIntervalBase > shutdownPollIntervalMax {\n\t\t\tpollIntervalBase = shutdownPollIntervalMax\n\t\t}\n\t\treturn interval\n\t}\n\n\ttimer := time.NewTimer(nextPollInterval())\n\tdefer timer.Stop()\n\tfor {\n\t\tif atomic.LoadUint64(&h.cnt) == 0 {\n\t\t\treturn nil\n\t\t}\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn ctx.Err()\n\t\tcase <-timer.C:\n\t\t\ttimer.Reset(nextPollInterval())\n\t\t}\n\t}\n}\n\nfunc (h *http2Listener) runHook(conn net.Conn, state http.ConnState) {\n\tif h.server.ConnState != nil {\n\t\th.server.ConnState(conn, state)\n\t}\n}\n\n// Path: modules/caddyhttp/httpredirectlistener.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(HTTPRedirectListenerWrapper{})\n}\n\n// HTTPRedirectListenerWrapper provides HTTP->HTTPS redirects for\n// connections that come on the TLS port as an HTTP request,\n// by detecting using the first few bytes that it's not a TLS\n// handshake, but instead an HTTP request.\n//\n// This is especially useful when using a non-standard HTTPS port.\n// A user may simply type the address in their browser without the\n// https:// scheme, which would cause the browser to attempt the\n// connection over HTTP, but this would cause a \"Client sent an\n// HTTP request to an HTTPS server\" error response.\n//\n// This listener wrapper must be placed BEFORE the \"tls\" listener\n// wrapper, for it to work properly.\ntype HTTPRedirectListenerWrapper struct {\n\t// MaxHeaderBytes is the maximum size to parse from a client's\n\t// HTTP request headers. Default: 1 MB\n\tMaxHeaderBytes int64 `json:\"max_header_bytes,omitempty\"`\n}\n\nfunc (HTTPRedirectListenerWrapper) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"caddy.listeners.http_redirect\",\n\t\tNew: func() caddy.Module { return new(HTTPRedirectListenerWrapper) },\n\t}\n}\n\nfunc (h *HTTPRedirectListenerWrapper) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\treturn nil\n}\n\nfunc (h *HTTPRedirectListenerWrapper) WrapListener(l net.Listener) net.Listener {\n\treturn &httpRedirectListener{l, h.MaxHeaderBytes}\n}\n\n// httpRedirectListener is listener that checks the first few bytes\n// of the request when the server is intended to accept HTTPS requests,\n// to respond to an HTTP request with a redirect.\ntype httpRedirectListener struct {\n\tnet.Listener\n\tmaxHeaderBytes int64\n}\n\n// Accept waits for and returns the next connection to the listener,\n// wrapping it with a httpRedirectConn.\nfunc (l *httpRedirectListener) Accept() (net.Conn, error) {\n\tc, err := l.Listener.Accept()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tmaxHeaderBytes := l.maxHeaderBytes\n\tif maxHeaderBytes == 0 {\n\t\tmaxHeaderBytes = 1024 * 1024\n\t}\n\n\treturn &httpRedirectConn{\n\t\tConn:  c,\n\t\tlimit: maxHeaderBytes,\n\t\tr:     bufio.NewReader(c),\n\t}, nil\n}\n\ntype httpRedirectConn struct {\n\tnet.Conn\n\tonce  bool\n\tlimit int64\n\tr     *bufio.Reader\n}\n\n// Read tries to peek at the first few bytes of the request, and if we get\n// an error reading the headers, and that error was due to the bytes looking\n// like an HTTP request, then we perform a HTTP->HTTPS redirect on the same\n// port as the original connection.\nfunc (c *httpRedirectConn) Read(p []byte) (int, error) {\n\tif c.once {\n\t\treturn c.r.Read(p)\n\t}\n\t// no need to use sync.Once - net.Conn is not read from concurrently.\n\tc.once = true\n\n\tfirstBytes, err := c.r.Peek(5)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\t// If the request doesn't look like HTTP, then it's probably\n\t// TLS bytes, and we don't need to do anything.\n\tif !firstBytesLookLikeHTTP(firstBytes) {\n\t\treturn c.r.Read(p)\n\t}\n\n\t// From now on, we can be almost certain the request is HTTP.\n\t// The returned error will be non nil and caller are expected to\n\t// close the connection.\n\n\t// Set the read limit, io.MultiReader is needed because\n\t// when resetting, *bufio.Reader discards buffered data.\n\tbuffered, _ := c.r.Peek(c.r.Buffered())\n\tmr := io.MultiReader(bytes.NewReader(buffered), c.Conn)\n\tc.r.Reset(io.LimitReader(mr, c.limit))\n\n\t// Parse the HTTP request, so we can get the Host and URL to redirect to.\n\treq, err := http.ReadRequest(c.r)\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"couldn't read HTTP request\")\n\t}\n\n\t// Build the redirect response, using the same Host and URL,\n\t// but replacing the scheme with https.\n\theaders := make(http.Header)\n\theaders.Add(\"Location\", \"https://\"+req.Host+req.URL.String())\n\tresp := &http.Response{\n\t\tProto:      \"HTTP/1.0\",\n\t\tStatus:     \"308 Permanent Redirect\",\n\t\tStatusCode: 308,\n\t\tProtoMajor: 1,\n\t\tProtoMinor: 0,\n\t\tHeader:     headers,\n\t}\n\n\terr = resp.Write(c.Conn)\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"couldn't write HTTP->HTTPS redirect\")\n\t}\n\n\treturn 0, fmt.Errorf(\"redirected HTTP request on HTTPS port\")\n}\n\n// firstBytesLookLikeHTTP reports whether a TLS record header\n// looks like it might've been a misdirected plaintext HTTP request.\nfunc firstBytesLookLikeHTTP(hdr []byte) bool {\n\tswitch string(hdr[:5]) {\n\tcase \"GET /\", \"HEAD \", \"POST \", \"PUT /\", \"OPTIO\":\n\t\treturn true\n\t}\n\treturn false\n}\n\nvar (\n\t_ caddy.ListenerWrapper = (*HTTPRedirectListenerWrapper)(nil)\n\t_ caddyfile.Unmarshaler = (*HTTPRedirectListenerWrapper)(nil)\n)\n\n// Path: modules/caddyhttp/invoke.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(Invoke{})\n}\n\n// Invoke implements a handler that compiles and executes a\n// named route that was defined on the server.\n//\n// EXPERIMENTAL: Subject to change or removal.\ntype Invoke struct {\n\t// Name is the key of the named route to execute\n\tName string `json:\"name,omitempty\"`\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (Invoke) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.handlers.invoke\",\n\t\tNew: func() caddy.Module { return new(Invoke) },\n\t}\n}\n\nfunc (invoke *Invoke) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {\n\tserver := r.Context().Value(ServerCtxKey).(*Server)\n\tif route, ok := server.NamedRoutes[invoke.Name]; ok {\n\t\treturn route.Compile(next).ServeHTTP(w, r)\n\t}\n\treturn fmt.Errorf(\"invoke: route '%s' not found\", invoke.Name)\n}\n\n// Interface guards\nvar (\n\t_ MiddlewareHandler = (*Invoke)(nil)\n)\n\n// Path: modules/caddyhttp/ip_matchers.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/netip\"\n\t\"reflect\"\n\t\"strings\"\n\n\t\"github.com/google/cel-go/cel\"\n\t\"github.com/google/cel-go/common/types/ref\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n)\n\n// MatchRemoteIP matches requests by the remote IP address,\n// i.e. the IP address of the direct connection to Caddy.\ntype MatchRemoteIP struct {\n\t// The IPs or CIDR ranges to match.\n\tRanges []string `json:\"ranges,omitempty\"`\n\n\t// cidrs and zones vars should aligned always in the same\n\t// length and indexes for matching later\n\tcidrs  []*netip.Prefix\n\tzones  []string\n\tlogger *zap.Logger\n}\n\n// MatchClientIP matches requests by the client IP address,\n// i.e. the resolved address, considering trusted proxies.\ntype MatchClientIP struct {\n\t// The IPs or CIDR ranges to match.\n\tRanges []string `json:\"ranges,omitempty\"`\n\n\t// cidrs and zones vars should aligned always in the same\n\t// length and indexes for matching later\n\tcidrs  []*netip.Prefix\n\tzones  []string\n\tlogger *zap.Logger\n}\n\nfunc init() {\n\tcaddy.RegisterModule(MatchRemoteIP{})\n\tcaddy.RegisterModule(MatchClientIP{})\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchRemoteIP) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.remote_ip\",\n\t\tNew: func() caddy.Module { return new(MatchRemoteIP) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchRemoteIP) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume matcher name\n\tfor d.NextArg() {\n\t\tif d.Val() == \"forwarded\" {\n\t\t\treturn d.Err(\"the 'forwarded' option is no longer supported; use the 'client_ip' matcher instead\")\n\t\t}\n\t\tif d.Val() == \"private_ranges\" {\n\t\t\tm.Ranges = append(m.Ranges, PrivateRangesCIDR()...)\n\t\t\tcontinue\n\t\t}\n\t\tm.Ranges = append(m.Ranges, d.Val())\n\t}\n\tif d.NextBlock(0) {\n\t\treturn d.Err(\"malformed remote_ip matcher: blocks are not supported\")\n\t}\n\treturn nil\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression remote_ip('192.168.0.0/16', '172.16.0.0/12', '10.0.0.0/8')\nfunc (MatchRemoteIP) CELLibrary(ctx caddy.Context) (cel.Library, error) {\n\treturn CELMatcherImpl(\n\t\t// name of the macro, this is the function name that users see when writing expressions.\n\t\t\"remote_ip\",\n\t\t// name of the function that the macro will be rewritten to call.\n\t\t\"remote_ip_match_request_list\",\n\t\t// internal data type of the MatchPath value.\n\t\t[]*cel.Type{cel.ListType(cel.StringType)},\n\t\t// function to convert a constant list of strings to a MatchPath instance.\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\trefStringList := reflect.TypeOf([]string{})\n\t\t\tstrList, err := data.ConvertToNative(refStringList)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tm := MatchRemoteIP{}\n\n\t\t\tfor _, input := range strList.([]string) {\n\t\t\t\tif input == \"forwarded\" {\n...\n", "needle_token_start": 5131, "needle_token_end": 5595, "code_context_ntokens": 15313, "output": ["```go\nfunc fileHidden(filename string, hide []string) bool {\n\tif len(hide) == 0 {\n\t\treturn false\n\t}\n\n\t// all path comparisons use the complete absolute path if possible\n\tfilenameAbs, err := filepath.Abs(filename)\n\tif err == nil {\n\t\tfilename = filenameAbs\n\t}\n\n\tvar components []string\n\n\tfor _, h := range hide {\n\t\tif !strings.Contains(h, separator) {\n\t\t\t// if there is no separator in h, then we assume the user\n\t\t\t// wants to hide any files or folders that match that\n\t\t\t// name; thus we have to compare against each component\n\t\t\t// of the filename, e.g. hiding \"bar\" would hide \"/bar\"\n\t\t\t// as well as \"/foo/bar/baz\" but not \"/barstool\".\n\t\t\tif len(components) == 0 {\n\t\t\t\tcomponents = strings.Split(filename, separator)\n\t\t\t}\n\t\t\tfor _, c := range components {\n\t\t\t\tif hidden, _ := filepath.Match(h, c); hidden {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t} else if strings.HasPrefix(filename, h) {\n\t\t\t// if there is a separator in h, and filename is exactly\n\t\t\t// prefixed with h, then we can do a prefix match so that\n\t\t\t// \"/foo\" matches \"/foo/bar\" but not \"/foobar\".\n\t\t\twithoutPrefix := strings.TrimPrefix(filename, h)\n\t\t\tif strings.HasPrefix(withoutPrefix, separator) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\n\t\t// in the general case, a glob match will suffice\n\t\tif hidden, _ := filepath.Match(h, filename); hidden {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "getHeaderFieldVals", "language": "go", "path": "modules/caddyhttp/matchers.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the values of a specified header field from an HTTP header, with special handling for the \"Host\" header.\n2. **Input**: The function takes three parameters: an HTTP header object, a string representing the name of the header field to retrieve, and a string representing the host.\n3. **Output**: Returns a list of strings containing the values of the specified header field. If the field is \"Host\" and the host parameter is provided, it returns the host as the value.\n4. **Procedure**: The function first normalizes the header field name to its canonical form. If the field name is \"Host\" and a non-empty host value is provided, it returns the host value in a list. Otherwise, it retrieves the values associated with the header field name directly from the HTTP header object.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression host('localhost')\nfunc (MatchHost) CELLibrary(ctx caddy.Context) (cel.Library, error) {\n\treturn CELMatcherImpl(\n\t\t\"host\",\n\t\t\"host_match_request_list\",\n\t\t[]*cel.Type{cel.ListType(cel.StringType)},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\trefStringList := reflect.TypeOf([]string{})\n\t\t\tstrList, err := data.ConvertToNative(refStringList)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tmatcher := MatchHost(strList.([]string))\n\t\t\terr = matcher.Provision(ctx)\n\t\t\treturn matcher, err\n\t\t},\n\t)\n}\n\n// fuzzy returns true if the given hostname h is not a specific\n// hostname, e.g. has placeholders or wildcards.\nfunc (MatchHost) fuzzy(h string) bool { return strings.ContainsAny(h, \"{*\") }\n\n// large returns true if m is considered to be large. Optimizing\n// the matcher for smaller lists has diminishing returns.\n// See related benchmark function in test file to conduct experiments.\nfunc (m MatchHost) large() bool { return len(m) > 100 }\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchPath) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.path\",\n\t\tNew: func() caddy.Module { return new(MatchPath) },\n\t}\n}\n\n// Provision lower-cases the paths in m to ensure case-insensitive matching.\nfunc (m MatchPath) Provision(_ caddy.Context) error {\n\tfor i := range m {\n\t\tif m[i] == \"*\" && i > 0 {\n\t\t\t// will always match, so just put it first\n\t\t\tm[0] = m[i]\n\t\t\tbreak\n\t\t}\n\t\tm[i] = strings.ToLower(m[i])\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m.\nfunc (m MatchPath) Match(r *http.Request) bool {\n\t// Even though RFC 9110 says that path matching is case-sensitive\n\t// (https://www.rfc-editor.org/rfc/rfc9110.html#section-4.2.3),\n\t// we do case-insensitive matching to mitigate security issues\n\t// related to differences between operating systems, applications,\n\t// etc; if case-sensitive matching is needed, the regex matcher\n\t// can be used instead.\n\treqPath := strings.ToLower(r.URL.Path)\n\n\t// See #2917; Windows ignores trailing dots and spaces\n\t// when accessing files (sigh), potentially causing a\n\t// security risk (cry) if PHP files end up being served\n\t// as static files, exposing the source code, instead of\n\t// being matched by *.php to be treated as PHP scripts.\n\tif runtime.GOOS == \"windows\" { // issue #5613\n\t\treqPath = strings.TrimRight(reqPath, \". \")\n\t}\n\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\n\tfor _, matchPattern := range m {\n\t\tmatchPattern = repl.ReplaceAll(matchPattern, \"\")\n\n\t\t// special case: whole path is wildcard; this is unnecessary\n\t\t// as it matches all requests, which is the same as no matcher\n\t\tif matchPattern == \"*\" {\n\t\t\treturn true\n\t\t}\n\n\t\t// Clean the path, merge doubled slashes, etc.\n\t\t// This ensures maliciously crafted requests can't bypass\n\t\t// the path matcher. See #4407. Good security posture\n\t\t// requires that we should do all we can to reduce any\n\t\t// funny-looking paths into \"normalized\" forms such that\n\t\t// weird variants can't sneak by.\n\t\t//\n\t\t// How we clean the path depends on the kind of pattern:\n\t\t// we either merge slashes or we don't. If the pattern\n\t\t// has double slashes, we preserve them in the path.\n\t\t//\n\t\t// TODO: Despite the fact that the *vast* majority of path\n\t\t// matchers have only 1 pattern, a possible optimization is\n\t\t// to remember the cleaned form of the path for future\n\t\t// iterations; it's just that the way we clean depends on\n\t\t// the kind of pattern.\n\n\t\tmergeSlashes := !strings.Contains(matchPattern, \"//\")\n\n\t\t// if '%' appears in the match pattern, we interpret that to mean\n\t\t// the intent is to compare that part of the path in raw/escaped\n\t\t// space; i.e. \"%40\"==\"%40\", not \"@\", and \"%2F\"==\"%2F\", not \"/\"\n\t\tif strings.Contains(matchPattern, \"%\") {\n\t\t\treqPathForPattern := CleanPath(r.URL.EscapedPath(), mergeSlashes)\n\t\t\tif m.matchPatternWithEscapeSequence(reqPathForPattern, matchPattern) {\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\t// doing prefix/suffix/substring matches doesn't make sense\n\t\t\tcontinue\n\t\t}\n\n\t\treqPathForPattern := CleanPath(reqPath, mergeSlashes)\n\n\t\t// for substring, prefix, and suffix matching, only perform those\n\t\t// special, fast matches if they are the only wildcards in the pattern;\n\t\t// otherwise we assume a globular match if any * appears in the middle\n\n\t\t// special case: first and last characters are wildcard,\n\t\t// treat it as a fast substring match\n\t\tif strings.Count(matchPattern, \"*\") == 2 &&\n\t\t\tstrings.HasPrefix(matchPattern, \"*\") &&\n\t\t\tstrings.HasSuffix(matchPattern, \"*\") {\n\t\t\tif strings.Contains(reqPathForPattern, matchPattern[1:len(matchPattern)-1]) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// only perform prefix/suffix match if it is the only wildcard...\n\t\t// I think that is more correct most of the time\n\t\tif strings.Count(matchPattern, \"*\") == 1 {\n\t\t\t// special case: first character is a wildcard,\n\t\t\t// treat it as a fast suffix match\n\t\t\tif strings.HasPrefix(matchPattern, \"*\") {\n\t\t\t\tif strings.HasSuffix(reqPathForPattern, matchPattern[1:]) {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// special case: last character is a wildcard,\n\t\t\t// treat it as a fast prefix match\n\t\t\tif strings.HasSuffix(matchPattern, \"*\") {\n\t\t\t\tif strings.HasPrefix(reqPathForPattern, matchPattern[:len(matchPattern)-1]) {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// at last, use globular matching, which also is exact matching\n\t\t// if there are no glob/wildcard chars; we ignore the error here\n\t\t// because we can't handle it anyway\n\t\tmatches, _ := path.Match(matchPattern, reqPathForPattern)\n\t\tif matches {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (MatchPath) matchPatternWithEscapeSequence(escapedPath, matchPath string) bool {\n\t// We would just compare the pattern against r.URL.Path,\n\t// but the pattern contains %, indicating that we should\n\t// compare at least some part of the path in raw/escaped\n\t// space, not normalized space; so we build the string we\n\t// will compare against by adding the normalized parts\n\t// of the path, then switching to the escaped parts where\n\t// the pattern hints to us wherever % is present.\n\tvar sb strings.Builder\n\n\t// iterate the pattern and escaped path in lock-step;\n\t// increment iPattern every time we consume a char from the pattern,\n\t// increment iPath every time we consume a char from the path;\n\t// iPattern and iPath are our cursors/iterator positions for each string\n\tvar iPattern, iPath int\n\tfor {\n\t\tif iPattern >= len(matchPath) || iPath >= len(escapedPath) {\n\t\t\tbreak\n\t\t}\n\n\t\t// get the next character from the request path\n\n\t\tpathCh := string(escapedPath[iPath])\n\t\tvar escapedPathCh string\n\n\t\t// normalize (decode) escape sequences\n\t\tif pathCh == \"%\" && len(escapedPath) >= iPath+3 {\n\t\t\t// hold onto this in case we find out the intent is to match in escaped space here;\n\t\t\t// we lowercase it even though technically the spec says: \"For consistency, URI\n\t\t\t// producers and normalizers should use uppercase hexadecimal digits for all percent-\n\t\t\t// encodings\" (RFC 3986 section 2.1) - we lowercased the matcher pattern earlier in\n\t\t\t// provisioning so we do the same here to gain case-insensitivity in equivalence;\n\t\t\t// besides, this string is never shown visibly\n\t\t\tescapedPathCh = strings.ToLower(escapedPath[iPath : iPath+3])\n\n\t\t\tvar err error\n\t\t\tpathCh, err = url.PathUnescape(escapedPathCh)\n\t\t\tif err != nil {\n\t\t\t\t// should be impossible unless EscapedPath() is giving us an invalid sequence!\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tiPath += 2 // escape sequence is 2 bytes longer than normal char\n\t\t}\n\n\t\t// now get the next character from the pattern\n\n\t\tnormalize := true\n\t\tswitch matchPath[iPattern] {\n\t\tcase '%':\n\t\t\t// escape sequence\n\n\t\t\t// if not a wildcard (\"%*\"), compare literally; consume next two bytes of pattern\n\t\t\tif len(matchPath) >= iPattern+3 && matchPath[iPattern+1] != '*' {\n\t\t\t\tsb.WriteString(escapedPathCh)\n\t\t\t\tiPath++\n\t\t\t\tiPattern += 2\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\t// escaped wildcard sequence; consume next byte only ('*')\n\t\t\tiPattern++\n\t\t\tnormalize = false\n\n\t\t\tfallthrough\n\t\tcase '*':\n\t\t\t// wildcard, so consume until next matching character\n\t\t\tremaining := escapedPath[iPath:]\n\t\t\tuntil := len(escapedPath) - iPath // go until end of string...\n\t\t\tif iPattern < len(matchPath)-1 {  // ...unless the * is not at the end\n\t\t\t\tnextCh := matchPath[iPattern+1]\n\t\t\t\tuntil = strings.IndexByte(remaining, nextCh)\n\t\t\t\tif until == -1 {\n\t\t\t\t\t// terminating char of wildcard span not found, so definitely no match\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\tif until == 0 {\n\t\t\t\t// empty span; nothing to add on this iteration\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tnext := remaining[:until]\n\t\t\tif normalize {\n\t\t\t\tvar err error\n\t\t\t\tnext, err = url.PathUnescape(next)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn false // should be impossible anyway\n\t\t\t\t}\n\t\t\t}\n\t\t\tsb.WriteString(next)\n\t\t\tiPath += until\n\t\tdefault:\n\t\t\tsb.WriteString(pathCh)\n\t\t\tiPath++\n\t\t}\n\n\t\tiPattern++\n\t}\n\n\t// we can now treat rawpath globs (%*) as regular globs (*)\n\tmatchPath = strings.ReplaceAll(matchPath, \"%*\", \"*\")\n\n\t// ignore error here because we can't handle it anyway=\n\tmatches, _ := path.Match(matchPath, sb.String())\n\treturn matches\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression path('*substring*', '*suffix')\nfunc (MatchPath) CELLibrary(ctx caddy.Context) (cel.Library, error) {\n\treturn CELMatcherImpl(\n\t\t// name of the macro, this is the function name that users see when writing expressions.\n\t\t\"path\",\n\t\t// name of the function that the macro will be rewritten to call.\n\t\t\"path_match_request_list\",\n\t\t// internal data type of the MatchPath value.\n\t\t[]*cel.Type{cel.ListType(cel.StringType)},\n\t\t// function to convert a constant list of strings to a MatchPath instance.\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\trefStringList := reflect.TypeOf([]string{})\n\t\t\tstrList, err := data.ConvertToNative(refStringList)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tmatcher := MatchPath(strList.([]string))\n\t\t\terr = matcher.Provision(ctx)\n\t\t\treturn matcher, err\n\t\t},\n\t)\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchPath) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\t*m = append(*m, d.RemainingArgs()...)\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed path matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchPathRE) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.path_regexp\",\n\t\tNew: func() caddy.Module { return new(MatchPathRE) },\n\t}\n}\n\n// Match returns true if r matches m.\nfunc (m MatchPathRE) Match(r *http.Request) bool {\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\n\t// Clean the path, merges doubled slashes, etc.\n\t// This ensures maliciously crafted requests can't bypass\n\t// the path matcher. See #4407\n\tcleanedPath := cleanPath(r.URL.Path)\n\n\treturn m.MatchRegexp.Match(cleanedPath, repl)\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression path_regexp('^/bar')\nfunc (MatchPathRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {\n\tunnamedPattern, err := CELMatcherImpl(\n\t\t\"path_regexp\",\n\t\t\"path_regexp_request_string\",\n\t\t[]*cel.Type{cel.StringType},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\tpattern := data.(types.String)\n\t\t\tmatcher := MatchPathRE{MatchRegexp{\n\t\t\t\tName:    ctx.Value(MatcherNameCtxKey).(string),\n\t\t\t\tPattern: string(pattern),\n\t\t\t}}\n\t\t\terr := matcher.Provision(ctx)\n\t\t\treturn matcher, err\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnamedPattern, err := CELMatcherImpl(\n\t\t\"path_regexp\",\n\t\t\"path_regexp_request_string_string\",\n\t\t[]*cel.Type{cel.StringType, cel.StringType},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\trefStringList := reflect.TypeOf([]string{})\n\t\t\tparams, err := data.ConvertToNative(refStringList)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tstrParams := params.([]string)\n\t\t\tname := strParams[0]\n\t\t\tif name == \"\" {\n\t\t\t\tname = ctx.Value(MatcherNameCtxKey).(string)\n\t\t\t}\n\t\t\tmatcher := MatchPathRE{MatchRegexp{\n\t\t\t\tName:    name,\n\t\t\t\tPattern: strParams[1],\n\t\t\t}}\n\t\t\terr = matcher.Provision(ctx)\n\t\t\treturn matcher, err\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tenvOpts := append(unnamedPattern.CompileOptions(), namedPattern.CompileOptions()...)\n\tprgOpts := append(unnamedPattern.ProgramOptions(), namedPattern.ProgramOptions()...)\n\treturn NewMatcherCELLibrary(envOpts, prgOpts), nil\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchMethod) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.method\",\n\t\tNew: func() caddy.Module { return new(MatchMethod) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchMethod) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\t*m = append(*m, d.RemainingArgs()...)\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed method matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m.\nfunc (m MatchMethod) Match(r *http.Request) bool {\n\tfor _, method := range m {\n\t\tif r.Method == method {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression method('PUT', 'POST')\nfunc (MatchMethod) CELLibrary(_ caddy.Context) (cel.Library, error) {\n\treturn CELMatcherImpl(\n\t\t\"method\",\n\t\t\"method_request_list\",\n\t\t[]*cel.Type{cel.ListType(cel.StringType)},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\trefStringList := reflect.TypeOf([]string{})\n\t\t\tstrList, err := data.ConvertToNative(refStringList)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn MatchMethod(strList.([]string)), nil\n\t\t},\n\t)\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchQuery) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.query\",\n\t\tNew: func() caddy.Module { return new(MatchQuery) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchQuery) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\tif *m == nil {\n\t\t*m = make(map[string][]string)\n\t}\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\tfor _, query := range d.RemainingArgs() {\n\t\t\tif query == \"\" {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tbefore, after, found := strings.Cut(query, \"=\")\n\t\t\tif !found {\n\t\t\t\treturn d.Errf(\"malformed query matcher token: %s; must be in param=val format\", d.Val())\n\t\t\t}\n\t\t\turl.Values(*m).Add(before, after)\n\t\t}\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed query matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m. An empty m matches an empty query string.\nfunc (m MatchQuery) Match(r *http.Request) bool {\n\t// If no query keys are configured, this only\n\t// matches an empty query string.\n\tif len(m) == 0 {\n\t\treturn len(r.URL.Query()) == 0\n\t}\n\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\n\t// parse query string just once, for efficiency\n\tparsed, err := url.ParseQuery(r.URL.RawQuery)\n\tif err != nil {\n\t\t// Illegal query string. Likely bad escape sequence or unescaped literals.\n\t\t// Note that semicolons in query string have a controversial history. Summaries:\n\t\t// - https://github.com/golang/go/issues/50034\n\t\t// - https://github.com/golang/go/issues/25192\n\t\t// Despite the URL WHATWG spec mandating the use of & separators for query strings,\n\t\t// every URL parser implementation is different, and Filippo Valsorda rightly wrote:\n\t\t// \"Relying on parser alignment for security is doomed.\" Overall conclusion is that\n\t\t// splitting on & and rejecting ; in key=value pairs is safer than accepting raw ;.\n\t\t// We regard the Go team's decision as sound and thus reject malformed query strings.\n\t\treturn false\n\t}\n\n\t// Count the amount of matched keys, to ensure we AND\n\t// between all configured query keys; all keys must\n\t// match at least one value.\n\tmatchedKeys := 0\n\tfor param, vals := range m {\n\t\tparam = repl.ReplaceAll(param, \"\")\n\t\tparamVal, found := parsed[param]\n\t\tif !found {\n\t\t\treturn false\n\t\t}\n\t\tfor _, v := range vals {\n\t\t\tv = repl.ReplaceAll(v, \"\")\n\t\t\tif slices.Contains(paramVal, v) || v == \"*\" {\n\t\t\t\tmatchedKeys++\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\treturn matchedKeys == len(m)\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression query({'sort': 'asc'}) || query({'foo': ['*bar*', 'baz']})\nfunc (MatchQuery) CELLibrary(_ caddy.Context) (cel.Library, error) {\n\treturn CELMatcherImpl(\n\t\t\"query\",\n\t\t\"query_matcher_request_map\",\n\t\t[]*cel.Type{CELTypeJSON},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\tmapStrListStr, err := CELValueToMapStrList(data)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn MatchQuery(url.Values(mapStrListStr)), nil\n\t\t},\n\t)\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchHeader) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.header\",\n\t\tNew: func() caddy.Module { return new(MatchHeader) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchHeader) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\tif *m == nil {\n\t\t*m = make(map[string][]string)\n\t}\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\tvar field, val string\n\t\tif !d.Args(&field) {\n\t\t\treturn d.Errf(\"malformed header matcher: expected field\")\n\t\t}\n\n\t\tif strings.HasPrefix(field, \"!\") {\n\t\t\tif len(field) == 1 {\n\t\t\t\treturn d.Errf(\"malformed header matcher: must have field name following ! character\")\n\t\t\t}\n\n\t\t\tfield = field[1:]\n\t\t\theaders := *m\n\t\t\theaders[field] = nil\n\t\t\tm = &headers\n\t\t\tif d.NextArg() {\n\t\t\t\treturn d.Errf(\"malformed header matcher: null matching headers cannot have a field value\")\n\t\t\t}\n\t\t} else {\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.Errf(\"malformed header matcher: expected both field and value\")\n\t\t\t}\n\n\t\t\t// If multiple header matchers with the same header field are defined,\n\t\t\t// we want to add the existing to the list of headers (will be OR'ed)\n\t\t\tval = d.Val()\n\t\t\thttp.Header(*m).Add(field, val)\n\t\t}\n\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed header matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m.\nfunc (m MatchHeader) Match(r *http.Request) bool {\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\treturn matchHeaders(r.Header, http.Header(m), r.Host, repl)\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression header({'content-type': 'image/png'})\n//\texpression header({'foo': ['bar', 'baz']}) // match bar or baz\nfunc (MatchHeader) CELLibrary(_ caddy.Context) (cel.Library, error) {\n\treturn CELMatcherImpl(\n\t\t\"header\",\n\t\t\"header_matcher_request_map\",\n\t\t[]*cel.Type{CELTypeJSON},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\tmapStrListStr, err := CELValueToMapStrList(data)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn MatchHeader(http.Header(mapStrListStr)), nil\n\t\t},\n\t)\n}\n\n// getHeaderFieldVals returns the field values for the given fieldName from input.\n// The host parameter should be obtained from the http.Request.Host field since\n// net/http removes it from the header map.\n\nfunc getHeaderFieldVals(input http.Header, fieldName, host string) []string {\n\tfieldName = textproto.CanonicalMIMEHeaderKey(fieldName)\n\tif fieldName == \"Host\" && host != \"\" {\n\t\treturn []string{host}\n\t}\n\treturn input[fieldName]\n}\n\n// matchHeaders returns true if input matches the criteria in against without regex.\n// The host parameter should be obtained from the http.Request.Host field since\n// net/http removes it from the header map.\nfunc matchHeaders(input, against http.Header, host string, repl *caddy.Replacer) bool {\n\tfor field, allowedFieldVals := range against {\n\t\tactualFieldVals := getHeaderFieldVals(input, field, host)\n\t\tif allowedFieldVals != nil && len(allowedFieldVals) == 0 && actualFieldVals != nil {\n\t\t\t// a non-nil but empty list of allowed values means\n\t\t\t// match if the header field exists at all\n\t\t\tcontinue\n\t\t}\n\t\tif allowedFieldVals == nil && actualFieldVals == nil {\n\t\t\t// a nil list means match if the header does not exist at all\n\t\t\tcontinue\n\t\t}\n\t\tvar match bool\n\tfieldVals:\n\t\tfor _, actualFieldVal := range actualFieldVals {\n\t\t\tfor _, allowedFieldVal := range allowedFieldVals {\n\t\t\t\tif repl != nil {\n\t\t\t\t\tallowedFieldVal = repl.ReplaceAll(allowedFieldVal, \"\")\n\t\t\t\t}\n\t\t\t\tswitch {\n\t\t\t\tcase allowedFieldVal == \"*\":\n\t\t\t\t\tmatch = true\n\t\t\t\tcase strings.HasPrefix(allowedFieldVal, \"*\") && strings.HasSuffix(allowedFieldVal, \"*\"):\n\t\t\t\t\tmatch = strings.Contains(actualFieldVal, allowedFieldVal[1:len(allowedFieldVal)-1])\n\t\t\t\tcase strings.HasPrefix(allowedFieldVal, \"*\"):\n\t\t\t\t\tmatch = strings.HasSuffix(actualFieldVal, allowedFieldVal[1:])\n\t\t\t\tcase strings.HasSuffix(allowedFieldVal, \"*\"):\n\t\t\t\t\tmatch = strings.HasPrefix(actualFieldVal, allowedFieldVal[:len(allowedFieldVal)-1])\n\t\t\t\tdefault:\n\t\t\t\t\tmatch = actualFieldVal == allowedFieldVal\n\t\t\t\t}\n\t\t\t\tif match {\n\t\t\t\t\tbreak fieldVals\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif !match {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchHeaderRE) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.header_regexp\",\n\t\tNew: func() caddy.Module { return new(MatchHeaderRE) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchHeaderRE) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\tif *m == nil {\n\t\t*m = make(map[string]*MatchRegexp)\n\t}\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\tvar first, second, third string\n\t\tif !d.Args(&first, &second) {\n\t\t\treturn d.ArgErr()\n\t\t}\n\n\t\tvar name, field, val string\n\t\tif d.Args(&third) {\n\t\t\tname = first\n\t\t\tfield = second\n\t\t\tval = third\n\t\t} else {\n\t\t\tfield = first\n\t\t\tval = second\n\t\t}\n\n\t\t// Default to the named matcher's name, if no regexp name is provided\n\t\tif name == \"\" {\n\t\t\tname = d.GetContextString(caddyfile.MatcherNameCtxKey)\n\t\t}\n\n\t\t// If there's already a pattern for this field\n\t\t// then we would end up overwriting the old one\n\t\tif (*m)[field] != nil {\n\t\t\treturn d.Errf(\"header_regexp matcher can only be used once per named matcher, per header field: %s\", field)\n\t\t}\n\n\t\t(*m)[field] = &MatchRegexp{Pattern: val, Name: name}\n\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed header_regexp matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m.\nfunc (m MatchHeaderRE) Match(r *http.Request) bool {\n\tfor field, rm := range m {\n\t\tactualFieldVals := getHeaderFieldVals(r.Header, field, r.Host)\n\t\tmatch := false\n\tfieldVal:\n\t\tfor _, actualFieldVal := range actualFieldVals {\n\t\t\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\t\t\tif rm.Match(actualFieldVal, repl) {\n\t\t\t\tmatch = true\n\t\t\t\tbreak fieldVal\n\t\t\t}\n\t\t}\n\t\tif !match {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Provision compiles m's regular expressions.\nfunc (m MatchHeaderRE) Provision(ctx caddy.Context) error {\n\tfor _, rm := range m {\n\t\terr := rm.Provision(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Validate validates m's regular expressions.\nfunc (m MatchHeaderRE) Validate() error {\n\tfor _, rm := range m {\n\t\terr := rm.Validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression header_regexp('foo', 'Field', 'fo+')\nfunc (MatchHeaderRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {\n\tunnamedPattern, err := CELMatcherImpl(\n\t\t\"header_regexp\",\n\t\t\"header_regexp_request_string_string\",\n\t\t[]*cel.Type{cel.StringType, cel.StringType},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\trefStringList := reflect.TypeOf([]string{})\n\t\t\tparams, err := data.ConvertToNative(refStringList)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tstrParams := params.([]string)\n\t\t\tmatcher := MatchHeaderRE{}\n\t\t\tmatcher[strParams[0]] = &MatchRegexp{\n\t\t\t\tPattern: strParams[1],\n\t\t\t\tName:    ctx.Value(MatcherNameCtxKey).(string),\n\t\t\t}\n\t\t\terr = matcher.Provision(ctx)\n\t\t\treturn matcher, err\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tnamedPattern, err := CELMatcherImpl(\n\t\t\"header_regexp\",\n\t\t\"header_regexp_request_string_string_string\",\n\t\t[]*cel.Type{cel.StringType, cel.StringType, cel.StringType},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\trefStringList := reflect.TypeOf([]string{})\n\t\t\tparams, err := data.ConvertToNative(refStringList)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tstrParams := params.([]string)\n\t\t\tname := strParams[0]\n\t\t\tif name == \"\" {\n\t\t\t\tname = ctx.Value(MatcherNameCtxKey).(string)\n\t\t\t}\n\t\t\tmatcher := MatchHeaderRE{}\n\t\t\tmatcher[strParams[1]] = &MatchRegexp{\n\t\t\t\tPattern: strParams[2],\n\t\t\t\tName:    name,\n\t\t\t}\n\t\t\terr = matcher.Provision(ctx)\n\t\t\treturn matcher, err\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tenvOpts := append(unnamedPattern.CompileOptions(), namedPattern.CompileOptions()...)\n\tprgOpts := append(unnamedPattern.ProgramOptions(), namedPattern.ProgramOptions()...)\n\treturn NewMatcherCELLibrary(envOpts, prgOpts), nil\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchProtocol) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.protocol\",\n\t\tNew: func() caddy.Module { return new(MatchProtocol) },\n\t}\n}\n\n// Match returns true if r matches m.\nfunc (m MatchProtocol) Match(r *http.Request) bool {\n\tswitch string(m) {\n\tcase \"grpc\":\n\t\treturn strings.HasPrefix(r.Header.Get(\"content-type\"), \"application/grpc\")\n\tcase \"https\":\n\t\treturn r.TLS != nil\n\tcase \"http\":\n\t\treturn r.TLS == nil\n\tcase \"http/1.0\":\n\t\treturn r.ProtoMajor == 1 && r.ProtoMinor == 0\n\tcase \"http/1.0+\":\n\t\treturn r.ProtoAtLeast(1, 0)\n\tcase \"http/1.1\":\n\t\treturn r.ProtoMajor == 1 && r.ProtoMinor == 1\n\tcase \"http/1.1+\":\n\t\treturn r.ProtoAtLeast(1, 1)\n\tcase \"http/2\":\n\t\treturn r.ProtoMajor == 2\n\tcase \"http/2+\":\n\t\treturn r.ProtoAtLeast(2, 0)\n\tcase \"http/3\":\n\t\treturn r.ProtoMajor == 3\n\tcase \"http/3+\":\n\t\treturn r.ProtoAtLeast(3, 0)\n\t}\n\treturn false\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchProtocol) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\tvar proto string\n\t\tif !d.Args(&proto) {\n\t\t\treturn d.Err(\"expected exactly one protocol\")\n\t\t}\n\t\t*m = MatchProtocol(proto)\n\t}\n\treturn nil\n}\n\n// CELLibrary produces options that expose this matcher for use in CEL\n// expression matchers.\n//\n// Example:\n//\n//\texpression protocol('https')\nfunc (MatchProtocol) CELLibrary(_ caddy.Context) (cel.Library, error) {\n\treturn CELMatcherImpl(\n\t\t\"protocol\",\n\t\t\"protocol_request_string\",\n\t\t[]*cel.Type{cel.StringType},\n\t\tfunc(data ref.Val) (RequestMatcher, error) {\n\t\t\tprotocolStr, ok := data.(types.String)\n\t\t\tif !ok {\n\t\t\t\treturn nil, errors.New(\"protocol argument was not a string\")\n\t\t\t}\n\t\t\treturn MatchProtocol(strings.ToLower(string(protocolStr))), nil\n\t\t},\n\t)\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchNot) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.not\",\n\t\tNew: func() caddy.Module { return new(MatchNot) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchNot) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\tmatcherSet, err := ParseCaddyfileNestedMatcherSet(d)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tm.MatcherSetsRaw = append(m.MatcherSetsRaw, matcherSet)\n\t}\n\treturn nil\n}\n\n// UnmarshalJSON satisfies json.Unmarshaler. It puts the JSON\n// bytes directly into m's MatcherSetsRaw field.\nfunc (m *MatchNot) UnmarshalJSON(data []byte) error {\n\treturn json.Unmarshal(data, &m.MatcherSetsRaw)\n}\n\n// MarshalJSON satisfies json.Marshaler by marshaling\n// m's raw matcher sets.\nfunc (m MatchNot) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(m.MatcherSetsRaw)\n}\n\n// Provision loads the matcher modules to be negated.\nfunc (m *MatchNot) Provision(ctx caddy.Context) error {\n\tmatcherSets, err := ctx.LoadModule(m, \"MatcherSetsRaw\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"loading matcher sets: %v\", err)\n\t}\n\tfor _, modMap := range matcherSets.([]map[string]any) {\n\t\tvar ms MatcherSet\n\t\tfor _, modIface := range modMap {\n\t\t\tms = append(ms, modIface.(RequestMatcher))\n\t\t}\n\t\tm.MatcherSets = append(m.MatcherSets, ms)\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m. Since this matcher negates\n// the embedded matchers, false is returned if any of its matcher\n// sets return true.\nfunc (m MatchNot) Match(r *http.Request) bool {\n\tfor _, ms := range m.MatcherSets {\n\t\tif ms.Match(r) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// MatchRegexp is an embedable type for matching\n// using regular expressions. It adds placeholders\n// to the request's replacer.\ntype MatchRegexp struct {\n\t// A unique name for this regular expression. Optional,\n\t// but useful to prevent overwriting captures from other\n\t// regexp matchers.\n\tName string `json:\"name,omitempty\"`\n\n\t// The regular expression to evaluate, in RE2 syntax,\n\t// which is the same general syntax used by Go, Perl,\n\t// and Python. For details, see\n\t// [Go's regexp package](https://golang.org/pkg/regexp/).\n\t// Captures are accessible via placeholders. Unnamed\n\t// capture groups are exposed as their numeric, 1-based\n\t// index, while named capture groups are available by\n\t// the capture group name.\n\tPattern string `json:\"pattern\"`\n\n\tcompiled *regexp.Regexp\n}\n\n// Provision compiles the regular expression.\nfunc (mre *MatchRegexp) Provision(caddy.Context) error {\n\tre, err := regexp.Compile(mre.Pattern)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"compiling matcher regexp %s: %v\", mre.Pattern, err)\n\t}\n\tmre.compiled = re\n\treturn nil\n}\n\n// Validate ensures mre is set up correctly.\nfunc (mre *MatchRegexp) Validate() error {\n\tif mre.Name != \"\" && !wordRE.MatchString(mre.Name) {\n\t\treturn fmt.Errorf(\"invalid regexp name (must contain only word characters): %s\", mre.Name)\n\t}\n\treturn nil\n}\n\n// Match returns true if input matches the compiled regular\n// expression in mre. It sets values on the replacer repl\n// associated with capture groups, using the given scope\n// (namespace).\nfunc (mre *MatchRegexp) Match(input string, repl *caddy.Replacer) bool {\n\tmatches := mre.compiled.FindStringSubmatch(input)\n\tif matches == nil {\n\t\treturn false\n\t}\n\n\t// save all capture groups, first by index\n\tfor i, match := range matches {\n\t\tkeySuffix := \".\" + strconv.Itoa(i)\n\t\tif mre.Name != \"\" {\n\t\t\trepl.Set(regexpPlaceholderPrefix+\".\"+mre.Name+keySuffix, match)\n\t\t}\n\t\trepl.Set(regexpPlaceholderPrefix+keySuffix, match)\n\t}\n\n\t// then by name\n\tfor i, name := range mre.compiled.SubexpNames() {\n\t\t// skip the first element (the full match), and empty names\n\t\tif i == 0 || name == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tkeySuffix := \".\" + name\n\t\tif mre.Name != \"\" {\n\t\t\trepl.Set(regexpPlaceholderPrefix+\".\"+mre.Name+keySuffix, matches[i])\n\t\t}\n\t\trepl.Set(regexpPlaceholderPrefix+keySuffix, matches[i])\n\t}\n\n\treturn true\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (mre *MatchRegexp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\t// If this is the second iteration of the loop\n\t\t// then there's more than one path_regexp matcher\n\t\t// and we would end up overwriting the old one\n\t\tif mre.Pattern != \"\" {\n\t\t\treturn d.Err(\"regular expression can only be used once per named matcher\")\n\t\t}\n\n\t\targs := d.RemainingArgs()\n\t\tswitch len(args) {\n\t\tcase 1:\n\t\t\tmre.Pattern = args[0]\n\t\tcase 2:\n\t\t\tmre.Name = args[0]\n\t\t\tmre.Pattern = args[1]\n\t\tdefault:\n\t\t\treturn d.ArgErr()\n\t\t}\n\n\t\t// Default to the named matcher's name, if no regexp name is provided\n\t\tif mre.Name == \"\" {\n\t\t\tmre.Name = d.GetContextString(caddyfile.MatcherNameCtxKey)\n\t\t}\n\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed path_regexp matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// ParseCaddyfileNestedMatcher parses the Caddyfile tokens for a nested\n// matcher set, and returns its raw module map value.\nfunc ParseCaddyfileNestedMatcherSet(d *caddyfile.Dispenser) (caddy.ModuleMap, error) {\n\tmatcherMap := make(map[string]RequestMatcher)\n\n\t// in case there are multiple instances of the same matcher, concatenate\n\t// their tokens (we expect that UnmarshalCaddyfile should be able to\n\t// handle more than one segment); otherwise, we'd overwrite other\n\t// instances of the matcher in this set\n\ttokensByMatcherName := make(map[string][]caddyfile.Token)\n\tfor nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {\n\t\tmatcherName := d.Val()\n\t\ttokensByMatcherName[matcherName] = append(tokensByMatcherName[matcherName], d.NextSegment()...)\n\t}\n\n\tfor matcherName, tokens := range tokensByMatcherName {\n\t\tmod, err := caddy.GetModule(\"http.matchers.\" + matcherName)\n\t\tif err != nil {\n\t\t\treturn nil, d.Errf(\"getting matcher module '%s': %v\", matcherName, err)\n\t\t}\n\t\tunm, ok := mod.New().(caddyfile.Unmarshaler)\n\t\tif !ok {\n\t\t\treturn nil, d.Errf(\"matcher module '%s' is not a Caddyfile unmarshaler\", matcherName)\n\t\t}\n\t\terr = unm.UnmarshalCaddyfile(caddyfile.NewDispenser(tokens))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trm, ok := unm.(RequestMatcher)\n\t\tif !ok {\n\t\t\treturn nil, fmt.Errorf(\"matcher module '%s' is not a request matcher\", matcherName)\n\t\t}\n\t\tmatcherMap[matcherName] = rm\n\t}\n\n\t// we should now have a functional matcher, but we also\n\t// need to be able to marshal as JSON, otherwise config\n\t// adaptation will be missing the matchers!\n\tmatcherSet := make(caddy.ModuleMap)\n\tfor name, matcher := range matcherMap {\n\t\tjsonBytes, err := json.Marshal(matcher)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"marshaling %T matcher: %v\", matcher, err)\n\t\t}\n\t\tmatcherSet[name] = jsonBytes\n\t}\n\n\treturn matcherSet, nil\n}\n\nvar wordRE = regexp.MustCompile(`\\w+`)\n\nconst regexpPlaceholderPrefix = \"http.regexp\"\n\n// MatcherErrorVarKey is the key used for the variable that\n// holds an optional error emitted from a request matcher,\n// to short-circuit the handler chain, since matchers cannot\n// return errors via the RequestMatcher interface.\nconst MatcherErrorVarKey = \"matchers.error\"\n\n// Interface guards\nvar (\n\t_ RequestMatcher    = (*MatchHost)(nil)\n\t_ caddy.Provisioner = (*MatchHost)(nil)\n\t_ RequestMatcher    = (*MatchPath)(nil)\n\t_ RequestMatcher    = (*MatchPathRE)(nil)\n\t_ caddy.Provisioner = (*MatchPathRE)(nil)\n\t_ RequestMatcher    = (*MatchMethod)(nil)\n\t_ RequestMatcher    = (*MatchQuery)(nil)\n\t_ RequestMatcher    = (*MatchHeader)(nil)\n\t_ RequestMatcher    = (*MatchHeaderRE)(nil)\n\t_ caddy.Provisioner = (*MatchHeaderRE)(nil)\n\t_ RequestMatcher    = (*MatchProtocol)(nil)\n\t_ RequestMatcher    = (*MatchNot)(nil)\n\t_ caddy.Provisioner = (*MatchNot)(nil)\n\t_ caddy.Provisioner = (*MatchRegexp)(nil)\n\n\t_ caddyfile.Unmarshaler = (*MatchHost)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchPath)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchPathRE)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchMethod)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchQuery)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchHeader)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchHeaderRE)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchProtocol)(nil)\n\t_ caddyfile.Unmarshaler = (*VarsMatcher)(nil)\n\t_ caddyfile.Unmarshaler = (*MatchVarsRE)(nil)\n\n\t_ CELLibraryProducer = (*MatchHost)(nil)\n\t_ CELLibraryProducer = (*MatchPath)(nil)\n\t_ CELLibraryProducer = (*MatchPathRE)(nil)\n\t_ CELLibraryProducer = (*MatchMethod)(nil)\n\t_ CELLibraryProducer = (*MatchQuery)(nil)\n\t_ CELLibraryProducer = (*MatchHeader)(nil)\n\t_ CELLibraryProducer = (*MatchHeaderRE)(nil)\n\t_ CELLibraryProducer = (*MatchProtocol)(nil)\n\t// _ CELLibraryProducer = (*VarsMatcher)(nil)\n\t// _ CELLibraryProducer = (*MatchVarsRE)(nil)\n\n\t_ json.Marshaler   = (*MatchNot)(nil)\n\t_ json.Unmarshaler = (*MatchNot)(nil)\n)\n\n// Path: modules/caddyhttp/matchers_test.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/url\"\n\t\"os\"\n\t\"runtime\"\n\t\"testing\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\nfunc TestHostMatcher(t *testing.T) {\n\terr := os.Setenv(\"GO_BENCHMARK_DOMAIN\", \"localhost\")\n\tif err != nil {\n\t\tt.Errorf(\"error while setting up environment: %v\", err)\n\t}\n\n\tfor i, tc := range []struct {\n\t\tmatch  MatchHost\n\t\tinput  string\n\t\texpect bool\n\t}{\n\t\t{\n\t\t\tmatch:  MatchHost{},\n\t\t\tinput:  \"example.com\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"example.com\"},\n\t\t\tinput:  \"example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"EXAMPLE.COM\"},\n\t\t\tinput:  \"example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"example.com\"},\n\t\t\tinput:  \"EXAMPLE.COM\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"example.com\"},\n\t\t\tinput:  \"foo.example.com\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"example.com\"},\n\t\t\tinput:  \"EXAMPLE.COM\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"foo.example.com\"},\n\t\t\tinput:  \"foo.example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"foo.example.com\"},\n\t\t\tinput:  \"bar.example.com\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.com\"},\n\t\t\tinput:  \"example.com\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.com\"},\n\t\t\tinput:  \"SUB.EXAMPLE.COM\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.com\"},\n\t\t\tinput:  \"foo.example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.com\"},\n\t\t\tinput:  \"foo.bar.example.com\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.com\", \"example.net\"},\n\t\t\tinput:  \"example.net\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"example.net\", \"*.example.com\"},\n\t\t\tinput:  \"foo.example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.net\", \"*.*.example.com\"},\n\t\t\tinput:  \"foo.bar.example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.net\", \"sub.*.example.com\"},\n\t\t\tinput:  \"sub.foo.example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"*.example.net\", \"sub.*.example.com\"},\n\t\t\tinput:  \"sub.foo.example.net\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"www.*.*\"},\n\t\t\tinput:  \"www.example.com\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"example.com\"},\n\t\t\tinput:  \"example.com:5555\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"{env.GO_BENCHMARK_DOMAIN}\"},\n\t\t\tinput:  \"localhost\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchHost{\"{env.GO_NONEXISTENT}\"},\n\t\t\tinput:  \"localhost\",\n\t\t\texpect: false,\n\t\t},\n\t} {\n\t\treq := &http.Request{Host: tc.input}\n\t\trepl := caddy.NewReplacer()\n\t\tctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)\n\t\treq = req.WithContext(ctx)\n\n\t\tactual := tc.match.Match(req)\n\t\tif actual != tc.expect {\n\t\t\tt.Errorf(\"Test %d %v: Expected %t, got %t for '%s'\", i, tc.match, tc.expect, actual, tc.input)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\nfunc TestPathMatcher(t *testing.T) {\n\tfor i, tc := range []struct {\n\t\tmatch        MatchPath // not URI-encoded because not parsing from a URI\n\t\tinput        string    // should be valid URI encoding (escaped) since it will become part of a request\n\t\texpect       bool\n\t\tprovisionErr bool\n\t}{\n\t\t{\n\t\t\tmatch:  MatchPath{},\n\t\t\tinput:  \"/\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/\"},\n\t\t\tinput:  \"/\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/bar\"},\n\t\t\tinput:  \"/\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/bar\"},\n\t\t\tinput:  \"/foo/bar\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/bar/\"},\n\t\t\tinput:  \"/foo/bar\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/bar/\"},\n\t\t\tinput:  \"/foo/bar/\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/bar/\", \"/other\"},\n\t\t\tinput:  \"/other/\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/bar/\", \"/other\"},\n\t\t\tinput:  \"/other\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"*.ext\"},\n\t\t\tinput:  \"/foo/bar.ext\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"*.php\"},\n\t\t\tinput:  \"/index.PHP\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"*.ext\"},\n\t\t\tinput:  \"/foo/bar.ext\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/*/baz\"},\n\t\t\tinput:  \"/foo/bar/baz\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo/*/baz/bam\"},\n\t\t\tinput:  \"/foo/bar/bam\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"*substring*\"},\n\t\t\tinput:  \"/foo/substring/bar.txt\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo\"},\n\t\t\tinput:  \"/foo/bar\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo\"},\n\t\t\tinput:  \"/foo/bar\",\n\t\t\texpect: false,\n\t\t},\n\t\t{\n\t\t\tmatch:  MatchPath{\"/foo\"},\n\t\t\tinput:  \"/FOO\",\n\t\t\texpect: true,\n\t\t},\n\t\t{\n...\n", "needle_token_start": 6817, "needle_token_end": 6893, "code_context_ntokens": 15188, "output": ["```go\nfunc getHeaderFieldVals(input http.Header, fieldName, host string) []string {\n\tfieldName = textproto.CanonicalMIMEHeaderKey(fieldName)\n\tif fieldName == \"Host\" && host != \"\" {\n\t\treturn []string{host}\n\t}\n\treturn input[fieldName]\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "cloneRequest", "language": "go", "path": "modules/caddyhttp/reverseproxy/reverseproxy.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To create a modified copy of an HTTP request for safe manipulation and forwarding to different destinations without altering the original request, ensuring thread safety and preventing data races.\n2. **Input**: A pointer to an HTTP request object.\n3. **Output**: A pointer to a new HTTP request object that is a semi-deep clone of the original.\n4. **Procedure**: \n   - A new HTTP request object is created.\n   - The new request is a shallow copy of the original request.\n   - The URL of the original request is deeply cloned, including the user information if present.\n   - The scheme and host of the URL are cleared to prevent unintended behavior in reverse proxy scenarios.\n   - Headers and trailers of the original request are deeply cloned to allow independent modification.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\tupstreams := h.Upstreams\n\tif h.DynamicUpstreams != nil {\n\t\tdUpstreams, err := h.DynamicUpstreams.GetUpstreams(r)\n\t\tif err != nil {\n\t\t\th.logger.Error(\"failed getting dynamic upstreams; falling back to static upstreams\", zap.Error(err))\n\t\t} else {\n\t\t\tupstreams = dUpstreams\n\t\t\tfor _, dUp := range dUpstreams {\n\t\t\t\th.provisionUpstream(dUp)\n\t\t\t}\n\t\t\th.logger.Debug(\"provisioned dynamic upstreams\", zap.Int(\"count\", len(dUpstreams)))\n\t\t\tdefer func() {\n\t\t\t\t// these upstreams are dynamic, so they are only used for this iteration\n\t\t\t\t// of the proxy loop; be sure to let them go away when we're done with them\n\t\t\t\tfor _, upstream := range dUpstreams {\n\t\t\t\t\t_, _ = hosts.Delete(upstream.String())\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t}\n\n\t// choose an available upstream\n\tupstream := h.LoadBalancing.SelectionPolicy.Select(upstreams, r, w)\n\tif upstream == nil {\n\t\tif proxyErr == nil {\n\t\t\tproxyErr = caddyhttp.Error(http.StatusServiceUnavailable, errNoUpstream)\n\t\t}\n\t\tif !h.LoadBalancing.tryAgain(h.ctx, start, retries, proxyErr, r) {\n\t\t\treturn true, proxyErr\n\t\t}\n\t\treturn false, proxyErr\n\t}\n\n\t// the dial address may vary per-request if placeholders are\n\t// used, so perform those replacements here; the resulting\n\t// DialInfo struct should have valid network address syntax\n\tdialInfo, err := upstream.fillDialInfo(r)\n\tif err != nil {\n\t\treturn true, fmt.Errorf(\"making dial info: %v\", err)\n\t}\n\n\th.logger.Debug(\"selected upstream\",\n\t\tzap.String(\"dial\", dialInfo.Address),\n\t\tzap.Int(\"total_upstreams\", len(upstreams)))\n\n\t// attach to the request information about how to dial the upstream;\n\t// this is necessary because the information cannot be sufficiently\n\t// or satisfactorily represented in a URL\n\tcaddyhttp.SetVar(r.Context(), dialInfoVarKey, dialInfo)\n\n\t// set placeholders with information about this upstream\n\trepl.Set(\"http.reverse_proxy.upstream.address\", dialInfo.String())\n\trepl.Set(\"http.reverse_proxy.upstream.hostport\", dialInfo.Address)\n\trepl.Set(\"http.reverse_proxy.upstream.host\", dialInfo.Host)\n\trepl.Set(\"http.reverse_proxy.upstream.port\", dialInfo.Port)\n\trepl.Set(\"http.reverse_proxy.upstream.requests\", upstream.Host.NumRequests())\n\trepl.Set(\"http.reverse_proxy.upstream.max_requests\", upstream.MaxRequests)\n\trepl.Set(\"http.reverse_proxy.upstream.fails\", upstream.Host.Fails())\n\n\t// mutate request headers according to this upstream;\n\t// because we're in a retry loop, we have to copy\n\t// headers (and the r.Host value) from the original\n\t// so that each retry is identical to the first\n\tif h.Headers != nil && h.Headers.Request != nil {\n\t\tr.Header = make(http.Header)\n\t\tcopyHeader(r.Header, reqHeader)\n\t\tr.Host = reqHost\n\t\th.Headers.Request.ApplyToRequest(r)\n\t}\n\n\t// proxy the request to that upstream\n\tproxyErr = h.reverseProxy(w, r, origReq, repl, dialInfo, next)\n\tif proxyErr == nil || errors.Is(proxyErr, context.Canceled) {\n\t\t// context.Canceled happens when the downstream client\n\t\t// cancels the request, which is not our failure\n\t\treturn true, nil\n\t}\n\n\t// if the roundtrip was successful, don't retry the request or\n\t// ding the health status of the upstream (an error can still\n\t// occur after the roundtrip if, for example, a response handler\n\t// after the roundtrip returns an error)\n\tif succ, ok := proxyErr.(roundtripSucceeded); ok {\n\t\treturn true, succ.error\n\t}\n\n\t// remember this failure (if enabled)\n\th.countFailure(upstream)\n\n\t// if we've tried long enough, break\n\tif !h.LoadBalancing.tryAgain(h.ctx, start, retries, proxyErr, r) {\n\t\treturn true, proxyErr\n\t}\n\n\treturn false, proxyErr\n}\n\n// prepareRequest clones req so that it can be safely modified without\n// changing the original request or introducing data races. It then\n// modifies it so that it is ready to be proxied, except for directing\n// to a specific upstream. This method adjusts headers and other relevant\n// properties of the cloned request and should be done just once (before\n// proxying) regardless of proxy retries. This assumes that no mutations\n// of the cloned request are performed by h during or after proxying.\nfunc (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http.Request, error) {\n\treq = cloneRequest(req)\n\n\t// if enabled, perform rewrites on the cloned request; if\n\t// the method is GET or HEAD, prevent the request body\n\t// from being copied to the upstream\n\tif h.Rewrite != nil {\n\t\tchanged := h.Rewrite.Rewrite(req, repl)\n\t\tif changed && (h.Rewrite.Method == \"GET\" || h.Rewrite.Method == \"HEAD\") {\n\t\t\treq.ContentLength = 0\n\t\t\treq.Body = nil\n\t\t}\n\t}\n\n\t// if enabled, buffer client request; this should only be\n\t// enabled if the upstream requires it and does not work\n\t// with \"slow clients\" (gunicorn, etc.) - this obviously\n\t// has a perf overhead and makes the proxy at risk of\n\t// exhausting memory and more susceptible to slowloris\n\t// attacks, so it is strongly recommended to only use this\n\t// feature if absolutely required, if read timeouts are\n\t// set, and if body size is limited\n\tif h.RequestBuffers != 0 && req.Body != nil {\n\t\tvar readBytes int64\n\t\treq.Body, readBytes = h.bufferedBody(req.Body, h.RequestBuffers)\n\t\tif h.RequestBuffers == -1 {\n\t\t\treq.ContentLength = readBytes\n\t\t\treq.Header.Set(\"Content-Length\", strconv.FormatInt(req.ContentLength, 10))\n\t\t}\n\t}\n\n\tif req.ContentLength == 0 {\n\t\treq.Body = nil // Issue golang/go#16036: nil Body for http.Transport retries\n\t}\n\n\treq.Close = false\n\n\t// if User-Agent is not set by client, then explicitly\n\t// disable it so it's not set to default value by std lib\n\tif _, ok := req.Header[\"User-Agent\"]; !ok {\n\t\treq.Header.Set(\"User-Agent\", \"\")\n\t}\n\n\treqUpType := upgradeType(req.Header)\n\tremoveConnectionHeaders(req.Header)\n\n\t// Remove hop-by-hop headers to the backend. Especially\n\t// important is \"Connection\" because we want a persistent\n\t// connection, regardless of what the client sent to us.\n\t// Issue golang/go#46313: don't skip if field is empty.\n\tfor _, h := range hopHeaders {\n\t\t// Issue golang/go#21096: tell backend applications that care about trailer support\n\t\t// that we support trailers. (We do, but we don't go out of our way to\n\t\t// advertise that unless the incoming client request thought it was worth\n\t\t// mentioning.)\n\t\tif h == \"Te\" && httpguts.HeaderValuesContainsToken(req.Header[\"Te\"], \"trailers\") {\n\t\t\treq.Header.Set(\"Te\", \"trailers\")\n\t\t\tcontinue\n\t\t}\n\t\treq.Header.Del(h)\n\t}\n\n\t// After stripping all the hop-by-hop connection headers above, add back any\n\t// necessary for protocol upgrades, such as for websockets.\n\tif reqUpType != \"\" {\n\t\treq.Header.Set(\"Connection\", \"Upgrade\")\n\t\treq.Header.Set(\"Upgrade\", reqUpType)\n\t}\n\n\t// Set up the PROXY protocol info\n\taddress := caddyhttp.GetVar(req.Context(), caddyhttp.ClientIPVarKey).(string)\n\taddrPort, err := netip.ParseAddrPort(address)\n\tif err != nil {\n\t\t// OK; probably didn't have a port\n\t\taddr, err := netip.ParseAddr(address)\n\t\tif err != nil {\n\t\t\t// Doesn't seem like a valid ip address at all\n\t\t} else {\n\t\t\t// Ok, only the port was missing\n\t\t\taddrPort = netip.AddrPortFrom(addr, 0)\n\t\t}\n\t}\n\tproxyProtocolInfo := ProxyProtocolInfo{AddrPort: addrPort}\n\tcaddyhttp.SetVar(req.Context(), proxyProtocolInfoVarKey, proxyProtocolInfo)\n\n\t// Add the supported X-Forwarded-* headers\n\terr = h.addForwardedHeaders(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn req, nil\n}\n\n// addForwardedHeaders adds the de-facto standard X-Forwarded-*\n// headers to the request before it is sent upstream.\n//\n// These headers are security sensitive, so care is taken to only\n// use existing values for these headers from the incoming request\n// if the client IP is trusted (i.e. coming from a trusted proxy\n// sitting in front of this server). If the request didn't have\n// the headers at all, then they will be added with the values\n// that we can glean from the request.\nfunc (h Handler) addForwardedHeaders(req *http.Request) error {\n\t// Parse the remote IP, ignore the error as non-fatal,\n\t// but the remote IP is required to continue, so we\n\t// just return early. This should probably never happen\n\t// though, unless some other module manipulated the request's\n\t// remote address and used an invalid value.\n\tclientIP, _, err := net.SplitHostPort(req.RemoteAddr)\n\tif err != nil {\n\t\t// Remove the `X-Forwarded-*` headers to avoid upstreams\n\t\t// potentially trusting a header that came from the client\n\t\treq.Header.Del(\"X-Forwarded-For\")\n\t\treq.Header.Del(\"X-Forwarded-Proto\")\n\t\treq.Header.Del(\"X-Forwarded-Host\")\n\t\treturn nil\n\t}\n\n\t// Client IP may contain a zone if IPv6, so we need\n\t// to pull that out before parsing the IP\n\tclientIP, _, _ = strings.Cut(clientIP, \"%\")\n\tipAddr, err := netip.ParseAddr(clientIP)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"invalid IP address: '%s': %v\", clientIP, err)\n\t}\n\n\t// Check if the client is a trusted proxy\n\ttrusted := caddyhttp.GetVar(req.Context(), caddyhttp.TrustedProxyVarKey).(bool)\n\tfor _, ipRange := range h.trustedProxies {\n\t\tif ipRange.Contains(ipAddr) {\n\t\t\ttrusted = true\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// If we aren't the first proxy, and the proxy is trusted,\n\t// retain prior X-Forwarded-For information as a comma+space\n\t// separated list and fold multiple headers into one.\n\tclientXFF := clientIP\n\tprior, ok, omit := allHeaderValues(req.Header, \"X-Forwarded-For\")\n\tif trusted && ok && prior != \"\" {\n\t\tclientXFF = prior + \", \" + clientXFF\n\t}\n\tif !omit {\n\t\treq.Header.Set(\"X-Forwarded-For\", clientXFF)\n\t}\n\n\t// Set X-Forwarded-Proto; many backend apps expect this,\n\t// so that they can properly craft URLs with the right\n\t// scheme to match the original request\n\tproto := \"https\"\n\tif req.TLS == nil {\n\t\tproto = \"http\"\n\t}\n\tprior, ok, omit = lastHeaderValue(req.Header, \"X-Forwarded-Proto\")\n\tif trusted && ok && prior != \"\" {\n\t\tproto = prior\n\t}\n\tif !omit {\n\t\treq.Header.Set(\"X-Forwarded-Proto\", proto)\n\t}\n\n\t// Set X-Forwarded-Host; often this is redundant because\n\t// we pass through the request Host as-is, but in situations\n\t// where we proxy over HTTPS, the user may need to override\n\t// Host themselves, so it's helpful to send the original too.\n\thost := req.Host\n\tprior, ok, omit = lastHeaderValue(req.Header, \"X-Forwarded-Host\")\n\tif trusted && ok && prior != \"\" {\n\t\thost = prior\n\t}\n\tif !omit {\n\t\treq.Header.Set(\"X-Forwarded-Host\", host)\n\t}\n\n\treturn nil\n}\n\n// reverseProxy performs a round-trip to the given backend and processes the response with the client.\n// (This method is mostly the beginning of what was borrowed from the net/http/httputil package in the\n// Go standard library which was used as the foundation.)\nfunc (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origReq *http.Request, repl *caddy.Replacer, di DialInfo, next caddyhttp.Handler) error {\n\t_ = di.Upstream.Host.countRequest(1)\n\t//nolint:errcheck\n\tdefer di.Upstream.Host.countRequest(-1)\n\n\t// point the request to this upstream\n\th.directRequest(req, di)\n\n\tserver := req.Context().Value(caddyhttp.ServerCtxKey).(*caddyhttp.Server)\n\tshouldLogCredentials := server.Logs != nil && server.Logs.ShouldLogCredentials\n\n\t// Forward 1xx status codes, backported from https://github.com/golang/go/pull/53164\n\ttrace := &httptrace.ClientTrace{\n\t\tGot1xxResponse: func(code int, header textproto.MIMEHeader) error {\n\t\t\th := rw.Header()\n\t\t\tcopyHeader(h, http.Header(header))\n\t\t\trw.WriteHeader(code)\n\n\t\t\t// Clear headers coming from the backend\n\t\t\t// (it's not automatically done by ResponseWriter.WriteHeader() for 1xx responses)\n\t\t\tfor k := range header {\n\t\t\t\tdelete(h, k)\n\t\t\t}\n\n\t\t\treturn nil\n\t\t},\n\t}\n\treq = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))\n\n\t// if FlushInterval is explicitly configured to -1 (i.e. flush continuously to achieve\n\t// low-latency streaming), don't let the transport cancel the request if the client\n\t// disconnects: user probably wants us to finish sending the data to the upstream\n\t// regardless, and we should expect client disconnection in low-latency streaming\n\t// scenarios (see issue #4922)\n\tif h.FlushInterval == -1 {\n\t\treq = req.WithContext(context.WithoutCancel(req.Context()))\n\t}\n\n\t// do the round-trip; emit debug log with values we know are\n\t// safe, or if there is no error, emit fuller log entry\n\tstart := time.Now()\n\tres, err := h.Transport.RoundTrip(req)\n\tduration := time.Since(start)\n\tlogger := h.logger.With(\n\t\tzap.String(\"upstream\", di.Upstream.String()),\n\t\tzap.Duration(\"duration\", duration),\n\t\tzap.Object(\"request\", caddyhttp.LoggableHTTPRequest{\n\t\t\tRequest:              req,\n\t\t\tShouldLogCredentials: shouldLogCredentials,\n\t\t}),\n\t)\n\tif err != nil {\n\t\tlogger.Debug(\"upstream roundtrip\", zap.Error(err))\n\t\treturn err\n\t}\n\tlogger.Debug(\"upstream roundtrip\",\n\t\tzap.Object(\"headers\", caddyhttp.LoggableHTTPHeader{\n\t\t\tHeader:               res.Header,\n\t\t\tShouldLogCredentials: shouldLogCredentials,\n\t\t}),\n\t\tzap.Int(\"status\", res.StatusCode))\n\n\t// duration until upstream wrote response headers (roundtrip duration)\n\trepl.Set(\"http.reverse_proxy.upstream.latency\", duration)\n\trepl.Set(\"http.reverse_proxy.upstream.latency_ms\", duration.Seconds()*1e3) // multiply seconds to preserve decimal (see #4666)\n\n\t// update circuit breaker on current conditions\n\tif di.Upstream.cb != nil {\n\t\tdi.Upstream.cb.RecordMetric(res.StatusCode, duration)\n\t}\n\n\t// perform passive health checks (if enabled)\n\tif h.HealthChecks != nil && h.HealthChecks.Passive != nil {\n\t\t// strike if the status code matches one that is \"bad\"\n\t\tfor _, badStatus := range h.HealthChecks.Passive.UnhealthyStatus {\n\t\t\tif caddyhttp.StatusCodeMatches(res.StatusCode, badStatus) {\n\t\t\t\th.countFailure(di.Upstream)\n\t\t\t}\n\t\t}\n\n\t\t// strike if the roundtrip took too long\n\t\tif h.HealthChecks.Passive.UnhealthyLatency > 0 &&\n\t\t\tduration >= time.Duration(h.HealthChecks.Passive.UnhealthyLatency) {\n\t\t\th.countFailure(di.Upstream)\n\t\t}\n\t}\n\n\t// if enabled, buffer the response body\n\tif h.ResponseBuffers != 0 {\n\t\tres.Body, _ = h.bufferedBody(res.Body, h.ResponseBuffers)\n\t}\n\n\t// see if any response handler is configured for this response from the backend\n\tfor i, rh := range h.HandleResponse {\n\t\tif rh.Match != nil && !rh.Match.Match(res.StatusCode, res.Header) {\n\t\t\tcontinue\n\t\t}\n\n\t\t// if configured to only change the status code,\n\t\t// do that then continue regular proxy response\n\t\tif statusCodeStr := rh.StatusCode.String(); statusCodeStr != \"\" {\n\t\t\tstatusCode, err := strconv.Atoi(repl.ReplaceAll(statusCodeStr, \"\"))\n\t\t\tif err != nil {\n\t\t\t\treturn caddyhttp.Error(http.StatusInternalServerError, err)\n\t\t\t}\n\t\t\tif statusCode != 0 {\n\t\t\t\tres.StatusCode = statusCode\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\t// set up the replacer so that parts of the original response can be\n\t\t// used for routing decisions\n\t\tfor field, value := range res.Header {\n\t\t\trepl.Set(\"http.reverse_proxy.header.\"+field, strings.Join(value, \",\"))\n\t\t}\n\t\trepl.Set(\"http.reverse_proxy.status_code\", res.StatusCode)\n\t\trepl.Set(\"http.reverse_proxy.status_text\", res.Status)\n\n\t\tlogger.Debug(\"handling response\", zap.Int(\"handler\", i))\n\n\t\t// we make some data available via request context to child routes\n\t\t// so that they may inherit some options and functions from the\n\t\t// handler, and be able to copy the response.\n\t\t// we use the original request here, so that any routes from 'next'\n\t\t// see the original request rather than the proxy cloned request.\n\t\thrc := &handleResponseContext{\n\t\t\thandler:  h,\n\t\t\tresponse: res,\n\t\t\tstart:    start,\n\t\t\tlogger:   logger,\n\t\t}\n\t\tctx := origReq.Context()\n\t\tctx = context.WithValue(ctx, proxyHandleResponseContextCtxKey, hrc)\n\n\t\t// pass the request through the response handler routes\n\t\trouteErr := rh.Routes.Compile(next).ServeHTTP(rw, origReq.WithContext(ctx))\n\n\t\t// close the response body afterwards, since we don't need it anymore;\n\t\t// either a route had 'copy_response' which already consumed the body,\n\t\t// or some other terminal handler ran which doesn't need the response\n\t\t// body after that point (e.g. 'file_server' for X-Accel-Redirect flow),\n\t\t// or we fell through to subsequent handlers past this proxy\n\t\t// (e.g. forward auth's 2xx response flow).\n\t\tif !hrc.isFinalized {\n\t\t\tres.Body.Close()\n\t\t}\n\n\t\t// wrap any route error in roundtripSucceeded so caller knows that\n\t\t// the roundtrip was successful and to not retry\n\t\tif routeErr != nil {\n\t\t\treturn roundtripSucceeded{routeErr}\n\t\t}\n\n\t\t// we're done handling the response, and we don't want to\n\t\t// fall through to the default finalize/copy behaviour\n\t\treturn nil\n\t}\n\n\t// copy the response body and headers back to the upstream client\n\treturn h.finalizeResponse(rw, req, res, repl, start, logger)\n}\n\n// finalizeResponse prepares and copies the response.\nfunc (h *Handler) finalizeResponse(\n\trw http.ResponseWriter,\n\treq *http.Request,\n\tres *http.Response,\n\trepl *caddy.Replacer,\n\tstart time.Time,\n\tlogger *zap.Logger,\n) error {\n\t// deal with 101 Switching Protocols responses: (WebSocket, h2c, etc)\n\tif res.StatusCode == http.StatusSwitchingProtocols {\n\t\tvar wg sync.WaitGroup\n\t\th.handleUpgradeResponse(logger, &wg, rw, req, res)\n\t\twg.Wait()\n\t\treturn nil\n\t}\n\n\tremoveConnectionHeaders(res.Header)\n\n\tfor _, h := range hopHeaders {\n\t\tres.Header.Del(h)\n\t}\n\n\t// apply any response header operations\n\tif h.Headers != nil && h.Headers.Response != nil {\n\t\tif h.Headers.Response.Require == nil ||\n\t\t\th.Headers.Response.Require.Match(res.StatusCode, res.Header) {\n\t\t\th.Headers.Response.ApplyTo(res.Header, repl)\n\t\t}\n\t}\n\n\tcopyHeader(rw.Header(), res.Header)\n\n\t// The \"Trailer\" header isn't included in the Transport's response,\n\t// at least for *http.Transport. Build it up from Trailer.\n\tannouncedTrailers := len(res.Trailer)\n\tif announcedTrailers > 0 {\n\t\ttrailerKeys := make([]string, 0, len(res.Trailer))\n\t\tfor k := range res.Trailer {\n\t\t\ttrailerKeys = append(trailerKeys, k)\n\t\t}\n\t\trw.Header().Add(\"Trailer\", strings.Join(trailerKeys, \", \"))\n\t}\n\n\trw.WriteHeader(res.StatusCode)\n\tif h.VerboseLogs {\n\t\tlogger.Debug(\"wrote header\")\n\t}\n\n\terr := h.copyResponse(rw, res.Body, h.flushInterval(req, res), logger)\n\terrClose := res.Body.Close() // close now, instead of defer, to populate res.Trailer\n\tif h.VerboseLogs || errClose != nil {\n\t\tlogger.Debug(\"closed response body from upstream\", zap.Error(errClose))\n\t}\n\tif err != nil {\n\t\t// we're streaming the response and we've already written headers, so\n\t\t// there's nothing an error handler can do to recover at this point;\n\t\t// we'll just log the error and abort the stream here and panic just as\n\t\t// the standard lib's proxy to propagate the stream error.\n\t\t// see issue https://github.com/caddyserver/caddy/issues/5951\n\t\tlogger.Error(\"aborting with incomplete response\", zap.Error(err))\n\t\t// no extra logging from stdlib\n\t\tpanic(http.ErrAbortHandler)\n\t}\n\n\tif len(res.Trailer) > 0 {\n\t\t// Force chunking if we saw a response trailer.\n\t\t// This prevents net/http from calculating the length for short\n\t\t// bodies and adding a Content-Length.\n\t\t//nolint:bodyclose\n\t\thttp.NewResponseController(rw).Flush()\n\t}\n\n\t// total duration spent proxying, including writing response body\n\trepl.Set(\"http.reverse_proxy.upstream.duration\", time.Since(start))\n\trepl.Set(\"http.reverse_proxy.upstream.duration_ms\", time.Since(start).Seconds()*1e3)\n\n\tif len(res.Trailer) == announcedTrailers {\n\t\tcopyHeader(rw.Header(), res.Trailer)\n\t\treturn nil\n\t}\n\n\tfor k, vv := range res.Trailer {\n\t\tk = http.TrailerPrefix + k\n\t\tfor _, v := range vv {\n\t\t\trw.Header().Add(k, v)\n\t\t}\n\t}\n\n\tif h.VerboseLogs {\n\t\tlogger.Debug(\"response finalized\")\n\t}\n\n\treturn nil\n}\n\n// tryAgain takes the time that the handler was initially invoked,\n// the amount of retries already performed, as well as any error\n// currently obtained, and the request being tried, and returns\n// true if another attempt should be made at proxying the request.\n// If true is returned, it has already blocked long enough before\n// the next retry (i.e. no more sleeping is needed). If false is\n// returned, the handler should stop trying to proxy the request.\nfunc (lb LoadBalancing) tryAgain(ctx caddy.Context, start time.Time, retries int, proxyErr error, req *http.Request) bool {\n\t// no retries are configured\n\tif lb.TryDuration == 0 && lb.Retries == 0 {\n\t\treturn false\n\t}\n\n\t// if we've tried long enough, break\n\tif lb.TryDuration > 0 && time.Since(start) >= time.Duration(lb.TryDuration) {\n\t\treturn false\n\t}\n\n\t// if we've reached the retry limit, break\n\tif lb.Retries > 0 && retries >= lb.Retries {\n\t\treturn false\n\t}\n\n\t// if the error occurred while dialing (i.e. a connection\n\t// could not even be established to the upstream), then it\n\t// should be safe to retry, since without a connection, no\n\t// HTTP request can be transmitted; but if the error is not\n\t// specifically a dialer error, we need to be careful\n\tif proxyErr != nil {\n\t\t_, isDialError := proxyErr.(DialError)\n\t\therr, isHandlerError := proxyErr.(caddyhttp.HandlerError)\n\n\t\t// if the error occurred after a connection was established,\n\t\t// we have to assume the upstream received the request, and\n\t\t// retries need to be carefully decided, because some requests\n\t\t// are not idempotent\n\t\tif !isDialError && !(isHandlerError && errors.Is(herr, errNoUpstream)) {\n\t\t\tif lb.RetryMatch == nil && req.Method != \"GET\" {\n\t\t\t\t// by default, don't retry requests if they aren't GET\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\tif !lb.RetryMatch.AnyMatch(req) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t}\n\n\t// fast path; if the interval is zero, we don't need to wait\n\tif lb.TryInterval == 0 {\n\t\treturn true\n\t}\n\n\t// otherwise, wait and try the next available host\n\ttimer := time.NewTimer(time.Duration(lb.TryInterval))\n\tselect {\n\tcase <-timer.C:\n\t\treturn true\n\tcase <-ctx.Done():\n\t\tif !timer.Stop() {\n\t\t\t// if the timer has been stopped then read from the channel\n\t\t\t<-timer.C\n\t\t}\n\t\treturn false\n\t}\n}\n\n// directRequest modifies only req.URL so that it points to the upstream\n// in the given DialInfo. It must modify ONLY the request URL.\nfunc (Handler) directRequest(req *http.Request, di DialInfo) {\n\t// we need a host, so set the upstream's host address\n\treqHost := di.Address\n\n\t// if the port equates to the scheme, strip the port because\n\t// it's weird to make a request like http://example.com:80/.\n\tif (req.URL.Scheme == \"http\" && di.Port == \"80\") ||\n\t\t(req.URL.Scheme == \"https\" && di.Port == \"443\") {\n\t\treqHost = di.Host\n\t}\n\n\treq.URL.Host = reqHost\n}\n\nfunc (h Handler) provisionUpstream(upstream *Upstream) {\n\t// create or get the host representation for this upstream\n\tupstream.fillHost()\n\n\t// give it the circuit breaker, if any\n\tupstream.cb = h.CB\n\n\t// if the passive health checker has a non-zero UnhealthyRequestCount\n\t// but the upstream has no MaxRequests set (they are the same thing,\n\t// but the passive health checker is a default value for upstreams\n\t// without MaxRequests), copy the value into this upstream, since the\n\t// value in the upstream (MaxRequests) is what is used during\n\t// availability checks\n\tif h.HealthChecks != nil &&\n\t\th.HealthChecks.Passive != nil &&\n\t\th.HealthChecks.Passive.UnhealthyRequestCount > 0 &&\n\t\tupstream.MaxRequests == 0 {\n\t\tupstream.MaxRequests = h.HealthChecks.Passive.UnhealthyRequestCount\n\t}\n\n\t// upstreams need independent access to the passive\n\t// health check policy because passive health checks\n\t// run without access to h.\n\tif h.HealthChecks != nil {\n\t\tupstream.healthCheckPolicy = h.HealthChecks.Passive\n\t}\n}\n\n// bufferedBody reads originalBody into a buffer with maximum size of limit (-1 for unlimited),\n// then returns a reader for the buffer along with how many bytes were buffered. Always close\n// the return value when done with it, just like if it was the original body! If limit is 0\n// (which it shouldn't be), this function returns its input; i.e. is a no-op, for safety.\nfunc (h Handler) bufferedBody(originalBody io.ReadCloser, limit int64) (io.ReadCloser, int64) {\n\tif limit == 0 {\n\t\treturn originalBody, 0\n\t}\n\tvar written int64\n\tbuf := bufPool.Get().(*bytes.Buffer)\n\tbuf.Reset()\n\tif limit > 0 {\n\t\tn, err := io.CopyN(buf, originalBody, limit)\n\t\tif (err != nil && err != io.EOF) || n == limit {\n\t\t\treturn bodyReadCloser{\n\t\t\t\tReader: io.MultiReader(buf, originalBody),\n\t\t\t\tbuf:    buf,\n\t\t\t\tbody:   originalBody,\n\t\t\t}, n\n\t\t}\n\t} else {\n\t\twritten, _ = io.Copy(buf, originalBody)\n\t}\n\toriginalBody.Close() // no point in keeping it open\n\treturn bodyReadCloser{\n\t\tReader: buf,\n\t\tbuf:    buf,\n\t}, written\n}\n\n// cloneRequest makes a semi-deep clone of origReq.\n//\n// Most of this code is borrowed from the Go stdlib reverse proxy,\n// but we make a shallow-ish clone the request (deep clone only\n// the headers and URL) so we can avoid manipulating the original\n// request when using it to proxy upstream. This prevents request\n// corruption and data races.\n\nfunc cloneRequest(origReq *http.Request) *http.Request {\n\treq := new(http.Request)\n\t*req = *origReq\n\tif origReq.URL != nil {\n\t\tnewURL := new(url.URL)\n\t\t*newURL = *origReq.URL\n\t\tif origReq.URL.User != nil {\n\t\t\tnewURL.User = new(url.Userinfo)\n\t\t\t*newURL.User = *origReq.URL.User\n\t\t}\n\t\t// sanitize the request URL; we expect it to not contain the\n\t\t// scheme and host since those should be determined by r.TLS\n\t\t// and r.Host respectively, but some clients may include it\n\t\t// in the request-line, which is technically valid in HTTP,\n\t\t// but breaks reverseproxy behaviour, overriding how the\n\t\t// dialer will behave. See #4237 for context.\n\t\tnewURL.Scheme = \"\"\n\t\tnewURL.Host = \"\"\n\t\treq.URL = newURL\n\t}\n\tif origReq.Header != nil {\n\t\treq.Header = origReq.Header.Clone()\n\t}\n\tif origReq.Trailer != nil {\n\t\treq.Trailer = origReq.Trailer.Clone()\n\t}\n\treturn req\n}\n\nfunc copyHeader(dst, src http.Header) {\n\tfor k, vv := range src {\n\t\tfor _, v := range vv {\n\t\t\tdst.Add(k, v)\n\t\t}\n\t}\n}\n\n// allHeaderValues gets all values for a given header field,\n// joined by a comma and space if more than one is set. If the\n// header field is nil, then the omit is true, meaning some\n// earlier logic in the server wanted to prevent this header from\n// getting written at all. If the header is empty, then ok is\n// false. Callers should still check that the value is not empty\n// (the header field may be set but have an empty value).\nfunc allHeaderValues(h http.Header, field string) (value string, ok bool, omit bool) {\n\tvalues, ok := h[http.CanonicalHeaderKey(field)]\n\tif ok && values == nil {\n\t\treturn \"\", true, true\n\t}\n\tif len(values) == 0 {\n\t\treturn \"\", false, false\n\t}\n\treturn strings.Join(values, \", \"), true, false\n}\n\n// lastHeaderValue gets the last value for a given header field\n// if more than one is set. If the header field is nil, then\n// the omit is true, meaning some earlier logic in the server\n// wanted to prevent this header from getting written at all.\n// If the header is empty, then ok is false. Callers should\n// still check that the value is not empty (the header field\n// may be set but have an empty value).\nfunc lastHeaderValue(h http.Header, field string) (value string, ok bool, omit bool) {\n\tvalues, ok := h[http.CanonicalHeaderKey(field)]\n\tif ok && values == nil {\n\t\treturn \"\", true, true\n\t}\n\tif len(values) == 0 {\n\t\treturn \"\", false, false\n\t}\n\treturn values[len(values)-1], true, false\n}\n\nfunc upgradeType(h http.Header) string {\n\tif !httpguts.HeaderValuesContainsToken(h[\"Connection\"], \"Upgrade\") {\n\t\treturn \"\"\n\t}\n\treturn strings.ToLower(h.Get(\"Upgrade\"))\n}\n\n// removeConnectionHeaders removes hop-by-hop headers listed in the \"Connection\" header of h.\n// See RFC 7230, section 6.1\nfunc removeConnectionHeaders(h http.Header) {\n\tfor _, f := range h[\"Connection\"] {\n\t\tfor _, sf := range strings.Split(f, \",\") {\n\t\t\tif sf = textproto.TrimString(sf); sf != \"\" {\n\t\t\t\th.Del(sf)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// statusError returns an error value that has a status code.\nfunc statusError(err error) error {\n\t// errors proxying usually mean there is a problem with the upstream(s)\n\tstatusCode := http.StatusBadGateway\n\n\t// timeout errors have a standard status code (see issue #4823)\n\tif err, ok := err.(net.Error); ok && err.Timeout() {\n\t\tstatusCode = http.StatusGatewayTimeout\n\t}\n\n\t// if the client canceled the request (usually this means they closed\n\t// the connection, so they won't see any response), we can report it\n\t// as a client error (4xx) and not a server error (5xx); unfortunately\n\t// the Go standard library, at least at time of writing in late 2020,\n\t// obnoxiously wraps the exported, standard context.Canceled error with\n\t// an unexported garbage value that we have to do a substring check for:\n\t// https://github.com/golang/go/blob/6965b01ea248cabb70c3749fd218b36089a21efb/src/net/net.go#L416-L430\n\tif errors.Is(err, context.Canceled) || strings.Contains(err.Error(), \"operation was canceled\") {\n\t\t// regrettably, there is no standard error code for \"client closed connection\", but\n\t\t// for historical reasons we can use a code that a lot of people are already using;\n\t\t// using 5xx is problematic for users; see #3748\n\t\tstatusCode = 499\n\t}\n\treturn caddyhttp.Error(statusCode, err)\n}\n\n// LoadBalancing has parameters related to load balancing.\ntype LoadBalancing struct {\n\t// A selection policy is how to choose an available backend.\n\t// The default policy is random selection.\n\tSelectionPolicyRaw json.RawMessage `json:\"selection_policy,omitempty\" caddy:\"namespace=http.reverse_proxy.selection_policies inline_key=policy\"`\n\n\t// How many times to retry selecting available backends for each\n\t// request if the next available host is down. If try_duration is\n\t// also configured, then retries may stop early if the duration\n\t// is reached. By default, retries are disabled (zero).\n\tRetries int `json:\"retries,omitempty\"`\n\n\t// How long to try selecting available backends for each request\n\t// if the next available host is down. Clients will wait for up\n\t// to this long while the load balancer tries to find an available\n\t// upstream host. If retries is also configured, tries may stop\n\t// early if the maximum retries is reached. By default, retries\n\t// are disabled (zero duration).\n\tTryDuration caddy.Duration `json:\"try_duration,omitempty\"`\n\n\t// How long to wait between selecting the next host from the pool.\n\t// Default is 250ms if try_duration is enabled, otherwise zero. Only\n\t// relevant when a request to an upstream host fails. Be aware that\n\t// setting this to 0 with a non-zero try_duration can cause the CPU\n\t// to spin if all backends are down and latency is very low.\n\tTryInterval caddy.Duration `json:\"try_interval,omitempty\"`\n\n\t// A list of matcher sets that restricts with which requests retries are\n\t// allowed. A request must match any of the given matcher sets in order\n\t// to be retried if the connection to the upstream succeeded but the\n\t// subsequent round-trip failed. If the connection to the upstream failed,\n\t// a retry is always allowed. If unspecified, only GET requests will be\n\t// allowed to be retried. Note that a retry is done with the next available\n\t// host according to the load balancing policy.\n\tRetryMatchRaw caddyhttp.RawMatcherSets `json:\"retry_match,omitempty\" caddy:\"namespace=http.matchers\"`\n\n\tSelectionPolicy Selector              `json:\"-\"`\n\tRetryMatch      caddyhttp.MatcherSets `json:\"-\"`\n}\n\n// Selector selects an available upstream from the pool.\ntype Selector interface {\n\tSelect(UpstreamPool, *http.Request, http.ResponseWriter) *Upstream\n}\n\n// UpstreamSource gets the list of upstreams that can be used when\n// proxying a request. Returned upstreams will be load balanced and\n// health-checked. This should be a very fast function -- instant\n// if possible -- and the return value must be as stable as possible.\n// In other words, the list of upstreams should ideally not change much\n// across successive calls. If the list of upstreams changes or the\n// ordering is not stable, load balancing will suffer. This function\n// may be called during each retry, multiple times per request, and as\n// such, needs to be instantaneous. The returned slice will not be\n// modified.\ntype UpstreamSource interface {\n\tGetUpstreams(*http.Request) ([]*Upstream, error)\n}\n\n// Hop-by-hop headers. These are removed when sent to the backend.\n// As of RFC 7230, hop-by-hop headers are required to appear in the\n// Connection header field. These are the headers defined by the\n// obsoleted RFC 2616 (section 13.5.1) and are used for backward\n// compatibility.\nvar hopHeaders = []string{\n\t\"Alt-Svc\",\n\t\"Connection\",\n\t\"Proxy-Connection\", // non-standard but still sent by libcurl and rejected by e.g. google\n\t\"Keep-Alive\",\n\t\"Proxy-Authenticate\",\n\t\"Proxy-Authorization\",\n\t\"Te\",      // canonicalized version of \"TE\"\n\t\"Trailer\", // not Trailers per URL above; https://www.rfc-editor.org/errata_search.php?eid=4522\n\t\"Transfer-Encoding\",\n\t\"Upgrade\",\n}\n\n// DialError is an error that specifically occurs\n// in a call to Dial or DialContext.\ntype DialError struct{ error }\n\n// TLSTransport is implemented by transports\n// that are capable of using TLS.\ntype TLSTransport interface {\n\t// TLSEnabled returns true if the transport\n\t// has TLS enabled, false otherwise.\n\tTLSEnabled() bool\n\n\t// EnableTLS enables TLS within the transport\n\t// if it is not already, using the provided\n\t// value as a basis for the TLS config.\n\tEnableTLS(base *TLSConfig) error\n}\n\n// roundtripSucceeded is an error type that is returned if the\n// roundtrip succeeded, but an error occurred after-the-fact.\ntype roundtripSucceeded struct{ error }\n\n// bodyReadCloser is a reader that, upon closing, will return\n// its buffer to the pool and close the underlying body reader.\ntype bodyReadCloser struct {\n\tio.Reader\n\tbuf  *bytes.Buffer\n\tbody io.ReadCloser\n}\n\nfunc (brc bodyReadCloser) Close() error {\n\tbufPool.Put(brc.buf)\n\tif brc.body != nil {\n\t\treturn brc.body.Close()\n\t}\n\treturn nil\n}\n\n// bufPool is used for buffering requests and responses.\nvar bufPool = sync.Pool{\n\tNew: func() any {\n\t\treturn new(bytes.Buffer)\n\t},\n}\n\n// handleResponseContext carries some contextual information about the\n// current proxy handling.\ntype handleResponseContext struct {\n\t// handler is the active proxy handler instance, so that\n\t// routes like copy_response may inherit some config\n\t// options and have access to handler methods.\n\thandler *Handler\n\n\t// response is the actual response received from the proxy\n\t// roundtrip, to potentially be copied if a copy_response\n\t// handler is in the handle_response routes.\n\tresponse *http.Response\n\n\t// start is the time just before the proxy roundtrip was\n\t// performed, used for logging.\n\tstart time.Time\n\n\t// logger is the prepared logger which is used to write logs\n\t// with the request, duration, and selected upstream attached.\n\tlogger *zap.Logger\n\n\t// isFinalized is whether the response has been finalized,\n\t// i.e. copied and closed, to make sure that it doesn't\n\t// happen twice.\n\tisFinalized bool\n}\n\n// proxyHandleResponseContextCtxKey is the context key for the active proxy handler\n// so that handle_response routes can inherit some config options\n// from the proxy handler.\nconst proxyHandleResponseContextCtxKey caddy.CtxKey = \"reverse_proxy_handle_response_context\"\n\n// errNoUpstream occurs when there are no upstream available.\nvar errNoUpstream = fmt.Errorf(\"no upstreams available\")\n\n// Interface guards\nvar (\n\t_ caddy.Provisioner           = (*Handler)(nil)\n\t_ caddy.CleanerUpper          = (*Handler)(nil)\n\t_ caddyhttp.MiddlewareHandler = (*Handler)(nil)\n)\n\n// Path: modules/caddyhttp/reverseproxy/forwardauth/caddyfile.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage forwardauth\n\nimport (\n\t\"encoding/json\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp/headers\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite\"\n)\n\nfunc init() {\n\thttpcaddyfile.RegisterDirective(\"forward_auth\", parseCaddyfile)\n}\n\n// parseCaddyfile parses the forward_auth directive, which has the same syntax\n// as the reverse_proxy directive (in fact, the reverse_proxy's directive\n// Unmarshaler is invoked by this function) but the resulting proxy is specially\n// configured for most\u2122\ufe0f auth gateways that support forward auth. The typical\n// config which looks something like this:\n//\n//\tforward_auth auth-gateway:9091 {\n//\t    uri /authenticate?redirect=https://auth.example.com\n//\t    copy_headers Remote-User Remote-Email\n//\t}\n//\n// is equivalent to a reverse_proxy directive like this:\n//\n//\treverse_proxy auth-gateway:9091 {\n//\t    method GET\n//\t    rewrite /authenticate?redirect=https://auth.example.com\n//\n//\t    header_up X-Forwarded-Method {method}\n//\t    header_up X-Forwarded-Uri {uri}\n//\n//\t    @good status 2xx\n//\t    handle_response @good {\n//\t        request_header {\n//\t            Remote-User {http.reverse_proxy.header.Remote-User}\n//\t            Remote-Email {http.reverse_proxy.header.Remote-Email}\n//\t        }\n//\t    }\n//\t}\nfunc parseCaddyfile(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error) {\n\tif !h.Next() {\n\t\treturn nil, h.ArgErr()\n\t}\n\n\t// if the user specified a matcher token, use that\n\t// matcher in a route that wraps both of our routes;\n\t// either way, strip the matcher token and pass\n\t// the remaining tokens to the unmarshaler so that\n\t// we can gain the rest of the reverse_proxy syntax\n\tuserMatcherSet, err := h.ExtractMatcherSet()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// make a new dispenser from the remaining tokens so that we\n\t// can reset the dispenser back to this point for the\n\t// reverse_proxy unmarshaler to read from it as well\n\tdispenser := h.NewFromNextSegment()\n\n\t// create the reverse proxy handler\n\trpHandler := &reverseproxy.Handler{\n\t\t// set up defaults for header_up; reverse_proxy already deals with\n\t\t// adding  the other three X-Forwarded-* headers, but for this flow,\n\t\t// we want to also send along the incoming method and URI since this\n\t\t// request will have a rewritten URI and method.\n\t\tHeaders: &headers.Handler{\n\t\t\tRequest: &headers.HeaderOps{\n\t\t\t\tSet: http.Header{\n\t\t\t\t\t\"X-Forwarded-Method\": []string{\"{http.request.method}\"},\n\t\t\t\t\t\"X-Forwarded-Uri\":    []string{\"{http.request.uri}\"},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\n\t\t// we always rewrite the method to GET, which implicitly\n\t\t// turns off sending the incoming request's body, which\n\t\t// allows later middleware handlers to consume it\n\t\tRewrite: &rewrite.Rewrite{\n\t\t\tMethod: \"GET\",\n\t\t},\n\n\t\tHandleResponse: []caddyhttp.ResponseHandler{},\n\t}\n\n\t// collect the headers to copy from the auth response\n\t// onto the original request, so they can get passed\n\t// through to a backend app\n\theadersToCopy := make(map[string]string)\n\n\t// read the subdirectives for configuring the forward_auth shortcut\n\t// NOTE: we delete the tokens as we go so that the reverse_proxy\n\t// unmarshal doesn't see these subdirectives which it cannot handle\n\tfor dispenser.Next() {\n\t\tfor dispenser.NextBlock(0) {\n\t\t\t// ignore any sub-subdirectives that might\n\t\t\t// have the same name somewhere within\n\t\t\t// the reverse_proxy passthrough tokens\n\t\t\tif dispenser.Nesting() != 1 {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// parse the forward_auth subdirectives\n\t\t\tswitch dispenser.Val() {\n\t\t\tcase \"uri\":\n\t\t\t\tif !dispenser.NextArg() {\n\t\t\t\t\treturn nil, dispenser.ArgErr()\n\t\t\t\t}\n\t\t\t\trpHandler.Rewrite.URI = dispenser.Val()\n\t\t\t\tdispenser.DeleteN(2)\n\n\t\t\tcase \"copy_headers\":\n\t\t\t\targs := dispenser.RemainingArgs()\n\t\t\t\thadBlock := false\n\t\t\t\tfor nesting := dispenser.Nesting(); dispenser.NextBlock(nesting); {\n\t\t\t\t\thadBlock = true\n\t\t\t\t\targs = append(args, dispenser.Val())\n\t\t\t\t}\n\n\t\t\t\t// directive name + args\n\t\t\t\tdispenser.DeleteN(len(args) + 1)\n\t\t\t\tif hadBlock {\n\t\t\t\t\t// opening & closing brace\n\t\t\t\t\tdispenser.DeleteN(2)\n\t\t\t\t}\n\n\t\t\t\tfor _, headerField := range args {\n\t\t\t\t\tif strings.Contains(headerField, \">\") {\n\t\t\t\t\t\tparts := strings.Split(headerField, \">\")\n\t\t\t\t\t\theadersToCopy[parts[0]] = parts[1]\n\t\t\t\t\t} else {\n\t\t\t\t\t\theadersToCopy[headerField] = headerField\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif len(headersToCopy) == 0 {\n\t\t\t\t\treturn nil, dispenser.ArgErr()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// reset the dispenser after we're done so that the reverse_proxy\n\t// unmarshaler can read it from the start\n\tdispenser.Reset()\n\n\t// the auth target URI must not be empty\n\tif rpHandler.Rewrite.URI == \"\" {\n\t\treturn nil, dispenser.Errf(\"the 'uri' subdirective is required\")\n\t}\n\n\t// set up handler for good responses; when a response\n\t// has 2xx status, then we will copy some headers from\n\t// the response onto the original request, and allow\n\t// handling to continue down the middleware chain,\n\t// by _not_ executing a terminal handler.\n\tgoodResponseHandler := caddyhttp.ResponseHandler{\n\t\tMatch: &caddyhttp.ResponseMatcher{\n\t\t\tStatusCode: []int{2},\n\t\t},\n\t\tRoutes: []caddyhttp.Route{},\n\t}\n\n\thandler := &headers.Handler{\n\t\tRequest: &headers.HeaderOps{\n\t\t\tSet: http.Header{},\n\t\t},\n\t}\n\n\t// the list of headers to copy may be empty, but that's okay; we\n\t// need at least one handler in the routes for the response handling\n\t// logic in reverse_proxy to not skip this entry as empty.\n\tfor from, to := range headersToCopy {\n\t\thandler.Request.Set.Set(to, \"{http.reverse_proxy.header.\"+http.CanonicalHeaderKey(from)+\"}\")\n\t}\n\n\tgoodResponseHandler.Routes = append(\n\t\tgoodResponseHandler.Routes,\n\t\tcaddyhttp.Route{\n\t\t\tHandlersRaw: []json.RawMessage{caddyconfig.JSONModuleObject(\n\t\t\t\thandler,\n\t\t\t\t\"handler\",\n\t\t\t\t\"headers\",\n\t\t\t\tnil,\n\t\t\t)},\n\t\t},\n\t)\n\n\t// note that when a response has any other status than 2xx, then we\n\t// use the reverse proxy's default behaviour of copying the response\n\t// back to the client, so we don't need to explicitly add a response\n\t// handler specifically for that behaviour; we do need the 2xx handler\n\t// though, to make handling fall through to handlers deeper in the chain.\n\trpHandler.HandleResponse = append(rpHandler.HandleResponse, goodResponseHandler)\n\n\t// the rest of the config is specified by the user\n\t// using the reverse_proxy directive syntax\n\tdispenser.Next() // consume the directive name\n\terr = rpHandler.UnmarshalCaddyfile(dispenser)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = rpHandler.FinalizeUnmarshalCaddyfile(h)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// create the final reverse proxy route\n\trpRoute := caddyhttp.Route{\n\t\tHandlersRaw: []json.RawMessage{caddyconfig.JSONModuleObject(\n\t\t\trpHandler,\n\t\t\t\"handler\",\n\t\t\t\"reverse_proxy\",\n\t\t\tnil,\n\t\t)},\n\t}\n\n\t// apply the user's matcher if any\n\tif userMatcherSet != nil {\n\t\trpRoute.MatcherSetsRaw = []caddy.ModuleMap{userMatcherSet}\n\t}\n\n\treturn []httpcaddyfile.ConfigValue{\n\t\t{\n\t\t\tClass: \"route\",\n\t\t\tValue: rpRoute,\n\t\t},\n\t}, nil\n}\n\n// Path: modules/caddyhttp/templates/caddyfile.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage templates\n\nimport (\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc init() {\n\thttpcaddyfile.RegisterHandlerDirective(\"templates\", parseCaddyfile)\n}\n\n// parseCaddyfile sets up the handler from Caddyfile tokens. Syntax:\n//\n//\ttemplates [<matcher>] {\n//\t    mime <types...>\n//\t    between <open_delim> <close_delim>\n//\t    root <path>\n//\t}\nfunc parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {\n\th.Next() // consume directive name\n\tt := new(Templates)\n\tfor h.NextBlock(0) {\n\t\tswitch h.Val() {\n\t\tcase \"mime\":\n\t\t\tt.MIMETypes = h.RemainingArgs()\n\t\t\tif len(t.MIMETypes) == 0 {\n\t\t\t\treturn nil, h.ArgErr()\n\t\t\t}\n\t\tcase \"between\":\n\t\t\tt.Delimiters = h.RemainingArgs()\n\t\t\tif len(t.Delimiters) != 2 {\n\t\t\t\treturn nil, h.ArgErr()\n\t\t\t}\n\t\tcase \"root\":\n\t\t\tif !h.Args(&t.FileRoot) {\n\t\t\t\treturn nil, h.ArgErr()\n\t\t\t}\n\t\tcase \"extensions\":\n\t\t\tif h.NextArg() {\n\t\t\t\treturn nil, h.ArgErr()\n\t\t\t}\n\t\t\tif t.ExtensionsRaw != nil {\n\t\t\t\treturn nil, h.Err(\"extensions already specified\")\n\t\t\t}\n\t\t\tfor nesting := h.Nesting(); h.NextBlock(nesting); {\n\t\t\t\textensionModuleName := h.Val()\n\t\t\t\tmodID := \"http.handlers.templates.functions.\" + extensionModuleName\n\t\t\t\tunm, err := caddyfile.UnmarshalModule(h.Dispenser, modID)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, err\n\t\t\t\t}\n\t\t\t\tcf, ok := unm.(CustomFunctions)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil, h.Errf(\"module %s (%T) does not provide template functions\", modID, unm)\n\t\t\t\t}\n\t\t\t\tif t.ExtensionsRaw == nil {\n\t\t\t\t\tt.ExtensionsRaw = make(caddy.ModuleMap)\n\t\t\t\t}\n\t\t\t\tt.ExtensionsRaw[extensionModuleName] = caddyconfig.JSON(cf, nil)\n\t\t\t}\n\t\t}\n\t}\n\treturn t, nil\n}\n\n// Path: modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage fastcgi\n\nimport (\n\t\"encoding/json\"\n\t\"net/http\"\n\t\"strconv\"\n\n...\n", "needle_token_start": 8213, "needle_token_end": 8539, "code_context_ntokens": 15310, "output": ["```go\nfunc cloneRequest(origReq *http.Request) *http.Request {\n\treq := new(http.Request)\n\t*req = *origReq\n\tif origReq.URL != nil {\n\t\tnewURL := new(url.URL)\n\t\t*newURL = *origReq.URL\n\t\tif origReq.URL.User != nil {\n\t\t\tnewURL.User = new(url.Userinfo)\n\t\t\t*newURL.User = *origReq.URL.User\n\t\t}\n\t\t// sanitize the request URL; we expect it to not contain the\n\t\t// scheme and host since those should be determined by r.TLS\n\t\t// and r.Host respectively, but some clients may include it\n\t\t// in the request-line, which is technically valid in HTTP,\n\t\t// but breaks reverseproxy behaviour, overriding how the\n\t\t// dialer will behave. See #4237 for context.\n\t\tnewURL.Scheme = \"\"\n\t\tnewURL.Host = \"\"\n\t\treq.URL = newURL\n\t}\n\tif origReq.Header != nil {\n\t\treq.Header = origReq.Header.Clone()\n\t}\n\tif origReq.Trailer != nil {\n\t\treq.Trailer = origReq.Trailer.Clone()\n\t}\n\treturn req\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "asciiEqualFold", "language": "go", "path": "modules/caddyhttp/reverseproxy/ascii.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To determine if two strings are equal, ignoring the case differences but only for ASCII characters.\n2. **Input**: Two strings.\n3. **Output**: A boolean value; `true` if the strings are equal ignoring case for ASCII characters, otherwise `false`.\n4. **Procedure**: The function first checks if the lengths of both strings are the same. If not, it returns `false`. If they are the same, it iterates through each character of the strings, converting each character to its lowercase ASCII equivalent and comparing them. If all corresponding characters match, it returns `true`; if any character does not match, it returns `false`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: modules/caddyhttp/responsewriter.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n)\n\n// ResponseWriterWrapper wraps an underlying ResponseWriter and\n// promotes its Pusher method as well. To use this type, embed\n// a pointer to it within your own struct type that implements\n// the http.ResponseWriter interface, then call methods on the\n// embedded value.\ntype ResponseWriterWrapper struct {\n\thttp.ResponseWriter\n}\n\n// Push implements http.Pusher. It simply calls the underlying\n// ResponseWriter's Push method if there is one, or returns\n// ErrNotImplemented otherwise.\nfunc (rww *ResponseWriterWrapper) Push(target string, opts *http.PushOptions) error {\n\tif pusher, ok := rww.ResponseWriter.(http.Pusher); ok {\n\t\treturn pusher.Push(target, opts)\n\t}\n\treturn ErrNotImplemented\n}\n\n// ReadFrom implements io.ReaderFrom. It simply calls io.Copy,\n// which uses io.ReaderFrom if available.\nfunc (rww *ResponseWriterWrapper) ReadFrom(r io.Reader) (n int64, err error) {\n\treturn io.Copy(rww.ResponseWriter, r)\n}\n\n// Unwrap returns the underlying ResponseWriter, necessary for\n// http.ResponseController to work correctly.\nfunc (rww *ResponseWriterWrapper) Unwrap() http.ResponseWriter {\n\treturn rww.ResponseWriter\n}\n\n// ErrNotImplemented is returned when an underlying\n// ResponseWriter does not implement the required method.\nvar ErrNotImplemented = fmt.Errorf(\"method not implemented\")\n\ntype responseRecorder struct {\n\t*ResponseWriterWrapper\n\tstatusCode   int\n\tbuf          *bytes.Buffer\n\tshouldBuffer ShouldBufferFunc\n\tsize         int\n\twroteHeader  bool\n\tstream       bool\n\n\treadSize *int\n}\n\n// NewResponseRecorder returns a new ResponseRecorder that can be\n// used instead of a standard http.ResponseWriter. The recorder is\n// useful for middlewares which need to buffer a response and\n// potentially process its entire body before actually writing the\n// response to the underlying writer. Of course, buffering the entire\n// body has a memory overhead, but sometimes there is no way to avoid\n// buffering the whole response, hence the existence of this type.\n// Still, if at all practical, handlers should strive to stream\n// responses by wrapping Write and WriteHeader methods instead of\n// buffering whole response bodies.\n//\n// Buffering is actually optional. The shouldBuffer function will\n// be called just before the headers are written. If it returns\n// true, the headers and body will be buffered by this recorder\n// and not written to the underlying writer; if false, the headers\n// will be written immediately and the body will be streamed out\n// directly to the underlying writer. If shouldBuffer is nil,\n// the response will never be buffered and will always be streamed\n// directly to the writer.\n//\n// You can know if shouldBuffer returned true by calling Buffered().\n//\n// The provided buffer buf should be obtained from a pool for best\n// performance (see the sync.Pool type).\n//\n// Proper usage of a recorder looks like this:\n//\n//\trec := caddyhttp.NewResponseRecorder(w, buf, shouldBuffer)\n//\terr := next.ServeHTTP(rec, req)\n//\tif err != nil {\n//\t    return err\n//\t}\n//\tif !rec.Buffered() {\n//\t    return nil\n//\t}\n//\t// process the buffered response here\n//\n// The header map is not buffered; i.e. the ResponseRecorder's Header()\n// method returns the same header map of the underlying ResponseWriter.\n// This is a crucial design decision to allow HTTP trailers to be\n// flushed properly (https://github.com/caddyserver/caddy/issues/3236).\n//\n// Once you are ready to write the response, there are two ways you can\n// do it. The easier way is to have the recorder do it:\n//\n//\trec.WriteResponse()\n//\n// This writes the recorded response headers as well as the buffered body.\n// Or, you may wish to do it yourself, especially if you manipulated the\n// buffered body. First you will need to write the headers with the\n// recorded status code, then write the body (this example writes the\n// recorder's body buffer, but you might have your own body to write\n// instead):\n//\n//\tw.WriteHeader(rec.Status())\n//\tio.Copy(w, rec.Buffer())\n//\n// As a special case, 1xx responses are not buffered nor recorded\n// because they are not the final response; they are passed through\n// directly to the underlying ResponseWriter.\nfunc NewResponseRecorder(w http.ResponseWriter, buf *bytes.Buffer, shouldBuffer ShouldBufferFunc) ResponseRecorder {\n\treturn &responseRecorder{\n\t\tResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: w},\n\t\tbuf:                   buf,\n\t\tshouldBuffer:          shouldBuffer,\n\t}\n}\n\n// WriteHeader writes the headers with statusCode to the wrapped\n// ResponseWriter unless the response is to be buffered instead.\n// 1xx responses are never buffered.\nfunc (rr *responseRecorder) WriteHeader(statusCode int) {\n\tif rr.wroteHeader {\n\t\treturn\n\t}\n\n\t// save statusCode always, in case HTTP middleware upgrades websocket\n\t// connections by manually setting headers and writing status 101\n\trr.statusCode = statusCode\n\n\t// 1xx responses aren't final; just informational\n\tif statusCode < 100 || statusCode > 199 {\n\t\trr.wroteHeader = true\n\n\t\t// decide whether we should buffer the response\n\t\tif rr.shouldBuffer == nil {\n\t\t\trr.stream = true\n\t\t} else {\n\t\t\trr.stream = !rr.shouldBuffer(rr.statusCode, rr.ResponseWriterWrapper.Header())\n\t\t}\n\t}\n\n\t// if informational or not buffered, immediately write header\n\tif rr.stream || (100 <= statusCode && statusCode <= 199) {\n\t\trr.ResponseWriterWrapper.WriteHeader(statusCode)\n\t}\n}\n\nfunc (rr *responseRecorder) Write(data []byte) (int, error) {\n\trr.WriteHeader(http.StatusOK)\n\tvar n int\n\tvar err error\n\tif rr.stream {\n\t\tn, err = rr.ResponseWriterWrapper.Write(data)\n\t} else {\n\t\tn, err = rr.buf.Write(data)\n\t}\n\n\trr.size += n\n\treturn n, err\n}\n\n...\n// Path: modules/caddyhttp/responsewriter_test.go\npackage caddyhttp\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n\t\"testing\"\n)\n\ntype responseWriterSpy interface {\n\thttp.ResponseWriter\n\tWritten() string\n\tCalledReadFrom() bool\n}\n\nvar (\n\t_ responseWriterSpy = (*baseRespWriter)(nil)\n\t_ responseWriterSpy = (*readFromRespWriter)(nil)\n)\n\n// a barebones http.ResponseWriter mock\ntype baseRespWriter []byte\n\nfunc (brw *baseRespWriter) Write(d []byte) (int, error) {\n\t*brw = append(*brw, d...)\n\treturn len(d), nil\n}\nfunc (brw *baseRespWriter) Header() http.Header        { return nil }\nfunc (brw *baseRespWriter) WriteHeader(statusCode int) {}\nfunc (brw *baseRespWriter) Written() string            { return string(*brw) }\nfunc (brw *baseRespWriter) CalledReadFrom() bool       { return false }\n\n// an http.ResponseWriter mock that supports ReadFrom\ntype readFromRespWriter struct {\n\tbaseRespWriter\n\tcalled bool\n}\n\nfunc (rf *readFromRespWriter) ReadFrom(r io.Reader) (int64, error) {\n\trf.called = true\n\treturn io.Copy(&rf.baseRespWriter, r)\n}\n\nfunc (rf *readFromRespWriter) CalledReadFrom() bool { return rf.called }\n\nfunc TestResponseWriterWrapperReadFrom(t *testing.T) {\n\ttests := map[string]struct {\n\t\tresponseWriter responseWriterSpy\n\t\twantReadFrom   bool\n\t}{\n\t\t\"no ReadFrom\": {\n\t\t\tresponseWriter: &baseRespWriter{},\n\t\t\twantReadFrom:   false,\n\t\t},\n\t\t\"has ReadFrom\": {\n\t\t\tresponseWriter: &readFromRespWriter{},\n\t\t\twantReadFrom:   true,\n\t\t},\n\t}\n\tfor name, tt := range tests {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\t// what we expect middlewares to do:\n\t\t\ttype myWrapper struct {\n\t\t\t\t*ResponseWriterWrapper\n\t\t\t}\n\n\t\t\twrapped := myWrapper{\n\t\t\t\tResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: tt.responseWriter},\n\t\t\t}\n\n\t\t\tconst srcData = \"boo!\"\n\t\t\t// hides everything but Read, since strings.Reader implements WriteTo it would\n\t\t\t// take precedence over our ReadFrom.\n\t\t\tsrc := struct{ io.Reader }{strings.NewReader(srcData)}\n\n\t\t\tif _, err := io.Copy(wrapped, src); err != nil {\n\t\t\t\tt.Errorf(\"%s: Copy() err = %v\", name, err)\n\t\t\t}\n\n\t\t\tif got := tt.responseWriter.Written(); got != srcData {\n\t\t\t\tt.Errorf(\"%s: data = %q, want %q\", name, got, srcData)\n\t\t\t}\n\n\t\t\tif tt.responseWriter.CalledReadFrom() != tt.wantReadFrom {\n\t\t\t\tif tt.wantReadFrom {\n\t\t\t\t\tt.Errorf(\"%s: ReadFrom() should have been called\", name)\n\t\t\t\t} else {\n\t\t\t\t\tt.Errorf(\"%s: ReadFrom() should not have been called\", name)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestResponseWriterWrapperUnwrap(t *testing.T) {\n\tw := &ResponseWriterWrapper{&baseRespWriter{}}\n\n\tif _, ok := w.Unwrap().(*baseRespWriter); !ok {\n\t\tt.Errorf(\"Unwrap() doesn't return the underlying ResponseWriter\")\n\t}\n}\n\nfunc TestResponseRecorderReadFrom(t *testing.T) {\n\ttests := map[string]struct {\n\t\tresponseWriter responseWriterSpy\n\t\tshouldBuffer   bool\n\t\twantReadFrom   bool\n\t}{\n\t\t\"buffered plain\": {\n\t\t\tresponseWriter: &baseRespWriter{},\n\t\t\tshouldBuffer:   true,\n\t\t\twantReadFrom:   false,\n\t\t},\n\t\t\"streamed plain\": {\n\t\t\tresponseWriter: &baseRespWriter{},\n\t\t\tshouldBuffer:   false,\n\t\t\twantReadFrom:   false,\n\t\t},\n\t\t\"buffered ReadFrom\": {\n\t\t\tresponseWriter: &readFromRespWriter{},\n\t\t\tshouldBuffer:   true,\n\t\t\twantReadFrom:   false,\n\t\t},\n\t\t\"streamed ReadFrom\": {\n\t\t\tresponseWriter: &readFromRespWriter{},\n\t\t\tshouldBuffer:   false,\n\t\t\twantReadFrom:   true,\n\t\t},\n\t}\n\tfor name, tt := range tests {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tvar buf bytes.Buffer\n\n\t\t\trr := NewResponseRecorder(tt.responseWriter, &buf, func(status int, header http.Header) bool {\n\t\t\t\treturn tt.shouldBuffer\n\t\t\t})\n\n\t\t\tconst srcData = \"boo!\"\n\t\t\t// hides everything but Read, since strings.Reader implements WriteTo it would\n\t\t\t// take precedence over our ReadFrom.\n\t\t\tsrc := struct{ io.Reader }{strings.NewReader(srcData)}\n\n\t\t\tif _, err := io.Copy(rr, src); err != nil {\n\t\t\t\tt.Errorf(\"Copy() err = %v\", err)\n\t\t\t}\n\n\t\t\twantStreamed := srcData\n\t\t\twantBuffered := \"\"\n\t\t\tif tt.shouldBuffer {\n\t\t\t\twantStreamed = \"\"\n\t\t\t\twantBuffered = srcData\n\t\t\t}\n\n\t\t\tif got := tt.responseWriter.Written(); got != wantStreamed {\n\t\t\t\tt.Errorf(\"streamed data = %q, want %q\", got, wantStreamed)\n\t\t\t}\n\t\t\tif got := buf.String(); got != wantBuffered {\n\t\t\t\tt.Errorf(\"buffered data = %q, want %q\", got, wantBuffered)\n\t\t\t}\n\n\t\t\tif tt.responseWriter.CalledReadFrom() != tt.wantReadFrom {\n\t\t\t\tif tt.wantReadFrom {\n\t\t\t\t\tt.Errorf(\"ReadFrom() should have been called\")\n\t\t\t\t} else {\n\t\t\t\t\tt.Errorf(\"ReadFrom() should not have been called\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: modules/caddyhttp/reverseproxy/addresses.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverseproxy\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\ntype parsedAddr struct {\n\tnetwork, scheme, host, port string\n\tvalid                       bool\n}\n\nfunc (p parsedAddr) dialAddr() string {\n\tif !p.valid {\n\t\treturn \"\"\n\t}\n\t// for simplest possible config, we only need to include\n\t// the network portion if the user specified one\n\tif p.network != \"\" {\n\t\treturn caddy.JoinNetworkAddress(p.network, p.host, p.port)\n\t}\n\n\t// if the host is a placeholder, then we don't want to join with an empty port,\n\t// because that would just append an extra ':' at the end of the address.\n\tif p.port == \"\" && strings.Contains(p.host, \"{\") {\n\t\treturn p.host\n\t}\n\treturn net.JoinHostPort(p.host, p.port)\n}\n\nfunc (p parsedAddr) rangedPort() bool {\n\treturn strings.Contains(p.port, \"-\")\n}\n\nfunc (p parsedAddr) replaceablePort() bool {\n\treturn strings.Contains(p.port, \"{\") && strings.Contains(p.port, \"}\")\n}\n\nfunc (p parsedAddr) isUnix() bool {\n\treturn caddy.IsUnixNetwork(p.network)\n}\n\n// parseUpstreamDialAddress parses configuration inputs for\n// the dial address, including support for a scheme in front\n// as a shortcut for the port number, and a network type,\n// for example 'unix' to dial a unix socket.\nfunc parseUpstreamDialAddress(upstreamAddr string) (parsedAddr, error) {\n\tvar network, scheme, host, port string\n\n\tif strings.Contains(upstreamAddr, \"://\") {\n\t\t// we get a parsing error if a placeholder is specified\n\t\t// so we return a more user-friendly error message instead\n\t\t// to explain what to do instead\n\t\tif strings.Contains(upstreamAddr, \"{\") {\n\t\t\treturn parsedAddr{}, fmt.Errorf(\"due to parsing difficulties, placeholders are not allowed when an upstream address contains a scheme\")\n\t\t}\n\n\t\ttoURL, err := url.Parse(upstreamAddr)\n\t\tif err != nil {\n\t\t\t// if the error seems to be due to a port range,\n\t\t\t// try to replace the port range with a dummy\n\t\t\t// single port so that url.Parse() will succeed\n\t\t\tif strings.Contains(err.Error(), \"invalid port\") && strings.Contains(err.Error(), \"-\") {\n\t\t\t\tindex := strings.LastIndex(upstreamAddr, \":\")\n\t\t\t\tif index == -1 {\n\t\t\t\t\treturn parsedAddr{}, fmt.Errorf(\"parsing upstream URL: %v\", err)\n\t\t\t\t}\n\t\t\t\tportRange := upstreamAddr[index+1:]\n\t\t\t\tif strings.Count(portRange, \"-\") != 1 {\n\t\t\t\t\treturn parsedAddr{}, fmt.Errorf(\"parsing upstream URL: parse \\\"%v\\\": port range invalid: %v\", upstreamAddr, portRange)\n\t\t\t\t}\n\t\t\t\ttoURL, err = url.Parse(strings.ReplaceAll(upstreamAddr, portRange, \"0\"))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn parsedAddr{}, fmt.Errorf(\"parsing upstream URL: %v\", err)\n\t\t\t\t}\n\t\t\t\tport = portRange\n\t\t\t} else {\n\t\t\t\treturn parsedAddr{}, fmt.Errorf(\"parsing upstream URL: %v\", err)\n\t\t\t}\n\t\t}\n\t\tif port == \"\" {\n\t\t\tport = toURL.Port()\n\t\t}\n\n\t\t// there is currently no way to perform a URL rewrite between choosing\n\t\t// a backend and proxying to it, so we cannot allow extra components\n\t\t// in backend URLs\n\t\tif toURL.Path != \"\" || toURL.RawQuery != \"\" || toURL.Fragment != \"\" {\n\t\t\treturn parsedAddr{}, fmt.Errorf(\"for now, URLs for proxy upstreams only support scheme, host, and port components\")\n\t\t}\n\n\t\t// ensure the port and scheme aren't in conflict\n\t\tif toURL.Scheme == \"http\" && port == \"443\" {\n\t\t\treturn parsedAddr{}, fmt.Errorf(\"upstream address has conflicting scheme (http://) and port (:443, the HTTPS port)\")\n\t\t}\n\t\tif toURL.Scheme == \"https\" && port == \"80\" {\n\t\t\treturn parsedAddr{}, fmt.Errorf(\"upstream address has conflicting scheme (https://) and port (:80, the HTTP port)\")\n\t\t}\n\t\tif toURL.Scheme == \"h2c\" && port == \"443\" {\n\t\t\treturn parsedAddr{}, fmt.Errorf(\"upstream address has conflicting scheme (h2c://) and port (:443, the HTTPS port)\")\n\t\t}\n\n\t\t// if port is missing, attempt to infer from scheme\n\t\tif port == \"\" {\n\t\t\tswitch toURL.Scheme {\n\t\t\tcase \"\", \"http\", \"h2c\":\n\t\t\t\tport = \"80\"\n\t\t\tcase \"https\":\n\t\t\t\tport = \"443\"\n\t\t\t}\n\t\t}\n\n\t\tscheme, host = toURL.Scheme, toURL.Hostname()\n\t} else {\n\t\tvar err error\n\t\tnetwork, host, port, err = caddy.SplitNetworkAddress(upstreamAddr)\n\t\tif err != nil {\n\t\t\thost = upstreamAddr\n\t\t}\n\t\t// we can assume a port if only a hostname is specified, but use of a\n\t\t// placeholder without a port likely means a port will be filled in\n\t\tif port == \"\" && !strings.Contains(host, \"{\") && !caddy.IsUnixNetwork(network) {\n\t\t\tport = \"80\"\n\t\t}\n\t}\n\n\t// special case network to support both unix and h2c at the same time\n\tif network == \"unix+h2c\" {\n\t\tnetwork = \"unix\"\n\t\tscheme = \"h2c\"\n\t}\n\treturn parsedAddr{network, scheme, host, port, true}, nil\n}\n\n// Path: modules/caddyhttp/reverseproxy/addresses_test.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//\thttp://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverseproxy\n\nimport \"testing\"\n\nfunc TestParseUpstreamDialAddress(t *testing.T) {\n\tfor i, tc := range []struct {\n\t\tinput          string\n\t\texpectHostPort string\n\t\texpectScheme   string\n\t\texpectErr      bool\n\t}{\n\t\t{\n\t\t\tinput:          \"foo\",\n\t\t\texpectHostPort: \"foo:80\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"foo:1234\",\n\t\t\texpectHostPort: \"foo:1234\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"127.0.0.1\",\n\t\t\texpectHostPort: \"127.0.0.1:80\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"127.0.0.1:1234\",\n\t\t\texpectHostPort: \"127.0.0.1:1234\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"[::1]\",\n\t\t\texpectHostPort: \"[::1]:80\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"[::1]:1234\",\n\t\t\texpectHostPort: \"[::1]:1234\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"{foo}\",\n\t\t\texpectHostPort: \"{foo}\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"{foo}:80\",\n\t\t\texpectHostPort: \"{foo}:80\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"{foo}:{bar}\",\n\t\t\texpectHostPort: \"{foo}:{bar}\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"http://foo\",\n\t\t\texpectHostPort: \"foo:80\",\n\t\t\texpectScheme:   \"http\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"http://foo:1234\",\n\t\t\texpectHostPort: \"foo:1234\",\n\t\t\texpectScheme:   \"http\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"http://127.0.0.1\",\n\t\t\texpectHostPort: \"127.0.0.1:80\",\n\t\t\texpectScheme:   \"http\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"http://127.0.0.1:1234\",\n\t\t\texpectHostPort: \"127.0.0.1:1234\",\n\t\t\texpectScheme:   \"http\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"http://[::1]\",\n\t\t\texpectHostPort: \"[::1]:80\",\n\t\t\texpectScheme:   \"http\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"http://[::1]:80\",\n\t\t\texpectHostPort: \"[::1]:80\",\n\t\t\texpectScheme:   \"http\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"https://foo\",\n\t\t\texpectHostPort: \"foo:443\",\n\t\t\texpectScheme:   \"https\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"https://foo:1234\",\n\t\t\texpectHostPort: \"foo:1234\",\n\t\t\texpectScheme:   \"https\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"https://127.0.0.1\",\n\t\t\texpectHostPort: \"127.0.0.1:443\",\n\t\t\texpectScheme:   \"https\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"https://127.0.0.1:1234\",\n\t\t\texpectHostPort: \"127.0.0.1:1234\",\n\t\t\texpectScheme:   \"https\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"https://[::1]\",\n\t\t\texpectHostPort: \"[::1]:443\",\n\t\t\texpectScheme:   \"https\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"https://[::1]:1234\",\n\t\t\texpectHostPort: \"[::1]:1234\",\n\t\t\texpectScheme:   \"https\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"h2c://foo\",\n\t\t\texpectHostPort: \"foo:80\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"h2c://foo:1234\",\n\t\t\texpectHostPort: \"foo:1234\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"h2c://127.0.0.1\",\n\t\t\texpectHostPort: \"127.0.0.1:80\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"h2c://127.0.0.1:1234\",\n\t\t\texpectHostPort: \"127.0.0.1:1234\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"h2c://[::1]\",\n\t\t\texpectHostPort: \"[::1]:80\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"h2c://[::1]:1234\",\n\t\t\texpectHostPort: \"[::1]:1234\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"localhost:1001-1009\",\n\t\t\texpectHostPort: \"localhost:1001-1009\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"{host}:1001-1009\",\n\t\t\texpectHostPort: \"{host}:1001-1009\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"http://localhost:1001-1009\",\n\t\t\texpectHostPort: \"localhost:1001-1009\",\n\t\t\texpectScheme:   \"http\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"https://localhost:1001-1009\",\n\t\t\texpectHostPort: \"localhost:1001-1009\",\n\t\t\texpectScheme:   \"https\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix//var/php.sock\",\n\t\t\texpectHostPort: \"unix//var/php.sock\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix+h2c//var/grpc.sock\",\n\t\t\texpectHostPort: \"unix//var/grpc.sock\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix/{foo}\",\n\t\t\texpectHostPort: \"unix/{foo}\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix+h2c/{foo}\",\n\t\t\texpectHostPort: \"unix/{foo}\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix//foo/{foo}/bar\",\n\t\t\texpectHostPort: \"unix//foo/{foo}/bar\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix+h2c//foo/{foo}/bar\",\n\t\t\texpectHostPort: \"unix//foo/{foo}/bar\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://{foo}\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http:// :80\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost/path\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost?key=value\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost#fragment\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost:8001-8002-8003\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost:8001-8002/foo:bar\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost:8001-8002/foo:1\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost:8001-8002/foo:1-2\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://localhost:8001-8002#foo:1\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"http://foo:443\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"https://foo:80\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:     \"h2c://foo:443\",\n\t\t\texpectErr: true,\n\t\t},\n\t\t{\n\t\t\tinput:          `unix/c:\\absolute\\path`,\n\t\t\texpectHostPort: `unix/c:\\absolute\\path`,\n\t\t},\n\t\t{\n\t\t\tinput:          `unix+h2c/c:\\absolute\\path`,\n\t\t\texpectHostPort: `unix/c:\\absolute\\path`,\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix/c:/absolute/path\",\n\t\t\texpectHostPort: \"unix/c:/absolute/path\",\n\t\t},\n\t\t{\n\t\t\tinput:          \"unix+h2c/c:/absolute/path\",\n\t\t\texpectHostPort: \"unix/c:/absolute/path\",\n\t\t\texpectScheme:   \"h2c\",\n\t\t},\n\t} {\n\t\tactualAddr, err := parseUpstreamDialAddress(tc.input)\n\t\tif tc.expectErr && err == nil {\n\t\t\tt.Errorf(\"Test %d: Expected error but got %v\", i, err)\n\t\t}\n\t\tif !tc.expectErr && err != nil {\n\t\t\tt.Errorf(\"Test %d: Expected no error but got %v\", i, err)\n\t\t}\n\t\tif actualAddr.dialAddr() != tc.expectHostPort {\n\t\t\tt.Errorf(\"Test %d: input %s: Expected host and port '%s' but got '%s'\", i, tc.input, tc.expectHostPort, actualAddr.dialAddr())\n\t\t}\n\t\tif actualAddr.scheme != tc.expectScheme {\n\t\t\tt.Errorf(\"Test %d: Expected scheme '%s' but got '%s'\", i, tc.expectScheme, actualAddr.scheme)\n\t\t}\n\t}\n}\n\n// Path: modules/caddyhttp/reverseproxy/admin.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverseproxy\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(adminUpstreams{})\n}\n\n// adminUpstreams is a module that provides the\n// /reverse_proxy/upstreams endpoint for the Caddy admin\n// API. This allows for checking the health of configured\n// reverse proxy upstreams in the pool.\ntype adminUpstreams struct{}\n\n// upstreamStatus holds the status of a particular upstream\ntype upstreamStatus struct {\n\tAddress     string `json:\"address\"`\n\tNumRequests int    `json:\"num_requests\"`\n\tFails       int    `json:\"fails\"`\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (adminUpstreams) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"admin.api.reverse_proxy\",\n\t\tNew: func() caddy.Module { return new(adminUpstreams) },\n\t}\n}\n\n// Routes returns a route for the /reverse_proxy/upstreams endpoint.\nfunc (al adminUpstreams) Routes() []caddy.AdminRoute {\n\treturn []caddy.AdminRoute{\n\t\t{\n\t\t\tPattern: \"/reverse_proxy/upstreams\",\n\t\t\tHandler: caddy.AdminHandlerFunc(al.handleUpstreams),\n\t\t},\n\t}\n}\n\n// handleUpstreams reports the status of the reverse proxy\n// upstream pool.\nfunc (adminUpstreams) handleUpstreams(w http.ResponseWriter, r *http.Request) error {\n\tif r.Method != http.MethodGet {\n\t\treturn caddy.APIError{\n\t\t\tHTTPStatus: http.StatusMethodNotAllowed,\n\t\t\tErr:        fmt.Errorf(\"method not allowed\"),\n\t\t}\n\t}\n\n\t// Prep for a JSON response\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tenc := json.NewEncoder(w)\n\n\t// Collect the results to respond with\n\tresults := []upstreamStatus{}\n\n\t// Iterate over the upstream pool (needs to be fast)\n\tvar rangeErr error\n\thosts.Range(func(key, val any) bool {\n\t\taddress, ok := key.(string)\n\t\tif !ok {\n\t\t\trangeErr = caddy.APIError{\n\t\t\t\tHTTPStatus: http.StatusInternalServerError,\n\t\t\t\tErr:        fmt.Errorf(\"could not type assert upstream address\"),\n\t\t\t}\n\t\t\treturn false\n\t\t}\n\n\t\tupstream, ok := val.(*Host)\n\t\tif !ok {\n\t\t\trangeErr = caddy.APIError{\n\t\t\t\tHTTPStatus: http.StatusInternalServerError,\n\t\t\t\tErr:        fmt.Errorf(\"could not type assert upstream struct\"),\n\t\t\t}\n\t\t\treturn false\n\t\t}\n\n\t\tresults = append(results, upstreamStatus{\n\t\t\tAddress:     address,\n\t\t\tNumRequests: upstream.NumRequests(),\n\t\t\tFails:       upstream.Fails(),\n\t\t})\n\t\treturn true\n\t})\n\n\t// If an error happened during the range, return it\n\tif rangeErr != nil {\n\t\treturn rangeErr\n\t}\n\n\terr := enc.Encode(results)\n\tif err != nil {\n\t\treturn caddy.APIError{\n\t\t\tHTTPStatus: http.StatusInternalServerError,\n\t\t\tErr:        err,\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Path: modules/caddyhttp/reverseproxy/ascii.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Most of the code in this file was initially borrowed from the Go\n// standard library and modified; It had this copyright notice:\n// Copyright 2021 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Original source, copied because the package was marked internal:\n// https://github.com/golang/go/blob/5c489514bc5e61ad9b5b07bd7d8ec65d66a0512a/src/net/http/internal/ascii/print.go\n\npackage reverseproxy\n\n// asciiEqualFold is strings.EqualFold, ASCII only. It reports whether s and t\n// are equal, ASCII-case-insensitively.\n\nfunc asciiEqualFold(s, t string) bool {\n\tif len(s) != len(t) {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(s); i++ {\n\t\tif asciiLower(s[i]) != asciiLower(t[i]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// asciiLower returns the ASCII lowercase version of b.\nfunc asciiLower(b byte) byte {\n\tif 'A' <= b && b <= 'Z' {\n\t\treturn b + ('a' - 'A')\n\t}\n\treturn b\n}\n\n// asciiIsPrint returns whether s is ASCII and printable according to\n// https://tools.ietf.org/html/rfc20#section-4.2.\nfunc asciiIsPrint(s string) bool {\n\tfor i := 0; i < len(s); i++ {\n\t\tif s[i] < ' ' || s[i] > '~' {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Path: modules/caddyhttp/reverseproxy/ascii_test.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n// Most of the code in this file was initially borrowed from the Go\n// standard library and modified; It had this copyright notice:\n// Copyright 2021 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// Original source, copied because the package was marked internal:\n// https://github.com/golang/go/blob/5c489514bc5e61ad9b5b07bd7d8ec65d66a0512a/src/net/http/internal/ascii/print_test.go\n\npackage reverseproxy\n\nimport \"testing\"\n\nfunc TestEqualFold(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\ta, b string\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tname: \"empty\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"simple match\",\n\t\t\ta:    \"CHUNKED\",\n\t\t\tb:    \"chunked\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"same string\",\n\t\t\ta:    \"chunked\",\n\t\t\tb:    \"chunked\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"Unicode Kelvin symbol\",\n\t\t\ta:    \"chun\u212aed\", // This \"\u212a\" is 'KELVIN SIGN' (\\u212A)\n\t\t\tb:    \"chunked\",\n\t\t\twant: false,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := asciiEqualFold(tt.a, tt.b); got != tt.want {\n\t\t\t\tt.Errorf(\"AsciiEqualFold(%q,%q): got %v want %v\", tt.a, tt.b, got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestIsPrint(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tin   string\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tname: \"empty\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"ASCII low\",\n\t\t\tin:   \"This is a space: ' '\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"ASCII high\",\n\t\t\tin:   \"This is a tilde: '~'\",\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"ASCII low non-print\",\n\t\t\tin:   \"This is a unit separator: \\x1F\",\n\t\t\twant: false,\n\t\t},\n\t\t{\n\t\t\tname: \"Ascii high non-print\",\n\t\t\tin:   \"This is a Delete: \\x7F\",\n\t\t\twant: false,\n\t\t},\n\t\t{\n\t\t\tname: \"Unicode letter\",\n\t\t\tin:   \"Today it's 280\u212a outside: it's freezing!\", // This \"\u212a\" is 'KELVIN SIGN' (\\u212A)\n\t\t\twant: false,\n\t\t},\n\t\t{\n\t\t\tname: \"Unicode emoji\",\n\t\t\tin:   \"Gophers like \ud83e\uddc0\",\n\t\t\twant: false,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := asciiIsPrint(tt.in); got != tt.want {\n\t\t\t\tt.Errorf(\"IsASCIIPrint(%q): got %v want %v\", tt.in, got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: modules/caddyhttp/reverseproxy/command.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverseproxy\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/spf13/cobra\"\n\t\"go.uber.org/zap\"\n\n\tcaddycmd \"github.com/caddyserver/caddy/v2/cmd\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp/headers\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddytls\"\n)\n\nfunc init() {\n\tcaddycmd.RegisterCommand(caddycmd.Command{\n\t\tName:  \"reverse-proxy\",\n\t\tUsage: `[--from <addr>] [--to <addr>] [--change-host-header] [--insecure] [--internal-certs] [--disable-redirects] [--header-up \"Field: value\"] [--header-down \"Field: value\"] [--access-log] [--debug]`,\n\t\tShort: \"A quick and production-ready reverse proxy\",\n\t\tLong: `\nA simple but production-ready reverse proxy. Useful for quick deployments,\ndemos, and development.\n\nSimply shuttles HTTP(S) traffic from the --from address to the --to address.\nMultiple --to addresses may be specified by repeating the flag.\n\nUnless otherwise specified in the addresses, the --from address will be\nassumed to be HTTPS if a hostname is given, and the --to address will be\nassumed to be HTTP.\n\nIf the --from address has a host or IP, Caddy will attempt to serve the\nproxy over HTTPS with a certificate (unless overridden by the HTTP scheme\nor port).\n\nIf serving HTTPS: \n  --disable-redirects can be used to avoid binding to the HTTP port.\n  --internal-certs can be used to force issuance certs using the internal\n    CA instead of attempting to issue a public certificate.\n\nFor proxying:\n  --header-up can be used to set a request header to send to the upstream.\n  --header-down can be used to set a response header to send back to the client.\n  --change-host-header sets the Host header on the request to the address\n    of the upstream, instead of defaulting to the incoming Host header.\n\tThis is a shortcut for --header-up \"Host: {http.reverse_proxy.upstream.hostport}\".\n  --insecure disables TLS verification with the upstream. WARNING: THIS\n    DISABLES SECURITY BY NOT VERIFYING THE UPSTREAM'S CERTIFICATE.\n`,\n\t\tCobraFunc: func(cmd *cobra.Command) {\n\t\t\tcmd.Flags().StringP(\"from\", \"f\", \"localhost\", \"Address on which to receive traffic\")\n\t\t\tcmd.Flags().StringSliceP(\"to\", \"t\", []string{}, \"Upstream address(es) to which traffic should be sent\")\n\t\t\tcmd.Flags().BoolP(\"change-host-header\", \"c\", false, \"Set upstream Host header to address of upstream\")\n\t\t\tcmd.Flags().BoolP(\"insecure\", \"\", false, \"Disable TLS verification (WARNING: DISABLES SECURITY BY NOT VERIFYING TLS CERTIFICATES!)\")\n\t\t\tcmd.Flags().BoolP(\"disable-redirects\", \"r\", false, \"Disable HTTP->HTTPS redirects\")\n\t\t\tcmd.Flags().BoolP(\"internal-certs\", \"i\", false, \"Use internal CA for issuing certs\")\n\t\t\tcmd.Flags().StringSliceP(\"header-up\", \"H\", []string{}, \"Set a request header to send to the upstream (format: \\\"Field: value\\\")\")\n\t\t\tcmd.Flags().StringSliceP(\"header-down\", \"d\", []string{}, \"Set a response header to send back to the client (format: \\\"Field: value\\\")\")\n\t\t\tcmd.Flags().BoolP(\"access-log\", \"\", false, \"Enable the access log\")\n\t\t\tcmd.Flags().BoolP(\"debug\", \"v\", false, \"Enable verbose debug logs\")\n\t\t\tcmd.RunE = caddycmd.WrapCommandFuncForCobra(cmdReverseProxy)\n\t\t},\n\t})\n}\n\nfunc cmdReverseProxy(fs caddycmd.Flags) (int, error) {\n\tcaddy.TrapSignals()\n\n\tfrom := fs.String(\"from\")\n\tchangeHost := fs.Bool(\"change-host-header\")\n\tinsecure := fs.Bool(\"insecure\")\n\tdisableRedir := fs.Bool(\"disable-redirects\")\n\tinternalCerts := fs.Bool(\"internal-certs\")\n\taccessLog := fs.Bool(\"access-log\")\n\tdebug := fs.Bool(\"debug\")\n\n\thttpPort := strconv.Itoa(caddyhttp.DefaultHTTPPort)\n\thttpsPort := strconv.Itoa(caddyhttp.DefaultHTTPSPort)\n\n\tto, err := fs.GetStringSlice(\"to\")\n\tif err != nil {\n\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"invalid to flag: %v\", err)\n\t}\n\tif len(to) == 0 {\n\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"--to is required\")\n\t}\n\n\t// set up the downstream address; assume missing information from given parts\n\tfromAddr, err := httpcaddyfile.ParseAddress(from)\n\tif err != nil {\n\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"invalid downstream address %s: %v\", from, err)\n\t}\n\tif fromAddr.Path != \"\" {\n\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"paths are not allowed: %s\", from)\n\t}\n\tif fromAddr.Scheme == \"\" {\n\t\tif fromAddr.Port == httpPort || fromAddr.Host == \"\" {\n\t\t\tfromAddr.Scheme = \"http\"\n\t\t} else {\n\t\t\tfromAddr.Scheme = \"https\"\n\t\t}\n\t}\n\tif fromAddr.Port == \"\" {\n\t\tif fromAddr.Scheme == \"http\" {\n\t\t\tfromAddr.Port = httpPort\n\t\t} else if fromAddr.Scheme == \"https\" {\n\t\t\tfromAddr.Port = httpsPort\n\t\t}\n\t}\n\n\t// set up the upstream address; assume missing information from given parts\n\t// mixing schemes isn't supported, so use first defined (if available)\n\ttoAddresses := make([]string, len(to))\n\tvar toScheme string\n\tfor i, toLoc := range to {\n\t\taddr, err := parseUpstreamDialAddress(toLoc)\n\t\tif err != nil {\n\t\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"invalid upstream address %s: %v\", toLoc, err)\n\t\t}\n\t\tif addr.scheme != \"\" && toScheme == \"\" {\n\t\t\ttoScheme = addr.scheme\n\t\t}\n\t\ttoAddresses[i] = addr.dialAddr()\n\t}\n\n\t// proceed to build the handler and server\n\tht := HTTPTransport{}\n\tif toScheme == \"https\" {\n\t\tht.TLS = new(TLSConfig)\n\t\tif insecure {\n\t\t\tht.TLS.InsecureSkipVerify = true\n\t\t}\n\t}\n\n\tupstreamPool := UpstreamPool{}\n\tfor _, toAddr := range toAddresses {\n\t\tparsedAddr, err := caddy.ParseNetworkAddress(toAddr)\n\t\tif err != nil {\n\t\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"invalid upstream address %s: %v\", toAddr, err)\n\t\t}\n\n\t\tif parsedAddr.StartPort == 0 && parsedAddr.EndPort == 0 {\n\t\t\t// unix networks don't have ports\n\t\t\tupstreamPool = append(upstreamPool, &Upstream{\n\t\t\t\tDial: toAddr,\n\t\t\t})\n\t\t} else {\n\t\t\t// expand a port range into multiple upstreams\n\t\t\tfor i := parsedAddr.StartPort; i <= parsedAddr.EndPort; i++ {\n\t\t\t\tupstreamPool = append(upstreamPool, &Upstream{\n\t\t\t\t\tDial: caddy.JoinNetworkAddress(\"\", parsedAddr.Host, fmt.Sprint(i)),\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n\n\thandler := Handler{\n\t\tTransportRaw: caddyconfig.JSONModuleObject(ht, \"protocol\", \"http\", nil),\n\t\tUpstreams:    upstreamPool,\n\t}\n\n\t// set up header_up\n\theaderUp, err := fs.GetStringSlice(\"header-up\")\n\tif err != nil {\n\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"invalid header flag: %v\", err)\n\t}\n\tif len(headerUp) > 0 {\n\t\treqHdr := make(http.Header)\n\t\tfor i, h := range headerUp {\n\t\t\tkey, val, found := strings.Cut(h, \":\")\n\t\t\tkey, val = strings.TrimSpace(key), strings.TrimSpace(val)\n\t\t\tif !found || key == \"\" || val == \"\" {\n\t\t\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"header-up %d: invalid format \\\"%s\\\" (expecting \\\"Field: value\\\")\", i, h)\n\t\t\t}\n\t\t\treqHdr.Set(key, val)\n\t\t}\n\t\thandler.Headers = &headers.Handler{\n\t\t\tRequest: &headers.HeaderOps{\n\t\t\t\tSet: reqHdr,\n\t\t\t},\n\t\t}\n\t}\n\n\t// set up header_down\n\theaderDown, err := fs.GetStringSlice(\"header-down\")\n\tif err != nil {\n\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"invalid header flag: %v\", err)\n\t}\n\tif len(headerDown) > 0 {\n\t\trespHdr := make(http.Header)\n\t\tfor i, h := range headerDown {\n\t\t\tkey, val, found := strings.Cut(h, \":\")\n\t\t\tkey, val = strings.TrimSpace(key), strings.TrimSpace(val)\n\t\t\tif !found || key == \"\" || val == \"\" {\n\t\t\t\treturn caddy.ExitCodeFailedStartup, fmt.Errorf(\"header-down %d: invalid format \\\"%s\\\" (expecting \\\"Field: value\\\")\", i, h)\n\t\t\t}\n\t\t\trespHdr.Set(key, val)\n\t\t}\n\t\tif handler.Headers == nil {\n\t\t\thandler.Headers = &headers.Handler{}\n\t\t}\n\t\thandler.Headers.Response = &headers.RespHeaderOps{\n\t\t\tHeaderOps: &headers.HeaderOps{\n\t\t\t\tSet: respHdr,\n\t\t\t},\n\t\t}\n\t}\n\n\tif changeHost {\n\t\tif handler.Headers == nil {\n\t\t\thandler.Headers = &headers.Handler{\n\t\t\t\tRequest: &headers.HeaderOps{\n\t\t\t\t\tSet: http.Header{},\n\t\t\t\t},\n\t\t\t}\n\t\t}\n\t\thandler.Headers.Request.Set.Set(\"Host\", \"{http.reverse_proxy.upstream.hostport}\")\n\t}\n\n\troute := caddyhttp.Route{\n\t\tHandlersRaw: []json.RawMessage{\n\t\t\tcaddyconfig.JSONModuleObject(handler, \"handler\", \"reverse_proxy\", nil),\n\t\t},\n\t}\n\tif fromAddr.Host != \"\" {\n\t\troute.MatcherSetsRaw = []caddy.ModuleMap{\n\t\t\t{\n\t\t\t\t\"host\": caddyconfig.JSON(caddyhttp.MatchHost{fromAddr.Host}, nil),\n\t\t\t},\n\t\t}\n\t}\n\n\tserver := &caddyhttp.Server{\n\t\tRoutes: caddyhttp.RouteList{route},\n\t\tListen: []string{\":\" + fromAddr.Port},\n\t}\n\tif accessLog {\n\t\tserver.Logs = &caddyhttp.ServerLogConfig{}\n\t}\n\n\tif fromAddr.Scheme == \"http\" {\n\t\tserver.AutoHTTPS = &caddyhttp.AutoHTTPSConfig{Disabled: true}\n\t} else if disableRedir {\n\t\tserver.AutoHTTPS = &caddyhttp.AutoHTTPSConfig{DisableRedir: true}\n\t}\n\n\thttpApp := caddyhttp.App{\n\t\tServers: map[string]*caddyhttp.Server{\"proxy\": server},\n\t}\n\n\tappsRaw := caddy.ModuleMap{\n\t\t\"http\": caddyconfig.JSON(httpApp, nil),\n\t}\n\tif internalCerts && fromAddr.Host != \"\" {\n\t\ttlsApp := caddytls.TLS{\n\t\t\tAutomation: &caddytls.AutomationConfig{\n\t\t\t\tPolicies: []*caddytls.AutomationPolicy{{\n\t\t\t\t\tSubjectsRaw: []string{fromAddr.Host},\n\t\t\t\t\tIssuersRaw:  []json.RawMessage{json.RawMessage(`{\"module\":\"internal\"}`)},\n\t\t\t\t}},\n\t\t\t},\n\t\t}\n\t\tappsRaw[\"tls\"] = caddyconfig.JSON(tlsApp, nil)\n\t}\n\n\tvar false bool\n\tcfg := &caddy.Config{\n\t\tAdmin: &caddy.AdminConfig{\n\t\t\tDisabled: true,\n\t\t\tConfig: &caddy.ConfigSettings{\n\t\t\t\tPersist: &false,\n\t\t\t},\n\t\t},\n\t\tAppsRaw: appsRaw,\n\t}\n\n\tif debug {\n\t\tcfg.Logging = &caddy.Logging{\n\t\t\tLogs: map[string]*caddy.CustomLog{\n\t\t\t\t\"default\": {BaseLog: caddy.BaseLog{Level: zap.DebugLevel.CapitalString()}},\n\t\t\t},\n\t\t}\n\t}\n\n\terr = caddy.Run(cfg)\n\tif err != nil {\n\t\treturn caddy.ExitCodeFailedStartup, err\n\t}\n\n\tcaddy.Log().Info(\"caddy proxying\", zap.String(\"from\", fromAddr.String()), zap.Strings(\"to\", toAddresses))\n\tif len(toAddresses) > 1 {\n\t\tcaddy.Log().Info(\"using default load balancing policy\", zap.String(\"policy\", \"random\"))\n\t}\n\n\tselect {}\n}\n\n...\n", "needle_token_start": 9899, "needle_token_end": 9992, "code_context_ntokens": 15298, "output": ["```go\n// asciiEqualFold is strings.EqualFold, ASCII only. It reports whether s and t\n// are equal, ASCII-case-insensitively.\nfunc asciiEqualFold(s, t string) bool {\n\tif len(s) != len(t) {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(s); i++ {\n\t\tif asciiLower(s[i]) != asciiLower(t[i]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// asciiLower returns the ASCII lowercase version of b.\nfunc asciiLower(b byte) byte {\n\tif 'A' <= b && b <= 'Z' {\n\t\treturn b + ('a' - 'A')\n\t}\n\treturn b\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "DefaultIssuers", "language": "go", "path": "modules/caddytls/automation.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To generate a list of issuer configurations for certificate management, primarily used when no specific issuer details are provided.\n2. **Input**: A string representing the user's email address.\n3. **Output**: A list of issuer objects configured for certificate issuance, potentially including an email address if provided.\n4. **Procedure**: The function initializes a list with a default issuer object. If the provided email address is not empty, it appends another issuer object configured with a specific Certificate Authority and the provided email.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: modules/caddytls/acmeissuer.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddytls\n\nimport (\n\t\"context\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/caddyserver/certmagic\"\n\t\"github.com/caddyserver/zerossl\"\n\t\"github.com/mholt/acmez/v2/acme\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(ACMEIssuer{})\n}\n\n// ACMEIssuer manages certificates using the ACME protocol (RFC 8555).\ntype ACMEIssuer struct {\n\t// The URL to the CA's ACME directory endpoint. Default:\n\t// https://acme-v02.api.letsencrypt.org/directory\n\tCA string `json:\"ca,omitempty\"`\n\n\t// The URL to the test CA's ACME directory endpoint.\n\t// This endpoint is only used during retries if there\n\t// is a failure using the primary CA. Default:\n\t// https://acme-staging-v02.api.letsencrypt.org/directory\n\tTestCA string `json:\"test_ca,omitempty\"`\n\n\t// Your email address, so the CA can contact you if necessary.\n\t// Not required, but strongly recommended to provide one so\n\t// you can be reached if there is a problem. Your email is\n\t// not sent to any Caddy mothership or used for any purpose\n\t// other than ACME transactions.\n\tEmail string `json:\"email,omitempty\"`\n\n\t// If you have an existing account with the ACME server, put\n\t// the private key here in PEM format. The ACME client will\n\t// look up your account information with this key first before\n\t// trying to create a new one. You can use placeholders here,\n\t// for example if you have it in an environment variable.\n\tAccountKey string `json:\"account_key,omitempty\"`\n\n\t// If using an ACME CA that requires an external account\n\t// binding, specify the CA-provided credentials here.\n\tExternalAccount *acme.EAB `json:\"external_account,omitempty\"`\n\n\t// Time to wait before timing out an ACME operation.\n\t// Default: 0 (no timeout)\n\tACMETimeout caddy.Duration `json:\"acme_timeout,omitempty\"`\n\n\t// Configures the various ACME challenge types.\n\tChallenges *ChallengesConfig `json:\"challenges,omitempty\"`\n\n\t// An array of files of CA certificates to accept when connecting to the\n\t// ACME CA. Generally, you should only use this if the ACME CA endpoint\n\t// is internal or for development/testing purposes.\n\tTrustedRootsPEMFiles []string `json:\"trusted_roots_pem_files,omitempty\"`\n\n\t// Preferences for selecting alternate certificate chains, if offered\n\t// by the CA. By default, the first offered chain will be selected.\n\t// If configured, the chains may be sorted and the first matching chain\n\t// will be selected.\n\tPreferredChains *ChainPreference `json:\"preferred_chains,omitempty\"`\n\n\t// The validity period to ask the CA to issue a certificate for.\n\t// Default: 0 (CA chooses lifetime).\n\t// This value is used to compute the \"notAfter\" field of the ACME order;\n\t// therefore the system must have a reasonably synchronized clock.\n\t// NOTE: Not all CAs support this. Check with your CA's ACME\n\t// documentation to see if this is allowed and what values may\n\t// be used. EXPERIMENTAL: Subject to change.\n\tCertificateLifetime caddy.Duration `json:\"certificate_lifetime,omitempty\"`\n\n\trootPool *x509.CertPool\n\tlogger   *zap.Logger\n\n\ttemplate certmagic.ACMEIssuer  // set at Provision\n\tmagic    *certmagic.Config     // set at PreCheck\n\tissuer   *certmagic.ACMEIssuer // set at PreCheck; result of template + magic\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (ACMEIssuer) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"tls.issuance.acme\",\n\t\tNew: func() caddy.Module { return new(ACMEIssuer) },\n\t}\n}\n\n// Provision sets up iss.\nfunc (iss *ACMEIssuer) Provision(ctx caddy.Context) error {\n\tiss.logger = ctx.Logger()\n\n\trepl := caddy.NewReplacer()\n\n\t// expand email address, if non-empty\n\tif iss.Email != \"\" {\n\t\temail, err := repl.ReplaceOrErr(iss.Email, true, true)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"expanding email address '%s': %v\", iss.Email, err)\n\t\t}\n\t\tiss.Email = email\n\t}\n\n\t// expand account key, if non-empty\n\tif iss.AccountKey != \"\" {\n\t\taccountKey, err := repl.ReplaceOrErr(iss.AccountKey, true, true)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"expanding account key PEM '%s': %v\", iss.AccountKey, err)\n\t\t}\n\t\tiss.AccountKey = accountKey\n\t}\n\n\t// DNS providers\n\tif iss.Challenges != nil && iss.Challenges.DNS != nil && iss.Challenges.DNS.ProviderRaw != nil {\n\t\tval, err := ctx.LoadModule(iss.Challenges.DNS, \"ProviderRaw\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"loading DNS provider module: %v\", err)\n\t\t}\n\t\tiss.Challenges.DNS.solver = &certmagic.DNS01Solver{\n\t\t\tDNSManager: certmagic.DNSManager{\n\t\t\t\tDNSProvider:        val.(certmagic.DNSProvider),\n\t\t\t\tTTL:                time.Duration(iss.Challenges.DNS.TTL),\n\t\t\t\tPropagationDelay:   time.Duration(iss.Challenges.DNS.PropagationDelay),\n\t\t\t\tPropagationTimeout: time.Duration(iss.Challenges.DNS.PropagationTimeout),\n\t\t\t\tResolvers:          iss.Challenges.DNS.Resolvers,\n\t\t\t\tOverrideDomain:     iss.Challenges.DNS.OverrideDomain,\n\t\t\t},\n\t\t}\n\t}\n\n\t// add any custom CAs to trust store\n\tif len(iss.TrustedRootsPEMFiles) > 0 {\n\t\tiss.rootPool = x509.NewCertPool()\n\t\tfor _, pemFile := range iss.TrustedRootsPEMFiles {\n\t\t\tpemData, err := os.ReadFile(pemFile)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"loading trusted root CA's PEM file: %s: %v\", pemFile, err)\n\t\t\t}\n\t\t\tif !iss.rootPool.AppendCertsFromPEM(pemData) {\n\t\t\t\treturn fmt.Errorf(\"unable to add %s to trust pool: %v\", pemFile, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tvar err error\n\tiss.template, err = iss.makeIssuerTemplate()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (iss *ACMEIssuer) makeIssuerTemplate() (certmagic.ACMEIssuer, error) {\n\ttemplate := certmagic.ACMEIssuer{\n\t\tCA:                iss.CA,\n\t\tTestCA:            iss.TestCA,\n\t\tEmail:             iss.Email,\n\t\tAccountKeyPEM:     iss.AccountKey,\n\t\tCertObtainTimeout: time.Duration(iss.ACMETimeout),\n\t\tTrustedRoots:      iss.rootPool,\n\t\tExternalAccount:   iss.ExternalAccount,\n\t\tNotAfter:          time.Duration(iss.CertificateLifetime),\n\t\tLogger:            iss.logger,\n\t}\n\n\tif iss.Challenges != nil {\n\t\tif iss.Challenges.HTTP != nil {\n\t\t\ttemplate.DisableHTTPChallenge = iss.Challenges.HTTP.Disabled\n\t\t\ttemplate.AltHTTPPort = iss.Challenges.HTTP.AlternatePort\n\t\t}\n\t\tif iss.Challenges.TLSALPN != nil {\n\t\t\ttemplate.DisableTLSALPNChallenge = iss.Challenges.TLSALPN.Disabled\n\t\t\ttemplate.AltTLSALPNPort = iss.Challenges.TLSALPN.AlternatePort\n\t\t}\n\t\tif iss.Challenges.DNS != nil {\n\t\t\ttemplate.DNS01Solver = iss.Challenges.DNS.solver\n\t\t}\n\t\ttemplate.ListenHost = iss.Challenges.BindHost\n\t}\n\n\tif iss.PreferredChains != nil {\n\t\ttemplate.PreferredChains = certmagic.ChainPreference{\n\t\t\tSmallest:       iss.PreferredChains.Smallest,\n\t\t\tAnyCommonName:  iss.PreferredChains.AnyCommonName,\n\t\t\tRootCommonName: iss.PreferredChains.RootCommonName,\n\t\t}\n\t}\n\n\t// ZeroSSL requires EAB, but we can generate that automatically (requires an email address be configured)\n\tif strings.HasPrefix(iss.CA, \"https://acme.zerossl.com/\") {\n\t\ttemplate.NewAccountFunc = func(ctx context.Context, acmeIss *certmagic.ACMEIssuer, acct acme.Account) (acme.Account, error) {\n\t\t\tif acmeIss.ExternalAccount != nil {\n\t\t\t\treturn acct, nil\n\t\t\t}\n\t\t\tvar err error\n\t\t\tacmeIss.ExternalAccount, acct, err = iss.generateZeroSSLEABCredentials(ctx, acct)\n\t\t\treturn acct, err\n\t\t}\n\t}\n\n\treturn template, nil\n}\n\n// SetConfig sets the associated certmagic config for this issuer.\n// This is required because ACME needs values from the config in\n// order to solve the challenges during issuance. This implements\n// the ConfigSetter interface.\nfunc (iss *ACMEIssuer) SetConfig(cfg *certmagic.Config) {\n\tiss.magic = cfg\n\tiss.issuer = certmagic.NewACMEIssuer(cfg, iss.template)\n}\n\n// PreCheck implements the certmagic.PreChecker interface.\nfunc (iss *ACMEIssuer) PreCheck(ctx context.Context, names []string, interactive bool) error {\n\treturn iss.issuer.PreCheck(ctx, names, interactive)\n}\n\n// Issue obtains a certificate for the given csr.\nfunc (iss *ACMEIssuer) Issue(ctx context.Context, csr *x509.CertificateRequest) (*certmagic.IssuedCertificate, error) {\n\treturn iss.issuer.Issue(ctx, csr)\n}\n\n// IssuerKey returns the unique issuer key for the configured CA endpoint.\nfunc (iss *ACMEIssuer) IssuerKey() string {\n\treturn iss.issuer.IssuerKey()\n}\n\n// Revoke revokes the given certificate.\nfunc (iss *ACMEIssuer) Revoke(ctx context.Context, cert certmagic.CertificateResource, reason int) error {\n\treturn iss.issuer.Revoke(ctx, cert, reason)\n}\n\n// GetACMEIssuer returns iss. This is useful when other types embed ACMEIssuer, because\n// type-asserting them to *ACMEIssuer will fail, but type-asserting them to an interface\n// with only this method will succeed, and will still allow the embedded ACMEIssuer\n// to be accessed and manipulated.\nfunc (iss *ACMEIssuer) GetACMEIssuer() *ACMEIssuer { return iss }\n\n// generateZeroSSLEABCredentials generates ZeroSSL EAB credentials for the primary contact email\n// on the issuer. It should only be usedif the CA endpoint is ZeroSSL. An email address is required.\nfunc (iss *ACMEIssuer) generateZeroSSLEABCredentials(ctx context.Context, acct acme.Account) (*acme.EAB, acme.Account, error) {\n\tif strings.TrimSpace(iss.Email) == \"\" {\n\t\treturn nil, acme.Account{}, fmt.Errorf(\"your email address is required to use ZeroSSL's ACME endpoint\")\n\t}\n\n\tif len(acct.Contact) == 0 {\n\t\t// we borrow the email from config or the default email, so ensure it's saved with the account\n\t\tacct.Contact = []string{\"mailto:\" + iss.Email}\n\t}\n\n\tendpoint := zerossl.BaseURL + \"/acme/eab-credentials-email\"\n\tform := url.Values{\"email\": []string{iss.Email}}\n\tbody := strings.NewReader(form.Encode())\n\n\treq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, body)\n\tif err != nil {\n\t\treturn nil, acct, fmt.Errorf(\"forming request: %v\", err)\n\t}\n\treq.Header.Set(\"Content-Type\", \"application/x-www-form-urlencoded\")\n\treq.Header.Set(\"User-Agent\", certmagic.UserAgent)\n\n\tresp, err := http.DefaultClient.Do(req)\n\tif err != nil {\n\t\treturn nil, acct, fmt.Errorf(\"performing EAB credentials request: %v\", err)\n\t}\n\tdefer resp.Body.Close()\n\n\tvar result struct {\n\t\tSuccess bool `json:\"success\"`\n\t\tError   struct {\n\t\t\tCode int    `json:\"code\"`\n\t\t\tType string `json:\"type\"`\n\t\t} `json:\"error\"`\n\t\tEABKID     string `json:\"eab_kid\"`\n\t\tEABHMACKey string `json:\"eab_hmac_key\"`\n\t}\n\terr = json.NewDecoder(resp.Body).Decode(&result)\n\tif err != nil {\n\t\treturn nil, acct, fmt.Errorf(\"decoding API response: %v\", err)\n\t}\n\tif result.Error.Code != 0 {\n\t\t// do this check first because ZeroSSL's API returns 200 on errors\n\t\treturn nil, acct, fmt.Errorf(\"failed getting EAB credentials: HTTP %d: %s (code %d)\",\n\t\t\tresp.StatusCode, result.Error.Type, result.Error.Code)\n\t}\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn nil, acct, fmt.Errorf(\"failed getting EAB credentials: HTTP %d\", resp.StatusCode)\n\t}\n\n\tiss.logger.Info(\"generated EAB credentials\", zap.String(\"key_id\", result.EABKID))\n\n\treturn &acme.EAB{\n\t\tKeyID:  result.EABKID,\n\t\tMACKey: result.EABHMACKey,\n\t}, acct, nil\n}\n\n// UnmarshalCaddyfile deserializes Caddyfile tokens into iss.\n//\n//\t... acme [<directory_url>] {\n//\t    dir <directory_url>\n//\t    test_dir <test_directory_url>\n//\t    email <email>\n//\t    timeout <duration>\n//\t    disable_http_challenge\n//\t    disable_tlsalpn_challenge\n//\t    alt_http_port    <port>\n//\t    alt_tlsalpn_port <port>\n//\t    eab <key_id> <mac_key>\n//\t    trusted_roots <pem_files...>\n//\t    dns <provider_name> [<options>]\n//\t    propagation_delay <duration>\n//\t    propagation_timeout <duration>\n//\t    resolvers <dns_servers...>\n//\t    dns_ttl <duration>\n//\t    dns_challenge_override_domain <domain>\n//\t    preferred_chains [smallest] {\n//\t        root_common_name <common_names...>\n//\t        any_common_name  <common_names...>\n//\t    }\n//\t}\nfunc (iss *ACMEIssuer) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume issuer name\n\n\tif d.NextArg() {\n\t\tiss.CA = d.Val()\n\t\tif d.NextArg() {\n\t\t\treturn d.ArgErr()\n\t\t}\n\t}\n\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"lifetime\":\n\t\t\tvar lifetimeStr string\n\t\t\tif !d.AllArgs(&lifetimeStr) {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tlifetime, err := caddy.ParseDuration(lifetimeStr)\n\t\t\tif err != nil {\n\t\t\t\treturn d.Errf(\"invalid lifetime %s: %v\", lifetimeStr, err)\n\t\t\t}\n\t\t\tif lifetime < 0 {\n\t\t\t\treturn d.Errf(\"lifetime must be >= 0: %s\", lifetime)\n\t\t\t}\n\t\t\tiss.CertificateLifetime = caddy.Duration(lifetime)\n\n\t\tcase \"dir\":\n\t\t\tif iss.CA != \"\" {\n\t\t\t\treturn d.Errf(\"directory is already specified: %s\", iss.CA)\n\t\t\t}\n\t\t\tif !d.AllArgs(&iss.CA) {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\n\t\tcase \"test_dir\":\n\t\t\tif !d.AllArgs(&iss.TestCA) {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\n\t\tcase \"email\":\n\t\t\tif !d.AllArgs(&iss.Email) {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\n\t\tcase \"timeout\":\n\t\t\tvar timeoutStr string\n\t\t\tif !d.AllArgs(&timeoutStr) {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\ttimeout, err := caddy.ParseDuration(timeoutStr)\n\t\t\tif err != nil {\n\t\t\t\treturn d.Errf(\"invalid timeout duration %s: %v\", timeoutStr, err)\n\t\t\t}\n\t\t\tiss.ACMETimeout = caddy.Duration(timeout)\n\n\t\tcase \"disable_http_challenge\":\n\t\t\tif d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tif iss.Challenges == nil {\n\t\t\t\tiss.Challenges = new(ChallengesConfig)\n\t\t\t}\n\t\t\tif iss.Challenges.HTTP == nil {\n\t\t\t\tiss.Challenges.HTTP = new(HTTPChallengeConfig)\n\t\t\t}\n\t\t\tiss.Challenges.HTTP.Disabled = true\n\n\t\tcase \"disable_tlsalpn_challenge\":\n\t\t\tif d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tif iss.Challenges == nil {\n\t\t\t\tiss.Challenges = new(ChallengesConfig)\n\t\t\t}\n\t\t\tif iss.Challenges.TLSALPN == nil {\n\t\t\t\tiss.Challenges.TLSALPN = new(TLSALPNChallengeConfig)\n\t\t\t}\n\t\t\tiss.Challenges.TLSALPN.Disabled = true\n\n\t\tcase \"alt_http_port\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tport, err := strconv.Atoi(d.Val())\n\t\t\tif err != nil {\n\t\t\t\treturn d.Errf(\"invalid port %s: %v\", d.Val(), err)\n\t\t\t}\n\t\t\tif iss.Challenges == nil {\n\t\t\t\tiss.Challenges = new(ChallengesConfig)\n\t\t\t}\n\t\t\tif iss.Challenges.HTTP == nil {\n\t\t\t\tiss.Challenges.HTTP = new(HTTPChallengeConfig)\n\t\t\t}\n\t\t\tiss.Challenges.HTTP.AlternatePort = port\n\n\t\tcase \"alt_tlsalpn_port\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tport, err := strconv.Atoi(d.Val())\n\t\t\tif err != nil {\n\t\t\t\treturn d.Errf(\"invalid port %s: %v\", d.Val(), err)\n\t\t\t}\n\t\t\tif iss.Challenges == nil {\n\t\t\t\tiss.Challenges = new(ChallengesConfig)\n\t\t\t}\n\t\t\tif iss.Challenges.TLSALPN == nil {\n\t\t\t\tiss.Challenges.TLSALPN = new(TLSALPNChallengeConfig)\n\t\t\t}\n\t\t\tiss.Challenges.TLSALPN.AlternatePort = port\n\n\t\tcase \"eab\":\n\t\t\tiss.ExternalAccount = new(acme.EAB)\n\t\t\tif !d.AllArgs(&iss.ExternalAccount.KeyID, &iss.ExternalAccount.MACKey) {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\n\t\tcase \"trusted_roots\":\n\t\t\tiss.TrustedRootsPEMFiles = d.RemainingArgs()\n\n\t\tcase \"dns\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tprovName := d.Val()\n\t\t\tif iss.Challenges == nil {\n\t\t\t\tiss.Challenges = new(ChallengesConfig)\n\t\t\t}\n\t\t\tif iss.Challenges.DNS == nil {\n\t\t\t\tiss.Challenges.DNS = new(DNSChallengeConfig)\n\t\t\t}\n\t\t\tunm, err := caddyfile.UnmarshalModule(d, \"dns.providers.\"+provName)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tiss.Challenges.DNS.ProviderRaw = caddyconfig.JSONModuleObject(unm, \"name\", provName, nil)\n\n\t\tcase \"propagation_delay\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tdelayStr := d.Val()\n\t\t\tdelay, err := caddy.ParseDuration(delayStr)\n\t\t\tif err != nil {\n\t\t\t\treturn d.Errf(\"invalid propagation_delay duration %s: %v\", delayStr, err)\n\t\t\t}\n\t\t\tif iss.Challenges == nil {\n\t\t\t\tiss.Challenges = new(ChallengesConfig)\n\t\t\t}\n\t\t\tif iss.Challenges.DNS == nil {\n\t\t\t\tiss.Challenges.DNS = new(DNSChallengeConfig)\n\t\t\t}\n\t\t\tiss.Challenges.DNS.PropagationDelay = caddy.Duration(delay)\n\n\t\tcase \"propagation_timeout\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\ttimeoutStr := d.Val()\n\t\t\tvar timeout time.Duration\n\t\t\tif timeoutStr == \"-1\" {\n\t\t\t\ttimeout = time.Duration(-1)\n\t\t\t} else {\n\t\t\t\tvar err error\n\t\t\t\ttimeout, err = caddy.ParseDuration(timeoutStr)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn d.Errf(\"invalid propagation_timeout duration %s: %v\", timeoutStr, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif iss.Challenges == nil {\n\t\t\t\tiss.Challenges = new(ChallengesConfig)\n\t\t\t}\n\t\t\tif iss.Challenges.DNS == nil {\n...\n// Path: modules/caddytls/automation.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddytls\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\n\t\"github.com/caddyserver/certmagic\"\n\t\"github.com/mholt/acmez/v2\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\n// AutomationConfig governs the automated management of TLS certificates.\ntype AutomationConfig struct {\n\t// The list of automation policies. The first policy matching\n\t// a certificate or subject name will be applied.\n\tPolicies []*AutomationPolicy `json:\"policies,omitempty\"`\n\n\t// On-Demand TLS defers certificate operations to the\n\t// moment they are needed, e.g. during a TLS handshake.\n\t// Useful when you don't know all the hostnames at\n\t// config-time, or when you are not in control of the\n\t// domain names you are managing certificates for.\n\t// In 2015, Caddy became the first web server to\n\t// implement this experimental technology.\n\t//\n\t// Note that this field does not enable on-demand TLS;\n\t// it only configures it for when it is used. To enable\n\t// it, create an automation policy with `on_demand`.\n\tOnDemand *OnDemandConfig `json:\"on_demand,omitempty\"`\n\n\t// Caddy staples OCSP (and caches the response) for all\n\t// qualifying certificates by default. This setting\n\t// changes how often it scans responses for freshness,\n\t// and updates them if they are getting stale. Default: 1h\n\tOCSPCheckInterval caddy.Duration `json:\"ocsp_interval,omitempty\"`\n\n\t// Every so often, Caddy will scan all loaded, managed\n\t// certificates for expiration. This setting changes how\n\t// frequently the scan for expiring certificates is\n\t// performed. Default: 10m\n\tRenewCheckInterval caddy.Duration `json:\"renew_interval,omitempty\"`\n\n\t// How often to scan storage units for old or expired\n\t// assets and remove them. These scans exert lots of\n\t// reads (and list operations) on the storage module, so\n\t// choose a longer interval for large deployments.\n\t// Default: 24h\n\t//\n\t// Storage will always be cleaned when the process first\n\t// starts. Then, a new cleaning will be started this\n\t// duration after the previous cleaning started if the\n\t// previous cleaning finished in less than half the time\n\t// of this interval (otherwise next start will be skipped).\n\tStorageCleanInterval caddy.Duration `json:\"storage_clean_interval,omitempty\"`\n\n\tdefaultPublicAutomationPolicy   *AutomationPolicy\n\tdefaultInternalAutomationPolicy *AutomationPolicy // only initialized if necessary\n}\n\n// AutomationPolicy designates the policy for automating the\n// management (obtaining, renewal, and revocation) of managed\n// TLS certificates.\n//\n// An AutomationPolicy value is not valid until it has been\n// provisioned; use the `AddAutomationPolicy()` method on the\n// TLS app to properly provision a new policy.\ntype AutomationPolicy struct {\n\t// Which subjects (hostnames or IP addresses) this policy applies to.\n\t//\n\t// This list is a filter, not a command. In other words, it is used\n\t// only to filter whether this policy should apply to a subject that\n\t// needs a certificate; it does NOT command the TLS app to manage a\n\t// certificate for that subject. To have Caddy automate a certificate\n\t// or specific subjects, use the \"automate\" certificate loader module\n\t// of the TLS app.\n\tSubjectsRaw []string `json:\"subjects,omitempty\"`\n\n\t// The modules that may issue certificates. Default: internal if all\n\t// subjects do not qualify for public certificates; othewise acme and\n\t// zerossl.\n\tIssuersRaw []json.RawMessage `json:\"issuers,omitempty\" caddy:\"namespace=tls.issuance inline_key=module\"`\n\n\t// Modules that can get a custom certificate to use for any\n\t// given TLS handshake at handshake-time. Custom certificates\n\t// can be useful if another entity is managing certificates\n\t// and Caddy need only get it and serve it. Specifying a Manager\n\t// enables on-demand TLS, i.e. it has the side-effect of setting\n\t// the on_demand parameter to `true`.\n\t//\n\t// TODO: This is an EXPERIMENTAL feature. Subject to change or removal.\n\tManagersRaw []json.RawMessage `json:\"get_certificate,omitempty\" caddy:\"namespace=tls.get_certificate inline_key=via\"`\n\n\t// If true, certificates will be requested with MustStaple. Not all\n\t// CAs support this, and there are potentially serious consequences\n\t// of enabling this feature without proper threat modeling.\n\tMustStaple bool `json:\"must_staple,omitempty\"`\n\n\t// How long before a certificate's expiration to try renewing it,\n\t// as a function of its total lifetime. As a general and conservative\n\t// rule, it is a good idea to renew a certificate when it has about\n\t// 1/3 of its total lifetime remaining. This utilizes the majority\n\t// of the certificate's lifetime while still saving time to\n\t// troubleshoot problems. However, for extremely short-lived certs,\n\t// you may wish to increase the ratio to ~1/2.\n\tRenewalWindowRatio float64 `json:\"renewal_window_ratio,omitempty\"`\n\n\t// The type of key to generate for certificates.\n\t// Supported values: `ed25519`, `p256`, `p384`, `rsa2048`, `rsa4096`.\n\tKeyType string `json:\"key_type,omitempty\"`\n\n\t// Optionally configure a separate storage module associated with this\n\t// manager, instead of using Caddy's global/default-configured storage.\n\tStorageRaw json.RawMessage `json:\"storage,omitempty\" caddy:\"namespace=caddy.storage inline_key=module\"`\n\n\t// If true, certificates will be managed \"on demand\"; that is, during\n\t// TLS handshakes or when needed, as opposed to at startup or config\n\t// load. This enables On-Demand TLS for this policy.\n\tOnDemand bool `json:\"on_demand,omitempty\"`\n\n\t// If true, private keys already existing in storage\n\t// will be reused. Otherwise, a new key will be\n\t// created for every new certificate to mitigate\n\t// pinning and reduce the scope of key compromise.\n\t// TEMPORARY: Key pinning is against industry best practices.\n\t// This property will likely be removed in the future.\n\t// Do not rely on it forever; watch the release notes.\n\tReusePrivateKeys bool `json:\"reuse_private_keys,omitempty\"`\n\n\t// Disables OCSP stapling. Disabling OCSP stapling puts clients at\n\t// greater risk, reduces their privacy, and usually lowers client\n\t// performance. It is NOT recommended to disable this unless you\n\t// are able to justify the costs.\n\t// EXPERIMENTAL. Subject to change.\n\tDisableOCSPStapling bool `json:\"disable_ocsp_stapling,omitempty\"`\n\n\t// Overrides the URLs of OCSP responders embedded in certificates.\n\t// Each key is a OCSP server URL to override, and its value is the\n\t// replacement. An empty value will disable querying of that server.\n\t// EXPERIMENTAL. Subject to change.\n\tOCSPOverrides map[string]string `json:\"ocsp_overrides,omitempty\"`\n\n\t// Issuers and Managers store the decoded issuer and manager modules;\n\t// they are only used to populate an underlying certmagic.Config's\n\t// fields during provisioning so that the modules can survive a\n\t// re-provisioning.\n\tIssuers  []certmagic.Issuer  `json:\"-\"`\n\tManagers []certmagic.Manager `json:\"-\"`\n\n\tsubjects []string\n\tmagic    *certmagic.Config\n\tstorage  certmagic.Storage\n}\n\n// Provision sets up ap and builds its underlying CertMagic config.\nfunc (ap *AutomationPolicy) Provision(tlsApp *TLS) error {\n\t// replace placeholders in subjects to allow environment variables\n\trepl := caddy.NewReplacer()\n\tsubjects := make([]string, len(ap.SubjectsRaw))\n\tfor i, sub := range ap.SubjectsRaw {\n\t\tsubjects[i] = repl.ReplaceAll(sub, \"\")\n\t}\n\tap.subjects = subjects\n\n\t// policy-specific storage implementation\n\tif ap.StorageRaw != nil {\n\t\tval, err := tlsApp.ctx.LoadModule(ap, \"StorageRaw\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"loading TLS storage module: %v\", err)\n\t\t}\n\t\tcmStorage, err := val.(caddy.StorageConverter).CertMagicStorage()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"creating TLS storage configuration: %v\", err)\n\t\t}\n\t\tap.storage = cmStorage\n\t}\n\n\t// we don't store loaded modules directly in the certmagic config since\n\t// policy provisioning may happen more than once (during auto-HTTPS) and\n\t// loading a module clears its config bytes; thus, load the module and\n\t// store them on the policy before putting it on the config\n\n\t// load and provision any cert manager modules\n\thadExplicitManagers := len(ap.ManagersRaw) > 0\n\tif ap.ManagersRaw != nil {\n\t\tvals, err := tlsApp.ctx.LoadModule(ap, \"ManagersRaw\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"loading external certificate manager modules: %v\", err)\n\t\t}\n\t\tfor _, getCertVal := range vals.([]any) {\n\t\t\tap.Managers = append(ap.Managers, getCertVal.(certmagic.Manager))\n\t\t}\n\t}\n\n\t// load and provision any explicitly-configured issuer modules\n\tif ap.IssuersRaw != nil {\n\t\tval, err := tlsApp.ctx.LoadModule(ap, \"IssuersRaw\")\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"loading TLS automation management module: %s\", err)\n\t\t}\n\t\tfor _, issVal := range val.([]any) {\n\t\t\tap.Issuers = append(ap.Issuers, issVal.(certmagic.Issuer))\n\t\t}\n\t}\n\n\tissuers := ap.Issuers\n\tif len(issuers) == 0 {\n\t\tvar err error\n\t\tissuers, err = DefaultIssuersProvisioned(tlsApp.ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tkeyType := ap.KeyType\n\tif keyType != \"\" {\n\t\tvar err error\n\t\tkeyType, err = caddy.NewReplacer().ReplaceOrErr(ap.KeyType, true, true)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"invalid key type %s: %s\", ap.KeyType, err)\n\t\t}\n\t\tif _, ok := supportedCertKeyTypes[keyType]; !ok {\n\t\t\treturn fmt.Errorf(\"unrecognized key type: %s\", keyType)\n\t\t}\n\t}\n\tkeySource := certmagic.StandardKeyGenerator{\n\t\tKeyType: supportedCertKeyTypes[keyType],\n\t}\n\n\tstorage := ap.storage\n\tif storage == nil {\n\t\tstorage = tlsApp.ctx.Storage()\n\t}\n\n\t// on-demand TLS\n\tvar ond *certmagic.OnDemandConfig\n\tif ap.OnDemand || len(ap.Managers) > 0 {\n\t\t// permission module is now required after a number of negligence cases that allowed abuse;\n\t\t// but it may still be optional for explicit subjects (bounded, non-wildcard), for the\n\t\t// internal issuer since it doesn't cause public PKI pressure on ACME servers; subtly, it\n\t\t// is useful to allow on-demand TLS to be enabled so Managers can be used, but to still\n\t\t// prevent issuance from Issuers (when Managers don't provide a certificate) if there's no\n\t\t// permission module configured\n\t\tnoProtections := ap.isWildcardOrDefault() && !ap.onlyInternalIssuer() && (tlsApp.Automation == nil || tlsApp.Automation.OnDemand == nil || tlsApp.Automation.OnDemand.permission == nil)\n\t\tfailClosed := noProtections && hadExplicitManagers // don't allow on-demand issuance (other than implicit managers) if no managers have been explicitly configured\n\t\tif noProtections {\n\t\t\tif !hadExplicitManagers {\n\t\t\t\t// no managers, no explicitly-configured permission module, this is a config error\n\t\t\t\treturn fmt.Errorf(\"on-demand TLS cannot be enabled without a permission module to prevent abuse; please refer to documentation for details\")\n\t\t\t}\n\t\t\t// allow on-demand to be enabled but only for the purpose of the Managers; issuance won't be allowed from Issuers\n\t\t\ttlsApp.logger.Warn(\"on-demand TLS can only get certificates from the configured external manager(s) because no ask endpoint / permission module is specified\")\n\t\t}\n\t\tond = &certmagic.OnDemandConfig{\n\t\t\tDecisionFunc: func(ctx context.Context, name string) error {\n\t\t\t\tif failClosed {\n\t\t\t\t\treturn fmt.Errorf(\"no permission module configured; certificates not allowed except from external Managers\")\n\t\t\t\t}\n\t\t\t\tif tlsApp.Automation == nil || tlsApp.Automation.OnDemand == nil {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\n\t\t\t\t// logging the remote IP can be useful for servers that want to count\n\t\t\t\t// attempts from clients to detect patterns of abuse -- it should NOT be\n\t\t\t\t// used solely for decision making, however\n\t\t\t\tvar remoteIP string\n\t\t\t\tif hello, ok := ctx.Value(certmagic.ClientHelloInfoCtxKey).(*tls.ClientHelloInfo); ok && hello != nil {\n\t\t\t\t\tif remote := hello.Conn.RemoteAddr(); remote != nil {\n\t\t\t\t\t\tremoteIP, _, _ = net.SplitHostPort(remote.String())\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ttlsApp.logger.Debug(\"asking for permission for on-demand certificate\",\n\t\t\t\t\tzap.String(\"remote_ip\", remoteIP),\n\t\t\t\t\tzap.String(\"domain\", name))\n\n\t\t\t\t// ask the permission module if this cert is allowed\n\t\t\t\tif err := tlsApp.Automation.OnDemand.permission.CertificateAllowed(ctx, name); err != nil {\n\t\t\t\t\t// distinguish true errors from denials, because it's important to elevate actual errors\n\t\t\t\t\tif errors.Is(err, ErrPermissionDenied) {\n\t\t\t\t\t\ttlsApp.logger.Debug(\"on-demand certificate issuance denied\",\n\t\t\t\t\t\t\tzap.String(\"domain\", name),\n\t\t\t\t\t\t\tzap.Error(err))\n\t\t\t\t\t} else {\n\t\t\t\t\t\ttlsApp.logger.Error(\"failed to get permission for on-demand certificate\",\n\t\t\t\t\t\t\tzap.String(\"domain\", name),\n\t\t\t\t\t\t\tzap.Error(err))\n\t\t\t\t\t}\n\t\t\t\t\treturn err\n\t\t\t\t}\n\n\t\t\t\t// check the rate limiter last because\n\t\t\t\t// doing so makes a reservation\n\t\t\t\tif !onDemandRateLimiter.Allow() {\n\t\t\t\t\treturn fmt.Errorf(\"on-demand rate limit exceeded\")\n\t\t\t\t}\n\n\t\t\t\treturn nil\n\t\t\t},\n\t\t\tManagers: ap.Managers,\n\t\t}\n\t}\n\n\ttemplate := certmagic.Config{\n\t\tMustStaple:         ap.MustStaple,\n\t\tRenewalWindowRatio: ap.RenewalWindowRatio,\n\t\tKeySource:          keySource,\n\t\tOnEvent:            tlsApp.onEvent,\n\t\tOnDemand:           ond,\n\t\tReusePrivateKeys:   ap.ReusePrivateKeys,\n\t\tOCSP: certmagic.OCSPConfig{\n\t\t\tDisableStapling:    ap.DisableOCSPStapling,\n\t\t\tResponderOverrides: ap.OCSPOverrides,\n\t\t},\n\t\tStorage: storage,\n\t\tIssuers: issuers,\n\t\tLogger:  tlsApp.logger,\n\t}\n\tcertCacheMu.RLock()\n\tap.magic = certmagic.New(certCache, template)\n\tcertCacheMu.RUnlock()\n\n\t// sometimes issuers may need the parent certmagic.Config in\n\t// order to function properly (for example, ACMEIssuer needs\n\t// access to the correct storage and cache so it can solve\n\t// ACME challenges -- it's an annoying, inelegant circular\n\t// dependency that I don't know how to resolve nicely!)\n\tfor _, issuer := range ap.magic.Issuers {\n\t\tif annoying, ok := issuer.(ConfigSetter); ok {\n\t\t\tannoying.SetConfig(ap.magic)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Subjects returns the list of subjects with all placeholders replaced.\nfunc (ap *AutomationPolicy) Subjects() []string {\n\treturn ap.subjects\n}\n\n// AllInternalSubjects returns true if all the subjects on this policy are internal.\nfunc (ap *AutomationPolicy) AllInternalSubjects() bool {\n\tfor _, subj := range ap.subjects {\n\t\tif !certmagic.SubjectIsInternal(subj) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc (ap *AutomationPolicy) onlyInternalIssuer() bool {\n\tif len(ap.Issuers) != 1 {\n\t\treturn false\n\t}\n\t_, ok := ap.Issuers[0].(*InternalIssuer)\n\treturn ok\n}\n\n// isWildcardOrDefault determines if the subjects include any wildcard domains,\n// or is the \"default\" policy (i.e. no subjects) which is unbounded.\nfunc (ap *AutomationPolicy) isWildcardOrDefault() bool {\n\tisWildcardOrDefault := false\n\tif len(ap.subjects) == 0 {\n\t\tisWildcardOrDefault = true\n\t}\n\tfor _, sub := range ap.subjects {\n\t\tif strings.HasPrefix(sub, \"*\") {\n\t\t\tisWildcardOrDefault = true\n\t\t\tbreak\n\t\t}\n\t}\n\treturn isWildcardOrDefault\n}\n\n// DefaultIssuers returns empty Issuers (not provisioned) to be used as defaults.\n// This function is experimental and has no compatibility promises.\n\nfunc DefaultIssuers(userEmail string) []certmagic.Issuer {\n\tissuers := []certmagic.Issuer{new(ACMEIssuer)}\n\tif strings.TrimSpace(userEmail) != \"\" {\n\t\tissuers = append(issuers, &ACMEIssuer{\n\t\t\tCA:    certmagic.ZeroSSLProductionCA,\n\t\t\tEmail: userEmail,\n\t\t})\n\t}\n\treturn issuers\n}\n\n// DefaultIssuersProvisioned returns empty but provisioned default Issuers from\n// DefaultIssuers(). This function is experimental and has no compatibility promises.\nfunc DefaultIssuersProvisioned(ctx caddy.Context) ([]certmagic.Issuer, error) {\n\tissuers := DefaultIssuers(\"\")\n\tfor i, iss := range issuers {\n\t\tif prov, ok := iss.(caddy.Provisioner); ok {\n\t\t\terr := prov.Provision(ctx)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"provisioning default issuer %d: %T: %v\", i, iss, err)\n\t\t\t}\n\t\t}\n\t}\n\treturn issuers, nil\n}\n\n// ChallengesConfig configures the ACME challenges.\ntype ChallengesConfig struct {\n\t// HTTP configures the ACME HTTP challenge. This\n\t// challenge is enabled and used automatically\n\t// and by default.\n\tHTTP *HTTPChallengeConfig `json:\"http,omitempty\"`\n\n\t// TLSALPN configures the ACME TLS-ALPN challenge.\n\t// This challenge is enabled and used automatically\n\t// and by default.\n\tTLSALPN *TLSALPNChallengeConfig `json:\"tls-alpn,omitempty\"`\n\n\t// Configures the ACME DNS challenge. Because this\n\t// challenge typically requires credentials for\n\t// interfacing with a DNS provider, this challenge is\n\t// not enabled by default. This is the only challenge\n\t// type which does not require a direct connection\n\t// to Caddy from an external server.\n\t//\n\t// NOTE: DNS providers are currently being upgraded,\n\t// and this API is subject to change, but should be\n\t// stabilized soon.\n\tDNS *DNSChallengeConfig `json:\"dns,omitempty\"`\n\n\t// Optionally customize the host to which a listener\n\t// is bound if required for solving a challenge.\n\tBindHost string `json:\"bind_host,omitempty\"`\n}\n\n// HTTPChallengeConfig configures the ACME HTTP challenge.\ntype HTTPChallengeConfig struct {\n\t// If true, the HTTP challenge will be disabled.\n\tDisabled bool `json:\"disabled,omitempty\"`\n\n\t// An alternate port on which to service this\n\t// challenge. Note that the HTTP challenge port is\n\t// hard-coded into the spec and cannot be changed,\n\t// so you would have to forward packets from the\n\t// standard HTTP challenge port to this one.\n\tAlternatePort int `json:\"alternate_port,omitempty\"`\n}\n\n// TLSALPNChallengeConfig configures the ACME TLS-ALPN challenge.\ntype TLSALPNChallengeConfig struct {\n\t// If true, the TLS-ALPN challenge will be disabled.\n\tDisabled bool `json:\"disabled,omitempty\"`\n\n\t// An alternate port on which to service this\n\t// challenge. Note that the TLS-ALPN challenge port\n\t// is hard-coded into the spec and cannot be changed,\n\t// so you would have to forward packets from the\n\t// standard TLS-ALPN challenge port to this one.\n\tAlternatePort int `json:\"alternate_port,omitempty\"`\n}\n\n// DNSChallengeConfig configures the ACME DNS challenge.\n//\n// NOTE: This API is still experimental and is subject to change.\ntype DNSChallengeConfig struct {\n\t// The DNS provider module to use which will manage\n\t// the DNS records relevant to the ACME challenge.\n\t// Required.\n\tProviderRaw json.RawMessage `json:\"provider,omitempty\" caddy:\"namespace=dns.providers inline_key=name\"`\n\n\t// The TTL of the TXT record used for the DNS challenge.\n\tTTL caddy.Duration `json:\"ttl,omitempty\"`\n\n\t// How long to wait before starting propagation checks.\n\t// Default: 0 (no wait).\n\tPropagationDelay caddy.Duration `json:\"propagation_delay,omitempty\"`\n\n\t// Maximum time to wait for temporary DNS record to appear.\n\t// Set to -1 to disable propagation checks.\n\t// Default: 2 minutes.\n\tPropagationTimeout caddy.Duration `json:\"propagation_timeout,omitempty\"`\n\n\t// Custom DNS resolvers to prefer over system/built-in defaults.\n\t// Often necessary to configure when using split-horizon DNS.\n\tResolvers []string `json:\"resolvers,omitempty\"`\n\n\t// Override the domain to use for the DNS challenge. This\n\t// is to delegate the challenge to a different domain,\n\t// e.g. one that updates faster or one with a provider API.\n\tOverrideDomain string `json:\"override_domain,omitempty\"`\n\n\tsolver acmez.Solver\n}\n\n// ConfigSetter is implemented by certmagic.Issuers that\n// need access to a parent certmagic.Config as part of\n// their provisioning phase. For example, the ACMEIssuer\n// requires a config so it can access storage and the\n// cache to solve ACME challenges.\ntype ConfigSetter interface {\n\tSetConfig(cfg *certmagic.Config)\n}\n\n// Path: modules/caddytls/capools.go\npackage caddytls\n\nimport (\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\n\t\"github.com/caddyserver/certmagic\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddypki\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(InlineCAPool{})\n\tcaddy.RegisterModule(FileCAPool{})\n\tcaddy.RegisterModule(PKIRootCAPool{})\n\tcaddy.RegisterModule(PKIIntermediateCAPool{})\n\tcaddy.RegisterModule(StoragePool{})\n\tcaddy.RegisterModule(HTTPCertPool{})\n\tcaddy.RegisterModule(LazyCertPool{})\n}\n\n// The interface to be implemented by all guest modules part of\n// the namespace 'tls.ca_pool.source.'\ntype CA interface {\n\tCertPool() *x509.CertPool\n}\n\n// InlineCAPool is a certificate authority pool provider coming from\n// a DER-encoded certificates in the config\ntype InlineCAPool struct {\n\t// A list of base64 DER-encoded CA certificates\n\t// against which to validate client certificates.\n\t// Client certs which are not signed by any of\n\t// these CAs will be rejected.\n\tTrustedCACerts []string `json:\"trusted_ca_certs,omitempty\"`\n\n\tpool *x509.CertPool\n}\n\n// CaddyModule implements caddy.Module.\nfunc (icp InlineCAPool) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID: \"tls.ca_pool.source.inline\",\n\t\tNew: func() caddy.Module {\n\t\t\treturn new(InlineCAPool)\n\t\t},\n\t}\n}\n\n// Provision implements caddy.Provisioner.\nfunc (icp *InlineCAPool) Provision(ctx caddy.Context) error {\n\tcaPool := x509.NewCertPool()\n\tfor i, clientCAString := range icp.TrustedCACerts {\n\t\tclientCA, err := decodeBase64DERCert(clientCAString)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"parsing certificate at index %d: %v\", i, err)\n\t\t}\n\t\tcaPool.AddCert(clientCA)\n\t}\n\ticp.pool = caPool\n\n\treturn nil\n}\n\n// Syntax:\n//\n//\ttrust_pool inline {\n//\t\ttrust_der <base64_der_cert>...\n//\t}\n//\n// The 'trust_der' directive can be specified multiple times.\nfunc (icp *InlineCAPool) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume module name\n\tif d.CountRemainingArgs() > 0 {\n\t\treturn d.ArgErr()\n\t}\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"trust_der\":\n\t\t\ticp.TrustedCACerts = append(icp.TrustedCACerts, d.RemainingArgs()...)\n\t\tdefault:\n\t\t\treturn d.Errf(\"unrecognized directive: %s\", d.Val())\n\t\t}\n\t}\n\tif len(icp.TrustedCACerts) == 0 {\n\t\treturn d.Err(\"no certificates specified\")\n\t}\n\treturn nil\n}\n\n// CertPool implements CA.\nfunc (icp InlineCAPool) CertPool() *x509.CertPool {\n\treturn icp.pool\n}\n\n// FileCAPool generates trusted root certificates pool from the designated DER and PEM file\ntype FileCAPool struct {\n\t// TrustedCACertPEMFiles is a list of PEM file names\n\t// from which to load certificates of trusted CAs.\n\t// Client certificates which are not signed by any of\n\t// these CA certificates will be rejected.\n\tTrustedCACertPEMFiles []string `json:\"pem_files,omitempty\"`\n\n\tpool *x509.CertPool\n}\n\n// CaddyModule implements caddy.Module.\nfunc (FileCAPool) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID: \"tls.ca_pool.source.file\",\n\t\tNew: func() caddy.Module {\n\t\t\treturn new(FileCAPool)\n\t\t},\n\t}\n}\n\n// Loads and decodes the DER and pem files to generate the certificate pool\nfunc (f *FileCAPool) Provision(ctx caddy.Context) error {\n\tcaPool := x509.NewCertPool()\n\tfor _, pemFile := range f.TrustedCACertPEMFiles {\n\t\tpemContents, err := os.ReadFile(pemFile)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"reading %s: %v\", pemFile, err)\n\t\t}\n\t\tcaPool.AppendCertsFromPEM(pemContents)\n\t}\n\tf.pool = caPool\n\treturn nil\n}\n\n// Syntax:\n//\n//\ttrust_pool file [<pem_file>...] {\n//\t\tpem_file <pem_file>...\n//\t}\n//\n// The 'pem_file' directive can be specified multiple times.\nfunc (fcap *FileCAPool) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume module name\n\tfcap.TrustedCACertPEMFiles = append(fcap.TrustedCACertPEMFiles, d.RemainingArgs()...)\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"pem_file\":\n\t\t\tfcap.TrustedCACertPEMFiles = append(fcap.TrustedCACertPEMFiles, d.RemainingArgs()...)\n\t\tdefault:\n\t\t\treturn d.Errf(\"unrecognized directive: %s\", d.Val())\n\t\t}\n\t}\n\tif len(fcap.TrustedCACertPEMFiles) == 0 {\n\t\treturn d.Err(\"no certificates specified\")\n\t}\n\treturn nil\n}\n\nfunc (f FileCAPool) CertPool() *x509.CertPool {\n\treturn f.pool\n}\n\n// PKIRootCAPool extracts the trusted root certificates from Caddy's native 'pki' app\ntype PKIRootCAPool struct {\n\t// List of the Authority names that are configured in the `pki` app whose root certificates are trusted\n\tAuthority []string `json:\"authority,omitempty\"`\n\n\tca   []*caddypki.CA\n\tpool *x509.CertPool\n}\n\n// CaddyModule implements caddy.Module.\nfunc (PKIRootCAPool) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID: \"tls.ca_pool.source.pki_root\",\n\t\tNew: func() caddy.Module {\n\t\t\treturn new(PKIRootCAPool)\n\t\t},\n\t}\n}\n\n// Loads the PKI app and load the root certificates into the certificate pool\nfunc (p *PKIRootCAPool) Provision(ctx caddy.Context) error {\n\tpkiApp := ctx.AppIfConfigured(\"pki\")\n\tif pkiApp == nil {\n\t\treturn fmt.Errorf(\"PKI app not configured\")\n\t}\n\tpki := pkiApp.(*caddypki.PKI)\n\tfor _, caID := range p.Authority {\n\t\tc, err := pki.GetCA(ctx, caID)\n\t\tif err != nil || c == nil {\n\t\t\treturn fmt.Errorf(\"getting CA %s: %v\", caID, err)\n\t\t}\n\t\tp.ca = append(p.ca, c)\n\t}\n\n\tcaPool := x509.NewCertPool()\n\tfor _, ca := range p.ca {\n\t\tcaPool.AddCert(ca.RootCertificate())\n\t}\n\tp.pool = caPool\n\n\treturn nil\n}\n\n// Syntax:\n//\n//\ttrust_pool pki_root [<ca_name>...] {\n//\t\tauthority <ca_name>...\n//\t}\n//\n// The 'authority' directive can be specified multiple times.\nfunc (pkir *PKIRootCAPool) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume module name\n\tpkir.Authority = append(pkir.Authority, d.RemainingArgs()...)\n\tfor nesting := d.Nesting(); d.NextBlock(nesting); {\n\t\tswitch d.Val() {\n\t\tcase \"authority\":\n\t\t\tpkir.Authority = append(pkir.Authority, d.RemainingArgs()...)\n\t\tdefault:\n\t\t\treturn d.Errf(\"unrecognized directive: %s\", d.Val())\n\t\t}\n\t}\n\tif len(pkir.Authority) == 0 {\n\t\treturn d.Err(\"no authorities specified\")\n\t}\n\treturn nil\n}\n\n// return the certificate pool generated with root certificates from the PKI app\nfunc (p PKIRootCAPool) CertPool() *x509.CertPool {\n\treturn p.pool\n}\n\n// PKIIntermediateCAPool extracts the trusted intermediate certificates from Caddy's native 'pki' app\ntype PKIIntermediateCAPool struct {\n...\n", "needle_token_start": 11424, "needle_token_end": 11542, "code_context_ntokens": 15307, "output": ["```go\n// DefaultIssuers returns empty Issuers (not provisioned) to be used as defaults.\n// This function is experimental and has no compatibility promises.\nfunc DefaultIssuers(userEmail string) []certmagic.Issuer {\n\tissuers := []certmagic.Issuer{new(ACMEIssuer)}\n\tif strings.TrimSpace(userEmail) != \"\" {\n\t\tissuers = append(issuers, &ACMEIssuer{\n\t\t\tCA:    certmagic.ZeroSSLProductionCA,\n\t\t\tEmail: userEmail,\n\t\t})\n\t}\n\treturn issuers\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "stringToChallenges", "language": "go", "path": "modules/caddypki/acmeserver/challenges.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function converts a list of challenge type strings into a list of challenge type constants.\n2. **Input**: A list of strings, each representing a type of challenge.\n3. **Output**: A list of constants, where each constant corresponds to a challenge type from the input list.\n4. **Procedure**: The function iterates over the input list of strings, converting each string into its corresponding challenge type constant, and collects these into a new list which is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: modules/caddyhttp/tracing/module.go\npackage tracing\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"go.uber.org/zap\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(Tracing{})\n\thttpcaddyfile.RegisterHandlerDirective(\"tracing\", parseCaddyfile)\n}\n\n// Tracing implements an HTTP handler that adds support for distributed tracing,\n// using OpenTelemetry. This module is responsible for the injection and\n// propagation of the trace context. Configure this module via environment\n// variables (see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md).\n// Some values can be overwritten in the configuration file.\ntype Tracing struct {\n\t// SpanName is a span name. It should follow the naming guidelines here:\n\t// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span\n\tSpanName string `json:\"span\"`\n\n\t// otel implements opentelemetry related logic.\n\totel openTelemetryWrapper\n\n\tlogger *zap.Logger\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (Tracing) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.handlers.tracing\",\n\t\tNew: func() caddy.Module { return new(Tracing) },\n\t}\n}\n...\n// Path: modules/caddyhttp/tracing/module_test.go\npackage tracing\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc TestTracing_UnmarshalCaddyfile(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\tspanName string\n\t\td        *caddyfile.Dispenser\n\t\twantErr  bool\n\t}{\n\t\t{\n\t\t\tname:     \"Full config\",\n\t\t\tspanName: \"my-span\",\n\t\t\td: caddyfile.NewTestDispenser(`\ntracing {\n\tspan my-span\n}`),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname:     \"Only span name in the config\",\n\t\t\tspanName: \"my-span\",\n\t\t\td: caddyfile.NewTestDispenser(`\ntracing {\n\tspan my-span\n}`),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"Empty config\",\n\t\t\td: caddyfile.NewTestDispenser(`\ntracing {\n}`),\n\t\t\twantErr: false,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tot := &Tracing{}\n\t\t\tif err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"UnmarshalCaddyfile() error = %v, wantErrType %v\", err, tt.wantErr)\n\t\t\t}\n\n\t\t\tif ot.SpanName != tt.spanName {\n\t\t\t\tt.Errorf(\"UnmarshalCaddyfile() SpanName = %v, want SpanName %v\", ot.SpanName, tt.spanName)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTracing_UnmarshalCaddyfile_Error(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\td       *caddyfile.Dispenser\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"Unknown parameter\",\n\t\t\td: caddyfile.NewTestDispenser(`\n\t\ttracing {\n\t\t\tfoo bar\n\t\t}`),\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"Missed argument\",\n\t\t\td: caddyfile.NewTestDispenser(`\ntracing {\n\tspan\n}`),\n\t\t\twantErr: true,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tot := &Tracing{}\n\t\t\tif err := ot.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"UnmarshalCaddyfile() error = %v, wantErrType %v\", err, tt.wantErr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTracing_ServeHTTP_Propagation_Without_Initial_Headers(t *testing.T) {\n\tot := &Tracing{\n\t\tSpanName: \"mySpan\",\n\t}\n\n\treq := createRequestWithContext(\"GET\", \"https://example.com/foo\")\n\tw := httptest.NewRecorder()\n\n\tvar handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {\n\t\ttraceparent := request.Header.Get(\"Traceparent\")\n\t\tif traceparent == \"\" || strings.HasPrefix(traceparent, \"00-00000000000000000000000000000000-0000000000000000\") {\n\t\t\tt.Errorf(\"Invalid traceparent: %v\", traceparent)\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})\n\tdefer cancel()\n\n\tif err := ot.Provision(ctx); err != nil {\n\t\tt.Errorf(\"Provision error: %v\", err)\n\t\tt.FailNow()\n\t}\n\n\tif err := ot.ServeHTTP(w, req, handler); err != nil {\n\t\tt.Errorf(\"ServeHTTP error: %v\", err)\n\t}\n}\n\nfunc TestTracing_ServeHTTP_Propagation_With_Initial_Headers(t *testing.T) {\n\tot := &Tracing{\n\t\tSpanName: \"mySpan\",\n\t}\n\n\treq := createRequestWithContext(\"GET\", \"https://example.com/foo\")\n\treq.Header.Set(\"traceparent\", \"00-11111111111111111111111111111111-1111111111111111-01\")\n\tw := httptest.NewRecorder()\n\n\tvar handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {\n\t\ttraceparent := request.Header.Get(\"Traceparent\")\n\t\tif !strings.HasPrefix(traceparent, \"00-11111111111111111111111111111111\") {\n\t\t\tt.Errorf(\"Invalid traceparent: %v\", traceparent)\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})\n\tdefer cancel()\n\n\tif err := ot.Provision(ctx); err != nil {\n\t\tt.Errorf(\"Provision error: %v\", err)\n\t\tt.FailNow()\n\t}\n\n\tif err := ot.ServeHTTP(w, req, handler); err != nil {\n\t\tt.Errorf(\"ServeHTTP error: %v\", err)\n\t}\n}\n\nfunc TestTracing_ServeHTTP_Next_Error(t *testing.T) {\n\tot := &Tracing{\n\t\tSpanName: \"mySpan\",\n\t}\n\n\treq := createRequestWithContext(\"GET\", \"https://example.com/foo\")\n\tw := httptest.NewRecorder()\n\n\texpectErr := errors.New(\"test error\")\n\n\tvar handler caddyhttp.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) error {\n\t\treturn expectErr\n\t}\n\n\tctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})\n\tdefer cancel()\n\n\tif err := ot.Provision(ctx); err != nil {\n\t\tt.Errorf(\"Provision error: %v\", err)\n\t\tt.FailNow()\n\t}\n\n\tif err := ot.ServeHTTP(w, req, handler); err == nil || !errors.Is(err, expectErr) {\n\t\tt.Errorf(\"expected error, got: %v\", err)\n\t}\n}\n\nfunc createRequestWithContext(method string, url string) *http.Request {\n\tr, _ := http.NewRequest(method, url, nil)\n\trepl := caddy.NewReplacer()\n\tctx := context.WithValue(r.Context(), caddy.ReplacerCtxKey, repl)\n\tr = r.WithContext(ctx)\n\treturn r\n}\n\n// Path: modules/caddyhttp/tracing/tracer.go\npackage tracing\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\"\n\t\"go.opentelemetry.io/contrib/propagators/autoprop\"\n\t\"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc\"\n\t\"go.opentelemetry.io/otel/propagation\"\n\t\"go.opentelemetry.io/otel/sdk/resource\"\n\tsdktrace \"go.opentelemetry.io/otel/sdk/trace\"\n\tsemconv \"go.opentelemetry.io/otel/semconv/v1.17.0\"\n\t\"go.opentelemetry.io/otel/trace\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nconst (\n\twebEngineName                = \"Caddy\"\n\tdefaultSpanName              = \"handler\"\n\tnextCallCtxKey  caddy.CtxKey = \"nextCall\"\n)\n\n// nextCall store the next handler, and the error value return on calling it (if any)\ntype nextCall struct {\n\tnext caddyhttp.Handler\n\terr  error\n}\n\n// openTelemetryWrapper is responsible for the tracing injection, extraction and propagation.\ntype openTelemetryWrapper struct {\n\tpropagators propagation.TextMapPropagator\n\n\thandler http.Handler\n\n\tspanName string\n}\n\n// newOpenTelemetryWrapper is responsible for the openTelemetryWrapper initialization using provided configuration.\nfunc newOpenTelemetryWrapper(\n\tctx context.Context,\n\tspanName string,\n) (openTelemetryWrapper, error) {\n\tif spanName == \"\" {\n\t\tspanName = defaultSpanName\n\t}\n\n\tot := openTelemetryWrapper{\n\t\tspanName: spanName,\n\t}\n\n\tversion, _ := caddy.Version()\n\tres, err := ot.newResource(webEngineName, version)\n\tif err != nil {\n\t\treturn ot, fmt.Errorf(\"creating resource error: %w\", err)\n\t}\n\n\ttraceExporter, err := otlptracegrpc.New(ctx)\n\tif err != nil {\n\t\treturn ot, fmt.Errorf(\"creating trace exporter error: %w\", err)\n\t}\n\n\tot.propagators = autoprop.NewTextMapPropagator()\n\n\ttracerProvider := globalTracerProvider.getTracerProvider(\n\t\tsdktrace.WithBatcher(traceExporter),\n\t\tsdktrace.WithResource(res),\n\t)\n\n\tot.handler = otelhttp.NewHandler(http.HandlerFunc(ot.serveHTTP),\n\t\tot.spanName,\n\t\totelhttp.WithTracerProvider(tracerProvider),\n\t\totelhttp.WithPropagators(ot.propagators),\n\t\totelhttp.WithSpanNameFormatter(ot.spanNameFormatter),\n\t)\n\n\treturn ot, nil\n}\n\n// serveHTTP injects a tracing context and call the next handler.\nfunc (ot *openTelemetryWrapper) serveHTTP(w http.ResponseWriter, r *http.Request) {\n\tctx := r.Context()\n\tot.propagators.Inject(ctx, propagation.HeaderCarrier(r.Header))\n\tspanCtx := trace.SpanContextFromContext(ctx)\n\tif spanCtx.IsValid() {\n\t\tif extra, ok := ctx.Value(caddyhttp.ExtraLogFieldsCtxKey).(*caddyhttp.ExtraLogFields); ok {\n\t\t\textra.Add(zap.String(\"traceID\", spanCtx.TraceID().String()))\n\t\t}\n\t}\n\tnext := ctx.Value(nextCallCtxKey).(*nextCall)\n\tnext.err = next.next.ServeHTTP(w, r)\n}\n\n// ServeHTTP propagates call to the by wrapped by `otelhttp` next handler.\nfunc (ot *openTelemetryWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {\n\tn := &nextCall{\n\t\tnext: next,\n\t\terr:  nil,\n\t}\n\tot.handler.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), nextCallCtxKey, n)))\n\n\treturn n.err\n}\n\n// cleanup flush all remaining data and shutdown a tracerProvider\nfunc (ot *openTelemetryWrapper) cleanup(logger *zap.Logger) error {\n\treturn globalTracerProvider.cleanupTracerProvider(logger)\n}\n\n// newResource creates a resource that describe current handler instance and merge it with a default attributes value.\nfunc (ot *openTelemetryWrapper) newResource(\n\twebEngineName,\n\twebEngineVersion string,\n) (*resource.Resource, error) {\n\treturn resource.Merge(resource.Default(), resource.NewSchemaless(\n\t\tsemconv.WebEngineName(webEngineName),\n\t\tsemconv.WebEngineVersion(webEngineVersion),\n\t))\n}\n\n// spanNameFormatter performs the replacement of placeholders in the span name\nfunc (ot *openTelemetryWrapper) spanNameFormatter(operation string, r *http.Request) string {\n\treturn r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer).ReplaceAll(operation, \"\")\n}\n\n// Path: modules/caddyhttp/tracing/tracer_test.go\npackage tracing\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n)\n\nfunc TestOpenTelemetryWrapper_newOpenTelemetryWrapper(t *testing.T) {\n\tctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})\n\tdefer cancel()\n\n\tvar otw openTelemetryWrapper\n\tvar err error\n\n\tif otw, err = newOpenTelemetryWrapper(ctx,\n\t\t\"\",\n\t); err != nil {\n\t\tt.Errorf(\"newOpenTelemetryWrapper() error = %v\", err)\n\t\tt.FailNow()\n\t}\n\n\tif otw.propagators == nil {\n\t\tt.Errorf(\"Propagators should not be empty\")\n\t}\n}\n\n// Path: modules/caddyhttp/tracing/tracerprovider.go\npackage tracing\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"sync\"\n\n\tsdktrace \"go.opentelemetry.io/otel/sdk/trace\"\n\t\"go.uber.org/zap\"\n)\n\n// globalTracerProvider stores global tracer provider and is responsible for graceful shutdown when nobody is using it.\nvar globalTracerProvider = &tracerProvider{}\n\ntype tracerProvider struct {\n\tmu                     sync.Mutex\n\ttracerProvider         *sdktrace.TracerProvider\n\ttracerProvidersCounter int\n}\n\n// getTracerProvider create or return an existing global TracerProvider\nfunc (t *tracerProvider) getTracerProvider(opts ...sdktrace.TracerProviderOption) *sdktrace.TracerProvider {\n\tt.mu.Lock()\n\tdefer t.mu.Unlock()\n\n\tt.tracerProvidersCounter++\n\n\tif t.tracerProvider == nil {\n\t\tt.tracerProvider = sdktrace.NewTracerProvider(\n\t\t\topts...,\n\t\t)\n\t}\n\n\treturn t.tracerProvider\n}\n\n// cleanupTracerProvider gracefully shutdown a TracerProvider\nfunc (t *tracerProvider) cleanupTracerProvider(logger *zap.Logger) error {\n\tt.mu.Lock()\n\tdefer t.mu.Unlock()\n\n\tif t.tracerProvidersCounter > 0 {\n\t\tt.tracerProvidersCounter--\n\t}\n\n\tif t.tracerProvidersCounter == 0 {\n\t\tif t.tracerProvider != nil {\n\t\t\t// tracerProvider.ForceFlush SHOULD be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#forceflush\n\t\t\tif err := t.tracerProvider.ForceFlush(context.Background()); err != nil {\n\t\t\t\tlogger.Error(\"forcing flush\", zap.Error(err))\n\t\t\t}\n\n\t\t\t// tracerProvider.Shutdown MUST be invoked according to https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#shutdown\n\t\t\tif err := t.tracerProvider.Shutdown(context.Background()); err != nil {\n\t\t\t\treturn fmt.Errorf(\"tracerProvider shutdown error: %w\", err)\n\t\t\t}\n\t\t}\n\n\t\tt.tracerProvider = nil\n\t}\n\n\treturn nil\n}\n\n// Path: modules/caddyhttp/tracing/tracerprovider_test.go\npackage tracing\n\nimport (\n\t\"testing\"\n\n\t\"go.uber.org/zap\"\n)\n\nfunc Test_tracersProvider_getTracerProvider(t *testing.T) {\n\ttp := tracerProvider{}\n\n\ttp.getTracerProvider()\n\ttp.getTracerProvider()\n\n\tif tp.tracerProvider == nil {\n\t\tt.Errorf(\"There should be tracer provider\")\n\t}\n\n\tif tp.tracerProvidersCounter != 2 {\n\t\tt.Errorf(\"Tracer providers counter should equal to 2\")\n\t}\n}\n\nfunc Test_tracersProvider_cleanupTracerProvider(t *testing.T) {\n\ttp := tracerProvider{}\n\n\ttp.getTracerProvider()\n\ttp.getTracerProvider()\n\n\terr := tp.cleanupTracerProvider(zap.NewNop())\n\tif err != nil {\n\t\tt.Errorf(\"There should be no error: %v\", err)\n\t}\n\n\tif tp.tracerProvider == nil {\n\t\tt.Errorf(\"There should be tracer provider\")\n\t}\n\n\tif tp.tracerProvidersCounter != 1 {\n\t\tt.Errorf(\"Tracer providers counter should equal to 1\")\n\t}\n}\n\n// Path: modules/caddyhttp/vars.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage caddyhttp\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(VarsMiddleware{})\n\tcaddy.RegisterModule(VarsMatcher{})\n\tcaddy.RegisterModule(MatchVarsRE{})\n}\n\n// VarsMiddleware is an HTTP middleware which sets variables to\n// have values that can be used in the HTTP request handler\n// chain. The primary way to access variables is with placeholders,\n// which have the form: `{http.vars.variable_name}`, or with\n// the `vars` and `vars_regexp` request matchers.\n//\n// The key is the variable name, and the value is the value of the\n// variable. Both the name and value may use or contain placeholders.\ntype VarsMiddleware map[string]any\n\n// CaddyModule returns the Caddy module information.\nfunc (VarsMiddleware) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.handlers.vars\",\n\t\tNew: func() caddy.Module { return new(VarsMiddleware) },\n\t}\n}\n\nfunc (m VarsMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error {\n\tvars := r.Context().Value(VarsCtxKey).(map[string]any)\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\tfor k, v := range m {\n\t\tkeyExpanded := repl.ReplaceAll(k, \"\")\n\t\tif valStr, ok := v.(string); ok {\n\t\t\tv = repl.ReplaceAll(valStr, \"\")\n\t\t}\n\t\tvars[keyExpanded] = v\n\n\t\t// Special case: the user ID is in the replacer, pulled from there\n\t\t// for access logs. Allow users to override it with the vars handler.\n\t\tif keyExpanded == \"http.auth.user.id\" {\n\t\t\trepl.Set(keyExpanded, v)\n\t\t}\n\t}\n\treturn next.ServeHTTP(w, r)\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler. Syntax:\n//\n//\tvars [<name> <val>] {\n//\t    <name> <val>\n//\t    ...\n//\t}\nfunc (m *VarsMiddleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume directive name\n\n\tif *m == nil {\n\t\t*m = make(VarsMiddleware)\n\t}\n\n\tnextVar := func(headerLine bool) error {\n\t\tif headerLine {\n\t\t\t// header line is optional\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t\tvarName := d.Val()\n\n\t\tif !d.NextArg() {\n\t\t\treturn d.ArgErr()\n\t\t}\n\t\tvarValue := d.ScalarVal()\n\n\t\t(*m)[varName] = varValue\n\n\t\tif d.NextArg() {\n\t\t\treturn d.ArgErr()\n\t\t}\n\t\treturn nil\n\t}\n\n\tif err := nextVar(true); err != nil {\n\t\treturn err\n\t}\n\tfor d.NextBlock(0) {\n\t\tif err := nextVar(false); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// VarsMatcher is an HTTP request matcher which can match\n// requests based on variables in the context or placeholder\n// values. The key is the placeholder or name of the variable,\n// and the values are possible values the variable can be in\n// order to match (logical OR'ed).\n//\n// If the key is surrounded by `{ }` it is assumed to be a\n// placeholder. Otherwise, it will be considered a variable\n// name.\n//\n// Placeholders in the keys are not expanded, but\n// placeholders in the values are.\ntype VarsMatcher map[string][]string\n\n// CaddyModule returns the Caddy module information.\nfunc (VarsMatcher) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.vars\",\n\t\tNew: func() caddy.Module { return new(VarsMatcher) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *VarsMatcher) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\tif *m == nil {\n\t\t*m = make(map[string][]string)\n\t}\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\tvar field string\n\t\tif !d.Args(&field) {\n\t\t\treturn d.Errf(\"malformed vars matcher: expected field name\")\n\t\t}\n\t\tvals := d.RemainingArgs()\n\t\tif len(vals) == 0 {\n\t\t\treturn d.Errf(\"malformed vars matcher: expected at least one value to match against\")\n\t\t}\n\t\t(*m)[field] = append((*m)[field], vals...)\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed vars matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// Match matches a request based on variables in the context,\n// or placeholders if the key is not a variable.\nfunc (m VarsMatcher) Match(r *http.Request) bool {\n\tif len(m) == 0 {\n\t\treturn true\n\t}\n\n\tvars := r.Context().Value(VarsCtxKey).(map[string]any)\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\n\tfor key, vals := range m {\n\t\tvar varValue any\n\t\tif strings.HasPrefix(key, \"{\") &&\n\t\t\tstrings.HasSuffix(key, \"}\") &&\n\t\t\tstrings.Count(key, \"{\") == 1 {\n\t\t\tvarValue, _ = repl.Get(strings.Trim(key, \"{}\"))\n\t\t} else {\n\t\t\tvarValue = vars[key]\n\t\t}\n\n\t\t// see if any of the values given in the matcher match the actual value\n\t\tfor _, v := range vals {\n\t\t\tmatcherValExpanded := repl.ReplaceAll(v, \"\")\n\t\t\tvar varStr string\n\t\t\tswitch vv := varValue.(type) {\n\t\t\tcase string:\n\t\t\t\tvarStr = vv\n\t\t\tcase fmt.Stringer:\n\t\t\t\tvarStr = vv.String()\n\t\t\tcase error:\n\t\t\t\tvarStr = vv.Error()\n\t\t\tcase nil:\n\t\t\t\tvarStr = \"\"\n\t\t\tdefault:\n\t\t\t\tvarStr = fmt.Sprintf(\"%v\", vv)\n\t\t\t}\n\t\t\tif varStr == matcherValExpanded {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// MatchVarsRE matches the value of the context variables by a given regular expression.\n//\n// Upon a match, it adds placeholders to the request: `{http.regexp.name.capture_group}`\n// where `name` is the regular expression's name, and `capture_group` is either\n// the named or positional capture group from the expression itself. If no name\n// is given, then the placeholder omits the name: `{http.regexp.capture_group}`\n// (potentially leading to collisions).\ntype MatchVarsRE map[string]*MatchRegexp\n\n// CaddyModule returns the Caddy module information.\nfunc (MatchVarsRE) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.matchers.vars_regexp\",\n\t\tNew: func() caddy.Module { return new(MatchVarsRE) },\n\t}\n}\n\n// UnmarshalCaddyfile implements caddyfile.Unmarshaler.\nfunc (m *MatchVarsRE) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\tif *m == nil {\n\t\t*m = make(map[string]*MatchRegexp)\n\t}\n\t// iterate to merge multiple matchers into one\n\tfor d.Next() {\n\t\tvar first, second, third string\n\t\tif !d.Args(&first, &second) {\n\t\t\treturn d.ArgErr()\n\t\t}\n\n\t\tvar name, field, val string\n\t\tif d.Args(&third) {\n\t\t\tname = first\n\t\t\tfield = second\n\t\t\tval = third\n\t\t} else {\n\t\t\tfield = first\n\t\t\tval = second\n\t\t}\n\n\t\t// Default to the named matcher's name, if no regexp name is provided\n\t\tif name == \"\" {\n\t\t\tname = d.GetContextString(caddyfile.MatcherNameCtxKey)\n\t\t}\n\n\t\t(*m)[field] = &MatchRegexp{Pattern: val, Name: name}\n\t\tif d.NextBlock(0) {\n\t\t\treturn d.Err(\"malformed vars_regexp matcher: blocks are not supported\")\n\t\t}\n\t}\n\treturn nil\n}\n\n// Provision compiles m's regular expressions.\nfunc (m MatchVarsRE) Provision(ctx caddy.Context) error {\n\tfor _, rm := range m {\n\t\terr := rm.Provision(ctx)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Match returns true if r matches m.\nfunc (m MatchVarsRE) Match(r *http.Request) bool {\n\tvars := r.Context().Value(VarsCtxKey).(map[string]any)\n\trepl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)\n\tfor key, val := range m {\n\t\tvar varValue any\n\t\tif strings.HasPrefix(key, \"{\") &&\n\t\t\tstrings.HasSuffix(key, \"}\") &&\n\t\t\tstrings.Count(key, \"{\") == 1 {\n\t\t\tvarValue, _ = repl.Get(strings.Trim(key, \"{}\"))\n\t\t} else {\n\t\t\tvarValue = vars[key]\n\t\t}\n\n\t\tvar varStr string\n\t\tswitch vv := varValue.(type) {\n\t\tcase string:\n\t\t\tvarStr = vv\n\t\tcase fmt.Stringer:\n\t\t\tvarStr = vv.String()\n\t\tcase error:\n\t\t\tvarStr = vv.Error()\n\t\tcase nil:\n\t\t\tvarStr = \"\"\n\t\tdefault:\n\t\t\tvarStr = fmt.Sprintf(\"%v\", vv)\n\t\t}\n\n\t\tvalExpanded := repl.ReplaceAll(varStr, \"\")\n\t\tif match := val.Match(valExpanded, repl); match {\n\t\t\treturn match\n\t\t}\n\t}\n\treturn false\n}\n\n// Validate validates m's regular expressions.\nfunc (m MatchVarsRE) Validate() error {\n\tfor _, rm := range m {\n\t\terr := rm.Validate()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// GetVar gets a value out of the context's variable table by key.\n// If the key does not exist, the return value will be nil.\nfunc GetVar(ctx context.Context, key string) any {\n\tvarMap, ok := ctx.Value(VarsCtxKey).(map[string]any)\n\tif !ok {\n\t\treturn nil\n\t}\n\treturn varMap[key]\n}\n\n// SetVar sets a value in the context's variable table with\n// the given key. It overwrites any previous value with the\n// same key.\n//\n// If the value is nil (note: non-nil interface with nil\n// underlying value does not count) and the key exists in\n// the table, the key+value will be deleted from the table.\nfunc SetVar(ctx context.Context, key string, value any) {\n\tvarMap, ok := ctx.Value(VarsCtxKey).(map[string]any)\n\tif !ok {\n\t\treturn\n\t}\n\tif value == nil {\n\t\tif _, ok := varMap[key]; ok {\n\t\t\tdelete(varMap, key)\n\t\t\treturn\n\t\t}\n\t}\n\tvarMap[key] = value\n}\n\n// Interface guards\nvar (\n\t_ MiddlewareHandler     = (*VarsMiddleware)(nil)\n\t_ caddyfile.Unmarshaler = (*VarsMiddleware)(nil)\n\t_ RequestMatcher        = (*VarsMatcher)(nil)\n\t_ caddyfile.Unmarshaler = (*VarsMatcher)(nil)\n)\n\n// Path: modules/caddypki/acmeserver/acmeserver.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage acmeserver\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\tweakrand \"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-chi/chi/v5\"\n\t\"github.com/smallstep/certificates/acme\"\n\t\"github.com/smallstep/certificates/acme/api\"\n\tacmeNoSQL \"github.com/smallstep/certificates/acme/db/nosql\"\n\t\"github.com/smallstep/certificates/authority\"\n\t\"github.com/smallstep/certificates/authority/provisioner\"\n\t\"github.com/smallstep/certificates/db\"\n\t\"github.com/smallstep/nosql\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddypki\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(Handler{})\n}\n\n// Handler is an ACME server handler.\ntype Handler struct {\n\t// The ID of the CA to use for signing. This refers to\n\t// the ID given to the CA in the `pki` app. If omitted,\n\t// the default ID is \"local\".\n\tCA string `json:\"ca,omitempty\"`\n\n\t// The lifetime for issued certificates\n\tLifetime caddy.Duration `json:\"lifetime,omitempty\"`\n\n\t// The hostname or IP address by which ACME clients\n\t// will access the server. This is used to populate\n\t// the ACME directory endpoint. If not set, the Host\n\t// header of the request will be used.\n\t// COMPATIBILITY NOTE / TODO: This property may go away in the\n\t// future. Do not rely on this property long-term; check release notes.\n\tHost string `json:\"host,omitempty\"`\n\n\t// The path prefix under which to serve all ACME\n\t// endpoints. All other requests will not be served\n\t// by this handler and will be passed through to\n\t// the next one. Default: \"/acme/\".\n\t// COMPATIBILITY NOTE / TODO: This property may go away in the\n\t// future, as it is currently only required due to\n\t// limitations in the underlying library. Do not rely\n\t// on this property long-term; check release notes.\n\tPathPrefix string `json:\"path_prefix,omitempty\"`\n\n\t// If true, the CA's root will be the issuer instead of\n\t// the intermediate. This is NOT recommended and should\n\t// only be used when devices/clients do not properly\n\t// validate certificate chains. EXPERIMENTAL: Might be\n\t// changed or removed in the future.\n\tSignWithRoot bool `json:\"sign_with_root,omitempty\"`\n\n\t// The addresses of DNS resolvers to use when looking up\n\t// the TXT records for solving DNS challenges.\n\t// It accepts [network addresses](/docs/conventions#network-addresses)\n\t// with port range of only 1. If the host is an IP address,\n\t// it will be dialed directly to resolve the upstream server.\n\t// If the host is not an IP address, the addresses are resolved\n\t// using the [name resolution convention](https://golang.org/pkg/net/#hdr-Name_Resolution)\n\t// of the Go standard library. If the array contains more\n\t// than 1 resolver address, one is chosen at random.\n\tResolvers []string `json:\"resolvers,omitempty\"`\n\n\t// Specify the set of enabled ACME challenges. An empty or absent value\n\t// means all challenges are enabled. Accepted values are:\n\t// \"http-01\", \"dns-01\", \"tls-alpn-01\"\n\tChallenges ACMEChallenges `json:\"challenges,omitempty\" `\n\n\t// The policy to use for issuing certificates\n\tPolicy *Policy `json:\"policy,omitempty\"`\n\n\tlogger    *zap.Logger\n\tresolvers []caddy.NetworkAddress\n\tctx       caddy.Context\n\n\tacmeDB        acme.DB\n\tacmeAuth      *authority.Authority\n\tacmeClient    acme.Client\n\tacmeLinker    acme.Linker\n\tacmeEndpoints http.Handler\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (Handler) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.handlers.acme_server\",\n\t\tNew: func() caddy.Module { return new(Handler) },\n\t}\n}\n\n// Provision sets up the ACME server handler.\nfunc (ash *Handler) Provision(ctx caddy.Context) error {\n\tash.ctx = ctx\n\tash.logger = ctx.Logger()\n\n\t// set some defaults\n\tif ash.CA == \"\" {\n\t\tash.CA = caddypki.DefaultCAID\n\t}\n\tif ash.PathPrefix == \"\" {\n\t\tash.PathPrefix = defaultPathPrefix\n\t}\n\tif ash.Lifetime == 0 {\n\t\tash.Lifetime = caddy.Duration(12 * time.Hour)\n\t}\n\tif len(ash.Challenges) > 0 {\n\t\tif err := ash.Challenges.validate(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// get a reference to the configured CA\n\tappModule, err := ctx.App(\"pki\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tpkiApp := appModule.(*caddypki.PKI)\n\tca, err := pkiApp.GetCA(ctx, ash.CA)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// make sure leaf cert lifetime is less than the intermediate cert lifetime. this check only\n\t// applies for caddy-managed intermediate certificates\n\tif ca.Intermediate == nil && ash.Lifetime >= ca.IntermediateLifetime {\n\t\treturn fmt.Errorf(\"certificate lifetime (%s) should be less than intermediate certificate lifetime (%s)\", time.Duration(ash.Lifetime), time.Duration(ca.IntermediateLifetime))\n\t}\n\n\tdatabase, err := ash.openDatabase()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tauthorityConfig := caddypki.AuthorityConfig{\n\t\tSignWithRoot: ash.SignWithRoot,\n\t\tAuthConfig: &authority.AuthConfig{\n\t\t\tProvisioners: provisioner.List{\n\t\t\t\t&provisioner.ACME{\n\t\t\t\t\tName:       ash.CA,\n\t\t\t\t\tChallenges: ash.Challenges.toSmallstepType(),\n\t\t\t\t\tOptions: &provisioner.Options{\n\t\t\t\t\t\tX509: ash.Policy.normalizeRules(),\n\t\t\t\t\t},\n\t\t\t\t\tType: provisioner.TypeACME.String(),\n\t\t\t\t\tClaims: &provisioner.Claims{\n\t\t\t\t\t\tMinTLSDur:     &provisioner.Duration{Duration: 5 * time.Minute},\n\t\t\t\t\t\tMaxTLSDur:     &provisioner.Duration{Duration: 24 * time.Hour * 365},\n\t\t\t\t\t\tDefaultTLSDur: &provisioner.Duration{Duration: time.Duration(ash.Lifetime)},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tDB: database,\n\t}\n\n\tash.acmeAuth, err = ca.NewAuthority(authorityConfig)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tash.acmeDB, err = acmeNoSQL.New(ash.acmeAuth.GetDatabase().(nosql.DB))\n\tif err != nil {\n\t\treturn fmt.Errorf(\"configuring ACME DB: %v\", err)\n\t}\n\n\tash.acmeClient, err = ash.makeClient()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tash.acmeLinker = acme.NewLinker(\n\t\tash.Host,\n\t\tstrings.Trim(ash.PathPrefix, \"/\"),\n\t)\n\n\t// extract its http.Handler so we can use it directly\n\tr := chi.NewRouter()\n\tr.Route(ash.PathPrefix, func(r chi.Router) {\n\t\tapi.Route(r)\n\t})\n\tash.acmeEndpoints = r\n\n\treturn nil\n}\n\nfunc (ash Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {\n\tif strings.HasPrefix(r.URL.Path, ash.PathPrefix) {\n\t\tacmeCtx := acme.NewContext(\n\t\t\tr.Context(),\n\t\t\tash.acmeDB,\n\t\t\tash.acmeClient,\n\t\t\tash.acmeLinker,\n\t\t\tnil,\n\t\t)\n\t\tacmeCtx = authority.NewContext(acmeCtx, ash.acmeAuth)\n\t\tr = r.WithContext(acmeCtx)\n\n\t\tash.acmeEndpoints.ServeHTTP(w, r)\n\t\treturn nil\n\t}\n\treturn next.ServeHTTP(w, r)\n}\n\nfunc (ash Handler) getDatabaseKey() string {\n\tkey := ash.CA\n\tkey = strings.ToLower(key)\n\tkey = strings.TrimSpace(key)\n\treturn keyCleaner.ReplaceAllLiteralString(key, \"\")\n}\n\n// Cleanup implements caddy.CleanerUpper and closes any idle databases.\nfunc (ash Handler) Cleanup() error {\n\tkey := ash.getDatabaseKey()\n\tdeleted, err := databasePool.Delete(key)\n\tif deleted {\n\t\tash.logger.Debug(\"unloading unused CA database\", zap.String(\"db_key\", key))\n\t}\n\tif err != nil {\n\t\tash.logger.Error(\"closing CA database\", zap.String(\"db_key\", key), zap.Error(err))\n\t}\n\treturn err\n}\n\nfunc (ash Handler) openDatabase() (*db.AuthDB, error) {\n\tkey := ash.getDatabaseKey()\n\tdatabase, loaded, err := databasePool.LoadOrNew(key, func() (caddy.Destructor, error) {\n\t\tdbFolder := filepath.Join(caddy.AppDataDir(), \"acme_server\", key)\n\t\tdbPath := filepath.Join(dbFolder, \"db\")\n\n\t\terr := os.MkdirAll(dbFolder, 0o755)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"making folder for CA database: %v\", err)\n\t\t}\n\n\t\tdbConfig := &db.Config{\n\t\t\tType:       \"bbolt\",\n\t\t\tDataSource: dbPath,\n\t\t}\n\t\tdatabase, err := db.New(dbConfig)\n\t\treturn databaseCloser{&database}, err\n\t})\n\n\tif loaded {\n\t\tash.logger.Debug(\"loaded preexisting CA database\", zap.String(\"db_key\", key))\n\t}\n\n\treturn database.(databaseCloser).DB, err\n}\n\n// makeClient creates an ACME client which will use a custom\n// resolver instead of net.DefaultResolver.\nfunc (ash Handler) makeClient() (acme.Client, error) {\n\tfor _, v := range ash.Resolvers {\n\t\taddr, err := caddy.ParseNetworkAddressWithDefaults(v, \"udp\", 53)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif addr.PortRangeSize() != 1 {\n\t\t\treturn nil, fmt.Errorf(\"resolver address must have exactly one address; cannot call %v\", addr)\n\t\t}\n\t\tash.resolvers = append(ash.resolvers, addr)\n\t}\n\n\tvar resolver *net.Resolver\n\tif len(ash.resolvers) != 0 {\n\t\tdialer := &net.Dialer{\n\t\t\tTimeout: 2 * time.Second,\n\t\t}\n\t\tresolver = &net.Resolver{\n\t\t\tPreferGo: true,\n\t\t\tDial: func(ctx context.Context, network, address string) (net.Conn, error) {\n\t\t\t\t//nolint:gosec\n\t\t\t\taddr := ash.resolvers[weakrand.Intn(len(ash.resolvers))]\n\t\t\t\treturn dialer.DialContext(ctx, addr.Network, addr.JoinHostPort(0))\n\t\t\t},\n\t\t}\n\t} else {\n\t\tresolver = net.DefaultResolver\n\t}\n\n\treturn resolverClient{\n\t\tClient:   acme.NewClient(),\n\t\tresolver: resolver,\n\t\tctx:      ash.ctx,\n\t}, nil\n}\n\ntype resolverClient struct {\n\tacme.Client\n\n\tresolver *net.Resolver\n\tctx      context.Context\n}\n\nfunc (c resolverClient) LookupTxt(name string) ([]string, error) {\n\treturn c.resolver.LookupTXT(c.ctx, name)\n}\n\nconst defaultPathPrefix = \"/acme/\"\n\nvar (\n\tkeyCleaner   = regexp.MustCompile(`[^\\w.-_]`)\n\tdatabasePool = caddy.NewUsagePool()\n)\n\ntype databaseCloser struct {\n\tDB *db.AuthDB\n}\n\nfunc (closer databaseCloser) Destruct() error {\n\treturn (*closer.DB).Shutdown()\n}\n\n// Interface guards\nvar (\n\t_ caddyhttp.MiddlewareHandler = (*Handler)(nil)\n\t_ caddy.Provisioner           = (*Handler)(nil)\n)\n\n// Path: modules/caddypki/acmeserver/challenges.go\npackage acmeserver\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/smallstep/certificates/authority/provisioner\"\n)\n\n// ACMEChallenge is an opaque string that represents supported ACME challenges.\ntype ACMEChallenge string\n\nconst (\n\tHTTP_01     ACMEChallenge = \"http-01\"\n\tDNS_01      ACMEChallenge = \"dns-01\"\n\tTLS_ALPN_01 ACMEChallenge = \"tls-alpn-01\"\n)\n\n// validate checks if the given challenge is supported.\nfunc (c ACMEChallenge) validate() error {\n\tswitch c {\n\tcase HTTP_01, DNS_01, TLS_ALPN_01:\n\t\treturn nil\n\tdefault:\n\t\treturn fmt.Errorf(\"acme challenge %q is not supported\", c)\n\t}\n}\n\n// The unmarshaller first marshals the value into a string. Then it\n// trims any space around it and lowercase it for normaliztion. The\n// method does not and should not validate the value within accepted enums.\nfunc (c *ACMEChallenge) UnmarshalJSON(b []byte) error {\n\tvar s string\n\tif err := json.Unmarshal(b, &s); err != nil {\n\t\treturn err\n\t}\n\t*c = ACMEChallenge(strings.ToLower(strings.TrimSpace(s)))\n\treturn nil\n}\n\n// String returns a string representation of the challenge.\nfunc (c ACMEChallenge) String() string {\n\treturn strings.ToLower(string(c))\n}\n\n// ACMEChallenges is a list of ACME challenges.\ntype ACMEChallenges []ACMEChallenge\n\n// validate checks if the given challenges are supported.\nfunc (c ACMEChallenges) validate() error {\n\tfor _, ch := range c {\n\t\tif err := ch.validate(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (c ACMEChallenges) toSmallstepType() []provisioner.ACMEChallenge {\n\tif len(c) == 0 {\n\t\treturn nil\n\t}\n\tac := make([]provisioner.ACMEChallenge, len(c))\n\tfor i, ch := range c {\n\t\tac[i] = provisioner.ACMEChallenge(ch)\n\t}\n\treturn ac\n}\n\n\nfunc stringToChallenges(chs []string) ACMEChallenges {\n\tchallenges := make(ACMEChallenges, len(chs))\n\tfor i, ch := range chs {\n\t\tchallenges[i] = ACMEChallenge(ch)\n\t}\n\treturn challenges\n}\n\n// Path: modules/caddypki/acmeserver/policy.go\npackage acmeserver\n\nimport (\n\t\"github.com/smallstep/certificates/authority/policy\"\n\t\"github.com/smallstep/certificates/authority/provisioner\"\n)\n\n// Policy defines the criteria for the ACME server\n// of when to issue a certificate. Refer to the\n// [Certificate Issuance Policy](https://smallstep.com/docs/step-ca/policies/)\n// on Smallstep website for the evaluation criteria.\ntype Policy struct {\n\t// If a rule set is configured to allow a certain type of name,\n\t// all other types of names are automatically denied.\n\tAllow *RuleSet `json:\"allow,omitempty\"`\n\n\t// If a rule set is configured to deny a certain type of name,\n\t// all other types of names are still allowed.\n\tDeny *RuleSet `json:\"deny,omitempty\"`\n\n\t// If set to true, the ACME server will allow issuing wildcard certificates.\n\tAllowWildcardNames bool `json:\"allow_wildcard_names,omitempty\"`\n}\n\n// RuleSet is the specific set of SAN criteria for a certificate\n// to be issued or denied.\ntype RuleSet struct {\n\t// Domains is a list of DNS domains that are allowed to be issued.\n\t// It can be in the form of FQDN for specific domain name, or\n\t// a wildcard domain name format, e.g. *.example.com, to allow\n\t// sub-domains of a domain.\n\tDomains []string `json:\"domains,omitempty\"`\n\n\t// IP ranges in the form of CIDR notation or specific IP addresses\n\t// to be approved or denied for certificates. Non-CIDR IP addresses\n\t// are matched exactly.\n\tIPRanges []string `json:\"ip_ranges,omitempty\"`\n}\n\n// normalizeAllowRules returns `nil` if policy is nil, the `Allow` rule is `nil`,\n// or all rules within the `Allow` rule are empty. Otherwise, it returns the X509NameOptions\n// with the content of the `Allow` rule.\nfunc (p *Policy) normalizeAllowRules() *policy.X509NameOptions {\n\tif (p == nil) || (p.Allow == nil) || (len(p.Allow.Domains) == 0 && len(p.Allow.IPRanges) == 0) {\n\t\treturn nil\n\t}\n\treturn &policy.X509NameOptions{\n\t\tDNSDomains: p.Allow.Domains,\n\t\tIPRanges:   p.Allow.IPRanges,\n\t}\n}\n\n// normalizeDenyRules returns `nil` if policy is nil, the `Deny` rule is `nil`,\n// or all rules within the `Deny` rule are empty. Otherwise, it returns the X509NameOptions\n// with the content of the `Deny` rule.\nfunc (p *Policy) normalizeDenyRules() *policy.X509NameOptions {\n\tif (p == nil) || (p.Deny == nil) || (len(p.Deny.Domains) == 0 && len(p.Deny.IPRanges) == 0) {\n\t\treturn nil\n\t}\n\treturn &policy.X509NameOptions{\n\t\tDNSDomains: p.Deny.Domains,\n\t\tIPRanges:   p.Deny.IPRanges,\n\t}\n}\n\n// normalizeRules returns `nil` if policy is nil, the `Allow` and `Deny` rules are `nil`,\nfunc (p *Policy) normalizeRules() *provisioner.X509Options {\n\tif p == nil {\n\t\treturn nil\n\t}\n\n\tallow := p.normalizeAllowRules()\n\tdeny := p.normalizeDenyRules()\n\tif allow == nil && deny == nil && !p.AllowWildcardNames {\n\t\treturn nil\n\t}\n\n\treturn &provisioner.X509Options{\n\t\tAllowedNames:       allow,\n\t\tDeniedNames:        deny,\n\t\tAllowWildcardNames: p.AllowWildcardNames,\n\t}\n}\n\n// Path: modules/caddypki/acmeserver/policy_test.go\npackage acmeserver\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/smallstep/certificates/authority/policy\"\n\t\"github.com/smallstep/certificates/authority/provisioner\"\n)\n\nfunc TestPolicyNormalizeAllowRules(t *testing.T) {\n\ttype fields struct {\n\t\tAllow              *RuleSet\n\t\tDeny               *RuleSet\n\t\tAllowWildcardNames bool\n\t}\n\ttests := []struct {\n\t\tname   string\n\t\tfields fields\n\t\twant   *policy.X509NameOptions\n\t}{\n\t\t{\n\t\t\tname:   \"providing no rules results in 'nil'\",\n\t\t\tfields: fields{},\n\t\t\twant:   nil,\n\t\t},\n\t\t{\n\t\t\tname: \"providing 'nil' Allow rules results in 'nil', regardless of Deny rules\",\n\t\t\tfields: fields{\n\t\t\t\tAllow:              nil,\n\t\t\t\tDeny:               &RuleSet{},\n\t\t\t\tAllowWildcardNames: true,\n\t\t\t},\n\t\t\twant: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"providing empty Allow rules results in 'nil', regardless of Deny rules\",\n\t\t\tfields: fields{\n\t\t\t\tAllow: &RuleSet{\n\t\t\t\t\tDomains:  []string{},\n\t\t\t\t\tIPRanges: []string{},\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"rules configured in Allow are returned in X509NameOptions\",\n\t\t\tfields: fields{\n\t\t\t\tAllow: &RuleSet{\n\t\t\t\t\tDomains:  []string{\"example.com\"},\n\t\t\t\t\tIPRanges: []string{\"127.0.0.1/32\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: &policy.X509NameOptions{\n\t\t\t\tDNSDomains: []string{\"example.com\"},\n\t\t\t\tIPRanges:   []string{\"127.0.0.1/32\"},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tp := &Policy{\n\t\t\t\tAllow:              tt.fields.Allow,\n\t\t\t\tDeny:               tt.fields.Deny,\n\t\t\t\tAllowWildcardNames: tt.fields.AllowWildcardNames,\n\t\t\t}\n\t\t\tif got := p.normalizeAllowRules(); !reflect.DeepEqual(got, tt.want) {\n\t\t\t\tt.Errorf(\"Policy.normalizeAllowRules() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestPolicy_normalizeDenyRules(t *testing.T) {\n\ttype fields struct {\n\t\tAllow              *RuleSet\n\t\tDeny               *RuleSet\n\t\tAllowWildcardNames bool\n\t}\n\ttests := []struct {\n\t\tname   string\n\t\tfields fields\n\t\twant   *policy.X509NameOptions\n\t}{\n\t\t{\n\t\t\tname:   \"providing no rules results in 'nil'\",\n\t\t\tfields: fields{},\n\t\t\twant:   nil,\n\t\t},\n\t\t{\n\t\t\tname: \"providing 'nil' Deny rules results in 'nil', regardless of Allow rules\",\n\t\t\tfields: fields{\n\t\t\t\tDeny:               nil,\n\t\t\t\tAllow:              &RuleSet{},\n\t\t\t\tAllowWildcardNames: true,\n\t\t\t},\n\t\t\twant: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"providing empty Deny rules results in 'nil', regardless of Allow rules\",\n\t\t\tfields: fields{\n\t\t\t\tDeny: &RuleSet{\n\t\t\t\t\tDomains:  []string{},\n\t\t\t\t\tIPRanges: []string{},\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: nil,\n\t\t},\n\t\t{\n\t\t\tname: \"rules configured in Deny are returned in X509NameOptions\",\n\t\t\tfields: fields{\n\t\t\t\tDeny: &RuleSet{\n\t\t\t\t\tDomains:  []string{\"example.com\"},\n\t\t\t\t\tIPRanges: []string{\"127.0.0.1/32\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\twant: &policy.X509NameOptions{\n\t\t\t\tDNSDomains: []string{\"example.com\"},\n...\n", "needle_token_start": 12898, "needle_token_end": 12971, "code_context_ntokens": 15220, "output": ["```go\nfunc stringToChallenges(chs []string) ACMEChallenges {\n\tchallenges := make(ACMEChallenges, len(chs))\n\tfor i, ch := range chs {\n\t\tchallenges[i] = ACMEChallenge(ch)\n\t}\n\treturn challenges\n}\n```"]}
{"repo": "caddyserver/caddy", "name": "hashCookie", "language": "go", "path": "modules/caddyhttp/reverseproxy/selectionpolicies.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a secure hash representation of a given string using a secret key. This is typically used for creating secure identifiers that can be used in cookies to maintain session integrity and security.\n2. **Input**: It takes two strings as inputs: a secret key and the data string that needs to be hashed.\n3. **Output**: Returns a hexadecimal string representation of the hashed data. If an error occurs during the hashing process, it returns an error alongside the hash.\n4. **Procedure**: The function initializes a new HMAC hasher using SHA-256 as the underlying hash function and the provided secret key. It then writes the data string to this hasher. After processing the data, it converts the resulting hash sum into a hexadecimal string format and returns this string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: modules/caddyhttp/reverseproxy/httptransport.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverseproxy\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\tweakrand \"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"reflect\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pires/go-proxyproto\"\n\t\"go.uber.org/zap\"\n\t\"golang.org/x/net/http2\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddytls\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(HTTPTransport{})\n}\n\n// HTTPTransport is essentially a configuration wrapper for http.Transport.\n// It defines a JSON structure useful when configuring the HTTP transport\n// for Caddy's reverse proxy. It builds its http.Transport at Provision.\ntype HTTPTransport struct {\n\t// TODO: It's possible that other transports (like fastcgi) might be\n\t// able to borrow/use at least some of these config fields; if so,\n\t// maybe move them into a type called CommonTransport and embed it?\n\n\t// Configures the DNS resolver used to resolve the IP address of upstream hostnames.\n\tResolver *UpstreamResolver `json:\"resolver,omitempty\"`\n\n\t// Configures TLS to the upstream. Setting this to an empty struct\n\t// is sufficient to enable TLS with reasonable defaults.\n\tTLS *TLSConfig `json:\"tls,omitempty\"`\n\n\t// Configures HTTP Keep-Alive (enabled by default). Should only be\n\t// necessary if rigorous testing has shown that tuning this helps\n\t// improve performance.\n\tKeepAlive *KeepAlive `json:\"keep_alive,omitempty\"`\n\n\t// Whether to enable compression to upstream. Default: true\n\tCompression *bool `json:\"compression,omitempty\"`\n\n\t// Maximum number of connections per host. Default: 0 (no limit)\n\tMaxConnsPerHost int `json:\"max_conns_per_host,omitempty\"`\n\n\t// If non-empty, which PROXY protocol version to send when\n\t// connecting to an upstream. Default: off.\n\tProxyProtocol string `json:\"proxy_protocol,omitempty\"`\n\n\t// URL to the server that the HTTP transport will use to proxy\n\t// requests to the upstream. See http.Transport.Proxy for\n\t// information regarding supported protocols. This value takes\n\t// precedence over `HTTP_PROXY`, etc.\n\t//\n\t// Providing a value to this parameter results in\n\t// requests flowing through the reverse_proxy in the following\n\t// way:\n\t//\n\t// User Agent ->\n\t//  reverse_proxy ->\n\t//  forward_proxy_url -> upstream\n\t//\n\t// Default: http.ProxyFromEnvironment\n\tForwardProxyURL string `json:\"forward_proxy_url,omitempty\"`\n\n\t// How long to wait before timing out trying to connect to\n\t// an upstream. Default: `3s`.\n\tDialTimeout caddy.Duration `json:\"dial_timeout,omitempty\"`\n\n\t// How long to wait before spawning an RFC 6555 Fast Fallback\n\t// connection. A negative value disables this. Default: `300ms`.\n\tFallbackDelay caddy.Duration `json:\"dial_fallback_delay,omitempty\"`\n\n\t// How long to wait for reading response headers from server. Default: No timeout.\n\tResponseHeaderTimeout caddy.Duration `json:\"response_header_timeout,omitempty\"`\n\n\t// The length of time to wait for a server's first response\n\t// headers after fully writing the request headers if the\n\t// request has a header \"Expect: 100-continue\". Default: No timeout.\n\tExpectContinueTimeout caddy.Duration `json:\"expect_continue_timeout,omitempty\"`\n\n\t// The maximum bytes to read from response headers. Default: `10MiB`.\n\tMaxResponseHeaderSize int64 `json:\"max_response_header_size,omitempty\"`\n\n\t// The size of the write buffer in bytes. Default: `4KiB`.\n\tWriteBufferSize int `json:\"write_buffer_size,omitempty\"`\n\n\t// The size of the read buffer in bytes. Default: `4KiB`.\n\tReadBufferSize int `json:\"read_buffer_size,omitempty\"`\n\n\t// The maximum time to wait for next read from backend. Default: no timeout.\n\tReadTimeout caddy.Duration `json:\"read_timeout,omitempty\"`\n\n\t// The maximum time to wait for next write to backend. Default: no timeout.\n\tWriteTimeout caddy.Duration `json:\"write_timeout,omitempty\"`\n\n\t// The versions of HTTP to support. As a special case, \"h2c\"\n\t// can be specified to use H2C (HTTP/2 over Cleartext) to the\n\t// upstream (this feature is experimental and subject to\n\t// change or removal). Default: [\"1.1\", \"2\"]\n\tVersions []string `json:\"versions,omitempty\"`\n\n\t// The pre-configured underlying HTTP transport.\n\tTransport *http.Transport `json:\"-\"`\n\n\th2cTransport *http2.Transport\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (HTTPTransport) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.transport.http\",\n\t\tNew: func() caddy.Module { return new(HTTPTransport) },\n\t}\n}\n\n// Provision sets up h.Transport with a *http.Transport\n// that is ready to use.\nfunc (h *HTTPTransport) Provision(ctx caddy.Context) error {\n\tif len(h.Versions) == 0 {\n\t\th.Versions = []string{\"1.1\", \"2\"}\n\t}\n\n\trt, err := h.NewTransport(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\th.Transport = rt\n\n\treturn nil\n}\n\n// NewTransport builds a standard-lib-compatible http.Transport value from h.\nfunc (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, error) {\n\t// Set keep-alive defaults if it wasn't otherwise configured\n\tif h.KeepAlive == nil {\n\t\th.KeepAlive = &KeepAlive{\n\t\t\tProbeInterval:       caddy.Duration(30 * time.Second),\n\t\t\tIdleConnTimeout:     caddy.Duration(2 * time.Minute),\n\t\t\tMaxIdleConnsPerHost: 32, // seems about optimal, see #2805\n\t\t}\n\t}\n\n\t// Set a relatively short default dial timeout.\n\t// This is helpful to make load-balancer retries more speedy.\n\tif h.DialTimeout == 0 {\n\t\th.DialTimeout = caddy.Duration(3 * time.Second)\n\t}\n\n\tdialer := &net.Dialer{\n\t\tTimeout:       time.Duration(h.DialTimeout),\n\t\tFallbackDelay: time.Duration(h.FallbackDelay),\n\t}\n\n\tif h.Resolver != nil {\n\t\terr := h.Resolver.ParseAddresses()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\td := &net.Dialer{\n\t\t\tTimeout:       time.Duration(h.DialTimeout),\n\t\t\tFallbackDelay: time.Duration(h.FallbackDelay),\n\t\t}\n\t\tdialer.Resolver = &net.Resolver{\n\t\t\tPreferGo: true,\n\t\t\tDial: func(ctx context.Context, _, _ string) (net.Conn, error) {\n\t\t\t\t//nolint:gosec\n\t\t\t\taddr := h.Resolver.netAddrs[weakrand.Intn(len(h.Resolver.netAddrs))]\n\t\t\t\treturn d.DialContext(ctx, addr.Network, addr.JoinHostPort(0))\n\t\t\t},\n\t\t}\n\t}\n\n\tdialContext := func(ctx context.Context, network, address string) (net.Conn, error) {\n\t\t// For unix socket upstreams, we need to recover the dial info from\n\t\t// the request's context, because the Host on the request's URL\n\t\t// will have been modified by directing the request, overwriting\n\t\t// the unix socket filename.\n\t\t// Also, we need to avoid overwriting the address at this point\n\t\t// when not necessary, because http.ProxyFromEnvironment may have\n\t\t// modified the address according to the user's env proxy config.\n\t\tif dialInfo, ok := GetDialInfo(ctx); ok {\n\t\t\tif strings.HasPrefix(dialInfo.Network, \"unix\") {\n\t\t\t\tnetwork = dialInfo.Network\n\t\t\t\taddress = dialInfo.Address\n\t\t\t}\n\t\t}\n\n\t\tconn, err := dialer.DialContext(ctx, network, address)\n\t\tif err != nil {\n\t\t\t// identify this error as one that occurred during\n\t\t\t// dialing, which can be important when trying to\n\t\t\t// decide whether to retry a request\n\t\t\treturn nil, DialError{err}\n\t\t}\n\n\t\tif h.ProxyProtocol != \"\" {\n\t\t\tproxyProtocolInfo, ok := caddyhttp.GetVar(ctx, proxyProtocolInfoVarKey).(ProxyProtocolInfo)\n\t\t\tif !ok {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to get proxy protocol info from context\")\n\t\t\t}\n\t\t\theader := proxyproto.Header{\n\t\t\t\tSourceAddr: &net.TCPAddr{\n\t\t\t\t\tIP:   proxyProtocolInfo.AddrPort.Addr().AsSlice(),\n\t\t\t\t\tPort: int(proxyProtocolInfo.AddrPort.Port()),\n\t\t\t\t\tZone: proxyProtocolInfo.AddrPort.Addr().Zone(),\n\t\t\t\t},\n\t\t\t}\n\t\t\t// The src and dst have to be of the same address family. As we don't know the original\n\t\t\t// dst address (it's kind of impossible to know) and this address is generally of very\n\t\t\t// little interest, we just set it to all zeros.\n\t\t\tswitch {\n\t\t\tcase proxyProtocolInfo.AddrPort.Addr().Is4():\n\t\t\t\theader.TransportProtocol = proxyproto.TCPv4\n\t\t\t\theader.DestinationAddr = &net.TCPAddr{\n\t\t\t\t\tIP: net.IPv4zero,\n\t\t\t\t}\n\t\t\tcase proxyProtocolInfo.AddrPort.Addr().Is6():\n\t\t\t\theader.TransportProtocol = proxyproto.TCPv6\n\t\t\t\theader.DestinationAddr = &net.TCPAddr{\n\t\t\t\t\tIP: net.IPv6zero,\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\treturn nil, fmt.Errorf(\"unexpected remote addr type in proxy protocol info\")\n\t\t\t}\n\n\t\t\tswitch h.ProxyProtocol {\n\t\t\tcase \"v1\":\n\t\t\t\theader.Version = 1\n\t\t\t\tcaddyCtx.Logger().Debug(\"sending proxy protocol header v1\", zap.Any(\"header\", header))\n\t\t\tcase \"v2\":\n\t\t\t\theader.Version = 2\n\t\t\t\tcaddyCtx.Logger().Debug(\"sending proxy protocol header v2\", zap.Any(\"header\", header))\n\t\t\tdefault:\n\t\t\t\treturn nil, fmt.Errorf(\"unexpected proxy protocol version\")\n\t\t\t}\n\t\t\t_, err = header.WriteTo(conn)\n\t\t\tif err != nil {\n\t\t\t\t// identify this error as one that occurred during\n\t\t\t\t// dialing, which can be important when trying to\n\t\t\t\t// decide whether to retry a request\n\t\t\t\treturn nil, DialError{err}\n\t\t\t}\n\t\t}\n\n\t\t// if read/write timeouts are configured and this is a TCP connection,\n\t\t// enforce the timeouts by wrapping the connection with our own type\n\t\tif tcpConn, ok := conn.(*net.TCPConn); ok && (h.ReadTimeout > 0 || h.WriteTimeout > 0) {\n\t\t\tconn = &tcpRWTimeoutConn{\n\t\t\t\tTCPConn:      tcpConn,\n\t\t\t\treadTimeout:  time.Duration(h.ReadTimeout),\n\t\t\t\twriteTimeout: time.Duration(h.WriteTimeout),\n\t\t\t\tlogger:       caddyCtx.Logger(),\n\t\t\t}\n\t\t}\n\n\t\treturn conn, nil\n\t}\n\n\t// negotiate any HTTP/SOCKS proxy for the HTTP transport\n\tvar proxy func(*http.Request) (*url.URL, error)\n\tif h.ForwardProxyURL != \"\" {\n\t\tpUrl, err := url.Parse(h.ForwardProxyURL)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"failed to parse transport proxy url: %v\", err)\n\t\t}\n\t\tcaddyCtx.Logger().Info(\"setting transport proxy url\", zap.String(\"url\", h.ForwardProxyURL))\n\t\tproxy = http.ProxyURL(pUrl)\n\t} else {\n\t\tproxy = http.ProxyFromEnvironment\n\t}\n\n\trt := &http.Transport{\n\t\tProxy:                  proxy,\n\t\tDialContext:            dialContext,\n\t\tMaxConnsPerHost:        h.MaxConnsPerHost,\n\t\tResponseHeaderTimeout:  time.Duration(h.ResponseHeaderTimeout),\n\t\tExpectContinueTimeout:  time.Duration(h.ExpectContinueTimeout),\n\t\tMaxResponseHeaderBytes: h.MaxResponseHeaderSize,\n\t\tWriteBufferSize:        h.WriteBufferSize,\n\t\tReadBufferSize:         h.ReadBufferSize,\n\t}\n\n\tif h.TLS != nil {\n\t\trt.TLSHandshakeTimeout = time.Duration(h.TLS.HandshakeTimeout)\n\t\tvar err error\n\t\trt.TLSClientConfig, err = h.TLS.MakeTLSClientConfig(caddyCtx)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"making TLS client config: %v\", err)\n\t\t}\n\t}\n\n\tif h.KeepAlive != nil {\n\t\tdialer.KeepAlive = time.Duration(h.KeepAlive.ProbeInterval)\n\t\tif h.KeepAlive.Enabled != nil {\n\t\t\trt.DisableKeepAlives = !*h.KeepAlive.Enabled\n\t\t}\n\t\trt.MaxIdleConns = h.KeepAlive.MaxIdleConns\n\t\trt.MaxIdleConnsPerHost = h.KeepAlive.MaxIdleConnsPerHost\n\t\trt.IdleConnTimeout = time.Duration(h.KeepAlive.IdleConnTimeout)\n\t}\n\n\t// The proxy protocol header can only be sent once right after opening the connection.\n\t// So single connection must not be used for multiple requests, which can potentially\n\t// come from different clients.\n\tif !rt.DisableKeepAlives && h.ProxyProtocol != \"\" {\n\t\tcaddyCtx.Logger().Warn(\"disabling keepalives, they are incompatible with using PROXY protocol\")\n\t\trt.DisableKeepAlives = true\n\t}\n\n\tif h.Compression != nil {\n\t\trt.DisableCompression = !*h.Compression\n\t}\n\n\tif sliceContains(h.Versions, \"2\") {\n\t\tif err := http2.ConfigureTransport(rt); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// if h2c is enabled, configure its transport (std lib http.Transport\n\t// does not \"HTTP/2 over cleartext TCP\")\n\tif sliceContains(h.Versions, \"h2c\") {\n\t\t// crafting our own http2.Transport doesn't allow us to utilize\n\t\t// most of the customizations/preferences on the http.Transport,\n\t\t// because, for some reason, only http2.ConfigureTransport()\n\t\t// is allowed to set the unexported field that refers to a base\n\t\t// http.Transport config; oh well\n\t\th2t := &http2.Transport{\n\t\t\t// kind of a hack, but for plaintext/H2C requests, pretend to dial TLS\n\t\t\tDialTLSContext: func(ctx context.Context, network, address string, _ *tls.Config) (net.Conn, error) {\n\t\t\t\treturn dialContext(ctx, network, address)\n\t\t\t},\n\t\t\tAllowHTTP: true,\n\t\t}\n\t\tif h.Compression != nil {\n\t\t\th2t.DisableCompression = !*h.Compression\n\t\t}\n\t\th.h2cTransport = h2t\n\t}\n\n\treturn rt, nil\n}\n\n// replaceTLSServername checks TLS servername to see if it needs replacing\n// if it does need replacing, it creates a new cloned HTTPTransport object to avoid any races\n// and does the replacing of the TLS servername on that and returns the new object\n// if no replacement is necessary it returns the original\nfunc (h *HTTPTransport) replaceTLSServername(repl *caddy.Replacer) *HTTPTransport {\n\t// check whether we have TLS and need to replace the servername in the TLSClientConfig\n\tif h.TLSEnabled() && strings.Contains(h.TLS.ServerName, \"{\") {\n\t\t// make a new h, \"copy\" the parts we don't need to touch, add a new *tls.Config and replace servername\n\t\tnewtransport := &HTTPTransport{\n\t\t\tResolver:              h.Resolver,\n\t\t\tTLS:                   h.TLS,\n\t\t\tKeepAlive:             h.KeepAlive,\n\t\t\tCompression:           h.Compression,\n\t\t\tMaxConnsPerHost:       h.MaxConnsPerHost,\n\t\t\tDialTimeout:           h.DialTimeout,\n\t\t\tFallbackDelay:         h.FallbackDelay,\n\t\t\tResponseHeaderTimeout: h.ResponseHeaderTimeout,\n\t\t\tExpectContinueTimeout: h.ExpectContinueTimeout,\n\t\t\tMaxResponseHeaderSize: h.MaxResponseHeaderSize,\n\t\t\tWriteBufferSize:       h.WriteBufferSize,\n\t\t\tReadBufferSize:        h.ReadBufferSize,\n\t\t\tVersions:              h.Versions,\n\t\t\tTransport:             h.Transport.Clone(),\n\t\t\th2cTransport:          h.h2cTransport,\n\t\t}\n\t\tnewtransport.Transport.TLSClientConfig.ServerName = repl.ReplaceAll(newtransport.Transport.TLSClientConfig.ServerName, \"\")\n\t\treturn newtransport\n\t}\n\n\treturn h\n}\n\n// RoundTrip implements http.RoundTripper.\n...\n// Path: modules/caddyhttp/reverseproxy/httptransport_test.go\npackage reverseproxy\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n)\n\nfunc TestHTTPTransportUnmarshalCaddyFileWithCaPools(t *testing.T) {\n\tconst test_der_1 = `MIIDSzCCAjOgAwIBAgIUfIRObjWNUA4jxQ/0x8BOCvE2Vw4wDQYJKoZIhvcNAQELBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwODI4MTYyNTU5WhcNMjkwODI1MTYyNTU5WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK5m5elxhQfMp/3aVJ4JnpN9PUSz6LlP6LePAPFU7gqohVVFVtDkChJAG3FNkNQNlieVTja/bgH9IcC6oKbROwdY1h0MvNV8AHHigvl03WuJD8g2ReVFXXwsnrPmKXCFzQyMI6TYk3m2gYrXsZOU1GLnfMRC3KAMRgE2F45twOs9hqG169YJ6mM2eQjzjCHWI6S2/iUYvYxRkCOlYUbLsMD/AhgAf1plzg6LPqNxtdlwxZnA0ytgkmhK67HtzJu0+ovUCsMv0RwcMhsEo9T8nyFAGt9XLZ63X5WpBCTUApaAUhnG0XnerjmUWb6eUWw4zev54sEfY5F3x002iQaW6cECAwEAAaOBkDCBjTAdBgNVHQ4EFgQU4CBUbZsS2GaNIkGRz/cBsD5ivjswUQYDVR0jBEowSIAU4CBUbZsS2GaNIkGRz/cBsD5ivjuhGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghR8hE5uNY1QDiPFD/THwE4K8TZXDjAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAQEAKB3V4HIzoiO/Ch6WMj9bLJ2FGbpkMrcb/Eq01hT5zcfKD66lVS1MlK+cRL446Z2b2KDP1oFyVs+qmrmtdwrWgD+nfe2sBmmIHo9m9KygMkEOfG3MghGTEcS+0cTKEcoHYWYyOqQh6jnedXY8Cdm4GM1hAc9MiL3/sqV8YCVSLNnkoNysmr06/rZ0MCUZPGUtRmfd0heWhrfzAKw2HLgX+RAmpOE2MZqWcjvqKGyaRiaZks4nJkP6521aC2Lgp0HhCz1j8/uQ5ldoDszCnu/iro0NAsNtudTMD+YoLQxLqdleIh6CW+illc2VdXwj7mn6J04yns9jfE2jRjW/yTLFuQ==`\n\ttype args struct {\n\t\td *caddyfile.Dispenser\n\t}\n\ttests := []struct {\n\t\tname              string\n\t\targs              args\n\t\texpectedTLSConfig TLSConfig\n\t\twantErr           bool\n\t}{\n\t\t{\n\t\t\tname: \"tls_trust_pool without a module argument returns an error\",\n\t\t\targs: args{\n\t\t\t\td: caddyfile.NewTestDispenser(\n\t\t\t\t\t`http {\n\t\t\t\t\ttls_trust_pool\n\t\t\t\t}`),\n\t\t\t},\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"providing both 'tls_trust_pool' and 'tls_trusted_ca_certs' returns an error\",\n\t\t\targs: args{\n\t\t\t\td: caddyfile.NewTestDispenser(fmt.Sprintf(\n\t\t\t\t\t`http {\n\t\t\t\t\ttls_trust_pool inline %s\n\t\t\t\t\ttls_trusted_ca_certs %s\n\t\t\t\t}`, test_der_1, test_der_1)),\n\t\t\t},\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"setting 'tls_trust_pool' and 'tls_trusted_ca_certs' produces an error\",\n\t\t\targs: args{\n\t\t\t\td: caddyfile.NewTestDispenser(fmt.Sprintf(\n\t\t\t\t\t`http {\n\t\t\t\t\ttls_trust_pool inline {\n\t\t\t\t\t\ttrust_der\t%s\n\t\t\t\t\t}\n\t\t\t\t\ttls_trusted_ca_certs %s\n\t\t\t\t}`, test_der_1, test_der_1)),\n\t\t\t},\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"using 'inline' tls_trust_pool loads the module successfully\",\n\t\t\targs: args{\n\t\t\t\td: caddyfile.NewTestDispenser(fmt.Sprintf(\n\t\t\t\t\t`http {\n\t\t\t\t\t\ttls_trust_pool inline {\n\t\t\t\t\t\t\ttrust_der\t%s\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t`, test_der_1)),\n\t\t\t},\n\t\t\texpectedTLSConfig: TLSConfig{CARaw: json.RawMessage(fmt.Sprintf(`{\"provider\":\"inline\",\"trusted_ca_certs\":[\"%s\"]}`, test_der_1))},\n\t\t},\n\t\t{\n\t\t\tname: \"setting 'tls_trusted_ca_certs' and 'tls_trust_pool' produces an error\",\n\t\t\targs: args{\n\t\t\t\td: caddyfile.NewTestDispenser(fmt.Sprintf(\n\t\t\t\t\t`http {\n\t\t\t\t\t\ttls_trusted_ca_certs %s\n\t\t\t\t\t\ttls_trust_pool inline {\n\t\t\t\t\t\t\ttrust_der\t%s\n\t\t\t\t\t\t}\n\t\t\t\t}`, test_der_1, test_der_1)),\n\t\t\t},\n\t\t\twantErr: true,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tht := &HTTPTransport{}\n\t\t\tif err := ht.UnmarshalCaddyfile(tt.args.d); (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"HTTPTransport.UnmarshalCaddyfile() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif !tt.wantErr && !reflect.DeepEqual(&tt.expectedTLSConfig, ht.TLS) {\n\t\t\t\tt.Errorf(\"HTTPTransport.UnmarshalCaddyfile() = %v, want %v\", ht, tt.expectedTLSConfig)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: modules/caddyhttp/reverseproxy/selectionpolicies.go\n// Copyright 2015 Matthew Holt and The Caddy Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//     http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage reverseproxy\n\nimport (\n\t\"crypto/hmac\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"fmt\"\n\tweakrand \"math/rand\"\n\t\"net\"\n\t\"net/http\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync/atomic\"\n\n\t\"github.com/cespare/xxhash/v2\"\n\n\t\"github.com/caddyserver/caddy/v2\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig\"\n\t\"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile\"\n\t\"github.com/caddyserver/caddy/v2/modules/caddyhttp\"\n)\n\nfunc init() {\n\tcaddy.RegisterModule(RandomSelection{})\n\tcaddy.RegisterModule(RandomChoiceSelection{})\n\tcaddy.RegisterModule(LeastConnSelection{})\n\tcaddy.RegisterModule(RoundRobinSelection{})\n\tcaddy.RegisterModule(WeightedRoundRobinSelection{})\n\tcaddy.RegisterModule(FirstSelection{})\n\tcaddy.RegisterModule(IPHashSelection{})\n\tcaddy.RegisterModule(ClientIPHashSelection{})\n\tcaddy.RegisterModule(URIHashSelection{})\n\tcaddy.RegisterModule(QueryHashSelection{})\n\tcaddy.RegisterModule(HeaderHashSelection{})\n\tcaddy.RegisterModule(CookieHashSelection{})\n}\n\n// RandomSelection is a policy that selects\n// an available host at random.\ntype RandomSelection struct{}\n\n// CaddyModule returns the Caddy module information.\nfunc (RandomSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.random\",\n\t\tNew: func() caddy.Module { return new(RandomSelection) },\n\t}\n}\n\n// Select returns an available host, if any.\nfunc (r RandomSelection) Select(pool UpstreamPool, request *http.Request, _ http.ResponseWriter) *Upstream {\n\treturn selectRandomHost(pool)\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *RandomSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\treturn nil\n}\n\n// WeightedRoundRobinSelection is a policy that selects\n// a host based on weighted round-robin ordering.\ntype WeightedRoundRobinSelection struct {\n\t// The weight of each upstream in order,\n\t// corresponding with the list of upstreams configured.\n\tWeights     []int `json:\"weights,omitempty\"`\n\tindex       uint32\n\ttotalWeight int\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (WeightedRoundRobinSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID: \"http.reverse_proxy.selection_policies.weighted_round_robin\",\n\t\tNew: func() caddy.Module {\n\t\t\treturn new(WeightedRoundRobinSelection)\n\t\t},\n\t}\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *WeightedRoundRobinSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\n\targs := d.RemainingArgs()\n\tif len(args) == 0 {\n\t\treturn d.ArgErr()\n\t}\n\n\tfor _, weight := range args {\n\t\tweightInt, err := strconv.Atoi(weight)\n\t\tif err != nil {\n\t\t\treturn d.Errf(\"invalid weight value '%s': %v\", weight, err)\n\t\t}\n\t\tif weightInt < 1 {\n\t\t\treturn d.Errf(\"invalid weight value '%s': weight should be non-zero and positive\", weight)\n\t\t}\n\t\tr.Weights = append(r.Weights, weightInt)\n\t}\n\treturn nil\n}\n\n// Provision sets up r.\nfunc (r *WeightedRoundRobinSelection) Provision(ctx caddy.Context) error {\n\tfor _, weight := range r.Weights {\n\t\tr.totalWeight += weight\n\t}\n\treturn nil\n}\n\n// Select returns an available host, if any.\nfunc (r *WeightedRoundRobinSelection) Select(pool UpstreamPool, _ *http.Request, _ http.ResponseWriter) *Upstream {\n\tif len(pool) == 0 {\n\t\treturn nil\n\t}\n\tif len(r.Weights) < 2 {\n\t\treturn pool[0]\n\t}\n\tvar index, totalWeight int\n\tcurrentWeight := int(atomic.AddUint32(&r.index, 1)) % r.totalWeight\n\tfor i, weight := range r.Weights {\n\t\ttotalWeight += weight\n\t\tif currentWeight < totalWeight {\n\t\t\tindex = i\n\t\t\tbreak\n\t\t}\n\t}\n\n\tupstreams := make([]*Upstream, 0, len(r.Weights))\n\tfor _, upstream := range pool {\n\t\tif !upstream.Available() {\n\t\t\tcontinue\n\t\t}\n\t\tupstreams = append(upstreams, upstream)\n\t\tif len(upstreams) == cap(upstreams) {\n\t\t\tbreak\n\t\t}\n\t}\n\tif len(upstreams) == 0 {\n\t\treturn nil\n\t}\n\treturn upstreams[index%len(upstreams)]\n}\n\n// RandomChoiceSelection is a policy that selects\n// two or more available hosts at random, then\n// chooses the one with the least load.\ntype RandomChoiceSelection struct {\n\t// The size of the sub-pool created from the larger upstream pool. The default value\n\t// is 2 and the maximum at selection time is the size of the upstream pool.\n\tChoose int `json:\"choose,omitempty\"`\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (RandomChoiceSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.random_choose\",\n\t\tNew: func() caddy.Module { return new(RandomChoiceSelection) },\n\t}\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *RandomChoiceSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\n\tif !d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\tchooseStr := d.Val()\n\tchoose, err := strconv.Atoi(chooseStr)\n\tif err != nil {\n\t\treturn d.Errf(\"invalid choice value '%s': %v\", chooseStr, err)\n\t}\n\tr.Choose = choose\n\treturn nil\n}\n\n// Provision sets up r.\nfunc (r *RandomChoiceSelection) Provision(ctx caddy.Context) error {\n\tif r.Choose == 0 {\n\t\tr.Choose = 2\n\t}\n\treturn nil\n}\n\n// Validate ensures that r's configuration is valid.\nfunc (r RandomChoiceSelection) Validate() error {\n\tif r.Choose < 2 {\n\t\treturn fmt.Errorf(\"choose must be at least 2\")\n\t}\n\treturn nil\n}\n\n// Select returns an available host, if any.\nfunc (r RandomChoiceSelection) Select(pool UpstreamPool, _ *http.Request, _ http.ResponseWriter) *Upstream {\n\tk := r.Choose\n\tif k > len(pool) {\n\t\tk = len(pool)\n\t}\n\tchoices := make([]*Upstream, k)\n\tfor i, upstream := range pool {\n\t\tif !upstream.Available() {\n\t\t\tcontinue\n\t\t}\n\t\tj := weakrand.Intn(i + 1) //nolint:gosec\n\t\tif j < k {\n\t\t\tchoices[j] = upstream\n\t\t}\n\t}\n\treturn leastRequests(choices)\n}\n\n// LeastConnSelection is a policy that selects the\n// host with the least active requests. If multiple\n// hosts have the same fewest number, one is chosen\n// randomly. The term \"conn\" or \"connection\" is used\n// in this policy name due to its similar meaning in\n// other software, but our load balancer actually\n// counts active requests rather than connections,\n// since these days requests are multiplexed onto\n// shared connections.\ntype LeastConnSelection struct{}\n\n// CaddyModule returns the Caddy module information.\nfunc (LeastConnSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.least_conn\",\n\t\tNew: func() caddy.Module { return new(LeastConnSelection) },\n\t}\n}\n\n// Select selects the up host with the least number of connections in the\n// pool. If more than one host has the same least number of connections,\n// one of the hosts is chosen at random.\nfunc (LeastConnSelection) Select(pool UpstreamPool, _ *http.Request, _ http.ResponseWriter) *Upstream {\n\tvar bestHost *Upstream\n\tvar count int\n\tleastReqs := -1\n\n\tfor _, host := range pool {\n\t\tif !host.Available() {\n\t\t\tcontinue\n\t\t}\n\t\tnumReqs := host.NumRequests()\n\t\tif leastReqs == -1 || numReqs < leastReqs {\n\t\t\tleastReqs = numReqs\n\t\t\tcount = 0\n\t\t}\n\n\t\t// among hosts with same least connections, perform a reservoir\n\t\t// sample: https://en.wikipedia.org/wiki/Reservoir_sampling\n\t\tif numReqs == leastReqs {\n\t\t\tcount++\n\t\t\tif count == 1 || (weakrand.Int()%count) == 0 { //nolint:gosec\n\t\t\t\tbestHost = host\n\t\t\t}\n\t\t}\n\t}\n\n\treturn bestHost\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *LeastConnSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\treturn nil\n}\n\n// RoundRobinSelection is a policy that selects\n// a host based on round-robin ordering.\ntype RoundRobinSelection struct {\n\trobin uint32\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (RoundRobinSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.round_robin\",\n\t\tNew: func() caddy.Module { return new(RoundRobinSelection) },\n\t}\n}\n\n// Select returns an available host, if any.\nfunc (r *RoundRobinSelection) Select(pool UpstreamPool, _ *http.Request, _ http.ResponseWriter) *Upstream {\n\tn := uint32(len(pool))\n\tif n == 0 {\n\t\treturn nil\n\t}\n\tfor i := uint32(0); i < n; i++ {\n\t\trobin := atomic.AddUint32(&r.robin, 1)\n\t\thost := pool[robin%n]\n\t\tif host.Available() {\n\t\t\treturn host\n\t\t}\n\t}\n\treturn nil\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *RoundRobinSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\treturn nil\n}\n\n// FirstSelection is a policy that selects\n// the first available host.\ntype FirstSelection struct{}\n\n// CaddyModule returns the Caddy module information.\nfunc (FirstSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.first\",\n\t\tNew: func() caddy.Module { return new(FirstSelection) },\n\t}\n}\n\n// Select returns an available host, if any.\nfunc (FirstSelection) Select(pool UpstreamPool, _ *http.Request, _ http.ResponseWriter) *Upstream {\n\tfor _, host := range pool {\n\t\tif host.Available() {\n\t\t\treturn host\n\t\t}\n\t}\n\treturn nil\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *FirstSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\treturn nil\n}\n\n// IPHashSelection is a policy that selects a host\n// based on hashing the remote IP of the request.\ntype IPHashSelection struct{}\n\n// CaddyModule returns the Caddy module information.\nfunc (IPHashSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.ip_hash\",\n\t\tNew: func() caddy.Module { return new(IPHashSelection) },\n\t}\n}\n\n// Select returns an available host, if any.\nfunc (IPHashSelection) Select(pool UpstreamPool, req *http.Request, _ http.ResponseWriter) *Upstream {\n\tclientIP, _, err := net.SplitHostPort(req.RemoteAddr)\n\tif err != nil {\n\t\tclientIP = req.RemoteAddr\n\t}\n\treturn hostByHashing(pool, clientIP)\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *IPHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\treturn nil\n}\n\n// ClientIPHashSelection is a policy that selects a host\n// based on hashing the client IP of the request, as determined\n// by the HTTP app's trusted proxies settings.\ntype ClientIPHashSelection struct{}\n\n// CaddyModule returns the Caddy module information.\nfunc (ClientIPHashSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.client_ip_hash\",\n\t\tNew: func() caddy.Module { return new(ClientIPHashSelection) },\n\t}\n}\n\n// Select returns an available host, if any.\nfunc (ClientIPHashSelection) Select(pool UpstreamPool, req *http.Request, _ http.ResponseWriter) *Upstream {\n\taddress := caddyhttp.GetVar(req.Context(), caddyhttp.ClientIPVarKey).(string)\n\tclientIP, _, err := net.SplitHostPort(address)\n\tif err != nil {\n\t\tclientIP = address // no port\n\t}\n\treturn hostByHashing(pool, clientIP)\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *ClientIPHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\treturn nil\n}\n\n// URIHashSelection is a policy that selects a\n// host by hashing the request URI.\ntype URIHashSelection struct{}\n\n// CaddyModule returns the Caddy module information.\nfunc (URIHashSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.uri_hash\",\n\t\tNew: func() caddy.Module { return new(URIHashSelection) },\n\t}\n}\n\n// Select returns an available host, if any.\nfunc (URIHashSelection) Select(pool UpstreamPool, req *http.Request, _ http.ResponseWriter) *Upstream {\n\treturn hostByHashing(pool, req.RequestURI)\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (r *URIHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\tif d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\treturn nil\n}\n\n// QueryHashSelection is a policy that selects\n// a host based on a given request query parameter.\ntype QueryHashSelection struct {\n\t// The query key whose value is to be hashed and used for upstream selection.\n\tKey string `json:\"key,omitempty\"`\n\n\t// The fallback policy to use if the query key is not present. Defaults to `random`.\n\tFallbackRaw json.RawMessage `json:\"fallback,omitempty\" caddy:\"namespace=http.reverse_proxy.selection_policies inline_key=policy\"`\n\tfallback    Selector\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (QueryHashSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.query\",\n\t\tNew: func() caddy.Module { return new(QueryHashSelection) },\n\t}\n}\n\n// Provision sets up the module.\nfunc (s *QueryHashSelection) Provision(ctx caddy.Context) error {\n\tif s.Key == \"\" {\n\t\treturn fmt.Errorf(\"query key is required\")\n\t}\n\tif s.FallbackRaw == nil {\n\t\ts.FallbackRaw = caddyconfig.JSONModuleObject(RandomSelection{}, \"policy\", \"random\", nil)\n\t}\n\tmod, err := ctx.LoadModule(s, \"FallbackRaw\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"loading fallback selection policy: %s\", err)\n\t}\n\ts.fallback = mod.(Selector)\n\treturn nil\n}\n\n// Select returns an available host, if any.\nfunc (s QueryHashSelection) Select(pool UpstreamPool, req *http.Request, _ http.ResponseWriter) *Upstream {\n\t// Since the query may have multiple values for the same key,\n\t// we'll join them to avoid a problem where the user can control\n\t// the upstream that the request goes to by sending multiple values\n\t// for the same key, when the upstream only considers the first value.\n\t// Keep in mind that a client changing the order of the values may\n\t// affect which upstream is selected, but this is a semantically\n\t// different request, because the order of the values is significant.\n\tvals := strings.Join(req.URL.Query()[s.Key], \",\")\n\tif vals == \"\" {\n\t\treturn s.fallback.Select(pool, req, nil)\n\t}\n\treturn hostByHashing(pool, vals)\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (s *QueryHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\n\tif !d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\ts.Key = d.Val()\n\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"fallback\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tif s.FallbackRaw != nil {\n\t\t\t\treturn d.Err(\"fallback selection policy already specified\")\n\t\t\t}\n\t\t\tmod, err := loadFallbackPolicy(d)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ts.FallbackRaw = mod\n\t\tdefault:\n\t\t\treturn d.Errf(\"unrecognized option '%s'\", d.Val())\n\t\t}\n\t}\n\treturn nil\n}\n\n// HeaderHashSelection is a policy that selects\n// a host based on a given request header.\ntype HeaderHashSelection struct {\n\t// The HTTP header field whose value is to be hashed and used for upstream selection.\n\tField string `json:\"field,omitempty\"`\n\n\t// The fallback policy to use if the header is not present. Defaults to `random`.\n\tFallbackRaw json.RawMessage `json:\"fallback,omitempty\" caddy:\"namespace=http.reverse_proxy.selection_policies inline_key=policy\"`\n\tfallback    Selector\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (HeaderHashSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.header\",\n\t\tNew: func() caddy.Module { return new(HeaderHashSelection) },\n\t}\n}\n\n// Provision sets up the module.\nfunc (s *HeaderHashSelection) Provision(ctx caddy.Context) error {\n\tif s.Field == \"\" {\n\t\treturn fmt.Errorf(\"header field is required\")\n\t}\n\tif s.FallbackRaw == nil {\n\t\ts.FallbackRaw = caddyconfig.JSONModuleObject(RandomSelection{}, \"policy\", \"random\", nil)\n\t}\n\tmod, err := ctx.LoadModule(s, \"FallbackRaw\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"loading fallback selection policy: %s\", err)\n\t}\n\ts.fallback = mod.(Selector)\n\treturn nil\n}\n\n// Select returns an available host, if any.\nfunc (s HeaderHashSelection) Select(pool UpstreamPool, req *http.Request, _ http.ResponseWriter) *Upstream {\n\t// The Host header should be obtained from the req.Host field\n\t// since net/http removes it from the header map.\n\tif s.Field == \"Host\" && req.Host != \"\" {\n\t\treturn hostByHashing(pool, req.Host)\n\t}\n\n\tval := req.Header.Get(s.Field)\n\tif val == \"\" {\n\t\treturn s.fallback.Select(pool, req, nil)\n\t}\n\treturn hostByHashing(pool, val)\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens.\nfunc (s *HeaderHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\td.Next() // consume policy name\n\n\tif !d.NextArg() {\n\t\treturn d.ArgErr()\n\t}\n\ts.Field = d.Val()\n\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"fallback\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tif s.FallbackRaw != nil {\n\t\t\t\treturn d.Err(\"fallback selection policy already specified\")\n\t\t\t}\n\t\t\tmod, err := loadFallbackPolicy(d)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ts.FallbackRaw = mod\n\t\tdefault:\n\t\t\treturn d.Errf(\"unrecognized option '%s'\", d.Val())\n\t\t}\n\t}\n\treturn nil\n}\n\n// CookieHashSelection is a policy that selects\n// a host based on a given cookie name.\ntype CookieHashSelection struct {\n\t// The HTTP cookie name whose value is to be hashed and used for upstream selection.\n\tName string `json:\"name,omitempty\"`\n\t// Secret to hash (Hmac256) chosen upstream in cookie\n\tSecret string `json:\"secret,omitempty\"`\n\n\t// The fallback policy to use if the cookie is not present. Defaults to `random`.\n\tFallbackRaw json.RawMessage `json:\"fallback,omitempty\" caddy:\"namespace=http.reverse_proxy.selection_policies inline_key=policy\"`\n\tfallback    Selector\n}\n\n// CaddyModule returns the Caddy module information.\nfunc (CookieHashSelection) CaddyModule() caddy.ModuleInfo {\n\treturn caddy.ModuleInfo{\n\t\tID:  \"http.reverse_proxy.selection_policies.cookie\",\n\t\tNew: func() caddy.Module { return new(CookieHashSelection) },\n\t}\n}\n\n// Provision sets up the module.\nfunc (s *CookieHashSelection) Provision(ctx caddy.Context) error {\n\tif s.Name == \"\" {\n\t\ts.Name = \"lb\"\n\t}\n\tif s.FallbackRaw == nil {\n\t\ts.FallbackRaw = caddyconfig.JSONModuleObject(RandomSelection{}, \"policy\", \"random\", nil)\n\t}\n\tmod, err := ctx.LoadModule(s, \"FallbackRaw\")\n\tif err != nil {\n\t\treturn fmt.Errorf(\"loading fallback selection policy: %s\", err)\n\t}\n\ts.fallback = mod.(Selector)\n\treturn nil\n}\n\n// Select returns an available host, if any.\nfunc (s CookieHashSelection) Select(pool UpstreamPool, req *http.Request, w http.ResponseWriter) *Upstream {\n\t// selects a new Host using the fallback policy (typically random)\n\t// and write a sticky session cookie to the response.\n\tselectNewHost := func() *Upstream {\n\t\tupstream := s.fallback.Select(pool, req, w)\n\t\tif upstream == nil {\n\t\t\treturn nil\n\t\t}\n\t\tsha, err := hashCookie(s.Secret, upstream.Dial)\n\t\tif err != nil {\n\t\t\treturn upstream\n\t\t}\n\t\tcookie := &http.Cookie{\n\t\t\tName:   s.Name,\n\t\t\tValue:  sha,\n\t\t\tPath:   \"/\",\n\t\t\tSecure: false,\n\t\t}\n\t\tisProxyHttps := false\n\t\tif trusted, ok := caddyhttp.GetVar(req.Context(), caddyhttp.TrustedProxyVarKey).(bool); ok && trusted {\n\t\t\txfp, xfpOk, _ := lastHeaderValue(req.Header, \"X-Forwarded-Proto\")\n\t\t\tisProxyHttps = xfpOk && xfp == \"https\"\n\t\t}\n\t\tif req.TLS != nil || isProxyHttps {\n\t\t\tcookie.Secure = true\n\t\t\tcookie.SameSite = http.SameSiteNoneMode\n\t\t}\n\t\thttp.SetCookie(w, cookie)\n\t\treturn upstream\n\t}\n\n\tcookie, err := req.Cookie(s.Name)\n\t// If there's no cookie, select a host using the fallback policy\n\tif err != nil || cookie == nil {\n\t\treturn selectNewHost()\n\t}\n\t// If the cookie is present, loop over the available upstreams until we find a match\n\tcookieValue := cookie.Value\n\tfor _, upstream := range pool {\n\t\tif !upstream.Available() {\n\t\t\tcontinue\n\t\t}\n\t\tsha, err := hashCookie(s.Secret, upstream.Dial)\n\t\tif err == nil && sha == cookieValue {\n\t\t\treturn upstream\n\t\t}\n\t}\n\t// If there is no matching host, select a host using the fallback policy\n\treturn selectNewHost()\n}\n\n// UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax:\n//\n//\tlb_policy cookie [<name> [<secret>]] {\n//\t\tfallback <policy>\n//\t}\n//\n// By default name is `lb`\nfunc (s *CookieHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {\n\targs := d.RemainingArgs()\n\tswitch len(args) {\n\tcase 1:\n\tcase 2:\n\t\ts.Name = args[1]\n\tcase 3:\n\t\ts.Name = args[1]\n\t\ts.Secret = args[2]\n\tdefault:\n\t\treturn d.ArgErr()\n\t}\n\tfor d.NextBlock(0) {\n\t\tswitch d.Val() {\n\t\tcase \"fallback\":\n\t\t\tif !d.NextArg() {\n\t\t\t\treturn d.ArgErr()\n\t\t\t}\n\t\t\tif s.FallbackRaw != nil {\n\t\t\t\treturn d.Err(\"fallback selection policy already specified\")\n\t\t\t}\n\t\t\tmod, err := loadFallbackPolicy(d)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ts.FallbackRaw = mod\n\t\tdefault:\n\t\t\treturn d.Errf(\"unrecognized option '%s'\", d.Val())\n\t\t}\n\t}\n\treturn nil\n}\n\n// hashCookie hashes (HMAC 256) some data with the secret\n\nfunc hashCookie(secret string, data string) (string, error) {\n\th := hmac.New(sha256.New, []byte(secret))\n\t_, err := h.Write([]byte(data))\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn hex.EncodeToString(h.Sum(nil)), nil\n}\n\n// selectRandomHost returns a random available host\nfunc selectRandomHost(pool []*Upstream) *Upstream {\n\t// use reservoir sampling because the number of available\n\t// hosts isn't known: https://en.wikipedia.org/wiki/Reservoir_sampling\n\tvar randomHost *Upstream\n\tvar count int\n\tfor _, upstream := range pool {\n\t\tif !upstream.Available() {\n\t\t\tcontinue\n\t\t}\n\t\t// (n % 1 == 0) holds for all n, therefore a\n\t\t// upstream will always be chosen if there is at\n\t\t// least one available\n\t\tcount++\n\t\tif (weakrand.Int() % count) == 0 { //nolint:gosec\n\t\t\trandomHost = upstream\n\t\t}\n\t}\n\treturn randomHost\n}\n\n// leastRequests returns the host with the\n// least number of active requests to it.\n// If more than one host has the same\n// least number of active requests, then\n// one of those is chosen at random.\nfunc leastRequests(upstreams []*Upstream) *Upstream {\n\tif len(upstreams) == 0 {\n\t\treturn nil\n\t}\n\tvar best []*Upstream\n\tvar bestReqs int = -1\n\tfor _, upstream := range upstreams {\n\t\tif upstream == nil {\n\t\t\tcontinue\n\t\t}\n\t\treqs := upstream.NumRequests()\n\t\tif reqs == 0 {\n\t\t\treturn upstream\n\t\t}\n\t\t// If bestReqs was just initialized to -1\n\t\t// we need to append upstream also\n\t\tif reqs <= bestReqs || bestReqs == -1 {\n\t\t\tbestReqs = reqs\n\t\t\tbest = append(best, upstream)\n\t\t}\n\t}\n\tif len(best) == 0 {\n\t\treturn nil\n\t}\n\tif len(best) == 1 {\n\t\treturn best[0]\n\t}\n\treturn best[weakrand.Intn(len(best))] //nolint:gosec\n}\n\n// hostByHashing returns an available host from pool based on a hashable string s.\nfunc hostByHashing(pool []*Upstream, s string) *Upstream {\n\t// Highest Random Weight (HRW, or \"Rendezvous\") hashing,\n\t// guarantees stability when the list of upstreams changes;\n\t// see https://medium.com/i0exception/rendezvous-hashing-8c00e2fb58b0,\n\t// https://randorithms.com/2020/12/26/rendezvous-hashing.html,\n\t// and https://en.wikipedia.org/wiki/Rendezvous_hashing.\n\tvar highestHash uint64\n\tvar upstream *Upstream\n\tfor _, up := range pool {\n\t\tif !up.Available() {\n...\n", "needle_token_start": 14517, "needle_token_end": 14602, "code_context_ntokens": 15323, "output": ["```go\n// hashCookie hashes (HMAC 256) some data with the secret\nfunc hashCookie(secret string, data string) (string, error) {\n\th := hmac.New(sha256.New, []byte(secret))\n\t_, err := h.Write([]byte(data))\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn hex.EncodeToString(h.Sum(nil)), nil\n}\n```"]}
{"repo": "nsqio/nsq", "name": "UniqRands", "language": "go", "path": "internal/util/rand.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function generates a list of unique random integers within a specified range.\n2. **Input**: Two integers, the first specifying the number of unique integers required, and the second the maximum value for the range.\n3. **Output**: A slice of integers containing the specified number of unique random values.\n4. **Procedure**: The function first checks if the requested number of integers is greater than the maximum value and adjusts if necessary. It initializes a slice with integers from 0 up to the maximum value minus one. It then randomly swaps elements within this slice for the number of times specified by the input quantity, decreasing the range of the swap with each iteration. The function finally returns a slice of the desired length containing the randomized unique integers.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/app/float_array.go\npackage app\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype FloatArray []float64\n\nfunc (a *FloatArray) Get() interface{} { return []float64(*a) }\n\nfunc (a *FloatArray) Set(param string) error {\n\tfor _, s := range strings.Split(param, \",\") {\n\t\tv, err := strconv.ParseFloat(s, 64)\n\t\tif err != nil {\n\t\t\tlog.Fatalf(\"Could not parse: %s\", s)\n\t\t\treturn nil\n\t\t}\n\t\t*a = append(*a, v)\n\t}\n\tsort.Sort(*a)\n\treturn nil\n}\n\nfunc (a FloatArray) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }\nfunc (a FloatArray) Less(i, j int) bool { return a[i] > a[j] }\nfunc (a FloatArray) Len() int           { return len(a) }\n\nfunc (a *FloatArray) String() string {\n\tvar s []string\n\tfor _, v := range *a {\n\t\ts = append(s, fmt.Sprintf(\"%f\", v))\n\t}\n\treturn strings.Join(s, \",\")\n}\n\n// Path: internal/app/string_array.go\npackage app\n\nimport (\n\t\"strings\"\n)\n\ntype StringArray []string\n\nfunc (a *StringArray) Get() interface{} { return []string(*a) }\n\nfunc (a *StringArray) Set(s string) error {\n\t*a = append(*a, s)\n\treturn nil\n}\n\nfunc (a *StringArray) String() string {\n\treturn strings.Join(*a, \",\")\n}\n\n// Path: internal/util/rand.go\npackage util\n\nimport (\n\t\"math/rand\"\n)\n\n\nfunc UniqRands(quantity int, maxval int) []int {\n\tif maxval < quantity {\n\t\tquantity = maxval\n\t}\n\n\tintSlice := make([]int, maxval)\n\tfor i := 0; i < maxval; i++ {\n\t\tintSlice[i] = i\n\t}\n\n\tfor i := 0; i < quantity; i++ {\n\t\tj := rand.Int()%maxval + i\n\t\t// swap\n\t\tintSlice[i], intSlice[j] = intSlice[j], intSlice[i]\n\t\tmaxval--\n\n\t}\n\treturn intSlice[0:quantity]\n}\n\n// Path: internal/protocol/errors.go\npackage protocol\n\ntype ChildErr interface {\n\tParent() error\n}\n\n// ClientErr provides a way for NSQ daemons to log a human reabable\n// error string and return a machine readable string to the client.\n//\n// see docs/protocol.md for error codes by command\ntype ClientErr struct {\n\tParentErr error\n\tCode      string\n\tDesc      string\n}\n\n// Error returns the machine readable form\nfunc (e *ClientErr) Error() string {\n\treturn e.Code + \" \" + e.Desc\n}\n\n// Parent returns the parent error\nfunc (e *ClientErr) Parent() error {\n\treturn e.ParentErr\n}\n\n// NewClientErr creates a ClientErr with the supplied human and machine readable strings\nfunc NewClientErr(parent error, code string, description string) *ClientErr {\n\treturn &ClientErr{parent, code, description}\n}\n\ntype FatalClientErr struct {\n\tParentErr error\n\tCode      string\n\tDesc      string\n}\n\n// Error returns the machine readable form\nfunc (e *FatalClientErr) Error() string {\n\treturn e.Code + \" \" + e.Desc\n}\n\n// Parent returns the parent error\nfunc (e *FatalClientErr) Parent() error {\n\treturn e.ParentErr\n}\n\n// NewClientErr creates a ClientErr with the supplied human and machine readable strings\nfunc NewFatalClientErr(parent error, code string, description string) *FatalClientErr {\n\treturn &FatalClientErr{parent, code, description}\n}\n\n// Path: internal/statsd/client.go\npackage statsd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n)\n\ntype Client struct {\n\tw      io.Writer\n\tprefix string\n}\n\nfunc NewClient(w io.Writer, prefix string) *Client {\n\treturn &Client{\n\t\tw:      w,\n\t\tprefix: prefix,\n\t}\n}\n\nfunc (c *Client) Incr(stat string, count int64) error {\n\treturn c.send(stat, \"%d|c\", count)\n}\n\nfunc (c *Client) Decr(stat string, count int64) error {\n\treturn c.send(stat, \"%d|c\", -count)\n}\n\nfunc (c *Client) Timing(stat string, delta int64) error {\n\treturn c.send(stat, \"%d|ms\", delta)\n}\n\nfunc (c *Client) Gauge(stat string, value int64) error {\n\treturn c.send(stat, \"%d|g\", value)\n}\n\nfunc (c *Client) send(stat string, format string, value int64) error {\n\tformat = fmt.Sprintf(\"%s%s:%s\\n\", c.prefix, stat, format)\n\t_, err := fmt.Fprintf(c.w, format, value)\n\treturn err\n}\n\n// Path: internal/auth/authorizations.go\npackage auth\n\nimport (\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/nsqio/nsq/internal/http_api\"\n)\n\ntype Authorization struct {\n\tTopic       string   `json:\"topic\"`\n\tChannels    []string `json:\"channels\"`\n\tPermissions []string `json:\"permissions\"`\n}\n\ntype State struct {\n\tTTL            int             `json:\"ttl\"`\n\tAuthorizations []Authorization `json:\"authorizations\"`\n\tIdentity       string          `json:\"identity\"`\n\tIdentityURL    string          `json:\"identity_url\"`\n\tExpires        time.Time\n}\n\nfunc (a *Authorization) HasPermission(permission string) bool {\n\tfor _, p := range a.Permissions {\n\t\tif permission == p {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (a *Authorization) IsAllowed(topic, channel string) bool {\n\tif channel != \"\" {\n\t\tif !a.HasPermission(\"subscribe\") {\n\t\t\treturn false\n\t\t}\n\t} else {\n\t\tif !a.HasPermission(\"publish\") {\n\t\t\treturn false\n\t\t}\n\t}\n\n\ttopicRegex := regexp.MustCompile(a.Topic)\n\n\tif !topicRegex.MatchString(topic) {\n\t\treturn false\n\t}\n\n\tfor _, c := range a.Channels {\n\t\tchannelRegex := regexp.MustCompile(c)\n\t\tif channelRegex.MatchString(channel) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (a *State) IsAllowed(topic, channel string) bool {\n\tfor _, aa := range a.Authorizations {\n\t\tif aa.IsAllowed(topic, channel) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (a *State) IsExpired() bool {\n\treturn a.Expires.Before(time.Now())\n}\n\nfunc QueryAnyAuthd(authd []string, remoteIP string, tlsEnabled bool, commonName string, authSecret string,\n\tclientTLSConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) (*State, error) {\n\tvar retErr error\n\tstart := rand.Int()\n\tn := len(authd)\n\tfor i := 0; i < n; i++ {\n\t\ta := authd[(i+start)%n]\n\t\tauthState, err := QueryAuthd(a, remoteIP, tlsEnabled, commonName, authSecret, clientTLSConfig, connectTimeout, requestTimeout)\n\t\tif err != nil {\n\t\t\tes := fmt.Sprintf(\"failed to auth against %s - %s\", a, err)\n\t\t\tif retErr != nil {\n\t\t\t\tes = fmt.Sprintf(\"%s; %s\", retErr, es)\n\t\t\t}\n\t\t\tretErr = errors.New(es)\n\t\t\tcontinue\n\t\t}\n\t\treturn authState, nil\n\t}\n\treturn nil, retErr\n}\n\nfunc QueryAuthd(authd string, remoteIP string, tlsEnabled bool, commonName string, authSecret string,\n\tclientTLSConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) (*State, error) {\n\tv := url.Values{}\n\tv.Set(\"remote_ip\", remoteIP)\n\tif tlsEnabled {\n\t\tv.Set(\"tls\", \"true\")\n\t} else {\n\t\tv.Set(\"tls\", \"false\")\n\t}\n\tv.Set(\"secret\", authSecret)\n\tv.Set(\"common_name\", commonName)\n\n\tvar endpoint string\n\tif strings.Contains(authd, \"://\") {\n\t\tendpoint = fmt.Sprintf(\"%s?%s\", authd, v.Encode())\n\t} else {\n\t\tendpoint = fmt.Sprintf(\"http://%s/auth?%s\", authd, v.Encode())\n\t}\n\n\tvar authState State\n\tclient := http_api.NewClient(clientTLSConfig, connectTimeout, requestTimeout)\n\tif err := client.GETV1(endpoint, &authState); err != nil {\n\t\treturn nil, err\n\t}\n\n\t// validation on response\n\tfor _, auth := range authState.Authorizations {\n\t\tfor _, p := range auth.Permissions {\n\t\t\tswitch p {\n\t\t\tcase \"subscribe\", \"publish\":\n\t\t\tdefault:\n\t\t\t\treturn nil, fmt.Errorf(\"unknown permission %s\", p)\n\t\t\t}\n\t\t}\n\n\t\tif _, err := regexp.Compile(auth.Topic); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unable to compile topic %q %s\", auth.Topic, err)\n\t\t}\n\n\t\tfor _, channel := range auth.Channels {\n\t\t\tif _, err := regexp.Compile(channel); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"unable to compile channel %q %s\", channel, err)\n\t\t\t}\n\t\t}\n\t}\n\n\tif authState.TTL <= 0 {\n\t\treturn nil, fmt.Errorf(\"invalid TTL %d (must be >0)\", authState.TTL)\n\t}\n\n\tauthState.Expires = time.Now().Add(time.Duration(authState.TTL) * time.Second)\n\treturn &authState, nil\n}\n\n// Path: internal/test/logger.go\npackage test\n\ntype Logger interface {\n\tOutput(maxdepth int, s string) error\n}\n\ntype tbLog interface {\n\tLog(...interface{})\n}\n\ntype testLogger struct {\n\ttbLog\n}\n\nfunc (tl *testLogger) Output(maxdepth int, s string) error {\n\ttl.Log(s)\n\treturn nil\n}\n\nfunc NewTestLogger(tbl tbLog) Logger {\n\treturn &testLogger{tbl}\n}\n\n// Path: internal/lg/lg.go\n// Package lg provides leveled logging\npackage lg\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"strings\"\n)\n\nconst (\n\tDEBUG = LogLevel(1)\n\tINFO  = LogLevel(2)\n\tWARN  = LogLevel(3)\n\tERROR = LogLevel(4)\n\tFATAL = LogLevel(5)\n)\n\ntype AppLogFunc func(lvl LogLevel, f string, args ...interface{})\n\ntype Logger interface {\n\tOutput(maxdepth int, s string) error\n}\n\ntype NilLogger struct{}\n\nfunc (l NilLogger) Output(maxdepth int, s string) error {\n\treturn nil\n}\n\ntype LogLevel int\n\nfunc (l *LogLevel) Get() interface{} { return *l }\n\nfunc (l *LogLevel) Set(s string) error {\n\tlvl, err := ParseLogLevel(s)\n\tif err != nil {\n\t\treturn err\n\t}\n\t*l = lvl\n\treturn nil\n}\n\nfunc (l *LogLevel) String() string {\n\tswitch *l {\n\tcase DEBUG:\n\t\treturn \"DEBUG\"\n\tcase INFO:\n\t\treturn \"INFO\"\n\tcase WARN:\n\t\treturn \"WARNING\"\n\tcase ERROR:\n\t\treturn \"ERROR\"\n\tcase FATAL:\n\t\treturn \"FATAL\"\n\t}\n\treturn \"invalid\"\n}\n\nfunc ParseLogLevel(levelstr string) (LogLevel, error) {\n\tswitch strings.ToLower(levelstr) {\n\tcase \"debug\":\n\t\treturn DEBUG, nil\n\tcase \"info\":\n\t\treturn INFO, nil\n\tcase \"warn\":\n\t\treturn WARN, nil\n\tcase \"error\":\n\t\treturn ERROR, nil\n\tcase \"fatal\":\n\t\treturn FATAL, nil\n\t}\n\treturn 0, fmt.Errorf(\"invalid log level '%s' (debug, info, warn, error, fatal)\", levelstr)\n}\n\nfunc Logf(logger Logger, cfgLevel LogLevel, msgLevel LogLevel, f string, args ...interface{}) {\n\tif cfgLevel > msgLevel {\n\t\treturn\n\t}\n\tlogger.Output(3, fmt.Sprintf(msgLevel.String()+\": \"+f, args...))\n}\n\nfunc LogFatal(prefix string, f string, args ...interface{}) {\n\tlogger := log.New(os.Stderr, prefix, log.Ldate|log.Ltime|log.Lmicroseconds)\n\tLogf(logger, FATAL, FATAL, f, args...)\n\tos.Exit(1)\n}\n\n// Path: internal/clusterinfo/data.go\npackage clusterinfo\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/http_api\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype PartialErr interface {\n\terror\n\tErrors() []error\n}\n\ntype ErrList []error\n\nfunc (l ErrList) Error() string {\n\tvar es []string\n\tfor _, e := range l {\n\t\tes = append(es, e.Error())\n\t}\n\treturn strings.Join(es, \"\\n\")\n}\n\nfunc (l ErrList) Errors() []error {\n\treturn l\n}\n\ntype ClusterInfo struct {\n\tlog    lg.AppLogFunc\n\tclient *http_api.Client\n}\n\nfunc New(log lg.AppLogFunc, client *http_api.Client) *ClusterInfo {\n\treturn &ClusterInfo{\n\t\tlog:    log,\n\t\tclient: client,\n\t}\n}\n\nfunc (c *ClusterInfo) logf(f string, args ...interface{}) {\n\tif c.log != nil {\n\t\tc.log(lg.INFO, f, args...)\n\t}\n}\n\n// GetVersion returns a semver.Version object by querying /info\nfunc (c *ClusterInfo) GetVersion(addr string) (semver.Version, error) {\n\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\tvar resp struct {\n\t\tVersion string `json:\"version\"`\n\t}\n\terr := c.client.GETV1(endpoint, &resp)\n\tif err != nil {\n\t\treturn semver.Version{}, err\n\t}\n\tif resp.Version == \"\" {\n\t\tresp.Version = \"unknown\"\n\t}\n\treturn semver.Parse(resp.Version)\n}\n\n// GetLookupdTopics returns a []string containing a union of all the topics\n// from all the given nsqlookupd\nfunc (c *ClusterInfo) GetLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []string `json:\"topics\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/topics\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\ttopics = append(topics, resp.Topics...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\ttopics = stringy.Uniq(topics)\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetLookupdTopicChannels returns a []string containing a union of all the channels\n// from all the given lookupd for the given topic\nfunc (c *ClusterInfo) GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {\n\tvar channels []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tChannels []string `json:\"channels\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/channels?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tchannels = append(channels, resp.Channels...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tchannels = stringy.Uniq(channels)\n\tsort.Strings(channels)\n\n\tif len(errs) > 0 {\n\t\treturn channels, ErrList(errs)\n\t}\n\treturn channels, nil\n}\n\n// GetLookupdProducers returns Producers of all the nsqd connected to the given lookupds\nfunc (c *ClusterInfo) GetLookupdProducers(lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers []*Producer\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\tproducersByAddr := make(map[string]*Producer)\n\tmaxVersion, _ := semver.Parse(\"0.0.0\")\n\n\ttype respType struct {\n\t\tProducers []*Producer `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/nodes\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, producer := range resp.Producers {\n\t\t\t\tkey := producer.TCPAddress()\n\t\t\t\tp, ok := producersByAddr[key]\n\t\t\t\tif !ok {\n\t\t\t\t\tproducersByAddr[key] = producer\n\t\t\t\t\tproducers = append(producers, producer)\n\t\t\t\t\tif maxVersion.LT(producer.VersionObj) {\n\t\t\t\t\t\tmaxVersion = producer.VersionObj\n\t\t\t\t\t}\n\t\t\t\t\tsort.Sort(producer.Topics)\n\t\t\t\t\tp = producer\n\t\t\t\t}\n\t\t\t\tp.RemoteAddresses = append(p.RemoteAddresses,\n\t\t\t\t\tfmt.Sprintf(\"%s/%s\", addr, producer.Address()))\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tfor _, producer := range producersByAddr {\n\t\tif producer.VersionObj.LT(maxVersion) {\n\t\t\tproducer.OutOfDate = true\n\t\t}\n\t}\n\tsort.Sort(ProducersByHost{producers})\n\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetLookupdTopicProducers returns Producers of all the nsqd for a given topic by\n// unioning the nodes returned from the given lookupd\nfunc (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tProducers Producers `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/lookup?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, p := range resp.Producers {\n\t\t\t\tfor _, pp := range producers {\n\t\t\t\t\tif p.HTTPAddress() == pp.HTTPAddress() {\n\t\t\t\t\t\tgoto skip\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tproducers = append(producers, p)\n\t\t\tskip:\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDTopics returns a []string containing all the topics produced by the given nsqd\nfunc (c *ClusterInfo) GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\ttopics = stringy.Add(topics, topic.Name)\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetNSQDProducers returns Producers of all the given nsqd\nfunc (c *ClusterInfo) GetNSQDProducers(nsqdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype infoRespType struct {\n\t\tVersion          string `json:\"version\"`\n\t\tBroadcastAddress string `json:\"broadcast_address\"`\n\t\tHostname         string `json:\"hostname\"`\n\t\tHTTPPort         int    `json:\"http_port\"`\n\t\tTCPPort          int    `json:\"tcp_port\"`\n\t}\n\n\ttype statsRespType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar infoResp infoRespType\n\t\t\terr := c.client.GETV1(endpoint, &infoResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tendpoint = fmt.Sprintf(\"http://%s/stats?format=json&include_clients=false\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar statsResp statsRespType\n\t\t\terr = c.client.GETV1(endpoint, &statsResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tvar producerTopics ProducerTopics\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tproducerTopics = append(producerTopics, ProducerTopic{Topic: t.Name})\n\t\t\t}\n\n\t\t\tversion, err := semver.Parse(infoResp.Version)\n\t\t\tif err != nil {\n\t\t\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tproducers = append(producers, &Producer{\n\t\t\t\tVersion:          infoResp.Version,\n\t\t\t\tVersionObj:       version,\n\t\t\t\tBroadcastAddress: infoResp.BroadcastAddress,\n\t\t\t\tHostname:         infoResp.Hostname,\n\t\t\t\tHTTPPort:         infoResp.HTTPPort,\n\t\t\t\tTCPPort:          infoResp.TCPPort,\n\t\t\t\tTopics:           producerTopics,\n\t\t\t})\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDTopicProducers returns Producers containing the addresses of all the nsqd\n// that produce the given topic\nfunc (c *ClusterInfo) GetNSQDTopicProducers(topic string, nsqdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype infoRespType struct {\n\t\tVersion          string `json:\"version\"`\n\t\tBroadcastAddress string `json:\"broadcast_address\"`\n\t\tHostname         string `json:\"hostname\"`\n\t\tHTTPPort         int    `json:\"http_port\"`\n\t\tTCPPort          int    `json:\"tcp_port\"`\n\t}\n\n\ttype statsRespType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json&topic=%s&include_clients=false\",\n\t\t\t\taddr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar statsResp statsRespType\n\t\t\terr := c.client.GETV1(endpoint, &statsResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tvar producerTopics ProducerTopics\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tproducerTopics = append(producerTopics, ProducerTopic{Topic: t.Name})\n\t\t\t}\n\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tif t.Name == topic {\n\t\t\t\t\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\t\t\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\t\t\tvar infoResp infoRespType\n\t\t\t\t\terr := c.client.GETV1(endpoint, &infoResp)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlock.Lock()\n\t\t\t\t\t\terrs = append(errs, err)\n\t\t\t\t\t\tlock.Unlock()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\tversion, err := semver.Parse(infoResp.Version)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t\t\t\t\t}\n\n\t\t\t\t\t// if BroadcastAddress/HTTPPort are missing, use the values from `addr` for\n\t\t\t\t\t// backwards compatibility\n\n\t\t\t\t\tif infoResp.BroadcastAddress == \"\" {\n\t\t\t\t\t\tvar p string\n\t\t\t\t\t\tinfoResp.BroadcastAddress, p, _ = net.SplitHostPort(addr)\n\t\t\t\t\t\tinfoResp.HTTPPort, _ = strconv.Atoi(p)\n\t\t\t\t\t}\n\t\t\t\t\tif infoResp.Hostname == \"\" {\n\t\t\t\t\t\tinfoResp.Hostname, _, _ = net.SplitHostPort(addr)\n\t\t\t\t\t}\n\n\t\t\t\t\tlock.Lock()\n\t\t\t\t\tproducers = append(producers, &Producer{\n\t\t\t\t\t\tVersion:          infoResp.Version,\n\t\t\t\t\t\tVersionObj:       version,\n\t\t\t\t\t\tBroadcastAddress: infoResp.BroadcastAddress,\n\t\t\t\t\t\tHostname:         infoResp.Hostname,\n\t\t\t\t\t\tHTTPPort:         infoResp.HTTPPort,\n\t\t\t\t\t\tTCPPort:          infoResp.TCPPort,\n\t\t\t\t\t\tTopics:           producerTopics,\n\t\t\t\t\t})\n\t\t\t\t\tlock.Unlock()\n\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDStats returns aggregate topic and channel stats from the given Producers\n//\n// if selectedChannel is empty, this will return stats for topic/channel\n// if selectedTopic is empty, this will return stats for *all* topic/channels\n// if includeClients is false, this will *not* return client stats for channels\n// and the ChannelStats dict will be keyed by topic + ':' + channel\nfunc (c *ClusterInfo) GetNSQDStats(producers Producers,\n\tselectedTopic string, selectedChannel string,\n\tincludeClients bool) ([]*TopicStats, map[string]*ChannelStats, error) {\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar topicStatsList TopicStatsList\n\tvar errs []error\n\n\tchannelStatsMap := make(map[string]*ChannelStats)\n\n\ttype respType struct {\n\t\tTopics []*TopicStats `json:\"topics\"`\n\t}\n\n\tfor _, p := range producers {\n\t\twg.Add(1)\n\t\tgo func(p *Producer) {\n\t\t\tdefer wg.Done()\n\n\t\t\taddr := p.HTTPAddress()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json\", addr)\n\t\t\tif selectedTopic != \"\" {\n\t\t\t\tendpoint += \"&topic=\" + url.QueryEscape(selectedTopic)\n\t\t\t\tif selectedChannel != \"\" {\n\t\t\t\t\tendpoint += \"&channel=\" + url.QueryEscape(selectedChannel)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !includeClients {\n\t\t\t\tendpoint += \"&include_clients=false\"\n\t\t\t}\n\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\ttopic.Node = addr\n\t\t\t\ttopic.Hostname = p.Hostname\n\t\t\t\ttopic.MemoryDepth = topic.Depth - topic.BackendDepth\n\t\t\t\tif selectedTopic != \"\" && topic.TopicName != selectedTopic {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\ttopicStatsList = append(topicStatsList, topic)\n\n\t\t\t\tfor _, channel := range topic.Channels {\n\t\t\t\t\tchannel.Node = addr\n\t\t\t\t\tchannel.Hostname = p.Hostname\n\t\t\t\t\tchannel.TopicName = topic.TopicName\n\t\t\t\t\tchannel.MemoryDepth = channel.Depth - channel.BackendDepth\n\t\t\t\t\tkey := channel.ChannelName\n\t\t\t\t\tif selectedTopic == \"\" {\n\t\t\t\t\t\tkey = fmt.Sprintf(\"%s:%s\", topic.TopicName, channel.ChannelName)\n\t\t\t\t\t}\n\t\t\t\t\tchannelStats, ok := channelStatsMap[key]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tchannelStats = &ChannelStats{\n\t\t\t\t\t\t\tNode:        addr,\n\t\t\t\t\t\t\tTopicName:   topic.TopicName,\n\t\t\t\t\t\t\tChannelName: channel.ChannelName,\n\t\t\t\t\t\t}\n\t\t\t\t\t\tchannelStatsMap[key] = channelStats\n\t\t\t\t\t}\n\t\t\t\t\tfor _, c := range channel.Clients {\n\t\t\t\t\t\tc.Node = addr\n\t\t\t\t\t}\n\t\t\t\t\tchannelStats.Add(channel)\n\t\t\t\t}\n\t\t\t}\n\t\t}(p)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(producers) {\n\t\treturn nil, nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\n\tsort.Sort(TopicStatsByHost{topicStatsList})\n\n\tif len(errs) > 0 {\n\t\treturn topicStatsList, channelStatsMap, ErrList(errs)\n\t}\n\treturn topicStatsList, channelStatsMap, nil\n}\n\n// TombstoneNodeForTopic tombstones the given node for the given topic on all the given nsqlookupd\n// and deletes the topic from the node\nfunc (c *ClusterInfo) TombstoneNodeForTopic(topic string, node string, lookupdHTTPAddrs []string) error {\n\tvar errs []error\n\n\t// tombstone the topic on all the lookupds\n\tqs := fmt.Sprintf(\"topic=%s&node=%s\", url.QueryEscape(topic), url.QueryEscape(node))\n\terr := c.nsqlookupdPOST(lookupdHTTPAddrs, \"topic/tombstone\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tproducers, err := c.GetNSQDProducers([]string{node})\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\t// delete the topic on the producer\n\tqs = fmt.Sprintf(\"topic=%s\", url.QueryEscape(topic))\n\terr = c.producersPOST(producers, \"topic/delete\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\nfunc (c *ClusterInfo) CreateTopicChannel(topicName string, channelName string, lookupdHTTPAddrs []string) error {\n\tvar errs []error\n\n\t// create the topic on all the nsqlookupd\n\tqs := fmt.Sprintf(\"topic=%s\", url.QueryEscape(topicName))\n\terr := c.nsqlookupdPOST(lookupdHTTPAddrs, \"topic/create\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tif len(channelName) > 0 {\n\t\tqs := fmt.Sprintf(\"topic=%s&channel=%s\", url.QueryEscape(topicName), url.QueryEscape(channelName))\n\n\t\t// create the channel on all the nsqlookupd\n\t\terr := c.nsqlookupdPOST(lookupdHTTPAddrs, \"channel/create\", qs)\n\t\tif err != nil {\n\t\t\tpe, ok := err.(PartialErr)\n\t\t\tif !ok {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terrs = append(errs, pe.Errors()...)\n\t\t}\n\n\t\t// create the channel on all the nsqd that produce the topic\n\t\tproducers, err := c.GetLookupdTopicProducers(topicName, lookupdHTTPAddrs)\n\t\tif err != nil {\n\t\t\tpe, ok := err.(PartialErr)\n\t\t\tif !ok {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terrs = append(errs, pe.Errors()...)\n\t\t}\n\t\terr = c.producersPOST(producers, \"channel/create\", qs)\n\t\tif err != nil {\n\t\t\tpe, ok := err.(PartialErr)\n\t\t\tif !ok {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terrs = append(errs, pe.Errors()...)\n\t\t}\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\nfunc (c *ClusterInfo) DeleteTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tvar errs []error\n\n\t// for topic removal, you need to get all the producers _first_\n\tproducers, err := c.GetTopicProducers(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tqs := fmt.Sprintf(\"topic=%s\", url.QueryEscape(topicName))\n\n\t// remove the topic from all the nsqlookupd\n\terr = c.nsqlookupdPOST(lookupdHTTPAddrs, \"topic/delete\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\t// remove the topic from all the nsqd that produce this topic\n\terr = c.producersPOST(producers, \"topic/delete\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\nfunc (c *ClusterInfo) DeleteChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tvar errs []error\n\n\tproducers, err := c.GetTopicProducers(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tqs := fmt.Sprintf(\"topic=%s&channel=%s\", url.QueryEscape(topicName), url.QueryEscape(channelName))\n\n\t// remove the channel from all the nsqlookupd\n\terr = c.nsqlookupdPOST(lookupdHTTPAddrs, \"channel/delete\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\t// remove the channel from all the nsqd that produce this topic\n\terr = c.producersPOST(producers, \"channel/delete\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\nfunc (c *ClusterInfo) PauseTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tqs := fmt.Sprintf(\"topic=%s\", url.QueryEscape(topicName))\n\treturn c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, \"topic/pause\", qs)\n}\n\nfunc (c *ClusterInfo) UnPauseTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tqs := fmt.Sprintf(\"topic=%s\", url.QueryEscape(topicName))\n\treturn c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, \"topic/unpause\", qs)\n}\n\nfunc (c *ClusterInfo) PauseChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tqs := fmt.Sprintf(\"topic=%s&channel=%s\", url.QueryEscape(topicName), url.QueryEscape(channelName))\n\treturn c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, \"channel/pause\", qs)\n}\n\nfunc (c *ClusterInfo) UnPauseChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tqs := fmt.Sprintf(\"topic=%s&channel=%s\", url.QueryEscape(topicName), url.QueryEscape(channelName))\n\treturn c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, \"channel/unpause\", qs)\n}\n\nfunc (c *ClusterInfo) EmptyTopic(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tqs := fmt.Sprintf(\"topic=%s\", url.QueryEscape(topicName))\n\treturn c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, \"topic/empty\", qs)\n}\n\nfunc (c *ClusterInfo) EmptyChannel(topicName string, channelName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) error {\n\tqs := fmt.Sprintf(\"topic=%s&channel=%s\", url.QueryEscape(topicName), url.QueryEscape(channelName))\n\treturn c.actionHelper(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs, \"channel/empty\", qs)\n}\n\nfunc (c *ClusterInfo) actionHelper(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string, uri string, qs string) error {\n\tvar errs []error\n\n\tproducers, err := c.GetTopicProducers(topicName, lookupdHTTPAddrs, nsqdHTTPAddrs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\terr = c.producersPOST(producers, uri, qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\nfunc (c *ClusterInfo) GetProducers(lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) (Producers, error) {\n\tif len(lookupdHTTPAddrs) != 0 {\n\t\treturn c.GetLookupdProducers(lookupdHTTPAddrs)\n\t}\n\treturn c.GetNSQDProducers(nsqdHTTPAddrs)\n}\n\nfunc (c *ClusterInfo) GetTopicProducers(topicName string, lookupdHTTPAddrs []string, nsqdHTTPAddrs []string) (Producers, error) {\n\tif len(lookupdHTTPAddrs) != 0 {\n\t\treturn c.GetLookupdTopicProducers(topicName, lookupdHTTPAddrs)\n\t}\n\treturn c.GetNSQDTopicProducers(topicName, nsqdHTTPAddrs)\n}\n\nfunc (c *ClusterInfo) nsqlookupdPOST(addrs []string, uri string, qs string) error {\n\tvar errs []error\n\tfor _, addr := range addrs {\n\t\tendpoint := fmt.Sprintf(\"http://%s/%s?%s\", addr, uri, qs)\n\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\t\terr := c.client.POSTV1(endpoint)\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\nfunc (c *ClusterInfo) producersPOST(pl Producers, uri string, qs string) error {\n\tvar errs []error\n\tfor _, p := range pl {\n\t\tendpoint := fmt.Sprintf(\"http://%s/%s?%s\", p.HTTPAddress(), uri, qs)\n\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\t\terr := c.client.POSTV1(endpoint)\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\n// Path: internal/clusterinfo/producer_test.go\npackage clusterinfo\n\nimport \"testing\"\n\nfunc TestHostNameAddresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"host.domain.com\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"host.domain.com:4151\" {\n\t\tt.Errorf(\"Incorrect HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"host.domain.com:4150\" {\n\t\tt.Errorf(\"Incorrect TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv4Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"192.168.1.17\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"192.168.1.17:4151\" {\n\t\tt.Errorf(\"Incorrect IPv4 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"192.168.1.17:4150\" {\n\t\tt.Errorf(\"Incorrect IPv4 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv6Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"fd4a:622f:d2f2::1\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\tif p.HTTPAddress() != \"[fd4a:622f:d2f2::1]:4151\" {\n\t\tt.Errorf(\"Incorrect IPv6 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"[fd4a:622f:d2f2::1]:4150\" {\n\t\tt.Errorf(\"Incorrect IPv6 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\n// Path: internal/quantile/quantile.go\npackage quantile\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/bmizerany/perks/quantile\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype Result struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n}\n\nfunc (r *Result) String() string {\n\tvar s []string\n\tfor _, item := range r.Percentiles {\n\t\ts = append(s, stringy.NanoSecondToHuman(item[\"value\"]))\n\t}\n\treturn strings.Join(s, \", \")\n}\n\ntype Quantile struct {\n\tsync.Mutex\n\tstreams        [2]quantile.Stream\n\tcurrentIndex   uint8\n\tlastMoveWindow time.Time\n\tcurrentStream  *quantile.Stream\n\n\tPercentiles    []float64\n\tMoveWindowTime time.Duration\n}\n\nfunc New(WindowTime time.Duration, Percentiles []float64) *Quantile {\n\tq := Quantile{\n\t\tcurrentIndex:   0,\n\t\tlastMoveWindow: time.Now(),\n\t\tMoveWindowTime: WindowTime / 2,\n\t\tPercentiles:    Percentiles,\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tq.streams[i] = *quantile.NewTargeted(Percentiles...)\n\t}\n\tq.currentStream = &q.streams[0]\n\treturn &q\n}\n\nfunc (q *Quantile) Result() *Result {\n\tif q == nil {\n\t\treturn &Result{}\n\t}\n\tqueryHandler := q.QueryHandler()\n\tresult := Result{\n\t\tCount:       queryHandler.Count(),\n\t\tPercentiles: make([]map[string]float64, len(q.Percentiles)),\n\t}\n\tfor i, p := range q.Percentiles {\n\t\tvalue := queryHandler.Query(p)\n\t\tresult.Percentiles[i] = map[string]float64{\"quantile\": p, \"value\": value}\n\t}\n\treturn &result\n}\n\nfunc (q *Quantile) Insert(msgStartTime int64) {\n\tq.Lock()\n\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tq.currentStream.Insert(float64(now.UnixNano() - msgStartTime))\n\tq.Unlock()\n}\n\nfunc (q *Quantile) QueryHandler() *quantile.Stream {\n\tq.Lock()\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tmerged := quantile.NewTargeted(q.Percentiles...)\n\tmerged.Merge(q.streams[0].Samples())\n\tmerged.Merge(q.streams[1].Samples())\n\tq.Unlock()\n\treturn merged\n}\n\nfunc (q *Quantile) IsDataStale(now time.Time) bool {\n\treturn now.After(q.lastMoveWindow.Add(q.MoveWindowTime))\n}\n\nfunc (q *Quantile) Merge(them *Quantile) {\n\tq.Lock()\n\tthem.Lock()\n\tiUs := q.currentIndex\n\tiThem := them.currentIndex\n\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tiUs ^= 0x1\n\tiThem ^= 0x1\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tif q.lastMoveWindow.Before(them.lastMoveWindow) {\n\t\tq.lastMoveWindow = them.lastMoveWindow\n\t}\n\tq.Unlock()\n\tthem.Unlock()\n}\n\nfunc (q *Quantile) moveWindow() {\n\tq.currentIndex ^= 0x1\n\tq.currentStream = &q.streams[q.currentIndex]\n\tq.lastMoveWindow = q.lastMoveWindow.Add(q.MoveWindowTime)\n\tq.currentStream.Reset()\n}\n\n// Path: internal/clusterinfo/types.go\npackage clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype ProducerTopic struct {\n\tTopic      string `json:\"topic\"`\n\tTombstoned bool   `json:\"tombstoned\"`\n}\n\ntype ProducerTopics []ProducerTopic\n\nfunc (pt ProducerTopics) Len() int           { return len(pt) }\nfunc (pt ProducerTopics) Swap(i, j int)      { pt[i], pt[j] = pt[j], pt[i] }\n...\n", "needle_token_start": 486, "needle_token_end": 652, "code_context_ntokens": 14895, "output": ["```go\npackage util\n\nimport (\n\t\"math/rand\"\n)\n\nfunc UniqRands(quantity int, maxval int) []int {\n\tif maxval < quantity {\n\t\tquantity = maxval\n\t}\n\n\tintSlice := make([]int, maxval)\n\tfor i := 0; i < maxval; i++ {\n\t\tintSlice[i] = i\n\t}\n\n\tfor i := 0; i < quantity; i++ {\n\t\tj := rand.Int()%maxval + i\n\t\t// swap\n\t\tintSlice[i], intSlice[j] = intSlice[j], intSlice[i]\n\t\tmaxval--\n\n\t}\n\treturn intSlice[0:quantity]\n}\n```"]}
{"repo": "nsqio/nsq", "name": "isNil", "language": "go", "path": "internal/test/assertions.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To determine if a given variable is considered \"nil\" or \"null\" in the context of Go, which includes checking not only for nil pointers but also nil values of other reference types such as slices, channels, maps, and interfaces.\n2. **Input**: A variable of any type.\n3. **Output**: A boolean value; `true` if the variable is nil, and `false` otherwise.\n4. **Procedure**: The function first checks if the variable is directly `nil`. If not, it uses reflection to examine the variable's type and value. It checks if the variable's type is one of the reference types (like channels, pointers, maps, slices, or interfaces) and then checks if the value is nil.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/quantile/aggregate.go\npackage quantile\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n)\n\ntype E2eProcessingLatencyAggregate struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\tTopic       string               `json:\"topic\"`\n\tChannel     string               `json:\"channel\"`\n\tAddr        string               `json:\"host\"`\n}\n\nfunc (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {\n\tvar resp struct {\n\t\tCount       int                  `json:\"count\"`\n\t\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\t\tTopic       string               `json:\"topic\"`\n\t\tChannel     string               `json:\"channel\"`\n\t\tAddr        string               `json:\"host\"`\n\t}\n\terr := json.Unmarshal(b, &resp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, p := range resp.Percentiles {\n\t\tp[\"min\"] = p[\"value\"]\n\t\tp[\"max\"] = p[\"value\"]\n\t\tp[\"average\"] = p[\"value\"]\n\t\tp[\"count\"] = float64(resp.Count)\n\t}\n\n\te.Count = resp.Count\n\te.Percentiles = resp.Percentiles\n\te.Topic = resp.Topic\n\te.Channel = resp.Channel\n\te.Addr = resp.Addr\n\n\treturn nil\n}\n\nfunc (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) }\nfunc (e *E2eProcessingLatencyAggregate) Swap(i, j int) {\n\te.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i]\n}\nfunc (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {\n\treturn e.Percentiles[i][\"percentile\"] > e.Percentiles[j][\"percentile\"]\n}\n\n// Add merges e2 into e by averaging the percentiles\nfunc (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) {\n\te.Addr = \"*\"\n\tp := e.Percentiles\n\te.Count += e2.Count\n\tfor _, value := range e2.Percentiles {\n\t\ti := -1\n\t\tfor j, v := range p {\n\t\t\tif value[\"quantile\"] == v[\"quantile\"] {\n\t\t\t\ti = j\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif i == -1 {\n\t\t\ti = len(p)\n\t\t\te.Percentiles = append(p, make(map[string]float64))\n\t\t\tp = e.Percentiles\n\t\t\tp[i][\"quantile\"] = value[\"quantile\"]\n\t\t}\n\t\tp[i][\"max\"] = math.Max(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"min\"] = math.Min(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"count\"] += value[\"count\"]\n...\n// Path: internal/statsd/host.go\npackage statsd\n\nimport (\n\t\"strings\"\n)\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \"_\", -1), \":\", \"_\", -1)\n}\n\n// Path: internal/stringy/slice.go\npackage stringy\n\nfunc Add(s []string, a string) []string {\n\tfor _, existing := range s {\n\t\tif a == existing {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn append(s, a)\n\n}\n\nfunc Union(s []string, a []string) []string {\n\tfor _, entry := range a {\n\t\tfound := false\n\t\tfor _, existing := range s {\n\t\t\tif entry == existing {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\ts = append(s, entry)\n\t\t}\n\t}\n\treturn s\n}\n\nfunc Uniq(s []string) (r []string) {\nouterLoop:\n\tfor _, entry := range s {\n\t\tfor _, existing := range r {\n\t\t\tif existing == entry {\n\t\t\t\tcontinue outerLoop\n\t\t\t}\n\t\t}\n\t\tr = append(r, entry)\n\t}\n\treturn\n}\n\n// Path: internal/stringy/slice_test.go\npackage stringy_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\nfunc BenchmarkUniq(b *testing.B) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"b\"}\n\tfor i := 0; i < b.N; i++ {\n\t\tvalues = stringy.Uniq(values)\n\t\tif len(values) != 2 {\n\t\t\tb.Fatal(\"values len is incorrect\")\n\t\t}\n\t}\n}\n\nfunc TestUniq(t *testing.T) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"b\", \"b\", \"b\", \"c\", \"c\", \"c\"}\n\tvalues = stringy.Uniq(values)\n\tif len(values) != 3 {\n\t\tt.Fatal(\"values len is incorrect\")\n\t}\n}\n\n// Path: internal/stringy/template.go\npackage stringy\n\nimport (\n\t\"fmt\"\n)\n\nfunc NanoSecondToHuman(v float64) string {\n\tvar suffix string\n\tswitch {\n\tcase v > 1000000000:\n\t\tv /= 1000000000\n\t\tsuffix = \"s\"\n\tcase v > 1000000:\n\t\tv /= 1000000\n\t\tsuffix = \"ms\"\n\tcase v > 1000:\n\t\tv /= 1000\n\t\tsuffix = \"us\"\n\tdefault:\n\t\tsuffix = \"ns\"\n\t}\n\treturn fmt.Sprintf(\"%0.1f%s\", v, suffix)\n}\n\n// Path: internal/test/assertions.go\npackage test\n\nimport (\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc Equal(t *testing.T, expected, actual interface{}) {\n\tif !reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   %#v (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotEqual(t *testing.T, expected, actual interface{}) {\n\tif reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tnexp: %#v\\n\\n\\tgot:  %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc Nil(t *testing.T, object interface{}) {\n\tif !isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   <nil> (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, object)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotNil(t *testing.T, object interface{}) {\n\tif isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tExpected value not to be <nil>\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line)\n\t\tt.FailNow()\n\t}\n}\n\n\nfunc isNil(object interface{}) bool {\n\tif object == nil {\n\t\treturn true\n\t}\n\n\tvalue := reflect.ValueOf(object)\n\tkind := value.Kind()\n\tif kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// Path: internal/test/fakes.go\npackage test\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\ntype FakeNetConn struct {\n\tReadFunc             func([]byte) (int, error)\n\tWriteFunc            func([]byte) (int, error)\n\tCloseFunc            func() error\n\tLocalAddrFunc        func() net.Addr\n\tRemoteAddrFunc       func() net.Addr\n\tSetDeadlineFunc      func(time.Time) error\n\tSetReadDeadlineFunc  func(time.Time) error\n\tSetWriteDeadlineFunc func(time.Time) error\n}\n\nfunc (f FakeNetConn) Read(b []byte) (int, error)         { return f.ReadFunc(b) }\nfunc (f FakeNetConn) Write(b []byte) (int, error)        { return f.WriteFunc(b) }\nfunc (f FakeNetConn) Close() error                       { return f.CloseFunc() }\nfunc (f FakeNetConn) LocalAddr() net.Addr                { return f.LocalAddrFunc() }\nfunc (f FakeNetConn) RemoteAddr() net.Addr               { return f.RemoteAddrFunc() }\nfunc (f FakeNetConn) SetDeadline(t time.Time) error      { return f.SetDeadlineFunc(t) }\nfunc (f FakeNetConn) SetReadDeadline(t time.Time) error  { return f.SetReadDeadlineFunc(t) }\nfunc (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) }\n\ntype fakeNetAddr struct{}\n\nfunc (fakeNetAddr) Network() string { return \"\" }\nfunc (fakeNetAddr) String() string  { return \"\" }\n\nfunc NewFakeNetConn() FakeNetConn {\n\tnetAddr := fakeNetAddr{}\n\treturn FakeNetConn{\n\t\tReadFunc:             func(b []byte) (int, error) { return 0, nil },\n\t\tWriteFunc:            func(b []byte) (int, error) { return len(b), nil },\n\t\tCloseFunc:            func() error { return nil },\n\t\tLocalAddrFunc:        func() net.Addr { return netAddr },\n\t\tRemoteAddrFunc:       func() net.Addr { return netAddr },\n\t\tSetDeadlineFunc:      func(time.Time) error { return nil },\n\t\tSetWriteDeadlineFunc: func(time.Time) error { return nil },\n\t\tSetReadDeadlineFunc:  func(time.Time) error { return nil },\n\t}\n}\n\n// Path: internal/util/unix_socket.go\npackage util\n\nimport (\n\t\"net\"\n)\n\nfunc TypeOfAddr(addr string) string {\n\tif _, _, err := net.SplitHostPort(addr); err == nil {\n\t\treturn \"tcp\"\n\t}\n\treturn \"unix\"\n}\n\n// Path: internal/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc BenchmarkUniqRands5of5(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(5, 5)\n\t}\n}\nfunc BenchmarkUniqRands20of20(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 20)\n\t}\n}\n\nfunc BenchmarkUniqRands20of50(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 50)\n\t}\n}\n\nfunc TestUniqRands(t *testing.T) {\n\tvar x []int\n\tx = UniqRands(3, 10)\n\ttest.Equal(t, 3, len(x))\n\n\tx = UniqRands(10, 5)\n\ttest.Equal(t, 5, len(x))\n\n\tx = UniqRands(10, 20)\n\ttest.Equal(t, 10, len(x))\n}\n\nfunc TestTypeOfAddr(t *testing.T) {\n\tvar x string\n\tx = TypeOfAddr(\"127.0.0.1:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"test:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"/var/run/nsqd.sock\")\n\ttest.Equal(t, \"unix\", x)\n\n\tx = TypeOfAddr(\"[::1%lo0]:80\")\n\ttest.Equal(t, \"tcp\", x)\n}\n\n// Path: internal/util/wait_group_wrapper.go\npackage util\n\nimport (\n\t\"sync\"\n)\n\ntype WaitGroupWrapper struct {\n\tsync.WaitGroup\n}\n\nfunc (w *WaitGroupWrapper) Wrap(cb func()) {\n\tw.Add(1)\n\tgo func() {\n\t\tcb()\n\t\tw.Done()\n\t}()\n}\n\n// Path: internal/writers/boundary_buffered_writer.go\npackage writers\n\nimport (\n\t\"bufio\"\n\t\"io\"\n)\n\ntype BoundaryBufferedWriter struct {\n\tbw *bufio.Writer\n}\n\nfunc NewBoundaryBufferedWriter(w io.Writer, size int) *BoundaryBufferedWriter {\n\treturn &BoundaryBufferedWriter{\n\t\tbw: bufio.NewWriterSize(w, size),\n\t}\n}\n\nfunc (b *BoundaryBufferedWriter) Write(p []byte) (int, error) {\n\tif len(p) > b.bw.Available() {\n\t\terr := b.bw.Flush()\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t}\n\treturn b.bw.Write(p)\n}\n\nfunc (b *BoundaryBufferedWriter) Flush() error {\n\treturn b.bw.Flush()\n}\n\n// Path: internal/writers/spread_writer.go\npackage writers\n\nimport (\n\t\"io\"\n\t\"time\"\n)\n\ntype SpreadWriter struct {\n\tw        io.Writer\n\tinterval time.Duration\n\tbuf      [][]byte\n\texitCh   chan int\n}\n\nfunc NewSpreadWriter(w io.Writer, interval time.Duration, exitCh chan int) *SpreadWriter {\n\treturn &SpreadWriter{\n\t\tw:        w,\n\t\tinterval: interval,\n\t\tbuf:      make([][]byte, 0),\n\t\texitCh:   exitCh,\n\t}\n}\n\nfunc (s *SpreadWriter) Write(p []byte) (int, error) {\n\tb := make([]byte, len(p))\n\tcopy(b, p)\n\ts.buf = append(s.buf, b)\n\treturn len(p), nil\n}\n\nfunc (s *SpreadWriter) Flush() {\n\tif len(s.buf) == 0 {\n\t\t// nothing to write, just wait\n\t\tselect {\n\t\tcase <-time.After(s.interval):\n\t\tcase <-s.exitCh:\n\t\t}\n\t\treturn\n\t}\n\tsleep := s.interval / time.Duration(len(s.buf))\n\tticker := time.NewTicker(sleep)\n\tfor _, b := range s.buf {\n\t\ts.w.Write(b)\n\t\tselect {\n\t\tcase <-ticker.C:\n\t\tcase <-s.exitCh: // skip sleeps finish writes\n\t\t}\n\t}\n\tticker.Stop()\n\ts.buf = s.buf[:0]\n}\n\n", "needle_token_start": 2243, "needle_token_end": 2326, "code_context_ntokens": 4307, "output": ["```go\nfunc isNil(object interface{}) bool {\n\tif object == nil {\n\t\treturn true\n\t}\n\n\tvalue := reflect.ValueOf(object)\n\tkind := value.Kind()\n\tif kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {\n\t\treturn true\n\t}\n\n\treturn false\n}\n```"]}
{"repo": "nsqio/nsq", "name": "NewFakeNetConn", "language": "go", "path": "internal/test/fakes.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To create a mock network connection object for testing purposes, allowing the simulation of network operations without real network interaction.\n2. **Input**: None.\n3. **Output**: Returns a mock network connection object with predefined behavior for its methods.\n4. **Procedure**: Initializes a mock network connection object with no-op or simple logic for all network-related methods (read, write, close, etc.), and returns this object for use in tests.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/protocol/byte_base10_test.go\npackage protocol\n\nimport (\n\t\"testing\"\n)\n\nvar result uint64\n\nfunc BenchmarkByteToBase10Valid(b *testing.B) {\n\tbt := []byte{'3', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\nfunc BenchmarkByteToBase10Invalid(b *testing.B) {\n...\n// Path: internal/protocol/names.go\npackage protocol\n\nimport (\n\t\"regexp\"\n)\n\nvar validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral)?$`)\n\n// IsValidTopicName checks a topic name for correctness\nfunc IsValidTopicName(name string) bool {\n\treturn isValidName(name)\n}\n\n// IsValidChannelName checks a channel name for correctness\nfunc IsValidChannelName(name string) bool {\n\treturn isValidName(name)\n}\n\nfunc isValidName(name string) bool {\n\tif len(name) > 64 || len(name) < 1 {\n\t\treturn false\n\t}\n\treturn validTopicChannelNameRegex.MatchString(name)\n}\n\n// Path: internal/protocol/tcp_server.go\npackage protocol\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype TCPHandler interface {\n\tHandle(net.Conn)\n}\n\nfunc TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"TCP: listening on %s\", listener.Addr())\n\n\tvar wg sync.WaitGroup\n\n\tfor {\n\t\tclientConn, err := listener.Accept()\n\t\tif err != nil {\n\t\t\t// net.Error.Temporary() is deprecated, but is valid for accept\n\t\t\t// this is a hack to avoid a staticcheck error\n\t\t\tif te, ok := err.(interface{ Temporary() bool }); ok && te.Temporary() {\n\t\t\t\tlogf(lg.WARN, \"temporary Accept() failure - %s\", err)\n\t\t\t\truntime.Gosched()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// theres no direct way to detect this error because it is not exposed\n\t\t\tif !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\t\t\treturn fmt.Errorf(\"listener.Accept() error - %s\", err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\thandler.Handle(clientConn)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\t// wait to return until all handler goroutines complete\n\twg.Wait()\n\n\tlogf(lg.INFO, \"TCP: closing %s\", listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/quantile/aggregate.go\npackage quantile\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n)\n\ntype E2eProcessingLatencyAggregate struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\tTopic       string               `json:\"topic\"`\n\tChannel     string               `json:\"channel\"`\n\tAddr        string               `json:\"host\"`\n}\n\nfunc (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {\n\tvar resp struct {\n\t\tCount       int                  `json:\"count\"`\n\t\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\t\tTopic       string               `json:\"topic\"`\n\t\tChannel     string               `json:\"channel\"`\n\t\tAddr        string               `json:\"host\"`\n\t}\n\terr := json.Unmarshal(b, &resp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, p := range resp.Percentiles {\n\t\tp[\"min\"] = p[\"value\"]\n\t\tp[\"max\"] = p[\"value\"]\n\t\tp[\"average\"] = p[\"value\"]\n\t\tp[\"count\"] = float64(resp.Count)\n\t}\n\n\te.Count = resp.Count\n\te.Percentiles = resp.Percentiles\n\te.Topic = resp.Topic\n\te.Channel = resp.Channel\n\te.Addr = resp.Addr\n\n\treturn nil\n}\n\nfunc (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) }\nfunc (e *E2eProcessingLatencyAggregate) Swap(i, j int) {\n\te.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i]\n}\nfunc (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {\n\treturn e.Percentiles[i][\"percentile\"] > e.Percentiles[j][\"percentile\"]\n}\n\n// Add merges e2 into e by averaging the percentiles\nfunc (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) {\n\te.Addr = \"*\"\n\tp := e.Percentiles\n\te.Count += e2.Count\n\tfor _, value := range e2.Percentiles {\n\t\ti := -1\n\t\tfor j, v := range p {\n\t\t\tif value[\"quantile\"] == v[\"quantile\"] {\n\t\t\t\ti = j\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif i == -1 {\n\t\t\ti = len(p)\n\t\t\te.Percentiles = append(p, make(map[string]float64))\n\t\t\tp = e.Percentiles\n\t\t\tp[i][\"quantile\"] = value[\"quantile\"]\n\t\t}\n\t\tp[i][\"max\"] = math.Max(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"min\"] = math.Min(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"count\"] += value[\"count\"]\n\t\tif p[i][\"count\"] == 0 {\n\t\t\tp[i][\"average\"] = 0\n\t\t\tcontinue\n\t\t}\n\t\tdelta := value[\"average\"] - p[i][\"average\"]\n\t\tR := delta * value[\"count\"] / p[i][\"count\"]\n\t\tp[i][\"average\"] = p[i][\"average\"] + R\n\t}\n\tsort.Sort(e)\n}\n\n// Path: internal/statsd/host.go\npackage statsd\n\nimport (\n\t\"strings\"\n)\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \"_\", -1), \":\", \"_\", -1)\n}\n\n// Path: internal/stringy/slice.go\npackage stringy\n\nfunc Add(s []string, a string) []string {\n\tfor _, existing := range s {\n\t\tif a == existing {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn append(s, a)\n\n}\n\nfunc Union(s []string, a []string) []string {\n\tfor _, entry := range a {\n\t\tfound := false\n\t\tfor _, existing := range s {\n\t\t\tif entry == existing {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\ts = append(s, entry)\n\t\t}\n\t}\n\treturn s\n}\n\nfunc Uniq(s []string) (r []string) {\nouterLoop:\n\tfor _, entry := range s {\n\t\tfor _, existing := range r {\n\t\t\tif existing == entry {\n\t\t\t\tcontinue outerLoop\n\t\t\t}\n\t\t}\n\t\tr = append(r, entry)\n\t}\n\treturn\n}\n\n// Path: internal/stringy/slice_test.go\npackage stringy_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\nfunc BenchmarkUniq(b *testing.B) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"b\"}\n\tfor i := 0; i < b.N; i++ {\n\t\tvalues = stringy.Uniq(values)\n\t\tif len(values) != 2 {\n\t\t\tb.Fatal(\"values len is incorrect\")\n\t\t}\n\t}\n}\n\nfunc TestUniq(t *testing.T) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"b\", \"b\", \"b\", \"c\", \"c\", \"c\"}\n\tvalues = stringy.Uniq(values)\n\tif len(values) != 3 {\n\t\tt.Fatal(\"values len is incorrect\")\n\t}\n}\n\n// Path: internal/stringy/template.go\npackage stringy\n\nimport (\n\t\"fmt\"\n)\n\nfunc NanoSecondToHuman(v float64) string {\n\tvar suffix string\n\tswitch {\n\tcase v > 1000000000:\n\t\tv /= 1000000000\n\t\tsuffix = \"s\"\n\tcase v > 1000000:\n\t\tv /= 1000000\n\t\tsuffix = \"ms\"\n\tcase v > 1000:\n\t\tv /= 1000\n\t\tsuffix = \"us\"\n\tdefault:\n\t\tsuffix = \"ns\"\n\t}\n\treturn fmt.Sprintf(\"%0.1f%s\", v, suffix)\n}\n\n// Path: internal/test/assertions.go\npackage test\n\nimport (\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc Equal(t *testing.T, expected, actual interface{}) {\n\tif !reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   %#v (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotEqual(t *testing.T, expected, actual interface{}) {\n\tif reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tnexp: %#v\\n\\n\\tgot:  %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc Nil(t *testing.T, object interface{}) {\n\tif !isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   <nil> (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, object)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotNil(t *testing.T, object interface{}) {\n\tif isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tExpected value not to be <nil>\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line)\n\t\tt.FailNow()\n\t}\n}\n\nfunc isNil(object interface{}) bool {\n\tif object == nil {\n\t\treturn true\n\t}\n\n\tvalue := reflect.ValueOf(object)\n\tkind := value.Kind()\n\tif kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// Path: internal/test/fakes.go\npackage test\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\ntype FakeNetConn struct {\n\tReadFunc             func([]byte) (int, error)\n\tWriteFunc            func([]byte) (int, error)\n\tCloseFunc            func() error\n\tLocalAddrFunc        func() net.Addr\n\tRemoteAddrFunc       func() net.Addr\n\tSetDeadlineFunc      func(time.Time) error\n\tSetReadDeadlineFunc  func(time.Time) error\n\tSetWriteDeadlineFunc func(time.Time) error\n}\n\nfunc (f FakeNetConn) Read(b []byte) (int, error)         { return f.ReadFunc(b) }\nfunc (f FakeNetConn) Write(b []byte) (int, error)        { return f.WriteFunc(b) }\nfunc (f FakeNetConn) Close() error                       { return f.CloseFunc() }\nfunc (f FakeNetConn) LocalAddr() net.Addr                { return f.LocalAddrFunc() }\nfunc (f FakeNetConn) RemoteAddr() net.Addr               { return f.RemoteAddrFunc() }\nfunc (f FakeNetConn) SetDeadline(t time.Time) error      { return f.SetDeadlineFunc(t) }\nfunc (f FakeNetConn) SetReadDeadline(t time.Time) error  { return f.SetReadDeadlineFunc(t) }\nfunc (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) }\n\ntype fakeNetAddr struct{}\n\nfunc (fakeNetAddr) Network() string { return \"\" }\nfunc (fakeNetAddr) String() string  { return \"\" }\n\n\nfunc NewFakeNetConn() FakeNetConn {\n\tnetAddr := fakeNetAddr{}\n\treturn FakeNetConn{\n\t\tReadFunc:             func(b []byte) (int, error) { return 0, nil },\n\t\tWriteFunc:            func(b []byte) (int, error) { return len(b), nil },\n\t\tCloseFunc:            func() error { return nil },\n\t\tLocalAddrFunc:        func() net.Addr { return netAddr },\n\t\tRemoteAddrFunc:       func() net.Addr { return netAddr },\n\t\tSetDeadlineFunc:      func(time.Time) error { return nil },\n\t\tSetWriteDeadlineFunc: func(time.Time) error { return nil },\n\t\tSetReadDeadlineFunc:  func(time.Time) error { return nil },\n\t}\n}\n\n// Path: internal/util/unix_socket.go\npackage util\n\nimport (\n\t\"net\"\n)\n\nfunc TypeOfAddr(addr string) string {\n\tif _, _, err := net.SplitHostPort(addr); err == nil {\n\t\treturn \"tcp\"\n\t}\n\treturn \"unix\"\n}\n\n// Path: internal/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc BenchmarkUniqRands5of5(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(5, 5)\n\t}\n}\nfunc BenchmarkUniqRands20of20(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 20)\n\t}\n}\n\nfunc BenchmarkUniqRands20of50(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 50)\n\t}\n}\n\nfunc TestUniqRands(t *testing.T) {\n\tvar x []int\n\tx = UniqRands(3, 10)\n\ttest.Equal(t, 3, len(x))\n\n\tx = UniqRands(10, 5)\n\ttest.Equal(t, 5, len(x))\n\n\tx = UniqRands(10, 20)\n\ttest.Equal(t, 10, len(x))\n}\n\nfunc TestTypeOfAddr(t *testing.T) {\n\tvar x string\n\tx = TypeOfAddr(\"127.0.0.1:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"test:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"/var/run/nsqd.sock\")\n\ttest.Equal(t, \"unix\", x)\n\n\tx = TypeOfAddr(\"[::1%lo0]:80\")\n\ttest.Equal(t, \"tcp\", x)\n}\n\n// Path: internal/util/wait_group_wrapper.go\npackage util\n\nimport (\n\t\"sync\"\n)\n\ntype WaitGroupWrapper struct {\n\tsync.WaitGroup\n}\n\nfunc (w *WaitGroupWrapper) Wrap(cb func()) {\n\tw.Add(1)\n\tgo func() {\n\t\tcb()\n\t\tw.Done()\n\t}()\n}\n\n// Path: internal/writers/boundary_buffered_writer.go\npackage writers\n\nimport (\n\t\"bufio\"\n\t\"io\"\n)\n\ntype BoundaryBufferedWriter struct {\n\tbw *bufio.Writer\n}\n\nfunc NewBoundaryBufferedWriter(w io.Writer, size int) *BoundaryBufferedWriter {\n\treturn &BoundaryBufferedWriter{\n\t\tbw: bufio.NewWriterSize(w, size),\n\t}\n}\n\nfunc (b *BoundaryBufferedWriter) Write(p []byte) (int, error) {\n\tif len(p) > b.bw.Available() {\n\t\terr := b.bw.Flush()\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t}\n\treturn b.bw.Write(p)\n}\n\nfunc (b *BoundaryBufferedWriter) Flush() error {\n\treturn b.bw.Flush()\n}\n\n// Path: internal/writers/spread_writer.go\npackage writers\n\nimport (\n\t\"io\"\n\t\"time\"\n)\n\ntype SpreadWriter struct {\n\tw        io.Writer\n\tinterval time.Duration\n\tbuf      [][]byte\n\texitCh   chan int\n}\n\nfunc NewSpreadWriter(w io.Writer, interval time.Duration, exitCh chan int) *SpreadWriter {\n\treturn &SpreadWriter{\n\t\tw:        w,\n\t\tinterval: interval,\n\t\tbuf:      make([][]byte, 0),\n\t\texitCh:   exitCh,\n\t}\n}\n\nfunc (s *SpreadWriter) Write(p []byte) (int, error) {\n\tb := make([]byte, len(p))\n\tcopy(b, p)\n\ts.buf = append(s.buf, b)\n\treturn len(p), nil\n}\n\nfunc (s *SpreadWriter) Flush() {\n\tif len(s.buf) == 0 {\n\t\t// nothing to write, just wait\n\t\tselect {\n\t\tcase <-time.After(s.interval):\n\t\tcase <-s.exitCh:\n\t\t}\n\t\treturn\n\t}\n\tsleep := s.interval / time.Duration(len(s.buf))\n\tticker := time.NewTicker(sleep)\n\tfor _, b := range s.buf {\n\t\ts.w.Write(b)\n\t\tselect {\n\t\tcase <-ticker.C:\n\t\tcase <-s.exitCh: // skip sleeps finish writes\n\t\t}\n\t}\n\tticker.Stop()\n\ts.buf = s.buf[:0]\n}\n\n", "needle_token_start": 3709, "needle_token_end": 3919, "code_context_ntokens": 5236, "output": ["```go\n// Path: internal/test/fakes.go\npackage test\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\ntype FakeNetConn struct {\n\tReadFunc             func([]byte) (int, error)\n\tWriteFunc            func([]byte) (int, error)\n\tCloseFunc            func() error\n\tLocalAddrFunc        func() net.Addr\n\tRemoteAddrFunc       func() net.Addr\n\tSetDeadlineFunc      func(time.Time) error\n\tSetReadDeadlineFunc  func(time.Time) error\n\tSetWriteDeadlineFunc func(time.Time) error\n}\n\nfunc (f FakeNetConn) Read(b []byte) (int, error)         { return f.ReadFunc(b) }\nfunc (f FakeNetConn) Write(b []byte) (int, error)        { return f.WriteFunc(b) }\nfunc (f FakeNetConn) Close() error                       { return f.CloseFunc() }\nfunc (f FakeNetConn) LocalAddr() net.Addr                { return f.LocalAddrFunc() }\nfunc (f FakeNetConn) RemoteAddr() net.Addr               { return f.RemoteAddrFunc() }\nfunc (f FakeNetConn) SetDeadline(t time.Time) error      { return f.SetDeadlineFunc(t) }\nfunc (f FakeNetConn) SetReadDeadline(t time.Time) error  { return f.SetReadDeadlineFunc(t) }\nfunc (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) }\n\ntype fakeNetAddr struct{}\n\nfunc (fakeNetAddr) Network() string { return \"\" }\nfunc (fakeNetAddr) String() string  { return \"\" }\n\nfunc NewFakeNetConn() FakeNetConn {\n\tnetAddr := fakeNetAddr{}\n\treturn FakeNetConn{\n\t\tReadFunc:             func(b []byte) (int, error) { return 0, nil },\n\t\tWriteFunc:            func(b []byte) (int, error) { return len(b), nil },\n\t\tCloseFunc:            func() error { return nil },\n\t\tLocalAddrFunc:        func() net.Addr { return netAddr },\n\t\tRemoteAddrFunc:       func() net.Addr { return netAddr },\n\t\tSetDeadlineFunc:      func(time.Time) error { return nil },\n\t\tSetWriteDeadlineFunc: func(time.Time) error { return nil },\n\t\tSetReadDeadlineFunc:  func(time.Time) error { return nil },\n\t}\n}\n```"]}
{"repo": "nsqio/nsq", "name": "HostKey", "language": "go", "path": "internal/statsd/host.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to format a hostname string into a standardized format that replaces certain characters with underscores, making it suitable for use in contexts where special characters like dots and colons may cause issues.\n2. **Input**: A single string representing a hostname, which may include dots and colons.\n3. **Output**: A string where all dots and colons have been replaced with underscores.\n4. **Procedure**: The function takes the input string and performs two sequential replacements: first, it replaces all dots with underscores, and then it replaces all colons with underscores. The result is returned as the output.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/http_api/http_server.go\npackage http_api\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype logWriter struct {\n\tlogf lg.AppLogFunc\n}\n\nfunc (l logWriter) Write(p []byte) (int, error) {\n\tl.logf(lg.WARN, \"%s\", string(p))\n\treturn len(p), nil\n}\n\nfunc Serve(listener net.Listener, handler http.Handler, proto string, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"%s: listening on %s\", proto, listener.Addr())\n\n\tserver := &http.Server{\n\t\tHandler:  handler,\n\t\tErrorLog: log.New(logWriter{logf}, \"\", 0),\n\t}\n\terr := server.Serve(listener)\n\t// theres no direct way to detect this error because it is not exposed\n...\n// Path: internal/http_api/req_params.go\npackage http_api\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n)\n\ntype ReqParams struct {\n\turl.Values\n\tBody []byte\n}\n\nfunc NewReqParams(req *http.Request) (*ReqParams, error) {\n\treqParams, err := url.ParseQuery(req.URL.RawQuery)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, err := io.ReadAll(req.Body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &ReqParams{reqParams, data}, nil\n}\n\nfunc (r *ReqParams) Get(key string) (string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn \"\", errors.New(\"key not in query params\")\n\t}\n\treturn v[0], nil\n}\n\nfunc (r *ReqParams) GetAll(key string) ([]string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn nil, errors.New(\"key not in query params\")\n\t}\n\treturn v, nil\n}\n\n// Path: internal/version/binary.go\npackage version\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n)\n\nconst Binary = \"1.3.0\"\n\nfunc String(app string) string {\n\treturn fmt.Sprintf(\"%s v%s (built w/%s)\", app, Binary, runtime.Version())\n}\n\n// Path: internal/protocol/protocol.go\npackage protocol\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"net\"\n)\n\ntype Client interface {\n\tClose() error\n}\n\n// Protocol describes the basic behavior of any protocol in the system\ntype Protocol interface {\n\tNewClient(net.Conn) Client\n\tIOLoop(Client) error\n}\n\n// SendResponse is a server side utility function to prefix data with a length header\n// and write to the supplied Writer\nfunc SendResponse(w io.Writer, data []byte) (int, error) {\n\terr := binary.Write(w, binary.BigEndian, int32(len(data)))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tn, err := w.Write(data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn (n + 4), nil\n}\n\n// SendFramedResponse is a server side utility function to prefix data with a length header\n// and frame header and write to the supplied Writer\nfunc SendFramedResponse(w io.Writer, frameType int32, data []byte) (int, error) {\n\tbeBuf := make([]byte, 4)\n\tsize := uint32(len(data)) + 4\n\n\tbinary.BigEndian.PutUint32(beBuf, size)\n\tn, err := w.Write(beBuf)\n\tif err != nil {\n\t\treturn n, err\n\t}\n\n\tbinary.BigEndian.PutUint32(beBuf, uint32(frameType))\n\tn, err = w.Write(beBuf)\n\tif err != nil {\n\t\treturn n + 4, err\n\t}\n\n\tn, err = w.Write(data)\n\treturn n + 8, err\n}\n\n// Path: internal/http_api/topic_channel_args.go\npackage http_api\n\nimport (\n\t\"errors\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\ntype getter interface {\n\tGet(key string) (string, error)\n}\n\nfunc GetTopicChannelArgs(rp getter) (string, string, error) {\n\ttopicName, err := rp.Get(\"topic\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_TOPIC\")\n\t}\n\n\tif !protocol.IsValidTopicName(topicName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_TOPIC\")\n\t}\n\n\tchannelName, err := rp.Get(\"channel\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_CHANNEL\")\n\t}\n\n\tif !protocol.IsValidChannelName(channelName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_CHANNEL\")\n\t}\n\n\treturn topicName, channelName, nil\n}\n\n// Path: internal/lg/lg_test.go\npackage lg\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\ntype mockLogger struct {\n\tCount int\n}\n\nfunc (l *mockLogger) Output(maxdepth int, s string) error {\n\tl.Count++\n\treturn nil\n}\n\nfunc TestLogging(t *testing.T) {\n\tlogger := &mockLogger{}\n\n\t// Test only fatal get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, FATAL, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 1, logger.Count)\n\n\t// Test only warnings or higher get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, WARN, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 3, logger.Count)\n\n\t// Test everything gets through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, DEBUG, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 5, logger.Count)\n}\n\n// Path: internal/pqueue/pqueue.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n)\n\ntype Item struct {\n\tValue    interface{}\n\tPriority int64\n\tIndex    int\n}\n\n// this is a priority queue as implemented by a min heap\n// ie. the 0th element is the *lowest* value\ntype PriorityQueue []*Item\n\nfunc New(capacity int) PriorityQueue {\n\treturn make(PriorityQueue, 0, capacity)\n}\n\nfunc (pq PriorityQueue) Len() int {\n\treturn len(pq)\n}\n\nfunc (pq PriorityQueue) Less(i, j int) bool {\n\treturn pq[i].Priority < pq[j].Priority\n}\n\nfunc (pq PriorityQueue) Swap(i, j int) {\n\tpq[i], pq[j] = pq[j], pq[i]\n\tpq[i].Index = i\n\tpq[j].Index = j\n}\n\nfunc (pq *PriorityQueue) Push(x interface{}) {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n+1 > c {\n\t\tnpq := make(PriorityQueue, n, c*2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\t*pq = (*pq)[0 : n+1]\n\titem := x.(*Item)\n\titem.Index = n\n\t(*pq)[n] = item\n}\n\nfunc (pq *PriorityQueue) Pop() interface{} {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n < (c/2) && c > 25 {\n\t\tnpq := make(PriorityQueue, n, c/2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\titem := (*pq)[n-1]\n\titem.Index = -1\n\t*pq = (*pq)[0 : n-1]\n\treturn item\n}\n\nfunc (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) {\n\tif pq.Len() == 0 {\n\t\treturn nil, 0\n\t}\n\n\titem := (*pq)[0]\n\tif item.Priority > max {\n\t\treturn nil, item.Priority - max\n\t}\n\theap.Remove(pq, 0)\n\n\treturn item, 0\n}\n\n// Path: internal/pqueue/pqueue_test.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n\t\"math/rand\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"testing\"\n)\n\nfunc equal(t *testing.T, act, exp interface{}) {\n\tif !reflect.DeepEqual(exp, act) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\texp: %#v\\n\\n\\tgot: %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, exp, act)\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestPriorityQueue(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c+1; i++ {\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(i)})\n\t}\n\tequal(t, pq.Len(), c+1)\n\tequal(t, cap(pq), c*2)\n\n\tfor i := 0; i < c+1; i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, item.(*Item).Value.(int), i)\n\t}\n\tequal(t, cap(pq), c/4)\n}\n\nfunc TestUnsortedInsert(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\tints := make([]int, 0, c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\tints = append(ints, v)\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(v)})\n\t}\n\tequal(t, pq.Len(), c)\n\tequal(t, cap(pq), c)\n\n\tsort.Ints(ints)\n\n\tfor i := 0; i < c; i++ {\n\t\titem, _ := pq.PeekAndShift(int64(ints[len(ints)-1]))\n\t\tequal(t, item.Priority, int64(ints[i]))\n\t}\n}\n\nfunc TestRemove(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\theap.Push(&pq, &Item{Value: \"test\", Priority: int64(v)})\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\theap.Remove(&pq, rand.Intn((c-1)-i))\n\t}\n\n\tlastPriority := heap.Pop(&pq).(*Item).Priority\n\tfor i := 0; i < (c - 10 - 1); i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, lastPriority < item.(*Item).Priority, true)\n\t\tlastPriority = item.(*Item).Priority\n\t}\n}\n\n// Path: internal/protocol/byte_base10.go\npackage protocol\n\nimport (\n\t\"errors\"\n)\n\nvar errBase10 = errors.New(\"failed to convert to Base10\")\n\nfunc ByteToBase10(b []byte) (n uint64, err error) {\n\tbase := uint64(10)\n\n\tn = 0\n\tfor i := 0; i < len(b); i++ {\n\t\tvar v byte\n\t\td := b[i]\n\t\tswitch {\n\t\tcase '0' <= d && d <= '9':\n\t\t\tv = d - '0'\n\t\tdefault:\n\t\t\tn = 0\n\t\t\terr = errBase10\n\t\t\treturn\n\t\t}\n\t\tn *= base\n\t\tn += uint64(v)\n\t}\n\n\treturn n, err\n}\n\n// Path: internal/protocol/byte_base10_test.go\npackage protocol\n\nimport (\n\t\"testing\"\n)\n\nvar result uint64\n\nfunc BenchmarkByteToBase10Valid(b *testing.B) {\n\tbt := []byte{'3', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\nfunc BenchmarkByteToBase10Invalid(b *testing.B) {\n\tbt := []byte{'?', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\n// Path: internal/protocol/names.go\npackage protocol\n\nimport (\n\t\"regexp\"\n)\n\nvar validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral)?$`)\n\n// IsValidTopicName checks a topic name for correctness\nfunc IsValidTopicName(name string) bool {\n\treturn isValidName(name)\n}\n\n// IsValidChannelName checks a channel name for correctness\nfunc IsValidChannelName(name string) bool {\n\treturn isValidName(name)\n}\n\nfunc isValidName(name string) bool {\n\tif len(name) > 64 || len(name) < 1 {\n\t\treturn false\n\t}\n\treturn validTopicChannelNameRegex.MatchString(name)\n}\n\n// Path: internal/protocol/tcp_server.go\npackage protocol\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype TCPHandler interface {\n\tHandle(net.Conn)\n}\n\nfunc TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"TCP: listening on %s\", listener.Addr())\n\n\tvar wg sync.WaitGroup\n\n\tfor {\n\t\tclientConn, err := listener.Accept()\n\t\tif err != nil {\n\t\t\t// net.Error.Temporary() is deprecated, but is valid for accept\n\t\t\t// this is a hack to avoid a staticcheck error\n\t\t\tif te, ok := err.(interface{ Temporary() bool }); ok && te.Temporary() {\n\t\t\t\tlogf(lg.WARN, \"temporary Accept() failure - %s\", err)\n\t\t\t\truntime.Gosched()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// theres no direct way to detect this error because it is not exposed\n\t\t\tif !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\t\t\treturn fmt.Errorf(\"listener.Accept() error - %s\", err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\thandler.Handle(clientConn)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\t// wait to return until all handler goroutines complete\n\twg.Wait()\n\n\tlogf(lg.INFO, \"TCP: closing %s\", listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/quantile/aggregate.go\npackage quantile\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n)\n\ntype E2eProcessingLatencyAggregate struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\tTopic       string               `json:\"topic\"`\n\tChannel     string               `json:\"channel\"`\n\tAddr        string               `json:\"host\"`\n}\n\nfunc (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {\n\tvar resp struct {\n\t\tCount       int                  `json:\"count\"`\n\t\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\t\tTopic       string               `json:\"topic\"`\n\t\tChannel     string               `json:\"channel\"`\n\t\tAddr        string               `json:\"host\"`\n\t}\n\terr := json.Unmarshal(b, &resp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, p := range resp.Percentiles {\n\t\tp[\"min\"] = p[\"value\"]\n\t\tp[\"max\"] = p[\"value\"]\n\t\tp[\"average\"] = p[\"value\"]\n\t\tp[\"count\"] = float64(resp.Count)\n\t}\n\n\te.Count = resp.Count\n\te.Percentiles = resp.Percentiles\n\te.Topic = resp.Topic\n\te.Channel = resp.Channel\n\te.Addr = resp.Addr\n\n\treturn nil\n}\n\nfunc (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) }\nfunc (e *E2eProcessingLatencyAggregate) Swap(i, j int) {\n\te.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i]\n}\nfunc (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {\n\treturn e.Percentiles[i][\"percentile\"] > e.Percentiles[j][\"percentile\"]\n}\n\n// Add merges e2 into e by averaging the percentiles\nfunc (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) {\n\te.Addr = \"*\"\n\tp := e.Percentiles\n\te.Count += e2.Count\n\tfor _, value := range e2.Percentiles {\n\t\ti := -1\n\t\tfor j, v := range p {\n\t\t\tif value[\"quantile\"] == v[\"quantile\"] {\n\t\t\t\ti = j\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif i == -1 {\n\t\t\ti = len(p)\n\t\t\te.Percentiles = append(p, make(map[string]float64))\n\t\t\tp = e.Percentiles\n\t\t\tp[i][\"quantile\"] = value[\"quantile\"]\n\t\t}\n\t\tp[i][\"max\"] = math.Max(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"min\"] = math.Min(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"count\"] += value[\"count\"]\n\t\tif p[i][\"count\"] == 0 {\n\t\t\tp[i][\"average\"] = 0\n\t\t\tcontinue\n\t\t}\n\t\tdelta := value[\"average\"] - p[i][\"average\"]\n\t\tR := delta * value[\"count\"] / p[i][\"count\"]\n\t\tp[i][\"average\"] = p[i][\"average\"] + R\n\t}\n\tsort.Sort(e)\n}\n\n// Path: internal/statsd/host.go\npackage statsd\n\nimport (\n\t\"strings\"\n)\n\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \"_\", -1), \":\", \"_\", -1)\n}\n\n// Path: internal/stringy/slice.go\npackage stringy\n\nfunc Add(s []string, a string) []string {\n\tfor _, existing := range s {\n\t\tif a == existing {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn append(s, a)\n\n}\n\nfunc Union(s []string, a []string) []string {\n\tfor _, entry := range a {\n\t\tfound := false\n\t\tfor _, existing := range s {\n\t\t\tif entry == existing {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\ts = append(s, entry)\n\t\t}\n\t}\n\treturn s\n}\n\nfunc Uniq(s []string) (r []string) {\nouterLoop:\n\tfor _, entry := range s {\n\t\tfor _, existing := range r {\n\t\t\tif existing == entry {\n\t\t\t\tcontinue outerLoop\n\t\t\t}\n\t\t}\n\t\tr = append(r, entry)\n\t}\n\treturn\n}\n\n// Path: internal/stringy/slice_test.go\npackage stringy_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\nfunc BenchmarkUniq(b *testing.B) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"b\"}\n\tfor i := 0; i < b.N; i++ {\n\t\tvalues = stringy.Uniq(values)\n\t\tif len(values) != 2 {\n\t\t\tb.Fatal(\"values len is incorrect\")\n\t\t}\n\t}\n}\n\nfunc TestUniq(t *testing.T) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"b\", \"b\", \"b\", \"c\", \"c\", \"c\"}\n\tvalues = stringy.Uniq(values)\n\tif len(values) != 3 {\n\t\tt.Fatal(\"values len is incorrect\")\n\t}\n}\n\n// Path: internal/stringy/template.go\npackage stringy\n\nimport (\n\t\"fmt\"\n)\n\nfunc NanoSecondToHuman(v float64) string {\n\tvar suffix string\n\tswitch {\n\tcase v > 1000000000:\n\t\tv /= 1000000000\n\t\tsuffix = \"s\"\n\tcase v > 1000000:\n\t\tv /= 1000000\n\t\tsuffix = \"ms\"\n\tcase v > 1000:\n\t\tv /= 1000\n\t\tsuffix = \"us\"\n\tdefault:\n\t\tsuffix = \"ns\"\n\t}\n\treturn fmt.Sprintf(\"%0.1f%s\", v, suffix)\n}\n\n// Path: internal/test/assertions.go\npackage test\n\nimport (\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc Equal(t *testing.T, expected, actual interface{}) {\n\tif !reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   %#v (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotEqual(t *testing.T, expected, actual interface{}) {\n\tif reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tnexp: %#v\\n\\n\\tgot:  %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc Nil(t *testing.T, object interface{}) {\n\tif !isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   <nil> (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, object)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotNil(t *testing.T, object interface{}) {\n\tif isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tExpected value not to be <nil>\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line)\n\t\tt.FailNow()\n\t}\n}\n\nfunc isNil(object interface{}) bool {\n\tif object == nil {\n\t\treturn true\n\t}\n\n\tvalue := reflect.ValueOf(object)\n\tkind := value.Kind()\n\tif kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// Path: internal/test/fakes.go\npackage test\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\ntype FakeNetConn struct {\n\tReadFunc             func([]byte) (int, error)\n\tWriteFunc            func([]byte) (int, error)\n\tCloseFunc            func() error\n\tLocalAddrFunc        func() net.Addr\n\tRemoteAddrFunc       func() net.Addr\n\tSetDeadlineFunc      func(time.Time) error\n\tSetReadDeadlineFunc  func(time.Time) error\n\tSetWriteDeadlineFunc func(time.Time) error\n}\n\nfunc (f FakeNetConn) Read(b []byte) (int, error)         { return f.ReadFunc(b) }\nfunc (f FakeNetConn) Write(b []byte) (int, error)        { return f.WriteFunc(b) }\nfunc (f FakeNetConn) Close() error                       { return f.CloseFunc() }\nfunc (f FakeNetConn) LocalAddr() net.Addr                { return f.LocalAddrFunc() }\nfunc (f FakeNetConn) RemoteAddr() net.Addr               { return f.RemoteAddrFunc() }\nfunc (f FakeNetConn) SetDeadline(t time.Time) error      { return f.SetDeadlineFunc(t) }\nfunc (f FakeNetConn) SetReadDeadline(t time.Time) error  { return f.SetReadDeadlineFunc(t) }\nfunc (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) }\n\ntype fakeNetAddr struct{}\n\nfunc (fakeNetAddr) Network() string { return \"\" }\nfunc (fakeNetAddr) String() string  { return \"\" }\n\nfunc NewFakeNetConn() FakeNetConn {\n\tnetAddr := fakeNetAddr{}\n\treturn FakeNetConn{\n\t\tReadFunc:             func(b []byte) (int, error) { return 0, nil },\n\t\tWriteFunc:            func(b []byte) (int, error) { return len(b), nil },\n\t\tCloseFunc:            func() error { return nil },\n\t\tLocalAddrFunc:        func() net.Addr { return netAddr },\n\t\tRemoteAddrFunc:       func() net.Addr { return netAddr },\n\t\tSetDeadlineFunc:      func(time.Time) error { return nil },\n\t\tSetWriteDeadlineFunc: func(time.Time) error { return nil },\n\t\tSetReadDeadlineFunc:  func(time.Time) error { return nil },\n\t}\n}\n\n// Path: internal/util/unix_socket.go\npackage util\n\nimport (\n\t\"net\"\n)\n\nfunc TypeOfAddr(addr string) string {\n\tif _, _, err := net.SplitHostPort(addr); err == nil {\n\t\treturn \"tcp\"\n\t}\n\treturn \"unix\"\n}\n\n// Path: internal/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc BenchmarkUniqRands5of5(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(5, 5)\n\t}\n}\nfunc BenchmarkUniqRands20of20(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 20)\n\t}\n}\n\nfunc BenchmarkUniqRands20of50(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 50)\n\t}\n}\n\nfunc TestUniqRands(t *testing.T) {\n\tvar x []int\n\tx = UniqRands(3, 10)\n\ttest.Equal(t, 3, len(x))\n\n\tx = UniqRands(10, 5)\n\ttest.Equal(t, 5, len(x))\n\n\tx = UniqRands(10, 20)\n\ttest.Equal(t, 10, len(x))\n}\n\nfunc TestTypeOfAddr(t *testing.T) {\n\tvar x string\n\tx = TypeOfAddr(\"127.0.0.1:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"test:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"/var/run/nsqd.sock\")\n\ttest.Equal(t, \"unix\", x)\n\n\tx = TypeOfAddr(\"[::1%lo0]:80\")\n\ttest.Equal(t, \"tcp\", x)\n}\n\n// Path: internal/util/wait_group_wrapper.go\npackage util\n\nimport (\n\t\"sync\"\n)\n\ntype WaitGroupWrapper struct {\n\tsync.WaitGroup\n}\n\nfunc (w *WaitGroupWrapper) Wrap(cb func()) {\n\tw.Add(1)\n\tgo func() {\n\t\tcb()\n\t\tw.Done()\n\t}()\n}\n\n// Path: internal/writers/boundary_buffered_writer.go\npackage writers\n\nimport (\n\t\"bufio\"\n\t\"io\"\n)\n\ntype BoundaryBufferedWriter struct {\n\tbw *bufio.Writer\n}\n\nfunc NewBoundaryBufferedWriter(w io.Writer, size int) *BoundaryBufferedWriter {\n\treturn &BoundaryBufferedWriter{\n\t\tbw: bufio.NewWriterSize(w, size),\n\t}\n}\n\nfunc (b *BoundaryBufferedWriter) Write(p []byte) (int, error) {\n\tif len(p) > b.bw.Available() {\n\t\terr := b.bw.Flush()\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t}\n\treturn b.bw.Write(p)\n}\n\nfunc (b *BoundaryBufferedWriter) Flush() error {\n\treturn b.bw.Flush()\n}\n\n// Path: internal/writers/spread_writer.go\npackage writers\n\nimport (\n\t\"io\"\n\t\"time\"\n)\n\ntype SpreadWriter struct {\n\tw        io.Writer\n\tinterval time.Duration\n\tbuf      [][]byte\n\texitCh   chan int\n}\n\nfunc NewSpreadWriter(w io.Writer, interval time.Duration, exitCh chan int) *SpreadWriter {\n\treturn &SpreadWriter{\n\t\tw:        w,\n\t\tinterval: interval,\n\t\tbuf:      make([][]byte, 0),\n\t\texitCh:   exitCh,\n\t}\n}\n\nfunc (s *SpreadWriter) Write(p []byte) (int, error) {\n\tb := make([]byte, len(p))\n\tcopy(b, p)\n\ts.buf = append(s.buf, b)\n\treturn len(p), nil\n}\n\nfunc (s *SpreadWriter) Flush() {\n\tif len(s.buf) == 0 {\n\t\t// nothing to write, just wait\n\t\tselect {\n\t\tcase <-time.After(s.interval):\n\t\tcase <-s.exitCh:\n\t\t}\n\t\treturn\n\t}\n\tsleep := s.interval / time.Duration(len(s.buf))\n\tticker := time.NewTicker(sleep)\n\tfor _, b := range s.buf {\n\t\ts.w.Write(b)\n\t\tselect {\n\t\tcase <-ticker.C:\n\t\tcase <-s.exitCh: // skip sleeps finish writes\n\t\t}\n\t}\n\tticker.Stop()\n\ts.buf = s.buf[:0]\n}\n\n", "needle_token_start": 5248, "needle_token_end": 5286, "code_context_ntokens": 8701, "output": ["```go\npackage statsd\n\nimport (\n\t\"strings\"\n)\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \"_\", -1), \":\", \"_\", -1)\n}\n```"]}
{"repo": "nsqio/nsq", "name": "TestHostNameAddresses", "language": "go", "path": "internal/clusterinfo/producer_test.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To verify that the methods responsible for constructing full HTTP and TCP addresses from a host's domain name and respective ports are functioning correctly.\n2. **Input**: A predefined object representing a producer with a broadcast address and specific TCP and HTTP ports.\n3. **Output**: There is no direct output from the test, but it internally checks if the constructed addresses match the expected format \"host.domain.com:4151\" for HTTP and \"host.domain.com:4150\" for TCP.\n4. **Procedure**: The test initializes an object with a broadcast address and port numbers, invokes the methods to construct the full HTTP and TCP addresses, and then compares these addresses against expected values. If there is a mismatch, the test records an error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/clusterinfo/data.go\npackage clusterinfo\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/http_api\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype PartialErr interface {\n\terror\n\tErrors() []error\n}\n\ntype ErrList []error\n\nfunc (l ErrList) Error() string {\n\tvar es []string\n\tfor _, e := range l {\n\t\tes = append(es, e.Error())\n\t}\n\treturn strings.Join(es, \"\\n\")\n}\n\nfunc (l ErrList) Errors() []error {\n\treturn l\n}\n\ntype ClusterInfo struct {\n\tlog    lg.AppLogFunc\n\tclient *http_api.Client\n}\n\nfunc New(log lg.AppLogFunc, client *http_api.Client) *ClusterInfo {\n\treturn &ClusterInfo{\n\t\tlog:    log,\n\t\tclient: client,\n\t}\n}\n\nfunc (c *ClusterInfo) logf(f string, args ...interface{}) {\n\tif c.log != nil {\n\t\tc.log(lg.INFO, f, args...)\n\t}\n}\n\n// GetVersion returns a semver.Version object by querying /info\nfunc (c *ClusterInfo) GetVersion(addr string) (semver.Version, error) {\n\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\tvar resp struct {\n\t\tVersion string `json:\"version\"`\n\t}\n\terr := c.client.GETV1(endpoint, &resp)\n\tif err != nil {\n\t\treturn semver.Version{}, err\n\t}\n\tif resp.Version == \"\" {\n\t\tresp.Version = \"unknown\"\n\t}\n\treturn semver.Parse(resp.Version)\n}\n\n// GetLookupdTopics returns a []string containing a union of all the topics\n// from all the given nsqlookupd\nfunc (c *ClusterInfo) GetLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []string `json:\"topics\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/topics\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\ttopics = append(topics, resp.Topics...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\ttopics = stringy.Uniq(topics)\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetLookupdTopicChannels returns a []string containing a union of all the channels\n// from all the given lookupd for the given topic\nfunc (c *ClusterInfo) GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {\n\tvar channels []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tChannels []string `json:\"channels\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/channels?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tchannels = append(channels, resp.Channels...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tchannels = stringy.Uniq(channels)\n\tsort.Strings(channels)\n\n\tif len(errs) > 0 {\n\t\treturn channels, ErrList(errs)\n\t}\n\treturn channels, nil\n}\n\n// GetLookupdProducers returns Producers of all the nsqd connected to the given lookupds\nfunc (c *ClusterInfo) GetLookupdProducers(lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers []*Producer\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\tproducersByAddr := make(map[string]*Producer)\n\tmaxVersion, _ := semver.Parse(\"0.0.0\")\n\n\ttype respType struct {\n\t\tProducers []*Producer `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/nodes\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, producer := range resp.Producers {\n\t\t\t\tkey := producer.TCPAddress()\n\t\t\t\tp, ok := producersByAddr[key]\n\t\t\t\tif !ok {\n\t\t\t\t\tproducersByAddr[key] = producer\n\t\t\t\t\tproducers = append(producers, producer)\n\t\t\t\t\tif maxVersion.LT(producer.VersionObj) {\n\t\t\t\t\t\tmaxVersion = producer.VersionObj\n\t\t\t\t\t}\n\t\t\t\t\tsort.Sort(producer.Topics)\n\t\t\t\t\tp = producer\n\t\t\t\t}\n\t\t\t\tp.RemoteAddresses = append(p.RemoteAddresses,\n\t\t\t\t\tfmt.Sprintf(\"%s/%s\", addr, producer.Address()))\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tfor _, producer := range producersByAddr {\n\t\tif producer.VersionObj.LT(maxVersion) {\n\t\t\tproducer.OutOfDate = true\n\t\t}\n\t}\n\tsort.Sort(ProducersByHost{producers})\n\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetLookupdTopicProducers returns Producers of all the nsqd for a given topic by\n// unioning the nodes returned from the given lookupd\nfunc (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tProducers Producers `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/lookup?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, p := range resp.Producers {\n\t\t\t\tfor _, pp := range producers {\n\t\t\t\t\tif p.HTTPAddress() == pp.HTTPAddress() {\n\t\t\t\t\t\tgoto skip\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tproducers = append(producers, p)\n\t\t\tskip:\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDTopics returns a []string containing all the topics produced by the given nsqd\nfunc (c *ClusterInfo) GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\ttopics = stringy.Add(topics, topic.Name)\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetNSQDProducers returns Producers of all the given nsqd\nfunc (c *ClusterInfo) GetNSQDProducers(nsqdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype infoRespType struct {\n\t\tVersion          string `json:\"version\"`\n\t\tBroadcastAddress string `json:\"broadcast_address\"`\n\t\tHostname         string `json:\"hostname\"`\n\t\tHTTPPort         int    `json:\"http_port\"`\n\t\tTCPPort          int    `json:\"tcp_port\"`\n\t}\n\n\ttype statsRespType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar infoResp infoRespType\n\t\t\terr := c.client.GETV1(endpoint, &infoResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tendpoint = fmt.Sprintf(\"http://%s/stats?format=json&include_clients=false\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar statsResp statsRespType\n\t\t\terr = c.client.GETV1(endpoint, &statsResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tvar producerTopics ProducerTopics\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tproducerTopics = append(producerTopics, ProducerTopic{Topic: t.Name})\n\t\t\t}\n\n\t\t\tversion, err := semver.Parse(infoResp.Version)\n\t\t\tif err != nil {\n\t\t\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tproducers = append(producers, &Producer{\n\t\t\t\tVersion:          infoResp.Version,\n\t\t\t\tVersionObj:       version,\n\t\t\t\tBroadcastAddress: infoResp.BroadcastAddress,\n\t\t\t\tHostname:         infoResp.Hostname,\n\t\t\t\tHTTPPort:         infoResp.HTTPPort,\n\t\t\t\tTCPPort:          infoResp.TCPPort,\n\t\t\t\tTopics:           producerTopics,\n\t\t\t})\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDTopicProducers returns Producers containing the addresses of all the nsqd\n// that produce the given topic\nfunc (c *ClusterInfo) GetNSQDTopicProducers(topic string, nsqdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype infoRespType struct {\n\t\tVersion          string `json:\"version\"`\n\t\tBroadcastAddress string `json:\"broadcast_address\"`\n\t\tHostname         string `json:\"hostname\"`\n\t\tHTTPPort         int    `json:\"http_port\"`\n\t\tTCPPort          int    `json:\"tcp_port\"`\n\t}\n\n\ttype statsRespType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json&topic=%s&include_clients=false\",\n\t\t\t\taddr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar statsResp statsRespType\n\t\t\terr := c.client.GETV1(endpoint, &statsResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tvar producerTopics ProducerTopics\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tproducerTopics = append(producerTopics, ProducerTopic{Topic: t.Name})\n\t\t\t}\n\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tif t.Name == topic {\n\t\t\t\t\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\t\t\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\t\t\tvar infoResp infoRespType\n\t\t\t\t\terr := c.client.GETV1(endpoint, &infoResp)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlock.Lock()\n\t\t\t\t\t\terrs = append(errs, err)\n\t\t\t\t\t\tlock.Unlock()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\tversion, err := semver.Parse(infoResp.Version)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t\t\t\t\t}\n\n\t\t\t\t\t// if BroadcastAddress/HTTPPort are missing, use the values from `addr` for\n\t\t\t\t\t// backwards compatibility\n\n\t\t\t\t\tif infoResp.BroadcastAddress == \"\" {\n\t\t\t\t\t\tvar p string\n\t\t\t\t\t\tinfoResp.BroadcastAddress, p, _ = net.SplitHostPort(addr)\n\t\t\t\t\t\tinfoResp.HTTPPort, _ = strconv.Atoi(p)\n\t\t\t\t\t}\n\t\t\t\t\tif infoResp.Hostname == \"\" {\n\t\t\t\t\t\tinfoResp.Hostname, _, _ = net.SplitHostPort(addr)\n\t\t\t\t\t}\n\n\t\t\t\t\tlock.Lock()\n\t\t\t\t\tproducers = append(producers, &Producer{\n\t\t\t\t\t\tVersion:          infoResp.Version,\n\t\t\t\t\t\tVersionObj:       version,\n\t\t\t\t\t\tBroadcastAddress: infoResp.BroadcastAddress,\n\t\t\t\t\t\tHostname:         infoResp.Hostname,\n\t\t\t\t\t\tHTTPPort:         infoResp.HTTPPort,\n\t\t\t\t\t\tTCPPort:          infoResp.TCPPort,\n\t\t\t\t\t\tTopics:           producerTopics,\n\t\t\t\t\t})\n\t\t\t\t\tlock.Unlock()\n\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDStats returns aggregate topic and channel stats from the given Producers\n//\n// if selectedChannel is empty, this will return stats for topic/channel\n// if selectedTopic is empty, this will return stats for *all* topic/channels\n// if includeClients is false, this will *not* return client stats for channels\n// and the ChannelStats dict will be keyed by topic + ':' + channel\nfunc (c *ClusterInfo) GetNSQDStats(producers Producers,\n\tselectedTopic string, selectedChannel string,\n\tincludeClients bool) ([]*TopicStats, map[string]*ChannelStats, error) {\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar topicStatsList TopicStatsList\n\tvar errs []error\n\n\tchannelStatsMap := make(map[string]*ChannelStats)\n\n\ttype respType struct {\n\t\tTopics []*TopicStats `json:\"topics\"`\n\t}\n\n\tfor _, p := range producers {\n\t\twg.Add(1)\n\t\tgo func(p *Producer) {\n\t\t\tdefer wg.Done()\n\n\t\t\taddr := p.HTTPAddress()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json\", addr)\n\t\t\tif selectedTopic != \"\" {\n\t\t\t\tendpoint += \"&topic=\" + url.QueryEscape(selectedTopic)\n\t\t\t\tif selectedChannel != \"\" {\n\t\t\t\t\tendpoint += \"&channel=\" + url.QueryEscape(selectedChannel)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !includeClients {\n\t\t\t\tendpoint += \"&include_clients=false\"\n\t\t\t}\n\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\ttopic.Node = addr\n\t\t\t\ttopic.Hostname = p.Hostname\n\t\t\t\ttopic.MemoryDepth = topic.Depth - topic.BackendDepth\n\t\t\t\tif selectedTopic != \"\" && topic.TopicName != selectedTopic {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\ttopicStatsList = append(topicStatsList, topic)\n\n\t\t\t\tfor _, channel := range topic.Channels {\n\t\t\t\t\tchannel.Node = addr\n\t\t\t\t\tchannel.Hostname = p.Hostname\n\t\t\t\t\tchannel.TopicName = topic.TopicName\n\t\t\t\t\tchannel.MemoryDepth = channel.Depth - channel.BackendDepth\n\t\t\t\t\tkey := channel.ChannelName\n\t\t\t\t\tif selectedTopic == \"\" {\n\t\t\t\t\t\tkey = fmt.Sprintf(\"%s:%s\", topic.TopicName, channel.ChannelName)\n\t\t\t\t\t}\n\t\t\t\t\tchannelStats, ok := channelStatsMap[key]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tchannelStats = &ChannelStats{\n\t\t\t\t\t\t\tNode:        addr,\n\t\t\t\t\t\t\tTopicName:   topic.TopicName,\n\t\t\t\t\t\t\tChannelName: channel.ChannelName,\n\t\t\t\t\t\t}\n\t\t\t\t\t\tchannelStatsMap[key] = channelStats\n\t\t\t\t\t}\n\t\t\t\t\tfor _, c := range channel.Clients {\n\t\t\t\t\t\tc.Node = addr\n\t\t\t\t\t}\n\t\t\t\t\tchannelStats.Add(channel)\n\t\t\t\t}\n\t\t\t}\n\t\t}(p)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(producers) {\n\t\treturn nil, nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\n\tsort.Sort(TopicStatsByHost{topicStatsList})\n\n\tif len(errs) > 0 {\n\t\treturn topicStatsList, channelStatsMap, ErrList(errs)\n\t}\n\treturn topicStatsList, channelStatsMap, nil\n}\n\n// TombstoneNodeForTopic tombstones the given node for the given topic on all the given nsqlookupd\n// and deletes the topic from the node\nfunc (c *ClusterInfo) TombstoneNodeForTopic(topic string, node string, lookupdHTTPAddrs []string) error {\n\tvar errs []error\n\n\t// tombstone the topic on all the lookupds\n\tqs := fmt.Sprintf(\"topic=%s&node=%s\", url.QueryEscape(topic), url.QueryEscape(node))\n\terr := c.nsqlookupdPOST(lookupdHTTPAddrs, \"topic/tombstone\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tproducers, err := c.GetNSQDProducers([]string{node})\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\t// delete the topic on the producer\n...\n// Path: internal/clusterinfo/producer_test.go\npackage clusterinfo\n\nimport \"testing\"\n\n\nfunc TestHostNameAddresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"host.domain.com\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"host.domain.com:4151\" {\n\t\tt.Errorf(\"Incorrect HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"host.domain.com:4150\" {\n\t\tt.Errorf(\"Incorrect TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv4Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"192.168.1.17\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"192.168.1.17:4151\" {\n\t\tt.Errorf(\"Incorrect IPv4 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"192.168.1.17:4150\" {\n\t\tt.Errorf(\"Incorrect IPv4 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv6Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"fd4a:622f:d2f2::1\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\tif p.HTTPAddress() != \"[fd4a:622f:d2f2::1]:4151\" {\n\t\tt.Errorf(\"Incorrect IPv6 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"[fd4a:622f:d2f2::1]:4150\" {\n\t\tt.Errorf(\"Incorrect IPv6 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\n// Path: internal/quantile/quantile.go\npackage quantile\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/bmizerany/perks/quantile\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype Result struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n}\n\nfunc (r *Result) String() string {\n\tvar s []string\n\tfor _, item := range r.Percentiles {\n\t\ts = append(s, stringy.NanoSecondToHuman(item[\"value\"]))\n\t}\n\treturn strings.Join(s, \", \")\n}\n\ntype Quantile struct {\n\tsync.Mutex\n\tstreams        [2]quantile.Stream\n\tcurrentIndex   uint8\n\tlastMoveWindow time.Time\n\tcurrentStream  *quantile.Stream\n\n\tPercentiles    []float64\n\tMoveWindowTime time.Duration\n}\n\nfunc New(WindowTime time.Duration, Percentiles []float64) *Quantile {\n\tq := Quantile{\n\t\tcurrentIndex:   0,\n\t\tlastMoveWindow: time.Now(),\n\t\tMoveWindowTime: WindowTime / 2,\n\t\tPercentiles:    Percentiles,\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tq.streams[i] = *quantile.NewTargeted(Percentiles...)\n\t}\n\tq.currentStream = &q.streams[0]\n\treturn &q\n}\n\nfunc (q *Quantile) Result() *Result {\n\tif q == nil {\n\t\treturn &Result{}\n\t}\n\tqueryHandler := q.QueryHandler()\n\tresult := Result{\n\t\tCount:       queryHandler.Count(),\n\t\tPercentiles: make([]map[string]float64, len(q.Percentiles)),\n\t}\n\tfor i, p := range q.Percentiles {\n\t\tvalue := queryHandler.Query(p)\n\t\tresult.Percentiles[i] = map[string]float64{\"quantile\": p, \"value\": value}\n\t}\n\treturn &result\n}\n\nfunc (q *Quantile) Insert(msgStartTime int64) {\n\tq.Lock()\n\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tq.currentStream.Insert(float64(now.UnixNano() - msgStartTime))\n\tq.Unlock()\n}\n\nfunc (q *Quantile) QueryHandler() *quantile.Stream {\n\tq.Lock()\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tmerged := quantile.NewTargeted(q.Percentiles...)\n\tmerged.Merge(q.streams[0].Samples())\n\tmerged.Merge(q.streams[1].Samples())\n\tq.Unlock()\n\treturn merged\n}\n\nfunc (q *Quantile) IsDataStale(now time.Time) bool {\n\treturn now.After(q.lastMoveWindow.Add(q.MoveWindowTime))\n}\n\nfunc (q *Quantile) Merge(them *Quantile) {\n\tq.Lock()\n\tthem.Lock()\n\tiUs := q.currentIndex\n\tiThem := them.currentIndex\n\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tiUs ^= 0x1\n\tiThem ^= 0x1\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tif q.lastMoveWindow.Before(them.lastMoveWindow) {\n\t\tq.lastMoveWindow = them.lastMoveWindow\n\t}\n\tq.Unlock()\n\tthem.Unlock()\n}\n\nfunc (q *Quantile) moveWindow() {\n\tq.currentIndex ^= 0x1\n\tq.currentStream = &q.streams[q.currentIndex]\n\tq.lastMoveWindow = q.lastMoveWindow.Add(q.MoveWindowTime)\n\tq.currentStream.Reset()\n}\n\n// Path: internal/clusterinfo/types.go\npackage clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype ProducerTopic struct {\n\tTopic      string `json:\"topic\"`\n\tTombstoned bool   `json:\"tombstoned\"`\n}\n\ntype ProducerTopics []ProducerTopic\n\nfunc (pt ProducerTopics) Len() int           { return len(pt) }\nfunc (pt ProducerTopics) Swap(i, j int)      { pt[i], pt[j] = pt[j], pt[i] }\nfunc (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt[j].Topic }\n\ntype Producer struct {\n\tRemoteAddresses  []string       `json:\"remote_addresses\"`\n\tRemoteAddress    string         `json:\"remote_address\"`\n\tHostname         string         `json:\"hostname\"`\n\tBroadcastAddress string         `json:\"broadcast_address\"`\n\tTCPPort          int            `json:\"tcp_port\"`\n\tHTTPPort         int            `json:\"http_port\"`\n\tVersion          string         `json:\"version\"`\n\tVersionObj       semver.Version `json:\"-\"`\n\tTopics           ProducerTopics `json:\"topics\"`\n\tOutOfDate        bool           `json:\"out_of_date\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses of ProducerTopics and VersionObj\nfunc (p *Producer) UnmarshalJSON(b []byte) error {\n\tvar r struct {\n\t\tRemoteAddress    string   `json:\"remote_address\"`\n\t\tHostname         string   `json:\"hostname\"`\n\t\tBroadcastAddress string   `json:\"broadcast_address\"`\n\t\tTCPPort          int      `json:\"tcp_port\"`\n\t\tHTTPPort         int      `json:\"http_port\"`\n\t\tVersion          string   `json:\"version\"`\n\t\tTopics           []string `json:\"topics\"`\n\t\tTombstoned       []bool   `json:\"tombstones\"`\n\t}\n\tif err := json.Unmarshal(b, &r); err != nil {\n\t\treturn err\n\t}\n\t*p = Producer{\n\t\tRemoteAddress:    r.RemoteAddress,\n\t\tHostname:         r.Hostname,\n\t\tBroadcastAddress: r.BroadcastAddress,\n\t\tTCPPort:          r.TCPPort,\n\t\tHTTPPort:         r.HTTPPort,\n\t\tVersion:          r.Version,\n\t}\n\tfor i, t := range r.Topics {\n\t\tp.Topics = append(p.Topics, ProducerTopic{Topic: t, Tombstoned: r.Tombstoned[i]})\n\t}\n\tversion, err := semver.Parse(p.Version)\n\tif err != nil {\n\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t}\n\tp.VersionObj = version\n\treturn nil\n}\n\nfunc (p *Producer) Address() string {\n\tif p.RemoteAddress == \"\" {\n\t\treturn \"N/A\"\n\t}\n\treturn p.RemoteAddress\n}\n\nfunc (p *Producer) HTTPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.HTTPPort))\n}\n\nfunc (p *Producer) TCPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.TCPPort))\n}\n\n// IsInconsistent checks for cases where an unexpected number of nsqd connections are\n// reporting the same information to nsqlookupd (ie: multiple instances are using the\n// same broadcast address), or cases where some nsqd are not reporting to all nsqlookupd.\nfunc (p *Producer) IsInconsistent(numLookupd int) bool {\n\treturn len(p.RemoteAddresses) != numLookupd\n}\n\ntype TopicStats struct {\n\tNode         string          `json:\"node\"`\n\tHostname     string          `json:\"hostname\"`\n\tTopicName    string          `json:\"topic_name\"`\n\tDepth        int64           `json:\"depth\"`\n\tMemoryDepth  int64           `json:\"memory_depth\"`\n\tBackendDepth int64           `json:\"backend_depth\"`\n\tMessageCount int64           `json:\"message_count\"`\n\tNodeStats    []*TopicStats   `json:\"nodes\"`\n\tChannels     []*ChannelStats `json:\"channels\"`\n\tPaused       bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (t *TopicStats) Add(a *TopicStats) {\n\tt.Node = \"*\"\n\tt.Depth += a.Depth\n\tt.MemoryDepth += a.MemoryDepth\n\tt.BackendDepth += a.BackendDepth\n\tt.MessageCount += a.MessageCount\n\tif a.Paused {\n\t\tt.Paused = a.Paused\n\t}\n\tfor _, aChannelStats := range a.Channels {\n\t\tfound := false\n\t\tfor _, channelStats := range t.Channels {\n\t\t\tif aChannelStats.ChannelName == channelStats.ChannelName {\n\t\t\t\tfound = true\n\t\t\t\tchannelStats.Add(aChannelStats)\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\tt.Channels = append(t.Channels, aChannelStats)\n\t\t}\n\t}\n\tt.NodeStats = append(t.NodeStats, a)\n\tsort.Sort(TopicStatsByHost{t.NodeStats})\n\tif t.E2eProcessingLatency == nil {\n\t\tt.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:  t.Node,\n\t\t\tTopic: t.TopicName,\n\t\t}\n\t}\n\tt.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n}\n\ntype ChannelStats struct {\n\tNode          string          `json:\"node\"`\n\tHostname      string          `json:\"hostname\"`\n\tTopicName     string          `json:\"topic_name\"`\n\tChannelName   string          `json:\"channel_name\"`\n\tDepth         int64           `json:\"depth\"`\n\tMemoryDepth   int64           `json:\"memory_depth\"`\n\tBackendDepth  int64           `json:\"backend_depth\"`\n\tInFlightCount int64           `json:\"in_flight_count\"`\n\tDeferredCount int64           `json:\"deferred_count\"`\n\tRequeueCount  int64           `json:\"requeue_count\"`\n\tTimeoutCount  int64           `json:\"timeout_count\"`\n\tMessageCount  int64           `json:\"message_count\"`\n\tClientCount   int             `json:\"client_count\"`\n\tSelected      bool            `json:\"-\"`\n\tNodeStats     []*ChannelStats `json:\"nodes\"`\n\tClients       []*ClientStats  `json:\"clients\"`\n\tPaused        bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (c *ChannelStats) Add(a *ChannelStats) {\n\tc.Node = \"*\"\n\tc.Depth += a.Depth\n\tc.MemoryDepth += a.MemoryDepth\n\tc.BackendDepth += a.BackendDepth\n\tc.InFlightCount += a.InFlightCount\n\tc.DeferredCount += a.DeferredCount\n\tc.RequeueCount += a.RequeueCount\n\tc.TimeoutCount += a.TimeoutCount\n\tc.MessageCount += a.MessageCount\n\tc.ClientCount += a.ClientCount\n\tif a.Paused {\n\t\tc.Paused = a.Paused\n\t}\n\tc.NodeStats = append(c.NodeStats, a)\n\tsort.Sort(ChannelStatsByHost{c.NodeStats})\n\tif c.E2eProcessingLatency == nil {\n\t\tc.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:    c.Node,\n\t\t\tTopic:   c.TopicName,\n\t\t\tChannel: c.ChannelName,\n\t\t}\n\t}\n\tc.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n\tc.Clients = append(c.Clients, a.Clients...)\n\tsort.Sort(ClientsByHost{c.Clients})\n}\n\ntype ClientStats struct {\n\tNode              string        `json:\"node\"`\n\tRemoteAddress     string        `json:\"remote_address\"`\n\tVersion           string        `json:\"version\"`\n\tClientID          string        `json:\"client_id\"`\n\tHostname          string        `json:\"hostname\"`\n\tUserAgent         string        `json:\"user_agent\"`\n\tConnectTs         int64         `json:\"connect_ts\"`\n\tConnectedDuration time.Duration `json:\"connected\"`\n\tInFlightCount     int           `json:\"in_flight_count\"`\n\tReadyCount        int           `json:\"ready_count\"`\n\tFinishCount       int64         `json:\"finish_count\"`\n\tRequeueCount      int64         `json:\"requeue_count\"`\n\tMessageCount      int64         `json:\"message_count\"`\n\tSampleRate        int32         `json:\"sample_rate\"`\n\tDeflate           bool          `json:\"deflate\"`\n\tSnappy            bool          `json:\"snappy\"`\n\tAuthed            bool          `json:\"authed\"`\n\tAuthIdentity      string        `json:\"auth_identity\"`\n\tAuthIdentityURL   string        `json:\"auth_identity_url\"`\n\n\tTLS                           bool   `json:\"tls\"`\n\tCipherSuite                   string `json:\"tls_cipher_suite\"`\n\tTLSVersion                    string `json:\"tls_version\"`\n\tTLSNegotiatedProtocol         string `json:\"tls_negotiated_protocol\"`\n\tTLSNegotiatedProtocolIsMutual bool   `json:\"tls_negotiated_protocol_is_mutual\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses ConnectedDuration\nfunc (s *ClientStats) UnmarshalJSON(b []byte) error {\n\ttype locaClientStats ClientStats // re-typed to prevent recursion from json.Unmarshal\n\tvar ss locaClientStats\n\tif err := json.Unmarshal(b, &ss); err != nil {\n\t\treturn err\n\t}\n\t*s = ClientStats(ss)\n\ts.ConnectedDuration = time.Now().Truncate(time.Second).Sub(time.Unix(s.ConnectTs, 0))\n\treturn nil\n}\n\nfunc (s *ClientStats) HasUserAgent() bool {\n\treturn s.UserAgent != \"\"\n}\n\nfunc (s *ClientStats) HasSampleRate() bool {\n\treturn s.SampleRate > 0\n}\n\ntype ChannelStatsList []*ChannelStats\n\nfunc (c ChannelStatsList) Len() int      { return len(c) }\nfunc (c ChannelStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ChannelStatsByHost struct {\n\tChannelStatsList\n}\n\nfunc (c ChannelStatsByHost) Less(i, j int) bool {\n\treturn c.ChannelStatsList[i].Hostname < c.ChannelStatsList[j].Hostname\n}\n\ntype ClientStatsList []*ClientStats\n\nfunc (c ClientStatsList) Len() int      { return len(c) }\nfunc (c ClientStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ClientsByHost struct {\n\tClientStatsList\n}\n\nfunc (c ClientsByHost) Less(i, j int) bool {\n\treturn c.ClientStatsList[i].Hostname < c.ClientStatsList[j].Hostname\n}\n\ntype TopicStatsList []*TopicStats\n\nfunc (t TopicStatsList) Len() int      { return len(t) }\nfunc (t TopicStatsList) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\ntype TopicStatsByHost struct {\n\tTopicStatsList\n}\n\nfunc (c TopicStatsByHost) Less(i, j int) bool {\n\treturn c.TopicStatsList[i].Hostname < c.TopicStatsList[j].Hostname\n}\n\ntype Producers []*Producer\n\nfunc (t Producers) Len() int      { return len(t) }\nfunc (t Producers) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\nfunc (t Producers) HTTPAddrs() []string {\n\tvar addrs []string\n\tfor _, p := range t {\n\t\taddrs = append(addrs, p.HTTPAddress())\n\t}\n\treturn addrs\n}\n\nfunc (t Producers) Search(needle string) *Producer {\n\tfor _, producer := range t {\n\t\tif needle == producer.HTTPAddress() {\n\t\t\treturn producer\n\t\t}\n\t}\n\treturn nil\n}\n\ntype ProducersByHost struct {\n\tProducers\n}\n\nfunc (c ProducersByHost) Less(i, j int) bool {\n\treturn c.Producers[i].Hostname < c.Producers[j].Hostname\n}\n\n// Path: internal/dirlock/dirlock.go\n//go:build !windows && !illumos\n// +build !windows,!illumos\n\npackage dirlock\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n)\n\ntype DirLock struct {\n\tdir string\n\tf   *os.File\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\tf, err := os.Open(l.dir)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl.f = f\n\terr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"cannot flock directory %s - %s (possibly in use by another instance of nsqd)\", l.dir, err)\n\t}\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\tdefer l.f.Close()\n\treturn syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)\n}\n\n// Path: internal/dirlock/dirlock_illumos.go\n//go:build illumos\n// +build illumos\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/dirlock/dirlock_windows.go\n//go:build windows\n// +build windows\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/http_api/api_request.go\npackage http_api\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// A custom http.Transport with support for deadline timeouts\nfunc NewDeadlineTransport(connectTimeout time.Duration, requestTimeout time.Duration) *http.Transport {\n\t// arbitrary values copied from http.DefaultTransport\n\ttransport := &http.Transport{\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout:   connectTimeout,\n\t\t\tKeepAlive: 30 * time.Second,\n\t\t\tDualStack: true,\n\t\t}).DialContext,\n\t\tResponseHeaderTimeout: requestTimeout,\n\t\tMaxIdleConns:          100,\n\t\tIdleConnTimeout:       90 * time.Second,\n\t\tTLSHandshakeTimeout:   10 * time.Second,\n\t}\n\treturn transport\n}\n\ntype Client struct {\n\tc *http.Client\n}\n\nfunc NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) *Client {\n\ttransport := NewDeadlineTransport(connectTimeout, requestTimeout)\n\ttransport.TLSClientConfig = tlsConfig\n\treturn &Client{\n\t\tc: &http.Client{\n\t\t\tTransport: transport,\n\t\t\tTimeout:   requestTimeout,\n\t\t},\n\t}\n}\n\n// GETV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) GETV1(endpoint string, v interface{}) error {\nretry:\n\treq, err := http.NewRequest(\"GET\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\terr = json.Unmarshal(body, &v)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// PostV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) POSTV1(endpoint string) error {\nretry:\n\treq, err := http.NewRequest(\"POST\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\n\treturn nil\n}\n\nfunc httpsEndpoint(endpoint string, body []byte) (string, error) {\n\tvar forbiddenResp struct {\n\t\tHTTPSPort int `json:\"https_port\"`\n\t}\n\terr := json.Unmarshal(body, &forbiddenResp)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thost, _, err := net.SplitHostPort(u.Host)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu.Scheme = \"https\"\n\tu.Host = net.JoinHostPort(host, strconv.Itoa(forbiddenResp.HTTPSPort))\n\treturn u.String(), nil\n}\n\n// Path: internal/http_api/api_response.go\npackage http_api\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/julienschmidt/httprouter\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Decorator func(APIHandler) APIHandler\n\ntype APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)\n\ntype Err struct {\n\tCode int\n\tText string\n}\n\nfunc (e Err) Error() string {\n\treturn e.Text\n}\n\nfunc PlainText(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tcode := 200\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tcode = err.(Err).Code\n\t\t\tdata = err.Error()\n\t\t}\n\t\tswitch d := data.(type) {\n\t\tcase string:\n\t\t\tw.WriteHeader(code)\n\t\t\tio.WriteString(w, d)\n\t\tcase []byte:\n\t\t\tw.WriteHeader(code)\n\t\t\tw.Write(d)\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"unknown response type %T\", data))\n\t\t}\n\t\treturn nil, nil\n\t}\n}\n\nfunc V1(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tRespondV1(w, err.(Err).Code, err)\n\t\t\treturn nil, nil\n\t\t}\n\t\tRespondV1(w, 200, data)\n\t\treturn nil, nil\n\t}\n}\n\nfunc RespondV1(w http.ResponseWriter, code int, data interface{}) {\n\tvar response []byte\n\tvar err error\n\tvar isJSON bool\n\n\tif code == 200 {\n\t\tswitch data := data.(type) {\n\t\tcase string:\n\t\t\tresponse = []byte(data)\n\t\tcase []byte:\n\t\t\tresponse = data\n\t\tcase nil:\n\t\t\tresponse = []byte{}\n\t\tdefault:\n\t\t\tisJSON = true\n\t\t\tresponse, err = json.Marshal(data)\n\t\t\tif err != nil {\n\t\t\t\tcode = 500\n\t\t\t\tdata = err\n\t\t\t}\n\t\t}\n\t}\n\n\tif code != 200 {\n\t\tisJSON = true\n\t\tresponse, _ = json.Marshal(struct {\n\t\t\tMessage string `json:\"message\"`\n\t\t}{fmt.Sprintf(\"%s\", data)})\n\t}\n\n\tif isJSON {\n\t\tw.Header().Set(\"Content-Type\", \"application/json; charset=utf-8\")\n\t}\n\tw.Header().Set(\"X-NSQ-Content-Type\", \"nsq; version=1.0\")\n\tw.WriteHeader(code)\n\tw.Write(response)\n}\n\nfunc Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {\n\tdecorated := f\n\tfor _, decorate := range ds {\n\t\tdecorated = decorate(decorated)\n\t}\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {\n\t\tdecorated(w, req, ps)\n\t}\n}\n\nfunc Log(logf lg.AppLogFunc) Decorator {\n\treturn func(f APIHandler) APIHandler {\n\t\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\tstart := time.Now()\n\t\t\tresponse, err := f(w, req, ps)\n\t\t\telapsed := time.Since(start)\n\t\t\tstatus := 200\n\t\t\tif e, ok := err.(Err); ok {\n\t\t\t\tstatus = e.Code\n\t\t\t}\n\t\t\tlogf(lg.INFO, \"%d %s %s (%s) %s\",\n\t\t\t\tstatus, req.Method, req.URL.RequestURI(), req.RemoteAddr, elapsed)\n\t\t\treturn response, err\n\t\t}\n\t}\n}\n\nfunc LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\treturn func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\t\tlogf(lg.ERROR, \"panic in HTTP handler - %s\", p)\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{500, \"INTERNAL_ERROR\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t}\n}\n\nfunc LogNotFoundHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{404, \"NOT_FOUND\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\nfunc LogMethodNotAllowedHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{405, \"METHOD_NOT_ALLOWED\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\n// Path: internal/http_api/compress.go\n// Copyright 2013 The Gorilla Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n...\n", "needle_token_start": 6686, "needle_token_end": 6848, "code_context_ntokens": 15110, "output": ["```go\nfunc TestHostNameAddresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"host.domain.com\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"host.domain.com:4151\" {\n\t\tt.Errorf(\"Incorrect HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"host.domain.com:4150\" {\n\t\tt.Errorf(\"Incorrect TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n```"]}
{"repo": "nsqio/nsq", "name": "SendResponse", "language": "go", "path": "internal/protocol/protocol.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to send data over a network connection with a prefixed length header to ensure the receiver knows the size of the incoming data packet.\n2. **Input**: It takes a writer interface and a byte slice as inputs. The writer interface is used to write data to a connection, and the byte slice contains the data to be sent.\n3. **Output**: It returns the total number of bytes written (including the length header) and an error if any occurs during the writing process.\n4. **Procedure**: The function first writes a 4-byte length header, representing the size of the data, in big-endian format. Then, it writes the actual data to the writer. It returns the sum of the bytes of the length header and the data written, along with any error encountered during the write operations.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/clusterinfo/types.go\npackage clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype ProducerTopic struct {\n\tTopic      string `json:\"topic\"`\n\tTombstoned bool   `json:\"tombstoned\"`\n}\n\ntype ProducerTopics []ProducerTopic\n\nfunc (pt ProducerTopics) Len() int           { return len(pt) }\nfunc (pt ProducerTopics) Swap(i, j int)      { pt[i], pt[j] = pt[j], pt[i] }\nfunc (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt[j].Topic }\n\ntype Producer struct {\n\tRemoteAddresses  []string       `json:\"remote_addresses\"`\n\tRemoteAddress    string         `json:\"remote_address\"`\n\tHostname         string         `json:\"hostname\"`\n\tBroadcastAddress string         `json:\"broadcast_address\"`\n\tTCPPort          int            `json:\"tcp_port\"`\n\tHTTPPort         int            `json:\"http_port\"`\n\tVersion          string         `json:\"version\"`\n\tVersionObj       semver.Version `json:\"-\"`\n\tTopics           ProducerTopics `json:\"topics\"`\n\tOutOfDate        bool           `json:\"out_of_date\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses of ProducerTopics and VersionObj\nfunc (p *Producer) UnmarshalJSON(b []byte) error {\n\tvar r struct {\n\t\tRemoteAddress    string   `json:\"remote_address\"`\n\t\tHostname         string   `json:\"hostname\"`\n\t\tBroadcastAddress string   `json:\"broadcast_address\"`\n\t\tTCPPort          int      `json:\"tcp_port\"`\n\t\tHTTPPort         int      `json:\"http_port\"`\n\t\tVersion          string   `json:\"version\"`\n\t\tTopics           []string `json:\"topics\"`\n\t\tTombstoned       []bool   `json:\"tombstones\"`\n\t}\n\tif err := json.Unmarshal(b, &r); err != nil {\n\t\treturn err\n\t}\n\t*p = Producer{\n\t\tRemoteAddress:    r.RemoteAddress,\n\t\tHostname:         r.Hostname,\n\t\tBroadcastAddress: r.BroadcastAddress,\n\t\tTCPPort:          r.TCPPort,\n\t\tHTTPPort:         r.HTTPPort,\n\t\tVersion:          r.Version,\n\t}\n\tfor i, t := range r.Topics {\n\t\tp.Topics = append(p.Topics, ProducerTopic{Topic: t, Tombstoned: r.Tombstoned[i]})\n\t}\n\tversion, err := semver.Parse(p.Version)\n\tif err != nil {\n\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t}\n\tp.VersionObj = version\n\treturn nil\n}\n\nfunc (p *Producer) Address() string {\n\tif p.RemoteAddress == \"\" {\n\t\treturn \"N/A\"\n\t}\n\treturn p.RemoteAddress\n}\n\nfunc (p *Producer) HTTPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.HTTPPort))\n}\n\nfunc (p *Producer) TCPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.TCPPort))\n}\n\n// IsInconsistent checks for cases where an unexpected number of nsqd connections are\n// reporting the same information to nsqlookupd (ie: multiple instances are using the\n// same broadcast address), or cases where some nsqd are not reporting to all nsqlookupd.\nfunc (p *Producer) IsInconsistent(numLookupd int) bool {\n\treturn len(p.RemoteAddresses) != numLookupd\n}\n\ntype TopicStats struct {\n\tNode         string          `json:\"node\"`\n\tHostname     string          `json:\"hostname\"`\n\tTopicName    string          `json:\"topic_name\"`\n\tDepth        int64           `json:\"depth\"`\n\tMemoryDepth  int64           `json:\"memory_depth\"`\n\tBackendDepth int64           `json:\"backend_depth\"`\n\tMessageCount int64           `json:\"message_count\"`\n\tNodeStats    []*TopicStats   `json:\"nodes\"`\n\tChannels     []*ChannelStats `json:\"channels\"`\n\tPaused       bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (t *TopicStats) Add(a *TopicStats) {\n\tt.Node = \"*\"\n\tt.Depth += a.Depth\n\tt.MemoryDepth += a.MemoryDepth\n\tt.BackendDepth += a.BackendDepth\n\tt.MessageCount += a.MessageCount\n\tif a.Paused {\n\t\tt.Paused = a.Paused\n\t}\n\tfor _, aChannelStats := range a.Channels {\n\t\tfound := false\n\t\tfor _, channelStats := range t.Channels {\n\t\t\tif aChannelStats.ChannelName == channelStats.ChannelName {\n\t\t\t\tfound = true\n\t\t\t\tchannelStats.Add(aChannelStats)\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\tt.Channels = append(t.Channels, aChannelStats)\n\t\t}\n\t}\n\tt.NodeStats = append(t.NodeStats, a)\n\tsort.Sort(TopicStatsByHost{t.NodeStats})\n\tif t.E2eProcessingLatency == nil {\n\t\tt.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:  t.Node,\n\t\t\tTopic: t.TopicName,\n\t\t}\n\t}\n\tt.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n}\n\ntype ChannelStats struct {\n\tNode          string          `json:\"node\"`\n\tHostname      string          `json:\"hostname\"`\n\tTopicName     string          `json:\"topic_name\"`\n\tChannelName   string          `json:\"channel_name\"`\n\tDepth         int64           `json:\"depth\"`\n\tMemoryDepth   int64           `json:\"memory_depth\"`\n\tBackendDepth  int64           `json:\"backend_depth\"`\n\tInFlightCount int64           `json:\"in_flight_count\"`\n\tDeferredCount int64           `json:\"deferred_count\"`\n\tRequeueCount  int64           `json:\"requeue_count\"`\n\tTimeoutCount  int64           `json:\"timeout_count\"`\n\tMessageCount  int64           `json:\"message_count\"`\n\tClientCount   int             `json:\"client_count\"`\n\tSelected      bool            `json:\"-\"`\n\tNodeStats     []*ChannelStats `json:\"nodes\"`\n\tClients       []*ClientStats  `json:\"clients\"`\n\tPaused        bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (c *ChannelStats) Add(a *ChannelStats) {\n\tc.Node = \"*\"\n\tc.Depth += a.Depth\n\tc.MemoryDepth += a.MemoryDepth\n\tc.BackendDepth += a.BackendDepth\n\tc.InFlightCount += a.InFlightCount\n\tc.DeferredCount += a.DeferredCount\n\tc.RequeueCount += a.RequeueCount\n\tc.TimeoutCount += a.TimeoutCount\n\tc.MessageCount += a.MessageCount\n\tc.ClientCount += a.ClientCount\n\tif a.Paused {\n\t\tc.Paused = a.Paused\n\t}\n\tc.NodeStats = append(c.NodeStats, a)\n\tsort.Sort(ChannelStatsByHost{c.NodeStats})\n\tif c.E2eProcessingLatency == nil {\n\t\tc.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:    c.Node,\n\t\t\tTopic:   c.TopicName,\n\t\t\tChannel: c.ChannelName,\n\t\t}\n\t}\n\tc.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n\tc.Clients = append(c.Clients, a.Clients...)\n\tsort.Sort(ClientsByHost{c.Clients})\n}\n\ntype ClientStats struct {\n\tNode              string        `json:\"node\"`\n\tRemoteAddress     string        `json:\"remote_address\"`\n\tVersion           string        `json:\"version\"`\n\tClientID          string        `json:\"client_id\"`\n\tHostname          string        `json:\"hostname\"`\n\tUserAgent         string        `json:\"user_agent\"`\n\tConnectTs         int64         `json:\"connect_ts\"`\n\tConnectedDuration time.Duration `json:\"connected\"`\n\tInFlightCount     int           `json:\"in_flight_count\"`\n\tReadyCount        int           `json:\"ready_count\"`\n\tFinishCount       int64         `json:\"finish_count\"`\n\tRequeueCount      int64         `json:\"requeue_count\"`\n\tMessageCount      int64         `json:\"message_count\"`\n\tSampleRate        int32         `json:\"sample_rate\"`\n\tDeflate           bool          `json:\"deflate\"`\n\tSnappy            bool          `json:\"snappy\"`\n\tAuthed            bool          `json:\"authed\"`\n\tAuthIdentity      string        `json:\"auth_identity\"`\n\tAuthIdentityURL   string        `json:\"auth_identity_url\"`\n\n\tTLS                           bool   `json:\"tls\"`\n\tCipherSuite                   string `json:\"tls_cipher_suite\"`\n\tTLSVersion                    string `json:\"tls_version\"`\n\tTLSNegotiatedProtocol         string `json:\"tls_negotiated_protocol\"`\n\tTLSNegotiatedProtocolIsMutual bool   `json:\"tls_negotiated_protocol_is_mutual\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses ConnectedDuration\nfunc (s *ClientStats) UnmarshalJSON(b []byte) error {\n\ttype locaClientStats ClientStats // re-typed to prevent recursion from json.Unmarshal\n\tvar ss locaClientStats\n\tif err := json.Unmarshal(b, &ss); err != nil {\n\t\treturn err\n\t}\n\t*s = ClientStats(ss)\n\ts.ConnectedDuration = time.Now().Truncate(time.Second).Sub(time.Unix(s.ConnectTs, 0))\n\treturn nil\n}\n\nfunc (s *ClientStats) HasUserAgent() bool {\n\treturn s.UserAgent != \"\"\n}\n\nfunc (s *ClientStats) HasSampleRate() bool {\n\treturn s.SampleRate > 0\n}\n\ntype ChannelStatsList []*ChannelStats\n\nfunc (c ChannelStatsList) Len() int      { return len(c) }\nfunc (c ChannelStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ChannelStatsByHost struct {\n\tChannelStatsList\n}\n\nfunc (c ChannelStatsByHost) Less(i, j int) bool {\n\treturn c.ChannelStatsList[i].Hostname < c.ChannelStatsList[j].Hostname\n}\n\ntype ClientStatsList []*ClientStats\n\nfunc (c ClientStatsList) Len() int      { return len(c) }\nfunc (c ClientStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ClientsByHost struct {\n\tClientStatsList\n}\n\nfunc (c ClientsByHost) Less(i, j int) bool {\n\treturn c.ClientStatsList[i].Hostname < c.ClientStatsList[j].Hostname\n}\n\ntype TopicStatsList []*TopicStats\n\nfunc (t TopicStatsList) Len() int      { return len(t) }\nfunc (t TopicStatsList) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\ntype TopicStatsByHost struct {\n\tTopicStatsList\n}\n\nfunc (c TopicStatsByHost) Less(i, j int) bool {\n\treturn c.TopicStatsList[i].Hostname < c.TopicStatsList[j].Hostname\n}\n\ntype Producers []*Producer\n\nfunc (t Producers) Len() int      { return len(t) }\n...\n// Path: internal/dirlock/dirlock.go\n//go:build !windows && !illumos\n// +build !windows,!illumos\n\npackage dirlock\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n)\n\ntype DirLock struct {\n\tdir string\n\tf   *os.File\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\tf, err := os.Open(l.dir)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl.f = f\n\terr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"cannot flock directory %s - %s (possibly in use by another instance of nsqd)\", l.dir, err)\n\t}\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\tdefer l.f.Close()\n\treturn syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)\n}\n\n// Path: internal/dirlock/dirlock_illumos.go\n//go:build illumos\n// +build illumos\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/dirlock/dirlock_windows.go\n//go:build windows\n// +build windows\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/http_api/api_request.go\npackage http_api\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// A custom http.Transport with support for deadline timeouts\nfunc NewDeadlineTransport(connectTimeout time.Duration, requestTimeout time.Duration) *http.Transport {\n\t// arbitrary values copied from http.DefaultTransport\n\ttransport := &http.Transport{\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout:   connectTimeout,\n\t\t\tKeepAlive: 30 * time.Second,\n\t\t\tDualStack: true,\n\t\t}).DialContext,\n\t\tResponseHeaderTimeout: requestTimeout,\n\t\tMaxIdleConns:          100,\n\t\tIdleConnTimeout:       90 * time.Second,\n\t\tTLSHandshakeTimeout:   10 * time.Second,\n\t}\n\treturn transport\n}\n\ntype Client struct {\n\tc *http.Client\n}\n\nfunc NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) *Client {\n\ttransport := NewDeadlineTransport(connectTimeout, requestTimeout)\n\ttransport.TLSClientConfig = tlsConfig\n\treturn &Client{\n\t\tc: &http.Client{\n\t\t\tTransport: transport,\n\t\t\tTimeout:   requestTimeout,\n\t\t},\n\t}\n}\n\n// GETV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) GETV1(endpoint string, v interface{}) error {\nretry:\n\treq, err := http.NewRequest(\"GET\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\terr = json.Unmarshal(body, &v)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// PostV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) POSTV1(endpoint string) error {\nretry:\n\treq, err := http.NewRequest(\"POST\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\n\treturn nil\n}\n\nfunc httpsEndpoint(endpoint string, body []byte) (string, error) {\n\tvar forbiddenResp struct {\n\t\tHTTPSPort int `json:\"https_port\"`\n\t}\n\terr := json.Unmarshal(body, &forbiddenResp)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thost, _, err := net.SplitHostPort(u.Host)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu.Scheme = \"https\"\n\tu.Host = net.JoinHostPort(host, strconv.Itoa(forbiddenResp.HTTPSPort))\n\treturn u.String(), nil\n}\n\n// Path: internal/http_api/api_response.go\npackage http_api\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/julienschmidt/httprouter\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Decorator func(APIHandler) APIHandler\n\ntype APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)\n\ntype Err struct {\n\tCode int\n\tText string\n}\n\nfunc (e Err) Error() string {\n\treturn e.Text\n}\n\nfunc PlainText(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tcode := 200\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tcode = err.(Err).Code\n\t\t\tdata = err.Error()\n\t\t}\n\t\tswitch d := data.(type) {\n\t\tcase string:\n\t\t\tw.WriteHeader(code)\n\t\t\tio.WriteString(w, d)\n\t\tcase []byte:\n\t\t\tw.WriteHeader(code)\n\t\t\tw.Write(d)\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"unknown response type %T\", data))\n\t\t}\n\t\treturn nil, nil\n\t}\n}\n\nfunc V1(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tRespondV1(w, err.(Err).Code, err)\n\t\t\treturn nil, nil\n\t\t}\n\t\tRespondV1(w, 200, data)\n\t\treturn nil, nil\n\t}\n}\n\nfunc RespondV1(w http.ResponseWriter, code int, data interface{}) {\n\tvar response []byte\n\tvar err error\n\tvar isJSON bool\n\n\tif code == 200 {\n\t\tswitch data := data.(type) {\n\t\tcase string:\n\t\t\tresponse = []byte(data)\n\t\tcase []byte:\n\t\t\tresponse = data\n\t\tcase nil:\n\t\t\tresponse = []byte{}\n\t\tdefault:\n\t\t\tisJSON = true\n\t\t\tresponse, err = json.Marshal(data)\n\t\t\tif err != nil {\n\t\t\t\tcode = 500\n\t\t\t\tdata = err\n\t\t\t}\n\t\t}\n\t}\n\n\tif code != 200 {\n\t\tisJSON = true\n\t\tresponse, _ = json.Marshal(struct {\n\t\t\tMessage string `json:\"message\"`\n\t\t}{fmt.Sprintf(\"%s\", data)})\n\t}\n\n\tif isJSON {\n\t\tw.Header().Set(\"Content-Type\", \"application/json; charset=utf-8\")\n\t}\n\tw.Header().Set(\"X-NSQ-Content-Type\", \"nsq; version=1.0\")\n\tw.WriteHeader(code)\n\tw.Write(response)\n}\n\nfunc Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {\n\tdecorated := f\n\tfor _, decorate := range ds {\n\t\tdecorated = decorate(decorated)\n\t}\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {\n\t\tdecorated(w, req, ps)\n\t}\n}\n\nfunc Log(logf lg.AppLogFunc) Decorator {\n\treturn func(f APIHandler) APIHandler {\n\t\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\tstart := time.Now()\n\t\t\tresponse, err := f(w, req, ps)\n\t\t\telapsed := time.Since(start)\n\t\t\tstatus := 200\n\t\t\tif e, ok := err.(Err); ok {\n\t\t\t\tstatus = e.Code\n\t\t\t}\n\t\t\tlogf(lg.INFO, \"%d %s %s (%s) %s\",\n\t\t\t\tstatus, req.Method, req.URL.RequestURI(), req.RemoteAddr, elapsed)\n\t\t\treturn response, err\n\t\t}\n\t}\n}\n\nfunc LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\treturn func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\t\tlogf(lg.ERROR, \"panic in HTTP handler - %s\", p)\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{500, \"INTERNAL_ERROR\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t}\n}\n\nfunc LogNotFoundHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{404, \"NOT_FOUND\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\nfunc LogMethodNotAllowedHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{405, \"METHOD_NOT_ALLOWED\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\n// Path: internal/http_api/compress.go\n// Copyright 2013 The Gorilla Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// copied from https://github.com/gorilla/handlers/blob/master/compress.go\n\npackage http_api\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n)\n\ntype compressResponseWriter struct {\n\tio.Writer\n\thttp.ResponseWriter\n\thttp.Hijacker\n}\n\nfunc (w *compressResponseWriter) Header() http.Header {\n\treturn w.ResponseWriter.Header()\n}\n\nfunc (w *compressResponseWriter) WriteHeader(c int) {\n\tw.ResponseWriter.Header().Del(\"Content-Length\")\n\tw.ResponseWriter.WriteHeader(c)\n}\n\nfunc (w *compressResponseWriter) Write(b []byte) (int, error) {\n\th := w.ResponseWriter.Header()\n\tif h.Get(\"Content-Type\") == \"\" {\n\t\th.Set(\"Content-Type\", http.DetectContentType(b))\n\t}\n\th.Del(\"Content-Length\")\n\treturn w.Writer.Write(b)\n}\n\n// CompressHandler gzip compresses HTTP responses for clients that support it\n// via the 'Accept-Encoding' header.\nfunc CompressHandler(h http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\tL:\n\t\tfor _, enc := range strings.Split(r.Header.Get(\"Accept-Encoding\"), \",\") {\n\t\t\tswitch strings.TrimSpace(enc) {\n\t\t\tcase \"gzip\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"gzip\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tgw := gzip.NewWriter(w)\n\t\t\t\tdefer gw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         gw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\tcase \"deflate\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"deflate\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tfw, _ := flate.NewWriter(w, flate.DefaultCompression)\n\t\t\t\tdefer fw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         fw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\t}\n\t\t}\n\n\t\th.ServeHTTP(w, r)\n\t})\n}\n\n// Path: internal/http_api/http_server.go\npackage http_api\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype logWriter struct {\n\tlogf lg.AppLogFunc\n}\n\nfunc (l logWriter) Write(p []byte) (int, error) {\n\tl.logf(lg.WARN, \"%s\", string(p))\n\treturn len(p), nil\n}\n\nfunc Serve(listener net.Listener, handler http.Handler, proto string, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"%s: listening on %s\", proto, listener.Addr())\n\n\tserver := &http.Server{\n\t\tHandler:  handler,\n\t\tErrorLog: log.New(logWriter{logf}, \"\", 0),\n\t}\n\terr := server.Serve(listener)\n\t// theres no direct way to detect this error because it is not exposed\n\tif err != nil && !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\treturn fmt.Errorf(\"http.Serve() error - %s\", err)\n\t}\n\n\tlogf(lg.INFO, \"%s: closing %s\", proto, listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/http_api/req_params.go\npackage http_api\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n)\n\ntype ReqParams struct {\n\turl.Values\n\tBody []byte\n}\n\nfunc NewReqParams(req *http.Request) (*ReqParams, error) {\n\treqParams, err := url.ParseQuery(req.URL.RawQuery)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, err := io.ReadAll(req.Body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &ReqParams{reqParams, data}, nil\n}\n\nfunc (r *ReqParams) Get(key string) (string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn \"\", errors.New(\"key not in query params\")\n\t}\n\treturn v[0], nil\n}\n\nfunc (r *ReqParams) GetAll(key string) ([]string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn nil, errors.New(\"key not in query params\")\n\t}\n\treturn v, nil\n}\n\n// Path: internal/version/binary.go\npackage version\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n)\n\nconst Binary = \"1.3.0\"\n\nfunc String(app string) string {\n\treturn fmt.Sprintf(\"%s v%s (built w/%s)\", app, Binary, runtime.Version())\n}\n\n// Path: internal/protocol/protocol.go\npackage protocol\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"net\"\n)\n\ntype Client interface {\n\tClose() error\n}\n\n// Protocol describes the basic behavior of any protocol in the system\ntype Protocol interface {\n\tNewClient(net.Conn) Client\n\tIOLoop(Client) error\n}\n\n// SendResponse is a server side utility function to prefix data with a length header\n// and write to the supplied Writer\n\nfunc SendResponse(w io.Writer, data []byte) (int, error) {\n\terr := binary.Write(w, binary.BigEndian, int32(len(data)))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tn, err := w.Write(data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn (n + 4), nil\n}\n\n// SendFramedResponse is a server side utility function to prefix data with a length header\n// and frame header and write to the supplied Writer\nfunc SendFramedResponse(w io.Writer, frameType int32, data []byte) (int, error) {\n\tbeBuf := make([]byte, 4)\n\tsize := uint32(len(data)) + 4\n\n\tbinary.BigEndian.PutUint32(beBuf, size)\n\tn, err := w.Write(beBuf)\n\tif err != nil {\n\t\treturn n, err\n\t}\n\n\tbinary.BigEndian.PutUint32(beBuf, uint32(frameType))\n\tn, err = w.Write(beBuf)\n\tif err != nil {\n\t\treturn n + 4, err\n\t}\n\n\tn, err = w.Write(data)\n\treturn n + 8, err\n}\n\n// Path: internal/http_api/topic_channel_args.go\npackage http_api\n\nimport (\n\t\"errors\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\ntype getter interface {\n\tGet(key string) (string, error)\n}\n\nfunc GetTopicChannelArgs(rp getter) (string, string, error) {\n\ttopicName, err := rp.Get(\"topic\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_TOPIC\")\n\t}\n\n\tif !protocol.IsValidTopicName(topicName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_TOPIC\")\n\t}\n\n\tchannelName, err := rp.Get(\"channel\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_CHANNEL\")\n\t}\n\n\tif !protocol.IsValidChannelName(channelName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_CHANNEL\")\n\t}\n\n\treturn topicName, channelName, nil\n}\n\n// Path: internal/lg/lg_test.go\npackage lg\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\ntype mockLogger struct {\n\tCount int\n}\n\nfunc (l *mockLogger) Output(maxdepth int, s string) error {\n\tl.Count++\n\treturn nil\n}\n\nfunc TestLogging(t *testing.T) {\n\tlogger := &mockLogger{}\n\n\t// Test only fatal get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, FATAL, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 1, logger.Count)\n\n\t// Test only warnings or higher get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, WARN, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 3, logger.Count)\n\n\t// Test everything gets through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, DEBUG, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 5, logger.Count)\n}\n\n// Path: internal/pqueue/pqueue.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n)\n\ntype Item struct {\n\tValue    interface{}\n\tPriority int64\n\tIndex    int\n}\n\n// this is a priority queue as implemented by a min heap\n// ie. the 0th element is the *lowest* value\ntype PriorityQueue []*Item\n\nfunc New(capacity int) PriorityQueue {\n\treturn make(PriorityQueue, 0, capacity)\n}\n\nfunc (pq PriorityQueue) Len() int {\n\treturn len(pq)\n}\n\nfunc (pq PriorityQueue) Less(i, j int) bool {\n\treturn pq[i].Priority < pq[j].Priority\n}\n\nfunc (pq PriorityQueue) Swap(i, j int) {\n\tpq[i], pq[j] = pq[j], pq[i]\n\tpq[i].Index = i\n\tpq[j].Index = j\n}\n\nfunc (pq *PriorityQueue) Push(x interface{}) {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n+1 > c {\n\t\tnpq := make(PriorityQueue, n, c*2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\t*pq = (*pq)[0 : n+1]\n\titem := x.(*Item)\n\titem.Index = n\n\t(*pq)[n] = item\n}\n\nfunc (pq *PriorityQueue) Pop() interface{} {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n < (c/2) && c > 25 {\n\t\tnpq := make(PriorityQueue, n, c/2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\titem := (*pq)[n-1]\n\titem.Index = -1\n\t*pq = (*pq)[0 : n-1]\n\treturn item\n}\n\nfunc (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) {\n\tif pq.Len() == 0 {\n\t\treturn nil, 0\n\t}\n\n\titem := (*pq)[0]\n\tif item.Priority > max {\n\t\treturn nil, item.Priority - max\n\t}\n\theap.Remove(pq, 0)\n\n\treturn item, 0\n}\n\n// Path: internal/pqueue/pqueue_test.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n\t\"math/rand\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"testing\"\n)\n\nfunc equal(t *testing.T, act, exp interface{}) {\n\tif !reflect.DeepEqual(exp, act) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\texp: %#v\\n\\n\\tgot: %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, exp, act)\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestPriorityQueue(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c+1; i++ {\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(i)})\n\t}\n\tequal(t, pq.Len(), c+1)\n\tequal(t, cap(pq), c*2)\n\n\tfor i := 0; i < c+1; i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, item.(*Item).Value.(int), i)\n\t}\n\tequal(t, cap(pq), c/4)\n}\n\nfunc TestUnsortedInsert(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\tints := make([]int, 0, c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\tints = append(ints, v)\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(v)})\n\t}\n\tequal(t, pq.Len(), c)\n\tequal(t, cap(pq), c)\n\n\tsort.Ints(ints)\n\n\tfor i := 0; i < c; i++ {\n\t\titem, _ := pq.PeekAndShift(int64(ints[len(ints)-1]))\n\t\tequal(t, item.Priority, int64(ints[i]))\n\t}\n}\n\nfunc TestRemove(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\theap.Push(&pq, &Item{Value: \"test\", Priority: int64(v)})\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\theap.Remove(&pq, rand.Intn((c-1)-i))\n\t}\n\n\tlastPriority := heap.Pop(&pq).(*Item).Priority\n\tfor i := 0; i < (c - 10 - 1); i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, lastPriority < item.(*Item).Priority, true)\n\t\tlastPriority = item.(*Item).Priority\n\t}\n}\n\n// Path: internal/protocol/byte_base10.go\npackage protocol\n\nimport (\n\t\"errors\"\n)\n\nvar errBase10 = errors.New(\"failed to convert to Base10\")\n\nfunc ByteToBase10(b []byte) (n uint64, err error) {\n\tbase := uint64(10)\n\n\tn = 0\n\tfor i := 0; i < len(b); i++ {\n\t\tvar v byte\n\t\td := b[i]\n\t\tswitch {\n\t\tcase '0' <= d && d <= '9':\n\t\t\tv = d - '0'\n\t\tdefault:\n\t\t\tn = 0\n\t\t\terr = errBase10\n\t\t\treturn\n\t\t}\n\t\tn *= base\n\t\tn += uint64(v)\n\t}\n\n\treturn n, err\n}\n\n// Path: internal/protocol/byte_base10_test.go\npackage protocol\n\nimport (\n\t\"testing\"\n)\n\nvar result uint64\n\nfunc BenchmarkByteToBase10Valid(b *testing.B) {\n\tbt := []byte{'3', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\nfunc BenchmarkByteToBase10Invalid(b *testing.B) {\n\tbt := []byte{'?', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\n// Path: internal/protocol/names.go\npackage protocol\n\nimport (\n\t\"regexp\"\n)\n\nvar validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral)?$`)\n\n// IsValidTopicName checks a topic name for correctness\nfunc IsValidTopicName(name string) bool {\n\treturn isValidName(name)\n}\n\n// IsValidChannelName checks a channel name for correctness\nfunc IsValidChannelName(name string) bool {\n\treturn isValidName(name)\n}\n\nfunc isValidName(name string) bool {\n\tif len(name) > 64 || len(name) < 1 {\n\t\treturn false\n\t}\n\treturn validTopicChannelNameRegex.MatchString(name)\n}\n\n// Path: internal/protocol/tcp_server.go\npackage protocol\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype TCPHandler interface {\n\tHandle(net.Conn)\n}\n\nfunc TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"TCP: listening on %s\", listener.Addr())\n\n\tvar wg sync.WaitGroup\n\n\tfor {\n\t\tclientConn, err := listener.Accept()\n\t\tif err != nil {\n\t\t\t// net.Error.Temporary() is deprecated, but is valid for accept\n\t\t\t// this is a hack to avoid a staticcheck error\n\t\t\tif te, ok := err.(interface{ Temporary() bool }); ok && te.Temporary() {\n\t\t\t\tlogf(lg.WARN, \"temporary Accept() failure - %s\", err)\n\t\t\t\truntime.Gosched()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// theres no direct way to detect this error because it is not exposed\n\t\t\tif !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\t\t\treturn fmt.Errorf(\"listener.Accept() error - %s\", err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\thandler.Handle(clientConn)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\t// wait to return until all handler goroutines complete\n\twg.Wait()\n\n\tlogf(lg.INFO, \"TCP: closing %s\", listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/quantile/aggregate.go\npackage quantile\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n)\n\ntype E2eProcessingLatencyAggregate struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\tTopic       string               `json:\"topic\"`\n\tChannel     string               `json:\"channel\"`\n\tAddr        string               `json:\"host\"`\n}\n\nfunc (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {\n\tvar resp struct {\n\t\tCount       int                  `json:\"count\"`\n\t\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\t\tTopic       string               `json:\"topic\"`\n\t\tChannel     string               `json:\"channel\"`\n\t\tAddr        string               `json:\"host\"`\n\t}\n\terr := json.Unmarshal(b, &resp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, p := range resp.Percentiles {\n\t\tp[\"min\"] = p[\"value\"]\n\t\tp[\"max\"] = p[\"value\"]\n\t\tp[\"average\"] = p[\"value\"]\n\t\tp[\"count\"] = float64(resp.Count)\n\t}\n\n\te.Count = resp.Count\n\te.Percentiles = resp.Percentiles\n\te.Topic = resp.Topic\n\te.Channel = resp.Channel\n\te.Addr = resp.Addr\n\n\treturn nil\n}\n\nfunc (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) }\nfunc (e *E2eProcessingLatencyAggregate) Swap(i, j int) {\n\te.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i]\n}\nfunc (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {\n\treturn e.Percentiles[i][\"percentile\"] > e.Percentiles[j][\"percentile\"]\n}\n\n// Add merges e2 into e by averaging the percentiles\nfunc (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) {\n\te.Addr = \"*\"\n\tp := e.Percentiles\n\te.Count += e2.Count\n\tfor _, value := range e2.Percentiles {\n\t\ti := -1\n\t\tfor j, v := range p {\n\t\t\tif value[\"quantile\"] == v[\"quantile\"] {\n\t\t\t\ti = j\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif i == -1 {\n\t\t\ti = len(p)\n\t\t\te.Percentiles = append(p, make(map[string]float64))\n\t\t\tp = e.Percentiles\n\t\t\tp[i][\"quantile\"] = value[\"quantile\"]\n\t\t}\n\t\tp[i][\"max\"] = math.Max(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"min\"] = math.Min(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"count\"] += value[\"count\"]\n\t\tif p[i][\"count\"] == 0 {\n\t\t\tp[i][\"average\"] = 0\n\t\t\tcontinue\n\t\t}\n\t\tdelta := value[\"average\"] - p[i][\"average\"]\n\t\tR := delta * value[\"count\"] / p[i][\"count\"]\n\t\tp[i][\"average\"] = p[i][\"average\"] + R\n\t}\n\tsort.Sort(e)\n}\n\n// Path: internal/statsd/host.go\npackage statsd\n\nimport (\n\t\"strings\"\n)\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \"_\", -1), \":\", \"_\", -1)\n}\n\n// Path: internal/stringy/slice.go\npackage stringy\n\nfunc Add(s []string, a string) []string {\n\tfor _, existing := range s {\n\t\tif a == existing {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn append(s, a)\n\n}\n\nfunc Union(s []string, a []string) []string {\n\tfor _, entry := range a {\n\t\tfound := false\n\t\tfor _, existing := range s {\n\t\t\tif entry == existing {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\ts = append(s, entry)\n\t\t}\n\t}\n\treturn s\n}\n\nfunc Uniq(s []string) (r []string) {\nouterLoop:\n\tfor _, entry := range s {\n\t\tfor _, existing := range r {\n\t\t\tif existing == entry {\n\t\t\t\tcontinue outerLoop\n\t\t\t}\n\t\t}\n\t\tr = append(r, entry)\n\t}\n\treturn\n}\n\n// Path: internal/stringy/slice_test.go\npackage stringy_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\nfunc BenchmarkUniq(b *testing.B) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"b\"}\n\tfor i := 0; i < b.N; i++ {\n\t\tvalues = stringy.Uniq(values)\n\t\tif len(values) != 2 {\n\t\t\tb.Fatal(\"values len is incorrect\")\n\t\t}\n\t}\n}\n\nfunc TestUniq(t *testing.T) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"b\", \"b\", \"b\", \"c\", \"c\", \"c\"}\n\tvalues = stringy.Uniq(values)\n\tif len(values) != 3 {\n\t\tt.Fatal(\"values len is incorrect\")\n\t}\n}\n\n// Path: internal/stringy/template.go\npackage stringy\n\nimport (\n\t\"fmt\"\n)\n\nfunc NanoSecondToHuman(v float64) string {\n\tvar suffix string\n\tswitch {\n\tcase v > 1000000000:\n\t\tv /= 1000000000\n\t\tsuffix = \"s\"\n\tcase v > 1000000:\n\t\tv /= 1000000\n\t\tsuffix = \"ms\"\n\tcase v > 1000:\n\t\tv /= 1000\n\t\tsuffix = \"us\"\n\tdefault:\n\t\tsuffix = \"ns\"\n\t}\n\treturn fmt.Sprintf(\"%0.1f%s\", v, suffix)\n}\n\n// Path: internal/test/assertions.go\npackage test\n\nimport (\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc Equal(t *testing.T, expected, actual interface{}) {\n\tif !reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   %#v (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotEqual(t *testing.T, expected, actual interface{}) {\n\tif reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tnexp: %#v\\n\\n\\tgot:  %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc Nil(t *testing.T, object interface{}) {\n\tif !isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   <nil> (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, object)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotNil(t *testing.T, object interface{}) {\n\tif isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tExpected value not to be <nil>\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line)\n\t\tt.FailNow()\n\t}\n}\n\nfunc isNil(object interface{}) bool {\n\tif object == nil {\n\t\treturn true\n\t}\n\n\tvalue := reflect.ValueOf(object)\n\tkind := value.Kind()\n\tif kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// Path: internal/test/fakes.go\npackage test\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\ntype FakeNetConn struct {\n\tReadFunc             func([]byte) (int, error)\n\tWriteFunc            func([]byte) (int, error)\n\tCloseFunc            func() error\n\tLocalAddrFunc        func() net.Addr\n\tRemoteAddrFunc       func() net.Addr\n\tSetDeadlineFunc      func(time.Time) error\n\tSetReadDeadlineFunc  func(time.Time) error\n\tSetWriteDeadlineFunc func(time.Time) error\n}\n\nfunc (f FakeNetConn) Read(b []byte) (int, error)         { return f.ReadFunc(b) }\nfunc (f FakeNetConn) Write(b []byte) (int, error)        { return f.WriteFunc(b) }\nfunc (f FakeNetConn) Close() error                       { return f.CloseFunc() }\nfunc (f FakeNetConn) LocalAddr() net.Addr                { return f.LocalAddrFunc() }\nfunc (f FakeNetConn) RemoteAddr() net.Addr               { return f.RemoteAddrFunc() }\nfunc (f FakeNetConn) SetDeadline(t time.Time) error      { return f.SetDeadlineFunc(t) }\nfunc (f FakeNetConn) SetReadDeadline(t time.Time) error  { return f.SetReadDeadlineFunc(t) }\nfunc (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) }\n\ntype fakeNetAddr struct{}\n\nfunc (fakeNetAddr) Network() string { return \"\" }\nfunc (fakeNetAddr) String() string  { return \"\" }\n\nfunc NewFakeNetConn() FakeNetConn {\n\tnetAddr := fakeNetAddr{}\n\treturn FakeNetConn{\n\t\tReadFunc:             func(b []byte) (int, error) { return 0, nil },\n\t\tWriteFunc:            func(b []byte) (int, error) { return len(b), nil },\n\t\tCloseFunc:            func() error { return nil },\n\t\tLocalAddrFunc:        func() net.Addr { return netAddr },\n\t\tRemoteAddrFunc:       func() net.Addr { return netAddr },\n\t\tSetDeadlineFunc:      func(time.Time) error { return nil },\n\t\tSetWriteDeadlineFunc: func(time.Time) error { return nil },\n\t\tSetReadDeadlineFunc:  func(time.Time) error { return nil },\n\t}\n}\n\n// Path: internal/util/unix_socket.go\npackage util\n\nimport (\n\t\"net\"\n)\n\nfunc TypeOfAddr(addr string) string {\n\tif _, _, err := net.SplitHostPort(addr); err == nil {\n\t\treturn \"tcp\"\n\t}\n\treturn \"unix\"\n}\n\n// Path: internal/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc BenchmarkUniqRands5of5(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(5, 5)\n\t}\n}\nfunc BenchmarkUniqRands20of20(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 20)\n\t}\n}\n\nfunc BenchmarkUniqRands20of50(b *testing.B) {\n...\n", "needle_token_start": 8263, "needle_token_end": 8368, "code_context_ntokens": 15131, "output": ["```go\n// SendResponse is a server side utility function to prefix data with a length header\n// and write to the supplied Writer\nfunc SendResponse(w io.Writer, data []byte) (int, error) {\n\terr := binary.Write(w, binary.BigEndian, int32(len(data)))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tn, err := w.Write(data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn (n + 4), nil\n}\n```"]}
{"repo": "nsqio/nsq", "name": "GetTopicChannelArgs", "language": "go", "path": "internal/http_api/topic_channel_args.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To validate and retrieve the names of a topic and a channel from the provided parameters, ensuring both are present and adhere to naming conventions.\n2. **Input**: An interface that allows retrieval of query parameters.\n3. **Output**: Returns the names of the topic and channel if both are valid and present, otherwise returns appropriate error messages.\n4. **Procedure**: \n   - Attempts to retrieve the 'topic' name from the input parameters.\n   - Checks if the 'topic' name is valid and present, returning an error if not.\n   - Attempts to retrieve the 'channel' name from the input parameters.\n   - Checks if the 'channel' name is valid and present, returning an error if not.\n   - If both names are valid and present, returns them; otherwise, returns the corresponding error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/quantile/quantile.go\npackage quantile\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/bmizerany/perks/quantile\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype Result struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n}\n\nfunc (r *Result) String() string {\n\tvar s []string\n\tfor _, item := range r.Percentiles {\n\t\ts = append(s, stringy.NanoSecondToHuman(item[\"value\"]))\n\t}\n\treturn strings.Join(s, \", \")\n}\n\ntype Quantile struct {\n\tsync.Mutex\n\tstreams        [2]quantile.Stream\n\tcurrentIndex   uint8\n\tlastMoveWindow time.Time\n\tcurrentStream  *quantile.Stream\n\n\tPercentiles    []float64\n\tMoveWindowTime time.Duration\n}\n\nfunc New(WindowTime time.Duration, Percentiles []float64) *Quantile {\n\tq := Quantile{\n\t\tcurrentIndex:   0,\n\t\tlastMoveWindow: time.Now(),\n\t\tMoveWindowTime: WindowTime / 2,\n\t\tPercentiles:    Percentiles,\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tq.streams[i] = *quantile.NewTargeted(Percentiles...)\n\t}\n\tq.currentStream = &q.streams[0]\n\treturn &q\n}\n\nfunc (q *Quantile) Result() *Result {\n\tif q == nil {\n\t\treturn &Result{}\n\t}\n\tqueryHandler := q.QueryHandler()\n\tresult := Result{\n\t\tCount:       queryHandler.Count(),\n\t\tPercentiles: make([]map[string]float64, len(q.Percentiles)),\n\t}\n\tfor i, p := range q.Percentiles {\n\t\tvalue := queryHandler.Query(p)\n\t\tresult.Percentiles[i] = map[string]float64{\"quantile\": p, \"value\": value}\n\t}\n\treturn &result\n}\n\nfunc (q *Quantile) Insert(msgStartTime int64) {\n\tq.Lock()\n\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tq.currentStream.Insert(float64(now.UnixNano() - msgStartTime))\n\tq.Unlock()\n}\n\nfunc (q *Quantile) QueryHandler() *quantile.Stream {\n\tq.Lock()\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tmerged := quantile.NewTargeted(q.Percentiles...)\n\tmerged.Merge(q.streams[0].Samples())\n\tmerged.Merge(q.streams[1].Samples())\n\tq.Unlock()\n\treturn merged\n}\n\nfunc (q *Quantile) IsDataStale(now time.Time) bool {\n\treturn now.After(q.lastMoveWindow.Add(q.MoveWindowTime))\n}\n\nfunc (q *Quantile) Merge(them *Quantile) {\n\tq.Lock()\n\tthem.Lock()\n\tiUs := q.currentIndex\n\tiThem := them.currentIndex\n\n...\n// Path: internal/clusterinfo/types.go\npackage clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype ProducerTopic struct {\n\tTopic      string `json:\"topic\"`\n\tTombstoned bool   `json:\"tombstoned\"`\n}\n\ntype ProducerTopics []ProducerTopic\n\nfunc (pt ProducerTopics) Len() int           { return len(pt) }\nfunc (pt ProducerTopics) Swap(i, j int)      { pt[i], pt[j] = pt[j], pt[i] }\nfunc (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt[j].Topic }\n\ntype Producer struct {\n\tRemoteAddresses  []string       `json:\"remote_addresses\"`\n\tRemoteAddress    string         `json:\"remote_address\"`\n\tHostname         string         `json:\"hostname\"`\n\tBroadcastAddress string         `json:\"broadcast_address\"`\n\tTCPPort          int            `json:\"tcp_port\"`\n\tHTTPPort         int            `json:\"http_port\"`\n\tVersion          string         `json:\"version\"`\n\tVersionObj       semver.Version `json:\"-\"`\n\tTopics           ProducerTopics `json:\"topics\"`\n\tOutOfDate        bool           `json:\"out_of_date\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses of ProducerTopics and VersionObj\nfunc (p *Producer) UnmarshalJSON(b []byte) error {\n\tvar r struct {\n\t\tRemoteAddress    string   `json:\"remote_address\"`\n\t\tHostname         string   `json:\"hostname\"`\n\t\tBroadcastAddress string   `json:\"broadcast_address\"`\n\t\tTCPPort          int      `json:\"tcp_port\"`\n\t\tHTTPPort         int      `json:\"http_port\"`\n\t\tVersion          string   `json:\"version\"`\n\t\tTopics           []string `json:\"topics\"`\n\t\tTombstoned       []bool   `json:\"tombstones\"`\n\t}\n\tif err := json.Unmarshal(b, &r); err != nil {\n\t\treturn err\n\t}\n\t*p = Producer{\n\t\tRemoteAddress:    r.RemoteAddress,\n\t\tHostname:         r.Hostname,\n\t\tBroadcastAddress: r.BroadcastAddress,\n\t\tTCPPort:          r.TCPPort,\n\t\tHTTPPort:         r.HTTPPort,\n\t\tVersion:          r.Version,\n\t}\n\tfor i, t := range r.Topics {\n\t\tp.Topics = append(p.Topics, ProducerTopic{Topic: t, Tombstoned: r.Tombstoned[i]})\n\t}\n\tversion, err := semver.Parse(p.Version)\n\tif err != nil {\n\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t}\n\tp.VersionObj = version\n\treturn nil\n}\n\nfunc (p *Producer) Address() string {\n\tif p.RemoteAddress == \"\" {\n\t\treturn \"N/A\"\n\t}\n\treturn p.RemoteAddress\n}\n\nfunc (p *Producer) HTTPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.HTTPPort))\n}\n\nfunc (p *Producer) TCPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.TCPPort))\n}\n\n// IsInconsistent checks for cases where an unexpected number of nsqd connections are\n// reporting the same information to nsqlookupd (ie: multiple instances are using the\n// same broadcast address), or cases where some nsqd are not reporting to all nsqlookupd.\nfunc (p *Producer) IsInconsistent(numLookupd int) bool {\n\treturn len(p.RemoteAddresses) != numLookupd\n}\n\ntype TopicStats struct {\n\tNode         string          `json:\"node\"`\n\tHostname     string          `json:\"hostname\"`\n\tTopicName    string          `json:\"topic_name\"`\n\tDepth        int64           `json:\"depth\"`\n\tMemoryDepth  int64           `json:\"memory_depth\"`\n\tBackendDepth int64           `json:\"backend_depth\"`\n\tMessageCount int64           `json:\"message_count\"`\n\tNodeStats    []*TopicStats   `json:\"nodes\"`\n\tChannels     []*ChannelStats `json:\"channels\"`\n\tPaused       bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (t *TopicStats) Add(a *TopicStats) {\n\tt.Node = \"*\"\n\tt.Depth += a.Depth\n\tt.MemoryDepth += a.MemoryDepth\n\tt.BackendDepth += a.BackendDepth\n\tt.MessageCount += a.MessageCount\n\tif a.Paused {\n\t\tt.Paused = a.Paused\n\t}\n\tfor _, aChannelStats := range a.Channels {\n\t\tfound := false\n\t\tfor _, channelStats := range t.Channels {\n\t\t\tif aChannelStats.ChannelName == channelStats.ChannelName {\n\t\t\t\tfound = true\n\t\t\t\tchannelStats.Add(aChannelStats)\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\tt.Channels = append(t.Channels, aChannelStats)\n\t\t}\n\t}\n\tt.NodeStats = append(t.NodeStats, a)\n\tsort.Sort(TopicStatsByHost{t.NodeStats})\n\tif t.E2eProcessingLatency == nil {\n\t\tt.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:  t.Node,\n\t\t\tTopic: t.TopicName,\n\t\t}\n\t}\n\tt.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n}\n\ntype ChannelStats struct {\n\tNode          string          `json:\"node\"`\n\tHostname      string          `json:\"hostname\"`\n\tTopicName     string          `json:\"topic_name\"`\n\tChannelName   string          `json:\"channel_name\"`\n\tDepth         int64           `json:\"depth\"`\n\tMemoryDepth   int64           `json:\"memory_depth\"`\n\tBackendDepth  int64           `json:\"backend_depth\"`\n\tInFlightCount int64           `json:\"in_flight_count\"`\n\tDeferredCount int64           `json:\"deferred_count\"`\n\tRequeueCount  int64           `json:\"requeue_count\"`\n\tTimeoutCount  int64           `json:\"timeout_count\"`\n\tMessageCount  int64           `json:\"message_count\"`\n\tClientCount   int             `json:\"client_count\"`\n\tSelected      bool            `json:\"-\"`\n\tNodeStats     []*ChannelStats `json:\"nodes\"`\n\tClients       []*ClientStats  `json:\"clients\"`\n\tPaused        bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (c *ChannelStats) Add(a *ChannelStats) {\n\tc.Node = \"*\"\n\tc.Depth += a.Depth\n\tc.MemoryDepth += a.MemoryDepth\n\tc.BackendDepth += a.BackendDepth\n\tc.InFlightCount += a.InFlightCount\n\tc.DeferredCount += a.DeferredCount\n\tc.RequeueCount += a.RequeueCount\n\tc.TimeoutCount += a.TimeoutCount\n\tc.MessageCount += a.MessageCount\n\tc.ClientCount += a.ClientCount\n\tif a.Paused {\n\t\tc.Paused = a.Paused\n\t}\n\tc.NodeStats = append(c.NodeStats, a)\n\tsort.Sort(ChannelStatsByHost{c.NodeStats})\n\tif c.E2eProcessingLatency == nil {\n\t\tc.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:    c.Node,\n\t\t\tTopic:   c.TopicName,\n\t\t\tChannel: c.ChannelName,\n\t\t}\n\t}\n\tc.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n\tc.Clients = append(c.Clients, a.Clients...)\n\tsort.Sort(ClientsByHost{c.Clients})\n}\n\ntype ClientStats struct {\n\tNode              string        `json:\"node\"`\n\tRemoteAddress     string        `json:\"remote_address\"`\n\tVersion           string        `json:\"version\"`\n\tClientID          string        `json:\"client_id\"`\n\tHostname          string        `json:\"hostname\"`\n\tUserAgent         string        `json:\"user_agent\"`\n\tConnectTs         int64         `json:\"connect_ts\"`\n\tConnectedDuration time.Duration `json:\"connected\"`\n\tInFlightCount     int           `json:\"in_flight_count\"`\n\tReadyCount        int           `json:\"ready_count\"`\n\tFinishCount       int64         `json:\"finish_count\"`\n\tRequeueCount      int64         `json:\"requeue_count\"`\n\tMessageCount      int64         `json:\"message_count\"`\n\tSampleRate        int32         `json:\"sample_rate\"`\n\tDeflate           bool          `json:\"deflate\"`\n\tSnappy            bool          `json:\"snappy\"`\n\tAuthed            bool          `json:\"authed\"`\n\tAuthIdentity      string        `json:\"auth_identity\"`\n\tAuthIdentityURL   string        `json:\"auth_identity_url\"`\n\n\tTLS                           bool   `json:\"tls\"`\n\tCipherSuite                   string `json:\"tls_cipher_suite\"`\n\tTLSVersion                    string `json:\"tls_version\"`\n\tTLSNegotiatedProtocol         string `json:\"tls_negotiated_protocol\"`\n\tTLSNegotiatedProtocolIsMutual bool   `json:\"tls_negotiated_protocol_is_mutual\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses ConnectedDuration\nfunc (s *ClientStats) UnmarshalJSON(b []byte) error {\n\ttype locaClientStats ClientStats // re-typed to prevent recursion from json.Unmarshal\n\tvar ss locaClientStats\n\tif err := json.Unmarshal(b, &ss); err != nil {\n\t\treturn err\n\t}\n\t*s = ClientStats(ss)\n\ts.ConnectedDuration = time.Now().Truncate(time.Second).Sub(time.Unix(s.ConnectTs, 0))\n\treturn nil\n}\n\nfunc (s *ClientStats) HasUserAgent() bool {\n\treturn s.UserAgent != \"\"\n}\n\nfunc (s *ClientStats) HasSampleRate() bool {\n\treturn s.SampleRate > 0\n}\n\ntype ChannelStatsList []*ChannelStats\n\nfunc (c ChannelStatsList) Len() int      { return len(c) }\nfunc (c ChannelStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ChannelStatsByHost struct {\n\tChannelStatsList\n}\n\nfunc (c ChannelStatsByHost) Less(i, j int) bool {\n\treturn c.ChannelStatsList[i].Hostname < c.ChannelStatsList[j].Hostname\n}\n\ntype ClientStatsList []*ClientStats\n\nfunc (c ClientStatsList) Len() int      { return len(c) }\nfunc (c ClientStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ClientsByHost struct {\n\tClientStatsList\n}\n\nfunc (c ClientsByHost) Less(i, j int) bool {\n\treturn c.ClientStatsList[i].Hostname < c.ClientStatsList[j].Hostname\n}\n\ntype TopicStatsList []*TopicStats\n\nfunc (t TopicStatsList) Len() int      { return len(t) }\nfunc (t TopicStatsList) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\ntype TopicStatsByHost struct {\n\tTopicStatsList\n}\n\nfunc (c TopicStatsByHost) Less(i, j int) bool {\n\treturn c.TopicStatsList[i].Hostname < c.TopicStatsList[j].Hostname\n}\n\ntype Producers []*Producer\n\nfunc (t Producers) Len() int      { return len(t) }\nfunc (t Producers) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\nfunc (t Producers) HTTPAddrs() []string {\n\tvar addrs []string\n\tfor _, p := range t {\n\t\taddrs = append(addrs, p.HTTPAddress())\n\t}\n\treturn addrs\n}\n\nfunc (t Producers) Search(needle string) *Producer {\n\tfor _, producer := range t {\n\t\tif needle == producer.HTTPAddress() {\n\t\t\treturn producer\n\t\t}\n\t}\n\treturn nil\n}\n\ntype ProducersByHost struct {\n\tProducers\n}\n\nfunc (c ProducersByHost) Less(i, j int) bool {\n\treturn c.Producers[i].Hostname < c.Producers[j].Hostname\n}\n\n// Path: internal/dirlock/dirlock.go\n//go:build !windows && !illumos\n// +build !windows,!illumos\n\npackage dirlock\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n)\n\ntype DirLock struct {\n\tdir string\n\tf   *os.File\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\tf, err := os.Open(l.dir)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl.f = f\n\terr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"cannot flock directory %s - %s (possibly in use by another instance of nsqd)\", l.dir, err)\n\t}\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\tdefer l.f.Close()\n\treturn syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)\n}\n\n// Path: internal/dirlock/dirlock_illumos.go\n//go:build illumos\n// +build illumos\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/dirlock/dirlock_windows.go\n//go:build windows\n// +build windows\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/http_api/api_request.go\npackage http_api\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// A custom http.Transport with support for deadline timeouts\nfunc NewDeadlineTransport(connectTimeout time.Duration, requestTimeout time.Duration) *http.Transport {\n\t// arbitrary values copied from http.DefaultTransport\n\ttransport := &http.Transport{\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout:   connectTimeout,\n\t\t\tKeepAlive: 30 * time.Second,\n\t\t\tDualStack: true,\n\t\t}).DialContext,\n\t\tResponseHeaderTimeout: requestTimeout,\n\t\tMaxIdleConns:          100,\n\t\tIdleConnTimeout:       90 * time.Second,\n\t\tTLSHandshakeTimeout:   10 * time.Second,\n\t}\n\treturn transport\n}\n\ntype Client struct {\n\tc *http.Client\n}\n\nfunc NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) *Client {\n\ttransport := NewDeadlineTransport(connectTimeout, requestTimeout)\n\ttransport.TLSClientConfig = tlsConfig\n\treturn &Client{\n\t\tc: &http.Client{\n\t\t\tTransport: transport,\n\t\t\tTimeout:   requestTimeout,\n\t\t},\n\t}\n}\n\n// GETV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) GETV1(endpoint string, v interface{}) error {\nretry:\n\treq, err := http.NewRequest(\"GET\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\terr = json.Unmarshal(body, &v)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// PostV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) POSTV1(endpoint string) error {\nretry:\n\treq, err := http.NewRequest(\"POST\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\n\treturn nil\n}\n\nfunc httpsEndpoint(endpoint string, body []byte) (string, error) {\n\tvar forbiddenResp struct {\n\t\tHTTPSPort int `json:\"https_port\"`\n\t}\n\terr := json.Unmarshal(body, &forbiddenResp)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thost, _, err := net.SplitHostPort(u.Host)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu.Scheme = \"https\"\n\tu.Host = net.JoinHostPort(host, strconv.Itoa(forbiddenResp.HTTPSPort))\n\treturn u.String(), nil\n}\n\n// Path: internal/http_api/api_response.go\npackage http_api\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/julienschmidt/httprouter\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Decorator func(APIHandler) APIHandler\n\ntype APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)\n\ntype Err struct {\n\tCode int\n\tText string\n}\n\nfunc (e Err) Error() string {\n\treturn e.Text\n}\n\nfunc PlainText(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tcode := 200\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tcode = err.(Err).Code\n\t\t\tdata = err.Error()\n\t\t}\n\t\tswitch d := data.(type) {\n\t\tcase string:\n\t\t\tw.WriteHeader(code)\n\t\t\tio.WriteString(w, d)\n\t\tcase []byte:\n\t\t\tw.WriteHeader(code)\n\t\t\tw.Write(d)\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"unknown response type %T\", data))\n\t\t}\n\t\treturn nil, nil\n\t}\n}\n\nfunc V1(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tRespondV1(w, err.(Err).Code, err)\n\t\t\treturn nil, nil\n\t\t}\n\t\tRespondV1(w, 200, data)\n\t\treturn nil, nil\n\t}\n}\n\nfunc RespondV1(w http.ResponseWriter, code int, data interface{}) {\n\tvar response []byte\n\tvar err error\n\tvar isJSON bool\n\n\tif code == 200 {\n\t\tswitch data := data.(type) {\n\t\tcase string:\n\t\t\tresponse = []byte(data)\n\t\tcase []byte:\n\t\t\tresponse = data\n\t\tcase nil:\n\t\t\tresponse = []byte{}\n\t\tdefault:\n\t\t\tisJSON = true\n\t\t\tresponse, err = json.Marshal(data)\n\t\t\tif err != nil {\n\t\t\t\tcode = 500\n\t\t\t\tdata = err\n\t\t\t}\n\t\t}\n\t}\n\n\tif code != 200 {\n\t\tisJSON = true\n\t\tresponse, _ = json.Marshal(struct {\n\t\t\tMessage string `json:\"message\"`\n\t\t}{fmt.Sprintf(\"%s\", data)})\n\t}\n\n\tif isJSON {\n\t\tw.Header().Set(\"Content-Type\", \"application/json; charset=utf-8\")\n\t}\n\tw.Header().Set(\"X-NSQ-Content-Type\", \"nsq; version=1.0\")\n\tw.WriteHeader(code)\n\tw.Write(response)\n}\n\nfunc Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {\n\tdecorated := f\n\tfor _, decorate := range ds {\n\t\tdecorated = decorate(decorated)\n\t}\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {\n\t\tdecorated(w, req, ps)\n\t}\n}\n\nfunc Log(logf lg.AppLogFunc) Decorator {\n\treturn func(f APIHandler) APIHandler {\n\t\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\tstart := time.Now()\n\t\t\tresponse, err := f(w, req, ps)\n\t\t\telapsed := time.Since(start)\n\t\t\tstatus := 200\n\t\t\tif e, ok := err.(Err); ok {\n\t\t\t\tstatus = e.Code\n\t\t\t}\n\t\t\tlogf(lg.INFO, \"%d %s %s (%s) %s\",\n\t\t\t\tstatus, req.Method, req.URL.RequestURI(), req.RemoteAddr, elapsed)\n\t\t\treturn response, err\n\t\t}\n\t}\n}\n\nfunc LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\treturn func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\t\tlogf(lg.ERROR, \"panic in HTTP handler - %s\", p)\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{500, \"INTERNAL_ERROR\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t}\n}\n\nfunc LogNotFoundHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{404, \"NOT_FOUND\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\nfunc LogMethodNotAllowedHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{405, \"METHOD_NOT_ALLOWED\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\n// Path: internal/http_api/compress.go\n// Copyright 2013 The Gorilla Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// copied from https://github.com/gorilla/handlers/blob/master/compress.go\n\npackage http_api\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n)\n\ntype compressResponseWriter struct {\n\tio.Writer\n\thttp.ResponseWriter\n\thttp.Hijacker\n}\n\nfunc (w *compressResponseWriter) Header() http.Header {\n\treturn w.ResponseWriter.Header()\n}\n\nfunc (w *compressResponseWriter) WriteHeader(c int) {\n\tw.ResponseWriter.Header().Del(\"Content-Length\")\n\tw.ResponseWriter.WriteHeader(c)\n}\n\nfunc (w *compressResponseWriter) Write(b []byte) (int, error) {\n\th := w.ResponseWriter.Header()\n\tif h.Get(\"Content-Type\") == \"\" {\n\t\th.Set(\"Content-Type\", http.DetectContentType(b))\n\t}\n\th.Del(\"Content-Length\")\n\treturn w.Writer.Write(b)\n}\n\n// CompressHandler gzip compresses HTTP responses for clients that support it\n// via the 'Accept-Encoding' header.\nfunc CompressHandler(h http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\tL:\n\t\tfor _, enc := range strings.Split(r.Header.Get(\"Accept-Encoding\"), \",\") {\n\t\t\tswitch strings.TrimSpace(enc) {\n\t\t\tcase \"gzip\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"gzip\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tgw := gzip.NewWriter(w)\n\t\t\t\tdefer gw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         gw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\tcase \"deflate\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"deflate\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tfw, _ := flate.NewWriter(w, flate.DefaultCompression)\n\t\t\t\tdefer fw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         fw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\t}\n\t\t}\n\n\t\th.ServeHTTP(w, r)\n\t})\n}\n\n// Path: internal/http_api/http_server.go\npackage http_api\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype logWriter struct {\n\tlogf lg.AppLogFunc\n}\n\nfunc (l logWriter) Write(p []byte) (int, error) {\n\tl.logf(lg.WARN, \"%s\", string(p))\n\treturn len(p), nil\n}\n\nfunc Serve(listener net.Listener, handler http.Handler, proto string, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"%s: listening on %s\", proto, listener.Addr())\n\n\tserver := &http.Server{\n\t\tHandler:  handler,\n\t\tErrorLog: log.New(logWriter{logf}, \"\", 0),\n\t}\n\terr := server.Serve(listener)\n\t// theres no direct way to detect this error because it is not exposed\n\tif err != nil && !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\treturn fmt.Errorf(\"http.Serve() error - %s\", err)\n\t}\n\n\tlogf(lg.INFO, \"%s: closing %s\", proto, listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/http_api/req_params.go\npackage http_api\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n)\n\ntype ReqParams struct {\n\turl.Values\n\tBody []byte\n}\n\nfunc NewReqParams(req *http.Request) (*ReqParams, error) {\n\treqParams, err := url.ParseQuery(req.URL.RawQuery)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, err := io.ReadAll(req.Body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &ReqParams{reqParams, data}, nil\n}\n\nfunc (r *ReqParams) Get(key string) (string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn \"\", errors.New(\"key not in query params\")\n\t}\n\treturn v[0], nil\n}\n\nfunc (r *ReqParams) GetAll(key string) ([]string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn nil, errors.New(\"key not in query params\")\n\t}\n\treturn v, nil\n}\n\n// Path: internal/version/binary.go\npackage version\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n)\n\nconst Binary = \"1.3.0\"\n\nfunc String(app string) string {\n\treturn fmt.Sprintf(\"%s v%s (built w/%s)\", app, Binary, runtime.Version())\n}\n\n// Path: internal/protocol/protocol.go\npackage protocol\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"net\"\n)\n\ntype Client interface {\n\tClose() error\n}\n\n// Protocol describes the basic behavior of any protocol in the system\ntype Protocol interface {\n\tNewClient(net.Conn) Client\n\tIOLoop(Client) error\n}\n\n// SendResponse is a server side utility function to prefix data with a length header\n// and write to the supplied Writer\nfunc SendResponse(w io.Writer, data []byte) (int, error) {\n\terr := binary.Write(w, binary.BigEndian, int32(len(data)))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tn, err := w.Write(data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn (n + 4), nil\n}\n\n// SendFramedResponse is a server side utility function to prefix data with a length header\n// and frame header and write to the supplied Writer\nfunc SendFramedResponse(w io.Writer, frameType int32, data []byte) (int, error) {\n\tbeBuf := make([]byte, 4)\n\tsize := uint32(len(data)) + 4\n\n\tbinary.BigEndian.PutUint32(beBuf, size)\n\tn, err := w.Write(beBuf)\n\tif err != nil {\n\t\treturn n, err\n\t}\n\n\tbinary.BigEndian.PutUint32(beBuf, uint32(frameType))\n\tn, err = w.Write(beBuf)\n\tif err != nil {\n\t\treturn n + 4, err\n\t}\n\n\tn, err = w.Write(data)\n\treturn n + 8, err\n}\n\n// Path: internal/http_api/topic_channel_args.go\npackage http_api\n\nimport (\n\t\"errors\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\ntype getter interface {\n\tGet(key string) (string, error)\n}\n\n\nfunc GetTopicChannelArgs(rp getter) (string, string, error) {\n\ttopicName, err := rp.Get(\"topic\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_TOPIC\")\n\t}\n\n\tif !protocol.IsValidTopicName(topicName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_TOPIC\")\n\t}\n\n\tchannelName, err := rp.Get(\"channel\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_CHANNEL\")\n\t}\n\n\tif !protocol.IsValidChannelName(channelName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_CHANNEL\")\n\t}\n\n\treturn topicName, channelName, nil\n}\n\n// Path: internal/lg/lg_test.go\npackage lg\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\ntype mockLogger struct {\n\tCount int\n}\n\nfunc (l *mockLogger) Output(maxdepth int, s string) error {\n\tl.Count++\n\treturn nil\n}\n\nfunc TestLogging(t *testing.T) {\n\tlogger := &mockLogger{}\n\n\t// Test only fatal get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, FATAL, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 1, logger.Count)\n\n\t// Test only warnings or higher get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, WARN, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 3, logger.Count)\n\n\t// Test everything gets through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, DEBUG, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 5, logger.Count)\n}\n\n// Path: internal/pqueue/pqueue.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n)\n\ntype Item struct {\n\tValue    interface{}\n\tPriority int64\n\tIndex    int\n}\n\n// this is a priority queue as implemented by a min heap\n// ie. the 0th element is the *lowest* value\ntype PriorityQueue []*Item\n\nfunc New(capacity int) PriorityQueue {\n\treturn make(PriorityQueue, 0, capacity)\n}\n\nfunc (pq PriorityQueue) Len() int {\n\treturn len(pq)\n}\n\nfunc (pq PriorityQueue) Less(i, j int) bool {\n\treturn pq[i].Priority < pq[j].Priority\n}\n\nfunc (pq PriorityQueue) Swap(i, j int) {\n\tpq[i], pq[j] = pq[j], pq[i]\n\tpq[i].Index = i\n\tpq[j].Index = j\n}\n\nfunc (pq *PriorityQueue) Push(x interface{}) {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n+1 > c {\n\t\tnpq := make(PriorityQueue, n, c*2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\t*pq = (*pq)[0 : n+1]\n\titem := x.(*Item)\n\titem.Index = n\n\t(*pq)[n] = item\n}\n\nfunc (pq *PriorityQueue) Pop() interface{} {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n < (c/2) && c > 25 {\n\t\tnpq := make(PriorityQueue, n, c/2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\titem := (*pq)[n-1]\n\titem.Index = -1\n\t*pq = (*pq)[0 : n-1]\n\treturn item\n}\n\nfunc (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) {\n\tif pq.Len() == 0 {\n\t\treturn nil, 0\n\t}\n\n\titem := (*pq)[0]\n\tif item.Priority > max {\n\t\treturn nil, item.Priority - max\n\t}\n\theap.Remove(pq, 0)\n\n\treturn item, 0\n}\n\n// Path: internal/pqueue/pqueue_test.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n\t\"math/rand\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"testing\"\n)\n\nfunc equal(t *testing.T, act, exp interface{}) {\n\tif !reflect.DeepEqual(exp, act) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\texp: %#v\\n\\n\\tgot: %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, exp, act)\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestPriorityQueue(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c+1; i++ {\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(i)})\n\t}\n\tequal(t, pq.Len(), c+1)\n\tequal(t, cap(pq), c*2)\n\n\tfor i := 0; i < c+1; i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, item.(*Item).Value.(int), i)\n\t}\n\tequal(t, cap(pq), c/4)\n}\n\nfunc TestUnsortedInsert(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\tints := make([]int, 0, c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\tints = append(ints, v)\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(v)})\n\t}\n\tequal(t, pq.Len(), c)\n\tequal(t, cap(pq), c)\n\n\tsort.Ints(ints)\n\n\tfor i := 0; i < c; i++ {\n\t\titem, _ := pq.PeekAndShift(int64(ints[len(ints)-1]))\n\t\tequal(t, item.Priority, int64(ints[i]))\n\t}\n}\n\nfunc TestRemove(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\theap.Push(&pq, &Item{Value: \"test\", Priority: int64(v)})\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\theap.Remove(&pq, rand.Intn((c-1)-i))\n\t}\n\n\tlastPriority := heap.Pop(&pq).(*Item).Priority\n\tfor i := 0; i < (c - 10 - 1); i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, lastPriority < item.(*Item).Priority, true)\n\t\tlastPriority = item.(*Item).Priority\n\t}\n}\n\n// Path: internal/protocol/byte_base10.go\npackage protocol\n\nimport (\n\t\"errors\"\n)\n\nvar errBase10 = errors.New(\"failed to convert to Base10\")\n\nfunc ByteToBase10(b []byte) (n uint64, err error) {\n\tbase := uint64(10)\n\n\tn = 0\n\tfor i := 0; i < len(b); i++ {\n\t\tvar v byte\n\t\td := b[i]\n\t\tswitch {\n\t\tcase '0' <= d && d <= '9':\n\t\t\tv = d - '0'\n\t\tdefault:\n\t\t\tn = 0\n\t\t\terr = errBase10\n\t\t\treturn\n\t\t}\n\t\tn *= base\n\t\tn += uint64(v)\n\t}\n\n\treturn n, err\n}\n\n// Path: internal/protocol/byte_base10_test.go\npackage protocol\n\nimport (\n\t\"testing\"\n)\n\nvar result uint64\n\nfunc BenchmarkByteToBase10Valid(b *testing.B) {\n\tbt := []byte{'3', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\nfunc BenchmarkByteToBase10Invalid(b *testing.B) {\n\tbt := []byte{'?', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\n// Path: internal/protocol/names.go\npackage protocol\n\nimport (\n\t\"regexp\"\n)\n\nvar validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral)?$`)\n\n// IsValidTopicName checks a topic name for correctness\nfunc IsValidTopicName(name string) bool {\n\treturn isValidName(name)\n}\n\n// IsValidChannelName checks a channel name for correctness\nfunc IsValidChannelName(name string) bool {\n\treturn isValidName(name)\n}\n\nfunc isValidName(name string) bool {\n\tif len(name) > 64 || len(name) < 1 {\n\t\treturn false\n\t}\n\treturn validTopicChannelNameRegex.MatchString(name)\n}\n\n// Path: internal/protocol/tcp_server.go\npackage protocol\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype TCPHandler interface {\n\tHandle(net.Conn)\n}\n\nfunc TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"TCP: listening on %s\", listener.Addr())\n\n\tvar wg sync.WaitGroup\n\n\tfor {\n\t\tclientConn, err := listener.Accept()\n\t\tif err != nil {\n\t\t\t// net.Error.Temporary() is deprecated, but is valid for accept\n\t\t\t// this is a hack to avoid a staticcheck error\n\t\t\tif te, ok := err.(interface{ Temporary() bool }); ok && te.Temporary() {\n\t\t\t\tlogf(lg.WARN, \"temporary Accept() failure - %s\", err)\n\t\t\t\truntime.Gosched()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// theres no direct way to detect this error because it is not exposed\n\t\t\tif !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\t\t\treturn fmt.Errorf(\"listener.Accept() error - %s\", err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\thandler.Handle(clientConn)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\t// wait to return until all handler goroutines complete\n\twg.Wait()\n\n\tlogf(lg.INFO, \"TCP: closing %s\", listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/quantile/aggregate.go\npackage quantile\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n)\n\ntype E2eProcessingLatencyAggregate struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\tTopic       string               `json:\"topic\"`\n\tChannel     string               `json:\"channel\"`\n\tAddr        string               `json:\"host\"`\n}\n\nfunc (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {\n\tvar resp struct {\n\t\tCount       int                  `json:\"count\"`\n\t\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\t\tTopic       string               `json:\"topic\"`\n\t\tChannel     string               `json:\"channel\"`\n\t\tAddr        string               `json:\"host\"`\n\t}\n\terr := json.Unmarshal(b, &resp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, p := range resp.Percentiles {\n\t\tp[\"min\"] = p[\"value\"]\n\t\tp[\"max\"] = p[\"value\"]\n\t\tp[\"average\"] = p[\"value\"]\n\t\tp[\"count\"] = float64(resp.Count)\n\t}\n\n\te.Count = resp.Count\n\te.Percentiles = resp.Percentiles\n\te.Topic = resp.Topic\n\te.Channel = resp.Channel\n\te.Addr = resp.Addr\n\n\treturn nil\n}\n\nfunc (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) }\nfunc (e *E2eProcessingLatencyAggregate) Swap(i, j int) {\n\te.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i]\n}\nfunc (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {\n\treturn e.Percentiles[i][\"percentile\"] > e.Percentiles[j][\"percentile\"]\n}\n\n// Add merges e2 into e by averaging the percentiles\nfunc (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) {\n\te.Addr = \"*\"\n\tp := e.Percentiles\n\te.Count += e2.Count\n\tfor _, value := range e2.Percentiles {\n\t\ti := -1\n\t\tfor j, v := range p {\n\t\t\tif value[\"quantile\"] == v[\"quantile\"] {\n\t\t\t\ti = j\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif i == -1 {\n\t\t\ti = len(p)\n\t\t\te.Percentiles = append(p, make(map[string]float64))\n\t\t\tp = e.Percentiles\n\t\t\tp[i][\"quantile\"] = value[\"quantile\"]\n\t\t}\n\t\tp[i][\"max\"] = math.Max(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"min\"] = math.Min(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"count\"] += value[\"count\"]\n\t\tif p[i][\"count\"] == 0 {\n\t\t\tp[i][\"average\"] = 0\n\t\t\tcontinue\n\t\t}\n\t\tdelta := value[\"average\"] - p[i][\"average\"]\n\t\tR := delta * value[\"count\"] / p[i][\"count\"]\n\t\tp[i][\"average\"] = p[i][\"average\"] + R\n\t}\n\tsort.Sort(e)\n}\n\n// Path: internal/statsd/host.go\npackage statsd\n\nimport (\n\t\"strings\"\n)\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \"_\", -1), \":\", \"_\", -1)\n}\n\n// Path: internal/stringy/slice.go\npackage stringy\n\nfunc Add(s []string, a string) []string {\n\tfor _, existing := range s {\n\t\tif a == existing {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn append(s, a)\n\n}\n\nfunc Union(s []string, a []string) []string {\n\tfor _, entry := range a {\n\t\tfound := false\n\t\tfor _, existing := range s {\n\t\t\tif entry == existing {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\ts = append(s, entry)\n\t\t}\n\t}\n\treturn s\n}\n\nfunc Uniq(s []string) (r []string) {\nouterLoop:\n\tfor _, entry := range s {\n\t\tfor _, existing := range r {\n\t\t\tif existing == entry {\n\t\t\t\tcontinue outerLoop\n\t\t\t}\n\t\t}\n\t\tr = append(r, entry)\n\t}\n\treturn\n}\n\n// Path: internal/stringy/slice_test.go\npackage stringy_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\nfunc BenchmarkUniq(b *testing.B) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"b\"}\n\tfor i := 0; i < b.N; i++ {\n\t\tvalues = stringy.Uniq(values)\n\t\tif len(values) != 2 {\n\t\t\tb.Fatal(\"values len is incorrect\")\n\t\t}\n\t}\n}\n\nfunc TestUniq(t *testing.T) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"b\", \"b\", \"b\", \"c\", \"c\", \"c\"}\n\tvalues = stringy.Uniq(values)\n\tif len(values) != 3 {\n\t\tt.Fatal(\"values len is incorrect\")\n\t}\n}\n\n// Path: internal/stringy/template.go\npackage stringy\n\nimport (\n\t\"fmt\"\n)\n\nfunc NanoSecondToHuman(v float64) string {\n\tvar suffix string\n\tswitch {\n\tcase v > 1000000000:\n\t\tv /= 1000000000\n\t\tsuffix = \"s\"\n\tcase v > 1000000:\n\t\tv /= 1000000\n\t\tsuffix = \"ms\"\n\tcase v > 1000:\n\t\tv /= 1000\n\t\tsuffix = \"us\"\n\tdefault:\n\t\tsuffix = \"ns\"\n\t}\n\treturn fmt.Sprintf(\"%0.1f%s\", v, suffix)\n}\n\n// Path: internal/test/assertions.go\npackage test\n\nimport (\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc Equal(t *testing.T, expected, actual interface{}) {\n\tif !reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   %#v (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotEqual(t *testing.T, expected, actual interface{}) {\n\tif reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tnexp: %#v\\n\\n\\tgot:  %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc Nil(t *testing.T, object interface{}) {\n\tif !isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   <nil> (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, object)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotNil(t *testing.T, object interface{}) {\n\tif isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n...\n", "needle_token_start": 9735, "needle_token_end": 9939, "code_context_ntokens": 15114, "output": ["```go\npackage http_api\n\nimport (\n\t\"errors\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\ntype getter interface {\n\tGet(key string) (string, error)\n}\n\nfunc GetTopicChannelArgs(rp getter) (string, string, error) {\n\ttopicName, err := rp.Get(\"topic\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_TOPIC\")\n\t}\n\n\tif !protocol.IsValidTopicName(topicName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_TOPIC\")\n\t}\n\n\tchannelName, err := rp.Get(\"channel\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_CHANNEL\")\n\t}\n\n\tif !protocol.IsValidChannelName(channelName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_CHANNEL\")\n\t}\n\n\treturn topicName, channelName, nil\n}\n```"]}
{"repo": "nsqio/nsq", "name": "Decorate", "language": "go", "path": "internal/http_api/api_response.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to enhance an API handler function by applying a series of modifications or enhancements, which are defined by additional functions provided as arguments. This allows for flexible configuration of handler behaviors such as error handling, response formatting, or logging.\n2. **Input**: The primary input is an API handler function that processes HTTP requests. Additional inputs are modifier functions that specify how the API handler should be enhanced or modified.\n3. **Output**: The output is a new HTTP router handler that incorporates all the specified enhancements into the original API handler, ready to be used in an HTTP server.\n4. **Procedure**: The function iteratively applies each modifier to the API handler, starting with the original handler and wrapping it with each subsequent modifier. The final result is a handler that has been decorated with all specified behaviors, which is then returned as a new router handler.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/clusterinfo/data.go\npackage clusterinfo\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/http_api\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype PartialErr interface {\n\terror\n\tErrors() []error\n}\n\ntype ErrList []error\n\nfunc (l ErrList) Error() string {\n\tvar es []string\n\tfor _, e := range l {\n\t\tes = append(es, e.Error())\n\t}\n\treturn strings.Join(es, \"\\n\")\n}\n\nfunc (l ErrList) Errors() []error {\n\treturn l\n}\n\ntype ClusterInfo struct {\n\tlog    lg.AppLogFunc\n\tclient *http_api.Client\n}\n\nfunc New(log lg.AppLogFunc, client *http_api.Client) *ClusterInfo {\n\treturn &ClusterInfo{\n\t\tlog:    log,\n\t\tclient: client,\n\t}\n}\n\nfunc (c *ClusterInfo) logf(f string, args ...interface{}) {\n\tif c.log != nil {\n\t\tc.log(lg.INFO, f, args...)\n\t}\n}\n\n// GetVersion returns a semver.Version object by querying /info\nfunc (c *ClusterInfo) GetVersion(addr string) (semver.Version, error) {\n\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\tvar resp struct {\n\t\tVersion string `json:\"version\"`\n\t}\n\terr := c.client.GETV1(endpoint, &resp)\n\tif err != nil {\n\t\treturn semver.Version{}, err\n\t}\n\tif resp.Version == \"\" {\n\t\tresp.Version = \"unknown\"\n\t}\n\treturn semver.Parse(resp.Version)\n}\n\n// GetLookupdTopics returns a []string containing a union of all the topics\n// from all the given nsqlookupd\nfunc (c *ClusterInfo) GetLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []string `json:\"topics\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/topics\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\ttopics = append(topics, resp.Topics...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\ttopics = stringy.Uniq(topics)\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetLookupdTopicChannels returns a []string containing a union of all the channels\n// from all the given lookupd for the given topic\nfunc (c *ClusterInfo) GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {\n\tvar channels []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tChannels []string `json:\"channels\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/channels?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tchannels = append(channels, resp.Channels...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tchannels = stringy.Uniq(channels)\n\tsort.Strings(channels)\n\n\tif len(errs) > 0 {\n\t\treturn channels, ErrList(errs)\n\t}\n\treturn channels, nil\n}\n\n// GetLookupdProducers returns Producers of all the nsqd connected to the given lookupds\nfunc (c *ClusterInfo) GetLookupdProducers(lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers []*Producer\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\tproducersByAddr := make(map[string]*Producer)\n\tmaxVersion, _ := semver.Parse(\"0.0.0\")\n\n\ttype respType struct {\n\t\tProducers []*Producer `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/nodes\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, producer := range resp.Producers {\n\t\t\t\tkey := producer.TCPAddress()\n\t\t\t\tp, ok := producersByAddr[key]\n\t\t\t\tif !ok {\n\t\t\t\t\tproducersByAddr[key] = producer\n\t\t\t\t\tproducers = append(producers, producer)\n\t\t\t\t\tif maxVersion.LT(producer.VersionObj) {\n\t\t\t\t\t\tmaxVersion = producer.VersionObj\n\t\t\t\t\t}\n\t\t\t\t\tsort.Sort(producer.Topics)\n\t\t\t\t\tp = producer\n\t\t\t\t}\n\t\t\t\tp.RemoteAddresses = append(p.RemoteAddresses,\n\t\t\t\t\tfmt.Sprintf(\"%s/%s\", addr, producer.Address()))\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tfor _, producer := range producersByAddr {\n\t\tif producer.VersionObj.LT(maxVersion) {\n\t\t\tproducer.OutOfDate = true\n\t\t}\n\t}\n\tsort.Sort(ProducersByHost{producers})\n\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetLookupdTopicProducers returns Producers of all the nsqd for a given topic by\n// unioning the nodes returned from the given lookupd\nfunc (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tProducers Producers `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/lookup?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, p := range resp.Producers {\n\t\t\t\tfor _, pp := range producers {\n\t\t\t\t\tif p.HTTPAddress() == pp.HTTPAddress() {\n\t\t\t\t\t\tgoto skip\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tproducers = append(producers, p)\n\t\t\tskip:\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDTopics returns a []string containing all the topics produced by the given nsqd\nfunc (c *ClusterInfo) GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\ttopics = stringy.Add(topics, topic.Name)\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetNSQDProducers returns Producers of all the given nsqd\nfunc (c *ClusterInfo) GetNSQDProducers(nsqdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype infoRespType struct {\n\t\tVersion          string `json:\"version\"`\n\t\tBroadcastAddress string `json:\"broadcast_address\"`\n\t\tHostname         string `json:\"hostname\"`\n\t\tHTTPPort         int    `json:\"http_port\"`\n\t\tTCPPort          int    `json:\"tcp_port\"`\n\t}\n\n\ttype statsRespType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar infoResp infoRespType\n\t\t\terr := c.client.GETV1(endpoint, &infoResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tendpoint = fmt.Sprintf(\"http://%s/stats?format=json&include_clients=false\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n...\n// Path: internal/clusterinfo/producer_test.go\npackage clusterinfo\n\nimport \"testing\"\n\nfunc TestHostNameAddresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"host.domain.com\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"host.domain.com:4151\" {\n\t\tt.Errorf(\"Incorrect HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"host.domain.com:4150\" {\n\t\tt.Errorf(\"Incorrect TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv4Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"192.168.1.17\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"192.168.1.17:4151\" {\n\t\tt.Errorf(\"Incorrect IPv4 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"192.168.1.17:4150\" {\n\t\tt.Errorf(\"Incorrect IPv4 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv6Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"fd4a:622f:d2f2::1\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\tif p.HTTPAddress() != \"[fd4a:622f:d2f2::1]:4151\" {\n\t\tt.Errorf(\"Incorrect IPv6 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"[fd4a:622f:d2f2::1]:4150\" {\n\t\tt.Errorf(\"Incorrect IPv6 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\n// Path: internal/quantile/quantile.go\npackage quantile\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/bmizerany/perks/quantile\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype Result struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n}\n\nfunc (r *Result) String() string {\n\tvar s []string\n\tfor _, item := range r.Percentiles {\n\t\ts = append(s, stringy.NanoSecondToHuman(item[\"value\"]))\n\t}\n\treturn strings.Join(s, \", \")\n}\n\ntype Quantile struct {\n\tsync.Mutex\n\tstreams        [2]quantile.Stream\n\tcurrentIndex   uint8\n\tlastMoveWindow time.Time\n\tcurrentStream  *quantile.Stream\n\n\tPercentiles    []float64\n\tMoveWindowTime time.Duration\n}\n\nfunc New(WindowTime time.Duration, Percentiles []float64) *Quantile {\n\tq := Quantile{\n\t\tcurrentIndex:   0,\n\t\tlastMoveWindow: time.Now(),\n\t\tMoveWindowTime: WindowTime / 2,\n\t\tPercentiles:    Percentiles,\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tq.streams[i] = *quantile.NewTargeted(Percentiles...)\n\t}\n\tq.currentStream = &q.streams[0]\n\treturn &q\n}\n\nfunc (q *Quantile) Result() *Result {\n\tif q == nil {\n\t\treturn &Result{}\n\t}\n\tqueryHandler := q.QueryHandler()\n\tresult := Result{\n\t\tCount:       queryHandler.Count(),\n\t\tPercentiles: make([]map[string]float64, len(q.Percentiles)),\n\t}\n\tfor i, p := range q.Percentiles {\n\t\tvalue := queryHandler.Query(p)\n\t\tresult.Percentiles[i] = map[string]float64{\"quantile\": p, \"value\": value}\n\t}\n\treturn &result\n}\n\nfunc (q *Quantile) Insert(msgStartTime int64) {\n\tq.Lock()\n\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tq.currentStream.Insert(float64(now.UnixNano() - msgStartTime))\n\tq.Unlock()\n}\n\nfunc (q *Quantile) QueryHandler() *quantile.Stream {\n\tq.Lock()\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tmerged := quantile.NewTargeted(q.Percentiles...)\n\tmerged.Merge(q.streams[0].Samples())\n\tmerged.Merge(q.streams[1].Samples())\n\tq.Unlock()\n\treturn merged\n}\n\nfunc (q *Quantile) IsDataStale(now time.Time) bool {\n\treturn now.After(q.lastMoveWindow.Add(q.MoveWindowTime))\n}\n\nfunc (q *Quantile) Merge(them *Quantile) {\n\tq.Lock()\n\tthem.Lock()\n\tiUs := q.currentIndex\n\tiThem := them.currentIndex\n\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tiUs ^= 0x1\n\tiThem ^= 0x1\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tif q.lastMoveWindow.Before(them.lastMoveWindow) {\n\t\tq.lastMoveWindow = them.lastMoveWindow\n\t}\n\tq.Unlock()\n\tthem.Unlock()\n}\n\nfunc (q *Quantile) moveWindow() {\n\tq.currentIndex ^= 0x1\n\tq.currentStream = &q.streams[q.currentIndex]\n\tq.lastMoveWindow = q.lastMoveWindow.Add(q.MoveWindowTime)\n\tq.currentStream.Reset()\n}\n\n// Path: internal/clusterinfo/types.go\npackage clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype ProducerTopic struct {\n\tTopic      string `json:\"topic\"`\n\tTombstoned bool   `json:\"tombstoned\"`\n}\n\ntype ProducerTopics []ProducerTopic\n\nfunc (pt ProducerTopics) Len() int           { return len(pt) }\nfunc (pt ProducerTopics) Swap(i, j int)      { pt[i], pt[j] = pt[j], pt[i] }\nfunc (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt[j].Topic }\n\ntype Producer struct {\n\tRemoteAddresses  []string       `json:\"remote_addresses\"`\n\tRemoteAddress    string         `json:\"remote_address\"`\n\tHostname         string         `json:\"hostname\"`\n\tBroadcastAddress string         `json:\"broadcast_address\"`\n\tTCPPort          int            `json:\"tcp_port\"`\n\tHTTPPort         int            `json:\"http_port\"`\n\tVersion          string         `json:\"version\"`\n\tVersionObj       semver.Version `json:\"-\"`\n\tTopics           ProducerTopics `json:\"topics\"`\n\tOutOfDate        bool           `json:\"out_of_date\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses of ProducerTopics and VersionObj\nfunc (p *Producer) UnmarshalJSON(b []byte) error {\n\tvar r struct {\n\t\tRemoteAddress    string   `json:\"remote_address\"`\n\t\tHostname         string   `json:\"hostname\"`\n\t\tBroadcastAddress string   `json:\"broadcast_address\"`\n\t\tTCPPort          int      `json:\"tcp_port\"`\n\t\tHTTPPort         int      `json:\"http_port\"`\n\t\tVersion          string   `json:\"version\"`\n\t\tTopics           []string `json:\"topics\"`\n\t\tTombstoned       []bool   `json:\"tombstones\"`\n\t}\n\tif err := json.Unmarshal(b, &r); err != nil {\n\t\treturn err\n\t}\n\t*p = Producer{\n\t\tRemoteAddress:    r.RemoteAddress,\n\t\tHostname:         r.Hostname,\n\t\tBroadcastAddress: r.BroadcastAddress,\n\t\tTCPPort:          r.TCPPort,\n\t\tHTTPPort:         r.HTTPPort,\n\t\tVersion:          r.Version,\n\t}\n\tfor i, t := range r.Topics {\n\t\tp.Topics = append(p.Topics, ProducerTopic{Topic: t, Tombstoned: r.Tombstoned[i]})\n\t}\n\tversion, err := semver.Parse(p.Version)\n\tif err != nil {\n\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t}\n\tp.VersionObj = version\n\treturn nil\n}\n\nfunc (p *Producer) Address() string {\n\tif p.RemoteAddress == \"\" {\n\t\treturn \"N/A\"\n\t}\n\treturn p.RemoteAddress\n}\n\nfunc (p *Producer) HTTPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.HTTPPort))\n}\n\nfunc (p *Producer) TCPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.TCPPort))\n}\n\n// IsInconsistent checks for cases where an unexpected number of nsqd connections are\n// reporting the same information to nsqlookupd (ie: multiple instances are using the\n// same broadcast address), or cases where some nsqd are not reporting to all nsqlookupd.\nfunc (p *Producer) IsInconsistent(numLookupd int) bool {\n\treturn len(p.RemoteAddresses) != numLookupd\n}\n\ntype TopicStats struct {\n\tNode         string          `json:\"node\"`\n\tHostname     string          `json:\"hostname\"`\n\tTopicName    string          `json:\"topic_name\"`\n\tDepth        int64           `json:\"depth\"`\n\tMemoryDepth  int64           `json:\"memory_depth\"`\n\tBackendDepth int64           `json:\"backend_depth\"`\n\tMessageCount int64           `json:\"message_count\"`\n\tNodeStats    []*TopicStats   `json:\"nodes\"`\n\tChannels     []*ChannelStats `json:\"channels\"`\n\tPaused       bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (t *TopicStats) Add(a *TopicStats) {\n\tt.Node = \"*\"\n\tt.Depth += a.Depth\n\tt.MemoryDepth += a.MemoryDepth\n\tt.BackendDepth += a.BackendDepth\n\tt.MessageCount += a.MessageCount\n\tif a.Paused {\n\t\tt.Paused = a.Paused\n\t}\n\tfor _, aChannelStats := range a.Channels {\n\t\tfound := false\n\t\tfor _, channelStats := range t.Channels {\n\t\t\tif aChannelStats.ChannelName == channelStats.ChannelName {\n\t\t\t\tfound = true\n\t\t\t\tchannelStats.Add(aChannelStats)\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\tt.Channels = append(t.Channels, aChannelStats)\n\t\t}\n\t}\n\tt.NodeStats = append(t.NodeStats, a)\n\tsort.Sort(TopicStatsByHost{t.NodeStats})\n\tif t.E2eProcessingLatency == nil {\n\t\tt.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:  t.Node,\n\t\t\tTopic: t.TopicName,\n\t\t}\n\t}\n\tt.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n}\n\ntype ChannelStats struct {\n\tNode          string          `json:\"node\"`\n\tHostname      string          `json:\"hostname\"`\n\tTopicName     string          `json:\"topic_name\"`\n\tChannelName   string          `json:\"channel_name\"`\n\tDepth         int64           `json:\"depth\"`\n\tMemoryDepth   int64           `json:\"memory_depth\"`\n\tBackendDepth  int64           `json:\"backend_depth\"`\n\tInFlightCount int64           `json:\"in_flight_count\"`\n\tDeferredCount int64           `json:\"deferred_count\"`\n\tRequeueCount  int64           `json:\"requeue_count\"`\n\tTimeoutCount  int64           `json:\"timeout_count\"`\n\tMessageCount  int64           `json:\"message_count\"`\n\tClientCount   int             `json:\"client_count\"`\n\tSelected      bool            `json:\"-\"`\n\tNodeStats     []*ChannelStats `json:\"nodes\"`\n\tClients       []*ClientStats  `json:\"clients\"`\n\tPaused        bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (c *ChannelStats) Add(a *ChannelStats) {\n\tc.Node = \"*\"\n\tc.Depth += a.Depth\n\tc.MemoryDepth += a.MemoryDepth\n\tc.BackendDepth += a.BackendDepth\n\tc.InFlightCount += a.InFlightCount\n\tc.DeferredCount += a.DeferredCount\n\tc.RequeueCount += a.RequeueCount\n\tc.TimeoutCount += a.TimeoutCount\n\tc.MessageCount += a.MessageCount\n\tc.ClientCount += a.ClientCount\n\tif a.Paused {\n\t\tc.Paused = a.Paused\n\t}\n\tc.NodeStats = append(c.NodeStats, a)\n\tsort.Sort(ChannelStatsByHost{c.NodeStats})\n\tif c.E2eProcessingLatency == nil {\n\t\tc.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:    c.Node,\n\t\t\tTopic:   c.TopicName,\n\t\t\tChannel: c.ChannelName,\n\t\t}\n\t}\n\tc.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n\tc.Clients = append(c.Clients, a.Clients...)\n\tsort.Sort(ClientsByHost{c.Clients})\n}\n\ntype ClientStats struct {\n\tNode              string        `json:\"node\"`\n\tRemoteAddress     string        `json:\"remote_address\"`\n\tVersion           string        `json:\"version\"`\n\tClientID          string        `json:\"client_id\"`\n\tHostname          string        `json:\"hostname\"`\n\tUserAgent         string        `json:\"user_agent\"`\n\tConnectTs         int64         `json:\"connect_ts\"`\n\tConnectedDuration time.Duration `json:\"connected\"`\n\tInFlightCount     int           `json:\"in_flight_count\"`\n\tReadyCount        int           `json:\"ready_count\"`\n\tFinishCount       int64         `json:\"finish_count\"`\n\tRequeueCount      int64         `json:\"requeue_count\"`\n\tMessageCount      int64         `json:\"message_count\"`\n\tSampleRate        int32         `json:\"sample_rate\"`\n\tDeflate           bool          `json:\"deflate\"`\n\tSnappy            bool          `json:\"snappy\"`\n\tAuthed            bool          `json:\"authed\"`\n\tAuthIdentity      string        `json:\"auth_identity\"`\n\tAuthIdentityURL   string        `json:\"auth_identity_url\"`\n\n\tTLS                           bool   `json:\"tls\"`\n\tCipherSuite                   string `json:\"tls_cipher_suite\"`\n\tTLSVersion                    string `json:\"tls_version\"`\n\tTLSNegotiatedProtocol         string `json:\"tls_negotiated_protocol\"`\n\tTLSNegotiatedProtocolIsMutual bool   `json:\"tls_negotiated_protocol_is_mutual\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses ConnectedDuration\nfunc (s *ClientStats) UnmarshalJSON(b []byte) error {\n\ttype locaClientStats ClientStats // re-typed to prevent recursion from json.Unmarshal\n\tvar ss locaClientStats\n\tif err := json.Unmarshal(b, &ss); err != nil {\n\t\treturn err\n\t}\n\t*s = ClientStats(ss)\n\ts.ConnectedDuration = time.Now().Truncate(time.Second).Sub(time.Unix(s.ConnectTs, 0))\n\treturn nil\n}\n\nfunc (s *ClientStats) HasUserAgent() bool {\n\treturn s.UserAgent != \"\"\n}\n\nfunc (s *ClientStats) HasSampleRate() bool {\n\treturn s.SampleRate > 0\n}\n\ntype ChannelStatsList []*ChannelStats\n\nfunc (c ChannelStatsList) Len() int      { return len(c) }\nfunc (c ChannelStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ChannelStatsByHost struct {\n\tChannelStatsList\n}\n\nfunc (c ChannelStatsByHost) Less(i, j int) bool {\n\treturn c.ChannelStatsList[i].Hostname < c.ChannelStatsList[j].Hostname\n}\n\ntype ClientStatsList []*ClientStats\n\nfunc (c ClientStatsList) Len() int      { return len(c) }\nfunc (c ClientStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ClientsByHost struct {\n\tClientStatsList\n}\n\nfunc (c ClientsByHost) Less(i, j int) bool {\n\treturn c.ClientStatsList[i].Hostname < c.ClientStatsList[j].Hostname\n}\n\ntype TopicStatsList []*TopicStats\n\nfunc (t TopicStatsList) Len() int      { return len(t) }\nfunc (t TopicStatsList) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\ntype TopicStatsByHost struct {\n\tTopicStatsList\n}\n\nfunc (c TopicStatsByHost) Less(i, j int) bool {\n\treturn c.TopicStatsList[i].Hostname < c.TopicStatsList[j].Hostname\n}\n\ntype Producers []*Producer\n\nfunc (t Producers) Len() int      { return len(t) }\nfunc (t Producers) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\nfunc (t Producers) HTTPAddrs() []string {\n\tvar addrs []string\n\tfor _, p := range t {\n\t\taddrs = append(addrs, p.HTTPAddress())\n\t}\n\treturn addrs\n}\n\nfunc (t Producers) Search(needle string) *Producer {\n\tfor _, producer := range t {\n\t\tif needle == producer.HTTPAddress() {\n\t\t\treturn producer\n\t\t}\n\t}\n\treturn nil\n}\n\ntype ProducersByHost struct {\n\tProducers\n}\n\nfunc (c ProducersByHost) Less(i, j int) bool {\n\treturn c.Producers[i].Hostname < c.Producers[j].Hostname\n}\n\n// Path: internal/dirlock/dirlock.go\n//go:build !windows && !illumos\n// +build !windows,!illumos\n\npackage dirlock\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n)\n\ntype DirLock struct {\n\tdir string\n\tf   *os.File\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\tf, err := os.Open(l.dir)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl.f = f\n\terr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"cannot flock directory %s - %s (possibly in use by another instance of nsqd)\", l.dir, err)\n\t}\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\tdefer l.f.Close()\n\treturn syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)\n}\n\n// Path: internal/dirlock/dirlock_illumos.go\n//go:build illumos\n// +build illumos\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/dirlock/dirlock_windows.go\n//go:build windows\n// +build windows\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/http_api/api_request.go\npackage http_api\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// A custom http.Transport with support for deadline timeouts\nfunc NewDeadlineTransport(connectTimeout time.Duration, requestTimeout time.Duration) *http.Transport {\n\t// arbitrary values copied from http.DefaultTransport\n\ttransport := &http.Transport{\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout:   connectTimeout,\n\t\t\tKeepAlive: 30 * time.Second,\n\t\t\tDualStack: true,\n\t\t}).DialContext,\n\t\tResponseHeaderTimeout: requestTimeout,\n\t\tMaxIdleConns:          100,\n\t\tIdleConnTimeout:       90 * time.Second,\n\t\tTLSHandshakeTimeout:   10 * time.Second,\n\t}\n\treturn transport\n}\n\ntype Client struct {\n\tc *http.Client\n}\n\nfunc NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) *Client {\n\ttransport := NewDeadlineTransport(connectTimeout, requestTimeout)\n\ttransport.TLSClientConfig = tlsConfig\n\treturn &Client{\n\t\tc: &http.Client{\n\t\t\tTransport: transport,\n\t\t\tTimeout:   requestTimeout,\n\t\t},\n\t}\n}\n\n// GETV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) GETV1(endpoint string, v interface{}) error {\nretry:\n\treq, err := http.NewRequest(\"GET\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\terr = json.Unmarshal(body, &v)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// PostV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) POSTV1(endpoint string) error {\nretry:\n\treq, err := http.NewRequest(\"POST\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\n\treturn nil\n}\n\nfunc httpsEndpoint(endpoint string, body []byte) (string, error) {\n\tvar forbiddenResp struct {\n\t\tHTTPSPort int `json:\"https_port\"`\n\t}\n\terr := json.Unmarshal(body, &forbiddenResp)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thost, _, err := net.SplitHostPort(u.Host)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu.Scheme = \"https\"\n\tu.Host = net.JoinHostPort(host, strconv.Itoa(forbiddenResp.HTTPSPort))\n\treturn u.String(), nil\n}\n\n// Path: internal/http_api/api_response.go\npackage http_api\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/julienschmidt/httprouter\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Decorator func(APIHandler) APIHandler\n\ntype APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)\n\ntype Err struct {\n\tCode int\n\tText string\n}\n\nfunc (e Err) Error() string {\n\treturn e.Text\n}\n\nfunc PlainText(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tcode := 200\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tcode = err.(Err).Code\n\t\t\tdata = err.Error()\n\t\t}\n\t\tswitch d := data.(type) {\n\t\tcase string:\n\t\t\tw.WriteHeader(code)\n\t\t\tio.WriteString(w, d)\n\t\tcase []byte:\n\t\t\tw.WriteHeader(code)\n\t\t\tw.Write(d)\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"unknown response type %T\", data))\n\t\t}\n\t\treturn nil, nil\n\t}\n}\n\nfunc V1(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tRespondV1(w, err.(Err).Code, err)\n\t\t\treturn nil, nil\n\t\t}\n\t\tRespondV1(w, 200, data)\n\t\treturn nil, nil\n\t}\n}\n\nfunc RespondV1(w http.ResponseWriter, code int, data interface{}) {\n\tvar response []byte\n\tvar err error\n\tvar isJSON bool\n\n\tif code == 200 {\n\t\tswitch data := data.(type) {\n\t\tcase string:\n\t\t\tresponse = []byte(data)\n\t\tcase []byte:\n\t\t\tresponse = data\n\t\tcase nil:\n\t\t\tresponse = []byte{}\n\t\tdefault:\n\t\t\tisJSON = true\n\t\t\tresponse, err = json.Marshal(data)\n\t\t\tif err != nil {\n\t\t\t\tcode = 500\n\t\t\t\tdata = err\n\t\t\t}\n\t\t}\n\t}\n\n\tif code != 200 {\n\t\tisJSON = true\n\t\tresponse, _ = json.Marshal(struct {\n\t\t\tMessage string `json:\"message\"`\n\t\t}{fmt.Sprintf(\"%s\", data)})\n\t}\n\n\tif isJSON {\n\t\tw.Header().Set(\"Content-Type\", \"application/json; charset=utf-8\")\n\t}\n\tw.Header().Set(\"X-NSQ-Content-Type\", \"nsq; version=1.0\")\n\tw.WriteHeader(code)\n\tw.Write(response)\n}\n\n\nfunc Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {\n\tdecorated := f\n\tfor _, decorate := range ds {\n\t\tdecorated = decorate(decorated)\n\t}\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {\n\t\tdecorated(w, req, ps)\n\t}\n}\n\nfunc Log(logf lg.AppLogFunc) Decorator {\n\treturn func(f APIHandler) APIHandler {\n\t\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\tstart := time.Now()\n\t\t\tresponse, err := f(w, req, ps)\n\t\t\telapsed := time.Since(start)\n\t\t\tstatus := 200\n\t\t\tif e, ok := err.(Err); ok {\n\t\t\t\tstatus = e.Code\n\t\t\t}\n\t\t\tlogf(lg.INFO, \"%d %s %s (%s) %s\",\n\t\t\t\tstatus, req.Method, req.URL.RequestURI(), req.RemoteAddr, elapsed)\n\t\t\treturn response, err\n\t\t}\n\t}\n}\n\nfunc LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\treturn func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\t\tlogf(lg.ERROR, \"panic in HTTP handler - %s\", p)\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{500, \"INTERNAL_ERROR\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t}\n}\n\nfunc LogNotFoundHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{404, \"NOT_FOUND\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\nfunc LogMethodNotAllowedHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{405, \"METHOD_NOT_ALLOWED\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\n// Path: internal/http_api/compress.go\n// Copyright 2013 The Gorilla Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// copied from https://github.com/gorilla/handlers/blob/master/compress.go\n\npackage http_api\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n)\n\ntype compressResponseWriter struct {\n\tio.Writer\n\thttp.ResponseWriter\n\thttp.Hijacker\n}\n\nfunc (w *compressResponseWriter) Header() http.Header {\n\treturn w.ResponseWriter.Header()\n}\n\nfunc (w *compressResponseWriter) WriteHeader(c int) {\n\tw.ResponseWriter.Header().Del(\"Content-Length\")\n\tw.ResponseWriter.WriteHeader(c)\n}\n\nfunc (w *compressResponseWriter) Write(b []byte) (int, error) {\n\th := w.ResponseWriter.Header()\n\tif h.Get(\"Content-Type\") == \"\" {\n\t\th.Set(\"Content-Type\", http.DetectContentType(b))\n\t}\n\th.Del(\"Content-Length\")\n\treturn w.Writer.Write(b)\n}\n\n// CompressHandler gzip compresses HTTP responses for clients that support it\n// via the 'Accept-Encoding' header.\nfunc CompressHandler(h http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\tL:\n\t\tfor _, enc := range strings.Split(r.Header.Get(\"Accept-Encoding\"), \",\") {\n\t\t\tswitch strings.TrimSpace(enc) {\n\t\t\tcase \"gzip\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"gzip\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tgw := gzip.NewWriter(w)\n\t\t\t\tdefer gw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         gw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\tcase \"deflate\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"deflate\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tfw, _ := flate.NewWriter(w, flate.DefaultCompression)\n\t\t\t\tdefer fw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         fw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\t}\n\t\t}\n\n\t\th.ServeHTTP(w, r)\n\t})\n}\n\n// Path: internal/http_api/http_server.go\npackage http_api\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype logWriter struct {\n\tlogf lg.AppLogFunc\n}\n\nfunc (l logWriter) Write(p []byte) (int, error) {\n\tl.logf(lg.WARN, \"%s\", string(p))\n\treturn len(p), nil\n}\n\nfunc Serve(listener net.Listener, handler http.Handler, proto string, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"%s: listening on %s\", proto, listener.Addr())\n\n\tserver := &http.Server{\n\t\tHandler:  handler,\n\t\tErrorLog: log.New(logWriter{logf}, \"\", 0),\n\t}\n\terr := server.Serve(listener)\n\t// theres no direct way to detect this error because it is not exposed\n\tif err != nil && !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\treturn fmt.Errorf(\"http.Serve() error - %s\", err)\n\t}\n\n\tlogf(lg.INFO, \"%s: closing %s\", proto, listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/http_api/req_params.go\npackage http_api\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n)\n\ntype ReqParams struct {\n\turl.Values\n\tBody []byte\n}\n\nfunc NewReqParams(req *http.Request) (*ReqParams, error) {\n\treqParams, err := url.ParseQuery(req.URL.RawQuery)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, err := io.ReadAll(req.Body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &ReqParams{reqParams, data}, nil\n}\n\nfunc (r *ReqParams) Get(key string) (string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn \"\", errors.New(\"key not in query params\")\n\t}\n\treturn v[0], nil\n}\n\nfunc (r *ReqParams) GetAll(key string) ([]string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn nil, errors.New(\"key not in query params\")\n\t}\n\treturn v, nil\n}\n\n// Path: internal/version/binary.go\npackage version\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n)\n\nconst Binary = \"1.3.0\"\n\nfunc String(app string) string {\n\treturn fmt.Sprintf(\"%s v%s (built w/%s)\", app, Binary, runtime.Version())\n}\n\n// Path: internal/protocol/protocol.go\npackage protocol\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"net\"\n)\n\ntype Client interface {\n\tClose() error\n}\n\n// Protocol describes the basic behavior of any protocol in the system\ntype Protocol interface {\n\tNewClient(net.Conn) Client\n\tIOLoop(Client) error\n}\n\n// SendResponse is a server side utility function to prefix data with a length header\n// and write to the supplied Writer\nfunc SendResponse(w io.Writer, data []byte) (int, error) {\n\terr := binary.Write(w, binary.BigEndian, int32(len(data)))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tn, err := w.Write(data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn (n + 4), nil\n}\n\n// SendFramedResponse is a server side utility function to prefix data with a length header\n// and frame header and write to the supplied Writer\nfunc SendFramedResponse(w io.Writer, frameType int32, data []byte) (int, error) {\n\tbeBuf := make([]byte, 4)\n\tsize := uint32(len(data)) + 4\n\n\tbinary.BigEndian.PutUint32(beBuf, size)\n\tn, err := w.Write(beBuf)\n\tif err != nil {\n\t\treturn n, err\n\t}\n\n\tbinary.BigEndian.PutUint32(beBuf, uint32(frameType))\n\tn, err = w.Write(beBuf)\n\tif err != nil {\n\t\treturn n + 4, err\n\t}\n\n\tn, err = w.Write(data)\n\treturn n + 8, err\n}\n\n// Path: internal/http_api/topic_channel_args.go\npackage http_api\n\nimport (\n\t\"errors\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\ntype getter interface {\n\tGet(key string) (string, error)\n}\n\nfunc GetTopicChannelArgs(rp getter) (string, string, error) {\n\ttopicName, err := rp.Get(\"topic\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_TOPIC\")\n\t}\n\n\tif !protocol.IsValidTopicName(topicName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_TOPIC\")\n\t}\n\n\tchannelName, err := rp.Get(\"channel\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_CHANNEL\")\n\t}\n\n\tif !protocol.IsValidChannelName(channelName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_CHANNEL\")\n\t}\n\n\treturn topicName, channelName, nil\n}\n\n// Path: internal/lg/lg_test.go\npackage lg\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\ntype mockLogger struct {\n\tCount int\n}\n\nfunc (l *mockLogger) Output(maxdepth int, s string) error {\n\tl.Count++\n\treturn nil\n}\n\nfunc TestLogging(t *testing.T) {\n\tlogger := &mockLogger{}\n\n\t// Test only fatal get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, FATAL, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 1, logger.Count)\n\n\t// Test only warnings or higher get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, WARN, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 3, logger.Count)\n\n\t// Test everything gets through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, DEBUG, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 5, logger.Count)\n}\n\n// Path: internal/pqueue/pqueue.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n)\n\ntype Item struct {\n\tValue    interface{}\n\tPriority int64\n\tIndex    int\n}\n\n// this is a priority queue as implemented by a min heap\n// ie. the 0th element is the *lowest* value\ntype PriorityQueue []*Item\n\nfunc New(capacity int) PriorityQueue {\n\treturn make(PriorityQueue, 0, capacity)\n}\n\nfunc (pq PriorityQueue) Len() int {\n\treturn len(pq)\n}\n\nfunc (pq PriorityQueue) Less(i, j int) bool {\n\treturn pq[i].Priority < pq[j].Priority\n}\n\nfunc (pq PriorityQueue) Swap(i, j int) {\n\tpq[i], pq[j] = pq[j], pq[i]\n\tpq[i].Index = i\n\tpq[j].Index = j\n}\n\nfunc (pq *PriorityQueue) Push(x interface{}) {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n+1 > c {\n\t\tnpq := make(PriorityQueue, n, c*2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\t*pq = (*pq)[0 : n+1]\n\titem := x.(*Item)\n\titem.Index = n\n\t(*pq)[n] = item\n}\n\nfunc (pq *PriorityQueue) Pop() interface{} {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n < (c/2) && c > 25 {\n...\n", "needle_token_start": 11271, "needle_token_end": 11365, "code_context_ntokens": 15090, "output": ["```go\nfunc Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {\n\tdecorated := f\n\tfor _, decorate := range ds {\n\t\tdecorated = decorate(decorated)\n\t}\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {\n\t\tdecorated(w, req, ps)\n\t}\n}\n```"]}
{"repo": "nsqio/nsq", "name": "BenchmarkUniqRands20of20", "language": "go", "path": "internal/util/util_test.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to measure the performance of generating a set of unique random integers where the total number of integers generated is equal to the maximum possible unique integers.\n2. **Input**: The function does not take any direct inputs from the user as it is a benchmark test; it internally uses fixed parameters to generate the integers.\n3. **Output**: There is no direct output returned to the user, as the primary goal is to assess the time efficiency and performance of the integer generation process over multiple iterations.\n4. **Procedure**: The function repeatedly generates a set of 20 unique random integers, where the range of possible integers is also 20. This process is repeated for a number of iterations determined by the benchmarking framework to evaluate the performance.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/clusterinfo/types.go\npackage clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype ProducerTopic struct {\n\tTopic      string `json:\"topic\"`\n\tTombstoned bool   `json:\"tombstoned\"`\n}\n\ntype ProducerTopics []ProducerTopic\n\nfunc (pt ProducerTopics) Len() int           { return len(pt) }\nfunc (pt ProducerTopics) Swap(i, j int)      { pt[i], pt[j] = pt[j], pt[i] }\nfunc (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt[j].Topic }\n\ntype Producer struct {\n\tRemoteAddresses  []string       `json:\"remote_addresses\"`\n\tRemoteAddress    string         `json:\"remote_address\"`\n\tHostname         string         `json:\"hostname\"`\n\tBroadcastAddress string         `json:\"broadcast_address\"`\n\tTCPPort          int            `json:\"tcp_port\"`\n\tHTTPPort         int            `json:\"http_port\"`\n\tVersion          string         `json:\"version\"`\n\tVersionObj       semver.Version `json:\"-\"`\n\tTopics           ProducerTopics `json:\"topics\"`\n\tOutOfDate        bool           `json:\"out_of_date\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses of ProducerTopics and VersionObj\nfunc (p *Producer) UnmarshalJSON(b []byte) error {\n\tvar r struct {\n\t\tRemoteAddress    string   `json:\"remote_address\"`\n\t\tHostname         string   `json:\"hostname\"`\n\t\tBroadcastAddress string   `json:\"broadcast_address\"`\n\t\tTCPPort          int      `json:\"tcp_port\"`\n\t\tHTTPPort         int      `json:\"http_port\"`\n\t\tVersion          string   `json:\"version\"`\n\t\tTopics           []string `json:\"topics\"`\n\t\tTombstoned       []bool   `json:\"tombstones\"`\n\t}\n\tif err := json.Unmarshal(b, &r); err != nil {\n\t\treturn err\n\t}\n\t*p = Producer{\n\t\tRemoteAddress:    r.RemoteAddress,\n\t\tHostname:         r.Hostname,\n\t\tBroadcastAddress: r.BroadcastAddress,\n\t\tTCPPort:          r.TCPPort,\n\t\tHTTPPort:         r.HTTPPort,\n\t\tVersion:          r.Version,\n\t}\n\tfor i, t := range r.Topics {\n\t\tp.Topics = append(p.Topics, ProducerTopic{Topic: t, Tombstoned: r.Tombstoned[i]})\n\t}\n\tversion, err := semver.Parse(p.Version)\n\tif err != nil {\n\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t}\n\tp.VersionObj = version\n\treturn nil\n}\n\nfunc (p *Producer) Address() string {\n\tif p.RemoteAddress == \"\" {\n\t\treturn \"N/A\"\n\t}\n\treturn p.RemoteAddress\n}\n\nfunc (p *Producer) HTTPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.HTTPPort))\n}\n\nfunc (p *Producer) TCPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.TCPPort))\n}\n\n// IsInconsistent checks for cases where an unexpected number of nsqd connections are\n// reporting the same information to nsqlookupd (ie: multiple instances are using the\n// same broadcast address), or cases where some nsqd are not reporting to all nsqlookupd.\nfunc (p *Producer) IsInconsistent(numLookupd int) bool {\n\treturn len(p.RemoteAddresses) != numLookupd\n}\n\n...\n// Path: internal/dirlock/dirlock.go\n//go:build !windows && !illumos\n// +build !windows,!illumos\n\npackage dirlock\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n)\n\ntype DirLock struct {\n\tdir string\n\tf   *os.File\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\tf, err := os.Open(l.dir)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl.f = f\n\terr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"cannot flock directory %s - %s (possibly in use by another instance of nsqd)\", l.dir, err)\n\t}\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\tdefer l.f.Close()\n\treturn syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)\n}\n\n// Path: internal/dirlock/dirlock_illumos.go\n//go:build illumos\n// +build illumos\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/dirlock/dirlock_windows.go\n//go:build windows\n// +build windows\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/http_api/api_request.go\npackage http_api\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// A custom http.Transport with support for deadline timeouts\nfunc NewDeadlineTransport(connectTimeout time.Duration, requestTimeout time.Duration) *http.Transport {\n\t// arbitrary values copied from http.DefaultTransport\n\ttransport := &http.Transport{\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout:   connectTimeout,\n\t\t\tKeepAlive: 30 * time.Second,\n\t\t\tDualStack: true,\n\t\t}).DialContext,\n\t\tResponseHeaderTimeout: requestTimeout,\n\t\tMaxIdleConns:          100,\n\t\tIdleConnTimeout:       90 * time.Second,\n\t\tTLSHandshakeTimeout:   10 * time.Second,\n\t}\n\treturn transport\n}\n\ntype Client struct {\n\tc *http.Client\n}\n\nfunc NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) *Client {\n\ttransport := NewDeadlineTransport(connectTimeout, requestTimeout)\n\ttransport.TLSClientConfig = tlsConfig\n\treturn &Client{\n\t\tc: &http.Client{\n\t\t\tTransport: transport,\n\t\t\tTimeout:   requestTimeout,\n\t\t},\n\t}\n}\n\n// GETV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) GETV1(endpoint string, v interface{}) error {\nretry:\n\treq, err := http.NewRequest(\"GET\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\terr = json.Unmarshal(body, &v)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// PostV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) POSTV1(endpoint string) error {\nretry:\n\treq, err := http.NewRequest(\"POST\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\n\treturn nil\n}\n\nfunc httpsEndpoint(endpoint string, body []byte) (string, error) {\n\tvar forbiddenResp struct {\n\t\tHTTPSPort int `json:\"https_port\"`\n\t}\n\terr := json.Unmarshal(body, &forbiddenResp)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thost, _, err := net.SplitHostPort(u.Host)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu.Scheme = \"https\"\n\tu.Host = net.JoinHostPort(host, strconv.Itoa(forbiddenResp.HTTPSPort))\n\treturn u.String(), nil\n}\n\n// Path: internal/http_api/api_response.go\npackage http_api\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/julienschmidt/httprouter\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Decorator func(APIHandler) APIHandler\n\ntype APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)\n\ntype Err struct {\n\tCode int\n\tText string\n}\n\nfunc (e Err) Error() string {\n\treturn e.Text\n}\n\nfunc PlainText(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tcode := 200\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tcode = err.(Err).Code\n\t\t\tdata = err.Error()\n\t\t}\n\t\tswitch d := data.(type) {\n\t\tcase string:\n\t\t\tw.WriteHeader(code)\n\t\t\tio.WriteString(w, d)\n\t\tcase []byte:\n\t\t\tw.WriteHeader(code)\n\t\t\tw.Write(d)\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"unknown response type %T\", data))\n\t\t}\n\t\treturn nil, nil\n\t}\n}\n\nfunc V1(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tRespondV1(w, err.(Err).Code, err)\n\t\t\treturn nil, nil\n\t\t}\n\t\tRespondV1(w, 200, data)\n\t\treturn nil, nil\n\t}\n}\n\nfunc RespondV1(w http.ResponseWriter, code int, data interface{}) {\n\tvar response []byte\n\tvar err error\n\tvar isJSON bool\n\n\tif code == 200 {\n\t\tswitch data := data.(type) {\n\t\tcase string:\n\t\t\tresponse = []byte(data)\n\t\tcase []byte:\n\t\t\tresponse = data\n\t\tcase nil:\n\t\t\tresponse = []byte{}\n\t\tdefault:\n\t\t\tisJSON = true\n\t\t\tresponse, err = json.Marshal(data)\n\t\t\tif err != nil {\n\t\t\t\tcode = 500\n\t\t\t\tdata = err\n\t\t\t}\n\t\t}\n\t}\n\n\tif code != 200 {\n\t\tisJSON = true\n\t\tresponse, _ = json.Marshal(struct {\n\t\t\tMessage string `json:\"message\"`\n\t\t}{fmt.Sprintf(\"%s\", data)})\n\t}\n\n\tif isJSON {\n\t\tw.Header().Set(\"Content-Type\", \"application/json; charset=utf-8\")\n\t}\n\tw.Header().Set(\"X-NSQ-Content-Type\", \"nsq; version=1.0\")\n\tw.WriteHeader(code)\n\tw.Write(response)\n}\n\nfunc Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {\n\tdecorated := f\n\tfor _, decorate := range ds {\n\t\tdecorated = decorate(decorated)\n\t}\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {\n\t\tdecorated(w, req, ps)\n\t}\n}\n\nfunc Log(logf lg.AppLogFunc) Decorator {\n\treturn func(f APIHandler) APIHandler {\n\t\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\tstart := time.Now()\n\t\t\tresponse, err := f(w, req, ps)\n\t\t\telapsed := time.Since(start)\n\t\t\tstatus := 200\n\t\t\tif e, ok := err.(Err); ok {\n\t\t\t\tstatus = e.Code\n\t\t\t}\n\t\t\tlogf(lg.INFO, \"%d %s %s (%s) %s\",\n\t\t\t\tstatus, req.Method, req.URL.RequestURI(), req.RemoteAddr, elapsed)\n\t\t\treturn response, err\n\t\t}\n\t}\n}\n\nfunc LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\treturn func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\t\tlogf(lg.ERROR, \"panic in HTTP handler - %s\", p)\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{500, \"INTERNAL_ERROR\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t}\n}\n\nfunc LogNotFoundHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{404, \"NOT_FOUND\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\nfunc LogMethodNotAllowedHandler(logf lg.AppLogFunc) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tDecorate(func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\treturn nil, Err{405, \"METHOD_NOT_ALLOWED\"}\n\t\t}, Log(logf), V1)(w, req, nil)\n\t})\n}\n\n// Path: internal/http_api/compress.go\n// Copyright 2013 The Gorilla Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n\n// copied from https://github.com/gorilla/handlers/blob/master/compress.go\n\npackage http_api\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"io\"\n\t\"net/http\"\n\t\"strings\"\n)\n\ntype compressResponseWriter struct {\n\tio.Writer\n\thttp.ResponseWriter\n\thttp.Hijacker\n}\n\nfunc (w *compressResponseWriter) Header() http.Header {\n\treturn w.ResponseWriter.Header()\n}\n\nfunc (w *compressResponseWriter) WriteHeader(c int) {\n\tw.ResponseWriter.Header().Del(\"Content-Length\")\n\tw.ResponseWriter.WriteHeader(c)\n}\n\nfunc (w *compressResponseWriter) Write(b []byte) (int, error) {\n\th := w.ResponseWriter.Header()\n\tif h.Get(\"Content-Type\") == \"\" {\n\t\th.Set(\"Content-Type\", http.DetectContentType(b))\n\t}\n\th.Del(\"Content-Length\")\n\treturn w.Writer.Write(b)\n}\n\n// CompressHandler gzip compresses HTTP responses for clients that support it\n// via the 'Accept-Encoding' header.\nfunc CompressHandler(h http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\tL:\n\t\tfor _, enc := range strings.Split(r.Header.Get(\"Accept-Encoding\"), \",\") {\n\t\t\tswitch strings.TrimSpace(enc) {\n\t\t\tcase \"gzip\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"gzip\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tgw := gzip.NewWriter(w)\n\t\t\t\tdefer gw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         gw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\tcase \"deflate\":\n\t\t\t\tw.Header().Set(\"Content-Encoding\", \"deflate\")\n\t\t\t\tw.Header().Add(\"Vary\", \"Accept-Encoding\")\n\n\t\t\t\tfw, _ := flate.NewWriter(w, flate.DefaultCompression)\n\t\t\t\tdefer fw.Close()\n\n\t\t\t\th, hok := w.(http.Hijacker)\n\t\t\t\tif !hok { /* w is not Hijacker... oh well... */\n\t\t\t\t\th = nil\n\t\t\t\t}\n\n\t\t\t\tw = &compressResponseWriter{\n\t\t\t\t\tWriter:         fw,\n\t\t\t\t\tResponseWriter: w,\n\t\t\t\t\tHijacker:       h,\n\t\t\t\t}\n\n\t\t\t\tbreak L\n\t\t\t}\n\t\t}\n\n\t\th.ServeHTTP(w, r)\n\t})\n}\n\n// Path: internal/http_api/http_server.go\npackage http_api\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype logWriter struct {\n\tlogf lg.AppLogFunc\n}\n\nfunc (l logWriter) Write(p []byte) (int, error) {\n\tl.logf(lg.WARN, \"%s\", string(p))\n\treturn len(p), nil\n}\n\nfunc Serve(listener net.Listener, handler http.Handler, proto string, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"%s: listening on %s\", proto, listener.Addr())\n\n\tserver := &http.Server{\n\t\tHandler:  handler,\n\t\tErrorLog: log.New(logWriter{logf}, \"\", 0),\n\t}\n\terr := server.Serve(listener)\n\t// theres no direct way to detect this error because it is not exposed\n\tif err != nil && !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\treturn fmt.Errorf(\"http.Serve() error - %s\", err)\n\t}\n\n\tlogf(lg.INFO, \"%s: closing %s\", proto, listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/http_api/req_params.go\npackage http_api\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n)\n\ntype ReqParams struct {\n\turl.Values\n\tBody []byte\n}\n\nfunc NewReqParams(req *http.Request) (*ReqParams, error) {\n\treqParams, err := url.ParseQuery(req.URL.RawQuery)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, err := io.ReadAll(req.Body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &ReqParams{reqParams, data}, nil\n}\n\nfunc (r *ReqParams) Get(key string) (string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn \"\", errors.New(\"key not in query params\")\n\t}\n\treturn v[0], nil\n}\n\nfunc (r *ReqParams) GetAll(key string) ([]string, error) {\n\tv, ok := r.Values[key]\n\tif !ok {\n\t\treturn nil, errors.New(\"key not in query params\")\n\t}\n\treturn v, nil\n}\n\n// Path: internal/version/binary.go\npackage version\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n)\n\nconst Binary = \"1.3.0\"\n\nfunc String(app string) string {\n\treturn fmt.Sprintf(\"%s v%s (built w/%s)\", app, Binary, runtime.Version())\n}\n\n// Path: internal/protocol/protocol.go\npackage protocol\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\t\"net\"\n)\n\ntype Client interface {\n\tClose() error\n}\n\n// Protocol describes the basic behavior of any protocol in the system\ntype Protocol interface {\n\tNewClient(net.Conn) Client\n\tIOLoop(Client) error\n}\n\n// SendResponse is a server side utility function to prefix data with a length header\n// and write to the supplied Writer\nfunc SendResponse(w io.Writer, data []byte) (int, error) {\n\terr := binary.Write(w, binary.BigEndian, int32(len(data)))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tn, err := w.Write(data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn (n + 4), nil\n}\n\n// SendFramedResponse is a server side utility function to prefix data with a length header\n// and frame header and write to the supplied Writer\nfunc SendFramedResponse(w io.Writer, frameType int32, data []byte) (int, error) {\n\tbeBuf := make([]byte, 4)\n\tsize := uint32(len(data)) + 4\n\n\tbinary.BigEndian.PutUint32(beBuf, size)\n\tn, err := w.Write(beBuf)\n\tif err != nil {\n\t\treturn n, err\n\t}\n\n\tbinary.BigEndian.PutUint32(beBuf, uint32(frameType))\n\tn, err = w.Write(beBuf)\n\tif err != nil {\n\t\treturn n + 4, err\n\t}\n\n\tn, err = w.Write(data)\n\treturn n + 8, err\n}\n\n// Path: internal/http_api/topic_channel_args.go\npackage http_api\n\nimport (\n\t\"errors\"\n\n\t\"github.com/nsqio/nsq/internal/protocol\"\n)\n\ntype getter interface {\n\tGet(key string) (string, error)\n}\n\nfunc GetTopicChannelArgs(rp getter) (string, string, error) {\n\ttopicName, err := rp.Get(\"topic\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_TOPIC\")\n\t}\n\n\tif !protocol.IsValidTopicName(topicName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_TOPIC\")\n\t}\n\n\tchannelName, err := rp.Get(\"channel\")\n\tif err != nil {\n\t\treturn \"\", \"\", errors.New(\"MISSING_ARG_CHANNEL\")\n\t}\n\n\tif !protocol.IsValidChannelName(channelName) {\n\t\treturn \"\", \"\", errors.New(\"INVALID_ARG_CHANNEL\")\n\t}\n\n\treturn topicName, channelName, nil\n}\n\n// Path: internal/lg/lg_test.go\npackage lg\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\ntype mockLogger struct {\n\tCount int\n}\n\nfunc (l *mockLogger) Output(maxdepth int, s string) error {\n\tl.Count++\n\treturn nil\n}\n\nfunc TestLogging(t *testing.T) {\n\tlogger := &mockLogger{}\n\n\t// Test only fatal get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, FATAL, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 1, logger.Count)\n\n\t// Test only warnings or higher get through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, WARN, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 3, logger.Count)\n\n\t// Test everything gets through\n\tlogger.Count = 0\n\tfor i := 1; i <= 5; i++ {\n\t\tLogf(logger, DEBUG, LogLevel(i), \"Test\")\n\t}\n\ttest.Equal(t, 5, logger.Count)\n}\n\n// Path: internal/pqueue/pqueue.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n)\n\ntype Item struct {\n\tValue    interface{}\n\tPriority int64\n\tIndex    int\n}\n\n// this is a priority queue as implemented by a min heap\n// ie. the 0th element is the *lowest* value\ntype PriorityQueue []*Item\n\nfunc New(capacity int) PriorityQueue {\n\treturn make(PriorityQueue, 0, capacity)\n}\n\nfunc (pq PriorityQueue) Len() int {\n\treturn len(pq)\n}\n\nfunc (pq PriorityQueue) Less(i, j int) bool {\n\treturn pq[i].Priority < pq[j].Priority\n}\n\nfunc (pq PriorityQueue) Swap(i, j int) {\n\tpq[i], pq[j] = pq[j], pq[i]\n\tpq[i].Index = i\n\tpq[j].Index = j\n}\n\nfunc (pq *PriorityQueue) Push(x interface{}) {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n+1 > c {\n\t\tnpq := make(PriorityQueue, n, c*2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\t*pq = (*pq)[0 : n+1]\n\titem := x.(*Item)\n\titem.Index = n\n\t(*pq)[n] = item\n}\n\nfunc (pq *PriorityQueue) Pop() interface{} {\n\tn := len(*pq)\n\tc := cap(*pq)\n\tif n < (c/2) && c > 25 {\n\t\tnpq := make(PriorityQueue, n, c/2)\n\t\tcopy(npq, *pq)\n\t\t*pq = npq\n\t}\n\titem := (*pq)[n-1]\n\titem.Index = -1\n\t*pq = (*pq)[0 : n-1]\n\treturn item\n}\n\nfunc (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) {\n\tif pq.Len() == 0 {\n\t\treturn nil, 0\n\t}\n\n\titem := (*pq)[0]\n\tif item.Priority > max {\n\t\treturn nil, item.Priority - max\n\t}\n\theap.Remove(pq, 0)\n\n\treturn item, 0\n}\n\n// Path: internal/pqueue/pqueue_test.go\npackage pqueue\n\nimport (\n\t\"container/heap\"\n\t\"math/rand\"\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"sort\"\n\t\"testing\"\n)\n\nfunc equal(t *testing.T, act, exp interface{}) {\n\tif !reflect.DeepEqual(exp, act) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\texp: %#v\\n\\n\\tgot: %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, exp, act)\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestPriorityQueue(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c+1; i++ {\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(i)})\n\t}\n\tequal(t, pq.Len(), c+1)\n\tequal(t, cap(pq), c*2)\n\n\tfor i := 0; i < c+1; i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, item.(*Item).Value.(int), i)\n\t}\n\tequal(t, cap(pq), c/4)\n}\n\nfunc TestUnsortedInsert(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\tints := make([]int, 0, c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\tints = append(ints, v)\n\t\theap.Push(&pq, &Item{Value: i, Priority: int64(v)})\n\t}\n\tequal(t, pq.Len(), c)\n\tequal(t, cap(pq), c)\n\n\tsort.Ints(ints)\n\n\tfor i := 0; i < c; i++ {\n\t\titem, _ := pq.PeekAndShift(int64(ints[len(ints)-1]))\n\t\tequal(t, item.Priority, int64(ints[i]))\n\t}\n}\n\nfunc TestRemove(t *testing.T) {\n\tc := 100\n\tpq := New(c)\n\n\tfor i := 0; i < c; i++ {\n\t\tv := rand.Int()\n\t\theap.Push(&pq, &Item{Value: \"test\", Priority: int64(v)})\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\theap.Remove(&pq, rand.Intn((c-1)-i))\n\t}\n\n\tlastPriority := heap.Pop(&pq).(*Item).Priority\n\tfor i := 0; i < (c - 10 - 1); i++ {\n\t\titem := heap.Pop(&pq)\n\t\tequal(t, lastPriority < item.(*Item).Priority, true)\n\t\tlastPriority = item.(*Item).Priority\n\t}\n}\n\n// Path: internal/protocol/byte_base10.go\npackage protocol\n\nimport (\n\t\"errors\"\n)\n\nvar errBase10 = errors.New(\"failed to convert to Base10\")\n\nfunc ByteToBase10(b []byte) (n uint64, err error) {\n\tbase := uint64(10)\n\n\tn = 0\n\tfor i := 0; i < len(b); i++ {\n\t\tvar v byte\n\t\td := b[i]\n\t\tswitch {\n\t\tcase '0' <= d && d <= '9':\n\t\t\tv = d - '0'\n\t\tdefault:\n\t\t\tn = 0\n\t\t\terr = errBase10\n\t\t\treturn\n\t\t}\n\t\tn *= base\n\t\tn += uint64(v)\n\t}\n\n\treturn n, err\n}\n\n// Path: internal/protocol/byte_base10_test.go\npackage protocol\n\nimport (\n\t\"testing\"\n)\n\nvar result uint64\n\nfunc BenchmarkByteToBase10Valid(b *testing.B) {\n\tbt := []byte{'3', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\nfunc BenchmarkByteToBase10Invalid(b *testing.B) {\n\tbt := []byte{'?', '1', '4', '1', '5', '9', '2', '5'}\n\tvar n uint64\n\tfor i := 0; i < b.N; i++ {\n\t\tn, _ = ByteToBase10(bt)\n\t}\n\tresult = n\n}\n\n// Path: internal/protocol/names.go\npackage protocol\n\nimport (\n\t\"regexp\"\n)\n\nvar validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral)?$`)\n\n// IsValidTopicName checks a topic name for correctness\nfunc IsValidTopicName(name string) bool {\n\treturn isValidName(name)\n}\n\n// IsValidChannelName checks a channel name for correctness\nfunc IsValidChannelName(name string) bool {\n\treturn isValidName(name)\n}\n\nfunc isValidName(name string) bool {\n\tif len(name) > 64 || len(name) < 1 {\n\t\treturn false\n\t}\n\treturn validTopicChannelNameRegex.MatchString(name)\n}\n\n// Path: internal/protocol/tcp_server.go\npackage protocol\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype TCPHandler interface {\n\tHandle(net.Conn)\n}\n\nfunc TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLogFunc) error {\n\tlogf(lg.INFO, \"TCP: listening on %s\", listener.Addr())\n\n\tvar wg sync.WaitGroup\n\n\tfor {\n\t\tclientConn, err := listener.Accept()\n\t\tif err != nil {\n\t\t\t// net.Error.Temporary() is deprecated, but is valid for accept\n\t\t\t// this is a hack to avoid a staticcheck error\n\t\t\tif te, ok := err.(interface{ Temporary() bool }); ok && te.Temporary() {\n\t\t\t\tlogf(lg.WARN, \"temporary Accept() failure - %s\", err)\n\t\t\t\truntime.Gosched()\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// theres no direct way to detect this error because it is not exposed\n\t\t\tif !strings.Contains(err.Error(), \"use of closed network connection\") {\n\t\t\t\treturn fmt.Errorf(\"listener.Accept() error - %s\", err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\thandler.Handle(clientConn)\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\t// wait to return until all handler goroutines complete\n\twg.Wait()\n\n\tlogf(lg.INFO, \"TCP: closing %s\", listener.Addr())\n\n\treturn nil\n}\n\n// Path: internal/quantile/aggregate.go\npackage quantile\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"sort\"\n)\n\ntype E2eProcessingLatencyAggregate struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\tTopic       string               `json:\"topic\"`\n\tChannel     string               `json:\"channel\"`\n\tAddr        string               `json:\"host\"`\n}\n\nfunc (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {\n\tvar resp struct {\n\t\tCount       int                  `json:\"count\"`\n\t\tPercentiles []map[string]float64 `json:\"percentiles\"`\n\t\tTopic       string               `json:\"topic\"`\n\t\tChannel     string               `json:\"channel\"`\n\t\tAddr        string               `json:\"host\"`\n\t}\n\terr := json.Unmarshal(b, &resp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, p := range resp.Percentiles {\n\t\tp[\"min\"] = p[\"value\"]\n\t\tp[\"max\"] = p[\"value\"]\n\t\tp[\"average\"] = p[\"value\"]\n\t\tp[\"count\"] = float64(resp.Count)\n\t}\n\n\te.Count = resp.Count\n\te.Percentiles = resp.Percentiles\n\te.Topic = resp.Topic\n\te.Channel = resp.Channel\n\te.Addr = resp.Addr\n\n\treturn nil\n}\n\nfunc (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) }\nfunc (e *E2eProcessingLatencyAggregate) Swap(i, j int) {\n\te.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i]\n}\nfunc (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {\n\treturn e.Percentiles[i][\"percentile\"] > e.Percentiles[j][\"percentile\"]\n}\n\n// Add merges e2 into e by averaging the percentiles\nfunc (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) {\n\te.Addr = \"*\"\n\tp := e.Percentiles\n\te.Count += e2.Count\n\tfor _, value := range e2.Percentiles {\n\t\ti := -1\n\t\tfor j, v := range p {\n\t\t\tif value[\"quantile\"] == v[\"quantile\"] {\n\t\t\t\ti = j\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif i == -1 {\n\t\t\ti = len(p)\n\t\t\te.Percentiles = append(p, make(map[string]float64))\n\t\t\tp = e.Percentiles\n\t\t\tp[i][\"quantile\"] = value[\"quantile\"]\n\t\t}\n\t\tp[i][\"max\"] = math.Max(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"min\"] = math.Min(value[\"max\"], p[i][\"max\"])\n\t\tp[i][\"count\"] += value[\"count\"]\n\t\tif p[i][\"count\"] == 0 {\n\t\t\tp[i][\"average\"] = 0\n\t\t\tcontinue\n\t\t}\n\t\tdelta := value[\"average\"] - p[i][\"average\"]\n\t\tR := delta * value[\"count\"] / p[i][\"count\"]\n\t\tp[i][\"average\"] = p[i][\"average\"] + R\n\t}\n\tsort.Sort(e)\n}\n\n// Path: internal/statsd/host.go\npackage statsd\n\nimport (\n\t\"strings\"\n)\n\nfunc HostKey(h string) string {\n\treturn strings.Replace(strings.Replace(h, \".\", \"_\", -1), \":\", \"_\", -1)\n}\n\n// Path: internal/stringy/slice.go\npackage stringy\n\nfunc Add(s []string, a string) []string {\n\tfor _, existing := range s {\n\t\tif a == existing {\n\t\t\treturn s\n\t\t}\n\t}\n\treturn append(s, a)\n\n}\n\nfunc Union(s []string, a []string) []string {\n\tfor _, entry := range a {\n\t\tfound := false\n\t\tfor _, existing := range s {\n\t\t\tif entry == existing {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\ts = append(s, entry)\n\t\t}\n\t}\n\treturn s\n}\n\nfunc Uniq(s []string) (r []string) {\nouterLoop:\n\tfor _, entry := range s {\n\t\tfor _, existing := range r {\n\t\t\tif existing == entry {\n\t\t\t\tcontinue outerLoop\n\t\t\t}\n\t\t}\n\t\tr = append(r, entry)\n\t}\n\treturn\n}\n\n// Path: internal/stringy/slice_test.go\npackage stringy_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\nfunc BenchmarkUniq(b *testing.B) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"a\", \"b\"}\n\tfor i := 0; i < b.N; i++ {\n\t\tvalues = stringy.Uniq(values)\n\t\tif len(values) != 2 {\n\t\t\tb.Fatal(\"values len is incorrect\")\n\t\t}\n\t}\n}\n\nfunc TestUniq(t *testing.T) {\n\tvalues := []string{\"a\", \"a\", \"a\", \"b\", \"b\", \"b\", \"c\", \"c\", \"c\"}\n\tvalues = stringy.Uniq(values)\n\tif len(values) != 3 {\n\t\tt.Fatal(\"values len is incorrect\")\n\t}\n}\n\n// Path: internal/stringy/template.go\npackage stringy\n\nimport (\n\t\"fmt\"\n)\n\nfunc NanoSecondToHuman(v float64) string {\n\tvar suffix string\n\tswitch {\n\tcase v > 1000000000:\n\t\tv /= 1000000000\n\t\tsuffix = \"s\"\n\tcase v > 1000000:\n\t\tv /= 1000000\n\t\tsuffix = \"ms\"\n\tcase v > 1000:\n\t\tv /= 1000\n\t\tsuffix = \"us\"\n\tdefault:\n\t\tsuffix = \"ns\"\n\t}\n\treturn fmt.Sprintf(\"%0.1f%s\", v, suffix)\n}\n\n// Path: internal/test/assertions.go\npackage test\n\nimport (\n\t\"path/filepath\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc Equal(t *testing.T, expected, actual interface{}) {\n\tif !reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   %#v (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotEqual(t *testing.T, expected, actual interface{}) {\n\tif reflect.DeepEqual(expected, actual) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tnexp: %#v\\n\\n\\tgot:  %#v\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, expected, actual)\n\t\tt.FailNow()\n\t}\n}\n\nfunc Nil(t *testing.T, object interface{}) {\n\tif !isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\t   <nil> (expected)\\n\\n\\t!= %#v (actual)\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line, object)\n\t\tt.FailNow()\n\t}\n}\n\nfunc NotNil(t *testing.T, object interface{}) {\n\tif isNil(object) {\n\t\t_, file, line, _ := runtime.Caller(1)\n\t\tt.Logf(\"\\033[31m%s:%d:\\n\\n\\tExpected value not to be <nil>\\033[39m\\n\\n\",\n\t\t\tfilepath.Base(file), line)\n\t\tt.FailNow()\n\t}\n}\n\nfunc isNil(object interface{}) bool {\n\tif object == nil {\n\t\treturn true\n\t}\n\n\tvalue := reflect.ValueOf(object)\n\tkind := value.Kind()\n\tif kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// Path: internal/test/fakes.go\npackage test\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\ntype FakeNetConn struct {\n\tReadFunc             func([]byte) (int, error)\n\tWriteFunc            func([]byte) (int, error)\n\tCloseFunc            func() error\n\tLocalAddrFunc        func() net.Addr\n\tRemoteAddrFunc       func() net.Addr\n\tSetDeadlineFunc      func(time.Time) error\n\tSetReadDeadlineFunc  func(time.Time) error\n\tSetWriteDeadlineFunc func(time.Time) error\n}\n\nfunc (f FakeNetConn) Read(b []byte) (int, error)         { return f.ReadFunc(b) }\nfunc (f FakeNetConn) Write(b []byte) (int, error)        { return f.WriteFunc(b) }\nfunc (f FakeNetConn) Close() error                       { return f.CloseFunc() }\nfunc (f FakeNetConn) LocalAddr() net.Addr                { return f.LocalAddrFunc() }\nfunc (f FakeNetConn) RemoteAddr() net.Addr               { return f.RemoteAddrFunc() }\nfunc (f FakeNetConn) SetDeadline(t time.Time) error      { return f.SetDeadlineFunc(t) }\nfunc (f FakeNetConn) SetReadDeadline(t time.Time) error  { return f.SetReadDeadlineFunc(t) }\nfunc (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) }\n\ntype fakeNetAddr struct{}\n\nfunc (fakeNetAddr) Network() string { return \"\" }\nfunc (fakeNetAddr) String() string  { return \"\" }\n\nfunc NewFakeNetConn() FakeNetConn {\n\tnetAddr := fakeNetAddr{}\n\treturn FakeNetConn{\n\t\tReadFunc:             func(b []byte) (int, error) { return 0, nil },\n\t\tWriteFunc:            func(b []byte) (int, error) { return len(b), nil },\n\t\tCloseFunc:            func() error { return nil },\n\t\tLocalAddrFunc:        func() net.Addr { return netAddr },\n\t\tRemoteAddrFunc:       func() net.Addr { return netAddr },\n\t\tSetDeadlineFunc:      func(time.Time) error { return nil },\n\t\tSetWriteDeadlineFunc: func(time.Time) error { return nil },\n\t\tSetReadDeadlineFunc:  func(time.Time) error { return nil },\n\t}\n}\n\n// Path: internal/util/unix_socket.go\npackage util\n\nimport (\n\t\"net\"\n)\n\nfunc TypeOfAddr(addr string) string {\n\tif _, _, err := net.SplitHostPort(addr); err == nil {\n\t\treturn \"tcp\"\n\t}\n\treturn \"unix\"\n}\n\n// Path: internal/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc BenchmarkUniqRands5of5(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(5, 5)\n\t}\n}\n\nfunc BenchmarkUniqRands20of20(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 20)\n\t}\n}\n\nfunc BenchmarkUniqRands20of50(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 50)\n\t}\n}\n\nfunc TestUniqRands(t *testing.T) {\n\tvar x []int\n\tx = UniqRands(3, 10)\n\ttest.Equal(t, 3, len(x))\n\n\tx = UniqRands(10, 5)\n\ttest.Equal(t, 5, len(x))\n\n\tx = UniqRands(10, 20)\n\ttest.Equal(t, 10, len(x))\n}\n\nfunc TestTypeOfAddr(t *testing.T) {\n\tvar x string\n\tx = TypeOfAddr(\"127.0.0.1:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"test:80\")\n\ttest.Equal(t, \"tcp\", x)\n\n\tx = TypeOfAddr(\"/var/run/nsqd.sock\")\n\ttest.Equal(t, \"unix\", x)\n\n\tx = TypeOfAddr(\"[::1%lo0]:80\")\n\ttest.Equal(t, \"tcp\", x)\n}\n\n// Path: internal/util/wait_group_wrapper.go\npackage util\n\nimport (\n\t\"sync\"\n)\n\ntype WaitGroupWrapper struct {\n\tsync.WaitGroup\n}\n\nfunc (w *WaitGroupWrapper) Wrap(cb func()) {\n\tw.Add(1)\n\tgo func() {\n\t\tcb()\n\t\tw.Done()\n\t}()\n}\n\n// Path: internal/writers/boundary_buffered_writer.go\npackage writers\n\nimport (\n\t\"bufio\"\n\t\"io\"\n)\n\ntype BoundaryBufferedWriter struct {\n\tbw *bufio.Writer\n}\n\nfunc NewBoundaryBufferedWriter(w io.Writer, size int) *BoundaryBufferedWriter {\n\treturn &BoundaryBufferedWriter{\n\t\tbw: bufio.NewWriterSize(w, size),\n\t}\n}\n\nfunc (b *BoundaryBufferedWriter) Write(p []byte) (int, error) {\n\tif len(p) > b.bw.Available() {\n\t\terr := b.bw.Flush()\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t}\n\treturn b.bw.Write(p)\n}\n\nfunc (b *BoundaryBufferedWriter) Flush() error {\n\treturn b.bw.Flush()\n}\n\n// Path: internal/writers/spread_writer.go\npackage writers\n\nimport (\n\t\"io\"\n\t\"time\"\n)\n\ntype SpreadWriter struct {\n\tw        io.Writer\n\tinterval time.Duration\n\tbuf      [][]byte\n\texitCh   chan int\n}\n\nfunc NewSpreadWriter(w io.Writer, interval time.Duration, exitCh chan int) *SpreadWriter {\n\treturn &SpreadWriter{\n\t\tw:        w,\n\t\tinterval: interval,\n\t\tbuf:      make([][]byte, 0),\n\t\texitCh:   exitCh,\n\t}\n}\n\nfunc (s *SpreadWriter) Write(p []byte) (int, error) {\n\tb := make([]byte, len(p))\n\tcopy(b, p)\n\ts.buf = append(s.buf, b)\n\treturn len(p), nil\n}\n\nfunc (s *SpreadWriter) Flush() {\n\tif len(s.buf) == 0 {\n\t\t// nothing to write, just wait\n\t\tselect {\n\t\tcase <-time.After(s.interval):\n\t\tcase <-s.exitCh:\n\t\t}\n\t\treturn\n\t}\n\tsleep := s.interval / time.Duration(len(s.buf))\n\tticker := time.NewTicker(sleep)\n\tfor _, b := range s.buf {\n\t\ts.w.Write(b)\n\t\tselect {\n\t\tcase <-ticker.C:\n\t\tcase <-s.exitCh: // skip sleeps finish writes\n\t\t}\n\t}\n\tticker.Stop()\n\ts.buf = s.buf[:0]\n}\n\n", "needle_token_start": 12826, "needle_token_end": 12883, "code_context_ntokens": 13962, "output": ["```go\n// Path: internal/util/util_test.go\npackage util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/nsqio/nsq/internal/test\"\n)\n\nfunc BenchmarkUniqRands5of5(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(5, 5)\n\t}\n}\n\nfunc BenchmarkUniqRands20of20(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 20)\n\t}\n}\n\nfunc BenchmarkUniqRands20of50(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tUniqRands(20, 50)\n\t}\n}\n\nfunc TestUniqRands(t *testing.T) {\n\tvar x []int\n\tx = UniqRands(3, 10)\n\ttest.Equal(t, 3, len(x))\n\n\tx = UniqRands(10, 5)\n\ttest.Equal(t, 5, len(x))\n\n\tx = UniqRands(10, 20)\n\ttest.Equal(t, 10, len(x))\n}\n```"]}
{"repo": "nsqio/nsq", "name": "V1", "language": "go", "path": "internal/http_api/api_response.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to standardize the HTTP response format for API endpoints, ensuring that all responses adhere to a specific version of the API's response structure.\n2. **Input**: It takes a handler function that processes the HTTP request and returns either data or an error.\n3. **Output**: The output is a standardized HTTP response sent to the client, which could be a success or error message formatted according to the API's version 1 specification.\n4. **Procedure**: \n   - The function invokes the provided handler function with the HTTP request details.\n   - If the handler function returns an error, the function formats this error using a specific response function tailored for errors, setting the appropriate HTTP status code.\n   - If the handler function succeeds, it formats the success data using the same specific response function, setting a successful HTTP status code.\n   - In both cases, the response is written directly to the HTTP response writer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: internal/clusterinfo/data.go\npackage clusterinfo\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/http_api\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype PartialErr interface {\n\terror\n\tErrors() []error\n}\n\ntype ErrList []error\n\nfunc (l ErrList) Error() string {\n\tvar es []string\n\tfor _, e := range l {\n\t\tes = append(es, e.Error())\n\t}\n\treturn strings.Join(es, \"\\n\")\n}\n\nfunc (l ErrList) Errors() []error {\n\treturn l\n}\n\ntype ClusterInfo struct {\n\tlog    lg.AppLogFunc\n\tclient *http_api.Client\n}\n\nfunc New(log lg.AppLogFunc, client *http_api.Client) *ClusterInfo {\n\treturn &ClusterInfo{\n\t\tlog:    log,\n\t\tclient: client,\n\t}\n}\n\nfunc (c *ClusterInfo) logf(f string, args ...interface{}) {\n\tif c.log != nil {\n\t\tc.log(lg.INFO, f, args...)\n\t}\n}\n\n// GetVersion returns a semver.Version object by querying /info\nfunc (c *ClusterInfo) GetVersion(addr string) (semver.Version, error) {\n\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\tvar resp struct {\n\t\tVersion string `json:\"version\"`\n\t}\n\terr := c.client.GETV1(endpoint, &resp)\n\tif err != nil {\n\t\treturn semver.Version{}, err\n\t}\n\tif resp.Version == \"\" {\n\t\tresp.Version = \"unknown\"\n\t}\n\treturn semver.Parse(resp.Version)\n}\n\n// GetLookupdTopics returns a []string containing a union of all the topics\n// from all the given nsqlookupd\nfunc (c *ClusterInfo) GetLookupdTopics(lookupdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []string `json:\"topics\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/topics\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\ttopics = append(topics, resp.Topics...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\ttopics = stringy.Uniq(topics)\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetLookupdTopicChannels returns a []string containing a union of all the channels\n// from all the given lookupd for the given topic\nfunc (c *ClusterInfo) GetLookupdTopicChannels(topic string, lookupdHTTPAddrs []string) ([]string, error) {\n\tvar channels []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tChannels []string `json:\"channels\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/channels?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tchannels = append(channels, resp.Channels...)\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tchannels = stringy.Uniq(channels)\n\tsort.Strings(channels)\n\n\tif len(errs) > 0 {\n\t\treturn channels, ErrList(errs)\n\t}\n\treturn channels, nil\n}\n\n// GetLookupdProducers returns Producers of all the nsqd connected to the given lookupds\nfunc (c *ClusterInfo) GetLookupdProducers(lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers []*Producer\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\tproducersByAddr := make(map[string]*Producer)\n\tmaxVersion, _ := semver.Parse(\"0.0.0\")\n\n\ttype respType struct {\n\t\tProducers []*Producer `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/nodes\", addr)\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, producer := range resp.Producers {\n\t\t\t\tkey := producer.TCPAddress()\n\t\t\t\tp, ok := producersByAddr[key]\n\t\t\t\tif !ok {\n\t\t\t\t\tproducersByAddr[key] = producer\n\t\t\t\t\tproducers = append(producers, producer)\n\t\t\t\t\tif maxVersion.LT(producer.VersionObj) {\n\t\t\t\t\t\tmaxVersion = producer.VersionObj\n\t\t\t\t\t}\n\t\t\t\t\tsort.Sort(producer.Topics)\n\t\t\t\t\tp = producer\n\t\t\t\t}\n\t\t\t\tp.RemoteAddresses = append(p.RemoteAddresses,\n\t\t\t\t\tfmt.Sprintf(\"%s/%s\", addr, producer.Address()))\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\n\tfor _, producer := range producersByAddr {\n\t\tif producer.VersionObj.LT(maxVersion) {\n\t\t\tproducer.OutOfDate = true\n\t\t}\n\t}\n\tsort.Sort(ProducersByHost{producers})\n\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetLookupdTopicProducers returns Producers of all the nsqd for a given topic by\n// unioning the nodes returned from the given lookupd\nfunc (c *ClusterInfo) GetLookupdTopicProducers(topic string, lookupdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tProducers Producers `json:\"producers\"`\n\t}\n\n\tfor _, addr := range lookupdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/lookup?topic=%s\", addr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqlookupd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, p := range resp.Producers {\n\t\t\t\tfor _, pp := range producers {\n\t\t\t\t\tif p.HTTPAddress() == pp.HTTPAddress() {\n\t\t\t\t\t\tgoto skip\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tproducers = append(producers, p)\n\t\t\tskip:\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(lookupdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqlookupd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDTopics returns a []string containing all the topics produced by the given nsqd\nfunc (c *ClusterInfo) GetNSQDTopics(nsqdHTTPAddrs []string) ([]string, error) {\n\tvar topics []string\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype respType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\ttopics = stringy.Add(topics, topic.Name)\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\n\tsort.Strings(topics)\n\n\tif len(errs) > 0 {\n\t\treturn topics, ErrList(errs)\n\t}\n\treturn topics, nil\n}\n\n// GetNSQDProducers returns Producers of all the given nsqd\nfunc (c *ClusterInfo) GetNSQDProducers(nsqdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype infoRespType struct {\n\t\tVersion          string `json:\"version\"`\n\t\tBroadcastAddress string `json:\"broadcast_address\"`\n\t\tHostname         string `json:\"hostname\"`\n\t\tHTTPPort         int    `json:\"http_port\"`\n\t\tTCPPort          int    `json:\"tcp_port\"`\n\t}\n\n\ttype statsRespType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar infoResp infoRespType\n\t\t\terr := c.client.GETV1(endpoint, &infoResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tendpoint = fmt.Sprintf(\"http://%s/stats?format=json&include_clients=false\", addr)\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar statsResp statsRespType\n\t\t\terr = c.client.GETV1(endpoint, &statsResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tvar producerTopics ProducerTopics\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tproducerTopics = append(producerTopics, ProducerTopic{Topic: t.Name})\n\t\t\t}\n\n\t\t\tversion, err := semver.Parse(infoResp.Version)\n\t\t\tif err != nil {\n\t\t\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tproducers = append(producers, &Producer{\n\t\t\t\tVersion:          infoResp.Version,\n\t\t\t\tVersionObj:       version,\n\t\t\t\tBroadcastAddress: infoResp.BroadcastAddress,\n\t\t\t\tHostname:         infoResp.Hostname,\n\t\t\t\tHTTPPort:         infoResp.HTTPPort,\n\t\t\t\tTCPPort:          infoResp.TCPPort,\n\t\t\t\tTopics:           producerTopics,\n\t\t\t})\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDTopicProducers returns Producers containing the addresses of all the nsqd\n// that produce the given topic\nfunc (c *ClusterInfo) GetNSQDTopicProducers(topic string, nsqdHTTPAddrs []string) (Producers, error) {\n\tvar producers Producers\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar errs []error\n\n\ttype infoRespType struct {\n\t\tVersion          string `json:\"version\"`\n\t\tBroadcastAddress string `json:\"broadcast_address\"`\n\t\tHostname         string `json:\"hostname\"`\n\t\tHTTPPort         int    `json:\"http_port\"`\n\t\tTCPPort          int    `json:\"tcp_port\"`\n\t}\n\n\ttype statsRespType struct {\n\t\tTopics []struct {\n\t\t\tName string `json:\"topic_name\"`\n\t\t} `json:\"topics\"`\n\t}\n\n\tfor _, addr := range nsqdHTTPAddrs {\n\t\twg.Add(1)\n\t\tgo func(addr string) {\n\t\t\tdefer wg.Done()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json&topic=%s&include_clients=false\",\n\t\t\t\taddr, url.QueryEscape(topic))\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar statsResp statsRespType\n\t\t\terr := c.client.GETV1(endpoint, &statsResp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tvar producerTopics ProducerTopics\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tproducerTopics = append(producerTopics, ProducerTopic{Topic: t.Name})\n\t\t\t}\n\n\t\t\tfor _, t := range statsResp.Topics {\n\t\t\t\tif t.Name == topic {\n\t\t\t\t\tendpoint := fmt.Sprintf(\"http://%s/info\", addr)\n\t\t\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\t\t\tvar infoResp infoRespType\n\t\t\t\t\terr := c.client.GETV1(endpoint, &infoResp)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlock.Lock()\n\t\t\t\t\t\terrs = append(errs, err)\n\t\t\t\t\t\tlock.Unlock()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\tversion, err := semver.Parse(infoResp.Version)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t\t\t\t\t}\n\n\t\t\t\t\t// if BroadcastAddress/HTTPPort are missing, use the values from `addr` for\n\t\t\t\t\t// backwards compatibility\n\n\t\t\t\t\tif infoResp.BroadcastAddress == \"\" {\n\t\t\t\t\t\tvar p string\n\t\t\t\t\t\tinfoResp.BroadcastAddress, p, _ = net.SplitHostPort(addr)\n\t\t\t\t\t\tinfoResp.HTTPPort, _ = strconv.Atoi(p)\n\t\t\t\t\t}\n\t\t\t\t\tif infoResp.Hostname == \"\" {\n\t\t\t\t\t\tinfoResp.Hostname, _, _ = net.SplitHostPort(addr)\n\t\t\t\t\t}\n\n\t\t\t\t\tlock.Lock()\n\t\t\t\t\tproducers = append(producers, &Producer{\n\t\t\t\t\t\tVersion:          infoResp.Version,\n\t\t\t\t\t\tVersionObj:       version,\n\t\t\t\t\t\tBroadcastAddress: infoResp.BroadcastAddress,\n\t\t\t\t\t\tHostname:         infoResp.Hostname,\n\t\t\t\t\t\tHTTPPort:         infoResp.HTTPPort,\n\t\t\t\t\t\tTCPPort:          infoResp.TCPPort,\n\t\t\t\t\t\tTopics:           producerTopics,\n\t\t\t\t\t})\n\t\t\t\t\tlock.Unlock()\n\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}(addr)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(nsqdHTTPAddrs) {\n\t\treturn nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\tif len(errs) > 0 {\n\t\treturn producers, ErrList(errs)\n\t}\n\treturn producers, nil\n}\n\n// GetNSQDStats returns aggregate topic and channel stats from the given Producers\n//\n// if selectedChannel is empty, this will return stats for topic/channel\n// if selectedTopic is empty, this will return stats for *all* topic/channels\n// if includeClients is false, this will *not* return client stats for channels\n// and the ChannelStats dict will be keyed by topic + ':' + channel\nfunc (c *ClusterInfo) GetNSQDStats(producers Producers,\n\tselectedTopic string, selectedChannel string,\n\tincludeClients bool) ([]*TopicStats, map[string]*ChannelStats, error) {\n\tvar lock sync.Mutex\n\tvar wg sync.WaitGroup\n\tvar topicStatsList TopicStatsList\n\tvar errs []error\n\n\tchannelStatsMap := make(map[string]*ChannelStats)\n\n\ttype respType struct {\n\t\tTopics []*TopicStats `json:\"topics\"`\n\t}\n\n\tfor _, p := range producers {\n\t\twg.Add(1)\n\t\tgo func(p *Producer) {\n\t\t\tdefer wg.Done()\n\n\t\t\taddr := p.HTTPAddress()\n\n\t\t\tendpoint := fmt.Sprintf(\"http://%s/stats?format=json\", addr)\n\t\t\tif selectedTopic != \"\" {\n\t\t\t\tendpoint += \"&topic=\" + url.QueryEscape(selectedTopic)\n\t\t\t\tif selectedChannel != \"\" {\n\t\t\t\t\tendpoint += \"&channel=\" + url.QueryEscape(selectedChannel)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !includeClients {\n\t\t\t\tendpoint += \"&include_clients=false\"\n\t\t\t}\n\n\t\t\tc.logf(\"CI: querying nsqd %s\", endpoint)\n\n\t\t\tvar resp respType\n\t\t\terr := c.client.GETV1(endpoint, &resp)\n\t\t\tif err != nil {\n\t\t\t\tlock.Lock()\n\t\t\t\terrs = append(errs, err)\n\t\t\t\tlock.Unlock()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlock.Lock()\n\t\t\tdefer lock.Unlock()\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\ttopic.Node = addr\n\t\t\t\ttopic.Hostname = p.Hostname\n\t\t\t\ttopic.MemoryDepth = topic.Depth - topic.BackendDepth\n\t\t\t\tif selectedTopic != \"\" && topic.TopicName != selectedTopic {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\ttopicStatsList = append(topicStatsList, topic)\n\n\t\t\t\tfor _, channel := range topic.Channels {\n\t\t\t\t\tchannel.Node = addr\n\t\t\t\t\tchannel.Hostname = p.Hostname\n\t\t\t\t\tchannel.TopicName = topic.TopicName\n\t\t\t\t\tchannel.MemoryDepth = channel.Depth - channel.BackendDepth\n\t\t\t\t\tkey := channel.ChannelName\n\t\t\t\t\tif selectedTopic == \"\" {\n\t\t\t\t\t\tkey = fmt.Sprintf(\"%s:%s\", topic.TopicName, channel.ChannelName)\n\t\t\t\t\t}\n\t\t\t\t\tchannelStats, ok := channelStatsMap[key]\n\t\t\t\t\tif !ok {\n\t\t\t\t\t\tchannelStats = &ChannelStats{\n\t\t\t\t\t\t\tNode:        addr,\n\t\t\t\t\t\t\tTopicName:   topic.TopicName,\n\t\t\t\t\t\t\tChannelName: channel.ChannelName,\n\t\t\t\t\t\t}\n\t\t\t\t\t\tchannelStatsMap[key] = channelStats\n\t\t\t\t\t}\n\t\t\t\t\tfor _, c := range channel.Clients {\n\t\t\t\t\t\tc.Node = addr\n\t\t\t\t\t}\n\t\t\t\t\tchannelStats.Add(channel)\n\t\t\t\t}\n\t\t\t}\n\t\t}(p)\n\t}\n\twg.Wait()\n\n\tif len(errs) == len(producers) {\n\t\treturn nil, nil, fmt.Errorf(\"failed to query any nsqd: %s\", ErrList(errs))\n\t}\n\n\tsort.Sort(TopicStatsByHost{topicStatsList})\n\n\tif len(errs) > 0 {\n\t\treturn topicStatsList, channelStatsMap, ErrList(errs)\n\t}\n\treturn topicStatsList, channelStatsMap, nil\n}\n\n// TombstoneNodeForTopic tombstones the given node for the given topic on all the given nsqlookupd\n// and deletes the topic from the node\nfunc (c *ClusterInfo) TombstoneNodeForTopic(topic string, node string, lookupdHTTPAddrs []string) error {\n\tvar errs []error\n\n\t// tombstone the topic on all the lookupds\n\tqs := fmt.Sprintf(\"topic=%s&node=%s\", url.QueryEscape(topic), url.QueryEscape(node))\n\terr := c.nsqlookupdPOST(lookupdHTTPAddrs, \"topic/tombstone\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tproducers, err := c.GetNSQDProducers([]string{node})\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\t// delete the topic on the producer\n\tqs = fmt.Sprintf(\"topic=%s\", url.QueryEscape(topic))\n\terr = c.producersPOST(producers, \"topic/delete\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tif len(errs) > 0 {\n\t\treturn ErrList(errs)\n\t}\n\treturn nil\n}\n\nfunc (c *ClusterInfo) CreateTopicChannel(topicName string, channelName string, lookupdHTTPAddrs []string) error {\n\tvar errs []error\n\n\t// create the topic on all the nsqlookupd\n\tqs := fmt.Sprintf(\"topic=%s\", url.QueryEscape(topicName))\n\terr := c.nsqlookupdPOST(lookupdHTTPAddrs, \"topic/create\", qs)\n\tif err != nil {\n\t\tpe, ok := err.(PartialErr)\n\t\tif !ok {\n\t\t\treturn err\n\t\t}\n\t\terrs = append(errs, pe.Errors()...)\n\t}\n\n\tif len(channelName) > 0 {\n\t\tqs := fmt.Sprintf(\"topic=%s&channel=%s\", url.QueryEscape(topicName), url.QueryEscape(channelName))\n\n\t\t// create the channel on all the nsqlookupd\n\t\terr := c.nsqlookupdPOST(lookupdHTTPAddrs, \"channel/create\", qs)\n\t\tif err != nil {\n...\n// Path: internal/clusterinfo/producer_test.go\npackage clusterinfo\n\nimport \"testing\"\n\nfunc TestHostNameAddresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"host.domain.com\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"host.domain.com:4151\" {\n\t\tt.Errorf(\"Incorrect HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"host.domain.com:4150\" {\n\t\tt.Errorf(\"Incorrect TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv4Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"192.168.1.17\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\n\tif p.HTTPAddress() != \"192.168.1.17:4151\" {\n\t\tt.Errorf(\"Incorrect IPv4 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"192.168.1.17:4150\" {\n\t\tt.Errorf(\"Incorrect IPv4 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\nfunc TestIPv6Addresses(t *testing.T) {\n\tp := &Producer{\n\t\tBroadcastAddress: \"fd4a:622f:d2f2::1\",\n\t\tTCPPort:          4150,\n\t\tHTTPPort:         4151,\n\t}\n\tif p.HTTPAddress() != \"[fd4a:622f:d2f2::1]:4151\" {\n\t\tt.Errorf(\"Incorrect IPv6 HTTPAddress: %s\", p.HTTPAddress())\n\t}\n\tif p.TCPAddress() != \"[fd4a:622f:d2f2::1]:4150\" {\n\t\tt.Errorf(\"Incorrect IPv6 TCPAddress: %s\", p.TCPAddress())\n\t}\n}\n\n// Path: internal/quantile/quantile.go\npackage quantile\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/bmizerany/perks/quantile\"\n\t\"github.com/nsqio/nsq/internal/stringy\"\n)\n\ntype Result struct {\n\tCount       int                  `json:\"count\"`\n\tPercentiles []map[string]float64 `json:\"percentiles\"`\n}\n\nfunc (r *Result) String() string {\n\tvar s []string\n\tfor _, item := range r.Percentiles {\n\t\ts = append(s, stringy.NanoSecondToHuman(item[\"value\"]))\n\t}\n\treturn strings.Join(s, \", \")\n}\n\ntype Quantile struct {\n\tsync.Mutex\n\tstreams        [2]quantile.Stream\n\tcurrentIndex   uint8\n\tlastMoveWindow time.Time\n\tcurrentStream  *quantile.Stream\n\n\tPercentiles    []float64\n\tMoveWindowTime time.Duration\n}\n\nfunc New(WindowTime time.Duration, Percentiles []float64) *Quantile {\n\tq := Quantile{\n\t\tcurrentIndex:   0,\n\t\tlastMoveWindow: time.Now(),\n\t\tMoveWindowTime: WindowTime / 2,\n\t\tPercentiles:    Percentiles,\n\t}\n\tfor i := 0; i < 2; i++ {\n\t\tq.streams[i] = *quantile.NewTargeted(Percentiles...)\n\t}\n\tq.currentStream = &q.streams[0]\n\treturn &q\n}\n\nfunc (q *Quantile) Result() *Result {\n\tif q == nil {\n\t\treturn &Result{}\n\t}\n\tqueryHandler := q.QueryHandler()\n\tresult := Result{\n\t\tCount:       queryHandler.Count(),\n\t\tPercentiles: make([]map[string]float64, len(q.Percentiles)),\n\t}\n\tfor i, p := range q.Percentiles {\n\t\tvalue := queryHandler.Query(p)\n\t\tresult.Percentiles[i] = map[string]float64{\"quantile\": p, \"value\": value}\n\t}\n\treturn &result\n}\n\nfunc (q *Quantile) Insert(msgStartTime int64) {\n\tq.Lock()\n\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tq.currentStream.Insert(float64(now.UnixNano() - msgStartTime))\n\tq.Unlock()\n}\n\nfunc (q *Quantile) QueryHandler() *quantile.Stream {\n\tq.Lock()\n\tnow := time.Now()\n\tfor q.IsDataStale(now) {\n\t\tq.moveWindow()\n\t}\n\n\tmerged := quantile.NewTargeted(q.Percentiles...)\n\tmerged.Merge(q.streams[0].Samples())\n\tmerged.Merge(q.streams[1].Samples())\n\tq.Unlock()\n\treturn merged\n}\n\nfunc (q *Quantile) IsDataStale(now time.Time) bool {\n\treturn now.After(q.lastMoveWindow.Add(q.MoveWindowTime))\n}\n\nfunc (q *Quantile) Merge(them *Quantile) {\n\tq.Lock()\n\tthem.Lock()\n\tiUs := q.currentIndex\n\tiThem := them.currentIndex\n\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tiUs ^= 0x1\n\tiThem ^= 0x1\n\tq.streams[iUs].Merge(them.streams[iThem].Samples())\n\n\tif q.lastMoveWindow.Before(them.lastMoveWindow) {\n\t\tq.lastMoveWindow = them.lastMoveWindow\n\t}\n\tq.Unlock()\n\tthem.Unlock()\n}\n\nfunc (q *Quantile) moveWindow() {\n\tq.currentIndex ^= 0x1\n\tq.currentStream = &q.streams[q.currentIndex]\n\tq.lastMoveWindow = q.lastMoveWindow.Add(q.MoveWindowTime)\n\tq.currentStream.Reset()\n}\n\n// Path: internal/clusterinfo/types.go\npackage clusterinfo\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"sort\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/blang/semver\"\n\t\"github.com/nsqio/nsq/internal/quantile\"\n)\n\ntype ProducerTopic struct {\n\tTopic      string `json:\"topic\"`\n\tTombstoned bool   `json:\"tombstoned\"`\n}\n\ntype ProducerTopics []ProducerTopic\n\nfunc (pt ProducerTopics) Len() int           { return len(pt) }\nfunc (pt ProducerTopics) Swap(i, j int)      { pt[i], pt[j] = pt[j], pt[i] }\nfunc (pt ProducerTopics) Less(i, j int) bool { return pt[i].Topic < pt[j].Topic }\n\ntype Producer struct {\n\tRemoteAddresses  []string       `json:\"remote_addresses\"`\n\tRemoteAddress    string         `json:\"remote_address\"`\n\tHostname         string         `json:\"hostname\"`\n\tBroadcastAddress string         `json:\"broadcast_address\"`\n\tTCPPort          int            `json:\"tcp_port\"`\n\tHTTPPort         int            `json:\"http_port\"`\n\tVersion          string         `json:\"version\"`\n\tVersionObj       semver.Version `json:\"-\"`\n\tTopics           ProducerTopics `json:\"topics\"`\n\tOutOfDate        bool           `json:\"out_of_date\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses of ProducerTopics and VersionObj\nfunc (p *Producer) UnmarshalJSON(b []byte) error {\n\tvar r struct {\n\t\tRemoteAddress    string   `json:\"remote_address\"`\n\t\tHostname         string   `json:\"hostname\"`\n\t\tBroadcastAddress string   `json:\"broadcast_address\"`\n\t\tTCPPort          int      `json:\"tcp_port\"`\n\t\tHTTPPort         int      `json:\"http_port\"`\n\t\tVersion          string   `json:\"version\"`\n\t\tTopics           []string `json:\"topics\"`\n\t\tTombstoned       []bool   `json:\"tombstones\"`\n\t}\n\tif err := json.Unmarshal(b, &r); err != nil {\n\t\treturn err\n\t}\n\t*p = Producer{\n\t\tRemoteAddress:    r.RemoteAddress,\n\t\tHostname:         r.Hostname,\n\t\tBroadcastAddress: r.BroadcastAddress,\n\t\tTCPPort:          r.TCPPort,\n\t\tHTTPPort:         r.HTTPPort,\n\t\tVersion:          r.Version,\n\t}\n\tfor i, t := range r.Topics {\n\t\tp.Topics = append(p.Topics, ProducerTopic{Topic: t, Tombstoned: r.Tombstoned[i]})\n\t}\n\tversion, err := semver.Parse(p.Version)\n\tif err != nil {\n\t\tversion, _ = semver.Parse(\"0.0.0\")\n\t}\n\tp.VersionObj = version\n\treturn nil\n}\n\nfunc (p *Producer) Address() string {\n\tif p.RemoteAddress == \"\" {\n\t\treturn \"N/A\"\n\t}\n\treturn p.RemoteAddress\n}\n\nfunc (p *Producer) HTTPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.HTTPPort))\n}\n\nfunc (p *Producer) TCPAddress() string {\n\treturn net.JoinHostPort(p.BroadcastAddress, strconv.Itoa(p.TCPPort))\n}\n\n// IsInconsistent checks for cases where an unexpected number of nsqd connections are\n// reporting the same information to nsqlookupd (ie: multiple instances are using the\n// same broadcast address), or cases where some nsqd are not reporting to all nsqlookupd.\nfunc (p *Producer) IsInconsistent(numLookupd int) bool {\n\treturn len(p.RemoteAddresses) != numLookupd\n}\n\ntype TopicStats struct {\n\tNode         string          `json:\"node\"`\n\tHostname     string          `json:\"hostname\"`\n\tTopicName    string          `json:\"topic_name\"`\n\tDepth        int64           `json:\"depth\"`\n\tMemoryDepth  int64           `json:\"memory_depth\"`\n\tBackendDepth int64           `json:\"backend_depth\"`\n\tMessageCount int64           `json:\"message_count\"`\n\tNodeStats    []*TopicStats   `json:\"nodes\"`\n\tChannels     []*ChannelStats `json:\"channels\"`\n\tPaused       bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (t *TopicStats) Add(a *TopicStats) {\n\tt.Node = \"*\"\n\tt.Depth += a.Depth\n\tt.MemoryDepth += a.MemoryDepth\n\tt.BackendDepth += a.BackendDepth\n\tt.MessageCount += a.MessageCount\n\tif a.Paused {\n\t\tt.Paused = a.Paused\n\t}\n\tfor _, aChannelStats := range a.Channels {\n\t\tfound := false\n\t\tfor _, channelStats := range t.Channels {\n\t\t\tif aChannelStats.ChannelName == channelStats.ChannelName {\n\t\t\t\tfound = true\n\t\t\t\tchannelStats.Add(aChannelStats)\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\tt.Channels = append(t.Channels, aChannelStats)\n\t\t}\n\t}\n\tt.NodeStats = append(t.NodeStats, a)\n\tsort.Sort(TopicStatsByHost{t.NodeStats})\n\tif t.E2eProcessingLatency == nil {\n\t\tt.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:  t.Node,\n\t\t\tTopic: t.TopicName,\n\t\t}\n\t}\n\tt.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n}\n\ntype ChannelStats struct {\n\tNode          string          `json:\"node\"`\n\tHostname      string          `json:\"hostname\"`\n\tTopicName     string          `json:\"topic_name\"`\n\tChannelName   string          `json:\"channel_name\"`\n\tDepth         int64           `json:\"depth\"`\n\tMemoryDepth   int64           `json:\"memory_depth\"`\n\tBackendDepth  int64           `json:\"backend_depth\"`\n\tInFlightCount int64           `json:\"in_flight_count\"`\n\tDeferredCount int64           `json:\"deferred_count\"`\n\tRequeueCount  int64           `json:\"requeue_count\"`\n\tTimeoutCount  int64           `json:\"timeout_count\"`\n\tMessageCount  int64           `json:\"message_count\"`\n\tClientCount   int             `json:\"client_count\"`\n\tSelected      bool            `json:\"-\"`\n\tNodeStats     []*ChannelStats `json:\"nodes\"`\n\tClients       []*ClientStats  `json:\"clients\"`\n\tPaused        bool            `json:\"paused\"`\n\n\tE2eProcessingLatency *quantile.E2eProcessingLatencyAggregate `json:\"e2e_processing_latency\"`\n}\n\nfunc (c *ChannelStats) Add(a *ChannelStats) {\n\tc.Node = \"*\"\n\tc.Depth += a.Depth\n\tc.MemoryDepth += a.MemoryDepth\n\tc.BackendDepth += a.BackendDepth\n\tc.InFlightCount += a.InFlightCount\n\tc.DeferredCount += a.DeferredCount\n\tc.RequeueCount += a.RequeueCount\n\tc.TimeoutCount += a.TimeoutCount\n\tc.MessageCount += a.MessageCount\n\tc.ClientCount += a.ClientCount\n\tif a.Paused {\n\t\tc.Paused = a.Paused\n\t}\n\tc.NodeStats = append(c.NodeStats, a)\n\tsort.Sort(ChannelStatsByHost{c.NodeStats})\n\tif c.E2eProcessingLatency == nil {\n\t\tc.E2eProcessingLatency = &quantile.E2eProcessingLatencyAggregate{\n\t\t\tAddr:    c.Node,\n\t\t\tTopic:   c.TopicName,\n\t\t\tChannel: c.ChannelName,\n\t\t}\n\t}\n\tc.E2eProcessingLatency.Add(a.E2eProcessingLatency)\n\tc.Clients = append(c.Clients, a.Clients...)\n\tsort.Sort(ClientsByHost{c.Clients})\n}\n\ntype ClientStats struct {\n\tNode              string        `json:\"node\"`\n\tRemoteAddress     string        `json:\"remote_address\"`\n\tVersion           string        `json:\"version\"`\n\tClientID          string        `json:\"client_id\"`\n\tHostname          string        `json:\"hostname\"`\n\tUserAgent         string        `json:\"user_agent\"`\n\tConnectTs         int64         `json:\"connect_ts\"`\n\tConnectedDuration time.Duration `json:\"connected\"`\n\tInFlightCount     int           `json:\"in_flight_count\"`\n\tReadyCount        int           `json:\"ready_count\"`\n\tFinishCount       int64         `json:\"finish_count\"`\n\tRequeueCount      int64         `json:\"requeue_count\"`\n\tMessageCount      int64         `json:\"message_count\"`\n\tSampleRate        int32         `json:\"sample_rate\"`\n\tDeflate           bool          `json:\"deflate\"`\n\tSnappy            bool          `json:\"snappy\"`\n\tAuthed            bool          `json:\"authed\"`\n\tAuthIdentity      string        `json:\"auth_identity\"`\n\tAuthIdentityURL   string        `json:\"auth_identity_url\"`\n\n\tTLS                           bool   `json:\"tls\"`\n\tCipherSuite                   string `json:\"tls_cipher_suite\"`\n\tTLSVersion                    string `json:\"tls_version\"`\n\tTLSNegotiatedProtocol         string `json:\"tls_negotiated_protocol\"`\n\tTLSNegotiatedProtocolIsMutual bool   `json:\"tls_negotiated_protocol_is_mutual\"`\n}\n\n// UnmarshalJSON implements json.Unmarshaler and postprocesses ConnectedDuration\nfunc (s *ClientStats) UnmarshalJSON(b []byte) error {\n\ttype locaClientStats ClientStats // re-typed to prevent recursion from json.Unmarshal\n\tvar ss locaClientStats\n\tif err := json.Unmarshal(b, &ss); err != nil {\n\t\treturn err\n\t}\n\t*s = ClientStats(ss)\n\ts.ConnectedDuration = time.Now().Truncate(time.Second).Sub(time.Unix(s.ConnectTs, 0))\n\treturn nil\n}\n\nfunc (s *ClientStats) HasUserAgent() bool {\n\treturn s.UserAgent != \"\"\n}\n\nfunc (s *ClientStats) HasSampleRate() bool {\n\treturn s.SampleRate > 0\n}\n\ntype ChannelStatsList []*ChannelStats\n\nfunc (c ChannelStatsList) Len() int      { return len(c) }\nfunc (c ChannelStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ChannelStatsByHost struct {\n\tChannelStatsList\n}\n\nfunc (c ChannelStatsByHost) Less(i, j int) bool {\n\treturn c.ChannelStatsList[i].Hostname < c.ChannelStatsList[j].Hostname\n}\n\ntype ClientStatsList []*ClientStats\n\nfunc (c ClientStatsList) Len() int      { return len(c) }\nfunc (c ClientStatsList) Swap(i, j int) { c[i], c[j] = c[j], c[i] }\n\ntype ClientsByHost struct {\n\tClientStatsList\n}\n\nfunc (c ClientsByHost) Less(i, j int) bool {\n\treturn c.ClientStatsList[i].Hostname < c.ClientStatsList[j].Hostname\n}\n\ntype TopicStatsList []*TopicStats\n\nfunc (t TopicStatsList) Len() int      { return len(t) }\nfunc (t TopicStatsList) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\ntype TopicStatsByHost struct {\n\tTopicStatsList\n}\n\nfunc (c TopicStatsByHost) Less(i, j int) bool {\n\treturn c.TopicStatsList[i].Hostname < c.TopicStatsList[j].Hostname\n}\n\ntype Producers []*Producer\n\nfunc (t Producers) Len() int      { return len(t) }\nfunc (t Producers) Swap(i, j int) { t[i], t[j] = t[j], t[i] }\n\nfunc (t Producers) HTTPAddrs() []string {\n\tvar addrs []string\n\tfor _, p := range t {\n\t\taddrs = append(addrs, p.HTTPAddress())\n\t}\n\treturn addrs\n}\n\nfunc (t Producers) Search(needle string) *Producer {\n\tfor _, producer := range t {\n\t\tif needle == producer.HTTPAddress() {\n\t\t\treturn producer\n\t\t}\n\t}\n\treturn nil\n}\n\ntype ProducersByHost struct {\n\tProducers\n}\n\nfunc (c ProducersByHost) Less(i, j int) bool {\n\treturn c.Producers[i].Hostname < c.Producers[j].Hostname\n}\n\n// Path: internal/dirlock/dirlock.go\n//go:build !windows && !illumos\n// +build !windows,!illumos\n\npackage dirlock\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"syscall\"\n)\n\ntype DirLock struct {\n\tdir string\n\tf   *os.File\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\tf, err := os.Open(l.dir)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl.f = f\n\terr = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"cannot flock directory %s - %s (possibly in use by another instance of nsqd)\", l.dir, err)\n\t}\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\tdefer l.f.Close()\n\treturn syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)\n}\n\n// Path: internal/dirlock/dirlock_illumos.go\n//go:build illumos\n// +build illumos\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/dirlock/dirlock_windows.go\n//go:build windows\n// +build windows\n\npackage dirlock\n\ntype DirLock struct {\n\tdir string\n}\n\nfunc New(dir string) *DirLock {\n\treturn &DirLock{\n\t\tdir: dir,\n\t}\n}\n\nfunc (l *DirLock) Lock() error {\n\treturn nil\n}\n\nfunc (l *DirLock) Unlock() error {\n\treturn nil\n}\n\n// Path: internal/http_api/api_request.go\npackage http_api\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// A custom http.Transport with support for deadline timeouts\nfunc NewDeadlineTransport(connectTimeout time.Duration, requestTimeout time.Duration) *http.Transport {\n\t// arbitrary values copied from http.DefaultTransport\n\ttransport := &http.Transport{\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout:   connectTimeout,\n\t\t\tKeepAlive: 30 * time.Second,\n\t\t\tDualStack: true,\n\t\t}).DialContext,\n\t\tResponseHeaderTimeout: requestTimeout,\n\t\tMaxIdleConns:          100,\n\t\tIdleConnTimeout:       90 * time.Second,\n\t\tTLSHandshakeTimeout:   10 * time.Second,\n\t}\n\treturn transport\n}\n\ntype Client struct {\n\tc *http.Client\n}\n\nfunc NewClient(tlsConfig *tls.Config, connectTimeout time.Duration, requestTimeout time.Duration) *Client {\n\ttransport := NewDeadlineTransport(connectTimeout, requestTimeout)\n\ttransport.TLSClientConfig = tlsConfig\n\treturn &Client{\n\t\tc: &http.Client{\n\t\t\tTransport: transport,\n\t\t\tTimeout:   requestTimeout,\n\t\t},\n\t}\n}\n\n// GETV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) GETV1(endpoint string, v interface{}) error {\nretry:\n\treq, err := http.NewRequest(\"GET\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\terr = json.Unmarshal(body, &v)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// PostV1 is a helper function to perform a V1 HTTP request\n// and parse our NSQ daemon's expected response format, with deadlines.\nfunc (c *Client) POSTV1(endpoint string) error {\nretry:\n\treq, err := http.NewRequest(\"POST\", endpoint, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treq.Header.Add(\"Accept\", \"application/vnd.nsq; version=1.0\")\n\n\tresp, err := c.c.Do(req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tresp.Body.Close()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif resp.StatusCode != 200 {\n\t\tif resp.StatusCode == 403 && !strings.HasPrefix(endpoint, \"https\") {\n\t\t\tendpoint, err = httpsEndpoint(endpoint, body)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgoto retry\n\t\t}\n\t\treturn fmt.Errorf(\"got response %s %q\", resp.Status, body)\n\t}\n\n\treturn nil\n}\n\nfunc httpsEndpoint(endpoint string, body []byte) (string, error) {\n\tvar forbiddenResp struct {\n\t\tHTTPSPort int `json:\"https_port\"`\n\t}\n\terr := json.Unmarshal(body, &forbiddenResp)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu, err := url.Parse(endpoint)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\thost, _, err := net.SplitHostPort(u.Host)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tu.Scheme = \"https\"\n\tu.Host = net.JoinHostPort(host, strconv.Itoa(forbiddenResp.HTTPSPort))\n\treturn u.String(), nil\n}\n\n// Path: internal/http_api/api_response.go\npackage http_api\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/julienschmidt/httprouter\"\n\t\"github.com/nsqio/nsq/internal/lg\"\n)\n\ntype Decorator func(APIHandler) APIHandler\n\ntype APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)\n\ntype Err struct {\n\tCode int\n\tText string\n}\n\nfunc (e Err) Error() string {\n\treturn e.Text\n}\n\nfunc PlainText(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tcode := 200\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tcode = err.(Err).Code\n\t\t\tdata = err.Error()\n\t\t}\n\t\tswitch d := data.(type) {\n\t\tcase string:\n\t\t\tw.WriteHeader(code)\n\t\t\tio.WriteString(w, d)\n\t\tcase []byte:\n\t\t\tw.WriteHeader(code)\n\t\t\tw.Write(d)\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"unknown response type %T\", data))\n\t\t}\n\t\treturn nil, nil\n\t}\n}\n\n\nfunc V1(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tRespondV1(w, err.(Err).Code, err)\n\t\t\treturn nil, nil\n\t\t}\n\t\tRespondV1(w, 200, data)\n\t\treturn nil, nil\n\t}\n}\n\nfunc RespondV1(w http.ResponseWriter, code int, data interface{}) {\n\tvar response []byte\n\tvar err error\n\tvar isJSON bool\n\n\tif code == 200 {\n\t\tswitch data := data.(type) {\n\t\tcase string:\n\t\t\tresponse = []byte(data)\n\t\tcase []byte:\n\t\t\tresponse = data\n\t\tcase nil:\n\t\t\tresponse = []byte{}\n\t\tdefault:\n\t\t\tisJSON = true\n\t\t\tresponse, err = json.Marshal(data)\n\t\t\tif err != nil {\n\t\t\t\tcode = 500\n\t\t\t\tdata = err\n\t\t\t}\n\t\t}\n\t}\n\n\tif code != 200 {\n\t\tisJSON = true\n\t\tresponse, _ = json.Marshal(struct {\n\t\t\tMessage string `json:\"message\"`\n\t\t}{fmt.Sprintf(\"%s\", data)})\n\t}\n\n\tif isJSON {\n\t\tw.Header().Set(\"Content-Type\", \"application/json; charset=utf-8\")\n\t}\n\tw.Header().Set(\"X-NSQ-Content-Type\", \"nsq; version=1.0\")\n\tw.WriteHeader(code)\n\tw.Write(response)\n}\n\nfunc Decorate(f APIHandler, ds ...Decorator) httprouter.Handle {\n\tdecorated := f\n\tfor _, decorate := range ds {\n\t\tdecorated = decorate(decorated)\n\t}\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {\n\t\tdecorated(w, req, ps)\n\t}\n}\n\nfunc Log(logf lg.AppLogFunc) Decorator {\n\treturn func(f APIHandler) APIHandler {\n\t\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\t\tstart := time.Now()\n\t\t\tresponse, err := f(w, req, ps)\n\t\t\telapsed := time.Since(start)\n\t\t\tstatus := 200\n\t\t\tif e, ok := err.(Err); ok {\n\t\t\t\tstatus = e.Code\n\t\t\t}\n\t\t\tlogf(lg.INFO, \"%d %s %s (%s) %s\",\n\t\t\t\tstatus, req.Method, req.URL.RequestURI(), req.RemoteAddr, elapsed)\n\t\t\treturn response, err\n\t\t}\n\t}\n}\n\nfunc LogPanicHandler(logf lg.AppLogFunc) func(w http.ResponseWriter, req *http.Request, p interface{}) {\n\treturn func(w http.ResponseWriter, req *http.Request, p interface{}) {\n...\n", "needle_token_start": 14282, "needle_token_end": 14407, "code_context_ntokens": 15088, "output": ["```go\nfunc V1(f APIHandler) APIHandler {\n\treturn func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) (interface{}, error) {\n\t\tdata, err := f(w, req, ps)\n\t\tif err != nil {\n\t\t\tRespondV1(w, err.(Err).Code, err)\n\t\t\treturn nil, nil\n\t\t}\n\t\tRespondV1(w, 200, data)\n\t\treturn nil, nil\n\t}\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "getTemplateDirs", "language": "go", "path": "pkg/protocols/common/automaticscan/util.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To retrieve directories containing templates based on default settings and additional specified paths.\n2. **Input**: An options structure that includes execution options and potentially custom template paths.\n3. **Output**: A list of template directory paths or an error if no templates are found or an issue occurs during the directory retrieval process.\n4. **Procedure**: \n   - Start by obtaining the default template directory.\n   - Append any custom template paths provided in the input options.\n   - For each directory, attempt to retrieve the template paths using a catalog function.\n   - Deduplicate the collected template paths.\n   - Return the list of unique template paths or an error if no templates are found or if an error occurs during retrieval.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/templates/templates.go\n//go:generate dstdocgen -path \"\" -structure Template -output templates_doc.go -package templates\npackage templates\n\nimport (\n\t\"encoding/json\"\n\t\"io\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\tvalidate \"github.com/go-playground/validator/v10\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/code\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/file\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/ssl\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/websocket\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n...\n// Path: pkg/protocols/common/automaticscan/util.go\npackage automaticscan\n\nimport (\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/templates\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\tsliceutil \"github.com/projectdiscovery/utils/slice\"\n)\n\n// getTemplateDirs returns template directories for given input\n// by default it returns default template directory\n\nfunc getTemplateDirs(opts Options) ([]string, error) {\n\tdefaultTemplatesDirectories := []string{config.DefaultConfig.GetTemplateDir()}\n\t// adding custom template path if available\n\tif len(opts.ExecuterOpts.Options.Templates) > 0 {\n\t\tdefaultTemplatesDirectories = append(defaultTemplatesDirectories, opts.ExecuterOpts.Options.Templates...)\n\t}\n\t// Collect path for default directories we want to look for templates in\n\tvar allTemplates []string\n\tfor _, directory := range defaultTemplatesDirectories {\n\t\ttemplates, err := opts.ExecuterOpts.Catalog.GetTemplatePath(directory)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"could not get templates in directory\")\n\t\t}\n\t\tallTemplates = append(allTemplates, templates...)\n\t}\n\tallTemplates = sliceutil.Dedupe(allTemplates)\n\tif len(allTemplates) == 0 {\n\t\treturn nil, errors.New(\"no templates found for given input\")\n\t}\n\treturn allTemplates, nil\n}\n\n// LoadTemplatesWithTags loads and returns templates with given tags\nfunc LoadTemplatesWithTags(opts Options, templateDirs []string, tags []string, logInfo bool) ([]*templates.Template, error) {\n\tfinalTemplates := opts.Store.LoadTemplatesWithTags(templateDirs, tags)\n\tif len(finalTemplates) == 0 {\n\t\treturn nil, errors.New(\"could not find any templates with tech tag\")\n\t}\n\n\tif !opts.ExecuterOpts.Options.DisableClustering {\n\t\t// cluster and reduce requests\n\t\ttotalReqBeforeCluster := getRequestCount(finalTemplates) * int(opts.Target.Count())\n\t\tfinalTemplates, clusterCount := templates.ClusterTemplates(finalTemplates, opts.ExecuterOpts)\n\t\ttotalReqAfterClustering := getRequestCount(finalTemplates) * int(opts.Target.Count())\n\t\tif totalReqAfterClustering < totalReqBeforeCluster && logInfo {\n\t\t\tgologger.Info().Msgf(\"Automatic scan tech-detect: Templates clustered: %d (Reduced %d Requests)\", clusterCount, totalReqBeforeCluster-totalReqAfterClustering)\n\t\t}\n\t}\n\n\t// log template loaded if VerboseVerbose flag is set\n\tif opts.ExecuterOpts.Options.VerboseVerbose {\n\t\tfor _, tpl := range finalTemplates {\n\t\t\tgologger.Print().Msgf(\"%s\\n\", templates.TemplateLogMessage(tpl.ID,\n\t\t\t\ttypes.ToString(tpl.Info.Name),\n\t\t\t\ttpl.Info.Authors.ToSlice(),\n\t\t\t\ttpl.Info.SeverityHolder.Severity))\n\t\t}\n\n\t}\n\treturn finalTemplates, nil\n}\n\n// returns total requests count\nfunc getRequestCount(templates []*templates.Template) int {\n\tcount := 0\n\tfor _, template := range templates {\n\t\t// ignore requests in workflows as total requests in workflow\n\t\t// depends on what templates will be called in workflow\n\t\tif len(template.Workflows) > 0 {\n\t\t\tcontinue\n\t\t}\n\t\tcount += template.TotalRequests\n\t}\n\treturn count\n}\n\n// Path: pkg/tmplexec/flow/util.go\npackage flow\n\nimport \"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\n// Checks if template has matchers\nfunc hasMatchers(all []*operators.Operators) bool {\n\tfor _, operator := range all {\n\t\tif len(operator.Matchers) > 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// hasOperators checks if template has operators (i.e matchers/extractors)\nfunc hasOperators(all []*operators.Operators) bool {\n\tfor _, operator := range all {\n\t\tif operator != nil {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc flatten(v interface{}) interface{} {\n\tswitch v := v.(type) {\n\tcase []interface{}:\n\t\tif len(v) == 1 {\n\t\t\treturn v[0]\n\t\t}\n\t\treturn v\n\tcase []string:\n\t\tif len(v) == 1 {\n\t\t\treturn v[0]\n\t\t}\n\t\treturn v\n\tdefault:\n\t\treturn v\n\t}\n}\n\n// Path: pkg/operators/extractors/util.go\npackage extractors\n\n// SupportsMap determines if the extractor type requires a map\nfunc SupportsMap(extractor *Extractor) bool {\n\treturn extractor.Type.ExtractorType == KValExtractor || extractor.Type.ExtractorType == DSLExtractor\n}\n\n// Path: pkg/reporting/trackers/filters/filters.go\npackage filters\n\nimport (\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\n\tsliceutil \"github.com/projectdiscovery/utils/slice\"\n)\n\n// CreateIssueResponse is a response to creating an issue\n// in a tracker\ntype CreateIssueResponse struct {\n\tIssueID  string `json:\"issue_id\"`\n\tIssueURL string `json:\"issue_url\"`\n}\n\n// Filter filters the received event and decides whether to perform\n// reporting for it or not.\ntype Filter struct {\n\tSeverities severity.Severities     `yaml:\"severity\"`\n\tTags       stringslice.StringSlice `yaml:\"tags\"`\n}\n\n// GetMatch returns true if a filter matches result event\nfunc (filter *Filter) GetMatch(event *output.ResultEvent) bool {\n\treturn isSeverityMatch(event, filter) && isTagMatch(event, filter) // TODO revisit this\n}\n\nfunc isTagMatch(event *output.ResultEvent, filter *Filter) bool {\n\tfilterTags := filter.Tags\n\tif filterTags.IsEmpty() {\n\t\treturn true\n\t}\n\n\ttags := event.Info.Tags.ToSlice()\n\tfor _, filterTag := range filterTags.ToSlice() {\n\t\tif sliceutil.Contains(tags, filterTag) {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc isSeverityMatch(event *output.ResultEvent, filter *Filter) bool {\n\tresultEventSeverity := event.Info.SeverityHolder.Severity // TODO review\n\n\tif len(filter.Severities) == 0 {\n\t\treturn true\n\t}\n\n\treturn sliceutil.Contains(filter.Severities, resultEventSeverity)\n}\n\n// Path: pkg/protocols/headless/engine/util.go\npackage engine\n\nimport (\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker\"\n\t\"github.com/valyala/fasttemplate\"\n)\n\nfunc replaceWithValues(data string, values map[string]interface{}) string {\n\treturn fasttemplate.ExecuteStringStd(data, marker.ParenthesisOpen, marker.ParenthesisClose, values)\n}\n\n// Path: pkg/external/customtemplates/github.go\npackage customtemplates\n\nimport (\n\t\"context\"\n\thttpclient \"net/http\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/go-git/go-git/v5\"\n\t\"github.com/go-git/go-git/v5/plumbing/transport/http\"\n\t\"github.com/google/go-github/github\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\tfileutil \"github.com/projectdiscovery/utils/file\"\n\tfolderutil \"github.com/projectdiscovery/utils/folder\"\n\t\"golang.org/x/oauth2\"\n)\n\nvar _ Provider = &customTemplateGitHubRepo{}\n\ntype customTemplateGitHubRepo struct {\n\towner       string\n\treponame    string\n\tgitCloneURL string\n\tgithubToken string\n}\n\n// This function download the custom github template repository\nfunc (customTemplate *customTemplateGitHubRepo) Download(ctx context.Context) {\n\tclonePath := customTemplate.getLocalRepoClonePath(config.DefaultConfig.CustomGitHubTemplatesDirectory)\n\n\tif !fileutil.FolderExists(clonePath) {\n\t\terr := customTemplate.cloneRepo(clonePath, customTemplate.githubToken)\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"%s\", err)\n\t\t} else {\n\t\t\tgologger.Info().Msgf(\"Repo %s/%s cloned successfully at %s\", customTemplate.owner, customTemplate.reponame, clonePath)\n\t\t}\n\t\treturn\n\t}\n}\n\nfunc (customTemplate *customTemplateGitHubRepo) Update(ctx context.Context) {\n\tdownloadPath := config.DefaultConfig.CustomGitHubTemplatesDirectory\n\tclonePath := customTemplate.getLocalRepoClonePath(downloadPath)\n\n\t// If folder does not exits then clone/download the repo\n\tif !fileutil.FolderExists(clonePath) {\n\t\tcustomTemplate.Download(ctx)\n\t\treturn\n\t}\n\terr := customTemplate.pullChanges(clonePath, customTemplate.githubToken)\n\tif err != nil {\n\t\tgologger.Error().Msgf(\"%s\", err)\n\t} else {\n\t\tgologger.Info().Msgf(\"Repo %s/%s successfully pulled the changes.\\n\", customTemplate.owner, customTemplate.reponame)\n\t}\n}\n\n// NewGitHubProviders returns new instance of GitHub providers for downloading custom templates\nfunc NewGitHubProviders(options *types.Options) ([]*customTemplateGitHubRepo, error) {\n\tproviders := []*customTemplateGitHubRepo{}\n\tgitHubClient := getGHClientIncognito()\n\n\tif options.GitHubTemplateDisableDownload {\n\t\treturn providers, nil\n\t}\n\n\tfor _, repoName := range options.GitHubTemplateRepo {\n\t\towner, repo, err := getOwnerAndRepo(repoName)\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"%s\", err)\n\t\t\tcontinue\n\t\t}\n\t\tgithubRepo, err := getGitHubRepo(gitHubClient, owner, repo, options.GitHubToken)\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"%s\", err)\n\t\t\tcontinue\n\t\t}\n\t\tcustomTemplateRepo := &customTemplateGitHubRepo{\n\t\t\towner:       owner,\n\t\t\treponame:    repo,\n\t\t\tgitCloneURL: githubRepo.GetCloneURL(),\n\t\t\tgithubToken: options.GitHubToken,\n\t\t}\n\t\tproviders = append(providers, customTemplateRepo)\n\n\t\tcustomTemplateRepo.restructureRepoDir()\n\t}\n\treturn providers, nil\n}\n\nfunc (customTemplateRepo *customTemplateGitHubRepo) restructureRepoDir() {\n\tcustomGitHubTemplatesDirectory := config.DefaultConfig.CustomGitHubTemplatesDirectory\n\toldRepoClonePath := filepath.Join(customGitHubTemplatesDirectory, customTemplateRepo.reponame+\"-\"+customTemplateRepo.owner)\n\tnewRepoClonePath := customTemplateRepo.getLocalRepoClonePath(customGitHubTemplatesDirectory)\n\n\tif fileutil.FolderExists(oldRepoClonePath) && !fileutil.FolderExists(newRepoClonePath) {\n\t\t_ = folderutil.SyncDirectory(oldRepoClonePath, newRepoClonePath)\n\t}\n}\n\n// getOwnerAndRepo returns the owner, repo, err from the given string\n// e.g., it takes input projectdiscovery/nuclei-templates and\n// returns owner => projectdiscovery, repo => nuclei-templates\nfunc getOwnerAndRepo(reponame string) (owner string, repo string, err error) {\n\ts := strings.Split(reponame, \"/\")\n\tif len(s) != 2 {\n\t\terr = errors.Errorf(\"wrong Repo name: %s\", reponame)\n\t\treturn\n\t}\n\towner = s[0]\n\trepo = s[1]\n\treturn\n}\n\n// returns *github.Repository if passed github repo name\nfunc getGitHubRepo(gitHubClient *github.Client, repoOwner, repoName, githubToken string) (*github.Repository, error) {\n\tvar retried bool\ngetRepo:\n\trepo, _, err := gitHubClient.Repositories.Get(context.Background(), repoOwner, repoName)\n\tif err != nil {\n\t\t// retry with authentication\n\t\tif gitHubClient = getGHClientWithToken(githubToken); gitHubClient != nil && !retried {\n\t\t\tretried = true\n\t\t\tgoto getRepo\n\t\t}\n\t\treturn nil, err\n\t}\n\tif repo == nil {\n\t\treturn nil, errors.Errorf(\"problem getting repository: %s/%s\", repoOwner, repoName)\n\t}\n\treturn repo, nil\n}\n\n// download the git repo to a given path\nfunc (ctr *customTemplateGitHubRepo) cloneRepo(clonePath, githubToken string) error {\n\tr, err := git.PlainClone(clonePath, false, &git.CloneOptions{\n\t\tURL:  ctr.gitCloneURL,\n\t\tAuth: getAuth(ctr.owner, githubToken),\n\t})\n\tif err != nil {\n\t\treturn errors.Errorf(\"%s/%s: %s\", ctr.owner, ctr.reponame, err.Error())\n\t}\n\t// Add the user as well in the config. By default, user is not set\n\tconfig, _ := r.Storer.Config()\n\tconfig.User.Name = ctr.owner\n\treturn r.SetConfig(config)\n}\n\n// performs the git pull on given repo\nfunc (ctr *customTemplateGitHubRepo) pullChanges(repoPath, githubToken string) error {\n\tr, err := git.PlainOpen(repoPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tw, err := r.Worktree()\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = w.Pull(&git.PullOptions{RemoteName: \"origin\", Auth: getAuth(ctr.owner, githubToken)})\n\tif err != nil {\n\t\treturn errors.Errorf(\"%s/%s: %s\", ctr.owner, ctr.reponame, err.Error())\n\t}\n\treturn nil\n}\n\n// All Custom github repos are cloned in the format of 'owner/reponame' for uniqueness\nfunc (ctr *customTemplateGitHubRepo) getLocalRepoClonePath(downloadPath string) string {\n\treturn filepath.Join(downloadPath, ctr.owner, ctr.reponame)\n}\n\n// returns the auth object with username and github token as password\nfunc getAuth(username, password string) *http.BasicAuth {\n\tif username != \"\" && password != \"\" {\n\t\treturn &http.BasicAuth{Username: username, Password: password}\n\t}\n\treturn nil\n}\n\nfunc getGHClientWithToken(token string) *github.Client {\n\tif token != \"\" {\n\t\tctx := context.Background()\n\t\tts := oauth2.StaticTokenSource(\n\t\t\t&oauth2.Token{AccessToken: token},\n\t\t)\n\t\toauthClient := oauth2.NewClient(ctx, ts)\n\t\treturn github.NewClient(oauthClient)\n\n\t}\n\treturn nil\n}\n\nfunc getGHClientIncognito() *github.Client {\n\tvar tc *httpclient.Client\n\treturn github.NewClient(tc)\n}\n\n// Path: pkg/reporting/format/format.go\npackage format\n\ntype ResultFormatter interface {\n\tMakeBold(text string) string\n\tCreateCodeBlock(title string, content string, language string) string\n\tCreateTable(headers []string, rows [][]string) (string, error)\n\tCreateLink(title string, url string) string\n\tCreateHorizontalLine() string\n}\n\n// Path: pkg/js/utils/util.go\npackage utils\n\nimport (\n\t\"database/sql\"\n)\n\n// SQLResult holds the result of a SQL query.\n//\n// It contains the count of rows, the columns present, and the actual row data.\ntype SQLResult struct {\n\tCount   int           // Count is the number of rows returned.\n\tColumns []string      // Columns is the slice of column names.\n\tRows    []interface{} // Rows is a slice of row data, where each row is a map of column name to value.\n}\n\n// UnmarshalSQLRows converts sql.Rows into a more structured SQLResult.\n//\n// This function takes *sql.Rows as input and attempts to unmarshal the data into\n// a SQLResult struct. It handles different SQL data types by using the appropriate\n// sql.Null* types during scanning. It returns a pointer to a SQLResult or an error.\n//\n// The function closes the sql.Rows when finished.\nfunc UnmarshalSQLRows(rows *sql.Rows) (*SQLResult, error) {\n\tdefer rows.Close()\n\tcolumnTypes, err := rows.ColumnTypes()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tresult := &SQLResult{}\n\tresult.Columns, err = rows.Columns()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcount := len(columnTypes)\n\tfor rows.Next() {\n\t\tresult.Count++\n\t\tscanArgs := make([]interface{}, count)\n\t\tfor i, v := range columnTypes {\n\t\t\tswitch v.DatabaseTypeName() {\n\t\t\tcase \"VARCHAR\", \"TEXT\", \"UUID\", \"TIMESTAMP\":\n\t\t\t\tscanArgs[i] = new(sql.NullString)\n\t\t\tcase \"BOOL\":\n\t\t\t\tscanArgs[i] = new(sql.NullBool)\n\t\t\tcase \"INT4\":\n\t\t\t\tscanArgs[i] = new(sql.NullInt64)\n\t\t\tdefault:\n\t\t\t\tscanArgs[i] = new(sql.NullString)\n\t\t\t}\n\t\t}\n\t\terr := rows.Scan(scanArgs...)\n\t\tif err != nil {\n\t\t\t// Return the result accumulated so far along with the error.\n\t\t\treturn result, err\n\t\t}\n\t\tmasterData := make(map[string]interface{})\n\t\tfor i, v := range columnTypes {\n\t\t\tif z, ok := (scanArgs[i]).(*sql.NullBool); ok {\n\t\t\t\tmasterData[v.Name()] = z.Bool\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif z, ok := (scanArgs[i]).(*sql.NullString); ok {\n\t\t\t\tmasterData[v.Name()] = z.String\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif z, ok := (scanArgs[i]).(*sql.NullInt64); ok {\n\t\t\t\tmasterData[v.Name()] = z.Int64\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif z, ok := (scanArgs[i]).(*sql.NullFloat64); ok {\n\t\t\t\tmasterData[v.Name()] = z.Float64\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif z, ok := (scanArgs[i]).(*sql.NullInt32); ok {\n\t\t\t\tmasterData[v.Name()] = z.Int32\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tmasterData[v.Name()] = scanArgs[i]\n\t\t}\n\t\tresult.Rows = append(result.Rows, masterData)\n\t}\n\treturn result, nil\n}\n\n// Path: pkg/reporting/trackers/github/github.go\npackage github\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/google/go-github/github\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n\t\"golang.org/x/oauth2\"\n)\n\n// Integration is a client for an issue tracker integration\ntype Integration struct {\n\tclient  *github.Client\n\toptions *Options\n}\n\n// Options contains the configuration options for GitHub issue tracker client\ntype Options struct {\n\t// BaseURL (optional) is the self-hosted GitHub application url\n\tBaseURL string `yaml:\"base-url\" validate:\"omitempty,url\"`\n\t// Username is the username of the GitHub user\n\tUsername string `yaml:\"username\" validate:\"required\"`\n\t// Owner is the owner name of the repository for issues.\n\tOwner string `yaml:\"owner\" validate:\"required\"`\n\t// Token is the token for GitHub account.\n\tToken string `yaml:\"token\" validate:\"required\"`\n\t// ProjectName is the name of the repository.\n\tProjectName string `yaml:\"project-name\" validate:\"required\"`\n\t// IssueLabel (optional) is the label of the created issue type\n\tIssueLabel string `yaml:\"issue-label\"`\n\t// SeverityAsLabel (optional) sends the severity as the label of the created\n\t// issue.\n\tSeverityAsLabel bool `yaml:\"severity-as-label\"`\n\t// AllowList contains a list of allowed events for this tracker\n\tAllowList *filters.Filter `yaml:\"allow-list\"`\n\t// DenyList contains a list of denied events for this tracker\n\tDenyList *filters.Filter `yaml:\"deny-list\"`\n\t// DuplicateIssueCheck (optional) comments under existing finding issue\n\t// instead of creating duplicates for subsequent runs.\n\tDuplicateIssueCheck bool `yaml:\"duplicate-issue-check\"`\n\n\tHttpClient *retryablehttp.Client `yaml:\"-\"`\n\tOmitRaw    bool                  `yaml:\"-\"`\n}\n\n// New creates a new issue tracker integration client based on options.\nfunc New(options *Options) (*Integration, error) {\n\tctx := context.Background()\n\tts := oauth2.StaticTokenSource(\n\t\t&oauth2.Token{AccessToken: options.Token},\n\t)\n\ttc := oauth2.NewClient(ctx, ts)\n\n\t// patch transport to support proxy - only http\n\t// TODO: investigate if it's possible to reuse existing retryablehttp\n\tif types.ProxyURL != \"\" {\n\t\tif proxyURL, err := url.Parse(types.ProxyURL); err == nil {\n\t\t\ttc.Transport.(*http.Transport).Proxy = http.ProxyURL(proxyURL)\n\t\t}\n\t}\n\n\tclient := github.NewClient(tc)\n\tif options.BaseURL != \"\" {\n\t\tparsed, err := url.Parse(options.BaseURL)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"could not parse custom baseurl\")\n\t\t}\n\t\tif !strings.HasSuffix(parsed.Path, \"/\") {\n\t\t\tparsed.Path += \"/\"\n\t\t}\n\t\tclient.BaseURL = parsed\n\t}\n\treturn &Integration{client: client, options: options}, nil\n}\n\n// CreateIssue creates an issue in the tracker\nfunc (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {\n\tsummary := format.Summary(event)\n\tdescription := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)\n\tlabels := []string{}\n\tseverityLabel := fmt.Sprintf(\"Severity: %s\", event.Info.SeverityHolder.Severity.String())\n\tif i.options.SeverityAsLabel && severityLabel != \"\" {\n\t\tlabels = append(labels, severityLabel)\n\t}\n\tif label := i.options.IssueLabel; label != \"\" {\n\t\tlabels = append(labels, label)\n\t}\n\n\tctx := context.Background()\n\n\tvar err error\n\tvar existingIssue *github.Issue\n\tif i.options.DuplicateIssueCheck {\n\t\texistingIssue, err = i.findIssueByTitle(ctx, summary)\n\t\tif err != nil && !errors.Is(err, io.EOF) {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif existingIssue == nil {\n\t\treq := &github.IssueRequest{\n\t\t\tTitle:     &summary,\n\t\t\tBody:      &description,\n\t\t\tLabels:    &labels,\n\t\t\tAssignees: &[]string{i.options.Username},\n\t\t}\n\t\tcreatedIssue, _, err := i.client.Issues.Create(ctx, i.options.Owner, i.options.ProjectName, req)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &filters.CreateIssueResponse{\n\t\t\tIssueID:  strconv.FormatInt(createdIssue.GetID(), 10),\n\t\t\tIssueURL: createdIssue.GetHTMLURL(),\n\t\t}, nil\n\t} else {\n\t\tif existingIssue.GetState() == \"closed\" {\n\t\t\tstateOpen := \"open\"\n\t\t\tif _, _, err := i.client.Issues.Edit(ctx, i.options.Owner, i.options.ProjectName, *existingIssue.Number, &github.IssueRequest{\n\t\t\t\tState: &stateOpen,\n\t\t\t}); err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"error reopening issue %d: %s\", *existingIssue.Number, err)\n\t\t\t}\n\t\t}\n\n\t\treq := &github.IssueComment{\n\t\t\tBody: &description,\n\t\t}\n\t\t_, _, err = i.client.Issues.CreateComment(ctx, i.options.Owner, i.options.ProjectName, *existingIssue.Number, req)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &filters.CreateIssueResponse{\n\t\t\tIssueID:  strconv.FormatInt(existingIssue.GetID(), 10),\n\t\t\tIssueURL: existingIssue.GetHTMLURL(),\n\t\t}, nil\n\t}\n}\n\nfunc (i *Integration) CloseIssue(event *output.ResultEvent) error {\n\tctx := context.Background()\n\tsummary := format.Summary(event)\n\n\texistingIssue, err := i.findIssueByTitle(ctx, summary)\n\tif err != nil && !errors.Is(err, io.EOF) {\n\t\treturn err\n\t}\n\tif existingIssue == nil {\n\t\treturn nil\n\t}\n\n\tstateClosed := \"closed\"\n\tif _, _, err := i.client.Issues.Edit(ctx, i.options.Owner, i.options.ProjectName, *existingIssue.Number, &github.IssueRequest{\n\t\tState: &stateClosed,\n\t}); err != nil {\n\t\treturn fmt.Errorf(\"error closing issue %d: %s\", *existingIssue.Number, err)\n\t}\n\treturn nil\n}\n\nfunc (i *Integration) Name() string {\n\treturn \"github\"\n}\n\n// ShouldFilter determines if an issue should be logged to this tracker\nfunc (i *Integration) ShouldFilter(event *output.ResultEvent) bool {\n\tif i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\tif i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\nfunc (i *Integration) findIssueByTitle(ctx context.Context, title string) (*github.Issue, error) {\n\treq := &github.SearchOptions{\n\t\tSort:      \"updated\",\n\t\tOrder:     \"desc\",\n\t\tTextMatch: false,\n\t\tListOptions: github.ListOptions{\n\t\t\tPage:    1,\n\t\t\tPerPage: 100,\n\t\t},\n\t}\n\n\tquery := fmt.Sprintf(`is:issue repo:%s/%s \"%s\"`, i.options.Owner, i.options.ProjectName, title)\n\n\tfor {\n\t\tissues, resp, err := i.client.Search.Issues(ctx, query, req)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"error listing issues for %s, %s: %w\", i.options.Owner, i.options.ProjectName, err)\n\t\t}\n\n\t\tfor _, issue := range issues.Issues {\n\t\t\tif issue.Title != nil && *issue.Title == title {\n\t\t\t\treturn &issue, nil\n\t\t\t}\n\t\t}\n\n\t\tif resp.NextPage <= req.Page || len(issues.Issues) == 0 {\n\t\t\treturn nil, io.EOF\n\t\t}\n\n\t\treq.ListOptions = github.ListOptions{\n\t\t\tPage:    resp.NextPage,\n\t\t\tPerPage: 100,\n\t\t}\n\t}\n}\n\n// Path: pkg/reporting/trackers/gitlab/gitlab.go\npackage gitlab\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/xanzy/go-gitlab\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n)\n\n// Integration is a client for an issue tracker integration\ntype Integration struct {\n\tclient  *gitlab.Client\n\tuserID  int\n\toptions *Options\n}\n\n// Options contains the configuration options for gitlab issue tracker client\ntype Options struct {\n\t// BaseURL (optional) is the self-hosted gitlab application url\n\tBaseURL string `yaml:\"base-url\" validate:\"omitempty,url\"`\n\t// Username is the username of the gitlab user\n\tUsername string `yaml:\"username\" validate:\"required\"`\n\t// Token is the token for gitlab account.\n\tToken string `yaml:\"token\" validate:\"required\"`\n\t// ProjectName is the name of the repository.\n\tProjectName string `yaml:\"project-name\" validate:\"required\"`\n\t// IssueLabel is the label of the created issue type\n\tIssueLabel string `yaml:\"issue-label\"`\n\t// SeverityAsLabel (optional) sends the severity as the label of the created\n\t// issue.\n\tSeverityAsLabel bool `yaml:\"severity-as-label\"`\n\t// AllowList contains a list of allowed events for this tracker\n\tAllowList *filters.Filter `yaml:\"allow-list\"`\n\t// DenyList contains a list of denied events for this tracker\n\tDenyList *filters.Filter `yaml:\"deny-list\"`\n\t// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest\n\tDuplicateIssueCheck bool `yaml:\"duplicate-issue-check\" default:\"false\"`\n\n\tHttpClient *retryablehttp.Client `yaml:\"-\"`\n\tOmitRaw    bool                  `yaml:\"-\"`\n}\n\n// New creates a new issue tracker integration client based on options.\nfunc New(options *Options) (*Integration, error) {\n\tgitlabOpts := []gitlab.ClientOptionFunc{}\n\tif options.BaseURL != \"\" {\n\t\tgitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL))\n\t}\n\tif options.HttpClient != nil {\n\t\tgitlabOpts = append(gitlabOpts, gitlab.WithHTTPClient(options.HttpClient.HTTPClient))\n\t}\n\tgit, err := gitlab.NewClient(options.Token, gitlabOpts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tuser, _, err := git.Users.CurrentUser()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Integration{client: git, userID: user.ID, options: options}, nil\n}\n\n// CreateIssue creates an issue in the tracker\nfunc (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {\n\tsummary := format.Summary(event)\n\tdescription := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)\n\tlabels := []string{}\n\tseverityLabel := fmt.Sprintf(\"Severity: %s\", event.Info.SeverityHolder.Severity.String())\n\tif i.options.SeverityAsLabel && severityLabel != \"\" {\n\t\tlabels = append(labels, severityLabel)\n\t}\n\tif label := i.options.IssueLabel; label != \"\" {\n\t\tlabels = append(labels, label)\n\t}\n\tcustomLabels := gitlab.Labels(labels)\n\tassigneeIDs := []int{i.userID}\n\tif i.options.DuplicateIssueCheck {\n\t\tsearchIn := \"title\"\n\t\tsearchState := \"all\"\n\t\tissues, _, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{\n\t\t\tIn:     &searchIn,\n\t\t\tState:  &searchState,\n\t\t\tSearch: &summary,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(issues) > 0 {\n\t\t\tissue := issues[0]\n\t\t\t_, _, err := i.client.Notes.CreateIssueNote(i.options.ProjectName, issue.IID, &gitlab.CreateIssueNoteOptions{\n\t\t\t\tBody: &description,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif issue.State == \"closed\" {\n\t\t\t\treopen := \"reopen\"\n\t\t\t\t_, resp, err := i.client.Issues.UpdateIssue(i.options.ProjectName, issue.IID, &gitlab.UpdateIssueOptions{\n\t\t\t\t\tStateEvent: &reopen,\n\t\t\t\t})\n\t\t\t\tfmt.Sprintln(resp, err)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn &filters.CreateIssueResponse{\n\t\t\t\tIssueID:  strconv.FormatInt(int64(issue.ID), 10),\n\t\t\t\tIssueURL: issue.WebURL,\n\t\t\t}, nil\n\t\t}\n\t}\n\tcreatedIssue, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{\n\t\tTitle:       &summary,\n\t\tDescription: &description,\n\t\tLabels:      &customLabels,\n\t\tAssigneeIDs: &assigneeIDs,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &filters.CreateIssueResponse{\n\t\tIssueID:  strconv.FormatInt(int64(createdIssue.ID), 10),\n\t\tIssueURL: createdIssue.WebURL,\n\t}, nil\n}\n\nfunc (i *Integration) Name() string {\n\treturn \"gitlab\"\n}\n\nfunc (i *Integration) CloseIssue(event *output.ResultEvent) error {\n\tsearchIn := \"title\"\n\tsearchState := \"all\"\n\n\tsummary := format.Summary(event)\n\tissues, _, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{\n\t\tIn:     &searchIn,\n\t\tState:  &searchState,\n\t\tSearch: &summary,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(issues) <= 0 {\n\t\treturn nil\n\t}\n\n\tissue := issues[0]\n\tstate := \"close\"\n\t_, _, err = i.client.Issues.UpdateIssue(i.options.ProjectName, issue.IID, &gitlab.UpdateIssueOptions{\n\t\tStateEvent: &state,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// ShouldFilter determines if an issue should be logged to this tracker\nfunc (i *Integration) ShouldFilter(event *output.ResultEvent) bool {\n\tif i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\tif i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// Path: pkg/projectfile/project.go\npackage projectfile\n\nimport (\n\t\"net/http\"\n\t\"regexp\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/hmap/store/hybrid\"\n)\n\nvar (\n\tErrNotFound          = errors.New(\"not found\")\n\tregexUserAgent       = regexp.MustCompile(`(?mi)\\r\\nUser-Agent: .+\\r\\n`)\n\tregexDefaultInteract = regexp.MustCompile(`(?mi)[a-zA-Z1-9%.]+interact.sh`)\n)\n\ntype Options struct {\n\tPath    string\n\tCleanup bool\n}\n\ntype ProjectFile struct {\n\tPath string\n\thm   *hybrid.HybridMap\n}\n\nfunc New(options *Options) (*ProjectFile, error) {\n\tvar p ProjectFile\n\thOptions := hybrid.DefaultDiskOptions\n\thOptions.Path = options.Path\n\thOptions.Cleanup = options.Cleanup\n\tvar err error\n\tp.hm, err = hybrid.New(hOptions)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &p, nil\n}\n\nfunc (pf *ProjectFile) cleanupData(data []byte) []byte {\n\t// ignore all user agents\n\tdata = regexUserAgent.ReplaceAll(data, []byte(\"\\r\\n\"))\n\t// ignore interact markers\n\treturn regexDefaultInteract.ReplaceAll(data, []byte(\"\"))\n}\n\nfunc (pf *ProjectFile) Get(req []byte) (*http.Response, error) {\n\treqHash, err := hash(pf.cleanupData(req))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, ok := pf.hm.Get(reqHash)\n\tif !ok {\n\t\treturn nil, ErrNotFound\n\t}\n\n\tvar httpRecord HTTPRecord\n\thttpRecord.Response = newInternalResponse()\n\tif err := unmarshal(data, &httpRecord); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn fromInternalResponse(httpRecord.Response), nil\n}\n\nfunc (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error {\n\treqHash, err := hash(pf.cleanupData(req))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar httpRecord HTTPRecord\n\thttpRecord.Request = req\n\thttpRecord.Response = toInternalResponse(resp, data)\n\tdata, err = marshal(httpRecord)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn pf.hm.Set(reqHash, data)\n}\n\nfunc (pf *ProjectFile) Close() {\n\tpf.hm.Close()\n}\n\n// Path: pkg/external/customtemplates/gitlab.go\npackage customtemplates\n\nimport (\n\t\"context\"\n\t\"encoding/base64\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n\t\"github.com/xanzy/go-gitlab\"\n)\n\nvar _ Provider = &customTemplateGitLabRepo{}\n\ntype customTemplateGitLabRepo struct {\n\tgitLabClient *gitlab.Client\n\tserverURL    string\n\tprojectIDs   []int\n}\n\n// NewGitLabProviders returns a new list of GitLab providers for downloading custom templates\nfunc NewGitLabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) {\n\tproviders := []*customTemplateGitLabRepo{}\n\tif options.GitLabToken != \"\" && !options.GitLabTemplateDisableDownload {\n\t\t// Establish a connection to GitLab and build a client object with which to download templates from GitLab\n\t\tgitLabClient, err := getGitLabClient(options.GitLabServerURL, options.GitLabToken)\n\t\tif err != nil {\n\t\t\treturn nil, errorutil.NewWithErr(err).Msgf(\"Error establishing GitLab client for %s %s\", options.GitLabServerURL, err)\n\t\t}\n\n\t\t// Create a new GitLab service client\n\t\tgitLabContainer := &customTemplateGitLabRepo{\n\t\t\tgitLabClient: gitLabClient,\n\t\t\tserverURL:    options.GitLabServerURL,\n\t\t\tprojectIDs:   options.GitLabTemplateRepositoryIDs,\n\t\t}\n\n\t\t// Add the GitLab service client to the list of custom templates\n\t\tproviders = append(providers, gitLabContainer)\n\t}\n\treturn providers, nil\n}\n\n// Download downloads all .yaml files from a GitLab repository\nfunc (bk *customTemplateGitLabRepo) Download(_ context.Context) {\n\n\t// Define the project and template count\n\tvar projectCount = 0\n\tvar templateCount = 0\n\n\t// Append the GitLab directory to the location\n\tlocation := config.DefaultConfig.CustomGitLabTemplatesDirectory\n\n\t// Ensure the CustomGitLabTemplateDirectory directory exists or create it if it doesn't yet exist\n\terr := os.MkdirAll(filepath.Dir(location), 0755)\n\tif err != nil {\n\t\tgologger.Error().Msgf(\"Error creating directory: %v\", err)\n\t\treturn\n\t}\n\n\t// Get the projects from the GitLab serverURL\n\tfor _, projectID := range bk.projectIDs {\n\n\t\t// Get the project information from the GitLab serverURL to get the default branch and the project name\n\t\tproject, _, err := bk.gitLabClient.Projects.GetProject(projectID, nil)\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"error retrieving GitLab project: %s %s\", project, err)\n\t\t\treturn\n\t\t}\n\n\t\t// Add a subdirectory with the project ID as the subdirectory within the location\n\t\tprojectOutputPath := filepath.Join(location, project.Path)\n\n\t\t// Ensure the subdirectory exists or create it if it doesn't yet exist\n\t\terr = os.MkdirAll(projectOutputPath, 0755)\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"Error creating subdirectory: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\t// Get the directory listing for the files in the project\n\t\ttree, _, err := bk.gitLabClient.Repositories.ListTree(projectID, &gitlab.ListTreeOptions{\n\t\t\tRef:       gitlab.String(project.DefaultBranch),\n\t\t\tRecursive: gitlab.Bool(true),\n\t\t})\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"error retrieving files from GitLab project: %s (%d) %s\", project.Name, projectID, err)\n\t\t}\n\n\t\t// Loop through the tree and download the files\n\t\tfor _, file := range tree {\n\t\t\t// If the object is not a file or file extension is not .yaml, skip it\n\t\t\tif file.Type == \"blob\" && filepath.Ext(file.Path) == \".yaml\" {\n\t\t\t\tgf := &gitlab.GetFileOptions{\n\t\t\t\t\tRef: gitlab.String(project.DefaultBranch),\n\t\t\t\t}\n\t\t\t\tf, _, err := bk.gitLabClient.RepositoryFiles.GetFile(projectID, file.Path, gf)\n\t\t\t\tif err != nil {\n\t\t\t\t\tgologger.Error().Msgf(\"error retrieving GitLab project file: %d %s\", projectID, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\t// Decode the file content from base64 into bytes so that it can be written to the local filesystem\n\t\t\t\tcontents, err := base64.StdEncoding.DecodeString(f.Content)\n\t\t\t\tif err != nil {\n\t\t\t\t\tgologger.Error().Msgf(\"error decoding GitLab project (%s) file: %s %s\", project.Name, f.FileName, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\t// Write the downloaded template to the local filesystem at the location with the filename of the blob name\n\t\t\t\terr = os.WriteFile(filepath.Join(projectOutputPath, f.FileName), contents, 0644)\n\t\t\t\tif err != nil {\n\t\t\t\t\tgologger.Error().Msgf(\"error writing GitLab project (%s) file: %s %s\", project.Name, f.FileName, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\t// Increment the number of templates downloaded\n\t\t\t\ttemplateCount++\n\t\t\t}\n\t\t}\n\n\t\t// Increment the number of projects downloaded\n\t\tprojectCount++\n\t\tgologger.Info().Msgf(\"GitLab project '%s' (%d) cloned successfully\", project.Name, projectID)\n\t}\n\n\t// Print the number of projects and templates downloaded\n\tgologger.Info().Msgf(\"%d templates downloaded from %d GitLab project(s) to: %s\", templateCount, projectCount, location)\n}\n\n// Update is a wrapper around Download since it doesn't maintain a diff of the templates downloaded versus in the\n// repository for simplicity.\nfunc (bk *customTemplateGitLabRepo) Update(ctx context.Context) {\n\tif len(bk.projectIDs) == 0 {\n\t\t// No projects to download or update\n\t\treturn\n\t}\n\tbk.Download(ctx)\n}\n\n// getGitLabClient returns a GitLab client for the given serverURL and token\nfunc getGitLabClient(server string, token string) (*gitlab.Client, error) {\n\tclient, err := gitlab.NewClient(token, gitlab.WithBaseURL(server))\n\treturn client, err\n}\n\n// Path: pkg/reporting/trackers/jira/jira.go\npackage jira\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/andygrunwald/go-jira\"\n\t\"github.com/trivago/tgo/tcontainer\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n)\n\ntype Formatter struct {\n\tutil.MarkdownFormatter\n}\n\nfunc (jiraFormatter *Formatter) MakeBold(text string) string {\n\treturn \"*\" + text + \"*\"\n}\n\nfunc (jiraFormatter *Formatter) CreateCodeBlock(title string, content string, _ string) string {\n\treturn fmt.Sprintf(\"\\n%s\\n{code}\\n%s\\n{code}\\n\", jiraFormatter.MakeBold(title), content)\n}\n\nfunc (jiraFormatter *Formatter) CreateTable(headers []string, rows [][]string) (string, error) {\n\ttable, err := jiraFormatter.MarkdownFormatter.CreateTable(headers, rows)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\ttableRows := strings.Split(table, \"\\n\")\n\ttableRowsWithoutHeaderSeparator := append(tableRows[:1], tableRows[2:]...)\n\treturn strings.Join(tableRowsWithoutHeaderSeparator, \"\\n\"), nil\n}\n\nfunc (jiraFormatter *Formatter) CreateLink(title string, url string) string {\n\treturn fmt.Sprintf(\"[%s|%s]\", title, url)\n}\n\n// Integration is a client for an issue tracker integration\ntype Integration struct {\n\tFormatter\n\tjira    *jira.Client\n\toptions *Options\n\n\tonce         *sync.Once\n\ttransitionID string\n}\n\n// Options contains the configuration options for jira client\ntype Options struct {\n\t// Cloud value (optional) is set to true when Jira cloud is used\n\tCloud bool `yaml:\"cloud\" json:\"cloud\"`\n\t// UpdateExisting value (optional) if true, the existing opened issue is updated\n\tUpdateExisting bool `yaml:\"update-existing\" json:\"update_existing\"`\n\t// URL is the URL of the jira server\n\tURL string `yaml:\"url\" json:\"url\" validate:\"required\"`\n\t// AccountID is the accountID of the jira user.\n\tAccountID string `yaml:\"account-id\" json:\"account_id\" validate:\"required\"`\n\t// Email is the email of the user for jira instance\n\tEmail string `yaml:\"email\" json:\"email\" validate:\"required,email\"`\n\t// Token is the token for jira instance.\n\tToken string `yaml:\"token\" json:\"token\" validate:\"required\"`\n\t// ProjectName is the name of the project.\n\tProjectName string `yaml:\"project-name\" json:\"project_name\" validate:\"required\"`\n\t// IssueType (optional) is the name of the created issue type\n\tIssueType string `yaml:\"issue-type\" json:\"issue_type\"`\n\t// SeverityAsLabel (optional) sends the severity as the label of the created\n\t// issue.\n\tSeverityAsLabel bool `yaml:\"severity-as-label\" json:\"severity_as_label\"`\n\t// AllowList contains a list of allowed events for this tracker\n\tAllowList *filters.Filter `yaml:\"allow-list\"`\n\t// DenyList contains a list of denied events for this tracker\n\tDenyList *filters.Filter `yaml:\"deny-list\"`\n\t// Severity (optional) is the severity of the issue.\n\tSeverity   []string              `yaml:\"severity\" json:\"severity\"`\n\tHttpClient *retryablehttp.Client `yaml:\"-\" json:\"-\"`\n\t// for each customfield specified in the configuration options\n\t// we will create a map of customfield name to the value\n\t// that will be used to create the issue\n\tCustomFields map[string]interface{} `yaml:\"custom-fields\" json:\"custom_fields\"`\n\tStatusNot    string                 `yaml:\"status-not\" json:\"status_not\"`\n\tOmitRaw      bool                   `yaml:\"-\"`\n}\n\n// New creates a new issue tracker integration client based on options.\nfunc New(options *Options) (*Integration, error) {\n\tusername := options.Email\n\tif !options.Cloud {\n\t\tusername = options.AccountID\n\t}\n\ttp := jira.BasicAuthTransport{\n\t\tUsername: username,\n\t\tPassword: options.Token,\n\t}\n\tif options.HttpClient != nil {\n\t\ttp.Transport = options.HttpClient.HTTPClient.Transport\n\t}\n\tjiraClient, err := jira.NewClient(tp.Client(), options.URL)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tintegration := &Integration{\n\t\tjira:    jiraClient,\n\t\toptions: options,\n\t\tonce:    &sync.Once{},\n\t}\n\treturn integration, nil\n}\n\nfunc (i *Integration) Name() string {\n\treturn \"jira\"\n}\n\n// CreateNewIssue creates a new issue in the tracker\nfunc (i *Integration) CreateNewIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {\n\tsummary := format.Summary(event)\n\tlabels := []string{}\n\tseverityLabel := fmt.Sprintf(\"Severity:%s\", event.Info.SeverityHolder.Severity.String())\n\tif i.options.SeverityAsLabel && severityLabel != \"\" {\n\t\tlabels = append(labels, severityLabel)\n\t}\n\tif label := i.options.IssueType; label != \"\" {\n\t\tlabels = append(labels, label)\n\t}\n\t// for each custom value, take the name of the custom field and\n\t// set the value of the custom field to the value specified in the\n\t// configuration options\n\tcustomFields := tcontainer.NewMarshalMap()\n\tfor name, value := range i.options.CustomFields {\n\t\t//customFields[name] = map[string]interface{}{\"value\": value}\n\t\tif valueMap, ok := value.(map[interface{}]interface{}); ok {\n\t\t\t// Iterate over nested map\n\t\t\tfor nestedName, nestedValue := range valueMap {\n\t\t\t\tfmtNestedValue, ok := nestedValue.(string)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil, fmt.Errorf(`couldn't iterate on nested item \"%s\": %s`, nestedName, nestedValue)\n\t\t\t\t}\n\t\t\t\tif strings.HasPrefix(fmtNestedValue, \"$\") {\n\t\t\t\t\tnestedValue = strings.TrimPrefix(fmtNestedValue, \"$\")\n\t\t\t\t\tswitch nestedValue {\n\t\t\t\t\tcase \"CVSSMetrics\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CVSSMetrics\n\t\t\t\t\tcase \"CVEID\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CVEID\n\t\t\t\t\tcase \"CWEID\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CWEID\n\t\t\t\t\tcase \"CVSSScore\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CVSSScore\n\t\t\t\t\tcase \"Host\":\n\t\t\t\t\t\tnestedValue = event.Host\n\t\t\t\t\tcase \"Severity\":\n\t\t\t\t\t\tnestedValue = event.Info.SeverityHolder\n\t\t\t\t\tcase \"Name\":\n\t\t\t\t\t\tnestedValue = event.Info.Name\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tswitch nestedName {\n\t\t\t\tcase \"id\":\n\t\t\t\t\tcustomFields[name] = map[string]interface{}{\"id\": nestedValue}\n\t\t\t\tcase \"name\":\n\t\t\t\t\tcustomFields[name] = map[string]interface{}{\"value\": nestedValue}\n\t\t\t\tcase \"freeform\":\n\t\t\t\t\tcustomFields[name] = nestedValue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tfields := &jira.IssueFields{\n\t\tAssignee:    &jira.User{Name: i.options.AccountID},\n\t\tDescription: format.CreateReportDescription(event, i, i.options.OmitRaw),\n\t\tUnknowns:    customFields,\n\t\tLabels:      labels,\n\t\tType:        jira.IssueType{Name: i.options.IssueType},\n\t\tProject:     jira.Project{Key: i.options.ProjectName},\n\t\tSummary:     summary,\n\t}\n\t// On-prem version of Jira server does not use AccountID\n\tif !i.options.Cloud {\n\t\tfields = &jira.IssueFields{\n\t\t\tAssignee:    &jira.User{Name: i.options.AccountID},\n\t\t\tDescription: format.CreateReportDescription(event, i, i.options.OmitRaw),\n\t\t\tType:        jira.IssueType{Name: i.options.IssueType},\n\t\t\tProject:     jira.Project{Key: i.options.ProjectName},\n\t\t\tSummary:     summary,\n\t\t\tLabels:      labels,\n\t\t\tUnknowns:    customFields,\n\t\t}\n\t}\n\n\tissueData := &jira.Issue{\n\t\tFields: fields,\n\t}\n\tcreatedIssue, resp, err := i.jira.Issue.Create(issueData)\n\tif err != nil {\n\t\tvar data string\n\t\tif resp != nil && resp.Body != nil {\n\t\t\td, _ := io.ReadAll(resp.Body)\n\t\t\tdata = string(d)\n\t\t}\n\t\treturn nil, fmt.Errorf(\"%w => %s\", err, data)\n...\n", "needle_token_start": 631, "needle_token_end": 900, "code_context_ntokens": 15242, "output": ["```go\n// getTemplateDirs returns template directories for given input\n// by default it returns default template directory\n\nfunc getTemplateDirs(opts Options) ([]string, error) {\n\tdefaultTemplatesDirectories := []string{config.DefaultConfig.GetTemplateDir()}\n\t// adding custom template path if available\n\tif len(opts.ExecuterOpts.Options.Templates) > 0 {\n\t\tdefaultTemplatesDirectories = append(defaultTemplatesDirectories, opts.ExecuterOpts.Options.Templates...)\n\t}\n\t// Collect path for default directories we want to look for templates in\n\tvar allTemplates []string\n\tfor _, directory := range defaultTemplatesDirectories {\n\t\ttemplates, err := opts.ExecuterOpts.Catalog.GetTemplatePath(directory)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"could not get templates in directory\")\n\t\t}\n\t\tallTemplates = append(allTemplates, templates...)\n\t}\n\tallTemplates = sliceutil.Dedupe(allTemplates)\n\tif len(allTemplates) == 0 {\n\t\treturn nil, errors.New(\"no templates found for given input\")\n\t}\n\treturn allTemplates, nil\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "NewGitLabProviders", "language": "go", "path": "pkg/external/customtemplates/gitlab.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a list of service clients for downloading custom templates from a version control platform.\n2. **Input**: It takes a configuration object containing a server URL, an access token, and a list of project IDs.\n3. **Output**: It returns a list of service clients configured to interact with the specified version control platform, or an error if the connection or client setup fails.\n4. **Procedure**: The function first checks if the token is provided and downloading is enabled. If conditions are met, it attempts to establish a connection to the version control platform using the provided server URL and token. If successful, it creates a service client for each project ID specified in the configuration, and then adds these clients to the list to be returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/reporting/trackers/gitlab/gitlab.go\npackage gitlab\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\n\t\"github.com/xanzy/go-gitlab\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n)\n\n// Integration is a client for an issue tracker integration\ntype Integration struct {\n\tclient  *gitlab.Client\n\tuserID  int\n\toptions *Options\n}\n\n// Options contains the configuration options for gitlab issue tracker client\ntype Options struct {\n\t// BaseURL (optional) is the self-hosted gitlab application url\n\tBaseURL string `yaml:\"base-url\" validate:\"omitempty,url\"`\n\t// Username is the username of the gitlab user\n\tUsername string `yaml:\"username\" validate:\"required\"`\n\t// Token is the token for gitlab account.\n\tToken string `yaml:\"token\" validate:\"required\"`\n\t// ProjectName is the name of the repository.\n\tProjectName string `yaml:\"project-name\" validate:\"required\"`\n\t// IssueLabel is the label of the created issue type\n\tIssueLabel string `yaml:\"issue-label\"`\n\t// SeverityAsLabel (optional) sends the severity as the label of the created\n\t// issue.\n\tSeverityAsLabel bool `yaml:\"severity-as-label\"`\n\t// AllowList contains a list of allowed events for this tracker\n\tAllowList *filters.Filter `yaml:\"allow-list\"`\n\t// DenyList contains a list of denied events for this tracker\n\tDenyList *filters.Filter `yaml:\"deny-list\"`\n\t// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest\n\tDuplicateIssueCheck bool `yaml:\"duplicate-issue-check\" default:\"false\"`\n\n\tHttpClient *retryablehttp.Client `yaml:\"-\"`\n\tOmitRaw    bool                  `yaml:\"-\"`\n}\n\n// New creates a new issue tracker integration client based on options.\nfunc New(options *Options) (*Integration, error) {\n\tgitlabOpts := []gitlab.ClientOptionFunc{}\n\tif options.BaseURL != \"\" {\n\t\tgitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL))\n\t}\n\tif options.HttpClient != nil {\n\t\tgitlabOpts = append(gitlabOpts, gitlab.WithHTTPClient(options.HttpClient.HTTPClient))\n\t}\n\tgit, err := gitlab.NewClient(options.Token, gitlabOpts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tuser, _, err := git.Users.CurrentUser()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Integration{client: git, userID: user.ID, options: options}, nil\n}\n\n// CreateIssue creates an issue in the tracker\nfunc (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {\n\tsummary := format.Summary(event)\n\tdescription := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)\n\tlabels := []string{}\n\tseverityLabel := fmt.Sprintf(\"Severity: %s\", event.Info.SeverityHolder.Severity.String())\n\tif i.options.SeverityAsLabel && severityLabel != \"\" {\n\t\tlabels = append(labels, severityLabel)\n\t}\n\tif label := i.options.IssueLabel; label != \"\" {\n\t\tlabels = append(labels, label)\n\t}\n\tcustomLabels := gitlab.Labels(labels)\n\tassigneeIDs := []int{i.userID}\n\tif i.options.DuplicateIssueCheck {\n\t\tsearchIn := \"title\"\n\t\tsearchState := \"all\"\n\t\tissues, _, err := i.client.Issues.ListProjectIssues(i.options.ProjectName, &gitlab.ListProjectIssuesOptions{\n\t\t\tIn:     &searchIn,\n\t\t\tState:  &searchState,\n\t\t\tSearch: &summary,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(issues) > 0 {\n\t\t\tissue := issues[0]\n\t\t\t_, _, err := i.client.Notes.CreateIssueNote(i.options.ProjectName, issue.IID, &gitlab.CreateIssueNoteOptions{\n\t\t\t\tBody: &description,\n...\n// Path: pkg/projectfile/project.go\npackage projectfile\n\nimport (\n\t\"net/http\"\n\t\"regexp\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/hmap/store/hybrid\"\n)\n\nvar (\n\tErrNotFound          = errors.New(\"not found\")\n\tregexUserAgent       = regexp.MustCompile(`(?mi)\\r\\nUser-Agent: .+\\r\\n`)\n\tregexDefaultInteract = regexp.MustCompile(`(?mi)[a-zA-Z1-9%.]+interact.sh`)\n)\n\ntype Options struct {\n\tPath    string\n\tCleanup bool\n}\n\ntype ProjectFile struct {\n\tPath string\n\thm   *hybrid.HybridMap\n}\n\nfunc New(options *Options) (*ProjectFile, error) {\n\tvar p ProjectFile\n\thOptions := hybrid.DefaultDiskOptions\n\thOptions.Path = options.Path\n\thOptions.Cleanup = options.Cleanup\n\tvar err error\n\tp.hm, err = hybrid.New(hOptions)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &p, nil\n}\n\nfunc (pf *ProjectFile) cleanupData(data []byte) []byte {\n\t// ignore all user agents\n\tdata = regexUserAgent.ReplaceAll(data, []byte(\"\\r\\n\"))\n\t// ignore interact markers\n\treturn regexDefaultInteract.ReplaceAll(data, []byte(\"\"))\n}\n\nfunc (pf *ProjectFile) Get(req []byte) (*http.Response, error) {\n\treqHash, err := hash(pf.cleanupData(req))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdata, ok := pf.hm.Get(reqHash)\n\tif !ok {\n\t\treturn nil, ErrNotFound\n\t}\n\n\tvar httpRecord HTTPRecord\n\thttpRecord.Response = newInternalResponse()\n\tif err := unmarshal(data, &httpRecord); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn fromInternalResponse(httpRecord.Response), nil\n}\n\nfunc (pf *ProjectFile) Set(req []byte, resp *http.Response, data []byte) error {\n\treqHash, err := hash(pf.cleanupData(req))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar httpRecord HTTPRecord\n\thttpRecord.Request = req\n\thttpRecord.Response = toInternalResponse(resp, data)\n\tdata, err = marshal(httpRecord)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn pf.hm.Set(reqHash, data)\n}\n\nfunc (pf *ProjectFile) Close() {\n\tpf.hm.Close()\n}\n\n// Path: pkg/external/customtemplates/gitlab.go\npackage customtemplates\n\nimport (\n\t\"context\"\n\t\"encoding/base64\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n\t\"github.com/xanzy/go-gitlab\"\n)\n\nvar _ Provider = &customTemplateGitLabRepo{}\n\ntype customTemplateGitLabRepo struct {\n\tgitLabClient *gitlab.Client\n\tserverURL    string\n\tprojectIDs   []int\n}\n\n// NewGitLabProviders returns a new list of GitLab providers for downloading custom templates\n\nfunc NewGitLabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) {\n\tproviders := []*customTemplateGitLabRepo{}\n\tif options.GitLabToken != \"\" && !options.GitLabTemplateDisableDownload {\n\t\t// Establish a connection to GitLab and build a client object with which to download templates from GitLab\n\t\tgitLabClient, err := getGitLabClient(options.GitLabServerURL, options.GitLabToken)\n\t\tif err != nil {\n\t\t\treturn nil, errorutil.NewWithErr(err).Msgf(\"Error establishing GitLab client for %s %s\", options.GitLabServerURL, err)\n\t\t}\n\n\t\t// Create a new GitLab service client\n\t\tgitLabContainer := &customTemplateGitLabRepo{\n\t\t\tgitLabClient: gitLabClient,\n\t\t\tserverURL:    options.GitLabServerURL,\n\t\t\tprojectIDs:   options.GitLabTemplateRepositoryIDs,\n\t\t}\n\n\t\t// Add the GitLab service client to the list of custom templates\n\t\tproviders = append(providers, gitLabContainer)\n\t}\n\treturn providers, nil\n}\n\n// Download downloads all .yaml files from a GitLab repository\nfunc (bk *customTemplateGitLabRepo) Download(_ context.Context) {\n\n\t// Define the project and template count\n\tvar projectCount = 0\n\tvar templateCount = 0\n\n\t// Append the GitLab directory to the location\n\tlocation := config.DefaultConfig.CustomGitLabTemplatesDirectory\n\n\t// Ensure the CustomGitLabTemplateDirectory directory exists or create it if it doesn't yet exist\n\terr := os.MkdirAll(filepath.Dir(location), 0755)\n\tif err != nil {\n\t\tgologger.Error().Msgf(\"Error creating directory: %v\", err)\n\t\treturn\n\t}\n\n\t// Get the projects from the GitLab serverURL\n\tfor _, projectID := range bk.projectIDs {\n\n\t\t// Get the project information from the GitLab serverURL to get the default branch and the project name\n\t\tproject, _, err := bk.gitLabClient.Projects.GetProject(projectID, nil)\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"error retrieving GitLab project: %s %s\", project, err)\n\t\t\treturn\n\t\t}\n\n\t\t// Add a subdirectory with the project ID as the subdirectory within the location\n\t\tprojectOutputPath := filepath.Join(location, project.Path)\n\n\t\t// Ensure the subdirectory exists or create it if it doesn't yet exist\n\t\terr = os.MkdirAll(projectOutputPath, 0755)\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"Error creating subdirectory: %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\t// Get the directory listing for the files in the project\n\t\ttree, _, err := bk.gitLabClient.Repositories.ListTree(projectID, &gitlab.ListTreeOptions{\n\t\t\tRef:       gitlab.String(project.DefaultBranch),\n\t\t\tRecursive: gitlab.Bool(true),\n\t\t})\n\t\tif err != nil {\n\t\t\tgologger.Error().Msgf(\"error retrieving files from GitLab project: %s (%d) %s\", project.Name, projectID, err)\n\t\t}\n\n\t\t// Loop through the tree and download the files\n\t\tfor _, file := range tree {\n\t\t\t// If the object is not a file or file extension is not .yaml, skip it\n\t\t\tif file.Type == \"blob\" && filepath.Ext(file.Path) == \".yaml\" {\n\t\t\t\tgf := &gitlab.GetFileOptions{\n\t\t\t\t\tRef: gitlab.String(project.DefaultBranch),\n\t\t\t\t}\n\t\t\t\tf, _, err := bk.gitLabClient.RepositoryFiles.GetFile(projectID, file.Path, gf)\n\t\t\t\tif err != nil {\n\t\t\t\t\tgologger.Error().Msgf(\"error retrieving GitLab project file: %d %s\", projectID, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\t// Decode the file content from base64 into bytes so that it can be written to the local filesystem\n\t\t\t\tcontents, err := base64.StdEncoding.DecodeString(f.Content)\n\t\t\t\tif err != nil {\n\t\t\t\t\tgologger.Error().Msgf(\"error decoding GitLab project (%s) file: %s %s\", project.Name, f.FileName, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\t// Write the downloaded template to the local filesystem at the location with the filename of the blob name\n\t\t\t\terr = os.WriteFile(filepath.Join(projectOutputPath, f.FileName), contents, 0644)\n\t\t\t\tif err != nil {\n\t\t\t\t\tgologger.Error().Msgf(\"error writing GitLab project (%s) file: %s %s\", project.Name, f.FileName, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\t// Increment the number of templates downloaded\n\t\t\t\ttemplateCount++\n\t\t\t}\n\t\t}\n\n\t\t// Increment the number of projects downloaded\n\t\tprojectCount++\n\t\tgologger.Info().Msgf(\"GitLab project '%s' (%d) cloned successfully\", project.Name, projectID)\n\t}\n\n\t// Print the number of projects and templates downloaded\n\tgologger.Info().Msgf(\"%d templates downloaded from %d GitLab project(s) to: %s\", templateCount, projectCount, location)\n}\n\n// Update is a wrapper around Download since it doesn't maintain a diff of the templates downloaded versus in the\n// repository for simplicity.\nfunc (bk *customTemplateGitLabRepo) Update(ctx context.Context) {\n\tif len(bk.projectIDs) == 0 {\n\t\t// No projects to download or update\n\t\treturn\n\t}\n\tbk.Download(ctx)\n}\n\n// getGitLabClient returns a GitLab client for the given serverURL and token\nfunc getGitLabClient(server string, token string) (*gitlab.Client, error) {\n\tclient, err := gitlab.NewClient(token, gitlab.WithBaseURL(server))\n\treturn client, err\n}\n\n// Path: pkg/reporting/trackers/jira/jira.go\npackage jira\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/andygrunwald/go-jira\"\n\t\"github.com/trivago/tgo/tcontainer\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n)\n\ntype Formatter struct {\n\tutil.MarkdownFormatter\n}\n\nfunc (jiraFormatter *Formatter) MakeBold(text string) string {\n\treturn \"*\" + text + \"*\"\n}\n\nfunc (jiraFormatter *Formatter) CreateCodeBlock(title string, content string, _ string) string {\n\treturn fmt.Sprintf(\"\\n%s\\n{code}\\n%s\\n{code}\\n\", jiraFormatter.MakeBold(title), content)\n}\n\nfunc (jiraFormatter *Formatter) CreateTable(headers []string, rows [][]string) (string, error) {\n\ttable, err := jiraFormatter.MarkdownFormatter.CreateTable(headers, rows)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\ttableRows := strings.Split(table, \"\\n\")\n\ttableRowsWithoutHeaderSeparator := append(tableRows[:1], tableRows[2:]...)\n\treturn strings.Join(tableRowsWithoutHeaderSeparator, \"\\n\"), nil\n}\n\nfunc (jiraFormatter *Formatter) CreateLink(title string, url string) string {\n\treturn fmt.Sprintf(\"[%s|%s]\", title, url)\n}\n\n// Integration is a client for an issue tracker integration\ntype Integration struct {\n\tFormatter\n\tjira    *jira.Client\n\toptions *Options\n\n\tonce         *sync.Once\n\ttransitionID string\n}\n\n// Options contains the configuration options for jira client\ntype Options struct {\n\t// Cloud value (optional) is set to true when Jira cloud is used\n\tCloud bool `yaml:\"cloud\" json:\"cloud\"`\n\t// UpdateExisting value (optional) if true, the existing opened issue is updated\n\tUpdateExisting bool `yaml:\"update-existing\" json:\"update_existing\"`\n\t// URL is the URL of the jira server\n\tURL string `yaml:\"url\" json:\"url\" validate:\"required\"`\n\t// AccountID is the accountID of the jira user.\n\tAccountID string `yaml:\"account-id\" json:\"account_id\" validate:\"required\"`\n\t// Email is the email of the user for jira instance\n\tEmail string `yaml:\"email\" json:\"email\" validate:\"required,email\"`\n\t// Token is the token for jira instance.\n\tToken string `yaml:\"token\" json:\"token\" validate:\"required\"`\n\t// ProjectName is the name of the project.\n\tProjectName string `yaml:\"project-name\" json:\"project_name\" validate:\"required\"`\n\t// IssueType (optional) is the name of the created issue type\n\tIssueType string `yaml:\"issue-type\" json:\"issue_type\"`\n\t// SeverityAsLabel (optional) sends the severity as the label of the created\n\t// issue.\n\tSeverityAsLabel bool `yaml:\"severity-as-label\" json:\"severity_as_label\"`\n\t// AllowList contains a list of allowed events for this tracker\n\tAllowList *filters.Filter `yaml:\"allow-list\"`\n\t// DenyList contains a list of denied events for this tracker\n\tDenyList *filters.Filter `yaml:\"deny-list\"`\n\t// Severity (optional) is the severity of the issue.\n\tSeverity   []string              `yaml:\"severity\" json:\"severity\"`\n\tHttpClient *retryablehttp.Client `yaml:\"-\" json:\"-\"`\n\t// for each customfield specified in the configuration options\n\t// we will create a map of customfield name to the value\n\t// that will be used to create the issue\n\tCustomFields map[string]interface{} `yaml:\"custom-fields\" json:\"custom_fields\"`\n\tStatusNot    string                 `yaml:\"status-not\" json:\"status_not\"`\n\tOmitRaw      bool                   `yaml:\"-\"`\n}\n\n// New creates a new issue tracker integration client based on options.\nfunc New(options *Options) (*Integration, error) {\n\tusername := options.Email\n\tif !options.Cloud {\n\t\tusername = options.AccountID\n\t}\n\ttp := jira.BasicAuthTransport{\n\t\tUsername: username,\n\t\tPassword: options.Token,\n\t}\n\tif options.HttpClient != nil {\n\t\ttp.Transport = options.HttpClient.HTTPClient.Transport\n\t}\n\tjiraClient, err := jira.NewClient(tp.Client(), options.URL)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tintegration := &Integration{\n\t\tjira:    jiraClient,\n\t\toptions: options,\n\t\tonce:    &sync.Once{},\n\t}\n\treturn integration, nil\n}\n\nfunc (i *Integration) Name() string {\n\treturn \"jira\"\n}\n\n// CreateNewIssue creates a new issue in the tracker\nfunc (i *Integration) CreateNewIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {\n\tsummary := format.Summary(event)\n\tlabels := []string{}\n\tseverityLabel := fmt.Sprintf(\"Severity:%s\", event.Info.SeverityHolder.Severity.String())\n\tif i.options.SeverityAsLabel && severityLabel != \"\" {\n\t\tlabels = append(labels, severityLabel)\n\t}\n\tif label := i.options.IssueType; label != \"\" {\n\t\tlabels = append(labels, label)\n\t}\n\t// for each custom value, take the name of the custom field and\n\t// set the value of the custom field to the value specified in the\n\t// configuration options\n\tcustomFields := tcontainer.NewMarshalMap()\n\tfor name, value := range i.options.CustomFields {\n\t\t//customFields[name] = map[string]interface{}{\"value\": value}\n\t\tif valueMap, ok := value.(map[interface{}]interface{}); ok {\n\t\t\t// Iterate over nested map\n\t\t\tfor nestedName, nestedValue := range valueMap {\n\t\t\t\tfmtNestedValue, ok := nestedValue.(string)\n\t\t\t\tif !ok {\n\t\t\t\t\treturn nil, fmt.Errorf(`couldn't iterate on nested item \"%s\": %s`, nestedName, nestedValue)\n\t\t\t\t}\n\t\t\t\tif strings.HasPrefix(fmtNestedValue, \"$\") {\n\t\t\t\t\tnestedValue = strings.TrimPrefix(fmtNestedValue, \"$\")\n\t\t\t\t\tswitch nestedValue {\n\t\t\t\t\tcase \"CVSSMetrics\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CVSSMetrics\n\t\t\t\t\tcase \"CVEID\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CVEID\n\t\t\t\t\tcase \"CWEID\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CWEID\n\t\t\t\t\tcase \"CVSSScore\":\n\t\t\t\t\t\tnestedValue = event.Info.Classification.CVSSScore\n\t\t\t\t\tcase \"Host\":\n\t\t\t\t\t\tnestedValue = event.Host\n\t\t\t\t\tcase \"Severity\":\n\t\t\t\t\t\tnestedValue = event.Info.SeverityHolder\n\t\t\t\t\tcase \"Name\":\n\t\t\t\t\t\tnestedValue = event.Info.Name\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tswitch nestedName {\n\t\t\t\tcase \"id\":\n\t\t\t\t\tcustomFields[name] = map[string]interface{}{\"id\": nestedValue}\n\t\t\t\tcase \"name\":\n\t\t\t\t\tcustomFields[name] = map[string]interface{}{\"value\": nestedValue}\n\t\t\t\tcase \"freeform\":\n\t\t\t\t\tcustomFields[name] = nestedValue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tfields := &jira.IssueFields{\n\t\tAssignee:    &jira.User{Name: i.options.AccountID},\n\t\tDescription: format.CreateReportDescription(event, i, i.options.OmitRaw),\n\t\tUnknowns:    customFields,\n\t\tLabels:      labels,\n\t\tType:        jira.IssueType{Name: i.options.IssueType},\n\t\tProject:     jira.Project{Key: i.options.ProjectName},\n\t\tSummary:     summary,\n\t}\n\t// On-prem version of Jira server does not use AccountID\n\tif !i.options.Cloud {\n\t\tfields = &jira.IssueFields{\n\t\t\tAssignee:    &jira.User{Name: i.options.AccountID},\n\t\t\tDescription: format.CreateReportDescription(event, i, i.options.OmitRaw),\n\t\t\tType:        jira.IssueType{Name: i.options.IssueType},\n\t\t\tProject:     jira.Project{Key: i.options.ProjectName},\n\t\t\tSummary:     summary,\n\t\t\tLabels:      labels,\n\t\t\tUnknowns:    customFields,\n\t\t}\n\t}\n\n\tissueData := &jira.Issue{\n\t\tFields: fields,\n\t}\n\tcreatedIssue, resp, err := i.jira.Issue.Create(issueData)\n\tif err != nil {\n\t\tvar data string\n\t\tif resp != nil && resp.Body != nil {\n\t\t\td, _ := io.ReadAll(resp.Body)\n\t\t\tdata = string(d)\n\t\t}\n\t\treturn nil, fmt.Errorf(\"%w => %s\", err, data)\n\t}\n\treturn getIssueResponseFromJira(createdIssue)\n}\n\nfunc getIssueResponseFromJira(issue *jira.Issue) (*filters.CreateIssueResponse, error) {\n\tparsed, err := url.Parse(issue.Self)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tparsed.Path = fmt.Sprintf(\"/browse/%s\", issue.Key)\n\tissueURL := parsed.String()\n\n\treturn &filters.CreateIssueResponse{\n\t\tIssueID:  issue.ID,\n\t\tIssueURL: issueURL,\n\t}, nil\n}\n\n// CreateIssue creates an issue in the tracker or updates the existing one\nfunc (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {\n\tif i.options.UpdateExisting {\n\t\tissue, err := i.FindExistingIssue(event)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t} else if issue.ID != \"\" {\n\t\t\t_, _, err = i.jira.Issue.AddComment(issue.ID, &jira.Comment{\n\t\t\t\tBody: format.CreateReportDescription(event, i, i.options.OmitRaw),\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\treturn getIssueResponseFromJira(&issue)\n\t\t}\n\t}\n\treturn i.CreateNewIssue(event)\n}\n\nfunc (i *Integration) CloseIssue(event *output.ResultEvent) error {\n\tif i.options.StatusNot == \"\" {\n\t\treturn nil\n\t}\n\n\tissue, err := i.FindExistingIssue(event)\n\tif err != nil {\n\t\treturn err\n\t} else if issue.ID != \"\" {\n\t\t// Lazy load the transitions ID in case it's not set\n\t\ti.once.Do(func() {\n\t\t\ttransitions, _, err := i.jira.Issue.GetTransitions(issue.ID)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor _, transition := range transitions {\n\t\t\t\tif transition.Name == i.options.StatusNot {\n\t\t\t\t\ti.transitionID = transition.ID\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t\tif i.transitionID == \"\" {\n\t\t\treturn nil\n\t\t}\n\t\ttransition := jira.CreateTransitionPayload{\n\t\t\tTransition: jira.TransitionPayload{\n\t\t\t\tID: i.transitionID,\n\t\t\t},\n\t\t}\n\n\t\t_, err = i.jira.Issue.DoTransitionWithPayload(issue.ID, transition)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// FindExistingIssue checks if the issue already exists and returns its ID\nfunc (i *Integration) FindExistingIssue(event *output.ResultEvent) (jira.Issue, error) {\n\ttemplate := format.GetMatchedTemplateName(event)\n\tjql := fmt.Sprintf(\"summary ~ \\\"%s\\\" AND summary ~ \\\"%s\\\" AND status != \\\"%s\\\" AND project = \\\"%s\\\"\", template, event.Host, i.options.StatusNot, i.options.ProjectName)\n\n\tsearchOptions := &jira.SearchOptions{\n\t\tMaxResults: 1, // if any issue exists, then we won't create a new one\n\t}\n\n\tchunk, resp, err := i.jira.Issue.Search(jql, searchOptions)\n\tif err != nil {\n\t\tvar data string\n\t\tif resp != nil && resp.Body != nil {\n\t\t\td, _ := io.ReadAll(resp.Body)\n\t\t\tdata = string(d)\n\t\t}\n\t\treturn jira.Issue{}, fmt.Errorf(\"%w => %s\", err, data)\n\t}\n\n\tswitch resp.Total {\n\tcase 0:\n\t\treturn jira.Issue{}, nil\n\tcase 1:\n\t\treturn chunk[0], nil\n\tdefault:\n\t\tgologger.Warning().Msgf(\"Discovered multiple opened issues %s for the host %s: The issue [%s] will be updated.\", template, event.Host, chunk[0].ID)\n\t\treturn chunk[0], nil\n\t}\n}\n\n// ShouldFilter determines if an issue should be logged to this tracker\nfunc (i *Integration) ShouldFilter(event *output.ResultEvent) bool {\n\tif i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\tif i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// Path: pkg/reporting/exporters/jsonl/jsonl.go\npackage jsonl\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n)\n\ntype Exporter struct {\n\toptions *Options\n\tmutex   *sync.Mutex\n\trows    []output.ResultEvent\n}\n\n// Options contains the configuration options for JSONL exporter client\ntype Options struct {\n\t// File is the file to export found JSONL result to\n\tFile    string `yaml:\"file\"`\n\tOmitRaw bool   `yaml:\"omit-raw\"`\n}\n\n// New creates a new JSONL exporter integration client based on options.\nfunc New(options *Options) (*Exporter, error) {\n\texporter := &Exporter{\n\t\tmutex:   &sync.Mutex{},\n\t\toptions: options,\n\t\trows:    []output.ResultEvent{},\n\t}\n\treturn exporter, nil\n}\n\n// Export appends the passed result event to the list of objects to be exported to\n// the resulting JSONL file\nfunc (exporter *Exporter) Export(event *output.ResultEvent) error {\n\texporter.mutex.Lock()\n\tdefer exporter.mutex.Unlock()\n\n\tif exporter.options.OmitRaw {\n\t\tevent.Request = \"\"\n\t\tevent.Response = \"\"\n\t}\n\n\t// Add the event to the rows\n\texporter.rows = append(exporter.rows, *event)\n\n\treturn nil\n}\n\n// Close writes the in-memory data to the JSONL file specified by options.JSONLExport\n// and closes the exporter after operation\nfunc (exporter *Exporter) Close() error {\n\texporter.mutex.Lock()\n\tdefer exporter.mutex.Unlock()\n\n\t// Open the JSONL file for writing and create it if it doesn't exist\n\tf, err := os.OpenFile(exporter.options.File, os.O_WRONLY|os.O_CREATE, 0644)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to create JSONL file\")\n\t}\n\n\t// Loop through the rows and convert each to a JSON byte array and write to file\n\tfor _, row := range exporter.rows {\n\t\t// Convert the row to JSON byte array and append a trailing newline. This is treated as a single line in JSONL\n\t\tobj, err := json.Marshal(row)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to generate row for JSONL report\")\n\t\t}\n\n\t\t// Add a trailing newline to the JSON byte array to confirm with the JSONL format\n\t\tobj = append(obj, '\\n')\n\n\t\t// Attempt to append the JSON line to file specified in options.JSONLExport\n\t\tif _, err = f.Write(obj); err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to append JSONL line\")\n\t\t}\n\t}\n\n\t// Close the file\n\tif err := f.Close(); err != nil {\n\t\treturn errors.Wrap(err, \"failed to close JSONL file\")\n\t}\n\n\treturn nil\n}\n\n// Path: pkg/reporting/dedupe/dedupe.go\n// Package dedupe implements deduplication layer for nuclei-generated\n// issues.\n//\n// The layer can be persisted to leveldb based storage for further use.\npackage dedupe\n\nimport (\n\t\"crypto/sha1\"\n\t\"os\"\n\n\t\"github.com/syndtr/goleveldb/leveldb\"\n\t\"github.com/syndtr/goleveldb/leveldb/errors\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\t\"github.com/projectdiscovery/utils/conversion\"\n)\n\n// Storage is a duplicate detecting storage for nuclei scan events.\ntype Storage struct {\n\ttemporary string\n\tstorage   *leveldb.DB\n}\n\n// New creates a new duplicate detecting storage for nuclei scan events.\nfunc New(dbPath string) (*Storage, error) {\n\tstorage := &Storage{}\n\n\tvar err error\n\tif dbPath == \"\" {\n\t\tdbPath, err = os.MkdirTemp(\"\", \"nuclei-report-*\")\n\t\tstorage.temporary = dbPath\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tstorage.storage, err = leveldb.OpenFile(dbPath, nil)\n\tif err != nil {\n\t\tif !errors.IsCorrupted(err) {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// If the metadata is corrupted, try to recover\n\t\tstorage.storage, err = leveldb.RecoverFile(dbPath, nil)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn storage, nil\n}\n\nfunc (s *Storage) Clear() {\n\tvar keys [][]byte\n\titer := s.storage.NewIterator(nil, nil)\n\tfor iter.Next() {\n\t\tkeys = append(keys, iter.Key())\n\t}\n\titer.Release()\n\tfor _, key := range keys {\n\t\t_ = s.storage.Delete(key, nil)\n\t}\n}\n\n// Close closes the storage for further operations\nfunc (s *Storage) Close() {\n\ts.storage.Close()\n\tif s.temporary != \"\" {\n\t\tos.RemoveAll(s.temporary)\n\t}\n}\n\n// Index indexes an item in storage and returns true if the item\n// was unique.\nfunc (s *Storage) Index(result *output.ResultEvent) (bool, error) {\n\thasher := sha1.New()\n\tif result.TemplateID != \"\" {\n\t\t_, _ = hasher.Write(conversion.Bytes(result.TemplateID))\n\t}\n\tif result.MatcherName != \"\" {\n\t\t_, _ = hasher.Write(conversion.Bytes(result.MatcherName))\n\t}\n\tif result.ExtractorName != \"\" {\n\t\t_, _ = hasher.Write(conversion.Bytes(result.ExtractorName))\n\t}\n\tif result.Type != \"\" {\n\t\t_, _ = hasher.Write(conversion.Bytes(result.Type))\n\t}\n\tif result.Host != \"\" {\n\t\t_, _ = hasher.Write(conversion.Bytes(result.Host))\n\t}\n\tif result.Matched != \"\" {\n\t\t_, _ = hasher.Write(conversion.Bytes(result.Matched))\n\t}\n\tfor _, v := range result.ExtractedResults {\n\t\t_, _ = hasher.Write(conversion.Bytes(v))\n\t}\n\tfor k, v := range result.Metadata {\n\t\t_, _ = hasher.Write(conversion.Bytes(k))\n\t\t_, _ = hasher.Write(conversion.Bytes(types.ToString(v)))\n\t}\n\thash := hasher.Sum(nil)\n\n\texists, err := s.storage.Has(hash, nil)\n\tif err != nil {\n\t\t// if we have an error, return with it but mark it as true\n\t\t// since we don't want to lose an issue considering it a dupe.\n\t\treturn true, err\n\t}\n\tif !exists {\n\t\treturn true, s.storage.Put(hash, nil, nil)\n\t}\n\treturn false, err\n}\n\n// Path: pkg/tmplexec/flow/builtin/dedupe.go\npackage builtin\n\nimport (\n\t\"crypto/md5\"\n\t\"reflect\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\n// Dedupe is a javascript builtin for deduping values\ntype Dedupe struct {\n\tm  map[string]goja.Value\n\tVM *goja.Runtime\n}\n\n// Add adds a value to the dedupe\nfunc (d *Dedupe) Add(call goja.FunctionCall) goja.Value {\n\tallVars := []any{}\n\tfor _, v := range call.Arguments {\n\t\tif v.Export() == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif v.ExportType().Kind() == reflect.Slice {\n\t\t\t// convert []datatype to []interface{}\n\t\t\t// since it cannot be type asserted to []interface{} directly\n\t\t\trfValue := reflect.ValueOf(v.Export())\n\t\t\tfor i := 0; i < rfValue.Len(); i++ {\n\t\t\t\tallVars = append(allVars, rfValue.Index(i).Interface())\n\t\t\t}\n\t\t} else {\n\t\t\tallVars = append(allVars, v.Export())\n\t\t}\n\t}\n\tfor _, v := range allVars {\n\t\thash := hashValue(v)\n\t\tif _, ok := d.m[hash]; ok {\n\t\t\tcontinue\n\t\t}\n\t\td.m[hash] = d.VM.ToValue(v)\n\t}\n\treturn d.VM.ToValue(true)\n}\n\n// Values returns all values from the dedupe\nfunc (d *Dedupe) Values(call goja.FunctionCall) goja.Value {\n\ttmp := []goja.Value{}\n\tfor _, v := range d.m {\n\t\ttmp = append(tmp, v)\n\t}\n\treturn d.VM.ToValue(tmp)\n}\n\n// NewDedupe creates a new dedupe builtin object\nfunc NewDedupe(vm *goja.Runtime) *Dedupe {\n\treturn &Dedupe{\n\t\tm:  make(map[string]goja.Value),\n\t\tVM: vm,\n\t}\n}\n\n// hashValue returns a hash of the value\nfunc hashValue(value interface{}) string {\n\tres := types.ToString(value)\n\tmd5sum := md5.Sum([]byte(res))\n\treturn string(md5sum[:])\n}\n\n// Path: pkg/reporting/trackers/gitea/gitea.go\npackage gitea\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"code.gitea.io/sdk/gitea\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n)\n\n// Integration is a client for an issue tracker integration\ntype Integration struct {\n\tclient  *gitea.Client\n\toptions *Options\n}\n\n// Options contains the configuration options for gitea issue tracker client\ntype Options struct {\n\t// BaseURL (optional) is the self-hosted Gitea application url\n\tBaseURL string `yaml:\"base-url\" validate:\"omitempty,url\"`\n\t// Token is the token for gitea account.\n\tToken string `yaml:\"token\" validate:\"required\"`\n\t// ProjectOwner is the owner (user or org) of the repository.\n\tProjectOwner string `yaml:\"project-owner\" validate:\"required\"`\n\t// ProjectName is the name of the repository.\n\tProjectName string `yaml:\"project-name\" validate:\"required\"`\n\t// IssueLabel is the label of the created issue type\n\tIssueLabel string `yaml:\"issue-label\"`\n\t// SeverityAsLabel (optional) adds the severity as the label of the created\n\t// issue.\n\tSeverityAsLabel bool `yaml:\"severity-as-label\"`\n\t// AllowList contains a list of allowed events for this tracker\n\tAllowList *filters.Filter `yaml:\"allow-list\"`\n\t// DenyList contains a list of denied events for this tracker\n\tDenyList *filters.Filter `yaml:\"deny-list\"`\n\t// DuplicateIssueCheck is a bool to enable duplicate tracking issue check and update the newest\n\tDuplicateIssueCheck bool `yaml:\"duplicate-issue-check\" default:\"false\"`\n\n\tHttpClient *retryablehttp.Client `yaml:\"-\"`\n\tOmitRaw    bool                  `yaml:\"-\"`\n}\n\n// New creates a new issue tracker integration client based on options.\nfunc New(options *Options) (*Integration, error) {\n\n\tvar opts []gitea.ClientOption\n\topts = append(opts, gitea.SetToken(options.Token))\n\n\tif options.HttpClient != nil {\n\t\topts = append(opts, gitea.SetHTTPClient(options.HttpClient.HTTPClient))\n\t}\n\n\tvar remote string\n\tif options.BaseURL != \"\" {\n\t\tparsed, err := url.Parse(options.BaseURL)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"could not parse custom baseurl\")\n\t\t}\n\t\tif !strings.HasSuffix(parsed.Path, \"/\") {\n\t\t\tparsed.Path += \"/\"\n\t\t}\n\t\tremote = parsed.String()\n\t} else {\n\t\tremote = `https://gitea.com/`\n\t}\n\n\tgit, err := gitea.NewClient(remote, opts...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &Integration{client: git, options: options}, nil\n}\n\n// CreateIssue creates an issue in the tracker\nfunc (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error) {\n\tsummary := format.Summary(event)\n\tdescription := format.CreateReportDescription(event, util.MarkdownFormatter{}, i.options.OmitRaw)\n\n\tlabels := []string{}\n\tseverityLabel := fmt.Sprintf(\"Severity: %s\", event.Info.SeverityHolder.Severity.String())\n\tif i.options.SeverityAsLabel && severityLabel != \"\" {\n\t\tlabels = append(labels, severityLabel)\n\t}\n\tif label := i.options.IssueLabel; label != \"\" {\n\t\tlabels = append(labels, label)\n\t}\n\tcustomLabels, err := i.getLabelIDsByNames(labels)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar issue *gitea.Issue\n\tif i.options.DuplicateIssueCheck {\n\t\tissue, err = i.findIssueByTitle(summary)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif issue == nil {\n\t\tcreatedIssue, _, err := i.client.CreateIssue(i.options.ProjectOwner, i.options.ProjectName, gitea.CreateIssueOption{\n\t\t\tTitle:  summary,\n\t\t\tBody:   description,\n\t\t\tLabels: customLabels,\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &filters.CreateIssueResponse{\n\t\t\tIssueID:  strconv.FormatInt(createdIssue.Index, 10),\n\t\t\tIssueURL: createdIssue.URL,\n\t\t}, nil\n\t}\n\n\t_, _, err = i.client.CreateIssueComment(i.options.ProjectOwner, i.options.ProjectName, issue.Index, gitea.CreateIssueCommentOption{\n\t\tBody: description,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &filters.CreateIssueResponse{\n\t\tIssueID:  strconv.FormatInt(issue.Index, 10),\n\t\tIssueURL: issue.URL,\n\t}, nil\n}\n\nfunc (i *Integration) CloseIssue(event *output.ResultEvent) error {\n\t// TODO: Implement\n\treturn nil\n}\n\n// ShouldFilter determines if an issue should be logged to this tracker\nfunc (i *Integration) ShouldFilter(event *output.ResultEvent) bool {\n\tif i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\tif i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\nfunc (i *Integration) findIssueByTitle(title string) (*gitea.Issue, error) {\n\n\tissueList, _, err := i.client.ListRepoIssues(i.options.ProjectOwner, i.options.ProjectName, gitea.ListIssueOption{\n\t\tState: \"all\",\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, issue := range issueList {\n\t\tif issue.Title == title {\n\t\t\treturn issue, nil\n\t\t}\n\t}\n\n\treturn nil, nil\n}\n\nfunc (i *Integration) getLabelIDsByNames(labels []string) ([]int64, error) {\n\n\tvar ids []int64\n\n\texistingLabels, _, err := i.client.ListRepoLabels(i.options.ProjectOwner, i.options.ProjectName, gitea.ListLabelsOptions{\n\t\tListOptions: gitea.ListOptions{Page: -1},\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tgetLabel := func(name string) int64 {\n\t\tfor _, existingLabel := range existingLabels {\n\t\t\tif existingLabel.Name == name {\n\t\t\t\treturn existingLabel.ID\n\t\t\t}\n\t\t}\n\t\treturn -1\n\t}\n\n\tfor _, label := range labels {\n\t\tlabelID := getLabel(label)\n\t\tif labelID == -1 {\n\t\t\tnewLabel, _, err := i.client.CreateLabel(i.options.ProjectOwner, i.options.ProjectName, gitea.CreateLabelOption{\n\t\t\t\tName:        label,\n\t\t\t\tColor:       `#00aabb`,\n\t\t\t\tDescription: label,\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tids = append(ids, newLabel.ID)\n\t\t} else {\n\t\t\tids = append(ids, labelID)\n\t\t}\n\t}\n\n\treturn ids, nil\n}\n\nfunc (i *Integration) Name() string {\n\treturn \"gitea\"\n}\n\n// Path: pkg/reporting/exporters/sarif/sarif.go\npackage sarif\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"os\"\n\t\"path\"\n\t\"sync\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/sarif\"\n)\n\n// Exporter is an exporter for nuclei sarif output format.\ntype Exporter struct {\n\tsarif   *sarif.Report\n\tmutex   *sync.Mutex\n\trulemap map[string]*int // contains rule-id && ruleIndex\n\trules   []sarif.ReportingDescriptor\n\toptions *Options\n}\n\n// Options contains the configuration options for sarif exporter client\ntype Options struct {\n\t// File is the file to export found sarif result to\n\tFile string `yaml:\"file\"`\n}\n\n// New creates a new sarif exporter integration client based on options.\nfunc New(options *Options) (*Exporter, error) {\n\treport := sarif.NewReport()\n\texporter := &Exporter{\n\t\tsarif:   report,\n\t\tmutex:   &sync.Mutex{},\n\t\trules:   []sarif.ReportingDescriptor{},\n\t\trulemap: map[string]*int{},\n\t\toptions: options,\n\t}\n\treturn exporter, nil\n}\n\n// addToolDetails adds details of static analysis tool (i.e nuclei)\nfunc (exporter *Exporter) addToolDetails() {\n\tdriver := sarif.ToolComponent{\n\t\tName:         \"Nuclei\",\n\t\tOrganization: \"ProjectDiscovery\",\n\t\tProduct:      \"Nuclei\",\n\t\tShortDescription: &sarif.MultiformatMessageString{\n\t\t\tText: \"Fast and Customizable Vulnerability Scanner\",\n\t\t},\n\t\tFullDescription: &sarif.MultiformatMessageString{\n\t\t\tText: \"Fast and customizable vulnerability scanner based on simple YAML based DSL\",\n\t\t},\n\t\tFullName:        \"Nuclei \" + config.Version,\n\t\tSemanticVersion: config.Version,\n\t\tDownloadURI:     \"https://github.com/projectdiscovery/nuclei/releases\",\n\t\tRules:           exporter.rules,\n\t}\n\texporter.sarif.RegisterTool(driver)\n\n\treportLocation := sarif.ArtifactLocation{\n\t\tUri: \"file:///\" + exporter.options.File,\n\t\tDescription: &sarif.Message{\n\t\t\tText: \"Nuclei Sarif Report\",\n\t\t},\n\t}\n\n\tinvocation := sarif.Invocation{\n\t\tCommandLine:   os.Args[0],\n\t\tArguments:     os.Args[1:],\n\t\tResponseFiles: []sarif.ArtifactLocation{reportLocation},\n\t}\n\texporter.sarif.RegisterToolInvocation(invocation)\n}\n\n// getSeverity in terms of sarif\nfunc (exporter *Exporter) getSeverity(severity string) (sarif.Level, string) {\n\tswitch severity {\n\tcase \"critical\":\n\t\treturn sarif.Error, \"9.4\"\n\tcase \"high\":\n\t\treturn sarif.Error, \"8\"\n\tcase \"medium\":\n\t\treturn sarif.Note, \"5\"\n\tcase \"low\":\n\t\treturn sarif.Note, \"2\"\n\tcase \"info\":\n\t\treturn sarif.None, \"1\"\n\t}\n\n\treturn sarif.None, \"9.5\"\n}\n\n// Export exports a passed result event to sarif structure\nfunc (exporter *Exporter) Export(event *output.ResultEvent) error {\n\texporter.mutex.Lock()\n\tdefer exporter.mutex.Unlock()\n\n\tseverity := event.Info.SeverityHolder.Severity.String()\n\tresultHeader := fmt.Sprintf(\"%v (%v) found on %v\", event.Info.Name, event.TemplateID, event.Host)\n\tresultLevel, vulnRating := exporter.getSeverity(severity)\n\n\t// Extra metadata if generated sarif is uploaded to GitHub security page\n\tghMeta := map[string]interface{}{}\n\tghMeta[\"tags\"] = []string{\"security\"}\n\tghMeta[\"security-severity\"] = vulnRating\n\n\t// rule contain details of template\n\trule := sarif.ReportingDescriptor{\n\t\tId:   event.TemplateID,\n\t\tName: event.Info.Name,\n\t\tFullDescription: &sarif.MultiformatMessageString{\n\t\t\t// Points to template URL\n\t\t\tText: event.Info.Description + \"\\nMore details at\\n\" + event.TemplateURL + \"\\n\",\n\t\t},\n\t\tProperties: ghMeta,\n\t}\n\n\t// GitHub Uses ShortDescription as title\n\tif event.Info.Description != \"\" {\n\t\trule.ShortDescription = &sarif.MultiformatMessageString{\n\t\t\tText: resultHeader,\n\t\t}\n\t}\n\n\t// If rule is added\n\truleIndex := int(math.Max(0, float64(len(exporter.rules)-1)))\n\tif exporter.rulemap[rule.Id] == nil {\n\t\texporter.rulemap[rule.Id] = &ruleIndex\n\t\texporter.rules = append(exporter.rules, rule)\n\t} else {\n\t\truleIndex = *exporter.rulemap[rule.Id]\n\t}\n\n\t// vulnerability target/location\n\tlocation := sarif.Location{\n\t\tMessage: &sarif.Message{\n\t\t\tText: path.Join(event.Host, event.Path),\n\t\t},\n\t\tPhysicalLocation: sarif.PhysicalLocation{\n\t\t\tArtifactLocation: sarif.ArtifactLocation{\n\t\t\t\t// GitHub only accepts file:// protocol and local & relative files only\n\t\t\t\t// to avoid errors // is used which also translates to file according to specification\n\t\t\t\tUri: \"/\" + event.Path,\n\t\t\t\tDescription: &sarif.Message{\n\t\t\t\t\tText: path.Join(event.Host, event.Path),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\t// vulnerability report/result\n\tresult := &sarif.Result{\n\t\tRuleId:    rule.Id,\n\t\tRuleIndex: ruleIndex,\n\t\tLevel:     resultLevel,\n\t\tKind:      sarif.Open,\n\t\tMessage: &sarif.Message{\n\t\t\tText: resultHeader,\n\t\t},\n\t\tLocations: []sarif.Location{location},\n\t\tRule: sarif.ReportingDescriptorReference{\n\t\t\tId: rule.Id,\n\t\t},\n\t}\n\n\texporter.sarif.RegisterResult(*result)\n\n\treturn nil\n\n}\n\n// Close Writes data and closes the exporter after operation\nfunc (exporter *Exporter) Close() error {\n\texporter.mutex.Lock()\n\tdefer exporter.mutex.Unlock()\n\n\tif len(exporter.rules) == 0 {\n\t\t// no output if there are no results\n\t\treturn nil\n\t}\n\t// links results and rules/templates\n\texporter.addToolDetails()\n\n\tbin, err := exporter.sarif.Export()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"failed to generate sarif report\")\n\t}\n\tif err := os.WriteFile(exporter.options.File, bin, 0644); err != nil {\n\t\treturn errors.Wrap(err, \"failed to create sarif file\")\n\t}\n\n\treturn nil\n}\n\n// Path: pkg/reporting/reporting.go\npackage reporting\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"sync/atomic\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\tjson_exporter \"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonexporter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl\"\n\n\t\"go.uber.org/multierr\"\n\t\"gopkg.in/yaml.v2\"\n\n\t\"errors\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/dedupe\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n\tfileutil \"github.com/projectdiscovery/utils/file\"\n)\n\nvar (\n\tErrReportingClientCreation = errors.New(\"could not create reporting client\")\n\tErrExportClientCreation    = errors.New(\"could not create exporting client\")\n)\n\n// Tracker is an interface implemented by an issue tracker\ntype Tracker interface {\n\t// Name returns the name of the tracker\n\tName() string\n\t// CreateIssue creates an issue in the tracker\n\tCreateIssue(event *output.ResultEvent) (*filters.CreateIssueResponse, error)\n\t// CloseIssue closes an issue in the tracker\n\tCloseIssue(event *output.ResultEvent) error\n\t// ShouldFilter determines if the event should be filtered out\n\tShouldFilter(event *output.ResultEvent) bool\n}\n\n// Exporter is an interface implemented by an issue exporter\ntype Exporter interface {\n\t// Close closes the exporter after operation\n\tClose() error\n\t// Export exports an issue to an exporter\n\tExport(event *output.ResultEvent) error\n}\n\n// ReportingClient is a client for nuclei issue tracking module\ntype ReportingClient struct {\n\ttrackers  []Tracker\n\texporters []Exporter\n\toptions   *Options\n\tdedupe    *dedupe.Storage\n\n\tstats map[string]*IssueTrackerStats\n}\n\ntype IssueTrackerStats struct {\n\tCreated atomic.Int32\n\tFailed  atomic.Int32\n}\n\n// New creates a new nuclei issue tracker reporting client\nfunc New(options *Options, db string, doNotDedupe bool) (Client, error) {\n\tclient := &ReportingClient{options: options}\n\n\tif options.GitHub != nil {\n\t\toptions.GitHub.HttpClient = options.HttpClient\n\t\toptions.GitHub.OmitRaw = options.OmitRaw\n...\n", "needle_token_start": 2145, "needle_token_end": 2430, "code_context_ntokens": 15188, "output": ["```go\n// NewGitLabProviders returns a new list of GitLab providers for downloading custom templates\nfunc NewGitLabProviders(options *types.Options) ([]*customTemplateGitLabRepo, error) {\n\tproviders := []*customTemplateGitLabRepo{}\n\tif options.GitLabToken != \"\" && !options.GitLabTemplateDisableDownload {\n\t\t// Establish a connection to GitLab and build a client object with which to download templates from GitLab\n\t\tgitLabClient, err := getGitLabClient(options.GitLabServerURL, options.GitLabToken)\n\t\tif err != nil {\n\t\t\treturn nil, errorutil.NewWithErr(err).Msgf(\"Error establishing GitLab client for %s %s\", options.GitLabServerURL, err)\n\t\t}\n\n\t\t// Create a new GitLab service client\n\t\tgitLabContainer := &customTemplateGitLabRepo{\n\t\t\tgitLabClient: gitLabClient,\n\t\t\tserverURL:    options.GitLabServerURL,\n\t\t\tprojectIDs:   options.GitLabTemplateRepositoryIDs,\n\t\t}\n\n\t\t// Add the GitLab service client to the list of custom templates\n\t\tproviders = append(providers, gitLabContainer)\n\t}\n\treturn providers, nil\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "RegisterNativeScripts", "language": "go", "path": "pkg/js/global/scripts.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to initialize a JavaScript runtime environment by loading and executing predefined JavaScript scripts and helper functions. This setup enhances the runtime with additional functionalities that can be used across various JavaScript scripts.\n2. **Input**: The function takes a JavaScript runtime environment as its input.\n3. **Output**: The function returns an error if there is a failure in reading directories, reading files, or executing scripts. If successful, it returns nil, indicating no errors occurred during the initialization process.\n4. **Procedure**: \n   - Initializes built-in functions within the runtime.\n   - Reads JavaScript files from a specified directory within an embedded file system.\n   - Executes each JavaScript file in the runtime to load scripts and helper functions.\n   - Sets up an exports object and imports default modules into the runtime.\n   - Handles errors at each step and returns them if any occur, otherwise concludes successfully.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/js/generated/go/libssh/ssh.go\npackage ssh\n\nimport (\n\tlib_ssh \"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/ssh\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs\"\n)\n\nvar (\n\tmodule = gojs.NewGojaModule(\"nuclei/ssh\")\n)\n\nfunc init() {\n\tmodule.Set(\n\t\tgojs.Objects{\n\t\t\t// Functions\n\n\t\t\t// Var and consts\n\n\t\t\t// Objects / Classes\n\t\t\t\"SSHClient\": gojs.GetClassConstructor[lib_ssh.SSHClient](&lib_ssh.SSHClient{}),\n\t\t},\n\t).Register()\n}\n\nfunc Enable(runtime *goja.Runtime) {\n...\n// Path: pkg/js/generated/go/libtelnet/telnet.go\npackage telnet\n\nimport (\n\tlib_telnet \"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/telnet\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs\"\n)\n\nvar (\n\tmodule = gojs.NewGojaModule(\"nuclei/telnet\")\n)\n\nfunc init() {\n\tmodule.Set(\n\t\tgojs.Objects{\n\t\t\t// Functions\n\t\t\t\"IsTelnet\": lib_telnet.IsTelnet,\n\n\t\t\t// Var and consts\n\n\t\t\t// Objects / Classes\n\t\t\t\"IsTelnetResponse\": gojs.GetClassConstructor[lib_telnet.IsTelnetResponse](&lib_telnet.IsTelnetResponse{}),\n\t\t},\n\t).Register()\n}\n\nfunc Enable(runtime *goja.Runtime) {\n\tmodule.Enable(runtime)\n}\n\n// Path: pkg/js/generated/go/libvnc/vnc.go\npackage vnc\n\nimport (\n\tlib_vnc \"github.com/projectdiscovery/nuclei/v3/pkg/js/libs/vnc\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs\"\n)\n\nvar (\n\tmodule = gojs.NewGojaModule(\"nuclei/vnc\")\n)\n\nfunc init() {\n\tmodule.Set(\n\t\tgojs.Objects{\n\t\t\t// Functions\n\t\t\t\"IsVNC\": lib_vnc.IsVNC,\n\n\t\t\t// Var and consts\n\n\t\t\t// Objects / Classes\n\t\t\t\"IsVNCResponse\": gojs.GetClassConstructor[lib_vnc.IsVNCResponse](&lib_vnc.IsVNCResponse{}),\n\t\t},\n\t).Register()\n}\n\nfunc Enable(runtime *goja.Runtime) {\n\tmodule.Enable(runtime)\n}\n\n// Path: pkg/js/global/helpers.go\npackage global\n\nimport (\n\t\"encoding/base64\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs\"\n)\n\nfunc registerAdditionalHelpers(runtime *goja.Runtime) {\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"atob\",\n\t\tSignatures: []string{\n\t\t\t\"atob(string) string\",\n\t\t},\n\t\tDescription: \"Base64 decodes a given string\",\n\t\tFuncDecl: func(call goja.FunctionCall) goja.Value {\n\t\t\tinput := call.Argument(0).String()\n\n\t\t\tdecoded, err := base64.StdEncoding.DecodeString(input)\n\t\t\tif err != nil {\n\t\t\t\treturn goja.Null()\n\t\t\t}\n\t\t\treturn runtime.ToValue(string(decoded))\n\t\t},\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"btoa\",\n\t\tSignatures: []string{\n\t\t\t\"bota(string) string\",\n\t\t},\n\t\tDescription: \"Base64 encodes a given string\",\n\t\tFuncDecl: func(call goja.FunctionCall) goja.Value {\n\t\t\tinput := call.Argument(0).String()\n\t\t\tencoded := base64.StdEncoding.EncodeToString([]byte(input))\n\t\t\treturn runtime.ToValue(encoded)\n\t\t},\n\t})\n}\n\nfunc init() {\n\t// these are dummy functions we use trigger documentation generation\n\t// actual definations are in exports.js\n\t_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{\n\t\tName: \"to_json\",\n\t\tSignatures: []string{\n\t\t\t\"to_json(any) object\",\n\t\t},\n\t\tDescription: \"Converts a given object to JSON\",\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{\n\t\tName: \"dump_json\",\n\t\tSignatures: []string{\n\t\t\t\"dump_json(any)\",\n\t\t},\n\t\tDescription: \"Prints a given object as JSON in console\",\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{\n\t\tName: \"to_array\",\n\t\tSignatures: []string{\n\t\t\t\"to_array(any) array\",\n\t\t},\n\t\tDescription: \"Sets/Updates objects prototype to array to enable Array.XXX functions\",\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{\n\t\tName: \"hex_to_ascii\",\n\t\tSignatures: []string{\n\t\t\t\"hex_to_ascii(string) string\",\n\t\t},\n\t\tDescription: \"Converts a given hex string to ascii\",\n\t})\n\n}\n\n// Path: pkg/js/global/scripts.go\npackage global\n\nimport (\n\t\"bytes\"\n\t\"embed\"\n\t\"math/rand\"\n\t\"net\"\n\t\"reflect\"\n\t\"time\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/logrusorgru/aurora\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\nvar (\n\t//go:embed js\n\tembedFS embed.FS\n\n\t//go:embed exports.js\n\texports string\n\t// knownPorts is a list of known ports for protocols implemented in nuclei\n\tknowPorts = []string{\"80\", \"443\", \"8080\", \"8081\", \"8443\", \"53\"}\n)\n\n// default imported modules\n// there might be other methods to achieve this\n// but this is most straightforward\nvar (\n\tdefaultImports = `\n\t  var structs = require(\"nuclei/structs\");\n\t  var bytes = require(\"nuclei/bytes\");\n\t`\n)\n\n// initBuiltInFunc initializes runtime with builtin functions\nfunc initBuiltInFunc(runtime *goja.Runtime) {\n\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName:        \"Rand\",\n\t\tSignatures:  []string{\"Rand(n int) []byte\"},\n\t\tDescription: \"Rand returns a random byte slice of length n\",\n\t\tFuncDecl: func(n int) []byte {\n\t\t\tb := make([]byte, n)\n\t\t\tfor i := range b {\n\t\t\t\tb[i] = byte(rand.Intn(255))\n\t\t\t}\n\t\t\treturn b\n\t\t},\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName:        \"RandInt\",\n\t\tSignatures:  []string{\"RandInt() int\"},\n\t\tDescription: \"RandInt returns a random int\",\n\t\tFuncDecl: func() int64 {\n\t\t\treturn rand.Int63()\n\t\t},\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"log\",\n\t\tSignatures: []string{\n\t\t\t\"log(msg string)\",\n\t\t\t\"log(msg map[string]interface{})\",\n\t\t},\n\t\tDescription: \"log prints given input to stdout with [JS] prefix for debugging purposes \",\n\t\tFuncDecl: func(call goja.FunctionCall) goja.Value {\n\t\t\targ := call.Argument(0).Export()\n\t\t\tswitch value := arg.(type) {\n\t\t\tcase string:\n\t\t\t\tgologger.DefaultLogger.Print().Msgf(\"[%v] %v\", aurora.BrightCyan(\"JS\"), value)\n\t\t\tcase map[string]interface{}:\n\t\t\t\tgologger.DefaultLogger.Print().Msgf(\"[%v] %v\", aurora.BrightCyan(\"JS\"), vardump.DumpVariables(value))\n\t\t\tdefault:\n\t\t\t\tgologger.DefaultLogger.Print().Msgf(\"[%v] %v\", aurora.BrightCyan(\"JS\"), value)\n\t\t\t}\n\t\t\treturn call.Argument(0)\n\t\t},\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"getNetworkPort\",\n\t\tSignatures: []string{\n\t\t\t\"getNetworkPort(port string, defaultPort string) string\",\n\t\t},\n\t\tDescription: \"getNetworkPort registers defaultPort and returns defaultPort if it is a colliding port with other protocols\",\n\t\tFuncDecl: func(call goja.FunctionCall) goja.Value {\n\t\t\tinputPort := call.Argument(0).String()\n\t\t\tif inputPort == \"\" || stringsutil.EqualFoldAny(inputPort, knowPorts...) {\n\t\t\t\t// if inputPort is empty or a know port of other protocol\n\t\t\t\t// return given defaultPort\n\t\t\t\treturn call.Argument(1)\n\t\t\t}\n\t\t\treturn call.Argument(0)\n\t\t},\n\t})\n\n\t// is port open check is port is actually open\n\t// it can be invoked as isPortOpen(host, port, [timeout])\n\t// where timeout is optional and defaults to 5 seconds\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"isPortOpen\",\n\t\tSignatures: []string{\n\t\t\t\"isPortOpen(host string, port string, [timeout int]) bool\",\n\t\t},\n\t\tDescription: \"isPortOpen checks if given port is open on host. timeout is optional and defaults to 5 seconds\",\n\t\tFuncDecl: func(host string, port string, timeout ...int) (bool, error) {\n\t\t\ttimeoutInSec := 5\n\t\t\tif len(timeout) > 0 {\n\t\t\t\ttimeoutInSec = timeout[0]\n\t\t\t}\n\t\t\tconn, err := net.DialTimeout(\"tcp\", net.JoinHostPort(host, port), time.Duration(timeoutInSec)*time.Second)\n\t\t\tif err != nil {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t\t_ = conn.Close()\n\t\t\treturn true, nil\n\t\t},\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"ToBytes\",\n\t\tSignatures: []string{\n\t\t\t\"ToBytes(...interface{}) []byte\",\n\t\t},\n\t\tDescription: \"ToBytes converts given input to byte slice\",\n\t\tFuncDecl: func(call goja.FunctionCall) goja.Value {\n\t\t\tvar buff bytes.Buffer\n\t\t\tallVars := []any{}\n\t\t\tfor _, v := range call.Arguments {\n\t\t\t\tif v.Export() == nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif v.ExportType().Kind() == reflect.Slice {\n\t\t\t\t\t// convert []datatype to []interface{}\n\t\t\t\t\t// since it cannot be type asserted to []interface{} directly\n\t\t\t\t\trfValue := reflect.ValueOf(v.Export())\n\t\t\t\t\tfor i := 0; i < rfValue.Len(); i++ {\n\t\t\t\t\t\tallVars = append(allVars, rfValue.Index(i).Interface())\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tallVars = append(allVars, v.Export())\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, v := range allVars {\n\t\t\t\tbuff.WriteString(types.ToString(v))\n\t\t\t}\n\t\t\treturn runtime.ToValue(buff.Bytes())\n\t\t},\n\t})\n\n\t_ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"ToString\",\n\t\tSignatures: []string{\n\t\t\t\"ToString(...interface{}) string\",\n\t\t},\n\t\tDescription: \"ToString converts given input to string\",\n\t\tFuncDecl: func(call goja.FunctionCall) goja.Value {\n\t\t\tvar buff bytes.Buffer\n\t\t\tfor _, v := range call.Arguments {\n\t\t\t\texported := v.Export()\n\t\t\t\tif exported != nil {\n\t\t\t\t\tbuff.WriteString(types.ToString(exported))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn runtime.ToValue(buff.String())\n\t\t},\n\t})\n\n\t// register additional helpers\n\tregisterAdditionalHelpers(runtime)\n}\n\n// RegisterNativeScripts are js scripts that were added for convenience\n// and abstraction purposes we execute them in every runtime and make them\n// available for use in any js script\n// see: scripts/ for examples\n\nfunc RegisterNativeScripts(runtime *goja.Runtime) error {\n\tinitBuiltInFunc(runtime)\n\n\tdirs, err := embedFS.ReadDir(\"js\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, dir := range dirs {\n\t\tif dir.IsDir() {\n\t\t\tcontinue\n\t\t}\n\t\t// embeds have / as path separator (on all os)\n\t\tcontents, err := embedFS.ReadFile(\"js\" + \"/\" + dir.Name())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// run all built in js helper functions or scripts\n\t\t_, err = runtime.RunString(string(contents))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// exports defines the exports object\n\t_, err = runtime.RunString(exports)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// import default modules\n\t_, err = runtime.RunString(defaultImports)\n\tif err != nil {\n\t\treturn errorutil.NewWithErr(err).Msgf(\"could not import default modules %v\", defaultImports)\n\t}\n\n\treturn nil\n}\n\n// Path: pkg/js/global/scripts_test.go\npackage global\n\nimport (\n\t\"testing\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/dop251/goja_nodejs/console\"\n\t\"github.com/dop251/goja_nodejs/require\"\n)\n\nfunc TestScriptsRuntime(t *testing.T) {\n\tdefaultImports = \"\"\n\truntime := goja.New()\n\n\tregistry := new(require.Registry)\n\tregistry.Enable(runtime)\n\tconsole.Enable(runtime)\n\n\terr := RegisterNativeScripts(runtime)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvalue, err := runtime.RunString(\"dump_json({a: 1, b: 2})\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t_ = value\n}\n\n// Path: pkg/js/gojs/set.go\npackage gojs\n\nimport (\n\t\"github.com/dop251/goja\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n)\n\nvar (\n\tErrInvalidFuncOpts = errorutil.NewWithFmt(\"invalid function options: %v\")\n\tErrNilRuntime      = errorutil.New(\"runtime is nil\")\n)\n\ntype FuncOpts struct {\n\tName        string\n\tSignatures  []string\n\tDescription string\n\tFuncDecl    interface{}\n}\n\n// valid checks if the function options are valid\nfunc (f *FuncOpts) valid() bool {\n\treturn f.Name != \"\" && f.FuncDecl != nil && len(f.Signatures) > 0 && f.Description != \"\"\n}\n\n// RegisterFunc registers a function with given name, signatures and description\nfunc RegisterFuncWithSignature(runtime *goja.Runtime, opts FuncOpts) error {\n\tif runtime == nil {\n\t\treturn ErrNilRuntime\n\t}\n\tif !opts.valid() {\n\t\treturn ErrInvalidFuncOpts.Msgf(\"name: %s, signatures: %v, description: %s\", opts.Name, opts.Signatures, opts.Description)\n\t}\n\treturn runtime.Set(opts.Name, opts.FuncDecl)\n}\n\n// Path: pkg/js/libs/ikev2/ikev2.go\npackage ikev2\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\n\t\"github.com/projectdiscovery/n3iwf/pkg/ike/message\"\n\t\"github.com/projectdiscovery/n3iwf/pkg/logger\"\n)\n\nfunc init() {\n\tlogger.Log.SetOutput(io.Discard)\n}\n\ntype (\n\t// IKEMessage is the IKEv2 message\n\t//\n\t// IKEv2 implements a limited subset of IKEv2 Protocol, specifically\n\t// the IKE_NOTIFY and IKE_NONCE payloads and the IKE_SA_INIT exchange.\n\tIKEMessage struct {\n\t\tInitiatorSPI uint64\n\t\tVersion      uint8\n\t\tExchangeType uint8\n\t\tFlags        uint8\n\t\tpayloads     []IKEPayload\n\t}\n)\n\n// AppendPayload appends a payload to the IKE message\n// payload can be any of the payloads like IKENotification, IKENonce, etc.\n// @example\n// ```javascript\n// const ikev2 = require('nuclei/ikev2');\n// const message = new ikev2.IKEMessage();\n// const nonce = new ikev2.IKENonce();\n// nonce.NonceData = [1, 2, 3];\n// message.AppendPayload(nonce);\n// ```\nfunc (m *IKEMessage) AppendPayload(payload any) error {\n\tif _, ok := payload.(IKEPayload); !ok {\n\t\treturn fmt.Errorf(\"invalid payload type only types defined in ikev module like IKENotification, IKENonce, etc. are allowed\")\n\t}\n\tm.payloads = append(m.payloads, payload.(IKEPayload))\n\treturn nil\n}\n\n// Encode encodes the final IKE message\n// @example\n// ```javascript\n// const ikev2 = require('nuclei/ikev2');\n// const message = new ikev2.IKEMessage();\n// const nonce = new ikev2.IKENonce();\n// nonce.NonceData = [1, 2, 3];\n// message.AppendPayload(nonce);\n// log(message.Encode());\n// ```\nfunc (m *IKEMessage) Encode() ([]byte, error) {\n\tvar payloads message.IKEPayloadContainer\n\tfor _, payload := range m.payloads {\n\t\tp, err := payload.encode()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tpayloads = append(payloads, p)\n\t}\n\n\tmsg := &message.IKEMessage{\n\t\tInitiatorSPI: m.InitiatorSPI,\n\t\tVersion:      m.Version,\n\t\tExchangeType: m.ExchangeType,\n\t\tFlags:        m.Flags,\n\t\tPayloads:     payloads,\n\t}\n\tencoded, err := msg.Encode()\n\treturn encoded, err\n}\n\n// IKEPayload is the IKEv2 payload interface\n// All the payloads like IKENotification, IKENonce, etc. implement\n// this interface.\ntype IKEPayload interface {\n\tencode() (message.IKEPayload, error)\n}\n\ntype (\n\t// IKEv2Notify is the IKEv2 Notification payload\n\t// this implements the IKEPayload interface\n\t// @example\n\t// ```javascript\n\t// const ikev2 = require('nuclei/ikev2');\n\t// const notify = new ikev2.IKENotification();\n\t// notify.NotifyMessageType = ikev2.IKE_NOTIFY_NO_PROPOSAL_CHOSEN;\n\t// notify.NotificationData = [1, 2, 3];\n\t// ```\n\tIKENotification struct {\n\t\tNotifyMessageType uint16\n\t\tNotificationData  []byte\n\t}\n)\n\n// encode encodes the IKEv2 Notification payload\nfunc (i *IKENotification) encode() (message.IKEPayload, error) {\n\tnotify := message.Notification{\n\t\tNotifyMessageType: i.NotifyMessageType,\n\t\tNotificationData:  i.NotificationData,\n\t}\n\treturn &notify, nil\n}\n\nconst (\n\t// Notify message types\n\tIKE_NOTIFY_NO_PROPOSAL_CHOSEN = 14\n\tIKE_NOTIFY_USE_TRANSPORT_MODE = 16391\n\n\tIKE_VERSION_2 = 0x20\n\n\t// Exchange Type\n\tIKE_EXCHANGE_SA_INIT         = 34\n\tIKE_EXCHANGE_AUTH            = 35\n\tIKE_EXCHANGE_CREATE_CHILD_SA = 36\n\tIKE_EXCHANGE_INFORMATIONAL   = 37\n\n\t// Flags\n\tIKE_FLAGS_InitiatorBitCheck = 0x08\n)\n\ntype (\n\t// IKENonce is the IKEv2 Nonce payload\n\t// this implements the IKEPayload interface\n\t// @example\n\t// ```javascript\n\t// const ikev2 = require('nuclei/ikev2');\n\t// const nonce = new ikev2.IKENonce();\n\t// nonce.NonceData = [1, 2, 3];\n\t// ```\n\tIKENonce struct {\n\t\tNonceData []byte\n\t}\n)\n\n// encode encodes the IKEv2 Nonce payload\nfunc (i *IKENonce) encode() (message.IKEPayload, error) {\n\tnonce := message.Nonce{\n\t\tNonceData: i.NonceData,\n\t}\n\treturn &nonce, nil\n}\n\n// Path: pkg/js/libs/kerberos/kerberosx.go\npackage kerberos\n\nimport (\n\t\"strings\"\n\n\t\"github.com/dop251/goja\"\n\tkclient \"github.com/jcmturner/gokrb5/v8/client\"\n\tkconfig \"github.com/jcmturner/gokrb5/v8/config\"\n\t\"github.com/jcmturner/gokrb5/v8/iana/errorcode\"\n\t\"github.com/jcmturner/gokrb5/v8/messages\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate\"\n\tConversionUtil \"github.com/projectdiscovery/utils/conversion\"\n)\n\ntype (\n\t// EnumerateUserResponse is the response from EnumerateUser\n\tEnumerateUserResponse struct {\n\t\tValid     bool   `json:\"valid\"`\n\t\tASREPHash string `json:\"asrep_hash\"`\n\t\tError     string `json:\"error\"`\n\t}\n)\n\ntype (\n\t// TGS is the response from GetServiceTicket\n\tTGS struct {\n\t\tTicket messages.Ticket `json:\"ticket\"`\n\t\tHash   string          `json:\"hash\"`\n\t\tErrMsg string          `json:\"error\"`\n\t}\n)\n\ntype (\n\t// Config is extra configuration for the kerberos client\n\tConfig struct {\n\t\tip      string\n\t\ttimeout int // in seconds\n\t}\n)\n\n// SetIPAddress sets the IP address for the kerberos client\n// @example\n// ```javascript\n// const kerberos = require('nuclei/kerberos');\n// const cfg = new kerberos.Config();\n// cfg.SetIPAddress('10.10.10.1');\n// ```\nfunc (c *Config) SetIPAddress(ip string) *Config {\n\tc.ip = ip\n\treturn c\n}\n\n// SetTimeout sets the RW timeout for the kerberos client\n// @example\n// ```javascript\n// const kerberos = require('nuclei/kerberos');\n// const cfg = new kerberos.Config();\n// cfg.SetTimeout(5);\n// ```\nfunc (c *Config) SetTimeout(timeout int) *Config {\n\tc.timeout = timeout\n\treturn c\n}\n\n// Example Values for jargons\n// Realm: ACME.COM (Authentical zone / security area)\n// Domain: acme.com (Public website / domain)\n// DomainController: dc.acme.com (Domain Controller / Active Directory Server)\n// KDC: kdc.acme.com (Key Distribution Center / Authentication Server)\n\ntype (\n\t// Known Issues:\n\t// Hardcoded timeout in gokrb5 library\n\t// TGT / Session Handling not exposed\n\t// Client is kerberos client\n\t// @example\n\t// ```javascript\n\t// const kerberos = require('nuclei/kerberos');\n\t// // if controller is empty a dns lookup for default kdc server will be performed\n\t// const client = new kerberos.Client('acme.com', 'kdc.acme.com');\n\t// ```\n\tClient struct {\n\t\tnj         *utils.NucleiJS // helper functions/bindings\n\t\tKrb5Config *kconfig.Config\n\t\tRealm      string\n\t\tconfig     Config\n\t}\n)\n\n// Constructor for Kerberos Client\n// Constructor: constructor(public domain: string, public controller?: string)\n// When controller is empty or not given krb5 will perform a DNS lookup for the default KDC server\n// and retrieve its address from the DNS server\nfunc NewKerberosClient(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {\n\t// setup nucleijs utils\n\tc := &Client{nj: utils.NewNucleiJS(runtime)}\n\tc.nj.ObjectSig = \"Client(domain, {controller})\" // will be included in error messages\n\n\t// get arguments (type assertion is efficient than reflection)\n\t// when accepting type as input like net.Conn we can use utils.GetArg\n\tdomain, _ := c.nj.GetArg(call.Arguments, 0).(string)\n\tcontroller, _ := c.nj.GetArg(call.Arguments, 1).(string)\n\n\t// validate arguments\n\tc.nj.Require(domain != \"\", \"domain cannot be empty\")\n\n\tcfg := kconfig.New()\n\n\tif controller != \"\" {\n\t\t// validate controller hostport\n\t\tif !protocolstate.IsHostAllowed(controller) {\n\t\t\tc.nj.Throw(\"domain controller address blacklisted by network policy\")\n\t\t}\n\n\t\ttmp := strings.Split(controller, \":\")\n\t\tif len(tmp) == 1 {\n\t\t\ttmp = append(tmp, \"88\")\n\t\t}\n\t\trealm := strings.ToUpper(domain)\n\t\tcfg.LibDefaults.DefaultRealm = realm // set default realm\n\t\tcfg.Realms = []kconfig.Realm{\n\t\t\t{\n\t\t\t\tRealm:         realm,\n\t\t\t\tKDC:           []string{tmp[0] + \":\" + tmp[1]},\n\t\t\t\tAdminServer:   []string{tmp[0] + \":\" + tmp[1]},\n\t\t\t\tKPasswdServer: []string{tmp[0] + \":464\"}, // default password server port\n\t\t\t},\n\t\t}\n\t\tcfg.DomainRealm = make(kconfig.DomainRealm)\n\t} else {\n\t\t// if controller is empty use DNS lookup\n\t\tcfg.LibDefaults.DNSLookupKDC = true\n\t\tcfg.LibDefaults.DefaultRealm = strings.ToUpper(domain)\n\t\tcfg.DomainRealm = make(kconfig.DomainRealm)\n\t}\n\tc.Krb5Config = cfg\n\tc.Realm = strings.ToUpper(domain)\n\n\t// Link Constructor to Client and return\n\treturn utils.LinkConstructor(call, runtime, c)\n}\n\n// NewKerberosClientFromString creates a new kerberos client from a string\n// by parsing krb5.conf\n// @example\n// ```javascript\n// const kerberos = require('nuclei/kerberos');\n// const client = kerberos.NewKerberosClientFromString(`\n// [libdefaults]\n// default_realm = ACME.COM\n// dns_lookup_kdc = true\n// `);\n// ```\nfunc NewKerberosClientFromString(cfg string) (*Client, error) {\n\tconfig, err := kconfig.NewFromString(cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Client{Krb5Config: config}, nil\n}\n\n// SetConfig sets additional config for the kerberos client\n// Note: as of now ip and timeout overrides are only supported\n// in EnumerateUser due to fastdialer but can be extended to other methods currently\n// @example\n// ```javascript\n// const kerberos = require('nuclei/kerberos');\n// const client = new kerberos.Client('acme.com', 'kdc.acme.com');\n// const cfg = new kerberos.Config();\n// cfg.SetIPAddress('192.168.100.22');\n// cfg.SetTimeout(5);\n// client.SetConfig(cfg);\n// ```\nfunc (c *Client) SetConfig(cfg *Config) {\n\tif cfg == nil {\n\t\tc.nj.Throw(\"config cannot be nil\")\n\t}\n\tc.config = *cfg\n}\n\n// EnumerateUser and attempt to get AS-REP hash by disabling PA-FX-FAST\n// @example\n// ```javascript\n// const kerberos = require('nuclei/kerberos');\n// const client = new kerberos.Client('acme.com', 'kdc.acme.com');\n// const resp = client.EnumerateUser('pdtm');\n// log(resp);\n// ```\nfunc (c *Client) EnumerateUser(username string) (EnumerateUserResponse, error) {\n\tc.nj.Require(c.Krb5Config != nil, \"Kerberos client not initialized\")\n\tpassword := \"password\"\n\t// client does not actually attempt connection it manages state here\n\tclient := kclient.NewWithPassword(username, c.Realm, password, c.Krb5Config, kclient.DisablePAFXFAST(true))\n\tdefer client.Destroy()\n\n\t// generate ASReq hash\n\treq, err := messages.NewASReqForTGT(client.Credentials.Domain(), client.Config, client.Credentials.CName())\n\tc.nj.HandleError(err, \"failed to generate TGT request\")\n\n\t// marshal request\n\tb, err := req.Marshal()\n\tc.nj.HandleError(err, \"failed to marshal TGT request\")\n\n\tdata, err := SendToKDC(c, string(b))\n\trb := ConversionUtil.Bytes(data)\n\n\tif err == nil {\n\t\tvar ASRep messages.ASRep\n\t\tresp := EnumerateUserResponse{Valid: true}\n\t\terr = ASRep.Unmarshal(rb)\n\t\tif err != nil {\n\t\t\tresp.Error = err.Error()\n\t\t\treturn resp, nil\n\t\t}\n\t\thashcatString, _ := ASRepToHashcat(ASRep)\n\t\tresp.ASREPHash = hashcatString\n\t\treturn resp, nil\n\t}\n\n\tresp := EnumerateUserResponse{}\n\te, ok := err.(messages.KRBError)\n\tif !ok {\n\t\treturn resp, err\n\t}\n\tif e.ErrorCode == errorcode.KDC_ERR_PREAUTH_REQUIRED {\n\t\tresp.Valid = true\n\t\tresp.Error = errorcode.Lookup(e.ErrorCode)\n\t\treturn resp, nil\n\t}\n\tresp.Error = errorcode.Lookup(e.ErrorCode)\n\treturn resp, nil\n}\n\n// GetServiceTicket returns a TGS for a given user, password and SPN\n// @example\n// ```javascript\n// const kerberos = require('nuclei/kerberos');\n// const client = new kerberos.Client('acme.com', 'kdc.acme.com');\n// const resp = client.GetServiceTicket('pdtm', 'password', 'HOST/CLIENT1');\n// log(resp);\n// ```\nfunc (c *Client) GetServiceTicket(User, Pass, SPN string) (TGS, error) {\n\tc.nj.Require(c.Krb5Config != nil, \"Kerberos client not initialized\")\n\tc.nj.Require(User != \"\", \"User cannot be empty\")\n\tc.nj.Require(Pass != \"\", \"Pass cannot be empty\")\n\tc.nj.Require(SPN != \"\", \"SPN cannot be empty\")\n\n\tif len(c.Krb5Config.Realms) > 0 {\n\t\t// this means dc address was given\n\t\tfor _, r := range c.Krb5Config.Realms {\n\t\t\tfor _, kdc := range r.KDC {\n\t\t\t\tif !protocolstate.IsHostAllowed(kdc) {\n\t\t\t\t\tc.nj.Throw(\"KDC address %v blacklisted by network policy\", kdc)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfor _, kpasswd := range r.KPasswdServer {\n\t\t\t\tif !protocolstate.IsHostAllowed(kpasswd) {\n\t\t\t\t\tc.nj.Throw(\"Kpasswd address %v blacklisted by network policy\", kpasswd)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// here net.Dialer is used instead of fastdialer hence get possible addresses\n\t\t// and check if they are allowed by network policy\n\t\t_, kdcs, _ := c.Krb5Config.GetKDCs(c.Realm, true)\n\t\tfor _, v := range kdcs {\n\t\t\tif !protocolstate.IsHostAllowed(v) {\n\t\t\t\tc.nj.Throw(\"KDC address %v blacklisted by network policy\", v)\n\t\t\t}\n\t\t}\n\t}\n\n\t// client does not actually attempt connection it manages state here\n\tclient := kclient.NewWithPassword(User, c.Realm, Pass, c.Krb5Config, kclient.DisablePAFXFAST(true))\n\tdefer client.Destroy()\n\n\tresp := TGS{}\n\n\tticket, _, err := client.GetServiceTicket(SPN)\n\tresp.Ticket = ticket\n\tif err != nil {\n\t\tif code, ok := err.(messages.KRBError); ok {\n\t\t\tresp.ErrMsg = errorcode.Lookup(code.ErrorCode)\n\t\t\treturn resp, err\n\t\t}\n\t\treturn resp, err\n\t}\n\t// convert AS-REP to hashcat format\n\thashcat, err := TGStoHashcat(ticket, c.Realm)\n\tif err != nil {\n\t\tif code, ok := err.(messages.KRBError); ok {\n\t\t\tresp.ErrMsg = errorcode.Lookup(code.ErrorCode)\n\t\t\treturn resp, err\n\t\t}\n\t\treturn resp, err\n\t}\n\tresp.Ticket = ticket\n\tresp.Hash = hashcat\n\treturn resp, nil\n}\n\n// // GetASREP returns AS-REP for a given user and password\n// // it contains Client's TGT , Principal and Session Key\n// // Signature: GetASREP(User, Pass)\n// // @param User: string\n// // @param Pass: string\n// func (c *Client) GetASREP(User, Pass string) messages.ASRep {\n// \tc.nj.Require(c.Krb5Config != nil, \"Kerberos client not initialized\")\n// \tc.nj.Require(User != \"\", \"User cannot be empty\")\n// \tc.nj.Require(Pass != \"\", \"Pass cannot be empty\")\n\n// \tif len(c.Krb5Config.Realms) > 0 {\n// \t\t// this means dc address was given\n// \t\tfor _, r := range c.Krb5Config.Realms {\n// \t\t\tfor _, kdc := range r.KDC {\n// \t\t\t\tif !protocolstate.IsHostAllowed(kdc) {\n// \t\t\t\t\tc.nj.Throw(\"KDC address blacklisted by network policy\")\n// \t\t\t\t}\n// \t\t\t}\n// \t\t\tfor _, kpasswd := range r.KPasswdServer {\n// \t\t\t\tif !protocolstate.IsHostAllowed(kpasswd) {\n// \t\t\t\t\tc.nj.Throw(\"Kpasswd address blacklisted by network policy\")\n// \t\t\t\t}\n// \t\t\t}\n// \t\t}\n// \t} else {\n// \t\t// here net.Dialer is used instead of fastdialer hence get possible addresses\n// \t\t// and check if they are allowed by network policy\n// \t\t_, kdcs, _ := c.Krb5Config.GetKDCs(c.Realm, true)\n// \t\tfor _, v := range kdcs {\n// \t\t\tif !protocolstate.IsHostAllowed(v) {\n// \t\t\t\tc.nj.Throw(\"KDC address blacklisted by network policy\")\n// \t\t\t}\n// \t\t}\n// \t}\n\n// \t// login to get TGT\n// \tcl := kclient.NewWithPassword(User, c.Realm, Pass, c.Krb5Config, kclient.DisablePAFXFAST(true))\n// \tdefer cl.Destroy()\n\n// \t// generate ASReq\n// \tASReq, err := messages.NewASReqForTGT(cl.Credentials.Domain(), cl.Config, cl.Credentials.CName())\n// \tc.nj.HandleError(err, \"failed to generate TGT request\")\n\n// \t// exchange AS-REQ for AS-REP\n// \tresp, err := cl.ASExchange(c.Realm, ASReq, 0)\n// \tc.nj.HandleError(err, \"failed to exchange AS-REQ\")\n\n// \t// try to decrypt encrypted parts of the response and TGT\n// \tkey, err := resp.DecryptEncPart(cl.Credentials)\n// \tif err == nil {\n// \t\t_ = resp.Ticket.Decrypt(key)\n// \t}\n// \treturn resp\n// }\n\n// Path: pkg/js/libs/kerberos/sendtokdc.go\npackage kerberos\n\n// the following code is adapted from the original library\n// https://github.com/jcmturner/gokrb5/blob/855dbc707a37a21467aef6c0245fcf3328dc39ed/v8/client/network.go\n// it is copied here because the library does not export \"SendToKDC()\"\n\nimport (\n\t\"context\"\n\t\"encoding/binary\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/jcmturner/gokrb5/v8/messages\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate\"\n)\n\n// sendtokdc.go deals with actual sending and receiving responses from KDC\n// SendToKDC sends a message to the KDC and returns the response.\n// It first tries to send the message over TCP, and if that fails, it falls back to UDP.(and vice versa)\n// @example\n// ```javascript\n// const kerberos = require('nuclei/kerberos');\n// const client = new kerberos.Client('acme.com');\n// const response = kerberos.SendToKDC(client, 'message');\n// ```\nfunc SendToKDC(kclient *Client, msg string) (string, error) {\n\tif kclient == nil || kclient.nj == nil || kclient.Krb5Config == nil || kclient.Realm == \"\" {\n\t\treturn \"\", fmt.Errorf(\"kerberos client is not initialized\")\n\t}\n\tif kclient.config.timeout == 0 {\n\t\tkclient.config.timeout = 5 // default timeout 5 seconds\n\t}\n\tvar response []byte\n\tvar err error\n\n\tresponse, err = sendToKDCTcp(kclient, msg)\n\tif err == nil {\n\t\t// if it related to tcp\n\t\tbin, err := CheckKrbError(response)\n\t\tif err == nil {\n\t\t\treturn string(bin), nil\n\t\t}\n\t\t// if it is krb error no need to do udp\n\t\tif e, ok := err.(messages.KRBError); ok {\n\t\t\treturn string(response), e\n\t\t}\n\t}\n\n\t// fallback to udp\n\tresponse, err = sendToKDCUdp(kclient, msg)\n\tif err == nil {\n\t\t// if it related to udp\n\t\tbin, err := CheckKrbError(response)\n\t\tif err == nil {\n\t\t\treturn string(bin), nil\n\t\t}\n\t}\n\treturn string(response), err\n}\n\n// sendToKDCTcp sends a message to the KDC via TCP.\nfunc sendToKDCTcp(kclient *Client, msg string) ([]byte, error) {\n\t_, kdcs, err := kclient.Krb5Config.GetKDCs(kclient.Realm, true)\n\tkclient.nj.HandleError(err, \"error getting KDCs\")\n\tkclient.nj.Require(len(kdcs) > 0, \"no KDCs found\")\n\n\tvar errs []string\n\tfor i := 1; i <= len(kdcs); i++ {\n\t\thost, port, err := net.SplitHostPort(kdcs[i])\n\t\tif err == nil && kclient.config.ip != \"\" {\n\t\t\t// use that ip address instead of realm/domain for resolving\n\t\t\thost = kclient.config.ip\n\t\t}\n\t\ttcpConn, err := protocolstate.Dialer.Dial(context.TODO(), \"tcp\", net.JoinHostPort(host, port))\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Sprintf(\"error establishing connection to %s: %v\", kdcs[i], err))\n\t\t\tcontinue\n\t\t}\n\t\tdefer tcpConn.Close()\n\t\t_ = tcpConn.SetDeadline(time.Now().Add(time.Duration(kclient.config.timeout) * time.Second)) //read and write deadline\n\t\trb, err := sendTCP(tcpConn.(*net.TCPConn), []byte(msg))\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Sprintf(\"error sending to %s: %v\", kdcs[i], err))\n\t\t\tcontinue\n\t\t}\n\t\treturn rb, nil\n\t}\n\tif len(errs) > 0 {\n\t\treturn nil, fmt.Errorf(\"error sending to a KDC: %s\", strings.Join(errs, \"; \"))\n\t}\n\treturn nil, nil\n}\n\n// sendToKDCUdp sends a message to the KDC via UDP.\nfunc sendToKDCUdp(kclient *Client, msg string) ([]byte, error) {\n\t_, kdcs, err := kclient.Krb5Config.GetKDCs(kclient.Realm, true)\n\tkclient.nj.HandleError(err, \"error getting KDCs\")\n\tkclient.nj.Require(len(kdcs) > 0, \"no KDCs found\")\n\n\tvar errs []string\n\tfor i := 1; i <= len(kdcs); i++ {\n\t\thost, port, err := net.SplitHostPort(kdcs[i])\n\t\tif err == nil && kclient.config.ip != \"\" {\n\t\t\t// use that ip address instead of realm/domain for resolving\n\t\t\thost = kclient.config.ip\n\t\t}\n\t\tudpConn, err := protocolstate.Dialer.Dial(context.TODO(), \"udp\", net.JoinHostPort(host, port))\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Sprintf(\"error establishing connection to %s: %v\", kdcs[i], err))\n\t\t\tcontinue\n\t\t}\n\t\tdefer udpConn.Close()\n\t\t_ = udpConn.SetDeadline(time.Now().Add(time.Duration(kclient.config.timeout) * time.Second)) //read and write deadline\n\t\trb, err := sendUDP(udpConn.(*net.UDPConn), []byte(msg))\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Sprintf(\"error sending to %s: %v\", kdcs[i], err))\n\t\t\tcontinue\n\t\t}\n\t\treturn rb, nil\n\t}\n\tif len(errs) > 0 {\n\t\t// fallback to tcp\n\t\treturn nil, fmt.Errorf(\"error sending to a KDC: %s\", strings.Join(errs, \"; \"))\n\t}\n\treturn nil, nil\n}\n\n// sendUDP sends bytes to connection over UDP.\nfunc sendUDP(conn *net.UDPConn, b []byte) ([]byte, error) {\n\tvar r []byte\n\tdefer conn.Close()\n\t_, err := conn.Write(b)\n\tif err != nil {\n\t\treturn r, fmt.Errorf(\"error sending to (%s): %v\", conn.RemoteAddr().String(), err)\n\t}\n\tudpbuf := make([]byte, 4096)\n\tn, _, err := conn.ReadFrom(udpbuf)\n\tr = udpbuf[:n]\n\tif err != nil {\n\t\treturn r, fmt.Errorf(\"sending over UDP failed to %s: %v\", conn.RemoteAddr().String(), err)\n\t}\n\tif len(r) < 1 {\n\t\treturn r, fmt.Errorf(\"no response data from %s\", conn.RemoteAddr().String())\n\t}\n\treturn r, nil\n}\n\n// sendTCP sends bytes to connection over TCP.\nfunc sendTCP(conn *net.TCPConn, b []byte) ([]byte, error) {\n\tdefer conn.Close()\n\tvar r []byte\n\t// RFC 4120 7.2.2 specifies the first 4 bytes indicate the length of the message in big endian order.\n\thb := make([]byte, 4)\n\tbinary.BigEndian.PutUint32(hb, uint32(len(b)))\n\tb = append(hb, b...)\n\n\t_, err := conn.Write(b)\n\tif err != nil {\n\t\treturn r, fmt.Errorf(\"error sending to KDC (%s): %v\", conn.RemoteAddr().String(), err)\n\t}\n\n\tsh := make([]byte, 4)\n\t_, err = conn.Read(sh)\n\tif err != nil {\n\t\treturn r, fmt.Errorf(\"error reading response size header: %v\", err)\n\t}\n\ts := binary.BigEndian.Uint32(sh)\n\n\trb := make([]byte, s)\n\t_, err = io.ReadFull(conn, rb)\n\tif err != nil {\n\t\treturn r, fmt.Errorf(\"error reading response: %v\", err)\n\t}\n\tif len(rb) < 1 {\n\t\treturn r, fmt.Errorf(\"no response data from KDC %s\", conn.RemoteAddr().String())\n\t}\n\treturn rb, nil\n}\n\n// CheckKrbError checks if the response bytes from the KDC are a KRBError.\nfunc CheckKrbError(b []byte) ([]byte, error) {\n\tvar KRBErr messages.KRBError\n\tif err := KRBErr.Unmarshal(b); err == nil {\n\t\treturn b, KRBErr\n\t}\n\treturn b, nil\n}\n\n// TGStoHashcat converts a TGS to a hashcat format.\nfunc TGStoHashcat(tgs messages.Ticket, username string) (string, error) {\n\treturn fmt.Sprintf(\"$krb5tgs$%d$*%s$%s$%s*$%s$%s\",\n\t\ttgs.EncPart.EType,\n\t\tusername,\n\t\ttgs.Realm,\n\t\tstrings.Join(tgs.SName.NameString[:], \"/\"),\n\t\thex.EncodeToString(tgs.EncPart.Cipher[:16]),\n\t\thex.EncodeToString(tgs.EncPart.Cipher[16:]),\n\t), nil\n}\n\n// ASRepToHashcat converts an AS-REP message to a hashcat format\nfunc ASRepToHashcat(asrep messages.ASRep) (string, error) {\n\treturn fmt.Sprintf(\"$krb5asrep$%d$%s@%s:%s$%s\",\n\t\tasrep.EncPart.EType,\n\t\tasrep.CName.PrincipalNameString(),\n\t\tasrep.CRealm,\n\t\thex.EncodeToString(asrep.EncPart.Cipher[:16]),\n\t\thex.EncodeToString(asrep.EncPart.Cipher[16:])), nil\n}\n\n// Path: pkg/js/libs/ldap/ldap.go\npackage ldap\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/go-ldap/ldap/v3\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate\"\n)\n\ntype (\n\t// Client is a client for ldap protocol in nuclei\n\t// @example\n\t// ```javascript\n\t// const ldap = require('nuclei/ldap');\n\t// // here ldap.example.com is the ldap server and acme.com is the realm\n\t// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');\n\t// ```\n\t// @example\n\t// ```javascript\n\t// const ldap = require('nuclei/ldap');\n\t// const cfg = new ldap.Config();\n\t// cfg.Timeout = 10;\n\t// cfg.ServerName = 'ldap.internal.acme.com';\n\t// // optional config can be passed as third argument\n\t// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com', cfg);\n\t// ```\n\tClient struct {\n\t\tHost   string // Hostname\n\t\tPort   int    // Port\n\t\tRealm  string // Realm\n\t\tBaseDN string // BaseDN (generated from Realm)\n\n\t\t// unexported\n\t\tnj   *utils.NucleiJS // nuclei js utils\n\t\tconn *ldap.Conn\n\t\tcfg  Config\n\t}\n)\n\ntype (\n\t// Config is extra configuration for the ldap client\n\t// @example\n\t// ```javascript\n\t// const ldap = require('nuclei/ldap');\n\t// const cfg = new ldap.Config();\n\t// cfg.Timeout = 10;\n\t// cfg.ServerName = 'ldap.internal.acme.com';\n\t// cfg.Upgrade = true; // upgrade to tls\n\t// ```\n\tConfig struct {\n\t\t// Timeout is the timeout for the ldap client in seconds\n\t\tTimeout    int\n\t\tServerName string // default to host (when using tls)\n\t\tUpgrade    bool   // when true first connects to non-tls and then upgrades to tls\n\t}\n)\n\n// Constructor for creating a new ldap client\n// The following schemas are supported for url: ldap://, ldaps://, ldapi://,\n// and cldap:// (RFC1798, deprecated but used by Active Directory).\n// ldaps uses TLS/SSL, ldapi uses a Unix domain socket, and cldap uses connectionless LDAP.\n// Constructor: constructor(public ldapUrl: string, public realm: string, public config?: Config)\nfunc NewClient(call goja.ConstructorCall, runtime *goja.Runtime) *goja.Object {\n\t// setup nucleijs utils\n\tc := &Client{nj: utils.NewNucleiJS(runtime)}\n\tc.nj.ObjectSig = \"Client(ldapUrl,Realm,{Config})\" // will be included in error messages\n\n\t// get arguments (type assertion is efficient than reflection)\n\tldapUrl, _ := c.nj.GetArg(call.Arguments, 0).(string)\n\trealm, _ := c.nj.GetArg(call.Arguments, 1).(string)\n\tc.cfg = utils.GetStructTypeSafe[Config](c.nj, call.Arguments, 2, Config{})\n\tc.Realm = realm\n\tc.BaseDN = fmt.Sprintf(\"dc=%s\", strings.Join(strings.Split(realm, \".\"), \",dc=\"))\n\n\t// validate arguments\n\tc.nj.Require(ldapUrl != \"\", \"ldap url cannot be empty\")\n\tc.nj.Require(realm != \"\", \"realm cannot be empty\")\n\n\tu, err := url.Parse(ldapUrl)\n\tc.nj.HandleError(err, \"invalid ldap url supported schemas are ldap://, ldaps://, ldapi://, and cldap://\")\n\n\tvar conn net.Conn\n\tif u.Scheme == \"ldapi\" {\n\t\tif u.Path == \"\" || u.Path == \"/\" {\n\t\t\tu.Path = \"/var/run/slapd/ldapi\"\n\t\t}\n\t\tconn, err = protocolstate.Dialer.Dial(context.TODO(), \"unix\", u.Path)\n\t\tc.nj.HandleError(err, \"failed to connect to ldap server\")\n\t} else {\n\t\thost, port, err := net.SplitHostPort(u.Host)\n\t\tif err != nil {\n\t\t\t// we assume that error is due to missing port\n\t\t\thost = u.Host\n\t\t\tport = \"\"\n\t\t}\n\t\tif u.Scheme == \"\" {\n\t\t\t// default to ldap\n\t\t\tu.Scheme = \"ldap\"\n\t\t}\n\n\t\tswitch u.Scheme {\n\t\tcase \"cldap\":\n\t\t\tif port == \"\" {\n\t\t\t\tport = ldap.DefaultLdapPort\n\t\t\t}\n\t\t\tconn, err = protocolstate.Dialer.Dial(context.TODO(), \"udp\", net.JoinHostPort(host, port))\n\t\tcase \"ldap\":\n\t\t\tif port == \"\" {\n\t\t\t\tport = ldap.DefaultLdapPort\n\t\t\t}\n\t\t\tconn, err = protocolstate.Dialer.Dial(context.TODO(), \"tcp\", net.JoinHostPort(host, port))\n\t\tcase \"ldaps\":\n\t\t\tif port == \"\" {\n\t\t\t\tport = ldap.DefaultLdapsPort\n\t\t\t}\n\t\t\tserverName := host\n\t\t\tif c.cfg.ServerName != \"\" {\n\t\t\t\tserverName = c.cfg.ServerName\n\t\t\t}\n\t\t\tconn, err = protocolstate.Dialer.DialTLSWithConfig(context.TODO(), \"tcp\", net.JoinHostPort(host, port),\n\t\t\t\t&tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10, ServerName: serverName})\n\t\tdefault:\n\t\t\terr = fmt.Errorf(\"unsupported ldap url schema %v\", u.Scheme)\n\t\t}\n\t\tc.nj.HandleError(err, \"failed to connect to ldap server\")\n\t}\n\tc.conn = ldap.NewConn(conn, u.Scheme == \"ldaps\")\n\tif u.Scheme != \"ldaps\" && c.cfg.Upgrade {\n\t\tserverName := u.Hostname()\n\t\tif c.cfg.ServerName != \"\" {\n\t\t\tserverName = c.cfg.ServerName\n\t\t}\n\t\tif err := c.conn.StartTLS(&tls.Config{InsecureSkipVerify: true, ServerName: serverName}); err != nil {\n\t\t\tc.nj.HandleError(err, \"failed to upgrade to tls\")\n\t\t}\n\t} else {\n\t\tc.conn.Start()\n\t}\n\n\treturn utils.LinkConstructor(call, runtime, c)\n}\n\n// Authenticate authenticates with the ldap server using the given username and password\n// performs NTLMBind first and then Bind/UnauthenticatedBind if NTLMBind fails\n// @example\n// ```javascript\n// const ldap = require('nuclei/ldap');\n// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');\n// client.Authenticate('user', 'password');\n// ```\nfunc (c *Client) Authenticate(username, password string) {\n\tc.nj.Require(c.conn != nil, \"no existing connection\")\n\tif c.BaseDN == \"\" {\n\t\tc.BaseDN = fmt.Sprintf(\"dc=%s\", strings.Join(strings.Split(c.Realm, \".\"), \",dc=\"))\n\t}\n\tif err := c.conn.NTLMBind(c.Realm, username, password); err == nil {\n\t\t// if bind with NTLMBind(), there is nothing\n\t\t// else to do, you are authenticated\n\t\treturn\n\t}\n\n\tswitch password {\n\tcase \"\":\n\t\tif err := c.conn.UnauthenticatedBind(username); err != nil {\n\t\t\tc.nj.ThrowError(err)\n\t\t}\n\tdefault:\n\t\tif err := c.conn.Bind(username, password); err != nil {\n\t\t\tc.nj.ThrowError(err)\n\t\t}\n\t}\n}\n\n// AuthenticateWithNTLMHash authenticates with the ldap server using the given username and NTLM hash\n// @example\n// ```javascript\n// const ldap = require('nuclei/ldap');\n// const client = new ldap.Client('ldap://ldap.example.com', 'acme.com');\n// client.AuthenticateWithNTLMHash('pdtm', 'hash');\n// ```\nfunc (c *Client) AuthenticateWithNTLMHash(username, hash string) {\n\tc.nj.Require(c.conn != nil, \"no existing connection\")\n\tif c.BaseDN == \"\" {\n\t\tc.BaseDN = fmt.Sprintf(\"dc=%s\", strings.Join(strings.Split(c.Realm, \".\"), \",dc=\"))\n\t}\n\tif err := c.conn.NTLMBindWithHash(c.Realm, username, hash); err != nil {\n\t\tc.nj.ThrowError(err)\n\t}\n}\n\n...\n", "needle_token_start": 3651, "needle_token_end": 3943, "code_context_ntokens": 15372, "output": ["```go\n// RegisterNativeScripts are js scripts that were added for convenience\n// and abstraction purposes we execute them in every runtime and make them\n// available for use in any js script\n// see: scripts/ for examples\n\nfunc RegisterNativeScripts(runtime *goja.Runtime) error {\n\tinitBuiltInFunc(runtime)\n\n\tdirs, err := embedFS.ReadDir(\"js\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, dir := range dirs {\n\t\tif dir.IsDir() {\n\t\t\tcontinue\n\t\t}\n\t\t// embeds have / as path separator (on all os)\n\t\tcontents, err := embedFS.ReadFile(\"js\" + \"/\" + dir.Name())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// run all built in js helper functions or scripts\n\t\t_, err = runtime.RunString(string(contents))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// exports defines the exports object\n\t_, err = runtime.RunString(exports)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// import default modules\n\t_, err = runtime.RunString(defaultImports)\n\tif err != nil {\n\t\treturn errorutil.NewWithErr(err).Msgf(\"could not import default modules %v\", defaultImports)\n\t}\n\n\treturn nil\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "isHostPort", "language": "go", "path": "pkg/protocols/http/request_annotations.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function checks if a given string includes both a host and a valid port.\n2. **Input**: A single string that potentially contains a host and a port.\n3. **Output**: A boolean value indicating whether the input string correctly includes a host and a valid port.\n4. **Procedure**: The function attempts to split the input string into host and port components. If the split is successful and the port is valid as per predefined port criteria, it returns true; otherwise, it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/protocols/http/operators_test.go\npackage http\n\nimport (\n\t\"net/http\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/testutils\"\n)\n\nfunc TestResponseToDSLMap(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\ttestutils.Init(options)\n\ttemplateID := \"testing-http\"\n\trequest := &Request{\n\t\tID:     templateID,\n\t\tName:   \"testing\",\n\t\tPath:   []string{\"{{BaseURL}}?test=1\"},\n\t\tMethod: HTTPMethodTypeHolder{MethodType: HTTPGet},\n\t}\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile file request\")\n\n\tresp := &http.Response{}\n\tresp.Header = make(http.Header)\n\tresp.Header.Set(\"Test\", \"Test-Response\")\n\thost := \"http://example.com/test/\"\n\tmatched := \"http://example.com/test/?test=1\"\n\n\tevent := request.responseToDSLMap(resp, host, matched, exampleRawRequest, exampleRawResponse, exampleResponseBody, exampleResponseHeader, 1*time.Second, map[string]interface{}{})\n\trequire.Len(t, event, 15, \"could not get correct number of items in dsl map\")\n\trequire.Equal(t, exampleRawResponse, event[\"response\"], \"could not get correct resp\")\n\trequire.Equal(t, \"Test-Response\", event[\"test\"], \"could not get correct resp for header\")\n}\n\nfunc TestHTTPOperatorMatch(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\ttestutils.Init(options)\n\ttemplateID := \"testing-http\"\n\trequest := &Request{\n\t\tID:     templateID,\n\t\tName:   \"testing\",\n\t\tPath:   []string{\"{{BaseURL}}?test=1\"},\n\t\tMethod: HTTPMethodTypeHolder{MethodType: HTTPGet},\n\t}\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile file request\")\n\n\tresp := &http.Response{}\n\tresp.Header = make(http.Header)\n\tresp.Header.Set(\"Test\", \"Test-Response\")\n\thost := \"http://example.com/test/\"\n...\n// Path: pkg/protocols/http/race/syncedreadcloser.go\npackage race\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"time\"\n)\n\n// SyncedReadCloser is compatible with io.ReadSeeker and performs\n// gate-based synced writes to enable race condition testing.\ntype SyncedReadCloser struct {\n\tdata           []byte\n\tp              int64\n\tlength         int64\n\topenGate       chan struct{}\n\tenableBlocking bool\n}\n\n// NewSyncedReadCloser creates a new SyncedReadCloser instance.\nfunc NewSyncedReadCloser(r io.ReadCloser) *SyncedReadCloser {\n\tvar (\n\t\ts   SyncedReadCloser\n\t\terr error\n\t)\n\ts.data, err = io.ReadAll(r)\n\tif err != nil {\n\t\treturn nil\n\t}\n\tr.Close()\n\ts.length = int64(len(s.data))\n\ts.openGate = make(chan struct{})\n\ts.enableBlocking = true\n\treturn &s\n}\n\n// NewOpenGateWithTimeout creates a new open gate with a timeout\nfunc NewOpenGateWithTimeout(r io.ReadCloser, d time.Duration) *SyncedReadCloser {\n\ts := NewSyncedReadCloser(r)\n\ts.OpenGateAfter(d)\n\treturn s\n}\n\n// SetOpenGate sets the status of the blocking gate\nfunc (s *SyncedReadCloser) SetOpenGate(status bool) {\n\ts.enableBlocking = status\n}\n\n// OpenGate opens the gate allowing all requests to be completed\nfunc (s *SyncedReadCloser) OpenGate() {\n\ts.openGate <- struct{}{}\n}\n\n// OpenGateAfter schedules gate to be opened after a duration\nfunc (s *SyncedReadCloser) OpenGateAfter(d time.Duration) {\n\ttime.AfterFunc(d, func() {\n\t\ts.openGate <- struct{}{}\n\t})\n}\n\n// Seek implements seek method for io.ReadSeeker\nfunc (s *SyncedReadCloser) Seek(offset int64, whence int) (int64, error) {\n\tvar err error\n\tswitch whence {\n\tcase io.SeekStart:\n\t\ts.p = 0\n\tcase io.SeekCurrent:\n\t\tif s.p+offset < s.length {\n\t\t\ts.p += offset\n\t\t\tbreak\n\t\t}\n\t\terr = fmt.Errorf(\"offset is too big\")\n\tcase io.SeekEnd:\n\t\tif s.length-offset >= 0 {\n\t\t\ts.p = s.length - offset\n\t\t\tbreak\n\t\t}\n\t\terr = fmt.Errorf(\"offset is too big\")\n\t}\n\treturn s.p, err\n}\n\n// Read implements read method for io.ReadSeeker\nfunc (s *SyncedReadCloser) Read(p []byte) (n int, err error) {\n\t// If the data fits in the buffer blocks awaiting the sync instruction\n\tif s.p+int64(len(p)) >= s.length && s.enableBlocking {\n\t\t<-s.openGate\n\t}\n\tn = copy(p, s.data[s.p:])\n\ts.p += int64(n)\n\tif s.p == s.length {\n\t\terr = io.EOF\n\t}\n\treturn n, err\n}\n\n// Close closes an io.ReadSeeker\nfunc (s *SyncedReadCloser) Close() error {\n\treturn nil\n}\n\n// Len returns the length of data in reader\nfunc (s *SyncedReadCloser) Len() int {\n\treturn int(s.length)\n}\n\n// Path: pkg/protocols/http/raw/doc.go\n// Package raw provides raw http request parsing abilities for nuclei.\npackage raw\n\n// Path: pkg/protocols/http/raw/raw_test.go\npackage raw\n\nimport (\n\t\"testing\"\n\n\turlutil \"github.com/projectdiscovery/utils/url\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestParseRawRequestWithPort(t *testing.T) {\n\trequest, err := Parse(`GET /gg/phpinfo.php HTTP/1.1\nHost: {{Hostname}}:123\nOrigin: {{BaseURL}}\nConnection: close\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko)\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\nAccept-Language: en-US,en;q=0.9`, parseURL(t, \"https://example.com:8080\"), false, false)\n\trequire.Nil(t, err, \"could not parse GET request\")\n\trequire.Equal(t, \"https://example.com:8080/gg/phpinfo.php\", request.FullURL, \"Could not parse request url correctly\")\n\trequire.Equal(t, \"/gg/phpinfo.php\", request.Path, \"Could not parse request path correctly\")\n\n\tt.Run(\"path-suffix\", func(t *testing.T) {\n\t\trequest, err := Parse(`GET /hello HTTP/1.1\nHost: {{Hostname}}`, parseURL(t, \"https://example.com:8080/test\"), false, false)\n\t\trequire.Nil(t, err, \"could not parse GET request\")\n\t\trequire.Equal(t, \"https://example.com:8080/test/hello\", request.FullURL, \"Could not parse request url correctly\")\n\t})\n\n\tt.Run(\"query-values\", func(t *testing.T) {\n\t\trequest, err := Parse(`GET ?username=test&password=test HTTP/1.1\nHost: {{Hostname}}:123`, parseURL(t, \"https://example.com:8080/test\"), false, false)\n\t\trequire.Nil(t, err, \"could not parse GET request\")\n\t\t// url.values are sorted to avoid randomness of using maps\n\t\trequire.Equal(t, \"https://example.com:8080/test?username=test&password=test\", request.FullURL, \"Could not parse request url correctly\")\n\n\t\trequest, err = Parse(`GET ?username=test&password=test HTTP/1.1\nHost: {{Hostname}}:123`, parseURL(t, \"https://example.com:8080/test/\"), false, false)\n\t\trequire.Nil(t, err, \"could not parse GET request\")\n\t\trequire.Equal(t, \"https://example.com:8080/test/?username=test&password=test\", request.FullURL, \"Could not parse request url correctly\")\n\n\t\trequest, err = Parse(`GET /?username=test&password=test HTTP/1.1\n\t\tHost: {{Hostname}}:123`, parseURL(t, \"https://example.com:8080/test/\"), false, false)\n\t\trequire.Nil(t, err, \"could not parse GET request\")\n\t\trequire.Equal(t, \"https://example.com:8080/test/?username=test&password=test\", request.FullURL, \"Could not parse request url correctly\")\n\t})\n}\n\nfunc TestParseRawRequest(t *testing.T) {\n\trequest, err := Parse(`GET /manager/html HTTP/1.1\nHost: {{Hostname}}\nAuthorization: Basic {{base64('username:password')}}\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0\nAccept-Language: en-US,en;q=0.9\nConnection: close`, parseURL(t, \"https://test.com\"), false, false)\n\trequire.Nil(t, err, \"could not parse GET request\")\n\trequire.Equal(t, \"GET\", request.Method, \"Could not parse GET method request correctly\")\n\trequire.Equal(t, \"/manager/html\", request.Path, \"Could not parse request path correctly\")\n\n\trequest, err = Parse(`POST /login HTTP/1.1\nHost: {{Hostname}}\nContent-Type: application/x-www-form-urlencoded\nConnection: close\n\nusername=admin&password=login`, parseURL(t, \"https://test.com\"), false, false)\n\trequire.Nil(t, err, \"could not parse POST request\")\n\trequire.Equal(t, \"POST\", request.Method, \"Could not parse POST method request correctly\")\n\trequire.Equal(t, \"username=admin&password=login\", request.Data, \"Could not parse request data correctly\")\n}\n\nfunc TestParseUnsafeRequestWithPath(t *testing.T) {\n\trequest, err := Parse(`GET /manager/html HTTP/1.1\nHost: {{Hostname}}\nAuthorization: Basic {{base64('username:password')}}\nUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0\nAccept-Language: en-US,en;q=0.9\nConnection: close`, parseURL(t, \"https://test.com/test/\"), true, false)\n\trequire.Nil(t, err, \"could not parse unsafe request\")\n\trequire.Contains(t, string(request.UnsafeRawBytes), \"GET /test/manager/html\", \"Could not parse unsafe method request path correctly\")\n\n\trequest, err = Parse(`GET ?a=b HTTP/1.1\n\tHost: {{Hostname}}\n\tOrigin: {{BaseURL}}`, parseURL(t, \"https://test.com/test.js\"), true, false)\n\trequire.Nil(t, err, \"could not parse unsafe request\")\n\trequire.Contains(t, string(request.UnsafeRawBytes), \"GET /test.js?a=b\", \"Could not parse unsafe method request path correctly\")\n}\n\nfunc TestTryFillCustomHeaders(t *testing.T) {\n\ttestValue := \"GET /manager/html HTTP/1.1\\r\\nHost: Test\\r\\n\"\n\texpected := \"GET /test/manager/html HTTP/1.1\\r\\nHost: Test\\r\\ntest: test\\r\\n\"\n\trequest, err := Parse(testValue, parseURL(t, \"https://test.com/test/\"), true, false)\n\trequire.Nil(t, err, \"could not parse unsafe request\")\n\terr = request.TryFillCustomHeaders([]string{\"test: test\"})\n\trequire.Nil(t, err, \"could not add custom headers\")\n\trequire.Equal(t, expected, string(request.UnsafeRawBytes), \"actual value and expected value are different\")\n}\n\nfunc TestDisableMergePath(t *testing.T) {\n\trequest, err := Parse(` GET /api/v1/id=123 HTTP/1.1\n\tHost: {{Hostname}}`, parseURL(t, \"https://example.com/api/v1/user\"), false, true)\n\trequire.Nil(t, err, \"could not parse GET request with disable merge path\")\n\trequire.Equal(t, \"https://example.com/api/v1/id=123\", request.FullURL, \"Could not parse request url with disable merge path correctly\")\n\n\trequest, err = Parse(` GET /api/v1/id=123 HTTP/1.1\n\tHost: {{Hostname}}`, parseURL(t, \"https://example.com/api/v1/user\"), false, false)\n\trequire.Nil(t, err, \"could not parse GET request with merge path\")\n\trequire.Equal(t, \"https://example.com/api/v1/user/api/v1/id=123\", request.FullURL, \"Could not parse request url with merge path correctly\")\n\n}\n\nfunc parseURL(t *testing.T, inputurl string) *urlutil.URL {\n\turlx, err := urlutil.Parse(inputurl)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to parse url %v\", urlx)\n\t}\n\treturn urlx\n}\n\n// Path: pkg/protocols/http/request_annotations.go\npackage http\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"net\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/projectdiscovery/fastdialer/fastdialer\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n\tiputil \"github.com/projectdiscovery/utils/ip\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\nvar (\n\t// @Host:target overrides the input target with the annotated one (similar to self-contained requests)\n\treHostAnnotation = regexp.MustCompile(`(?m)^@Host:\\s*(.+)\\s*$`)\n\t// @tls-sni:target overrides the input target with the annotated one\n\t// special values:\n\t// request.host: takes the value from the host header\n\t// target: overrides with the specific value\n\treSniAnnotation = regexp.MustCompile(`(?m)^@tls-sni:\\s*(.+)\\s*$`)\n\t// @timeout:duration overrides the input timeout with a custom duration\n\treTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\\s*(.+)\\s*$`)\n\t// @once sets the request to be executed only once for a specific URL\n\treOnceAnnotation = regexp.MustCompile(`(?m)^@once\\s*$`)\n)\n\ntype flowMark int\n\nconst (\n\tOnce flowMark = iota\n)\n\n// parseFlowAnnotations and override requests flow\nfunc parseFlowAnnotations(rawRequest string) (flowMark, bool) {\n\tvar fm flowMark\n\t// parse request for known override annotations\n\tvar hasFlowOverride bool\n\t// @once\n\tif reOnceAnnotation.MatchString(rawRequest) {\n\t\tfm = Once\n\t\thasFlowOverride = true\n\t}\n\n\treturn fm, hasFlowOverride\n}\n\ntype annotationOverrides struct {\n\trequest        *retryablehttp.Request\n\tcancelFunc     context.CancelFunc\n\tinteractshURLs []string\n}\n\n// parseAnnotations and override requests settings\nfunc (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Request) (overrides annotationOverrides, modified bool) {\n\t// parse request for known override annotations\n\n\t// @Host:target\n\tif hosts := reHostAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {\n\t\tvalue := strings.TrimSpace(hosts[1])\n\t\t// handle scheme\n\t\tswitch {\n\t\tcase stringsutil.HasPrefixI(value, \"http://\"):\n\t\t\trequest.URL.Scheme = \"http\"\n\t\tcase stringsutil.HasPrefixI(value, \"https://\"):\n\t\t\trequest.URL.Scheme = \"https\"\n\t\t}\n\n\t\tvalue = stringsutil.TrimPrefixAny(value, \"http://\", \"https://\")\n\n\t\tif isHostPort(value) {\n\t\t\trequest.URL.Host = value\n\t\t} else {\n\t\t\thostPort := value\n\t\t\tport := request.URL.Port()\n\t\t\tif port != \"\" {\n\t\t\t\thostPort = net.JoinHostPort(hostPort, port)\n\t\t\t}\n\t\t\trequest.URL.Host = hostPort\n\t\t}\n\t\tmodified = true\n\t}\n\n\t// @tls-sni:target\n\tif hosts := reSniAnnotation.FindStringSubmatch(rawRequest); len(hosts) > 0 {\n\t\tvalue := strings.TrimSpace(hosts[1])\n\t\tvalue = stringsutil.TrimPrefixAny(value, \"http://\", \"https://\")\n\n\t\tvar literal bool\n\t\tswitch value {\n\t\tcase \"request.host\":\n\t\t\tvalue = request.Host\n\t\tcase \"interactsh-url\":\n\t\t\tif interactshURL, err := r.options.Interactsh.NewURLWithData(\"interactsh-url\"); err == nil {\n\t\t\t\tvalue = interactshURL\n\t\t\t}\n\t\t\toverrides.interactshURLs = append(overrides.interactshURLs, value)\n\t\tdefault:\n\t\t\tliteral = true\n\t\t}\n\t\tctx := context.WithValue(request.Context(), fastdialer.SniName, value)\n\t\trequest = request.Clone(ctx)\n\n\t\tif literal {\n\t\t\trequest.TLS = &tls.ConnectionState{ServerName: value}\n\t\t}\n\t\tmodified = true\n\t}\n\n\t// @timeout:duration\n\tif r.connConfiguration.NoTimeout {\n\t\tmodified = true\n\t\tvar ctx context.Context\n\n\t\tif duration := reTimeoutAnnotation.FindStringSubmatch(rawRequest); len(duration) > 0 {\n\t\t\tvalue := strings.TrimSpace(duration[1])\n\t\t\tif parsed, err := time.ParseDuration(value); err == nil {\n\t\t\t\t//nolint:govet // cancelled automatically by withTimeout\n\t\t\t\tctx, overrides.cancelFunc = context.WithTimeout(context.Background(), parsed)\n\t\t\t\trequest = request.Clone(ctx)\n\t\t\t}\n\t\t} else {\n\t\t\t//nolint:govet // cancelled automatically by withTimeout\n\t\t\tctx, overrides.cancelFunc = context.WithTimeout(context.Background(), time.Duration(r.options.Options.Timeout)*time.Second)\n\t\t\trequest = request.Clone(ctx)\n\t\t}\n\t}\n\n\toverrides.request = request\n\n\treturn\n}\n\n\nfunc isHostPort(value string) bool {\n\t_, port, err := net.SplitHostPort(value)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif !iputil.IsPort(port) {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// Path: pkg/protocols/http/request_annotations_test.go\npackage http\n\nimport (\n\t\"context\"\n\t\"net/http\"\n\t\"testing\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestRequestParseAnnotationsSNI(t *testing.T) {\n\tt.Run(\"compliant-SNI-value\", func(t *testing.T) {\n\t\treq := &Request{connConfiguration: &httpclientpool.Configuration{}}\n\t\trawRequest := `@tls-sni: github.com\n\t\tGET / HTTP/1.1\n\t\tHost: {{Hostname}}`\n\n\t\thttpReq, err := retryablehttp.NewRequest(http.MethodGet, \"https://example.com\", nil)\n\t\trequire.Nil(t, err, \"could not create http request\")\n\n\t\toverrides, modified := req.parseAnnotations(rawRequest, httpReq)\n\t\trequire.True(t, modified, \"could not apply request annotations\")\n\t\trequire.Equal(t, \"github.com\", overrides.request.TLS.ServerName)\n\t\trequire.Equal(t, \"example.com\", overrides.request.URL.Hostname())\n\t})\n\tt.Run(\"non-compliant-SNI-value\", func(t *testing.T) {\n\t\treq := &Request{connConfiguration: &httpclientpool.Configuration{}}\n\t\trawRequest := `@tls-sni: ${jndi:ldap://${hostName}.test.com}\n\t\tGET / HTTP/1.1\n\t\tHost: {{Hostname}}`\n\n\t\thttpReq, err := retryablehttp.NewRequest(http.MethodGet, \"https://example.com\", nil)\n\t\trequire.Nil(t, err, \"could not create http request\")\n\n\t\toverrides, modified := req.parseAnnotations(rawRequest, httpReq)\n\t\trequire.True(t, modified, \"could not apply request annotations\")\n\t\trequire.Equal(t, \"${jndi:ldap://${hostName}.test.com}\", overrides.request.TLS.ServerName)\n\t\trequire.Equal(t, \"example.com\", overrides.request.URL.Hostname())\n\t})\n}\n\nfunc TestRequestParseAnnotationsTimeout(t *testing.T) {\n\tt.Run(\"positive\", func(t *testing.T) {\n\t\trequest := &Request{\n\t\t\tconnConfiguration: &httpclientpool.Configuration{NoTimeout: true},\n\t\t}\n\t\trawRequest := `@timeout: 2s\n\t\tGET / HTTP/1.1\n\t\tHost: {{Hostname}}`\n\n\t\thttpReq, err := retryablehttp.NewRequest(http.MethodGet, \"https://example.com\", nil)\n\t\trequire.Nil(t, err, \"could not create http request\")\n\n\t\toverrides, modified := request.parseAnnotations(rawRequest, httpReq)\n\t\trequire.NotNil(t, overrides.cancelFunc, \"could not initialize valid cancel function\")\n\t\trequire.True(t, modified, \"could not get correct modified value\")\n\t\t_, deadlined := overrides.request.Context().Deadline()\n\t\trequire.True(t, deadlined, \"could not get set request deadline\")\n\t})\n\n\tt.Run(\"negative\", func(t *testing.T) {\n\t\trequest := &Request{\n\t\t\tconnConfiguration: &httpclientpool.Configuration{},\n\t\t}\n\t\trawRequest := `GET / HTTP/1.1\n\t\tHost: {{Hostname}}`\n\n\t\thttpReq, err := retryablehttp.NewRequestWithContext(context.Background(), http.MethodGet, \"https://example.com\", nil)\n\t\trequire.Nil(t, err, \"could not create http request\")\n\n\t\tnewRequestWithOverrides, modified := request.parseAnnotations(rawRequest, httpReq)\n\t\trequire.Nil(t, newRequestWithOverrides.cancelFunc, \"cancel function should be nil\")\n\t\trequire.False(t, modified, \"could not get correct modified value\")\n\t\t_, deadlined := newRequestWithOverrides.request.Context().Deadline()\n\t\trequire.False(t, deadlined, \"could not get set request deadline\")\n\t})\n}\n\n// Path: pkg/protocols/http/request_condition.go\npackage http\n\nimport (\n\t\"regexp\"\n)\n\nvar (\n\t// Determines if request condition are needed by detecting the pattern _xxx\n\treRequestCondition = regexp.MustCompile(`(?m)_\\d+`)\n)\n\n// NeedsRequestCondition determines if request condition should be enabled\nfunc (request *Request) NeedsRequestCondition() bool {\n\tfor _, matcher := range request.Matchers {\n\t\tif checkRequestConditionExpressions(matcher.DSL...) {\n\t\t\treturn true\n\t\t}\n\t\tif checkRequestConditionExpressions(matcher.Part) {\n\t\t\treturn true\n\t\t}\n\t}\n\tfor _, extractor := range request.Extractors {\n\t\tif checkRequestConditionExpressions(extractor.DSL...) {\n\t\t\treturn true\n\t\t}\n\t\tif checkRequestConditionExpressions(extractor.Part) {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc checkRequestConditionExpressions(expressions ...string) bool {\n\tfor _, expression := range expressions {\n\t\tif reRequestCondition.MatchString(expression) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Path: pkg/protocols/http/request_fuzz.go\npackage http\n\n// === Fuzzing Documentation (Scoped to this File) =====\n// -> request.executeFuzzingRule   [iterates over payloads(+requests) and executes]\n//\t-> request.executePayloadUsingRules [executes single payload on all rules (if more than 1)]\n//\t\t-> request.executeGeneratedFuzzingRequest [execute final generated fuzzing request and get result]\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/fuzz\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n\turlutil \"github.com/projectdiscovery/utils/url\"\n)\n\n// executeFuzzingRule executes fuzzing request for a URL\n// TODO:\n// 1. use SPMHandler and rewrite stop at first match logic here\n// 2. use scanContext instead of contextargs.Context\nfunc (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\t// methdology:\n\t// to check applicablity of rule, we first try to execute it with one value\n\t// if it is applicable, we execute all requests\n\t// if it is not applicable, we log and fail silently\n\n\t// check if target should be fuzzed or not\n\tif !request.ShouldFuzzTarget(input) {\n\t\turlx, _ := input.MetaInput.URL()\n\t\tif urlx != nil {\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: target(%s) not applicable for fuzzing\\n\", request.options.TemplateID, urlx.String())\n\t\t} else {\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: target(%s) not applicable for fuzzing\\n\", request.options.TemplateID, input.MetaInput.Input)\n\t\t}\n\t\treturn nil\n\t}\n\n\tif input.MetaInput.Input == \"\" && input.MetaInput.ReqResp == nil {\n\t\treturn errors.New(\"empty input provided for fuzzing\")\n\t}\n\n\t// ==== fuzzing when full HTTP request is provided =====\n\n\tif input.MetaInput.ReqResp != nil {\n\t\tbaseRequest, err := input.MetaInput.ReqResp.BuildRequest()\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"fuzz: could not build request obtained from target file\")\n\t\t}\n\t\tinput.MetaInput.Input = baseRequest.URL.String()\n\t\t// execute with one value first to checks its applicability\n\t\terr = request.executeAllFuzzingRules(input, previous, baseRequest, callback)\n\t\tif err != nil {\n\t\t\t// in case of any error, return it\n\t\t\tif fuzz.IsErrRuleNotApplicable(err) {\n\t\t\t\t// log and fail silently\n\t\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: %s\\n\", request.options.TemplateID, err)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif errors.Is(err, ErrMissingVars) {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: payload request execution failed : %s\\n\", request.options.TemplateID, err)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// ==== fuzzing when only URL is provided =====\n\n\t// we need to use this url instead of input\n\tinputx := input.Clone()\n\tparsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, true)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fuzz: could not parse input url\")\n\t}\n\tbaseRequest, err := retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fuzz: could not build request from url\")\n\t}\n\t// execute with one value first to checks its applicability\n\terr = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback)\n\tif err != nil {\n\t\t// in case of any error, return it\n\t\tif fuzz.IsErrRuleNotApplicable(err) {\n\t\t\t// log and fail silently\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: rule not applicable : %s\\n\", request.options.TemplateID, err)\n\t\t\treturn nil\n\t\t}\n\t\tif errors.Is(err, ErrMissingVars) {\n\t\t\treturn err\n\t\t}\n\t\tgologger.Verbose().Msgf(\"[%s] fuzz: payload request execution failed : %s\\n\", request.options.TemplateID, err)\n\t}\n\treturn nil\n}\n\n// executeAllFuzzingRules executes all fuzzing rules defined in template for a given base request\nfunc (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error {\n\tapplicable := false\n\tfor _, rule := range request.Fuzzing {\n\t\terr := rule.Execute(&fuzz.ExecuteRuleInput{\n\t\t\tInput: input,\n\t\t\tCallback: func(gr fuzz.GeneratedRequest) bool {\n\t\t\t\t// TODO: replace this after scanContext Refactor\n\t\t\t\treturn request.executeGeneratedFuzzingRequest(gr, input, callback)\n\t\t\t},\n\t\t\tValues:      values,\n\t\t\tBaseRequest: baseRequest.Clone(context.TODO()),\n\t\t})\n\t\tif err == nil {\n\t\t\tapplicable = true\n\t\t\tcontinue\n\t\t}\n\t\tif fuzz.IsErrRuleNotApplicable(err) {\n\t\t\tcontinue\n\t\t}\n\t\tif err == types.ErrNoMoreRequests {\n\t\t\treturn nil\n\t\t}\n\t\treturn errors.Wrap(err, \"could not execute rule\")\n\t}\n\n\tif !applicable {\n\t\treturn fuzz.ErrRuleNotApplicable.Msgf(fmt.Sprintf(\"no rule was applicable for this request: %v\", input.MetaInput.Input))\n\t}\n\treturn nil\n}\n\n// executeGeneratedFuzzingRequest executes a generated fuzzing request after building it using rules and payloads\nfunc (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool {\n\thasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)\n\thasInteractMarkers := len(gr.InteractURLs) > 0\n\tif request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {\n\t\treturn false\n\t}\n\trequest.options.RateLimitTake()\n\treq := &generatedRequest{\n\t\trequest:        gr.Request,\n\t\tdynamicValues:  gr.DynamicValues,\n\t\tinteractshURLs: gr.InteractURLs,\n\t\toriginal:       request,\n\t}\n\tvar gotMatches bool\n\trequestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {\n\t\tif hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {\n\t\t\trequestData := &interactsh.RequestData{\n\t\t\t\tMakeResultFunc: request.MakeResultEvent,\n\t\t\t\tEvent:          event,\n\t\t\t\tOperators:      request.CompiledOperators,\n\t\t\t\tMatchFunc:      request.Match,\n\t\t\t\tExtractFunc:    request.Extract,\n\t\t\t}\n\t\t\trequest.options.Interactsh.RequestEvent(gr.InteractURLs, requestData)\n\t\t\tgotMatches = request.options.Interactsh.AlreadyMatched(requestData)\n\t\t} else {\n\t\t\tcallback(event)\n\t\t}\n\t\t// Add the extracts to the dynamic values if any.\n\t\tif event.OperatorsResult != nil {\n\t\t\tgotMatches = event.OperatorsResult.Matched\n\t\t}\n\t}, 0)\n\t// If a variable is unresolved, skip all further requests\n\tif errors.Is(requestErr, ErrMissingVars) {\n\t\treturn false\n\t}\n\tif requestErr != nil {\n\t\tif request.options.HostErrorsCache != nil {\n\t\t\trequest.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr)\n\t\t}\n\t\tgologger.Verbose().Msgf(\"[%s] Error occurred in request: %s\\n\", request.options.TemplateID, requestErr)\n\t}\n\trequest.options.Progress.IncrementRequests()\n\n\t// If this was a match, and we want to stop at first match, skip all further requests.\n\tshouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch\n\tif shouldStopAtFirstMatch && gotMatches {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// ShouldFuzzTarget checks if given target should be fuzzed or not using `filter` field in template\nfunc (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool {\n\tif len(request.FuzzPreCondition) == 0 {\n\t\treturn true\n\t}\n\tstatus := []bool{}\n\tfor index, filter := range request.FuzzPreCondition {\n\t\tisMatch, _ := request.Match(request.filterDataMap(input), filter)\n\t\tstatus = append(status, isMatch)\n\t\tif request.options.Options.MatcherStatus {\n\t\t\tgologger.Debug().Msgf(\"[%s] [%s] Filter => %s : %v\", input.MetaInput.Target(), request.options.TemplateID, operators.GetMatcherName(filter, index), isMatch)\n\t\t}\n\t}\n\tif len(status) == 0 {\n\t\treturn true\n\t}\n\tvar matched bool\n\tif request.fuzzPreConditionOperator == matchers.ANDCondition {\n\t\tmatched = operators.EvalBoolSlice(status, true)\n\t} else {\n\t\tmatched = operators.EvalBoolSlice(status, false)\n\t}\n\tif request.options.Options.MatcherStatus {\n\t\tgologger.Debug().Msgf(\"[%s] [%s] Final Filter Status =>  %v\", input.MetaInput.Target(), request.options.TemplateID, matched)\n\t}\n\treturn matched\n}\n\n// input data map returns map[string]interface{} from input\nfunc (request *Request) filterDataMap(input *contextargs.Context) map[string]interface{} {\n\tm := make(map[string]interface{})\n\tparsed, err := input.MetaInput.URL()\n\tif err != nil {\n\t\tm[\"host\"] = input.MetaInput.Input\n\t\treturn m\n\t}\n\tm = protocolutils.GenerateVariables(parsed, true, m)\n\tfor k, v := range m {\n\t\tm[strings.ToLower(k)] = v\n\t}\n\tm[\"path\"] = parsed.Path // override existing\n\tm[\"query\"] = parsed.RawQuery\n\t// add request data like headers, body etc\n\tif input.MetaInput.ReqResp != nil && input.MetaInput.ReqResp.Request != nil {\n\t\treq := input.MetaInput.ReqResp.Request\n\t\tm[\"method\"] = req.Method\n\t\tm[\"body\"] = req.Body\n\n\t\tsb := &strings.Builder{}\n\t\treq.Headers.Iterate(func(k, v string) bool {\n\t\t\tk = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(k), \"-\", \"_\"))\n\t\t\tif strings.EqualFold(k, \"Cookie\") {\n\t\t\t\tm[\"cookie\"] = v\n\t\t\t}\n\t\t\tif strings.EqualFold(k, \"User_Agent\") {\n\t\t\t\tm[\"user_agent\"] = v\n\t\t\t}\n\t\t\tif strings.EqualFold(k, \"content_type\") {\n\t\t\t\tm[\"content_type\"] = v\n\t\t\t}\n\t\t\tsb.WriteString(fmt.Sprintf(\"%s: %s\\n\", k, v))\n\t\t\treturn true\n\t\t})\n\t\tm[\"header\"] = sb.String()\n\t} else {\n\t\t// add default method value\n\t\tm[\"method\"] = http.MethodGet\n\t}\n\n\t// dump if svd is enabled\n\tif request.options.Options.ShowVarDump {\n\t\tgologger.Debug().Msgf(\"Fuzz Filter Variables: \\n%s\\n\", vardump.DumpVariables(m))\n\t}\n\treturn m\n}\n\n// Path: pkg/protocols/http/request_generator.go\npackage http\n\nimport (\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n)\n\n// requestGenerator generates requests sequentially based on various\n// configurations for a http request template.\n//\n// If payload values are present, an iterator is created for the payload\n// values. Paths and Raw requests are supported as base input, so\n// it will automatically select between them based on the template.\ntype requestGenerator struct {\n\tcurrentIndex     int\n\tcurrentPayloads  map[string]interface{}\n\tokCurrentPayload bool\n\trequest          *Request\n\toptions          *protocols.ExecutorOptions\n\tpayloadIterator  *generators.Iterator\n\tinteractshURLs   []string\n\tonceFlow         map[string]struct{}\n}\n\n// LeaveDefaultPorts skips normalization of default standard ports\nvar LeaveDefaultPorts = false\n\n// newGenerator creates a new request generator instance\nfunc (request *Request) newGenerator(disablePayloads bool) *requestGenerator {\n\tgenerator := &requestGenerator{\n\t\trequest:  request,\n\t\toptions:  request.options,\n\t\tonceFlow: make(map[string]struct{}),\n\t}\n\n\tif len(request.Payloads) > 0 && !disablePayloads {\n\t\tgenerator.payloadIterator = request.generator.NewIterator()\n\t}\n\treturn generator\n}\n\n// nextValue returns the next path or the next raw request depending on user input\n// It returns false if all the inputs have been exhausted by the generator instance.\nfunc (r *requestGenerator) nextValue() (value string, payloads map[string]interface{}, result bool) {\n\t// Iterate each payload sequentially for each request path/raw\n\t//\n\t// If the sequence has finished for the current payload values\n\t// then restart the sequence from the beginning and move on to the next payloads values\n\t// otherwise use the last request.\n\tvar sequence []string\n\tswitch {\n\tcase len(r.request.Path) > 0:\n\t\tsequence = r.request.Path\n\tcase len(r.request.Raw) > 0:\n\t\tsequence = r.request.Raw\n\tdefault:\n\t\treturn \"\", nil, false\n\t}\n\n\thasPayloadIterator := r.payloadIterator != nil\n\n\tif hasPayloadIterator && r.currentPayloads == nil {\n\t\tr.currentPayloads, r.okCurrentPayload = r.payloadIterator.Value()\n\t}\n\n\tvar request string\n\tvar shouldContinue bool\n\tif nextRequest, nextIndex, found := r.findNextIteration(sequence, r.currentIndex); found {\n\t\tr.currentIndex = nextIndex + 1\n\t\trequest = nextRequest\n\t\tshouldContinue = true\n\t} else {\n\t\t// if found is false which happens at end of iteration of reqData(path or raw request)\n\t\t// try again from start with index 0\n\t\tif nextRequest, nextIndex, found := r.findNextIteration(sequence, 0); found && hasPayloadIterator {\n\t\t\tr.currentIndex = nextIndex + 1\n\t\t\trequest = nextRequest\n\t\t\tshouldContinue = true\n\t\t}\n\t}\n\n\tif shouldContinue {\n\t\tif r.hasMarker(request, Once) {\n\t\t\tr.applyMark(request, Once)\n\t\t}\n\t\tif hasPayloadIterator {\n\t\t\treturn request, generators.MergeMaps(r.currentPayloads), r.okCurrentPayload\n\t\t}\n\t\t// next should return a copy of payloads and not pointer to payload to avoid data race\n\t\treturn request, generators.MergeMaps(r.currentPayloads), true\n\t} else {\n\t\treturn \"\", nil, false\n\t}\n}\n\n// findNextIteration iterates and returns next Request(path or raw request)\n// at end of each iteration payload is incremented\nfunc (r *requestGenerator) findNextIteration(sequence []string, index int) (string, int, bool) {\n\tfor i, request := range sequence[index:] {\n\t\tif r.wasMarked(request, Once) {\n\t\t\t// if request contains flowmark i.e `@once` and is marked skip it\n\t\t\tcontinue\n\t\t}\n\t\treturn request, index + i, true\n\n\t}\n\t// move on to next payload if current payload is applied/returned for all Requests(path or raw request)\n\tif r.payloadIterator != nil {\n\t\tr.currentPayloads, r.okCurrentPayload = r.payloadIterator.Value()\n\t}\n\treturn \"\", 0, false\n}\n\n// applyMark marks given request i.e blacklist request\nfunc (r *requestGenerator) applyMark(request string, mark flowMark) {\n\tswitch mark {\n\tcase Once:\n\t\tr.onceFlow[request] = struct{}{}\n\t}\n\n}\n\n// wasMarked checks if request is marked using request blacklist\nfunc (r *requestGenerator) wasMarked(request string, mark flowMark) bool {\n\tswitch mark {\n\tcase Once:\n\t\t_, ok := r.onceFlow[request]\n\t\treturn ok\n\t}\n\treturn false\n}\n\n// hasMarker returns true if request has a marker (ex: @once which means request should only be executed once)\nfunc (r *requestGenerator) hasMarker(request string, mark flowMark) bool {\n\tfo, hasOverrides := parseFlowAnnotations(request)\n\treturn hasOverrides && fo == mark\n}\n\n// Path: pkg/protocols/http/request_generator_test.go\npackage http\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\nfunc TestRequestGeneratorPaths(t *testing.T) {\n\treq := &Request{\n\t\tPath: []string{\"{{BaseURL}}/test\", \"{{BaseURL}}/test.php\"},\n\t}\n\tgenerator := req.newGenerator(false)\n\tvar payloads []string\n\tfor {\n\t\traw, _, ok := generator.nextValue()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tpayloads = append(payloads, raw)\n\t}\n\trequire.Equal(t, req.Path, payloads, \"Could not get correct paths\")\n}\n\nfunc TestRequestGeneratorClusterBombSingle(t *testing.T) {\n\tvar err error\n\n\treq := &Request{\n\t\tPayloads:   map[string]interface{}{\"username\": []string{\"admin\", \"tomcat\", \"manager\"}, \"password\": []string{\"password\", \"test\", \"secret\"}},\n\t\tAttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},\n\t\tRaw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`},\n\t}\n\tcatalogInstance := disk.NewCatalog(\"\")\n\treq.generator, err = generators.New(req.Payloads, req.AttackType.Value, \"\", catalogInstance, \"\", types.DefaultOptions())\n\trequire.Nil(t, err, \"could not create generator\")\n\n\tgenerator := req.newGenerator(false)\n\tvar payloads []map[string]interface{}\n\tfor {\n\t\t_, data, ok := generator.nextValue()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tpayloads = append(payloads, data)\n\t}\n\trequire.Equal(t, 9, len(payloads), \"Could not get correct number of payloads\")\n}\n\nfunc TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) {\n\tvar err error\n\n\treq := &Request{\n\t\tPayloads:   map[string]interface{}{\"username\": []string{\"admin\", \"tomcat\", \"manager\"}, \"password\": []string{\"password\", \"test\", \"secret\"}},\n\t\tAttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},\n\t\tRaw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},\n\t}\n\tcatalogInstance := disk.NewCatalog(\"\")\n\treq.generator, err = generators.New(req.Payloads, req.AttackType.Value, \"\", catalogInstance, \"\", types.DefaultOptions())\n\trequire.Nil(t, err, \"could not create generator\")\n\n\tgenerator := req.newGenerator(false)\n\tvar payloads []map[string]interface{}\n\tfor {\n\t\t_, data, ok := generator.nextValue()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tpayloads = append(payloads, data)\n\t}\n\trequire.Equal(t, 18, len(payloads), \"Could not get correct number of payloads\")\n}\n\n// Path: pkg/protocols/http/request_test.go\npackage http\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/testutils\"\n)\n\nfunc TestHTTPExtractMultipleReuse(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\ttestutils.Init(options)\n\ttemplateID := \"testing-http\"\n\trequest := &Request{\n\t\tID: templateID,\n\t\tRaw: []string{\n\t\t\t`GET /robots.txt HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\t\t\t`,\n\n\t\t\t`GET {{endpoint}} HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\t\t\t`,\n\t\t},\n\t\tOperators: operators.Operators{\n\t\t\tMatchers: []*matchers.Matcher{{\n\t\t\t\tPart:  \"body\",\n\t\t\t\tType:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},\n\t\t\t\tWords: []string{\"match /a\", \"match /b\", \"match /c\"},\n\t\t\t}},\n\t\t\tExtractors: []*extractors.Extractor{{\n\t\t\t\tPart:     \"body\",\n\t\t\t\tName:     \"endpoint\",\n\t\t\t\tType:     extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},\n\t\t\t\tRegex:    []string{\"(?m)/([a-zA-Z0-9-_/\\\\\\\\]+)\"},\n\t\t\t\tInternal: true,\n\t\t\t}},\n\t\t},\n\t\tIterateAll: true,\n\t}\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tswitch r.URL.Path {\n\t\tcase \"/robots.txt\":\n\t\t\t_, _ = w.Write([]byte(`User-agent: Googlebot\nDisallow: /a\nDisallow: /b\nDisallow: /c`))\n\t\tdefault:\n\t\t\t_, _ = w.Write([]byte(fmt.Sprintf(`match %v`, r.URL.Path)))\n\t\t}\n\t}))\n\tdefer ts.Close()\n\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile network request\")\n\n\tvar finalEvent *output.InternalWrappedEvent\n\tvar matchCount int\n\tt.Run(\"test\", func(t *testing.T) {\n\t\tmetadata := make(output.InternalEvent)\n\t\tprevious := make(output.InternalEvent)\n\t\tctxArgs := contextargs.NewWithInput(ts.URL)\n\t\terr := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {\n\t\t\tif event.OperatorsResult != nil && event.OperatorsResult.Matched {\n\t\t\t\tmatchCount++\n\t\t\t}\n\t\t\tfinalEvent = event\n\t\t})\n\t\trequire.Nil(t, err, \"could not execute network request\")\n\t})\n\trequire.NotNil(t, finalEvent, \"could not get event output from request\")\n\trequire.Equal(t, 3, matchCount, \"could not get correct match count\")\n}\n\nfunc TestDisableTE(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\ttestutils.Init(options)\n\ttemplateID := \"http-disable-transfer-encoding\"\n\n\t// in raw request format\n\trequest := &Request{\n\t\tID: templateID,\n\t\tRaw: []string{\n\t\t\t`POST / HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\n\t\t\tlogin=1&username=admin&password=admin\n\t\t\t`,\n\t\t},\n\t\tOperators: operators.Operators{\n\t\t\tMatchers: []*matchers.Matcher{{\n\t\t\t\tType:   matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},\n\t\t\t\tStatus: []int{200},\n\t\t\t}},\n\t\t},\n\t}\n\n\t// in base request format\n\trequest2 := &Request{\n\t\tID:     templateID,\n\t\tMethod: HTTPMethodTypeHolder{MethodType: HTTPPost},\n\t\tPath:   []string{\"{{BaseURL}}\"},\n\t\tBody:   \"login=1&username=admin&password=admin\",\n\t\tOperators: operators.Operators{\n\t\t\tMatchers: []*matchers.Matcher{{\n\t\t\t\tType:   matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},\n\t\t\t\tStatus: []int{200},\n\t\t\t}},\n\t\t},\n\t}\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif len(r.TransferEncoding) > 0 || r.ContentLength <= 0 {\n\t\t\tt.Error(\"Transfer-Encoding header should not be set\")\n\t\t}\n\t}))\n\tdefer ts.Close()\n\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile http raw request\")\n\n\terr = request2.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile http base request\")\n\n\tvar finalEvent *output.InternalWrappedEvent\n\tvar matchCount int\n\tt.Run(\"test\", func(t *testing.T) {\n\t\tmetadata := make(output.InternalEvent)\n\t\tprevious := make(output.InternalEvent)\n\t\tctxArgs := contextargs.NewWithInput(ts.URL)\n\t\terr := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {\n\t\t\tif event.OperatorsResult != nil && event.OperatorsResult.Matched {\n\t\t\t\tmatchCount++\n\t\t\t}\n\t\t\tfinalEvent = event\n\t\t})\n\t\trequire.Nil(t, err, \"could not execute network request\")\n\t})\n\n\tt.Run(\"test2\", func(t *testing.T) {\n\t\tmetadata := make(output.InternalEvent)\n\t\tprevious := make(output.InternalEvent)\n\t\tctxArgs := contextargs.NewWithInput(ts.URL)\n\t\terr := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {\n\t\t\tif event.OperatorsResult != nil && event.OperatorsResult.Matched {\n\t\t\t\tmatchCount++\n\t\t\t}\n\t\t\tfinalEvent = event\n\t\t})\n\t\trequire.Nil(t, err, \"could not execute network request\")\n\t})\n\n\trequire.NotNil(t, finalEvent, \"could not get event output from request\")\n\trequire.Equal(t, 2, matchCount, \"could not get correct match count\")\n}\n\n// consult @Ice3man543 before making any breaking changes to this test (context: vuln_hash)\nfunc TestReqURLPattern(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\t// assume this was a preprocessor\n\t// {{randstr}} => 2eNU2kbrOcUDzhnUL1RGvSo1it7\n\ttestutils.Init(options)\n\ttemplateID := \"testing-http\"\n\trequest := &Request{\n\t\tID: templateID,\n\t\tRaw: []string{\n\t\t\t`GET /{{rand_char(\"abc\")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data=2eNU2kbrOcUDzhnUL1RGvSo1it7 HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\t\t\t`,\n\t\t},\n...\n", "needle_token_start": 5357, "needle_token_end": 5425, "code_context_ntokens": 15322, "output": ["```go\nfunc isHostPort(value string) bool {\n\t_, port, err := net.SplitHostPort(value)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif !iputil.IsPort(port) {\n\t\treturn false\n\t}\n\treturn true\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "Eval", "language": "go", "path": "pkg/protocols/common/expressions/expressions.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to compile and evaluate a given string expression using specified values, while preserving the return type of the result.\n2. **Input**: It takes a string representing the expression to be evaluated and a map where keys are strings and values are of any type, representing the variables to be used in the expression.\n3. **Output**: Returns the result of the evaluated expression, which can be of any type depending on the expression, or an error if the expression cannot be compiled or evaluated correctly.\n4. **Procedure**: The function first compiles the string expression into an evaluable form using a library that supports custom functions. If the compilation is successful, it then evaluates this compiled expression using the provided variable values. Errors during compilation or evaluation are handled and returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/installer/template.go\npackage installer\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"crypto/md5\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/charmbracelet/glamour\"\n\t\"github.com/olekukonko/tablewriter\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n\tfileutil \"github.com/projectdiscovery/utils/file\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n\tupdateutils \"github.com/projectdiscovery/utils/update\"\n)\n\nconst (\n\tcheckSumFilePerm = 0644\n)\n\nvar (\n\tHideProgressBar        = true\n\tHideUpdateChangesTable = false\n...\n// Path: pkg/operators/matchers/validate.go\npackage matchers\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"strings\"\n\n\t\"github.com/antchfx/xpath\"\n\tsliceutil \"github.com/projectdiscovery/utils/slice\"\n\t\"gopkg.in/yaml.v3\"\n)\n\nvar commonExpectedFields = []string{\"Type\", \"Condition\", \"Name\", \"MatchAll\", \"Negative\", \"Internal\"}\n\n// Validate perform initial validation on the matcher structure\nfunc (matcher *Matcher) Validate() error {\n\t// uses yaml marshaling to convert the struct to map[string]interface to have same field names\n\tmatcherMap := make(map[string]interface{})\n\tmarshaledMatcher, err := yaml.Marshal(matcher)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := yaml.Unmarshal(marshaledMatcher, &matcherMap); err != nil {\n\t\treturn err\n\t}\n\n\tvar expectedFields []string\n\tswitch matcher.matcherType {\n\tcase DSLMatcher:\n\t\texpectedFields = append(commonExpectedFields, \"DSL\")\n\tcase StatusMatcher:\n\t\texpectedFields = append(commonExpectedFields, \"Status\", \"Part\")\n\tcase SizeMatcher:\n\t\texpectedFields = append(commonExpectedFields, \"Size\", \"Part\")\n\tcase WordsMatcher:\n\t\texpectedFields = append(commonExpectedFields, \"Words\", \"Part\", \"Encoding\", \"CaseInsensitive\")\n\tcase BinaryMatcher:\n\t\texpectedFields = append(commonExpectedFields, \"Binary\", \"Part\", \"Encoding\", \"CaseInsensitive\")\n\tcase RegexMatcher:\n\t\texpectedFields = append(commonExpectedFields, \"Regex\", \"Part\", \"Encoding\", \"CaseInsensitive\")\n\tcase XPathMatcher:\n\t\texpectedFields = append(commonExpectedFields, \"XPath\", \"Part\")\n\t}\n\n\tif err = checkFields(matcher, matcherMap, expectedFields...); err != nil {\n\t\treturn err\n\t}\n\n\t// validate the XPath query\n\tif matcher.matcherType == XPathMatcher {\n\t\tfor _, query := range matcher.XPath {\n\t\t\tif _, err = xpath.Compile(query); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc checkFields(m *Matcher, matcherMap map[string]interface{}, expectedFields ...string) error {\n\tvar foundUnexpectedFields []string\n\tfor marshaledFieldName := range matcherMap {\n\t\t// revert back the marshaled name to the original field\n\t\tstructFieldName, err := getFieldNameFromYamlTag(marshaledFieldName, *m)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !sliceutil.Contains(expectedFields, structFieldName) {\n\t\t\tfoundUnexpectedFields = append(foundUnexpectedFields, structFieldName)\n\t\t}\n\t}\n\tif len(foundUnexpectedFields) > 0 {\n\t\treturn fmt.Errorf(\"matcher %s has unexpected fields: %s\", m.matcherType, strings.Join(foundUnexpectedFields, \",\"))\n\t}\n\treturn nil\n}\n\nfunc getFieldNameFromYamlTag(tagName string, object interface{}) (string, error) {\n\treflectType := reflect.TypeOf(object)\n\tif reflectType.Kind() != reflect.Struct {\n\t\treturn \"\", errors.New(\"the object must be a struct\")\n\t}\n\tfor idx := 0; idx < reflectType.NumField(); idx++ {\n\t\tfield := reflectType.Field(idx)\n\t\ttagParts := strings.Split(field.Tag.Get(\"yaml\"), \",\")\n\t\tif len(tagParts) > 0 && tagParts[0] == tagName {\n\t\t\treturn field.Name, nil\n\t\t}\n\t}\n\treturn \"\", fmt.Errorf(\"field %s not found\", tagName)\n}\n\n// Path: pkg/protocols/dns/dns.go\npackage dns\n\nimport (\n\t\"strings\"\n\n\t\"github.com/miekg/dns\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool\"\n\t\"github.com/projectdiscovery/retryabledns\"\n\tfileutil \"github.com/projectdiscovery/utils/file\"\n)\n\n// Request contains a DNS protocol request to be made from a template\ntype Request struct {\n\t// Operators for the current request go here.\n\toperators.Operators `yaml:\",inline\"`\n\n\t// ID is the optional id of the request\n\tID string `yaml:\"id,omitempty\" json:\"id,omitempty\" jsonschema:\"title=id of the dns request,description=ID is the optional ID of the DNS Request\"`\n\n\t// description: |\n\t//   Name is the Hostname to make DNS request for.\n\t//\n\t//   Generally, it is set to {{FQDN}} which is the domain we get from input.\n\t// examples:\n\t//   - value: \"\\\"{{FQDN}}\\\"\"\n\tName string `yaml:\"name,omitempty\" json:\"name,omitempty\" jsonschema:\"title=hostname to make dns request for,description=Name is the Hostname to make DNS request for\"`\n\t// description: |\n\t//   RequestType is the type of DNS request to make.\n\tRequestType DNSRequestTypeHolder `yaml:\"type,omitempty\" json:\"type,omitempty\" jsonschema:\"title=type of dns request to make,description=Type is the type of DNS request to make,enum=A,enum=NS,enum=DS,enum=CNAME,enum=SOA,enum=PTR,enum=MX,enum=TXT,enum=AAAA\"`\n\t// description: |\n\t//   Class is the class of the DNS request.\n\t//\n\t//   Usually it's enough to just leave it as INET.\n\t// values:\n\t//   - \"inet\"\n\t//   - \"csnet\"\n\t//   - \"chaos\"\n\t//   - \"hesiod\"\n\t//   - \"none\"\n\t//   - \"any\"\n\tClass string `yaml:\"class,omitempty\" json:\"class,omitempty\" jsonschema:\"title=class of DNS request,description=Class is the class of the DNS request,enum=inet,enum=csnet,enum=chaos,enum=hesiod,enum=none,enum=any\"`\n\t// description: |\n\t//   Retries is the number of retries for the DNS request\n\t// examples:\n\t//   - name: Use a retry of 3 to 5 generally\n\t//     value: 5\n\tRetries int `yaml:\"retries,omitempty\" json:\"retries,omitempty\" jsonschema:\"title=retries for dns request,description=Retries is the number of retries for the DNS request\"`\n\t// description: |\n\t//   Trace performs a trace operation for the target.\n\tTrace bool `yaml:\"trace,omitempty\" json:\"trace,omitempty\" jsonschema:\"title=trace operation,description=Trace performs a trace operation for the target.\"`\n\t// description: |\n\t//   TraceMaxRecursion is the number of max recursion allowed for trace operations\n\t// examples:\n\t//   - name: Use a retry of 100 to 150 generally\n\t//     value: 100\n\tTraceMaxRecursion int `yaml:\"trace-max-recursion,omitempty\" json:\"trace-max-recursion,omitempty\"  jsonschema:\"title=trace-max-recursion level for dns request,description=TraceMaxRecursion is the number of max recursion allowed for trace operations\"`\n\n\t// description: |\n\t//   Attack is the type of payload combinations to perform.\n\t//\n\t//   Batteringram is inserts the same payload into all defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\n\t//   permutations and combinations for all payloads.\n\tAttackType generators.AttackTypeHolder `yaml:\"attack,omitempty\" json:\"attack,omitempty\" jsonschema:\"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb\"`\n\t// description: |\n\t//   Payloads contains any payloads for the current request.\n\t//\n\t//   Payloads support both key-values combinations where a list\n\t//   of payloads is provided, or optionally a single file can also\n\t//   be provided as payload which will be read on run-time.\n\tPayloads map[string]interface{} `yaml:\"payloads,omitempty\" json:\"payloads,omitempty\" jsonschema:\"title=payloads for the network request,description=Payloads contains any payloads for the current request\"`\n\t// description: |\n\t//    Threads to use when sending iterating over payloads\n\t// examples:\n\t//   - name: Send requests using 10 concurrent threads\n\t//     value: 10\n\tThreads   int `yaml:\"threads,omitempty\" json:\"threads,omitempty\" jsonschema:\"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling\"`\n\tgenerator *generators.PayloadGenerator\n\n\tCompiledOperators *operators.Operators `yaml:\"-\" json:\"-\"`\n\tdnsClient         *retryabledns.Client\n\toptions           *protocols.ExecutorOptions\n\n\t// cache any variables that may be needed for operation.\n\tclass    uint16\n\tquestion uint16\n\n\t// description: |\n\t//   Recursion determines if resolver should recurse all records to get fresh results.\n\tRecursion *bool `yaml:\"recursion,omitempty\" json:\"recursion,omitempty\" jsonschema:\"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results\"`\n\t// Resolvers to use for the dns requests\n\tResolvers []string `yaml:\"resolvers,omitempty\" json:\"resolvers,omitempty\" jsonschema:\"title=Resolvers,description=Define resolvers to use within the template\"`\n}\n\n// RequestPartDefinitions contains a mapping of request part definitions and their\n// description. Multiple definitions are separated by commas.\n// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>.\nvar RequestPartDefinitions = map[string]string{\n\t\"template-id\":   \"ID of the template executed\",\n\t\"template-info\": \"Info Block of the template executed\",\n\t\"template-path\": \"Path of the template executed\",\n\t\"host\":          \"Host is the input to the template\",\n\t\"matched\":       \"Matched is the input which was matched upon\",\n\t\"request\":       \"Request contains the DNS request in text format\",\n\t\"type\":          \"Type is the type of request made\",\n\t\"rcode\":         \"Rcode field returned for the DNS request\",\n\t\"question\":      \"Question contains the DNS question field\",\n\t\"extra\":         \"Extra contains the DNS response extra field\",\n\t\"answer\":        \"Answer contains the DNS response answer field\",\n\t\"ns\":            \"NS contains the DNS response NS field\",\n\t\"raw,body,all\":  \"Raw contains the raw DNS response (default)\",\n\t\"trace\":         \"Trace contains trace data for DNS request if enabled\",\n}\n\nfunc (request *Request) GetCompiledOperators() []*operators.Operators {\n\treturn []*operators.Operators{request.CompiledOperators}\n}\n\n// GetID returns the unique ID of the request if any.\nfunc (request *Request) GetID() string {\n\treturn request.ID\n}\n\n// Options returns executer options for http request\nfunc (r *Request) Options() *protocols.ExecutorOptions {\n\treturn r.options\n}\n\n// Compile compiles the protocol request for further execution.\nfunc (request *Request) Compile(options *protocols.ExecutorOptions) error {\n\tif request.Retries == 0 {\n\t\trequest.Retries = 3\n\t}\n\tif request.Recursion == nil {\n\t\trecursion := true\n\t\trequest.Recursion = &recursion\n\t}\n\tdnsClientOptions := &dnsclientpool.Configuration{\n\t\tRetries: request.Retries,\n\t}\n\tif len(request.Resolvers) > 0 {\n\t\tdnsClientOptions.Resolvers = request.Resolvers\n\t}\n\t// Create a dns client for the class\n\tclient, err := request.getDnsClient(options, nil)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"could not get dns client\")\n\t}\n\trequest.dnsClient = client\n\n\tif len(request.Matchers) > 0 || len(request.Extractors) > 0 {\n\t\tcompiled := &request.Operators\n\t\tcompiled.ExcludeMatchers = options.ExcludeMatchers\n\t\tcompiled.TemplateID = options.TemplateID\n\t\tif err := compiled.Compile(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile operators\")\n\t\t}\n\t\trequest.CompiledOperators = compiled\n\t}\n\trequest.class = classToInt(request.Class)\n\trequest.options = options\n\trequest.question = questionTypeToInt(request.RequestType.String())\n\tfor name, payload := range options.Options.Vars.AsMap() {\n\t\tpayloadStr, ok := payload.(string)\n\t\t// check if inputs contains the payload\n\t\tif ok && fileutil.FileExists(payloadStr) {\n\t\t\tif request.Payloads == nil {\n\t\t\t\trequest.Payloads = make(map[string]interface{})\n\t\t\t}\n\t\t\trequest.Payloads[name] = payloadStr\n\t\t}\n\t}\n\n\tif len(request.Payloads) > 0 {\n\t\trequest.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"could not parse payloads\")\n\t\t}\n\t\t// default to 20 threads for payload requests\n\t\trequest.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)\n\t}\n\treturn nil\n}\n\nfunc (request *Request) getDnsClient(options *protocols.ExecutorOptions, metadata map[string]interface{}) (*retryabledns.Client, error) {\n\tdnsClientOptions := &dnsclientpool.Configuration{\n\t\tRetries: request.Retries,\n\t}\n\tif len(request.Resolvers) > 0 {\n\t\tif len(request.Resolvers) > 0 {\n\t\t\tfor _, resolver := range request.Resolvers {\n\t\t\t\tif expressions.ContainsUnresolvedVariables(resolver) != nil {\n\t\t\t\t\tvar err error\n\t\t\t\t\tresolver, err = expressions.Evaluate(resolver, metadata)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn nil, errors.Wrap(err, \"could not resolve resolvers expressions\")\n\t\t\t\t\t}\n\t\t\t\t\tdnsClientOptions.Resolvers = append(dnsClientOptions.Resolvers, resolver)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tdnsClientOptions.Resolvers = request.Resolvers\n\t}\n\treturn dnsclientpool.Get(options.Options, dnsClientOptions)\n}\n\n// Requests returns the total number of requests the YAML rule will perform\nfunc (request *Request) Requests() int {\n\tif request.generator != nil {\n\t\tpayloadRequests := request.generator.NewIterator().Total()\n\t\treturn payloadRequests\n\t}\n\n\treturn 1\n}\n\n// Make returns the request to be sent for the protocol\nfunc (request *Request) Make(host string, vars map[string]interface{}) (*dns.Msg, error) {\n\t// Build a request on the specified URL\n\treq := new(dns.Msg)\n\treq.Id = dns.Id()\n\treq.RecursionDesired = *request.Recursion\n\n\tvar q dns.Question\n\tfinal := replacer.Replace(request.Name, vars)\n\n\tq.Name = dns.Fqdn(final)\n\tq.Qclass = request.class\n\tq.Qtype = request.question\n\treq.Question = append(req.Question, q)\n\n\treq.SetEdns0(4096, false)\n\n\tswitch request.question {\n\tcase dns.TypeTXT:\n\t\treq.AuthenticatedData = true\n\t}\n\n\treturn req, nil\n}\n\n// questionTypeToInt converts DNS question type to internal representation\nfunc questionTypeToInt(questionType string) uint16 {\n\tquestionType = strings.TrimSpace(strings.ToUpper(questionType))\n\tquestion := dns.TypeA\n\n\tswitch questionType {\n\tcase \"A\":\n\t\tquestion = dns.TypeA\n\tcase \"NS\":\n\t\tquestion = dns.TypeNS\n\tcase \"CNAME\":\n\t\tquestion = dns.TypeCNAME\n\tcase \"SOA\":\n\t\tquestion = dns.TypeSOA\n\tcase \"PTR\":\n\t\tquestion = dns.TypePTR\n\tcase \"MX\":\n\t\tquestion = dns.TypeMX\n\tcase \"TXT\":\n\t\tquestion = dns.TypeTXT\n\tcase \"DS\":\n\t\tquestion = dns.TypeDS\n\tcase \"AAAA\":\n\t\tquestion = dns.TypeAAAA\n\tcase \"CAA\":\n\t\tquestion = dns.TypeCAA\n\tcase \"TLSA\":\n\t\tquestion = dns.TypeTLSA\n\tcase \"ANY\":\n\t\tquestion = dns.TypeANY\n\tcase \"SRV\":\n\t\tquestion = dns.TypeSRV\n\t}\n\treturn question\n}\n\n// classToInt converts a dns class name to its internal representation\nfunc classToInt(class string) uint16 {\n\tclass = strings.TrimSpace(strings.ToUpper(class))\n\tresult := dns.ClassINET\n\n\tswitch class {\n\tcase \"INET\":\n\t\tresult = dns.ClassINET\n\tcase \"CSNET\":\n\t\tresult = dns.ClassCSNET\n\tcase \"CHAOS\":\n\t\tresult = dns.ClassCHAOS\n\tcase \"HESIOD\":\n\t\tresult = dns.ClassHESIOD\n\tcase \"NONE\":\n\t\tresult = dns.ClassNONE\n\tcase \"ANY\":\n\t\tresult = dns.ClassANY\n\t}\n\treturn uint16(result)\n}\n\n// Path: pkg/operators/common/dsl/dsl.go\npackage dsl\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/Knetic/govaluate\"\n\t\"github.com/miekg/dns\"\n\t\"github.com/projectdiscovery/dsl\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns/dnsclientpool\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\tsliceutil \"github.com/projectdiscovery/utils/slice\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\nvar (\n\tHelperFunctions map[string]govaluate.ExpressionFunction\n\tFunctionNames   []string\n\t// knownPorts is a list of known ports for protocols implemented in nuclei\n\tknowPorts = []string{\"80\", \"443\", \"8080\", \"8081\", \"8443\", \"53\"}\n)\n\nfunc init() {\n\t_ = dsl.AddFunction(dsl.NewWithMultipleSignatures(\"resolve\", []string{\n\t\t\"(host string) string\",\n\t\t\"(format string) string\",\n\t}, false, func(args ...interface{}) (interface{}, error) {\n\t\targCount := len(args)\n\t\tif argCount == 0 || argCount > 2 {\n\t\t\treturn nil, dsl.ErrInvalidDslFunction\n\t\t}\n\t\tformat := \"4\"\n\t\tvar dnsType uint16\n\t\tif len(args) > 1 {\n\t\t\tformat = strings.ToLower(types.ToString(args[1]))\n\t\t}\n\n\t\tswitch format {\n\t\tcase \"4\", \"a\":\n\t\t\tdnsType = dns.TypeA\n\t\tcase \"6\", \"aaaa\":\n\t\t\tdnsType = dns.TypeAAAA\n\t\tcase \"cname\":\n\t\t\tdnsType = dns.TypeCNAME\n\t\tcase \"ns\":\n\t\t\tdnsType = dns.TypeNS\n\t\tcase \"txt\":\n\t\t\tdnsType = dns.TypeTXT\n\t\tcase \"srv\":\n\t\t\tdnsType = dns.TypeSRV\n\t\tcase \"ptr\":\n\t\t\tdnsType = dns.TypePTR\n\t\tcase \"mx\":\n\t\t\tdnsType = dns.TypeMX\n\t\tcase \"soa\":\n\t\t\tdnsType = dns.TypeSOA\n\t\tcase \"caa\":\n\t\t\tdnsType = dns.TypeCAA\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"invalid dns type\")\n\t\t}\n\n\t\terr := dnsclientpool.Init(&types.Options{})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdnsClient, err := dnsclientpool.Get(nil, &dnsclientpool.Configuration{})\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// query\n\t\trawResp, err := dnsClient.Query(types.ToString(args[0]), dnsType)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdnsValues := map[uint16][]string{\n\t\t\tdns.TypeA:     rawResp.A,\n\t\t\tdns.TypeAAAA:  rawResp.AAAA,\n\t\t\tdns.TypeCNAME: rawResp.CNAME,\n\t\t\tdns.TypeNS:    rawResp.NS,\n\t\t\tdns.TypeTXT:   rawResp.TXT,\n\t\t\tdns.TypeSRV:   rawResp.SRV,\n\t\t\tdns.TypePTR:   rawResp.PTR,\n\t\t\tdns.TypeMX:    rawResp.MX,\n\t\t\tdns.TypeCAA:   rawResp.CAA,\n\t\t\tdns.TypeSOA:   rawResp.GetSOARecords(),\n\t\t}\n\n\t\tif values, ok := dnsValues[dnsType]; ok {\n\t\t\tfirstFound, found := sliceutil.FirstNonZero(values)\n\t\t\tif found {\n\t\t\t\treturn firstFound, nil\n\t\t\t}\n\t\t}\n\n\t\treturn \"\", fmt.Errorf(\"no records found\")\n\t}))\n\t_ = dsl.AddFunction(dsl.NewWithMultipleSignatures(\"getNetworkPort\", []string{\n\t\t\"(Port string,defaultPort string) string)\",\n\t\t\"(Port int,defaultPort int) int\",\n\t}, false, func(args ...interface{}) (interface{}, error) {\n\t\tif len(args) != 2 {\n\t\t\treturn nil, dsl.ErrInvalidDslFunction\n\t\t}\n\t\tport := types.ToString(args[0])\n\t\tdefaultPort := types.ToString(args[1])\n\t\tif port == \"\" || stringsutil.EqualFoldAny(port, knowPorts...) {\n\t\t\treturn defaultPort, nil\n\t\t}\n\t\treturn port, nil\n\t}))\n\n\tdsl.PrintDebugCallback = func(args ...interface{}) error {\n\t\tgologger.Info().Msgf(\"print_debug value: %s\", fmt.Sprint(args))\n\t\treturn nil\n\t}\n\n\tHelperFunctions = dsl.HelperFunctions()\n\tFunctionNames = dsl.GetFunctionNames(HelperFunctions)\n}\n\ntype CompilationError struct {\n\tDslSignature string\n\tWrappedError error\n}\n\nfunc (e *CompilationError) Error() string {\n\treturn fmt.Sprintf(\"could not compile DSL expression %q: %v\", e.DslSignature, e.WrappedError)\n}\n\nfunc (e *CompilationError) Unwrap() error {\n\treturn e.WrappedError\n}\n\nfunc GetPrintableDslFunctionSignatures(noColor bool) string {\n\treturn dsl.GetPrintableDslFunctionSignatures(noColor)\n}\n\n// Path: pkg/protocols/common/expressions/expressions.go\npackage expressions\n\nimport (\n\t\"strings\"\n\n\t\"github.com/Knetic/govaluate\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\n// Eval compiles the given expression and evaluate it with the given values preserving the return type\n\nfunc Eval(expression string, values map[string]interface{}) (interface{}, error) {\n\tcompiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn compiled.Evaluate(values)\n}\n\n// Evaluate checks if the match contains a dynamic variable, for each\n// found one we will check if it's an expression and can\n// be compiled, it will be evaluated and the results will be returned.\n//\n// The provided keys from finalValues will be used as variable names\n// for substitution inside the expression.\nfunc Evaluate(data string, base map[string]interface{}) (string, error) {\n\treturn evaluate(data, base)\n}\n\n// EvaluateByte checks if the match contains a dynamic variable, for each\n// found one we will check if it's an expression and can\n// be compiled, it will be evaluated and the results will be returned.\n//\n// The provided keys from finalValues will be used as variable names\n// for substitution inside the expression.\nfunc EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {\n\tfinalData, err := evaluate(string(data), base)\n\treturn []byte(finalData), err\n}\n\nfunc evaluate(data string, base map[string]interface{}) (string, error) {\n\t// replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value\n\tdata = replacer.Replace(data, base)\n\n\t// expressions can be:\n\t// - simple: containing base values keys (variables)\n\t// - complex: containing helper functions [ + variables]\n\t// literals like {{2+2}} are not considered expressions\n\texpressions := FindExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)\n\tfor _, expression := range expressions {\n\t\t// replace variable placeholders with base values\n\t\texpression = replacer.Replace(expression, base)\n\t\t// turns expressions (either helper functions+base values or base values)\n\t\tcompiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tresult, err := compiled.Evaluate(base)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\t// replace incrementally\n\t\tdata = replacer.ReplaceOne(data, expression, result)\n\t}\n\treturn data, nil\n}\n\n// maxIterations to avoid infinite loop\nconst maxIterations = 250\n\nfunc FindExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string {\n\tvar (\n\t\titerations int\n\t\texps       []string\n\t)\n\tfor {\n\t\t// check if we reached the maximum number of iterations\n\t\tif iterations > maxIterations {\n\t\t\tbreak\n\t\t}\n\t\titerations++\n\t\t// attempt to find open markers\n\t\tindexOpenMarker := strings.Index(data, OpenMarker)\n\t\t// exits if not found\n\t\tif indexOpenMarker < 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tindexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)\n\n\t\tshouldSearchCloseMarker := true\n\t\tcloseMarkerFound := false\n\t\tinnerData := data\n\t\tvar potentialMatch string\n\t\tvar indexCloseMarker, indexCloseMarkerOffset int\n\t\tskip := indexOpenMarkerOffset\n\t\tfor shouldSearchCloseMarker {\n\t\t\t// attempt to find close marker\n\t\t\tindexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)\n\t\t\t// if no close markers are found exit\n\t\t\tif indexCloseMarker < 0 {\n\t\t\t\tshouldSearchCloseMarker = false\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tindexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)\n\n\t\t\tpotentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]\n\t\t\tif isExpression(potentialMatch, base) {\n\t\t\t\tcloseMarkerFound = true\n\t\t\t\tshouldSearchCloseMarker = false\n\t\t\t\texps = append(exps, potentialMatch)\n\t\t\t} else {\n\t\t\t\tskip = indexCloseMarkerOffset\n\t\t\t}\n\t\t}\n\n\t\tif closeMarkerFound {\n\t\t\t// move after the close marker\n\t\t\tdata = data[indexCloseMarkerOffset:]\n\t\t} else {\n\t\t\t// move after the open marker\n\t\t\tdata = data[indexOpenMarkerOffset:]\n\t\t}\n\t}\n\treturn exps\n}\n\nfunc isExpression(data string, base map[string]interface{}) bool {\n\tif _, err := govaluate.NewEvaluableExpression(data); err == nil {\n\t\tif stringsutil.ContainsAny(data, getFunctionsNames(base)...) {\n\t\t\treturn true\n\t\t} else if stringsutil.ContainsAny(data, dsl.FunctionNames...) {\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\t_, err := govaluate.NewEvaluableExpressionWithFunctions(data, dsl.HelperFunctions)\n\treturn err == nil\n}\n\nfunc getFunctionsNames(m map[string]interface{}) []string {\n\tkeys := make([]string, 0, len(m))\n\tfor k := range m {\n\t\tkeys = append(keys, k)\n\t}\n\treturn keys\n}\n\n// Path: pkg/protocols/websocket/websocket.go\npackage websocket\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"path\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/gobwas/ws\"\n\t\"github.com/gobwas/ws/wsutil\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/fastdialer/fastdialer\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\turlutil \"github.com/projectdiscovery/utils/url\"\n)\n\n// Request is a request for the Websocket protocol\ntype Request struct {\n\t// Operators for the current request go here.\n\toperators.Operators `yaml:\",inline,omitempty\" json:\",inline,omitempty\"`\n\tCompiledOperators   *operators.Operators `yaml:\"-\" json:\"-\"`\n\n\t// ID is the optional id of the request\n\tID string `yaml:\"id,omitempty\" json:\"id,omitempty\" jsonschema:\"title=id of the request,description=ID of the network request\"`\n\t// description: |\n\t//   Address contains address for the request\n\tAddress string `yaml:\"address,omitempty\" json:\"address,omitempty\" jsonschema:\"title=address for the websocket request,description=Address contains address for the request\"`\n\t// description: |\n\t//   Inputs contains inputs for the websocket protocol\n\tInputs []*Input `yaml:\"inputs,omitempty\" json:\"inputs,omitempty\" jsonschema:\"title=inputs for the websocket request,description=Inputs contains any input/output for the current request\"`\n\t// description: |\n\t//   Headers contains headers for the request.\n\tHeaders map[string]string `yaml:\"headers,omitempty\" json:\"headers,omitempty\" jsonschema:\"title=headers contains the request headers,description=Headers contains headers for the request\"`\n\n\t// description: |\n\t//   Attack is the type of payload combinations to perform.\n\t//\n\t//   Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\n\t//   permutations and combinations for all payloads.\n\tAttackType generators.AttackTypeHolder `yaml:\"attack,omitempty\" json:\"attack,omitempty\" jsonschema:\"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb\"`\n\t// description: |\n\t//   Payloads contains any payloads for the current request.\n\t//\n\t//   Payloads support both key-values combinations where a list\n\t//   of payloads is provided, or optionally a single file can also\n\t//   be provided as payload which will be read on run-time.\n\tPayloads map[string]interface{} `yaml:\"payloads,omitempty\" json:\"payloads,omitempty\" jsonschema:\"title=payloads for the websocket request,description=Payloads contains any payloads for the current request\"`\n\n\tgenerator *generators.PayloadGenerator\n\n\t// cache any variables that may be needed for operation.\n\tdialer  *fastdialer.Dialer\n\toptions *protocols.ExecutorOptions\n}\n\n// Input is an input for the websocket protocol\ntype Input struct {\n\t// description: |\n\t//   Data is the data to send as the input.\n\t//\n\t//   It supports DSL Helper Functions as well as normal expressions.\n\t// examples:\n\t//   - value: \"\\\"TEST\\\"\"\n\t//   - value: \"\\\"hex_decode('50494e47')\\\"\"\n\tData string `yaml:\"data,omitempty\" json:\"data,omitempty\" jsonschema:\"title=data to send as input,description=Data is the data to send as the input\"`\n\t// description: |\n\t//   Name is the optional name of the data read to provide matching on.\n\t// examples:\n\t//   - value: \"\\\"prefix\\\"\"\n\tName string `yaml:\"name,omitempty\" json:\"name,omitempty\" jsonschema:\"title=optional name for data read,description=Optional name of the data read to provide matching on\"`\n}\n\nconst (\n\tparseUrlErrorMessage                   = \"could not parse input url\"\n\tevaluateTemplateExpressionErrorMessage = \"could not evaluate template expressions\"\n)\n\n// Compile compiles the request generators preparing any requests possible.\nfunc (request *Request) Compile(options *protocols.ExecutorOptions) error {\n\trequest.options = options\n\n\tclient, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"could not get network client\")\n\t}\n\trequest.dialer = client\n\n\tif len(request.Payloads) > 0 {\n\t\trequest.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, types.DefaultOptions())\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"could not parse payloads\")\n\t\t}\n\t}\n\n\tif len(request.Matchers) > 0 || len(request.Extractors) > 0 {\n\t\tcompiled := &request.Operators\n\t\tcompiled.ExcludeMatchers = options.ExcludeMatchers\n\t\tcompiled.TemplateID = options.TemplateID\n\t\tif err := compiled.Compile(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile operators\")\n\t\t}\n\t\trequest.CompiledOperators = compiled\n\t}\n\treturn nil\n}\n\n// Requests returns the total number of requests the rule will perform\nfunc (request *Request) Requests() int {\n\tif request.generator != nil {\n\t\treturn request.generator.NewIterator().Total()\n\t}\n\treturn 1\n}\n\n// GetID returns the ID for the request if any.\nfunc (request *Request) GetID() string {\n\treturn \"\"\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\thostname, err := getAddress(input.MetaInput.Input)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif request.generator != nil {\n\t\titerator := request.generator.NewIterator()\n\n\t\tfor {\n\t\t\tvalue, ok := iterator.Value()\n\t\t\tif !ok {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t} else {\n\t\tvalue := make(map[string]interface{})\n\t\tif err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) executeRequestWithPayloads(target *contextargs.Context, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\theader := http.Header{}\n\tinput := target.MetaInput.Input\n\n\tparsed, err := urlutil.Parse(input)\n\tif err != nil {\n\t\treturn errors.Wrap(err, parseUrlErrorMessage)\n\t}\n\tdefaultVars := protocolutils.GenerateVariables(parsed, false, nil)\n\toptionVars := generators.BuildPayloadFromOptions(request.options.Options)\n\t// add templatecontext variables to varMap\n\tvariables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(target.MetaInput).GetAll()))\n\tpayloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants)\n\n\trequestOptions := request.options\n\tfor key, value := range request.Headers {\n\t\tfinalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues)\n\t\tif dataErr != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)\n\t\t}\n\t\theader.Set(key, string(finalData))\n\t}\n\ttlsConfig := &tls.Config{\n\t\tInsecureSkipVerify: true,\n\t\tServerName:         hostname,\n\t\tMinVersion:         tls.VersionTLS10,\n\t}\n\tif requestOptions.Options.SNI != \"\" {\n\t\ttlsConfig.ServerName = requestOptions.Options.SNI\n\t}\n\twebsocketDialer := ws.Dialer{\n\t\tHeader:    ws.HandshakeHeaderHTTP(header),\n\t\tTimeout:   time.Duration(requestOptions.Options.Timeout) * time.Second,\n\t\tNetDial:   request.dialer.Dial,\n\t\tTLSConfig: tlsConfig,\n\t}\n\n\tif vardump.EnableVarDump {\n\t\tgologger.Debug().Msgf(\"Websocket Protocol request variables: \\n%s\\n\", vardump.DumpVariables(payloadValues))\n\t}\n\n\tfinalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)\n\tif dataErr != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)\n\t}\n\n\taddressToDial := string(finalAddress)\n\tparsedAddress, err := url.Parse(addressToDial)\n\tif err != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, parseUrlErrorMessage)\n\t}\n\tparsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path)\n\taddressToDial = parsedAddress.String()\n\n\tconn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial)\n\tif err != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, \"could not connect to server\")\n\t}\n\tdefer conn.Close()\n\n\tresponseBuilder := &strings.Builder{}\n\tif readBuffer != nil {\n\t\t_, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response\n\t}\n\n\tevents, requestOutput, err := request.readWriteInputWebsocket(conn, payloadValues, input, responseBuilder)\n\tif err != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, \"could not read write response\")\n\t}\n\trequestOptions.Progress.IncrementRequests()\n\n\tif requestOptions.Options.Debug || requestOptions.Options.DebugRequests {\n\t\tgologger.Debug().Str(\"address\", input).Msgf(\"[%s] Dumped Websocket request for %s\", requestOptions.TemplateID, input)\n\t\tgologger.Print().Msgf(\"%s\", requestOutput)\n\t}\n\n\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\tgologger.Verbose().Msgf(\"Sent Websocket request to %s\", input)\n\n\tdata := make(map[string]interface{})\n\n\tdata[\"type\"] = request.Type().String()\n\tdata[\"success\"] = \"true\"\n\tdata[\"request\"] = requestOutput\n\tdata[\"response\"] = responseBuilder.String()\n\tdata[\"host\"] = input\n\tdata[\"matched\"] = addressToDial\n\tdata[\"ip\"] = request.dialer.GetDialedIP(hostname)\n\n\t// add response fields to template context and merge templatectx variables to output event\n\trequest.options.AddTemplateVars(target.MetaInput, request.Type(), request.ID, data)\n\tdata = generators.MergeMaps(data, request.options.GetTemplateCtx(target.MetaInput).GetAll())\n\n\tfor k, v := range previous {\n\t\tdata[k] = v\n\t}\n\tfor k, v := range events {\n\t\tdata[k] = v\n\t}\n\n\tevent := eventcreator.CreateEventWithAdditionalOptions(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {\n\t\tinternalWrappedEvent.OperatorsResult.PayloadValues = payloadValues\n\t})\n\tif requestOptions.Options.Debug || requestOptions.Options.DebugResponse {\n\t\tresponseOutput := responseBuilder.String()\n\t\tgologger.Debug().Msgf(\"[%s] Dumped Websocket response for %s\", requestOptions.TemplateID, input)\n\t\tgologger.Print().Msgf(\"%s\", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, requestOptions.Options.NoColor, false))\n\t}\n\n\tcallback(event)\n\treturn nil\n}\n\nfunc (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map[string]interface{}, input string, respBuilder *strings.Builder) (events map[string]interface{}, req string, err error) {\n\treqBuilder := &strings.Builder{}\n\tinputEvents := make(map[string]interface{})\n\n\trequestOptions := request.options\n\tfor _, req := range request.Inputs {\n\t\treqBuilder.Grow(len(req.Data))\n\n\t\tfinalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues)\n\t\tif dataErr != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn nil, \"\", errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)\n\t\t}\n\t\treqBuilder.WriteString(string(finalData))\n\n\t\terr = wsutil.WriteClientMessage(conn, ws.OpText, finalData)\n\t\tif err != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn nil, \"\", errors.Wrap(err, \"could not write request to server\")\n\t\t}\n\n\t\tmsg, opCode, err := wsutil.ReadServerData(conn)\n\t\tif err != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn nil, \"\", errors.Wrap(err, \"could not write request to server\")\n\t\t}\n\t\t// Only perform matching and writes in case we receive\n\t\t// text or binary opcode from the websocket server.\n\t\tif opCode != ws.OpText && opCode != ws.OpBinary {\n\t\t\tcontinue\n\t\t}\n\n\t\trespBuilder.Write(msg)\n\t\tif req.Name != \"\" {\n\t\t\tbufferStr := string(msg)\n\t\t\tinputEvents[req.Name] = bufferStr\n\n\t\t\t// Run any internal extractors for the request here and add found values to map.\n\t\t\tif request.CompiledOperators != nil {\n\t\t\t\tvalues := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc)\n\t\t\t\tfor k, v := range values {\n\t\t\t\t\tinputEvents[k] = v\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn inputEvents, reqBuilder.String(), nil\n}\n\n// getAddress returns the address of the host to make request to\nfunc getAddress(toTest string) (string, error) {\n\tparsed, err := url.Parse(toTest)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, parseUrlErrorMessage)\n\t}\n\tscheme := strings.ToLower(parsed.Scheme)\n\n\tif scheme != \"ws\" && scheme != \"wss\" {\n\t\treturn \"\", fmt.Errorf(\"invalid url scheme provided: %s\", scheme)\n\t}\n\tif parsed != nil && parsed.Host != \"\" {\n\t\treturn parsed.Host, nil\n\t}\n\treturn \"\", nil\n}\n\n// Match performs matching operation for a matcher on model and returns:\n// true and a list of matched snippets if the matcher type is supports it\n// otherwise false and an empty string slice\nfunc (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {\n\treturn protocols.MakeDefaultMatchFunc(data, matcher)\n}\n\n// Extract performs extracting operation for an extractor on model and returns true or false.\nfunc (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {\n\treturn protocols.MakeDefaultExtractFunc(data, matcher)\n}\n\n// MakeResultEvent creates a result event from internal wrapped event\nfunc (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {\n\treturn protocols.MakeDefaultResultEvent(request, wrapped)\n}\n\n// GetCompiledOperators returns a list of the compiled operators\nfunc (request *Request) GetCompiledOperators() []*operators.Operators {\n\treturn []*operators.Operators{request.CompiledOperators}\n}\n\n// RequestPartDefinitions contains a mapping of request part definitions and their\n// description. Multiple definitions are separated by commas.\n// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>.\nvar RequestPartDefinitions = map[string]string{\n\t\"type\":     \"Type is the type of request made\",\n\t\"success\":  \"Success specifies whether websocket connection was successful\",\n\t\"request\":  \"Websocket request made to the server\",\n\t\"response\": \"Websocket response received from the server\",\n\t\"host\":     \"Host is the input to the template\",\n\t\"matched\":  \"Matched is the input which was matched upon\",\n}\n\nfunc (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {\n\tfields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent[\"host\"]))\n\tif types.ToString(wrapped.InternalEvent[\"ip\"]) != \"\" {\n\t\tfields.Ip = types.ToString(wrapped.InternalEvent[\"ip\"])\n\t}\n\tdata := &output.ResultEvent{\n\t\tTemplateID:       types.ToString(request.options.TemplateID),\n\t\tTemplatePath:     types.ToString(request.options.TemplatePath),\n\t\tInfo:             request.options.TemplateInfo,\n\t\tType:             types.ToString(wrapped.InternalEvent[\"type\"]),\n\t\tHost:             fields.Host,\n\t\tPort:             fields.Port,\n\t\tMatched:          types.ToString(wrapped.InternalEvent[\"matched\"]),\n\t\tMetadata:         wrapped.OperatorsResult.PayloadValues,\n\t\tExtractedResults: wrapped.OperatorsResult.OutputExtracts,\n\t\tTimestamp:        time.Now(),\n\t\tMatcherStatus:    true,\n\t\tIP:               fields.Ip,\n\t\tRequest:          types.ToString(wrapped.InternalEvent[\"request\"]),\n\t\tResponse:         types.ToString(wrapped.InternalEvent[\"response\"]),\n\t\tTemplateEncoded:  request.options.EncodeTemplate(),\n\t\tError:            types.ToString(wrapped.InternalEvent[\"error\"]),\n\t}\n\treturn data\n}\n\n// Type returns the type of the protocol request\nfunc (request *Request) Type() templateTypes.ProtocolType {\n\treturn templateTypes.WebsocketProtocol\n}\n\n// Path: pkg/protocols/whois/whois.go\npackage whois\n\nimport (\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\tjsoniter \"github.com/json-iterator/go\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/rdap\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\n// Request is a request for the WHOIS protocol\ntype Request struct {\n\t// Operators for the current request go here.\n\toperators.Operators `yaml:\",inline,omitempty\" json:\",inline,omitempty\"`\n\tCompiledOperators   *operators.Operators `yaml:\"-\" json:\"-\"`\n\n\t// ID is the optional id of the request\n\tID string `yaml:\"id,omitempty\" json:\"id,omitempty\" jsonschema:\"title=id of the request,description=ID of the network request\"`\n\n\t// description: |\n\t//   Query contains query for the request\n\tQuery string `yaml:\"query,omitempty\" json:\"query,omitempty\" jsonschema:\"title=query for the WHOIS request,description=Query contains query for the request\"`\n\n\t// description: |\n\t// \t Optional WHOIS server URL.\n\t//\n\t// \t If present, specifies the WHOIS server to execute the Request on.\n\t//   Otherwise, nil enables bootstrapping\n\tServer string `yaml:\"server,omitempty\" json:\"server,omitempty\" jsonschema:\"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on\"`\n\t// cache any variables that may be needed for operation.\n\tclient          *rdap.Client\n\toptions         *protocols.ExecutorOptions\n\tparsedServerURL *url.URL\n}\n\n// Compile compiles the request generators preparing any requests possible.\nfunc (request *Request) Compile(options *protocols.ExecutorOptions) error {\n\tvar err error\n\tif request.Server != \"\" {\n\t\trequest.parsedServerURL, err = url.Parse(request.Server)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to parse server URL\")\n\t\t}\n\t}\n\n\trequest.options = options\n\trequest.client, _ = rdapclientpool.Get(options.Options, nil)\n\n\tif len(request.Matchers) > 0 || len(request.Extractors) > 0 {\n\t\tcompiled := &request.Operators\n\t\tcompiled.ExcludeMatchers = options.ExcludeMatchers\n\t\tcompiled.TemplateID = options.TemplateID\n\t\tif err := compiled.Compile(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile operators\")\n\t\t}\n\t\trequest.CompiledOperators = compiled\n\t}\n\treturn nil\n}\n\n// Requests returns the total number of requests the rule will perform\nfunc (request *Request) Requests() int {\n\treturn 1\n}\n\n// GetID returns the ID for the request if any.\nfunc (request *Request) GetID() string {\n\treturn \"\"\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\t// generate variables\n\tdefaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)\n\toptionVars := generators.BuildPayloadFromOptions(request.options.Options)\n\t// add templatectx variables to varMap\n\tvars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(input.MetaInput).GetAll()))\n\n\tvariables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)\n\n\tif vardump.EnableVarDump {\n\t\tgologger.Debug().Msgf(\"Whois Protocol request variables: \\n%s\\n\", vardump.DumpVariables(variables))\n\t}\n\n\t// and replace placeholders\n\tquery := replacer.Replace(request.Query, variables)\n\t// build an rdap request\n\trdapReq := rdap.NewAutoRequest(query)\n\trdapReq.Server = request.parsedServerURL\n\tres, err := request.client.Do(rdapReq)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"could not make whois request\")\n\t}\n...\n", "needle_token_start": 6817, "needle_token_end": 6898, "code_context_ntokens": 15279, "output": ["```go\n// Eval compiles the given expression and evaluate it with the given values preserving the return type\nfunc Eval(expression string, values map[string]interface{}) (interface{}, error) {\n\tcompiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn compiled.Evaluate(values)\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "dump", "language": "go", "path": "pkg/protocols/http/utils.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a byte representation of an HTTP request, which can be used for logging, debugging, or transmission over networks.\n2. **Input**: The function takes two parameters: a structured representation of an HTTP request and a string representing the URL.\n3. **Output**: It returns a byte slice containing the HTTP request data and an error if the operation fails.\n4. **Procedure**: The function first checks if the structured HTTP request is non-nil and attempts to convert it into a byte slice using a method specific to the HTTP request type. If the structured request is nil, it constructs the request using provided method, URL, path, headers, and data, and then converts this into a byte slice. Errors during the conversion process are captured and returned along with a contextual error message.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/catalog/disk/catalog.go\npackage disk\n\nimport (\n\t\"io\"\n\t\"io/fs\"\n\t\"os\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n)\n\n// DiskCatalog is a template catalog helper implementation based on disk\ntype DiskCatalog struct {\n\ttemplatesDirectory string\n\ttemplatesFS        fs.FS // TODO: Refactor to use this\n}\n\n// NewCatalog creates a new Catalog structure using provided input items\n// using disk based items\nfunc NewCatalog(directory string) *DiskCatalog {\n\tcatalog := &DiskCatalog{templatesDirectory: directory}\n\tif directory != \"\" {\n\t\tcatalog.templatesFS = os.DirFS(directory)\n...\n// Path: pkg/loader/parser/parser.go\npackage parser\n\nimport (\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog\"\n)\n\ntype Parser interface {\n\tLoadTemplate(templatePath string, tagFilter any, extraTags []string, catalog catalog.Catalog) (bool, error)\n\tParseTemplate(templatePath string, catalog catalog.Catalog) (any, error)\n\tLoadWorkflow(templatePath string, catalog catalog.Catalog) (bool, error)\n}\n\n// Path: pkg/js/libs/goconsole/log.go\npackage goconsole\n\nimport (\n\t\"github.com/dop251/goja_nodejs/console\"\n\t\"github.com/projectdiscovery/gologger\"\n)\n\nvar _ console.Printer = &GoConsolePrinter{}\n\n// GoConsolePrinter is a console printer for nuclei using gologger\ntype GoConsolePrinter struct {\n\tlogger *gologger.Logger\n}\n\nfunc NewGoConsolePrinter() *GoConsolePrinter {\n\treturn &GoConsolePrinter{\n\t\tlogger: gologger.DefaultLogger,\n\t}\n}\n\nfunc (p *GoConsolePrinter) Log(msg string) {\n\tp.logger.Info().Msg(msg)\n}\n\nfunc (p *GoConsolePrinter) Warn(msg string) {\n\tp.logger.Warning().Msg(msg)\n}\n\nfunc (p *GoConsolePrinter) Error(msg string) {\n\tp.logger.Error().Msg(msg)\n}\n\n// Path: pkg/templates/log.go\npackage templates\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/logrusorgru/aurora\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity\"\n\tmapsutil \"github.com/projectdiscovery/utils/maps\"\n)\n\nvar (\n\tColorizer                       aurora.Aurora\n\tSeverityColorizer               func(severity.Severity) string\n\tdeprecatedProtocolNameTemplates = mapsutil.SyncLockMap[string, bool]{Map: mapsutil.Map[string, bool]{}} //templates that still use deprecated protocol names\n)\n\n// TemplateLogMessage returns a beautified log string for a template\nfunc TemplateLogMessage(id, name string, authors []string, templateSeverity severity.Severity) string {\n\tif Colorizer == nil || SeverityColorizer == nil {\n\t\treturn \"\"\n\t}\n\t// Display the message for the template\n\treturn fmt.Sprintf(\"[%s] %s (%s) [%s]\",\n\t\tColorizer.BrightBlue(id).String(),\n\t\tColorizer.Bold(name).String(),\n\t\tColorizer.BrightYellow(appendAtSignToAuthors(authors)).String(),\n\t\tSeverityColorizer(templateSeverity))\n}\n\n// appendAtSignToAuthors appends @ before each author and returns the final string\nfunc appendAtSignToAuthors(authors []string) string {\n\tif len(authors) == 0 {\n\t\treturn \"@none\"\n\t}\n\n\tvalues := make([]string, 0, len(authors))\n\tfor _, k := range authors {\n\t\tif !strings.HasPrefix(k, \"@\") {\n\t\t\tvalues = append(values, fmt.Sprintf(\"@%s\", k))\n\t\t} else {\n\t\t\tvalues = append(values, k)\n\t\t}\n\t}\n\treturn strings.Join(values, \",\")\n}\n\n// PrintDeprecatedProtocolNameMsgIfApplicable prints a message if deprecated protocol names are used\n// Unless mode is silent we print a message for deprecated protocol name\nfunc PrintDeprecatedProtocolNameMsgIfApplicable(isSilent bool, verbose bool) {\n\tcount := 0\n\t_ = deprecatedProtocolNameTemplates.Iterate(func(k string, v bool) error {\n\t\tcount++\n\t\treturn nil\n\t})\n\tif count > 0 && !isSilent {\n\t\tgologger.Print().Msgf(\"[%v] Found %v templates loaded with deprecated protocol syntax, update before v3 for continued support.\\n\", aurora.Yellow(\"WRN\").String(), count)\n\t}\n\tif config.DefaultConfig.LogAllEvents {\n\t\t_ = deprecatedProtocolNameTemplates.Iterate(func(k string, v bool) error {\n\t\t\tgologger.Print().Msgf(\"  - %s\\n\", k)\n\t\t\treturn nil\n\t\t})\n\t}\n\tdeprecatedProtocolNameTemplates.Lock()\n\tdeprecatedProtocolNameTemplates.Map = make(map[string]bool)\n\tdeprecatedProtocolNameTemplates.Unlock()\n}\n\n// Path: pkg/js/devtools/bindgen/generator.go\npackage generator\n\nimport (\n\t\"fmt\"\n\t\"go/ast\"\n\t\"go/importer\"\n\t\"go/parser\"\n\t\"go/token\"\n\t\"go/types\"\n\t\"log\"\n\t\"os\"\n\t\"strings\"\n\n\t_ \"embed\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler\"\n)\n\nvar (\n\t//go:embed templates/js_class.tmpl\n\tjsClassFile string\n\t//go:embed templates/go_class.tmpl\n\tgoClassFile string\n\t//go:embed templates/markdown_class.tmpl\n\tmarkdownClassFile string\n)\n\n// TemplateData contains the parameters for the JS code generator\ntype TemplateData struct {\n\tPackageName               string\n\tPackagePath               string\n\tHasObjects                bool\n\tPackageFuncs              map[string]string\n\tPackageInterfaces         map[string]string\n\tPackageFuncsExtraNoType   map[string]PackageFunctionExtra\n\tPackageFuncsExtra         map[string]PackageFuncExtra\n\tPackageVars               map[string]string\n\tPackageVarsValues         map[string]string\n\tPackageTypes              map[string]string\n\tPackageTypesExtra         map[string]PackageTypeExtra\n\tPackageDefinedConstructor map[string]struct{}\n\n\ttypesPackage *types.Package\n\n\t// NativeScripts contains the list of native scripts\n\t// that should be included in the package.\n\tNativeScripts []string\n}\n\n// PackageTypeExtra contains extra information about a type\ntype PackageTypeExtra struct {\n\tFields map[string]string\n}\n\n// PackageFuncExtra contains extra information about a function\ntype PackageFuncExtra struct {\n\tItems map[string]PackageFunctionExtra\n\tDoc   string\n}\n\n// PackageFunctionExtra contains extra information about a function\ntype PackageFunctionExtra struct {\n\tArgs    []string\n\tName    string\n\tReturns []string\n\tDoc     string\n}\n\n// newTemplateData creates a new template data structure\nfunc newTemplateData(packagePrefix, pkgName string) *TemplateData {\n\treturn &TemplateData{\n\t\tPackageName:               pkgName,\n\t\tPackagePath:               packagePrefix + pkgName,\n\t\tPackageFuncs:              make(map[string]string),\n\t\tPackageFuncsExtraNoType:   make(map[string]PackageFunctionExtra),\n\t\tPackageFuncsExtra:         make(map[string]PackageFuncExtra),\n\t\tPackageVars:               make(map[string]string),\n\t\tPackageVarsValues:         make(map[string]string),\n\t\tPackageTypes:              make(map[string]string),\n\t\tPackageInterfaces:         make(map[string]string),\n\t\tPackageTypesExtra:         make(map[string]PackageTypeExtra),\n\t\tPackageDefinedConstructor: make(map[string]struct{}),\n\t}\n}\n\n// GetLibraryModules takes a directory and returns subdirectories as modules\nfunc GetLibraryModules(directory string) ([]string, error) {\n\tdirs, err := os.ReadDir(directory)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"could not read directory\")\n\t}\n\tvar modules []string\n\tfor _, dir := range dirs {\n\t\tif dir.IsDir() {\n\t\t\tmodules = append(modules, dir.Name())\n\t\t}\n\t}\n\treturn modules, nil\n}\n\n// CreateTemplateData creates a TemplateData structure from a directory\n// of go source code.\nfunc CreateTemplateData(directory string, packagePrefix string) (*TemplateData, error) {\n\tfmt.Println(directory)\n\tfset := token.NewFileSet()\n\n\tpkgs, err := parser.ParseDir(fset, directory, nil, parser.ParseComments)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"could not parse directory\")\n\t}\n\tif len(pkgs) != 1 {\n\t\treturn nil, fmt.Errorf(\"expected 1 package, got %d\", len(pkgs))\n\t}\n\n\tconfig := &types.Config{\n\t\tImporter: importer.ForCompiler(fset, \"source\", nil),\n\t}\n\tvar packageName string\n\tvar files []*ast.File\n\tfor k, v := range pkgs {\n\t\tpackageName = k\n\t\tfor _, f := range v.Files {\n\t\t\tfiles = append(files, f)\n\t\t}\n\t\tbreak\n\t}\n\n\tpkg, err := config.Check(packageName, fset, files, nil)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"could not check package\")\n\t}\n\n\tvar pkgMain *ast.Package\n\tfor _, p := range pkgs {\n\t\tpkgMain = p\n\t\tbreak\n\t}\n\n\tlog.Printf(\"[create] [discover] Package: %s\\n\", pkgMain.Name)\n\tdata := newTemplateData(packagePrefix, pkgMain.Name)\n\tdata.typesPackage = pkg\n\tdata.gatherPackageData(pkgMain, data)\n\n\tfor item, v := range data.PackageFuncsExtra {\n\t\tif len(v.Items) == 0 {\n\t\t\tdelete(data.PackageFuncsExtra, item)\n\t\t}\n\t}\n\n\t// map types with corresponding constructors\n\tfor constructor := range data.PackageDefinedConstructor {\n\tobject:\n\t\tfor k := range data.PackageTypes {\n\t\t\tif strings.Contains(constructor, k) {\n\t\t\t\tdata.PackageTypes[k] = constructor\n\t\t\t\tbreak object\n\t\t\t}\n\t\t}\n\t}\n\tfor k, v := range data.PackageTypes {\n\t\tif k == v || v == \"\" {\n\t\t\tdata.HasObjects = true\n\t\t\tdata.PackageTypes[k] = \"\"\n\t\t}\n\t}\n\n\treturn data, nil\n}\n\n// InitNativeScripts initializes the native scripts array\n// with all the exported functions from the runtime\nfunc (d *TemplateData) InitNativeScripts() {\n\truntime := compiler.InternalGetGeneratorRuntime()\n\n\texports := runtime.Get(\"exports\")\n\tif exports == nil {\n\t\treturn\n\t}\n\texportsObj := exports.Export()\n\tif exportsObj == nil {\n\t\treturn\n\t}\n\tfor v := range exportsObj.(map[string]interface{}) {\n\t\td.NativeScripts = append(d.NativeScripts, v)\n\t}\n}\n\n// gatherPackageData gathers data about the package\nfunc (d *TemplateData) gatherPackageData(pkg *ast.Package, data *TemplateData) {\n\tast.Inspect(pkg, func(node ast.Node) bool {\n\t\tswitch node := node.(type) {\n\t\tcase *ast.FuncDecl:\n\t\t\textra := d.collectFuncDecl(node)\n\t\t\tif extra.Name == \"\" {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tdata.PackageFuncsExtraNoType[node.Name.Name] = extra\n\t\t\tdata.PackageFuncs[node.Name.Name] = node.Name.Name\n\t\tcase *ast.TypeSpec:\n\t\t\tif !node.Name.IsExported() {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif node.Type == nil {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tstructDecl, ok := node.Type.(*ast.StructType)\n\t\t\tif !ok {\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\tpackageTypes := PackageTypeExtra{\n\t\t\t\tFields: make(map[string]string),\n\t\t\t}\n\t\t\tfor _, field := range structDecl.Fields.List {\n\t\t\t\tfieldName := field.Names[0].Name\n\n\t\t\t\tvar fieldTypeValue string\n\t\t\t\tswitch fieldType := field.Type.(type) {\n\t\t\t\tcase *ast.Ident: // Field type is a simple identifier\n\t\t\t\t\tfieldTypeValue = fieldType.Name\n\t\t\t\tcase *ast.ArrayType:\n\t\t\t\t\tswitch fieldType.Elt.(type) {\n\t\t\t\t\tcase *ast.Ident:\n\t\t\t\t\t\tfieldTypeValue = fmt.Sprintf(\"[]%s\", fieldType.Elt.(*ast.Ident).Name)\n\t\t\t\t\tcase *ast.StarExpr:\n\t\t\t\t\t\tfieldTypeValue = fmt.Sprintf(\"[]%s\", d.handleStarExpr(fieldType.Elt.(*ast.StarExpr)))\n\t\t\t\t\t}\n\t\t\t\tcase *ast.SelectorExpr: // Field type is a qualified identifier\n\t\t\t\t\tfieldTypeValue = fmt.Sprintf(\"%s.%s\", fieldType.X, fieldType.Sel)\n\t\t\t\t}\n\t\t\t\tpackageTypes.Fields[fieldName] = fieldTypeValue\n\t\t\t}\n\t\t\tif len(packageTypes.Fields) == 0 {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tdata.PackageTypesExtra[node.Name.Name] = packageTypes\n\t\tcase *ast.GenDecl:\n\t\t\tidentifyGenDecl(pkg, node, data)\n\t\t}\n\t\treturn true\n\t})\n}\n\nfunc identifyGenDecl(pkg *ast.Package, decl *ast.GenDecl, data *TemplateData) {\n\tfor _, spec := range decl.Specs {\n\t\tswitch spec := spec.(type) {\n\t\tcase *ast.ValueSpec:\n\t\t\tif !spec.Names[0].IsExported() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif spec.Values == nil || len(spec.Values) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tdata.PackageVars[spec.Names[0].Name] = spec.Names[0].Name\n\t\t\tdata.PackageVarsValues[spec.Names[0].Name] = spec.Values[0].(*ast.BasicLit).Value\n\t\tcase *ast.TypeSpec:\n\t\t\tif !spec.Name.IsExported() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif spec.Type == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tswitch spec.Type.(type) {\n\t\t\tcase *ast.InterfaceType:\n\t\t\t\tdata.PackageInterfaces[spec.Name.Name] = convertCommentsToJavascript(decl.Doc.Text())\n\n\t\t\tcase *ast.StructType:\n\t\t\t\tdata.PackageFuncsExtra[spec.Name.Name] = PackageFuncExtra{\n\t\t\t\t\tItems: make(map[string]PackageFunctionExtra),\n\t\t\t\t\tDoc:   convertCommentsToJavascript(decl.Doc.Text()),\n\t\t\t\t}\n\n\t\t\t\t// Traverse the AST.\n\t\t\t\tcollectStructFuncsFromAST(pkg, spec, data)\n\t\t\t\tdata.PackageTypes[spec.Name.Name] = spec.Name.Name\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc collectStructFuncsFromAST(pkg *ast.Package, spec *ast.TypeSpec, data *TemplateData) {\n\tast.Inspect(pkg, func(n ast.Node) bool {\n\t\tif fn, isFunc := n.(*ast.FuncDecl); isFunc && fn.Name.IsExported() {\n\t\t\tprocessFunc(fn, spec, data)\n\t\t}\n\t\treturn true\n\t})\n}\n\nfunc processFunc(fn *ast.FuncDecl, spec *ast.TypeSpec, data *TemplateData) {\n\tif fn.Recv == nil || len(fn.Recv.List) == 0 {\n\t\treturn\n\t}\n\n\tif t, ok := fn.Recv.List[0].Type.(*ast.StarExpr); ok {\n\t\tif ident, ok := t.X.(*ast.Ident); ok && spec.Name.Name == ident.Name {\n\t\t\tprocessFunctionDetails(fn, ident, data)\n\t\t}\n\t}\n}\n\nfunc processFunctionDetails(fn *ast.FuncDecl, ident *ast.Ident, data *TemplateData) {\n\textra := PackageFunctionExtra{\n\t\tName:    fn.Name.Name,\n\t\tArgs:    extractArgs(fn),\n\t\tDoc:     convertCommentsToJavascript(fn.Doc.Text()),\n\t\tReturns: data.extractReturns(fn),\n\t}\n\tdata.PackageFuncsExtra[ident.Name].Items[fn.Name.Name] = extra\n}\n\nfunc extractArgs(fn *ast.FuncDecl) []string {\n\targs := make([]string, 0)\n\tfor _, arg := range fn.Type.Params.List {\n\t\tfor _, name := range arg.Names {\n\t\t\targs = append(args, name.Name)\n\t\t}\n\t}\n\treturn args\n}\n\nfunc (d *TemplateData) extractReturns(fn *ast.FuncDecl) []string {\n\treturns := make([]string, 0)\n\tif fn.Type.Results == nil {\n\t\treturn returns\n\t}\n\tfor _, ret := range fn.Type.Results.List {\n\t\treturnType := d.extractReturnType(ret)\n\t\tif returnType != \"\" {\n\t\t\treturns = append(returns, returnType)\n\t\t}\n\t}\n\treturn returns\n}\n\nfunc (d *TemplateData) extractReturnType(ret *ast.Field) string {\n\tswitch v := ret.Type.(type) {\n\tcase *ast.ArrayType:\n\t\tif v, ok := v.Elt.(*ast.Ident); ok {\n\t\t\treturn fmt.Sprintf(\"[]%s\", v.Name)\n\t\t}\n\t\tif v, ok := v.Elt.(*ast.StarExpr); ok {\n\t\t\treturn fmt.Sprintf(\"[]%s\", d.handleStarExpr(v))\n\t\t}\n\tcase *ast.Ident:\n\t\treturn v.Name\n\tcase *ast.StarExpr:\n\t\treturn d.handleStarExpr(v)\n\t}\n\treturn \"\"\n}\n\nfunc (d *TemplateData) handleStarExpr(v *ast.StarExpr) string {\n\tswitch vk := v.X.(type) {\n\tcase *ast.Ident:\n\t\treturn vk.Name\n\tcase *ast.SelectorExpr:\n\t\tif vk.X != nil {\n\t\t\td.collectTypeFromExternal(d.typesPackage, vk.X.(*ast.Ident).Name, vk.Sel.Name)\n\t\t}\n\t\treturn vk.Sel.Name\n\t}\n\treturn \"\"\n}\n\nfunc (d *TemplateData) collectTypeFromExternal(pkg *types.Package, pkgName, name string) {\n\tif pkgName == \"goja\" {\n\t\t// no need to attempt to collect types from goja ( this is metadata )\n\t\treturn\n\t}\n\textra := PackageTypeExtra{\n\t\tFields: make(map[string]string),\n\t}\n\n\tfor _, importValue := range pkg.Imports() {\n\t\tif importValue.Name() != pkgName {\n\t\t\tcontinue\n\t\t}\n\t\tobj := importValue.Scope().Lookup(name)\n\t\tif obj == nil || !obj.Exported() {\n\t\t\tcontinue\n\t\t}\n\t\ttypeName, ok := obj.(*types.TypeName)\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\tunderlying, ok := typeName.Type().Underlying().(*types.Struct)\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\tfor i := 0; i < underlying.NumFields(); i++ {\n\t\t\tfield := underlying.Field(i)\n\t\t\tfieldType := field.Type().String()\n\n\t\t\tif val, ok := field.Type().Underlying().(*types.Pointer); ok {\n\t\t\t\tfieldType = field.Name()\n\t\t\t\td.collectTypeFromExternal(pkg, pkgName, val.Elem().(*types.Named).Obj().Name())\n\t\t\t}\n\t\t\tif _, ok := field.Type().Underlying().(*types.Struct); ok {\n\t\t\t\tfieldType = field.Name()\n\t\t\t\td.collectTypeFromExternal(pkg, pkgName, field.Name())\n\t\t\t}\n\t\t\textra.Fields[field.Name()] = fieldType\n\t\t}\n\t\tif len(extra.Fields) > 0 {\n\t\t\td.PackageTypesExtra[name] = extra\n\t\t}\n\t}\n}\n\nfunc (d *TemplateData) collectFuncDecl(decl *ast.FuncDecl) (extra PackageFunctionExtra) {\n\tif decl.Recv != nil {\n\t\treturn\n\t}\n\tif !decl.Name.IsExported() {\n\t\treturn\n\t}\n\textra.Name = decl.Name.Name\n\textra.Doc = convertCommentsToJavascript(decl.Doc.Text())\n\n\tisConstructor := false\n\n\tfor _, arg := range decl.Type.Params.List {\n\t\tp := exprToString(arg.Type)\n\t\tif strings.Contains(p, \"goja.ConstructorCall\") {\n\t\t\tisConstructor = true\n\t\t}\n\t\tfor _, name := range arg.Names {\n\t\t\textra.Args = append(extra.Args, name.Name)\n\t\t}\n\t}\n\tif isConstructor {\n\t\td.PackageDefinedConstructor[decl.Name.Name] = struct{}{}\n\t}\n\n\textra.Returns = d.extractReturns(decl)\n\treturn extra\n}\n\n// convertCommentsToJavascript converts comments to javascript comments.\nfunc convertCommentsToJavascript(comments string) string {\n\tsuffix := strings.Trim(strings.TrimSuffix(strings.ReplaceAll(comments, \"\\n\", \"\\n// \"), \"// \"), \"\\n\")\n\treturn fmt.Sprintf(\"// %s\", suffix)\n}\n\n// exprToString converts an expression to a string\nfunc exprToString(expr ast.Expr) string {\n\tswitch t := expr.(type) {\n\tcase *ast.Ident:\n\t\treturn t.Name\n\tcase *ast.SelectorExpr:\n\t\treturn exprToString(t.X) + \".\" + t.Sel.Name\n\tcase *ast.StarExpr:\n\t\treturn exprToString(t.X)\n\tcase *ast.ArrayType:\n\t\treturn \"[]\" + exprToString(t.Elt)\n\tcase *ast.InterfaceType:\n\t\treturn \"interface{}\"\n\t// Add more cases to handle other types\n\tdefault:\n\t\treturn fmt.Sprintf(\"%T\", expr)\n\t}\n}\n\n// Path: pkg/protocols/common/generators/generators.go\n// Inspired from https://github.com/ffuf/ffuf/blob/master/pkg/input/input.go\n\npackage generators\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\n// PayloadGenerator is the generator struct for generating payloads\ntype PayloadGenerator struct {\n\tType     AttackType\n\tcatalog  catalog.Catalog\n\tpayloads map[string][]string\n\toptions  *types.Options\n}\n\n// New creates a new generator structure for payload generation\nfunc New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog, customAttackType string, opts *types.Options) (*PayloadGenerator, error) {\n\tif attackType.String() == \"\" {\n\t\tattackType = BatteringRamAttack\n\t}\n\n\t// Resolve payload paths if they are files.\n\tpayloadsFinal := make(map[string]interface{})\n\tfor name, payload := range payloads {\n\t\tpayloadsFinal[name] = payload\n\t}\n\n\tgenerator := &PayloadGenerator{catalog: catalog, options: opts}\n\tif err := generator.validate(payloadsFinal, templatePath); err != nil {\n\t\treturn nil, err\n\t}\n\n\tcompiled, err := generator.loadPayloads(payloadsFinal, templatePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tgenerator.Type = attackType\n\tgenerator.payloads = compiled\n\n\tif customAttackType != \"\" {\n\t\tattackTypeNew, err := toAttackType(customAttackType)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"could not parse custom attack-type\")\n\t\t}\n\t\tgenerator.Type = attackTypeNew\n\t}\n\t// Validate the batteringram payload set\n\tif attackType == BatteringRamAttack {\n\t\tif len(payloads) != 1 {\n\t\t\treturn nil, errors.New(\"batteringram must have single payload set\")\n\t\t}\n\t}\n\treturn generator, nil\n}\n\n// Iterator is a single instance of an iterator for a generator structure\ntype Iterator struct {\n\tType        AttackType\n\tposition    int\n\tmsbIterator int\n\ttotal       int\n\tpayloads    []*payloadIterator\n}\n\n// NewIterator creates a new iterator for the payloads generator\nfunc (g *PayloadGenerator) NewIterator() *Iterator {\n\tvar payloads []*payloadIterator\n\n\tfor name, values := range g.payloads {\n\t\tpayloads = append(payloads, &payloadIterator{name: name, values: values})\n\t}\n\titerator := &Iterator{\n\t\tType:     g.Type,\n\t\tpayloads: payloads,\n\t}\n\titerator.total = iterator.Total()\n\treturn iterator\n}\n\n// Reset resets the iterator back to its initial value\nfunc (i *Iterator) Reset() {\n\ti.position = 0\n\ti.msbIterator = 0\n\n\tfor _, payload := range i.payloads {\n\t\tpayload.resetPosition()\n\t}\n}\n\n// Remaining returns the amount of requests left for the generator.\nfunc (i *Iterator) Remaining() int {\n\treturn i.total - i.position\n}\n\n// Total returns the amount of input combinations available\nfunc (i *Iterator) Total() int {\n\tcount := 0\n\tswitch i.Type {\n\tcase BatteringRamAttack:\n\t\tfor _, p := range i.payloads {\n\t\t\tcount += len(p.values)\n\t\t}\n\tcase PitchForkAttack:\n\t\tcount = len(i.payloads[0].values)\n\t\tfor _, p := range i.payloads {\n\t\t\tif count > len(p.values) {\n\t\t\t\tcount = len(p.values)\n\t\t\t}\n\t\t}\n\tcase ClusterBombAttack:\n\t\tcount = 1\n\t\tfor _, p := range i.payloads {\n\t\t\tcount *= len(p.values)\n\t\t}\n\t}\n\treturn count\n}\n\n// Value returns the next value for an iterator\nfunc (i *Iterator) Value() (map[string]interface{}, bool) {\n\tswitch i.Type {\n\tcase BatteringRamAttack:\n\t\treturn i.batteringRamValue()\n\tcase PitchForkAttack:\n\t\treturn i.pitchforkValue()\n\tcase ClusterBombAttack:\n\t\treturn i.clusterbombValue()\n\tdefault:\n\t\treturn i.batteringRamValue()\n\t}\n}\n\n// batteringRamValue returns a list of all payloads for the iterator\nfunc (i *Iterator) batteringRamValue() (map[string]interface{}, bool) {\n\tvalues := make(map[string]interface{}, 1)\n\n\tcurrentIndex := i.msbIterator\n\tpayload := i.payloads[currentIndex]\n\tif !payload.next() {\n\t\ti.msbIterator++\n\t\tif i.msbIterator == len(i.payloads) {\n\t\t\treturn nil, false\n\t\t}\n\t\treturn i.batteringRamValue()\n\t}\n\tvalues[payload.name] = payload.value()\n\tpayload.incrementPosition()\n\ti.position++\n\treturn values, true\n}\n\n// pitchforkValue returns a map of keyword:value pairs in same index\nfunc (i *Iterator) pitchforkValue() (map[string]interface{}, bool) {\n\tvalues := make(map[string]interface{}, len(i.payloads))\n\n\tfor _, p := range i.payloads {\n\t\tif !p.next() {\n\t\t\treturn nil, false\n\t\t}\n\t\tvalues[p.name] = p.value()\n\t\tp.incrementPosition()\n\t}\n\ti.position++\n\treturn values, true\n}\n\n// clusterbombValue returns a combination of all input pairs in key:value format.\nfunc (i *Iterator) clusterbombValue() (map[string]interface{}, bool) {\n\tif i.position >= i.total {\n\t\treturn nil, false\n\t}\n\tvalues := make(map[string]interface{}, len(i.payloads))\n\n\t// Should we signal the next InputProvider in the slice to increment\n\tsignalNext := false\n\tfirst := true\n\tfor index, p := range i.payloads {\n\t\tif signalNext {\n\t\t\tp.incrementPosition()\n\t\t\tsignalNext = false\n\t\t}\n\t\tif !p.next() {\n\t\t\t// No more inputs in this input provider\n\t\t\tif index == i.msbIterator {\n\t\t\t\t// Reset all previous wordlists and increment the msb counter\n\t\t\t\ti.msbIterator++\n\t\t\t\ti.clusterbombIteratorReset()\n\t\t\t\t// Start again\n\t\t\t\treturn i.clusterbombValue()\n\t\t\t}\n\t\t\tp.resetPosition()\n\t\t\tsignalNext = true\n\t\t}\n\t\tvalues[p.name] = p.value()\n\t\tif first {\n\t\t\tp.incrementPosition()\n\t\t\tfirst = false\n\t\t}\n\t}\n\ti.position++\n\treturn values, true\n}\n\nfunc (i *Iterator) clusterbombIteratorReset() {\n\tfor index, p := range i.payloads {\n\t\tif index < i.msbIterator {\n\t\t\tp.resetPosition()\n\t\t}\n\t\tif index == i.msbIterator {\n\t\t\tp.incrementPosition()\n\t\t}\n\t}\n}\n\n// payloadIterator is a single instance of an iterator for a single payload list.\ntype payloadIterator struct {\n\tindex  int\n\tname   string\n\tvalues []string\n}\n\n// next returns true if there are more values in payload iterator\nfunc (i *payloadIterator) next() bool {\n\treturn i.index < len(i.values)\n}\n\n// resetPosition resets the position of the payload iterator\nfunc (i *payloadIterator) resetPosition() {\n\ti.index = 0\n}\n\n// incrementPosition increments the position of the payload iterator\nfunc (i *payloadIterator) incrementPosition() {\n\ti.index++\n}\n\n// value returns the value of the payload at an index\nfunc (i *payloadIterator) value() string {\n\treturn i.values[i.index]\n}\n\n// Path: pkg/protocols/http/utils.go\npackage http\n\nimport (\n\t\"io\"\n\t\"strings\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/rawhttp\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n)\n\n// dump creates a dump of the http request in form of a byte slice\n\nfunc dump(req *generatedRequest, reqURL string) ([]byte, error) {\n\tif req.request != nil {\n\t\tbin, err := req.request.Dump()\n\t\tif err != nil {\n\t\t\treturn nil, errorutil.NewWithErr(err).WithTag(\"http\").Msgf(\"could not dump request: %v\", req.request.URL.String())\n\t\t}\n\t\treturn bin, nil\n\t}\n\trawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes}\n\tbin, err := rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)\n\tif err != nil {\n\t\treturn nil, errorutil.NewWithErr(err).WithTag(\"http\").Msgf(\"could not dump request: %v\", reqURL)\n\t}\n\treturn bin, nil\n}\n\n// Path: pkg/protocols/headless/engine/action.go\npackage engine\n\nimport (\n\t\"strings\"\n\n\t\"github.com/invopop/jsonschema\"\n)\n\n// Action is an action taken by the browser to reach a navigation\n//\n// Each step that the browser executes is an action. Most navigations\n// usually start from the ActionLoadURL event, and further navigations\n// are discovered on the found page. We also keep track and only\n// scrape new navigation from pages we haven't crawled yet.\ntype Action struct {\n\t// description:\n\t//   Args contain arguments for the headless action.\n\t//\n\t//   Per action arguments are described in detail [here](https://nuclei.projectdiscovery.io/templating-guide/protocols/headless/).\n\tData map[string]string `yaml:\"args,omitempty\" json:\"args,omitempty\" jsonschema:\"title=arguments for headless action,description=Args contain arguments for the headless action\"`\n\t// description: |\n\t//   Name is the name assigned to the headless action.\n\t//\n\t//   This can be used to execute code, for instance in browser\n\t//   DOM using script action, and get the result in a variable\n\t//   which can be matched upon by nuclei. An Example template [here](https://github.com/projectdiscovery/nuclei-templates/blob/main/headless/prototype-pollution-check.yaml).\n\tName string `yaml:\"name,omitempty\" json:\"name,omitempty\" jsonschema:\"title=name for headless action,description=Name is the name assigned to the headless action\"`\n\t// description: |\n\t//   Description is the optional description of the headless action\n\tDescription string `yaml:\"description,omitempty\" json:\"description,omitempty\" jsonschema:\"title=description for headless action,description=Description of the headless action\"`\n\t// description: |\n\t//   Action is the type of the action to perform.\n\tActionType ActionTypeHolder `yaml:\"action\" json:\"action\" jsonschema:\"title=action to perform,description=Type of actions to perform,enum=navigate,enum=script,enum=click,enum=rightclick,enum=text,enum=screenshot,enum=time,enum=select,enum=files,enum=waitload,enum=getresource,enum=extract,enum=setmethod,enum=addheader,enum=setheader,enum=deleteheader,enum=setbody,enum=waitevent,enum=keyboard,enum=debug,enum=sleep\"`\n}\n\nfunc (a Action) JSONSchemaExtend(schema *jsonschema.Schema) {\n\targsSchema, ok := schema.Properties.Get(\"args\")\n\tif !ok {\n\t\treturn\n\t}\n\targsSchema.PatternProperties = map[string]*jsonschema.Schema{\n\t\t\".*\": {\n\t\t\tOneOf: []*jsonschema.Schema{\n\t\t\t\t{\n\t\t\t\t\tType: \"string\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tType: \"integer\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tType: \"boolean\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\targsSchema.Ref = \"\"\n}\n\n// String returns the string representation of an action\nfunc (a *Action) String() string {\n\tbuilder := &strings.Builder{}\n\tbuilder.WriteString(a.ActionType.String())\n\tif a.Name != \"\" {\n\t\tbuilder.WriteString(\" Name:\")\n\t\tbuilder.WriteString(a.Name)\n\t}\n\tbuilder.WriteString(\" \")\n\tfor k, v := range a.Data {\n\t\tbuilder.WriteString(k)\n\t\tbuilder.WriteString(\":\")\n\t\tbuilder.WriteString(v)\n\t\tbuilder.WriteString(\",\")\n\t}\n\treturn strings.TrimSuffix(builder.String(), \",\")\n}\n\n// GetArg returns an arg for a name\nfunc (a *Action) GetArg(name string) string {\n\tv, ok := a.Data[name]\n\tif !ok {\n\t\treturn \"\"\n\t}\n\treturn v\n}\n\n// Path: pkg/protocols/headless/engine/page.go\npackage engine\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/go-rod/rod\"\n\t\"github.com/go-rod/rod/lib/proto\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\n// Page is a single page in an isolated browser instance\ntype Page struct {\n\tinput          *contextargs.Context\n\toptions        *Options\n\tpage           *rod.Page\n\trules          []rule\n\tinstance       *Instance\n\thijackRouter   *rod.HijackRouter\n\thijackNative   *Hijack\n\tmutex          *sync.RWMutex\n\tHistory        []HistoryData\n\tInteractshURLs []string\n\tpayloads       map[string]interface{}\n}\n\n// HistoryData contains the page request/response pairs\ntype HistoryData struct {\n\tRawRequest  string\n\tRawResponse string\n}\n\n// Options contains additional configuration options for the browser instance\ntype Options struct {\n\tTimeout       time.Duration\n\tDisableCookie bool\n\tOptions       *types.Options\n}\n\n// Run runs a list of actions by creating a new page in the browser.\nfunc (i *Instance) Run(input *contextargs.Context, actions []*Action, payloads map[string]interface{}, options *Options) (map[string]string, *Page, error) {\n\tpage, err := i.engine.Page(proto.TargetCreateTarget{})\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tpage = page.Timeout(options.Timeout)\n\n\tif i.browser.customAgent != \"\" {\n\t\tif userAgentErr := page.SetUserAgent(&proto.NetworkSetUserAgentOverride{UserAgent: i.browser.customAgent}); userAgentErr != nil {\n\t\t\treturn nil, nil, userAgentErr\n\t\t}\n\t}\n\n\tcreatedPage := &Page{\n\t\toptions:  options,\n\t\tpage:     page,\n\t\tinput:    input,\n\t\tinstance: i,\n\t\tmutex:    &sync.RWMutex{},\n\t\tpayloads: payloads,\n\t}\n\n\t// in case the page has request/response modification rules - enable global hijacking\n\tif createdPage.hasModificationRules() || containsModificationActions(actions...) {\n\t\thijackRouter := page.HijackRequests()\n\t\tif err := hijackRouter.Add(\"*\", \"\", createdPage.routingRuleHandler); err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tcreatedPage.hijackRouter = hijackRouter\n\t\tgo hijackRouter.Run()\n\t} else {\n\t\thijackRouter := NewHijack(page)\n\t\thijackRouter.SetPattern(&proto.FetchRequestPattern{\n\t\t\tURLPattern:   \"*\",\n\t\t\tRequestStage: proto.FetchRequestStageResponse,\n\t\t})\n\t\tcreatedPage.hijackNative = hijackRouter\n\t\thijackRouterHandler := hijackRouter.Start(createdPage.routingRuleHandlerNative)\n\t\tgo func() {\n\t\t\t_ = hijackRouterHandler()\n\t\t}()\n\t}\n\n\tif err := page.SetViewport(&proto.EmulationSetDeviceMetricsOverride{Viewport: &proto.PageViewport{\n\t\tScale:  1,\n\t\tWidth:  float64(1920),\n\t\tHeight: float64(1080),\n\t}}); err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif _, err := page.SetExtraHeaders([]string{\"Accept-Language\", \"en, en-GB, en-us;\"}); err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\t// inject cookies\n\t// each http request is performed via the native go http client\n\t// we first inject the shared cookies\n\tURL, err := url.Parse(input.MetaInput.Input)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif !options.DisableCookie {\n\t\tif cookies := input.CookieJar.Cookies(URL); len(cookies) > 0 {\n\t\t\tvar NetworkCookies []*proto.NetworkCookie\n\t\t\tfor _, cookie := range cookies {\n\t\t\t\tnetworkCookie := &proto.NetworkCookie{\n\t\t\t\t\tName:     cookie.Name,\n\t\t\t\t\tValue:    cookie.Value,\n\t\t\t\t\tDomain:   cookie.Domain,\n\t\t\t\t\tPath:     cookie.Path,\n\t\t\t\t\tHTTPOnly: cookie.HttpOnly,\n\t\t\t\t\tSecure:   cookie.Secure,\n\t\t\t\t\tExpires:  proto.TimeSinceEpoch(cookie.Expires.Unix()),\n\t\t\t\t\tSameSite: proto.NetworkCookieSameSite(GetSameSite(cookie)),\n\t\t\t\t\tPriority: proto.NetworkCookiePriorityLow,\n\t\t\t\t}\n\t\t\t\tNetworkCookies = append(NetworkCookies, networkCookie)\n\t\t\t}\n\t\t\tparams := proto.CookiesToParams(NetworkCookies)\n\t\t\tfor _, param := range params {\n\t\t\t\tparam.URL = input.MetaInput.Input\n\t\t\t}\n\t\t\terr := page.SetCookies(params)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, nil, err\n\t\t\t}\n\t\t}\n\t}\n\n\tdata, err := createdPage.ExecuteActions(input, actions, payloads)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tif !options.DisableCookie {\n\t\t// at the end of actions pull out updated cookies from the browser and inject them into the shared cookie jar\n\t\tif cookies, err := page.Cookies([]string{URL.String()}); !options.DisableCookie && err == nil && len(cookies) > 0 {\n\t\t\tvar httpCookies []*http.Cookie\n\t\t\tfor _, cookie := range cookies {\n\t\t\t\thttpCookie := &http.Cookie{\n\t\t\t\t\tName:     cookie.Name,\n\t\t\t\t\tValue:    cookie.Value,\n\t\t\t\t\tDomain:   cookie.Domain,\n\t\t\t\t\tPath:     cookie.Path,\n\t\t\t\t\tHttpOnly: cookie.HTTPOnly,\n\t\t\t\t\tSecure:   cookie.Secure,\n\t\t\t\t}\n\t\t\t\thttpCookies = append(httpCookies, httpCookie)\n\t\t\t}\n\t\t\tinput.CookieJar.SetCookies(URL, httpCookies)\n\t\t}\n\t}\n\n\t// The first item of history data will contain the very first request from the browser\n\t// we assume it's the one matching the initial URL\n\tif len(createdPage.History) > 0 {\n\t\tfirstItem := createdPage.History[0]\n\t\tif resp, err := http.ReadResponse(bufio.NewReader(strings.NewReader(firstItem.RawResponse)), nil); err == nil {\n\t\t\tdata[\"header\"] = utils.HeadersToString(resp.Header)\n\t\t\tdata[\"status_code\"] = fmt.Sprint(resp.StatusCode)\n\t\t\tresp.Body.Close()\n\t\t}\n\t}\n\n\treturn data, createdPage, nil\n}\n\n// Close closes a browser page\nfunc (p *Page) Close() {\n\tif p.hijackRouter != nil {\n\t\t_ = p.hijackRouter.Stop()\n\t}\n\tif p.hijackNative != nil {\n\t\t_ = p.hijackNative.Stop()\n\t}\n\tp.page.Close()\n}\n\n// Page returns the current page for the actions\nfunc (p *Page) Page() *rod.Page {\n\treturn p.page\n}\n\n// Browser returns the browser that created the current page\nfunc (p *Page) Browser() *rod.Browser {\n\treturn p.instance.engine\n}\n\n// URL returns the URL for the current page.\nfunc (p *Page) URL() string {\n\tinfo, err := p.page.Info()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\treturn info.URL\n}\n\n// DumpHistory returns the full page navigation history\nfunc (p *Page) DumpHistory() string {\n\tp.mutex.RLock()\n\tdefer p.mutex.RUnlock()\n\n\tvar historyDump strings.Builder\n\tfor _, historyData := range p.History {\n\t\thistoryDump.WriteString(historyData.RawRequest)\n\t\thistoryDump.WriteString(historyData.RawResponse)\n\t}\n\treturn historyDump.String()\n}\n\n// addToHistory adds a request/response pair to the page history\nfunc (p *Page) addToHistory(historyData ...HistoryData) {\n\tp.mutex.Lock()\n\tdefer p.mutex.Unlock()\n\n\tp.History = append(p.History, historyData...)\n}\n\nfunc (p *Page) addInteractshURL(URLs ...string) {\n\tp.mutex.Lock()\n\tdefer p.mutex.Unlock()\n\n\tp.InteractshURLs = append(p.InteractshURLs, URLs...)\n}\n\nfunc (p *Page) hasModificationRules() bool {\n\tfor _, rule := range p.rules {\n\t\tif containsAnyModificationActionType(rule.Action) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc containsModificationActions(actions ...*Action) bool {\n\tfor _, action := range actions {\n\t\tif containsAnyModificationActionType(action.ActionType.ActionType) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc containsAnyModificationActionType(actionTypes ...ActionType) bool {\n\tfor _, actionType := range actionTypes {\n\t\tswitch actionType {\n\t\tcase ActionSetMethod:\n\t\t\treturn true\n\t\tcase ActionAddHeader:\n\t\t\treturn true\n\t\tcase ActionSetHeader:\n\t\t\treturn true\n\t\tcase ActionDeleteHeader:\n\t\t\treturn true\n\t\tcase ActionSetBody:\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc GetSameSite(cookie *http.Cookie) string {\n\tswitch cookie.SameSite {\n\tcase http.SameSiteNoneMode:\n\t\treturn \"none\"\n\tcase http.SameSiteLaxMode:\n\t\treturn \"lax\"\n\tcase http.SameSiteStrictMode:\n\t\treturn \"strict\"\n\tcase http.SameSiteDefaultMode:\n\t\tfallthrough\n\tdefault:\n\t\treturn \"\"\n\t}\n}\n\n// Path: pkg/protocols/headless/engine/instance.go\npackage engine\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"time\"\n\n\t\"github.com/go-rod/rod\"\n\t\"github.com/go-rod/rod/lib/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n)\n\n// Instance is an isolated browser instance opened for doing operations with it.\ntype Instance struct {\n\tbrowser *Browser\n\tengine  *rod.Browser\n\n\t// redundant due to dependency cycle\n\tinteractsh *interactsh.Client\n\trequestLog map[string]string // contains actual request that was sent\n}\n\n// NewInstance creates a new instance for the current browser.\n//\n// The login process is repeated only once for a browser, and the created\n// isolated browser instance is used for entire navigation one by one.\n//\n// Users can also choose to run the login->actions process again\n// which uses a new incognito browser instance to run actions.\nfunc (b *Browser) NewInstance() (*Instance, error) {\n\tbrowser, err := b.engine.Incognito()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// We use a custom sleeper that sleeps from 100ms to 500 ms waiting\n\t// for an interaction. Used throughout rod for clicking, etc.\n\tbrowser = browser.Sleeper(func() utils.Sleeper { return maxBackoffSleeper(10) })\n\treturn &Instance{browser: b, engine: browser, requestLog: map[string]string{}}, nil\n}\n\n// returns a map of [template-defined-urls] -> [actual-request-sent]\n// Note: this does not include CORS or other requests while rendering that were not explicitly\n// specified in template\nfunc (i *Instance) GetRequestLog() map[string]string {\n\treturn i.requestLog\n}\n\n// Close closes all the tabs and pages for a browser instance\nfunc (i *Instance) Close() error {\n\treturn i.engine.Close()\n}\n\n// SetInteractsh client\nfunc (i *Instance) SetInteractsh(interactsh *interactsh.Client) {\n\ti.interactsh = interactsh\n}\n\n// maxBackoffSleeper is a backoff sleeper respecting max backoff values\nfunc maxBackoffSleeper(max int) utils.Sleeper {\n\tcount := 0\n\tbackoffSleeper := utils.BackoffSleeper(100*time.Millisecond, 500*time.Millisecond, nil)\n\n\treturn func(ctx context.Context) error {\n\t\tif ctx.Err() != nil {\n\t\t\treturn ctx.Err()\n\t\t}\n\t\tif count == max {\n\t\t\treturn errors.New(\"max sleep count\")\n\t\t}\n\t\tcount++\n\t\treturn backoffSleeper(ctx)\n\t}\n}\n\n// Path: pkg/protocols/headless/request.go\npackage headless\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/exp/maps\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/fuzz\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\turlutil \"github.com/projectdiscovery/utils/url\"\n)\n\nvar _ protocols.Request = &Request{}\n\nconst errCouldGetHtmlElement = \"could get html element\"\n\n// Type returns the type of the protocol request\nfunc (request *Request) Type() templateTypes.ProtocolType {\n\treturn templateTypes.HeadlessProtocol\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\tif request.SelfContained {\n\t\turl, err := extractBaseURLFromActions(request.Steps)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tinput = contextargs.NewWithInput(url)\n\t}\n\n\tif request.options.Browser.UserAgent() == \"\" {\n\t\trequest.options.Browser.SetUserAgent(request.compiledUserAgent)\n\t}\n\n\tvars := protocolutils.GenerateVariablesWithContextArgs(input, false)\n\tpayloads := generators.BuildPayloadFromOptions(request.options.Options)\n\t// add templatecontext variables to varMap\n\tvalues := generators.MergeMaps(vars, metadata, payloads)\n\tif request.options.HasTemplateCtx(input.MetaInput) {\n\t\tvalues = generators.MergeMaps(values, request.options.GetTemplateCtx(input.MetaInput).GetAll())\n\t}\n\tvariablesMap := request.options.Variables.Evaluate(values)\n\tpayloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants)\n\n\t// check for operator matches by wrapping callback\n\tgotmatches := false\n\twrappedCallback := func(results *output.InternalWrappedEvent) {\n\t\tcallback(results)\n\t\tif results != nil && results.OperatorsResult != nil {\n\t\t\tgotmatches = results.OperatorsResult.Matched\n\t\t}\n\t}\n\t// verify if fuzz elaboration was requested\n\tif len(request.Fuzzing) > 0 {\n\t\treturn request.executeFuzzingRule(input, payloads, previous, wrappedCallback)\n\t}\n\tif request.generator != nil {\n\t\titerator := request.generator.NewIterator()\n\t\tfor {\n\t\t\tvalue, ok := iterator.Value()\n\t\t\tif !ok {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif gotmatches && (request.StopAtFirstMatch || request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tvalue = generators.MergeMaps(value, payloads)\n\t\t\tif err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t} else {\n\t\tvalue := maps.Clone(payloads)\n\t\tif err := request.executeRequestWithPayloads(input, value, previous, wrappedCallback); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// This function extracts the base URL from actions.\nfunc extractBaseURLFromActions(steps []*engine.Action) (string, error) {\n\tfor _, action := range steps {\n\t\tif action.ActionType.ActionType == engine.ActionNavigate {\n\t\t\tnavigateURL := action.GetArg(\"url\")\n\t\t\turl, err := urlutil.Parse(navigateURL)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", errors.Errorf(\"could not parse URL '%s': %s\", navigateURL, err.Error())\n\t\t\t}\n\t\t\treturn fmt.Sprintf(\"%s://%s\", url.Scheme, url.Host), nil\n\t\t}\n\t}\n\treturn \"\", errors.New(\"no navigation action found\")\n}\n\nfunc (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\tinstance, err := request.options.Browser.NewInstance()\n\tif err != nil {\n\t\trequest.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)\n\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, errCouldGetHtmlElement)\n\t}\n\tdefer instance.Close()\n\n\tif vardump.EnableVarDump {\n\t\tgologger.Debug().Msgf(\"Headless Protocol request variables: \\n%s\\n\", vardump.DumpVariables(payloads))\n\t}\n\n\tinstance.SetInteractsh(request.options.Interactsh)\n\n\tif _, err := url.Parse(input.MetaInput.Input); err != nil {\n\t\trequest.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)\n\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, errCouldGetHtmlElement)\n\t}\n\toptions := &engine.Options{\n\t\tTimeout:       time.Duration(request.options.Options.PageTimeout) * time.Second,\n\t\tDisableCookie: request.DisableCookie,\n\t\tOptions:       request.options.Options,\n\t}\n\n\tif !options.DisableCookie && input.CookieJar == nil {\n\t\treturn errors.New(\"cookie reuse enabled but cookie-jar is nil\")\n\t}\n\n\tout, page, err := instance.Run(input, request.Steps, payloads, options)\n\tif err != nil {\n\t\trequest.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)\n\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, errCouldGetHtmlElement)\n\t}\n\tdefer page.Close()\n\n\treqLog := instance.GetRequestLog()\n\tnavigatedURL := request.getLastNavigationURLWithLog(reqLog) // also known as matchedURL if there is a match\n\n\trequest.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), nil)\n\trequest.options.Progress.IncrementRequests()\n\tgologger.Verbose().Msgf(\"Sent Headless request to %s\", navigatedURL)\n\n\treqBuilder := &strings.Builder{}\n\tif request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.DebugResponse {\n\t\tgologger.Info().Msgf(\"[%s] Dumped Headless request for %s\", request.options.TemplateID, navigatedURL)\n\n\t\tfor _, act := range request.Steps {\n\t\t\tif act.ActionType.ActionType == engine.ActionNavigate {\n\t\t\t\tvalue := act.GetArg(\"url\")\n\t\t\t\tif reqLog[value] != \"\" {\n\t\t\t\t\treqBuilder.WriteString(fmt.Sprintf(\"\\tnavigate => %v\\n\", reqLog[value]))\n\t\t\t\t} else {\n\t\t\t\t\treqBuilder.WriteString(fmt.Sprintf(\"%v not found in %v\\n\", value, reqLog))\n\t\t\t\t}\n...\n", "needle_token_start": 8177, "needle_token_end": 8442, "code_context_ntokens": 15173, "output": ["```go\n// dump creates a dump of the http request in form of a byte slice\nfunc dump(req *generatedRequest, reqURL string) ([]byte, error) {\n\tif req.request != nil {\n\t\tbin, err := req.request.Dump()\n\t\tif err != nil {\n\t\t\treturn nil, errorutil.NewWithErr(err).WithTag(\"http\").Msgf(\"could not dump request: %v\", req.request.URL.String())\n\t\t}\n\t\treturn bin, nil\n\t}\n\trawHttpOptions := &rawhttp.Options{CustomHeaders: req.rawRequest.UnsafeHeaders, CustomRawBytes: req.rawRequest.UnsafeRawBytes}\n\tbin, err := rawhttp.DumpRequestRaw(req.rawRequest.Method, reqURL, req.rawRequest.Path, generators.ExpandMapValues(req.rawRequest.Headers), io.NopCloser(strings.NewReader(req.rawRequest.Data)), rawHttpOptions)\n\tif err != nil {\n\t\treturn nil, errorutil.NewWithErr(err).WithTag(\"http\").Msgf(\"could not dump request: %v\", reqURL)\n\t}\n\treturn bin, nil\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "GetVariablesNamesSkipList", "language": "go", "path": "pkg/protocols/http/signature.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to retrieve a list of variable names that should be skipped or ignored based on a specified type of digital signature.\n2. **Input**: It accepts an enumeration value representing a specific type of digital signature.\n3. **Output**: It returns a map where keys are the names of variables to be skipped and the values are typically set to `nil`, indicating that these variables should be ignored. If the signature type is not recognized, it returns `nil`.\n4. **Procedure**: The function checks the provided signature type and, depending on the type, returns a predefined list of variable names that should be ignored for that signature type. If the signature type does not match any predefined types, it returns `nil`.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/protocols/http/request_fuzz.go\npackage http\n\n// === Fuzzing Documentation (Scoped to this File) =====\n// -> request.executeFuzzingRule   [iterates over payloads(+requests) and executes]\n//\t-> request.executePayloadUsingRules [executes single payload on all rules (if more than 1)]\n//\t\t-> request.executeGeneratedFuzzingRequest [execute final generated fuzzing request and get result]\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/fuzz\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\t\"github.com/projectdiscovery/retryablehttp-go\"\n\turlutil \"github.com/projectdiscovery/utils/url\"\n)\n\n// executeFuzzingRule executes fuzzing request for a URL\n// TODO:\n// 1. use SPMHandler and rewrite stop at first match logic here\n// 2. use scanContext instead of contextargs.Context\nfunc (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\t// methdology:\n\t// to check applicablity of rule, we first try to execute it with one value\n\t// if it is applicable, we execute all requests\n\t// if it is not applicable, we log and fail silently\n\n\t// check if target should be fuzzed or not\n\tif !request.ShouldFuzzTarget(input) {\n\t\turlx, _ := input.MetaInput.URL()\n\t\tif urlx != nil {\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: target(%s) not applicable for fuzzing\\n\", request.options.TemplateID, urlx.String())\n\t\t} else {\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: target(%s) not applicable for fuzzing\\n\", request.options.TemplateID, input.MetaInput.Input)\n\t\t}\n\t\treturn nil\n\t}\n\n\tif input.MetaInput.Input == \"\" && input.MetaInput.ReqResp == nil {\n\t\treturn errors.New(\"empty input provided for fuzzing\")\n\t}\n\n\t// ==== fuzzing when full HTTP request is provided =====\n\n\tif input.MetaInput.ReqResp != nil {\n\t\tbaseRequest, err := input.MetaInput.ReqResp.BuildRequest()\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"fuzz: could not build request obtained from target file\")\n\t\t}\n\t\tinput.MetaInput.Input = baseRequest.URL.String()\n\t\t// execute with one value first to checks its applicability\n\t\terr = request.executeAllFuzzingRules(input, previous, baseRequest, callback)\n\t\tif err != nil {\n\t\t\t// in case of any error, return it\n\t\t\tif fuzz.IsErrRuleNotApplicable(err) {\n\t\t\t\t// log and fail silently\n\t\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: %s\\n\", request.options.TemplateID, err)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif errors.Is(err, ErrMissingVars) {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: payload request execution failed : %s\\n\", request.options.TemplateID, err)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// ==== fuzzing when only URL is provided =====\n\n\t// we need to use this url instead of input\n\tinputx := input.Clone()\n\tparsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, true)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fuzz: could not parse input url\")\n\t}\n\tbaseRequest, err := retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fuzz: could not build request from url\")\n\t}\n\t// execute with one value first to checks its applicability\n\terr = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback)\n\tif err != nil {\n\t\t// in case of any error, return it\n\t\tif fuzz.IsErrRuleNotApplicable(err) {\n\t\t\t// log and fail silently\n\t\t\tgologger.Verbose().Msgf(\"[%s] fuzz: rule not applicable : %s\\n\", request.options.TemplateID, err)\n\t\t\treturn nil\n\t\t}\n\t\tif errors.Is(err, ErrMissingVars) {\n\t\t\treturn err\n\t\t}\n\t\tgologger.Verbose().Msgf(\"[%s] fuzz: payload request execution failed : %s\\n\", request.options.TemplateID, err)\n\t}\n\treturn nil\n}\n\n// executeAllFuzzingRules executes all fuzzing rules defined in template for a given base request\nfunc (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error {\n\tapplicable := false\n\tfor _, rule := range request.Fuzzing {\n\t\terr := rule.Execute(&fuzz.ExecuteRuleInput{\n\t\t\tInput: input,\n\t\t\tCallback: func(gr fuzz.GeneratedRequest) bool {\n\t\t\t\t// TODO: replace this after scanContext Refactor\n\t\t\t\treturn request.executeGeneratedFuzzingRequest(gr, input, callback)\n\t\t\t},\n\t\t\tValues:      values,\n\t\t\tBaseRequest: baseRequest.Clone(context.TODO()),\n\t\t})\n\t\tif err == nil {\n\t\t\tapplicable = true\n\t\t\tcontinue\n\t\t}\n\t\tif fuzz.IsErrRuleNotApplicable(err) {\n\t\t\tcontinue\n\t\t}\n\t\tif err == types.ErrNoMoreRequests {\n\t\t\treturn nil\n\t\t}\n\t\treturn errors.Wrap(err, \"could not execute rule\")\n\t}\n\n\tif !applicable {\n\t\treturn fuzz.ErrRuleNotApplicable.Msgf(fmt.Sprintf(\"no rule was applicable for this request: %v\", input.MetaInput.Input))\n\t}\n\treturn nil\n}\n\n// executeGeneratedFuzzingRequest executes a generated fuzzing request after building it using rules and payloads\nfunc (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool {\n\thasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)\n\thasInteractMarkers := len(gr.InteractURLs) > 0\n\tif request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {\n\t\treturn false\n\t}\n\trequest.options.RateLimitTake()\n\treq := &generatedRequest{\n\t\trequest:        gr.Request,\n\t\tdynamicValues:  gr.DynamicValues,\n\t\tinteractshURLs: gr.InteractURLs,\n\t\toriginal:       request,\n\t}\n\tvar gotMatches bool\n\trequestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {\n\t\tif hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {\n\t\t\trequestData := &interactsh.RequestData{\n\t\t\t\tMakeResultFunc: request.MakeResultEvent,\n\t\t\t\tEvent:          event,\n\t\t\t\tOperators:      request.CompiledOperators,\n\t\t\t\tMatchFunc:      request.Match,\n\t\t\t\tExtractFunc:    request.Extract,\n\t\t\t}\n\t\t\trequest.options.Interactsh.RequestEvent(gr.InteractURLs, requestData)\n\t\t\tgotMatches = request.options.Interactsh.AlreadyMatched(requestData)\n\t\t} else {\n\t\t\tcallback(event)\n\t\t}\n\t\t// Add the extracts to the dynamic values if any.\n\t\tif event.OperatorsResult != nil {\n\t\t\tgotMatches = event.OperatorsResult.Matched\n\t\t}\n\t}, 0)\n\t// If a variable is unresolved, skip all further requests\n\tif errors.Is(requestErr, ErrMissingVars) {\n\t\treturn false\n\t}\n\tif requestErr != nil {\n\t\tif request.options.HostErrorsCache != nil {\n\t\t\trequest.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr)\n\t\t}\n\t\tgologger.Verbose().Msgf(\"[%s] Error occurred in request: %s\\n\", request.options.TemplateID, requestErr)\n\t}\n\trequest.options.Progress.IncrementRequests()\n\n\t// If this was a match, and we want to stop at first match, skip all further requests.\n\tshouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch\n\tif shouldStopAtFirstMatch && gotMatches {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// ShouldFuzzTarget checks if given target should be fuzzed or not using `filter` field in template\nfunc (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool {\n\tif len(request.FuzzPreCondition) == 0 {\n\t\treturn true\n\t}\n\tstatus := []bool{}\n\tfor index, filter := range request.FuzzPreCondition {\n\t\tisMatch, _ := request.Match(request.filterDataMap(input), filter)\n\t\tstatus = append(status, isMatch)\n\t\tif request.options.Options.MatcherStatus {\n\t\t\tgologger.Debug().Msgf(\"[%s] [%s] Filter => %s : %v\", input.MetaInput.Target(), request.options.TemplateID, operators.GetMatcherName(filter, index), isMatch)\n\t\t}\n\t}\n\tif len(status) == 0 {\n\t\treturn true\n\t}\n\tvar matched bool\n\tif request.fuzzPreConditionOperator == matchers.ANDCondition {\n\t\tmatched = operators.EvalBoolSlice(status, true)\n\t} else {\n\t\tmatched = operators.EvalBoolSlice(status, false)\n\t}\n\tif request.options.Options.MatcherStatus {\n\t\tgologger.Debug().Msgf(\"[%s] [%s] Final Filter Status =>  %v\", input.MetaInput.Target(), request.options.TemplateID, matched)\n\t}\n\treturn matched\n}\n\n// input data map returns map[string]interface{} from input\nfunc (request *Request) filterDataMap(input *contextargs.Context) map[string]interface{} {\n\tm := make(map[string]interface{})\n\tparsed, err := input.MetaInput.URL()\n\tif err != nil {\n\t\tm[\"host\"] = input.MetaInput.Input\n\t\treturn m\n\t}\n\tm = protocolutils.GenerateVariables(parsed, true, m)\n\tfor k, v := range m {\n\t\tm[strings.ToLower(k)] = v\n\t}\n\tm[\"path\"] = parsed.Path // override existing\n\tm[\"query\"] = parsed.RawQuery\n\t// add request data like headers, body etc\n\tif input.MetaInput.ReqResp != nil && input.MetaInput.ReqResp.Request != nil {\n\t\treq := input.MetaInput.ReqResp.Request\n\t\tm[\"method\"] = req.Method\n\t\tm[\"body\"] = req.Body\n\n\t\tsb := &strings.Builder{}\n\t\treq.Headers.Iterate(func(k, v string) bool {\n\t\t\tk = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(k), \"-\", \"_\"))\n\t\t\tif strings.EqualFold(k, \"Cookie\") {\n\t\t\t\tm[\"cookie\"] = v\n\t\t\t}\n\t\t\tif strings.EqualFold(k, \"User_Agent\") {\n\t\t\t\tm[\"user_agent\"] = v\n\t\t\t}\n\t\t\tif strings.EqualFold(k, \"content_type\") {\n\t\t\t\tm[\"content_type\"] = v\n...\n// Path: pkg/protocols/http/request_generator.go\npackage http\n\nimport (\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n)\n\n// requestGenerator generates requests sequentially based on various\n// configurations for a http request template.\n//\n// If payload values are present, an iterator is created for the payload\n// values. Paths and Raw requests are supported as base input, so\n// it will automatically select between them based on the template.\ntype requestGenerator struct {\n\tcurrentIndex     int\n\tcurrentPayloads  map[string]interface{}\n\tokCurrentPayload bool\n\trequest          *Request\n\toptions          *protocols.ExecutorOptions\n\tpayloadIterator  *generators.Iterator\n\tinteractshURLs   []string\n\tonceFlow         map[string]struct{}\n}\n\n// LeaveDefaultPorts skips normalization of default standard ports\nvar LeaveDefaultPorts = false\n\n// newGenerator creates a new request generator instance\nfunc (request *Request) newGenerator(disablePayloads bool) *requestGenerator {\n\tgenerator := &requestGenerator{\n\t\trequest:  request,\n\t\toptions:  request.options,\n\t\tonceFlow: make(map[string]struct{}),\n\t}\n\n\tif len(request.Payloads) > 0 && !disablePayloads {\n\t\tgenerator.payloadIterator = request.generator.NewIterator()\n\t}\n\treturn generator\n}\n\n// nextValue returns the next path or the next raw request depending on user input\n// It returns false if all the inputs have been exhausted by the generator instance.\nfunc (r *requestGenerator) nextValue() (value string, payloads map[string]interface{}, result bool) {\n\t// Iterate each payload sequentially for each request path/raw\n\t//\n\t// If the sequence has finished for the current payload values\n\t// then restart the sequence from the beginning and move on to the next payloads values\n\t// otherwise use the last request.\n\tvar sequence []string\n\tswitch {\n\tcase len(r.request.Path) > 0:\n\t\tsequence = r.request.Path\n\tcase len(r.request.Raw) > 0:\n\t\tsequence = r.request.Raw\n\tdefault:\n\t\treturn \"\", nil, false\n\t}\n\n\thasPayloadIterator := r.payloadIterator != nil\n\n\tif hasPayloadIterator && r.currentPayloads == nil {\n\t\tr.currentPayloads, r.okCurrentPayload = r.payloadIterator.Value()\n\t}\n\n\tvar request string\n\tvar shouldContinue bool\n\tif nextRequest, nextIndex, found := r.findNextIteration(sequence, r.currentIndex); found {\n\t\tr.currentIndex = nextIndex + 1\n\t\trequest = nextRequest\n\t\tshouldContinue = true\n\t} else {\n\t\t// if found is false which happens at end of iteration of reqData(path or raw request)\n\t\t// try again from start with index 0\n\t\tif nextRequest, nextIndex, found := r.findNextIteration(sequence, 0); found && hasPayloadIterator {\n\t\t\tr.currentIndex = nextIndex + 1\n\t\t\trequest = nextRequest\n\t\t\tshouldContinue = true\n\t\t}\n\t}\n\n\tif shouldContinue {\n\t\tif r.hasMarker(request, Once) {\n\t\t\tr.applyMark(request, Once)\n\t\t}\n\t\tif hasPayloadIterator {\n\t\t\treturn request, generators.MergeMaps(r.currentPayloads), r.okCurrentPayload\n\t\t}\n\t\t// next should return a copy of payloads and not pointer to payload to avoid data race\n\t\treturn request, generators.MergeMaps(r.currentPayloads), true\n\t} else {\n\t\treturn \"\", nil, false\n\t}\n}\n\n// findNextIteration iterates and returns next Request(path or raw request)\n// at end of each iteration payload is incremented\nfunc (r *requestGenerator) findNextIteration(sequence []string, index int) (string, int, bool) {\n\tfor i, request := range sequence[index:] {\n\t\tif r.wasMarked(request, Once) {\n\t\t\t// if request contains flowmark i.e `@once` and is marked skip it\n\t\t\tcontinue\n\t\t}\n\t\treturn request, index + i, true\n\n\t}\n\t// move on to next payload if current payload is applied/returned for all Requests(path or raw request)\n\tif r.payloadIterator != nil {\n\t\tr.currentPayloads, r.okCurrentPayload = r.payloadIterator.Value()\n\t}\n\treturn \"\", 0, false\n}\n\n// applyMark marks given request i.e blacklist request\nfunc (r *requestGenerator) applyMark(request string, mark flowMark) {\n\tswitch mark {\n\tcase Once:\n\t\tr.onceFlow[request] = struct{}{}\n\t}\n\n}\n\n// wasMarked checks if request is marked using request blacklist\nfunc (r *requestGenerator) wasMarked(request string, mark flowMark) bool {\n\tswitch mark {\n\tcase Once:\n\t\t_, ok := r.onceFlow[request]\n\t\treturn ok\n\t}\n\treturn false\n}\n\n// hasMarker returns true if request has a marker (ex: @once which means request should only be executed once)\nfunc (r *requestGenerator) hasMarker(request string, mark flowMark) bool {\n\tfo, hasOverrides := parseFlowAnnotations(request)\n\treturn hasOverrides && fo == mark\n}\n\n// Path: pkg/protocols/http/request_generator_test.go\npackage http\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\nfunc TestRequestGeneratorPaths(t *testing.T) {\n\treq := &Request{\n\t\tPath: []string{\"{{BaseURL}}/test\", \"{{BaseURL}}/test.php\"},\n\t}\n\tgenerator := req.newGenerator(false)\n\tvar payloads []string\n\tfor {\n\t\traw, _, ok := generator.nextValue()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tpayloads = append(payloads, raw)\n\t}\n\trequire.Equal(t, req.Path, payloads, \"Could not get correct paths\")\n}\n\nfunc TestRequestGeneratorClusterBombSingle(t *testing.T) {\n\tvar err error\n\n\treq := &Request{\n\t\tPayloads:   map[string]interface{}{\"username\": []string{\"admin\", \"tomcat\", \"manager\"}, \"password\": []string{\"password\", \"test\", \"secret\"}},\n\t\tAttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},\n\t\tRaw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`},\n\t}\n\tcatalogInstance := disk.NewCatalog(\"\")\n\treq.generator, err = generators.New(req.Payloads, req.AttackType.Value, \"\", catalogInstance, \"\", types.DefaultOptions())\n\trequire.Nil(t, err, \"could not create generator\")\n\n\tgenerator := req.newGenerator(false)\n\tvar payloads []map[string]interface{}\n\tfor {\n\t\t_, data, ok := generator.nextValue()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tpayloads = append(payloads, data)\n\t}\n\trequire.Equal(t, 9, len(payloads), \"Could not get correct number of payloads\")\n}\n\nfunc TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) {\n\tvar err error\n\n\treq := &Request{\n\t\tPayloads:   map[string]interface{}{\"username\": []string{\"admin\", \"tomcat\", \"manager\"}, \"password\": []string{\"password\", \"test\", \"secret\"}},\n\t\tAttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},\n\t\tRaw:        []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`},\n\t}\n\tcatalogInstance := disk.NewCatalog(\"\")\n\treq.generator, err = generators.New(req.Payloads, req.AttackType.Value, \"\", catalogInstance, \"\", types.DefaultOptions())\n\trequire.Nil(t, err, \"could not create generator\")\n\n\tgenerator := req.newGenerator(false)\n\tvar payloads []map[string]interface{}\n\tfor {\n\t\t_, data, ok := generator.nextValue()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tpayloads = append(payloads, data)\n\t}\n\trequire.Equal(t, 18, len(payloads), \"Could not get correct number of payloads\")\n}\n\n// Path: pkg/protocols/http/request_test.go\npackage http\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/testutils\"\n)\n\nfunc TestHTTPExtractMultipleReuse(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\ttestutils.Init(options)\n\ttemplateID := \"testing-http\"\n\trequest := &Request{\n\t\tID: templateID,\n\t\tRaw: []string{\n\t\t\t`GET /robots.txt HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\t\t\t`,\n\n\t\t\t`GET {{endpoint}} HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\t\t\t`,\n\t\t},\n\t\tOperators: operators.Operators{\n\t\t\tMatchers: []*matchers.Matcher{{\n\t\t\t\tPart:  \"body\",\n\t\t\t\tType:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},\n\t\t\t\tWords: []string{\"match /a\", \"match /b\", \"match /c\"},\n\t\t\t}},\n\t\t\tExtractors: []*extractors.Extractor{{\n\t\t\t\tPart:     \"body\",\n\t\t\t\tName:     \"endpoint\",\n\t\t\t\tType:     extractors.ExtractorTypeHolder{ExtractorType: extractors.RegexExtractor},\n\t\t\t\tRegex:    []string{\"(?m)/([a-zA-Z0-9-_/\\\\\\\\]+)\"},\n\t\t\t\tInternal: true,\n\t\t\t}},\n\t\t},\n\t\tIterateAll: true,\n\t}\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tswitch r.URL.Path {\n\t\tcase \"/robots.txt\":\n\t\t\t_, _ = w.Write([]byte(`User-agent: Googlebot\nDisallow: /a\nDisallow: /b\nDisallow: /c`))\n\t\tdefault:\n\t\t\t_, _ = w.Write([]byte(fmt.Sprintf(`match %v`, r.URL.Path)))\n\t\t}\n\t}))\n\tdefer ts.Close()\n\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile network request\")\n\n\tvar finalEvent *output.InternalWrappedEvent\n\tvar matchCount int\n\tt.Run(\"test\", func(t *testing.T) {\n\t\tmetadata := make(output.InternalEvent)\n\t\tprevious := make(output.InternalEvent)\n\t\tctxArgs := contextargs.NewWithInput(ts.URL)\n\t\terr := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {\n\t\t\tif event.OperatorsResult != nil && event.OperatorsResult.Matched {\n\t\t\t\tmatchCount++\n\t\t\t}\n\t\t\tfinalEvent = event\n\t\t})\n\t\trequire.Nil(t, err, \"could not execute network request\")\n\t})\n\trequire.NotNil(t, finalEvent, \"could not get event output from request\")\n\trequire.Equal(t, 3, matchCount, \"could not get correct match count\")\n}\n\nfunc TestDisableTE(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\ttestutils.Init(options)\n\ttemplateID := \"http-disable-transfer-encoding\"\n\n\t// in raw request format\n\trequest := &Request{\n\t\tID: templateID,\n\t\tRaw: []string{\n\t\t\t`POST / HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\n\t\t\tlogin=1&username=admin&password=admin\n\t\t\t`,\n\t\t},\n\t\tOperators: operators.Operators{\n\t\t\tMatchers: []*matchers.Matcher{{\n\t\t\t\tType:   matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},\n\t\t\t\tStatus: []int{200},\n\t\t\t}},\n\t\t},\n\t}\n\n\t// in base request format\n\trequest2 := &Request{\n\t\tID:     templateID,\n\t\tMethod: HTTPMethodTypeHolder{MethodType: HTTPPost},\n\t\tPath:   []string{\"{{BaseURL}}\"},\n\t\tBody:   \"login=1&username=admin&password=admin\",\n\t\tOperators: operators.Operators{\n\t\t\tMatchers: []*matchers.Matcher{{\n\t\t\t\tType:   matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},\n\t\t\t\tStatus: []int{200},\n\t\t\t}},\n\t\t},\n\t}\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tif len(r.TransferEncoding) > 0 || r.ContentLength <= 0 {\n\t\t\tt.Error(\"Transfer-Encoding header should not be set\")\n\t\t}\n\t}))\n\tdefer ts.Close()\n\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile http raw request\")\n\n\terr = request2.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile http base request\")\n\n\tvar finalEvent *output.InternalWrappedEvent\n\tvar matchCount int\n\tt.Run(\"test\", func(t *testing.T) {\n\t\tmetadata := make(output.InternalEvent)\n\t\tprevious := make(output.InternalEvent)\n\t\tctxArgs := contextargs.NewWithInput(ts.URL)\n\t\terr := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {\n\t\t\tif event.OperatorsResult != nil && event.OperatorsResult.Matched {\n\t\t\t\tmatchCount++\n\t\t\t}\n\t\t\tfinalEvent = event\n\t\t})\n\t\trequire.Nil(t, err, \"could not execute network request\")\n\t})\n\n\tt.Run(\"test2\", func(t *testing.T) {\n\t\tmetadata := make(output.InternalEvent)\n\t\tprevious := make(output.InternalEvent)\n\t\tctxArgs := contextargs.NewWithInput(ts.URL)\n\t\terr := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {\n\t\t\tif event.OperatorsResult != nil && event.OperatorsResult.Matched {\n\t\t\t\tmatchCount++\n\t\t\t}\n\t\t\tfinalEvent = event\n\t\t})\n\t\trequire.Nil(t, err, \"could not execute network request\")\n\t})\n\n\trequire.NotNil(t, finalEvent, \"could not get event output from request\")\n\trequire.Equal(t, 2, matchCount, \"could not get correct match count\")\n}\n\n// consult @Ice3man543 before making any breaking changes to this test (context: vuln_hash)\nfunc TestReqURLPattern(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\t// assume this was a preprocessor\n\t// {{randstr}} => 2eNU2kbrOcUDzhnUL1RGvSo1it7\n\ttestutils.Init(options)\n\ttemplateID := \"testing-http\"\n\trequest := &Request{\n\t\tID: templateID,\n\t\tRaw: []string{\n\t\t\t`GET /{{rand_char(\"abc\")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data=2eNU2kbrOcUDzhnUL1RGvSo1it7 HTTP/1.1\n\t\t\tHost: {{Hostname}}\n\t\t\tUser-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0\n\t\t\tAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\n\t\t\tAccept-Language: en-US,en;q=0.5\n\t\t\t`,\n\t\t},\n\t\tOperators: operators.Operators{\n\t\t\tMatchers: []*matchers.Matcher{{\n\t\t\t\tType: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},\n\t\t\t\tDSL:  []string{\"true\"},\n\t\t\t}},\n\t\t},\n\t\tIterateAll: true,\n\t}\n\tts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t// always return 200\n\t\tw.WriteHeader(200)\n\t\t_, _ = w.Write([]byte(`match`))\n\t}))\n\tdefer ts.Close()\n\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\tclient, _ := interactsh.New(interactsh.DefaultOptions(executerOpts.Output, nil, executerOpts.Progress))\n\texecuterOpts.Interactsh = client\n\tdefer client.Close()\n\texecuterOpts.ExportReqURLPattern = true\n\n\t// this is how generated constants are added to template\n\t// generated constants are preprocessors that are executed while loading once\n\texecuterOpts.Constants = map[string]interface{}{\n\t\t\"{{randstr}}\": \"2eNU2kbrOcUDzhnUL1RGvSo1it7\",\n\t}\n\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile network request\")\n\n\tvar finalEvent *output.InternalWrappedEvent\n\tvar matchCount int\n\tt.Run(\"test\", func(t *testing.T) {\n\t\tmetadata := make(output.InternalEvent)\n\t\tprevious := make(output.InternalEvent)\n\t\tctxArgs := contextargs.NewWithInput(ts.URL)\n\t\terr := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {\n\t\t\tif event.OperatorsResult != nil && event.OperatorsResult.Matched {\n\t\t\t\tmatchCount++\n\t\t\t}\n\t\t\tfinalEvent = event\n\t\t})\n\t\trequire.Nil(t, err, \"could not execute network request\")\n\t})\n\trequire.NotNil(t, finalEvent, \"could not get event output from request\")\n\trequire.Equal(t, 1, matchCount, \"could not get correct match count\")\n\trequire.NotEmpty(t, finalEvent.Results[0].ReqURLPattern, \"could not get req url pattern\")\n\trequire.Equal(t, `/{{rand_char(\"abc\")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data={{randstr}}`, finalEvent.Results[0].ReqURLPattern)\n}\n\n// Path: pkg/protocols/http/signature.go\npackage http\n\nimport (\n\t\"encoding/json\"\n\t\"strings\"\n\n\t\"github.com/invopop/jsonschema\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer\"\n)\n\n// SignatureType is the type of signature\ntype SignatureType int\n\n// Supported values for the SignatureType\nconst (\n\tAWSSignature SignatureType = iota + 1\n\tsignatureLimit\n)\n\n// signatureTypeMappings is a table for conversion of signature type from string.\nvar signatureTypeMappings = map[SignatureType]string{\n\tAWSSignature: \"AWS\",\n}\n\nfunc GetSupportedSignaturesTypes() []SignatureType {\n\tvar result []SignatureType\n\tfor index := SignatureType(1); index < signatureLimit; index++ {\n\t\tresult = append(result, index)\n\t}\n\treturn result\n}\n\nfunc toSignatureType(valueToMap string) (SignatureType, error) {\n\tnormalizedValue := normalizeValue(valueToMap)\n\tfor key, currentValue := range signatureTypeMappings {\n\t\tif normalizedValue == currentValue {\n\t\t\treturn key, nil\n\t\t}\n\t}\n\treturn -1, errors.New(\"invalid signature type: \" + valueToMap)\n}\n\nfunc (t SignatureType) String() string {\n\treturn signatureTypeMappings[t]\n}\n\n// SignatureTypeHolder is used to hold internal type of the signature\ntype SignatureTypeHolder struct {\n\tValue SignatureType\n}\n\nfunc (holder SignatureTypeHolder) JSONSchema() *jsonschema.Schema {\n\tgotType := &jsonschema.Schema{\n\t\tType:        \"string\",\n\t\tTitle:       \"type of the signature\",\n\t\tDescription: \"Type of the signature\",\n\t}\n\tfor _, types := range GetSupportedSignaturesTypes() {\n\t\tgotType.Enum = append(gotType.Enum, types.String())\n\t}\n\treturn gotType\n}\n\nfunc (holder *SignatureTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {\n\tvar marshalledTypes string\n\tif err := unmarshal(&marshalledTypes); err != nil {\n\t\treturn err\n\t}\n\n\tcomputedType, err := toSignatureType(marshalledTypes)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tholder.Value = computedType\n\treturn nil\n}\n\nfunc (holder *SignatureTypeHolder) UnmarshalJSON(data []byte) error {\n\ts := strings.Trim(string(data), `\"`)\n\tif s == \"\" {\n\t\treturn nil\n\t}\n\tcomputedType, err := toSignatureType(s)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tholder.Value = computedType\n\treturn nil\n}\n\nfunc (holder SignatureTypeHolder) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(holder.Value.String())\n}\n\nfunc (holder SignatureTypeHolder) MarshalYAML() (interface{}, error) {\n\treturn holder.Value.String(), nil\n}\n\nvar ErrNoIgnoreList = errors.New(\"unknown signature types\")\n\n// GetVariablesNamesSkipList depending on the signature type\n\nfunc GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} {\n\tswitch signature {\n\tcase AWSSignature:\n\t\treturn signer.AwsSkipList\n\tdefault:\n\t\treturn nil\n\t}\n}\n\n// GetDefaultSignerVars returns the default signer variables\nfunc GetDefaultSignerVars(signatureType SignatureType) map[string]interface{} {\n\tif signatureType == AWSSignature {\n\t\treturn signer.AwsDefaultVars\n\t}\n\treturn map[string]interface{}{}\n}\n\n// Path: pkg/protocols/http/signer/aws-sign.go\npackage signer\n\nimport (\n\t\"context\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tv4 \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\tawsconfig \"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n\t\"github.com/projectdiscovery/gologger\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n)\n\nconst defaultEmptyPayloadHash = \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\"\n\n// AWSOptions\ntype AWSOptions struct {\n\tAwsID          string\n\tAwsSecretToken string\n\tService        string\n\tRegion         string\n}\n\n// Validate Signature Arguments\nfunc (a *AWSOptions) Validate() error {\n\tif a.Service == \"\" {\n\t\treturn errors.New(\"aws service cannot be empty\")\n\t}\n\tif a.Region == \"\" {\n\t\treturn errors.New(\"aws region cannot be empty\")\n\t}\n\n\treturn nil\n}\n\n// AWS v4 signer\ntype AWSSigner struct {\n\tcreds   *aws.Credentials\n\tsigner  *v4.Signer\n\toptions *AWSOptions\n}\n\n// SignHTTP\nfunc (a *AWSSigner) SignHTTP(ctx context.Context, request *http.Request) error {\n\tif region, ok := ctx.Value(SignerArg(\"region\")).(string); ok && region != \"\" {\n\t\ta.options.Region = region\n\t}\n\tif service, ok := ctx.Value(SignerArg(\"service\")).(string); ok && service != \"\" {\n\t\ta.options.Service = service\n\t}\n\tif err := a.options.Validate(); err != nil {\n\t\treturn err\n\t}\n\t// contentHash is sha256 hash of response body\n\tcontentHash := a.getPayloadHash(request)\n\tif err := a.signer.SignHTTP(ctx, *a.creds, request, contentHash, a.options.Service, a.options.Region, time.Now()); err != nil {\n\t\treturn errorutil.NewWithErr(err).Msgf(\"failed to sign http request using aws v4 signer\")\n\t}\n\t// add x-amz-content-sha256 header to request\n\trequest.Header.Set(\"x-amz-content-sha256\", contentHash)\n\treturn nil\n}\n\n// getPayloadHash returns hex encoded SHA-256 of request body\nfunc (a *AWSSigner) getPayloadHash(request *http.Request) string {\n\tif request.Body == nil {\n\t\t// Default Hash of Empty Payload\n\t\treturn defaultEmptyPayloadHash\n\t}\n\n\t// no need to close request body since it is a reusablereadercloser\n\tbin, err := io.ReadAll(request.Body)\n\tif err != nil {\n\t\tgologger.Error().Msgf(\"aws signer: failed to read request body: %s\", err)\n\t}\n\tsha256Hash := sha256.Sum256(bin)\n\treturn hex.EncodeToString(sha256Hash[:])\n}\n\n// NewAwsSigner\nfunc NewAwsSigner(opts *AWSOptions) (*AWSSigner, error) {\n\tcredcache := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(opts.AwsID, opts.AwsSecretToken, \"\"))\n\tawscred, err := credcache.Retrieve(context.TODO())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &AWSSigner{\n\t\tcreds:   &awscred,\n\t\toptions: opts,\n\t\tsigner:  v4.NewSigner(),\n\t}, nil\n}\n\n// NewAwsSignerFromConfig\nfunc NewAwsSignerFromConfig(opts *AWSOptions) (*AWSSigner, error) {\n\t/*\n\t\tNewAwsSignerFromConfig fetches credentials from both\n\t\t1. Environment Variables (old & new)\n\t\t2. Shared Credentials ($HOME/.aws)\n\t*/\n\tcfg, err := awsconfig.LoadDefaultConfig(context.TODO())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcredcache := aws.NewCredentialsCache(cfg.Credentials)\n\tawscred, err := credcache.Retrieve(context.TODO())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &AWSSigner{\n\t\tcreds:   &awscred,\n\t\toptions: opts,\n\t\tsigner: v4.NewSigner(func(signer *v4.SignerOptions) {\n\t\t\t// signer.DisableURIPathEscaping = true\n\t\t}),\n\t}, nil\n}\n\nvar AwsSkipList = map[string]interface{}{\n\t\"region\": struct{}{},\n}\n\nvar AwsDefaultVars = map[string]interface{}{\n\t\"region\":  \"us-east-2\",\n\t\"service\": \"sts\",\n}\n\nvar AwsInternalOnlyVars = map[string]interface{}{\n\t\"aws-id\":     struct{}{},\n\t\"aws-secret\": struct{}{},\n}\n\n// Path: pkg/protocols/javascript/js.go\npackage javascript\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/alecthomas/chroma/quick\"\n\t\"github.com/ditashi/jsbeautifier-go/jsbeautifier\"\n\t\"github.com/dop251/goja\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n\tiputil \"github.com/projectdiscovery/utils/ip\"\n\tsyncutil \"github.com/projectdiscovery/utils/sync\"\n\turlutil \"github.com/projectdiscovery/utils/url\"\n)\n\n// Request is a request for the javascript protocol\ntype Request struct {\n\t// Operators for the current request go here.\n\toperators.Operators `yaml:\",inline,omitempty\" json:\",inline,omitempty\"`\n\tCompiledOperators   *operators.Operators `yaml:\"-\" json:\"-\"`\n\n\t// description: |\n\t// ID is request id in that protocol\n\tID string `yaml:\"id,omitempty\" json:\"id,omitempty\" jsonschema:\"title=id of the request,description=ID is the optional ID of the Request\"`\n\n\t// description: |\n\t//  Init is javascript code to execute after compiling template and before executing it on any target\n\t//  This is helpful for preparing payloads or other setup that maybe required for exploits\n\tInit string `yaml:\"init,omitempty\" json:\"init,omitempty\" jsonschema:\"title=init javascript code,description=Init is the javascript code to execute after compiling template\"`\n\n\t// description: |\n\t//   PreCondition is a condition which is evaluated before sending the request.\n\tPreCondition string `yaml:\"pre-condition,omitempty\" json:\"pre-condition,omitempty\" jsonschema:\"title=pre-condition for the request,description=PreCondition is a condition which is evaluated before sending the request\"`\n\n\t// description: |\n\t//   Args contains the arguments to pass to the javascript code.\n\tArgs map[string]interface{} `yaml:\"args,omitempty\" json:\"args,omitempty\"`\n\t// description: |\n\t//   Code contains code to execute for the javascript request.\n\tCode string `yaml:\"code,omitempty\" json:\"code,omitempty\" jsonschema:\"title=code to execute in javascript,description=Executes inline javascript code for the request\"`\n\t// description: |\n\t//   Timeout in seconds is optional timeout for each  javascript script execution (i.e init, pre-condition, code)\n\tTimeout int `yaml:\"timeout,omitempty\" json:\"timeout,omitempty\" jsonschema:\"title=timeout for javascript execution,description=Timeout in seconds is optional timeout for entire javascript script execution\"`\n\t// description: |\n\t//   StopAtFirstMatch stops processing the request at first match.\n\tStopAtFirstMatch bool `yaml:\"stop-at-first-match,omitempty\" json:\"stop-at-first-match,omitempty\" jsonschema:\"title=stop at first match,description=Stop the execution after a match is found\"`\n\t// description: |\n\t//   Attack is the type of payload combinations to perform.\n\t//\n\t//   Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\n\t//   permutations and combinations for all payloads.\n\tAttackType generators.AttackTypeHolder `yaml:\"attack,omitempty\" json:\"attack,omitempty\" jsonschema:\"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb\"`\n\t// description: |\n\t//   Payload concurreny i.e threads for sending requests.\n\t// examples:\n\t//   - name: Send requests using 10 concurrent threads\n\t//     value: 10\n\tThreads int `yaml:\"threads,omitempty\" json:\"threads,omitempty\" jsonschema:\"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling\"`\n\t// description: |\n\t//   Payloads contains any payloads for the current request.\n\t//\n\t//   Payloads support both key-values combinations where a list\n\t//   of payloads is provided, or optionally a single file can also\n\t//   be provided as payload which will be read on run-time.\n\tPayloads map[string]interface{} `yaml:\"payloads,omitempty\" json:\"payloads,omitempty\" jsonschema:\"title=payloads for the webosocket request,description=Payloads contains any payloads for the current request\"`\n\n\tgenerator *generators.PayloadGenerator\n\n\t// cache any variables that may be needed for operation.\n\toptions *protocols.ExecutorOptions `yaml:\"-\" json:\"-\"`\n\n\tpreConditionCompiled *goja.Program `yaml:\"-\" json:\"-\"`\n\n\tscriptCompiled *goja.Program `yaml:\"-\" json:\"-\"`\n}\n\n// Compile compiles the request generators preparing any requests possible.\nfunc (request *Request) Compile(options *protocols.ExecutorOptions) error {\n\trequest.options = options\n\n\tvar err error\n\tif len(request.Payloads) > 0 {\n\t\trequest.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, options.Options)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"could not parse payloads\")\n\t\t}\n\t\t// default to 20 threads for payload requests\n\t\trequest.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)\n\t}\n\n\tif len(request.Matchers) > 0 || len(request.Extractors) > 0 {\n\t\tcompiled := &request.Operators\n\t\tcompiled.ExcludeMatchers = options.ExcludeMatchers\n\t\tcompiled.TemplateID = options.TemplateID\n\t\tfor _, matcher := range compiled.Matchers {\n\t\t\tif matcher.Part == \"\" && matcher.Type.MatcherType != matchers.DSLMatcher {\n\t\t\t\tmatcher.Part = \"response\"\n\t\t\t}\n\t\t}\n\t\tfor _, extractor := range compiled.Extractors {\n\t\t\tif extractor.Part == \"\" {\n\t\t\t\textractor.Part = \"response\"\n\t\t\t}\n\t\t}\n\t\tif err := compiled.Compile(); err != nil {\n\t\t\treturn errorutil.NewWithTag(request.TemplateID, \"could not compile operators got %v\", err)\n\t\t}\n\t\trequest.CompiledOperators = compiled\n\t}\n\n\t// \"Port\" is a special variable and it should not contains any dsl expressions\n\tif strings.Contains(request.getPort(), \"{{\") {\n\t\treturn errorutil.NewWithTag(request.TemplateID, \"'Port' variable cannot contain any dsl expressions\")\n\t}\n\n\tif request.Init != \"\" {\n\t\t// execute init code if any\n\t\tif request.options.Options.Debug || request.options.Options.DebugRequests {\n\t\t\tgologger.Debug().Msgf(\"[%s] Executing Template Init\\n\", request.TemplateID)\n\t\t\tvar highlightFormatter = \"terminal256\"\n\t\t\tif request.options.Options.NoColor {\n\t\t\t\thighlightFormatter = \"text\"\n\t\t\t}\n\t\t\tvar buff bytes.Buffer\n\t\t\t_ = quick.Highlight(&buff, beautifyJavascript(request.Init), \"javascript\", highlightFormatter, \"monokai\")\n\t\t\tprettyPrint(request.TemplateID, buff.String())\n\t\t}\n\n\t\topts := &compiler.ExecuteOptions{\n\t\t\tTimeout: request.Timeout,\n\t\t\tSource:  &request.Init,\n\t\t}\n\t\t// register 'export' function to export variables from init code\n\t\t// these are saved in args and are available in pre-condition and request code\n\t\topts.Callback = func(runtime *goja.Runtime) error {\n\t\t\terr := gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\t\t\tName: \"set\",\n\t\t\t\tSignatures: []string{\n\t\t\t\t\t\"set(string, interface{})\",\n\t\t\t\t},\n\t\t\t\tDescription: \"set variable from init code. this function is available in init code block only\",\n\t\t\t\tFuncDecl: func(varname string, value any) error {\n\t\t\t\t\tif varname == \"\" {\n\t\t\t\t\t\treturn fmt.Errorf(\"variable name cannot be empty\")\n\t\t\t\t\t}\n\t\t\t\t\tif value == nil {\n\t\t\t\t\t\treturn fmt.Errorf(\"variable value cannot be empty\")\n\t\t\t\t\t}\n\t\t\t\t\tif request.Args == nil {\n\t\t\t\t\t\trequest.Args = make(map[string]interface{})\n\t\t\t\t\t}\n\t\t\t\t\trequest.Args[varname] = value\n\t\t\t\t\treturn nil\n\t\t\t\t},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\treturn gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\t\t\tName: \"updatePayload\",\n\t\t\t\tSignatures: []string{\n\t\t\t\t\t\"updatePayload(string, interface{})\",\n\t\t\t\t},\n\t\t\t\tDescription: \"update/override any payload from init code. this function is available in init code block only\",\n\t\t\t\tFuncDecl: func(varname string, Value any) error {\n\t\t\t\t\tif request.Payloads == nil {\n\t\t\t\t\t\trequest.Payloads = make(map[string]interface{})\n\t\t\t\t\t}\n\t\t\t\t\tif request.generator != nil {\n\t\t\t\t\t\trequest.Payloads[varname] = Value\n\t\t\t\t\t\trequest.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, options.Options)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn fmt.Errorf(\"payloads not defined and cannot be updated\")\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t},\n\t\t\t})\n\t\t}\n\t\topts.Cleanup = func(runtime *goja.Runtime) {\n\t\t\t_ = runtime.GlobalObject().Delete(\"set\")\n\t\t\t_ = runtime.GlobalObject().Delete(\"updatePayload\")\n\t\t}\n\n\t\targs := compiler.NewExecuteArgs()\n\t\tallVars := generators.MergeMaps(options.Variables.GetAll(), options.Options.Vars.AsMap(), request.options.Constants)\n\t\t// proceed with whatever args we have\n\t\targs.Args, _ = request.evaluateArgs(allVars, options, true)\n\n\t\tinitCompiled, err := compiler.WrapScriptNCompile(request.Init, false)\n\t\tif err != nil {\n\t\t\treturn errorutil.NewWithTag(request.TemplateID, \"could not compile init code: %s\", err)\n\t\t}\n\t\tresult, err := request.options.JsCompiler.ExecuteWithOptions(initCompiled, args, opts)\n\t\tif err != nil {\n\t\t\treturn errorutil.NewWithTag(request.TemplateID, \"could not execute pre-condition: %s\", err)\n\t\t}\n\t\tif types.ToString(result[\"error\"]) != \"\" {\n\t\t\tgologger.Warning().Msgf(\"[%s] Init failed with error %v\\n\", request.TemplateID, result[\"error\"])\n\t\t\treturn nil\n\t\t} else {\n\t\t\tif request.options.Options.Debug || request.options.Options.DebugResponse {\n\t\t\t\tgologger.Debug().Msgf(\"[%s] Init executed successfully\\n\", request.TemplateID)\n\t\t\t\tgologger.Debug().Msgf(\"[%s] Init result: %v\\n\", request.TemplateID, result[\"response\"])\n\t\t\t}\n\t\t}\n\t}\n\n\t// compile pre-condition if any\n\tif request.PreCondition != \"\" {\n\t\tpreConditionCompiled, err := compiler.WrapScriptNCompile(request.PreCondition, false)\n\t\tif err != nil {\n\t\t\treturn errorutil.NewWithTag(request.TemplateID, \"could not compile pre-condition: %s\", err)\n\t\t}\n\t\trequest.preConditionCompiled = preConditionCompiled\n\t}\n\n\t// compile actual source code\n\tif request.Code != \"\" {\n\t\tscriptCompiled, err := compiler.WrapScriptNCompile(request.Code, false)\n\t\tif err != nil {\n\t\t\treturn errorutil.NewWithTag(request.TemplateID, \"could not compile javascript code: %s\", err)\n\t\t}\n\t\trequest.scriptCompiled = scriptCompiled\n\t}\n\n\treturn nil\n}\n\n// Options returns executer options for http request\nfunc (r *Request) Options() *protocols.ExecutorOptions {\n\treturn r.options\n}\n\n// Requests returns the total number of requests the rule will perform\nfunc (request *Request) Requests() int {\n\tpre_conditions := 0\n...\n", "needle_token_start": 9896, "needle_token_end": 9958, "code_context_ntokens": 15299, "output": ["```go\n// GetVariablesNamesSkipList depending on the signature type\nfunc GetVariablesNamesSkipList(signature SignatureType) map[string]interface{} {\n\tswitch signature {\n\tcase AWSSignature:\n\t\treturn signer.AwsSkipList\n\tdefault:\n\t\treturn nil\n\t}\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "NewExecuteArgs", "language": "go", "path": "pkg/js/compiler/compiler.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to initialize and return a structure that holds two specific types of data maps used for passing arguments to a script.\n2. **Input**: There are no inputs required for this function.\n3. **Output**: The output is a newly created structure containing two empty maps intended for storing script execution arguments and template context variables.\n4. **Procedure**: The function creates a new instance of a structure, initializes two maps within this structure to hold data, and then returns this initialized structure.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/protocols/common/expressions/expressions.go\npackage expressions\n\nimport (\n\t\"strings\"\n\n\t\"github.com/Knetic/govaluate\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\n// Eval compiles the given expression and evaluate it with the given values preserving the return type\nfunc Eval(expression string, values map[string]interface{}) (interface{}, error) {\n\tcompiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn compiled.Evaluate(values)\n}\n\n// Evaluate checks if the match contains a dynamic variable, for each\n// found one we will check if it's an expression and can\n// be compiled, it will be evaluated and the results will be returned.\n//\n// The provided keys from finalValues will be used as variable names\n// for substitution inside the expression.\nfunc Evaluate(data string, base map[string]interface{}) (string, error) {\n\treturn evaluate(data, base)\n}\n\n// EvaluateByte checks if the match contains a dynamic variable, for each\n// found one we will check if it's an expression and can\n// be compiled, it will be evaluated and the results will be returned.\n//\n// The provided keys from finalValues will be used as variable names\n// for substitution inside the expression.\nfunc EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) {\n\tfinalData, err := evaluate(string(data), base)\n\treturn []byte(finalData), err\n}\n\nfunc evaluate(data string, base map[string]interface{}) (string, error) {\n\t// replace simple placeholders (key => value) MarkerOpen + key + MarkerClose and General + key + General to value\n\tdata = replacer.Replace(data, base)\n\n\t// expressions can be:\n\t// - simple: containing base values keys (variables)\n\t// - complex: containing helper functions [ + variables]\n\t// literals like {{2+2}} are not considered expressions\n\texpressions := FindExpressions(data, marker.ParenthesisOpen, marker.ParenthesisClose, base)\n\tfor _, expression := range expressions {\n\t\t// replace variable placeholders with base values\n\t\texpression = replacer.Replace(expression, base)\n\t\t// turns expressions (either helper functions+base values or base values)\n\t\tcompiled, err := govaluate.NewEvaluableExpressionWithFunctions(expression, dsl.HelperFunctions)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tresult, err := compiled.Evaluate(base)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\t// replace incrementally\n\t\tdata = replacer.ReplaceOne(data, expression, result)\n\t}\n\treturn data, nil\n}\n\n// maxIterations to avoid infinite loop\nconst maxIterations = 250\n\nfunc FindExpressions(data, OpenMarker, CloseMarker string, base map[string]interface{}) []string {\n\tvar (\n\t\titerations int\n\t\texps       []string\n\t)\n\tfor {\n\t\t// check if we reached the maximum number of iterations\n\t\tif iterations > maxIterations {\n\t\t\tbreak\n\t\t}\n\t\titerations++\n\t\t// attempt to find open markers\n\t\tindexOpenMarker := strings.Index(data, OpenMarker)\n\t\t// exits if not found\n\t\tif indexOpenMarker < 0 {\n\t\t\tbreak\n\t\t}\n\n\t\tindexOpenMarkerOffset := indexOpenMarker + len(OpenMarker)\n\n\t\tshouldSearchCloseMarker := true\n\t\tcloseMarkerFound := false\n\t\tinnerData := data\n\t\tvar potentialMatch string\n\t\tvar indexCloseMarker, indexCloseMarkerOffset int\n\t\tskip := indexOpenMarkerOffset\n\t\tfor shouldSearchCloseMarker {\n\t\t\t// attempt to find close marker\n\t\t\tindexCloseMarker = stringsutil.IndexAt(innerData, CloseMarker, skip)\n\t\t\t// if no close markers are found exit\n\t\t\tif indexCloseMarker < 0 {\n\t\t\t\tshouldSearchCloseMarker = false\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tindexCloseMarkerOffset = indexCloseMarker + len(CloseMarker)\n\n\t\t\tpotentialMatch = innerData[indexOpenMarkerOffset:indexCloseMarker]\n\t\t\tif isExpression(potentialMatch, base) {\n\t\t\t\tcloseMarkerFound = true\n\t\t\t\tshouldSearchCloseMarker = false\n\t\t\t\texps = append(exps, potentialMatch)\n\t\t\t} else {\n\t\t\t\tskip = indexCloseMarkerOffset\n\t\t\t}\n\t\t}\n\n\t\tif closeMarkerFound {\n\t\t\t// move after the close marker\n\t\t\tdata = data[indexCloseMarkerOffset:]\n\t\t} else {\n\t\t\t// move after the open marker\n\t\t\tdata = data[indexOpenMarkerOffset:]\n\t\t}\n\t}\n\treturn exps\n}\n\nfunc isExpression(data string, base map[string]interface{}) bool {\n\tif _, err := govaluate.NewEvaluableExpression(data); err == nil {\n\t\tif stringsutil.ContainsAny(data, getFunctionsNames(base)...) {\n\t\t\treturn true\n\t\t} else if stringsutil.ContainsAny(data, dsl.FunctionNames...) {\n\t\t\treturn true\n\t\t}\n\t\treturn false\n...\n// Path: pkg/protocols/websocket/websocket.go\npackage websocket\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"path\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/gobwas/ws\"\n\t\"github.com/gobwas/ws/wsutil\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/fastdialer/fastdialer\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\turlutil \"github.com/projectdiscovery/utils/url\"\n)\n\n// Request is a request for the Websocket protocol\ntype Request struct {\n\t// Operators for the current request go here.\n\toperators.Operators `yaml:\",inline,omitempty\" json:\",inline,omitempty\"`\n\tCompiledOperators   *operators.Operators `yaml:\"-\" json:\"-\"`\n\n\t// ID is the optional id of the request\n\tID string `yaml:\"id,omitempty\" json:\"id,omitempty\" jsonschema:\"title=id of the request,description=ID of the network request\"`\n\t// description: |\n\t//   Address contains address for the request\n\tAddress string `yaml:\"address,omitempty\" json:\"address,omitempty\" jsonschema:\"title=address for the websocket request,description=Address contains address for the request\"`\n\t// description: |\n\t//   Inputs contains inputs for the websocket protocol\n\tInputs []*Input `yaml:\"inputs,omitempty\" json:\"inputs,omitempty\" jsonschema:\"title=inputs for the websocket request,description=Inputs contains any input/output for the current request\"`\n\t// description: |\n\t//   Headers contains headers for the request.\n\tHeaders map[string]string `yaml:\"headers,omitempty\" json:\"headers,omitempty\" jsonschema:\"title=headers contains the request headers,description=Headers contains headers for the request\"`\n\n\t// description: |\n\t//   Attack is the type of payload combinations to perform.\n\t//\n\t//   Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\n\t//   permutations and combinations for all payloads.\n\tAttackType generators.AttackTypeHolder `yaml:\"attack,omitempty\" json:\"attack,omitempty\" jsonschema:\"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb\"`\n\t// description: |\n\t//   Payloads contains any payloads for the current request.\n\t//\n\t//   Payloads support both key-values combinations where a list\n\t//   of payloads is provided, or optionally a single file can also\n\t//   be provided as payload which will be read on run-time.\n\tPayloads map[string]interface{} `yaml:\"payloads,omitempty\" json:\"payloads,omitempty\" jsonschema:\"title=payloads for the websocket request,description=Payloads contains any payloads for the current request\"`\n\n\tgenerator *generators.PayloadGenerator\n\n\t// cache any variables that may be needed for operation.\n\tdialer  *fastdialer.Dialer\n\toptions *protocols.ExecutorOptions\n}\n\n// Input is an input for the websocket protocol\ntype Input struct {\n\t// description: |\n\t//   Data is the data to send as the input.\n\t//\n\t//   It supports DSL Helper Functions as well as normal expressions.\n\t// examples:\n\t//   - value: \"\\\"TEST\\\"\"\n\t//   - value: \"\\\"hex_decode('50494e47')\\\"\"\n\tData string `yaml:\"data,omitempty\" json:\"data,omitempty\" jsonschema:\"title=data to send as input,description=Data is the data to send as the input\"`\n\t// description: |\n\t//   Name is the optional name of the data read to provide matching on.\n\t// examples:\n\t//   - value: \"\\\"prefix\\\"\"\n\tName string `yaml:\"name,omitempty\" json:\"name,omitempty\" jsonschema:\"title=optional name for data read,description=Optional name of the data read to provide matching on\"`\n}\n\nconst (\n\tparseUrlErrorMessage                   = \"could not parse input url\"\n\tevaluateTemplateExpressionErrorMessage = \"could not evaluate template expressions\"\n)\n\n// Compile compiles the request generators preparing any requests possible.\nfunc (request *Request) Compile(options *protocols.ExecutorOptions) error {\n\trequest.options = options\n\n\tclient, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"could not get network client\")\n\t}\n\trequest.dialer = client\n\n\tif len(request.Payloads) > 0 {\n\t\trequest.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, types.DefaultOptions())\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"could not parse payloads\")\n\t\t}\n\t}\n\n\tif len(request.Matchers) > 0 || len(request.Extractors) > 0 {\n\t\tcompiled := &request.Operators\n\t\tcompiled.ExcludeMatchers = options.ExcludeMatchers\n\t\tcompiled.TemplateID = options.TemplateID\n\t\tif err := compiled.Compile(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile operators\")\n\t\t}\n\t\trequest.CompiledOperators = compiled\n\t}\n\treturn nil\n}\n\n// Requests returns the total number of requests the rule will perform\nfunc (request *Request) Requests() int {\n\tif request.generator != nil {\n\t\treturn request.generator.NewIterator().Total()\n\t}\n\treturn 1\n}\n\n// GetID returns the ID for the request if any.\nfunc (request *Request) GetID() string {\n\treturn \"\"\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\thostname, err := getAddress(input.MetaInput.Input)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif request.generator != nil {\n\t\titerator := request.generator.NewIterator()\n\n\t\tfor {\n\t\t\tvalue, ok := iterator.Value()\n\t\t\tif !ok {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t} else {\n\t\tvalue := make(map[string]interface{})\n\t\tif err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) executeRequestWithPayloads(target *contextargs.Context, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\theader := http.Header{}\n\tinput := target.MetaInput.Input\n\n\tparsed, err := urlutil.Parse(input)\n\tif err != nil {\n\t\treturn errors.Wrap(err, parseUrlErrorMessage)\n\t}\n\tdefaultVars := protocolutils.GenerateVariables(parsed, false, nil)\n\toptionVars := generators.BuildPayloadFromOptions(request.options.Options)\n\t// add templatecontext variables to varMap\n\tvariables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(target.MetaInput).GetAll()))\n\tpayloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants)\n\n\trequestOptions := request.options\n\tfor key, value := range request.Headers {\n\t\tfinalData, dataErr := expressions.EvaluateByte([]byte(value), payloadValues)\n\t\tif dataErr != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)\n\t\t}\n\t\theader.Set(key, string(finalData))\n\t}\n\ttlsConfig := &tls.Config{\n\t\tInsecureSkipVerify: true,\n\t\tServerName:         hostname,\n\t\tMinVersion:         tls.VersionTLS10,\n\t}\n\tif requestOptions.Options.SNI != \"\" {\n\t\ttlsConfig.ServerName = requestOptions.Options.SNI\n\t}\n\twebsocketDialer := ws.Dialer{\n\t\tHeader:    ws.HandshakeHeaderHTTP(header),\n\t\tTimeout:   time.Duration(requestOptions.Options.Timeout) * time.Second,\n\t\tNetDial:   request.dialer.Dial,\n\t\tTLSConfig: tlsConfig,\n\t}\n\n\tif vardump.EnableVarDump {\n\t\tgologger.Debug().Msgf(\"Websocket Protocol request variables: \\n%s\\n\", vardump.DumpVariables(payloadValues))\n\t}\n\n\tfinalAddress, dataErr := expressions.EvaluateByte([]byte(request.Address), payloadValues)\n\tif dataErr != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)\n\t}\n\n\taddressToDial := string(finalAddress)\n\tparsedAddress, err := url.Parse(addressToDial)\n\tif err != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, parseUrlErrorMessage)\n\t}\n\tparsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path)\n\taddressToDial = parsedAddress.String()\n\n\tconn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial)\n\tif err != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, \"could not connect to server\")\n\t}\n\tdefer conn.Close()\n\n\tresponseBuilder := &strings.Builder{}\n\tif readBuffer != nil {\n\t\t_, _ = io.Copy(responseBuilder, readBuffer) // Copy initial response\n\t}\n\n\tevents, requestOutput, err := request.readWriteInputWebsocket(conn, payloadValues, input, responseBuilder)\n\tif err != nil {\n\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, \"could not read write response\")\n\t}\n\trequestOptions.Progress.IncrementRequests()\n\n\tif requestOptions.Options.Debug || requestOptions.Options.DebugRequests {\n\t\tgologger.Debug().Str(\"address\", input).Msgf(\"[%s] Dumped Websocket request for %s\", requestOptions.TemplateID, input)\n\t\tgologger.Print().Msgf(\"%s\", requestOutput)\n\t}\n\n\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\tgologger.Verbose().Msgf(\"Sent Websocket request to %s\", input)\n\n\tdata := make(map[string]interface{})\n\n\tdata[\"type\"] = request.Type().String()\n\tdata[\"success\"] = \"true\"\n\tdata[\"request\"] = requestOutput\n\tdata[\"response\"] = responseBuilder.String()\n\tdata[\"host\"] = input\n\tdata[\"matched\"] = addressToDial\n\tdata[\"ip\"] = request.dialer.GetDialedIP(hostname)\n\n\t// add response fields to template context and merge templatectx variables to output event\n\trequest.options.AddTemplateVars(target.MetaInput, request.Type(), request.ID, data)\n\tdata = generators.MergeMaps(data, request.options.GetTemplateCtx(target.MetaInput).GetAll())\n\n\tfor k, v := range previous {\n\t\tdata[k] = v\n\t}\n\tfor k, v := range events {\n\t\tdata[k] = v\n\t}\n\n\tevent := eventcreator.CreateEventWithAdditionalOptions(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {\n\t\tinternalWrappedEvent.OperatorsResult.PayloadValues = payloadValues\n\t})\n\tif requestOptions.Options.Debug || requestOptions.Options.DebugResponse {\n\t\tresponseOutput := responseBuilder.String()\n\t\tgologger.Debug().Msgf(\"[%s] Dumped Websocket response for %s\", requestOptions.TemplateID, input)\n\t\tgologger.Print().Msgf(\"%s\", responsehighlighter.Highlight(event.OperatorsResult, responseOutput, requestOptions.Options.NoColor, false))\n\t}\n\n\tcallback(event)\n\treturn nil\n}\n\nfunc (request *Request) readWriteInputWebsocket(conn net.Conn, payloadValues map[string]interface{}, input string, respBuilder *strings.Builder) (events map[string]interface{}, req string, err error) {\n\treqBuilder := &strings.Builder{}\n\tinputEvents := make(map[string]interface{})\n\n\trequestOptions := request.options\n\tfor _, req := range request.Inputs {\n\t\treqBuilder.Grow(len(req.Data))\n\n\t\tfinalData, dataErr := expressions.EvaluateByte([]byte(req.Data), payloadValues)\n\t\tif dataErr != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), dataErr)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn nil, \"\", errors.Wrap(dataErr, evaluateTemplateExpressionErrorMessage)\n\t\t}\n\t\treqBuilder.WriteString(string(finalData))\n\n\t\terr = wsutil.WriteClientMessage(conn, ws.OpText, finalData)\n\t\tif err != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn nil, \"\", errors.Wrap(err, \"could not write request to server\")\n\t\t}\n\n\t\tmsg, opCode, err := wsutil.ReadServerData(conn)\n\t\tif err != nil {\n\t\t\trequestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)\n\t\t\trequestOptions.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn nil, \"\", errors.Wrap(err, \"could not write request to server\")\n\t\t}\n\t\t// Only perform matching and writes in case we receive\n\t\t// text or binary opcode from the websocket server.\n\t\tif opCode != ws.OpText && opCode != ws.OpBinary {\n\t\t\tcontinue\n\t\t}\n\n\t\trespBuilder.Write(msg)\n\t\tif req.Name != \"\" {\n\t\t\tbufferStr := string(msg)\n\t\t\tinputEvents[req.Name] = bufferStr\n\n\t\t\t// Run any internal extractors for the request here and add found values to map.\n\t\t\tif request.CompiledOperators != nil {\n\t\t\t\tvalues := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{req.Name: bufferStr}, protocols.MakeDefaultExtractFunc)\n\t\t\t\tfor k, v := range values {\n\t\t\t\t\tinputEvents[k] = v\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn inputEvents, reqBuilder.String(), nil\n}\n\n// getAddress returns the address of the host to make request to\nfunc getAddress(toTest string) (string, error) {\n\tparsed, err := url.Parse(toTest)\n\tif err != nil {\n\t\treturn \"\", errors.Wrap(err, parseUrlErrorMessage)\n\t}\n\tscheme := strings.ToLower(parsed.Scheme)\n\n\tif scheme != \"ws\" && scheme != \"wss\" {\n\t\treturn \"\", fmt.Errorf(\"invalid url scheme provided: %s\", scheme)\n\t}\n\tif parsed != nil && parsed.Host != \"\" {\n\t\treturn parsed.Host, nil\n\t}\n\treturn \"\", nil\n}\n\n// Match performs matching operation for a matcher on model and returns:\n// true and a list of matched snippets if the matcher type is supports it\n// otherwise false and an empty string slice\nfunc (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {\n\treturn protocols.MakeDefaultMatchFunc(data, matcher)\n}\n\n// Extract performs extracting operation for an extractor on model and returns true or false.\nfunc (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {\n\treturn protocols.MakeDefaultExtractFunc(data, matcher)\n}\n\n// MakeResultEvent creates a result event from internal wrapped event\nfunc (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {\n\treturn protocols.MakeDefaultResultEvent(request, wrapped)\n}\n\n// GetCompiledOperators returns a list of the compiled operators\nfunc (request *Request) GetCompiledOperators() []*operators.Operators {\n\treturn []*operators.Operators{request.CompiledOperators}\n}\n\n// RequestPartDefinitions contains a mapping of request part definitions and their\n// description. Multiple definitions are separated by commas.\n// Definitions not having a name (generated on runtime) are prefixed & suffixed by <>.\nvar RequestPartDefinitions = map[string]string{\n\t\"type\":     \"Type is the type of request made\",\n\t\"success\":  \"Success specifies whether websocket connection was successful\",\n\t\"request\":  \"Websocket request made to the server\",\n\t\"response\": \"Websocket response received from the server\",\n\t\"host\":     \"Host is the input to the template\",\n\t\"matched\":  \"Matched is the input which was matched upon\",\n}\n\nfunc (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {\n\tfields := protocolutils.GetJsonFieldsFromURL(types.ToString(wrapped.InternalEvent[\"host\"]))\n\tif types.ToString(wrapped.InternalEvent[\"ip\"]) != \"\" {\n\t\tfields.Ip = types.ToString(wrapped.InternalEvent[\"ip\"])\n\t}\n\tdata := &output.ResultEvent{\n\t\tTemplateID:       types.ToString(request.options.TemplateID),\n\t\tTemplatePath:     types.ToString(request.options.TemplatePath),\n\t\tInfo:             request.options.TemplateInfo,\n\t\tType:             types.ToString(wrapped.InternalEvent[\"type\"]),\n\t\tHost:             fields.Host,\n\t\tPort:             fields.Port,\n\t\tMatched:          types.ToString(wrapped.InternalEvent[\"matched\"]),\n\t\tMetadata:         wrapped.OperatorsResult.PayloadValues,\n\t\tExtractedResults: wrapped.OperatorsResult.OutputExtracts,\n\t\tTimestamp:        time.Now(),\n\t\tMatcherStatus:    true,\n\t\tIP:               fields.Ip,\n\t\tRequest:          types.ToString(wrapped.InternalEvent[\"request\"]),\n\t\tResponse:         types.ToString(wrapped.InternalEvent[\"response\"]),\n\t\tTemplateEncoded:  request.options.EncodeTemplate(),\n\t\tError:            types.ToString(wrapped.InternalEvent[\"error\"]),\n\t}\n\treturn data\n}\n\n// Type returns the type of the protocol request\nfunc (request *Request) Type() templateTypes.ProtocolType {\n\treturn templateTypes.WebsocketProtocol\n}\n\n// Path: pkg/protocols/whois/whois.go\npackage whois\n\nimport (\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\tjsoniter \"github.com/json-iterator/go\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/rdap\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/whois/rdapclientpool\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\n// Request is a request for the WHOIS protocol\ntype Request struct {\n\t// Operators for the current request go here.\n\toperators.Operators `yaml:\",inline,omitempty\" json:\",inline,omitempty\"`\n\tCompiledOperators   *operators.Operators `yaml:\"-\" json:\"-\"`\n\n\t// ID is the optional id of the request\n\tID string `yaml:\"id,omitempty\" json:\"id,omitempty\" jsonschema:\"title=id of the request,description=ID of the network request\"`\n\n\t// description: |\n\t//   Query contains query for the request\n\tQuery string `yaml:\"query,omitempty\" json:\"query,omitempty\" jsonschema:\"title=query for the WHOIS request,description=Query contains query for the request\"`\n\n\t// description: |\n\t// \t Optional WHOIS server URL.\n\t//\n\t// \t If present, specifies the WHOIS server to execute the Request on.\n\t//   Otherwise, nil enables bootstrapping\n\tServer string `yaml:\"server,omitempty\" json:\"server,omitempty\" jsonschema:\"title=server url to execute the WHOIS request on,description=Server contains the server url to execute the WHOIS request on\"`\n\t// cache any variables that may be needed for operation.\n\tclient          *rdap.Client\n\toptions         *protocols.ExecutorOptions\n\tparsedServerURL *url.URL\n}\n\n// Compile compiles the request generators preparing any requests possible.\nfunc (request *Request) Compile(options *protocols.ExecutorOptions) error {\n\tvar err error\n\tif request.Server != \"\" {\n\t\trequest.parsedServerURL, err = url.Parse(request.Server)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"failed to parse server URL\")\n\t\t}\n\t}\n\n\trequest.options = options\n\trequest.client, _ = rdapclientpool.Get(options.Options, nil)\n\n\tif len(request.Matchers) > 0 || len(request.Extractors) > 0 {\n\t\tcompiled := &request.Operators\n\t\tcompiled.ExcludeMatchers = options.ExcludeMatchers\n\t\tcompiled.TemplateID = options.TemplateID\n\t\tif err := compiled.Compile(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile operators\")\n\t\t}\n\t\trequest.CompiledOperators = compiled\n\t}\n\treturn nil\n}\n\n// Requests returns the total number of requests the rule will perform\nfunc (request *Request) Requests() int {\n\treturn 1\n}\n\n// GetID returns the ID for the request if any.\nfunc (request *Request) GetID() string {\n\treturn \"\"\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\t// generate variables\n\tdefaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)\n\toptionVars := generators.BuildPayloadFromOptions(request.options.Options)\n\t// add templatectx variables to varMap\n\tvars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(input.MetaInput).GetAll()))\n\n\tvariables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants)\n\n\tif vardump.EnableVarDump {\n\t\tgologger.Debug().Msgf(\"Whois Protocol request variables: \\n%s\\n\", vardump.DumpVariables(variables))\n\t}\n\n\t// and replace placeholders\n\tquery := replacer.Replace(request.Query, variables)\n\t// build an rdap request\n\trdapReq := rdap.NewAutoRequest(query)\n\trdapReq.Server = request.parsedServerURL\n\tres, err := request.client.Do(rdapReq)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"could not make whois request\")\n\t}\n\tgologger.Verbose().Msgf(\"Sent WHOIS request to %s\", query)\n\tif request.options.Options.Debug || request.options.Options.DebugRequests {\n\t\tgologger.Debug().Msgf(\"[%s] Dumped WHOIS request for %s\", request.options.TemplateID, query)\n\t}\n\n\tdata := make(map[string]interface{})\n\tvar response interface{}\n\tswitch rdapReq.Type {\n\tcase rdap.DomainRequest:\n\t\t// convert the rdap response to a whois style response (for domain request type only)\n\t\twhoisResp := res.ToWhoisStyleResponse()\n\t\tfor k, v := range whoisResp.Data {\n\t\t\tdata[strings.ToLower(k)] = strings.Join(v, \",\")\n\t\t}\n\t\tresponse = whoisResp\n\tdefault:\n\t\tresponse = res.Object\n\t}\n\tjsonData, _ := jsoniter.Marshal(response)\n\tjsonDataString := string(jsonData)\n\n\tdata[\"type\"] = request.Type().String()\n\tdata[\"host\"] = query\n\tdata[\"response\"] = jsonDataString\n\n\t// add response fields to template context and merge templatectx variables to output event\n\trequest.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data)\n\tdata = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll())\n\n\tevent := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse)\n\tif request.options.Options.Debug || request.options.Options.DebugResponse {\n\t\tgologger.Debug().Msgf(\"[%s] Dumped WHOIS response for %s\", request.options.TemplateID, query)\n\t\tgologger.Print().Msgf(\"%s\", responsehighlighter.Highlight(event.OperatorsResult, jsonDataString, request.options.Options.NoColor, false))\n\t}\n\n\tcallback(event)\n\treturn nil\n}\n\n// Match performs matching operation for a matcher on model and returns:\n// true and a list of matched snippets if the matcher type is supports it\n// otherwise false and an empty string slice\nfunc (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {\n\treturn protocols.MakeDefaultMatchFunc(data, matcher)\n}\n\n// Extract performs extracting operation for an extractor on model and returns true or false.\nfunc (request *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} {\n\treturn protocols.MakeDefaultExtractFunc(data, matcher)\n}\n\n// MakeResultEvent creates a result event from internal wrapped event\nfunc (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {\n\treturn protocols.MakeDefaultResultEvent(request, wrapped)\n}\n\n// GetCompiledOperators returns a list of the compiled operators\nfunc (request *Request) GetCompiledOperators() []*operators.Operators {\n\treturn []*operators.Operators{request.CompiledOperators}\n}\n\nfunc (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {\n\tdata := &output.ResultEvent{\n\t\tTemplateID:       types.ToString(request.options.TemplateID),\n\t\tTemplatePath:     types.ToString(request.options.TemplatePath),\n\t\tInfo:             request.options.TemplateInfo,\n\t\tType:             types.ToString(wrapped.InternalEvent[\"type\"]),\n\t\tHost:             types.ToString(wrapped.InternalEvent[\"host\"]),\n\t\tMetadata:         wrapped.OperatorsResult.PayloadValues,\n\t\tExtractedResults: wrapped.OperatorsResult.OutputExtracts,\n\t\tTimestamp:        time.Now(),\n\t\tMatcherStatus:    true,\n\t\tRequest:          types.ToString(wrapped.InternalEvent[\"request\"]),\n\t\tResponse:         types.ToString(wrapped.InternalEvent[\"response\"]),\n\t\tTemplateEncoded:  request.options.EncodeTemplate(),\n\t\tError:            types.ToString(wrapped.InternalEvent[\"error\"]),\n\t}\n\treturn data\n}\n\n// Type returns the type of the protocol request\nfunc (request *Request) Type() templateTypes.ProtocolType {\n\treturn templateTypes.WHOISProtocol\n}\n\n// Path: pkg/workflows/workflows.go\npackage workflows\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n)\n\n// Workflow is a workflow to execute with chained requests, etc.\ntype Workflow struct {\n\t// description: |\n\t//   Workflows is a list of workflows to execute for a template.\n\tWorkflows []*WorkflowTemplate `yaml:\"workflows,omitempty\" json:\"workflows,omitempty\" jsonschema:\"title=list of workflows to execute,description=List of workflows to execute for template\"`\n\n\tOptions *protocols.ExecutorOptions `yaml:\"-\" json:\"-\"`\n}\n\n// WorkflowTemplate is a template to be run as part of a workflow\ntype WorkflowTemplate struct {\n\t// description: |\n\t//   Template is a single template or directory to execute as part of workflow.\n\t// examples:\n\t//   - name: A single template\n\t//     value: \"\\\"dns/worksites-detection.yaml\\\"\"\n\t//   - name: A template directory\n\t//     value: \"\\\"misconfigurations/aem\\\"\"\n\tTemplate string `yaml:\"template,omitempty\" json:\"template,omitempty\" jsonschema:\"title=template/directory to execute,description=Template or directory to execute as part of workflow\"`\n\t// description: |\n\t//    Tags to run templates based on.\n\tTags stringslice.StringSlice `yaml:\"tags,omitempty\" json:\"tags,omitempty\" jsonschema:\"title=tags to execute,description=Tags to run template based on\"`\n\t// description: |\n\t//    Matchers perform name based matching to run subtemplates for a workflow.\n\tMatchers []*Matcher `yaml:\"matchers,omitempty\" json:\"matchers,omitempty\" jsonschema:\"title=name based template result matchers,description=Matchers perform name based matching to run subtemplates for a workflow\"`\n\t// description: |\n\t//    Subtemplates are run if the `template` field Template matches.\n\tSubtemplates []*WorkflowTemplate `yaml:\"subtemplates,omitempty\" json:\"subtemplates,omitempty\" jsonschema:\"title=subtemplate based result matchers,description=Subtemplates are ran if the template field Template matches\"`\n\t// Executers perform the actual execution for the workflow template\n\tExecuters []*ProtocolExecuterPair `yaml:\"-\" json:\"-\"`\n}\n\n// ProtocolExecuterPair is a pair of protocol executer and its options\ntype ProtocolExecuterPair struct {\n\tExecuter     protocols.Executer\n\tOptions      *protocols.ExecutorOptions\n\tTemplateType templateTypes.ProtocolType\n}\n\n// Matcher performs conditional matching on the workflow template results.\ntype Matcher struct {\n\t// description: |\n\t//    Name is the name of the items to match.\n\tName stringslice.StringSlice `yaml:\"name,omitempty\" json:\"name,omitempty\" jsonschema:\"title=name of items to match,description=Name of items to match\"`\n\t// description: |\n\t//   Condition is the optional condition between names. By default,\n\t//   the condition is assumed to be OR.\n\t// values:\n\t//   - \"and\"\n\t//   - \"or\"\n\tCondition string `yaml:\"condition,omitempty\" json:\"condition,omitempty\" jsonschema:\"title=condition between names,description=Condition between the names,enum=and,enum=or\"`\n\t// description: |\n\t//    Subtemplates are run if the name of matcher matches.\n\tSubtemplates []*WorkflowTemplate `yaml:\"subtemplates,omitempty\" json:\"subtemplates,omitempty\" jsonschema:\"title=templates to run after match,description=Templates to run after match\"`\n\n\tcondition ConditionType\n}\n\n// ConditionType is the type of condition for matcher\ntype ConditionType int\n\nconst (\n\t// ANDCondition matches responses with AND condition in arguments.\n\tANDCondition ConditionType = iota + 1\n\t// ORCondition matches responses with AND condition in arguments.\n\tORCondition\n)\n\n// ConditionTypes is a table for conversion of condition type from string.\nvar ConditionTypes = map[string]ConditionType{\n\t\"and\": ANDCondition,\n\t\"or\":  ORCondition,\n}\n\n// Compile compiles the matcher for workflow\nfunc (matcher *Matcher) Compile() error {\n\tvar ok bool\n\tif matcher.Condition != \"\" {\n\t\tmatcher.condition, ok = ConditionTypes[matcher.Condition]\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"unknown condition specified: %s\", matcher.Condition)\n\t\t}\n\t} else {\n\t\tmatcher.condition = ORCondition\n\t}\n\treturn nil\n}\n\n// Match matches a name for matcher names or name\nfunc (matcher *Matcher) Match(result *operators.Result) bool {\n\tnames := matcher.Name.ToSlice()\n\tif len(names) == 0 {\n\t\treturn false\n\t}\n\n\tfor i, name := range names {\n\t\tmatchOK := result.HasMatch(name)\n\t\textractOK := result.HasExtract(name)\n\n\t\tif !matchOK && !extractOK {\n\t\t\tif matcher.condition == ANDCondition {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif matcher.condition == ORCondition {\n\t\t\treturn true\n\t\t} else if len(names)-1 == i {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// Path: pkg/js/compiler/compiler.go\n// Package compiler provides a compiler for the goja runtime.\npackage compiler\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/dop251/goja\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\tcontextutil \"github.com/projectdiscovery/utils/context\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\n// Compiler provides a runtime to execute goja runtime\n// based javascript scripts efficiently while also\n// providing them access to custom modules defined in libs/.\ntype Compiler struct{}\n\n// New creates a new compiler for the goja runtime.\nfunc New() *Compiler {\n\treturn &Compiler{}\n}\n\n// ExecuteOptions provides options for executing a script.\ntype ExecuteOptions struct {\n\t// Callback can be used to register new runtime helper functions\n\t// ex: export etc\n\tCallback func(runtime *goja.Runtime) error\n\n\t// Cleanup is extra cleanup function to be called after execution\n\tCleanup func(runtime *goja.Runtime)\n\n\t/// Timeout for this script execution\n\tTimeout int\n\t// Source is original source of the script\n\tSource *string\n\n\t// Manually exported objects\n\texports map[string]interface{}\n}\n\n// ExecuteArgs is the arguments to pass to the script.\ntype ExecuteArgs struct {\n\tArgs        map[string]interface{} //these are protocol variables\n\tTemplateCtx map[string]interface{} // templateCtx contains template scoped variables\n}\n\n// NewExecuteArgs returns a new execute arguments.\n\nfunc NewExecuteArgs() *ExecuteArgs {\n\treturn &ExecuteArgs{\n\t\tArgs:        make(map[string]interface{}),\n\t\tTemplateCtx: make(map[string]interface{}),\n\t}\n}\n\n// ExecuteResult is the result of executing a script.\ntype ExecuteResult map[string]interface{}\n\nfunc NewExecuteResult() ExecuteResult {\n\treturn make(map[string]interface{})\n}\n\n// GetSuccess returns whether the script was successful or not.\nfunc (e ExecuteResult) GetSuccess() bool {\n\tval, ok := e[\"success\"].(bool)\n\tif !ok {\n\t\treturn false\n\t}\n\treturn val\n}\n\n// Execute executes a script with the default options.\nfunc (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error) {\n\tp, err := WrapScriptNCompile(code, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.ExecuteWithOptions(p, args, &ExecuteOptions{})\n}\n\n// ExecuteWithOptions executes a script with the provided options.\nfunc (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {\n\tif opts == nil {\n\t\topts = &ExecuteOptions{}\n\t}\n\tif args == nil {\n\t\targs = NewExecuteArgs()\n\t}\n\t// handle nil maps\n\tif args.TemplateCtx == nil {\n\t\targs.TemplateCtx = make(map[string]interface{})\n\t}\n\tif args.Args == nil {\n\t\targs.Args = make(map[string]interface{})\n\t}\n\t// merge all args into templatectx\n\targs.TemplateCtx = generators.MergeMaps(args.TemplateCtx, args.Args)\n\n\tif opts.Timeout <= 0 || opts.Timeout > 180 {\n\t\t// some js scripts can take longer time so allow configuring timeout\n\t\t// from template but keep it within sane limits (180s)\n\t\topts.Timeout = JsProtocolTimeout\n\t}\n\n\t// execute with context and timeout\n\tctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.Timeout)*time.Second)\n\tdefer cancel()\n\t// execute the script\n\tresults, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\terr = fmt.Errorf(\"panic: %v\", r)\n\t\t\t}\n\t\t}()\n\t\treturn ExecuteProgram(program, args, opts)\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar res ExecuteResult\n\tif opts.exports != nil {\n\t\tres = ExecuteResult(opts.exports)\n\t\topts.exports = nil\n\t} else {\n\t\tres = NewExecuteResult()\n\t}\n\tres[\"response\"] = results.Export()\n\tres[\"success\"] = results.ToBoolean()\n\treturn res, nil\n}\n\n// Wraps a script in a function and compiles it.\nfunc WrapScriptNCompile(script string, strict bool) (*goja.Program, error) {\n\tif !stringsutil.ContainsAny(script, exportAsToken, exportToken) {\n\t\t// this will not be run in a pooled runtime\n\t\treturn goja.Compile(\"\", script, strict)\n\t}\n\tval := fmt.Sprintf(`\n\t\t(function() {\n\t\t\t%s\n\t\t})()\n\t`, script)\n\treturn goja.Compile(\"\", val, strict)\n}\n\n// Path: pkg/protocols/code/code.go\npackage code\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/alecthomas/chroma/quick\"\n\t\"github.com/ditashi/jsbeautifier-go/jsbeautifier\"\n\t\"github.com/dop251/goja\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/gozero\"\n\tgozerotypes \"github.com/projectdiscovery/gozero/types\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\tcontextutil \"github.com/projectdiscovery/utils/context\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n)\n\nconst (\n\tpythonEnvRegex    = `os\\.getenv\\(['\"]([^'\"]+)['\"]\\)`\n\tTimeoutMultiplier = 6 // timeout multiplier for code protocol\n)\n\nvar (\n\tpythonEnvRegexCompiled = regexp.MustCompile(pythonEnvRegex)\n)\n\n// Request is a request for the SSL protocol\ntype Request struct {\n\t// Operators for the current request go here.\n\toperators.Operators `yaml:\",inline,omitempty\"`\n\tCompiledOperators   *operators.Operators `yaml:\"-\" json:\"-\"`\n\n\t// ID is the optional id of the request\n\tID string `yaml:\"id,omitempty\" json:\"id,omitempty\" jsonschema:\"title=id of the request,description=ID is the optional ID of the Request\"`\n\t// description: |\n\t//   Engine type\n\tEngine []string `yaml:\"engine,omitempty\" json:\"engine,omitempty\" jsonschema:\"title=engine,description=Engine\"`\n\t// description: |\n\t//   PreCondition is a condition which is evaluated before sending the request.\n\tPreCondition string `yaml:\"pre-condition,omitempty\" json:\"pre-condition,omitempty\" jsonschema:\"title=pre-condition for the request,description=PreCondition is a condition which is evaluated before sending the request\"`\n\t// description: |\n\t//   Engine Arguments\n\tArgs []string `yaml:\"args,omitempty\" json:\"args,omitempty\" jsonschema:\"title=args,description=Args\"`\n\t// description: |\n\t//   Pattern preferred for file name\n\tPattern string `yaml:\"pattern,omitempty\" json:\"pattern,omitempty\" jsonschema:\"title=pattern,description=Pattern\"`\n\t// description: |\n\t//   Source File/Snippet\n\tSource string `yaml:\"source,omitempty\" json:\"source,omitempty\" jsonschema:\"title=source file/snippet,description=Source snippet\"`\n\n\toptions              *protocols.ExecutorOptions `yaml:\"-\" json:\"-\"`\n\tpreConditionCompiled *goja.Program              `yaml:\"-\" json:\"-\"`\n\tgozero               *gozero.Gozero             `yaml:\"-\" json:\"-\"`\n\tsrc                  *gozero.Source             `yaml:\"-\" json:\"-\"`\n}\n\n// Compile compiles the request generators preparing any requests possible.\nfunc (request *Request) Compile(options *protocols.ExecutorOptions) error {\n\trequest.options = options\n\n\tgozeroOptions := &gozero.Options{\n\t\tEngines:                  request.Engine,\n\t\tArgs:                     request.Args,\n\t\tEarlyCloseFileDescriptor: true,\n\t}\n\tengine, err := gozero.New(gozeroOptions)\n\tif err != nil {\n\t\treturn errorutil.NewWithErr(err).Msgf(\"[%s] engines '%s' not available on host\", options.TemplateID, strings.Join(request.Engine, \",\"))\n\t}\n\trequest.gozero = engine\n\n\tvar src *gozero.Source\n\n\tsrc, err = gozero.NewSourceWithString(request.Source, request.Pattern, request.options.TemporaryDirectory)\n\tif err != nil {\n\t\treturn err\n\t}\n\trequest.src = src\n\n\tif len(request.Matchers) > 0 || len(request.Extractors) > 0 {\n\t\tcompiled := &request.Operators\n\t\tcompiled.ExcludeMatchers = options.ExcludeMatchers\n\t\tcompiled.TemplateID = options.TemplateID\n\t\tif err := compiled.Compile(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile operators\")\n\t\t}\n\t\tfor _, matcher := range compiled.Matchers {\n\t\t\t// default matcher part for code protocol is response\n\t\t\tif matcher.Part == \"\" || matcher.Part == \"body\" {\n\t\t\t\tmatcher.Part = \"response\"\n\t\t\t}\n\t\t}\n\t\tfor _, extractor := range compiled.Extractors {\n\t\t\t// default extractor part for code protocol is response\n\t\t\tif extractor.Part == \"\" || extractor.Part == \"body\" {\n\t\t\t\textractor.Part = \"response\"\n\t\t\t}\n\t\t}\n\t\trequest.CompiledOperators = compiled\n\t}\n\n\t// compile pre-condition if any\n\tif request.PreCondition != \"\" {\n\t\tpreConditionCompiled, err := compiler.WrapScriptNCompile(request.PreCondition, false)\n\t\tif err != nil {\n\t\t\treturn errorutil.NewWithTag(request.TemplateID, \"could not compile pre-condition: %s\", err)\n\t\t}\n\t\trequest.preConditionCompiled = preConditionCompiled\n\t}\n\treturn nil\n}\n\n// Requests returns the total number of requests the rule will perform\nfunc (request *Request) Requests() int {\n\treturn 1\n}\n\n// GetID returns the ID for the request if any.\nfunc (request *Request) GetID() string {\n\treturn request.ID\n}\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) (err error) {\n\tmetaSrc, err := gozero.NewSourceWithString(input.MetaInput.Input, \"\", request.options.TemporaryDirectory)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tif err := metaSrc.Cleanup(); err != nil {\n\t\t\tgologger.Warning().Msgf(\"%s\\n\", err)\n\t\t}\n\t}()\n\n\tvar interactshURLs []string\n\n\t// inject all template context values as gozero env allvars\n\tallvars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil)\n\t// add template context values if available\n\tif request.options.HasTemplateCtx(input.MetaInput) {\n\t\tallvars = generators.MergeMaps(allvars, request.options.GetTemplateCtx(input.MetaInput).GetAll())\n\t}\n\t// add dynamic and previous variables\n\tallvars = generators.MergeMaps(allvars, dynamicValues, previous)\n\t// optionvars are vars passed from CLI or env variables\n\toptionVars := generators.BuildPayloadFromOptions(request.options.Options)\n\tvariablesMap := request.options.Variables.Evaluate(allvars)\n\t// since we evaluate variables using allvars, give precedence to variablesMap\n\tallvars = generators.MergeMaps(allvars, variablesMap, optionVars, request.options.Constants)\n\tfor name, value := range allvars {\n\t\tv := fmt.Sprint(value)\n\t\tv, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs)\n\t\t// if value is updated by interactsh, update allvars to reflect the change downstream\n\t\tallvars[name] = v\n\t\tmetaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v})\n\t}\n\n\t// set timeout using multiplier\n\ttimeout := TimeoutMultiplier * request.options.Options.Timeout\n\n\tif request.PreCondition != \"\" {\n\t\tif request.options.Options.Debug || request.options.Options.DebugRequests {\n\t\t\tgologger.Debug().Msgf(\"[%s] Executing Precondition for Code request\\n\", request.TemplateID)\n\t\t\tvar highlightFormatter = \"terminal256\"\n\t\t\tif request.options.Options.NoColor {\n\t\t\t\thighlightFormatter = \"text\"\n\t\t\t}\n\t\t\tvar buff bytes.Buffer\n\t\t\t_ = quick.Highlight(&buff, beautifyJavascript(request.PreCondition), \"javascript\", highlightFormatter, \"monokai\")\n\t\t\tprettyPrint(request.TemplateID, buff.String())\n\t\t}\n\n\t\targs := compiler.NewExecuteArgs()\n\t\targs.TemplateCtx = allvars\n\n\t\tresult, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args,\n\t\t\t&compiler.ExecuteOptions{\n\t\t\t\tTimeout:  timeout,\n\t\t\t\tSource:   &request.PreCondition,\n\t\t\t\tCallback: registerPreConditionFunctions,\n\t\t\t\tCleanup:  cleanUpPreConditionFunctions,\n\t\t\t})\n\t\tif err != nil {\n\t\t\treturn errorutil.NewWithTag(request.TemplateID, \"could not execute pre-condition: %s\", err)\n\t\t}\n\t\tif !result.GetSuccess() || types.ToString(result[\"error\"]) != \"\" {\n\t\t\tgologger.Warning().Msgf(\"[%s] Precondition for request %s was not satisfied\\n\", request.TemplateID, request.PreCondition)\n\t\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\t\treturn nil\n\t\t}\n\t\tif request.options.Options.Debug || request.options.Options.DebugRequests {\n\t\t\tgologger.Debug().Msgf(\"[%s] Precondition for request was satisfied\\n\", request.TemplateID)\n\t\t}\n\t}\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)\n\tdefer cancel()\n\t// Note: we use contextutil despite the fact that gozero accepts context as argument\n...\n", "needle_token_start": 11491, "needle_token_end": 11543, "code_context_ntokens": 15334, "output": ["```go\n// Path: pkg/js/compiler/compiler.go\n// Package compiler provides a compiler for the goja runtime.\npackage compiler\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/dop251/goja\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\tcontextutil \"github.com/projectdiscovery/utils/context\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\n// Compiler provides a runtime to execute goja runtime\n// based javascript scripts efficiently while also\n// providing them access to custom modules defined in libs/.\ntype Compiler struct{}\n\n// New creates a new compiler for the goja runtime.\nfunc New() *Compiler {\n\treturn &Compiler{}\n}\n\n// ExecuteOptions provides options for executing a script.\ntype ExecuteOptions struct {\n\t// Callback can be used to register new runtime helper functions\n\t// ex: export etc\n\tCallback func(runtime *goja.Runtime) error\n\n\t// Cleanup is extra cleanup function to be called after execution\n\tCleanup func(runtime *goja.Runtime)\n\n\t/// Timeout for this script execution\n\tTimeout int\n\t// Source is original source of the script\n\tSource *string\n\n\t// Manually exported objects\n\texports map[string]interface{}\n}\n\n// ExecuteArgs is the arguments to pass to the script.\ntype ExecuteArgs struct {\n\tArgs        map[string]interface{} //these are protocol variables\n\tTemplateCtx map[string]interface{} // templateCtx contains template scoped variables\n}\n\n// NewExecuteArgs returns a new execute arguments.\n\nfunc NewExecuteArgs() *ExecuteArgs {\n\treturn &ExecuteArgs{\n\t\tArgs:        make(map[string]interface{}),\n\t\tTemplateCtx: make(map[string]interface{}),\n\t}\n}\n\n// ExecuteResult is the result of executing a script.\ntype ExecuteResult map[string]interface{}\n\nfunc NewExecuteResult() ExecuteResult {\n\treturn make(map[string]interface{})\n}\n\n// GetSuccess returns whether the script was successful or not.\nfunc (e ExecuteResult) GetSuccess() bool {\n\tval, ok := e[\"success\"].(bool)\n\tif !ok {\n\t\treturn false\n\t}\n\treturn val\n}\n\n// Execute executes a script with the default options.\nfunc (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error) {\n\tp, err := WrapScriptNCompile(code, false)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.ExecuteWithOptions(p, args, &ExecuteOptions{})\n}\n\n// ExecuteWithOptions executes a script with the provided options.\nfunc (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {\n\tif opts == nil {\n\t\topts = &ExecuteOptions{}\n\t}\n\tif args == nil {\n\t\targs = NewExecuteArgs()\n\t}\n\t// handle nil maps\n\tif args.TemplateCtx == nil {\n\t\targs.TemplateCtx = make(map[string]interface{})\n\t}\n\tif args.Args == nil {\n\t\targs.Args = make(map[string]interface{})\n\t}\n\t// merge all args into templatectx\n\targs.TemplateCtx = generators.MergeMaps(args.TemplateCtx, args.Args)\n\n\tif opts.Timeout <= 0 || opts.Timeout > 180 {\n\t\t// some js scripts can take longer time so allow configuring timeout\n\t\t// from template but keep it within sane limits (180s)\n\t\topts.Timeout = JsProtocolTimeout\n\t}\n\n\t// execute with context and timeout\n\tctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.Timeout)*time.Second)\n\tdefer cancel()\n\t// execute the script\n\tresults, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\terr = fmt.Errorf(\"panic: %v\", r)\n\t\t\t}\n\t\t}()\n\t\treturn ExecuteProgram(program, args, opts)\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar res ExecuteResult\n\tif opts.exports != nil {\n\t\tres = ExecuteResult(opts.exports)\n\t\topts.exports = nil\n\t} else {\n\t\tres = NewExecuteResult()\n\t}\n\tres[\"response\"] = results.Export()\n\tres[\"success\"] = results.ToBoolean()\n\treturn res, nil\n}\n\n// Wraps a script in a function and compiles it.\nfunc WrapScriptNCompile(script string, strict bool) (*goja.Program, error) {\n\tif !stringsutil.ContainsAny(script, exportAsToken, exportToken) {\n\t\t// this will not be run in a pooled runtime\n\t\treturn goja.Compile(\"\", script, strict)\n\t}\n\tval := fmt.Sprintf(`\n\t\t(function() {\n\t\t\t%s\n\t\t})()\n\t`, script)\n\treturn goja.Compile(\"\", val, strict)\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "TestStandardWriterRequest", "language": "go", "path": "pkg/output/output_test.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate the behavior of a logging component when handling requests, specifically focusing on how it logs different scenarios such as requests without errors, requests with tracing enabled, and requests that encounter errors.\n2. **Input**: The function takes no direct inputs as it is a test function, but it internally uses predefined inputs such as template IDs, URLs, request types, and error conditions to simulate different logging scenarios.\n3. **Output**: The output of this function is not a return value but the content written to log files, which is checked to ensure it matches expected log messages for each test scenario.\n4. **Procedure**: The function executes several sub-tests where it:\n   - Initializes a logging component.\n   - Simulates logging of a request without any associated errors or trace files.\n   - Simulates logging of a request with trace files enabled but no errors.\n   - Simulates logging of a request that results in an error and checks how the error is logged.\n   Each sub-test checks that the log output is as expected, verifying the correct functionality of the logging component under different conditions.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/operators/extractors/extract_test.go\npackage extractors\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestExtractor_ExtractRegex(t *testing.T) {\n\te := &Extractor{Type: ExtractorTypeHolder{ExtractorType: RegexExtractor}, Regex: []string{`([A-Z])\\w+`}}\n\terr := e.CompileExtractors()\n\trequire.Nil(t, err)\n\n\tgot := e.ExtractRegex(\"RegEx\")\n\trequire.Equal(t, map[string]struct{}{\"RegEx\": {}}, got)\n\n\tgot = e.ExtractRegex(\"regex\")\n\trequire.Equal(t, map[string]struct{}{}, got)\n}\n\nfunc TestExtractor_ExtractKval(t *testing.T) {\n\te := &Extractor{Type: ExtractorTypeHolder{ExtractorType: KValExtractor}, KVal: []string{\"content_type\"}}\n\terr := e.CompileExtractors()\n\trequire.Nil(t, err)\n\n\tgot := e.ExtractKval(map[string]interface{}{\"content_type\": \"text/html\"})\n\trequire.Equal(t, map[string]struct{}{\"text/html\": {}}, got)\n\n\tgot = e.ExtractKval(map[string]interface{}{\"authorization\": \"Basic YWxhZGRpbjpvcGVuc2VzYW1l\"})\n\trequire.Equal(t, map[string]struct{}{}, got)\n\n}\n\nfunc TestExtractor_ExtractXPath(t *testing.T) {\n...\n// Path: pkg/operators/extractors/extractor_types.go\npackage extractors\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"strings\"\n\n\t\"github.com/invopop/jsonschema\"\n)\n\n// ExtractorType is the type of the extractor specified\ntype ExtractorType int\n\n// name:ExtractorType\nconst (\n\t// name:regex\n\tRegexExtractor ExtractorType = iota + 1\n\t// name:kval\n\tKValExtractor\n\t// name:xpath\n\tXPathExtractor\n\t// name:json\n\tJSONExtractor\n\t// name:dsl\n\tDSLExtractor\n\tlimit\n)\n\n// extractorMappings is a table for conversion of extractor type from string.\nvar extractorMappings = map[ExtractorType]string{\n\tRegexExtractor: \"regex\",\n\tKValExtractor:  \"kval\",\n\tXPathExtractor: \"xpath\",\n\tJSONExtractor:  \"json\",\n\tDSLExtractor:   \"dsl\",\n}\n\n// GetType returns the type of the matcher\nfunc (e *Extractor) GetType() ExtractorType {\n\treturn e.Type.ExtractorType\n}\n\n// GetSupportedExtractorTypes returns list of supported types\nfunc GetSupportedExtractorTypes() []ExtractorType {\n\tvar result []ExtractorType\n\tfor index := ExtractorType(1); index < limit; index++ {\n\t\tresult = append(result, index)\n\t}\n\treturn result\n}\n\nfunc toExtractorTypes(valueToMap string) (ExtractorType, error) {\n\tnormalizedValue := normalizeValue(valueToMap)\n\tfor key, currentValue := range extractorMappings {\n\t\tif normalizedValue == currentValue {\n\t\t\treturn key, nil\n\t\t}\n\t}\n\treturn -1, errors.New(\"Invalid extractor type: \" + valueToMap)\n}\n\nfunc normalizeValue(value string) string {\n\treturn strings.TrimSpace(strings.ToLower(value))\n}\n\nfunc (t ExtractorType) String() string {\n\treturn extractorMappings[t]\n}\n\n// ExtractorTypeHolder is used to hold internal type of the extractor\ntype ExtractorTypeHolder struct {\n\tExtractorType ExtractorType `mapping:\"true\"`\n}\n\nfunc (holder ExtractorTypeHolder) JSONSchema() *jsonschema.Schema {\n\tgotType := &jsonschema.Schema{\n\t\tType:        \"string\",\n\t\tTitle:       \"type of the extractor\",\n\t\tDescription: \"Type of the extractor\",\n\t}\n\tfor _, types := range GetSupportedExtractorTypes() {\n\t\tgotType.Enum = append(gotType.Enum, types.String())\n\t}\n\treturn gotType\n}\n\nfunc (holder *ExtractorTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {\n\tvar marshalledTypes string\n\tif err := unmarshal(&marshalledTypes); err != nil {\n\t\treturn err\n\t}\n\n\tcomputedType, err := toExtractorTypes(marshalledTypes)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tholder.ExtractorType = computedType\n\treturn nil\n}\n\nfunc (holder *ExtractorTypeHolder) UnmarshalJSON(data []byte) error {\n\ts := strings.Trim(string(data), `\"`)\n\tif s == \"\" {\n\t\treturn nil\n\t}\n\tcomputedType, err := toExtractorTypes(s)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tholder.ExtractorType = computedType\n\treturn nil\n}\n\nfunc (holder *ExtractorTypeHolder) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(holder.ExtractorType.String())\n}\n\nfunc (holder ExtractorTypeHolder) MarshalYAML() (interface{}, error) {\n\treturn holder.ExtractorType.String(), nil\n}\n\n// Path: pkg/operators/matchers/compile.go\npackage matchers\n\nimport (\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/Knetic/govaluate\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl\"\n)\n\n// CompileMatchers performs the initial setup operation on a matcher\nfunc (matcher *Matcher) CompileMatchers() error {\n\tvar ok bool\n\n\t// Support hexadecimal encoding for matchers too.\n\tif matcher.Encoding == \"hex\" {\n\t\tfor i, word := range matcher.Words {\n\t\t\tif decoded, err := hex.DecodeString(word); err == nil && len(decoded) > 0 {\n\t\t\t\tmatcher.Words[i] = string(decoded)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Set up the matcher type\n\tcomputedType, err := toMatcherTypes(matcher.GetType().String())\n\tif err != nil {\n\t\treturn fmt.Errorf(\"unknown matcher type specified: %s\", matcher.Type)\n\t}\n\n\tmatcher.matcherType = computedType\n\n\t// Validate the matcher structure\n\tif err := matcher.Validate(); err != nil {\n\t\treturn err\n\t}\n\n\t// By default, match on body if user hasn't provided any specific items\n\tif matcher.Part == \"\" && matcher.GetType() != DSLMatcher {\n\t\tmatcher.Part = \"body\"\n\t}\n\n\t// Compile the regexes\n\tfor _, regex := range matcher.Regex {\n\t\tcompiled, err := regexp.Compile(regex)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"could not compile regex: %s\", regex)\n\t\t}\n\t\tmatcher.regexCompiled = append(matcher.regexCompiled, compiled)\n\t}\n\n\t// Compile and validate binary Values in matcher\n\tfor _, value := range matcher.Binary {\n\t\tif decoded, err := hex.DecodeString(value); err != nil {\n\t\t\treturn fmt.Errorf(\"could not hex decode binary: %s\", value)\n\t\t} else {\n\t\t\tmatcher.binaryDecoded = append(matcher.binaryDecoded, string(decoded))\n\t\t}\n\t}\n\n\t// Compile the dsl expressions\n\tfor _, dslExpression := range matcher.DSL {\n\t\tcompiledExpression, err := govaluate.NewEvaluableExpressionWithFunctions(dslExpression, dsl.HelperFunctions)\n\t\tif err != nil {\n\t\t\treturn &dsl.CompilationError{DslSignature: dslExpression, WrappedError: err}\n\t\t}\n\t\tmatcher.dslCompiled = append(matcher.dslCompiled, compiledExpression)\n\t}\n\n\t// Set up the condition type, if any.\n\tif matcher.Condition != \"\" {\n\t\tmatcher.condition, ok = ConditionTypes[matcher.Condition]\n\t\tif !ok {\n\t\t\treturn fmt.Errorf(\"unknown condition specified: %s\", matcher.Condition)\n\t\t}\n\t} else {\n\t\tmatcher.condition = ORCondition\n\t}\n\n\tif matcher.CaseInsensitive {\n\t\tif matcher.GetType() != WordsMatcher {\n\t\t\treturn fmt.Errorf(\"case-insensitive flag is supported only for 'word' matchers (not '%s')\", matcher.Type)\n\t\t}\n\t\tfor i := range matcher.Words {\n\t\t\tmatcher.Words[i] = strings.ToLower(matcher.Words[i])\n\t\t}\n\t}\n\treturn nil\n}\n\n// GetType returns the condition type of the matcher\n// todo: the field should be exposed natively\nfunc (matcher *Matcher) GetCondition() ConditionType {\n\treturn matcher.condition\n}\n\n// Path: pkg/operators/matchers/doc.go\n// Package matchers implements matchers for http response\n// matching with templates.\npackage matchers\n\n// Path: pkg/operators/matchers/match.go\npackage matchers\n\nimport (\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/Knetic/govaluate\"\n\t\"github.com/antchfx/htmlquery\"\n\t\"github.com/antchfx/xmlquery\"\n\n\tdslRepo \"github.com/projectdiscovery/dsl\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n)\n\nvar (\n\t// showDSLErr controls whether to show hidden DSL errors or not\n\tshowDSLErr = strings.EqualFold(os.Getenv(\"SHOW_DSL_ERRORS\"), \"true\")\n)\n\n// MatchStatusCode matches a status code check against a corpus\nfunc (matcher *Matcher) MatchStatusCode(statusCode int) bool {\n\t// Iterate over all the status codes accepted as valid\n\t//\n\t// Status codes don't support AND conditions.\n\tfor _, status := range matcher.Status {\n\t\t// Continue if the status codes don't match\n\t\tif statusCode != status {\n\t\t\tcontinue\n\t\t}\n\t\t// Return on the first match.\n\t\treturn true\n\t}\n\treturn false\n}\n\n// MatchSize matches a size check against a corpus\nfunc (matcher *Matcher) MatchSize(length int) bool {\n\t// Iterate over all the sizes accepted as valid\n\t//\n\t// Sizes codes don't support AND conditions.\n\tfor _, size := range matcher.Size {\n\t\t// Continue if the size doesn't match\n\t\tif length != size {\n\t\t\tcontinue\n\t\t}\n\t\t// Return on the first match.\n\t\treturn true\n\t}\n\treturn false\n}\n\n// MatchWords matches a word check against a corpus.\nfunc (matcher *Matcher) MatchWords(corpus string, data map[string]interface{}) (bool, []string) {\n\tif matcher.CaseInsensitive {\n\t\tcorpus = strings.ToLower(corpus)\n\t}\n\n\tvar matchedWords []string\n\t// Iterate over all the words accepted as valid\n\tfor i, word := range matcher.Words {\n\t\tif data == nil {\n\t\t\tdata = make(map[string]interface{})\n\t\t}\n\n\t\tvar err error\n\t\tword, err = expressions.Evaluate(word, data)\n\t\tif err != nil {\n\t\t\tgologger.Warning().Msgf(\"Error while evaluating word matcher: %q\", word)\n\t\t\tif matcher.condition == ANDCondition {\n\t\t\t\treturn false, []string{}\n\t\t\t}\n\t\t}\n\t\t// Continue if the word doesn't match\n\t\tif !strings.Contains(corpus, word) {\n\t\t\t// If we are in an AND request and a match failed,\n\t\t\t// return false as the AND condition fails on any single mismatch.\n\t\t\tswitch matcher.condition {\n\t\t\tcase ANDCondition:\n\t\t\t\treturn false, []string{}\n\t\t\tcase ORCondition:\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// If the condition was an OR, return on the first match.\n\t\tif matcher.condition == ORCondition && !matcher.MatchAll {\n\t\t\treturn true, []string{word}\n\t\t}\n\t\tmatchedWords = append(matchedWords, word)\n\n\t\t// If we are at the end of the words, return with true\n\t\tif len(matcher.Words)-1 == i && !matcher.MatchAll {\n\t\t\treturn true, matchedWords\n\t\t}\n\t}\n\tif len(matchedWords) > 0 && matcher.MatchAll {\n\t\treturn true, matchedWords\n\t}\n\treturn false, []string{}\n}\n\n// MatchRegex matches a regex check against a corpus\nfunc (matcher *Matcher) MatchRegex(corpus string) (bool, []string) {\n\tvar matchedRegexes []string\n\t// Iterate over all the regexes accepted as valid\n\tfor i, regex := range matcher.regexCompiled {\n\t\t// Continue if the regex doesn't match\n\t\tif !regex.MatchString(corpus) {\n\t\t\t// If we are in an AND request and a match failed,\n\t\t\t// return false as the AND condition fails on any single mismatch.\n\t\t\tswitch matcher.condition {\n\t\t\tcase ANDCondition:\n\t\t\t\treturn false, []string{}\n\t\t\tcase ORCondition:\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tcurrentMatches := regex.FindAllString(corpus, -1)\n\t\t// If the condition was an OR, return on the first match.\n\t\tif matcher.condition == ORCondition && !matcher.MatchAll {\n\t\t\treturn true, currentMatches\n\t\t}\n\n\t\tmatchedRegexes = append(matchedRegexes, currentMatches...)\n\n\t\t// If we are at the end of the regex, return with true\n\t\tif len(matcher.regexCompiled)-1 == i && !matcher.MatchAll {\n\t\t\treturn true, matchedRegexes\n\t\t}\n\t}\n\tif len(matchedRegexes) > 0 && matcher.MatchAll {\n\t\treturn true, matchedRegexes\n\t}\n\treturn false, []string{}\n}\n\n// MatchBinary matches a binary check against a corpus\nfunc (matcher *Matcher) MatchBinary(corpus string) (bool, []string) {\n\tvar matchedBinary []string\n\t// Iterate over all the words accepted as valid\n\tfor i, binary := range matcher.binaryDecoded {\n\t\tif !strings.Contains(corpus, binary) {\n\t\t\t// If we are in an AND request and a match failed,\n\t\t\t// return false as the AND condition fails on any single mismatch.\n\t\t\tswitch matcher.condition {\n\t\t\tcase ANDCondition:\n\t\t\t\treturn false, []string{}\n\t\t\tcase ORCondition:\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// If the condition was an OR, return on the first match.\n\t\tif matcher.condition == ORCondition {\n\t\t\treturn true, []string{binary}\n\t\t}\n\n\t\tmatchedBinary = append(matchedBinary, binary)\n\n\t\t// If we are at the end of the words, return with true\n\t\tif len(matcher.Binary)-1 == i {\n\t\t\treturn true, matchedBinary\n\t\t}\n\t}\n\treturn false, []string{}\n}\n\n// MatchDSL matches on a generic map result\nfunc (matcher *Matcher) MatchDSL(data map[string]interface{}) bool {\n\tlogExpressionEvaluationFailure := func(matcherName string, err error) {\n\t\tgologger.Warning().Msgf(\"Could not evaluate expression: %s, error: %s\", matcherName, err.Error())\n\t}\n\n\t// Iterate over all the expressions accepted as valid\n\tfor i, expression := range matcher.dslCompiled {\n\t\tif varErr := expressions.ContainsUnresolvedVariables(expression.String()); varErr != nil {\n\t\t\tresolvedExpression, err := expressions.Evaluate(expression.String(), data)\n\t\t\tif err != nil {\n\t\t\t\tlogExpressionEvaluationFailure(matcher.Name, err)\n\t\t\t\treturn false\n\t\t\t}\n\t\t\texpression, err = govaluate.NewEvaluableExpressionWithFunctions(resolvedExpression, dsl.HelperFunctions)\n\t\t\tif err != nil {\n\t\t\t\tlogExpressionEvaluationFailure(matcher.Name, err)\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\n\t\tresult, err := expression.Evaluate(data)\n\t\tif err != nil {\n\t\t\tif matcher.condition == ANDCondition {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tif !matcher.ignoreErr(err) {\n\t\t\t\tgologger.Warning().Msgf(\"[%s] %s\", data[\"template-id\"], err.Error())\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tif boolResult, ok := result.(bool); !ok {\n\t\t\tgologger.Error().Label(\"WRN\").Msgf(\"[%s] The return value of a DSL statement must return a boolean value.\", data[\"template-id\"])\n\t\t\tcontinue\n\t\t} else if !boolResult {\n\t\t\t// If we are in an AND request and a match failed,\n\t\t\t// return false as the AND condition fails on any single mismatch.\n\t\t\tswitch matcher.condition {\n\t\t\tcase ANDCondition:\n\t\t\t\treturn false\n\t\t\tcase ORCondition:\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// If the condition was an OR, return on the first match.\n\t\tif matcher.condition == ORCondition {\n\t\t\treturn true\n\t\t}\n\n\t\t// If we are at the end of the dsl, return with true\n\t\tif len(matcher.dslCompiled)-1 == i {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// MatchXPath matches on a generic map result\nfunc (matcher *Matcher) MatchXPath(corpus string) bool {\n\tif strings.HasPrefix(corpus, \"<?xml\") {\n\t\treturn matcher.MatchXML(corpus)\n\t}\n\treturn matcher.MatchHTML(corpus)\n}\n\n// MatchHTML matches items from HTML using XPath selectors\nfunc (matcher *Matcher) MatchHTML(corpus string) bool {\n\tdoc, err := htmlquery.Parse(strings.NewReader(corpus))\n\tif err != nil {\n\t\treturn false\n\t}\n\n\tmatches := 0\n\n\tfor _, k := range matcher.XPath {\n\t\tnodes, err := htmlquery.QueryAll(doc, k)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Continue if the xpath doesn't return any nodes\n\t\tif len(nodes) == 0 {\n\t\t\t// If we are in an AND request and a match failed,\n\t\t\t// return false as the AND condition fails on any single mismatch.\n\t\t\tswitch matcher.condition {\n\t\t\tcase ANDCondition:\n\t\t\t\treturn false\n\t\t\tcase ORCondition:\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// If the condition was an OR, return on the first match.\n\t\tif matcher.condition == ORCondition && !matcher.MatchAll {\n\t\t\treturn true\n\t\t}\n\n\t\tmatches = matches + len(nodes)\n\t}\n\treturn matches > 0\n}\n\n// MatchXML matches items from XML using XPath selectors\nfunc (matcher *Matcher) MatchXML(corpus string) bool {\n\tdoc, err := xmlquery.Parse(strings.NewReader(corpus))\n\tif err != nil {\n\t\treturn false\n\t}\n\n\tmatches := 0\n\n\tfor _, k := range matcher.XPath {\n\t\tnodes, err := xmlquery.QueryAll(doc, k)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Continue if the xpath doesn't return any nodes\n\t\tif len(nodes) == 0 {\n\t\t\t// If we are in an AND request and a match failed,\n\t\t\t// return false as the AND condition fails on any single mismatch.\n\t\t\tswitch matcher.condition {\n\t\t\tcase ANDCondition:\n\t\t\t\treturn false\n\t\t\tcase ORCondition:\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// If the condition was an OR, return on the first match.\n\t\tif matcher.condition == ORCondition && !matcher.MatchAll {\n\t\t\treturn true\n\t\t}\n\t\tmatches = matches + len(nodes)\n\t}\n\n\treturn matches > 0\n}\n\n// ignoreErr checks if the error is to be ignored or not\n// Reference: https://github.com/projectdiscovery/nuclei/issues/3950\nfunc (m *Matcher) ignoreErr(err error) bool {\n\tif showDSLErr {\n\t\treturn false\n\t}\n\tif stringsutil.ContainsAny(err.Error(), \"No parameter\", dslRepo.ErrParsingArg.Error()) {\n\t\treturn true\n\t}\n\treturn false\n}\n\n// Path: pkg/operators/matchers/match_test.go\npackage matchers\n\nimport (\n\t\"testing\"\n\n\t\"github.com/Knetic/govaluate\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestWordANDCondition(t *testing.T) {\n\tm := &Matcher{condition: ANDCondition, Words: []string{\"a\", \"b\"}}\n\n\tisMatched, matched := m.MatchWords(\"a b\", nil)\n\trequire.True(t, isMatched, \"Could not match words with valid AND condition\")\n\trequire.Equal(t, m.Words, matched)\n\n\tisMatched, matched = m.MatchWords(\"b\", nil)\n\trequire.False(t, isMatched, \"Could match words with invalid AND condition\")\n\trequire.Equal(t, []string{}, matched)\n}\n\nfunc TestRegexANDCondition(t *testing.T) {\n\tm := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: \"and\", Regex: []string{\"[a-z]{3}\", \"\\\\d{2}\"}}\n\terr := m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched, matched := m.MatchRegex(\"abc abcd 123\")\n\trequire.True(t, isMatched, \"Could not match regex with valid AND condition\")\n\trequire.Equal(t, []string{\"abc\", \"abc\", \"12\"}, matched)\n\n\tisMatched, matched = m.MatchRegex(\"bc 1\")\n\trequire.False(t, isMatched, \"Could match regex with invalid AND condition\")\n\trequire.Equal(t, []string{}, matched)\n}\n\nfunc TestORCondition(t *testing.T) {\n\tm := &Matcher{condition: ORCondition, Words: []string{\"a\", \"b\"}}\n\n\tisMatched, matched := m.MatchWords(\"a b\", nil)\n\trequire.True(t, isMatched, \"Could not match valid word OR condition\")\n\trequire.Equal(t, []string{\"a\"}, matched)\n\n\tisMatched, matched = m.MatchWords(\"b\", nil)\n\trequire.True(t, isMatched, \"Could not match valid word OR condition\")\n\trequire.Equal(t, []string{\"b\"}, matched)\n\n\tisMatched, matched = m.MatchWords(\"c\", nil)\n\trequire.False(t, isMatched, \"Could match invalid word OR condition\")\n\trequire.Equal(t, []string{}, matched)\n}\n\nfunc TestRegexOrCondition(t *testing.T) {\n\tm := &Matcher{Type: MatcherTypeHolder{MatcherType: RegexMatcher}, Condition: \"or\", Regex: []string{\"[a-z]{3}\", \"\\\\d{2}\"}}\n\terr := m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched, matched := m.MatchRegex(\"ab 123\")\n\trequire.True(t, isMatched, \"Could not match valid regex OR condition\")\n\trequire.Equal(t, []string{\"12\"}, matched)\n\n\tisMatched, matched = m.MatchRegex(\"bc 1\")\n\trequire.False(t, isMatched, \"Could match invalid regex OR condition\")\n\trequire.Equal(t, []string{}, matched)\n}\n\nfunc TestHexEncoding(t *testing.T) {\n\tm := &Matcher{Encoding: \"hex\", Type: MatcherTypeHolder{MatcherType: WordsMatcher}, Part: \"body\", Words: []string{\"50494e47\"}}\n\terr := m.CompileMatchers()\n\trequire.Nil(t, err, \"could not compile matcher\")\n\n\tisMatched, matched := m.MatchWords(\"PING\", nil)\n\trequire.True(t, isMatched, \"Could not match valid Hex condition\")\n\trequire.Equal(t, m.Words, matched)\n}\n\nfunc TestMatcher_MatchDSL(t *testing.T) {\n\tcompiled, err := govaluate.NewEvaluableExpressionWithFunctions(\"contains(body, \\\"{{VARIABLE}}\\\")\", dsl.HelperFunctions)\n\trequire.Nil(t, err, \"couldn't compile expression\")\n\n\tm := &Matcher{Type: MatcherTypeHolder{MatcherType: DSLMatcher}, dslCompiled: []*govaluate.EvaluableExpression{compiled}}\n\terr = m.CompileMatchers()\n\trequire.Nil(t, err, \"could not compile matcher\")\n\n\tvalues := []string{\"PING\", \"pong\"}\n\n\tfor value := range values {\n\t\tisMatched := m.MatchDSL(map[string]interface{}{\"body\": value, \"VARIABLE\": value})\n\t\trequire.True(t, isMatched)\n\t}\n}\n\nfunc TestMatcher_MatchXPath_HTML(t *testing.T) {\n\tbody := `<!doctype html>\n<html>\n<head>\n    <title>Example Domain</title>\n\n    <meta charset=\"utf-8\" />\n    <meta http-equiv=\"Content-type\" content=\"text/html; charset=utf-8\" />\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n</head>\n\n<body>\n<div>\n    <h1>Example Domain</h1>\n    <p>This domain is for use in illustrative examples in documents. You may use this\n    domain in literature without prior coordination or asking for permission.</p>\n    <p><a href=\"https://www.iana.org/domains/example\">More information...</a></p>\n</div>\n</body>\n</html>\n`\n\tbody2 := `<!doctype html>\n<html>\n<head>\n    <title>Example Domain</title>\n</head>\n<body>\n<h1> It's test time! </h1>\n</body>\n</html>\n`\n\n\t// single match\n\tm := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{\"/html/body/div/p[2]/a\"}}\n\terr := m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched := m.MatchXPath(body)\n\trequire.True(t, isMatched, \"Could not match valid XPath\")\n\n\tisMatched = m.MatchXPath(\"<h1>aaaaaaaaa\")\n\trequire.False(t, isMatched, \"Could match invalid XPath\")\n\n\t// OR match\n\tm = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: \"or\", XPath: []string{\"/html/head/title[contains(text(), 'PATRICAAA')]\", \"/html/body/div/p[2]/a\"}}\n\terr = m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched = m.MatchXPath(body)\n\trequire.True(t, isMatched, \"Could not match valid multi-XPath with OR condition\")\n\n\tisMatched = m.MatchXPath(body2)\n\trequire.False(t, isMatched, \"Could match invalid multi-XPath with OR condition\")\n\n\t// AND match\n\tm = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: \"and\", XPath: []string{\"/html/head/title[contains(text(), 'Example Domain')]\", \"/html/body/div/p[2]/a\"}}\n\terr = m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched = m.MatchXPath(body)\n\trequire.True(t, isMatched, \"Could not match valid multi-XPath with AND condition\")\n\n\tisMatched = m.MatchXPath(body2)\n\trequire.False(t, isMatched, \"Could match invalid multi-XPath with AND condition\")\n\n\t// invalid xpath\n\tm = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{\"//a[@a==1]\"}}\n\t_ = m.CompileMatchers()\n\tisMatched = m.MatchXPath(body)\n\trequire.False(t, isMatched, \"Invalid xpath did not return false\")\n}\n\nfunc TestMatcher_MatchXPath_XML(t *testing.T) {\n\tbody := `<?xml version=\"1.0\" encoding=\"utf-8\"?><foo>bar</foo><wibble id=\"1\" /><parent><child>baz</child></parent>`\n\tbody2 := `<?xml version=\"1.0\" encoding=\"utf-8\"?><test>bar</test><wibble2 id=\"1\" /><roditelj><dijete>alo</dijete></roditelj>`\n\n\t// single match\n\tm := &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{\"//foo[contains(text(), 'bar')]\"}}\n\terr := m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched := m.MatchXPath(body)\n\trequire.True(t, isMatched, \"Could not match valid XPath\")\n\n\tisMatched = m.MatchXPath(\"<h1>aaaaaaaaa</h1>\")\n\trequire.False(t, isMatched, \"Could match invalid XPath\")\n\n\t// OR match\n\tm = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: \"or\", XPath: []string{\"/foo[contains(text(), 'PATRICAAA')]\", \"/parent/child\"}}\n\terr = m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched = m.MatchXPath(body)\n\trequire.True(t, isMatched, \"Could not match valid multi-XPath with OR condition\")\n\n\tisMatched = m.MatchXPath(body2)\n\trequire.False(t, isMatched, \"Could match invalid multi-XPath with OR condition\")\n\n\t// AND match\n\tm = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, Condition: \"and\", XPath: []string{\"/foo[contains(text(), 'bar')]\", \"/parent/child\"}}\n\terr = m.CompileMatchers()\n\trequire.Nil(t, err)\n\n\tisMatched = m.MatchXPath(body)\n\trequire.True(t, isMatched, \"Could not match valid multi-XPath with AND condition\")\n\n\tisMatched = m.MatchXPath(body2)\n\trequire.False(t, isMatched, \"Could match invalid multi-XPath with AND condition\")\n\n\t// invalid xpath\n\tm = &Matcher{Type: MatcherTypeHolder{MatcherType: XPathMatcher}, XPath: []string{\"//a[@a==1]\"}}\n\t_ = m.CompileMatchers()\n\tisMatched = m.MatchXPath(body)\n\trequire.False(t, isMatched, \"Invalid xpath did not return false\")\n\n\t// invalid xml\n\tisMatched = m.MatchXPath(\"<h1> not right <q id=2/>notvalid\")\n\trequire.False(t, isMatched, \"Invalid xpath did not return false\")\n}\n\n// Path: pkg/operators/matchers/matchers_types.go\npackage matchers\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"strings\"\n\n\t\"github.com/invopop/jsonschema\"\n)\n\n// MatcherType is the type of the matcher specified\ntype MatcherType int\n\n// name:MatcherType\nconst (\n\t// name:word\n\tWordsMatcher MatcherType = iota + 1\n\t// name:regex\n\tRegexMatcher\n\t// name:binary\n\tBinaryMatcher\n\t// name:status\n\tStatusMatcher\n\t// name:size\n\tSizeMatcher\n\t// name:dsl\n\tDSLMatcher\n\t// name:xpath\n\tXPathMatcher\n\tlimit\n)\n\n// MatcherTypes is a table for conversion of matcher type from string.\nvar MatcherTypes = map[MatcherType]string{\n\tStatusMatcher: \"status\",\n\tSizeMatcher:   \"size\",\n\tWordsMatcher:  \"word\",\n\tRegexMatcher:  \"regex\",\n\tBinaryMatcher: \"binary\",\n\tDSLMatcher:    \"dsl\",\n\tXPathMatcher:  \"xpath\",\n}\n\n// GetType returns the type of the matcher\nfunc (matcher *Matcher) GetType() MatcherType {\n\treturn matcher.Type.MatcherType\n}\n\n// GetSupportedMatcherTypes returns list of supported types\nfunc GetSupportedMatcherTypes() []MatcherType {\n\tvar result []MatcherType\n\tfor index := MatcherType(1); index < limit; index++ {\n\t\tresult = append(result, index)\n\t}\n\treturn result\n}\n\nfunc toMatcherTypes(valueToMap string) (MatcherType, error) {\n\tnormalizedValue := normalizeValue(valueToMap)\n\tfor key, currentValue := range MatcherTypes {\n\t\tif normalizedValue == currentValue {\n\t\t\treturn key, nil\n\t\t}\n\t}\n\treturn -1, errors.New(\"Invalid matcher type: \" + valueToMap)\n}\n\nfunc normalizeValue(value string) string {\n\treturn strings.TrimSpace(strings.ToLower(value))\n}\n\nfunc (t MatcherType) String() string {\n\treturn MatcherTypes[t]\n}\n\n// MatcherTypeHolder is used to hold internal type of the matcher\ntype MatcherTypeHolder struct {\n\tMatcherType MatcherType `mapping:\"true\"`\n}\n\nfunc (t MatcherTypeHolder) String() string {\n\treturn t.MatcherType.String()\n}\n\nfunc (holder MatcherTypeHolder) JSONSchema() *jsonschema.Schema {\n\tgotType := &jsonschema.Schema{\n\t\tType:        \"string\",\n\t\tTitle:       \"type of the matcher\",\n\t\tDescription: \"Type of the matcher\",\n\t}\n\tfor _, types := range GetSupportedMatcherTypes() {\n\t\tgotType.Enum = append(gotType.Enum, types.String())\n\t}\n\treturn gotType\n}\n\nfunc (holder *MatcherTypeHolder) UnmarshalYAML(unmarshal func(interface{}) error) error {\n\tvar marshalledTypes string\n\tif err := unmarshal(&marshalledTypes); err != nil {\n\t\treturn err\n\t}\n\n\tcomputedType, err := toMatcherTypes(marshalledTypes)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tholder.MatcherType = computedType\n\treturn nil\n}\n\nfunc (holder *MatcherTypeHolder) UnmarshalJSON(data []byte) error {\n\ts := strings.Trim(string(data), `\"`)\n\tif s == \"\" {\n\t\treturn nil\n\t}\n\tcomputedType, err := toMatcherTypes(s)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tholder.MatcherType = computedType\n\treturn nil\n}\n\nfunc (holder MatcherTypeHolder) MarshalJSON() ([]byte, error) {\n\treturn json.Marshal(holder.MatcherType.String())\n}\n\nfunc (holder MatcherTypeHolder) MarshalYAML() (interface{}, error) {\n\treturn holder.MatcherType.String(), nil\n}\n\n// Path: pkg/operators/matchers/validate_test.go\npackage matchers\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestValidate(t *testing.T) {\n\tm := &Matcher{matcherType: DSLMatcher, DSL: []string{\"anything\"}}\n\n\terr := m.Validate()\n\trequire.Nil(t, err, \"Could not validate correct template\")\n\n\tm = &Matcher{matcherType: DSLMatcher, Part: \"test\"}\n\terr = m.Validate()\n\trequire.NotNil(t, err, \"Invalid template was correctly validated\")\n\n\tm = &Matcher{matcherType: XPathMatcher, XPath: []string{\"//q[@id=\\\"foo\\\"]\"}}\n\n\terr = m.Validate()\n\trequire.Nil(t, err, \"Could not validate correct XPath template\")\n\n\tm = &Matcher{matcherType: XPathMatcher, Status: []int{123}}\n\terr = m.Validate()\n\trequire.NotNil(t, err, \"Invalid XPath template was correctly validated\")\n\n\tm = &Matcher{matcherType: XPathMatcher, XPath: []string{\"//a[@a==1]\"}}\n\terr = m.Validate()\n\trequire.NotNil(t, err, \"Invalid XPath query was correctly validated\")\n}\n\n// Path: pkg/operators/operators_test.go\npackage operators\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestMakeDynamicValuesCallback(t *testing.T) {\n\tinput := map[string][]string{\n\t\t\"a\": {\"1\", \"2\"},\n\t\t\"b\": {\"3\"},\n\t\t\"c\": {},\n\t\t\"d\": {\"A\", \"B\", \"C\"},\n\t}\n\n\tcount := 0\n\tMakeDynamicValuesCallback(input, true, func(data map[string]interface{}) bool {\n\t\tcount++\n\t\trequire.Len(t, data, 3, \"could not get correct output length\")\n\t\treturn false\n\t})\n\trequire.Equal(t, 3, count, \"could not get correct result count\")\n\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tinput := map[string][]string{\n\t\t\t\"a\": {\"1\"},\n\t\t\t\"b\": {\"2\"},\n\t\t\t\"c\": {\"3\"},\n\t\t}\n\n\t\tcount := 0\n\t\tMakeDynamicValuesCallback(input, true, func(data map[string]interface{}) bool {\n\t\t\tcount++\n\t\t\trequire.Len(t, data, 3, \"could not get correct output length\")\n\t\t\treturn false\n\t\t})\n\t\trequire.Equal(t, 1, count, \"could not get correct result count\")\n\t})\n\n\tt.Run(\"first\", func(t *testing.T) {\n\t\tinput := map[string][]string{\n\t\t\t\"a\": {\"1\", \"2\"},\n\t\t\t\"b\": {\"3\"},\n\t\t\t\"c\": {},\n\t\t\t\"d\": {\"A\", \"B\", \"C\"},\n\t\t}\n\n\t\tcount := 0\n\t\tMakeDynamicValuesCallback(input, false, func(data map[string]interface{}) bool {\n\t\t\tcount++\n\t\t\trequire.Len(t, data, 3, \"could not get correct output length\")\n\t\t\treturn false\n\t\t})\n\t\trequire.Equal(t, 1, count, \"could not get correct result count\")\n\t})\n}\n\n// Path: pkg/output/doc.go\n// Package output implements output writing interfaces for nuclei.\npackage output\n\n// Path: pkg/output/file_output_writer.go\npackage output\n\nimport (\n\t\"os\"\n\t\"sync\"\n)\n\n// fileWriter is a concurrent file based output writer.\ntype fileWriter struct {\n\tfile *os.File\n\tmu   sync.Mutex\n}\n\n// NewFileOutputWriter creates a new buffered writer for a file\nfunc newFileOutputWriter(file string, resume bool) (*fileWriter, error) {\n\tvar output *os.File\n\tvar err error\n\tif resume {\n\t\toutput, err = os.OpenFile(file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)\n\t} else {\n\t\toutput, err = os.Create(file)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &fileWriter{file: output}, nil\n}\n\n// WriteString writes an output to the underlying file\nfunc (w *fileWriter) Write(data []byte) (int, error) {\n\tw.mu.Lock()\n\tdefer w.mu.Unlock()\n\tif _, err := w.file.Write(data); err != nil {\n\t\treturn 0, err\n\t}\n\tif _, err := w.file.Write([]byte(\"\\n\")); err != nil {\n\t\treturn 0, err\n\t}\n\treturn len(data) + 1, nil\n}\n\n// Close closes the underlying writer flushing everything to disk\nfunc (w *fileWriter) Close() error {\n\tw.mu.Lock()\n\tdefer w.mu.Unlock()\n\t//nolint:errcheck // we don't care whether sync failed or succeeded.\n\tw.file.Sync()\n\treturn w.file.Close()\n}\n\n// Path: pkg/output/format_json.go\npackage output\n\nimport (\n\tjsoniter \"github.com/json-iterator/go\"\n)\n\n// formatJSON formats the output for json based formatting\nfunc (w *StandardWriter) formatJSON(output *ResultEvent) ([]byte, error) {\n\tif !w.jsonReqResp { // don't show request-response in json if not asked\n\t\toutput.Request = \"\"\n\t\toutput.Response = \"\"\n\t}\n\treturn jsoniter.Marshal(output)\n}\n\n// Path: pkg/output/format_screen.go\npackage output\n\nimport (\n\t\"bytes\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\tmapsutil \"github.com/projectdiscovery/utils/maps\"\n)\n\n// formatScreen formats the output for showing on screen.\nfunc (w *StandardWriter) formatScreen(output *ResultEvent) []byte {\n\tbuilder := &bytes.Buffer{}\n\n\tif !w.noMetadata {\n\t\tif w.timestamp {\n\t\t\tbuilder.WriteRune('[')\n\t\t\tbuilder.WriteString(w.aurora.Cyan(output.Timestamp.Format(\"2006-01-02 15:04:05\")).String())\n\t\t\tbuilder.WriteString(\"] \")\n\t\t}\n\t\tbuilder.WriteRune('[')\n\t\tbuilder.WriteString(w.aurora.BrightGreen(output.TemplateID).String())\n\n\t\tif output.MatcherName != \"\" {\n\t\t\tbuilder.WriteString(\":\")\n\t\t\tbuilder.WriteString(w.aurora.BrightGreen(output.MatcherName).Bold().String())\n\t\t} else if output.ExtractorName != \"\" {\n\t\t\tbuilder.WriteString(\":\")\n\t\t\tbuilder.WriteString(w.aurora.BrightGreen(output.ExtractorName).Bold().String())\n\t\t}\n\n\t\tif w.matcherStatus {\n\t\t\tbuilder.WriteString(\"] [\")\n\t\t\tif !output.MatcherStatus {\n\t\t\t\tbuilder.WriteString(w.aurora.Red(\"failed\").String())\n\t\t\t} else {\n\t\t\t\tbuilder.WriteString(w.aurora.Green(\"matched\").String())\n\t\t\t}\n\t\t}\n\n\t\tbuilder.WriteString(\"] [\")\n\t\tbuilder.WriteString(w.aurora.BrightBlue(output.Type).String())\n\t\tbuilder.WriteString(\"] \")\n\n\t\tbuilder.WriteString(\"[\")\n\t\tbuilder.WriteString(w.severityColors(output.Info.SeverityHolder.Severity))\n\t\tbuilder.WriteString(\"] \")\n\t}\n\tif output.Matched != \"\" {\n\t\tbuilder.WriteString(output.Matched)\n\t} else {\n\t\tbuilder.WriteString(output.Host)\n\t}\n\n\t// If any extractors, write the results\n\tif len(output.ExtractedResults) > 0 {\n\t\tbuilder.WriteString(\" [\")\n\n\t\tfor i, item := range output.ExtractedResults {\n\t\t\t// trim trailing space\n\t\t\t// quote non-ascii and non printable characters and then\n\t\t\t// unquote quotes (`\"`) for readability\n\t\t\titem = strings.TrimSpace(item)\n\t\t\titem = strconv.QuoteToASCII(item)\n\t\t\titem = strings.ReplaceAll(item, `\\\"`, `\"`)\n\n\t\t\tbuilder.WriteString(w.aurora.BrightCyan(item).String())\n\n\t\t\tif i != len(output.ExtractedResults)-1 {\n\t\t\t\tbuilder.WriteRune(',')\n\t\t\t}\n\t\t}\n\t\tbuilder.WriteString(\"]\")\n\t}\n\n\tif len(output.Lines) > 0 {\n\t\tbuilder.WriteString(\" [LN: \")\n\n\t\tfor i, line := range output.Lines {\n\t\t\tbuilder.WriteString(strconv.Itoa(line))\n\n\t\t\tif i != len(output.Lines)-1 {\n\t\t\t\tbuilder.WriteString(\",\")\n\t\t\t}\n\t\t}\n\t\tbuilder.WriteString(\"]\")\n\t}\n\n\t// Write meta if any\n\tif len(output.Metadata) > 0 {\n\t\tbuilder.WriteString(\" [\")\n\n\t\tfirst := true\n\t\t// sort to get predictable output\n\t\tfor _, name := range mapsutil.GetSortedKeys(output.Metadata) {\n\t\t\tvalue := output.Metadata[name]\n\t\t\tif !first {\n\t\t\t\tbuilder.WriteRune(',')\n\t\t\t}\n\t\t\tfirst = false\n\n\t\t\tbuilder.WriteString(w.aurora.BrightYellow(name).String())\n\t\t\tbuilder.WriteRune('=')\n\t\t\tbuilder.WriteString(w.aurora.BrightYellow(strconv.QuoteToASCII(types.ToString(value))).String())\n\t\t}\n\t\tbuilder.WriteString(\"]\")\n\t}\n\treturn builder.Bytes()\n}\n\n// Path: pkg/output/multi_writer.go\npackage output\n\nimport (\n\t\"github.com/logrusorgru/aurora\"\n)\n\ntype MultiWriter struct {\n\twriters []Writer\n}\n\n// NewMultiWriter creates a new MultiWriter instance\nfunc NewMultiWriter(writers ...Writer) *MultiWriter {\n\treturn &MultiWriter{writers: writers}\n}\n\nfunc (mw *MultiWriter) Close() {\n\tfor _, writer := range mw.writers {\n\t\twriter.Close()\n\t}\n}\n\nfunc (mw *MultiWriter) Colorizer() aurora.Aurora {\n\t// Return the colorizer of the first writer\n\tif len(mw.writers) > 0 {\n\t\treturn mw.writers[0].Colorizer()\n\t}\n\t// Default to a no-op colorizer\n\treturn aurora.NewAurora(false)\n}\n\nfunc (mw *MultiWriter) Write(event *ResultEvent) error {\n\tfor _, writer := range mw.writers {\n\t\tif err := writer.Write(event); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (mw *MultiWriter) WriteFailure(event *InternalWrappedEvent) error {\n\tfor _, writer := range mw.writers {\n\t\tif err := writer.WriteFailure(event); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (mw *MultiWriter) Request(templateID, url, requestType string, err error) {\n\tfor _, writer := range mw.writers {\n\t\twriter.Request(templateID, url, requestType, err)\n\t}\n}\n\nfunc (mw *MultiWriter) WriteStoreDebugData(host, templateID, eventType string, data string) {\n\tfor _, writer := range mw.writers {\n\t\twriter.WriteStoreDebugData(host, templateID, eventType, data)\n\t}\n}\n\n// Path: pkg/output/output_test.go\npackage output\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\t\"github.com/stretchr/testify/require\"\n)\n\n\nfunc TestStandardWriterRequest(t *testing.T) {\n\tt.Run(\"WithoutTraceAndError\", func(t *testing.T) {\n\t\tw, err := NewStandardWriter(&types.Options{})\n\t\trequire.NoError(t, err)\n\t\trequire.NotPanics(t, func() {\n\t\t\tw.Request(\"path\", \"input\", \"http\", nil)\n\t\t\tw.Close()\n\t\t})\n\t})\n\n\tt.Run(\"TraceAndErrorWithoutError\", func(t *testing.T) {\n\t\ttraceWriter := &testWriteCloser{}\n\t\terrorWriter := &testWriteCloser{}\n\n\t\tw, err := NewStandardWriter(&types.Options{})\n\t\tw.traceFile = traceWriter\n\t\tw.errorFile = errorWriter\n\t\trequire.NoError(t, err)\n\t\tw.Request(\"path\", \"input\", \"http\", nil)\n\n\t\trequire.Equal(t, `{\"template\":\"path\",\"input\":\"input\",\"error\":\"none\",\"type\":\"http\"}`, traceWriter.String())\n\t\trequire.Empty(t, errorWriter.String())\n\t})\n\n\tt.Run(\"ErrorWithWrappedError\", func(t *testing.T) {\n\t\terrorWriter := &testWriteCloser{}\n\n\t\tw, err := NewStandardWriter(&types.Options{})\n\t\tw.errorFile = errorWriter\n\t\trequire.NoError(t, err)\n\t\tw.Request(\n\t\t\t\"misconfiguration/tcpconfig.yaml\",\n\t\t\t\"https://example.com/tcpconfig.html\",\n\t\t\t\"http\",\n\t\t\tfmt.Errorf(\"GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w\", errors.New(\"context deadline exceeded (Client.Timeout exceeded while awaiting headers)\")),\n\t\t)\n\n\t\trequire.Equal(t, `{\"template\":\"misconfiguration/tcpconfig.yaml\",\"input\":\"https://example.com/tcpconfig.html\",\"error\":\"context deadline exceeded (Client.Timeout exceeded while awaiting headers)\",\"type\":\"http\"}`, errorWriter.String())\n\t})\n}\n\ntype testWriteCloser struct {\n\tstrings.Builder\n}\n\nfunc (w testWriteCloser) Close() error {\n\treturn nil\n}\n\n// Path: pkg/output/standard_writer.go\npackage output\n\nimport (\n\t\"io\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/logrusorgru/aurora\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity\"\n\tfileutil \"github.com/projectdiscovery/utils/file\"\n)\n\n// WriterOptions contains configuration options for a writer\ntype WriterOptions func(s *StandardWriter) error\n\n// WithJson writes output in json format\nfunc WithJson(json bool, dumpReqResp bool) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.json = json\n\t\ts.jsonReqResp = dumpReqResp\n\t\treturn nil\n\t}\n}\n\n// WithTimestamp writes output with timestamp\nfunc WithTimestamp(timestamp bool) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.timestamp = timestamp\n\t\treturn nil\n\t}\n}\n\n// WithNoMetadata disables metadata output\nfunc WithNoMetadata(noMetadata bool) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.noMetadata = noMetadata\n\t\treturn nil\n\t}\n}\n\n// WithMatcherStatus writes output with matcher status\nfunc WithMatcherStatus(matcherStatus bool) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.matcherStatus = matcherStatus\n\t\treturn nil\n\t}\n}\n\n// WithAurora sets the aurora instance for the writer\nfunc WithAurora(aurora aurora.Aurora) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.aurora = aurora\n\t\treturn nil\n\t}\n}\n\n// WithWriter sets the writer for the writer\nfunc WithWriter(outputFile io.WriteCloser) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.outputFile = outputFile\n\t\treturn nil\n\t}\n}\n\n// WithTraceSink sets the writer where trace output is written\nfunc WithTraceSink(traceFile io.WriteCloser) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.traceFile = traceFile\n\t\treturn nil\n\t}\n}\n\n// WithErrorSink sets the writer where error output is written\nfunc WithErrorSink(errorFile io.WriteCloser) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.errorFile = errorFile\n\t\treturn nil\n\t}\n}\n\n// WithSeverityColors sets the color function for severity\nfunc WithSeverityColors(severityColors func(severity.Severity) string) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.severityColors = severityColors\n\t\treturn nil\n\t}\n}\n\n// WithStoreResponse sets the store response option\nfunc WithStoreResponse(storeResponse bool, respDir string) WriterOptions {\n\treturn func(s *StandardWriter) error {\n\t\ts.storeResponse = storeResponse\n\t\ts.storeResponseDir = respDir\n\t\treturn nil\n\t}\n}\n\n// NewWriter creates a new output writer\n// if no writer is specified it writes to stdout\nfunc NewWriter(opts ...WriterOptions) (*StandardWriter, error) {\n\ts := &StandardWriter{\n\t\tmutex:                 &sync.Mutex{},\n\t\tDisableStdout:         true,\n\t\tAddNewLinesOutputFile: true,\n\t}\n\tfor _, opt := range opts {\n\t\tif err := opt(s); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tif s.aurora == nil {\n\t\ts.aurora = aurora.NewAurora(false)\n\t}\n\tif s.outputFile == nil {\n\t\ts.outputFile = os.Stdout\n\t}\n\t// Try to create output folder if it doesn't exist\n\tif s.storeResponse && !fileutil.FolderExists(s.storeResponseDir) {\n\t\tif err := fileutil.CreateFolder(s.storeResponseDir); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn s, nil\n}\n\n// Path: pkg/progress/doc.go\n// Package progress implements progress display mechanism with very\n// simple command line statistics printing on runtime.\npackage progress\n\n// Path: pkg/protocols/code/code_test.go\n//go:build linux || darwin\n\npackage code\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/testutils\"\n)\n\nfunc TestCodeProtocol(t *testing.T) {\n\toptions := testutils.DefaultOptions\n\n\ttestutils.Init(options)\n\ttemplateID := \"testing-code\"\n\trequest := &Request{\n\t\tEngine: []string{\"sh\"},\n\t\tSource: \"echo test\",\n\t}\n\texecuterOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{\n\t\tID:   templateID,\n\t\tInfo: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: \"test\"},\n\t})\n\terr := request.Compile(executerOpts)\n\trequire.Nil(t, err, \"could not compile code request\")\n\n\tvar gotEvent output.InternalEvent\n\tctxArgs := contextargs.NewWithInput(\"\")\n\terr = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {\n\t\tgotEvent = event.InternalEvent\n\t})\n\trequire.Nil(t, err, \"could not run code request\")\n\trequire.NotEmpty(t, gotEvent, \"could not get event items\")\n}\n\n// Path: pkg/protocols/code/helpers.go\npackage code\n\nimport (\n\tgoruntime \"runtime\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs\"\n\tosutils \"github.com/projectdiscovery/utils/os\"\n)\n\n// registerPreConditionFunctions registers the pre-condition functions\nfunc registerPreConditionFunctions(runtime *goja.Runtime) error {\n\t// Note: the only reason we are not using forloop to generate these functions is because\n\t// 'scrapefuncs' uses this function to find all dsl helper functions  and document them.\n\n\t// === OS ===\n\terr := gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"OS\",\n\t\tSignatures: []string{\n\t\t\t\"OS() string\",\n\t\t},\n\t\tDescription: \"OS returns the current OS\",\n\t\tFuncDecl: func() string {\n\t\t\treturn goruntime.GOOS\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// IsLinux checks if the current OS is Linux\n\terr = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{\n\t\tName: \"IsLinux\",\n\t\tSignatures: []string{\n\t\t\t\"IsLinux() bool\",\n\t\t},\n\t\tDescription: \"IsLinux checks if the current OS is Linux\",\n\t\tFuncDecl: func() bool {\n\t\t\treturn osutils.IsLinux()\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// IsWindows checks if the current OS is Windows\n...\n", "needle_token_start": 12627, "needle_token_end": 13149, "code_context_ntokens": 15171, "output": ["```go\nfunc TestStandardWriterRequest(t *testing.T) {\n\tt.Run(\"WithoutTraceAndError\", func(t *testing.T) {\n\t\tw, err := NewStandardWriter(&types.Options{})\n\t\trequire.NoError(t, err)\n\t\trequire.NotPanics(t, func() {\n\t\t\tw.Request(\"path\", \"input\", \"http\", nil)\n\t\t\tw.Close()\n\t\t})\n\t})\n\n\tt.Run(\"TraceAndErrorWithoutError\", func(t *testing.T) {\n\t\ttraceWriter := &testWriteCloser{}\n\t\terrorWriter := &testWriteCloser{}\n\n\t\tw, err := NewStandardWriter(&types.Options{})\n\t\tw.traceFile = traceWriter\n\t\tw.errorFile = errorWriter\n\t\trequire.NoError(t, err)\n\t\tw.Request(\"path\", \"input\", \"http\", nil)\n\n\t\trequire.Equal(t, `{\"template\":\"path\",\"input\":\"input\",\"error\":\"none\",\"type\":\"http\"}`, traceWriter.String())\n\t\trequire.Empty(t, errorWriter.String())\n\t})\n\n\tt.Run(\"ErrorWithWrappedError\", func(t *testing.T) {\n\t\terrorWriter := &testWriteCloser{}\n\n\t\tw, err := NewStandardWriter(&types.Options{})\n\t\tw.errorFile = errorWriter\n\t\trequire.NoError(t, err)\n\t\tw.Request(\n\t\t\t\"misconfiguration/tcpconfig.yaml\",\n\t\t\t\"https://example.com/tcpconfig.html\",\n\t\t\t\"http\",\n\t\t\tfmt.Errorf(\"GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w\", errors.New(\"context deadline exceeded (Client.Timeout exceeded while awaiting headers)\")),\n\t\t)\n\n\t\trequire.Equal(t, `{\"template\":\"misconfiguration/tcpconfig.yaml\",\"input\":\"https://example.com/tcpconfig.html\",\"error\":\"context deadline exceeded (Client.Timeout exceeded while awaiting headers)\",\"type\":\"http\"}`, errorWriter.String())\n\t})\n}\n\ntype testWriteCloser struct {\n\tstrings.Builder\n}\n\nfunc (w testWriteCloser) Close() error {\n\treturn nil\n}\n```"]}
{"repo": "projectdiscovery/nuclei", "name": "MergeMapsMany", "language": "go", "path": "pkg/protocols/common/generators/maps.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to combine multiple maps into a single map, where each key maps to a list of strings. It aggregates values from similar keys across all input maps.\n2. **Input**: The function accepts a variable number of map-like structures as input.\n3. **Output**: It returns a new map where each key is associated with a list of strings, representing the combined values from all input maps for that key.\n4. **Procedure**: The function iterates over each input map. For each key-value pair in these maps, it checks the type of the value (string, slice, or array) and appends each value to a list associated with the key in the resulting map. If a key does not exist in the result map, it initializes a new list. The function handles different data types by using reflection to dynamically determine the type and extract values accordingly.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/protocols/http/request.go\npackage http\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\t\"go.uber.org/multierr\"\n\t\"moul.io/http2curl\"\n\n\t\"github.com/projectdiscovery/fastdialer/fastdialer\"\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n\t\"github.com/projectdiscovery/rawhttp\"\n\tconvUtil \"github.com/projectdiscovery/utils/conversion\"\n\terrorutil \"github.com/projectdiscovery/utils/errors\"\n\thttpUtils \"github.com/projectdiscovery/utils/http\"\n\t\"github.com/projectdiscovery/utils/reader\"\n\tsliceutil \"github.com/projectdiscovery/utils/slice\"\n\tstringsutil \"github.com/projectdiscovery/utils/strings\"\n\turlutil \"github.com/projectdiscovery/utils/url\"\n)\n\nconst (\n\tdefaultMaxWorkers = 150\n\t// max unique errors to store & combine\n\t// when executing requests in parallel\n\tmaxErrorsWhenParallel = 3\n)\n\nvar (\n\tMaxBodyRead = int64(1 << 22) // 4MB using shift operator\n\t// ErrMissingVars is error occured when variables are missing\n\tErrMissingVars = errors.New(\"stop execution due to unresolved variables\")\n)\n\n// Type returns the type of the protocol request\nfunc (request *Request) Type() templateTypes.ProtocolType {\n\treturn templateTypes.HTTPProtocol\n}\n\n// executeRaceRequest executes race condition request for a URL\nfunc (request *Request) executeRaceRequest(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\treqURL := input.MetaInput.Input\n\tvar generatedRequests []*generatedRequest\n\n\t// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work\n\t// This will introduce a delay and will populate in hacky way the field \"request\" of outputEvent\n\tgenerator := request.newGenerator(false)\n\n\tinputData, payloads, ok := generator.nextValue()\n\tif !ok {\n\t\treturn nil\n\t}\n\tctx := request.newContext(input)\n\trequestForDump, err := generator.Make(ctx, input, inputData, payloads, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\trequest.setCustomHeaders(requestForDump)\n\tdumpedRequest, err := dump(requestForDump, reqURL)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif request.options.Options.Debug || request.options.Options.DebugRequests || request.options.Options.StoreResponse {\n\t\tmsg := fmt.Sprintf(\"[%s] Dumped HTTP request for %s\\n\\n\", request.options.TemplateID, reqURL)\n\t\tif request.options.Options.Debug || request.options.Options.DebugRequests {\n\t\t\tgologger.Info().Msg(msg)\n\t\t\tgologger.Print().Msgf(\"%s\", string(dumpedRequest))\n\t\t}\n\t\tif request.options.Options.StoreResponse {\n\t\t\trequest.options.Output.WriteStoreDebugData(reqURL, request.options.TemplateID, request.Type().String(), fmt.Sprintf(\"%s\\n%s\", msg, dumpedRequest))\n\t\t}\n\t}\n\tprevious[\"request\"] = string(dumpedRequest)\n\n\t// Pre-Generate requests\n\tfor i := 0; i < request.RaceNumberRequests; i++ {\n\t\tgenerator := request.newGenerator(false)\n\t\tinputData, payloads, ok := generator.nextValue()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tctx := request.newContext(input)\n\t\tgeneratedRequest, err := generator.Make(ctx, input, inputData, payloads, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tgeneratedRequests = append(generatedRequests, generatedRequest)\n\t}\n\n\tshouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)\n\tspmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, maxErrorsWhenParallel, shouldStop)\n\tgotMatches := &atomic.Bool{}\n\t// wrappedCallback is a callback that wraps the original callback\n\t// to implement stop at first match logic\n\twrappedCallback := func(event *output.InternalWrappedEvent) {\n\t\tif !event.HasOperatorResult() {\n\t\t\tcallback(event) // not required but we can allow it\n\t\t\treturn\n\t\t}\n\t\t// this will execute match condition such that if stop at first match is enabled\n\t\t// this will be only executed once\n\t\tspmHandler.MatchCallback(func() {\n\t\t\tgotMatches.Store(true)\n\t\t\tcallback(event)\n\t\t})\n\t\tif shouldStop {\n\t\t\t// stop all running requests and exit\n\t\t\tspmHandler.Trigger()\n\t\t}\n\t}\n\n\t// look for unresponsive hosts and cancel inflight requests as well\n\tspmHandler.SetOnResultCallback(func(err error) {\n\t\tif err == nil {\n\t\t\treturn\n\t\t}\n\t\t// marks thsi host as unresponsive if applicable\n\t\trequest.markUnresponsiveHost(input, err)\n\t\tif request.isUnresponsiveHost(input) {\n\t\t\t// stop all inflight requests\n\t\t\tspmHandler.Cancel()\n\t\t}\n\t})\n\n\tfor i := 0; i < request.RaceNumberRequests; i++ {\n\t\tif spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {\n\t\t\t// stop sending more requests condition is met\n\t\t\tbreak\n\t\t}\n\t\tspmHandler.Acquire()\n\t\t// execute http request\n\t\tgo func(httpRequest *generatedRequest) {\n\t\t\tdefer spmHandler.Release()\n\t\t\tif spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {\n\t\t\t\t// stop sending more requests condition is met\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tselect {\n\t\t\tcase <-spmHandler.Done():\n\t\t\t\treturn\n\t\t\tcase spmHandler.ResultChan <- request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0):\n\t\t\t\treturn\n\t\t\t}\n\t\t}(generatedRequests[i])\n\t\trequest.options.Progress.IncrementRequests()\n\t}\n\tspmHandler.Wait()\n\n\tif spmHandler.FoundFirstMatch() {\n\t\t// ignore any context cancellation and in-transit execution errors\n\t\treturn nil\n\t}\n\treturn multierr.Combine(spmHandler.CombinedResults()...)\n}\n\n// executeRaceRequest executes parallel requests for a template\n...\n// Path: pkg/protocols/offlinehttp/operators.go\npackage offlinehttp\n\nimport (\n\t\"net/http\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/model\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/types\"\n)\n\n// Match matches a generic data response again a given matcher\nfunc (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {\n\titem, ok := getMatchPart(matcher.Part, data)\n\tif !ok && matcher.Type.MatcherType != matchers.DSLMatcher {\n\t\treturn false, []string{}\n\t}\n\n\tswitch matcher.GetType() {\n\tcase matchers.StatusMatcher:\n\t\tstatusCode, ok := getStatusCode(data)\n\t\tif !ok {\n\t\t\treturn false, []string{}\n\t\t}\n\t\treturn matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data[\"response\"].(string), statusCode)}\n\tcase matchers.SizeMatcher:\n\t\treturn matcher.Result(matcher.MatchSize(len(item))), []string{}\n\tcase matchers.WordsMatcher:\n\t\treturn matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, nil))\n\tcase matchers.RegexMatcher:\n\t\treturn matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item))\n\tcase matchers.BinaryMatcher:\n\t\treturn matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))\n\tcase matchers.DSLMatcher:\n\t\treturn matcher.Result(matcher.MatchDSL(data)), []string{}\n\tcase matchers.XPathMatcher:\n\t\treturn matcher.Result(matcher.MatchXPath(item)), []string{}\n\t}\n\treturn false, []string{}\n}\n\nfunc getStatusCode(data map[string]interface{}) (int, bool) {\n\tstatusCodeValue, ok := data[\"status_code\"]\n\tif !ok {\n\t\treturn 0, false\n\t}\n\tstatusCode, ok := statusCodeValue.(int)\n\tif !ok {\n\t\treturn 0, false\n\t}\n\treturn statusCode, true\n}\n\n// Extract performs extracting operation for an extractor on model and returns true or false.\nfunc (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {\n\titem, ok := getMatchPart(extractor.Part, data)\n\tif !ok && !extractors.SupportsMap(extractor) {\n\t\treturn nil\n\t}\n\tswitch extractor.GetType() {\n\tcase extractors.RegexExtractor:\n\t\treturn extractor.ExtractRegex(item)\n\tcase extractors.KValExtractor:\n\t\treturn extractor.ExtractKval(data)\n\tcase extractors.DSLExtractor:\n\t\treturn extractor.ExtractDSL(data)\n\t}\n\treturn nil\n}\n\n// getMatchPart returns the match part honoring \"all\" matchers + others.\nfunc getMatchPart(part string, data output.InternalEvent) (string, bool) {\n\tif part == \"header\" {\n\t\tpart = \"all_headers\"\n\t}\n\tvar itemStr string\n\n\tif part == \"all\" {\n\t\tbuilder := &strings.Builder{}\n\t\tbuilder.WriteString(types.ToString(data[\"body\"]))\n\t\tbuilder.WriteString(types.ToString(data[\"all_headers\"]))\n\t\titemStr = builder.String()\n\t} else {\n\t\titem, ok := data[part]\n\t\tif !ok {\n\t\t\treturn \"\", false\n\t\t}\n\t\titemStr = types.ToString(item)\n\t}\n\treturn itemStr, true\n}\n\n// responseToDSLMap converts an HTTP response to a map for use in DSL matching\nfunc (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent {\n\tdata := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies()))\n\tfor k, v := range extra {\n\t\tdata[k] = v\n\t}\n\tfor _, cookie := range resp.Cookies() {\n\t\tdata[strings.ToLower(cookie.Name)] = cookie.Value\n\t}\n\tfor k, v := range resp.Header {\n\t\tk = strings.ToLower(strings.TrimSpace(k))\n\t\tdata[k] = strings.Join(v, \" \")\n\t}\n\n\tdata[\"path\"] = host\n\tdata[\"matched\"] = matched\n\tdata[\"request\"] = rawReq\n\tdata[\"response\"] = rawResp\n\tdata[\"status_code\"] = resp.StatusCode\n\tdata[\"body\"] = body\n\tdata[\"type\"] = request.Type().String()\n\tdata[\"all_headers\"] = headers\n\tdata[\"duration\"] = duration.Seconds()\n\tdata[\"template-id\"] = request.options.TemplateID\n\tdata[\"template-info\"] = request.options.TemplateInfo\n\tdata[\"template-path\"] = request.options.TemplatePath\n\tdata[\"content_length\"] = utils.CalculateContentLength(resp.ContentLength, int64(len(body)))\n\n\treturn data\n}\n\n// MakeResultEvent creates a result event from internal wrapped event\nfunc (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {\n\treturn protocols.MakeDefaultResultEvent(request, wrapped)\n}\n\nfunc (request *Request) GetCompiledOperators() []*operators.Operators {\n\treturn request.compiledOperators\n}\n\nfunc (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {\n\tdata := &output.ResultEvent{\n\t\tTemplateID:       types.ToString(wrapped.InternalEvent[\"template-id\"]),\n\t\tTemplatePath:     types.ToString(wrapped.InternalEvent[\"template-path\"]),\n\t\tInfo:             wrapped.InternalEvent[\"template-info\"].(model.Info),\n\t\tType:             types.ToString(wrapped.InternalEvent[\"type\"]),\n\t\tPath:             types.ToString(wrapped.InternalEvent[\"path\"]),\n\t\tMatched:          types.ToString(wrapped.InternalEvent[\"matched\"]),\n\t\tMetadata:         wrapped.OperatorsResult.PayloadValues,\n\t\tExtractedResults: wrapped.OperatorsResult.OutputExtracts,\n\t\tMatcherStatus:    true,\n\t\tIP:               types.ToString(wrapped.InternalEvent[\"ip\"]),\n\t\tRequest:          types.ToString(wrapped.InternalEvent[\"request\"]),\n\t\tResponse:         types.ToString(wrapped.InternalEvent[\"raw\"]),\n\t\tTemplateEncoded:  request.options.EncodeTemplate(),\n\t\tError:            types.ToString(wrapped.InternalEvent[\"error\"]),\n\t}\n\treturn data\n}\n\n// Path: pkg/protocols/common/utils/excludematchers/excludematchers.go\npackage excludematchers\n\nimport (\n\t\"strings\"\n)\n\n// ExcludeMatchers is an instance for excluding matchers with template IDs\ntype ExcludeMatchers struct {\n\tvalues       map[string]struct{}\n\ttemplateIDs  map[string]struct{}\n\tmatcherNames map[string]struct{}\n}\n\n// New returns a new exclude matchers instance\n//\n// Wildcard and non-wildcard values are supported.\n// <template-id>:<matcher-name> is the syntax. Wildcards can be specified\n// using * character for either value.\n//\n//\tEx- http-missing-security-headers:* skips all http-missing-security-header templates\nfunc New(values []string) *ExcludeMatchers {\n\texcludeMatchers := &ExcludeMatchers{\n\t\tvalues:       make(map[string]struct{}),\n\t\ttemplateIDs:  make(map[string]struct{}),\n\t\tmatcherNames: make(map[string]struct{}),\n\t}\n\tfor _, value := range values {\n\t\tpartValues := strings.SplitN(value, \":\", 2)\n\t\tif len(partValues) < 2 {\n\t\t\t// If there is no matcher name, consider it as template ID\n\t\t\tif _, ok := excludeMatchers.templateIDs[value]; !ok {\n\t\t\t\texcludeMatchers.templateIDs[value] = struct{}{}\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\ttemplateID, matcherName := partValues[0], partValues[1]\n\n\t\t// Handle wildcards\n\t\tif templateID == \"*\" {\n\t\t\tif _, ok := excludeMatchers.matcherNames[matcherName]; !ok {\n\t\t\t\texcludeMatchers.matcherNames[matcherName] = struct{}{}\n\t\t\t}\n\t\t} else if matcherName == \"*\" {\n\t\t\tif _, ok := excludeMatchers.templateIDs[templateID]; !ok {\n\t\t\t\texcludeMatchers.templateIDs[templateID] = struct{}{}\n\t\t\t}\n\t\t} else {\n\t\t\tif _, ok := excludeMatchers.values[value]; !ok {\n\t\t\t\texcludeMatchers.values[value] = struct{}{}\n\t\t\t}\n\t\t}\n\t}\n\treturn excludeMatchers\n}\n\n// Match returns true if templateID and matcherName matches the blocklist\nfunc (e *ExcludeMatchers) Match(templateID, matcherName string) bool {\n\tif _, ok := e.templateIDs[templateID]; ok {\n\t\treturn true\n\t}\n\tif _, ok := e.matcherNames[matcherName]; ok {\n\t\treturn true\n\t}\n\tmatchName := strings.Join([]string{templateID, matcherName}, \":\")\n\t_, found := e.values[matchName]\n\treturn found\n}\n\n// Path: pkg/operators/operators.go\npackage operators\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers\"\n\tsliceutil \"github.com/projectdiscovery/utils/slice\"\n)\n\n// Operators contains the operators that can be applied on protocols\ntype Operators struct {\n\t// description: |\n\t//   Matchers contains the detection mechanism for the request to identify\n\t//   whether the request was successful by doing pattern matching\n\t//   on request/responses.\n\t//\n\t//   Multiple matchers can be combined with `matcher-condition` flag\n\t//   which accepts either `and` or `or` as argument.\n\tMatchers []*matchers.Matcher `yaml:\"matchers,omitempty\" json:\"matchers,omitempty\" jsonschema:\"title=matchers to run on response,description=Detection mechanism to identify whether the request was successful by doing pattern matching\"`\n\t// description: |\n\t//   Extractors contains the extraction mechanism for the request to identify\n\t//   and extract parts of the response.\n\tExtractors []*extractors.Extractor `yaml:\"extractors,omitempty\" json:\"extractors,omitempty\" jsonschema:\"title=extractors to run on response,description=Extractors contains the extraction mechanism for the request to identify and extract parts of the response\"`\n\t// description: |\n\t//   MatchersCondition is the condition between the matchers. Default is OR.\n\t// values:\n\t//   - \"and\"\n\t//   - \"or\"\n\tMatchersCondition string `yaml:\"matchers-condition,omitempty\" json:\"matchers-condition,omitempty\" jsonschema:\"title=condition between the matchers,description=Conditions between the matchers,enum=and,enum=or\"`\n\t// cached variables that may be used along with request.\n\tmatchersCondition matchers.ConditionType\n\n\t// TemplateID is the ID of the template for matcher\n\tTemplateID string `json:\"-\" yaml:\"-\" jsonschema:\"-\"`\n\t// ExcludeMatchers is a list of excludeMatchers items\n\tExcludeMatchers *excludematchers.ExcludeMatchers `json:\"-\" yaml:\"-\" jsonschema:\"-\"`\n}\n\n// Compile compiles the operators as well as their corresponding matchers and extractors\nfunc (operators *Operators) Compile() error {\n\tif operators.MatchersCondition != \"\" {\n\t\toperators.matchersCondition = matchers.ConditionTypes[operators.MatchersCondition]\n\t} else {\n\t\toperators.matchersCondition = matchers.ORCondition\n\t}\n\n\tfor _, matcher := range operators.Matchers {\n\t\tif err := matcher.CompileMatchers(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile matcher\")\n\t\t}\n\t}\n\tfor _, extractor := range operators.Extractors {\n\t\tif err := extractor.CompileExtractors(); err != nil {\n\t\t\treturn errors.Wrap(err, \"could not compile extractor\")\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (operators *Operators) HasDSL() bool {\n\tfor _, matcher := range operators.Matchers {\n\t\tif len(matcher.DSL) > 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\n\tfor _, extractor := range operators.Extractors {\n\t\tif len(extractor.DSL) > 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\n// GetMatchersCondition returns the condition for the matchers\nfunc (operators *Operators) GetMatchersCondition() matchers.ConditionType {\n\treturn operators.matchersCondition\n}\n\n// Result is a result structure created from operators running on data.\ntype Result struct {\n\t// Matched is true if any matchers matched\n\tMatched bool\n\t// Extracted is true if any result type values were extracted\n\tExtracted bool\n\t// Matches is a map of matcher names that we matched\n\tMatches map[string][]string\n\t// Extracts contains all the data extracted from inputs\n\tExtracts map[string][]string\n\t// OutputExtracts is the list of extracts to be displayed on screen.\n\tOutputExtracts []string\n\toutputUnique   map[string]struct{}\n\n\t// DynamicValues contains any dynamic values to be templated\n\tDynamicValues map[string][]string\n\t// PayloadValues contains payload values provided by user. (Optional)\n\tPayloadValues map[string]interface{}\n\n\t// Optional lineCounts for file protocol\n\tLineCount string\n\t// Operators is reference to operators that generated this result (Read-Only)\n\tOperators *Operators\n}\n\nfunc (result *Result) HasMatch(name string) bool {\n\treturn result.hasItem(name, result.Matches)\n}\n\nfunc (result *Result) HasExtract(name string) bool {\n\treturn result.hasItem(name, result.Extracts)\n}\n\nfunc (result *Result) hasItem(name string, m map[string][]string) bool {\n\tfor matchName := range m {\n\t\tif strings.EqualFold(name, matchName) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n// MakeDynamicValuesCallback takes an input dynamic values map and calls\n// the callback function with all variations of the data in input in form\n// of map[string]string (interface{}).\nfunc MakeDynamicValuesCallback(input map[string][]string, iterateAllValues bool, callback func(map[string]interface{}) bool) {\n\toutput := make(map[string]interface{}, len(input))\n\n\tif !iterateAllValues {\n\t\tfor k, v := range input {\n\t\t\tif len(v) > 0 {\n\t\t\t\toutput[k] = v[0]\n\t\t\t}\n\t\t}\n\t\tcallback(output)\n\t\treturn\n\t}\n\tinputIndex := make(map[string]int, len(input))\n\n\tvar maxValue int\n\tfor _, v := range input {\n\t\tif len(v) > maxValue {\n\t\t\tmaxValue = len(v)\n\t\t}\n\t}\n\n\tfor i := 0; i < maxValue; i++ {\n\t\tfor k, v := range input {\n\t\t\tif len(v) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif len(v) == 1 {\n\t\t\t\toutput[k] = v[0]\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif gotIndex, ok := inputIndex[k]; !ok {\n\t\t\t\tinputIndex[k] = 0\n\t\t\t\toutput[k] = v[0]\n\t\t\t} else {\n\t\t\t\tnewIndex := gotIndex + 1\n\t\t\t\tif newIndex >= len(v) {\n\t\t\t\t\toutput[k] = v[len(v)-1]\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\toutput[k] = v[newIndex]\n\t\t\t\tinputIndex[k] = newIndex\n\t\t\t}\n\t\t}\n\t\t// skip if the callback says so\n\t\tif callback(output) {\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Merge merges a result structure into the other.\nfunc (r *Result) Merge(result *Result) {\n\tif !r.Matched && result.Matched {\n\t\tr.Matched = result.Matched\n\t}\n\tif !r.Extracted && result.Extracted {\n\t\tr.Extracted = result.Extracted\n\t}\n\n\tfor k, v := range result.Matches {\n\t\tr.Matches[k] = sliceutil.Dedupe(append(r.Matches[k], v...))\n\t}\n\tfor k, v := range result.Extracts {\n\t\tr.Extracts[k] = sliceutil.Dedupe(append(r.Extracts[k], v...))\n\t}\n\n\tr.outputUnique = make(map[string]struct{})\n\toutput := r.OutputExtracts\n\tr.OutputExtracts = make([]string, 0, len(output))\n\tfor _, v := range output {\n\t\tif _, ok := r.outputUnique[v]; !ok {\n\t\t\tr.outputUnique[v] = struct{}{}\n\t\t\tr.OutputExtracts = append(r.OutputExtracts, v)\n\t\t}\n\t}\n\tfor _, v := range result.OutputExtracts {\n\t\tif _, ok := r.outputUnique[v]; !ok {\n\t\t\tr.outputUnique[v] = struct{}{}\n\t\t\tr.OutputExtracts = append(r.OutputExtracts, v)\n\t\t}\n\t}\n\tfor k, v := range result.DynamicValues {\n\t\tif _, ok := r.DynamicValues[k]; !ok {\n\t\t\tr.DynamicValues[k] = v\n\t\t} else {\n\t\t\tr.DynamicValues[k] = sliceutil.Dedupe(append(r.DynamicValues[k], v...))\n\t\t}\n\t}\n\tfor k, v := range result.PayloadValues {\n\t\tr.PayloadValues[k] = v\n\t}\n}\n\n// MatchFunc performs matching operation for a matcher on model and returns true or false.\ntype MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string)\n\n// ExtractFunc performs extracting operation for an extractor on model and returns true or false.\ntype ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}\n\n// Execute executes the operators on data and returns a result structure\nfunc (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) {\n\tmatcherCondition := operators.GetMatchersCondition()\n\n\tvar matches bool\n\tresult := &Result{\n\t\tMatches:       make(map[string][]string),\n\t\tExtracts:      make(map[string][]string),\n\t\tDynamicValues: make(map[string][]string),\n\t\toutputUnique:  make(map[string]struct{}),\n\t\tOperators:     operators,\n\t}\n\n\t// state variable to check if all extractors are internal\n\tvar allInternalExtractors bool = true\n\n\t// Start with the extractors first and evaluate them.\n\tfor _, extractor := range operators.Extractors {\n\t\tif !extractor.Internal && allInternalExtractors {\n\t\t\tallInternalExtractors = false\n\t\t}\n\t\tvar extractorResults []string\n\t\tfor match := range extract(data, extractor) {\n\t\t\textractorResults = append(extractorResults, match)\n\n\t\t\tif extractor.Internal {\n\t\t\t\tif data, ok := result.DynamicValues[extractor.Name]; !ok {\n\t\t\t\t\tresult.DynamicValues[extractor.Name] = []string{match}\n\t\t\t\t} else {\n\t\t\t\t\tresult.DynamicValues[extractor.Name] = append(data, match)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif _, ok := result.outputUnique[match]; !ok {\n\t\t\t\t\tresult.OutputExtracts = append(result.OutputExtracts, match)\n\t\t\t\t\tresult.outputUnique[match] = struct{}{}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif len(extractorResults) > 0 && !extractor.Internal && extractor.Name != \"\" {\n\t\t\tresult.Extracts[extractor.Name] = extractorResults\n\t\t}\n\t\t// update data with whatever was extracted doesn't matter if it is internal or not (skip unless it empty)\n\t\tif len(extractorResults) > 0 {\n\t\t\tdata[extractor.Name] = getExtractedValue(extractorResults)\n\t\t}\n\t}\n\n\t// expose dynamic values to same request matchers\n\tif len(result.DynamicValues) > 0 {\n\t\tdataDynamicValues := make(map[string]interface{})\n\t\tfor dynName, dynValues := range result.DynamicValues {\n\t\t\tif len(dynValues) > 1 {\n\t\t\t\tfor dynIndex, dynValue := range dynValues {\n\t\t\t\t\tdynKeyName := fmt.Sprintf(\"%s%d\", dynName, dynIndex)\n\t\t\t\t\tdataDynamicValues[dynKeyName] = dynValue\n\t\t\t\t}\n\t\t\t\tdataDynamicValues[dynName] = dynValues\n\t\t\t} else {\n\t\t\t\tdataDynamicValues[dynName] = dynValues[0]\n\t\t\t}\n\n\t\t}\n\t\tdata = generators.MergeMaps(data, dataDynamicValues)\n\t}\n\n\tfor matcherIndex, matcher := range operators.Matchers {\n\t\t// Skip matchers that are in the blocklist\n\t\tif operators.ExcludeMatchers != nil {\n\t\t\tif operators.ExcludeMatchers.Match(operators.TemplateID, matcher.Name) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif isMatch, matched := match(data, matcher); isMatch {\n\t\t\tif isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled\n\t\t\t\tmatcherName := GetMatcherName(matcher, matcherIndex)\n\t\t\t\tresult.Matches[matcherName] = matched\n\t\t\t} else { // if it's a \"named\" matcher with OR condition, then display it\n\t\t\t\tif matcherCondition == matchers.ORCondition && matcher.Name != \"\" {\n\t\t\t\t\tresult.Matches[matcher.Name] = matched\n\t\t\t\t}\n\t\t\t}\n\t\t\tmatches = true\n\t\t} else if matcherCondition == matchers.ANDCondition {\n\t\t\tif len(result.DynamicValues) > 0 {\n\t\t\t\treturn result, true\n\t\t\t}\n\t\t\treturn result, false\n\t\t}\n\t}\n\n\tresult.Matched = matches\n\tresult.Extracted = len(result.OutputExtracts) > 0\n\tif len(result.DynamicValues) > 0 && allInternalExtractors {\n\t\t// only return early if all extractors are internal\n\t\t// if some are internal and some are not then followthrough\n\t\treturn result, true\n\t}\n\n\t// Don't print if we have matchers, and they have not matched, regardless of extractor\n\tif len(operators.Matchers) > 0 && !matches {\n\t\t// if dynamic values are present then it is not a failure\n\t\tif len(result.DynamicValues) > 0 {\n\t\t\treturn result, true\n\t\t}\n\t\treturn nil, false\n\t}\n\t// Write a final string of output if matcher type is\n\t// AND or if we have extractors for the mechanism too.\n\tif len(result.Extracts) > 0 || len(result.OutputExtracts) > 0 || matches {\n\t\treturn result, true\n\t}\n\t// if dynamic values are present then it is not a failure\n\tif len(result.DynamicValues) > 0 {\n\t\treturn result, true\n\t}\n\treturn nil, false\n}\n\n// GetMatcherName returns matchername of given matcher\nfunc GetMatcherName(matcher *matchers.Matcher, matcherIndex int) string {\n\tif matcher.Name != \"\" {\n\t\treturn matcher.Name\n\t} else {\n\t\treturn matcher.Type.String() + \"-\" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable\n\t}\n}\n\n// ExecuteInternalExtractors executes internal dynamic extractors\nfunc (operators *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} {\n\tdynamicValues := make(map[string]interface{})\n\n\t// Start with the extractors first and evaluate them.\n\tfor _, extractor := range operators.Extractors {\n\t\tif !extractor.Internal {\n\t\t\tcontinue\n\t\t}\n\t\tfor match := range extract(data, extractor) {\n\t\t\tif _, ok := dynamicValues[extractor.Name]; !ok {\n\t\t\t\tdynamicValues[extractor.Name] = match\n\t\t\t}\n\t\t}\n\t}\n\treturn dynamicValues\n}\n\n// IsEmpty determines if the operator has matchers or extractors\nfunc (operators *Operators) IsEmpty() bool {\n\treturn operators.Len() == 0\n}\n\n// Len calculates the sum of the number of matchers and extractors\nfunc (operators *Operators) Len() int {\n\treturn len(operators.Matchers) + len(operators.Extractors)\n}\n\n// getExtractedValue takes array of extracted values if it only has one value\n// then it is flattened and returned as a string else original type is returned\nfunc getExtractedValue(values []string) any {\n\tif len(values) == 1 {\n\t\treturn values[0]\n\t} else {\n\t\treturn values\n\t}\n}\n\n// EvalBoolSlice evaluates a slice of bools using a logical AND\nfunc EvalBoolSlice(slice []bool, isAnd bool) bool {\n\tif len(slice) == 0 {\n\t\treturn false\n\t}\n\n\tresult := slice[0]\n\tfor _, b := range slice[1:] {\n\t\tif isAnd {\n\t\t\tresult = result && b\n\t\t} else {\n\t\t\tresult = result || b\n\t\t}\n\t}\n\treturn result\n}\n\n// Path: pkg/protocols/file/request.go\npackage file\n\nimport (\n\t\"bufio\"\n\t\"encoding/hex\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/docker/go-units\"\n\t\"github.com/mholt/archiver\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\ttemplateTypes \"github.com/projectdiscovery/nuclei/v3/pkg/templates/types\"\n\tsliceutil \"github.com/projectdiscovery/utils/slice\"\n\tsyncutil \"github.com/projectdiscovery/utils/sync\"\n)\n\nvar _ protocols.Request = &Request{}\n\n// Type returns the type of the protocol request\nfunc (request *Request) Type() templateTypes.ProtocolType {\n\treturn templateTypes.FileProtocol\n}\n\ntype FileMatch struct {\n\tData      string\n\tLine      int\n\tByteIndex int\n\tMatch     bool\n\tExtract   bool\n\tExpr      string\n\tRaw       string\n}\n\nvar errEmptyResult = errors.New(\"Empty result\")\n\n// ExecuteWithResults executes the protocol requests and returns results instead of writing them.\nfunc (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {\n\twg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize))\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = request.getInputPaths(input.MetaInput.Input, func(filePath string) {\n\t\twg.Add()\n\t\tfunc(filePath string) {\n\t\t\tdefer wg.Done()\n\t\t\tarchiveReader, _ := archiver.ByExtension(filePath)\n\t\t\tswitch {\n\t\t\tcase archiveReader != nil:\n\t\t\t\tswitch archiveInstance := archiveReader.(type) {\n\t\t\t\tcase archiver.Walker:\n\t\t\t\t\terr := archiveInstance.Walk(filePath, func(file archiver.File) error {\n\t\t\t\t\t\tif !request.validatePath(\"/\", file.Name(), true) {\n\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t}\n\t\t\t\t\t\t// every new file in the compressed multi-file archive counts 1\n\t\t\t\t\t\trequest.options.Progress.AddToTotal(1)\n\t\t\t\t\t\tarchiveFileName := filepath.Join(filePath, file.Name())\n\t\t\t\t\t\tevent, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tif errors.Is(err, errEmptyResult) {\n\t\t\t\t\t\t\t\t// no matches but one file elaborated\n\t\t\t\t\t\t\t\trequest.options.Progress.IncrementRequests()\n\t\t\t\t\t\t\t\treturn nil\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tgologger.Error().Msgf(\"%s\\n\", err)\n\t\t\t\t\t\t\t// error while elaborating the file\n\t\t\t\t\t\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t\tdefer file.Close()\n\t\t\t\t\t\tdumpResponse(event, request.options, fileMatches, filePath)\n\t\t\t\t\t\tcallback(event)\n\t\t\t\t\t\t// file elaborated and matched\n\t\t\t\t\t\trequest.options.Progress.IncrementRequests()\n\t\t\t\t\t\treturn nil\n\t\t\t\t\t})\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tgologger.Error().Msgf(\"%s\\n\", err)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\tcase archiver.Decompressor:\n\t\t\t\t\t// compressed archive - contains only one file => increments the counter by 1\n\t\t\t\t\trequest.options.Progress.AddToTotal(1)\n\t\t\t\t\tfile, err := os.Open(filePath)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tgologger.Error().Msgf(\"%s\\n\", err)\n\t\t\t\t\t\t// error while elaborating the file\n\t\t\t\t\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tdefer file.Close()\n\t\t\t\t\tfileStat, _ := file.Stat()\n\t\t\t\t\ttmpFileOut, err := os.CreateTemp(\"\", \"\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tgologger.Error().Msgf(\"%s\\n\", err)\n\t\t\t\t\t\t// error while elaborating the file\n\t\t\t\t\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tdefer tmpFileOut.Close()\n\t\t\t\t\tdefer os.RemoveAll(tmpFileOut.Name())\n\t\t\t\t\tif err := archiveInstance.Decompress(file, tmpFileOut); err != nil {\n\t\t\t\t\t\tgologger.Error().Msgf(\"%s\\n\", err)\n\t\t\t\t\t\t// error while elaborating the file\n\t\t\t\t\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\t_ = tmpFileOut.Sync()\n\t\t\t\t\t// rewind the file\n\t\t\t\t\t_, _ = tmpFileOut.Seek(0, 0)\n\t\t\t\t\tevent, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tif errors.Is(err, errEmptyResult) {\n\t\t\t\t\t\t\t// no matches but one file elaborated\n\t\t\t\t\t\t\trequest.options.Progress.IncrementRequests()\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\t\t\t\t\t\tgologger.Error().Msgf(\"%s\\n\", err)\n\t\t\t\t\t\t// error while elaborating the file\n\t\t\t\t\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tdumpResponse(event, request.options, fileMatches, filePath)\n\t\t\t\t\tcallback(event)\n\t\t\t\t\t// file elaborated and matched\n\t\t\t\t\trequest.options.Progress.IncrementRequests()\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\t// normal file - increments the counter by 1\n\t\t\t\trequest.options.Progress.AddToTotal(1)\n\t\t\t\tevent, fileMatches, err := request.processFile(filePath, input, previous)\n\t\t\t\tif err != nil {\n\t\t\t\t\tif errors.Is(err, errEmptyResult) {\n\t\t\t\t\t\t// no matches but one file elaborated\n\t\t\t\t\t\trequest.options.Progress.IncrementRequests()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tgologger.Error().Msgf(\"%s\\n\", err)\n\t\t\t\t\t// error while elaborating the file\n\t\t\t\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tdumpResponse(event, request.options, fileMatches, filePath)\n\t\t\t\tcallback(event)\n\t\t\t\t// file elaborated and matched\n\t\t\t\trequest.options.Progress.IncrementRequests()\n\t\t\t}\n\t\t}(filePath)\n\t})\n\n\twg.Wait()\n\tif err != nil {\n\t\trequest.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)\n\t\trequest.options.Progress.IncrementFailedRequestsBy(1)\n\t\treturn errors.Wrap(err, \"could not send file request\")\n\t}\n\treturn nil\n}\n\nfunc (request *Request) processFile(filePath string, input *contextargs.Context, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) {\n\tfile, err := os.Open(filePath)\n\tif err != nil {\n\t\treturn nil, nil, errors.Errorf(\"Could not open file path %s: %s\\n\", filePath, err)\n\t}\n\tdefer file.Close()\n\n\tstat, err := file.Stat()\n\tif err != nil {\n\t\treturn nil, nil, errors.Errorf(\"Could not stat file path %s: %s\\n\", filePath, err)\n\t}\n\tif stat.Size() >= request.maxSize {\n\t\tmaxSizeString := units.HumanSize(float64(request.maxSize))\n\t\tgologger.Verbose().Msgf(\"Limiting %s processed data to %s bytes: exceeded max size\\n\", filePath, maxSizeString)\n\t}\n\n\treturn request.processReader(file, filePath, input, stat.Size(), previousInternalEvent)\n}\n\nfunc (request *Request) processReader(reader io.Reader, filePath string, input *contextargs.Context, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) {\n\tfileReader := io.LimitReader(reader, request.maxSize)\n\tfileMatches, opResult := request.findMatchesWithReader(fileReader, input, filePath, totalBytes, previousInternalEvent)\n\tif opResult == nil && len(fileMatches) == 0 {\n\t\treturn nil, nil, errEmptyResult\n\t}\n\n\t// build event structure to interface with internal logic\n\treturn request.buildEvent(input.MetaInput.Input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil\n}\n\nfunc (request *Request) findMatchesWithReader(reader io.Reader, input *contextargs.Context, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) {\n\tvar bytesCount, linesCount, wordsCount int\n\tisResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse\n\ttotalBytesString := units.BytesSize(float64(totalBytes))\n\n\t// we are forced to check if the whole file needs to be elaborated\n\t// - matchers-condition option set to AND\n\thasAndCondition := request.CompiledOperators.GetMatchersCondition() == matchers.ANDCondition\n\t// - any matcher has AND condition\n\tfor _, matcher := range request.CompiledOperators.Matchers {\n\t\tif hasAndCondition {\n\t\t\tbreak\n\t\t}\n\t\tif matcher.GetCondition() == matchers.ANDCondition {\n\t\t\thasAndCondition = true\n\t\t}\n\t}\n\n\tscanner := bufio.NewScanner(reader)\n\tbuffer := []byte{}\n\tif hasAndCondition {\n\t\tscanner.Buffer(buffer, int(defaultMaxReadSize))\n\t\tscanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {\n\t\t\tdefaultMaxReadSizeInt := int(defaultMaxReadSize)\n\t\t\tif len(data) > defaultMaxReadSizeInt {\n\t\t\t\treturn defaultMaxReadSizeInt, data[0:defaultMaxReadSizeInt], nil\n\t\t\t}\n\t\t\tif !atEOF {\n\t\t\t\treturn 0, nil, nil\n\t\t\t}\n\t\t\treturn len(data), data, bufio.ErrFinalToken\n\t\t})\n\t} else {\n\t\tscanner.Buffer(buffer, int(chunkSize))\n\t}\n\n\tvar fileMatches []FileMatch\n\tvar opResult *operators.Result\n\tfor scanner.Scan() {\n\t\tlineContent := scanner.Text()\n\t\tn := len(lineContent)\n\n\t\t// update counters\n\t\tcurrentBytes := bytesCount + n\n\t\tprocessedBytes := units.BytesSize(float64(currentBytes))\n\n\t\tgologger.Verbose().Msgf(\"[%s] Processing file %s chunk %s/%s\", request.options.TemplateID, filePath, processedBytes, totalBytesString)\n\t\tdslMap := request.responseToDSLMap(lineContent, input.MetaInput.Input, filePath)\n\t\tfor k, v := range previous {\n\t\t\tdslMap[k] = v\n\t\t}\n\t\t// add template context variables to DSL map\n\t\tif request.options.HasTemplateCtx(input.MetaInput) {\n\t\t\tdslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll())\n\t\t}\n\t\tdiscardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug)\n\t\tnewOpResult := discardEvent.OperatorsResult\n\t\tif newOpResult != nil {\n\t\t\tif opResult == nil {\n\t\t\t\topResult = newOpResult\n\t\t\t} else {\n\t\t\t\topResult.Merge(newOpResult)\n\t\t\t}\n\t\t\tif newOpResult.Matched || newOpResult.Extracted {\n\t\t\t\tif newOpResult.Extracts != nil {\n\t\t\t\t\tfor expr, extracts := range newOpResult.Extracts {\n\t\t\t\t\t\tfor _, extract := range extracts {\n\t\t\t\t\t\t\tfileMatches = append(fileMatches, FileMatch{\n\t\t\t\t\t\t\t\tData:      extract,\n\t\t\t\t\t\t\t\tExtract:   true,\n\t\t\t\t\t\t\t\tLine:      linesCount + 1,\n\t\t\t\t\t\t\t\tByteIndex: bytesCount,\n\t\t\t\t\t\t\t\tExpr:      expr,\n\t\t\t\t\t\t\t\tRaw:       lineContent,\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif newOpResult.Matches != nil {\n\t\t\t\t\tfor expr, matches := range newOpResult.Matches {\n\t\t\t\t\t\tfor _, match := range matches {\n\t\t\t\t\t\t\tfileMatches = append(fileMatches, FileMatch{\n\t\t\t\t\t\t\t\tData:      match,\n\t\t\t\t\t\t\t\tMatch:     true,\n\t\t\t\t\t\t\t\tLine:      linesCount + 1,\n\t\t\t\t\t\t\t\tByteIndex: bytesCount,\n\t\t\t\t\t\t\t\tExpr:      expr,\n\t\t\t\t\t\t\t\tRaw:       lineContent,\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tfor _, outputExtract := range newOpResult.OutputExtracts {\n\t\t\t\t\tfileMatches = append(fileMatches, FileMatch{\n\t\t\t\t\t\tData:      outputExtract,\n\t\t\t\t\t\tMatch:     true,\n\t\t\t\t\t\tLine:      linesCount + 1,\n\t\t\t\t\t\tByteIndex: bytesCount,\n\t\t\t\t\t\tExpr:      outputExtract,\n\t\t\t\t\t\tRaw:       lineContent,\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcurrentLinesCount := 1 + strings.Count(lineContent, \"\\n\")\n\t\tlinesCount += currentLinesCount\n\t\twordsCount += strings.Count(lineContent, \" \")\n\t\tbytesCount = currentBytes\n\t}\n\treturn fileMatches, opResult\n}\n\nfunc (request *Request) buildEvent(input, filePath string, fileMatches []FileMatch, operatorResult *operators.Result, previous output.InternalEvent) *output.InternalWrappedEvent {\n\texprLines := make(map[string][]int)\n\texprBytes := make(map[string][]int)\n\tinternalEvent := request.responseToDSLMap(\"\", input, filePath)\n\tfor k, v := range previous {\n\t\tinternalEvent[k] = v\n\t}\n\tfor _, fileMatch := range fileMatches {\n\t\texprLines[fileMatch.Expr] = append(exprLines[fileMatch.Expr], fileMatch.Line)\n\t\texprBytes[fileMatch.Expr] = append(exprBytes[fileMatch.Expr], fileMatch.ByteIndex)\n\t}\n\n\tevent := eventcreator.CreateEventWithOperatorResults(request, internalEvent, operatorResult)\n\t// Annotate with line numbers if asked by the user\n\tif request.options.Options.ShowMatchLine {\n\t\tfor _, result := range event.Results {\n\t\t\tswitch {\n\t\t\tcase result.MatcherName != \"\":\n\t\t\t\tresult.Lines = exprLines[result.MatcherName]\n\t\t\tcase result.ExtractorName != \"\":\n\t\t\t\tresult.Lines = exprLines[result.ExtractorName]\n\t\t\tdefault:\n\t\t\t\tfor _, extractedResult := range result.ExtractedResults {\n\t\t\t\t\tresult.Lines = append(result.Lines, exprLines[extractedResult]...)\n\t\t\t\t}\n\t\t\t}\n\t\t\tresult.Lines = sliceutil.Dedupe(result.Lines)\n\t\t}\n\t}\n\treturn event\n}\n\nfunc dumpResponse(event *output.InternalWrappedEvent, requestOptions *protocols.ExecutorOptions, filematches []FileMatch, filePath string) {\n\tcliOptions := requestOptions.Options\n\tif cliOptions.Debug || cliOptions.DebugResponse {\n\t\tfor _, fileMatch := range filematches {\n\t\t\tlineContent := fileMatch.Raw\n\t\t\thexDump := false\n\t\t\tif responsehighlighter.HasBinaryContent(lineContent) {\n\t\t\t\thexDump = true\n\t\t\t\tlineContent = hex.Dump([]byte(lineContent))\n\t\t\t}\n\t\t\thighlightedResponse := responsehighlighter.Highlight(event.OperatorsResult, lineContent, cliOptions.NoColor, hexDump)\n\t\t\tgologger.Debug().Msgf(\"[%s] Dumped match/extract file snippet for %s at line %d\\n\\n%s\", requestOptions.TemplateID, filePath, fileMatch.Line, highlightedResponse)\n\t\t}\n\t}\n}\n\n// Path: pkg/protocols/common/generators/maps.go\npackage generators\n\nimport (\n\t\"reflect\"\n)\n\n// MergeMapsMany merges many maps into a new map\n\nfunc MergeMapsMany(maps ...interface{}) map[string][]string {\n\tm := make(map[string][]string)\n\tfor _, gotMap := range maps {\n\t\tval := reflect.ValueOf(gotMap)\n\t\tif val.Kind() != reflect.Map {\n\t\t\tcontinue\n\t\t}\n\t\tappendToSlice := func(key, value string) {\n\t\t\tif values, ok := m[key]; !ok {\n\t\t\t\tm[key] = []string{value}\n\t\t\t} else {\n\t\t\t\tm[key] = append(values, value)\n\t\t\t}\n\t\t}\n\t\tfor _, e := range val.MapKeys() {\n\t\t\tv := val.MapIndex(e)\n\t\t\tswitch v.Kind() {\n\t\t\tcase reflect.Slice, reflect.Array:\n\t\t\t\tfor i := 0; i < v.Len(); i++ {\n\t\t\t\t\tappendToSlice(e.String(), v.Index(i).String())\n\t\t\t\t}\n\t\t\tcase reflect.String:\n\t\t\t\tappendToSlice(e.String(), v.String())\n\t\t\tcase reflect.Interface:\n\t\t\t\tswitch data := v.Interface().(type) {\n\t\t\t\tcase string:\n\t\t\t\t\tappendToSlice(e.String(), data)\n\t\t\t\tcase []string:\n\t\t\t\t\tfor _, value := range data {\n\t\t\t\t\t\tappendToSlice(e.String(), value)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn m\n}\n\n// MergeMaps merges two maps into a new map\nfunc MergeMaps(maps ...map[string]interface{}) map[string]interface{} {\n\tmerged := make(map[string]interface{})\n\tfor _, m := range maps {\n\t\tfor k, v := range m {\n\t\t\tmerged[k] = v\n\t\t}\n\t}\n\treturn merged\n}\n\n// ExpandMapValues converts values from flat string to string slice\nfunc ExpandMapValues(m map[string]string) map[string][]string {\n\tm1 := make(map[string][]string, len(m))\n\tfor k, v := range m {\n\t\tm1[k] = []string{v}\n\t}\n\treturn m1\n}\n\n// Path: pkg/protocols/dns/request.go\npackage dns\n\nimport (\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/miekg/dns\"\n\t\"github.com/pkg/errors\"\n\t\"go.uber.org/multierr\"\n\t\"golang.org/x/exp/maps\"\n\n\t\"github.com/projectdiscovery/gologger\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/output\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter\"\n\t\"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump\"\n\tprotocolutils \"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils\"\n...\n", "needle_token_start": 14295, "needle_token_end": 14699, "code_context_ntokens": 15272, "output": ["```go\n// MergeMapsMany merges many maps into a new map\nfunc MergeMapsMany(maps ...interface{}) map[string][]string {\n\tm := make(map[string][]string)\n\tfor _, gotMap := range maps {\n\t\tval := reflect.ValueOf(gotMap)\n\t\tif val.Kind() != reflect.Map {\n\t\t\tcontinue\n\t\t}\n\t\tappendToSlice := func(key, value string) {\n\t\t\tif values, ok := m[key]; !ok {\n\t\t\t\tm[key] = []string{value}\n\t\t\t} else {\n\t\t\t\tm[key] = append(values, value)\n\t\t\t}\n\t\t}\n\t\tfor _, e := range val.MapKeys() {\n\t\t\tv := val.MapIndex(e)\n\t\t\tswitch v.Kind() {\n\t\t\tcase reflect.Slice, reflect.Array:\n\t\t\t\tfor i := 0; i < v.Len(); i++ {\n\t\t\t\t\tappendToSlice(e.String(), v.Index(i).String())\n\t\t\t\t}\n\t\t\tcase reflect.String:\n\t\t\t\tappendToSlice(e.String(), v.String())\n\t\t\tcase reflect.Interface:\n\t\t\t\tswitch data := v.Interface().(type) {\n\t\t\t\tcase string:\n\t\t\t\t\tappendToSlice(e.String(), data)\n\t\t\t\tcase []string:\n\t\t\t\t\tfor _, value := range data {\n\t\t\t\t\t\tappendToSlice(e.String(), value)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn m\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "NewDockerCommand", "language": "go", "path": "pkg/commands/docker.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function initializes and configures an interface for managing Docker operations, setting up necessary components like logging, SSH tunneling for Docker commands, and error handling.\n2. **Input**: It takes several configuration and operational parameters including a logging instance, command execution handler, translation set, application configuration, and an error channel.\n3. **Output**: Returns an instance configured for Docker operations and an error if any issues occur during the setup.\n4. **Procedure**: \n   - Establishes an SSH tunnel for Docker host communication.\n   - Determines the Docker host address.\n   - Creates a Docker client with specified API version and host.\n   - Checks if Docker Compose is applicable for the current project using a templated command.\n   - Configures and returns the Docker operations interface, handling any errors that might affect the initialization.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\"github.com/docker/docker/client\"\n\t\"github.com/imdario/mergo\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands/ssh\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\nconst (\n\tAPIVersion = \"1.25\"\n)\n\n// DockerCommand is our main docker interface\ntype DockerCommand struct {\n\tLog                    *logrus.Entry\n\tOSCommand              *OSCommand\n\tTr                     *i18n.TranslationSet\n\tConfig                 *config.AppConfig\n\tClient                 *client.Client\n\tInDockerComposeProject bool\n\tErrorChan              chan error\n\tContainerMutex         deadlock.Mutex\n\tServiceMutex           deadlock.Mutex\n\n\tClosers []io.Closer\n}\n\nvar _ io.Closer = &DockerCommand{}\n\n// LimitedDockerCommand is a stripped-down DockerCommand with just the methods the container/service/image might need\ntype LimitedDockerCommand interface {\n\tNewCommandObject(CommandObject) CommandObject\n}\n\n// CommandObject is what we pass to our template resolvers when we are running a custom command. We do not guarantee that all fields will be populated: just the ones that make sense for the current context\ntype CommandObject struct {\n\tDockerCompose string\n\tService       *Service\n\tContainer     *Container\n\tImage         *Image\n\tVolume        *Volume\n\tNetwork       *Network\n}\n\n// NewCommandObject takes a command object and returns a default command object with the passed command object merged in\nfunc (c *DockerCommand) NewCommandObject(obj CommandObject) CommandObject {\n\tdefaultObj := CommandObject{DockerCompose: c.Config.UserConfig.CommandTemplates.DockerCompose}\n\t_ = mergo.Merge(&defaultObj, obj)\n\treturn defaultObj\n}\n\n// NewDockerCommand it runs docker commands\n\nfunc NewDockerCommand(log *logrus.Entry, osCommand *OSCommand, tr *i18n.TranslationSet, config *config.AppConfig, errorChan chan error) (*DockerCommand, error) {\n\ttunnelCloser, err := ssh.NewSSHHandler(osCommand).HandleSSHDockerHost()\n\tif err != nil {\n\t\togLog.Fatal(err)\n\t}\n\n\tdockerHost, err := determineDockerHost()\n\tif err != nil {\n\t\togLog.Printf(\"> could not determine host %v\", err)\n\t}\n\n\tcli, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion(APIVersion), client.WithHost(dockerHost))\n\tif err != nil {\n\t\togLog.Fatal(err)\n\t}\n\n\tdockerCommand := &DockerCommand{\n\t\tLog:                    log,\n\t\tOSCommand:              osCommand,\n\t\tTr:                     tr,\n\t\tConfig:                 config,\n\t\tClient:                 cli,\n\t\tErrorChan:              errorChan,\n\t\tInDockerComposeProject: true,\n\t\tClosers:                []io.Closer{tunnelCloser},\n\t}\n\n\tcommand := utils.ApplyTemplate(\n\t\tconfig.UserConfig.CommandTemplates.CheckDockerComposeConfig,\n\t\tdockerCommand.NewCommandObject(CommandObject{}),\n\t)\n\n\tlog.Warn(command)\n\n\terr = osCommand.RunCommand(\n\t\tutils.ApplyTemplate(\n\t\t\tconfig.UserConfig.CommandTemplates.CheckDockerComposeConfig,\n\t\t\tdockerCommand.NewCommandObject(CommandObject{}),\n\t\t),\n\t)\n\tif err != nil {\n\t\tdockerCommand.InDockerComposeProject = false\n\t\tlog.Warn(err.Error())\n\t}\n\n\treturn dockerCommand, nil\n}\n\nfunc (c *DockerCommand) Close() error {\n\treturn utils.CloseMany(c.Closers)\n}\n\nfunc (c *DockerCommand) CreateClientStatMonitor(container *Container) {\n\tcontainer.MonitoringStats = true\n\tstream, err := c.Client.ContainerStats(context.Background(), container.ID, true)\n\tif err != nil {\n\t\t// not creating error panel because if we've disconnected from docker we'll\n\t\t// have already created an error panel\n\t\tc.Log.Error(err)\n\t\tcontainer.MonitoringStats = false\n\t\treturn\n\t}\n\n\tdefer stream.Body.Close()\n\n\tscanner := bufio.NewScanner(stream.Body)\n\tfor scanner.Scan() {\n\t\tdata := scanner.Bytes()\n\t\tvar stats ContainerStats\n\t\t_ = json.Unmarshal(data, &stats)\n\n\t\trecordedStats := &RecordedStats{\n\t\t\tClientStats: stats,\n\t\t\tDerivedStats: DerivedStats{\n\t\t\t\tCPUPercentage:    stats.CalculateContainerCPUPercentage(),\n\t\t\t\tMemoryPercentage: stats.CalculateContainerMemoryUsage(),\n\t\t\t},\n\t\t\tRecordedAt: time.Now(),\n\t\t}\n\n\t\tcontainer.appendStats(recordedStats, c.Config.UserConfig.Stats.MaxDuration)\n\t}\n\n\tcontainer.MonitoringStats = false\n}\n\nfunc (c *DockerCommand) RefreshContainersAndServices(currentServices []*Service, currentContainers []*Container) ([]*Container, []*Service, error) {\n\tc.ServiceMutex.Lock()\n\tdefer c.ServiceMutex.Unlock()\n\n\tcontainers, err := c.GetContainers(currentContainers)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tvar services []*Service\n\t// we only need to get these services once because they won't change in the runtime of the program\n\tif currentServices != nil {\n\t\tservices = currentServices\n\t} else {\n\t\tservices, err = c.GetServices()\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tc.assignContainersToServices(containers, services)\n\n\treturn containers, services, nil\n}\n\nfunc (c *DockerCommand) assignContainersToServices(containers []*Container, services []*Service) {\nL:\n\tfor _, service := range services {\n\t\tfor _, container := range containers {\n\t\t\tif !container.OneOff && container.ServiceName == service.Name {\n\t\t\t\tservice.Container = container\n\t\t\t\tcontinue L\n\t\t\t}\n\t\t}\n\t\tservice.Container = nil\n\t}\n}\n\n// GetContainers gets the docker containers\nfunc (c *DockerCommand) GetContainers(existingContainers []*Container) ([]*Container, error) {\n\tc.ContainerMutex.Lock()\n\tdefer c.ContainerMutex.Unlock()\n\n\tcontainers, err := c.Client.ContainerList(context.Background(), dockerTypes.ContainerListOptions{All: true})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\townContainers := make([]*Container, len(containers))\n\n\tfor i, container := range containers {\n\t\tvar newContainer *Container\n\n\t\t// check if we already data stored against the container\n\t\tfor _, existingContainer := range existingContainers {\n\t\t\tif existingContainer.ID == container.ID {\n\t\t\t\tnewContainer = existingContainer\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// initialise the container if it's completely new\n\t\tif newContainer == nil {\n\t\t\tnewContainer = &Container{\n\t\t\t\tID:            container.ID,\n\t\t\t\tClient:        c.Client,\n\t\t\t\tOSCommand:     c.OSCommand,\n\t\t\t\tLog:           c.Log,\n\t\t\t\tDockerCommand: c,\n\t\t\t\tTr:            c.Tr,\n\t\t\t}\n\t\t}\n\n\t\tnewContainer.Container = container\n\t\t// if the container is made with a name label we will use that\n\t\tif name, ok := container.Labels[\"name\"]; ok {\n\t\t\tnewContainer.Name = name\n\t\t} else {\n\t\t\tnewContainer.Name = strings.TrimLeft(container.Names[0], \"/\")\n\t\t}\n\t\tnewContainer.ServiceName = container.Labels[\"com.docker.compose.service\"]\n\t\tnewContainer.ProjectName = container.Labels[\"com.docker.compose.project\"]\n\t\tnewContainer.ContainerNumber = container.Labels[\"com.docker.compose.container\"]\n\t\tnewContainer.OneOff = container.Labels[\"com.docker.compose.oneoff\"] == \"True\"\n\n\t\townContainers[i] = newContainer\n\t}\n\n\treturn ownContainers, nil\n}\n\n// GetServices gets services\nfunc (c *DockerCommand) GetServices() ([]*Service, error) {\n\tif !c.InDockerComposeProject {\n\t\treturn nil, nil\n\t}\n\n\tcomposeCommand := c.Config.UserConfig.CommandTemplates.DockerCompose\n\toutput, err := c.OSCommand.RunCommandWithOutput(fmt.Sprintf(\"%s config --services\", composeCommand))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// output looks like:\n\t// service1\n\t// service2\n\n\tlines := utils.SplitLines(output)\n\tservices := make([]*Service, len(lines))\n\tfor i, str := range lines {\n\t\tservices[i] = &Service{\n\t\t\tName:          str,\n\t\t\tID:            str,\n\t\t\tOSCommand:     c.OSCommand,\n\t\t\tLog:           c.Log,\n\t\t\tDockerCommand: c,\n\t\t}\n\t}\n\n\treturn services, nil\n}\n\n// UpdateContainerDetails attaches the details returned from docker inspect to each of the containers\n// this contains a bit more info than what you get from the go-docker client\nfunc (c *DockerCommand) UpdateContainerDetails(containers []*Container) error {\n\tc.ContainerMutex.Lock()\n\tdefer c.ContainerMutex.Unlock()\n\n\tfor _, container := range containers {\n\t\tdetails, err := c.Client.ContainerInspect(context.Background(), container.ID)\n\t\tif err != nil {\n\t\t\tc.Log.Error(err)\n\t\t} else {\n\t\t\tcontainer.Details = details\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// ViewAllLogs attaches to a subprocess viewing all the logs from docker-compose\nfunc (c *DockerCommand) ViewAllLogs() (*exec.Cmd, error) {\n\tcmd := c.OSCommand.ExecutableFromString(\n\t\tutils.ApplyTemplate(\n\t\t\tc.OSCommand.Config.UserConfig.CommandTemplates.ViewAllLogs,\n\t\t\tc.NewCommandObject(CommandObject{}),\n\t\t),\n\t)\n\n\tc.OSCommand.PrepareForChildren(cmd)\n\n\treturn cmd, nil\n}\n\n// DockerComposeConfig returns the result of 'docker-compose config'\nfunc (c *DockerCommand) DockerComposeConfig() string {\n\toutput, err := c.OSCommand.RunCommandWithOutput(\n\t\tutils.ApplyTemplate(\n\t\t\tc.OSCommand.Config.UserConfig.CommandTemplates.DockerComposeConfig,\n\t\t\tc.NewCommandObject(CommandObject{}),\n\t\t),\n\t)\n\tif err != nil {\n\t\toutput = err.Error()\n\t}\n\treturn output\n}\n\n// determineDockerHost tries to the determine the docker host that we should connect to\n// in the following order of decreasing precedence:\n//   - value of \"DOCKER_HOST\" environment variable\n//   - host retrieved from the current context (specified via DOCKER_CONTEXT)\n//   - \"default docker host\" for the host operating system, otherwise\nfunc determineDockerHost() (string, error) {\n\t// If the docker host is explicitly set via the \"DOCKER_HOST\" environment variable,\n\t// then its a no-brainer :shrug:\n\tif os.Getenv(\"DOCKER_HOST\") != \"\" {\n\t\treturn os.Getenv(\"DOCKER_HOST\"), nil\n\t}\n\n\tcurrentContext := os.Getenv(\"DOCKER_CONTEXT\")\n\tif currentContext == \"\" {\n\t\tcf, err := cliconfig.Load(cliconfig.Dir())\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tcurrentContext = cf.CurrentContext\n\t}\n\n\tif currentContext == \"\" {\n\t\t// If a docker context is neither specified via the \"DOCKER_CONTEXT\" environment variable nor via the\n\t\t// $HOME/.docker/config file, then we fall back to connecting to the \"default docker host\" meant for\n\t\t// the host operating system.\n\t\treturn defaultDockerHost, nil\n\t}\n\n\tstoreConfig := ctxstore.NewConfig(\n\t\tfunc() interface{} { return &ddocker.EndpointMeta{} },\n\t\tctxstore.EndpointTypeGetter(ddocker.DockerEndpoint, func() interface{} { return &ddocker.EndpointMeta{} }),\n\t)\n\n\tst := ctxstore.New(cliconfig.ContextStoreDir(), storeConfig)\n\tmd, err := st.GetMetadata(currentContext)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdockerEP, ok := md.Endpoints[ddocker.DockerEndpoint]\n\tif !ok {\n\t\treturn \"\", err\n\t}\n\tdockerEPMeta, ok := dockerEP.(ddocker.EndpointMeta)\n\tif !ok {\n\t\treturn \"\", fmt.Errorf(\"expected docker.EndpointMeta, got %T\", dockerEP)\n\t}\n\n\tif dockerEPMeta.Host != \"\" {\n\t\treturn dockerEPMeta.Host, nil\n\t}\n\n\t// We might end up here, if the context was created with the `host` set to an empty value (i.e. '').\n\t// For example:\n\t// ```sh\n\t// docker context create foo --docker \"host=\"\n\t// ```\n\t// In such scenario, we mimic the `docker` cli and try to connect to the \"default docker host\".\n\treturn defaultDockerHost, nil\n}\n\n// Path: pkg/commands/docker_host_unix.go\n//go:build !windows\n\npackage commands\n\nconst (\n\tdefaultDockerHost = \"unix:///var/run/docker.sock\"\n)\n\n// Path: pkg/commands/docker_host_windows.go\npackage commands\n\nconst (\n\tdefaultDockerHost = \"npipe:////./pipe/docker_engine\"\n)\n\n// Path: pkg/commands/dummies.go\npackage commands\n\nimport (\n\t\"io\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// This file exports dummy constructors for use by tests in other packages\n\n// NewDummyOSCommand creates a new dummy OSCommand for testing\nfunc NewDummyOSCommand() *OSCommand {\n\treturn NewOSCommand(NewDummyLog(), NewDummyAppConfig())\n}\n\n// NewDummyAppConfig creates a new dummy AppConfig for testing\nfunc NewDummyAppConfig() *config.AppConfig {\n\tappConfig := &config.AppConfig{\n\t\tName:        \"lazydocker\",\n\t\tVersion:     \"unversioned\",\n\t\tCommit:      \"\",\n\t\tBuildDate:   \"\",\n\t\tDebug:       false,\n\t\tBuildSource: \"\",\n\t}\n\treturn appConfig\n}\n\n// NewDummyLog creates a new dummy Log for testing\nfunc NewDummyLog() *logrus.Entry {\n\tlog := logrus.New()\n\tlog.Out = io.Discard\n\treturn log.WithField(\"test\", \"test\")\n}\n\n// NewDummyDockerCommand creates a new dummy DockerCommand for testing\nfunc NewDummyDockerCommand() *DockerCommand {\n\treturn NewDummyDockerCommandWithOSCommand(NewDummyOSCommand())\n}\n\n// NewDummyDockerCommandWithOSCommand creates a new dummy DockerCommand for testing\nfunc NewDummyDockerCommandWithOSCommand(osCommand *OSCommand) *DockerCommand {\n\tnewAppConfig := NewDummyAppConfig()\n\treturn &DockerCommand{\n\t\tLog:       NewDummyLog(),\n\t\tOSCommand: osCommand,\n\t\tTr:        i18n.NewTranslationSet(NewDummyLog(), newAppConfig.UserConfig.Gui.Language),\n\t\tConfig:    newAppConfig,\n\t}\n}\n\n// Path: pkg/commands/image.go\npackage commands\n\nimport (\n\t\"context\"\n\t\"strings\"\n\n\t\"github.com/docker/docker/api/types/image\"\n\t\"github.com/samber/lo\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/docker/docker/api/types/filters\"\n\t\"github.com/docker/docker/client\"\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Image : A docker Image\ntype Image struct {\n\tName          string\n\tTag           string\n\tID            string\n\tImage         dockerTypes.ImageSummary\n\tClient        *client.Client\n\tOSCommand     *OSCommand\n\tLog           *logrus.Entry\n\tDockerCommand LimitedDockerCommand\n}\n\n// Remove removes the image\nfunc (i *Image) Remove(options dockerTypes.ImageRemoveOptions) error {\n\tif _, err := i.Client.ImageRemove(context.Background(), i.ID, options); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc getHistoryResponseItemDisplayStrings(layer image.HistoryResponseItem) []string {\n\ttag := \"\"\n\tif len(layer.Tags) > 0 {\n\t\ttag = layer.Tags[0]\n\t}\n\n\tid := strings.TrimPrefix(layer.ID, \"sha256:\")\n\tif len(id) > 10 {\n\t\tid = id[0:10]\n\t}\n\tidColor := color.FgWhite\n\tif id == \"<missing>\" {\n\t\tidColor = color.FgBlue\n\t}\n\n\tdockerFileCommandPrefix := \"/bin/sh -c #(nop) \"\n\tcreatedBy := layer.CreatedBy\n\tif strings.Contains(layer.CreatedBy, dockerFileCommandPrefix) {\n\t\tcreatedBy = strings.Trim(strings.TrimPrefix(layer.CreatedBy, dockerFileCommandPrefix), \" \")\n\t\tsplit := strings.Split(createdBy, \" \")\n\t\tcreatedBy = utils.ColoredString(split[0], color.FgYellow) + \" \" + strings.Join(split[1:], \" \")\n\t}\n\n\tcreatedBy = strings.Replace(createdBy, \"\\t\", \" \", -1)\n\n\tsize := utils.FormatBinaryBytes(int(layer.Size))\n\tsizeColor := color.FgWhite\n\tif size == \"0B\" {\n\t\tsizeColor = color.FgBlue\n\t}\n\n\treturn []string{\n\t\tutils.ColoredString(id, idColor),\n\t\tutils.ColoredString(tag, color.FgGreen),\n\t\tutils.ColoredString(size, sizeColor),\n\t\tcreatedBy,\n\t}\n}\n\n// RenderHistory renders the history of the image\nfunc (i *Image) RenderHistory() (string, error) {\n\thistory, err := i.Client.ImageHistory(context.Background(), i.ID)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\ttableBody := lo.Map(history, func(layer image.HistoryResponseItem, _ int) []string {\n\t\treturn getHistoryResponseItemDisplayStrings(layer)\n\t})\n\n\theaders := [][]string{{\"ID\", \"TAG\", \"SIZE\", \"COMMAND\"}}\n\ttable := append(headers, tableBody...)\n\n\treturn utils.RenderTable(table)\n}\n\n// RefreshImages returns a slice of docker images\nfunc (c *DockerCommand) RefreshImages() ([]*Image, error) {\n\timages, err := c.Client.ImageList(context.Background(), dockerTypes.ImageListOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\townImages := make([]*Image, len(images))\n\n\tfor i, image := range images {\n\t\tfirstTag := \"\"\n\t\ttags := image.RepoTags\n\t\tif len(tags) > 0 {\n\t\t\tfirstTag = tags[0]\n\t\t}\n\n\t\tnameParts := strings.Split(firstTag, \":\")\n\t\ttag := \"\"\n\t\tname := \"none\"\n\t\tif len(nameParts) > 1 {\n\t\t\ttag = nameParts[len(nameParts)-1]\n\t\t\tname = strings.Join(nameParts[:len(nameParts)-1], \":\")\n\n\t\t\tfor prefix, replacement := range c.Config.UserConfig.Replacements.ImageNamePrefixes {\n\t\t\t\tif strings.HasPrefix(name, prefix) {\n\t\t\t\t\tname = strings.Replace(name, prefix, replacement, 1)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\townImages[i] = &Image{\n\t\t\tID:            image.ID,\n\t\t\tName:          name,\n\t\t\tTag:           tag,\n\t\t\tImage:         image,\n\t\t\tClient:        c.Client,\n\t\t\tOSCommand:     c.OSCommand,\n\t\t\tLog:           c.Log,\n\t\t\tDockerCommand: c,\n\t\t}\n\t}\n\n\treturn ownImages, nil\n}\n\n// PruneImages prunes images\nfunc (c *DockerCommand) PruneImages() error {\n\t_, err := c.Client.ImagesPrune(context.Background(), filters.Args{})\n\treturn err\n}\n\n// Path: pkg/commands/network.go\npackage commands\n\nimport (\n\t\"context\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/docker/docker/api/types/filters\"\n\t\"github.com/docker/docker/client\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Network : A docker Network\ntype Network struct {\n\tName          string\n\tNetwork       dockerTypes.NetworkResource\n\tClient        *client.Client\n\tOSCommand     *OSCommand\n\tLog           *logrus.Entry\n\tDockerCommand LimitedDockerCommand\n}\n\n// RefreshNetworks gets the networks and stores them\nfunc (c *DockerCommand) RefreshNetworks() ([]*Network, error) {\n\tnetworks, err := c.Client.NetworkList(context.Background(), dockerTypes.NetworkListOptions{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\townNetworks := make([]*Network, len(networks))\n\n\tfor i, network := range networks {\n\t\townNetworks[i] = &Network{\n\t\t\tName:          network.Name,\n\t\t\tNetwork:       network,\n\t\t\tClient:        c.Client,\n\t\t\tOSCommand:     c.OSCommand,\n\t\t\tLog:           c.Log,\n\t\t\tDockerCommand: c,\n\t\t}\n\t}\n\n\treturn ownNetworks, nil\n}\n\n// PruneNetworks prunes networks\nfunc (c *DockerCommand) PruneNetworks() error {\n\t_, err := c.Client.NetworksPrune(context.Background(), filters.Args{})\n\treturn err\n}\n\n// Remove removes the network\nfunc (v *Network) Remove() error {\n\treturn v.Client.NetworkRemove(context.Background(), v.Name)\n}\n\n// Path: pkg/commands/os_default_platform.go\n//go:build !windows\n// +build !windows\n\npackage commands\n\nimport (\n\t\"runtime\"\n)\n\nfunc getPlatform() *Platform {\n\treturn &Platform{\n\t\tos:              runtime.GOOS,\n\t\tshell:           \"bash\",\n\t\tshellArg:        \"-c\",\n\t\topenCommand:     \"open {{filename}}\",\n\t\topenLinkCommand: \"open {{link}}\",\n\t}\n}\n\n// Path: pkg/commands/os_test.go\npackage commands\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\n// TestOSCommandRunCommandWithOutput is a function.\nfunc TestOSCommandRunCommandWithOutput(t *testing.T) {\n\ttype scenario struct {\n\t\tcommand string\n\t\ttest    func(string, error)\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"echo -n '123'\",\n\t\t\tfunc(output string, err error) {\n\t\t\t\tassert.NoError(t, err)\n\t\t\t\tassert.EqualValues(t, \"123\", output)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"rmdir unexisting-folder\",\n\t\t\tfunc(output string, err error) {\n\t\t\t\tassert.Regexp(t, \"rmdir.*unexisting-folder.*\", err.Error())\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\ts.test(NewDummyOSCommand().RunCommandWithOutput(s.command))\n\t}\n}\n\n// TestOSCommandRunCommand is a function.\nfunc TestOSCommandRunCommand(t *testing.T) {\n\ttype scenario struct {\n\t\tcommand string\n\t\ttest    func(error)\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"rmdir unexisting-folder\",\n\t\t\tfunc(err error) {\n\t\t\t\tassert.Regexp(t, \"rmdir.*unexisting-folder.*\", err.Error())\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\ts.test(NewDummyOSCommand().RunCommand(s.command))\n\t}\n}\n\n// TestOSCommandEditFile is a function.\nfunc TestOSCommandEditFile(t *testing.T) {\n\ttype scenario struct {\n\t\tfilename string\n\t\tcommand  func(string, ...string) *exec.Cmd\n\t\tgetenv   func(string) string\n\t\ttest     func(*exec.Cmd, error)\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"test\",\n\t\t\tfunc(name string, arg ...string) *exec.Cmd {\n\t\t\t\treturn exec.Command(\"exit\", \"1\")\n\t\t\t},\n\t\t\tfunc(env string) string {\n\t\t\t\treturn \"\"\n\t\t\t},\n\t\t\tfunc(cmd *exec.Cmd, err error) {\n\t\t\t\tassert.EqualError(t, err, \"No editor defined in $VISUAL or $EDITOR\")\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"test\",\n\t\t\tfunc(name string, arg ...string) *exec.Cmd {\n\t\t\t\tif name == \"which\" {\n\t\t\t\t\treturn exec.Command(\"exit\", \"1\")\n\t\t\t\t}\n\n\t\t\t\tassert.EqualValues(t, \"nano\", name)\n\n\t\t\t\treturn exec.Command(\"exit\", \"0\")\n\t\t\t},\n\t\t\tfunc(env string) string {\n\t\t\t\tif env == \"VISUAL\" {\n\t\t\t\t\treturn \"nano\"\n\t\t\t\t}\n\n\t\t\t\treturn \"\"\n\t\t\t},\n\t\t\tfunc(cmd *exec.Cmd, err error) {\n\t\t\t\tassert.NoError(t, err)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"test\",\n\t\t\tfunc(name string, arg ...string) *exec.Cmd {\n\t\t\t\tif name == \"which\" {\n\t\t\t\t\treturn exec.Command(\"exit\", \"1\")\n\t\t\t\t}\n\n\t\t\t\tassert.EqualValues(t, \"emacs\", name)\n\n\t\t\t\treturn exec.Command(\"exit\", \"0\")\n\t\t\t},\n\t\t\tfunc(env string) string {\n\t\t\t\tif env == \"EDITOR\" {\n\t\t\t\t\treturn \"emacs\"\n\t\t\t\t}\n\n\t\t\t\treturn \"\"\n\t\t\t},\n\t\t\tfunc(cmd *exec.Cmd, err error) {\n\t\t\t\tassert.NoError(t, err)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"test\",\n\t\t\tfunc(name string, arg ...string) *exec.Cmd {\n\t\t\t\tif name == \"which\" {\n\t\t\t\t\treturn exec.Command(\"echo\")\n\t\t\t\t}\n\n\t\t\t\tassert.EqualValues(t, \"vi\", name)\n\n\t\t\t\treturn exec.Command(\"exit\", \"0\")\n\t\t\t},\n\t\t\tfunc(env string) string {\n\t\t\t\treturn \"\"\n\t\t\t},\n\t\t\tfunc(cmd *exec.Cmd, err error) {\n\t\t\t\tassert.NoError(t, err)\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tOSCmd := NewDummyOSCommand()\n\t\tOSCmd.command = s.command\n\t\tOSCmd.getenv = s.getenv\n\n\t\ts.test(OSCmd.EditFile(s.filename))\n\t}\n}\n\nfunc TestOSCommandQuote(t *testing.T) {\n\tosCommand := NewDummyOSCommand()\n\n\tosCommand.Platform.os = \"linux\"\n\n\tactual := osCommand.Quote(\"hello `test`\")\n\n\texpected := \"\\\"hello \\\\`test\\\\`\\\"\"\n\n\tassert.EqualValues(t, expected, actual)\n}\n\n// TestOSCommandQuoteSingleQuote tests the quote function with ' quotes explicitly for Linux\nfunc TestOSCommandQuoteSingleQuote(t *testing.T) {\n\tosCommand := NewDummyOSCommand()\n\n\tosCommand.Platform.os = \"linux\"\n\n\tactual := osCommand.Quote(\"hello 'test'\")\n\n\texpected := `\"hello 'test'\"`\n\n\tassert.EqualValues(t, expected, actual)\n}\n\n// TestOSCommandQuoteDoubleQuote tests the quote function with \" quotes explicitly for Linux\nfunc TestOSCommandQuoteDoubleQuote(t *testing.T) {\n\tosCommand := NewDummyOSCommand()\n\n\tosCommand.Platform.os = \"linux\"\n\n\tactual := osCommand.Quote(`hello \"test\"`)\n\n\texpected := `\"hello \\\"test\\\"\"`\n\n\tassert.EqualValues(t, expected, actual)\n}\n\n// TestOSCommandQuoteWindows tests the quote function for Windows\nfunc TestOSCommandQuoteWindows(t *testing.T) {\n\tosCommand := NewDummyOSCommand()\n\n\tosCommand.Platform.os = \"windows\"\n\n\tactual := osCommand.Quote(`hello \"test\" 'test2'`)\n\n\texpected := `\\\"hello \"'\"'\"test\"'\"'\" 'test2'\\\"`\n\n\tassert.EqualValues(t, expected, actual)\n}\n\n// TestOSCommandUnquote is a function.\nfunc TestOSCommandUnquote(t *testing.T) {\n\tosCommand := NewDummyOSCommand()\n\n\tactual := osCommand.Unquote(`hello \"test\"`)\n\n\texpected := \"hello test\"\n\n\tassert.EqualValues(t, expected, actual)\n}\n\n// TestOSCommandFileType is a function.\nfunc TestOSCommandFileType(t *testing.T) {\n\ttype scenario struct {\n\t\tpath  string\n\t\tsetup func()\n\t\ttest  func(string)\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"testFile\",\n\t\t\tfunc() {\n\t\t\t\tif _, err := os.Create(\"testFile\"); err != nil {\n\t\t\t\t\tpanic(err)\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunc(output string) {\n\t\t\t\tassert.EqualValues(t, \"file\", output)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"file with spaces\",\n\t\t\tfunc() {\n\t\t\t\tif _, err := os.Create(\"file with spaces\"); err != nil {\n\t\t\t\t\tpanic(err)\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunc(output string) {\n\t\t\t\tassert.EqualValues(t, \"file\", output)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"testDirectory\",\n\t\t\tfunc() {\n\t\t\t\tif err := os.Mkdir(\"testDirectory\", 0o644); err != nil {\n\t\t\t\t\tpanic(err)\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunc(output string) {\n\t\t\t\tassert.EqualValues(t, \"directory\", output)\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"nonExistant\",\n\t\t\tfunc() {},\n\t\t\tfunc(output string) {\n\t\t\t\tassert.EqualValues(t, \"other\", output)\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\ts.setup()\n\t\ts.test(NewDummyOSCommand().FileType(s.path))\n\t\t_ = os.RemoveAll(s.path)\n\t}\n}\n\nfunc TestOSCommandCreateTempFile(t *testing.T) {\n\ttype scenario struct {\n\t\ttestName string\n\t\tfilename string\n\t\tcontent  string\n\t\ttest     func(string, error)\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"valid case\",\n\t\t\t\"filename\",\n\t\t\t\"content\",\n\t\t\tfunc(path string, err error) {\n\t\t\t\tassert.NoError(t, err)\n\n\t\t\t\tcontent, err := os.ReadFile(path)\n\t\t\t\tassert.NoError(t, err)\n\n\t\t\t\tassert.Equal(t, \"content\", string(content))\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tt.Run(s.testName, func(t *testing.T) {\n\t\t\ts.test(NewDummyOSCommand().CreateTempFile(s.filename, s.content))\n\t\t})\n\t}\n}\n\nfunc TestOSCommandExecutableFromStringWithShellLinux(t *testing.T) {\n\tosCommand := NewDummyOSCommand()\n\n\tosCommand.Platform.os = \"linux\"\n\n\ttests := []struct {\n\t\tname       string\n\t\tcommandStr string\n\t\twant       string\n\t}{\n\t\t{\n\t\t\t\"success\",\n\t\t\t\"pwd\",\n\t\t\tfmt.Sprintf(\"%v %v %v\", osCommand.Platform.shell, osCommand.Platform.shellArg, \"\\\"pwd\\\"\"),\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgot := osCommand.NewCommandStringWithShell(tt.commandStr)\n\t\t\tassert.Equal(t, tt.want, got)\n\t\t})\n\t}\n}\n\nfunc TestOSCommandNewCommandStringWithShellWindows(t *testing.T) {\n\tosCommand := NewDummyOSCommand()\n\n\tosCommand.Platform.os = \"windows\"\n\n\ttests := []struct {\n\t\tname       string\n\t\tcommandStr string\n\t\twant       string\n\t}{\n\t\t{\n\t\t\t\"success\",\n\t\t\t\"pwd\",\n\t\t\tfmt.Sprintf(\"%v %v %v\", osCommand.Platform.shell, osCommand.Platform.shellArg, \"pwd\"),\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgot := osCommand.NewCommandStringWithShell(tt.commandStr)\n\t\t\tassert.Equal(t, tt.want, got)\n\t\t})\n\t}\n}\n\n// Path: pkg/commands/os_windows.go\npackage commands\n\nfunc getPlatform() *Platform {\n\treturn &Platform{\n\t\tos:       \"windows\",\n\t\tshell:    \"cmd\",\n\t\tshellArg: \"/c\",\n\t}\n}\n\n// Path: pkg/commands/project.go\npackage commands\n\ntype Project struct {\n\tName string\n}\n\n// Path: pkg/commands/ssh/ssh_test.go\npackage ssh\n\nimport (\n\t\"context\"\n\t\"io\"\n\t\"os/exec\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestSSHHandlerHandleSSHDockerHost(t *testing.T) {\n\ttype scenario struct {\n\t\ttestName                 string\n\t\tenvVarValue              string\n\t\texpectedDialContextCount int\n\t\texpectedStartCmdCount    int\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\ttestName:                 \"No env var set\",\n\t\t\tenvVarValue:              \"\",\n\t\t\texpectedDialContextCount: 0,\n\t\t\texpectedStartCmdCount:    0,\n\t\t},\n\t\t{\n\t\t\ttestName:                 \"Env var set with https scheme\",\n\t\t\tenvVarValue:              \"https://myhost.com\",\n\t\t\texpectedStartCmdCount:    0,\n\t\t\texpectedDialContextCount: 0,\n\t\t},\n\t\t{\n\t\t\ttestName:                 \"Env var set with ssh scheme\",\n\t\t\tenvVarValue:              \"ssh://myhost@192.168.5.178\",\n\t\t\texpectedStartCmdCount:    1,\n\t\t\texpectedDialContextCount: 1,\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\ts := s\n\t\tt.Run(s.testName, func(t *testing.T) {\n\t\t\tgetenv := func(key string) string {\n\t\t\t\tif key != \"DOCKER_HOST\" {\n\t\t\t\t\tt.Errorf(\"Expected key to be DOCKER_HOST, got %s\", key)\n\t\t\t\t}\n\n\t\t\t\treturn s.envVarValue\n\t\t\t}\n\n\t\t\ttempDir := func(dir string, pattern string) (string, error) {\n\t\t\t\tassert.Equal(t, \"/tmp\", dir)\n\t\t\t\tassert.Equal(t, \"lazydocker-sshtunnel-\", pattern)\n\n\t\t\t\treturn \"/tmp/lazydocker-ssh-tunnel-12345\", nil\n\t\t\t}\n\n\t\t\tsetenv := func(key, value string) error {\n\t\t\t\tassert.Equal(t, \"DOCKER_HOST\", key)\n\t\t\t\tassert.Equal(t, \"unix:///tmp/lazydocker-ssh-tunnel-12345/dockerhost.sock\", value)\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tstartCmdCount := 0\n\t\t\tstartCmd := func(cmd *exec.Cmd) error {\n\t\t\t\tassert.EqualValues(t, []string{\"ssh\", \"-L\", \"/tmp/lazydocker-ssh-tunnel-12345/dockerhost.sock:/var/run/docker.sock\", \"192.168.5.178\", \"-N\"}, cmd.Args)\n\n\t\t\t\tstartCmdCount++\n\n\t\t\t\treturn nil\n\t\t\t}\n\n\t\t\tdialContextCount := 0\n\t\t\tdialContext := func(ctx context.Context, network string, address string) (io.Closer, error) {\n\t\t\t\tassert.Equal(t, \"unix\", network)\n\t\t\t\tassert.Equal(t, \"/tmp/lazydocker-ssh-tunnel-12345/dockerhost.sock\", address)\n\n\t\t\t\tdialContextCount++\n\n\t\t\t\treturn noopCloser{}, nil\n\t\t\t}\n\n\t\t\thandler := &SSHHandler{\n\t\t\t\toSCommand: &fakeCmdKiller{},\n\n\t\t\t\tdialContext: dialContext,\n\t\t\t\tstartCmd:    startCmd,\n\t\t\t\ttempDir:     tempDir,\n\t\t\t\tgetenv:      getenv,\n\t\t\t\tsetenv:      setenv,\n\t\t\t}\n\n\t\t\t_, err := handler.HandleSSHDockerHost()\n\t\t\tassert.NoError(t, err)\n\n\t\t\tassert.Equal(t, s.expectedDialContextCount, dialContextCount)\n\t\t\tassert.Equal(t, s.expectedStartCmdCount, startCmdCount)\n\t\t})\n\t}\n}\n\ntype fakeCmdKiller struct{}\n\nfunc (self *fakeCmdKiller) Kill(cmd *exec.Cmd) error {\n\treturn nil\n}\n\nfunc (self *fakeCmdKiller) PrepareForChildren(cmd *exec.Cmd) {}\n\n// Path: pkg/commands/volume.go\npackage commands\n\nimport (\n\t\"context\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/docker/docker/api/types/filters\"\n\t\"github.com/docker/docker/client\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Volume : A docker Volume\ntype Volume struct {\n\tName          string\n\tVolume        *dockerTypes.Volume\n\tClient        *client.Client\n\tOSCommand     *OSCommand\n\tLog           *logrus.Entry\n\tDockerCommand LimitedDockerCommand\n}\n\n// RefreshVolumes gets the volumes and stores them\nfunc (c *DockerCommand) RefreshVolumes() ([]*Volume, error) {\n\tresult, err := c.Client.VolumeList(context.Background(), filters.Args{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvolumes := result.Volumes\n\n\townVolumes := make([]*Volume, len(volumes))\n\n\tfor i, volume := range volumes {\n\t\townVolumes[i] = &Volume{\n\t\t\tName:          volume.Name,\n\t\t\tVolume:        volume,\n\t\t\tClient:        c.Client,\n\t\t\tOSCommand:     c.OSCommand,\n\t\t\tLog:           c.Log,\n\t\t\tDockerCommand: c,\n\t\t}\n\t}\n\n\treturn ownVolumes, nil\n}\n\n// PruneVolumes prunes volumes\nfunc (c *DockerCommand) PruneVolumes() error {\n\t_, err := c.Client.VolumesPrune(context.Background(), filters.Args{})\n\treturn err\n}\n\n// Remove removes the volume\nfunc (v *Volume) Remove(force bool) error {\n\treturn v.Client.VolumeRemove(context.Background(), v.Name, force)\n}\n\n// Path: pkg/config/app_config.go\n// Package config handles all the user-configuration. The fields here are\n// all in PascalCase but in your actual config.yml they'll be in camelCase.\n// You can view the default config with `lazydocker --config`.\n// You can open your config file by going to the status panel (using left-arrow)\n// and pressing 'o'.\n// You can directly edit the file (e.g. in vim) by pressing 'e' instead.\n// To see the final config after your user-specific options have been merged\n// with the defaults, go to the 'about' tab in the status panel.\n// Because of the way we merge your user config with the defaults you may need\n// to be careful: if for example you set a `commandTemplates:` yaml key but then\n// give it no child values, it will scrap all of the defaults and the app will\n// probably crash.\npackage config\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/OpenPeeDeeP/xdg\"\n\t\"github.com/jesseduffield/yaml\"\n)\n\n// UserConfig holds all of the user-configurable options\ntype UserConfig struct {\n\t// Gui is for configuring visual things like colors and whether we show or\n\t// hide things\n\tGui GuiConfig `yaml:\"gui,omitempty\"`\n\n\t// ConfirmOnQuit when enabled prompts you to confirm you want to quit when you\n\t// hit esc or q when no confirmation panels are open\n\tConfirmOnQuit bool `yaml:\"confirmOnQuit,omitempty\"`\n\n\t// Logs determines how we render/filter a container's logs\n\tLogs LogsConfig `yaml:\"logs,omitempty\"`\n\n\t// CommandTemplates determines what commands actually get called when we run\n\t// certain commands\n\tCommandTemplates CommandTemplatesConfig `yaml:\"commandTemplates,omitempty\"`\n\n\t// CustomCommands determines what shows up in your custom commands menu when\n\t// you press 'c'. You can use go templates to access three items on the\n\t// struct: the DockerCompose command (defaulted to 'docker-compose'), the\n\t// Service if present, and the Container if present. The struct types for\n\t// those are found in the commands package\n\tCustomCommands CustomCommands `yaml:\"customCommands,omitempty\"`\n\n\t// BulkCommands are commands that apply to all items in a panel e.g.\n\t// killing all containers, stopping all services, or pruning all images\n\tBulkCommands CustomCommands `yaml:\"bulkCommands,omitempty\"`\n\n\t// OS determines what defaults are set for opening files and links\n\tOS OSConfig `yaml:\"oS,omitempty\"`\n\n\t// Stats determines how long lazydocker will gather container stats for, and\n\t// what stat info to graph\n\tStats StatsConfig `yaml:\"stats,omitempty\"`\n\n\t// Replacements determines how we render an item's info\n\tReplacements Replacements `yaml:\"replacements,omitempty\"`\n\n\t// For demo purposes: any list item with one of these strings as a substring\n\t// will be filtered out and not displayed.\n\t// Not documented because it's subject to change\n\tIgnore []string `yaml:\"ignore,omitempty\"`\n}\n\n// ThemeConfig is for setting the colors of panels and some text.\ntype ThemeConfig struct {\n\tActiveBorderColor   []string `yaml:\"activeBorderColor,omitempty\"`\n\tInactiveBorderColor []string `yaml:\"inactiveBorderColor,omitempty\"`\n\tSelectedLineBgColor []string `yaml:\"selectedLineBgColor,omitempty\"`\n\tOptionsTextColor    []string `yaml:\"optionsTextColor,omitempty\"`\n}\n\n// GuiConfig is for configuring visual things like colors and whether we show or\n// hide things\ntype GuiConfig struct {\n\t// ScrollHeight determines how many characters you scroll at a time when\n\t// scrolling the main panel\n\tScrollHeight int `yaml:\"scrollHeight,omitempty\"`\n\n\t// Language determines which language the GUI displayed.\n\tLanguage string `yaml:\"language,omitempty\"`\n\n\t// ScrollPastBottom determines whether you can scroll past the bottom of the\n\t// main view\n\tScrollPastBottom bool `yaml:\"scrollPastBottom,omitempty\"`\n\n\t// IgnoreMouseEvents is for when you do not want to use your mouse to interact\n\t// with anything\n\tIgnoreMouseEvents bool `yaml:\"mouseEvents,omitempty\"`\n\n\t// Theme determines what colors and color attributes your panel borders have.\n\t// I always set inactiveBorderColor to black because in my terminal it's more\n\t// of a grey, but that doesn't work in your average terminal. I highly\n\t// recommended finding a combination that works for you\n\tTheme ThemeConfig `yaml:\"theme,omitempty\"`\n\n\t// ShowAllContainers determines whether the Containers panel contains all the\n\t// containers returned by `docker ps -a`, or just those containers that aren't\n\t// directly linked to a service. It is probably desirable to enable this if\n\t// you have multiple containers per service, but otherwise it can cause a lot\n\t// of clutter\n\tShowAllContainers bool `yaml:\"showAllContainers,omitempty\"`\n\n\t// ReturnImmediately determines whether you get the 'press enter to return to\n\t// lazydocker' message after a subprocess has completed. You would set this to\n\t// true if you often want to see the output of subprocesses before returning\n\t// to lazydocker. I would default this to false but then people who want it\n\t// set to true won't even know the config option exists.\n\tReturnImmediately bool `yaml:\"returnImmediately,omitempty\"`\n\n\t// WrapMainPanel determines whether we use word wrap on the main panel\n\tWrapMainPanel bool `yaml:\"wrapMainPanel,omitempty\"`\n\n\t// LegacySortContainers determines if containers should be sorted using legacy approach.\n\t// By default, containers are now sorted by status. This setting allows users to\n\t// use legacy behaviour instead.\n\tLegacySortContainers bool `yaml:\"legacySortContainers,omitempty\"`\n\n\t// If 0.333, then the side panels will be 1/3 of the screen's width\n\tSidePanelWidth float64 `yaml:\"sidePanelWidth\"`\n\n\t// Determines whether we show the bottom line (the one containing keybinding\n\t// info and the status of the app).\n\tShowBottomLine bool `yaml:\"showBottomLine\"`\n\n\t// When true, increases vertical space used by focused side panel,\n\t// creating an accordion effect\n\tExpandFocusedSidePanel bool `yaml:\"expandFocusedSidePanel\"`\n\n\t// ScreenMode allow user to specify which screen mode will be used on startup\n\tScreenMode string `yaml:\"screenMode,omitempty\"`\n\n\t// Determines the style of the container status and container health display in the\n\t// containers panel. \"long\": full words (default), \"short\": one or two characters,\n\t// \"icon\": unicode emoji.\n\tContainerStatusHealthStyle string `yaml:\"containerStatusHealthStyle\"`\n}\n\n// CommandTemplatesConfig determines what commands actually get called when we\n// run certain commands\ntype CommandTemplatesConfig struct {\n\t// RestartService is for restarting a service. docker-compose restart {{\n\t// .Service.Name }} works but I prefer docker-compose up --force-recreate {{\n\t// .Service.Name }}\n\tRestartService string `yaml:\"restartService,omitempty\"`\n\n\t// StartService is just like the above but for starting\n\tStartService string `yaml:\"startService,omitempty\"`\n\n\t// UpService ups the service (creates and starts)\n\tUpService string `yaml:\"upService,omitempty\"`\n\n\t// Runs \"docker-compose up -d\"\n\tUp string `yaml:\"up,omitempty\"`\n\n\t// downs everything\n\tDown string `yaml:\"down,omitempty\"`\n\t// downs and removes volumes\n\tDownWithVolumes string `yaml:\"downWithVolumes,omitempty\"`\n\n\t// DockerCompose is for your docker-compose command. You may want to combine a\n\t// few different docker-compose.yml files together, in which case you can set\n\t// this to \"docker-compose -f foo/docker-compose.yml -f\n\t// bah/docker-compose.yml\". The reason that the other docker-compose command\n\t// templates all start with {{ .DockerCompose }} is so that they can make use\n\t// of whatever you've set in this value rather than you having to copy and\n\t// paste it to all the other commands\n\tDockerCompose string `yaml:\"dockerCompose,omitempty\"`\n\n\t// StopService is the command for stopping a service\n\tStopService string `yaml:\"stopService,omitempty\"`\n\n\t// ServiceLogs get the logs for a service. This is actually not currently\n\t// used; we just get the logs of the corresponding container. But we should\n\t// probably support explicitly returning the logs of the service when you've\n\t// selected the service, given that a service may have multiple containers.\n\tServiceLogs string `yaml:\"serviceLogs,omitempty\"`\n\n\t// ViewServiceLogs is for when you want to view the logs of a service as a\n\t// subprocess. This defaults to having no filter, unlike the in-app logs\n\t// commands which will usually filter down to the last hour for the sake of\n\t// performance.\n\tViewServiceLogs string `yaml:\"viewServiceLogs,omitempty\"`\n\n\t// RebuildService is the command for rebuilding a service. Defaults to\n\t// something along the lines of `{{ .DockerCompose }} up --build {{\n\t// .Service.Name }}`\n\tRebuildService string `yaml:\"rebuildService,omitempty\"`\n\n\t// RecreateService is for force-recreating a service. I prefer this to\n\t// restarting a service because it will also restart any dependent services\n\t// and ensure they're running before trying to run the service at hand\n\tRecreateService string `yaml:\"recreateService,omitempty\"`\n\n\t// AllLogs is for showing what you get from doing `docker-compose logs`. It\n\t// combines all the logs together\n\tAllLogs string `yaml:\"allLogs,omitempty\"`\n\n\t// ViewAllLogs is the command we use when you want to see all logs in a subprocess with no filtering\n\tViewAllLogs string `yaml:\"viewAlLogs,omitempty\"`\n\n\t// DockerComposeConfig is the command for viewing the config of your docker\n\t// compose. It basically prints out the yaml from your docker-compose.yml\n\t// file(s)\n\tDockerComposeConfig string `yaml:\"dockerComposeConfig,omitempty\"`\n\n\t// CheckDockerComposeConfig is what we use to check whether we are in a\n\t// docker-compose context. If the command returns an error then we clearly\n\t// aren't in a docker-compose config and we then just hide the services panel\n\t// and only show containers\n\tCheckDockerComposeConfig string `yaml:\"checkDockerComposeConfig,omitempty\"`\n\n\t// ServiceTop is the command for viewing the processes under a given service\n\tServiceTop string `yaml:\"serviceTop,omitempty\"`\n}\n\n// OSConfig contains config on the level of the os\ntype OSConfig struct {\n\t// OpenCommand is the command for opening a file\n\tOpenCommand string `yaml:\"openCommand,omitempty\"`\n\n\t// OpenCommand is the command for opening a link\n\tOpenLinkCommand string `yaml:\"openLinkCommand,omitempty\"`\n}\n\n// GraphConfig specifies how to make a graph of recorded container stats\ntype GraphConfig struct {\n\t// Min sets the minimum value that you want to display. If you want to set\n\t// this, you should also set MinType to \"static\". The reason for this is that\n\t// if Min == 0, it's not clear if it has not been set (given that the\n\t// zero-value of an int is 0) or if it's intentionally been set to 0.\n\tMin float64 `yaml:\"min,omitempty\"`\n\n\t// Max sets the maximum value that you want to display. If you want to set\n\t// this, you should also set MaxType to \"static\". The reason for this is that\n\t// if Max == 0, it's not clear if it has not been set (given that the\n\t// zero-value of an int is 0) or if it's intentionally been set to 0.\n\tMax float64 `yaml:\"max,omitempty\"`\n\n\t// Height sets the height of the graph in ascii characters\n\tHeight int `yaml:\"height,omitempty\"`\n\n\t// Caption sets the caption of the graph. If you want to show CPU Percentage\n\t// you could set this to \"CPU (%)\"\n\tCaption string `yaml:\"caption,omitempty\"`\n\n\t// This is the path to the stat that you want to display. It is based on the\n\t// RecordedStats struct in container_stats.go, so feel free to look there to\n\t// see all the options available. Alternatively if you go into lazydocker and\n\t// go to the stats tab, you'll see that same struct in JSON format, so you can\n\t// just PascalCase the path and you'll have a valid path. E.g.\n\t// ClientStats.blkio_stats -> \"ClientStats.BlkioStats\"\n\tStatPath string `yaml:\"statPath,omitempty\"`\n\n\t// This determines the color of the graph. This can be any color attribute,\n\t// e.g. 'blue', 'green'\n\tColor string `yaml:\"color,omitempty\"`\n\n\t// MinType and MaxType are each one of \"\", \"static\". blank means the min/max\n\t// of the data set will be used. \"static\" means the min/max specified will be\n\t// used\n\tMinType string `yaml:\"minType,omitempty\"`\n\n\t// MaxType is just like MinType but for the max value\n\tMaxType string `yaml:\"maxType,omitempty\"`\n}\n\n// StatsConfig contains the stuff relating to stats and graphs\ntype StatsConfig struct {\n\t// Graphs contains the configuration for the stats graphs we want to show in\n\t// the app\n\tGraphs []GraphConfig\n\n\t// MaxDuration tells us how long to collect stats for. Currently this defaults\n\t// to \"5m\" i.e. 5 minutes.\n\tMaxDuration time.Duration `yaml:\"maxDuration,omitempty\"`\n}\n\n// CustomCommands contains the custom commands that you might want to use on any\n// given service or container\ntype CustomCommands struct {\n\t// Containers contains the custom commands for containers\n\tContainers []CustomCommand `yaml:\"containers,omitempty\"`\n\n\t// Services contains the custom commands for services\n\tServices []CustomCommand `yaml:\"services,omitempty\"`\n\n\t// Images contains the custom commands for images\n\tImages []CustomCommand `yaml:\"images,omitempty\"`\n\n\t// Volumes contains the custom commands for volumes\n\tVolumes []CustomCommand `yaml:\"volumes,omitempty\"`\n\n\t// Networks contains the custom commands for networks\n\tNetworks []CustomCommand `yaml:\"networks,omitempty\"`\n}\n\n// Replacements contains the stuff relating to rendering a container's info\ntype Replacements struct {\n\t// ImageNamePrefixes tells us how to replace a prefix in the Docker image name\n\tImageNamePrefixes map[string]string `yaml:\"imageNamePrefixes,omitempty\"`\n}\n\n// CustomCommand is a template for a command we want to run against a service or\n// container\ntype CustomCommand struct {\n\t// Name is the name of the command, purely for visual display\n\tName string `yaml:\"name\"`\n\n\t// Attach tells us whether to switch to a subprocess to interact with the\n\t// called program, or just read its output. If Attach is set to false, the\n\t// command will run in the background. I'm open to the idea of having a third\n\t// option where the output plays in the main panel.\n\tAttach bool `yaml:\"attach\"`\n\n\t// Shell indicates whether to invoke the Command on a shell or not.\n\t// Example of a bash invoked command: `/bin/bash -c \"{Command}\".\n\tShell bool `yaml:\"shell\"`\n\n\t// Command is the command we want to run. We can use the go templates here as\n\t// well. One example might be `{{ .DockerCompose }} exec {{ .Service.Name }}\n\t// /bin/sh`\n\tCommand string `yaml:\"command\"`\n\n\t// ServiceNames is used to restrict this command to just one or more services.\n\t// An example might be 'rails migrate' for your rails api service(s). This\n\t// field has no effect on customcommands under the 'communications' part of\n\t// the customCommand config.\n\tServiceNames []string `yaml:\"serviceNames\"`\n\n\t// InternalFunction is the name of a function inside lazydocker that we want to run, as opposed to a command-line command. This is only used internally and can't be configured by the user\n\tInternalFunction func() error `yaml:\"-\"`\n}\n\ntype LogsConfig struct {\n\tTimestamps bool   `yaml:\"timestamps,omitempty\"`\n\tSince      string `yaml:\"since,omitempty\"`\n\tTail       string `yaml:\"tail,omitempty\"`\n}\n\n// GetDefaultConfig returns the application default configuration NOTE (to\n// contributors, not users): do not default a boolean to true, because false is\n// the boolean zero value and this will be ignored when parsing the user's\n// config\nfunc GetDefaultConfig() UserConfig {\n\tduration, err := time.ParseDuration(\"3m\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\treturn UserConfig{\n\t\tGui: GuiConfig{\n\t\t\tScrollHeight:      2,\n\t\t\tLanguage:          \"auto\",\n\t\t\tScrollPastBottom:  false,\n\t\t\tIgnoreMouseEvents: false,\n\t\t\tTheme: ThemeConfig{\n...\n", "needle_token_start": 547, "needle_token_end": 1015, "code_context_ntokens": 15099, "output": ["```go\nfunc NewDockerCommand(log *logrus.Entry, osCommand *OSCommand, tr *i18n.TranslationSet, config *config.AppConfig, errorChan chan error) (*DockerCommand, error) {\n\ttunnelCloser, err := ssh.NewSSHHandler(osCommand).HandleSSHDockerHost()\n\tif err != nil {\n\t\togLog.Fatal(err)\n\t}\n\n\tdockerHost, err := determineDockerHost()\n\tif err != nil {\n\t\togLog.Printf(\"> could not determine host %v\", err)\n\t}\n\n\tcli, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion(APIVersion), client.WithHost(dockerHost))\n\tif err != nil {\n\t\togLog.Fatal(err)\n\t}\n\n\tdockerCommand := &DockerCommand{\n\t\tLog:                    log,\n\t\tOSCommand:              osCommand,\n\t\tTr:                     tr,\n\t\tConfig:                 config,\n\t\tClient:                 cli,\n\t\tErrorChan:              errorChan,\n\t\tInDockerComposeProject: true,\n\t\tClosers:                []io.Closer{tunnelCloser},\n\t}\n\n\tcommand := utils.ApplyTemplate(\n\t\tconfig.UserConfig.CommandTemplates.CheckDockerComposeConfig,\n\t\tdockerCommand.NewCommandObject(CommandObject{}),\n\t)\n\n\tlog.Warn(command)\n\n\terr = osCommand.RunCommand(\n\t\tutils.ApplyTemplate(\n\t\t\tconfig.UserConfig.CommandTemplates.CheckDockerComposeConfig,\n\t\t\tdockerCommand.NewCommandObject(CommandObject{}),\n\t\t),\n\t)\n\tif err != nil {\n\t\tdockerCommand.InDockerComposeProject = false\n\t\tlog.Warn(err.Error())\n\t}\n\n\treturn dockerCommand, nil\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "RenderStats", "language": "go", "path": "pkg/gui/presentation/container_stats.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To generate a visual representation and detailed statistics of a container's performance metrics.\n2. **Input**: User configuration settings, a container object, and the width of the display area.\n3. **Output**: A formatted string containing graphical representations of performance metrics and detailed statistical data, or an error if the process fails.\n4. **Procedure**: \n   - Retrieve the latest statistics from the container.\n   - If statistics are available, generate graphs for specified metrics using the provided width.\n   - Format network traffic data and the current number of process IDs.\n   - Convert all statistical data into a YAML formatted string.\n   - Combine all elements (graphs, formatted data strings, and YAML data) into a single string for display.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/gui/panels/side_list_panel.go\npackage panels\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\ntype ISideListPanel interface {\n\tSetMainTabIndex(int)\n\tHandleSelect() error\n\tGetView() *gocui.View\n\tRefocus()\n\tRerenderList() error\n\tIsFilterDisabled() bool\n\tIsHidden() bool\n\tHandleNextLine() error\n\tHandlePrevLine() error\n\tHandleClick() error\n\tHandlePrevMainTab() error\n\tHandleNextMainTab() error\n}\n\n// list panel at the side of the screen that renders content to the main panel\ntype SideListPanel[T comparable] struct {\n\tContextState *ContextState[T]\n\n\tListPanel[T]\n\n\t// message to render in the main view if there are no items in the panel\n\t// and it has focus. Leave empty if you don't want to render anything\n\tNoItemsMessage string\n\n\t// a representation of the gui\n\tGui IGui\n\n\t// this Filter is applied on top of additional default filters\n\tFilter func(T) bool\n\tSort   func(a, b T) bool\n\n\t// a callback to invoke when the item is clicked\n\tOnClick func(T) error\n\n\t// returns the cells that we render to the view in a table format. The cells will\n\t// be rendered with padding.\n\tGetTableCells func(T) []string\n\n\t// function to be called after re-rendering list. Can be nil\n\tOnRerender func() error\n\n\t// set this to true if you don't want to allow manual filtering via '/'\n\tDisableFilter bool\n\n\t// This can be nil if you want to always show the panel\n\tHide func() bool\n}\n\nvar _ ISideListPanel = &SideListPanel[int]{}\n\ntype IGui interface {\n\tHandleClick(v *gocui.View, itemCount int, selectedLine *int, handleSelect func() error) error\n\tNewSimpleRenderStringTask(getContent func() string) tasks.TaskFunc\n\tFocusY(selectedLine int, itemCount int, view *gocui.View)\n\tShouldRefresh(contextKey string) bool\n\tGetMainView() *gocui.View\n\tIsCurrentView(*gocui.View) bool\n\tFilterString(view *gocui.View) string\n\tIgnoreStrings() []string\n\tUpdate(func() error)\n\n\tQueueTask(f func(ctx context.Context)) error\n}\n\nfunc (self *SideListPanel[T]) HandleClick() error {\n\titemCount := self.List.Len()\n\thandleSelect := self.HandleSelect\n\tselectedLine := &self.SelectedIdx\n\n\tif err := self.Gui.HandleClick(self.View, itemCount, selectedLine, handleSelect); err != nil {\n\t\treturn err\n\t}\n\n\tif self.OnClick != nil {\n\t\tselectedItem, err := self.GetSelectedItem()\n\t\tif err == nil {\n\t\t\treturn self.OnClick(selectedItem)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (self *SideListPanel[T]) GetView() *gocui.View {\n\treturn self.View\n}\n\nfunc (self *SideListPanel[T]) HandleSelect() error {\n\titem, err := self.GetSelectedItem()\n\tif err != nil {\n\t\tif err.Error() != self.NoItemsMessage {\n\t\t\treturn err\n\t\t}\n\n\t\tif self.NoItemsMessage != \"\" {\n\t\t\tself.Gui.NewSimpleRenderStringTask(func() string { return self.NoItemsMessage })\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tself.Refocus()\n\n\treturn self.renderContext(item)\n}\n\nfunc (self *SideListPanel[T]) renderContext(item T) error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tkey := self.ContextState.GetCurrentContextKey(item)\n\tif !self.Gui.ShouldRefresh(key) {\n\t\treturn nil\n\t}\n\n\tmainView := self.Gui.GetMainView()\n\tmainView.Tabs = self.ContextState.GetMainTabTitles()\n\tmainView.TabIndex = self.ContextState.mainTabIdx\n\n\ttask := self.ContextState.GetCurrentMainTab().Render(item)\n\n\treturn self.Gui.QueueTask(task)\n}\n\nfunc (self *SideListPanel[T]) GetSelectedItem() (T, error) {\n\tvar zero T\n\n\titem, ok := self.List.TryGet(self.SelectedIdx)\n\tif !ok {\n\t\t// could probably have a better error here\n\t\treturn zero, errors.New(self.NoItemsMessage)\n\t}\n\n\treturn item, nil\n}\n\nfunc (self *SideListPanel[T]) HandleNextLine() error {\n\tself.SelectNextLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevLine() error {\n\tself.SelectPrevLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandleNextMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandleNextMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandlePrevMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) Refocus() {\n\tself.Gui.FocusY(self.SelectedIdx, self.List.Len(), self.View)\n}\n\nfunc (self *SideListPanel[T]) SetItems(items []T) {\n\tself.List.SetItems(items)\n\tself.FilterAndSort()\n}\n\nfunc (self *SideListPanel[T]) FilterAndSort() {\n\tfilterString := self.Gui.FilterString(self.View)\n\n\tself.List.Filter(func(item T, index int) bool {\n\t\tif self.Filter != nil && !self.Filter(item) {\n\t\t\treturn false\n\t\t}\n\n\t\tif lo.SomeBy(self.Gui.IgnoreStrings(), func(ignore string) bool {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, ignore)\n\t\t\t})\n\t\t}) {\n\t\t\treturn false\n\t\t}\n\n\t\tif filterString != \"\" {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, filterString)\n\t\t\t})\n\t\t}\n\n\t\treturn true\n\t})\n\n\tself.List.Sort(self.Sort)\n\n\tself.clampSelectedLineIdx()\n}\n\nfunc (self *SideListPanel[T]) RerenderList() error {\n\tself.FilterAndSort()\n\n\tself.Gui.Update(func() error {\n\t\tself.View.Clear()\n...\n// Path: pkg/gui/presentation/container_stats.go\npackage presentation\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/asciigraph\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/mcuadros/go-lookup\"\n\t\"github.com/samber/lo\"\n)\n\n\nfunc RenderStats(userConfig *config.UserConfig, container *commands.Container, viewWidth int) (string, error) {\n\tstats, ok := container.GetLastStats()\n\tif !ok {\n\t\treturn \"\", nil\n\t}\n\n\tgraphSpecs := userConfig.Stats.Graphs\n\tgraphs := make([]string, len(graphSpecs))\n\tfor i, spec := range graphSpecs {\n\t\tgraph, err := plotGraph(container, spec, viewWidth-10)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tgraphs[i] = utils.ColoredString(graph, utils.GetColorAttribute(spec.Color))\n\t}\n\n\tpidsCount := fmt.Sprintf(\"PIDs: %d\", stats.ClientStats.PidsStats.Current)\n\tdataReceived := fmt.Sprintf(\"Traffic received: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.RxBytes))\n\tdataSent := fmt.Sprintf(\"Traffic sent: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.TxBytes))\n\n\toriginalStats, err := utils.MarshalIntoYaml(stats)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tcontents := fmt.Sprintf(\"\\n\\n%s\\n\\n%s\\n\\n%s\\n%s\\n\\n%s\",\n\t\tutils.ColoredString(strings.Join(graphs, \"\\n\\n\"), color.FgGreen),\n\t\tpidsCount,\n\t\tdataReceived,\n\t\tdataSent,\n\t\tutils.ColoredYamlString(string(originalStats)),\n\t)\n\n\treturn contents, nil\n}\n\n// plotGraph returns the plotted graph based on the graph spec and the stat history\nfunc plotGraph(container *commands.Container, spec config.GraphConfig, width int) (string, error) {\n\tcontainer.StatsMutex.Lock()\n\tdefer container.StatsMutex.Unlock()\n\n\tdata := make([]float64, len(container.StatHistory))\n\n\tfor i, stats := range container.StatHistory {\n\t\tvalue, err := lookup.LookupString(stats, spec.StatPath)\n\t\tif err != nil {\n\t\t\treturn \"Could not find key: \" + spec.StatPath, nil\n\t\t}\n\t\tfloatValue, err := getFloat(value.Interface())\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tdata[i] = floatValue\n\t}\n\n\tmax := spec.Max\n\tif spec.MaxType == \"\" {\n\t\tmax = lo.Max(data)\n\t}\n\n\tmin := spec.Min\n\tif spec.MinType == \"\" {\n\t\tmin = lo.Min(data)\n\t}\n\n\theight := 10\n\tif spec.Height > 0 {\n\t\theight = spec.Height\n\t}\n\n\tcaption := fmt.Sprintf(\n\t\t\"%s: %0.2f (%v)\",\n\t\tspec.Caption,\n\t\tdata[len(data)-1],\n\t\ttime.Since(container.StatHistory[0].RecordedAt).Round(time.Second),\n\t)\n\n\treturn asciigraph.Plot(\n\t\tdata,\n\t\tasciigraph.Height(height),\n\t\tasciigraph.Width(width),\n\t\tasciigraph.Min(min),\n\t\tasciigraph.Max(max),\n\t\tasciigraph.Caption(caption),\n\t), nil\n}\n\n// from Dave C's answer at https://stackoverflow.com/questions/20767724/converting-unknown-interface-to-float64-in-golang\nfunc getFloat(unk interface{}) (float64, error) {\n\tfloatType := reflect.TypeOf(float64(0))\n\tstringType := reflect.TypeOf(\"\")\n\n\tswitch i := unk.(type) {\n\tcase float64:\n\t\treturn i, nil\n\tcase float32:\n\t\treturn float64(i), nil\n\tcase int64:\n\t\treturn float64(i), nil\n\tcase int32:\n\t\treturn float64(i), nil\n\tcase int:\n\t\treturn float64(i), nil\n\tcase uint64:\n\t\treturn float64(i), nil\n\tcase uint32:\n\t\treturn float64(i), nil\n\tcase uint:\n\t\treturn float64(i), nil\n\tcase string:\n\t\treturn strconv.ParseFloat(i, 64)\n\tdefault:\n\t\tv := reflect.ValueOf(unk)\n\t\tv = reflect.Indirect(v)\n\t\tif v.Type().ConvertibleTo(floatType) {\n\t\t\tfv := v.Convert(floatType)\n\t\t\treturn fv.Float(), nil\n\t\t} else if v.Type().ConvertibleTo(stringType) {\n\t\t\tsv := v.Convert(stringType)\n\t\t\ts := sv.String()\n\t\t\treturn strconv.ParseFloat(s, 64)\n\t\t} else {\n\t\t\treturn math.NaN(), fmt.Errorf(\"Can't convert %v to float64\", v.Type())\n\t\t}\n\t}\n}\n\n// Path: pkg/gui/presentation/containers.go\npackage presentation\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\nfunc GetContainerDisplayStrings(guiConfig *config.GuiConfig, container *commands.Container) []string {\n\treturn []string{\n\t\tgetContainerDisplayStatus(guiConfig, container),\n\t\tgetContainerDisplaySubstatus(guiConfig, container),\n\t\tcontainer.Name,\n\t\tgetDisplayCPUPerc(container),\n\t\tutils.ColoredString(displayPorts(container), color.FgYellow),\n\t\tutils.ColoredString(displayContainerImage(container), color.FgMagenta),\n\t}\n}\n\nfunc displayContainerImage(container *commands.Container) string {\n\treturn strings.TrimPrefix(container.Container.Image, \"sha256:\")\n}\n\nfunc displayPorts(c *commands.Container) string {\n\tportStrings := lo.Map(c.Container.Ports, func(port dockerTypes.Port, _ int) string {\n\t\tif port.PublicPort == 0 {\n\t\t\treturn fmt.Sprintf(\"%d/%s\", port.PrivatePort, port.Type)\n\t\t}\n\n\t\t// docker ps will show '0.0.0.0:80->80/tcp' but we'll show\n\t\t// '80->80/tcp' instead to save space (unless the IP is something other than\n\t\t// 0.0.0.0)\n\t\tipString := \"\"\n\t\tif port.IP != \"0.0.0.0\" {\n\t\t\tipString = port.IP + \":\"\n\t\t}\n\t\treturn fmt.Sprintf(\"%s%d->%d/%s\", ipString, port.PublicPort, port.PrivatePort, port.Type)\n\t})\n\n\t// sorting because the order of the ports is not deterministic\n\t// and we don't want to have them constantly swapping\n\tsort.Strings(portStrings)\n\n\treturn strings.Join(portStrings, \", \")\n}\n\n// getContainerDisplayStatus returns the colored status of the container\nfunc getContainerDisplayStatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tshortStatusMap := map[string]string{\n\t\t\"paused\":     \"P\",\n\t\t\"exited\":     \"X\",\n\t\t\"created\":    \"C\",\n\t\t\"removing\":   \"RM\",\n\t\t\"restarting\": \"RS\",\n\t\t\"running\":    \"R\",\n\t\t\"dead\":       \"D\",\n\t}\n\n\ticonStatusMap := map[string]rune{\n\t\t\"paused\":     '\u25eb',\n\t\t\"exited\":     '\u2a2f',\n\t\t\"created\":    '+',\n\t\t\"removing\":   '\u2212',\n\t\t\"restarting\": '\u27f3',\n\t\t\"running\":    '\u25b6',\n\t\t\"dead\":       '!',\n\t}\n\n\tvar containerState string\n\tswitch guiConfig.ContainerStatusHealthStyle {\n\tcase \"short\":\n\t\tcontainerState = shortStatusMap[c.Container.State]\n\tcase \"icon\":\n\t\tcontainerState = string(iconStatusMap[c.Container.State])\n\tcase \"long\":\n\t\tfallthrough\n\tdefault:\n\t\tcontainerState = c.Container.State\n\t}\n\n\treturn utils.ColoredString(containerState, getContainerColor(c))\n}\n\n// GetDisplayStatus returns the exit code if the container has exited, and the health status if the container is running (and has a health check)\nfunc getContainerDisplaySubstatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tif !c.DetailsLoaded() {\n\t\treturn \"\"\n\t}\n\n\tswitch c.Container.State {\n\tcase \"exited\":\n\t\treturn utils.ColoredString(\n\t\t\tfmt.Sprintf(\"(%s)\", strconv.Itoa(c.Details.State.ExitCode)), getContainerColor(c),\n\t\t)\n\tcase \"running\":\n\t\treturn getHealthStatus(guiConfig, c)\n\tdefault:\n\t\treturn \"\"\n\t}\n}\n\nfunc getHealthStatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tif !c.DetailsLoaded() {\n\t\treturn \"\"\n\t}\n\n\thealthStatusColorMap := map[string]color.Attribute{\n\t\t\"healthy\":   color.FgGreen,\n\t\t\"unhealthy\": color.FgRed,\n\t\t\"starting\":  color.FgYellow,\n\t}\n\n\tif c.Details.State.Health == nil {\n\t\treturn \"\"\n\t}\n\n\tshortHealthStatusMap := map[string]string{\n\t\t\"healthy\":   \"H\",\n\t\t\"unhealthy\": \"U\",\n\t\t\"starting\":  \"S\",\n\t}\n\n\ticonHealthStatusMap := map[string]rune{\n\t\t\"healthy\":   '\u2714',\n\t\t\"unhealthy\": '?',\n\t\t\"starting\":  '\u2026',\n\t}\n\n\tvar healthStatus string\n\tswitch guiConfig.ContainerStatusHealthStyle {\n\tcase \"short\":\n\t\thealthStatus = shortHealthStatusMap[c.Details.State.Health.Status]\n\tcase \"icon\":\n\t\thealthStatus = string(iconHealthStatusMap[c.Details.State.Health.Status])\n\tcase \"long\":\n\t\tfallthrough\n\tdefault:\n\t\thealthStatus = c.Details.State.Health.Status\n\t}\n\n\tif healthStatusColor, ok := healthStatusColorMap[c.Details.State.Health.Status]; ok {\n\t\treturn utils.ColoredString(fmt.Sprintf(\"(%s)\", healthStatus), healthStatusColor)\n\t}\n\treturn \"\"\n}\n\n// getDisplayCPUPerc colors the cpu percentage based on how extreme it is\nfunc getDisplayCPUPerc(c *commands.Container) string {\n\tstats, ok := c.GetLastStats()\n\tif !ok {\n\t\treturn \"\"\n\t}\n\n\tpercentage := stats.DerivedStats.CPUPercentage\n\tformattedPercentage := fmt.Sprintf(\"%.2f%%\", stats.DerivedStats.CPUPercentage)\n\n\tvar clr color.Attribute\n\tif percentage > 90 {\n\t\tclr = color.FgRed\n\t} else if percentage > 50 {\n\t\tclr = color.FgYellow\n\t} else {\n\t\tclr = color.FgWhite\n\t}\n\n\treturn utils.ColoredString(formattedPercentage, clr)\n}\n\n// getContainerColor Container color\nfunc getContainerColor(c *commands.Container) color.Attribute {\n\tswitch c.Container.State {\n\tcase \"exited\":\n\t\t// This means the colour may be briefly yellow and then switch to red upon starting\n\t\t// Not sure what a better alternative is.\n\t\tif !c.DetailsLoaded() || c.Details.State.ExitCode == 0 {\n\t\t\treturn color.FgYellow\n\t\t}\n\t\treturn color.FgRed\n\tcase \"created\":\n\t\treturn color.FgCyan\n\tcase \"running\":\n\t\treturn color.FgGreen\n\tcase \"paused\":\n\t\treturn color.FgYellow\n\tcase \"dead\":\n\t\treturn color.FgRed\n\tcase \"restarting\":\n\t\treturn color.FgBlue\n\tcase \"removing\":\n\t\treturn color.FgMagenta\n\tdefault:\n\t\treturn color.FgWhite\n\t}\n}\n\n// Path: pkg/gui/presentation/images.go\npackage presentation\n\nimport (\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\nfunc GetImageDisplayStrings(image *commands.Image) []string {\n\treturn []string{\n\t\timage.Name,\n\t\timage.Tag,\n\t\tutils.FormatDecimalBytes(int(image.Image.Size)),\n\t}\n}\n\n// Path: pkg/gui/presentation/menu_items.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\nfunc GetMenuItemDisplayStrings(menuItem *types.MenuItem) []string {\n\treturn menuItem.LabelColumns\n}\n\n// Path: pkg/gui/presentation/networks.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/commands\"\n\nfunc GetNetworkDisplayStrings(network *commands.Network) []string {\n\treturn []string{network.Network.Driver, network.Name}\n}\n\n// Path: pkg/gui/presentation/projects.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/commands\"\n\nfunc GetProjectDisplayStrings(project *commands.Project) []string {\n\treturn []string{project.Name}\n}\n\n// Path: pkg/gui/presentation/services.go\npackage presentation\n\nimport (\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\nfunc GetServiceDisplayStrings(guiConfig *config.GuiConfig, service *commands.Service) []string {\n\tif service.Container == nil {\n\t\tvar containerState string\n\t\tswitch guiConfig.ContainerStatusHealthStyle {\n\t\tcase \"short\":\n\t\t\tcontainerState = \"n\"\n\t\tcase \"icon\":\n\t\t\tcontainerState = \".\"\n\t\tcase \"long\":\n\t\t\tfallthrough\n\t\tdefault:\n\t\t\tcontainerState = \"none\"\n\t\t}\n\n\t\treturn []string{\n\t\t\tutils.ColoredString(containerState, color.FgBlue),\n\t\t\t\"\",\n\t\t\tservice.Name,\n\t\t\t\"\",\n\t\t\t\"\",\n\t\t\t\"\",\n\t\t}\n\t}\n\n\tcontainer := service.Container\n\treturn []string{\n\t\tgetContainerDisplayStatus(guiConfig, container),\n\t\tgetContainerDisplaySubstatus(guiConfig, container),\n\t\tservice.Name,\n\t\tgetDisplayCPUPerc(container),\n\t\tutils.ColoredString(displayPorts(container), color.FgYellow),\n\t\tutils.ColoredString(displayContainerImage(container), color.FgMagenta),\n\t}\n}\n\n// Path: pkg/gui/presentation/volumes.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/commands\"\n\nfunc GetVolumeDisplayStrings(volume *commands.Volume) []string {\n\treturn []string{volume.Volume.Driver, volume.Name}\n}\n\n// Path: pkg/gui/project_panel.go\npackage gui\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"path\"\n\t\"strings\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/jesseduffield/yaml\"\n)\n\n// Although at the moment we'll only have one project, in future we could have\n// a list of projects in the project panel.\n\nfunc (gui *Gui) getProjectPanel() *panels.SideListPanel[*commands.Project] {\n\treturn &panels.SideListPanel[*commands.Project]{\n\t\tContextState: &panels.ContextState[*commands.Project]{\n\t\t\tGetMainTabs: func() []panels.MainTab[*commands.Project] {\n\t\t\t\tif gui.DockerCommand.InDockerComposeProject {\n\t\t\t\t\treturn []panels.MainTab[*commands.Project]{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey:    \"logs\",\n\t\t\t\t\t\t\tTitle:  gui.Tr.LogsTitle,\n\t\t\t\t\t\t\tRender: gui.renderAllLogs,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey:    \"config\",\n\t\t\t\t\t\t\tTitle:  gui.Tr.DockerComposeConfigTitle,\n\t\t\t\t\t\t\tRender: gui.renderDockerComposeConfig,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey:    \"credits\",\n\t\t\t\t\t\t\tTitle:  gui.Tr.CreditsTitle,\n\t\t\t\t\t\t\tRender: gui.renderCredits,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn []panels.MainTab[*commands.Project]{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"credits\",\n\t\t\t\t\t\tTitle:  gui.Tr.CreditsTitle,\n\t\t\t\t\t\tRender: gui.renderCredits,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t},\n\t\t\tGetItemContextCacheKey: func(project *commands.Project) string {\n\t\t\t\treturn \"projects-\" + project.Name\n\t\t\t},\n\t\t},\n\n\t\tListPanel: panels.ListPanel[*commands.Project]{\n\t\t\tList: panels.NewFilteredList[*commands.Project](),\n\t\t\tView: gui.Views.Project,\n\t\t},\n\t\tNoItemsMessage: \"\",\n\t\tGui:            gui.intoInterface(),\n\n\t\tSort: func(a *commands.Project, b *commands.Project) bool {\n\t\t\treturn false\n\t\t},\n\t\tGetTableCells: presentation.GetProjectDisplayStrings,\n\t\t// It doesn't make sense to filter a list of only one item.\n\t\tDisableFilter: true,\n\t}\n}\n\nfunc (gui *Gui) refreshProject() error {\n\tgui.Panels.Projects.SetItems([]*commands.Project{{Name: gui.getProjectName()}})\n\treturn gui.Panels.Projects.RerenderList()\n}\n\nfunc (gui *Gui) getProjectName() string {\n\tprojectName := path.Base(gui.Config.ProjectDir)\n\tif gui.DockerCommand.InDockerComposeProject {\n\t\tfor _, service := range gui.Panels.Services.List.GetAllItems() {\n\t\t\tcontainer := service.Container\n\t\t\tif container != nil && container.DetailsLoaded() {\n\t\t\t\treturn container.Details.Config.Labels[\"com.docker.compose.project\"]\n\t\t\t}\n\t\t}\n\t}\n\n\treturn projectName\n}\n\nfunc (gui *Gui) renderCredits(_project *commands.Project) tasks.TaskFunc {\n\treturn gui.NewSimpleRenderStringTask(func() string { return gui.creditsStr() })\n}\n\nfunc (gui *Gui) creditsStr() string {\n\tvar configBuf bytes.Buffer\n\t_ = yaml.NewEncoder(&configBuf, yaml.IncludeOmitted).Encode(gui.Config.UserConfig)\n\n\treturn strings.Join(\n\t\t[]string{\n\t\t\tlazydockerTitle(),\n\t\t\t\"Copyright (c) 2019 Jesse Duffield\",\n\t\t\t\"Keybindings: https://github.com/jesseduffield/lazydocker/blob/master/docs/keybindings\",\n\t\t\t\"Config Options: https://github.com/jesseduffield/lazydocker/blob/master/docs/Config.md\",\n\t\t\t\"Raise an Issue: https://github.com/jesseduffield/lazydocker/issues\",\n\t\t\tutils.ColoredString(\"Buy Jesse a coffee: https://github.com/sponsors/jesseduffield\", color.FgMagenta), // caffeine ain't free\n\t\t\t\"Here's your lazydocker config when merged in with the defaults (you can open your config by pressing 'o'):\",\n\t\t\tutils.ColoredYamlString(configBuf.String()),\n\t\t}, \"\\n\\n\")\n}\n\nfunc (gui *Gui) renderAllLogs(_project *commands.Project) tasks.TaskFunc {\n\treturn gui.NewTask(TaskOpts{\n\t\tAutoscroll: true,\n\t\tWrap:       gui.Config.UserConfig.Gui.WrapMainPanel,\n\t\tFunc: func(ctx context.Context) {\n\t\t\tgui.clearMainView()\n\n\t\t\tcmd := gui.OSCommand.RunCustomCommand(\n\t\t\t\tutils.ApplyTemplate(\n\t\t\t\t\tgui.Config.UserConfig.CommandTemplates.AllLogs,\n\t\t\t\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{}),\n\t\t\t\t),\n\t\t\t)\n\n\t\t\tcmd.Stdout = gui.Views.Main\n\t\t\tcmd.Stderr = gui.Views.Main\n\n\t\t\tgui.OSCommand.PrepareForChildren(cmd)\n\t\t\t_ = cmd.Start()\n\n\t\t\tgo func() {\n\t\t\t\t<-ctx.Done()\n\t\t\t\tif err := gui.OSCommand.Kill(cmd); err != nil {\n\t\t\t\t\tgui.Log.Error(err)\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\t_ = cmd.Wait()\n\t\t},\n\t})\n}\n\nfunc (gui *Gui) renderDockerComposeConfig(_project *commands.Project) tasks.TaskFunc {\n\treturn gui.NewSimpleRenderStringTask(func() string {\n\t\treturn utils.ColoredYamlString(gui.DockerCommand.DockerComposeConfig())\n\t})\n}\n\nfunc (gui *Gui) handleOpenConfig(g *gocui.Gui, v *gocui.View) error {\n\treturn gui.openFile(gui.Config.ConfigFilename())\n}\n\nfunc (gui *Gui) handleEditConfig(g *gocui.Gui, v *gocui.View) error {\n\treturn gui.editFile(gui.Config.ConfigFilename())\n}\n\nfunc lazydockerTitle() string {\n\treturn `\n   _                     _            _\n  | |                   | |          | |\n  | | __ _ _____   _  __| | ___   ___| | _____ _ __\n  | |/ _` + \"`\" + ` |_  / | | |/ _` + \"`\" + ` |/ _ \\ / __| |/ / _ \\ '__|\n  | | (_| |/ /| |_| | (_| | (_) | (__|   <  __/ |\n  |_|\\__,_/___|\\__, |\\__,_|\\___/ \\___|_|\\_\\___|_|\n                __/ |\n               |___/\n`\n}\n\n// handleViewAllLogs switches to a subprocess viewing all the logs from docker-compose\nfunc (gui *Gui) handleViewAllLogs(g *gocui.Gui, v *gocui.View) error {\n\tc, err := gui.DockerCommand.ViewAllLogs()\n\tif err != nil {\n\t\treturn gui.createErrorPanel(err.Error())\n\t}\n\n\treturn gui.runSubprocess(c)\n}\n\n// Path: pkg/gui/services_panel.go\npackage gui\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\nfunc (gui *Gui) getServicesPanel() *panels.SideListPanel[*commands.Service] {\n\treturn &panels.SideListPanel[*commands.Service]{\n\t\tContextState: &panels.ContextState[*commands.Service]{\n\t\t\tGetMainTabs: func() []panels.MainTab[*commands.Service] {\n\t\t\t\treturn []panels.MainTab[*commands.Service]{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"logs\",\n\t\t\t\t\t\tTitle:  gui.Tr.LogsTitle,\n\t\t\t\t\t\tRender: gui.renderServiceLogs,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"stats\",\n\t\t\t\t\t\tTitle:  gui.Tr.StatsTitle,\n\t\t\t\t\t\tRender: gui.renderServiceStats,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"container-env\",\n\t\t\t\t\t\tTitle:  gui.Tr.ContainerEnvTitle,\n\t\t\t\t\t\tRender: gui.renderServiceContainerEnv,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"container-config\",\n\t\t\t\t\t\tTitle:  gui.Tr.ContainerConfigTitle,\n\t\t\t\t\t\tRender: gui.renderServiceContainerConfig,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"top\",\n\t\t\t\t\t\tTitle:  gui.Tr.TopTitle,\n\t\t\t\t\t\tRender: gui.renderServiceTop,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t},\n\t\t\tGetItemContextCacheKey: func(service *commands.Service) string {\n\t\t\t\tif service.Container == nil {\n\t\t\t\t\treturn \"services-\" + service.ID\n\t\t\t\t}\n\t\t\t\treturn \"services-\" + service.ID + \"-\" + service.Container.ID + \"-\" + service.Container.Container.State\n\t\t\t},\n\t\t},\n\t\tListPanel: panels.ListPanel[*commands.Service]{\n\t\t\tList: panels.NewFilteredList[*commands.Service](),\n\t\t\tView: gui.Views.Services,\n\t\t},\n\t\tNoItemsMessage: gui.Tr.NoServices,\n\t\tGui:            gui.intoInterface(),\n\t\t// sort services first by whether they have a linked container, and second by alphabetical order\n\t\tSort: func(a *commands.Service, b *commands.Service) bool {\n\t\t\tif a.Container != nil && b.Container == nil {\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\tif a.Container == nil && b.Container != nil {\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\treturn a.Name < b.Name\n\t\t},\n\t\tGetTableCells: func(service *commands.Service) []string {\n\t\t\treturn presentation.GetServiceDisplayStrings(&gui.Config.UserConfig.Gui, service)\n\t\t},\n\t\tHide: func() bool {\n\t\t\treturn !gui.DockerCommand.InDockerComposeProject\n\t\t},\n\t}\n}\n\nfunc (gui *Gui) renderServiceContainerConfig(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainer })\n\t}\n\n\treturn gui.renderContainerConfig(service.Container)\n}\n\nfunc (gui *Gui) renderServiceContainerEnv(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainer })\n\t}\n\n\treturn gui.renderContainerEnv(service.Container)\n}\n\nfunc (gui *Gui) renderServiceStats(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainer })\n\t}\n\n\treturn gui.renderContainerStats(service.Container)\n}\n\nfunc (gui *Gui) renderServiceTop(service *commands.Service) tasks.TaskFunc {\n\treturn gui.NewTickerTask(TickerTaskOpts{\n\t\tFunc: func(ctx context.Context, notifyStopped chan struct{}) {\n\t\t\tcontents, err := service.RenderTop(ctx)\n\t\t\tif err != nil {\n\t\t\t\tgui.RenderStringMain(err.Error())\n\t\t\t}\n\n\t\t\tgui.reRenderStringMain(contents)\n\t\t},\n\t\tDuration:   time.Second,\n\t\tBefore:     func(ctx context.Context) { gui.clearMainView() },\n\t\tWrap:       gui.Config.UserConfig.Gui.WrapMainPanel,\n\t\tAutoscroll: false,\n\t})\n}\n\nfunc (gui *Gui) renderServiceLogs(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainerForService })\n\t}\n\n\treturn gui.renderContainerLogsToMain(service.Container)\n}\n\ntype commandOption struct {\n\tdescription string\n\tcommand     string\n\tonPress     func() error\n}\n\nfunc (r *commandOption) getDisplayStrings() []string {\n\treturn []string{r.description, color.New(color.FgCyan).Sprint(r.command)}\n}\n\nfunc (gui *Gui) handleServiceRemoveMenu(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tcomposeCommand := gui.Config.UserConfig.CommandTemplates.DockerCompose\n\n\toptions := []*commandOption{\n\t\t{\n\t\t\tdescription: gui.Tr.Remove,\n\t\t\tcommand:     fmt.Sprintf(\"%s rm --stop --force %s\", composeCommand, service.Name),\n\t\t},\n\t\t{\n\t\t\tdescription: gui.Tr.RemoveWithVolumes,\n\t\t\tcommand:     fmt.Sprintf(\"%s rm --stop --force -v %s\", composeCommand, service.Name),\n\t\t},\n\t}\n\n\tmenuItems := lo.Map(options, func(option *commandOption, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: option.getDisplayStrings(),\n\t\t\tOnPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.RemovingStatus, func() error {\n\t\t\t\t\tif err := gui.OSCommand.RunCommand(option.command); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle: \"\",\n\t\tItems: menuItems,\n\t})\n}\n\nfunc (gui *Gui) handleServicePause(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\tif service.Container == nil {\n\t\treturn nil\n\t}\n\n\treturn gui.PauseContainer(service.Container)\n}\n\nfunc (gui *Gui) handleServiceStop(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.createConfirmationPanel(gui.Tr.Confirm, gui.Tr.StopService, func(g *gocui.Gui, v *gocui.View) error {\n\t\treturn gui.WithWaitingStatus(gui.Tr.StoppingStatus, func() error {\n\t\t\tif err := service.Stop(); err != nil {\n\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t}\n\n\t\t\treturn nil\n\t\t})\n\t}, nil)\n}\n\nfunc (gui *Gui) handleServiceUp(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.WithWaitingStatus(gui.Tr.UppingServiceStatus, func() error {\n\t\tif err := service.Up(); err != nil {\n\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t}\n\n\t\treturn nil\n\t})\n}\n\nfunc (gui *Gui) handleServiceRestart(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.WithWaitingStatus(gui.Tr.RestartingStatus, func() error {\n\t\tif err := service.Restart(); err != nil {\n\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t}\n\n\t\treturn nil\n\t})\n}\n\nfunc (gui *Gui) handleServiceStart(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.WithWaitingStatus(gui.Tr.StartingStatus, func() error {\n\t\tif err := service.Start(); err != nil {\n\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t}\n\n\t\treturn nil\n\t})\n}\n\nfunc (gui *Gui) handleServiceAttach(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tif service.Container == nil {\n\t\treturn gui.createErrorPanel(gui.Tr.NoContainers)\n\t}\n\n\tc, err := service.Attach()\n\tif err != nil {\n\t\treturn gui.createErrorPanel(err.Error())\n\t}\n\n\treturn gui.runSubprocess(c)\n}\n\nfunc (gui *Gui) handleServiceRenderLogsToMain(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tc, err := service.ViewLogs()\n\tif err != nil {\n\t\treturn gui.createErrorPanel(err.Error())\n\t}\n\n\treturn gui.runSubprocess(c)\n}\n\nfunc (gui *Gui) handleProjectUp(g *gocui.Gui, v *gocui.View) error {\n\treturn gui.createConfirmationPanel(gui.Tr.Confirm, gui.Tr.ConfirmUpProject, func(g *gocui.Gui, v *gocui.View) error {\n\t\tcmdStr := utils.ApplyTemplate(\n\t\t\tgui.Config.UserConfig.CommandTemplates.Up,\n\t\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{}),\n\t\t)\n\n\t\treturn gui.WithWaitingStatus(gui.Tr.UppingProjectStatus, func() error {\n\t\t\tif err := gui.OSCommand.RunCommand(cmdStr); err != nil {\n\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}, nil)\n}\n\nfunc (gui *Gui) handleProjectDown(g *gocui.Gui, v *gocui.View) error {\n\tdownCommand := utils.ApplyTemplate(\n\t\tgui.Config.UserConfig.CommandTemplates.Down,\n\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{}),\n\t)\n\n\tdownWithVolumesCommand := utils.ApplyTemplate(\n\t\tgui.Config.UserConfig.CommandTemplates.DownWithVolumes,\n\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{}),\n\t)\n\n\toptions := []*commandOption{\n\t\t{\n\t\t\tdescription: gui.Tr.Down,\n\t\t\tcommand:     downCommand,\n\t\t\tonPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.DowningStatus, func() error {\n\t\t\t\t\tif err := gui.OSCommand.RunCommand(downCommand); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdescription: gui.Tr.DownWithVolumes,\n\t\t\tcommand:     downWithVolumesCommand,\n\t\t\tonPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.DowningStatus, func() error {\n\t\t\t\t\tif err := gui.OSCommand.RunCommand(downWithVolumesCommand); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t},\n\t}\n\n\tmenuItems := lo.Map(options, func(option *commandOption, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: option.getDisplayStrings(),\n\t\t\tOnPress:      option.onPress,\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle: \"\",\n\t\tItems: menuItems,\n\t})\n}\n\nfunc (gui *Gui) handleServiceRestartMenu(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\trebuildCommand := utils.ApplyTemplate(\n\t\tgui.Config.UserConfig.CommandTemplates.RebuildService,\n\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{Service: service}),\n\t)\n\n\trecreateCommand := utils.ApplyTemplate(\n\t\tgui.Config.UserConfig.CommandTemplates.RecreateService,\n\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{Service: service}),\n\t)\n\n\toptions := []*commandOption{\n\t\t{\n\t\t\tdescription: gui.Tr.Restart,\n\t\t\tcommand: utils.ApplyTemplate(\n\t\t\t\tgui.Config.UserConfig.CommandTemplates.RestartService,\n\t\t\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{Service: service}),\n\t\t\t),\n\t\t\tonPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.RestartingStatus, func() error {\n\t\t\t\t\tif err := service.Restart(); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdescription: gui.Tr.Recreate,\n\t\t\tcommand: utils.ApplyTemplate(\n\t\t\t\tgui.Config.UserConfig.CommandTemplates.RecreateService,\n\t\t\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{Service: service}),\n\t\t\t),\n\t\t\tonPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.RestartingStatus, func() error {\n\t\t\t\t\tif err := gui.OSCommand.RunCommand(recreateCommand); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdescription: gui.Tr.Rebuild,\n\t\t\tcommand: utils.ApplyTemplate(\n\t\t\t\tgui.Config.UserConfig.CommandTemplates.RebuildService,\n\t\t\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{Service: service}),\n\t\t\t),\n\t\t\tonPress: func() error {\n\t\t\t\treturn gui.runSubprocess(gui.OSCommand.RunCustomCommand(rebuildCommand))\n\t\t\t},\n\t\t},\n\t}\n\n\tmenuItems := lo.Map(options, func(option *commandOption, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: option.getDisplayStrings(),\n\t\t\tOnPress:      option.onPress,\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle: \"\",\n\t\tItems: menuItems,\n\t})\n}\n\nfunc (gui *Gui) handleServicesCustomCommand(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tcommandObject := gui.DockerCommand.NewCommandObject(commands.CommandObject{\n\t\tService:   service,\n\t\tContainer: service.Container,\n\t})\n\n\tvar customCommands []config.CustomCommand\n\n\tcustomServiceCommands := gui.Config.UserConfig.CustomCommands.Services\n\t// we only include service commands if they have no serviceNames defined or if our service happens to be one of the serviceNames defined\nL:\n\tfor _, cmd := range customServiceCommands {\n\t\tif len(cmd.ServiceNames) == 0 {\n\t\t\tcustomCommands = append(customCommands, cmd)\n\t\t\tcontinue L\n\t\t}\n\t\tfor _, serviceName := range cmd.ServiceNames {\n\t\t\tif serviceName == service.Name {\n\t\t\t\t// appending these to the top given they're more likely to be selected\n\t\t\t\tcustomCommands = append([]config.CustomCommand{cmd}, customCommands...)\n\t\t\t\tcontinue L\n\t\t\t}\n\t\t}\n\t}\n\n\tif service.Container != nil {\n\t\tcustomCommands = append(customCommands, gui.Config.UserConfig.CustomCommands.Containers...)\n\t}\n\n\treturn gui.createCustomCommandMenu(customCommands, commandObject)\n}\n\nfunc (gui *Gui) handleServicesBulkCommand(g *gocui.Gui, v *gocui.View) error {\n\tbulkCommands := gui.Config.UserConfig.BulkCommands.Services\n\tcommandObject := gui.DockerCommand.NewCommandObject(commands.CommandObject{})\n\n\treturn gui.createBulkCommandMenu(bulkCommands, commandObject)\n}\n\nfunc (gui *Gui) handleServicesExecShell(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tcontainer := service.Container\n\tif container == nil {\n\t\treturn gui.createErrorPanel(gui.Tr.NoContainers)\n\t}\n\n\treturn gui.containerExecShell(container)\n}\n\nfunc (gui *Gui) handleServicesOpenInBrowserCommand(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tcontainer := service.Container\n\tif container == nil {\n\t\treturn gui.createErrorPanel(gui.Tr.NoContainers)\n\t}\n\n\treturn gui.openContainerInBrowser(container)\n}\n\n// Path: pkg/gui/sort_container_test.go\npackage gui\n\nimport (\n\t\"sort\"\n\t\"testing\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc sampleContainers() []*commands.Container {\n\treturn []*commands.Container{\n\t\t{\n\t\t\tID:   \"1\",\n\t\t\tName: \"1\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"exited\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"2\",\n\t\t\tName: \"2\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"running\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"3\",\n\t\t\tName: \"3\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"running\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"4\",\n\t\t\tName: \"4\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"created\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc expectedPerStatusContainers() []*commands.Container {\n\treturn []*commands.Container{\n\t\t{\n\t\t\tID:   \"2\",\n\t\t\tName: \"2\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"running\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"3\",\n\t\t\tName: \"3\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"running\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"1\",\n\t\t\tName: \"1\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"exited\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"4\",\n\t\t\tName: \"4\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"created\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc expectedLegacySortedContainers() []*commands.Container {\n\treturn []*commands.Container{\n\t\t{\n\t\t\tID:   \"1\",\n\t\t\tName: \"1\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"exited\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"2\",\n\t\t\tName: \"2\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"running\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"3\",\n\t\t\tName: \"3\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"running\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tID:   \"4\",\n\t\t\tName: \"4\",\n\t\t\tContainer: dockerTypes.Container{\n\t\t\t\tState: \"created\",\n\t\t\t},\n\t\t},\n\t}\n}\n\nfunc assertEqualContainers(t *testing.T, left *commands.Container, right *commands.Container) {\n\tt.Helper()\n\tassert.Equal(t, left.Container.State, right.Container.State)\n\tassert.Equal(t, left.Container.ID, right.Container.ID)\n\tassert.Equal(t, left.Name, right.Name)\n}\n\nfunc TestSortContainers(t *testing.T) {\n\tactual := sampleContainers()\n\n\texpected := expectedPerStatusContainers()\n\n\tsort.Slice(actual, func(i, j int) bool {\n\t\treturn sortContainers(actual[i], actual[j], false)\n\t})\n\n\tassert.Equal(t, len(actual), len(expected))\n\n\tfor i := 0; i < len(actual); i++ {\n\t\tassertEqualContainers(t, expected[i], actual[i])\n\t}\n}\n\nfunc TestLegacySortedContainers(t *testing.T) {\n\tactual := sampleContainers()\n\n\texpected := expectedLegacySortedContainers()\n\n\tsort.Slice(actual, func(i, j int) bool {\n\t\treturn sortContainers(actual[i], actual[j], true)\n\t})\n\n\tassert.Equal(t, len(actual), len(expected))\n\n\tfor i := 0; i < len(actual); i++ {\n\t\tassertEqualContainers(t, expected[i], actual[i])\n\t}\n}\n\n// Path: pkg/gui/subprocess.go\npackage gui\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"os/signal\"\n\t\"strings\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\nfunc (gui *Gui) runSubprocess(cmd *exec.Cmd) error {\n\treturn gui.runSubprocessWithMessage(cmd, \"\")\n}\n\nfunc (gui *Gui) runSubprocessWithMessage(cmd *exec.Cmd, msg string) error {\n\tgui.Mutexes.SubprocessMutex.Lock()\n\tdefer gui.Mutexes.SubprocessMutex.Unlock()\n\n\tif err := gui.g.Suspend(); err != nil {\n\t\treturn gui.createErrorPanel(err.Error())\n\t}\n\n\tgui.PauseBackgroundThreads = true\n\n\tgui.runCommand(cmd, msg)\n\n\tif err := gui.g.Resume(); err != nil {\n\t\treturn gui.createErrorPanel(err.Error())\n\t}\n\n\tgui.PauseBackgroundThreads = false\n\n\treturn nil\n}\n\nfunc (gui *Gui) runCommand(cmd *exec.Cmd, msg string) {\n\tcmd.Stdout = os.Stdout\n...\n", "needle_token_start": 2022, "needle_token_end": 2450, "code_context_ntokens": 15063, "output": ["```go\nfunc RenderStats(userConfig *config.UserConfig, container *commands.Container, viewWidth int) (string, error) {\n\tstats, ok := container.GetLastStats()\n\tif !ok {\n\t\treturn \"\", nil\n\t}\n\n\tgraphSpecs := userConfig.Stats.Graphs\n\tgraphs := make([]string, len(graphSpecs))\n\tfor i, spec := range graphSpecs {\n\t\tgraph, err := plotGraph(container, spec, viewWidth-10)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tgraphs[i] = utils.ColoredString(graph, utils.GetColorAttribute(spec.Color))\n\t}\n\n\tpidsCount := fmt.Sprintf(\"PIDs: %d\", stats.ClientStats.PidsStats.Current)\n\tdataReceived := fmt.Sprintf(\"Traffic received: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.RxBytes))\n\tdataSent := fmt.Sprintf(\"Traffic sent: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.TxBytes))\n\n\toriginalStats, err := utils.MarshalIntoYaml(stats)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tcontents := fmt.Sprintf(\"\\n\\n%s\\n\\n%s\\n\\n%s\\n%s\\n\\n%s\",\n\t\tutils.ColoredString(strings.Join(graphs, \"\\n\\n\"), color.FgGreen),\n\t\tpidsCount,\n\t\tdataReceived,\n\t\tdataSent,\n\t\tutils.ColoredYamlString(string(originalStats)),\n\t)\n\n\treturn contents, nil\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "NewApp", "language": "go", "path": "pkg/app/app.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: Initializes and configures a new application instance that manages Docker containers and commands, integrating various services and user configurations.\n2. **Input**: Configuration settings for the application, including user preferences and system settings.\n3. **Output**: Returns a fully initialized application instance or an error if initialization fails.\n4. **Procedure**: \n   - Initializes logging based on the provided configuration.\n   - Sets up translations based on the user's selected language.\n   - Creates command handlers for operating system and Docker-related tasks.\n   - Initializes the graphical user interface to interact with Docker containers and services.\n   - Manages error handling through a dedicated channel.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/gui/gui.go\npackage gui\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\n\t\"github.com/go-errors/errors\"\n\n\tthrottle \"github.com/boz/go-throttle\"\n\t\"github.com/jesseduffield/gocui\"\n\tlcUtils \"github.com/jesseduffield/lazycore/pkg/utils\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// OverlappingEdges determines if panel edges overlap\nvar OverlappingEdges = false\n\n// Gui wraps the gocui Gui object which handles rendering and events\ntype Gui struct {\n\tg             *gocui.Gui\n\tLog           *logrus.Entry\n\tDockerCommand *commands.DockerCommand\n\tOSCommand     *commands.OSCommand\n\tState         guiState\n\tConfig        *config.AppConfig\n\tTr            *i18n.TranslationSet\n\tstatusManager *statusManager\n\ttaskManager   *tasks.TaskManager\n\tErrorChan     chan error\n\tViews         Views\n\n\t// if we've suspended the gui (e.g. because we've switched to a subprocess)\n\t// we typically want to pause some things that are running like background\n\t// file refreshes\n\tPauseBackgroundThreads bool\n\n\tMutexes\n\n\tPanels Panels\n}\n\ntype Panels struct {\n\tProjects   *panels.SideListPanel[*commands.Project]\n\tServices   *panels.SideListPanel[*commands.Service]\n\tContainers *panels.SideListPanel[*commands.Container]\n\tImages     *panels.SideListPanel[*commands.Image]\n\tVolumes    *panels.SideListPanel[*commands.Volume]\n\tNetworks   *panels.SideListPanel[*commands.Network]\n\tMenu       *panels.SideListPanel[*types.MenuItem]\n}\n\ntype Mutexes struct {\n\tSubprocessMutex deadlock.Mutex\n\tViewStackMutex  deadlock.Mutex\n}\n\ntype mainPanelState struct {\n\t// ObjectKey tells us what context we are in. For example, if we are looking at the logs of a particular service in the services panel this key might be 'services-<service id>-logs'. The key is made so that if something changes which might require us to re-run the logs command or run a different command, the key will be different, and we'll then know to do whatever is required. Object key probably isn't the best name for this but Context is already used to refer to tabs. Maybe I should just call them tabs.\n\tObjectKey string\n}\n\ntype panelStates struct {\n\tMain *mainPanelState\n}\n\ntype guiState struct {\n\t// the names of views in the current focus stack (last item is the current view)\n\tViewStack        []string\n\tPlatform         commands.Platform\n\tPanels           *panelStates\n\tSubProcessOutput string\n\tStats            map[string]commands.ContainerStats\n\n\t// if true, we show containers with an 'exited' status in the containers panel\n\tShowExitedContainers bool\n\n\tScreenMode WindowMaximisation\n\n\t// Maintains the state of manual filtering i.e. typing in a substring\n\t// to filter on in the current panel.\n\tFilter filterState\n}\n\ntype filterState struct {\n\t// If true then we're either currently inside the filter view\n\t// or we've committed the filter and we're back in the list view\n\tactive bool\n\t// The panel that we're filtering.\n\tpanel panels.ISideListPanel\n\t// The string that we're filtering on\n\tneedle string\n}\n\n// screen sizing determines how much space your selected window takes up (window\n// as in panel, not your terminal's window). Sometimes you want a bit more space\n// to see the contents of a panel, and this keeps track of how much maximisation\n// you've set\ntype WindowMaximisation int\n\nconst (\n\tSCREEN_NORMAL WindowMaximisation = iota\n\tSCREEN_HALF\n\tSCREEN_FULL\n)\n\nfunc getScreenMode(config *config.AppConfig) WindowMaximisation {\n\tswitch config.UserConfig.Gui.ScreenMode {\n\tcase \"normal\":\n\t\treturn SCREEN_NORMAL\n\tcase \"half\":\n\t\treturn SCREEN_HALF\n\tcase \"fullscreen\":\n\t\treturn SCREEN_FULL\n\tdefault:\n\t\treturn SCREEN_NORMAL\n\t}\n}\n\n// NewGui builds a new gui handler\nfunc NewGui(log *logrus.Entry, dockerCommand *commands.DockerCommand, oSCommand *commands.OSCommand, tr *i18n.TranslationSet, config *config.AppConfig, errorChan chan error) (*Gui, error) {\n\tinitialState := guiState{\n\t\tPlatform: *oSCommand.Platform,\n\t\tPanels: &panelStates{\n\t\t\tMain: &mainPanelState{\n\t\t\t\tObjectKey: \"\",\n\t\t\t},\n\t\t},\n\t\tViewStack: []string{},\n\n\t\tShowExitedContainers: true,\n\t\tScreenMode:           getScreenMode(config),\n\t}\n\n\tgui := &Gui{\n\t\tLog:           log,\n\t\tDockerCommand: dockerCommand,\n\t\tOSCommand:     oSCommand,\n\t\tState:         initialState,\n\t\tConfig:        config,\n\t\tTr:            tr,\n\t\tstatusManager: &statusManager{},\n\t\ttaskManager:   tasks.NewTaskManager(log, tr),\n\t\tErrorChan:     errorChan,\n\t}\n\n\tdeadlock.Opts.Disable = !gui.Config.Debug\n\tdeadlock.Opts.DeadlockTimeout = 10 * time.Second\n\n\treturn gui, nil\n}\n\nfunc (gui *Gui) renderGlobalOptions() error {\n\treturn gui.renderOptionsMap(map[string]string{\n\t\t\"PgUp/PgDn\": gui.Tr.Scroll,\n\t\t\"\u2190 \u2192 \u2191 \u2193\":   gui.Tr.Navigate,\n\t\t\"q\":         gui.Tr.Quit,\n\t\t\"b\":         gui.Tr.ViewBulkCommands,\n\t\t\"x\":         gui.Tr.Menu,\n\t})\n}\n\nfunc (gui *Gui) goEvery(interval time.Duration, function func() error) {\n\t_ = function() // time.Tick doesn't run immediately so we'll do that here // TODO: maybe change\n\tgo func() {\n\t\tticker := time.NewTicker(interval)\n\t\tdefer ticker.Stop()\n\t\tfor range ticker.C {\n\t\t\tif gui.PauseBackgroundThreads {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t_ = function()\n\t\t}\n\t}()\n}\n\n// Run setup the gui with keybindings and start the mainloop\nfunc (gui *Gui) Run() error {\n\t// closing our task manager which in turn closes the current task if there is any, so we aren't leaving processes lying around after closing lazydocker\n\tdefer gui.taskManager.Close()\n\n\tg, err := gocui.NewGui(gocui.OutputTrue, OverlappingEdges, gocui.NORMAL, false, map[rune]string{})\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer g.Close()\n\n\t// forgive the double-negative, this is because of my yaml `omitempty` woes\n\tif !gui.Config.UserConfig.Gui.IgnoreMouseEvents {\n\t\tg.Mouse = true\n\t}\n\n\tgui.g = g // TODO: always use gui.g rather than passing g around everywhere\n\n\t// if the deadlock package wants to report a deadlock, we first need to\n\t// close the gui so that we can actually read what it prints.\n\tdeadlock.Opts.LogBuf = lcUtils.NewOnceWriter(os.Stderr, func() {\n\t\tgui.g.Close()\n\t})\n\n\tif err := gui.SetColorScheme(); err != nil {\n\t\treturn err\n\t}\n\n\tthrottledRefresh := throttle.ThrottleFunc(time.Millisecond*50, true, gui.refresh)\n\tdefer throttledRefresh.Stop()\n\n\tgo func() {\n\t\tfor err := range gui.ErrorChan {\n\t\t\tif err == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif strings.Contains(err.Error(), \"No such container\") {\n\t\t\t\t// this happens all the time when e.g. restarting containers so we won't worry about it\n\t\t\t\tgui.Log.Warn(err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_ = gui.createErrorPanel(err.Error())\n\t\t}\n\t}()\n\n\tg.SetManager(gocui.ManagerFunc(gui.layout), gocui.ManagerFunc(gui.getFocusLayout()))\n\n\tif err := gui.createAllViews(); err != nil {\n\t\treturn err\n\t}\n\tif err := gui.setInitialViewContent(); err != nil {\n\t\treturn err\n\t}\n\n\t// TODO: see if we can avoid the circular dependency\n\tgui.setPanels()\n\n\tif err = gui.keybindings(g); err != nil {\n\t\treturn err\n\t}\n\n\tif gui.g.CurrentView() == nil {\n\t\tviewName := gui.initiallyFocusedViewName()\n\t\tview, err := gui.g.View(viewName)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif err := gui.switchFocus(view); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tctx, finish := context.WithCancel(context.Background())\n\tdefer finish()\n\n\tgo gui.listenForEvents(ctx, throttledRefresh.Trigger)\n\tgo gui.monitorContainerStats(ctx)\n\n\tgo func() {\n\t\tthrottledRefresh.Trigger()\n\n\t\tgui.goEvery(time.Millisecond*30, gui.reRenderMain)\n\t\tgui.goEvery(time.Millisecond*1000, gui.updateContainerDetails)\n\t\tgui.goEvery(time.Millisecond*1000, gui.checkForContextChange)\n\t\t// we need to regularly re-render these because their stats will be changed in the background\n\t\tgui.goEvery(time.Millisecond*1000, gui.renderContainersAndServices)\n\t}()\n\n\terr = g.MainLoop()\n\tif err == gocui.ErrQuit {\n\t\treturn nil\n\t}\n\treturn err\n}\n\nfunc (gui *Gui) setPanels() {\n\tgui.Panels = Panels{\n\t\tProjects:   gui.getProjectPanel(),\n\t\tServices:   gui.getServicesPanel(),\n\t\tContainers: gui.getContainersPanel(),\n\t\tImages:     gui.getImagesPanel(),\n\t\tVolumes:    gui.getVolumesPanel(),\n\t\tNetworks:   gui.getNetworksPanel(),\n\t\tMenu:       gui.getMenuPanel(),\n\t}\n}\n\nfunc (gui *Gui) updateContainerDetails() error {\n\treturn gui.DockerCommand.UpdateContainerDetails(gui.Panels.Containers.List.GetAllItems())\n}\n\nfunc (gui *Gui) refresh() {\n\tgo func() {\n\t\tif err := gui.refreshProject(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.refreshContainersAndServices(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.reloadVolumes(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.reloadNetworks(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.reloadImages(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n}\n\nfunc (gui *Gui) listenForEvents(ctx context.Context, refresh func()) {\n\terrorCount := 0\n\n\tonError := func(err error) {\n\t\tif err != nil {\n\t\t\tgui.ErrorChan <- errors.Errorf(\"Docker event stream returned error: %s\\nRetry count: %d\", err.Error(), errorCount)\n\t\t}\n\t\terrorCount++\n...\n// Path: pkg/app/app.go\npackage app\n\nimport (\n\t\"io\"\n\t\"strings\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/log\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// App struct\ntype App struct {\n\tclosers []io.Closer\n\n\tConfig        *config.AppConfig\n\tLog           *logrus.Entry\n\tOSCommand     *commands.OSCommand\n\tDockerCommand *commands.DockerCommand\n\tGui           *gui.Gui\n\tTr            *i18n.TranslationSet\n\tErrorChan     chan error\n}\n\n// NewApp bootstrap a new application\n\nfunc NewApp(config *config.AppConfig) (*App, error) {\n\tapp := &App{\n\t\tclosers:   []io.Closer{},\n\t\tConfig:    config,\n\t\tErrorChan: make(chan error),\n\t}\n\tvar err error\n\tapp.Log = log.NewLogger(config, \"23432119147a4367abf7c0de2aa99a2d\")\n\tapp.Tr, err = i18n.NewTranslationSetFromConfig(app.Log, config.UserConfig.Gui.Language)\n\tif err != nil {\n\t\treturn app, err\n\t}\n\tapp.OSCommand = commands.NewOSCommand(app.Log, config)\n\n\t// here is the place to make use of the docker-compose.yml file in the current directory\n\n\tapp.DockerCommand, err = commands.NewDockerCommand(app.Log, app.OSCommand, app.Tr, app.Config, app.ErrorChan)\n\tif err != nil {\n\t\treturn app, err\n\t}\n\tapp.closers = append(app.closers, app.DockerCommand)\n\tapp.Gui, err = gui.NewGui(app.Log, app.DockerCommand, app.OSCommand, app.Tr, config, app.ErrorChan)\n\tif err != nil {\n\t\treturn app, err\n\t}\n\treturn app, nil\n}\n\nfunc (app *App) Run() error {\n\treturn app.Gui.Run()\n}\n\nfunc (app *App) Close() error {\n\treturn utils.CloseMany(app.closers)\n}\n\ntype errorMapping struct {\n\toriginalError string\n\tnewError      string\n}\n\n// KnownError takes an error and tells us whether it's an error that we know about where we can print a nicely formatted version of it rather than panicking with a stack trace\nfunc (app *App) KnownError(err error) (string, bool) {\n\terrorMessage := err.Error()\n\n\tmappings := []errorMapping{\n\t\t{\n\t\t\toriginalError: \"Got permission denied while trying to connect to the Docker daemon socket\",\n\t\t\tnewError:      app.Tr.CannotAccessDockerSocketError,\n\t\t},\n\t}\n\n\tfor _, mapping := range mappings {\n\t\tif strings.Contains(errorMessage, mapping.originalError) {\n\t\t\treturn mapping.newError, true\n\t\t}\n\t}\n\n\treturn \"\", false\n}\n\n// Path: pkg/cheatsheet/generate.go\n// This \"script\" generates a file called Keybindings_{{.LANG}}.md\n// in current working directory.\n//\n// The content of this generated file is a keybindings cheatsheet.\n//\n// To generate cheatsheet in english run:\n//   LANG=en go run scripts/cheatsheet/main.go generate\n\npackage cheatsheet\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/app\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n)\n\nconst (\n\tgenerateCheatsheetCmd = \"go run scripts/cheatsheet/main.go generate\"\n)\n\ntype bindingSection struct {\n\ttitle    string\n\tbindings []*gui.Binding\n}\n\nfunc Generate() {\n\tgenerateAtDir(GetKeybindingsDir())\n}\n\nfunc generateAtDir(dir string) {\n\tmConfig, err := config.NewAppConfig(\"lazydocker\", \"\", \"\", \"\", \"\", true, nil, \"\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tfor lang := range i18n.GetTranslationSets() {\n\t\tos.Setenv(\"LC_ALL\", lang)\n\t\tmApp, _ := app.NewApp(mConfig)\n\t\tmApp.Gui.SetupFakeGui()\n\n\t\tfile, err := os.Create(dir + \"/Keybindings_\" + lang + \".md\")\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\n\t\tbindingSections := getBindingSections(mApp)\n\t\tcontent := formatSections(mApp, bindingSections)\n\t\tcontent = fmt.Sprintf(\n\t\t\t\"_This file is auto-generated. To update, make the changes in the \"+\n\t\t\t\t\"pkg/i18n directory and then run `%s` from the project root._\\n\\n%s\",\n\t\t\tgenerateCheatsheetCmd,\n\t\t\tcontent,\n\t\t)\n\t\twriteString(file, content)\n\t}\n}\n\nfunc writeString(file *os.File, str string) {\n\t_, err := file.WriteString(str)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n\nfunc formatTitle(title string) string {\n\treturn fmt.Sprintf(\"\\n## %s\\n\\n\", title)\n}\n\nfunc formatBinding(binding *gui.Binding) string {\n\treturn fmt.Sprintf(\"  <kbd>%s</kbd>: %s\\n\", binding.GetKey(), binding.Description)\n}\n\nfunc getBindingSections(mApp *app.App) []*bindingSection {\n\tbindingSections := []*bindingSection{}\n\n\tfor _, binding := range mApp.Gui.GetInitialKeybindings() {\n\t\tif binding.Description == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tviewName := binding.ViewName\n\t\tif viewName == \"\" {\n\t\t\tviewName = \"global\"\n\t\t}\n\n\t\ttitleMap := map[string]string{\n\t\t\t\"global\":     mApp.Tr.GlobalTitle,\n\t\t\t\"main\":       mApp.Tr.MainTitle,\n\t\t\t\"project\":    mApp.Tr.ProjectTitle,\n\t\t\t\"services\":   mApp.Tr.ServicesTitle,\n\t\t\t\"containers\": mApp.Tr.ContainersTitle,\n\t\t\t\"images\":     mApp.Tr.ImagesTitle,\n\t\t\t\"volumes\":    mApp.Tr.VolumesTitle,\n\t\t\t\"networks\":   mApp.Tr.NetworksTitle,\n\t\t}\n\n\t\tbindingSections = addBinding(titleMap[viewName], bindingSections, binding)\n\t}\n\n\treturn bindingSections\n}\n\nfunc addBinding(title string, bindingSections []*bindingSection, binding *gui.Binding) []*bindingSection {\n\tif binding.Description == \"\" {\n\t\treturn bindingSections\n\t}\n\n\tfor _, section := range bindingSections {\n\t\tif title == section.title {\n\t\t\tsection.bindings = append(section.bindings, binding)\n\t\t\treturn bindingSections\n\t\t}\n\t}\n\n\tsection := &bindingSection{\n\t\ttitle:    title,\n\t\tbindings: []*gui.Binding{binding},\n\t}\n\n\treturn append(bindingSections, section)\n}\n\nfunc formatSections(mApp *app.App, bindingSections []*bindingSection) string {\n\tcontent := fmt.Sprintf(\"# Lazydocker %s\\n\", mApp.Tr.Menu)\n\n\tfor _, section := range bindingSections {\n\t\tcontent += formatTitle(section.title)\n\t\tcontent += \"<pre>\\n\"\n\t\tfor _, binding := range section.bindings {\n\t\t\tcontent += formatBinding(binding)\n\t\t}\n\t\tcontent += \"</pre>\\n\"\n\t}\n\n\treturn content\n}\n\n// Path: pkg/cheatsheet/validate.go\npackage cheatsheet\n\nimport (\n\t\"fmt\"\n\t\"io/fs\"\n\t\"log\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\n\t\"github.com/jesseduffield/lazycore/pkg/utils\"\n\t\"github.com/pmezard/go-difflib/difflib\"\n)\n\nfunc Check() {\n\tdir := GetKeybindingsDir()\n\ttmpDir := filepath.Join(os.TempDir(), \"lazydocker_cheatsheet\")\n\n\terr := os.RemoveAll(tmpDir)\n\tif err != nil {\n\t\tlog.Fatalf(\"Error occurred while checking if cheatsheets are up to date: %v\", err)\n\t}\n\tdefer os.RemoveAll(tmpDir)\n\n\tif err = os.Mkdir(tmpDir, 0o700); err != nil {\n\t\tlog.Fatalf(\"Error occurred while checking if cheatsheets are up to date: %v\", err)\n\t}\n\n\tgenerateAtDir(tmpDir)\n\n\tactualContent := obtainContent(dir)\n\texpectedContent := obtainContent(tmpDir)\n\n\tif expectedContent == \"\" {\n\t\tlog.Fatal(\"empty expected content\")\n\t}\n\n\tif actualContent != expectedContent {\n\t\tif err := difflib.WriteUnifiedDiff(os.Stdout, difflib.UnifiedDiff{\n\t\t\tA:        difflib.SplitLines(expectedContent),\n\t\t\tB:        difflib.SplitLines(actualContent),\n\t\t\tFromFile: \"Expected\",\n\t\t\tFromDate: \"\",\n\t\t\tToFile:   \"Actual\",\n\t\t\tToDate:   \"\",\n\t\t\tContext:  1,\n\t\t}); err != nil {\n\t\t\tlog.Fatalf(\"Error occurred while checking if cheatsheets are up to date: %v\", err)\n\t\t}\n\t\tfmt.Printf(\n\t\t\t\"\\nCheatsheets are out of date. Please run `%s` at the project root and commit the changes. \"+\n\t\t\t\t\"If you run the script and no keybindings files are updated as a result, try rebasing onto master\"+\n\t\t\t\t\"and trying again.\\n\",\n\t\t\tgenerateCheatsheetCmd,\n\t\t)\n\t\tos.Exit(1)\n\t}\n\n\tfmt.Println(\"\\nCheatsheets are up to date\")\n}\n\nfunc GetKeybindingsDir() string {\n\treturn utils.GetLazyRootDirectory() + \"/docs/keybindings\"\n}\n\nfunc obtainContent(dir string) string {\n\tre := regexp.MustCompile(`Keybindings_\\w+\\.md$`)\n\n\tcontent := \"\"\n\terr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {\n\t\tif re.MatchString(path) {\n\t\t\tbytes, err := os.ReadFile(path)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalf(\"Error occurred while checking if cheatsheets are up to date: %v\", err)\n\t\t\t}\n\t\t\tcontent += fmt.Sprintf(\"\\n%s\\n\\n\", filepath.Base(path))\n\t\t\tcontent += string(bytes)\n\t\t}\n\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlog.Fatalf(\"Error occurred while checking if cheatsheets are up to date: %v\", err)\n\t}\n\n\treturn content\n}\n\n// Path: pkg/commands/container_stats.go\npackage commands\n\nimport (\n\t\"math\"\n\t\"time\"\n)\n\n// RecordedStats contains both the container stats we've received from docker, and our own derived stats  from those container stats. When configuring a graph, you're basically specifying the path of a value in this struct\ntype RecordedStats struct {\n\tClientStats  ContainerStats\n\tDerivedStats DerivedStats\n\tRecordedAt   time.Time\n}\n\n// DerivedStats contains some useful stats that we've calculated based on the raw container stats that we got back from docker\ntype DerivedStats struct {\n\tCPUPercentage    float64\n\tMemoryPercentage float64\n}\n\n// ContainerStats autogenerated at https://mholt.github.io/json-to-go/\ntype ContainerStats struct {\n\tRead      time.Time `json:\"read\"`\n\tPreread   time.Time `json:\"preread\"`\n\tPidsStats struct {\n\t\tCurrent int `json:\"current\"`\n\t} `json:\"pids_stats\"`\n\tBlkioStats struct {\n\t\tIoServiceBytesRecursive []struct {\n\t\t\tMajor int    `json:\"major\"`\n\t\t\tMinor int    `json:\"minor\"`\n\t\t\tOp    string `json:\"op\"`\n\t\t\tValue int    `json:\"value\"`\n\t\t} `json:\"io_service_bytes_recursive\"`\n\t\tIoServicedRecursive []struct {\n\t\t\tMajor int    `json:\"major\"`\n\t\t\tMinor int    `json:\"minor\"`\n\t\t\tOp    string `json:\"op\"`\n\t\t\tValue int    `json:\"value\"`\n\t\t} `json:\"io_serviced_recursive\"`\n\t\tIoQueueRecursive       []interface{} `json:\"io_queue_recursive\"`\n\t\tIoServiceTimeRecursive []interface{} `json:\"io_service_time_recursive\"`\n\t\tIoWaitTimeRecursive    []interface{} `json:\"io_wait_time_recursive\"`\n\t\tIoMergedRecursive      []interface{} `json:\"io_merged_recursive\"`\n\t\tIoTimeRecursive        []interface{} `json:\"io_time_recursive\"`\n\t\tSectorsRecursive       []interface{} `json:\"sectors_recursive\"`\n\t} `json:\"blkio_stats\"`\n\tNumProcs     int      `json:\"num_procs\"`\n\tStorageStats struct{} `json:\"storage_stats\"`\n\tCPUStats     struct {\n\t\tCPUUsage struct {\n\t\t\tTotalUsage        int64   `json:\"total_usage\"`\n\t\t\tPercpuUsage       []int64 `json:\"percpu_usage\"`\n\t\t\tUsageInKernelmode int64   `json:\"usage_in_kernelmode\"`\n\t\t\tUsageInUsermode   int64   `json:\"usage_in_usermode\"`\n\t\t} `json:\"cpu_usage\"`\n\t\tSystemCPUUsage int64 `json:\"system_cpu_usage\"`\n\t\tOnlineCpus     int   `json:\"online_cpus\"`\n\t\tThrottlingData struct {\n\t\t\tPeriods          int `json:\"periods\"`\n\t\t\tThrottledPeriods int `json:\"throttled_periods\"`\n\t\t\tThrottledTime    int `json:\"throttled_time\"`\n\t\t} `json:\"throttling_data\"`\n\t} `json:\"cpu_stats\"`\n\tPrecpuStats struct {\n\t\tCPUUsage struct {\n\t\t\tTotalUsage        int64   `json:\"total_usage\"`\n\t\t\tPercpuUsage       []int64 `json:\"percpu_usage\"`\n\t\t\tUsageInKernelmode int64   `json:\"usage_in_kernelmode\"`\n\t\t\tUsageInUsermode   int64   `json:\"usage_in_usermode\"`\n\t\t} `json:\"cpu_usage\"`\n\t\tSystemCPUUsage int64 `json:\"system_cpu_usage\"`\n\t\tOnlineCpus     int   `json:\"online_cpus\"`\n\t\tThrottlingData struct {\n\t\t\tPeriods          int `json:\"periods\"`\n\t\t\tThrottledPeriods int `json:\"throttled_periods\"`\n\t\t\tThrottledTime    int `json:\"throttled_time\"`\n\t\t} `json:\"throttling_data\"`\n\t} `json:\"precpu_stats\"`\n\tMemoryStats struct {\n\t\tUsage    int `json:\"usage\"`\n\t\tMaxUsage int `json:\"max_usage\"`\n\t\tStats    struct {\n\t\t\tActiveAnon              int   `json:\"active_anon\"`\n\t\t\tActiveFile              int   `json:\"active_file\"`\n\t\t\tCache                   int   `json:\"cache\"`\n\t\t\tDirty                   int   `json:\"dirty\"`\n\t\t\tHierarchicalMemoryLimit int64 `json:\"hierarchical_memory_limit\"`\n\t\t\tHierarchicalMemswLimit  int64 `json:\"hierarchical_memsw_limit\"`\n\t\t\tInactiveAnon            int   `json:\"inactive_anon\"`\n\t\t\tInactiveFile            int   `json:\"inactive_file\"`\n\t\t\tMappedFile              int   `json:\"mapped_file\"`\n\t\t\tPgfault                 int   `json:\"pgfault\"`\n\t\t\tPgmajfault              int   `json:\"pgmajfault\"`\n\t\t\tPgpgin                  int   `json:\"pgpgin\"`\n\t\t\tPgpgout                 int   `json:\"pgpgout\"`\n\t\t\tRss                     int   `json:\"rss\"`\n\t\t\tRssHuge                 int   `json:\"rss_huge\"`\n\t\t\tTotalActiveAnon         int   `json:\"total_active_anon\"`\n\t\t\tTotalActiveFile         int   `json:\"total_active_file\"`\n\t\t\tTotalCache              int   `json:\"total_cache\"`\n\t\t\tTotalDirty              int   `json:\"total_dirty\"`\n\t\t\tTotalInactiveAnon       int   `json:\"total_inactive_anon\"`\n\t\t\tTotalInactiveFile       int   `json:\"total_inactive_file\"`\n\t\t\tTotalMappedFile         int   `json:\"total_mapped_file\"`\n\t\t\tTotalPgfault            int   `json:\"total_pgfault\"`\n\t\t\tTotalPgmajfault         int   `json:\"total_pgmajfault\"`\n\t\t\tTotalPgpgin             int   `json:\"total_pgpgin\"`\n\t\t\tTotalPgpgout            int   `json:\"total_pgpgout\"`\n\t\t\tTotalRss                int   `json:\"total_rss\"`\n\t\t\tTotalRssHuge            int   `json:\"total_rss_huge\"`\n\t\t\tTotalUnevictable        int   `json:\"total_unevictable\"`\n\t\t\tTotalWriteback          int   `json:\"total_writeback\"`\n\t\t\tUnevictable             int   `json:\"unevictable\"`\n\t\t\tWriteback               int   `json:\"writeback\"`\n\t\t} `json:\"stats\"`\n\t\tLimit int64 `json:\"limit\"`\n\t} `json:\"memory_stats\"`\n\tName     string `json:\"name\"`\n\tID       string `json:\"id\"`\n\tNetworks struct {\n\t\tEth0 struct {\n\t\t\tRxBytes   int `json:\"rx_bytes\"`\n\t\t\tRxPackets int `json:\"rx_packets\"`\n\t\t\tRxErrors  int `json:\"rx_errors\"`\n\t\t\tRxDropped int `json:\"rx_dropped\"`\n\t\t\tTxBytes   int `json:\"tx_bytes\"`\n\t\t\tTxPackets int `json:\"tx_packets\"`\n\t\t\tTxErrors  int `json:\"tx_errors\"`\n\t\t\tTxDropped int `json:\"tx_dropped\"`\n\t\t} `json:\"eth0\"`\n\t} `json:\"networks\"`\n}\n\n// CalculateContainerCPUPercentage calculates the cpu usage of the container as a percent of total CPU usage\n// to calculate CPU usage, we take the increase in CPU time from the container since the last poll, divide that by the total increase in CPU time since the last poll, times by the number of cores, and times by 100 to get a percentage\n// I'm not entirely sure why we need to multiply by the number of cores, but the numbers work\nfunc (s *ContainerStats) CalculateContainerCPUPercentage() float64 {\n\tcpuUsageDelta := s.CPUStats.CPUUsage.TotalUsage - s.PrecpuStats.CPUUsage.TotalUsage\n\tcpuTotalUsageDelta := s.CPUStats.SystemCPUUsage - s.PrecpuStats.SystemCPUUsage\n\n\tvalue := float64(cpuUsageDelta*100) / float64(cpuTotalUsageDelta)\n\tif math.IsNaN(value) {\n\t\treturn 0\n\t}\n\treturn value\n}\n\n// CalculateContainerMemoryUsage calculates the memory usage of the container as a percent of total available memory\nfunc (s *ContainerStats) CalculateContainerMemoryUsage() float64 {\n\tvalue := float64(s.MemoryStats.Usage*100) / float64(s.MemoryStats.Limit)\n\tif math.IsNaN(value) {\n\t\treturn 0\n\t}\n\treturn value\n}\n\nfunc (c *Container) appendStats(stats *RecordedStats, maxDuration time.Duration) {\n\tc.StatsMutex.Lock()\n\tdefer c.StatsMutex.Unlock()\n\n\tc.StatHistory = append(c.StatHistory, stats)\n\tc.eraseOldHistory(maxDuration)\n}\n\n// eraseOldHistory removes any history before the user-specified max duration\nfunc (c *Container) eraseOldHistory(maxDuration time.Duration) {\n\tif maxDuration == 0 {\n\t\treturn\n\t}\n\n\tfor i, stat := range c.StatHistory {\n\t\tif time.Since(stat.RecordedAt) < maxDuration {\n\t\t\tc.StatHistory = c.StatHistory[i:]\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (c *Container) GetLastStats() (*RecordedStats, bool) {\n\tc.StatsMutex.Lock()\n\tdefer c.StatsMutex.Unlock()\n\thistory := c.StatHistory\n\tif len(history) == 0 {\n\t\treturn nil, false\n\t}\n\treturn history[len(history)-1], true\n}\n\n// Path: pkg/commands/container_stats_test.go\npackage commands\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestCalculateContainerCPUPercentage(t *testing.T) {\n\tcontainer := &ContainerStats{}\n\tcontainer.CPUStats.CPUUsage.TotalUsage = 10\n\tcontainer.CPUStats.SystemCPUUsage = 10\n\tcontainer.PrecpuStats.CPUUsage.TotalUsage = 5\n\tcontainer.PrecpuStats.SystemCPUUsage = 2\n\n\tassert.EqualValues(t, 62.5, container.CalculateContainerCPUPercentage())\n}\n\n// Path: pkg/commands/service.go\npackage commands\n\nimport (\n\t\"context\"\n\t\"os/exec\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Service : A docker Service\ntype Service struct {\n\tName          string\n\tID            string\n\tOSCommand     *OSCommand\n\tLog           *logrus.Entry\n\tContainer     *Container\n\tDockerCommand LimitedDockerCommand\n}\n\n// Remove removes the service's containers\nfunc (s *Service) Remove(options dockerTypes.ContainerRemoveOptions) error {\n\treturn s.Container.Remove(options)\n}\n\n// Stop stops the service's containers\nfunc (s *Service) Stop() error {\n\treturn s.runCommand(s.OSCommand.Config.UserConfig.CommandTemplates.StopService)\n}\n\n// Up up's the service\nfunc (s *Service) Up() error {\n\treturn s.runCommand(s.OSCommand.Config.UserConfig.CommandTemplates.UpService)\n}\n\n// Restart restarts the service\nfunc (s *Service) Restart() error {\n\treturn s.runCommand(s.OSCommand.Config.UserConfig.CommandTemplates.RestartService)\n}\n\n// Restart starts the service\nfunc (s *Service) Start() error {\n\treturn s.runCommand(s.OSCommand.Config.UserConfig.CommandTemplates.StartService)\n}\n\nfunc (s *Service) runCommand(templateCmdStr string) error {\n\tcommand := utils.ApplyTemplate(\n\t\ttemplateCmdStr,\n\t\ts.DockerCommand.NewCommandObject(CommandObject{Service: s}),\n\t)\n\treturn s.OSCommand.RunCommand(command)\n}\n\n// Attach attaches to the service\nfunc (s *Service) Attach() (*exec.Cmd, error) {\n\treturn s.Container.Attach()\n}\n\n// ViewLogs attaches to a subprocess viewing the service's logs\nfunc (s *Service) ViewLogs() (*exec.Cmd, error) {\n\ttemplateString := s.OSCommand.Config.UserConfig.CommandTemplates.ViewServiceLogs\n\tcommand := utils.ApplyTemplate(\n\t\ttemplateString,\n\t\ts.DockerCommand.NewCommandObject(CommandObject{Service: s}),\n\t)\n\n\tcmd := s.OSCommand.ExecutableFromString(command)\n\ts.OSCommand.PrepareForChildren(cmd)\n\n\treturn cmd, nil\n}\n\n// RenderTop renders the process list of the service\nfunc (s *Service) RenderTop(ctx context.Context) (string, error) {\n\ttemplateString := s.OSCommand.Config.UserConfig.CommandTemplates.ServiceTop\n\tcommand := utils.ApplyTemplate(\n\t\ttemplateString,\n\t\ts.DockerCommand.NewCommandObject(CommandObject{Service: s}),\n\t)\n\n\treturn s.OSCommand.RunCommandWithOutputContext(ctx, command)\n}\n\n// Path: pkg/commands/ssh/ssh.go\npackage ssh\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path\"\n\t\"time\"\n)\n\n// we only need these two methods from our OSCommand struct, for killing commands\ntype CmdKiller interface {\n\tKill(cmd *exec.Cmd) error\n\tPrepareForChildren(cmd *exec.Cmd)\n}\n\ntype SSHHandler struct {\n\toSCommand CmdKiller\n\n\tdialContext func(ctx context.Context, network, addr string) (io.Closer, error)\n\tstartCmd    func(*exec.Cmd) error\n\ttempDir     func(dir string, pattern string) (name string, err error)\n\tgetenv      func(key string) string\n\tsetenv      func(key, value string) error\n}\n\nfunc NewSSHHandler(oSCommand CmdKiller) *SSHHandler {\n\treturn &SSHHandler{\n\t\toSCommand: oSCommand,\n\n\t\tdialContext: func(ctx context.Context, network, addr string) (io.Closer, error) {\n\t\t\treturn (&net.Dialer{}).DialContext(ctx, network, addr)\n\t\t},\n\t\tstartCmd: func(cmd *exec.Cmd) error { return cmd.Start() },\n\t\ttempDir:  os.MkdirTemp,\n\t\tgetenv:   os.Getenv,\n\t\tsetenv:   os.Setenv,\n\t}\n}\n\n// HandleSSHDockerHost overrides the DOCKER_HOST environment variable\n// to point towards a local unix socket tunneled over SSH to the specified ssh host.\nfunc (self *SSHHandler) HandleSSHDockerHost() (io.Closer, error) {\n\tconst key = \"DOCKER_HOST\"\n\tctx := context.Background()\n\tu, err := url.Parse(self.getenv(key))\n\tif err != nil {\n\t\t// if no or an invalid docker host is specified, continue nominally\n\t\treturn noopCloser{}, nil\n\t}\n\n\t// if the docker host scheme is \"ssh\", forward the docker socket before creating the client\n\tif u.Scheme == \"ssh\" {\n\t\ttunnel, err := self.createDockerHostTunnel(ctx, u.Host)\n\t\tif err != nil {\n\t\t\treturn noopCloser{}, fmt.Errorf(\"tunnel ssh docker host: %w\", err)\n\t\t}\n\t\terr = self.setenv(key, tunnel.socketPath)\n\t\tif err != nil {\n\t\t\treturn noopCloser{}, fmt.Errorf(\"override DOCKER_HOST to tunneled socket: %w\", err)\n\t\t}\n\n\t\treturn tunnel, nil\n\t}\n\treturn noopCloser{}, nil\n}\n\ntype noopCloser struct{}\n\nfunc (noopCloser) Close() error { return nil }\n\ntype tunneledDockerHost struct {\n\tsocketPath string\n\tcmd        *exec.Cmd\n\toSCommand  CmdKiller\n}\n\nvar _ io.Closer = (*tunneledDockerHost)(nil)\n\nfunc (t *tunneledDockerHost) Close() error {\n\treturn t.oSCommand.Kill(t.cmd)\n}\n\nfunc (self *SSHHandler) createDockerHostTunnel(ctx context.Context, remoteHost string) (*tunneledDockerHost, error) {\n\tsocketDir, err := self.tempDir(\"/tmp\", \"lazydocker-sshtunnel-\")\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"create ssh tunnel tmp file: %w\", err)\n\t}\n\tlocalSocket := path.Join(socketDir, \"dockerhost.sock\")\n\n\tcmd, err := self.tunnelSSH(ctx, remoteHost, localSocket)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"tunnel docker host over ssh: %w\", err)\n\t}\n\n\t// set a reasonable timeout, then wait for the socket to dial successfully\n\t// before attempting to create a new docker client\n\tconst socketTunnelTimeout = 8 * time.Second\n\tctx, cancel := context.WithTimeout(ctx, socketTunnelTimeout)\n\tdefer cancel()\n\n\terr = self.retrySocketDial(ctx, localSocket)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"ssh tunneled socket never became available: %w\", err)\n\t}\n\n\t// construct the new DOCKER_HOST url with the proper scheme\n\tnewDockerHostURL := url.URL{Scheme: \"unix\", Path: localSocket}\n\treturn &tunneledDockerHost{\n\t\tsocketPath: newDockerHostURL.String(),\n\t\tcmd:        cmd,\n\t\toSCommand:  self.oSCommand,\n\t}, nil\n}\n\n// Attempt to dial the socket until it becomes available.\n// The retry loop will continue until the parent context is canceled.\nfunc (self *SSHHandler) retrySocketDial(ctx context.Context, socketPath string) error {\n\tt := time.NewTicker(1 * time.Second)\n\tdefer t.Stop()\n\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn ctx.Err()\n\t\tcase <-t.C:\n\t\t}\n\t\t// attempt to dial the socket, exit on success\n\t\terr := self.tryDial(ctx, socketPath)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\treturn nil\n\t}\n}\n\n// Try to dial the specified unix socket, immediately close the connection if successfully created.\nfunc (self *SSHHandler) tryDial(ctx context.Context, socketPath string) error {\n\tconn, err := self.dialContext(ctx, \"unix\", socketPath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer conn.Close()\n\treturn nil\n}\n\nfunc (self *SSHHandler) tunnelSSH(ctx context.Context, host, localSocket string) (*exec.Cmd, error) {\n\tcmd := exec.CommandContext(ctx, \"ssh\", \"-L\", localSocket+\":/var/run/docker.sock\", host, \"-N\")\n\tself.oSCommand.PrepareForChildren(cmd)\n\terr := self.startCmd(cmd)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn cmd, nil\n}\n\n// Path: pkg/commands/docker.go\npackage commands\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\togLog \"log\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"time\"\n\n\tcliconfig \"github.com/docker/cli/cli/config\"\n\tddocker \"github.com/docker/cli/cli/context/docker\"\n\tctxstore \"github.com/docker/cli/cli/context/store\"\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/docker/docker/client\"\n\t\"github.com/imdario/mergo\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands/ssh\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\nconst (\n\tAPIVersion = \"1.25\"\n)\n\n// DockerCommand is our main docker interface\ntype DockerCommand struct {\n\tLog                    *logrus.Entry\n\tOSCommand              *OSCommand\n\tTr                     *i18n.TranslationSet\n\tConfig                 *config.AppConfig\n\tClient                 *client.Client\n\tInDockerComposeProject bool\n\tErrorChan              chan error\n\tContainerMutex         deadlock.Mutex\n\tServiceMutex           deadlock.Mutex\n\n\tClosers []io.Closer\n}\n\nvar _ io.Closer = &DockerCommand{}\n\n// LimitedDockerCommand is a stripped-down DockerCommand with just the methods the container/service/image might need\ntype LimitedDockerCommand interface {\n\tNewCommandObject(CommandObject) CommandObject\n}\n\n// CommandObject is what we pass to our template resolvers when we are running a custom command. We do not guarantee that all fields will be populated: just the ones that make sense for the current context\ntype CommandObject struct {\n\tDockerCompose string\n\tService       *Service\n\tContainer     *Container\n\tImage         *Image\n\tVolume        *Volume\n\tNetwork       *Network\n}\n\n// NewCommandObject takes a command object and returns a default command object with the passed command object merged in\nfunc (c *DockerCommand) NewCommandObject(obj CommandObject) CommandObject {\n\tdefaultObj := CommandObject{DockerCompose: c.Config.UserConfig.CommandTemplates.DockerCompose}\n\t_ = mergo.Merge(&defaultObj, obj)\n\treturn defaultObj\n}\n\n// NewDockerCommand it runs docker commands\nfunc NewDockerCommand(log *logrus.Entry, osCommand *OSCommand, tr *i18n.TranslationSet, config *config.AppConfig, errorChan chan error) (*DockerCommand, error) {\n\ttunnelCloser, err := ssh.NewSSHHandler(osCommand).HandleSSHDockerHost()\n\tif err != nil {\n\t\togLog.Fatal(err)\n\t}\n\n\tdockerHost, err := determineDockerHost()\n\tif err != nil {\n\t\togLog.Printf(\"> could not determine host %v\", err)\n\t}\n\n\tcli, err := client.NewClientWithOpts(client.FromEnv, client.WithVersion(APIVersion), client.WithHost(dockerHost))\n\tif err != nil {\n\t\togLog.Fatal(err)\n\t}\n\n\tdockerCommand := &DockerCommand{\n\t\tLog:                    log,\n\t\tOSCommand:              osCommand,\n\t\tTr:                     tr,\n\t\tConfig:                 config,\n\t\tClient:                 cli,\n\t\tErrorChan:              errorChan,\n\t\tInDockerComposeProject: true,\n\t\tClosers:                []io.Closer{tunnelCloser},\n\t}\n\n\tcommand := utils.ApplyTemplate(\n\t\tconfig.UserConfig.CommandTemplates.CheckDockerComposeConfig,\n\t\tdockerCommand.NewCommandObject(CommandObject{}),\n\t)\n\n\tlog.Warn(command)\n\n\terr = osCommand.RunCommand(\n\t\tutils.ApplyTemplate(\n\t\t\tconfig.UserConfig.CommandTemplates.CheckDockerComposeConfig,\n\t\t\tdockerCommand.NewCommandObject(CommandObject{}),\n\t\t),\n\t)\n\tif err != nil {\n\t\tdockerCommand.InDockerComposeProject = false\n\t\tlog.Warn(err.Error())\n\t}\n\n\treturn dockerCommand, nil\n}\n\nfunc (c *DockerCommand) Close() error {\n\treturn utils.CloseMany(c.Closers)\n}\n\nfunc (c *DockerCommand) CreateClientStatMonitor(container *Container) {\n\tcontainer.MonitoringStats = true\n\tstream, err := c.Client.ContainerStats(context.Background(), container.ID, true)\n\tif err != nil {\n\t\t// not creating error panel because if we've disconnected from docker we'll\n\t\t// have already created an error panel\n\t\tc.Log.Error(err)\n\t\tcontainer.MonitoringStats = false\n\t\treturn\n\t}\n\n\tdefer stream.Body.Close()\n\n\tscanner := bufio.NewScanner(stream.Body)\n\tfor scanner.Scan() {\n\t\tdata := scanner.Bytes()\n\t\tvar stats ContainerStats\n\t\t_ = json.Unmarshal(data, &stats)\n\n\t\trecordedStats := &RecordedStats{\n\t\t\tClientStats: stats,\n\t\t\tDerivedStats: DerivedStats{\n\t\t\t\tCPUPercentage:    stats.CalculateContainerCPUPercentage(),\n\t\t\t\tMemoryPercentage: stats.CalculateContainerMemoryUsage(),\n\t\t\t},\n\t\t\tRecordedAt: time.Now(),\n\t\t}\n\n\t\tcontainer.appendStats(recordedStats, c.Config.UserConfig.Stats.MaxDuration)\n\t}\n\n\tcontainer.MonitoringStats = false\n}\n\nfunc (c *DockerCommand) RefreshContainersAndServices(currentServices []*Service, currentContainers []*Container) ([]*Container, []*Service, error) {\n\tc.ServiceMutex.Lock()\n\tdefer c.ServiceMutex.Unlock()\n\n\tcontainers, err := c.GetContainers(currentContainers)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tvar services []*Service\n\t// we only need to get these services once because they won't change in the runtime of the program\n\tif currentServices != nil {\n\t\tservices = currentServices\n\t} else {\n\t\tservices, err = c.GetServices()\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t}\n\n\tc.assignContainersToServices(containers, services)\n\n\treturn containers, services, nil\n}\n\nfunc (c *DockerCommand) assignContainersToServices(containers []*Container, services []*Service) {\nL:\n\tfor _, service := range services {\n\t\tfor _, container := range containers {\n\t\t\tif !container.OneOff && container.ServiceName == service.Name {\n\t\t\t\tservice.Container = container\n\t\t\t\tcontinue L\n\t\t\t}\n\t\t}\n\t\tservice.Container = nil\n\t}\n}\n\n// GetContainers gets the docker containers\nfunc (c *DockerCommand) GetContainers(existingContainers []*Container) ([]*Container, error) {\n\tc.ContainerMutex.Lock()\n\tdefer c.ContainerMutex.Unlock()\n\n\tcontainers, err := c.Client.ContainerList(context.Background(), dockerTypes.ContainerListOptions{All: true})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\townContainers := make([]*Container, len(containers))\n\n\tfor i, container := range containers {\n\t\tvar newContainer *Container\n\n\t\t// check if we already data stored against the container\n\t\tfor _, existingContainer := range existingContainers {\n\t\t\tif existingContainer.ID == container.ID {\n\t\t\t\tnewContainer = existingContainer\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\t// initialise the container if it's completely new\n\t\tif newContainer == nil {\n\t\t\tnewContainer = &Container{\n\t\t\t\tID:            container.ID,\n\t\t\t\tClient:        c.Client,\n\t\t\t\tOSCommand:     c.OSCommand,\n\t\t\t\tLog:           c.Log,\n\t\t\t\tDockerCommand: c,\n\t\t\t\tTr:            c.Tr,\n\t\t\t}\n\t\t}\n\n\t\tnewContainer.Container = container\n\t\t// if the container is made with a name label we will use that\n\t\tif name, ok := container.Labels[\"name\"]; ok {\n\t\t\tnewContainer.Name = name\n\t\t} else {\n\t\t\tnewContainer.Name = strings.TrimLeft(container.Names[0], \"/\")\n\t\t}\n\t\tnewContainer.ServiceName = container.Labels[\"com.docker.compose.service\"]\n\t\tnewContainer.ProjectName = container.Labels[\"com.docker.compose.project\"]\n\t\tnewContainer.ContainerNumber = container.Labels[\"com.docker.compose.container\"]\n\t\tnewContainer.OneOff = container.Labels[\"com.docker.compose.oneoff\"] == \"True\"\n\n\t\townContainers[i] = newContainer\n\t}\n\n\treturn ownContainers, nil\n}\n\n// GetServices gets services\nfunc (c *DockerCommand) GetServices() ([]*Service, error) {\n\tif !c.InDockerComposeProject {\n\t\treturn nil, nil\n\t}\n\n\tcomposeCommand := c.Config.UserConfig.CommandTemplates.DockerCompose\n\toutput, err := c.OSCommand.RunCommandWithOutput(fmt.Sprintf(\"%s config --services\", composeCommand))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// output looks like:\n\t// service1\n\t// service2\n\n\tlines := utils.SplitLines(output)\n\tservices := make([]*Service, len(lines))\n\tfor i, str := range lines {\n\t\tservices[i] = &Service{\n\t\t\tName:          str,\n\t\t\tID:            str,\n\t\t\tOSCommand:     c.OSCommand,\n\t\t\tLog:           c.Log,\n\t\t\tDockerCommand: c,\n\t\t}\n\t}\n\n\treturn services, nil\n}\n\n// UpdateContainerDetails attaches the details returned from docker inspect to each of the containers\n// this contains a bit more info than what you get from the go-docker client\nfunc (c *DockerCommand) UpdateContainerDetails(containers []*Container) error {\n\tc.ContainerMutex.Lock()\n\tdefer c.ContainerMutex.Unlock()\n\n\tfor _, container := range containers {\n\t\tdetails, err := c.Client.ContainerInspect(context.Background(), container.ID)\n\t\tif err != nil {\n\t\t\tc.Log.Error(err)\n\t\t} else {\n\t\t\tcontainer.Details = details\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// ViewAllLogs attaches to a subprocess viewing all the logs from docker-compose\nfunc (c *DockerCommand) ViewAllLogs() (*exec.Cmd, error) {\n\tcmd := c.OSCommand.ExecutableFromString(\n\t\tutils.ApplyTemplate(\n\t\t\tc.OSCommand.Config.UserConfig.CommandTemplates.ViewAllLogs,\n\t\t\tc.NewCommandObject(CommandObject{}),\n\t\t),\n\t)\n\n\tc.OSCommand.PrepareForChildren(cmd)\n\n\treturn cmd, nil\n}\n\n// DockerComposeConfig returns the result of 'docker-compose config'\nfunc (c *DockerCommand) DockerComposeConfig() string {\n\toutput, err := c.OSCommand.RunCommandWithOutput(\n\t\tutils.ApplyTemplate(\n\t\t\tc.OSCommand.Config.UserConfig.CommandTemplates.DockerComposeConfig,\n\t\t\tc.NewCommandObject(CommandObject{}),\n\t\t),\n\t)\n\tif err != nil {\n\t\toutput = err.Error()\n\t}\n\treturn output\n}\n\n// determineDockerHost tries to the determine the docker host that we should connect to\n// in the following order of decreasing precedence:\n//   - value of \"DOCKER_HOST\" environment variable\n//   - host retrieved from the current context (specified via DOCKER_CONTEXT)\n//   - \"default docker host\" for the host operating system, otherwise\nfunc determineDockerHost() (string, error) {\n\t// If the docker host is explicitly set via the \"DOCKER_HOST\" environment variable,\n\t// then its a no-brainer :shrug:\n\tif os.Getenv(\"DOCKER_HOST\") != \"\" {\n\t\treturn os.Getenv(\"DOCKER_HOST\"), nil\n\t}\n\n\tcurrentContext := os.Getenv(\"DOCKER_CONTEXT\")\n\tif currentContext == \"\" {\n\t\tcf, err := cliconfig.Load(cliconfig.Dir())\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tcurrentContext = cf.CurrentContext\n\t}\n\n\tif currentContext == \"\" {\n\t\t// If a docker context is neither specified via the \"DOCKER_CONTEXT\" environment variable nor via the\n\t\t// $HOME/.docker/config file, then we fall back to connecting to the \"default docker host\" meant for\n\t\t// the host operating system.\n\t\treturn defaultDockerHost, nil\n\t}\n\n\tstoreConfig := ctxstore.NewConfig(\n\t\tfunc() interface{} { return &ddocker.EndpointMeta{} },\n\t\tctxstore.EndpointTypeGetter(ddocker.DockerEndpoint, func() interface{} { return &ddocker.EndpointMeta{} }),\n\t)\n\n\tst := ctxstore.New(cliconfig.ContextStoreDir(), storeConfig)\n\tmd, err := st.GetMetadata(currentContext)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdockerEP, ok := md.Endpoints[ddocker.DockerEndpoint]\n\tif !ok {\n\t\treturn \"\", err\n\t}\n\tdockerEPMeta, ok := dockerEP.(ddocker.EndpointMeta)\n\tif !ok {\n\t\treturn \"\", fmt.Errorf(\"expected docker.EndpointMeta, got %T\", dockerEP)\n\t}\n\n\tif dockerEPMeta.Host != \"\" {\n\t\treturn dockerEPMeta.Host, nil\n\t}\n\n\t// We might end up here, if the context was created with the `host` set to an empty value (i.e. '').\n\t// For example:\n\t// ```sh\n\t// docker context create foo --docker \"host=\"\n\t// ```\n...\n", "needle_token_start": 3609, "needle_token_end": 3956, "code_context_ntokens": 15157, "output": ["```go\n// NewApp bootstrap a new application\nfunc NewApp(config *config.AppConfig) (*App, error) {\n\tapp := &App{\n\t\tclosers:   []io.Closer{},\n\t\tConfig:    config,\n\t\tErrorChan: make(chan error),\n\t}\n\tvar err error\n\tapp.Log = log.NewLogger(config, \"23432119147a4367abf7c0de2aa99a2d\")\n\tapp.Tr, err = i18n.NewTranslationSetFromConfig(app.Log, config.UserConfig.Gui.Language)\n\tif err != nil {\n\t\treturn app, err\n\t}\n\tapp.OSCommand = commands.NewOSCommand(app.Log, config)\n\n\t// here is the place to make use of the docker-compose.yml file in the current directory\n\n\tapp.DockerCommand, err = commands.NewDockerCommand(app.Log, app.OSCommand, app.Tr, app.Config, app.ErrorChan)\n\tif err != nil {\n\t\treturn app, err\n\t}\n\tapp.closers = append(app.closers, app.DockerCommand)\n\tapp.Gui, err = gui.NewGui(app.Log, app.DockerCommand, app.OSCommand, app.Tr, config, app.ErrorChan)\n\tif err != nil {\n\t\treturn app, err\n\t}\n\treturn app, nil\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "sanitisedCommandOutput", "language": "go", "path": "pkg/commands/os.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to process the output from executing a system command, ensuring that any errors are clearly reported and the output is returned as a clean string.\n2. **Input**: It takes two parameters: a byte array containing the output from a command execution, and an error object that might have been generated during the command's execution.\n3. **Output**: It returns a string representing the command's output and an error object that might contain a more descriptive error message if the command execution failed.\n4. **Procedure**: The function first converts the byte array of the command output into a string. If the error object indicates a failure (specifically an exit error), it extracts a more descriptive error message from the error object to return alongside the output string. If there is no error, it simply returns the output string and a nil error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/utils/utils.go\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"html/template\"\n\t\"io\"\n\t\"math\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/mattn/go-runewidth\"\n\n\t// \"github.com/jesseduffield/yaml\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/goccy/go-yaml\"\n\t\"github.com/goccy/go-yaml/lexer\"\n\t\"github.com/goccy/go-yaml/printer\"\n)\n\n// SplitLines takes a multiline string and splits it on newlines\n// currently we are also stripping \\r's which may have adverse effects for\n// windows users (but no issues have been raised yet)\nfunc SplitLines(multilineString string) []string {\n\tmultilineString = strings.Replace(multilineString, \"\\r\", \"\", -1)\n\tif multilineString == \"\" || multilineString == \"\\n\" {\n\t\treturn make([]string, 0)\n\t}\n\tlines := strings.Split(multilineString, \"\\n\")\n\tif lines[len(lines)-1] == \"\" {\n\t\treturn lines[:len(lines)-1]\n\t}\n\treturn lines\n}\n\n// WithPadding pads a string as much as you want\nfunc WithPadding(str string, padding int) string {\n\tuncoloredStr := Decolorise(str)\n\tif padding < runewidth.StringWidth(uncoloredStr) {\n\t\treturn str\n\t}\n\treturn str + strings.Repeat(\" \", padding-runewidth.StringWidth(uncoloredStr))\n}\n\n// ColoredString takes a string and a colour attribute and returns a colored\n// string with that attribute\nfunc ColoredString(str string, colorAttribute color.Attribute) string {\n\t// fatih/color does not have a color.Default attribute, so unless we fork that repo the only way for us to express that we don't want to color a string different to the terminal's default is to not call the function in the first place, but that's annoying when you want a streamlined code path. Because I'm too lazy to fork the repo right now, we'll just assume that by FgWhite you really mean Default, for the sake of supporting users with light themed terminals.\n\tif colorAttribute == color.FgWhite {\n\t\treturn str\n\t}\n\tcolour := color.New(colorAttribute)\n\treturn ColoredStringDirect(str, colour)\n}\n\n// ColoredYamlString takes an YAML formatted string and returns a colored string\n// with colors hardcoded as:\n// keys: cyan\n// Booleans: magenta\n// Numbers: yellow\n// Strings: green\nfunc ColoredYamlString(str string) string {\n\tformat := func(attr color.Attribute) string {\n\t\treturn fmt.Sprintf(\"%s[%dm\", \"\\x1b\", attr)\n\t}\n\ttokens := lexer.Tokenize(str)\n\tvar p printer.Printer\n\tp.Bool = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgMagenta),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.Number = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgYellow),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.MapKey = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgCyan),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.String = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgGreen),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\treturn p.PrintTokens(tokens)\n}\n\n// MultiColoredString takes a string and an array of colour attributes and returns a colored\n// string with those attributes\nfunc MultiColoredString(str string, colorAttribute ...color.Attribute) string {\n\tcolour := color.New(colorAttribute...)\n\treturn ColoredStringDirect(str, colour)\n}\n\n// ColoredStringDirect used for aggregating a few color attributes rather than\n// just sending a single one\nfunc ColoredStringDirect(str string, colour *color.Color) string {\n\treturn colour.SprintFunc()(fmt.Sprint(str))\n}\n\n// NormalizeLinefeeds - Removes all Windows and Mac style line feeds\nfunc NormalizeLinefeeds(str string) string {\n\tstr = strings.Replace(str, \"\\r\\n\", \"\\n\", -1)\n\tstr = strings.Replace(str, \"\\r\", \"\", -1)\n\treturn str\n}\n\n// Loader dumps a string to be displayed as a loader\nfunc Loader() string {\n\tcharacters := \"|/-\\\\\"\n\tnow := time.Now()\n\tnanos := now.UnixNano()\n\tindex := nanos / 50000000 % int64(len(characters))\n\treturn characters[index : index+1]\n}\n\n// ResolvePlaceholderString populates a template with values\nfunc ResolvePlaceholderString(str string, arguments map[string]string) string {\n\tfor key, value := range arguments {\n\t\tstr = strings.Replace(str, \"{{\"+key+\"}}\", value, -1)\n\t}\n\treturn str\n}\n\n// Max returns the maximum of two integers\nfunc Max(x, y int) int {\n\tif x > y {\n\t\treturn x\n\t}\n\treturn y\n}\n\n// RenderTable takes an array of string arrays and returns a table containing the values\nfunc RenderTable(rows [][]string) (string, error) {\n\tif len(rows) == 0 {\n\t\treturn \"\", nil\n\t}\n\tif !displayArraysAligned(rows) {\n\t\treturn \"\", errors.New(\"Each item must return the same number of strings to display\")\n\t}\n\n\tcolumnPadWidths := getPadWidths(rows)\n\tpaddedDisplayRows := getPaddedDisplayStrings(rows, columnPadWidths)\n\n\treturn strings.Join(paddedDisplayRows, \"\\n\"), nil\n}\n\n// Decolorise strips a string of color\nfunc Decolorise(str string) string {\n\tre := regexp.MustCompile(`\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mK]`)\n\treturn re.ReplaceAllString(str, \"\")\n}\n\nfunc getPadWidths(rows [][]string) []int {\n\tif len(rows[0]) <= 1 {\n\t\treturn []int{}\n\t}\n\tcolumnPadWidths := make([]int, len(rows[0])-1)\n\tfor i := range columnPadWidths {\n\t\tfor _, cells := range rows {\n\t\t\tuncoloredCell := Decolorise(cells[i])\n\n\t\t\tif runewidth.StringWidth(uncoloredCell) > columnPadWidths[i] {\n\t\t\t\tcolumnPadWidths[i] = runewidth.StringWidth(uncoloredCell)\n\t\t\t}\n\t\t}\n\t}\n\treturn columnPadWidths\n}\n\nfunc getPaddedDisplayStrings(rows [][]string, columnPadWidths []int) []string {\n\tpaddedDisplayRows := make([]string, len(rows))\n\tfor i, cells := range rows {\n\t\tfor j, columnPadWidth := range columnPadWidths {\n\t\t\tpaddedDisplayRows[i] += WithPadding(cells[j], columnPadWidth) + \" \"\n\t\t}\n\t\tpaddedDisplayRows[i] += cells[len(columnPadWidths)]\n\t}\n\treturn paddedDisplayRows\n}\n\n// displayArraysAligned returns true if every string array returned from our\n// list of displayables has the same length\nfunc displayArraysAligned(stringArrays [][]string) bool {\n\tfor _, strings := range stringArrays {\n\t\tif len(strings) != len(stringArrays[0]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc FormatBinaryBytes(b int) string {\n\tn := float64(b)\n\tunits := []string{\"B\", \"kiB\", \"MiB\", \"GiB\", \"TiB\", \"PiB\", \"EiB\", \"ZiB\", \"YiB\"}\n\tfor _, unit := range units {\n\t\tif n > math.Pow(2, 10) {\n\t\t\tn /= math.Pow(2, 10)\n\t\t} else {\n\t\t\tval := fmt.Sprintf(\"%.2f%s\", n, unit)\n\t\t\tif val == \"0.00B\" {\n\t\t\t\treturn \"0B\"\n\t\t\t}\n\t\t\treturn val\n\t\t}\n\t}\n\treturn \"a lot\"\n}\n\nfunc FormatDecimalBytes(b int) string {\n\tn := float64(b)\n\tunits := []string{\"B\", \"kB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\", \"ZB\", \"YB\"}\n\tfor _, unit := range units {\n\t\tif n > math.Pow(10, 3) {\n\t\t\tn /= math.Pow(10, 3)\n\t\t} else {\n\t\t\tval := fmt.Sprintf(\"%.2f%s\", n, unit)\n\t\t\tif val == \"0.00B\" {\n\t\t\t\treturn \"0B\"\n\t\t\t}\n\t\t\treturn val\n\t\t}\n\t}\n\treturn \"a lot\"\n}\n\nfunc ApplyTemplate(str string, object interface{}) string {\n\tvar buf bytes.Buffer\n\t_ = template.Must(template.New(\"\").Parse(str)).Execute(&buf, object)\n\treturn buf.String()\n}\n\n// GetGocuiAttribute gets the gocui color attribute from the string\nfunc GetGocuiAttribute(key string) gocui.Attribute {\n\tcolorMap := map[string]gocui.Attribute{\n\t\t\"default\":   gocui.ColorDefault,\n\t\t\"black\":     gocui.ColorBlack,\n\t\t\"red\":       gocui.ColorRed,\n\t\t\"green\":     gocui.ColorGreen,\n\t\t\"yellow\":    gocui.ColorYellow,\n\t\t\"blue\":      gocui.ColorBlue,\n\t\t\"magenta\":   gocui.ColorMagenta,\n\t\t\"cyan\":      gocui.ColorCyan,\n\t\t\"white\":     gocui.ColorWhite,\n\t\t\"bold\":      gocui.AttrBold,\n\t\t\"reverse\":   gocui.AttrReverse,\n\t\t\"underline\": gocui.AttrUnderline,\n\t}\n\tvalue, present := colorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn gocui.ColorDefault\n}\n\n// GetColorAttribute gets the color attribute from the string\nfunc GetColorAttribute(key string) color.Attribute {\n\tcolorMap := map[string]color.Attribute{\n\t\t\"default\":   color.FgWhite,\n\t\t\"black\":     color.FgBlack,\n\t\t\"red\":       color.FgRed,\n\t\t\"green\":     color.FgGreen,\n\t\t\"yellow\":    color.FgYellow,\n\t\t\"blue\":      color.FgBlue,\n\t\t\"magenta\":   color.FgMagenta,\n\t\t\"cyan\":      color.FgCyan,\n\t\t\"white\":     color.FgWhite,\n\t\t\"bold\":      color.Bold,\n\t\t\"underline\": color.Underline,\n\t}\n\tvalue, present := colorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn color.FgWhite\n}\n\n// WithShortSha returns a command but with a shorter SHA. in the terminal we're all used to 10 character SHAs but under the hood they're actually 64 characters long. No need including all the characters when we're just displaying a command\nfunc WithShortSha(str string) string {\n\tsplit := strings.Split(str, \" \")\n\tfor i, word := range split {\n\t\t// good enough proxy for now\n\t\tif len(word) == 64 {\n\t\t\tsplit[i] = word[0:10]\n\t\t}\n\t}\n\treturn strings.Join(split, \" \")\n}\n\n// FormatMapItem is for displaying items in a map\nfunc FormatMapItem(padding int, k string, v interface{}) string {\n\treturn fmt.Sprintf(\"%s%s %v\\n\", strings.Repeat(\" \", padding), ColoredString(k+\":\", color.FgYellow), fmt.Sprintf(\"%v\", v))\n}\n\n// FormatMap is for displaying a map\nfunc FormatMap(padding int, m map[string]string) string {\n\tif len(m) == 0 {\n\t\treturn \"none\\n\"\n\t}\n\n\toutput := \"\\n\"\n\n\tkeys := make([]string, 0, len(m))\n\tfor key := range m {\n\t\tkeys = append(keys, key)\n\t}\n\tsort.Strings(keys)\n\tfor _, key := range keys {\n\t\toutput += FormatMapItem(padding, key, m[key])\n\t}\n\n\treturn output\n}\n\ntype multiErr []error\n\nfunc (m multiErr) Error() string {\n\tvar b bytes.Buffer\n\tb.WriteString(\"encountered multiple errors:\")\n\tfor _, err := range m {\n\t\tb.WriteString(\"\\n\\t... \" + err.Error())\n\t}\n\treturn b.String()\n}\n\nfunc CloseMany(closers []io.Closer) error {\n\terrs := make([]error, 0, len(closers))\n\tfor _, c := range closers {\n\t\terr := c.Close()\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn multiErr(errs)\n\t}\n\treturn nil\n}\n\nfunc SafeTruncate(str string, limit int) string {\n\tif len(str) > limit {\n\t\treturn str[0:limit]\n\t} else {\n\t\treturn str\n\t}\n}\n\nfunc IsValidHexValue(v string) bool {\n\tif len(v) != 4 && len(v) != 7 {\n\t\treturn false\n\t}\n\n\tif v[0] != '#' {\n\t\treturn false\n\t}\n\n\tfor _, char := range v[1:] {\n\t\tswitch char {\n\t\tcase '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F':\n\t\t\tcontinue\n\t\tdefault:\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// Style used on menu items that open another menu\nfunc OpensMenuStyle(str string) string {\n\treturn ColoredString(fmt.Sprintf(\"%s...\", str), color.FgMagenta)\n}\n\n// MarshalIntoYaml gets any json-tagged data and marshal it into yaml saving original json structure.\n// Useful for structs from 3rd-party libs without yaml tags.\nfunc MarshalIntoYaml(data interface{}) ([]byte, error) {\n...\n// Path: pkg/commands/os.go\npackage commands\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/go-errors/errors\"\n\n\t\"github.com/jesseduffield/kill\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/mgutz/str\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Platform stores the os state\ntype Platform struct {\n\tos              string\n\tshell           string\n\tshellArg        string\n\topenCommand     string\n\topenLinkCommand string\n}\n\n// OSCommand holds all the os commands\ntype OSCommand struct {\n\tLog      *logrus.Entry\n\tPlatform *Platform\n\tConfig   *config.AppConfig\n\tcommand  func(string, ...string) *exec.Cmd\n\tgetenv   func(string) string\n}\n\n// NewOSCommand os command runner\nfunc NewOSCommand(log *logrus.Entry, config *config.AppConfig) *OSCommand {\n\treturn &OSCommand{\n\t\tLog:      log,\n\t\tPlatform: getPlatform(),\n\t\tConfig:   config,\n\t\tcommand:  exec.Command,\n\t\tgetenv:   os.Getenv,\n\t}\n}\n\n// SetCommand sets the command function used by the struct.\n// To be used for testing only\nfunc (c *OSCommand) SetCommand(cmd func(string, ...string) *exec.Cmd) {\n\tc.command = cmd\n}\n\n// RunCommandWithOutput wrapper around commands returning their output and error\nfunc (c *OSCommand) RunCommandWithOutput(command string) (string, error) {\n\tcmd := c.ExecutableFromString(command)\n\tbefore := time.Now()\n\toutput, err := sanitisedCommandOutput(cmd.Output())\n\tc.Log.Warn(fmt.Sprintf(\"'%s': %s\", command, time.Since(before)))\n\treturn output, err\n}\n\n// RunCommandWithOutput wrapper around commands returning their output and error\nfunc (c *OSCommand) RunCommandWithOutputContext(ctx context.Context, command string) (string, error) {\n\tcmd := c.ExecutableFromStringContext(ctx, command)\n\tbefore := time.Now()\n\toutput, err := sanitisedCommandOutput(cmd.Output())\n\tc.Log.Warn(fmt.Sprintf(\"'%s': %s\", command, time.Since(before)))\n\treturn output, err\n}\n\n// RunExecutableWithOutput runs an executable file and returns its output\nfunc (c *OSCommand) RunExecutableWithOutput(cmd *exec.Cmd) (string, error) {\n\treturn sanitisedCommandOutput(cmd.CombinedOutput())\n}\n\n// RunExecutable runs an executable file and returns an error if there was one\nfunc (c *OSCommand) RunExecutable(cmd *exec.Cmd) error {\n\t_, err := c.RunExecutableWithOutput(cmd)\n\treturn err\n}\n\n// ExecutableFromString takes a string like `docker ps -a` and returns an executable command for it\nfunc (c *OSCommand) ExecutableFromString(commandStr string) *exec.Cmd {\n\tsplitCmd := str.ToArgv(commandStr)\n\treturn c.NewCmd(splitCmd[0], splitCmd[1:]...)\n}\n\n// Same as ExecutableFromString but cancellable via a context\nfunc (c *OSCommand) ExecutableFromStringContext(ctx context.Context, commandStr string) *exec.Cmd {\n\tsplitCmd := str.ToArgv(commandStr)\n\treturn exec.CommandContext(ctx, splitCmd[0], splitCmd[1:]...)\n}\n\nfunc (c *OSCommand) NewCmd(cmdName string, commandArgs ...string) *exec.Cmd {\n\tcmd := c.command(cmdName, commandArgs...)\n\tcmd.Env = os.Environ()\n\treturn cmd\n}\n\nfunc (c *OSCommand) NewCommandStringWithShell(commandStr string) string {\n\tvar quotedCommand string\n\t// Windows does not seem to like quotes around the command\n\tif c.Platform.os == \"windows\" {\n\t\tquotedCommand = strings.NewReplacer(\n\t\t\t\"^\", \"^^\",\n\t\t\t\"&\", \"^&\",\n\t\t\t\"|\", \"^|\",\n\t\t\t\"<\", \"^<\",\n\t\t\t\">\", \"^>\",\n\t\t\t\"%\", \"^%\",\n\t\t).Replace(commandStr)\n\t} else {\n\t\tquotedCommand = c.Quote(commandStr)\n\t}\n\n\treturn fmt.Sprintf(\"%s %s %s\", c.Platform.shell, c.Platform.shellArg, quotedCommand)\n}\n\n// RunCommand runs a command and just returns the error\nfunc (c *OSCommand) RunCommand(command string) error {\n\t_, err := c.RunCommandWithOutput(command)\n\treturn err\n}\n\n// FileType tells us if the file is a file, directory or other\nfunc (c *OSCommand) FileType(path string) string {\n\tfileInfo, err := os.Stat(path)\n\tif err != nil {\n\t\treturn \"other\"\n\t}\n\tif fileInfo.IsDir() {\n\t\treturn \"directory\"\n\t}\n\treturn \"file\"\n}\n\n\nfunc sanitisedCommandOutput(output []byte, err error) (string, error) {\n\toutputString := string(output)\n\tif err != nil {\n\t\t// errors like 'exit status 1' are not very useful so we'll create an error\n\t\t// from stderr if we got an ExitError\n\t\texitError, ok := err.(*exec.ExitError)\n\t\tif ok {\n\t\t\treturn outputString, errors.New(string(exitError.Stderr))\n\t\t}\n\t\treturn \"\", WrapError(err)\n\t}\n\treturn outputString, nil\n}\n\n// OpenFile opens a file with the given\nfunc (c *OSCommand) OpenFile(filename string) error {\n\tcommandTemplate := c.Config.UserConfig.OS.OpenCommand\n\ttemplateValues := map[string]string{\n\t\t\"filename\": c.Quote(filename),\n\t}\n\n\tcommand := utils.ResolvePlaceholderString(commandTemplate, templateValues)\n\terr := c.RunCommand(command)\n\treturn err\n}\n\n// OpenLink opens a file with the given\nfunc (c *OSCommand) OpenLink(link string) error {\n\tcommandTemplate := c.Config.UserConfig.OS.OpenLinkCommand\n\ttemplateValues := map[string]string{\n\t\t\"link\": c.Quote(link),\n\t}\n\n\tcommand := utils.ResolvePlaceholderString(commandTemplate, templateValues)\n\terr := c.RunCommand(command)\n\treturn err\n}\n\n// EditFile opens a file in a subprocess using whatever editor is available,\n// falling back to core.editor, VISUAL, EDITOR, then vi\nfunc (c *OSCommand) EditFile(filename string) (*exec.Cmd, error) {\n\teditor := c.getenv(\"VISUAL\")\n\tif editor == \"\" {\n\t\teditor = c.getenv(\"EDITOR\")\n\t}\n\tif editor == \"\" {\n\t\tif err := c.RunCommand(\"which vi\"); err == nil {\n\t\t\teditor = \"vi\"\n\t\t}\n\t}\n\tif editor == \"\" {\n\t\treturn nil, errors.New(\"No editor defined in $VISUAL or $EDITOR\")\n\t}\n\n\treturn c.NewCmd(editor, filename), nil\n}\n\n// Quote wraps a message in platform-specific quotation marks\nfunc (c *OSCommand) Quote(message string) string {\n\tvar quote string\n\tif c.Platform.os == \"windows\" {\n\t\tquote = `\\\"`\n\t\tmessage = strings.NewReplacer(\n\t\t\t`\"`, `\"'\"'\"`,\n\t\t\t`\\\"`, `\\\\\"`,\n\t\t).Replace(message)\n\t} else {\n\t\tquote = `\"`\n\t\tmessage = strings.NewReplacer(\n\t\t\t`\\`, `\\\\`,\n\t\t\t`\"`, `\\\"`,\n\t\t\t`$`, `\\$`,\n\t\t\t\"`\", \"\\\\`\",\n\t\t).Replace(message)\n\t}\n\treturn quote + message + quote\n}\n\n// Unquote removes wrapping quotations marks if they are present\n// this is needed for removing quotes from staged filenames with spaces\nfunc (c *OSCommand) Unquote(message string) string {\n\treturn strings.Replace(message, `\"`, \"\", -1)\n}\n\n// AppendLineToFile adds a new line in file\nfunc (c *OSCommand) AppendLineToFile(filename, line string) error {\n\tf, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600)\n\tif err != nil {\n\t\treturn WrapError(err)\n\t}\n\tdefer f.Close()\n\n\t_, err = f.WriteString(\"\\n\" + line)\n\tif err != nil {\n\t\treturn WrapError(err)\n\t}\n\treturn nil\n}\n\n// CreateTempFile writes a string to a new temp file and returns the file's name\nfunc (c *OSCommand) CreateTempFile(filename, content string) (string, error) {\n\ttmpfile, err := os.CreateTemp(\"\", filename)\n\tif err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\n\tif _, err := tmpfile.WriteString(content); err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\tif err := tmpfile.Close(); err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\n\treturn tmpfile.Name(), nil\n}\n\n// Remove removes a file or directory at the specified path\nfunc (c *OSCommand) Remove(filename string) error {\n\terr := os.RemoveAll(filename)\n\treturn WrapError(err)\n}\n\n// FileExists checks whether a file exists at the specified path\nfunc (c *OSCommand) FileExists(path string) (bool, error) {\n\tif _, err := os.Stat(path); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, err\n\t}\n\treturn true, nil\n}\n\n// RunPreparedCommand takes a pointer to an exec.Cmd and runs it\n// this is useful if you need to give your command some environment variables\n// before running it\nfunc (c *OSCommand) RunPreparedCommand(cmd *exec.Cmd) error {\n\tout, err := cmd.CombinedOutput()\n\toutString := string(out)\n\tc.Log.Info(outString)\n\tif err != nil {\n\t\tif len(outString) == 0 {\n\t\t\treturn err\n\t\t}\n\t\treturn errors.New(outString)\n\t}\n\treturn nil\n}\n\n// GetLazydockerPath returns the path of the currently executed file\nfunc (c *OSCommand) GetLazydockerPath() string {\n\tex, err := os.Executable() // get the executable path for docker to use\n\tif err != nil {\n\t\tex = os.Args[0] // fallback to the first call argument if needed\n\t}\n\treturn filepath.ToSlash(ex)\n}\n\n// RunCustomCommand returns the pointer to a custom command\nfunc (c *OSCommand) RunCustomCommand(command string) *exec.Cmd {\n\treturn c.NewCmd(c.Platform.shell, c.Platform.shellArg, command)\n}\n\n// PipeCommands runs a heap of commands and pipes their inputs/outputs together like A | B | C\nfunc (c *OSCommand) PipeCommands(commandStrings ...string) error {\n\tcmds := make([]*exec.Cmd, len(commandStrings))\n\n\tfor i, str := range commandStrings {\n\t\tcmds[i] = c.ExecutableFromString(str)\n\t}\n\n\tfor i := 0; i < len(cmds)-1; i++ {\n\t\tstdout, err := cmds[i].StdoutPipe()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tcmds[i+1].Stdin = stdout\n\t}\n\n\t// keeping this here in case I adapt this code for some other purpose in the future\n\t// cmds[len(cmds)-1].Stdout = os.Stdout\n\n\tfinalErrors := []string{}\n\n\twg := sync.WaitGroup{}\n\twg.Add(len(cmds))\n\n\tfor _, cmd := range cmds {\n\t\tcurrentCmd := cmd\n\t\tgo func() {\n\t\t\tstderr, err := currentCmd.StderrPipe()\n\t\t\tif err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\tif err := currentCmd.Start(); err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\tif b, err := io.ReadAll(stderr); err == nil {\n\t\t\t\tif len(b) > 0 {\n\t\t\t\t\tfinalErrors = append(finalErrors, string(b))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif err := currentCmd.Wait(); err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\twg.Wait()\n\n\tif len(finalErrors) > 0 {\n\t\treturn errors.New(strings.Join(finalErrors, \"\\n\"))\n\t}\n\treturn nil\n}\n\n// Kill kills a process. If the process has Setpgid == true, then we have anticipated that it might spawn its own child processes, so we've given it a process group ID (PGID) equal to its process id (PID) and given its child processes will inherit the PGID, we can kill that group, rather than killing the process itself.\nfunc (c *OSCommand) Kill(cmd *exec.Cmd) error {\n\treturn kill.Kill(cmd)\n}\n\n// PrepareForChildren sets Setpgid to true on the cmd, so that when we run it as a subprocess, we can kill its group rather than the process itself. This is because some commands, like `docker-compose logs` spawn multiple children processes, and killing the parent process isn't sufficient for killing those child processes. We set the group id here, and then in subprocess.go we check if the group id is set and if so, we kill the whole group rather than just the one process.\nfunc (c *OSCommand) PrepareForChildren(cmd *exec.Cmd) {\n\tkill.PrepareForChildren(cmd)\n}\n\n// Path: pkg/log/log.go\npackage log\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// NewLogger returns a new logger\nfunc NewLogger(config *config.AppConfig, rollrusHook string) *logrus.Entry {\n\tvar log *logrus.Logger\n\tif config.Debug || os.Getenv(\"DEBUG\") == \"TRUE\" {\n\t\tlog = newDevelopmentLogger(config)\n\t} else {\n\t\tlog = newProductionLogger()\n\t}\n\n\t// highly recommended: tail -f development.log | humanlog\n\t// https://github.com/aybabtme/humanlog\n\tlog.Formatter = &logrus.JSONFormatter{}\n\n\treturn log.WithFields(logrus.Fields{\n\t\t\"debug\":     config.Debug,\n\t\t\"version\":   config.Version,\n\t\t\"commit\":    config.Commit,\n\t\t\"buildDate\": config.BuildDate,\n\t})\n}\n\nfunc getLogLevel() logrus.Level {\n\tstrLevel := os.Getenv(\"LOG_LEVEL\")\n\tlevel, err := logrus.ParseLevel(strLevel)\n\tif err != nil {\n\t\treturn logrus.DebugLevel\n\t}\n\treturn level\n}\n\nfunc newDevelopmentLogger(config *config.AppConfig) *logrus.Logger {\n\tlog := logrus.New()\n\tlog.SetLevel(getLogLevel())\n\tfile, err := os.OpenFile(filepath.Join(config.ConfigDir, \"development.log\"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o666)\n\tif err != nil {\n\t\tfmt.Println(\"unable to log to file\")\n\t\tos.Exit(1)\n\t}\n\tlog.SetOutput(file)\n\treturn log\n}\n\nfunc newProductionLogger() *logrus.Logger {\n\tlog := logrus.New()\n\tlog.Out = io.Discard\n\tlog.SetLevel(logrus.ErrorLevel)\n\treturn log\n}\n\n// Path: pkg/i18n/i18n.go\npackage i18n\n\nimport (\n\t\"strings\"\n\n\t\"github.com/imdario/mergo\"\n\n\t\"github.com/cloudfoundry/jibber_jabber\"\n\t\"github.com/go-errors/errors\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Localizer will translate a message into the user's language\ntype Localizer struct {\n\tLog *logrus.Entry\n\tS   TranslationSet\n}\n\nfunc NewTranslationSetFromConfig(log *logrus.Entry, configLanguage string) (*TranslationSet, error) {\n\tif configLanguage == \"auto\" {\n\t\tlanguage := detectLanguage(jibber_jabber.DetectLanguage)\n\t\treturn NewTranslationSet(log, language), nil\n\t}\n\n\tfor key := range GetTranslationSets() {\n\t\tif key == configLanguage {\n\t\t\treturn NewTranslationSet(log, configLanguage), nil\n\t\t}\n\t}\n\n\treturn NewTranslationSet(log, \"en\"), errors.New(\"Language not found: \" + configLanguage)\n}\n\nfunc NewTranslationSet(log *logrus.Entry, language string) *TranslationSet {\n\tlog.Info(\"language: \" + language)\n\n\tbaseSet := englishSet()\n\n\tfor languageCode, translationSet := range GetTranslationSets() {\n\t\tif strings.HasPrefix(language, languageCode) {\n\t\t\t_ = mergo.Merge(&baseSet, translationSet, mergo.WithOverride)\n\t\t}\n\t}\n\n\treturn &baseSet\n}\n\n// GetTranslationSets gets all the translation sets, keyed by language code\nfunc GetTranslationSets() map[string]TranslationSet {\n\treturn map[string]TranslationSet{\n\t\t\"pl\": polishSet(),\n\t\t\"nl\": dutchSet(),\n\t\t\"de\": germanSet(),\n\t\t\"tr\": turkishSet(),\n\t\t\"en\": englishSet(),\n\t\t\"fr\": frenchSet(),\n\t\t\"zh\": chineseSet(),\n\t\t\"es\": spanishSet(),\n\t\t\"pt\": portugueseSet(),\n\t}\n}\n\n// detectLanguage extracts user language from environment\nfunc detectLanguage(langDetector func() (string, error)) string {\n\tif userLang, err := langDetector(); err == nil {\n\t\treturn userLang\n\t}\n\n\treturn \"C\"\n}\n\n// Path: pkg/tasks/tasks.go\npackage tasks\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\ntype TaskManager struct {\n\tcurrentTask  *Task\n\twaitingMutex deadlock.Mutex\n\ttaskIDMutex  deadlock.Mutex\n\tLog          *logrus.Entry\n\tTr           *i18n.TranslationSet\n\tnewTaskId    int\n}\n\ntype Task struct {\n\tctx           context.Context\n\tcancel        context.CancelFunc\n\tstopped       bool\n\tstopMutex     deadlock.Mutex\n\tnotifyStopped chan struct{}\n\tLog           *logrus.Entry\n\tf             func(ctx context.Context)\n}\n\ntype TaskFunc func(ctx context.Context)\n\nfunc NewTaskManager(log *logrus.Entry, translationSet *i18n.TranslationSet) *TaskManager {\n\treturn &TaskManager{Log: log, Tr: translationSet}\n}\n\n// Close closes the task manager, killing whatever task may currently be running\nfunc (t *TaskManager) Close() {\n\tif t.currentTask == nil {\n\t\treturn\n\t}\n\n\tc := make(chan struct{}, 1)\n\n\tgo func() {\n\t\tt.currentTask.Stop()\n\t\tc <- struct{}{}\n\t}()\n\n\tselect {\n\tcase <-c:\n\t\treturn\n\tcase <-time.After(3 * time.Second):\n\t\tfmt.Println(t.Tr.CannotKillChildError)\n\t}\n}\n\nfunc (t *TaskManager) NewTask(f func(ctx context.Context)) error {\n\tgo func() {\n\t\tt.taskIDMutex.Lock()\n\t\tt.newTaskId++\n\t\ttaskID := t.newTaskId\n\t\tt.taskIDMutex.Unlock()\n\n\t\tt.waitingMutex.Lock()\n\t\tdefer t.waitingMutex.Unlock()\n\t\tif taskID < t.newTaskId {\n\t\t\treturn\n\t\t}\n\n\t\tctx, cancel := context.WithCancel(context.Background())\n\t\tnotifyStopped := make(chan struct{})\n\n\t\tif t.currentTask != nil {\n\t\t\tt.Log.Info(\"asking task to stop\")\n\t\t\tt.currentTask.Stop()\n\t\t\tt.Log.Info(\"task stopped\")\n\t\t}\n\n\t\tt.currentTask = &Task{\n\t\t\tctx:           ctx,\n\t\t\tcancel:        cancel,\n\t\t\tnotifyStopped: notifyStopped,\n\t\t\tLog:           t.Log,\n\t\t\tf:             f,\n\t\t}\n\n\t\tgo func() {\n\t\t\tf(ctx)\n\t\t\tt.Log.Info(\"returned from function, closing notifyStopped\")\n\t\t\tclose(notifyStopped)\n\t\t}()\n\t}()\n\n\treturn nil\n}\n\nfunc (t *Task) Stop() {\n\tt.stopMutex.Lock()\n\tdefer t.stopMutex.Unlock()\n\tif t.stopped {\n\t\treturn\n\t}\n\n\tt.cancel()\n\tt.Log.Info(\"closed stop channel, waiting for notifyStopped message\")\n\t<-t.notifyStopped\n\tt.Log.Info(\"received notifystopped message\")\n\tt.stopped = true\n}\n\n// NewTickerTask is a convenience function for making a new task that repeats some action once per e.g. second\n// the before function gets called after the lock is obtained, but before the ticker starts.\n// if you handle a message on the stop channel in f() you need to send a message on the notifyStopped channel because returning is not sufficient. Here, unlike in a regular task, simply returning means we're now going to wait till the next tick to run again.\nfunc (t *TaskManager) NewTickerTask(duration time.Duration, before func(ctx context.Context), f func(ctx context.Context, notifyStopped chan struct{})) error {\n\tnotifyStopped := make(chan struct{}, 10)\n\n\treturn t.NewTask(func(ctx context.Context) {\n\t\tif before != nil {\n\t\t\tbefore(ctx)\n\t\t}\n\t\ttickChan := time.NewTicker(duration)\n\t\tdefer tickChan.Stop()\n\t\t// calling f first so that we're not waiting for the first tick\n\t\tf(ctx, notifyStopped)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-notifyStopped:\n\t\t\t\tt.Log.Info(\"exiting ticker task due to notifyStopped channel\")\n\t\t\t\treturn\n\t\t\tcase <-ctx.Done():\n\t\t\t\tt.Log.Info(\"exiting ticker task due to stopped cahnnel\")\n\t\t\t\treturn\n\t\t\tcase <-tickChan.C:\n\t\t\t\tt.Log.Info(\"running ticker task again\")\n\t\t\t\tf(ctx, notifyStopped)\n\t\t\t}\n\t\t}\n\t})\n}\n\n// Path: pkg/gui/types/types.go\npackage types\n\ntype MenuItem struct {\n\tLabel string\n\n\t// alternative to Label. Allows specifying columns which will be auto-aligned\n\tLabelColumns []string\n\n\tOnPress func() error\n\n\t// Only applies when Label is used\n\tOpensMenu bool\n}\n\n// Path: pkg/commands/container.go\npackage commands\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os/exec\"\n\t\"strings\"\n\n\t\"github.com/docker/docker/api/types/container\"\n\t\"github.com/sasha-s/go-deadlock\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/docker/docker/api/types/filters\"\n\t\"github.com/docker/docker/client\"\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/sirupsen/logrus\"\n\t\"golang.org/x/xerrors\"\n)\n\n// Container : A docker Container\ntype Container struct {\n\tName            string\n\tServiceName     string\n\tContainerNumber string // might make this an int in the future if need be\n\n\t// OneOff tells us if the container is just a job container or is actually bound to the service\n\tOneOff          bool\n\tProjectName     string\n\tID              string\n\tContainer       dockerTypes.Container\n\tClient          *client.Client\n\tOSCommand       *OSCommand\n\tLog             *logrus.Entry\n\tStatHistory     []*RecordedStats\n\tDetails         dockerTypes.ContainerJSON\n\tMonitoringStats bool\n\tDockerCommand   LimitedDockerCommand\n\tTr              *i18n.TranslationSet\n\n\tStatsMutex deadlock.Mutex\n}\n\n// Remove removes the container\nfunc (c *Container) Remove(options dockerTypes.ContainerRemoveOptions) error {\n\tc.Log.Warn(fmt.Sprintf(\"removing container %s\", c.Name))\n\tif err := c.Client.ContainerRemove(context.Background(), c.ID, options); err != nil {\n\t\tif strings.Contains(err.Error(), \"Stop the container before attempting removal or force remove\") {\n\t\t\treturn ComplexError{\n\t\t\t\tCode:    MustStopContainer,\n\t\t\t\tMessage: err.Error(),\n\t\t\t\tframe:   xerrors.Caller(1),\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// Stop stops the container\nfunc (c *Container) Stop() error {\n\tc.Log.Warn(fmt.Sprintf(\"stopping container %s\", c.Name))\n\treturn c.Client.ContainerStop(context.Background(), c.ID, nil)\n}\n\n// Pause pauses the container\nfunc (c *Container) Pause() error {\n\tc.Log.Warn(fmt.Sprintf(\"pausing container %s\", c.Name))\n\treturn c.Client.ContainerPause(context.Background(), c.ID)\n}\n\n// Unpause unpauses the container\nfunc (c *Container) Unpause() error {\n\tc.Log.Warn(fmt.Sprintf(\"unpausing container %s\", c.Name))\n\treturn c.Client.ContainerUnpause(context.Background(), c.ID)\n}\n\n// Restart restarts the container\nfunc (c *Container) Restart() error {\n\tc.Log.Warn(fmt.Sprintf(\"restarting container %s\", c.Name))\n\treturn c.Client.ContainerRestart(context.Background(), c.ID, nil)\n}\n\n// Attach attaches the container\nfunc (c *Container) Attach() (*exec.Cmd, error) {\n\tif !c.DetailsLoaded() {\n\t\treturn nil, errors.New(c.Tr.WaitingForContainerInfo)\n\t}\n\n\t// verify that we can in fact attach to this container\n\tif !c.Details.Config.OpenStdin {\n\t\treturn nil, errors.New(c.Tr.UnattachableContainerError)\n\t}\n\n\tif c.Container.State == \"exited\" {\n\t\treturn nil, errors.New(c.Tr.CannotAttachStoppedContainerError)\n\t}\n\n\tc.Log.Warn(fmt.Sprintf(\"attaching to container %s\", c.Name))\n\t// TODO: use SDK\n\tcmd := c.OSCommand.NewCmd(\"docker\", \"attach\", \"--sig-proxy=false\", c.ID)\n\treturn cmd, nil\n}\n\n// Top returns process information\nfunc (c *Container) Top(ctx context.Context) (container.ContainerTopOKBody, error) {\n\tdetail, err := c.Inspect()\n\tif err != nil {\n\t\treturn container.ContainerTopOKBody{}, err\n\t}\n\n\t// check container status\n\tif !detail.State.Running {\n\t\treturn container.ContainerTopOKBody{}, errors.New(\"container is not running\")\n\t}\n\n\treturn c.Client.ContainerTop(ctx, c.ID, []string{})\n}\n\n// PruneContainers prunes containers\nfunc (c *DockerCommand) PruneContainers() error {\n\t_, err := c.Client.ContainersPrune(context.Background(), filters.Args{})\n\treturn err\n}\n\n// Inspect returns details about the container\nfunc (c *Container) Inspect() (dockerTypes.ContainerJSON, error) {\n\treturn c.Client.ContainerInspect(context.Background(), c.ID)\n}\n\n// RenderTop returns details about the container\nfunc (c *Container) RenderTop(ctx context.Context) (string, error) {\n\tresult, err := c.Top(ctx)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn utils.RenderTable(append([][]string{result.Titles}, result.Processes...))\n}\n\n// DetailsLoaded tells us whether we have yet loaded the details for a container. Because this is an asynchronous operation, sometimes we have the container before we have its details.\nfunc (c *Container) DetailsLoaded() bool {\n\treturn c.Details.ContainerJSONBase != nil\n}\n\n// Path: pkg/gui/panels.go\npackage gui\n\nimport \"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\nfunc (gui *Gui) intoInterface() panels.IGui {\n\treturn gui\n}\n\n// Path: pkg/gui/gui.go\npackage gui\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\n\t\"github.com/go-errors/errors\"\n\n\tthrottle \"github.com/boz/go-throttle\"\n\t\"github.com/jesseduffield/gocui\"\n\tlcUtils \"github.com/jesseduffield/lazycore/pkg/utils\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// OverlappingEdges determines if panel edges overlap\nvar OverlappingEdges = false\n\n// Gui wraps the gocui Gui object which handles rendering and events\ntype Gui struct {\n\tg             *gocui.Gui\n\tLog           *logrus.Entry\n\tDockerCommand *commands.DockerCommand\n\tOSCommand     *commands.OSCommand\n\tState         guiState\n\tConfig        *config.AppConfig\n\tTr            *i18n.TranslationSet\n\tstatusManager *statusManager\n\ttaskManager   *tasks.TaskManager\n\tErrorChan     chan error\n\tViews         Views\n\n\t// if we've suspended the gui (e.g. because we've switched to a subprocess)\n\t// we typically want to pause some things that are running like background\n\t// file refreshes\n\tPauseBackgroundThreads bool\n\n\tMutexes\n\n\tPanels Panels\n}\n\ntype Panels struct {\n\tProjects   *panels.SideListPanel[*commands.Project]\n\tServices   *panels.SideListPanel[*commands.Service]\n\tContainers *panels.SideListPanel[*commands.Container]\n\tImages     *panels.SideListPanel[*commands.Image]\n\tVolumes    *panels.SideListPanel[*commands.Volume]\n\tNetworks   *panels.SideListPanel[*commands.Network]\n\tMenu       *panels.SideListPanel[*types.MenuItem]\n}\n\ntype Mutexes struct {\n\tSubprocessMutex deadlock.Mutex\n\tViewStackMutex  deadlock.Mutex\n}\n\ntype mainPanelState struct {\n\t// ObjectKey tells us what context we are in. For example, if we are looking at the logs of a particular service in the services panel this key might be 'services-<service id>-logs'. The key is made so that if something changes which might require us to re-run the logs command or run a different command, the key will be different, and we'll then know to do whatever is required. Object key probably isn't the best name for this but Context is already used to refer to tabs. Maybe I should just call them tabs.\n\tObjectKey string\n}\n\ntype panelStates struct {\n\tMain *mainPanelState\n}\n\ntype guiState struct {\n\t// the names of views in the current focus stack (last item is the current view)\n\tViewStack        []string\n\tPlatform         commands.Platform\n\tPanels           *panelStates\n\tSubProcessOutput string\n\tStats            map[string]commands.ContainerStats\n\n\t// if true, we show containers with an 'exited' status in the containers panel\n\tShowExitedContainers bool\n\n\tScreenMode WindowMaximisation\n\n\t// Maintains the state of manual filtering i.e. typing in a substring\n\t// to filter on in the current panel.\n\tFilter filterState\n}\n\ntype filterState struct {\n\t// If true then we're either currently inside the filter view\n\t// or we've committed the filter and we're back in the list view\n\tactive bool\n\t// The panel that we're filtering.\n\tpanel panels.ISideListPanel\n\t// The string that we're filtering on\n\tneedle string\n}\n\n// screen sizing determines how much space your selected window takes up (window\n// as in panel, not your terminal's window). Sometimes you want a bit more space\n// to see the contents of a panel, and this keeps track of how much maximisation\n// you've set\ntype WindowMaximisation int\n\nconst (\n\tSCREEN_NORMAL WindowMaximisation = iota\n\tSCREEN_HALF\n\tSCREEN_FULL\n)\n\nfunc getScreenMode(config *config.AppConfig) WindowMaximisation {\n\tswitch config.UserConfig.Gui.ScreenMode {\n\tcase \"normal\":\n\t\treturn SCREEN_NORMAL\n\tcase \"half\":\n\t\treturn SCREEN_HALF\n\tcase \"fullscreen\":\n\t\treturn SCREEN_FULL\n\tdefault:\n\t\treturn SCREEN_NORMAL\n\t}\n}\n\n// NewGui builds a new gui handler\nfunc NewGui(log *logrus.Entry, dockerCommand *commands.DockerCommand, oSCommand *commands.OSCommand, tr *i18n.TranslationSet, config *config.AppConfig, errorChan chan error) (*Gui, error) {\n\tinitialState := guiState{\n\t\tPlatform: *oSCommand.Platform,\n\t\tPanels: &panelStates{\n\t\t\tMain: &mainPanelState{\n\t\t\t\tObjectKey: \"\",\n\t\t\t},\n\t\t},\n\t\tViewStack: []string{},\n\n\t\tShowExitedContainers: true,\n\t\tScreenMode:           getScreenMode(config),\n\t}\n\n\tgui := &Gui{\n\t\tLog:           log,\n\t\tDockerCommand: dockerCommand,\n\t\tOSCommand:     oSCommand,\n\t\tState:         initialState,\n\t\tConfig:        config,\n\t\tTr:            tr,\n\t\tstatusManager: &statusManager{},\n\t\ttaskManager:   tasks.NewTaskManager(log, tr),\n\t\tErrorChan:     errorChan,\n\t}\n\n\tdeadlock.Opts.Disable = !gui.Config.Debug\n\tdeadlock.Opts.DeadlockTimeout = 10 * time.Second\n\n\treturn gui, nil\n}\n\nfunc (gui *Gui) renderGlobalOptions() error {\n\treturn gui.renderOptionsMap(map[string]string{\n\t\t\"PgUp/PgDn\": gui.Tr.Scroll,\n\t\t\"\u2190 \u2192 \u2191 \u2193\":   gui.Tr.Navigate,\n\t\t\"q\":         gui.Tr.Quit,\n\t\t\"b\":         gui.Tr.ViewBulkCommands,\n\t\t\"x\":         gui.Tr.Menu,\n\t})\n}\n\nfunc (gui *Gui) goEvery(interval time.Duration, function func() error) {\n\t_ = function() // time.Tick doesn't run immediately so we'll do that here // TODO: maybe change\n\tgo func() {\n\t\tticker := time.NewTicker(interval)\n\t\tdefer ticker.Stop()\n\t\tfor range ticker.C {\n\t\t\tif gui.PauseBackgroundThreads {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t_ = function()\n\t\t}\n\t}()\n}\n\n// Run setup the gui with keybindings and start the mainloop\nfunc (gui *Gui) Run() error {\n\t// closing our task manager which in turn closes the current task if there is any, so we aren't leaving processes lying around after closing lazydocker\n\tdefer gui.taskManager.Close()\n\n\tg, err := gocui.NewGui(gocui.OutputTrue, OverlappingEdges, gocui.NORMAL, false, map[rune]string{})\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer g.Close()\n\n\t// forgive the double-negative, this is because of my yaml `omitempty` woes\n\tif !gui.Config.UserConfig.Gui.IgnoreMouseEvents {\n\t\tg.Mouse = true\n\t}\n\n\tgui.g = g // TODO: always use gui.g rather than passing g around everywhere\n\n\t// if the deadlock package wants to report a deadlock, we first need to\n\t// close the gui so that we can actually read what it prints.\n\tdeadlock.Opts.LogBuf = lcUtils.NewOnceWriter(os.Stderr, func() {\n\t\tgui.g.Close()\n\t})\n\n\tif err := gui.SetColorScheme(); err != nil {\n\t\treturn err\n\t}\n\n\tthrottledRefresh := throttle.ThrottleFunc(time.Millisecond*50, true, gui.refresh)\n\tdefer throttledRefresh.Stop()\n\n\tgo func() {\n\t\tfor err := range gui.ErrorChan {\n\t\t\tif err == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif strings.Contains(err.Error(), \"No such container\") {\n\t\t\t\t// this happens all the time when e.g. restarting containers so we won't worry about it\n\t\t\t\tgui.Log.Warn(err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_ = gui.createErrorPanel(err.Error())\n\t\t}\n\t}()\n\n\tg.SetManager(gocui.ManagerFunc(gui.layout), gocui.ManagerFunc(gui.getFocusLayout()))\n\n\tif err := gui.createAllViews(); err != nil {\n\t\treturn err\n\t}\n\tif err := gui.setInitialViewContent(); err != nil {\n\t\treturn err\n\t}\n\n\t// TODO: see if we can avoid the circular dependency\n\tgui.setPanels()\n\n\tif err = gui.keybindings(g); err != nil {\n\t\treturn err\n\t}\n\n\tif gui.g.CurrentView() == nil {\n\t\tviewName := gui.initiallyFocusedViewName()\n\t\tview, err := gui.g.View(viewName)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif err := gui.switchFocus(view); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tctx, finish := context.WithCancel(context.Background())\n\tdefer finish()\n\n\tgo gui.listenForEvents(ctx, throttledRefresh.Trigger)\n\tgo gui.monitorContainerStats(ctx)\n\n\tgo func() {\n\t\tthrottledRefresh.Trigger()\n\n\t\tgui.goEvery(time.Millisecond*30, gui.reRenderMain)\n\t\tgui.goEvery(time.Millisecond*1000, gui.updateContainerDetails)\n\t\tgui.goEvery(time.Millisecond*1000, gui.checkForContextChange)\n\t\t// we need to regularly re-render these because their stats will be changed in the background\n\t\tgui.goEvery(time.Millisecond*1000, gui.renderContainersAndServices)\n\t}()\n\n\terr = g.MainLoop()\n\tif err == gocui.ErrQuit {\n\t\treturn nil\n\t}\n\treturn err\n}\n\nfunc (gui *Gui) setPanels() {\n\tgui.Panels = Panels{\n\t\tProjects:   gui.getProjectPanel(),\n\t\tServices:   gui.getServicesPanel(),\n\t\tContainers: gui.getContainersPanel(),\n\t\tImages:     gui.getImagesPanel(),\n\t\tVolumes:    gui.getVolumesPanel(),\n\t\tNetworks:   gui.getNetworksPanel(),\n\t\tMenu:       gui.getMenuPanel(),\n\t}\n}\n\nfunc (gui *Gui) updateContainerDetails() error {\n\treturn gui.DockerCommand.UpdateContainerDetails(gui.Panels.Containers.List.GetAllItems())\n}\n\nfunc (gui *Gui) refresh() {\n\tgo func() {\n\t\tif err := gui.refreshProject(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.refreshContainersAndServices(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.reloadVolumes(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.reloadNetworks(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n\tgo func() {\n\t\tif err := gui.reloadImages(); err != nil {\n\t\t\tgui.Log.Error(err)\n\t\t}\n\t}()\n}\n\nfunc (gui *Gui) listenForEvents(ctx context.Context, refresh func()) {\n\terrorCount := 0\n\n\tonError := func(err error) {\n\t\tif err != nil {\n\t\t\tgui.ErrorChan <- errors.Errorf(\"Docker event stream returned error: %s\\nRetry count: %d\", err.Error(), errorCount)\n\t\t}\n\t\terrorCount++\n\t\ttime.Sleep(time.Second * 2)\n\t}\n\nouter:\n\tfor {\n\t\tmessageChan, errChan := gui.DockerCommand.Client.Events(context.Background(), dockerTypes.EventsOptions{})\n\n\t\tif errorCount > 0 {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase err := <-errChan:\n\t\t\t\tonError(err)\n\t\t\t\tcontinue outer\n\t\t\tdefault:\n\t\t\t\t// If we're here then we lost connection to docker and we just got it back.\n...\n", "needle_token_start": 5258, "needle_token_end": 5399, "code_context_ntokens": 15149, "output": ["```go\nfunc sanitisedCommandOutput(output []byte, err error) (string, error) {\n\toutputString := string(output)\n\tif err != nil {\n\t\t// errors like 'exit status 1' are not very useful so we'll create an error\n\t\t// from stderr if we got an ExitError\n\t\texitError, ok := err.(*exec.ExitError)\n\t\tif ok {\n\t\t\treturn outputString, errors.New(string(exitError.Stderr))\n\t\t}\n\t\treturn \"\", WrapError(err)\n\t}\n\treturn outputString, nil\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "MultiColoredString", "language": "go", "path": "pkg/utils/utils.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To create a string that is styled with multiple color attributes, allowing for a visually distinct representation in supported text viewers or terminals.\n2. **Input**: A string to be colored and a variable number of color attributes that determine the styling of the string.\n3. **Output**: A string that has been styled with the specified color attributes.\n4. **Procedure**: The function initializes a new color object with the provided color attributes. It then applies these color settings directly to the input string, resulting in a string that is styled according to the specified attributes.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/commands/errors.go\npackage commands\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/go-errors/errors\"\n\t\"golang.org/x/xerrors\"\n)\n\nconst (\n\t// MustStopContainer tells us that we must stop the container before removing it\n\tMustStopContainer = iota\n)\n\n// WrapError wraps an error for the sake of showing a stack trace at the top level\n// the go-errors package, for some reason, does not return nil when you try to wrap\n// a non-error, so we're just doing it here\nfunc WrapError(err error) error {\n\tif err == nil {\n\t\treturn err\n\t}\n\n\treturn errors.Wrap(err, 0)\n}\n\n// ComplexError an error which carries a code so that calling code has an easier job to do\n// adapted from https://medium.com/yakka/better-go-error-handling-with-xerrors-1987650e0c79\ntype ComplexError struct {\n\tMessage string\n\tCode    int\n\tframe   xerrors.Frame\n}\n\n// FormatError is a function\nfunc (ce ComplexError) FormatError(p xerrors.Printer) error {\n\tp.Printf(\"%d %s\", ce.Code, ce.Message)\n\tce.frame.Format(p)\n\treturn nil\n}\n\n// Format is a function\nfunc (ce ComplexError) Format(f fmt.State, c rune) {\n\txerrors.FormatError(ce, f, c)\n}\n\nfunc (ce ComplexError) Error() string {\n\treturn fmt.Sprint(ce)\n}\n\n// HasErrorCode is a function\nfunc HasErrorCode(err error, code int) bool {\n\tvar originalErr ComplexError\n\tif xerrors.As(err, &originalErr) {\n\t\treturn originalErr.Code == MustStopContainer\n\t}\n\treturn false\n}\n\n// Path: pkg/gui/gocui.go\npackage gui\n\nimport (\n\t\"github.com/gookit/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\nvar gocuiColorMap = map[string]gocui.Attribute{\n\t\"default\":   gocui.ColorDefault,\n\t\"black\":     gocui.ColorBlack,\n\t\"red\":       gocui.ColorRed,\n\t\"green\":     gocui.ColorGreen,\n\t\"yellow\":    gocui.ColorYellow,\n\t\"blue\":      gocui.ColorBlue,\n\t\"magenta\":   gocui.ColorMagenta,\n\t\"cyan\":      gocui.ColorCyan,\n\t\"white\":     gocui.ColorWhite,\n\t\"bold\":      gocui.AttrBold,\n\t\"reverse\":   gocui.AttrReverse,\n\t\"underline\": gocui.AttrUnderline,\n}\n\n// GetAttribute gets the gocui color attribute from the string\nfunc GetGocuiAttribute(key string) gocui.Attribute {\n\tif utils.IsValidHexValue(key) {\n\t\tvalues := color.HEX(key).Values()\n\t\treturn gocui.NewRGBColor(int32(values[0]), int32(values[1]), int32(values[2]))\n\t}\n\n\tvalue, present := gocuiColorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn gocui.ColorDefault\n}\n\n// GetGocuiStyle bitwise OR's a list of attributes obtained via the given keys\nfunc GetGocuiStyle(keys []string) gocui.Attribute {\n\tvar attribute gocui.Attribute\n\tfor _, key := range keys {\n\t\tattribute |= GetGocuiAttribute(key)\n\t}\n\treturn attribute\n}\n\n// Path: pkg/utils/utils.go\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"html/template\"\n\t\"io\"\n\t\"math\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/mattn/go-runewidth\"\n\n\t// \"github.com/jesseduffield/yaml\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/goccy/go-yaml\"\n\t\"github.com/goccy/go-yaml/lexer\"\n\t\"github.com/goccy/go-yaml/printer\"\n)\n\n// SplitLines takes a multiline string and splits it on newlines\n// currently we are also stripping \\r's which may have adverse effects for\n// windows users (but no issues have been raised yet)\nfunc SplitLines(multilineString string) []string {\n\tmultilineString = strings.Replace(multilineString, \"\\r\", \"\", -1)\n\tif multilineString == \"\" || multilineString == \"\\n\" {\n\t\treturn make([]string, 0)\n\t}\n\tlines := strings.Split(multilineString, \"\\n\")\n\tif lines[len(lines)-1] == \"\" {\n\t\treturn lines[:len(lines)-1]\n\t}\n\treturn lines\n}\n\n// WithPadding pads a string as much as you want\nfunc WithPadding(str string, padding int) string {\n\tuncoloredStr := Decolorise(str)\n\tif padding < runewidth.StringWidth(uncoloredStr) {\n\t\treturn str\n\t}\n\treturn str + strings.Repeat(\" \", padding-runewidth.StringWidth(uncoloredStr))\n}\n\n// ColoredString takes a string and a colour attribute and returns a colored\n// string with that attribute\nfunc ColoredString(str string, colorAttribute color.Attribute) string {\n\t// fatih/color does not have a color.Default attribute, so unless we fork that repo the only way for us to express that we don't want to color a string different to the terminal's default is to not call the function in the first place, but that's annoying when you want a streamlined code path. Because I'm too lazy to fork the repo right now, we'll just assume that by FgWhite you really mean Default, for the sake of supporting users with light themed terminals.\n\tif colorAttribute == color.FgWhite {\n\t\treturn str\n\t}\n\tcolour := color.New(colorAttribute)\n\treturn ColoredStringDirect(str, colour)\n}\n\n// ColoredYamlString takes an YAML formatted string and returns a colored string\n// with colors hardcoded as:\n// keys: cyan\n// Booleans: magenta\n// Numbers: yellow\n// Strings: green\nfunc ColoredYamlString(str string) string {\n\tformat := func(attr color.Attribute) string {\n\t\treturn fmt.Sprintf(\"%s[%dm\", \"\\x1b\", attr)\n\t}\n\ttokens := lexer.Tokenize(str)\n\tvar p printer.Printer\n\tp.Bool = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgMagenta),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.Number = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgYellow),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.MapKey = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgCyan),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.String = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgGreen),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\treturn p.PrintTokens(tokens)\n}\n\n// MultiColoredString takes a string and an array of colour attributes and returns a colored\n// string with those attributes\n\nfunc MultiColoredString(str string, colorAttribute ...color.Attribute) string {\n\tcolour := color.New(colorAttribute...)\n\treturn ColoredStringDirect(str, colour)\n}\n\n// ColoredStringDirect used for aggregating a few color attributes rather than\n// just sending a single one\nfunc ColoredStringDirect(str string, colour *color.Color) string {\n\treturn colour.SprintFunc()(fmt.Sprint(str))\n}\n\n// NormalizeLinefeeds - Removes all Windows and Mac style line feeds\nfunc NormalizeLinefeeds(str string) string {\n\tstr = strings.Replace(str, \"\\r\\n\", \"\\n\", -1)\n\tstr = strings.Replace(str, \"\\r\", \"\", -1)\n\treturn str\n}\n\n// Loader dumps a string to be displayed as a loader\nfunc Loader() string {\n\tcharacters := \"|/-\\\\\"\n\tnow := time.Now()\n\tnanos := now.UnixNano()\n\tindex := nanos / 50000000 % int64(len(characters))\n\treturn characters[index : index+1]\n}\n\n// ResolvePlaceholderString populates a template with values\nfunc ResolvePlaceholderString(str string, arguments map[string]string) string {\n\tfor key, value := range arguments {\n\t\tstr = strings.Replace(str, \"{{\"+key+\"}}\", value, -1)\n\t}\n\treturn str\n}\n\n// Max returns the maximum of two integers\nfunc Max(x, y int) int {\n\tif x > y {\n\t\treturn x\n\t}\n\treturn y\n}\n\n// RenderTable takes an array of string arrays and returns a table containing the values\nfunc RenderTable(rows [][]string) (string, error) {\n\tif len(rows) == 0 {\n\t\treturn \"\", nil\n\t}\n\tif !displayArraysAligned(rows) {\n\t\treturn \"\", errors.New(\"Each item must return the same number of strings to display\")\n\t}\n\n\tcolumnPadWidths := getPadWidths(rows)\n\tpaddedDisplayRows := getPaddedDisplayStrings(rows, columnPadWidths)\n\n\treturn strings.Join(paddedDisplayRows, \"\\n\"), nil\n}\n\n// Decolorise strips a string of color\nfunc Decolorise(str string) string {\n\tre := regexp.MustCompile(`\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mK]`)\n\treturn re.ReplaceAllString(str, \"\")\n}\n\nfunc getPadWidths(rows [][]string) []int {\n\tif len(rows[0]) <= 1 {\n\t\treturn []int{}\n\t}\n\tcolumnPadWidths := make([]int, len(rows[0])-1)\n\tfor i := range columnPadWidths {\n\t\tfor _, cells := range rows {\n\t\t\tuncoloredCell := Decolorise(cells[i])\n\n\t\t\tif runewidth.StringWidth(uncoloredCell) > columnPadWidths[i] {\n\t\t\t\tcolumnPadWidths[i] = runewidth.StringWidth(uncoloredCell)\n\t\t\t}\n\t\t}\n\t}\n\treturn columnPadWidths\n}\n\nfunc getPaddedDisplayStrings(rows [][]string, columnPadWidths []int) []string {\n\tpaddedDisplayRows := make([]string, len(rows))\n\tfor i, cells := range rows {\n\t\tfor j, columnPadWidth := range columnPadWidths {\n\t\t\tpaddedDisplayRows[i] += WithPadding(cells[j], columnPadWidth) + \" \"\n\t\t}\n\t\tpaddedDisplayRows[i] += cells[len(columnPadWidths)]\n\t}\n\treturn paddedDisplayRows\n}\n\n// displayArraysAligned returns true if every string array returned from our\n// list of displayables has the same length\nfunc displayArraysAligned(stringArrays [][]string) bool {\n\tfor _, strings := range stringArrays {\n\t\tif len(strings) != len(stringArrays[0]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc FormatBinaryBytes(b int) string {\n\tn := float64(b)\n\tunits := []string{\"B\", \"kiB\", \"MiB\", \"GiB\", \"TiB\", \"PiB\", \"EiB\", \"ZiB\", \"YiB\"}\n\tfor _, unit := range units {\n\t\tif n > math.Pow(2, 10) {\n\t\t\tn /= math.Pow(2, 10)\n\t\t} else {\n\t\t\tval := fmt.Sprintf(\"%.2f%s\", n, unit)\n\t\t\tif val == \"0.00B\" {\n\t\t\t\treturn \"0B\"\n\t\t\t}\n\t\t\treturn val\n\t\t}\n\t}\n\treturn \"a lot\"\n}\n\nfunc FormatDecimalBytes(b int) string {\n\tn := float64(b)\n\tunits := []string{\"B\", \"kB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\", \"ZB\", \"YB\"}\n\tfor _, unit := range units {\n\t\tif n > math.Pow(10, 3) {\n\t\t\tn /= math.Pow(10, 3)\n\t\t} else {\n\t\t\tval := fmt.Sprintf(\"%.2f%s\", n, unit)\n\t\t\tif val == \"0.00B\" {\n\t\t\t\treturn \"0B\"\n\t\t\t}\n\t\t\treturn val\n\t\t}\n\t}\n\treturn \"a lot\"\n}\n\nfunc ApplyTemplate(str string, object interface{}) string {\n\tvar buf bytes.Buffer\n\t_ = template.Must(template.New(\"\").Parse(str)).Execute(&buf, object)\n\treturn buf.String()\n}\n\n// GetGocuiAttribute gets the gocui color attribute from the string\nfunc GetGocuiAttribute(key string) gocui.Attribute {\n\tcolorMap := map[string]gocui.Attribute{\n\t\t\"default\":   gocui.ColorDefault,\n\t\t\"black\":     gocui.ColorBlack,\n\t\t\"red\":       gocui.ColorRed,\n\t\t\"green\":     gocui.ColorGreen,\n\t\t\"yellow\":    gocui.ColorYellow,\n\t\t\"blue\":      gocui.ColorBlue,\n\t\t\"magenta\":   gocui.ColorMagenta,\n\t\t\"cyan\":      gocui.ColorCyan,\n\t\t\"white\":     gocui.ColorWhite,\n\t\t\"bold\":      gocui.AttrBold,\n\t\t\"reverse\":   gocui.AttrReverse,\n\t\t\"underline\": gocui.AttrUnderline,\n\t}\n\tvalue, present := colorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn gocui.ColorDefault\n}\n\n// GetColorAttribute gets the color attribute from the string\nfunc GetColorAttribute(key string) color.Attribute {\n\tcolorMap := map[string]color.Attribute{\n\t\t\"default\":   color.FgWhite,\n\t\t\"black\":     color.FgBlack,\n\t\t\"red\":       color.FgRed,\n\t\t\"green\":     color.FgGreen,\n\t\t\"yellow\":    color.FgYellow,\n\t\t\"blue\":      color.FgBlue,\n\t\t\"magenta\":   color.FgMagenta,\n\t\t\"cyan\":      color.FgCyan,\n\t\t\"white\":     color.FgWhite,\n\t\t\"bold\":      color.Bold,\n\t\t\"underline\": color.Underline,\n\t}\n\tvalue, present := colorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn color.FgWhite\n}\n\n// WithShortSha returns a command but with a shorter SHA. in the terminal we're all used to 10 character SHAs but under the hood they're actually 64 characters long. No need including all the characters when we're just displaying a command\nfunc WithShortSha(str string) string {\n\tsplit := strings.Split(str, \" \")\n\tfor i, word := range split {\n\t\t// good enough proxy for now\n\t\tif len(word) == 64 {\n\t\t\tsplit[i] = word[0:10]\n\t\t}\n\t}\n\treturn strings.Join(split, \" \")\n}\n\n// FormatMapItem is for displaying items in a map\nfunc FormatMapItem(padding int, k string, v interface{}) string {\n\treturn fmt.Sprintf(\"%s%s %v\\n\", strings.Repeat(\" \", padding), ColoredString(k+\":\", color.FgYellow), fmt.Sprintf(\"%v\", v))\n}\n\n// FormatMap is for displaying a map\nfunc FormatMap(padding int, m map[string]string) string {\n\tif len(m) == 0 {\n\t\treturn \"none\\n\"\n\t}\n\n\toutput := \"\\n\"\n\n\tkeys := make([]string, 0, len(m))\n\tfor key := range m {\n\t\tkeys = append(keys, key)\n\t}\n\tsort.Strings(keys)\n\tfor _, key := range keys {\n\t\toutput += FormatMapItem(padding, key, m[key])\n\t}\n\n\treturn output\n}\n\ntype multiErr []error\n\nfunc (m multiErr) Error() string {\n\tvar b bytes.Buffer\n\tb.WriteString(\"encountered multiple errors:\")\n\tfor _, err := range m {\n\t\tb.WriteString(\"\\n\\t... \" + err.Error())\n\t}\n\treturn b.String()\n}\n\nfunc CloseMany(closers []io.Closer) error {\n\terrs := make([]error, 0, len(closers))\n\tfor _, c := range closers {\n\t\terr := c.Close()\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn multiErr(errs)\n\t}\n\treturn nil\n}\n\nfunc SafeTruncate(str string, limit int) string {\n\tif len(str) > limit {\n\t\treturn str[0:limit]\n\t} else {\n\t\treturn str\n\t}\n}\n\nfunc IsValidHexValue(v string) bool {\n\tif len(v) != 4 && len(v) != 7 {\n\t\treturn false\n\t}\n\n\tif v[0] != '#' {\n\t\treturn false\n\t}\n\n\tfor _, char := range v[1:] {\n\t\tswitch char {\n\t\tcase '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F':\n\t\t\tcontinue\n\t\tdefault:\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// Style used on menu items that open another menu\nfunc OpensMenuStyle(str string) string {\n\treturn ColoredString(fmt.Sprintf(\"%s...\", str), color.FgMagenta)\n}\n\n// MarshalIntoYaml gets any json-tagged data and marshal it into yaml saving original json structure.\n// Useful for structs from 3rd-party libs without yaml tags.\nfunc MarshalIntoYaml(data interface{}) ([]byte, error) {\n\treturn marshalIntoFormat(data, \"yaml\")\n}\n\nfunc marshalIntoFormat(data interface{}, format string) ([]byte, error) {\n\t// First marshal struct->json to get the resulting structure declared by json tags\n\tdataJSON, err := json.MarshalIndent(data, \"\", \"  \")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tswitch format {\n\tcase \"json\":\n\t\treturn dataJSON, err\n\tcase \"yaml\":\n\t\t// Use Unmarshal->Marshal hack to convert json into yaml with the original structure preserved\n\t\tvar dataMirror yaml.MapSlice\n\t\tif err := yaml.Unmarshal(dataJSON, &dataMirror); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn yaml.Marshal(dataMirror)\n\tdefault:\n\t\treturn nil, errors.New(fmt.Sprintf(\"Unsupported detailization format: %s\", format))\n\t}\n}\n\n// Path: pkg/commands/os.go\npackage commands\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/go-errors/errors\"\n\n\t\"github.com/jesseduffield/kill\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/mgutz/str\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Platform stores the os state\ntype Platform struct {\n\tos              string\n\tshell           string\n\tshellArg        string\n\topenCommand     string\n\topenLinkCommand string\n}\n\n// OSCommand holds all the os commands\ntype OSCommand struct {\n\tLog      *logrus.Entry\n\tPlatform *Platform\n\tConfig   *config.AppConfig\n\tcommand  func(string, ...string) *exec.Cmd\n\tgetenv   func(string) string\n}\n\n// NewOSCommand os command runner\nfunc NewOSCommand(log *logrus.Entry, config *config.AppConfig) *OSCommand {\n\treturn &OSCommand{\n\t\tLog:      log,\n\t\tPlatform: getPlatform(),\n\t\tConfig:   config,\n\t\tcommand:  exec.Command,\n\t\tgetenv:   os.Getenv,\n\t}\n}\n\n// SetCommand sets the command function used by the struct.\n// To be used for testing only\nfunc (c *OSCommand) SetCommand(cmd func(string, ...string) *exec.Cmd) {\n\tc.command = cmd\n}\n\n// RunCommandWithOutput wrapper around commands returning their output and error\nfunc (c *OSCommand) RunCommandWithOutput(command string) (string, error) {\n\tcmd := c.ExecutableFromString(command)\n\tbefore := time.Now()\n\toutput, err := sanitisedCommandOutput(cmd.Output())\n\tc.Log.Warn(fmt.Sprintf(\"'%s': %s\", command, time.Since(before)))\n\treturn output, err\n}\n\n// RunCommandWithOutput wrapper around commands returning their output and error\nfunc (c *OSCommand) RunCommandWithOutputContext(ctx context.Context, command string) (string, error) {\n\tcmd := c.ExecutableFromStringContext(ctx, command)\n\tbefore := time.Now()\n\toutput, err := sanitisedCommandOutput(cmd.Output())\n\tc.Log.Warn(fmt.Sprintf(\"'%s': %s\", command, time.Since(before)))\n\treturn output, err\n}\n\n// RunExecutableWithOutput runs an executable file and returns its output\nfunc (c *OSCommand) RunExecutableWithOutput(cmd *exec.Cmd) (string, error) {\n\treturn sanitisedCommandOutput(cmd.CombinedOutput())\n}\n\n// RunExecutable runs an executable file and returns an error if there was one\nfunc (c *OSCommand) RunExecutable(cmd *exec.Cmd) error {\n\t_, err := c.RunExecutableWithOutput(cmd)\n\treturn err\n}\n\n// ExecutableFromString takes a string like `docker ps -a` and returns an executable command for it\nfunc (c *OSCommand) ExecutableFromString(commandStr string) *exec.Cmd {\n\tsplitCmd := str.ToArgv(commandStr)\n\treturn c.NewCmd(splitCmd[0], splitCmd[1:]...)\n}\n\n// Same as ExecutableFromString but cancellable via a context\nfunc (c *OSCommand) ExecutableFromStringContext(ctx context.Context, commandStr string) *exec.Cmd {\n\tsplitCmd := str.ToArgv(commandStr)\n\treturn exec.CommandContext(ctx, splitCmd[0], splitCmd[1:]...)\n}\n\nfunc (c *OSCommand) NewCmd(cmdName string, commandArgs ...string) *exec.Cmd {\n\tcmd := c.command(cmdName, commandArgs...)\n\tcmd.Env = os.Environ()\n\treturn cmd\n}\n\nfunc (c *OSCommand) NewCommandStringWithShell(commandStr string) string {\n\tvar quotedCommand string\n\t// Windows does not seem to like quotes around the command\n\tif c.Platform.os == \"windows\" {\n\t\tquotedCommand = strings.NewReplacer(\n\t\t\t\"^\", \"^^\",\n\t\t\t\"&\", \"^&\",\n\t\t\t\"|\", \"^|\",\n\t\t\t\"<\", \"^<\",\n\t\t\t\">\", \"^>\",\n\t\t\t\"%\", \"^%\",\n\t\t).Replace(commandStr)\n\t} else {\n\t\tquotedCommand = c.Quote(commandStr)\n\t}\n\n\treturn fmt.Sprintf(\"%s %s %s\", c.Platform.shell, c.Platform.shellArg, quotedCommand)\n}\n\n// RunCommand runs a command and just returns the error\nfunc (c *OSCommand) RunCommand(command string) error {\n\t_, err := c.RunCommandWithOutput(command)\n\treturn err\n}\n\n// FileType tells us if the file is a file, directory or other\nfunc (c *OSCommand) FileType(path string) string {\n\tfileInfo, err := os.Stat(path)\n\tif err != nil {\n\t\treturn \"other\"\n\t}\n\tif fileInfo.IsDir() {\n\t\treturn \"directory\"\n\t}\n\treturn \"file\"\n}\n\nfunc sanitisedCommandOutput(output []byte, err error) (string, error) {\n\toutputString := string(output)\n\tif err != nil {\n\t\t// errors like 'exit status 1' are not very useful so we'll create an error\n\t\t// from stderr if we got an ExitError\n\t\texitError, ok := err.(*exec.ExitError)\n\t\tif ok {\n\t\t\treturn outputString, errors.New(string(exitError.Stderr))\n\t\t}\n\t\treturn \"\", WrapError(err)\n\t}\n\treturn outputString, nil\n}\n\n// OpenFile opens a file with the given\nfunc (c *OSCommand) OpenFile(filename string) error {\n\tcommandTemplate := c.Config.UserConfig.OS.OpenCommand\n\ttemplateValues := map[string]string{\n\t\t\"filename\": c.Quote(filename),\n\t}\n\n\tcommand := utils.ResolvePlaceholderString(commandTemplate, templateValues)\n\terr := c.RunCommand(command)\n\treturn err\n}\n\n// OpenLink opens a file with the given\nfunc (c *OSCommand) OpenLink(link string) error {\n\tcommandTemplate := c.Config.UserConfig.OS.OpenLinkCommand\n\ttemplateValues := map[string]string{\n\t\t\"link\": c.Quote(link),\n\t}\n\n\tcommand := utils.ResolvePlaceholderString(commandTemplate, templateValues)\n\terr := c.RunCommand(command)\n\treturn err\n}\n\n// EditFile opens a file in a subprocess using whatever editor is available,\n// falling back to core.editor, VISUAL, EDITOR, then vi\nfunc (c *OSCommand) EditFile(filename string) (*exec.Cmd, error) {\n\teditor := c.getenv(\"VISUAL\")\n\tif editor == \"\" {\n\t\teditor = c.getenv(\"EDITOR\")\n\t}\n\tif editor == \"\" {\n\t\tif err := c.RunCommand(\"which vi\"); err == nil {\n\t\t\teditor = \"vi\"\n\t\t}\n\t}\n\tif editor == \"\" {\n\t\treturn nil, errors.New(\"No editor defined in $VISUAL or $EDITOR\")\n\t}\n\n\treturn c.NewCmd(editor, filename), nil\n}\n\n// Quote wraps a message in platform-specific quotation marks\nfunc (c *OSCommand) Quote(message string) string {\n\tvar quote string\n\tif c.Platform.os == \"windows\" {\n\t\tquote = `\\\"`\n\t\tmessage = strings.NewReplacer(\n\t\t\t`\"`, `\"'\"'\"`,\n\t\t\t`\\\"`, `\\\\\"`,\n\t\t).Replace(message)\n\t} else {\n\t\tquote = `\"`\n\t\tmessage = strings.NewReplacer(\n\t\t\t`\\`, `\\\\`,\n\t\t\t`\"`, `\\\"`,\n\t\t\t`$`, `\\$`,\n\t\t\t\"`\", \"\\\\`\",\n\t\t).Replace(message)\n\t}\n\treturn quote + message + quote\n}\n\n// Unquote removes wrapping quotations marks if they are present\n// this is needed for removing quotes from staged filenames with spaces\nfunc (c *OSCommand) Unquote(message string) string {\n\treturn strings.Replace(message, `\"`, \"\", -1)\n}\n\n// AppendLineToFile adds a new line in file\nfunc (c *OSCommand) AppendLineToFile(filename, line string) error {\n\tf, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600)\n\tif err != nil {\n\t\treturn WrapError(err)\n\t}\n\tdefer f.Close()\n\n\t_, err = f.WriteString(\"\\n\" + line)\n\tif err != nil {\n\t\treturn WrapError(err)\n\t}\n\treturn nil\n}\n\n// CreateTempFile writes a string to a new temp file and returns the file's name\nfunc (c *OSCommand) CreateTempFile(filename, content string) (string, error) {\n\ttmpfile, err := os.CreateTemp(\"\", filename)\n\tif err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\n\tif _, err := tmpfile.WriteString(content); err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\tif err := tmpfile.Close(); err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\n\treturn tmpfile.Name(), nil\n}\n\n// Remove removes a file or directory at the specified path\nfunc (c *OSCommand) Remove(filename string) error {\n\terr := os.RemoveAll(filename)\n\treturn WrapError(err)\n}\n\n// FileExists checks whether a file exists at the specified path\nfunc (c *OSCommand) FileExists(path string) (bool, error) {\n\tif _, err := os.Stat(path); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, err\n\t}\n\treturn true, nil\n}\n\n// RunPreparedCommand takes a pointer to an exec.Cmd and runs it\n// this is useful if you need to give your command some environment variables\n// before running it\nfunc (c *OSCommand) RunPreparedCommand(cmd *exec.Cmd) error {\n\tout, err := cmd.CombinedOutput()\n\toutString := string(out)\n\tc.Log.Info(outString)\n\tif err != nil {\n\t\tif len(outString) == 0 {\n\t\t\treturn err\n\t\t}\n\t\treturn errors.New(outString)\n\t}\n\treturn nil\n}\n\n// GetLazydockerPath returns the path of the currently executed file\nfunc (c *OSCommand) GetLazydockerPath() string {\n\tex, err := os.Executable() // get the executable path for docker to use\n\tif err != nil {\n\t\tex = os.Args[0] // fallback to the first call argument if needed\n\t}\n\treturn filepath.ToSlash(ex)\n}\n\n// RunCustomCommand returns the pointer to a custom command\nfunc (c *OSCommand) RunCustomCommand(command string) *exec.Cmd {\n\treturn c.NewCmd(c.Platform.shell, c.Platform.shellArg, command)\n}\n\n// PipeCommands runs a heap of commands and pipes their inputs/outputs together like A | B | C\nfunc (c *OSCommand) PipeCommands(commandStrings ...string) error {\n\tcmds := make([]*exec.Cmd, len(commandStrings))\n\n\tfor i, str := range commandStrings {\n\t\tcmds[i] = c.ExecutableFromString(str)\n\t}\n\n\tfor i := 0; i < len(cmds)-1; i++ {\n\t\tstdout, err := cmds[i].StdoutPipe()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tcmds[i+1].Stdin = stdout\n\t}\n\n\t// keeping this here in case I adapt this code for some other purpose in the future\n\t// cmds[len(cmds)-1].Stdout = os.Stdout\n\n\tfinalErrors := []string{}\n\n\twg := sync.WaitGroup{}\n\twg.Add(len(cmds))\n\n\tfor _, cmd := range cmds {\n\t\tcurrentCmd := cmd\n\t\tgo func() {\n\t\t\tstderr, err := currentCmd.StderrPipe()\n\t\t\tif err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\tif err := currentCmd.Start(); err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\tif b, err := io.ReadAll(stderr); err == nil {\n\t\t\t\tif len(b) > 0 {\n\t\t\t\t\tfinalErrors = append(finalErrors, string(b))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif err := currentCmd.Wait(); err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\twg.Wait()\n\n\tif len(finalErrors) > 0 {\n\t\treturn errors.New(strings.Join(finalErrors, \"\\n\"))\n\t}\n\treturn nil\n}\n\n// Kill kills a process. If the process has Setpgid == true, then we have anticipated that it might spawn its own child processes, so we've given it a process group ID (PGID) equal to its process id (PID) and given its child processes will inherit the PGID, we can kill that group, rather than killing the process itself.\nfunc (c *OSCommand) Kill(cmd *exec.Cmd) error {\n\treturn kill.Kill(cmd)\n}\n\n// PrepareForChildren sets Setpgid to true on the cmd, so that when we run it as a subprocess, we can kill its group rather than the process itself. This is because some commands, like `docker-compose logs` spawn multiple children processes, and killing the parent process isn't sufficient for killing those child processes. We set the group id here, and then in subprocess.go we check if the group id is set and if so, we kill the whole group rather than just the one process.\nfunc (c *OSCommand) PrepareForChildren(cmd *exec.Cmd) {\n\tkill.PrepareForChildren(cmd)\n}\n\n// Path: pkg/log/log.go\npackage log\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// NewLogger returns a new logger\nfunc NewLogger(config *config.AppConfig, rollrusHook string) *logrus.Entry {\n\tvar log *logrus.Logger\n\tif config.Debug || os.Getenv(\"DEBUG\") == \"TRUE\" {\n\t\tlog = newDevelopmentLogger(config)\n\t} else {\n\t\tlog = newProductionLogger()\n\t}\n\n\t// highly recommended: tail -f development.log | humanlog\n\t// https://github.com/aybabtme/humanlog\n\tlog.Formatter = &logrus.JSONFormatter{}\n\n\treturn log.WithFields(logrus.Fields{\n\t\t\"debug\":     config.Debug,\n\t\t\"version\":   config.Version,\n\t\t\"commit\":    config.Commit,\n\t\t\"buildDate\": config.BuildDate,\n\t})\n}\n\nfunc getLogLevel() logrus.Level {\n\tstrLevel := os.Getenv(\"LOG_LEVEL\")\n\tlevel, err := logrus.ParseLevel(strLevel)\n\tif err != nil {\n\t\treturn logrus.DebugLevel\n\t}\n\treturn level\n}\n\nfunc newDevelopmentLogger(config *config.AppConfig) *logrus.Logger {\n\tlog := logrus.New()\n\tlog.SetLevel(getLogLevel())\n\tfile, err := os.OpenFile(filepath.Join(config.ConfigDir, \"development.log\"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o666)\n\tif err != nil {\n\t\tfmt.Println(\"unable to log to file\")\n\t\tos.Exit(1)\n\t}\n\tlog.SetOutput(file)\n\treturn log\n}\n\nfunc newProductionLogger() *logrus.Logger {\n\tlog := logrus.New()\n\tlog.Out = io.Discard\n\tlog.SetLevel(logrus.ErrorLevel)\n\treturn log\n}\n\n// Path: pkg/i18n/i18n.go\npackage i18n\n\nimport (\n\t\"strings\"\n\n\t\"github.com/imdario/mergo\"\n\n\t\"github.com/cloudfoundry/jibber_jabber\"\n\t\"github.com/go-errors/errors\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Localizer will translate a message into the user's language\ntype Localizer struct {\n\tLog *logrus.Entry\n\tS   TranslationSet\n}\n\nfunc NewTranslationSetFromConfig(log *logrus.Entry, configLanguage string) (*TranslationSet, error) {\n\tif configLanguage == \"auto\" {\n\t\tlanguage := detectLanguage(jibber_jabber.DetectLanguage)\n\t\treturn NewTranslationSet(log, language), nil\n\t}\n\n\tfor key := range GetTranslationSets() {\n\t\tif key == configLanguage {\n\t\t\treturn NewTranslationSet(log, configLanguage), nil\n\t\t}\n\t}\n\n\treturn NewTranslationSet(log, \"en\"), errors.New(\"Language not found: \" + configLanguage)\n}\n\nfunc NewTranslationSet(log *logrus.Entry, language string) *TranslationSet {\n\tlog.Info(\"language: \" + language)\n\n\tbaseSet := englishSet()\n\n\tfor languageCode, translationSet := range GetTranslationSets() {\n\t\tif strings.HasPrefix(language, languageCode) {\n\t\t\t_ = mergo.Merge(&baseSet, translationSet, mergo.WithOverride)\n\t\t}\n\t}\n\n\treturn &baseSet\n}\n\n// GetTranslationSets gets all the translation sets, keyed by language code\nfunc GetTranslationSets() map[string]TranslationSet {\n\treturn map[string]TranslationSet{\n\t\t\"pl\": polishSet(),\n\t\t\"nl\": dutchSet(),\n\t\t\"de\": germanSet(),\n\t\t\"tr\": turkishSet(),\n\t\t\"en\": englishSet(),\n\t\t\"fr\": frenchSet(),\n\t\t\"zh\": chineseSet(),\n\t\t\"es\": spanishSet(),\n\t\t\"pt\": portugueseSet(),\n\t}\n}\n\n// detectLanguage extracts user language from environment\nfunc detectLanguage(langDetector func() (string, error)) string {\n\tif userLang, err := langDetector(); err == nil {\n\t\treturn userLang\n\t}\n\n\treturn \"C\"\n}\n\n// Path: pkg/tasks/tasks.go\npackage tasks\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\ntype TaskManager struct {\n\tcurrentTask  *Task\n\twaitingMutex deadlock.Mutex\n\ttaskIDMutex  deadlock.Mutex\n\tLog          *logrus.Entry\n\tTr           *i18n.TranslationSet\n\tnewTaskId    int\n}\n\ntype Task struct {\n\tctx           context.Context\n\tcancel        context.CancelFunc\n\tstopped       bool\n\tstopMutex     deadlock.Mutex\n\tnotifyStopped chan struct{}\n\tLog           *logrus.Entry\n\tf             func(ctx context.Context)\n}\n\ntype TaskFunc func(ctx context.Context)\n\nfunc NewTaskManager(log *logrus.Entry, translationSet *i18n.TranslationSet) *TaskManager {\n\treturn &TaskManager{Log: log, Tr: translationSet}\n}\n\n// Close closes the task manager, killing whatever task may currently be running\nfunc (t *TaskManager) Close() {\n\tif t.currentTask == nil {\n\t\treturn\n\t}\n\n\tc := make(chan struct{}, 1)\n\n\tgo func() {\n\t\tt.currentTask.Stop()\n\t\tc <- struct{}{}\n\t}()\n\n\tselect {\n\tcase <-c:\n\t\treturn\n\tcase <-time.After(3 * time.Second):\n...\n", "needle_token_start": 2026, "needle_token_end": 2070, "code_context_ntokens": 10383, "output": ["```go\n// MultiColoredString takes a string and an array of colour attributes and returns a colored\n// string with those attributes\nfunc MultiColoredString(str string, colorAttribute ...color.Attribute) string {\n\tcolour := color.New(colorAttribute...)\n\treturn ColoredStringDirect(str, colour)\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "getContainerDisplaySubstatus", "language": "go", "path": "pkg/gui/presentation/containers.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to provide a detailed status string for a container, focusing on specific states like \"exited\" or \"running\" and providing additional information such as exit codes or health status.\n2. **Input**: The function takes a configuration object and a container object as inputs.\n3. **Output**: It returns a string that represents the substatus of the container, which could be an exit code or health status, formatted with appropriate colors.\n4. **Procedure**: The function first checks if the container details are loaded. If not, it returns an empty string. If details are loaded, it checks the container's state. If the state is \"exited\", it returns the exit code formatted with a color. If the state is \"running\", it calls another function to get the health status of the container. For other states, it returns an empty string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/gui/panels/context_state.go\npackage panels\n\nimport (\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/samber/lo\"\n)\n\n// A 'context' generally corresponds to an item and the tab in the main panel that we're\n// displaying. So if we switch to a new item, or change the tab in the panel panel\n...\n// Path: pkg/gui/panels/filtered_list.go\npackage panels\n\nimport (\n\t\"sort\"\n\t\"sync\"\n)\n\ntype FilteredList[T comparable] struct {\n\tallItems []T\n\t// indices of items in the allItems slice that are included in the filtered list\n\tindices []int\n\n\tmutex sync.RWMutex\n}\n\nfunc NewFilteredList[T comparable]() *FilteredList[T] {\n\treturn &FilteredList[T]{}\n}\n\nfunc (self *FilteredList[T]) SetItems(items []T) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tself.allItems = items\n\tself.indices = make([]int, len(items))\n\tfor i := range self.indices {\n\t\tself.indices[i] = i\n\t}\n}\n\nfunc (self *FilteredList[T]) Filter(filter func(T, int) bool) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tself.indices = self.indices[:0]\n\tfor i, item := range self.allItems {\n\t\tif filter(item, i) {\n\t\t\tself.indices = append(self.indices, i)\n\t\t}\n\t}\n}\n\nfunc (self *FilteredList[T]) Sort(less func(T, T) bool) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tif less == nil {\n\t\treturn\n\t}\n\n\tsort.Slice(self.indices, func(i, j int) bool {\n\t\treturn less(self.allItems[self.indices[i]], self.allItems[self.indices[j]])\n\t})\n}\n\nfunc (self *FilteredList[T]) Get(index int) T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn self.allItems[self.indices[index]]\n}\n\nfunc (self *FilteredList[T]) TryGet(index int) (T, bool) {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tif index < 0 || index >= len(self.indices) {\n\t\tvar zero T\n\t\treturn zero, false\n\t}\n\n\treturn self.allItems[self.indices[index]], true\n}\n\n// returns the length of the filtered list\nfunc (self *FilteredList[T]) Len() int {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn len(self.indices)\n}\n\nfunc (self *FilteredList[T]) GetIndex(item T) int {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tfor i, index := range self.indices {\n\t\tif self.allItems[index] == item {\n\t\t\treturn i\n\t\t}\n\t}\n\treturn -1\n}\n\nfunc (self *FilteredList[T]) GetItems() []T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tresult := make([]T, len(self.indices))\n\tfor i, index := range self.indices {\n\t\tresult[i] = self.allItems[index]\n\t}\n\treturn result\n}\n\nfunc (self *FilteredList[T]) GetAllItems() []T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn self.allItems\n}\n\n// Path: pkg/gui/panels/filtered_list_test.go\npackage panels\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestFilteredListGet(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs int\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 1,\n\t\t\twant: 2,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 2,\n\t\t\twant: 3,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: 0,\n\t\t\twant: 2,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.Get(tt.args); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.Get() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListLen(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\twant: 3,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\twant: 1,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.Len(); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.Len() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListFilter(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs func(int, int) bool\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 0 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 1 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 2}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.Filter(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t}\n}\n\nfunc TestFilteredListSort(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs func(int, int) bool\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, j int) bool { return i < j },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, j int) bool { return i > j },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{2, 1, 0}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.Sort(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t}\n}\n\nfunc TestFilteredListGetIndex(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs int\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 1,\n\t\t\twant: 0,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 2,\n\t\t\twant: 1,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: 0,\n\t\t\twant: -1,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.GetIndex(tt.args); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.GetIndex() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListGetItems(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\twant []int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\twant: []int{1, 2, 3},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\twant: []int{2},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tgot := tt.f.GetItems()\n\t\tassert.EqualValues(t, got, tt.want)\n\t}\n}\n\nfunc TestFilteredListSetItems(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs []int\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: []int{4, 5, 6},\n\t\t\twant: &FilteredList[int]{allItems: []int{4, 5, 6}, indices: []int{0, 1, 2}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: []int{4},\n\t\t\twant: &FilteredList[int]{allItems: []int{4}, indices: []int{0}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.SetItems(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t\tassert.EqualValues(t, tt.f.allItems, tt.want.allItems)\n\t}\n}\n\n// Path: pkg/gui/panels/list_panel.go\npackage panels\n\nimport (\n\t\"github.com/jesseduffield/gocui\"\n\tlcUtils \"github.com/jesseduffield/lazycore/pkg/utils\"\n)\n\ntype ListPanel[T comparable] struct {\n\tSelectedIdx int\n\tList        *FilteredList[T]\n\tView        *gocui.View\n}\n\nfunc (self *ListPanel[T]) SetSelectedLineIdx(value int) {\n\tclampedValue := 0\n\tif self.List.Len() > 0 {\n\t\tclampedValue = lcUtils.Clamp(value, 0, self.List.Len()-1)\n\t}\n\n\tself.SelectedIdx = clampedValue\n}\n\nfunc (self *ListPanel[T]) clampSelectedLineIdx() {\n\tclamped := lcUtils.Clamp(self.SelectedIdx, 0, self.List.Len()-1)\n\n\tif clamped != self.SelectedIdx {\n\t\tself.SelectedIdx = clamped\n\t}\n}\n\n// moves the cursor up or down by the given amount (up for negative values)\nfunc (self *ListPanel[T]) moveSelectedLine(delta int) {\n\tself.SetSelectedLineIdx(self.SelectedIdx + delta)\n}\n\nfunc (self *ListPanel[T]) SelectNextLine() {\n\tself.moveSelectedLine(1)\n}\n\nfunc (self *ListPanel[T]) SelectPrevLine() {\n\tself.moveSelectedLine(-1)\n}\n\n// Path: pkg/gui/panels/side_list_panel.go\npackage panels\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\ntype ISideListPanel interface {\n\tSetMainTabIndex(int)\n\tHandleSelect() error\n\tGetView() *gocui.View\n\tRefocus()\n\tRerenderList() error\n\tIsFilterDisabled() bool\n\tIsHidden() bool\n\tHandleNextLine() error\n\tHandlePrevLine() error\n\tHandleClick() error\n\tHandlePrevMainTab() error\n\tHandleNextMainTab() error\n}\n\n// list panel at the side of the screen that renders content to the main panel\ntype SideListPanel[T comparable] struct {\n\tContextState *ContextState[T]\n\n\tListPanel[T]\n\n\t// message to render in the main view if there are no items in the panel\n\t// and it has focus. Leave empty if you don't want to render anything\n\tNoItemsMessage string\n\n\t// a representation of the gui\n\tGui IGui\n\n\t// this Filter is applied on top of additional default filters\n\tFilter func(T) bool\n\tSort   func(a, b T) bool\n\n\t// a callback to invoke when the item is clicked\n\tOnClick func(T) error\n\n\t// returns the cells that we render to the view in a table format. The cells will\n\t// be rendered with padding.\n\tGetTableCells func(T) []string\n\n\t// function to be called after re-rendering list. Can be nil\n\tOnRerender func() error\n\n\t// set this to true if you don't want to allow manual filtering via '/'\n\tDisableFilter bool\n\n\t// This can be nil if you want to always show the panel\n\tHide func() bool\n}\n\nvar _ ISideListPanel = &SideListPanel[int]{}\n\ntype IGui interface {\n\tHandleClick(v *gocui.View, itemCount int, selectedLine *int, handleSelect func() error) error\n\tNewSimpleRenderStringTask(getContent func() string) tasks.TaskFunc\n\tFocusY(selectedLine int, itemCount int, view *gocui.View)\n\tShouldRefresh(contextKey string) bool\n\tGetMainView() *gocui.View\n\tIsCurrentView(*gocui.View) bool\n\tFilterString(view *gocui.View) string\n\tIgnoreStrings() []string\n\tUpdate(func() error)\n\n\tQueueTask(f func(ctx context.Context)) error\n}\n\nfunc (self *SideListPanel[T]) HandleClick() error {\n\titemCount := self.List.Len()\n\thandleSelect := self.HandleSelect\n\tselectedLine := &self.SelectedIdx\n\n\tif err := self.Gui.HandleClick(self.View, itemCount, selectedLine, handleSelect); err != nil {\n\t\treturn err\n\t}\n\n\tif self.OnClick != nil {\n\t\tselectedItem, err := self.GetSelectedItem()\n\t\tif err == nil {\n\t\t\treturn self.OnClick(selectedItem)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (self *SideListPanel[T]) GetView() *gocui.View {\n\treturn self.View\n}\n\nfunc (self *SideListPanel[T]) HandleSelect() error {\n\titem, err := self.GetSelectedItem()\n\tif err != nil {\n\t\tif err.Error() != self.NoItemsMessage {\n\t\t\treturn err\n\t\t}\n\n\t\tif self.NoItemsMessage != \"\" {\n\t\t\tself.Gui.NewSimpleRenderStringTask(func() string { return self.NoItemsMessage })\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tself.Refocus()\n\n\treturn self.renderContext(item)\n}\n\nfunc (self *SideListPanel[T]) renderContext(item T) error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tkey := self.ContextState.GetCurrentContextKey(item)\n\tif !self.Gui.ShouldRefresh(key) {\n\t\treturn nil\n\t}\n\n\tmainView := self.Gui.GetMainView()\n\tmainView.Tabs = self.ContextState.GetMainTabTitles()\n\tmainView.TabIndex = self.ContextState.mainTabIdx\n\n\ttask := self.ContextState.GetCurrentMainTab().Render(item)\n\n\treturn self.Gui.QueueTask(task)\n}\n\nfunc (self *SideListPanel[T]) GetSelectedItem() (T, error) {\n\tvar zero T\n\n\titem, ok := self.List.TryGet(self.SelectedIdx)\n\tif !ok {\n\t\t// could probably have a better error here\n\t\treturn zero, errors.New(self.NoItemsMessage)\n\t}\n\n\treturn item, nil\n}\n\nfunc (self *SideListPanel[T]) HandleNextLine() error {\n\tself.SelectNextLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevLine() error {\n\tself.SelectPrevLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandleNextMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandleNextMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandlePrevMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) Refocus() {\n\tself.Gui.FocusY(self.SelectedIdx, self.List.Len(), self.View)\n}\n\nfunc (self *SideListPanel[T]) SetItems(items []T) {\n\tself.List.SetItems(items)\n\tself.FilterAndSort()\n}\n\nfunc (self *SideListPanel[T]) FilterAndSort() {\n\tfilterString := self.Gui.FilterString(self.View)\n\n\tself.List.Filter(func(item T, index int) bool {\n\t\tif self.Filter != nil && !self.Filter(item) {\n\t\t\treturn false\n\t\t}\n\n\t\tif lo.SomeBy(self.Gui.IgnoreStrings(), func(ignore string) bool {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, ignore)\n\t\t\t})\n\t\t}) {\n\t\t\treturn false\n\t\t}\n\n\t\tif filterString != \"\" {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, filterString)\n\t\t\t})\n\t\t}\n\n\t\treturn true\n\t})\n\n\tself.List.Sort(self.Sort)\n\n\tself.clampSelectedLineIdx()\n}\n\nfunc (self *SideListPanel[T]) RerenderList() error {\n\tself.FilterAndSort()\n\n\tself.Gui.Update(func() error {\n\t\tself.View.Clear()\n\t\ttable := lo.Map(self.List.GetItems(), func(item T, index int) []string {\n\t\t\treturn self.GetTableCells(item)\n\t\t})\n\t\trenderedTable, err := utils.RenderTable(table)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfmt.Fprint(self.View, renderedTable)\n\n\t\tif self.OnRerender != nil {\n\t\t\tif err := self.OnRerender(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif self.Gui.IsCurrentView(self.View) {\n\t\t\treturn self.HandleSelect()\n\t\t}\n\t\treturn nil\n\t})\n\n\treturn nil\n}\n\nfunc (self *SideListPanel[T]) SetMainTabIndex(index int) {\n\tif self.ContextState == nil {\n\t\treturn\n\t}\n\n\tself.ContextState.SetMainTabIndex(index)\n}\n\nfunc (self *SideListPanel[T]) IsFilterDisabled() bool {\n\treturn self.DisableFilter\n}\n\nfunc (self *SideListPanel[T]) IsHidden() bool {\n\tif self.Hide == nil {\n\t\treturn false\n\t}\n\n\treturn self.Hide()\n}\n\n// Path: pkg/gui/presentation/container_stats.go\npackage presentation\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/asciigraph\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/mcuadros/go-lookup\"\n\t\"github.com/samber/lo\"\n)\n\nfunc RenderStats(userConfig *config.UserConfig, container *commands.Container, viewWidth int) (string, error) {\n\tstats, ok := container.GetLastStats()\n\tif !ok {\n\t\treturn \"\", nil\n\t}\n\n\tgraphSpecs := userConfig.Stats.Graphs\n\tgraphs := make([]string, len(graphSpecs))\n\tfor i, spec := range graphSpecs {\n\t\tgraph, err := plotGraph(container, spec, viewWidth-10)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tgraphs[i] = utils.ColoredString(graph, utils.GetColorAttribute(spec.Color))\n\t}\n\n\tpidsCount := fmt.Sprintf(\"PIDs: %d\", stats.ClientStats.PidsStats.Current)\n\tdataReceived := fmt.Sprintf(\"Traffic received: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.RxBytes))\n\tdataSent := fmt.Sprintf(\"Traffic sent: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.TxBytes))\n\n\toriginalStats, err := utils.MarshalIntoYaml(stats)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tcontents := fmt.Sprintf(\"\\n\\n%s\\n\\n%s\\n\\n%s\\n%s\\n\\n%s\",\n\t\tutils.ColoredString(strings.Join(graphs, \"\\n\\n\"), color.FgGreen),\n\t\tpidsCount,\n\t\tdataReceived,\n\t\tdataSent,\n\t\tutils.ColoredYamlString(string(originalStats)),\n\t)\n\n\treturn contents, nil\n}\n\n// plotGraph returns the plotted graph based on the graph spec and the stat history\nfunc plotGraph(container *commands.Container, spec config.GraphConfig, width int) (string, error) {\n\tcontainer.StatsMutex.Lock()\n\tdefer container.StatsMutex.Unlock()\n\n\tdata := make([]float64, len(container.StatHistory))\n\n\tfor i, stats := range container.StatHistory {\n\t\tvalue, err := lookup.LookupString(stats, spec.StatPath)\n\t\tif err != nil {\n\t\t\treturn \"Could not find key: \" + spec.StatPath, nil\n\t\t}\n\t\tfloatValue, err := getFloat(value.Interface())\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tdata[i] = floatValue\n\t}\n\n\tmax := spec.Max\n\tif spec.MaxType == \"\" {\n\t\tmax = lo.Max(data)\n\t}\n\n\tmin := spec.Min\n\tif spec.MinType == \"\" {\n\t\tmin = lo.Min(data)\n\t}\n\n\theight := 10\n\tif spec.Height > 0 {\n\t\theight = spec.Height\n\t}\n\n\tcaption := fmt.Sprintf(\n\t\t\"%s: %0.2f (%v)\",\n\t\tspec.Caption,\n\t\tdata[len(data)-1],\n\t\ttime.Since(container.StatHistory[0].RecordedAt).Round(time.Second),\n\t)\n\n\treturn asciigraph.Plot(\n\t\tdata,\n\t\tasciigraph.Height(height),\n\t\tasciigraph.Width(width),\n\t\tasciigraph.Min(min),\n\t\tasciigraph.Max(max),\n\t\tasciigraph.Caption(caption),\n\t), nil\n}\n\n// from Dave C's answer at https://stackoverflow.com/questions/20767724/converting-unknown-interface-to-float64-in-golang\nfunc getFloat(unk interface{}) (float64, error) {\n\tfloatType := reflect.TypeOf(float64(0))\n\tstringType := reflect.TypeOf(\"\")\n\n\tswitch i := unk.(type) {\n\tcase float64:\n\t\treturn i, nil\n\tcase float32:\n\t\treturn float64(i), nil\n\tcase int64:\n\t\treturn float64(i), nil\n\tcase int32:\n\t\treturn float64(i), nil\n\tcase int:\n\t\treturn float64(i), nil\n\tcase uint64:\n\t\treturn float64(i), nil\n\tcase uint32:\n\t\treturn float64(i), nil\n\tcase uint:\n\t\treturn float64(i), nil\n\tcase string:\n\t\treturn strconv.ParseFloat(i, 64)\n\tdefault:\n\t\tv := reflect.ValueOf(unk)\n\t\tv = reflect.Indirect(v)\n\t\tif v.Type().ConvertibleTo(floatType) {\n\t\t\tfv := v.Convert(floatType)\n\t\t\treturn fv.Float(), nil\n\t\t} else if v.Type().ConvertibleTo(stringType) {\n\t\t\tsv := v.Convert(stringType)\n\t\t\ts := sv.String()\n\t\t\treturn strconv.ParseFloat(s, 64)\n\t\t} else {\n\t\t\treturn math.NaN(), fmt.Errorf(\"Can't convert %v to float64\", v.Type())\n\t\t}\n\t}\n}\n\n// Path: pkg/gui/presentation/containers.go\npackage presentation\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\nfunc GetContainerDisplayStrings(guiConfig *config.GuiConfig, container *commands.Container) []string {\n\treturn []string{\n\t\tgetContainerDisplayStatus(guiConfig, container),\n\t\tgetContainerDisplaySubstatus(guiConfig, container),\n\t\tcontainer.Name,\n\t\tgetDisplayCPUPerc(container),\n\t\tutils.ColoredString(displayPorts(container), color.FgYellow),\n\t\tutils.ColoredString(displayContainerImage(container), color.FgMagenta),\n\t}\n}\n\nfunc displayContainerImage(container *commands.Container) string {\n\treturn strings.TrimPrefix(container.Container.Image, \"sha256:\")\n}\n\nfunc displayPorts(c *commands.Container) string {\n\tportStrings := lo.Map(c.Container.Ports, func(port dockerTypes.Port, _ int) string {\n\t\tif port.PublicPort == 0 {\n\t\t\treturn fmt.Sprintf(\"%d/%s\", port.PrivatePort, port.Type)\n\t\t}\n\n\t\t// docker ps will show '0.0.0.0:80->80/tcp' but we'll show\n\t\t// '80->80/tcp' instead to save space (unless the IP is something other than\n\t\t// 0.0.0.0)\n\t\tipString := \"\"\n\t\tif port.IP != \"0.0.0.0\" {\n\t\t\tipString = port.IP + \":\"\n\t\t}\n\t\treturn fmt.Sprintf(\"%s%d->%d/%s\", ipString, port.PublicPort, port.PrivatePort, port.Type)\n\t})\n\n\t// sorting because the order of the ports is not deterministic\n\t// and we don't want to have them constantly swapping\n\tsort.Strings(portStrings)\n\n\treturn strings.Join(portStrings, \", \")\n}\n\n// getContainerDisplayStatus returns the colored status of the container\nfunc getContainerDisplayStatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tshortStatusMap := map[string]string{\n\t\t\"paused\":     \"P\",\n\t\t\"exited\":     \"X\",\n\t\t\"created\":    \"C\",\n\t\t\"removing\":   \"RM\",\n\t\t\"restarting\": \"RS\",\n\t\t\"running\":    \"R\",\n\t\t\"dead\":       \"D\",\n\t}\n\n\ticonStatusMap := map[string]rune{\n\t\t\"paused\":     '\u25eb',\n\t\t\"exited\":     '\u2a2f',\n\t\t\"created\":    '+',\n\t\t\"removing\":   '\u2212',\n\t\t\"restarting\": '\u27f3',\n\t\t\"running\":    '\u25b6',\n\t\t\"dead\":       '!',\n\t}\n\n\tvar containerState string\n\tswitch guiConfig.ContainerStatusHealthStyle {\n\tcase \"short\":\n\t\tcontainerState = shortStatusMap[c.Container.State]\n\tcase \"icon\":\n\t\tcontainerState = string(iconStatusMap[c.Container.State])\n\tcase \"long\":\n\t\tfallthrough\n\tdefault:\n\t\tcontainerState = c.Container.State\n\t}\n\n\treturn utils.ColoredString(containerState, getContainerColor(c))\n}\n\n// GetDisplayStatus returns the exit code if the container has exited, and the health status if the container is running (and has a health check)\nfunc getCo\nntainerDisplaySubstatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tif !c.DetailsLoaded() {\n\t\treturn \"\"\n\t}\n\n\tswitch c.Container.State {\n\tcase \"exited\":\n\t\treturn utils.ColoredString(\n\t\t\tfmt.Sprintf(\"(%s)\", strconv.Itoa(c.Details.State.ExitCode)), getContainerColor(c),\n\t\t)\n\tcase \"running\":\n\t\treturn getHealthStatus(guiConfig, c)\n\tdefault:\n\t\treturn \"\"\n\t}\n}\n\nfunc getHealthStatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tif !c.DetailsLoaded() {\n\t\treturn \"\"\n\t}\n\n\thealthStatusColorMap := map[string]color.Attribute{\n\t\t\"healthy\":   color.FgGreen,\n\t\t\"unhealthy\": color.FgRed,\n\t\t\"starting\":  color.FgYellow,\n\t}\n\n\tif c.Details.State.Health == nil {\n\t\treturn \"\"\n\t}\n\n\tshortHealthStatusMap := map[string]string{\n\t\t\"healthy\":   \"H\",\n\t\t\"unhealthy\": \"U\",\n\t\t\"starting\":  \"S\",\n\t}\n\n\ticonHealthStatusMap := map[string]rune{\n\t\t\"healthy\":   '\u2714',\n\t\t\"unhealthy\": '?',\n\t\t\"starting\":  '\u2026',\n\t}\n\n\tvar healthStatus string\n\tswitch guiConfig.ContainerStatusHealthStyle {\n\tcase \"short\":\n\t\thealthStatus = shortHealthStatusMap[c.Details.State.Health.Status]\n\tcase \"icon\":\n\t\thealthStatus = string(iconHealthStatusMap[c.Details.State.Health.Status])\n\tcase \"long\":\n\t\tfallthrough\n\tdefault:\n\t\thealthStatus = c.Details.State.Health.Status\n\t}\n\n\tif healthStatusColor, ok := healthStatusColorMap[c.Details.State.Health.Status]; ok {\n\t\treturn utils.ColoredString(fmt.Sprintf(\"(%s)\", healthStatus), healthStatusColor)\n\t}\n\treturn \"\"\n}\n\n// getDisplayCPUPerc colors the cpu percentage based on how extreme it is\nfunc getDisplayCPUPerc(c *commands.Container) string {\n\tstats, ok := c.GetLastStats()\n\tif !ok {\n\t\treturn \"\"\n\t}\n\n\tpercentage := stats.DerivedStats.CPUPercentage\n\tformattedPercentage := fmt.Sprintf(\"%.2f%%\", stats.DerivedStats.CPUPercentage)\n\n\tvar clr color.Attribute\n\tif percentage > 90 {\n\t\tclr = color.FgRed\n\t} else if percentage > 50 {\n\t\tclr = color.FgYellow\n\t} else {\n\t\tclr = color.FgWhite\n\t}\n\n\treturn utils.ColoredString(formattedPercentage, clr)\n}\n\n// getContainerColor Container color\nfunc getContainerColor(c *commands.Container) color.Attribute {\n\tswitch c.Container.State {\n\tcase \"exited\":\n\t\t// This means the colour may be briefly yellow and then switch to red upon starting\n\t\t// Not sure what a better alternative is.\n\t\tif !c.DetailsLoaded() || c.Details.State.ExitCode == 0 {\n\t\t\treturn color.FgYellow\n\t\t}\n\t\treturn color.FgRed\n\tcase \"created\":\n\t\treturn color.FgCyan\n\tcase \"running\":\n\t\treturn color.FgGreen\n\tcase \"paused\":\n\t\treturn color.FgYellow\n\tcase \"dead\":\n\t\treturn color.FgRed\n\tcase \"restarting\":\n\t\treturn color.FgBlue\n\tcase \"removing\":\n\t\treturn color.FgMagenta\n\tdefault:\n\t\treturn color.FgWhite\n\t}\n}\n\n// Path: pkg/gui/presentation/images.go\npackage presentation\n\nimport (\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\nfunc GetImageDisplayStrings(image *commands.Image) []string {\n\treturn []string{\n\t\timage.Name,\n\t\timage.Tag,\n\t\tutils.FormatDecimalBytes(int(image.Image.Size)),\n\t}\n}\n\n// Path: pkg/gui/presentation/menu_items.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\nfunc GetMenuItemDisplayStrings(menuItem *types.MenuItem) []string {\n\treturn menuItem.LabelColumns\n}\n\n// Path: pkg/gui/presentation/networks.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/commands\"\n\nfunc GetNetworkDisplayStrings(network *commands.Network) []string {\n\treturn []string{network.Network.Driver, network.Name}\n}\n\n// Path: pkg/gui/presentation/projects.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/commands\"\n\nfunc GetProjectDisplayStrings(project *commands.Project) []string {\n\treturn []string{project.Name}\n}\n\n// Path: pkg/gui/presentation/services.go\npackage presentation\n\nimport (\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\nfunc GetServiceDisplayStrings(guiConfig *config.GuiConfig, service *commands.Service) []string {\n\tif service.Container == nil {\n\t\tvar containerState string\n\t\tswitch guiConfig.ContainerStatusHealthStyle {\n\t\tcase \"short\":\n\t\t\tcontainerState = \"n\"\n\t\tcase \"icon\":\n\t\t\tcontainerState = \".\"\n\t\tcase \"long\":\n\t\t\tfallthrough\n\t\tdefault:\n\t\t\tcontainerState = \"none\"\n\t\t}\n\n\t\treturn []string{\n\t\t\tutils.ColoredString(containerState, color.FgBlue),\n\t\t\t\"\",\n\t\t\tservice.Name,\n\t\t\t\"\",\n\t\t\t\"\",\n\t\t\t\"\",\n\t\t}\n\t}\n\n\tcontainer := service.Container\n\treturn []string{\n\t\tgetContainerDisplayStatus(guiConfig, container),\n\t\tgetContainerDisplaySubstatus(guiConfig, container),\n\t\tservice.Name,\n\t\tgetDisplayCPUPerc(container),\n\t\tutils.ColoredString(displayPorts(container), color.FgYellow),\n\t\tutils.ColoredString(displayContainerImage(container), color.FgMagenta),\n\t}\n}\n\n// Path: pkg/gui/presentation/volumes.go\npackage presentation\n\nimport \"github.com/jesseduffield/lazydocker/pkg/commands\"\n\nfunc GetVolumeDisplayStrings(volume *commands.Volume) []string {\n\treturn []string{volume.Volume.Driver, volume.Name}\n}\n\n// Path: pkg/gui/project_panel.go\npackage gui\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"path\"\n\t\"strings\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/jesseduffield/yaml\"\n)\n\n// Although at the moment we'll only have one project, in future we could have\n// a list of projects in the project panel.\n\nfunc (gui *Gui) getProjectPanel() *panels.SideListPanel[*commands.Project] {\n\treturn &panels.SideListPanel[*commands.Project]{\n\t\tContextState: &panels.ContextState[*commands.Project]{\n\t\t\tGetMainTabs: func() []panels.MainTab[*commands.Project] {\n\t\t\t\tif gui.DockerCommand.InDockerComposeProject {\n\t\t\t\t\treturn []panels.MainTab[*commands.Project]{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey:    \"logs\",\n\t\t\t\t\t\t\tTitle:  gui.Tr.LogsTitle,\n\t\t\t\t\t\t\tRender: gui.renderAllLogs,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey:    \"config\",\n\t\t\t\t\t\t\tTitle:  gui.Tr.DockerComposeConfigTitle,\n\t\t\t\t\t\t\tRender: gui.renderDockerComposeConfig,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tKey:    \"credits\",\n\t\t\t\t\t\t\tTitle:  gui.Tr.CreditsTitle,\n\t\t\t\t\t\t\tRender: gui.renderCredits,\n\t\t\t\t\t\t},\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn []panels.MainTab[*commands.Project]{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"credits\",\n\t\t\t\t\t\tTitle:  gui.Tr.CreditsTitle,\n\t\t\t\t\t\tRender: gui.renderCredits,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t},\n\t\t\tGetItemContextCacheKey: func(project *commands.Project) string {\n\t\t\t\treturn \"projects-\" + project.Name\n\t\t\t},\n\t\t},\n\n\t\tListPanel: panels.ListPanel[*commands.Project]{\n\t\t\tList: panels.NewFilteredList[*commands.Project](),\n\t\t\tView: gui.Views.Project,\n\t\t},\n\t\tNoItemsMessage: \"\",\n\t\tGui:            gui.intoInterface(),\n\n\t\tSort: func(a *commands.Project, b *commands.Project) bool {\n\t\t\treturn false\n\t\t},\n\t\tGetTableCells: presentation.GetProjectDisplayStrings,\n\t\t// It doesn't make sense to filter a list of only one item.\n\t\tDisableFilter: true,\n\t}\n}\n\nfunc (gui *Gui) refreshProject() error {\n\tgui.Panels.Projects.SetItems([]*commands.Project{{Name: gui.getProjectName()}})\n\treturn gui.Panels.Projects.RerenderList()\n}\n\nfunc (gui *Gui) getProjectName() string {\n\tprojectName := path.Base(gui.Config.ProjectDir)\n\tif gui.DockerCommand.InDockerComposeProject {\n\t\tfor _, service := range gui.Panels.Services.List.GetAllItems() {\n\t\t\tcontainer := service.Container\n\t\t\tif container != nil && container.DetailsLoaded() {\n\t\t\t\treturn container.Details.Config.Labels[\"com.docker.compose.project\"]\n\t\t\t}\n\t\t}\n\t}\n\n\treturn projectName\n}\n\nfunc (gui *Gui) renderCredits(_project *commands.Project) tasks.TaskFunc {\n\treturn gui.NewSimpleRenderStringTask(func() string { return gui.creditsStr() })\n}\n\nfunc (gui *Gui) creditsStr() string {\n\tvar configBuf bytes.Buffer\n\t_ = yaml.NewEncoder(&configBuf, yaml.IncludeOmitted).Encode(gui.Config.UserConfig)\n\n\treturn strings.Join(\n\t\t[]string{\n\t\t\tlazydockerTitle(),\n\t\t\t\"Copyright (c) 2019 Jesse Duffield\",\n\t\t\t\"Keybindings: https://github.com/jesseduffield/lazydocker/blob/master/docs/keybindings\",\n\t\t\t\"Config Options: https://github.com/jesseduffield/lazydocker/blob/master/docs/Config.md\",\n\t\t\t\"Raise an Issue: https://github.com/jesseduffield/lazydocker/issues\",\n\t\t\tutils.ColoredString(\"Buy Jesse a coffee: https://github.com/sponsors/jesseduffield\", color.FgMagenta), // caffeine ain't free\n\t\t\t\"Here's your lazydocker config when merged in with the defaults (you can open your config by pressing 'o'):\",\n\t\t\tutils.ColoredYamlString(configBuf.String()),\n\t\t}, \"\\n\\n\")\n}\n\nfunc (gui *Gui) renderAllLogs(_project *commands.Project) tasks.TaskFunc {\n\treturn gui.NewTask(TaskOpts{\n\t\tAutoscroll: true,\n\t\tWrap:       gui.Config.UserConfig.Gui.WrapMainPanel,\n\t\tFunc: func(ctx context.Context) {\n\t\t\tgui.clearMainView()\n\n\t\t\tcmd := gui.OSCommand.RunCustomCommand(\n\t\t\t\tutils.ApplyTemplate(\n\t\t\t\t\tgui.Config.UserConfig.CommandTemplates.AllLogs,\n\t\t\t\t\tgui.DockerCommand.NewCommandObject(commands.CommandObject{}),\n\t\t\t\t),\n\t\t\t)\n\n\t\t\tcmd.Stdout = gui.Views.Main\n\t\t\tcmd.Stderr = gui.Views.Main\n\n\t\t\tgui.OSCommand.PrepareForChildren(cmd)\n\t\t\t_ = cmd.Start()\n\n\t\t\tgo func() {\n\t\t\t\t<-ctx.Done()\n\t\t\t\tif err := gui.OSCommand.Kill(cmd); err != nil {\n\t\t\t\t\tgui.Log.Error(err)\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\t_ = cmd.Wait()\n\t\t},\n\t})\n}\n\nfunc (gui *Gui) renderDockerComposeConfig(_project *commands.Project) tasks.TaskFunc {\n\treturn gui.NewSimpleRenderStringTask(func() string {\n\t\treturn utils.ColoredYamlString(gui.DockerCommand.DockerComposeConfig())\n\t})\n}\n\nfunc (gui *Gui) handleOpenConfig(g *gocui.Gui, v *gocui.View) error {\n\treturn gui.openFile(gui.Config.ConfigFilename())\n}\n\nfunc (gui *Gui) handleEditConfig(g *gocui.Gui, v *gocui.View) error {\n\treturn gui.editFile(gui.Config.ConfigFilename())\n}\n\nfunc lazydockerTitle() string {\n\treturn `\n   _                     _            _\n  | |                   | |          | |\n  | | __ _ _____   _  __| | ___   ___| | _____ _ __\n  | |/ _` + \"`\" + ` |_  / | | |/ _` + \"`\" + ` |/ _ \\ / __| |/ / _ \\ '__|\n  | | (_| |/ /| |_| | (_| | (_) | (__|   <  __/ |\n  |_|\\__,_/___|\\__, |\\__,_|\\___/ \\___|_|\\_\\___|_|\n                __/ |\n               |___/\n`\n}\n\n// handleViewAllLogs switches to a subprocess viewing all the logs from docker-compose\nfunc (gui *Gui) handleViewAllLogs(g *gocui.Gui, v *gocui.View) error {\n\tc, err := gui.DockerCommand.ViewAllLogs()\n\tif err != nil {\n\t\treturn gui.createErrorPanel(err.Error())\n\t}\n\n\treturn gui.runSubprocess(c)\n}\n\n// Path: pkg/gui/services_panel.go\npackage gui\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\nfunc (gui *Gui) getServicesPanel() *panels.SideListPanel[*commands.Service] {\n\treturn &panels.SideListPanel[*commands.Service]{\n\t\tContextState: &panels.ContextState[*commands.Service]{\n\t\t\tGetMainTabs: func() []panels.MainTab[*commands.Service] {\n\t\t\t\treturn []panels.MainTab[*commands.Service]{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"logs\",\n\t\t\t\t\t\tTitle:  gui.Tr.LogsTitle,\n\t\t\t\t\t\tRender: gui.renderServiceLogs,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"stats\",\n\t\t\t\t\t\tTitle:  gui.Tr.StatsTitle,\n\t\t\t\t\t\tRender: gui.renderServiceStats,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"container-env\",\n\t\t\t\t\t\tTitle:  gui.Tr.ContainerEnvTitle,\n\t\t\t\t\t\tRender: gui.renderServiceContainerEnv,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"container-config\",\n\t\t\t\t\t\tTitle:  gui.Tr.ContainerConfigTitle,\n\t\t\t\t\t\tRender: gui.renderServiceContainerConfig,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"top\",\n\t\t\t\t\t\tTitle:  gui.Tr.TopTitle,\n\t\t\t\t\t\tRender: gui.renderServiceTop,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t},\n\t\t\tGetItemContextCacheKey: func(service *commands.Service) string {\n\t\t\t\tif service.Container == nil {\n\t\t\t\t\treturn \"services-\" + service.ID\n\t\t\t\t}\n\t\t\t\treturn \"services-\" + service.ID + \"-\" + service.Container.ID + \"-\" + service.Container.Container.State\n\t\t\t},\n\t\t},\n\t\tListPanel: panels.ListPanel[*commands.Service]{\n\t\t\tList: panels.NewFilteredList[*commands.Service](),\n\t\t\tView: gui.Views.Services,\n\t\t},\n\t\tNoItemsMessage: gui.Tr.NoServices,\n\t\tGui:            gui.intoInterface(),\n\t\t// sort services first by whether they have a linked container, and second by alphabetical order\n\t\tSort: func(a *commands.Service, b *commands.Service) bool {\n\t\t\tif a.Container != nil && b.Container == nil {\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\tif a.Container == nil && b.Container != nil {\n\t\t\t\treturn false\n\t\t\t}\n\n\t\t\treturn a.Name < b.Name\n\t\t},\n\t\tGetTableCells: func(service *commands.Service) []string {\n\t\t\treturn presentation.GetServiceDisplayStrings(&gui.Config.UserConfig.Gui, service)\n\t\t},\n\t\tHide: func() bool {\n\t\t\treturn !gui.DockerCommand.InDockerComposeProject\n\t\t},\n\t}\n}\n\nfunc (gui *Gui) renderServiceContainerConfig(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainer })\n\t}\n\n\treturn gui.renderContainerConfig(service.Container)\n}\n\nfunc (gui *Gui) renderServiceContainerEnv(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainer })\n\t}\n\n\treturn gui.renderContainerEnv(service.Container)\n}\n\nfunc (gui *Gui) renderServiceStats(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainer })\n\t}\n\n\treturn gui.renderContainerStats(service.Container)\n}\n\nfunc (gui *Gui) renderServiceTop(service *commands.Service) tasks.TaskFunc {\n\treturn gui.NewTickerTask(TickerTaskOpts{\n\t\tFunc: func(ctx context.Context, notifyStopped chan struct{}) {\n\t\t\tcontents, err := service.RenderTop(ctx)\n\t\t\tif err != nil {\n\t\t\t\tgui.RenderStringMain(err.Error())\n\t\t\t}\n\n\t\t\tgui.reRenderStringMain(contents)\n\t\t},\n\t\tDuration:   time.Second,\n\t\tBefore:     func(ctx context.Context) { gui.clearMainView() },\n\t\tWrap:       gui.Config.UserConfig.Gui.WrapMainPanel,\n\t\tAutoscroll: false,\n\t})\n}\n\nfunc (gui *Gui) renderServiceLogs(service *commands.Service) tasks.TaskFunc {\n\tif service.Container == nil {\n\t\treturn gui.NewSimpleRenderStringTask(func() string { return gui.Tr.NoContainerForService })\n\t}\n\n\treturn gui.renderContainerLogsToMain(service.Container)\n}\n\ntype commandOption struct {\n\tdescription string\n\tcommand     string\n\tonPress     func() error\n}\n\nfunc (r *commandOption) getDisplayStrings() []string {\n\treturn []string{r.description, color.New(color.FgCyan).Sprint(r.command)}\n}\n\nfunc (gui *Gui) handleServiceRemoveMenu(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tcomposeCommand := gui.Config.UserConfig.CommandTemplates.DockerCompose\n\n\toptions := []*commandOption{\n\t\t{\n\t\t\tdescription: gui.Tr.Remove,\n\t\t\tcommand:     fmt.Sprintf(\"%s rm --stop --force %s\", composeCommand, service.Name),\n\t\t},\n\t\t{\n\t\t\tdescription: gui.Tr.RemoveWithVolumes,\n\t\t\tcommand:     fmt.Sprintf(\"%s rm --stop --force -v %s\", composeCommand, service.Name),\n\t\t},\n\t}\n\n\tmenuItems := lo.Map(options, func(option *commandOption, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: option.getDisplayStrings(),\n\t\t\tOnPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.RemovingStatus, func() error {\n\t\t\t\t\tif err := gui.OSCommand.RunCommand(option.command); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle: \"\",\n\t\tItems: menuItems,\n\t})\n}\n\nfunc (gui *Gui) handleServicePause(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\tif service.Container == nil {\n\t\treturn nil\n\t}\n\n\treturn gui.PauseContainer(service.Container)\n}\n\nfunc (gui *Gui) handleServiceStop(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.createConfirmationPanel(gui.Tr.Confirm, gui.Tr.StopService, func(g *gocui.Gui, v *gocui.View) error {\n\t\treturn gui.WithWaitingStatus(gui.Tr.StoppingStatus, func() error {\n\t\t\tif err := service.Stop(); err != nil {\n\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t}\n\n\t\t\treturn nil\n\t\t})\n\t}, nil)\n}\n\nfunc (gui *Gui) handleServiceUp(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.WithWaitingStatus(gui.Tr.UppingServiceStatus, func() error {\n\t\tif err := service.Up(); err != nil {\n\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t}\n\n\t\treturn nil\n\t})\n}\n\nfunc (gui *Gui) handleServiceRestart(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.WithWaitingStatus(gui.Tr.RestartingStatus, func() error {\n\t\tif err := service.Restart(); err != nil {\n\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t}\n\n\t\treturn nil\n\t})\n}\n\nfunc (gui *Gui) handleServiceStart(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.WithWaitingStatus(gui.Tr.StartingStatus, func() error {\n\t\tif err := service.Start(); err != nil {\n\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t}\n\n\t\treturn nil\n\t})\n}\n\nfunc (gui *Gui) handleServiceAttach(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tif service.Container == nil {\n\t\treturn gui.createErrorPanel(gui.Tr.NoContainers)\n\t}\n\n\tc, err := service.Attach()\n\tif err != nil {\n\t\treturn gui.createErrorPanel(err.Error())\n\t}\n\n\treturn gui.runSubprocess(c)\n}\n\nfunc (gui *Gui) handleServiceRenderLogsToMain(g *gocui.Gui, v *gocui.View) error {\n\tservice, err := gui.Panels.Services.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n...\n", "needle_token_start": 8210, "needle_token_end": 8355, "code_context_ntokens": 15108, "output": ["```go\nfunc getContainerDisplaySubstatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tif !c.DetailsLoaded() {\n\t\treturn \"\"\n\t}\n\n\tswitch c.Container.State {\n\tcase \"exited\":\n\t\treturn utils.ColoredString(\n\t\t\tfmt.Sprintf(\"(%s)\", strconv.Itoa(c.Details.State.ExitCode)), getContainerColor(c),\n\t\t)\n\tcase \"running\":\n\t\treturn getHealthStatus(guiConfig, c)\n\tdefault:\n\t\treturn \"\"\n\t}\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "TestSplitLines", "language": "go", "path": "pkg/utils/utils_test.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To verify that a function correctly splits a string containing multiple lines into a list of individual lines, excluding the newline characters.\n2. **Input**: A series of strings that may contain multiple lines separated by newline characters.\n3. **Output**: Lists of strings, where each list contains the separate lines from the input string as its elements, with newline characters removed.\n4. **Procedure**: The test function defines several scenarios with different multiline strings. For each scenario, it checks if the function correctly splits the input string into a list of lines as expected, comparing the function's output to a predefined list of expected results.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/i18n/german.go\npackage i18n\n\nfunc germanSet() TranslationSet {\n\treturn TranslationSet{\n\t\tPruningStatus:              \"zerst\u00f6ren\",\n\t\tRemovingStatus:             \"entfernen\",\n\t\tRestartingStatus:           \"neustarten\",\n\t\tStoppingStatus:             \"anhalten\",\n\t\tRunningCustomCommandStatus: \"f\u00fchrt benutzerdefinierten Befehl aus\",\n\n\t\tNoViewMachingNewLineFocusedSwitchStatement: \"No view matching newLineFocused switch statement\",\n\n\t\tErrorOccurred:                     \"Es ist ein Fehler aufgetreten! Bitte erstelle ein Issue hier: https://github.com/jesseduffield/lazydocker/issues\",\n\t\tConnectionFailed:                  \"Verbindung zum Docker Client fehlgeschlagen. Du musst ggf. den Docker Client neustarten.\",\n\t\tUnattachableContainerError:        \"Der Container bietet keine Unterst\u00fctzung f\u00fcr das Anbinden. Du musst den Dienst entweder mit der '-it' Flagge benutzen oder `stdin_open: true, tty: true` in der docker-compose.yml Datei setzen.\",\n\t\tCannotAttachStoppedContainerError: \"Du kannst keinen angehaltenen Container anbinden. Du musst ihn erst starten (was du tun kannst, indem du 'r' dr\u00fcckst), (ja, ich bin zu faul um das zu automatisieren) (aber ist schon cool, dass ich so eine Konversation durch eine Fehlermeldung mit dir f\u00fchren kann)\",\n\t\tCannotAccessDockerSocketError:     \"Kann nicht auf den Socket zugreifen: unix:///var/run/docker.sock\\nF\u00fchre lazydocker als root aus oder lese https://docs.docker.com/install/linux/linux-postinstall/\",\n\n\t\tDonate:  \"Spenden\",\n\t\tConfirm: \"Best\u00e4tigen\",\n\n\t\tReturn:             \"zur\u00fcck\",\n\t\tFocusMain:          \"fokussieren aufs Hauptpanel\",\n\t\tNavigate:           \"navigieren\",\n\t\tExecute:            \"ausf\u00fchren\",\n\t\tClose:              \"schlie\u00dfen\",\n\t\tMenu:               \"men\u00fc\",\n\t\tMenuTitle:          \"Men\u00fc\",\n\t\tScroll:             \"scrollen\",\n\t\tOpenConfig:         \"\u00f6ffne lazydocker Konfiguration\",\n\t\tEditConfig:         \"bearbeite lazydocker Konfiguration\",\n\t\tCancel:             \"abbrechen\",\n\t\tRemove:             \"entfernen\",\n\t\tForceRemove:        \"Entfernen erzwingen\",\n\t\tRemoveWithVolumes:  \"entferne mit Volumes\",\n\t\tRemoveService:      \"entferne Container\",\n\t\tStop:               \"anhalten\",\n\t\tRestart:            \"neustarten\",\n\t\tRebuild:            \"neubauen\",\n\t\tRecreate:           \"neuerstellen\",\n\t\tPreviousContext:    \"vorheriges Tab\",\n\t\tNextContext:        \"n\u00e4chstes Tab\",\n\t\tAttach:             \"anbinden\",\n\t\tViewLogs:           \"zeige Protokolle\",\n\t\tRemoveImage:        \"entferne Image\",\n\t\tRemoveVolume:       \"entferne Volume\",\n\t\tRemoveNetwork:      \"entferne Netzwerk\",\n\t\tRemoveWithoutPrune: \"entfernen, ohne die unmarkierten Eltern zu entfernen\",\n\t\tPruneContainers:    \"entferne verlassene Container\",\n\t\tPruneVolumes:       \"entferne unbenutzte Volumes\",\n\t\tPruneNetworks:      \"entferne unbenutzte Netzwerk\",\n\t\tPruneImages:        \"entferne unbenutzte Images\",\n\t\tViewRestartOptions: \"zeige Neustartoptionen\",\n\t\tRunCustomCommand:   \"f\u00fchre vordefinierten benutzerdefinierten Befehl aus\",\n\n\t\tGlobalTitle:               \"Global\",\n\t\tMainTitle:                 \"Haupt\",\n\t\tProjectTitle:              \"Projekt\",\n\t\tServicesTitle:             \"Dienste\",\n\t\tContainersTitle:           \"Container\",\n\t\tStandaloneContainersTitle: \"Alleinstehende Container\",\n\t\tImagesTitle:               \"Images\",\n\t\tVolumesTitle:              \"Volumes\",\n\t\tNetworksTitle:             \"Netzwerk\",\n\t\tCustomCommandTitle:        \"Benutzerdefinierter Befehl\",\n\t\tErrorTitle:                \"Fehler\",\n\t\tLogsTitle:                 \"Protokoll\",\n\t\tConfigTitle:               \"Konfiguration\",\n\t\tEnvTitle:                  \"Env\",\n\t\tDockerComposeConfigTitle:  \"Docker-Compose Konfiguration\",\n\t\tTopTitle:                  \"Top\",\n\t\tStatsTitle:                \"Statistiken\",\n\t\tCreditsTitle:              \"\u00dcber Uns\",\n\t\tContainerConfigTitle:      \"Container Konfiguration\",\n\t\tContainerEnvTitle:         \"Container Env\",\n\t\tNothingToDisplay:          \"Nothing to display\",\n\t\tCannotDisplayEnvVariables: \"Something went wrong while displaying environment variables\",\n\n\t\tNoContainers: \"Keine Container\",\n\t\tNoContainer:  \"Kein Container\",\n\t\tNoImages:     \"Keine Images\",\n\t\tNoVolumes:    \"Keine Volumes\",\n\t\tNoNetworks:   \"Keine Netzwerk\",\n\n\t\tConfirmQuit:                 \"Bist du dir sicher, dass du verlassen m\u00f6chtest?\",\n\t\tMustForceToRemoveContainer:  \"Du kannst keinen Container entfernen, der noch ausgef\u00fchrt wird au\u00dfer du erzwingst es. M\u00f6chtest du es erzwingen?\",\n\t\tNotEnoughSpace:              \"Nicht genug Platz um die Panel darzustellen\",\n\t\tConfirmPruneImages:          \"Bist du dir sicher, dass du alle unbenutzten Images entfernen m\u00f6chtest?\",\n\t\tConfirmPruneContainers:      \"Bist du dir sicher, dass du alle angehaltenen Container entfernen m\u00f6chtes?\",\n...\n// Path: pkg/i18n/polish.go\npackage i18n\n\nfunc polishSet() TranslationSet {\n\treturn TranslationSet{\n\t\tPruningStatus:              \"czyszczenie\",\n\t\tRemovingStatus:             \"usuwanie\",\n\t\tRestartingStatus:           \"restartowanie\",\n\t\tStoppingStatus:             \"zatrzymywanie\",\n\t\tRunningCustomCommandStatus: \"uruchamianie w\u0142asnej komendty\",\n\n\t\tNoViewMachingNewLineFocusedSwitchStatement: \"\u017baden widok nie odpowiada instrukcji prze\u0142\u0105czenia newLineFocused\",\n\n\t\tErrorOccurred:                     \"Wyst\u0105pi\u0142 b\u0142\u0105d! Prosz\u0119 go zg\u0142osi\u0107 na https://github.com/jesseduffield/lazydocker/issues\",\n\t\tConnectionFailed:                  \"B\u0142\u0105d po\u0142\u0105czenia z Dockerem. By\u0107 mo\u017ce nale\u017cy go zrestartowa\u0107.\",\n\t\tUnattachableContainerError:        \"Kontener nie obs\u0142uguje przyczepiania (attach). Musisz albo u\u017cy\u0107 flag '-it', albo `stdin_open: true, tty: true` w pliku docker-compose.yml.\",\n\t\tCannotAttachStoppedContainerError: \"Nie mo\u017cna przyczepi\u0107 si\u0119 do zatrzymanego kontenera, nale\u017cy go najpierw uruchomi\u0107 (co mo\u017cna wykona\u0107 wciskaj\u0105c przycisk 'r')\",\n\t\tCannotAccessDockerSocketError:     \"Nie uda\u0142o si\u0119\u00a0uzyska\u0107 dost\u0119pu do unix:///var/run/docker.sock\\nUruchom program jako root lub przeczytaj https://docs.docker.com/install/linux/linux-postinstall/\",\n\n\t\tDonate:  \"Dotacja\",\n\t\tConfirm: \"Potwierd\u017a\",\n\n\t\tReturn:             \"powr\u00f3t\",\n\t\tFocusMain:          \"skup na g\u0142\u00f3wnym panelu\",\n\t\tNavigate:           \"nawigowanie\",\n\t\tExecute:            \"wykonaj\",\n\t\tClose:              \"zamknij\",\n\t\tMenu:               \"menu\",\n\t\tMenuTitle:          \"Menu\",\n\t\tScroll:             \"przewi\u0144\",\n\t\tOpenConfig:         \"otw\u00f3rz konfiguracj\u0119\",\n\t\tEditConfig:         \"edytuj konfiguracj\u0119\",\n\t\tCancel:             \"anuluj\",\n\t\tRemove:             \"usu\u0144\",\n\t\tForceRemove:        \"usu\u0144 si\u0142\u0105\",\n\t\tRemoveWithVolumes:  \"usu\u0144 z wolumenami\",\n\t\tRemoveService:      \"usu\u0144 kontenery\",\n\t\tStop:               \"zatrzymaj\",\n\t\tRestart:            \"restartuj\",\n\t\tRebuild:            \"przebuduj\",\n\t\tRecreate:           \"odtw\u00f3rz\",\n\t\tPreviousContext:    \"poprzednia zak\u0142adka\",\n\t\tNextContext:        \"nast\u0119pna zak\u0142adka\",\n\t\tAttach:             \"przyczep\",\n\t\tViewLogs:           \"poka\u017c logi\",\n\t\tRemoveImage:        \"usu\u0144 obraz\",\n\t\tRemoveVolume:       \"usu\u0144 wolumen\",\n\t\tRemoveNetwork:      \"usu\u0144 sieci\",\n\t\tRemoveWithoutPrune: \"usu\u0144 bez kasowania nieoznaczonych rodzic\u00f3w\",\n\t\tPruneContainers:    \"wyczy\u015b\u0107\u00a0kontenery\",\n\t\tPruneVolumes:       \"wyczy\u015b\u0107 nieu\u017cywane wolumeny\",\n\t\tPruneNetworks:      \"wyczy\u015b\u0107 nieu\u017cywane sieci\",\n\t\tPruneImages:        \"wyczy\u015b\u0107 nieu\u017cywane obrazy\",\n\t\tViewRestartOptions: \"poka\u017c opcje restartu\",\n\t\tRunCustomCommand:   \"wykonaj predefiniowan\u0105 w\u0142asn\u0105 komende\",\n\n\t\tGlobalTitle:               \"Globalne\",\n\t\tMainTitle:                 \"G\u0142\u00f3wne\",\n\t\tProjectTitle:              \"Projekt\",\n\t\tServicesTitle:             \"Serwisy\",\n\t\tContainersTitle:           \"Kontenery\",\n\t\tStandaloneContainersTitle: \"Kontenery samodzielne\",\n\t\tImagesTitle:               \"Obrazy\",\n\t\tVolumesTitle:              \"Wolumeny\",\n\t\tNetworksTitle:             \"Sieci\",\n\t\tCustomCommandTitle:        \"W\u0142asna komenda:\",\n\t\tErrorTitle:                \"B\u0142\u0105d\",\n\t\tLogsTitle:                 \"Logi\",\n\t\tConfigTitle:               \"Konfiguracja\",\n\t\tEnvTitle:                  \"Env\",\n\t\tDockerComposeConfigTitle:  \"Konfiguracja docker-compose\",\n\t\tTopTitle:                  \"Top\",\n\t\tStatsTitle:                \"Staty\",\n\t\tCreditsTitle:              \"O\",\n\t\tContainerConfigTitle:      \"Konfiguracja kontenera\",\n\t\tContainerEnvTitle:         \"Container Env\",\n\t\tNothingToDisplay:          \"Nothing to display\",\n\t\tCannotDisplayEnvVariables: \"Something went wrong while displaying environment variables\",\n\n\t\tNoContainers: \"Brak kontener\u00f3w\",\n\t\tNoContainer:  \"Brak kontenera\",\n\t\tNoImages:     \"Brak obraz\u00f3w\",\n\t\tNoVolumes:    \"Brak wolumen\u00f3w\",\n\t\tNoNetworks:   \"Brak sieci\",\n\n\t\tConfirmQuit:                 \"Na pewno chcesz wyj\u015b\u0107?\",\n\t\tMustForceToRemoveContainer:  \"Nie mo\u017cesz usun\u0105\u0107\u00a0uruchomionego kontenera dop\u00f3ki nie zrobisz tego si\u0142\u0105. Chcesz wykona\u0107 to z si\u0142\u0105?\",\n\t\tNotEnoughSpace:              \"Niedostateczna ilo\u015b\u0107 miejsca do wy\u015bwietlenia paneli\",\n\t\tConfirmPruneImages:          \"Na pewno wyczy\u015bci\u0107 wszystkie nieu\u017cywane obrazy?\",\n\t\tConfirmPruneContainers:      \"Na pewno wyczy\u015bci\u0107 wszystkie nieuruchomione kontenery?\",\n\t\tConfirmPruneVolumes:         \"Na pewno wyczy\u015bci\u0107 wszystkie nieu\u017cywane wolumeny?\",\n\t\tConfirmPruneNetworks:        \"Na pewno wyczy\u015bci\u0107 wszystkie nieu\u017cywane sieci?\",\n\t\tStopService:                 \"Na pewno zatrzyma\u0107 kontenery tego serwisu?\",\n\t\tStopContainer:               \"Na pewno zatrzyma\u0107 ten kontener?\",\n\t\tPressEnterToReturn:          \"Wci\u015bnij enter aby powr\u00f3ci\u0107 do lazydockera (ten komunikat mo\u017ce by\u0107 wy\u0142\u0105czony w konfiguracji poprzez ustawienie `gui.returnImmediately: true`)\",\n\t\tDetachFromContainerShortCut: \"Domy\u015blnie, aby od\u0142\u0105czy\u0107 si\u0119 od kontenera, naci\u015bnij ctrl-p, a nast\u0119pnie ctrl-q\",\n\t}\n}\n\n// Path: pkg/i18n/portuguese.go\npackage i18n\n\nfunc portugueseSet() TranslationSet {\n\treturn TranslationSet{\n\t\tPruningStatus:              \"destruindo\",\n\t\tRemovingStatus:             \"removendo\",\n\t\tRestartingStatus:           \"reiniciando\",\n\t\tStartingStatus:             \"iniciando\",\n\t\tStoppingStatus:             \"parando\",\n\t\tUppingServiceStatus:        \"subindo servi\u00e7o\",\n\t\tUppingProjectStatus:        \"subindo projeto\",\n\t\tDowningStatus:              \"derrubando\",\n\t\tPausingStatus:              \"pausando\",\n\t\tRunningCustomCommandStatus: \"executando comando personalizado\",\n\t\tRunningBulkCommandStatus:   \"executando comando em massa\",\n\n\t\tNoViewMachingNewLineFocusedSwitchStatement: \"No view matching newLineFocused switch statement\",\n\n\t\tErrorOccurred:                     \"Um erro ocorreu! Por favor, crie uma issue em https://github.com/jesseduffield/lazydocker/issues\",\n\t\tConnectionFailed:                  \"Falha na conex\u00e3o com o cliente Docker. Voc\u00ea pode precisar reiniciar o seu cliente Docker\",\n\t\tUnattachableContainerError:        \"O cont\u00eainer n\u00e3o suporta anexa\u00e7\u00e3o. Voc\u00ea deve executar o servi\u00e7o com a flag '-it' ou usar `stdin_open: true, tty: true` no arquivo docker-compose.yml\",\n\t\tWaitingForContainerInfo:           \"N\u00e3o \u00e9 poss\u00edvel prosseguir at\u00e9 que o Docker forne\u00e7a mais informa\u00e7\u00f5es sobre o cont\u00eainer. Por favor, tente novamente em alguns momentos.\",\n\t\tCannotAttachStoppedContainerError: \"Voc\u00ea n\u00e3o pode anexar a um cont\u00eainer parado, voc\u00ea precisa inici\u00e1-lo primeiro (o que voc\u00ea pode fazer com a tecla 'r') (sim, sou pregui\u00e7oso demais para fazer isso automaticamente para voc\u00ea) (ali\u00e1s, bem legal que eu posso me comunicar diretamente com voc\u00ea na forma de uma mensagem de erro)\",\n\t\tCannotAccessDockerSocketError:     \"N\u00e3o \u00e9 poss\u00edvel acessar o s\u00f4quete docker em: unix:///var/run/docker.sock\\nExecute o lazydocker como root ou leia https://docs.docker.com/install/linux/linux-postinstall/\",\n\t\tCannotKillChildError:              \"Tr\u00eas segundos foram esperarados para que os processos filhos parassem. Pode haver um processo \u00f3rf\u00e3o que continua em execu\u00e7\u00e3o em seu sistema.\",\n\n\t\tDonate:  \"Doar\",\n\t\tConfirm: \"Confirmar\",\n\n\t\tReturn:                      \"retornar\",\n\t\tFocusMain:                   \"focar no painel principal\",\n\t\tLcFilter:                    \"filtrar lista\",\n\t\tNavigate:                    \"navegar\",\n\t\tExecute:                     \"executar\",\n\t\tClose:                       \"fechar\",\n\t\tQuit:                        \"sair\",\n\t\tMenu:                        \"menu\",\n\t\tMenuTitle:                   \"Menu\",\n\t\tScroll:                      \"rolar\",\n\t\tOpenConfig:                  \"abrir configura\u00e7\u00e3o do lazydocker\",\n\t\tEditConfig:                  \"editar configura\u00e7\u00e3o do lazydocker\",\n\t\tCancel:                      \"cancelar\",\n\t\tRemove:                      \"remover\",\n\t\tHideStopped:                 \"ocultar/mostrar cont\u00eaineres parados\",\n\t\tForceRemove:                 \"for\u00e7ar remo\u00e7\u00e3o\",\n\t\tRemoveWithVolumes:           \"remover com volumes\",\n\t\tRemoveService:               \"remover cont\u00eaineres\",\n\t\tUpService:                   \"subir servi\u00e7o\",\n\t\tStop:                        \"parar\",\n\t\tPause:                       \"pausar\",\n\t\tRestart:                     \"reiniciar\",\n\t\tDown:                        \"derrubar projeto\",\n\t\tDownWithVolumes:             \"derrubar projetos com volumes\",\n\t\tStart:                       \"iniciar\",\n\t\tRebuild:                     \"reconstuir\",\n\t\tRecreate:                    \"recriar\",\n\t\tPreviousContext:             \"aba anterior\",\n\t\tNextContext:                 \"pr\u00f3xima aba\",\n\t\tAttach:                      \"anexar\",\n\t\tViewLogs:                    \"ver logs\",\n\t\tUpProject:                   \"subir projeto\",\n\t\tDownProject:                 \"derrubar projeto\",\n\t\tRemoveImage:                 \"remover imagem\",\n\t\tRemoveVolume:                \"remover volume\",\n\t\tRemoveNetwork:               \"remover rede\",\n\t\tRemoveWithoutPrune:          \"remover sem deletar pais n\u00e3o etiquetados\",\n\t\tRemoveWithoutPruneWithForce: \"remover (for\u00e7ado) sem deletar pais n\u00e3o etiquetados\",\n\t\tRemoveWithForce:             \"remover (for\u00e7ado)\",\n\t\tPruneContainers:             \"destruir cont\u00eaineres encerrados\",\n\t\tPruneVolumes:                \"destruir volumes n\u00e3o utilizados\",\n\t\tPruneNetworks:               \"destruir redes n\u00e3o utilizadas\",\n\t\tPruneImages:                 \"destruir imagens n\u00e3o utilizadas\",\n\t\tStopAllContainers:           \"parar todos os cont\u00eaineres\",\n\t\tRemoveAllContainers:         \"remover todos os cont\u00eaineres (for\u00e7ado)\",\n\t\tViewRestartOptions:          \"ver op\u00e7\u00f5es de rein\u00edcio\",\n\t\tExecShell:                   \"executar shell\",\n\t\tRunCustomCommand:            \"executar comando personalizado predefinido\",\n\t\tViewBulkCommands:            \"ver comandos em massa\",\n\t\tFilterList:                  \"filtrar lista\",\n\t\tOpenInBrowser:               \"abrir no navegador (primeira porta \u00e9 http)\",\n\t\tSortContainersByState:       \"ordenar cont\u00eaineres por estado\",\n\n\t\tGlobalTitle:               \"Global\",\n\t\tMainTitle:                 \"Principal\",\n\t\tProjectTitle:              \"Projeto\",\n\t\tServicesTitle:             \"Servi\u00e7os\",\n\t\tContainersTitle:           \"Cont\u00eaineres\",\n\t\tStandaloneContainersTitle: \"Cont\u00eaineres Avulsos\",\n\t\tImagesTitle:               \"Imagens\",\n\t\tVolumesTitle:              \"Volumes\",\n\t\tNetworksTitle:             \"Redes\",\n\t\tCustomCommandTitle:        \"Comando Personalizado:\",\n\t\tBulkCommandTitle:          \"Comando em Massa:\",\n\t\tErrorTitle:                \"Erro\",\n\t\tLogsTitle:                 \"Registros\",\n\t\tConfigTitle:               \"Config\",\n\t\tEnvTitle:                  \"Env\",\n\t\tDockerComposeConfigTitle:  \"Docker-Compose Config\",\n\t\tTopTitle:                  \"Topo\",\n\t\tStatsTitle:                \"Estat\u00edsticas\",\n\t\tCreditsTitle:              \"Sobre\",\n\t\tContainerConfigTitle:      \"Configura\u00e7\u00e3o do Cont\u00eainer\",\n\t\tContainerEnvTitle:         \"Cont\u00eainer Env\",\n\t\tNothingToDisplay:          \"Nada a exibir\",\n\t\tNoContainerForService:     \"Nenhum log para exibir; o servi\u00e7o n\u00e3o est\u00e1 associado a nenhum cont\u00eainer\",\n\t\tCannotDisplayEnvVariables: \"Algo deu errado ao exibir as vari\u00e1veis de ambiente\",\n\n\t\tNoContainers: \"Sem cont\u00eaineres\",\n\t\tNoContainer:  \"Sem cont\u00eainer\",\n\t\tNoImages:     \"Sem imagens\",\n\t\tNoVolumes:    \"Sem volumes\",\n\t\tNoNetworks:   \"Sem redes\",\n\t\tNoServices:   \"Sem servi\u00e7os\",\n\n\t\tConfirmQuit:                 \"Tem certeza que deseja sair?\",\n\t\tConfirmUpProject:            \"Tem certeza que deseja 'iniciar' seu projeto docker compose?\",\n\t\tMustForceToRemoveContainer:  \"Voc\u00ea n\u00e3o pode remover um cont\u00eainer em execu\u00e7\u00e3o a menos que o force. Deseja for\u00e7ar?\",\n\t\tNotEnoughSpace:              \"Sem espa\u00e7o suficiente para renderizar os pain\u00e9is\",\n\t\tConfirmPruneImages:          \"Tem certeza que deseja eliminar todas as imagens n\u00e3o utilizadas?\",\n\t\tConfirmPruneContainers:      \"Tem certeza que deseja destruir todos os cont\u00eaineres parados?\",\n\t\tConfirmStopContainers:       \"Tem certeza que deseja parar todos os cont\u00eaineres?\",\n\t\tConfirmRemoveContainers:     \"Tem certeza que deseja remover todos os cont\u00eaineres?\",\n\t\tConfirmPruneVolumes:         \"Tem certeza que deseja destruir todos os volumes n\u00e3o utilizados?\",\n\t\tConfirmPruneNetworks:        \"Tem certeza que deseja destruir todos as redes n\u00e3o utilizadas?\",\n\t\tStopService:                 \"Tem certeza que deseja parar os cont\u00eaineres deste servi\u00e7o?\",\n\t\tStopContainer:               \"Tem certeza que deseja parar este cont\u00eainer?\",\n\t\tPressEnterToReturn:          \"Pressione enter para retornar ao lazydocker (este prompt pode ser desativado em sua configura\u00e7\u00e3o definindo `gui.returnImmediately: true`)\",\n\t\tDetachFromContainerShortCut: \"Por padr\u00e3o, para desanexar do cont\u00eainer, pressione ctrl-p e depois ctrl-q\",\n\n\t\tNo:  \"n\u00e3o\",\n\t\tYes: \"sim\",\n\n\t\tLcNextScreenMode: \"modo de tela seguinte (normal/meia/tela cheia)\",\n\t\tLcPrevScreenMode: \"modo de tela anterior\",\n\t\tFilterPrompt:     \"filtro\",\n\t}\n}\n\n// Path: pkg/i18n/spanish.go\npackage i18n\n\nfunc spanishSet() TranslationSet {\n\treturn TranslationSet{\n\t\tPruningStatus:              \"limpiando\",\n\t\tRemovingStatus:             \"eliminando\",\n\t\tRestartingStatus:           \"reiniciando\",\n\t\tStartingStatus:             \"iniciando\",\n\t\tStoppingStatus:             \"terminando\",\n\t\tUppingServiceStatus:        \"levantando servicio\",\n\t\tUppingProjectStatus:        \"levantando proyecto\",\n\t\tDowningStatus:              \"dando de baja\",\n\t\tPausingStatus:              \"pausando\",\n\t\tRunningCustomCommandStatus: \"ejecutando comando personalizado\",\n\t\tRunningBulkCommandStatus:   \"ejecutando comando masivo\",\n\n\t\tErrorOccurred:                 \"\u00a1Hubo un error! Por favor crea un issue en https://github.com/jesseduffield/lazydocker/issues\",\n\t\tConnectionFailed:              \"Fall\u00f3 la conexi\u00f3n con el docker client. Quiz\u00e1 necesitas reiniciar tu docker client\",\n\t\tUnattachableContainerError:    \"Container does not support attaching. You must either run the service with the '-it' flag or use `stdin_open: true, tty: true` in the docker-compose.yml file\",\n\t\tWaitingForContainerInfo:       \"No podemos proceder hasta que docker nos de m\u00e1s informaci\u00f3n sobre el contenedor. Int\u00e9ntalo otra vez en unos segundos.\",\n\t\tCannotAccessDockerSocketError: \"No es posible acceder al docker socket en: unix:///var/run/docker.sock\\nEjecuta lazydocker como root o lee https://docs.docker.com/install/linux/linux-postinstall/\",\n\t\tCannotKillChildError:          \"Esperamos tres segundos a que el proceso hijo se detenga. Debe de haber un proceso hu\u00e9rfano que continua activo en tu sistema.\",\n\n\t\tDonate:  \"Donar\",\n\t\tConfirm: \"Confirmar\",\n\n\t\tReturn:                      \"regresar\",\n\t\tFocusMain:                   \"enfocar panel principal\",\n\t\tLcFilter:                    \"filtrar lista\",\n\t\tNavigate:                    \"navegar\",\n\t\tExecute:                     \"ejecutar\",\n\t\tClose:                       \"cerrar\",\n\t\tQuit:                        \"salir\",\n\t\tMenu:                        \"men\u00fa\",\n\t\tMenuTitle:                   \"Men\u00fa\",\n\t\tOpenConfig:                  \"abrir configuraci\u00f3n de lazydocker\",\n\t\tEditConfig:                  \"editar configuraci\u00f3n de lazydocker\",\n\t\tCancel:                      \"cancelar\",\n\t\tRemove:                      \"borrar\",\n\t\tHideStopped:                 \"esconder/mostrar contenedores parados\",\n\t\tForceRemove:                 \"borrar(forzado)\",\n\t\tRemoveWithVolumes:           \"borrar con vol\u00famenes\",\n\t\tRemoveService:               \"borrar contenedores\",\n\t\tUpService:                   \"levantar servicio\",\n\t\tStop:                        \"parar\",\n\t\tPause:                       \"pausa\",\n\t\tRestart:                     \"reiniciar\",\n\t\tDown:                        \"bajar proyecto\",\n\t\tDownWithVolumes:             \"bajar proyecto con vol\u00famenes\",\n\t\tStart:                       \"iniciar\",\n\t\tRebuild:                     \"recompilar\",\n\t\tRecreate:                    \"recrear\",\n\t\tPreviousContext:             \"anterior pesta\u00f1a\",\n\t\tNextContext:                 \"siguiente pesta\u00f1a\",\n\t\tViewLogs:                    \"ver logs\",\n\t\tUpProject:                   \"levantar proyecto\",\n\t\tDownProject:                 \"dar de baja el proyecto\",\n\t\tRemoveImage:                 \"limpiar imagen\",\n\t\tRemoveVolume:                \"limpiar vol\u00famen\",\n\t\tRemoveNetwork:               \"limpiar red\",\n\t\tRemoveWithoutPrune:          \"limpiar sin borrar padres sin etiqueta\",\n\t\tRemoveWithoutPruneWithForce: \"limpiar (forzado) sin borrar padres sin etiqueta\",\n\t\tRemoveWithForce:             \"limpiar (forzado)\",\n\t\tPruneContainers:             \"limpiar contenedores finalizados\",\n\t\tPruneVolumes:                \"limpiar vol\u00famenes sin usar\",\n\t\tPruneNetworks:               \"limpiar redes sin usar\",\n\t\tPruneImages:                 \"limpiar im\u00e1genes sin usar\",\n\t\tStopAllContainers:           \"detener todos los contenedores\",\n\t\tRemoveAllContainers:         \"borrar todos los contenedores (forzado)\",\n\t\tViewRestartOptions:          \"ver opciones de reinicio\",\n\t\tExecShell:                   \"ejecutar shell\",\n\t\tRunCustomCommand:            \"ejecutar comando personalizado\",\n\t\tViewBulkCommands:            \"ver comandos masivos\",\n\t\tFilterList:                  \"filtar list\",\n\t\tOpenInBrowser:               \"abrir en navegador (first port is http)\",\n\t\tSortContainersByState:       \"ordenar contenedores por estado\",\n\n\t\tGlobalTitle:               \"Global\",\n\t\tMainTitle:                 \"Inicio\",\n\t\tProjectTitle:              \"Proyecto\",\n\t\tServicesTitle:             \"Servicios\",\n\t\tContainersTitle:           \"Contenedores\",\n\t\tStandaloneContainersTitle: \"Contenedores independientes\",\n\t\tImagesTitle:               \"Im\u00e1genes\",\n\t\tVolumesTitle:              \"Vol\u00famenes\",\n\t\tNetworksTitle:             \"Redes\",\n\t\tCustomCommandTitle:        \"Comando personalizado:\",\n\t\tBulkCommandTitle:          \"Comando masivo:\",\n\t\tErrorTitle:                \"Error\",\n\t\tLogsTitle:                 \"Logs\",\n\t\tConfigTitle:               \"Configuraci\u00f3n\",\n\t\tEnvTitle:                  \"Variables de entorno\",\n\t\tDockerComposeConfigTitle:  \"Docker-Compose Config\",\n\t\tTopTitle:                  \"Top\",\n\t\tStatsTitle:                \"Estad\u00edsticas\",\n\t\tCreditsTitle:              \"Acerca\",\n\t\tContainerConfigTitle:      \"Configuraci\u00f3n\",\n\t\tContainerEnvTitle:         \"Variables de entorno\",\n\t\tNothingToDisplay:          \"Nada que mostrar\",\n\t\tNoContainerForService:     \"No hay logs que mostrar; el servicio no est\u00e1 asociado con un contenedor\",\n\t\tCannotDisplayEnvVariables: \"Algo sali\u00f3 mal mientras se mostraban las variables de entorno\",\n\n\t\tNoContainers: \"Sin contenedores\",\n\t\tNoContainer:  \"Sin contenedor\",\n\t\tNoImages:     \"Sin im\u00e1genes\",\n\t\tNoVolumes:    \"Sin vol\u00famenes\",\n\t\tNoNetworks:   \"Sin redes\",\n\t\tNoServices:   \"Sin servicios\",\n\n\t\tConfirmQuit:                \"\u00bfRealmente quieres salir?\",\n\t\tConfirmUpProject:           \"\u00bfRealmente quieres levantar tu proyecto docker compose?\",\n\t\tMustForceToRemoveContainer: \"No puedes borrar un contenedor en ejecuci\u00f3n a menos de que lo fuerces, \u00bfquieres hacerlo?\",\n\t\tNotEnoughSpace:             \"No hay suficiente espacio para renderizar los paneles\",\n\t\tConfirmPruneImages:         \"\u00bfRealmente quieres limpiar todas tus im\u00e1genes?\",\n\t\tConfirmPruneContainers:     \"\u00bfRealmente quieres limpiar todos los contenedores finalizados?\",\n\t\tConfirmStopContainers:      \"\u00bfRealmente quieres detener todos los contenedores?\",\n\t\tConfirmRemoveContainers:    \"\u00bfRealmente quieres borrar todos los contenedores?\",\n\t\tConfirmPruneVolumes:        \"\u00bfRealmente quieres limpiar todos los v\u00f3lumenes sin usar?\",\n\t\tConfirmPruneNetworks:       \"\u00bfRealmente quieres limpiar todas las redes sin usar?\",\n\t\tStopService:                \"\u00bfRealmente quieres detener los contenedores de este servicio?\",\n\t\tStopContainer:              \"\u00bfRealmente quieres detener este contenedor?\",\n\t\tPressEnterToReturn:         \"Presionar [enter] para volver a lazydocker (este mensaje puede ser desactivado en tu configuraci\u00f3n poniendo `gui.returnImmediately: true`)\",\n\n\t\tNo:  \"no\",\n\t\tYes: \"s\u00ed\",\n\n\t\tFilterPrompt: \"filtrar\",\n\t}\n}\n\n// Path: pkg/i18n/turkish.go\npackage i18n\n\nfunc turkishSet() TranslationSet {\n\treturn TranslationSet{\n\t\tPruningStatus:              \"temizleniyor\",\n\t\tRemovingStatus:             \"kald\u0131r\u0131l\u0131yor\",\n\t\tRestartingStatus:           \"yeniden ba\u015flat\u0131l\u0131yor\",\n\t\tStoppingStatus:             \"durduruluyor\",\n\t\tRunningCustomCommandStatus: \"\u00f6zel komut \u00e7al\u0131\u015ft\u0131r\",\n\n\t\tNoViewMachingNewLineFocusedSwitchStatement: \"NewLineFocused anahtar deyimi ile e\u015fle\u015fen g\u00f6r\u00fcn\u00fcm yok\",\n\n\t\tErrorOccurred:                     \"Bir hata olu\u015ftu! L\u00fctfen https://github.com/jesseduffield/lazydocker/issues adresinden bir hataya ili\u015fkin konu olu\u015fturun\",\n\t\tConnectionFailed:                  \"Docker ba\u011flant\u0131s\u0131 ba\u015far\u0131s\u0131z oldu. Docker' \u0131 yeniden ba\u015flatman\u0131z gerekebilir\",\n\t\tUnattachableContainerError:        \"Konteyner attaching modunda \u00e7al\u0131\u015fmay\u0131 desteklemiyor. Hizmeti '-it' opsiyonu ile \u00e7al\u0131\u015ft\u0131rman\u0131z veya docker-compose.yml dosyas\u0131nda `stdin_open: true, tty: true` kullanman\u0131z gerekir.\",\n\t\tCannotAttachStoppedContainerError: \"Durdurulan konteynera ba\u011flanamazs\u0131n\u0131z, ilk \u00f6nce ba\u015flatman\u0131z gerekir (asl\u0131nda ba\u015flatmay\u0131 r tu\u015fu ile yapabilirsiniz) (evet, senin i\u00e7in bunu otomatik olarak yapabilirim fakat \u00e7ok tembelim) (hata mesaj\u0131 ile seninle birebir ileti\u015fim kurmam \u00e7ok daha g\u00fczel)\",\n\t\tCannotAccessDockerSocketError:     \"Docker' a \u015fu adresten eri\u015filemiyor : unix:///var/run/docker.sock\\n lazydocker' \u0131 root(k\u00f6k kullan\u0131c\u0131) olarak \u00e7al\u0131\u015ft\u0131r veya \u015fu adresteki ad\u0131mlar\u0131 takip et : https://docs.docker.com/install/linux/linux-postinstall/\",\n\n\t\tDonate:  \"Ba\u011f\u0131\u015f\",\n\t\tConfirm: \"Onayla\",\n\n\t\tReturn:             \"d\u00f6n\u00fc\u015f\",\n\t\tFocusMain:          \"ana panele odaklan\",\n\t\tNavigate:           \"gezin\",\n\t\tExecute:            \"\u00e7al\u0131\u015ft\u0131r\",\n\t\tClose:              \"kapat\",\n\t\tMenu:               \"men\u00fc\",\n\t\tMenuTitle:          \"Men\u00fc\",\n\t\tScroll:             \"kayd\u0131r\",\n\t\tOpenConfig:         \"lazydocker ayarlar\u0131n\u0131 a\u00e7\",\n\t\tEditConfig:         \"lazzydocker ayarlar\u0131n\u0131 d\u00fczenle\",\n\t\tCancel:             \"iptal\",\n\t\tRemove:             \"kald\u0131r\",\n\t\tForceRemove:        \"kald\u0131rmaya zorla\",\n\t\tRemoveWithVolumes:  \"alanlar\u0131 ile birlikte kald\u0131r\",\n\t\tRemoveService:      \"konteynerleri kald\u0131r\",\n\t\tStop:               \"durdur\",\n\t\tRestart:            \"yeniden ba\u015flat\",\n\t\tRebuild:            \"yeniden yap\u0131land\u0131r\",\n\t\tRecreate:           \"yeniden olu\u015ftur\",\n\t\tPreviousContext:    \"\u00f6nceki sekme\",\n\t\tNextContext:        \"sonraki sekme\",\n\t\tAttach:             \"ba\u011flan/ili\u015ftir\",\n\t\tViewLogs:           \"kay\u0131t defterini g\u00f6r\u00fcnt\u00fcle\",\n\t\tRemoveImage:        \"imaj\u0131 kald\u0131r\",\n\t\tRemoveVolume:       \"alan\u0131 kald\u0131r\",\n\t\tRemoveNetwork:      \"a\u011f\u0131 kald\u0131r\",\n\t\tRemoveWithoutPrune: \"etkisiz ebeveynleri silmeden kald\u0131r\",\n\t\tPruneContainers:    \"\u00e7al\u0131\u015fmayan konteynerleri temizle\",\n\t\tPruneVolumes:       \"kullan\u0131lmayan alanlar\u0131 temizle\",\n\t\tPruneNetworks:      \"kullan\u0131lmayan a\u011flar\u0131 temizle\",\n\t\tPruneImages:        \"kullan\u0131lmayan imajlar\u0131 temizle\",\n\t\tViewRestartOptions: \"yeniden ba\u015flatma se\u00e7eneklerini g\u00f6r\u00fcnt\u00fcle\",\n\t\tRunCustomCommand:   \"\u00f6nceden tan\u0131mlanm\u0131\u015f \u00f6zel komutu \u00e7al\u0131\u015ft\u0131r\",\n\n\t\tGlobalTitle:               \"Global\",\n\t\tMainTitle:                 \"Ana\",\n\t\tProjectTitle:              \"Proje\",\n\t\tServicesTitle:             \"Servisler\",\n\t\tContainersTitle:           \"Konteynerler\",\n\t\tStandaloneContainersTitle: \"Ba\u011f\u0131ms\u0131z Konteynerler\",\n\t\tImagesTitle:               \"Imajlar\",\n\t\tVolumesTitle:              \"Alanlar\",\n\t\tNetworksTitle:             \"A\u011flar\u0131\",\n\t\tCustomCommandTitle:        \"\u00d6zel Komut:\",\n\t\tErrorTitle:                \"Hata\",\n\t\tLogsTitle:                 \"Kayitlar\",\n\t\tConfigTitle:               \"Ayarlar\",\n\t\tEnvTitle:                  \"Env\",\n\t\tDockerComposeConfigTitle:  \"Docker-Compose Ayar\",\n\t\tTopTitle:                  \"Top\",\n\t\tStatsTitle:                \"Durumlar\",\n\t\tCreditsTitle:              \"Hakkinda\",\n\t\tContainerConfigTitle:      \"Konteyner Ayar\",\n\t\tContainerEnvTitle:         \"Konteyner Env\",\n\t\tNothingToDisplay:          \"Nothing to display\",\n\t\tCannotDisplayEnvVariables: \"Something went wrong while displaying environment variables\",\n\n\t\tNoContainers: \"Konteynerler yok\",\n\t\tNoContainer:  \"Konteyner yok\",\n\t\tNoImages:     \"Imajlar yok\",\n\t\tNoVolumes:    \"Alanlar yok\",\n\t\tNoNetworks:   \"A\u011flar\u0131 yok\",\n\n\t\tConfirmQuit:                 \"\u00c7\u0131kmak istedi\u011fine emin misin?\",\n\t\tMustForceToRemoveContainer:  \"Zorlamadan \u00e7al\u0131\u015fan bir konteyneri kald\u0131ramazs\u0131n\u0131z. Zorlamak ister misin?\",\n\t\tNotEnoughSpace:              \"Panelleri olu\u015fturmak i\u00e7in yeterli alan yok\",\n\t\tConfirmPruneImages:          \"Kullan\u0131lmayan t\u00fcm g\u00f6r\u00fcnt\u00fcleri temizlemek istedi\u011finize emin misiniz?\",\n\t\tConfirmPruneContainers:      \"Durdurulan t\u00fcm konteynerlar\u0131 temizlemek istedi\u011finizden emin misiniz?\",\n\t\tConfirmPruneVolumes:         \"Kullan\u0131lmayan t\u00fcm alanlar\u0131 temizlemek istedi\u011finizden emin misiniz?\",\n\t\tConfirmPruneNetworks:        \"Kullan\u0131lmayan t\u00fcm a\u011flar\u0131 temizlemek istedi\u011finizden emin misiniz?\",\n\t\tStopService:                 \"Bu servisin konteynerlerini durdurmak istedi\u011finize emin misiniz?\",\n\t\tStopContainer:               \"Bu konteyneri durdurmak istedi\u011finize emin misiniz?\",\n\t\tPressEnterToReturn:          \"lazydocker' a geri d\u00f6nmek i\u00e7in enter tu\u015funa bas\u0131n ( Bu uyar\u0131, `gui.return Immediately: true` ayar\u0131yla devre d\u0131\u015f\u0131 b\u0131rak\u0131labilir)\",\n\t\tDetachFromContainerShortCut: \"Varsay\u0131lan olarak, kaptan ay\u0131rmak i\u00e7in ctrl-p ve ard\u0131ndan ctrl-q tu\u015flar\u0131na bas\u0131n\",\n\t}\n}\n\n// Path: pkg/utils/utils_test.go\npackage utils\n\nimport (\n\t\"testing\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\n// TestSplitLines is a function.\n\nfunc TestSplitLines(t *testing.T) {\n\ttype scenario struct {\n\t\tmultilineString string\n\t\texpected        []string\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"\",\n\t\t\t[]string{},\n\t\t},\n\t\t{\n\t\t\t\"\\n\",\n\t\t\t[]string{},\n\t\t},\n\t\t{\n\t\t\t\"hello world !\\nhello universe !\\n\",\n\t\t\t[]string{\n\t\t\t\t\"hello world !\",\n\t\t\t\t\"hello universe !\",\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, s.expected, SplitLines(s.multilineString))\n\t}\n}\n\n// TestWithPadding is a function.\nfunc TestWithPadding(t *testing.T) {\n\ttype scenario struct {\n\t\tstr      string\n\t\tpadding  int\n\t\texpected string\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"hello world !\",\n\t\t\t1,\n\t\t\t\"hello world !\",\n\t\t},\n\t\t{\n\t\t\t\"hello world !\",\n\t\t\t14,\n\t\t\t\"hello world ! \",\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, s.expected, WithPadding(s.str, s.padding))\n\t}\n}\n\n// TestNormalizeLinefeeds is a function.\nfunc TestNormalizeLinefeeds(t *testing.T) {\n\ttype scenario struct {\n\t\tbyteArray []byte\n\t\texpected  []byte\n\t}\n\tscenarios := []scenario{\n\t\t{\n\t\t\t// \\r\\n\n\t\t\t[]byte{97, 115, 100, 102, 13, 10},\n\t\t\t[]byte{97, 115, 100, 102, 10},\n\t\t},\n\t\t{\n\t\t\t// bash\\r\\nblah\n\t\t\t[]byte{97, 115, 100, 102, 13, 10, 97, 115, 100, 102},\n\t\t\t[]byte{97, 115, 100, 102, 10, 97, 115, 100, 102},\n\t\t},\n\t\t{\n\t\t\t// \\r\n\t\t\t[]byte{97, 115, 100, 102, 13},\n\t\t\t[]byte{97, 115, 100, 102},\n\t\t},\n\t\t{\n\t\t\t// \\n\n\t\t\t[]byte{97, 115, 100, 102, 10},\n\t\t\t[]byte{97, 115, 100, 102, 10},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, string(s.expected), NormalizeLinefeeds(string(s.byteArray)))\n\t}\n}\n\n// TestResolvePlaceholderString is a function.\nfunc TestResolvePlaceholderString(t *testing.T) {\n\ttype scenario struct {\n\t\ttemplateString string\n\t\targuments      map[string]string\n\t\texpected       string\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"\",\n\t\t\tmap[string]string{},\n\t\t\t\"\",\n\t\t},\n\t\t{\n\t\t\t\"hello\",\n\t\t\tmap[string]string{},\n\t\t\t\"hello\",\n\t\t},\n\t\t{\n\t\t\t\"hello {{arg}}\",\n\t\t\tmap[string]string{},\n\t\t\t\"hello {{arg}}\",\n\t\t},\n\t\t{\n\t\t\t\"hello {{arg}}\",\n\t\t\tmap[string]string{\"arg\": \"there\"},\n\t\t\t\"hello there\",\n\t\t},\n\t\t{\n\t\t\t\"hello\",\n\t\t\tmap[string]string{\"arg\": \"there\"},\n\t\t\t\"hello\",\n\t\t},\n\t\t{\n\t\t\t\"{{nothing}}\",\n\t\t\tmap[string]string{\"nothing\": \"\"},\n\t\t\t\"\",\n\t\t},\n\t\t{\n\t\t\t\"{{}} {{ this }} { should not throw}} an {{{{}}}} error\",\n\t\t\tmap[string]string{\n\t\t\t\t\"blah\": \"blah\",\n\t\t\t\t\"this\": \"won't match\",\n\t\t\t},\n\t\t\t\"{{}} {{ this }} { should not throw}} an {{{{}}}} error\",\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, s.expected, ResolvePlaceholderString(s.templateString, s.arguments))\n\t}\n}\n\n// TestDisplayArraysAligned is a function.\nfunc TestDisplayArraysAligned(t *testing.T) {\n\ttype scenario struct {\n\t\tinput    [][]string\n\t\texpected bool\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t[][]string{{\"\", \"\"}, {\"\", \"\"}},\n\t\t\ttrue,\n\t\t},\n\t\t{\n\t\t\t[][]string{{\"\"}, {\"\", \"\"}},\n\t\t\tfalse,\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, s.expected, displayArraysAligned(s.input))\n\t}\n}\n\n// TestGetPaddedDisplayStrings is a function.\nfunc TestGetPaddedDisplayStrings(t *testing.T) {\n\ttype scenario struct {\n\t\tstringArrays [][]string\n\t\tpadWidths    []int\n\t\texpected     []string\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t[][]string{{\"a\", \"b\"}, {\"c\", \"d\"}},\n\t\t\t[]int{1},\n\t\t\t[]string{\"a b\", \"c d\"},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, s.expected, getPaddedDisplayStrings(s.stringArrays, s.padWidths))\n\t}\n}\n\n// TestGetPadWidths is a function.\nfunc TestGetPadWidths(t *testing.T) {\n\ttype scenario struct {\n\t\tstringArrays [][]string\n\t\texpected     []int\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t[][]string{{\"\"}, {\"\"}},\n\t\t\t[]int{},\n\t\t},\n\t\t{\n\t\t\t[][]string{{\"a\"}, {\"\"}},\n\t\t\t[]int{},\n\t\t},\n\t\t{\n\t\t\t[][]string{{\"aa\", \"b\", \"ccc\"}, {\"c\", \"d\", \"e\"}},\n\t\t\t[]int{2, 1},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, s.expected, getPadWidths(s.stringArrays))\n\t}\n}\n\nfunc TestRenderTable(t *testing.T) {\n\ttype scenario struct {\n\t\tinput       [][]string\n\t\texpected    string\n\t\texpectedErr error\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\tinput:       [][]string{{\"a\", \"b\"}, {\"c\", \"d\"}},\n\t\t\texpected:    \"a b\\nc d\",\n\t\t\texpectedErr: nil,\n\t\t},\n\t\t{\n\t\t\tinput:       [][]string{{\"aaaa\", \"b\"}, {\"c\", \"d\"}},\n\t\t\texpected:    \"aaaa b\\nc    d\",\n\t\t\texpectedErr: nil,\n\t\t},\n\t\t{\n\t\t\tinput:       [][]string{{\"a\"}, {\"c\", \"d\"}},\n\t\t\texpected:    \"\",\n\t\t\texpectedErr: errors.New(\"Each item must return the same number of strings to display\"),\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\toutput, err := RenderTable(s.input)\n\t\tassert.EqualValues(t, s.expected, output)\n\t\tif s.expectedErr != nil {\n\t\t\tassert.EqualError(t, err, s.expectedErr.Error())\n\t\t} else {\n\t\t\tassert.NoError(t, err)\n\t\t}\n\t}\n}\n\nfunc TestMarshalIntoFormat(t *testing.T) {\n\ttype innerData struct {\n\t\tFoo int    `json:\"foo\"`\n\t\tBar string `json:\"bar\"`\n\t\tBaz bool   `json:\"baz\"`\n\t}\n\ttype data struct {\n\t\tQux  int       `json:\"quz\"`\n\t\tQuux innerData `json:\"quux\"`\n\t}\n\n\ttype scenario struct {\n\t\tinput       interface{}\n\t\tformat      string\n\t\texpected    []byte\n\t\texpectedErr error\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\tinput:  data{1, innerData{2, \"foo\", true}},\n\t\t\tformat: \"json\",\n\t\t\texpected: []byte(`{\n  \"quz\": 1,\n  \"quux\": {\n    \"foo\": 2,\n    \"bar\": \"foo\",\n    \"baz\": true\n  }\n}`),\n\t\t\texpectedErr: nil,\n\t\t},\n\t\t{\n\t\t\tinput:  data{1, innerData{2, \"foo\", true}},\n\t\t\tformat: \"yaml\",\n\t\t\texpected: []byte(`quz: 1\nquux:\n  bar: foo\n  baz: true\n  foo: 2\n`),\n\t\t\texpectedErr: nil,\n\t\t},\n\t\t{\n\t\t\tinput:       data{1, innerData{2, \"foo\", true}},\n\t\t\tformat:      \"xml\",\n\t\t\texpected:    nil,\n\t\t\texpectedErr: errors.New(\"Unsupported detailization format: xml\"),\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\toutput, err := marshalIntoFormat(s.input, s.format)\n\t\tassert.EqualValues(t, s.expected, output)\n\t\tif s.expectedErr != nil {\n\t\t\tassert.EqualError(t, err, s.expectedErr.Error())\n\t\t} else {\n\t\t\tassert.NoError(t, err)\n\t\t}\n\t}\n}\n\n", "needle_token_start": 9995, "needle_token_end": 10185, "code_context_ntokens": 12620, "output": ["```go\nfunc TestSplitLines(t *testing.T) {\n\ttype scenario struct {\n\t\tmultilineString string\n\t\texpected        []string\n\t}\n\n\tscenarios := []scenario{\n\t\t{\n\t\t\t\"\",\n\t\t\t[]string{},\n\t\t},\n\t\t{\n\t\t\t\"\\n\",\n\t\t\t[]string{},\n\t\t},\n\t\t{\n\t\t\t\"hello world !\\nhello universe !\\n\",\n\t\t\t[]string{\n\t\t\t\t\"hello world !\",\n\t\t\t\t\"hello universe !\",\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, s := range scenarios {\n\t\tassert.EqualValues(t, s.expected, SplitLines(s.multilineString))\n\t}\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "TestFilteredListFilter", "language": "go", "path": "pkg/gui/panels/filtered_list_test.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To verify that the method correctly filters the list based on a provided condition, ensuring that only indices meeting the condition remain in the list.\n2. **Input**: A list with predefined items and indices, along with a filtering function that specifies a condition to retain certain indices.\n3. **Output**: The method does not directly return an output but modifies the internal state of the list to reflect the filtered indices.\n4. **Procedure**: The test initializes a list with specific items and indices, applies a filtering function, and then checks if the internal indices of the list match the expected outcome after filtering.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/gui/keybindings.go\npackage gui\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/jesseduffield/gocui\"\n)\n\n// Binding - a keybinding mapping a key and modifier to a handler. The keypress\n// is only handled if the given view has focus, or handled globally if the view\n// is \"\"\ntype Binding struct {\n\tViewName    string\n\tHandler     func(*gocui.Gui, *gocui.View) error\n\tKey         interface{} // FIXME: find out how to get `gocui.Key | rune`\n\tModifier    gocui.Modifier\n\tDescription string\n}\n\n// GetKey is a function.\nfunc (b *Binding) GetKey() string {\n\tkey := 0\n\n\tswitch b.Key.(type) {\n\tcase rune:\n\t\tkey = int(b.Key.(rune))\n\tcase gocui.Key:\n\t\tkey = int(b.Key.(gocui.Key))\n\t}\n\n\t// special keys\n\tswitch key {\n\tcase 27:\n\t\treturn \"esc\"\n\tcase 13:\n\t\treturn \"enter\"\n\tcase 32:\n\t\treturn \"space\"\n\tcase 65514:\n\t\treturn \"\u25ba\"\n\tcase 65515:\n\t\treturn \"\u25c4\"\n\tcase 65517:\n\t\treturn \"\u25b2\"\n\tcase 65516:\n\t\treturn \"\u25bc\"\n\tcase 65508:\n\t\treturn \"PgUp\"\n\tcase 65507:\n\t\treturn \"PgDn\"\n\t}\n\n\treturn fmt.Sprintf(\"%c\", key)\n}\n\n// GetInitialKeybindings is a function.\nfunc (gui *Gui) GetInitialKeybindings() []*Binding {\n\tbindings := []*Binding{\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyEsc,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.escape),\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      'q',\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.quit,\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyCtrlC,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.quit,\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyPgup,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.scrollUpMain),\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyPgdn,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.scrollDownMain),\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyCtrlU,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.scrollUpMain),\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyCtrlD,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.scrollDownMain),\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyEnd,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.autoScrollMain,\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      gocui.KeyHome,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.jumpToTopMain,\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      'x',\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.handleCreateOptionsMenu,\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      '?',\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.handleCreateOptionsMenu,\n\t\t},\n\t\t{\n\t\t\tViewName: \"\",\n\t\t\tKey:      'X',\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.handleCustomCommand,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"project\",\n\t\t\tKey:         'e',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleEditConfig,\n\t\t\tDescription: gui.Tr.EditConfig,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"project\",\n\t\t\tKey:         'o',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleOpenConfig,\n\t\t\tDescription: gui.Tr.OpenConfig,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"project\",\n\t\t\tKey:         'm',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleViewAllLogs,\n\t\t\tDescription: gui.Tr.ViewLogs,\n\t\t},\n\t\t{\n\t\t\tViewName: \"menu\",\n\t\t\tKey:      gocui.KeyEsc,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.handleMenuClose),\n\t\t},\n\t\t{\n\t\t\tViewName: \"menu\",\n\t\t\tKey:      'q',\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.handleMenuClose),\n\t\t},\n\t\t{\n\t\t\tViewName: \"menu\",\n\t\t\tKey:      ' ',\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.handleMenuPress),\n\t\t},\n\t\t{\n\t\t\tViewName: \"menu\",\n\t\t\tKey:      gocui.KeyEnter,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.handleMenuPress),\n\t\t},\n\t\t{\n\t\t\tViewName: \"menu\",\n\t\t\tKey:      'y',\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  wrappedHandler(gui.handleMenuPress),\n\t\t},\n\t\t{\n\t\t\tViewName: \"information\",\n\t\t\tKey:      gocui.MouseLeft,\n\t\t\tModifier: gocui.ModNone,\n\t\t\tHandler:  gui.handleDonate,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'd',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainersRemoveMenu,\n\t\t\tDescription: gui.Tr.Remove,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'e',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleHideStoppedContainers,\n\t\t\tDescription: gui.Tr.HideStopped,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'p',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainerPause,\n\t\t\tDescription: gui.Tr.Pause,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         's',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainerStop,\n\t\t\tDescription: gui.Tr.Stop,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'r',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainerRestart,\n\t\t\tDescription: gui.Tr.Restart,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'a',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainerAttach,\n\t\t\tDescription: gui.Tr.Attach,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'm',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainerViewLogs,\n\t\t\tDescription: gui.Tr.ViewLogs,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'E',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainersExecShell,\n\t\t\tDescription: gui.Tr.ExecShell,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'c',\n\t\t\tModifier:    gocui.ModNone,\n\t\t\tHandler:     gui.handleContainersCustomCommand,\n\t\t\tDescription: gui.Tr.RunCustomCommand,\n\t\t},\n\t\t{\n\t\t\tViewName:    \"containers\",\n\t\t\tKey:         'b',\n...\n// Path: pkg/gui/layout.go\npackage gui\n\nimport (\n\t\"github.com/jesseduffield/gocui\"\n)\n\nconst UNKNOWN_VIEW_ERROR_MSG = \"unknown view\"\n\n// getFocusLayout returns a manager function for when view gain and lose focus\nfunc (gui *Gui) getFocusLayout() func(g *gocui.Gui) error {\n\tvar previousView *gocui.View\n\treturn func(g *gocui.Gui) error {\n\t\tnewView := gui.g.CurrentView()\n\t\tif err := gui.onFocusChange(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// for now we don't consider losing focus to a popup panel as actually losing focus\n\t\tif newView != previousView && !gui.isPopupPanel(newView.Name()) {\n\t\t\tgui.onFocusLost(previousView, newView)\n\t\t\tgui.onFocus(newView)\n\t\t\tpreviousView = newView\n\t\t}\n\t\treturn nil\n\t}\n}\n\nfunc (gui *Gui) onFocusChange() error {\n\tcurrentView := gui.g.CurrentView()\n\tfor _, view := range gui.g.Views() {\n\t\tview.Highlight = view == currentView && view.Name() != \"main\"\n\t}\n\treturn nil\n}\n\nfunc (gui *Gui) onFocusLost(v *gocui.View, newView *gocui.View) {\n\tif v == nil {\n\t\treturn\n\t}\n\n\tif !gui.isPopupPanel(newView.Name()) {\n\t\tv.ParentView = nil\n\t}\n\n\t// refocusing because in responsive mode (when the window is very short) we want to ensure that after the view size changes we can still see the last selected item\n\tgui.focusPointInView(v)\n\n\tgui.Log.Info(v.Name() + \" focus lost\")\n}\n\nfunc (gui *Gui) onFocus(v *gocui.View) {\n\tif v == nil {\n\t\treturn\n\t}\n\n\tgui.focusPointInView(v)\n\n\tgui.Log.Info(v.Name() + \" focus gained\")\n}\n\n// layout is called for every screen re-render e.g. when the screen is resized\nfunc (gui *Gui) layout(g *gocui.Gui) error {\n\tg.Highlight = true\n\twidth, height := g.Size()\n\n\tappStatus := gui.statusManager.getStatusString()\n\n\tviewDimensions := gui.getWindowDimensions(gui.getInformationContent(), appStatus)\n\t// we assume that the view has already been created.\n\tsetViewFromDimensions := func(viewName string, windowName string) (*gocui.View, error) {\n\t\tdimensionsObj, ok := viewDimensions[windowName]\n\n\t\tview, err := g.View(viewName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif !ok {\n\t\t\t// view not specified in dimensions object: so create the view and hide it\n\t\t\t// making the view take up the whole space in the background in case it needs\n\t\t\t// to render content as soon as it appears, because lazyloaded content (via a pty task)\n\t\t\t// cares about the size of the view.\n\t\t\t_, err := g.SetView(viewName, 0, 0, width, height, 0)\n\t\t\tview.Visible = false\n\t\t\treturn view, err\n\t\t}\n\n\t\tframeOffset := 1\n\t\tif view.Frame {\n\t\t\tframeOffset = 0\n\t\t}\n\t\t_, err = g.SetView(\n\t\t\tviewName,\n\t\t\tdimensionsObj.X0-frameOffset,\n\t\t\tdimensionsObj.Y0-frameOffset,\n\t\t\tdimensionsObj.X1+frameOffset,\n\t\t\tdimensionsObj.Y1+frameOffset,\n\t\t\t0,\n\t\t)\n\t\tview.Visible = true\n\n\t\treturn view, err\n\t}\n\n\tfor _, viewName := range gui.autoPositionedViewNames() {\n\t\t_, err := setViewFromDimensions(viewName, viewName)\n\t\tif err != nil && err.Error() != UNKNOWN_VIEW_ERROR_MSG {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// here is a good place log some stuff\n\t// if you download humanlog and do tail -f development.log | humanlog\n\t// this will let you see these branches as prettified json\n\t// gui.Log.Info(utils.AsJson(gui.State.Branches[0:4]))\n\treturn gui.resizeCurrentPopupPanel(g)\n}\n\nfunc (gui *Gui) focusPointInView(view *gocui.View) {\n\tif view == nil {\n\t\treturn\n\t}\n\n\tcurrentPanel, ok := gui.currentListPanel()\n\tif ok {\n\t\tcurrentPanel.Refocus()\n\t}\n}\n\nfunc (gui *Gui) prepareView(viewName string) (*gocui.View, error) {\n\t// arbitrarily giving the view enough size so that we don't get an error, but\n\t// it's expected that the view will be given the correct size before being shown\n\treturn gui.g.SetView(viewName, 0, 0, 10, 10, 0)\n}\n\n// Path: pkg/gui/main_panel.go\npackage gui\n\nimport (\n\t\"math\"\n\n\t\"github.com/jesseduffield/gocui\"\n)\n\nfunc (gui *Gui) scrollUpMain() error {\n\tmainView := gui.Views.Main\n\tmainView.Autoscroll = false\n\tox, oy := mainView.Origin()\n\tnewOy := int(math.Max(0, float64(oy-gui.Config.UserConfig.Gui.ScrollHeight)))\n\treturn mainView.SetOrigin(ox, newOy)\n}\n\nfunc (gui *Gui) scrollDownMain() error {\n\tmainView := gui.Views.Main\n\tmainView.Autoscroll = false\n\tox, oy := mainView.Origin()\n\n\treservedLines := 0\n\tif !gui.Config.UserConfig.Gui.ScrollPastBottom {\n\t\t_, sizeY := mainView.Size()\n\t\treservedLines = sizeY\n\t}\n\n\ttotalLines := mainView.ViewLinesHeight()\n\tif oy+reservedLines >= totalLines {\n\t\treturn nil\n\t}\n\n\treturn mainView.SetOrigin(ox, oy+gui.Config.UserConfig.Gui.ScrollHeight)\n}\n\nfunc (gui *Gui) scrollLeftMain(g *gocui.Gui, v *gocui.View) error {\n\tmainView := gui.Views.Main\n\tox, oy := mainView.Origin()\n\tnewOx := int(math.Max(0, float64(ox-gui.Config.UserConfig.Gui.ScrollHeight)))\n\n\treturn mainView.SetOrigin(newOx, oy)\n}\n\nfunc (gui *Gui) scrollRightMain(g *gocui.Gui, v *gocui.View) error {\n\tmainView := gui.Views.Main\n\tox, oy := mainView.Origin()\n\n\tcontent := mainView.ViewBufferLines()\n\tvar largestNumberOfCharacters int\n\tfor _, txt := range content {\n\t\tif len(txt) > largestNumberOfCharacters {\n\t\t\tlargestNumberOfCharacters = len(txt)\n\t\t}\n\t}\n\n\tsizeX, _ := mainView.Size()\n\tif ox+sizeX >= largestNumberOfCharacters {\n\t\treturn nil\n\t}\n\n\treturn mainView.SetOrigin(ox+gui.Config.UserConfig.Gui.ScrollHeight, oy)\n}\n\nfunc (gui *Gui) autoScrollMain(g *gocui.Gui, v *gocui.View) error {\n\tgui.Views.Main.Autoscroll = true\n\treturn nil\n}\n\nfunc (gui *Gui) jumpToTopMain(g *gocui.Gui, v *gocui.View) error {\n\tgui.Views.Main.Autoscroll = false\n\t_ = gui.Views.Main.SetOrigin(0, 0)\n\t_ = gui.Views.Main.SetCursor(0, 0)\n\treturn nil\n}\n\nfunc (gui *Gui) onMainTabClick(tabIndex int) error {\n\tgui.Log.Warn(tabIndex)\n\n\tcurrentSidePanel, ok := gui.currentSidePanel()\n\n\tif !ok {\n\t\treturn nil\n\t}\n\n\tcurrentSidePanel.SetMainTabIndex(tabIndex)\n\treturn currentSidePanel.HandleSelect()\n}\n\nfunc (gui *Gui) handleEnterMain(g *gocui.Gui, v *gocui.View) error {\n\tmainView := gui.Views.Main\n\tmainView.ParentView = v\n\n\treturn gui.switchFocus(mainView)\n}\n\nfunc (gui *Gui) handleExitMain(g *gocui.Gui, v *gocui.View) error {\n\tv.ParentView = nil\n\treturn gui.returnFocus()\n}\n\nfunc (gui *Gui) handleMainClick() error {\n\tif gui.popupPanelFocused() {\n\t\treturn nil\n\t}\n\n\tcurrentView := gui.g.CurrentView()\n\n\tif currentView.Name() != \"main\" {\n\t\tgui.Views.Main.ParentView = currentView\n\t}\n\n\treturn gui.switchFocus(gui.Views.Main)\n}\n\n// Path: pkg/gui/menu_panel.go\npackage gui\n\nimport (\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\ntype CreateMenuOptions struct {\n\tTitle      string\n\tItems      []*types.MenuItem\n\tHideCancel bool\n}\n\nfunc (gui *Gui) getMenuPanel() *panels.SideListPanel[*types.MenuItem] {\n\treturn &panels.SideListPanel[*types.MenuItem]{\n\t\tListPanel: panels.ListPanel[*types.MenuItem]{\n\t\t\tList: panels.NewFilteredList[*types.MenuItem](),\n\t\t\tView: gui.Views.Menu,\n\t\t},\n\t\tNoItemsMessage: \"\",\n\t\tGui:            gui.intoInterface(),\n\t\tOnClick:        gui.onMenuPress,\n\t\tSort:           nil,\n\t\tGetTableCells:  presentation.GetMenuItemDisplayStrings,\n\t\tOnRerender: func() error {\n\t\t\treturn gui.resizePopupPanel(gui.Views.Menu)\n\t\t},\n\t\t// so that we can avoid some UI trickiness, the menu will not have filtering\n\t\t// abillity yet. To support it, we would need to have filter state against\n\t\t// each panel (e.g. for when you filter the images panel, then bring up\n\t\t// the options menu, then try to filter that too.\n\t\tDisableFilter: true,\n\t}\n}\n\nfunc (gui *Gui) onMenuPress(menuItem *types.MenuItem) error {\n\tif err := gui.handleMenuClose(); err != nil {\n\t\treturn err\n\t}\n\n\tif menuItem.OnPress != nil {\n\t\treturn menuItem.OnPress()\n\t}\n\n\treturn nil\n}\n\nfunc (gui *Gui) handleMenuPress() error {\n\tselectedMenuItem, err := gui.Panels.Menu.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.onMenuPress(selectedMenuItem)\n}\n\nfunc (gui *Gui) Menu(opts CreateMenuOptions) error {\n\tif !opts.HideCancel {\n\t\t// this is mutative but I'm okay with that for now\n\t\topts.Items = append(opts.Items, &types.MenuItem{\n\t\t\tLabelColumns: []string{gui.Tr.Cancel},\n\t\t\tOnPress: func() error {\n\t\t\t\treturn nil\n\t\t\t},\n\t\t})\n\t}\n\n\tmaxColumnSize := 1\n\n\tfor _, item := range opts.Items {\n\t\tif item.LabelColumns == nil {\n\t\t\titem.LabelColumns = []string{item.Label}\n\t\t}\n\n\t\tif item.OpensMenu {\n\t\t\titem.LabelColumns[0] = utils.OpensMenuStyle(item.LabelColumns[0])\n\t\t}\n\n\t\tmaxColumnSize = utils.Max(maxColumnSize, len(item.LabelColumns))\n\t}\n\n\tfor _, item := range opts.Items {\n\t\tif len(item.LabelColumns) < maxColumnSize {\n\t\t\t// we require that each item has the same number of columns so we're padding out with blank strings\n\t\t\t// if this item has too few\n\t\t\titem.LabelColumns = append(item.LabelColumns, make([]string, maxColumnSize-len(item.LabelColumns))...)\n\t\t}\n\t}\n\n\tgui.Panels.Menu.SetItems(opts.Items)\n\tgui.Panels.Menu.SetSelectedLineIdx(0)\n\n\tif err := gui.Panels.Menu.RerenderList(); err != nil {\n\t\treturn err\n\t}\n\n\tgui.Views.Menu.Title = opts.Title\n\tgui.Views.Menu.Visible = true\n\n\treturn gui.switchFocus(gui.Views.Menu)\n}\n\n// specific functions\n\nfunc (gui *Gui) renderMenuOptions() error {\n\toptionsMap := map[string]string{\n\t\t\"esc\":   gui.Tr.Close,\n\t\t\"\u2191 \u2193\":   gui.Tr.Navigate,\n\t\t\"enter\": gui.Tr.Execute,\n\t}\n\treturn gui.renderOptionsMap(optionsMap)\n}\n\nfunc (gui *Gui) handleMenuClose() error {\n\tgui.Views.Menu.Visible = false\n\n\t// this code is here for when we do add filter ability to the menu panel,\n\t// though it's currently disabled\n\tif gui.State.Filter.panel == gui.Panels.Menu {\n\t\tif err := gui.clearFilter(); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// we need to remove the view from the view stack because we're about to\n\t\t// return focus and don't want to land in the search view when it was searching\n\t\t// the menu in the first place\n\t\tgui.removeViewFromStack(gui.Views.Filter)\n\t}\n\n\treturn gui.returnFocus()\n}\n\n// Path: pkg/gui/networks_panel.go\npackage gui\n\nimport (\n\t\"strconv\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\nfunc (gui *Gui) getNetworksPanel() *panels.SideListPanel[*commands.Network] {\n\treturn &panels.SideListPanel[*commands.Network]{\n\t\tContextState: &panels.ContextState[*commands.Network]{\n\t\t\tGetMainTabs: func() []panels.MainTab[*commands.Network] {\n\t\t\t\treturn []panels.MainTab[*commands.Network]{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"config\",\n\t\t\t\t\t\tTitle:  gui.Tr.ConfigTitle,\n\t\t\t\t\t\tRender: gui.renderNetworkConfig,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t},\n\t\t\tGetItemContextCacheKey: func(network *commands.Network) string {\n\t\t\t\treturn \"networks-\" + network.Name\n\t\t\t},\n\t\t},\n\t\tListPanel: panels.ListPanel[*commands.Network]{\n\t\t\tList: panels.NewFilteredList[*commands.Network](),\n\t\t\tView: gui.Views.Networks,\n\t\t},\n\t\tNoItemsMessage: gui.Tr.NoNetworks,\n\t\tGui:            gui.intoInterface(),\n\t\t// we're sorting these networks based on whether they have labels defined,\n\t\t// because those are the ones you typically care about.\n\t\t// Within that, we also sort them alphabetically\n\t\tSort: func(a *commands.Network, b *commands.Network) bool {\n\t\t\treturn a.Name < b.Name\n\t\t},\n\t\tGetTableCells: presentation.GetNetworkDisplayStrings,\n\t}\n}\n\nfunc (gui *Gui) renderNetworkConfig(network *commands.Network) tasks.TaskFunc {\n\treturn gui.NewSimpleRenderStringTask(func() string { return gui.networkConfigStr(network) })\n}\n\nfunc (gui *Gui) networkConfigStr(network *commands.Network) string {\n\tpadding := 15\n\toutput := \"\"\n\toutput += utils.WithPadding(\"ID: \", padding) + network.Network.ID + \"\\n\"\n\toutput += utils.WithPadding(\"Name: \", padding) + network.Name + \"\\n\"\n\toutput += utils.WithPadding(\"Driver: \", padding) + network.Network.Driver + \"\\n\"\n\toutput += utils.WithPadding(\"Scope: \", padding) + network.Network.Scope + \"\\n\"\n\toutput += utils.WithPadding(\"EnabledIPV6: \", padding) + strconv.FormatBool(network.Network.EnableIPv6) + \"\\n\"\n\toutput += utils.WithPadding(\"Internal: \", padding) + strconv.FormatBool(network.Network.Internal) + \"\\n\"\n\toutput += utils.WithPadding(\"Attachable: \", padding) + strconv.FormatBool(network.Network.Attachable) + \"\\n\"\n\toutput += utils.WithPadding(\"Ingress: \", padding) + strconv.FormatBool(network.Network.Ingress) + \"\\n\"\n\n\toutput += utils.WithPadding(\"Containers: \", padding)\n\tif len(network.Network.Containers) > 0 {\n\t\toutput += \"\\n\"\n\t\tfor _, v := range network.Network.Containers {\n\t\t\toutput += utils.FormatMapItem(padding, v.Name, v.EndpointID)\n\t\t}\n\t} else {\n\t\toutput += \"none\\n\"\n\t}\n\n\toutput += \"\\n\"\n\toutput += utils.WithPadding(\"Labels: \", padding) + utils.FormatMap(padding, network.Network.Labels) + \"\\n\"\n\toutput += utils.WithPadding(\"Options: \", padding) + utils.FormatMap(padding, network.Network.Options)\n\n\treturn output\n}\n\nfunc (gui *Gui) reloadNetworks() error {\n\tif err := gui.refreshStateNetworks(); err != nil {\n\t\treturn err\n\t}\n\n\treturn gui.Panels.Networks.RerenderList()\n}\n\nfunc (gui *Gui) refreshStateNetworks() error {\n\tnetworks, err := gui.DockerCommand.RefreshNetworks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tgui.Panels.Networks.SetItems(networks)\n\n\treturn nil\n}\n\nfunc (gui *Gui) handleNetworksRemoveMenu(g *gocui.Gui, v *gocui.View) error {\n\tnetwork, err := gui.Panels.Networks.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\ttype removeNetworkOption struct {\n\t\tdescription string\n\t\tcommand     string\n\t}\n\n\toptions := []*removeNetworkOption{\n\t\t{\n\t\t\tdescription: gui.Tr.Remove,\n\t\t\tcommand:     utils.WithShortSha(\"docker network rm \" + network.Name),\n\t\t},\n\t}\n\n\tmenuItems := lo.Map(options, func(option *removeNetworkOption, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: []string{option.description, color.New(color.FgRed).Sprint(option.command)},\n\t\t\tOnPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.RemovingStatus, func() error {\n\t\t\t\t\tif err := network.Remove(); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle: \"\",\n\t\tItems: menuItems,\n\t})\n}\n\nfunc (gui *Gui) handlePruneNetworks() error {\n\treturn gui.createConfirmationPanel(gui.Tr.Confirm, gui.Tr.ConfirmPruneNetworks, func(g *gocui.Gui, v *gocui.View) error {\n\t\treturn gui.WithWaitingStatus(gui.Tr.PruningStatus, func() error {\n\t\t\terr := gui.DockerCommand.PruneNetworks()\n\t\t\tif err != nil {\n\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}, nil)\n}\n\nfunc (gui *Gui) handleNetworksCustomCommand(g *gocui.Gui, v *gocui.View) error {\n\tnetwork, err := gui.Panels.Networks.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tcommandObject := gui.DockerCommand.NewCommandObject(commands.CommandObject{\n\t\tNetwork: network,\n\t})\n\n\tcustomCommands := gui.Config.UserConfig.CustomCommands.Networks\n\n\treturn gui.createCustomCommandMenu(customCommands, commandObject)\n}\n\nfunc (gui *Gui) handleNetworksBulkCommand(g *gocui.Gui, v *gocui.View) error {\n\tbaseBulkCommands := []config.CustomCommand{\n\t\t{\n\t\t\tName:             gui.Tr.PruneNetworks,\n\t\t\tInternalFunction: gui.handlePruneNetworks,\n\t\t},\n\t}\n\n\tbulkCommands := append(baseBulkCommands, gui.Config.UserConfig.BulkCommands.Networks...)\n\tcommandObject := gui.DockerCommand.NewCommandObject(commands.CommandObject{})\n\n\treturn gui.createBulkCommandMenu(bulkCommands, commandObject)\n}\n\n// Path: pkg/gui/options_menu_panel.go\npackage gui\n\nimport (\n\t\"github.com/samber/lo\"\n\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n)\n\nfunc (gui *Gui) getBindings(v *gocui.View) []*Binding {\n\tvar bindingsGlobal, bindingsPanel []*Binding\n\n\tbindings := gui.GetInitialKeybindings()\n\n\tfor _, binding := range bindings {\n\t\tif binding.GetKey() != \"\" && binding.Description != \"\" {\n\t\t\tswitch binding.ViewName {\n\t\t\tcase \"\":\n\t\t\t\tbindingsGlobal = append(bindingsGlobal, binding)\n\t\t\tcase v.Name():\n\t\t\t\tbindingsPanel = append(bindingsPanel, binding)\n\t\t\t}\n\t\t}\n\t}\n\n\t// check if we have any keybindings from our parent view to add\n\tif v.ParentView != nil {\n\tL:\n\t\tfor _, binding := range bindings {\n\t\t\tif binding.GetKey() != \"\" && binding.Description != \"\" {\n\t\t\t\tif binding.ViewName == v.ParentView.Name() {\n\t\t\t\t\t// if we haven't got a conflict we will display the binding\n\t\t\t\t\tfor _, ownBinding := range bindingsPanel {\n\t\t\t\t\t\tif ownBinding.GetKey() == binding.GetKey() {\n\t\t\t\t\t\t\tcontinue L\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tbindingsPanel = append(bindingsPanel, binding)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// append dummy element to have a separator between\n\t// panel and global keybindings\n\tbindingsPanel = append(bindingsPanel, &Binding{})\n\treturn append(bindingsPanel, bindingsGlobal...)\n}\n\nfunc (gui *Gui) handleCreateOptionsMenu(g *gocui.Gui, v *gocui.View) error {\n\tif gui.isPopupPanel(v.Name()) {\n\t\treturn nil\n\t}\n\n\tmenuItems := lo.Map(gui.getBindings(v), func(binding *Binding, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: []string{binding.GetKey(), binding.Description},\n\t\t\tOnPress: func() error {\n\t\t\t\tif binding.Key == nil {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\n\t\t\t\treturn binding.Handler(g, v)\n\t\t\t},\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle:      gui.Tr.MenuTitle,\n\t\tItems:      menuItems,\n\t\tHideCancel: true,\n\t})\n}\n\n// Path: pkg/gui/panels/context_state.go\npackage panels\n\nimport (\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/samber/lo\"\n)\n\n// A 'context' generally corresponds to an item and the tab in the main panel that we're\n// displaying. So if we switch to a new item, or change the tab in the panel panel\n// for the current item, we end up with a new context. When we have a new context,\n// we render new content to the main panel.\ntype ContextState[T any] struct {\n\t// index of the currently selected tab in the main view.\n\tmainTabIdx int\n\t// this function returns the tabs that we can display for an item (the tabs\n\t// are shown on the main view)\n\tGetMainTabs func() []MainTab[T]\n\t// This tells us whether we need to re-render to the main panel for a given item.\n\t// This should include the item's ID and if you want to invalidate the cache for\n\t// some other reason, you can add that to the key as well (e.g. the container's state).\n\tGetItemContextCacheKey func(item T) string\n}\n\ntype MainTab[T any] struct {\n\t// key used as part of the context cache key\n\tKey string\n\t// title of the tab, rendered in the main view\n\tTitle string\n\t// function to render the content of the tab\n\tRender func(item T) tasks.TaskFunc\n}\n\nfunc (self *ContextState[T]) GetMainTabTitles() []string {\n\treturn lo.Map(self.GetMainTabs(), func(tab MainTab[T], _ int) string {\n\t\treturn tab.Title\n\t})\n}\n\nfunc (self *ContextState[T]) GetCurrentContextKey(item T) string {\n\treturn self.GetItemContextCacheKey(item) + \"-\" + self.GetCurrentMainTab().Key\n}\n\nfunc (self *ContextState[T]) GetCurrentMainTab() MainTab[T] {\n\treturn self.GetMainTabs()[self.mainTabIdx]\n}\n\nfunc (self *ContextState[T]) HandleNextMainTab() {\n\ttabs := self.GetMainTabs()\n\n\tif len(tabs) == 0 {\n\t\treturn\n\t}\n\n\tself.mainTabIdx = (self.mainTabIdx + 1) % len(tabs)\n}\n\nfunc (self *ContextState[T]) HandlePrevMainTab() {\n\ttabs := self.GetMainTabs()\n\n\tif len(tabs) == 0 {\n\t\treturn\n\t}\n\n\tself.mainTabIdx = (self.mainTabIdx - 1 + len(tabs)) % len(tabs)\n}\n\nfunc (self *ContextState[T]) SetMainTabIndex(index int) {\n\tself.mainTabIdx = index\n}\n\n// Path: pkg/gui/panels/filtered_list.go\npackage panels\n\nimport (\n\t\"sort\"\n\t\"sync\"\n)\n\ntype FilteredList[T comparable] struct {\n\tallItems []T\n\t// indices of items in the allItems slice that are included in the filtered list\n\tindices []int\n\n\tmutex sync.RWMutex\n}\n\nfunc NewFilteredList[T comparable]() *FilteredList[T] {\n\treturn &FilteredList[T]{}\n}\n\nfunc (self *FilteredList[T]) SetItems(items []T) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tself.allItems = items\n\tself.indices = make([]int, len(items))\n\tfor i := range self.indices {\n\t\tself.indices[i] = i\n\t}\n}\n\nfunc (self *FilteredList[T]) Filter(filter func(T, int) bool) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tself.indices = self.indices[:0]\n\tfor i, item := range self.allItems {\n\t\tif filter(item, i) {\n\t\t\tself.indices = append(self.indices, i)\n\t\t}\n\t}\n}\n\nfunc (self *FilteredList[T]) Sort(less func(T, T) bool) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tif less == nil {\n\t\treturn\n\t}\n\n\tsort.Slice(self.indices, func(i, j int) bool {\n\t\treturn less(self.allItems[self.indices[i]], self.allItems[self.indices[j]])\n\t})\n}\n\nfunc (self *FilteredList[T]) Get(index int) T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn self.allItems[self.indices[index]]\n}\n\nfunc (self *FilteredList[T]) TryGet(index int) (T, bool) {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tif index < 0 || index >= len(self.indices) {\n\t\tvar zero T\n\t\treturn zero, false\n\t}\n\n\treturn self.allItems[self.indices[index]], true\n}\n\n// returns the length of the filtered list\nfunc (self *FilteredList[T]) Len() int {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn len(self.indices)\n}\n\nfunc (self *FilteredList[T]) GetIndex(item T) int {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tfor i, index := range self.indices {\n\t\tif self.allItems[index] == item {\n\t\t\treturn i\n\t\t}\n\t}\n\treturn -1\n}\n\nfunc (self *FilteredList[T]) GetItems() []T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tresult := make([]T, len(self.indices))\n\tfor i, index := range self.indices {\n\t\tresult[i] = self.allItems[index]\n\t}\n\treturn result\n}\n\nfunc (self *FilteredList[T]) GetAllItems() []T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn self.allItems\n}\n\n// Path: pkg/gui/panels/filtered_list_test.go\npackage panels\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestFilteredListGet(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs int\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 1,\n\t\t\twant: 2,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 2,\n\t\t\twant: 3,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: 0,\n\t\t\twant: 2,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.Get(tt.args); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.Get() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListLen(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\twant: 3,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\twant: 1,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.Len(); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.Len() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\n\nfunc TestFilteredListFilter(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs func(int, int) bool\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 0 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 1 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 2}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.Filter(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t}\n}\n\nfunc TestFilteredListSort(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs func(int, int) bool\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, j int) bool { return i < j },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, j int) bool { return i > j },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{2, 1, 0}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.Sort(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t}\n}\n\nfunc TestFilteredListGetIndex(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs int\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 1,\n\t\t\twant: 0,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 2,\n\t\t\twant: 1,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: 0,\n\t\t\twant: -1,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.GetIndex(tt.args); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.GetIndex() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListGetItems(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\twant []int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\twant: []int{1, 2, 3},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\twant: []int{2},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tgot := tt.f.GetItems()\n\t\tassert.EqualValues(t, got, tt.want)\n\t}\n}\n\nfunc TestFilteredListSetItems(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs []int\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: []int{4, 5, 6},\n\t\t\twant: &FilteredList[int]{allItems: []int{4, 5, 6}, indices: []int{0, 1, 2}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: []int{4},\n\t\t\twant: &FilteredList[int]{allItems: []int{4}, indices: []int{0}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.SetItems(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t\tassert.EqualValues(t, tt.f.allItems, tt.want.allItems)\n\t}\n}\n\n// Path: pkg/gui/panels/list_panel.go\npackage panels\n\nimport (\n\t\"github.com/jesseduffield/gocui\"\n\tlcUtils \"github.com/jesseduffield/lazycore/pkg/utils\"\n)\n\ntype ListPanel[T comparable] struct {\n\tSelectedIdx int\n\tList        *FilteredList[T]\n\tView        *gocui.View\n}\n\nfunc (self *ListPanel[T]) SetSelectedLineIdx(value int) {\n\tclampedValue := 0\n\tif self.List.Len() > 0 {\n\t\tclampedValue = lcUtils.Clamp(value, 0, self.List.Len()-1)\n\t}\n\n\tself.SelectedIdx = clampedValue\n}\n\nfunc (self *ListPanel[T]) clampSelectedLineIdx() {\n\tclamped := lcUtils.Clamp(self.SelectedIdx, 0, self.List.Len()-1)\n\n\tif clamped != self.SelectedIdx {\n\t\tself.SelectedIdx = clamped\n\t}\n}\n\n// moves the cursor up or down by the given amount (up for negative values)\nfunc (self *ListPanel[T]) moveSelectedLine(delta int) {\n\tself.SetSelectedLineIdx(self.SelectedIdx + delta)\n}\n\nfunc (self *ListPanel[T]) SelectNextLine() {\n\tself.moveSelectedLine(1)\n}\n\nfunc (self *ListPanel[T]) SelectPrevLine() {\n\tself.moveSelectedLine(-1)\n}\n\n// Path: pkg/gui/panels/side_list_panel.go\npackage panels\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\ntype ISideListPanel interface {\n\tSetMainTabIndex(int)\n\tHandleSelect() error\n\tGetView() *gocui.View\n\tRefocus()\n\tRerenderList() error\n\tIsFilterDisabled() bool\n\tIsHidden() bool\n\tHandleNextLine() error\n\tHandlePrevLine() error\n\tHandleClick() error\n\tHandlePrevMainTab() error\n\tHandleNextMainTab() error\n}\n\n// list panel at the side of the screen that renders content to the main panel\ntype SideListPanel[T comparable] struct {\n\tContextState *ContextState[T]\n\n\tListPanel[T]\n\n\t// message to render in the main view if there are no items in the panel\n\t// and it has focus. Leave empty if you don't want to render anything\n\tNoItemsMessage string\n\n\t// a representation of the gui\n\tGui IGui\n\n\t// this Filter is applied on top of additional default filters\n\tFilter func(T) bool\n\tSort   func(a, b T) bool\n\n\t// a callback to invoke when the item is clicked\n\tOnClick func(T) error\n\n\t// returns the cells that we render to the view in a table format. The cells will\n\t// be rendered with padding.\n\tGetTableCells func(T) []string\n\n\t// function to be called after re-rendering list. Can be nil\n\tOnRerender func() error\n\n\t// set this to true if you don't want to allow manual filtering via '/'\n\tDisableFilter bool\n\n\t// This can be nil if you want to always show the panel\n\tHide func() bool\n}\n\nvar _ ISideListPanel = &SideListPanel[int]{}\n\ntype IGui interface {\n\tHandleClick(v *gocui.View, itemCount int, selectedLine *int, handleSelect func() error) error\n\tNewSimpleRenderStringTask(getContent func() string) tasks.TaskFunc\n\tFocusY(selectedLine int, itemCount int, view *gocui.View)\n\tShouldRefresh(contextKey string) bool\n\tGetMainView() *gocui.View\n\tIsCurrentView(*gocui.View) bool\n\tFilterString(view *gocui.View) string\n\tIgnoreStrings() []string\n\tUpdate(func() error)\n\n\tQueueTask(f func(ctx context.Context)) error\n}\n\nfunc (self *SideListPanel[T]) HandleClick() error {\n\titemCount := self.List.Len()\n\thandleSelect := self.HandleSelect\n\tselectedLine := &self.SelectedIdx\n\n\tif err := self.Gui.HandleClick(self.View, itemCount, selectedLine, handleSelect); err != nil {\n\t\treturn err\n\t}\n\n\tif self.OnClick != nil {\n\t\tselectedItem, err := self.GetSelectedItem()\n\t\tif err == nil {\n\t\t\treturn self.OnClick(selectedItem)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (self *SideListPanel[T]) GetView() *gocui.View {\n\treturn self.View\n}\n\nfunc (self *SideListPanel[T]) HandleSelect() error {\n\titem, err := self.GetSelectedItem()\n\tif err != nil {\n\t\tif err.Error() != self.NoItemsMessage {\n\t\t\treturn err\n\t\t}\n\n\t\tif self.NoItemsMessage != \"\" {\n\t\t\tself.Gui.NewSimpleRenderStringTask(func() string { return self.NoItemsMessage })\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tself.Refocus()\n\n\treturn self.renderContext(item)\n}\n\nfunc (self *SideListPanel[T]) renderContext(item T) error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tkey := self.ContextState.GetCurrentContextKey(item)\n\tif !self.Gui.ShouldRefresh(key) {\n\t\treturn nil\n\t}\n\n\tmainView := self.Gui.GetMainView()\n\tmainView.Tabs = self.ContextState.GetMainTabTitles()\n\tmainView.TabIndex = self.ContextState.mainTabIdx\n\n\ttask := self.ContextState.GetCurrentMainTab().Render(item)\n\n\treturn self.Gui.QueueTask(task)\n}\n\nfunc (self *SideListPanel[T]) GetSelectedItem() (T, error) {\n\tvar zero T\n\n\titem, ok := self.List.TryGet(self.SelectedIdx)\n\tif !ok {\n\t\t// could probably have a better error here\n\t\treturn zero, errors.New(self.NoItemsMessage)\n\t}\n\n\treturn item, nil\n}\n\nfunc (self *SideListPanel[T]) HandleNextLine() error {\n\tself.SelectNextLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevLine() error {\n\tself.SelectPrevLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandleNextMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandleNextMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandlePrevMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) Refocus() {\n\tself.Gui.FocusY(self.SelectedIdx, self.List.Len(), self.View)\n}\n\nfunc (self *SideListPanel[T]) SetItems(items []T) {\n\tself.List.SetItems(items)\n\tself.FilterAndSort()\n}\n\nfunc (self *SideListPanel[T]) FilterAndSort() {\n\tfilterString := self.Gui.FilterString(self.View)\n\n\tself.List.Filter(func(item T, index int) bool {\n\t\tif self.Filter != nil && !self.Filter(item) {\n\t\t\treturn false\n\t\t}\n\n\t\tif lo.SomeBy(self.Gui.IgnoreStrings(), func(ignore string) bool {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, ignore)\n\t\t\t})\n\t\t}) {\n\t\t\treturn false\n\t\t}\n\n\t\tif filterString != \"\" {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, filterString)\n\t\t\t})\n\t\t}\n\n\t\treturn true\n\t})\n\n\tself.List.Sort(self.Sort)\n\n\tself.clampSelectedLineIdx()\n}\n\nfunc (self *SideListPanel[T]) RerenderList() error {\n\tself.FilterAndSort()\n\n\tself.Gui.Update(func() error {\n\t\tself.View.Clear()\n\t\ttable := lo.Map(self.List.GetItems(), func(item T, index int) []string {\n\t\t\treturn self.GetTableCells(item)\n\t\t})\n\t\trenderedTable, err := utils.RenderTable(table)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfmt.Fprint(self.View, renderedTable)\n\n\t\tif self.OnRerender != nil {\n\t\t\tif err := self.OnRerender(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n...\n", "needle_token_start": 11135, "needle_token_end": 11472, "code_context_ntokens": 15074, "output": ["```go\nfunc TestFilteredListFilter(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs func(int, int) bool\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 0 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 1 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 2}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.Filter(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t}\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "plotGraph", "language": "go", "path": "pkg/gui/presentation/container_stats.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To generate a visual representation of statistical data over time for a specific metric of a container.\n2. **Input**: Takes a container object, a configuration for the graph, and the desired width of the graph.\n3. **Output**: Returns a string that represents the plotted graph or an error message if the data cannot be processed.\n4. **Procedure**: \n   - Locks the container's statistics for thread safety.\n   - Extracts the specified metric from the container's historical statistics using a lookup mechanism.\n   - Converts the extracted values into a float format.\n   - Determines the maximum and minimum values for the graph scale based on configuration or data.\n   - Configures the graph's height and other visual properties.\n   - Uses a library to generate an ASCII graph from the data.\n   - Unlocks the statistics and returns the generated graph or an error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/gui/layout.go\npackage gui\n\nimport (\n\t\"github.com/jesseduffield/gocui\"\n)\n\nconst UNKNOWN_VIEW_ERROR_MSG = \"unknown view\"\n\n// getFocusLayout returns a manager function for when view gain and lose focus\nfunc (gui *Gui) getFocusLayout() func(g *gocui.Gui) error {\n\tvar previousView *gocui.View\n\treturn func(g *gocui.Gui) error {\n\t\tnewView := gui.g.CurrentView()\n\t\tif err := gui.onFocusChange(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// for now we don't consider losing focus to a popup panel as actually losing focus\n\t\tif newView != previousView && !gui.isPopupPanel(newView.Name()) {\n\t\t\tgui.onFocusLost(previousView, newView)\n\t\t\tgui.onFocus(newView)\n\t\t\tpreviousView = newView\n\t\t}\n\t\treturn nil\n\t}\n}\n\nfunc (gui *Gui) onFocusChange() error {\n\tcurrentView := gui.g.CurrentView()\n\tfor _, view := range gui.g.Views() {\n\t\tview.Highlight = view == currentView && view.Name() != \"main\"\n\t}\n\treturn nil\n}\n\nfunc (gui *Gui) onFocusLost(v *gocui.View, newView *gocui.View) {\n\tif v == nil {\n\t\treturn\n\t}\n\n\tif !gui.isPopupPanel(newView.Name()) {\n\t\tv.ParentView = nil\n\t}\n\n\t// refocusing because in responsive mode (when the window is very short) we want to ensure that after the view size changes we can still see the last selected item\n\tgui.focusPointInView(v)\n\n\tgui.Log.Info(v.Name() + \" focus lost\")\n}\n\nfunc (gui *Gui) onFocus(v *gocui.View) {\n\tif v == nil {\n\t\treturn\n\t}\n\n\tgui.focusPointInView(v)\n\n\tgui.Log.Info(v.Name() + \" focus gained\")\n...\n// Path: pkg/gui/main_panel.go\npackage gui\n\nimport (\n\t\"math\"\n\n\t\"github.com/jesseduffield/gocui\"\n)\n\nfunc (gui *Gui) scrollUpMain() error {\n\tmainView := gui.Views.Main\n\tmainView.Autoscroll = false\n\tox, oy := mainView.Origin()\n\tnewOy := int(math.Max(0, float64(oy-gui.Config.UserConfig.Gui.ScrollHeight)))\n\treturn mainView.SetOrigin(ox, newOy)\n}\n\nfunc (gui *Gui) scrollDownMain() error {\n\tmainView := gui.Views.Main\n\tmainView.Autoscroll = false\n\tox, oy := mainView.Origin()\n\n\treservedLines := 0\n\tif !gui.Config.UserConfig.Gui.ScrollPastBottom {\n\t\t_, sizeY := mainView.Size()\n\t\treservedLines = sizeY\n\t}\n\n\ttotalLines := mainView.ViewLinesHeight()\n\tif oy+reservedLines >= totalLines {\n\t\treturn nil\n\t}\n\n\treturn mainView.SetOrigin(ox, oy+gui.Config.UserConfig.Gui.ScrollHeight)\n}\n\nfunc (gui *Gui) scrollLeftMain(g *gocui.Gui, v *gocui.View) error {\n\tmainView := gui.Views.Main\n\tox, oy := mainView.Origin()\n\tnewOx := int(math.Max(0, float64(ox-gui.Config.UserConfig.Gui.ScrollHeight)))\n\n\treturn mainView.SetOrigin(newOx, oy)\n}\n\nfunc (gui *Gui) scrollRightMain(g *gocui.Gui, v *gocui.View) error {\n\tmainView := gui.Views.Main\n\tox, oy := mainView.Origin()\n\n\tcontent := mainView.ViewBufferLines()\n\tvar largestNumberOfCharacters int\n\tfor _, txt := range content {\n\t\tif len(txt) > largestNumberOfCharacters {\n\t\t\tlargestNumberOfCharacters = len(txt)\n\t\t}\n\t}\n\n\tsizeX, _ := mainView.Size()\n\tif ox+sizeX >= largestNumberOfCharacters {\n\t\treturn nil\n\t}\n\n\treturn mainView.SetOrigin(ox+gui.Config.UserConfig.Gui.ScrollHeight, oy)\n}\n\nfunc (gui *Gui) autoScrollMain(g *gocui.Gui, v *gocui.View) error {\n\tgui.Views.Main.Autoscroll = true\n\treturn nil\n}\n\nfunc (gui *Gui) jumpToTopMain(g *gocui.Gui, v *gocui.View) error {\n\tgui.Views.Main.Autoscroll = false\n\t_ = gui.Views.Main.SetOrigin(0, 0)\n\t_ = gui.Views.Main.SetCursor(0, 0)\n\treturn nil\n}\n\nfunc (gui *Gui) onMainTabClick(tabIndex int) error {\n\tgui.Log.Warn(tabIndex)\n\n\tcurrentSidePanel, ok := gui.currentSidePanel()\n\n\tif !ok {\n\t\treturn nil\n\t}\n\n\tcurrentSidePanel.SetMainTabIndex(tabIndex)\n\treturn currentSidePanel.HandleSelect()\n}\n\nfunc (gui *Gui) handleEnterMain(g *gocui.Gui, v *gocui.View) error {\n\tmainView := gui.Views.Main\n\tmainView.ParentView = v\n\n\treturn gui.switchFocus(mainView)\n}\n\nfunc (gui *Gui) handleExitMain(g *gocui.Gui, v *gocui.View) error {\n\tv.ParentView = nil\n\treturn gui.returnFocus()\n}\n\nfunc (gui *Gui) handleMainClick() error {\n\tif gui.popupPanelFocused() {\n\t\treturn nil\n\t}\n\n\tcurrentView := gui.g.CurrentView()\n\n\tif currentView.Name() != \"main\" {\n\t\tgui.Views.Main.ParentView = currentView\n\t}\n\n\treturn gui.switchFocus(gui.Views.Main)\n}\n\n// Path: pkg/gui/menu_panel.go\npackage gui\n\nimport (\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\ntype CreateMenuOptions struct {\n\tTitle      string\n\tItems      []*types.MenuItem\n\tHideCancel bool\n}\n\nfunc (gui *Gui) getMenuPanel() *panels.SideListPanel[*types.MenuItem] {\n\treturn &panels.SideListPanel[*types.MenuItem]{\n\t\tListPanel: panels.ListPanel[*types.MenuItem]{\n\t\t\tList: panels.NewFilteredList[*types.MenuItem](),\n\t\t\tView: gui.Views.Menu,\n\t\t},\n\t\tNoItemsMessage: \"\",\n\t\tGui:            gui.intoInterface(),\n\t\tOnClick:        gui.onMenuPress,\n\t\tSort:           nil,\n\t\tGetTableCells:  presentation.GetMenuItemDisplayStrings,\n\t\tOnRerender: func() error {\n\t\t\treturn gui.resizePopupPanel(gui.Views.Menu)\n\t\t},\n\t\t// so that we can avoid some UI trickiness, the menu will not have filtering\n\t\t// abillity yet. To support it, we would need to have filter state against\n\t\t// each panel (e.g. for when you filter the images panel, then bring up\n\t\t// the options menu, then try to filter that too.\n\t\tDisableFilter: true,\n\t}\n}\n\nfunc (gui *Gui) onMenuPress(menuItem *types.MenuItem) error {\n\tif err := gui.handleMenuClose(); err != nil {\n\t\treturn err\n\t}\n\n\tif menuItem.OnPress != nil {\n\t\treturn menuItem.OnPress()\n\t}\n\n\treturn nil\n}\n\nfunc (gui *Gui) handleMenuPress() error {\n\tselectedMenuItem, err := gui.Panels.Menu.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\treturn gui.onMenuPress(selectedMenuItem)\n}\n\nfunc (gui *Gui) Menu(opts CreateMenuOptions) error {\n\tif !opts.HideCancel {\n\t\t// this is mutative but I'm okay with that for now\n\t\topts.Items = append(opts.Items, &types.MenuItem{\n\t\t\tLabelColumns: []string{gui.Tr.Cancel},\n\t\t\tOnPress: func() error {\n\t\t\t\treturn nil\n\t\t\t},\n\t\t})\n\t}\n\n\tmaxColumnSize := 1\n\n\tfor _, item := range opts.Items {\n\t\tif item.LabelColumns == nil {\n\t\t\titem.LabelColumns = []string{item.Label}\n\t\t}\n\n\t\tif item.OpensMenu {\n\t\t\titem.LabelColumns[0] = utils.OpensMenuStyle(item.LabelColumns[0])\n\t\t}\n\n\t\tmaxColumnSize = utils.Max(maxColumnSize, len(item.LabelColumns))\n\t}\n\n\tfor _, item := range opts.Items {\n\t\tif len(item.LabelColumns) < maxColumnSize {\n\t\t\t// we require that each item has the same number of columns so we're padding out with blank strings\n\t\t\t// if this item has too few\n\t\t\titem.LabelColumns = append(item.LabelColumns, make([]string, maxColumnSize-len(item.LabelColumns))...)\n\t\t}\n\t}\n\n\tgui.Panels.Menu.SetItems(opts.Items)\n\tgui.Panels.Menu.SetSelectedLineIdx(0)\n\n\tif err := gui.Panels.Menu.RerenderList(); err != nil {\n\t\treturn err\n\t}\n\n\tgui.Views.Menu.Title = opts.Title\n\tgui.Views.Menu.Visible = true\n\n\treturn gui.switchFocus(gui.Views.Menu)\n}\n\n// specific functions\n\nfunc (gui *Gui) renderMenuOptions() error {\n\toptionsMap := map[string]string{\n\t\t\"esc\":   gui.Tr.Close,\n\t\t\"\u2191 \u2193\":   gui.Tr.Navigate,\n\t\t\"enter\": gui.Tr.Execute,\n\t}\n\treturn gui.renderOptionsMap(optionsMap)\n}\n\nfunc (gui *Gui) handleMenuClose() error {\n\tgui.Views.Menu.Visible = false\n\n\t// this code is here for when we do add filter ability to the menu panel,\n\t// though it's currently disabled\n\tif gui.State.Filter.panel == gui.Panels.Menu {\n\t\tif err := gui.clearFilter(); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// we need to remove the view from the view stack because we're about to\n\t\t// return focus and don't want to land in the search view when it was searching\n\t\t// the menu in the first place\n\t\tgui.removeViewFromStack(gui.Views.Filter)\n\t}\n\n\treturn gui.returnFocus()\n}\n\n// Path: pkg/gui/networks_panel.go\npackage gui\n\nimport (\n\t\"strconv\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/presentation\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\nfunc (gui *Gui) getNetworksPanel() *panels.SideListPanel[*commands.Network] {\n\treturn &panels.SideListPanel[*commands.Network]{\n\t\tContextState: &panels.ContextState[*commands.Network]{\n\t\t\tGetMainTabs: func() []panels.MainTab[*commands.Network] {\n\t\t\t\treturn []panels.MainTab[*commands.Network]{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey:    \"config\",\n\t\t\t\t\t\tTitle:  gui.Tr.ConfigTitle,\n\t\t\t\t\t\tRender: gui.renderNetworkConfig,\n\t\t\t\t\t},\n\t\t\t\t}\n\t\t\t},\n\t\t\tGetItemContextCacheKey: func(network *commands.Network) string {\n\t\t\t\treturn \"networks-\" + network.Name\n\t\t\t},\n\t\t},\n\t\tListPanel: panels.ListPanel[*commands.Network]{\n\t\t\tList: panels.NewFilteredList[*commands.Network](),\n\t\t\tView: gui.Views.Networks,\n\t\t},\n\t\tNoItemsMessage: gui.Tr.NoNetworks,\n\t\tGui:            gui.intoInterface(),\n\t\t// we're sorting these networks based on whether they have labels defined,\n\t\t// because those are the ones you typically care about.\n\t\t// Within that, we also sort them alphabetically\n\t\tSort: func(a *commands.Network, b *commands.Network) bool {\n\t\t\treturn a.Name < b.Name\n\t\t},\n\t\tGetTableCells: presentation.GetNetworkDisplayStrings,\n\t}\n}\n\nfunc (gui *Gui) renderNetworkConfig(network *commands.Network) tasks.TaskFunc {\n\treturn gui.NewSimpleRenderStringTask(func() string { return gui.networkConfigStr(network) })\n}\n\nfunc (gui *Gui) networkConfigStr(network *commands.Network) string {\n\tpadding := 15\n\toutput := \"\"\n\toutput += utils.WithPadding(\"ID: \", padding) + network.Network.ID + \"\\n\"\n\toutput += utils.WithPadding(\"Name: \", padding) + network.Name + \"\\n\"\n\toutput += utils.WithPadding(\"Driver: \", padding) + network.Network.Driver + \"\\n\"\n\toutput += utils.WithPadding(\"Scope: \", padding) + network.Network.Scope + \"\\n\"\n\toutput += utils.WithPadding(\"EnabledIPV6: \", padding) + strconv.FormatBool(network.Network.EnableIPv6) + \"\\n\"\n\toutput += utils.WithPadding(\"Internal: \", padding) + strconv.FormatBool(network.Network.Internal) + \"\\n\"\n\toutput += utils.WithPadding(\"Attachable: \", padding) + strconv.FormatBool(network.Network.Attachable) + \"\\n\"\n\toutput += utils.WithPadding(\"Ingress: \", padding) + strconv.FormatBool(network.Network.Ingress) + \"\\n\"\n\n\toutput += utils.WithPadding(\"Containers: \", padding)\n\tif len(network.Network.Containers) > 0 {\n\t\toutput += \"\\n\"\n\t\tfor _, v := range network.Network.Containers {\n\t\t\toutput += utils.FormatMapItem(padding, v.Name, v.EndpointID)\n\t\t}\n\t} else {\n\t\toutput += \"none\\n\"\n\t}\n\n\toutput += \"\\n\"\n\toutput += utils.WithPadding(\"Labels: \", padding) + utils.FormatMap(padding, network.Network.Labels) + \"\\n\"\n\toutput += utils.WithPadding(\"Options: \", padding) + utils.FormatMap(padding, network.Network.Options)\n\n\treturn output\n}\n\nfunc (gui *Gui) reloadNetworks() error {\n\tif err := gui.refreshStateNetworks(); err != nil {\n\t\treturn err\n\t}\n\n\treturn gui.Panels.Networks.RerenderList()\n}\n\nfunc (gui *Gui) refreshStateNetworks() error {\n\tnetworks, err := gui.DockerCommand.RefreshNetworks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tgui.Panels.Networks.SetItems(networks)\n\n\treturn nil\n}\n\nfunc (gui *Gui) handleNetworksRemoveMenu(g *gocui.Gui, v *gocui.View) error {\n\tnetwork, err := gui.Panels.Networks.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\ttype removeNetworkOption struct {\n\t\tdescription string\n\t\tcommand     string\n\t}\n\n\toptions := []*removeNetworkOption{\n\t\t{\n\t\t\tdescription: gui.Tr.Remove,\n\t\t\tcommand:     utils.WithShortSha(\"docker network rm \" + network.Name),\n\t\t},\n\t}\n\n\tmenuItems := lo.Map(options, func(option *removeNetworkOption, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: []string{option.description, color.New(color.FgRed).Sprint(option.command)},\n\t\t\tOnPress: func() error {\n\t\t\t\treturn gui.WithWaitingStatus(gui.Tr.RemovingStatus, func() error {\n\t\t\t\t\tif err := network.Remove(); err != nil {\n\t\t\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t\t\t}\n\t\t\t\t\treturn nil\n\t\t\t\t})\n\t\t\t},\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle: \"\",\n\t\tItems: menuItems,\n\t})\n}\n\nfunc (gui *Gui) handlePruneNetworks() error {\n\treturn gui.createConfirmationPanel(gui.Tr.Confirm, gui.Tr.ConfirmPruneNetworks, func(g *gocui.Gui, v *gocui.View) error {\n\t\treturn gui.WithWaitingStatus(gui.Tr.PruningStatus, func() error {\n\t\t\terr := gui.DockerCommand.PruneNetworks()\n\t\t\tif err != nil {\n\t\t\t\treturn gui.createErrorPanel(err.Error())\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}, nil)\n}\n\nfunc (gui *Gui) handleNetworksCustomCommand(g *gocui.Gui, v *gocui.View) error {\n\tnetwork, err := gui.Panels.Networks.GetSelectedItem()\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tcommandObject := gui.DockerCommand.NewCommandObject(commands.CommandObject{\n\t\tNetwork: network,\n\t})\n\n\tcustomCommands := gui.Config.UserConfig.CustomCommands.Networks\n\n\treturn gui.createCustomCommandMenu(customCommands, commandObject)\n}\n\nfunc (gui *Gui) handleNetworksBulkCommand(g *gocui.Gui, v *gocui.View) error {\n\tbaseBulkCommands := []config.CustomCommand{\n\t\t{\n\t\t\tName:             gui.Tr.PruneNetworks,\n\t\t\tInternalFunction: gui.handlePruneNetworks,\n\t\t},\n\t}\n\n\tbulkCommands := append(baseBulkCommands, gui.Config.UserConfig.BulkCommands.Networks...)\n\tcommandObject := gui.DockerCommand.NewCommandObject(commands.CommandObject{})\n\n\treturn gui.createBulkCommandMenu(bulkCommands, commandObject)\n}\n\n// Path: pkg/gui/options_menu_panel.go\npackage gui\n\nimport (\n\t\"github.com/samber/lo\"\n\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n)\n\nfunc (gui *Gui) getBindings(v *gocui.View) []*Binding {\n\tvar bindingsGlobal, bindingsPanel []*Binding\n\n\tbindings := gui.GetInitialKeybindings()\n\n\tfor _, binding := range bindings {\n\t\tif binding.GetKey() != \"\" && binding.Description != \"\" {\n\t\t\tswitch binding.ViewName {\n\t\t\tcase \"\":\n\t\t\t\tbindingsGlobal = append(bindingsGlobal, binding)\n\t\t\tcase v.Name():\n\t\t\t\tbindingsPanel = append(bindingsPanel, binding)\n\t\t\t}\n\t\t}\n\t}\n\n\t// check if we have any keybindings from our parent view to add\n\tif v.ParentView != nil {\n\tL:\n\t\tfor _, binding := range bindings {\n\t\t\tif binding.GetKey() != \"\" && binding.Description != \"\" {\n\t\t\t\tif binding.ViewName == v.ParentView.Name() {\n\t\t\t\t\t// if we haven't got a conflict we will display the binding\n\t\t\t\t\tfor _, ownBinding := range bindingsPanel {\n\t\t\t\t\t\tif ownBinding.GetKey() == binding.GetKey() {\n\t\t\t\t\t\t\tcontinue L\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tbindingsPanel = append(bindingsPanel, binding)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// append dummy element to have a separator between\n\t// panel and global keybindings\n\tbindingsPanel = append(bindingsPanel, &Binding{})\n\treturn append(bindingsPanel, bindingsGlobal...)\n}\n\nfunc (gui *Gui) handleCreateOptionsMenu(g *gocui.Gui, v *gocui.View) error {\n\tif gui.isPopupPanel(v.Name()) {\n\t\treturn nil\n\t}\n\n\tmenuItems := lo.Map(gui.getBindings(v), func(binding *Binding, _ int) *types.MenuItem {\n\t\treturn &types.MenuItem{\n\t\t\tLabelColumns: []string{binding.GetKey(), binding.Description},\n\t\t\tOnPress: func() error {\n\t\t\t\tif binding.Key == nil {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\n\t\t\t\treturn binding.Handler(g, v)\n\t\t\t},\n\t\t}\n\t})\n\n\treturn gui.Menu(CreateMenuOptions{\n\t\tTitle:      gui.Tr.MenuTitle,\n\t\tItems:      menuItems,\n\t\tHideCancel: true,\n\t})\n}\n\n// Path: pkg/gui/panels/context_state.go\npackage panels\n\nimport (\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/samber/lo\"\n)\n\n// A 'context' generally corresponds to an item and the tab in the main panel that we're\n// displaying. So if we switch to a new item, or change the tab in the panel panel\n// for the current item, we end up with a new context. When we have a new context,\n// we render new content to the main panel.\ntype ContextState[T any] struct {\n\t// index of the currently selected tab in the main view.\n\tmainTabIdx int\n\t// this function returns the tabs that we can display for an item (the tabs\n\t// are shown on the main view)\n\tGetMainTabs func() []MainTab[T]\n\t// This tells us whether we need to re-render to the main panel for a given item.\n\t// This should include the item's ID and if you want to invalidate the cache for\n\t// some other reason, you can add that to the key as well (e.g. the container's state).\n\tGetItemContextCacheKey func(item T) string\n}\n\ntype MainTab[T any] struct {\n\t// key used as part of the context cache key\n\tKey string\n\t// title of the tab, rendered in the main view\n\tTitle string\n\t// function to render the content of the tab\n\tRender func(item T) tasks.TaskFunc\n}\n\nfunc (self *ContextState[T]) GetMainTabTitles() []string {\n\treturn lo.Map(self.GetMainTabs(), func(tab MainTab[T], _ int) string {\n\t\treturn tab.Title\n\t})\n}\n\nfunc (self *ContextState[T]) GetCurrentContextKey(item T) string {\n\treturn self.GetItemContextCacheKey(item) + \"-\" + self.GetCurrentMainTab().Key\n}\n\nfunc (self *ContextState[T]) GetCurrentMainTab() MainTab[T] {\n\treturn self.GetMainTabs()[self.mainTabIdx]\n}\n\nfunc (self *ContextState[T]) HandleNextMainTab() {\n\ttabs := self.GetMainTabs()\n\n\tif len(tabs) == 0 {\n\t\treturn\n\t}\n\n\tself.mainTabIdx = (self.mainTabIdx + 1) % len(tabs)\n}\n\nfunc (self *ContextState[T]) HandlePrevMainTab() {\n\ttabs := self.GetMainTabs()\n\n\tif len(tabs) == 0 {\n\t\treturn\n\t}\n\n\tself.mainTabIdx = (self.mainTabIdx - 1 + len(tabs)) % len(tabs)\n}\n\nfunc (self *ContextState[T]) SetMainTabIndex(index int) {\n\tself.mainTabIdx = index\n}\n\n// Path: pkg/gui/panels/filtered_list.go\npackage panels\n\nimport (\n\t\"sort\"\n\t\"sync\"\n)\n\ntype FilteredList[T comparable] struct {\n\tallItems []T\n\t// indices of items in the allItems slice that are included in the filtered list\n\tindices []int\n\n\tmutex sync.RWMutex\n}\n\nfunc NewFilteredList[T comparable]() *FilteredList[T] {\n\treturn &FilteredList[T]{}\n}\n\nfunc (self *FilteredList[T]) SetItems(items []T) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tself.allItems = items\n\tself.indices = make([]int, len(items))\n\tfor i := range self.indices {\n\t\tself.indices[i] = i\n\t}\n}\n\nfunc (self *FilteredList[T]) Filter(filter func(T, int) bool) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tself.indices = self.indices[:0]\n\tfor i, item := range self.allItems {\n\t\tif filter(item, i) {\n\t\t\tself.indices = append(self.indices, i)\n\t\t}\n\t}\n}\n\nfunc (self *FilteredList[T]) Sort(less func(T, T) bool) {\n\tself.mutex.Lock()\n\tdefer self.mutex.Unlock()\n\n\tif less == nil {\n\t\treturn\n\t}\n\n\tsort.Slice(self.indices, func(i, j int) bool {\n\t\treturn less(self.allItems[self.indices[i]], self.allItems[self.indices[j]])\n\t})\n}\n\nfunc (self *FilteredList[T]) Get(index int) T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn self.allItems[self.indices[index]]\n}\n\nfunc (self *FilteredList[T]) TryGet(index int) (T, bool) {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tif index < 0 || index >= len(self.indices) {\n\t\tvar zero T\n\t\treturn zero, false\n\t}\n\n\treturn self.allItems[self.indices[index]], true\n}\n\n// returns the length of the filtered list\nfunc (self *FilteredList[T]) Len() int {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn len(self.indices)\n}\n\nfunc (self *FilteredList[T]) GetIndex(item T) int {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tfor i, index := range self.indices {\n\t\tif self.allItems[index] == item {\n\t\t\treturn i\n\t\t}\n\t}\n\treturn -1\n}\n\nfunc (self *FilteredList[T]) GetItems() []T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\tresult := make([]T, len(self.indices))\n\tfor i, index := range self.indices {\n\t\tresult[i] = self.allItems[index]\n\t}\n\treturn result\n}\n\nfunc (self *FilteredList[T]) GetAllItems() []T {\n\tself.mutex.RLock()\n\tdefer self.mutex.RUnlock()\n\n\treturn self.allItems\n}\n\n// Path: pkg/gui/panels/filtered_list_test.go\npackage panels\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestFilteredListGet(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs int\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 1,\n\t\t\twant: 2,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 2,\n\t\t\twant: 3,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: 0,\n\t\t\twant: 2,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.Get(tt.args); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.Get() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListLen(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\twant: 3,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\twant: 1,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.Len(); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.Len() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListFilter(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs func(int, int) bool\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 0 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, _ int) bool { return i%2 == 1 },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 2}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.Filter(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t}\n}\n\nfunc TestFilteredListSort(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs func(int, int) bool\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, j int) bool { return i < j },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: func(i int, j int) bool { return i > j },\n\t\t\twant: &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{2, 1, 0}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.Sort(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t}\n}\n\nfunc TestFilteredListGetIndex(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs int\n\t\twant int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 1,\n\t\t\twant: 0,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: 2,\n\t\t\twant: 1,\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: 0,\n\t\t\twant: -1,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tif got := tt.f.GetIndex(tt.args); got != tt.want {\n\t\t\tt.Errorf(\"FilteredList.GetIndex() = %v, want %v\", got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestFilteredListGetItems(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\twant []int\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\twant: []int{1, 2, 3},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\twant: []int{2},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tgot := tt.f.GetItems()\n\t\tassert.EqualValues(t, got, tt.want)\n\t}\n}\n\nfunc TestFilteredListSetItems(t *testing.T) {\n\ttests := []struct {\n\t\tf    *FilteredList[int]\n\t\targs []int\n\t\twant *FilteredList[int]\n\t}{\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{0, 1, 2}},\n\t\t\targs: []int{4, 5, 6},\n\t\t\twant: &FilteredList[int]{allItems: []int{4, 5, 6}, indices: []int{0, 1, 2}},\n\t\t},\n\t\t{\n\t\t\tf:    &FilteredList[int]{allItems: []int{1, 2, 3}, indices: []int{1}},\n\t\t\targs: []int{4},\n\t\t\twant: &FilteredList[int]{allItems: []int{4}, indices: []int{0}},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\ttt.f.SetItems(tt.args)\n\t\tassert.EqualValues(t, tt.f.indices, tt.want.indices)\n\t\tassert.EqualValues(t, tt.f.allItems, tt.want.allItems)\n\t}\n}\n\n// Path: pkg/gui/panels/list_panel.go\npackage panels\n\nimport (\n\t\"github.com/jesseduffield/gocui\"\n\tlcUtils \"github.com/jesseduffield/lazycore/pkg/utils\"\n)\n\ntype ListPanel[T comparable] struct {\n\tSelectedIdx int\n\tList        *FilteredList[T]\n\tView        *gocui.View\n}\n\nfunc (self *ListPanel[T]) SetSelectedLineIdx(value int) {\n\tclampedValue := 0\n\tif self.List.Len() > 0 {\n\t\tclampedValue = lcUtils.Clamp(value, 0, self.List.Len()-1)\n\t}\n\n\tself.SelectedIdx = clampedValue\n}\n\nfunc (self *ListPanel[T]) clampSelectedLineIdx() {\n\tclamped := lcUtils.Clamp(self.SelectedIdx, 0, self.List.Len()-1)\n\n\tif clamped != self.SelectedIdx {\n\t\tself.SelectedIdx = clamped\n\t}\n}\n\n// moves the cursor up or down by the given amount (up for negative values)\nfunc (self *ListPanel[T]) moveSelectedLine(delta int) {\n\tself.SetSelectedLineIdx(self.SelectedIdx + delta)\n}\n\nfunc (self *ListPanel[T]) SelectNextLine() {\n\tself.moveSelectedLine(1)\n}\n\nfunc (self *ListPanel[T]) SelectPrevLine() {\n\tself.moveSelectedLine(-1)\n}\n\n// Path: pkg/gui/panels/side_list_panel.go\npackage panels\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\ntype ISideListPanel interface {\n\tSetMainTabIndex(int)\n\tHandleSelect() error\n\tGetView() *gocui.View\n\tRefocus()\n\tRerenderList() error\n\tIsFilterDisabled() bool\n\tIsHidden() bool\n\tHandleNextLine() error\n\tHandlePrevLine() error\n\tHandleClick() error\n\tHandlePrevMainTab() error\n\tHandleNextMainTab() error\n}\n\n// list panel at the side of the screen that renders content to the main panel\ntype SideListPanel[T comparable] struct {\n\tContextState *ContextState[T]\n\n\tListPanel[T]\n\n\t// message to render in the main view if there are no items in the panel\n\t// and it has focus. Leave empty if you don't want to render anything\n\tNoItemsMessage string\n\n\t// a representation of the gui\n\tGui IGui\n\n\t// this Filter is applied on top of additional default filters\n\tFilter func(T) bool\n\tSort   func(a, b T) bool\n\n\t// a callback to invoke when the item is clicked\n\tOnClick func(T) error\n\n\t// returns the cells that we render to the view in a table format. The cells will\n\t// be rendered with padding.\n\tGetTableCells func(T) []string\n\n\t// function to be called after re-rendering list. Can be nil\n\tOnRerender func() error\n\n\t// set this to true if you don't want to allow manual filtering via '/'\n\tDisableFilter bool\n\n\t// This can be nil if you want to always show the panel\n\tHide func() bool\n}\n\nvar _ ISideListPanel = &SideListPanel[int]{}\n\ntype IGui interface {\n\tHandleClick(v *gocui.View, itemCount int, selectedLine *int, handleSelect func() error) error\n\tNewSimpleRenderStringTask(getContent func() string) tasks.TaskFunc\n\tFocusY(selectedLine int, itemCount int, view *gocui.View)\n\tShouldRefresh(contextKey string) bool\n\tGetMainView() *gocui.View\n\tIsCurrentView(*gocui.View) bool\n\tFilterString(view *gocui.View) string\n\tIgnoreStrings() []string\n\tUpdate(func() error)\n\n\tQueueTask(f func(ctx context.Context)) error\n}\n\nfunc (self *SideListPanel[T]) HandleClick() error {\n\titemCount := self.List.Len()\n\thandleSelect := self.HandleSelect\n\tselectedLine := &self.SelectedIdx\n\n\tif err := self.Gui.HandleClick(self.View, itemCount, selectedLine, handleSelect); err != nil {\n\t\treturn err\n\t}\n\n\tif self.OnClick != nil {\n\t\tselectedItem, err := self.GetSelectedItem()\n\t\tif err == nil {\n\t\t\treturn self.OnClick(selectedItem)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (self *SideListPanel[T]) GetView() *gocui.View {\n\treturn self.View\n}\n\nfunc (self *SideListPanel[T]) HandleSelect() error {\n\titem, err := self.GetSelectedItem()\n\tif err != nil {\n\t\tif err.Error() != self.NoItemsMessage {\n\t\t\treturn err\n\t\t}\n\n\t\tif self.NoItemsMessage != \"\" {\n\t\t\tself.Gui.NewSimpleRenderStringTask(func() string { return self.NoItemsMessage })\n\t\t}\n\n\t\treturn nil\n\t}\n\n\tself.Refocus()\n\n\treturn self.renderContext(item)\n}\n\nfunc (self *SideListPanel[T]) renderContext(item T) error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tkey := self.ContextState.GetCurrentContextKey(item)\n\tif !self.Gui.ShouldRefresh(key) {\n\t\treturn nil\n\t}\n\n\tmainView := self.Gui.GetMainView()\n\tmainView.Tabs = self.ContextState.GetMainTabTitles()\n\tmainView.TabIndex = self.ContextState.mainTabIdx\n\n\ttask := self.ContextState.GetCurrentMainTab().Render(item)\n\n\treturn self.Gui.QueueTask(task)\n}\n\nfunc (self *SideListPanel[T]) GetSelectedItem() (T, error) {\n\tvar zero T\n\n\titem, ok := self.List.TryGet(self.SelectedIdx)\n\tif !ok {\n\t\t// could probably have a better error here\n\t\treturn zero, errors.New(self.NoItemsMessage)\n\t}\n\n\treturn item, nil\n}\n\nfunc (self *SideListPanel[T]) HandleNextLine() error {\n\tself.SelectNextLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevLine() error {\n\tself.SelectPrevLine()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandleNextMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandleNextMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) HandlePrevMainTab() error {\n\tif self.ContextState == nil {\n\t\treturn nil\n\t}\n\n\tself.ContextState.HandlePrevMainTab()\n\n\treturn self.HandleSelect()\n}\n\nfunc (self *SideListPanel[T]) Refocus() {\n\tself.Gui.FocusY(self.SelectedIdx, self.List.Len(), self.View)\n}\n\nfunc (self *SideListPanel[T]) SetItems(items []T) {\n\tself.List.SetItems(items)\n\tself.FilterAndSort()\n}\n\nfunc (self *SideListPanel[T]) FilterAndSort() {\n\tfilterString := self.Gui.FilterString(self.View)\n\n\tself.List.Filter(func(item T, index int) bool {\n\t\tif self.Filter != nil && !self.Filter(item) {\n\t\t\treturn false\n\t\t}\n\n\t\tif lo.SomeBy(self.Gui.IgnoreStrings(), func(ignore string) bool {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, ignore)\n\t\t\t})\n\t\t}) {\n\t\t\treturn false\n\t\t}\n\n\t\tif filterString != \"\" {\n\t\t\treturn lo.SomeBy(self.GetTableCells(item), func(searchString string) bool {\n\t\t\t\treturn strings.Contains(searchString, filterString)\n\t\t\t})\n\t\t}\n\n\t\treturn true\n\t})\n\n\tself.List.Sort(self.Sort)\n\n\tself.clampSelectedLineIdx()\n}\n\nfunc (self *SideListPanel[T]) RerenderList() error {\n\tself.FilterAndSort()\n\n\tself.Gui.Update(func() error {\n\t\tself.View.Clear()\n\t\ttable := lo.Map(self.List.GetItems(), func(item T, index int) []string {\n\t\t\treturn self.GetTableCells(item)\n\t\t})\n\t\trenderedTable, err := utils.RenderTable(table)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfmt.Fprint(self.View, renderedTable)\n\n\t\tif self.OnRerender != nil {\n\t\t\tif err := self.OnRerender(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif self.Gui.IsCurrentView(self.View) {\n\t\t\treturn self.HandleSelect()\n\t\t}\n\t\treturn nil\n\t})\n\n\treturn nil\n}\n\nfunc (self *SideListPanel[T]) SetMainTabIndex(index int) {\n\tif self.ContextState == nil {\n\t\treturn\n\t}\n\n\tself.ContextState.SetMainTabIndex(index)\n}\n\nfunc (self *SideListPanel[T]) IsFilterDisabled() bool {\n\treturn self.DisableFilter\n}\n\nfunc (self *SideListPanel[T]) IsHidden() bool {\n\tif self.Hide == nil {\n\t\treturn false\n\t}\n\n\treturn self.Hide()\n}\n\n// Path: pkg/gui/presentation/container_stats.go\npackage presentation\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/asciigraph\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/mcuadros/go-lookup\"\n\t\"github.com/samber/lo\"\n)\n\nfunc RenderStats(userConfig *config.UserConfig, container *commands.Container, viewWidth int) (string, error) {\n\tstats, ok := container.GetLastStats()\n\tif !ok {\n\t\treturn \"\", nil\n\t}\n\n\tgraphSpecs := userConfig.Stats.Graphs\n\tgraphs := make([]string, len(graphSpecs))\n\tfor i, spec := range graphSpecs {\n\t\tgraph, err := plotGraph(container, spec, viewWidth-10)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tgraphs[i] = utils.ColoredString(graph, utils.GetColorAttribute(spec.Color))\n\t}\n\n\tpidsCount := fmt.Sprintf(\"PIDs: %d\", stats.ClientStats.PidsStats.Current)\n\tdataReceived := fmt.Sprintf(\"Traffic received: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.RxBytes))\n\tdataSent := fmt.Sprintf(\"Traffic sent: %s\", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.TxBytes))\n\n\toriginalStats, err := utils.MarshalIntoYaml(stats)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tcontents := fmt.Sprintf(\"\\n\\n%s\\n\\n%s\\n\\n%s\\n%s\\n\\n%s\",\n\t\tutils.ColoredString(strings.Join(graphs, \"\\n\\n\"), color.FgGreen),\n\t\tpidsCount,\n\t\tdataReceived,\n\t\tdataSent,\n\t\tutils.ColoredYamlString(string(originalStats)),\n\t)\n\n\treturn contents, nil\n}\n\n// plotGraph returns the plotted graph based on the graph spec and the stat history\n\nfunc plotGraph(container *commands.Container, spec config.GraphConfig, width int) (string, error) {\n\tcontainer.StatsMutex.Lock()\n\tdefer container.StatsMutex.Unlock()\n\n\tdata := make([]float64, len(container.StatHistory))\n\n\tfor i, stats := range container.StatHistory {\n\t\tvalue, err := lookup.LookupString(stats, spec.StatPath)\n\t\tif err != nil {\n\t\t\treturn \"Could not find key: \" + spec.StatPath, nil\n\t\t}\n\t\tfloatValue, err := getFloat(value.Interface())\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tdata[i] = floatValue\n\t}\n\n\tmax := spec.Max\n\tif spec.MaxType == \"\" {\n\t\tmax = lo.Max(data)\n\t}\n\n\tmin := spec.Min\n\tif spec.MinType == \"\" {\n\t\tmin = lo.Min(data)\n\t}\n\n\theight := 10\n\tif spec.Height > 0 {\n\t\theight = spec.Height\n\t}\n\n\tcaption := fmt.Sprintf(\n\t\t\"%s: %0.2f (%v)\",\n\t\tspec.Caption,\n\t\tdata[len(data)-1],\n\t\ttime.Since(container.StatHistory[0].RecordedAt).Round(time.Second),\n\t)\n\n\treturn asciigraph.Plot(\n\t\tdata,\n\t\tasciigraph.Height(height),\n\t\tasciigraph.Width(width),\n\t\tasciigraph.Min(min),\n\t\tasciigraph.Max(max),\n\t\tasciigraph.Caption(caption),\n\t), nil\n}\n\n// from Dave C's answer at https://stackoverflow.com/questions/20767724/converting-unknown-interface-to-float64-in-golang\nfunc getFloat(unk interface{}) (float64, error) {\n\tfloatType := reflect.TypeOf(float64(0))\n\tstringType := reflect.TypeOf(\"\")\n\n\tswitch i := unk.(type) {\n\tcase float64:\n\t\treturn i, nil\n\tcase float32:\n\t\treturn float64(i), nil\n\tcase int64:\n\t\treturn float64(i), nil\n\tcase int32:\n\t\treturn float64(i), nil\n\tcase int:\n\t\treturn float64(i), nil\n\tcase uint64:\n\t\treturn float64(i), nil\n\tcase uint32:\n\t\treturn float64(i), nil\n\tcase uint:\n\t\treturn float64(i), nil\n\tcase string:\n\t\treturn strconv.ParseFloat(i, 64)\n\tdefault:\n\t\tv := reflect.ValueOf(unk)\n\t\tv = reflect.Indirect(v)\n\t\tif v.Type().ConvertibleTo(floatType) {\n\t\t\tfv := v.Convert(floatType)\n\t\t\treturn fv.Float(), nil\n\t\t} else if v.Type().ConvertibleTo(stringType) {\n\t\t\tsv := v.Convert(stringType)\n\t\t\ts := sv.String()\n\t\t\treturn strconv.ParseFloat(s, 64)\n\t\t} else {\n\t\t\treturn math.NaN(), fmt.Errorf(\"Can't convert %v to float64\", v.Type())\n\t\t}\n\t}\n}\n\n// Path: pkg/gui/presentation/containers.go\npackage presentation\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/fatih/color\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/samber/lo\"\n)\n\nfunc GetContainerDisplayStrings(guiConfig *config.GuiConfig, container *commands.Container) []string {\n\treturn []string{\n\t\tgetContainerDisplayStatus(guiConfig, container),\n\t\tgetContainerDisplaySubstatus(guiConfig, container),\n\t\tcontainer.Name,\n\t\tgetDisplayCPUPerc(container),\n\t\tutils.ColoredString(displayPorts(container), color.FgYellow),\n\t\tutils.ColoredString(displayContainerImage(container), color.FgMagenta),\n\t}\n}\n\nfunc displayContainerImage(container *commands.Container) string {\n\treturn strings.TrimPrefix(container.Container.Image, \"sha256:\")\n}\n\nfunc displayPorts(c *commands.Container) string {\n\tportStrings := lo.Map(c.Container.Ports, func(port dockerTypes.Port, _ int) string {\n\t\tif port.PublicPort == 0 {\n\t\t\treturn fmt.Sprintf(\"%d/%s\", port.PrivatePort, port.Type)\n\t\t}\n\n\t\t// docker ps will show '0.0.0.0:80->80/tcp' but we'll show\n\t\t// '80->80/tcp' instead to save space (unless the IP is something other than\n\t\t// 0.0.0.0)\n\t\tipString := \"\"\n\t\tif port.IP != \"0.0.0.0\" {\n\t\t\tipString = port.IP + \":\"\n\t\t}\n\t\treturn fmt.Sprintf(\"%s%d->%d/%s\", ipString, port.PublicPort, port.PrivatePort, port.Type)\n\t})\n\n\t// sorting because the order of the ports is not deterministic\n\t// and we don't want to have them constantly swapping\n\tsort.Strings(portStrings)\n\n\treturn strings.Join(portStrings, \", \")\n}\n\n// getContainerDisplayStatus returns the colored status of the container\nfunc getContainerDisplayStatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tshortStatusMap := map[string]string{\n\t\t\"paused\":     \"P\",\n\t\t\"exited\":     \"X\",\n\t\t\"created\":    \"C\",\n\t\t\"removing\":   \"RM\",\n\t\t\"restarting\": \"RS\",\n\t\t\"running\":    \"R\",\n\t\t\"dead\":       \"D\",\n\t}\n\n\ticonStatusMap := map[string]rune{\n\t\t\"paused\":     '\u25eb',\n\t\t\"exited\":     '\u2a2f',\n\t\t\"created\":    '+',\n\t\t\"removing\":   '\u2212',\n\t\t\"restarting\": '\u27f3',\n\t\t\"running\":    '\u25b6',\n\t\t\"dead\":       '!',\n\t}\n\n\tvar containerState string\n\tswitch guiConfig.ContainerStatusHealthStyle {\n\tcase \"short\":\n\t\tcontainerState = shortStatusMap[c.Container.State]\n\tcase \"icon\":\n\t\tcontainerState = string(iconStatusMap[c.Container.State])\n\tcase \"long\":\n\t\tfallthrough\n\tdefault:\n\t\tcontainerState = c.Container.State\n\t}\n\n\treturn utils.ColoredString(containerState, getContainerColor(c))\n}\n\n// GetDisplayStatus returns the exit code if the container has exited, and the health status if the container is running (and has a health check)\nfunc getContainerDisplaySubstatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tif !c.DetailsLoaded() {\n\t\treturn \"\"\n\t}\n\n\tswitch c.Container.State {\n\tcase \"exited\":\n\t\treturn utils.ColoredString(\n\t\t\tfmt.Sprintf(\"(%s)\", strconv.Itoa(c.Details.State.ExitCode)), getContainerColor(c),\n\t\t)\n\tcase \"running\":\n\t\treturn getHealthStatus(guiConfig, c)\n\tdefault:\n\t\treturn \"\"\n\t}\n}\n\nfunc getHealthStatus(guiConfig *config.GuiConfig, c *commands.Container) string {\n\tif !c.DetailsLoaded() {\n\t\treturn \"\"\n\t}\n\n\thealthStatusColorMap := map[string]color.Attribute{\n\t\t\"healthy\":   color.FgGreen,\n\t\t\"unhealthy\": color.FgRed,\n\t\t\"starting\":  color.FgYellow,\n\t}\n\n\tif c.Details.State.Health == nil {\n\t\treturn \"\"\n\t}\n\n\tshortHealthStatusMap := map[string]string{\n\t\t\"healthy\":   \"H\",\n\t\t\"unhealthy\": \"U\",\n\t\t\"starting\":  \"S\",\n\t}\n\n\ticonHealthStatusMap := map[string]rune{\n\t\t\"healthy\":   '\u2714',\n\t\t\"unhealthy\": '?',\n\t\t\"starting\":  '\u2026',\n\t}\n\n\tvar healthStatus string\n\tswitch guiConfig.ContainerStatusHealthStyle {\n\tcase \"short\":\n\t\thealthStatus = shortHealthStatusMap[c.Details.State.Health.Status]\n\tcase \"icon\":\n\t\thealthStatus = string(iconHealthStatusMap[c.Details.State.Health.Status])\n\tcase \"long\":\n\t\tfallthrough\n\tdefault:\n\t\thealthStatus = c.Details.State.Health.Status\n\t}\n\n\tif healthStatusColor, ok := healthStatusColorMap[c.Details.State.Health.Status]; ok {\n\t\treturn utils.ColoredString(fmt.Sprintf(\"(%s)\", healthStatus), healthStatusColor)\n\t}\n\treturn \"\"\n}\n\n// getDisplayCPUPerc colors the cpu percentage based on how extreme it is\nfunc getDisplayCPUPerc(c *commands.Container) string {\n\tstats, ok := c.GetLastStats()\n\tif !ok {\n\t\treturn \"\"\n\t}\n\n\tpercentage := stats.DerivedStats.CPUPercentage\n...\n", "needle_token_start": 12627, "needle_token_end": 13058, "code_context_ntokens": 15104, "output": ["```go\nfunc plotGraph(container *commands.Container, spec config.GraphConfig, width int) (string, error) {\n\tcontainer.StatsMutex.Lock()\n\tdefer container.StatsMutex.Unlock()\n\n\tdata := make([]float64, len(container.StatHistory))\n\n\tfor i, stats := range container.StatHistory {\n\t\tvalue, err := lookup.LookupString(stats, spec.StatPath)\n\t\tif err != nil {\n\t\t\treturn \"Could not find key: \" + spec.StatPath, nil\n\t\t}\n\t\tfloatValue, err := getFloat(value.Interface())\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\n\t\tdata[i] = floatValue\n\t}\n\n\tmax := spec.Max\n\tif spec.MaxType == \"\" {\n\t\tmax = lo.Max(data)\n\t}\n\n\tmin := spec.Min\n\tif spec.MinType == \"\" {\n\t\tmin = lo.Min(data)\n\t}\n\n\theight := 10\n\tif spec.Height > 0 {\n\t\theight = spec.Height\n\t}\n\n\tcaption := fmt.Sprintf(\n\t\t\"%s: %0.2f (%v)\",\n\t\tspec.Caption,\n\t\tdata[len(data)-1],\n\t\ttime.Since(container.StatHistory[0].RecordedAt).Round(time.Second),\n\t)\n\n\treturn asciigraph.Plot(\n\t\tdata,\n\t\tasciigraph.Height(height),\n\t\tasciigraph.Width(width),\n\t\tasciigraph.Min(min),\n\t\tasciigraph.Max(max),\n\t\tasciigraph.Caption(caption),\n\t), nil\n}\n```"]}
{"repo": "jesseduffield/lazydocker", "name": "getScreenMode", "language": "go", "path": "pkg/gui/gui.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to determine the display size setting for a user interface based on user preferences.\n2. **Input**: It takes a configuration object that includes user settings.\n3. **Output**: Returns an enumeration value that represents the screen size mode (normal, half, or full).\n4. **Procedure**: The function reads the screen mode setting from the user's configuration. It then maps this setting to a corresponding enumeration value that indicates how much of the screen a panel should occupy. If the setting is unrecognized, it defaults to the normal screen size.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: pkg/commands/errors.go\npackage commands\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/go-errors/errors\"\n\t\"golang.org/x/xerrors\"\n)\n\nconst (\n\t// MustStopContainer tells us that we must stop the container before removing it\n\tMustStopContainer = iota\n)\n\n// WrapError wraps an error for the sake of showing a stack trace at the top level\n// the go-errors package, for some reason, does not return nil when you try to wrap\n// a non-error, so we're just doing it here\nfunc WrapError(err error) error {\n\tif err == nil {\n\t\treturn err\n\t}\n\n\treturn errors.Wrap(err, 0)\n}\n\n// ComplexError an error which carries a code so that calling code has an easier job to do\n// adapted from https://medium.com/yakka/better-go-error-handling-with-xerrors-1987650e0c79\ntype ComplexError struct {\n\tMessage string\n\tCode    int\n\tframe   xerrors.Frame\n}\n\n// FormatError is a function\nfunc (ce ComplexError) FormatError(p xerrors.Printer) error {\n\tp.Printf(\"%d %s\", ce.Code, ce.Message)\n\tce.frame.Format(p)\n\treturn nil\n}\n\n// Format is a function\nfunc (ce ComplexError) Format(f fmt.State, c rune) {\n\txerrors.FormatError(ce, f, c)\n}\n\nfunc (ce ComplexError) Error() string {\n\treturn fmt.Sprint(ce)\n}\n\n// HasErrorCode is a function\nfunc HasErrorCode(err error, code int) bool {\n\tvar originalErr ComplexError\n\tif xerrors.As(err, &originalErr) {\n\t\treturn originalErr.Code == MustStopContainer\n\t}\n\treturn false\n}\n\n// Path: pkg/gui/gocui.go\npackage gui\n\nimport (\n\t\"github.com/gookit/color\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n)\n\nvar gocuiColorMap = map[string]gocui.Attribute{\n\t\"default\":   gocui.ColorDefault,\n\t\"black\":     gocui.ColorBlack,\n\t\"red\":       gocui.ColorRed,\n\t\"green\":     gocui.ColorGreen,\n\t\"yellow\":    gocui.ColorYellow,\n\t\"blue\":      gocui.ColorBlue,\n\t\"magenta\":   gocui.ColorMagenta,\n\t\"cyan\":      gocui.ColorCyan,\n\t\"white\":     gocui.ColorWhite,\n\t\"bold\":      gocui.AttrBold,\n\t\"reverse\":   gocui.AttrReverse,\n\t\"underline\": gocui.AttrUnderline,\n}\n\n// GetAttribute gets the gocui color attribute from the string\nfunc GetGocuiAttribute(key string) gocui.Attribute {\n\tif utils.IsValidHexValue(key) {\n\t\tvalues := color.HEX(key).Values()\n\t\treturn gocui.NewRGBColor(int32(values[0]), int32(values[1]), int32(values[2]))\n\t}\n\n\tvalue, present := gocuiColorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn gocui.ColorDefault\n}\n\n// GetGocuiStyle bitwise OR's a list of attributes obtained via the given keys\nfunc GetGocuiStyle(keys []string) gocui.Attribute {\n\tvar attribute gocui.Attribute\n\tfor _, key := range keys {\n\t\tattribute |= GetGocuiAttribute(key)\n\t}\n\treturn attribute\n}\n\n// Path: pkg/utils/utils.go\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"html/template\"\n\t\"io\"\n\t\"math\"\n\t\"regexp\"\n\t\"sort\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/gocui\"\n\t\"github.com/mattn/go-runewidth\"\n\n\t// \"github.com/jesseduffield/yaml\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/goccy/go-yaml\"\n\t\"github.com/goccy/go-yaml/lexer\"\n\t\"github.com/goccy/go-yaml/printer\"\n)\n\n// SplitLines takes a multiline string and splits it on newlines\n// currently we are also stripping \\r's which may have adverse effects for\n// windows users (but no issues have been raised yet)\nfunc SplitLines(multilineString string) []string {\n\tmultilineString = strings.Replace(multilineString, \"\\r\", \"\", -1)\n\tif multilineString == \"\" || multilineString == \"\\n\" {\n\t\treturn make([]string, 0)\n\t}\n\tlines := strings.Split(multilineString, \"\\n\")\n\tif lines[len(lines)-1] == \"\" {\n\t\treturn lines[:len(lines)-1]\n\t}\n\treturn lines\n}\n\n// WithPadding pads a string as much as you want\nfunc WithPadding(str string, padding int) string {\n\tuncoloredStr := Decolorise(str)\n\tif padding < runewidth.StringWidth(uncoloredStr) {\n\t\treturn str\n\t}\n\treturn str + strings.Repeat(\" \", padding-runewidth.StringWidth(uncoloredStr))\n}\n\n// ColoredString takes a string and a colour attribute and returns a colored\n// string with that attribute\nfunc ColoredString(str string, colorAttribute color.Attribute) string {\n\t// fatih/color does not have a color.Default attribute, so unless we fork that repo the only way for us to express that we don't want to color a string different to the terminal's default is to not call the function in the first place, but that's annoying when you want a streamlined code path. Because I'm too lazy to fork the repo right now, we'll just assume that by FgWhite you really mean Default, for the sake of supporting users with light themed terminals.\n\tif colorAttribute == color.FgWhite {\n\t\treturn str\n\t}\n\tcolour := color.New(colorAttribute)\n\treturn ColoredStringDirect(str, colour)\n}\n\n// ColoredYamlString takes an YAML formatted string and returns a colored string\n// with colors hardcoded as:\n// keys: cyan\n// Booleans: magenta\n// Numbers: yellow\n// Strings: green\nfunc ColoredYamlString(str string) string {\n\tformat := func(attr color.Attribute) string {\n\t\treturn fmt.Sprintf(\"%s[%dm\", \"\\x1b\", attr)\n\t}\n\ttokens := lexer.Tokenize(str)\n\tvar p printer.Printer\n\tp.Bool = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgMagenta),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.Number = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgYellow),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.MapKey = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgCyan),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.String = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgGreen),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\treturn p.PrintTokens(tokens)\n}\n\n// MultiColoredString takes a string and an array of colour attributes and returns a colored\n// string with those attributes\nfunc MultiColoredString(str string, colorAttribute ...color.Attribute) string {\n\tcolour := color.New(colorAttribute...)\n\treturn ColoredStringDirect(str, colour)\n}\n\n// ColoredStringDirect used for aggregating a few color attributes rather than\n// just sending a single one\nfunc ColoredStringDirect(str string, colour *color.Color) string {\n\treturn colour.SprintFunc()(fmt.Sprint(str))\n}\n\n// NormalizeLinefeeds - Removes all Windows and Mac style line feeds\nfunc NormalizeLinefeeds(str string) string {\n\tstr = strings.Replace(str, \"\\r\\n\", \"\\n\", -1)\n\tstr = strings.Replace(str, \"\\r\", \"\", -1)\n\treturn str\n}\n\n// Loader dumps a string to be displayed as a loader\nfunc Loader() string {\n\tcharacters := \"|/-\\\\\"\n\tnow := time.Now()\n\tnanos := now.UnixNano()\n\tindex := nanos / 50000000 % int64(len(characters))\n\treturn characters[index : index+1]\n}\n\n// ResolvePlaceholderString populates a template with values\nfunc ResolvePlaceholderString(str string, arguments map[string]string) string {\n\tfor key, value := range arguments {\n\t\tstr = strings.Replace(str, \"{{\"+key+\"}}\", value, -1)\n\t}\n\treturn str\n}\n\n// Max returns the maximum of two integers\nfunc Max(x, y int) int {\n\tif x > y {\n\t\treturn x\n\t}\n\treturn y\n}\n\n// RenderTable takes an array of string arrays and returns a table containing the values\nfunc RenderTable(rows [][]string) (string, error) {\n\tif len(rows) == 0 {\n\t\treturn \"\", nil\n\t}\n\tif !displayArraysAligned(rows) {\n\t\treturn \"\", errors.New(\"Each item must return the same number of strings to display\")\n\t}\n\n\tcolumnPadWidths := getPadWidths(rows)\n\tpaddedDisplayRows := getPaddedDisplayStrings(rows, columnPadWidths)\n\n\treturn strings.Join(paddedDisplayRows, \"\\n\"), nil\n}\n\n// Decolorise strips a string of color\nfunc Decolorise(str string) string {\n\tre := regexp.MustCompile(`\\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mK]`)\n\treturn re.ReplaceAllString(str, \"\")\n}\n\nfunc getPadWidths(rows [][]string) []int {\n\tif len(rows[0]) <= 1 {\n\t\treturn []int{}\n\t}\n\tcolumnPadWidths := make([]int, len(rows[0])-1)\n\tfor i := range columnPadWidths {\n\t\tfor _, cells := range rows {\n\t\t\tuncoloredCell := Decolorise(cells[i])\n\n\t\t\tif runewidth.StringWidth(uncoloredCell) > columnPadWidths[i] {\n\t\t\t\tcolumnPadWidths[i] = runewidth.StringWidth(uncoloredCell)\n\t\t\t}\n\t\t}\n\t}\n\treturn columnPadWidths\n}\n\nfunc getPaddedDisplayStrings(rows [][]string, columnPadWidths []int) []string {\n\tpaddedDisplayRows := make([]string, len(rows))\n\tfor i, cells := range rows {\n\t\tfor j, columnPadWidth := range columnPadWidths {\n\t\t\tpaddedDisplayRows[i] += WithPadding(cells[j], columnPadWidth) + \" \"\n\t\t}\n\t\tpaddedDisplayRows[i] += cells[len(columnPadWidths)]\n\t}\n\treturn paddedDisplayRows\n}\n\n// displayArraysAligned returns true if every string array returned from our\n// list of displayables has the same length\nfunc displayArraysAligned(stringArrays [][]string) bool {\n\tfor _, strings := range stringArrays {\n\t\tif len(strings) != len(stringArrays[0]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc FormatBinaryBytes(b int) string {\n\tn := float64(b)\n\tunits := []string{\"B\", \"kiB\", \"MiB\", \"GiB\", \"TiB\", \"PiB\", \"EiB\", \"ZiB\", \"YiB\"}\n\tfor _, unit := range units {\n\t\tif n > math.Pow(2, 10) {\n\t\t\tn /= math.Pow(2, 10)\n\t\t} else {\n\t\t\tval := fmt.Sprintf(\"%.2f%s\", n, unit)\n\t\t\tif val == \"0.00B\" {\n\t\t\t\treturn \"0B\"\n\t\t\t}\n\t\t\treturn val\n\t\t}\n\t}\n\treturn \"a lot\"\n}\n\nfunc FormatDecimalBytes(b int) string {\n\tn := float64(b)\n\tunits := []string{\"B\", \"kB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\", \"ZB\", \"YB\"}\n\tfor _, unit := range units {\n\t\tif n > math.Pow(10, 3) {\n\t\t\tn /= math.Pow(10, 3)\n\t\t} else {\n\t\t\tval := fmt.Sprintf(\"%.2f%s\", n, unit)\n\t\t\tif val == \"0.00B\" {\n\t\t\t\treturn \"0B\"\n\t\t\t}\n\t\t\treturn val\n\t\t}\n\t}\n\treturn \"a lot\"\n}\n\nfunc ApplyTemplate(str string, object interface{}) string {\n\tvar buf bytes.Buffer\n\t_ = template.Must(template.New(\"\").Parse(str)).Execute(&buf, object)\n\treturn buf.String()\n}\n\n// GetGocuiAttribute gets the gocui color attribute from the string\nfunc GetGocuiAttribute(key string) gocui.Attribute {\n\tcolorMap := map[string]gocui.Attribute{\n\t\t\"default\":   gocui.ColorDefault,\n\t\t\"black\":     gocui.ColorBlack,\n\t\t\"red\":       gocui.ColorRed,\n\t\t\"green\":     gocui.ColorGreen,\n\t\t\"yellow\":    gocui.ColorYellow,\n\t\t\"blue\":      gocui.ColorBlue,\n\t\t\"magenta\":   gocui.ColorMagenta,\n\t\t\"cyan\":      gocui.ColorCyan,\n\t\t\"white\":     gocui.ColorWhite,\n\t\t\"bold\":      gocui.AttrBold,\n\t\t\"reverse\":   gocui.AttrReverse,\n\t\t\"underline\": gocui.AttrUnderline,\n\t}\n\tvalue, present := colorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn gocui.ColorDefault\n}\n\n// GetColorAttribute gets the color attribute from the string\nfunc GetColorAttribute(key string) color.Attribute {\n\tcolorMap := map[string]color.Attribute{\n\t\t\"default\":   color.FgWhite,\n\t\t\"black\":     color.FgBlack,\n\t\t\"red\":       color.FgRed,\n\t\t\"green\":     color.FgGreen,\n\t\t\"yellow\":    color.FgYellow,\n\t\t\"blue\":      color.FgBlue,\n\t\t\"magenta\":   color.FgMagenta,\n\t\t\"cyan\":      color.FgCyan,\n\t\t\"white\":     color.FgWhite,\n\t\t\"bold\":      color.Bold,\n\t\t\"underline\": color.Underline,\n\t}\n\tvalue, present := colorMap[key]\n\tif present {\n\t\treturn value\n\t}\n\treturn color.FgWhite\n}\n\n// WithShortSha returns a command but with a shorter SHA. in the terminal we're all used to 10 character SHAs but under the hood they're actually 64 characters long. No need including all the characters when we're just displaying a command\nfunc WithShortSha(str string) string {\n\tsplit := strings.Split(str, \" \")\n\tfor i, word := range split {\n\t\t// good enough proxy for now\n\t\tif len(word) == 64 {\n\t\t\tsplit[i] = word[0:10]\n\t\t}\n\t}\n\treturn strings.Join(split, \" \")\n}\n\n// FormatMapItem is for displaying items in a map\nfunc FormatMapItem(padding int, k string, v interface{}) string {\n\treturn fmt.Sprintf(\"%s%s %v\\n\", strings.Repeat(\" \", padding), ColoredString(k+\":\", color.FgYellow), fmt.Sprintf(\"%v\", v))\n}\n\n// FormatMap is for displaying a map\nfunc FormatMap(padding int, m map[string]string) string {\n\tif len(m) == 0 {\n\t\treturn \"none\\n\"\n\t}\n\n\toutput := \"\\n\"\n\n\tkeys := make([]string, 0, len(m))\n\tfor key := range m {\n\t\tkeys = append(keys, key)\n\t}\n\tsort.Strings(keys)\n\tfor _, key := range keys {\n\t\toutput += FormatMapItem(padding, key, m[key])\n\t}\n\n\treturn output\n}\n\ntype multiErr []error\n\nfunc (m multiErr) Error() string {\n\tvar b bytes.Buffer\n\tb.WriteString(\"encountered multiple errors:\")\n\tfor _, err := range m {\n\t\tb.WriteString(\"\\n\\t... \" + err.Error())\n\t}\n\treturn b.String()\n}\n\nfunc CloseMany(closers []io.Closer) error {\n\terrs := make([]error, 0, len(closers))\n\tfor _, c := range closers {\n\t\terr := c.Close()\n\t\tif err != nil {\n\t\t\terrs = append(errs, err)\n\t\t}\n\t}\n\tif len(errs) > 0 {\n\t\treturn multiErr(errs)\n\t}\n\treturn nil\n}\n\nfunc SafeTruncate(str string, limit int) string {\n\tif len(str) > limit {\n\t\treturn str[0:limit]\n\t} else {\n\t\treturn str\n\t}\n}\n\nfunc IsValidHexValue(v string) bool {\n\tif len(v) != 4 && len(v) != 7 {\n\t\treturn false\n\t}\n\n\tif v[0] != '#' {\n\t\treturn false\n\t}\n\n\tfor _, char := range v[1:] {\n\t\tswitch char {\n\t\tcase '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F':\n\t\t\tcontinue\n\t\tdefault:\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// Style used on menu items that open another menu\nfunc OpensMenuStyle(str string) string {\n\treturn ColoredString(fmt.Sprintf(\"%s...\", str), color.FgMagenta)\n}\n\n// MarshalIntoYaml gets any json-tagged data and marshal it into yaml saving original json structure.\n// Useful for structs from 3rd-party libs without yaml tags.\nfunc MarshalIntoYaml(data interface{}) ([]byte, error) {\n\treturn marshalIntoFormat(data, \"yaml\")\n}\n\nfunc marshalIntoFormat(data interface{}, format string) ([]byte, error) {\n\t// First marshal struct->json to get the resulting structure declared by json tags\n\tdataJSON, err := json.MarshalIndent(data, \"\", \"  \")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tswitch format {\n\tcase \"json\":\n\t\treturn dataJSON, err\n\tcase \"yaml\":\n\t\t// Use Unmarshal->Marshal hack to convert json into yaml with the original structure preserved\n\t\tvar dataMirror yaml.MapSlice\n\t\tif err := yaml.Unmarshal(dataJSON, &dataMirror); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn yaml.Marshal(dataMirror)\n\tdefault:\n\t\treturn nil, errors.New(fmt.Sprintf(\"Unsupported detailization format: %s\", format))\n\t}\n}\n\n// Path: pkg/commands/os.go\npackage commands\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/go-errors/errors\"\n\n\t\"github.com/jesseduffield/kill\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/mgutz/str\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Platform stores the os state\ntype Platform struct {\n\tos              string\n\tshell           string\n\tshellArg        string\n\topenCommand     string\n\topenLinkCommand string\n}\n\n// OSCommand holds all the os commands\ntype OSCommand struct {\n\tLog      *logrus.Entry\n\tPlatform *Platform\n\tConfig   *config.AppConfig\n\tcommand  func(string, ...string) *exec.Cmd\n\tgetenv   func(string) string\n}\n\n// NewOSCommand os command runner\nfunc NewOSCommand(log *logrus.Entry, config *config.AppConfig) *OSCommand {\n\treturn &OSCommand{\n\t\tLog:      log,\n\t\tPlatform: getPlatform(),\n\t\tConfig:   config,\n\t\tcommand:  exec.Command,\n\t\tgetenv:   os.Getenv,\n\t}\n}\n\n// SetCommand sets the command function used by the struct.\n// To be used for testing only\nfunc (c *OSCommand) SetCommand(cmd func(string, ...string) *exec.Cmd) {\n\tc.command = cmd\n}\n\n// RunCommandWithOutput wrapper around commands returning their output and error\nfunc (c *OSCommand) RunCommandWithOutput(command string) (string, error) {\n\tcmd := c.ExecutableFromString(command)\n\tbefore := time.Now()\n\toutput, err := sanitisedCommandOutput(cmd.Output())\n\tc.Log.Warn(fmt.Sprintf(\"'%s': %s\", command, time.Since(before)))\n\treturn output, err\n}\n\n// RunCommandWithOutput wrapper around commands returning their output and error\nfunc (c *OSCommand) RunCommandWithOutputContext(ctx context.Context, command string) (string, error) {\n\tcmd := c.ExecutableFromStringContext(ctx, command)\n\tbefore := time.Now()\n\toutput, err := sanitisedCommandOutput(cmd.Output())\n\tc.Log.Warn(fmt.Sprintf(\"'%s': %s\", command, time.Since(before)))\n\treturn output, err\n}\n\n// RunExecutableWithOutput runs an executable file and returns its output\nfunc (c *OSCommand) RunExecutableWithOutput(cmd *exec.Cmd) (string, error) {\n\treturn sanitisedCommandOutput(cmd.CombinedOutput())\n}\n\n// RunExecutable runs an executable file and returns an error if there was one\nfunc (c *OSCommand) RunExecutable(cmd *exec.Cmd) error {\n\t_, err := c.RunExecutableWithOutput(cmd)\n\treturn err\n}\n\n// ExecutableFromString takes a string like `docker ps -a` and returns an executable command for it\nfunc (c *OSCommand) ExecutableFromString(commandStr string) *exec.Cmd {\n\tsplitCmd := str.ToArgv(commandStr)\n\treturn c.NewCmd(splitCmd[0], splitCmd[1:]...)\n}\n\n// Same as ExecutableFromString but cancellable via a context\nfunc (c *OSCommand) ExecutableFromStringContext(ctx context.Context, commandStr string) *exec.Cmd {\n\tsplitCmd := str.ToArgv(commandStr)\n\treturn exec.CommandContext(ctx, splitCmd[0], splitCmd[1:]...)\n}\n\nfunc (c *OSCommand) NewCmd(cmdName string, commandArgs ...string) *exec.Cmd {\n\tcmd := c.command(cmdName, commandArgs...)\n\tcmd.Env = os.Environ()\n\treturn cmd\n}\n\nfunc (c *OSCommand) NewCommandStringWithShell(commandStr string) string {\n\tvar quotedCommand string\n\t// Windows does not seem to like quotes around the command\n\tif c.Platform.os == \"windows\" {\n\t\tquotedCommand = strings.NewReplacer(\n\t\t\t\"^\", \"^^\",\n\t\t\t\"&\", \"^&\",\n\t\t\t\"|\", \"^|\",\n\t\t\t\"<\", \"^<\",\n\t\t\t\">\", \"^>\",\n\t\t\t\"%\", \"^%\",\n\t\t).Replace(commandStr)\n\t} else {\n\t\tquotedCommand = c.Quote(commandStr)\n\t}\n\n\treturn fmt.Sprintf(\"%s %s %s\", c.Platform.shell, c.Platform.shellArg, quotedCommand)\n}\n\n// RunCommand runs a command and just returns the error\nfunc (c *OSCommand) RunCommand(command string) error {\n\t_, err := c.RunCommandWithOutput(command)\n\treturn err\n}\n\n// FileType tells us if the file is a file, directory or other\nfunc (c *OSCommand) FileType(path string) string {\n\tfileInfo, err := os.Stat(path)\n\tif err != nil {\n\t\treturn \"other\"\n\t}\n\tif fileInfo.IsDir() {\n\t\treturn \"directory\"\n\t}\n\treturn \"file\"\n}\n\nfunc sanitisedCommandOutput(output []byte, err error) (string, error) {\n\toutputString := string(output)\n\tif err != nil {\n\t\t// errors like 'exit status 1' are not very useful so we'll create an error\n\t\t// from stderr if we got an ExitError\n\t\texitError, ok := err.(*exec.ExitError)\n\t\tif ok {\n\t\t\treturn outputString, errors.New(string(exitError.Stderr))\n\t\t}\n\t\treturn \"\", WrapError(err)\n\t}\n\treturn outputString, nil\n}\n\n// OpenFile opens a file with the given\nfunc (c *OSCommand) OpenFile(filename string) error {\n\tcommandTemplate := c.Config.UserConfig.OS.OpenCommand\n\ttemplateValues := map[string]string{\n\t\t\"filename\": c.Quote(filename),\n\t}\n\n\tcommand := utils.ResolvePlaceholderString(commandTemplate, templateValues)\n\terr := c.RunCommand(command)\n\treturn err\n}\n\n// OpenLink opens a file with the given\nfunc (c *OSCommand) OpenLink(link string) error {\n\tcommandTemplate := c.Config.UserConfig.OS.OpenLinkCommand\n\ttemplateValues := map[string]string{\n\t\t\"link\": c.Quote(link),\n\t}\n\n\tcommand := utils.ResolvePlaceholderString(commandTemplate, templateValues)\n\terr := c.RunCommand(command)\n\treturn err\n}\n\n// EditFile opens a file in a subprocess using whatever editor is available,\n// falling back to core.editor, VISUAL, EDITOR, then vi\nfunc (c *OSCommand) EditFile(filename string) (*exec.Cmd, error) {\n\teditor := c.getenv(\"VISUAL\")\n\tif editor == \"\" {\n\t\teditor = c.getenv(\"EDITOR\")\n\t}\n\tif editor == \"\" {\n\t\tif err := c.RunCommand(\"which vi\"); err == nil {\n\t\t\teditor = \"vi\"\n\t\t}\n\t}\n\tif editor == \"\" {\n\t\treturn nil, errors.New(\"No editor defined in $VISUAL or $EDITOR\")\n\t}\n\n\treturn c.NewCmd(editor, filename), nil\n}\n\n// Quote wraps a message in platform-specific quotation marks\nfunc (c *OSCommand) Quote(message string) string {\n\tvar quote string\n\tif c.Platform.os == \"windows\" {\n\t\tquote = `\\\"`\n\t\tmessage = strings.NewReplacer(\n\t\t\t`\"`, `\"'\"'\"`,\n\t\t\t`\\\"`, `\\\\\"`,\n\t\t).Replace(message)\n\t} else {\n\t\tquote = `\"`\n\t\tmessage = strings.NewReplacer(\n\t\t\t`\\`, `\\\\`,\n\t\t\t`\"`, `\\\"`,\n\t\t\t`$`, `\\$`,\n\t\t\t\"`\", \"\\\\`\",\n\t\t).Replace(message)\n\t}\n\treturn quote + message + quote\n}\n\n// Unquote removes wrapping quotations marks if they are present\n// this is needed for removing quotes from staged filenames with spaces\nfunc (c *OSCommand) Unquote(message string) string {\n\treturn strings.Replace(message, `\"`, \"\", -1)\n}\n\n// AppendLineToFile adds a new line in file\nfunc (c *OSCommand) AppendLineToFile(filename, line string) error {\n\tf, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600)\n\tif err != nil {\n\t\treturn WrapError(err)\n\t}\n\tdefer f.Close()\n\n\t_, err = f.WriteString(\"\\n\" + line)\n\tif err != nil {\n\t\treturn WrapError(err)\n\t}\n\treturn nil\n}\n\n// CreateTempFile writes a string to a new temp file and returns the file's name\nfunc (c *OSCommand) CreateTempFile(filename, content string) (string, error) {\n\ttmpfile, err := os.CreateTemp(\"\", filename)\n\tif err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\n\tif _, err := tmpfile.WriteString(content); err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\tif err := tmpfile.Close(); err != nil {\n\t\tc.Log.Error(err)\n\t\treturn \"\", WrapError(err)\n\t}\n\n\treturn tmpfile.Name(), nil\n}\n\n// Remove removes a file or directory at the specified path\nfunc (c *OSCommand) Remove(filename string) error {\n\terr := os.RemoveAll(filename)\n\treturn WrapError(err)\n}\n\n// FileExists checks whether a file exists at the specified path\nfunc (c *OSCommand) FileExists(path string) (bool, error) {\n\tif _, err := os.Stat(path); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false, nil\n\t\t}\n\t\treturn false, err\n\t}\n\treturn true, nil\n}\n\n// RunPreparedCommand takes a pointer to an exec.Cmd and runs it\n// this is useful if you need to give your command some environment variables\n// before running it\nfunc (c *OSCommand) RunPreparedCommand(cmd *exec.Cmd) error {\n\tout, err := cmd.CombinedOutput()\n\toutString := string(out)\n\tc.Log.Info(outString)\n\tif err != nil {\n\t\tif len(outString) == 0 {\n\t\t\treturn err\n\t\t}\n\t\treturn errors.New(outString)\n\t}\n\treturn nil\n}\n\n// GetLazydockerPath returns the path of the currently executed file\nfunc (c *OSCommand) GetLazydockerPath() string {\n\tex, err := os.Executable() // get the executable path for docker to use\n\tif err != nil {\n\t\tex = os.Args[0] // fallback to the first call argument if needed\n\t}\n\treturn filepath.ToSlash(ex)\n}\n\n// RunCustomCommand returns the pointer to a custom command\nfunc (c *OSCommand) RunCustomCommand(command string) *exec.Cmd {\n\treturn c.NewCmd(c.Platform.shell, c.Platform.shellArg, command)\n}\n\n// PipeCommands runs a heap of commands and pipes their inputs/outputs together like A | B | C\nfunc (c *OSCommand) PipeCommands(commandStrings ...string) error {\n\tcmds := make([]*exec.Cmd, len(commandStrings))\n\n\tfor i, str := range commandStrings {\n\t\tcmds[i] = c.ExecutableFromString(str)\n\t}\n\n\tfor i := 0; i < len(cmds)-1; i++ {\n\t\tstdout, err := cmds[i].StdoutPipe()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tcmds[i+1].Stdin = stdout\n\t}\n\n\t// keeping this here in case I adapt this code for some other purpose in the future\n\t// cmds[len(cmds)-1].Stdout = os.Stdout\n\n\tfinalErrors := []string{}\n\n\twg := sync.WaitGroup{}\n\twg.Add(len(cmds))\n\n\tfor _, cmd := range cmds {\n\t\tcurrentCmd := cmd\n\t\tgo func() {\n\t\t\tstderr, err := currentCmd.StderrPipe()\n\t\t\tif err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\tif err := currentCmd.Start(); err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\tif b, err := io.ReadAll(stderr); err == nil {\n\t\t\t\tif len(b) > 0 {\n\t\t\t\t\tfinalErrors = append(finalErrors, string(b))\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif err := currentCmd.Wait(); err != nil {\n\t\t\t\tc.Log.Error(err)\n\t\t\t}\n\n\t\t\twg.Done()\n\t\t}()\n\t}\n\n\twg.Wait()\n\n\tif len(finalErrors) > 0 {\n\t\treturn errors.New(strings.Join(finalErrors, \"\\n\"))\n\t}\n\treturn nil\n}\n\n// Kill kills a process. If the process has Setpgid == true, then we have anticipated that it might spawn its own child processes, so we've given it a process group ID (PGID) equal to its process id (PID) and given its child processes will inherit the PGID, we can kill that group, rather than killing the process itself.\nfunc (c *OSCommand) Kill(cmd *exec.Cmd) error {\n\treturn kill.Kill(cmd)\n}\n\n// PrepareForChildren sets Setpgid to true on the cmd, so that when we run it as a subprocess, we can kill its group rather than the process itself. This is because some commands, like `docker-compose logs` spawn multiple children processes, and killing the parent process isn't sufficient for killing those child processes. We set the group id here, and then in subprocess.go we check if the group id is set and if so, we kill the whole group rather than just the one process.\nfunc (c *OSCommand) PrepareForChildren(cmd *exec.Cmd) {\n\tkill.PrepareForChildren(cmd)\n}\n\n// Path: pkg/log/log.go\npackage log\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// NewLogger returns a new logger\nfunc NewLogger(config *config.AppConfig, rollrusHook string) *logrus.Entry {\n\tvar log *logrus.Logger\n\tif config.Debug || os.Getenv(\"DEBUG\") == \"TRUE\" {\n\t\tlog = newDevelopmentLogger(config)\n\t} else {\n\t\tlog = newProductionLogger()\n\t}\n\n\t// highly recommended: tail -f development.log | humanlog\n\t// https://github.com/aybabtme/humanlog\n\tlog.Formatter = &logrus.JSONFormatter{}\n\n\treturn log.WithFields(logrus.Fields{\n\t\t\"debug\":     config.Debug,\n\t\t\"version\":   config.Version,\n\t\t\"commit\":    config.Commit,\n\t\t\"buildDate\": config.BuildDate,\n\t})\n}\n\nfunc getLogLevel() logrus.Level {\n\tstrLevel := os.Getenv(\"LOG_LEVEL\")\n\tlevel, err := logrus.ParseLevel(strLevel)\n\tif err != nil {\n\t\treturn logrus.DebugLevel\n\t}\n\treturn level\n}\n\nfunc newDevelopmentLogger(config *config.AppConfig) *logrus.Logger {\n\tlog := logrus.New()\n\tlog.SetLevel(getLogLevel())\n\tfile, err := os.OpenFile(filepath.Join(config.ConfigDir, \"development.log\"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o666)\n\tif err != nil {\n\t\tfmt.Println(\"unable to log to file\")\n\t\tos.Exit(1)\n\t}\n\tlog.SetOutput(file)\n\treturn log\n}\n\nfunc newProductionLogger() *logrus.Logger {\n\tlog := logrus.New()\n\tlog.Out = io.Discard\n\tlog.SetLevel(logrus.ErrorLevel)\n\treturn log\n}\n\n// Path: pkg/i18n/i18n.go\npackage i18n\n\nimport (\n\t\"strings\"\n\n\t\"github.com/imdario/mergo\"\n\n\t\"github.com/cloudfoundry/jibber_jabber\"\n\t\"github.com/go-errors/errors\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Localizer will translate a message into the user's language\ntype Localizer struct {\n\tLog *logrus.Entry\n\tS   TranslationSet\n}\n\nfunc NewTranslationSetFromConfig(log *logrus.Entry, configLanguage string) (*TranslationSet, error) {\n\tif configLanguage == \"auto\" {\n\t\tlanguage := detectLanguage(jibber_jabber.DetectLanguage)\n\t\treturn NewTranslationSet(log, language), nil\n\t}\n\n\tfor key := range GetTranslationSets() {\n\t\tif key == configLanguage {\n\t\t\treturn NewTranslationSet(log, configLanguage), nil\n\t\t}\n\t}\n\n\treturn NewTranslationSet(log, \"en\"), errors.New(\"Language not found: \" + configLanguage)\n}\n\nfunc NewTranslationSet(log *logrus.Entry, language string) *TranslationSet {\n\tlog.Info(\"language: \" + language)\n\n\tbaseSet := englishSet()\n\n\tfor languageCode, translationSet := range GetTranslationSets() {\n\t\tif strings.HasPrefix(language, languageCode) {\n\t\t\t_ = mergo.Merge(&baseSet, translationSet, mergo.WithOverride)\n\t\t}\n\t}\n\n\treturn &baseSet\n}\n\n// GetTranslationSets gets all the translation sets, keyed by language code\nfunc GetTranslationSets() map[string]TranslationSet {\n\treturn map[string]TranslationSet{\n\t\t\"pl\": polishSet(),\n\t\t\"nl\": dutchSet(),\n\t\t\"de\": germanSet(),\n\t\t\"tr\": turkishSet(),\n\t\t\"en\": englishSet(),\n\t\t\"fr\": frenchSet(),\n\t\t\"zh\": chineseSet(),\n\t\t\"es\": spanishSet(),\n\t\t\"pt\": portugueseSet(),\n\t}\n}\n\n// detectLanguage extracts user language from environment\nfunc detectLanguage(langDetector func() (string, error)) string {\n\tif userLang, err := langDetector(); err == nil {\n\t\treturn userLang\n\t}\n\n\treturn \"C\"\n}\n\n// Path: pkg/tasks/tasks.go\npackage tasks\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\ntype TaskManager struct {\n\tcurrentTask  *Task\n\twaitingMutex deadlock.Mutex\n\ttaskIDMutex  deadlock.Mutex\n\tLog          *logrus.Entry\n\tTr           *i18n.TranslationSet\n\tnewTaskId    int\n}\n\ntype Task struct {\n\tctx           context.Context\n\tcancel        context.CancelFunc\n\tstopped       bool\n\tstopMutex     deadlock.Mutex\n\tnotifyStopped chan struct{}\n\tLog           *logrus.Entry\n\tf             func(ctx context.Context)\n}\n\ntype TaskFunc func(ctx context.Context)\n\nfunc NewTaskManager(log *logrus.Entry, translationSet *i18n.TranslationSet) *TaskManager {\n\treturn &TaskManager{Log: log, Tr: translationSet}\n}\n\n// Close closes the task manager, killing whatever task may currently be running\nfunc (t *TaskManager) Close() {\n\tif t.currentTask == nil {\n\t\treturn\n\t}\n\n\tc := make(chan struct{}, 1)\n\n\tgo func() {\n\t\tt.currentTask.Stop()\n\t\tc <- struct{}{}\n\t}()\n\n\tselect {\n\tcase <-c:\n\t\treturn\n\tcase <-time.After(3 * time.Second):\n\t\tfmt.Println(t.Tr.CannotKillChildError)\n\t}\n}\n\nfunc (t *TaskManager) NewTask(f func(ctx context.Context)) error {\n\tgo func() {\n\t\tt.taskIDMutex.Lock()\n\t\tt.newTaskId++\n\t\ttaskID := t.newTaskId\n\t\tt.taskIDMutex.Unlock()\n\n\t\tt.waitingMutex.Lock()\n\t\tdefer t.waitingMutex.Unlock()\n\t\tif taskID < t.newTaskId {\n\t\t\treturn\n\t\t}\n\n\t\tctx, cancel := context.WithCancel(context.Background())\n\t\tnotifyStopped := make(chan struct{})\n\n\t\tif t.currentTask != nil {\n\t\t\tt.Log.Info(\"asking task to stop\")\n\t\t\tt.currentTask.Stop()\n\t\t\tt.Log.Info(\"task stopped\")\n\t\t}\n\n\t\tt.currentTask = &Task{\n\t\t\tctx:           ctx,\n\t\t\tcancel:        cancel,\n\t\t\tnotifyStopped: notifyStopped,\n\t\t\tLog:           t.Log,\n\t\t\tf:             f,\n\t\t}\n\n\t\tgo func() {\n\t\t\tf(ctx)\n\t\t\tt.Log.Info(\"returned from function, closing notifyStopped\")\n\t\t\tclose(notifyStopped)\n\t\t}()\n\t}()\n\n\treturn nil\n}\n\nfunc (t *Task) Stop() {\n\tt.stopMutex.Lock()\n\tdefer t.stopMutex.Unlock()\n\tif t.stopped {\n\t\treturn\n\t}\n\n\tt.cancel()\n\tt.Log.Info(\"closed stop channel, waiting for notifyStopped message\")\n\t<-t.notifyStopped\n\tt.Log.Info(\"received notifystopped message\")\n\tt.stopped = true\n}\n\n// NewTickerTask is a convenience function for making a new task that repeats some action once per e.g. second\n// the before function gets called after the lock is obtained, but before the ticker starts.\n// if you handle a message on the stop channel in f() you need to send a message on the notifyStopped channel because returning is not sufficient. Here, unlike in a regular task, simply returning means we're now going to wait till the next tick to run again.\nfunc (t *TaskManager) NewTickerTask(duration time.Duration, before func(ctx context.Context), f func(ctx context.Context, notifyStopped chan struct{})) error {\n\tnotifyStopped := make(chan struct{}, 10)\n\n\treturn t.NewTask(func(ctx context.Context) {\n\t\tif before != nil {\n\t\t\tbefore(ctx)\n\t\t}\n\t\ttickChan := time.NewTicker(duration)\n\t\tdefer tickChan.Stop()\n\t\t// calling f first so that we're not waiting for the first tick\n\t\tf(ctx, notifyStopped)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-notifyStopped:\n\t\t\t\tt.Log.Info(\"exiting ticker task due to notifyStopped channel\")\n\t\t\t\treturn\n\t\t\tcase <-ctx.Done():\n\t\t\t\tt.Log.Info(\"exiting ticker task due to stopped cahnnel\")\n\t\t\t\treturn\n\t\t\tcase <-tickChan.C:\n\t\t\t\tt.Log.Info(\"running ticker task again\")\n\t\t\t\tf(ctx, notifyStopped)\n\t\t\t}\n\t\t}\n\t})\n}\n\n// Path: pkg/gui/types/types.go\npackage types\n\ntype MenuItem struct {\n\tLabel string\n\n\t// alternative to Label. Allows specifying columns which will be auto-aligned\n\tLabelColumns []string\n\n\tOnPress func() error\n\n\t// Only applies when Label is used\n\tOpensMenu bool\n}\n\n// Path: pkg/commands/container.go\npackage commands\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os/exec\"\n\t\"strings\"\n\n\t\"github.com/docker/docker/api/types/container\"\n\t\"github.com/sasha-s/go-deadlock\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\t\"github.com/docker/docker/api/types/filters\"\n\t\"github.com/docker/docker/client\"\n\t\"github.com/go-errors/errors\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/utils\"\n\t\"github.com/sirupsen/logrus\"\n\t\"golang.org/x/xerrors\"\n)\n\n// Container : A docker Container\ntype Container struct {\n\tName            string\n\tServiceName     string\n\tContainerNumber string // might make this an int in the future if need be\n\n\t// OneOff tells us if the container is just a job container or is actually bound to the service\n\tOneOff          bool\n\tProjectName     string\n\tID              string\n\tContainer       dockerTypes.Container\n\tClient          *client.Client\n\tOSCommand       *OSCommand\n\tLog             *logrus.Entry\n\tStatHistory     []*RecordedStats\n\tDetails         dockerTypes.ContainerJSON\n\tMonitoringStats bool\n\tDockerCommand   LimitedDockerCommand\n\tTr              *i18n.TranslationSet\n\n\tStatsMutex deadlock.Mutex\n}\n\n// Remove removes the container\nfunc (c *Container) Remove(options dockerTypes.ContainerRemoveOptions) error {\n\tc.Log.Warn(fmt.Sprintf(\"removing container %s\", c.Name))\n\tif err := c.Client.ContainerRemove(context.Background(), c.ID, options); err != nil {\n\t\tif strings.Contains(err.Error(), \"Stop the container before attempting removal or force remove\") {\n\t\t\treturn ComplexError{\n\t\t\t\tCode:    MustStopContainer,\n\t\t\t\tMessage: err.Error(),\n\t\t\t\tframe:   xerrors.Caller(1),\n\t\t\t}\n\t\t}\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\n// Stop stops the container\nfunc (c *Container) Stop() error {\n\tc.Log.Warn(fmt.Sprintf(\"stopping container %s\", c.Name))\n\treturn c.Client.ContainerStop(context.Background(), c.ID, nil)\n}\n\n// Pause pauses the container\nfunc (c *Container) Pause() error {\n\tc.Log.Warn(fmt.Sprintf(\"pausing container %s\", c.Name))\n\treturn c.Client.ContainerPause(context.Background(), c.ID)\n}\n\n// Unpause unpauses the container\nfunc (c *Container) Unpause() error {\n\tc.Log.Warn(fmt.Sprintf(\"unpausing container %s\", c.Name))\n\treturn c.Client.ContainerUnpause(context.Background(), c.ID)\n}\n\n// Restart restarts the container\nfunc (c *Container) Restart() error {\n\tc.Log.Warn(fmt.Sprintf(\"restarting container %s\", c.Name))\n\treturn c.Client.ContainerRestart(context.Background(), c.ID, nil)\n}\n\n// Attach attaches the container\nfunc (c *Container) Attach() (*exec.Cmd, error) {\n\tif !c.DetailsLoaded() {\n\t\treturn nil, errors.New(c.Tr.WaitingForContainerInfo)\n\t}\n\n\t// verify that we can in fact attach to this container\n\tif !c.Details.Config.OpenStdin {\n\t\treturn nil, errors.New(c.Tr.UnattachableContainerError)\n\t}\n\n\tif c.Container.State == \"exited\" {\n\t\treturn nil, errors.New(c.Tr.CannotAttachStoppedContainerError)\n\t}\n\n\tc.Log.Warn(fmt.Sprintf(\"attaching to container %s\", c.Name))\n\t// TODO: use SDK\n\tcmd := c.OSCommand.NewCmd(\"docker\", \"attach\", \"--sig-proxy=false\", c.ID)\n\treturn cmd, nil\n}\n\n// Top returns process information\nfunc (c *Container) Top(ctx context.Context) (container.ContainerTopOKBody, error) {\n\tdetail, err := c.Inspect()\n\tif err != nil {\n\t\treturn container.ContainerTopOKBody{}, err\n\t}\n\n\t// check container status\n\tif !detail.State.Running {\n\t\treturn container.ContainerTopOKBody{}, errors.New(\"container is not running\")\n\t}\n\n\treturn c.Client.ContainerTop(ctx, c.ID, []string{})\n}\n\n// PruneContainers prunes containers\nfunc (c *DockerCommand) PruneContainers() error {\n\t_, err := c.Client.ContainersPrune(context.Background(), filters.Args{})\n\treturn err\n}\n\n// Inspect returns details about the container\nfunc (c *Container) Inspect() (dockerTypes.ContainerJSON, error) {\n\treturn c.Client.ContainerInspect(context.Background(), c.ID)\n}\n\n// RenderTop returns details about the container\nfunc (c *Container) RenderTop(ctx context.Context) (string, error) {\n\tresult, err := c.Top(ctx)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn utils.RenderTable(append([][]string{result.Titles}, result.Processes...))\n}\n\n// DetailsLoaded tells us whether we have yet loaded the details for a container. Because this is an asynchronous operation, sometimes we have the container before we have its details.\nfunc (c *Container) DetailsLoaded() bool {\n\treturn c.Details.ContainerJSONBase != nil\n}\n\n// Path: pkg/gui/panels.go\npackage gui\n\nimport \"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\nfunc (gui *Gui) intoInterface() panels.IGui {\n\treturn gui\n}\n\n// Path: pkg/gui/gui.go\npackage gui\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\tdockerTypes \"github.com/docker/docker/api/types\"\n\n\t\"github.com/go-errors/errors\"\n\n\tthrottle \"github.com/boz/go-throttle\"\n\t\"github.com/jesseduffield/gocui\"\n\tlcUtils \"github.com/jesseduffield/lazycore/pkg/utils\"\n\t\"github.com/jesseduffield/lazydocker/pkg/commands\"\n\t\"github.com/jesseduffield/lazydocker/pkg/config\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/panels\"\n\t\"github.com/jesseduffield/lazydocker/pkg/gui/types\"\n\t\"github.com/jesseduffield/lazydocker/pkg/i18n\"\n\t\"github.com/jesseduffield/lazydocker/pkg/tasks\"\n\t\"github.com/sasha-s/go-deadlock\"\n\t\"github.com/sirupsen/logrus\"\n)\n\n// OverlappingEdges determines if panel edges overlap\nvar OverlappingEdges = false\n\n// Gui wraps the gocui Gui object which handles rendering and events\ntype Gui struct {\n\tg             *gocui.Gui\n\tLog           *logrus.Entry\n\tDockerCommand *commands.DockerCommand\n\tOSCommand     *commands.OSCommand\n\tState         guiState\n\tConfig        *config.AppConfig\n\tTr            *i18n.TranslationSet\n\tstatusManager *statusManager\n\ttaskManager   *tasks.TaskManager\n\tErrorChan     chan error\n\tViews         Views\n\n\t// if we've suspended the gui (e.g. because we've switched to a subprocess)\n\t// we typically want to pause some things that are running like background\n\t// file refreshes\n\tPauseBackgroundThreads bool\n\n\tMutexes\n\n\tPanels Panels\n}\n\ntype Panels struct {\n\tProjects   *panels.SideListPanel[*commands.Project]\n\tServices   *panels.SideListPanel[*commands.Service]\n\tContainers *panels.SideListPanel[*commands.Container]\n\tImages     *panels.SideListPanel[*commands.Image]\n\tVolumes    *panels.SideListPanel[*commands.Volume]\n\tNetworks   *panels.SideListPanel[*commands.Network]\n\tMenu       *panels.SideListPanel[*types.MenuItem]\n}\n\ntype Mutexes struct {\n\tSubprocessMutex deadlock.Mutex\n\tViewStackMutex  deadlock.Mutex\n}\n\ntype mainPanelState struct {\n\t// ObjectKey tells us what context we are in. For example, if we are looking at the logs of a particular service in the services panel this key might be 'services-<service id>-logs'. The key is made so that if something changes which might require us to re-run the logs command or run a different command, the key will be different, and we'll then know to do whatever is required. Object key probably isn't the best name for this but Context is already used to refer to tabs. Maybe I should just call them tabs.\n\tObjectKey string\n}\n\ntype panelStates struct {\n\tMain *mainPanelState\n}\n\ntype guiState struct {\n\t// the names of views in the current focus stack (last item is the current view)\n\tViewStack        []string\n\tPlatform         commands.Platform\n\tPanels           *panelStates\n\tSubProcessOutput string\n\tStats            map[string]commands.ContainerStats\n\n\t// if true, we show containers with an 'exited' status in the containers panel\n\tShowExitedContainers bool\n\n\tScreenMode WindowMaximisation\n\n\t// Maintains the state of manual filtering i.e. typing in a substring\n\t// to filter on in the current panel.\n\tFilter filterState\n}\n\ntype filterState struct {\n\t// If true then we're either currently inside the filter view\n\t// or we've committed the filter and we're back in the list view\n\tactive bool\n\t// The panel that we're filtering.\n\tpanel panels.ISideListPanel\n\t// The string that we're filtering on\n\tneedle string\n}\n\n// screen sizing determines how much space your selected window takes up (window\n// as in panel, not your terminal's window). Sometimes you want a bit more space\n// to see the contents of a panel, and this keeps track of how much maximisation\n// you've set\ntype WindowMaximisation int\n\nconst (\n\tSCREEN_NORMAL WindowMaximisation = iota\n\tSCREEN_HALF\n\tSCREEN_FULL\n)\n\n\nfunc getScreenMode(config *config.AppConfig) WindowMaximisation {\n\tswitch config.UserConfig.Gui.ScreenMode {\n\tcase \"normal\":\n\t\treturn SCREEN_NORMAL\n\tcase \"half\":\n\t\treturn SCREEN_HALF\n\tcase \"fullscreen\":\n\t\treturn SCREEN_FULL\n\tdefault:\n\t\treturn SCREEN_NORMAL\n\t}\n}\n\n// NewGui builds a new gui handler\nfunc NewGui(log *logrus.Entry, dockerCommand *commands.DockerCommand, oSCommand *commands.OSCommand, tr *i18n.TranslationSet, config *config.AppConfig, errorChan chan error) (*Gui, error) {\n\tinitialState := guiState{\n\t\tPlatform: *oSCommand.Platform,\n\t\tPanels: &panelStates{\n\t\t\tMain: &mainPanelState{\n\t\t\t\tObjectKey: \"\",\n\t\t\t},\n\t\t},\n\t\tViewStack: []string{},\n\n\t\tShowExitedContainers: true,\n\t\tScreenMode:           getScreenMode(config),\n\t}\n\n\tgui := &Gui{\n\t\tLog:           log,\n\t\tDockerCommand: dockerCommand,\n\t\tOSCommand:     oSCommand,\n\t\tState:         initialState,\n\t\tConfig:        config,\n\t\tTr:            tr,\n\t\tstatusManager: &statusManager{},\n\t\ttaskManager:   tasks.NewTaskManager(log, tr),\n\t\tErrorChan:     errorChan,\n\t}\n\n\tdeadlock.Opts.Disable = !gui.Config.Debug\n\tdeadlock.Opts.DeadlockTimeout = 10 * time.Second\n\n\treturn gui, nil\n}\n\nfunc (gui *Gui) renderGlobalOptions() error {\n\treturn gui.renderOptionsMap(map[string]string{\n\t\t\"PgUp/PgDn\": gui.Tr.Scroll,\n\t\t\"\u2190 \u2192 \u2191 \u2193\":   gui.Tr.Navigate,\n\t\t\"q\":         gui.Tr.Quit,\n\t\t\"b\":         gui.Tr.ViewBulkCommands,\n\t\t\"x\":         gui.Tr.Menu,\n\t})\n}\n\nfunc (gui *Gui) goEvery(interval time.Duration, function func() error) {\n\t_ = function() // time.Tick doesn't run immediately so we'll do that here // TODO: maybe change\n\tgo func() {\n\t\tticker := time.NewTicker(interval)\n\t\tdefer ticker.Stop()\n\t\tfor range ticker.C {\n\t\t\tif gui.PauseBackgroundThreads {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t_ = function()\n\t\t}\n\t}()\n}\n\n// Run setup the gui with keybindings and start the mainloop\nfunc (gui *Gui) Run() error {\n\t// closing our task manager which in turn closes the current task if there is any, so we aren't leaving processes lying around after closing lazydocker\n\tdefer gui.taskManager.Close()\n\n\tg, err := gocui.NewGui(gocui.OutputTrue, OverlappingEdges, gocui.NORMAL, false, map[rune]string{})\n...\n", "needle_token_start": 14031, "needle_token_end": 14134, "code_context_ntokens": 14842, "output": ["```go\n// screen sizing determines how much space your selected window takes up (window\n// as in panel, not your terminal's window). Sometimes you want a bit more space\n// to see the contents of a panel, and this keeps track of how much maximisation\n// you've set\ntype WindowMaximisation int\n\nconst (\n\tSCREEN_NORMAL WindowMaximisation = iota\n\tSCREEN_HALF\n\tSCREEN_FULL\n)\n\nfunc getScreenMode(config *config.AppConfig) WindowMaximisation {\n\tswitch config.UserConfig.Gui.ScreenMode {\n\tcase \"normal\":\n\t\treturn SCREEN_NORMAL\n\tcase \"half\":\n\t\treturn SCREEN_HALF\n\tcase \"fullscreen\":\n\t\treturn SCREEN_FULL\n\tdefault:\n\t\treturn SCREEN_NORMAL\n\t}\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "NewGaugeVec", "language": "go", "path": "core/metric/gauge.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To create and register a new gauge vector for monitoring and tracking metrics that can increase, decrease, or be set to specific values.\n2. **Input**: A configuration object containing metadata such as namespace, subsystem, name, help description, and labels for the metrics.\n3. **Output**: Returns an interface that allows manipulation of the gauge metrics, such as setting, incrementing, decrementing, adding, and subtracting values based on labels.\n4. **Procedure**: The function initializes a new gauge vector using the provided configuration. It registers this vector with Prometheus for metric collection and ensures it is properly cleaned up on system shutdown.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/metric/counter_test.go\npackage metric\n\nimport (\n\t\"testing\"\n\n\t\"github.com/prometheus/client_golang/prometheus/testutil\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/proc\"\n\t\"github.com/zeromicro/go-zero/core/prometheus\"\n)\n\nfunc TestNewCounterVec(t *testing.T) {\n\tcounterVec := NewCounterVec(&CounterVecOpts{\n\t\tNamespace: \"http_server\",\n\t\tSubsystem: \"requests\",\n\t\tName:      \"total\",\n\t\tHelp:      \"rpc client requests error count.\",\n\t})\n\tdefer counterVec.close()\n\tcounterVecNil := NewCounterVec(nil)\n\tcounterVec.Inc(\"path\", \"code\")\n\tcounterVec.Add(1, \"path\", \"code\")\n\tproc.Shutdown()\n\tassert.NotNil(t, counterVec)\n\tassert.Nil(t, counterVecNil)\n}\n\nfunc TestCounterIncr(t *testing.T) {\n\tstartAgent()\n\tcounterVec := NewCounterVec(&CounterVecOpts{\n\t\tNamespace: \"http_client\",\n\t\tSubsystem: \"call\",\n\t\tName:      \"code_total\",\n\t\tHelp:      \"http client requests error count.\",\n\t\tLabels:    []string{\"path\", \"code\"},\n\t})\n\tdefer counterVec.close()\n\tcv, _ := counterVec.(*promCounterVec)\n...\n// Path: core/metric/gauge.go\npackage metric\n\nimport (\n\tprom \"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/zeromicro/go-zero/core/proc\"\n)\n\ntype (\n\t// GaugeVecOpts is an alias of VectorOpts.\n\tGaugeVecOpts VectorOpts\n\n\t// GaugeVec represents a gauge vector.\n\tGaugeVec interface {\n\t\t// Set sets v to labels.\n\t\tSet(v float64, labels ...string)\n\t\t// Inc increments labels.\n\t\tInc(labels ...string)\n\t\t// Dec decrements labels.\n\t\tDec(labels ...string)\n\t\t// Add adds v to labels.\n\t\tAdd(v float64, labels ...string)\n\t\t// Sub subtracts v to labels.\n\t\tSub(v float64, labels ...string)\n\t\tclose() bool\n\t}\n\n\tpromGaugeVec struct {\n\t\tgauge *prom.GaugeVec\n\t}\n)\n\n// NewGaugeVec returns a GaugeVec.\n\nfunc NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {\n\tif cfg == nil {\n\t\treturn nil\n\t}\n\n\tvec := prom.NewGaugeVec(prom.GaugeOpts{\n\t\tNamespace: cfg.Namespace,\n\t\tSubsystem: cfg.Subsystem,\n\t\tName:      cfg.Name,\n\t\tHelp:      cfg.Help,\n\t}, cfg.Labels)\n\tprom.MustRegister(vec)\n\tgv := &promGaugeVec{\n\t\tgauge: vec,\n\t}\n\tproc.AddShutdownListener(func() {\n\t\tgv.close()\n\t})\n\n\treturn gv\n}\n\nfunc (gv *promGaugeVec) Add(v float64, labels ...string) {\n\tupdate(func() {\n\t\tgv.gauge.WithLabelValues(labels...).Add(v)\n\t})\n}\n\nfunc (gv *promGaugeVec) Dec(labels ...string) {\n\tupdate(func() {\n\t\tgv.gauge.WithLabelValues(labels...).Dec()\n\t})\n}\n\nfunc (gv *promGaugeVec) Inc(labels ...string) {\n\tupdate(func() {\n\t\tgv.gauge.WithLabelValues(labels...).Inc()\n\t})\n}\n\nfunc (gv *promGaugeVec) Set(v float64, labels ...string) {\n\tupdate(func() {\n\t\tgv.gauge.WithLabelValues(labels...).Set(v)\n\t})\n}\n\nfunc (gv *promGaugeVec) Sub(v float64, labels ...string) {\n\tupdate(func() {\n\t\tgv.gauge.WithLabelValues(labels...).Sub(v)\n\t})\n}\n\nfunc (gv *promGaugeVec) close() bool {\n\treturn prom.Unregister(gv.gauge)\n}\n\n// Path: core/metric/gauge_test.go\npackage metric\n\nimport (\n\t\"testing\"\n\n\t\"github.com/prometheus/client_golang/prometheus/testutil\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/proc\"\n)\n\nfunc TestNewGaugeVec(t *testing.T) {\n\tgaugeVec := NewGaugeVec(&GaugeVecOpts{\n\t\tNamespace: \"rpc_server\",\n\t\tSubsystem: \"requests\",\n\t\tName:      \"duration\",\n\t\tHelp:      \"rpc server requests duration(ms).\",\n\t})\n\tdefer gaugeVec.close()\n\tgaugeVecNil := NewGaugeVec(nil)\n\tassert.NotNil(t, gaugeVec)\n\tassert.Nil(t, gaugeVecNil)\n\n\tproc.Shutdown()\n}\n\nfunc TestGaugeInc(t *testing.T) {\n\tstartAgent()\n\tgaugeVec := NewGaugeVec(&GaugeVecOpts{\n\t\tNamespace: \"rpc_client2\",\n\t\tSubsystem: \"requests\",\n\t\tName:      \"duration_ms\",\n\t\tHelp:      \"rpc server requests duration(ms).\",\n\t\tLabels:    []string{\"path\"},\n\t})\n\tdefer gaugeVec.close()\n\tgv, _ := gaugeVec.(*promGaugeVec)\n\tgv.Inc(\"/users\")\n\tgv.Inc(\"/users\")\n\tr := testutil.ToFloat64(gv.gauge)\n\tassert.Equal(t, float64(2), r)\n}\n\nfunc TestGaugeDec(t *testing.T) {\n\tstartAgent()\n\tgaugeVec := NewGaugeVec(&GaugeVecOpts{\n\t\tNamespace: \"rpc_client\",\n\t\tSubsystem: \"requests\",\n\t\tName:      \"duration_ms\",\n\t\tHelp:      \"rpc server requests duration(ms).\",\n\t\tLabels:    []string{\"path\"},\n\t})\n\tdefer gaugeVec.close()\n\tgv, _ := gaugeVec.(*promGaugeVec)\n\tgv.Dec(\"/users\")\n\tgv.Dec(\"/users\")\n\tr := testutil.ToFloat64(gv.gauge)\n\tassert.Equal(t, float64(-2), r)\n}\n\nfunc TestGaugeAdd(t *testing.T) {\n\tstartAgent()\n\tgaugeVec := NewGaugeVec(&GaugeVecOpts{\n\t\tNamespace: \"rpc_client\",\n\t\tSubsystem: \"request\",\n\t\tName:      \"duration_ms\",\n\t\tHelp:      \"rpc server requests duration(ms).\",\n\t\tLabels:    []string{\"path\"},\n\t})\n\tdefer gaugeVec.close()\n\tgv, _ := gaugeVec.(*promGaugeVec)\n\tgv.Add(-10, \"/classroom\")\n\tgv.Add(30, \"/classroom\")\n\tr := testutil.ToFloat64(gv.gauge)\n\tassert.Equal(t, float64(20), r)\n}\n\nfunc TestGaugeSub(t *testing.T) {\n\tstartAgent()\n\tgaugeVec := NewGaugeVec(&GaugeVecOpts{\n\t\tNamespace: \"rpc_client\",\n\t\tSubsystem: \"request\",\n\t\tName:      \"duration_ms\",\n\t\tHelp:      \"rpc server requests duration(ms).\",\n\t\tLabels:    []string{\"path\"},\n\t})\n\tdefer gaugeVec.close()\n\tgv, _ := gaugeVec.(*promGaugeVec)\n\tgv.Sub(-100, \"/classroom\")\n\tgv.Sub(30, \"/classroom\")\n\tr := testutil.ToFloat64(gv.gauge)\n\tassert.Equal(t, float64(70), r)\n}\n\nfunc TestGaugeSet(t *testing.T) {\n\tstartAgent()\n\tgaugeVec := NewGaugeVec(&GaugeVecOpts{\n\t\tNamespace: \"http_client\",\n\t\tSubsystem: \"request\",\n\t\tName:      \"duration_ms\",\n\t\tHelp:      \"rpc server requests duration(ms).\",\n\t\tLabels:    []string{\"path\"},\n\t})\n\tgaugeVec.close()\n\tgv, _ := gaugeVec.(*promGaugeVec)\n\tgv.Set(666, \"/users\")\n\tr := testutil.ToFloat64(gv.gauge)\n\tassert.Equal(t, float64(666), r)\n}\n\n// Path: core/metric/histogram.go\npackage metric\n\nimport (\n\tprom \"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/zeromicro/go-zero/core/proc\"\n)\n\ntype (\n\t// A HistogramVecOpts is a histogram vector options.\n\tHistogramVecOpts struct {\n\t\tNamespace   string\n\t\tSubsystem   string\n\t\tName        string\n\t\tHelp        string\n\t\tLabels      []string\n\t\tBuckets     []float64\n\t\tConstLabels map[string]string\n\t}\n\n\t// A HistogramVec interface represents a histogram vector.\n\tHistogramVec interface {\n\t\t// Observe adds observation v to labels.\n\t\tObserve(v int64, labels ...string)\n\t\t// ObserveFloat allow to observe float64 values.\n\t\tObserveFloat(v float64, labels ...string)\n\t\tclose() bool\n\t}\n\n\tpromHistogramVec struct {\n\t\thistogram *prom.HistogramVec\n\t}\n)\n\n// NewHistogramVec returns a HistogramVec.\nfunc NewHistogramVec(cfg *HistogramVecOpts) HistogramVec {\n\tif cfg == nil {\n\t\treturn nil\n\t}\n\n\tvec := prom.NewHistogramVec(prom.HistogramOpts{\n\t\tNamespace:   cfg.Namespace,\n\t\tSubsystem:   cfg.Subsystem,\n\t\tName:        cfg.Name,\n\t\tHelp:        cfg.Help,\n\t\tBuckets:     cfg.Buckets,\n\t\tConstLabels: cfg.ConstLabels,\n\t}, cfg.Labels)\n\tprom.MustRegister(vec)\n\thv := &promHistogramVec{\n\t\thistogram: vec,\n\t}\n\tproc.AddShutdownListener(func() {\n\t\thv.close()\n\t})\n\n\treturn hv\n}\n\nfunc (hv *promHistogramVec) Observe(v int64, labels ...string) {\n\tupdate(func() {\n\t\thv.histogram.WithLabelValues(labels...).Observe(float64(v))\n\t})\n}\n\nfunc (hv *promHistogramVec) ObserveFloat(v float64, labels ...string) {\n\tupdate(func() {\n\t\thv.histogram.WithLabelValues(labels...).Observe(v)\n\t})\n}\n\nfunc (hv *promHistogramVec) close() bool {\n\treturn prom.Unregister(hv.histogram)\n}\n\n// Path: core/metric/histogram_test.go\npackage metric\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/prometheus/client_golang/prometheus/testutil\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestNewHistogramVec(t *testing.T) {\n\thistogramVec := NewHistogramVec(&HistogramVecOpts{\n\t\tName:    \"duration_ms\",\n\t\tHelp:    \"rpc server requests duration(ms).\",\n\t\tBuckets: []float64{1, 2, 3},\n\t})\n\tdefer histogramVec.(*promHistogramVec).close()\n\thistogramVecNil := NewHistogramVec(nil)\n\tassert.NotNil(t, histogramVec)\n\tassert.Nil(t, histogramVecNil)\n}\n\nfunc TestHistogramObserve(t *testing.T) {\n\tstartAgent()\n\thistogramVec := NewHistogramVec(&HistogramVecOpts{\n\t\tName:    \"counts\",\n\t\tHelp:    \"rpc server requests duration(ms).\",\n\t\tBuckets: []float64{1, 2, 3},\n\t\tLabels:  []string{\"method\"},\n\t})\n\tdefer histogramVec.(*promHistogramVec).close()\n\thv, _ := histogramVec.(*promHistogramVec)\n\thv.Observe(2, \"/Users\")\n\thv.ObserveFloat(1.1, \"/Users\")\n\n\tmetadata := `\n\t\t# HELP counts rpc server requests duration(ms).\n        # TYPE counts histogram\n`\n\tval := `\n\t\tcounts_bucket{method=\"/Users\",le=\"1\"} 0\n\t\tcounts_bucket{method=\"/Users\",le=\"2\"} 2\n\t\tcounts_bucket{method=\"/Users\",le=\"3\"} 2\n\t\tcounts_bucket{method=\"/Users\",le=\"+Inf\"} 2\n\t\tcounts_sum{method=\"/Users\"} 3.1\n        counts_count{method=\"/Users\"} 2\n`\n\n\terr := testutil.CollectAndCompare(hv.histogram, strings.NewReader(metadata+val))\n\tassert.Nil(t, err)\n}\n\n// Path: core/metric/metric.go\npackage metric\n\nimport \"github.com/zeromicro/go-zero/core/prometheus\"\n\n// A VectorOpts is a general configuration.\ntype VectorOpts struct {\n\tNamespace string\n\tSubsystem string\n\tName      string\n\tHelp      string\n\tLabels    []string\n}\n\nfunc update(fn func()) {\n\tif !prometheus.Enabled() {\n\t\treturn\n\t}\n\n\tfn()\n}\n\n// Path: core/metric/summary.go\npackage metric\n\nimport (\n\tprom \"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/zeromicro/go-zero/core/proc\"\n)\n\ntype (\n\t// A SummaryVecOpts is a summary vector options\n\tSummaryVecOpts struct {\n\t\tVecOpt     VectorOpts\n\t\tObjectives map[float64]float64\n\t}\n\n\t// A SummaryVec interface represents a summary vector.\n\tSummaryVec interface {\n\t\t// Observe adds observation v to labels.\n\t\tObserve(v float64, labels ...string)\n\t\tclose() bool\n\t}\n\n\tpromSummaryVec struct {\n\t\tsummary *prom.SummaryVec\n\t}\n)\n\n// NewSummaryVec return a SummaryVec\nfunc NewSummaryVec(cfg *SummaryVecOpts) SummaryVec {\n\tif cfg == nil {\n\t\treturn nil\n\t}\n\n\tvec := prom.NewSummaryVec(\n\t\tprom.SummaryOpts{\n\t\t\tNamespace:  cfg.VecOpt.Namespace,\n\t\t\tSubsystem:  cfg.VecOpt.Subsystem,\n\t\t\tName:       cfg.VecOpt.Name,\n\t\t\tHelp:       cfg.VecOpt.Help,\n\t\t\tObjectives: cfg.Objectives,\n\t\t},\n\t\tcfg.VecOpt.Labels,\n\t)\n\tprom.MustRegister(vec)\n\tsv := &promSummaryVec{\n\t\tsummary: vec,\n\t}\n\tproc.AddShutdownListener(func() {\n\t\tsv.close()\n\t})\n\n\treturn sv\n}\n\nfunc (sv *promSummaryVec) Observe(v float64, labels ...string) {\n\tupdate(func() {\n\t\tsv.summary.WithLabelValues(labels...).Observe(v)\n\t})\n}\n\nfunc (sv *promSummaryVec) close() bool {\n\treturn prom.Unregister(sv.summary)\n}\n\n// Path: core/metric/summary_test.go\npackage metric\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/prometheus/client_golang/prometheus/testutil\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/proc\"\n)\n\nfunc TestNewSummaryVec(t *testing.T) {\n\tsummaryVec := NewSummaryVec(&SummaryVecOpts{\n\t\tVecOpt: VectorOpts{\n\t\t\tNamespace: \"http_server\",\n\t\t\tSubsystem: \"requests\",\n\t\t\tName:      \"duration_quantiles\",\n\t\t\tHelp:      \"rpc client requests duration(ms) \u03c6 quantiles \",\n\t\t\tLabels:    []string{\"method\"},\n\t\t},\n\t\tObjectives: map[float64]float64{\n\t\t\t0.5: 0.01,\n\t\t\t0.9: 0.01,\n\t\t},\n\t})\n\tdefer summaryVec.close()\n\tsummaryVecNil := NewSummaryVec(nil)\n\tassert.NotNil(t, summaryVec)\n\tassert.Nil(t, summaryVecNil)\n}\n\nfunc TestSummaryObserve(t *testing.T) {\n\tstartAgent()\n\tsummaryVec := NewSummaryVec(&SummaryVecOpts{\n\t\tVecOpt: VectorOpts{\n\t\t\tNamespace: \"http_server\",\n\t\t\tSubsystem: \"requests\",\n\t\t\tName:      \"duration_quantiles\",\n\t\t\tHelp:      \"rpc client requests duration(ms) \u03c6 quantiles \",\n\t\t\tLabels:    []string{\"method\"},\n\t\t},\n\t\tObjectives: map[float64]float64{\n\t\t\t0.3: 0.01,\n\t\t\t0.6: 0.01,\n\t\t\t1:   0.01,\n\t\t},\n\t})\n\tdefer summaryVec.close()\n\tsv := summaryVec.(*promSummaryVec)\n\tsv.Observe(100, \"GET\")\n\tsv.Observe(200, \"GET\")\n\tsv.Observe(300, \"GET\")\n\tmetadata := `\n\t\t# HELP http_server_requests_duration_quantiles rpc client requests duration(ms) \u03c6 quantiles \n\t\t# TYPE http_server_requests_duration_quantiles summary\n`\n\tval := `\n\t\thttp_server_requests_duration_quantiles{method=\"GET\",quantile=\"0.3\"} 100\n\t\thttp_server_requests_duration_quantiles{method=\"GET\",quantile=\"0.6\"} 200\n\t\thttp_server_requests_duration_quantiles{method=\"GET\",quantile=\"1\"} 300\n\t\thttp_server_requests_duration_quantiles_sum{method=\"GET\"} 600\n\t\thttp_server_requests_duration_quantiles_count{method=\"GET\"} 3\n`\n\n\terr := testutil.CollectAndCompare(sv.summary, strings.NewReader(metadata+val))\n\tassert.Nil(t, err)\n\tproc.Shutdown()\n}\n\n// Path: core/syncx/once.go\npackage syncx\n\nimport \"sync\"\n\n// Once returns a func that guarantees fn can only called once.\nfunc Once(fn func()) func() {\n\tonce := new(sync.Once)\n\treturn func() {\n\t\tonce.Do(fn)\n\t}\n}\n\n// Path: core/mr/mapreduce.go\npackage mr\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/zeromicro/go-zero/core/errorx\"\n)\n\nconst (\n\tdefaultWorkers = 16\n\tminWorkers     = 1\n)\n\nvar (\n\t// ErrCancelWithNil is an error that mapreduce was cancelled with nil.\n\tErrCancelWithNil = errors.New(\"mapreduce cancelled with nil\")\n\t// ErrReduceNoOutput is an error that reduce did not output a value.\n\tErrReduceNoOutput = errors.New(\"reduce not writing value\")\n)\n\ntype (\n\t// ForEachFunc is used to do element processing, but no output.\n\tForEachFunc[T any] func(item T)\n\t// GenerateFunc is used to let callers send elements into source.\n\tGenerateFunc[T any] func(source chan<- T)\n\t// MapFunc is used to do element processing and write the output to writer.\n\tMapFunc[T, U any] func(item T, writer Writer[U])\n\t// MapperFunc is used to do element processing and write the output to writer,\n\t// use cancel func to cancel the processing.\n\tMapperFunc[T, U any] func(item T, writer Writer[U], cancel func(error))\n\t// ReducerFunc is used to reduce all the mapping output and write to writer,\n\t// use cancel func to cancel the processing.\n\tReducerFunc[U, V any] func(pipe <-chan U, writer Writer[V], cancel func(error))\n\t// VoidReducerFunc is used to reduce all the mapping output, but no output.\n\t// Use cancel func to cancel the processing.\n\tVoidReducerFunc[U any] func(pipe <-chan U, cancel func(error))\n\t// Option defines the method to customize the mapreduce.\n\tOption func(opts *mapReduceOptions)\n\n\tmapperContext[T, U any] struct {\n\t\tctx       context.Context\n\t\tmapper    MapFunc[T, U]\n\t\tsource    <-chan T\n\t\tpanicChan *onceChan\n\t\tcollector chan<- U\n\t\tdoneChan  <-chan struct{}\n\t\tworkers   int\n\t}\n\n\tmapReduceOptions struct {\n\t\tctx     context.Context\n\t\tworkers int\n\t}\n\n\t// Writer interface wraps Write method.\n\tWriter[T any] interface {\n\t\tWrite(v T)\n\t}\n)\n\n// Finish runs fns parallelly, cancelled on any error.\nfunc Finish(fns ...func() error) error {\n\tif len(fns) == 0 {\n\t\treturn nil\n\t}\n\n\treturn MapReduceVoid(func(source chan<- func() error) {\n\t\tfor _, fn := range fns {\n\t\t\tsource <- fn\n\t\t}\n\t}, func(fn func() error, writer Writer[any], cancel func(error)) {\n\t\tif err := fn(); err != nil {\n\t\t\tcancel(err)\n\t\t}\n\t}, func(pipe <-chan any, cancel func(error)) {\n\t}, WithWorkers(len(fns)))\n}\n\n// FinishVoid runs fns parallelly.\nfunc FinishVoid(fns ...func()) {\n\tif len(fns) == 0 {\n\t\treturn\n\t}\n\n\tForEach(func(source chan<- func()) {\n\t\tfor _, fn := range fns {\n\t\t\tsource <- fn\n\t\t}\n\t}, func(fn func()) {\n\t\tfn()\n\t}, WithWorkers(len(fns)))\n}\n\n// ForEach maps all elements from given generate but no output.\nfunc ForEach[T any](generate GenerateFunc[T], mapper ForEachFunc[T], opts ...Option) {\n\toptions := buildOptions(opts...)\n\tpanicChan := &onceChan{channel: make(chan any)}\n\tsource := buildSource(generate, panicChan)\n\tcollector := make(chan any)\n\tdone := make(chan struct{})\n\n\tgo executeMappers(mapperContext[T, any]{\n\t\tctx: options.ctx,\n\t\tmapper: func(item T, _ Writer[any]) {\n\t\t\tmapper(item)\n\t\t},\n\t\tsource:    source,\n\t\tpanicChan: panicChan,\n\t\tcollector: collector,\n\t\tdoneChan:  done,\n\t\tworkers:   options.workers,\n\t})\n\n\tfor {\n\t\tselect {\n\t\tcase v := <-panicChan.channel:\n\t\t\tpanic(v)\n\t\tcase _, ok := <-collector:\n\t\t\tif !ok {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// MapReduce maps all elements generated from given generate func,\n// and reduces the output elements with given reducer.\nfunc MapReduce[T, U, V any](generate GenerateFunc[T], mapper MapperFunc[T, U], reducer ReducerFunc[U, V],\n\topts ...Option) (V, error) {\n\tpanicChan := &onceChan{channel: make(chan any)}\n\tsource := buildSource(generate, panicChan)\n\treturn mapReduceWithPanicChan(source, panicChan, mapper, reducer, opts...)\n}\n\n// MapReduceChan maps all elements from source, and reduce the output elements with given reducer.\nfunc MapReduceChan[T, U, V any](source <-chan T, mapper MapperFunc[T, U], reducer ReducerFunc[U, V],\n\topts ...Option) (V, error) {\n\tpanicChan := &onceChan{channel: make(chan any)}\n\treturn mapReduceWithPanicChan(source, panicChan, mapper, reducer, opts...)\n}\n\n// mapReduceWithPanicChan maps all elements from source, and reduce the output elements with given reducer.\nfunc mapReduceWithPanicChan[T, U, V any](source <-chan T, panicChan *onceChan, mapper MapperFunc[T, U],\n\treducer ReducerFunc[U, V], opts ...Option) (val V, err error) {\n\toptions := buildOptions(opts...)\n\t// output is used to write the final result\n\toutput := make(chan V)\n\tdefer func() {\n\t\t// reducer can only write once, if more, panic\n\t\tfor range output {\n\t\t\tpanic(\"more than one element written in reducer\")\n\t\t}\n\t}()\n\n\t// collector is used to collect data from mapper, and consume in reducer\n\tcollector := make(chan U, options.workers)\n\t// if done is closed, all mappers and reducer should stop processing\n\tdone := make(chan struct{})\n\twriter := newGuardedWriter(options.ctx, output, done)\n\tvar closeOnce sync.Once\n\t// use atomic type to avoid data race\n\tvar retErr errorx.AtomicError\n\tfinish := func() {\n\t\tcloseOnce.Do(func() {\n\t\t\tclose(done)\n\t\t\tclose(output)\n\t\t})\n\t}\n\tcancel := once(func(err error) {\n\t\tif err != nil {\n\t\t\tretErr.Set(err)\n\t\t} else {\n\t\t\tretErr.Set(ErrCancelWithNil)\n\t\t}\n\n\t\tdrain(source)\n\t\tfinish()\n\t})\n\n\tgo func() {\n\t\tdefer func() {\n\t\t\tdrain(collector)\n\t\t\tif r := recover(); r != nil {\n\t\t\t\tpanicChan.write(r)\n\t\t\t}\n\t\t\tfinish()\n\t\t}()\n\n\t\treducer(collector, writer, cancel)\n\t}()\n\n\tgo executeMappers(mapperContext[T, U]{\n\t\tctx: options.ctx,\n\t\tmapper: func(item T, w Writer[U]) {\n\t\t\tmapper(item, w, cancel)\n\t\t},\n\t\tsource:    source,\n\t\tpanicChan: panicChan,\n\t\tcollector: collector,\n\t\tdoneChan:  done,\n\t\tworkers:   options.workers,\n\t})\n\n\tselect {\n\tcase <-options.ctx.Done():\n\t\tcancel(context.DeadlineExceeded)\n\t\terr = context.DeadlineExceeded\n\tcase v := <-panicChan.channel:\n\t\t// drain output here, otherwise for loop panic in defer\n\t\tdrain(output)\n\t\tpanic(v)\n\tcase v, ok := <-output:\n\t\tif e := retErr.Load(); e != nil {\n\t\t\terr = e\n\t\t} else if ok {\n\t\t\tval = v\n\t\t} else {\n\t\t\terr = ErrReduceNoOutput\n\t\t}\n\t}\n\n\treturn\n}\n\n// MapReduceVoid maps all elements generated from given generate,\n// and reduce the output elements with given reducer.\nfunc MapReduceVoid[T, U any](generate GenerateFunc[T], mapper MapperFunc[T, U],\n\treducer VoidReducerFunc[U], opts ...Option) error {\n\t_, err := MapReduce(generate, mapper, func(input <-chan U, writer Writer[any], cancel func(error)) {\n\t\treducer(input, cancel)\n\t}, opts...)\n\tif errors.Is(err, ErrReduceNoOutput) {\n\t\treturn nil\n\t}\n\n\treturn err\n}\n\n// WithContext customizes a mapreduce processing accepts a given ctx.\nfunc WithContext(ctx context.Context) Option {\n\treturn func(opts *mapReduceOptions) {\n\t\topts.ctx = ctx\n\t}\n}\n\n// WithWorkers customizes a mapreduce processing with given workers.\nfunc WithWorkers(workers int) Option {\n\treturn func(opts *mapReduceOptions) {\n\t\tif workers < minWorkers {\n\t\t\topts.workers = minWorkers\n\t\t} else {\n\t\t\topts.workers = workers\n\t\t}\n\t}\n}\n\nfunc buildOptions(opts ...Option) *mapReduceOptions {\n\toptions := newOptions()\n\tfor _, opt := range opts {\n\t\topt(options)\n\t}\n\n\treturn options\n}\n\nfunc buildSource[T any](generate GenerateFunc[T], panicChan *onceChan) chan T {\n\tsource := make(chan T)\n\tgo func() {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\tpanicChan.write(r)\n\t\t\t}\n\t\t\tclose(source)\n\t\t}()\n\n\t\tgenerate(source)\n\t}()\n\n\treturn source\n}\n\n// drain drains the channel.\nfunc drain[T any](channel <-chan T) {\n\t// drain the channel\n\tfor range channel {\n\t}\n}\n\nfunc executeMappers[T, U any](mCtx mapperContext[T, U]) {\n\tvar wg sync.WaitGroup\n\tdefer func() {\n\t\twg.Wait()\n\t\tclose(mCtx.collector)\n\t\tdrain(mCtx.source)\n\t}()\n\n\tvar failed int32\n\tpool := make(chan struct{}, mCtx.workers)\n\twriter := newGuardedWriter(mCtx.ctx, mCtx.collector, mCtx.doneChan)\n\tfor atomic.LoadInt32(&failed) == 0 {\n\t\tselect {\n\t\tcase <-mCtx.ctx.Done():\n\t\t\treturn\n\t\tcase <-mCtx.doneChan:\n\t\t\treturn\n\t\tcase pool <- struct{}{}:\n\t\t\titem, ok := <-mCtx.source\n\t\t\tif !ok {\n\t\t\t\t<-pool\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer func() {\n\t\t\t\t\tif r := recover(); r != nil {\n\t\t\t\t\t\tatomic.AddInt32(&failed, 1)\n\t\t\t\t\t\tmCtx.panicChan.write(r)\n\t\t\t\t\t}\n\t\t\t\t\twg.Done()\n\t\t\t\t\t<-pool\n\t\t\t\t}()\n\n\t\t\t\tmCtx.mapper(item, writer)\n\t\t\t}()\n\t\t}\n\t}\n}\n\nfunc newOptions() *mapReduceOptions {\n\treturn &mapReduceOptions{\n\t\tctx:     context.Background(),\n\t\tworkers: defaultWorkers,\n\t}\n}\n\nfunc once(fn func(error)) func(error) {\n\tonce := new(sync.Once)\n\treturn func(err error) {\n\t\tonce.Do(func() {\n\t\t\tfn(err)\n\t\t})\n\t}\n}\n\ntype guardedWriter[T any] struct {\n\tctx     context.Context\n\tchannel chan<- T\n\tdone    <-chan struct{}\n}\n\nfunc newGuardedWriter[T any](ctx context.Context, channel chan<- T, done <-chan struct{}) guardedWriter[T] {\n\treturn guardedWriter[T]{\n\t\tctx:     ctx,\n\t\tchannel: channel,\n\t\tdone:    done,\n\t}\n}\n\nfunc (gw guardedWriter[T]) Write(v T) {\n\tselect {\n\tcase <-gw.ctx.Done():\n\t\treturn\n\tcase <-gw.done:\n\t\treturn\n\tdefault:\n\t\tgw.channel <- v\n\t}\n}\n\ntype onceChan struct {\n\tchannel chan any\n\twrote   int32\n}\n\nfunc (oc *onceChan) write(val any) {\n\tif atomic.CompareAndSwapInt32(&oc.wrote, 0, 1) {\n\t\toc.channel <- val\n\t}\n}\n\n// Path: core/mr/mapreduce_fuzz_test.go\npackage mr\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"runtime\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"go.uber.org/goleak\"\n)\n\nfunc FuzzMapReduce(f *testing.F) {\n\trand.NewSource(time.Now().UnixNano())\n\n\tf.Add(int64(10), runtime.NumCPU())\n\tf.Fuzz(func(t *testing.T, n int64, workers int) {\n\t\tn = n%5000 + 5000\n\t\tgenPanic := rand.Intn(100) == 0\n\t\tmapperPanic := rand.Intn(100) == 0\n\t\treducerPanic := rand.Intn(100) == 0\n\t\tgenIdx := rand.Int63n(n)\n\t\tmapperIdx := rand.Int63n(n)\n\t\treducerIdx := rand.Int63n(n)\n\t\tsquareSum := (n - 1) * n * (2*n - 1) / 6\n\n\t\tfn := func() (int64, error) {\n\t\t\tdefer goleak.VerifyNone(t, goleak.IgnoreCurrent())\n\n\t\t\treturn MapReduce(func(source chan<- int64) {\n\t\t\t\tfor i := int64(0); i < n; i++ {\n\t\t\t\t\tsource <- i\n\t\t\t\t\tif genPanic && i == genIdx {\n\t\t\t\t\t\tpanic(\"foo\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}, func(v int64, writer Writer[int64], cancel func(error)) {\n\t\t\t\tif mapperPanic && v == mapperIdx {\n\t\t\t\t\tpanic(\"bar\")\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t}, func(pipe <-chan int64, writer Writer[int64], cancel func(error)) {\n\t\t\t\tvar idx int64\n\t\t\t\tvar total int64\n\t\t\t\tfor v := range pipe {\n\t\t\t\t\tif reducerPanic && idx == reducerIdx {\n\t\t\t\t\t\tpanic(\"baz\")\n\t\t\t\t\t}\n\t\t\t\t\ttotal += v\n\t\t\t\t\tidx++\n\t\t\t\t}\n\t\t\t\twriter.Write(total)\n\t\t\t}, WithWorkers(workers%50+runtime.NumCPU()))\n\t\t}\n\n\t\tif genPanic || mapperPanic || reducerPanic {\n\t\t\tvar buf strings.Builder\n\t\t\tbuf.WriteString(fmt.Sprintf(\"n: %d\", n))\n\t\t\tbuf.WriteString(fmt.Sprintf(\", genPanic: %t\", genPanic))\n\t\t\tbuf.WriteString(fmt.Sprintf(\", mapperPanic: %t\", mapperPanic))\n\t\t\tbuf.WriteString(fmt.Sprintf(\", reducerPanic: %t\", reducerPanic))\n\t\t\tbuf.WriteString(fmt.Sprintf(\", genIdx: %d\", genIdx))\n\t\t\tbuf.WriteString(fmt.Sprintf(\", mapperIdx: %d\", mapperIdx))\n\t\t\tbuf.WriteString(fmt.Sprintf(\", reducerIdx: %d\", reducerIdx))\n\t\t\tassert.Panicsf(t, func() { fn() }, buf.String())\n\t\t} else {\n\t\t\tval, err := fn()\n\t\t\tassert.Nil(t, err)\n\t\t\tassert.Equal(t, squareSum, val)\n\t\t}\n\t})\n}\n\n// Path: core/mr/mapreduce_fuzzcase_test.go\n//go:build fuzz\n\npackage mr\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/threading\"\n\t\"gopkg.in/cheggaaa/pb.v1\"\n)\n\n// If Fuzz stuck, we don't know why, because it only returns hung or unexpected,\n// so we need to simulate the fuzz test in test mode.\nfunc TestMapReduceRandom(t *testing.T) {\n\trand.NewSource(time.Now().UnixNano())\n\n\tconst (\n\t\ttimes  = 10000\n\t\tnRange = 500\n\t\tmega   = 1024 * 1024\n\t)\n\n\tbar := pb.New(times).Start()\n\trunner := threading.NewTaskRunner(runtime.NumCPU())\n\tvar wg sync.WaitGroup\n\twg.Add(times)\n\tfor i := 0; i < times; i++ {\n\t\trunner.Schedule(func() {\n\t\t\tstart := time.Now()\n\t\t\tdefer func() {\n\t\t\t\tif time.Since(start) > time.Minute {\n\t\t\t\t\tt.Fatal(\"timeout\")\n\t\t\t\t}\n\t\t\t\twg.Done()\n\t\t\t}()\n\n\t\t\tt.Run(strconv.Itoa(i), func(t *testing.T) {\n\t\t\t\tn := rand.Int63n(nRange)%nRange + nRange\n\t\t\t\tworkers := rand.Int()%50 + runtime.NumCPU()/2\n\t\t\t\tgenPanic := rand.Intn(100) == 0\n\t\t\t\tmapperPanic := rand.Intn(100) == 0\n\t\t\t\treducerPanic := rand.Intn(100) == 0\n\t\t\t\tgenIdx := rand.Int63n(n)\n\t\t\t\tmapperIdx := rand.Int63n(n)\n\t\t\t\treducerIdx := rand.Int63n(n)\n\t\t\t\tsquareSum := (n - 1) * n * (2*n - 1) / 6\n\n\t\t\t\tfn := func() (int64, error) {\n\t\t\t\t\treturn MapReduce(func(source chan<- int64) {\n\t\t\t\t\t\tfor i := int64(0); i < n; i++ {\n\t\t\t\t\t\t\tsource <- i\n\t\t\t\t\t\t\tif genPanic && i == genIdx {\n\t\t\t\t\t\t\t\tpanic(\"foo\")\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}, func(v int64, writer Writer[int64], cancel func(error)) {\n\t\t\t\t\t\tif mapperPanic && v == mapperIdx {\n\t\t\t\t\t\t\tpanic(\"bar\")\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.Write(v * v)\n\t\t\t\t\t}, func(pipe <-chan int64, writer Writer[int64], cancel func(error)) {\n\t\t\t\t\t\tvar idx int64\n\t\t\t\t\t\tvar total int64\n\t\t\t\t\t\tfor v := range pipe {\n\t\t\t\t\t\t\tif reducerPanic && idx == reducerIdx {\n\t\t\t\t\t\t\t\tpanic(\"baz\")\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\ttotal += v\n\t\t\t\t\t\t\tidx++\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.Write(total)\n\t\t\t\t\t}, WithWorkers(int(workers)%50+runtime.NumCPU()/2))\n\t\t\t\t}\n\n\t\t\t\tif genPanic || mapperPanic || reducerPanic {\n\t\t\t\t\tvar buf strings.Builder\n\t\t\t\t\tbuf.WriteString(fmt.Sprintf(\"n: %d\", n))\n\t\t\t\t\tbuf.WriteString(fmt.Sprintf(\", genPanic: %t\", genPanic))\n\t\t\t\t\tbuf.WriteString(fmt.Sprintf(\", mapperPanic: %t\", mapperPanic))\n\t\t\t\t\tbuf.WriteString(fmt.Sprintf(\", reducerPanic: %t\", reducerPanic))\n\t\t\t\t\tbuf.WriteString(fmt.Sprintf(\", genIdx: %d\", genIdx))\n\t\t\t\t\tbuf.WriteString(fmt.Sprintf(\", mapperIdx: %d\", mapperIdx))\n\t\t\t\t\tbuf.WriteString(fmt.Sprintf(\", reducerIdx: %d\", reducerIdx))\n\t\t\t\t\tassert.Panicsf(t, func() { fn() }, buf.String())\n\t\t\t\t} else {\n\t\t\t\t\tval, err := fn()\n\t\t\t\t\tassert.Nil(t, err)\n\t\t\t\t\tassert.Equal(t, squareSum, val)\n\t\t\t\t}\n\t\t\t\tbar.Increment()\n\t\t\t})\n\t\t})\n\t}\n\n\twg.Wait()\n\tbar.Finish()\n}\n\n// Path: core/mr/mapreduce_test.go\npackage mr\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"log\"\n\t\"runtime\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"go.uber.org/goleak\"\n)\n\nvar errDummy = errors.New(\"dummy\")\n\nfunc init() {\n\tlog.SetOutput(io.Discard)\n}\n\nfunc TestFinish(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar total uint32\n\terr := Finish(func() error {\n\t\tatomic.AddUint32(&total, 2)\n\t\treturn nil\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 3)\n\t\treturn nil\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 5)\n\t\treturn nil\n\t})\n\n\tassert.Equal(t, uint32(10), atomic.LoadUint32(&total))\n\tassert.Nil(t, err)\n}\n\nfunc TestFinishNone(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tassert.Nil(t, Finish())\n}\n\nfunc TestFinishVoidNone(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tFinishVoid()\n}\n\nfunc TestFinishErr(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar total uint32\n\terr := Finish(func() error {\n\t\tatomic.AddUint32(&total, 2)\n\t\treturn nil\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 3)\n\t\treturn errDummy\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 5)\n\t\treturn nil\n\t})\n\n\tassert.Equal(t, errDummy, err)\n}\n\nfunc TestFinishVoid(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar total uint32\n\tFinishVoid(func() {\n\t\tatomic.AddUint32(&total, 2)\n\t}, func() {\n\t\tatomic.AddUint32(&total, 3)\n\t}, func() {\n\t\tatomic.AddUint32(&total, 5)\n\t})\n\n\tassert.Equal(t, uint32(10), atomic.LoadUint32(&total))\n}\n\nfunc TestForEach(t *testing.T) {\n\tconst tasks = 1000\n\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tdefer goleak.VerifyNone(t)\n\n\t\tvar count uint32\n\t\tForEach(func(source chan<- int) {\n\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\tsource <- i\n\t\t\t}\n\t\t}, func(item int) {\n\t\t\tatomic.AddUint32(&count, 1)\n\t\t}, WithWorkers(-1))\n\n\t\tassert.Equal(t, tasks, int(count))\n\t})\n\n\tt.Run(\"odd\", func(t *testing.T) {\n\t\tdefer goleak.VerifyNone(t)\n\n\t\tvar count uint32\n\t\tForEach(func(source chan<- int) {\n\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\tsource <- i\n\t\t\t}\n\t\t}, func(item int) {\n\t\t\tif item%2 == 0 {\n\t\t\t\tatomic.AddUint32(&count, 1)\n\t\t\t}\n\t\t})\n\n\t\tassert.Equal(t, tasks/2, int(count))\n\t})\n\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tdefer goleak.VerifyNone(t)\n\n\t\tassert.PanicsWithValue(t, \"foo\", func() {\n\t\t\tForEach(func(source chan<- int) {\n\t\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\t\tsource <- i\n\t\t\t\t}\n\t\t\t}, func(item int) {\n\t\t\t\tpanic(\"foo\")\n\t\t\t})\n\t\t})\n\t})\n}\n\nfunc TestGeneratePanic(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tassert.PanicsWithValue(t, \"foo\", func() {\n\t\t\tForEach(func(source chan<- int) {\n\t\t\t\tpanic(\"foo\")\n\t\t\t}, func(item int) {\n\t\t\t})\n\t\t})\n\t})\n}\n\nfunc TestMapperPanic(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tconst tasks = 1000\n\tvar run int32\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tassert.PanicsWithValue(t, \"foo\", func() {\n\t\t\t_, _ = MapReduce(func(source chan<- int) {\n\t\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\t\tsource <- i\n\t\t\t\t}\n\t\t\t}, func(item int, writer Writer[int], cancel func(error)) {\n\t\t\t\tatomic.AddInt32(&run, 1)\n\t\t\t\tpanic(\"foo\")\n\t\t\t}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t})\n\t\t})\n\t\tassert.True(t, atomic.LoadInt32(&run) < tasks/2)\n\t})\n}\n\nfunc TestMapReduce(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\ttests := []struct {\n\t\tname        string\n\t\tmapper      MapperFunc[int, int]\n\t\treducer     ReducerFunc[int, int]\n\t\texpectErr   error\n\t\texpectValue int\n\t}{\n\t\t{\n\t\t\tname:        \"simple\",\n\t\t\texpectErr:   nil,\n\t\t\texpectValue: 30,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with error\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(errDummy)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with nil\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(nil)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: ErrCancelWithNil,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with more\",\n\t\t\treducer: func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t\tvar result int\n\t\t\t\tfor item := range pipe {\n\t\t\t\t\tresult += item\n\t\t\t\t\tif result > 10 {\n\t\t\t\t\t\tcancel(errDummy)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\twriter.Write(result)\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t}\n\n\tt.Run(\"MapReduce\", func(t *testing.T) {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\tif test.mapper == nil {\n\t\t\t\t\ttest.mapper = func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\twriter.Write(v * v)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif test.reducer == nil {\n\t\t\t\t\ttest.reducer = func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\tvar result int\n\t\t\t\t\t\tfor item := range pipe {\n\t\t\t\t\t\t\tresult += item\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.Write(result)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tvalue, err := MapReduce(func(source chan<- int) {\n\t\t\t\t\tfor i := 1; i < 5; i++ {\n\t\t\t\t\t\tsource <- i\n\t\t\t\t\t}\n\t\t\t\t}, test.mapper, test.reducer, WithWorkers(runtime.NumCPU()))\n\n\t\t\t\tassert.Equal(t, test.expectErr, err)\n\t\t\t\tassert.Equal(t, test.expectValue, value)\n\t\t\t})\n\t\t}\n\t})\n\n\tt.Run(\"MapReduce\", func(t *testing.T) {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\tif test.mapper == nil {\n\t\t\t\t\ttest.mapper = func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\twriter.Write(v * v)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif test.reducer == nil {\n\t\t\t\t\ttest.reducer = func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\tvar result int\n\t\t\t\t\t\tfor item := range pipe {\n\t\t\t\t\t\t\tresult += item\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.Write(result)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tsource := make(chan int)\n\t\t\t\tgo func() {\n\t\t\t\t\tfor i := 1; i < 5; i++ {\n\t\t\t\t\t\tsource <- i\n\t\t\t\t\t}\n\t\t\t\t\tclose(source)\n\t\t\t\t}()\n\n\t\t\t\tvalue, err := MapReduceChan(source, test.mapper, test.reducer, WithWorkers(-1))\n\t\t\t\tassert.Equal(t, test.expectErr, err)\n\t\t\t\tassert.Equal(t, test.expectValue, value)\n\t\t\t})\n\t\t}\n\t})\n}\n\nfunc TestMapReduceWithReduerWriteMoreThanOnce(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tassert.Panics(t, func() {\n\t\tMapReduce(func(source chan<- int) {\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsource <- i\n\t\t\t}\n\t\t}, func(item int, writer Writer[int], cancel func(error)) {\n\t\t\twriter.Write(item)\n\t\t}, func(pipe <-chan int, writer Writer[string], cancel func(error)) {\n\t\t\tdrain(pipe)\n\t\t\twriter.Write(\"one\")\n\t\t\twriter.Write(\"two\")\n\t\t})\n\t})\n}\n\nfunc TestMapReduceVoid(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar value uint32\n\ttests := []struct {\n\t\tname        string\n\t\tmapper      MapperFunc[int, int]\n\t\treducer     VoidReducerFunc[int]\n\t\texpectValue uint32\n\t\texpectErr   error\n\t}{\n\t\t{\n\t\t\tname:        \"simple\",\n\t\t\texpectValue: 30,\n\t\t\texpectErr:   nil,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with error\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(errDummy)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with nil\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(nil)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: ErrCancelWithNil,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with more\",\n\t\t\treducer: func(pipe <-chan int, cancel func(error)) {\n\t\t\t\tfor item := range pipe {\n\t\t\t\t\tresult := atomic.AddUint32(&value, uint32(item))\n\t\t\t\t\tif result > 10 {\n\t\t\t\t\t\tcancel(errDummy)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tatomic.StoreUint32(&value, 0)\n\n\t\t\tif test.mapper == nil {\n\t\t\t\ttest.mapper = func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\twriter.Write(v * v)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif test.reducer == nil {\n\t\t\t\ttest.reducer = func(pipe <-chan int, cancel func(error)) {\n\t\t\t\t\tfor item := range pipe {\n\t\t\t\t\t\tatomic.AddUint32(&value, uint32(item))\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\terr := MapReduceVoid(func(source chan<- int) {\n\t\t\t\tfor i := 1; i < 5; i++ {\n\t\t\t\t\tsource <- i\n\t\t\t\t}\n\t\t\t}, test.mapper, test.reducer)\n\n\t\t\tassert.Equal(t, test.expectErr, err)\n\t\t\tif err == nil {\n\t\t\t\tassert.Equal(t, test.expectValue, atomic.LoadUint32(&value))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMapReduceVoidWithDelay(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar result []int\n\terr := MapReduceVoid(func(source chan<- int) {\n\t\tsource <- 0\n\t\tsource <- 1\n\t}, func(i int, writer Writer[int], cancel func(error)) {\n\t\tif i == 0 {\n\t\t\ttime.Sleep(time.Millisecond * 50)\n\t\t}\n\t\twriter.Write(i)\n\t}, func(pipe <-chan int, cancel func(error)) {\n\t\tfor item := range pipe {\n\t\t\ti := item\n\t\t\tresult = append(result, i)\n\t\t}\n\t})\n\tassert.Nil(t, err)\n\tassert.Equal(t, 2, len(result))\n\tassert.Equal(t, 1, result[0])\n\tassert.Equal(t, 0, result[1])\n}\n\nfunc TestMapReducePanic(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tassert.Panics(t, func() {\n\t\t_, _ = MapReduce(func(source chan<- int) {\n\t\t\tsource <- 0\n\t\t\tsource <- 1\n\t\t}, func(i int, writer Writer[int], cancel func(error)) {\n\t\t\twriter.Write(i)\n\t\t}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\tfor range pipe {\n\t\t\t\tpanic(\"panic\")\n\t\t\t}\n\t\t})\n\t})\n}\n\nfunc TestMapReducePanicOnce(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tassert.Panics(t, func() {\n\t\t_, _ = MapReduce(func(source chan<- int) {\n...\n", "needle_token_start": 664, "needle_token_end": 841, "code_context_ntokens": 15064, "output": ["```go\nfunc NewGaugeVec(cfg *GaugeVecOpts) GaugeVec {\n\tif cfg == nil {\n\t\treturn nil\n\t}\n\n\tvec := prom.NewGaugeVec(prom.GaugeOpts{\n\t\tNamespace: cfg.Namespace,\n\t\tSubsystem: cfg.Subsystem,\n\t\tName:      cfg.Name,\n\t\tHelp:      cfg.Help,\n\t}, cfg.Labels)\n\tprom.MustRegister(vec)\n\tgv := &promGaugeVec{\n\t\tgauge: vec,\n\t}\n\tproc.AddShutdownListener(func() {\n\t\tgv.close()\n\t})\n\n\treturn gv\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "newGoogleBreaker", "language": "go", "path": "core/breaker/googlebreaker.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function creates an instance of a circuit breaker based on the Google SRE guidelines for client-side throttling. This circuit breaker helps manage the flow of requests to ensure that a system remains stable and responsive under high load.\n2. **Input**: There are no direct inputs as it is a constructor function.\n3. **Output**: Returns an instance of the circuit breaker, initialized with default parameters.\n4. **Procedure**: \n   - Calculates the duration of each bucket by dividing the total window duration by the number of buckets.\n   - Initializes a rolling window with the calculated bucket duration, where each bucket can track the number of accepted and rejected requests.\n   - Sets the initial multiplier for the probability of accepting a request based on the system's performance and initializes the probability calculator.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/stores/mon/collection.go\npackage mon\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/breaker\"\n\t\"github.com/zeromicro/go-zero/core/timex\"\n\t\"go.mongodb.org/mongo-driver/mongo\"\n\tmopt \"go.mongodb.org/mongo-driver/mongo/options\"\n\t\"go.mongodb.org/mongo-driver/x/mongo/driver/session\"\n)\n\nconst (\n\tdefaultSlowThreshold = time.Millisecond * 500\n\t// spanName is the span name of the mongo calls.\n\tspanName = \"mongo\"\n\n\t// mongodb method names\n\taggregate              = \"Aggregate\"\n\tbulkWrite              = \"BulkWrite\"\n\tcountDocuments         = \"CountDocuments\"\n\tdeleteMany             = \"DeleteMany\"\n\tdeleteOne              = \"DeleteOne\"\n\tdistinct               = \"Distinct\"\n\testimatedDocumentCount = \"EstimatedDocumentCount\"\n\tfind                   = \"Find\"\n\tfindOne                = \"FindOne\"\n\tfindOneAndDelete       = \"FindOneAndDelete\"\n\tfindOneAndReplace      = \"FindOneAndReplace\"\n\tfindOneAndUpdate       = \"FindOneAndUpdate\"\n\tinsertMany             = \"InsertMany\"\n\tinsertOne              = \"InsertOne\"\n\treplaceOne             = \"ReplaceOne\"\n\tupdateByID             = \"UpdateByID\"\n\tupdateMany             = \"UpdateMany\"\n\tupdateOne              = \"UpdateOne\"\n)\n\n// ErrNotFound is an alias of mongo.ErrNoDocuments\nvar ErrNotFound = mongo.ErrNoDocuments\n\ntype (\n\t// Collection defines a MongoDB collection.\n\tCollection interface {\n\t\t// Aggregate executes an aggregation pipeline.\n\t\tAggregate(ctx context.Context, pipeline any, opts ...*mopt.AggregateOptions) (\n\t\t\t*mongo.Cursor, error)\n\t\t// BulkWrite performs a bulk write operation.\n\t\tBulkWrite(ctx context.Context, models []mongo.WriteModel, opts ...*mopt.BulkWriteOptions) (\n\t\t\t*mongo.BulkWriteResult, error)\n\t\t// Clone creates a copy of this collection with the same settings.\n\t\tClone(opts ...*mopt.CollectionOptions) (*mongo.Collection, error)\n\t\t// CountDocuments returns the number of documents in the collection that match the filter.\n\t\tCountDocuments(ctx context.Context, filter any, opts ...*mopt.CountOptions) (int64, error)\n\t\t// Database returns the database that this collection is a part of.\n\t\tDatabase() *mongo.Database\n\t\t// DeleteMany deletes documents from the collection that match the filter.\n\t\tDeleteMany(ctx context.Context, filter any, opts ...*mopt.DeleteOptions) (\n\t\t\t*mongo.DeleteResult, error)\n\t\t// DeleteOne deletes at most one document from the collection that matches the filter.\n\t\tDeleteOne(ctx context.Context, filter any, opts ...*mopt.DeleteOptions) (\n\t\t\t*mongo.DeleteResult, error)\n\t\t// Distinct returns a list of distinct values for the given key across the collection.\n\t\tDistinct(ctx context.Context, fieldName string, filter any,\n\t\t\topts ...*mopt.DistinctOptions) ([]any, error)\n\t\t// Drop drops this collection from database.\n\t\tDrop(ctx context.Context) error\n\t\t// EstimatedDocumentCount returns an estimate of the count of documents in a collection\n\t\t// using collection metadata.\n\t\tEstimatedDocumentCount(ctx context.Context, opts ...*mopt.EstimatedDocumentCountOptions) (int64, error)\n\t\t// Find finds the documents matching the provided filter.\n\t\tFind(ctx context.Context, filter any, opts ...*mopt.FindOptions) (*mongo.Cursor, error)\n\t\t// FindOne returns up to one document that matches the provided filter.\n\t\tFindOne(ctx context.Context, filter any, opts ...*mopt.FindOneOptions) (\n\t\t\t*mongo.SingleResult, error)\n\t\t// FindOneAndDelete returns at most one document that matches the filter. If the filter\n\t\t// matches multiple documents, only the first document is deleted.\n\t\tFindOneAndDelete(ctx context.Context, filter any, opts ...*mopt.FindOneAndDeleteOptions) (\n\t\t\t*mongo.SingleResult, error)\n\t\t// FindOneAndReplace returns at most one document that matches the filter. If the filter\n\t\t// matches multiple documents, FindOneAndReplace returns the first document in the\n\t\t// collection that matches the filter.\n\t\tFindOneAndReplace(ctx context.Context, filter, replacement any,\n\t\t\topts ...*mopt.FindOneAndReplaceOptions) (*mongo.SingleResult, error)\n\t\t// FindOneAndUpdate returns at most one document that matches the filter. If the filter\n\t\t// matches multiple documents, FindOneAndUpdate returns the first document in the\n\t\t// collection that matches the filter.\n\t\tFindOneAndUpdate(ctx context.Context, filter, update any,\n\t\t\topts ...*mopt.FindOneAndUpdateOptions) (*mongo.SingleResult, error)\n\t\t// Indexes returns the index view for this collection.\n\t\tIndexes() mongo.IndexView\n\t\t// InsertMany inserts the provided documents.\n\t\tInsertMany(ctx context.Context, documents []any, opts ...*mopt.InsertManyOptions) (\n\t\t\t*mongo.InsertManyResult, error)\n\t\t// InsertOne inserts the provided document.\n\t\tInsertOne(ctx context.Context, document any, opts ...*mopt.InsertOneOptions) (\n\t\t\t*mongo.InsertOneResult, error)\n\t\t// ReplaceOne replaces at most one document that matches the filter.\n\t\tReplaceOne(ctx context.Context, filter, replacement any,\n\t\t\topts ...*mopt.ReplaceOptions) (*mongo.UpdateResult, error)\n\t\t// UpdateByID updates a single document matching the provided filter.\n\t\tUpdateByID(ctx context.Context, id, update any,\n\t\t\topts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)\n\t\t// UpdateMany updates the provided documents.\n\t\tUpdateMany(ctx context.Context, filter, update any,\n\t\t\topts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)\n\t\t// UpdateOne updates a single document matching the provided filter.\n\t\tUpdateOne(ctx context.Context, filter, update any,\n\t\t\topts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)\n\t\t// Watch returns a change stream cursor used to receive notifications of changes to the collection.\n\t\tWatch(ctx context.Context, pipeline any, opts ...*mopt.ChangeStreamOptions) (\n\t\t\t*mongo.ChangeStream, error)\n\t}\n\n\tdecoratedCollection struct {\n\t\t*mongo.Collection\n\t\tname string\n\t\tbrk  breaker.Breaker\n\t}\n\n\tkeepablePromise struct {\n\t\tpromise breaker.Promise\n\t\tlog     func(error)\n\t}\n)\n\nfunc newCollection(collection *mongo.Collection, brk breaker.Breaker) Collection {\n\treturn &decoratedCollection{\n\t\tCollection: collection,\n\t\tname:       collection.Name(),\n\t\tbrk:        brk,\n\t}\n}\n\nfunc (c *decoratedCollection) Aggregate(ctx context.Context, pipeline any,\n\topts ...*mopt.AggregateOptions) (cur *mongo.Cursor, err error) {\n\tctx, span := startSpan(ctx, aggregate)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n...\n// Path: core/breaker/googlebreaker.go\npackage breaker\n\nimport (\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/collection\"\n\t\"github.com/zeromicro/go-zero/core/mathx\"\n)\n\nconst (\n\t// 250ms for bucket duration\n\twindow                    = time.Second * 10\n\tbuckets                   = 40\n\tmaxFailBucketsToDecreaseK = 30\n\tminBucketsToSpeedUp       = 3\n\tk                         = 1.5\n\tminK                      = 1.1\n\trecoveryK                 = 3 - k\n\tprotection                = 5\n)\n\n// googleBreaker is a netflixBreaker pattern from google.\n// see Client-Side Throttling section in https://landing.google.com/sre/sre-book/chapters/handling-overload/\ntype (\n\tgoogleBreaker struct {\n\t\tk     float64\n\t\tstat  *collection.RollingWindow[int64, *bucket]\n\t\tproba *mathx.Proba\n\t}\n\n\twindowResult struct {\n\t\taccepts        int64\n\t\ttotal          int64\n\t\tfailingBuckets int64\n\t\tworkingBuckets int64\n\t}\n)\n\n\nfunc newGoogleBreaker() *googleBreaker {\n\tbucketDuration := time.Duration(int64(window) / int64(buckets))\n\tst := collection.NewRollingWindow[int64, *bucket](func() *bucket {\n\t\treturn new(bucket)\n\t}, buckets, bucketDuration)\n\treturn &googleBreaker{\n\t\tstat:  st,\n\t\tk:     k,\n\t\tproba: mathx.NewProba(),\n\t}\n}\n\nfunc (b *googleBreaker) accept() error {\n\tvar w float64\n\thistory := b.history()\n\tif history.failingBuckets >= minBucketsToSpeedUp {\n\t\tw = b.k - float64(history.failingBuckets-1)*(b.k-minK)/maxFailBucketsToDecreaseK\n\t\tw = mathx.AtLeast(w, minK)\n\t} else {\n\t\tw = b.k\n\t}\n\tweightedAccepts := w * float64(history.accepts)\n\t// https://landing.google.com/sre/sre-book/chapters/handling-overload/#eq2101\n\t// for better performance, no need to care about the negative ratio\n\tdropRatio := (float64(history.total-protection) - weightedAccepts) / float64(history.total+1)\n\tif dropRatio <= 0 {\n\t\treturn nil\n\t}\n\n\t// If we have more than 2 working buckets, we are in recovery mode,\n\t// the latest bucket is the current one, so we ignore it.\n\tif history.workingBuckets >= minBucketsToSpeedUp {\n\t\tdropRatio /= recoveryK\n\t}\n\n\tif b.proba.TrueOnProba(dropRatio) {\n\t\treturn ErrServiceUnavailable\n\t}\n\n\treturn nil\n}\n\nfunc (b *googleBreaker) allow() (internalPromise, error) {\n\tif err := b.accept(); err != nil {\n\t\tb.markDrop()\n\t\treturn nil, err\n\t}\n\n\treturn googlePromise{\n\t\tb: b,\n\t}, nil\n}\n\nfunc (b *googleBreaker) doReq(req func() error, fallback Fallback, acceptable Acceptable) error {\n\tif err := b.accept(); err != nil {\n\t\tb.markDrop()\n\t\tif fallback != nil {\n\t\t\treturn fallback(err)\n\t\t}\n\n\t\treturn err\n\t}\n\n\tvar succ bool\n\tdefer func() {\n\t\t// if req() panic, success is false, mark as failure\n\t\tif succ {\n\t\t\tb.markSuccess()\n\t\t} else {\n\t\t\tb.markFailure()\n\t\t}\n\t}()\n\n\terr := req()\n\tif acceptable(err) {\n\t\tsucc = true\n\t}\n\n\treturn err\n}\n\nfunc (b *googleBreaker) markDrop() {\n\tb.stat.Add(drop)\n}\n\nfunc (b *googleBreaker) markFailure() {\n\tb.stat.Add(fail)\n}\n\nfunc (b *googleBreaker) markSuccess() {\n\tb.stat.Add(success)\n}\n\nfunc (b *googleBreaker) history() windowResult {\n\tvar result windowResult\n\n\tb.stat.Reduce(func(b *bucket) {\n\t\tresult.accepts += b.Success\n\t\tresult.total += b.Sum\n\t\tif b.Failure > 0 {\n\t\t\tresult.workingBuckets = 0\n\t\t} else if b.Success > 0 {\n\t\t\tresult.workingBuckets++\n\t\t}\n\t\tif b.Drop > 0 && b.Failure > 0 {\n\t\t\tresult.failingBuckets++\n\t\t} else {\n\t\t\tresult.failingBuckets = 0\n\t\t}\n\t})\n\n\treturn result\n}\n\ntype googlePromise struct {\n\tb *googleBreaker\n}\n\nfunc (p googlePromise) Accept() {\n\tp.b.markSuccess()\n}\n\nfunc (p googlePromise) Reject() {\n\tp.b.markFailure()\n}\n\n// Path: core/breaker/googlebreaker_test.go\npackage breaker\n\nimport (\n\t\"errors\"\n\t\"math/rand\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/collection\"\n\t\"github.com/zeromicro/go-zero/core/mathx\"\n\t\"github.com/zeromicro/go-zero/core/stat\"\n)\n\nconst (\n\ttestBuckets  = 10\n\ttestInterval = time.Millisecond * 10\n)\n\nfunc init() {\n\tstat.SetReporter(nil)\n}\n\nfunc getGoogleBreaker() *googleBreaker {\n\tst := collection.NewRollingWindow[int64, *bucket](func() *bucket {\n\t\treturn new(bucket)\n\t}, testBuckets, testInterval)\n\treturn &googleBreaker{\n\t\tstat:  st,\n\t\tk:     5,\n\t\tproba: mathx.NewProba(),\n\t}\n}\n\nfunc markSuccessWithDuration(b *googleBreaker, count int, sleep time.Duration) {\n\tfor i := 0; i < count; i++ {\n\t\tb.markSuccess()\n\t\ttime.Sleep(sleep)\n\t}\n}\n\nfunc markFailedWithDuration(b *googleBreaker, count int, sleep time.Duration) {\n\tfor i := 0; i < count; i++ {\n\t\tb.markFailure()\n\t\ttime.Sleep(sleep)\n\t}\n}\n\nfunc TestGoogleBreakerClose(t *testing.T) {\n\tb := getGoogleBreaker()\n\tmarkSuccess(b, 80)\n\tassert.Nil(t, b.accept())\n\tmarkSuccess(b, 120)\n\tassert.Nil(t, b.accept())\n}\n\nfunc TestGoogleBreakerOpen(t *testing.T) {\n\tb := getGoogleBreaker()\n\tmarkSuccess(b, 10)\n\tassert.Nil(t, b.accept())\n\tmarkFailed(b, 100000)\n\ttime.Sleep(testInterval * 2)\n\tverify(t, func() bool {\n\t\treturn b.accept() != nil\n\t})\n}\n\nfunc TestGoogleBreakerRecover(t *testing.T) {\n\tst := collection.NewRollingWindow[int64, *bucket](func() *bucket {\n\t\treturn new(bucket)\n\t}, testBuckets*2, testInterval)\n\tb := &googleBreaker{\n\t\tstat:  st,\n\t\tk:     k,\n\t\tproba: mathx.NewProba(),\n\t}\n\tfor i := 0; i < testBuckets; i++ {\n\t\tfor j := 0; j < 100; j++ {\n\t\t\tb.stat.Add(1)\n\t\t}\n\t\ttime.Sleep(testInterval)\n\t}\n\tfor i := 0; i < testBuckets; i++ {\n\t\tfor j := 0; j < 100; j++ {\n\t\t\tb.stat.Add(0)\n\t\t}\n\t\ttime.Sleep(testInterval)\n\t}\n\tverify(t, func() bool {\n\t\treturn b.accept() == nil\n\t})\n}\n\nfunc TestGoogleBreakerFallback(t *testing.T) {\n\tb := getGoogleBreaker()\n\tmarkSuccess(b, 1)\n\tassert.Nil(t, b.accept())\n\tmarkFailed(b, 10000)\n\ttime.Sleep(testInterval * 2)\n\tverify(t, func() bool {\n\t\treturn b.doReq(func() error {\n\t\t\treturn errors.New(\"any\")\n\t\t}, func(err error) error {\n\t\t\treturn nil\n\t\t}, defaultAcceptable) == nil\n\t})\n}\n\nfunc TestGoogleBreakerReject(t *testing.T) {\n\tb := getGoogleBreaker()\n\tmarkSuccess(b, 100)\n\tassert.Nil(t, b.accept())\n\tmarkFailed(b, 10000)\n\ttime.Sleep(testInterval)\n\tassert.Equal(t, ErrServiceUnavailable, b.doReq(func() error {\n\t\treturn ErrServiceUnavailable\n\t}, nil, defaultAcceptable))\n}\n\nfunc TestGoogleBreakerAcceptable(t *testing.T) {\n\tb := getGoogleBreaker()\n\terrAcceptable := errors.New(\"any\")\n\tassert.Equal(t, errAcceptable, b.doReq(func() error {\n\t\treturn errAcceptable\n\t}, nil, func(err error) bool {\n\t\treturn errors.Is(err, errAcceptable)\n\t}))\n}\n\nfunc TestGoogleBreakerNotAcceptable(t *testing.T) {\n\tb := getGoogleBreaker()\n\terrAcceptable := errors.New(\"any\")\n\tassert.Equal(t, errAcceptable, b.doReq(func() error {\n\t\treturn errAcceptable\n\t}, nil, func(err error) bool {\n\t\treturn !errors.Is(err, errAcceptable)\n\t}))\n}\n\nfunc TestGoogleBreakerPanic(t *testing.T) {\n\tb := getGoogleBreaker()\n\tassert.Panics(t, func() {\n\t\t_ = b.doReq(func() error {\n\t\t\tpanic(\"fail\")\n\t\t}, nil, defaultAcceptable)\n\t})\n}\n\nfunc TestGoogleBreakerHalfOpen(t *testing.T) {\n\tb := getGoogleBreaker()\n\tassert.Nil(t, b.accept())\n\tt.Run(\"accept single failed/accept\", func(t *testing.T) {\n\t\tmarkFailed(b, 10000)\n\t\ttime.Sleep(testInterval * 2)\n\t\tverify(t, func() bool {\n\t\t\treturn b.accept() != nil\n\t\t})\n\t})\n\tt.Run(\"accept single failed/allow\", func(t *testing.T) {\n\t\tmarkFailed(b, 10000)\n\t\ttime.Sleep(testInterval * 2)\n\t\tverify(t, func() bool {\n\t\t\t_, err := b.allow()\n\t\t\treturn err != nil\n\t\t})\n\t})\n\ttime.Sleep(testInterval * testBuckets)\n\tt.Run(\"accept single succeed\", func(t *testing.T) {\n\t\tassert.Nil(t, b.accept())\n\t\tmarkSuccess(b, 10000)\n\t\tverify(t, func() bool {\n\t\t\treturn b.accept() == nil\n\t\t})\n\t})\n}\n\nfunc TestGoogleBreakerSelfProtection(t *testing.T) {\n\tt.Run(\"total request < 100\", func(t *testing.T) {\n\t\tb := getGoogleBreaker()\n\t\tmarkFailed(b, 4)\n\t\ttime.Sleep(testInterval)\n\t\tassert.Nil(t, b.accept())\n\t})\n\tt.Run(\"total request > 100, total < 2 * success\", func(t *testing.T) {\n\t\tb := getGoogleBreaker()\n\t\tsize := rand.Intn(10000)\n\t\taccepts := size + 1\n\t\tmarkSuccess(b, accepts)\n\t\tmarkFailed(b, size-accepts)\n\t\tassert.Nil(t, b.accept())\n\t})\n}\n\nfunc TestGoogleBreakerHistory(t *testing.T) {\n\tsleep := testInterval\n\tt.Run(\"accepts == total\", func(t *testing.T) {\n\t\tb := getGoogleBreaker()\n\t\tmarkSuccessWithDuration(b, 10, sleep/2)\n\t\tresult := b.history()\n\t\tassert.Equal(t, int64(10), result.accepts)\n\t\tassert.Equal(t, int64(10), result.total)\n\t})\n\n\tt.Run(\"fail == total\", func(t *testing.T) {\n\t\tb := getGoogleBreaker()\n\t\tmarkFailedWithDuration(b, 10, sleep/2)\n\t\tresult := b.history()\n\t\tassert.Equal(t, int64(0), result.accepts)\n\t\tassert.Equal(t, int64(10), result.total)\n\t})\n\n\tt.Run(\"accepts = 1/2 * total, fail = 1/2 * total\", func(t *testing.T) {\n\t\tb := getGoogleBreaker()\n\t\tmarkFailedWithDuration(b, 5, sleep/2)\n\t\tmarkSuccessWithDuration(b, 5, sleep/2)\n\t\tresult := b.history()\n\t\tassert.Equal(t, int64(5), result.accepts)\n\t\tassert.Equal(t, int64(10), result.total)\n\t})\n\n\tt.Run(\"auto reset rolling counter\", func(t *testing.T) {\n\t\tb := getGoogleBreaker()\n\t\ttime.Sleep(testInterval * testBuckets)\n\t\tresult := b.history()\n\t\tassert.Equal(t, int64(0), result.accepts)\n\t\tassert.Equal(t, int64(0), result.total)\n\t})\n}\n\nfunc BenchmarkGoogleBreakerAllow(b *testing.B) {\n\tbreaker := getGoogleBreaker()\n\tb.ResetTimer()\n\tfor i := 0; i <= b.N; i++ {\n\t\t_ = breaker.accept()\n\t\tif i%2 == 0 {\n\t\t\tbreaker.markSuccess()\n\t\t} else {\n\t\t\tbreaker.markFailure()\n\t\t}\n\t}\n}\n\nfunc BenchmarkGoogleBreakerDoReq(b *testing.B) {\n\tbreaker := getGoogleBreaker()\n\tb.ResetTimer()\n\tfor i := 0; i <= b.N; i++ {\n\t\t_ = breaker.doReq(func() error {\n\t\t\treturn nil\n\t\t}, nil, defaultAcceptable)\n\t}\n}\n\nfunc markSuccess(b *googleBreaker, count int) {\n\tfor i := 0; i < count; i++ {\n\t\tp, err := b.allow()\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tp.Accept()\n\t}\n}\n\nfunc markFailed(b *googleBreaker, count int) {\n\tfor i := 0; i < count; i++ {\n\t\tp, err := b.allow()\n\t\tif err == nil {\n\t\t\tp.Reject()\n\t\t}\n\t}\n}\n\n// Path: core/breaker/nopbreaker.go\npackage breaker\n\nimport \"context\"\n\nconst nopBreakerName = \"nopBreaker\"\n\ntype nopBreaker struct{}\n\n// NopBreaker returns a breaker that never trigger breaker circuit.\nfunc NopBreaker() Breaker {\n\treturn nopBreaker{}\n}\n\nfunc (b nopBreaker) Name() string {\n\treturn nopBreakerName\n}\n\nfunc (b nopBreaker) Allow() (Promise, error) {\n\treturn nopPromise{}, nil\n}\n\nfunc (b nopBreaker) AllowCtx(_ context.Context) (Promise, error) {\n\treturn nopPromise{}, nil\n}\n\nfunc (b nopBreaker) Do(req func() error) error {\n\treturn req()\n}\n\nfunc (b nopBreaker) DoCtx(_ context.Context, req func() error) error {\n\treturn req()\n}\n\nfunc (b nopBreaker) DoWithAcceptable(req func() error, _ Acceptable) error {\n\treturn req()\n}\n\nfunc (b nopBreaker) DoWithAcceptableCtx(_ context.Context, req func() error, _ Acceptable) error {\n\treturn req()\n}\n\nfunc (b nopBreaker) DoWithFallback(req func() error, _ Fallback) error {\n\treturn req()\n}\n\nfunc (b nopBreaker) DoWithFallbackCtx(_ context.Context, req func() error, _ Fallback) error {\n\treturn req()\n}\n\nfunc (b nopBreaker) DoWithFallbackAcceptable(req func() error, _ Fallback, _ Acceptable) error {\n\treturn req()\n}\n\nfunc (b nopBreaker) DoWithFallbackAcceptableCtx(_ context.Context, req func() error,\n\t_ Fallback, _ Acceptable) error {\n\treturn req()\n}\n\ntype nopPromise struct{}\n\nfunc (p nopPromise) Accept() {\n}\n\nfunc (p nopPromise) Reject(_ string) {\n}\n\n// Path: core/breaker/nopbreaker_test.go\npackage breaker\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestNopBreaker(t *testing.T) {\n\tb := NopBreaker()\n\tassert.Equal(t, nopBreakerName, b.Name())\n\tp, err := b.Allow()\n\tassert.Nil(t, err)\n\tp, err = b.AllowCtx(context.Background())\n\tassert.Nil(t, err)\n\tp.Accept()\n\tfor i := 0; i < 1000; i++ {\n\t\tp, err := b.Allow()\n\t\tassert.Nil(t, err)\n\t\tp.Reject(\"any\")\n\t}\n\tassert.Nil(t, b.Do(func() error {\n\t\treturn nil\n\t}))\n\tassert.Nil(t, b.DoCtx(context.Background(), func() error {\n\t\treturn nil\n\t}))\n\tassert.Nil(t, b.DoWithAcceptable(func() error {\n\t\treturn nil\n\t}, defaultAcceptable))\n\tassert.Nil(t, b.DoWithAcceptableCtx(context.Background(), func() error {\n\t\treturn nil\n\t}, defaultAcceptable))\n\terrDummy := errors.New(\"any\")\n\tassert.Equal(t, errDummy, b.DoWithFallback(func() error {\n\t\treturn errDummy\n\t}, func(err error) error {\n\t\treturn nil\n\t}))\n\tassert.Equal(t, errDummy, b.DoWithFallbackCtx(context.Background(), func() error {\n\t\treturn errDummy\n\t}, func(err error) error {\n\t\treturn nil\n\t}))\n\tassert.Equal(t, errDummy, b.DoWithFallbackAcceptable(func() error {\n\t\treturn errDummy\n\t}, func(err error) error {\n\t\treturn nil\n\t}, defaultAcceptable))\n\tassert.Equal(t, errDummy, b.DoWithFallbackAcceptableCtx(context.Background(), func() error {\n\t\treturn errDummy\n\t}, func(err error) error {\n\t\treturn nil\n\t}, defaultAcceptable))\n}\n\n// Path: core/cmdline/input.go\npackage cmdline\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n)\n\n// EnterToContinue let stdin waiting for an enter key to continue.\nfunc EnterToContinue() {\n\tfmt.Print(\"Press 'Enter' to continue...\")\n\tbufio.NewReader(os.Stdin).ReadBytes('\\n')\n}\n\n// ReadLine shows prompt to stdout and read a line from stdin.\nfunc ReadLine(prompt string) string {\n\tfmt.Print(prompt)\n\tinput, _ := bufio.NewReader(os.Stdin).ReadString('\\n')\n\treturn strings.TrimSpace(input)\n}\n\n// Path: core/cmdline/input_test.go\npackage cmdline\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/iox\"\n\t\"github.com/zeromicro/go-zero/core/lang\"\n)\n\nfunc TestEnterToContinue(t *testing.T) {\n\trestore, err := iox.RedirectInOut()\n\tassert.Nil(t, err)\n\tdefer restore()\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfmt.Println()\n\t}()\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tEnterToContinue()\n\t}()\n\n\twait := make(chan lang.PlaceholderType)\n\tgo func() {\n\t\twg.Wait()\n\t\tclose(wait)\n\t}()\n\n\tselect {\n\tcase <-time.After(time.Second):\n\t\tt.Error(\"timeout\")\n\tcase <-wait:\n\t}\n}\n\nfunc TestReadLine(t *testing.T) {\n\tr, w, err := os.Pipe()\n\tassert.Nil(t, err)\n\tow := os.Stdout\n\tos.Stdout = w\n\tor := os.Stdin\n\tos.Stdin = r\n\tdefer func() {\n\t\tos.Stdin = or\n\t\tos.Stdout = ow\n\t}()\n\n\tconst message = \"hello\"\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfmt.Println(message)\n\t}()\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tinput := ReadLine(\"\")\n\t\tassert.Equal(t, message, input)\n\t}()\n\n\twait := make(chan lang.PlaceholderType)\n\tgo func() {\n\t\twg.Wait()\n\t\tclose(wait)\n\t}()\n\n\tselect {\n\tcase <-time.After(time.Second):\n\t\tt.Error(\"timeout\")\n\tcase <-wait:\n\t}\n}\n\n// Path: core/codec/aesecb.go\npackage codec\n\nimport (\n\t\"bytes\"\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"encoding/base64\"\n\t\"errors\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n)\n\n// ErrPaddingSize indicates bad padding size.\nvar ErrPaddingSize = errors.New(\"padding size error\")\n\ntype ecb struct {\n\tb         cipher.Block\n\tblockSize int\n}\n\nfunc newECB(b cipher.Block) *ecb {\n\treturn &ecb{\n\t\tb:         b,\n\t\tblockSize: b.BlockSize(),\n\t}\n}\n\ntype ecbEncrypter ecb\n\n// NewECBEncrypter returns an ECB encrypter.\nfunc NewECBEncrypter(b cipher.Block) cipher.BlockMode {\n\treturn (*ecbEncrypter)(newECB(b))\n}\n\n// BlockSize returns the mode's block size.\nfunc (x *ecbEncrypter) BlockSize() int { return x.blockSize }\n\n// CryptBlocks encrypts a number of blocks. The length of src must be a multiple of\n// the block size. Dst and src must overlap entirely or not at all.\nfunc (x *ecbEncrypter) CryptBlocks(dst, src []byte) {\n\tif len(src)%x.blockSize != 0 {\n\t\tlogx.Error(\"crypto/cipher: input not full blocks\")\n\t\treturn\n\t}\n\tif len(dst) < len(src) {\n\t\tlogx.Error(\"crypto/cipher: output smaller than input\")\n\t\treturn\n\t}\n\n\tfor len(src) > 0 {\n\t\tx.b.Encrypt(dst, src[:x.blockSize])\n\t\tsrc = src[x.blockSize:]\n\t\tdst = dst[x.blockSize:]\n\t}\n}\n\ntype ecbDecrypter ecb\n\n// NewECBDecrypter returns an ECB decrypter.\nfunc NewECBDecrypter(b cipher.Block) cipher.BlockMode {\n\treturn (*ecbDecrypter)(newECB(b))\n}\n\n// BlockSize returns the mode's block size.\nfunc (x *ecbDecrypter) BlockSize() int {\n\treturn x.blockSize\n}\n\n// CryptBlocks decrypts a number of blocks. The length of src must be a multiple of\n// the block size. Dst and src must overlap entirely or not at all.\nfunc (x *ecbDecrypter) CryptBlocks(dst, src []byte) {\n\tif len(src)%x.blockSize != 0 {\n\t\tlogx.Error(\"crypto/cipher: input not full blocks\")\n\t\treturn\n\t}\n\tif len(dst) < len(src) {\n\t\tlogx.Error(\"crypto/cipher: output smaller than input\")\n\t\treturn\n\t}\n\n\tfor len(src) > 0 {\n\t\tx.b.Decrypt(dst, src[:x.blockSize])\n\t\tsrc = src[x.blockSize:]\n\t\tdst = dst[x.blockSize:]\n\t}\n}\n\n// EcbDecrypt decrypts src with the given key.\nfunc EcbDecrypt(key, src []byte) ([]byte, error) {\n\tblock, err := aes.NewCipher(key)\n\tif err != nil {\n\t\tlogx.Errorf(\"Decrypt key error: % x\", key)\n\t\treturn nil, err\n\t}\n\n\tdecrypter := NewECBDecrypter(block)\n\tdecrypted := make([]byte, len(src))\n\tdecrypter.CryptBlocks(decrypted, src)\n\n\treturn pkcs5Unpadding(decrypted, decrypter.BlockSize())\n}\n\n// EcbDecryptBase64 decrypts base64 encoded src with the given base64 encoded key.\n// The returned string is also base64 encoded.\nfunc EcbDecryptBase64(key, src string) (string, error) {\n\tkeyBytes, err := getKeyBytes(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tencryptedBytes, err := base64.StdEncoding.DecodeString(src)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tdecryptedBytes, err := EcbDecrypt(keyBytes, encryptedBytes)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn base64.StdEncoding.EncodeToString(decryptedBytes), nil\n}\n\n// EcbEncrypt encrypts src with the given key.\nfunc EcbEncrypt(key, src []byte) ([]byte, error) {\n\tblock, err := aes.NewCipher(key)\n\tif err != nil {\n\t\tlogx.Errorf(\"Encrypt key error: % x\", key)\n\t\treturn nil, err\n\t}\n\n\tpadded := pkcs5Padding(src, block.BlockSize())\n\tcrypted := make([]byte, len(padded))\n\tencrypter := NewECBEncrypter(block)\n\tencrypter.CryptBlocks(crypted, padded)\n\n\treturn crypted, nil\n}\n\n// EcbEncryptBase64 encrypts base64 encoded src with the given base64 encoded key.\n// The returned string is also base64 encoded.\nfunc EcbEncryptBase64(key, src string) (string, error) {\n\tkeyBytes, err := getKeyBytes(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tsrcBytes, err := base64.StdEncoding.DecodeString(src)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tencryptedBytes, err := EcbEncrypt(keyBytes, srcBytes)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn base64.StdEncoding.EncodeToString(encryptedBytes), nil\n}\n\nfunc getKeyBytes(key string) ([]byte, error) {\n\tif len(key) <= 32 {\n\t\treturn []byte(key), nil\n\t}\n\n\tkeyBytes, err := base64.StdEncoding.DecodeString(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn keyBytes, nil\n}\n\nfunc pkcs5Padding(ciphertext []byte, blockSize int) []byte {\n\tpadding := blockSize - len(ciphertext)%blockSize\n\tpadtext := bytes.Repeat([]byte{byte(padding)}, padding)\n\treturn append(ciphertext, padtext...)\n}\n\nfunc pkcs5Unpadding(src []byte, blockSize int) ([]byte, error) {\n\tlength := len(src)\n\tunpadding := int(src[length-1])\n\tif unpadding >= length || unpadding > blockSize {\n\t\treturn nil, ErrPaddingSize\n\t}\n\n\treturn src[:length-unpadding], nil\n}\n\n// Path: core/codec/aesecb_test.go\npackage codec\n\nimport (\n\t\"crypto/aes\"\n\t\"encoding/base64\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestAesEcb(t *testing.T) {\n\tvar (\n\t\tkey     = []byte(\"q4t7w!z%C*F-JaNdRgUjXn2r5u8x/A?D\")\n\t\tval     = []byte(\"helloworld\")\n\t\tvalLong = []byte(\"helloworldlong..\")\n\t\tbadKey1 = []byte(\"aaaaaaaaa\")\n\t\t// more than 32 chars\n\t\tbadKey2 = []byte(\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\")\n\t)\n\t_, err := EcbEncrypt(badKey1, val)\n\tassert.NotNil(t, err)\n\t_, err = EcbEncrypt(badKey2, val)\n\tassert.NotNil(t, err)\n\tdst, err := EcbEncrypt(key, val)\n\tassert.Nil(t, err)\n\t_, err = EcbDecrypt(badKey1, dst)\n\tassert.NotNil(t, err)\n\t_, err = EcbDecrypt(badKey2, dst)\n\tassert.NotNil(t, err)\n\t_, err = EcbDecrypt(key, val)\n\t// not enough block, just nil\n\tassert.Nil(t, err)\n\tsrc, err := EcbDecrypt(key, dst)\n\tassert.Nil(t, err)\n\tassert.Equal(t, val, src)\n\tblock, err := aes.NewCipher(key)\n\tassert.NoError(t, err)\n\tencrypter := NewECBEncrypter(block)\n\tassert.Equal(t, 16, encrypter.BlockSize())\n\tdecrypter := NewECBDecrypter(block)\n\tassert.Equal(t, 16, decrypter.BlockSize())\n\n\tdst = make([]byte, 8)\n\tencrypter.CryptBlocks(dst, val)\n\tfor _, b := range dst {\n\t\tassert.Equal(t, byte(0), b)\n\t}\n\n\tdst = make([]byte, 8)\n\tencrypter.CryptBlocks(dst, valLong)\n\tfor _, b := range dst {\n\t\tassert.Equal(t, byte(0), b)\n\t}\n\n\tdst = make([]byte, 8)\n\tdecrypter.CryptBlocks(dst, val)\n\tfor _, b := range dst {\n\t\tassert.Equal(t, byte(0), b)\n\t}\n\n\tdst = make([]byte, 8)\n\tdecrypter.CryptBlocks(dst, valLong)\n\tfor _, b := range dst {\n\t\tassert.Equal(t, byte(0), b)\n\t}\n\n\t_, err = EcbEncryptBase64(\"cTR0N3dDKkYtSmFOZFJnVWpYbjJyNXU4eC9BP0QK\", \"aGVsbG93b3JsZGxvbmcuLgo=\")\n\tassert.Error(t, err)\n}\n\nfunc TestAesEcbBase64(t *testing.T) {\n\tconst (\n\t\tval     = \"hello\"\n\t\tbadKey1 = \"aaaaaaaaa\"\n\t\t// more than 32 chars\n\t\tbadKey2 = \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n\t)\n\tkey := []byte(\"q4t7w!z%C*F-JaNdRgUjXn2r5u8x/A?D\")\n\tb64Key := base64.StdEncoding.EncodeToString(key)\n\tb64Val := base64.StdEncoding.EncodeToString([]byte(val))\n\t_, err := EcbEncryptBase64(badKey1, val)\n\tassert.NotNil(t, err)\n\t_, err = EcbEncryptBase64(badKey2, val)\n\tassert.NotNil(t, err)\n\t_, err = EcbEncryptBase64(b64Key, val)\n\tassert.NotNil(t, err)\n\tdst, err := EcbEncryptBase64(b64Key, b64Val)\n\tassert.Nil(t, err)\n\t_, err = EcbDecryptBase64(badKey1, dst)\n\tassert.NotNil(t, err)\n\t_, err = EcbDecryptBase64(badKey2, dst)\n\tassert.NotNil(t, err)\n\t_, err = EcbDecryptBase64(b64Key, val)\n\tassert.NotNil(t, err)\n\tsrc, err := EcbDecryptBase64(b64Key, dst)\n\tassert.Nil(t, err)\n\tb, err := base64.StdEncoding.DecodeString(src)\n\tassert.Nil(t, err)\n\tassert.Equal(t, val, string(b))\n}\n\n// Path: core/codec/dh.go\npackage codec\n\nimport (\n\t\"crypto/rand\"\n\t\"errors\"\n\t\"math/big\"\n)\n\n// see https://www.zhihu.com/question/29383090/answer/70435297\n// see https://www.ietf.org/rfc/rfc3526.txt\n// 2048-bit MODP Group\n\nvar (\n\t// ErrInvalidPriKey indicates the invalid private key.\n\tErrInvalidPriKey = errors.New(\"invalid private key\")\n\t// ErrInvalidPubKey indicates the invalid public key.\n\tErrInvalidPubKey = errors.New(\"invalid public key\")\n\t// ErrPubKeyOutOfBound indicates the public key is out of bound.\n\tErrPubKeyOutOfBound = errors.New(\"public key out of bound\")\n\n\tp, _ = new(big.Int).SetString(\"FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF\", 16)\n\tg, _ = new(big.Int).SetString(\"2\", 16)\n\tzero = big.NewInt(0)\n)\n\n// DhKey defines the Diffie-Hellman key.\ntype DhKey struct {\n\tPriKey *big.Int\n\tPubKey *big.Int\n}\n\n// ComputeKey returns a key from public key and private key.\nfunc ComputeKey(pubKey, priKey *big.Int) (*big.Int, error) {\n\tif pubKey == nil {\n\t\treturn nil, ErrInvalidPubKey\n\t}\n\n\tif pubKey.Sign() <= 0 && p.Cmp(pubKey) <= 0 {\n\t\treturn nil, ErrPubKeyOutOfBound\n\t}\n\n\tif priKey == nil {\n\t\treturn nil, ErrInvalidPriKey\n\t}\n\n\treturn new(big.Int).Exp(pubKey, priKey, p), nil\n}\n\n// GenerateKey returns a Diffie-Hellman key.\nfunc GenerateKey() (*DhKey, error) {\n\tvar err error\n\tvar x *big.Int\n\n\tfor {\n\t\tx, err = rand.Int(rand.Reader, p)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif zero.Cmp(x) < 0 {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tkey := new(DhKey)\n\tkey.PriKey = x\n\tkey.PubKey = new(big.Int).Exp(g, x, p)\n\n\treturn key, nil\n}\n\n// NewPublicKey returns a public key from the given bytes.\nfunc NewPublicKey(bs []byte) *big.Int {\n\treturn new(big.Int).SetBytes(bs)\n}\n\n// Bytes returns public key bytes.\nfunc (k *DhKey) Bytes() []byte {\n\tif k.PubKey == nil {\n\t\treturn nil\n\t}\n\n\tbyteLen := (p.BitLen() + 7) >> 3\n\tret := make([]byte, byteLen)\n\tcopyWithLeftPad(ret, k.PubKey.Bytes())\n\n\treturn ret\n}\n\nfunc copyWithLeftPad(dst, src []byte) {\n\tpadBytes := len(dst) - len(src)\n\tfor i := 0; i < padBytes; i++ {\n\t\tdst[i] = 0\n\t}\n\tcopy(dst[padBytes:], src)\n}\n\n// Path: core/codec/dh_test.go\npackage codec\n\nimport (\n\t\"math/big\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestDiffieHellman(t *testing.T) {\n\tkey1, err := GenerateKey()\n\tassert.Nil(t, err)\n\tkey2, err := GenerateKey()\n\tassert.Nil(t, err)\n\n\tpubKey1, err := ComputeKey(key1.PubKey, key2.PriKey)\n\tassert.Nil(t, err)\n\tpubKey2, err := ComputeKey(key2.PubKey, key1.PriKey)\n\tassert.Nil(t, err)\n\n\tassert.Equal(t, pubKey1, pubKey2)\n}\n\nfunc TestDiffieHellman1024(t *testing.T) {\n\told := p\n\tp, _ = new(big.Int).SetString(\"F488FD584E49DBCD20B49DE49107366B336C380D451D0F7C88B31C7C5B2D8EF6F3C923C043F0A55B188D8EBB558CB85D38D334FD7C175743A31D186CDE33212CB52AFF3CE1B1294018118D7C84A70A72D686C40319C807297ACA950CD9969FABD00A509B0246D3083D66A45D419F9C7CBD894B221926BAABA25EC355E92F78C7\", 16)\n\tdefer func() {\n\t\tp = old\n\t}()\n\n\tkey1, err := GenerateKey()\n\tassert.Nil(t, err)\n\tkey2, err := GenerateKey()\n\tassert.Nil(t, err)\n\n\tpubKey1, err := ComputeKey(key1.PubKey, key2.PriKey)\n\tassert.Nil(t, err)\n\tpubKey2, err := ComputeKey(key2.PubKey, key1.PriKey)\n\tassert.Nil(t, err)\n\n\tassert.Equal(t, pubKey1, pubKey2)\n}\n\nfunc TestDiffieHellmanMiddleManAttack(t *testing.T) {\n\tkey1, err := GenerateKey()\n\tassert.Nil(t, err)\n\tkeyMiddle, err := GenerateKey()\n\tassert.Nil(t, err)\n\tkey2, err := GenerateKey()\n\tassert.Nil(t, err)\n\n\tconst aesByteLen = 32\n\tpubKey1, err := ComputeKey(keyMiddle.PubKey, key1.PriKey)\n\tassert.Nil(t, err)\n\tsrc := []byte(`hello, world!`)\n\tencryptedSrc, err := EcbEncrypt(pubKey1.Bytes()[:aesByteLen], src)\n\tassert.Nil(t, err)\n\tpubKeyMiddle, err := ComputeKey(key1.PubKey, keyMiddle.PriKey)\n\tassert.Nil(t, err)\n\tdecryptedSrc, err := EcbDecrypt(pubKeyMiddle.Bytes()[:aesByteLen], encryptedSrc)\n\tassert.Nil(t, err)\n\tassert.Equal(t, string(src), string(decryptedSrc))\n\n\tpubKeyMiddle, err = ComputeKey(key2.PubKey, keyMiddle.PriKey)\n\tassert.Nil(t, err)\n\tencryptedSrc, err = EcbEncrypt(pubKeyMiddle.Bytes()[:aesByteLen], decryptedSrc)\n\tassert.Nil(t, err)\n\tpubKey2, err := ComputeKey(keyMiddle.PubKey, key2.PriKey)\n\tassert.Nil(t, err)\n\tdecryptedSrc, err = EcbDecrypt(pubKey2.Bytes()[:aesByteLen], encryptedSrc)\n\tassert.Nil(t, err)\n\tassert.Equal(t, string(src), string(decryptedSrc))\n}\n\nfunc TestKeyBytes(t *testing.T) {\n\tvar empty DhKey\n\tassert.Equal(t, 0, len(empty.Bytes()))\n\n\tkey, err := GenerateKey()\n\tassert.Nil(t, err)\n\tassert.True(t, len(key.Bytes()) > 0)\n}\n\nfunc TestDHOnErrors(t *testing.T) {\n\tkey, err := GenerateKey()\n\tassert.Nil(t, err)\n\tassert.NotEmpty(t, key.Bytes())\n\t_, err = ComputeKey(key.PubKey, key.PriKey)\n\tassert.NoError(t, err)\n\t_, err = ComputeKey(nil, key.PriKey)\n\tassert.Error(t, err)\n\t_, err = ComputeKey(key.PubKey, nil)\n\tassert.Error(t, err)\n\n\tassert.NotNil(t, NewPublicKey([]byte(\"\")))\n}\n\n// Path: core/codec/gzip.go\npackage codec\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"io\"\n)\n\nconst unzipLimit = 100 * 1024 * 1024 // 100MB\n\n// Gzip compresses bs.\nfunc Gzip(bs []byte) []byte {\n\tvar b bytes.Buffer\n\n\tw := gzip.NewWriter(&b)\n\tw.Write(bs)\n\tw.Close()\n\n\treturn b.Bytes()\n}\n\n// Gunzip uncompresses bs.\nfunc Gunzip(bs []byte) ([]byte, error) {\n\tr, err := gzip.NewReader(bytes.NewBuffer(bs))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer r.Close()\n\n\tvar c bytes.Buffer\n\tif _, err = io.Copy(&c, io.LimitReader(r, unzipLimit)); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn c.Bytes(), nil\n}\n\n// Path: core/codec/gzip_test.go\npackage codec\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"errors\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestGzip(t *testing.T) {\n\tvar buf bytes.Buffer\n\tfor i := 0; i < 10000; i++ {\n\t\tfmt.Fprint(&buf, i)\n\t}\n\n\tbs := Gzip(buf.Bytes())\n\tactual, err := Gunzip(bs)\n\n\tassert.Nil(t, err)\n\tassert.True(t, len(bs) < buf.Len())\n\tassert.Equal(t, buf.Bytes(), actual)\n}\n\nfunc TestGunzip(t *testing.T) {\n\ttests := []struct {\n\t\tname        string\n\t\tinput       []byte\n\t\texpected    []byte\n\t\texpectedErr error\n\t}{\n\t\t{\n\t\t\tname: \"valid input\",\n\t\t\tinput: func() []byte {\n\t\t\t\tvar buf bytes.Buffer\n\t\t\t\tgz := gzip.NewWriter(&buf)\n\t\t\t\tgz.Write([]byte(\"hello\"))\n\t\t\t\tgz.Close()\n\t\t\t\treturn buf.Bytes()\n\t\t\t}(),\n\t\t\texpected:    []byte(\"hello\"),\n\t\t\texpectedErr: nil,\n\t\t},\n\t\t{\n\t\t\tname:        \"invalid input\",\n\t\t\tinput:       []byte(\"invalid input\"),\n\t\t\texpected:    nil,\n\t\t\texpectedErr: gzip.ErrHeader,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tresult, err := Gunzip(test.input)\n\n\t\t\tif !bytes.Equal(result, test.expected) {\n\t\t\t\tt.Errorf(\"unexpected result: %v\", result)\n\t\t\t}\n\n\t\t\tif !errors.Is(err, test.expectedErr) {\n\t\t\t\tt.Errorf(\"unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: core/codec/hmac.go\npackage codec\n\nimport (\n\t\"crypto/hmac\"\n\t\"crypto/sha256\"\n\t\"encoding/base64\"\n\t\"io\"\n)\n\n// Hmac returns HMAC bytes for body with the given key.\nfunc Hmac(key []byte, body string) []byte {\n\th := hmac.New(sha256.New, key)\n\tio.WriteString(h, body)\n\treturn h.Sum(nil)\n}\n\n// HmacBase64 returns the base64 encoded string of HMAC for body with the given key.\nfunc HmacBase64(key []byte, body string) string {\n\treturn base64.StdEncoding.EncodeToString(Hmac(key, body))\n}\n\n// Path: core/codec/hmac_test.go\npackage codec\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestHmac(t *testing.T) {\n\tret := Hmac([]byte(\"foo\"), \"bar\")\n\tassert.Equal(t, \"f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317\",\n\t\tfmt.Sprintf(\"%x\", ret))\n}\n\nfunc TestHmacBase64(t *testing.T) {\n\tret := HmacBase64([]byte(\"foo\"), \"bar\")\n\tassert.Equal(t, \"+TILrwJJFp5zhQzWFW3tAQbiu2rYyrAbe7vr5tEGUxc=\", ret)\n}\n\n// Path: core/codec/rsa.go\npackage codec\n\nimport (\n\t\"crypto/rand\"\n\t\"crypto/rsa\"\n\t\"crypto/x509\"\n\t\"encoding/base64\"\n\t\"encoding/pem\"\n\t\"errors\"\n\t\"os\"\n)\n\nvar (\n\t// ErrPrivateKey indicates the invalid private key.\n\tErrPrivateKey = errors.New(\"private key error\")\n\t// ErrPublicKey indicates the invalid public key.\n\tErrPublicKey = errors.New(\"failed to parse PEM block containing the public key\")\n\t// ErrNotRsaKey indicates the invalid RSA key.\n\tErrNotRsaKey = errors.New(\"key type is not RSA\")\n)\n\ntype (\n\t// RsaDecrypter represents a RSA decrypter.\n\tRsaDecrypter interface {\n\t\tDecrypt(input []byte) ([]byte, error)\n\t\tDecryptBase64(input string) ([]byte, error)\n\t}\n\n\t// RsaEncrypter represents a RSA encrypter.\n\tRsaEncrypter interface {\n\t\tEncrypt(input []byte) ([]byte, error)\n\t}\n\n\trsaBase struct {\n\t\tbytesLimit int\n\t}\n\n\trsaDecrypter struct {\n\t\trsaBase\n\t\tprivateKey *rsa.PrivateKey\n\t}\n\n\trsaEncrypter struct {\n\t\trsaBase\n\t\tpublicKey *rsa.PublicKey\n\t}\n)\n\n// NewRsaDecrypter returns a RsaDecrypter with the given file.\nfunc NewRsaDecrypter(file string) (RsaDecrypter, error) {\n\tcontent, err := os.ReadFile(file)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tblock, _ := pem.Decode(content)\n\tif block == nil {\n\t\treturn nil, ErrPrivateKey\n\t}\n\n\tprivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)\n\tif err != nil {\n...\n", "needle_token_start": 2228, "needle_token_end": 2345, "code_context_ntokens": 15161, "output": ["```go\nfunc newGoogleBreaker() *googleBreaker {\n\tbucketDuration := time.Duration(int64(window) / int64(buckets))\n\tst := collection.NewRollingWindow[int64, *bucket](func() *bucket {\n\t\treturn new(bucket)\n\t}, buckets, bucketDuration)\n\treturn &googleBreaker{\n\t\tstat:  st,\n\t\tk:     k,\n\t\tproba: mathx.NewProba(),\n\t}\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "TestCircuitBreaker_Do", "language": "go", "path": "core/breaker/breaker_test.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To verify the functionality of a circuit breaker in executing a function under normal conditions and simulated context timeouts and cancellations.\n2. **Input**: The test does not take external inputs but internally uses a context (with and without timeouts and cancellations) and a function that returns no error.\n3. **Output**: The test checks for the absence or presence of errors based on the context's state (whether it is active, timed out, or cancelled).\n4. **Procedure**: \n   - First, a circuit breaker instance is created.\n   - The test then attempts to execute a no-operation function under various conditions:\n     - With a normal context.\n     - With a context that has an extremely short timeout, followed by a delay to ensure the timeout is exceeded.\n     - With a context that is immediately cancelled.\n   - The test asserts that there are no errors when the context is normal, and appropriate errors (timeout or cancellation) are returned in other scenarios.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/hash/hash.go\npackage hash\n\nimport (\n\t\"crypto/md5\"\n\t\"fmt\"\n\n\t\"github.com/spaolacci/murmur3\"\n)\n\n// Hash returns the hash value of data.\nfunc Hash(data []byte) uint64 {\n\treturn murmur3.Sum64(data)\n}\n\n// Md5 returns the md5 bytes of data.\nfunc Md5(data []byte) []byte {\n\tdigest := md5.New()\n\tdigest.Write(data)\n\treturn digest.Sum(nil)\n}\n\n// Md5Hex returns the md5 hex string of data.\nfunc Md5Hex(data []byte) string {\n...\n// Path: core/bloom/bloom.go\npackage bloom\n\nimport (\n\t\"context\"\n\t_ \"embed\"\n\t\"errors\"\n\t\"strconv\"\n\n\t\"github.com/zeromicro/go-zero/core/hash\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis\"\n)\n\n// for detailed error rate table, see http://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html\n// maps as k in the error rate table\nconst maps = 14\n\nvar (\n\t// ErrTooLargeOffset indicates the offset is too large in bitset.\n\tErrTooLargeOffset = errors.New(\"too large offset\")\n\n\t//go:embed setscript.lua\n\tsetLuaScript string\n\tsetScript    = redis.NewScript(setLuaScript)\n\n\t//go:embed testscript.lua\n\ttestLuaScript string\n\ttestScript    = redis.NewScript(testLuaScript)\n)\n\ntype (\n\t// A Filter is a bloom filter.\n\tFilter struct {\n\t\tbits   uint\n\t\tbitSet bitSetProvider\n\t}\n\n\tbitSetProvider interface {\n\t\tcheck(ctx context.Context, offsets []uint) (bool, error)\n\t\tset(ctx context.Context, offsets []uint) error\n\t}\n)\n\n// New create a Filter, store is the backed redis, key is the key for the bloom filter,\n// bits is how many bits will be used, maps is how many hashes for each addition.\n// best practices:\n// elements - means how many actual elements\n// when maps = 14, formula: 0.7*(bits/maps), bits = 20*elements, the error rate is 0.000067 < 1e-4\n// for detailed error rate table, see http://pages.cs.wisc.edu/~cao/papers/summary-cache/node8.html\nfunc New(store *redis.Redis, key string, bits uint) *Filter {\n\treturn &Filter{\n\t\tbits:   bits,\n\t\tbitSet: newRedisBitSet(store, key, bits),\n\t}\n}\n\n// Add adds data into f.\nfunc (f *Filter) Add(data []byte) error {\n\treturn f.AddCtx(context.Background(), data)\n}\n\n// AddCtx adds data into f with context.\nfunc (f *Filter) AddCtx(ctx context.Context, data []byte) error {\n\tlocations := f.getLocations(data)\n\treturn f.bitSet.set(ctx, locations)\n}\n\n// Exists checks if data is in f.\nfunc (f *Filter) Exists(data []byte) (bool, error) {\n\treturn f.ExistsCtx(context.Background(), data)\n}\n\n// ExistsCtx checks if data is in f with context.\nfunc (f *Filter) ExistsCtx(ctx context.Context, data []byte) (bool, error) {\n\tlocations := f.getLocations(data)\n\tisSet, err := f.bitSet.check(ctx, locations)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn isSet, nil\n}\n\nfunc (f *Filter) getLocations(data []byte) []uint {\n\tlocations := make([]uint, maps)\n\tfor i := uint(0); i < maps; i++ {\n\t\thashValue := hash.Hash(append(data, byte(i)))\n\t\tlocations[i] = uint(hashValue % uint64(f.bits))\n\t}\n\n\treturn locations\n}\n\ntype redisBitSet struct {\n\tstore *redis.Redis\n\tkey   string\n\tbits  uint\n}\n\nfunc newRedisBitSet(store *redis.Redis, key string, bits uint) *redisBitSet {\n\treturn &redisBitSet{\n\t\tstore: store,\n\t\tkey:   key,\n\t\tbits:  bits,\n\t}\n}\n\nfunc (r *redisBitSet) buildOffsetArgs(offsets []uint) ([]string, error) {\n\tvar args []string\n\n\tfor _, offset := range offsets {\n\t\tif offset >= r.bits {\n\t\t\treturn nil, ErrTooLargeOffset\n\t\t}\n\n\t\targs = append(args, strconv.FormatUint(uint64(offset), 10))\n\t}\n\n\treturn args, nil\n}\n\nfunc (r *redisBitSet) check(ctx context.Context, offsets []uint) (bool, error) {\n\targs, err := r.buildOffsetArgs(offsets)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tresp, err := r.store.ScriptRunCtx(ctx, testScript, []string{r.key}, args)\n\tif errors.Is(err, redis.Nil) {\n\t\treturn false, nil\n\t} else if err != nil {\n\t\treturn false, err\n\t}\n\n\texists, ok := resp.(int64)\n\tif !ok {\n\t\treturn false, nil\n\t}\n\n\treturn exists == 1, nil\n}\n\n// del only use for testing.\nfunc (r *redisBitSet) del() error {\n\t_, err := r.store.Del(r.key)\n\treturn err\n}\n\n// expire only use for testing.\nfunc (r *redisBitSet) expire(seconds int) error {\n\treturn r.store.Expire(r.key, seconds)\n}\n\nfunc (r *redisBitSet) set(ctx context.Context, offsets []uint) error {\n\targs, err := r.buildOffsetArgs(offsets)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = r.store.ScriptRunCtx(ctx, setScript, []string{r.key}, args)\n\tif errors.Is(err, redis.Nil) {\n\t\treturn nil\n\t}\n\n\treturn err\n}\n\n// Path: core/stores/redis/redistest/redistest.go\npackage redistest\n\nimport (\n\t\"testing\"\n\n\t\"github.com/alicebob/miniredis/v2\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis\"\n)\n\n// CreateRedis returns an in process redis.Redis.\nfunc CreateRedis(t *testing.T) *redis.Redis {\n\tr, _ := CreateRedisWithClean(t)\n\treturn r\n}\n\n// CreateRedisWithClean returns an in process redis.Redis and a clean function.\nfunc CreateRedisWithClean(t *testing.T) (r *redis.Redis, clean func()) {\n\tmr := miniredis.RunT(t)\n\treturn redis.New(mr.Addr()), mr.Close\n}\n\n// Path: core/bloom/bloom_test.go\npackage bloom\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis/redistest\"\n)\n\nfunc TestRedisBitSet_New_Set_Test(t *testing.T) {\n\tstore := redistest.CreateRedis(t)\n\tctx := context.Background()\n\n\tbitSet := newRedisBitSet(store, \"test_key\", 1024)\n\tisSetBefore, err := bitSet.check(ctx, []uint{0})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif isSetBefore {\n\t\tt.Fatal(\"Bit should not be set\")\n\t}\n\terr = bitSet.set(ctx, []uint{512})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tisSetAfter, err := bitSet.check(ctx, []uint{512})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif !isSetAfter {\n\t\tt.Fatal(\"Bit should be set\")\n\t}\n\terr = bitSet.expire(3600)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = bitSet.del()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestRedisBitSet_Add(t *testing.T) {\n\tstore := redistest.CreateRedis(t)\n\n\tfilter := New(store, \"test_key\", 64)\n\tassert.Nil(t, filter.Add([]byte(\"hello\")))\n\tassert.Nil(t, filter.Add([]byte(\"world\")))\n\tok, err := filter.Exists([]byte(\"hello\"))\n\tassert.Nil(t, err)\n\tassert.True(t, ok)\n}\n\nfunc TestFilter_Exists(t *testing.T) {\n\tstore, clean := redistest.CreateRedisWithClean(t)\n\n\trbs := New(store, \"test\", 64)\n\t_, err := rbs.Exists([]byte{0, 1, 2})\n\tassert.NoError(t, err)\n\n\tclean()\n\trbs = New(store, \"test\", 64)\n\t_, err = rbs.Exists([]byte{0, 1, 2})\n\tassert.Error(t, err)\n}\n\nfunc TestRedisBitSet_check(t *testing.T) {\n\tstore, clean := redistest.CreateRedisWithClean(t)\n\tctx := context.Background()\n\n\trbs := newRedisBitSet(store, \"test\", 0)\n\tassert.Error(t, rbs.set(ctx, []uint{0, 1, 2}))\n\t_, err := rbs.check(ctx, []uint{0, 1, 2})\n\tassert.Error(t, err)\n\n\trbs = newRedisBitSet(store, \"test\", 64)\n\t_, err = rbs.check(ctx, []uint{0, 1, 2})\n\tassert.NoError(t, err)\n\n\tclean()\n\trbs = newRedisBitSet(store, \"test\", 64)\n\t_, err = rbs.check(ctx, []uint{0, 1, 2})\n\tassert.Error(t, err)\n}\n\nfunc TestRedisBitSet_set(t *testing.T) {\n\tlogx.Disable()\n\tstore, clean := redistest.CreateRedisWithClean(t)\n\tctx := context.Background()\n\n\trbs := newRedisBitSet(store, \"test\", 0)\n\tassert.Error(t, rbs.set(ctx, []uint{0, 1, 2}))\n\n\trbs = newRedisBitSet(store, \"test\", 64)\n\tassert.NoError(t, rbs.set(ctx, []uint{0, 1, 2}))\n\n\tclean()\n\trbs = newRedisBitSet(store, \"test\", 64)\n\tassert.Error(t, rbs.set(ctx, []uint{0, 1, 2}))\n}\n\n// Path: core/breaker/breaker_test.go\npackage breaker\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/stat\"\n)\n\nfunc init() {\n\tstat.SetReporter(nil)\n}\n\nfunc TestCircuitBreaker_Allow(t *testing.T) {\n\tt.Run(\"allow\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\t_, err := b.Allow()\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"allow with ctx\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\t_, err := b.AllowCtx(context.Background())\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"allow with ctx timeout\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Microsecond)\n\t\tdefer cancel()\n\t\ttime.Sleep(time.Millisecond)\n\t\t_, err := b.AllowCtx(ctx)\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded)\n\t})\n\n\tt.Run(\"allow with ctx cancel\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\t\tcancel()\n\t\t\t_, err := b.AllowCtx(ctx)\n\t\t\tassert.ErrorIs(t, err, context.Canceled)\n\t\t}\n\t\t_, err := b.AllowCtx(context.Background())\n\t\tassert.NoError(t, err)\n\t})\n}\n\n\nfunc TestCircuitBreaker_Do(t *testing.T) {\n\tt.Run(\"do\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.Do(func() error {\n\t\t\treturn nil\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"do with ctx\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.DoCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"do with ctx timeout\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Microsecond)\n\t\tdefer cancel()\n\t\ttime.Sleep(time.Millisecond)\n\t\terr := b.DoCtx(ctx, func() error {\n\t\t\treturn nil\n\t\t})\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded)\n\t})\n\n\tt.Run(\"do with ctx cancel\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\t\tcancel()\n\t\t\terr := b.DoCtx(ctx, func() error {\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tassert.ErrorIs(t, err, context.Canceled)\n\t\t}\n\t\tassert.NoError(t, b.DoCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t}))\n\t})\n}\n\nfunc TestCircuitBreaker_DoWithAcceptable(t *testing.T) {\n\tt.Run(\"doWithAcceptable\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.DoWithAcceptable(func() error {\n\t\t\treturn nil\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"doWithAcceptable with ctx\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.DoWithAcceptableCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"doWithAcceptable with ctx timeout\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Microsecond)\n\t\tdefer cancel()\n\t\ttime.Sleep(time.Millisecond)\n\t\terr := b.DoWithAcceptableCtx(ctx, func() error {\n\t\t\treturn nil\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t})\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded)\n\t})\n\n\tt.Run(\"doWithAcceptable with ctx cancel\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\t\tcancel()\n\t\t\terr := b.DoWithAcceptableCtx(ctx, func() error {\n\t\t\t\treturn nil\n\t\t\t}, func(err error) bool {\n\t\t\t\treturn true\n\t\t\t})\n\t\t\tassert.ErrorIs(t, err, context.Canceled)\n\t\t}\n\t\tassert.NoError(t, b.DoWithAcceptableCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t}))\n\t})\n}\n\nfunc TestCircuitBreaker_DoWithFallback(t *testing.T) {\n\tt.Run(\"doWithFallback\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.DoWithFallback(func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"doWithFallback with ctx\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.DoWithFallbackCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"doWithFallback with ctx timeout\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Microsecond)\n\t\tdefer cancel()\n\t\ttime.Sleep(time.Millisecond)\n\t\terr := b.DoWithFallbackCtx(ctx, func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t})\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded)\n\t})\n\n\tt.Run(\"doWithFallback with ctx cancel\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\t\tcancel()\n\t\t\terr := b.DoWithFallbackCtx(ctx, func() error {\n\t\t\t\treturn nil\n\t\t\t}, func(err error) error {\n\t\t\t\treturn err\n\t\t\t})\n\t\t\tassert.ErrorIs(t, err, context.Canceled)\n\t\t}\n\t\tassert.NoError(t, b.DoWithFallbackCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t}))\n\t})\n}\n\nfunc TestCircuitBreaker_DoWithFallbackAcceptable(t *testing.T) {\n\tt.Run(\"doWithFallbackAcceptable\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.DoWithFallbackAcceptable(func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"doWithFallbackAcceptable with ctx\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\terr := b.DoWithFallbackAcceptableCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t})\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"doWithFallbackAcceptable with ctx timeout\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Microsecond)\n\t\tdefer cancel()\n\t\ttime.Sleep(time.Millisecond)\n\t\terr := b.DoWithFallbackAcceptableCtx(ctx, func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t})\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded)\n\t})\n\n\tt.Run(\"doWithFallbackAcceptable with ctx cancel\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\t\tcancel()\n\t\t\terr := b.DoWithFallbackAcceptableCtx(ctx, func() error {\n\t\t\t\treturn nil\n\t\t\t}, func(err error) error {\n\t\t\t\treturn err\n\t\t\t}, func(err error) bool {\n\t\t\t\treturn true\n\t\t\t})\n\t\t\tassert.ErrorIs(t, err, context.Canceled)\n\t\t}\n\t\tassert.NoError(t, b.DoWithFallbackAcceptableCtx(context.Background(), func() error {\n\t\t\treturn nil\n\t\t}, func(err error) error {\n\t\t\treturn err\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t}))\n\t})\n}\n\nfunc TestLogReason(t *testing.T) {\n\tb := NewBreaker()\n\tassert.True(t, len(b.Name()) > 0)\n\n\tfor i := 0; i < 1000; i++ {\n\t\t_ = b.Do(func() error {\n\t\t\treturn errors.New(strconv.Itoa(i))\n\t\t})\n\t}\n\terrs := b.(*circuitBreaker).throttle.(loggedThrottle).errWin\n\tassert.Equal(t, numHistoryReasons, errs.count)\n}\n\nfunc TestErrorWindow(t *testing.T) {\n\ttests := []struct {\n\t\tname    string\n\t\treasons []string\n\t}{\n\t\t{\n\t\t\tname: \"no error\",\n\t\t},\n\t\t{\n\t\t\tname:    \"one error\",\n\t\t\treasons: []string{\"foo\"},\n\t\t},\n\t\t{\n\t\t\tname:    \"two errors\",\n\t\t\treasons: []string{\"foo\", \"bar\"},\n\t\t},\n\t\t{\n\t\t\tname:    \"five errors\",\n\t\t\treasons: []string{\"first\", \"second\", \"third\", \"fourth\", \"fifth\"},\n\t\t},\n\t\t{\n\t\t\tname:    \"six errors\",\n\t\t\treasons: []string{\"first\", \"second\", \"third\", \"fourth\", \"fifth\", \"sixth\"},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tvar ew errorWindow\n\t\t\tfor _, reason := range test.reasons {\n\t\t\t\tew.add(reason)\n\t\t\t}\n\t\t\tvar reasons []string\n\t\t\tif len(test.reasons) > numHistoryReasons {\n\t\t\t\treasons = test.reasons[len(test.reasons)-numHistoryReasons:]\n\t\t\t} else {\n\t\t\t\treasons = test.reasons\n\t\t\t}\n\t\t\tfor _, reason := range reasons {\n\t\t\t\tassert.True(t, strings.Contains(ew.String(), reason), fmt.Sprintf(\"actual: %s\", ew.String()))\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestPromiseWithReason(t *testing.T) {\n\ttests := []struct {\n\t\tname   string\n\t\treason string\n\t\texpect string\n\t}{\n\t\t{\n\t\t\tname: \"success\",\n\t\t},\n\t\t{\n\t\t\tname:   \"success\",\n\t\t\treason: \"fail\",\n\t\t\texpect: \"fail\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tpromise := promiseWithReason{\n\t\t\t\tpromise: new(mockedPromise),\n\t\t\t\terrWin:  new(errorWindow),\n\t\t\t}\n\t\t\tif len(test.reason) == 0 {\n\t\t\t\tpromise.Accept()\n\t\t\t} else {\n\t\t\t\tpromise.Reject(test.reason)\n\t\t\t}\n\n\t\t\tassert.True(t, strings.Contains(promise.errWin.String(), test.expect))\n\t\t})\n\t}\n}\n\nfunc BenchmarkGoogleBreaker(b *testing.B) {\n\tbr := NewBreaker()\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = br.Do(func() error {\n\t\t\treturn nil\n\t\t})\n\t}\n}\n\ntype mockedPromise struct{}\n\nfunc (m *mockedPromise) Accept() {\n}\n\nfunc (m *mockedPromise) Reject() {\n}\n\n// Path: core/breaker/breakers.go\npackage breaker\n\nimport (\n\t\"context\"\n\t\"sync\"\n)\n\nvar (\n\tlock     sync.RWMutex\n\tbreakers = make(map[string]Breaker)\n)\n\n// Do calls Breaker.Do on the Breaker with given name.\nfunc Do(name string, req func() error) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.Do(req)\n\t})\n}\n\n// DoCtx calls Breaker.DoCtx on the Breaker with given name.\nfunc DoCtx(ctx context.Context, name string, req func() error) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.DoCtx(ctx, req)\n\t})\n}\n\n// DoWithAcceptable calls Breaker.DoWithAcceptable on the Breaker with given name.\nfunc DoWithAcceptable(name string, req func() error, acceptable Acceptable) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.DoWithAcceptable(req, acceptable)\n\t})\n}\n\n// DoWithAcceptableCtx calls Breaker.DoWithAcceptableCtx on the Breaker with given name.\nfunc DoWithAcceptableCtx(ctx context.Context, name string, req func() error,\n\tacceptable Acceptable) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.DoWithAcceptableCtx(ctx, req, acceptable)\n\t})\n}\n\n// DoWithFallback calls Breaker.DoWithFallback on the Breaker with given name.\nfunc DoWithFallback(name string, req func() error, fallback Fallback) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.DoWithFallback(req, fallback)\n\t})\n}\n\n// DoWithFallbackCtx calls Breaker.DoWithFallbackCtx on the Breaker with given name.\nfunc DoWithFallbackCtx(ctx context.Context, name string, req func() error, fallback Fallback) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.DoWithFallbackCtx(ctx, req, fallback)\n\t})\n}\n\n// DoWithFallbackAcceptable calls Breaker.DoWithFallbackAcceptable on the Breaker with given name.\nfunc DoWithFallbackAcceptable(name string, req func() error, fallback Fallback,\n\tacceptable Acceptable) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.DoWithFallbackAcceptable(req, fallback, acceptable)\n\t})\n}\n\n// DoWithFallbackAcceptableCtx calls Breaker.DoWithFallbackAcceptableCtx on the Breaker with given name.\nfunc DoWithFallbackAcceptableCtx(ctx context.Context, name string, req func() error,\n\tfallback Fallback, acceptable Acceptable) error {\n\treturn do(name, func(b Breaker) error {\n\t\treturn b.DoWithFallbackAcceptableCtx(ctx, req, fallback, acceptable)\n\t})\n}\n\n// GetBreaker returns the Breaker with the given name.\nfunc GetBreaker(name string) Breaker {\n\tlock.RLock()\n\tb, ok := breakers[name]\n\tlock.RUnlock()\n\tif ok {\n\t\treturn b\n\t}\n\n\tlock.Lock()\n\tb, ok = breakers[name]\n\tif !ok {\n\t\tb = NewBreaker(WithName(name))\n\t\tbreakers[name] = b\n\t}\n\tlock.Unlock()\n\n\treturn b\n}\n\n// NoBreakerFor disables the circuit breaker for the given name.\nfunc NoBreakerFor(name string) {\n\tlock.Lock()\n\tbreakers[name] = NopBreaker()\n\tlock.Unlock()\n}\n\nfunc do(name string, execute func(b Breaker) error) error {\n\treturn execute(GetBreaker(name))\n}\n\n// Path: core/breaker/breakers_test.go\npackage breaker\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/stat\"\n)\n\nfunc init() {\n\tstat.SetReporter(nil)\n}\n\nfunc TestBreakersDo(t *testing.T) {\n\tassert.Nil(t, Do(\"any\", func() error {\n\t\treturn nil\n\t}))\n\n\terrDummy := errors.New(\"any\")\n\tassert.Equal(t, errDummy, Do(\"any\", func() error {\n\t\treturn errDummy\n\t}))\n\tassert.Equal(t, errDummy, DoCtx(context.Background(), \"any\", func() error {\n\t\treturn errDummy\n\t}))\n}\n\nfunc TestBreakersDoWithAcceptable(t *testing.T) {\n\terrDummy := errors.New(\"anyone\")\n\tfor i := 0; i < 10000; i++ {\n\t\tassert.Equal(t, errDummy, GetBreaker(\"anyone\").DoWithAcceptable(func() error {\n\t\t\treturn errDummy\n\t\t}, func(err error) bool {\n\t\t\treturn err == nil || errors.Is(err, errDummy)\n\t\t}))\n\t}\n\tverify(t, func() bool {\n\t\treturn Do(\"anyone\", func() error {\n\t\t\treturn nil\n\t\t}) == nil\n\t})\n\tverify(t, func() bool {\n\t\treturn DoWithAcceptableCtx(context.Background(), \"anyone\", func() error {\n\t\t\treturn nil\n\t\t}, func(err error) bool {\n\t\t\treturn true\n\t\t}) == nil\n\t})\n\n\tfor i := 0; i < 10000; i++ {\n\t\terr := DoWithAcceptable(\"another\", func() error {\n\t\t\treturn errDummy\n\t\t}, func(err error) bool {\n\t\t\treturn err == nil\n\t\t})\n\t\tassert.True(t, errors.Is(err, errDummy) || errors.Is(err, ErrServiceUnavailable))\n\t}\n\tverify(t, func() bool {\n\t\treturn errors.Is(Do(\"another\", func() error {\n\t\t\treturn nil\n\t\t}), ErrServiceUnavailable)\n\t})\n}\n\nfunc TestBreakersNoBreakerFor(t *testing.T) {\n\tNoBreakerFor(\"any\")\n\terrDummy := errors.New(\"any\")\n\tfor i := 0; i < 10000; i++ {\n\t\tassert.Equal(t, errDummy, GetBreaker(\"any\").Do(func() error {\n\t\t\treturn errDummy\n\t\t}))\n\t}\n\tassert.Equal(t, nil, Do(\"any\", func() error {\n\t\treturn nil\n\t}))\n}\n\nfunc TestBreakersFallback(t *testing.T) {\n\terrDummy := errors.New(\"any\")\n\tfor i := 0; i < 10000; i++ {\n\t\terr := DoWithFallback(\"fallback\", func() error {\n\t\t\treturn errDummy\n\t\t}, func(err error) error {\n\t\t\treturn nil\n\t\t})\n\t\tassert.True(t, err == nil || errors.Is(err, errDummy))\n\t\terr = DoWithFallbackCtx(context.Background(), \"fallback\", func() error {\n\t\t\treturn errDummy\n\t\t}, func(err error) error {\n\t\t\treturn nil\n\t\t})\n\t\tassert.True(t, err == nil || errors.Is(err, errDummy))\n\t}\n\tverify(t, func() bool {\n\t\treturn errors.Is(Do(\"fallback\", func() error {\n\t\t\treturn nil\n\t\t}), ErrServiceUnavailable)\n\t})\n}\n\nfunc TestBreakersAcceptableFallback(t *testing.T) {\n\terrDummy := errors.New(\"any\")\n\tfor i := 0; i < 5000; i++ {\n\t\terr := DoWithFallbackAcceptable(\"acceptablefallback\", func() error {\n\t\t\treturn errDummy\n\t\t}, func(err error) error {\n\t\t\treturn nil\n\t\t}, func(err error) bool {\n\t\t\treturn err == nil\n\t\t})\n\t\tassert.True(t, err == nil || errors.Is(err, errDummy))\n\t\terr = DoWithFallbackAcceptableCtx(context.Background(), \"acceptablefallback\", func() error {\n\t\t\treturn errDummy\n\t\t}, func(err error) error {\n\t\t\treturn nil\n\t\t}, func(err error) bool {\n\t\t\treturn err == nil\n\t\t})\n\t\tassert.True(t, err == nil || errors.Is(err, errDummy))\n\t}\n\tverify(t, func() bool {\n\t\treturn errors.Is(Do(\"acceptablefallback\", func() error {\n\t\t\treturn nil\n\t\t}), ErrServiceUnavailable)\n\t})\n}\n\nfunc verify(t *testing.T, fn func() bool) {\n\tvar count int\n\tfor i := 0; i < 100; i++ {\n\t\tif fn() {\n\t\t\tcount++\n\t\t}\n\t}\n\tassert.True(t, count >= 80, fmt.Sprintf(\"should be greater than 80, actual %d\", count))\n}\n\n// Path: core/breaker/bucket.go\npackage breaker\n\nconst (\n\tsuccess = iota\n\tfail\n\tdrop\n)\n\n// bucket defines the bucket that holds sum and num of additions.\ntype bucket struct {\n\tSum     int64\n\tSuccess int64\n\tFailure int64\n\tDrop    int64\n}\n\nfunc (b *bucket) Add(v int64) {\n\tswitch v {\n\tcase fail:\n\t\tb.fail()\n\tcase drop:\n\t\tb.drop()\n\tdefault:\n\t\tb.succeed()\n\t}\n}\n\nfunc (b *bucket) Reset() {\n\tb.Sum = 0\n\tb.Success = 0\n\tb.Failure = 0\n\tb.Drop = 0\n}\n\nfunc (b *bucket) drop() {\n\tb.Sum++\n\tb.Drop++\n}\n\nfunc (b *bucket) fail() {\n\tb.Sum++\n\tb.Failure++\n}\n\nfunc (b *bucket) succeed() {\n\tb.Sum++\n\tb.Success++\n}\n\n// Path: core/breaker/bucket_test.go\npackage breaker\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestBucketAdd(t *testing.T) {\n\tb := &bucket{}\n\n\t// Test succeed\n\tb.Add(0) // Using 0 for success\n\tassert.Equal(t, int64(1), b.Sum, \"Sum should be incremented\")\n\tassert.Equal(t, int64(1), b.Success, \"Success should be incremented\")\n\tassert.Equal(t, int64(0), b.Failure, \"Failure should not be incremented\")\n\tassert.Equal(t, int64(0), b.Drop, \"Drop should not be incremented\")\n\n\t// Test failure\n\tb.Add(fail)\n\tassert.Equal(t, int64(2), b.Sum, \"Sum should be incremented\")\n\tassert.Equal(t, int64(1), b.Failure, \"Failure should be incremented\")\n\tassert.Equal(t, int64(0), b.Drop, \"Drop should not be incremented\")\n\n\t// Test drop\n\tb.Add(drop)\n\tassert.Equal(t, int64(3), b.Sum, \"Sum should be incremented\")\n\tassert.Equal(t, int64(1), b.Drop, \"Drop should be incremented\")\n}\n\nfunc TestBucketReset(t *testing.T) {\n\tb := &bucket{\n\t\tSum:     3,\n\t\tSuccess: 1,\n\t\tFailure: 1,\n\t\tDrop:    1,\n\t}\n\tb.Reset()\n\tassert.Equal(t, int64(0), b.Sum, \"Sum should be reset to 0\")\n\tassert.Equal(t, int64(0), b.Success, \"Success should be reset to 0\")\n\tassert.Equal(t, int64(0), b.Failure, \"Failure should be reset to 0\")\n\tassert.Equal(t, int64(0), b.Drop, \"Drop should be reset to 0\")\n}\n\n// Path: core/stores/mon/collection.go\npackage mon\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/breaker\"\n\t\"github.com/zeromicro/go-zero/core/timex\"\n\t\"go.mongodb.org/mongo-driver/mongo\"\n\tmopt \"go.mongodb.org/mongo-driver/mongo/options\"\n\t\"go.mongodb.org/mongo-driver/x/mongo/driver/session\"\n)\n\nconst (\n\tdefaultSlowThreshold = time.Millisecond * 500\n\t// spanName is the span name of the mongo calls.\n\tspanName = \"mongo\"\n\n\t// mongodb method names\n\taggregate              = \"Aggregate\"\n\tbulkWrite              = \"BulkWrite\"\n\tcountDocuments         = \"CountDocuments\"\n\tdeleteMany             = \"DeleteMany\"\n\tdeleteOne              = \"DeleteOne\"\n\tdistinct               = \"Distinct\"\n\testimatedDocumentCount = \"EstimatedDocumentCount\"\n\tfind                   = \"Find\"\n\tfindOne                = \"FindOne\"\n\tfindOneAndDelete       = \"FindOneAndDelete\"\n\tfindOneAndReplace      = \"FindOneAndReplace\"\n\tfindOneAndUpdate       = \"FindOneAndUpdate\"\n\tinsertMany             = \"InsertMany\"\n\tinsertOne              = \"InsertOne\"\n\treplaceOne             = \"ReplaceOne\"\n\tupdateByID             = \"UpdateByID\"\n\tupdateMany             = \"UpdateMany\"\n\tupdateOne              = \"UpdateOne\"\n)\n\n// ErrNotFound is an alias of mongo.ErrNoDocuments\nvar ErrNotFound = mongo.ErrNoDocuments\n\ntype (\n\t// Collection defines a MongoDB collection.\n\tCollection interface {\n\t\t// Aggregate executes an aggregation pipeline.\n\t\tAggregate(ctx context.Context, pipeline any, opts ...*mopt.AggregateOptions) (\n\t\t\t*mongo.Cursor, error)\n\t\t// BulkWrite performs a bulk write operation.\n\t\tBulkWrite(ctx context.Context, models []mongo.WriteModel, opts ...*mopt.BulkWriteOptions) (\n\t\t\t*mongo.BulkWriteResult, error)\n\t\t// Clone creates a copy of this collection with the same settings.\n\t\tClone(opts ...*mopt.CollectionOptions) (*mongo.Collection, error)\n\t\t// CountDocuments returns the number of documents in the collection that match the filter.\n\t\tCountDocuments(ctx context.Context, filter any, opts ...*mopt.CountOptions) (int64, error)\n\t\t// Database returns the database that this collection is a part of.\n\t\tDatabase() *mongo.Database\n\t\t// DeleteMany deletes documents from the collection that match the filter.\n\t\tDeleteMany(ctx context.Context, filter any, opts ...*mopt.DeleteOptions) (\n\t\t\t*mongo.DeleteResult, error)\n\t\t// DeleteOne deletes at most one document from the collection that matches the filter.\n\t\tDeleteOne(ctx context.Context, filter any, opts ...*mopt.DeleteOptions) (\n\t\t\t*mongo.DeleteResult, error)\n\t\t// Distinct returns a list of distinct values for the given key across the collection.\n\t\tDistinct(ctx context.Context, fieldName string, filter any,\n\t\t\topts ...*mopt.DistinctOptions) ([]any, error)\n\t\t// Drop drops this collection from database.\n\t\tDrop(ctx context.Context) error\n\t\t// EstimatedDocumentCount returns an estimate of the count of documents in a collection\n\t\t// using collection metadata.\n\t\tEstimatedDocumentCount(ctx context.Context, opts ...*mopt.EstimatedDocumentCountOptions) (int64, error)\n\t\t// Find finds the documents matching the provided filter.\n\t\tFind(ctx context.Context, filter any, opts ...*mopt.FindOptions) (*mongo.Cursor, error)\n\t\t// FindOne returns up to one document that matches the provided filter.\n\t\tFindOne(ctx context.Context, filter any, opts ...*mopt.FindOneOptions) (\n\t\t\t*mongo.SingleResult, error)\n\t\t// FindOneAndDelete returns at most one document that matches the filter. If the filter\n\t\t// matches multiple documents, only the first document is deleted.\n\t\tFindOneAndDelete(ctx context.Context, filter any, opts ...*mopt.FindOneAndDeleteOptions) (\n\t\t\t*mongo.SingleResult, error)\n\t\t// FindOneAndReplace returns at most one document that matches the filter. If the filter\n\t\t// matches multiple documents, FindOneAndReplace returns the first document in the\n\t\t// collection that matches the filter.\n\t\tFindOneAndReplace(ctx context.Context, filter, replacement any,\n\t\t\topts ...*mopt.FindOneAndReplaceOptions) (*mongo.SingleResult, error)\n\t\t// FindOneAndUpdate returns at most one document that matches the filter. If the filter\n\t\t// matches multiple documents, FindOneAndUpdate returns the first document in the\n\t\t// collection that matches the filter.\n\t\tFindOneAndUpdate(ctx context.Context, filter, update any,\n\t\t\topts ...*mopt.FindOneAndUpdateOptions) (*mongo.SingleResult, error)\n\t\t// Indexes returns the index view for this collection.\n\t\tIndexes() mongo.IndexView\n\t\t// InsertMany inserts the provided documents.\n\t\tInsertMany(ctx context.Context, documents []any, opts ...*mopt.InsertManyOptions) (\n\t\t\t*mongo.InsertManyResult, error)\n\t\t// InsertOne inserts the provided document.\n\t\tInsertOne(ctx context.Context, document any, opts ...*mopt.InsertOneOptions) (\n\t\t\t*mongo.InsertOneResult, error)\n\t\t// ReplaceOne replaces at most one document that matches the filter.\n\t\tReplaceOne(ctx context.Context, filter, replacement any,\n\t\t\topts ...*mopt.ReplaceOptions) (*mongo.UpdateResult, error)\n\t\t// UpdateByID updates a single document matching the provided filter.\n\t\tUpdateByID(ctx context.Context, id, update any,\n\t\t\topts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)\n\t\t// UpdateMany updates the provided documents.\n\t\tUpdateMany(ctx context.Context, filter, update any,\n\t\t\topts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)\n\t\t// UpdateOne updates a single document matching the provided filter.\n\t\tUpdateOne(ctx context.Context, filter, update any,\n\t\t\topts ...*mopt.UpdateOptions) (*mongo.UpdateResult, error)\n\t\t// Watch returns a change stream cursor used to receive notifications of changes to the collection.\n\t\tWatch(ctx context.Context, pipeline any, opts ...*mopt.ChangeStreamOptions) (\n\t\t\t*mongo.ChangeStream, error)\n\t}\n\n\tdecoratedCollection struct {\n\t\t*mongo.Collection\n\t\tname string\n\t\tbrk  breaker.Breaker\n\t}\n\n\tkeepablePromise struct {\n\t\tpromise breaker.Promise\n\t\tlog     func(error)\n\t}\n)\n\nfunc newCollection(collection *mongo.Collection, brk breaker.Breaker) Collection {\n\treturn &decoratedCollection{\n\t\tCollection: collection,\n\t\tname:       collection.Name(),\n\t\tbrk:        brk,\n\t}\n}\n\nfunc (c *decoratedCollection) Aggregate(ctx context.Context, pipeline any,\n\topts ...*mopt.AggregateOptions) (cur *mongo.Cursor, err error) {\n\tctx, span := startSpan(ctx, aggregate)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstarTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDurationSimple(ctx, aggregate, starTime, err)\n\t\t}()\n\n\t\tcur, err = c.Collection.Aggregate(ctx, pipeline, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) BulkWrite(ctx context.Context, models []mongo.WriteModel,\n\topts ...*mopt.BulkWriteOptions) (res *mongo.BulkWriteResult, err error) {\n\tctx, span := startSpan(ctx, bulkWrite)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDurationSimple(ctx, bulkWrite, startTime, err)\n\t\t}()\n\n\t\tres, err = c.Collection.BulkWrite(ctx, models, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) CountDocuments(ctx context.Context, filter any,\n\topts ...*mopt.CountOptions) (count int64, err error) {\n\tctx, span := startSpan(ctx, countDocuments)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDurationSimple(ctx, countDocuments, startTime, err)\n\t\t}()\n\n\t\tcount, err = c.Collection.CountDocuments(ctx, filter, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) DeleteMany(ctx context.Context, filter any,\n\topts ...*mopt.DeleteOptions) (res *mongo.DeleteResult, err error) {\n\tctx, span := startSpan(ctx, deleteMany)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDurationSimple(ctx, deleteMany, startTime, err)\n\t\t}()\n\n\t\tres, err = c.Collection.DeleteMany(ctx, filter, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) DeleteOne(ctx context.Context, filter any,\n\topts ...*mopt.DeleteOptions) (res *mongo.DeleteResult, err error) {\n\tctx, span := startSpan(ctx, deleteOne)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, deleteOne, startTime, err, filter)\n\t\t}()\n\n\t\tres, err = c.Collection.DeleteOne(ctx, filter, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) Distinct(ctx context.Context, fieldName string, filter any,\n\topts ...*mopt.DistinctOptions) (val []any, err error) {\n\tctx, span := startSpan(ctx, distinct)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDurationSimple(ctx, distinct, startTime, err)\n\t\t}()\n\n\t\tval, err = c.Collection.Distinct(ctx, fieldName, filter, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) EstimatedDocumentCount(ctx context.Context,\n\topts ...*mopt.EstimatedDocumentCountOptions) (val int64, err error) {\n\tctx, span := startSpan(ctx, estimatedDocumentCount)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDurationSimple(ctx, estimatedDocumentCount, startTime, err)\n\t\t}()\n\n\t\tval, err = c.Collection.EstimatedDocumentCount(ctx, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) Find(ctx context.Context, filter any,\n\topts ...*mopt.FindOptions) (cur *mongo.Cursor, err error) {\n\tctx, span := startSpan(ctx, find)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, find, startTime, err, filter)\n\t\t}()\n\n\t\tcur, err = c.Collection.Find(ctx, filter, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) FindOne(ctx context.Context, filter any,\n\topts ...*mopt.FindOneOptions) (res *mongo.SingleResult, err error) {\n\tctx, span := startSpan(ctx, findOne)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, findOne, startTime, err, filter)\n\t\t}()\n\n\t\tres = c.Collection.FindOne(ctx, filter, opts...)\n\t\terr = res.Err()\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) FindOneAndDelete(ctx context.Context, filter any,\n\topts ...*mopt.FindOneAndDeleteOptions) (res *mongo.SingleResult, err error) {\n\tctx, span := startSpan(ctx, findOneAndDelete)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, findOneAndDelete, startTime, err, filter)\n\t\t}()\n\n\t\tres = c.Collection.FindOneAndDelete(ctx, filter, opts...)\n\t\terr = res.Err()\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) FindOneAndReplace(ctx context.Context, filter any,\n\treplacement any, opts ...*mopt.FindOneAndReplaceOptions) (\n\tres *mongo.SingleResult, err error) {\n\tctx, span := startSpan(ctx, findOneAndReplace)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, findOneAndReplace, startTime, err, filter, replacement)\n\t\t}()\n\n\t\tres = c.Collection.FindOneAndReplace(ctx, filter, replacement, opts...)\n\t\terr = res.Err()\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) FindOneAndUpdate(ctx context.Context, filter, update any,\n\topts ...*mopt.FindOneAndUpdateOptions) (res *mongo.SingleResult, err error) {\n\tctx, span := startSpan(ctx, findOneAndUpdate)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, findOneAndUpdate, startTime, err, filter, update)\n\t\t}()\n\n\t\tres = c.Collection.FindOneAndUpdate(ctx, filter, update, opts...)\n\t\terr = res.Err()\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) InsertMany(ctx context.Context, documents []any,\n\topts ...*mopt.InsertManyOptions) (res *mongo.InsertManyResult, err error) {\n\tctx, span := startSpan(ctx, insertMany)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDurationSimple(ctx, insertMany, startTime, err)\n\t\t}()\n\n\t\tres, err = c.Collection.InsertMany(ctx, documents, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) InsertOne(ctx context.Context, document any,\n\topts ...*mopt.InsertOneOptions) (res *mongo.InsertOneResult, err error) {\n\tctx, span := startSpan(ctx, insertOne)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, insertOne, startTime, err, document)\n\t\t}()\n\n\t\tres, err = c.Collection.InsertOne(ctx, document, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) ReplaceOne(ctx context.Context, filter, replacement any,\n\topts ...*mopt.ReplaceOptions) (res *mongo.UpdateResult, err error) {\n\tctx, span := startSpan(ctx, replaceOne)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, replaceOne, startTime, err, filter, replacement)\n\t\t}()\n\n\t\tres, err = c.Collection.ReplaceOne(ctx, filter, replacement, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) UpdateByID(ctx context.Context, id, update any,\n\topts ...*mopt.UpdateOptions) (res *mongo.UpdateResult, err error) {\n\tctx, span := startSpan(ctx, updateByID)\n\tdefer func() {\n\t\tendSpan(span, err)\n\t}()\n\n\terr = c.brk.DoWithAcceptableCtx(ctx, func() error {\n\t\tstartTime := timex.Now()\n\t\tdefer func() {\n\t\t\tc.logDuration(ctx, updateByID, startTime, err, id, update)\n\t\t}()\n\n\t\tres, err = c.Collection.UpdateByID(ctx, id, update, opts...)\n\t\treturn err\n\t}, acceptable)\n\n\treturn\n}\n\nfunc (c *decoratedCollection) UpdateMany(ctx context.Context, filter, update any,\n\topts ...*mopt.UpdateOptions) (res *mongo.UpdateResult, err error) {\n\tctx, span := startSpan(ctx, updateMany)\n\tdefer func() {\n...\n", "needle_token_start": 3548, "needle_token_end": 4066, "code_context_ntokens": 15136, "output": ["```go\nfunc TestCircuitBreaker_Allow(t *testing.T) {\n\tt.Run(\"allow\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\t_, err := b.Allow()\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"allow with ctx\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\t_, err := b.AllowCtx(context.Background())\n\t\tassert.Nil(t, err)\n\t})\n\n\tt.Run(\"allow with ctx timeout\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tctx, cancel := context.WithTimeout(context.Background(), time.Microsecond)\n\t\tdefer cancel()\n\t\ttime.Sleep(time.Millisecond)\n\t\t_, err := b.AllowCtx(ctx)\n\t\tassert.ErrorIs(t, err, context.DeadlineExceeded)\n\t})\n\n\tt.Run(\"allow with ctx cancel\", func(t *testing.T) {\n\t\tb := NewBreaker()\n\t\tassert.True(t, len(b.Name()) > 0)\n\t\tfor i := 0; i < 100; i++ {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), time.Second)\n\t\t\tcancel()\n\t\t\t_, err := b.AllowCtx(ctx)\n\t\t\tassert.ErrorIs(t, err, context.Canceled)\n\t\t}\n\t\t_, err := b.AllowCtx(context.Background())\n\t\tassert.NoError(t, err)\n\t})\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "NewSubscriber", "language": "go", "path": "core/discov/subscriber.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To create an object that subscribes to a specific key on an etcd cluster, allowing for updates and changes to be monitored.\n2. **Input**: A list of etcd cluster endpoints, a key to subscribe to, and optional customization functions for the subscription object.\n3. **Output**: Returns a subscription object if successful, or an error if the monitoring setup fails.\n4. **Procedure**: Initializes a subscription object with the provided endpoints, applies any customization options, sets up a container to hold subscription items, and attempts to monitor the specified key on the etcd cluster using an internal registry. If monitoring is successful, the subscription object is returned; otherwise, an error is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/discov/publisher_test.go\npackage discov\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net\"\n\t\"os\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/golang/mock/gomock\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/discov/internal\"\n\t\"github.com/zeromicro/go-zero/core/lang\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/stringx\"\n\tclientv3 \"go.etcd.io/etcd/client/v3\"\n\t\"golang.org/x/net/http2\"\n\t\"google.golang.org/grpc\"\n\t\"google.golang.org/grpc/credentials/insecure\"\n\t\"google.golang.org/grpc/resolver\"\n\t\"google.golang.org/grpc/resolver/manual\"\n)\n\nconst (\n\tcertContent = `-----BEGIN CERTIFICATE-----\nMIIDazCCAlOgAwIBAgIUEg9GVO2oaPn+YSmiqmFIuAo10WIwDQYJKoZIhvcNAQEM\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMzAzMTExMzIxMjNaGA8yMTIz\nMDIxNTEzMjEyM1owRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx\nITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN\nAQEBBQADggEPADCCAQoCggEBALplXlWsIf0O/IgnIplmiZHKGnxyfyufyE2FBRNk\nOofRqbKuPH8GNqbkvZm7N29fwTDAQ+mViAggCkDht4hOzoWJMA7KYJt8JnTSWL48\nM1lcrpc9DL2gszC/JF/FGvyANbBtLklkZPFBGdHUX14pjrT937wqPtm+SqUHSvRT\nB7bmwmm2drRcmhpVm98LSlV7uQ2EgnJgsLjBPITKUejLmVLHfgX0RwQ2xIpX9pS4\nFCe1BTacwl2gGp7Mje7y4Mfv3o0ArJW6Tuwbjx59ZXwb1KIP71b7bT04AVS8ZeYO\nUMLKKuB5UR9x9Rn6cLXOTWBpcMVyzDgrAFLZjnE9LPUolZMCAwEAAaNRME8wHwYD\nVR0jBBgwFoAUeW8w8pmhncbRgTsl48k4/7wnfx8wCQYDVR0TBAIwADALBgNVHQ8E\nBAMCBPAwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBDAUAA4IBAQAI\ny9xaoS88CLPBsX6mxfcTAFVfGNTRW9VN9Ng1cCnUR+YGoXGM/l+qP4f7p8ocdGwK\niYZErVTzXYIn+D27//wpY3klJk3gAnEUBT3QRkStBw7XnpbeZ2oPBK+cmDnCnZPS\nBIF1wxPX7vIgaxs5Zsdqwk3qvZ4Djr2wP7LabNWTLSBKgQoUY45Liw6pffLwcGF9\nUKlu54bvGze2SufISCR3ib+I+FLvqpvJhXToZWYb/pfI/HccuCL1oot1x8vx6DQy\nU+TYxlZsKS5mdNxAX3dqEkEMsgEi+g/tzDPXJImfeCGGBhIOXLm8SRypiuGdEbc9\nxkWYxRPegajuEZGvCqVs\n-----END CERTIFICATE-----`\n\tkeyContent = `-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAumVeVawh/Q78iCcimWaJkcoafHJ/K5/ITYUFE2Q6h9Gpsq48\nfwY2puS9mbs3b1/BMMBD6ZWICCAKQOG3iE7OhYkwDspgm3wmdNJYvjwzWVyulz0M\nvaCzML8kX8Ua/IA1sG0uSWRk8UEZ0dRfXimOtP3fvCo+2b5KpQdK9FMHtubCabZ2\ntFyaGlWb3wtKVXu5DYSCcmCwuME8hMpR6MuZUsd+BfRHBDbEilf2lLgUJ7UFNpzC\nXaAansyN7vLgx+/ejQCslbpO7BuPHn1lfBvUog/vVvttPTgBVLxl5g5Qwsoq4HlR\nH3H1Gfpwtc5NYGlwxXLMOCsAUtmOcT0s9SiVkwIDAQABAoIBAD5meTJNMgO55Kjg\nESExxpRcCIno+tHr5+6rvYtEXqPheOIsmmwb9Gfi4+Z3WpOaht5/Pz0Ppj6yGzyl\nU//6AgGKb+BDuBvVcDpjwPnOxZIBCSHwejdxeQu0scSuA97MPS0XIAvJ5FEv7ijk\n5Bht6SyGYURpECltHygoTNuGgGqmO+McCJRLE9L09lTBI6UQ/JQwWJqSr7wx6iPU\nM1Ze/srIV+7cyEPu6i0DGjS1gSQKkX68Lqn1w6oE290O+OZvleO0gZ02fLDWCZke\naeD9+EU/Pw+rqm3H6o0szOFIpzhRp41FUdW9sybB3Yp3u7c/574E+04Z/e30LMKs\nTCtE1QECgYEA3K7KIpw0NH2HXL5C3RHcLmr204xeBfS70riBQQuVUgYdmxak2ima\n80RInskY8hRhSGTg0l+VYIH8cmjcUyqMSOELS5XfRH99r4QPiK8AguXg80T4VumY\nW3Pf+zEC2ssgP/gYthV0g0Xj5m2QxktOF9tRw5nkg739ZR4dI9lm/iECgYEA2Dnf\nuwEDGqHiQRF6/fh5BG/nGVMvrefkqx6WvTJQ3k/M/9WhxB+lr/8yH46TuS8N2b29\nFoTf3Mr9T7pr/PWkOPzoY3P56nYbKU8xSwCim9xMzhBMzj8/N9ukJvXy27/VOz56\neQaKqnvdXNGtPJrIMDGHps2KKWlKLyAlapzjVTMCgYAA/W++tACv85g13EykfT4F\nn0k4LbsGP9DP4zABQLIMyiY72eAncmRVjwrcW36XJ2xATOONTgx3gF3HjZzfaqNy\neD/6uNNllUTVEryXGmHgNHPL45VRnn6memCY2eFvZdXhM5W4y2PYaunY0MkDercA\n+GTngbs6tBF88KOk04bYwQKBgFl68cRgsdkmnwwQYNaTKfmVGYzYaQXNzkqmWPko\nxmCJo6tHzC7ubdG8iRCYHzfmahPuuj6EdGPZuSRyYFgJi5Ftz/nAN+84OxtIQ3zn\nYWOgskQgaLh9YfsKsQ7Sf1NDOsnOnD5TX7UXl07fEpLe9vNCvAFiU8e5Y9LGudU5\n4bYTAoGBAMdX3a3bXp4cZvXNBJ/QLVyxC6fP1Q4haCR1Od3m+T00Jth2IX2dk/fl\np6xiJT1av5JtYabv1dFKaXOS5s1kLGGuCCSKpkvFZm826aQ2AFm0XGqEQDLeei5b\nA52Kpy/YJ+RkG4BTFtAooFq6DmA0cnoP6oPvG2h6XtDJwDTPInJb\n-----END RSA PRIVATE KEY-----`\n\tcaContent = `-----BEGIN CERTIFICATE-----\nMIIDbTCCAlWgAwIBAgIUBJvFoCowKich7MMfseJ+DYzzirowDQYJKoZIhvcNAQEM\nBQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM\nGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAgFw0yMzAzMTExMzIxMDNaGA8yMTIz\nMDIxNTEzMjEwM1owRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx\nITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN\nAQEBBQADggEPADCCAQoCggEBAO4to2YMYj0bxgr2FCiweSTSFuPx33zSw2x/s9Wf\nOR41bm2DFsyYT5f3sOIKlXZEdLmOKty2e3ho3yC0EyNpVHdykkkHT3aDI17quZax\nkYi/URqqtl1Z08A22txolc04hAZisg2BypGi3vql81UW1t3zyloGnJoIAeXR9uca\nljP6Bk3bwsxoVBLi1JtHrO0hHLQaeHmKhAyrys06X0LRdn7Px48yRZlt6FaLSa8X\nYiRM0G44bVy/h6BkoQjMYGwVmCVk6zjJ9U7ZPFqdnDMNxAfR+hjDnYodqdLDMTTR\n1NPVrnEnNwFx0AMLvgt/ba/45vZCEAmSZnFXFAJJcM7ai9ECAwEAAaNTMFEwHQYD\nVR0OBBYEFHlvMPKZoZ3G0YE7JePJOP+8J38fMB8GA1UdIwQYMBaAFHlvMPKZoZ3G\n0YE7JePJOP+8J38fMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggEB\nAMX8dNulADOo9uQgBMyFb9TVra7iY0zZjzv4GY5XY7scd52n6CnfAPvYBBDnTr/O\nBgNp5jaujb4+9u/2qhV3f9n+/3WOb2CmPehBgVSzlXqHeQ9lshmgwZPeem2T+8Tm\nNnc/xQnsUfCFszUDxpkr55+aLVM22j02RWqcZ4q7TAaVYL+kdFVMc8FoqG/0ro6A\nBjE/Qn0Nn7ciX1VUjDt8l+k7ummPJTmzdi6i6E4AwO9dzrGNgGJ4aWL8cC6xYcIX\ngoVIRTFeONXSDno/oPjWHpIPt7L15heMpKBHNuzPkKx2YVqPHE5QZxWfS+Lzgx+Q\nE2oTTM0rYKOZ8p6000mhvKI=\n-----END CERTIFICATE-----`\n)\n\nfunc init() {\n\tlogx.Disable()\n}\n\nfunc TestPublisher_register(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tconst id = 1\n\tcli := internal.NewMockEtcdClient(ctrl)\n\trestore := setMockClient(cli)\n\tdefer restore()\n\tcli.EXPECT().Ctx().AnyTimes()\n\tcli.EXPECT().Grant(gomock.Any(), timeToLive).Return(&clientv3.LeaseGrantResponse{\n\t\tID: id,\n\t}, nil)\n\tcli.EXPECT().Put(gomock.Any(), makeEtcdKey(\"thekey\", id), \"thevalue\", gomock.Any())\n\tpub := NewPublisher(nil, \"thekey\", \"thevalue\",\n\t\tWithPubEtcdAccount(stringx.Rand(), \"bar\"))\n\t_, err := pub.register(cli)\n\tassert.Nil(t, err)\n}\n\nfunc TestPublisher_registerWithOptions(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tconst id = 2\n\tcli := internal.NewMockEtcdClient(ctrl)\n\trestore := setMockClient(cli)\n\tdefer restore()\n\tcli.EXPECT().Ctx().AnyTimes()\n\tcli.EXPECT().Grant(gomock.Any(), timeToLive).Return(&clientv3.LeaseGrantResponse{\n\t\tID: 1,\n\t}, nil)\n\tcli.EXPECT().Put(gomock.Any(), makeEtcdKey(\"thekey\", id), \"thevalue\", gomock.Any())\n\n\tcertFile := createTempFile(t, []byte(certContent))\n\tdefer os.Remove(certFile)\n\tkeyFile := createTempFile(t, []byte(keyContent))\n\tdefer os.Remove(keyFile)\n\tcaFile := createTempFile(t, []byte(caContent))\n\tdefer os.Remove(caFile)\n\tpub := NewPublisher(nil, \"thekey\", \"thevalue\", WithId(id),\n\t\tWithPubEtcdTLS(certFile, keyFile, caFile, true))\n\t_, err := pub.register(cli)\n\tassert.Nil(t, err)\n}\n\nfunc TestPublisher_registerError(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tcli := internal.NewMockEtcdClient(ctrl)\n\trestore := setMockClient(cli)\n\tdefer restore()\n\tcli.EXPECT().Ctx().AnyTimes()\n\tcli.EXPECT().Grant(gomock.Any(), timeToLive).Return(nil, errors.New(\"error\"))\n\tpub := NewPublisher(nil, \"thekey\", \"thevalue\")\n\tval, err := pub.register(cli)\n\tassert.NotNil(t, err)\n\tassert.Equal(t, clientv3.NoLease, val)\n}\n\nfunc TestPublisher_revoke(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tconst id clientv3.LeaseID = 1\n\tcli := internal.NewMockEtcdClient(ctrl)\n\trestore := setMockClient(cli)\n\tdefer restore()\n\tcli.EXPECT().Ctx().AnyTimes()\n\tcli.EXPECT().Revoke(gomock.Any(), id)\n\tpub := NewPublisher(nil, \"thekey\", \"thevalue\")\n\tpub.lease = id\n\tpub.revoke(cli)\n}\n\nfunc TestPublisher_revokeError(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tconst id clientv3.LeaseID = 1\n\tcli := internal.NewMockEtcdClient(ctrl)\n\trestore := setMockClient(cli)\n\tdefer restore()\n\tcli.EXPECT().Ctx().AnyTimes()\n\tcli.EXPECT().Revoke(gomock.Any(), id).Return(nil, errors.New(\"error\"))\n\tpub := NewPublisher(nil, \"thekey\", \"thevalue\")\n\tpub.lease = id\n\tpub.revoke(cli)\n}\n\nfunc TestPublisher_keepAliveAsyncError(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tconst id clientv3.LeaseID = 1\n\tcli := internal.NewMockEtcdClient(ctrl)\n\trestore := setMockClient(cli)\n\tdefer restore()\n\tcli.EXPECT().Ctx().AnyTimes()\n\tcli.EXPECT().KeepAlive(gomock.Any(), id).Return(nil, errors.New(\"error\"))\n\tpub := NewPublisher(nil, \"thekey\", \"thevalue\")\n\tpub.lease = id\n\tassert.NotNil(t, pub.keepAliveAsync(cli))\n}\n\nfunc TestPublisher_keepAliveAsyncQuit(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tconst id clientv3.LeaseID = 1\n\tcli := internal.NewMockEtcdClient(ctrl)\n\tcli.EXPECT().ActiveConnection()\n\tcli.EXPECT().Close()\n\tdefer cli.Close()\n\tcli.ActiveConnection()\n\trestore := setMockClient(cli)\n\tdefer restore()\n\tcli.EXPECT().Ctx().AnyTimes()\n\tcli.EXPECT().KeepAlive(gomock.Any(), id)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tcli.EXPECT().Revoke(gomock.Any(), id).Do(func(_, _ any) {\n\t\twg.Done()\n\t})\n\tpub := NewPublisher(nil, \"thekey\", \"thevalue\")\n\tpub.lease = id\n\tpub.Stop()\n\tassert.Nil(t, pub.keepAliveAsync(cli))\n\twg.Wait()\n}\n\nfunc TestPublisher_keepAliveAsyncPause(t *testing.T) {\n\tctrl := gomock.NewController(t)\n\tdefer ctrl.Finish()\n\tconst id clientv3.LeaseID = 1\n\tcli := internal.NewMockEtcdClient(ctrl)\n\trestore := setMockClient(cli)\n\tdefer restore()\n...\n// Path: core/discov/subscriber.go\npackage discov\n\nimport (\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/zeromicro/go-zero/core/discov/internal\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n)\n\ntype (\n\t// SubOption defines the method to customize a Subscriber.\n\tSubOption func(sub *Subscriber)\n\n\t// A Subscriber is used to subscribe the given key on an etcd cluster.\n\tSubscriber struct {\n\t\tendpoints []string\n\t\texclusive bool\n\t\titems     *container\n\t}\n)\n\n// NewSubscriber returns a Subscriber.\n// endpoints is the hosts of the etcd cluster.\n// key is the key to subscribe.\n// opts are used to customize the Subscriber.\n\nfunc NewSubscriber(endpoints []string, key string, opts ...SubOption) (*Subscriber, error) {\n\tsub := &Subscriber{\n\t\tendpoints: endpoints,\n\t}\n\tfor _, opt := range opts {\n\t\topt(sub)\n\t}\n\tsub.items = newContainer(sub.exclusive)\n\n\tif err := internal.GetRegistry().Monitor(endpoints, key, sub.items); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn sub, nil\n}\n\n// AddListener adds listener to s.\nfunc (s *Subscriber) AddListener(listener func()) {\n\ts.items.addListener(listener)\n}\n\n// Values returns all the subscription values.\nfunc (s *Subscriber) Values() []string {\n\treturn s.items.getValues()\n}\n\n// Exclusive means that key value can only be 1:1,\n// which means later added value will remove the keys associated with the same value previously.\nfunc Exclusive() SubOption {\n\treturn func(sub *Subscriber) {\n\t\tsub.exclusive = true\n\t}\n}\n\n// WithSubEtcdAccount provides the etcd username/password.\nfunc WithSubEtcdAccount(user, pass string) SubOption {\n\treturn func(sub *Subscriber) {\n\t\tRegisterAccount(sub.endpoints, user, pass)\n\t}\n}\n\n// WithSubEtcdTLS provides the etcd CertFile/CertKeyFile/CACertFile.\nfunc WithSubEtcdTLS(certFile, certKeyFile, caFile string, insecureSkipVerify bool) SubOption {\n\treturn func(sub *Subscriber) {\n\t\tlogx.Must(RegisterTLS(sub.endpoints, certFile, certKeyFile, caFile, insecureSkipVerify))\n\t}\n}\n\ntype container struct {\n\texclusive bool\n\tvalues    map[string][]string\n\tmapping   map[string]string\n\tsnapshot  atomic.Value\n\tdirty     *syncx.AtomicBool\n\tlisteners []func()\n\tlock      sync.Mutex\n}\n\nfunc newContainer(exclusive bool) *container {\n\treturn &container{\n\t\texclusive: exclusive,\n\t\tvalues:    make(map[string][]string),\n\t\tmapping:   make(map[string]string),\n\t\tdirty:     syncx.ForAtomicBool(true),\n\t}\n}\n\nfunc (c *container) OnAdd(kv internal.KV) {\n\tc.addKv(kv.Key, kv.Val)\n\tc.notifyChange()\n}\n\nfunc (c *container) OnDelete(kv internal.KV) {\n\tc.removeKey(kv.Key)\n\tc.notifyChange()\n}\n\n// addKv adds the kv, returns if there are already other keys associate with the value\nfunc (c *container) addKv(key, value string) ([]string, bool) {\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\n\tc.dirty.Set(true)\n\tkeys := c.values[value]\n\tprevious := append([]string(nil), keys...)\n\tearly := len(keys) > 0\n\tif c.exclusive && early {\n\t\tfor _, each := range keys {\n\t\t\tc.doRemoveKey(each)\n\t\t}\n\t}\n\tc.values[value] = append(c.values[value], key)\n\tc.mapping[key] = value\n\n\tif early {\n\t\treturn previous, true\n\t}\n\n\treturn nil, false\n}\n\nfunc (c *container) addListener(listener func()) {\n\tc.lock.Lock()\n\tc.listeners = append(c.listeners, listener)\n\tc.lock.Unlock()\n}\n\nfunc (c *container) doRemoveKey(key string) {\n\tserver, ok := c.mapping[key]\n\tif !ok {\n\t\treturn\n\t}\n\n\tdelete(c.mapping, key)\n\tkeys := c.values[server]\n\tremain := keys[:0]\n\n\tfor _, k := range keys {\n\t\tif k != key {\n\t\t\tremain = append(remain, k)\n\t\t}\n\t}\n\n\tif len(remain) > 0 {\n\t\tc.values[server] = remain\n\t} else {\n\t\tdelete(c.values, server)\n\t}\n}\n\nfunc (c *container) getValues() []string {\n\tif !c.dirty.True() {\n\t\treturn c.snapshot.Load().([]string)\n\t}\n\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\n\tvar vals []string\n\tfor each := range c.values {\n\t\tvals = append(vals, each)\n\t}\n\tc.snapshot.Store(vals)\n\tc.dirty.Set(false)\n\n\treturn vals\n}\n\nfunc (c *container) notifyChange() {\n\tc.lock.Lock()\n\tlisteners := append(([]func())(nil), c.listeners...)\n\tc.lock.Unlock()\n\n\tfor _, listener := range listeners {\n\t\tlistener()\n\t}\n}\n\n// removeKey removes the kv, returns true if there are still other keys associate with the value\nfunc (c *container) removeKey(key string) {\n\tc.lock.Lock()\n\tdefer c.lock.Unlock()\n\n\tc.dirty.Set(true)\n\tc.doRemoveKey(key)\n}\n\n// Path: core/discov/subscriber_test.go\npackage discov\n\nimport (\n\t\"sync/atomic\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/discov/internal\"\n\t\"github.com/zeromicro/go-zero/core/stringx\"\n)\n\nconst (\n\tactionAdd = iota\n\tactionDel\n)\n\nfunc TestContainer(t *testing.T) {\n\ttype action struct {\n\t\tact int\n\t\tkey string\n\t\tval string\n\t}\n\ttests := []struct {\n\t\tname   string\n\t\tdo     []action\n\t\texpect []string\n\t}{\n\t\t{\n\t\t\tname: \"add one\",\n\t\t\tdo: []action{\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: []string{\n\t\t\t\t\"a\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"add two\",\n\t\t\tdo: []action{\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t\tval: \"b\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: []string{\n\t\t\t\t\"a\",\n\t\t\t\t\"b\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"add two, delete one\",\n\t\t\tdo: []action{\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t\tval: \"b\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: []string{\"b\"},\n\t\t},\n\t\t{\n\t\t\tname: \"add two, delete two\",\n\t\t\tdo: []action{\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t\tval: \"b\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: []string{},\n\t\t},\n\t\t{\n\t\t\tname: \"add three, dup values, delete two\",\n\t\t\tdo: []action{\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t\tval: \"b\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"third\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: []string{\"a\"},\n\t\t},\n\t\t{\n\t\t\tname: \"add three, dup values, delete two, delete not added\",\n\t\t\tdo: []action{\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t\tval: \"b\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionAdd,\n\t\t\t\t\tkey: \"third\",\n\t\t\t\t\tval: \"a\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"first\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"second\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tact: actionDel,\n\t\t\t\t\tkey: \"forth\",\n\t\t\t\t},\n\t\t\t},\n\t\t\texpect: []string{\"a\"},\n\t\t},\n\t}\n\n\texclusives := []bool{true, false}\n\tfor _, test := range tests {\n\t\tfor _, exclusive := range exclusives {\n\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\tvar changed bool\n\t\t\t\tc := newContainer(exclusive)\n\t\t\t\tc.addListener(func() {\n\t\t\t\t\tchanged = true\n\t\t\t\t})\n\t\t\t\tassert.Nil(t, c.getValues())\n\t\t\t\tassert.False(t, changed)\n\n\t\t\t\tfor _, order := range test.do {\n\t\t\t\t\tif order.act == actionAdd {\n\t\t\t\t\t\tc.OnAdd(internal.KV{\n\t\t\t\t\t\t\tKey: order.key,\n\t\t\t\t\t\t\tVal: order.val,\n\t\t\t\t\t\t})\n\t\t\t\t\t} else {\n\t\t\t\t\t\tc.OnDelete(internal.KV{\n\t\t\t\t\t\t\tKey: order.key,\n\t\t\t\t\t\t\tVal: order.val,\n\t\t\t\t\t\t})\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tassert.True(t, changed)\n\t\t\t\tassert.True(t, c.dirty.True())\n\t\t\t\tassert.ElementsMatch(t, test.expect, c.getValues())\n\t\t\t\tassert.False(t, c.dirty.True())\n\t\t\t\tassert.ElementsMatch(t, test.expect, c.getValues())\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestSubscriber(t *testing.T) {\n\tsub := new(Subscriber)\n\tExclusive()(sub)\n\tsub.items = newContainer(sub.exclusive)\n\tvar count int32\n\tsub.AddListener(func() {\n\t\tatomic.AddInt32(&count, 1)\n\t})\n\tsub.items.notifyChange()\n\tassert.Empty(t, sub.Values())\n\tassert.Equal(t, int32(1), atomic.LoadInt32(&count))\n}\n\nfunc TestWithSubEtcdAccount(t *testing.T) {\n\tendpoints := []string{\"localhost:2379\"}\n\tuser := stringx.Rand()\n\tWithSubEtcdAccount(user, \"bar\")(&Subscriber{\n\t\tendpoints: endpoints,\n\t})\n\taccount, ok := internal.GetAccount(endpoints)\n\tassert.True(t, ok)\n\tassert.Equal(t, user, account.User)\n\tassert.Equal(t, \"bar\", account.Pass)\n}\n\n// Path: core/errorx/atomicerror.go\npackage errorx\n\nimport \"sync/atomic\"\n\n// AtomicError defines an atomic error.\ntype AtomicError struct {\n\terr atomic.Value // error\n}\n\n// Set sets the error.\nfunc (ae *AtomicError) Set(err error) {\n\tif err != nil {\n\t\tae.err.Store(err)\n\t}\n}\n\n// Load returns the error.\nfunc (ae *AtomicError) Load() error {\n\tif v := ae.err.Load(); v != nil {\n\t\treturn v.(error)\n\t}\n\treturn nil\n}\n\n// Path: core/errorx/atomicerror_test.go\npackage errorx\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar errDummy = errors.New(\"hello\")\n\nfunc TestAtomicError(t *testing.T) {\n\tvar err AtomicError\n\terr.Set(errDummy)\n\tassert.Equal(t, errDummy, err.Load())\n}\n\nfunc TestAtomicErrorSetNil(t *testing.T) {\n\tvar (\n\t\terrNil error\n\t\terr    AtomicError\n\t)\n\terr.Set(errNil)\n\tassert.Equal(t, errNil, err.Load())\n}\n\nfunc TestAtomicErrorNil(t *testing.T) {\n\tvar err AtomicError\n\tassert.Nil(t, err.Load())\n}\n\nfunc BenchmarkAtomicError(b *testing.B) {\n\tvar aerr AtomicError\n\twg := sync.WaitGroup{}\n\n\tb.Run(\"Load\", func(b *testing.B) {\n\t\tvar done uint32\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tif atomic.LoadUint32(&done) != 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\taerr.Set(errDummy)\n\t\t\t\t\twg.Done()\n\t\t\t\t}()\n\t\t\t}\n\t\t}()\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\t_ = aerr.Load()\n\t\t}\n\t\tb.StopTimer()\n\t\tatomic.StoreUint32(&done, 1)\n\t\twg.Wait()\n\t})\n\tb.Run(\"Set\", func(b *testing.B) {\n\t\tvar done uint32\n\t\tgo func() {\n\t\t\tfor {\n\t\t\t\tif atomic.LoadUint32(&done) != 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\twg.Add(1)\n\t\t\t\tgo func() {\n\t\t\t\t\t_ = aerr.Load()\n\t\t\t\t\twg.Done()\n\t\t\t\t}()\n\t\t\t}\n\t\t}()\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\taerr.Set(errDummy)\n\t\t}\n\t\tb.StopTimer()\n\t\tatomic.StoreUint32(&done, 1)\n\t\twg.Wait()\n\t})\n}\n\n// Path: core/errorx/batcherror.go\npackage errorx\n\nimport (\n\t\"bytes\"\n\t\"sync\"\n)\n\ntype (\n\t// A BatchError is an error that can hold multiple errors.\n\tBatchError struct {\n\t\terrs errorArray\n\t\tlock sync.Mutex\n\t}\n\n\terrorArray []error\n)\n\n// Add adds errs to be, nil errors are ignored.\nfunc (be *BatchError) Add(errs ...error) {\n\tbe.lock.Lock()\n\tdefer be.lock.Unlock()\n\n\tfor _, err := range errs {\n\t\tif err != nil {\n\t\t\tbe.errs = append(be.errs, err)\n\t\t}\n\t}\n}\n\n// Err returns an error that represents all errors.\nfunc (be *BatchError) Err() error {\n\tbe.lock.Lock()\n\tdefer be.lock.Unlock()\n\n\tswitch len(be.errs) {\n\tcase 0:\n\t\treturn nil\n\tcase 1:\n\t\treturn be.errs[0]\n\tdefault:\n\t\treturn be.errs\n\t}\n}\n\n// NotNil checks if any error inside.\nfunc (be *BatchError) NotNil() bool {\n\tbe.lock.Lock()\n\tdefer be.lock.Unlock()\n\n\treturn len(be.errs) > 0\n}\n\n// Error returns a string that represents inside errors.\nfunc (ea errorArray) Error() string {\n\tvar buf bytes.Buffer\n\n\tfor i := range ea {\n\t\tif i > 0 {\n\t\t\tbuf.WriteByte('\\n')\n\t\t}\n\t\tbuf.WriteString(ea[i].Error())\n\t}\n\n\treturn buf.String()\n}\n\n// Path: core/errorx/batcherror_test.go\npackage errorx\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst (\n\terr1 = \"first error\"\n\terr2 = \"second error\"\n)\n\nfunc TestBatchErrorNil(t *testing.T) {\n\tvar batch BatchError\n\tassert.Nil(t, batch.Err())\n\tassert.False(t, batch.NotNil())\n\tbatch.Add(nil)\n\tassert.Nil(t, batch.Err())\n\tassert.False(t, batch.NotNil())\n}\n\nfunc TestBatchErrorNilFromFunc(t *testing.T) {\n\terr := func() error {\n\t\tvar be BatchError\n\t\treturn be.Err()\n\t}()\n\tassert.True(t, err == nil)\n}\n\nfunc TestBatchErrorOneError(t *testing.T) {\n\tvar batch BatchError\n\tbatch.Add(errors.New(err1))\n\tassert.NotNil(t, batch.Err())\n\tassert.Equal(t, err1, batch.Err().Error())\n\tassert.True(t, batch.NotNil())\n}\n\nfunc TestBatchErrorWithErrors(t *testing.T) {\n\tvar batch BatchError\n\tbatch.Add(errors.New(err1))\n\tbatch.Add(errors.New(err2))\n\tassert.NotNil(t, batch.Err())\n\tassert.Equal(t, fmt.Sprintf(\"%s\\n%s\", err1, err2), batch.Err().Error())\n\tassert.True(t, batch.NotNil())\n}\n\nfunc TestBatchErrorConcurrentAdd(t *testing.T) {\n\tconst count = 10000\n\tvar batch BatchError\n\tvar wg sync.WaitGroup\n\n\twg.Add(count)\n\tfor i := 0; i < count; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tbatch.Add(errors.New(err1))\n\t\t}()\n\t}\n\twg.Wait()\n\n\tassert.NotNil(t, batch.Err())\n\tassert.Equal(t, count, len(batch.errs))\n\tassert.True(t, batch.NotNil())\n}\n\n// Path: core/errorx/callchain.go\npackage errorx\n\n// Chain runs funs one by one until an error occurred.\nfunc Chain(fns ...func() error) error {\n\tfor _, fn := range fns {\n\t\tif err := fn(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// Path: core/errorx/callchain_test.go\npackage errorx\n\nimport (\n\t\"errors\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestChain(t *testing.T) {\n\terrDummy := errors.New(\"dummy\")\n\tassert.Nil(t, Chain(func() error {\n\t\treturn nil\n\t}, func() error {\n\t\treturn nil\n\t}))\n\tassert.Equal(t, errDummy, Chain(func() error {\n\t\treturn errDummy\n\t}, func() error {\n\t\treturn nil\n\t}))\n\tassert.Equal(t, errDummy, Chain(func() error {\n\t\treturn nil\n\t}, func() error {\n\t\treturn errDummy\n\t}))\n}\n\n// Path: core/errorx/wrap.go\npackage errorx\n\nimport \"fmt\"\n\n// Wrap returns an error that wraps err with given message.\nfunc Wrap(err error, message string) error {\n\tif err == nil {\n\t\treturn nil\n\t}\n\n\treturn fmt.Errorf(\"%s: %w\", message, err)\n}\n\n// Wrapf returns an error that wraps err with given format and args.\nfunc Wrapf(err error, format string, args ...any) error {\n\tif err == nil {\n\t\treturn nil\n\t}\n\n\treturn fmt.Errorf(\"%s: %w\", fmt.Sprintf(format, args...), err)\n}\n\n// Path: core/errorx/wrap_test.go\npackage errorx\n\nimport (\n\t\"errors\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestWrap(t *testing.T) {\n\tassert.Nil(t, Wrap(nil, \"test\"))\n\tassert.Equal(t, \"foo: bar\", Wrap(errors.New(\"bar\"), \"foo\").Error())\n\n\terr := errors.New(\"foo\")\n\tassert.True(t, errors.Is(Wrap(err, \"bar\"), err))\n}\n\nfunc TestWrapf(t *testing.T) {\n\tassert.Nil(t, Wrapf(nil, \"%s\", \"test\"))\n\tassert.Equal(t, \"foo bar: quz\", Wrapf(errors.New(\"quz\"), \"foo %s\", \"bar\").Error())\n\n\terr := errors.New(\"foo\")\n\tassert.True(t, errors.Is(Wrapf(err, \"foo %s\", \"bar\"), err))\n}\n\n// Path: core/executors/bulkexecutor.go\npackage executors\n\nimport \"time\"\n\nconst defaultBulkTasks = 1000\n\ntype (\n\t// BulkOption defines the method to customize a BulkExecutor.\n\tBulkOption func(options *bulkOptions)\n\n\t// A BulkExecutor is an executor that can execute tasks on either requirement meets:\n\t// 1. up to given size of tasks\n\t// 2. flush interval time elapsed\n\tBulkExecutor struct {\n\t\texecutor  *PeriodicalExecutor\n\t\tcontainer *bulkContainer\n\t}\n\n\tbulkOptions struct {\n\t\tcachedTasks   int\n\t\tflushInterval time.Duration\n\t}\n)\n\n// NewBulkExecutor returns a BulkExecutor.\nfunc NewBulkExecutor(execute Execute, opts ...BulkOption) *BulkExecutor {\n\toptions := newBulkOptions()\n\tfor _, opt := range opts {\n\t\topt(&options)\n\t}\n\n\tcontainer := &bulkContainer{\n\t\texecute:  execute,\n\t\tmaxTasks: options.cachedTasks,\n\t}\n\texecutor := &BulkExecutor{\n\t\texecutor:  NewPeriodicalExecutor(options.flushInterval, container),\n\t\tcontainer: container,\n\t}\n\n\treturn executor\n}\n\n// Add adds task into be.\nfunc (be *BulkExecutor) Add(task any) error {\n\tbe.executor.Add(task)\n\treturn nil\n}\n\n// Flush forces be to flush and execute tasks.\nfunc (be *BulkExecutor) Flush() {\n\tbe.executor.Flush()\n}\n\n// Wait waits be to done with the task execution.\nfunc (be *BulkExecutor) Wait() {\n\tbe.executor.Wait()\n}\n\n// WithBulkTasks customizes a BulkExecutor with given tasks limit.\nfunc WithBulkTasks(tasks int) BulkOption {\n\treturn func(options *bulkOptions) {\n\t\toptions.cachedTasks = tasks\n\t}\n}\n\n// WithBulkInterval customizes a BulkExecutor with given flush interval.\nfunc WithBulkInterval(duration time.Duration) BulkOption {\n\treturn func(options *bulkOptions) {\n\t\toptions.flushInterval = duration\n\t}\n}\n\nfunc newBulkOptions() bulkOptions {\n\treturn bulkOptions{\n\t\tcachedTasks:   defaultBulkTasks,\n\t\tflushInterval: defaultFlushInterval,\n\t}\n}\n\ntype bulkContainer struct {\n\ttasks    []any\n\texecute  Execute\n\tmaxTasks int\n}\n\nfunc (bc *bulkContainer) AddTask(task any) bool {\n\tbc.tasks = append(bc.tasks, task)\n\treturn len(bc.tasks) >= bc.maxTasks\n}\n\nfunc (bc *bulkContainer) Execute(tasks any) {\n\tvals := tasks.([]any)\n\tbc.execute(vals)\n}\n\nfunc (bc *bulkContainer) RemoveAll() any {\n\ttasks := bc.tasks\n\tbc.tasks = nil\n\treturn tasks\n}\n\n// Path: core/executors/bulkexecutor_test.go\npackage executors\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestBulkExecutor(t *testing.T) {\n\tvar values []int\n\tvar lock sync.Mutex\n\n\texecutor := NewBulkExecutor(func(items []any) {\n\t\tlock.Lock()\n\t\tvalues = append(values, len(items))\n\t\tlock.Unlock()\n\t}, WithBulkTasks(10), WithBulkInterval(time.Minute))\n\n\tfor i := 0; i < 50; i++ {\n\t\texecutor.Add(1)\n\t\ttime.Sleep(time.Millisecond)\n\t}\n\n\tlock.Lock()\n\tassert.True(t, len(values) > 0)\n\t// ignore last value\n\tfor i := 0; i < len(values); i++ {\n\t\tassert.Equal(t, 10, values[i])\n\t}\n\tlock.Unlock()\n}\n\nfunc TestBulkExecutorFlushInterval(t *testing.T) {\n\tconst (\n\t\tcaches = 10\n\t\tsize   = 5\n\t)\n\tvar wait sync.WaitGroup\n\n\twait.Add(1)\n\texecutor := NewBulkExecutor(func(items []any) {\n\t\tassert.Equal(t, size, len(items))\n\t\twait.Done()\n\t}, WithBulkTasks(caches), WithBulkInterval(time.Millisecond*100))\n\n\tfor i := 0; i < size; i++ {\n\t\texecutor.Add(1)\n\t}\n\n\twait.Wait()\n}\n\nfunc TestBulkExecutorEmpty(t *testing.T) {\n\tNewBulkExecutor(func(items []any) {\n\t\tassert.Fail(t, \"should not called\")\n\t}, WithBulkTasks(10), WithBulkInterval(time.Millisecond))\n\ttime.Sleep(time.Millisecond * 100)\n}\n\nfunc TestBulkExecutorFlush(t *testing.T) {\n\tconst (\n\t\tcaches = 10\n\t\ttasks  = 5\n\t)\n\n\tvar wait sync.WaitGroup\n\twait.Add(1)\n\tbe := NewBulkExecutor(func(items []any) {\n\t\tassert.Equal(t, tasks, len(items))\n\t\twait.Done()\n\t}, WithBulkTasks(caches), WithBulkInterval(time.Minute))\n\tfor i := 0; i < tasks; i++ {\n\t\tbe.Add(1)\n\t}\n\tbe.Flush()\n\twait.Wait()\n}\n\nfunc TestBulkExecutorFlushSlowTasks(t *testing.T) {\n\tconst total = 1500\n\tlock := new(sync.Mutex)\n\tresult := make([]any, 0, 10000)\n\texec := NewBulkExecutor(func(tasks []any) {\n\t\ttime.Sleep(time.Millisecond * 100)\n\t\tlock.Lock()\n\t\tdefer lock.Unlock()\n\t\tresult = append(result, tasks...)\n\t}, WithBulkTasks(1000))\n\tfor i := 0; i < total; i++ {\n\t\tassert.Nil(t, exec.Add(i))\n\t}\n\n\texec.Flush()\n\texec.Wait()\n\tassert.Equal(t, total, len(result))\n}\n\nfunc BenchmarkBulkExecutor(b *testing.B) {\n\tb.ReportAllocs()\n\n\tbe := NewBulkExecutor(func(tasks []any) {\n\t\ttime.Sleep(time.Millisecond * time.Duration(len(tasks)))\n\t})\n\tfor i := 0; i < b.N; i++ {\n\t\ttime.Sleep(time.Microsecond * 200)\n\t\tbe.Add(1)\n\t}\n\tbe.Flush()\n}\n\n// Path: core/executors/chunkexecutor.go\npackage executors\n\nimport \"time\"\n\nconst defaultChunkSize = 1024 * 1024 // 1M\n\ntype (\n\t// ChunkOption defines the method to customize a ChunkExecutor.\n\tChunkOption func(options *chunkOptions)\n\n\t// A ChunkExecutor is an executor to execute tasks when either requirement meets:\n\t// 1. up to given chunk size\n\t// 2. flush interval elapsed\n\tChunkExecutor struct {\n\t\texecutor  *PeriodicalExecutor\n\t\tcontainer *chunkContainer\n\t}\n\n\tchunkOptions struct {\n\t\tchunkSize     int\n\t\tflushInterval time.Duration\n\t}\n)\n\n// NewChunkExecutor returns a ChunkExecutor.\nfunc NewChunkExecutor(execute Execute, opts ...ChunkOption) *ChunkExecutor {\n\toptions := newChunkOptions()\n\tfor _, opt := range opts {\n\t\topt(&options)\n\t}\n\n\tcontainer := &chunkContainer{\n\t\texecute:      execute,\n\t\tmaxChunkSize: options.chunkSize,\n\t}\n\texecutor := &ChunkExecutor{\n\t\texecutor:  NewPeriodicalExecutor(options.flushInterval, container),\n\t\tcontainer: container,\n\t}\n\n\treturn executor\n}\n\n// Add adds task with given chunk size into ce.\nfunc (ce *ChunkExecutor) Add(task any, size int) error {\n\tce.executor.Add(chunk{\n\t\tval:  task,\n\t\tsize: size,\n\t})\n\treturn nil\n}\n\n// Flush forces ce to flush and execute tasks.\nfunc (ce *ChunkExecutor) Flush() {\n\tce.executor.Flush()\n}\n\n// Wait waits the execution to be done.\nfunc (ce *ChunkExecutor) Wait() {\n\tce.executor.Wait()\n}\n\n// WithChunkBytes customizes a ChunkExecutor with the given chunk size.\nfunc WithChunkBytes(size int) ChunkOption {\n\treturn func(options *chunkOptions) {\n\t\toptions.chunkSize = size\n\t}\n}\n\n// WithFlushInterval customizes a ChunkExecutor with the given flush interval.\nfunc WithFlushInterval(duration time.Duration) ChunkOption {\n\treturn func(options *chunkOptions) {\n\t\toptions.flushInterval = duration\n\t}\n}\n\nfunc newChunkOptions() chunkOptions {\n\treturn chunkOptions{\n\t\tchunkSize:     defaultChunkSize,\n\t\tflushInterval: defaultFlushInterval,\n\t}\n}\n\ntype chunkContainer struct {\n\ttasks        []any\n\texecute      Execute\n\tsize         int\n\tmaxChunkSize int\n}\n\nfunc (bc *chunkContainer) AddTask(task any) bool {\n\tck := task.(chunk)\n\tbc.tasks = append(bc.tasks, ck.val)\n\tbc.size += ck.size\n\treturn bc.size >= bc.maxChunkSize\n}\n\nfunc (bc *chunkContainer) Execute(tasks any) {\n\tvals := tasks.([]any)\n\tbc.execute(vals)\n}\n\nfunc (bc *chunkContainer) RemoveAll() any {\n\ttasks := bc.tasks\n\tbc.tasks = nil\n\tbc.size = 0\n\treturn tasks\n}\n\ntype chunk struct {\n\tval  any\n\tsize int\n}\n\n// Path: core/executors/chunkexecutor_test.go\npackage executors\n\nimport (\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestChunkExecutor(t *testing.T) {\n\tvar values []int\n\tvar lock sync.Mutex\n\n\texecutor := NewChunkExecutor(func(items []any) {\n\t\tlock.Lock()\n\t\tvalues = append(values, len(items))\n\t\tlock.Unlock()\n\t}, WithChunkBytes(10), WithFlushInterval(time.Minute))\n\n\tfor i := 0; i < 50; i++ {\n\t\texecutor.Add(1, 1)\n\t\ttime.Sleep(time.Millisecond)\n\t}\n\n\tlock.Lock()\n\tassert.True(t, len(values) > 0)\n\t// ignore last value\n\tfor i := 0; i < len(values); i++ {\n\t\tassert.Equal(t, 10, values[i])\n\t}\n\tlock.Unlock()\n}\n\nfunc TestChunkExecutorFlushInterval(t *testing.T) {\n\tconst (\n\t\tcaches = 10\n\t\tsize   = 5\n\t)\n\tvar wait sync.WaitGroup\n\n\twait.Add(1)\n\texecutor := NewChunkExecutor(func(items []any) {\n\t\tassert.Equal(t, size, len(items))\n\t\twait.Done()\n\t}, WithChunkBytes(caches), WithFlushInterval(time.Millisecond*100))\n\n\tfor i := 0; i < size; i++ {\n\t\texecutor.Add(1, 1)\n\t}\n\n\twait.Wait()\n}\n\nfunc TestChunkExecutorEmpty(t *testing.T) {\n\texecutor := NewChunkExecutor(func(items []any) {\n\t\tassert.Fail(t, \"should not called\")\n\t}, WithChunkBytes(10), WithFlushInterval(time.Millisecond))\n\ttime.Sleep(time.Millisecond * 100)\n\texecutor.Wait()\n}\n\nfunc TestChunkExecutorFlush(t *testing.T) {\n\tconst (\n\t\tcaches = 10\n\t\ttasks  = 5\n\t)\n\n\tvar wait sync.WaitGroup\n\twait.Add(1)\n\tbe := NewChunkExecutor(func(items []any) {\n\t\tassert.Equal(t, tasks, len(items))\n\t\twait.Done()\n\t}, WithChunkBytes(caches), WithFlushInterval(time.Minute))\n\tfor i := 0; i < tasks; i++ {\n\t\tbe.Add(1, 1)\n\t}\n\tbe.Flush()\n\twait.Wait()\n}\n\nfunc BenchmarkChunkExecutor(b *testing.B) {\n\tb.ReportAllocs()\n\n\tbe := NewChunkExecutor(func(tasks []any) {\n\t\ttime.Sleep(time.Millisecond * time.Duration(len(tasks)))\n\t})\n\tfor i := 0; i < b.N; i++ {\n\t\ttime.Sleep(time.Microsecond * 200)\n\t\tbe.Add(1, 1)\n\t}\n\tbe.Flush()\n}\n\n// Path: core/executors/delayexecutor.go\npackage executors\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/threading\"\n)\n\n// A DelayExecutor delays a tasks on given delay interval.\ntype DelayExecutor struct {\n\tfn        func()\n\tdelay     time.Duration\n\ttriggered bool\n\tlock      sync.Mutex\n}\n\n// NewDelayExecutor returns a DelayExecutor with given fn and delay.\nfunc NewDelayExecutor(fn func(), delay time.Duration) *DelayExecutor {\n\treturn &DelayExecutor{\n\t\tfn:    fn,\n\t\tdelay: delay,\n\t}\n}\n\n// Trigger triggers the task to be executed after given delay, safe to trigger more than once.\nfunc (de *DelayExecutor) Trigger() {\n\tde.lock.Lock()\n\tdefer de.lock.Unlock()\n\n\tif de.triggered {\n\t\treturn\n\t}\n\n\tde.triggered = true\n\tthreading.GoSafe(func() {\n\t\ttimer := time.NewTimer(de.delay)\n\t\tdefer timer.Stop()\n\t\t<-timer.C\n\n\t\t// set triggered to false before calling fn to ensure no triggers are missed.\n\t\tde.lock.Lock()\n\t\tde.triggered = false\n\t\tde.lock.Unlock()\n\t\tde.fn()\n\t})\n}\n\n// Path: core/executors/delayexecutor_test.go\npackage executors\n\nimport (\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\n...\n", "needle_token_start": 5491, "needle_token_end": 5619, "code_context_ntokens": 15250, "output": ["```go\nfunc NewSubscriber(endpoints []string, key string, opts ...SubOption) (*Subscriber, error) {\n\tsub := &Subscriber{\n\t\tendpoints: endpoints,\n\t}\n\tfor _, opt := range opts {\n\t\topt(sub)\n\t}\n\tsub.items = newContainer(sub.exclusive)\n\n\tif err := internal.GetRegistry().Monitor(endpoints, key, sub.items); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn sub, nil\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "RunTest", "language": "go", "path": "core/stores/dbtest/sql.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to facilitate the testing of database-related operations by providing a simulated database environment. This allows developers to verify the behavior of database interactions without the need for a real database connection.\n2. **Input**: The function takes a testing framework pointer and a user-defined function. The user-defined function should include the logic to be tested using the mock database.\n3. **Output**: There is no direct output from the function itself; however, it indirectly affects the testing process by asserting whether the database interactions meet the expectations set in the test.\n4. **Procedure**: \n   - Initializes a mock database connection.\n   - Executes the user-defined function, passing the mock database and its control interface.\n   - After the function execution, it checks if all predefined expectations on the mock database were met.\n   - Reports errors if the expectations were not satisfied or if there was an issue in setting up the mock database.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/stores/dbtest/sql.go\npackage dbtest\n\nimport (\n\t\"database/sql\"\n\t\"testing\"\n\n\t\"github.com/DATA-DOG/go-sqlmock\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\n// RunTest runs a test function with a mock database.\n\nfunc RunTest(t *testing.T, fn func(db *sql.DB, mock sqlmock.Sqlmock)) {\n\tdb, mock, err := sqlmock.New()\n\tif err != nil {\n\t\tt.Fatalf(\"an error '%s' was not expected when opening a stub database connection\", err)\n\t}\n\tdefer func() {\n\t\t_ = db.Close()\n\t}()\n\n\tfn(db, mock)\n\n\tif err = mock.ExpectationsWereMet(); err != nil {\n\t\tt.Errorf(\"there were unfulfilled expectations: %s\", err)\n\t}\n}\n\n// RunTxTest runs a test function with a mock database in a transaction.\nfunc RunTxTest(t *testing.T, f func(tx *sql.Tx, mock sqlmock.Sqlmock)) {\n\tRunTest(t, func(db *sql.DB, mock sqlmock.Sqlmock) {\n\t\tmock.ExpectBegin()\n\t\ttx, err := db.Begin()\n\t\tif assert.NoError(t, err) {\n\t\t\tf(tx, mock)\n\t\t}\n\t})\n}\n\n// Path: core/stores/sqlx/errors.go\npackage sqlx\n\nimport (\n\t\"database/sql\"\n\t\"errors\"\n)\n\nvar (\n\t// ErrNotFound is an alias of sql.ErrNoRows\n\tErrNotFound = sql.ErrNoRows\n\n\terrCantNestTx    = errors.New(\"cannot nest transactions\")\n\terrNoRawDBFromTx = errors.New(\"cannot get raw db from transaction\")\n)\n\n// Path: core/lang/lang.go\npackage lang\n\nimport (\n\t\"fmt\"\n\t\"reflect\"\n\t\"strconv\"\n)\n\n// Placeholder is a placeholder object that can be used globally.\nvar Placeholder PlaceholderType\n\ntype (\n\t// AnyType can be used to hold any type.\n\tAnyType = any\n\t// PlaceholderType represents a placeholder type.\n\tPlaceholderType = struct{}\n)\n\n// Repr returns the string representation of v.\nfunc Repr(v any) string {\n\tif v == nil {\n\t\treturn \"\"\n\t}\n\n\t// if func (v *Type) String() string, we can't use Elem()\n\tswitch vt := v.(type) {\n\tcase fmt.Stringer:\n\t\treturn vt.String()\n\t}\n\n\tval := reflect.ValueOf(v)\n\tfor val.Kind() == reflect.Ptr && !val.IsNil() {\n\t\tval = val.Elem()\n\t}\n\n\treturn reprOfValue(val)\n}\n\nfunc reprOfValue(val reflect.Value) string {\n\tswitch vt := val.Interface().(type) {\n\tcase bool:\n\t\treturn strconv.FormatBool(vt)\n\tcase error:\n\t\treturn vt.Error()\n\tcase float32:\n\t\treturn strconv.FormatFloat(float64(vt), 'f', -1, 32)\n\tcase float64:\n\t\treturn strconv.FormatFloat(vt, 'f', -1, 64)\n\tcase fmt.Stringer:\n\t\treturn vt.String()\n\tcase int:\n\t\treturn strconv.Itoa(vt)\n\tcase int8:\n\t\treturn strconv.Itoa(int(vt))\n\tcase int16:\n\t\treturn strconv.Itoa(int(vt))\n\tcase int32:\n\t\treturn strconv.Itoa(int(vt))\n\tcase int64:\n\t\treturn strconv.FormatInt(vt, 10)\n\tcase string:\n\t\treturn vt\n\tcase uint:\n\t\treturn strconv.FormatUint(uint64(vt), 10)\n\tcase uint8:\n\t\treturn strconv.FormatUint(uint64(vt), 10)\n\tcase uint16:\n\t\treturn strconv.FormatUint(uint64(vt), 10)\n\tcase uint32:\n\t\treturn strconv.FormatUint(uint64(vt), 10)\n\tcase uint64:\n\t\treturn strconv.FormatUint(vt, 10)\n\tcase []byte:\n\t\treturn string(vt)\n\tdefault:\n\t\treturn fmt.Sprint(val.Interface())\n\t}\n}\n\n// Path: core/stringx/strings.go\npackage stringx\n\nimport (\n\t\"errors\"\n\t\"unicode\"\n\n\t\"github.com/zeromicro/go-zero/core/lang\"\n)\n\nvar (\n\t// ErrInvalidStartPosition is an error that indicates the start position is invalid.\n\tErrInvalidStartPosition = errors.New(\"start position is invalid\")\n\t// ErrInvalidStopPosition is an error that indicates the stop position is invalid.\n\tErrInvalidStopPosition = errors.New(\"stop position is invalid\")\n)\n\n// Contains checks if str is in list.\nfunc Contains(list []string, str string) bool {\n\tfor _, each := range list {\n\t\tif each == str {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\n// Filter filters chars from s with given filter function.\nfunc Filter(s string, filter func(r rune) bool) string {\n\tvar n int\n\tchars := []rune(s)\n\tfor i, x := range chars {\n\t\tif n < i {\n\t\t\tchars[n] = x\n\t\t}\n\t\tif !filter(x) {\n\t\t\tn++\n\t\t}\n\t}\n\n\treturn string(chars[:n])\n}\n\n// FirstN returns first n runes from s.\nfunc FirstN(s string, n int, ellipsis ...string) string {\n\tvar i int\n\n\tfor j := range s {\n\t\tif i == n {\n\t\t\tret := s[:j]\n\t\t\tfor _, each := range ellipsis {\n\t\t\t\tret += each\n\t\t\t}\n\t\t\treturn ret\n\t\t}\n\t\ti++\n\t}\n\n\treturn s\n}\n\n// HasEmpty checks if there are empty strings in args.\nfunc HasEmpty(args ...string) bool {\n\tfor _, arg := range args {\n\t\tif len(arg) == 0 {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\n// Join joins any number of elements into a single string, separating them with given sep.\n// Empty elements are ignored. However, if the argument list is empty or all its elements are empty,\n// Join returns an empty string.\nfunc Join(sep byte, elem ...string) string {\n\tvar size int\n\tfor _, e := range elem {\n\t\tsize += len(e)\n\t}\n\tif size == 0 {\n\t\treturn \"\"\n\t}\n\n\tbuf := make([]byte, 0, size+len(elem)-1)\n\tfor _, e := range elem {\n\t\tif len(e) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tif len(buf) > 0 {\n\t\t\tbuf = append(buf, sep)\n\t\t}\n\t\tbuf = append(buf, e...)\n\t}\n\n\treturn string(buf)\n}\n\n// NotEmpty checks if all strings are not empty in args.\nfunc NotEmpty(args ...string) bool {\n\treturn !HasEmpty(args...)\n}\n\n// Remove removes given strs from strings.\nfunc Remove(strings []string, strs ...string) []string {\n\tout := append([]string(nil), strings...)\n\n\tfor _, str := range strs {\n\t\tvar n int\n\t\tfor _, v := range out {\n\t\t\tif v != str {\n\t\t\t\tout[n] = v\n\t\t\t\tn++\n\t\t\t}\n\t\t}\n\t\tout = out[:n]\n\t}\n\n\treturn out\n}\n\n// Reverse reverses s.\nfunc Reverse(s string) string {\n\trunes := []rune(s)\n\n\tfor from, to := 0, len(runes)-1; from < to; from, to = from+1, to-1 {\n\t\trunes[from], runes[to] = runes[to], runes[from]\n\t}\n\n\treturn string(runes)\n}\n\n// Substr returns runes between start and stop [start, stop)\n// regardless of the chars are ascii or utf8.\nfunc Substr(str string, start, stop int) (string, error) {\n\trs := []rune(str)\n\tlength := len(rs)\n\n\tif start < 0 || start > length {\n\t\treturn \"\", ErrInvalidStartPosition\n\t}\n\n\tif stop < 0 || stop > length {\n\t\treturn \"\", ErrInvalidStopPosition\n\t}\n\n\treturn string(rs[start:stop]), nil\n}\n\n// TakeOne returns valid string if not empty or later one.\nfunc TakeOne(valid, or string) string {\n\tif len(valid) > 0 {\n\t\treturn valid\n\t}\n\n\treturn or\n}\n\n// TakeWithPriority returns the first not empty result from fns.\nfunc TakeWithPriority(fns ...func() string) string {\n\tfor _, fn := range fns {\n\t\tval := fn()\n\t\tif len(val) > 0 {\n\t\t\treturn val\n\t\t}\n\t}\n\n\treturn \"\"\n}\n\n// ToCamelCase returns the string that converts the first letter to lowercase.\nfunc ToCamelCase(s string) string {\n\tfor i, v := range s {\n\t\treturn string(unicode.ToLower(v)) + s[i+1:]\n\t}\n\n\treturn \"\"\n}\n\n// Union merges the strings in first and second.\nfunc Union(first, second []string) []string {\n\tset := make(map[string]lang.PlaceholderType)\n\n\tfor _, each := range first {\n\t\tset[each] = lang.Placeholder\n\t}\n\tfor _, each := range second {\n\t\tset[each] = lang.Placeholder\n\t}\n\n\tmerged := make([]string, 0, len(set))\n\tfor k := range set {\n\t\tmerged = append(merged, k)\n\t}\n\n\treturn merged\n}\n\n// Path: core/breaker/breaker.go\npackage breaker\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/mathx\"\n\t\"github.com/zeromicro/go-zero/core/proc\"\n\t\"github.com/zeromicro/go-zero/core/stat\"\n\t\"github.com/zeromicro/go-zero/core/stringx\"\n)\n\nconst (\n\tnumHistoryReasons = 5\n\ttimeFormat        = \"15:04:05\"\n)\n\n// ErrServiceUnavailable is returned when the Breaker state is open.\nvar ErrServiceUnavailable = errors.New(\"circuit breaker is open\")\n\ntype (\n\t// Acceptable is the func to check if the error can be accepted.\n\tAcceptable func(err error) bool\n\n\t// A Breaker represents a circuit breaker.\n\tBreaker interface {\n\t\t// Name returns the name of the Breaker.\n\t\tName() string\n\n\t\t// Allow checks if the request is allowed.\n\t\t// If allowed, a promise will be returned,\n\t\t// otherwise ErrServiceUnavailable will be returned as the error.\n\t\t// The caller needs to call promise.Accept() on success,\n\t\t// or call promise.Reject() on failure.\n\t\tAllow() (Promise, error)\n\t\t// AllowCtx checks if the request is allowed when ctx isn't done.\n\t\tAllowCtx(ctx context.Context) (Promise, error)\n\n\t\t// Do runs the given request if the Breaker accepts it.\n\t\t// Do returns an error instantly if the Breaker rejects the request.\n\t\t// If a panic occurs in the request, the Breaker handles it as an error\n\t\t// and causes the same panic again.\n\t\tDo(req func() error) error\n\t\t// DoCtx runs the given request if the Breaker accepts it when ctx isn't done.\n\t\tDoCtx(ctx context.Context, req func() error) error\n\n\t\t// DoWithAcceptable runs the given request if the Breaker accepts it.\n\t\t// DoWithAcceptable returns an error instantly if the Breaker rejects the request.\n\t\t// If a panic occurs in the request, the Breaker handles it as an error\n\t\t// and causes the same panic again.\n\t\t// acceptable checks if it's a successful call, even if the error is not nil.\n\t\tDoWithAcceptable(req func() error, acceptable Acceptable) error\n\t\t// DoWithAcceptableCtx runs the given request if the Breaker accepts it when ctx isn't done.\n\t\tDoWithAcceptableCtx(ctx context.Context, req func() error, acceptable Acceptable) error\n\n\t\t// DoWithFallback runs the given request if the Breaker accepts it.\n\t\t// DoWithFallback runs the fallback if the Breaker rejects the request.\n\t\t// If a panic occurs in the request, the Breaker handles it as an error\n\t\t// and causes the same panic again.\n\t\tDoWithFallback(req func() error, fallback Fallback) error\n\t\t// DoWithFallbackCtx runs the given request if the Breaker accepts it when ctx isn't done.\n\t\tDoWithFallbackCtx(ctx context.Context, req func() error, fallback Fallback) error\n\n\t\t// DoWithFallbackAcceptable runs the given request if the Breaker accepts it.\n\t\t// DoWithFallbackAcceptable runs the fallback if the Breaker rejects the request.\n\t\t// If a panic occurs in the request, the Breaker handles it as an error\n\t\t// and causes the same panic again.\n\t\t// acceptable checks if it's a successful call, even if the error is not nil.\n\t\tDoWithFallbackAcceptable(req func() error, fallback Fallback, acceptable Acceptable) error\n\t\t// DoWithFallbackAcceptableCtx runs the given request if the Breaker accepts it when ctx isn't done.\n\t\tDoWithFallbackAcceptableCtx(ctx context.Context, req func() error, fallback Fallback,\n\t\t\tacceptable Acceptable) error\n\t}\n\n\t// Fallback is the func to be called if the request is rejected.\n\tFallback func(err error) error\n\n\t// Option defines the method to customize a Breaker.\n\tOption func(breaker *circuitBreaker)\n\n\t// Promise interface defines the callbacks that returned by Breaker.Allow.\n\tPromise interface {\n\t\t// Accept tells the Breaker that the call is successful.\n\t\tAccept()\n\t\t// Reject tells the Breaker that the call is failed.\n\t\tReject(reason string)\n\t}\n\n\tinternalPromise interface {\n\t\tAccept()\n\t\tReject()\n\t}\n\n\tcircuitBreaker struct {\n\t\tname string\n\t\tthrottle\n\t}\n\n\tinternalThrottle interface {\n\t\tallow() (internalPromise, error)\n\t\tdoReq(req func() error, fallback Fallback, acceptable Acceptable) error\n\t}\n\n\tthrottle interface {\n\t\tallow() (Promise, error)\n\t\tdoReq(req func() error, fallback Fallback, acceptable Acceptable) error\n\t}\n)\n\n// NewBreaker returns a Breaker object.\n// opts can be used to customize the Breaker.\nfunc NewBreaker(opts ...Option) Breaker {\n\tvar b circuitBreaker\n\tfor _, opt := range opts {\n\t\topt(&b)\n\t}\n\tif len(b.name) == 0 {\n\t\tb.name = stringx.Rand()\n\t}\n\tb.throttle = newLoggedThrottle(b.name, newGoogleBreaker())\n\n\treturn &b\n}\n\nfunc (cb *circuitBreaker) Allow() (Promise, error) {\n\treturn cb.throttle.allow()\n}\n\nfunc (cb *circuitBreaker) AllowCtx(ctx context.Context) (Promise, error) {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn nil, ctx.Err()\n\tdefault:\n\t\treturn cb.Allow()\n\t}\n}\n\nfunc (cb *circuitBreaker) Do(req func() error) error {\n\treturn cb.throttle.doReq(req, nil, defaultAcceptable)\n}\n\nfunc (cb *circuitBreaker) DoCtx(ctx context.Context, req func() error) error {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tdefault:\n\t\treturn cb.Do(req)\n\t}\n}\n\nfunc (cb *circuitBreaker) DoWithAcceptable(req func() error, acceptable Acceptable) error {\n\treturn cb.throttle.doReq(req, nil, acceptable)\n}\n\nfunc (cb *circuitBreaker) DoWithAcceptableCtx(ctx context.Context, req func() error,\n\tacceptable Acceptable) error {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tdefault:\n\t\treturn cb.DoWithAcceptable(req, acceptable)\n\t}\n}\n\nfunc (cb *circuitBreaker) DoWithFallback(req func() error, fallback Fallback) error {\n\treturn cb.throttle.doReq(req, fallback, defaultAcceptable)\n}\n\nfunc (cb *circuitBreaker) DoWithFallbackCtx(ctx context.Context, req func() error,\n\tfallback Fallback) error {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tdefault:\n\t\treturn cb.DoWithFallback(req, fallback)\n\t}\n}\n\nfunc (cb *circuitBreaker) DoWithFallbackAcceptable(req func() error, fallback Fallback,\n\tacceptable Acceptable) error {\n\treturn cb.throttle.doReq(req, fallback, acceptable)\n}\n\nfunc (cb *circuitBreaker) DoWithFallbackAcceptableCtx(ctx context.Context, req func() error,\n\tfallback Fallback, acceptable Acceptable) error {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tdefault:\n\t\treturn cb.DoWithFallbackAcceptable(req, fallback, acceptable)\n\t}\n}\n\nfunc (cb *circuitBreaker) Name() string {\n\treturn cb.name\n}\n\n// WithName returns a function to set the name of a Breaker.\nfunc WithName(name string) Option {\n\treturn func(b *circuitBreaker) {\n\t\tb.name = name\n\t}\n}\n\nfunc defaultAcceptable(err error) bool {\n\treturn err == nil\n}\n\ntype loggedThrottle struct {\n\tname string\n\tinternalThrottle\n\terrWin *errorWindow\n}\n\nfunc newLoggedThrottle(name string, t internalThrottle) loggedThrottle {\n\treturn loggedThrottle{\n\t\tname:             name,\n\t\tinternalThrottle: t,\n\t\terrWin:           new(errorWindow),\n\t}\n}\n\nfunc (lt loggedThrottle) allow() (Promise, error) {\n\tpromise, err := lt.internalThrottle.allow()\n\treturn promiseWithReason{\n\t\tpromise: promise,\n\t\terrWin:  lt.errWin,\n\t}, lt.logError(err)\n}\n\nfunc (lt loggedThrottle) doReq(req func() error, fallback Fallback, acceptable Acceptable) error {\n\treturn lt.logError(lt.internalThrottle.doReq(req, fallback, func(err error) bool {\n\t\taccept := acceptable(err)\n\t\tif !accept && err != nil {\n\t\t\tlt.errWin.add(err.Error())\n\t\t}\n\t\treturn accept\n\t}))\n}\n\nfunc (lt loggedThrottle) logError(err error) error {\n\tif errors.Is(err, ErrServiceUnavailable) {\n\t\t// if circuit open, not possible to have empty error window\n\t\tstat.Report(fmt.Sprintf(\n\t\t\t\"proc(%s/%d), callee: %s, breaker is open and requests dropped\\nlast errors:\\n%s\",\n\t\t\tproc.ProcessName(), proc.Pid(), lt.name, lt.errWin))\n\t}\n\n\treturn err\n}\n\ntype errorWindow struct {\n\treasons [numHistoryReasons]string\n\tindex   int\n\tcount   int\n\tlock    sync.Mutex\n}\n\nfunc (ew *errorWindow) add(reason string) {\n\tew.lock.Lock()\n\tew.reasons[ew.index] = fmt.Sprintf(\"%s %s\", time.Now().Format(timeFormat), reason)\n\tew.index = (ew.index + 1) % numHistoryReasons\n\tew.count = mathx.MinInt(ew.count+1, numHistoryReasons)\n\tew.lock.Unlock()\n}\n\nfunc (ew *errorWindow) String() string {\n\tvar reasons []string\n\n\tew.lock.Lock()\n\t// reverse order\n\tfor i := ew.index - 1; i >= ew.index-ew.count; i-- {\n\t\treasons = append(reasons, ew.reasons[(i+numHistoryReasons)%numHistoryReasons])\n\t}\n\tew.lock.Unlock()\n\n\treturn strings.Join(reasons, \"\\n\")\n}\n\ntype promiseWithReason struct {\n\tpromise internalPromise\n\terrWin  *errorWindow\n}\n\nfunc (p promiseWithReason) Accept() {\n\tp.promise.Accept()\n}\n\nfunc (p promiseWithReason) Reject(reason string) {\n\tp.errWin.add(reason)\n\tp.promise.Reject()\n}\n\n// Path: core/stores/redis/conf.go\npackage redis\n\nimport (\n\t\"errors\"\n\t\"time\"\n)\n\nvar (\n\t// ErrEmptyHost is an error that indicates no redis host is set.\n\tErrEmptyHost = errors.New(\"empty redis host\")\n\t// ErrEmptyType is an error that indicates no redis type is set.\n\tErrEmptyType = errors.New(\"empty redis type\")\n\t// ErrEmptyKey is an error that indicates no redis key is set.\n\tErrEmptyKey = errors.New(\"empty redis key\")\n)\n\ntype (\n\t// A RedisConf is a redis config.\n\tRedisConf struct {\n\t\tHost     string\n\t\tType     string `json:\",default=node,options=node|cluster\"`\n\t\tPass     string `json:\",optional\"`\n\t\tTls      bool   `json:\",optional\"`\n\t\tNonBlock bool   `json:\",default=true\"`\n\t\t// PingTimeout is the timeout for ping redis.\n\t\tPingTimeout time.Duration `json:\",default=1s\"`\n\t}\n\n\t// A RedisKeyConf is a redis config with key.\n\tRedisKeyConf struct {\n\t\tRedisConf\n\t\tKey string\n\t}\n)\n\n// NewRedis returns a Redis.\n// Deprecated: use MustNewRedis or NewRedis instead.\nfunc (rc RedisConf) NewRedis() *Redis {\n\tvar opts []Option\n\tif rc.Type == ClusterType {\n\t\topts = append(opts, Cluster())\n\t}\n\tif len(rc.Pass) > 0 {\n\t\topts = append(opts, WithPass(rc.Pass))\n\t}\n\tif rc.Tls {\n\t\topts = append(opts, WithTLS())\n\t}\n\n\treturn newRedis(rc.Host, opts...)\n}\n\n// Validate validates the RedisConf.\nfunc (rc RedisConf) Validate() error {\n\tif len(rc.Host) == 0 {\n\t\treturn ErrEmptyHost\n\t}\n\n\tif len(rc.Type) == 0 {\n\t\treturn ErrEmptyType\n\t}\n\n\treturn nil\n}\n\n// Validate validates the RedisKeyConf.\nfunc (rkc RedisKeyConf) Validate() error {\n\tif err := rkc.RedisConf.Validate(); err != nil {\n\t\treturn err\n\t}\n\n\tif len(rkc.Key) == 0 {\n\t\treturn ErrEmptyKey\n\t}\n\n\treturn nil\n}\n\n// Path: core/stringx/node.go\npackage stringx\n\ntype node struct {\n\tchildren map[rune]*node\n\tfail     *node\n\tdepth    int\n\tend      bool\n}\n\nfunc (n *node) add(word string) {\n\tchars := []rune(word)\n\tif len(chars) == 0 {\n\t\treturn\n\t}\n\n\tnd := n\n\tfor i, char := range chars {\n\t\tif nd.children == nil {\n\t\t\tchild := new(node)\n\t\t\tchild.depth = i + 1\n\t\t\tnd.children = map[rune]*node{char: child}\n\t\t\tnd = child\n\t\t} else if child, ok := nd.children[char]; ok {\n\t\t\tnd = child\n\t\t} else {\n\t\t\tchild := new(node)\n\t\t\tchild.depth = i + 1\n\t\t\tnd.children[char] = child\n\t\t\tnd = child\n\t\t}\n\t}\n\n\tnd.end = true\n}\n\nfunc (n *node) build() {\n\tvar nodes []*node\n\tfor _, child := range n.children {\n\t\tchild.fail = n\n\t\tnodes = append(nodes, child)\n\t}\n\tfor len(nodes) > 0 {\n\t\tnd := nodes[0]\n\t\tnodes = nodes[1:]\n\t\tfor key, child := range nd.children {\n\t\t\tnodes = append(nodes, child)\n\t\t\tcur := nd\n\t\t\tfor cur != nil {\n\t\t\t\tif cur.fail == nil {\n\t\t\t\t\tchild.fail = n\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif fail, ok := cur.fail.children[key]; ok {\n\t\t\t\t\tchild.fail = fail\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tcur = cur.fail\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (n *node) find(chars []rune) []scope {\n\tvar scopes []scope\n\tsize := len(chars)\n\tcur := n\n\n\tfor i := 0; i < size; i++ {\n\t\tchild, ok := cur.children[chars[i]]\n\t\tif ok {\n\t\t\tcur = child\n\t\t} else {\n\t\t\tfor cur != n {\n\t\t\t\tcur = cur.fail\n\t\t\t\tif child, ok = cur.children[chars[i]]; ok {\n\t\t\t\t\tcur = child\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif child == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tfor child != n {\n\t\t\tif child.end {\n\t\t\t\tscopes = append(scopes, scope{\n\t\t\t\t\tstart: i + 1 - child.depth,\n\t\t\t\t\tstop:  i + 1,\n\t\t\t\t})\n\t\t\t}\n\t\t\tchild = child.fail\n\t\t}\n\t}\n\n\treturn scopes\n}\n\n// Path: core/stores/redis/redis.go\npackage redis\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"time\"\n\n\tred \"github.com/redis/go-redis/v9\"\n\t\"github.com/zeromicro/go-zero/core/breaker\"\n\t\"github.com/zeromicro/go-zero/core/errorx\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/mapping\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n)\n\nconst (\n\t// ClusterType means redis cluster.\n\tClusterType = \"cluster\"\n\t// NodeType means redis node.\n\tNodeType = \"node\"\n\t// Nil is an alias of redis.Nil.\n\tNil = red.Nil\n\n\tblockingQueryTimeout = 5 * time.Second\n\treadWriteTimeout     = 2 * time.Second\n\tdefaultSlowThreshold = time.Millisecond * 100\n\tdefaultPingTimeout   = time.Second\n)\n\nvar (\n\t// ErrNilNode is an error that indicates a nil redis node.\n\tErrNilNode    = errors.New(\"nil redis node\")\n\tslowThreshold = syncx.ForAtomicDuration(defaultSlowThreshold)\n)\n\ntype (\n\t// Option defines the method to customize a Redis.\n\tOption func(r *Redis)\n\n\t// A Pair is a key/pair set used in redis zset.\n\tPair struct {\n\t\tKey   string\n\t\tScore int64\n\t}\n\n\t// A FloatPair is a key/pair for float set used in redis zet.\n\tFloatPair struct {\n\t\tKey   string\n\t\tScore float64\n\t}\n\n\t// Redis defines a redis node/cluster. It is thread-safe.\n\tRedis struct {\n\t\tAddr  string\n\t\tType  string\n\t\tPass  string\n\t\ttls   bool\n\t\tbrk   breaker.Breaker\n\t\thooks []red.Hook\n\t}\n\n\t// RedisNode interface represents a redis node.\n\tRedisNode interface {\n\t\tred.Cmdable\n\t\tred.BitMapCmdable\n\t}\n\n\t// GeoLocation is used with GeoAdd to add geospatial location.\n\tGeoLocation = red.GeoLocation\n\t// GeoRadiusQuery is used with GeoRadius to query geospatial index.\n\tGeoRadiusQuery = red.GeoRadiusQuery\n\t// GeoPos is used to represent a geo position.\n\tGeoPos = red.GeoPos\n\n\t// Pipeliner is an alias of redis.Pipeliner.\n\tPipeliner = red.Pipeliner\n\n\t// Z represents sorted set member.\n\tZ = red.Z\n\t// ZStore is an alias of redis.ZStore.\n\tZStore = red.ZStore\n\n\t// IntCmd is an alias of redis.IntCmd.\n\tIntCmd = red.IntCmd\n\t// FloatCmd is an alias of redis.FloatCmd.\n\tFloatCmd = red.FloatCmd\n\t// StringCmd is an alias of redis.StringCmd.\n\tStringCmd = red.StringCmd\n\t// Script is an alias of redis.Script.\n\tScript = red.Script\n)\n\n// MustNewRedis returns a Redis with given options.\nfunc MustNewRedis(conf RedisConf, opts ...Option) *Redis {\n\trds, err := NewRedis(conf, opts...)\n\tlogx.Must(err)\n\treturn rds\n}\n\n// New returns a Redis with given options.\n// Deprecated: use MustNewRedis or NewRedis instead.\nfunc New(addr string, opts ...Option) *Redis {\n\treturn newRedis(addr, opts...)\n}\n\n// NewRedis returns a Redis with given options.\nfunc NewRedis(conf RedisConf, opts ...Option) (*Redis, error) {\n\tif err := conf.Validate(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif conf.Type == ClusterType {\n\t\topts = append([]Option{Cluster()}, opts...)\n\t}\n\tif len(conf.Pass) > 0 {\n\t\topts = append([]Option{WithPass(conf.Pass)}, opts...)\n\t}\n\tif conf.Tls {\n\t\topts = append([]Option{WithTLS()}, opts...)\n\t}\n\n\trds := newRedis(conf.Host, opts...)\n\tif !conf.NonBlock {\n\t\tif err := rds.checkConnection(conf.PingTimeout); err != nil {\n\t\t\treturn nil, errorx.Wrap(err, fmt.Sprintf(\"redis connect error, addr: %s\", conf.Host))\n\t\t}\n\t}\n\n\treturn rds, nil\n}\n\nfunc newRedis(addr string, opts ...Option) *Redis {\n\tr := &Redis{\n\t\tAddr: addr,\n\t\tType: NodeType,\n\t\tbrk:  breaker.NewBreaker(),\n\t}\n\n\tfor _, opt := range opts {\n\t\topt(r)\n\t}\n\n\treturn r\n}\n\n// NewScript returns a new Script instance.\nfunc NewScript(script string) *Script {\n\treturn red.NewScript(script)\n}\n\n// BitCount is redis bitcount command implementation.\nfunc (s *Redis) BitCount(key string, start, end int64) (int64, error) {\n\treturn s.BitCountCtx(context.Background(), key, start, end)\n}\n\n// BitCountCtx is redis bitcount command implementation.\nfunc (s *Redis) BitCountCtx(ctx context.Context, key string, start, end int64) (int64, error) {\n\tconn, err := getRedis(s)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn conn.BitCount(ctx, key, &red.BitCount{\n\t\tStart: start,\n\t\tEnd:   end,\n\t}).Result()\n}\n\n// BitOpAnd is redis bit operation (and) command implementation.\n...\n", "needle_token_start": 86, "needle_token_end": 236, "code_context_ntokens": 8403, "output": ["```go\n// RunTest runs a test function with a mock database.\nfunc RunTest(t *testing.T, fn func(db *sql.DB, mock sqlmock.Sqlmock)) {\n\tdb, mock, err := sqlmock.New()\n\tif err != nil {\n\t\tt.Fatalf(\"an error '%s' was not expected when opening a stub database connection\", err)\n\t}\n\tdefer func() {\n\t\t_ = db.Close()\n\t}()\n\n\tfn(db, mock)\n\n\tif err = mock.ExpectationsWereMet(); err != nil {\n\t\tt.Errorf(\"there were unfulfilled expectations: %s\", err)\n\t}\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "format", "language": "go", "path": "core/stores/sqlx/utils.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to replace placeholders in a SQL query string with actual values from a list of arguments, ensuring that the final query string is properly formatted and safe for execution.\n2. **Input**: The function takes a SQL query string with placeholders and a variable number of arguments that are intended to replace the placeholders in the query.\n3. **Output**: It returns a formatted SQL query string with all placeholders replaced by the provided arguments. If there is an error, such as a mismatch between the number of placeholders and the number of arguments provided, it returns an error.\n4. **Procedure**: The function iterates over each character in the query string. When it encounters a placeholder, it replaces it with the corresponding argument from the provided list. It handles different types of placeholders (like `?`, `:1`, `$1`) and ensures that strings within the query are correctly escaped to prevent SQL injection. If there are more arguments than placeholders or vice versa, it generates an error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/mr/mapreduce_test.go\npackage mr\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"log\"\n\t\"runtime\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"go.uber.org/goleak\"\n)\n\nvar errDummy = errors.New(\"dummy\")\n\nfunc init() {\n\tlog.SetOutput(io.Discard)\n}\n\nfunc TestFinish(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar total uint32\n\terr := Finish(func() error {\n\t\tatomic.AddUint32(&total, 2)\n\t\treturn nil\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 3)\n\t\treturn nil\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 5)\n\t\treturn nil\n\t})\n\n\tassert.Equal(t, uint32(10), atomic.LoadUint32(&total))\n\tassert.Nil(t, err)\n}\n\nfunc TestFinishNone(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tassert.Nil(t, Finish())\n}\n\nfunc TestFinishVoidNone(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tFinishVoid()\n}\n\nfunc TestFinishErr(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar total uint32\n\terr := Finish(func() error {\n\t\tatomic.AddUint32(&total, 2)\n\t\treturn nil\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 3)\n\t\treturn errDummy\n\t}, func() error {\n\t\tatomic.AddUint32(&total, 5)\n\t\treturn nil\n\t})\n\n\tassert.Equal(t, errDummy, err)\n}\n\nfunc TestFinishVoid(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar total uint32\n\tFinishVoid(func() {\n\t\tatomic.AddUint32(&total, 2)\n\t}, func() {\n\t\tatomic.AddUint32(&total, 3)\n\t}, func() {\n\t\tatomic.AddUint32(&total, 5)\n\t})\n\n\tassert.Equal(t, uint32(10), atomic.LoadUint32(&total))\n}\n\nfunc TestForEach(t *testing.T) {\n\tconst tasks = 1000\n\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tdefer goleak.VerifyNone(t)\n\n\t\tvar count uint32\n\t\tForEach(func(source chan<- int) {\n\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\tsource <- i\n\t\t\t}\n\t\t}, func(item int) {\n\t\t\tatomic.AddUint32(&count, 1)\n\t\t}, WithWorkers(-1))\n\n\t\tassert.Equal(t, tasks, int(count))\n\t})\n\n\tt.Run(\"odd\", func(t *testing.T) {\n\t\tdefer goleak.VerifyNone(t)\n\n\t\tvar count uint32\n\t\tForEach(func(source chan<- int) {\n\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\tsource <- i\n\t\t\t}\n\t\t}, func(item int) {\n\t\t\tif item%2 == 0 {\n\t\t\t\tatomic.AddUint32(&count, 1)\n\t\t\t}\n\t\t})\n\n\t\tassert.Equal(t, tasks/2, int(count))\n\t})\n\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tdefer goleak.VerifyNone(t)\n\n\t\tassert.PanicsWithValue(t, \"foo\", func() {\n\t\t\tForEach(func(source chan<- int) {\n\t\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\t\tsource <- i\n\t\t\t\t}\n\t\t\t}, func(item int) {\n\t\t\t\tpanic(\"foo\")\n\t\t\t})\n\t\t})\n\t})\n}\n\nfunc TestGeneratePanic(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tassert.PanicsWithValue(t, \"foo\", func() {\n\t\t\tForEach(func(source chan<- int) {\n\t\t\t\tpanic(\"foo\")\n\t\t\t}, func(item int) {\n\t\t\t})\n\t\t})\n\t})\n}\n\nfunc TestMapperPanic(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tconst tasks = 1000\n\tvar run int32\n\tt.Run(\"all\", func(t *testing.T) {\n\t\tassert.PanicsWithValue(t, \"foo\", func() {\n\t\t\t_, _ = MapReduce(func(source chan<- int) {\n\t\t\t\tfor i := 0; i < tasks; i++ {\n\t\t\t\t\tsource <- i\n\t\t\t\t}\n\t\t\t}, func(item int, writer Writer[int], cancel func(error)) {\n\t\t\t\tatomic.AddInt32(&run, 1)\n\t\t\t\tpanic(\"foo\")\n\t\t\t}, func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t})\n\t\t})\n\t\tassert.True(t, atomic.LoadInt32(&run) < tasks/2)\n\t})\n}\n\nfunc TestMapReduce(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\ttests := []struct {\n\t\tname        string\n\t\tmapper      MapperFunc[int, int]\n\t\treducer     ReducerFunc[int, int]\n\t\texpectErr   error\n\t\texpectValue int\n\t}{\n\t\t{\n\t\t\tname:        \"simple\",\n\t\t\texpectErr:   nil,\n\t\t\texpectValue: 30,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with error\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(errDummy)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with nil\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(nil)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: ErrCancelWithNil,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with more\",\n\t\t\treducer: func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t\tvar result int\n\t\t\t\tfor item := range pipe {\n\t\t\t\t\tresult += item\n\t\t\t\t\tif result > 10 {\n\t\t\t\t\t\tcancel(errDummy)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\twriter.Write(result)\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t}\n\n\tt.Run(\"MapReduce\", func(t *testing.T) {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\tif test.mapper == nil {\n\t\t\t\t\ttest.mapper = func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\twriter.Write(v * v)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif test.reducer == nil {\n\t\t\t\t\ttest.reducer = func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\tvar result int\n\t\t\t\t\t\tfor item := range pipe {\n\t\t\t\t\t\t\tresult += item\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.Write(result)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tvalue, err := MapReduce(func(source chan<- int) {\n\t\t\t\t\tfor i := 1; i < 5; i++ {\n\t\t\t\t\t\tsource <- i\n\t\t\t\t\t}\n\t\t\t\t}, test.mapper, test.reducer, WithWorkers(runtime.NumCPU()))\n\n\t\t\t\tassert.Equal(t, test.expectErr, err)\n\t\t\t\tassert.Equal(t, test.expectValue, value)\n\t\t\t})\n\t\t}\n\t})\n\n\tt.Run(\"MapReduce\", func(t *testing.T) {\n\t\tfor _, test := range tests {\n\t\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t\tif test.mapper == nil {\n\t\t\t\t\ttest.mapper = func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\twriter.Write(v * v)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tif test.reducer == nil {\n\t\t\t\t\ttest.reducer = func(pipe <-chan int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\t\tvar result int\n\t\t\t\t\t\tfor item := range pipe {\n\t\t\t\t\t\t\tresult += item\n\t\t\t\t\t\t}\n\t\t\t\t\t\twriter.Write(result)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tsource := make(chan int)\n\t\t\t\tgo func() {\n\t\t\t\t\tfor i := 1; i < 5; i++ {\n\t\t\t\t\t\tsource <- i\n\t\t\t\t\t}\n\t\t\t\t\tclose(source)\n\t\t\t\t}()\n\n\t\t\t\tvalue, err := MapReduceChan(source, test.mapper, test.reducer, WithWorkers(-1))\n\t\t\t\tassert.Equal(t, test.expectErr, err)\n\t\t\t\tassert.Equal(t, test.expectValue, value)\n\t\t\t})\n\t\t}\n\t})\n}\n\nfunc TestMapReduceWithReduerWriteMoreThanOnce(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tassert.Panics(t, func() {\n\t\tMapReduce(func(source chan<- int) {\n\t\t\tfor i := 0; i < 10; i++ {\n\t\t\t\tsource <- i\n\t\t\t}\n\t\t}, func(item int, writer Writer[int], cancel func(error)) {\n\t\t\twriter.Write(item)\n\t\t}, func(pipe <-chan int, writer Writer[string], cancel func(error)) {\n\t\t\tdrain(pipe)\n\t\t\twriter.Write(\"one\")\n\t\t\twriter.Write(\"two\")\n\t\t})\n\t})\n}\n\nfunc TestMapReduceVoid(t *testing.T) {\n\tdefer goleak.VerifyNone(t)\n\n\tvar value uint32\n\ttests := []struct {\n\t\tname        string\n\t\tmapper      MapperFunc[int, int]\n\t\treducer     VoidReducerFunc[int]\n\t\texpectValue uint32\n\t\texpectErr   error\n\t}{\n\t\t{\n\t\t\tname:        \"simple\",\n\t\t\texpectValue: 30,\n\t\t\texpectErr:   nil,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with error\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(errDummy)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with nil\",\n\t\t\tmapper: func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\tif v%3 == 0 {\n\t\t\t\t\tcancel(nil)\n\t\t\t\t}\n\t\t\t\twriter.Write(v * v)\n\t\t\t},\n\t\t\texpectErr: ErrCancelWithNil,\n\t\t},\n\t\t{\n\t\t\tname: \"cancel with more\",\n\t\t\treducer: func(pipe <-chan int, cancel func(error)) {\n\t\t\t\tfor item := range pipe {\n\t\t\t\t\tresult := atomic.AddUint32(&value, uint32(item))\n\t\t\t\t\tif result > 10 {\n\t\t\t\t\t\tcancel(errDummy)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t},\n\t\t\texpectErr: errDummy,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tatomic.StoreUint32(&value, 0)\n\n\t\t\tif test.mapper == nil {\n\t\t\t\ttest.mapper = func(v int, writer Writer[int], cancel func(error)) {\n\t\t\t\t\twriter.Write(v * v)\n\t\t\t\t}\n\t\t\t}\n\t\t\tif test.reducer == nil {\n\t\t\t\ttest.reducer = func(pipe <-chan int, cancel func(error)) {\n\t\t\t\t\tfor item := range pipe {\n...\n// Path: core/naming/namer.go\npackage naming\n\n// Namer interface wraps the method Name.\ntype Namer interface {\n\tName() string\n}\n\n// Path: core/netx/ip.go\npackage netx\n\nimport \"net\"\n\n// InternalIp returns an internal ip.\nfunc InternalIp() string {\n\tinfs, err := net.Interfaces()\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\n\tfor _, inf := range infs {\n\t\tif isEthDown(inf.Flags) || isLoopback(inf.Flags) {\n\t\t\tcontinue\n\t\t}\n\n\t\taddrs, err := inf.Addrs()\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tfor _, addr := range addrs {\n\t\t\tif ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {\n\t\t\t\tif ipnet.IP.To4() != nil {\n\t\t\t\t\treturn ipnet.IP.String()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn \"\"\n}\n\nfunc isEthDown(f net.Flags) bool {\n\treturn f&net.FlagUp != net.FlagUp\n}\n\nfunc isLoopback(f net.Flags) bool {\n\treturn f&net.FlagLoopback == net.FlagLoopback\n}\n\n// Path: core/netx/ip_test.go\npackage netx\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestInternalIp(t *testing.T) {\n\tassert.True(t, len(InternalIp()) > 0)\n}\n\n// Path: core/proc/env.go\npackage proc\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"sync\"\n)\n\nvar (\n\tenvs    = make(map[string]string)\n\tenvLock sync.RWMutex\n)\n\n// Env returns the value of the given environment variable.\nfunc Env(name string) string {\n\tenvLock.RLock()\n\tval, ok := envs[name]\n\tenvLock.RUnlock()\n\n\tif ok {\n\t\treturn val\n\t}\n\n\tval = os.Getenv(name)\n\tenvLock.Lock()\n\tenvs[name] = val\n\tenvLock.Unlock()\n\n\treturn val\n}\n\n// EnvInt returns an int value of the given environment variable.\nfunc EnvInt(name string) (int, bool) {\n\tval := Env(name)\n\tif len(val) == 0 {\n\t\treturn 0, false\n\t}\n\n\tn, err := strconv.Atoi(val)\n\tif err != nil {\n\t\treturn 0, false\n\t}\n\n\treturn n, true\n}\n\n// Path: core/proc/env_test.go\npackage proc\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestEnv(t *testing.T) {\n\tassert.True(t, len(Env(\"any\")) == 0)\n\tenvLock.RLock()\n\tval, ok := envs[\"any\"]\n\tenvLock.RUnlock()\n\tassert.True(t, len(val) == 0)\n\tassert.True(t, ok)\n\tassert.True(t, len(Env(\"any\")) == 0)\n}\n\nfunc TestEnvInt(t *testing.T) {\n\tval, ok := EnvInt(\"any\")\n\tassert.Equal(t, 0, val)\n\tassert.False(t, ok)\n\tt.Setenv(\"anyInt\", \"10\")\n\tval, ok = EnvInt(\"anyInt\")\n\tassert.Equal(t, 10, val)\n\tassert.True(t, ok)\n\tt.Setenv(\"anyString\", \"a\")\n\tval, ok = EnvInt(\"anyString\")\n\tassert.Equal(t, 0, val)\n\tassert.False(t, ok)\n}\n\n// Path: core/proc/goroutines.go\n//go:build linux || darwin\n\npackage proc\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path\"\n\t\"runtime/pprof\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n)\n\nconst (\n\tgoroutineProfile = \"goroutine\"\n\tdebugLevel       = 2\n)\n\ntype creator interface {\n\tCreate(name string) (file *os.File, err error)\n}\n\nfunc dumpGoroutines(ctor creator) {\n\tcommand := path.Base(os.Args[0])\n\tpid := syscall.Getpid()\n\tdumpFile := path.Join(os.TempDir(), fmt.Sprintf(\"%s-%d-goroutines-%s.dump\",\n\t\tcommand, pid, time.Now().Format(timeFormat)))\n\n\tlogx.Infof(\"Got dump goroutine signal, printing goroutine profile to %s\", dumpFile)\n\n\tif f, err := ctor.Create(dumpFile); err != nil {\n\t\tlogx.Errorf(\"Failed to dump goroutine profile, error: %v\", err)\n\t} else {\n\t\tdefer f.Close()\n\t\tpprof.Lookup(goroutineProfile).WriteTo(f, debugLevel)\n\t}\n}\n\ntype fileCreator struct{}\n\nfunc (fc fileCreator) Create(name string) (file *os.File, err error) {\n\treturn os.Create(name)\n}\n\n// Path: core/proc/goroutines_test.go\n//go:build linux || darwin\n\npackage proc\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/logx/logtest\"\n)\n\nfunc TestDumpGoroutines(t *testing.T) {\n\tt.Run(\"real file\", func(t *testing.T) {\n\t\tbuf := logtest.NewCollector(t)\n\t\tdumpGoroutines(fileCreator{})\n\t\tassert.True(t, strings.Contains(buf.String(), \".dump\"))\n\t})\n\n\tt.Run(\"fake file\", func(t *testing.T) {\n\t\tconst msg = \"any message\"\n\t\tbuf := logtest.NewCollector(t)\n\t\terr := errors.New(msg)\n\t\tdumpGoroutines(fakeCreator{\n\t\t\tfile: &os.File{},\n\t\t\terr:  err,\n\t\t})\n\t\tassert.True(t, strings.Contains(buf.String(), msg))\n\t})\n}\n\ntype fakeCreator struct {\n\tfile *os.File\n\terr  error\n}\n\nfunc (fc fakeCreator) Create(_ string) (file *os.File, err error) {\n\treturn fc.file, fc.err\n}\n\n// Path: core/proc/process.go\npackage proc\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n)\n\nvar (\n\tprocName string\n\tpid      int\n)\n\nfunc init() {\n\tprocName = filepath.Base(os.Args[0])\n\tpid = os.Getpid()\n}\n\n// Pid returns pid of current process.\nfunc Pid() int {\n\treturn pid\n}\n\n// ProcessName returns the processname, same as the command name.\nfunc ProcessName() string {\n\treturn procName\n}\n\n// Path: core/proc/process_test.go\npackage proc\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestProcessName(t *testing.T) {\n\tassert.True(t, len(ProcessName()) > 0)\n}\n\nfunc TestPid(t *testing.T) {\n\tassert.True(t, Pid() > 0)\n}\n\n// Path: core/proc/profile+polyfill.go\n//go:build windows\n\npackage proc\n\nfunc StartProfile() Stopper {\n\treturn noopStopper\n}\n\n// Path: core/proc/profile.go\n//go:build linux || darwin\n\npackage proc\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/signal\"\n\t\"path\"\n\t\"runtime\"\n\t\"runtime/pprof\"\n\t\"runtime/trace\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n)\n\n// DefaultMemProfileRate is the default memory profiling rate.\n// See also http://golang.org/pkg/runtime/#pkg-variables\nconst DefaultMemProfileRate = 4096\n\n// started is non zero if a profile is running.\nvar started uint32\n\n// Profile represents an active profiling session.\ntype Profile struct {\n\t// closers holds cleanup functions that run after each profile\n\tclosers []func()\n\n\t// stopped records if a call to profile.Stop has been made\n\tstopped uint32\n}\n\nfunc (p *Profile) close() {\n\tfor _, closer := range p.closers {\n\t\tcloser()\n\t}\n}\n\nfunc (p *Profile) startBlockProfile() {\n\tfn := createDumpFile(\"block\")\n\tf, err := os.Create(fn)\n\tif err != nil {\n\t\tlogx.Errorf(\"profile: could not create block profile %q: %v\", fn, err)\n\t\treturn\n\t}\n\n\truntime.SetBlockProfileRate(1)\n\tlogx.Infof(\"profile: block profiling enabled, %s\", fn)\n\tp.closers = append(p.closers, func() {\n\t\tpprof.Lookup(\"block\").WriteTo(f, 0)\n\t\tf.Close()\n\t\truntime.SetBlockProfileRate(0)\n\t\tlogx.Infof(\"profile: block profiling disabled, %s\", fn)\n\t})\n}\n\nfunc (p *Profile) startCpuProfile() {\n\tfn := createDumpFile(\"cpu\")\n\tf, err := os.Create(fn)\n\tif err != nil {\n\t\tlogx.Errorf(\"profile: could not create cpu profile %q: %v\", fn, err)\n\t\treturn\n\t}\n\n\tlogx.Infof(\"profile: cpu profiling enabled, %s\", fn)\n\tpprof.StartCPUProfile(f)\n\tp.closers = append(p.closers, func() {\n\t\tpprof.StopCPUProfile()\n\t\tf.Close()\n\t\tlogx.Infof(\"profile: cpu profiling disabled, %s\", fn)\n\t})\n}\n\nfunc (p *Profile) startMemProfile() {\n\tfn := createDumpFile(\"mem\")\n\tf, err := os.Create(fn)\n\tif err != nil {\n\t\tlogx.Errorf(\"profile: could not create memory profile %q: %v\", fn, err)\n\t\treturn\n\t}\n\n\told := runtime.MemProfileRate\n\truntime.MemProfileRate = DefaultMemProfileRate\n\tlogx.Infof(\"profile: memory profiling enabled (rate %d), %s\", runtime.MemProfileRate, fn)\n\tp.closers = append(p.closers, func() {\n\t\tpprof.Lookup(\"heap\").WriteTo(f, 0)\n\t\tf.Close()\n\t\truntime.MemProfileRate = old\n\t\tlogx.Infof(\"profile: memory profiling disabled, %s\", fn)\n\t})\n}\n\nfunc (p *Profile) startMutexProfile() {\n\tfn := createDumpFile(\"mutex\")\n\tf, err := os.Create(fn)\n\tif err != nil {\n\t\tlogx.Errorf(\"profile: could not create mutex profile %q: %v\", fn, err)\n\t\treturn\n\t}\n\n\truntime.SetMutexProfileFraction(1)\n\tlogx.Infof(\"profile: mutex profiling enabled, %s\", fn)\n\tp.closers = append(p.closers, func() {\n\t\tif mp := pprof.Lookup(\"mutex\"); mp != nil {\n\t\t\tmp.WriteTo(f, 0)\n\t\t}\n\t\tf.Close()\n\t\truntime.SetMutexProfileFraction(0)\n\t\tlogx.Infof(\"profile: mutex profiling disabled, %s\", fn)\n\t})\n}\n\nfunc (p *Profile) startThreadCreateProfile() {\n\tfn := createDumpFile(\"threadcreate\")\n\tf, err := os.Create(fn)\n\tif err != nil {\n\t\tlogx.Errorf(\"profile: could not create threadcreate profile %q: %v\", fn, err)\n\t\treturn\n\t}\n\n\tlogx.Infof(\"profile: threadcreate profiling enabled, %s\", fn)\n\tp.closers = append(p.closers, func() {\n\t\tif mp := pprof.Lookup(\"threadcreate\"); mp != nil {\n\t\t\tmp.WriteTo(f, 0)\n\t\t}\n\t\tf.Close()\n\t\tlogx.Infof(\"profile: threadcreate profiling disabled, %s\", fn)\n\t})\n}\n\nfunc (p *Profile) startTraceProfile() {\n\tfn := createDumpFile(\"trace\")\n\tf, err := os.Create(fn)\n\tif err != nil {\n\t\tlogx.Errorf(\"profile: could not create trace output file %q: %v\", fn, err)\n\t\treturn\n\t}\n\n\tif err := trace.Start(f); err != nil {\n\t\tlogx.Errorf(\"profile: could not start trace: %v\", err)\n\t\treturn\n\t}\n\n\tlogx.Infof(\"profile: trace enabled, %s\", fn)\n\tp.closers = append(p.closers, func() {\n\t\ttrace.Stop()\n\t\tlogx.Infof(\"profile: trace disabled, %s\", fn)\n\t})\n}\n\n// Stop stops the profile and flushes any unwritten data.\nfunc (p *Profile) Stop() {\n\tif !atomic.CompareAndSwapUint32(&p.stopped, 0, 1) {\n\t\t// someone has already called close\n\t\treturn\n\t}\n\tp.close()\n\tatomic.StoreUint32(&started, 0)\n}\n\n// StartProfile starts a new profiling session.\n// The caller should call the Stop method on the value returned\n// to cleanly stop profiling.\nfunc StartProfile() Stopper {\n\tif !atomic.CompareAndSwapUint32(&started, 0, 1) {\n\t\tlogx.Error(\"profile: Start() already called\")\n\t\treturn noopStopper\n\t}\n\n\tvar prof Profile\n\tprof.startCpuProfile()\n\tprof.startMemProfile()\n\tprof.startMutexProfile()\n\tprof.startBlockProfile()\n\tprof.startTraceProfile()\n\tprof.startThreadCreateProfile()\n\n\tgo func() {\n\t\tc := make(chan os.Signal, 1)\n\t\tsignal.Notify(c, syscall.SIGINT)\n\t\t<-c\n\n\t\tlogx.Info(\"profile: caught interrupt, stopping profiles\")\n\t\tprof.Stop()\n\n\t\tsignal.Reset()\n\t\tsyscall.Kill(os.Getpid(), syscall.SIGINT)\n\t}()\n\n\treturn &prof\n}\n\nfunc createDumpFile(kind string) string {\n\tcommand := path.Base(os.Args[0])\n\tpid := syscall.Getpid()\n\treturn path.Join(os.TempDir(), fmt.Sprintf(\"%s-%d-%s-%s.pprof\",\n\t\tcommand, pid, kind, time.Now().Format(timeFormat)))\n}\n\n// Path: core/stores/sqlx/utils.go\npackage sqlx\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/mapping\"\n)\n\nvar errUnbalancedEscape = errors.New(\"no char after escape char\")\n\nfunc desensitize(datasource string) string {\n\t// remove account\n\tpos := strings.LastIndex(datasource, \"@\")\n\tif 0 <= pos && pos+1 < len(datasource) {\n\t\tdatasource = datasource[pos+1:]\n\t}\n\n\treturn datasource\n}\n\nfunc escape(input string) string {\n\tvar b strings.Builder\n\n\tfor _, ch := range input {\n\t\tswitch ch {\n\t\tcase '\\x00':\n\t\t\tb.WriteString(`\\x00`)\n\t\tcase '\\r':\n\t\t\tb.WriteString(`\\r`)\n\t\tcase '\\n':\n\t\t\tb.WriteString(`\\n`)\n\t\tcase '\\\\':\n\t\t\tb.WriteString(`\\\\`)\n\t\tcase '\\'':\n\t\t\tb.WriteString(`\\'`)\n\t\tcase '\"':\n\t\t\tb.WriteString(`\\\"`)\n\t\tcase '\\x1a':\n\t\t\tb.WriteString(`\\x1a`)\n\t\tdefault:\n\t\t\tb.WriteRune(ch)\n\t\t}\n\t}\n\n\treturn b.String()\n}\n\n\nfunc format(query string, args ...any) (val string, err error) {\n\tdefer func() {\n\t\tif err != nil {\n\t\t\terr = newAcceptableError(err)\n\t\t}\n\t}()\n\n\tnumArgs := len(args)\n\tif numArgs == 0 {\n\t\treturn query, nil\n\t}\n\n\tvar b strings.Builder\n\tvar argIndex int\n\tbytes := len(query)\n\n\tfor i := 0; i < bytes; i++ {\n\t\tch := query[i]\n\t\tswitch ch {\n\t\tcase '?':\n\t\t\tif argIndex >= numArgs {\n\t\t\t\treturn \"\", fmt.Errorf(\"%d ? in sql, but only %d arguments provided\",\n\t\t\t\t\targIndex+1, numArgs)\n\t\t\t}\n\n\t\t\twriteValue(&b, args[argIndex])\n\t\t\targIndex++\n\t\tcase ':', '$':\n\t\t\tvar j int\n\t\t\tfor j = i + 1; j < bytes; j++ {\n\t\t\t\tchar := query[j]\n\t\t\t\tif char < '0' || '9' < char {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif j > i+1 {\n\t\t\t\tindex, err := strconv.Atoi(query[i+1 : j])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\n\t\t\t\t// index starts from 1 for pg or oracle\n\t\t\t\tif index > argIndex {\n\t\t\t\t\targIndex = index\n\t\t\t\t}\n\n\t\t\t\tindex--\n\t\t\t\tif index < 0 || numArgs <= index {\n\t\t\t\t\treturn \"\", fmt.Errorf(\"wrong index %d in sql\", index)\n\t\t\t\t}\n\n\t\t\t\twriteValue(&b, args[index])\n\t\t\t\ti = j - 1\n\t\t\t}\n\t\tcase '\\'', '\"', '`':\n\t\t\tb.WriteByte(ch)\n\n\t\t\tfor j := i + 1; j < bytes; j++ {\n\t\t\t\tcur := query[j]\n\t\t\t\tb.WriteByte(cur)\n\n\t\t\t\tif cur == '\\\\' {\n\t\t\t\t\tj++\n\t\t\t\t\tif j >= bytes {\n\t\t\t\t\t\treturn \"\", errUnbalancedEscape\n\t\t\t\t\t}\n\n\t\t\t\t\tb.WriteByte(query[j])\n\t\t\t\t} else if cur == ch {\n\t\t\t\t\ti = j\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tb.WriteByte(ch)\n\t\t}\n\t}\n\n\tif argIndex < numArgs {\n\t\treturn \"\", fmt.Errorf(\"%d arguments provided, not matching sql\", argIndex)\n\t}\n\n\treturn b.String(), nil\n}\n\nfunc logInstanceError(ctx context.Context, datasource string, err error) {\n\tdatasource = desensitize(datasource)\n\tlogx.WithContext(ctx).Errorf(\"Error on getting sql instance of %s: %v\", datasource, err)\n}\n\nfunc logSqlError(ctx context.Context, stmt string, err error) {\n\tif err != nil && !errors.Is(err, ErrNotFound) {\n\t\tlogx.WithContext(ctx).Errorf(\"stmt: %s, error: %s\", stmt, err.Error())\n\t}\n}\n\nfunc writeValue(buf *strings.Builder, arg any) {\n\tswitch v := arg.(type) {\n\tcase bool:\n\t\tif v {\n\t\t\tbuf.WriteByte('1')\n\t\t} else {\n\t\t\tbuf.WriteByte('0')\n\t\t}\n\tcase string:\n\t\tbuf.WriteByte('\\'')\n\t\tbuf.WriteString(escape(v))\n\t\tbuf.WriteByte('\\'')\n\tcase time.Time:\n\t\tbuf.WriteByte('\\'')\n\t\tbuf.WriteString(v.String())\n\t\tbuf.WriteByte('\\'')\n\tcase *time.Time:\n\t\tbuf.WriteByte('\\'')\n\t\tbuf.WriteString(v.String())\n\t\tbuf.WriteByte('\\'')\n\tdefault:\n\t\tbuf.WriteString(mapping.Repr(v))\n\t}\n}\n\ntype acceptableError struct {\n\terr error\n}\n\nfunc newAcceptableError(err error) error {\n\treturn acceptableError{\n\t\terr: err,\n\t}\n}\n\nfunc (e acceptableError) Error() string {\n\treturn e.err.Error()\n}\n\n// Path: core/trace/utils.go\npackage trace\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"strings\"\n\n\tztrace \"github.com/zeromicro/go-zero/internal/trace\"\n\t\"go.opentelemetry.io/otel\"\n\t\"go.opentelemetry.io/otel/attribute\"\n\tsemconv \"go.opentelemetry.io/otel/semconv/v1.4.0\"\n\t\"go.opentelemetry.io/otel/trace\"\n\t\"google.golang.org/grpc/peer\"\n)\n\nconst localhost = \"127.0.0.1\"\n\nvar (\n\t// SpanIDFromContext returns the span id from ctx.\n\tSpanIDFromContext = ztrace.SpanIDFromContext\n\t// TraceIDFromContext returns the trace id from ctx.\n\tTraceIDFromContext = ztrace.TraceIDFromContext\n)\n\n// ParseFullMethod returns the method name and attributes.\nfunc ParseFullMethod(fullMethod string) (string, []attribute.KeyValue) {\n\tname := strings.TrimLeft(fullMethod, \"/\")\n\tparts := strings.SplitN(name, \"/\", 2)\n\tif len(parts) != 2 {\n\t\t// Invalid format, does not follow `/package.service/method`.\n\t\treturn name, []attribute.KeyValue(nil)\n\t}\n\n\tvar attrs []attribute.KeyValue\n\tif service := parts[0]; service != \"\" {\n\t\tattrs = append(attrs, semconv.RPCServiceKey.String(service))\n\t}\n\tif method := parts[1]; method != \"\" {\n\t\tattrs = append(attrs, semconv.RPCMethodKey.String(method))\n\t}\n\n\treturn name, attrs\n}\n\n// PeerAttr returns the peer attributes.\nfunc PeerAttr(addr string) []attribute.KeyValue {\n\thost, port, err := net.SplitHostPort(addr)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tif len(host) == 0 {\n\t\thost = localhost\n\t}\n\n\treturn []attribute.KeyValue{\n\t\tsemconv.NetPeerIPKey.String(host),\n\t\tsemconv.NetPeerPortKey.String(port),\n\t}\n}\n\n// PeerFromCtx returns the peer from ctx.\nfunc PeerFromCtx(ctx context.Context) string {\n\tp, ok := peer.FromContext(ctx)\n\tif !ok || p == nil {\n\t\treturn \"\"\n\t}\n\n\treturn p.Addr.String()\n}\n\n// SpanInfo returns the span info.\nfunc SpanInfo(fullMethod, peerAddress string) (string, []attribute.KeyValue) {\n\tattrs := []attribute.KeyValue{RPCSystemGRPC}\n\tname, mAttrs := ParseFullMethod(fullMethod)\n\tattrs = append(attrs, mAttrs...)\n\tattrs = append(attrs, PeerAttr(peerAddress)...)\n\treturn name, attrs\n}\n\n// TracerFromContext returns a tracer in ctx, otherwise returns a global tracer.\nfunc TracerFromContext(ctx context.Context) (tracer trace.Tracer) {\n\tif span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() {\n\t\ttracer = span.TracerProvider().Tracer(TraceName)\n\t} else {\n\t\ttracer = otel.Tracer(TraceName)\n\t}\n\n\treturn\n}\n\n// Path: core/prof/profiler.go\npackage prof\n\nimport \"github.com/zeromicro/go-zero/core/utils\"\n\ntype (\n\t// A ProfilePoint is a profile time point.\n\tProfilePoint struct {\n\t\t*utils.ElapsedTimer\n\t}\n\n\t// A Profiler interface represents a profiler that used to report profile points.\n\tProfiler interface {\n\t\tStart() ProfilePoint\n\t\tReport(name string, point ProfilePoint)\n\t}\n\n\trealProfiler struct{}\n\n\tnullProfiler struct{}\n)\n\nvar profiler = newNullProfiler()\n\n// EnableProfiling enables profiling.\nfunc EnableProfiling() {\n\tprofiler = newRealProfiler()\n}\n\n// Start starts a Profiler, and returns a start profiling point.\nfunc Start() ProfilePoint {\n\treturn profiler.Start()\n}\n\n// Report reports a ProfilePoint with given name.\nfunc Report(name string, point ProfilePoint) {\n\tprofiler.Report(name, point)\n}\n\nfunc newRealProfiler() Profiler {\n\treturn &realProfiler{}\n}\n\nfunc (rp *realProfiler) Start() ProfilePoint {\n\treturn ProfilePoint{\n\t\tElapsedTimer: utils.NewElapsedTimer(),\n\t}\n}\n\nfunc (rp *realProfiler) Report(name string, point ProfilePoint) {\n\tduration := point.Duration()\n\treport(name, duration)\n}\n\nfunc newNullProfiler() Profiler {\n\treturn &nullProfiler{}\n}\n\nfunc (np *nullProfiler) Start() ProfilePoint {\n\treturn ProfilePoint{}\n}\n\nfunc (np *nullProfiler) Report(string, ProfilePoint) {\n}\n\n// Path: core/proc/profile_test.go\npackage proc\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/logx/logtest\"\n)\n\nfunc TestProfile(t *testing.T) {\n\tc := logtest.NewCollector(t)\n\tprofiler := StartProfile()\n\t// start again should not work\n\tassert.NotNil(t, StartProfile())\n\tprofiler.Stop()\n\t// stop twice\n\tprofiler.Stop()\n\tassert.True(t, strings.Contains(c.String(), \".pprof\"))\n}\n\n// Path: core/proc/shutdown+polyfill.go\n//go:build windows\n\npackage proc\n\nimport \"time\"\n\n// AddShutdownListener returns fn itself on windows, lets callers call fn on their own.\nfunc AddShutdownListener(fn func()) func() {\n\treturn fn\n}\n\n// AddWrapUpListener returns fn itself on windows, lets callers call fn on their own.\nfunc AddWrapUpListener(fn func()) func() {\n\treturn fn\n}\n\n// SetTimeToForceQuit does nothing on windows.\nfunc SetTimeToForceQuit(duration time.Duration) {\n}\n\n// Shutdown does nothing on windows.\nfunc Shutdown() {\n}\n\n// WrapUp does nothing on windows.\nfunc WrapUp() {\n}\n\n// Path: core/proc/shutdown.go\n//go:build linux || darwin\n\npackage proc\n\nimport (\n\t\"os\"\n\t\"os/signal\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/threading\"\n)\n\nconst (\n\twrapUpTime = time.Second\n\t// why we use 5500 milliseconds is because most of our queue are blocking mode with 5 seconds\n\twaitTime = 5500 * time.Millisecond\n)\n\nvar (\n\twrapUpListeners          = new(listenerManager)\n\tshutdownListeners        = new(listenerManager)\n\tdelayTimeBeforeForceQuit = waitTime\n)\n\n// AddShutdownListener adds fn as a shutdown listener.\n// The returned func can be used to wait for fn getting called.\nfunc AddShutdownListener(fn func()) (waitForCalled func()) {\n\treturn shutdownListeners.addListener(fn)\n}\n\n// AddWrapUpListener adds fn as a wrap up listener.\n// The returned func can be used to wait for fn getting called.\nfunc AddWrapUpListener(fn func()) (waitForCalled func()) {\n\treturn wrapUpListeners.addListener(fn)\n}\n\n// SetTimeToForceQuit sets the waiting time before force quitting.\nfunc SetTimeToForceQuit(duration time.Duration) {\n\tdelayTimeBeforeForceQuit = duration\n}\n\n// Shutdown calls the registered shutdown listeners, only for test purpose.\nfunc Shutdown() {\n\tshutdownListeners.notifyListeners()\n}\n\n// WrapUp wraps up the process, only for test purpose.\nfunc WrapUp() {\n\twrapUpListeners.notifyListeners()\n}\n\nfunc gracefulStop(signals chan os.Signal, sig syscall.Signal) {\n\tsignal.Stop(signals)\n\n\tlogx.Infof(\"Got signal %d, shutting down...\", sig)\n\tgo wrapUpListeners.notifyListeners()\n\n\ttime.Sleep(wrapUpTime)\n\tgo shutdownListeners.notifyListeners()\n\n\ttime.Sleep(delayTimeBeforeForceQuit - wrapUpTime)\n\tlogx.Infof(\"Still alive after %v, going to force kill the process...\", delayTimeBeforeForceQuit)\n\t_ = syscall.Kill(syscall.Getpid(), sig)\n}\n\ntype listenerManager struct {\n\tlock      sync.Mutex\n\twaitGroup sync.WaitGroup\n\tlisteners []func()\n}\n\nfunc (lm *listenerManager) addListener(fn func()) (waitForCalled func()) {\n\tlm.waitGroup.Add(1)\n\n\tlm.lock.Lock()\n\tlm.listeners = append(lm.listeners, func() {\n\t\tdefer lm.waitGroup.Done()\n\t\tfn()\n\t})\n\tlm.lock.Unlock()\n\n\treturn func() {\n\t\tlm.waitGroup.Wait()\n\t}\n}\n\nfunc (lm *listenerManager) notifyListeners() {\n\tlm.lock.Lock()\n\tdefer lm.lock.Unlock()\n\n\tgroup := threading.NewRoutineGroup()\n\tfor _, listener := range lm.listeners {\n\t\tgroup.RunSafe(listener)\n\t}\n\tgroup.Wait()\n\n\tlm.listeners = nil\n}\n\n// Path: core/proc/shutdown_test.go\n//go:build linux || darwin\n\npackage proc\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestShutdown(t *testing.T) {\n\tSetTimeToForceQuit(time.Hour)\n\tassert.Equal(t, time.Hour, delayTimeBeforeForceQuit)\n\n\tvar val int\n\tcalled := AddWrapUpListener(func() {\n\t\tval++\n\t})\n\tWrapUp()\n\tcalled()\n\tassert.Equal(t, 1, val)\n\n\tcalled = AddShutdownListener(func() {\n\t\tval += 2\n\t})\n\tShutdown()\n\tcalled()\n\tassert.Equal(t, 3, val)\n}\n\nfunc TestNotifyMoreThanOnce(t *testing.T) {\n\tch := make(chan struct{}, 1)\n\n\tgo func() {\n\t\tvar val int\n\t\tcalled := AddWrapUpListener(func() {\n\t\t\tval++\n\t\t})\n\t\tWrapUp()\n\t\tWrapUp()\n\t\tcalled()\n\t\tassert.Equal(t, 1, val)\n\n\t\tcalled = AddShutdownListener(func() {\n\t\t\tval += 2\n\t\t})\n\t\tShutdown()\n\t\tShutdown()\n\t\tcalled()\n\t\tassert.Equal(t, 3, val)\n\t\tch <- struct{}{}\n\t}()\n\n\tselect {\n\tcase <-ch:\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"timeout, check error logs\")\n\t}\n}\n\n// Path: core/proc/signals+polyfill.go\n//go:build windows\n\npackage proc\n\nimport \"context\"\n\nfunc Done() <-chan struct{} {\n\treturn context.Background().Done()\n}\n\n// Path: core/proc/signals.go\n//go:build linux || darwin\n\npackage proc\n\nimport (\n\t\"os\"\n\t\"os/signal\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n)\n\nconst (\n\tprofileDuration = time.Minute\n\ttimeFormat      = \"0102150405\"\n)\n\nvar done = make(chan struct{})\n\nfunc init() {\n\tgo func() {\n\t\t// https://golang.org/pkg/os/signal/#Notify\n\t\tsignals := make(chan os.Signal, 1)\n\t\tsignal.Notify(signals, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGTERM, syscall.SIGINT)\n\n\t\tfor {\n\t\t\tv := <-signals\n\t\t\tswitch v {\n\t\t\tcase syscall.SIGUSR1:\n\t\t\t\tdumpGoroutines(fileCreator{})\n\t\t\tcase syscall.SIGUSR2:\n\t\t\t\tprofiler := StartProfile()\n\t\t\t\ttime.AfterFunc(profileDuration, profiler.Stop)\n\t\t\tcase syscall.SIGTERM:\n\t\t\t\tstopOnSignal()\n\t\t\t\tgracefulStop(signals, syscall.SIGTERM)\n\t\t\tcase syscall.SIGINT:\n\t\t\t\tstopOnSignal()\n\t\t\t\tgracefulStop(signals, syscall.SIGINT)\n\t\t\tdefault:\n\t\t\t\tlogx.Error(\"Got unregistered signal:\", v)\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// Done returns the channel that notifies the process quitting.\nfunc Done() <-chan struct{} {\n\treturn done\n}\n\nfunc stopOnSignal() {\n\tselect {\n\tcase <-done:\n\t\t// already closed\n\tdefault:\n\t\tclose(done)\n\t}\n}\n\n// Path: core/proc/signals_test.go\n//go:build linux || darwin\n\npackage proc\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestDone(t *testing.T) {\n\tselect {\n\tcase <-Done():\n\t\tassert.Fail(t, \"should run\")\n\tdefault:\n\t}\n\tassert.NotNil(t, Done())\n}\n\n// Path: core/proc/stopper.go\npackage proc\n\nvar noopStopper nilStopper\n\ntype (\n\t// Stopper interface wraps the method Stop.\n\tStopper interface {\n\t\tStop()\n\t}\n\n\tnilStopper struct{}\n)\n\nfunc (ns nilStopper) Stop() {\n}\n\n// Path: core/proc/stopper_test.go\npackage proc\n\nimport \"testing\"\n\nfunc TestNopStopper(t *testing.T) {\n\t// no panic\n\tnoopStopper.Stop()\n}\n\n// Path: core/prof/profilecenter.go\npackage prof\n\nimport (\n\t\"bytes\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/olekukonko/tablewriter\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/threading\"\n)\n\ntype (\n\tprofileSlot struct {\n\t\tlifecount int64\n\t\tlastcount int64\n\t\tlifecycle int64\n\t\tlastcycle int64\n\t}\n\n\tprofileCenter struct {\n\t\tlock  sync.RWMutex\n\t\tslots map[string]*profileSlot\n\t}\n)\n\nconst flushInterval = 5 * time.Minute\n\nvar (\n\tpc = &profileCenter{\n\t\tslots: make(map[string]*profileSlot),\n\t}\n\tonce sync.Once\n)\n\nfunc report(name string, duration time.Duration) {\n\tupdated := func() bool {\n\t\tpc.lock.RLock()\n\t\tdefer pc.lock.RUnlock()\n\n\t\tslot, ok := pc.slots[name]\n\t\tif ok {\n\t\t\tatomic.AddInt64(&slot.lifecount, 1)\n\t\t\tatomic.AddInt64(&slot.lastcount, 1)\n\t\t\tatomic.AddInt64(&slot.lifecycle, int64(duration))\n\t\t\tatomic.AddInt64(&slot.lastcycle, int64(duration))\n\t\t}\n\t\treturn ok\n\t}()\n\n\tif !updated {\n\t\tfunc() {\n\t\t\tpc.lock.Lock()\n\t\t\tdefer pc.lock.Unlock()\n\n\t\t\tpc.slots[name] = &profileSlot{\n\t\t\t\tlifecount: 1,\n\t\t\t\tlastcount: 1,\n\t\t\t\tlifecycle: int64(duration),\n\t\t\t\tlastcycle: int64(duration),\n\t\t\t}\n\t\t}()\n\t}\n\n\tonce.Do(flushRepeatly)\n}\n\nfunc flushRepeatly() {\n\tthreading.GoSafe(func() {\n\t\tfor {\n\t\t\ttime.Sleep(flushInterval)\n\t\t\tlogx.Stat(generateReport())\n\t\t}\n\t})\n}\n\nfunc generateReport() string {\n\tvar buffer bytes.Buffer\n\tbuffer.WriteString(\"Profiling report\\n\")\n\tvar data [][]string\n\tcalcFn := func(total, count int64) string {\n\t\tif count == 0 {\n\t\t\treturn \"-\"\n\t\t}\n\n\t\treturn (time.Duration(total) / time.Duration(count)).String()\n\t}\n\n\tfunc() {\n\t\tpc.lock.Lock()\n\t\tdefer pc.lock.Unlock()\n\n\t\tfor key, slot := range pc.slots {\n\t\t\tdata = append(data, []string{\n\t\t\t\tkey,\n\t\t\t\tstrconv.FormatInt(slot.lifecount, 10),\n\t\t\t\tcalcFn(slot.lifecycle, slot.lifecount),\n\t\t\t\tstrconv.FormatInt(slot.lastcount, 10),\n\t\t\t\tcalcFn(slot.lastcycle, slot.lastcount),\n\t\t\t})\n\n\t\t\t// reset the data for last cycle\n\t\t\tslot.lastcount = 0\n\t\t\tslot.lastcycle = 0\n\t\t}\n\t}()\n\n\ttable := tablewriter.NewWriter(&buffer)\n\ttable.SetHeader([]string{\"QUEUE\", \"LIFECOUNT\", \"LIFECYCLE\", \"LASTCOUNT\", \"LASTCYCLE\"})\n\ttable.SetBorder(false)\n\ttable.AppendBulk(data)\n\ttable.Render()\n\n\treturn buffer.String()\n}\n\n// Path: core/prof/profilecenter_test.go\npackage prof\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestReport(t *testing.T) {\n\tonce.Do(func() {})\n\tassert.NotContains(t, generateReport(), \"foo\")\n\treport(\"foo\", time.Second)\n\tassert.Contains(t, generateReport(), \"foo\")\n\treport(\"foo\", time.Second)\n}\n\n// Path: core/prof/profiler_test.go\npackage prof\n\nimport (\n\t\"testing\"\n\n\t\"github.com/zeromicro/go-zero/core/utils\"\n)\n\nfunc TestProfiler(t *testing.T) {\n\tEnableProfiling()\n\tStart()\n\tReport(\"foo\", ProfilePoint{\n\t\tElapsedTimer: utils.NewElapsedTimer(),\n\t})\n}\n\nfunc TestNullProfiler(t *testing.T) {\n\tp := newNullProfiler()\n\tp.Start()\n\tp.Report(\"foo\", ProfilePoint{\n\t\tElapsedTimer: utils.NewElapsedTimer(),\n\t})\n}\n\n// Path: core/prof/runtime_test.go\npackage prof\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestDisplayStats(t *testing.T) {\n\twriter := &threadSafeBuffer{\n\t\tbuf: strings.Builder{},\n\t}\n\tdisplayStatsWithWriter(writer, time.Millisecond*10)\n\ttime.Sleep(time.Millisecond * 50)\n\tassert.Contains(t, writer.String(), \"Goroutines: \")\n}\n\ntype threadSafeBuffer struct {\n\tbuf  strings.Builder\n\tlock sync.Mutex\n}\n\nfunc (b *threadSafeBuffer) String() string {\n\tb.lock.Lock()\n\tdefer b.lock.Unlock()\n\treturn b.buf.String()\n}\n\nfunc (b *threadSafeBuffer) Write(p []byte) (n int, err error) {\n\tb.lock.Lock()\n\tdefer b.lock.Unlock()\n\treturn b.buf.Write(p)\n}\n\n// Path: core/prometheus/agent.go\npackage prometheus\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"sync\"\n\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n\t\"github.com/zeromicro/go-zero/core/threading\"\n)\n\nvar (\n\tonce    sync.Once\n\tenabled syncx.AtomicBool\n)\n\n// Enabled returns if prometheus is enabled.\nfunc Enabled() bool {\n\treturn enabled.True()\n}\n\n// Enable enables prometheus.\nfunc Enable() {\n\tenabled.Set(true)\n}\n\n// StartAgent starts a prometheus agent.\nfunc StartAgent(c Config) {\n\tif len(c.Host) == 0 {\n\t\treturn\n\t}\n\n\tonce.Do(func() {\n\t\tenabled.Set(true)\n\t\tthreading.GoSafe(func() {\n\t\t\thttp.Handle(c.Path, promhttp.Handler())\n\t\t\taddr := fmt.Sprintf(\"%s:%d\", c.Host, c.Port)\n\t\t\tlogx.Infof(\"Starting prometheus agent at %s\", addr)\n\t\t\tif err := http.ListenAndServe(addr, nil); err != nil {\n\t\t\t\tlogx.Error(err)\n\t\t\t}\n\t\t})\n\t})\n}\n\n// Path: core/queue/balancedpusher.go\npackage queue\n\nimport (\n\t\"errors\"\n\t\"sync/atomic\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n)\n\n// ErrNoAvailablePusher indicates no pusher available.\nvar ErrNoAvailablePusher = errors.New(\"no available pusher\")\n\n// A BalancedPusher is used to push messages to multiple pusher with round robin algorithm.\ntype BalancedPusher struct {\n\tname    string\n\tpushers []Pusher\n\tindex   uint64\n}\n\n// NewBalancedPusher returns a BalancedPusher.\nfunc NewBalancedPusher(pushers []Pusher) Pusher {\n\treturn &BalancedPusher{\n\t\tname:    generateName(pushers),\n\t\tpushers: pushers,\n\t}\n}\n\n// Name returns the name of pusher.\nfunc (pusher *BalancedPusher) Name() string {\n\treturn pusher.name\n}\n\n// Push pushes message to one of the underlying pushers.\nfunc (pusher *BalancedPusher) Push(message string) error {\n\tsize := len(pusher.pushers)\n\n...\n", "needle_token_start": 7882, "needle_token_end": 8614, "code_context_ntokens": 15026, "output": ["```go\nfunc format(query string, args ...any) (val string, err error) {\n\tdefer func() {\n\t\tif err != nil {\n\t\t\terr = newAcceptableError(err)\n\t\t}\n\t}()\n\n\tnumArgs := len(args)\n\tif numArgs == 0 {\n\t\treturn query, nil\n\t}\n\n\tvar b strings.Builder\n\tvar argIndex int\n\tbytes := len(query)\n\n\tfor i := 0; i < bytes; i++ {\n\t\tch := query[i]\n\t\tswitch ch {\n\t\tcase '?':\n\t\t\tif argIndex >= numArgs {\n\t\t\t\treturn \"\", fmt.Errorf(\"%d ? in sql, but only %d arguments provided\",\n\t\t\t\t\targIndex+1, numArgs)\n\t\t\t}\n\n\t\t\twriteValue(&b, args[argIndex])\n\t\t\targIndex++\n\t\tcase ':', '$':\n\t\t\tvar j int\n\t\t\tfor j = i + 1; j < bytes; j++ {\n\t\t\t\tchar := query[j]\n\t\t\t\tif char < '0' || '9' < char {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif j > i+1 {\n\t\t\t\tindex, err := strconv.Atoi(query[i+1 : j])\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\n\t\t\t\t// index starts from 1 for pg or oracle\n\t\t\t\tif index > argIndex {\n\t\t\t\t\targIndex = index\n\t\t\t\t}\n\n\t\t\t\tindex--\n\t\t\t\tif index < 0 || numArgs <= index {\n\t\t\t\t\treturn \"\", fmt.Errorf(\"wrong index %d in sql\", index)\n\t\t\t\t}\n\n\t\t\t\twriteValue(&b, args[index])\n\t\t\t\ti = j - 1\n\t\t\t}\n\t\tcase '\\'', '\"', '`':\n\t\t\tb.WriteByte(ch)\n\n\t\t\tfor j := i + 1; j < bytes; j++ {\n\t\t\t\tcur := query[j]\n\t\t\t\tb.WriteByte(cur)\n\n\t\t\t\tif cur == '\\\\' {\n\t\t\t\t\tj++\n\t\t\t\t\tif j >= bytes {\n\t\t\t\t\t\treturn \"\", errUnbalancedEscape\n\t\t\t\t\t}\n\n\t\t\t\t\tb.WriteByte(query[j])\n\t\t\t\t} else if cur == ch {\n\t\t\t\t\ti = j\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tb.WriteByte(ch)\n\t\t}\n\t}\n\n\tif argIndex < numArgs {\n\t\treturn \"\", fmt.Errorf(\"%d arguments provided, not matching sql\", argIndex)\n\t}\n\n\treturn b.String(), nil\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "TestPostgreSqlJoin", "language": "go", "path": "core/stores/builder/builder_test.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To verify that a function correctly formats a list of column names into a PostgreSQL update statement format, appending parameter placeholders starting from `$2`.\n2. **Input**: An array of strings representing column names.\n3. **Output**: A string that concatenates the input column names with their respective PostgreSQL placeholders, formatted for use in an SQL update statement.\n4. **Procedure**: The test checks three scenarios: an empty array, an array with a single string, and an array with multiple strings. For each case, it asserts that the output string is formatted correctly, with each column name followed by a `= $n` pattern, where `n` starts from 2 and increments for each subsequent column.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/stat/internal/cgroup_linux.go\npackage internal\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"os\"\n\t\"path\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/iox\"\n\t\"github.com/zeromicro/go-zero/core/lang\"\n\t\"golang.org/x/sys/unix\"\n)\n\nconst (\n\tcgroupDir   = \"/sys/fs/cgroup\"\n\tcpuMaxFile  = cgroupDir + \"/cpu.max\"\n\tcpuStatFile = cgroupDir + \"/cpu.stat\"\n\tcpusetFile  = cgroupDir + \"/cpuset.cpus.effective\"\n)\n\nvar (\n\tisUnifiedOnce sync.Once\n\tisUnified     bool\n\tinUserNS      bool\n\tnsOnce        sync.Once\n)\n\ntype cgroup interface {\n\tcpuQuota() (float64, error)\n\tcpuUsage() (uint64, error)\n\teffectiveCpus() (int, error)\n}\n\nfunc currentCgroup() (cgroup, error) {\n\tif isCgroup2UnifiedMode() {\n\t\treturn currentCgroupV2()\n\t}\n\n\treturn currentCgroupV1()\n}\n\ntype cgroupV1 struct {\n\tcgroups map[string]string\n}\n\nfunc (c *cgroupV1) cpuQuota() (float64, error) {\n\tquotaUs, err := c.cpuQuotaUs()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tif quotaUs == -1 {\n\t\treturn -1, nil\n\t}\n\n\tperiodUs, err := c.cpuPeriodUs()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn float64(quotaUs) / float64(periodUs), nil\n}\n\nfunc (c *cgroupV1) cpuPeriodUs() (uint64, error) {\n\tdata, err := iox.ReadText(path.Join(c.cgroups[\"cpu\"], \"cpu.cfs_period_us\"))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn parseUint(data)\n}\n\nfunc (c *cgroupV1) cpuQuotaUs() (int64, error) {\n\tdata, err := iox.ReadText(path.Join(c.cgroups[\"cpu\"], \"cpu.cfs_quota_us\"))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn strconv.ParseInt(data, 10, 64)\n}\n\nfunc (c *cgroupV1) cpuUsage() (uint64, error) {\n\tdata, err := iox.ReadText(path.Join(c.cgroups[\"cpuacct\"], \"cpuacct.usage\"))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn parseUint(data)\n}\n\nfunc (c *cgroupV1) effectiveCpus() (int, error) {\n\tdata, err := iox.ReadText(path.Join(c.cgroups[\"cpuset\"], \"cpuset.cpus\"))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tcpus, err := parseUints(data)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn len(cpus), nil\n}\n\ntype cgroupV2 struct {\n\tcgroups map[string]string\n}\n\nfunc (c *cgroupV2) cpuQuota() (float64, error) {\n\tdata, err := iox.ReadText(cpuMaxFile)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tfields := strings.Fields(data)\n\tif len(fields) != 2 {\n\t\treturn 0, fmt.Errorf(\"cgroup: bad /sys/fs/cgroup/cpu.max file: %s\", data)\n\t}\n\n\tif fields[0] == \"max\" {\n\t\treturn -1, nil\n\t}\n\n...\n// Path: core/stat/internal/cgroup_linux_test.go\npackage internal\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRunningInUserNS(t *testing.T) {\n\t// should be false in docker\n\tassert.False(t, runningInUserNS())\n}\n\nfunc TestCgroups(t *testing.T) {\n\t// test cgroup legacy(v1) & hybrid\n\tif !isCgroup2UnifiedMode() {\n\t\tcg, err := currentCgroupV1()\n\t\tassert.NoError(t, err)\n\t\t_, err = cg.effectiveCpus()\n\t\tassert.NoError(t, err)\n\t\t_, err = cg.cpuQuota()\n\t\tassert.NoError(t, err)\n\t\t_, err = cg.cpuUsage()\n\t\tassert.NoError(t, err)\n\t}\n\n\t// test cgroup v2\n\tif isCgroup2UnifiedMode() {\n\t\tcg, err := currentCgroupV2()\n\t\tassert.NoError(t, err)\n\t\t_, err = cg.effectiveCpus()\n\t\tassert.NoError(t, err)\n\t\t_, err = cg.cpuQuota()\n\t\tassert.Error(t, err)\n\t\t_, err = cg.cpuUsage()\n\t\tassert.NoError(t, err)\n\t}\n}\n\nfunc TestParseUint(t *testing.T) {\n\ttests := []struct {\n\t\tinput string\n\t\twant  uint64\n\t\terr   error\n\t}{\n\t\t{\"0\", 0, nil},\n\t\t{\"123\", 123, nil},\n\t\t{\"-1\", 0, nil},\n\t\t{\"-18446744073709551616\", 0, nil},\n\t\t{\"foo\", 0, fmt.Errorf(\"cgroup: bad int format: foo\")},\n\t}\n\n\tfor _, tt := range tests {\n\t\tgot, err := parseUint(tt.input)\n\t\tassert.Equal(t, tt.err, err)\n\t\tassert.Equal(t, tt.want, got)\n\t}\n}\n\nfunc TestParseUints(t *testing.T) {\n\ttests := []struct {\n\t\tinput string\n\t\twant  []uint64\n\t\terr   error\n\t}{\n\t\t{\"\", nil, nil},\n\t\t{\"1,2,3\", []uint64{1, 2, 3}, nil},\n\t\t{\"1-3\", []uint64{1, 2, 3}, nil},\n\t\t{\"1-3,5,7-9\", []uint64{1, 2, 3, 5, 7, 8, 9}, nil},\n\t\t{\"foo\", nil, fmt.Errorf(\"cgroup: bad int format: foo\")},\n\t\t{\"1-bar\", nil, fmt.Errorf(\"cgroup: bad int list format: 1-bar\")},\n\t\t{\"bar-3\", nil, fmt.Errorf(\"cgroup: bad int list format: bar-3\")},\n\t\t{\"3-1\", nil, fmt.Errorf(\"cgroup: bad int list format: 3-1\")},\n\t}\n\n\tfor _, tt := range tests {\n\t\tgot, err := parseUints(tt.input)\n\t\tassert.Equal(t, tt.err, err)\n\t\tassert.Equal(t, tt.want, got)\n\t}\n}\n\n// Path: core/stat/internal/cpu_linux.go\npackage internal\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/iox\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n)\n\nconst (\n\tcpuTicks  = 100\n\tcpuFields = 8\n\tcpuMax    = 1000\n\tstatFile  = \"/proc/stat\"\n)\n\nvar (\n\tpreSystem uint64\n\tpreTotal  uint64\n\tlimit     float64\n\tcores     uint64\n\tnoCgroup  bool\n\tinitOnce  sync.Once\n)\n\n// RefreshCpu refreshes cpu usage and returns.\nfunc RefreshCpu() uint64 {\n\tinitializeOnce()\n\n\tif noCgroup {\n\t\treturn 0\n\t}\n\n\ttotal, err := cpuUsage()\n\tif err != nil {\n\t\treturn 0\n\t}\n\n\tsystem, err := systemCpuUsage()\n\tif err != nil {\n\t\treturn 0\n\t}\n\n\tvar usage uint64\n\tcpuDelta := total - preTotal\n\tsystemDelta := system - preSystem\n\tif cpuDelta > 0 && systemDelta > 0 {\n\t\tusage = uint64(float64(cpuDelta*cores*cpuMax) / (float64(systemDelta) * limit))\n\t\tif usage > cpuMax {\n\t\t\tusage = cpuMax\n\t\t}\n\t}\n\tpreSystem = system\n\tpreTotal = total\n\n\treturn usage\n}\n\nfunc cpuQuota() (float64, error) {\n\tcg, err := currentCgroup()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn cg.cpuQuota()\n}\n\nfunc cpuUsage() (uint64, error) {\n\tcg, err := currentCgroup()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn cg.cpuUsage()\n}\n\nfunc effectiveCpus() (int, error) {\n\tcg, err := currentCgroup()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn cg.effectiveCpus()\n}\n\n// if /proc not present, ignore the cpu calculation, like wsl linux\nfunc initialize() error {\n\tcpus, err := effectiveCpus()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tcores = uint64(cpus)\n\tlimit = float64(cpus)\n\tquota, err := cpuQuota()\n\tif err == nil && quota > 0 {\n\t\tif quota < limit {\n\t\t\tlimit = quota\n\t\t}\n\t}\n\n\tpreSystem, err = systemCpuUsage()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpreTotal, err = cpuUsage()\n\treturn err\n}\n\nfunc initializeOnce() {\n\tinitOnce.Do(func() {\n\t\tdefer func() {\n\t\t\tif p := recover(); p != nil {\n\t\t\t\tnoCgroup = true\n\t\t\t\tlogx.Error(p)\n\t\t\t}\n\t\t}()\n\n\t\tif err := initialize(); err != nil {\n\t\t\tnoCgroup = true\n\t\t\tlogx.Error(err)\n\t\t}\n\t})\n}\n\nfunc systemCpuUsage() (uint64, error) {\n\tlines, err := iox.ReadTextLines(statFile, iox.WithoutBlank())\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tfor _, line := range lines {\n\t\tfields := strings.Fields(line)\n\t\tif fields[0] == \"cpu\" {\n\t\t\tif len(fields) < cpuFields {\n\t\t\t\treturn 0, fmt.Errorf(\"bad format of cpu stats\")\n\t\t\t}\n\n\t\t\tvar totalClockTicks uint64\n\t\t\tfor _, i := range fields[1:cpuFields] {\n\t\t\t\tv, err := parseUint(i)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn 0, err\n\t\t\t\t}\n\n\t\t\t\ttotalClockTicks += v\n\t\t\t}\n\n\t\t\treturn (totalClockTicks * uint64(time.Second)) / cpuTicks, nil\n\t\t}\n\t}\n\n\treturn 0, errors.New(\"bad stats format\")\n}\n\n// Path: core/stat/internal/cpu_linux_test.go\npackage internal\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestRefreshCpu(t *testing.T) {\n\tassert.NotPanics(t, func() {\n\t\tRefreshCpu()\n\t})\n}\n\nfunc BenchmarkRefreshCpu(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tRefreshCpu()\n\t}\n}\n\n// Path: core/stat/internal/cpu_other.go\n//go:build !linux\n\npackage internal\n\n// RefreshCpu returns cpu usage, always returns 0 on systems other than linux.\nfunc RefreshCpu() uint64 {\n\treturn 0\n}\n\n// Path: core/stat/metrics.go\npackage stat\n\nimport (\n\t\"os\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/executors\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n)\n\nvar (\n\tlogInterval  = time.Minute\n\twriterLock   sync.Mutex\n\treportWriter Writer = nil\n\tlogEnabled          = syncx.ForAtomicBool(true)\n)\n\ntype (\n\t// Writer interface wraps the Write method.\n\tWriter interface {\n\t\tWrite(report *StatReport) error\n\t}\n\n\t// A StatReport is a stat report entry.\n\tStatReport struct {\n\t\tName          string  `json:\"name\"`\n\t\tTimestamp     int64   `json:\"tm\"`\n\t\tPid           int     `json:\"pid\"`\n\t\tReqsPerSecond float32 `json:\"qps\"`\n\t\tDrops         int     `json:\"drops\"`\n\t\tAverage       float32 `json:\"avg\"`\n\t\tMedian        float32 `json:\"med\"`\n\t\tTop90th       float32 `json:\"t90\"`\n\t\tTop99th       float32 `json:\"t99\"`\n\t\tTop99p9th     float32 `json:\"t99p9\"`\n\t}\n\n\t// A Metrics is used to log and report stat reports.\n\tMetrics struct {\n\t\texecutor  *executors.PeriodicalExecutor\n\t\tcontainer *metricsContainer\n\t}\n)\n\n// DisableLog disables logs of stats.\nfunc DisableLog() {\n\tlogEnabled.Set(false)\n}\n\n// SetReportWriter sets the report writer.\nfunc SetReportWriter(writer Writer) {\n\twriterLock.Lock()\n\treportWriter = writer\n\twriterLock.Unlock()\n}\n\n// NewMetrics returns a Metrics.\nfunc NewMetrics(name string) *Metrics {\n\tcontainer := &metricsContainer{\n\t\tname: name,\n\t\tpid:  os.Getpid(),\n\t}\n\n\treturn &Metrics{\n\t\texecutor:  executors.NewPeriodicalExecutor(logInterval, container),\n\t\tcontainer: container,\n\t}\n}\n\n// Add adds task to m.\nfunc (m *Metrics) Add(task Task) {\n\tm.executor.Add(task)\n}\n\n// AddDrop adds a drop to m.\nfunc (m *Metrics) AddDrop() {\n\tm.executor.Add(Task{\n\t\tDrop: true,\n\t})\n}\n\n// SetName sets the name of m.\nfunc (m *Metrics) SetName(name string) {\n\tm.executor.Sync(func() {\n\t\tm.container.name = name\n\t})\n}\n\ntype (\n\ttasksDurationPair struct {\n\t\ttasks    []Task\n\t\tduration time.Duration\n\t\tdrops    int\n\t}\n\n\tmetricsContainer struct {\n\t\tname     string\n\t\tpid      int\n\t\ttasks    []Task\n\t\tduration time.Duration\n\t\tdrops    int\n\t}\n)\n\nfunc (c *metricsContainer) AddTask(v any) bool {\n\tif task, ok := v.(Task); ok {\n\t\tif task.Drop {\n\t\t\tc.drops++\n\t\t} else {\n\t\t\tc.tasks = append(c.tasks, task)\n\t\t\tc.duration += task.Duration\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc (c *metricsContainer) Execute(v any) {\n\tpair := v.(tasksDurationPair)\n\ttasks := pair.tasks\n\tduration := pair.duration\n\tdrops := pair.drops\n\tsize := len(tasks)\n\treport := &StatReport{\n\t\tName:          c.name,\n\t\tTimestamp:     time.Now().Unix(),\n\t\tPid:           c.pid,\n\t\tReqsPerSecond: float32(size) / float32(logInterval/time.Second),\n\t\tDrops:         drops,\n\t}\n\n\tif size > 0 {\n\t\treport.Average = float32(duration/time.Millisecond) / float32(size)\n\n\t\tfiftyPercent := size >> 1\n\t\tif fiftyPercent > 0 {\n\t\t\ttop50pTasks := topK(tasks, fiftyPercent)\n\t\t\tmedianTask := top50pTasks[0]\n\t\t\treport.Median = float32(medianTask.Duration) / float32(time.Millisecond)\n\t\t\ttenPercent := fiftyPercent / 5\n\t\t\tif tenPercent > 0 {\n\t\t\t\ttop10pTasks := topK(top50pTasks, tenPercent)\n\t\t\t\ttask90th := top10pTasks[0]\n\t\t\t\treport.Top90th = float32(task90th.Duration) / float32(time.Millisecond)\n\t\t\t\tonePercent := tenPercent / 10\n\t\t\t\tif onePercent > 0 {\n\t\t\t\t\ttop1pTasks := topK(top10pTasks, onePercent)\n\t\t\t\t\ttask99th := top1pTasks[0]\n\t\t\t\t\treport.Top99th = float32(task99th.Duration) / float32(time.Millisecond)\n\t\t\t\t\tpointOnePercent := onePercent / 10\n\t\t\t\t\tif pointOnePercent > 0 {\n\t\t\t\t\t\ttopPointOneTasks := topK(top1pTasks, pointOnePercent)\n\t\t\t\t\t\ttask99Point9th := topPointOneTasks[0]\n\t\t\t\t\t\treport.Top99p9th = float32(task99Point9th.Duration) / float32(time.Millisecond)\n\t\t\t\t\t} else {\n\t\t\t\t\t\treport.Top99p9th = getTopDuration(top1pTasks)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tmostDuration := getTopDuration(top10pTasks)\n\t\t\t\t\treport.Top99th = mostDuration\n\t\t\t\t\treport.Top99p9th = mostDuration\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tmostDuration := getTopDuration(top50pTasks)\n\t\t\t\treport.Top90th = mostDuration\n\t\t\t\treport.Top99th = mostDuration\n\t\t\t\treport.Top99p9th = mostDuration\n\t\t\t}\n\t\t} else {\n\t\t\tmostDuration := getTopDuration(tasks)\n\t\t\treport.Median = mostDuration\n\t\t\treport.Top90th = mostDuration\n\t\t\treport.Top99th = mostDuration\n\t\t\treport.Top99p9th = mostDuration\n\t\t}\n\t}\n\n\tlog(report)\n}\n\nfunc (c *metricsContainer) RemoveAll() any {\n\ttasks := c.tasks\n\tduration := c.duration\n\tdrops := c.drops\n\tc.tasks = nil\n\tc.duration = 0\n\tc.drops = 0\n\n\treturn tasksDurationPair{\n\t\ttasks:    tasks,\n\t\tduration: duration,\n\t\tdrops:    drops,\n\t}\n}\n\nfunc getTopDuration(tasks []Task) float32 {\n\ttop := topK(tasks, 1)\n\tif len(top) < 1 {\n\t\treturn 0\n\t}\n\n\treturn float32(top[0].Duration) / float32(time.Millisecond)\n}\n\nfunc log(report *StatReport) {\n\twriteReport(report)\n\tif logEnabled.True() {\n\t\tlogx.Statf(\"(%s) - qps: %.1f/s, drops: %d, avg time: %.1fms, med: %.1fms, \"+\n\t\t\t\"90th: %.1fms, 99th: %.1fms, 99.9th: %.1fms\",\n\t\t\treport.Name, report.ReqsPerSecond, report.Drops, report.Average, report.Median,\n\t\t\treport.Top90th, report.Top99th, report.Top99p9th)\n\t}\n}\n\nfunc writeReport(report *StatReport) {\n\twriterLock.Lock()\n\tdefer writerLock.Unlock()\n\n\tif reportWriter != nil {\n\t\tif err := reportWriter.Write(report); err != nil {\n\t\t\tlogx.Error(err)\n\t\t}\n\t}\n}\n\n// Path: core/stat/metrics_test.go\npackage stat\n\nimport (\n\t\"errors\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/logx/logtest\"\n)\n\nfunc TestMetrics(t *testing.T) {\n\tDisableLog()\n\tdefer logEnabled.Set(true)\n\n\tcounts := []int{1, 5, 10, 100, 1000, 1000}\n\tfor _, count := range counts {\n\t\tm := NewMetrics(\"foo\")\n\t\tm.SetName(\"bar\")\n\t\tfor i := 0; i < count; i++ {\n\t\t\tm.Add(Task{\n\t\t\t\tDuration:    time.Millisecond * time.Duration(i),\n\t\t\t\tDescription: strconv.Itoa(i),\n\t\t\t})\n\t\t}\n\t\tm.AddDrop()\n\t\tvar writer mockedWriter\n\t\tSetReportWriter(&writer)\n\t\tm.executor.Flush()\n\t\tassert.Equal(t, \"bar\", writer.report.Name)\n\t}\n}\n\nfunc TestTopDurationWithEmpty(t *testing.T) {\n\tassert.Equal(t, float32(0), getTopDuration(nil))\n\tassert.Equal(t, float32(0), getTopDuration([]Task{}))\n}\n\nfunc TestLogAndReport(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\told := logEnabled.True()\n\tlogEnabled.Set(true)\n\tt.Cleanup(func() {\n\t\tlogEnabled.Set(old)\n\t})\n\n\tlog(&StatReport{})\n\tassert.NotEmpty(t, buf.String())\n\n\twriterLock.Lock()\n\twriter := reportWriter\n\twriterLock.Unlock()\n\tbuf = logtest.NewCollector(t)\n\tt.Cleanup(func() {\n\t\tSetReportWriter(writer)\n\t})\n\tSetReportWriter(&badWriter{})\n\twriteReport(&StatReport{})\n\tassert.NotEmpty(t, buf.String())\n}\n\ntype mockedWriter struct {\n\treport *StatReport\n}\n\nfunc (m *mockedWriter) Write(report *StatReport) error {\n\tm.report = report\n\treturn nil\n}\n\ntype badWriter struct{}\n\nfunc (b *badWriter) Write(_ *StatReport) error {\n\treturn errors.New(\"bad\")\n}\n\n// Path: core/stat/remotewriter.go\npackage stat\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n)\n\nconst (\n\thttpTimeout     = time.Second * 5\n\tjsonContentType = \"application/json; charset=utf-8\"\n)\n\n// ErrWriteFailed is an error that indicates failed to submit a StatReport.\nvar ErrWriteFailed = errors.New(\"submit failed\")\n\n// A RemoteWriter is a writer to write StatReport.\ntype RemoteWriter struct {\n\tendpoint string\n}\n\n// NewRemoteWriter returns a RemoteWriter.\nfunc NewRemoteWriter(endpoint string) Writer {\n\treturn &RemoteWriter{\n\t\tendpoint: endpoint,\n\t}\n}\n\nfunc (rw *RemoteWriter) Write(report *StatReport) error {\n\tbs, err := json.Marshal(report)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tclient := &http.Client{\n\t\tTimeout: httpTimeout,\n\t}\n\tresp, err := client.Post(rw.endpoint, jsonContentType, bytes.NewReader(bs))\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != http.StatusOK {\n\t\tlogx.Errorf(\"write report failed, code: %d, reason: %s\", resp.StatusCode, resp.Status)\n\t\treturn ErrWriteFailed\n\t}\n\n\treturn nil\n}\n\n// Path: core/stat/remotewriter_test.go\npackage stat\n\nimport (\n\t\"errors\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"gopkg.in/h2non/gock.v1\"\n)\n\nfunc TestRemoteWriter(t *testing.T) {\n\tdefer gock.Off()\n\n\tgock.New(\"http://foo.com\").Reply(200).BodyString(\"foo\")\n\twriter := NewRemoteWriter(\"http://foo.com\")\n\terr := writer.Write(&StatReport{\n\t\tName: \"bar\",\n\t})\n\tassert.Nil(t, err)\n}\n\nfunc TestRemoteWriterFail(t *testing.T) {\n\tdefer gock.Off()\n\n\tgock.New(\"http://foo.com\").Reply(503).BodyString(\"foo\")\n\twriter := NewRemoteWriter(\"http://foo.com\")\n\terr := writer.Write(&StatReport{\n\t\tName: \"bar\",\n\t})\n\tassert.NotNil(t, err)\n}\n\nfunc TestRemoteWriterError(t *testing.T) {\n\tdefer gock.Off()\n\n\tgock.New(\"http://foo.com\").ReplyError(errors.New(\"foo\"))\n\twriter := NewRemoteWriter(\"http://foo.com\")\n\terr := writer.Write(&StatReport{\n\t\tName: \"bar\",\n\t})\n\tassert.NotNil(t, err)\n}\n\n// Path: core/stat/topk.go\npackage stat\n\nimport \"container/heap\"\n\ntype taskHeap []Task\n\nfunc (h *taskHeap) Len() int {\n\treturn len(*h)\n}\n\nfunc (h *taskHeap) Less(i, j int) bool {\n\treturn (*h)[i].Duration < (*h)[j].Duration\n}\n\nfunc (h *taskHeap) Swap(i, j int) {\n\t(*h)[i], (*h)[j] = (*h)[j], (*h)[i]\n}\n\nfunc (h *taskHeap) Push(x any) {\n\t*h = append(*h, x.(Task))\n}\n\nfunc (h *taskHeap) Pop() any {\n\told := *h\n\tn := len(old)\n\tx := old[n-1]\n\t*h = old[0 : n-1]\n\treturn x\n}\n\nfunc topK(all []Task, k int) []Task {\n\th := new(taskHeap)\n\theap.Init(h)\n\n\tfor _, each := range all {\n\t\tif h.Len() < k {\n\t\t\theap.Push(h, each)\n\t\t} else if (*h)[0].Duration < each.Duration {\n\t\t\theap.Pop(h)\n\t\t\theap.Push(h, each)\n\t\t}\n\t}\n\n\treturn *h\n}\n\n// Path: core/stat/topk_test.go\npackage stat\n\nimport (\n\t\"math/rand\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst (\n\tnumSamples = 10000\n\ttopNum     = 100\n)\n\nvar samples []Task\n\nfunc init() {\n\tfor i := 0; i < numSamples; i++ {\n\t\ttask := Task{\n\t\t\tDuration: time.Duration(rand.Int63()),\n\t\t}\n\t\tsamples = append(samples, task)\n\t}\n}\n\nfunc TestTopK(t *testing.T) {\n\ttasks := []Task{\n\t\t{false, 1, \"a\"},\n\t\t{false, 4, \"a\"},\n\t\t{false, 2, \"a\"},\n\t\t{false, 5, \"a\"},\n\t\t{false, 9, \"a\"},\n\t\t{false, 10, \"a\"},\n\t\t{false, 12, \"a\"},\n\t\t{false, 3, \"a\"},\n\t\t{false, 6, \"a\"},\n\t\t{false, 11, \"a\"},\n\t\t{false, 8, \"a\"},\n\t}\n\n\tresult := topK(tasks, 3)\n\tif len(result) != 3 {\n\t\tt.Fail()\n\t}\n\n\tset := make(map[time.Duration]struct{})\n\tfor _, each := range result {\n\t\tset[each.Duration] = struct{}{}\n\t}\n\n\tfor _, v := range []time.Duration{10, 11, 12} {\n\t\t_, ok := set[v]\n\t\tassert.True(t, ok)\n\t}\n}\n\nfunc BenchmarkTopkHeap(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\ttopK(samples, topNum)\n\t}\n}\n\n// Path: core/stat/usage.go\npackage stat\n\nimport (\n\t\"runtime\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/stat/internal\"\n\t\"github.com/zeromicro/go-zero/core/threading\"\n)\n\nconst (\n\t// 250ms and 0.95 as beta will count the average cpu load for past 5 seconds\n\tcpuRefreshInterval = time.Millisecond * 250\n\tallRefreshInterval = time.Minute\n\t// moving average beta hyperparameter\n\tbeta = 0.95\n)\n\nvar cpuUsage int64\n\nfunc init() {\n\tgo func() {\n\t\tcpuTicker := time.NewTicker(cpuRefreshInterval)\n\t\tdefer cpuTicker.Stop()\n\t\tallTicker := time.NewTicker(allRefreshInterval)\n\t\tdefer allTicker.Stop()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-cpuTicker.C:\n\t\t\t\tthreading.RunSafe(func() {\n\t\t\t\t\tcurUsage := internal.RefreshCpu()\n\t\t\t\t\tprevUsage := atomic.LoadInt64(&cpuUsage)\n\t\t\t\t\t// cpu = cpu\u1d57\u207b\u00b9 * beta + cpu\u1d57 * (1 - beta)\n\t\t\t\t\tusage := int64(float64(prevUsage)*beta + float64(curUsage)*(1-beta))\n\t\t\t\t\tatomic.StoreInt64(&cpuUsage, usage)\n\t\t\t\t})\n\t\t\tcase <-allTicker.C:\n\t\t\t\tif logEnabled.True() {\n\t\t\t\t\tprintUsage()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n}\n\n// CpuUsage returns current cpu usage.\nfunc CpuUsage() int64 {\n\treturn atomic.LoadInt64(&cpuUsage)\n}\n\nfunc bToMb(b uint64) float32 {\n\treturn float32(b) / 1024 / 1024\n}\n\nfunc printUsage() {\n\tvar m runtime.MemStats\n\truntime.ReadMemStats(&m)\n\tlogx.Statf(\"CPU: %dm, MEMORY: Alloc=%.1fMi, TotalAlloc=%.1fMi, Sys=%.1fMi, NumGC=%d\",\n\t\tCpuUsage(), bToMb(m.Alloc), bToMb(m.TotalAlloc), bToMb(m.Sys), m.NumGC)\n}\n\n// Path: core/stat/usage_test.go\npackage stat\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/logx/logtest\"\n)\n\nfunc TestBToMb(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\tbytes    uint64\n\t\texpected float32\n\t}{\n\t\t{\n\t\t\tname:     \"Test 1: Convert 0 bytes to MB\",\n\t\t\tbytes:    0,\n\t\t\texpected: 0,\n\t\t},\n\t\t{\n\t\t\tname:     \"Test 2: Convert 1048576 bytes to MB\",\n\t\t\tbytes:    1048576,\n\t\t\texpected: 1,\n\t\t},\n\t\t{\n\t\t\tname:     \"Test 3: Convert 2097152 bytes to MB\",\n\t\t\tbytes:    2097152,\n\t\t\texpected: 2,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tresult := bToMb(test.bytes)\n\t\t\tassert.Equal(t, test.expected, result)\n\t\t})\n\t}\n}\n\nfunc TestPrintUsage(t *testing.T) {\n\tc := logtest.NewCollector(t)\n\n\tprintUsage()\n\n\toutput := c.String()\n\tassert.Contains(t, output, \"CPU:\")\n\tassert.Contains(t, output, \"MEMORY:\")\n\tassert.Contains(t, output, \"Alloc=\")\n\tassert.Contains(t, output, \"TotalAlloc=\")\n\tassert.Contains(t, output, \"Sys=\")\n\tassert.Contains(t, output, \"NumGC=\")\n\n\tlines := strings.Split(output, \"\\n\")\n\tassert.Len(t, lines, 2)\n\tfields := strings.Split(lines[0], \", \")\n\tassert.Len(t, fields, 5)\n}\n\n// Path: core/stores/builder/builder_test.go\npackage builder\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype mockedUser struct {\n\tID       string `db:\"id\" json:\"id,omitempty\"`\n\tUserName string `db:\"user_name\" json:\"userName,omitempty\"`\n\tSex      int    `db:\"sex\" json:\"sex,omitempty\"`\n\tUUID     string `db:\"uuid\" uuid:\"uuid,omitempty\"`\n\tAge      int    `db:\"age\" json:\"age\"`\n}\n\nfunc TestFieldNames(t *testing.T) {\n\tt.Run(\"new\", func(t *testing.T) {\n\t\tvar u mockedUser\n\t\tout := RawFieldNames(&u)\n\t\texpected := []string{\"`id`\", \"`user_name`\", \"`sex`\", \"`uuid`\", \"`age`\"}\n\t\tassert.Equal(t, expected, out)\n\t})\n}\n\ntype mockedUserWithOptions struct {\n\tID       string `db:\"id\" json:\"id,omitempty\"`\n\tUserName string `db:\"user_name,type=varchar,length=255\" json:\"userName,omitempty\"`\n\tSex      int    `db:\"sex\" json:\"sex,omitempty\"`\n\tUUID     string `db:\",type=varchar,length=16\" uuid:\"uuid,omitempty\"`\n\tAge      int    `db:\"age\" json:\"age\"`\n}\n\nfunc TestFieldNamesWithTagOptions(t *testing.T) {\n\tt.Run(\"new\", func(t *testing.T) {\n\t\tvar u mockedUserWithOptions\n\t\tout := RawFieldNames(&u)\n\t\texpected := []string{\"`id`\", \"`user_name`\", \"`sex`\", \"`UUID`\", \"`age`\"}\n\t\tassert.Equal(t, expected, out)\n\t})\n}\n\ntype mockedUserWithDashTag struct {\n\tID       string `db:\"id\" json:\"id,omitempty\"`\n\tUserName string `db:\"user_name\" json:\"userName,omitempty\"`\n\tMobile   string `db:\"-\" json:\"mobile,omitempty\"`\n}\n\nfunc TestFieldNamesWithDashTag(t *testing.T) {\n\tt.Run(\"new\", func(t *testing.T) {\n\t\tvar u mockedUserWithDashTag\n\t\tout := RawFieldNames(&u)\n\t\texpected := []string{\"`id`\", \"`user_name`\"}\n\t\tassert.Equal(t, expected, out)\n\t})\n}\n\ntype mockedUserWithDashTagAndOptions struct {\n\tID       string `db:\"id\" json:\"id,omitempty\"`\n\tUserName string `db:\"user_name,type=varchar,length=255\" json:\"userName,omitempty\"`\n\tMobile   string `db:\"-,type=varchar,length=255\" json:\"mobile,omitempty\"`\n}\n\nfunc TestFieldNamesWithDashTagAndOptions(t *testing.T) {\n\tt.Run(\"new\", func(t *testing.T) {\n\t\tvar u mockedUserWithDashTagAndOptions\n\t\tout := RawFieldNames(&u)\n\t\texpected := []string{\"`id`\", \"`user_name`\"}\n\t\tassert.Equal(t, expected, out)\n\t})\n}\n\n\nfunc TestPostgreSqlJoin(t *testing.T) {\n\t// Test with empty input array\n\tvar input []string\n\tvar expectedOutput string\n\tassert.Equal(t, expectedOutput, PostgreSqlJoin(input))\n\n\t// Test with single element input array\n\tinput = []string{\"foo\"}\n\texpectedOutput = \"foo = $2\"\n\tassert.Equal(t, expectedOutput, PostgreSqlJoin(input))\n\n\t// Test with multiple elements input array\n\tinput = []string{\"foo\", \"bar\", \"baz\"}\n\texpectedOutput = \"foo = $2, bar = $3, baz = $4\"\n\tassert.Equal(t, expectedOutput, PostgreSqlJoin(input))\n}\n\ntype testStruct struct {\n\tFoo string `db:\"foo\"`\n\tBar int    `db:\"bar\"`\n\tBaz bool   `db:\"-\"`\n}\n\nfunc TestRawFieldNames(t *testing.T) {\n\t// Test with a struct without tags\n\tin := struct {\n\t\tFoo string\n\t\tBar int\n\t}{}\n\texpectedOutput := []string{\"`Foo`\", \"`Bar`\"}\n\tassert.ElementsMatch(t, expectedOutput, RawFieldNames(in))\n\n\t// Test pg without db tag\n\texpectedOutput = []string{\"Foo\", \"Bar\"}\n\tassert.ElementsMatch(t, expectedOutput, RawFieldNames(in, true))\n\n\t// Test with a struct with tags\n\tinput := testStruct{}\n\texpectedOutput = []string{\"`foo`\", \"`bar`\"}\n\tassert.ElementsMatch(t, expectedOutput, RawFieldNames(input))\n\n\t// Test with nil input (pointer)\n\tvar nilInput *testStruct\n\tassert.Panics(t, func() {\n\t\tRawFieldNames(nilInput)\n\t}, \"RawFieldNames should panic with nil input\")\n\n\t// Test with non-struct input\n\tinputInt := 42\n\tassert.Panics(t, func() {\n\t\tRawFieldNames(inputInt)\n\t}, \"RawFieldNames should panic with non-struct input\")\n\n\t// Test with PostgreSQL flag\n\tinput = testStruct{}\n\texpectedOutput = []string{\"foo\", \"bar\"}\n\tassert.ElementsMatch(t, expectedOutput, RawFieldNames(input, true))\n}\n\n// Path: core/stores/cache/cache_test.go\npackage cache\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/alicebob/miniredis/v2\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/errorx\"\n\t\"github.com/zeromicro/go-zero/core/hash\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis/redistest\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n)\n\nvar _ Cache = (*mockedNode)(nil)\n\ntype mockedNode struct {\n\tvals        map[string][]byte\n\terrNotFound error\n}\n\nfunc (mc *mockedNode) Del(keys ...string) error {\n\treturn mc.DelCtx(context.Background(), keys...)\n}\n\nfunc (mc *mockedNode) DelCtx(_ context.Context, keys ...string) error {\n\tvar be errorx.BatchError\n\n\tfor _, key := range keys {\n\t\tif _, ok := mc.vals[key]; !ok {\n\t\t\tbe.Add(mc.errNotFound)\n\t\t} else {\n\t\t\tdelete(mc.vals, key)\n\t\t}\n\t}\n\n\treturn be.Err()\n}\n\nfunc (mc *mockedNode) Get(key string, val any) error {\n\treturn mc.GetCtx(context.Background(), key, val)\n}\n\nfunc (mc *mockedNode) GetCtx(ctx context.Context, key string, val any) error {\n\tbs, ok := mc.vals[key]\n\tif ok {\n\t\treturn json.Unmarshal(bs, val)\n\t}\n\n\treturn mc.errNotFound\n}\n\nfunc (mc *mockedNode) IsNotFound(err error) bool {\n\treturn errors.Is(err, mc.errNotFound)\n}\n\nfunc (mc *mockedNode) Set(key string, val any) error {\n\treturn mc.SetCtx(context.Background(), key, val)\n}\n\nfunc (mc *mockedNode) SetCtx(ctx context.Context, key string, val any) error {\n\tdata, err := json.Marshal(val)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmc.vals[key] = data\n\treturn nil\n}\n\nfunc (mc *mockedNode) SetWithExpire(key string, val any, expire time.Duration) error {\n\treturn mc.SetWithExpireCtx(context.Background(), key, val, expire)\n}\n\nfunc (mc *mockedNode) SetWithExpireCtx(ctx context.Context, key string, val any, expire time.Duration) error {\n\treturn mc.Set(key, val)\n}\n\nfunc (mc *mockedNode) Take(val any, key string, query func(val any) error) error {\n\treturn mc.TakeCtx(context.Background(), val, key, query)\n}\n\nfunc (mc *mockedNode) TakeCtx(ctx context.Context, val any, key string, query func(val any) error) error {\n\tif _, ok := mc.vals[key]; ok {\n\t\treturn mc.GetCtx(ctx, key, val)\n\t}\n\n\tif err := query(val); err != nil {\n\t\treturn err\n\t}\n\n\treturn mc.SetCtx(ctx, key, val)\n}\n\nfunc (mc *mockedNode) TakeWithExpire(val any, key string, query func(val any, expire time.Duration) error) error {\n\treturn mc.TakeWithExpireCtx(context.Background(), val, key, query)\n}\n\nfunc (mc *mockedNode) TakeWithExpireCtx(ctx context.Context, val any, key string, query func(val any, expire time.Duration) error) error {\n\treturn mc.Take(val, key, func(val any) error {\n\t\treturn query(val, 0)\n\t})\n}\n\nfunc TestCache_SetDel(t *testing.T) {\n\tt.Run(\"test set del\", func(t *testing.T) {\n\t\tconst total = 1000\n\t\tr1 := redistest.CreateRedis(t)\n\t\tr2 := redistest.CreateRedis(t)\n\t\tconf := ClusterConf{\n\t\t\t{\n\t\t\t\tRedisConf: redis.RedisConf{\n\t\t\t\t\tHost: r1.Addr,\n\t\t\t\t\tType: redis.NodeType,\n\t\t\t\t},\n\t\t\t\tWeight: 100,\n\t\t\t},\n\t\t\t{\n\t\t\t\tRedisConf: redis.RedisConf{\n\t\t\t\t\tHost: r2.Addr,\n\t\t\t\t\tType: redis.NodeType,\n\t\t\t\t},\n\t\t\t\tWeight: 100,\n\t\t\t},\n\t\t}\n\t\tc := New(conf, syncx.NewSingleFlight(), NewStat(\"mock\"), errPlaceholder)\n\t\tfor i := 0; i < total; i++ {\n\t\t\tif i%2 == 0 {\n\t\t\t\tassert.Nil(t, c.Set(fmt.Sprintf(\"key/%d\", i), i))\n\t\t\t} else {\n\t\t\t\tassert.Nil(t, c.SetWithExpire(fmt.Sprintf(\"key/%d\", i), i, 0))\n\t\t\t}\n\t\t}\n\t\tfor i := 0; i < total; i++ {\n\t\t\tvar val int\n\t\t\tassert.Nil(t, c.Get(fmt.Sprintf(\"key/%d\", i), &val))\n\t\t\tassert.Equal(t, i, val)\n\t\t}\n\t\tassert.Nil(t, c.Del())\n\t\tfor i := 0; i < total; i++ {\n\t\t\tassert.Nil(t, c.Del(fmt.Sprintf(\"key/%d\", i)))\n\t\t}\n\t\tassert.Nil(t, c.Del(\"a\", \"b\", \"c\"))\n\t\tfor i := 0; i < total; i++ {\n\t\t\tvar val int\n\t\t\tassert.True(t, c.IsNotFound(c.Get(fmt.Sprintf(\"key/%d\", i), &val)))\n\t\t\tassert.Equal(t, 0, val)\n\t\t}\n\t})\n\n\tt.Run(\"test set del error\", func(t *testing.T) {\n\t\tr1, err := miniredis.Run()\n\t\tassert.NoError(t, err)\n\t\tdefer r1.Close()\n\n\t\tr2, err := miniredis.Run()\n\t\tassert.NoError(t, err)\n\t\tdefer r2.Close()\n\n\t\tconf := ClusterConf{\n\t\t\t{\n\t\t\t\tRedisConf: redis.RedisConf{\n\t\t\t\t\tHost: r1.Addr(),\n\t\t\t\t\tType: redis.NodeType,\n\t\t\t\t},\n\t\t\t\tWeight: 100,\n\t\t\t},\n\t\t\t{\n\t\t\t\tRedisConf: redis.RedisConf{\n\t\t\t\t\tHost: r2.Addr(),\n\t\t\t\t\tType: redis.NodeType,\n\t\t\t\t},\n\t\t\t\tWeight: 100,\n\t\t\t},\n\t\t}\n\t\tc := New(conf, syncx.NewSingleFlight(), NewStat(\"mock\"), errPlaceholder)\n\t\tr1.SetError(\"mock error\")\n\t\tr2.SetError(\"mock error\")\n\t\tassert.NoError(t, c.Del(\"a\", \"b\", \"c\"))\n\t})\n}\n\nfunc TestCache_OneNode(t *testing.T) {\n\tconst total = 1000\n\tr := redistest.CreateRedis(t)\n\tconf := ClusterConf{\n\t\t{\n\t\t\tRedisConf: redis.RedisConf{\n\t\t\t\tHost: r.Addr,\n\t\t\t\tType: redis.NodeType,\n\t\t\t},\n\t\t\tWeight: 100,\n\t\t},\n\t}\n\tc := New(conf, syncx.NewSingleFlight(), NewStat(\"mock\"), errPlaceholder)\n\tfor i := 0; i < total; i++ {\n\t\tif i%2 == 0 {\n\t\t\tassert.Nil(t, c.Set(fmt.Sprintf(\"key/%d\", i), i))\n\t\t} else {\n\t\t\tassert.Nil(t, c.SetWithExpire(fmt.Sprintf(\"key/%d\", i), i, 0))\n\t\t}\n\t}\n\tfor i := 0; i < total; i++ {\n\t\tvar val int\n\t\tassert.Nil(t, c.Get(fmt.Sprintf(\"key/%d\", i), &val))\n\t\tassert.Equal(t, i, val)\n\t}\n\tassert.Nil(t, c.Del())\n\tfor i := 0; i < total; i++ {\n\t\tassert.Nil(t, c.Del(fmt.Sprintf(\"key/%d\", i)))\n\t}\n\tfor i := 0; i < total; i++ {\n\t\tvar val int\n\t\tassert.True(t, c.IsNotFound(c.Get(fmt.Sprintf(\"key/%d\", i), &val)))\n\t\tassert.Equal(t, 0, val)\n\t}\n}\n\nfunc TestCache_Balance(t *testing.T) {\n\tconst (\n\t\tnumNodes = 100\n\t\ttotal    = 10000\n\t)\n\tdispatcher := hash.NewConsistentHash()\n\tmaps := make([]map[string][]byte, numNodes)\n\tfor i := 0; i < numNodes; i++ {\n\t\tmaps[i] = map[string][]byte{\n\t\t\tstrconv.Itoa(i): []byte(strconv.Itoa(i)),\n\t\t}\n\t}\n\tfor i := 0; i < numNodes; i++ {\n\t\tdispatcher.AddWithWeight(&mockedNode{\n\t\t\tvals:        maps[i],\n\t\t\terrNotFound: errPlaceholder,\n\t\t}, 100)\n\t}\n\n\tc := cacheCluster{\n\t\tdispatcher:  dispatcher,\n\t\terrNotFound: errPlaceholder,\n\t}\n\tfor i := 0; i < total; i++ {\n\t\tassert.Nil(t, c.Set(strconv.Itoa(i), i))\n\t}\n\n\tcounts := make(map[int]int)\n\tfor i, m := range maps {\n\t\tcounts[i] = len(m)\n\t}\n\tentropy := calcEntropy(counts, total)\n\tassert.True(t, len(counts) > 1)\n\tassert.True(t, entropy > .95, fmt.Sprintf(\"entropy should be greater than 0.95, but got %.2f\", entropy))\n\n\tfor i := 0; i < total; i++ {\n\t\tvar val int\n\t\tassert.Nil(t, c.Get(strconv.Itoa(i), &val))\n\t\tassert.Equal(t, i, val)\n\t}\n\n\tfor i := 0; i < total/10; i++ {\n\t\tassert.Nil(t, c.Del(strconv.Itoa(i*10), strconv.Itoa(i*10+1), strconv.Itoa(i*10+2)))\n\t\tassert.Nil(t, c.Del(strconv.Itoa(i*10+9)))\n\t}\n\n\tvar count int\n\tfor i := 0; i < total/10; i++ {\n\t\tvar val int\n\t\tif i%2 == 0 {\n\t\t\tassert.Nil(t, c.Take(&val, strconv.Itoa(i*10), func(val any) error {\n\t\t\t\t*val.(*int) = i\n\t\t\t\tcount++\n\t\t\t\treturn nil\n\t\t\t}))\n\t\t} else {\n\t\t\tassert.Nil(t, c.TakeWithExpire(&val, strconv.Itoa(i*10), func(val any, expire time.Duration) error {\n\t\t\t\t*val.(*int) = i\n\t\t\t\tcount++\n\t\t\t\treturn nil\n\t\t\t}))\n\t\t}\n\t\tassert.Equal(t, i, val)\n\t}\n\tassert.Equal(t, total/10, count)\n}\n\nfunc TestCacheNoNode(t *testing.T) {\n\tdispatcher := hash.NewConsistentHash()\n\tc := cacheCluster{\n\t\tdispatcher:  dispatcher,\n\t\terrNotFound: errPlaceholder,\n\t}\n\tassert.NotNil(t, c.Del(\"foo\"))\n\tassert.NotNil(t, c.Del(\"foo\", \"bar\", \"any\"))\n\tassert.NotNil(t, c.Get(\"foo\", nil))\n\tassert.NotNil(t, c.Set(\"foo\", nil))\n\tassert.NotNil(t, c.SetWithExpire(\"foo\", nil, time.Second))\n\tassert.NotNil(t, c.Take(nil, \"foo\", func(val any) error {\n\t\treturn nil\n\t}))\n\tassert.NotNil(t, c.TakeWithExpire(nil, \"foo\", func(val any, duration time.Duration) error {\n\t\treturn nil\n\t}))\n}\n\nfunc calcEntropy(m map[int]int, total int) float64 {\n\tvar entropy float64\n\n\tfor _, val := range m {\n\t\tproba := float64(val) / float64(total)\n\t\tentropy -= proba * math.Log2(proba)\n\t}\n\n\treturn entropy / math.Log2(float64(len(m)))\n}\n\n// Path: core/stores/cache/cacheconf.go\npackage cache\n\n// CacheConf is an alias of ClusterConf.\ntype CacheConf = ClusterConf\n\n// Path: core/stores/cache/cachenode.go\npackage cache\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/jsonx\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/mathx\"\n\t\"github.com/zeromicro/go-zero/core/stat\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n)\n\nconst (\n\tnotFoundPlaceholder = \"*\"\n\t// make the expiry unstable to avoid lots of cached items expire at the same time\n\t// make the unstable expiry to be [0.95, 1.05] * seconds\n\texpiryDeviation = 0.05\n)\n\n// indicates there is no such value associate with the key\nvar errPlaceholder = errors.New(\"placeholder\")\n\ntype cacheNode struct {\n\trds            *redis.Redis\n\texpiry         time.Duration\n\tnotFoundExpiry time.Duration\n\tbarrier        syncx.SingleFlight\n\tr              *rand.Rand\n\tlock           *sync.Mutex\n\tunstableExpiry mathx.Unstable\n\tstat           *Stat\n\terrNotFound    error\n}\n\n// NewNode returns a cacheNode.\n// rds is the underlying redis node or cluster.\n// barrier is the barrier that maybe shared with other cache nodes on cache cluster.\n// st is used to stat the cache.\n// errNotFound defines the error that returned on cache not found.\n// opts are the options that customize the cacheNode.\nfunc NewNode(rds *redis.Redis, barrier syncx.SingleFlight, st *Stat,\n\terrNotFound error, opts ...Option) Cache {\n\to := newOptions(opts...)\n\treturn cacheNode{\n\t\trds:            rds,\n\t\texpiry:         o.Expiry,\n\t\tnotFoundExpiry: o.NotFoundExpiry,\n\t\tbarrier:        barrier,\n\t\tr:              rand.New(rand.NewSource(time.Now().UnixNano())),\n\t\tlock:           new(sync.Mutex),\n\t\tunstableExpiry: mathx.NewUnstable(expiryDeviation),\n\t\tstat:           st,\n\t\terrNotFound:    errNotFound,\n\t}\n}\n\n// Del deletes cached values with keys.\nfunc (c cacheNode) Del(keys ...string) error {\n\treturn c.DelCtx(context.Background(), keys...)\n}\n\n// DelCtx deletes cached values with keys.\nfunc (c cacheNode) DelCtx(ctx context.Context, keys ...string) error {\n\tif len(keys) == 0 {\n\t\treturn nil\n\t}\n\n\tlogger := logx.WithContext(ctx)\n\tif len(keys) > 1 && c.rds.Type == redis.ClusterType {\n\t\tfor _, key := range keys {\n\t\t\tif _, err := c.rds.DelCtx(ctx, key); err != nil {\n\t\t\t\tlogger.Errorf(\"failed to clear cache with key: %q, error: %v\", key, err)\n\t\t\t\tc.asyncRetryDelCache(key)\n\t\t\t}\n\t\t}\n\t} else if _, err := c.rds.DelCtx(ctx, keys...); err != nil {\n\t\tlogger.Errorf(\"failed to clear cache with keys: %q, error: %v\", formatKeys(keys), err)\n\t\tc.asyncRetryDelCache(keys...)\n\t}\n\n\treturn nil\n}\n\n// Get gets the cache with key and fills into v.\nfunc (c cacheNode) Get(key string, val any) error {\n\treturn c.GetCtx(context.Background(), key, val)\n}\n\n// GetCtx gets the cache with key and fills into v.\nfunc (c cacheNode) GetCtx(ctx context.Context, key string, val any) error {\n\terr := c.doGetCache(ctx, key, val)\n\tif errors.Is(err, errPlaceholder) {\n\t\treturn c.errNotFound\n\t}\n\n\treturn err\n}\n\n// IsNotFound checks if the given error is the defined errNotFound.\nfunc (c cacheNode) IsNotFound(err error) bool {\n\treturn errors.Is(err, c.errNotFound)\n}\n\n// Set sets the cache with key and v, using c.expiry.\nfunc (c cacheNode) Set(key string, val any) error {\n\treturn c.SetCtx(context.Background(), key, val)\n}\n\n// SetCtx sets the cache with key and v, using c.expiry.\nfunc (c cacheNode) SetCtx(ctx context.Context, key string, val any) error {\n\treturn c.SetWithExpireCtx(ctx, key, val, c.aroundDuration(c.expiry))\n}\n\n// SetWithExpire sets the cache with key and v, using given expire.\nfunc (c cacheNode) SetWithExpire(key string, val any, expire time.Duration) error {\n\treturn c.SetWithExpireCtx(context.Background(), key, val, expire)\n}\n...\n", "needle_token_start": 9674, "needle_token_end": 9840, "code_context_ntokens": 15090, "output": ["```go\nfunc PostgreSqlJoin(input []string) string {\n\tvar output strings.Builder\n\tfor i, field := range input {\n\t\tif i > 0 {\n\t\t\toutput.WriteString(\", \")\n\t\t}\n\t\toutput.WriteString(field)\n\t\toutput.WriteString(\" = $\")\n\t\toutput.WriteString(strconv.Itoa(i + 2))\n\t}\n\treturn output.String()\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "TestField", "language": "go", "path": "core/logx/logs_test.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to verify the correct formatting and handling of different types of log fields, ensuring that each type is converted and represented accurately in the log output.\n2. **Input**: The function takes a series of predefined test cases, each containing a specific log field type and the expected output representation of that field.\n3. **Output**: There is no direct output from the function; instead, it checks if the log output matches the expected results for each test case, indicating success or failure of the test.\n4. **Procedure**: The function iterates over a list of test cases. For each case, it sets up a mock log writer, injects a log entry using the specified field, and then checks if the resulting log output matches the expected map of values. The test ensures that various data types (errors, strings, durations, times, and custom stringers) are logged correctly.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/logc/logs_test.go\npackage logc\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"runtime\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/logx/logtest\"\n)\n\nfunc TestAddGlobalFields(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\n\tInfo(context.Background(), \"hello\")\n\tbuf.Reset()\n\n\tAddGlobalFields(Field(\"a\", \"1\"), Field(\"b\", \"2\"))\n\tAddGlobalFields(Field(\"c\", \"3\"))\n\tInfo(context.Background(), \"world\")\n\tvar m map[string]any\n\tassert.NoError(t, json.Unmarshal(buf.Bytes(), &m))\n\tassert.Equal(t, \"1\", m[\"a\"])\n\tassert.Equal(t, \"2\", m[\"b\"])\n\tassert.Equal(t, \"3\", m[\"c\"])\n}\n\nfunc TestAlert(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tAlert(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), \"foo\"), buf.String())\n}\n\nfunc TestError(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tError(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestErrorf(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tErrorf(context.Background(), \"foo %s\", \"bar\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestErrorv(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tErrorv(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestErrorw(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tErrorw(context.Background(), \"foo\", Field(\"a\", \"b\"))\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestInfo(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tInfo(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestInfof(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tInfof(context.Background(), \"foo %s\", \"bar\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestInfov(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tInfov(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestInfow(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tInfow(context.Background(), \"foo\", Field(\"a\", \"b\"))\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestDebug(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tDebug(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestDebugf(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tDebugf(context.Background(), \"foo %s\", \"bar\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestDebugv(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tDebugv(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestDebugw(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tDebugw(context.Background(), \"foo\", Field(\"a\", \"b\"))\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestMust(t *testing.T) {\n\tassert.NotPanics(t, func() {\n\t\tMust(nil)\n\t})\n\tassert.NotPanics(t, func() {\n\t\tMustSetup(LogConf{})\n\t})\n}\n\nfunc TestMisc(t *testing.T) {\n\tSetLevel(logx.DebugLevel)\n\tassert.NoError(t, SetUp(LogConf{}))\n\tassert.NoError(t, Close())\n}\n\nfunc TestSlow(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tSlow(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)), buf.String())\n}\n\nfunc TestSlowf(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tSlowf(context.Background(), \"foo %s\", \"bar\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)), buf.String())\n}\n\nfunc TestSlowv(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tSlowv(context.Background(), \"foo\")\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)), buf.String())\n}\n\nfunc TestSloww(t *testing.T) {\n\tbuf := logtest.NewCollector(t)\n\tfile, line := getFileLine()\n\tSloww(context.Background(), \"foo\", Field(\"a\", \"b\"))\n\tassert.True(t, strings.Contains(buf.String(), fmt.Sprintf(\"%s:%d\", file, line+1)), buf.String())\n}\n\nfunc getFileLine() (string, int) {\n\t_, file, line, _ := runtime.Caller(1)\n\tshort := file\n\n\tfor i := len(file) - 1; i > 0; i-- {\n\t\tif file[i] == '/' {\n\t\t\tshort = file[i+1:]\n...\n// Path: core/logx/color_test.go\npackage logx\n\nimport (\n\t\"sync/atomic\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/color\"\n)\n\nfunc TestWithColor(t *testing.T) {\n\told := atomic.SwapUint32(&encoding, plainEncodingType)\n\tdefer atomic.StoreUint32(&encoding, old)\n\n\toutput := WithColor(\"hello\", color.BgBlue)\n\tassert.Equal(t, \"hello\", output)\n\n\tatomic.StoreUint32(&encoding, jsonEncodingType)\n\toutput = WithColor(\"hello\", color.BgBlue)\n\tassert.Equal(t, \"hello\", output)\n}\n\nfunc TestWithColorPadding(t *testing.T) {\n\told := atomic.SwapUint32(&encoding, plainEncodingType)\n\tdefer atomic.StoreUint32(&encoding, old)\n\n\toutput := WithColorPadding(\"hello\", color.BgBlue)\n\tassert.Equal(t, \" hello \", output)\n\n\tatomic.StoreUint32(&encoding, jsonEncodingType)\n\toutput = WithColorPadding(\"hello\", color.BgBlue)\n\tassert.Equal(t, \"hello\", output)\n}\n\n// Path: core/logx/fields_test.go\npackage logx\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestAddGlobalFields(t *testing.T) {\n\tvar buf bytes.Buffer\n\twriter := NewWriter(&buf)\n\told := Reset()\n\tSetWriter(writer)\n\tdefer SetWriter(old)\n\n\tInfo(\"hello\")\n\tbuf.Reset()\n\n\tAddGlobalFields(Field(\"a\", \"1\"), Field(\"b\", \"2\"))\n\tAddGlobalFields(Field(\"c\", \"3\"))\n\tInfo(\"world\")\n\tvar m map[string]any\n\tassert.NoError(t, json.Unmarshal(buf.Bytes(), &m))\n\tassert.Equal(t, \"1\", m[\"a\"])\n\tassert.Equal(t, \"2\", m[\"b\"])\n\tassert.Equal(t, \"3\", m[\"c\"])\n}\n\nfunc TestContextWithFields(t *testing.T) {\n\tctx := ContextWithFields(context.Background(), Field(\"a\", 1), Field(\"b\", 2))\n\tvals := ctx.Value(fieldsContextKey)\n\tassert.NotNil(t, vals)\n\tfields, ok := vals.([]LogField)\n\tassert.True(t, ok)\n\tassert.EqualValues(t, []LogField{Field(\"a\", 1), Field(\"b\", 2)}, fields)\n}\n\nfunc TestWithFields(t *testing.T) {\n\tctx := WithFields(context.Background(), Field(\"a\", 1), Field(\"b\", 2))\n\tvals := ctx.Value(fieldsContextKey)\n\tassert.NotNil(t, vals)\n\tfields, ok := vals.([]LogField)\n\tassert.True(t, ok)\n\tassert.EqualValues(t, []LogField{Field(\"a\", 1), Field(\"b\", 2)}, fields)\n}\n\nfunc TestWithFieldsAppend(t *testing.T) {\n\tvar dummyKey struct{}\n\tctx := context.WithValue(context.Background(), dummyKey, \"dummy\")\n\tctx = ContextWithFields(ctx, Field(\"a\", 1), Field(\"b\", 2))\n\tctx = ContextWithFields(ctx, Field(\"c\", 3), Field(\"d\", 4))\n\tvals := ctx.Value(fieldsContextKey)\n\tassert.NotNil(t, vals)\n\tfields, ok := vals.([]LogField)\n\tassert.True(t, ok)\n\tassert.Equal(t, \"dummy\", ctx.Value(dummyKey))\n\tassert.EqualValues(t, []LogField{\n\t\tField(\"a\", 1),\n\t\tField(\"b\", 2),\n\t\tField(\"c\", 3),\n\t\tField(\"d\", 4),\n\t}, fields)\n}\n\nfunc TestWithFieldsAppendCopy(t *testing.T) {\n\tconst count = 10\n\tctx := context.Background()\n\tfor i := 0; i < count; i++ {\n\t\tctx = ContextWithFields(ctx, Field(strconv.Itoa(i), 1))\n\t}\n\n\taf := Field(\"foo\", 1)\n\tbf := Field(\"bar\", 2)\n\tctxa := ContextWithFields(ctx, af)\n\tctxb := ContextWithFields(ctx, bf)\n\n\tassert.EqualValues(t, af, ctxa.Value(fieldsContextKey).([]LogField)[count])\n\tassert.EqualValues(t, bf, ctxb.Value(fieldsContextKey).([]LogField)[count])\n}\n\nfunc BenchmarkAtomicValue(b *testing.B) {\n\tb.ReportAllocs()\n\n\tvar container atomic.Value\n\tvals := []LogField{\n\t\tField(\"a\", \"b\"),\n\t\tField(\"c\", \"d\"),\n\t\tField(\"e\", \"f\"),\n\t}\n\tcontainer.Store(&vals)\n\n\tfor i := 0; i < b.N; i++ {\n\t\tval := container.Load()\n\t\tif val != nil {\n\t\t\t_ = *val.(*[]LogField)\n\t\t}\n\t}\n}\n\nfunc BenchmarkRWMutex(b *testing.B) {\n\tb.ReportAllocs()\n\n\tvar lock sync.RWMutex\n\tvals := []LogField{\n\t\tField(\"a\", \"b\"),\n\t\tField(\"c\", \"d\"),\n\t\tField(\"e\", \"f\"),\n\t}\n\n\tfor i := 0; i < b.N; i++ {\n\t\tlock.RLock()\n\t\t_ = vals\n\t\tlock.RUnlock()\n\t}\n}\n\n// Path: core/logx/logger.go\npackage logx\n\nimport (\n\t\"context\"\n\t\"time\"\n)\n\n// A Logger represents a logger.\ntype Logger interface {\n\t// Debug logs a message at info level.\n\tDebug(...any)\n\t// Debugf logs a message at info level.\n\tDebugf(string, ...any)\n\t// Debugv logs a message at info level.\n\tDebugv(any)\n\t// Debugw logs a message at info level.\n\tDebugw(string, ...LogField)\n\t// Error logs a message at error level.\n\tError(...any)\n\t// Errorf logs a message at error level.\n\tErrorf(string, ...any)\n\t// Errorv logs a message at error level.\n\tErrorv(any)\n\t// Errorw logs a message at error level.\n\tErrorw(string, ...LogField)\n\t// Info logs a message at info level.\n\tInfo(...any)\n\t// Infof logs a message at info level.\n\tInfof(string, ...any)\n\t// Infov logs a message at info level.\n\tInfov(any)\n\t// Infow logs a message at info level.\n\tInfow(string, ...LogField)\n\t// Slow logs a message at slow level.\n\tSlow(...any)\n\t// Slowf logs a message at slow level.\n\tSlowf(string, ...any)\n\t// Slowv logs a message at slow level.\n\tSlowv(any)\n\t// Sloww logs a message at slow level.\n\tSloww(string, ...LogField)\n\t// WithCallerSkip returns a new logger with the given caller skip.\n\tWithCallerSkip(skip int) Logger\n\t// WithContext returns a new logger with the given context.\n\tWithContext(ctx context.Context) Logger\n\t// WithDuration returns a new logger with the given duration.\n\tWithDuration(d time.Duration) Logger\n\t// WithFields returns a new logger with the given fields.\n\tWithFields(fields ...LogField) Logger\n}\n\n// Path: core/logx/lesslogger.go\npackage logx\n\n// A LessLogger is a logger that controls to log once during the given duration.\ntype LessLogger struct {\n\t*limitedExecutor\n}\n\n// NewLessLogger returns a LessLogger.\nfunc NewLessLogger(milliseconds int) *LessLogger {\n\treturn &LessLogger{\n\t\tlimitedExecutor: newLimitedExecutor(milliseconds),\n\t}\n}\n\n// Error logs v into error log or discard it if more than once in the given duration.\nfunc (logger *LessLogger) Error(v ...any) {\n\tlogger.logOrDiscard(func() {\n\t\tError(v...)\n\t})\n}\n\n// Errorf logs v with format into error log or discard it if more than once in the given duration.\nfunc (logger *LessLogger) Errorf(format string, v ...any) {\n\tlogger.logOrDiscard(func() {\n\t\tErrorf(format, v...)\n\t})\n}\n\n// Path: core/logx/lesslogger_test.go\npackage logx\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestLessLogger_Error(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tl := NewLessLogger(500)\n\tfor i := 0; i < 100; i++ {\n\t\tl.Error(\"hello\")\n\t}\n\n\tassert.Equal(t, 1, strings.Count(w.String(), \"\\n\"))\n}\n\nfunc TestLessLogger_Errorf(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tl := NewLessLogger(500)\n\tfor i := 0; i < 100; i++ {\n\t\tl.Errorf(\"hello\")\n\t}\n\n\tassert.Equal(t, 1, strings.Count(w.String(), \"\\n\"))\n}\n\n// Path: core/logx/lesswriter.go\npackage logx\n\nimport \"io\"\n\ntype lessWriter struct {\n\t*limitedExecutor\n\twriter io.Writer\n}\n\nfunc newLessWriter(writer io.Writer, milliseconds int) *lessWriter {\n\treturn &lessWriter{\n\t\tlimitedExecutor: newLimitedExecutor(milliseconds),\n\t\twriter:          writer,\n\t}\n}\n\nfunc (w *lessWriter) Write(p []byte) (n int, err error) {\n\tw.logOrDiscard(func() {\n\t\tw.writer.Write(p)\n\t})\n\treturn len(p), nil\n}\n\n// Path: core/logx/lesswriter_test.go\npackage logx\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestLessWriter(t *testing.T) {\n\tvar builder strings.Builder\n\tw := newLessWriter(&builder, 500)\n\tfor i := 0; i < 100; i++ {\n\t\t_, err := w.Write([]byte(\"hello\"))\n\t\tassert.Nil(t, err)\n\t}\n\n\tassert.Equal(t, \"hello\", builder.String())\n}\n\n// Path: core/logx/limitedexecutor.go\npackage logx\n\nimport (\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n\t\"github.com/zeromicro/go-zero/core/timex\"\n)\n\ntype limitedExecutor struct {\n\tthreshold time.Duration\n\tlastTime  *syncx.AtomicDuration\n\tdiscarded uint32\n}\n\nfunc newLimitedExecutor(milliseconds int) *limitedExecutor {\n\treturn &limitedExecutor{\n\t\tthreshold: time.Duration(milliseconds) * time.Millisecond,\n\t\tlastTime:  syncx.NewAtomicDuration(),\n\t}\n}\n\nfunc (le *limitedExecutor) logOrDiscard(execute func()) {\n\tif le == nil || le.threshold <= 0 {\n\t\texecute()\n\t\treturn\n\t}\n\n\tnow := timex.Now()\n\tif now-le.lastTime.Load() <= le.threshold {\n\t\tatomic.AddUint32(&le.discarded, 1)\n\t} else {\n\t\tle.lastTime.Set(now)\n\t\tdiscarded := atomic.SwapUint32(&le.discarded, 0)\n\t\tif discarded > 0 {\n\t\t\tErrorf(\"Discarded %d error messages\", discarded)\n\t\t}\n\n\t\texecute()\n\t}\n}\n\n// Path: core/logx/limitedexecutor_test.go\npackage logx\n\nimport (\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/timex\"\n)\n\nfunc TestLimitedExecutor_logOrDiscard(t *testing.T) {\n\ttests := []struct {\n\t\tname      string\n\t\tthreshold time.Duration\n\t\tlastTime  time.Duration\n\t\tdiscarded uint32\n\t\texecuted  bool\n\t}{\n\t\t{\n\t\t\tname:     \"nil executor\",\n\t\t\texecuted: true,\n\t\t},\n\t\t{\n\t\t\tname:      \"regular\",\n\t\t\tthreshold: time.Hour,\n\t\t\tlastTime:  timex.Now(),\n\t\t\tdiscarded: 10,\n\t\t\texecuted:  false,\n\t\t},\n\t\t{\n\t\t\tname:      \"slow\",\n\t\t\tthreshold: time.Duration(1),\n\t\t\tlastTime:  -1000,\n\t\t\tdiscarded: 10,\n\t\t\texecuted:  true,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttest := test\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tt.Parallel()\n\n\t\t\texecutor := newLimitedExecutor(0)\n\t\t\texecutor.threshold = test.threshold\n\t\t\texecutor.discarded = test.discarded\n\t\t\texecutor.lastTime.Set(test.lastTime)\n\n\t\t\tvar run int32\n\t\t\texecutor.logOrDiscard(func() {\n\t\t\t\tatomic.AddInt32(&run, 1)\n\t\t\t})\n\t\t\tif test.executed {\n\t\t\t\tassert.Equal(t, int32(1), atomic.LoadInt32(&run))\n\t\t\t} else {\n\t\t\t\tassert.Equal(t, int32(0), atomic.LoadInt32(&run))\n\t\t\t\tassert.Equal(t, test.discarded+1, atomic.LoadUint32(&executor.discarded))\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: core/logx/logs.go\npackage logx\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"os\"\n\t\"path\"\n\t\"runtime/debug\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/sysx\"\n)\n\nconst callerDepth = 4\n\nvar (\n\ttimeFormat        = \"2006-01-02T15:04:05.000Z07:00\"\n\tencoding   uint32 = jsonEncodingType\n\t// maxContentLength is used to truncate the log content, 0 for not truncating.\n\tmaxContentLength uint32\n\t// use uint32 for atomic operations\n\tdisableStat uint32\n\tlogLevel    uint32\n\toptions     logOptions\n\twriter      = new(atomicWriter)\n\tsetupOnce   sync.Once\n)\n\ntype (\n\t// LogField is a key-value pair that will be added to the log entry.\n\tLogField struct {\n\t\tKey   string\n\t\tValue any\n\t}\n\n\t// LogOption defines the method to customize the logging.\n\tLogOption func(options *logOptions)\n\n\tlogEntry map[string]any\n\n\tlogOptions struct {\n\t\tgzipEnabled           bool\n\t\tlogStackCooldownMills int\n\t\tkeepDays              int\n\t\tmaxBackups            int\n\t\tmaxSize               int\n\t\trotationRule          string\n\t}\n)\n\n// Alert alerts v in alert level, and the message is written to error log.\nfunc Alert(v string) {\n\tgetWriter().Alert(v)\n}\n\n// Close closes the logging.\nfunc Close() error {\n\tif w := writer.Swap(nil); w != nil {\n\t\treturn w.(io.Closer).Close()\n\t}\n\n\treturn nil\n}\n\n// Debug writes v into access log.\nfunc Debug(v ...any) {\n\tif shallLog(DebugLevel) {\n\t\twriteDebug(fmt.Sprint(v...))\n\t}\n}\n\n// Debugf writes v with format into access log.\nfunc Debugf(format string, v ...any) {\n\tif shallLog(DebugLevel) {\n\t\twriteDebug(fmt.Sprintf(format, v...))\n\t}\n}\n\n// Debugv writes v into access log with json content.\nfunc Debugv(v any) {\n\tif shallLog(DebugLevel) {\n\t\twriteDebug(v)\n\t}\n}\n\n// Debugw writes msg along with fields into the access log.\nfunc Debugw(msg string, fields ...LogField) {\n\tif shallLog(DebugLevel) {\n\t\twriteDebug(msg, fields...)\n\t}\n}\n\n// Disable disables the logging.\nfunc Disable() {\n\tatomic.StoreUint32(&logLevel, disableLevel)\n\twriter.Store(nopWriter{})\n}\n\n// DisableStat disables the stat logs.\nfunc DisableStat() {\n\tatomic.StoreUint32(&disableStat, 1)\n}\n\n// Error writes v into error log.\nfunc Error(v ...any) {\n\tif shallLog(ErrorLevel) {\n\t\twriteError(fmt.Sprint(v...))\n\t}\n}\n\n// Errorf writes v with format into error log.\nfunc Errorf(format string, v ...any) {\n\tif shallLog(ErrorLevel) {\n\t\twriteError(fmt.Errorf(format, v...).Error())\n\t}\n}\n\n// ErrorStack writes v along with call stack into error log.\nfunc ErrorStack(v ...any) {\n\tif shallLog(ErrorLevel) {\n\t\t// there is newline in stack string\n\t\twriteStack(fmt.Sprint(v...))\n\t}\n}\n\n// ErrorStackf writes v along with call stack in format into error log.\nfunc ErrorStackf(format string, v ...any) {\n\tif shallLog(ErrorLevel) {\n\t\t// there is newline in stack string\n\t\twriteStack(fmt.Sprintf(format, v...))\n\t}\n}\n\n// Errorv writes v into error log with json content.\n// No call stack attached, because not elegant to pack the messages.\nfunc Errorv(v any) {\n\tif shallLog(ErrorLevel) {\n\t\twriteError(v)\n\t}\n}\n\n// Errorw writes msg along with fields into the error log.\nfunc Errorw(msg string, fields ...LogField) {\n\tif shallLog(ErrorLevel) {\n\t\twriteError(msg, fields...)\n\t}\n}\n\n// Field returns a LogField for the given key and value.\nfunc Field(key string, value any) LogField {\n\tswitch val := value.(type) {\n\tcase error:\n\t\treturn LogField{Key: key, Value: val.Error()}\n\tcase []error:\n\t\tvar errs []string\n\t\tfor _, err := range val {\n\t\t\terrs = append(errs, err.Error())\n\t\t}\n\t\treturn LogField{Key: key, Value: errs}\n\tcase time.Duration:\n\t\treturn LogField{Key: key, Value: fmt.Sprint(val)}\n\tcase []time.Duration:\n\t\tvar durs []string\n\t\tfor _, dur := range val {\n\t\t\tdurs = append(durs, fmt.Sprint(dur))\n\t\t}\n\t\treturn LogField{Key: key, Value: durs}\n\tcase []time.Time:\n\t\tvar times []string\n\t\tfor _, t := range val {\n\t\t\ttimes = append(times, fmt.Sprint(t))\n\t\t}\n\t\treturn LogField{Key: key, Value: times}\n\tcase fmt.Stringer:\n\t\treturn LogField{Key: key, Value: val.String()}\n\tcase []fmt.Stringer:\n\t\tvar strs []string\n\t\tfor _, str := range val {\n\t\t\tstrs = append(strs, str.String())\n\t\t}\n\t\treturn LogField{Key: key, Value: strs}\n\tdefault:\n\t\treturn LogField{Key: key, Value: val}\n\t}\n}\n\n// Info writes v into access log.\nfunc Info(v ...any) {\n\tif shallLog(InfoLevel) {\n\t\twriteInfo(fmt.Sprint(v...))\n\t}\n}\n\n// Infof writes v with format into access log.\nfunc Infof(format string, v ...any) {\n\tif shallLog(InfoLevel) {\n\t\twriteInfo(fmt.Sprintf(format, v...))\n\t}\n}\n\n// Infov writes v into access log with json content.\nfunc Infov(v any) {\n\tif shallLog(InfoLevel) {\n\t\twriteInfo(v)\n\t}\n}\n\n// Infow writes msg along with fields into the access log.\nfunc Infow(msg string, fields ...LogField) {\n\tif shallLog(InfoLevel) {\n\t\twriteInfo(msg, fields...)\n\t}\n}\n\n// Must checks if err is nil, otherwise logs the error and exits.\nfunc Must(err error) {\n\tif err == nil {\n\t\treturn\n\t}\n\n\tmsg := fmt.Sprintf(\"%+v\\n\\n%s\", err.Error(), debug.Stack())\n\tlog.Print(msg)\n\tgetWriter().Severe(msg)\n\n\tif ExitOnFatal.True() {\n\t\tos.Exit(1)\n\t} else {\n\t\tpanic(msg)\n\t}\n}\n\n// MustSetup sets up logging with given config c. It exits on error.\nfunc MustSetup(c LogConf) {\n\tMust(SetUp(c))\n}\n\n// Reset clears the writer and resets the log level.\nfunc Reset() Writer {\n\treturn writer.Swap(nil)\n}\n\n// SetLevel sets the logging level. It can be used to suppress some logs.\nfunc SetLevel(level uint32) {\n\tatomic.StoreUint32(&logLevel, level)\n}\n\n// SetWriter sets the logging writer. It can be used to customize the logging.\nfunc SetWriter(w Writer) {\n\tif atomic.LoadUint32(&logLevel) != disableLevel {\n\t\twriter.Store(w)\n\t}\n}\n\n// SetUp sets up the logx.\n// If already set up, return nil.\n// We allow SetUp to be called multiple times, because, for example,\n// we need to allow different service frameworks to initialize logx respectively.\nfunc SetUp(c LogConf) (err error) {\n\t// Ignore the later SetUp calls.\n\t// Because multiple services in one process might call SetUp respectively.\n\t// Need to wait for the first caller to complete the execution.\n\tsetupOnce.Do(func() {\n\t\tsetupLogLevel(c)\n\n\t\tif !c.Stat {\n\t\t\tDisableStat()\n\t\t}\n\n\t\tif len(c.TimeFormat) > 0 {\n\t\t\ttimeFormat = c.TimeFormat\n\t\t}\n\n\t\tatomic.StoreUint32(&maxContentLength, c.MaxContentLength)\n\n\t\tswitch c.Encoding {\n\t\tcase plainEncoding:\n\t\t\tatomic.StoreUint32(&encoding, plainEncodingType)\n\t\tdefault:\n\t\t\tatomic.StoreUint32(&encoding, jsonEncodingType)\n\t\t}\n\n\t\tswitch c.Mode {\n\t\tcase fileMode:\n\t\t\terr = setupWithFiles(c)\n\t\tcase volumeMode:\n\t\t\terr = setupWithVolume(c)\n\t\tdefault:\n\t\t\tsetupWithConsole()\n\t\t}\n\t})\n\n\treturn\n}\n\n// Severe writes v into severe log.\nfunc Severe(v ...any) {\n\tif shallLog(SevereLevel) {\n\t\twriteSevere(fmt.Sprint(v...))\n\t}\n}\n\n// Severef writes v with format into severe log.\nfunc Severef(format string, v ...any) {\n\tif shallLog(SevereLevel) {\n\t\twriteSevere(fmt.Sprintf(format, v...))\n\t}\n}\n\n// Slow writes v into slow log.\nfunc Slow(v ...any) {\n\tif shallLog(ErrorLevel) {\n\t\twriteSlow(fmt.Sprint(v...))\n\t}\n}\n\n// Slowf writes v with format into slow log.\nfunc Slowf(format string, v ...any) {\n\tif shallLog(ErrorLevel) {\n\t\twriteSlow(fmt.Sprintf(format, v...))\n\t}\n}\n\n// Slowv writes v into slow log with json content.\nfunc Slowv(v any) {\n\tif shallLog(ErrorLevel) {\n\t\twriteSlow(v)\n\t}\n}\n\n// Sloww writes msg along with fields into slow log.\nfunc Sloww(msg string, fields ...LogField) {\n\tif shallLog(ErrorLevel) {\n\t\twriteSlow(msg, fields...)\n\t}\n}\n\n// Stat writes v into stat log.\nfunc Stat(v ...any) {\n\tif shallLogStat() && shallLog(InfoLevel) {\n\t\twriteStat(fmt.Sprint(v...))\n\t}\n}\n\n// Statf writes v with format into stat log.\nfunc Statf(format string, v ...any) {\n\tif shallLogStat() && shallLog(InfoLevel) {\n\t\twriteStat(fmt.Sprintf(format, v...))\n\t}\n}\n\n// WithCooldownMillis customizes logging on writing call stack interval.\nfunc WithCooldownMillis(millis int) LogOption {\n\treturn func(opts *logOptions) {\n\t\topts.logStackCooldownMills = millis\n\t}\n}\n\n// WithKeepDays customizes logging to keep logs with days.\nfunc WithKeepDays(days int) LogOption {\n\treturn func(opts *logOptions) {\n\t\topts.keepDays = days\n\t}\n}\n\n// WithGzip customizes logging to automatically gzip the log files.\nfunc WithGzip() LogOption {\n\treturn func(opts *logOptions) {\n\t\topts.gzipEnabled = true\n\t}\n}\n\n// WithMaxBackups customizes how many log files backups will be kept.\nfunc WithMaxBackups(count int) LogOption {\n\treturn func(opts *logOptions) {\n\t\topts.maxBackups = count\n\t}\n}\n\n// WithMaxSize customizes how much space the writing log file can take up.\nfunc WithMaxSize(size int) LogOption {\n\treturn func(opts *logOptions) {\n\t\topts.maxSize = size\n\t}\n}\n\n// WithRotation customizes which log rotation rule to use.\nfunc WithRotation(r string) LogOption {\n\treturn func(opts *logOptions) {\n\t\topts.rotationRule = r\n\t}\n}\n\nfunc addCaller(fields ...LogField) []LogField {\n\treturn append(fields, Field(callerKey, getCaller(callerDepth)))\n}\n\nfunc createOutput(path string) (io.WriteCloser, error) {\n\tif len(path) == 0 {\n\t\treturn nil, ErrLogPathNotSet\n\t}\n\n\tvar rule RotateRule\n\tswitch options.rotationRule {\n\tcase sizeRotationRule:\n\t\trule = NewSizeLimitRotateRule(path, backupFileDelimiter, options.keepDays, options.maxSize,\n\t\t\toptions.maxBackups, options.gzipEnabled)\n\tdefault:\n\t\trule = DefaultRotateRule(path, backupFileDelimiter, options.keepDays, options.gzipEnabled)\n\t}\n\n\treturn NewLogger(path, rule, options.gzipEnabled)\n}\n\nfunc getWriter() Writer {\n\tw := writer.Load()\n\tif w == nil {\n\t\tw = writer.StoreIfNil(newConsoleWriter())\n\t}\n\n\treturn w\n}\n\nfunc handleOptions(opts []LogOption) {\n\tfor _, opt := range opts {\n\t\topt(&options)\n\t}\n}\n\nfunc setupLogLevel(c LogConf) {\n\tswitch c.Level {\n\tcase levelDebug:\n\t\tSetLevel(DebugLevel)\n\tcase levelInfo:\n\t\tSetLevel(InfoLevel)\n\tcase levelError:\n\t\tSetLevel(ErrorLevel)\n\tcase levelSevere:\n\t\tSetLevel(SevereLevel)\n\t}\n}\n\nfunc setupWithConsole() {\n\tSetWriter(newConsoleWriter())\n}\n\nfunc setupWithFiles(c LogConf) error {\n\tw, err := newFileWriter(c)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tSetWriter(w)\n\treturn nil\n}\n\nfunc setupWithVolume(c LogConf) error {\n\tif len(c.ServiceName) == 0 {\n\t\treturn ErrLogServiceNameNotSet\n\t}\n\n\tc.Path = path.Join(c.Path, c.ServiceName, sysx.Hostname())\n\treturn setupWithFiles(c)\n}\n\nfunc shallLog(level uint32) bool {\n\treturn atomic.LoadUint32(&logLevel) <= level\n}\n\nfunc shallLogStat() bool {\n\treturn atomic.LoadUint32(&disableStat) == 0\n}\n\n// writeDebug writes v into debug log.\n// Not checking shallLog here is for performance consideration.\n// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.\n// The caller should check shallLog before calling this function.\nfunc writeDebug(val any, fields ...LogField) {\n\tgetWriter().Debug(val, addCaller(fields...)...)\n}\n\n// writeError writes v into the error log.\n// Not checking shallLog here is for performance consideration.\n// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.\n// The caller should check shallLog before calling this function.\nfunc writeError(val any, fields ...LogField) {\n\tgetWriter().Error(val, addCaller(fields...)...)\n}\n\n// writeInfo writes v into info log.\n// Not checking shallLog here is for performance consideration.\n// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.\n// The caller should check shallLog before calling this function.\nfunc writeInfo(val any, fields ...LogField) {\n\tgetWriter().Info(val, addCaller(fields...)...)\n}\n\n// writeSevere writes v into severe log.\n// Not checking shallLog here is for performance consideration.\n// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.\n// The caller should check shallLog before calling this function.\nfunc writeSevere(msg string) {\n\tgetWriter().Severe(fmt.Sprintf(\"%s\\n%s\", msg, string(debug.Stack())))\n}\n\n// writeSlow writes v into slow log.\n// Not checking shallLog here is for performance consideration.\n// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.\n// The caller should check shallLog before calling this function.\nfunc writeSlow(val any, fields ...LogField) {\n\tgetWriter().Slow(val, addCaller(fields...)...)\n}\n\n// writeStack writes v into stack log.\n// Not checking shallLog here is for performance consideration.\n// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.\n// The caller should check shallLog before calling this function.\nfunc writeStack(msg string) {\n\tgetWriter().Stack(fmt.Sprintf(\"%s\\n%s\", msg, string(debug.Stack())))\n}\n\n// writeStat writes v into the stat log.\n// Not checking shallLog here is for performance consideration.\n// If we check shallLog here, the fmt.Sprint might be called even if the log level is not enabled.\n// The caller should check shallLog before calling this function.\nfunc writeStat(msg string) {\n\tgetWriter().Stat(msg, addCaller()...)\n}\n\n// Path: core/logx/logs_test.go\npackage logx\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"os\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar (\n\ts           = []byte(\"Sending #11 notification (id: 1451875113812010473) in #1 connection\")\n\tpool        = make(chan []byte, 1)\n\t_    Writer = (*mockWriter)(nil)\n)\n\nfunc init() {\n\tExitOnFatal.Set(false)\n}\n\ntype mockWriter struct {\n\tlock    sync.Mutex\n\tbuilder strings.Builder\n}\n\nfunc (mw *mockWriter) Alert(v any) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelAlert, v)\n}\n\nfunc (mw *mockWriter) Debug(v any, fields ...LogField) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelDebug, v, fields...)\n}\n\nfunc (mw *mockWriter) Error(v any, fields ...LogField) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelError, v, fields...)\n}\n\nfunc (mw *mockWriter) Info(v any, fields ...LogField) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelInfo, v, fields...)\n}\n\nfunc (mw *mockWriter) Severe(v any) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelSevere, v)\n}\n\nfunc (mw *mockWriter) Slow(v any, fields ...LogField) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelSlow, v, fields...)\n}\n\nfunc (mw *mockWriter) Stack(v any) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelError, v)\n}\n\nfunc (mw *mockWriter) Stat(v any, fields ...LogField) {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\toutput(&mw.builder, levelStat, v, fields...)\n}\n\nfunc (mw *mockWriter) Close() error {\n\treturn nil\n}\n\nfunc (mw *mockWriter) Contains(text string) bool {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\treturn strings.Contains(mw.builder.String(), text)\n}\n\nfunc (mw *mockWriter) Reset() {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\tmw.builder.Reset()\n}\n\nfunc (mw *mockWriter) String() string {\n\tmw.lock.Lock()\n\tdefer mw.lock.Unlock()\n\treturn mw.builder.String()\n}\n\n\nfunc TestField(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tf    LogField\n\t\twant map[string]any\n\t}{\n\t\t{\n\t\t\tname: \"error\",\n\t\t\tf:    Field(\"foo\", errors.New(\"bar\")),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": \"bar\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"errors\",\n\t\t\tf:    Field(\"foo\", []error{errors.New(\"bar\"), errors.New(\"baz\")}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"bar\", \"baz\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"strings\",\n\t\t\tf:    Field(\"foo\", []string{\"bar\", \"baz\"}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"bar\", \"baz\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"duration\",\n\t\t\tf:    Field(\"foo\", time.Second),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": \"1s\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"durations\",\n\t\t\tf:    Field(\"foo\", []time.Duration{time.Second, 2 * time.Second}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"1s\", \"2s\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"times\",\n\t\t\tf: Field(\"foo\", []time.Time{\n\t\t\t\ttime.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC),\n\t\t\t\ttime.Date(2020, time.January, 2, 0, 0, 0, 0, time.UTC),\n\t\t\t}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"2020-01-01 00:00:00 +0000 UTC\", \"2020-01-02 00:00:00 +0000 UTC\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stringer\",\n\t\t\tf:    Field(\"foo\", ValStringer{val: \"bar\"}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": \"bar\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stringers\",\n\t\t\tf:    Field(\"foo\", []fmt.Stringer{ValStringer{val: \"bar\"}, ValStringer{val: \"baz\"}}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"bar\", \"baz\"},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttest := test\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tw := new(mockWriter)\n\t\t\told := writer.Swap(w)\n\t\t\tdefer writer.Store(old)\n\n\t\t\tInfow(\"foo\", test.f)\n\t\t\tvalidateFields(t, w.String(), test.want)\n\t\t})\n\t}\n}\n\nfunc TestFileLineFileMode(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tfile, line := getFileLine()\n\tError(\"anything\")\n\tassert.True(t, w.Contains(fmt.Sprintf(\"%s:%d\", file, line+1)))\n\n\tfile, line = getFileLine()\n\tErrorf(\"anything %s\", \"format\")\n\tassert.True(t, w.Contains(fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestFileLineConsoleMode(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tfile, line := getFileLine()\n\tError(\"anything\")\n\tassert.True(t, w.Contains(fmt.Sprintf(\"%s:%d\", file, line+1)))\n\n\tw.Reset()\n\tfile, line = getFileLine()\n\tErrorf(\"anything %s\", \"format\")\n\tassert.True(t, w.Contains(fmt.Sprintf(\"%s:%d\", file, line+1)))\n}\n\nfunc TestMust(t *testing.T) {\n\tassert.Panics(t, func() {\n\t\tMust(errors.New(\"foo\"))\n\t})\n}\n\nfunc TestStructedLogAlert(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelAlert, w, func(v ...any) {\n\t\tAlert(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogDebug(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelDebug, w, func(v ...any) {\n\t\tDebug(v...)\n\t})\n}\n\nfunc TestStructedLogDebugf(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelDebug, w, func(v ...any) {\n\t\tDebugf(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogDebugv(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelDebug, w, func(v ...any) {\n\t\tDebugv(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogDebugw(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelDebug, w, func(v ...any) {\n\t\tDebugw(fmt.Sprint(v...), Field(\"foo\", time.Second))\n\t})\n}\n\nfunc TestStructedLogError(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelError, w, func(v ...any) {\n\t\tError(v...)\n\t})\n}\n\nfunc TestStructedLogErrorf(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelError, w, func(v ...any) {\n\t\tErrorf(\"%s\", fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogErrorv(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelError, w, func(v ...any) {\n\t\tErrorv(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogErrorw(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelError, w, func(v ...any) {\n\t\tErrorw(fmt.Sprint(v...), Field(\"foo\", \"bar\"))\n\t})\n}\n\nfunc TestStructedLogInfo(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelInfo, w, func(v ...any) {\n\t\tInfo(v...)\n\t})\n}\n\nfunc TestStructedLogInfof(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelInfo, w, func(v ...any) {\n\t\tInfof(\"%s\", fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogInfov(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelInfo, w, func(v ...any) {\n\t\tInfov(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogInfow(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelInfo, w, func(v ...any) {\n\t\tInfow(fmt.Sprint(v...), Field(\"foo\", \"bar\"))\n\t})\n}\n\nfunc TestStructedLogInfoConsoleAny(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLogConsole(t, w, func(v ...any) {\n\t\told := atomic.LoadUint32(&encoding)\n\t\tatomic.StoreUint32(&encoding, plainEncodingType)\n\t\tdefer func() {\n\t\t\tatomic.StoreUint32(&encoding, old)\n\t\t}()\n\n\t\tInfov(v)\n\t})\n}\n\nfunc TestStructedLogInfoConsoleAnyString(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLogConsole(t, w, func(v ...any) {\n\t\told := atomic.LoadUint32(&encoding)\n\t\tatomic.StoreUint32(&encoding, plainEncodingType)\n\t\tdefer func() {\n\t\t\tatomic.StoreUint32(&encoding, old)\n\t\t}()\n\n\t\tInfov(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogInfoConsoleAnyError(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLogConsole(t, w, func(v ...any) {\n\t\told := atomic.LoadUint32(&encoding)\n\t\tatomic.StoreUint32(&encoding, plainEncodingType)\n\t\tdefer func() {\n\t\t\tatomic.StoreUint32(&encoding, old)\n\t\t}()\n\n\t\tInfov(errors.New(fmt.Sprint(v...)))\n\t})\n}\n\nfunc TestStructedLogInfoConsoleAnyStringer(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLogConsole(t, w, func(v ...any) {\n\t\told := atomic.LoadUint32(&encoding)\n\t\tatomic.StoreUint32(&encoding, plainEncodingType)\n\t\tdefer func() {\n\t\t\tatomic.StoreUint32(&encoding, old)\n\t\t}()\n\n\t\tInfov(ValStringer{\n\t\t\tval: fmt.Sprint(v...),\n\t\t})\n\t})\n}\n\nfunc TestStructedLogInfoConsoleText(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLogConsole(t, w, func(v ...any) {\n\t\told := atomic.LoadUint32(&encoding)\n\t\tatomic.StoreUint32(&encoding, plainEncodingType)\n\t\tdefer func() {\n\t\t\tatomic.StoreUint32(&encoding, old)\n\t\t}()\n\n\t\tInfo(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogSlow(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelSlow, w, func(v ...any) {\n\t\tSlow(v...)\n\t})\n}\n\nfunc TestStructedLogSlowf(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelSlow, w, func(v ...any) {\n\t\tSlowf(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogSlowv(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelSlow, w, func(v ...any) {\n\t\tSlowv(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogSloww(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelSlow, w, func(v ...any) {\n\t\tSloww(fmt.Sprint(v...), Field(\"foo\", time.Second))\n\t})\n}\n\nfunc TestStructedLogStat(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelStat, w, func(v ...any) {\n\t\tStat(v...)\n\t})\n}\n\nfunc TestStructedLogStatf(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelStat, w, func(v ...any) {\n\t\tStatf(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogSevere(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelSevere, w, func(v ...any) {\n\t\tSevere(v...)\n\t})\n}\n\nfunc TestStructedLogSeveref(t *testing.T) {\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tdoTestStructedLog(t, levelSevere, w, func(v ...any) {\n\t\tSeveref(fmt.Sprint(v...))\n\t})\n}\n\nfunc TestStructedLogWithDuration(t *testing.T) {\n\tconst message = \"hello there\"\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tWithDuration(time.Second).Info(message)\n\tvar entry map[string]any\n\tif err := json.Unmarshal([]byte(w.String()), &entry); err != nil {\n\t\tt.Error(err)\n\t}\n\tassert.Equal(t, levelInfo, entry[levelKey])\n\tassert.Equal(t, message, entry[contentKey])\n\tassert.Equal(t, \"1000.0ms\", entry[durationKey])\n}\n\nfunc TestSetLevel(t *testing.T) {\n\tSetLevel(ErrorLevel)\n\tconst message = \"hello there\"\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n\tdefer writer.Store(old)\n\n\tInfo(message)\n\tassert.Equal(t, 0, w.builder.Len())\n}\n\nfunc TestSetLevelTwiceWithMode(t *testing.T) {\n\ttestModes := []string{\n\t\t\"console\",\n\t\t\"volumn\",\n\t\t\"mode\",\n\t}\n\tw := new(mockWriter)\n\told := writer.Swap(w)\n...\n", "needle_token_start": 10973, "needle_token_end": 11843, "code_context_ntokens": 15204, "output": ["```go\nfunc TestField(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tf    LogField\n\t\twant map[string]any\n\t}{\n\t\t{\n\t\t\tname: \"error\",\n\t\t\tf:    Field(\"foo\", errors.New(\"bar\")),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": \"bar\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"errors\",\n\t\t\tf:    Field(\"foo\", []error{errors.New(\"bar\"), errors.New(\"baz\")}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"bar\", \"baz\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"strings\",\n\t\t\tf:    Field(\"foo\", []string{\"bar\", \"baz\"}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"bar\", \"baz\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"duration\",\n\t\t\tf:    Field(\"foo\", time.Second),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": \"1s\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"durations\",\n\t\t\tf:    Field(\"foo\", []time.Duration{time.Second, 2 * time.Second}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"1s\", \"2s\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"times\",\n\t\t\tf: Field(\"foo\", []time.Time{\n\t\t\t\ttime.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC),\n\t\t\t\ttime.Date(2020, time.January, 2, 0, 0, 0, 0, time.UTC),\n\t\t\t}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"2020-01-01 00:00:00 +0000 UTC\", \"2020-01-02 00:00:00 +0000 UTC\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stringer\",\n\t\t\tf:    Field(\"foo\", ValStringer{val: \"bar\"}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": \"bar\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stringers\",\n\t\t\tf:    Field(\"foo\", []fmt.Stringer{ValStringer{val: \"bar\"}, ValStringer{val: \"baz\"}}),\n\t\t\twant: map[string]any{\n\t\t\t\t\"foo\": []any{\"bar\", \"baz\"},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttest := test\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\tw := new(mockWriter)\n\t\t\told := writer.Swap(w)\n\t\t\tdefer writer.Store(old)\n\n\t\t\tInfow(\"foo\", test.f)\n\t\t\tvalidateFields(t, w.String(), test.want)\n\t\t})\n\t}\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "TestMapValuerWithInherit_Value", "language": "go", "path": "core/mapping/valuer_test.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To verify that a value retriever correctly inherits and merges values from both the current context and its parent context when the requested key is present in both.\n2. **Input**: A nested map structure representing settings for a component and its discovery service, with the component nested within the main settings.\n3. **Output**: The test checks that the correct merged map is returned for a given key, ensuring that values from the parent context are correctly inherited and merged into the current context's values.\n4. **Procedure**: The test initializes a nested map structure and sets up a value retriever with a parent-child relationship. It then retrieves a value using a key that exists in both the parent and child contexts. The test verifies that the retrieved value is a merged map containing values from both contexts, and asserts the correctness of specific values within the map.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/mapping/unmarshaler_test.go\npackage mapping\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\t\"unicode\"\n\n\t\"github.com/google/uuid\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/stringx\"\n)\n\n// because json.Number doesn't support strconv.ParseUint(...),\n// so we only can test to 62 bits.\nconst maxUintBitsToTest = 62\n\nfunc TestUnmarshalWithFullNameNotStruct(t *testing.T) {\n\tvar s map[string]any\n\tcontent := []byte(`{\"name\":\"xiaoming\"}`)\n\terr := UnmarshalJsonBytes(content, &s)\n\tassert.Equal(t, errTypeMismatch, err)\n}\n\nfunc TestUnmarshalValueNotSettable(t *testing.T) {\n\tvar s map[string]any\n\tcontent := []byte(`{\"name\":\"xiaoming\"}`)\n\terr := UnmarshalJsonBytes(content, s)\n\tassert.Equal(t, errValueNotSettable, err)\n}\n\nfunc TestUnmarshalWithoutTagName(t *testing.T) {\n\ttype inner struct {\n\t\tOptional   bool   `key:\",optional\"`\n\t\tOptionalP  *bool  `key:\",optional\"`\n\t\tOptionalPP **bool `key:\",optional\"`\n\t}\n\tm := map[string]any{\n\t\t\"Optional\":   true,\n\t\t\"OptionalP\":  true,\n\t\t\"OptionalPP\": true,\n\t}\n\n\tvar in inner\n\tif assert.NoError(t, UnmarshalKey(m, &in)) {\n\t\tassert.True(t, in.Optional)\n\t\tassert.True(t, *in.OptionalP)\n\t\tassert.True(t, **in.OptionalPP)\n\t}\n}\n\nfunc TestUnmarshalWithLowerField(t *testing.T) {\n\ttype (\n\t\tLower struct {\n\t\t\tvalue int `key:\"lower\"`\n\t\t}\n\n\t\tinner struct {\n\t\t\tLower\n\t\t\tOptional bool `key:\",optional\"`\n\t\t}\n\t)\n\tm := map[string]any{\n\t\t\"Optional\": true,\n\t\t\"lower\":    1,\n\t}\n\n\tvar in inner\n\tif assert.NoError(t, UnmarshalKey(m, &in)) {\n\t\tassert.True(t, in.Optional)\n\t\tassert.Equal(t, 0, in.value)\n\t}\n}\n\nfunc TestUnmarshalWithLowerAnonymousStruct(t *testing.T) {\n\ttype (\n\t\tlower struct {\n\t\t\tValue int `key:\"lower\"`\n\t\t}\n\n\t\tinner struct {\n\t\t\tlower\n\t\t\tOptional bool `key:\",optional\"`\n\t\t}\n\t)\n\tm := map[string]any{\n\t\t\"Optional\": true,\n\t\t\"lower\":    1,\n\t}\n\n\tvar in inner\n\tif assert.NoError(t, UnmarshalKey(m, &in)) {\n\t\tassert.True(t, in.Optional)\n\t\tassert.Equal(t, 1, in.Value)\n\t}\n}\n\nfunc TestUnmarshalWithoutTagNameWithCanonicalKey(t *testing.T) {\n\ttype inner struct {\n\t\tName string `key:\"name\"`\n\t}\n\tm := map[string]any{\n\t\t\"Name\": \"go-zero\",\n\t}\n\n\tvar in inner\n\tunmarshaler := NewUnmarshaler(defaultKeyName, WithCanonicalKeyFunc(func(s string) string {\n\t\tfirst := true\n\t\treturn strings.Map(func(r rune) rune {\n\t\t\tif first {\n\t\t\t\tfirst = false\n\t\t\t\treturn unicode.ToTitle(r)\n\t\t\t}\n\t\t\treturn r\n\t\t}, s)\n\t}))\n\tif assert.NoError(t, unmarshaler.Unmarshal(m, &in)) {\n\t\tassert.Equal(t, \"go-zero\", in.Name)\n\t}\n}\n\nfunc TestUnmarshalWithoutTagNameWithCanonicalKeyOptionalDep(t *testing.T) {\n\ttype inner struct {\n\t\tFirstName string `key:\",optional\"`\n\t\tLastName  string `key:\",optional=FirstName\"`\n\t}\n\tm := map[string]any{\n\t\t\"firstname\": \"go\",\n\t\t\"lastname\":  \"zero\",\n\t}\n\n\tvar in inner\n\tunmarshaler := NewUnmarshaler(defaultKeyName, WithCanonicalKeyFunc(func(s string) string {\n\t\treturn strings.ToLower(s)\n\t}))\n\tif assert.NoError(t, unmarshaler.Unmarshal(m, &in)) {\n\t\tassert.Equal(t, \"go\", in.FirstName)\n\t\tassert.Equal(t, \"zero\", in.LastName)\n\t}\n}\n\nfunc TestUnmarshalBool(t *testing.T) {\n\ttype inner struct {\n\t\tTrue           bool `key:\"yes\"`\n\t\tFalse          bool `key:\"no\"`\n\t\tTrueFromOne    bool `key:\"yesone,string\"`\n\t\tFalseFromZero  bool `key:\"nozero,string\"`\n\t\tTrueFromTrue   bool `key:\"yestrue,string\"`\n\t\tFalseFromFalse bool `key:\"nofalse,string\"`\n\t\tDefaultTrue    bool `key:\"defaulttrue,default=1\"`\n\t\tOptional       bool `key:\"optional,optional\"`\n\t}\n\tm := map[string]any{\n\t\t\"yes\":     true,\n\t\t\"no\":      false,\n\t\t\"yesone\":  \"1\",\n\t\t\"nozero\":  \"0\",\n\t\t\"yestrue\": \"true\",\n\t\t\"nofalse\": \"false\",\n\t}\n\n\tvar in inner\n\tast := assert.New(t)\n\tif ast.NoError(UnmarshalKey(m, &in)) {\n\t\tast.True(in.True)\n\t\tast.False(in.False)\n\t\tast.True(in.TrueFromOne)\n\t\tast.False(in.FalseFromZero)\n\t\tast.True(in.TrueFromTrue)\n\t\tast.False(in.FalseFromFalse)\n\t\tast.True(in.DefaultTrue)\n\t}\n}\n\nfunc TestUnmarshalDuration(t *testing.T) {\n\ttype inner struct {\n\t\tDuration       time.Duration   `key:\"duration\"`\n\t\tLessDuration   time.Duration   `key:\"less\"`\n\t\tMoreDuration   time.Duration   `key:\"more\"`\n\t\tPtrDuration    *time.Duration  `key:\"ptr\"`\n\t\tPtrPtrDuration **time.Duration `key:\"ptrptr\"`\n\t}\n\tm := map[string]any{\n\t\t\"duration\": \"5s\",\n\t\t\"less\":     \"100ms\",\n\t\t\"more\":     \"24h\",\n\t\t\"ptr\":      \"1h\",\n\t\t\"ptrptr\":   \"2h\",\n\t}\n\tvar in inner\n\tif assert.NoError(t, UnmarshalKey(m, &in)) {\n\t\tassert.Equal(t, time.Second*5, in.Duration)\n\t\tassert.Equal(t, time.Millisecond*100, in.LessDuration)\n\t\tassert.Equal(t, time.Hour*24, in.MoreDuration)\n\t\tassert.Equal(t, time.Hour, *in.PtrDuration)\n\t\tassert.Equal(t, time.Hour*2, **in.PtrPtrDuration)\n\t}\n}\n\nfunc TestUnmarshalDurationDefault(t *testing.T) {\n\ttype inner struct {\n\t\tInt      int           `key:\"int\"`\n\t\tDuration time.Duration `key:\"duration,default=5s\"`\n\t}\n\tm := map[string]any{\n\t\t\"int\": 5,\n\t}\n\tvar in inner\n\tif assert.NoError(t, UnmarshalKey(m, &in)) {\n\t\tassert.Equal(t, 5, in.Int)\n\t\tassert.Equal(t, time.Second*5, in.Duration)\n\t}\n}\n\nfunc TestUnmarshalDurationPtr(t *testing.T) {\n\ttype inner struct {\n\t\tDuration *time.Duration `key:\"duration\"`\n\t}\n\tm := map[string]any{\n\t\t\"duration\": \"5s\",\n\t}\n\tvar in inner\n\tif assert.NoError(t, UnmarshalKey(m, &in)) {\n\t\tassert.Equal(t, time.Second*5, *in.Duration)\n\t}\n}\n\nfunc TestUnmarshalDurationPtrDefault(t *testing.T) {\n\ttype inner struct {\n\t\tInt      int            `key:\"int\"`\n\t\tValue    *int           `key:\",default=5\"`\n\t\tDuration *time.Duration `key:\"duration,default=5s\"`\n\t}\n\tm := map[string]any{\n\t\t\"int\": 5,\n\t}\n\tvar in inner\n\tif assert.NoError(t, UnmarshalKey(m, &in)) {\n\t\tassert.Equal(t, 5, in.Int)\n\t\tassert.Equal(t, 5, *in.Value)\n\t\tassert.Equal(t, time.Second*5, *in.Duration)\n\t}\n}\n\nfunc TestUnmarshalInt(t *testing.T) {\n\ttype inner struct {\n\t\tInt          int   `key:\"int\"`\n\t\tIntFromStr   int   `key:\"intstr,string\"`\n\t\tInt8         int8  `key:\"int8\"`\n\t\tInt8FromStr  int8  `key:\"int8str,string\"`\n\t\tInt16        int16 `key:\"int16\"`\n\t\tInt16FromStr int16 `key:\"int16str,string\"`\n\t\tInt32        int32 `key:\"int32\"`\n\t\tInt32FromStr int32 `key:\"int32str,string\"`\n\t\tInt64        int64 `key:\"int64\"`\n\t\tInt64FromStr int64 `key:\"int64str,string\"`\n\t\tDefaultInt   int64 `key:\"defaultint,default=11\"`\n\t\tOptional     int   `key:\"optional,optional\"`\n\t}\n\tm := map[string]any{\n\t\t\"int\":      1,\n\t\t\"intstr\":   \"2\",\n\t\t\"int8\":     int8(3),\n\t\t\"int8str\":  \"4\",\n\t\t\"int16\":    int16(5),\n\t\t\"int16str\": \"6\",\n\t\t\"int32\":    int32(7),\n\t\t\"int32str\": \"8\",\n\t\t\"int64\":    int64(9),\n\t\t\"int64str\": \"10\",\n\t}\n\n\tvar in inner\n\tast := assert.New(t)\n\tif ast.NoError(UnmarshalKey(m, &in)) {\n\t\tast.Equal(1, in.Int)\n\t\tast.Equal(2, in.IntFromStr)\n\t\tast.Equal(int8(3), in.Int8)\n\t\tast.Equal(int8(4), in.Int8FromStr)\n\t\tast.Equal(int16(5), in.Int16)\n\t\tast.Equal(int16(6), in.Int16FromStr)\n\t\tast.Equal(int32(7), in.Int32)\n\t\tast.Equal(int32(8), in.Int32FromStr)\n\t\tast.Equal(int64(9), in.Int64)\n...\n// Path: core/mapping/utils.go\npackage mapping\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/zeromicro/go-zero/core/lang\"\n\t\"github.com/zeromicro/go-zero/core/stringx\"\n)\n\nconst (\n\tdefaultOption      = \"default\"\n\tenvOption          = \"env\"\n\tinheritOption      = \"inherit\"\n\tstringOption       = \"string\"\n\toptionalOption     = \"optional\"\n\toptionsOption      = \"options\"\n\trangeOption        = \"range\"\n\toptionSeparator    = \"|\"\n\tequalToken         = \"=\"\n\tescapeChar         = '\\\\'\n\tleftBracket        = '('\n\trightBracket       = ')'\n\tleftSquareBracket  = '['\n\trightSquareBracket = ']'\n\tsegmentSeparator   = ','\n\tintSize            = 32 << (^uint(0) >> 63) // 32 or 64\n)\n\nvar (\n\terrUnsupportedType  = errors.New(\"unsupported type on setting field value\")\n\terrNumberRange      = errors.New(\"wrong number range setting\")\n\terrNilSliceElement  = errors.New(\"null element for slice\")\n\toptionsCache        = make(map[string]optionsCacheValue)\n\tcacheLock           sync.RWMutex\n\tstructRequiredCache = make(map[reflect.Type]requiredCacheValue)\n\tstructCacheLock     sync.RWMutex\n)\n\ntype (\n\toptionsCacheValue struct {\n\t\tkey     string\n\t\toptions *fieldOptions\n\t\terr     error\n\t}\n\n\trequiredCacheValue struct {\n\t\trequired bool\n\t\terr      error\n\t}\n)\n\n// Deref dereferences a type, if pointer type, returns its element type.\nfunc Deref(t reflect.Type) reflect.Type {\n\tfor t.Kind() == reflect.Ptr {\n\t\tt = t.Elem()\n\t}\n\n\treturn t\n}\n\n// Repr returns the string representation of v.\nfunc Repr(v any) string {\n\treturn lang.Repr(v)\n}\n\n// SetValue sets target to value, pointers are processed automatically.\nfunc SetValue(tp reflect.Type, value, target reflect.Value) {\n\tvalue.Set(convertTypeOfPtr(tp, target))\n}\n\n// SetMapIndexValue sets target to value at key position, pointers are processed automatically.\nfunc SetMapIndexValue(tp reflect.Type, value, key, target reflect.Value) {\n\tvalue.SetMapIndex(key, convertTypeOfPtr(tp, target))\n}\n\n// ValidatePtr validates v if it's a valid pointer.\nfunc ValidatePtr(v reflect.Value) error {\n\t// sequence is very important, IsNil must be called after checking Kind() with reflect.Ptr,\n\t// panic otherwise\n\tif !v.IsValid() || v.Kind() != reflect.Ptr || v.IsNil() {\n\t\treturn fmt.Errorf(\"not a valid pointer: %v\", v)\n\t}\n\n\treturn nil\n}\n\nfunc convertTypeFromString(kind reflect.Kind, str string) (any, error) {\n\tswitch kind {\n\tcase reflect.Bool:\n\t\tswitch strings.ToLower(str) {\n\t\tcase \"1\", \"true\":\n\t\t\treturn true, nil\n\t\tcase \"0\", \"false\":\n\t\t\treturn false, nil\n\t\tdefault:\n\t\t\treturn false, errTypeMismatch\n\t\t}\n\tcase reflect.Int:\n\t\treturn strconv.ParseInt(str, 10, intSize)\n\tcase reflect.Int8:\n\t\treturn strconv.ParseInt(str, 10, 8)\n\tcase reflect.Int16:\n\t\treturn strconv.ParseInt(str, 10, 16)\n\tcase reflect.Int32:\n\t\treturn strconv.ParseInt(str, 10, 32)\n\tcase reflect.Int64:\n\t\treturn strconv.ParseInt(str, 10, 64)\n\tcase reflect.Uint:\n\t\treturn strconv.ParseUint(str, 10, intSize)\n\tcase reflect.Uint8:\n\t\treturn strconv.ParseUint(str, 10, 8)\n\tcase reflect.Uint16:\n\t\treturn strconv.ParseUint(str, 10, 16)\n\tcase reflect.Uint32:\n\t\treturn strconv.ParseUint(str, 10, 32)\n\tcase reflect.Uint64:\n\t\treturn strconv.ParseUint(str, 10, 64)\n\tcase reflect.Float32:\n\t\treturn strconv.ParseFloat(str, 32)\n\tcase reflect.Float64:\n\t\treturn strconv.ParseFloat(str, 64)\n\tcase reflect.String:\n\t\treturn str, nil\n\tdefault:\n\t\treturn nil, errUnsupportedType\n\t}\n}\n\nfunc convertTypeOfPtr(tp reflect.Type, target reflect.Value) reflect.Value {\n\t// keep the original value is a pointer\n\tif tp.Kind() == reflect.Ptr && target.CanAddr() {\n\t\ttp = tp.Elem()\n\t\ttarget = target.Addr()\n\t}\n\n\tfor tp.Kind() == reflect.Ptr {\n\t\tp := reflect.New(target.Type())\n\t\tp.Elem().Set(target)\n\t\ttarget = p\n\t\ttp = tp.Elem()\n\t}\n\n\treturn target\n}\n\nfunc doParseKeyAndOptions(field reflect.StructField, value string) (string, *fieldOptions, error) {\n\tsegments := parseSegments(value)\n\tkey := strings.TrimSpace(segments[0])\n\toptions := segments[1:]\n\n\tif len(options) == 0 {\n\t\treturn key, nil, nil\n\t}\n\n\tvar fieldOpts fieldOptions\n\tfor _, segment := range options {\n\t\toption := strings.TrimSpace(segment)\n\t\tif err := parseOption(&fieldOpts, field.Name, option); err != nil {\n\t\t\treturn \"\", nil, err\n\t\t}\n\t}\n\n\treturn key, &fieldOpts, nil\n}\n\n// ensureValue ensures nested members not to be nil.\n// If pointer value is nil, set to a new value.\nfunc ensureValue(v reflect.Value) reflect.Value {\n\tfor {\n\t\tif v.Kind() != reflect.Ptr {\n\t\t\tbreak\n\t\t}\n\n\t\tif v.IsNil() {\n\t\t\tv.Set(reflect.New(v.Type().Elem()))\n\t\t}\n\t\tv = v.Elem()\n\t}\n\n\treturn v\n}\n\nfunc implicitValueRequiredStruct(tag string, tp reflect.Type) (bool, error) {\n\tnumFields := tp.NumField()\n\tfor i := 0; i < numFields; i++ {\n\t\tchildField := tp.Field(i)\n\t\tif usingDifferentKeys(tag, childField) {\n\t\t\treturn true, nil\n\t\t}\n\n\t\t_, opts, err := parseKeyAndOptions(tag, childField)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\tif opts == nil {\n\t\t\tif childField.Type.Kind() != reflect.Struct {\n\t\t\t\treturn true, nil\n\t\t\t}\n\n\t\t\tif required, err := implicitValueRequiredStruct(tag, childField.Type); err != nil {\n\t\t\t\treturn false, err\n\t\t\t} else if required {\n\t\t\t\treturn true, nil\n\t\t\t}\n\t\t} else if !opts.Optional && len(opts.Default) == 0 {\n\t\t\treturn true, nil\n\t\t} else if len(opts.OptionalDep) > 0 && opts.OptionalDep[0] == notSymbol {\n\t\t\treturn true, nil\n\t\t}\n\t}\n\n\treturn false, nil\n}\n\nfunc isLeftInclude(b byte) (bool, error) {\n\tswitch b {\n\tcase '[':\n\t\treturn true, nil\n\tcase '(':\n\t\treturn false, nil\n\tdefault:\n\t\treturn false, errNumberRange\n\t}\n}\n\nfunc isRightInclude(b byte) (bool, error) {\n\tswitch b {\n\tcase ']':\n\t\treturn true, nil\n\tcase ')':\n\t\treturn false, nil\n\tdefault:\n\t\treturn false, errNumberRange\n\t}\n}\n\nfunc maybeNewValue(fieldType reflect.Type, value reflect.Value) {\n\tif fieldType.Kind() == reflect.Ptr && value.IsNil() {\n\t\tvalue.Set(reflect.New(value.Type().Elem()))\n\t}\n}\n\nfunc parseGroupedSegments(val string) []string {\n\tval = strings.TrimLeftFunc(val, func(r rune) bool {\n\t\treturn r == leftBracket || r == leftSquareBracket\n\t})\n\tval = strings.TrimRightFunc(val, func(r rune) bool {\n\t\treturn r == rightBracket || r == rightSquareBracket\n\t})\n\treturn parseSegments(val)\n}\n\n// don't modify returned fieldOptions, it's cached and shared among different calls.\nfunc parseKeyAndOptions(tagName string, field reflect.StructField) (string, *fieldOptions, error) {\n\tvalue := strings.TrimSpace(field.Tag.Get(tagName))\n\tif len(value) == 0 {\n\t\treturn field.Name, nil, nil\n\t}\n\n\tcacheLock.RLock()\n\tcache, ok := optionsCache[value]\n\tcacheLock.RUnlock()\n\tif ok {\n\t\treturn stringx.TakeOne(cache.key, field.Name), cache.options, cache.err\n\t}\n\n\tkey, options, err := doParseKeyAndOptions(field, value)\n\tcacheLock.Lock()\n\toptionsCache[value] = optionsCacheValue{\n\t\tkey:     key,\n\t\toptions: options,\n\t\terr:     err,\n\t}\n\tcacheLock.Unlock()\n\n\treturn stringx.TakeOne(key, field.Name), options, err\n}\n\n// support below notations:\n// [:5] (:5] [:5) (:5)\n// [1:] [1:) (1:] (1:)\n// [1:5] [1:5) (1:5] (1:5)\nfunc parseNumberRange(str string) (*numberRange, error) {\n\tif len(str) == 0 {\n\t\treturn nil, errNumberRange\n\t}\n\n\tleftInclude, err := isLeftInclude(str[0])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tstr = str[1:]\n\tif len(str) == 0 {\n\t\treturn nil, errNumberRange\n\t}\n\n\trightInclude, err := isRightInclude(str[len(str)-1])\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tstr = str[:len(str)-1]\n\tfields := strings.Split(str, \":\")\n\tif len(fields) != 2 {\n\t\treturn nil, errNumberRange\n\t}\n\n\tif len(fields[0]) == 0 && len(fields[1]) == 0 {\n\t\treturn nil, errNumberRange\n\t}\n\n\tvar left float64\n\tif len(fields[0]) > 0 {\n\t\tvar err error\n\t\tif left, err = strconv.ParseFloat(fields[0], 64); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tleft = -math.MaxFloat64\n\t}\n\n\tvar right float64\n\tif len(fields[1]) > 0 {\n\t\tvar err error\n\t\tif right, err = strconv.ParseFloat(fields[1], 64); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tright = math.MaxFloat64\n\t}\n\n\tif left > right {\n\t\treturn nil, errNumberRange\n\t}\n\n\t// [2:2] valid\n\t// [2:2) invalid\n\t// (2:2] invalid\n\t// (2:2) invalid\n\tif left == right {\n\t\tif !leftInclude || !rightInclude {\n\t\t\treturn nil, errNumberRange\n\t\t}\n\t}\n\n\treturn &numberRange{\n\t\tleft:         left,\n\t\tleftInclude:  leftInclude,\n\t\tright:        right,\n\t\trightInclude: rightInclude,\n\t}, nil\n}\n\nfunc parseOption(fieldOpts *fieldOptions, fieldName, option string) error {\n\tswitch {\n\tcase option == inheritOption:\n\t\tfieldOpts.Inherit = true\n\tcase option == stringOption:\n\t\tfieldOpts.FromString = true\n\tcase strings.HasPrefix(option, optionalOption):\n\t\tsegs := strings.Split(option, equalToken)\n\t\tswitch len(segs) {\n\t\tcase 1:\n\t\t\tfieldOpts.Optional = true\n\t\tcase 2:\n\t\t\tfieldOpts.Optional = true\n\t\t\tfieldOpts.OptionalDep = segs[1]\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"field %q has wrong optional\", fieldName)\n\t\t}\n\tcase strings.HasPrefix(option, optionsOption):\n\t\tval, err := parseProperty(fieldName, optionsOption, option)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfieldOpts.Options = parseOptions(val)\n\tcase strings.HasPrefix(option, defaultOption):\n\t\tval, err := parseProperty(fieldName, defaultOption, option)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfieldOpts.Default = val\n\tcase strings.HasPrefix(option, envOption):\n\t\tval, err := parseProperty(fieldName, envOption, option)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfieldOpts.EnvVar = val\n\tcase strings.HasPrefix(option, rangeOption):\n\t\tval, err := parseProperty(fieldName, rangeOption, option)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tnr, err := parseNumberRange(val)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfieldOpts.Range = nr\n\t}\n\n\treturn nil\n}\n\n// parseOptions parses the given options in tag.\n// for example, `json:\"name,options=foo|bar\"` or `json:\"name,options=[foo,bar]\"`\nfunc parseOptions(val string) []string {\n\tif len(val) == 0 {\n\t\treturn nil\n\t}\n\n\tif val[0] == leftSquareBracket {\n\t\treturn parseGroupedSegments(val)\n\t}\n\n\treturn strings.Split(val, optionSeparator)\n}\n\nfunc parseProperty(field, tag, val string) (string, error) {\n\tsegs := strings.Split(val, equalToken)\n\tif len(segs) != 2 {\n\t\treturn \"\", fmt.Errorf(\"field %q has wrong tag value %q\", field, tag)\n\t}\n\n\treturn strings.TrimSpace(segs[1]), nil\n}\n\nfunc parseSegments(val string) []string {\n\tvar segments []string\n\tvar escaped, grouped bool\n\tvar buf strings.Builder\n\n\tfor _, ch := range val {\n\t\tif escaped {\n\t\t\tbuf.WriteRune(ch)\n\t\t\tescaped = false\n\t\t\tcontinue\n\t\t}\n\n\t\tswitch ch {\n\t\tcase segmentSeparator:\n\t\t\tif grouped {\n\t\t\t\tbuf.WriteRune(ch)\n\t\t\t} else {\n\t\t\t\t// need to trim spaces, but we cannot ignore empty string,\n\t\t\t\t// because the first segment stands for the key might be empty.\n\t\t\t\t// if ignored, the later tag will be used as the key.\n\t\t\t\tsegments = append(segments, strings.TrimSpace(buf.String()))\n\t\t\t\tbuf.Reset()\n\t\t\t}\n\t\tcase escapeChar:\n\t\t\tif grouped {\n\t\t\t\tbuf.WriteRune(ch)\n\t\t\t} else {\n\t\t\t\tescaped = true\n\t\t\t}\n\t\tcase leftBracket, leftSquareBracket:\n\t\t\tbuf.WriteRune(ch)\n\t\t\tgrouped = true\n\t\tcase rightBracket, rightSquareBracket:\n\t\t\tbuf.WriteRune(ch)\n\t\t\tgrouped = false\n\t\tdefault:\n\t\t\tbuf.WriteRune(ch)\n\t\t}\n\t}\n\n\tlast := strings.TrimSpace(buf.String())\n\t// ignore last empty string\n\tif len(last) > 0 {\n\t\tsegments = append(segments, last)\n\t}\n\n\treturn segments\n}\n\nfunc setMatchedPrimitiveValue(kind reflect.Kind, value reflect.Value, v any) error {\n\tswitch kind {\n\tcase reflect.Bool:\n\t\tvalue.SetBool(v.(bool))\n\t\treturn nil\n\tcase reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:\n\t\tvalue.SetInt(v.(int64))\n\t\treturn nil\n\tcase reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:\n\t\tvalue.SetUint(v.(uint64))\n\t\treturn nil\n\tcase reflect.Float32, reflect.Float64:\n\t\tvalue.SetFloat(v.(float64))\n\t\treturn nil\n\tcase reflect.String:\n\t\tvalue.SetString(v.(string))\n\t\treturn nil\n\tdefault:\n\t\treturn errUnsupportedType\n\t}\n}\n\nfunc setValueFromString(kind reflect.Kind, value reflect.Value, str string) error {\n\tif !value.CanSet() {\n\t\treturn errValueNotSettable\n\t}\n\n\tvalue = ensureValue(value)\n\tv, err := convertTypeFromString(kind, str)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn setMatchedPrimitiveValue(kind, value, v)\n}\n\nfunc structValueRequired(tag string, tp reflect.Type) (bool, error) {\n\tstructCacheLock.RLock()\n\tval, ok := structRequiredCache[tp]\n\tstructCacheLock.RUnlock()\n\tif ok {\n\t\treturn val.required, val.err\n\t}\n\n\trequired, err := implicitValueRequiredStruct(tag, tp)\n\tstructCacheLock.Lock()\n\tstructRequiredCache[tp] = requiredCacheValue{\n\t\trequired: required,\n\t\terr:      err,\n\t}\n\tstructCacheLock.Unlock()\n\n\treturn required, err\n}\n\nfunc toFloat64(v any) (float64, bool) {\n\tswitch val := v.(type) {\n\tcase int:\n\t\treturn float64(val), true\n\tcase int8:\n\t\treturn float64(val), true\n\tcase int16:\n\t\treturn float64(val), true\n\tcase int32:\n\t\treturn float64(val), true\n\tcase int64:\n\t\treturn float64(val), true\n\tcase uint:\n\t\treturn float64(val), true\n\tcase uint8:\n\t\treturn float64(val), true\n\tcase uint16:\n\t\treturn float64(val), true\n\tcase uint32:\n\t\treturn float64(val), true\n\tcase uint64:\n\t\treturn float64(val), true\n\tcase float32:\n\t\treturn float64(val), true\n\tcase float64:\n\t\treturn val, true\n\tdefault:\n\t\treturn 0, false\n\t}\n}\n\nfunc usingDifferentKeys(key string, field reflect.StructField) bool {\n\tif len(field.Tag) > 0 {\n\t\tif _, ok := field.Tag.Lookup(key); !ok {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc validateAndSetValue(kind reflect.Kind, value reflect.Value, str string,\n\topts *fieldOptionsWithContext) error {\n\tif !value.CanSet() {\n\t\treturn errValueNotSettable\n\t}\n\n\tv, err := convertTypeFromString(kind, str)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif err := validateValueRange(v, opts); err != nil {\n\t\treturn err\n\t}\n\n\treturn setMatchedPrimitiveValue(kind, value, v)\n}\n\nfunc validateJsonNumberRange(v json.Number, opts *fieldOptionsWithContext) error {\n\tif opts == nil || opts.Range == nil {\n\t\treturn nil\n\t}\n\n\tfv, err := v.Float64()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn validateNumberRange(fv, opts.Range)\n}\n\nfunc validateNumberRange(fv float64, nr *numberRange) error {\n\tif nr == nil {\n\t\treturn nil\n\t}\n\n\tif (nr.leftInclude && fv < nr.left) || (!nr.leftInclude && fv <= nr.left) {\n\t\treturn errNumberRange\n\t}\n\n\tif (nr.rightInclude && fv > nr.right) || (!nr.rightInclude && fv >= nr.right) {\n\t\treturn errNumberRange\n\t}\n\n\treturn nil\n}\n\nfunc validateValueInOptions(val any, options []string) error {\n\tif len(options) > 0 {\n\t\tswitch v := val.(type) {\n\t\tcase string:\n\t\t\tif !stringx.Contains(options, v) {\n\t\t\t\treturn fmt.Errorf(`error: value %q is not defined in options \"%v\"`, v, options)\n\t\t\t}\n\t\tdefault:\n\t\t\tif !stringx.Contains(options, Repr(v)) {\n\t\t\t\treturn fmt.Errorf(`error: value \"%v\" is not defined in options \"%v\"`, val, options)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc validateValueRange(mapValue any, opts *fieldOptionsWithContext) error {\n\tif opts == nil || opts.Range == nil {\n\t\treturn nil\n\t}\n\n\tfv, ok := toFloat64(mapValue)\n\tif !ok {\n\t\treturn errNumberRange\n\t}\n\n\treturn validateNumberRange(fv, opts.Range)\n}\n\n// Path: core/mapping/utils_test.go\npackage mapping\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst testTagName = \"key\"\n\ntype Foo struct {\n\tStr                 string\n\tStrWithTag          string `key:\"stringwithtag\"`\n\tStrWithTagAndOption string `key:\"stringwithtag,string\"`\n}\n\nfunc TestDerefInt(t *testing.T) {\n\ti := 1\n\ts := \"hello\"\n\tnumber := struct {\n\t\tf float64\n\t}{\n\t\tf: 6.4,\n\t}\n\tcases := []struct {\n\t\tt      reflect.Type\n\t\texpect reflect.Kind\n\t}{\n\t\t{\n\t\t\tt:      reflect.TypeOf(i),\n\t\t\texpect: reflect.Int,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.TypeOf(&i),\n\t\t\texpect: reflect.Int,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.TypeOf(s),\n\t\t\texpect: reflect.String,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.TypeOf(&s),\n\t\t\texpect: reflect.String,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.TypeOf(number.f),\n\t\t\texpect: reflect.Float64,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.TypeOf(&number.f),\n\t\t\texpect: reflect.Float64,\n\t\t},\n\t}\n\n\tfor _, each := range cases {\n\t\tt.Run(each.t.String(), func(t *testing.T) {\n\t\t\tassert.Equal(t, each.expect, Deref(each.t).Kind())\n\t\t})\n\t}\n}\n\nfunc TestDerefValInt(t *testing.T) {\n\ti := 1\n\ts := \"hello\"\n\tnumber := struct {\n\t\tf float64\n\t}{\n\t\tf: 6.4,\n\t}\n\tcases := []struct {\n\t\tt      reflect.Value\n\t\texpect reflect.Kind\n\t}{\n\t\t{\n\t\t\tt:      reflect.ValueOf(i),\n\t\t\texpect: reflect.Int,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.ValueOf(&i),\n\t\t\texpect: reflect.Int,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.ValueOf(s),\n\t\t\texpect: reflect.String,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.ValueOf(&s),\n\t\t\texpect: reflect.String,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.ValueOf(number.f),\n\t\t\texpect: reflect.Float64,\n\t\t},\n\t\t{\n\t\t\tt:      reflect.ValueOf(&number.f),\n\t\t\texpect: reflect.Float64,\n\t\t},\n\t}\n\n\tfor _, each := range cases {\n\t\tt.Run(each.t.String(), func(t *testing.T) {\n\t\t\tassert.Equal(t, each.expect, ensureValue(each.t).Kind())\n\t\t})\n\t}\n}\n\nfunc TestParseKeyAndOptionWithoutTag(t *testing.T) {\n\tvar foo Foo\n\trte := reflect.TypeOf(&foo).Elem()\n\tfield, _ := rte.FieldByName(\"Str\")\n\tkey, options, err := parseKeyAndOptions(testTagName, field)\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"Str\", key)\n\tassert.Nil(t, options)\n}\n\nfunc TestParseKeyAndOptionWithTagWithoutOption(t *testing.T) {\n\tvar foo Foo\n\trte := reflect.TypeOf(&foo).Elem()\n\tfield, _ := rte.FieldByName(\"StrWithTag\")\n\tkey, options, err := parseKeyAndOptions(testTagName, field)\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"stringwithtag\", key)\n\tassert.Nil(t, options)\n}\n\nfunc TestParseKeyAndOptionWithTagAndOption(t *testing.T) {\n\tvar foo Foo\n\trte := reflect.TypeOf(&foo).Elem()\n\tfield, _ := rte.FieldByName(\"StrWithTagAndOption\")\n\tkey, options, err := parseKeyAndOptions(testTagName, field)\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"stringwithtag\", key)\n\tassert.True(t, options.FromString)\n}\n\nfunc TestParseSegments(t *testing.T) {\n\ttests := []struct {\n\t\tinput  string\n\t\texpect []string\n\t}{\n\t\t{\n\t\t\tinput:  \"\",\n\t\t\texpect: []string{},\n\t\t},\n\t\t{\n\t\t\tinput:  \"   \",\n\t\t\texpect: []string{},\n\t\t},\n\t\t{\n\t\t\tinput:  \",\",\n\t\t\texpect: []string{\"\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"foo,\",\n\t\t\texpect: []string{\"foo\"},\n\t\t},\n\t\t{\n\t\t\tinput: \",foo\",\n\t\t\t// the first empty string cannot be ignored, it's the key.\n\t\t\texpect: []string{\"\", \"foo\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"foo\",\n\t\t\texpect: []string{\"foo\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"foo,bar\",\n\t\t\texpect: []string{\"foo\", \"bar\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"foo,bar,baz\",\n\t\t\texpect: []string{\"foo\", \"bar\", \"baz\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"foo,options=a|b\",\n\t\t\texpect: []string{\"foo\", \"options=a|b\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"foo,bar,default=[baz,qux]\",\n\t\t\texpect: []string{\"foo\", \"bar\", \"default=[baz,qux]\"},\n\t\t},\n\t\t{\n\t\t\tinput:  \"foo,bar,options=[baz,qux]\",\n\t\t\texpect: []string{\"foo\", \"bar\", \"options=[baz,qux]\"},\n\t\t},\n\t\t{\n\t\t\tinput:  `foo\\,bar,options=[baz,qux]`,\n\t\t\texpect: []string{`foo,bar`, \"options=[baz,qux]\"},\n\t\t},\n\t\t{\n\t\t\tinput:  `foo,bar,options=\\[baz,qux]`,\n\t\t\texpect: []string{\"foo\", \"bar\", \"options=[baz\", \"qux]\"},\n\t\t},\n\t\t{\n\t\t\tinput:  `foo,bar,options=[baz\\,qux]`,\n\t\t\texpect: []string{\"foo\", \"bar\", `options=[baz\\,qux]`},\n\t\t},\n\t\t{\n\t\t\tinput:  `foo\\,bar,options=[baz,qux],default=baz`,\n\t\t\texpect: []string{`foo,bar`, \"options=[baz,qux]\", \"default=baz\"},\n\t\t},\n\t\t{\n\t\t\tinput:  `foo\\,bar,options=[baz,qux, quux],default=[qux, baz]`,\n\t\t\texpect: []string{`foo,bar`, \"options=[baz,qux, quux]\", \"default=[qux, baz]\"},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttest := test\n\t\tt.Run(test.input, func(t *testing.T) {\n\t\t\tassert.ElementsMatch(t, test.expect, parseSegments(test.input))\n\t\t})\n\t}\n}\n\nfunc TestValidatePtrWithNonPtr(t *testing.T) {\n\tvar foo string\n\trve := reflect.ValueOf(foo)\n\tassert.NotNil(t, ValidatePtr(rve))\n}\n\nfunc TestValidatePtrWithPtr(t *testing.T) {\n\tvar foo string\n\trve := reflect.ValueOf(&foo)\n\tassert.Nil(t, ValidatePtr(rve))\n}\n\nfunc TestValidatePtrWithNilPtr(t *testing.T) {\n\tvar foo *string\n\trve := reflect.ValueOf(foo)\n\tassert.NotNil(t, ValidatePtr(rve))\n}\n\nfunc TestValidatePtrWithZeroValue(t *testing.T) {\n\tvar s string\n\te := reflect.Zero(reflect.TypeOf(s))\n\tassert.NotNil(t, ValidatePtr(e))\n}\n\nfunc TestSetValueNotSettable(t *testing.T) {\n\tvar i int\n\tassert.Error(t, setValueFromString(reflect.Int, reflect.ValueOf(i), \"1\"))\n\tassert.Error(t, validateAndSetValue(reflect.Int, reflect.ValueOf(i), \"1\", nil))\n}\n\nfunc TestParseKeyAndOptionsErrors(t *testing.T) {\n\ttype Bar struct {\n\t\tOptionsValue string `key:\",options=a=b\"`\n\t\tDefaultValue string `key:\",default=a=b\"`\n\t}\n\n\tvar bar Bar\n\t_, _, err := parseKeyAndOptions(\"key\", reflect.TypeOf(&bar).Elem().Field(0))\n\tassert.NotNil(t, err)\n\t_, _, err = parseKeyAndOptions(\"key\", reflect.TypeOf(&bar).Elem().Field(1))\n\tassert.NotNil(t, err)\n}\n\nfunc TestSetValueFormatErrors(t *testing.T) {\n\ttype Bar struct {\n\t\tIntValue   int\n\t\tUintValue  uint\n\t\tFloatValue float32\n\t\tMapValue   map[string]any\n\t}\n\n\tvar bar Bar\n\ttests := []struct {\n\t\tkind   reflect.Kind\n\t\ttarget reflect.Value\n\t\tvalue  string\n\t}{\n\t\t{\n\t\t\tkind:   reflect.Int,\n\t\t\ttarget: reflect.ValueOf(&bar.IntValue).Elem(),\n\t\t\tvalue:  \"a\",\n\t\t},\n\t\t{\n\t\t\tkind:   reflect.Uint,\n\t\t\ttarget: reflect.ValueOf(&bar.UintValue).Elem(),\n\t\t\tvalue:  \"a\",\n\t\t},\n\t\t{\n\t\t\tkind:   reflect.Float32,\n\t\t\ttarget: reflect.ValueOf(&bar.FloatValue).Elem(),\n\t\t\tvalue:  \"a\",\n\t\t},\n\t\t{\n\t\t\tkind:   reflect.Map,\n\t\t\ttarget: reflect.ValueOf(&bar.MapValue).Elem(),\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.kind.String(), func(t *testing.T) {\n\t\t\terr := setValueFromString(test.kind, test.target, test.value)\n\t\t\tassert.NotEqual(t, errValueNotSettable, err)\n\t\t\tassert.NotNil(t, err)\n\t\t})\n\t}\n}\n\nfunc TestValidateValueRange(t *testing.T) {\n\tt.Run(\"float\", func(t *testing.T) {\n\t\tassert.NoError(t, validateValueRange(1.2, nil))\n\t})\n\n\tt.Run(\"float number range\", func(t *testing.T) {\n\t\tassert.NoError(t, validateNumberRange(1.2, nil))\n\t})\n\n\tt.Run(\"bad float\", func(t *testing.T) {\n\t\tassert.Error(t, validateValueRange(\"a\", &fieldOptionsWithContext{\n\t\t\tRange: &numberRange{},\n\t\t}))\n\t})\n\n\tt.Run(\"bad float validate\", func(t *testing.T) {\n\t\tvar v struct {\n\t\t\tFoo float32\n\t\t}\n\t\tassert.Error(t, validateAndSetValue(reflect.Int, reflect.ValueOf(&v).Elem().Field(0),\n\t\t\t\"1\", &fieldOptionsWithContext{\n\t\t\t\tRange: &numberRange{\n\t\t\t\t\tleft:  2,\n\t\t\t\t\tright: 3,\n\t\t\t\t},\n\t\t\t}))\n\t})\n}\n\nfunc TestSetMatchedPrimitiveValue(t *testing.T) {\n\tassert.Error(t, setMatchedPrimitiveValue(reflect.Func, reflect.ValueOf(2), \"1\"))\n}\n\n// Path: core/mapping/valuer.go\npackage mapping\n\ntype (\n\t// A Valuer interface defines the way to get values from the underlying object with keys.\n\tValuer interface {\n\t\t// Value gets the value associated with the given key.\n\t\tValue(key string) (any, bool)\n\t}\n\n\t// A valuerWithParent defines a node that has a parent node.\n\tvaluerWithParent interface {\n\t\tValuer\n\t\t// Parent get the parent valuer for current node.\n\t\tParent() valuerWithParent\n\t}\n\n\t// A node is a map that can use Value method to get values with given keys.\n\tnode struct {\n\t\tcurrent Valuer\n\t\tparent  valuerWithParent\n\t}\n\n\t// A valueWithParent is used to wrap the value with its parent.\n\tvalueWithParent struct {\n\t\tvalue  any\n\t\tparent valuerWithParent\n\t}\n\n\t// mapValuer is a type for the map to meet the Valuer interface.\n\tmapValuer map[string]any\n\t// simpleValuer is a type to get value from the current node.\n\tsimpleValuer node\n\t// recursiveValuer is a type to get the value recursively from current and parent nodes.\n\trecursiveValuer node\n)\n\n// Value gets the value associated with the given key from mv.\nfunc (mv mapValuer) Value(key string) (any, bool) {\n\tv, ok := mv[key]\n\treturn v, ok\n}\n\n// Value gets the value associated with the given key from sv.\nfunc (sv simpleValuer) Value(key string) (any, bool) {\n\tv, ok := sv.current.Value(key)\n\treturn v, ok\n}\n\n// Parent get the parent valuer from sv.\nfunc (sv simpleValuer) Parent() valuerWithParent {\n\tif sv.parent == nil {\n\t\treturn nil\n\t}\n\n\treturn recursiveValuer{\n\t\tcurrent: sv.parent,\n\t\tparent:  sv.parent.Parent(),\n\t}\n}\n\n// Value gets the value associated with the given key from rv,\n// and it will inherit the value from parent nodes.\nfunc (rv recursiveValuer) Value(key string) (any, bool) {\n\tval, ok := rv.current.Value(key)\n\tif !ok {\n\t\tif parent := rv.Parent(); parent != nil {\n\t\t\treturn parent.Value(key)\n\t\t}\n\n\t\treturn nil, false\n\t}\n\n\tvm, ok := val.(map[string]any)\n\tif !ok {\n\t\treturn val, true\n\t}\n\n\tparent := rv.Parent()\n\tif parent == nil {\n\t\treturn val, true\n\t}\n\n\tpv, ok := parent.Value(key)\n\tif !ok {\n\t\treturn val, true\n\t}\n\n\tpm, ok := pv.(map[string]any)\n\tif !ok {\n\t\treturn val, true\n\t}\n\n\tfor k, v := range pm {\n\t\tif _, ok := vm[k]; !ok {\n\t\t\tvm[k] = v\n\t\t}\n\t}\n\n\treturn vm, true\n}\n\n// Parent get the parent valuer from rv.\nfunc (rv recursiveValuer) Parent() valuerWithParent {\n\tif rv.parent == nil {\n\t\treturn nil\n\t}\n\n\treturn recursiveValuer{\n\t\tcurrent: rv.parent,\n\t\tparent:  rv.parent.Parent(),\n\t}\n}\n\n// Path: core/mapping/valuer_test.go\npackage mapping\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\n\nfunc TestMapValuerWithInherit_Value(t *testing.T) {\n\tinput := map[string]any{\n\t\t\"discovery\": map[string]any{\n\t\t\t\"host\": \"localhost\",\n\t\t\t\"port\": 8080,\n\t\t},\n\t\t\"component\": map[string]any{\n\t\t\t\"name\": \"test\",\n\t\t},\n\t}\n\tvaluer := recursiveValuer{\n\t\tcurrent: mapValuer(input[\"component\"].(map[string]any)),\n\t\tparent: simpleValuer{\n\t\t\tcurrent: mapValuer(input),\n\t\t},\n\t}\n\n\tval, ok := valuer.Value(\"discovery\")\n\tassert.True(t, ok)\n\n\tm, ok := val.(map[string]any)\n\tassert.True(t, ok)\n\tassert.Equal(t, \"localhost\", m[\"host\"])\n\tassert.Equal(t, 8080, m[\"port\"])\n}\n\nfunc TestRecursiveValuer_Value(t *testing.T) {\n\tinput := map[string]any{\n\t\t\"component\": map[string]any{\n\t\t\t\"name\": \"test\",\n\t\t\t\"foo\": map[string]any{\n\t\t\t\t\"bar\": \"baz\",\n\t\t\t},\n\t\t},\n\t\t\"foo\": \"value\",\n\t}\n\tvaluer := recursiveValuer{\n\t\tcurrent: mapValuer(input[\"component\"].(map[string]any)),\n\t\tparent: simpleValuer{\n\t\t\tcurrent: mapValuer(input),\n\t\t},\n\t}\n\n\tval, ok := valuer.Value(\"foo\")\n\tassert.True(t, ok)\n\tassert.EqualValues(t, map[string]any{\n\t\t\"bar\": \"baz\",\n\t}, val)\n}\n\n// Path: core/mapping/yamlunmarshaler.go\npackage mapping\n\nimport (\n\t\"io\"\n\n\t\"github.com/zeromicro/go-zero/internal/encoding\"\n)\n\n// UnmarshalYamlBytes unmarshals content into v.\nfunc UnmarshalYamlBytes(content []byte, v any, opts ...UnmarshalOption) error {\n\tb, err := encoding.YamlToJson(content)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn UnmarshalJsonBytes(b, v, opts...)\n}\n\n// UnmarshalYamlReader unmarshals content from reader into v.\nfunc UnmarshalYamlReader(reader io.Reader, v any, opts ...UnmarshalOption) error {\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn UnmarshalYamlBytes(b, v, opts...)\n}\n\n// Path: core/mapping/yamlunmarshaler_test.go\npackage mapping\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"k8s.io/utils/io\"\n)\n\nfunc TestUnmarshalYamlBytes(t *testing.T) {\n\tvar c struct {\n\t\tName string\n\t}\n\tcontent := []byte(`Name: liao`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, \"liao\", c.Name)\n}\n\nfunc TestUnmarshalYamlBytesErrorInput(t *testing.T) {\n\tvar c struct {\n\t\tName string\n\t}\n\tcontent := []byte(`liao`)\n\tassert.NotNil(t, UnmarshalYamlBytes(content, &c))\n}\n\nfunc TestUnmarshalYamlBytesEmptyInput(t *testing.T) {\n\tvar c struct {\n\t\tName string\n\t}\n\tcontent := []byte(``)\n\tassert.NotNil(t, UnmarshalYamlBytes(content, &c))\n}\n\nfunc TestUnmarshalYamlBytesOptional(t *testing.T) {\n\tvar c struct {\n\t\tName string\n\t\tAge  int `json:\",optional\"`\n\t}\n\tcontent := []byte(`Name: liao`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, \"liao\", c.Name)\n}\n\nfunc TestUnmarshalYamlBytesOptionalDefault(t *testing.T) {\n\tvar c struct {\n\t\tName string\n\t\tAge  int `json:\",optional,default=1\"`\n\t}\n\tcontent := []byte(`Name: liao`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, \"liao\", c.Name)\n\tassert.Equal(t, 1, c.Age)\n}\n\nfunc TestUnmarshalYamlBytesDefaultOptional(t *testing.T) {\n\tvar c struct {\n\t\tName string\n\t\tAge  int `json:\",default=1,optional\"`\n\t}\n\tcontent := []byte(`Name: liao`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, \"liao\", c.Name)\n\tassert.Equal(t, 1, c.Age)\n}\n\nfunc TestUnmarshalYamlBytesDefault(t *testing.T) {\n\tvar c struct {\n\t\tName string `json:\",default=liao\"`\n\t}\n\tcontent := []byte(`{}`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, \"liao\", c.Name)\n}\n\nfunc TestUnmarshalYamlBytesBool(t *testing.T) {\n\tvar c struct {\n\t\tGreat bool\n\t}\n\tcontent := []byte(`Great: true`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.True(t, c.Great)\n}\n\nfunc TestUnmarshalYamlBytesInt(t *testing.T) {\n\tvar c struct {\n\t\tAge int\n\t}\n\tcontent := []byte(`Age: 1`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, 1, c.Age)\n}\n\nfunc TestUnmarshalYamlBytesUint(t *testing.T) {\n\tvar c struct {\n\t\tAge uint\n\t}\n\tcontent := []byte(`Age: 1`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, uint(1), c.Age)\n}\n\nfunc TestUnmarshalYamlBytesFloat(t *testing.T) {\n\tvar c struct {\n\t\tAge float32\n\t}\n\tcontent := []byte(`Age: 1.5`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, float32(1.5), c.Age)\n}\n\nfunc TestUnmarshalYamlBytesMustInOptional(t *testing.T) {\n\tvar c struct {\n\t\tInner struct {\n\t\t\tThere    string\n\t\t\tMust     string\n\t\t\tOptional string `json:\",optional\"`\n\t\t} `json:\",optional\"`\n\t}\n\tcontent := []byte(`{}`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n}\n\nfunc TestUnmarshalYamlBytesMustInOptionalMissedPart(t *testing.T) {\n\tvar c struct {\n\t\tInner struct {\n\t\t\tThere    string\n\t\t\tMust     string\n\t\t\tOptional string `json:\",optional\"`\n\t\t} `json:\",optional\"`\n\t}\n\tcontent := []byte(`Inner:\n  There: sure`)\n\n\tassert.NotNil(t, UnmarshalYamlBytes(content, &c))\n}\n\nfunc TestUnmarshalYamlBytesMustInOptionalOnlyOptionalFilled(t *testing.T) {\n\tvar c struct {\n\t\tInner struct {\n\t\t\tThere    string\n\t\t\tMust     string\n\t\t\tOptional string `json:\",optional\"`\n\t\t} `json:\",optional\"`\n\t}\n\tcontent := []byte(`Inner:\n  Optional: sure`)\n\n\tassert.NotNil(t, UnmarshalYamlBytes(content, &c))\n}\n\nfunc TestUnmarshalYamlBytesPartial(t *testing.T) {\n\tvar c struct {\n\t\tName string\n\t\tAge  float32\n\t}\n\tcontent := []byte(`Age: 1.5`)\n\n\tassert.NotNil(t, UnmarshalYamlBytes(content, &c))\n}\n\nfunc TestUnmarshalYamlBytesStruct(t *testing.T) {\n\tvar c struct {\n\t\tInner struct {\n\t\t\tName string\n\t\t}\n\t}\n\tcontent := []byte(`Inner:\n  Name: liao`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, \"liao\", c.Inner.Name)\n}\n\nfunc TestUnmarshalYamlBytesStructOptional(t *testing.T) {\n\tvar c struct {\n\t\tInner struct {\n\t\t\tName string\n\t\t\tAge  int `json:\",optional\"`\n\t\t}\n\t}\n\tcontent := []byte(`Inner:\n  Name: liao`)\n\n\tassert.Nil(t, UnmarshalYamlBytes(content, &c))\n\tassert.Equal(t, \"liao\", c.Inner.Name)\n}\n\n...\n", "needle_token_start": 12564, "needle_token_end": 12805, "code_context_ntokens": 14931, "output": ["```go\nfunc TestMapValuerWithInherit_Value(t *testing.T) {\n\tinput := map[string]any{\n\t\t\"discovery\": map[string]any{\n\t\t\t\"host\": \"localhost\",\n\t\t\t\"port\": 8080,\n\t\t},\n\t\t\"component\": map[string]any{\n\t\t\t\"name\": \"test\",\n\t\t},\n\t}\n\tvaluer := recursiveValuer{\n\t\tcurrent: mapValuer(input[\"component\"].(map[string]any)),\n\t\tparent: simpleValuer{\n\t\t\tcurrent: mapValuer(input),\n\t\t},\n\t}\n\n\tval, ok := valuer.Value(\"discovery\")\n\tassert.True(t, ok)\n\n\tm, ok := val.(map[string]any)\n\tassert.True(t, ok)\n\tassert.Equal(t, \"localhost\", m[\"host\"])\n\tassert.Equal(t, 8080, m[\"port\"])\n}\n```"]}
{"repo": "zeromicro/go-zero", "name": "TestCacheNode_DelCache", "language": "go", "path": "core/stores/cache/cachenode_test.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to test the deletion functionality of a caching mechanism in a simulated Redis environment, ensuring that keys can be properly removed from the cache.\n2. **Input**: The input includes various scenarios of keys to be deleted, such as no keys, empty key lists, and specific keys.\n3. **Output**: The output is the result of the deletion operations, typically checking for any errors that might occur during the deletion process.\n4. **Procedure**: The procedure involves setting up a mock Redis server, initializing a cache node, and performing deletion operations under different conditions. It checks for the successful removal of keys and handles potential errors, such as simulated Redis failures, to ensure robustness.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: core/stores/cache/cachenode.go\npackage cache\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"math\"\n\t\"math/rand\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/zeromicro/go-zero/core/jsonx\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/mathx\"\n\t\"github.com/zeromicro/go-zero/core/stat\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n)\n\nconst (\n\tnotFoundPlaceholder = \"*\"\n\t// make the expiry unstable to avoid lots of cached items expire at the same time\n\t// make the unstable expiry to be [0.95, 1.05] * seconds\n\texpiryDeviation = 0.05\n)\n\n// indicates there is no such value associate with the key\nvar errPlaceholder = errors.New(\"placeholder\")\n\ntype cacheNode struct {\n\trds            *redis.Redis\n\texpiry         time.Duration\n\tnotFoundExpiry time.Duration\n\tbarrier        syncx.SingleFlight\n\tr              *rand.Rand\n\tlock           *sync.Mutex\n\tunstableExpiry mathx.Unstable\n\tstat           *Stat\n\terrNotFound    error\n}\n\n// NewNode returns a cacheNode.\n// rds is the underlying redis node or cluster.\n// barrier is the barrier that maybe shared with other cache nodes on cache cluster.\n// st is used to stat the cache.\n// errNotFound defines the error that returned on cache not found.\n// opts are the options that customize the cacheNode.\nfunc NewNode(rds *redis.Redis, barrier syncx.SingleFlight, st *Stat,\n\terrNotFound error, opts ...Option) Cache {\n\to := newOptions(opts...)\n\treturn cacheNode{\n\t\trds:            rds,\n\t\texpiry:         o.Expiry,\n\t\tnotFoundExpiry: o.NotFoundExpiry,\n\t\tbarrier:        barrier,\n\t\tr:              rand.New(rand.NewSource(time.Now().UnixNano())),\n\t\tlock:           new(sync.Mutex),\n\t\tunstableExpiry: mathx.NewUnstable(expiryDeviation),\n\t\tstat:           st,\n\t\terrNotFound:    errNotFound,\n\t}\n}\n\n// Del deletes cached values with keys.\nfunc (c cacheNode) Del(keys ...string) error {\n\treturn c.DelCtx(context.Background(), keys...)\n}\n\n// DelCtx deletes cached values with keys.\nfunc (c cacheNode) DelCtx(ctx context.Context, keys ...string) error {\n\tif len(keys) == 0 {\n\t\treturn nil\n\t}\n\n\tlogger := logx.WithContext(ctx)\n\tif len(keys) > 1 && c.rds.Type == redis.ClusterType {\n\t\tfor _, key := range keys {\n\t\t\tif _, err := c.rds.DelCtx(ctx, key); err != nil {\n\t\t\t\tlogger.Errorf(\"failed to clear cache with key: %q, error: %v\", key, err)\n\t\t\t\tc.asyncRetryDelCache(key)\n\t\t\t}\n\t\t}\n\t} else if _, err := c.rds.DelCtx(ctx, keys...); err != nil {\n\t\tlogger.Errorf(\"failed to clear cache with keys: %q, error: %v\", formatKeys(keys), err)\n\t\tc.asyncRetryDelCache(keys...)\n\t}\n\n\treturn nil\n}\n\n// Get gets the cache with key and fills into v.\nfunc (c cacheNode) Get(key string, val any) error {\n\treturn c.GetCtx(context.Background(), key, val)\n}\n\n// GetCtx gets the cache with key and fills into v.\nfunc (c cacheNode) GetCtx(ctx context.Context, key string, val any) error {\n\terr := c.doGetCache(ctx, key, val)\n\tif errors.Is(err, errPlaceholder) {\n\t\treturn c.errNotFound\n\t}\n\n\treturn err\n}\n\n// IsNotFound checks if the given error is the defined errNotFound.\nfunc (c cacheNode) IsNotFound(err error) bool {\n\treturn errors.Is(err, c.errNotFound)\n}\n\n// Set sets the cache with key and v, using c.expiry.\nfunc (c cacheNode) Set(key string, val any) error {\n\treturn c.SetCtx(context.Background(), key, val)\n}\n\n// SetCtx sets the cache with key and v, using c.expiry.\nfunc (c cacheNode) SetCtx(ctx context.Context, key string, val any) error {\n\treturn c.SetWithExpireCtx(ctx, key, val, c.aroundDuration(c.expiry))\n}\n\n// SetWithExpire sets the cache with key and v, using given expire.\nfunc (c cacheNode) SetWithExpire(key string, val any, expire time.Duration) error {\n\treturn c.SetWithExpireCtx(context.Background(), key, val, expire)\n}\n\n// SetWithExpireCtx sets the cache with key and v, using given expire.\nfunc (c cacheNode) SetWithExpireCtx(ctx context.Context, key string, val any,\n\texpire time.Duration) error {\n\tdata, err := jsonx.Marshal(val)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn c.rds.SetexCtx(ctx, key, string(data), int(math.Ceil(expire.Seconds())))\n}\n\n// String returns a string that represents the cacheNode.\nfunc (c cacheNode) String() string {\n\treturn c.rds.Addr\n}\n\n// Take takes the result from cache first, if not found,\n// query from DB and set cache using c.expiry, then return the result.\nfunc (c cacheNode) Take(val any, key string, query func(val any) error) error {\n\treturn c.TakeCtx(context.Background(), val, key, query)\n}\n\n// TakeCtx takes the result from cache first, if not found,\n// query from DB and set cache using c.expiry, then return the result.\nfunc (c cacheNode) TakeCtx(ctx context.Context, val any, key string,\n\tquery func(val any) error) error {\n\treturn c.doTake(ctx, val, key, query, func(v any) error {\n\t\treturn c.SetCtx(ctx, key, v)\n\t})\n}\n\n// TakeWithExpire takes the result from cache first, if not found,\n// query from DB and set cache using given expire, then return the result.\nfunc (c cacheNode) TakeWithExpire(val any, key string, query func(val any,\n\texpire time.Duration) error) error {\n\treturn c.TakeWithExpireCtx(context.Background(), val, key, query)\n}\n\n// TakeWithExpireCtx takes the result from cache first, if not found,\n// query from DB and set cache using given expire, then return the result.\nfunc (c cacheNode) TakeWithExpireCtx(ctx context.Context, val any, key string,\n\tquery func(val any, expire time.Duration) error) error {\n\texpire := c.aroundDuration(c.expiry)\n\treturn c.doTake(ctx, val, key, func(v any) error {\n\t\treturn query(v, expire)\n\t}, func(v any) error {\n\t\treturn c.SetWithExpireCtx(ctx, key, v, expire)\n\t})\n}\n\nfunc (c cacheNode) aroundDuration(duration time.Duration) time.Duration {\n\treturn c.unstableExpiry.AroundDuration(duration)\n}\n\nfunc (c cacheNode) asyncRetryDelCache(keys ...string) {\n\tAddCleanTask(func() error {\n\t\t_, err := c.rds.Del(keys...)\n\t\treturn err\n\t}, keys...)\n}\n\nfunc (c cacheNode) doGetCache(ctx context.Context, key string, v any) error {\n\tc.stat.IncrementTotal()\n...\n// Path: core/stores/kv/store.go\npackage kv\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"log\"\n\n\t\"github.com/zeromicro/go-zero/core/errorx\"\n\t\"github.com/zeromicro/go-zero/core/hash\"\n\t\"github.com/zeromicro/go-zero/core/stores/cache\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis\"\n)\n\n// ErrNoRedisNode is an error that indicates no redis node.\nvar ErrNoRedisNode = errors.New(\"no redis node\")\n\ntype (\n\t// Store interface represents a KV store.\n\tStore interface {\n\t\tDecr(key string) (int64, error)\n\t\tDecrCtx(ctx context.Context, key string) (int64, error)\n\t\tDecrby(key string, decrement int64) (int64, error)\n\t\tDecrbyCtx(ctx context.Context, key string, decrement int64) (int64, error)\n\t\tDel(keys ...string) (int, error)\n\t\tDelCtx(ctx context.Context, keys ...string) (int, error)\n\t\tEval(script, key string, args ...any) (any, error)\n\t\tEvalCtx(ctx context.Context, script, key string, args ...any) (any, error)\n\t\tExists(key string) (bool, error)\n\t\tExistsCtx(ctx context.Context, key string) (bool, error)\n\t\tExpire(key string, seconds int) error\n\t\tExpireCtx(ctx context.Context, key string, seconds int) error\n\t\tExpireat(key string, expireTime int64) error\n\t\tExpireatCtx(ctx context.Context, key string, expireTime int64) error\n\t\tGet(key string) (string, error)\n\t\tGetCtx(ctx context.Context, key string) (string, error)\n\t\tGetSet(key, value string) (string, error)\n\t\tGetSetCtx(ctx context.Context, key, value string) (string, error)\n\t\tHdel(key, field string) (bool, error)\n\t\tHdelCtx(ctx context.Context, key, field string) (bool, error)\n\t\tHexists(key, field string) (bool, error)\n\t\tHexistsCtx(ctx context.Context, key, field string) (bool, error)\n\t\tHget(key, field string) (string, error)\n\t\tHgetCtx(ctx context.Context, key, field string) (string, error)\n\t\tHgetall(key string) (map[string]string, error)\n\t\tHgetallCtx(ctx context.Context, key string) (map[string]string, error)\n\t\tHincrby(key, field string, increment int) (int, error)\n\t\tHincrbyCtx(ctx context.Context, key, field string, increment int) (int, error)\n\t\tHkeys(key string) ([]string, error)\n\t\tHkeysCtx(ctx context.Context, key string) ([]string, error)\n\t\tHlen(key string) (int, error)\n\t\tHlenCtx(ctx context.Context, key string) (int, error)\n\t\tHmget(key string, fields ...string) ([]string, error)\n\t\tHmgetCtx(ctx context.Context, key string, fields ...string) ([]string, error)\n\t\tHset(key, field, value string) error\n\t\tHsetCtx(ctx context.Context, key, field, value string) error\n\t\tHsetnx(key, field, value string) (bool, error)\n\t\tHsetnxCtx(ctx context.Context, key, field, value string) (bool, error)\n\t\tHmset(key string, fieldsAndValues map[string]string) error\n\t\tHmsetCtx(ctx context.Context, key string, fieldsAndValues map[string]string) error\n\t\tHvals(key string) ([]string, error)\n\t\tHvalsCtx(ctx context.Context, key string) ([]string, error)\n\t\tIncr(key string) (int64, error)\n\t\tIncrCtx(ctx context.Context, key string) (int64, error)\n\t\tIncrby(key string, increment int64) (int64, error)\n\t\tIncrbyCtx(ctx context.Context, key string, increment int64) (int64, error)\n\t\tLindex(key string, index int64) (string, error)\n\t\tLindexCtx(ctx context.Context, key string, index int64) (string, error)\n\t\tLlen(key string) (int, error)\n\t\tLlenCtx(ctx context.Context, key string) (int, error)\n\t\tLpop(key string) (string, error)\n\t\tLpopCtx(ctx context.Context, key string) (string, error)\n\t\tLpush(key string, values ...any) (int, error)\n\t\tLpushCtx(ctx context.Context, key string, values ...any) (int, error)\n\t\tLrange(key string, start, stop int) ([]string, error)\n\t\tLrangeCtx(ctx context.Context, key string, start, stop int) ([]string, error)\n\t\tLrem(key string, count int, value string) (int, error)\n\t\tLremCtx(ctx context.Context, key string, count int, value string) (int, error)\n\t\tPersist(key string) (bool, error)\n\t\tPersistCtx(ctx context.Context, key string) (bool, error)\n\t\tPfadd(key string, values ...any) (bool, error)\n\t\tPfaddCtx(ctx context.Context, key string, values ...any) (bool, error)\n\t\tPfcount(key string) (int64, error)\n\t\tPfcountCtx(ctx context.Context, key string) (int64, error)\n\t\tRpush(key string, values ...any) (int, error)\n\t\tRpushCtx(ctx context.Context, key string, values ...any) (int, error)\n\t\tSadd(key string, values ...any) (int, error)\n\t\tSaddCtx(ctx context.Context, key string, values ...any) (int, error)\n\t\tScard(key string) (int64, error)\n\t\tScardCtx(ctx context.Context, key string) (int64, error)\n\t\tSet(key, value string) error\n\t\tSetCtx(ctx context.Context, key, value string) error\n\t\tSetex(key, value string, seconds int) error\n\t\tSetexCtx(ctx context.Context, key, value string, seconds int) error\n\t\tSetnx(key, value string) (bool, error)\n\t\tSetnxCtx(ctx context.Context, key, value string) (bool, error)\n\t\tSetnxEx(key, value string, seconds int) (bool, error)\n\t\tSetnxExCtx(ctx context.Context, key, value string, seconds int) (bool, error)\n\t\tSismember(key string, value any) (bool, error)\n\t\tSismemberCtx(ctx context.Context, key string, value any) (bool, error)\n\t\tSmembers(key string) ([]string, error)\n\t\tSmembersCtx(ctx context.Context, key string) ([]string, error)\n\t\tSpop(key string) (string, error)\n\t\tSpopCtx(ctx context.Context, key string) (string, error)\n\t\tSrandmember(key string, count int) ([]string, error)\n\t\tSrandmemberCtx(ctx context.Context, key string, count int) ([]string, error)\n\t\tSrem(key string, values ...any) (int, error)\n\t\tSremCtx(ctx context.Context, key string, values ...any) (int, error)\n\t\tSscan(key string, cursor uint64, match string, count int64) (keys []string, cur uint64, err error)\n\t\tSscanCtx(ctx context.Context, key string, cursor uint64, match string, count int64) (keys []string, cur uint64, err error)\n\t\tTtl(key string) (int, error)\n\t\tTtlCtx(ctx context.Context, key string) (int, error)\n\t\tZadd(key string, score int64, value string) (bool, error)\n\t\tZaddFloat(key string, score float64, value string) (bool, error)\n\t\tZaddCtx(ctx context.Context, key string, score int64, value string) (bool, error)\n\t\tZaddFloatCtx(ctx context.Context, key string, score float64, value string) (bool, error)\n\t\tZadds(key string, ps ...redis.Pair) (int64, error)\n\t\tZaddsCtx(ctx context.Context, key string, ps ...redis.Pair) (int64, error)\n\t\tZcard(key string) (int, error)\n\t\tZcardCtx(ctx context.Context, key string) (int, error)\n\t\tZcount(key string, start, stop int64) (int, error)\n\t\tZcountCtx(ctx context.Context, key string, start, stop int64) (int, error)\n\t\tZincrby(key string, increment int64, field string) (int64, error)\n\t\tZincrbyCtx(ctx context.Context, key string, increment int64, field string) (int64, error)\n\t\tZrange(key string, start, stop int64) ([]string, error)\n\t\tZrangeCtx(ctx context.Context, key string, start, stop int64) ([]string, error)\n\t\tZrangeWithScores(key string, start, stop int64) ([]redis.Pair, error)\n\t\tZrangeWithScoresCtx(ctx context.Context, key string, start, stop int64) ([]redis.Pair, error)\n\t\tZrangebyscoreWithScores(key string, start, stop int64) ([]redis.Pair, error)\n\t\tZrangebyscoreWithScoresCtx(ctx context.Context, key string, start, stop int64) ([]redis.Pair, error)\n\t\tZrangebyscoreWithScoresAndLimit(key string, start, stop int64, page, size int) ([]redis.Pair, error)\n\t\tZrangebyscoreWithScoresAndLimitCtx(ctx context.Context, key string, start, stop int64, page, size int) ([]redis.Pair, error)\n\t\tZrank(key, field string) (int64, error)\n\t\tZrankCtx(ctx context.Context, key, field string) (int64, error)\n\t\tZrem(key string, values ...any) (int, error)\n\t\tZremCtx(ctx context.Context, key string, values ...any) (int, error)\n\t\tZremrangebyrank(key string, start, stop int64) (int, error)\n\t\tZremrangebyrankCtx(ctx context.Context, key string, start, stop int64) (int, error)\n\t\tZremrangebyscore(key string, start, stop int64) (int, error)\n\t\tZremrangebyscoreCtx(ctx context.Context, key string, start, stop int64) (int, error)\n\t\tZrevrange(key string, start, stop int64) ([]string, error)\n\t\tZrevrangeCtx(ctx context.Context, key string, start, stop int64) ([]string, error)\n\t\tZrevrangebyscoreWithScores(key string, start, stop int64) ([]redis.Pair, error)\n\t\tZrevrangebyscoreWithScoresCtx(ctx context.Context, key string, start, stop int64) ([]redis.Pair, error)\n\t\tZrevrangebyscoreWithScoresAndLimit(key string, start, stop int64, page, size int) ([]redis.Pair, error)\n\t\tZrevrangebyscoreWithScoresAndLimitCtx(ctx context.Context, key string, start, stop int64, page, size int) ([]redis.Pair, error)\n\t\tZscore(key, value string) (int64, error)\n\t\tZscoreCtx(ctx context.Context, key, value string) (int64, error)\n\t\tZrevrank(key, field string) (int64, error)\n\t\tZrevrankCtx(ctx context.Context, key, field string) (int64, error)\n\t}\n\n\tclusterStore struct {\n\t\tdispatcher *hash.ConsistentHash\n\t}\n)\n\n// NewStore returns a Store.\nfunc NewStore(c KvConf) Store {\n\tif len(c) == 0 || cache.TotalWeights(c) <= 0 {\n\t\tlog.Fatal(\"no cache nodes\")\n\t}\n\n\t// even if only one node, we chose to use consistent hash,\n\t// because Store and redis.Redis has different methods.\n\tdispatcher := hash.NewConsistentHash()\n\tfor _, node := range c {\n\t\tcn := redis.MustNewRedis(node.RedisConf)\n\t\tdispatcher.AddWithWeight(cn, node.Weight)\n\t}\n\n\treturn clusterStore{\n\t\tdispatcher: dispatcher,\n\t}\n}\n\nfunc (cs clusterStore) Decr(key string) (int64, error) {\n\treturn cs.DecrCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) DecrCtx(ctx context.Context, key string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.DecrCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Decrby(key string, decrement int64) (int64, error) {\n\treturn cs.DecrbyCtx(context.Background(), key, decrement)\n}\n\nfunc (cs clusterStore) DecrbyCtx(ctx context.Context, key string, decrement int64) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.DecrbyCtx(ctx, key, decrement)\n}\n\nfunc (cs clusterStore) Del(keys ...string) (int, error) {\n\treturn cs.DelCtx(context.Background(), keys...)\n}\n\nfunc (cs clusterStore) DelCtx(ctx context.Context, keys ...string) (int, error) {\n\tvar val int\n\tvar be errorx.BatchError\n\n\tfor _, key := range keys {\n\t\tnode, e := cs.getRedis(key)\n\t\tif e != nil {\n\t\t\tbe.Add(e)\n\t\t\tcontinue\n\t\t}\n\n\t\tif v, e := node.DelCtx(ctx, key); e != nil {\n\t\t\tbe.Add(e)\n\t\t} else {\n\t\t\tval += v\n\t\t}\n\t}\n\n\treturn val, be.Err()\n}\n\nfunc (cs clusterStore) Eval(script, key string, args ...any) (any, error) {\n\treturn cs.EvalCtx(context.Background(), script, key, args...)\n}\n\nfunc (cs clusterStore) EvalCtx(ctx context.Context, script, key string, args ...any) (any, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.EvalCtx(ctx, script, []string{key}, args...)\n}\n\nfunc (cs clusterStore) Exists(key string) (bool, error) {\n\treturn cs.ExistsCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) ExistsCtx(ctx context.Context, key string) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.ExistsCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Expire(key string, seconds int) error {\n\treturn cs.ExpireCtx(context.Background(), key, seconds)\n}\n\nfunc (cs clusterStore) ExpireCtx(ctx context.Context, key string, seconds int) error {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn node.ExpireCtx(ctx, key, seconds)\n}\n\nfunc (cs clusterStore) Expireat(key string, expireTime int64) error {\n\treturn cs.ExpireatCtx(context.Background(), key, expireTime)\n}\n\nfunc (cs clusterStore) ExpireatCtx(ctx context.Context, key string, expireTime int64) error {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn node.ExpireatCtx(ctx, key, expireTime)\n}\n\nfunc (cs clusterStore) Get(key string) (string, error) {\n\treturn cs.GetCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) GetCtx(ctx context.Context, key string) (string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn node.GetCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Hdel(key, field string) (bool, error) {\n\treturn cs.HdelCtx(context.Background(), key, field)\n}\n\nfunc (cs clusterStore) HdelCtx(ctx context.Context, key, field string) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.HdelCtx(ctx, key, field)\n}\n\nfunc (cs clusterStore) Hexists(key, field string) (bool, error) {\n\treturn cs.HexistsCtx(context.Background(), key, field)\n}\n\nfunc (cs clusterStore) HexistsCtx(ctx context.Context, key, field string) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.HexistsCtx(ctx, key, field)\n}\n\nfunc (cs clusterStore) Hget(key, field string) (string, error) {\n\treturn cs.HgetCtx(context.Background(), key, field)\n}\n\nfunc (cs clusterStore) HgetCtx(ctx context.Context, key, field string) (string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn node.HgetCtx(ctx, key, field)\n}\n\nfunc (cs clusterStore) Hgetall(key string) (map[string]string, error) {\n\treturn cs.HgetallCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) HgetallCtx(ctx context.Context, key string) (map[string]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.HgetallCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Hincrby(key, field string, increment int) (int, error) {\n\treturn cs.HincrbyCtx(context.Background(), key, field, increment)\n}\n\nfunc (cs clusterStore) HincrbyCtx(ctx context.Context, key, field string, increment int) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.HincrbyCtx(ctx, key, field, increment)\n}\n\nfunc (cs clusterStore) Hkeys(key string) ([]string, error) {\n\treturn cs.HkeysCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) HkeysCtx(ctx context.Context, key string) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.HkeysCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Hlen(key string) (int, error) {\n\treturn cs.HlenCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) HlenCtx(ctx context.Context, key string) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.HlenCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Hmget(key string, fields ...string) ([]string, error) {\n\treturn cs.HmgetCtx(context.Background(), key, fields...)\n}\n\nfunc (cs clusterStore) HmgetCtx(ctx context.Context, key string, fields ...string) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.HmgetCtx(ctx, key, fields...)\n}\n\nfunc (cs clusterStore) Hset(key, field, value string) error {\n\treturn cs.HsetCtx(context.Background(), key, field, value)\n}\n\nfunc (cs clusterStore) HsetCtx(ctx context.Context, key, field, value string) error {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn node.HsetCtx(ctx, key, field, value)\n}\n\nfunc (cs clusterStore) Hsetnx(key, field, value string) (bool, error) {\n\treturn cs.HsetnxCtx(context.Background(), key, field, value)\n}\n\nfunc (cs clusterStore) HsetnxCtx(ctx context.Context, key, field, value string) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.HsetnxCtx(ctx, key, field, value)\n}\n\nfunc (cs clusterStore) Hmset(key string, fieldsAndValues map[string]string) error {\n\treturn cs.HmsetCtx(context.Background(), key, fieldsAndValues)\n}\n\nfunc (cs clusterStore) HmsetCtx(ctx context.Context, key string, fieldsAndValues map[string]string) error {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn node.HmsetCtx(ctx, key, fieldsAndValues)\n}\n\nfunc (cs clusterStore) Hvals(key string) ([]string, error) {\n\treturn cs.HvalsCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) HvalsCtx(ctx context.Context, key string) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.HvalsCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Incr(key string) (int64, error) {\n\treturn cs.IncrCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) IncrCtx(ctx context.Context, key string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.IncrCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Incrby(key string, increment int64) (int64, error) {\n\treturn cs.IncrbyCtx(context.Background(), key, increment)\n}\n\nfunc (cs clusterStore) IncrbyCtx(ctx context.Context, key string, increment int64) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.IncrbyCtx(ctx, key, increment)\n}\n\nfunc (cs clusterStore) Llen(key string) (int, error) {\n\treturn cs.LlenCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) LlenCtx(ctx context.Context, key string) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.LlenCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Lindex(key string, index int64) (string, error) {\n\treturn cs.LindexCtx(context.Background(), key, index)\n}\n\nfunc (cs clusterStore) LindexCtx(ctx context.Context, key string, index int64) (string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn node.LindexCtx(ctx, key, index)\n}\n\nfunc (cs clusterStore) Lpop(key string) (string, error) {\n\treturn cs.LpopCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) LpopCtx(ctx context.Context, key string) (string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn node.LpopCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Lpush(key string, values ...any) (int, error) {\n\treturn cs.LpushCtx(context.Background(), key, values...)\n}\n\nfunc (cs clusterStore) LpushCtx(ctx context.Context, key string, values ...any) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.LpushCtx(ctx, key, values...)\n}\n\nfunc (cs clusterStore) Lrange(key string, start, stop int) ([]string, error) {\n\treturn cs.LrangeCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) LrangeCtx(ctx context.Context, key string, start, stop int) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.LrangeCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) Lrem(key string, count int, value string) (int, error) {\n\treturn cs.LremCtx(context.Background(), key, count, value)\n}\n\nfunc (cs clusterStore) LremCtx(ctx context.Context, key string, count int, value string) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.LremCtx(ctx, key, count, value)\n}\n\nfunc (cs clusterStore) Persist(key string) (bool, error) {\n\treturn cs.PersistCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) PersistCtx(ctx context.Context, key string) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.PersistCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Pfadd(key string, values ...any) (bool, error) {\n\treturn cs.PfaddCtx(context.Background(), key, values...)\n}\n\nfunc (cs clusterStore) PfaddCtx(ctx context.Context, key string, values ...any) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.PfaddCtx(ctx, key, values...)\n}\n\nfunc (cs clusterStore) Pfcount(key string) (int64, error) {\n\treturn cs.PfcountCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) PfcountCtx(ctx context.Context, key string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.PfcountCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Rpush(key string, values ...any) (int, error) {\n\treturn cs.RpushCtx(context.Background(), key, values...)\n}\n\nfunc (cs clusterStore) RpushCtx(ctx context.Context, key string, values ...any) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.RpushCtx(ctx, key, values...)\n}\n\nfunc (cs clusterStore) Sadd(key string, values ...any) (int, error) {\n\treturn cs.SaddCtx(context.Background(), key, values...)\n}\n\nfunc (cs clusterStore) SaddCtx(ctx context.Context, key string, values ...any) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.SaddCtx(ctx, key, values...)\n}\n\nfunc (cs clusterStore) Scard(key string) (int64, error) {\n\treturn cs.ScardCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) ScardCtx(ctx context.Context, key string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ScardCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Set(key, value string) error {\n\treturn cs.SetCtx(context.Background(), key, value)\n}\n\nfunc (cs clusterStore) SetCtx(ctx context.Context, key, value string) error {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn node.SetCtx(ctx, key, value)\n}\n\nfunc (cs clusterStore) Setex(key, value string, seconds int) error {\n\treturn cs.SetexCtx(context.Background(), key, value, seconds)\n}\n\nfunc (cs clusterStore) SetexCtx(ctx context.Context, key, value string, seconds int) error {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn node.SetexCtx(ctx, key, value, seconds)\n}\n\nfunc (cs clusterStore) Setnx(key, value string) (bool, error) {\n\treturn cs.SetnxCtx(context.Background(), key, value)\n}\n\nfunc (cs clusterStore) SetnxCtx(ctx context.Context, key, value string) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.SetnxCtx(ctx, key, value)\n}\n\nfunc (cs clusterStore) SetnxEx(key, value string, seconds int) (bool, error) {\n\treturn cs.SetnxExCtx(context.Background(), key, value, seconds)\n}\n\nfunc (cs clusterStore) SetnxExCtx(ctx context.Context, key, value string, seconds int) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.SetnxExCtx(ctx, key, value, seconds)\n}\n\nfunc (cs clusterStore) GetSet(key, value string) (string, error) {\n\treturn cs.GetSetCtx(context.Background(), key, value)\n}\n\nfunc (cs clusterStore) GetSetCtx(ctx context.Context, key, value string) (string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn node.GetSetCtx(ctx, key, value)\n}\n\nfunc (cs clusterStore) Sismember(key string, value any) (bool, error) {\n\treturn cs.SismemberCtx(context.Background(), key, value)\n}\n\nfunc (cs clusterStore) SismemberCtx(ctx context.Context, key string, value any) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.SismemberCtx(ctx, key, value)\n}\n\nfunc (cs clusterStore) Smembers(key string) ([]string, error) {\n\treturn cs.SmembersCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) SmembersCtx(ctx context.Context, key string) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.SmembersCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Spop(key string) (string, error) {\n\treturn cs.SpopCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) SpopCtx(ctx context.Context, key string) (string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn node.SpopCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Srandmember(key string, count int) ([]string, error) {\n\treturn cs.SrandmemberCtx(context.Background(), key, count)\n}\n\nfunc (cs clusterStore) SrandmemberCtx(ctx context.Context, key string, count int) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.SrandmemberCtx(ctx, key, count)\n}\n\nfunc (cs clusterStore) Srem(key string, values ...any) (int, error) {\n\treturn cs.SremCtx(context.Background(), key, values...)\n}\n\nfunc (cs clusterStore) SremCtx(ctx context.Context, key string, values ...any) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.SremCtx(ctx, key, values...)\n}\n\nfunc (cs clusterStore) Sscan(key string, cursor uint64, match string, count int64) (\n\tkeys []string, cur uint64, err error) {\n\treturn cs.SscanCtx(context.Background(), key, cursor, match, count)\n}\n\nfunc (cs clusterStore) SscanCtx(ctx context.Context, key string, cursor uint64, match string, count int64) (\n\tkeys []string, cur uint64, err error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\n\treturn node.SscanCtx(ctx, key, cursor, match, count)\n}\n\nfunc (cs clusterStore) Ttl(key string) (int, error) {\n\treturn cs.TtlCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) TtlCtx(ctx context.Context, key string) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.TtlCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Zadd(key string, score int64, value string) (bool, error) {\n\treturn cs.ZaddCtx(context.Background(), key, score, value)\n}\n\nfunc (cs clusterStore) ZaddFloat(key string, score float64, value string) (bool, error) {\n\treturn cs.ZaddFloatCtx(context.Background(), key, score, value)\n}\n\nfunc (cs clusterStore) ZaddCtx(ctx context.Context, key string, score int64, value string) (bool, error) {\n\treturn cs.ZaddFloatCtx(ctx, key, float64(score), value)\n}\n\nfunc (cs clusterStore) ZaddFloatCtx(ctx context.Context, key string, score float64, value string) (bool, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\treturn node.ZaddFloatCtx(ctx, key, score, value)\n}\n\nfunc (cs clusterStore) Zadds(key string, ps ...redis.Pair) (int64, error) {\n\treturn cs.ZaddsCtx(context.Background(), key, ps...)\n}\n\nfunc (cs clusterStore) ZaddsCtx(ctx context.Context, key string, ps ...redis.Pair) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZaddsCtx(ctx, key, ps...)\n}\n\nfunc (cs clusterStore) Zcard(key string) (int, error) {\n\treturn cs.ZcardCtx(context.Background(), key)\n}\n\nfunc (cs clusterStore) ZcardCtx(ctx context.Context, key string) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZcardCtx(ctx, key)\n}\n\nfunc (cs clusterStore) Zcount(key string, start, stop int64) (int, error) {\n\treturn cs.ZcountCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZcountCtx(ctx context.Context, key string, start, stop int64) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZcountCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) Zincrby(key string, increment int64, field string) (int64, error) {\n\treturn cs.ZincrbyCtx(context.Background(), key, increment, field)\n}\n\nfunc (cs clusterStore) ZincrbyCtx(ctx context.Context, key string, increment int64, field string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZincrbyCtx(ctx, key, increment, field)\n}\n\nfunc (cs clusterStore) Zrank(key, field string) (int64, error) {\n\treturn cs.ZrankCtx(context.Background(), key, field)\n}\n\nfunc (cs clusterStore) ZrankCtx(ctx context.Context, key, field string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZrankCtx(ctx, key, field)\n}\n\nfunc (cs clusterStore) Zrange(key string, start, stop int64) ([]string, error) {\n\treturn cs.ZrangeCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZrangeCtx(ctx context.Context, key string, start, stop int64) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.ZrangeCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) ZrangeWithScores(key string, start, stop int64) ([]redis.Pair, error) {\n\treturn cs.ZrangeWithScoresCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZrangeWithScoresCtx(ctx context.Context, key string, start, stop int64) ([]redis.Pair, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.ZrangeWithScoresCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) ZrangebyscoreWithScores(key string, start, stop int64) ([]redis.Pair, error) {\n\treturn cs.ZrangebyscoreWithScoresCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZrangebyscoreWithScoresCtx(ctx context.Context, key string, start, stop int64) ([]redis.Pair, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.ZrangebyscoreWithScoresCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) ZrangebyscoreWithScoresAndLimit(key string, start, stop int64, page, size int) (\n\t[]redis.Pair, error) {\n\treturn cs.ZrangebyscoreWithScoresAndLimitCtx(context.Background(), key, start, stop, page, size)\n}\n\nfunc (cs clusterStore) ZrangebyscoreWithScoresAndLimitCtx(ctx context.Context, key string, start, stop int64, page, size int) (\n\t[]redis.Pair, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.ZrangebyscoreWithScoresAndLimitCtx(ctx, key, start, stop, page, size)\n}\n\nfunc (cs clusterStore) Zrem(key string, values ...any) (int, error) {\n\treturn cs.ZremCtx(context.Background(), key, values...)\n}\n\nfunc (cs clusterStore) ZremCtx(ctx context.Context, key string, values ...any) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZremCtx(ctx, key, values...)\n}\n\nfunc (cs clusterStore) Zremrangebyrank(key string, start, stop int64) (int, error) {\n\treturn cs.ZremrangebyrankCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZremrangebyrankCtx(ctx context.Context, key string, start, stop int64) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZremrangebyrankCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) Zremrangebyscore(key string, start, stop int64) (int, error) {\n\treturn cs.ZremrangebyscoreCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZremrangebyscoreCtx(ctx context.Context, key string, start, stop int64) (int, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZremrangebyscoreCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) Zrevrange(key string, start, stop int64) ([]string, error) {\n\treturn cs.ZrevrangeCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZrevrangeCtx(ctx context.Context, key string, start, stop int64) ([]string, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.ZrevrangeCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) ZrevrangebyscoreWithScores(key string, start, stop int64) ([]redis.Pair, error) {\n\treturn cs.ZrevrangebyscoreWithScoresCtx(context.Background(), key, start, stop)\n}\n\nfunc (cs clusterStore) ZrevrangebyscoreWithScoresCtx(ctx context.Context, key string, start, stop int64) ([]redis.Pair, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.ZrevrangebyscoreWithScoresCtx(ctx, key, start, stop)\n}\n\nfunc (cs clusterStore) ZrevrangebyscoreWithScoresAndLimit(key string, start, stop int64, page, size int) (\n\t[]redis.Pair, error) {\n\treturn cs.ZrevrangebyscoreWithScoresAndLimitCtx(context.Background(), key, start, stop, page, size)\n}\n\nfunc (cs clusterStore) ZrevrangebyscoreWithScoresAndLimitCtx(ctx context.Context, key string, start, stop int64, page, size int) (\n\t[]redis.Pair, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn node.ZrevrangebyscoreWithScoresAndLimitCtx(ctx, key, start, stop, page, size)\n}\n\nfunc (cs clusterStore) Zrevrank(key, field string) (int64, error) {\n\treturn cs.ZrevrankCtx(context.Background(), key, field)\n}\n\nfunc (cs clusterStore) ZrevrankCtx(ctx context.Context, key, field string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZrevrankCtx(ctx, key, field)\n}\n\nfunc (cs clusterStore) Zscore(key, value string) (int64, error) {\n\treturn cs.ZscoreCtx(context.Background(), key, value)\n}\n\nfunc (cs clusterStore) ZscoreCtx(ctx context.Context, key, value string) (int64, error) {\n\tnode, err := cs.getRedis(key)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\treturn node.ZscoreCtx(ctx, key, value)\n}\n\nfunc (cs clusterStore) getRedis(key string) (*redis.Redis, error) {\n\tval, ok := cs.dispatcher.Get(key)\n\tif !ok {\n\t\treturn nil, ErrNoRedisNode\n\t}\n\n\treturn val.(*redis.Redis), nil\n}\n\n// Path: core/stores/cache/cachenode_test.go\npackage cache\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/alicebob/miniredis/v2\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/zeromicro/go-zero/core/collection\"\n\t\"github.com/zeromicro/go-zero/core/logx\"\n\t\"github.com/zeromicro/go-zero/core/mathx\"\n\t\"github.com/zeromicro/go-zero/core/stat\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis\"\n\t\"github.com/zeromicro/go-zero/core/stores/redis/redistest\"\n\t\"github.com/zeromicro/go-zero/core/syncx\"\n\t\"github.com/zeromicro/go-zero/core/timex\"\n)\n\nvar errTestNotFound = errors.New(\"not found\")\n\nfunc init() {\n\tlogx.Disable()\n\tstat.SetReporter(nil)\n}\n\n\nfunc TestCacheNode_DelCache(t *testing.T) {\n\tt.Run(\"del cache\", func(t *testing.T) {\n\t\tr, err := miniredis.Run()\n\t\tassert.NoError(t, err)\n\t\tdefer r.Close()\n\t\tstore := redis.New(r.Addr(), redis.Cluster())\n\n\t\tcn := cacheNode{\n\t\t\trds:            store,\n\t\t\tr:              rand.New(rand.NewSource(time.Now().UnixNano())),\n\t\t\tlock:           new(sync.Mutex),\n\t\t\tunstableExpiry: mathx.NewUnstable(expiryDeviation),\n\t\t\tstat:           NewStat(\"any\"),\n\t\t\terrNotFound:    errTestNotFound,\n\t\t}\n\t\tassert.Nil(t, cn.Del())\n\t\tassert.Nil(t, cn.Del([]string{}...))\n\t\tassert.Nil(t, cn.Del(make([]string, 0)...))\n\t\tcn.Set(\"first\", \"one\")\n\t\tassert.Nil(t, cn.Del(\"first\"))\n\t\tcn.Set(\"first\", \"one\")\n\t\tcn.Set(\"second\", \"two\")\n\t\tassert.Nil(t, cn.Del(\"first\", \"second\"))\n\t})\n\n\tt.Run(\"del cache with errors\", func(t *testing.T) {\n\t\told := timingWheel.Load()\n\t\tticker := timex.NewFakeTicker()\n\t\ttw, err := collection.NewTimingWheelWithTicker(\n\t\t\ttime.Millisecond, timingWheelSlots, func(key, value any) {\n\t\t\t\tclean(key, value)\n\t\t\t}, ticker)\n\t\ttimingWheel.Store(tw)\n\t\tassert.NoError(t, err)\n\t\tt.Cleanup(func() {\n\t\t\ttimingWheel.Store(old)\n\t\t})\n\n\t\tr, err := miniredis.Run()\n\t\tassert.NoError(t, err)\n\t\tdefer r.Close()\n\t\tr.SetError(\"mock error\")\n\n\t\tnode := NewNode(redis.New(r.Addr(), redis.Cluster()), syncx.NewSingleFlight(),\n\t\t\tNewStat(\"any\"), errTestNotFound)\n\t\tassert.NoError(t, node.Del(\"foo\", \"bar\"))\n\t\tticker.Tick()\n\t\truntime.Gosched()\n\t})\n}\n\nfunc TestCacheNode_DelCacheWithErrors(t *testing.T) {\n\tstore := redistest.CreateRedis(t)\n\tstore.Type = redis.ClusterType\n\n\tcn := cacheNode{\n\t\trds:            store,\n\t\tr:              rand.New(rand.NewSource(time.Now().UnixNano())),\n\t\tlock:           new(sync.Mutex),\n\t\tunstableExpiry: mathx.NewUnstable(expiryDeviation),\n\t\tstat:           NewStat(\"any\"),\n\t\terrNotFound:    errTestNotFound,\n\t}\n\tassert.Nil(t, cn.Del(\"third\", \"fourth\"))\n}\n\nfunc TestCacheNode_InvalidCache(t *testing.T) {\n\ts, err := miniredis.Run()\n\tassert.Nil(t, err)\n\tdefer s.Close()\n\n\tcn := cacheNode{\n\t\trds:            redis.New(s.Addr()),\n\t\tr:              rand.New(rand.NewSource(time.Now().UnixNano())),\n\t\tlock:           new(sync.Mutex),\n\t\tunstableExpiry: mathx.NewUnstable(expiryDeviation),\n\t\tstat:           NewStat(\"any\"),\n\t\terrNotFound:    errTestNotFound,\n\t}\n\ts.Set(\"any\", \"value\")\n\tvar str string\n\tassert.NotNil(t, cn.Get(\"any\", &str))\n\tassert.Equal(t, \"\", str)\n\t_, err = s.Get(\"any\")\n\tassert.Equal(t, miniredis.ErrKeyNotFound, err)\n}\n\nfunc TestCacheNode_SetWithExpire(t *testing.T) {\n\tstore := redistest.CreateRedis(t)\n\n\tcn := cacheNode{\n\t\trds:            store,\n...\n", "needle_token_start": 14337, "needle_token_end": 14956, "code_context_ntokens": 15408, "output": ["```go\nfunc TestCacheNode_DelCache(t *testing.T) {\n\tt.Run(\"del cache\", func(t *testing.T) {\n\t\tr, err := miniredis.Run()\n\t\tassert.NoError(t, err)\n\t\tdefer r.Close()\n\t\tstore := redis.New(r.Addr(), redis.Cluster())\n\n\t\tcn := cacheNode{\n\t\t\trds:            store,\n\t\t\tr:              rand.New(rand.NewSource(time.Now().UnixNano())),\n\t\t\tlock:           new(sync.Mutex),\n\t\t\tunstableExpiry: mathx.NewUnstable(expiryDeviation),\n\t\t\tstat:           NewStat(\"any\"),\n\t\t\terrNotFound:    errTestNotFound,\n\t\t}\n\t\tassert.Nil(t, cn.Del())\n\t\tassert.Nil(t, cn.Del([]string{}...))\n\t\tassert.Nil(t, cn.Del(make([]string, 0)...))\n\t\tcn.Set(\"first\", \"one\")\n\t\tassert.Nil(t, cn.Del(\"first\"))\n\t\tcn.Set(\"first\", \"one\")\n\t\tcn.Set(\"second\", \"two\")\n\t\tassert.Nil(t, cn.Del(\"first\", \"second\"))\n\t})\n\n\tt.Run(\"del cache with errors\", func(t *testing.T) {\n\t\told := timingWheel.Load()\n\t\tticker := timex.NewFakeTicker()\n\t\ttw, err := collection.NewTimingWheelWithTicker(\n\t\t\ttime.Millisecond, timingWheelSlots, func(key, value any) {\n\t\t\t\tclean(key, value)\n\t\t\t}, ticker)\n\t\ttimingWheel.Store(tw)\n\t\tassert.NoError(t, err)\n\t\tt.Cleanup(func() {\n\t\t\ttimingWheel.Store(old)\n\t\t})\n\n\t\tr, err := miniredis.Run()\n\t\tassert.NoError(t, err)\n\t\tdefer r.Close()\n\t\tr.SetError(\"mock error\")\n\n\t\tnode := NewNode(redis.New(r.Addr(), redis.Cluster()), syncx.NewSingleFlight(),\n\t\t\tNewStat(\"any\"), errTestNotFound)\n\t\tassert.NoError(t, node.Del(\"foo\", \"bar\"))\n\t\tticker.Tick()\n\t\truntime.Gosched()\n\t})\n}\n```"]}
{"repo": "schollz/croc", "name": "getStdin", "language": "go", "path": "src/cli/cli.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to read data from the standard input stream and save it to a temporary file.\n2. **Input**: There are no direct parameters passed to the function; it implicitly reads from the standard input.\n3. **Output**: Returns a list containing the path of the created temporary file and an error object if an issue occurs during the process.\n4. **Procedure**: \n   - Creates a temporary file in the current directory with a specific prefix.\n   - Copies all data from the standard input stream to the newly created temporary file.\n   - Closes the file to ensure all data is written properly.\n   - Adds the file path to a list which is then returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\t}\n\t\tif !c.IsSet(\"overwrite\") {\n\t\t\tcrocOptions.Overwrite = rememberedOptions.Overwrite\n\t\t}\n\t\tif !c.IsSet(\"curve\") && rememberedOptions.Curve != \"\" {\n\t\t\tcrocOptions.Curve = rememberedOptions.Curve\n\t\t}\n\t\tif !c.IsSet(\"local\") {\n\t\t\tcrocOptions.OnlyLocal = rememberedOptions.OnlyLocal\n\t\t}\n\t\tif !c.IsSet(\"hash\") {\n\t\t\tcrocOptions.HashAlgorithm = rememberedOptions.HashAlgorithm\n\t\t}\n\t\tif !c.IsSet(\"git\") {\n\t\t\tcrocOptions.GitIgnore = rememberedOptions.GitIgnore\n\t\t}\n\t}\n\n\tvar fnames []string\n\tstat, _ := os.Stdin.Stat()\n\tif ((stat.Mode() & os.ModeCharDevice) == 0) && !c.Bool(\"ignore-stdin\") {\n\t\tfnames, err = getStdin()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tdefer func() {\n\t\t\te := os.Remove(fnames[0])\n\t\t\tif e != nil {\n\t\t\t\tlog.Error(e)\n\t\t\t}\n\t\t}()\n\t} else if c.String(\"text\") != \"\" {\n\t\tfnames, err = makeTempFileWithString(c.String(\"text\"))\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tdefer func() {\n\t\t\te := os.Remove(fnames[0])\n\t\t\tif e != nil {\n\t\t\t\tlog.Error(e)\n\t\t\t}\n\t\t}()\n\n\t} else {\n\t\tfnames = c.Args().Slice()\n\t}\n\tif len(fnames) == 0 {\n\t\treturn errors.New(\"must specify file: croc send [filename(s) or folder]\")\n\t}\n\n\tif len(crocOptions.SharedSecret) == 0 {\n\t\t// generate code phrase\n\t\tcrocOptions.SharedSecret = utils.GetRandomName()\n\t}\n\tminimalFileInfos, emptyFoldersToTransfer, totalNumberFolders, err := croc.GetFilesInfo(fnames, crocOptions.ZipFolder, crocOptions.GitIgnore)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tcr, err := croc.New(crocOptions)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// save the config\n\tsaveConfig(c, crocOptions)\n\n\terr = cr.Send(minimalFileInfos, emptyFoldersToTransfer, totalNumberFolders)\n\n\treturn\n}\n\n\nfunc getStdin() (fnames []string, err error) {\n\tf, err := os.CreateTemp(\".\", \"croc-stdin-\")\n\tif err != nil {\n\t\treturn\n\t}\n\t_, err = io.Copy(f, os.Stdin)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = f.Close()\n\tif err != nil {\n\t\treturn\n\t}\n\tfnames = []string{f.Name()}\n\treturn\n}\n\nfunc makeTempFileWithString(s string) (fnames []string, err error) {\n\tf, err := os.CreateTemp(\".\", \"croc-stdin-\")\n\tif err != nil {\n\t\treturn\n\t}\n\n\t_, err = f.WriteString(s)\n\tif err != nil {\n\t\treturn\n\t}\n\n\terr = f.Close()\n\tif err != nil {\n\t\treturn\n\t}\n\tfnames = []string{f.Name()}\n\treturn\n}\n\nfunc saveConfig(c *cli.Context, crocOptions croc.Options) {\n\tif c.Bool(\"remember\") {\n\t\tconfigFile := getSendConfigFile(true)\n\t\tlog.Debug(\"saving config file\")\n\t\tvar bConfig []byte\n\t\t// if the code wasn't set, don't save it\n\t\tif c.String(\"code\") == \"\" {\n\t\t\tcrocOptions.SharedSecret = \"\"\n\t\t}\n\t\tbConfig, err := json.MarshalIndent(crocOptions, \"\", \"    \")\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\terr = os.WriteFile(configFile, bConfig, 0o644)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debugf(\"wrote %s\", configFile)\n\t}\n}\n\ntype TabComplete struct{}\n\nfunc (t TabComplete) Do(line []rune, pos int) ([][]rune, int) {\n\tvar words = strings.SplitAfter(string(line), \"-\")\n\tvar lastPartialWord = words[len(words)-1]\n\tvar nbCharacter = len(lastPartialWord)\n\tif nbCharacter == 0 {\n\t\t// No completion\n\t\treturn [][]rune{[]rune(\"\")}, 0\n\t}\n\tif len(words) == 1 && nbCharacter == utils.NbPinNumbers {\n\t\t// Check if word is indeed a number\n\t\t_, err := strconv.Atoi(lastPartialWord)\n\t\tif err == nil {\n\t\t\treturn [][]rune{[]rune(\"-\")}, nbCharacter\n\t\t}\n\t}\n\tvar strArray [][]rune\n\tfor _, s := range mnemonicode.WordList {\n\t\tif strings.HasPrefix(s, lastPartialWord) {\n\t\t\tvar completionCandidate = s[nbCharacter:]\n\t\t\tif len(words) <= mnemonicode.WordsRequired(utils.NbBytesWords) {\n\t\t\t\tcompletionCandidate += \"-\"\n\t\t\t}\n\t\t\tstrArray = append(strArray, []rune(completionCandidate))\n\t\t}\n\t}\n\treturn strArray, nbCharacter\n}\n\nfunc receive(c *cli.Context) (err error) {\n\tcomm.Socks5Proxy = c.String(\"socks5\")\n\tcomm.HttpProxy = c.String(\"connect\")\n\tcrocOptions := croc.Options{\n\t\tSharedSecret:  c.String(\"code\"),\n\t\tIsSender:      false,\n\t\tDebug:         c.Bool(\"debug\"),\n\t\tNoPrompt:      c.Bool(\"yes\"),\n\t\tRelayAddress:  c.String(\"relay\"),\n\t\tRelayAddress6: c.String(\"relay6\"),\n\t\tStdout:        c.Bool(\"stdout\"),\n\t\tAsk:           c.Bool(\"ask\"),\n\t\tRelayPassword: determinePass(c),\n\t\tOnlyLocal:     c.Bool(\"local\"),\n\t\tIP:            c.String(\"ip\"),\n\t\tOverwrite:     c.Bool(\"overwrite\"),\n\t\tCurve:         c.String(\"curve\"),\n\t\tTestFlag:      c.Bool(\"testing\"),\n\t}\n\tif crocOptions.RelayAddress != models.DEFAULT_RELAY {\n\t\tcrocOptions.RelayAddress6 = \"\"\n\t} else if crocOptions.RelayAddress6 != models.DEFAULT_RELAY6 {\n\t\tcrocOptions.RelayAddress = \"\"\n\t}\n\n\tswitch c.Args().Len() {\n\tcase 1:\n\t\tcrocOptions.SharedSecret = c.Args().First()\n\tcase 3:\n\t\tfallthrough\n\tcase 4:\n\t\tvar phrase []string\n\t\tphrase = append(phrase, c.Args().First())\n\t\tphrase = append(phrase, c.Args().Tail()...)\n\t\tcrocOptions.SharedSecret = strings.Join(phrase, \"-\")\n\t}\n\n\t// load options here\n\tsetDebugLevel(c)\n\n\tdoRemember := c.Bool(\"remember\")\n\tconfigFile, err := getReceiveConfigFile(doRemember)\n\tif err != nil && doRemember {\n\t\treturn\n\t}\n\tb, errOpen := os.ReadFile(configFile)\n\tif errOpen == nil && !doRemember {\n\t\tvar rememberedOptions croc.Options\n\t\terr = json.Unmarshal(b, &rememberedOptions)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\t// update anything that isn't explicitly Globally set\n\t\tif !c.IsSet(\"relay\") && rememberedOptions.RelayAddress != \"\" {\n\t\t\tcrocOptions.RelayAddress = rememberedOptions.RelayAddress\n\t\t}\n\t\tif !c.IsSet(\"yes\") {\n\t\t\tcrocOptions.NoPrompt = rememberedOptions.NoPrompt\n\t\t}\n\t\tif crocOptions.SharedSecret == \"\" {\n\t\t\tcrocOptions.SharedSecret = rememberedOptions.SharedSecret\n\t\t}\n\t\tif !c.IsSet(\"pass\") && rememberedOptions.RelayPassword != \"\" {\n\t\t\tcrocOptions.RelayPassword = rememberedOptions.RelayPassword\n\t\t}\n\t\tif !c.IsSet(\"relay6\") && rememberedOptions.RelayAddress6 != \"\" {\n\t\t\tcrocOptions.RelayAddress6 = rememberedOptions.RelayAddress6\n\t\t}\n\t\tif !c.IsSet(\"overwrite\") {\n\t\t\tcrocOptions.Overwrite = rememberedOptions.Overwrite\n\t\t}\n\t\tif !c.IsSet(\"curve\") && rememberedOptions.Curve != \"\" {\n\t\t\tcrocOptions.Curve = rememberedOptions.Curve\n\t\t}\n\t\tif !c.IsSet(\"local\") {\n\t\t\tcrocOptions.OnlyLocal = rememberedOptions.OnlyLocal\n\t\t}\n\t}\n\n\tif crocOptions.SharedSecret == \"\" {\n\t\tl, err := readline.NewEx(&readline.Config{\n\t\t\tPrompt:       \"Enter receive code: \",\n\t\t\tAutoComplete: TabComplete{},\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tcrocOptions.SharedSecret, err = l.Readline()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif c.String(\"out\") != \"\" {\n\t\tif err = os.Chdir(c.String(\"out\")); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tcr, err := croc.New(crocOptions)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// save the config\n\tif doRemember {\n\t\tlog.Debug(\"saving config file\")\n\t\tvar bConfig []byte\n\t\tbConfig, err = json.MarshalIndent(crocOptions, \"\", \"    \")\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\terr = os.WriteFile(configFile, bConfig, 0o644)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debugf(\"wrote %s\", configFile)\n\t}\n\n\terr = cr.Receive()\n\treturn\n}\n\nfunc relay(c *cli.Context) (err error) {\n\tlog.Infof(\"starting croc relay version %v\", Version)\n\tdebugString := \"info\"\n\tif c.Bool(\"debug\") {\n\t\tdebugString = \"debug\"\n\t}\n\thost := c.String(\"host\")\n\tports := strings.Split(c.String(\"ports\"), \",\")\n\ttcpPorts := strings.Join(ports[1:], \",\")\n\tfor i, port := range ports {\n\t\tif i == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tgo func(portStr string) {\n\t\t\terr := tcp.Run(debugString, host, portStr, determinePass(c))\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t}(port)\n\t}\n\treturn tcp.Run(debugString, host, ports[0], determinePass(c), tcpPorts)\n}\n\n// Path: src/comm/comm_test.go\npackage comm\n\nimport (\n\t\"crypto/rand\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestComm(t *testing.T) {\n\ttoken := make([]byte, 3000)\n\tif _, err := rand.Read(token); err != nil {\n\t\tt.Error(err)\n\t}\n\n\tport := \"8001\"\n\tgo func() {\n\t\tlog.Debugf(\"starting TCP server on \" + port)\n\t\tserver, err := net.Listen(\"tcp\", \"0.0.0.0:\"+port)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t}\n\t\tdefer server.Close()\n\t\t// spawn a new goroutine whenever a client connects\n\t\tfor {\n\t\t\tconnection, err := server.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\t\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\t\tgo func(_ string, connection net.Conn) {\n\t\t\t\tc := New(connection)\n\t\t\t\terr = c.Send([]byte(\"hello, world\"))\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tdata, err := c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte(\"hello, computer\"), data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte{'\\x00'}, data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, token, data)\n\t\t\t}(port, connection)\n\t\t}\n\t}()\n\n\ttime.Sleep(300 * time.Millisecond)\n\ta, err := NewConnection(\"127.0.0.1:\"+port, 10*time.Minute)\n\tassert.Nil(t, err)\n\tdata, err := a.Receive()\n\tassert.Equal(t, []byte(\"hello, world\"), data)\n\tassert.Nil(t, err)\n\tassert.Nil(t, a.Send([]byte(\"hello, computer\")))\n\tassert.Nil(t, a.Send([]byte{'\\x00'}))\n\n\tassert.Nil(t, a.Send(token))\n\t_ = a.Connection()\n\ta.Close()\n\tassert.NotNil(t, a.Send(token))\n\t_, err = a.Write(token)\n\tassert.NotNil(t, err)\n}\n\n// Path: src/compress/compress_test.go\npackage compress\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar fable = []byte(`The Frog and the Crocodile\nOnce, there was a frog who lived in the middle of a swamp. His entire family had lived in that swamp for generations, but this particular frog decided that he had had quite enough wetness to last him a lifetime. He decided that he was going to find a dry place to live instead.\n\nThe only thing that separated him from dry land was a swampy, muddy, swiftly flowing river. But the river was home to all sorts of slippery, slittering snakes that loved nothing better than a good, plump frog for dinner, so Frog didn't dare try to swim across.\n\nSo for many days, the frog stayed put, hopping along the bank, trying to think of a way to get across.\n\nThe snakes hissed and jeered at him, daring him to come closer, but he refused. Occasionally they would slither closer, jaws open to attack, but the frog always leaped out of the way. But no matter how far upstream he searched or how far downstream, the frog wasn't able to find a way across the water.\n\nHe had felt certain that there would be a bridge, or a place where the banks came together, yet all he found was more reeds and water. After a while, even the snakes stopped teasing him and went off in search of easier prey.\n\nThe frog sighed in frustration and sat to sulk in the rushes. Suddenly, he spotted two big eyes staring at him from the water. The giant log-shaped animal opened its mouth and asked him, \"What are you doing, Frog? Surely there are enough flies right there for a meal.\"\n\nThe frog croaked in surprise and leaped away from the crocodile. That creature could swallow him whole in a moment without thinking about it! Once he was a satisfied that he was a safe distance away, he answered. \"I'm tired of living in swampy waters, and I want to travel to the other side of the river. But if I swim across, the snakes will eat me.\"\n\nThe crocodile harrumphed in agreement and sat, thinking, for a while. \"Well, if you're afraid of the snakes, I could give you a ride across,\" he suggested.\n\n\"Oh no, I don't think so,\" Frog answered quickly. \"You'd eat me on the way over, or go underwater so the snakes could get me!\"\n\n\"Now why would I let the snakes get you? I think they're a terrible nuisance with all their hissing and slithering! The river would be much better off without them altogether! Anyway, if you're so worried that I might eat you, you can ride on my tail.\"\n\nThe frog considered his offer. He did want to get to dry ground very badly, and there didn't seem to be any other way across the river. He looked at the crocodile from his short, squat buggy eyes and wondered about the crocodile's motives. But if he rode on the tail, the croc couldn't eat him anyway. And he was right about the snakes--no self-respecting crocodile would give a meal to the snakes.\n\n\"Okay, it sounds like a good plan to me. Turn around so I can hop on your tail.\"\n\nThe crocodile flopped his tail into the marshy mud and let the frog climb on, then he waddled out to the river. But he couldn't stick his tail into the water as a rudder because the frog was on it -- and if he put his tail in the water, the snakes would eat the frog. They clumsily floated downstream for a ways, until the crocodile said, \"Hop onto my back so I can steer straight with my tail.\" The frog moved, and the journey smoothed out.\n\nFrom where he was sitting, the frog couldn't see much except the back of Crocodile's head. \"Why don't you hop up on my head so you can see everything around us?\" Crocodile invited. `)\n\nfunc BenchmarkCompressLevelMinusTwo(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(fable, -2)\n\t}\n}\n\nfunc BenchmarkCompressLevelNine(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(fable, 9)\n\t}\n}\n\nfunc BenchmarkCompressLevelMinusTwoBinary(b *testing.B) {\n\tdata := make([]byte, 1000000)\n\tif _, err := rand.Read(data); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(data, -2)\n\t}\n}\n\nfunc BenchmarkCompressLevelNineBinary(b *testing.B) {\n\tdata := make([]byte, 1000000)\n\tif _, err := rand.Read(data); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(data, 9)\n\t}\n}\n\nfunc TestCompress(t *testing.T) {\n\tcompressedB := CompressWithOption(fable, 9)\n\tdataRateSavings := 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))\n\tfmt.Printf(\"Level 9: %2.0f%% percent space savings\\n\", dataRateSavings)\n\tassert.True(t, len(compressedB) < len(fable))\n\tassert.Equal(t, fable, Decompress(compressedB))\n\n\tcompressedB = CompressWithOption(fable, -2)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))\n\tfmt.Printf(\"Level -2: %2.0f%% percent space savings\\n\", dataRateSavings)\n\tassert.True(t, len(compressedB) < len(fable))\n\n\tcompressedB = Compress(fable)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))\n\tfmt.Printf(\"Level -2: %2.0f%% percent space savings\\n\", dataRateSavings)\n\tassert.True(t, len(compressedB) < len(fable))\n\n\tdata := make([]byte, 4096)\n\tif _, err := rand.Read(data); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcompressedB = CompressWithOption(data, -2)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(data)))\n\tfmt.Printf(\"random, Level -2: %2.0f%% percent space savings\\n\", dataRateSavings)\n\n\tif _, err := rand.Read(data); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcompressedB = CompressWithOption(data, 9)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(data)))\n\n\tfmt.Printf(\"random, Level 9: %2.0f%% percent space savings\\n\", dataRateSavings)\n\n}\n\n// Path: src/croc/croc_test.go\npackage croc\n\nimport (\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/tcp\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc init() {\n\tlog.SetLevel(\"trace\")\n\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8281\", \"pass123\", \"8282,8283,8284,8285\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8282\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8284\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8285\", \"pass123\")\n\ttime.Sleep(1 * time.Second)\n}\n\nfunc TestCrocReadme(t *testing.T) {\n\tdefer os.Remove(\"README.md\")\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../README.md\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocEmptyFolder(t *testing.T) {\n\tpathName := \"../../testEmpty\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./testEmpty\")\n\tos.MkdirAll(pathName, 0o755)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocSymlink(t *testing.T) {\n\tpathName := \"../link-in-folder\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./link-in-folder\")\n\tos.MkdirAll(pathName, 0o755)\n\tos.Symlink(\"../../README.md\", filepath.Join(pathName, \"README.link\"))\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr = receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\ts, err := filepath.EvalSymlinks(path.Join(pathName, \"README.link\"))\n\tif s != \"../../README.md\" && s != \"..\\\\..\\\\README.md\" {\n\t\tlog.Debug(s)\n\t\tt.Errorf(\"symlink failed to transfer in folder\")\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"symlink transfer failed: %s\", err.Error())\n\t}\n}\nfunc testCrocIgnoreGit(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\".gitignore\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\ttime.Sleep(1 * time.Second)\n\tfile, err := os.Create(\".gitignore\")\n\tif err != nil {\n\t\tlog.Errorf(\"error creating file\")\n\t}\n\t_, err = file.WriteString(\"LICENSE\")\n\tif err != nil {\n\t\tlog.Errorf(\"error writing to file\")\n\t}\n\ttime.Sleep(1 * time.Second)\n\t// due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo.\n\tfilesInfo, _, _, errGet := GetFilesInfo([]string{\"../../LICENSE\", \".gitignore\", \"croc.go\"}, false, true)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\tfor _, file := range filesInfo {\n\t\tif strings.Contains(file.Name, \"LICENSE\") {\n\t\t\tt.Errorf(\"test failed, should ignore LICENSE\")\n\t\t}\n\t}\n}\n\nfunc TestCrocLocal(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\"LICENSE\")\n\tdefer os.Remove(\"touched\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPorts:    []string{\"8181\", \"8182\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttime.Sleep(1 * time.Second)\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\tos.Create(\"touched\")\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../LICENSE\", \"touched\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocError(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err = tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err = tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\n\tDebug(false)\n\tlog.SetLevel(\"warn\")\n\tsender, _ := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc2\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"doesntexistok.com:8381\",\n\t\tRelayPorts:    []string{\"8381\", \"8382\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false, false)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\tlog.Debug(err)\n\tassert.NotNil(t, err)\n}\n\nfunc TestCleanUp(t *testing.T) {\n\t// windows allows files to be deleted only if they\n\t// are not open by another program so the remove actions\n\t// from the above tests will not always do a good clean up\n\t// This \"test\" will make sure\n\toperatingSystem := runtime.GOOS\n\tlog.Debugf(\"The operating system is %s\", operatingSystem)\n\tif operatingSystem == \"windows\" {\n\t\ttime.Sleep(1 * time.Second)\n\t\tlog.Debug(\"Full cleanup\")\n\t\tvar err error\n\n\t\tfor _, file := range []string{\"README.md\", \"./README.md\"} {\n\t\t\terr = os.Remove(file)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", file)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", file)\n\t\t\t}\n\t\t}\n\t\tfor _, folder := range []string{\"./testEmpty\", \"./link-in-folder\"} {\n\t\t\terr = os.RemoveAll(folder)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", folder)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", folder)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Path: src/crypt/crypt_test.go\npackage crypt\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkEncrypt(b *testing.B) {\n\tbob, _, _ := New([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncrypt([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecrypt(b *testing.B) {\n\tkey, _, _ := New([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := Encrypt(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecrypt(enc, key)\n\t}\n}\n\nfunc BenchmarkEncryptChaCha(b *testing.B) {\n\tbob, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncryptChaCha([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecryptChaCha(b *testing.B) {\n\tkey, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := EncryptChaCha(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecryptChaCha(enc, key)\n\t}\n}\n\nfunc TestEncryption(t *testing.T) {\n\tkey, salt, err := New([]byte(\"password\"), nil)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := Encrypt(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := Decrypt(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := New([]byte(\"password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = New([]byte(\"wrong password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = Decrypt([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = New([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\nfunc TestEncryptionChaCha(t *testing.T) {\n\tkey, salt, err := NewArgon2([]byte(\"password\"), nil)\n\tfmt.Printf(\"key: %x\\n\", key)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := EncryptChaCha(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := DecryptChaCha(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := NewArgon2([]byte(\"password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = NewArgon2([]byte(\"wrong password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = DecryptChaCha([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = NewArgon2([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\n// Path: src/install/updateversion.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n)\n\nfunc main() {\n\terr := run()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t}\n}\n\nfunc run() (err error) {\n\tversionNew := \"v\" + os.Getenv(\"VERSION\")\n\tversionHash, err := exec.Command(\"git\", \"rev-parse\", \"--short\", \"HEAD\").Output()\n\tif err != nil {\n\t\treturn\n\t}\n\tversionHashNew := strings.TrimSpace(string(versionHash))\n\tfmt.Println(versionNew)\n\tfmt.Println(versionHashNew)\n\n\terr = replaceInFile(\"src/cli/cli.go\", `Version = \"`, `\"`, versionNew+\"-\"+versionHashNew)\n\tif err == nil {\n\t\tfmt.Printf(\"updated cli.go to version %s\\n\", versionNew)\n\t}\n\terr = replaceInFile(\"README.md\", `version-`, `-b`, strings.Split(versionNew, \"-\")[0])\n\tif err == nil {\n\t\tfmt.Printf(\"updated README to version %s\\n\", strings.Split(versionNew, \"-\")[0])\n\t}\n\n\terr = replaceInFile(\"src/install/default.txt\", `croc_version=\"`, `\"`, strings.Split(versionNew, \"-\")[0][1:])\n\tif err == nil {\n\t\tfmt.Printf(\"updated default.txt to version %s\\n\", strings.Split(versionNew, \"-\")[0][1:])\n\t}\n\n\treturn\n}\n\nfunc replaceInFile(fname, start, end, replacement string) (err error) {\n\tb, err := os.ReadFile(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\toldVersion := GetStringInBetween(string(b), start, end)\n\tif oldVersion == \"\" {\n\t\terr = fmt.Errorf(\"nothing\")\n\t\treturn\n\t}\n\tnewF := strings.Replace(\n\t\tstring(b),\n\t\tfmt.Sprintf(\"%s%s%s\", start, oldVersion, end),\n\t\tfmt.Sprintf(\"%s%s%s\", start, replacement, end),\n\t\t1,\n\t)\n\terr = os.WriteFile(fname, []byte(newF), 0o644)\n\treturn\n}\n\n// GetStringInBetween Returns empty string if no start string found\nfunc GetStringInBetween(str string, start string, end string) (result string) {\n\ts := strings.Index(str, start)\n\tif s == -1 {\n\t\treturn\n\t}\n\ts += len(start)\n\te := strings.Index(str[s:], end)\n\tif e == -1 {\n\t\treturn\n\t}\n\te += s\n\treturn str[s:e]\n}\n\n// Path: src/message/message_test.go\npackage message\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar TypeMessage Type = \"message\"\n\nfunc TestMessage(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tassert.Nil(t, err)\n\tfmt.Println(string(salt))\n\tb, err := Encode(e, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(e, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n\t_, err = Decode([]byte(\"not pass\"), b)\n\tassert.NotNil(t, err)\n\t_, err = Encode([]byte(\"0\"), m)\n\tassert.NotNil(t, err)\n}\n\nfunc TestMessageNoPass(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\tb, err := Encode(nil, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(nil, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n}\n\nfunc TestSend(t *testing.T) {\n\ttoken := make([]byte, 40000000)\n\trand.Read(token)\n\n\tport := \"8801\"\n\tgo func() {\n\t\tlog.Debugf(\"starting TCP server on \" + port)\n\t\tserver, err := net.Listen(\"tcp\", \"0.0.0.0:\"+port)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t}\n\t\tdefer server.Close()\n\t\t// spawn a new goroutine whenever a client connects\n\t\tfor {\n\t\t\tconnection, err := server.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\t\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\t\tgo func(_ string, connection net.Conn) {\n\t\t\t\tc := comm.New(connection)\n\t\t\t\terr = c.Send([]byte(\"hello, world\"))\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tdata, err := c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte(\"hello, computer\"), data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte{'\\x00'}, data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, token, data)\n\t\t\t}(port, connection)\n\t\t}\n\t}()\n\n\ttime.Sleep(300 * time.Millisecond)\n\ta, err := comm.NewConnection(\"127.0.0.1:\"+port, 10*time.Minute)\n\tassert.Nil(t, err)\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tlog.Debug(salt)\n\tassert.Nil(t, err)\n\n\tassert.Nil(t, Send(a, e, m))\n}\n\n// Path: src/models/constants.go\npackage models\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\n\t\"github.com/schollz/croc/v9/src/utils\"\n)\n\n// TCP_BUFFER_SIZE is the maximum packet size\nconst TCP_BUFFER_SIZE = 1024 * 64\n\n// DEFAULT_RELAY is the default relay used (can be set using --relay)\nvar (\n\tDEFAULT_RELAY      = \"croc.schollz.com\"\n\tDEFAULT_RELAY6     = \"croc6.schollz.com\"\n\tDEFAULT_PORT       = \"9009\"\n\tDEFAULT_PASSPHRASE = \"pass123\"\n\tINTERNAL_DNS       = false\n)\n\n// publicDNS are servers to be queried if a local lookup fails\nvar publicDNS = []string{\n\t\"1.0.0.1\",                // Cloudflare\n\t\"1.1.1.1\",                // Cloudflare\n\t\"[2606:4700:4700::1111]\", // Cloudflare\n\t\"[2606:4700:4700::1001]\", // Cloudflare\n\t\"8.8.4.4\",                // Google\n\t\"8.8.8.8\",                // Google\n\t\"[2001:4860:4860::8844]\", // Google\n\t\"[2001:4860:4860::8888]\", // Google\n\t\"9.9.9.9\",                // Quad9\n\t\"149.112.112.112\",        // Quad9\n\t\"[2620:fe::fe]\",          // Quad9\n\t\"[2620:fe::fe:9]\",        // Quad9\n\t\"8.26.56.26\",             // Comodo\n\t\"8.20.247.20\",            // Comodo\n\t\"208.67.220.220\",         // Cisco OpenDNS\n\t\"208.67.222.222\",         // Cisco OpenDNS\n\t\"[2620:119:35::35]\",      // Cisco OpenDNS\n\t\"[2620:119:53::53]\",      // Cisco OpenDNS\n}\n\nfunc getConfigFile(requireValidPath bool) (fname string, err error) {\n\tconfigFile, err := utils.GetConfigDir(requireValidPath)\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = path.Join(configFile, \"internal-dns\")\n\treturn\n}\n\nfunc init() {\n\tdoRemember := false\n\tfor _, flag := range os.Args {\n\t\tif flag == \"--internal-dns\" {\n\t\t\tINTERNAL_DNS = true\n\t\t\tbreak\n\t\t}\n\t\tif flag == \"--remember\" {\n\t\t\tdoRemember = true\n\t\t}\n\t}\n\tif doRemember {\n\t\t// save in config file\n\t\tfname, err := getConfigFile(true)\n\t\tif err == nil {\n\t\t\tf, _ := os.Create(fname)\n\t\t\tf.Close()\n\t\t}\n\t}\n\tif !INTERNAL_DNS {\n\t\tfname, err := getConfigFile(false)\n\t\tif err == nil {\n\t\t\tINTERNAL_DNS = utils.Exists(fname)\n\t\t}\n\t}\n\tvar err error\n\tvar addr string\n\taddr, err = lookup(DEFAULT_RELAY)\n\tif err == nil {\n\t\tDEFAULT_RELAY = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY = \"\"\n\t}\n\taddr, err = lookup(DEFAULT_RELAY6)\n\tif err == nil {\n\t\tDEFAULT_RELAY6 = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY6 = \"\"\n\t}\n}\n\n// Resolve a hostname to an IP address using DNS.\nfunc lookup(address string) (ipaddress string, err error) {\n\tif !INTERNAL_DNS {\n\t\treturn localLookupIP(address)\n\t}\n\ttype Result struct {\n\t\ts   string\n\t\terr error\n\t}\n\tresult := make(chan Result, len(publicDNS))\n\tfor _, dns := range publicDNS {\n\t\tgo func(dns string) {\n\t\t\tvar r Result\n\t\t\tr.s, r.err = remoteLookupIP(address, dns)\n\t\t\tresult <- r\n\t\t}(dns)\n\t}\n\tfor i := 0; i < len(publicDNS); i++ {\n\t\tipaddress = (<-result).s\n\t\tif ipaddress != \"\" {\n\t\t\treturn\n\t\t}\n\t}\n\terr = fmt.Errorf(\"failed to resolve %s: all DNS servers exhausted\", address)\n\treturn\n}\n\n// localLookupIP returns a host's IP address based on the local resolver.\nfunc localLookupIP(address string) (ipaddress string, err error) {\n\tip, err := net.LookupHost(address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// remoteLookupIP returns a host's IP address based on a remote DNS server.\nfunc remoteLookupIP(address, dns string) (ipaddress string, err error) {\n\tr := &net.Resolver{\n\t\tPreferGo: true,\n\t\tDial: func(ctx context.Context, network, _ string) (net.Conn, error) {\n\t\t\td := new(net.Dialer)\n\t\t\treturn d.DialContext(ctx, network, dns+\":53\")\n\t\t},\n\t}\n\tip, err := r.LookupHost(context.Background(), address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// Path: src/tcp/tcp_test.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkConnection(b *testing.B) {\n\tlog.SetLevel(\"trace\")\n\tgo Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\", \"8284\")\n\ttime.Sleep(100 * time.Millisecond)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tc, _, _, _ := ConnectToTCPServer(\"127.0.0.1:8283\", \"pass123\", fmt.Sprintf(\"testroom%d\", i), 1*time.Minute)\n\t\tc.Close()\n\t}\n}\n\nfunc TestTCP(t *testing.T) {\n\tlog.SetLevel(\"error\")\n\ttimeToRoomDeletion = 100 * time.Millisecond\n\tgo Run(\"debug\", \"127.0.0.1\", \"8381\", \"pass123\", \"8382\")\n\ttime.Sleep(timeToRoomDeletion)\n\terr := PingServer(\"127.0.0.1:8381\")\n\tassert.Nil(t, err)\n\terr = PingServer(\"127.0.0.1:8333\")\n\tassert.NotNil(t, err)\n\n\ttime.Sleep(timeToRoomDeletion)\n\tc1, banner, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Minute)\n\tassert.Equal(t, banner, \"8382\")\n\tassert.Nil(t, err)\n\tc2, _, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.Nil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.NotNil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Nanosecond)\n\tassert.NotNil(t, err)\n\n\t// try sending data\n\tassert.Nil(t, c1.Send([]byte(\"hello, c2\")))\n\tvar data []byte\n\tfor {\n\t\tdata, err = c2.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c2\"), data)\n\n\tassert.Nil(t, c2.Send([]byte(\"hello, c1\")))\n\tfor {\n\t\tdata, err = c1.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c1\"), data)\n\n\tc1.Close()\n\ttime.Sleep(300 * time.Millisecond)\n}\n\n...\n", "needle_token_start": 675, "needle_token_end": 793, "code_context_ntokens": 15122, "output": ["```go\nfunc getStdin() (fnames []string, err error) {\n\tf, err := os.CreateTemp(\".\", \"croc-stdin-\")\n\tif err != nil {\n\t\treturn\n\t}\n\t_, err = io.Copy(f, os.Stdin)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = f.Close()\n\tif err != nil {\n\t\treturn\n\t}\n\tfnames = []string{f.Name()}\n\treturn\n}\n```"]}
{"repo": "schollz/croc", "name": "TestCrocSymlink", "language": "go", "path": "src/croc/croc_test.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To verify the correct transfer of symbolic links between a sender and a receiver in a controlled test environment.\n2. **Input**: A symbolic link created in a test directory pointing to a README file.\n3. **Output**: The test outputs whether the symbolic link was successfully transferred and correctly points to the intended target file after the transfer.\n4. **Procedure**: \n   - A test directory and a symbolic link within it are created.\n   - Setup configurations for both sender and receiver are initialized.\n   - The sender gathers file information and attempts to send it.\n   - The receiver attempts to receive the file information.\n   - After the transfer, the test checks if the symbolic link still points to the original target file, verifying the integrity of the link transfer.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/compress/compress_test.go\npackage compress\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar fable = []byte(`The Frog and the Crocodile\nOnce, there was a frog who lived in the middle of a swamp. His entire family had lived in that swamp for generations, but this particular frog decided that he had had quite enough wetness to last him a lifetime. He decided that he was going to find a dry place to live instead.\n\nThe only thing that separated him from dry land was a swampy, muddy, swiftly flowing river. But the river was home to all sorts of slippery, slittering snakes that loved nothing better than a good, plump frog for dinner, so Frog didn't dare try to swim across.\n\nSo for many days, the frog stayed put, hopping along the bank, trying to think of a way to get across.\n\n...\n// Path: src/croc/croc_test.go\npackage croc\n\nimport (\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/tcp\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc init() {\n\tlog.SetLevel(\"trace\")\n\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8281\", \"pass123\", \"8282,8283,8284,8285\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8282\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8284\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8285\", \"pass123\")\n\ttime.Sleep(1 * time.Second)\n}\n\nfunc TestCrocReadme(t *testing.T) {\n\tdefer os.Remove(\"README.md\")\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../README.md\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocEmptyFolder(t *testing.T) {\n\tpathName := \"../../testEmpty\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./testEmpty\")\n\tos.MkdirAll(pathName, 0o755)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\n\nfunc TestCrocSymlink(t *testing.T) {\n\tpathName := \"../link-in-folder\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./link-in-folder\")\n\tos.MkdirAll(pathName, 0o755)\n\tos.Symlink(\"../../README.md\", filepath.Join(pathName, \"README.link\"))\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr = receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\ts, err := filepath.EvalSymlinks(path.Join(pathName, \"README.link\"))\n\tif s != \"../../README.md\" && s != \"..\\\\..\\\\README.md\" {\n\t\tlog.Debug(s)\n\t\tt.Errorf(\"symlink failed to transfer in folder\")\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"symlink transfer failed: %s\", err.Error())\n\t}\n}\nfunc testCrocIgnoreGit(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\".gitignore\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\ttime.Sleep(1 * time.Second)\n\tfile, err := os.Create(\".gitignore\")\n\tif err != nil {\n\t\tlog.Errorf(\"error creating file\")\n\t}\n\t_, err = file.WriteString(\"LICENSE\")\n\tif err != nil {\n\t\tlog.Errorf(\"error writing to file\")\n\t}\n\ttime.Sleep(1 * time.Second)\n\t// due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo.\n\tfilesInfo, _, _, errGet := GetFilesInfo([]string{\"../../LICENSE\", \".gitignore\", \"croc.go\"}, false, true)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\tfor _, file := range filesInfo {\n\t\tif strings.Contains(file.Name, \"LICENSE\") {\n\t\t\tt.Errorf(\"test failed, should ignore LICENSE\")\n\t\t}\n\t}\n}\n\nfunc TestCrocLocal(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\"LICENSE\")\n\tdefer os.Remove(\"touched\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPorts:    []string{\"8181\", \"8182\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttime.Sleep(1 * time.Second)\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\tos.Create(\"touched\")\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../LICENSE\", \"touched\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocError(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err = tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err = tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\n\tDebug(false)\n\tlog.SetLevel(\"warn\")\n\tsender, _ := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc2\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"doesntexistok.com:8381\",\n\t\tRelayPorts:    []string{\"8381\", \"8382\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false, false)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\tlog.Debug(err)\n\tassert.NotNil(t, err)\n}\n\nfunc TestCleanUp(t *testing.T) {\n\t// windows allows files to be deleted only if they\n\t// are not open by another program so the remove actions\n\t// from the above tests will not always do a good clean up\n\t// This \"test\" will make sure\n\toperatingSystem := runtime.GOOS\n\tlog.Debugf(\"The operating system is %s\", operatingSystem)\n\tif operatingSystem == \"windows\" {\n\t\ttime.Sleep(1 * time.Second)\n\t\tlog.Debug(\"Full cleanup\")\n\t\tvar err error\n\n\t\tfor _, file := range []string{\"README.md\", \"./README.md\"} {\n\t\t\terr = os.Remove(file)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", file)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", file)\n\t\t\t}\n\t\t}\n\t\tfor _, folder := range []string{\"./testEmpty\", \"./link-in-folder\"} {\n\t\t\terr = os.RemoveAll(folder)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", folder)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", folder)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Path: src/crypt/crypt_test.go\npackage crypt\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkEncrypt(b *testing.B) {\n\tbob, _, _ := New([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncrypt([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecrypt(b *testing.B) {\n\tkey, _, _ := New([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := Encrypt(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecrypt(enc, key)\n\t}\n}\n\nfunc BenchmarkEncryptChaCha(b *testing.B) {\n\tbob, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncryptChaCha([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecryptChaCha(b *testing.B) {\n\tkey, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := EncryptChaCha(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecryptChaCha(enc, key)\n\t}\n}\n\nfunc TestEncryption(t *testing.T) {\n\tkey, salt, err := New([]byte(\"password\"), nil)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := Encrypt(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := Decrypt(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := New([]byte(\"password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = New([]byte(\"wrong password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = Decrypt([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = New([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\nfunc TestEncryptionChaCha(t *testing.T) {\n\tkey, salt, err := NewArgon2([]byte(\"password\"), nil)\n\tfmt.Printf(\"key: %x\\n\", key)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := EncryptChaCha(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := DecryptChaCha(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := NewArgon2([]byte(\"password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = NewArgon2([]byte(\"wrong password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = DecryptChaCha([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = NewArgon2([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\n// Path: src/install/updateversion.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n)\n\nfunc main() {\n\terr := run()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t}\n}\n\nfunc run() (err error) {\n\tversionNew := \"v\" + os.Getenv(\"VERSION\")\n\tversionHash, err := exec.Command(\"git\", \"rev-parse\", \"--short\", \"HEAD\").Output()\n\tif err != nil {\n\t\treturn\n\t}\n\tversionHashNew := strings.TrimSpace(string(versionHash))\n\tfmt.Println(versionNew)\n\tfmt.Println(versionHashNew)\n\n\terr = replaceInFile(\"src/cli/cli.go\", `Version = \"`, `\"`, versionNew+\"-\"+versionHashNew)\n\tif err == nil {\n\t\tfmt.Printf(\"updated cli.go to version %s\\n\", versionNew)\n\t}\n\terr = replaceInFile(\"README.md\", `version-`, `-b`, strings.Split(versionNew, \"-\")[0])\n\tif err == nil {\n\t\tfmt.Printf(\"updated README to version %s\\n\", strings.Split(versionNew, \"-\")[0])\n\t}\n\n\terr = replaceInFile(\"src/install/default.txt\", `croc_version=\"`, `\"`, strings.Split(versionNew, \"-\")[0][1:])\n\tif err == nil {\n\t\tfmt.Printf(\"updated default.txt to version %s\\n\", strings.Split(versionNew, \"-\")[0][1:])\n\t}\n\n\treturn\n}\n\nfunc replaceInFile(fname, start, end, replacement string) (err error) {\n\tb, err := os.ReadFile(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\toldVersion := GetStringInBetween(string(b), start, end)\n\tif oldVersion == \"\" {\n\t\terr = fmt.Errorf(\"nothing\")\n\t\treturn\n\t}\n\tnewF := strings.Replace(\n\t\tstring(b),\n\t\tfmt.Sprintf(\"%s%s%s\", start, oldVersion, end),\n\t\tfmt.Sprintf(\"%s%s%s\", start, replacement, end),\n\t\t1,\n\t)\n\terr = os.WriteFile(fname, []byte(newF), 0o644)\n\treturn\n}\n\n// GetStringInBetween Returns empty string if no start string found\nfunc GetStringInBetween(str string, start string, end string) (result string) {\n\ts := strings.Index(str, start)\n\tif s == -1 {\n\t\treturn\n\t}\n\ts += len(start)\n\te := strings.Index(str[s:], end)\n\tif e == -1 {\n\t\treturn\n\t}\n\te += s\n\treturn str[s:e]\n}\n\n// Path: src/message/message_test.go\npackage message\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar TypeMessage Type = \"message\"\n\nfunc TestMessage(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tassert.Nil(t, err)\n\tfmt.Println(string(salt))\n\tb, err := Encode(e, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(e, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n\t_, err = Decode([]byte(\"not pass\"), b)\n\tassert.NotNil(t, err)\n\t_, err = Encode([]byte(\"0\"), m)\n\tassert.NotNil(t, err)\n}\n\nfunc TestMessageNoPass(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\tb, err := Encode(nil, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(nil, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n}\n\nfunc TestSend(t *testing.T) {\n\ttoken := make([]byte, 40000000)\n\trand.Read(token)\n\n\tport := \"8801\"\n\tgo func() {\n\t\tlog.Debugf(\"starting TCP server on \" + port)\n\t\tserver, err := net.Listen(\"tcp\", \"0.0.0.0:\"+port)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t}\n\t\tdefer server.Close()\n\t\t// spawn a new goroutine whenever a client connects\n\t\tfor {\n\t\t\tconnection, err := server.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\t\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\t\tgo func(_ string, connection net.Conn) {\n\t\t\t\tc := comm.New(connection)\n\t\t\t\terr = c.Send([]byte(\"hello, world\"))\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tdata, err := c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte(\"hello, computer\"), data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte{'\\x00'}, data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, token, data)\n\t\t\t}(port, connection)\n\t\t}\n\t}()\n\n\ttime.Sleep(300 * time.Millisecond)\n\ta, err := comm.NewConnection(\"127.0.0.1:\"+port, 10*time.Minute)\n\tassert.Nil(t, err)\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tlog.Debug(salt)\n\tassert.Nil(t, err)\n\n\tassert.Nil(t, Send(a, e, m))\n}\n\n// Path: src/models/constants.go\npackage models\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\n\t\"github.com/schollz/croc/v9/src/utils\"\n)\n\n// TCP_BUFFER_SIZE is the maximum packet size\nconst TCP_BUFFER_SIZE = 1024 * 64\n\n// DEFAULT_RELAY is the default relay used (can be set using --relay)\nvar (\n\tDEFAULT_RELAY      = \"croc.schollz.com\"\n\tDEFAULT_RELAY6     = \"croc6.schollz.com\"\n\tDEFAULT_PORT       = \"9009\"\n\tDEFAULT_PASSPHRASE = \"pass123\"\n\tINTERNAL_DNS       = false\n)\n\n// publicDNS are servers to be queried if a local lookup fails\nvar publicDNS = []string{\n\t\"1.0.0.1\",                // Cloudflare\n\t\"1.1.1.1\",                // Cloudflare\n\t\"[2606:4700:4700::1111]\", // Cloudflare\n\t\"[2606:4700:4700::1001]\", // Cloudflare\n\t\"8.8.4.4\",                // Google\n\t\"8.8.8.8\",                // Google\n\t\"[2001:4860:4860::8844]\", // Google\n\t\"[2001:4860:4860::8888]\", // Google\n\t\"9.9.9.9\",                // Quad9\n\t\"149.112.112.112\",        // Quad9\n\t\"[2620:fe::fe]\",          // Quad9\n\t\"[2620:fe::fe:9]\",        // Quad9\n\t\"8.26.56.26\",             // Comodo\n\t\"8.20.247.20\",            // Comodo\n\t\"208.67.220.220\",         // Cisco OpenDNS\n\t\"208.67.222.222\",         // Cisco OpenDNS\n\t\"[2620:119:35::35]\",      // Cisco OpenDNS\n\t\"[2620:119:53::53]\",      // Cisco OpenDNS\n}\n\nfunc getConfigFile(requireValidPath bool) (fname string, err error) {\n\tconfigFile, err := utils.GetConfigDir(requireValidPath)\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = path.Join(configFile, \"internal-dns\")\n\treturn\n}\n\nfunc init() {\n\tdoRemember := false\n\tfor _, flag := range os.Args {\n\t\tif flag == \"--internal-dns\" {\n\t\t\tINTERNAL_DNS = true\n\t\t\tbreak\n\t\t}\n\t\tif flag == \"--remember\" {\n\t\t\tdoRemember = true\n\t\t}\n\t}\n\tif doRemember {\n\t\t// save in config file\n\t\tfname, err := getConfigFile(true)\n\t\tif err == nil {\n\t\t\tf, _ := os.Create(fname)\n\t\t\tf.Close()\n\t\t}\n\t}\n\tif !INTERNAL_DNS {\n\t\tfname, err := getConfigFile(false)\n\t\tif err == nil {\n\t\t\tINTERNAL_DNS = utils.Exists(fname)\n\t\t}\n\t}\n\tvar err error\n\tvar addr string\n\taddr, err = lookup(DEFAULT_RELAY)\n\tif err == nil {\n\t\tDEFAULT_RELAY = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY = \"\"\n\t}\n\taddr, err = lookup(DEFAULT_RELAY6)\n\tif err == nil {\n\t\tDEFAULT_RELAY6 = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY6 = \"\"\n\t}\n}\n\n// Resolve a hostname to an IP address using DNS.\nfunc lookup(address string) (ipaddress string, err error) {\n\tif !INTERNAL_DNS {\n\t\treturn localLookupIP(address)\n\t}\n\ttype Result struct {\n\t\ts   string\n\t\terr error\n\t}\n\tresult := make(chan Result, len(publicDNS))\n\tfor _, dns := range publicDNS {\n\t\tgo func(dns string) {\n\t\t\tvar r Result\n\t\t\tr.s, r.err = remoteLookupIP(address, dns)\n\t\t\tresult <- r\n\t\t}(dns)\n\t}\n\tfor i := 0; i < len(publicDNS); i++ {\n\t\tipaddress = (<-result).s\n\t\tif ipaddress != \"\" {\n\t\t\treturn\n\t\t}\n\t}\n\terr = fmt.Errorf(\"failed to resolve %s: all DNS servers exhausted\", address)\n\treturn\n}\n\n// localLookupIP returns a host's IP address based on the local resolver.\nfunc localLookupIP(address string) (ipaddress string, err error) {\n\tip, err := net.LookupHost(address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// remoteLookupIP returns a host's IP address based on a remote DNS server.\nfunc remoteLookupIP(address, dns string) (ipaddress string, err error) {\n\tr := &net.Resolver{\n\t\tPreferGo: true,\n\t\tDial: func(ctx context.Context, network, _ string) (net.Conn, error) {\n\t\t\td := new(net.Dialer)\n\t\t\treturn d.DialContext(ctx, network, dns+\":53\")\n\t\t},\n\t}\n\tip, err := r.LookupHost(context.Background(), address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// Path: src/tcp/tcp_test.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkConnection(b *testing.B) {\n\tlog.SetLevel(\"trace\")\n\tgo Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\", \"8284\")\n\ttime.Sleep(100 * time.Millisecond)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tc, _, _, _ := ConnectToTCPServer(\"127.0.0.1:8283\", \"pass123\", fmt.Sprintf(\"testroom%d\", i), 1*time.Minute)\n\t\tc.Close()\n\t}\n}\n\nfunc TestTCP(t *testing.T) {\n\tlog.SetLevel(\"error\")\n\ttimeToRoomDeletion = 100 * time.Millisecond\n\tgo Run(\"debug\", \"127.0.0.1\", \"8381\", \"pass123\", \"8382\")\n\ttime.Sleep(timeToRoomDeletion)\n\terr := PingServer(\"127.0.0.1:8381\")\n\tassert.Nil(t, err)\n\terr = PingServer(\"127.0.0.1:8333\")\n\tassert.NotNil(t, err)\n\n\ttime.Sleep(timeToRoomDeletion)\n\tc1, banner, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Minute)\n\tassert.Equal(t, banner, \"8382\")\n\tassert.Nil(t, err)\n\tc2, _, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.Nil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.NotNil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Nanosecond)\n\tassert.NotNil(t, err)\n\n\t// try sending data\n\tassert.Nil(t, c1.Send([]byte(\"hello, c2\")))\n\tvar data []byte\n\tfor {\n\t\tdata, err = c2.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c2\"), data)\n\n\tassert.Nil(t, c2.Send([]byte(\"hello, c1\")))\n\tfor {\n\t\tdata, err = c1.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c1\"), data)\n\n\tc1.Close()\n\ttime.Sleep(300 * time.Millisecond)\n}\n\n// Path: src/utils/utils_test.go\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst TCP_BUFFER_SIZE = 1024 * 64\n\nvar bigFileSize = 75000000\n\nfunc bigFile() {\n\tos.WriteFile(\"bigfile.test\", bytes.Repeat([]byte(\"z\"), bigFileSize), 0o666)\n}\n\nfunc BenchmarkMD5(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMD5HashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkXXHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tXXHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHashFull(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFileFull(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkSha256(b *testing.B) {\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tSHA256(\"hello,world\")\n\t}\n}\n\nfunc BenchmarkMissingChunks(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMissingChunks(\"bigfile.test\", int64(bigFileSize), TCP_BUFFER_SIZE/2)\n\t}\n}\n\nfunc TestExists(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tfmt.Println(GetLocalIPs())\n\tassert.True(t, Exists(\"bigfile.test\"))\n\tassert.False(t, Exists(\"doesnotexist\"))\n}\n\nfunc TestMD5HashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := MD5HashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"8304ff018e02baad0e3555bade29a405\", fmt.Sprintf(\"%x\", b))\n\t_, err = MD5HashFile(\"bigfile.test.nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestIMOHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := IMOHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"c0d1e123ca94148ffea146137684ebb9\", fmt.Sprintf(\"%x\", b))\n}\n\nfunc TestXXHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := XXHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"4918740eb5ccb6f7\", fmt.Sprintf(\"%x\", b))\n\t_, err = XXHashFile(\"nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestSHA256(t *testing.T) {\n\tassert.Equal(t, \"09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b\", SHA256(\"hello, world\"))\n}\n\nfunc TestByteCountDecimal(t *testing.T) {\n\tassert.Equal(t, \"10.0 kB\", ByteCountDecimal(10240))\n\tassert.Equal(t, \"50 B\", ByteCountDecimal(50))\n\tassert.Equal(t, \"12.4 MB\", ByteCountDecimal(13002343))\n}\n\nfunc TestMissingChunks(t *testing.T) {\n\tfileSize := 100\n\tchunkSize := 10\n\trand.Seed(1)\n\tbigBuff := make([]byte, fileSize)\n\trand.Read(bigBuff)\n\tos.WriteFile(\"missing.test\", bigBuff, 0o644)\n\tempty := make([]byte, chunkSize)\n\tf, err := os.OpenFile(\"missing.test\", os.O_RDWR, 0o644)\n\tassert.Nil(t, err)\n\tfor block := 0; block < fileSize/chunkSize; block++ {\n\t\tif block == 0 || block == 4 || block == 5 || block >= 7 {\n\t\t\tf.WriteAt(empty, int64(block*chunkSize))\n\t\t}\n\t}\n\tf.Close()\n\n\tchunkRanges := MissingChunks(\"missing.test\", int64(fileSize), chunkSize)\n\tassert.Equal(t, []int64{10, 0, 1, 40, 2, 70, 3}, chunkRanges)\n\n\tchunks := ChunkRangesToChunks(chunkRanges)\n\tassert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks)\n\n\tos.Remove(\"missing.test\")\n\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err := tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err := tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\tchunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)+10), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunkRanges = MissingChunks(tmpfile.Name()+\"ok\", int64(len(content)), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunks = ChunkRangesToChunks(chunkRanges)\n\tassert.Empty(t, chunks)\n}\n\n// func Test1(t *testing.T) {\n// \tchunkRanges := MissingChunks(\"../../m/bigfile.test\", int64(75000000), 1024*64/2)\n// \tfmt.Println(chunkRanges)\n// \tfmt.Println(ChunkRangesToChunks((chunkRanges)))\n// \tassert.Nil(t, nil)\n// }\n\nfunc TestHashFile(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err = tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err = tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\thashed, err := HashFile(tmpfile.Name(), \"xxhash\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"e66c561610ad51e2\", fmt.Sprintf(\"%x\", hashed))\n}\n\nfunc TestPublicIP(t *testing.T) {\n\tip, err := PublicIP()\n\tfmt.Println(ip)\n\tassert.True(t, strings.Contains(ip, \".\") || strings.Contains(ip, \":\"))\n\tassert.Nil(t, err)\n}\n\nfunc TestLocalIP(t *testing.T) {\n\tip := LocalIP()\n\tfmt.Println(ip)\n\tassert.True(t, strings.Contains(ip, \".\") || strings.Contains(ip, \":\"))\n}\n\nfunc TestGetRandomName(t *testing.T) {\n\tname := GetRandomName()\n\tfmt.Println(name)\n\tassert.NotEmpty(t, name)\n}\n\nfunc TestFindOpenPorts(t *testing.T) {\n\topenPorts := FindOpenPorts(\"127.0.0.1\", 9009, 4)\n\tassert.Equal(t, []int{9009, 9010, 9011, 9012}, openPorts)\n}\n\nfunc TestIsLocalIP(t *testing.T) {\n\tassert.True(t, IsLocalIP(\"192.168.0.14:9009\"))\n}\n\n", "needle_token_start": 1863, "needle_token_end": 2676, "code_context_ntokens": 12147, "output": ["```go\nfunc TestCrocSymlink(t *testing.T) {\n\tpathName := \"../link-in-folder\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./link-in-folder\")\n\tos.MkdirAll(pathName, 0o755)\n\tos.Symlink(\"../../README.md\", filepath.Join(pathName, \"README.link\"))\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\ts, err := filepath.EvalSymlinks(path.Join(pathName, \"README.link\"))\n\tif s != \"../../README.md\" && s != \"..\\\\..\\\\README.md\" {\n\t\tlog.Debug(s)\n\t\tt.Errorf(\"symlink failed to transfer in folder\")\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"symlink transfer failed: %s\", err.Error())\n\t}\n}\n```"]}
{"repo": "schollz/croc", "name": "BenchmarkCompressLevelNineBinary", "language": "go", "path": "src/compress/compress_test.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: To measure the performance of a compression algorithm at its highest compression level using a large binary dataset.\n2. **Input**: A randomly generated binary dataset of one million bytes.\n3. **Output**: There is no direct output from this function as it is designed to assess performance; however, it internally generates the compressed version of the input data.\n4. **Procedure**: The function first generates a binary dataset of one million bytes filled with random data. It then compresses this data repeatedly in a loop, using the highest compression level setting, to evaluate the efficiency and speed of the compression process under these conditions.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/cli/cli.go\npackage cli\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/chzyer/readline\"\n\t\"github.com/schollz/cli/v2\"\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/croc\"\n\t\"github.com/schollz/croc/v9/src/models\"\n\t\"github.com/schollz/croc/v9/src/tcp\"\n\t\"github.com/schollz/croc/v9/src/utils\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/schollz/mnemonicode\"\n\t\"github.com/schollz/pake/v3\"\n)\n\n// Version specifies the version\nvar Version string\n\n// Run will run the command line program\nfunc Run() (err error) {\n\t// use all of the processors\n\truntime.GOMAXPROCS(runtime.NumCPU())\n\n\tapp := cli.NewApp()\n\tapp.Name = \"croc\"\n\tif Version == \"\" {\n\t\tVersion = \"v9.6.15\"\n\t}\n\tapp.Version = Version\n\tapp.Compiled = time.Now()\n\tapp.Usage = \"easily and securely transfer stuff from one computer to another\"\n\tapp.UsageText = `Send a file:\n      croc send file.txt\n\n      -git to respect your .gitignore\n   Send multiple files:\n      croc send file1.txt file2.txt file3.txt\n    or\n      croc send *.jpg\n\n   Send everything in a folder:\n      croc send example-folder-name\n\n   Send a file with a custom code:\n      croc send --code secret-code file.txt\n\n   Receive a file using code:\n      croc secret-code`\n\tapp.Commands = []*cli.Command{\n\t\t{\n\t\t\tName:        \"send\",\n\t\t\tUsage:       \"send file(s), or folder (see options with croc send -h)\",\n\t\t\tDescription: \"send file(s), or folder, over the relay\",\n\t\t\tArgsUsage:   \"[filename(s) or folder]\",\n\t\t\tFlags: []cli.Flag{\n\t\t\t\t&cli.BoolFlag{Name: \"zip\", Usage: \"zip folder before sending\"},\n\t\t\t\t&cli.StringFlag{Name: \"code\", Aliases: []string{\"c\"}, Usage: \"codephrase used to connect to relay\"},\n\t\t\t\t&cli.StringFlag{Name: \"hash\", Value: \"xxhash\", Usage: \"hash algorithm (xxhash, imohash, md5)\"},\n\t\t\t\t&cli.StringFlag{Name: \"text\", Aliases: []string{\"t\"}, Usage: \"send some text\"},\n\t\t\t\t&cli.BoolFlag{Name: \"no-local\", Usage: \"disable local relay when sending\"},\n\t\t\t\t&cli.BoolFlag{Name: \"no-multi\", Usage: \"disable multiplexing\"},\n\t\t\t\t&cli.BoolFlag{Name: \"git\", Usage: \"enable .gitignore respect / don't send ignored files\"},\n\t\t\t\t&cli.IntFlag{Name: \"port\", Value: 9009, Usage: \"base port for the relay\"},\n\t\t\t\t&cli.IntFlag{Name: \"transfers\", Value: 4, Usage: \"number of ports to use for transfers\"},\n\t\t\t},\n\t\t\tHelpName: \"croc send\",\n\t\t\tAction:   send,\n\t\t},\n\t\t{\n\t\t\tName:        \"relay\",\n\t\t\tUsage:       \"start your own relay (optional)\",\n\t\t\tDescription: \"start relay\",\n\t\t\tHelpName:    \"croc relay\",\n\t\t\tAction:      relay,\n\t\t\tFlags: []cli.Flag{\n\t\t\t\t&cli.StringFlag{Name: \"host\", Usage: \"host of the relay\"},\n\t\t\t\t&cli.StringFlag{Name: \"ports\", Value: \"9009,9010,9011,9012,9013\", Usage: \"ports of the relay\"},\n\t\t\t},\n\t\t},\n\t}\n\tapp.Flags = []cli.Flag{\n\t\t&cli.BoolFlag{Name: \"internal-dns\", Usage: \"use a built-in DNS stub resolver rather than the host operating system\"},\n\t\t&cli.BoolFlag{Name: \"remember\", Usage: \"save these settings to reuse next time\"},\n\t\t&cli.BoolFlag{Name: \"debug\", Usage: \"toggle debug mode\"},\n\t\t&cli.BoolFlag{Name: \"yes\", Usage: \"automatically agree to all prompts\"},\n\t\t&cli.BoolFlag{Name: \"stdout\", Usage: \"redirect file to stdout\"},\n\t\t&cli.BoolFlag{Name: \"no-compress\", Usage: \"disable compression\"},\n\t\t&cli.BoolFlag{Name: \"ask\", Usage: \"make sure sender and recipient are prompted\"},\n\t\t&cli.BoolFlag{Name: \"local\", Usage: \"force to use only local connections\"},\n\t\t&cli.BoolFlag{Name: \"ignore-stdin\", Usage: \"ignore piped stdin\"},\n\t\t&cli.BoolFlag{Name: \"overwrite\", Usage: \"do not prompt to overwrite\"},\n\t\t&cli.BoolFlag{Name: \"testing\", Usage: \"flag for testing purposes\"},\n\t\t&cli.StringFlag{Name: \"curve\", Value: \"p256\", Usage: \"choose an encryption curve (\" + strings.Join(pake.AvailableCurves(), \", \") + \")\"},\n\t\t&cli.StringFlag{Name: \"ip\", Value: \"\", Usage: \"set sender ip if known e.g. 10.0.0.1:9009, [::1]:9009\"},\n\t\t&cli.StringFlag{Name: \"relay\", Value: models.DEFAULT_RELAY, Usage: \"address of the relay\", EnvVars: []string{\"CROC_RELAY\"}},\n\t\t&cli.StringFlag{Name: \"relay6\", Value: models.DEFAULT_RELAY6, Usage: \"ipv6 address of the relay\", EnvVars: []string{\"CROC_RELAY6\"}},\n\t\t&cli.StringFlag{Name: \"out\", Value: \".\", Usage: \"specify an output folder to receive the file\"},\n\t\t&cli.StringFlag{Name: \"pass\", Value: models.DEFAULT_PASSPHRASE, Usage: \"password for the relay\", EnvVars: []string{\"CROC_PASS\"}},\n\t\t&cli.StringFlag{Name: \"socks5\", Value: \"\", Usage: \"add a socks5 proxy\", EnvVars: []string{\"SOCKS5_PROXY\"}},\n\t\t&cli.StringFlag{Name: \"connect\", Value: \"\", Usage: \"add a http proxy\", EnvVars: []string{\"HTTP_PROXY\"}},\n\t\t&cli.StringFlag{Name: \"throttleUpload\", Value: \"\", Usage: \"Throttle the upload speed e.g. 500k\"},\n\t}\n\tapp.EnableBashCompletion = true\n\tapp.HideHelp = false\n\tapp.HideVersion = false\n\tapp.Action = func(c *cli.Context) error {\n\t\tallStringsAreFiles := func(strs []string) bool {\n\t\t\tfor _, str := range strs {\n...\n// Path: src/comm/comm_test.go\npackage comm\n\nimport (\n\t\"crypto/rand\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestComm(t *testing.T) {\n\ttoken := make([]byte, 3000)\n\tif _, err := rand.Read(token); err != nil {\n\t\tt.Error(err)\n\t}\n\n\tport := \"8001\"\n\tgo func() {\n\t\tlog.Debugf(\"starting TCP server on \" + port)\n\t\tserver, err := net.Listen(\"tcp\", \"0.0.0.0:\"+port)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t}\n\t\tdefer server.Close()\n\t\t// spawn a new goroutine whenever a client connects\n\t\tfor {\n\t\t\tconnection, err := server.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\t\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\t\tgo func(_ string, connection net.Conn) {\n\t\t\t\tc := New(connection)\n\t\t\t\terr = c.Send([]byte(\"hello, world\"))\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tdata, err := c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte(\"hello, computer\"), data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte{'\\x00'}, data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, token, data)\n\t\t\t}(port, connection)\n\t\t}\n\t}()\n\n\ttime.Sleep(300 * time.Millisecond)\n\ta, err := NewConnection(\"127.0.0.1:\"+port, 10*time.Minute)\n\tassert.Nil(t, err)\n\tdata, err := a.Receive()\n\tassert.Equal(t, []byte(\"hello, world\"), data)\n\tassert.Nil(t, err)\n\tassert.Nil(t, a.Send([]byte(\"hello, computer\")))\n\tassert.Nil(t, a.Send([]byte{'\\x00'}))\n\n\tassert.Nil(t, a.Send(token))\n\t_ = a.Connection()\n\ta.Close()\n\tassert.NotNil(t, a.Send(token))\n\t_, err = a.Write(token)\n\tassert.NotNil(t, err)\n}\n\n// Path: src/compress/compress_test.go\npackage compress\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar fable = []byte(`The Frog and the Crocodile\nOnce, there was a frog who lived in the middle of a swamp. His entire family had lived in that swamp for generations, but this particular frog decided that he had had quite enough wetness to last him a lifetime. He decided that he was going to find a dry place to live instead.\n\nThe only thing that separated him from dry land was a swampy, muddy, swiftly flowing river. But the river was home to all sorts of slippery, slittering snakes that loved nothing better than a good, plump frog for dinner, so Frog didn't dare try to swim across.\n\nSo for many days, the frog stayed put, hopping along the bank, trying to think of a way to get across.\n\nThe snakes hissed and jeered at him, daring him to come closer, but he refused. Occasionally they would slither closer, jaws open to attack, but the frog always leaped out of the way. But no matter how far upstream he searched or how far downstream, the frog wasn't able to find a way across the water.\n\nHe had felt certain that there would be a bridge, or a place where the banks came together, yet all he found was more reeds and water. After a while, even the snakes stopped teasing him and went off in search of easier prey.\n\nThe frog sighed in frustration and sat to sulk in the rushes. Suddenly, he spotted two big eyes staring at him from the water. The giant log-shaped animal opened its mouth and asked him, \"What are you doing, Frog? Surely there are enough flies right there for a meal.\"\n\nThe frog croaked in surprise and leaped away from the crocodile. That creature could swallow him whole in a moment without thinking about it! Once he was a satisfied that he was a safe distance away, he answered. \"I'm tired of living in swampy waters, and I want to travel to the other side of the river. But if I swim across, the snakes will eat me.\"\n\nThe crocodile harrumphed in agreement and sat, thinking, for a while. \"Well, if you're afraid of the snakes, I could give you a ride across,\" he suggested.\n\n\"Oh no, I don't think so,\" Frog answered quickly. \"You'd eat me on the way over, or go underwater so the snakes could get me!\"\n\n\"Now why would I let the snakes get you? I think they're a terrible nuisance with all their hissing and slithering! The river would be much better off without them altogether! Anyway, if you're so worried that I might eat you, you can ride on my tail.\"\n\nThe frog considered his offer. He did want to get to dry ground very badly, and there didn't seem to be any other way across the river. He looked at the crocodile from his short, squat buggy eyes and wondered about the crocodile's motives. But if he rode on the tail, the croc couldn't eat him anyway. And he was right about the snakes--no self-respecting crocodile would give a meal to the snakes.\n\n\"Okay, it sounds like a good plan to me. Turn around so I can hop on your tail.\"\n\nThe crocodile flopped his tail into the marshy mud and let the frog climb on, then he waddled out to the river. But he couldn't stick his tail into the water as a rudder because the frog was on it -- and if he put his tail in the water, the snakes would eat the frog. They clumsily floated downstream for a ways, until the crocodile said, \"Hop onto my back so I can steer straight with my tail.\" The frog moved, and the journey smoothed out.\n\nFrom where he was sitting, the frog couldn't see much except the back of Crocodile's head. \"Why don't you hop up on my head so you can see everything around us?\" Crocodile invited. `)\n\nfunc BenchmarkCompressLevelMinusTwo(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(fable, -2)\n\t}\n}\n\nfunc BenchmarkCompressLevelNine(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(fable, 9)\n\t}\n}\n\nfunc BenchmarkCompressLevelMinusTwoBinary(b *testing.B) {\n\tdata := make([]byte, 1000000)\n\tif _, err := rand.Read(data); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(data, -2)\n\t}\n}\n\n\nfunc BenchmarkCompressLevelNineBinary(b *testing.B) {\n\tdata := make([]byte, 1000000)\n\tif _, err := rand.Read(data); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(data, 9)\n\t}\n}\n\nfunc TestCompress(t *testing.T) {\n\tcompressedB := CompressWithOption(fable, 9)\n\tdataRateSavings := 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))\n\tfmt.Printf(\"Level 9: %2.0f%% percent space savings\\n\", dataRateSavings)\n\tassert.True(t, len(compressedB) < len(fable))\n\tassert.Equal(t, fable, Decompress(compressedB))\n\n\tcompressedB = CompressWithOption(fable, -2)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))\n\tfmt.Printf(\"Level -2: %2.0f%% percent space savings\\n\", dataRateSavings)\n\tassert.True(t, len(compressedB) < len(fable))\n\n\tcompressedB = Compress(fable)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))\n\tfmt.Printf(\"Level -2: %2.0f%% percent space savings\\n\", dataRateSavings)\n\tassert.True(t, len(compressedB) < len(fable))\n\n\tdata := make([]byte, 4096)\n\tif _, err := rand.Read(data); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcompressedB = CompressWithOption(data, -2)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(data)))\n\tfmt.Printf(\"random, Level -2: %2.0f%% percent space savings\\n\", dataRateSavings)\n\n\tif _, err := rand.Read(data); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tcompressedB = CompressWithOption(data, 9)\n\tdataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(data)))\n\n\tfmt.Printf(\"random, Level 9: %2.0f%% percent space savings\\n\", dataRateSavings)\n\n}\n\n// Path: src/croc/croc_test.go\npackage croc\n\nimport (\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/tcp\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc init() {\n\tlog.SetLevel(\"trace\")\n\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8281\", \"pass123\", \"8282,8283,8284,8285\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8282\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8284\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8285\", \"pass123\")\n\ttime.Sleep(1 * time.Second)\n}\n\nfunc TestCrocReadme(t *testing.T) {\n\tdefer os.Remove(\"README.md\")\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../README.md\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocEmptyFolder(t *testing.T) {\n\tpathName := \"../../testEmpty\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./testEmpty\")\n\tos.MkdirAll(pathName, 0o755)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocSymlink(t *testing.T) {\n\tpathName := \"../link-in-folder\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./link-in-folder\")\n\tos.MkdirAll(pathName, 0o755)\n\tos.Symlink(\"../../README.md\", filepath.Join(pathName, \"README.link\"))\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr = receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\ts, err := filepath.EvalSymlinks(path.Join(pathName, \"README.link\"))\n\tif s != \"../../README.md\" && s != \"..\\\\..\\\\README.md\" {\n\t\tlog.Debug(s)\n\t\tt.Errorf(\"symlink failed to transfer in folder\")\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"symlink transfer failed: %s\", err.Error())\n\t}\n}\nfunc testCrocIgnoreGit(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\".gitignore\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\ttime.Sleep(1 * time.Second)\n\tfile, err := os.Create(\".gitignore\")\n\tif err != nil {\n\t\tlog.Errorf(\"error creating file\")\n\t}\n\t_, err = file.WriteString(\"LICENSE\")\n\tif err != nil {\n\t\tlog.Errorf(\"error writing to file\")\n\t}\n\ttime.Sleep(1 * time.Second)\n\t// due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo.\n\tfilesInfo, _, _, errGet := GetFilesInfo([]string{\"../../LICENSE\", \".gitignore\", \"croc.go\"}, false, true)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\tfor _, file := range filesInfo {\n\t\tif strings.Contains(file.Name, \"LICENSE\") {\n\t\t\tt.Errorf(\"test failed, should ignore LICENSE\")\n\t\t}\n\t}\n}\n\nfunc TestCrocLocal(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\"LICENSE\")\n\tdefer os.Remove(\"touched\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPorts:    []string{\"8181\", \"8182\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttime.Sleep(1 * time.Second)\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\tos.Create(\"touched\")\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../LICENSE\", \"touched\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocError(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err = tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err = tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\n\tDebug(false)\n\tlog.SetLevel(\"warn\")\n\tsender, _ := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc2\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"doesntexistok.com:8381\",\n\t\tRelayPorts:    []string{\"8381\", \"8382\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false, false)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\tlog.Debug(err)\n\tassert.NotNil(t, err)\n}\n\nfunc TestCleanUp(t *testing.T) {\n\t// windows allows files to be deleted only if they\n\t// are not open by another program so the remove actions\n\t// from the above tests will not always do a good clean up\n\t// This \"test\" will make sure\n\toperatingSystem := runtime.GOOS\n\tlog.Debugf(\"The operating system is %s\", operatingSystem)\n\tif operatingSystem == \"windows\" {\n\t\ttime.Sleep(1 * time.Second)\n\t\tlog.Debug(\"Full cleanup\")\n\t\tvar err error\n\n\t\tfor _, file := range []string{\"README.md\", \"./README.md\"} {\n\t\t\terr = os.Remove(file)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", file)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", file)\n\t\t\t}\n\t\t}\n\t\tfor _, folder := range []string{\"./testEmpty\", \"./link-in-folder\"} {\n\t\t\terr = os.RemoveAll(folder)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", folder)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", folder)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Path: src/crypt/crypt_test.go\npackage crypt\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkEncrypt(b *testing.B) {\n\tbob, _, _ := New([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncrypt([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecrypt(b *testing.B) {\n\tkey, _, _ := New([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := Encrypt(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecrypt(enc, key)\n\t}\n}\n\nfunc BenchmarkEncryptChaCha(b *testing.B) {\n\tbob, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncryptChaCha([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecryptChaCha(b *testing.B) {\n\tkey, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := EncryptChaCha(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecryptChaCha(enc, key)\n\t}\n}\n\nfunc TestEncryption(t *testing.T) {\n\tkey, salt, err := New([]byte(\"password\"), nil)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := Encrypt(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := Decrypt(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := New([]byte(\"password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = New([]byte(\"wrong password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = Decrypt([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = New([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\nfunc TestEncryptionChaCha(t *testing.T) {\n\tkey, salt, err := NewArgon2([]byte(\"password\"), nil)\n\tfmt.Printf(\"key: %x\\n\", key)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := EncryptChaCha(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := DecryptChaCha(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := NewArgon2([]byte(\"password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = NewArgon2([]byte(\"wrong password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = DecryptChaCha([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = NewArgon2([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\n// Path: src/install/updateversion.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n)\n\nfunc main() {\n\terr := run()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t}\n}\n\nfunc run() (err error) {\n\tversionNew := \"v\" + os.Getenv(\"VERSION\")\n\tversionHash, err := exec.Command(\"git\", \"rev-parse\", \"--short\", \"HEAD\").Output()\n\tif err != nil {\n\t\treturn\n\t}\n\tversionHashNew := strings.TrimSpace(string(versionHash))\n\tfmt.Println(versionNew)\n\tfmt.Println(versionHashNew)\n\n\terr = replaceInFile(\"src/cli/cli.go\", `Version = \"`, `\"`, versionNew+\"-\"+versionHashNew)\n\tif err == nil {\n\t\tfmt.Printf(\"updated cli.go to version %s\\n\", versionNew)\n\t}\n\terr = replaceInFile(\"README.md\", `version-`, `-b`, strings.Split(versionNew, \"-\")[0])\n\tif err == nil {\n\t\tfmt.Printf(\"updated README to version %s\\n\", strings.Split(versionNew, \"-\")[0])\n\t}\n\n\terr = replaceInFile(\"src/install/default.txt\", `croc_version=\"`, `\"`, strings.Split(versionNew, \"-\")[0][1:])\n\tif err == nil {\n\t\tfmt.Printf(\"updated default.txt to version %s\\n\", strings.Split(versionNew, \"-\")[0][1:])\n\t}\n\n\treturn\n}\n\nfunc replaceInFile(fname, start, end, replacement string) (err error) {\n\tb, err := os.ReadFile(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\toldVersion := GetStringInBetween(string(b), start, end)\n\tif oldVersion == \"\" {\n\t\terr = fmt.Errorf(\"nothing\")\n\t\treturn\n\t}\n\tnewF := strings.Replace(\n\t\tstring(b),\n\t\tfmt.Sprintf(\"%s%s%s\", start, oldVersion, end),\n\t\tfmt.Sprintf(\"%s%s%s\", start, replacement, end),\n\t\t1,\n\t)\n\terr = os.WriteFile(fname, []byte(newF), 0o644)\n\treturn\n}\n\n// GetStringInBetween Returns empty string if no start string found\nfunc GetStringInBetween(str string, start string, end string) (result string) {\n\ts := strings.Index(str, start)\n\tif s == -1 {\n\t\treturn\n\t}\n\ts += len(start)\n\te := strings.Index(str[s:], end)\n\tif e == -1 {\n\t\treturn\n\t}\n\te += s\n\treturn str[s:e]\n}\n\n// Path: src/message/message_test.go\npackage message\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar TypeMessage Type = \"message\"\n\nfunc TestMessage(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tassert.Nil(t, err)\n\tfmt.Println(string(salt))\n\tb, err := Encode(e, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(e, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n\t_, err = Decode([]byte(\"not pass\"), b)\n\tassert.NotNil(t, err)\n\t_, err = Encode([]byte(\"0\"), m)\n\tassert.NotNil(t, err)\n}\n\nfunc TestMessageNoPass(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\tb, err := Encode(nil, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(nil, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n}\n\nfunc TestSend(t *testing.T) {\n\ttoken := make([]byte, 40000000)\n\trand.Read(token)\n\n\tport := \"8801\"\n\tgo func() {\n\t\tlog.Debugf(\"starting TCP server on \" + port)\n\t\tserver, err := net.Listen(\"tcp\", \"0.0.0.0:\"+port)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t}\n\t\tdefer server.Close()\n\t\t// spawn a new goroutine whenever a client connects\n\t\tfor {\n\t\t\tconnection, err := server.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\t\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\t\tgo func(_ string, connection net.Conn) {\n\t\t\t\tc := comm.New(connection)\n\t\t\t\terr = c.Send([]byte(\"hello, world\"))\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tdata, err := c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte(\"hello, computer\"), data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte{'\\x00'}, data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, token, data)\n\t\t\t}(port, connection)\n\t\t}\n\t}()\n\n\ttime.Sleep(300 * time.Millisecond)\n\ta, err := comm.NewConnection(\"127.0.0.1:\"+port, 10*time.Minute)\n\tassert.Nil(t, err)\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tlog.Debug(salt)\n\tassert.Nil(t, err)\n\n\tassert.Nil(t, Send(a, e, m))\n}\n\n// Path: src/models/constants.go\npackage models\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\n\t\"github.com/schollz/croc/v9/src/utils\"\n)\n\n// TCP_BUFFER_SIZE is the maximum packet size\nconst TCP_BUFFER_SIZE = 1024 * 64\n\n// DEFAULT_RELAY is the default relay used (can be set using --relay)\nvar (\n\tDEFAULT_RELAY      = \"croc.schollz.com\"\n\tDEFAULT_RELAY6     = \"croc6.schollz.com\"\n\tDEFAULT_PORT       = \"9009\"\n\tDEFAULT_PASSPHRASE = \"pass123\"\n\tINTERNAL_DNS       = false\n)\n\n// publicDNS are servers to be queried if a local lookup fails\nvar publicDNS = []string{\n\t\"1.0.0.1\",                // Cloudflare\n\t\"1.1.1.1\",                // Cloudflare\n\t\"[2606:4700:4700::1111]\", // Cloudflare\n\t\"[2606:4700:4700::1001]\", // Cloudflare\n\t\"8.8.4.4\",                // Google\n\t\"8.8.8.8\",                // Google\n\t\"[2001:4860:4860::8844]\", // Google\n\t\"[2001:4860:4860::8888]\", // Google\n\t\"9.9.9.9\",                // Quad9\n\t\"149.112.112.112\",        // Quad9\n\t\"[2620:fe::fe]\",          // Quad9\n\t\"[2620:fe::fe:9]\",        // Quad9\n\t\"8.26.56.26\",             // Comodo\n\t\"8.20.247.20\",            // Comodo\n\t\"208.67.220.220\",         // Cisco OpenDNS\n\t\"208.67.222.222\",         // Cisco OpenDNS\n\t\"[2620:119:35::35]\",      // Cisco OpenDNS\n\t\"[2620:119:53::53]\",      // Cisco OpenDNS\n}\n\nfunc getConfigFile(requireValidPath bool) (fname string, err error) {\n\tconfigFile, err := utils.GetConfigDir(requireValidPath)\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = path.Join(configFile, \"internal-dns\")\n\treturn\n}\n\nfunc init() {\n\tdoRemember := false\n\tfor _, flag := range os.Args {\n\t\tif flag == \"--internal-dns\" {\n\t\t\tINTERNAL_DNS = true\n\t\t\tbreak\n\t\t}\n\t\tif flag == \"--remember\" {\n\t\t\tdoRemember = true\n\t\t}\n\t}\n\tif doRemember {\n\t\t// save in config file\n\t\tfname, err := getConfigFile(true)\n\t\tif err == nil {\n\t\t\tf, _ := os.Create(fname)\n\t\t\tf.Close()\n\t\t}\n\t}\n\tif !INTERNAL_DNS {\n\t\tfname, err := getConfigFile(false)\n\t\tif err == nil {\n\t\t\tINTERNAL_DNS = utils.Exists(fname)\n\t\t}\n\t}\n\tvar err error\n\tvar addr string\n\taddr, err = lookup(DEFAULT_RELAY)\n\tif err == nil {\n\t\tDEFAULT_RELAY = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY = \"\"\n\t}\n\taddr, err = lookup(DEFAULT_RELAY6)\n\tif err == nil {\n\t\tDEFAULT_RELAY6 = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY6 = \"\"\n\t}\n}\n\n// Resolve a hostname to an IP address using DNS.\nfunc lookup(address string) (ipaddress string, err error) {\n\tif !INTERNAL_DNS {\n\t\treturn localLookupIP(address)\n\t}\n\ttype Result struct {\n\t\ts   string\n\t\terr error\n\t}\n\tresult := make(chan Result, len(publicDNS))\n\tfor _, dns := range publicDNS {\n\t\tgo func(dns string) {\n\t\t\tvar r Result\n\t\t\tr.s, r.err = remoteLookupIP(address, dns)\n\t\t\tresult <- r\n\t\t}(dns)\n\t}\n\tfor i := 0; i < len(publicDNS); i++ {\n\t\tipaddress = (<-result).s\n\t\tif ipaddress != \"\" {\n\t\t\treturn\n\t\t}\n\t}\n\terr = fmt.Errorf(\"failed to resolve %s: all DNS servers exhausted\", address)\n\treturn\n}\n\n// localLookupIP returns a host's IP address based on the local resolver.\nfunc localLookupIP(address string) (ipaddress string, err error) {\n\tip, err := net.LookupHost(address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// remoteLookupIP returns a host's IP address based on a remote DNS server.\nfunc remoteLookupIP(address, dns string) (ipaddress string, err error) {\n\tr := &net.Resolver{\n\t\tPreferGo: true,\n\t\tDial: func(ctx context.Context, network, _ string) (net.Conn, error) {\n\t\t\td := new(net.Dialer)\n\t\t\treturn d.DialContext(ctx, network, dns+\":53\")\n\t\t},\n\t}\n\tip, err := r.LookupHost(context.Background(), address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// Path: src/tcp/tcp_test.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkConnection(b *testing.B) {\n\tlog.SetLevel(\"trace\")\n\tgo Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\", \"8284\")\n\ttime.Sleep(100 * time.Millisecond)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tc, _, _, _ := ConnectToTCPServer(\"127.0.0.1:8283\", \"pass123\", fmt.Sprintf(\"testroom%d\", i), 1*time.Minute)\n\t\tc.Close()\n\t}\n}\n\nfunc TestTCP(t *testing.T) {\n\tlog.SetLevel(\"error\")\n\ttimeToRoomDeletion = 100 * time.Millisecond\n\tgo Run(\"debug\", \"127.0.0.1\", \"8381\", \"pass123\", \"8382\")\n\ttime.Sleep(timeToRoomDeletion)\n\terr := PingServer(\"127.0.0.1:8381\")\n\tassert.Nil(t, err)\n\terr = PingServer(\"127.0.0.1:8333\")\n\tassert.NotNil(t, err)\n\n\ttime.Sleep(timeToRoomDeletion)\n\tc1, banner, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Minute)\n\tassert.Equal(t, banner, \"8382\")\n\tassert.Nil(t, err)\n\tc2, _, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.Nil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.NotNil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Nanosecond)\n\tassert.NotNil(t, err)\n\n\t// try sending data\n\tassert.Nil(t, c1.Send([]byte(\"hello, c2\")))\n\tvar data []byte\n\tfor {\n\t\tdata, err = c2.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c2\"), data)\n\n\tassert.Nil(t, c2.Send([]byte(\"hello, c1\")))\n\tfor {\n\t\tdata, err = c1.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c1\"), data)\n\n\tc1.Close()\n\ttime.Sleep(300 * time.Millisecond)\n}\n\n// Path: src/utils/utils_test.go\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst TCP_BUFFER_SIZE = 1024 * 64\n\nvar bigFileSize = 75000000\n\nfunc bigFile() {\n\tos.WriteFile(\"bigfile.test\", bytes.Repeat([]byte(\"z\"), bigFileSize), 0o666)\n}\n\nfunc BenchmarkMD5(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMD5HashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkXXHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tXXHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHashFull(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFileFull(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkSha256(b *testing.B) {\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tSHA256(\"hello,world\")\n\t}\n}\n\nfunc BenchmarkMissingChunks(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMissingChunks(\"bigfile.test\", int64(bigFileSize), TCP_BUFFER_SIZE/2)\n\t}\n}\n\nfunc TestExists(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tfmt.Println(GetLocalIPs())\n\tassert.True(t, Exists(\"bigfile.test\"))\n\tassert.False(t, Exists(\"doesnotexist\"))\n}\n\nfunc TestMD5HashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := MD5HashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"8304ff018e02baad0e3555bade29a405\", fmt.Sprintf(\"%x\", b))\n\t_, err = MD5HashFile(\"bigfile.test.nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestIMOHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := IMOHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"c0d1e123ca94148ffea146137684ebb9\", fmt.Sprintf(\"%x\", b))\n}\n\nfunc TestXXHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := XXHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"4918740eb5ccb6f7\", fmt.Sprintf(\"%x\", b))\n\t_, err = XXHashFile(\"nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestSHA256(t *testing.T) {\n\tassert.Equal(t, \"09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b\", SHA256(\"hello, world\"))\n}\n\nfunc TestByteCountDecimal(t *testing.T) {\n\tassert.Equal(t, \"10.0 kB\", ByteCountDecimal(10240))\n\tassert.Equal(t, \"50 B\", ByteCountDecimal(50))\n\tassert.Equal(t, \"12.4 MB\", ByteCountDecimal(13002343))\n}\n\nfunc TestMissingChunks(t *testing.T) {\n\tfileSize := 100\n\tchunkSize := 10\n\trand.Seed(1)\n...\n", "needle_token_start": 3843, "needle_token_end": 3942, "code_context_ntokens": 15242, "output": ["```go\nfunc BenchmarkCompressLevelNineBinary(b *testing.B) {\n\tdata := make([]byte, 1000000)\n\tif _, err := rand.Read(data); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tfor i := 0; i < b.N; i++ {\n\t\tCompressWithOption(data, 9)\n\t}\n}\n```"]}
{"repo": "schollz/croc", "name": "GetStringInBetween", "language": "go", "path": "src/install/updateversion.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to extract a substring from a given string, which is located between two specified marker strings.\n2. **Input**: The function takes three parameters: a main string from which to extract the substring, a starting marker string, and an ending marker string.\n3. **Output**: It returns the extracted substring. If the starting marker is not found, or if the ending marker is not found after the starting marker, it returns an empty string.\n4. **Procedure**: The function first locates the starting marker in the main string. If found, it then searches for the ending marker starting from the position just after the end of the starting marker. If both markers are found, the substring between these markers is returned. If either marker is not found, an empty string is returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/croc/croc_test.go\npackage croc\n\nimport (\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/tcp\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc init() {\n\tlog.SetLevel(\"trace\")\n\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8281\", \"pass123\", \"8282,8283,8284,8285\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8282\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8284\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8285\", \"pass123\")\n\ttime.Sleep(1 * time.Second)\n}\n\nfunc TestCrocReadme(t *testing.T) {\n\tdefer os.Remove(\"README.md\")\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../README.md\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocEmptyFolder(t *testing.T) {\n\tpathName := \"../../testEmpty\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./testEmpty\")\n\tos.MkdirAll(pathName, 0o755)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocSymlink(t *testing.T) {\n\tpathName := \"../link-in-folder\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./link-in-folder\")\n\tos.MkdirAll(pathName, 0o755)\n\tos.Symlink(\"../../README.md\", filepath.Join(pathName, \"README.link\"))\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr = receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\ts, err := filepath.EvalSymlinks(path.Join(pathName, \"README.link\"))\n\tif s != \"../../README.md\" && s != \"..\\\\..\\\\README.md\" {\n\t\tlog.Debug(s)\n\t\tt.Errorf(\"symlink failed to transfer in folder\")\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"symlink transfer failed: %s\", err.Error())\n\t}\n}\nfunc testCrocIgnoreGit(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\".gitignore\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\ttime.Sleep(1 * time.Second)\n\tfile, err := os.Create(\".gitignore\")\n\tif err != nil {\n\t\tlog.Errorf(\"error creating file\")\n\t}\n\t_, err = file.WriteString(\"LICENSE\")\n\tif err != nil {\n\t\tlog.Errorf(\"error writing to file\")\n\t}\n\ttime.Sleep(1 * time.Second)\n\t// due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo.\n\tfilesInfo, _, _, errGet := GetFilesInfo([]string{\"../../LICENSE\", \".gitignore\", \"croc.go\"}, false, true)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\tfor _, file := range filesInfo {\n\t\tif strings.Contains(file.Name, \"LICENSE\") {\n\t\t\tt.Errorf(\"test failed, should ignore LICENSE\")\n\t\t}\n\t}\n}\n\nfunc TestCrocLocal(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\"LICENSE\")\n\tdefer os.Remove(\"touched\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPorts:    []string{\"8181\", \"8182\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttime.Sleep(1 * time.Second)\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\tos.Create(\"touched\")\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../LICENSE\", \"touched\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocError(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n...\n// Path: src/crypt/crypt_test.go\npackage crypt\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkEncrypt(b *testing.B) {\n\tbob, _, _ := New([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncrypt([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecrypt(b *testing.B) {\n\tkey, _, _ := New([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := Encrypt(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecrypt(enc, key)\n\t}\n}\n\nfunc BenchmarkEncryptChaCha(b *testing.B) {\n\tbob, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncryptChaCha([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecryptChaCha(b *testing.B) {\n\tkey, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := EncryptChaCha(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecryptChaCha(enc, key)\n\t}\n}\n\nfunc TestEncryption(t *testing.T) {\n\tkey, salt, err := New([]byte(\"password\"), nil)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := Encrypt(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := Decrypt(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := New([]byte(\"password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = New([]byte(\"wrong password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = Decrypt([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = New([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\nfunc TestEncryptionChaCha(t *testing.T) {\n\tkey, salt, err := NewArgon2([]byte(\"password\"), nil)\n\tfmt.Printf(\"key: %x\\n\", key)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := EncryptChaCha(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := DecryptChaCha(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := NewArgon2([]byte(\"password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = NewArgon2([]byte(\"wrong password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = DecryptChaCha([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = NewArgon2([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\n// Path: src/install/updateversion.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n)\n\nfunc main() {\n\terr := run()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t}\n}\n\nfunc run() (err error) {\n\tversionNew := \"v\" + os.Getenv(\"VERSION\")\n\tversionHash, err := exec.Command(\"git\", \"rev-parse\", \"--short\", \"HEAD\").Output()\n\tif err != nil {\n\t\treturn\n\t}\n\tversionHashNew := strings.TrimSpace(string(versionHash))\n\tfmt.Println(versionNew)\n\tfmt.Println(versionHashNew)\n\n\terr = replaceInFile(\"src/cli/cli.go\", `Version = \"`, `\"`, versionNew+\"-\"+versionHashNew)\n\tif err == nil {\n\t\tfmt.Printf(\"updated cli.go to version %s\\n\", versionNew)\n\t}\n\terr = replaceInFile(\"README.md\", `version-`, `-b`, strings.Split(versionNew, \"-\")[0])\n\tif err == nil {\n\t\tfmt.Printf(\"updated README to version %s\\n\", strings.Split(versionNew, \"-\")[0])\n\t}\n\n\terr = replaceInFile(\"src/install/default.txt\", `croc_version=\"`, `\"`, strings.Split(versionNew, \"-\")[0][1:])\n\tif err == nil {\n\t\tfmt.Printf(\"updated default.txt to version %s\\n\", strings.Split(versionNew, \"-\")[0][1:])\n\t}\n\n\treturn\n}\n\nfunc replaceInFile(fname, start, end, replacement string) (err error) {\n\tb, err := os.ReadFile(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\toldVersion := GetStringInBetween(string(b), start, end)\n\tif oldVersion == \"\" {\n\t\terr = fmt.Errorf(\"nothing\")\n\t\treturn\n\t}\n\tnewF := strings.Replace(\n\t\tstring(b),\n\t\tfmt.Sprintf(\"%s%s%s\", start, oldVersion, end),\n\t\tfmt.Sprintf(\"%s%s%s\", start, replacement, end),\n\t\t1,\n\t)\n\terr = os.WriteFile(fname, []byte(newF), 0o644)\n\treturn\n}\n\n// GetStringInBetween Returns empty string if no start string found\n\nfunc GetStringInBetween(str string, start string, end string) (result string) {\n\ts := strings.Index(str, start)\n\tif s == -1 {\n\t\treturn\n\t}\n\ts += len(start)\n\te := strings.Index(str[s:], end)\n\tif e == -1 {\n\t\treturn\n\t}\n\te += s\n\treturn str[s:e]\n}\n\n// Path: src/message/message_test.go\npackage message\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar TypeMessage Type = \"message\"\n\nfunc TestMessage(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tassert.Nil(t, err)\n\tfmt.Println(string(salt))\n\tb, err := Encode(e, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(e, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n\t_, err = Decode([]byte(\"not pass\"), b)\n\tassert.NotNil(t, err)\n\t_, err = Encode([]byte(\"0\"), m)\n\tassert.NotNil(t, err)\n}\n\nfunc TestMessageNoPass(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\tb, err := Encode(nil, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(nil, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n}\n\nfunc TestSend(t *testing.T) {\n\ttoken := make([]byte, 40000000)\n\trand.Read(token)\n\n\tport := \"8801\"\n\tgo func() {\n\t\tlog.Debugf(\"starting TCP server on \" + port)\n\t\tserver, err := net.Listen(\"tcp\", \"0.0.0.0:\"+port)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t}\n\t\tdefer server.Close()\n\t\t// spawn a new goroutine whenever a client connects\n\t\tfor {\n\t\t\tconnection, err := server.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\t\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\t\tgo func(_ string, connection net.Conn) {\n\t\t\t\tc := comm.New(connection)\n\t\t\t\terr = c.Send([]byte(\"hello, world\"))\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tdata, err := c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte(\"hello, computer\"), data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte{'\\x00'}, data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, token, data)\n\t\t\t}(port, connection)\n\t\t}\n\t}()\n\n\ttime.Sleep(300 * time.Millisecond)\n\ta, err := comm.NewConnection(\"127.0.0.1:\"+port, 10*time.Minute)\n\tassert.Nil(t, err)\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tlog.Debug(salt)\n\tassert.Nil(t, err)\n\n\tassert.Nil(t, Send(a, e, m))\n}\n\n// Path: src/models/constants.go\npackage models\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\n\t\"github.com/schollz/croc/v9/src/utils\"\n)\n\n// TCP_BUFFER_SIZE is the maximum packet size\nconst TCP_BUFFER_SIZE = 1024 * 64\n\n// DEFAULT_RELAY is the default relay used (can be set using --relay)\nvar (\n\tDEFAULT_RELAY      = \"croc.schollz.com\"\n\tDEFAULT_RELAY6     = \"croc6.schollz.com\"\n\tDEFAULT_PORT       = \"9009\"\n\tDEFAULT_PASSPHRASE = \"pass123\"\n\tINTERNAL_DNS       = false\n)\n\n// publicDNS are servers to be queried if a local lookup fails\nvar publicDNS = []string{\n\t\"1.0.0.1\",                // Cloudflare\n\t\"1.1.1.1\",                // Cloudflare\n\t\"[2606:4700:4700::1111]\", // Cloudflare\n\t\"[2606:4700:4700::1001]\", // Cloudflare\n\t\"8.8.4.4\",                // Google\n\t\"8.8.8.8\",                // Google\n\t\"[2001:4860:4860::8844]\", // Google\n\t\"[2001:4860:4860::8888]\", // Google\n\t\"9.9.9.9\",                // Quad9\n\t\"149.112.112.112\",        // Quad9\n\t\"[2620:fe::fe]\",          // Quad9\n\t\"[2620:fe::fe:9]\",        // Quad9\n\t\"8.26.56.26\",             // Comodo\n\t\"8.20.247.20\",            // Comodo\n\t\"208.67.220.220\",         // Cisco OpenDNS\n\t\"208.67.222.222\",         // Cisco OpenDNS\n\t\"[2620:119:35::35]\",      // Cisco OpenDNS\n\t\"[2620:119:53::53]\",      // Cisco OpenDNS\n}\n\nfunc getConfigFile(requireValidPath bool) (fname string, err error) {\n\tconfigFile, err := utils.GetConfigDir(requireValidPath)\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = path.Join(configFile, \"internal-dns\")\n\treturn\n}\n\nfunc init() {\n\tdoRemember := false\n\tfor _, flag := range os.Args {\n\t\tif flag == \"--internal-dns\" {\n\t\t\tINTERNAL_DNS = true\n\t\t\tbreak\n\t\t}\n\t\tif flag == \"--remember\" {\n\t\t\tdoRemember = true\n\t\t}\n\t}\n\tif doRemember {\n\t\t// save in config file\n\t\tfname, err := getConfigFile(true)\n\t\tif err == nil {\n\t\t\tf, _ := os.Create(fname)\n\t\t\tf.Close()\n\t\t}\n\t}\n\tif !INTERNAL_DNS {\n\t\tfname, err := getConfigFile(false)\n\t\tif err == nil {\n\t\t\tINTERNAL_DNS = utils.Exists(fname)\n\t\t}\n\t}\n\tvar err error\n\tvar addr string\n\taddr, err = lookup(DEFAULT_RELAY)\n\tif err == nil {\n\t\tDEFAULT_RELAY = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY = \"\"\n\t}\n\taddr, err = lookup(DEFAULT_RELAY6)\n\tif err == nil {\n\t\tDEFAULT_RELAY6 = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY6 = \"\"\n\t}\n}\n\n// Resolve a hostname to an IP address using DNS.\nfunc lookup(address string) (ipaddress string, err error) {\n\tif !INTERNAL_DNS {\n\t\treturn localLookupIP(address)\n\t}\n\ttype Result struct {\n\t\ts   string\n\t\terr error\n\t}\n\tresult := make(chan Result, len(publicDNS))\n\tfor _, dns := range publicDNS {\n\t\tgo func(dns string) {\n\t\t\tvar r Result\n\t\t\tr.s, r.err = remoteLookupIP(address, dns)\n\t\t\tresult <- r\n\t\t}(dns)\n\t}\n\tfor i := 0; i < len(publicDNS); i++ {\n\t\tipaddress = (<-result).s\n\t\tif ipaddress != \"\" {\n\t\t\treturn\n\t\t}\n\t}\n\terr = fmt.Errorf(\"failed to resolve %s: all DNS servers exhausted\", address)\n\treturn\n}\n\n// localLookupIP returns a host's IP address based on the local resolver.\nfunc localLookupIP(address string) (ipaddress string, err error) {\n\tip, err := net.LookupHost(address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// remoteLookupIP returns a host's IP address based on a remote DNS server.\nfunc remoteLookupIP(address, dns string) (ipaddress string, err error) {\n\tr := &net.Resolver{\n\t\tPreferGo: true,\n\t\tDial: func(ctx context.Context, network, _ string) (net.Conn, error) {\n\t\t\td := new(net.Dialer)\n\t\t\treturn d.DialContext(ctx, network, dns+\":53\")\n\t\t},\n\t}\n\tip, err := r.LookupHost(context.Background(), address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// Path: src/tcp/tcp_test.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkConnection(b *testing.B) {\n\tlog.SetLevel(\"trace\")\n\tgo Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\", \"8284\")\n\ttime.Sleep(100 * time.Millisecond)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tc, _, _, _ := ConnectToTCPServer(\"127.0.0.1:8283\", \"pass123\", fmt.Sprintf(\"testroom%d\", i), 1*time.Minute)\n\t\tc.Close()\n\t}\n}\n\nfunc TestTCP(t *testing.T) {\n\tlog.SetLevel(\"error\")\n\ttimeToRoomDeletion = 100 * time.Millisecond\n\tgo Run(\"debug\", \"127.0.0.1\", \"8381\", \"pass123\", \"8382\")\n\ttime.Sleep(timeToRoomDeletion)\n\terr := PingServer(\"127.0.0.1:8381\")\n\tassert.Nil(t, err)\n\terr = PingServer(\"127.0.0.1:8333\")\n\tassert.NotNil(t, err)\n\n\ttime.Sleep(timeToRoomDeletion)\n\tc1, banner, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Minute)\n\tassert.Equal(t, banner, \"8382\")\n\tassert.Nil(t, err)\n\tc2, _, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.Nil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.NotNil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Nanosecond)\n\tassert.NotNil(t, err)\n\n\t// try sending data\n\tassert.Nil(t, c1.Send([]byte(\"hello, c2\")))\n\tvar data []byte\n\tfor {\n\t\tdata, err = c2.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c2\"), data)\n\n\tassert.Nil(t, c2.Send([]byte(\"hello, c1\")))\n\tfor {\n\t\tdata, err = c1.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c1\"), data)\n\n\tc1.Close()\n\ttime.Sleep(300 * time.Millisecond)\n}\n\n// Path: src/utils/utils_test.go\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst TCP_BUFFER_SIZE = 1024 * 64\n\nvar bigFileSize = 75000000\n\nfunc bigFile() {\n\tos.WriteFile(\"bigfile.test\", bytes.Repeat([]byte(\"z\"), bigFileSize), 0o666)\n}\n\nfunc BenchmarkMD5(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMD5HashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkXXHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tXXHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHashFull(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFileFull(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkSha256(b *testing.B) {\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tSHA256(\"hello,world\")\n\t}\n}\n\nfunc BenchmarkMissingChunks(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMissingChunks(\"bigfile.test\", int64(bigFileSize), TCP_BUFFER_SIZE/2)\n\t}\n}\n\nfunc TestExists(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tfmt.Println(GetLocalIPs())\n\tassert.True(t, Exists(\"bigfile.test\"))\n\tassert.False(t, Exists(\"doesnotexist\"))\n}\n\nfunc TestMD5HashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := MD5HashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"8304ff018e02baad0e3555bade29a405\", fmt.Sprintf(\"%x\", b))\n\t_, err = MD5HashFile(\"bigfile.test.nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestIMOHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := IMOHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"c0d1e123ca94148ffea146137684ebb9\", fmt.Sprintf(\"%x\", b))\n}\n\nfunc TestXXHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := XXHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"4918740eb5ccb6f7\", fmt.Sprintf(\"%x\", b))\n\t_, err = XXHashFile(\"nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestSHA256(t *testing.T) {\n\tassert.Equal(t, \"09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b\", SHA256(\"hello, world\"))\n}\n\nfunc TestByteCountDecimal(t *testing.T) {\n\tassert.Equal(t, \"10.0 kB\", ByteCountDecimal(10240))\n\tassert.Equal(t, \"50 B\", ByteCountDecimal(50))\n\tassert.Equal(t, \"12.4 MB\", ByteCountDecimal(13002343))\n}\n\nfunc TestMissingChunks(t *testing.T) {\n\tfileSize := 100\n\tchunkSize := 10\n\trand.Seed(1)\n\tbigBuff := make([]byte, fileSize)\n\trand.Read(bigBuff)\n\tos.WriteFile(\"missing.test\", bigBuff, 0o644)\n\tempty := make([]byte, chunkSize)\n\tf, err := os.OpenFile(\"missing.test\", os.O_RDWR, 0o644)\n\tassert.Nil(t, err)\n\tfor block := 0; block < fileSize/chunkSize; block++ {\n\t\tif block == 0 || block == 4 || block == 5 || block >= 7 {\n\t\t\tf.WriteAt(empty, int64(block*chunkSize))\n\t\t}\n\t}\n\tf.Close()\n\n\tchunkRanges := MissingChunks(\"missing.test\", int64(fileSize), chunkSize)\n\tassert.Equal(t, []int64{10, 0, 1, 40, 2, 70, 3}, chunkRanges)\n\n\tchunks := ChunkRangesToChunks(chunkRanges)\n\tassert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks)\n\n\tos.Remove(\"missing.test\")\n\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err := tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err := tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\tchunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)+10), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunkRanges = MissingChunks(tmpfile.Name()+\"ok\", int64(len(content)), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunks = ChunkRangesToChunks(chunkRanges)\n\tassert.Empty(t, chunks)\n}\n\n// func Test1(t *testing.T) {\n// \tchunkRanges := MissingChunks(\"../../m/bigfile.test\", int64(75000000), 1024*64/2)\n// \tfmt.Println(chunkRanges)\n// \tfmt.Println(ChunkRangesToChunks((chunkRanges)))\n// \tassert.Nil(t, nil)\n// }\n\nfunc TestHashFile(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err = tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err = tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\thashed, err := HashFile(tmpfile.Name(), \"xxhash\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"e66c561610ad51e2\", fmt.Sprintf(\"%x\", hashed))\n}\n\nfunc TestPublicIP(t *testing.T) {\n\tip, err := PublicIP()\n\tfmt.Println(ip)\n\tassert.True(t, strings.Contains(ip, \".\") || strings.Contains(ip, \":\"))\n\tassert.Nil(t, err)\n}\n\nfunc TestLocalIP(t *testing.T) {\n\tip := LocalIP()\n\tfmt.Println(ip)\n\tassert.True(t, strings.Contains(ip, \".\") || strings.Contains(ip, \":\"))\n}\n\nfunc TestGetRandomName(t *testing.T) {\n\tname := GetRandomName()\n\tfmt.Println(name)\n\tassert.NotEmpty(t, name)\n}\n\nfunc TestFindOpenPorts(t *testing.T) {\n\topenPorts := FindOpenPorts(\"127.0.0.1\", 9009, 4)\n\tassert.Equal(t, []int{9009, 9010, 9011, 9012}, openPorts)\n}\n\nfunc TestIsLocalIP(t *testing.T) {\n\tassert.True(t, IsLocalIP(\"192.168.0.14:9009\"))\n}\n\n", "needle_token_start": 5205, "needle_token_end": 5308, "code_context_ntokens": 11225, "output": ["```go\n// GetStringInBetween Returns empty string if no start string found\nfunc GetStringInBetween(str string, start string, end string) (result string) {\n\ts := strings.Index(str, start)\n\tif s == -1 {\n\t\treturn\n\t}\n\ts += len(start)\n\te := strings.Index(str[s:], end)\n\tif e == -1 {\n\t\treturn\n\t}\n\te += s\n\treturn str[s:e]\n}\n```"]}
{"repo": "schollz/croc", "name": "ZipDirectory", "language": "go", "path": "src/utils/utils.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to compress a specified directory into a ZIP file, ensuring that the directory's structure is maintained in the archive without applying additional compression to the contents.\n2. **Input**: It takes two string parameters: the first specifies the path where the ZIP file will be created, and the second specifies the path of the directory to be compressed.\n3. **Output**: The function does not return any value but will output a ZIP file at the specified destination. It logs progress and errors directly to the standard error stream.\n4. **Procedure**: The function first checks if the output file already exists and logs an error if so. It then creates a new ZIP file at the specified destination and iterates over each file in the source directory. Each file is added to the ZIP file with its relative path preserved. The function logs the addition of each file to the standard error. If an error occurs during the file walking or file writing process, it logs the error and halts execution.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/crypt/crypt.go\npackage crypt\n\nimport (\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"golang.org/x/crypto/argon2\"\n\t\"golang.org/x/crypto/chacha20poly1305\"\n\t\"golang.org/x/crypto/pbkdf2\"\n)\n\n// New generates a new key based on a passphrase and salt\nfunc New(passphrase []byte, usersalt []byte) (key []byte, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err := rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\tkey = pbkdf2.Key(passphrase, salt, 100, 32, sha256.New)\n\treturn\n}\n\n// Encrypt will encrypt using the pre-generated key\nfunc Encrypt(plaintext []byte, key []byte) (encrypted []byte, err error) {\n\t// generate a random iv each time\n\t// http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf\n\t// Section 8.2\n\tivBytes := make([]byte, 12)\n\tif _, err = rand.Read(ivBytes); err != nil {\n\t\tlog.Fatalf(\"can't initialize crypto: %v\", err)\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tencrypted = aesgcm.Seal(nil, ivBytes, plaintext, nil)\n\tencrypted = append(ivBytes, encrypted...)\n\treturn\n}\n\n// Decrypt using the pre-generated key\nfunc Decrypt(encrypted []byte, key []byte) (plaintext []byte, err error) {\n\tif len(encrypted) < 13 {\n\t\terr = fmt.Errorf(\"incorrect passphrase\")\n\t\treturn\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tplaintext, err = aesgcm.Open(nil, encrypted[:12], encrypted[12:], nil)\n\treturn\n}\n\n// NewArgon2 generates a new key based on a passphrase and salt\n// using argon2\n// https://pkg.go.dev/golang.org/x/crypto/argon2\nfunc NewArgon2(passphrase []byte, usersalt []byte) (aead cipher.AEAD, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err = rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\taead, err = chacha20poly1305.NewX(argon2.IDKey(passphrase, salt, 1, 64*1024, 4, 32))\n\treturn\n}\n\n// EncryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc EncryptChaCha(plaintext []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tnonce := make([]byte, aead.NonceSize(), aead.NonceSize()+len(plaintext)+aead.Overhead())\n\tif _, err := rand.Read(nonce); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Encrypt the message and append the ciphertext to the nonce.\n\tencrypted = aead.Seal(nonce, nonce, plaintext, nil)\n\treturn\n}\n\n// DecryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc DecryptChaCha(encryptedMsg []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tif len(encryptedMsg) < aead.NonceSize() {\n\t\terr = fmt.Errorf(\"ciphertext too short\")\n\t\treturn\n\t}\n\n\t// Split nonce and ciphertext.\n\tnonce, ciphertext := encryptedMsg[:aead.NonceSize()], encryptedMsg[aead.NonceSize():]\n\n\t// Decrypt the message and check it wasn't tampered with.\n\tencrypted, err = aead.Open(nil, nonce, ciphertext, nil)\n\treturn\n}\n\n// Path: src/compress/compress.go\npackage compress\n\nimport (\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"io\"\n\n\tlog \"github.com/schollz/logger\"\n)\n\n// CompressWithOption returns compressed data using the specified level\nfunc CompressWithOption(src []byte, level int) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, level)\n\treturn compressedData.Bytes()\n}\n\n// Compress returns a compressed byte slice.\nfunc Compress(src []byte) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, -2)\n\treturn compressedData.Bytes()\n}\n\n// Decompress returns a decompressed byte slice.\nfunc Decompress(src []byte) []byte {\n\tcompressedData := bytes.NewBuffer(src)\n\tdeCompressedData := new(bytes.Buffer)\n\tdecompress(compressedData, deCompressedData)\n\treturn deCompressedData.Bytes()\n}\n\n// compress uses flate to compress a byte slice to a corresponding level\nfunc compress(src []byte, dest io.Writer, level int) {\n\tcompressor, err := flate.NewWriter(dest, level)\n\tif err != nil {\n\t\tlog.Debugf(\"error level data: %v\", err)\n\t\treturn\n\t}\n\tif _, err := compressor.Write(src); err != nil {\n\t\tlog.Debugf(\"error writing data: %v\", err)\n\t}\n\tcompressor.Close()\n}\n\n// compress uses flate to decompress an io.Reader\nfunc decompress(src io.Reader, dest io.Writer) {\n\tdecompressor := flate.NewReader(src)\n\tif _, err := io.Copy(dest, decompressor); err != nil {\n\t\tlog.Debugf(\"error copying data: %v\", err)\n\t}\n\tdecompressor.Close()\n}\n\n// Path: src/utils/utils.go\npackage utils\n\nimport (\n\t\"archive/zip\"\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"crypto/md5\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math\"\n\t\"math/big\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cespare/xxhash\"\n\t\"github.com/kalafut/imohash\"\n\t\"github.com/schollz/mnemonicode\"\n)\n\nconst NbPinNumbers = 4\nconst NbBytesWords = 4\n\n// Get or create home directory\nfunc GetConfigDir(requireValidPath bool) (homedir string, err error) {\n\tif envHomedir, isSet := os.LookupEnv(\"CROC_CONFIG_DIR\"); isSet {\n\t\thomedir = envHomedir\n\t} else if xdgConfigHome, isSet := os.LookupEnv(\"XDG_CONFIG_HOME\"); isSet {\n\t\thomedir = path.Join(xdgConfigHome, \"croc\")\n\t} else {\n\t\thomedir, err = os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tif !requireValidPath {\n\t\t\t\terr = nil\n\t\t\t\thomedir = \"\"\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\thomedir = path.Join(homedir, \".config\", \"croc\")\n\t}\n\n\tif requireValidPath {\n\t\tif _, err = os.Stat(homedir); os.IsNotExist(err) {\n\t\t\terr = os.MkdirAll(homedir, 0o700)\n\t\t}\n\t}\n\treturn\n}\n\n// Exists reports whether the named file or directory exists.\nfunc Exists(name string) bool {\n\tif _, err := os.Stat(name); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// GetInput returns the input with a given prompt\nfunc GetInput(prompt string) string {\n\treader := bufio.NewReader(os.Stdin)\n\tfmt.Fprintf(os.Stderr, \"%s\", prompt)\n\ttext, _ := reader.ReadString('\\n')\n\treturn strings.TrimSpace(text)\n}\n\n// HashFile returns the hash of a file or, in case of a symlink, the\n// SHA256 hash of its target. Takes an argument to specify the algorithm to use.\nfunc HashFile(fname string, algorithm string) (hash256 []byte, err error) {\n\tvar fstats os.FileInfo\n\tfstats, err = os.Lstat(fname)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif fstats.Mode()&os.ModeSymlink != 0 {\n\t\tvar target string\n\t\ttarget, err = os.Readlink(fname)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn []byte(SHA256(target)), nil\n\t}\n\tswitch algorithm {\n\tcase \"imohash\":\n\t\treturn IMOHashFile(fname)\n\tcase \"md5\":\n\t\treturn MD5HashFile(fname)\n\tcase \"xxhash\":\n\t\treturn XXHashFile(fname)\n\t}\n\terr = fmt.Errorf(\"unspecified algorithm\")\n\treturn\n}\n\n// MD5HashFile returns MD5 hash\nfunc MD5HashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := md5.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// IMOHashFile returns imohash\nfunc IMOHashFile(fname string) (hash []byte, err error) {\n\tb, err := imohash.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\nvar imofull = imohash.NewCustom(0, 0)\n\n// IMOHashFileFull returns imohash of full file\nfunc IMOHashFileFull(fname string) (hash []byte, err error) {\n\tb, err := imofull.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\n// XXHashFile returns the xxhash of a file\nfunc XXHashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := xxhash.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// SHA256 returns sha256 sum\nfunc SHA256(s string) string {\n\tsha := sha256.New()\n\tsha.Write([]byte(s))\n\treturn hex.EncodeToString(sha.Sum(nil))\n}\n\n// PublicIP returns public ip address\nfunc PublicIP() (ip string, err error) {\n\tresp, err := http.Get(\"https://canhazip.com\")\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode == http.StatusOK {\n\t\tbodyBytes, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tip = strings.TrimSpace(string(bodyBytes))\n\t}\n\treturn\n}\n\n// LocalIP returns local ip address\nfunc LocalIP() string {\n\tconn, err := net.Dial(\"udp\", \"8.8.8.8:80\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tlocalAddr := conn.LocalAddr().(*net.UDPAddr)\n\n\treturn localAddr.IP.String()\n}\n\nfunc GenerateRandomPin() string {\n\ts := \"\"\n\tmax := new(big.Int)\n\tmax.SetInt64(9)\n\tfor i := 0; i < NbPinNumbers; i++ {\n\t\tv, err := rand.Int(rand.Reader, max)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\ts += fmt.Sprintf(\"%d\", v)\n\t}\n\treturn s\n}\n\n// GetRandomName returns mnemonicoded random name\nfunc GetRandomName() string {\n\tvar result []string\n\tbs := make([]byte, NbBytesWords)\n\trand.Read(bs)\n\tresult = mnemonicode.EncodeWordList(result, bs)\n\treturn GenerateRandomPin() + \"-\" + strings.Join(result, \"-\")\n}\n\n// ByteCountDecimal converts bytes to human readable byte string\nfunc ByteCountDecimal(b int64) string {\n\tconst unit = 1024\n\tif b < unit {\n\t\treturn fmt.Sprintf(\"%d B\", b)\n\t}\n\tdiv, exp := int64(unit), 0\n\tfor n := b / unit; n >= unit; n /= unit {\n\t\tdiv *= unit\n\t\texp++\n\t}\n\treturn fmt.Sprintf(\"%.1f %cB\", float64(b)/float64(div), \"kMGTPE\"[exp])\n}\n\n// MissingChunks returns the positions of missing chunks.\n// If file doesn't exist, it returns an empty chunk list (all chunks).\n// If the file size is not the same as requested, it returns an empty chunk list (all chunks).\nfunc MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\tfstat, err := os.Stat(fname)\n\tif err != nil || fstat.Size() != fsize {\n\t\treturn\n\t}\n\n\temptyBuffer := make([]byte, chunkSize)\n\tchunkNum := 0\n\tchunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))\n\tvar currentLocation int64\n\tfor {\n\t\tbuffer := make([]byte, chunkSize)\n\t\tbytesread, err := f.Read(buffer)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tif bytes.Equal(buffer[:bytesread], emptyBuffer[:bytesread]) {\n\t\t\tchunks[chunkNum] = currentLocation\n\t\t\tchunkNum++\n\t\t}\n\t\tcurrentLocation += int64(bytesread)\n\t}\n\tif chunkNum == 0 {\n\t\tchunkRanges = []int64{}\n\t} else {\n\t\tchunks = chunks[:chunkNum]\n\t\tchunkRanges = []int64{int64(chunkSize), chunks[0]}\n\t\tcurCount := 0\n\t\tfor i, chunk := range chunks {\n\t\t\tif i == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurCount++\n\t\t\tif chunk-chunks[i-1] > int64(chunkSize) {\n\t\t\t\tchunkRanges = append(chunkRanges, int64(curCount))\n\t\t\t\tchunkRanges = append(chunkRanges, chunk)\n\t\t\t\tcurCount = 0\n\t\t\t}\n\t\t}\n\t\tchunkRanges = append(chunkRanges, int64(curCount+1))\n\t}\n\treturn\n}\n\n// ChunkRangesToChunks converts chunk ranges to list\nfunc ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) {\n\tif len(chunkRanges) == 0 {\n\t\treturn\n\t}\n\tchunkSize := chunkRanges[0]\n\tchunks = []int64{}\n\tfor i := 1; i < len(chunkRanges); i += 2 {\n\t\tfor j := int64(0); j < (chunkRanges[i+1]); j++ {\n\t\t\tchunks = append(chunks, chunkRanges[i]+j*chunkSize)\n\t\t}\n\t}\n\treturn\n}\n\n// GetLocalIPs returns all local ips\nfunc GetLocalIPs() (ips []string, err error) {\n\taddrs, err := net.InterfaceAddrs()\n\tif err != nil {\n\t\treturn\n\t}\n\tips = []string{}\n\tfor _, address := range addrs {\n\t\t// check the address type and if it is not a loopback the display it\n\t\tif ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {\n\t\t\tif ipnet.IP.To4() != nil {\n\t\t\t\tips = append(ips, ipnet.IP.String())\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\nfunc RandomFileName() (fname string, err error) {\n\tf, err := os.CreateTemp(\".\", \"croc-stdin-\")\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = f.Name()\n\t_ = f.Close()\n\treturn\n}\n\nfunc FindOpenPorts(host string, portNumStart, numPorts int) (openPorts []int) {\n\topenPorts = []int{}\n\tfor port := portNumStart; port-portNumStart < 200; port++ {\n\t\ttimeout := 100 * time.Millisecond\n\t\tconn, err := net.DialTimeout(\"tcp\", net.JoinHostPort(host, fmt.Sprint(port)), timeout)\n\t\tif conn != nil {\n\t\t\tconn.Close()\n\t\t} else if err != nil {\n\t\t\topenPorts = append(openPorts, port)\n\t\t}\n\t\tif len(openPorts) >= numPorts {\n\t\t\treturn\n\t\t}\n\t}\n\treturn\n}\n\n// local ip determination\n// https://stackoverflow.com/questions/41240761/check-if-ip-address-is-in-private-network-space\nvar privateIPBlocks []*net.IPNet\n\nfunc init() {\n\tfor _, cidr := range []string{\n\t\t\"127.0.0.0/8\",    // IPv4 loopback\n\t\t\"10.0.0.0/8\",     // RFC1918\n\t\t\"172.16.0.0/12\",  // RFC1918\n\t\t\"192.168.0.0/16\", // RFC1918\n\t\t\"169.254.0.0/16\", // RFC3927 link-local\n\t\t\"::1/128\",        // IPv6 loopback\n\t\t\"fe80::/10\",      // IPv6 link-local\n\t\t\"fc00::/7\",       // IPv6 unique local addr\n\t} {\n\t\t_, block, err := net.ParseCIDR(cidr)\n\t\tif err != nil {\n\t\t\tpanic(fmt.Errorf(\"parse error on %q: %v\", cidr, err))\n\t\t}\n\t\tprivateIPBlocks = append(privateIPBlocks, block)\n\t}\n}\n\nfunc IsLocalIP(ipaddress string) bool {\n\tif strings.Contains(ipaddress, \"127.0.0.1\") {\n\t\treturn true\n\t}\n\thost, _, _ := net.SplitHostPort(ipaddress)\n\tip := net.ParseIP(host)\n\tif ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {\n\t\treturn true\n\t}\n\tfor _, block := range privateIPBlocks {\n\t\tif block.Contains(ip) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\n\nfunc ZipDirectory(destination string, source string) (err error) {\n\tif _, err = os.Stat(destination); err == nil {\n\t\tlog.Fatalf(\"%s file already exists!\\n\", destination)\n\t}\n\tfmt.Fprintf(os.Stderr, \"Zipping %s to %s\\n\", source, destination)\n\tfile, err := os.Create(destination)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tdefer file.Close()\n\twriter := zip.NewWriter(file)\n\t// no compression because croc does its compression on the fly\n\twriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {\n\t\treturn flate.NewWriter(out, flate.NoCompression)\n\t})\n\tdefer writer.Close()\n\terr = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\t\tif info.Mode().IsRegular() {\n\t\t\tf1, err := os.Open(path)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tdefer f1.Close()\n\t\t\tzipPath := strings.ReplaceAll(path, source, strings.TrimSuffix(destination, \".zip\"))\n\t\t\tzipPath = filepath.ToSlash(zipPath)\n\t\t\tw1, err := writer.Create(zipPath)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tif _, err := io.Copy(w1, f1); err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tfmt.Fprintf(os.Stderr, \"\\r\\033[2K\")\n\t\t\tfmt.Fprintf(os.Stderr, \"\\rAdding %s\", zipPath)\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tfmt.Fprintf(os.Stderr, \"\\n\")\n\treturn nil\n}\n\nfunc UnzipDirectory(destination string, source string) error {\n\tarchive, err := zip.OpenReader(source)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tdefer archive.Close()\n\n\tfor _, f := range archive.File {\n\t\tfilePath := filepath.Join(destination, f.Name)\n\t\tfmt.Fprintf(os.Stderr, \"\\r\\033[2K\")\n\t\tfmt.Fprintf(os.Stderr, \"\\rUnzipping file %s\", filePath)\n\t\tif f.FileInfo().IsDir() {\n\t\t\tos.MkdirAll(filePath, os.ModePerm)\n\t\t\tcontinue\n\t\t}\n\n\t\tif err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tdstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tfileInArchive, err := f.Open()\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tif _, err := io.Copy(dstFile, fileInArchive); err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tdstFile.Close()\n\t\tfileInArchive.Close()\n\t}\n\tfmt.Fprintf(os.Stderr, \"\\n\")\n\treturn nil\n}\n\n// Path: src/comm/comm.go\npackage comm\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/magisterquis/connectproxy\"\n\t\"github.com/schollz/croc/v9/src/utils\"\n\tlog \"github.com/schollz/logger\"\n\t\"golang.org/x/net/proxy\"\n)\n\nvar Socks5Proxy = \"\"\nvar HttpProxy = \"\"\n\nvar MAGIC_BYTES = []byte(\"croc\")\n\n// Comm is some basic TCP communication\ntype Comm struct {\n\tconnection net.Conn\n}\n\n// NewConnection gets a new comm to a tcp address\nfunc NewConnection(address string, timelimit ...time.Duration) (c *Comm, err error) {\n\ttlimit := 30 * time.Second\n\tif len(timelimit) > 0 {\n\t\ttlimit = timelimit[0]\n\t}\n\tvar connection net.Conn\n\tif Socks5Proxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(Socks5Proxy, `://`) {\n\t\t\tSocks5Proxy = `socks5://` + Socks5Proxy\n\t\t}\n\t\tsocks5ProxyURL, urlParseError := url.Parse(Socks5Proxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse socks proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = proxy.FromURL(socks5ProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\t} else if HttpProxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(HttpProxy, `://`) {\n\t\t\tHttpProxy = `http://` + HttpProxy\n\t\t}\n\t\tHttpProxyURL, urlParseError := url.Parse(HttpProxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse http proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = connectproxy.New(HttpProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\n\t} else {\n\t\tlog.Debugf(\"dialing to %s with timelimit %s\", address, tlimit)\n\t\tconnection, err = net.DialTimeout(\"tcp\", address, tlimit)\n\t}\n\tif err != nil {\n\t\terr = fmt.Errorf(\"comm.NewConnection failed: %w\", err)\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tc = New(connection)\n\tlog.Debugf(\"connected to '%s'\", address)\n\treturn\n}\n\n// New returns a new comm\nfunc New(c net.Conn) *Comm {\n\tif err := c.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\tif err := c.SetDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting overall deadline: %v\", err)\n\t}\n\tif err := c.SetWriteDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Errorf(\"error setting write deadline: %v\", err)\n\t}\n\tcomm := new(Comm)\n\tcomm.connection = c\n\treturn comm\n}\n\n// Connection returns the net.Conn connection\nfunc (c *Comm) Connection() net.Conn {\n\treturn c.connection\n}\n\n// Close closes the connection\nfunc (c *Comm) Close() {\n\tif err := c.connection.Close(); err != nil {\n\t\tlog.Warnf(\"error closing connection: %v\", err)\n\t}\n}\n\nfunc (c *Comm) Write(b []byte) (n int, err error) {\n\theader := new(bytes.Buffer)\n\terr = binary.Write(header, binary.LittleEndian, uint32(len(b)))\n\tif err != nil {\n\t\tfmt.Println(\"binary.Write failed:\", err)\n\t}\n\ttmpCopy := append(header.Bytes(), b...)\n\ttmpCopy = append(MAGIC_BYTES, tmpCopy...)\n\tn, err = c.connection.Write(tmpCopy)\n\tif err != nil {\n\t\terr = fmt.Errorf(\"connection.Write failed: %w\", err)\n\t\treturn\n\t}\n\tif n != len(tmpCopy) {\n\t\terr = fmt.Errorf(\"wanted to write %d but wrote %d\", len(b), n)\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (c *Comm) Read() (buf []byte, numBytes int, bs []byte, err error) {\n\t// long read deadline in case waiting for file\n\tif err = c.connection.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\t// must clear the timeout setting\n\tdefer c.connection.SetDeadline(time.Time{})\n\n\t// read until we get 4 bytes for the magic\n\theader := make([]byte, 4)\n\t_, err = io.ReadFull(c.connection, header)\n\tif err != nil {\n\t\tlog.Debugf(\"initial read error: %v\", err)\n\t\treturn\n\t}\n\tif !bytes.Equal(header, MAGIC_BYTES) {\n\t\terr = fmt.Errorf(\"initial bytes are not magic: %x\", header)\n\t\treturn\n\t}\n\n\t// read until we get 4 bytes for the header\n\theader = make([]byte, 4)\n\t_, err = io.ReadFull(c.connection, header)\n\tif err != nil {\n\t\tlog.Debugf(\"initial read error: %v\", err)\n\t\treturn\n\t}\n\n\tvar numBytesUint32 uint32\n\trbuf := bytes.NewReader(header)\n\terr = binary.Read(rbuf, binary.LittleEndian, &numBytesUint32)\n\tif err != nil {\n\t\terr = fmt.Errorf(\"binary.Read failed: %w\", err)\n\t\tlog.Debug(err.Error())\n\t\treturn\n\t}\n\tnumBytes = int(numBytesUint32)\n\n\t// shorten the reading deadline in case getting weird data\n\tif err = c.connection.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\tbuf = make([]byte, numBytes)\n\t_, err = io.ReadFull(c.connection, buf)\n\tif err != nil {\n\t\tlog.Debugf(\"consecutive read error: %v\", err)\n\t\treturn\n\t}\n\treturn\n}\n\n// Send a message\nfunc (c *Comm) Send(message []byte) (err error) {\n\t_, err = c.Write(message)\n\treturn\n}\n\n// Receive a message\nfunc (c *Comm) Receive() (b []byte, err error) {\n\tb, _, _, err = c.Read()\n\treturn\n}\n\n// Path: src/tcp/tcp.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/schollz/pake/v3\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\t\"github.com/schollz/croc/v9/src/models\"\n)\n\ntype server struct {\n\thost       string\n\tport       string\n\tdebugLevel string\n\tbanner     string\n\tpassword   string\n\trooms      roomMap\n}\n\ntype roomInfo struct {\n\tfirst  *comm.Comm\n\tsecond *comm.Comm\n\topened time.Time\n\tfull   bool\n}\n\ntype roomMap struct {\n\trooms map[string]roomInfo\n\tsync.Mutex\n}\n\nconst pingRoom = \"pinglkasjdlfjsaldjf\"\n\nvar timeToRoomDeletion = 10 * time.Minute\n\n// Run starts a tcp listener, run async\nfunc Run(debugLevel, host, port, password string, banner ...string) (err error) {\n\ts := new(server)\n\ts.host = host\n\ts.port = port\n\ts.password = password\n\ts.debugLevel = debugLevel\n\tif len(banner) > 0 {\n\t\ts.banner = banner[0]\n\t}\n\treturn s.start()\n}\n\nfunc (s *server) start() (err error) {\n\tlog.SetLevel(s.debugLevel)\n\tlog.Debugf(\"starting with password '%s'\", s.password)\n\ts.rooms.Lock()\n\ts.rooms.rooms = make(map[string]roomInfo)\n\ts.rooms.Unlock()\n\n\t// delete old rooms\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(timeToRoomDeletion)\n\t\t\tvar roomsToDelete []string\n\t\t\ts.rooms.Lock()\n\t\t\tfor room := range s.rooms.rooms {\n\t\t\t\tif time.Since(s.rooms.rooms[room].opened) > 3*time.Hour {\n\t\t\t\t\troomsToDelete = append(roomsToDelete, room)\n\t\t\t\t}\n\t\t\t}\n\t\t\ts.rooms.Unlock()\n\n\t\t\tfor _, room := range roomsToDelete {\n\t\t\t\ts.deleteRoom(room)\n\t\t\t}\n\t\t}\n\t}()\n\n\terr = s.run()\n\tif err != nil {\n\t\tlog.Error(err)\n\t}\n\treturn\n}\n\nfunc (s *server) run() (err error) {\n\tnetwork := \"tcp\"\n\taddr := net.JoinHostPort(s.host, s.port)\n\tif s.host != \"\" {\n\t\tip := net.ParseIP(s.host)\n\t\tif ip == nil {\n\t\t\tvar tcpIP *net.IPAddr\n\t\t\ttcpIP, err = net.ResolveIPAddr(\"ip\", s.host)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tip = tcpIP.IP\n\t\t}\n\t\taddr = net.JoinHostPort(ip.String(), s.port)\n\t\tif s.host != \"\" {\n\t\t\tif ip.To4() != nil {\n\t\t\t\tnetwork = \"tcp4\"\n\t\t\t} else {\n\t\t\t\tnetwork = \"tcp6\"\n\t\t\t}\n\t\t}\n\t}\n\taddr = strings.Replace(addr, \"127.0.0.1\", \"0.0.0.0\", 1)\n\tlog.Infof(\"starting TCP server on \" + addr)\n\tserver, err := net.Listen(network, addr)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error listening on %s: %w\", addr, err)\n\t}\n\tdefer server.Close()\n\t// spawn a new goroutine whenever a client connects\n\tfor {\n\t\tconnection, err := server.Accept()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"problem accepting connection: %w\", err)\n\t\t}\n\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\tgo func(port string, connection net.Conn) {\n\t\t\tc := comm.New(connection)\n\t\t\troom, errCommunication := s.clientCommunication(port, c)\n\t\t\tlog.Debugf(\"room: %+v\", room)\n\t\t\tlog.Debugf(\"err: %+v\", errCommunication)\n\t\t\tif errCommunication != nil {\n\t\t\t\tlog.Debugf(\"relay-%s: %s\", connection.RemoteAddr().String(), errCommunication.Error())\n\t\t\t\tconnection.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif room == pingRoom {\n\t\t\t\tlog.Debugf(\"got ping\")\n\t\t\t\tconnection.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor {\n\t\t\t\t// check connection\n\t\t\t\tlog.Debugf(\"checking connection of room %s for %+v\", room, c)\n\t\t\t\tdeleteIt := false\n\t\t\t\ts.rooms.Lock()\n\t\t\t\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\t\t\t\tlog.Debug(\"room is gone\")\n\t\t\t\t\ts.rooms.Unlock()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tlog.Debugf(\"room: %+v\", s.rooms.rooms[room])\n\t\t\t\tif s.rooms.rooms[room].first != nil && s.rooms.rooms[room].second != nil {\n\t\t\t\t\tlog.Debug(\"rooms ready\")\n\t\t\t\t\ts.rooms.Unlock()\n\t\t\t\t\tbreak\n\t\t\t\t} else {\n\t\t\t\t\tif s.rooms.rooms[room].first != nil {\n\t\t\t\t\t\terrSend := s.rooms.rooms[room].first.Send([]byte{1})\n\t\t\t\t\t\tif errSend != nil {\n\t\t\t\t\t\t\tlog.Debug(errSend)\n\t\t\t\t\t\t\tdeleteIt = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.rooms.Unlock()\n\t\t\t\tif deleteIt {\n\t\t\t\t\ts.deleteRoom(room)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\ttime.Sleep(1 * time.Second)\n\t\t\t}\n\t\t}(s.port, connection)\n\t}\n}\n\nvar weakKey = []byte{1, 2, 3}\n\nfunc (s *server) clientCommunication(port string, c *comm.Comm) (room string, err error) {\n\t// establish secure password with PAKE for communication with relay\n\tB, err := pake.InitCurve(weakKey, 1, \"siec\")\n\tif err != nil {\n\t\treturn\n\t}\n\tAbytes, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tlog.Debugf(\"Abytes: %s\", Abytes)\n\tif bytes.Equal(Abytes, []byte(\"ping\")) {\n\t\troom = pingRoom\n\t\tlog.Debug(\"sending back pong\")\n\t\tc.Send([]byte(\"pong\"))\n\t\treturn\n\t}\n\terr = B.Update(Abytes)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(B.Bytes())\n\tif err != nil {\n\t\treturn\n\t}\n\tstrongKey, err := B.SessionKey()\n\tif err != nil {\n\t\treturn\n\t}\n\tlog.Debugf(\"strongkey: %x\", strongKey)\n\n\t// receive salt\n\tsalt, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tstrongKeyForEncryption, _, err := crypt.New(strongKey, salt)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tlog.Debugf(\"waiting for password\")\n\tpasswordBytesEnc, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tpasswordBytes, err := crypt.Decrypt(passwordBytesEnc, strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\tif strings.TrimSpace(string(passwordBytes)) != s.password {\n\t\terr = fmt.Errorf(\"bad password\")\n\t\tenc, _ := crypt.Encrypt([]byte(err.Error()), strongKeyForEncryption)\n\t\tif err = c.Send(enc); err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"send error: %w\", err)\n\t\t}\n\t\treturn\n\t}\n\n\t// send ok to tell client they are connected\n\tbanner := s.banner\n\tif len(banner) == 0 {\n\t\tbanner = \"ok\"\n\t}\n\tlog.Debugf(\"sending '%s'\", banner)\n\tbSend, err := crypt.Encrypt([]byte(banner+\"|||\"+c.Connection().RemoteAddr().String()), strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// wait for client to tell me which room they want\n\tlog.Debug(\"waiting for answer\")\n\tenc, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\troomBytes, err := crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\troom = string(roomBytes)\n\n\ts.rooms.Lock()\n\t// create the room if it is new\n\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\ts.rooms.rooms[room] = roomInfo{\n\t\t\tfirst:  c,\n\t\t\topened: time.Now(),\n\t\t}\n\t\ts.rooms.Unlock()\n\t\t// tell the client that they got the room\n\n\t\tbSend, err = crypt.Encrypt([]byte(\"ok\"), strongKeyForEncryption)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\terr = c.Send(bSend)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\ts.deleteRoom(room)\n\t\t\treturn\n\t\t}\n\t\tlog.Debugf(\"room %s has 1\", room)\n\t\treturn\n\t}\n\tif s.rooms.rooms[room].full {\n\t\ts.rooms.Unlock()\n\t\tbSend, err = crypt.Encrypt([]byte(\"room full\"), strongKeyForEncryption)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\terr = c.Send(bSend)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\treturn\n\t}\n\tlog.Debugf(\"room %s has 2\", room)\n\ts.rooms.rooms[room] = roomInfo{\n\t\tfirst:  s.rooms.rooms[room].first,\n\t\tsecond: c,\n\t\topened: s.rooms.rooms[room].opened,\n\t\tfull:   true,\n\t}\n\totherConnection := s.rooms.rooms[room].first\n\ts.rooms.Unlock()\n\n\t// second connection is the sender, time to staple connections\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\t// start piping\n\tgo func(com1, com2 *comm.Comm, wg *sync.WaitGroup) {\n\t\tlog.Debug(\"starting pipes\")\n\t\tpipe(com1.Connection(), com2.Connection())\n\t\twg.Done()\n\t\tlog.Debug(\"done piping\")\n\t}(otherConnection, c, &wg)\n\n\t// tell the sender everything is ready\n\tbSend, err = crypt.Encrypt([]byte(\"ok\"), strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\ts.deleteRoom(room)\n\t\treturn\n\t}\n\twg.Wait()\n\n\t// delete room\n\ts.deleteRoom(room)\n\treturn\n}\n\nfunc (s *server) deleteRoom(room string) {\n\ts.rooms.Lock()\n\tdefer s.rooms.Unlock()\n\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\treturn\n\t}\n\tlog.Debugf(\"deleting room: %s\", room)\n\tif s.rooms.rooms[room].first != nil {\n\t\ts.rooms.rooms[room].first.Close()\n\t}\n\tif s.rooms.rooms[room].second != nil {\n\t\ts.rooms.rooms[room].second.Close()\n\t}\n\ts.rooms.rooms[room] = roomInfo{first: nil, second: nil}\n\tdelete(s.rooms.rooms, room)\n}\n\n// chanFromConn creates a channel from a Conn object, and sends everything it\n//\n//\tRead()s from the socket to the channel.\nfunc chanFromConn(conn net.Conn) chan []byte {\n\tc := make(chan []byte, 1)\n\tif err := conn.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"can't set read deadline: %v\", err)\n\t}\n\n\tgo func() {\n\t\tb := make([]byte, models.TCP_BUFFER_SIZE)\n\t\tfor {\n\t\t\tn, err := conn.Read(b)\n\t\t\tif n > 0 {\n\t\t\t\tres := make([]byte, n)\n\t\t\t\t// Copy the buffer so it doesn't get changed while read by the recipient.\n\t\t\t\tcopy(res, b[:n])\n\t\t\t\tc <- res\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tlog.Debug(err)\n\t\t\t\tc <- nil\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tlog.Debug(\"exiting\")\n\t}()\n\n\treturn c\n}\n\n// pipe creates a full-duplex pipe between the two sockets and\n// transfers data from one to the other.\nfunc pipe(conn1 net.Conn, conn2 net.Conn) {\n\tchan1 := chanFromConn(conn1)\n\tchan2 := chanFromConn(conn2)\n\n\tfor {\n\t\tselect {\n\t\tcase b1 := <-chan1:\n\t\t\tif b1 == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif _, err := conn2.Write(b1); err != nil {\n\t\t\t\tlog.Errorf(\"write error on channel 1: %v\", err)\n\t\t\t}\n\n\t\tcase b2 := <-chan2:\n\t\t\tif b2 == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif _, err := conn1.Write(b2); err != nil {\n\t\t\t\tlog.Errorf(\"write error on channel 2: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc PingServer(address string) (err error) {\n\tlog.Debugf(\"pinging %s\", address)\n\tc, err := comm.NewConnection(address, 300*time.Millisecond)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send([]byte(\"ping\"))\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tb, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif bytes.Equal(b, []byte(\"pong\")) {\n\t\treturn nil\n\t}\n\treturn fmt.Errorf(\"no pong\")\n}\n\n// ConnectToTCPServer will initiate a new connection\n// to the specified address, room with optional time limit\nfunc ConnectToTCPServer(address, password, room string, timelimit ...time.Duration) (c *comm.Comm, banner string, ipaddr string, err error) {\n\tif len(timelimit) > 0 {\n\t\tc, err = comm.NewConnection(address, timelimit[0])\n\t} else {\n\t\tc, err = comm.NewConnection(address)\n\t}\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\n\t// get PAKE connection with server to establish strong key to transfer info\n\tA, err := pake.InitCurve(weakKey, 0, \"siec\")\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(A.Bytes())\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tBbytes, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = A.Update(Bbytes)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tstrongKey, err := A.SessionKey()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debugf(\"strong key: %x\", strongKey)\n\n\tstrongKeyForEncryption, salt, err := crypt.New(strongKey, nil)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\t// send salt\n\terr = c.Send(salt)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\n\tlog.Debug(\"sending password\")\n\tbSend, err := crypt.Encrypt([]byte(password), strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"waiting for first ok\")\n\tenc, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tdata, err := crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif !strings.Contains(string(data), \"|||\") {\n\t\terr = fmt.Errorf(\"bad response: %s\", string(data))\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tbanner = strings.Split(string(data), \"|||\")[0]\n\tipaddr = strings.Split(string(data), \"|||\")[1]\n\tlog.Debug(\"sending room\")\n\tbSend, err = crypt.Encrypt([]byte(room), strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"waiting for room confirmation\")\n\tenc, err = c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tdata, err = crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif !bytes.Equal(data, []byte(\"ok\")) {\n\t\terr = fmt.Errorf(\"got bad response: %s\", data)\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"all set\")\n\treturn\n}\n\n// Path: src/message/message.go\npackage message\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/compress\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n)\n\n// Type is a message type\ntype Type string\n\nconst (\n\tTypePAKE           Type = \"pake\"\n\tTypeExternalIP     Type = \"externalip\"\n\tTypeFinished       Type = \"finished\"\n\tTypeError          Type = \"error\"\n\tTypeCloseRecipient Type = \"close-recipient\"\n\tTypeCloseSender    Type = \"close-sender\"\n\tTypeRecipientReady Type = \"recipientready\"\n\tTypeFileInfo       Type = \"fileinfo\"\n)\n\n// Message is the possible payload for messaging\ntype Message struct {\n\tType    Type   `json:\"t,omitempty\"`\n\tMessage string `json:\"m,omitempty\"`\n\tBytes   []byte `json:\"b,omitempty\"`\n\tBytes2  []byte `json:\"b2,omitempty\"`\n\tNum     int    `json:\"n,omitempty\"`\n}\n\nfunc (m Message) String() string {\n\tb, _ := json.Marshal(m)\n\treturn string(b)\n}\n\n// Send will send out\nfunc Send(c *comm.Comm, key []byte, m Message) (err error) {\n\tmSend, err := Encode(key, m)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(mSend)\n\treturn\n}\n\n// Encode will convert to bytes\nfunc Encode(key []byte, m Message) (b []byte, err error) {\n\tb, err = json.Marshal(m)\n\tif err != nil {\n...\n", "needle_token_start": 5612, "needle_token_end": 6175, "code_context_ntokens": 14147, "output": ["```go\n// ZipDirectory compresses a directory into a ZIP file without additional compression\nfunc ZipDirectory(destination string, source string) (err error) {\n\tif _, err = os.Stat(destination); err == nil {\n\t\tlog.Fatalf(\"%s file already exists!\\n\", destination)\n\t}\n\tfmt.Fprintf(os.Stderr, \"Zipping %s to %s\\n\", source, destination)\n\tfile, err := os.Create(destination)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tdefer file.Close()\n\twriter := zip.NewWriter(file)\n\t// no compression because croc does its compression on the fly\n\twriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {\n\t\treturn flate.NewWriter(out, flate.NoCompression)\n\t})\n\tdefer writer.Close()\n\terr = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\t\tif info.Mode().IsRegular() {\n\t\t\tf1, err := os.Open(path)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tdefer f1.Close()\n\t\t\tzipPath := strings.ReplaceAll(path, source, strings.TrimSuffix(destination, \".zip\"))\n\t\t\tzipPath = filepath.ToSlash(zipPath)\n\t\t\tw1, err := writer.Create(zipPath)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tif _, err := io.Copy(w1, f1); err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tfmt.Fprintf(os.Stderr, \"\\r\\033[2K\")\n\t\t\tfmt.Fprintf(os.Stderr, \"\\rAdding %s\", zipPath)\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tfmt.Fprintf(os.Stderr, \"\\n\")\n\treturn nil\n}\n```"]}
{"repo": "schollz/croc", "name": "NewConnection", "language": "go", "path": "src/comm/comm.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: Establishes a TCP connection to a specified address, optionally using SOCKS5 or HTTP proxies if configured and the target is not a local IP.\n2. **Input**: A string representing the TCP address to connect to and an optional duration specifying the connection timeout.\n3. **Output**: Returns a communication object representing the TCP connection and an error if the connection fails.\n4. **Procedure**: \n   - Determines the timeout for the connection.\n   - Checks if a SOCKS5 proxy is configured and the address is not local, then attempts to connect using the SOCKS5 proxy.\n   - If a SOCKS5 proxy is not used but an HTTP proxy is configured and the address is not local, attempts to connect using the HTTP proxy.\n   - If no proxies are used or applicable, attempts to connect directly to the TCP address with the specified timeout.\n   - If the connection is successful, returns a communication object; otherwise, returns an error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/crypt/crypt.go\npackage crypt\n\nimport (\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"golang.org/x/crypto/argon2\"\n\t\"golang.org/x/crypto/chacha20poly1305\"\n\t\"golang.org/x/crypto/pbkdf2\"\n)\n\n// New generates a new key based on a passphrase and salt\nfunc New(passphrase []byte, usersalt []byte) (key []byte, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err := rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\tkey = pbkdf2.Key(passphrase, salt, 100, 32, sha256.New)\n\treturn\n}\n\n// Encrypt will encrypt using the pre-generated key\nfunc Encrypt(plaintext []byte, key []byte) (encrypted []byte, err error) {\n\t// generate a random iv each time\n\t// http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf\n\t// Section 8.2\n\tivBytes := make([]byte, 12)\n\tif _, err = rand.Read(ivBytes); err != nil {\n\t\tlog.Fatalf(\"can't initialize crypto: %v\", err)\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tencrypted = aesgcm.Seal(nil, ivBytes, plaintext, nil)\n\tencrypted = append(ivBytes, encrypted...)\n\treturn\n}\n\n// Decrypt using the pre-generated key\nfunc Decrypt(encrypted []byte, key []byte) (plaintext []byte, err error) {\n\tif len(encrypted) < 13 {\n\t\terr = fmt.Errorf(\"incorrect passphrase\")\n\t\treturn\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tplaintext, err = aesgcm.Open(nil, encrypted[:12], encrypted[12:], nil)\n\treturn\n}\n\n// NewArgon2 generates a new key based on a passphrase and salt\n// using argon2\n// https://pkg.go.dev/golang.org/x/crypto/argon2\nfunc NewArgon2(passphrase []byte, usersalt []byte) (aead cipher.AEAD, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err = rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\taead, err = chacha20poly1305.NewX(argon2.IDKey(passphrase, salt, 1, 64*1024, 4, 32))\n\treturn\n}\n\n// EncryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc EncryptChaCha(plaintext []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tnonce := make([]byte, aead.NonceSize(), aead.NonceSize()+len(plaintext)+aead.Overhead())\n\tif _, err := rand.Read(nonce); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Encrypt the message and append the ciphertext to the nonce.\n\tencrypted = aead.Seal(nonce, nonce, plaintext, nil)\n\treturn\n}\n\n// DecryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc DecryptChaCha(encryptedMsg []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tif len(encryptedMsg) < aead.NonceSize() {\n\t\terr = fmt.Errorf(\"ciphertext too short\")\n\t\treturn\n\t}\n\n\t// Split nonce and ciphertext.\n\tnonce, ciphertext := encryptedMsg[:aead.NonceSize()], encryptedMsg[aead.NonceSize():]\n\n\t// Decrypt the message and check it wasn't tampered with.\n\tencrypted, err = aead.Open(nil, nonce, ciphertext, nil)\n\treturn\n}\n\n// Path: src/compress/compress.go\npackage compress\n\nimport (\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"io\"\n\n\tlog \"github.com/schollz/logger\"\n)\n\n// CompressWithOption returns compressed data using the specified level\nfunc CompressWithOption(src []byte, level int) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, level)\n\treturn compressedData.Bytes()\n}\n\n// Compress returns a compressed byte slice.\nfunc Compress(src []byte) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, -2)\n\treturn compressedData.Bytes()\n}\n\n// Decompress returns a decompressed byte slice.\nfunc Decompress(src []byte) []byte {\n\tcompressedData := bytes.NewBuffer(src)\n\tdeCompressedData := new(bytes.Buffer)\n\tdecompress(compressedData, deCompressedData)\n\treturn deCompressedData.Bytes()\n}\n\n// compress uses flate to compress a byte slice to a corresponding level\nfunc compress(src []byte, dest io.Writer, level int) {\n\tcompressor, err := flate.NewWriter(dest, level)\n\tif err != nil {\n\t\tlog.Debugf(\"error level data: %v\", err)\n\t\treturn\n\t}\n\tif _, err := compressor.Write(src); err != nil {\n\t\tlog.Debugf(\"error writing data: %v\", err)\n\t}\n\tcompressor.Close()\n}\n\n// compress uses flate to decompress an io.Reader\nfunc decompress(src io.Reader, dest io.Writer) {\n\tdecompressor := flate.NewReader(src)\n\tif _, err := io.Copy(dest, decompressor); err != nil {\n\t\tlog.Debugf(\"error copying data: %v\", err)\n\t}\n\tdecompressor.Close()\n}\n\n// Path: src/utils/utils.go\npackage utils\n\nimport (\n\t\"archive/zip\"\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"crypto/md5\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math\"\n\t\"math/big\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cespare/xxhash\"\n\t\"github.com/kalafut/imohash\"\n\t\"github.com/schollz/mnemonicode\"\n)\n\nconst NbPinNumbers = 4\nconst NbBytesWords = 4\n\n// Get or create home directory\nfunc GetConfigDir(requireValidPath bool) (homedir string, err error) {\n\tif envHomedir, isSet := os.LookupEnv(\"CROC_CONFIG_DIR\"); isSet {\n\t\thomedir = envHomedir\n\t} else if xdgConfigHome, isSet := os.LookupEnv(\"XDG_CONFIG_HOME\"); isSet {\n\t\thomedir = path.Join(xdgConfigHome, \"croc\")\n\t} else {\n\t\thomedir, err = os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tif !requireValidPath {\n\t\t\t\terr = nil\n\t\t\t\thomedir = \"\"\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\thomedir = path.Join(homedir, \".config\", \"croc\")\n\t}\n\n\tif requireValidPath {\n\t\tif _, err = os.Stat(homedir); os.IsNotExist(err) {\n\t\t\terr = os.MkdirAll(homedir, 0o700)\n\t\t}\n\t}\n\treturn\n}\n\n// Exists reports whether the named file or directory exists.\nfunc Exists(name string) bool {\n\tif _, err := os.Stat(name); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// GetInput returns the input with a given prompt\nfunc GetInput(prompt string) string {\n\treader := bufio.NewReader(os.Stdin)\n\tfmt.Fprintf(os.Stderr, \"%s\", prompt)\n\ttext, _ := reader.ReadString('\\n')\n\treturn strings.TrimSpace(text)\n}\n\n// HashFile returns the hash of a file or, in case of a symlink, the\n// SHA256 hash of its target. Takes an argument to specify the algorithm to use.\nfunc HashFile(fname string, algorithm string) (hash256 []byte, err error) {\n\tvar fstats os.FileInfo\n\tfstats, err = os.Lstat(fname)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif fstats.Mode()&os.ModeSymlink != 0 {\n\t\tvar target string\n\t\ttarget, err = os.Readlink(fname)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn []byte(SHA256(target)), nil\n\t}\n\tswitch algorithm {\n\tcase \"imohash\":\n\t\treturn IMOHashFile(fname)\n\tcase \"md5\":\n\t\treturn MD5HashFile(fname)\n\tcase \"xxhash\":\n\t\treturn XXHashFile(fname)\n\t}\n\terr = fmt.Errorf(\"unspecified algorithm\")\n\treturn\n}\n\n// MD5HashFile returns MD5 hash\nfunc MD5HashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := md5.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// IMOHashFile returns imohash\nfunc IMOHashFile(fname string) (hash []byte, err error) {\n\tb, err := imohash.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\nvar imofull = imohash.NewCustom(0, 0)\n\n// IMOHashFileFull returns imohash of full file\nfunc IMOHashFileFull(fname string) (hash []byte, err error) {\n\tb, err := imofull.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\n// XXHashFile returns the xxhash of a file\nfunc XXHashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := xxhash.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// SHA256 returns sha256 sum\nfunc SHA256(s string) string {\n\tsha := sha256.New()\n\tsha.Write([]byte(s))\n\treturn hex.EncodeToString(sha.Sum(nil))\n}\n\n// PublicIP returns public ip address\nfunc PublicIP() (ip string, err error) {\n\tresp, err := http.Get(\"https://canhazip.com\")\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode == http.StatusOK {\n\t\tbodyBytes, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tip = strings.TrimSpace(string(bodyBytes))\n\t}\n\treturn\n}\n\n// LocalIP returns local ip address\nfunc LocalIP() string {\n\tconn, err := net.Dial(\"udp\", \"8.8.8.8:80\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tlocalAddr := conn.LocalAddr().(*net.UDPAddr)\n\n\treturn localAddr.IP.String()\n}\n\nfunc GenerateRandomPin() string {\n\ts := \"\"\n\tmax := new(big.Int)\n\tmax.SetInt64(9)\n\tfor i := 0; i < NbPinNumbers; i++ {\n\t\tv, err := rand.Int(rand.Reader, max)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\ts += fmt.Sprintf(\"%d\", v)\n\t}\n\treturn s\n}\n\n// GetRandomName returns mnemonicoded random name\nfunc GetRandomName() string {\n\tvar result []string\n\tbs := make([]byte, NbBytesWords)\n\trand.Read(bs)\n\tresult = mnemonicode.EncodeWordList(result, bs)\n\treturn GenerateRandomPin() + \"-\" + strings.Join(result, \"-\")\n}\n\n// ByteCountDecimal converts bytes to human readable byte string\nfunc ByteCountDecimal(b int64) string {\n\tconst unit = 1024\n\tif b < unit {\n\t\treturn fmt.Sprintf(\"%d B\", b)\n\t}\n\tdiv, exp := int64(unit), 0\n\tfor n := b / unit; n >= unit; n /= unit {\n\t\tdiv *= unit\n\t\texp++\n\t}\n\treturn fmt.Sprintf(\"%.1f %cB\", float64(b)/float64(div), \"kMGTPE\"[exp])\n}\n\n// MissingChunks returns the positions of missing chunks.\n// If file doesn't exist, it returns an empty chunk list (all chunks).\n// If the file size is not the same as requested, it returns an empty chunk list (all chunks).\nfunc MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\tfstat, err := os.Stat(fname)\n\tif err != nil || fstat.Size() != fsize {\n\t\treturn\n\t}\n\n\temptyBuffer := make([]byte, chunkSize)\n\tchunkNum := 0\n\tchunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))\n\tvar currentLocation int64\n\tfor {\n\t\tbuffer := make([]byte, chunkSize)\n\t\tbytesread, err := f.Read(buffer)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tif bytes.Equal(buffer[:bytesread], emptyBuffer[:bytesread]) {\n\t\t\tchunks[chunkNum] = currentLocation\n\t\t\tchunkNum++\n\t\t}\n\t\tcurrentLocation += int64(bytesread)\n\t}\n\tif chunkNum == 0 {\n\t\tchunkRanges = []int64{}\n\t} else {\n\t\tchunks = chunks[:chunkNum]\n\t\tchunkRanges = []int64{int64(chunkSize), chunks[0]}\n\t\tcurCount := 0\n\t\tfor i, chunk := range chunks {\n\t\t\tif i == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurCount++\n\t\t\tif chunk-chunks[i-1] > int64(chunkSize) {\n\t\t\t\tchunkRanges = append(chunkRanges, int64(curCount))\n\t\t\t\tchunkRanges = append(chunkRanges, chunk)\n\t\t\t\tcurCount = 0\n\t\t\t}\n\t\t}\n\t\tchunkRanges = append(chunkRanges, int64(curCount+1))\n\t}\n\treturn\n}\n\n// ChunkRangesToChunks converts chunk ranges to list\nfunc ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) {\n\tif len(chunkRanges) == 0 {\n\t\treturn\n\t}\n\tchunkSize := chunkRanges[0]\n\tchunks = []int64{}\n\tfor i := 1; i < len(chunkRanges); i += 2 {\n\t\tfor j := int64(0); j < (chunkRanges[i+1]); j++ {\n\t\t\tchunks = append(chunks, chunkRanges[i]+j*chunkSize)\n\t\t}\n\t}\n\treturn\n}\n\n// GetLocalIPs returns all local ips\nfunc GetLocalIPs() (ips []string, err error) {\n\taddrs, err := net.InterfaceAddrs()\n\tif err != nil {\n\t\treturn\n\t}\n\tips = []string{}\n\tfor _, address := range addrs {\n\t\t// check the address type and if it is not a loopback the display it\n\t\tif ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {\n\t\t\tif ipnet.IP.To4() != nil {\n\t\t\t\tips = append(ips, ipnet.IP.String())\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\nfunc RandomFileName() (fname string, err error) {\n\tf, err := os.CreateTemp(\".\", \"croc-stdin-\")\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = f.Name()\n\t_ = f.Close()\n\treturn\n}\n\nfunc FindOpenPorts(host string, portNumStart, numPorts int) (openPorts []int) {\n\topenPorts = []int{}\n\tfor port := portNumStart; port-portNumStart < 200; port++ {\n\t\ttimeout := 100 * time.Millisecond\n\t\tconn, err := net.DialTimeout(\"tcp\", net.JoinHostPort(host, fmt.Sprint(port)), timeout)\n\t\tif conn != nil {\n\t\t\tconn.Close()\n\t\t} else if err != nil {\n\t\t\topenPorts = append(openPorts, port)\n\t\t}\n\t\tif len(openPorts) >= numPorts {\n\t\t\treturn\n\t\t}\n\t}\n\treturn\n}\n\n// local ip determination\n// https://stackoverflow.com/questions/41240761/check-if-ip-address-is-in-private-network-space\nvar privateIPBlocks []*net.IPNet\n\nfunc init() {\n\tfor _, cidr := range []string{\n\t\t\"127.0.0.0/8\",    // IPv4 loopback\n\t\t\"10.0.0.0/8\",     // RFC1918\n\t\t\"172.16.0.0/12\",  // RFC1918\n\t\t\"192.168.0.0/16\", // RFC1918\n\t\t\"169.254.0.0/16\", // RFC3927 link-local\n\t\t\"::1/128\",        // IPv6 loopback\n\t\t\"fe80::/10\",      // IPv6 link-local\n\t\t\"fc00::/7\",       // IPv6 unique local addr\n\t} {\n\t\t_, block, err := net.ParseCIDR(cidr)\n\t\tif err != nil {\n\t\t\tpanic(fmt.Errorf(\"parse error on %q: %v\", cidr, err))\n\t\t}\n\t\tprivateIPBlocks = append(privateIPBlocks, block)\n\t}\n}\n\nfunc IsLocalIP(ipaddress string) bool {\n\tif strings.Contains(ipaddress, \"127.0.0.1\") {\n\t\treturn true\n\t}\n\thost, _, _ := net.SplitHostPort(ipaddress)\n\tip := net.ParseIP(host)\n\tif ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {\n\t\treturn true\n\t}\n\tfor _, block := range privateIPBlocks {\n\t\tif block.Contains(ip) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc ZipDirectory(destination string, source string) (err error) {\n\tif _, err = os.Stat(destination); err == nil {\n\t\tlog.Fatalf(\"%s file already exists!\\n\", destination)\n\t}\n\tfmt.Fprintf(os.Stderr, \"Zipping %s to %s\\n\", source, destination)\n\tfile, err := os.Create(destination)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tdefer file.Close()\n\twriter := zip.NewWriter(file)\n\t// no compression because croc does its compression on the fly\n\twriter.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {\n\t\treturn flate.NewWriter(out, flate.NoCompression)\n\t})\n\tdefer writer.Close()\n\terr = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\t\tif info.Mode().IsRegular() {\n\t\t\tf1, err := os.Open(path)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tdefer f1.Close()\n\t\t\tzipPath := strings.ReplaceAll(path, source, strings.TrimSuffix(destination, \".zip\"))\n\t\t\tzipPath = filepath.ToSlash(zipPath)\n\t\t\tw1, err := writer.Create(zipPath)\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tif _, err := io.Copy(w1, f1); err != nil {\n\t\t\t\tlog.Fatalln(err)\n\t\t\t}\n\t\t\tfmt.Fprintf(os.Stderr, \"\\r\\033[2K\")\n\t\t\tfmt.Fprintf(os.Stderr, \"\\rAdding %s\", zipPath)\n\t\t}\n\t\treturn nil\n\t})\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tfmt.Fprintf(os.Stderr, \"\\n\")\n\treturn nil\n}\n\nfunc UnzipDirectory(destination string, source string) error {\n\tarchive, err := zip.OpenReader(source)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tdefer archive.Close()\n\n\tfor _, f := range archive.File {\n\t\tfilePath := filepath.Join(destination, f.Name)\n\t\tfmt.Fprintf(os.Stderr, \"\\r\\033[2K\")\n\t\tfmt.Fprintf(os.Stderr, \"\\rUnzipping file %s\", filePath)\n\t\tif f.FileInfo().IsDir() {\n\t\t\tos.MkdirAll(filePath, os.ModePerm)\n\t\t\tcontinue\n\t\t}\n\n\t\tif err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tdstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tfileInArchive, err := f.Open()\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tif _, err := io.Copy(dstFile, fileInArchive); err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\n\t\tdstFile.Close()\n\t\tfileInArchive.Close()\n\t}\n\tfmt.Fprintf(os.Stderr, \"\\n\")\n\treturn nil\n}\n\n// Path: src/comm/comm.go\npackage comm\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/magisterquis/connectproxy\"\n\t\"github.com/schollz/croc/v9/src/utils\"\n\tlog \"github.com/schollz/logger\"\n\t\"golang.org/x/net/proxy\"\n)\n\nvar Socks5Proxy = \"\"\nvar HttpProxy = \"\"\n\nvar MAGIC_BYTES = []byte(\"croc\")\n\n// Comm is some basic TCP communication\ntype Comm struct {\n\tconnection net.Conn\n}\n\n// NewConnection gets a new comm to a tcp address\n\nfunc NewConnection(address string, timelimit ...time.Duration) (c *Comm, err error) {\n\ttlimit := 30 * time.Second\n\tif len(timelimit) > 0 {\n\t\ttlimit = timelimit[0]\n\t}\n\tvar connection net.Conn\n\tif Socks5Proxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(Socks5Proxy, `://`) {\n\t\t\tSocks5Proxy = `socks5://` + Socks5Proxy\n\t\t}\n\t\tsocks5ProxyURL, urlParseError := url.Parse(Socks5Proxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse socks proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = proxy.FromURL(socks5ProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\t} else if HttpProxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(HttpProxy, `://`) {\n\t\t\tHttpProxy = `http://` + HttpProxy\n\t\t}\n\t\tHttpProxyURL, urlParseError := url.Parse(HttpProxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse http proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = connectproxy.New(HttpProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\n\t} else {\n\t\tlog.Debugf(\"dialing to %s with timelimit %s\", address, tlimit)\n\t\tconnection, err = net.DialTimeout(\"tcp\", address, tlimit)\n\t}\n\tif err != nil {\n\t\terr = fmt.Errorf(\"comm.NewConnection failed: %w\", err)\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tc = New(connection)\n\tlog.Debugf(\"connected to '%s'\", address)\n\treturn\n}\n\n// New returns a new comm\nfunc New(c net.Conn) *Comm {\n\tif err := c.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\tif err := c.SetDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting overall deadline: %v\", err)\n\t}\n\tif err := c.SetWriteDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Errorf(\"error setting write deadline: %v\", err)\n\t}\n\tcomm := new(Comm)\n\tcomm.connection = c\n\treturn comm\n}\n\n// Connection returns the net.Conn connection\nfunc (c *Comm) Connection() net.Conn {\n\treturn c.connection\n}\n\n// Close closes the connection\nfunc (c *Comm) Close() {\n\tif err := c.connection.Close(); err != nil {\n\t\tlog.Warnf(\"error closing connection: %v\", err)\n\t}\n}\n\nfunc (c *Comm) Write(b []byte) (n int, err error) {\n\theader := new(bytes.Buffer)\n\terr = binary.Write(header, binary.LittleEndian, uint32(len(b)))\n\tif err != nil {\n\t\tfmt.Println(\"binary.Write failed:\", err)\n\t}\n\ttmpCopy := append(header.Bytes(), b...)\n\ttmpCopy = append(MAGIC_BYTES, tmpCopy...)\n\tn, err = c.connection.Write(tmpCopy)\n\tif err != nil {\n\t\terr = fmt.Errorf(\"connection.Write failed: %w\", err)\n\t\treturn\n\t}\n\tif n != len(tmpCopy) {\n\t\terr = fmt.Errorf(\"wanted to write %d but wrote %d\", len(b), n)\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (c *Comm) Read() (buf []byte, numBytes int, bs []byte, err error) {\n\t// long read deadline in case waiting for file\n\tif err = c.connection.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\t// must clear the timeout setting\n\tdefer c.connection.SetDeadline(time.Time{})\n\n\t// read until we get 4 bytes for the magic\n\theader := make([]byte, 4)\n\t_, err = io.ReadFull(c.connection, header)\n\tif err != nil {\n\t\tlog.Debugf(\"initial read error: %v\", err)\n\t\treturn\n\t}\n\tif !bytes.Equal(header, MAGIC_BYTES) {\n\t\terr = fmt.Errorf(\"initial bytes are not magic: %x\", header)\n\t\treturn\n\t}\n\n\t// read until we get 4 bytes for the header\n\theader = make([]byte, 4)\n\t_, err = io.ReadFull(c.connection, header)\n\tif err != nil {\n\t\tlog.Debugf(\"initial read error: %v\", err)\n\t\treturn\n\t}\n\n\tvar numBytesUint32 uint32\n\trbuf := bytes.NewReader(header)\n\terr = binary.Read(rbuf, binary.LittleEndian, &numBytesUint32)\n\tif err != nil {\n\t\terr = fmt.Errorf(\"binary.Read failed: %w\", err)\n\t\tlog.Debug(err.Error())\n\t\treturn\n\t}\n\tnumBytes = int(numBytesUint32)\n\n\t// shorten the reading deadline in case getting weird data\n\tif err = c.connection.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\tbuf = make([]byte, numBytes)\n\t_, err = io.ReadFull(c.connection, buf)\n\tif err != nil {\n\t\tlog.Debugf(\"consecutive read error: %v\", err)\n\t\treturn\n\t}\n\treturn\n}\n\n// Send a message\nfunc (c *Comm) Send(message []byte) (err error) {\n\t_, err = c.Write(message)\n\treturn\n}\n\n// Receive a message\nfunc (c *Comm) Receive() (b []byte, err error) {\n\tb, _, _, err = c.Read()\n\treturn\n}\n\n// Path: src/tcp/tcp.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/schollz/pake/v3\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\t\"github.com/schollz/croc/v9/src/models\"\n)\n\ntype server struct {\n\thost       string\n\tport       string\n\tdebugLevel string\n\tbanner     string\n\tpassword   string\n\trooms      roomMap\n}\n\ntype roomInfo struct {\n\tfirst  *comm.Comm\n\tsecond *comm.Comm\n\topened time.Time\n\tfull   bool\n}\n\ntype roomMap struct {\n\trooms map[string]roomInfo\n\tsync.Mutex\n}\n\nconst pingRoom = \"pinglkasjdlfjsaldjf\"\n\nvar timeToRoomDeletion = 10 * time.Minute\n\n// Run starts a tcp listener, run async\nfunc Run(debugLevel, host, port, password string, banner ...string) (err error) {\n\ts := new(server)\n\ts.host = host\n\ts.port = port\n\ts.password = password\n\ts.debugLevel = debugLevel\n\tif len(banner) > 0 {\n\t\ts.banner = banner[0]\n\t}\n\treturn s.start()\n}\n\nfunc (s *server) start() (err error) {\n\tlog.SetLevel(s.debugLevel)\n\tlog.Debugf(\"starting with password '%s'\", s.password)\n\ts.rooms.Lock()\n\ts.rooms.rooms = make(map[string]roomInfo)\n\ts.rooms.Unlock()\n\n\t// delete old rooms\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(timeToRoomDeletion)\n\t\t\tvar roomsToDelete []string\n\t\t\ts.rooms.Lock()\n\t\t\tfor room := range s.rooms.rooms {\n\t\t\t\tif time.Since(s.rooms.rooms[room].opened) > 3*time.Hour {\n\t\t\t\t\troomsToDelete = append(roomsToDelete, room)\n\t\t\t\t}\n\t\t\t}\n\t\t\ts.rooms.Unlock()\n\n\t\t\tfor _, room := range roomsToDelete {\n\t\t\t\ts.deleteRoom(room)\n\t\t\t}\n\t\t}\n\t}()\n\n\terr = s.run()\n\tif err != nil {\n\t\tlog.Error(err)\n\t}\n\treturn\n}\n\nfunc (s *server) run() (err error) {\n\tnetwork := \"tcp\"\n\taddr := net.JoinHostPort(s.host, s.port)\n\tif s.host != \"\" {\n\t\tip := net.ParseIP(s.host)\n\t\tif ip == nil {\n\t\t\tvar tcpIP *net.IPAddr\n\t\t\ttcpIP, err = net.ResolveIPAddr(\"ip\", s.host)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tip = tcpIP.IP\n\t\t}\n\t\taddr = net.JoinHostPort(ip.String(), s.port)\n\t\tif s.host != \"\" {\n\t\t\tif ip.To4() != nil {\n\t\t\t\tnetwork = \"tcp4\"\n\t\t\t} else {\n\t\t\t\tnetwork = \"tcp6\"\n\t\t\t}\n\t\t}\n\t}\n\taddr = strings.Replace(addr, \"127.0.0.1\", \"0.0.0.0\", 1)\n\tlog.Infof(\"starting TCP server on \" + addr)\n\tserver, err := net.Listen(network, addr)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error listening on %s: %w\", addr, err)\n\t}\n\tdefer server.Close()\n\t// spawn a new goroutine whenever a client connects\n\tfor {\n\t\tconnection, err := server.Accept()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"problem accepting connection: %w\", err)\n\t\t}\n\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\tgo func(port string, connection net.Conn) {\n\t\t\tc := comm.New(connection)\n\t\t\troom, errCommunication := s.clientCommunication(port, c)\n\t\t\tlog.Debugf(\"room: %+v\", room)\n\t\t\tlog.Debugf(\"err: %+v\", errCommunication)\n\t\t\tif errCommunication != nil {\n\t\t\t\tlog.Debugf(\"relay-%s: %s\", connection.RemoteAddr().String(), errCommunication.Error())\n\t\t\t\tconnection.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif room == pingRoom {\n\t\t\t\tlog.Debugf(\"got ping\")\n\t\t\t\tconnection.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor {\n\t\t\t\t// check connection\n\t\t\t\tlog.Debugf(\"checking connection of room %s for %+v\", room, c)\n\t\t\t\tdeleteIt := false\n\t\t\t\ts.rooms.Lock()\n\t\t\t\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\t\t\t\tlog.Debug(\"room is gone\")\n\t\t\t\t\ts.rooms.Unlock()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tlog.Debugf(\"room: %+v\", s.rooms.rooms[room])\n\t\t\t\tif s.rooms.rooms[room].first != nil && s.rooms.rooms[room].second != nil {\n\t\t\t\t\tlog.Debug(\"rooms ready\")\n\t\t\t\t\ts.rooms.Unlock()\n\t\t\t\t\tbreak\n\t\t\t\t} else {\n\t\t\t\t\tif s.rooms.rooms[room].first != nil {\n\t\t\t\t\t\terrSend := s.rooms.rooms[room].first.Send([]byte{1})\n\t\t\t\t\t\tif errSend != nil {\n\t\t\t\t\t\t\tlog.Debug(errSend)\n\t\t\t\t\t\t\tdeleteIt = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.rooms.Unlock()\n\t\t\t\tif deleteIt {\n\t\t\t\t\ts.deleteRoom(room)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\ttime.Sleep(1 * time.Second)\n\t\t\t}\n\t\t}(s.port, connection)\n\t}\n}\n\nvar weakKey = []byte{1, 2, 3}\n\nfunc (s *server) clientCommunication(port string, c *comm.Comm) (room string, err error) {\n\t// establish secure password with PAKE for communication with relay\n\tB, err := pake.InitCurve(weakKey, 1, \"siec\")\n\tif err != nil {\n\t\treturn\n\t}\n\tAbytes, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tlog.Debugf(\"Abytes: %s\", Abytes)\n\tif bytes.Equal(Abytes, []byte(\"ping\")) {\n\t\troom = pingRoom\n\t\tlog.Debug(\"sending back pong\")\n\t\tc.Send([]byte(\"pong\"))\n\t\treturn\n\t}\n\terr = B.Update(Abytes)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(B.Bytes())\n\tif err != nil {\n\t\treturn\n\t}\n\tstrongKey, err := B.SessionKey()\n\tif err != nil {\n\t\treturn\n\t}\n\tlog.Debugf(\"strongkey: %x\", strongKey)\n\n\t// receive salt\n\tsalt, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tstrongKeyForEncryption, _, err := crypt.New(strongKey, salt)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tlog.Debugf(\"waiting for password\")\n\tpasswordBytesEnc, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tpasswordBytes, err := crypt.Decrypt(passwordBytesEnc, strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\tif strings.TrimSpace(string(passwordBytes)) != s.password {\n\t\terr = fmt.Errorf(\"bad password\")\n\t\tenc, _ := crypt.Encrypt([]byte(err.Error()), strongKeyForEncryption)\n\t\tif err = c.Send(enc); err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"send error: %w\", err)\n\t\t}\n\t\treturn\n\t}\n\n\t// send ok to tell client they are connected\n\tbanner := s.banner\n\tif len(banner) == 0 {\n\t\tbanner = \"ok\"\n\t}\n\tlog.Debugf(\"sending '%s'\", banner)\n\tbSend, err := crypt.Encrypt([]byte(banner+\"|||\"+c.Connection().RemoteAddr().String()), strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// wait for client to tell me which room they want\n\tlog.Debug(\"waiting for answer\")\n\tenc, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\troomBytes, err := crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\troom = string(roomBytes)\n\n\ts.rooms.Lock()\n\t// create the room if it is new\n\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\ts.rooms.rooms[room] = roomInfo{\n\t\t\tfirst:  c,\n\t\t\topened: time.Now(),\n\t\t}\n\t\ts.rooms.Unlock()\n\t\t// tell the client that they got the room\n\n\t\tbSend, err = crypt.Encrypt([]byte(\"ok\"), strongKeyForEncryption)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\terr = c.Send(bSend)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\ts.deleteRoom(room)\n\t\t\treturn\n\t\t}\n\t\tlog.Debugf(\"room %s has 1\", room)\n\t\treturn\n\t}\n\tif s.rooms.rooms[room].full {\n\t\ts.rooms.Unlock()\n\t\tbSend, err = crypt.Encrypt([]byte(\"room full\"), strongKeyForEncryption)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\terr = c.Send(bSend)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\treturn\n\t}\n\tlog.Debugf(\"room %s has 2\", room)\n\ts.rooms.rooms[room] = roomInfo{\n\t\tfirst:  s.rooms.rooms[room].first,\n\t\tsecond: c,\n\t\topened: s.rooms.rooms[room].opened,\n\t\tfull:   true,\n\t}\n\totherConnection := s.rooms.rooms[room].first\n\ts.rooms.Unlock()\n\n\t// second connection is the sender, time to staple connections\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\t// start piping\n\tgo func(com1, com2 *comm.Comm, wg *sync.WaitGroup) {\n\t\tlog.Debug(\"starting pipes\")\n\t\tpipe(com1.Connection(), com2.Connection())\n\t\twg.Done()\n\t\tlog.Debug(\"done piping\")\n\t}(otherConnection, c, &wg)\n\n\t// tell the sender everything is ready\n\tbSend, err = crypt.Encrypt([]byte(\"ok\"), strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\ts.deleteRoom(room)\n\t\treturn\n\t}\n\twg.Wait()\n\n\t// delete room\n\ts.deleteRoom(room)\n\treturn\n}\n\nfunc (s *server) deleteRoom(room string) {\n\ts.rooms.Lock()\n\tdefer s.rooms.Unlock()\n\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\treturn\n\t}\n\tlog.Debugf(\"deleting room: %s\", room)\n\tif s.rooms.rooms[room].first != nil {\n\t\ts.rooms.rooms[room].first.Close()\n\t}\n\tif s.rooms.rooms[room].second != nil {\n\t\ts.rooms.rooms[room].second.Close()\n\t}\n\ts.rooms.rooms[room] = roomInfo{first: nil, second: nil}\n\tdelete(s.rooms.rooms, room)\n}\n\n// chanFromConn creates a channel from a Conn object, and sends everything it\n//\n//\tRead()s from the socket to the channel.\nfunc chanFromConn(conn net.Conn) chan []byte {\n\tc := make(chan []byte, 1)\n\tif err := conn.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"can't set read deadline: %v\", err)\n\t}\n\n\tgo func() {\n\t\tb := make([]byte, models.TCP_BUFFER_SIZE)\n\t\tfor {\n\t\t\tn, err := conn.Read(b)\n\t\t\tif n > 0 {\n\t\t\t\tres := make([]byte, n)\n\t\t\t\t// Copy the buffer so it doesn't get changed while read by the recipient.\n\t\t\t\tcopy(res, b[:n])\n\t\t\t\tc <- res\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tlog.Debug(err)\n\t\t\t\tc <- nil\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tlog.Debug(\"exiting\")\n\t}()\n\n\treturn c\n}\n\n// pipe creates a full-duplex pipe between the two sockets and\n// transfers data from one to the other.\nfunc pipe(conn1 net.Conn, conn2 net.Conn) {\n\tchan1 := chanFromConn(conn1)\n\tchan2 := chanFromConn(conn2)\n\n\tfor {\n\t\tselect {\n\t\tcase b1 := <-chan1:\n\t\t\tif b1 == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif _, err := conn2.Write(b1); err != nil {\n\t\t\t\tlog.Errorf(\"write error on channel 1: %v\", err)\n\t\t\t}\n\n\t\tcase b2 := <-chan2:\n\t\t\tif b2 == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif _, err := conn1.Write(b2); err != nil {\n\t\t\t\tlog.Errorf(\"write error on channel 2: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc PingServer(address string) (err error) {\n\tlog.Debugf(\"pinging %s\", address)\n\tc, err := comm.NewConnection(address, 300*time.Millisecond)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send([]byte(\"ping\"))\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tb, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif bytes.Equal(b, []byte(\"pong\")) {\n\t\treturn nil\n\t}\n\treturn fmt.Errorf(\"no pong\")\n}\n\n// ConnectToTCPServer will initiate a new connection\n// to the specified address, room with optional time limit\nfunc ConnectToTCPServer(address, password, room string, timelimit ...time.Duration) (c *comm.Comm, banner string, ipaddr string, err error) {\n\tif len(timelimit) > 0 {\n\t\tc, err = comm.NewConnection(address, timelimit[0])\n\t} else {\n\t\tc, err = comm.NewConnection(address)\n\t}\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\n\t// get PAKE connection with server to establish strong key to transfer info\n\tA, err := pake.InitCurve(weakKey, 0, \"siec\")\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(A.Bytes())\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tBbytes, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = A.Update(Bbytes)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tstrongKey, err := A.SessionKey()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debugf(\"strong key: %x\", strongKey)\n\n\tstrongKeyForEncryption, salt, err := crypt.New(strongKey, nil)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\t// send salt\n\terr = c.Send(salt)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\n\tlog.Debug(\"sending password\")\n\tbSend, err := crypt.Encrypt([]byte(password), strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"waiting for first ok\")\n\tenc, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tdata, err := crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif !strings.Contains(string(data), \"|||\") {\n\t\terr = fmt.Errorf(\"bad response: %s\", string(data))\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tbanner = strings.Split(string(data), \"|||\")[0]\n\tipaddr = strings.Split(string(data), \"|||\")[1]\n\tlog.Debug(\"sending room\")\n\tbSend, err = crypt.Encrypt([]byte(room), strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"waiting for room confirmation\")\n\tenc, err = c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tdata, err = crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif !bytes.Equal(data, []byte(\"ok\")) {\n\t\terr = fmt.Errorf(\"got bad response: %s\", data)\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"all set\")\n\treturn\n}\n\n// Path: src/message/message.go\npackage message\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/compress\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n)\n\n// Type is a message type\ntype Type string\n\nconst (\n\tTypePAKE           Type = \"pake\"\n\tTypeExternalIP     Type = \"externalip\"\n\tTypeFinished       Type = \"finished\"\n\tTypeError          Type = \"error\"\n\tTypeCloseRecipient Type = \"close-recipient\"\n\tTypeCloseSender    Type = \"close-sender\"\n\tTypeRecipientReady Type = \"recipientready\"\n\tTypeFileInfo       Type = \"fileinfo\"\n)\n\n...\n", "needle_token_start": 6790, "needle_token_end": 7517, "code_context_ntokens": 13899, "output": ["```go\n// NewConnection gets a new comm to a tcp address\nfunc NewConnection(address string, timelimit ...time.Duration) (c *Comm, err error) {\n\ttlimit := 30 * time.Second\n\tif len(timelimit) > 0 {\n\t\ttlimit = timelimit[0]\n\t}\n\tvar connection net.Conn\n\tif Socks5Proxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(Socks5Proxy, `://`) {\n\t\t\tSocks5Proxy = `socks5://` + Socks5Proxy\n\t\t}\n\t\tsocks5ProxyURL, urlParseError := url.Parse(Socks5Proxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse socks proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = proxy.FromURL(socks5ProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\t} else if HttpProxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(HttpProxy, `://`) {\n\t\t\tHttpProxy = `http://` + HttpProxy\n\t\t}\n\t\tHttpProxyURL, urlParseError := url.Parse(HttpProxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse http proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = connectproxy.New(HttpProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\n\t} else {\n\t\tlog.Debugf(\"dialing to %s with timelimit %s\", address, tlimit)\n\t\tconnection, err = net.DialTimeout(\"tcp\", address, tlimit)\n\t}\n\tif err != nil {\n\t\terr = fmt.Errorf(\"comm.NewConnection failed: %w\", err)\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tc = New(connection)\n\tlog.Debugf(\"connected to '%s'\", address)\n\treturn\n}\n```"]}
{"repo": "schollz/croc", "name": "TestTCP", "language": "go", "path": "src/tcp/tcp_test.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to test the functionality of a TCP server by simulating client connections, sending and receiving data, and handling room deletions and connection timeouts.\n2. **Input**: There are no direct inputs as it is a test function, but it internally uses predefined server settings (IP address, ports, passwords) and simulates client operations.\n3. **Output**: The function does not return any values but uses assertions to verify the correctness of server responses, connection handling, and data transmission.\n4. **Procedure**: \n   - Initializes the server with specific configurations.\n   - Waits briefly to allow for server setup.\n   - Tests server responsiveness by pinging the server.\n   - Simulates multiple client connections to the server to test room creation, data transmission, and error handling for duplicate connections and rapid reconnections.\n   - Sends and receives messages between clients to verify data integrity.\n   - Closes connections and checks server behavior after a set duration to ensure proper room deletion.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/compress/compress_test.go\npackage compress\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar fable = []byte(`The Frog and the Crocodile\nOnce, there was a frog who lived in the middle of a swamp. His entire family had lived in that swamp for generations, but this particular frog decided that he had had quite enough wetness to last him a lifetime. He decided that he was going to find a dry place to live instead.\n\nThe only thing that separated him from dry land was a swampy, muddy, swiftly flowing river. But the river was home to all sorts of slippery, slittering snakes that loved nothing better than a good, plump frog for dinner, so Frog didn't dare try to swim across.\n\nSo for many days, the frog stayed put, hopping along the bank, trying to think of a way to get across.\n\nThe snakes hissed and jeered at him, daring him to come closer, but he refused. Occasionally they would slither closer, jaws open to attack, but the frog always leaped out of the way. But no matter how far upstream he searched or how far downstream, the frog wasn't able to find a way across the water.\n\nHe had felt certain that there would be a bridge, or a place where the banks came together, yet all he found was more reeds and water. After a while, even the snakes stopped teasing him and went off in search of easier prey.\n\nThe frog sighed in frustration and sat to sulk in the rushes. Suddenly, he spotted two big eyes staring at him from the water. The giant log-shaped animal opened its mouth and asked him, \"What are you doing, Frog? Surely there are enough flies right there for a meal.\"\n\nThe frog croaked in surprise and leaped away from the crocodile. That creature could swallow him whole in a moment without thinking about it! Once he was a satisfied that he was a safe distance away, he answered. \"I'm tired of living in swampy waters, and I want to travel to the other side of the river. But if I swim across, the snakes will eat me.\"\n\nThe crocodile harrumphed in agreement and sat, thinking, for a while. \"Well, if you're afraid of the snakes, I could give you a ride across,\" he suggested.\n\n\"Oh no, I don't think so,\" Frog answered quickly. \"You'd eat me on the way over, or go underwater so the snakes could get me!\"\n\n...\n// Path: src/croc/croc_test.go\npackage croc\n\nimport (\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/tcp\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc init() {\n\tlog.SetLevel(\"trace\")\n\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8281\", \"pass123\", \"8282,8283,8284,8285\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8282\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8284\", \"pass123\")\n\tgo tcp.Run(\"debug\", \"127.0.0.1\", \"8285\", \"pass123\")\n\ttime.Sleep(1 * time.Second)\n}\n\nfunc TestCrocReadme(t *testing.T) {\n\tdefer os.Remove(\"README.md\")\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../README.md\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocEmptyFolder(t *testing.T) {\n\tpathName := \"../../testEmpty\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./testEmpty\")\n\tos.MkdirAll(pathName, 0o755)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocSymlink(t *testing.T) {\n\tpathName := \"../link-in-folder\"\n\tdefer os.RemoveAll(pathName)\n\tdefer os.RemoveAll(\"./link-in-folder\")\n\tos.MkdirAll(pathName, 0o755)\n\tos.Symlink(\"../../README.md\", filepath.Join(pathName, \"README.link\"))\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPorts:    []string{\"8281\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8124-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8281\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        false,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr = receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"receive failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n\n\ts, err := filepath.EvalSymlinks(path.Join(pathName, \"README.link\"))\n\tif s != \"../../README.md\" && s != \"..\\\\..\\\\README.md\" {\n\t\tlog.Debug(s)\n\t\tt.Errorf(\"symlink failed to transfer in folder\")\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"symlink transfer failed: %s\", err.Error())\n\t}\n}\nfunc testCrocIgnoreGit(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\".gitignore\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\ttime.Sleep(1 * time.Second)\n\tfile, err := os.Create(\".gitignore\")\n\tif err != nil {\n\t\tlog.Errorf(\"error creating file\")\n\t}\n\t_, err = file.WriteString(\"LICENSE\")\n\tif err != nil {\n\t\tlog.Errorf(\"error writing to file\")\n\t}\n\ttime.Sleep(1 * time.Second)\n\t// due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo.\n\tfilesInfo, _, _, errGet := GetFilesInfo([]string{\"../../LICENSE\", \".gitignore\", \"croc.go\"}, false, true)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\tfor _, file := range filesInfo {\n\t\tif strings.Contains(file.Name, \"LICENSE\") {\n\t\t\tt.Errorf(\"test failed, should ignore LICENSE\")\n\t\t}\n\t}\n}\n\nfunc TestCrocLocal(t *testing.T) {\n\tlog.SetLevel(\"trace\")\n\tdefer os.Remove(\"LICENSE\")\n\tdefer os.Remove(\"touched\")\n\ttime.Sleep(300 * time.Millisecond)\n\n\tlog.Debug(\"setting up sender\")\n\tsender, err := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPorts:    []string{\"8181\", \"8182\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t\tGitIgnore:     false,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttime.Sleep(1 * time.Second)\n\n\tlog.Debug(\"setting up receiver\")\n\treceiver, err := New(Options{\n\t\tIsSender:      false,\n\t\tSharedSecret:  \"8123-testingthecroc\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"127.0.0.1:8181\",\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  false,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tvar wg sync.WaitGroup\n\tos.Create(\"touched\")\n\twg.Add(2)\n\tgo func() {\n\t\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{\"../../LICENSE\", \"touched\"}, false, false)\n\t\tif errGet != nil {\n\t\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t\t}\n\t\terr := sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\ttime.Sleep(100 * time.Millisecond)\n\tgo func() {\n\t\terr := receiver.Receive()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"send failed: %v\", err)\n\t\t}\n\t\twg.Done()\n\t}()\n\n\twg.Wait()\n}\n\nfunc TestCrocError(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err = tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err = tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\n\tDebug(false)\n\tlog.SetLevel(\"warn\")\n\tsender, _ := New(Options{\n\t\tIsSender:      true,\n\t\tSharedSecret:  \"8123-testingthecroc2\",\n\t\tDebug:         true,\n\t\tRelayAddress:  \"doesntexistok.com:8381\",\n\t\tRelayPorts:    []string{\"8381\", \"8382\"},\n\t\tRelayPassword: \"pass123\",\n\t\tStdout:        true,\n\t\tNoPrompt:      true,\n\t\tDisableLocal:  true,\n\t\tCurve:         \"siec\",\n\t\tOverwrite:     true,\n\t})\n\tfilesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false, false)\n\tif errGet != nil {\n\t\tt.Errorf(\"failed to get minimal info: %v\", errGet)\n\t}\n\terr = sender.Send(filesInfo, emptyFolders, totalNumberFolders)\n\tlog.Debug(err)\n\tassert.NotNil(t, err)\n}\n\nfunc TestCleanUp(t *testing.T) {\n\t// windows allows files to be deleted only if they\n\t// are not open by another program so the remove actions\n\t// from the above tests will not always do a good clean up\n\t// This \"test\" will make sure\n\toperatingSystem := runtime.GOOS\n\tlog.Debugf(\"The operating system is %s\", operatingSystem)\n\tif operatingSystem == \"windows\" {\n\t\ttime.Sleep(1 * time.Second)\n\t\tlog.Debug(\"Full cleanup\")\n\t\tvar err error\n\n\t\tfor _, file := range []string{\"README.md\", \"./README.md\"} {\n\t\t\terr = os.Remove(file)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", file)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", file)\n\t\t\t}\n\t\t}\n\t\tfor _, folder := range []string{\"./testEmpty\", \"./link-in-folder\"} {\n\t\t\terr = os.RemoveAll(folder)\n\t\t\tif err == nil {\n\t\t\t\tlog.Debugf(\"Successfully purged %s\", folder)\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"%s was already purged.\", folder)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// Path: src/crypt/crypt_test.go\npackage crypt\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkEncrypt(b *testing.B) {\n\tbob, _, _ := New([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncrypt([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecrypt(b *testing.B) {\n\tkey, _, _ := New([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := Encrypt(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecrypt(enc, key)\n\t}\n}\n\nfunc BenchmarkEncryptChaCha(b *testing.B) {\n\tbob, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tfor i := 0; i < b.N; i++ {\n\t\tEncryptChaCha([]byte(\"hello, world\"), bob)\n\t}\n}\n\nfunc BenchmarkDecryptChaCha(b *testing.B) {\n\tkey, _, _ := NewArgon2([]byte(\"password\"), nil)\n\tmsg := []byte(\"hello, world\")\n\tenc, _ := EncryptChaCha(msg, key)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tDecryptChaCha(enc, key)\n\t}\n}\n\nfunc TestEncryption(t *testing.T) {\n\tkey, salt, err := New([]byte(\"password\"), nil)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := Encrypt(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := Decrypt(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := New([]byte(\"password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = New([]byte(\"wrong password\"), salt)\n\tdec, err = Decrypt(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = Decrypt([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = New([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\nfunc TestEncryptionChaCha(t *testing.T) {\n\tkey, salt, err := NewArgon2([]byte(\"password\"), nil)\n\tfmt.Printf(\"key: %x\\n\", key)\n\tassert.Nil(t, err)\n\tmsg := []byte(\"hello, world\")\n\tenc, err := EncryptChaCha(msg, key)\n\tassert.Nil(t, err)\n\tdec, err := DecryptChaCha(enc, key)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ := NewArgon2([]byte(\"password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.Nil(t, err)\n\tassert.Equal(t, msg, dec)\n\n\t// check reusing the salt\n\tkey2, _, _ = NewArgon2([]byte(\"wrong password\"), salt)\n\tdec, err = DecryptChaCha(enc, key2)\n\tassert.NotNil(t, err)\n\tassert.NotEqual(t, msg, dec)\n\n\t// error with no password\n\t_, err = DecryptChaCha([]byte(\"\"), key)\n\tassert.NotNil(t, err)\n\n\t// error with small password\n\t_, _, err = NewArgon2([]byte(\"\"), nil)\n\tassert.NotNil(t, err)\n}\n\n// Path: src/install/updateversion.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n)\n\nfunc main() {\n\terr := run()\n\tif err != nil {\n\t\tfmt.Println(err)\n\t}\n}\n\nfunc run() (err error) {\n\tversionNew := \"v\" + os.Getenv(\"VERSION\")\n\tversionHash, err := exec.Command(\"git\", \"rev-parse\", \"--short\", \"HEAD\").Output()\n\tif err != nil {\n\t\treturn\n\t}\n\tversionHashNew := strings.TrimSpace(string(versionHash))\n\tfmt.Println(versionNew)\n\tfmt.Println(versionHashNew)\n\n\terr = replaceInFile(\"src/cli/cli.go\", `Version = \"`, `\"`, versionNew+\"-\"+versionHashNew)\n\tif err == nil {\n\t\tfmt.Printf(\"updated cli.go to version %s\\n\", versionNew)\n\t}\n\terr = replaceInFile(\"README.md\", `version-`, `-b`, strings.Split(versionNew, \"-\")[0])\n\tif err == nil {\n\t\tfmt.Printf(\"updated README to version %s\\n\", strings.Split(versionNew, \"-\")[0])\n\t}\n\n\terr = replaceInFile(\"src/install/default.txt\", `croc_version=\"`, `\"`, strings.Split(versionNew, \"-\")[0][1:])\n\tif err == nil {\n\t\tfmt.Printf(\"updated default.txt to version %s\\n\", strings.Split(versionNew, \"-\")[0][1:])\n\t}\n\n\treturn\n}\n\nfunc replaceInFile(fname, start, end, replacement string) (err error) {\n\tb, err := os.ReadFile(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\toldVersion := GetStringInBetween(string(b), start, end)\n\tif oldVersion == \"\" {\n\t\terr = fmt.Errorf(\"nothing\")\n\t\treturn\n\t}\n\tnewF := strings.Replace(\n\t\tstring(b),\n\t\tfmt.Sprintf(\"%s%s%s\", start, oldVersion, end),\n\t\tfmt.Sprintf(\"%s%s%s\", start, replacement, end),\n\t\t1,\n\t)\n\terr = os.WriteFile(fname, []byte(newF), 0o644)\n\treturn\n}\n\n// GetStringInBetween Returns empty string if no start string found\nfunc GetStringInBetween(str string, start string, end string) (result string) {\n\ts := strings.Index(str, start)\n\tif s == -1 {\n\t\treturn\n\t}\n\ts += len(start)\n\te := strings.Index(str[s:], end)\n\tif e == -1 {\n\t\treturn\n\t}\n\te += s\n\treturn str[s:e]\n}\n\n// Path: src/message/message_test.go\npackage message\n\nimport (\n\t\"crypto/rand\"\n\t\"fmt\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar TypeMessage Type = \"message\"\n\nfunc TestMessage(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tassert.Nil(t, err)\n\tfmt.Println(string(salt))\n\tb, err := Encode(e, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(e, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n\t_, err = Decode([]byte(\"not pass\"), b)\n\tassert.NotNil(t, err)\n\t_, err = Encode([]byte(\"0\"), m)\n\tassert.NotNil(t, err)\n}\n\nfunc TestMessageNoPass(t *testing.T) {\n\tlog.SetLevel(\"debug\")\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\tb, err := Encode(nil, m)\n\tassert.Nil(t, err)\n\tfmt.Printf(\"%x\\n\", b)\n\n\tm2, err := Decode(nil, b)\n\tassert.Nil(t, err)\n\tassert.Equal(t, m, m2)\n\tassert.Equal(t, `{\"t\":\"message\",\"m\":\"hello, world\"}`, m.String())\n}\n\nfunc TestSend(t *testing.T) {\n\ttoken := make([]byte, 40000000)\n\trand.Read(token)\n\n\tport := \"8801\"\n\tgo func() {\n\t\tlog.Debugf(\"starting TCP server on \" + port)\n\t\tserver, err := net.Listen(\"tcp\", \"0.0.0.0:\"+port)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t}\n\t\tdefer server.Close()\n\t\t// spawn a new goroutine whenever a client connects\n\t\tfor {\n\t\t\tconnection, err := server.Accept()\n\t\t\tif err != nil {\n\t\t\t\tlog.Error(err)\n\t\t\t}\n\t\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\t\tgo func(_ string, connection net.Conn) {\n\t\t\t\tc := comm.New(connection)\n\t\t\t\terr = c.Send([]byte(\"hello, world\"))\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tdata, err := c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte(\"hello, computer\"), data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, []byte{'\\x00'}, data)\n\t\t\t\tdata, err = c.Receive()\n\t\t\t\tassert.Nil(t, err)\n\t\t\t\tassert.Equal(t, token, data)\n\t\t\t}(port, connection)\n\t\t}\n\t}()\n\n\ttime.Sleep(300 * time.Millisecond)\n\ta, err := comm.NewConnection(\"127.0.0.1:\"+port, 10*time.Minute)\n\tassert.Nil(t, err)\n\tm := Message{Type: TypeMessage, Message: \"hello, world\"}\n\te, salt, err := crypt.New([]byte(\"pass\"), nil)\n\tlog.Debug(salt)\n\tassert.Nil(t, err)\n\n\tassert.Nil(t, Send(a, e, m))\n}\n\n// Path: src/models/constants.go\npackage models\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\n\t\"github.com/schollz/croc/v9/src/utils\"\n)\n\n// TCP_BUFFER_SIZE is the maximum packet size\nconst TCP_BUFFER_SIZE = 1024 * 64\n\n// DEFAULT_RELAY is the default relay used (can be set using --relay)\nvar (\n\tDEFAULT_RELAY      = \"croc.schollz.com\"\n\tDEFAULT_RELAY6     = \"croc6.schollz.com\"\n\tDEFAULT_PORT       = \"9009\"\n\tDEFAULT_PASSPHRASE = \"pass123\"\n\tINTERNAL_DNS       = false\n)\n\n// publicDNS are servers to be queried if a local lookup fails\nvar publicDNS = []string{\n\t\"1.0.0.1\",                // Cloudflare\n\t\"1.1.1.1\",                // Cloudflare\n\t\"[2606:4700:4700::1111]\", // Cloudflare\n\t\"[2606:4700:4700::1001]\", // Cloudflare\n\t\"8.8.4.4\",                // Google\n\t\"8.8.8.8\",                // Google\n\t\"[2001:4860:4860::8844]\", // Google\n\t\"[2001:4860:4860::8888]\", // Google\n\t\"9.9.9.9\",                // Quad9\n\t\"149.112.112.112\",        // Quad9\n\t\"[2620:fe::fe]\",          // Quad9\n\t\"[2620:fe::fe:9]\",        // Quad9\n\t\"8.26.56.26\",             // Comodo\n\t\"8.20.247.20\",            // Comodo\n\t\"208.67.220.220\",         // Cisco OpenDNS\n\t\"208.67.222.222\",         // Cisco OpenDNS\n\t\"[2620:119:35::35]\",      // Cisco OpenDNS\n\t\"[2620:119:53::53]\",      // Cisco OpenDNS\n}\n\nfunc getConfigFile(requireValidPath bool) (fname string, err error) {\n\tconfigFile, err := utils.GetConfigDir(requireValidPath)\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = path.Join(configFile, \"internal-dns\")\n\treturn\n}\n\nfunc init() {\n\tdoRemember := false\n\tfor _, flag := range os.Args {\n\t\tif flag == \"--internal-dns\" {\n\t\t\tINTERNAL_DNS = true\n\t\t\tbreak\n\t\t}\n\t\tif flag == \"--remember\" {\n\t\t\tdoRemember = true\n\t\t}\n\t}\n\tif doRemember {\n\t\t// save in config file\n\t\tfname, err := getConfigFile(true)\n\t\tif err == nil {\n\t\t\tf, _ := os.Create(fname)\n\t\t\tf.Close()\n\t\t}\n\t}\n\tif !INTERNAL_DNS {\n\t\tfname, err := getConfigFile(false)\n\t\tif err == nil {\n\t\t\tINTERNAL_DNS = utils.Exists(fname)\n\t\t}\n\t}\n\tvar err error\n\tvar addr string\n\taddr, err = lookup(DEFAULT_RELAY)\n\tif err == nil {\n\t\tDEFAULT_RELAY = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY = \"\"\n\t}\n\taddr, err = lookup(DEFAULT_RELAY6)\n\tif err == nil {\n\t\tDEFAULT_RELAY6 = net.JoinHostPort(addr, DEFAULT_PORT)\n\t} else {\n\t\tDEFAULT_RELAY6 = \"\"\n\t}\n}\n\n// Resolve a hostname to an IP address using DNS.\nfunc lookup(address string) (ipaddress string, err error) {\n\tif !INTERNAL_DNS {\n\t\treturn localLookupIP(address)\n\t}\n\ttype Result struct {\n\t\ts   string\n\t\terr error\n\t}\n\tresult := make(chan Result, len(publicDNS))\n\tfor _, dns := range publicDNS {\n\t\tgo func(dns string) {\n\t\t\tvar r Result\n\t\t\tr.s, r.err = remoteLookupIP(address, dns)\n\t\t\tresult <- r\n\t\t}(dns)\n\t}\n\tfor i := 0; i < len(publicDNS); i++ {\n\t\tipaddress = (<-result).s\n\t\tif ipaddress != \"\" {\n\t\t\treturn\n\t\t}\n\t}\n\terr = fmt.Errorf(\"failed to resolve %s: all DNS servers exhausted\", address)\n\treturn\n}\n\n// localLookupIP returns a host's IP address based on the local resolver.\nfunc localLookupIP(address string) (ipaddress string, err error) {\n\tip, err := net.LookupHost(address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// remoteLookupIP returns a host's IP address based on a remote DNS server.\nfunc remoteLookupIP(address, dns string) (ipaddress string, err error) {\n\tr := &net.Resolver{\n\t\tPreferGo: true,\n\t\tDial: func(ctx context.Context, network, _ string) (net.Conn, error) {\n\t\t\td := new(net.Dialer)\n\t\t\treturn d.DialContext(ctx, network, dns+\":53\")\n\t\t},\n\t}\n\tip, err := r.LookupHost(context.Background(), address)\n\tif err != nil {\n\t\treturn\n\t}\n\tipaddress = ip[0]\n\treturn\n}\n\n// Path: src/tcp/tcp_test.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc BenchmarkConnection(b *testing.B) {\n\tlog.SetLevel(\"trace\")\n\tgo Run(\"debug\", \"127.0.0.1\", \"8283\", \"pass123\", \"8284\")\n\ttime.Sleep(100 * time.Millisecond)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tc, _, _, _ := ConnectToTCPServer(\"127.0.0.1:8283\", \"pass123\", fmt.Sprintf(\"testroom%d\", i), 1*time.Minute)\n\t\tc.Close()\n\t}\n}\n\n\nfunc TestTCP(t *testing.T) {\n\tlog.SetLevel(\"error\")\n\ttimeToRoomDeletion = 100 * time.Millisecond\n\tgo Run(\"debug\", \"127.0.0.1\", \"8381\", \"pass123\", \"8382\")\n\ttime.Sleep(timeToRoomDeletion)\n\terr := PingServer(\"127.0.0.1:8381\")\n\tassert.Nil(t, err)\n\terr = PingServer(\"127.0.0.1:8333\")\n\tassert.NotNil(t, err)\n\n\ttime.Sleep(timeToRoomDeletion)\n\tc1, banner, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Minute)\n\tassert.Equal(t, banner, \"8382\")\n\tassert.Nil(t, err)\n\tc2, _, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.Nil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.NotNil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Nanosecond)\n\tassert.NotNil(t, err)\n\n\t// try sending data\n\tassert.Nil(t, c1.Send([]byte(\"hello, c2\")))\n\tvar data []byte\n\tfor {\n\t\tdata, err = c2.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c2\"), data)\n\n\tassert.Nil(t, c2.Send([]byte(\"hello, c1\")))\n\tfor {\n\t\tdata, err = c1.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c1\"), data)\n\n\tc1.Close()\n\ttime.Sleep(300 * time.Millisecond)\n}\n\n// Path: src/utils/utils_test.go\npackage utils\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst TCP_BUFFER_SIZE = 1024 * 64\n\nvar bigFileSize = 75000000\n\nfunc bigFile() {\n\tos.WriteFile(\"bigfile.test\", bytes.Repeat([]byte(\"z\"), bigFileSize), 0o666)\n}\n\nfunc BenchmarkMD5(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMD5HashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkXXHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tXXHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHash(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFile(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkImoHashFull(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tIMOHashFileFull(\"bigfile.test\")\n\t}\n}\n\nfunc BenchmarkSha256(b *testing.B) {\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tSHA256(\"hello,world\")\n\t}\n}\n\nfunc BenchmarkMissingChunks(b *testing.B) {\n\tbigFile()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tMissingChunks(\"bigfile.test\", int64(bigFileSize), TCP_BUFFER_SIZE/2)\n\t}\n}\n\nfunc TestExists(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tfmt.Println(GetLocalIPs())\n\tassert.True(t, Exists(\"bigfile.test\"))\n\tassert.False(t, Exists(\"doesnotexist\"))\n}\n\nfunc TestMD5HashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := MD5HashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"8304ff018e02baad0e3555bade29a405\", fmt.Sprintf(\"%x\", b))\n\t_, err = MD5HashFile(\"bigfile.test.nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestIMOHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := IMOHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"c0d1e123ca94148ffea146137684ebb9\", fmt.Sprintf(\"%x\", b))\n}\n\nfunc TestXXHashFile(t *testing.T) {\n\tbigFile()\n\tdefer os.Remove(\"bigfile.test\")\n\tb, err := XXHashFile(\"bigfile.test\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"4918740eb5ccb6f7\", fmt.Sprintf(\"%x\", b))\n\t_, err = XXHashFile(\"nofile\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestSHA256(t *testing.T) {\n\tassert.Equal(t, \"09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b\", SHA256(\"hello, world\"))\n}\n\nfunc TestByteCountDecimal(t *testing.T) {\n\tassert.Equal(t, \"10.0 kB\", ByteCountDecimal(10240))\n\tassert.Equal(t, \"50 B\", ByteCountDecimal(50))\n\tassert.Equal(t, \"12.4 MB\", ByteCountDecimal(13002343))\n}\n\nfunc TestMissingChunks(t *testing.T) {\n\tfileSize := 100\n\tchunkSize := 10\n\trand.Seed(1)\n\tbigBuff := make([]byte, fileSize)\n\trand.Read(bigBuff)\n\tos.WriteFile(\"missing.test\", bigBuff, 0o644)\n\tempty := make([]byte, chunkSize)\n\tf, err := os.OpenFile(\"missing.test\", os.O_RDWR, 0o644)\n\tassert.Nil(t, err)\n\tfor block := 0; block < fileSize/chunkSize; block++ {\n\t\tif block == 0 || block == 4 || block == 5 || block >= 7 {\n\t\t\tf.WriteAt(empty, int64(block*chunkSize))\n\t\t}\n\t}\n\tf.Close()\n\n\tchunkRanges := MissingChunks(\"missing.test\", int64(fileSize), chunkSize)\n\tassert.Equal(t, []int64{10, 0, 1, 40, 2, 70, 3}, chunkRanges)\n\n\tchunks := ChunkRangesToChunks(chunkRanges)\n\tassert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks)\n\n\tos.Remove(\"missing.test\")\n\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err := tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err := tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\tchunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)+10), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunkRanges = MissingChunks(tmpfile.Name()+\"ok\", int64(len(content)), chunkSize)\n\tassert.Empty(t, chunkRanges)\n\tchunks = ChunkRangesToChunks(chunkRanges)\n\tassert.Empty(t, chunks)\n}\n\n// func Test1(t *testing.T) {\n// \tchunkRanges := MissingChunks(\"../../m/bigfile.test\", int64(75000000), 1024*64/2)\n// \tfmt.Println(chunkRanges)\n// \tfmt.Println(ChunkRangesToChunks((chunkRanges)))\n// \tassert.Nil(t, nil)\n// }\n\nfunc TestHashFile(t *testing.T) {\n\tcontent := []byte(\"temporary file's content\")\n\ttmpfile, err := os.CreateTemp(\"\", \"example\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tdefer os.Remove(tmpfile.Name()) // clean up\n\n\tif _, err = tmpfile.Write(content); err != nil {\n\t\tpanic(err)\n\t}\n\tif err = tmpfile.Close(); err != nil {\n\t\tpanic(err)\n\t}\n\thashed, err := HashFile(tmpfile.Name(), \"xxhash\")\n\tassert.Nil(t, err)\n\tassert.Equal(t, \"e66c561610ad51e2\", fmt.Sprintf(\"%x\", hashed))\n}\n\nfunc TestPublicIP(t *testing.T) {\n\tip, err := PublicIP()\n\tfmt.Println(ip)\n\tassert.True(t, strings.Contains(ip, \".\") || strings.Contains(ip, \":\"))\n\tassert.Nil(t, err)\n}\n\nfunc TestLocalIP(t *testing.T) {\n\tip := LocalIP()\n\tfmt.Println(ip)\n\tassert.True(t, strings.Contains(ip, \".\") || strings.Contains(ip, \":\"))\n}\n\nfunc TestGetRandomName(t *testing.T) {\n\tname := GetRandomName()\n\tfmt.Println(name)\n\tassert.NotEmpty(t, name)\n}\n\nfunc TestFindOpenPorts(t *testing.T) {\n\topenPorts := FindOpenPorts(\"127.0.0.1\", 9009, 4)\n\tassert.Equal(t, []int{9009, 9010, 9011, 9012}, openPorts)\n}\n\nfunc TestIsLocalIP(t *testing.T) {\n\tassert.True(t, IsLocalIP(\"192.168.0.14:9009\"))\n}\n\n", "needle_token_start": 9476, "needle_token_end": 10125, "code_context_ntokens": 12535, "output": ["```go\nfunc TestTCP(t *testing.T) {\n\tlog.SetLevel(\"error\")\n\ttimeToRoomDeletion = 100 * time.Millisecond\n\tgo Run(\"debug\", \"127.0.0.1\", \"8381\", \"pass123\", \"8382\")\n\ttime.Sleep(timeToRoomDeletion)\n\terr := PingServer(\"127.0.0.1:8381\")\n\tassert.Nil(t, err)\n\terr = PingServer(\"127.0.0.1:8333\")\n\tassert.NotNil(t, err)\n\n\ttime.Sleep(timeToRoomDeletion)\n\tc1, banner, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Minute)\n\tassert.Equal(t, banner, \"8382\")\n\tassert.Nil(t, err)\n\tc2, _, _, err := ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.Nil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\")\n\tassert.NotNil(t, err)\n\t_, _, _, err = ConnectToTCPServer(\"127.0.0.1:8381\", \"pass123\", \"testRoom\", 1*time.Nanosecond)\n\tassert.NotNil(t, err)\n\n\t// try sending data\n\tassert.Nil(t, c1.Send([]byte(\"hello, c2\")))\n\tvar data []byte\n\tfor {\n\t\tdata, err = c2.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c2\"), data)\n\n\tassert.Nil(t, c2.Send([]byte(\"hello, c1\")))\n\tfor {\n\t\tdata, err = c1.Receive()\n\t\tif bytes.Equal(data, []byte{1}) {\n\t\t\tcontinue\n\t\t}\n\t\tbreak\n\t}\n\tassert.Nil(t, err)\n\tassert.Equal(t, []byte(\"hello, c1\"), data)\n\n\tc1.Close()\n\ttime.Sleep(300 * time.Millisecond)\n}\n```"]}
{"repo": "schollz/croc", "name": "Send", "language": "go", "path": "src/message/message.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to securely transmit encrypted messages over a communication channel.\n2. **Input**: It takes a communication object, an encryption key, and a message structure containing various potential fields such as message type, content, and numerical identifiers.\n3. **Output**: The function returns an error if the message encoding or sending fails, otherwise it completes without returning any data.\n4. **Procedure**: The function first encodes the message using the provided encryption key. If encoding is successful, it sends the encoded message through the communication object. Any errors encountered during these steps are returned.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/utils/utils.go\npackage utils\n\nimport (\n\t\"archive/zip\"\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"crypto/md5\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math\"\n\t\"math/big\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cespare/xxhash\"\n\t\"github.com/kalafut/imohash\"\n\t\"github.com/schollz/mnemonicode\"\n)\n\nconst NbPinNumbers = 4\nconst NbBytesWords = 4\n\n// Get or create home directory\nfunc GetConfigDir(requireValidPath bool) (homedir string, err error) {\n\tif envHomedir, isSet := os.LookupEnv(\"CROC_CONFIG_DIR\"); isSet {\n\t\thomedir = envHomedir\n\t} else if xdgConfigHome, isSet := os.LookupEnv(\"XDG_CONFIG_HOME\"); isSet {\n\t\thomedir = path.Join(xdgConfigHome, \"croc\")\n\t} else {\n\t\thomedir, err = os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tif !requireValidPath {\n\t\t\t\terr = nil\n\t\t\t\thomedir = \"\"\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\thomedir = path.Join(homedir, \".config\", \"croc\")\n\t}\n\n\tif requireValidPath {\n\t\tif _, err = os.Stat(homedir); os.IsNotExist(err) {\n\t\t\terr = os.MkdirAll(homedir, 0o700)\n\t\t}\n\t}\n\treturn\n}\n\n// Exists reports whether the named file or directory exists.\nfunc Exists(name string) bool {\n\tif _, err := os.Stat(name); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// GetInput returns the input with a given prompt\nfunc GetInput(prompt string) string {\n\treader := bufio.NewReader(os.Stdin)\n\tfmt.Fprintf(os.Stderr, \"%s\", prompt)\n\ttext, _ := reader.ReadString('\\n')\n\treturn strings.TrimSpace(text)\n}\n\n// HashFile returns the hash of a file or, in case of a symlink, the\n// SHA256 hash of its target. Takes an argument to specify the algorithm to use.\nfunc HashFile(fname string, algorithm string) (hash256 []byte, err error) {\n\tvar fstats os.FileInfo\n\tfstats, err = os.Lstat(fname)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif fstats.Mode()&os.ModeSymlink != 0 {\n\t\tvar target string\n\t\ttarget, err = os.Readlink(fname)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn []byte(SHA256(target)), nil\n\t}\n\tswitch algorithm {\n\tcase \"imohash\":\n\t\treturn IMOHashFile(fname)\n\tcase \"md5\":\n\t\treturn MD5HashFile(fname)\n\tcase \"xxhash\":\n\t\treturn XXHashFile(fname)\n\t}\n\terr = fmt.Errorf(\"unspecified algorithm\")\n\treturn\n}\n\n// MD5HashFile returns MD5 hash\nfunc MD5HashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := md5.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// IMOHashFile returns imohash\nfunc IMOHashFile(fname string) (hash []byte, err error) {\n\tb, err := imohash.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\nvar imofull = imohash.NewCustom(0, 0)\n\n// IMOHashFileFull returns imohash of full file\nfunc IMOHashFileFull(fname string) (hash []byte, err error) {\n\tb, err := imofull.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\n// XXHashFile returns the xxhash of a file\nfunc XXHashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := xxhash.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// SHA256 returns sha256 sum\nfunc SHA256(s string) string {\n\tsha := sha256.New()\n\tsha.Write([]byte(s))\n\treturn hex.EncodeToString(sha.Sum(nil))\n}\n\n// PublicIP returns public ip address\nfunc PublicIP() (ip string, err error) {\n\tresp, err := http.Get(\"https://canhazip.com\")\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode == http.StatusOK {\n\t\tbodyBytes, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tip = strings.TrimSpace(string(bodyBytes))\n\t}\n\treturn\n}\n\n// LocalIP returns local ip address\nfunc LocalIP() string {\n\tconn, err := net.Dial(\"udp\", \"8.8.8.8:80\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tlocalAddr := conn.LocalAddr().(*net.UDPAddr)\n\n\treturn localAddr.IP.String()\n}\n\nfunc GenerateRandomPin() string {\n\ts := \"\"\n\tmax := new(big.Int)\n\tmax.SetInt64(9)\n\tfor i := 0; i < NbPinNumbers; i++ {\n\t\tv, err := rand.Int(rand.Reader, max)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\ts += fmt.Sprintf(\"%d\", v)\n\t}\n\treturn s\n}\n\n// GetRandomName returns mnemonicoded random name\nfunc GetRandomName() string {\n\tvar result []string\n\tbs := make([]byte, NbBytesWords)\n\trand.Read(bs)\n\tresult = mnemonicode.EncodeWordList(result, bs)\n\treturn GenerateRandomPin() + \"-\" + strings.Join(result, \"-\")\n}\n\n// ByteCountDecimal converts bytes to human readable byte string\nfunc ByteCountDecimal(b int64) string {\n\tconst unit = 1024\n\tif b < unit {\n\t\treturn fmt.Sprintf(\"%d B\", b)\n\t}\n\tdiv, exp := int64(unit), 0\n\tfor n := b / unit; n >= unit; n /= unit {\n\t\tdiv *= unit\n\t\texp++\n\t}\n\treturn fmt.Sprintf(\"%.1f %cB\", float64(b)/float64(div), \"kMGTPE\"[exp])\n}\n\n// MissingChunks returns the positions of missing chunks.\n// If file doesn't exist, it returns an empty chunk list (all chunks).\n// If the file size is not the same as requested, it returns an empty chunk list (all chunks).\nfunc MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\tfstat, err := os.Stat(fname)\n\tif err != nil || fstat.Size() != fsize {\n\t\treturn\n\t}\n\n\temptyBuffer := make([]byte, chunkSize)\n\tchunkNum := 0\n\tchunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))\n\tvar currentLocation int64\n\tfor {\n\t\tbuffer := make([]byte, chunkSize)\n\t\tbytesread, err := f.Read(buffer)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tif bytes.Equal(buffer[:bytesread], emptyBuffer[:bytesread]) {\n\t\t\tchunks[chunkNum] = currentLocation\n\t\t\tchunkNum++\n\t\t}\n\t\tcurrentLocation += int64(bytesread)\n\t}\n\tif chunkNum == 0 {\n\t\tchunkRanges = []int64{}\n\t} else {\n\t\tchunks = chunks[:chunkNum]\n\t\tchunkRanges = []int64{int64(chunkSize), chunks[0]}\n\t\tcurCount := 0\n\t\tfor i, chunk := range chunks {\n\t\t\tif i == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurCount++\n\t\t\tif chunk-chunks[i-1] > int64(chunkSize) {\n\t\t\t\tchunkRanges = append(chunkRanges, int64(curCount))\n\t\t\t\tchunkRanges = append(chunkRanges, chunk)\n\t\t\t\tcurCount = 0\n\t\t\t}\n\t\t}\n\t\tchunkRanges = append(chunkRanges, int64(curCount+1))\n\t}\n\treturn\n}\n\n// ChunkRangesToChunks converts chunk ranges to list\nfunc ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) {\n\tif len(chunkRanges) == 0 {\n\t\treturn\n\t}\n\tchunkSize := chunkRanges[0]\n\tchunks = []int64{}\n\tfor i := 1; i < len(chunkRanges); i += 2 {\n\t\tfor j := int64(0); j < (chunkRanges[i+1]); j++ {\n\t\t\tchunks = append(chunks, chunkRanges[i]+j*chunkSize)\n\t\t}\n\t}\n\treturn\n}\n\n// GetLocalIPs returns all local ips\nfunc GetLocalIPs() (ips []string, err error) {\n\taddrs, err := net.InterfaceAddrs()\n\tif err != nil {\n\t\treturn\n\t}\n\tips = []string{}\n\tfor _, address := range addrs {\n\t\t// check the address type and if it is not a loopback the display it\n\t\tif ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {\n\t\t\tif ipnet.IP.To4() != nil {\n\t\t\t\tips = append(ips, ipnet.IP.String())\n\t\t\t}\n\t\t}\n\t}\n\treturn\n}\n\nfunc RandomFileName() (fname string, err error) {\n\tf, err := os.CreateTemp(\".\", \"croc-stdin-\")\n\tif err != nil {\n\t\treturn\n\t}\n\tfname = f.Name()\n\t_ = f.Close()\n\treturn\n}\n\nfunc FindOpenPorts(host string, portNumStart, numPorts int) (openPorts []int) {\n\topenPorts = []int{}\n\tfor port := portNumStart; port-portNumStart < 200; port++ {\n\t\ttimeout := 100 * time.Millisecond\n\t\tconn, err := net.DialTimeout(\"tcp\", net.JoinHostPort(host, fmt.Sprint(port)), timeout)\n\t\tif conn != nil {\n\t\t\tconn.Close()\n\t\t} else if err != nil {\n\t\t\topenPorts = append(openPorts, port)\n\t\t}\n\t\tif len(openPorts) >= numPorts {\n\t\t\treturn\n\t\t}\n\t}\n\treturn\n}\n\n// local ip determination\n// https://stackoverflow.com/questions/41240761/check-if-ip-address-is-in-private-network-space\nvar privateIPBlocks []*net.IPNet\n\nfunc init() {\n\tfor _, cidr := range []string{\n\t\t\"127.0.0.0/8\",    // IPv4 loopback\n\t\t\"10.0.0.0/8\",     // RFC1918\n\t\t\"172.16.0.0/12\",  // RFC1918\n\t\t\"192.168.0.0/16\", // RFC1918\n\t\t\"169.254.0.0/16\", // RFC3927 link-local\n\t\t\"::1/128\",        // IPv6 loopback\n\t\t\"fe80::/10\",      // IPv6 link-local\n\t\t\"fc00::/7\",       // IPv6 unique local addr\n\t} {\n\t\t_, block, err := net.ParseCIDR(cidr)\n\t\tif err != nil {\n\t\t\tpanic(fmt.Errorf(\"parse error on %q: %v\", cidr, err))\n\t\t}\n\t\tprivateIPBlocks = append(privateIPBlocks, block)\n\t}\n}\n\nfunc IsLocalIP(ipaddress string) bool {\n\tif strings.Contains(ipaddress, \"127.0.0.1\") {\n\t\treturn true\n\t}\n\thost, _, _ := net.SplitHostPort(ipaddress)\n\tip := net.ParseIP(host)\n\tif ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {\n\t\treturn true\n\t}\n\tfor _, block := range privateIPBlocks {\n\t\tif block.Contains(ip) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc ZipDirectory(destination string, source string) (err error) {\n\tif _, err = os.Stat(destination); err == nil {\n\t\tlog.Fatalf(\"%s file already exists!\\n\", destination)\n\t}\n\tfmt.Fprintf(os.Stderr, \"Zipping %s to %s\\n\", source, destination)\n\tfile, err := os.Create(destination)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tdefer file.Close()\n\twriter := zip.NewWriter(file)\n\t// no compression because croc does its compression on the fly\n...\n// Path: src/comm/comm.go\npackage comm\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/magisterquis/connectproxy\"\n\t\"github.com/schollz/croc/v9/src/utils\"\n\tlog \"github.com/schollz/logger\"\n\t\"golang.org/x/net/proxy\"\n)\n\nvar Socks5Proxy = \"\"\nvar HttpProxy = \"\"\n\nvar MAGIC_BYTES = []byte(\"croc\")\n\n// Comm is some basic TCP communication\ntype Comm struct {\n\tconnection net.Conn\n}\n\n// NewConnection gets a new comm to a tcp address\nfunc NewConnection(address string, timelimit ...time.Duration) (c *Comm, err error) {\n\ttlimit := 30 * time.Second\n\tif len(timelimit) > 0 {\n\t\ttlimit = timelimit[0]\n\t}\n\tvar connection net.Conn\n\tif Socks5Proxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(Socks5Proxy, `://`) {\n\t\t\tSocks5Proxy = `socks5://` + Socks5Proxy\n\t\t}\n\t\tsocks5ProxyURL, urlParseError := url.Parse(Socks5Proxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse socks proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = proxy.FromURL(socks5ProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\t} else if HttpProxy != \"\" && !utils.IsLocalIP(address) {\n\t\tvar dialer proxy.Dialer\n\t\t// prepend schema if no schema is given\n\t\tif !strings.Contains(HttpProxy, `://`) {\n\t\t\tHttpProxy = `http://` + HttpProxy\n\t\t}\n\t\tHttpProxyURL, urlParseError := url.Parse(HttpProxy)\n\t\tif urlParseError != nil {\n\t\t\terr = fmt.Errorf(\"unable to parse http proxy url: %s\", urlParseError)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tdialer, err = connectproxy.New(HttpProxyURL, proxy.Direct)\n\t\tif err != nil {\n\t\t\terr = fmt.Errorf(\"proxy failed: %w\", err)\n\t\t\tlog.Debug(err)\n\t\t\treturn\n\t\t}\n\t\tlog.Debug(\"dialing with dialer.Dial\")\n\t\tconnection, err = dialer.Dial(\"tcp\", address)\n\n\t} else {\n\t\tlog.Debugf(\"dialing to %s with timelimit %s\", address, tlimit)\n\t\tconnection, err = net.DialTimeout(\"tcp\", address, tlimit)\n\t}\n\tif err != nil {\n\t\terr = fmt.Errorf(\"comm.NewConnection failed: %w\", err)\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tc = New(connection)\n\tlog.Debugf(\"connected to '%s'\", address)\n\treturn\n}\n\n// New returns a new comm\nfunc New(c net.Conn) *Comm {\n\tif err := c.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\tif err := c.SetDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting overall deadline: %v\", err)\n\t}\n\tif err := c.SetWriteDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Errorf(\"error setting write deadline: %v\", err)\n\t}\n\tcomm := new(Comm)\n\tcomm.connection = c\n\treturn comm\n}\n\n// Connection returns the net.Conn connection\nfunc (c *Comm) Connection() net.Conn {\n\treturn c.connection\n}\n\n// Close closes the connection\nfunc (c *Comm) Close() {\n\tif err := c.connection.Close(); err != nil {\n\t\tlog.Warnf(\"error closing connection: %v\", err)\n\t}\n}\n\nfunc (c *Comm) Write(b []byte) (n int, err error) {\n\theader := new(bytes.Buffer)\n\terr = binary.Write(header, binary.LittleEndian, uint32(len(b)))\n\tif err != nil {\n\t\tfmt.Println(\"binary.Write failed:\", err)\n\t}\n\ttmpCopy := append(header.Bytes(), b...)\n\ttmpCopy = append(MAGIC_BYTES, tmpCopy...)\n\tn, err = c.connection.Write(tmpCopy)\n\tif err != nil {\n\t\terr = fmt.Errorf(\"connection.Write failed: %w\", err)\n\t\treturn\n\t}\n\tif n != len(tmpCopy) {\n\t\terr = fmt.Errorf(\"wanted to write %d but wrote %d\", len(b), n)\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (c *Comm) Read() (buf []byte, numBytes int, bs []byte, err error) {\n\t// long read deadline in case waiting for file\n\tif err = c.connection.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\t// must clear the timeout setting\n\tdefer c.connection.SetDeadline(time.Time{})\n\n\t// read until we get 4 bytes for the magic\n\theader := make([]byte, 4)\n\t_, err = io.ReadFull(c.connection, header)\n\tif err != nil {\n\t\tlog.Debugf(\"initial read error: %v\", err)\n\t\treturn\n\t}\n\tif !bytes.Equal(header, MAGIC_BYTES) {\n\t\terr = fmt.Errorf(\"initial bytes are not magic: %x\", header)\n\t\treturn\n\t}\n\n\t// read until we get 4 bytes for the header\n\theader = make([]byte, 4)\n\t_, err = io.ReadFull(c.connection, header)\n\tif err != nil {\n\t\tlog.Debugf(\"initial read error: %v\", err)\n\t\treturn\n\t}\n\n\tvar numBytesUint32 uint32\n\trbuf := bytes.NewReader(header)\n\terr = binary.Read(rbuf, binary.LittleEndian, &numBytesUint32)\n\tif err != nil {\n\t\terr = fmt.Errorf(\"binary.Read failed: %w\", err)\n\t\tlog.Debug(err.Error())\n\t\treturn\n\t}\n\tnumBytes = int(numBytesUint32)\n\n\t// shorten the reading deadline in case getting weird data\n\tif err = c.connection.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {\n\t\tlog.Warnf(\"error setting read deadline: %v\", err)\n\t}\n\tbuf = make([]byte, numBytes)\n\t_, err = io.ReadFull(c.connection, buf)\n\tif err != nil {\n\t\tlog.Debugf(\"consecutive read error: %v\", err)\n\t\treturn\n\t}\n\treturn\n}\n\n// Send a message\nfunc (c *Comm) Send(message []byte) (err error) {\n\t_, err = c.Write(message)\n\treturn\n}\n\n// Receive a message\nfunc (c *Comm) Receive() (b []byte, err error) {\n\tb, _, _, err = c.Read()\n\treturn\n}\n\n// Path: src/tcp/tcp.go\npackage tcp\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/schollz/pake/v3\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\t\"github.com/schollz/croc/v9/src/models\"\n)\n\ntype server struct {\n\thost       string\n\tport       string\n\tdebugLevel string\n\tbanner     string\n\tpassword   string\n\trooms      roomMap\n}\n\ntype roomInfo struct {\n\tfirst  *comm.Comm\n\tsecond *comm.Comm\n\topened time.Time\n\tfull   bool\n}\n\ntype roomMap struct {\n\trooms map[string]roomInfo\n\tsync.Mutex\n}\n\nconst pingRoom = \"pinglkasjdlfjsaldjf\"\n\nvar timeToRoomDeletion = 10 * time.Minute\n\n// Run starts a tcp listener, run async\nfunc Run(debugLevel, host, port, password string, banner ...string) (err error) {\n\ts := new(server)\n\ts.host = host\n\ts.port = port\n\ts.password = password\n\ts.debugLevel = debugLevel\n\tif len(banner) > 0 {\n\t\ts.banner = banner[0]\n\t}\n\treturn s.start()\n}\n\nfunc (s *server) start() (err error) {\n\tlog.SetLevel(s.debugLevel)\n\tlog.Debugf(\"starting with password '%s'\", s.password)\n\ts.rooms.Lock()\n\ts.rooms.rooms = make(map[string]roomInfo)\n\ts.rooms.Unlock()\n\n\t// delete old rooms\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(timeToRoomDeletion)\n\t\t\tvar roomsToDelete []string\n\t\t\ts.rooms.Lock()\n\t\t\tfor room := range s.rooms.rooms {\n\t\t\t\tif time.Since(s.rooms.rooms[room].opened) > 3*time.Hour {\n\t\t\t\t\troomsToDelete = append(roomsToDelete, room)\n\t\t\t\t}\n\t\t\t}\n\t\t\ts.rooms.Unlock()\n\n\t\t\tfor _, room := range roomsToDelete {\n\t\t\t\ts.deleteRoom(room)\n\t\t\t}\n\t\t}\n\t}()\n\n\terr = s.run()\n\tif err != nil {\n\t\tlog.Error(err)\n\t}\n\treturn\n}\n\nfunc (s *server) run() (err error) {\n\tnetwork := \"tcp\"\n\taddr := net.JoinHostPort(s.host, s.port)\n\tif s.host != \"\" {\n\t\tip := net.ParseIP(s.host)\n\t\tif ip == nil {\n\t\t\tvar tcpIP *net.IPAddr\n\t\t\ttcpIP, err = net.ResolveIPAddr(\"ip\", s.host)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tip = tcpIP.IP\n\t\t}\n\t\taddr = net.JoinHostPort(ip.String(), s.port)\n\t\tif s.host != \"\" {\n\t\t\tif ip.To4() != nil {\n\t\t\t\tnetwork = \"tcp4\"\n\t\t\t} else {\n\t\t\t\tnetwork = \"tcp6\"\n\t\t\t}\n\t\t}\n\t}\n\taddr = strings.Replace(addr, \"127.0.0.1\", \"0.0.0.0\", 1)\n\tlog.Infof(\"starting TCP server on \" + addr)\n\tserver, err := net.Listen(network, addr)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"error listening on %s: %w\", addr, err)\n\t}\n\tdefer server.Close()\n\t// spawn a new goroutine whenever a client connects\n\tfor {\n\t\tconnection, err := server.Accept()\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"problem accepting connection: %w\", err)\n\t\t}\n\t\tlog.Debugf(\"client %s connected\", connection.RemoteAddr().String())\n\t\tgo func(port string, connection net.Conn) {\n\t\t\tc := comm.New(connection)\n\t\t\troom, errCommunication := s.clientCommunication(port, c)\n\t\t\tlog.Debugf(\"room: %+v\", room)\n\t\t\tlog.Debugf(\"err: %+v\", errCommunication)\n\t\t\tif errCommunication != nil {\n\t\t\t\tlog.Debugf(\"relay-%s: %s\", connection.RemoteAddr().String(), errCommunication.Error())\n\t\t\t\tconnection.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif room == pingRoom {\n\t\t\t\tlog.Debugf(\"got ping\")\n\t\t\t\tconnection.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfor {\n\t\t\t\t// check connection\n\t\t\t\tlog.Debugf(\"checking connection of room %s for %+v\", room, c)\n\t\t\t\tdeleteIt := false\n\t\t\t\ts.rooms.Lock()\n\t\t\t\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\t\t\t\tlog.Debug(\"room is gone\")\n\t\t\t\t\ts.rooms.Unlock()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tlog.Debugf(\"room: %+v\", s.rooms.rooms[room])\n\t\t\t\tif s.rooms.rooms[room].first != nil && s.rooms.rooms[room].second != nil {\n\t\t\t\t\tlog.Debug(\"rooms ready\")\n\t\t\t\t\ts.rooms.Unlock()\n\t\t\t\t\tbreak\n\t\t\t\t} else {\n\t\t\t\t\tif s.rooms.rooms[room].first != nil {\n\t\t\t\t\t\terrSend := s.rooms.rooms[room].first.Send([]byte{1})\n\t\t\t\t\t\tif errSend != nil {\n\t\t\t\t\t\t\tlog.Debug(errSend)\n\t\t\t\t\t\t\tdeleteIt = true\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\ts.rooms.Unlock()\n\t\t\t\tif deleteIt {\n\t\t\t\t\ts.deleteRoom(room)\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\ttime.Sleep(1 * time.Second)\n\t\t\t}\n\t\t}(s.port, connection)\n\t}\n}\n\nvar weakKey = []byte{1, 2, 3}\n\nfunc (s *server) clientCommunication(port string, c *comm.Comm) (room string, err error) {\n\t// establish secure password with PAKE for communication with relay\n\tB, err := pake.InitCurve(weakKey, 1, \"siec\")\n\tif err != nil {\n\t\treturn\n\t}\n\tAbytes, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tlog.Debugf(\"Abytes: %s\", Abytes)\n\tif bytes.Equal(Abytes, []byte(\"ping\")) {\n\t\troom = pingRoom\n\t\tlog.Debug(\"sending back pong\")\n\t\tc.Send([]byte(\"pong\"))\n\t\treturn\n\t}\n\terr = B.Update(Abytes)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(B.Bytes())\n\tif err != nil {\n\t\treturn\n\t}\n\tstrongKey, err := B.SessionKey()\n\tif err != nil {\n\t\treturn\n\t}\n\tlog.Debugf(\"strongkey: %x\", strongKey)\n\n\t// receive salt\n\tsalt, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tstrongKeyForEncryption, _, err := crypt.New(strongKey, salt)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tlog.Debugf(\"waiting for password\")\n\tpasswordBytesEnc, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\tpasswordBytes, err := crypt.Decrypt(passwordBytesEnc, strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\tif strings.TrimSpace(string(passwordBytes)) != s.password {\n\t\terr = fmt.Errorf(\"bad password\")\n\t\tenc, _ := crypt.Encrypt([]byte(err.Error()), strongKeyForEncryption)\n\t\tif err = c.Send(enc); err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"send error: %w\", err)\n\t\t}\n\t\treturn\n\t}\n\n\t// send ok to tell client they are connected\n\tbanner := s.banner\n\tif len(banner) == 0 {\n\t\tbanner = \"ok\"\n\t}\n\tlog.Debugf(\"sending '%s'\", banner)\n\tbSend, err := crypt.Encrypt([]byte(banner+\"|||\"+c.Connection().RemoteAddr().String()), strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// wait for client to tell me which room they want\n\tlog.Debug(\"waiting for answer\")\n\tenc, err := c.Receive()\n\tif err != nil {\n\t\treturn\n\t}\n\troomBytes, err := crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\troom = string(roomBytes)\n\n\ts.rooms.Lock()\n\t// create the room if it is new\n\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\ts.rooms.rooms[room] = roomInfo{\n\t\t\tfirst:  c,\n\t\t\topened: time.Now(),\n\t\t}\n\t\ts.rooms.Unlock()\n\t\t// tell the client that they got the room\n\n\t\tbSend, err = crypt.Encrypt([]byte(\"ok\"), strongKeyForEncryption)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\terr = c.Send(bSend)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\ts.deleteRoom(room)\n\t\t\treturn\n\t\t}\n\t\tlog.Debugf(\"room %s has 1\", room)\n\t\treturn\n\t}\n\tif s.rooms.rooms[room].full {\n\t\ts.rooms.Unlock()\n\t\tbSend, err = crypt.Encrypt([]byte(\"room full\"), strongKeyForEncryption)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\terr = c.Send(bSend)\n\t\tif err != nil {\n\t\t\tlog.Error(err)\n\t\t\treturn\n\t\t}\n\t\treturn\n\t}\n\tlog.Debugf(\"room %s has 2\", room)\n\ts.rooms.rooms[room] = roomInfo{\n\t\tfirst:  s.rooms.rooms[room].first,\n\t\tsecond: c,\n\t\topened: s.rooms.rooms[room].opened,\n\t\tfull:   true,\n\t}\n\totherConnection := s.rooms.rooms[room].first\n\ts.rooms.Unlock()\n\n\t// second connection is the sender, time to staple connections\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\n\t// start piping\n\tgo func(com1, com2 *comm.Comm, wg *sync.WaitGroup) {\n\t\tlog.Debug(\"starting pipes\")\n\t\tpipe(com1.Connection(), com2.Connection())\n\t\twg.Done()\n\t\tlog.Debug(\"done piping\")\n\t}(otherConnection, c, &wg)\n\n\t// tell the sender everything is ready\n\tbSend, err = crypt.Encrypt([]byte(\"ok\"), strongKeyForEncryption)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\ts.deleteRoom(room)\n\t\treturn\n\t}\n\twg.Wait()\n\n\t// delete room\n\ts.deleteRoom(room)\n\treturn\n}\n\nfunc (s *server) deleteRoom(room string) {\n\ts.rooms.Lock()\n\tdefer s.rooms.Unlock()\n\tif _, ok := s.rooms.rooms[room]; !ok {\n\t\treturn\n\t}\n\tlog.Debugf(\"deleting room: %s\", room)\n\tif s.rooms.rooms[room].first != nil {\n\t\ts.rooms.rooms[room].first.Close()\n\t}\n\tif s.rooms.rooms[room].second != nil {\n\t\ts.rooms.rooms[room].second.Close()\n\t}\n\ts.rooms.rooms[room] = roomInfo{first: nil, second: nil}\n\tdelete(s.rooms.rooms, room)\n}\n\n// chanFromConn creates a channel from a Conn object, and sends everything it\n//\n//\tRead()s from the socket to the channel.\nfunc chanFromConn(conn net.Conn) chan []byte {\n\tc := make(chan []byte, 1)\n\tif err := conn.SetReadDeadline(time.Now().Add(3 * time.Hour)); err != nil {\n\t\tlog.Warnf(\"can't set read deadline: %v\", err)\n\t}\n\n\tgo func() {\n\t\tb := make([]byte, models.TCP_BUFFER_SIZE)\n\t\tfor {\n\t\t\tn, err := conn.Read(b)\n\t\t\tif n > 0 {\n\t\t\t\tres := make([]byte, n)\n\t\t\t\t// Copy the buffer so it doesn't get changed while read by the recipient.\n\t\t\t\tcopy(res, b[:n])\n\t\t\t\tc <- res\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tlog.Debug(err)\n\t\t\t\tc <- nil\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tlog.Debug(\"exiting\")\n\t}()\n\n\treturn c\n}\n\n// pipe creates a full-duplex pipe between the two sockets and\n// transfers data from one to the other.\nfunc pipe(conn1 net.Conn, conn2 net.Conn) {\n\tchan1 := chanFromConn(conn1)\n\tchan2 := chanFromConn(conn2)\n\n\tfor {\n\t\tselect {\n\t\tcase b1 := <-chan1:\n\t\t\tif b1 == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif _, err := conn2.Write(b1); err != nil {\n\t\t\t\tlog.Errorf(\"write error on channel 1: %v\", err)\n\t\t\t}\n\n\t\tcase b2 := <-chan2:\n\t\t\tif b2 == nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif _, err := conn1.Write(b2); err != nil {\n\t\t\t\tlog.Errorf(\"write error on channel 2: %v\", err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc PingServer(address string) (err error) {\n\tlog.Debugf(\"pinging %s\", address)\n\tc, err := comm.NewConnection(address, 300*time.Millisecond)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send([]byte(\"ping\"))\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tb, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif bytes.Equal(b, []byte(\"pong\")) {\n\t\treturn nil\n\t}\n\treturn fmt.Errorf(\"no pong\")\n}\n\n// ConnectToTCPServer will initiate a new connection\n// to the specified address, room with optional time limit\nfunc ConnectToTCPServer(address, password, room string, timelimit ...time.Duration) (c *comm.Comm, banner string, ipaddr string, err error) {\n\tif len(timelimit) > 0 {\n\t\tc, err = comm.NewConnection(address, timelimit[0])\n\t} else {\n\t\tc, err = comm.NewConnection(address)\n\t}\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\n\t// get PAKE connection with server to establish strong key to transfer info\n\tA, err := pake.InitCurve(weakKey, 0, \"siec\")\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(A.Bytes())\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tBbytes, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = A.Update(Bbytes)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tstrongKey, err := A.SessionKey()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debugf(\"strong key: %x\", strongKey)\n\n\tstrongKeyForEncryption, salt, err := crypt.New(strongKey, nil)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\t// send salt\n\terr = c.Send(salt)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\n\tlog.Debug(\"sending password\")\n\tbSend, err := crypt.Encrypt([]byte(password), strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"waiting for first ok\")\n\tenc, err := c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tdata, err := crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif !strings.Contains(string(data), \"|||\") {\n\t\terr = fmt.Errorf(\"bad response: %s\", string(data))\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tbanner = strings.Split(string(data), \"|||\")[0]\n\tipaddr = strings.Split(string(data), \"|||\")[1]\n\tlog.Debug(\"sending room\")\n\tbSend, err = crypt.Encrypt([]byte(room), strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\terr = c.Send(bSend)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"waiting for room confirmation\")\n\tenc, err = c.Receive()\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tdata, err = crypt.Decrypt(enc, strongKeyForEncryption)\n\tif err != nil {\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tif !bytes.Equal(data, []byte(\"ok\")) {\n\t\terr = fmt.Errorf(\"got bad response: %s\", data)\n\t\tlog.Debug(err)\n\t\treturn\n\t}\n\tlog.Debug(\"all set\")\n\treturn\n}\n\n// Path: src/message/message.go\npackage message\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/compress\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\tlog \"github.com/schollz/logger\"\n)\n\n// Type is a message type\ntype Type string\n\nconst (\n\tTypePAKE           Type = \"pake\"\n\tTypeExternalIP     Type = \"externalip\"\n\tTypeFinished       Type = \"finished\"\n\tTypeError          Type = \"error\"\n\tTypeCloseRecipient Type = \"close-recipient\"\n\tTypeCloseSender    Type = \"close-sender\"\n\tTypeRecipientReady Type = \"recipientready\"\n\tTypeFileInfo       Type = \"fileinfo\"\n)\n\n// Message is the possible payload for messaging\ntype Message struct {\n\tType    Type   `json:\"t,omitempty\"`\n\tMessage string `json:\"m,omitempty\"`\n\tBytes   []byte `json:\"b,omitempty\"`\n\tBytes2  []byte `json:\"b2,omitempty\"`\n\tNum     int    `json:\"n,omitempty\"`\n}\n\nfunc (m Message) String() string {\n\tb, _ := json.Marshal(m)\n\treturn string(b)\n}\n\n// Send will send out\n\nfunc Send(c *comm.Comm, key []byte, m Message) (err error) {\n\tmSend, err := Encode(key, m)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(mSend)\n\treturn\n}\n\n// Encode will convert to bytes\nfunc Encode(key []byte, m Message) (b []byte, err error) {\n\tb, err = json.Marshal(m)\n\tif err != nil {\n\t\treturn\n\t}\n\tb = compress.Compress(b)\n\tif key != nil {\n\t\tlog.Debugf(\"writing %s message (encrypted)\", m.Type)\n\t\tb, err = crypt.Encrypt(b, key)\n\t} else {\n\t\tlog.Debugf(\"writing %s message (unencrypted)\", m.Type)\n\t}\n\treturn\n}\n\n// Decode will convert from bytes\nfunc Decode(key []byte, b []byte) (m Message, err error) {\n\tif key != nil {\n\t\tb, err = crypt.Decrypt(b, key)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\tb = compress.Decompress(b)\n\terr = json.Unmarshal(b, &m)\n\tif err == nil {\n\t\tif key != nil {\n\t\t\tlog.Debugf(\"read %s message (encrypted)\", m.Type)\n\t\t} else {\n\t\t\tlog.Debugf(\"read %s message (unencrypted)\", m.Type)\n\t\t}\n\t}\n\treturn\n}\n\n// Path: src/croc/croc.go\npackage croc\n\nimport (\n\t\"bytes\"\n\t\"crypto/rand\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\n\t\"golang.org/x/time/rate\"\n\n\t\"github.com/denisbrodbeck/machineid\"\n\tignore \"github.com/sabhiram/go-gitignore\"\n\tlog \"github.com/schollz/logger\"\n\t\"github.com/schollz/pake/v3\"\n\t\"github.com/schollz/peerdiscovery\"\n\t\"github.com/schollz/progressbar/v3\"\n\n\t\"github.com/schollz/croc/v9/src/comm\"\n\t\"github.com/schollz/croc/v9/src/compress\"\n\t\"github.com/schollz/croc/v9/src/crypt\"\n\t\"github.com/schollz/croc/v9/src/message\"\n\t\"github.com/schollz/croc/v9/src/models\"\n\t\"github.com/schollz/croc/v9/src/tcp\"\n\t\"github.com/schollz/croc/v9/src/utils\"\n)\n\nvar (\n\tipRequest        = []byte(\"ips?\")\n\thandshakeRequest = []byte(\"handshake\")\n)\n\nfunc init() {\n\tlog.SetLevel(\"debug\")\n}\n\n// Debug toggles debug mode\nfunc Debug(debug bool) {\n\tif debug {\n\t\tlog.SetLevel(\"debug\")\n\t} else {\n\t\tlog.SetLevel(\"warn\")\n\t}\n}\n\n// Options specifies user specific options\ntype Options struct {\n\tIsSender       bool\n\tSharedSecret   string\n\tDebug          bool\n\tRelayAddress   string\n\tRelayAddress6  string\n\tRelayPorts     []string\n\tRelayPassword  string\n\tStdout         bool\n\tNoPrompt       bool\n\tNoMultiplexing bool\n\tDisableLocal   bool\n\tOnlyLocal      bool\n\tIgnoreStdin    bool\n\tAsk            bool\n\tSendingText    bool\n\tNoCompress     bool\n\tIP             string\n\tOverwrite      bool\n\tCurve          string\n\tHashAlgorithm  string\n\tThrottleUpload string\n\tZipFolder      bool\n\tTestFlag       bool\n\tGitIgnore      bool\n}\n\n// Client holds the state of the croc transfer\ntype Client struct {\n\tOptions                         Options\n\tPake                            *pake.Pake\n\tKey                             []byte\n\tExternalIP, ExternalIPConnected string\n\n\t// steps involved in forming relationship\n\tStep1ChannelSecured       bool\n\tStep2FileInfoTransferred  bool\n\tStep3RecipientRequestFile bool\n\tStep4FileTransferred      bool\n\tStep5CloseChannels        bool\n\tSuccessfulTransfer        bool\n\n\t// send / receive information of all files\n\tFilesToTransfer           []FileInfo\n\tEmptyFoldersToTransfer    []FileInfo\n\tTotalNumberOfContents     int\n\tTotalNumberFolders        int\n\tFilesToTransferCurrentNum int\n\tFilesHasFinished          map[int]struct{}\n\tTotalFilesIgnored         int\n\n\t// send / receive information of current file\n\tCurrentFile            *os.File\n\tCurrentFileChunkRanges []int64\n\tCurrentFileChunks      []int64\n\tCurrentFileIsClosed    bool\n\tLastFolder             string\n\n\tTotalSent              int64\n\tTotalChunksTransferred int\n\tchunkMap               map[uint64]struct{}\n\tlimiter                *rate.Limiter\n\n\t// tcp connections\n\tconn []*comm.Comm\n\n\tbar             *progressbar.ProgressBar\n\tlongestFilename int\n\tfirstSend       bool\n\n\tmutex                    *sync.Mutex\n\tfread                    *os.File\n\tnumfinished              int\n\tquit                     chan bool\n\tfinishedNum              int\n\tnumberOfTransferredFiles int\n}\n\n// Chunk contains information about the\n// needed bytes\ntype Chunk struct {\n\tBytes    []byte `json:\"b,omitempty\"`\n\tLocation int64  `json:\"l,omitempty\"`\n}\n\n// FileInfo registers the information about the file\ntype FileInfo struct {\n\tName         string      `json:\"n,omitempty\"`\n\tFolderRemote string      `json:\"fr,omitempty\"`\n\tFolderSource string      `json:\"fs,omitempty\"`\n\tHash         []byte      `json:\"h,omitempty\"`\n\tSize         int64       `json:\"s,omitempty\"`\n\tModTime      time.Time   `json:\"m,omitempty\"`\n\tIsCompressed bool        `json:\"c,omitempty\"`\n\tIsEncrypted  bool        `json:\"e,omitempty\"`\n\tSymlink      string      `json:\"sy,omitempty\"`\n\tMode         os.FileMode `json:\"md,omitempty\"`\n\tTempFile     bool        `json:\"tf,omitempty\"`\n\tIsIgnored    bool        `json:\"ig,omitempty\"`\n}\n\n// RemoteFileRequest requests specific bytes\ntype RemoteFileRequest struct {\n\tCurrentFileChunkRanges    []int64\n\tFilesToTransferCurrentNum int\n\tMachineID                 string\n}\n\n// SenderInfo lists the files to be transferred\ntype SenderInfo struct {\n\tFilesToTransfer        []FileInfo\n\tEmptyFoldersToTransfer []FileInfo\n\tTotalNumberFolders     int\n\tMachineID              string\n\tAsk                    bool\n\tSendingText            bool\n\tNoCompress             bool\n\tHashAlgorithm          string\n}\n\n// New establishes a new connection for transferring files between two instances.\nfunc New(ops Options) (c *Client, err error) {\n\tc = new(Client)\n\tc.FilesHasFinished = make(map[int]struct{})\n\n\t// setup basic info\n\tc.Options = ops\n\tDebug(c.Options.Debug)\n\tlog.Debugf(\"options: %+v\", c.Options)\n\n\tif len(c.Options.SharedSecret) < 6 {\n\t\terr = fmt.Errorf(\"code is too short\")\n\t\treturn\n\t}\n\n\tc.conn = make([]*comm.Comm, 16)\n\n\t// initialize throttler\n\tif len(c.Options.ThrottleUpload) > 1 && c.Options.IsSender {\n\t\tupload := c.Options.ThrottleUpload[:len(c.Options.ThrottleUpload)-1]\n\t\tvar uploadLimit int64\n\t\tuploadLimit, err = strconv.ParseInt(upload, 10, 64)\n\t\tif err != nil {\n\t\t\tpanic(\"Could not parse given Upload Limit\")\n\t\t}\n\t\tminBurstSize := models.TCP_BUFFER_SIZE\n\t\tvar rt rate.Limit\n\t\tswitch unit := string(c.Options.ThrottleUpload[len(c.Options.ThrottleUpload)-1:]); unit {\n\t\tcase \"g\", \"G\":\n\t\t\tuploadLimit = uploadLimit * 1024 * 1024 * 1024\n\t\tcase \"m\", \"M\":\n\t\t\tuploadLimit = uploadLimit * 1024 * 1024\n\t\tcase \"k\", \"K\":\n\t\t\tuploadLimit = uploadLimit * 1024\n\t\tdefault:\n\t\t\tuploadLimit, err = strconv.ParseInt(c.Options.ThrottleUpload, 10, 64)\n\t\t\tif err != nil {\n\t\t\t\tpanic(\"Could not parse given Upload Limit\")\n\t\t\t}\n\t\t}\n\t\t// Somehow 4* is necessary\n\t\trt = rate.Every(time.Second / (4 * time.Duration(uploadLimit)))\n\t\tif int(uploadLimit) > minBurstSize {\n\t\t\tminBurstSize = int(uploadLimit)\n\t\t}\n\t\tc.limiter = rate.NewLimiter(rt, minBurstSize)\n\t\tlog.Debugf(\"Throttling Upload to %#v\", c.limiter.Limit())\n\t}\n\n\t// initialize pake for recipient\n\tif !c.Options.IsSender {\n\t\tc.Pake, err = pake.InitCurve([]byte(c.Options.SharedSecret[5:]), 0, c.Options.Curve)\n\t}\n\tif err != nil {\n\t\treturn\n\t}\n\n\tc.mutex = &sync.Mutex{}\n\treturn\n}\n\n// TransferOptions for sending\ntype TransferOptions struct {\n\tPathToFiles      []string\n\tKeepPathInRemote bool\n}\n\nfunc isEmptyFolder(folderPath string) (bool, error) {\n\tf, err := os.Open(folderPath)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\tdefer f.Close()\n\n\t_, err = f.Readdirnames(1)\n\tif err == io.EOF {\n\t\treturn true, nil\n\t}\n\treturn false, nil\n}\n\n// helper function to walk each subfolder and parses against an ignore file.\n// returns a hashmap Key: Absolute filepath, Value: boolean (true=ignore)\nfunc gitWalk(dir string, gitObj *ignore.GitIgnore, files map[string]bool) {\n\tvar ignoredDir bool\n\tvar current string\n\terr := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif isChild(current, path) && ignoredDir {\n\t\t\tfiles[path] = true\n\t\t\treturn nil\n\t\t}\n\t\tif info.IsDir() && filepath.Base(path) == filepath.Base(dir) {\n\t\t\tignoredDir = false // Skip applying ignore rules for root directory\n\t\t\treturn nil\n\t\t}\n\t\tif gitObj.MatchesPath(info.Name()) {\n\t\t\tfiles[path] = true\n\t\t\tignoredDir = true\n\t\t\tcurrent = path\n\t\t\treturn nil\n\t\t} else {\n\t\t\tfiles[path] = false\n\t\t\tignoredDir = false\n\t\t\treturn nil\n\t\t}\n\t})\n\tif err != nil {\n\t\tlog.Errorf(\"filepath error\")\n\t}\n}\n\nfunc isChild(parentPath, childPath string) bool {\n\trelPath, err := filepath.Rel(parentPath, childPath)\n\tif err != nil {\n\t\treturn false\n\t}\n\treturn !strings.HasPrefix(relPath, \"..\")\n\n}\n\n// This function retrieves the important file information\n// for every file that will be transferred\nfunc GetFilesInfo(fnames []string, zipfolder bool, ignoreGit bool) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) {\n\t// fnames: the relative/absolute paths of files/folders that will be transferred\n\ttotalNumberFolders = 0\n\tvar paths []string\n\tfor _, fname := range fnames {\n\t\t// Support wildcard\n\t\tif strings.Contains(fname, \"*\") {\n\t\t\tmatches, errGlob := filepath.Glob(fname)\n\t\t\tif errGlob != nil {\n\t\t\t\terr = errGlob\n\t\t\t\treturn\n\t\t\t}\n\t\t\tpaths = append(paths, matches...)\n\t\t\tcontinue\n\t\t} else {\n\t\t\tpaths = append(paths, fname)\n\t\t}\n\t}\n\tvar ignoredPaths = make(map[string]bool)\n\tif ignoreGit {\n\t\twd, wdErr := os.Stat(\".gitignore\")\n\t\tif wdErr == nil {\n\t\t\tgitIgnore, gitErr := ignore.CompileIgnoreFile(wd.Name())\n\t\t\tif gitErr == nil {\n\t\t\t\tfor _, path := range paths {\n\t\t\t\t\tabs, absErr := filepath.Abs(path)\n\t\t\t\t\tif absErr != nil {\n\t\t\t\t\t\terr = absErr\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tif gitIgnore.MatchesPath(path) {\n\t\t\t\t\t\tignoredPaths[abs] = true\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tfor _, path := range paths {\n\t\t\tabs, absErr := filepath.Abs(path)\n\t\t\tif absErr != nil {\n\t\t\t\terr = absErr\n\t\t\t\treturn\n\t\t\t}\n\t\t\tfile, fileErr := os.Stat(path)\n\t\t\tif fileErr == nil && file.IsDir() {\n\t\t\t\t_, subErr := os.Stat(filepath.Join(path, \".gitignore\"))\n\t\t\t\tif subErr == nil {\n\t\t\t\t\tgitObj, gitObjErr := ignore.CompileIgnoreFile(filepath.Join(path, \".gitignore\"))\n\t\t\t\t\tif gitObjErr != nil {\n\t\t\t\t\t\terr = gitObjErr\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\tgitWalk(abs, gitObj, ignoredPaths)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tfor _, fpath := range paths {\n\t\tstat, errStat := os.Lstat(fpath)\n\n\t\tif errStat != nil {\n\t\t\terr = errStat\n\t\t\treturn\n\t\t}\n\n...\n", "needle_token_start": 11205, "needle_token_end": 11270, "code_context_ntokens": 14984, "output": ["```go\nfunc Send(c *comm.Comm, key []byte, m Message) (err error) {\n\tmSend, err := Encode(key, m)\n\tif err != nil {\n\t\treturn\n\t}\n\terr = c.Send(mSend)\n\treturn\n}\n```"]}
{"repo": "schollz/croc", "name": "decompress", "language": "go", "path": "src/compress/compress.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: To restore data from its compressed form back to its original or usable format.\n2. **Input**: A byte slice containing compressed data.\n3. **Output**: A byte slice containing the original uncompressed data.\n4. **Procedure**: The function reads the compressed byte slice using a decompression reader, then writes the decompressed data into a buffer, which is returned as a byte slice.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/crypt/crypt.go\npackage crypt\n\nimport (\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"golang.org/x/crypto/argon2\"\n\t\"golang.org/x/crypto/chacha20poly1305\"\n\t\"golang.org/x/crypto/pbkdf2\"\n)\n\n// New generates a new key based on a passphrase and salt\nfunc New(passphrase []byte, usersalt []byte) (key []byte, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err := rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\tkey = pbkdf2.Key(passphrase, salt, 100, 32, sha256.New)\n\treturn\n}\n\n// Encrypt will encrypt using the pre-generated key\nfunc Encrypt(plaintext []byte, key []byte) (encrypted []byte, err error) {\n\t// generate a random iv each time\n\t// http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf\n\t// Section 8.2\n\tivBytes := make([]byte, 12)\n\tif _, err = rand.Read(ivBytes); err != nil {\n\t\tlog.Fatalf(\"can't initialize crypto: %v\", err)\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tencrypted = aesgcm.Seal(nil, ivBytes, plaintext, nil)\n\tencrypted = append(ivBytes, encrypted...)\n\treturn\n}\n\n// Decrypt using the pre-generated key\nfunc Decrypt(encrypted []byte, key []byte) (plaintext []byte, err error) {\n\tif len(encrypted) < 13 {\n\t\terr = fmt.Errorf(\"incorrect passphrase\")\n\t\treturn\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tplaintext, err = aesgcm.Open(nil, encrypted[:12], encrypted[12:], nil)\n\treturn\n}\n\n// NewArgon2 generates a new key based on a passphrase and salt\n// using argon2\n// https://pkg.go.dev/golang.org/x/crypto/argon2\nfunc NewArgon2(passphrase []byte, usersalt []byte) (aead cipher.AEAD, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err = rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\taead, err = chacha20poly1305.NewX(argon2.IDKey(passphrase, salt, 1, 64*1024, 4, 32))\n\treturn\n}\n\n// EncryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc EncryptChaCha(plaintext []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tnonce := make([]byte, aead.NonceSize(), aead.NonceSize()+len(plaintext)+aead.Overhead())\n\tif _, err := rand.Read(nonce); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Encrypt the message and append the ciphertext to the nonce.\n\tencrypted = aead.Seal(nonce, nonce, plaintext, nil)\n\treturn\n}\n\n// DecryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc DecryptChaCha(encryptedMsg []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tif len(encryptedMsg) < aead.NonceSize() {\n\t\terr = fmt.Errorf(\"ciphertext too short\")\n\t\treturn\n\t}\n\n\t// Split nonce and ciphertext.\n\tnonce, ciphertext := encryptedMsg[:aead.NonceSize()], encryptedMsg[aead.NonceSize():]\n\n\t// Decrypt the message and check it wasn't tampered with.\n\tencrypted, err = aead.Open(nil, nonce, ciphertext, nil)\n\treturn\n}\n\n// Path: src/compress/compress.go\npackage compress\n\nimport (\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"io\"\n\n\tlog \"github.com/schollz/logger\"\n)\n\n// CompressWithOption returns compressed data using the specified level\nfunc CompressWithOption(src []byte, level int) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, level)\n\treturn compressedData.Bytes()\n}\n\n// Compress returns a compressed byte slice.\nfunc Compress(src []byte) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, -2)\n\treturn compressedData.Bytes()\n}\n\n// Decompress returns a decompressed byte slice.\nfunc Decompress(src []byte) []byte {\n\tcompressedData := bytes.NewBuffer(src)\n\tdeCompressedData := new(bytes.Buffer)\n\tdecompress(compressedData, deCompressedData)\n\treturn deCompressedData.Bytes()\n}\n\n// compress uses flate to compress a byte slice to a corresponding level\nfunc compress(src []byte, dest io.Writer, level int) {\n\tcompressor, err := flate.NewWriter(dest, level)\n\tif err != nil {\n\t\tlog.Debugf(\"error level data: %v\", err)\n\t\treturn\n\t}\n\tif _, err := compressor.Write(src); err != nil {\n\t\tlog.Debugf(\"error writing data: %v\", err)\n\t}\n\tcompressor.Close()\n}\n\n// compress uses flate to decompress an io.Reader\n\nfunc decompress(src io.Reader, dest io.Writer) {\n\tdecompressor := flate.NewReader(src)\n\tif _, err := io.Copy(dest, decompressor); err != nil {\n\t\tlog.Debugf(\"error copying data: %v\", err)\n\t}\n\tdecompressor.Close()\n}\n\n// Path: src/utils/utils.go\npackage utils\n\nimport (\n\t\"archive/zip\"\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"crypto/md5\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math\"\n\t\"math/big\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cespare/xxhash\"\n\t\"github.com/kalafut/imohash\"\n\t\"github.com/schollz/mnemonicode\"\n)\n\nconst NbPinNumbers = 4\nconst NbBytesWords = 4\n\n// Get or create home directory\nfunc GetConfigDir(requireValidPath bool) (homedir string, err error) {\n\tif envHomedir, isSet := os.LookupEnv(\"CROC_CONFIG_DIR\"); isSet {\n\t\thomedir = envHomedir\n\t} else if xdgConfigHome, isSet := os.LookupEnv(\"XDG_CONFIG_HOME\"); isSet {\n\t\thomedir = path.Join(xdgConfigHome, \"croc\")\n\t} else {\n\t\thomedir, err = os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tif !requireValidPath {\n\t\t\t\terr = nil\n\t\t\t\thomedir = \"\"\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\thomedir = path.Join(homedir, \".config\", \"croc\")\n\t}\n\n\tif requireValidPath {\n\t\tif _, err = os.Stat(homedir); os.IsNotExist(err) {\n\t\t\terr = os.MkdirAll(homedir, 0o700)\n\t\t}\n\t}\n\treturn\n}\n\n// Exists reports whether the named file or directory exists.\nfunc Exists(name string) bool {\n\tif _, err := os.Stat(name); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// GetInput returns the input with a given prompt\nfunc GetInput(prompt string) string {\n\treader := bufio.NewReader(os.Stdin)\n\tfmt.Fprintf(os.Stderr, \"%s\", prompt)\n\ttext, _ := reader.ReadString('\\n')\n\treturn strings.TrimSpace(text)\n}\n\n// HashFile returns the hash of a file or, in case of a symlink, the\n// SHA256 hash of its target. Takes an argument to specify the algorithm to use.\nfunc HashFile(fname string, algorithm string) (hash256 []byte, err error) {\n\tvar fstats os.FileInfo\n\tfstats, err = os.Lstat(fname)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif fstats.Mode()&os.ModeSymlink != 0 {\n\t\tvar target string\n\t\ttarget, err = os.Readlink(fname)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn []byte(SHA256(target)), nil\n\t}\n\tswitch algorithm {\n\tcase \"imohash\":\n\t\treturn IMOHashFile(fname)\n\tcase \"md5\":\n\t\treturn MD5HashFile(fname)\n\tcase \"xxhash\":\n\t\treturn XXHashFile(fname)\n\t}\n\terr = fmt.Errorf(\"unspecified algorithm\")\n\treturn\n}\n\n// MD5HashFile returns MD5 hash\nfunc MD5HashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := md5.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// IMOHashFile returns imohash\nfunc IMOHashFile(fname string) (hash []byte, err error) {\n\tb, err := imohash.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\nvar imofull = imohash.NewCustom(0, 0)\n\n// IMOHashFileFull returns imohash of full file\nfunc IMOHashFileFull(fname string) (hash []byte, err error) {\n\tb, err := imofull.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\n// XXHashFile returns the xxhash of a file\nfunc XXHashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := xxhash.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// SHA256 returns sha256 sum\nfunc SHA256(s string) string {\n\tsha := sha256.New()\n\tsha.Write([]byte(s))\n\treturn hex.EncodeToString(sha.Sum(nil))\n}\n\n// PublicIP returns public ip address\nfunc PublicIP() (ip string, err error) {\n\tresp, err := http.Get(\"https://canhazip.com\")\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode == http.StatusOK {\n\t\tbodyBytes, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tip = strings.TrimSpace(string(bodyBytes))\n\t}\n\treturn\n}\n\n// LocalIP returns local ip address\nfunc LocalIP() string {\n\tconn, err := net.Dial(\"udp\", \"8.8.8.8:80\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tlocalAddr := conn.LocalAddr().(*net.UDPAddr)\n\n\treturn localAddr.IP.String()\n}\n\nfunc GenerateRandomPin() string {\n\ts := \"\"\n\tmax := new(big.Int)\n\tmax.SetInt64(9)\n\tfor i := 0; i < NbPinNumbers; i++ {\n\t\tv, err := rand.Int(rand.Reader, max)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\ts += fmt.Sprintf(\"%d\", v)\n\t}\n\treturn s\n}\n\n// GetRandomName returns mnemonicoded random name\nfunc GetRandomName() string {\n\tvar result []string\n\tbs := make([]byte, NbBytesWords)\n\trand.Read(bs)\n\tresult = mnemonicode.EncodeWordList(result, bs)\n\treturn GenerateRandomPin() + \"-\" + strings.Join(result, \"-\")\n}\n\n// ByteCountDecimal converts bytes to human readable byte string\nfunc ByteCountDecimal(b int64) string {\n\tconst unit = 1024\n\tif b < unit {\n\t\treturn fmt.Sprintf(\"%d B\", b)\n\t}\n\tdiv, exp := int64(unit), 0\n\tfor n := b / unit; n >= unit; n /= unit {\n\t\tdiv *= unit\n\t\texp++\n\t}\n\treturn fmt.Sprintf(\"%.1f %cB\", float64(b)/float64(div), \"kMGTPE\"[exp])\n}\n\n// MissingChunks returns the positions of missing chunks.\n// If file doesn't exist, it returns an empty chunk list (all chunks).\n// If the file size is not the same as requested, it returns an empty chunk list (all chunks).\nfunc MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\tfstat, err := os.Stat(fname)\n\tif err != nil || fstat.Size() != fsize {\n\t\treturn\n\t}\n\n\temptyBuffer := make([]byte, chunkSize)\n\tchunkNum := 0\n\tchunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))\n\tvar currentLocation int64\n\tfor {\n...\n", "needle_token_start": 1910, "needle_token_end": 1991, "code_context_ntokens": 4207, "output": ["```go\n// Decompress returns a decompressed byte slice.\nfunc Decompress(src []byte) []byte {\n\tcompressedData := bytes.NewBuffer(src)\n\tdeCompressedData := new(bytes.Buffer)\n\tdecompress(compressedData, deCompressedData)\n\treturn deCompressedData.Bytes()\n}\n```"]}
{"repo": "schollz/croc", "name": "GetRandomName", "language": "go", "path": "src/utils/utils.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To generate a unique identifier that combines a random numeric pin with a mnemonic-encoded string derived from random bytes.\n2. **Input**: None.\n3. **Output**: A string that consists of a random numeric pin followed by a hyphen and a mnemonic-encoded sequence.\n4. **Procedure**: \n   - Generate a random numeric pin.\n   - Create a byte array of predefined size and populate it with random bytes.\n   - Encode these bytes into a mnemonic word list.\n   - Concatenate the numeric pin, a hyphen, and the mnemonic word list into a single string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: src/crypt/crypt.go\npackage crypt\n\nimport (\n\t\"crypto/aes\"\n\t\"crypto/cipher\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"golang.org/x/crypto/argon2\"\n\t\"golang.org/x/crypto/chacha20poly1305\"\n\t\"golang.org/x/crypto/pbkdf2\"\n)\n\n// New generates a new key based on a passphrase and salt\nfunc New(passphrase []byte, usersalt []byte) (key []byte, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err := rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\tkey = pbkdf2.Key(passphrase, salt, 100, 32, sha256.New)\n\treturn\n}\n\n// Encrypt will encrypt using the pre-generated key\nfunc Encrypt(plaintext []byte, key []byte) (encrypted []byte, err error) {\n\t// generate a random iv each time\n\t// http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf\n\t// Section 8.2\n\tivBytes := make([]byte, 12)\n\tif _, err = rand.Read(ivBytes); err != nil {\n\t\tlog.Fatalf(\"can't initialize crypto: %v\", err)\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tencrypted = aesgcm.Seal(nil, ivBytes, plaintext, nil)\n\tencrypted = append(ivBytes, encrypted...)\n\treturn\n}\n\n// Decrypt using the pre-generated key\nfunc Decrypt(encrypted []byte, key []byte) (plaintext []byte, err error) {\n\tif len(encrypted) < 13 {\n\t\terr = fmt.Errorf(\"incorrect passphrase\")\n\t\treturn\n\t}\n\tb, err := aes.NewCipher(key)\n\tif err != nil {\n\t\treturn\n\t}\n\taesgcm, err := cipher.NewGCM(b)\n\tif err != nil {\n\t\treturn\n\t}\n\tplaintext, err = aesgcm.Open(nil, encrypted[:12], encrypted[12:], nil)\n\treturn\n}\n\n// NewArgon2 generates a new key based on a passphrase and salt\n// using argon2\n// https://pkg.go.dev/golang.org/x/crypto/argon2\nfunc NewArgon2(passphrase []byte, usersalt []byte) (aead cipher.AEAD, salt []byte, err error) {\n\tif len(passphrase) < 1 {\n\t\terr = fmt.Errorf(\"need more than that for passphrase\")\n\t\treturn\n\t}\n\tif usersalt == nil {\n\t\tsalt = make([]byte, 8)\n\t\t// http://www.ietf.org/rfc/rfc2898.txt\n\t\t// Salt.\n\t\tif _, err = rand.Read(salt); err != nil {\n\t\t\tlog.Fatalf(\"can't get random salt: %v\", err)\n\t\t}\n\t} else {\n\t\tsalt = usersalt\n\t}\n\taead, err = chacha20poly1305.NewX(argon2.IDKey(passphrase, salt, 1, 64*1024, 4, 32))\n\treturn\n}\n\n// EncryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc EncryptChaCha(plaintext []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tnonce := make([]byte, aead.NonceSize(), aead.NonceSize()+len(plaintext)+aead.Overhead())\n\tif _, err := rand.Read(nonce); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// Encrypt the message and append the ciphertext to the nonce.\n\tencrypted = aead.Seal(nonce, nonce, plaintext, nil)\n\treturn\n}\n\n// DecryptChaCha will encrypt ChaCha20-Poly1305 using the pre-generated key\n// https://pkg.go.dev/golang.org/x/crypto/chacha20poly1305\nfunc DecryptChaCha(encryptedMsg []byte, aead cipher.AEAD) (encrypted []byte, err error) {\n\tif len(encryptedMsg) < aead.NonceSize() {\n\t\terr = fmt.Errorf(\"ciphertext too short\")\n\t\treturn\n\t}\n\n\t// Split nonce and ciphertext.\n\tnonce, ciphertext := encryptedMsg[:aead.NonceSize()], encryptedMsg[aead.NonceSize():]\n\n\t// Decrypt the message and check it wasn't tampered with.\n\tencrypted, err = aead.Open(nil, nonce, ciphertext, nil)\n\treturn\n}\n\n// Path: src/compress/compress.go\npackage compress\n\nimport (\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"io\"\n\n\tlog \"github.com/schollz/logger\"\n)\n\n// CompressWithOption returns compressed data using the specified level\nfunc CompressWithOption(src []byte, level int) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, level)\n\treturn compressedData.Bytes()\n}\n\n// Compress returns a compressed byte slice.\nfunc Compress(src []byte) []byte {\n\tcompressedData := new(bytes.Buffer)\n\tcompress(src, compressedData, -2)\n\treturn compressedData.Bytes()\n}\n\n// Decompress returns a decompressed byte slice.\nfunc Decompress(src []byte) []byte {\n\tcompressedData := bytes.NewBuffer(src)\n\tdeCompressedData := new(bytes.Buffer)\n\tdecompress(compressedData, deCompressedData)\n\treturn deCompressedData.Bytes()\n}\n\n// compress uses flate to compress a byte slice to a corresponding level\nfunc compress(src []byte, dest io.Writer, level int) {\n\tcompressor, err := flate.NewWriter(dest, level)\n\tif err != nil {\n\t\tlog.Debugf(\"error level data: %v\", err)\n\t\treturn\n\t}\n\tif _, err := compressor.Write(src); err != nil {\n\t\tlog.Debugf(\"error writing data: %v\", err)\n\t}\n\tcompressor.Close()\n}\n\n// compress uses flate to decompress an io.Reader\nfunc decompress(src io.Reader, dest io.Writer) {\n\tdecompressor := flate.NewReader(src)\n\tif _, err := io.Copy(dest, decompressor); err != nil {\n\t\tlog.Debugf(\"error copying data: %v\", err)\n\t}\n\tdecompressor.Close()\n}\n\n// Path: src/utils/utils.go\npackage utils\n\nimport (\n\t\"archive/zip\"\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"crypto/md5\"\n\t\"crypto/rand\"\n\t\"crypto/sha256\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"math\"\n\t\"math/big\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/cespare/xxhash\"\n\t\"github.com/kalafut/imohash\"\n\t\"github.com/schollz/mnemonicode\"\n)\n\nconst NbPinNumbers = 4\nconst NbBytesWords = 4\n\n// Get or create home directory\nfunc GetConfigDir(requireValidPath bool) (homedir string, err error) {\n\tif envHomedir, isSet := os.LookupEnv(\"CROC_CONFIG_DIR\"); isSet {\n\t\thomedir = envHomedir\n\t} else if xdgConfigHome, isSet := os.LookupEnv(\"XDG_CONFIG_HOME\"); isSet {\n\t\thomedir = path.Join(xdgConfigHome, \"croc\")\n\t} else {\n\t\thomedir, err = os.UserHomeDir()\n\t\tif err != nil {\n\t\t\tif !requireValidPath {\n\t\t\t\terr = nil\n\t\t\t\thomedir = \"\"\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\thomedir = path.Join(homedir, \".config\", \"croc\")\n\t}\n\n\tif requireValidPath {\n\t\tif _, err = os.Stat(homedir); os.IsNotExist(err) {\n\t\t\terr = os.MkdirAll(homedir, 0o700)\n\t\t}\n\t}\n\treturn\n}\n\n// Exists reports whether the named file or directory exists.\nfunc Exists(name string) bool {\n\tif _, err := os.Stat(name); err != nil {\n\t\tif os.IsNotExist(err) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// GetInput returns the input with a given prompt\nfunc GetInput(prompt string) string {\n\treader := bufio.NewReader(os.Stdin)\n\tfmt.Fprintf(os.Stderr, \"%s\", prompt)\n\ttext, _ := reader.ReadString('\\n')\n\treturn strings.TrimSpace(text)\n}\n\n// HashFile returns the hash of a file or, in case of a symlink, the\n// SHA256 hash of its target. Takes an argument to specify the algorithm to use.\nfunc HashFile(fname string, algorithm string) (hash256 []byte, err error) {\n\tvar fstats os.FileInfo\n\tfstats, err = os.Lstat(fname)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif fstats.Mode()&os.ModeSymlink != 0 {\n\t\tvar target string\n\t\ttarget, err = os.Readlink(fname)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn []byte(SHA256(target)), nil\n\t}\n\tswitch algorithm {\n\tcase \"imohash\":\n\t\treturn IMOHashFile(fname)\n\tcase \"md5\":\n\t\treturn MD5HashFile(fname)\n\tcase \"xxhash\":\n\t\treturn XXHashFile(fname)\n\t}\n\terr = fmt.Errorf(\"unspecified algorithm\")\n\treturn\n}\n\n// MD5HashFile returns MD5 hash\nfunc MD5HashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := md5.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// IMOHashFile returns imohash\nfunc IMOHashFile(fname string) (hash []byte, err error) {\n\tb, err := imohash.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\nvar imofull = imohash.NewCustom(0, 0)\n\n// IMOHashFileFull returns imohash of full file\nfunc IMOHashFileFull(fname string) (hash []byte, err error) {\n\tb, err := imofull.SumFile(fname)\n\thash = b[:]\n\treturn\n}\n\n// XXHashFile returns the xxhash of a file\nfunc XXHashFile(fname string) (hash256 []byte, err error) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\th := xxhash.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn\n\t}\n\n\thash256 = h.Sum(nil)\n\treturn\n}\n\n// SHA256 returns sha256 sum\nfunc SHA256(s string) string {\n\tsha := sha256.New()\n\tsha.Write([]byte(s))\n\treturn hex.EncodeToString(sha.Sum(nil))\n}\n\n// PublicIP returns public ip address\nfunc PublicIP() (ip string, err error) {\n\tresp, err := http.Get(\"https://canhazip.com\")\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode == http.StatusOK {\n\t\tbodyBytes, err := io.ReadAll(resp.Body)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tip = strings.TrimSpace(string(bodyBytes))\n\t}\n\treturn\n}\n\n// LocalIP returns local ip address\nfunc LocalIP() string {\n\tconn, err := net.Dial(\"udp\", \"8.8.8.8:80\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tlocalAddr := conn.LocalAddr().(*net.UDPAddr)\n\n\treturn localAddr.IP.String()\n}\n\nfunc GenerateRandomPin() string {\n\ts := \"\"\n\tmax := new(big.Int)\n\tmax.SetInt64(9)\n\tfor i := 0; i < NbPinNumbers; i++ {\n\t\tv, err := rand.Int(rand.Reader, max)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\ts += fmt.Sprintf(\"%d\", v)\n\t}\n\treturn s\n}\n\n// GetRandomName returns mnemonicoded random name\n\nfunc GetRandomName() string {\n\tvar result []string\n\tbs := make([]byte, NbBytesWords)\n\trand.Read(bs)\n\tresult = mnemonicode.EncodeWordList(result, bs)\n\treturn GenerateRandomPin() + \"-\" + strings.Join(result, \"-\")\n}\n\n// ByteCountDecimal converts bytes to human readable byte string\nfunc ByteCountDecimal(b int64) string {\n\tconst unit = 1024\n\tif b < unit {\n\t\treturn fmt.Sprintf(\"%d B\", b)\n\t}\n\tdiv, exp := int64(unit), 0\n\tfor n := b / unit; n >= unit; n /= unit {\n\t\tdiv *= unit\n\t\texp++\n\t}\n\treturn fmt.Sprintf(\"%.1f %cB\", float64(b)/float64(div), \"kMGTPE\"[exp])\n}\n\n// MissingChunks returns the positions of missing chunks.\n// If file doesn't exist, it returns an empty chunk list (all chunks).\n// If the file size is not the same as requested, it returns an empty chunk list (all chunks).\nfunc MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {\n\tf, err := os.Open(fname)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\tfstat, err := os.Stat(fname)\n\tif err != nil || fstat.Size() != fsize {\n\t\treturn\n\t}\n\n\temptyBuffer := make([]byte, chunkSize)\n\tchunkNum := 0\n\tchunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))\n\tvar currentLocation int64\n\tfor {\n\t\tbuffer := make([]byte, chunkSize)\n\t\tbytesread, err := f.Read(buffer)\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\tif bytes.Equal(buffer[:bytesread], emptyBuffer[:bytesread]) {\n\t\t\tchunks[chunkNum] = currentLocation\n\t\t\tchunkNum++\n\t\t}\n\t\tcurrentLocation += int64(bytesread)\n\t}\n\tif chunkNum == 0 {\n\t\tchunkRanges = []int64{}\n\t} else {\n\t\tchunks = chunks[:chunkNum]\n\t\tchunkRanges = []int64{int64(chunkSize), chunks[0]}\n\t\tcurCount := 0\n\t\tfor i, chunk := range chunks {\n\t\t\tif i == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcurCount++\n\t\t\tif chunk-chunks[i-1] > int64(chunkSize) {\n\t\t\t\tchunkRanges = append(chunkRanges, int64(curCount))\n\t\t\t\tchunkRanges = append(chunkRanges, chunk)\n\t\t\t\tcurCount = 0\n\t\t\t}\n\t\t}\n\t\tchunkRanges = append(chunkRanges, int64(curCount+1))\n\t}\n\treturn\n}\n\n// ChunkRangesToChunks converts chunk ranges to list\n...\n", "needle_token_start": 3749, "needle_token_end": 3826, "code_context_ntokens": 4547, "output": ["```go\n// GetRandomName returns mnemonicoded random name\nfunc GetRandomName() string {\n\tvar result []string\n\tbs := make([]byte, NbBytesWords)\n\trand.Read(bs)\n\tresult = mnemonicode.EncodeWordList(result, bs)\n\treturn GenerateRandomPin() + \"-\" + strings.Join(result, \"-\")\n}\n```"]}
{"repo": "iawia002/lux", "name": "TestHuya", "language": "go", "path": "extractors/huya/huya_test.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate the functionality of a video extraction tool for a specific website, ensuring it can correctly fetch and parse video data from given URLs.\n2. **Input**: The function takes predefined test cases, each containing a URL of a webpage and an expected title.\n3. **Output**: There is no direct output from this function as it primarily runs tests to check the extraction process. However, it implicitly verifies if the video data extracted matches expected criteria such as title and video URL.\n4. **Procedure**: The function iterates over a set of test cases, each specifying a webpage URL. For each test case, it invokes the video extraction tool to fetch and parse the video data from the URL. It checks if the extracted video information (like title and URL) is correct, ensuring the tool functions as expected.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/huya/huya.go\npackage huya\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"huya\", New())\n}\n\ntype extractor struct{}\n\nconst huyaVideoHost = \"https://videotx-platform.cdn.huya.com/\"\n\n// New returns a huya extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar title string\n\ttitleDesc := utils.MatchOneOf(html, `<h1>(.+?)</h1>`)\n\tif len(titleDesc) > 1 {\n\t\ttitle = titleDesc[1]\n\t} else {\n\t\ttitle = \"huya video\"\n\t}\n\n\tvar videoUrl string\n\tvideoDesc := utils.MatchOneOf(html, `//videotx-platform.cdn.huya.com/(.*)\" poster=(.+?)`)\n\tif len(videoDesc) > 1 {\n\t\tvideoUrl = huyaVideoHost + videoDesc[1]\n\t} else {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tsize, err := request.Size(videoUrl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoUrl,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n...\n// Path: extractors/huya/huya_test.go\npackage huya\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\n\nfunc TestHuya(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://m.v.huya.com/play/fans/630103747.html/?shareid=4597484513543964249&shareUid=2179142017&source=ios&sharetype=other&platform=2\",\n\t\t\t\tTitle: \"12.28 \u96c6\u68a6\u859b\u5c0f\u8c26\u3010\u5c01\u53f7\u6597\u7f57\u3011\u76f4\u64ad\u540d\u573a\u9762\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/instagram/instagram.go\npackage instagram\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nvar client *http.Client\n\nfunc init() {\n\textractors.Register(\"instagram\", New())\n\tclient = &http.Client{\n\t\tTimeout: 10 * time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tDial: (&net.Dialer{\n\t\t\t\tTimeout: 5 * time.Second,\n\t\t\t}).Dial,\n\t\t\tTLSHandshakeTimeout: 5 * time.Second,\n\t\t},\n\t}\n}\n\n// sliderItemNode contains information about the Instagram post\ntype sliderItemNode struct {\n\tDisplayURL string `json:\"display_url\"` // URL of the Media (resolution is dynamic)\n\n\tIsVideo  bool   `json:\"is_video\"`  // Is type of the Media equals to video\n\tVideoURL string `json:\"video_url\"` // Direct URL to the Video\n}\n\nfunc (s sliderItemNode) extractMediaURL() string {\n\tif s.IsVideo {\n\t\treturn s.VideoURL\n\t}\n\n\treturn s.DisplayURL\n}\n\ntype instagramPayload struct {\n\tMedia struct {\n\t\tID          string `json:\"id\"` // Unique ID of the Media\n\t\tSliderItems struct {\n\t\t\tEdges []struct {\n\t\t\t\tNode sliderItemNode `json:\"node\"`\n\t\t\t} `json:\"edges\"`\n\t\t} `json:\"edge_sidecar_to_children\"` // Children of the Media\n\t} `json:\"shortcode_media\"` // Media\n}\n\nfunc (s instagramPayload) isEmpty() bool {\n\treturn s.Media.ID == \"\"\n}\n\nfunc getPostWithCode(code string) ([]string, error) {\n\tURL := fmt.Sprintf(\"https://www.instagram.com/p/%v/embed/captioned/\", code)\n\n\tvar embeddedMediaImage string\n\tvar embedResponse = instagramPayload{}\n\tcollector := colly.NewCollector()\n\tcollector.SetClient(client)\n\tvar collectorErr error\n\n\tcollector.OnHTML(\"img.EmbeddedMediaImage\", func(e *colly.HTMLElement) {\n\t\tembeddedMediaImage = e.Attr(\"src\")\n\t})\n\n\tcollector.OnHTML(\"script\", func(e *colly.HTMLElement) {\n\t\tr := regexp.MustCompile(`\\\\\\\"gql_data\\\\\\\":([\\s\\S]*)\\}\\\"\\}\\]\\]\\,\\[\\\"NavigationMetrics`)\n\t\tmatch := r.FindStringSubmatch(e.Text)\n\n\t\tif len(match) < 2 {\n\t\t\treturn\n\t\t}\n\n\t\ts := strings.ReplaceAll(match[1], `\\\"`, `\"`)\n\t\ts = strings.ReplaceAll(s, `\\\\/`, `/`)\n\t\ts = strings.ReplaceAll(s, `\\\\`, `\\`)\n\n\t\terr := json.Unmarshal([]byte(s), &embedResponse)\n\t\tif err != nil {\n\t\t\tcollectorErr = err\n\t\t}\n\t})\n\n\tcollector.OnRequest(func(r *colly.Request) {\n\t\tr.Headers.Set(\"User-Agent\", browser.Chrome())\n\t})\n\n\tif err := collector.Visit(URL); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to send HTTP request to the Instagram: %v\", err)\n\t}\n\n\tif collectorErr != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse the Instagram response: %v\", collectorErr)\n\t}\n\n\t// If the method one which is JSON parsing didn't fail\n\tif !embedResponse.isEmpty() {\n\t\tresult := make([]string, 0, len(embedResponse.Media.SliderItems.Edges))\n\t\tfor _, item := range embedResponse.Media.SliderItems.Edges {\n\t\t\tresult = append(result, item.Node.extractMediaURL())\n\t\t}\n\n\t\treturn result, nil\n\t}\n\n\tif embeddedMediaImage != \"\" {\n\t\treturn []string{embeddedMediaImage}, nil\n\t}\n\n\t// If every two methods have failed, then return an error\n\treturn nil, errors.New(\"failed to fetch the post, the page might be \\\"private\\\", or the link is completely wrong\")\n}\n\nfunc extractShortCodeFromLink(link string) (string, error) {\n\tvalues := regexp.MustCompile(`(p|tv|reel|reels\\/videos)\\/([A-Za-z0-9-_]+)`).FindStringSubmatch(link)\n\tif len(values) != 3 {\n\t\treturn \"\", errors.New(\"couldn't extract the media short code from the link\")\n\t}\n\n\treturn values[2], nil\n}\n\ntype extractor struct{}\n\n// New returns a instagram extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tu, err := netURL.Parse(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tshortCode, err := extractShortCodeFromLink(u.String())\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turls, err := getPostWithCode(shortCode)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar totalSize int64\n\tvar parts []*extractors.Part\n\n\tfor _, u := range urls {\n\t\t_, ext, err := utils.GetNameAndExt(u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfileSize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tpart := &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: fileSize,\n\t\t\tExt:  ext,\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tfor _, part := range parts {\n\t\ttotalSize += part.Size\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Instagram instagram.com\",\n\t\t\tTitle:   \"Instagram \" + shortCode,\n\t\t\tType:    extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/instagram/instagram_test.go\npackage instagram\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.instagram.com/p/BlIka1ZFCNr\",\n\t\t\t\tTitle: \"Instagram BlIka1ZFCNr\",\n\t\t\t\tSize:  992330,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.instagram.com/p/Bl5oVUyl9Yx\",\n\t\t\t\tTitle: \"Instagram Bl5oVUyl9Yx\",\n\t\t\t\tSize:  250596,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image album test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.instagram.com/p/Bjyr-gxF4Rb\",\n\t\t\t\tTitle: \"Instagram Bjyr-gxF4Rb\",\n\t\t\t\tSize:  656476,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/iqiyi/iqiyi.go\npackage iqiyi\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"iqiyi\", New(SiteTypeIqiyi))\n\textractors.Register(\"iq\", New(SiteTypeIQ))\n}\n\ntype iqiyi struct {\n\tCode string `json:\"code\"`\n\tData struct {\n\t\tVP struct {\n\t\t\tDu  string `json:\"du\"`\n\t\t\tTkl []struct {\n\t\t\t\tVs []struct {\n\t\t\t\t\tBid   int    `json:\"bid\"`\n\t\t\t\t\tScrsz string `json:\"scrsz\"`\n\t\t\t\t\tVsize int64  `json:\"vsize\"`\n\t\t\t\t\tFs    []struct {\n\t\t\t\t\t\tL string `json:\"l\"`\n\t\t\t\t\t\tB int64  `json:\"b\"`\n\t\t\t\t\t} `json:\"fs\"`\n\t\t\t\t} `json:\"vs\"`\n\t\t\t} `json:\"tkl\"`\n\t\t} `json:\"vp\"`\n\t} `json:\"data\"`\n\tMsg string `json:\"msg\"`\n}\n\ntype iqiyiURL struct {\n\tL string `json:\"l\"`\n}\n\n// SiteType indicates the site type of iqiyi\ntype SiteType int\n\nconst (\n\t// SiteTypeIQ indicates the site is iq.com\n\tSiteTypeIQ SiteType = iota\n\t// SiteTypeIqiyi indicates the site is iqiyi.com\n\tSiteTypeIqiyi\n\tiqReferer    = \"https://www.iq.com\"\n\tiqiyiReferer = \"https://www.iqiyi.com\"\n)\n\nfunc getMacID() string {\n\tvar macID string\n\tchars := []string{\n\t\t\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"n\", \"m\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\",\n\t\t\"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\",\n\t}\n\tsize := len(chars)\n\tfor i := 0; i < 32; i++ {\n\t\tmacID += chars[rand.Intn(size)]\n\t}\n\treturn macID\n}\n\nfunc getVF(params string) string {\n\tvar suffix string\n\tfor j := 0; j < 8; j++ {\n\t\tfor k := 0; k < 4; k++ {\n\t\t\tvar v8 int\n\t\t\tv4 := 13 * (66*k + 27*j) % 35\n\t\t\tif v4 >= 10 {\n\t\t\t\tv8 = v4 + 88\n\t\t\t} else {\n\t\t\t\tv8 = v4 + 49\n\t\t\t}\n\t\t\tsuffix += string(rune(v8)) // string(97) -> \"a\"\n\t\t}\n\t}\n\tparams += suffix\n\n\treturn utils.Md5(params)\n}\n\nfunc getVPS(tvid, vid, refer string) (*iqiyi, error) {\n\tt := time.Now().Unix() * 1000\n\thost := \"http://cache.video.qiyi.com\"\n\tparams := fmt.Sprintf(\n\t\t\"/vps?tvid=%s&vid=%s&v=0&qypid=%s_12&src=01012001010000000000&t=%d&k_tag=1&k_uid=%s&rs=1\",\n\t\ttvid, vid, tvid, t, getMacID(),\n\t)\n\tvf := getVF(params)\n\tapiURL := fmt.Sprintf(\"%s%s&vf=%s\", host, params, vf)\n\tinfo, err := request.Get(apiURL, refer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdata := new(iqiyi)\n\tif err := json.Unmarshal([]byte(info), data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn data, nil\n}\n\ntype extractor struct {\n\tsiteType SiteType\n}\n\n// New returns a iqiyi extractor.\nfunc New(siteType SiteType) extractors.Extractor {\n\treturn &extractor{\n\t\tsiteType: siteType,\n\t}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\trefer := iqiyiReferer\n\theaders := make(map[string]string)\n\tif e.siteType == SiteTypeIQ {\n\t\theaders = map[string]string{\n\t\t\t\"Accept-Language\": \"zh-TW\",\n\t\t}\n\t\trefer = iqReferer\n\t}\n\thtml, err := request.Get(url, refer, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttvid := utils.MatchOneOf(\n\t\turl,\n\t\t`#curid=(.+)_`,\n\t\t`tvid=([^&]+)`,\n\t)\n\tif tvid == nil {\n\t\ttvid = utils.MatchOneOf(\n\t\t\thtml,\n\t\t\t`data-player-tvid=\"([^\"]+)\"`,\n\t\t\t`param\\['tvid'\\]\\s*=\\s*\"(.+?)\"`,\n\t\t\t`\"tvid\":\"(\\d+)\"`,\n\t\t\t`\"tvId\":(\\d+)`,\n\t\t)\n\t}\n\tif tvid == nil || len(tvid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvid := utils.MatchOneOf(\n\t\turl,\n\t\t`#curid=.+_(.*)$`,\n\t\t`vid=([^&]+)`,\n\t)\n\tif vid == nil {\n\t\tvid = utils.MatchOneOf(\n\t\t\thtml,\n\t\t\t`data-player-videoid=\"([^\"]+)\"`,\n\t\t\t`param\\['vid'\\]\\s*=\\s*\"(.+?)\"`,\n\t\t\t`\"vid\":\"(\\w+)\"`,\n\t\t)\n\t}\n\tif vid == nil || len(vid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tif e.siteType == SiteTypeIqiyi {\n\t\ttitle = strings.TrimSpace(doc.Find(\"h1>a\").First().Text())\n\t\tvar sub string\n\t\tfor _, k := range []string{\"span\", \"em\"} {\n\t\t\tif sub != \"\" {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tsub = strings.TrimSpace(doc.Find(\"h1>\" + k).First().Text())\n\t\t}\n\t\ttitle += sub\n\t} else {\n\t\ttitle = strings.TrimSpace(doc.Find(\"span#pageMetaTitle\").First().Text())\n\t\tsub := utils.MatchOneOf(html, `\"subTitle\":\"([^\"]+)\",\"isoDuration\":`)\n\t\tif len(sub) > 1 {\n\t\t\ttitle += fmt.Sprintf(\" %s\", sub[1])\n\t\t}\n\t}\n\tif title == \"\" {\n\t\ttitle = doc.Find(\"title\").Text()\n\t}\n\tvideoDatas, err := getVPS(tvid[1], vid[1], refer)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif videoDatas.Code != \"A00000\" {\n\t\treturn nil, errors.Errorf(\"can't play this video: %s\", videoDatas.Msg)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\turlPrefix := videoDatas.Data.VP.Du\n\tfor _, video := range videoDatas.Data.VP.Tkl[0].Vs {\n\t\turls := make([]*extractors.Part, len(video.Fs))\n\t\tfor index, v := range video.Fs {\n\t\t\trealURLData, err := request.Get(urlPrefix+v.L, refer, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tvar realURL iqiyiURL\n\t\t\tif err = json.Unmarshal([]byte(realURLData), &realURL); err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\t_, ext, err := utils.GetNameAndExt(realURL.L)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL:  realURL.L,\n\t\t\t\tSize: v.B,\n\t\t\t\tExt:  ext,\n\t\t\t}\n\t\t}\n\t\tstreams[strconv.Itoa(video.Bid)] = &extractors.Stream{\n\t\t\tParts:   urls,\n\t\t\tSize:    video.Vsize,\n\t\t\tQuality: video.Scrsz,\n\t\t}\n\t}\n\n\tsiteName := \"\u7231\u5947\u827a iqiyi.com\"\n\tif e.siteType == SiteTypeIQ {\n\t\tsiteName = \"\u7231\u5947\u827a iq.com\"\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    siteName,\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/iqiyi/iqiyi_test.go\npackage iqiyi\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"http://www.iqiyi.com/v_19rrbdmaj0.html\",\n\t\t\t\tTitle:   \"\u65b0\u4e00\u8f6e\u964d\u6c34\u5c06\u81f3 \u51b7\u7a7a\u6c14\u5f71\u54cd\u4e2d\u4e1c\u90e8\u5730\u533a\",\n\t\t\t\tSize:    2952228,\n\t\t\t\tQuality: \"896x504\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"title test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"http://www.iqiyi.com/v_19rqy2z83w.html\",\n\t\t\t\tTitle:   \"\u6536\u4e86\u521b\u610f\u89c6\u98912018 :58\u5929\u73af\u7403\u98de\u884c\u8bb0\",\n\t\t\t\tSize:    76186786,\n\t\t\t\tQuality: \"1920x1080\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"curid test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.iqiyi.com/v_19rro0jdls.html#curid=350289100_6e6601aae889d0b1004586a52027c321\",\n\t\t\t\tTitle:   \"Shawn Mendes - Never Be Alone\",\n\t\t\t\tSize:    79921894,\n\t\t\t\tQuality: \"1920x800\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New(SiteTypeIqiyi).Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/ixigua/ixigua.go\npackage ixigua\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"ixigua\", New())\n\textractors.Register(\"toutiao\", New())\n}\n\ntype extractor struct{}\n\ntype Video struct {\n\tTitle     string `json:\"title\"`\n\tQualities []struct {\n\t\tQuality string `json:\"quality\"`\n\t\tSize    int64  `json:\"size\"`\n\t\tURL     string `json:\"url\"`\n\t\tExt     string `json:\"ext\"`\n\t} `json:\"qualities\"`\n}\n\n// New returns a ixigua extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\theaders := map[string]string{\n\t\t\"User-Agent\": browser.Chrome(),\n\t\t\"Cookie\":     option.Cookie,\n\t}\n\n\t// ixigua \u6709\u4e09\u79cd\u683c\u5f0f\u7684 URL\n\t// \u683c\u5f0f\u4e00 https://www.ixigua.com/7053389963487871502\n\t// \u683c\u5f0f\u4e8c https://v.ixigua.com/RedcbWM/\n\t// \u683c\u5f0f\u4e09 https://m.toutiao.com/is/dtj1pND/\n\t// \u683c\u5f0f\u4e8c\u4f1a\u8df3\u8f6c\u5230\u683c\u5f0f\u4e00\n\t// \u683c\u5f0f\u4e09\u4f1a\u8df3\u8f6c\u5230 https://www.toutiao.com/a7053389963487871502\n\n\tvar finalURL string\n\tif strings.HasPrefix(url, \"https://www.ixigua.com/\") {\n\t\tfinalURL = url\n\t}\n\n\tif strings.HasPrefix(url, \"https://v.ixigua.com/\") || strings.HasPrefix(url, \"https://m.toutiao.com/\") {\n\t\tresp, err := http.Get(url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tdefer resp.Body.Close() // nolint\n\t\t// follow redirects, https://stackoverflow.com/a/16785343\n\t\tfinalURL = resp.Request.URL.String()\n\t}\n\n\tfinalURL = strings.ReplaceAll(finalURL, \"https://www.toutiao.com/video/\", \"https://www.ixigua.com/\")\n\n\tr := regexp.MustCompile(`(ixigua.com/)(\\w+)?`)\n\tid := r.FindSubmatch([]byte(finalURL))[2]\n\turl2 := fmt.Sprintf(\"https://www.ixigua.com/%s\", string(id))\n\n\tbody, err := request.Get(url2, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvideoListJson := utils.MatchOneOf(body, `window._SSR_HYDRATED_DATA=(\\{.*?\\})\\<\\/script\\>`)\n\tif videoListJson == nil || len(videoListJson) != 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrBodyParseFailed)\n\t}\n\n\tvideoUrl := videoListJson[1]\n\tvideoUrl = strings.Replace(videoUrl, \":undefined\", \":\\\"undefined\\\"\", -1)\n\n\tvar data xiguanData\n\tif err = json.Unmarshal([]byte(videoUrl), &data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := data.AnyVideo.GidInformation.PackerData.Video.Title\n\tvideoList := data.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList\n\n\tstreams := make(map[string]*extractors.Stream)\n\tfor _, v := range videoList {\n\t\tstreams[v.Definition] = &extractors.Stream{\n\t\t\tQuality: v.Definition,\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  base64Decode(v.MainUrl),\n\t\t\t\t\tSize: v.Size,\n\t\t\t\t\tExt:  v.Vtype,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u897f\u74dc\u89c6\u9891 ixigua.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\nfunc base64Decode(t string) string {\n\td, _ := base64.StdEncoding.DecodeString(t)\n\treturn string(d)\n}\n\n// Path: extractors/ixigua/ixigua_test.go\npackage ixigua\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.ixigua.com/7053389963487871502\",\n\t\t\t\tTitle:   \"\u6f2b\u5a01\u65a5\u5de8\u8d44\u62cd\u7684\u300a\u6c38\u6052\u65cf\u300b\uff0c\u521a\u4e0a\u67b6\u5c31\u88ab\u591a\u56fd\u7981\u64ad\uff0c\u7a76\u7adf\u62cd\u4e86\u4ec0\u4e48\uff1f\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    313091514,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.ixigua.com/RedcbWM/\",\n\t\t\t\tTitle:   \"\u4e3a\u957f\u751f\u4e0d\u8001\uff0c\u7adf\u7136\u8fde\u5c0f\u9c9b\u4eba\u90fd\u6740@\u4e2d\u89c6\u9891\u4f19\u4f34\u8ba1\u5212\u5b98\u53f7\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    64980732,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"test 3\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://m.toutiao.com/is/dtj1pND/\",\n\t\t\t\tTitle:   \"\u5361\u5c14\uff1a59\u67404200\u6cd5\u5f3a\u5c0f\u6cd5\u5e08\uff0c\u70b9\u5854\u53ea\u9700\u4e00\u4e0b\uff0cW\u6280\u80fd\u77ac\u79d2\u5bf9\u65b9\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    468324298,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/ixigua/types.go\npackage ixigua\n\ntype xiguanData struct {\n\tAnyVideo struct {\n\t\tGidInformation struct {\n\t\t\tGid        string `json:\"gid\"`\n\t\t\tPackerData struct {\n\t\t\t\tVideo struct {\n\t\t\t\t\tTitle         string `json:\"title\"`\n\t\t\t\t\tPosterUrl     string `json:\"poster_url\"`\n\t\t\t\t\tVideoResource struct {\n\t\t\t\t\t\tVid    string `json:\"vid\"`\n\t\t\t\t\t\tNormal struct {\n\t\t\t\t\t\t\tVideoId   string `json:\"video_id\"`\n\t\t\t\t\t\t\tVideoList map[string]struct {\n\t\t\t\t\t\t\t\tDefinition  string `json:\"definition\"`\n\t\t\t\t\t\t\t\tQuality     string `json:\"quality\"`\n\t\t\t\t\t\t\t\tVtype       string `json:\"vtype\"`\n\t\t\t\t\t\t\t\tVwidth      int    `json:\"vwidth\"`\n\t\t\t\t\t\t\t\tVheight     int    `json:\"vheight\"`\n\t\t\t\t\t\t\t\tBitrate     int64  `json:\"bitrate\"`\n\t\t\t\t\t\t\t\tRealBitrate int64  `json:\"real_bitrate\"`\n\t\t\t\t\t\t\t\tFps         int    `json:\"fps\"`\n\t\t\t\t\t\t\t\tCodecType   string `json:\"codec_type\"`\n\t\t\t\t\t\t\t\tSize        int64  `json:\"size\"`\n\t\t\t\t\t\t\t\tMainUrl     string `json:\"main_url\"`\n\t\t\t\t\t\t\t\tBackupUrl1  string `json:\"backup_url_1\"`\n\t\t\t\t\t\t\t} `json:\"video_list\"`\n\t\t\t\t\t\t} `json:\"normal\"`\n\t\t\t\t\t} `json:\"videoResource\"`\n\t\t\t\t} `json:\"video\"`\n\t\t\t\tKey string `json:\"key\"`\n\t\t\t} `json:\"packerData\"`\n\t\t} `json:\"gidInformation\"`\n\t} `json:\"anyVideo\"`\n}\n\n// Path: extractors/kuaishou/kuaishou.go\npackage kuaishou\n\nimport (\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"kuaishou\", New())\n}\n\ntype extractor struct{}\n\n// New returns a kuaishou extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// fetch url and get the cookie that write by server\nfunc fetchCookies(url string, headers map[string]string) (string, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, headers)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tdefer res.Body.Close() // nolint\n\n\tcookiesArr := make([]string, 0)\n\tcookies := res.Cookies()\n\n\tfor _, c := range cookies {\n\t\tcookiesArr = append(cookiesArr, c.Name+\"=\"+c.Value)\n\t}\n\n\treturn strings.Join(cookiesArr, \"; \"), nil\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\theaders := map[string]string{\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t}\n\n\tcookies, err := fetchCookies(url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\theaders[\"Cookie\"] = cookies\n\n\thtml, err := request.Get(url, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitles := utils.MatchOneOf(html, `<title>([^<]+)</title>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.New(\"can not found title\")\n\t}\n\n\ttitle := regexp.MustCompile(`\\n+`).ReplaceAllString(strings.TrimSpace(titles[1]), \" \")\n\n\tqualityRegMap := map[string]*regexp.Regexp{\n\t\t\"sd\": regexp.MustCompile(`\"photoUrl\":\\s*\"([^\"]+)\"`),\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 1)\n\tfor quality, qualityReg := range qualityRegMap {\n\t\tmatcher := qualityReg.FindStringSubmatch(html)\n\t\tif len(matcher) != 2 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\n\t\tu := strings.ReplaceAll(matcher[1], `\\u002F`, \"/\")\n\n\t\tsize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u5feb\u624b kuaishou.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/kuaishou/kuaishou_test.go\npackage kuaishou\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.kuaishou.com/short-video/3x43cyvcyph57i4?authorId=3xtq3uqyjmhbimq&streamSource=find&area=homexxbrilliant\",\n\t\t\t\tTitle:   \"\u73b0\u5728\u8fde\u6234\u53e3\u7f69\u90fd\u5f00\u59cb\u5185\u5377\u4e86\u5417\uff1f\uff01\u5febget\u53e3\u7f69\u5fc3\u673a\u6234\u6cd5\uff0c\u76f4\u63a5\u6234\u51fa\u5c0fV\u8138\u554a \uff01 #\u53e3\u7f69 #\u663e\u8138\u5c0f-\u5feb\u624b\",\n\t\t\t\tSize:    1077774,\n\t\t\t\tQuality: \"sd\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/mgtv/mgtv.go\npackage mgtv\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"mgtv\", New())\n}\n\ntype mgtvVideoStream struct {\n\tName string `json:\"name\"`\n\tURL  string `json:\"url\"`\n\tDef  string `json:\"def\"`\n}\n\ntype mgtvVideoInfo struct {\n\tTitle string `json:\"title\"`\n\tDesc  string `json:\"desc\"`\n}\n\ntype mgtvVideoData struct {\n\tStream       []mgtvVideoStream `json:\"stream\"`\n\tStreamDomain []string          `json:\"stream_domain\"`\n\tInfo         mgtvVideoInfo     `json:\"info\"`\n}\n\ntype mgtv struct {\n\tData mgtvVideoData `json:\"data\"`\n}\n\ntype mgtvVideoAddr struct {\n\tInfo string `json:\"info\"`\n}\n\ntype mgtvURLInfo struct {\n\tURL  string\n\tSize int64\n}\n\ntype mgtvPm2Data struct {\n\tData struct {\n\t\tAtc struct {\n\t\t\tPm2 string `json:\"pm2\"`\n\t\t} `json:\"atc\"`\n\t\tInfo mgtvVideoInfo `json:\"info\"`\n\t} `json:\"data\"`\n}\n\nfunc mgtvM3u8(url string) ([]mgtvURLInfo, int64, error) {\n\tvar data []mgtvURLInfo\n\tvar temp mgtvURLInfo\n\tvar size, totalSize int64\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tm3u8String, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tsizes := utils.MatchAll(m3u8String, `#EXT-MGTV-File-SIZE:(\\d+)`)\n\t// sizes: [[#EXT-MGTV-File-SIZE:1893724, 1893724]]\n\tfor index, u := range urls {\n\t\tsize, err = strconv.ParseInt(sizes[index][1], 10, 64)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\ttotalSize += size\n\t\ttemp = mgtvURLInfo{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, totalSize, nil\n}\n\nfunc encodeTk2(str string) string {\n\tencodeString := base64.StdEncoding.EncodeToString([]byte(str))\n\tr1 := regexp.MustCompile(`/\\+/g`)\n\tr2 := regexp.MustCompile(`///g`)\n\tr3 := regexp.MustCompile(`/=/g`)\n\tr1.ReplaceAllString(encodeString, \"_\")\n\tr2.ReplaceAllString(encodeString, \"~\")\n\tr3.ReplaceAllString(encodeString, \"-\")\n\tencodeString = utils.Reverse(encodeString)\n\treturn encodeString\n}\n\ntype extractor struct{}\n\n// New returns a mgtv extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvid := utils.MatchOneOf(\n\t\turl,\n\t\t`https?://www.mgtv.com/(?:b|l)/\\d+/(\\d+).html`,\n\t\t`https?://www.mgtv.com/hz/bdpz/\\d+/(\\d+).html`,\n\t)\n\tif vid == nil {\n\t\tvid = utils.MatchOneOf(html, `vid: (\\d+),`)\n\t}\n\tif vid == nil || len(vid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// API extract from https://js.mgtv.com/imgotv-miniv6/global/page/play-tv.js\n\t// getSource and getPlayInfo function\n\t// Chrome Network JS panel\n\theaders := map[string]string{\n\t\t\"Cookie\": \"PM_CHKID=1\",\n\t}\n\tclit := fmt.Sprintf(\"clit=%d\", time.Now().Unix()/1000)\n\tpm2DataString, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"https://pcweb.api.mgtv.com/player/video?video_id=%s&tk2=%s\",\n\t\t\tvid[1],\n\t\t\tencodeTk2(fmt.Sprintf(\n\t\t\t\t\"did=f11dee65-4e0d-4d25-bfce-719ad9dc991d|pno=1030|ver=5.5.1|%s\", clit,\n\t\t\t)),\n\t\t),\n\t\turl,\n\t\theaders,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar pm2 mgtvPm2Data\n\tif err = json.Unmarshal([]byte(pm2DataString), &pm2); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdataString, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"https://pcweb.api.mgtv.com/player/getSource?video_id=%s&tk2=%s&pm2=%s\",\n\t\t\tvid[1], encodeTk2(clit), pm2.Data.Atc.Pm2,\n\t\t),\n\t\turl,\n\t\theaders,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar mgtvData mgtv\n\tif err = json.Unmarshal([]byte(dataString), &mgtvData); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := strings.TrimSpace(\n\t\tpm2.Data.Info.Title + \" \" + pm2.Data.Info.Desc,\n\t)\n\tmgtvStreams := mgtvData.Data.Stream\n\tvar addr mgtvVideoAddr\n\tstreams := make(map[string]*extractors.Stream)\n\tfor _, stream := range mgtvStreams {\n\t\tif stream.URL == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\t// real download address\n\t\taddr = mgtvVideoAddr{}\n\t\taddrInfo, err := request.GetByte(mgtvData.Data.StreamDomain[0]+stream.URL, url, headers)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tif err = json.Unmarshal(addrInfo, &addr); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tm3u8URLs, totalSize, err := mgtvM3u8(addr.Info)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turls := make([]*extractors.Part, len(m3u8URLs))\n\t\tfor index, u := range m3u8URLs {\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL:  u.URL,\n\t\t\t\tSize: u.Size,\n\t\t\t\tExt:  \"ts\",\n\t\t\t}\n\t\t}\n\t\tstreams[stream.Def] = &extractors.Stream{\n\t\t\tParts:   urls,\n\t\t\tSize:    totalSize,\n\t\t\tQuality: stream.Name,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u8292\u679cTV mgtv.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/mgtv/mgtv_test.go\npackage mgtv\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.mgtv.com/b/322712/4317248.html\",\n\t\t\t\tTitle:   \"\u6211\u662f\u5927\u4fa6\u63a2 \u5148\u5bfc\u7247\uff1a\u4f55\u7085\u5434\u78ca\u9093\u4f26\u7a7f\u8d8a\u7834\u6848\",\n\t\t\t\tSize:    86169236,\n\t\t\t\tQuality: \"\u8d85\u6e05\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.mgtv.com/b/308703/4197072.html\",\n\t\t\t\tTitle:   \"\u8292\u679c\u635e\u661f\u95fb 2017 \u8bfa\u4e00\u4e3a\u7237\u7237\u548c\u59e5\u7237\u505a\u7ffb\u8bd1\u8d85\u840c\",\n\t\t\t\tSize:    6486376,\n\t\t\t\tQuality: \"\u8d85\u6e05\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"vip test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.mgtv.com/b/322865/4352046.html\",\n\t\t\t\tTitle:   \"\u5411\u5f80\u7684\u751f\u6d3b \u7b2c\u4e8c\u5b63 \u5148\u5bfc\u7247\uff1a\u4f55\u7085\u9ec4\u78ca\u56de\u5f52\u8d28\u6734\u751f\u6d3b\",\n\t\t\t\tSize:    453246944,\n\t\t\t\tQuality: \"\u8d85\u6e05\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/miaopai/miaopai.go\npackage miaopai\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"miaopai\", New())\n}\n\ntype miaopaiData struct {\n\tData struct {\n\t\tDescription string `json:\"description\"`\n\t\tMetaData    []struct {\n\t\t\tURLs struct {\n\t\t\t\tM string `json:\"m\"`\n\t\t\t} `json:\"play_urls\"`\n\t\t} `json:\"meta_data\"`\n\t} `json:\"data\"`\n}\n\nfunc getRandomString(l int) string {\n\ts := make([]string, 0)\n\tchars := []string{\n\t\t\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"n\", \"m\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\",\n\t\t\"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\",\n\t}\n\tfor i := 0; i < l; i++ {\n\t\ts = append(s, chars[rand.Intn(len(chars)-1)])\n\t}\n\treturn strings.Join(s, \"\")\n}\n\ntype extractor struct{}\n\n// New returns a miaopai extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tids := utils.MatchOneOf(url, `/media/([^\\./]+)`, `/show(?:/channel)?/([^\\./]+)`)\n\tif ids == nil || len(ids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tid := ids[1]\n\n\trandomString := getRandomString(10)\n\n\tvar data miaopaiData\n\tjsonString, err := request.Get(\n\t\tfmt.Sprintf(\"https://n.miaopai.com/api/aj_media/info.json?smid=%s&appid=530&_cb=_jsonp%s\", id, randomString),\n\t\turl, nil,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tmatch := utils.MatchOneOf(jsonString, randomString+`\\((.*)\\);$`)\n\tif match == nil || len(match) < 2 {\n\t\treturn nil, errors.New(\"\u83b7\u53d6\u89c6\u9891\u4fe1\u606f\u5931\u8d25\u3002\")\n\t}\n\n\terr = json.Unmarshal([]byte(match[1]), &data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\trealURL := data.Data.MetaData[0].URLs.M\n\tsize, err := request.Size(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  realURL,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize:  size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u79d2\u62cd miaopai.com\",\n\t\t\tTitle:   data.Data.Description,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/miaopai/miaopai_test.go\npackage miaopai\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"http://n.miaopai.com/media/Dqg5Pmb~I6lChdvOb-~r1BpKzzDu~MPr\",\n\t\t\t\tTitle: \"\u5c0f\u5b66\u97386\u70b9\u534a\u8d77\u5e8a\u5b66\u4e60:\u60f3\u8d62\u5728\u8d77\u8dd1\u7ebf\",\n\t\t\t\tSize:  6743958,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/netease/netease.go\npackage netease\n\nimport (\n\tnetURL \"net/url\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"163\", New())\n}\n\ntype extractor struct{}\n\n// New returns a netease extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\turl = strings.Replace(url, \"/#/\", \"/\", 1)\n\tvid := utils.MatchOneOf(url, `/(mv|video)\\?id=(\\w+)`)\n\tif vid == nil {\n...\n", "needle_token_start": 628, "needle_token_end": 893, "code_context_ntokens": 15156, "output": ["```go\nfunc TestHuya(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://m.v.huya.com/play/fans/630103747.html/?shareid=4597484513543964249&shareUid=2179142017&source=ios&sharetype=other&platform=2\",\n\t\t\t\tTitle: \"12.28 \u96c6\u68a6\u859b\u5c0f\u8c26\u3010\u5c01\u53f7\u6597\u7f57\u3011\u76f4\u64ad\u540d\u573a\u9762\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n```"]}
{"repo": "iawia002/lux", "name": "tangdouDownload", "language": "go", "path": "extractors/tangdou/tangdou.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to extract video data from a specified URL of a video sharing website.\n2. **Input**: A single string representing the URL of the video page.\n3. **Output**: Returns a data structure containing details about the video such as the site name, video title, type of data, available video streams, and the original URL. If an error occurs during the process, it returns an error-specific data structure.\n4. **Procedure**: \n   - Fetches the HTML content of the given URL using a HTTP GET request with specific headers.\n   - Extracts the video title from the HTML using regex patterns.\n   - Searches for video URLs in the HTML content using multiple regex patterns.\n   - Converts any escaped characters in the URL to their actual representation.\n   - Retrieves the size of the video file from the extracted URL.\n   - Constructs a stream object containing the video URL, size, and file extension.\n   - Compiles all extracted information into a structured format and returns it.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/rumble/rumble.go\npackage rumble\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"path\"\n\t\"regexp\"\n\t\"strconv\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"rumble\", New())\n}\n\ntype extractor struct{}\n\n// New returns a rumble extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\ntype rumbleData struct {\n...\n// Path: extractors/rumble/rumble_test.go\npackage rumble\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestRumble(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://rumble.com/v24swn0-just-say-yes-to-climate-lockdowns.html\",\n\t\t\t\tTitle: \"Just Say YES to Climate Lockdowns!\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/streamtape/streamtape.go\npackage streamtape\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/robertkrimen/otto\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"streamtape\", e)\n\textractors.Register(\"streamta\", e) // streamta.pe\n}\n\ntype extractor struct{}\n\n// New returns a StreamTape extractor\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tscripts := utils.MatchOneOf(html, `document.getElementById\\('norobotlink'\\).innerHTML = (.+?);`)\n\tif len(scripts) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvm := otto.New()\n\t_, err = vm.Run(fmt.Sprintf(\"var __VM__OUTPUT = %s\", scripts[1]))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvalue, err := vm.Get(\"__VM__OUTPUT\")\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu, err := value.ToString() // //streamtape.com/get_video?id=xx&expires=xx&ip=xx&token=xx\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu = fmt.Sprintf(\"https:%s&stream=1\", u)\n\n\t// get title\n\tvar title = \"StreamTape Video\"\n\ttitleMatch := utils.MatchOneOf(html,\n\t\t`\\<meta name=\"og:title\" content=\"(.*)\"\\>`)\n\tif len(titleMatch) >= 2 {\n\t\ttitle = titleMatch[1]\n\t}\n\n\tsize, err := request.Size(u, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlData := &extractors.Part{\n\t\tURL:  u,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize:  size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tURL:     u,\n\t\t\tSite:    \"StreamTape streamtape.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/streamtape/streamtape_test.go\npackage streamtape\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestStreamtape(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://streamtape.com/v/YKLDrr4X9gSvm9q/00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t\tTitle: \"00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/tangdou/tangdou.go\npackage tangdou\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"tangdou\", New())\n}\n\ntype extractor struct{}\n\n// New returns a tangdou extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nvar defaultHeader = map[string]string{\n\t\"Sec-Fetch-Dest\": \"document\",\n\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\"Sec-Fetch-Site\": \"cross-site\",\n\t\"Sec-GPC\":        \"1\",\n\t\"User-Agent\":     \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\treturn []*extractors.Data{tangdouDownload(url)}, nil\n}\n\n// tangdouDownload download function for single url\n\nfunc tangdouDownload(uri string) *extractors.Data {\n\thtml, err := request.Get(uri, uri, defaultHeader)\n\tif err != nil {\n\t\treturn extractors.EmptyData(uri, err)\n\t}\n\n\ttitles := utils.MatchOneOf(\n\t\thtml, `<div class=\"title\">(.+?)</div>`, `<meta name=\"description\" content=\"(.+?)\"`, `<title>(.+?)</title>`,\n\t)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed))\n\t}\n\ttitle := titles[1]\n\n\tvideoURLs := utils.MatchOneOf(\n\t\thtml, `video:'(.+?)'`, `video:\"(.+?)\"`, `<video[^>]*src=\"(.+?)\"`, `play_url:\\s*\"(.+?)\",`,\n\t)\n\n\tif len(videoURLs) < 2 {\n\t\treturn extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed))\n\t}\n\n\trealURL := strings.ReplaceAll(videoURLs[1], `\\u002F`, \"/\")\n\n\tsize, err := request.Size(realURL, uri)\n\tif err != nil {\n\t\treturn extractors.EmptyData(uri, err)\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  realURL,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt:  \"mp4\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: size,\n\t\t},\n\t}\n\n\treturn &extractors.Data{\n\t\tSite:    \"\u7cd6\u8c46\u5e7f\u573a\u821e tangdou.com\",\n\t\tTitle:   title,\n\t\tType:    extractors.DataTypeVideo,\n\t\tStreams: streams,\n\t\tURL:     uri,\n\t}\n}\n\n// Path: extractors/tangdou/tangdou_test.go\npackage tangdou\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestTangDou(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\targs     test.Args\n\t\tplaylist bool\n\t}{\n\t\t{\n\t\t\tname: \"need call share url first and get the signed video URL test and can get title from head's title tag\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://m.tangdou.com/play/1500676338077\",\n\t\t\t\tTitle: \"\u66b4\u7626\u51cf\u809a\u5b50\uff0c\u4e0d\u7528\u8dd1\u4e0d\u7528\u8df3\uff0c8\u5929\u7626\u4e8616\u65a4 \u6b63\u9762\u6f14\u793a \u80cc\u9762\u6f14\u793a \u5206\u89e3\u6559\u5b66__\u5e7f\u573a\u821e_\u7cd6\u8c46\u5e7f\u573a\u821e-\u7cd6\u8c46\u89c6\u9891\",\n\t\t\t\tSize:  62258444,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar (\n\t\t\t\tdata []*extractors.Data\n\t\t\t\terr  error\n\t\t\t)\n\t\t\tif tt.playlist {\n\t\t\t\t// playlist mode\n\t\t\t\t_, err = New().Extract(tt.args.URL, extractors.Options{\n\t\t\t\t\tPlaylist:     true,\n\t\t\t\t\tThreadNumber: 9,\n\t\t\t\t})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t} else {\n\t\t\t\tdata, err = New().Extract(tt.args.URL, extractors.Options{})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t\ttest.Check(t, tt.args, data[0])\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: extractors/tiktok/tiktok.go\npackage tiktok\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n)\n\nfunc init() {\n\textractors.Register(\"tiktok\", New())\n}\n\ntype extractor struct{}\n\n// New returns a tiktok extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, map[string]string{\n\t\t// tiktok require a user agent\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlMatcherRegExp := regexp.MustCompile(`\"downloadAddr\":\\s*\"([^\"]+)\"`)\n\n\tdownloadURLMatcher := urlMatcherRegExp.FindStringSubmatch(html)\n\n\tif len(downloadURLMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvideoURL := strings.ReplaceAll(downloadURLMatcher[1], `\\u002F`, \"/\")\n\n\ttitleMatcherRegExp := regexp.MustCompile(`<title[^>]*>([^<]+)</title>`)\n\n\ttitleMatcher := titleMatcherRegExp.FindStringSubmatch(html)\n\n\tif len(titleMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\ttitle := titleMatcher[1]\n\n\ttitleArr := strings.Split(title, \"|\")\n\n\tif len(titleArr) == 1 {\n\t\ttitle = titleArr[0]\n\t} else {\n\t\ttitle = strings.TrimSpace(strings.Join(titleArr[:len(titleArr)-1], \"|\"))\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\n\tsize, err := request.Size(videoURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoURL,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize:  size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"TikTok tiktok.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/tiktok/tiktok_test.go\npackage tiktok\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.tiktok.com/@ginjiro_koyama/video/7164293510617763073?is_copy_url=1&is_from_webapp=v1\",\n\t\t\t\tTitle: \"\u30a4\u30b1\u3059\u304e\u305fXOXO#xoxo #repezenfoxx #\u80cc\u4e2d\u7537 #kfam #yoshiki\u3055\u3093\u3092\u6ce3\u304b\u305b\u305f\u30c1\u30fc\u30e0 @K fam @\u3010Repezen Foxx\u3011\ud83e\udd8a\",\n\t\t\t\tSize:  4356253,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.tiktok.com/@enhypen/video/7165445991238356225?is_copy_url=1&is_from_webapp=v1\",\n\t\t\t\tTitle: \"\uae5c\uc9dd \ud1f4\uc7a5 \ud83d\udc4b #ENHYPEN #SUNGHOON #NI_KI #Make_the_change\",\n\t\t\t\tSize:  3848307,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/tumblr/tumblr.go\npackage tumblr\n\nimport (\n\t\"encoding/json\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"tumblr\", New())\n}\n\ntype imageList struct {\n\tList []string `json:\"@list\"`\n}\n\ntype tumblrImageList struct {\n\tImage imageList `json:\"image\"`\n}\n\ntype tumblrImage struct {\n\tImage string `json:\"image\"`\n}\n\nfunc genURLData(url, referer string) (*extractors.Part, int64, error) {\n\tsize, err := request.Size(url, referer)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\t_, ext, err := utils.GetNameAndExt(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\treturn &extractors.Part{\n\t\tURL:  url,\n\t\tSize: size,\n\t\tExt:  ext,\n\t}, size, nil\n}\n\nfunc tumblrImageDownload(url, html, title string) ([]*extractors.Data, error) {\n\tjsonStrings := utils.MatchOneOf(\n\t\thtml, `<script type=\"application/ld\\+json\">\\s*(.+?)</script>`,\n\t)\n\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := jsonStrings[1]\n\n\tvar totalSize int64\n\turls := make([]*extractors.Part, 0, 1)\n\tif strings.Contains(jsonString, `\"image\":{\"@list\"`) {\n\t\t// there are two data structures in the same field(image)\n\t\tvar imageList tumblrImageList\n\t\tif err := json.Unmarshal([]byte(jsonString), &imageList); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfor _, u := range imageList.Image.List {\n\t\t\turlData, size, err := genURLData(u, url)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\ttotalSize += size\n\t\t\turls = append(urls, urlData)\n\t\t}\n\t} else {\n\t\tvar image tumblrImage\n\t\tif err := json.Unmarshal([]byte(jsonString), &image); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData, size, err := genURLData(image.Image, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\ttotalSize = size\n\t\turls = append(urls, urlData)\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urls,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Tumblr tumblr.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\nfunc tumblrVideoDownload(url, html, title string) ([]*extractors.Data, error) {\n\tvideoURLs := utils.MatchOneOf(html, `<iframe src='(.+?)'`)\n\tif videoURLs == nil || len(videoURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvideoURL := videoURLs[1]\n\n\tif !strings.Contains(videoURL, \"tumblr.com/video\") {\n\t\treturn nil, errors.New(\"lux doesn't support this URL right now\")\n\t}\n\tvideoHTML, err := request.Get(videoURL, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\trealURLs := utils.MatchOneOf(videoHTML, `source src=\"(.+?)\"`)\n\tif realURLs == nil || len(realURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\trealURL := realURLs[1]\n\n\turlData, size, err := genURLData(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize:  size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Tumblr tumblr.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\ntype extractor struct{}\n\n// New returns a tumblr extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\t// get the title\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitle := parser.Title(doc)\n\tif strings.Contains(html, \"<iframe src=\") {\n\t\t// Data\n\t\treturn tumblrVideoDownload(url, html, title)\n\t}\n\t// Image\n\treturn tumblrImageDownload(url, html, title)\n}\n\n// Path: extractors/tumblr/tumblr_test.go\npackage tumblr\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"image test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"http://fuckyeah-fx.tumblr.com/post/170392654141/180202-%E5%AE%8B%E8%8C%9C\",\n\t\t\t\tTitle: \"f(x)\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"http://therealautoblog.tumblr.com/post/171623222197/paganis-new-projects-huayra-successor-with\",\n\t\t\t\tTitle: \"Autoblog \u2022 Pagani\u2019s new projects: Huayra successor with...\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://boomgoestheprower.tumblr.com/post/174127507696\",\n\t\t\t\tTitle: \"Out of Context Sonic Boom\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/twitter/twitter.go\npackage twitter\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"twitter\", New())\n}\n\ntype twitter struct {\n\tTrack struct {\n\t\tURL string `json:\"playbackUrl\"`\n\t} `json:\"track\"`\n\tTweetID  string\n\tUsername string\n}\n\ntype extractor struct{}\n\n// New returns a twitter extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tusernames := utils.MatchOneOf(html, `property=\"og:title\"\\s+content=\"(.+)\"`)\n\tif usernames == nil || len(usernames) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tusername := usernames[1]\n\n\ttweetIDs := utils.MatchOneOf(url, `(status|statuses)/(\\d+)`)\n\tif tweetIDs == nil || len(tweetIDs) < 3 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttweetID := tweetIDs[2]\n\n\tapi := fmt.Sprintf(\n\t\t\"https://api.twitter.com/1.1/videos/tweet/config/%s.json\", tweetID,\n\t)\n\theaders := map[string]string{\n\t\t\"Authorization\": \"Bearer AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE\",\n\t}\n\tjsonString, err := request.Get(api, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar twitterData twitter\n\tif err := json.Unmarshal([]byte(jsonString), &twitterData); err != nil {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttwitterData.TweetID = tweetID\n\ttwitterData.Username = username\n\textractedData, err := download(twitterData, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn extractedData, nil\n}\n\nfunc download(data twitter, uri string) ([]*extractors.Data, error) {\n\tvar (\n\t\terr  error\n\t\tsize int64\n\t)\n\tstreams := make(map[string]*extractors.Stream)\n\tswitch {\n\t// if video file is m3u8 and ts\n\tcase strings.Contains(data.Track.URL, \".m3u8\"):\n\t\tm3u8urls, err := utils.M3u8URLs(data.Track.URL)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfor index, m3u8 := range m3u8urls {\n\t\t\tvar totalSize int64\n\t\t\tts, err := utils.M3u8URLs(m3u8)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turls := make([]*extractors.Part, 0, len(ts))\n\t\t\tfor _, i := range ts {\n\t\t\t\tsize, err := request.Size(i, uri)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t\t}\n\t\t\t\ttemp := &extractors.Part{\n\t\t\t\t\tURL:  i,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt:  \"ts\",\n\t\t\t\t}\n\t\t\t\ttotalSize += size\n\t\t\t\turls = append(urls, temp)\n\t\t\t}\n\t\t\tqualityString := utils.MatchOneOf(m3u8, `/(\\d+x\\d+)/`)[1]\n\t\t\tquality := strconv.Itoa(index + 1)\n\t\t\tstreams[quality] = &extractors.Stream{\n\t\t\t\tParts:   urls,\n\t\t\t\tSize:    totalSize,\n\t\t\t\tQuality: qualityString,\n\t\t\t}\n\t\t}\n\n\t// if video file is mp4\n\tcase strings.Contains(data.Track.URL, \".mp4\"):\n\t\tsize, err = request.Size(data.Track.URL, uri)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL:  data.Track.URL,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[\"default\"] = &extractors.Stream{\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize:  size,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Twitter twitter.com\",\n\t\t\tTitle:   fmt.Sprintf(\"%s %s\", data.Username, data.TweetID),\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     uri,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/twitter/twitter_test.go\npackage twitter\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://twitter.com/justinbieber/status/898217160060698624\",\n\t\t\t\tTitle:   \"Justin Bieber on Twitter 898217160060698624\",\n\t\t\t\tQuality: \"720x1280\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"abnormal uri test1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://twitter.com/twitter/statuses/898567934192177153\",\n\t\t\t\tTitle:   \"Justin Bieber on Twitter 898567934192177153\",\n\t\t\t\tQuality: \"1280x720\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"abnormal uri test2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://twitter.com/kyoudera/status/971819131711373312/video/1/\",\n\t\t\t\tTitle:   \"\u30cd\u30e1\u30b7\u30b9 \u4eac\u5bfa on Twitter 971819131711373312\",\n\t\t\t\tQuality: \"1280x720\",\n\t\t\t},\n\t\t},\n\t}\n\t// The file size changes every time (caused by CDN?), so the size is not checked here\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/types.go\npackage extractors\n\n// Part is the data structure for a single part of the video stream information.\ntype Part struct {\n\tURL  string `json:\"url\"`\n\tSize int64  `json:\"size\"`\n\tExt  string `json:\"ext\"`\n}\n\ntype CaptionPart struct {\n\tPart\n\tTransform func([]byte) ([]byte, error) `json:\"-\"`\n}\n\n// Stream is the data structure for each video stream, eg: 720P, 1080P.\ntype Stream struct {\n\t// eg: \"1080\"\n\tID string `json:\"id\"`\n\t// eg: \"1080P xxx\"\n\tQuality string `json:\"quality\"`\n\t// [Part: {URL, Size, Ext}, ...]\n\t// Some video stream have multiple parts,\n\t// and can also be used to download multiple image files at once\n\tParts []*Part `json:\"parts\"`\n\t// total size of all urls\n\tSize int64 `json:\"size\"`\n\t// the file extension after video parts merged\n\tExt string `json:\"ext\"`\n\t// if the parts need mux\n\tNeedMux bool\n}\n\n// DataType indicates the type of extracted data, eg: video or image.\ntype DataType string\n\nconst (\n\t// DataTypeVideo indicates the type of extracted data is the video.\n\tDataTypeVideo DataType = \"video\"\n\t// DataTypeImage indicates the type of extracted data is the image.\n\tDataTypeImage DataType = \"image\"\n\t// DataTypeAudio indicates the type of extracted data is the audio.\n\tDataTypeAudio DataType = \"audio\"\n)\n\n// Data is the main data structure for the whole video data.\ntype Data struct {\n\t// URL is used to record the address of this download\n\tURL   string   `json:\"url\"`\n\tSite  string   `json:\"site\"`\n\tTitle string   `json:\"title\"`\n\tType  DataType `json:\"type\"`\n\t// each stream has it's own Parts and Quality\n\tStreams map[string]*Stream `json:\"streams\"`\n\t// danmaku, subtitles, etc\n\tCaptions map[string]*CaptionPart `json:\"caption\"`\n\t// Err is used to record whether an error occurred when extracting the list data\n\tErr error `json:\"err\"`\n}\n\n// FillUpStreamsData fills up some data automatically.\nfunc (d *Data) FillUpStreamsData() {\n\tfor id, stream := range d.Streams {\n\t\t// fill up ID\n\t\tstream.ID = id\n\t\tif stream.Quality == \"\" {\n\t\t\tstream.Quality = id\n\t\t}\n\n\t\t// generate the merged file extension\n\t\tif d.Type == DataTypeVideo && stream.Ext == \"\" {\n\t\t\text := stream.Parts[0].Ext\n\t\t\t// The file extension in `Parts` is used as the merged file extension by default, except for the following formats\n\t\t\tswitch ext {\n\t\t\t// ts and flv files should be merged into an mp4 file\n\t\t\tcase \"ts\", \"flv\", \"f4v\":\n\t\t\t\text = \"mp4\"\n\t\t\t}\n\t\t\tstream.Ext = ext\n\t\t}\n\n\t\t// calculate total size\n\t\tif stream.Size > 0 {\n\t\t\tcontinue\n\t\t}\n\t\tvar size int64\n\t\tfor _, part := range stream.Parts {\n\t\t\tsize += part.Size\n\t\t}\n\t\tstream.Size = size\n\t}\n}\n\n// EmptyData returns an \"empty\" Data object with the given URL and error.\nfunc EmptyData(url string, err error) *Data {\n\treturn &Data{\n\t\tURL: url,\n\t\tErr: err,\n\t}\n}\n\n// Options defines optional options that can be used in the extraction function.\ntype Options struct {\n\t// Playlist indicates if we need to extract the whole playlist rather than the single video.\n\tPlaylist bool\n\t// Items defines wanted items from a playlist. Separated by commas like: 1,5,6,8-10.\n\tItems string\n\t// ItemStart defines the starting item of a playlist.\n\tItemStart int\n\t// ItemEnd defines the ending item of a playlist.\n\tItemEnd int\n\n\t// ThreadNumber defines how many threads will use in the extraction, only works when Playlist is true.\n\tThreadNumber int\n\tCookie       string\n\n\t// EpisodeTitleOnly indicates file name of each bilibili episode doesn't include the playlist title\n\tEpisodeTitleOnly bool\n\n\tYoukuCcode    string\n\tYoukuCkey     string\n\tYoukuPassword string\n}\n\n// Extractor implements video data extraction related operations.\ntype Extractor interface {\n\t// Extract is the main function to extract the data.\n\tExtract(url string, option Options) ([]*Data, error)\n}\n\n// Path: extractors/udn/udn.go\npackage udn\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"udn\", New())\n}\n\nconst (\n\tstartFlag = `',\n            mp4: '//`\n\tendFlag = `'\n        },\n        subtitles`\n)\n\nfunc getCDNUrl(html string) string {\n\tif cdnURLs := utils.MatchOneOf(html, startFlag+\"(.+?)\"+endFlag); len(cdnURLs) > 1 && cdnURLs[1] != \"\" {\n\t\treturn cdnURLs[1]\n\t}\n\treturn \"\"\n}\n\nfunc prepareEmbedURL(url string) string {\n\tif !strings.Contains(url, \"https://video.udn.com/embed/\") {\n\t\tnewIDs := strings.Split(url, \"/\")\n\t\tif len(newIDs) < 1 {\n\t\t\treturn \"\"\n\t\t}\n\t\treturn \"https://video.udn.com/embed/news/\" + newIDs[len(newIDs)-1]\n\t}\n\treturn url\n}\n\ntype extractor struct{}\n\n// New returns a udn extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\turl = prepareEmbedURL(url)\n\tif len(url) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tdesc := utils.MatchOneOf(html, `title: '(.+?)',\n        link:`)\n\tif len(desc) > 1 {\n\t\ttitle = desc[1]\n\t} else {\n\t\ttitle = \"udn\"\n\t}\n\tcdnURL := getCDNUrl(html)\n\tif cdnURL == \"\" {\n\t\treturn nil, errors.New(\"empty list\")\n\t}\n\tsrcURL, err := request.Get(\"http://\"+cdnURL, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsize, err := request.Size(srcURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  srcURL,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"udn udn.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/udn/udn_test.go\npackage udn\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestExtract(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://video.udn.com/embed/news/300040\",\n\t\t\t\tTitle: `\u751f\u7269\u8001\u5e2b\u7537\u8b8a\u5973 \u5168\u6821\u633a\"\u505a\u81ea\u5df1\"`,\n\t\t\t\tSize:  12740874,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/universal/universal.go\npackage universal\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"\", New())\n}\n\ntype extractor struct{}\n\n// New returns a universal extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tfilename, ext, err := utils.GetNameAndExt(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsize, err := request.Size(url, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  url,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt:  ext,\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: size,\n\t\t},\n\t}\n\tcontentType, err := request.ContentType(url, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Universal\",\n\t\t\tTitle:   filename,\n\t\t\tType:    extractors.DataType(contentType),\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/universal/universal_test.go\npackage universal\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://img9.bcyimg.com/drawer/15294/post/1799t/1f5a87801a0711e898b12b640777720f.jpg\",\n\t\t\t\tTitle: \"1f5a87801a0711e898b12b640777720f\",\n\t\t\t\tSize:  1051042,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/vimeo/vimeo.go\npackage vimeo\n\nimport (\n\t\"encoding/json\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"vimeo\", New())\n}\n\ntype vimeoProgressive struct {\n\tWidth   int    `json:\"width\"`\n\tHeight  int    `json:\"height\"`\n\tProfile string `json:\"profile\"`\n\tQuality string `json:\"quality\"`\n\tURL     string `json:\"url\"`\n}\n\ntype vimeoFiles struct {\n\tProgressive []vimeoProgressive `json:\"progressive\"`\n}\n\ntype vimeoRequest struct {\n\tFiles vimeoFiles `json:\"files\"`\n}\n\ntype vimeoVideo struct {\n\tTitle string `json:\"title\"`\n}\n\ntype vimeo struct {\n\tRequest vimeoRequest `json:\"request\"`\n\tVideo   vimeoVideo   `json:\"video\"`\n}\n\ntype extractor struct{}\n\n// New returns a vimeo extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar (\n\t\thtml, vid string\n\t\terr       error\n\t)\n\tif strings.Contains(url, \"player.vimeo.com\") {\n\t\thtml, err = request.Get(url, url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t} else {\n\t\tvid = utils.MatchOneOf(url, `vimeo\\.com/(\\d+)`)[1]\n\t\thtml, err = request.Get(\"https://player.vimeo.com/video/\"+vid, url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t}\n\tjsonStrings := utils.MatchOneOf(html, `var \\w+\\s?=\\s?({.+?});`)\n\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := jsonStrings[1]\n\n\tvar vimeoData vimeo\n\tif err = json.Unmarshal([]byte(jsonString), &vimeoData); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, len(vimeoData.Request.Files.Progressive))\n\tvar size int64\n\tfor _, video := range vimeoData.Request.Files.Progressive {\n\t\tsize, err = request.Size(video.URL, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL:  video.URL,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[video.Profile] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: video.Quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Vimeo vimeo.com\",\n\t\t\tTitle:   vimeoData.Video.Title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/vimeo/vimeo_test.go\npackage vimeo\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://player.vimeo.com/video/259325107\",\n\t\t\t\tTitle:   \"prfm 20180309\",\n\t\t\t\tSize:    131051118,\n\t\t\t\tQuality: \"1080p\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://vimeo.com/254865724\",\n\t\t\t\tTitle:   \"MAGIC DINER PT. II\",\n\t\t\t\tSize:    138966306,\n\t\t\t\tQuality: \"1080p\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/vk/vk.go\npackage vk\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/config\"\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"vk\", New())\n}\n\nvar qualityNames = map[int]string{\n\t0: \"Highest\",\n\t1: \"High\",\n\t2: \"Medium\",\n\t3: \"Low\",\n\t4: \"Lowest\",\n\t5: \"Legacy\",\n}\n\ntype extractor struct{}\n\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\t// If url comes from feed or search, its id stored in url parameter.\n\t// We need to convert it to direct link to make it work with m.vk.com.\n\tif strings.Contains(url, \"z=\") {\n\t\tsplit := strings.Split(url, \"z=\")\n\t\turl = split[len(split)-1]\n\t\turl = strings.Split(url, \"%2F\")[0]\n\t}\n\n\t// Convert url to mobile version.\n\tsplit := strings.Split(url, \"vk.com\")\n\turl = split[len(split)-1]\n\tif url[0] == '/' {\n\t\turl = url[1:]\n\t}\n\turl = \"https://m.vk.com/\" + url\n\n\t// Set custom cookies required to download high-res video.\n\tconfig.FakeHeaders[\"Cookie\"] += \"remixlang=0; remixaudio_show_alert_today=0; remixff=0; remixmdevice=1920/1080/1/!!-!!!!!!\"\n\n\t// Get html.\n\thtml, err := request.Get(url, url, config.FakeHeaders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// Get video title.\n\ttitles := utils.MatchOneOf(html, `<h1 class=\"VideoPageInfoRow__title\">(.*)</h1>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\t// Get video urls.\n\tsources := utils.MatchAll(html, `<source(.*?)/>`)\n\tsrcs := make([]string, len(sources))\n\tj := 0\n\tfor i := range sources {\n\t\tsrcs[j] = utils.MatchOneOf(sources[i][1], `src=\"(.*?)\"`)[1]\n\t\tsrcs[j] = strings.Replace(srcs[j], \"&amp;\", \"&\", -1)\n\t\t// Some videos have some technical preview on domain vk.com.\n\t\t// We need to remove it.\n\t\tif strings.Contains(srcs[j], \"vk.com\") {\n\t\t\tsrcs = append(srcs[:j], srcs[j+1:]...)\n\t\t} else {\n\t\t\tj++\n\t\t}\n\t}\n\n\t// Create download streams.\n\tstreams := make(map[string]*extractors.Stream)\n\tfor i := range srcs {\n\t\tsize, err := request.Size(srcs[i], \"m.vk.vom\")\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL:  srcs[i],\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[qualityNames[i]] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: qualityNames[i],\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"VK vk.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/vk/vk_test.go\npackage vk\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestVK(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 0\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://vk.com/video/&z=video9671026_161348481%2Fclub43218296%2Fpl_-43218296_-2\",\n\t\t\t\tTitle: \"Rick Ashley - RickRoll\",\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/weibo/weibo.go\npackage weibo\n\nimport (\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"weibo\", New())\n}\n\ntype playInfo struct {\n\tTitle string            `json:\"title\"`\n\tURLs  map[string]string `json:\"urls\"`\n}\n\ntype playData struct {\n\tPlayInfo playInfo `json:\"Component_Play_Playinfo\"`\n}\n\ntype weiboData struct {\n\tCode string   `json:\"code\"`\n\tData playData `json:\"data\"`\n\tMsg  string   `json:\"msg\"`\n}\n\n...\n", "needle_token_start": 2028, "needle_token_end": 2541, "code_context_ntokens": 15131, "output": ["```go\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\t// Fetch the HTML content of the given URL\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// Extract the video title from the HTML\n\ttitles := utils.MatchOneOf(html, `<h1 class=\"VideoPageInfoRow__title\">(.*)</h1>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\t// Search for video URLs in the HTML content\n\tsources := utils.MatchAll(html, `<source(.*?)/>`)\n\tsrcs := make([]string, len(sources))\n\tj := 0\n\tfor i := range sources {\n\t\tsrcs[j] = utils.MatchOneOf(sources[i][1], `src=\"(.*?)\"`)[1]\n\t\tsrcs[j] = strings.Replace(srcs[j], \"&amp;\", \"&\", -1)\n\t\t// Remove technical preview URLs\n\t\tif strings.Contains(srcs[j], \"vk.com\") {\n\t\t\tsrcs = append(srcs[:j], srcs[j+1:]...)\n\t\t} else {\n\t\t\tj++\n\t\t}\n\t}\n\n\t// Create download streams\n\tstreams := make(map[string]*extractors.Stream)\n\tfor i := range srcs {\n\t\tsize, err := request.Size(srcs[i], \"m.vk.vom\")\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL:  srcs[i],\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[qualityNames[i]] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: qualityNames[i],\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"VK vk.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n```"]}
{"repo": "iawia002/lux", "name": "getMultiPageData", "language": "go", "path": "extractors/bilibili/bilibili.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to parse a web page's HTML content to extract and decode information about multiple pages or episodes available on a video streaming platform.\n2. **Input**: The function takes a single string parameter, which is the HTML content of the webpage.\n3. **Output**: It returns a structured object containing details about multiple pages or episodes, or an error if the required data cannot be extracted or parsed from the HTML.\n4. **Procedure**: \n   - The function first searches the HTML content for a specific pattern that contains the initial state data of the webpage.\n   - If the pattern is not found, it returns an error indicating the absence of a playlist.\n   - If the pattern is found, it extracts the JSON string that contains the multi-page data.\n   - This JSON string is then parsed into a structured object.\n   - The function returns this object if the parsing is successful; otherwise, it returns an error.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/acfun/types.go\npackage acfun\n\ntype episodeData struct {\n\tItemID      int64  `json:\"itemId\"`\n\tEpisodeName string `json:\"episodeName\"`\n\tBangumiID   int64  `json:\"bangumiId\"`\n\tVideoID     int64  `json:\"videoId\"`\n}\n\ntype bangumiData struct {\n\tepisodeData\n\tBangumiTitle     string `json:\"bangumiTitle\"`\n\tCurrentVideoInfo struct {\n\t\tKsPlayJSON string `json:\"ksPlayJson\"`\n\t} `json:\"currentVideoInfo\"`\n}\n\ntype videoInfo struct {\n\tAdaptationSet []struct {\n\t\tStreams streams `json:\"representation\"`\n\t} `json:\"adaptationSet\"`\n}\n\ntype streams []stream\n\ntype episodeList struct {\n\tEpisodes []*episodeData `json:\"items\"`\n}\n\ntype stream struct {\n\tID           int64  `json:\"id\"`\n\tBackURL      string `json:\"backUrl\"`\n\tCodecs       string `json:\"codecs\"`\n...\n// Path: extractors/bcy/bcy.go\npackage bcy\n\nimport (\n\t\"encoding/json\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"bcy\", New())\n}\n\ntype bcyData struct {\n\tDetail struct {\n\t\tPostData struct {\n\t\t\tMulti []struct {\n\t\t\t\tOriginalPath string `json:\"original_path\"`\n\t\t\t} `json:\"multi\"`\n\t\t} `json:\"post_data\"`\n\t} `json:\"detail\"`\n}\n\ntype extractor struct{}\n\n// New returns a bcy extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// parse json data\n\trep := strings.NewReplacer(`\\\"`, `\"`, `\\\\`, `\\`)\n\trealURLs := utils.MatchOneOf(html, `JSON.parse\\(\"(.+?)\"\\);`)\n\tif realURLs == nil || len(realURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := rep.Replace(realURLs[1])\n\n\tvar data bcyData\n\tif err = json.Unmarshal([]byte(jsonString), &data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitle := strings.Replace(parser.Title(doc), \" - \u534a\u6b21\u5143 banciyuan - ACG\u7231\u597d\u8005\u793e\u533a\", \"\", -1)\n\n\tparts := make([]*extractors.Part, 0, len(data.Detail.PostData.Multi))\n\tvar totalSize int64\n\tfor _, img := range data.Detail.PostData.Multi {\n\t\tsize, err := request.Size(img.OriginalPath, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\ttotalSize += size\n\t\t_, ext, err := utils.GetNameAndExt(img.OriginalPath)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL:  img.OriginalPath,\n\t\t\tSize: size,\n\t\t\tExt:  ext,\n\t\t})\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u534a\u6b21\u5143 bcy.net\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/bcy/bcy_test.go\npackage bcy\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://bcy.net/item/detail/6558738153367142664\",\n\t\t\t\tTitle: \"cos\u6b63\u7247 \u547d\u8fd0\u77f3\u4e4b\u95e8 \u7267\u6fd1\u7ea2\u8389\u6816 \u514b\u91cc\u65af\u8482\u5a1c\u2026 - \u534a\u6b21\u5143 - ACG\u7231\u597d\u8005\u793e\u533a\",\n\t\t\t\tSize:  13035763,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/bilibili/bilibili.go\npackage bilibili\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"slices\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\tbilibiliExtractor := New()\n\textractors.Register(\"bilibili\", bilibiliExtractor)\n\textractors.Register(\"b23\", bilibiliExtractor)\n}\n\nconst (\n\tbilibiliAPI        = \"https://api.bilibili.com/x/player/playurl?\"\n\tbilibiliBangumiAPI = \"https://api.bilibili.com/pgc/player/web/playurl?\"\n\tbilibiliTokenAPI   = \"https://api.bilibili.com/x/player/playurl/token?\"\n)\n\nconst referer = \"https://www.bilibili.com\"\n\nvar utoken string\n\nfunc genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (string, error) {\n\tvar (\n\t\terr        error\n\t\tbaseAPIURL string\n\t\tparams     string\n\t)\n\tif cookie != \"\" && utoken == \"\" {\n\t\tutoken, err = request.Get(\n\t\t\tfmt.Sprintf(\"%said=%d&cid=%d\", bilibiliTokenAPI, aid, cid),\n\t\t\treferer,\n\t\t\tnil,\n\t\t)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tvar t token\n\t\terr = json.Unmarshal([]byte(utoken), &t)\n\t\tif err != nil {\n\t\t\treturn \"\", err\n\t\t}\n\t\tif t.Code != 0 {\n\t\t\treturn \"\", errors.Errorf(\"cookie error: %s\", t.Message)\n\t\t}\n\t\tutoken = t.Data.Token\n\t}\n\tvar api string\n\tif bangumi {\n\t\t// The parameters need to be sorted by name\n\t\t// qn=0 flag makes the CDN address different every time\n\t\t// quality=120(4k) is the highest quality so far\n\t\tparams = fmt.Sprintf(\n\t\t\t\"cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=16\",\n\t\t\tcid, bvid, quality,\n\t\t)\n\t\tbaseAPIURL = bilibiliBangumiAPI\n\t} else {\n\t\tparams = fmt.Sprintf(\n\t\t\t\"avid=%d&cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=2000\",\n\t\t\taid, cid, bvid, quality,\n\t\t)\n\t\tbaseAPIURL = bilibiliAPI\n\t}\n\tapi = baseAPIURL + params\n\t// bangumi utoken also need to put in params to sign, but the ordinary video doesn't need\n\tif !bangumi && utoken != \"\" {\n\t\tapi = fmt.Sprintf(\"%s&utoken=%s\", api, utoken)\n\t}\n\treturn api, nil\n}\n\ntype bilibiliOptions struct {\n\turl      string\n\thtml     string\n\tbangumi  bool\n\taid      int\n\tcid      int\n\tbvid     string\n\tpage     int\n\tsubtitle string\n}\n\nfunc extractBangumi(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tdataString := utils.MatchOneOf(html, `<script\\s+id=\"__NEXT_DATA__\"\\s+type=\"application/json\"\\s*>(.*?)</script\\s*>`)[1]\n\tepArrayString := utils.MatchOneOf(dataString, `\"episodes\"\\s*:\\s*(.+?)\\s*,\\s*\"user_status\"`)[1]\n\tfullVideoIdString := utils.MatchOneOf(dataString, `\"videoId\"\\s*:\\s*\"(ep|ss)(\\d+)\"`)\n\tepSsString := fullVideoIdString[1] // \"ep\" or \"ss\"\n\tvideoIdString := fullVideoIdString[2]\n\n\tvar epArray []json.RawMessage\n\terr := json.Unmarshal([]byte(epArrayString), &epArray)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar data bangumiData\n\tfor _, jsonByte := range epArray {\n\t\tvar epInfo bangumiEpData\n\t\terr := json.Unmarshal(jsonByte, &epInfo)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tvideoId, err := strconv.ParseInt(videoIdString, 10, 0)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tif epInfo.ID == int(videoId) || (epSsString == \"ss\" && epInfo.TitleFormat == \"\u7b2c1\u8bdd\") {\n\t\t\tdata.EpInfo = epInfo\n\t\t}\n\t\tdata.EpList = append(data.EpList, epInfo)\n\t}\n\n\tsort.Slice(data.EpList, func(i, j int) bool {\n\t\treturn data.EpList[i].EpID < data.EpList[j].EpID\n\t})\n\n\tif !extractOption.Playlist {\n\t\taid := data.EpInfo.Aid\n\t\tcid := data.EpInfo.Cid\n\t\tbvid := data.EpInfo.BVid\n\t\ttitleFormat := data.EpInfo.TitleFormat\n\t\tlongTitle := data.EpInfo.LongTitle\n\t\tif aid <= 0 || cid <= 0 || bvid == \"\" {\n\t\t\taid = data.EpList[0].Aid\n\t\t\tcid = data.EpList[0].Cid\n\t\t\tbvid = data.EpList[0].BVid\n\t\t\ttitleFormat = data.EpList[0].TitleFormat\n\t\t\tlongTitle = data.EpList[0].LongTitle\n\t\t}\n\t\toptions := bilibiliOptions{\n\t\t\turl:     url,\n\t\t\thtml:    html,\n\t\t\tbangumi: true,\n\t\t\taid:     aid,\n\t\t\tcid:     cid,\n\t\t\tbvid:    bvid,\n\n\t\t\tsubtitle: fmt.Sprintf(\"%s %s\", titleFormat, longTitle),\n\t\t}\n\t\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n\t}\n\n\t// handle bangumi playlist\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(data.EpList))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range data.EpList {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\tid := u.EpID\n\t\tif id == 0 {\n\t\t\tid = u.ID\n\t\t}\n\t\t// html content can't be reused here\n\t\toptions := bilibiliOptions{\n\t\t\turl:     fmt.Sprintf(\"https://www.bilibili.com/bangumi/play/ep%d\", id),\n\t\t\tbangumi: true,\n\t\t\taid:     u.Aid,\n\t\t\tcid:     u.Cid,\n\t\t\tbvid:    u.BVid,\n\n\t\t\tsubtitle: fmt.Sprintf(\"%s %s\", u.TitleFormat, u.LongTitle),\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\nfunc\n getMultiPageData(html string) (*multiPage, error) {\n\tvar data multiPage\n\tmultiPageDataString := utils.MatchOneOf(\n\t\thtml, `window.__INITIAL_STATE__=(.+?);\\(function`,\n\t)\n\tif multiPageDataString == nil {\n\t\treturn &data, errors.New(\"this page has no playlist\")\n\t}\n\terr := json.Unmarshal([]byte(multiPageDataString[1]), &data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn &data, nil\n}\n\nfunc extractFestival(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tmatches := utils.MatchAll(html, \"<\\\\s*script[^>]*>\\\\s*window\\\\.__INITIAL_STATE__=([\\\\s\\\\S]*?);\\\\s?\\\\(function[\\\\s\\\\S]*?<\\\\/\\\\s*script\\\\s*>\")\n\tif len(matches) < 1 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tif len(matches[0]) < 2 {\n\t\treturn nil, errors.New(\"could not find video in page\")\n\t}\n\n\tvar festivalData festival\n\terr := json.Unmarshal([]byte(matches[0][1]), &festivalData)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\toptions := bilibiliOptions{\n\t\turl:  url,\n\t\thtml: html,\n\t\taid:  festivalData.VideoInfo.Aid,\n\t\tbvid: festivalData.VideoInfo.BVid,\n\t\tcid:  festivalData.VideoInfo.Cid,\n\t\tpage: 0,\n\t}\n\n\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n}\n\nfunc extractNormalVideo(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tpageData, err := getMultiPageData(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif !extractOption.Playlist {\n\t\t// handle URL that has a playlist, mainly for unified titles\n\t\t// <h1> tag does not include subtitles\n\t\t// bangumi doesn't need this\n\t\tpageString := utils.MatchOneOf(url, `\\?p=(\\d+)`)\n\t\tvar p int\n\t\tif pageString == nil {\n\t\t\t// https://www.bilibili.com/video/av20827366/\n\t\t\tp = 1\n\t\t} else {\n\t\t\t// https://www.bilibili.com/video/av20827366/?p=2\n\t\t\tp, _ = strconv.Atoi(pageString[1])\n\t\t}\n\n\t\tif len(pageData.VideoData.Pages) < p || p < 1 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\n\t\tpage := pageData.VideoData.Pages[p-1]\n\t\toptions := bilibiliOptions{\n\t\t\turl:  url,\n\t\t\thtml: html,\n\t\t\taid:  pageData.Aid,\n\t\t\tbvid: pageData.BVid,\n\t\t\tcid:  page.Cid,\n\t\t\tpage: p,\n\t\t}\n\t\t// \"part\":\"\" or \"part\":\"Untitled\"\n\t\tif page.Part == \"Untitled\" || len(pageData.VideoData.Pages) == 1 {\n\t\t\toptions.subtitle = \"\"\n\t\t} else {\n\t\t\toptions.subtitle = page.Part\n\t\t}\n\t\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n\t}\n\n\t// handle normal video playlist\n\tif len(pageData.Sections) == 0 {\n\t\t// https://www.bilibili.com/video/av20827366/?p=* each video in playlist has different p=?\n\t\treturn multiPageDownload(url, html, extractOption, pageData)\n\t}\n\t// handle another kind of playlist\n\t// https://www.bilibili.com/video/av*** each video in playlist has different av/bv id\n\treturn multiEpisodeDownload(url, html, extractOption, pageData)\n}\n\n// handle multi episode download\nfunc multiEpisodeDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) {\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.Sections[0].Episodes))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range pageData.Sections[0].Episodes {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\toptions := bilibiliOptions{\n\t\t\turl:      url,\n\t\t\thtml:     html,\n\t\t\taid:      u.Aid,\n\t\t\tbvid:     u.BVid,\n\t\t\tcid:      u.Cid,\n\t\t\tsubtitle: fmt.Sprintf(\"%s P%d\", u.Title, index+1),\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\n// handle multi page download\nfunc multiPageDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) {\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.VideoData.Pages))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range pageData.VideoData.Pages {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\toptions := bilibiliOptions{\n\t\t\turl:      url,\n\t\t\thtml:     html,\n\t\t\taid:      pageData.Aid,\n\t\t\tbvid:     pageData.BVid,\n\t\t\tcid:      u.Cid,\n\t\t\tsubtitle: u.Part,\n\t\t\tpage:     u.Page,\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\ntype extractor struct{}\n\n// New returns a bilibili extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\thtml, err := request.Get(url, referer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// set thread number to 1 manually to avoid http 412 error\n\toption.ThreadNumber = 1\n\n\tif strings.Contains(url, \"bangumi\") {\n\t\t// handle bangumi\n\t\treturn extractBangumi(url, html, option)\n\t} else if strings.Contains(url, \"festival\") {\n\t\treturn extractFestival(url, html, option)\n\t} else {\n\t\t// handle normal video\n\t\treturn extractNormalVideo(url, html, option)\n\t}\n}\n\n// bilibiliDownload is the download function for a single URL\nfunc bilibiliDownload(options bilibiliOptions, extractOption extractors.Options) *extractors.Data {\n\tvar (\n\t\terr  error\n\t\thtml string\n\t)\n\tif options.html != \"\" {\n\t\t// reuse html string, but this can't be reused in case of playlist\n\t\thtml = options.html\n\t} else {\n\t\thtml, err = request.Get(options.url, referer, nil)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t}\n\n\t// Get \"accept_quality\" and \"accept_description\"\n\t// \"accept_description\":[\"\u8d85\u9ad8\u6e05 8K\",\"\u8d85\u6e05 4K\",\"\u9ad8\u6e05 1080P+\",\"\u9ad8\u6e05 1080P\",\"\u9ad8\u6e05 720P\",\"\u6e05\u6670 480P\",\"\u6d41\u7545 360P\"],\n\t// \"accept_quality\":[127\uff0c120,112,80,48,32,16],\n\tapi, err := genAPI(options.aid, options.cid, 127, options.bvid, options.bangumi, extractOption.Cookie)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\tjsonString, err := request.Get(api, referer, nil)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\n\tvar data dash\n\terr = json.Unmarshal([]byte(jsonString), &data)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\tvar dashData dashInfo\n\tif data.Data.Description == nil {\n\t\tdashData = data.Result\n\t} else {\n\t\tdashData = data.Data\n\t}\n\n\tvar audioPart *extractors.Part\n\tif dashData.Streams.Audio != nil {\n\t\t// Get audio part\n\t\tvar audioID int\n\t\taudios := map[int]string{}\n\t\tbandwidth := 0\n\t\tfor _, stream := range dashData.Streams.Audio {\n\t\t\tif stream.Bandwidth > bandwidth {\n\t\t\t\taudioID = stream.ID\n\t\t\t\tbandwidth = stream.Bandwidth\n\t\t\t}\n\t\t\taudios[stream.ID] = stream.BaseURL\n\t\t}\n\t\ts, err := request.Size(audios[audioID], referer)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t\taudioPart = &extractors.Part{\n\t\t\tURL:  audios[audioID],\n\t\t\tSize: s,\n\t\t\tExt:  \"m4a\",\n\t\t}\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, len(dashData.Quality))\n\tfor _, stream := range dashData.Streams.Video {\n\t\ts, err := request.Size(stream.BaseURL, referer)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t\tparts := make([]*extractors.Part, 0, 2)\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL:  stream.BaseURL,\n\t\t\tSize: s,\n\t\t\tExt:  getExtFromMimeType(stream.MimeType),\n\t\t})\n\t\tif audioPart != nil {\n\t\t\tparts = append(parts, audioPart)\n\t\t}\n\t\tvar size int64\n\t\tfor _, part := range parts {\n\t\t\tsize += part.Size\n\t\t}\n\t\tid := fmt.Sprintf(\"%d-%d\", stream.ID, stream.Codecid)\n\t\tstreams[id] = &extractors.Stream{\n\t\t\tParts:   parts,\n\t\t\tSize:    size,\n\t\t\tQuality: fmt.Sprintf(\"%s %s\", qualityString[stream.ID], stream.Codecs),\n\t\t}\n\t\tif audioPart != nil {\n\t\t\tstreams[id].NeedMux = true\n\t\t}\n\t}\n\n\tfor _, durl := range dashData.DURLs {\n\t\tvar ext string\n\t\tswitch dashData.DURLFormat {\n\t\tcase \"flv\", \"flv480\":\n\t\t\text = \"flv\"\n\t\tcase \"mp4\", \"hdmp4\": // nolint\n\t\t\text = \"mp4\"\n\t\t}\n\n\t\tparts := make([]*extractors.Part, 0, 1)\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL:  durl.URL,\n\t\t\tSize: durl.Size,\n\t\t\tExt:  ext,\n\t\t})\n\n\t\tstreams[strconv.Itoa(dashData.CurQuality)] = &extractors.Stream{\n\t\t\tParts:   parts,\n\t\t\tSize:    durl.Size,\n\t\t\tQuality: qualityString[dashData.CurQuality],\n\t\t}\n\t}\n\n\t// get the title\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\ttitle := parser.Title(doc)\n\tif options.subtitle != \"\" {\n\t\tpageString := \"\"\n\t\tif options.page > 0 {\n\t\t\tpageString = fmt.Sprintf(\"P%d \", options.page)\n\t\t}\n\t\tif extractOption.EpisodeTitleOnly {\n\t\t\ttitle = fmt.Sprintf(\"%s%s\", pageString, options.subtitle)\n\t\t} else {\n\t\t\ttitle = fmt.Sprintf(\"%s %s%s\", title, pageString, options.subtitle)\n\t\t}\n\t}\n\n\treturn &extractors.Data{\n\t\tSite:    \"\u54d4\u54e9\u54d4\u54e9 bilibili.com\",\n\t\tTitle:   title,\n\t\tType:    extractors.DataTypeVideo,\n\t\tStreams: streams,\n\t\tCaptions: map[string]*extractors.CaptionPart{\n\t\t\t\"danmaku\": {\n\t\t\t\tPart: extractors.Part{\n\t\t\t\t\tURL: fmt.Sprintf(\"https://comment.bilibili.com/%d.xml\", options.cid),\n\t\t\t\t\tExt: \"xml\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"subtitle\": getSubTitleCaptionPart(options.aid, options.cid),\n\t\t},\n\t\tURL: options.url,\n\t}\n}\n\nfunc getExtFromMimeType(mimeType string) string {\n\texts := strings.Split(mimeType, \"/\")\n\tif len(exts) == 2 {\n\t\treturn exts[1]\n\t}\n\treturn \"mp4\"\n}\n\nfunc getSubTitleCaptionPart(aid int, cid int) *extractors.CaptionPart {\n\tjsonString, err := request.Get(\n\t\tfmt.Sprintf(\"http://api.bilibili.com/x/player/wbi/v2?aid=%d&cid=%d\", aid, cid), referer, nil,\n\t)\n\tif err != nil {\n\t\treturn nil\n\t}\n\tstu := bilibiliWebInterface{}\n\terr = json.Unmarshal([]byte(jsonString), &stu)\n\tif err != nil || len(stu.Data.SubtitleInfo.SubtitleList) == 0 {\n\t\treturn nil\n\t}\n\treturn &extractors.CaptionPart{\n\t\tPart: extractors.Part{\n\t\t\tURL: fmt.Sprintf(\"https:%s\", stu.Data.SubtitleInfo.SubtitleList[0].SubtitleUrl),\n\t\t\tExt: \"srt\",\n\t\t},\n\t\tTransform: subtitleTransform,\n\t}\n}\n\nfunc subtitleTransform(body []byte) ([]byte, error) {\n\tbytes := \"\"\n\tcaptionData := bilibiliSubtitleFormat{}\n\terr := json.Unmarshal(body, &captionData)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tfor i := 0; i < len(captionData.Body); i++ {\n\t\tbytes += fmt.Sprintf(\"%d\\n%s --> %s\\n%s\\n\\n\",\n\t\t\ti,\n\t\t\ttime.Unix(0, int64(captionData.Body[i].From*1000)*int64(time.Millisecond)).UTC().Format(\"15:04:05.000\"),\n\t\t\ttime.Unix(0, int64(captionData.Body[i].To*1000)*int64(time.Millisecond)).UTC().Format(\"15:04:05.000\"),\n\t\t\tcaptionData.Body[i].Content,\n\t\t)\n\t}\n\treturn []byte(bytes), nil\n}\n\n// Path: extractors/bilibili/bilibili_test.go\npackage bilibili\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestBilibili(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\targs     test.Args\n\t\tplaylist bool\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/av20203945/\",\n\t\t\t\tTitle: \"\u30102018\u62dc\u5e74\u796d\u5355\u54c1\u3011\u76f8\u9047day by day\",\n\t\t\t},\n\t\t\tplaylist: false,\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/av41301960\",\n\t\t\t\tTitle: \"\u3010\u82f1\u96c4\u8054\u76df\u30112019\u8d5b\u5b63CG \u300a\u89c9\u9192\u300b\",\n\t\t\t},\n\t\t\tplaylist: false,\n\t\t},\n\t\t{\n\t\t\tname: \"bangumi test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/bangumi/play/ep167000\",\n\t\t\t\tTitle: \"\u72d0\u5996\u5c0f\u7ea2\u5a18 \u7b2c70\u8bdd \u82cf\u82cf\u667a\u5546\u4e0a\u7ebf\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"bangumi playlist test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/bangumi/play/ss5050\",\n\t\t\t\tTitle: \"\u4e00\u4eba\u4e4b\u4e0b\uff1a\u7b2c1\u8bdd \u5f02\u4eba\u5200\u5175\u8d77\uff0c\u9053\u7081\u643a\u9634\u9633\",\n\t\t\t},\n\t\t\tplaylist: true,\n\t\t},\n\t\t{\n\t\t\tname: \"playlist test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/av16907446/\",\n\t\t\t\tTitle: \"\\\"\u4e0d\u8981\u76f8\u4fe1\u6b4c\u8bcd\uff0c\u4ed6\u4eec\u4e3a\u4e86\u62bc\u97f5\u4ec0\u4e48\u90fd\u5e72\u5f97\u51fa\u6765\\\"\",\n\t\t\t},\n\t\t\tplaylist: true,\n\t\t},\n\t\t{\n\t\t\tname: \"8k test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/BV1qM4y1w716\",\n\t\t\t\tTitle: \"\u30108K\u6f14\u793a\u7247\u3011B\u7ad9\u9996\u53d1\uff01\u4f60\u7684\u8bbe\u5907\u8fd8\u9876\u5f97\u4f4f\u5417\uff1f\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"b23 test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://b23.tv/Fc9i7QF\",\n\t\t\t\tTitle: \"\u3010\u5341\u5e74\u699c\u30112000-2009\u5e74\u6700\u5f3a\u534e\u8bed\u91d1\u66f2TOP100 P1 100\u7231\u8f6c\u89d2-\u7f57\u5fd7\u7965\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"festival test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/festival/lty10th?bvid=BV1dZ4y1Y7bt\",\n\t\t\t\tTitle: \"\u6d1b\u5929\u4f9d\u5341\u5468\u5e74\u5b98\u65b9\u6f14\u5531\u4f1a\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar (\n\t\t\t\tdata []*extractors.Data\n\t\t\t\terr  error\n\t\t\t)\n\t\t\tif tt.playlist {\n\t\t\t\t// for playlist, we don't check the data\n\t\t\t\t_, err = New().Extract(tt.args.URL, extractors.Options{\n\t\t\t\t\tPlaylist:     true,\n\t\t\t\t\tThreadNumber: 9,\n\t\t\t\t})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t} else {\n\t\t\t\tdata, err = New().Extract(tt.args.URL, extractors.Options{})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t\ttest.Check(t, tt.args, data[0])\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: extractors/bilibili/types.go\npackage bilibili\n\n// {\"code\":0,\"message\":\"0\",\"ttl\":1,\"data\":{\"token\":\"aaa\"}}\n// {\"code\":-101,\"message\":\"\u8d26\u53f7\u672a\u767b\u5f55\",\"ttl\":1}\ntype tokenData struct {\n\tToken string `json:\"token\"`\n}\n\ntype token struct {\n\tCode    int       `json:\"code\"`\n\tMessage string    `json:\"message\"`\n\tData    tokenData `json:\"data\"`\n}\n\ntype bangumiEpData struct {\n\tAid         int    `json:\"aid\"`\n\tCid         int    `json:\"cid\"`\n\tBVid        string `json:\"bvid\"`\n\tID          int    `json:\"id\"`\n\tEpID        int    `json:\"ep_id\"`\n\tTitleFormat string `json:\"titleFormat\"`\n\tLongTitle   string `json:\"long_title\"`\n}\n\ntype bangumiData struct {\n\tEpInfo bangumiEpData   `json:\"epInfo\"`\n\tEpList []bangumiEpData `json:\"epList\"`\n}\n\ntype videoPagesData struct {\n\tCid  int    `json:\"cid\"`\n\tPart string `json:\"part\"`\n\tPage int    `json:\"page\"`\n}\n\ntype multiPageVideoData struct {\n\tTitle string           `json:\"title\"`\n\tPages []videoPagesData `json:\"pages\"`\n}\n\ntype episode struct {\n\tAid   int    `json:\"aid\"`\n\tCid   int    `json:\"cid\"`\n\tTitle string `json:\"title\"`\n\tBVid  string `json:\"bvid\"`\n}\n\ntype multiEpisodeData struct {\n\tSeasionid int       `json:\"season_id\"`\n\tEpisodes  []episode `json:\"episodes\"`\n}\n\ntype multiPage struct {\n\tAid       int                `json:\"aid\"`\n\tBVid      string             `json:\"bvid\"`\n\tSections  []multiEpisodeData `json:\"sections\"`\n\tVideoData multiPageVideoData `json:\"videoData\"`\n}\n\ntype dashStream struct {\n\tID        int    `json:\"id\"`\n\tBaseURL   string `json:\"baseUrl\"`\n\tBandwidth int    `json:\"bandwidth\"`\n\tMimeType  string `json:\"mimeType\"`\n\tCodecid   int    `json:\"codecid\"`\n\tCodecs    string `json:\"codecs\"`\n}\n\ntype dashStreams struct {\n\tVideo []dashStream `json:\"video\"`\n\tAudio []dashStream `json:\"audio\"`\n}\n\ntype dashInfo struct {\n\tCurQuality  int         `json:\"quality\"`\n\tDescription []string    `json:\"accept_description\"`\n\tQuality     []int       `json:\"accept_quality\"`\n\tStreams     dashStreams `json:\"dash\"`\n\tDURLFormat  string      `json:\"format\"`\n\tDURLs       []dURL      `json:\"durl\"`\n}\n\ntype dURL struct {\n\tURL  string `json:\"url\"`\n\tSize int64  `json:\"size\"`\n}\n\ntype dash struct {\n\tCode    int      `json:\"code\"`\n\tMessage string   `json:\"message\"`\n\tData    dashInfo `json:\"data\"`\n\tResult  dashInfo `json:\"result\"`\n}\n\nvar qualityString = map[int]string{\n\t127: \"\u8d85\u9ad8\u6e05 8K\",\n\t120: \"\u8d85\u6e05 4K\",\n\t116: \"\u9ad8\u6e05 1080P60\",\n\t74:  \"\u9ad8\u6e05 720P60\",\n\t112: \"\u9ad8\u6e05 1080P+\",\n\t80:  \"\u9ad8\u6e05 1080P\",\n\t64:  \"\u9ad8\u6e05 720P\",\n\t48:  \"\u9ad8\u6e05 720P\",\n\t32:  \"\u6e05\u6670 480P\",\n\t16:  \"\u6d41\u7545 360P\",\n\t15:  \"\u6d41\u7545 360P\",\n}\n\ntype subtitleData struct {\n\tFrom     float32 `json:\"from\"`\n\tTo       float32 `json:\"to\"`\n\tLocation int     `json:\"location\"`\n\tContent  string  `json:\"content\"`\n}\n\ntype bilibiliSubtitleFormat struct {\n\tFontSize        float32        `json:\"font_size\"`\n\tFontColor       string         `json:\"font_color\"`\n\tBackgroundAlpha float32        `json:\"background_alpha\"`\n\tBackgroundColor string         `json:\"background_color\"`\n\tStroke          string         `json:\"Stroke\"`\n\tBody            []subtitleData `json:\"body\"`\n}\n\ntype subtitleProperty struct {\n\tID          int64  `json:\"id\"`\n\tLan         string `json:\"lan\"`\n\tLanDoc      string `json:\"lan_doc\"`\n\tSubtitleUrl string `json:\"subtitle_url\"`\n}\n\ntype subtitleInfo struct {\n\tAllowSubmit  bool               `json:\"allow_submit\"`\n\tSubtitleList []subtitleProperty `json:\"subtitles\"`\n}\n\ntype bilibiliWebInterfaceData struct {\n\tBvid         string       `json:\"bvid\"`\n\tSubtitleInfo subtitleInfo `json:\"subtitle\"`\n}\n\ntype bilibiliWebInterface struct {\n\tCode int                      `json:\"code\"`\n\tData bilibiliWebInterfaceData `json:\"data\"`\n}\n\ntype festival struct {\n\tVideoSections []struct {\n\t\tId    int64  `json:\"id\"`\n\t\tTitle string `json:\"title\"`\n\t\tType  int    `json:\"type\"`\n\t} `json:\"videoSections\"`\n\tEpisodes  []episode `json:\"episodes\"`\n\tVideoInfo struct {\n\t\tAid   int    `json:\"aid\"`\n\t\tBVid  string `json:\"bvid\"`\n\t\tCid   int    `json:\"cid\"`\n\t\tTitle string `json:\"title\"`\n\t\tDesc  string `json:\"desc\"`\n\t\tPages []struct {\n\t\t\tCid       int    `json:\"cid\"`\n\t\t\tDuration  int    `json:\"duration\"`\n\t\t\tPage      int    `json:\"page\"`\n\t\t\tPart      string `json:\"part\"`\n\t\t\tDimension struct {\n\t\t\t\tWidth  int `json:\"width\"`\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tRotate int `json:\"rotate\"`\n\t\t\t} `json:\"dimension\"`\n\t\t} `json:\"pages\"`\n\t} `json:\"videoInfo\"`\n}\n\n// Path: extractors/douyin/douyin.go\npackage douyin\n\nimport (\n\t\"crypto/rand\"\n\t_ \"embed\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"douyin\", e)\n\textractors.Register(\"iesdouyin\", e)\n}\n\n//go:embed sign.js\nvar script string\n\ntype extractor struct{}\n\n// New returns a douyin extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tif strings.Contains(url, \"v.douyin.com\") {\n\t\treq, err := http.NewRequest(\"GET\", url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tc := http.Client{\n\t\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\t\treturn http.ErrUseLastResponse\n\t\t\t},\n\t\t}\n\t\tresp, err := c.Do(req)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tdefer resp.Body.Close() // nolint\n\t\turl = resp.Header.Get(\"location\")\n\t}\n\n\titemIds := utils.MatchOneOf(url, `/video/(\\d+)`)\n\tif len(itemIds) == 0 {\n\t\treturn nil, errors.New(\"unable to get video ID\")\n\t}\n\tif itemIds == nil || len(itemIds) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\titemId := itemIds[len(itemIds)-1]\n\n\t// dynamic generate cookie\n\tcookie, err := createCookie()\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tapi := \"https://www.douyin.com/aweme/v1/web/aweme/detail/?aweme_id=\" + itemId\n\t// parse api query params string\n\tquery, err := netURL.Parse(api)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed)\n\t}\n\t// define request headers and sign agent\n\theaders := map[string]string{}\n\theaders[\"Cookie\"] = cookie\n\theaders[\"Referer\"] = \"https://www.douyin.com/\"\n\theaders[\"User-Agent\"] = \"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36 Edg/87.0.664.66\"\n\n\t// init JavaScripts runtime\n\tvm := goja.New()\n\t// load sign scripts\n\t_, _ = vm.RunString(script)\n\t// sign\n\tsign, err := vm.RunString(fmt.Sprintf(\"sign('%s', '%s')\", query.RawQuery, headers[\"User-Agent\"]))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tapi = fmt.Sprintf(\"%s&X-Bogus=%s\", api, sign)\n\n\tjsonData, err := request.Get(api, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar douyin douyinData\n\tif err = json.Unmarshal([]byte(jsonData), &douyin); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlData := make([]*extractors.Part, 0)\n\tvar douyinType extractors.DataType\n\tvar totalSize int64\n\t// AwemeType: 0:video 68:image\n\tif douyin.AwemeDetail.AwemeType == 68 {\n\t\tdouyinType = extractors.DataTypeImage\n\t\tfor _, img := range douyin.AwemeDetail.Images {\n\t\t\trealURL := img.URLList[len(img.URLList)-1]\n\t\t\tsize, err := request.Size(realURL, url)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\ttotalSize += size\n\t\t\t_, ext, err := utils.GetNameAndExt(realURL)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turlData = append(urlData, &extractors.Part{\n\t\t\t\tURL:  realURL,\n\t\t\t\tSize: size,\n\t\t\t\tExt:  ext,\n\t\t\t})\n\t\t}\n\t} else {\n\t\tdouyinType = extractors.DataTypeVideo\n\t\trealURL := douyin.AwemeDetail.Video.PlayAddr.URLList[0]\n\t\ttotalSize, err = request.Size(realURL, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData = append(urlData, &extractors.Part{\n\t\t\tURL:  realURL,\n\t\t\tSize: totalSize,\n\t\t\tExt:  \"mp4\",\n\t\t})\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urlData,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u6296\u97f3 douyin.com\",\n\t\t\tTitle:   douyin.AwemeDetail.Desc,\n\t\t\tType:    douyinType,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\nfunc createCookie() (string, error) {\n\tv1, err := msToken(107)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tv2, err := ttwid()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tv3 := \"324fb4ea4a89c0c05827e18a1ed9cf9bf8a17f7705fcc793fec935b637867e2a5a9b8168c885554d029919117a18ba69\"\n\tv4 := \"eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWNsaWVudC1jc3IiOiItLS0tLUJFR0lOIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLVxyXG5NSUlCRFRDQnRRSUJBREFuTVFzd0NRWURWUVFHRXdKRFRqRVlNQllHQTFVRUF3d1BZbVJmZEdsamEyVjBYMmQxXHJcbllYSmtNRmt3RXdZSEtvWkl6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVKUDZzbjNLRlFBNUROSEcyK2F4bXAwNG5cclxud1hBSTZDU1IyZW1sVUE5QTZ4aGQzbVlPUlI4NVRLZ2tXd1FJSmp3Nyszdnc0Z2NNRG5iOTRoS3MvSjFJc3FBc1xyXG5NQ29HQ1NxR1NJYjNEUUVKRGpFZE1Cc3dHUVlEVlIwUkJCSXdFSUlPZDNkM0xtUnZkWGxwYmk1amIyMHdDZ1lJXHJcbktvWkl6ajBFQXdJRFJ3QXdSQUlnVmJkWTI0c0RYS0c0S2h3WlBmOHpxVDRBU0ROamNUb2FFRi9MQnd2QS8xSUNcclxuSURiVmZCUk1PQVB5cWJkcytld1QwSDZqdDg1czZZTVNVZEo5Z2dmOWlmeTBcclxuLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tXHJcbiJ9\"\n\tcookie := fmt.Sprintf(\"msToken=%s;ttwid=%s;odin_tt=%s;bd_ticket_guard_client_data=%s;\", v1, v2, v3, v4)\n\treturn cookie, nil\n}\n\nfunc msToken(length int) (string, error) {\n\tconst characters = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\"\n\trandomBytes := make([]byte, length)\n\tif _, err := rand.Read(randomBytes); err != nil {\n\t\treturn \"\", err\n\t}\n\ttoken := make([]byte, length)\n\tfor i, b := range randomBytes {\n\t\ttoken[i] = characters[int(b)%len(characters)]\n\t}\n\treturn string(token), nil\n}\n\nfunc ttwid() (string, error) {\n\tbody := map[string]interface{}{\n\t\t\"aid\":           1768,\n\t\t\"union\":         true,\n\t\t\"needFid\":       false,\n\t\t\"region\":        \"cn\",\n\t\t\"cbUrlProtocol\": \"https\",\n\t\t\"service\":       \"www.ixigua.com\",\n\t\t\"migrate_info\":  map[string]string{\"ticket\": \"\", \"source\": \"node\"},\n\t}\n\tbytes, err := json.Marshal(body)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tpayload := strings.NewReader(string(bytes))\n\tresp, err := request.Request(http.MethodPost, \"https://ttwid.bytedance.com/ttwid/union/register/\", payload, nil)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer resp.Body.Close() // nolint\n\tcookie := resp.Header.Get(\"Set-Cookie\")\n\tre := regexp.MustCompile(`ttwid=([^;]+)`)\n\tif match := re.FindStringSubmatch(cookie); match != nil {\n\t\treturn match[1], nil\n\t}\n\treturn \"\", errors.New(\"douyin ttwid request failed\")\n}\n\n// Path: extractors/douyin/douyin_test.go\npackage douyin\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.douyin.com/video/6967223681286278436?previous_page=main_page&tab_name=home\",\n\t\t\t\tTitle: \"\u662f\u7231\u60c5\uff0c\u8ba9\u7236\u5b50\u76f8\u8ba4#\u9648\u7fd4\u516d\u70b9\u534a  #\u5173\u4e8e\u7231\u60c5\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://v.douyin.com/LvCYKvV\",\n\t\t\t\tTitle: \"\u9ed1\u53d1\u9650\u5b9a#\u5f00\u6625\u5fc5\u5907\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/douyin/types.go\npackage douyin\n\ntype douyinData struct {\n\tStatusCode  int `json:\"status_code\"`\n\tAwemeDetail struct {\n\t\tAdmireAuth struct {\n\t\t\tAdmireButton       int `json:\"admire_button\"`\n\t\t\tIsAdmire           int `json:\"is_admire\"`\n\t\t\tIsShowAdmireButton int `json:\"is_show_admire_button\"`\n\t\t\tIsShowAdmireTab    int `json:\"is_show_admire_tab\"`\n\t\t} `json:\"admire_auth\"`\n\t\tAnchors interface{} `json:\"anchors\"`\n\t\tAuthor  struct {\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tCfList          interface{} `json:\"cf_list\"`\n\t\t\tCloseFriendType int         `json:\"close_friend_type\"`\n\t\t\tContactsStatus  int         `json:\"contacts_status\"`\n\t\t\tContrailList    interface{} `json:\"contrail_list\"`\n\t\t\tCoverURL        []struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_url\"`\n\t\t\tCreateTime                             int         `json:\"create_time\"`\n\t\t\tCustomVerify                           string      `json:\"custom_verify\"`\n\t\t\tDataLabelList                          interface{} `json:\"data_label_list\"`\n\t\t\tEndorsementInfoList                    interface{} `json:\"endorsement_info_list\"`\n\t\t\tEnterpriseVerifyReason                 string      `json:\"enterprise_verify_reason\"`\n\t\t\tFavoritingCount                        int         `json:\"favoriting_count\"`\n\t\t\tFollowStatus                           int         `json:\"follow_status\"`\n\t\t\tFollowerCount                          int         `json:\"follower_count\"`\n\t\t\tFollowerListSecondaryInformationStruct interface{} `json:\"follower_list_secondary_information_struct\"`\n\t\t\tFollowerStatus                         int         `json:\"follower_status\"`\n\t\t\tFollowingCount                         int         `json:\"following_count\"`\n\t\t\tImRoleIds                              interface{} `json:\"im_role_ids\"`\n\t\t\tIsAdFake                               bool        `json:\"is_ad_fake\"`\n\t\t\tIsBlockedV2                            bool        `json:\"is_blocked_v2\"`\n\t\t\tIsBlockingV2                           bool        `json:\"is_blocking_v2\"`\n\t\t\tIsCf                                   int         `json:\"is_cf\"`\n\t\t\tMaxFollowerCount                       int         `json:\"max_follower_count\"`\n\t\t\tNickname                               string      `json:\"nickname\"`\n\t\t\tNotSeenItemIDList                      interface{} `json:\"not_seen_item_id_list\"`\n\t\t\tNotSeenItemIDListV2                    interface{} `json:\"not_seen_item_id_list_v2\"`\n\t\t\tOfflineInfoList                        interface{} `json:\"offline_info_list\"`\n\t\t\tPersonalTagList                        interface{} `json:\"personal_tag_list\"`\n\t\t\tPreventDownload                        bool        `json:\"prevent_download\"`\n...\n", "needle_token_start": 3709, "needle_token_end": 3859, "code_context_ntokens": 15227, "output": ["```go\nfunc getMultiPageData(html string) (*multiPage, error) {\n\tvar data multiPage\n\tmultiPageDataString := utils.MatchOneOf(\n\t\thtml, `window.__INITIAL_STATE__=(.+?);\\(function`,\n\t)\n\tif multiPageDataString == nil {\n\t\treturn &data, errors.New(\"this page has no playlist\")\n\t}\n\terr := json.Unmarshal([]byte(multiPageDataString[1]), &data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn &data, nil\n}\n```"]}
{"repo": "iawia002/lux", "name": "getExtFromMimeType", "language": "go", "path": "extractors/bilibili/bilibili.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to extract the file extension from a given MIME type string.\n2. **Input**: A single string representing a MIME type (e.g., \"video/mp4\").\n3. **Output**: A string representing the extracted file extension (e.g., \"mp4\").\n4. **Procedure**: The function splits the input MIME type string using the \"/\" delimiter. If the resulting array has two elements, it returns the second element as the file extension. If not, it defaults to returning \"mp4\".\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\t\t\"avid=%d&cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=2000\",\n\t\t\taid, cid, bvid, quality,\n\t\t)\n\t\tbaseAPIURL = bilibiliAPI\n\t}\n\tapi = baseAPIURL + params\n\t// bangumi utoken also need to put in params to sign, but the ordinary video doesn't need\n\tif !bangumi && utoken != \"\" {\n\t\tapi = fmt.Sprintf(\"%s&utoken=%s\", api, utoken)\n\t}\n\treturn api, nil\n}\n\ntype bilibiliOptions struct {\n\turl      string\n\thtml     string\n\tbangumi  bool\n\taid      int\n\tcid      int\n\tbvid     string\n\tpage     int\n\tsubtitle string\n}\n\nfunc extractBangumi(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tdataString := utils.MatchOneOf(html, `<script\\s+id=\"__NEXT_DATA__\"\\s+type=\"application/json\"\\s*>(.*?)</script\\s*>`)[1]\n\tepArrayString := utils.MatchOneOf(dataString, `\"episodes\"\\s*:\\s*(.+?)\\s*,\\s*\"user_status\"`)[1]\n\tfullVideoIdString := utils.MatchOneOf(dataString, `\"videoId\"\\s*:\\s*\"(ep|ss)(\\d+)\"`)\n\tepSsString := fullVideoIdString[1] // \"ep\" or \"ss\"\n\tvideoIdString := fullVideoIdString[2]\n\n\tvar epArray []json.RawMessage\n\terr := json.Unmarshal([]byte(epArrayString), &epArray)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar data bangumiData\n\tfor _, jsonByte := range epArray {\n\t\tvar epInfo bangumiEpData\n\t\terr := json.Unmarshal(jsonByte, &epInfo)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tvideoId, err := strconv.ParseInt(videoIdString, 10, 0)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tif epInfo.ID == int(videoId) || (epSsString == \"ss\" && epInfo.TitleFormat == \"\u7b2c1\u8bdd\") {\n\t\t\tdata.EpInfo = epInfo\n\t\t}\n\t\tdata.EpList = append(data.EpList, epInfo)\n\t}\n\n\tsort.Slice(data.EpList, func(i, j int) bool {\n\t\treturn data.EpList[i].EpID < data.EpList[j].EpID\n\t})\n\n\tif !extractOption.Playlist {\n\t\taid := data.EpInfo.Aid\n\t\tcid := data.EpInfo.Cid\n\t\tbvid := data.EpInfo.BVid\n\t\ttitleFormat := data.EpInfo.TitleFormat\n\t\tlongTitle := data.EpInfo.LongTitle\n\t\tif aid <= 0 || cid <= 0 || bvid == \"\" {\n\t\t\taid = data.EpList[0].Aid\n\t\t\tcid = data.EpList[0].Cid\n\t\t\tbvid = data.EpList[0].BVid\n\t\t\ttitleFormat = data.EpList[0].TitleFormat\n\t\t\tlongTitle = data.EpList[0].LongTitle\n\t\t}\n\t\toptions := bilibiliOptions{\n\t\t\turl:     url,\n\t\t\thtml:    html,\n\t\t\tbangumi: true,\n\t\t\taid:     aid,\n\t\t\tcid:     cid,\n\t\t\tbvid:    bvid,\n\n\t\t\tsubtitle: fmt.Sprintf(\"%s %s\", titleFormat, longTitle),\n\t\t}\n\t\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n\t}\n\n\t// handle bangumi playlist\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(data.EpList))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range data.EpList {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\tid := u.EpID\n\t\tif id == 0 {\n\t\t\tid = u.ID\n\t\t}\n\t\t// html content can't be reused here\n\t\toptions := bilibiliOptions{\n\t\t\turl:     fmt.Sprintf(\"https://www.bilibili.com/bangumi/play/ep%d\", id),\n\t\t\tbangumi: true,\n\t\t\taid:     u.Aid,\n\t\t\tcid:     u.Cid,\n\t\t\tbvid:    u.BVid,\n\n\t\t\tsubtitle: fmt.Sprintf(\"%s %s\", u.TitleFormat, u.LongTitle),\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\nfunc getMultiPageData(html string) (*multiPage, error) {\n\tvar data multiPage\n\tmultiPageDataString := utils.MatchOneOf(\n\t\thtml, `window.__INITIAL_STATE__=(.+?);\\(function`,\n\t)\n\tif multiPageDataString == nil {\n\t\treturn &data, errors.New(\"this page has no playlist\")\n\t}\n\terr := json.Unmarshal([]byte(multiPageDataString[1]), &data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn &data, nil\n}\n\nfunc extractFestival(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tmatches := utils.MatchAll(html, \"<\\\\s*script[^>]*>\\\\s*window\\\\.__INITIAL_STATE__=([\\\\s\\\\S]*?);\\\\s?\\\\(function[\\\\s\\\\S]*?<\\\\/\\\\s*script\\\\s*>\")\n\tif len(matches) < 1 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tif len(matches[0]) < 2 {\n\t\treturn nil, errors.New(\"could not find video in page\")\n\t}\n\n\tvar festivalData festival\n\terr := json.Unmarshal([]byte(matches[0][1]), &festivalData)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\toptions := bilibiliOptions{\n\t\turl:  url,\n\t\thtml: html,\n\t\taid:  festivalData.VideoInfo.Aid,\n\t\tbvid: festivalData.VideoInfo.BVid,\n\t\tcid:  festivalData.VideoInfo.Cid,\n\t\tpage: 0,\n\t}\n\n\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n}\n\nfunc extractNormalVideo(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) {\n\tpageData, err := getMultiPageData(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif !extractOption.Playlist {\n\t\t// handle URL that has a playlist, mainly for unified titles\n\t\t// <h1> tag does not include subtitles\n\t\t// bangumi doesn't need this\n\t\tpageString := utils.MatchOneOf(url, `\\?p=(\\d+)`)\n\t\tvar p int\n\t\tif pageString == nil {\n\t\t\t// https://www.bilibili.com/video/av20827366/\n\t\t\tp = 1\n\t\t} else {\n\t\t\t// https://www.bilibili.com/video/av20827366/?p=2\n\t\t\tp, _ = strconv.Atoi(pageString[1])\n\t\t}\n\n\t\tif len(pageData.VideoData.Pages) < p || p < 1 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\n\t\tpage := pageData.VideoData.Pages[p-1]\n\t\toptions := bilibiliOptions{\n\t\t\turl:  url,\n\t\t\thtml: html,\n\t\t\taid:  pageData.Aid,\n\t\t\tbvid: pageData.BVid,\n\t\t\tcid:  page.Cid,\n\t\t\tpage: p,\n\t\t}\n\t\t// \"part\":\"\" or \"part\":\"Untitled\"\n\t\tif page.Part == \"Untitled\" || len(pageData.VideoData.Pages) == 1 {\n\t\t\toptions.subtitle = \"\"\n\t\t} else {\n\t\t\toptions.subtitle = page.Part\n\t\t}\n\t\treturn []*extractors.Data{bilibiliDownload(options, extractOption)}, nil\n\t}\n\n\t// handle normal video playlist\n\tif len(pageData.Sections) == 0 {\n\t\t// https://www.bilibili.com/video/av20827366/?p=* each video in playlist has different p=?\n\t\treturn multiPageDownload(url, html, extractOption, pageData)\n\t}\n\t// handle another kind of playlist\n\t// https://www.bilibili.com/video/av*** each video in playlist has different av/bv id\n\treturn multiEpisodeDownload(url, html, extractOption, pageData)\n}\n\n// handle multi episode download\nfunc multiEpisodeDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) {\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.Sections[0].Episodes))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range pageData.Sections[0].Episodes {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\toptions := bilibiliOptions{\n\t\t\turl:      url,\n\t\t\thtml:     html,\n\t\t\taid:      u.Aid,\n\t\t\tbvid:     u.BVid,\n\t\t\tcid:      u.Cid,\n\t\t\tsubtitle: fmt.Sprintf(\"%s P%d\", u.Title, index+1),\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\n// handle multi page download\nfunc multiPageDownload(url, html string, extractOption extractors.Options, pageData *multiPage) ([]*extractors.Data, error) {\n\tneedDownloadItems := utils.NeedDownloadList(extractOption.Items, extractOption.ItemStart, extractOption.ItemEnd, len(pageData.VideoData.Pages))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(extractOption.ThreadNumber)\n\tdataIndex := 0\n\tfor index, u := range pageData.VideoData.Pages {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\t\twgp.Add()\n\t\toptions := bilibiliOptions{\n\t\t\turl:      url,\n\t\t\thtml:     html,\n\t\t\taid:      pageData.Aid,\n\t\t\tbvid:     pageData.BVid,\n\t\t\tcid:      u.Cid,\n\t\t\tsubtitle: u.Part,\n\t\t\tpage:     u.Page,\n\t\t}\n\t\tgo func(index int, options bilibiliOptions, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\textractedData[index] = bilibiliDownload(options, extractOption)\n\t\t}(dataIndex, options, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\ntype extractor struct{}\n\n// New returns a bilibili extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\thtml, err := request.Get(url, referer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// set thread number to 1 manually to avoid http 412 error\n\toption.ThreadNumber = 1\n\n\tif strings.Contains(url, \"bangumi\") {\n\t\t// handle bangumi\n\t\treturn extractBangumi(url, html, option)\n\t} else if strings.Contains(url, \"festival\") {\n\t\treturn extractFestival(url, html, option)\n\t} else {\n\t\t// handle normal video\n\t\treturn extractNormalVideo(url, html, option)\n\t}\n}\n\n// bilibiliDownload is the download function for a single URL\nfunc bilibiliDownload(options bilibiliOptions, extractOption extractors.Options) *extractors.Data {\n\tvar (\n\t\terr  error\n\t\thtml string\n\t)\n\tif options.html != \"\" {\n\t\t// reuse html string, but this can't be reused in case of playlist\n\t\thtml = options.html\n\t} else {\n\t\thtml, err = request.Get(options.url, referer, nil)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t}\n\n\t// Get \"accept_quality\" and \"accept_description\"\n\t// \"accept_description\":[\"\u8d85\u9ad8\u6e05 8K\",\"\u8d85\u6e05 4K\",\"\u9ad8\u6e05 1080P+\",\"\u9ad8\u6e05 1080P\",\"\u9ad8\u6e05 720P\",\"\u6e05\u6670 480P\",\"\u6d41\u7545 360P\"],\n\t// \"accept_quality\":[127\uff0c120,112,80,48,32,16],\n\tapi, err := genAPI(options.aid, options.cid, 127, options.bvid, options.bangumi, extractOption.Cookie)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\tjsonString, err := request.Get(api, referer, nil)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\n\tvar data dash\n\terr = json.Unmarshal([]byte(jsonString), &data)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\tvar dashData dashInfo\n\tif data.Data.Description == nil {\n\t\tdashData = data.Result\n\t} else {\n\t\tdashData = data.Data\n\t}\n\n\tvar audioPart *extractors.Part\n\tif dashData.Streams.Audio != nil {\n\t\t// Get audio part\n\t\tvar audioID int\n\t\taudios := map[int]string{}\n\t\tbandwidth := 0\n\t\tfor _, stream := range dashData.Streams.Audio {\n\t\t\tif stream.Bandwidth > bandwidth {\n\t\t\t\taudioID = stream.ID\n\t\t\t\tbandwidth = stream.Bandwidth\n\t\t\t}\n\t\t\taudios[stream.ID] = stream.BaseURL\n\t\t}\n\t\ts, err := request.Size(audios[audioID], referer)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t\taudioPart = &extractors.Part{\n\t\t\tURL:  audios[audioID],\n\t\t\tSize: s,\n\t\t\tExt:  \"m4a\",\n\t\t}\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, len(dashData.Quality))\n\tfor _, stream := range dashData.Streams.Video {\n\t\ts, err := request.Size(stream.BaseURL, referer)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(options.url, err)\n\t\t}\n\t\tparts := make([]*extractors.Part, 0, 2)\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL:  stream.BaseURL,\n\t\t\tSize: s,\n\t\t\tExt:  getExtFromMimeType(stream.MimeType),\n\t\t})\n\t\tif audioPart != nil {\n\t\t\tparts = append(parts, audioPart)\n\t\t}\n\t\tvar size int64\n\t\tfor _, part := range parts {\n\t\t\tsize += part.Size\n\t\t}\n\t\tid := fmt.Sprintf(\"%d-%d\", stream.ID, stream.Codecid)\n\t\tstreams[id] = &extractors.Stream{\n\t\t\tParts:   parts,\n\t\t\tSize:    size,\n\t\t\tQuality: fmt.Sprintf(\"%s %s\", qualityString[stream.ID], stream.Codecs),\n\t\t}\n\t\tif audioPart != nil {\n\t\t\tstreams[id].NeedMux = true\n\t\t}\n\t}\n\n\tfor _, durl := range dashData.DURLs {\n\t\tvar ext string\n\t\tswitch dashData.DURLFormat {\n\t\tcase \"flv\", \"flv480\":\n\t\t\text = \"flv\"\n\t\tcase \"mp4\", \"hdmp4\": // nolint\n\t\t\text = \"mp4\"\n\t\t}\n\n\t\tparts := make([]*extractors.Part, 0, 1)\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL:  durl.URL,\n\t\t\tSize: durl.Size,\n\t\t\tExt:  ext,\n\t\t})\n\n\t\tstreams[strconv.Itoa(dashData.CurQuality)] = &extractors.Stream{\n\t\t\tParts:   parts,\n\t\t\tSize:    durl.Size,\n\t\t\tQuality: qualityString[dashData.CurQuality],\n\t\t}\n\t}\n\n\t// get the title\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn extractors.EmptyData(options.url, err)\n\t}\n\ttitle := parser.Title(doc)\n\tif options.subtitle != \"\" {\n\t\tpageString := \"\"\n\t\tif options.page > 0 {\n\t\t\tpageString = fmt.Sprintf(\"P%d \", options.page)\n\t\t}\n\t\tif extractOption.EpisodeTitleOnly {\n\t\t\ttitle = fmt.Sprintf(\"%s%s\", pageString, options.subtitle)\n\t\t} else {\n\t\t\ttitle = fmt.Sprintf(\"%s %s%s\", title, pageString, options.subtitle)\n\t\t}\n\t}\n\n\treturn &extractors.Data{\n\t\tSite:    \"\u54d4\u54e9\u54d4\u54e9 bilibili.com\",\n\t\tTitle:   title,\n\t\tType:    extractors.DataTypeVideo,\n\t\tStreams: streams,\n\t\tCaptions: map[string]*extractors.CaptionPart{\n\t\t\t\"danmaku\": {\n\t\t\t\tPart: extractors.Part{\n\t\t\t\t\tURL: fmt.Sprintf(\"https://comment.bilibili.com/%d.xml\", options.cid),\n\t\t\t\t\tExt: \"xml\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t\"subtitle\": getSubTitleCaptionPart(options.aid, options.cid),\n\t\t},\n\t\tURL: options.url,\n\t}\n}\n\nfunc getExtFromMimeType(mimeType string) str\ning {\n\texts := strings.Split(mimeType, \"/\")\n\tif len(exts) == 2 {\n\t\treturn exts[1]\n\t}\n\treturn \"mp4\"\n}\n\nfunc getSubTitleCaptionPart(aid int, cid int) *extractors.CaptionPart {\n\tjsonString, err := request.Get(\n\t\tfmt.Sprintf(\"http://api.bilibili.com/x/player/wbi/v2?aid=%d&cid=%d\", aid, cid), referer, nil,\n\t)\n\tif err != nil {\n\t\treturn nil\n\t}\n\tstu := bilibiliWebInterface{}\n\terr = json.Unmarshal([]byte(jsonString), &stu)\n\tif err != nil || len(stu.Data.SubtitleInfo.SubtitleList) == 0 {\n\t\treturn nil\n\t}\n\treturn &extractors.CaptionPart{\n\t\tPart: extractors.Part{\n\t\t\tURL: fmt.Sprintf(\"https:%s\", stu.Data.SubtitleInfo.SubtitleList[0].SubtitleUrl),\n\t\t\tExt: \"srt\",\n\t\t},\n\t\tTransform: subtitleTransform,\n\t}\n}\n\nfunc subtitleTransform(body []byte) ([]byte, error) {\n\tbytes := \"\"\n\tcaptionData := bilibiliSubtitleFormat{}\n\terr := json.Unmarshal(body, &captionData)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tfor i := 0; i < len(captionData.Body); i++ {\n\t\tbytes += fmt.Sprintf(\"%d\\n%s --> %s\\n%s\\n\\n\",\n\t\t\ti,\n\t\t\ttime.Unix(0, int64(captionData.Body[i].From*1000)*int64(time.Millisecond)).UTC().Format(\"15:04:05.000\"),\n\t\t\ttime.Unix(0, int64(captionData.Body[i].To*1000)*int64(time.Millisecond)).UTC().Format(\"15:04:05.000\"),\n\t\t\tcaptionData.Body[i].Content,\n\t\t)\n\t}\n\treturn []byte(bytes), nil\n}\n\n// Path: extractors/bilibili/bilibili_test.go\npackage bilibili\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestBilibili(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\targs     test.Args\n\t\tplaylist bool\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/av20203945/\",\n\t\t\t\tTitle: \"\u30102018\u62dc\u5e74\u796d\u5355\u54c1\u3011\u76f8\u9047day by day\",\n\t\t\t},\n\t\t\tplaylist: false,\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/av41301960\",\n\t\t\t\tTitle: \"\u3010\u82f1\u96c4\u8054\u76df\u30112019\u8d5b\u5b63CG \u300a\u89c9\u9192\u300b\",\n\t\t\t},\n\t\t\tplaylist: false,\n\t\t},\n\t\t{\n\t\t\tname: \"bangumi test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/bangumi/play/ep167000\",\n\t\t\t\tTitle: \"\u72d0\u5996\u5c0f\u7ea2\u5a18 \u7b2c70\u8bdd \u82cf\u82cf\u667a\u5546\u4e0a\u7ebf\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"bangumi playlist test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/bangumi/play/ss5050\",\n\t\t\t\tTitle: \"\u4e00\u4eba\u4e4b\u4e0b\uff1a\u7b2c1\u8bdd \u5f02\u4eba\u5200\u5175\u8d77\uff0c\u9053\u7081\u643a\u9634\u9633\",\n\t\t\t},\n\t\t\tplaylist: true,\n\t\t},\n\t\t{\n\t\t\tname: \"playlist test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/av16907446/\",\n\t\t\t\tTitle: \"\\\"\u4e0d\u8981\u76f8\u4fe1\u6b4c\u8bcd\uff0c\u4ed6\u4eec\u4e3a\u4e86\u62bc\u97f5\u4ec0\u4e48\u90fd\u5e72\u5f97\u51fa\u6765\\\"\",\n\t\t\t},\n\t\t\tplaylist: true,\n\t\t},\n\t\t{\n\t\t\tname: \"8k test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/video/BV1qM4y1w716\",\n\t\t\t\tTitle: \"\u30108K\u6f14\u793a\u7247\u3011B\u7ad9\u9996\u53d1\uff01\u4f60\u7684\u8bbe\u5907\u8fd8\u9876\u5f97\u4f4f\u5417\uff1f\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"b23 test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://b23.tv/Fc9i7QF\",\n\t\t\t\tTitle: \"\u3010\u5341\u5e74\u699c\u30112000-2009\u5e74\u6700\u5f3a\u534e\u8bed\u91d1\u66f2TOP100 P1 100\u7231\u8f6c\u89d2-\u7f57\u5fd7\u7965\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"festival test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.bilibili.com/festival/lty10th?bvid=BV1dZ4y1Y7bt\",\n\t\t\t\tTitle: \"\u6d1b\u5929\u4f9d\u5341\u5468\u5e74\u5b98\u65b9\u6f14\u5531\u4f1a\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar (\n\t\t\t\tdata []*extractors.Data\n\t\t\t\terr  error\n\t\t\t)\n\t\t\tif tt.playlist {\n\t\t\t\t// for playlist, we don't check the data\n\t\t\t\t_, err = New().Extract(tt.args.URL, extractors.Options{\n\t\t\t\t\tPlaylist:     true,\n\t\t\t\t\tThreadNumber: 9,\n\t\t\t\t})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t} else {\n\t\t\t\tdata, err = New().Extract(tt.args.URL, extractors.Options{})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t\ttest.Check(t, tt.args, data[0])\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: extractors/bilibili/types.go\npackage bilibili\n\n// {\"code\":0,\"message\":\"0\",\"ttl\":1,\"data\":{\"token\":\"aaa\"}}\n// {\"code\":-101,\"message\":\"\u8d26\u53f7\u672a\u767b\u5f55\",\"ttl\":1}\ntype tokenData struct {\n\tToken string `json:\"token\"`\n}\n\ntype token struct {\n\tCode    int       `json:\"code\"`\n\tMessage string    `json:\"message\"`\n\tData    tokenData `json:\"data\"`\n}\n\ntype bangumiEpData struct {\n\tAid         int    `json:\"aid\"`\n\tCid         int    `json:\"cid\"`\n\tBVid        string `json:\"bvid\"`\n\tID          int    `json:\"id\"`\n\tEpID        int    `json:\"ep_id\"`\n\tTitleFormat string `json:\"titleFormat\"`\n\tLongTitle   string `json:\"long_title\"`\n}\n\ntype bangumiData struct {\n\tEpInfo bangumiEpData   `json:\"epInfo\"`\n\tEpList []bangumiEpData `json:\"epList\"`\n}\n\ntype videoPagesData struct {\n\tCid  int    `json:\"cid\"`\n\tPart string `json:\"part\"`\n\tPage int    `json:\"page\"`\n}\n\ntype multiPageVideoData struct {\n\tTitle string           `json:\"title\"`\n\tPages []videoPagesData `json:\"pages\"`\n}\n\ntype episode struct {\n\tAid   int    `json:\"aid\"`\n\tCid   int    `json:\"cid\"`\n\tTitle string `json:\"title\"`\n\tBVid  string `json:\"bvid\"`\n}\n\ntype multiEpisodeData struct {\n\tSeasionid int       `json:\"season_id\"`\n\tEpisodes  []episode `json:\"episodes\"`\n}\n\ntype multiPage struct {\n\tAid       int                `json:\"aid\"`\n\tBVid      string             `json:\"bvid\"`\n\tSections  []multiEpisodeData `json:\"sections\"`\n\tVideoData multiPageVideoData `json:\"videoData\"`\n}\n\ntype dashStream struct {\n\tID        int    `json:\"id\"`\n\tBaseURL   string `json:\"baseUrl\"`\n\tBandwidth int    `json:\"bandwidth\"`\n\tMimeType  string `json:\"mimeType\"`\n\tCodecid   int    `json:\"codecid\"`\n\tCodecs    string `json:\"codecs\"`\n}\n\ntype dashStreams struct {\n\tVideo []dashStream `json:\"video\"`\n\tAudio []dashStream `json:\"audio\"`\n}\n\ntype dashInfo struct {\n\tCurQuality  int         `json:\"quality\"`\n\tDescription []string    `json:\"accept_description\"`\n\tQuality     []int       `json:\"accept_quality\"`\n\tStreams     dashStreams `json:\"dash\"`\n\tDURLFormat  string      `json:\"format\"`\n\tDURLs       []dURL      `json:\"durl\"`\n}\n\ntype dURL struct {\n\tURL  string `json:\"url\"`\n\tSize int64  `json:\"size\"`\n}\n\ntype dash struct {\n\tCode    int      `json:\"code\"`\n\tMessage string   `json:\"message\"`\n\tData    dashInfo `json:\"data\"`\n\tResult  dashInfo `json:\"result\"`\n}\n\nvar qualityString = map[int]string{\n\t127: \"\u8d85\u9ad8\u6e05 8K\",\n\t120: \"\u8d85\u6e05 4K\",\n\t116: \"\u9ad8\u6e05 1080P60\",\n\t74:  \"\u9ad8\u6e05 720P60\",\n\t112: \"\u9ad8\u6e05 1080P+\",\n\t80:  \"\u9ad8\u6e05 1080P\",\n\t64:  \"\u9ad8\u6e05 720P\",\n\t48:  \"\u9ad8\u6e05 720P\",\n\t32:  \"\u6e05\u6670 480P\",\n\t16:  \"\u6d41\u7545 360P\",\n\t15:  \"\u6d41\u7545 360P\",\n}\n\ntype subtitleData struct {\n\tFrom     float32 `json:\"from\"`\n\tTo       float32 `json:\"to\"`\n\tLocation int     `json:\"location\"`\n\tContent  string  `json:\"content\"`\n}\n\ntype bilibiliSubtitleFormat struct {\n\tFontSize        float32        `json:\"font_size\"`\n\tFontColor       string         `json:\"font_color\"`\n\tBackgroundAlpha float32        `json:\"background_alpha\"`\n\tBackgroundColor string         `json:\"background_color\"`\n\tStroke          string         `json:\"Stroke\"`\n\tBody            []subtitleData `json:\"body\"`\n}\n\ntype subtitleProperty struct {\n\tID          int64  `json:\"id\"`\n\tLan         string `json:\"lan\"`\n\tLanDoc      string `json:\"lan_doc\"`\n\tSubtitleUrl string `json:\"subtitle_url\"`\n}\n\ntype subtitleInfo struct {\n\tAllowSubmit  bool               `json:\"allow_submit\"`\n\tSubtitleList []subtitleProperty `json:\"subtitles\"`\n}\n\ntype bilibiliWebInterfaceData struct {\n\tBvid         string       `json:\"bvid\"`\n\tSubtitleInfo subtitleInfo `json:\"subtitle\"`\n}\n\ntype bilibiliWebInterface struct {\n\tCode int                      `json:\"code\"`\n\tData bilibiliWebInterfaceData `json:\"data\"`\n}\n\ntype festival struct {\n\tVideoSections []struct {\n\t\tId    int64  `json:\"id\"`\n\t\tTitle string `json:\"title\"`\n\t\tType  int    `json:\"type\"`\n\t} `json:\"videoSections\"`\n\tEpisodes  []episode `json:\"episodes\"`\n\tVideoInfo struct {\n\t\tAid   int    `json:\"aid\"`\n\t\tBVid  string `json:\"bvid\"`\n\t\tCid   int    `json:\"cid\"`\n\t\tTitle string `json:\"title\"`\n\t\tDesc  string `json:\"desc\"`\n\t\tPages []struct {\n\t\t\tCid       int    `json:\"cid\"`\n\t\t\tDuration  int    `json:\"duration\"`\n\t\t\tPage      int    `json:\"page\"`\n\t\t\tPart      string `json:\"part\"`\n\t\t\tDimension struct {\n\t\t\t\tWidth  int `json:\"width\"`\n\t\t\t\tHeight int `json:\"height\"`\n\t\t\t\tRotate int `json:\"rotate\"`\n\t\t\t} `json:\"dimension\"`\n\t\t} `json:\"pages\"`\n\t} `json:\"videoInfo\"`\n}\n\n// Path: extractors/douyin/douyin.go\npackage douyin\n\nimport (\n\t\"crypto/rand\"\n\t_ \"embed\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/dop251/goja\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"douyin\", e)\n\textractors.Register(\"iesdouyin\", e)\n}\n\n//go:embed sign.js\nvar script string\n\ntype extractor struct{}\n\n// New returns a douyin extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tif strings.Contains(url, \"v.douyin.com\") {\n\t\treq, err := http.NewRequest(\"GET\", url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tc := http.Client{\n\t\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\t\treturn http.ErrUseLastResponse\n\t\t\t},\n\t\t}\n\t\tresp, err := c.Do(req)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tdefer resp.Body.Close() // nolint\n\t\turl = resp.Header.Get(\"location\")\n\t}\n\n\titemIds := utils.MatchOneOf(url, `/video/(\\d+)`)\n\tif len(itemIds) == 0 {\n\t\treturn nil, errors.New(\"unable to get video ID\")\n\t}\n\tif itemIds == nil || len(itemIds) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\titemId := itemIds[len(itemIds)-1]\n\n\t// dynamic generate cookie\n\tcookie, err := createCookie()\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tapi := \"https://www.douyin.com/aweme/v1/web/aweme/detail/?aweme_id=\" + itemId\n\t// parse api query params string\n\tquery, err := netURL.Parse(api)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed)\n\t}\n\t// define request headers and sign agent\n\theaders := map[string]string{}\n\theaders[\"Cookie\"] = cookie\n\theaders[\"Referer\"] = \"https://www.douyin.com/\"\n\theaders[\"User-Agent\"] = \"Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36 Edg/87.0.664.66\"\n\n\t// init JavaScripts runtime\n\tvm := goja.New()\n\t// load sign scripts\n\t_, _ = vm.RunString(script)\n\t// sign\n\tsign, err := vm.RunString(fmt.Sprintf(\"sign('%s', '%s')\", query.RawQuery, headers[\"User-Agent\"]))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tapi = fmt.Sprintf(\"%s&X-Bogus=%s\", api, sign)\n\n\tjsonData, err := request.Get(api, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar douyin douyinData\n\tif err = json.Unmarshal([]byte(jsonData), &douyin); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlData := make([]*extractors.Part, 0)\n\tvar douyinType extractors.DataType\n\tvar totalSize int64\n\t// AwemeType: 0:video 68:image\n\tif douyin.AwemeDetail.AwemeType == 68 {\n\t\tdouyinType = extractors.DataTypeImage\n\t\tfor _, img := range douyin.AwemeDetail.Images {\n\t\t\trealURL := img.URLList[len(img.URLList)-1]\n\t\t\tsize, err := request.Size(realURL, url)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\ttotalSize += size\n\t\t\t_, ext, err := utils.GetNameAndExt(realURL)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turlData = append(urlData, &extractors.Part{\n\t\t\t\tURL:  realURL,\n\t\t\t\tSize: size,\n\t\t\t\tExt:  ext,\n\t\t\t})\n\t\t}\n\t} else {\n\t\tdouyinType = extractors.DataTypeVideo\n\t\trealURL := douyin.AwemeDetail.Video.PlayAddr.URLList[0]\n\t\ttotalSize, err = request.Size(realURL, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData = append(urlData, &extractors.Part{\n\t\t\tURL:  realURL,\n\t\t\tSize: totalSize,\n\t\t\tExt:  \"mp4\",\n\t\t})\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urlData,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u6296\u97f3 douyin.com\",\n\t\t\tTitle:   douyin.AwemeDetail.Desc,\n\t\t\tType:    douyinType,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\nfunc createCookie() (string, error) {\n\tv1, err := msToken(107)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tv2, err := ttwid()\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tv3 := \"324fb4ea4a89c0c05827e18a1ed9cf9bf8a17f7705fcc793fec935b637867e2a5a9b8168c885554d029919117a18ba69\"\n\tv4 := \"eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWNsaWVudC1jc3IiOiItLS0tLUJFR0lOIENFUlRJRklDQVRFIFJFUVVFU1QtLS0tLVxyXG5NSUlCRFRDQnRRSUJBREFuTVFzd0NRWURWUVFHRXdKRFRqRVlNQllHQTFVRUF3d1BZbVJmZEdsamEyVjBYMmQxXHJcbllYSmtNRmt3RXdZSEtvWkl6ajBDQVFZSUtvWkl6ajBEQVFjRFFnQUVKUDZzbjNLRlFBNUROSEcyK2F4bXAwNG5cclxud1hBSTZDU1IyZW1sVUE5QTZ4aGQzbVlPUlI4NVRLZ2tXd1FJSmp3Nyszdnc0Z2NNRG5iOTRoS3MvSjFJc3FBc1xyXG5NQ29HQ1NxR1NJYjNEUUVKRGpFZE1Cc3dHUVlEVlIwUkJCSXdFSUlPZDNkM0xtUnZkWGxwYmk1amIyMHdDZ1lJXHJcbktvWkl6ajBFQXdJRFJ3QXdSQUlnVmJkWTI0c0RYS0c0S2h3WlBmOHpxVDRBU0ROamNUb2FFRi9MQnd2QS8xSUNcclxuSURiVmZCUk1PQVB5cWJkcytld1QwSDZqdDg1czZZTVNVZEo5Z2dmOWlmeTBcclxuLS0tLS1FTkQgQ0VSVElGSUNBVEUgUkVRVUVTVC0tLS0tXHJcbiJ9\"\n\tcookie := fmt.Sprintf(\"msToken=%s;ttwid=%s;odin_tt=%s;bd_ticket_guard_client_data=%s;\", v1, v2, v3, v4)\n\treturn cookie, nil\n}\n\nfunc msToken(length int) (string, error) {\n\tconst characters = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\"\n\trandomBytes := make([]byte, length)\n\tif _, err := rand.Read(randomBytes); err != nil {\n\t\treturn \"\", err\n\t}\n\ttoken := make([]byte, length)\n\tfor i, b := range randomBytes {\n\t\ttoken[i] = characters[int(b)%len(characters)]\n\t}\n\treturn string(token), nil\n}\n\nfunc ttwid() (string, error) {\n\tbody := map[string]interface{}{\n\t\t\"aid\":           1768,\n\t\t\"union\":         true,\n\t\t\"needFid\":       false,\n\t\t\"region\":        \"cn\",\n\t\t\"cbUrlProtocol\": \"https\",\n\t\t\"service\":       \"www.ixigua.com\",\n\t\t\"migrate_info\":  map[string]string{\"ticket\": \"\", \"source\": \"node\"},\n\t}\n\tbytes, err := json.Marshal(body)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tpayload := strings.NewReader(string(bytes))\n\tresp, err := request.Request(http.MethodPost, \"https://ttwid.bytedance.com/ttwid/union/register/\", payload, nil)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer resp.Body.Close() // nolint\n\tcookie := resp.Header.Get(\"Set-Cookie\")\n\tre := regexp.MustCompile(`ttwid=([^;]+)`)\n\tif match := re.FindStringSubmatch(cookie); match != nil {\n\t\treturn match[1], nil\n\t}\n\treturn \"\", errors.New(\"douyin ttwid request failed\")\n}\n\n// Path: extractors/douyin/douyin_test.go\npackage douyin\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.douyin.com/video/6967223681286278436?previous_page=main_page&tab_name=home\",\n\t\t\t\tTitle: \"\u662f\u7231\u60c5\uff0c\u8ba9\u7236\u5b50\u76f8\u8ba4#\u9648\u7fd4\u516d\u70b9\u534a  #\u5173\u4e8e\u7231\u60c5\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://v.douyin.com/LvCYKvV\",\n\t\t\t\tTitle: \"\u9ed1\u53d1\u9650\u5b9a#\u5f00\u6625\u5fc5\u5907\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/douyin/types.go\npackage douyin\n\ntype douyinData struct {\n\tStatusCode  int `json:\"status_code\"`\n\tAwemeDetail struct {\n\t\tAdmireAuth struct {\n\t\t\tAdmireButton       int `json:\"admire_button\"`\n\t\t\tIsAdmire           int `json:\"is_admire\"`\n\t\t\tIsShowAdmireButton int `json:\"is_show_admire_button\"`\n\t\t\tIsShowAdmireTab    int `json:\"is_show_admire_tab\"`\n\t\t} `json:\"admire_auth\"`\n\t\tAnchors interface{} `json:\"anchors\"`\n\t\tAuthor  struct {\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tCfList          interface{} `json:\"cf_list\"`\n\t\t\tCloseFriendType int         `json:\"close_friend_type\"`\n\t\t\tContactsStatus  int         `json:\"contacts_status\"`\n\t\t\tContrailList    interface{} `json:\"contrail_list\"`\n\t\t\tCoverURL        []struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_url\"`\n\t\t\tCreateTime                             int         `json:\"create_time\"`\n\t\t\tCustomVerify                           string      `json:\"custom_verify\"`\n\t\t\tDataLabelList                          interface{} `json:\"data_label_list\"`\n\t\t\tEndorsementInfoList                    interface{} `json:\"endorsement_info_list\"`\n\t\t\tEnterpriseVerifyReason                 string      `json:\"enterprise_verify_reason\"`\n\t\t\tFavoritingCount                        int         `json:\"favoriting_count\"`\n\t\t\tFollowStatus                           int         `json:\"follow_status\"`\n\t\t\tFollowerCount                          int         `json:\"follower_count\"`\n\t\t\tFollowerListSecondaryInformationStruct interface{} `json:\"follower_list_secondary_information_struct\"`\n\t\t\tFollowerStatus                         int         `json:\"follower_status\"`\n\t\t\tFollowingCount                         int         `json:\"following_count\"`\n\t\t\tImRoleIds                              interface{} `json:\"im_role_ids\"`\n\t\t\tIsAdFake                               bool        `json:\"is_ad_fake\"`\n\t\t\tIsBlockedV2                            bool        `json:\"is_blocked_v2\"`\n\t\t\tIsBlockingV2                           bool        `json:\"is_blocking_v2\"`\n\t\t\tIsCf                                   int         `json:\"is_cf\"`\n\t\t\tMaxFollowerCount                       int         `json:\"max_follower_count\"`\n\t\t\tNickname                               string      `json:\"nickname\"`\n\t\t\tNotSeenItemIDList                      interface{} `json:\"not_seen_item_id_list\"`\n\t\t\tNotSeenItemIDListV2                    interface{} `json:\"not_seen_item_id_list_v2\"`\n\t\t\tOfflineInfoList                        interface{} `json:\"offline_info_list\"`\n\t\t\tPersonalTagList                        interface{} `json:\"personal_tag_list\"`\n\t\t\tPreventDownload                        bool        `json:\"prevent_download\"`\n\t\t\tRiskNoticeText                         string      `json:\"risk_notice_text\"`\n\t\t\tSecUID                                 string      `json:\"sec_uid\"`\n\t\t\tSecret                                 int         `json:\"secret\"`\n\t\t\tShareInfo                              struct {\n\t\t\t\tShareDesc      string `json:\"share_desc\"`\n\t\t\t\tShareDescInfo  string `json:\"share_desc_info\"`\n\t\t\t\tShareQrcodeURL struct {\n\t\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t\t} `json:\"share_qrcode_url\"`\n\t\t\t\tShareTitle       string `json:\"share_title\"`\n\t\t\t\tShareTitleMyself string `json:\"share_title_myself\"`\n\t\t\t\tShareTitleOther  string `json:\"share_title_other\"`\n\t\t\t\tShareURL         string `json:\"share_url\"`\n\t\t\t\tShareWeiboDesc   string `json:\"share_weibo_desc\"`\n\t\t\t} `json:\"share_info\"`\n\t\t\tShortID             string      `json:\"short_id\"`\n\t\t\tSignature           string      `json:\"signature\"`\n\t\t\tSignatureExtra      interface{} `json:\"signature_extra\"`\n\t\t\tSpecialPeopleLabels interface{} `json:\"special_people_labels\"`\n\t\t\tStatus              int         `json:\"status\"`\n\t\t\tTextExtra           interface{} `json:\"text_extra\"`\n\t\t\tTotalFavorited      int         `json:\"total_favorited\"`\n\t\t\tUID                 string      `json:\"uid\"`\n\t\t\tUniqueID            string      `json:\"unique_id\"`\n\t\t\tUserAge             int         `json:\"user_age\"`\n\t\t\tUserCanceled        bool        `json:\"user_canceled\"`\n\t\t\tUserPermissions     interface{} `json:\"user_permissions\"`\n\t\t\tVerificationType    int         `json:\"verification_type\"`\n\t\t} `json:\"author\"`\n\t\tAuthorMaskTag int   `json:\"author_mask_tag\"`\n\t\tAuthorUserID  int64 `json:\"author_user_id\"`\n\t\tAwemeACL      struct {\n\t\t\tDownloadMaskPanel struct {\n\t\t\t\tCode     int `json:\"code\"`\n\t\t\t\tShowType int `json:\"show_type\"`\n\t\t\t} `json:\"download_mask_panel\"`\n\t\t} `json:\"aweme_acl\"`\n\t\tAwemeControl struct {\n\t\t\tCanComment     bool `json:\"can_comment\"`\n\t\t\tCanForward     bool `json:\"can_forward\"`\n\t\t\tCanShare       bool `json:\"can_share\"`\n\t\t\tCanShowComment bool `json:\"can_show_comment\"`\n\t\t} `json:\"aweme_control\"`\n\t\tAwemeID               string      `json:\"aweme_id\"`\n\t\tAwemeType             int         `json:\"aweme_type\"`\n\t\tChallengePosition     interface{} `json:\"challenge_position\"`\n\t\tChapterList           interface{} `json:\"chapter_list\"`\n\t\tCollectStat           int         `json:\"collect_stat\"`\n\t\tCommentGid            int64       `json:\"comment_gid\"`\n\t\tCommentList           interface{} `json:\"comment_list\"`\n\t\tCommentPermissionInfo struct {\n\t\t\tCanComment              bool `json:\"can_comment\"`\n\t\t\tCommentPermissionStatus int  `json:\"comment_permission_status\"`\n\t\t\tItemDetailEntry         bool `json:\"item_detail_entry\"`\n\t\t\tPressEntry              bool `json:\"press_entry\"`\n\t\t\tToastGuide              bool `json:\"toast_guide\"`\n\t\t} `json:\"comment_permission_info\"`\n\t\tCommerceConfigData interface{} `json:\"commerce_config_data\"`\n\t\tCommonBarInfo      string      `json:\"common_bar_info\"`\n\t\tComponentInfoV2    string      `json:\"component_info_v2\"`\n\t\tCoverLabels        interface{} `json:\"cover_labels\"`\n\t\tCreateTime         int         `json:\"create_time\"`\n\t\tDesc               string      `json:\"desc\"`\n\t\tDiggLottie         struct {\n\t\t\tCanBomb  int    `json:\"can_bomb\"`\n\t\t\tLottieID string `json:\"lottie_id\"`\n\t\t} `json:\"digg_lottie\"`\n\t\tDisableRelationBar      int         `json:\"disable_relation_bar\"`\n\t\tDislikeDimensionList    interface{} `json:\"dislike_dimension_list\"`\n\t\tDuetAggregateInMusicTab bool        `json:\"duet_aggregate_in_music_tab\"`\n\t\tDuration                int         `json:\"duration\"`\n\t\tFeedCommentConfig       struct {\n\t\t\tAuthorAuditStatus int    `json:\"author_audit_status\"`\n\t\t\tInputConfigText   string `json:\"input_config_text\"`\n\t\t} `json:\"feed_comment_config\"`\n\t\tGeofencing          []interface{} `json:\"geofencing\"`\n\t\tGeofencingRegions   interface{}   `json:\"geofencing_regions\"`\n\t\tGroupID             string        `json:\"group_id\"`\n\t\tHybridLabel         interface{}   `json:\"hybrid_label\"`\n\t\tImageAlbumMusicInfo struct {\n\t\t\tBeginTime int `json:\"begin_time\"`\n\t\t\tEndTime   int `json:\"end_time\"`\n\t\t\tVolume    int `json:\"volume\"`\n\t\t} `json:\"image_album_music_info\"`\n\t\tImageInfos interface{} `json:\"image_infos\"`\n\t\tImageList  interface{} `json:\"image_list\"`\n\t\tImages     []struct {\n\t\t\tDownloadURLList []string    `json:\"download_url_list\"`\n\t\t\tHeight          int         `json:\"height\"`\n\t\t\tMaskURLList     interface{} `json:\"mask_url_list\"`\n\t\t\tURI             string      `json:\"uri\"`\n\t\t\tURLList         []string    `json:\"url_list\"`\n\t\t\tWidth           int         `json:\"width\"`\n\t\t} `json:\"images\"`\n\t\tImgBitrate []struct {\n\t\t\tImages []struct {\n\t\t\t\tDownloadURLList []string    `json:\"download_url_list\"`\n\t\t\t\tHeight          int         `json:\"height\"`\n\t\t\t\tMaskURLList     interface{} `json:\"mask_url_list\"`\n\t\t\t\tURI             string      `json:\"uri\"`\n\t\t\t\tURLList         []string    `json:\"url_list\"`\n\t\t\t\tWidth           int         `json:\"width\"`\n\t\t\t} `json:\"images\"`\n\t\t\tName string `json:\"name\"`\n\t\t} `json:\"img_bitrate\"`\n\t\tImpressionData struct {\n\t\t\tGroupIDListA   []int64     `json:\"group_id_list_a\"`\n\t\t\tGroupIDListB   []int64     `json:\"group_id_list_b\"`\n\t\t\tSimilarIDListA interface{} `json:\"similar_id_list_a\"`\n\t\t\tSimilarIDListB interface{} `json:\"similar_id_list_b\"`\n\t\t} `json:\"impression_data\"`\n\t\tInteractionStickers  interface{} `json:\"interaction_stickers\"`\n\t\tIsAds                bool        `json:\"is_ads\"`\n\t\tIsCollectsSelected   int         `json:\"is_collects_selected\"`\n\t\tIsDuetSing           bool        `json:\"is_duet_sing\"`\n\t\tIsImageBeat          bool        `json:\"is_image_beat\"`\n\t\tIsLifeItem           bool        `json:\"is_life_item\"`\n\t\tIsMultiContent       int         `json:\"is_multi_content\"`\n\t\tIsStory              int         `json:\"is_story\"`\n\t\tIsTop                int         `json:\"is_top\"`\n\t\tItemWarnNotification struct {\n\t\t\tContent string `json:\"content\"`\n\t\t\tShow    bool   `json:\"show\"`\n\t\t\tType    int    `json:\"type\"`\n\t\t} `json:\"item_warn_notification\"`\n\t\tLabelTopText interface{} `json:\"label_top_text\"`\n\t\tLongVideo    interface{} `json:\"long_video\"`\n\t\tMusic        struct {\n\t\t\tAlbum            string        `json:\"album\"`\n\t\t\tArtistUserInfos  interface{}   `json:\"artist_user_infos\"`\n\t\t\tArtists          []interface{} `json:\"artists\"`\n\t\t\tAuditionDuration int           `json:\"audition_duration\"`\n\t\t\tAuthor           string        `json:\"author\"`\n\t\t\tAuthorDeleted    bool          `json:\"author_deleted\"`\n\t\t\tAuthorPosition   interface{}   `json:\"author_position\"`\n\t\t\tAuthorStatus     int           `json:\"author_status\"`\n\t\t\tAvatarLarge      struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_large\"`\n\t\t\tAvatarMedium struct {\n...\n", "needle_token_start": 5267, "needle_token_end": 5334, "code_context_ntokens": 15262, "output": ["```go\nfunc getExtFromMimeType(mimeType string) string {\n\texts := strings.Split(mimeType, \"/\")\n\tif len(exts) == 2 {\n\t\treturn exts[1]\n\t}\n\treturn \"mp4\"\n}\n```"]}
{"repo": "iawia002/lux", "name": "getSrcMeta", "language": "go", "path": "extractors/eporner/eporner.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to parse a string containing video quality and size information, extracting these details and converting the size into bytes.\n2. **Input**: A single string that includes the video quality and size enclosed in parentheses, separated by a comma.\n3. **Output**: An object containing the video quality as a string, the size as a string, and the size in bytes as an integer.\n4. **Procedure**: \n   - The function first locates and extracts the substring within parentheses.\n   - It then splits this substring by a comma to separate the quality and size components.\n   - The size string is further split to separate the numerical value from its unit (KB, MB, GB).\n   - The numerical value is converted to a float and then multiplied by the appropriate factor based on its unit to convert it to bytes.\n   - If any errors occur during parsing or conversion, the size is set to zero.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/douyin/types.go\npackage douyin\n\ntype douyinData struct {\n\tStatusCode  int `json:\"status_code\"`\n\tAwemeDetail struct {\n\t\tAdmireAuth struct {\n\t\t\tAdmireButton       int `json:\"admire_button\"`\n\t\t\tIsAdmire           int `json:\"is_admire\"`\n\t\t\tIsShowAdmireButton int `json:\"is_show_admire_button\"`\n\t\t\tIsShowAdmireTab    int `json:\"is_show_admire_tab\"`\n\t\t} `json:\"admire_auth\"`\n\t\tAnchors interface{} `json:\"anchors\"`\n\t\tAuthor  struct {\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tCfList          interface{} `json:\"cf_list\"`\n\t\t\tCloseFriendType int         `json:\"close_friend_type\"`\n\t\t\tContactsStatus  int         `json:\"contacts_status\"`\n\t\t\tContrailList    interface{} `json:\"contrail_list\"`\n\t\t\tCoverURL        []struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_url\"`\n\t\t\tCreateTime                             int         `json:\"create_time\"`\n\t\t\tCustomVerify                           string      `json:\"custom_verify\"`\n\t\t\tDataLabelList                          interface{} `json:\"data_label_list\"`\n\t\t\tEndorsementInfoList                    interface{} `json:\"endorsement_info_list\"`\n\t\t\tEnterpriseVerifyReason                 string      `json:\"enterprise_verify_reason\"`\n\t\t\tFavoritingCount                        int         `json:\"favoriting_count\"`\n\t\t\tFollowStatus                           int         `json:\"follow_status\"`\n\t\t\tFollowerCount                          int         `json:\"follower_count\"`\n\t\t\tFollowerListSecondaryInformationStruct interface{} `json:\"follower_list_secondary_information_struct\"`\n\t\t\tFollowerStatus                         int         `json:\"follower_status\"`\n\t\t\tFollowingCount                         int         `json:\"following_count\"`\n\t\t\tImRoleIds                              interface{} `json:\"im_role_ids\"`\n\t\t\tIsAdFake                               bool        `json:\"is_ad_fake\"`\n\t\t\tIsBlockedV2                            bool        `json:\"is_blocked_v2\"`\n\t\t\tIsBlockingV2                           bool        `json:\"is_blocking_v2\"`\n\t\t\tIsCf                                   int         `json:\"is_cf\"`\n\t\t\tMaxFollowerCount                       int         `json:\"max_follower_count\"`\n\t\t\tNickname                               string      `json:\"nickname\"`\n\t\t\tNotSeenItemIDList                      interface{} `json:\"not_seen_item_id_list\"`\n\t\t\tNotSeenItemIDListV2                    interface{} `json:\"not_seen_item_id_list_v2\"`\n\t\t\tOfflineInfoList                        interface{} `json:\"offline_info_list\"`\n\t\t\tPersonalTagList                        interface{} `json:\"personal_tag_list\"`\n\t\t\tPreventDownload                        bool        `json:\"prevent_download\"`\n\t\t\tRiskNoticeText                         string      `json:\"risk_notice_text\"`\n\t\t\tSecUID                                 string      `json:\"sec_uid\"`\n\t\t\tSecret                                 int         `json:\"secret\"`\n\t\t\tShareInfo                              struct {\n\t\t\t\tShareDesc      string `json:\"share_desc\"`\n\t\t\t\tShareDescInfo  string `json:\"share_desc_info\"`\n\t\t\t\tShareQrcodeURL struct {\n\t\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t\t} `json:\"share_qrcode_url\"`\n\t\t\t\tShareTitle       string `json:\"share_title\"`\n\t\t\t\tShareTitleMyself string `json:\"share_title_myself\"`\n\t\t\t\tShareTitleOther  string `json:\"share_title_other\"`\n\t\t\t\tShareURL         string `json:\"share_url\"`\n\t\t\t\tShareWeiboDesc   string `json:\"share_weibo_desc\"`\n\t\t\t} `json:\"share_info\"`\n\t\t\tShortID             string      `json:\"short_id\"`\n\t\t\tSignature           string      `json:\"signature\"`\n\t\t\tSignatureExtra      interface{} `json:\"signature_extra\"`\n\t\t\tSpecialPeopleLabels interface{} `json:\"special_people_labels\"`\n\t\t\tStatus              int         `json:\"status\"`\n\t\t\tTextExtra           interface{} `json:\"text_extra\"`\n\t\t\tTotalFavorited      int         `json:\"total_favorited\"`\n\t\t\tUID                 string      `json:\"uid\"`\n\t\t\tUniqueID            string      `json:\"unique_id\"`\n\t\t\tUserAge             int         `json:\"user_age\"`\n\t\t\tUserCanceled        bool        `json:\"user_canceled\"`\n\t\t\tUserPermissions     interface{} `json:\"user_permissions\"`\n\t\t\tVerificationType    int         `json:\"verification_type\"`\n\t\t} `json:\"author\"`\n\t\tAuthorMaskTag int   `json:\"author_mask_tag\"`\n\t\tAuthorUserID  int64 `json:\"author_user_id\"`\n\t\tAwemeACL      struct {\n\t\t\tDownloadMaskPanel struct {\n\t\t\t\tCode     int `json:\"code\"`\n\t\t\t\tShowType int `json:\"show_type\"`\n\t\t\t} `json:\"download_mask_panel\"`\n\t\t} `json:\"aweme_acl\"`\n\t\tAwemeControl struct {\n\t\t\tCanComment     bool `json:\"can_comment\"`\n\t\t\tCanForward     bool `json:\"can_forward\"`\n\t\t\tCanShare       bool `json:\"can_share\"`\n\t\t\tCanShowComment bool `json:\"can_show_comment\"`\n\t\t} `json:\"aweme_control\"`\n\t\tAwemeID               string      `json:\"aweme_id\"`\n\t\tAwemeType             int         `json:\"aweme_type\"`\n\t\tChallengePosition     interface{} `json:\"challenge_position\"`\n\t\tChapterList           interface{} `json:\"chapter_list\"`\n\t\tCollectStat           int         `json:\"collect_stat\"`\n\t\tCommentGid            int64       `json:\"comment_gid\"`\n\t\tCommentList           interface{} `json:\"comment_list\"`\n\t\tCommentPermissionInfo struct {\n\t\t\tCanComment              bool `json:\"can_comment\"`\n\t\t\tCommentPermissionStatus int  `json:\"comment_permission_status\"`\n\t\t\tItemDetailEntry         bool `json:\"item_detail_entry\"`\n\t\t\tPressEntry              bool `json:\"press_entry\"`\n\t\t\tToastGuide              bool `json:\"toast_guide\"`\n\t\t} `json:\"comment_permission_info\"`\n\t\tCommerceConfigData interface{} `json:\"commerce_config_data\"`\n\t\tCommonBarInfo      string      `json:\"common_bar_info\"`\n\t\tComponentInfoV2    string      `json:\"component_info_v2\"`\n\t\tCoverLabels        interface{} `json:\"cover_labels\"`\n\t\tCreateTime         int         `json:\"create_time\"`\n\t\tDesc               string      `json:\"desc\"`\n\t\tDiggLottie         struct {\n\t\t\tCanBomb  int    `json:\"can_bomb\"`\n\t\t\tLottieID string `json:\"lottie_id\"`\n\t\t} `json:\"digg_lottie\"`\n\t\tDisableRelationBar      int         `json:\"disable_relation_bar\"`\n\t\tDislikeDimensionList    interface{} `json:\"dislike_dimension_list\"`\n\t\tDuetAggregateInMusicTab bool        `json:\"duet_aggregate_in_music_tab\"`\n\t\tDuration                int         `json:\"duration\"`\n\t\tFeedCommentConfig       struct {\n\t\t\tAuthorAuditStatus int    `json:\"author_audit_status\"`\n\t\t\tInputConfigText   string `json:\"input_config_text\"`\n\t\t} `json:\"feed_comment_config\"`\n\t\tGeofencing          []interface{} `json:\"geofencing\"`\n\t\tGeofencingRegions   interface{}   `json:\"geofencing_regions\"`\n\t\tGroupID             string        `json:\"group_id\"`\n\t\tHybridLabel         interface{}   `json:\"hybrid_label\"`\n\t\tImageAlbumMusicInfo struct {\n\t\t\tBeginTime int `json:\"begin_time\"`\n\t\t\tEndTime   int `json:\"end_time\"`\n\t\t\tVolume    int `json:\"volume\"`\n\t\t} `json:\"image_album_music_info\"`\n\t\tImageInfos interface{} `json:\"image_infos\"`\n\t\tImageList  interface{} `json:\"image_list\"`\n\t\tImages     []struct {\n\t\t\tDownloadURLList []string    `json:\"download_url_list\"`\n\t\t\tHeight          int         `json:\"height\"`\n\t\t\tMaskURLList     interface{} `json:\"mask_url_list\"`\n\t\t\tURI             string      `json:\"uri\"`\n\t\t\tURLList         []string    `json:\"url_list\"`\n\t\t\tWidth           int         `json:\"width\"`\n\t\t} `json:\"images\"`\n\t\tImgBitrate []struct {\n\t\t\tImages []struct {\n\t\t\t\tDownloadURLList []string    `json:\"download_url_list\"`\n\t\t\t\tHeight          int         `json:\"height\"`\n\t\t\t\tMaskURLList     interface{} `json:\"mask_url_list\"`\n\t\t\t\tURI             string      `json:\"uri\"`\n\t\t\t\tURLList         []string    `json:\"url_list\"`\n\t\t\t\tWidth           int         `json:\"width\"`\n\t\t\t} `json:\"images\"`\n\t\t\tName string `json:\"name\"`\n\t\t} `json:\"img_bitrate\"`\n\t\tImpressionData struct {\n\t\t\tGroupIDListA   []int64     `json:\"group_id_list_a\"`\n\t\t\tGroupIDListB   []int64     `json:\"group_id_list_b\"`\n\t\t\tSimilarIDListA interface{} `json:\"similar_id_list_a\"`\n\t\t\tSimilarIDListB interface{} `json:\"similar_id_list_b\"`\n\t\t} `json:\"impression_data\"`\n\t\tInteractionStickers  interface{} `json:\"interaction_stickers\"`\n\t\tIsAds                bool        `json:\"is_ads\"`\n\t\tIsCollectsSelected   int         `json:\"is_collects_selected\"`\n\t\tIsDuetSing           bool        `json:\"is_duet_sing\"`\n\t\tIsImageBeat          bool        `json:\"is_image_beat\"`\n\t\tIsLifeItem           bool        `json:\"is_life_item\"`\n\t\tIsMultiContent       int         `json:\"is_multi_content\"`\n\t\tIsStory              int         `json:\"is_story\"`\n\t\tIsTop                int         `json:\"is_top\"`\n\t\tItemWarnNotification struct {\n\t\t\tContent string `json:\"content\"`\n\t\t\tShow    bool   `json:\"show\"`\n\t\t\tType    int    `json:\"type\"`\n\t\t} `json:\"item_warn_notification\"`\n\t\tLabelTopText interface{} `json:\"label_top_text\"`\n\t\tLongVideo    interface{} `json:\"long_video\"`\n\t\tMusic        struct {\n\t\t\tAlbum            string        `json:\"album\"`\n\t\t\tArtistUserInfos  interface{}   `json:\"artist_user_infos\"`\n\t\t\tArtists          []interface{} `json:\"artists\"`\n\t\t\tAuditionDuration int           `json:\"audition_duration\"`\n\t\t\tAuthor           string        `json:\"author\"`\n\t\t\tAuthorDeleted    bool          `json:\"author_deleted\"`\n\t\t\tAuthorPosition   interface{}   `json:\"author_position\"`\n\t\t\tAuthorStatus     int           `json:\"author_status\"`\n\t\t\tAvatarLarge      struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_large\"`\n\t\t\tAvatarMedium struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_medium\"`\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tBindedChallengeID int  `json:\"binded_challenge_id\"`\n\t\t\tCanBackgroundPlay bool `json:\"can_background_play\"`\n\t\t\tCollectStat       int  `json:\"collect_stat\"`\n\t\t\tCoverColorHsv     struct {\n\t\t\t\tH int `json:\"h\"`\n\t\t\t\tS int `json:\"s\"`\n\t\t\t\tV int `json:\"v\"`\n\t\t\t} `json:\"cover_color_hsv\"`\n\t\t\tCoverHd struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_hd\"`\n\t\t\tCoverLarge struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_large\"`\n\t\t\tCoverMedium struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_medium\"`\n\t\t\tCoverThumb struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_thumb\"`\n\t\t\tDmvAutoShow          bool          `json:\"dmv_auto_show\"`\n\t\t\tDspStatus            int           `json:\"dsp_status\"`\n\t\t\tDuration             int           `json:\"duration\"`\n\t\t\tEndTime              int           `json:\"end_time\"`\n\t\t\tExternalSongInfo     []interface{} `json:\"external_song_info\"`\n\t\t\tExtra                string        `json:\"extra\"`\n\t\t\tID                   int64         `json:\"id\"`\n\t\t\tIDStr                string        `json:\"id_str\"`\n\t\t\tIsAudioURLWithCookie bool          `json:\"is_audio_url_with_cookie\"`\n\t\t\tIsCommerceMusic      bool          `json:\"is_commerce_music\"`\n\t\t\tIsDelVideo           bool          `json:\"is_del_video\"`\n\t\t\tIsMatchedMetadata    bool          `json:\"is_matched_metadata\"`\n\t\t\tIsOriginal           bool          `json:\"is_original\"`\n\t\t\tIsOriginalSound      bool          `json:\"is_original_sound\"`\n\t\t\tIsPgc                bool          `json:\"is_pgc\"`\n\t\t\tIsRestricted         bool          `json:\"is_restricted\"`\n\t\t\tIsVideoSelfSee       bool          `json:\"is_video_self_see\"`\n\t\t\tLunaInfo             struct {\n\t\t\t\tHasCopyright bool `json:\"has_copyright\"`\n\t\t\t\tIsLunaUser   bool `json:\"is_luna_user\"`\n\t\t\t} `json:\"luna_info\"`\n\t\t\tLyricShortPosition interface{} `json:\"lyric_short_position\"`\n\t\t\tMatchedPgcSound    struct {\n\t\t\t\tAuthor      string `json:\"author\"`\n\t\t\t\tCoverMedium struct {\n\t\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t\t} `json:\"cover_medium\"`\n\t\t\t\tMixedAuthor string `json:\"mixed_author\"`\n\t\t\t\tMixedTitle  string `json:\"mixed_title\"`\n\t\t\t\tTitle       string `json:\"title\"`\n\t\t\t} `json:\"matched_pgc_sound\"`\n\t\t\tMid               string      `json:\"mid\"`\n\t\t\tMusicChartRanks   interface{} `json:\"music_chart_ranks\"`\n\t\t\tMusicStatus       int         `json:\"music_status\"`\n\t\t\tMusicianUserInfos interface{} `json:\"musician_user_infos\"`\n\t\t\tMuteShare         bool        `json:\"mute_share\"`\n\t\t\tOfflineDesc       string      `json:\"offline_desc\"`\n\t\t\tOwnerHandle       string      `json:\"owner_handle\"`\n\t\t\tOwnerID           string      `json:\"owner_id\"`\n\t\t\tOwnerNickname     string      `json:\"owner_nickname\"`\n\t\t\tPgcMusicType      int         `json:\"pgc_music_type\"`\n\t\t\tPlayURL           struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLKey  string   `json:\"url_key\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"play_url\"`\n\t\t\tPosition                  interface{} `json:\"position\"`\n\t\t\tPreventDownload           bool        `json:\"prevent_download\"`\n\t\t\tPreventItemDownloadStatus int         `json:\"prevent_item_download_status\"`\n\t\t\tPreviewEndTime            int         `json:\"preview_end_time\"`\n\t\t\tPreviewStartTime          float64     `json:\"preview_start_time\"`\n\t\t\tReasonType                int         `json:\"reason_type\"`\n\t\t\tRedirect                  bool        `json:\"redirect\"`\n\t\t\tSchemaURL                 string      `json:\"schema_url\"`\n\t\t\tSearchImpr                struct {\n\t\t\t\tEntityID string `json:\"entity_id\"`\n\t\t\t} `json:\"search_impr\"`\n\t\t\tSecUID        string `json:\"sec_uid\"`\n\t\t\tShootDuration int    `json:\"shoot_duration\"`\n\t\t\tSong          struct {\n\t\t\t\tArtists interface{} `json:\"artists\"`\n\t\t\t\tID      int64       `json:\"id\"`\n\t\t\t\tIDStr   string      `json:\"id_str\"`\n\t\t\t} `json:\"song\"`\n\t\t\tSourcePlatform    int         `json:\"source_platform\"`\n\t\t\tStartTime         int         `json:\"start_time\"`\n\t\t\tStatus            int         `json:\"status\"`\n\t\t\tTagList           interface{} `json:\"tag_list\"`\n...\n// Path: extractors/douyu/douyu.go\npackage douyu\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"douyu\", New())\n}\n\ntype douyuData struct {\n\tError int `json:\"error\"`\n\tData  struct {\n\t\tVideoURL string `json:\"video_url\"`\n\t} `json:\"data\"`\n}\n\ntype douyuURLInfo struct {\n\tURL  string\n\tSize int64\n}\n\nfunc douyuM3u8(url string) ([]douyuURLInfo, int64, error) {\n\tvar (\n\t\tdata            []douyuURLInfo\n\t\ttemp            douyuURLInfo\n\t\tsize, totalSize int64\n\t\terr             error\n\t)\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tfor _, u := range urls {\n\t\tsize, err = request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\ttotalSize += size\n\t\ttemp = douyuURLInfo{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, totalSize, nil\n}\n\ntype extractor struct{}\n\n// New returns a douyu extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\tliveVid := utils.MatchOneOf(url, `https?://www.douyu.com/(\\S+)`)\n\tif liveVid != nil {\n\t\treturn nil, errors.New(\"\u6682\u4e0d\u652f\u6301\u6597\u9c7c\u76f4\u64ad\")\n\t}\n\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitles := utils.MatchOneOf(html, `<title>(.*?)</title>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\tvids := utils.MatchOneOf(url, `https?://v.douyu.com/show/(\\S+)`)\n\tif vids == nil || len(vids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvid := vids[1]\n\n\tdataString, err := request.Get(\"http://vmobile.douyu.com/video/getInfo?vid=\"+vid, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdataDict := new(douyuData)\n\tif err := json.Unmarshal([]byte(dataString), dataDict); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tm3u8URLs, totalSize, err := douyuM3u8(dataDict.Data.VideoURL)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turls := make([]*extractors.Part, len(m3u8URLs))\n\tfor index, u := range m3u8URLs {\n\t\turls[index] = &extractors.Part{\n\t\t\tURL:  u.URL,\n\t\t\tSize: u.Size,\n\t\t\tExt:  \"ts\",\n\t\t}\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urls,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u6597\u9c7c douyu.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/douyu/douyu_test.go\npackage douyu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://v.douyu.com/show/l0Q8mMY3wZqv49Ad\",\n\t\t\t\tTitle: \"\u6bcf\u65e5\u64b8\u62a5_\u6bcf\u65e5\u64b8\u62a5\uff1a\u6709\u4e9b\u4eba\u6b7b\u4e86\u5176\u5b9e\u5b83\u8fd8\u53ef\u4ee5\u628a\u4f60\u5e26\u8d70_\u6597\u9c7c\u89c6\u9891 - \u67006\u7684\u5f39\u5e55\u89c6\u9891\u7f51\u7ad9\",\n\t\t\t\tSize:  10558080,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/eporner/eporner.go\npackage eporner\n\nimport (\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/PuerkitoBio/goquery\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"eporner\", New())\n}\n\nconst (\n\tdownloadclass = \".dloaddivcol\"\n)\n\ntype src struct {\n\turl     string\n\tquality string\n\tsizestr string\n\tsize    int64\n}\n\n\nfunc getSrcMeta(text string) *src {\n\tsti := strings.Index(text, \"(\")\n\tste := strings.Index(text, \")\")\n\titext := text[sti+1 : ste]\n\tstrs := strings.Split(itext, \",\")\n\ts := &src{}\n\n\tif len(strs) == 2 {\n\t\ts.quality = strings.Trim(strs[0], \" \")\n\t\ts.sizestr = strings.Trim(strs[1], \" \")\n\t}\n\n\tif s.sizestr == \"\" {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\n\tvalunit := strings.Split(s.sizestr, \" \")\n\tval, err := strconv.ParseFloat(valunit[0], 64)\n\tif err != nil {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\tunit := valunit[1]\n\tswitch unit {\n\tcase \"KB\":\n\t\ts.size = int64(val * 1024)\n\tcase \"MB\":\n\t\ts.size = int64(val * 1024 * 1024)\n\tcase \"GB\":\n\t\ts.size = int64(val * 1024 * 1024 * 1024)\n\tdefault:\n\t\ts.size = int64(val)\n\t}\n\treturn s\n}\n\nfunc getSrc(html string) []*src {\n\tsrcs := []*src{}\n\td, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\td.Find(downloadclass).Each(func(i int, s *goquery.Selection) {\n\t\ts.Contents().Each(func(i int, s *goquery.Selection) {\n\t\t\tfor ns := range s.Nodes {\n\t\t\t\tn := s.Get(ns)\n\t\t\t\tif n.Data == \"a\" {\n\t\t\t\t\tvar sr *src\n\t\t\t\t\tif n.FirstChild != nil {\n\t\t\t\t\t\tsr = getSrcMeta(n.FirstChild.Data)\n\t\t\t\t\t}\n\t\t\t\t\tfor _, a := range n.Attr {\n\t\t\t\t\t\tif a.Key == \"href\" {\n\t\t\t\t\t\t\tsr.url = a.Val\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tsrcs = append(srcs, sr)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t})\n\n\treturn srcs\n}\n\ntype extractor struct{}\n\n// New returns a eporner extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(u, u, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tdesc := utils.MatchOneOf(html, `<title>(.+?)</title>`)\n\tif len(desc) > 1 {\n\t\ttitle = desc[1]\n\t} else {\n\t\ttitle = \"eporner\"\n\t}\n\tuu, err := url.Parse(u)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsrcs := getSrc(html)\n\tstreams := make(map[string]*extractors.Stream, len(srcs))\n\tfor _, src := range srcs {\n\t\tsrcurl := uu.Scheme + \"://\" + uu.Host + src.url\n\t\t// skipping an extra HEAD request to the URL.\n\t\t// size, err := request.Size(srcurl, u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL:  srcurl,\n\t\t\tSize: src.size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[src.quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    src.size,\n\t\t\tQuality: src.quality,\n\t\t}\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"EPORNER eporner.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     u,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/eporner/eporner_test.go\npackage eporner\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.eporner.com/video-mbubfvXYFip/dirtywivesclub-becky-bandini/\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    1525510307,\n\t\t\t\tTitle:   \"DirtyWivesClub - Becky Bandini - EPORNER\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/facebook/facebook.go\npackage facebook\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"facebook\", New())\n}\n\ntype extractor struct{}\n\n// New returns a facebook extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitles := utils.MatchOneOf(html, `<title>([^<]+)</title>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\ttitle := strings.TrimSpace(titles[1])\n\n\ttitle = regexp.MustCompile(`\\n+`).ReplaceAllString(title, \" \")\n\n\tqualityRegMap := map[string]*regexp.Regexp{\n\t\t\"sd\": regexp.MustCompile(`\"playable_url\":\\s*\"([^\"]+)\"`),\n\t\t// \"hd\": regexp.MustCompile(`\"playable_url_quality_hd\":\\s*\"([^\"]+)\"`),\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 2)\n\tfor quality, qualityReg := range qualityRegMap {\n\t\tmatcher := qualityReg.FindStringSubmatch(html)\n\n\t\tif len(matcher) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tu := strings.ReplaceAll(matcher[1], \"\\\\\", \"\")\n\n\t\tsize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Facebook facebook.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/facebook/facebook_test.go\npackage facebook\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.facebook.com/100058251872436/videos/424557726111987\",\n\t\t\t\tTitle:   \"\u0420\u043e\u043c\u0430\u043d \u0413\u0440\u0438\u0449\u0443\u043a - \u041f\u0456\u0434\u0442\u0440\u0438\u043c\u043a\u0430 \u0437 \u042f\u043f\u043e\u043d\u0456\u0457 \ud83c\uddef\ud83c\uddf5 \u0413\u0443\u0440\u0442 Yokohama Sisters \ud83d\udc4f\",\n\t\t\t\tSize:    1441128,\n\t\t\t\tQuality: \"sd\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/geekbang/geekbang.go\npackage geekbang\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"geekbang\", New())\n}\n\ntype geekData struct {\n\tCode  int             `json:\"code\"`\n\tError json.RawMessage `json:\"error\"`\n\tData  struct {\n\t\tVideoID      string `json:\"video_id\"`\n\t\tTitle        string `json:\"article_sharetitle\"`\n\t\tColumnHadSub bool   `json:\"column_had_sub\"`\n\t} `json:\"data\"`\n}\n\ntype videoPlayAuth struct {\n\tCode  int             `json:\"code\"`\n\tError json.RawMessage `json:\"error\"`\n\tData  struct {\n\t\tPlayAuth string `json:\"play_auth\"`\n\t} `json:\"data\"`\n}\n\ntype playInfo struct {\n\tVideoBase struct {\n\t\tVideoID  string `json:\"VideoId\"`\n\t\tTitle    string `json:\"Title\"`\n\t\tCoverURL string `josn:\"CoverURL\"`\n\t} `json:\"VideoBase\"`\n\tPlayInfoList struct {\n\t\tPlayInfo []struct {\n\t\t\tURL        string `json:\"PlayURL\"`\n\t\t\tSize       int64  `json:\"Size\"`\n\t\t\tDefinition string `json:\"Definition\"`\n\t\t} `json:\"PlayInfo\"`\n\t} `json:\"PlayInfoList\"`\n}\n\ntype geekURLInfo struct {\n\tURL  string\n\tSize int64\n}\n\nfunc geekM3u8(url string) ([]geekURLInfo, error) {\n\tvar (\n\t\tdata []geekURLInfo\n\t\ttemp geekURLInfo\n\t\tsize int64\n\t\terr  error\n\t)\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor _, u := range urls {\n\t\ttemp = geekURLInfo{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, nil\n}\n\ntype extractor struct{}\n\n// New returns a geekbang extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\tmatches := utils.MatchOneOf(url, `https?://time.geekbang.org/course/detail/(\\d+)-(\\d+)`)\n\tif matches == nil || len(matches) < 3 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Get video information\n\theanders := map[string]string{\"Origin\": \"https://time.geekbang.org\", \"Content-Type\": \"application/json\", \"Referer\": url}\n\tparams := strings.NewReader(fmt.Sprintf(`{\"id\": %q}`, matches[2]))\n\tres, err := request.Request(http.MethodPost, \"https://time.geekbang.org/serv/v1/article\", params, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar data geekData\n\tif err = json.NewDecoder(res.Body).Decode(&data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif data.Code < 0 {\n\t\treturn nil, errors.New(string(data.Error))\n\t}\n\n\tif data.Data.VideoID == \"\" && !data.Data.ColumnHadSub {\n\t\treturn nil, errors.New(\"\u8bf7\u5148\u8d2d\u4e70\u8bfe\u7a0b\uff0c\u6216\u4f7f\u7528Cookie\u767b\u5f55\u3002\")\n\t}\n\n\t// Get video license token information\n\tparams = strings.NewReader(\"{\\\"source_type\\\":1,\\\"aid\\\":\" + matches[2] + \",\\\"video_id\\\":\\\"\" + data.Data.VideoID + \"\\\"}\")\n\tres, err = request.Request(http.MethodPost, \"https://time.geekbang.org/serv/v3/source_auth/video_play_auth\", params, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar playAuth videoPlayAuth\n\tif err = json.NewDecoder(res.Body).Decode(&playAuth); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif playAuth.Code < 0 {\n\t\treturn nil, errors.New(string(playAuth.Error))\n\t}\n\n\t// Get video playback information\n\theanders = map[string]string{\"Accept-Encoding\": \"\"}\n\tres, err = request.Request(http.MethodGet, \"http://ali.mantv.top/play/info?playAuth=\"+playAuth.Data.PlayAuth, nil, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar playInfo playInfo\n\tif err = json.NewDecoder(res.Body).Decode(&playInfo); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := data.Data.Title\n\n\tstreams := make(map[string]*extractors.Stream, len(playInfo.PlayInfoList.PlayInfo))\n\n\tfor _, media := range playInfo.PlayInfoList.PlayInfo {\n\t\tm3u8URLs, err := geekM3u8(media.URL)\n\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turls := make([]*extractors.Part, len(m3u8URLs))\n\t\tfor index, u := range m3u8URLs {\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL:  u.URL,\n\t\t\t\tSize: u.Size,\n\t\t\t\tExt:  \"ts\",\n\t\t\t}\n\t\t}\n\n\t\tstreams[media.Definition] = &extractors.Stream{\n\t\t\tParts: urls,\n\t\t\tSize:  media.Size,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u6781\u5ba2\u65f6\u95f4 geekbang.org\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/geekbang/geekbang_test.go\npackage geekbang\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://time.geekbang.org/course/detail/190-97203\",\n\t\t\t\tTitle: \"02 | \u5185\u5bb9\u7efc\u8ff0 - \u73a9\u8f6cwebpack\",\n\t\t\t\tSize:  10752472,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/haokan/haokan.go\npackage haokan\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"haokan\", New())\n}\n\ntype extractor struct{}\n\n// New returns a haokan extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitles := utils.MatchOneOf(html, `property=\"og:title\"\\s+content=\"(.+?)\"`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\t// \u4e4b\u524d\u7684\u597d\u770b\u7f51\u9875\u4e2d\uff0c\u89c6\u9891\u5730\u5740\u662f\u653e\u5728 video \u6807\u7b7e\u4e0b\n\turls := utils.MatchOneOf(html, `<video\\s*class=\"video\"\\s*src=\"?(.+?)\"?\\s*>`)\n\n\tif urls == nil || len(urls) < 2 {\n\t\t// fallbak: \u65b0\u7684\u597d\u770b\u7f51\u9875\u4e2d\uff0c\u89c6\u9891\u5730\u5740\u5728 json \u6570\u636e\u91cc\n\t\turls = utils.MatchOneOf(html, `\"playurl\":\"(http.+?)\"`)\n\t}\n\n\tif urls == nil || len(urls) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tplayurl := strings.Replace(urls[1], `\\/`, `/`, -1)\n\n\tsize, err := request.Size(playurl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t_, ext, err := utils.GetNameAndExt(playurl)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  playurl,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt:  ext,\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u597d\u770b\u89c6\u9891 haokan.baidu.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/haokan/haokan_test.go\npackage haokan\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://haokan.baidu.com/v?vid=10057409468467026969\",\n\t\t\t\tTitle: \"\u542c\u6b4c\u5b66\u82f1\u8bed\u5c0f\u5b66\u7bc7\uff086\uff09\uff1amy new pen pal\",\n\t\t\t\tSize:  2027354,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/hupu/hupu.go\npackage hupu\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"hupu\", New())\n}\n\ntype extractor struct{}\n\n// New returns a hupu extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar title string\n\ttitleDesc := utils.MatchOneOf(html, `<span class=\"post-user-comp-info-bottom-title\">(.+?)</span>`)\n\tif len(titleDesc) > 1 {\n\t\ttitle = titleDesc[1]\n\t} else {\n\t\ttitle = \"hupu video\"\n\t}\n\n\tvar videoUrl string\n\turlDesc := utils.MatchOneOf(html, `<video src=\"(.+?)\" controls=\"\" poster=(.+?)></video>`)\n\tif len(urlDesc) > 1 {\n\t\tvideoUrl = urlDesc[1]\n\t} else {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tsize, err := request.Size(videoUrl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoUrl,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u864e\u6251 hupu.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/hupu/hupu_test.go\npackage hupu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestHupu(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://bbs.hupu.com/47401018.html?is_reflow=1&cid=84752419&bddid=56KXU5QUJH4VGM26SFPTYTKNI5CFNJMX736TIZ52DXLGUAAMBJVA01&puid=16522089&client=8577E496-4D9B-4E5C-A9DB-A8EF5C1956D2\",\n\t\t\t\tTitle: \"\u7ed3\u5c40\u5f15\u8d77\u8212\u9002\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/huya/huya.go\npackage huya\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"huya\", New())\n}\n\ntype extractor struct{}\n\nconst huyaVideoHost = \"https://videotx-platform.cdn.huya.com/\"\n\n// New returns a huya extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar title string\n\ttitleDesc := utils.MatchOneOf(html, `<h1>(.+?)</h1>`)\n\tif len(titleDesc) > 1 {\n\t\ttitle = titleDesc[1]\n\t} else {\n\t\ttitle = \"huya video\"\n\t}\n\n\tvar videoUrl string\n\tvideoDesc := utils.MatchOneOf(html, `//videotx-platform.cdn.huya.com/(.*)\" poster=(.+?)`)\n\tif len(videoDesc) > 1 {\n\t\tvideoUrl = huyaVideoHost + videoDesc[1]\n\t} else {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tsize, err := request.Size(videoUrl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoUrl,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u864e\u7259 huya.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/huya/huya_test.go\npackage huya\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestHuya(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://m.v.huya.com/play/fans/630103747.html/?shareid=4597484513543964249&shareUid=2179142017&source=ios&sharetype=other&platform=2\",\n\t\t\t\tTitle: \"12.28 \u96c6\u68a6\u859b\u5c0f\u8c26\u3010\u5c01\u53f7\u6597\u7f57\u3011\u76f4\u64ad\u540d\u573a\u9762\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/instagram/instagram.go\npackage instagram\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nvar client *http.Client\n\nfunc init() {\n\textractors.Register(\"instagram\", New())\n\tclient = &http.Client{\n\t\tTimeout: 10 * time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tDial: (&net.Dialer{\n\t\t\t\tTimeout: 5 * time.Second,\n\t\t\t}).Dial,\n\t\t\tTLSHandshakeTimeout: 5 * time.Second,\n\t\t},\n\t}\n}\n\n// sliderItemNode contains information about the Instagram post\ntype sliderItemNode struct {\n\tDisplayURL string `json:\"display_url\"` // URL of the Media (resolution is dynamic)\n\n\tIsVideo  bool   `json:\"is_video\"`  // Is type of the Media equals to video\n\tVideoURL string `json:\"video_url\"` // Direct URL to the Video\n}\n\nfunc (s sliderItemNode) extractMediaURL() string {\n\tif s.IsVideo {\n\t\treturn s.VideoURL\n\t}\n\n\treturn s.DisplayURL\n}\n\ntype instagramPayload struct {\n\tMedia struct {\n\t\tID          string `json:\"id\"` // Unique ID of the Media\n\t\tSliderItems struct {\n\t\t\tEdges []struct {\n\t\t\t\tNode sliderItemNode `json:\"node\"`\n\t\t\t} `json:\"edges\"`\n\t\t} `json:\"edge_sidecar_to_children\"` // Children of the Media\n\t} `json:\"shortcode_media\"` // Media\n}\n\nfunc (s instagramPayload) isEmpty() bool {\n\treturn s.Media.ID == \"\"\n}\n\n...\n", "needle_token_start": 6724, "needle_token_end": 7078, "code_context_ntokens": 15202, "output": ["```go\nfunc getSrcMeta(text string) *src {\n\tsti := strings.Index(text, \"(\")\n\tste := strings.Index(text, \")\")\n\titext := text[sti+1 : ste]\n\tstrs := strings.Split(itext, \",\")\n\ts := &src{}\n\n\tif len(strs) == 2 {\n\t\ts.quality = strings.Trim(strs[0], \" \")\n\t\ts.sizestr = strings.Trim(strs[1], \" \")\n\t}\n\n\tif s.sizestr == \"\" {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\n\tvalunit := strings.Split(s.sizestr, \" \")\n\tval, err := strconv.ParseFloat(valunit[0], 64)\n\tif err != nil {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\tunit := valunit[1]\n\tswitch unit {\n\tcase \"KB\":\n\t\ts.size = int64(val * 1024)\n\tcase \"MB\":\n\t\ts.size = int64(val * 1024 * 1024)\n\tcase \"GB\":\n\t\ts.size = int64(val * 1024 * 1024 * 1024)\n\tdefault:\n\t\ts.size = int64(val)\n\t}\n\treturn s\n}\n```"]}
{"repo": "iawia002/lux", "name": "TestRumble", "language": "go", "path": "extractors/rumble/rumble_test.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to test the video extraction functionality from a specific video hosting platform, ensuring that the video data can be successfully retrieved and processed.\n2. **Input**: The function takes a predefined set of test cases, each containing a URL pointing to a video on the platform and an expected title for that video.\n3. **Output**: There is no direct output from this function; however, it is expected to validate if the video extraction process works correctly by running the extraction and potentially checking against expected results.\n4. **Procedure**: The function iterates over a list of test cases. For each test case, it invokes the video extraction method with the provided URL and checks if the extraction process completes without errors and potentially if the extracted video data matches expected values such as title or quality.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/qq/qq.go\npackage qq\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"qq\", New())\n}\n\ntype qqVideoInfo struct {\n\tFl struct {\n\t\tFi []struct {\n\t\t\tID    int    `json:\"id\"`\n\t\t\tName  string `json:\"name\"`\n\t\t\tCname string `json:\"cname\"`\n\t\t\tFs    int64  `json:\"fs\"`\n\t\t} `json:\"fi\"`\n\t} `json:\"fl\"`\n\tVl struct {\n\t\tVi []struct {\n\t\t\tFn    string `json:\"fn\"`\n\t\t\tTi    string `json:\"ti\"`\n\t\t\tFvkey string `json:\"fvkey\"`\n\t\t\tCl    struct {\n\t\t\t\tFc int `json:\"fc\"`\n\t\t\t\tCi []struct {\n\t\t\t\t\tIdx int `json:\"idx\"`\n\t\t\t\t} `json:\"ci\"`\n\t\t\t} `json:\"cl\"`\n\t\t\tUl struct {\n\t\t\t\tUI []struct {\n\t\t\t\t\tURL string `json:\"url\"`\n\t\t\t\t} `json:\"ui\"`\n\t\t\t} `json:\"ul\"`\n\t\t} `json:\"vi\"`\n\t} `json:\"vl\"`\n\tMsg string `json:\"msg\"`\n}\n\ntype qqKeyInfo struct {\n\tKey string `json:\"key\"`\n}\n\nconst qqPlayerVersion string = \"3.2.19.333\"\n\nfunc getVinfo(vid, defn, refer string) (qqVideoInfo, error) {\n\thtml, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"http://vv.video.qq.com/getinfo?otype=json&platform=11&defnpayver=1&appver=%s&defn=%s&vid=%s\",\n\t\t\tqqPlayerVersion, defn, vid,\n\t\t), refer, nil,\n\t)\n\tif err != nil {\n\t\treturn qqVideoInfo{}, err\n\t}\n\tjsonStrings := utils.MatchOneOf(html, `QZOutputJson=(.+);$`)\n\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\treturn qqVideoInfo{}, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := jsonStrings[1]\n\tvar data qqVideoInfo\n\tif err = json.Unmarshal([]byte(jsonString), &data); err != nil {\n\t\treturn qqVideoInfo{}, err\n\t}\n\treturn data, nil\n}\n\nfunc genStreams(vid, cdn string, data qqVideoInfo) (map[string]*extractors.Stream, error) {\n\tstreams := make(map[string]*extractors.Stream)\n\tvar vkey string\n\t// number of fragments\n\tvar clips int\n\n\tfor _, fi := range data.Fl.Fi {\n\t\tvar fmtIDPrefix string\n\t\tvar fns []string\n\t\tif slices.Contains([]string{\"shd\", \"fhd\"}, fi.Name) {\n\t\t\tfmtIDPrefix = \"p\"\n\t\t\tfmtIDName := fmt.Sprintf(\"%s%d\", fmtIDPrefix, fi.ID%10000)\n\t\t\tfns = []string{strings.Split(data.Vl.Vi[0].Fn, \".\")[0], fmtIDName, \"mp4\"}\n\t\t\tif len(fns) > 3 {\n\t\t\t\t// delete ID part\n\t\t\t\t// e0765r4mwcr.2.mp4 -> e0765r4mwcr.mp4\n\t\t\t\tfns = append(fns[:1], fns[2:]...)\n\t\t\t}\n\t\t\tclips = data.Vl.Vi[0].Cl.Fc\n\t\t\tif clips == 0 {\n\t\t\t\tclips = 1\n\t\t\t}\n\t\t} else {\n\t\t\ttmpData, err := getVinfo(vid, fi.Name, cdn)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tfns = strings.Split(tmpData.Vl.Vi[0].Fn, \".\")\n\t\t\tif len(fns) >= 3 && utils.MatchOneOf(fns[1], `^p(\\d{3})$`) != nil {\n\t\t\t\tfmtIDPrefix = \"p\"\n\t\t\t}\n\t\t\tclips = tmpData.Vl.Vi[0].Cl.Fc\n\t\t\tif clips == 0 {\n\t\t\t\tclips = 1\n\t\t\t}\n\t\t}\n\n\t\tvar urls []*extractors.Part\n\t\tvar totalSize int64\n\t\tvar filename string\n\t\tfor part := 1; part < clips+1; part++ {\n\t\t\t// Multiple fragments per streams\n\t\t\tif fmtIDPrefix == \"p\" {\n\t\t\t\tif len(fns) < 4 {\n\t\t\t\t\t// If the number of fragments > 0, the filename needs to add the number of fragments\n\t\t\t\t\t// n0687peq62x.p709.mp4 -> n0687peq62x.p709.1.mp4\n\t\t\t\t\tfns = append(fns[:2], append([]string{strconv.Itoa(part)}, fns[2:]...)...)\n\t\t\t\t} else {\n\t\t\t\t\tfns[2] = strconv.Itoa(part)\n\t\t\t\t}\n...\n// Path: extractors/qq/qq_test.go\npackage qq\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.qq.com/x/page/n0687peq62x.html\",\n\t\t\t\tTitle:   \"\u4e16\u754c\u676f\u7b2c\u4e00\u671f\uff1a100\u79d2\u901f\u6210\uff01\u201c\u4f2a\u7403\u8ff7\u201d\u4e16\u754c\u676f\u751f\u5b58\u6307\u5357\",\n\t\t\t\tSize:    23759683,\n\t\t\t\tQuality: \"\u84dd\u5149;(1080P)\",\n\t\t\t},\n\t\t},\n\t\t// {\n\t\t// \tname: \"movie and vid test\",\n\t\t// \targs: test.Args{\n\t\t// \t\tURL:     \"https://v.qq.com/x/cover/e5qmd3z5jr0uigk.html\",\n\t\t// \t\tTitle:   \"\u8d4c\u4fa0\uff08\u7ca4\u8bed\u7248\uff09\",\n\t\t// \t\tSize:    1046910811,\n\t\t// \t\tQuality: \"\u8d85\u6e05;(720P)\",\n\t\t// \t},\n\t\t// },\n\t\t{\n\t\t\tname: \"fmt ID test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.qq.com/x/cover/2aya3ibdmft6vdw/e0765r4mwcr.html\",\n\t\t\t\tTitle:   \"\u300a\u5361\u8def\u91cc\u300b\u51fa\u5708\uff01\u5996\u5a06\u7537\u5b50\u6559\u5b66\u5e7f\u573a\u821e\u7248\uff0c\u5927\u5988\u8868\u60c5\u4eae\u4e86\uff01\",\n\t\t\t\tSize:    14112979,\n\t\t\t\tQuality: \"\u8d85\u6e05;(720P)\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/reddit/reddit.go\npackage reddit\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"reddit\", New())\n}\n\nconst (\n\treferer  = \"https://www.reddit.com\"\n\tsiteName = \"Reddit reddit.com\"\n\n\tredditMP4API = \"https://v.redd.it/\"\n\tredditIMGAPI = \"https://i.redd.it/\"\n\taudioURLPart = \"/DASH_audio.mp4\"\n)\n\nvar resMap = map[string]string{\n\t\"720p\": \"/DASH_720.mp4\",\n\t\"480p\": \"/DASH_480.mp4\",\n\t\"360p\": \"/DASH_360.mp4\",\n\t\"240p\": \"/DASH_240.mp4\",\n\t\"220p\": \"/DASH_220.mp4\",\n}\n\ntype extractor struct{}\n\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, referer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// set thread number to 1 manually to avoid http 412 error\n\toption.ThreadNumber = 1\n\n\ttitle := utils.MatchOneOf(html, `<title>(.+?)<\\/title>`)[1]\n\n\tif utils.MatchOneOf(html, `meta property=\"og:video\" content=.*HLSPlaylist`) != nil {\n\t\tmp4URL := utils.MatchOneOf(html, `https://v.redd.it/(.+?)/HLSPlaylist`)[1]\n\t\tif mp4URL == \"\" {\n\t\t\treturn nil, errors.New(\"can't match mp4 content downloadable url\")\n\t\t}\n\n\t\taudioURL := fmt.Sprintf(\"%s%s%s\", redditMP4API, mp4URL, audioURLPart)\n\t\tsize, err := request.Size(audioURL, referer)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\taudioPart := &extractors.Part{\n\t\t\tURL:  audioURL,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp3\",\n\t\t}\n\n\t\tstreams := make(map[string]*extractors.Stream, len(resMap))\n\t\tfor res, urlParts := range resMap {\n\t\t\tresURL := fmt.Sprintf(\"%s%s%s\", redditMP4API, mp4URL, urlParts)\n\t\t\tsize, err := request.Size(resURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tstreams[res] = &extractors.Stream{\n\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t{\n\t\t\t\t\t\tURL:  resURL,\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\tExt:  \"mp4\",\n\t\t\t\t\t},\n\t\t\t\t\taudioPart,\n\t\t\t\t},\n\t\t\t\tSize:    size + audioPart.Size,\n\t\t\t\tQuality: res,\n\t\t\t\tNeedMux: true,\n\t\t\t}\n\t\t}\n\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite:    siteName,\n\t\t\t\tTitle:   title,\n\t\t\t\tType:    extractors.DataTypeVideo,\n\t\t\t\tStreams: streams,\n\t\t\t\tURL:     url,\n\t\t\t},\n\t\t}, nil\n\t} else if utils.MatchOneOf(html, `<meta property=\"og:type\" content=\"image\"/>`) != nil {\n\t\tvar imgURL string\n\t\tvar size int64\n\t\tif utils.MatchOneOf(html, `content\":\"https:\\/\\/i.redd.it\\/(.+?)\",\"type\":\"image\"`) != nil {\n\t\t\timgURL = redditIMGAPI + utils.MatchOneOf(html, `content\":\"https:\\/\\/i.redd.it\\/(.+?)\",\"type\":\"image\"`)[1]\n\t\t\tsize, err = request.Size(imgURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t} else {\n\t\t\timgURL = utils.MatchOneOf(html, `content\":\"(.+?)\",\"type\":\"image\"`)[1]\n\t\t\timgURL = strings.ReplaceAll(imgURL, \"auto=webp\\\\u0026s\", \"auto=webp&s\")\n\t\t\tsize, err = request.Size(imgURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t}\n\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite:  siteName,\n\t\t\t\tTitle: title,\n\t\t\t\tType:  extractors.DataTypeImage,\n\t\t\t\tStreams: map[string]*extractors.Stream{\n\t\t\t\t\t\"default\": {\n\t\t\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL:  imgURL,\n\t\t\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\t\t\tExt:  \"jpg\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tURL: url,\n\t\t\t},\n\t\t}, nil\n\t} else if utils.MatchOneOf(html, `https:\\/\\/preview\\.redd\\.it\\/.*gif`) != nil {\n\t\tgifURL := utils.MatchOneOf(html, `https:\\/\\/preview\\.redd\\.it\\/.*?\\.gif\\?format=mp4.*?\"`)[0]\n\t\tif gifURL == \"\" {\n\t\t\treturn nil, errors.New(\"can't match gif content downloadable url\")\n\t\t}\n\n\t\tgifURL = strings.ReplaceAll(gifURL, \"&amp;\", \"&\")\n\t\tgifURL = strings.ReplaceAll(gifURL, \"\\\"\", \"\")\n\n\t\tsize, err := request.Size(gifURL, \"reddit.com\")\n\t\tif err != nil {\n\t\t\treturn nil, errors.New(\"can't get video size\")\n\t\t}\n\n\t\tstreams := map[string]*extractors.Stream{\n\t\t\t\"default\": {\n\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t{\n\t\t\t\t\t\tURL:  gifURL,\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\tExt:  \"mp4\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tSize: size,\n\t\t\t},\n\t\t}\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite:    siteName,\n\t\t\t\tTitle:   title,\n\t\t\t\tType:    extractors.DataTypeVideo,\n\t\t\t\tStreams: streams,\n\t\t\t\tURL:     url,\n\t\t\t},\n\t\t}, nil\n\t}\n\n\treturn nil, fmt.Errorf(\"unable to handle url: %s\", url)\n}\n\n// Path: extractors/reddit/reddit_test.go\npackage reddit\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestReddit(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 0\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/space/comments/uj8sod/a_couple_of_days_ago_i_visited_this_place_an/\",\n\t\t\t\tTitle: \"A couple of days ago I visited this place. An abandoned space shuttle : space\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/DotA2/comments/uq012r/til_how_useful_hurricane_bird_is/\",\n\t\t\t\tTitle: \"TIL how useful hurricane bird is : DotA2\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/ProgrammerHumor/comments/uqovco/my_code_works/\",\n\t\t\t\tTitle: \"My code works : ProgrammerHumor\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 3\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/AnimatedPixelArt/comments/uomu32/animation_for_astral_ascent/\",\n\t\t\t\tTitle: \"Animation for Astral Ascent : AnimatedPixelArt\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 4\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/linuxmemes/comments/v1a4wh/please_olive_do_something/\",\n\t\t\t\tTitle: \"Please Olive, do something... : linuxmemes\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 5\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/gaming/comments/v27m79/skyrim_probably/\",\n\t\t\t\tTitle: \"Skyrim, probably : gaming\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/rumble/rumble.go\npackage rumble\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"path\"\n\t\"regexp\"\n\t\"strconv\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"rumble\", New())\n}\n\ntype extractor struct{}\n\n// New returns a rumble extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\ntype rumbleData struct {\n\tFormat       string `json:\"format\"`\n\tName         string `json:\"name\"`\n\tEmbedURL     string `json:\"embedUrl\"`\n\tThumbnailURL string `json:\"thumbnailUrl\"`\n\tType         string `json:\"@type\"`\n\tVideoURL     string `json:\"videoUrl\"`\n\tQuality      string `json:\"quality\"`\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\thtml := string(b)\n\tvar title string\n\tmatchTitle := utils.MatchOneOf(html, `<title>(.+?)</title>`)\n\tif len(matchTitle) > 1 {\n\t\ttitle = matchTitle[1]\n\t} else {\n\t\ttitle = \"rumble video\"\n\t}\n\n\tpayload, err := readPayload(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvideoID, err := getVideoID(payload.EmbedURL)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams, err := fetchVideoQuality(videoID)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Rumble rumble.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Read JSON object from the video webpage\nfunc readPayload(html string) (*rumbleData, error) {\n\tmatchPayload := utils.MatchOneOf(html, `\\<script\\stype=\"?application\\/ld\\+json\"?\\>(.+?)\\<\\/script>`)\n\tif len(matchPayload) < 1 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed)\n\t}\n\n\trumbles := make([]rumbleData, 0)\n\tif err := json.Unmarshal([]byte(matchPayload[1]), &rumbles); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tfor _, it := range rumbles {\n\t\tif it.Type == \"VideoObject\" {\n\t\t\treturn &it, nil\n\t\t}\n\t}\n\n\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n}\n\nfunc getVideoID(embedURL string) (string, error) {\n\tu, err := url.Parse(embedURL)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\treturn path.Base(u.Path), nil\n}\n\n// Rumble response contains the streams in `rumbleStreams`\ntype rumbleResponse struct {\n\tStreams *json.RawMessage `json:\"ua\"`\n}\n\n// Common video meta data\ntype streamInfo struct {\n\tURL  string `json:\"url\"`\n\tMeta struct {\n\t\tBitrate uint16 `json:\"bitrate\"`\n\t\tSize    int64  `json:\"size\"`\n\t\tWidth   uint16 `json:\"w\"`\n\t\tHeight  uint16 `json:\"h\"`\n\t} `json:\"meta\"`\n}\n\n// common video qualities for `mp4`, `webm`\ntype videoQualities struct {\n\tQ240  struct{ streamInfo } `json:\"240\"`\n\tQ360  struct{ streamInfo } `json:\"360\"`\n\tQ480  struct{ streamInfo } `json:\"480\"`\n\tQ720  struct{ streamInfo } `json:\"720\"`\n\tQ1080 struct{ streamInfo } `json:\"1080\"`\n\tQ1440 struct{ streamInfo } `json:\"1440\"`\n\tQ2160 struct{ streamInfo } `json:\"2160\"`\n\tQ2161 struct{ streamInfo } `json:\"2161\"`\n}\n\n// Video payload for adaptive stream and different qualities\ntype rumbleStreams struct {\n\tFMp4 struct {\n\t\tvideoQualities\n\t} `json:\"mp4\"`\n\tFWebm struct {\n\t\tvideoQualities\n\t} `json:\"webm\"`\n\tFHLS struct {\n\t\tQAuto struct{ streamInfo } `json:\"auto\"`\n\t} `json:\"hls\"`\n}\n\n// Unmarshall the video response\n// Some properties like `mp4`, `webm` are either array or an object\nfunc (r *rumbleStreams) UnmarshalJSON(b []byte) error {\n\tvar resp *rumbleResponse\n\tif err := json.Unmarshal(b, &resp); err != nil {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Get individual stream from the response\n\tvar obj map[string]*json.RawMessage\n\tif err := json.Unmarshal(*resp.Streams, &obj); err != nil {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tif v, ok := obj[\"mp4\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FMp4)\n\t}\n\tif v, ok := obj[\"webm\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FWebm)\n\t}\n\tif v, ok := obj[\"hls\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FHLS)\n\t}\n\n\treturn nil\n}\n\n// Use this to create all the streams for `mp4`, `webm`\nfunc (rs *rumbleStreams) makeAllVODStreams(m map[string]*extractors.Stream) {\n\tm[\"webm\"] = makeStreamMeta(\"480\", \"webm\", &rs.FWebm.Q480.streamInfo)\n\tm[\"240\"] = makeStreamMeta(\"240\", \"mp4\", &rs.FMp4.Q240.streamInfo)\n\tm[\"360\"] = makeStreamMeta(\"360\", \"mp4\", &rs.FMp4.Q360.streamInfo)\n\tm[\"480\"] = makeStreamMeta(\"480\", \"mp4\", &rs.FMp4.Q480.streamInfo)\n\tm[\"720\"] = makeStreamMeta(\"720\", \"mp4\", &rs.FMp4.Q720.streamInfo)\n\tm[\"1080\"] = makeStreamMeta(\"1080\", \"mp4\", &rs.FMp4.Q1080.streamInfo)\n\tm[\"1440\"] = makeStreamMeta(\"1440\", \"mp4\", &rs.FMp4.Q1440.streamInfo)\n\tm[\"2160\"] = makeStreamMeta(\"2160\", \"mp4\", &rs.FMp4.Q2160.streamInfo)\n\tm[\"2161\"] = makeStreamMeta(\"2161\", \"mp4\", &rs.FMp4.Q2161.streamInfo)\n}\n\nvar reResolution = regexp.MustCompile(`_(\\d{3,4})p\\/`) // ex. _720p/\n\n// Use this to create all the streams for live videos\nfunc (rs *rumbleStreams) makeAllLiveStreams(m map[string]*extractors.Stream) error {\n\tplaylists, err := utils.M3u8URLs(rs.FHLS.QAuto.URL)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tif len(playlists) == 0 {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Find the highest resolution\n\tplaylistURL := playlists[0]\n\tmaxRes := 0\n\tfor _, x := range playlists {\n\t\tmatched := reResolution.FindStringSubmatch(x)\n\t\tif len(matched) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tres, err := strconv.Atoi(matched[1])\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tif maxRes < res {\n\t\t\tmaxRes = res\n\t\t\tplaylistURL = x\n\t\t}\n\t}\n\n\ttsURLs, err := utils.M3u8URLs(playlistURL)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tvar parts []*extractors.Part\n\tfor _, x := range tsURLs {\n\t\tpart := &extractors.Part{\n\t\t\tURL:  x,\n\t\t\tSize: rs.FHLS.QAuto.streamInfo.Meta.Size,\n\t\t\tExt:  \"ts\",\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tm[\"hls\"] = &extractors.Stream{\n\t\tParts:   parts,\n\t\tSize:    rs.FHLS.QAuto.streamInfo.Meta.Size,\n\t\tQuality: strconv.Itoa(maxRes),\n\t}\n\n\treturn nil\n}\n\n// Request video formats and qualities\nfunc fetchVideoQuality(videoID string) (map[string]*extractors.Stream, error) {\n\treqURL := fmt.Sprintf(`https://rumble.com/embedJS/u3/?request=video&ver=2&v=%s&ext={\"ad_count\":null}&ad_wt=0`, videoID)\n\n\tres, err := request.Request(http.MethodGet, reqURL, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar rs rumbleStreams\n\tif err := json.Unmarshal(b, &rs); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 9)\n\trs.makeAllVODStreams(streams)\n\t_ = rs.makeAllLiveStreams(streams)\n\n\treturn streams, nil\n}\n\nfunc makeStreamMeta(q, ext string, info *streamInfo) *extractors.Stream {\n\turlMeta := &extractors.Part{\n\t\tURL:  info.URL,\n\t\tSize: info.Meta.Size,\n\t\tExt:  ext,\n\t}\n\n\treturn &extractors.Stream{\n\t\tParts:   []*extractors.Part{urlMeta},\n\t\tSize:    info.Meta.Size,\n\t\tQuality: q,\n\t}\n}\n\n// Path: extractors/rumble/rumble_test.go\npackage rumble\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\n\nfunc TestRumble(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://rumble.com/v24swn0-just-say-yes-to-climate-lockdowns.html\",\n\t\t\t\tTitle: \"Just Say YES to Climate Lockdowns!\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/streamtape/streamtape.go\npackage streamtape\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/robertkrimen/otto\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"streamtape\", e)\n\textractors.Register(\"streamta\", e) // streamta.pe\n}\n\ntype extractor struct{}\n\n// New returns a StreamTape extractor\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tscripts := utils.MatchOneOf(html, `document.getElementById\\('norobotlink'\\).innerHTML = (.+?);`)\n\tif len(scripts) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvm := otto.New()\n\t_, err = vm.Run(fmt.Sprintf(\"var __VM__OUTPUT = %s\", scripts[1]))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvalue, err := vm.Get(\"__VM__OUTPUT\")\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu, err := value.ToString() // //streamtape.com/get_video?id=xx&expires=xx&ip=xx&token=xx\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu = fmt.Sprintf(\"https:%s&stream=1\", u)\n\n\t// get title\n\tvar title = \"StreamTape Video\"\n\ttitleMatch := utils.MatchOneOf(html,\n\t\t`\\<meta name=\"og:title\" content=\"(.*)\"\\>`)\n\tif len(titleMatch) >= 2 {\n\t\ttitle = titleMatch[1]\n\t}\n\n\tsize, err := request.Size(u, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlData := &extractors.Part{\n\t\tURL:  u,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize:  size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tURL:     u,\n\t\t\tSite:    \"StreamTape streamtape.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/streamtape/streamtape_test.go\npackage streamtape\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestStreamtape(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://streamtape.com/v/YKLDrr4X9gSvm9q/00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t\tTitle: \"00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/tangdou/tangdou.go\npackage tangdou\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"tangdou\", New())\n}\n\ntype extractor struct{}\n\n// New returns a tangdou extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nvar defaultHeader = map[string]string{\n\t\"Sec-Fetch-Dest\": \"document\",\n\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\"Sec-Fetch-Site\": \"cross-site\",\n\t\"Sec-GPC\":        \"1\",\n\t\"User-Agent\":     \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\treturn []*extractors.Data{tangdouDownload(url)}, nil\n}\n\n// tangdouDownload download function for single url\nfunc tangdouDownload(uri string) *extractors.Data {\n\thtml, err := request.Get(uri, uri, defaultHeader)\n\tif err != nil {\n\t\treturn extractors.EmptyData(uri, err)\n\t}\n\n\ttitles := utils.MatchOneOf(\n\t\thtml, `<div class=\"title\">(.+?)</div>`, `<meta name=\"description\" content=\"(.+?)\"`, `<title>(.+?)</title>`,\n\t)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed))\n\t}\n\ttitle := titles[1]\n\n\tvideoURLs := utils.MatchOneOf(\n\t\thtml, `video:'(.+?)'`, `video:\"(.+?)\"`, `<video[^>]*src=\"(.+?)\"`, `play_url:\\s*\"(.+?)\",`,\n\t)\n\n\tif len(videoURLs) < 2 {\n\t\treturn extractors.EmptyData(uri, errors.WithStack(extractors.ErrURLParseFailed))\n\t}\n\n\trealURL := strings.ReplaceAll(videoURLs[1], `\\u002F`, \"/\")\n\n\tsize, err := request.Size(realURL, uri)\n\tif err != nil {\n\t\treturn extractors.EmptyData(uri, err)\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  realURL,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt:  \"mp4\",\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: size,\n\t\t},\n\t}\n\n\treturn &extractors.Data{\n\t\tSite:    \"\u7cd6\u8c46\u5e7f\u573a\u821e tangdou.com\",\n\t\tTitle:   title,\n\t\tType:    extractors.DataTypeVideo,\n\t\tStreams: streams,\n\t\tURL:     uri,\n\t}\n}\n\n// Path: extractors/tangdou/tangdou_test.go\npackage tangdou\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestTangDou(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\targs     test.Args\n\t\tplaylist bool\n\t}{\n\t\t{\n\t\t\tname: \"need call share url first and get the signed video URL test and can get title from head's title tag\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://m.tangdou.com/play/1500676338077\",\n\t\t\t\tTitle: \"\u66b4\u7626\u51cf\u809a\u5b50\uff0c\u4e0d\u7528\u8dd1\u4e0d\u7528\u8df3\uff0c8\u5929\u7626\u4e8616\u65a4 \u6b63\u9762\u6f14\u793a \u80cc\u9762\u6f14\u793a \u5206\u89e3\u6559\u5b66__\u5e7f\u573a\u821e_\u7cd6\u8c46\u5e7f\u573a\u821e-\u7cd6\u8c46\u89c6\u9891\",\n\t\t\t\tSize:  62258444,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar (\n\t\t\t\tdata []*extractors.Data\n\t\t\t\terr  error\n\t\t\t)\n\t\t\tif tt.playlist {\n\t\t\t\t// playlist mode\n\t\t\t\t_, err = New().Extract(tt.args.URL, extractors.Options{\n\t\t\t\t\tPlaylist:     true,\n\t\t\t\t\tThreadNumber: 9,\n\t\t\t\t})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t} else {\n\t\t\t\tdata, err = New().Extract(tt.args.URL, extractors.Options{})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t\ttest.Check(t, tt.args, data[0])\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: extractors/tiktok/tiktok.go\npackage tiktok\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n)\n\nfunc init() {\n\textractors.Register(\"tiktok\", New())\n}\n\ntype extractor struct{}\n\n// New returns a tiktok extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, map[string]string{\n\t\t// tiktok require a user agent\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlMatcherRegExp := regexp.MustCompile(`\"downloadAddr\":\\s*\"([^\"]+)\"`)\n\n\tdownloadURLMatcher := urlMatcherRegExp.FindStringSubmatch(html)\n\n\tif len(downloadURLMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvideoURL := strings.ReplaceAll(downloadURLMatcher[1], `\\u002F`, \"/\")\n\n\ttitleMatcherRegExp := regexp.MustCompile(`<title[^>]*>([^<]+)</title>`)\n\n\ttitleMatcher := titleMatcherRegExp.FindStringSubmatch(html)\n\n\tif len(titleMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\ttitle := titleMatcher[1]\n\n\ttitleArr := strings.Split(title, \"|\")\n\n\tif len(titleArr) == 1 {\n\t\ttitle = titleArr[0]\n\t} else {\n\t\ttitle = strings.TrimSpace(strings.Join(titleArr[:len(titleArr)-1], \"|\"))\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\n\tsize, err := request.Size(videoURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoURL,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize:  size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"TikTok tiktok.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/tiktok/tiktok_test.go\npackage tiktok\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.tiktok.com/@ginjiro_koyama/video/7164293510617763073?is_copy_url=1&is_from_webapp=v1\",\n\t\t\t\tTitle: \"\u30a4\u30b1\u3059\u304e\u305fXOXO#xoxo #repezenfoxx #\u80cc\u4e2d\u7537 #kfam #yoshiki\u3055\u3093\u3092\u6ce3\u304b\u305b\u305f\u30c1\u30fc\u30e0 @K fam @\u3010Repezen Foxx\u3011\ud83e\udd8a\",\n\t\t\t\tSize:  4356253,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.tiktok.com/@enhypen/video/7165445991238356225?is_copy_url=1&is_from_webapp=v1\",\n\t\t\t\tTitle: \"\uae5c\uc9dd \ud1f4\uc7a5 \ud83d\udc4b #ENHYPEN #SUNGHOON #NI_KI #Make_the_change\",\n\t\t\t\tSize:  3848307,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/tumblr/tumblr.go\npackage tumblr\n\nimport (\n\t\"encoding/json\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"tumblr\", New())\n}\n\ntype imageList struct {\n\tList []string `json:\"@list\"`\n}\n\ntype tumblrImageList struct {\n\tImage imageList `json:\"image\"`\n}\n\ntype tumblrImage struct {\n\tImage string `json:\"image\"`\n}\n\nfunc genURLData(url, referer string) (*extractors.Part, int64, error) {\n\tsize, err := request.Size(url, referer)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\t_, ext, err := utils.GetNameAndExt(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\treturn &extractors.Part{\n\t\tURL:  url,\n\t\tSize: size,\n\t\tExt:  ext,\n\t}, size, nil\n}\n\nfunc tumblrImageDownload(url, html, title string) ([]*extractors.Data, error) {\n\tjsonStrings := utils.MatchOneOf(\n\t\thtml, `<script type=\"application/ld\\+json\">\\s*(.+?)</script>`,\n\t)\n\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := jsonStrings[1]\n\n\tvar totalSize int64\n\turls := make([]*extractors.Part, 0, 1)\n\tif strings.Contains(jsonString, `\"image\":{\"@list\"`) {\n\t\t// there are two data structures in the same field(image)\n\t\tvar imageList tumblrImageList\n\t\tif err := json.Unmarshal([]byte(jsonString), &imageList); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfor _, u := range imageList.Image.List {\n\t\t\turlData, size, err := genURLData(u, url)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\ttotalSize += size\n\t\t\turls = append(urls, urlData)\n\t\t}\n\t} else {\n\t\tvar image tumblrImage\n\t\tif err := json.Unmarshal([]byte(jsonString), &image); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData, size, err := genURLData(image.Image, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\ttotalSize = size\n\t\turls = append(urls, urlData)\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urls,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Tumblr tumblr.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\nfunc tumblrVideoDownload(url, html, title string) ([]*extractors.Data, error) {\n\tvideoURLs := utils.MatchOneOf(html, `<iframe src='(.+?)'`)\n\tif videoURLs == nil || len(videoURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvideoURL := videoURLs[1]\n\n\tif !strings.Contains(videoURL, \"tumblr.com/video\") {\n\t\treturn nil, errors.New(\"lux doesn't support this URL right now\")\n\t}\n\tvideoHTML, err := request.Get(videoURL, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\trealURLs := utils.MatchOneOf(videoHTML, `source src=\"(.+?)\"`)\n\tif realURLs == nil || len(realURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\trealURL := realURLs[1]\n\n\turlData, size, err := genURLData(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize:  size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Tumblr tumblr.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\ntype extractor struct{}\n\n// New returns a tumblr extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\t// get the title\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitle := parser.Title(doc)\n\tif strings.Contains(html, \"<iframe src=\") {\n\t\t// Data\n\t\treturn tumblrVideoDownload(url, html, title)\n\t}\n\t// Image\n\treturn tumblrImageDownload(url, html, title)\n}\n\n// Path: extractors/tumblr/tumblr_test.go\npackage tumblr\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"image test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"http://fuckyeah-fx.tumblr.com/post/170392654141/180202-%E5%AE%8B%E8%8C%9C\",\n\t\t\t\tTitle: \"f(x)\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"http://therealautoblog.tumblr.com/post/171623222197/paganis-new-projects-huayra-successor-with\",\n\t\t\t\tTitle: \"Autoblog \u2022 Pagani\u2019s new projects: Huayra successor with...\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://boomgoestheprower.tumblr.com/post/174127507696\",\n\t\t\t\tTitle: \"Out of Context Sonic Boom\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/twitter/twitter.go\npackage twitter\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"twitter\", New())\n}\n\ntype twitter struct {\n\tTrack struct {\n\t\tURL string `json:\"playbackUrl\"`\n\t} `json:\"track\"`\n\tTweetID  string\n\tUsername string\n}\n\ntype extractor struct{}\n\n// New returns a twitter extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tusernames := utils.MatchOneOf(html, `property=\"og:title\"\\s+content=\"(.+)\"`)\n\tif usernames == nil || len(usernames) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tusername := usernames[1]\n\n\ttweetIDs := utils.MatchOneOf(url, `(status|statuses)/(\\d+)`)\n\tif tweetIDs == nil || len(tweetIDs) < 3 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttweetID := tweetIDs[2]\n\n\tapi := fmt.Sprintf(\n\t\t\"https://api.twitter.com/1.1/videos/tweet/config/%s.json\", tweetID,\n\t)\n\theaders := map[string]string{\n...\n", "needle_token_start": 8248, "needle_token_end": 8437, "code_context_ntokens": 15091, "output": ["```go\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.qq.com/x/page/n0687peq62x.html\",\n\t\t\t\tTitle:   \"\u4e16\u754c\u676f\u7b2c\u4e00\u671f\uff1a100\u79d2\u901f\u6210\uff01\u201c\u4f2a\u7403\u8ff7\u201d\u4e16\u754c\u676f\u751f\u5b58\u6307\u5357\",\n\t\t\t\tSize:    23759683,\n\t\t\t\tQuality: \"\u84dd\u5149;(1080P)\",\n\t\t\t},\n\t\t},\n\t\t// {\n\t\t// \tname: \"movie and vid test\",\n\t\t// \targs: test.Args{\n\t\t// \t\tURL:     \"https://v.qq.com/x/cover/e5qmd3z5jr0uigk.html\",\n\t\t// \t\tTitle:   \"\u8d4c\u4fa0\uff08\u7ca4\u8bed\u7248\uff09\",\n\t\t// \t\tSize:    1046910811,\n\t\t// \t\tQuality: \"\u8d85\u6e05;(720P)\",\n\t\t// \t},\n\t\t// },\n\t\t{\n\t\t\tname: \"fmt ID test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.qq.com/x/cover/2aya3ibdmft6vdw/e0765r4mwcr.html\",\n\t\t\t\tTitle:   \"\u300a\u5361\u8def\u91cc\u300b\u51fa\u5708\uff01\u5996\u5a06\u7537\u5b50\u6559\u5b66\u5e7f\u573a\u821e\u7248\uff0c\u5927\u5988\u8868\u60c5\u4eae\u4e86\uff01\",\n\t\t\t\tSize:    14112979,\n\t\t\t\tQuality: \"\u8d85\u6e05;(720P)\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n```"]}
{"repo": "iawia002/lux", "name": "TestReddit", "language": "go", "path": "extractors/reddit/reddit_test.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to validate the functionality of a module that extracts media data from specific URLs. It ensures that the module correctly identifies and processes different types of media (video, image, GIF) from a popular social media platform.\n2. **Input**: The function takes a set of predefined test cases, each containing a URL pointing to a specific media post on the social media platform and the expected title of that post.\n3. **Output**: The function does not return any values but checks if the media extraction from each URL matches the expected results specified in the test cases. Errors are reported if the outcomes do not align with expectations.\n4. **Procedure**: The function iterates over each test case, uses the media extraction module to process the URL, and then compares the extracted media data against the expected title and type. It verifies that the module can handle different media formats and correctly parse the necessary data from the platform.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/netease/netease_test.go\npackage netease\n\nimport (\n\t\"testing\"\n\n...\n// Path: extractors/pinterest/pinterest.go\npackage pinterest\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n)\n\nfunc init() {\n\textractors.Register(\"pinterest\", New())\n}\n\ntype extractor struct{}\n\n// New returns a pinterest extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, map[string]string{\n\t\t// pinterest require a user agent\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlMatcherRegExp := regexp.MustCompile(`\"contentUrl\":\"https:\\/\\/v1\\.pinimg\\.com\\/videos\\/mc\\/720p\\/[a-zA-Z0-9\\/]+\\.mp4`)\n\n\tdownloadURLMatcher := urlMatcherRegExp.FindStringSubmatch(html)\n\n\tif len(downloadURLMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvideoURL := strings.ReplaceAll(downloadURLMatcher[0], `\"contentUrl\":\"`, \"\")\n\n\ttitleMatcherRegExp := regexp.MustCompile(`<title[^>]*>([^<]+)</title>`)\n\n\ttitleMatcher := titleMatcherRegExp.FindStringSubmatch(html)\n\n\tif len(titleMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\ttitle := strings.ReplaceAll(strings.ReplaceAll(titleMatcher[0], \"<title>\", \"\"), \"</title>\", \"\")\n\n\ttitleArr := strings.Split(title, \"|\")\n\n\tif len(titleArr) > 0 {\n\t\ttitle = titleArr[0]\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\n\tsize, err := request.Size(videoURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoURL,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize:  size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Pinterest pinterest.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/pinterest/pinterest_test.go\npackage pinterest\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.pinterest.com/pin/creamy-cheesy-pretzel-bites-video--368450813272292084/\",\n\t\t\t\tTitle: \"Creamy Cheesy Pretzel Bites [Video] \",\n\t\t\t\tSize:  30247497,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.pinterest.com/pin/532198880988430823/\",\n\t\t\t\tTitle: \"Pin on TikTok ~ The world of food\",\n\t\t\t\tSize:  4676927,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/pixivision/pixivision.go\npackage pixivision\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"pixivision\", New())\n}\n\ntype extractor struct{}\n\n// New returns a pixivision extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitle, urls, err := parser.GetImages(html, \"am__work__illust  \", nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tparts := make([]*extractors.Part, 0, len(urls))\n\tfor _, u := range urls {\n\t\t_, ext, err := utils.GetNameAndExt(u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tsize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tparts = append(parts, &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t\tExt:  ext,\n\t\t})\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"pixivision pixivision.net\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/pixivision/pixivision_test.go\npackage pixivision\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.pixivision.net/zh/a/3271\",\n\t\t\t\tTitle: \"Don't ask me to choose! Tiny Breasts VS Huge Breasts\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/pornhub/pornhub.go\npackage pornhub\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/robertkrimen/otto\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"pornhub\", New())\n}\n\ntype pornhubData struct {\n\tDefaultQuality bool   `json:\"defaultQuality\"`\n\tFormat         string `json:\"format\"`\n\tVideoURL       string `json:\"videoUrl\"`\n\tQuality        string `json:\"quality\"`\n}\n\ntype extractor struct{}\n\n// New returns a pornhub extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\thtml := string(b)\n\n\tcookiesArr := make([]string, 0)\n\tcookies := res.Cookies()\n\n\tfor _, c := range cookies {\n\t\tcookiesArr = append(cookiesArr, c.Name+\"=\"+c.Value)\n\t}\n\n\tvar title string\n\tdesc := utils.MatchOneOf(html, `<span class=\"inlineFree\">(.+?)</span>`)\n\tif len(desc) > 1 {\n\t\ttitle = desc[1]\n\t} else {\n\t\ttitle = \"pornhub video\"\n\t}\n\n\treg, err := regexp.Compile(`<script\\b[^>]*>([\\s\\S]*?)</script>`)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(extractors.ErrInvalidRegularExpression)\n\t}\n\n\tmatchers := reg.FindAllStringSubmatch(html, -1)\n\tvar encryptedScript string\n\n\tfor _, scripts := range matchers {\n\t\tscript := scripts[1]\n\t\tif !strings.Contains(script, \"flashvars_\") {\n\t\t\tcontinue\n\t\t} else {\n\t\t\tencryptedScript = script\n\t\t\tbreak\n\t\t}\n\t}\n\n\tflashId := regexp.MustCompile(`flashvars_\\d+`).FindString(encryptedScript)\n\n\tvm := otto.New()\n\t_, err = vm.Run(`var playerObjList = {};` + encryptedScript + fmt.Sprintf(`;var __VM__OUTPUT = JSON.stringify(%s.mediaDefinitions)`, flashId))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvalue, err := vm.Get(\"__VM__OUTPUT\")\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttype MediaDefinition struct {\n\t\tFormat   string `json:\"format\"`\n\t\tVideoURL string `json:\"videoUrl\"`\n\t}\n\n\tmediaDefinitions := make([]MediaDefinition, 0)\n\n\tif str, err := value.ToString(); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t} else {\n\t\tif err := json.Unmarshal([]byte(str), &mediaDefinitions); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t}\n\n\tvar mp4MediaDefinition *MediaDefinition\n\n\tfor _, mediaDefinition := range mediaDefinitions {\n\t\tif mediaDefinition.Format == \"mp4\" {\n\t\t\tmp4MediaDefinition = &mediaDefinition\n\t\t}\n\t}\n\n\tif mp4MediaDefinition == nil {\n\t\treturn nil, errors.New(\"can not found media\")\n\t}\n\n\tresApi, err := request.Get(mp4MediaDefinition.VideoURL, mp4MediaDefinition.VideoURL, map[string]string{\n\t\t\"Cookie\": strings.Join(cookiesArr, \"; \"),\n\t})\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tpornhubs := make([]pornhubData, 0)\n\n\tif err := json.Unmarshal([]byte(resApi), &pornhubs); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, len(pornhubs))\n\n\tfor _, data := range pornhubs {\n\t\tsize, err := request.Size(data.VideoURL, data.VideoURL)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL:  data.VideoURL,\n\t\t\tSize: size,\n\t\t\tExt:  data.Format,\n\t\t}\n\n\t\tstreams[data.Quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: data.Quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Pornhub pornhub.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/pornhub/pornhub_test.go\npackage pornhub\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestPornhub(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.pornhub.com/view_video.php?viewkey=ph5cb5fc41c6ebd\",\n\t\t\t\tTitle: \"Must watch Milf drilled by the fireplace\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/qq/qq.go\npackage qq\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"slices\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"qq\", New())\n}\n\ntype qqVideoInfo struct {\n\tFl struct {\n\t\tFi []struct {\n\t\t\tID    int    `json:\"id\"`\n\t\t\tName  string `json:\"name\"`\n\t\t\tCname string `json:\"cname\"`\n\t\t\tFs    int64  `json:\"fs\"`\n\t\t} `json:\"fi\"`\n\t} `json:\"fl\"`\n\tVl struct {\n\t\tVi []struct {\n\t\t\tFn    string `json:\"fn\"`\n\t\t\tTi    string `json:\"ti\"`\n\t\t\tFvkey string `json:\"fvkey\"`\n\t\t\tCl    struct {\n\t\t\t\tFc int `json:\"fc\"`\n\t\t\t\tCi []struct {\n\t\t\t\t\tIdx int `json:\"idx\"`\n\t\t\t\t} `json:\"ci\"`\n\t\t\t} `json:\"cl\"`\n\t\t\tUl struct {\n\t\t\t\tUI []struct {\n\t\t\t\t\tURL string `json:\"url\"`\n\t\t\t\t} `json:\"ui\"`\n\t\t\t} `json:\"ul\"`\n\t\t} `json:\"vi\"`\n\t} `json:\"vl\"`\n\tMsg string `json:\"msg\"`\n}\n\ntype qqKeyInfo struct {\n\tKey string `json:\"key\"`\n}\n\nconst qqPlayerVersion string = \"3.2.19.333\"\n\nfunc getVinfo(vid, defn, refer string) (qqVideoInfo, error) {\n\thtml, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"http://vv.video.qq.com/getinfo?otype=json&platform=11&defnpayver=1&appver=%s&defn=%s&vid=%s\",\n\t\t\tqqPlayerVersion, defn, vid,\n\t\t), refer, nil,\n\t)\n\tif err != nil {\n\t\treturn qqVideoInfo{}, err\n\t}\n\tjsonStrings := utils.MatchOneOf(html, `QZOutputJson=(.+);$`)\n\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\treturn qqVideoInfo{}, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tjsonString := jsonStrings[1]\n\tvar data qqVideoInfo\n\tif err = json.Unmarshal([]byte(jsonString), &data); err != nil {\n\t\treturn qqVideoInfo{}, err\n\t}\n\treturn data, nil\n}\n\nfunc genStreams(vid, cdn string, data qqVideoInfo) (map[string]*extractors.Stream, error) {\n\tstreams := make(map[string]*extractors.Stream)\n\tvar vkey string\n\t// number of fragments\n\tvar clips int\n\n\tfor _, fi := range data.Fl.Fi {\n\t\tvar fmtIDPrefix string\n\t\tvar fns []string\n\t\tif slices.Contains([]string{\"shd\", \"fhd\"}, fi.Name) {\n\t\t\tfmtIDPrefix = \"p\"\n\t\t\tfmtIDName := fmt.Sprintf(\"%s%d\", fmtIDPrefix, fi.ID%10000)\n\t\t\tfns = []string{strings.Split(data.Vl.Vi[0].Fn, \".\")[0], fmtIDName, \"mp4\"}\n\t\t\tif len(fns) > 3 {\n\t\t\t\t// delete ID part\n\t\t\t\t// e0765r4mwcr.2.mp4 -> e0765r4mwcr.mp4\n\t\t\t\tfns = append(fns[:1], fns[2:]...)\n\t\t\t}\n\t\t\tclips = data.Vl.Vi[0].Cl.Fc\n\t\t\tif clips == 0 {\n\t\t\t\tclips = 1\n\t\t\t}\n\t\t} else {\n\t\t\ttmpData, err := getVinfo(vid, fi.Name, cdn)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tfns = strings.Split(tmpData.Vl.Vi[0].Fn, \".\")\n\t\t\tif len(fns) >= 3 && utils.MatchOneOf(fns[1], `^p(\\d{3})$`) != nil {\n\t\t\t\tfmtIDPrefix = \"p\"\n\t\t\t}\n\t\t\tclips = tmpData.Vl.Vi[0].Cl.Fc\n\t\t\tif clips == 0 {\n\t\t\t\tclips = 1\n\t\t\t}\n\t\t}\n\n\t\tvar urls []*extractors.Part\n\t\tvar totalSize int64\n\t\tvar filename string\n\t\tfor part := 1; part < clips+1; part++ {\n\t\t\t// Multiple fragments per streams\n\t\t\tif fmtIDPrefix == \"p\" {\n\t\t\t\tif len(fns) < 4 {\n\t\t\t\t\t// If the number of fragments > 0, the filename needs to add the number of fragments\n\t\t\t\t\t// n0687peq62x.p709.mp4 -> n0687peq62x.p709.1.mp4\n\t\t\t\t\tfns = append(fns[:2], append([]string{strconv.Itoa(part)}, fns[2:]...)...)\n\t\t\t\t} else {\n\t\t\t\t\tfns[2] = strconv.Itoa(part)\n\t\t\t\t}\n\t\t\t}\n\t\t\tfilename = strings.Join(fns, \".\")\n\t\t\thtml, err := request.Get(\n\t\t\t\tfmt.Sprintf(\n\t\t\t\t\t\"http://vv.video.qq.com/getkey?otype=json&platform=11&appver=%s&filename=%s&format=%d&vid=%s\",\n\t\t\t\t\tqqPlayerVersion, filename, fi.ID, vid,\n\t\t\t\t), \"\", nil,\n\t\t\t)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tjsonStrings := utils.MatchOneOf(html, `QZOutputJson=(.+);$`)\n\t\t\tif jsonStrings == nil || len(jsonStrings) < 2 {\n\t\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t\t}\n\t\t\tjsonString := jsonStrings[1]\n\n\t\t\tvar keyData qqKeyInfo\n\t\t\tif err = json.Unmarshal([]byte(jsonString), &keyData); err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\n\t\t\tvkey = keyData.Key\n\t\t\tif vkey == \"\" {\n\t\t\t\tvkey = data.Vl.Vi[0].Fvkey\n\t\t\t}\n\t\t\trealURL := fmt.Sprintf(\"%s%s?vkey=%s\", cdn, filename, vkey)\n\t\t\tsize, err := request.Size(realURL, cdn)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turlData := &extractors.Part{\n\t\t\t\tURL:  realURL,\n\t\t\t\tSize: size,\n\t\t\t\tExt:  \"mp4\",\n\t\t\t}\n\t\t\turls = append(urls, urlData)\n\t\t\ttotalSize += size\n\t\t}\n\t\tstreams[fi.Name] = &extractors.Stream{\n\t\t\tParts:   urls,\n\t\t\tSize:    totalSize,\n\t\t\tQuality: fi.Cname,\n\t\t}\n\t}\n\treturn streams, nil\n}\n\ntype extractor struct{}\n\n// New returns a qq extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvids := utils.MatchOneOf(url, `vid=(\\w+)`, `/(\\w+)\\.html`)\n\tif vids == nil || len(vids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvid := vids[1]\n\n\tif len(vid) != 11 {\n\t\tu, err := request.Get(url, url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tvids = utils.MatchOneOf(\n\t\t\tu, `vid=(\\w+)`, `vid:\\s*[\"'](\\w+)`, `vid\\s*=\\s*[\"']\\s*(\\w+)`,\n\t\t)\n\t\tif vids == nil || len(vids) < 2 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\t\tvid = vids[1]\n\t}\n\n\tdata, err := getVinfo(vid, \"shd\", url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// API request error\n\tif data.Msg != \"\" {\n\t\treturn nil, errors.New(data.Msg)\n\t}\n\tcdn := data.Vl.Vi[0].Ul.UI[0].URL\n\tstreams, err := genStreams(vid, cdn, data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u817e\u8baf\u89c6\u9891 v.qq.com\",\n\t\t\tTitle:   data.Vl.Vi[0].Ti,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/qq/qq_test.go\npackage qq\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.qq.com/x/page/n0687peq62x.html\",\n\t\t\t\tTitle:   \"\u4e16\u754c\u676f\u7b2c\u4e00\u671f\uff1a100\u79d2\u901f\u6210\uff01\u201c\u4f2a\u7403\u8ff7\u201d\u4e16\u754c\u676f\u751f\u5b58\u6307\u5357\",\n\t\t\t\tSize:    23759683,\n\t\t\t\tQuality: \"\u84dd\u5149;(1080P)\",\n\t\t\t},\n\t\t},\n\t\t// {\n\t\t// \tname: \"movie and vid test\",\n\t\t// \targs: test.Args{\n\t\t// \t\tURL:     \"https://v.qq.com/x/cover/e5qmd3z5jr0uigk.html\",\n\t\t// \t\tTitle:   \"\u8d4c\u4fa0\uff08\u7ca4\u8bed\u7248\uff09\",\n\t\t// \t\tSize:    1046910811,\n\t\t// \t\tQuality: \"\u8d85\u6e05;(720P)\",\n\t\t// \t},\n\t\t// },\n\t\t{\n\t\t\tname: \"fmt ID test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.qq.com/x/cover/2aya3ibdmft6vdw/e0765r4mwcr.html\",\n\t\t\t\tTitle:   \"\u300a\u5361\u8def\u91cc\u300b\u51fa\u5708\uff01\u5996\u5a06\u7537\u5b50\u6559\u5b66\u5e7f\u573a\u821e\u7248\uff0c\u5927\u5988\u8868\u60c5\u4eae\u4e86\uff01\",\n\t\t\t\tSize:    14112979,\n\t\t\t\tQuality: \"\u8d85\u6e05;(720P)\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/reddit/reddit.go\npackage reddit\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"reddit\", New())\n}\n\nconst (\n\treferer  = \"https://www.reddit.com\"\n\tsiteName = \"Reddit reddit.com\"\n\n\tredditMP4API = \"https://v.redd.it/\"\n\tredditIMGAPI = \"https://i.redd.it/\"\n\taudioURLPart = \"/DASH_audio.mp4\"\n)\n\nvar resMap = map[string]string{\n\t\"720p\": \"/DASH_720.mp4\",\n\t\"480p\": \"/DASH_480.mp4\",\n\t\"360p\": \"/DASH_360.mp4\",\n\t\"240p\": \"/DASH_240.mp4\",\n\t\"220p\": \"/DASH_220.mp4\",\n}\n\ntype extractor struct{}\n\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, referer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// set thread number to 1 manually to avoid http 412 error\n\toption.ThreadNumber = 1\n\n\ttitle := utils.MatchOneOf(html, `<title>(.+?)<\\/title>`)[1]\n\n\tif utils.MatchOneOf(html, `meta property=\"og:video\" content=.*HLSPlaylist`) != nil {\n\t\tmp4URL := utils.MatchOneOf(html, `https://v.redd.it/(.+?)/HLSPlaylist`)[1]\n\t\tif mp4URL == \"\" {\n\t\t\treturn nil, errors.New(\"can't match mp4 content downloadable url\")\n\t\t}\n\n\t\taudioURL := fmt.Sprintf(\"%s%s%s\", redditMP4API, mp4URL, audioURLPart)\n\t\tsize, err := request.Size(audioURL, referer)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\taudioPart := &extractors.Part{\n\t\t\tURL:  audioURL,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp3\",\n\t\t}\n\n\t\tstreams := make(map[string]*extractors.Stream, len(resMap))\n\t\tfor res, urlParts := range resMap {\n\t\t\tresURL := fmt.Sprintf(\"%s%s%s\", redditMP4API, mp4URL, urlParts)\n\t\t\tsize, err := request.Size(resURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tstreams[res] = &extractors.Stream{\n\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t{\n\t\t\t\t\t\tURL:  resURL,\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\tExt:  \"mp4\",\n\t\t\t\t\t},\n\t\t\t\t\taudioPart,\n\t\t\t\t},\n\t\t\t\tSize:    size + audioPart.Size,\n\t\t\t\tQuality: res,\n\t\t\t\tNeedMux: true,\n\t\t\t}\n\t\t}\n\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite:    siteName,\n\t\t\t\tTitle:   title,\n\t\t\t\tType:    extractors.DataTypeVideo,\n\t\t\t\tStreams: streams,\n\t\t\t\tURL:     url,\n\t\t\t},\n\t\t}, nil\n\t} else if utils.MatchOneOf(html, `<meta property=\"og:type\" content=\"image\"/>`) != nil {\n\t\tvar imgURL string\n\t\tvar size int64\n\t\tif utils.MatchOneOf(html, `content\":\"https:\\/\\/i.redd.it\\/(.+?)\",\"type\":\"image\"`) != nil {\n\t\t\timgURL = redditIMGAPI + utils.MatchOneOf(html, `content\":\"https:\\/\\/i.redd.it\\/(.+?)\",\"type\":\"image\"`)[1]\n\t\t\tsize, err = request.Size(imgURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t} else {\n\t\t\timgURL = utils.MatchOneOf(html, `content\":\"(.+?)\",\"type\":\"image\"`)[1]\n\t\t\timgURL = strings.ReplaceAll(imgURL, \"auto=webp\\\\u0026s\", \"auto=webp&s\")\n\t\t\tsize, err = request.Size(imgURL, referer)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t}\n\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite:  siteName,\n\t\t\t\tTitle: title,\n\t\t\t\tType:  extractors.DataTypeImage,\n\t\t\t\tStreams: map[string]*extractors.Stream{\n\t\t\t\t\t\"default\": {\n\t\t\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tURL:  imgURL,\n\t\t\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\t\t\tExt:  \"jpg\",\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tURL: url,\n\t\t\t},\n\t\t}, nil\n\t} else if utils.MatchOneOf(html, `https:\\/\\/preview\\.redd\\.it\\/.*gif`) != nil {\n\t\tgifURL := utils.MatchOneOf(html, `https:\\/\\/preview\\.redd\\.it\\/.*?\\.gif\\?format=mp4.*?\"`)[0]\n\t\tif gifURL == \"\" {\n\t\t\treturn nil, errors.New(\"can't match gif content downloadable url\")\n\t\t}\n\n\t\tgifURL = strings.ReplaceAll(gifURL, \"&amp;\", \"&\")\n\t\tgifURL = strings.ReplaceAll(gifURL, \"\\\"\", \"\")\n\n\t\tsize, err := request.Size(gifURL, \"reddit.com\")\n\t\tif err != nil {\n\t\t\treturn nil, errors.New(\"can't get video size\")\n\t\t}\n\n\t\tstreams := map[string]*extractors.Stream{\n\t\t\t\"default\": {\n\t\t\t\tParts: []*extractors.Part{\n\t\t\t\t\t{\n\t\t\t\t\t\tURL:  gifURL,\n\t\t\t\t\t\tSize: size,\n\t\t\t\t\t\tExt:  \"mp4\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tSize: size,\n\t\t\t},\n\t\t}\n\t\treturn []*extractors.Data{\n\t\t\t{\n\t\t\t\tSite:    siteName,\n\t\t\t\tTitle:   title,\n\t\t\t\tType:    extractors.DataTypeVideo,\n\t\t\t\tStreams: streams,\n\t\t\t\tURL:     url,\n\t\t\t},\n\t\t}, nil\n\t}\n\n\treturn nil, fmt.Errorf(\"unable to handle url: %s\", url)\n}\n\n// Path: extractors/reddit/reddit_test.go\npackage reddit\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\n\nfunc TestReddit(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 0\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/space/comments/uj8sod/a_couple_of_days_ago_i_visited_this_place_an/\",\n\t\t\t\tTitle: \"A couple of days ago I visited this place. An abandoned space shuttle : space\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/DotA2/comments/uq012r/til_how_useful_hurricane_bird_is/\",\n\t\t\t\tTitle: \"TIL how useful hurricane bird is : DotA2\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/ProgrammerHumor/comments/uqovco/my_code_works/\",\n\t\t\t\tTitle: \"My code works : ProgrammerHumor\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 3\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/AnimatedPixelArt/comments/uomu32/animation_for_astral_ascent/\",\n\t\t\t\tTitle: \"Animation for Astral Ascent : AnimatedPixelArt\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 4\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/linuxmemes/comments/v1a4wh/please_olive_do_something/\",\n\t\t\t\tTitle: \"Please Olive, do something... : linuxmemes\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 5\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.reddit.com/r/gaming/comments/v27m79/skyrim_probably/\",\n\t\t\t\tTitle: \"Skyrim, probably : gaming\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/rumble/rumble.go\npackage rumble\n\nimport (\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"path\"\n\t\"regexp\"\n\t\"strconv\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"rumble\", New())\n}\n\ntype extractor struct{}\n\n// New returns a rumble extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\ntype rumbleData struct {\n\tFormat       string `json:\"format\"`\n\tName         string `json:\"name\"`\n\tEmbedURL     string `json:\"embedUrl\"`\n\tThumbnailURL string `json:\"thumbnailUrl\"`\n\tType         string `json:\"@type\"`\n\tVideoURL     string `json:\"videoUrl\"`\n\tQuality      string `json:\"quality\"`\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\thtml := string(b)\n\tvar title string\n\tmatchTitle := utils.MatchOneOf(html, `<title>(.+?)</title>`)\n\tif len(matchTitle) > 1 {\n\t\ttitle = matchTitle[1]\n\t} else {\n\t\ttitle = \"rumble video\"\n\t}\n\n\tpayload, err := readPayload(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvideoID, err := getVideoID(payload.EmbedURL)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams, err := fetchVideoQuality(videoID)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Rumble rumble.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Read JSON object from the video webpage\nfunc readPayload(html string) (*rumbleData, error) {\n\tmatchPayload := utils.MatchOneOf(html, `\\<script\\stype=\"?application\\/ld\\+json\"?\\>(.+?)\\<\\/script>`)\n\tif len(matchPayload) < 1 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLQueryParamsParseFailed)\n\t}\n\n\trumbles := make([]rumbleData, 0)\n\tif err := json.Unmarshal([]byte(matchPayload[1]), &rumbles); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tfor _, it := range rumbles {\n\t\tif it.Type == \"VideoObject\" {\n\t\t\treturn &it, nil\n\t\t}\n\t}\n\n\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n}\n\nfunc getVideoID(embedURL string) (string, error) {\n\tu, err := url.Parse(embedURL)\n\tif err != nil {\n\t\treturn \"\", errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\treturn path.Base(u.Path), nil\n}\n\n// Rumble response contains the streams in `rumbleStreams`\ntype rumbleResponse struct {\n\tStreams *json.RawMessage `json:\"ua\"`\n}\n\n// Common video meta data\ntype streamInfo struct {\n\tURL  string `json:\"url\"`\n\tMeta struct {\n\t\tBitrate uint16 `json:\"bitrate\"`\n\t\tSize    int64  `json:\"size\"`\n\t\tWidth   uint16 `json:\"w\"`\n\t\tHeight  uint16 `json:\"h\"`\n\t} `json:\"meta\"`\n}\n\n// common video qualities for `mp4`, `webm`\ntype videoQualities struct {\n\tQ240  struct{ streamInfo } `json:\"240\"`\n\tQ360  struct{ streamInfo } `json:\"360\"`\n\tQ480  struct{ streamInfo } `json:\"480\"`\n\tQ720  struct{ streamInfo } `json:\"720\"`\n\tQ1080 struct{ streamInfo } `json:\"1080\"`\n\tQ1440 struct{ streamInfo } `json:\"1440\"`\n\tQ2160 struct{ streamInfo } `json:\"2160\"`\n\tQ2161 struct{ streamInfo } `json:\"2161\"`\n}\n\n// Video payload for adaptive stream and different qualities\ntype rumbleStreams struct {\n\tFMp4 struct {\n\t\tvideoQualities\n\t} `json:\"mp4\"`\n\tFWebm struct {\n\t\tvideoQualities\n\t} `json:\"webm\"`\n\tFHLS struct {\n\t\tQAuto struct{ streamInfo } `json:\"auto\"`\n\t} `json:\"hls\"`\n}\n\n// Unmarshall the video response\n// Some properties like `mp4`, `webm` are either array or an object\nfunc (r *rumbleStreams) UnmarshalJSON(b []byte) error {\n\tvar resp *rumbleResponse\n\tif err := json.Unmarshal(b, &resp); err != nil {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Get individual stream from the response\n\tvar obj map[string]*json.RawMessage\n\tif err := json.Unmarshal(*resp.Streams, &obj); err != nil {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tif v, ok := obj[\"mp4\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FMp4)\n\t}\n\tif v, ok := obj[\"webm\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FWebm)\n\t}\n\tif v, ok := obj[\"hls\"]; ok {\n\t\t_ = json.Unmarshal(*v, &r.FHLS)\n\t}\n\n\treturn nil\n}\n\n// Use this to create all the streams for `mp4`, `webm`\nfunc (rs *rumbleStreams) makeAllVODStreams(m map[string]*extractors.Stream) {\n\tm[\"webm\"] = makeStreamMeta(\"480\", \"webm\", &rs.FWebm.Q480.streamInfo)\n\tm[\"240\"] = makeStreamMeta(\"240\", \"mp4\", &rs.FMp4.Q240.streamInfo)\n\tm[\"360\"] = makeStreamMeta(\"360\", \"mp4\", &rs.FMp4.Q360.streamInfo)\n\tm[\"480\"] = makeStreamMeta(\"480\", \"mp4\", &rs.FMp4.Q480.streamInfo)\n\tm[\"720\"] = makeStreamMeta(\"720\", \"mp4\", &rs.FMp4.Q720.streamInfo)\n\tm[\"1080\"] = makeStreamMeta(\"1080\", \"mp4\", &rs.FMp4.Q1080.streamInfo)\n\tm[\"1440\"] = makeStreamMeta(\"1440\", \"mp4\", &rs.FMp4.Q1440.streamInfo)\n\tm[\"2160\"] = makeStreamMeta(\"2160\", \"mp4\", &rs.FMp4.Q2160.streamInfo)\n\tm[\"2161\"] = makeStreamMeta(\"2161\", \"mp4\", &rs.FMp4.Q2161.streamInfo)\n}\n\nvar reResolution = regexp.MustCompile(`_(\\d{3,4})p\\/`) // ex. _720p/\n\n// Use this to create all the streams for live videos\nfunc (rs *rumbleStreams) makeAllLiveStreams(m map[string]*extractors.Stream) error {\n\tplaylists, err := utils.M3u8URLs(rs.FHLS.QAuto.URL)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tif len(playlists) == 0 {\n\t\treturn errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Find the highest resolution\n\tplaylistURL := playlists[0]\n\tmaxRes := 0\n\tfor _, x := range playlists {\n\t\tmatched := reResolution.FindStringSubmatch(x)\n\t\tif len(matched) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tres, err := strconv.Atoi(matched[1])\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tif maxRes < res {\n\t\t\tmaxRes = res\n\t\t\tplaylistURL = x\n\t\t}\n\t}\n\n\ttsURLs, err := utils.M3u8URLs(playlistURL)\n\tif err != nil {\n\t\treturn errors.WithStack(err)\n\t}\n\n\tvar parts []*extractors.Part\n\tfor _, x := range tsURLs {\n\t\tpart := &extractors.Part{\n\t\t\tURL:  x,\n\t\t\tSize: rs.FHLS.QAuto.streamInfo.Meta.Size,\n\t\t\tExt:  \"ts\",\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tm[\"hls\"] = &extractors.Stream{\n\t\tParts:   parts,\n\t\tSize:    rs.FHLS.QAuto.streamInfo.Meta.Size,\n\t\tQuality: strconv.Itoa(maxRes),\n\t}\n\n\treturn nil\n}\n\n// Request video formats and qualities\nfunc fetchVideoQuality(videoID string) (map[string]*extractors.Stream, error) {\n\treqURL := fmt.Sprintf(`https://rumble.com/embedJS/u3/?request=video&ver=2&v=%s&ext={\"ad_count\":null}&ad_wt=0`, videoID)\n\n\tres, err := request.Request(http.MethodGet, reqURL, nil, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar reader io.ReadCloser\n\tswitch res.Header.Get(\"Content-Encoding\") {\n\tcase \"gzip\":\n\t\treader, _ = gzip.NewReader(res.Body)\n\tcase \"deflate\":\n\t\treader = flate.NewReader(res.Body)\n\tdefault:\n\t\treader = res.Body\n\t}\n\tdefer reader.Close() // nolint\n\n\tb, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar rs rumbleStreams\n\tif err := json.Unmarshal(b, &rs); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 9)\n\trs.makeAllVODStreams(streams)\n\t_ = rs.makeAllLiveStreams(streams)\n\n\treturn streams, nil\n}\n\nfunc makeStreamMeta(q, ext string, info *streamInfo) *extractors.Stream {\n\turlMeta := &extractors.Part{\n\t\tURL:  info.URL,\n\t\tSize: info.Meta.Size,\n\t\tExt:  ext,\n\t}\n\n\treturn &extractors.Stream{\n\t\tParts:   []*extractors.Part{urlMeta},\n\t\tSize:    info.Meta.Size,\n\t\tQuality: q,\n\t}\n}\n\n// Path: extractors/rumble/rumble_test.go\npackage rumble\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestRumble(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://rumble.com/v24swn0-just-say-yes-to-climate-lockdowns.html\",\n\t\t\t\tTitle: \"Just Say YES to Climate Lockdowns!\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/streamtape/streamtape.go\npackage streamtape\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/robertkrimen/otto\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"streamtape\", e)\n\textractors.Register(\"streamta\", e) // streamta.pe\n}\n\ntype extractor struct{}\n\n// New returns a StreamTape extractor\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tscripts := utils.MatchOneOf(html, `document.getElementById\\('norobotlink'\\).innerHTML = (.+?);`)\n\tif len(scripts) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvm := otto.New()\n\t_, err = vm.Run(fmt.Sprintf(\"var __VM__OUTPUT = %s\", scripts[1]))\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvalue, err := vm.Get(\"__VM__OUTPUT\")\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu, err := value.ToString() // //streamtape.com/get_video?id=xx&expires=xx&ip=xx&token=xx\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tu = fmt.Sprintf(\"https:%s&stream=1\", u)\n\n\t// get title\n\tvar title = \"StreamTape Video\"\n\ttitleMatch := utils.MatchOneOf(html,\n\t\t`\\<meta name=\"og:title\" content=\"(.*)\"\\>`)\n\tif len(titleMatch) >= 2 {\n\t\ttitle = titleMatch[1]\n\t}\n\n\tsize, err := request.Size(u, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turlData := &extractors.Part{\n\t\tURL:  u,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\tstreams[\"default\"] = &extractors.Stream{\n\t\tParts: []*extractors.Part{urlData},\n\t\tSize:  size,\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tURL:     u,\n\t\t\tSite:    \"StreamTape streamtape.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/streamtape/streamtape_test.go\npackage streamtape\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestStreamtape(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://streamtape.com/v/YKLDrr4X9gSvm9q/00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t\tTitle: \"00819gb0ly1gd4okz3fqbg30b405jnpj.mp4\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/tangdou/tangdou.go\npackage tangdou\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"tangdou\", New())\n}\n\ntype extractor struct{}\n\n// New returns a tangdou extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nvar defaultHeader = map[string]string{\n\t\"Sec-Fetch-Dest\": \"document\",\n\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\"Sec-Fetch-Site\": \"cross-site\",\n\t\"Sec-GPC\":        \"1\",\n\t\"User-Agent\":     \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n}\n\n...\n", "needle_token_start": 9449, "needle_token_end": 10187, "code_context_ntokens": 15130, "output": ["```go\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.pinterest.com/pin/creamy-cheesy-pretzel-bites-video--368450813272292084/\",\n\t\t\t\tTitle: \"Creamy Cheesy Pretzel Bites [Video] \",\n\t\t\t\tSize:  30247497,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.pinterest.com/pin/532198880988430823/\",\n\t\t\t\tTitle: \"Pin on TikTok ~ The world of food\",\n\t\t\t\tSize:  4676927,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n```"]}
{"repo": "iawia002/lux", "name": "getVideoAudio", "language": "go", "path": "extractors/youtube/youtube.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: To retrieve the best audio format available for a given video and specified MIME type.\n2. **Input**: A video object and a string representing the MIME type.\n3. **Output**: Returns the highest quality audio format available or an error if no suitable format is found.\n4. **Procedure**: The function filters the video's formats to find those that match the specified MIME type and are of type 'audio'. It then sorts these audio formats by quality and returns the highest quality format.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/xiaohongshu/xiaohongshu.go\npackage xiaohongshu\n\nimport (\n\t\"encoding/json\"\n\tneturl \"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/config\"\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"xiaohongshu\", New())\n}\n\ntype extractor struct{}\n\n// New returns a xiaohognshu extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nconst mp4VideoType = \"mp4\"\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, config.FakeHeaders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// title\n\ttitles := utils.MatchOneOf(html, `<title>(.*?)</title>`)\n\tif titles == nil || len(titles) != 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrBodyParseFailed)\n\t}\n\ttitle := titles[1]\n\n\t// video url\n\turlsJSON := utils.MatchOneOf(html, `\"backupUrls\":(\\[.+?\\])`)\n...\n// Path: extractors/xiaohongshu/xiaohongshu_test.go\npackage xiaohongshu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.xiaohongshu.com/explore/64e9f1e50000000003023b3f\",\n\t\t\t\tTitle: \"\u4e03\u661f\u7ea7\u5927\u53a8\u90fd\u4e0d\u4f1a\u544a\u8bc9\u4f60\u7684\uff0c\u4e94\u82b1\u8089\u76848\u79cd\u505a\u6cd5\",\n\t\t\t\tSize:  59410194,\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/ximalaya/types.go\npackage ximalaya\n\ntype ximalayaData struct {\n\tStatusCode int `json:\"ret\"`\n\tData       struct {\n\t\tTrackId         int    `json:\"trackId\"`\n\t\tCanPlay         bool   `json:\"canPlay\"`\n\t\tSrc             string `json:\"src\"`\n\t\tXimiVipFreeType int    `json:\"ximiVipFreeType\"`\n\t\tSampleDuration  int    `json:\"sampleDuration\"`\n\t} `json:\"data\"`\n}\n\n// Path: extractors/ximalaya/ximalaya.go\npackage ximalaya\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"ximalaya\", New())\n}\n\ntype extractor struct{}\n\n// New returns a ximalaya extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t// get the title\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitle := parser.Title(doc)\n\n\titemIds := utils.MatchOneOf(url, `/sound/(\\d+)`)\n\tif len(itemIds) == 0 {\n\t\treturn nil, errors.New(\"unable to get audio ID\")\n\t}\n\tif itemIds == nil || len(itemIds) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\titemId := itemIds[len(itemIds)-1]\n\n\tjsonData, err := request.Get(\"https://www.ximalaya.com/revision/play/v1/audio?id=\"+itemId+\"&ptype=1\", url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar ximalaya ximalayaData\n\tif err = json.Unmarshal([]byte(jsonData), &ximalaya); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\trealURL := ximalaya.Data.Src\n\turlData := make([]*extractors.Part, 0)\n\ttotalSize, err := request.Size(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\t_, ext, err := utils.GetNameAndExt(realURL)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData = append(urlData, &extractors.Part{\n\t\tURL:  realURL,\n\t\tSize: totalSize,\n\t\tExt:  ext,\n\t})\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urlData,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u559c\u9a6c\u62c9\u96c5 ximalaya.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeAudio,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/ximalaya/ximalaya_test.go\npackage ximalaya\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.ximalaya.com/sound/211583675\",\n\t\t\t\tTitle: \"\u72fc\u7684\u773c\u775b\u4e3a\u4ec0\u4e48\u4f1a\u53d1\u5149 - \u5341\u4e07\u4e2a\u4e3a\u4ec0\u4e48\u3010\u5b9d\u5b9d\u5df4\u58eb\u767e\u79d1\u6545\u4e8b\u3011\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/xinpianchang/xinpianchang.go\npackage xinpianchang\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/itchyny/gojq\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n)\n\nfunc init() {\n\textractors.Register(\"xinpianchang\", New())\n}\n\ntype extractor struct{}\n\ntype Video struct {\n\tTitle     string `json:\"title\"`\n\tQualities []struct {\n\t\tQuality string `json:\"quality\"`\n\t\tSize    int64  `json:\"size\"`\n\t\tURL     string `json:\"url\"`\n\t\tExt     string `json:\"ext\"`\n\t} `json:\"qualities\"`\n}\n\n// New returns a xinpianchang extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\theaders := map[string]string{\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t}\n\n\thtml, err := request.Get(url, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tr1 := regexp.MustCompile(`vid = \"(.+?)\";`)\n\tr2 := regexp.MustCompile(`modeServerAppKey = \"(.+?)\";`)\n\n\tvid := r1.FindSubmatch([]byte(html))[1]\n\tappKey := r2.FindSubmatch([]byte(html))[1]\n\n\tvideo_url := fmt.Sprintf(\"https://mod-api.xinpianchang.com/mod/api/v2/media/%s?appKey=%s\", string(vid), string(appKey))\n\tbody, err := request.Get(video_url, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar m interface{}\n\terr = json.Unmarshal([]byte(body), &m)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tquery, err := gojq.Parse(\"{title: .data.title} + {qualities: [(.data.resource.progressive[] | {quality: .quality, size: .filesize, url: .url, ext: .mime})]}\")\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\titer := query.Run(m)\n\tvideo := Video{}\n\n\tfor {\n\t\tv, ok := iter.Next()\n\t\tif !ok {\n\t\t\tbreak\n\t\t}\n\t\tif err, ok := v.(error); ok {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tjsonbody, err := json.Marshal(v)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tif err := json.Unmarshal(jsonbody, &video); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\tfor _, quality := range video.Qualities {\n\t\tstreams[quality.Quality] = &extractors.Stream{\n\t\t\tSize:    quality.Size,\n\t\t\tQuality: quality.Quality,\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  quality.URL,\n\t\t\t\t\tSize: quality.Size,\n\t\t\t\t\tExt:  strings.Split(quality.Ext, \"/\")[1],\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u65b0\u7247\u573a xinpianchang.com\",\n\t\t\tTitle:   video.Title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/xinpianchang/xinpianchang_test.go\npackage xinpianchang\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.xinpianchang.com/a10880684?from=ArticlePageSimilar\",\n\t\t\t\tTitle:   \"\u8d85\u70ab\u9177\u89c6\u89c9\u7cfb\u521b\u610f\u77ed\u7247\u300a\u9057\u7559\u300b\",\n\t\t\t\tQuality: \"720p\",\n\t\t\t\tSize:    79595290,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/xvideos/xvideos.go\npackage xvideos\n\nimport (\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"xvideos\", New())\n}\n\nconst (\n\tlowFlag      = \"html5player.setVideoUrlLow('\"\n\tlowFinalFlag = `');\n\t    html5player.setVideoUrlHigh(`\n\thighFlag      = \"html5player.setVideoUrlHigh('\"\n\thighFinalFlag = `');\n\t    html5player.setVideoHLS(`\n\tqualityLow  = \"low\"\n\tqualityHigh = \"high\"\n)\n\nvar (\n\tlowFlagLength  = len(lowFlag)\n\thighFlagLength = len(highFlag)\n)\n\ntype src struct {\n\turl     string\n\tquality string\n}\n\nfunc getSrc(html string) []*src {\n\tvar wg sync.WaitGroup\n\twg.Add(4)\n\n\tstartIndexLow := 0\n\tgo func() {\n\t\tstartIndexLow = strings.Index(html, lowFlag)\n\t\tstartIndexLow += lowFlagLength\n\t\twg.Done()\n\t}()\n\tendIndexLow := 0\n\tgo func() {\n\t\tendIndexLow = strings.Index(html, lowFinalFlag)\n\t\twg.Done()\n\t}()\n\n\tstartIndexHigh := 0\n\tgo func() {\n\t\tstartIndexHigh = strings.Index(html, highFlag)\n\t\tstartIndexHigh += highFlagLength\n\t\twg.Done()\n\t}()\n\tendIndexHigh := 0\n\tgo func() {\n\t\tendIndexHigh = strings.Index(html, highFinalFlag)\n\t\twg.Done()\n\t}()\n\twg.Wait()\n\n\tvar srcs []*src\n\tif startIndexLow != -1 {\n\t\tsrcs = append(srcs, &src{\n\t\t\turl:     html[startIndexLow:endIndexLow],\n\t\t\tquality: qualityLow,\n\t\t})\n\t}\n\tif startIndexHigh != -1 {\n\t\tsrcs = append(srcs, &src{\n\t\t\turl:     html[startIndexHigh:endIndexHigh],\n\t\t\tquality: qualityHigh,\n\t\t})\n\t}\n\treturn srcs\n}\n\ntype extractor struct{}\n\n// New returns a xvideos extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tdesc := utils.MatchOneOf(html, `<title>(.+?)</title>`)\n\tif len(desc) > 1 {\n\t\ttitle = desc[1]\n\t} else {\n\t\ttitle = \"xvideos\"\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, len(getSrc(html)))\n\tfor _, src := range getSrc(html) {\n\t\tsize, err := request.Size(src.url, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL:  src.url,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[src.quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: src.quality,\n\t\t}\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"XVIDEOS xvideos.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/xvideos/xvideos_test.go\npackage xvideos\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestExtract(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.xvideos.com/video29018757/asian_chick_enjoying_sex_debut._hd_full_at_nanairo.co\",\n\t\t\t\tTitle: \"Asian chick enjoying sex debut&period; HD FULL at&colon; nanairo&period;co - XVIDEOS.COM\",\n\t\t\t\tSize:  16574766,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/yinyuetai/types.go\npackage yinyuetai\n\ntype yinyuetaiMvData struct {\n\tError     bool      `json:\"error\"`\n\tMessage   string    `json:\"message\"`\n\tVideoInfo videoInfo `json:\"videoInfo\"`\n}\n\ntype videoInfo struct {\n\tCoreVideoInfo coreVideoInfo `json:\"coreVideoInfo\"`\n}\n\ntype coreVideoInfo struct {\n\tArtistNames    string          `json:\"artistNames\"`\n\tDuration       int             `json:\"duration\"`\n\tError          bool            `json:\"error\"`\n\tErrorMsg       string          `json:\"errorMsg\"`\n\tVideoID        int             `json:\"videoID\"`\n\tVideoName      string          `json:\"videoName\"`\n\tVideoURLModels []videoURLModel `json:\"videoURLModels\"`\n}\n\ntype videoURLModel struct {\n\tBitrate          int    `json:\"bitrate\"`\n\tBitrateType      int    `json:\"bitrateType\"`\n\tFileSize         int64  `json:\"fileSize\"`\n\tMD5              string `json:\"md5\"`\n\tSHA1             string `json:\"sha1\"`\n\tQualityLevel     string `json:\"qualityLevel\"`\n\tQualityLevelName string `json:\"qualityLevelName\"`\n\tVideoURL         string `json:\"videoURL\"`\n}\n\n// Path: extractors/yinyuetai/yinyuetai.go\npackage yinyuetai\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"yinyuetai\", New())\n}\n\nconst yinyuetaiAPI = \"https://ext.yinyuetai.com/main/\"\n\nconst (\n\tactionGetMvInfo = \"get-h-mv-info\"\n)\n\nfunc genAPI(action string, param string) string {\n\treturn fmt.Sprintf(\"%s%s?json=true&%s\", yinyuetaiAPI, action, param)\n}\n\ntype extractor struct{}\n\n// New returns a yinyuetai extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvid := utils.MatchOneOf(\n\t\turl,\n\t\t`https?://v.yinyuetai.com/video/(\\d+)(?:\\?vid=\\d+)?`,\n\t\t`https?://v.yinyuetai.com/video/h5/(\\d+)(?:\\?vid=\\d+)?`,\n\t\t`https?://m2.yinyuetai.com/video.html\\?id=(\\d+)`,\n\t)\n\tif vid == nil || len(vid) < 2 {\n\t\treturn nil, errors.New(\"invalid url for yinyuetai\")\n\t}\n\tparams := fmt.Sprintf(\"videoId=%s\", vid[1])\n\t// generate api url\n\tapiURL := genAPI(actionGetMvInfo, params)\n\tvar err error\n\thtml, err := request.Get(apiURL, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\t// parse yinyuetai data\n\tdata := yinyuetaiMvData{}\n\tif err = json.Unmarshal([]byte(html), &data); err != nil {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\t// handle api error\n\tif data.Error {\n\t\treturn nil, errors.New(data.Message)\n\t}\n\tif data.VideoInfo.CoreVideoInfo.Error {\n\t\treturn nil, errors.New(data.VideoInfo.CoreVideoInfo.ErrorMsg)\n\t}\n\ttitle := data.VideoInfo.CoreVideoInfo.VideoName\n\tstreams := make(map[string]*extractors.Stream, len(data.VideoInfo.CoreVideoInfo.VideoURLModels))\n\t// set streams\n\tfor _, model := range data.VideoInfo.CoreVideoInfo.VideoURLModels {\n\t\turlData := &extractors.Part{\n\t\t\tURL:  model.VideoURL,\n\t\t\tSize: model.FileSize,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[model.QualityLevel] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    model.FileSize,\n\t\t\tQuality: model.QualityLevelName,\n\t\t}\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u97f3\u60a6\u53f0 yinyuetai.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/yinyuetai/yinyuetai_test.go\npackage yinyuetai\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"http://v.yinyuetai.com/video/3386385\",\n\t\t\t\tTitle:   \"\u4ec0\u4e48\u662f\u7231/ What is Love\",\n\t\t\t\tSize:    20028736,\n\t\t\t\tQuality: \"\u6d41\u7545\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/youku/youku.go\npackage youku\n\nimport (\n\t\"bytes\"\n\t\"crypto/hmac\"\n\t\"crypto/sha1\"\n\t\"encoding/base64\"\n\t\"encoding/binary\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\tnetURL \"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"youku\", New())\n}\n\ntype errorData struct {\n\tNote string `json:\"note\"`\n\tCode int    `json:\"code\"`\n}\n\ntype segs struct {\n\tSize int64  `json:\"size\"`\n\tURL  string `json:\"cdn_url\"`\n}\n\ntype stream struct {\n\tSize      int64  `json:\"size\"`\n\tWidth     int    `json:\"width\"`\n\tHeight    int    `json:\"height\"`\n\tSegs      []segs `json:\"segs\"`\n\tType      string `json:\"stream_type\"`\n\tAudioLang string `json:\"audio_lang\"`\n}\n\ntype youkuVideo struct {\n\tTitle string `json:\"title\"`\n}\n\ntype youkuShow struct {\n\tTitle string `json:\"title\"`\n}\n\ntype data struct {\n\tError  errorData  `json:\"error\"`\n\tStream []stream   `json:\"stream\"`\n\tVideo  youkuVideo `json:\"video\"`\n\tShow   youkuShow  `json:\"show\"`\n}\n\ntype youkuData struct {\n\tData data `json:\"data\"`\n}\n\nconst youkuReferer = \"https://v.youku.com\"\n\nfunc getAudioLang(lang string) string {\n\tvar youkuAudioLang = map[string]string{\n\t\t\"guoyu\": \"\u56fd\u8bed\",\n\t\t\"ja\":    \"\u65e5\u8bed\",\n\t\t\"yue\":   \"\u7ca4\u8bed\",\n\t}\n\ttranslate, ok := youkuAudioLang[lang]\n\tif !ok {\n\t\treturn lang\n\t}\n\treturn translate\n}\n\n// https://g.alicdn.com/player/ykplayer/0.5.61/youku-player.min.js\n// {\"0505\":\"interior\",\"050F\":\"interior\",\"0501\":\"interior\",\"0502\":\"interior\",\"0503\":\"interior\",\"0510\":\"adshow\",\"0512\":\"BDskin\",\"0590\":\"BDskin\"}\n\n// var ccodes = []string{\"0510\", \"0502\", \"0507\", \"0508\", \"0512\", \"0513\", \"0514\", \"0503\", \"0590\"}\n\nfunc youkuUps(vid string, option extractors.Options) (*youkuData, error) {\n\tvar (\n\t\turl   string\n\t\tutid  string\n\t\tutids []string\n\t\tdata  youkuData\n\t)\n\tif strings.Contains(option.Cookie, \"cna\") {\n\t\tutids = utils.MatchOneOf(option.Cookie, `cna=(.+?);`, `cna\\s+(.+?)\\s`, `cna\\s+(.+?)$`)\n\t} else {\n\t\theaders, err := request.Headers(\"http://log.mmstat.com/eg.js\", youkuReferer)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tsetCookie := headers.Get(\"Set-Cookie\")\n\t\tutids = utils.MatchOneOf(setCookie, `cna=(.+?);`)\n\t}\n\tif utids == nil || len(utids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tutid = utids[1]\n\n\t// https://g.alicdn.com/player/ykplayer/0.5.61/youku-player.min.js\n\t// grep -oE '\"[0-9a-zA-Z+/=]{256}\"' youku-player.min.js\n\tfor _, ccode := range []string{option.YoukuCcode} {\n\t\tif ccode == \"0103010102\" {\n\t\t\tutid = generateUtdid()\n\t\t}\n\t\turl = fmt.Sprintf(\n\t\t\t\"https://ups.youku.com/ups/get.json?vid=%s&ccode=%s&client_ip=192.168.1.1&client_ts=%d&utid=%s&ckey=%s\",\n\t\t\tvid, ccode, time.Now().Unix()/1000, netURL.QueryEscape(utid), netURL.QueryEscape(option.YoukuCkey),\n\t\t)\n\t\tif option.YoukuPassword != \"\" {\n\t\t\turl = fmt.Sprintf(\"%s&password=%s\", url, option.YoukuPassword)\n\t\t}\n\t\thtml, err := request.GetByte(url, youkuReferer, nil)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\t// data must be emptied before reassignment, otherwise it will contain the previous value(the 'error' data)\n\t\tdata = youkuData{}\n\t\tif err = json.Unmarshal(html, &data); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tif data.Data.Error == (errorData{}) {\n\t\t\treturn &data, nil\n\t\t}\n\t}\n\treturn &data, nil\n}\n\nfunc getBytes(val int32) []byte {\n\tvar buff bytes.Buffer\n\tbinary.Write(&buff, binary.BigEndian, val) // nolint\n\treturn buff.Bytes()\n}\n\nfunc hashCode(s string) int32 {\n\tvar result int32\n\tfor _, c := range s {\n\t\tresult = result*0x1f + c\n\t}\n\treturn result\n}\n\nfunc hmacSha1(key []byte, msg []byte) []byte {\n\tmac := hmac.New(sha1.New, key)\n\tmac.Write(msg) // nolint\n\treturn mac.Sum(nil)\n}\n\nfunc generateUtdid() string {\n\ttimestamp := int32(time.Now().Unix())\n\tvar buffer bytes.Buffer\n\tbuffer.Write(getBytes(timestamp - 60*60*8))\n\tbuffer.Write(getBytes(rand.Int31()))\n\tbuffer.WriteByte(0x03)\n\tbuffer.WriteByte(0x00)\n\timei := fmt.Sprintf(\"%d\", rand.Int31())\n\tbuffer.Write(getBytes(hashCode(imei)))\n\tdata := hmacSha1([]byte(\"d6fc3a4a06adbde89223bvefedc24fecde188aaa9161\"), buffer.Bytes())\n\tbuffer.Write(getBytes(hashCode(base64.StdEncoding.EncodeToString(data))))\n\treturn base64.StdEncoding.EncodeToString(buffer.Bytes())\n}\n\nfunc genData(youkuData data) map[string]*extractors.Stream {\n\tvar (\n\t\tstreamString string\n\t\tquality      string\n\t)\n\tstreams := make(map[string]*extractors.Stream, len(youkuData.Stream))\n\tfor _, stream := range youkuData.Stream {\n\t\tif stream.AudioLang == \"default\" {\n\t\t\tstreamString = stream.Type\n\t\t\tquality = fmt.Sprintf(\n\t\t\t\t\"%s %dx%d\", stream.Type, stream.Width, stream.Height,\n\t\t\t)\n\t\t} else {\n\t\t\tstreamString = fmt.Sprintf(\"%s-%s\", stream.Type, stream.AudioLang)\n\t\t\tquality = fmt.Sprintf(\n\t\t\t\t\"%s %dx%d %s\", stream.Type, stream.Width, stream.Height,\n\t\t\t\tgetAudioLang(stream.AudioLang),\n\t\t\t)\n\t\t}\n\n\t\text := strings.Split(\n\t\t\tstrings.Split(stream.Segs[0].URL, \"?\")[0],\n\t\t\t\".\",\n\t\t)\n\t\turls := make([]*extractors.Part, len(stream.Segs))\n\t\tfor index, data := range stream.Segs {\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL:  data.URL,\n\t\t\t\tSize: data.Size,\n\t\t\t\tExt:  ext[len(ext)-1],\n\t\t\t}\n\t\t}\n\t\tstreams[streamString] = &extractors.Stream{\n\t\t\tParts:   urls,\n\t\t\tSize:    stream.Size,\n\t\t\tQuality: quality,\n\t\t}\n\t}\n\treturn streams\n}\n\ntype extractor struct{}\n\n// New returns a youku extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvids := utils.MatchOneOf(\n\t\turl, `id_(.+?)\\.html`, `id_(.+)`,\n\t)\n\tif vids == nil || len(vids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvid := vids[1]\n\n\tyoukuData, err := youkuUps(vid, option)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif youkuData.Data.Error.Code != 0 {\n\t\treturn nil, errors.New(youkuData.Data.Error.Note)\n\t}\n\tstreams := genData(youkuData.Data)\n\tvar title string\n\tif youkuData.Data.Show.Title == \"\" || strings.Contains(\n\t\tyoukuData.Data.Video.Title, youkuData.Data.Show.Title,\n\t) {\n\t\ttitle = youkuData.Data.Video.Title\n\t} else {\n\t\ttitle = fmt.Sprintf(\"%s %s\", youkuData.Data.Show.Title, youkuData.Data.Video.Title)\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u4f18\u9177 youku.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/youku/youku_test.go\npackage youku\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"http://v.youku.com/v_show/id_XMzUzMjE3NDczNg==.html\",\n\t\t\t\tTitle:   \"\u8f66\u4e8b\u513f: \u667a\u80fd\u6c7d\u8f66\u5df2\u7ecf\u4e0d\u5728\u9065\u8fdc \u4e1c\u98ce\u98ce\u5149iX5\u53d1\u5e03\",\n\t\t\t\tSize:    22692900,\n\t\t\t\tQuality: \"mp4hd2v2 1280x720\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{\n\t\t\t\tYoukuCcode: \"0590\",\n\t\t\t})\n\t\t})\n\t}\n}\n\n// Path: extractors/youtube/youtube.go\npackage youtube\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"slices\"\n\t\"strconv\"\n\n\t\"github.com/kkdai/youtube/v2\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\te := New()\n\textractors.Register(\"youtube\", e)\n\textractors.Register(\"youtu\", e) // youtu.be\n}\n\nconst referer = \"https://www.youtube.com\"\n\ntype extractor struct {\n\tclient *youtube.Client\n}\n\n// New returns a youtube extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{\n\t\tclient: &youtube.Client{\n\t\t\tHTTPClient: &http.Client{\n\t\t\t\tTransport: &http.Transport{\n\t\t\t\t\tProxy: http.ProxyFromEnvironment,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tif !option.Playlist {\n\t\tvideo, err := e.client.GetVideo(url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\treturn []*extractors.Data{e.youtubeDownload(url, video)}, nil\n\t}\n\n\tplaylist, err := e.client.GetPlaylist(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tneedDownloadItems := utils.NeedDownloadList(option.Items, option.ItemStart, option.ItemEnd, len(playlist.Videos))\n\textractedData := make([]*extractors.Data, len(needDownloadItems))\n\twgp := utils.NewWaitGroupPool(option.ThreadNumber)\n\tdataIndex := 0\n\tfor index, videoEntry := range playlist.Videos {\n\t\tif !slices.Contains(needDownloadItems, index+1) {\n\t\t\tcontinue\n\t\t}\n\n\t\twgp.Add()\n\t\tgo func(index int, entry *youtube.PlaylistEntry, extractedData []*extractors.Data) {\n\t\t\tdefer wgp.Done()\n\t\t\tvideo, err := e.client.VideoFromPlaylistEntry(entry)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\textractedData[index] = e.youtubeDownload(url, video)\n\t\t}(dataIndex, videoEntry, extractedData)\n\t\tdataIndex++\n\t}\n\twgp.Wait()\n\treturn extractedData, nil\n}\n\n// youtubeDownload download function for single url\nfunc (e *extractor) youtubeDownload(url string, video *youtube.Video) *extractors.Data {\n\tstreams := make(map[string]*extractors.Stream, len(video.Formats))\n\taudioCache := make(map[string]*extractors.Part)\n\n\tfor i := range video.Formats {\n\t\tf := &video.Formats[i]\n\t\titag := strconv.Itoa(f.ItagNo)\n\t\tquality := f.MimeType\n\t\tif f.QualityLabel != \"\" {\n\t\t\tquality = fmt.Sprintf(\"%s %s\", f.QualityLabel, f.MimeType)\n\t\t}\n\n\t\tpart, err := e.genPartByFormat(video, f)\n\t\tif err != nil {\n\t\t\treturn extractors.EmptyData(url, err)\n\t\t}\n\t\tstream := &extractors.Stream{\n\t\t\tID:      itag,\n\t\t\tParts:   []*extractors.Part{part},\n\t\t\tQuality: quality,\n\t\t\tExt:     part.Ext,\n\t\t\tNeedMux: true,\n\t\t}\n\n\t\t// Unlike `url_encoded_fmt_stream_map`, all videos in `adaptive_fmts` have no sound,\n\t\t// we need download video and audio both and then merge them.\n\t\t// video format with audio:\n\t\t//   AudioSampleRate: \"44100\", AudioChannels: 2\n\t\t// video format without audio:\n\t\t//   AudioSampleRate: \"\", AudioChannels: 0\n\t\tif f.AudioChannels == 0 {\n\t\t\taudioPart, ok := audioCache[part.Ext]\n\t\t\tif !ok {\n\t\t\t\taudio, err := getVideoAudio(video, part.Ext)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn extractors.EmptyData(url, err)\n\t\t\t\t}\n\t\t\t\taudioPart, err = e.genPartByFormat(video, audio)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn extractors.EmptyData(url, err)\n\t\t\t\t}\n\t\t\t\taudioCache[part.Ext] = audioPart\n\t\t\t}\n\t\t\tstream.Parts = append(stream.Parts, audioPart)\n\t\t}\n\t\tstreams[itag] = stream\n\t}\n\n\treturn &extractors.Data{\n\t\tSite:    \"YouTube youtube.com\",\n\t\tTitle:   video.Title,\n\t\tType:    \"video\",\n\t\tStreams: streams,\n\t\tURL:     url,\n\t}\n}\n\nfunc (e *extractor) genPartByFormat(video *youtube.Video, f *youtube.Format) (*extractors.Part, error) {\n\text := getStreamExt(f.MimeType)\n\turl, err := e.client.GetStreamURL(video, f)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsize := f.ContentLength\n\tif size == 0 {\n\t\tsize, _ = request.Size(url, referer)\n\t}\n\treturn &extractors.Part{\n\t\tURL:  url,\n\t\tSize: size,\n\t\tExt:  ext,\n\t}, nil\n}\n\n\nfunc getVideoAudio(v *youtube.Video, mimeType string) (*youtube.Format, error) {\n\taudioFormats := v.Formats.Type(mimeType).Type(\"audio\")\n\tif len(audioFormats) == 0 {\n\t\treturn nil, errors.New(\"no audio format found after filtering\")\n\t}\n\taudioFormats.Sort()\n\treturn &audioFormats[0], nil\n}\n\nfunc getStreamExt(streamType string) string {\n\t// video/webm; codecs=\"vp8.0, vorbis\" --> webm\n\texts := utils.MatchOneOf(streamType, `(\\w+)/(\\w+);`)\n\tif exts == nil || len(exts) < 3 {\n\t\treturn \"\"\n\t}\n\treturn exts[2]\n}\n\n// Path: extractors/youtube/youtube_test.go\npackage youtube\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestYoutube(t *testing.T) {\n\ttests := []struct {\n\t\tname     string\n\t\targs     test.Args\n\t\tplaylist bool\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.youtube.com/watch?v=Gnbch2osEeo\",\n\t\t\t\tTitle: \"Multifandom Mashup 2017\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"signature test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.youtube.com/watch?v=ZtgzKBrU1GY\",\n\t\t\t\tTitle: \"Halo Infinite - E3 2019 - Discover Hope\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"playlist test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.youtube.com/watch?v=Lt2pwLxJxgA&list=PLIYAO-qLriEtYm7UcXPH3SOJxgqjwRrIw\",\n\t\t\t\tTitle: \"papi\u9171 - \u4f60\u6709\u9171\u5a76\u513f\u7684\u670b\u53cb\u5417\uff1f\",\n\t\t\t},\n\t\t\tplaylist: true,\n\t\t},\n\t\t{\n\t\t\tname: \"url_encoded_fmt_stream_map test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://youtu.be/DNaOZovrSVo\",\n\t\t\t\tTitle: \"QNAP Customer Story | Scorptec\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"stream 404 test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.youtube.com/watch?v=MRJ8NnUXacY\",\n\t\t\t\tTitle: \"FreeFileSync: Mirror Synchronization\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar (\n\t\t\t\tdata []*extractors.Data\n\t\t\t\terr  error\n\t\t\t)\n\t\t\tif tt.playlist {\n\t\t\t\t// playlist mode\n\t\t\t\t_, err = New().Extract(tt.args.URL, extractors.Options{\n\t\t\t\t\tPlaylist:     true,\n\t\t\t\t\tThreadNumber: 9,\n\t\t\t\t})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t} else {\n\t\t\t\tdata, err = New().Extract(tt.args.URL, extractors.Options{})\n\t\t\t\ttest.CheckError(t, err)\n\t\t\t\ttest.Check(t, tt.args, data[0])\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Path: extractors/zhihu/types.go\npackage zhihu\n\n// minimum field\ntype video struct {\n\tPlayList struct {\n\t\tFHD resolution `json:\"FHD\"`\n\t\tHD  resolution `json:\"HD\"`\n\t\tSD  resolution `json:\"SD\"`\n\t} `json:\"playlist_v2\"`\n}\n\n// minimum field\ntype resolution struct {\n\tSize    int64  `json:\"size\"`\n\tFormat  string `json:\"format\"`\n\tPlayURL string `json:\"play_url\"`\n}\n\n// Path: extractors/zhihu/zhihu.go\npackage zhihu\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nconst (\n\tvideoURL = \"www.zhihu.com/zvideo\"\n\tapi      = \"https://lens.zhihu.com/api/v4/videos/\"\n)\n\nfunc init() {\n\textractors.Register(\"zhihu\", New())\n}\n\ntype extractor struct{}\n\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tif !strings.Contains(url, videoURL) {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvideoID := utils.MatchOneOf(html, `\"videoId\":\"(\\d+)\"`)\n\ttitleMatch := utils.MatchOneOf(html, `<title.*?>(.*?)</title>`)\n\n\tif len(videoID) <= 1 {\n\t\treturn nil, errors.New(\"zhihu video id extract failed\")\n\t}\n\n\ttitle := \"Unknown\"\n\tif len(titleMatch) > 1 {\n\t\ttitle = titleMatch[1]\n\t}\n\n\tresp, err := request.GetByte(fmt.Sprintf(\"%s%s\", api, videoID[1]), url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar data video\n\tif err = json.Unmarshal(resp, &data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\tresolutions := map[string]resolution{\n\t\t\"FHD\": data.PlayList.FHD,\n\t\t\"HD\":  data.PlayList.HD,\n\t\t\"SD\":  data.PlayList.SD,\n\t}\n\n\tfor k, v := range resolutions {\n\t\tstream := &extractors.Stream{\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  v.PlayURL,\n\t\t\t\t\tSize: v.Size,\n\t\t\t\t\tExt:  v.Format,\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: v.Size,\n\t\t}\n\t\tstreams[k] = stream\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u77e5\u4e4e zhihu.com\",\n\t\t\tTitle:   title,\n\t\t\tStreams: streams,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/zhihu/zhihu_test.go\npackage zhihu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.zhihu.com/zvideo/1620162752064061440\",\n\t\t\t\tTitle: `Cursor, GPT-4 \u9a71\u52a8\u7684\u5f3a\u5927\u4ee3\u7801\u7f16\u8f91\u5668 - \u77e5\u4e4e`,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/zingmp3/zingmp3.go\npackage zingmp3\n\nimport (\n\t\"crypto/hmac\"\n\t\"crypto/sha256\"\n\t\"crypto/sha512\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"maps\"\n\t\"net/http\"\n\tneturl \"net/url\"\n\t\"regexp\"\n\t\"sort\"\n\n\t\"github.com/buger/jsonparser\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\tzingmp3Extractor := New()\n\textractors.Register(\"zingmp3\", zingmp3Extractor)\n\textractors.Register(\"zing\", zingmp3Extractor)\n}\n\ntype extractor struct{}\n\n// New returns a zingmp3 extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\ntype params map[string]string\n\nvar ApiSlugs = map[string]string{\n\t\"bai-hat\":        \"/api/v2/page/get/song\",\n\t\"embed\":          \"/api/v2/page/get/song\",\n\t\"video-clip\":     \"/api/v2/page/get/video\",\n\t\"lyric\":          \"/api/v2/lyric/get/lyric\",\n\t\"song-streaming\": \"/api/v2/song/get/streaming\",\n}\n\nconst Domain = \"https://zingmp3.vn\"\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\turlRegExp := regexp.MustCompile(`https?://(?:mp3\\.zing|zingmp3)\\.vn/(?P<type>(?:bai-hat|video-clip|embed))/[^/?#]+/(?P<id>\\w+)(?:\\.html|\\?)`)\n\turlMatcher := urlRegExp.FindStringSubmatch(url)\n\tif len(urlMatcher) == 0 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\turlType := urlMatcher[1]\n\tid := urlMatcher[2]\n\tif err := updatingCookies(); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdata := callApi(urlType, params{\"id\": id})\n\ttitle, _ := jsonparser.GetString(data, \"title\")\n\tvar contentType extractors.DataType\n\tvar source []byte\n\tif urlType == \"video-clip\" {\n\t\tsource, _, _, _ = jsonparser.Get(data, \"streaming\")\n\t\tapi := fmt.Sprintf(`http://api.mp3.zing.vn/api/mobile/video/getvideoinfo?requestdata={\"id\":\"%s\"}`, id)\n\t\tres, _ := request.Get(api, api, nil)\n\t\tnewSource, _, _, _ := jsonparser.Get([]byte(res), \"source\")\n\t\tsource, _ = jsonparser.Set(source, newSource, \"mp4\")\n\t\tcontentType = extractors.DataTypeVideo\n\t} else {\n\t\tcontentType = extractors.DataTypeAudio\n\t\tsource = callApi(\"song-streaming\", params{\"id\": id})\n\t}\n\tstreams := make(map[string]*extractors.Stream)\n\tif err := jsonparser.ObjectEach(source, func(k []byte, v []byte, dataType jsonparser.ValueType, offset int) error {\n\t\tkey := string(k)\n\t\tvalue := string(v)\n\t\tif value == \"\" || value == \"VIP\" {\n\t\t\treturn nil\n\t\t}\n\n\t\t// Handle for audio\n\t\tif key != \"mp4\" && key != \"hls\" {\n\t\t\tsize, _ := request.Size(value, url)\n\t\t\turlData := &extractors.Part{\n\t\t\t\tURL:  value,\n\t\t\t\tExt:  \"mp3\",\n\t\t\t\tSize: size,\n\t\t\t}\n\t\t\tstreams[\"default\"] = &extractors.Stream{\n\t\t\t\tParts: []*extractors.Part{urlData},\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\n\t\t// Handle for video\n\t\treturn jsonparser.ObjectEach(v, func(kSource []byte, vSource []byte, _ jsonparser.ValueType, _ int) error {\n\t\t\tresolution := string(kSource)\n\t\t\tvideoUrl := string(vSource)\n\t\t\tif resolution == \"\" {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif resolution == \"hls\" {\n\t\t\t\turls, _ := utils.M3u8URLs(videoUrl)\n\t\t\t\tparts := make([]*extractors.Part, 0)\n\t\t\t\tfor _, u := range urls {\n\t\t\t\t\tparts = append(parts, &extractors.Part{\n\t\t\t\t\t\tURL: u,\n\t\t\t\t\t\tExt: \"ts\",\n\t\t\t\t\t})\n\t\t\t\t}\n\t\t\t\tstreams[resolution] = &extractors.Stream{\n\t\t\t\t\tID:      resolution,\n\t\t\t\t\tParts:   parts,\n\t\t\t\t\tNeedMux: false,\n\t\t\t\t}\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tsize, _ := request.Size(videoUrl, url)\n\t\t\tstreams[fmt.Sprintf(\"mp4-%s\", resolution)] = &extractors.Stream{\n\t\t\t\tParts: []*extractors.Part{{\n\t\t\t\t\tURL:  videoUrl,\n\t\t\t\t\tExt:  \"mp4\",\n\t\t\t\t\tSize: size,\n\t\t\t\t}},\n\t\t\t}\n\t\t\treturn nil\n\t\t})\n\t}); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n...\n", "needle_token_start": 11290, "needle_token_end": 11389, "code_context_ntokens": 15117, "output": ["```go\nfunc getVideoAudio(v *youtube.Video, mimeType string) (*youtube.Format, error) {\n\taudioFormats := v.Formats.Type(mimeType).Type(\"audio\")\n\tif len(audioFormats) == 0 {\n\t\treturn nil, errors.New(\"no audio format found after filtering\")\n\t}\n\taudioFormats.Sort()\n\treturn &audioFormats[0], nil\n}\n```"]}
{"repo": "iawia002/lux", "name": "getRandomString", "language": "go", "path": "extractors/miaopai/miaopai.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate a random sequence of alphanumeric characters.\n2. **Input**: It accepts an integer that specifies the length of the string to be generated.\n3. **Output**: It returns a string composed of randomly selected alphanumeric characters, with the length determined by the input.\n4. **Procedure**: The function initializes an empty list to store characters. It defines a list of possible characters (letters and digits). For the number of times specified by the input, it randomly selects a character from the list and appends it to the initialized list. Finally, it concatenates all characters in the list into a single string and returns this string.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/instagram/instagram.go\npackage instagram\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nvar client *http.Client\n\nfunc init() {\n\textractors.Register(\"instagram\", New())\n\tclient = &http.Client{\n\t\tTimeout: 10 * time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tDial: (&net.Dialer{\n\t\t\t\tTimeout: 5 * time.Second,\n\t\t\t}).Dial,\n\t\t\tTLSHandshakeTimeout: 5 * time.Second,\n\t\t},\n\t}\n}\n\n// sliderItemNode contains information about the Instagram post\ntype sliderItemNode struct {\n\tDisplayURL string `json:\"display_url\"` // URL of the Media (resolution is dynamic)\n\n\tIsVideo  bool   `json:\"is_video\"`  // Is type of the Media equals to video\n\tVideoURL string `json:\"video_url\"` // Direct URL to the Video\n}\n\nfunc (s sliderItemNode) extractMediaURL() string {\n\tif s.IsVideo {\n\t\treturn s.VideoURL\n\t}\n\n\treturn s.DisplayURL\n}\n\ntype instagramPayload struct {\n\tMedia struct {\n\t\tID          string `json:\"id\"` // Unique ID of the Media\n\t\tSliderItems struct {\n\t\t\tEdges []struct {\n\t\t\t\tNode sliderItemNode `json:\"node\"`\n\t\t\t} `json:\"edges\"`\n\t\t} `json:\"edge_sidecar_to_children\"` // Children of the Media\n\t} `json:\"shortcode_media\"` // Media\n}\n\nfunc (s instagramPayload) isEmpty() bool {\n\treturn s.Media.ID == \"\"\n}\n\nfunc getPostWithCode(code string) ([]string, error) {\n\tURL := fmt.Sprintf(\"https://www.instagram.com/p/%v/embed/captioned/\", code)\n\n\tvar embeddedMediaImage string\n\tvar embedResponse = instagramPayload{}\n\tcollector := colly.NewCollector()\n\tcollector.SetClient(client)\n\tvar collectorErr error\n\n\tcollector.OnHTML(\"img.EmbeddedMediaImage\", func(e *colly.HTMLElement) {\n\t\tembeddedMediaImage = e.Attr(\"src\")\n\t})\n\n\tcollector.OnHTML(\"script\", func(e *colly.HTMLElement) {\n\t\tr := regexp.MustCompile(`\\\\\\\"gql_data\\\\\\\":([\\s\\S]*)\\}\\\"\\}\\]\\]\\,\\[\\\"NavigationMetrics`)\n\t\tmatch := r.FindStringSubmatch(e.Text)\n\n\t\tif len(match) < 2 {\n\t\t\treturn\n\t\t}\n\n\t\ts := strings.ReplaceAll(match[1], `\\\"`, `\"`)\n\t\ts = strings.ReplaceAll(s, `\\\\/`, `/`)\n\t\ts = strings.ReplaceAll(s, `\\\\`, `\\`)\n\n\t\terr := json.Unmarshal([]byte(s), &embedResponse)\n\t\tif err != nil {\n\t\t\tcollectorErr = err\n\t\t}\n\t})\n\n\tcollector.OnRequest(func(r *colly.Request) {\n\t\tr.Headers.Set(\"User-Agent\", browser.Chrome())\n\t})\n\n\tif err := collector.Visit(URL); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to send HTTP request to the Instagram: %v\", err)\n\t}\n\n\tif collectorErr != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse the Instagram response: %v\", collectorErr)\n\t}\n\n\t// If the method one which is JSON parsing didn't fail\n\tif !embedResponse.isEmpty() {\n\t\tresult := make([]string, 0, len(embedResponse.Media.SliderItems.Edges))\n\t\tfor _, item := range embedResponse.Media.SliderItems.Edges {\n\t\t\tresult = append(result, item.Node.extractMediaURL())\n\t\t}\n\n\t\treturn result, nil\n\t}\n\n\tif embeddedMediaImage != \"\" {\n\t\treturn []string{embeddedMediaImage}, nil\n\t}\n\n\t// If every two methods have failed, then return an error\n\treturn nil, errors.New(\"failed to fetch the post, the page might be \\\"private\\\", or the link is completely wrong\")\n}\n\nfunc extractShortCodeFromLink(link string) (string, error) {\n\tvalues := regexp.MustCompile(`(p|tv|reel|reels\\/videos)\\/([A-Za-z0-9-_]+)`).FindStringSubmatch(link)\n\tif len(values) != 3 {\n\t\treturn \"\", errors.New(\"couldn't extract the media short code from the link\")\n\t}\n\n\treturn values[2], nil\n}\n\ntype extractor struct{}\n\n// New returns a instagram extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tu, err := netURL.Parse(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tshortCode, err := extractShortCodeFromLink(u.String())\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turls, err := getPostWithCode(shortCode)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar totalSize int64\n\tvar parts []*extractors.Part\n\n\tfor _, u := range urls {\n\t\t_, ext, err := utils.GetNameAndExt(u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfileSize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tpart := &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: fileSize,\n\t\t\tExt:  ext,\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tfor _, part := range parts {\n\t\ttotalSize += part.Size\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Instagram instagram.com\",\n\t\t\tTitle:   \"Instagram \" + shortCode,\n...\n// Path: extractors/instagram/instagram_test.go\npackage instagram\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.instagram.com/p/BlIka1ZFCNr\",\n\t\t\t\tTitle: \"Instagram BlIka1ZFCNr\",\n\t\t\t\tSize:  992330,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.instagram.com/p/Bl5oVUyl9Yx\",\n\t\t\t\tTitle: \"Instagram Bl5oVUyl9Yx\",\n\t\t\t\tSize:  250596,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"image album test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.instagram.com/p/Bjyr-gxF4Rb\",\n\t\t\t\tTitle: \"Instagram Bjyr-gxF4Rb\",\n\t\t\t\tSize:  656476,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/iqiyi/iqiyi.go\npackage iqiyi\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"iqiyi\", New(SiteTypeIqiyi))\n\textractors.Register(\"iq\", New(SiteTypeIQ))\n}\n\ntype iqiyi struct {\n\tCode string `json:\"code\"`\n\tData struct {\n\t\tVP struct {\n\t\t\tDu  string `json:\"du\"`\n\t\t\tTkl []struct {\n\t\t\t\tVs []struct {\n\t\t\t\t\tBid   int    `json:\"bid\"`\n\t\t\t\t\tScrsz string `json:\"scrsz\"`\n\t\t\t\t\tVsize int64  `json:\"vsize\"`\n\t\t\t\t\tFs    []struct {\n\t\t\t\t\t\tL string `json:\"l\"`\n\t\t\t\t\t\tB int64  `json:\"b\"`\n\t\t\t\t\t} `json:\"fs\"`\n\t\t\t\t} `json:\"vs\"`\n\t\t\t} `json:\"tkl\"`\n\t\t} `json:\"vp\"`\n\t} `json:\"data\"`\n\tMsg string `json:\"msg\"`\n}\n\ntype iqiyiURL struct {\n\tL string `json:\"l\"`\n}\n\n// SiteType indicates the site type of iqiyi\ntype SiteType int\n\nconst (\n\t// SiteTypeIQ indicates the site is iq.com\n\tSiteTypeIQ SiteType = iota\n\t// SiteTypeIqiyi indicates the site is iqiyi.com\n\tSiteTypeIqiyi\n\tiqReferer    = \"https://www.iq.com\"\n\tiqiyiReferer = \"https://www.iqiyi.com\"\n)\n\nfunc getMacID() string {\n\tvar macID string\n\tchars := []string{\n\t\t\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"n\", \"m\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\",\n\t\t\"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\",\n\t}\n\tsize := len(chars)\n\tfor i := 0; i < 32; i++ {\n\t\tmacID += chars[rand.Intn(size)]\n\t}\n\treturn macID\n}\n\nfunc getVF(params string) string {\n\tvar suffix string\n\tfor j := 0; j < 8; j++ {\n\t\tfor k := 0; k < 4; k++ {\n\t\t\tvar v8 int\n\t\t\tv4 := 13 * (66*k + 27*j) % 35\n\t\t\tif v4 >= 10 {\n\t\t\t\tv8 = v4 + 88\n\t\t\t} else {\n\t\t\t\tv8 = v4 + 49\n\t\t\t}\n\t\t\tsuffix += string(rune(v8)) // string(97) -> \"a\"\n\t\t}\n\t}\n\tparams += suffix\n\n\treturn utils.Md5(params)\n}\n\nfunc getVPS(tvid, vid, refer string) (*iqiyi, error) {\n\tt := time.Now().Unix() * 1000\n\thost := \"http://cache.video.qiyi.com\"\n\tparams := fmt.Sprintf(\n\t\t\"/vps?tvid=%s&vid=%s&v=0&qypid=%s_12&src=01012001010000000000&t=%d&k_tag=1&k_uid=%s&rs=1\",\n\t\ttvid, vid, tvid, t, getMacID(),\n\t)\n\tvf := getVF(params)\n\tapiURL := fmt.Sprintf(\"%s%s&vf=%s\", host, params, vf)\n\tinfo, err := request.Get(apiURL, refer, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdata := new(iqiyi)\n\tif err := json.Unmarshal([]byte(info), data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\treturn data, nil\n}\n\ntype extractor struct {\n\tsiteType SiteType\n}\n\n// New returns a iqiyi extractor.\nfunc New(siteType SiteType) extractors.Extractor {\n\treturn &extractor{\n\t\tsiteType: siteType,\n\t}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\trefer := iqiyiReferer\n\theaders := make(map[string]string)\n\tif e.siteType == SiteTypeIQ {\n\t\theaders = map[string]string{\n\t\t\t\"Accept-Language\": \"zh-TW\",\n\t\t}\n\t\trefer = iqReferer\n\t}\n\thtml, err := request.Get(url, refer, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttvid := utils.MatchOneOf(\n\t\turl,\n\t\t`#curid=(.+)_`,\n\t\t`tvid=([^&]+)`,\n\t)\n\tif tvid == nil {\n\t\ttvid = utils.MatchOneOf(\n\t\t\thtml,\n\t\t\t`data-player-tvid=\"([^\"]+)\"`,\n\t\t\t`param\\['tvid'\\]\\s*=\\s*\"(.+?)\"`,\n\t\t\t`\"tvid\":\"(\\d+)\"`,\n\t\t\t`\"tvId\":(\\d+)`,\n\t\t)\n\t}\n\tif tvid == nil || len(tvid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tvid := utils.MatchOneOf(\n\t\turl,\n\t\t`#curid=.+_(.*)$`,\n\t\t`vid=([^&]+)`,\n\t)\n\tif vid == nil {\n\t\tvid = utils.MatchOneOf(\n\t\t\thtml,\n\t\t\t`data-player-videoid=\"([^\"]+)\"`,\n\t\t\t`param\\['vid'\\]\\s*=\\s*\"(.+?)\"`,\n\t\t\t`\"vid\":\"(\\w+)\"`,\n\t\t)\n\t}\n\tif vid == nil || len(vid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tdoc, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tif e.siteType == SiteTypeIqiyi {\n\t\ttitle = strings.TrimSpace(doc.Find(\"h1>a\").First().Text())\n\t\tvar sub string\n\t\tfor _, k := range []string{\"span\", \"em\"} {\n\t\t\tif sub != \"\" {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tsub = strings.TrimSpace(doc.Find(\"h1>\" + k).First().Text())\n\t\t}\n\t\ttitle += sub\n\t} else {\n\t\ttitle = strings.TrimSpace(doc.Find(\"span#pageMetaTitle\").First().Text())\n\t\tsub := utils.MatchOneOf(html, `\"subTitle\":\"([^\"]+)\",\"isoDuration\":`)\n\t\tif len(sub) > 1 {\n\t\t\ttitle += fmt.Sprintf(\" %s\", sub[1])\n\t\t}\n\t}\n\tif title == \"\" {\n\t\ttitle = doc.Find(\"title\").Text()\n\t}\n\tvideoDatas, err := getVPS(tvid[1], vid[1], refer)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif videoDatas.Code != \"A00000\" {\n\t\treturn nil, errors.Errorf(\"can't play this video: %s\", videoDatas.Msg)\n\t}\n\n\tstreams := make(map[string]*extractors.Stream)\n\turlPrefix := videoDatas.Data.VP.Du\n\tfor _, video := range videoDatas.Data.VP.Tkl[0].Vs {\n\t\turls := make([]*extractors.Part, len(video.Fs))\n\t\tfor index, v := range video.Fs {\n\t\t\trealURLData, err := request.Get(urlPrefix+v.L, refer, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\tvar realURL iqiyiURL\n\t\t\tif err = json.Unmarshal([]byte(realURLData), &realURL); err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\t_, ext, err := utils.GetNameAndExt(realURL.L)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, errors.WithStack(err)\n\t\t\t}\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL:  realURL.L,\n\t\t\t\tSize: v.B,\n\t\t\t\tExt:  ext,\n\t\t\t}\n\t\t}\n\t\tstreams[strconv.Itoa(video.Bid)] = &extractors.Stream{\n\t\t\tParts:   urls,\n\t\t\tSize:    video.Vsize,\n\t\t\tQuality: video.Scrsz,\n\t\t}\n\t}\n\n\tsiteName := \"\u7231\u5947\u827a iqiyi.com\"\n\tif e.siteType == SiteTypeIQ {\n\t\tsiteName = \"\u7231\u5947\u827a iq.com\"\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    siteName,\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/iqiyi/iqiyi_test.go\npackage iqiyi\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"http://www.iqiyi.com/v_19rrbdmaj0.html\",\n\t\t\t\tTitle:   \"\u65b0\u4e00\u8f6e\u964d\u6c34\u5c06\u81f3 \u51b7\u7a7a\u6c14\u5f71\u54cd\u4e2d\u4e1c\u90e8\u5730\u533a\",\n\t\t\t\tSize:    2952228,\n\t\t\t\tQuality: \"896x504\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"title test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"http://www.iqiyi.com/v_19rqy2z83w.html\",\n\t\t\t\tTitle:   \"\u6536\u4e86\u521b\u610f\u89c6\u98912018 :58\u5929\u73af\u7403\u98de\u884c\u8bb0\",\n\t\t\t\tSize:    76186786,\n\t\t\t\tQuality: \"1920x1080\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"curid test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.iqiyi.com/v_19rro0jdls.html#curid=350289100_6e6601aae889d0b1004586a52027c321\",\n\t\t\t\tTitle:   \"Shawn Mendes - Never Be Alone\",\n\t\t\t\tSize:    79921894,\n\t\t\t\tQuality: \"1920x800\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New(SiteTypeIqiyi).Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/ixigua/ixigua.go\npackage ixigua\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"ixigua\", New())\n\textractors.Register(\"toutiao\", New())\n}\n\ntype extractor struct{}\n\ntype Video struct {\n\tTitle     string `json:\"title\"`\n\tQualities []struct {\n\t\tQuality string `json:\"quality\"`\n\t\tSize    int64  `json:\"size\"`\n\t\tURL     string `json:\"url\"`\n\t\tExt     string `json:\"ext\"`\n\t} `json:\"qualities\"`\n}\n\n// New returns a ixigua extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\theaders := map[string]string{\n\t\t\"User-Agent\": browser.Chrome(),\n\t\t\"Cookie\":     option.Cookie,\n\t}\n\n\t// ixigua \u6709\u4e09\u79cd\u683c\u5f0f\u7684 URL\n\t// \u683c\u5f0f\u4e00 https://www.ixigua.com/7053389963487871502\n\t// \u683c\u5f0f\u4e8c https://v.ixigua.com/RedcbWM/\n\t// \u683c\u5f0f\u4e09 https://m.toutiao.com/is/dtj1pND/\n\t// \u683c\u5f0f\u4e8c\u4f1a\u8df3\u8f6c\u5230\u683c\u5f0f\u4e00\n\t// \u683c\u5f0f\u4e09\u4f1a\u8df3\u8f6c\u5230 https://www.toutiao.com/a7053389963487871502\n\n\tvar finalURL string\n\tif strings.HasPrefix(url, \"https://www.ixigua.com/\") {\n\t\tfinalURL = url\n\t}\n\n\tif strings.HasPrefix(url, \"https://v.ixigua.com/\") || strings.HasPrefix(url, \"https://m.toutiao.com/\") {\n\t\tresp, err := http.Get(url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tdefer resp.Body.Close() // nolint\n\t\t// follow redirects, https://stackoverflow.com/a/16785343\n\t\tfinalURL = resp.Request.URL.String()\n\t}\n\n\tfinalURL = strings.ReplaceAll(finalURL, \"https://www.toutiao.com/video/\", \"https://www.ixigua.com/\")\n\n\tr := regexp.MustCompile(`(ixigua.com/)(\\w+)?`)\n\tid := r.FindSubmatch([]byte(finalURL))[2]\n\turl2 := fmt.Sprintf(\"https://www.ixigua.com/%s\", string(id))\n\n\tbody, err := request.Get(url2, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvideoListJson := utils.MatchOneOf(body, `window._SSR_HYDRATED_DATA=(\\{.*?\\})\\<\\/script\\>`)\n\tif videoListJson == nil || len(videoListJson) != 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrBodyParseFailed)\n\t}\n\n\tvideoUrl := videoListJson[1]\n\tvideoUrl = strings.Replace(videoUrl, \":undefined\", \":\\\"undefined\\\"\", -1)\n\n\tvar data xiguanData\n\tif err = json.Unmarshal([]byte(videoUrl), &data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := data.AnyVideo.GidInformation.PackerData.Video.Title\n\tvideoList := data.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList\n\n\tstreams := make(map[string]*extractors.Stream)\n\tfor _, v := range videoList {\n\t\tstreams[v.Definition] = &extractors.Stream{\n\t\t\tQuality: v.Definition,\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  base64Decode(v.MainUrl),\n\t\t\t\t\tSize: v.Size,\n\t\t\t\t\tExt:  v.Vtype,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u897f\u74dc\u89c6\u9891 ixigua.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\nfunc base64Decode(t string) string {\n\td, _ := base64.StdEncoding.DecodeString(t)\n\treturn string(d)\n}\n\n// Path: extractors/ixigua/ixigua_test.go\npackage ixigua\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.ixigua.com/7053389963487871502\",\n\t\t\t\tTitle:   \"\u6f2b\u5a01\u65a5\u5de8\u8d44\u62cd\u7684\u300a\u6c38\u6052\u65cf\u300b\uff0c\u521a\u4e0a\u67b6\u5c31\u88ab\u591a\u56fd\u7981\u64ad\uff0c\u7a76\u7adf\u62cd\u4e86\u4ec0\u4e48\uff1f\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    313091514,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://v.ixigua.com/RedcbWM/\",\n\t\t\t\tTitle:   \"\u4e3a\u957f\u751f\u4e0d\u8001\uff0c\u7adf\u7136\u8fde\u5c0f\u9c9b\u4eba\u90fd\u6740@\u4e2d\u89c6\u9891\u4f19\u4f34\u8ba1\u5212\u5b98\u53f7\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    64980732,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"test 3\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://m.toutiao.com/is/dtj1pND/\",\n\t\t\t\tTitle:   \"\u5361\u5c14\uff1a59\u67404200\u6cd5\u5f3a\u5c0f\u6cd5\u5e08\uff0c\u70b9\u5854\u53ea\u9700\u4e00\u4e0b\uff0cW\u6280\u80fd\u77ac\u79d2\u5bf9\u65b9\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    468324298,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/ixigua/types.go\npackage ixigua\n\ntype xiguanData struct {\n\tAnyVideo struct {\n\t\tGidInformation struct {\n\t\t\tGid        string `json:\"gid\"`\n\t\t\tPackerData struct {\n\t\t\t\tVideo struct {\n\t\t\t\t\tTitle         string `json:\"title\"`\n\t\t\t\t\tPosterUrl     string `json:\"poster_url\"`\n\t\t\t\t\tVideoResource struct {\n\t\t\t\t\t\tVid    string `json:\"vid\"`\n\t\t\t\t\t\tNormal struct {\n\t\t\t\t\t\t\tVideoId   string `json:\"video_id\"`\n\t\t\t\t\t\t\tVideoList map[string]struct {\n\t\t\t\t\t\t\t\tDefinition  string `json:\"definition\"`\n\t\t\t\t\t\t\t\tQuality     string `json:\"quality\"`\n\t\t\t\t\t\t\t\tVtype       string `json:\"vtype\"`\n\t\t\t\t\t\t\t\tVwidth      int    `json:\"vwidth\"`\n\t\t\t\t\t\t\t\tVheight     int    `json:\"vheight\"`\n\t\t\t\t\t\t\t\tBitrate     int64  `json:\"bitrate\"`\n\t\t\t\t\t\t\t\tRealBitrate int64  `json:\"real_bitrate\"`\n\t\t\t\t\t\t\t\tFps         int    `json:\"fps\"`\n\t\t\t\t\t\t\t\tCodecType   string `json:\"codec_type\"`\n\t\t\t\t\t\t\t\tSize        int64  `json:\"size\"`\n\t\t\t\t\t\t\t\tMainUrl     string `json:\"main_url\"`\n\t\t\t\t\t\t\t\tBackupUrl1  string `json:\"backup_url_1\"`\n\t\t\t\t\t\t\t} `json:\"video_list\"`\n\t\t\t\t\t\t} `json:\"normal\"`\n\t\t\t\t\t} `json:\"videoResource\"`\n\t\t\t\t} `json:\"video\"`\n\t\t\t\tKey string `json:\"key\"`\n\t\t\t} `json:\"packerData\"`\n\t\t} `json:\"gidInformation\"`\n\t} `json:\"anyVideo\"`\n}\n\n// Path: extractors/kuaishou/kuaishou.go\npackage kuaishou\n\nimport (\n\t\"net/http\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"kuaishou\", New())\n}\n\ntype extractor struct{}\n\n// New returns a kuaishou extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// fetch url and get the cookie that write by server\nfunc fetchCookies(url string, headers map[string]string) (string, error) {\n\tres, err := request.Request(http.MethodGet, url, nil, headers)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tdefer res.Body.Close() // nolint\n\n\tcookiesArr := make([]string, 0)\n\tcookies := res.Cookies()\n\n\tfor _, c := range cookies {\n\t\tcookiesArr = append(cookiesArr, c.Name+\"=\"+c.Value)\n\t}\n\n\treturn strings.Join(cookiesArr, \"; \"), nil\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\theaders := map[string]string{\n\t\t\"User-Agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0\",\n\t}\n\n\tcookies, err := fetchCookies(url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\theaders[\"Cookie\"] = cookies\n\n\thtml, err := request.Get(url, url, headers)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitles := utils.MatchOneOf(html, `<title>([^<]+)</title>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.New(\"can not found title\")\n\t}\n\n\ttitle := regexp.MustCompile(`\\n+`).ReplaceAllString(strings.TrimSpace(titles[1]), \" \")\n\n\tqualityRegMap := map[string]*regexp.Regexp{\n\t\t\"sd\": regexp.MustCompile(`\"photoUrl\":\\s*\"([^\"]+)\"`),\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 1)\n\tfor quality, qualityReg := range qualityRegMap {\n\t\tmatcher := qualityReg.FindStringSubmatch(html)\n\t\tif len(matcher) != 2 {\n\t\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t\t}\n\n\t\tu := strings.ReplaceAll(matcher[1], `\\u002F`, \"/\")\n\n\t\tsize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u5feb\u624b kuaishou.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/kuaishou/kuaishou_test.go\npackage kuaishou\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.kuaishou.com/short-video/3x43cyvcyph57i4?authorId=3xtq3uqyjmhbimq&streamSource=find&area=homexxbrilliant\",\n\t\t\t\tTitle:   \"\u73b0\u5728\u8fde\u6234\u53e3\u7f69\u90fd\u5f00\u59cb\u5185\u5377\u4e86\u5417\uff1f\uff01\u5febget\u53e3\u7f69\u5fc3\u673a\u6234\u6cd5\uff0c\u76f4\u63a5\u6234\u51fa\u5c0fV\u8138\u554a \uff01 #\u53e3\u7f69 #\u663e\u8138\u5c0f-\u5feb\u624b\",\n\t\t\t\tSize:    1077774,\n\t\t\t\tQuality: \"sd\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/mgtv/mgtv.go\npackage mgtv\n\nimport (\n\t\"encoding/base64\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"mgtv\", New())\n}\n\ntype mgtvVideoStream struct {\n\tName string `json:\"name\"`\n\tURL  string `json:\"url\"`\n\tDef  string `json:\"def\"`\n}\n\ntype mgtvVideoInfo struct {\n\tTitle string `json:\"title\"`\n\tDesc  string `json:\"desc\"`\n}\n\ntype mgtvVideoData struct {\n\tStream       []mgtvVideoStream `json:\"stream\"`\n\tStreamDomain []string          `json:\"stream_domain\"`\n\tInfo         mgtvVideoInfo     `json:\"info\"`\n}\n\ntype mgtv struct {\n\tData mgtvVideoData `json:\"data\"`\n}\n\ntype mgtvVideoAddr struct {\n\tInfo string `json:\"info\"`\n}\n\ntype mgtvURLInfo struct {\n\tURL  string\n\tSize int64\n}\n\ntype mgtvPm2Data struct {\n\tData struct {\n\t\tAtc struct {\n\t\t\tPm2 string `json:\"pm2\"`\n\t\t} `json:\"atc\"`\n\t\tInfo mgtvVideoInfo `json:\"info\"`\n\t} `json:\"data\"`\n}\n\nfunc mgtvM3u8(url string) ([]mgtvURLInfo, int64, error) {\n\tvar data []mgtvURLInfo\n\tvar temp mgtvURLInfo\n\tvar size, totalSize int64\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tm3u8String, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tsizes := utils.MatchAll(m3u8String, `#EXT-MGTV-File-SIZE:(\\d+)`)\n\t// sizes: [[#EXT-MGTV-File-SIZE:1893724, 1893724]]\n\tfor index, u := range urls {\n\t\tsize, err = strconv.ParseInt(sizes[index][1], 10, 64)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\ttotalSize += size\n\t\ttemp = mgtvURLInfo{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, totalSize, nil\n}\n\nfunc encodeTk2(str string) string {\n\tencodeString := base64.StdEncoding.EncodeToString([]byte(str))\n\tr1 := regexp.MustCompile(`/\\+/g`)\n\tr2 := regexp.MustCompile(`///g`)\n\tr3 := regexp.MustCompile(`/=/g`)\n\tr1.ReplaceAllString(encodeString, \"_\")\n\tr2.ReplaceAllString(encodeString, \"~\")\n\tr3.ReplaceAllString(encodeString, \"-\")\n\tencodeString = utils.Reverse(encodeString)\n\treturn encodeString\n}\n\ntype extractor struct{}\n\n// New returns a mgtv extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvid := utils.MatchOneOf(\n\t\turl,\n\t\t`https?://www.mgtv.com/(?:b|l)/\\d+/(\\d+).html`,\n\t\t`https?://www.mgtv.com/hz/bdpz/\\d+/(\\d+).html`,\n\t)\n\tif vid == nil {\n\t\tvid = utils.MatchOneOf(html, `vid: (\\d+),`)\n\t}\n\tif vid == nil || len(vid) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// API extract from https://js.mgtv.com/imgotv-miniv6/global/page/play-tv.js\n\t// getSource and getPlayInfo function\n\t// Chrome Network JS panel\n\theaders := map[string]string{\n\t\t\"Cookie\": \"PM_CHKID=1\",\n\t}\n\tclit := fmt.Sprintf(\"clit=%d\", time.Now().Unix()/1000)\n\tpm2DataString, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"https://pcweb.api.mgtv.com/player/video?video_id=%s&tk2=%s\",\n\t\t\tvid[1],\n\t\t\tencodeTk2(fmt.Sprintf(\n\t\t\t\t\"did=f11dee65-4e0d-4d25-bfce-719ad9dc991d|pno=1030|ver=5.5.1|%s\", clit,\n\t\t\t)),\n\t\t),\n\t\turl,\n\t\theaders,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar pm2 mgtvPm2Data\n\tif err = json.Unmarshal([]byte(pm2DataString), &pm2); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tdataString, err := request.Get(\n\t\tfmt.Sprintf(\n\t\t\t\"https://pcweb.api.mgtv.com/player/getSource?video_id=%s&tk2=%s&pm2=%s\",\n\t\t\tvid[1], encodeTk2(clit), pm2.Data.Atc.Pm2,\n\t\t),\n\t\turl,\n\t\theaders,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar mgtvData mgtv\n\tif err = json.Unmarshal([]byte(dataString), &mgtvData); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := strings.TrimSpace(\n\t\tpm2.Data.Info.Title + \" \" + pm2.Data.Info.Desc,\n\t)\n\tmgtvStreams := mgtvData.Data.Stream\n\tvar addr mgtvVideoAddr\n\tstreams := make(map[string]*extractors.Stream)\n\tfor _, stream := range mgtvStreams {\n\t\tif stream.URL == \"\" {\n\t\t\tcontinue\n\t\t}\n\t\t// real download address\n\t\taddr = mgtvVideoAddr{}\n\t\taddrInfo, err := request.GetByte(mgtvData.Data.StreamDomain[0]+stream.URL, url, headers)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tif err = json.Unmarshal(addrInfo, &addr); err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tm3u8URLs, totalSize, err := mgtvM3u8(addr.Info)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turls := make([]*extractors.Part, len(m3u8URLs))\n\t\tfor index, u := range m3u8URLs {\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL:  u.URL,\n\t\t\t\tSize: u.Size,\n\t\t\t\tExt:  \"ts\",\n\t\t\t}\n\t\t}\n\t\tstreams[stream.Def] = &extractors.Stream{\n\t\t\tParts:   urls,\n\t\t\tSize:    totalSize,\n\t\t\tQuality: stream.Name,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u8292\u679cTV mgtv.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/mgtv/mgtv_test.go\npackage mgtv\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.mgtv.com/b/322712/4317248.html\",\n\t\t\t\tTitle:   \"\u6211\u662f\u5927\u4fa6\u63a2 \u5148\u5bfc\u7247\uff1a\u4f55\u7085\u5434\u78ca\u9093\u4f26\u7a7f\u8d8a\u7834\u6848\",\n\t\t\t\tSize:    86169236,\n\t\t\t\tQuality: \"\u8d85\u6e05\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"normal test 2\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.mgtv.com/b/308703/4197072.html\",\n\t\t\t\tTitle:   \"\u8292\u679c\u635e\u661f\u95fb 2017 \u8bfa\u4e00\u4e3a\u7237\u7237\u548c\u59e5\u7237\u505a\u7ffb\u8bd1\u8d85\u840c\",\n\t\t\t\tSize:    6486376,\n\t\t\t\tQuality: \"\u8d85\u6e05\",\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"vip test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.mgtv.com/b/322865/4352046.html\",\n\t\t\t\tTitle:   \"\u5411\u5f80\u7684\u751f\u6d3b \u7b2c\u4e8c\u5b63 \u5148\u5bfc\u7247\uff1a\u4f55\u7085\u9ec4\u78ca\u56de\u5f52\u8d28\u6734\u751f\u6d3b\",\n\t\t\t\tSize:    453246944,\n\t\t\t\tQuality: \"\u8d85\u6e05\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/miaopai/miaopai.go\npackage miaopai\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"miaopai\", New())\n}\n\ntype miaopaiData struct {\n\tData struct {\n\t\tDescription string `json:\"description\"`\n\t\tMetaData    []struct {\n\t\t\tURLs struct {\n\t\t\t\tM string `json:\"m\"`\n\t\t\t} `json:\"play_urls\"`\n\t\t} `json:\"meta_data\"`\n\t} `json:\"data\"`\n}\n\n\nfunc getRandomString(l int) string {\n\ts := make([]string, 0)\n\tchars := []string{\n\t\t\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"n\", \"m\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\",\n\t\t\"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\",\n\t}\n\tfor i := 0; i < l; i++ {\n\t\ts = append(s, chars[rand.Intn(len(chars)-1)])\n\t}\n\treturn strings.Join(s, \"\")\n}\n\ntype extractor struct{}\n\n// New returns a miaopai extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tids := utils.MatchOneOf(url, `/media/([^\\./]+)`, `/show(?:/channel)?/([^\\./]+)`)\n\tif ids == nil || len(ids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tid := ids[1]\n\n\trandomString := getRandomString(10)\n\n\tvar data miaopaiData\n\tjsonString, err := request.Get(\n\t\tfmt.Sprintf(\"https://n.miaopai.com/api/aj_media/info.json?smid=%s&appid=530&_cb=_jsonp%s\", id, randomString),\n\t\turl, nil,\n\t)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tmatch := utils.MatchOneOf(jsonString, randomString+`\\((.*)\\);$`)\n\tif match == nil || len(match) < 2 {\n\t\treturn nil, errors.New(\"\u83b7\u53d6\u89c6\u9891\u4fe1\u606f\u5931\u8d25\u3002\")\n\t}\n\n\terr = json.Unmarshal([]byte(match[1]), &data)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\trealURL := data.Data.MetaData[0].URLs.M\n\tsize, err := request.Size(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  realURL,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize:  size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u79d2\u62cd miaopai.com\",\n\t\t\tTitle:   data.Data.Description,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/miaopai/miaopai_test.go\npackage miaopai\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"http://n.miaopai.com/media/Dqg5Pmb~I6lChdvOb-~r1BpKzzDu~MPr\",\n\t\t\t\tTitle: \"\u5c0f\u5b66\u97386\u70b9\u534a\u8d77\u5e8a\u5b66\u4e60:\u60f3\u8d62\u5728\u8d77\u8dd1\u7ebf\",\n\t\t\t\tSize:  6743958,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/netease/netease.go\npackage netease\n\nimport (\n\tnetURL \"net/url\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"163\", New())\n}\n\ntype extractor struct{}\n\n// New returns a netease extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\turl = strings.Replace(url, \"/#/\", \"/\", 1)\n\tvid := utils.MatchOneOf(url, `/(mv|video)\\?id=(\\w+)`)\n\tif vid == nil {\n\t\treturn nil, errors.New(\"invalid url for netease music\")\n\t}\n\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tif strings.Contains(html, \"u-errlg-404\") {\n\t\treturn nil, errors.New(\"404 music not found\")\n\t}\n\n\ttitles := utils.MatchOneOf(html, `<meta property=\"og:title\" content=\"(.+?)\" />`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\trealURLs := utils.MatchOneOf(html, `<meta property=\"og:video\" content=\"(.+?)\" />`)\n\tif realURLs == nil || len(realURLs) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\trealURL, _ := netURL.QueryUnescape(realURLs[1])\n\n\tsize, err := request.Size(realURL, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  realURL,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{urlData},\n\t\t\tSize:  size,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u7f51\u6613\u4e91\u97f3\u4e50 music.163.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/netease/netease_test.go\npackage netease\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"mv test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://music.163.com/#/mv?id=5547010\",\n\t\t\t\tTitle: \"There For You\",\n\t\t\t\tSize:  24249078,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"video test 1\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://music.163.com/#/video?id=C8C9D11629798595BD28451DE3AC9FF4\",\n\t\t\t\tTitle: \"\uff03\u91d1\u66dc\u65e5\u306e\u65b0\u57a3\u7ed3\u8863 \u603b\u96c6\u7de8\u3008\u51689\u7de8\u3009\",\n\t\t\t\tSize:  37408123,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n...\n", "needle_token_start": 12763, "needle_token_end": 12964, "code_context_ntokens": 15136, "output": ["```go\nfunc getRandomString(l int) string {\n\ts := make([]string, 0)\n\tchars := []string{\n\t\t\"a\", \"b\", \"c\", \"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\", \"n\", \"m\", \"o\", \"p\", \"q\", \"r\", \"s\", \"t\", \"u\", \"v\",\n\t\t\"w\", \"x\", \"y\", \"z\", \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\",\n\t}\n\tfor i := 0; i < l; i++ {\n\t\ts = append(s, chars[rand.Intn(len(chars)-1)])\n\t}\n\treturn strings.Join(s, \"\")\n}\n```"]}
{"repo": "iawia002/lux", "name": "extractShortCodeFromLink", "language": "go", "path": "extractors/instagram/instagram.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: To retrieve a specific identifier from a given social media URL, which is used to uniquely identify media content such as photos or videos.\n2. **Input**: A string representing a URL from a social media platform.\n3. **Output**: A string representing the unique identifier extracted from the URL, or an error message if the identifier cannot be extracted.\n4. **Procedure**: The function uses a regular expression to search for patterns in the URL that match the expected format for media identifiers. If a match is found, the identifier is extracted and returned. If no match is found, an error is returned indicating the failure to extract the identifier.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: extractors/douyin/types.go\npackage douyin\n\ntype douyinData struct {\n\tStatusCode  int `json:\"status_code\"`\n\tAwemeDetail struct {\n\t\tAdmireAuth struct {\n\t\t\tAdmireButton       int `json:\"admire_button\"`\n\t\t\tIsAdmire           int `json:\"is_admire\"`\n\t\t\tIsShowAdmireButton int `json:\"is_show_admire_button\"`\n\t\t\tIsShowAdmireTab    int `json:\"is_show_admire_tab\"`\n\t\t} `json:\"admire_auth\"`\n\t\tAnchors interface{} `json:\"anchors\"`\n\t\tAuthor  struct {\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tCfList          interface{} `json:\"cf_list\"`\n\t\t\tCloseFriendType int         `json:\"close_friend_type\"`\n\t\t\tContactsStatus  int         `json:\"contacts_status\"`\n\t\t\tContrailList    interface{} `json:\"contrail_list\"`\n\t\t\tCoverURL        []struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"cover_url\"`\n\t\t\tCreateTime                             int         `json:\"create_time\"`\n\t\t\tCustomVerify                           string      `json:\"custom_verify\"`\n\t\t\tDataLabelList                          interface{} `json:\"data_label_list\"`\n\t\t\tEndorsementInfoList                    interface{} `json:\"endorsement_info_list\"`\n\t\t\tEnterpriseVerifyReason                 string      `json:\"enterprise_verify_reason\"`\n\t\t\tFavoritingCount                        int         `json:\"favoriting_count\"`\n\t\t\tFollowStatus                           int         `json:\"follow_status\"`\n\t\t\tFollowerCount                          int         `json:\"follower_count\"`\n\t\t\tFollowerListSecondaryInformationStruct interface{} `json:\"follower_list_secondary_information_struct\"`\n\t\t\tFollowerStatus                         int         `json:\"follower_status\"`\n\t\t\tFollowingCount                         int         `json:\"following_count\"`\n\t\t\tImRoleIds                              interface{} `json:\"im_role_ids\"`\n\t\t\tIsAdFake                               bool        `json:\"is_ad_fake\"`\n\t\t\tIsBlockedV2                            bool        `json:\"is_blocked_v2\"`\n\t\t\tIsBlockingV2                           bool        `json:\"is_blocking_v2\"`\n\t\t\tIsCf                                   int         `json:\"is_cf\"`\n\t\t\tMaxFollowerCount                       int         `json:\"max_follower_count\"`\n\t\t\tNickname                               string      `json:\"nickname\"`\n\t\t\tNotSeenItemIDList                      interface{} `json:\"not_seen_item_id_list\"`\n\t\t\tNotSeenItemIDListV2                    interface{} `json:\"not_seen_item_id_list_v2\"`\n\t\t\tOfflineInfoList                        interface{} `json:\"offline_info_list\"`\n\t\t\tPersonalTagList                        interface{} `json:\"personal_tag_list\"`\n\t\t\tPreventDownload                        bool        `json:\"prevent_download\"`\n\t\t\tRiskNoticeText                         string      `json:\"risk_notice_text\"`\n\t\t\tSecUID                                 string      `json:\"sec_uid\"`\n\t\t\tSecret                                 int         `json:\"secret\"`\n\t\t\tShareInfo                              struct {\n\t\t\t\tShareDesc      string `json:\"share_desc\"`\n\t\t\t\tShareDescInfo  string `json:\"share_desc_info\"`\n\t\t\t\tShareQrcodeURL struct {\n\t\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t\t} `json:\"share_qrcode_url\"`\n\t\t\t\tShareTitle       string `json:\"share_title\"`\n\t\t\t\tShareTitleMyself string `json:\"share_title_myself\"`\n\t\t\t\tShareTitleOther  string `json:\"share_title_other\"`\n\t\t\t\tShareURL         string `json:\"share_url\"`\n\t\t\t\tShareWeiboDesc   string `json:\"share_weibo_desc\"`\n\t\t\t} `json:\"share_info\"`\n\t\t\tShortID             string      `json:\"short_id\"`\n\t\t\tSignature           string      `json:\"signature\"`\n\t\t\tSignatureExtra      interface{} `json:\"signature_extra\"`\n\t\t\tSpecialPeopleLabels interface{} `json:\"special_people_labels\"`\n\t\t\tStatus              int         `json:\"status\"`\n\t\t\tTextExtra           interface{} `json:\"text_extra\"`\n\t\t\tTotalFavorited      int         `json:\"total_favorited\"`\n\t\t\tUID                 string      `json:\"uid\"`\n\t\t\tUniqueID            string      `json:\"unique_id\"`\n\t\t\tUserAge             int         `json:\"user_age\"`\n\t\t\tUserCanceled        bool        `json:\"user_canceled\"`\n\t\t\tUserPermissions     interface{} `json:\"user_permissions\"`\n\t\t\tVerificationType    int         `json:\"verification_type\"`\n\t\t} `json:\"author\"`\n\t\tAuthorMaskTag int   `json:\"author_mask_tag\"`\n\t\tAuthorUserID  int64 `json:\"author_user_id\"`\n\t\tAwemeACL      struct {\n\t\t\tDownloadMaskPanel struct {\n\t\t\t\tCode     int `json:\"code\"`\n\t\t\t\tShowType int `json:\"show_type\"`\n\t\t\t} `json:\"download_mask_panel\"`\n\t\t} `json:\"aweme_acl\"`\n\t\tAwemeControl struct {\n\t\t\tCanComment     bool `json:\"can_comment\"`\n\t\t\tCanForward     bool `json:\"can_forward\"`\n\t\t\tCanShare       bool `json:\"can_share\"`\n\t\t\tCanShowComment bool `json:\"can_show_comment\"`\n\t\t} `json:\"aweme_control\"`\n\t\tAwemeID               string      `json:\"aweme_id\"`\n\t\tAwemeType             int         `json:\"aweme_type\"`\n\t\tChallengePosition     interface{} `json:\"challenge_position\"`\n\t\tChapterList           interface{} `json:\"chapter_list\"`\n\t\tCollectStat           int         `json:\"collect_stat\"`\n\t\tCommentGid            int64       `json:\"comment_gid\"`\n\t\tCommentList           interface{} `json:\"comment_list\"`\n\t\tCommentPermissionInfo struct {\n\t\t\tCanComment              bool `json:\"can_comment\"`\n\t\t\tCommentPermissionStatus int  `json:\"comment_permission_status\"`\n\t\t\tItemDetailEntry         bool `json:\"item_detail_entry\"`\n\t\t\tPressEntry              bool `json:\"press_entry\"`\n\t\t\tToastGuide              bool `json:\"toast_guide\"`\n\t\t} `json:\"comment_permission_info\"`\n\t\tCommerceConfigData interface{} `json:\"commerce_config_data\"`\n\t\tCommonBarInfo      string      `json:\"common_bar_info\"`\n\t\tComponentInfoV2    string      `json:\"component_info_v2\"`\n\t\tCoverLabels        interface{} `json:\"cover_labels\"`\n\t\tCreateTime         int         `json:\"create_time\"`\n\t\tDesc               string      `json:\"desc\"`\n\t\tDiggLottie         struct {\n\t\t\tCanBomb  int    `json:\"can_bomb\"`\n\t\t\tLottieID string `json:\"lottie_id\"`\n\t\t} `json:\"digg_lottie\"`\n\t\tDisableRelationBar      int         `json:\"disable_relation_bar\"`\n\t\tDislikeDimensionList    interface{} `json:\"dislike_dimension_list\"`\n\t\tDuetAggregateInMusicTab bool        `json:\"duet_aggregate_in_music_tab\"`\n\t\tDuration                int         `json:\"duration\"`\n\t\tFeedCommentConfig       struct {\n\t\t\tAuthorAuditStatus int    `json:\"author_audit_status\"`\n\t\t\tInputConfigText   string `json:\"input_config_text\"`\n\t\t} `json:\"feed_comment_config\"`\n\t\tGeofencing          []interface{} `json:\"geofencing\"`\n\t\tGeofencingRegions   interface{}   `json:\"geofencing_regions\"`\n\t\tGroupID             string        `json:\"group_id\"`\n\t\tHybridLabel         interface{}   `json:\"hybrid_label\"`\n\t\tImageAlbumMusicInfo struct {\n\t\t\tBeginTime int `json:\"begin_time\"`\n\t\t\tEndTime   int `json:\"end_time\"`\n\t\t\tVolume    int `json:\"volume\"`\n\t\t} `json:\"image_album_music_info\"`\n\t\tImageInfos interface{} `json:\"image_infos\"`\n\t\tImageList  interface{} `json:\"image_list\"`\n\t\tImages     []struct {\n\t\t\tDownloadURLList []string    `json:\"download_url_list\"`\n\t\t\tHeight          int         `json:\"height\"`\n\t\t\tMaskURLList     interface{} `json:\"mask_url_list\"`\n\t\t\tURI             string      `json:\"uri\"`\n\t\t\tURLList         []string    `json:\"url_list\"`\n\t\t\tWidth           int         `json:\"width\"`\n\t\t} `json:\"images\"`\n\t\tImgBitrate []struct {\n\t\t\tImages []struct {\n\t\t\t\tDownloadURLList []string    `json:\"download_url_list\"`\n\t\t\t\tHeight          int         `json:\"height\"`\n\t\t\t\tMaskURLList     interface{} `json:\"mask_url_list\"`\n\t\t\t\tURI             string      `json:\"uri\"`\n\t\t\t\tURLList         []string    `json:\"url_list\"`\n\t\t\t\tWidth           int         `json:\"width\"`\n\t\t\t} `json:\"images\"`\n\t\t\tName string `json:\"name\"`\n\t\t} `json:\"img_bitrate\"`\n\t\tImpressionData struct {\n\t\t\tGroupIDListA   []int64     `json:\"group_id_list_a\"`\n\t\t\tGroupIDListB   []int64     `json:\"group_id_list_b\"`\n\t\t\tSimilarIDListA interface{} `json:\"similar_id_list_a\"`\n\t\t\tSimilarIDListB interface{} `json:\"similar_id_list_b\"`\n\t\t} `json:\"impression_data\"`\n\t\tInteractionStickers  interface{} `json:\"interaction_stickers\"`\n\t\tIsAds                bool        `json:\"is_ads\"`\n\t\tIsCollectsSelected   int         `json:\"is_collects_selected\"`\n\t\tIsDuetSing           bool        `json:\"is_duet_sing\"`\n\t\tIsImageBeat          bool        `json:\"is_image_beat\"`\n\t\tIsLifeItem           bool        `json:\"is_life_item\"`\n\t\tIsMultiContent       int         `json:\"is_multi_content\"`\n\t\tIsStory              int         `json:\"is_story\"`\n\t\tIsTop                int         `json:\"is_top\"`\n\t\tItemWarnNotification struct {\n\t\t\tContent string `json:\"content\"`\n\t\t\tShow    bool   `json:\"show\"`\n\t\t\tType    int    `json:\"type\"`\n\t\t} `json:\"item_warn_notification\"`\n\t\tLabelTopText interface{} `json:\"label_top_text\"`\n\t\tLongVideo    interface{} `json:\"long_video\"`\n\t\tMusic        struct {\n\t\t\tAlbum            string        `json:\"album\"`\n\t\t\tArtistUserInfos  interface{}   `json:\"artist_user_infos\"`\n\t\t\tArtists          []interface{} `json:\"artists\"`\n\t\t\tAuditionDuration int           `json:\"audition_duration\"`\n\t\t\tAuthor           string        `json:\"author\"`\n\t\t\tAuthorDeleted    bool          `json:\"author_deleted\"`\n\t\t\tAuthorPosition   interface{}   `json:\"author_position\"`\n\t\t\tAuthorStatus     int           `json:\"author_status\"`\n\t\t\tAvatarLarge      struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_large\"`\n\t\t\tAvatarMedium struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_medium\"`\n\t\t\tAvatarThumb struct {\n\t\t\t\tHeight  int      `json:\"height\"`\n\t\t\t\tURI     string   `json:\"uri\"`\n\t\t\t\tURLList []string `json:\"url_list\"`\n\t\t\t\tWidth   int      `json:\"width\"`\n\t\t\t} `json:\"avatar_thumb\"`\n\t\t\tBindedChallengeID int  `json:\"binded_challenge_id\"`\n\t\t\tCanBackgroundPlay bool `json:\"can_background_play\"`\n\t\t\tCollectStat       int  `json:\"collect_stat\"`\n\t\t\tCoverColorHsv     struct {\n\t\t\t\tH int `json:\"h\"`\n\t\t\t\tS int `json:\"s\"`\n\t\t\t\tV int `json:\"v\"`\n...\n// Path: extractors/douyu/douyu.go\npackage douyu\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"douyu\", New())\n}\n\ntype douyuData struct {\n\tError int `json:\"error\"`\n\tData  struct {\n\t\tVideoURL string `json:\"video_url\"`\n\t} `json:\"data\"`\n}\n\ntype douyuURLInfo struct {\n\tURL  string\n\tSize int64\n}\n\nfunc douyuM3u8(url string) ([]douyuURLInfo, int64, error) {\n\tvar (\n\t\tdata            []douyuURLInfo\n\t\ttemp            douyuURLInfo\n\t\tsize, totalSize int64\n\t\terr             error\n\t)\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, 0, err\n\t}\n\tfor _, u := range urls {\n\t\tsize, err = request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, 0, err\n\t\t}\n\t\ttotalSize += size\n\t\ttemp = douyuURLInfo{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, totalSize, nil\n}\n\ntype extractor struct{}\n\n// New returns a douyu extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\tliveVid := utils.MatchOneOf(url, `https?://www.douyu.com/(\\S+)`)\n\tif liveVid != nil {\n\t\treturn nil, errors.New(\"\u6682\u4e0d\u652f\u6301\u6597\u9c7c\u76f4\u64ad\")\n\t}\n\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitles := utils.MatchOneOf(html, `<title>(.*?)</title>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\tvids := utils.MatchOneOf(url, `https?://v.douyu.com/show/(\\S+)`)\n\tif vids == nil || len(vids) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\tvid := vids[1]\n\n\tdataString, err := request.Get(\"http://vmobile.douyu.com/video/getInfo?vid=\"+vid, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdataDict := new(douyuData)\n\tif err := json.Unmarshal([]byte(dataString), dataDict); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tm3u8URLs, totalSize, err := douyuM3u8(dataDict.Data.VideoURL)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turls := make([]*extractors.Part, len(m3u8URLs))\n\tfor index, u := range m3u8URLs {\n\t\turls[index] = &extractors.Part{\n\t\t\tURL:  u.URL,\n\t\t\tSize: u.Size,\n\t\t\tExt:  \"ts\",\n\t\t}\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: urls,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u6597\u9c7c douyu.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/douyu/douyu_test.go\npackage douyu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://v.douyu.com/show/l0Q8mMY3wZqv49Ad\",\n\t\t\t\tTitle: \"\u6bcf\u65e5\u64b8\u62a5_\u6bcf\u65e5\u64b8\u62a5\uff1a\u6709\u4e9b\u4eba\u6b7b\u4e86\u5176\u5b9e\u5b83\u8fd8\u53ef\u4ee5\u628a\u4f60\u5e26\u8d70_\u6597\u9c7c\u89c6\u9891 - \u67006\u7684\u5f39\u5e55\u89c6\u9891\u7f51\u7ad9\",\n\t\t\t\tSize:  10558080,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/eporner/eporner.go\npackage eporner\n\nimport (\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/PuerkitoBio/goquery\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/parser\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"eporner\", New())\n}\n\nconst (\n\tdownloadclass = \".dloaddivcol\"\n)\n\ntype src struct {\n\turl     string\n\tquality string\n\tsizestr string\n\tsize    int64\n}\n\nfunc getSrcMeta(text string) *src {\n\tsti := strings.Index(text, \"(\")\n\tste := strings.Index(text, \")\")\n\titext := text[sti+1 : ste]\n\tstrs := strings.Split(itext, \",\")\n\ts := &src{}\n\n\tif len(strs) == 2 {\n\t\ts.quality = strings.Trim(strs[0], \" \")\n\t\ts.sizestr = strings.Trim(strs[1], \" \")\n\t}\n\n\tif s.sizestr == \"\" {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\n\tvalunit := strings.Split(s.sizestr, \" \")\n\tval, err := strconv.ParseFloat(valunit[0], 64)\n\tif err != nil {\n\t\ts.size = 0\n\t\treturn s\n\t}\n\tunit := valunit[1]\n\tswitch unit {\n\tcase \"KB\":\n\t\ts.size = int64(val * 1024)\n\tcase \"MB\":\n\t\ts.size = int64(val * 1024 * 1024)\n\tcase \"GB\":\n\t\ts.size = int64(val * 1024 * 1024 * 1024)\n\tdefault:\n\t\ts.size = int64(val)\n\t}\n\treturn s\n}\n\nfunc getSrc(html string) []*src {\n\tsrcs := []*src{}\n\td, err := parser.GetDoc(html)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\td.Find(downloadclass).Each(func(i int, s *goquery.Selection) {\n\t\ts.Contents().Each(func(i int, s *goquery.Selection) {\n\t\t\tfor ns := range s.Nodes {\n\t\t\t\tn := s.Get(ns)\n\t\t\t\tif n.Data == \"a\" {\n\t\t\t\t\tvar sr *src\n\t\t\t\t\tif n.FirstChild != nil {\n\t\t\t\t\t\tsr = getSrcMeta(n.FirstChild.Data)\n\t\t\t\t\t}\n\t\t\t\t\tfor _, a := range n.Attr {\n\t\t\t\t\t\tif a.Key == \"href\" {\n\t\t\t\t\t\t\tsr.url = a.Val\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tsrcs = append(srcs, sr)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t})\n\n\treturn srcs\n}\n\ntype extractor struct{}\n\n// New returns a eporner extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(u string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(u, u, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tvar title string\n\tdesc := utils.MatchOneOf(html, `<title>(.+?)</title>`)\n\tif len(desc) > 1 {\n\t\ttitle = desc[1]\n\t} else {\n\t\ttitle = \"eporner\"\n\t}\n\tuu, err := url.Parse(u)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tsrcs := getSrc(html)\n\tstreams := make(map[string]*extractors.Stream, len(srcs))\n\tfor _, src := range srcs {\n\t\tsrcurl := uu.Scheme + \"://\" + uu.Host + src.url\n\t\t// skipping an extra HEAD request to the URL.\n\t\t// size, err := request.Size(srcurl, u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\turlData := &extractors.Part{\n\t\t\tURL:  srcurl,\n\t\t\tSize: src.size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[src.quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    src.size,\n\t\t\tQuality: src.quality,\n\t\t}\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"EPORNER eporner.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     u,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/eporner/eporner_test.go\npackage eporner\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.eporner.com/video-mbubfvXYFip/dirtywivesclub-becky-bandini/\",\n\t\t\t\tQuality: \"1080p\",\n\t\t\t\tSize:    1525510307,\n\t\t\t\tTitle:   \"DirtyWivesClub - Becky Bandini - EPORNER\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/facebook/facebook.go\npackage facebook\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"facebook\", New())\n}\n\ntype extractor struct{}\n\n// New returns a facebook extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\ttitles := utils.MatchOneOf(html, `<title>([^<]+)</title>`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\ttitle := strings.TrimSpace(titles[1])\n\n\ttitle = regexp.MustCompile(`\\n+`).ReplaceAllString(title, \" \")\n\n\tqualityRegMap := map[string]*regexp.Regexp{\n\t\t\"sd\": regexp.MustCompile(`\"playable_url\":\\s*\"([^\"]+)\"`),\n\t\t// \"hd\": regexp.MustCompile(`\"playable_url_quality_hd\":\\s*\"([^\"]+)\"`),\n\t}\n\n\tstreams := make(map[string]*extractors.Stream, 2)\n\tfor quality, qualityReg := range qualityRegMap {\n\t\tmatcher := qualityReg.FindStringSubmatch(html)\n\n\t\tif len(matcher) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tu := strings.ReplaceAll(matcher[1], \"\\\\\", \"\")\n\n\t\tsize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turlData := &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t\tExt:  \"mp4\",\n\t\t}\n\t\tstreams[quality] = &extractors.Stream{\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Facebook facebook.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/facebook/facebook_test.go\npackage facebook\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:     \"https://www.facebook.com/100058251872436/videos/424557726111987\",\n\t\t\t\tTitle:   \"\u0420\u043e\u043c\u0430\u043d \u0413\u0440\u0438\u0449\u0443\u043a - \u041f\u0456\u0434\u0442\u0440\u0438\u043c\u043a\u0430 \u0437 \u042f\u043f\u043e\u043d\u0456\u0457 \ud83c\uddef\ud83c\uddf5 \u0413\u0443\u0440\u0442 Yokohama Sisters \ud83d\udc4f\",\n\t\t\t\tSize:    1441128,\n\t\t\t\tQuality: \"sd\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/geekbang/geekbang.go\npackage geekbang\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"geekbang\", New())\n}\n\ntype geekData struct {\n\tCode  int             `json:\"code\"`\n\tError json.RawMessage `json:\"error\"`\n\tData  struct {\n\t\tVideoID      string `json:\"video_id\"`\n\t\tTitle        string `json:\"article_sharetitle\"`\n\t\tColumnHadSub bool   `json:\"column_had_sub\"`\n\t} `json:\"data\"`\n}\n\ntype videoPlayAuth struct {\n\tCode  int             `json:\"code\"`\n\tError json.RawMessage `json:\"error\"`\n\tData  struct {\n\t\tPlayAuth string `json:\"play_auth\"`\n\t} `json:\"data\"`\n}\n\ntype playInfo struct {\n\tVideoBase struct {\n\t\tVideoID  string `json:\"VideoId\"`\n\t\tTitle    string `json:\"Title\"`\n\t\tCoverURL string `josn:\"CoverURL\"`\n\t} `json:\"VideoBase\"`\n\tPlayInfoList struct {\n\t\tPlayInfo []struct {\n\t\t\tURL        string `json:\"PlayURL\"`\n\t\t\tSize       int64  `json:\"Size\"`\n\t\t\tDefinition string `json:\"Definition\"`\n\t\t} `json:\"PlayInfo\"`\n\t} `json:\"PlayInfoList\"`\n}\n\ntype geekURLInfo struct {\n\tURL  string\n\tSize int64\n}\n\nfunc geekM3u8(url string) ([]geekURLInfo, error) {\n\tvar (\n\t\tdata []geekURLInfo\n\t\ttemp geekURLInfo\n\t\tsize int64\n\t\terr  error\n\t)\n\turls, err := utils.M3u8URLs(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tfor _, u := range urls {\n\t\ttemp = geekURLInfo{\n\t\t\tURL:  u,\n\t\t\tSize: size,\n\t\t}\n\t\tdata = append(data, temp)\n\t}\n\treturn data, nil\n}\n\ntype extractor struct{}\n\n// New returns a geekbang extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, _ extractors.Options) ([]*extractors.Data, error) {\n\tvar err error\n\tmatches := utils.MatchOneOf(url, `https?://time.geekbang.org/course/detail/(\\d+)-(\\d+)`)\n\tif matches == nil || len(matches) < 3 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\t// Get video information\n\theanders := map[string]string{\"Origin\": \"https://time.geekbang.org\", \"Content-Type\": \"application/json\", \"Referer\": url}\n\tparams := strings.NewReader(fmt.Sprintf(`{\"id\": %q}`, matches[2]))\n\tres, err := request.Request(http.MethodPost, \"https://time.geekbang.org/serv/v1/article\", params, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar data geekData\n\tif err = json.NewDecoder(res.Body).Decode(&data); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif data.Code < 0 {\n\t\treturn nil, errors.New(string(data.Error))\n\t}\n\n\tif data.Data.VideoID == \"\" && !data.Data.ColumnHadSub {\n\t\treturn nil, errors.New(\"\u8bf7\u5148\u8d2d\u4e70\u8bfe\u7a0b\uff0c\u6216\u4f7f\u7528Cookie\u767b\u5f55\u3002\")\n\t}\n\n\t// Get video license token information\n\tparams = strings.NewReader(\"{\\\"source_type\\\":1,\\\"aid\\\":\" + matches[2] + \",\\\"video_id\\\":\\\"\" + data.Data.VideoID + \"\\\"}\")\n\tres, err = request.Request(http.MethodPost, \"https://time.geekbang.org/serv/v3/source_auth/video_play_auth\", params, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar playAuth videoPlayAuth\n\tif err = json.NewDecoder(res.Body).Decode(&playAuth); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tif playAuth.Code < 0 {\n\t\treturn nil, errors.New(string(playAuth.Error))\n\t}\n\n\t// Get video playback information\n\theanders = map[string]string{\"Accept-Encoding\": \"\"}\n\tres, err = request.Request(http.MethodGet, \"http://ali.mantv.top/play/info?playAuth=\"+playAuth.Data.PlayAuth, nil, heanders)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\tdefer res.Body.Close() // nolint\n\n\tvar playInfo playInfo\n\tif err = json.NewDecoder(res.Body).Decode(&playInfo); err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitle := data.Data.Title\n\n\tstreams := make(map[string]*extractors.Stream, len(playInfo.PlayInfoList.PlayInfo))\n\n\tfor _, media := range playInfo.PlayInfoList.PlayInfo {\n\t\tm3u8URLs, err := geekM3u8(media.URL)\n\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\turls := make([]*extractors.Part, len(m3u8URLs))\n\t\tfor index, u := range m3u8URLs {\n\t\t\turls[index] = &extractors.Part{\n\t\t\t\tURL:  u.URL,\n\t\t\t\tSize: u.Size,\n\t\t\t\tExt:  \"ts\",\n\t\t\t}\n\t\t}\n\n\t\tstreams[media.Definition] = &extractors.Stream{\n\t\t\tParts: urls,\n\t\t\tSize:  media.Size,\n\t\t}\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u6781\u5ba2\u65f6\u95f4 geekbang.org\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/geekbang/geekbang_test.go\npackage geekbang\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://time.geekbang.org/course/detail/190-97203\",\n\t\t\t\tTitle: \"02 | \u5185\u5bb9\u7efc\u8ff0 - \u73a9\u8f6cwebpack\",\n\t\t\t\tSize:  10752472,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/haokan/haokan.go\npackage haokan\n\nimport (\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"haokan\", New())\n}\n\ntype extractor struct{}\n\n// New returns a haokan extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\ttitles := utils.MatchOneOf(html, `property=\"og:title\"\\s+content=\"(.+?)\"`)\n\tif titles == nil || len(titles) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\ttitle := titles[1]\n\n\t// \u4e4b\u524d\u7684\u597d\u770b\u7f51\u9875\u4e2d\uff0c\u89c6\u9891\u5730\u5740\u662f\u653e\u5728 video \u6807\u7b7e\u4e0b\n\turls := utils.MatchOneOf(html, `<video\\s*class=\"video\"\\s*src=\"?(.+?)\"?\\s*>`)\n\n\tif urls == nil || len(urls) < 2 {\n\t\t// fallbak: \u65b0\u7684\u597d\u770b\u7f51\u9875\u4e2d\uff0c\u89c6\u9891\u5730\u5740\u5728 json \u6570\u636e\u91cc\n\t\turls = utils.MatchOneOf(html, `\"playurl\":\"(http.+?)\"`)\n\t}\n\n\tif urls == nil || len(urls) < 2 {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tplayurl := strings.Replace(urls[1], `\\/`, `/`, -1)\n\n\tsize, err := request.Size(playurl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\t_, ext, err := utils.GetNameAndExt(playurl)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: []*extractors.Part{\n\t\t\t\t{\n\t\t\t\t\tURL:  playurl,\n\t\t\t\t\tSize: size,\n\t\t\t\t\tExt:  ext,\n\t\t\t\t},\n\t\t\t},\n\t\t\tSize: size,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u597d\u770b\u89c6\u9891 haokan.baidu.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/haokan/haokan_test.go\npackage haokan\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://haokan.baidu.com/v?vid=10057409468467026969\",\n\t\t\t\tTitle: \"\u542c\u6b4c\u5b66\u82f1\u8bed\u5c0f\u5b66\u7bc7\uff086\uff09\uff1amy new pen pal\",\n\t\t\t\tSize:  2027354,\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tdata, err := New().Extract(tt.args.URL, extractors.Options{})\n\t\t\ttest.CheckError(t, err)\n\t\t\ttest.Check(t, tt.args, data[0])\n\t\t})\n\t}\n}\n\n// Path: extractors/hupu/hupu.go\npackage hupu\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"hupu\", New())\n}\n\ntype extractor struct{}\n\n// New returns a hupu extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar title string\n\ttitleDesc := utils.MatchOneOf(html, `<span class=\"post-user-comp-info-bottom-title\">(.+?)</span>`)\n\tif len(titleDesc) > 1 {\n\t\ttitle = titleDesc[1]\n\t} else {\n\t\ttitle = \"hupu video\"\n\t}\n\n\tvar videoUrl string\n\turlDesc := utils.MatchOneOf(html, `<video src=\"(.+?)\" controls=\"\" poster=(.+?)></video>`)\n\tif len(urlDesc) > 1 {\n\t\tvideoUrl = urlDesc[1]\n\t} else {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tsize, err := request.Size(videoUrl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoUrl,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u864e\u6251 hupu.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/hupu/hupu_test.go\npackage hupu\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestHupu(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://bbs.hupu.com/47401018.html?is_reflow=1&cid=84752419&bddid=56KXU5QUJH4VGM26SFPTYTKNI5CFNJMX736TIZ52DXLGUAAMBJVA01&puid=16522089&client=8577E496-4D9B-4E5C-A9DB-A8EF5C1956D2\",\n\t\t\t\tTitle: \"\u7ed3\u5c40\u5f15\u8d77\u8212\u9002\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/huya/huya.go\npackage huya\n\nimport (\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nfunc init() {\n\textractors.Register(\"huya\", New())\n}\n\ntype extractor struct{}\n\nconst huyaVideoHost = \"https://videotx-platform.cdn.huya.com/\"\n\n// New returns a huya extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\thtml, err := request.Get(url, url, nil)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar title string\n\ttitleDesc := utils.MatchOneOf(html, `<h1>(.+?)</h1>`)\n\tif len(titleDesc) > 1 {\n\t\ttitle = titleDesc[1]\n\t} else {\n\t\ttitle = \"huya video\"\n\t}\n\n\tvar videoUrl string\n\tvideoDesc := utils.MatchOneOf(html, `//videotx-platform.cdn.huya.com/(.*)\" poster=(.+?)`)\n\tif len(videoDesc) > 1 {\n\t\tvideoUrl = huyaVideoHost + videoDesc[1]\n\t} else {\n\t\treturn nil, errors.WithStack(extractors.ErrURLParseFailed)\n\t}\n\n\tsize, err := request.Size(videoUrl, url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\turlData := &extractors.Part{\n\t\tURL:  videoUrl,\n\t\tSize: size,\n\t\tExt:  \"mp4\",\n\t}\n\tquality := \"normal\"\n\tstreams := map[string]*extractors.Stream{\n\t\tquality: {\n\t\t\tParts:   []*extractors.Part{urlData},\n\t\t\tSize:    size,\n\t\t\tQuality: quality,\n\t\t},\n\t}\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"\u864e\u7259 huya.com\",\n\t\t\tTitle:   title,\n\t\t\tType:    extractors.DataTypeVideo,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/huya/huya_test.go\npackage huya\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestHuya(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"normal test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://m.v.huya.com/play/fans/630103747.html/?shareid=4597484513543964249&shareUid=2179142017&source=ios&sharetype=other&platform=2\",\n\t\t\t\tTitle: \"12.28 \u96c6\u68a6\u859b\u5c0f\u8c26\u3010\u5c01\u53f7\u6597\u7f57\u3011\u76f4\u64ad\u540d\u573a\u9762\",\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tNew().Extract(tt.args.URL, extractors.Options{})\n\t\t})\n\t}\n}\n\n// Path: extractors/instagram/instagram.go\npackage instagram\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\tnetURL \"net/url\"\n\t\"regexp\"\n\t\"strings\"\n\t\"time\"\n\n\tbrowser \"github.com/EDDYCJY/fake-useragent\"\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/request\"\n\t\"github.com/iawia002/lux/utils\"\n)\n\nvar client *http.Client\n\nfunc init() {\n\textractors.Register(\"instagram\", New())\n\tclient = &http.Client{\n\t\tTimeout: 10 * time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tDial: (&net.Dialer{\n\t\t\t\tTimeout: 5 * time.Second,\n\t\t\t}).Dial,\n\t\t\tTLSHandshakeTimeout: 5 * time.Second,\n\t\t},\n\t}\n}\n\n// sliderItemNode contains information about the Instagram post\ntype sliderItemNode struct {\n\tDisplayURL string `json:\"display_url\"` // URL of the Media (resolution is dynamic)\n\n\tIsVideo  bool   `json:\"is_video\"`  // Is type of the Media equals to video\n\tVideoURL string `json:\"video_url\"` // Direct URL to the Video\n}\n\nfunc (s sliderItemNode) extractMediaURL() string {\n\tif s.IsVideo {\n\t\treturn s.VideoURL\n\t}\n\n\treturn s.DisplayURL\n}\n\ntype instagramPayload struct {\n\tMedia struct {\n\t\tID          string `json:\"id\"` // Unique ID of the Media\n\t\tSliderItems struct {\n\t\t\tEdges []struct {\n\t\t\t\tNode sliderItemNode `json:\"node\"`\n\t\t\t} `json:\"edges\"`\n\t\t} `json:\"edge_sidecar_to_children\"` // Children of the Media\n\t} `json:\"shortcode_media\"` // Media\n}\n\nfunc (s instagramPayload) isEmpty() bool {\n\treturn s.Media.ID == \"\"\n}\n\nfunc getPostWithCode(code string) ([]string, error) {\n\tURL := fmt.Sprintf(\"https://www.instagram.com/p/%v/embed/captioned/\", code)\n\n\tvar embeddedMediaImage string\n\tvar embedResponse = instagramPayload{}\n\tcollector := colly.NewCollector()\n\tcollector.SetClient(client)\n\tvar collectorErr error\n\n\tcollector.OnHTML(\"img.EmbeddedMediaImage\", func(e *colly.HTMLElement) {\n\t\tembeddedMediaImage = e.Attr(\"src\")\n\t})\n\n\tcollector.OnHTML(\"script\", func(e *colly.HTMLElement) {\n\t\tr := regexp.MustCompile(`\\\\\\\"gql_data\\\\\\\":([\\s\\S]*)\\}\\\"\\}\\]\\]\\,\\[\\\"NavigationMetrics`)\n\t\tmatch := r.FindStringSubmatch(e.Text)\n\n\t\tif len(match) < 2 {\n\t\t\treturn\n\t\t}\n\n\t\ts := strings.ReplaceAll(match[1], `\\\"`, `\"`)\n\t\ts = strings.ReplaceAll(s, `\\\\/`, `/`)\n\t\ts = strings.ReplaceAll(s, `\\\\`, `\\`)\n\n\t\terr := json.Unmarshal([]byte(s), &embedResponse)\n\t\tif err != nil {\n\t\t\tcollectorErr = err\n\t\t}\n\t})\n\n\tcollector.OnRequest(func(r *colly.Request) {\n\t\tr.Headers.Set(\"User-Agent\", browser.Chrome())\n\t})\n\n\tif err := collector.Visit(URL); err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to send HTTP request to the Instagram: %v\", err)\n\t}\n\n\tif collectorErr != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse the Instagram response: %v\", collectorErr)\n\t}\n\n\t// If the method one which is JSON parsing didn't fail\n\tif !embedResponse.isEmpty() {\n\t\tresult := make([]string, 0, len(embedResponse.Media.SliderItems.Edges))\n\t\tfor _, item := range embedResponse.Media.SliderItems.Edges {\n\t\t\tresult = append(result, item.Node.extractMediaURL())\n\t\t}\n\n\t\treturn result, nil\n\t}\n\n\tif embeddedMediaImage != \"\" {\n\t\treturn []string{embeddedMediaImage}, nil\n\t}\n\n\t// If every two methods have failed, then return an error\n\treturn nil, errors.New(\"failed to fetch the post, the page might be \\\"private\\\", or the link is completely wrong\")\n}\n\n\nfunc extractShortCodeFromLink(link string) (string, error) {\n\tvalues := regexp.MustCompile(`(p|tv|reel|reels\\/videos)\\/([A-Za-z0-9-_]+)`).FindStringSubmatch(link)\n\tif len(values) != 3 {\n\t\treturn \"\", errors.New(\"couldn't extract the media short code from the link\")\n\t}\n\n\treturn values[2], nil\n}\n\ntype extractor struct{}\n\n// New returns a instagram extractor.\nfunc New() extractors.Extractor {\n\treturn &extractor{}\n}\n\n// Extract is the main function to extract the data.\nfunc (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {\n\tu, err := netURL.Parse(url)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tshortCode, err := extractShortCodeFromLink(u.String())\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\turls, err := getPostWithCode(shortCode)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(err)\n\t}\n\n\tvar totalSize int64\n\tvar parts []*extractors.Part\n\n\tfor _, u := range urls {\n\t\t_, ext, err := utils.GetNameAndExt(u)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\t\tfileSize, err := request.Size(u, url)\n\t\tif err != nil {\n\t\t\treturn nil, errors.WithStack(err)\n\t\t}\n\n\t\tpart := &extractors.Part{\n\t\t\tURL:  u,\n\t\t\tSize: fileSize,\n\t\t\tExt:  ext,\n\t\t}\n\t\tparts = append(parts, part)\n\t}\n\n\tfor _, part := range parts {\n\t\ttotalSize += part.Size\n\t}\n\n\tstreams := map[string]*extractors.Stream{\n\t\t\"default\": {\n\t\t\tParts: parts,\n\t\t\tSize:  totalSize,\n\t\t},\n\t}\n\n\treturn []*extractors.Data{\n\t\t{\n\t\t\tSite:    \"Instagram instagram.com\",\n\t\t\tTitle:   \"Instagram \" + shortCode,\n\t\t\tType:    extractors.DataTypeImage,\n\t\t\tStreams: streams,\n\t\t\tURL:     url,\n\t\t},\n\t}, nil\n}\n\n// Path: extractors/instagram/instagram_test.go\npackage instagram\n\nimport (\n\t\"testing\"\n\n\t\"github.com/iawia002/lux/extractors\"\n\t\"github.com/iawia002/lux/test\"\n)\n\nfunc TestDownload(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs test.Args\n\t}{\n\t\t{\n\t\t\tname: \"video test\",\n\t\t\targs: test.Args{\n\t\t\t\tURL:   \"https://www.instagram.com/p/BlIka1ZFCNr\",\n...\n", "needle_token_start": 14323, "needle_token_end": 14435, "code_context_ntokens": 15125, "output": ["```go\nfunc extractShortCodeFromLink(link string) (string, error) {\n\tvalues := regexp.MustCompile(`(p|tv|reel|reels\\/videos)\\/([A-Za-z0-9-_]+)`).FindStringSubmatch(link)\n\tif len(values) != 3 {\n\t\treturn \"\", errors.New(\"couldn't extract the media short code from the link\")\n\t}\n\n\treturn values[2], nil\n}\n```"]}
{"repo": "lima-vm/lima", "name": "initLogrus", "language": "go", "path": "cmd/limactl/hostagent.go", "position_ratio": 0.05, "description": "\nFunction Description:\n1. **Purpose**: To configure the logging system used throughout the application, setting the output destination, format, and verbosity level.\n2. **Input**: An `io.Writer` interface where log messages will be directed.\n3. **Output**: There is no direct output from this function; however, it affects the global logging behavior by configuring the logging output and format.\n4. **Procedure**: \n   - Sets the destination for log messages to the provided writer.\n   - Configures the log format to JSON, which is suitable for parsing by other components.\n   - Adjusts the logging level based on the current level; if it is set to debug, it increases the verbosity to trace, otherwise, it sets it to debug.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\n\tinstName := args[0]\n\n\trunGUI, err := cmd.Flags().GetBool(\"run-gui\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif runGUI {\n\t\t// Without this the call to vz.RunGUI fails. Adding it here, as this has to be called before the vz cgo loads.\n\t\truntime.LockOSThread()\n\t\tdefer runtime.UnlockOSThread()\n\t}\n\n\tsignalCh := make(chan os.Signal, 1)\n\tsignal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)\n\n\tstdout := &syncWriter{w: cmd.OutOrStdout()}\n\tstderr := &syncWriter{w: cmd.ErrOrStderr()}\n\n\tinitLogrus(stderr)\n\tvar opts []hostagent.Opt\n\tnerdctlArchive, err := cmd.Flags().GetString(\"nerdctl-archive\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif nerdctlArchive != \"\" {\n\t\topts = append(opts, hostagent.WithNerdctlArchive(nerdctlArchive))\n\t}\n\tha, err := hostagent.New(instName, stdout, signalCh, opts...)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbackend := &server.Backend{\n\t\tAgent: ha,\n\t}\n\tr := http.NewServeMux()\n\tserver.AddRoutes(r, backend)\n\tsrv := &http.Server{Handler: r}\n\terr = os.RemoveAll(socket)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl, err := net.Listen(\"unix\", socket)\n\tlogrus.Infof(\"hostagent socket created at %s\", socket)\n\tif err != nil {\n\t\treturn err\n\t}\n\tgo func() {\n\t\tdefer os.RemoveAll(socket)\n\t\tdefer srv.Close()\n\t\tif serveErr := srv.Serve(l); serveErr != nil {\n\t\t\tlogrus.WithError(serveErr).Warn(\"hostagent API server exited with an error\")\n\t\t}\n\t}()\n\treturn ha.Run(cmd.Context())\n}\n\n// syncer is implemented by *os.File\ntype syncer interface {\n\tSync() error\n}\n\ntype syncWriter struct {\n\tw io.Writer\n}\n\nfunc (w *syncWriter) Write(p []byte) (int, error) {\n\twritten, err := w.w.Write(p)\n\tif err == nil {\n\t\tif s, ok := w.w.(syncer); ok {\n\t\t\t_ = s.Sync()\n\t\t}\n\t}\n\treturn written, err\n}\n\n\nfunc initLogrus(stderr io.Writer) {\n\tlogrus.SetOutput(stderr)\n\t// JSON logs are parsed in pkg/hostagent/events.Watcher()\n\tlogrus.SetFormatter(new(logrus.JSONFormatter))\n\t// HostAgent logging is one level more verbose than the start command itself\n\tif logrus.GetLevel() == logrus.DebugLevel {\n\t\tlogrus.SetLevel(logrus.TraceLevel)\n\t} else {\n\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t}\n}\n\n// Path: cmd/limactl/list.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/cheggaaa/pb/v3/termutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc fieldNames() []string {\n\tnames := []string{}\n\tvar data store.FormatData\n\tt := reflect.TypeOf(data)\n\tfor i := 0; i < t.NumField(); i++ {\n\t\tf := t.Field(i)\n\t\tif f.Anonymous {\n\t\t\tfor j := 0; j < f.Type.NumField(); j++ {\n\t\t\t\tnames = append(names, f.Type.Field(j).Name)\n\t\t\t}\n\t\t} else {\n\t\t\tnames = append(names, t.Field(i).Name)\n\t\t}\n\t}\n\treturn names\n}\n\nfunc newListCommand() *cobra.Command {\n\tlistCommand := &cobra.Command{\n\t\tUse:     \"list [flags] [INSTANCE]...\",\n\t\tAliases: []string{\"ls\"},\n\t\tShort:   \"List instances of Lima.\",\n\t\tLong: `List instances of Lima.\n\nThe output can be presented in one of several formats, using the --format <format> flag.\n\n  --format json  - output in json format\n  --format yaml  - output in yaml format\n  --format table - output in table format\n  --format '{{ <go template> }}' - if the format begins and ends with '{{ }}', then it is used as a go template.\n` + store.FormatHelp + `\nThe following legacy flags continue to function:\n  --json - equal to '--format json'`,\n\t\tArgs:              WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:              listAction,\n\t\tValidArgsFunction: listBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tlistCommand.Flags().StringP(\"format\", \"f\", \"table\", \"output format, one of: json, yaml, table, go-template\")\n\tlistCommand.Flags().Bool(\"list-fields\", false, \"List fields available for format\")\n\tlistCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\tlistCommand.Flags().BoolP(\"quiet\", \"q\", false, \"Only show names\")\n\tlistCommand.Flags().Bool(\"all-fields\", false, \"Show all fields\")\n\n\treturn listCommand\n}\n\nfunc instanceMatches(arg string, instances []string) []string {\n\tmatches := []string{}\n\tfor _, instance := range instances {\n\t\tif instance == arg {\n\t\t\tmatches = append(matches, instance)\n\t\t}\n\t}\n\treturn matches\n}\n\nfunc listAction(cmd *cobra.Command, args []string) error {\n\tquiet, err := cmd.Flags().GetBool(\"quiet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tlistFields, err := cmd.Flags().GetBool(\"list-fields\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif jsonFormat {\n\t\tformat = \"json\"\n\t}\n\n\t// conflicts\n\tif jsonFormat && cmd.Flags().Changed(\"format\") {\n\t\treturn errors.New(\"option --json conflicts with option --format\")\n\t}\n\tif listFields && cmd.Flags().Changed(\"format\") {\n\t\treturn errors.New(\"option --list-fields conflicts with option --format\")\n\t}\n\n\tif quiet && format != \"table\" {\n\t\treturn errors.New(\"option --quiet can only be used with '--format table'\")\n\t}\n\n\tif listFields {\n\t\tnames := fieldNames()\n\t\tsort.Strings(names)\n\t\tfmt.Println(strings.Join(names, \"\\n\"))\n\t\treturn nil\n\t}\n\n\tallinstances, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(allinstances) == 0 {\n\t\tlogrus.Warn(\"No instance found. Run `limactl create` to create an instance.\")\n\t\treturn nil\n\t}\n\n\tinstanceNames := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := instanceMatches(arg, allinstances)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tinstanceNames = append(instanceNames, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No instance matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tinstanceNames = allinstances\n\t}\n\n\tif quiet {\n\t\tfor _, instName := range instanceNames {\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), instName)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// get the state and config for all the requested instances\n\tvar instances []*store.Instance\n\tfor _, instanceName := range instanceNames {\n\t\tinstance, err := store.Inspect(instanceName)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"unable to load instance %s: %w\", instanceName, err)\n\t\t}\n\t\tinstances = append(instances, instance)\n\t}\n\n\tfor _, instance := range instances {\n\t\tif len(instance.Errors) > 0 {\n\t\t\tlogrus.WithField(\"errors\", instance.Errors).Warnf(\"instance %q has errors\", instance.Name)\n\t\t}\n\t}\n\n\tallFields, err := cmd.Flags().GetBool(\"all-fields\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\toptions := store.PrintOptions{AllFields: allFields}\n\tout := cmd.OutOrStdout()\n\tif out == os.Stdout {\n\t\tif isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t\tif w, err := termutil.TerminalWidth(); err == nil {\n\t\t\t\toptions.TerminalWidth = w\n\t\t\t}\n\t\t}\n\t}\n\treturn store.PrintInstances(out, instances, format, &options)\n}\n\nfunc listBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/main.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store/dirnames\"\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst (\n\tDefaultInstanceName = \"default\"\n\tbasicCommand        = \"basic\"\n\tadvancedCommand     = \"advanced\"\n)\n\nfunc main() {\n\tif err := newApp().Execute(); err != nil {\n\t\thandleExitCoder(err)\n\t\tlogrus.Fatal(err)\n\t}\n}\n\nfunc newApp() *cobra.Command {\n\ttemplatesDir := \"$PREFIX/share/lima/templates\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n\t\ttemplatesDir = filepath.Join(prefixDir, \"share/lima/templates\")\n\t}\n\n\trootCmd := &cobra.Command{\n\t\tUse:     \"limactl\",\n\t\tShort:   \"Lima: Linux virtual machines\",\n\t\tVersion: strings.TrimPrefix(version.Version, \"v\"),\n\t\tExample: fmt.Sprintf(`  Start the default instance:\n  $ limactl start\n\n  Open a shell:\n  $ lima\n\n  Run a container:\n  $ lima nerdctl run -d --name nginx -p 8080:80 nginx:alpine\n\n  Stop the default instance:\n  $ limactl stop\n\n  See also template YAMLs: %s`, templatesDir),\n\t\tSilenceUsage:      true,\n\t\tSilenceErrors:     true,\n\t\tDisableAutoGenTag: true,\n\t}\n\trootCmd.PersistentFlags().String(\"log-level\", \"\", \"Set the logging level [trace, debug, info, warn, error]\")\n\trootCmd.PersistentFlags().Bool(\"debug\", false, \"debug mode\")\n\t// TODO: \"survey\" does not support using cygwin terminal on windows yet\n\trootCmd.PersistentFlags().Bool(\"tty\", isatty.IsTerminal(os.Stdout.Fd()), \"Enable TUI interactions such as opening an editor. Defaults to true when stdout is a terminal. Set to false for automation.\")\n\trootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n\t\tl, _ := cmd.Flags().GetString(\"log-level\")\n\t\tif l != \"\" {\n\t\t\tlvl, err := logrus.ParseLevel(l)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tlogrus.SetLevel(lvl)\n\t\t}\n\t\tdebug, _ := cmd.Flags().GetBool(\"debug\")\n\t\tif debug {\n\t\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t\t}\n\n\t\tif osutil.IsBeingRosettaTranslated() {\n\t\t\t// running under rosetta would provide inappropriate runtime.GOARCH info, see: https://github.com/lima-vm/lima/issues/543\n\t\t\treturn errors.New(\"limactl is running under rosetta, please reinstall lima with native arch\")\n\t\t}\n\n\t\tif runtime.GOOS == \"windows\" && isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t\tformatter := new(logrus.TextFormatter)\n\t\t\t// the default setting does not recognize cygwin on windows\n\t\t\tformatter.ForceColors = true\n\t\t\tlogrus.StandardLogger().SetFormatter(formatter)\n\t\t}\n\t\tif os.Geteuid() == 0 && cmd.Name() != \"generate-doc\" {\n\t\t\treturn errors.New(\"must not run as the root\")\n\t\t}\n\t\t// Make sure either $HOME or $LIMA_HOME is defined, so we don't need\n\t\t// to check for errors later\n\t\tif _, err := dirnames.LimaDir(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\trootCmd.AddGroup(&cobra.Group{ID: \"basic\", Title: \"Basic Commands:\"})\n\trootCmd.AddGroup(&cobra.Group{ID: \"advanced\", Title: \"Advanced Commands:\"})\n\trootCmd.AddCommand(\n\t\tnewCreateCommand(),\n\t\tnewStartCommand(),\n\t\tnewStopCommand(),\n\t\tnewShellCommand(),\n\t\tnewCopyCommand(),\n\t\tnewListCommand(),\n\t\tnewDeleteCommand(),\n\t\tnewValidateCommand(),\n\t\tnewSudoersCommand(),\n\t\tnewPruneCommand(),\n\t\tnewHostagentCommand(),\n\t\tnewInfoCommand(),\n\t\tnewShowSSHCommand(),\n\t\tnewDebugCommand(),\n\t\tnewEditCommand(),\n\t\tnewFactoryResetCommand(),\n\t\tnewDiskCommand(),\n\t\tnewUsernetCommand(),\n\t\tnewGenDocCommand(),\n\t\tnewSnapshotCommand(),\n\t\tnewProtectCommand(),\n\t\tnewUnprotectCommand(),\n\t)\n\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\trootCmd.AddCommand(startAtLoginCommand())\n\t}\n\n\treturn rootCmd\n}\n\ntype ExitCoder interface {\n\terror\n\tExitCode() int\n}\n\nfunc handleExitCoder(err error) {\n\tif err == nil {\n\t\treturn\n\t}\n\n\tif exitErr, ok := err.(ExitCoder); ok {\n\t\tos.Exit(exitErr.ExitCode())\n\t\treturn\n\t}\n}\n\n// WrapArgsError annotates cobra args error with some context, so the error message is more user-friendly\nfunc WrapArgsError(argFn cobra.PositionalArgs) cobra.PositionalArgs {\n\treturn func(cmd *cobra.Command, args []string) error {\n\t\terr := argFn(cmd, args)\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\treturn fmt.Errorf(\"%q %s.\\nSee '%s --help'.\\n\\nUsage:  %s\\n\\n%s\",\n\t\t\tcmd.CommandPath(), err.Error(),\n\t\t\tcmd.CommandPath(),\n\t\t\tcmd.UseLine(), cmd.Short,\n\t\t)\n\t}\n}\n\n// Path: cmd/limactl/protect.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newProtectCommand() *cobra.Command {\n\tprotectCommand := &cobra.Command{\n\t\tUse:   \"protect INSTANCE [INSTANCE, ...]\",\n\t\tShort: \"Protect an instance to prohibit accidental removal\",\n\t\tLong: `Protect an instance to prohibit accidental removal via the 'limactl delete' command.\nThe instance is not being protected against removal via '/bin/rm', Finder, etc.`,\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              protectAction,\n\t\tValidArgsFunction: protectBashComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn protectCommand\n}\n\nfunc protectAction(_ *cobra.Command, args []string) error {\n\tvar errs []error\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to inspect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tif inst.Protected {\n\t\t\tlogrus.Warnf(\"Instance %q is already protected. Skipping.\", instName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := inst.Protect(); err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to protect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tlogrus.Infof(\"Protected %q\", instName)\n\t}\n\treturn errors.Join(errs...)\n}\n\nfunc protectBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/prune.go\npackage main\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newPruneCommand() *cobra.Command {\n\tpruneCommand := &cobra.Command{\n\t\tUse:               \"prune\",\n\t\tShort:             \"Prune garbage objects\",\n\t\tArgs:              WrapArgsError(cobra.NoArgs),\n\t\tRunE:              pruneAction,\n\t\tValidArgsFunction: cobra.NoFileCompletions,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn pruneCommand\n}\n\nfunc pruneAction(_ *cobra.Command, _ []string) error {\n\tucd, err := os.UserCacheDir()\n\tif err != nil {\n\t\treturn err\n\t}\n\tcacheDir := filepath.Join(ucd, \"lima\")\n\tlogrus.Infof(\"Pruning %q\", cacheDir)\n\treturn os.RemoveAll(cacheDir)\n}\n\n// Path: cmd/limactl/shell.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/alessio/shellescape\"\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/mattn/go-shellwords\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\n// Environment variable that allows configuring the command (alias) to execute\n// in place of the 'ssh' executable.\nconst envShellSSH = \"SSH\"\n\nconst shellHelp = `Execute shell in Lima\n\nlima command is provided as an alias for limactl shell $LIMA_INSTANCE. $LIMA_INSTANCE defaults to \"` + DefaultInstanceName + `\".\n\nBy default, the first 'ssh' executable found in the host's PATH is used to connect to the Lima instance.\nA custom ssh alias can be used instead by setting the $` + envShellSSH + ` environment variable.\n\nHint: try --debug to show the detailed logs, if it seems hanging (mostly due to some SSH issue).\n`\n\nfunc newShellCommand() *cobra.Command {\n\tshellCmd := &cobra.Command{\n\t\tUse:               \"shell [flags] INSTANCE [COMMAND...]\",\n\t\tShort:             \"Execute shell in Lima\",\n\t\tLong:              shellHelp,\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              shellAction,\n\t\tValidArgsFunction: shellBashComplete,\n\t\tSilenceErrors:     true,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tshellCmd.Flags().SetInterspersed(false)\n\n\tshellCmd.Flags().String(\"shell\", \"\", \"shell interpreter, e.g. /bin/bash\")\n\tshellCmd.Flags().String(\"workdir\", \"\", \"working directory\")\n\treturn shellCmd\n}\n\nfunc shellAction(cmd *cobra.Command, args []string) error {\n\t// simulate the behavior of double dash\n\tnewArg := []string{}\n\tif len(args) >= 2 && args[1] == \"--\" {\n\t\tnewArg = append(newArg, args[:1]...)\n\t\tnewArg = append(newArg, args[2:]...)\n\t\targs = newArg\n\t}\n\tinstName := args[0]\n\n\tif len(args) >= 2 {\n\t\tswitch args[1] {\n\t\tcase \"create\", \"start\", \"delete\", \"shell\":\n\t\t\t// `lima start` (alias of `limactl $LIMA_INSTANCE start`) is probably a typo of `limactl start`\n\t\t\tlogrus.Warnf(\"Perhaps you meant `limactl %s`?\", strings.Join(args[1:], \" \"))\n\t\t}\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t}\n\t\treturn err\n\t}\n\tif inst.Status == store.StatusStopped {\n\t\treturn fmt.Errorf(\"instance %q is stopped, run `limactl start %s` to start the instance\", instName, instName)\n\t}\n\ty, err := inst.LoadYAML()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// When workDir is explicitly set, the shell MUST have workDir as the cwd, or exit with an error.\n\t//\n\t// changeDirCmd := \"cd workDir || exit 1\"                  if workDir != \"\"\n\t//              := \"cd hostCurrentDir || cd hostHomeDir\"   if workDir == \"\"\n\tvar changeDirCmd string\n\tworkDir, err := cmd.Flags().GetString(\"workdir\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif workDir != \"\" {\n\t\tchangeDirCmd = fmt.Sprintf(\"cd %s || exit 1\", shellescape.Quote(workDir))\n\t\t// FIXME: check whether y.Mounts contains the home, not just len > 0\n\t} else if len(y.Mounts) > 0 {\n\t\thostCurrentDir, err := os.Getwd()\n\t\tif err == nil {\n\t\t\tchangeDirCmd = fmt.Sprintf(\"cd %s\", shellescape.Quote(hostCurrentDir))\n\t\t} else {\n\t\t\tchangeDirCmd = \"false\"\n\t\t\tlogrus.WithError(err).Warn(\"failed to get the current directory\")\n\t\t}\n\t\thostHomeDir, err := os.UserHomeDir()\n\t\tif err == nil {\n\t\t\tchangeDirCmd = fmt.Sprintf(\"%s || cd %s\", changeDirCmd, shellescape.Quote(hostHomeDir))\n\t\t} else {\n\t\t\tlogrus.WithError(err).Warn(\"failed to get the home directory\")\n\t\t}\n\t} else {\n\t\tlogrus.Debug(\"the host home does not seem mounted, so the guest shell will have a different cwd\")\n\t}\n\n\tif changeDirCmd == \"\" {\n\t\tchangeDirCmd = \"false\"\n\t}\n\tlogrus.Debugf(\"changeDirCmd=%q\", changeDirCmd)\n\n\tshell, err := cmd.Flags().GetString(\"shell\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif shell == \"\" {\n\t\tshell = `\"$SHELL\"`\n\t} else {\n\t\tshell = shellescape.Quote(shell)\n\t}\n\tscript := fmt.Sprintf(\"%s ; exec %s --login\", changeDirCmd, shell)\n\tif len(args) > 1 {\n\t\tquotedArgs := make([]string, len(args[1:]))\n\t\tparsingEnv := true\n\t\tfor i, arg := range args[1:] {\n\t\t\tif parsingEnv && isEnv(arg) {\n\t\t\t\tquotedArgs[i] = quoteEnv(arg)\n\t\t\t} else {\n\t\t\t\tparsingEnv = false\n\t\t\t\tquotedArgs[i] = shellescape.Quote(arg)\n\t\t\t}\n\t\t}\n\t\tscript += fmt.Sprintf(\n\t\t\t\" -c %s\",\n\t\t\tshellescape.Quote(strings.Join(quotedArgs, \" \")),\n\t\t)\n\t}\n\n\tvar arg0 string\n\tvar arg0Args []string\n\n\tif sshShell := os.Getenv(envShellSSH); sshShell != \"\" {\n\t\tsshShellFields, err := shellwords.Parse(sshShell)\n\t\tswitch {\n\t\tcase err != nil:\n\t\t\tlogrus.WithError(err).Warnf(\"Failed to split %s variable into shell tokens. \"+\n\t\t\t\t\"Falling back to 'ssh' command\", envShellSSH)\n\t\tcase len(sshShellFields) > 0:\n\t\t\targ0 = sshShellFields[0]\n\t\t\tif len(sshShellFields) > 1 {\n\t\t\t\targ0Args = sshShellFields[1:]\n\t\t\t}\n\t\t}\n\t}\n\n\tif arg0 == \"\" {\n\t\targ0, err = exec.LookPath(\"ssh\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tsshOpts, err := sshutil.SSHOpts(inst.Dir, *y.SSH.LoadDotSSHPubKeys, *y.SSH.ForwardAgent, *y.SSH.ForwardX11, *y.SSH.ForwardX11Trusted)\n\tif err != nil {\n\t\treturn err\n\t}\n\tsshArgs := sshutil.SSHArgsFromOpts(sshOpts)\n\tif isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t// required for showing the shell prompt: https://stackoverflow.com/a/626574\n\t\tsshArgs = append(sshArgs, \"-t\")\n\t}\n\tif _, present := os.LookupEnv(\"COLORTERM\"); present {\n\t\t// SendEnv config is cumulative, with already existing options in ssh_config\n\t\tsshArgs = append(sshArgs, \"-o\", \"SendEnv=COLORTERM\")\n\t}\n\tsshArgs = append(sshArgs, []string{\n\t\t\"-q\",\n\t\t\"-p\", strconv.Itoa(inst.SSHLocalPort),\n\t\tinst.SSHAddress,\n\t\t\"--\",\n\t\tscript,\n\t}...)\n\tsshCmd := exec.Command(arg0, append(arg0Args, sshArgs...)...)\n\tsshCmd.Stdin = os.Stdin\n\tsshCmd.Stdout = os.Stdout\n\tsshCmd.Stderr = os.Stderr\n\tlogrus.Debugf(\"executing ssh (may take a long)): %+v\", sshCmd.Args)\n\n\t// TODO: use syscall.Exec directly (results in losing tty?)\n\treturn sshCmd.Run()\n}\n\nfunc shellBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\nfunc isEnv(arg string) bool {\n\treturn len(strings.Split(arg, \"=\")) > 1\n}\n\nfunc quoteEnv(arg string) string {\n\tenv := strings.SplitN(arg, \"=\", 2)\n\tenv[1] = shellescape.Quote(env[1])\n\treturn strings.Join(env, \"=\")\n}\n\n// Path: cmd/limactl/show_ssh.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/dirnames\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst showSSHExample = `\n  \"cmd\" format (default): Full ssh command line.\n    $ limactl show-ssh --format=cmd default\n    ssh -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022 lima-default\n\n  \"args\" format: Similar to the cmd format but omits \"ssh\" and the destination address\n    $ limactl show-ssh --format=args default\n    -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022\n\n  \"options\" format: ssh option key value pairs\n    $ limactl show-ssh --format=options default\n    IdentityFile=\"/Users/example/.lima/_config/user\"\n    User=example\n    Hostname=127.0.0.1\n    Port=60022\n\n  \"config\" format: ~/.ssh/config format\n    $ limactl show-ssh --format=config default\n    Host lima-default\n      IdentityFile \"/Users/example/.lima/_config/user \"\n      User example\n      Hostname 127.0.0.1\n      Port 60022\n\n  To show the config file path:\n    $ limactl ls --format='{{.SSHConfigFile}}' default\n    /Users/example/.lima/default/ssh.config\n`\n\nfunc newShowSSHCommand() *cobra.Command {\n\tlimaHome := \"~/\" + dirnames.DotLima\n\tif s, err := dirnames.LimaDir(); err == nil {\n\t\tlimaHome = s\n\t}\n\tshellCmd := &cobra.Command{\n\t\tUse:   \"show-ssh [flags] INSTANCE\",\n\t\tShort: \"Show the ssh command line (DEPRECATED; use `ssh -F` instead)\",\n\t\tLong: fmt.Sprintf(`Show the ssh command line (DEPRECATED)\n\nWARNING: 'limactl show-ssh' is deprecated.\nInstead, use 'ssh -F %s/default/ssh.config lima-default' .\n`, limaHome),\n\t\tExample:           showSSHExample,\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:              showSSHAction,\n\t\tValidArgsFunction: showSSHBashComplete,\n\t\tSilenceErrors:     true,\n\t\tGroupID:           advancedCommand,\n\t}\n\n\tshellCmd.Flags().StringP(\"format\", \"f\", sshutil.FormatCmd, \"Format: \"+strings.Join(sshutil.Formats, \", \"))\n\t_ = shellCmd.RegisterFlagCompletionFunc(\"format\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn sshutil.Formats, cobra.ShellCompDirectiveNoFileComp\n\t})\n\treturn shellCmd\n}\n\nfunc showSSHAction(cmd *cobra.Command, args []string) error {\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tinstName := args[0]\n\tw := cmd.OutOrStdout()\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t}\n\t\treturn err\n\t}\n\tlogrus.Warnf(\"`limactl show-ssh` is deprecated. Instead, use `ssh -F %s lima-%s`.\",\n\t\tfilepath.Join(inst.Dir, filenames.SSHConfig), inst.Name)\n\ty, err := inst.LoadYAML()\n\tif err != nil {\n\t\treturn err\n\t}\n\topts, err := sshutil.SSHOpts(inst.Dir, *y.SSH.LoadDotSSHPubKeys, *y.SSH.ForwardAgent, *y.SSH.ForwardX11, *y.SSH.ForwardX11Trusted)\n\tif err != nil {\n\t\treturn err\n\t}\n\topts = append(opts, \"Hostname=127.0.0.1\")\n\topts = append(opts, fmt.Sprintf(\"Port=%d\", inst.SSHLocalPort))\n\treturn sshutil.Format(w, instName, format, opts)\n}\n\nfunc showSSHBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/snapshot.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/snapshot\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSnapshotCommand() *cobra.Command {\n\tsnapshotCmd := &cobra.Command{\n\t\tUse:   \"snapshot\",\n\t\tShort: \"Manage instance snapshots\",\n\t\tPersistentPreRun: func(cmd *cobra.Command, args []string) {\n\t\t\tlogrus.Warn(\"`limactl snapshot` is experimental\")\n\t\t},\n\t\tGroupID: advancedCommand,\n\t}\n\tsnapshotCmd.AddCommand(newSnapshotApplyCommand())\n\tsnapshotCmd.AddCommand(newSnapshotCreateCommand())\n\tsnapshotCmd.AddCommand(newSnapshotDeleteCommand())\n\tsnapshotCmd.AddCommand(newSnapshotListCommand())\n\n\treturn snapshotCmd\n}\n\nfunc newSnapshotCreateCommand() *cobra.Command {\n\tcreateCmd := &cobra.Command{\n\t\tUse:               \"create INSTANCE\",\n\t\tAliases:           []string{\"save\"},\n\t\tShort:             \"Create (save) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotCreateAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tcreateCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn createCmd\n}\n\nfunc snapshotCreateAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Save(ctx, inst, tag)\n}\n\nfunc newSnapshotDeleteCommand() *cobra.Command {\n\tdeleteCmd := &cobra.Command{\n\t\tUse:               \"delete INSTANCE\",\n\t\tAliases:           []string{\"del\"},\n\t\tShort:             \"Delete (del) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotDeleteAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tdeleteCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn deleteCmd\n}\n\nfunc snapshotDeleteAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Del(ctx, inst, tag)\n}\n\nfunc newSnapshotApplyCommand() *cobra.Command {\n\tapplyCmd := &cobra.Command{\n\t\tUse:               \"apply INSTANCE\",\n\t\tAliases:           []string{\"load\"},\n\t\tShort:             \"Apply (load) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotApplyAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tapplyCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn applyCmd\n}\n\nfunc snapshotApplyAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Load(ctx, inst, tag)\n}\n\nfunc newSnapshotListCommand() *cobra.Command {\n\tlistCmd := &cobra.Command{\n\t\tUse:               \"list INSTANCE\",\n\t\tAliases:           []string{\"ls\"},\n\t\tShort:             \"List existing snapshots\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotListAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tlistCmd.Flags().BoolP(\"quiet\", \"q\", false, \"Only show tags\")\n\n\treturn listCmd\n}\n\nfunc snapshotListAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tquiet, err := cmd.Flags().GetBool(\"quiet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tctx := cmd.Context()\n\tout, err := snapshot.List(ctx, inst)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif quiet {\n\t\tfor i, line := range strings.Split(out, \"\\n\") {\n\t\t\t// \"ID\", \"TAG\", \"VM SIZE\", \"DATE\", \"VM CLOCK\", \"ICOUNT\"\n\t\t\tfields := strings.Fields(line)\n\t\t\tif i == 0 && len(fields) > 1 && fields[1] != \"TAG\" {\n\t\t\t\t// make sure that output matches the expected\n\t\t\t\treturn fmt.Errorf(\"unknown header: %s\", line)\n\t\t\t}\n\t\t\tif i == 0 || line == \"\" {\n\t\t\t\t// skip header and empty line after using split\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttag := fields[1]\n\t\t\tfmt.Printf(\"%s\\n\", tag)\n\t\t}\n\t\treturn nil\n\t}\n\tfmt.Print(out)\n\treturn nil\n}\n\nfunc snapshotBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/start-at-login.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc startAtLoginCommand() *cobra.Command {\n\tstartAtLoginCommand := &cobra.Command{\n\t\tUse:               \"start-at-login INSTANCE\",\n\t\tShort:             \"Register/Unregister an autostart file for the instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              startAtLoginAction,\n\t\tValidArgsFunction: startAtLoginComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\n\tstartAtLoginCommand.Flags().Bool(\n\t\t\"enabled\", true,\n\t\t\"Automatically start the instance when the user logs in\",\n\t)\n\n\treturn startAtLoginCommand\n}\n\nfunc startAtLoginAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\tlogrus.Infof(\"Instance %q not found\", instName)\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\n\tflags := cmd.Flags()\n\tstartAtLogin, err := flags.GetBool(\"enabled\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif startAtLogin {\n\t\tif err := autostart.CreateStartAtLoginEntry(runtime.GOOS, inst.Name, inst.Dir); err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"Can't create an autostart file for instance %q\", inst.Name)\n\t\t} else {\n\t\t\tlogrus.Infof(\"The autostart file %q has been created or updated\", autostart.GetFilePath(runtime.GOOS, inst.Name))\n\t\t}\n\t} else {\n\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"The autostart file %q could not be deleted\", instName)\n\t\t} else if deleted {\n\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc startAtLoginComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/sudoers.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/networks\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSudoersCommand() *cobra.Command {\n\tnetworksMD := \"$PREFIX/share/doc/lima/docs/network.md\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n\t\tnetworksMD = filepath.Join(prefixDir, \"share/doc/lima/docs/network.md\")\n\t}\n\tsudoersCommand := &cobra.Command{\n\t\tUse: \"sudoers [--check [SUDOERSFILE-TO-CHECK]]\",\n\t\tExample: `\nTo generate the /etc/sudoers.d/lima file:\n$ limactl sudoers | sudo tee /etc/sudoers.d/lima\n\nTo validate the existing /etc/sudoers.d/lima file:\n$ limactl sudoers --check /etc/sudoers.d/lima\n`,\n\t\tShort: \"Generate the content of the /etc/sudoers.d/lima file\",\n\t\tLong: fmt.Sprintf(`Generate the content of the /etc/sudoers.d/lima file for enabling vmnet.framework support.\nThe content is written to stdout, NOT to the file.\nThis command must not run as the root.\nSee %s for the usage.`, networksMD),\n\t\tArgs:    WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:    sudoersAction,\n\t\tGroupID: advancedCommand,\n\t}\n\tconfigFile, _ := networks.ConfigFile()\n\tsudoersCommand.Flags().Bool(\"check\", false,\n\t\tfmt.Sprintf(\"check that the sudoers file is up-to-date with %q\", configFile))\n\treturn sudoersCommand\n}\n\nfunc sudoersAction(cmd *cobra.Command, args []string) error {\n\tif runtime.GOOS != \"darwin\" {\n\t\treturn errors.New(\"sudoers command is only supported on macOS right now\")\n\t}\n\tcheck, err := cmd.Flags().GetBool(\"check\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif check {\n\t\treturn verifySudoAccess(args)\n\t}\n\tswitch len(args) {\n\tcase 0:\n\t\t// NOP\n\tcase 1:\n\t\treturn fmt.Errorf(\"the file argument can be specified only for --check mode\")\n\tdefault:\n\t\treturn fmt.Errorf(\"unexpected arguments %v\", args)\n\t}\n\tsudoers, err := networks.Sudoers()\n\tif err != nil {\n\t\treturn err\n\t}\n\tfmt.Print(sudoers)\n\treturn nil\n}\n\nfunc verifySudoAccess(args []string) error {\n\tconfig, err := networks.Config()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := config.Validate(); err != nil {\n\t\treturn err\n\t}\n\tvar file string\n\tswitch len(args) {\n\tcase 0:\n\t\tfile = config.Paths.Sudoers\n\t\tif file == \"\" {\n\t\t\tconfigFile, _ := networks.ConfigFile()\n\t\t\treturn fmt.Errorf(\"no sudoers file defined in %q\", configFile)\n\t\t}\n\tcase 1:\n\t\tfile = args[0]\n\tdefault:\n\t\treturn errors.New(\"can check only a single sudoers file\")\n\t}\n\tif err := config.VerifySudoAccess(file); err != nil {\n\t\treturn err\n\t}\n\tfmt.Printf(\"%q is up-to-date (or sudo doesn't require a password)\\n\", file)\n\treturn nil\n}\n\n// Path: cmd/limactl/unprotect.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newUnprotectCommand() *cobra.Command {\n\tunprotectCommand := &cobra.Command{\n\t\tUse:               \"unprotect INSTANCE [INSTANCE, ...]\",\n\t\tShort:             \"Unprotect an instance\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              unprotectAction,\n\t\tValidArgsFunction: unprotectBashComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn unprotectCommand\n}\n\nfunc unprotectAction(_ *cobra.Command, args []string) error {\n\tvar errs []error\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to inspect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tif !inst.Protected {\n\t\t\tlogrus.Warnf(\"Instance %q isn't protected. Skipping.\", instName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := inst.Unprotect(); err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to unprotect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tlogrus.Infof(\"Unprotected %q\", instName)\n\t}\n\treturn errors.Join(errs...)\n}\n\nfunc unprotectBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/usernet.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\n\t\"github.com/lima-vm/lima/pkg/networks/usernet\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newUsernetCommand() *cobra.Command {\n\thostagentCommand := &cobra.Command{\n\t\tUse:    \"usernet\",\n\t\tShort:  \"run usernet\",\n\t\tArgs:   cobra.ExactArgs(0),\n\t\tRunE:   usernetAction,\n\t\tHidden: true,\n\t}\n\thostagentCommand.Flags().StringP(\"pidfile\", \"p\", \"\", \"write pid to file\")\n\thostagentCommand.Flags().StringP(\"endpoint\", \"e\", \"\", \"exposes usernet api(s) on this endpoint\")\n\thostagentCommand.Flags().String(\"listen-qemu\", \"\", \"listen for qemu connections\")\n\thostagentCommand.Flags().String(\"listen\", \"\", \"listen on a Unix socket and receive Bess-compatible FDs as SCM_RIGHTS messages\")\n\thostagentCommand.Flags().String(\"subnet\", \"192.168.5.0/24\", \"sets subnet value for the usernet network\")\n\thostagentCommand.Flags().Int(\"mtu\", 1500, \"mtu\")\n\thostagentCommand.Flags().StringToString(\"leases\", nil, \"pass default static leases for startup. Eg: '192.168.104.1=52:55:55:b3:bc:d9,192.168.104.2=5a:94:ef:e4:0c:df' \")\n\treturn hostagentCommand\n}\n\nfunc usernetAction(cmd *cobra.Command, _ []string) error {\n\tpidfile, err := cmd.Flags().GetString(\"pidfile\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif pidfile != \"\" {\n\t\tif _, err := os.Stat(pidfile); !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"pidfile %q already exists\", pidfile)\n\t\t}\n\t\tif err := os.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())+\"\\n\"), 0o644); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer os.RemoveAll(pidfile)\n\t}\n\tendpoint, err := cmd.Flags().GetString(\"endpoint\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tqemuSocket, err := cmd.Flags().GetString(\"listen-qemu\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfdSocket, err := cmd.Flags().GetString(\"listen\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tsubnet, err := cmd.Flags().GetString(\"subnet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tleases, err := cmd.Flags().GetStringToString(\"leases\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmtu, err := cmd.Flags().GetInt(\"mtu\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tos.RemoveAll(endpoint)\n\tos.RemoveAll(qemuSocket)\n\tos.RemoveAll(fdSocket)\n\n\treturn usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{\n\t\tMTU:           mtu,\n\t\tEndpoint:      endpoint,\n\t\tQemuSocket:    qemuSocket,\n\t\tFdSocket:      fdSocket,\n\t\tSubnet:        subnet,\n\t\tDefaultLeases: leases,\n\t})\n}\n\n// Path: cmd/limactl/validate.go\npackage main\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/cmd/limactl/guessarg\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/spf13/cobra\"\n\n\t\"github.com/sirupsen/logrus\"\n)\n\nfunc newValidateCommand() *cobra.Command {\n\tvalidateCommand := &cobra.Command{\n\t\tUse:     \"validate FILE.yaml [FILE.yaml, ...]\",\n\t\tShort:   \"Validate YAML files\",\n\t\tArgs:    WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:    validateAction,\n\t\tGroupID: advancedCommand,\n\t}\n\treturn validateCommand\n}\n\nfunc validateAction(_ *cobra.Command, args []string) error {\n\tfor _, f := range args {\n\t\t_, err := store.LoadYAMLByFilePath(f)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to load YAML file %q: %w\", f, err)\n\t\t}\n\t\tif _, err := guessarg.InstNameFromYAMLPath(f); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tlogrus.Infof(\"%q: OK\", f)\n\t}\n\n\treturn nil\n}\n\n", "needle_token_start": 679, "needle_token_end": 816, "code_context_ntokens": 14101, "output": ["```go\nfunc initLogrus(stderr io.Writer) {\n\tlogrus.SetOutput(stderr)\n\t// JSON logs are parsed in pkg/hostagent/events.Watcher()\n\tlogrus.SetFormatter(new(logrus.JSONFormatter))\n\t// HostAgent logging is one level more verbose than the start command itself\n\tif logrus.GetLevel() == logrus.DebugLevel {\n\t\tlogrus.SetLevel(logrus.TraceLevel)\n\t} else {\n\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t}\n}\n```"]}
{"repo": "lima-vm/lima", "name": "newDiskResizeCommand", "language": "go", "path": "cmd/limactl/disk.go", "position_ratio": 0.15, "description": "\nFunction Description:\n1. **Purpose**: To adjust the storage capacity of an existing virtual disk used by a virtual machine.\n2. **Input**: The name of the disk to be resized and the new size specification.\n3. **Output**: There is no direct output returned by the function, but it modifies the disk size, potentially logging the process and any errors encountered.\n4. **Procedure**: The function first validates that exactly one disk name is provided and that a new size is specified. It then attempts to resize the specified disk to the new size, handling any errors that might occur during this process.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "\t\t}\n\t}\n\treturn matches\n}\n\nfunc diskListAction(cmd *cobra.Command, args []string) error {\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tallDisks, err := store.Disks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdisks := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := diskMatches(arg, allDisks)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tdisks = append(disks, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No disk matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tdisks = allDisks\n\t}\n\n\tif jsonFormat {\n\t\tfor _, diskName := range disks {\n\t\t\tdisk, err := store.InspectDisk(diskName)\n\t\t\tif err != nil {\n\t\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tj, err := json.Marshal(disk)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), string(j))\n\t\t}\n\t\treturn nil\n\t}\n\n\tw := tabwriter.NewWriter(cmd.OutOrStdout(), 4, 8, 4, ' ', 0)\n\tfmt.Fprintln(w, \"NAME\\tSIZE\\tFORMAT\\tDIR\\tIN-USE-BY\")\n\n\tif len(disks) == 0 {\n\t\tlogrus.Warn(\"No disk found. Run `limactl disk create DISK --size SIZE` to create a disk.\")\n\t}\n\n\tfor _, diskName := range disks {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Fprintf(w, \"%s\\t%s\\t%s\\t%s\\t%s\\n\", disk.Name, units.BytesSize(float64(disk.Size)), disk.Format, disk.Dir, disk.Instance)\n\t}\n\n\treturn w.Flush()\n}\n\nfunc newDiskDeleteCommand() *cobra.Command {\n\tdiskDeleteCommand := &cobra.Command{\n\t\tUse: \"delete DISK [DISK, ...]\",\n\t\tExample: `\nTo delete a disk:\n$ limactl disk delete DISK\n\nTo delete multiple disks:\n$ limactl disk delete DISK1 DISK2 ...\n`,\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskDeleteAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskDeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"force delete\")\n\treturn diskDeleteCommand\n}\n\nfunc diskDeleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tinstNames, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar instances []*store.Instance\n\tfor _, instName := range instNames {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tinstances = append(instances, inst)\n\t}\n\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tif !force {\n\t\t\tif disk.Instance != \"\" {\n\t\t\t\treturn fmt.Errorf(\"cannot delete disk %q in use by instance %q\", disk.Name, disk.Instance)\n\t\t\t}\n\t\t\tvar refInstances []string\n\t\t\tfor _, inst := range instances {\n\t\t\t\tif len(inst.AdditionalDisks) > 0 {\n\t\t\t\t\tfor _, d := range inst.AdditionalDisks {\n\t\t\t\t\t\tif d.Name == diskName {\n\t\t\t\t\t\t\trefInstances = append(refInstances, inst.Name)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(refInstances) > 0 {\n\t\t\t\tlogrus.Warnf(\"Skipping deleting disk %q, disk is referenced by one or more non-running instances: %q\",\n\t\t\t\t\tdiskName, refInstances)\n\t\t\t\tlogrus.Warnf(\"To delete anyway, run %q\", forceDeleteCommand(diskName))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif err := deleteDisk(disk); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\nfunc deleteDisk(disk *store.Disk) error {\n\tif err := os.RemoveAll(disk.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", disk.Dir, err)\n\t}\n\treturn nil\n}\n\nfunc forceDeleteCommand(diskName string) string {\n\treturn fmt.Sprintf(\"limactl disk delete --force %v\", diskName)\n}\n\nfunc newDiskUnlockCommand() *cobra.Command {\n\tdiskUnlockCommand := &cobra.Command{\n\t\tUse: \"unlock DISK [DISK, ...]\",\n\t\tExample: `\nEmergency recovery! If an instance is force stopped, it may leave a disk locked while not actually using it.\n\nTo unlock a disk:\n$ limactl disk unlock DISK\n\nTo unlock multiple disks:\n$ limactl disk unlock DISK1 DISK2 ...\n`,\n\t\tShort:             \"Unlock one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskUnlockAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\treturn diskUnlockCommand\n}\n\nfunc diskUnlockAction(_ *cobra.Command, args []string) error {\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif disk.Instance == \"\" {\n\t\t\tlogrus.Warnf(\"Ignoring unlocked disk %q\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\t// if store.Inspect throws an error, the instance does not exist, and it is safe to unlock\n\t\tinst, err := store.Inspect(disk.Instance)\n\t\tif err == nil {\n\t\t\tif len(inst.Errors) > 0 {\n\t\t\t\tlogrus.Warnf(\"Cannot unlock disk %q, attached instance %q has errors: %+v\",\n\t\t\t\t\tdiskName, disk.Instance, inst.Errors)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif inst.Status == store.StatusRunning {\n\t\t\t\tlogrus.Warnf(\"Cannot unlock disk %q used by running instance %q\", diskName, disk.Instance)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to unlock disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Unlocked disk %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\n\nfunc newDiskResizeCommand() *cobra.Command {\n\tdiskResizeCommand := &cobra.Command{\n\t\tUse: \"resize DISK\",\n\t\tExample: `\nResize a disk:\n$ limactl disk resize DISK --size SIZE`,\n\t\tShort:             \"Resize existing Lima disk\",\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:              diskResizeAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskResizeCommand.Flags().String(\"size\", \"\", \"Disk size\")\n\t_ = diskResizeCommand.MarkFlagRequired(\"size\")\n\treturn diskResizeCommand\n}\n\nfunc diskResizeAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskName := args[0]\n\tdisk, err := store.InspectDisk(diskName)\n\tif err != nil {\n\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"disk %q does not exists\", diskName)\n\t\t}\n\t\treturn err\n\t}\n\n\t// Shrinking can cause a disk failure\n\tif diskSize < disk.Size {\n\t\treturn fmt.Errorf(\"specified size %q is less than the current disk size %q. Disk shrinking is currently unavailable\", units.BytesSize(float64(diskSize)), units.BytesSize(float64(disk.Size)))\n\t}\n\n\tif disk.Instance != \"\" {\n\t\tinst, err := store.Inspect(disk.Instance)\n\t\tif err == nil {\n\t\t\tif inst.Status == store.StatusRunning {\n\t\t\t\treturn fmt.Errorf(\"cannot resize disk %q used by running instance %q. Please stop the VM instance\", diskName, disk.Instance)\n\t\t\t}\n\t\t}\n\t}\n\tif err := qemu.ResizeDataDisk(disk.Dir, disk.Format, int(diskSize)); err != nil {\n\t\treturn fmt.Errorf(\"failed to resize disk %q: %w\", diskName, err)\n\t}\n\tlogrus.Infof(\"Resized disk %q (%q)\", diskName, disk.Dir)\n\treturn nil\n}\n\nfunc diskBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteDiskNames(cmd)\n}\n\n// Path: cmd/limactl/stop.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\thostagentevents \"github.com/lima-vm/lima/pkg/hostagent/events\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newStopCommand() *cobra.Command {\n\tstopCmd := &cobra.Command{\n\t\tUse:               \"stop INSTANCE\",\n\t\tShort:             \"Stop an instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              stopAction,\n\t\tValidArgsFunction: stopBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tstopCmd.Flags().BoolP(\"force\", \"f\", false, \"force stop the instance\")\n\treturn stopCmd\n}\n\nfunc stopAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif force {\n\t\tstopInstanceForcibly(inst)\n\t} else {\n\t\terr = stopInstanceGracefully(inst)\n\t}\n\t// TODO: should we also reconcile networks if graceful stop returned an error?\n\tif err == nil {\n\t\terr = networks.Reconcile(cmd.Context(), \"\")\n\t}\n\treturn err\n}\n\nfunc stopInstanceGracefully(inst *store.Instance) error {\n\tif inst.Status != store.StatusRunning {\n\t\treturn fmt.Errorf(\"expected status %q, got %q (maybe use `limactl stop -f`?)\", store.StatusRunning, inst.Status)\n\t}\n\n\tbegin := time.Now() // used for logrus propagation\n\tlogrus.Infof(\"Sending SIGINT to hostagent process %d\", inst.HostAgentPID)\n\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigInt); err != nil {\n\t\tlogrus.Error(err)\n\t}\n\n\tlogrus.Info(\"Waiting for the host agent and the driver processes to shut down\")\n\treturn waitForHostAgentTermination(context.TODO(), inst, begin)\n}\n\nfunc waitForHostAgentTermination(ctx context.Context, inst *store.Instance, begin time.Time) error {\n\tctx2, cancel := context.WithTimeout(ctx, 3*time.Minute)\n\tdefer cancel()\n\n\tvar receivedExitingEvent bool\n\tonEvent := func(ev hostagentevents.Event) bool {\n\t\tif len(ev.Status.Errors) > 0 {\n\t\t\tlogrus.Errorf(\"%+v\", ev.Status.Errors)\n\t\t}\n\t\tif ev.Status.Exiting {\n\t\t\treceivedExitingEvent = true\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\n\thaStdoutPath := filepath.Join(inst.Dir, filenames.HostAgentStdoutLog)\n\thaStderrPath := filepath.Join(inst.Dir, filenames.HostAgentStderrLog)\n\n\tif err := hostagentevents.Watch(ctx2, haStdoutPath, haStderrPath, begin, onEvent); err != nil {\n\t\treturn err\n\t}\n\n\tif !receivedExitingEvent {\n\t\treturn errors.New(\"did not receive an event with the \\\"exiting\\\" status\")\n\t}\n\n\treturn nil\n}\n\nfunc stopInstanceForcibly(inst *store.Instance) {\n\tif inst.DriverPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the %s driver process %d\", inst.VMType, inst.DriverPID)\n\t\tif err := osutil.SysKill(inst.DriverPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Infof(\"The %s driver process seems already stopped\", inst.VMType)\n\t}\n\n\tfor _, d := range inst.AdditionalDisks {\n\t\tdiskName := d.Name\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.Warnf(\"Disk %q does not exist\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\tlogrus.Warnf(\"Failed to unlock disk %q. To use, run `limactl disk unlock %v`\", diskName, diskName)\n\t\t}\n\t}\n\n\tif inst.HostAgentPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the host agent process %d\", inst.HostAgentPID)\n\t\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"The host agent process seems already stopped\")\n\t}\n\n\tsuffixesToBeRemoved := []string{\".pid\", \".sock\", \".tmp\"}\n\tlogrus.Infof(\"Removing %s under %q\", inst.Dir, strings.ReplaceAll(strings.Join(suffixesToBeRemoved, \" \"), \".\", \"*.\"))\n\tfi, err := os.ReadDir(inst.Dir)\n\tif err != nil {\n\t\tlogrus.Error(err)\n\t\treturn\n\t}\n\tfor _, f := range fi {\n\t\tpath := filepath.Join(inst.Dir, f.Name())\n\t\tfor _, suffix := range suffixesToBeRemoved {\n\t\t\tif strings.HasSuffix(path, suffix) {\n\t\t\t\tlogrus.Infof(\"Removing %q\", path)\n\t\t\t\tif err := os.Remove(path); err != nil {\n\t\t\t\t\tlogrus.Error(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc stopBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/delete.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/stop\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDeleteCommand() *cobra.Command {\n\tdeleteCommand := &cobra.Command{\n\t\tUse:               \"delete INSTANCE [INSTANCE, ...]\",\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete an instance of Lima.\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              deleteAction,\n\t\tValidArgsFunction: deleteBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\tdeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"forcibly kill the processes\")\n\treturn deleteCommand\n}\n\nfunc deleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent instance %q\", instName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif err := deleteInstance(cmd.Context(), inst, force); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete instance %q: %w\", instName, err)\n\t\t}\n\t\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\t\tif err != nil && !errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.WithError(err).Warnf(\"The autostart file for instance %q does not exist\", instName)\n\t\t\t} else if deleted {\n\t\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t\t}\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", instName, inst.Dir)\n\t}\n\treturn networks.Reconcile(cmd.Context(), \"\")\n}\n\nfunc deleteInstance(ctx context.Context, inst *store.Instance, force bool) error {\n\tif inst.Protected {\n\t\treturn fmt.Errorf(\"instance is protected to prohibit accidental removal (Hint: use `limactl unprotect`)\")\n\t}\n\tif !force && inst.Status != store.StatusStopped {\n\t\treturn fmt.Errorf(\"expected status %q, got %q\", store.StatusStopped, inst.Status)\n\t}\n\n\tstopInstanceForcibly(inst)\n\n\tif err := stop.Unregister(ctx, inst); err != nil {\n\t\treturn fmt.Errorf(\"failed to unregister %q: %w\", inst.Dir, err)\n\t}\n\n\tif err := os.RemoveAll(inst.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", inst.Dir, err)\n\t}\n\n\treturn nil\n}\n\nfunc deleteBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/editflags/editflags.go\npackage editflags\n\nimport (\n\t\"fmt\"\n\t\"math/bits\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pbnjay/memory\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n\tflag \"github.com/spf13/pflag\"\n)\n\n// RegisterEdit registers flags related to in-place YAML modification, for `limactl edit`.\nfunc RegisterEdit(cmd *cobra.Command) {\n\tregisterEdit(cmd, \"\")\n}\n\nfunc registerEdit(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\n\tflags.Int(\"cpus\", 0, commentPrefix+\"number of CPUs\") // Similar to colima's --cpu, but the flag name is slightly different (cpu vs cpus)\n\t_ = cmd.RegisterFlagCompletionFunc(\"cpus\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeCPUs(runtime.NumCPU()) {\n\t\t\tres = append(res, strconv.Itoa(f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.IPSlice(\"dns\", nil, commentPrefix+\"specify custom DNS (disable host resolver)\") // colima-compatible\n\n\tflags.Float32(\"memory\", 0, commentPrefix+\"memory in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeMemoryGiB(memory.TotalMemory()) {\n\t\t\tres = append(res, fmt.Sprintf(\"%.1f\", f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.StringSlice(\"mount\", nil, commentPrefix+\"directories to mount, suffix ':w' for writable (Do not specify directories that overlap with the existing mounts)\") // colima-compatible\n\n\tflags.String(\"mount-type\", \"\", commentPrefix+\"mount type (reverse-sshfs, 9p, virtiofs)\") // Similar to colima's --mount-type=(sshfs|9p|virtiofs), but \"reverse-sshfs\" is Lima is called \"sshfs\" in colima\n\t_ = cmd.RegisterFlagCompletionFunc(\"mount-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"reverse-sshfs\", \"9p\", \"virtiofs\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"mount-writable\", false, commentPrefix+\"make all mounts writable\")\n\tflags.Bool(\"mount-inotify\", false, commentPrefix+\"enable inotify for mounts\")\n\n\tflags.StringSlice(\"network\", nil, commentPrefix+\"additional networks, e.g., \\\"vzNAT\\\" or \\\"lima:shared\\\" to assign vmnet IP\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"network\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\t// TODO: retrieve the lima:* network list from networks.yaml\n\t\treturn []string{\"lima:shared\", \"lima:bridged\", \"lima:host\", \"lima:user-v2\", \"vzNAT\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"rosetta\", false, commentPrefix+\"enable Rosetta (for vz instances)\")\n\n\tflags.String(\"set\", \"\", commentPrefix+\"modify the template inplace, using yq syntax\")\n\n\t// negative performance impact: https://gitlab.com/qemu-project/qemu/-/issues/334\n\tflags.Bool(\"video\", false, commentPrefix+\"enable video output (has negative performance impact for QEMU)\")\n}\n\n// RegisterCreate registers flags related to in-place YAML modification, for `limactl create`.\nfunc RegisterCreate(cmd *cobra.Command, commentPrefix string) {\n\tregisterEdit(cmd, commentPrefix)\n\tflags := cmd.Flags()\n\n\tflags.String(\"arch\", \"\", commentPrefix+\"machine architecture (x86_64, aarch64, riscv64)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"arch\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"x86_64\", \"aarch64\", \"riscv64\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"containerd\", \"\", commentPrefix+\"containerd mode (user, system, user+system, none)\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"user\", \"system\", \"user+system\", \"none\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Float32(\"disk\", 0, commentPrefix+\"disk size in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"10\", \"30\", \"50\", \"100\", \"200\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"vm-type\", \"\", commentPrefix+\"virtual machine type (qemu, vz)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"qemu\", \"vz\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"plain\", false, commentPrefix+\"plain mode. Disable mounts, port forwarding, containerd, etc.\")\n}\n\nfunc defaultExprFunc(expr string) func(v *flag.Flag) (string, error) {\n\treturn func(v *flag.Flag) (string, error) {\n\t\treturn fmt.Sprintf(expr, v.Value), nil\n\t}\n}\n\n// YQExpressions returns YQ expressions.\nfunc YQExpressions(flags *flag.FlagSet, newInstance bool) ([]string, error) {\n\ttype def struct {\n\t\tflagName                 string\n\t\texprFunc                 func(*flag.Flag) (string, error)\n\t\tonlyValidForNewInstances bool\n\t\texperimental             bool\n\t}\n\td := defaultExprFunc\n\tdefs := []def{\n\t\t{\"cpus\", d(\".cpus = %s\"), false, false},\n\t\t{\n\t\t\t\"dns\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tipSlice, err := flags.GetIPSlice(\"dns\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.dns += [`\n\t\t\t\tfor i, ip := range ipSlice {\n\t\t\t\t\texpr += fmt.Sprintf(\"%q\", ip)\n\t\t\t\t\tif i < len(ipSlice)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .dns |= unique | .hostResolver.enabled=false`\n\t\t\t\tlogrus.Warnf(\"Disabling HostResolver, as custom DNS addresses (%v) are specified\", ipSlice)\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"memory\", d(\".memory = \\\"%sGiB\\\"\"), false, false},\n\t\t{\n\t\t\t\"mount\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"mount\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.mounts += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\twritable := strings.HasSuffix(s, \":w\")\n\t\t\t\t\tloc := strings.TrimSuffix(s, \":w\")\n\t\t\t\t\texpr += fmt.Sprintf(`{\"location\": %q, \"writable\": %v}`, loc, writable)\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .mounts |= unique_by(.location)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"mount-type\", d(\".mountType = %q\"), false, false},\n\t\t{\"mount-inotify\", d(\".mountInotify = %s\"), false, true},\n\t\t{\"mount-writable\", d(\".mounts[].writable = %s\"), false, false},\n\t\t{\n\t\t\t\"network\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"network\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.networks += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\t// CLI syntax is still experimental (YAML syntax is out of experimental)\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase s == \"vzNAT\":\n\t\t\t\t\t\texpr += `{\"vzNAT\": true}`\n\t\t\t\t\tcase strings.HasPrefix(s, \"lima:\"):\n\t\t\t\t\t\tnetwork := strings.TrimPrefix(s, \"lima:\")\n\t\t\t\t\t\texpr += fmt.Sprintf(`{\"lima\": %q}`, network)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\treturn \"\", fmt.Errorf(\"network name must be \\\"vzNAT\\\" or \\\"lima:*\\\", got %q\", s)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .networks |= unique_by(.lima)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t\"rosetta\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tb, err := flags.GetBool(\"rosetta\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\treturn fmt.Sprintf(\".rosetta.enabled = %v | .rosetta.binfmt = %v\", b, b), nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\ttrue,\n\t\t},\n\t\t{\"set\", d(\"%s\"), false, false},\n\t\t{\n\t\t\t\"video\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tb, err := flags.GetBool(\"video\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tif b {\n\t\t\t\t\treturn \".video.display = \\\"default\\\"\", nil\n\t\t\t\t}\n\t\t\t\treturn \".video.display = \\\"none\\\"\", nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"arch\", d(\".arch = %q\"), true, false},\n\t\t{\n\t\t\t\"containerd\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\ts, err := flags.GetString(\"containerd\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tswitch s {\n\t\t\t\tcase \"user\":\n\t\t\t\t\treturn `.containerd.user = true | .containerd.system = false`, nil\n\t\t\t\tcase \"system\":\n\t\t\t\t\treturn `.containerd.user = false | .containerd.system = true`, nil\n\t\t\t\tcase \"user+system\", \"system+user\":\n\t\t\t\t\treturn `.containerd.user = true | .containerd.system = true`, nil\n\t\t\t\tcase \"none\":\n\t\t\t\t\treturn `.containerd.user = false | .containerd.system = false`, nil\n\t\t\t\tdefault:\n\t\t\t\t\treturn \"\", fmt.Errorf(`expected one of [\"user\", \"system\", \"user+system\", \"none\"], got %q`, s)\n\t\t\t\t}\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t},\n\t\t{\"disk\", d(\".disk= \\\"%sGiB\\\"\"), true, false},\n\t\t{\"vm-type\", d(\".vmType = %q\"), true, false},\n\t\t{\"plain\", d(\".plain = %s\"), true, false},\n\t}\n\tvar exprs []string\n\tfor _, def := range defs {\n\t\tv := flags.Lookup(def.flagName)\n\t\tif v != nil && v.Changed {\n\t\t\tif def.experimental {\n\t\t\t\tlogrus.Warnf(\"`--%s` is experimental\", def.flagName)\n\t\t\t}\n\t\t\tif def.onlyValidForNewInstances && !newInstance {\n\t\t\t\tlogrus.Warnf(\"`--%s` is not applicable to an existing instance (Hint: create a new instance with `limactl create --%s=%s --name=NAME`)\",\n\t\t\t\t\tdef.flagName, def.flagName, v.Value.String())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\texpr, err := def.exprFunc(v)\n\t\t\tif err != nil {\n\t\t\t\treturn exprs, fmt.Errorf(\"error while processing flag %q: %w\", def.flagName, err)\n\t\t\t}\n\t\t\texprs = append(exprs, expr)\n\t\t}\n\t}\n\treturn exprs, nil\n}\n\nfunc isPowerOfTwo(x int) bool {\n\treturn bits.OnesCount(uint(x)) == 1\n}\n\nfunc completeCPUs(hostCPUs int) []int {\n\tvar res []int\n\tfor i := 1; i <= hostCPUs; i *= 2 {\n\t\tres = append(res, i)\n\t}\n\tif !isPowerOfTwo(hostCPUs) {\n\t\tres = append(res, hostCPUs)\n\t}\n\treturn res\n}\n\nfunc completeMemoryGiB(hostMemory uint64) []float32 {\n\thostMemoryHalfGiB := int(hostMemory / 2 / 1024 / 1024 / 1024)\n\tvar res []float32\n\tif hostMemoryHalfGiB < 1 {\n\t\tres = append(res, 0.5)\n\t}\n\tfor i := 1; i <= hostMemoryHalfGiB; i *= 2 {\n\t\tres = append(res, float32(i))\n\t}\n\tif hostMemoryHalfGiB > 1 && !isPowerOfTwo(hostMemoryHalfGiB) {\n\t\tres = append(res, float32(hostMemoryHalfGiB))\n\t}\n\treturn res\n}\n\n// Path: cmd/limactl/guessarg/guessarg.go\npackage guessarg\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/containerd/containerd/identifiers\"\n)\n\nfunc SeemsTemplateURL(arg string) (bool, *url.URL) {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false, u\n\t}\n\treturn u.Scheme == \"template\", u\n}\n\nfunc SeemsHTTPURL(arg string) bool {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif u.Scheme != \"http\" && u.Scheme != \"https\" {\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc SeemsFileURL(arg string) bool {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false\n\t}\n\treturn u.Scheme == \"file\"\n}\n\nfunc SeemsYAMLPath(arg string) bool {\n\tif strings.Contains(arg, \"/\") {\n\t\treturn true\n\t}\n\tlower := strings.ToLower(arg)\n\treturn strings.HasSuffix(lower, \".yml\") || strings.HasSuffix(lower, \".yaml\")\n}\n\nfunc InstNameFromURL(urlStr string) (string, error) {\n\tu, err := url.Parse(urlStr)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn InstNameFromYAMLPath(path.Base(u.Path))\n}\n\nfunc InstNameFromYAMLPath(yamlPath string) (string, error) {\n\ts := strings.ToLower(filepath.Base(yamlPath))\n\ts = strings.TrimSuffix(strings.TrimSuffix(s, \".yml\"), \".yaml\")\n\ts = strings.ReplaceAll(s, \".\", \"-\")\n\tif err := identifiers.Validate(s); err != nil {\n\t\treturn \"\", fmt.Errorf(\"filename %q is invalid: %w\", yamlPath, err)\n\t}\n\treturn s, nil\n}\n\n// Path: cmd/limactl/start.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/containerd/containerd/identifiers\"\n\t\"github.com/lima-vm/lima/cmd/limactl/editflags\"\n\t\"github.com/lima-vm/lima/cmd/limactl/guessarg\"\n\t\"github.com/lima-vm/lima/pkg/editutil\"\n\t\"github.com/lima-vm/lima/pkg/ioutilx\"\n\t\"github.com/lima-vm/lima/pkg/limayaml\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/start\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/lima-vm/lima/pkg/templatestore\"\n\t\"github.com/lima-vm/lima/pkg/uiutil\"\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/lima-vm/lima/pkg/yqutil\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc registerCreateFlags(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\tflags.String(\"name\", \"\", commentPrefix+\"override the instance name\")\n\tflags.Bool(\"list-templates\", false, commentPrefix+\"list available templates and exit\")\n\teditflags.RegisterCreate(cmd, commentPrefix)\n}\n\nfunc newCreateCommand() *cobra.Command {\n\tcreateCommand := &cobra.Command{\n\t\tUse: \"create FILE.yaml|URL\",\n\t\tExample: `\nTo create an instance \"default\" from the default Ubuntu template:\n$ limactl create\n\nTo create an instance \"default\" from a template \"docker\":\n$ limactl create --name=default template://docker\n\nTo create an instance \"default\" with modified parameters:\n$ limactl create --cpus=2 --memory=2\n\nTo create an instance \"default\" with yq expressions:\n$ limactl create --set='.cpus = 2 | .memory = \"2GiB\"'\n\nTo see the template list:\n$ limactl create --list-templates\n\nTo create an instance \"default\" from a local file:\n$ limactl create --name=default /usr/local/share/lima/templates/fedora.yaml\n\nTo create an instance \"default\" from a remote URL (use carefully, with a trustable source):\n$ limactl create --name=default https://raw.githubusercontent.com/lima-vm/lima/master/examples/alpine.yaml\n\nTo create an instance \"local\" from a template passed to stdin (--name parameter is required):\n$ cat template.yaml | limactl create --name=local -\n`,\n\t\tShort:             \"Create an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tValidArgsFunction: createBashComplete,\n\t\tRunE:              createAction,\n\t\tGroupID:           basicCommand,\n\t}\n\tregisterCreateFlags(createCommand, \"\")\n\treturn createCommand\n}\n\nfunc newStartCommand() *cobra.Command {\n\tstartCommand := &cobra.Command{\n\t\tUse: \"start NAME|FILE.yaml|URL\",\n\t\tExample: `\nTo create an instance \"default\" (if not created yet) from the default Ubuntu template, and start it:\n$ limactl start\n\nTo create an instance \"default\" from a template \"docker\", and start it:\n$ limactl start --name=default template://docker\n\n'limactl start' also accepts the 'limactl create' flags such as '--set'.\nSee the examples in 'limactl create --help'.\n`,\n\t\tShort:             \"Start an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tValidArgsFunction: startBashComplete,\n\t\tRunE:              startAction,\n\t\tGroupID:           basicCommand,\n\t}\n\tregisterCreateFlags(startCommand, \"[limactl create] \")\n\tif runtime.GOOS != \"windows\" {\n\t\tstartCommand.Flags().Bool(\"foreground\", false, \"run the hostagent in the foreground\")\n\t}\n\tstartCommand.Flags().Duration(\"timeout\", start.DefaultWatchHostAgentEventsTimeout, \"duration to wait for the instance to be running before timing out\")\n\treturn startCommand\n}\n\nfunc loadOrCreateInstance(cmd *cobra.Command, args []string, createOnly bool) (*store.Instance, error) {\n\tvar arg string // can be empty\n\tif len(args) > 0 {\n\t\targ = args[0]\n\t}\n\n\tvar (\n\t\tst  = &creatorState{}\n\t\terr error\n\t)\n\n\tflags := cmd.Flags()\n\n\t// Create an instance, with menu TUI when TTY is available\n\ttty, err := flags.GetBool(\"tty\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tst.instName, err = flags.GetString(\"name\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconst yBytesLimit = 4 * 1024 * 1024 // 4MiB\n\n\tif ok, u := guessarg.SeemsTemplateURL(arg); ok {\n\t\t// No need to use SecureJoin here. https://github.com/lima-vm/lima/pull/805#discussion_r853411702\n\t\ttemplateName := filepath.Join(u.Host, u.Path)\n\t\tlogrus.Debugf(\"interpreting argument %q as a template name %q\", arg, templateName)\n\t\tif st.instName == \"\" {\n\t\t\t// e.g., templateName = \"deprecated/centos-7\" , st.instName = \"centos-7\"\n\t\t\tst.instName = filepath.Base(templateName)\n\t\t}\n\t\tst.yBytes, err = templatestore.Read(templateName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsHTTPURL(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromURL(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a http url for instance %q\", arg, st.instName)\n\t\treq, err := http.NewRequestWithContext(cmd.Context(), http.MethodGet, arg, http.NoBody)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tresp, err := http.DefaultClient.Do(req)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer resp.Body.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(resp.Body, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsFileURL(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromURL(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a file url for instance %q\", arg, st.instName)\n\t\tr, err := os.Open(strings.TrimPrefix(arg, \"file://\"))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer r.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(r, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsYAMLPath(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromYAMLPath(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a file path for instance %q\", arg, st.instName)\n\t\tr, err := os.Open(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer r.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(r, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if arg == \"-\" {\n\t\tif st.instName == \"\" {\n\t\t\treturn nil, errors.New(\"must pass instance name with --name when reading template from stdin\")\n\t\t}\n\t\tst.yBytes, err = io.ReadAll(os.Stdin)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unexpected error reading stdin: %w\", err)\n\t\t}\n\t\t// see if the tty was set explicitly or not\n\t\tttySet := cmd.Flags().Changed(\"tty\")\n\t\tif ttySet && tty {\n\t\t\treturn nil, errors.New(\"cannot use --tty=true and read template from stdin together\")\n\t\t}\n\t\ttty = false\n\t} else {\n\t\tif arg == \"\" {\n\t\t\tif st.instName == \"\" {\n\t\t\t\tst.instName = DefaultInstanceName\n\t\t\t}\n\t\t} else {\n\t\t\tlogrus.Debugf(\"interpreting argument %q as an instance name\", arg)\n\t\t\tif st.instName != \"\" && st.instName != arg {\n\t\t\t\treturn nil, fmt.Errorf(\"instance name %q and CLI flag --name=%q cannot be specified together\", arg, st.instName)\n\t\t\t}\n\t\t\tst.instName = arg\n\t\t}\n\t\tif err := identifiers.Validate(st.instName); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"argument must be either an instance name, a YAML file path, or a URL, got %q: %w\", st.instName, err)\n\t\t}\n\t\tinst, err := store.Inspect(st.instName)\n\t\tif err == nil {\n\t\t\tif createOnly {\n\t\t\t\treturn nil, fmt.Errorf(\"Instance %q already exists\", st.instName)\n\t\t\t}\n\t\t\tlogrus.Infof(\"Using the existing instance %q\", st.instName)\n\t\t\tyqExprs, err := editflags.YQExpressions(flags, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif len(yqExprs) > 0 {\n\t\t\t\tyq := yqutil.Join(yqExprs)\n\t\t\t\tinst, err = applyYQExpressionToExistingInstance(inst, yq)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"failed to apply yq expression %q to instance %q: %w\", yq, st.instName, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn inst, nil\n\t\t}\n\t\tif !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn nil, err\n\t\t}\n\t\tif arg != \"\" && arg != DefaultInstanceName {\n\t\t\tlogrus.Infof(\"Creating an instance %q from template://default (Not from template://%s)\", st.instName, st.instName)\n\t\t\tlogrus.Warnf(\"This form is deprecated. Use `limactl create --name=%s template://default` instead\", st.instName)\n\t\t}\n\t\t// Read the default template for creating a new instance\n\t\tst.yBytes, err = templatestore.Read(templatestore.Default)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tyqExprs, err := editflags.YQExpressions(flags, true)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tyq := yqutil.Join(yqExprs)\n\tif tty {\n\t\tvar err error\n\t\tst, err = chooseNextCreatorState(st, yq)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"Terminal is not available, proceeding without opening an editor\")\n\t\tif err := modifyInPlace(st, yq); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tsaveBrokenEditorBuffer := tty\n\treturn createInstance(cmd.Context(), st, saveBrokenEditorBuffer)\n}\n\nfunc applyYQExpressionToExistingInstance(inst *store.Instance, yq string) (*store.Instance, error) {\n\tif strings.TrimSpace(yq) == \"\" {\n\t\treturn inst, nil\n\t}\n\tfilePath := filepath.Join(inst.Dir, filenames.LimaYAML)\n\tyContent, err := os.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlogrus.Debugf(\"Applying yq expression %q to an existing instance %q\", yq, inst.Name)\n\tyBytes, err := yqutil.EvaluateExpression(yq, yContent)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filePath, yBytes, 0o644); err != nil {\n\t\treturn nil, err\n\t}\n\t// Reload\n\treturn store.Inspect(inst.Name)\n}\n\nfunc createInstance(ctx context.Context, st *creatorState, saveBrokenEditorBuffer bool) (*store.Instance, error) {\n\tif st.instName == \"\" {\n\t\treturn nil, errors.New(\"got empty st.instName\")\n\t}\n\tif len(st.yBytes) == 0 {\n\t\treturn nil, errors.New(\"got empty st.yBytes\")\n\t}\n\n\tinstDir, err := store.InstanceDir(st.instName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// the full path of the socket name must be less than UNIX_PATH_MAX chars.\n\tmaxSockName := filepath.Join(instDir, filenames.LongestSock)\n\tif len(maxSockName) >= osutil.UnixPathMax {\n\t\treturn nil, fmt.Errorf(\"instance name %q too long: %q must be less than UNIX_PATH_MAX=%d characters, but is %d\",\n\t\t\tst.instName, maxSockName, osutil.UnixPathMax, len(maxSockName))\n\t}\n\tif _, err := os.Stat(instDir); !errors.Is(err, os.ErrNotExist) {\n\t\treturn nil, fmt.Errorf(\"instance %q already exists (%q)\", st.instName, instDir)\n\t}\n\t// limayaml.Load() needs to pass the store file path to limayaml.FillDefault() to calculate default MAC addresses\n\tfilePath := filepath.Join(instDir, filenames.LimaYAML)\n\ty, err := limayaml.Load(st.yBytes, filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := limayaml.Validate(*y, true); err != nil {\n\t\tif !saveBrokenEditorBuffer {\n\t\t\treturn nil, err\n\t\t}\n\t\trejectedYAML := \"lima.REJECTED.yaml\"\n\t\tif writeErr := os.WriteFile(rejectedYAML, st.yBytes, 0o644); writeErr != nil {\n\t\t\treturn nil, fmt.Errorf(\"the YAML is invalid, attempted to save the buffer as %q but failed: %w: %w\", rejectedYAML, writeErr, err)\n\t\t}\n\t\treturn nil, fmt.Errorf(\"the YAML is invalid, saved the buffer as %q: %w\", rejectedYAML, err)\n\t}\n\tif err := os.MkdirAll(instDir, 0o700); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filePath, st.yBytes, 0o644); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filepath.Join(instDir, filenames.LimaVersion), []byte(version.Version), 0o444); err != nil {\n\t\treturn nil, err\n\t}\n\n\tinst, err := store.Inspect(st.instName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := start.Register(ctx, inst); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn inst, nil\n}\n\ntype creatorState struct {\n\tinstName string // instance name\n\tyBytes   []byte // yaml bytes\n}\n\nfunc modifyInPlace(st *creatorState, yq string) error {\n\tout, err := yqutil.EvaluateExpression(yq, st.yBytes)\n\tif err != nil {\n\t\treturn err\n\t}\n\tst.yBytes = out\n\treturn nil\n}\n\nfunc chooseNextCreatorState(st *creatorState, yq string) (*creatorState, error) {\n\tfor {\n\t\tif err := modifyInPlace(st, yq); err != nil {\n\t\t\tlogrus.WithError(err).Warn(\"Failed to evaluate yq expression\")\n\t\t\treturn st, err\n\t\t}\n\t\tmessage := fmt.Sprintf(\"Creating an instance %q\", st.instName)\n\t\toptions := []string{\n\t\t\t\"Proceed with the current configuration\",\n\t\t\t\"Open an editor to review or modify the current configuration\",\n\t\t\t\"Choose another template (docker, podman, archlinux, fedora, ...)\",\n\t\t\t\"Exit\",\n\t\t}\n\t\tans, err := uiutil.Select(message, options)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, uiutil.InterruptErr) {\n\t\t\t\tlogrus.Fatal(\"Interrupted by user\")\n\t\t\t}\n\t\t\tlogrus.WithError(err).Warn(\"Failed to open TUI\")\n\t\t\treturn st, nil\n\t\t}\n\t\tswitch ans {\n\t\tcase 0: // \"Proceed with the current configuration\"\n\t\t\treturn st, nil\n\t\tcase 1: // \"Open an editor ...\"\n\t\t\thdr := fmt.Sprintf(\"# Review and modify the following configuration for Lima instance %q.\\n\", st.instName)\n\t\t\tif st.instName == DefaultInstanceName {\n\t\t\t\thdr += \"# - In most cases, you do not need to modify this file.\\n\"\n\t\t\t}\n\t\t\thdr += \"# - To cancel starting Lima, just save this file as an empty file.\\n\"\n\t\t\thdr += \"\\n\"\n\t\t\thdr += editutil.GenerateEditorWarningHeader()\n\t\t\tvar err error\n\t\t\tst.yBytes, err = editutil.OpenEditor(st.yBytes, hdr)\n\t\t\tif err != nil {\n\t\t\t\treturn st, err\n\t\t\t}\n\t\t\tif len(st.yBytes) == 0 {\n\t\t\t\tlogrus.Info(\"Aborting, as requested by saving the file with empty content\")\n\t\t\t\tos.Exit(0)\n\t\t\t\treturn st, errors.New(\"should not reach here\")\n\t\t\t}\n\t\t\treturn st, nil\n\t\tcase 2: // \"Choose another template...\"\n\t\t\ttemplates, err := templatestore.Templates()\n\t\t\tif err != nil {\n...\n", "needle_token_start": 2180, "needle_token_end": 2349, "code_context_ntokens": 15219, "output": ["```go\nfunc newDiskResizeCommand() *cobra.Command {\n\tdiskResizeCommand := &cobra.Command{\n\t\tUse: \"resize DISK\",\n\t\tExample: `\nResize a disk:\n$ limactl disk resize DISK --size SIZE`,\n\t\tShort:             \"Resize existing Lima disk\",\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:              diskResizeAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskResizeCommand.Flags().String(\"size\", \"\", \"Disk size\")\n\t_ = diskResizeCommand.MarkFlagRequired(\"size\")\n\treturn diskResizeCommand\n}\n\nfunc diskResizeAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskName := args[0]\n\tdisk, err := store.InspectDisk(diskName)\n\tif err != nil {\n\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"disk %q does not exists\", diskName)\n\t\t}\n\t\treturn err\n\t}\n\n\t// Shrinking can cause a disk failure\n\tif diskSize < disk.Size {\n\t\treturn fmt.Errorf(\"specified size %q is less than the current disk size %q. Disk shrinking is currently unavailable\", units.BytesSize(float64(diskSize)), units.BytesSize(float64(disk.Size)))\n\t}\n\n\tif disk.Instance != \"\" {\n\t\tinst, err := store.Inspect(disk.Instance)\n\t\tif err == nil {\n\t\t\tif inst.Status == store.StatusRunning {\n\t\t\t\treturn fmt.Errorf(\"cannot resize disk %q used by running instance %q. Please stop the VM instance\", diskName, disk.Instance)\n\t\t\t}\n\t\t}\n\t}\n\tif err := qemu.ResizeDataDisk(disk.Dir, disk.Format, int(diskSize)); err != nil {\n\t\treturn fmt.Errorf(\"failed to resize disk %q: %w\", diskName, err)\n\t}\n\tlogrus.Infof(\"Resized disk %q (%q)\", diskName, disk.Dir)\n\treturn nil\n}\n```"]}
{"repo": "lima-vm/lima", "name": "sudoersAction", "language": "go", "path": "cmd/limactl/sudoers.go", "position_ratio": 0.25, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to generate or verify the content of a specific system configuration file that enables support for a virtual machine network framework on macOS.\n2. **Input**: The function accepts optional arguments to either generate the content or check an existing configuration file against an expected template. It can be invoked with no arguments to generate the content, or with the `--check` flag followed by a file path to verify an existing file.\n3. **Output**: If generating content, the output is the expected content of the system configuration file, printed to standard output. If verifying, the function checks if the specified file matches the expected content and returns an error if it does not.\n4. **Procedure**: The function first checks if it is running on macOS, as it is only supported on this operating system. It then checks if it is in verification mode (`--check`). If so, it verifies the specified file against the expected content. If not in verification mode, it generates and prints the expected content for the system configuration file.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/limactl/show_ssh.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/dirnames\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst showSSHExample = `\n  \"cmd\" format (default): Full ssh command line.\n    $ limactl show-ssh --format=cmd default\n    ssh -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022 lima-default\n\n  \"args\" format: Similar to the cmd format but omits \"ssh\" and the destination address\n    $ limactl show-ssh --format=args default\n    -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022\n\n  \"options\" format: ssh option key value pairs\n    $ limactl show-ssh --format=options default\n    IdentityFile=\"/Users/example/.lima/_config/user\"\n    User=example\n    Hostname=127.0.0.1\n    Port=60022\n\n  \"config\" format: ~/.ssh/config format\n    $ limactl show-ssh --format=config default\n    Host lima-default\n      IdentityFile \"/Users/example/.lima/_config/user \"\n      User example\n      Hostname 127.0.0.1\n      Port 60022\n\n  To show the config file path:\n    $ limactl ls --format='{{.SSHConfigFile}}' default\n    /Users/example/.lima/default/ssh.config\n`\n\nfunc newShowSSHCommand() *cobra.Command {\n\tlimaHome := \"~/\" + dirnames.DotLima\n\tif s, err := dirnames.LimaDir(); err == nil {\n\t\tlimaHome = s\n\t}\n\tshellCmd := &cobra.Command{\n\t\tUse:   \"show-ssh [flags] INSTANCE\",\n\t\tShort: \"Show the ssh command line (DEPRECATED; use `ssh -F` instead)\",\n\t\tLong: fmt.Sprintf(`Show the ssh command line (DEPRECATED)\n\nWARNING: 'limactl show-ssh' is deprecated.\nInstead, use 'ssh -F %s/default/ssh.config lima-default' .\n`, limaHome),\n\t\tExample:           showSSHExample,\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n...\n// Path: cmd/limactl/snapshot.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/snapshot\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSnapshotCommand() *cobra.Command {\n\tsnapshotCmd := &cobra.Command{\n\t\tUse:   \"snapshot\",\n\t\tShort: \"Manage instance snapshots\",\n\t\tPersistentPreRun: func(cmd *cobra.Command, args []string) {\n\t\t\tlogrus.Warn(\"`limactl snapshot` is experimental\")\n\t\t},\n\t\tGroupID: advancedCommand,\n\t}\n\tsnapshotCmd.AddCommand(newSnapshotApplyCommand())\n\tsnapshotCmd.AddCommand(newSnapshotCreateCommand())\n\tsnapshotCmd.AddCommand(newSnapshotDeleteCommand())\n\tsnapshotCmd.AddCommand(newSnapshotListCommand())\n\n\treturn snapshotCmd\n}\n\nfunc newSnapshotCreateCommand() *cobra.Command {\n\tcreateCmd := &cobra.Command{\n\t\tUse:               \"create INSTANCE\",\n\t\tAliases:           []string{\"save\"},\n\t\tShort:             \"Create (save) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotCreateAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tcreateCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn createCmd\n}\n\nfunc snapshotCreateAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Save(ctx, inst, tag)\n}\n\nfunc newSnapshotDeleteCommand() *cobra.Command {\n\tdeleteCmd := &cobra.Command{\n\t\tUse:               \"delete INSTANCE\",\n\t\tAliases:           []string{\"del\"},\n\t\tShort:             \"Delete (del) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotDeleteAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tdeleteCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn deleteCmd\n}\n\nfunc snapshotDeleteAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Del(ctx, inst, tag)\n}\n\nfunc newSnapshotApplyCommand() *cobra.Command {\n\tapplyCmd := &cobra.Command{\n\t\tUse:               \"apply INSTANCE\",\n\t\tAliases:           []string{\"load\"},\n\t\tShort:             \"Apply (load) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotApplyAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tapplyCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn applyCmd\n}\n\nfunc snapshotApplyAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Load(ctx, inst, tag)\n}\n\nfunc newSnapshotListCommand() *cobra.Command {\n\tlistCmd := &cobra.Command{\n\t\tUse:               \"list INSTANCE\",\n\t\tAliases:           []string{\"ls\"},\n\t\tShort:             \"List existing snapshots\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotListAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tlistCmd.Flags().BoolP(\"quiet\", \"q\", false, \"Only show tags\")\n\n\treturn listCmd\n}\n\nfunc snapshotListAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tquiet, err := cmd.Flags().GetBool(\"quiet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tctx := cmd.Context()\n\tout, err := snapshot.List(ctx, inst)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif quiet {\n\t\tfor i, line := range strings.Split(out, \"\\n\") {\n\t\t\t// \"ID\", \"TAG\", \"VM SIZE\", \"DATE\", \"VM CLOCK\", \"ICOUNT\"\n\t\t\tfields := strings.Fields(line)\n\t\t\tif i == 0 && len(fields) > 1 && fields[1] != \"TAG\" {\n\t\t\t\t// make sure that output matches the expected\n\t\t\t\treturn fmt.Errorf(\"unknown header: %s\", line)\n\t\t\t}\n\t\t\tif i == 0 || line == \"\" {\n\t\t\t\t// skip header and empty line after using split\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttag := fields[1]\n\t\t\tfmt.Printf(\"%s\\n\", tag)\n\t\t}\n\t\treturn nil\n\t}\n\tfmt.Print(out)\n\treturn nil\n}\n\nfunc snapshotBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/start-at-login.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc startAtLoginCommand() *cobra.Command {\n\tstartAtLoginCommand := &cobra.Command{\n\t\tUse:               \"start-at-login INSTANCE\",\n\t\tShort:             \"Register/Unregister an autostart file for the instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              startAtLoginAction,\n\t\tValidArgsFunction: startAtLoginComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\n\tstartAtLoginCommand.Flags().Bool(\n\t\t\"enabled\", true,\n\t\t\"Automatically start the instance when the user logs in\",\n\t)\n\n\treturn startAtLoginCommand\n}\n\nfunc startAtLoginAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\tlogrus.Infof(\"Instance %q not found\", instName)\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\n\tflags := cmd.Flags()\n\tstartAtLogin, err := flags.GetBool(\"enabled\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif startAtLogin {\n\t\tif err := autostart.CreateStartAtLoginEntry(runtime.GOOS, inst.Name, inst.Dir); err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"Can't create an autostart file for instance %q\", inst.Name)\n\t\t} else {\n\t\t\tlogrus.Infof(\"The autostart file %q has been created or updated\", autostart.GetFilePath(runtime.GOOS, inst.Name))\n\t\t}\n\t} else {\n\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"The autostart file %q could not be deleted\", instName)\n\t\t} else if deleted {\n\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc startAtLoginComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/sudoers.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/networks\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSudoersCommand() *cobra.Command {\n\tnetworksMD := \"$PREFIX/share/doc/lima/docs/network.md\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n\t\tnetworksMD = filepath.Join(prefixDir, \"share/doc/lima/docs/network.md\")\n\t}\n\tsudoersCommand := &cobra.Command{\n\t\tUse: \"sudoers [--check [SUDOERSFILE-TO-CHECK]]\",\n\t\tExample: `\nTo generate the /etc/sudoers.d/lima file:\n$ limactl sudoers | sudo tee /etc/sudoers.d/lima\n\nTo validate the existing /etc/sudoers.d/lima file:\n$ limactl sudoers --check /etc/sudoers.d/lima\n`,\n\t\tShort: \"Generate the content of the /etc/sudoers.d/lima file\",\n\t\tLong: fmt.Sprintf(`Generate the content of the /etc/sudoers.d/lima file for enabling vmnet.framework support.\nThe content is written to stdout, NOT to the file.\nThis command must not run as the root.\nSee %s for the usage.`, networksMD),\n\t\tArgs:    WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:    sudoersAction,\n\t\tGroupID: advancedCommand,\n\t}\n\tconfigFile, _ := networks.ConfigFile()\n\tsudoersCommand.Flags().Bool(\"check\", false,\n\t\tfmt.Sprintf(\"check that the sudoers file is up-to-date with %q\", configFile))\n\treturn sudoersCommand\n}\n\n\nfunc sudoersAction(cmd *cobra.Command, args []string) error {\n\tif runtime.GOOS != \"darwin\" {\n\t\treturn errors.New(\"sudoers command is only supported on macOS right now\")\n\t}\n\tcheck, err := cmd.Flags().GetBool(\"check\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif check {\n\t\treturn verifySudoAccess(args)\n\t}\n\tswitch len(args) {\n\tcase 0:\n\t\t// NOP\n\tcase 1:\n\t\treturn fmt.Errorf(\"the file argument can be specified only for --check mode\")\n\tdefault:\n\t\treturn fmt.Errorf(\"unexpected arguments %v\", args)\n\t}\n\tsudoers, err := networks.Sudoers()\n\tif err != nil {\n\t\treturn err\n\t}\n\tfmt.Print(sudoers)\n\treturn nil\n}\n\nfunc verifySudoAccess(args []string) error {\n\tconfig, err := networks.Config()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := config.Validate(); err != nil {\n\t\treturn err\n\t}\n\tvar file string\n\tswitch len(args) {\n\tcase 0:\n\t\tfile = config.Paths.Sudoers\n\t\tif file == \"\" {\n\t\t\tconfigFile, _ := networks.ConfigFile()\n\t\t\treturn fmt.Errorf(\"no sudoers file defined in %q\", configFile)\n\t\t}\n\tcase 1:\n\t\tfile = args[0]\n\tdefault:\n\t\treturn errors.New(\"can check only a single sudoers file\")\n\t}\n\tif err := config.VerifySudoAccess(file); err != nil {\n\t\treturn err\n\t}\n\tfmt.Printf(\"%q is up-to-date (or sudo doesn't require a password)\\n\", file)\n\treturn nil\n}\n\n// Path: cmd/limactl/unprotect.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newUnprotectCommand() *cobra.Command {\n\tunprotectCommand := &cobra.Command{\n\t\tUse:               \"unprotect INSTANCE [INSTANCE, ...]\",\n\t\tShort:             \"Unprotect an instance\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              unprotectAction,\n\t\tValidArgsFunction: unprotectBashComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn unprotectCommand\n}\n\nfunc unprotectAction(_ *cobra.Command, args []string) error {\n\tvar errs []error\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to inspect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tif !inst.Protected {\n\t\t\tlogrus.Warnf(\"Instance %q isn't protected. Skipping.\", instName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := inst.Unprotect(); err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to unprotect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tlogrus.Infof(\"Unprotected %q\", instName)\n\t}\n\treturn errors.Join(errs...)\n}\n\nfunc unprotectBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/usernet.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\n\t\"github.com/lima-vm/lima/pkg/networks/usernet\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newUsernetCommand() *cobra.Command {\n\thostagentCommand := &cobra.Command{\n\t\tUse:    \"usernet\",\n\t\tShort:  \"run usernet\",\n\t\tArgs:   cobra.ExactArgs(0),\n\t\tRunE:   usernetAction,\n\t\tHidden: true,\n\t}\n\thostagentCommand.Flags().StringP(\"pidfile\", \"p\", \"\", \"write pid to file\")\n\thostagentCommand.Flags().StringP(\"endpoint\", \"e\", \"\", \"exposes usernet api(s) on this endpoint\")\n\thostagentCommand.Flags().String(\"listen-qemu\", \"\", \"listen for qemu connections\")\n\thostagentCommand.Flags().String(\"listen\", \"\", \"listen on a Unix socket and receive Bess-compatible FDs as SCM_RIGHTS messages\")\n\thostagentCommand.Flags().String(\"subnet\", \"192.168.5.0/24\", \"sets subnet value for the usernet network\")\n\thostagentCommand.Flags().Int(\"mtu\", 1500, \"mtu\")\n\thostagentCommand.Flags().StringToString(\"leases\", nil, \"pass default static leases for startup. Eg: '192.168.104.1=52:55:55:b3:bc:d9,192.168.104.2=5a:94:ef:e4:0c:df' \")\n\treturn hostagentCommand\n}\n\nfunc usernetAction(cmd *cobra.Command, _ []string) error {\n\tpidfile, err := cmd.Flags().GetString(\"pidfile\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif pidfile != \"\" {\n\t\tif _, err := os.Stat(pidfile); !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"pidfile %q already exists\", pidfile)\n\t\t}\n\t\tif err := os.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())+\"\\n\"), 0o644); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer os.RemoveAll(pidfile)\n\t}\n\tendpoint, err := cmd.Flags().GetString(\"endpoint\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tqemuSocket, err := cmd.Flags().GetString(\"listen-qemu\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfdSocket, err := cmd.Flags().GetString(\"listen\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tsubnet, err := cmd.Flags().GetString(\"subnet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tleases, err := cmd.Flags().GetStringToString(\"leases\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmtu, err := cmd.Flags().GetInt(\"mtu\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tos.RemoveAll(endpoint)\n\tos.RemoveAll(qemuSocket)\n\tos.RemoveAll(fdSocket)\n\n\treturn usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{\n\t\tMTU:           mtu,\n\t\tEndpoint:      endpoint,\n\t\tQemuSocket:    qemuSocket,\n\t\tFdSocket:      fdSocket,\n\t\tSubnet:        subnet,\n\t\tDefaultLeases: leases,\n\t})\n}\n\n// Path: cmd/limactl/validate.go\npackage main\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/cmd/limactl/guessarg\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/spf13/cobra\"\n\n\t\"github.com/sirupsen/logrus\"\n)\n\nfunc newValidateCommand() *cobra.Command {\n\tvalidateCommand := &cobra.Command{\n\t\tUse:     \"validate FILE.yaml [FILE.yaml, ...]\",\n\t\tShort:   \"Validate YAML files\",\n\t\tArgs:    WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:    validateAction,\n\t\tGroupID: advancedCommand,\n\t}\n\treturn validateCommand\n}\n\nfunc validateAction(_ *cobra.Command, args []string) error {\n\tfor _, f := range args {\n\t\t_, err := store.LoadYAMLByFilePath(f)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to load YAML file %q: %w\", f, err)\n\t\t}\n\t\tif _, err := guessarg.InstNameFromYAMLPath(f); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tlogrus.Infof(\"%q: OK\", f)\n\t}\n\n\treturn nil\n}\n\n", "needle_token_start": 3682, "needle_token_end": 3900, "code_context_ntokens": 5973, "output": ["```go\nfunc newSudoersCommand() *cobra.Command {\n\tnetworksMD := \"$PREFIX/share/doc/lima/docs/network.md\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n\t\tnetworksMD = filepath.Join(prefixDir, \"share/doc/lima/docs/network.md\")\n\t}\n\tsudoersCommand := &cobra.Command{\n\t\tUse: \"sudoers [--check [SUDOERSFILE-TO-CHECK]]\",\n\t\tExample: `\nTo generate the /etc/sudoers.d/lima file:\n$ limactl sudoers | sudo tee /etc/sudoers.d/lima\n\nTo validate the existing /etc/sudoers.d/lima file:\n$ limactl sudoers --check /etc/sudoers.d/lima\n`,\n\t\tShort: \"Generate the content of the /etc/sudoers.d/lima file\",\n\t\tLong: fmt.Sprintf(`Generate the content of the /etc/sudoers.d/lima file for enabling vmnet.framework support.\nThe content is written to stdout, NOT to the file.\nThis command must not run as the root.\nSee %s for the usage.`, networksMD),\n\t\tArgs:    WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:    sudoersAction,\n\t\tGroupID: advancedCommand,\n\t}\n\tconfigFile, _ := networks.ConfigFile()\n\tsudoersCommand.Flags().Bool(\"check\", false,\n\t\tfmt.Sprintf(\"check that the sudoers file is up-to-date with %q\", configFile))\n\treturn sudoersCommand\n}\n\nfunc sudoersAction(cmd *cobra.Command, args []string) error {\n\tif runtime.GOOS != \"darwin\" {\n\t\treturn errors.New(\"sudoers command is only supported on macOS right now\")\n\t}\n\tcheck, err := cmd.Flags().GetBool(\"check\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif check {\n\t\treturn verifySudoAccess(args)\n\t}\n\tswitch len(args) {\n\tcase 0:\n\t\t// NOP\n\tcase 1:\n\t\treturn fmt.Errorf(\"the file argument can be specified only for --check mode\")\n\tdefault:\n\t\treturn fmt.Errorf(\"unexpected arguments %v\", args)\n\t}\n\tsudoers, err := networks.Sudoers()\n\tif err != nil {\n\t\treturn err\n\t}\n\tfmt.Print(sudoers)\n\treturn nil\n}\n\nfunc verifySudoAccess(args []string) error {\n\tconfig, err := networks.Config()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := config.Validate(); err != nil {\n\t\treturn err\n\t}\n\tvar file string\n\tswitch len(args) {\n\tcase 0:\n\t\tfile = config.Paths.Sudoers\n\t\tif file == \"\" {\n\t\t\tconfigFile, _ := networks.ConfigFile()\n\t\t\treturn fmt.Errorf(\"no sudoers file defined in %q\", configFile)\n\t\t}\n\tcase 1:\n\t\tfile = args[0]\n\tdefault:\n\t\treturn errors.New(\"can check only a single sudoers file\")\n\t}\n\tif err := config.VerifySudoAccess(file); err != nil {\n\t\treturn err\n\t}\n\tfmt.Printf(\"%q is up-to-date (or sudo doesn't require a password)\\n\", file)\n\treturn nil\n}\n```"]}
{"repo": "lima-vm/lima", "name": "deleteInstance", "language": "go", "path": "cmd/limactl/delete.go", "position_ratio": 0.35, "description": "\nFunction Description:\n1. **Purpose**: To remove a virtual machine instance from the system, including its directory and unregistering it from the system's startup configuration.\n2. **Input**: The function takes a context for handling cancellations and timeouts, an instance object representing the virtual machine, and a boolean indicating whether to forcibly remove the instance.\n3. **Output**: Returns an error if the deletion process fails, otherwise returns nil indicating successful deletion.\n4. **Procedure**: \n   - Checks if the instance is protected and if so, returns an error to prevent accidental deletion.\n   - Verifies the instance's status to ensure it is stopped unless the force removal option is enabled.\n   - Calls a function to forcibly stop the instance if necessary.\n   - Unregisters the instance from the system's startup configuration.\n   - Deletes the instance's directory from the file system.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/limactl/disk.go\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"os\"\n\t\"text/tabwriter\"\n\n\t\"github.com/docker/go-units\"\n\t\"github.com/lima-vm/lima/pkg/qemu\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDiskCommand() *cobra.Command {\n\tdiskCommand := &cobra.Command{\n\t\tUse:   \"disk\",\n\t\tShort: \"Lima disk management\",\n\t\tExample: `  Create a disk:\n  $ limactl disk create DISK --size SIZE [--format qcow2]\n\n  List existing disks:\n  $ limactl disk ls\n\n  Delete a disk:\n  $ limactl disk delete DISK\n  \n  Resize a disk:\n  $ limactl disk resize DISK --size SIZE`,\n\t\tSilenceUsage:  true,\n\t\tSilenceErrors: true,\n\t\tGroupID:       advancedCommand,\n\t}\n\tdiskCommand.AddCommand(\n\t\tnewDiskCreateCommand(),\n\t\tnewDiskListCommand(),\n\t\tnewDiskDeleteCommand(),\n\t\tnewDiskUnlockCommand(),\n\t\tnewDiskResizeCommand(),\n\t)\n\treturn diskCommand\n}\n\nfunc newDiskCreateCommand() *cobra.Command {\n\tdiskCreateCommand := &cobra.Command{\n\t\tUse: \"create DISK\",\n\t\tExample: `\nTo create a new disk:\n$ limactl disk create DISK --size SIZE [--format qcow2]\n`,\n\t\tShort: \"Create a Lima disk\",\n\t\tArgs:  WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:  diskCreateAction,\n\t}\n\tdiskCreateCommand.Flags().String(\"size\", \"\", \"configure the disk size\")\n\t_ = diskCreateCommand.MarkFlagRequired(\"size\")\n\tdiskCreateCommand.Flags().String(\"format\", \"qcow2\", \"specify the disk format\")\n\treturn diskCreateCommand\n}\n\nfunc diskCreateAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tswitch format {\n\tcase \"qcow2\", \"raw\":\n\tdefault:\n\t\treturn fmt.Errorf(`disk format %q not supported, use \"qcow2\" or \"raw\" instead`, format)\n\t}\n\n\t// only exactly one arg is allowed\n\tname := args[0]\n\n\tdiskDir, err := store.DiskDir(name)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif _, err := os.Stat(diskDir); !errors.Is(err, fs.ErrNotExist) {\n\t\treturn fmt.Errorf(\"disk %q already exists (%q)\", name, diskDir)\n\t}\n\n\tlogrus.Infof(\"Creating %s disk %q with size %s\", format, name, units.BytesSize(float64(diskSize)))\n\n\tif err := os.MkdirAll(diskDir, 0o700); err != nil {\n\t\treturn err\n\t}\n\n\tif err := qemu.CreateDataDisk(diskDir, format, int(diskSize)); err != nil {\n\t\trerr := os.RemoveAll(diskDir)\n\t\tif rerr != nil {\n\t\t\terr = errors.Join(err, fmt.Errorf(\"failed to remove a directory %q: %w\", diskDir, rerr))\n\t\t}\n\t\treturn fmt.Errorf(\"Failed to create %s disk in %q: %w\", format, diskDir, err)\n\t}\n\n\treturn nil\n}\n\nfunc newDiskListCommand() *cobra.Command {\n\tdiskListCommand := &cobra.Command{\n\t\tUse: \"list\",\n\t\tExample: `\nTo list existing disks:\n$ limactl disk list\n`,\n\t\tShort:   \"List existing Lima disks\",\n\t\tAliases: []string{\"ls\"},\n\t\tArgs:    WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:    diskListAction,\n\t}\n\tdiskListCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\treturn diskListCommand\n}\n\nfunc diskMatches(diskName string, disks []string) []string {\n\tmatches := []string{}\n\tfor _, disk := range disks {\n\t\tif disk == diskName {\n\t\t\tmatches = append(matches, disk)\n\t\t}\n\t}\n\treturn matches\n}\n\nfunc diskListAction(cmd *cobra.Command, args []string) error {\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tallDisks, err := store.Disks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdisks := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := diskMatches(arg, allDisks)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tdisks = append(disks, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No disk matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tdisks = allDisks\n\t}\n\n\tif jsonFormat {\n\t\tfor _, diskName := range disks {\n\t\t\tdisk, err := store.InspectDisk(diskName)\n\t\t\tif err != nil {\n\t\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tj, err := json.Marshal(disk)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), string(j))\n\t\t}\n\t\treturn nil\n\t}\n\n\tw := tabwriter.NewWriter(cmd.OutOrStdout(), 4, 8, 4, ' ', 0)\n\tfmt.Fprintln(w, \"NAME\\tSIZE\\tFORMAT\\tDIR\\tIN-USE-BY\")\n\n\tif len(disks) == 0 {\n\t\tlogrus.Warn(\"No disk found. Run `limactl disk create DISK --size SIZE` to create a disk.\")\n\t}\n\n\tfor _, diskName := range disks {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Fprintf(w, \"%s\\t%s\\t%s\\t%s\\t%s\\n\", disk.Name, units.BytesSize(float64(disk.Size)), disk.Format, disk.Dir, disk.Instance)\n\t}\n\n\treturn w.Flush()\n}\n\nfunc newDiskDeleteCommand() *cobra.Command {\n\tdiskDeleteCommand := &cobra.Command{\n\t\tUse: \"delete DISK [DISK, ...]\",\n\t\tExample: `\nTo delete a disk:\n$ limactl disk delete DISK\n\nTo delete multiple disks:\n$ limactl disk delete DISK1 DISK2 ...\n`,\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskDeleteAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskDeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"force delete\")\n\treturn diskDeleteCommand\n}\n\nfunc diskDeleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tinstNames, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar instances []*store.Instance\n\tfor _, instName := range instNames {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tinstances = append(instances, inst)\n\t}\n\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tif !force {\n\t\t\tif disk.Instance != \"\" {\n\t\t\t\treturn fmt.Errorf(\"cannot delete disk %q in use by instance %q\", disk.Name, disk.Instance)\n\t\t\t}\n\t\t\tvar refInstances []string\n\t\t\tfor _, inst := range instances {\n\t\t\t\tif len(inst.AdditionalDisks) > 0 {\n\t\t\t\t\tfor _, d := range inst.AdditionalDisks {\n\t\t\t\t\t\tif d.Name == diskName {\n\t\t\t\t\t\t\trefInstances = append(refInstances, inst.Name)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(refInstances) > 0 {\n\t\t\t\tlogrus.Warnf(\"Skipping deleting disk %q, disk is referenced by one or more non-running instances: %q\",\n\t\t\t\t\tdiskName, refInstances)\n\t\t\t\tlogrus.Warnf(\"To delete anyway, run %q\", forceDeleteCommand(diskName))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif err := deleteDisk(disk); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\n...\n// Path: cmd/limactl/stop.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\thostagentevents \"github.com/lima-vm/lima/pkg/hostagent/events\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newStopCommand() *cobra.Command {\n\tstopCmd := &cobra.Command{\n\t\tUse:               \"stop INSTANCE\",\n\t\tShort:             \"Stop an instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              stopAction,\n\t\tValidArgsFunction: stopBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tstopCmd.Flags().BoolP(\"force\", \"f\", false, \"force stop the instance\")\n\treturn stopCmd\n}\n\nfunc stopAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif force {\n\t\tstopInstanceForcibly(inst)\n\t} else {\n\t\terr = stopInstanceGracefully(inst)\n\t}\n\t// TODO: should we also reconcile networks if graceful stop returned an error?\n\tif err == nil {\n\t\terr = networks.Reconcile(cmd.Context(), \"\")\n\t}\n\treturn err\n}\n\nfunc stopInstanceGracefully(inst *store.Instance) error {\n\tif inst.Status != store.StatusRunning {\n\t\treturn fmt.Errorf(\"expected status %q, got %q (maybe use `limactl stop -f`?)\", store.StatusRunning, inst.Status)\n\t}\n\n\tbegin := time.Now() // used for logrus propagation\n\tlogrus.Infof(\"Sending SIGINT to hostagent process %d\", inst.HostAgentPID)\n\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigInt); err != nil {\n\t\tlogrus.Error(err)\n\t}\n\n\tlogrus.Info(\"Waiting for the host agent and the driver processes to shut down\")\n\treturn waitForHostAgentTermination(context.TODO(), inst, begin)\n}\n\nfunc waitForHostAgentTermination(ctx context.Context, inst *store.Instance, begin time.Time) error {\n\tctx2, cancel := context.WithTimeout(ctx, 3*time.Minute)\n\tdefer cancel()\n\n\tvar receivedExitingEvent bool\n\tonEvent := func(ev hostagentevents.Event) bool {\n\t\tif len(ev.Status.Errors) > 0 {\n\t\t\tlogrus.Errorf(\"%+v\", ev.Status.Errors)\n\t\t}\n\t\tif ev.Status.Exiting {\n\t\t\treceivedExitingEvent = true\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\n\thaStdoutPath := filepath.Join(inst.Dir, filenames.HostAgentStdoutLog)\n\thaStderrPath := filepath.Join(inst.Dir, filenames.HostAgentStderrLog)\n\n\tif err := hostagentevents.Watch(ctx2, haStdoutPath, haStderrPath, begin, onEvent); err != nil {\n\t\treturn err\n\t}\n\n\tif !receivedExitingEvent {\n\t\treturn errors.New(\"did not receive an event with the \\\"exiting\\\" status\")\n\t}\n\n\treturn nil\n}\n\nfunc stopInstanceForcibly(inst *store.Instance) {\n\tif inst.DriverPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the %s driver process %d\", inst.VMType, inst.DriverPID)\n\t\tif err := osutil.SysKill(inst.DriverPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Infof(\"The %s driver process seems already stopped\", inst.VMType)\n\t}\n\n\tfor _, d := range inst.AdditionalDisks {\n\t\tdiskName := d.Name\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.Warnf(\"Disk %q does not exist\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\tlogrus.Warnf(\"Failed to unlock disk %q. To use, run `limactl disk unlock %v`\", diskName, diskName)\n\t\t}\n\t}\n\n\tif inst.HostAgentPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the host agent process %d\", inst.HostAgentPID)\n\t\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"The host agent process seems already stopped\")\n\t}\n\n\tsuffixesToBeRemoved := []string{\".pid\", \".sock\", \".tmp\"}\n\tlogrus.Infof(\"Removing %s under %q\", inst.Dir, strings.ReplaceAll(strings.Join(suffixesToBeRemoved, \" \"), \".\", \"*.\"))\n\tfi, err := os.ReadDir(inst.Dir)\n\tif err != nil {\n\t\tlogrus.Error(err)\n\t\treturn\n\t}\n\tfor _, f := range fi {\n\t\tpath := filepath.Join(inst.Dir, f.Name())\n\t\tfor _, suffix := range suffixesToBeRemoved {\n\t\t\tif strings.HasSuffix(path, suffix) {\n\t\t\t\tlogrus.Infof(\"Removing %q\", path)\n\t\t\t\tif err := os.Remove(path); err != nil {\n\t\t\t\t\tlogrus.Error(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc stopBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/delete.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/stop\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDeleteCommand() *cobra.Command {\n\tdeleteCommand := &cobra.Command{\n\t\tUse:               \"delete INSTANCE [INSTANCE, ...]\",\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete an instance of Lima.\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              deleteAction,\n\t\tValidArgsFunction: deleteBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\tdeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"forcibly kill the processes\")\n\treturn deleteCommand\n}\n\nfunc deleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent instance %q\", instName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif err := deleteInstance(cmd.Context(), inst, force); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete instance %q: %w\", instName, err)\n\t\t}\n\t\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\t\tif err != nil && !errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.WithError(err).Warnf(\"The autostart file for instance %q does not exist\", instName)\n\t\t\t} else if deleted {\n\t\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t\t}\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", instName, inst.Dir)\n\t}\n\treturn networks.Reconcile(cmd.Context(), \"\")\n}\n\n\nfunc deleteInstance(ctx context.Context, inst *store.Instance, force bool) error {\n\tif inst.Protected {\n\t\treturn fmt.Errorf(\"instance is protected to prohibit accidental removal (Hint: use `limactl unprotect`)\")\n\t}\n\tif !force && inst.Status != store.StatusStopped {\n\t\treturn fmt.Errorf(\"expected status %q, got %q\", store.StatusStopped, inst.Status)\n\t}\n\n\tstopInstanceForcibly(inst)\n\n\tif err := stop.Unregister(ctx, inst); err != nil {\n\t\treturn fmt.Errorf(\"failed to unregister %q: %w\", inst.Dir, err)\n\t}\n\n\tif err := os.RemoveAll(inst.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", inst.Dir, err)\n\t}\n\n\treturn nil\n}\n\nfunc deleteBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/editflags/editflags.go\npackage editflags\n\nimport (\n\t\"fmt\"\n\t\"math/bits\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pbnjay/memory\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n\tflag \"github.com/spf13/pflag\"\n)\n\n// RegisterEdit registers flags related to in-place YAML modification, for `limactl edit`.\nfunc RegisterEdit(cmd *cobra.Command) {\n\tregisterEdit(cmd, \"\")\n}\n\nfunc registerEdit(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\n\tflags.Int(\"cpus\", 0, commentPrefix+\"number of CPUs\") // Similar to colima's --cpu, but the flag name is slightly different (cpu vs cpus)\n\t_ = cmd.RegisterFlagCompletionFunc(\"cpus\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeCPUs(runtime.NumCPU()) {\n\t\t\tres = append(res, strconv.Itoa(f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.IPSlice(\"dns\", nil, commentPrefix+\"specify custom DNS (disable host resolver)\") // colima-compatible\n\n\tflags.Float32(\"memory\", 0, commentPrefix+\"memory in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeMemoryGiB(memory.TotalMemory()) {\n\t\t\tres = append(res, fmt.Sprintf(\"%.1f\", f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.StringSlice(\"mount\", nil, commentPrefix+\"directories to mount, suffix ':w' for writable (Do not specify directories that overlap with the existing mounts)\") // colima-compatible\n\n\tflags.String(\"mount-type\", \"\", commentPrefix+\"mount type (reverse-sshfs, 9p, virtiofs)\") // Similar to colima's --mount-type=(sshfs|9p|virtiofs), but \"reverse-sshfs\" is Lima is called \"sshfs\" in colima\n\t_ = cmd.RegisterFlagCompletionFunc(\"mount-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"reverse-sshfs\", \"9p\", \"virtiofs\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"mount-writable\", false, commentPrefix+\"make all mounts writable\")\n\tflags.Bool(\"mount-inotify\", false, commentPrefix+\"enable inotify for mounts\")\n\n\tflags.StringSlice(\"network\", nil, commentPrefix+\"additional networks, e.g., \\\"vzNAT\\\" or \\\"lima:shared\\\" to assign vmnet IP\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"network\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\t// TODO: retrieve the lima:* network list from networks.yaml\n\t\treturn []string{\"lima:shared\", \"lima:bridged\", \"lima:host\", \"lima:user-v2\", \"vzNAT\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"rosetta\", false, commentPrefix+\"enable Rosetta (for vz instances)\")\n\n\tflags.String(\"set\", \"\", commentPrefix+\"modify the template inplace, using yq syntax\")\n\n\t// negative performance impact: https://gitlab.com/qemu-project/qemu/-/issues/334\n\tflags.Bool(\"video\", false, commentPrefix+\"enable video output (has negative performance impact for QEMU)\")\n}\n\n// RegisterCreate registers flags related to in-place YAML modification, for `limactl create`.\nfunc RegisterCreate(cmd *cobra.Command, commentPrefix string) {\n\tregisterEdit(cmd, commentPrefix)\n\tflags := cmd.Flags()\n\n\tflags.String(\"arch\", \"\", commentPrefix+\"machine architecture (x86_64, aarch64, riscv64)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"arch\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"x86_64\", \"aarch64\", \"riscv64\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"containerd\", \"\", commentPrefix+\"containerd mode (user, system, user+system, none)\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"user\", \"system\", \"user+system\", \"none\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Float32(\"disk\", 0, commentPrefix+\"disk size in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"10\", \"30\", \"50\", \"100\", \"200\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"vm-type\", \"\", commentPrefix+\"virtual machine type (qemu, vz)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"qemu\", \"vz\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"plain\", false, commentPrefix+\"plain mode. Disable mounts, port forwarding, containerd, etc.\")\n}\n\nfunc defaultExprFunc(expr string) func(v *flag.Flag) (string, error) {\n\treturn func(v *flag.Flag) (string, error) {\n\t\treturn fmt.Sprintf(expr, v.Value), nil\n\t}\n}\n\n// YQExpressions returns YQ expressions.\nfunc YQExpressions(flags *flag.FlagSet, newInstance bool) ([]string, error) {\n\ttype def struct {\n\t\tflagName                 string\n\t\texprFunc                 func(*flag.Flag) (string, error)\n\t\tonlyValidForNewInstances bool\n\t\texperimental             bool\n\t}\n\td := defaultExprFunc\n\tdefs := []def{\n\t\t{\"cpus\", d(\".cpus = %s\"), false, false},\n\t\t{\n\t\t\t\"dns\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tipSlice, err := flags.GetIPSlice(\"dns\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.dns += [`\n\t\t\t\tfor i, ip := range ipSlice {\n\t\t\t\t\texpr += fmt.Sprintf(\"%q\", ip)\n\t\t\t\t\tif i < len(ipSlice)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .dns |= unique | .hostResolver.enabled=false`\n\t\t\t\tlogrus.Warnf(\"Disabling HostResolver, as custom DNS addresses (%v) are specified\", ipSlice)\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"memory\", d(\".memory = \\\"%sGiB\\\"\"), false, false},\n\t\t{\n\t\t\t\"mount\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"mount\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.mounts += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\twritable := strings.HasSuffix(s, \":w\")\n\t\t\t\t\tloc := strings.TrimSuffix(s, \":w\")\n\t\t\t\t\texpr += fmt.Sprintf(`{\"location\": %q, \"writable\": %v}`, loc, writable)\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .mounts |= unique_by(.location)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"mount-type\", d(\".mountType = %q\"), false, false},\n\t\t{\"mount-inotify\", d(\".mountInotify = %s\"), false, true},\n\t\t{\"mount-writable\", d(\".mounts[].writable = %s\"), false, false},\n\t\t{\n\t\t\t\"network\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"network\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.networks += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\t// CLI syntax is still experimental (YAML syntax is out of experimental)\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase s == \"vzNAT\":\n\t\t\t\t\t\texpr += `{\"vzNAT\": true}`\n\t\t\t\t\tcase strings.HasPrefix(s, \"lima:\"):\n\t\t\t\t\t\tnetwork := strings.TrimPrefix(s, \"lima:\")\n\t\t\t\t\t\texpr += fmt.Sprintf(`{\"lima\": %q}`, network)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\treturn \"\", fmt.Errorf(\"network name must be \\\"vzNAT\\\" or \\\"lima:*\\\", got %q\", s)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .networks |= unique_by(.lima)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t\"rosetta\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tb, err := flags.GetBool(\"rosetta\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\treturn fmt.Sprintf(\".rosetta.enabled = %v | .rosetta.binfmt = %v\", b, b), nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\ttrue,\n\t\t},\n\t\t{\"set\", d(\"%s\"), false, false},\n\t\t{\n\t\t\t\"video\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tb, err := flags.GetBool(\"video\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tif b {\n\t\t\t\t\treturn \".video.display = \\\"default\\\"\", nil\n\t\t\t\t}\n\t\t\t\treturn \".video.display = \\\"none\\\"\", nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"arch\", d(\".arch = %q\"), true, false},\n\t\t{\n\t\t\t\"containerd\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\ts, err := flags.GetString(\"containerd\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tswitch s {\n\t\t\t\tcase \"user\":\n\t\t\t\t\treturn `.containerd.user = true | .containerd.system = false`, nil\n\t\t\t\tcase \"system\":\n\t\t\t\t\treturn `.containerd.user = false | .containerd.system = true`, nil\n\t\t\t\tcase \"user+system\", \"system+user\":\n\t\t\t\t\treturn `.containerd.user = true | .containerd.system = true`, nil\n\t\t\t\tcase \"none\":\n\t\t\t\t\treturn `.containerd.user = false | .containerd.system = false`, nil\n\t\t\t\tdefault:\n\t\t\t\t\treturn \"\", fmt.Errorf(`expected one of [\"user\", \"system\", \"user+system\", \"none\"], got %q`, s)\n\t\t\t\t}\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t},\n\t\t{\"disk\", d(\".disk= \\\"%sGiB\\\"\"), true, false},\n\t\t{\"vm-type\", d(\".vmType = %q\"), true, false},\n\t\t{\"plain\", d(\".plain = %s\"), true, false},\n\t}\n\tvar exprs []string\n\tfor _, def := range defs {\n\t\tv := flags.Lookup(def.flagName)\n\t\tif v != nil && v.Changed {\n\t\t\tif def.experimental {\n\t\t\t\tlogrus.Warnf(\"`--%s` is experimental\", def.flagName)\n\t\t\t}\n\t\t\tif def.onlyValidForNewInstances && !newInstance {\n\t\t\t\tlogrus.Warnf(\"`--%s` is not applicable to an existing instance (Hint: create a new instance with `limactl create --%s=%s --name=NAME`)\",\n\t\t\t\t\tdef.flagName, def.flagName, v.Value.String())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\texpr, err := def.exprFunc(v)\n\t\t\tif err != nil {\n\t\t\t\treturn exprs, fmt.Errorf(\"error while processing flag %q: %w\", def.flagName, err)\n\t\t\t}\n\t\t\texprs = append(exprs, expr)\n\t\t}\n\t}\n\treturn exprs, nil\n}\n\nfunc isPowerOfTwo(x int) bool {\n\treturn bits.OnesCount(uint(x)) == 1\n}\n\nfunc completeCPUs(hostCPUs int) []int {\n\tvar res []int\n\tfor i := 1; i <= hostCPUs; i *= 2 {\n\t\tres = append(res, i)\n\t}\n\tif !isPowerOfTwo(hostCPUs) {\n\t\tres = append(res, hostCPUs)\n\t}\n\treturn res\n}\n\nfunc completeMemoryGiB(hostMemory uint64) []float32 {\n\thostMemoryHalfGiB := int(hostMemory / 2 / 1024 / 1024 / 1024)\n\tvar res []float32\n\tif hostMemoryHalfGiB < 1 {\n\t\tres = append(res, 0.5)\n\t}\n\tfor i := 1; i <= hostMemoryHalfGiB; i *= 2 {\n\t\tres = append(res, float32(i))\n\t}\n\tif hostMemoryHalfGiB > 1 && !isPowerOfTwo(hostMemoryHalfGiB) {\n\t\tres = append(res, float32(hostMemoryHalfGiB))\n\t}\n\treturn res\n}\n\n// Path: cmd/limactl/guessarg/guessarg.go\npackage guessarg\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/containerd/containerd/identifiers\"\n)\n\nfunc SeemsTemplateURL(arg string) (bool, *url.URL) {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false, u\n\t}\n\treturn u.Scheme == \"template\", u\n}\n\nfunc SeemsHTTPURL(arg string) bool {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif u.Scheme != \"http\" && u.Scheme != \"https\" {\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc SeemsFileURL(arg string) bool {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false\n\t}\n\treturn u.Scheme == \"file\"\n}\n\nfunc SeemsYAMLPath(arg string) bool {\n\tif strings.Contains(arg, \"/\") {\n\t\treturn true\n\t}\n\tlower := strings.ToLower(arg)\n\treturn strings.HasSuffix(lower, \".yml\") || strings.HasSuffix(lower, \".yaml\")\n}\n\nfunc InstNameFromURL(urlStr string) (string, error) {\n\tu, err := url.Parse(urlStr)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn InstNameFromYAMLPath(path.Base(u.Path))\n}\n\nfunc InstNameFromYAMLPath(yamlPath string) (string, error) {\n\ts := strings.ToLower(filepath.Base(yamlPath))\n\ts = strings.TrimSuffix(strings.TrimSuffix(s, \".yml\"), \".yaml\")\n\ts = strings.ReplaceAll(s, \".\", \"-\")\n\tif err := identifiers.Validate(s); err != nil {\n\t\treturn \"\", fmt.Errorf(\"filename %q is invalid: %w\", yamlPath, err)\n\t}\n\treturn s, nil\n}\n\n// Path: cmd/limactl/start.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/containerd/containerd/identifiers\"\n\t\"github.com/lima-vm/lima/cmd/limactl/editflags\"\n\t\"github.com/lima-vm/lima/cmd/limactl/guessarg\"\n\t\"github.com/lima-vm/lima/pkg/editutil\"\n\t\"github.com/lima-vm/lima/pkg/ioutilx\"\n\t\"github.com/lima-vm/lima/pkg/limayaml\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/start\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/lima-vm/lima/pkg/templatestore\"\n\t\"github.com/lima-vm/lima/pkg/uiutil\"\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/lima-vm/lima/pkg/yqutil\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc registerCreateFlags(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\tflags.String(\"name\", \"\", commentPrefix+\"override the instance name\")\n\tflags.Bool(\"list-templates\", false, commentPrefix+\"list available templates and exit\")\n\teditflags.RegisterCreate(cmd, commentPrefix)\n}\n\nfunc newCreateCommand() *cobra.Command {\n\tcreateCommand := &cobra.Command{\n\t\tUse: \"create FILE.yaml|URL\",\n\t\tExample: `\nTo create an instance \"default\" from the default Ubuntu template:\n$ limactl create\n\nTo create an instance \"default\" from a template \"docker\":\n$ limactl create --name=default template://docker\n\nTo create an instance \"default\" with modified parameters:\n$ limactl create --cpus=2 --memory=2\n\nTo create an instance \"default\" with yq expressions:\n$ limactl create --set='.cpus = 2 | .memory = \"2GiB\"'\n\nTo see the template list:\n$ limactl create --list-templates\n\nTo create an instance \"default\" from a local file:\n$ limactl create --name=default /usr/local/share/lima/templates/fedora.yaml\n\nTo create an instance \"default\" from a remote URL (use carefully, with a trustable source):\n$ limactl create --name=default https://raw.githubusercontent.com/lima-vm/lima/master/examples/alpine.yaml\n\nTo create an instance \"local\" from a template passed to stdin (--name parameter is required):\n$ cat template.yaml | limactl create --name=local -\n`,\n\t\tShort:             \"Create an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tValidArgsFunction: createBashComplete,\n\t\tRunE:              createAction,\n\t\tGroupID:           basicCommand,\n\t}\n\tregisterCreateFlags(createCommand, \"\")\n\treturn createCommand\n}\n\nfunc newStartCommand() *cobra.Command {\n\tstartCommand := &cobra.Command{\n\t\tUse: \"start NAME|FILE.yaml|URL\",\n\t\tExample: `\nTo create an instance \"default\" (if not created yet) from the default Ubuntu template, and start it:\n$ limactl start\n\nTo create an instance \"default\" from a template \"docker\", and start it:\n$ limactl start --name=default template://docker\n\n'limactl start' also accepts the 'limactl create' flags such as '--set'.\nSee the examples in 'limactl create --help'.\n`,\n\t\tShort:             \"Start an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tValidArgsFunction: startBashComplete,\n\t\tRunE:              startAction,\n\t\tGroupID:           basicCommand,\n\t}\n\tregisterCreateFlags(startCommand, \"[limactl create] \")\n\tif runtime.GOOS != \"windows\" {\n\t\tstartCommand.Flags().Bool(\"foreground\", false, \"run the hostagent in the foreground\")\n\t}\n\tstartCommand.Flags().Duration(\"timeout\", start.DefaultWatchHostAgentEventsTimeout, \"duration to wait for the instance to be running before timing out\")\n\treturn startCommand\n}\n\nfunc loadOrCreateInstance(cmd *cobra.Command, args []string, createOnly bool) (*store.Instance, error) {\n\tvar arg string // can be empty\n\tif len(args) > 0 {\n\t\targ = args[0]\n\t}\n\n\tvar (\n\t\tst  = &creatorState{}\n\t\terr error\n\t)\n\n\tflags := cmd.Flags()\n\n\t// Create an instance, with menu TUI when TTY is available\n\ttty, err := flags.GetBool(\"tty\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tst.instName, err = flags.GetString(\"name\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconst yBytesLimit = 4 * 1024 * 1024 // 4MiB\n\n\tif ok, u := guessarg.SeemsTemplateURL(arg); ok {\n\t\t// No need to use SecureJoin here. https://github.com/lima-vm/lima/pull/805#discussion_r853411702\n\t\ttemplateName := filepath.Join(u.Host, u.Path)\n\t\tlogrus.Debugf(\"interpreting argument %q as a template name %q\", arg, templateName)\n\t\tif st.instName == \"\" {\n\t\t\t// e.g., templateName = \"deprecated/centos-7\" , st.instName = \"centos-7\"\n\t\t\tst.instName = filepath.Base(templateName)\n\t\t}\n\t\tst.yBytes, err = templatestore.Read(templateName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsHTTPURL(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromURL(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a http url for instance %q\", arg, st.instName)\n\t\treq, err := http.NewRequestWithContext(cmd.Context(), http.MethodGet, arg, http.NoBody)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tresp, err := http.DefaultClient.Do(req)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer resp.Body.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(resp.Body, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsFileURL(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromURL(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a file url for instance %q\", arg, st.instName)\n\t\tr, err := os.Open(strings.TrimPrefix(arg, \"file://\"))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer r.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(r, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsYAMLPath(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromYAMLPath(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a file path for instance %q\", arg, st.instName)\n\t\tr, err := os.Open(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer r.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(r, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if arg == \"-\" {\n\t\tif st.instName == \"\" {\n\t\t\treturn nil, errors.New(\"must pass instance name with --name when reading template from stdin\")\n\t\t}\n\t\tst.yBytes, err = io.ReadAll(os.Stdin)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unexpected error reading stdin: %w\", err)\n\t\t}\n\t\t// see if the tty was set explicitly or not\n\t\tttySet := cmd.Flags().Changed(\"tty\")\n\t\tif ttySet && tty {\n\t\t\treturn nil, errors.New(\"cannot use --tty=true and read template from stdin together\")\n\t\t}\n\t\ttty = false\n\t} else {\n\t\tif arg == \"\" {\n\t\t\tif st.instName == \"\" {\n\t\t\t\tst.instName = DefaultInstanceName\n\t\t\t}\n\t\t} else {\n\t\t\tlogrus.Debugf(\"interpreting argument %q as an instance name\", arg)\n\t\t\tif st.instName != \"\" && st.instName != arg {\n\t\t\t\treturn nil, fmt.Errorf(\"instance name %q and CLI flag --name=%q cannot be specified together\", arg, st.instName)\n\t\t\t}\n\t\t\tst.instName = arg\n\t\t}\n\t\tif err := identifiers.Validate(st.instName); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"argument must be either an instance name, a YAML file path, or a URL, got %q: %w\", st.instName, err)\n\t\t}\n\t\tinst, err := store.Inspect(st.instName)\n\t\tif err == nil {\n\t\t\tif createOnly {\n\t\t\t\treturn nil, fmt.Errorf(\"Instance %q already exists\", st.instName)\n\t\t\t}\n\t\t\tlogrus.Infof(\"Using the existing instance %q\", st.instName)\n\t\t\tyqExprs, err := editflags.YQExpressions(flags, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif len(yqExprs) > 0 {\n\t\t\t\tyq := yqutil.Join(yqExprs)\n\t\t\t\tinst, err = applyYQExpressionToExistingInstance(inst, yq)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"failed to apply yq expression %q to instance %q: %w\", yq, st.instName, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn inst, nil\n\t\t}\n\t\tif !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn nil, err\n\t\t}\n\t\tif arg != \"\" && arg != DefaultInstanceName {\n\t\t\tlogrus.Infof(\"Creating an instance %q from template://default (Not from template://%s)\", st.instName, st.instName)\n\t\t\tlogrus.Warnf(\"This form is deprecated. Use `limactl create --name=%s template://default` instead\", st.instName)\n\t\t}\n\t\t// Read the default template for creating a new instance\n\t\tst.yBytes, err = templatestore.Read(templatestore.Default)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tyqExprs, err := editflags.YQExpressions(flags, true)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tyq := yqutil.Join(yqExprs)\n\tif tty {\n\t\tvar err error\n\t\tst, err = chooseNextCreatorState(st, yq)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"Terminal is not available, proceeding without opening an editor\")\n\t\tif err := modifyInPlace(st, yq); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tsaveBrokenEditorBuffer := tty\n\treturn createInstance(cmd.Context(), st, saveBrokenEditorBuffer)\n}\n\nfunc applyYQExpressionToExistingInstance(inst *store.Instance, yq string) (*store.Instance, error) {\n\tif strings.TrimSpace(yq) == \"\" {\n\t\treturn inst, nil\n\t}\n\tfilePath := filepath.Join(inst.Dir, filenames.LimaYAML)\n\tyContent, err := os.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlogrus.Debugf(\"Applying yq expression %q to an existing instance %q\", yq, inst.Name)\n\tyBytes, err := yqutil.EvaluateExpression(yq, yContent)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filePath, yBytes, 0o644); err != nil {\n\t\treturn nil, err\n\t}\n\t// Reload\n\treturn store.Inspect(inst.Name)\n}\n\nfunc createInstance(ctx context.Context, st *creatorState, saveBrokenEditorBuffer bool) (*store.Instance, error) {\n\tif st.instName == \"\" {\n\t\treturn nil, errors.New(\"got empty st.instName\")\n\t}\n\tif len(st.yBytes) == 0 {\n\t\treturn nil, errors.New(\"got empty st.yBytes\")\n\t}\n\n\tinstDir, err := store.InstanceDir(st.instName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// the full path of the socket name must be less than UNIX_PATH_MAX chars.\n\tmaxSockName := filepath.Join(instDir, filenames.LongestSock)\n\tif len(maxSockName) >= osutil.UnixPathMax {\n\t\treturn nil, fmt.Errorf(\"instance name %q too long: %q must be less than UNIX_PATH_MAX=%d characters, but is %d\",\n\t\t\tst.instName, maxSockName, osutil.UnixPathMax, len(maxSockName))\n\t}\n\tif _, err := os.Stat(instDir); !errors.Is(err, os.ErrNotExist) {\n\t\treturn nil, fmt.Errorf(\"instance %q already exists (%q)\", st.instName, instDir)\n\t}\n\t// limayaml.Load() needs to pass the store file path to limayaml.FillDefault() to calculate default MAC addresses\n\tfilePath := filepath.Join(instDir, filenames.LimaYAML)\n\ty, err := limayaml.Load(st.yBytes, filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := limayaml.Validate(*y, true); err != nil {\n\t\tif !saveBrokenEditorBuffer {\n\t\t\treturn nil, err\n\t\t}\n\t\trejectedYAML := \"lima.REJECTED.yaml\"\n\t\tif writeErr := os.WriteFile(rejectedYAML, st.yBytes, 0o644); writeErr != nil {\n\t\t\treturn nil, fmt.Errorf(\"the YAML is invalid, attempted to save the buffer as %q but failed: %w: %w\", rejectedYAML, writeErr, err)\n\t\t}\n\t\treturn nil, fmt.Errorf(\"the YAML is invalid, saved the buffer as %q: %w\", rejectedYAML, err)\n\t}\n\tif err := os.MkdirAll(instDir, 0o700); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filePath, st.yBytes, 0o644); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filepath.Join(instDir, filenames.LimaVersion), []byte(version.Version), 0o444); err != nil {\n\t\treturn nil, err\n\t}\n\n\tinst, err := store.Inspect(st.instName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := start.Register(ctx, inst); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn inst, nil\n}\n\ntype creatorState struct {\n\tinstName string // instance name\n\tyBytes   []byte // yaml bytes\n}\n\nfunc modifyInPlace(st *creatorState, yq string) error {\n\tout, err := yqutil.EvaluateExpression(yq, st.yBytes)\n\tif err != nil {\n\t\treturn err\n\t}\n\tst.yBytes = out\n\treturn nil\n}\n\nfunc chooseNextCreatorState(st *creatorState, yq string) (*creatorState, error) {\n\tfor {\n\t\tif err := modifyInPlace(st, yq); err != nil {\n\t\t\tlogrus.WithError(err).Warn(\"Failed to evaluate yq expression\")\n\t\t\treturn st, err\n\t\t}\n\t\tmessage := fmt.Sprintf(\"Creating an instance %q\", st.instName)\n\t\toptions := []string{\n\t\t\t\"Proceed with the current configuration\",\n\t\t\t\"Open an editor to review or modify the current configuration\",\n\t\t\t\"Choose another template (docker, podman, archlinux, fedora, ...)\",\n\t\t\t\"Exit\",\n\t\t}\n\t\tans, err := uiutil.Select(message, options)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, uiutil.InterruptErr) {\n\t\t\t\tlogrus.Fatal(\"Interrupted by user\")\n\t\t\t}\n\t\t\tlogrus.WithError(err).Warn(\"Failed to open TUI\")\n\t\t\treturn st, nil\n\t\t}\n\t\tswitch ans {\n\t\tcase 0: // \"Proceed with the current configuration\"\n\t\t\treturn st, nil\n\t\tcase 1: // \"Open an editor ...\"\n\t\t\thdr := fmt.Sprintf(\"# Review and modify the following configuration for Lima instance %q.\\n\", st.instName)\n\t\t\tif st.instName == DefaultInstanceName {\n\t\t\t\thdr += \"# - In most cases, you do not need to modify this file.\\n\"\n\t\t\t}\n\t\t\thdr += \"# - To cancel starting Lima, just save this file as an empty file.\\n\"\n\t\t\thdr += \"\\n\"\n\t\t\thdr += editutil.GenerateEditorWarningHeader()\n\t\t\tvar err error\n\t\t\tst.yBytes, err = editutil.OpenEditor(st.yBytes, hdr)\n\t\t\tif err != nil {\n\t\t\t\treturn st, err\n\t\t\t}\n\t\t\tif len(st.yBytes) == 0 {\n\t\t\t\tlogrus.Info(\"Aborting, as requested by saving the file with empty content\")\n\t\t\t\tos.Exit(0)\n\t\t\t\treturn st, errors.New(\"should not reach here\")\n\t\t\t}\n\t\t\treturn st, nil\n\t\tcase 2: // \"Choose another template...\"\n\t\t\ttemplates, err := templatestore.Templates()\n\t\t\tif err != nil {\n\t\t\t\treturn st, err\n\t\t\t}\n\t\t\tmessage := \"Choose a template\"\n\t\t\toptions := make([]string, len(templates))\n\t\t\tfor i := range templates {\n\t\t\t\toptions[i] = templates[i].Name\n\t\t\t}\n\t\t\tansEx, err := uiutil.Select(message, options)\n\t\t\tif err != nil {\n\t\t\t\treturn st, err\n\t\t\t}\n\t\t\tif ansEx > len(templates)-1 {\n...\n", "needle_token_start": 5179, "needle_token_end": 5401, "code_context_ntokens": 15198, "output": ["```go\nfunc deleteInstance(ctx context.Context, inst *store.Instance, force bool) error {\n\tif inst.Protected {\n\t\treturn fmt.Errorf(\"instance is protected to prohibit accidental removal (Hint: use `limactl unprotect`)\")\n\t}\n\tif !force && inst.Status != store.StatusStopped {\n\t\treturn fmt.Errorf(\"expected status %q, got %q\", store.StatusStopped, inst.Status)\n\t}\n\n\tstopInstanceForcibly(inst)\n\n\tif err := stop.Unregister(ctx, inst); err != nil {\n\t\treturn fmt.Errorf(\"failed to unregister %q: %w\", inst.Dir, err)\n\t}\n\n\tif err := os.RemoveAll(inst.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", inst.Dir, err)\n\t}\n\n\treturn nil\n}\n```"]}
{"repo": "lima-vm/lima", "name": "newDiskListCommand", "language": "go", "path": "cmd/limactl/disk.go", "position_ratio": 0.45, "description": "\nFunction Description:\n1. **Purpose**: To display a list of all existing Lima disks managed by the application.\n2. **Input**: Optional command-line flags, specifically a flag to format the output as JSON.\n3. **Output**: A list of existing disks, either in a standard text format or in JSON format, depending on the input flag.\n4. **Procedure**: The command checks for an optional flag to determine the output format. It then retrieves the list of disks from the disk storage directory and formats the output accordingly before displaying it to the user.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/lima-guestagent/daemon_linux.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"net\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/lima-vm/lima/pkg/guestagent\"\n\t\"github.com/lima-vm/lima/pkg/guestagent/api/server\"\n\t\"github.com/lima-vm/lima/pkg/guestagent/serialport\"\n\t\"github.com/mdlayher/vsock\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDaemonCommand() *cobra.Command {\n\tdaemonCommand := &cobra.Command{\n\t\tUse:   \"daemon\",\n\t\tShort: \"run the daemon\",\n\t\tRunE:  daemonAction,\n\t}\n\tdaemonCommand.Flags().Duration(\"tick\", 3*time.Second, \"tick for polling events\")\n\tdaemonCommand.Flags().Int(\"vsock-port\", 0, \"use vsock server instead a UNIX socket\")\n\tdaemonCommand.Flags().String(\"virtio-port\", \"\", \"use virtio server instead a UNIX socket\")\n\treturn daemonCommand\n}\n\nfunc daemonAction(cmd *cobra.Command, _ []string) error {\n\tsocket := \"/run/lima-guestagent.sock\"\n\ttick, err := cmd.Flags().GetDuration(\"tick\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvSockPort, err := cmd.Flags().GetInt(\"vsock-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvirtioPort, err := cmd.Flags().GetString(\"virtio-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif tick == 0 {\n\t\treturn errors.New(\"tick must be specified\")\n\t}\n\tif os.Geteuid() != 0 {\n\t\treturn errors.New(\"must run as the root\")\n\t}\n\tlogrus.Infof(\"event tick: %v\", tick)\n\n\tnewTicker := func() (<-chan time.Time, func()) {\n\t\t// TODO: use an equivalent of `bpftrace -e 'tracepoint:syscalls:sys_*_bind { printf(\"tick\\n\"); }')`,\n\t\t// without depending on `bpftrace` binary.\n\t\t// The agent binary will need CAP_BPF file cap.\n\t\tticker := time.NewTicker(tick)\n\t\treturn ticker.C, ticker.Stop\n\t}\n\n\tagent, err := guestagent.New(newTicker, tick*20)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = os.RemoveAll(socket)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar l net.Listener\n\tif virtioPort != \"\" {\n\t\tqemuL, err := serialport.Listen(\"/dev/virtio-ports/\" + virtioPort)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = qemuL\n\t\tlogrus.Infof(\"serving the guest agent on qemu serial file: %s\", virtioPort)\n\t} else if vSockPort != 0 {\n\t\tvsockL, err := vsock.Listen(uint32(vSockPort), nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = vsockL\n\t\tlogrus.Infof(\"serving the guest agent on vsock port: %d\", vSockPort)\n\t} else {\n\t\tsocketL, err := net.Listen(\"unix\", socket)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := os.Chmod(socket, 0o777); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = socketL\n\t\tlogrus.Infof(\"serving the guest agent on %q\", socket)\n\t}\n\treturn server.StartServer(l, &server.GuestServer{Agent: agent})\n}\n\n// Path: cmd/lima-guestagent/install_systemd_linux.go\npackage main\n\nimport (\n\t_ \"embed\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/textutil\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newInstallSystemdCommand() *cobra.Command {\n\tinstallSystemdCommand := &cobra.Command{\n\t\tUse:   \"install-systemd\",\n\t\tShort: \"install a systemd unit (user)\",\n\t\tRunE:  installSystemdAction,\n\t}\n\tinstallSystemdCommand.Flags().Int(\"vsock-port\", 0, \"use vsock server on specified port\")\n\tinstallSystemdCommand.Flags().String(\"virtio-port\", \"\", \"use virtio server instead a UNIX socket\")\n\treturn installSystemdCommand\n}\n\nfunc installSystemdAction(cmd *cobra.Command, _ []string) error {\n\tvsockPort, err := cmd.Flags().GetInt(\"vsock-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvirtioPort, err := cmd.Flags().GetString(\"virtio-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tunit, err := generateSystemdUnit(vsockPort, virtioPort)\n\tif err != nil {\n\t\treturn err\n\t}\n\tunitPath := \"/etc/systemd/system/lima-guestagent.service\"\n\tif _, err := os.Stat(unitPath); !errors.Is(err, os.ErrNotExist) {\n\t\tlogrus.Infof(\"File %q already exists, overwriting\", unitPath)\n\t} else {\n\t\tunitDir := filepath.Dir(unitPath)\n\t\tif err := os.MkdirAll(unitDir, 0o755); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err := os.WriteFile(unitPath, unit, 0o644); err != nil {\n\t\treturn err\n\t}\n\tlogrus.Infof(\"Written file %q\", unitPath)\n\targs := [][]string{\n\t\t{\"daemon-reload\"},\n\t\t{\"enable\", \"--now\", \"lima-guestagent.service\"},\n\t\t{\"try-restart\", \"lima-guestagent.service\"},\n\t}\n\tfor _, args := range args {\n\t\tcmd := exec.Command(\"systemctl\", append([]string{\"--system\"}, args...)...)\n\t\tcmd.Stdout = os.Stdout\n\t\tcmd.Stderr = os.Stderr\n\t\tlogrus.Infof(\"Executing: %s\", strings.Join(cmd.Args, \" \"))\n\t\tif err := cmd.Run(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tlogrus.Info(\"Done\")\n\treturn nil\n}\n\n//go:embed lima-guestagent.TEMPLATE.service\nvar systemdUnitTemplate string\n\nfunc generateSystemdUnit(vsockPort int, virtioPort string) ([]byte, error) {\n\tselfExeAbs, err := os.Executable()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar args []string\n\tif vsockPort != 0 {\n\t\targs = append(args, fmt.Sprintf(\"--vsock-port %d\", vsockPort))\n\t}\n\tif virtioPort != \"\" {\n\t\targs = append(args, fmt.Sprintf(\"--virtio-port %s\", virtioPort))\n\t}\n\n\tm := map[string]string{\n\t\t\"Binary\": selfExeAbs,\n\t\t\"Args\":   strings.Join(args, \" \"),\n\t}\n\treturn textutil.ExecuteTemplate(systemdUnitTemplate, m)\n}\n\n// Path: cmd/lima-guestagent/main_linux.go\npackage main\n\nimport (\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc main() {\n\tif err := newApp().Execute(); err != nil {\n\t\tlogrus.Fatal(err)\n\t}\n}\n\nfunc newApp() *cobra.Command {\n\trootCmd := &cobra.Command{\n\t\tUse:     \"lima-guestagent\",\n\t\tShort:   \"Do not launch manually\",\n\t\tVersion: strings.TrimPrefix(version.Version, \"v\"),\n\t}\n\trootCmd.PersistentFlags().Bool(\"debug\", false, \"debug mode\")\n\trootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n\t\tdebug, _ := cmd.Flags().GetBool(\"debug\")\n\t\tif debug {\n\t\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t\t}\n\t\treturn nil\n\t}\n\trootCmd.AddCommand(\n\t\tnewDaemonCommand(),\n\t\tnewInstallSystemdCommand(),\n\t)\n\treturn rootCmd\n}\n\n// Path: cmd/limactl/completion.go\npackage main\n\nimport (\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/templatestore\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc bashCompleteInstanceNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tinstances, err := store.Instances()\n\tif err != nil {\n\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t}\n\treturn instances, cobra.ShellCompDirectiveNoFileComp\n}\n\nfunc bashCompleteTemplateNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tvar comp []string\n\tif templates, err := templatestore.Templates(); err == nil {\n\t\tfor _, f := range templates {\n\t\t\tcomp = append(comp, \"template://\"+f.Name)\n\t\t}\n\t}\n\treturn comp, cobra.ShellCompDirectiveDefault\n}\n\nfunc bashCompleteDiskNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tdisks, err := store.Disks()\n\tif err != nil {\n\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t}\n\treturn disks, cobra.ShellCompDirectiveNoFileComp\n}\n\n// Path: cmd/limactl/copy.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\n\t\"github.com/coreos/go-semver/semver\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst copyHelp = `Copy files between host and guest\n\nPrefix guest filenames with the instance name and a colon.\n\nExample: limactl copy default:/etc/os-release .\n`\n\nfunc newCopyCommand() *cobra.Command {\n\tcopyCommand := &cobra.Command{\n\t\tUse:     \"copy SOURCE ... TARGET\",\n\t\tAliases: []string{\"cp\"},\n\t\tShort:   \"Copy files between host and guest\",\n\t\tLong:    copyHelp,\n\t\tArgs:    WrapArgsError(cobra.MinimumNArgs(2)),\n\t\tRunE:    copyAction,\n\t\tGroupID: advancedCommand,\n\t}\n\n\tcopyCommand.Flags().BoolP(\"recursive\", \"r\", false, \"copy directories recursively\")\n\n\treturn copyCommand\n}\n\nfunc copyAction(cmd *cobra.Command, args []string) error {\n\trecursive, err := cmd.Flags().GetBool(\"recursive\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\targ0, err := exec.LookPath(\"scp\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tu, err := osutil.LimaUser(false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tinstDirs := make(map[string]string)\n\tscpFlags := []string{}\n\tscpArgs := []string{}\n\tdebug, err := cmd.Flags().GetBool(\"debug\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif debug {\n\t\tscpFlags = append(scpFlags, \"-v\")\n\t}\n\tif recursive {\n\t\tscpFlags = append(scpFlags, \"-r\")\n\t}\n\tlegacySSH := false\n\tif sshutil.DetectOpenSSHVersion().LessThan(*semver.New(\"8.0.0\")) {\n\t\tlegacySSH = true\n\t}\n\tfor _, arg := range args {\n\t\tpath := strings.Split(arg, \":\")\n\t\tswitch len(path) {\n\t\tcase 1:\n\t\t\tscpArgs = append(scpArgs, arg)\n\t\tcase 2:\n\t\t\tinstName := path[0]\n\t\t\tinst, err := store.Inspect(instName)\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif inst.Status == store.StatusStopped {\n\t\t\t\treturn fmt.Errorf(\"instance %q is stopped, run `limactl start %s` to start the instance\", instName, instName)\n\t\t\t}\n\t\t\tif legacySSH {\n\t\t\t\tscpFlags = append(scpFlags, \"-P\", fmt.Sprintf(\"%d\", inst.SSHLocalPort))\n\t\t\t\tscpArgs = append(scpArgs, fmt.Sprintf(\"%s@127.0.0.1:%s\", u.Username, path[1]))\n\t\t\t} else {\n\t\t\t\tscpArgs = append(scpArgs, fmt.Sprintf(\"scp://%s@127.0.0.1:%d/%s\", u.Username, inst.SSHLocalPort, path[1]))\n\t\t\t}\n\t\t\tinstDirs[instName] = inst.Dir\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"path %q contains multiple colons\", arg)\n\t\t}\n\t}\n\tif legacySSH && len(instDirs) > 1 {\n\t\treturn fmt.Errorf(\"More than one (instance) host is involved in this command, this is only supported for openSSH v8.0 or higher\")\n\t}\n\tscpFlags = append(scpFlags, \"-3\", \"--\")\n\tscpArgs = append(scpFlags, scpArgs...)\n\n\tvar sshOpts []string\n\tif len(instDirs) == 1 {\n\t\t// Only one (instance) host is involved; we can use the instance-specific\n\t\t// arguments such as ControlPath.  This is preferred as we can multiplex\n\t\t// sessions without re-authenticating (MaxSessions permitting).\n\t\tfor _, instDir := range instDirs {\n\t\t\tsshOpts, err = sshutil.SSHOpts(instDir, false, false, false, false)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// Copying among multiple hosts; we can't pass in host-specific options.\n\t\tsshOpts, err = sshutil.CommonOpts(false)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tsshArgs := sshutil.SSHArgsFromOpts(sshOpts)\n\n\tsshCmd := exec.Command(arg0, append(sshArgs, scpArgs...)...)\n\tsshCmd.Stdin = cmd.InOrStdin()\n\tsshCmd.Stdout = cmd.OutOrStdout()\n\tsshCmd.Stderr = cmd.ErrOrStderr()\n\tlogrus.Debugf(\"executing scp (may take a long time)): %+v\", sshCmd.Args)\n\n\t// TODO: use syscall.Exec directly (results in losing tty?)\n\treturn sshCmd.Run()\n}\n\n// Path: cmd/limactl/debug.go\npackage main\n\nimport (\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/lima-vm/lima/pkg/hostagent/dns\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDebugCommand() *cobra.Command {\n\tcmd := &cobra.Command{\n\t\tUse:    \"debug\",\n\t\tShort:  \"Debug utilities\",\n\t\tLong:   \"DO NOT USE! THE COMMAND SYNTAX IS SUBJECT TO CHANGE!\",\n\t\tHidden: true,\n\t}\n\tcmd.AddCommand(newDebugDNSCommand())\n\treturn cmd\n}\n\nfunc newDebugDNSCommand() *cobra.Command {\n\tcmd := &cobra.Command{\n\t\tUse:   \"dns UDPPORT [TCPPORT]\",\n\t\tShort: \"Debug built-in DNS\",\n\t\tLong:  \"DO NOT USE! THE COMMAND SYNTAX IS SUBJECT TO CHANGE!\",\n\t\tArgs:  WrapArgsError(cobra.RangeArgs(1, 2)),\n\t\tRunE:  debugDNSAction,\n\t}\n\tcmd.Flags().BoolP(\"ipv6\", \"6\", false, \"lookup IPv6 addresses too\")\n\treturn cmd\n}\n\nfunc debugDNSAction(cmd *cobra.Command, args []string) error {\n\tipv6, err := cmd.Flags().GetBool(\"ipv6\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tudpLocalPort, err := strconv.Atoi(args[0])\n\tif err != nil {\n\t\treturn err\n\t}\n\ttcpLocalPort := 0\n\tif len(args) > 1 {\n\t\ttcpLocalPort, err = strconv.Atoi(args[1])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tsrvOpts := dns.ServerOptions{\n\t\tUDPPort: udpLocalPort,\n\t\tTCPPort: tcpLocalPort,\n\t\tAddress: \"127.0.0.1\",\n\t\tHandlerOptions: dns.HandlerOptions{\n\t\t\tIPv6:        ipv6,\n\t\t\tStaticHosts: map[string]string{},\n\t\t},\n\t}\n\tsrv, err := dns.Start(srvOpts)\n\tif err != nil {\n\t\treturn err\n\t}\n\tlogrus.Infof(\"Started srv %+v (UDP %d, TCP %d)\", srv, udpLocalPort, tcpLocalPort)\n\tfor {\n\t\ttime.Sleep(time.Hour)\n\t}\n}\n\n// Path: cmd/limactl/disk.go\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"os\"\n\t\"text/tabwriter\"\n\n\t\"github.com/docker/go-units\"\n\t\"github.com/lima-vm/lima/pkg/qemu\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDiskCommand() *cobra.Command {\n\tdiskCommand := &cobra.Command{\n\t\tUse:   \"disk\",\n\t\tShort: \"Lima disk management\",\n\t\tExample: `  Create a disk:\n  $ limactl disk create DISK --size SIZE [--format qcow2]\n\n  List existing disks:\n  $ limactl disk ls\n\n  Delete a disk:\n  $ limactl disk delete DISK\n  \n  Resize a disk:\n  $ limactl disk resize DISK --size SIZE`,\n\t\tSilenceUsage:  true,\n\t\tSilenceErrors: true,\n\t\tGroupID:       advancedCommand,\n\t}\n\tdiskCommand.AddCommand(\n\t\tnewDiskCreateCommand(),\n\t\tnewDiskListCommand(),\n\t\tnewDiskDeleteCommand(),\n\t\tnewDiskUnlockCommand(),\n\t\tnewDiskResizeCommand(),\n\t)\n\treturn diskCommand\n}\n\nfunc newDiskCreateCommand() *cobra.Command {\n\tdiskCreateCommand := &cobra.Command{\n\t\tUse: \"create DISK\",\n\t\tExample: `\nTo create a new disk:\n$ limactl disk create DISK --size SIZE [--format qcow2]\n`,\n\t\tShort: \"Create a Lima disk\",\n\t\tArgs:  WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:  diskCreateAction,\n\t}\n\tdiskCreateCommand.Flags().String(\"size\", \"\", \"configure the disk size\")\n\t_ = diskCreateCommand.MarkFlagRequired(\"size\")\n\tdiskCreateCommand.Flags().String(\"format\", \"qcow2\", \"specify the disk format\")\n\treturn diskCreateCommand\n}\n\nfunc diskCreateAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tswitch format {\n\tcase \"qcow2\", \"raw\":\n\tdefault:\n\t\treturn fmt.Errorf(`disk format %q not supported, use \"qcow2\" or \"raw\" instead`, format)\n\t}\n\n\t// only exactly one arg is allowed\n\tname := args[0]\n\n\tdiskDir, err := store.DiskDir(name)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif _, err := os.Stat(diskDir); !errors.Is(err, fs.ErrNotExist) {\n\t\treturn fmt.Errorf(\"disk %q already exists (%q)\", name, diskDir)\n\t}\n\n\tlogrus.Infof(\"Creating %s disk %q with size %s\", format, name, units.BytesSize(float64(diskSize)))\n\n\tif err := os.MkdirAll(diskDir, 0o700); err != nil {\n\t\treturn err\n\t}\n\n\tif err := qemu.CreateDataDisk(diskDir, format, int(diskSize)); err != nil {\n\t\trerr := os.RemoveAll(diskDir)\n\t\tif rerr != nil {\n\t\t\terr = errors.Join(err, fmt.Errorf(\"failed to remove a directory %q: %w\", diskDir, rerr))\n\t\t}\n\t\treturn fmt.Errorf(\"Failed to create %s disk in %q: %w\", format, diskDir, err)\n\t}\n\n\treturn nil\n}\n\n\nfunc newDiskListCommand() *cobra.Command {\n\tdiskListCommand := &cobra.Command{\n\t\tUse: \"list\",\n\t\tExample: `\nTo list existing disks:\n$ limactl disk list\n`,\n\t\tShort:   \"List existing Lima disks\",\n\t\tAliases: []string{\"ls\"},\n\t\tArgs:    WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:    diskListAction,\n\t}\n\tdiskListCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\treturn diskListCommand\n}\n\nfunc diskMatches(diskName string, disks []string) []string {\n\tmatches := []string{}\n\tfor _, disk := range disks {\n\t\tif disk == diskName {\n\t\t\tmatches = append(matches, disk)\n\t\t}\n\t}\n\treturn matches\n}\n\nfunc diskListAction(cmd *cobra.Command, args []string) error {\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tallDisks, err := store.Disks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdisks := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := diskMatches(arg, allDisks)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tdisks = append(disks, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No disk matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tdisks = allDisks\n\t}\n\n\tif jsonFormat {\n\t\tfor _, diskName := range disks {\n\t\t\tdisk, err := store.InspectDisk(diskName)\n\t\t\tif err != nil {\n\t\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tj, err := json.Marshal(disk)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), string(j))\n\t\t}\n\t\treturn nil\n\t}\n\n\tw := tabwriter.NewWriter(cmd.OutOrStdout(), 4, 8, 4, ' ', 0)\n\tfmt.Fprintln(w, \"NAME\\tSIZE\\tFORMAT\\tDIR\\tIN-USE-BY\")\n\n\tif len(disks) == 0 {\n\t\tlogrus.Warn(\"No disk found. Run `limactl disk create DISK --size SIZE` to create a disk.\")\n\t}\n\n\tfor _, diskName := range disks {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Fprintf(w, \"%s\\t%s\\t%s\\t%s\\t%s\\n\", disk.Name, units.BytesSize(float64(disk.Size)), disk.Format, disk.Dir, disk.Instance)\n\t}\n\n\treturn w.Flush()\n}\n\nfunc newDiskDeleteCommand() *cobra.Command {\n\tdiskDeleteCommand := &cobra.Command{\n\t\tUse: \"delete DISK [DISK, ...]\",\n\t\tExample: `\nTo delete a disk:\n$ limactl disk delete DISK\n\nTo delete multiple disks:\n$ limactl disk delete DISK1 DISK2 ...\n`,\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskDeleteAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskDeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"force delete\")\n\treturn diskDeleteCommand\n}\n\nfunc diskDeleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tinstNames, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar instances []*store.Instance\n\tfor _, instName := range instNames {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tinstances = append(instances, inst)\n\t}\n\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tif !force {\n\t\t\tif disk.Instance != \"\" {\n\t\t\t\treturn fmt.Errorf(\"cannot delete disk %q in use by instance %q\", disk.Name, disk.Instance)\n\t\t\t}\n\t\t\tvar refInstances []string\n\t\t\tfor _, inst := range instances {\n\t\t\t\tif len(inst.AdditionalDisks) > 0 {\n\t\t\t\t\tfor _, d := range inst.AdditionalDisks {\n\t\t\t\t\t\tif d.Name == diskName {\n\t\t\t\t\t\t\trefInstances = append(refInstances, inst.Name)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(refInstances) > 0 {\n\t\t\t\tlogrus.Warnf(\"Skipping deleting disk %q, disk is referenced by one or more non-running instances: %q\",\n\t\t\t\t\tdiskName, refInstances)\n\t\t\t\tlogrus.Warnf(\"To delete anyway, run %q\", forceDeleteCommand(diskName))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif err := deleteDisk(disk); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\nfunc deleteDisk(disk *store.Disk) error {\n\tif err := os.RemoveAll(disk.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", disk.Dir, err)\n\t}\n\treturn nil\n}\n\nfunc forceDeleteCommand(diskName string) string {\n\treturn fmt.Sprintf(\"limactl disk delete --force %v\", diskName)\n}\n\nfunc newDiskUnlockCommand() *cobra.Command {\n\tdiskUnlockCommand := &cobra.Command{\n\t\tUse: \"unlock DISK [DISK, ...]\",\n\t\tExample: `\nEmergency recovery! If an instance is force stopped, it may leave a disk locked while not actually using it.\n\nTo unlock a disk:\n$ limactl disk unlock DISK\n\nTo unlock multiple disks:\n$ limactl disk unlock DISK1 DISK2 ...\n`,\n\t\tShort:             \"Unlock one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskUnlockAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\treturn diskUnlockCommand\n}\n\nfunc diskUnlockAction(_ *cobra.Command, args []string) error {\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif disk.Instance == \"\" {\n\t\t\tlogrus.Warnf(\"Ignoring unlocked disk %q\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\t// if store.Inspect throws an error, the instance does not exist, and it is safe to unlock\n\t\tinst, err := store.Inspect(disk.Instance)\n\t\tif err == nil {\n\t\t\tif len(inst.Errors) > 0 {\n\t\t\t\tlogrus.Warnf(\"Cannot unlock disk %q, attached instance %q has errors: %+v\",\n\t\t\t\t\tdiskName, disk.Instance, inst.Errors)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif inst.Status == store.StatusRunning {\n\t\t\t\tlogrus.Warnf(\"Cannot unlock disk %q used by running instance %q\", diskName, disk.Instance)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to unlock disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Unlocked disk %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\nfunc newDiskResizeCommand() *cobra.Command {\n\tdiskResizeCommand := &cobra.Command{\n\t\tUse: \"resize DISK\",\n\t\tExample: `\nResize a disk:\n$ limactl disk resize DISK --size SIZE`,\n\t\tShort:             \"Resize existing Lima disk\",\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:              diskResizeAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskResizeCommand.Flags().String(\"size\", \"\", \"Disk size\")\n\t_ = diskResizeCommand.MarkFlagRequired(\"size\")\n\treturn diskResizeCommand\n}\n\nfunc diskResizeAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskName := args[0]\n\tdisk, err := store.InspectDisk(diskName)\n\tif err != nil {\n\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"disk %q does not exists\", diskName)\n\t\t}\n\t\treturn err\n\t}\n\n\t// Shrinking can cause a disk failure\n\tif diskSize < disk.Size {\n\t\treturn fmt.Errorf(\"specified size %q is less than the current disk size %q. Disk shrinking is currently unavailable\", units.BytesSize(float64(diskSize)), units.BytesSize(float64(disk.Size)))\n\t}\n\n\tif disk.Instance != \"\" {\n\t\tinst, err := store.Inspect(disk.Instance)\n\t\tif err == nil {\n\t\t\tif inst.Status == store.StatusRunning {\n\t\t\t\treturn fmt.Errorf(\"cannot resize disk %q used by running instance %q. Please stop the VM instance\", diskName, disk.Instance)\n\t\t\t}\n\t\t}\n\t}\n\tif err := qemu.ResizeDataDisk(disk.Dir, disk.Format, int(diskSize)); err != nil {\n\t\treturn fmt.Errorf(\"failed to resize disk %q: %w\", diskName, err)\n\t}\n\tlogrus.Infof(\"Resized disk %q (%q)\", diskName, disk.Dir)\n\treturn nil\n}\n\nfunc diskBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteDiskNames(cmd)\n}\n\n// Path: cmd/limactl/stop.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\thostagentevents \"github.com/lima-vm/lima/pkg/hostagent/events\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newStopCommand() *cobra.Command {\n\tstopCmd := &cobra.Command{\n\t\tUse:               \"stop INSTANCE\",\n\t\tShort:             \"Stop an instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              stopAction,\n\t\tValidArgsFunction: stopBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tstopCmd.Flags().BoolP(\"force\", \"f\", false, \"force stop the instance\")\n\treturn stopCmd\n}\n\nfunc stopAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif force {\n\t\tstopInstanceForcibly(inst)\n\t} else {\n\t\terr = stopInstanceGracefully(inst)\n\t}\n\t// TODO: should we also reconcile networks if graceful stop returned an error?\n\tif err == nil {\n\t\terr = networks.Reconcile(cmd.Context(), \"\")\n\t}\n\treturn err\n}\n\nfunc stopInstanceGracefully(inst *store.Instance) error {\n\tif inst.Status != store.StatusRunning {\n\t\treturn fmt.Errorf(\"expected status %q, got %q (maybe use `limactl stop -f`?)\", store.StatusRunning, inst.Status)\n\t}\n\n\tbegin := time.Now() // used for logrus propagation\n\tlogrus.Infof(\"Sending SIGINT to hostagent process %d\", inst.HostAgentPID)\n\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigInt); err != nil {\n\t\tlogrus.Error(err)\n\t}\n\n\tlogrus.Info(\"Waiting for the host agent and the driver processes to shut down\")\n\treturn waitForHostAgentTermination(context.TODO(), inst, begin)\n}\n\nfunc waitForHostAgentTermination(ctx context.Context, inst *store.Instance, begin time.Time) error {\n\tctx2, cancel := context.WithTimeout(ctx, 3*time.Minute)\n\tdefer cancel()\n\n\tvar receivedExitingEvent bool\n\tonEvent := func(ev hostagentevents.Event) bool {\n\t\tif len(ev.Status.Errors) > 0 {\n\t\t\tlogrus.Errorf(\"%+v\", ev.Status.Errors)\n\t\t}\n\t\tif ev.Status.Exiting {\n\t\t\treceivedExitingEvent = true\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\n\thaStdoutPath := filepath.Join(inst.Dir, filenames.HostAgentStdoutLog)\n\thaStderrPath := filepath.Join(inst.Dir, filenames.HostAgentStderrLog)\n\n\tif err := hostagentevents.Watch(ctx2, haStdoutPath, haStderrPath, begin, onEvent); err != nil {\n\t\treturn err\n\t}\n\n\tif !receivedExitingEvent {\n\t\treturn errors.New(\"did not receive an event with the \\\"exiting\\\" status\")\n\t}\n\n\treturn nil\n}\n\nfunc stopInstanceForcibly(inst *store.Instance) {\n\tif inst.DriverPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the %s driver process %d\", inst.VMType, inst.DriverPID)\n\t\tif err := osutil.SysKill(inst.DriverPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Infof(\"The %s driver process seems already stopped\", inst.VMType)\n\t}\n\n\tfor _, d := range inst.AdditionalDisks {\n\t\tdiskName := d.Name\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.Warnf(\"Disk %q does not exist\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\tlogrus.Warnf(\"Failed to unlock disk %q. To use, run `limactl disk unlock %v`\", diskName, diskName)\n\t\t}\n\t}\n\n\tif inst.HostAgentPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the host agent process %d\", inst.HostAgentPID)\n\t\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"The host agent process seems already stopped\")\n\t}\n\n\tsuffixesToBeRemoved := []string{\".pid\", \".sock\", \".tmp\"}\n\tlogrus.Infof(\"Removing %s under %q\", inst.Dir, strings.ReplaceAll(strings.Join(suffixesToBeRemoved, \" \"), \".\", \"*.\"))\n\tfi, err := os.ReadDir(inst.Dir)\n\tif err != nil {\n\t\tlogrus.Error(err)\n\t\treturn\n\t}\n\tfor _, f := range fi {\n\t\tpath := filepath.Join(inst.Dir, f.Name())\n\t\tfor _, suffix := range suffixesToBeRemoved {\n\t\t\tif strings.HasSuffix(path, suffix) {\n\t\t\t\tlogrus.Infof(\"Removing %q\", path)\n\t\t\t\tif err := os.Remove(path); err != nil {\n\t\t\t\t\tlogrus.Error(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc stopBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/delete.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/stop\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDeleteCommand() *cobra.Command {\n\tdeleteCommand := &cobra.Command{\n\t\tUse:               \"delete INSTANCE [INSTANCE, ...]\",\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete an instance of Lima.\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              deleteAction,\n\t\tValidArgsFunction: deleteBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\tdeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"forcibly kill the processes\")\n\treturn deleteCommand\n}\n\nfunc deleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent instance %q\", instName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif err := deleteInstance(cmd.Context(), inst, force); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete instance %q: %w\", instName, err)\n\t\t}\n\t\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\t\tif err != nil && !errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.WithError(err).Warnf(\"The autostart file for instance %q does not exist\", instName)\n\t\t\t} else if deleted {\n\t\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t\t}\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", instName, inst.Dir)\n\t}\n\treturn networks.Reconcile(cmd.Context(), \"\")\n}\n\nfunc deleteInstance(ctx context.Context, inst *store.Instance, force bool) error {\n\tif inst.Protected {\n\t\treturn fmt.Errorf(\"instance is protected to prohibit accidental removal (Hint: use `limactl unprotect`)\")\n\t}\n\tif !force && inst.Status != store.StatusStopped {\n\t\treturn fmt.Errorf(\"expected status %q, got %q\", store.StatusStopped, inst.Status)\n\t}\n\n\tstopInstanceForcibly(inst)\n\n\tif err := stop.Unregister(ctx, inst); err != nil {\n\t\treturn fmt.Errorf(\"failed to unregister %q: %w\", inst.Dir, err)\n\t}\n\n\tif err := os.RemoveAll(inst.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", inst.Dir, err)\n\t}\n\n\treturn nil\n}\n\nfunc deleteBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/editflags/editflags.go\npackage editflags\n\nimport (\n\t\"fmt\"\n\t\"math/bits\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pbnjay/memory\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n\tflag \"github.com/spf13/pflag\"\n)\n\n// RegisterEdit registers flags related to in-place YAML modification, for `limactl edit`.\nfunc RegisterEdit(cmd *cobra.Command) {\n\tregisterEdit(cmd, \"\")\n}\n\nfunc registerEdit(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\n\tflags.Int(\"cpus\", 0, commentPrefix+\"number of CPUs\") // Similar to colima's --cpu, but the flag name is slightly different (cpu vs cpus)\n\t_ = cmd.RegisterFlagCompletionFunc(\"cpus\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeCPUs(runtime.NumCPU()) {\n\t\t\tres = append(res, strconv.Itoa(f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.IPSlice(\"dns\", nil, commentPrefix+\"specify custom DNS (disable host resolver)\") // colima-compatible\n\n\tflags.Float32(\"memory\", 0, commentPrefix+\"memory in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeMemoryGiB(memory.TotalMemory()) {\n\t\t\tres = append(res, fmt.Sprintf(\"%.1f\", f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.StringSlice(\"mount\", nil, commentPrefix+\"directories to mount, suffix ':w' for writable (Do not specify directories that overlap with the existing mounts)\") // colima-compatible\n\n\tflags.String(\"mount-type\", \"\", commentPrefix+\"mount type (reverse-sshfs, 9p, virtiofs)\") // Similar to colima's --mount-type=(sshfs|9p|virtiofs), but \"reverse-sshfs\" is Lima is called \"sshfs\" in colima\n\t_ = cmd.RegisterFlagCompletionFunc(\"mount-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"reverse-sshfs\", \"9p\", \"virtiofs\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"mount-writable\", false, commentPrefix+\"make all mounts writable\")\n\tflags.Bool(\"mount-inotify\", false, commentPrefix+\"enable inotify for mounts\")\n\n\tflags.StringSlice(\"network\", nil, commentPrefix+\"additional networks, e.g., \\\"vzNAT\\\" or \\\"lima:shared\\\" to assign vmnet IP\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"network\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\t// TODO: retrieve the lima:* network list from networks.yaml\n\t\treturn []string{\"lima:shared\", \"lima:bridged\", \"lima:host\", \"lima:user-v2\", \"vzNAT\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"rosetta\", false, commentPrefix+\"enable Rosetta (for vz instances)\")\n\n\tflags.String(\"set\", \"\", commentPrefix+\"modify the template inplace, using yq syntax\")\n\n\t// negative performance impact: https://gitlab.com/qemu-project/qemu/-/issues/334\n\tflags.Bool(\"video\", false, commentPrefix+\"enable video output (has negative performance impact for QEMU)\")\n}\n\n// RegisterCreate registers flags related to in-place YAML modification, for `limactl create`.\nfunc RegisterCreate(cmd *cobra.Command, commentPrefix string) {\n\tregisterEdit(cmd, commentPrefix)\n\tflags := cmd.Flags()\n\n\tflags.String(\"arch\", \"\", commentPrefix+\"machine architecture (x86_64, aarch64, riscv64)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"arch\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"x86_64\", \"aarch64\", \"riscv64\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"containerd\", \"\", commentPrefix+\"containerd mode (user, system, user+system, none)\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"user\", \"system\", \"user+system\", \"none\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Float32(\"disk\", 0, commentPrefix+\"disk size in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"10\", \"30\", \"50\", \"100\", \"200\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"vm-type\", \"\", commentPrefix+\"virtual machine type (qemu, vz)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"qemu\", \"vz\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"plain\", false, commentPrefix+\"plain mode. Disable mounts, port forwarding, containerd, etc.\")\n}\n\nfunc defaultExprFunc(expr string) func(v *flag.Flag) (string, error) {\n\treturn func(v *flag.Flag) (string, error) {\n\t\treturn fmt.Sprintf(expr, v.Value), nil\n\t}\n}\n\n// YQExpressions returns YQ expressions.\nfunc YQExpressions(flags *flag.FlagSet, newInstance bool) ([]string, error) {\n\ttype def struct {\n\t\tflagName                 string\n\t\texprFunc                 func(*flag.Flag) (string, error)\n\t\tonlyValidForNewInstances bool\n\t\texperimental             bool\n\t}\n\td := defaultExprFunc\n\tdefs := []def{\n\t\t{\"cpus\", d(\".cpus = %s\"), false, false},\n\t\t{\n\t\t\t\"dns\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tipSlice, err := flags.GetIPSlice(\"dns\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.dns += [`\n\t\t\t\tfor i, ip := range ipSlice {\n\t\t\t\t\texpr += fmt.Sprintf(\"%q\", ip)\n\t\t\t\t\tif i < len(ipSlice)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .dns |= unique | .hostResolver.enabled=false`\n\t\t\t\tlogrus.Warnf(\"Disabling HostResolver, as custom DNS addresses (%v) are specified\", ipSlice)\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"memory\", d(\".memory = \\\"%sGiB\\\"\"), false, false},\n\t\t{\n\t\t\t\"mount\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"mount\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.mounts += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\twritable := strings.HasSuffix(s, \":w\")\n\t\t\t\t\tloc := strings.TrimSuffix(s, \":w\")\n\t\t\t\t\texpr += fmt.Sprintf(`{\"location\": %q, \"writable\": %v}`, loc, writable)\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .mounts |= unique_by(.location)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"mount-type\", d(\".mountType = %q\"), false, false},\n\t\t{\"mount-inotify\", d(\".mountInotify = %s\"), false, true},\n\t\t{\"mount-writable\", d(\".mounts[].writable = %s\"), false, false},\n\t\t{\n\t\t\t\"network\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"network\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.networks += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\t// CLI syntax is still experimental (YAML syntax is out of experimental)\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase s == \"vzNAT\":\n\t\t\t\t\t\texpr += `{\"vzNAT\": true}`\n\t\t\t\t\tcase strings.HasPrefix(s, \"lima:\"):\n\t\t\t\t\t\tnetwork := strings.TrimPrefix(s, \"lima:\")\n\t\t\t\t\t\texpr += fmt.Sprintf(`{\"lima\": %q}`, network)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\treturn \"\", fmt.Errorf(\"network name must be \\\"vzNAT\\\" or \\\"lima:*\\\", got %q\", s)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(ss)-1 {\n...\n", "needle_token_start": 5933, "needle_token_end": 6076, "code_context_ntokens": 14391, "output": ["```go\nfunc newDiskListCommand() *cobra.Command {\n\tdiskListCommand := &cobra.Command{\n\t\tUse: \"list\",\n\t\tExample: `\nTo list existing disks:\n$ limactl disk list\n`,\n\t\tShort:   \"List existing Lima disks\",\n\t\tAliases: []string{\"ls\"},\n\t\tArgs:    WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:    diskListAction,\n\t}\n\tdiskListCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\treturn diskListCommand\n}\n\nfunc diskListAction(cmd *cobra.Command, args []string) error {\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tallDisks, err := store.Disks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdisks := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := diskMatches(arg, allDisks)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tdisks = append(disks, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No disk matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tdisks = allDisks\n\t}\n\n\tif jsonFormat {\n\t\tfor _, diskName := range disks {\n\t\t\tdisk, err := store.InspectDisk(diskName)\n\t\t\tif err != nil {\n\t\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tj, err := json.Marshal(disk)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), string(j))\n\t\t}\n\t\treturn nil\n\t}\n\n\tw := tabwriter.NewWriter(cmd.OutOrStdout(), 4, 8, 4, ' ', 0)\n\tfmt.Fprintln(w, \"NAME\\tSIZE\\tFORMAT\\tDIR\\tIN-USE-BY\")\n\n\tif len(disks) == 0 {\n\t\tlogrus.Warn(\"No disk found. Run `limactl disk create DISK --size SIZE` to create a disk.\")\n\t}\n\n\tfor _, diskName := range disks {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Fprintf(w, \"%s\\t%s\\t%s\\t%s\\t%s\\n\", disk.Name, units.BytesSize(float64(disk.Size)), disk.Format, disk.Dir, disk.Instance)\n\t}\n\n\treturn w.Flush()\n}\n```"]}
{"repo": "lima-vm/lima", "name": "snapshotDeleteAction", "language": "go", "path": "cmd/limactl/snapshot.go", "position_ratio": 0.55, "description": "\nFunction Description:\n1. **Purpose**: To remove a specified snapshot of a virtual machine instance.\n2. **Input**: The name of the instance and the tag of the snapshot to be deleted.\n3. **Output**: The function returns an error if the deletion fails, otherwise it completes without any output, indicating a successful deletion.\n4. **Procedure**: \n   - The function first retrieves the instance details using the instance name.\n   - It checks if the snapshot tag is provided; if not, it returns an error.\n   - It then proceeds to delete the snapshot associated with the given tag and instance.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/limactl/list.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/cheggaaa/pb/v3/termutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc fieldNames() []string {\n\tnames := []string{}\n\tvar data store.FormatData\n\tt := reflect.TypeOf(data)\n\tfor i := 0; i < t.NumField(); i++ {\n\t\tf := t.Field(i)\n\t\tif f.Anonymous {\n\t\t\tfor j := 0; j < f.Type.NumField(); j++ {\n\t\t\t\tnames = append(names, f.Type.Field(j).Name)\n\t\t\t}\n\t\t} else {\n\t\t\tnames = append(names, t.Field(i).Name)\n\t\t}\n\t}\n\treturn names\n}\n\nfunc newListCommand() *cobra.Command {\n\tlistCommand := &cobra.Command{\n\t\tUse:     \"list [flags] [INSTANCE]...\",\n\t\tAliases: []string{\"ls\"},\n\t\tShort:   \"List instances of Lima.\",\n\t\tLong: `List instances of Lima.\n\nThe output can be presented in one of several formats, using the --format <format> flag.\n\n  --format json  - output in json format\n  --format yaml  - output in yaml format\n  --format table - output in table format\n  --format '{{ <go template> }}' - if the format begins and ends with '{{ }}', then it is used as a go template.\n` + store.FormatHelp + `\nThe following legacy flags continue to function:\n  --json - equal to '--format json'`,\n\t\tArgs:              WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:              listAction,\n\t\tValidArgsFunction: listBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tlistCommand.Flags().StringP(\"format\", \"f\", \"table\", \"output format, one of: json, yaml, table, go-template\")\n\tlistCommand.Flags().Bool(\"list-fields\", false, \"List fields available for format\")\n\tlistCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\tlistCommand.Flags().BoolP(\"quiet\", \"q\", false, \"Only show names\")\n\tlistCommand.Flags().Bool(\"all-fields\", false, \"Show all fields\")\n\n\treturn listCommand\n}\n\nfunc instanceMatches(arg string, instances []string) []string {\n\tmatches := []string{}\n\tfor _, instance := range instances {\n\t\tif instance == arg {\n\t\t\tmatches = append(matches, instance)\n\t\t}\n\t}\n\treturn matches\n}\n\nfunc listAction(cmd *cobra.Command, args []string) error {\n\tquiet, err := cmd.Flags().GetBool(\"quiet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tlistFields, err := cmd.Flags().GetBool(\"list-fields\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif jsonFormat {\n\t\tformat = \"json\"\n\t}\n\n\t// conflicts\n\tif jsonFormat && cmd.Flags().Changed(\"format\") {\n\t\treturn errors.New(\"option --json conflicts with option --format\")\n\t}\n\tif listFields && cmd.Flags().Changed(\"format\") {\n\t\treturn errors.New(\"option --list-fields conflicts with option --format\")\n\t}\n\n\tif quiet && format != \"table\" {\n\t\treturn errors.New(\"option --quiet can only be used with '--format table'\")\n\t}\n\n\tif listFields {\n\t\tnames := fieldNames()\n\t\tsort.Strings(names)\n\t\tfmt.Println(strings.Join(names, \"\\n\"))\n\t\treturn nil\n\t}\n\n\tallinstances, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(allinstances) == 0 {\n\t\tlogrus.Warn(\"No instance found. Run `limactl create` to create an instance.\")\n\t\treturn nil\n\t}\n\n\tinstanceNames := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := instanceMatches(arg, allinstances)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tinstanceNames = append(instanceNames, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No instance matching %v found.\", arg)\n\t\t\t}\n...\n// Path: cmd/limactl/main.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store/dirnames\"\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst (\n\tDefaultInstanceName = \"default\"\n\tbasicCommand        = \"basic\"\n\tadvancedCommand     = \"advanced\"\n)\n\nfunc main() {\n\tif err := newApp().Execute(); err != nil {\n\t\thandleExitCoder(err)\n\t\tlogrus.Fatal(err)\n\t}\n}\n\nfunc newApp() *cobra.Command {\n\ttemplatesDir := \"$PREFIX/share/lima/templates\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n\t\ttemplatesDir = filepath.Join(prefixDir, \"share/lima/templates\")\n\t}\n\n\trootCmd := &cobra.Command{\n\t\tUse:     \"limactl\",\n\t\tShort:   \"Lima: Linux virtual machines\",\n\t\tVersion: strings.TrimPrefix(version.Version, \"v\"),\n\t\tExample: fmt.Sprintf(`  Start the default instance:\n  $ limactl start\n\n  Open a shell:\n  $ lima\n\n  Run a container:\n  $ lima nerdctl run -d --name nginx -p 8080:80 nginx:alpine\n\n  Stop the default instance:\n  $ limactl stop\n\n  See also template YAMLs: %s`, templatesDir),\n\t\tSilenceUsage:      true,\n\t\tSilenceErrors:     true,\n\t\tDisableAutoGenTag: true,\n\t}\n\trootCmd.PersistentFlags().String(\"log-level\", \"\", \"Set the logging level [trace, debug, info, warn, error]\")\n\trootCmd.PersistentFlags().Bool(\"debug\", false, \"debug mode\")\n\t// TODO: \"survey\" does not support using cygwin terminal on windows yet\n\trootCmd.PersistentFlags().Bool(\"tty\", isatty.IsTerminal(os.Stdout.Fd()), \"Enable TUI interactions such as opening an editor. Defaults to true when stdout is a terminal. Set to false for automation.\")\n\trootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n\t\tl, _ := cmd.Flags().GetString(\"log-level\")\n\t\tif l != \"\" {\n\t\t\tlvl, err := logrus.ParseLevel(l)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tlogrus.SetLevel(lvl)\n\t\t}\n\t\tdebug, _ := cmd.Flags().GetBool(\"debug\")\n\t\tif debug {\n\t\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t\t}\n\n\t\tif osutil.IsBeingRosettaTranslated() {\n\t\t\t// running under rosetta would provide inappropriate runtime.GOARCH info, see: https://github.com/lima-vm/lima/issues/543\n\t\t\treturn errors.New(\"limactl is running under rosetta, please reinstall lima with native arch\")\n\t\t}\n\n\t\tif runtime.GOOS == \"windows\" && isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t\tformatter := new(logrus.TextFormatter)\n\t\t\t// the default setting does not recognize cygwin on windows\n\t\t\tformatter.ForceColors = true\n\t\t\tlogrus.StandardLogger().SetFormatter(formatter)\n\t\t}\n\t\tif os.Geteuid() == 0 && cmd.Name() != \"generate-doc\" {\n\t\t\treturn errors.New(\"must not run as the root\")\n\t\t}\n\t\t// Make sure either $HOME or $LIMA_HOME is defined, so we don't need\n\t\t// to check for errors later\n\t\tif _, err := dirnames.LimaDir(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\trootCmd.AddGroup(&cobra.Group{ID: \"basic\", Title: \"Basic Commands:\"})\n\trootCmd.AddGroup(&cobra.Group{ID: \"advanced\", Title: \"Advanced Commands:\"})\n\trootCmd.AddCommand(\n\t\tnewCreateCommand(),\n\t\tnewStartCommand(),\n\t\tnewStopCommand(),\n\t\tnewShellCommand(),\n\t\tnewCopyCommand(),\n\t\tnewListCommand(),\n\t\tnewDeleteCommand(),\n\t\tnewValidateCommand(),\n\t\tnewSudoersCommand(),\n\t\tnewPruneCommand(),\n\t\tnewHostagentCommand(),\n\t\tnewInfoCommand(),\n\t\tnewShowSSHCommand(),\n\t\tnewDebugCommand(),\n\t\tnewEditCommand(),\n\t\tnewFactoryResetCommand(),\n\t\tnewDiskCommand(),\n\t\tnewUsernetCommand(),\n\t\tnewGenDocCommand(),\n\t\tnewSnapshotCommand(),\n\t\tnewProtectCommand(),\n\t\tnewUnprotectCommand(),\n\t)\n\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\trootCmd.AddCommand(startAtLoginCommand())\n\t}\n\n\treturn rootCmd\n}\n\ntype ExitCoder interface {\n\terror\n\tExitCode() int\n}\n\nfunc handleExitCoder(err error) {\n\tif err == nil {\n\t\treturn\n\t}\n\n\tif exitErr, ok := err.(ExitCoder); ok {\n\t\tos.Exit(exitErr.ExitCode())\n\t\treturn\n\t}\n}\n\n// WrapArgsError annotates cobra args error with some context, so the error message is more user-friendly\nfunc WrapArgsError(argFn cobra.PositionalArgs) cobra.PositionalArgs {\n\treturn func(cmd *cobra.Command, args []string) error {\n\t\terr := argFn(cmd, args)\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\treturn fmt.Errorf(\"%q %s.\\nSee '%s --help'.\\n\\nUsage:  %s\\n\\n%s\",\n\t\t\tcmd.CommandPath(), err.Error(),\n\t\t\tcmd.CommandPath(),\n\t\t\tcmd.UseLine(), cmd.Short,\n\t\t)\n\t}\n}\n\n// Path: cmd/limactl/protect.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newProtectCommand() *cobra.Command {\n\tprotectCommand := &cobra.Command{\n\t\tUse:   \"protect INSTANCE [INSTANCE, ...]\",\n\t\tShort: \"Protect an instance to prohibit accidental removal\",\n\t\tLong: `Protect an instance to prohibit accidental removal via the 'limactl delete' command.\nThe instance is not being protected against removal via '/bin/rm', Finder, etc.`,\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              protectAction,\n\t\tValidArgsFunction: protectBashComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn protectCommand\n}\n\nfunc protectAction(_ *cobra.Command, args []string) error {\n\tvar errs []error\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to inspect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tif inst.Protected {\n\t\t\tlogrus.Warnf(\"Instance %q is already protected. Skipping.\", instName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := inst.Protect(); err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to protect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tlogrus.Infof(\"Protected %q\", instName)\n\t}\n\treturn errors.Join(errs...)\n}\n\nfunc protectBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/prune.go\npackage main\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newPruneCommand() *cobra.Command {\n\tpruneCommand := &cobra.Command{\n\t\tUse:               \"prune\",\n\t\tShort:             \"Prune garbage objects\",\n\t\tArgs:              WrapArgsError(cobra.NoArgs),\n\t\tRunE:              pruneAction,\n\t\tValidArgsFunction: cobra.NoFileCompletions,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn pruneCommand\n}\n\nfunc pruneAction(_ *cobra.Command, _ []string) error {\n\tucd, err := os.UserCacheDir()\n\tif err != nil {\n\t\treturn err\n\t}\n\tcacheDir := filepath.Join(ucd, \"lima\")\n\tlogrus.Infof(\"Pruning %q\", cacheDir)\n\treturn os.RemoveAll(cacheDir)\n}\n\n// Path: cmd/limactl/shell.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/alessio/shellescape\"\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/mattn/go-shellwords\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\n// Environment variable that allows configuring the command (alias) to execute\n// in place of the 'ssh' executable.\nconst envShellSSH = \"SSH\"\n\nconst shellHelp = `Execute shell in Lima\n\nlima command is provided as an alias for limactl shell $LIMA_INSTANCE. $LIMA_INSTANCE defaults to \"` + DefaultInstanceName + `\".\n\nBy default, the first 'ssh' executable found in the host's PATH is used to connect to the Lima instance.\nA custom ssh alias can be used instead by setting the $` + envShellSSH + ` environment variable.\n\nHint: try --debug to show the detailed logs, if it seems hanging (mostly due to some SSH issue).\n`\n\nfunc newShellCommand() *cobra.Command {\n\tshellCmd := &cobra.Command{\n\t\tUse:               \"shell [flags] INSTANCE [COMMAND...]\",\n\t\tShort:             \"Execute shell in Lima\",\n\t\tLong:              shellHelp,\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              shellAction,\n\t\tValidArgsFunction: shellBashComplete,\n\t\tSilenceErrors:     true,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tshellCmd.Flags().SetInterspersed(false)\n\n\tshellCmd.Flags().String(\"shell\", \"\", \"shell interpreter, e.g. /bin/bash\")\n\tshellCmd.Flags().String(\"workdir\", \"\", \"working directory\")\n\treturn shellCmd\n}\n\nfunc shellAction(cmd *cobra.Command, args []string) error {\n\t// simulate the behavior of double dash\n\tnewArg := []string{}\n\tif len(args) >= 2 && args[1] == \"--\" {\n\t\tnewArg = append(newArg, args[:1]...)\n\t\tnewArg = append(newArg, args[2:]...)\n\t\targs = newArg\n\t}\n\tinstName := args[0]\n\n\tif len(args) >= 2 {\n\t\tswitch args[1] {\n\t\tcase \"create\", \"start\", \"delete\", \"shell\":\n\t\t\t// `lima start` (alias of `limactl $LIMA_INSTANCE start`) is probably a typo of `limactl start`\n\t\t\tlogrus.Warnf(\"Perhaps you meant `limactl %s`?\", strings.Join(args[1:], \" \"))\n\t\t}\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t}\n\t\treturn err\n\t}\n\tif inst.Status == store.StatusStopped {\n\t\treturn fmt.Errorf(\"instance %q is stopped, run `limactl start %s` to start the instance\", instName, instName)\n\t}\n\ty, err := inst.LoadYAML()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// When workDir is explicitly set, the shell MUST have workDir as the cwd, or exit with an error.\n\t//\n\t// changeDirCmd := \"cd workDir || exit 1\"                  if workDir != \"\"\n\t//              := \"cd hostCurrentDir || cd hostHomeDir\"   if workDir == \"\"\n\tvar changeDirCmd string\n\tworkDir, err := cmd.Flags().GetString(\"workdir\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif workDir != \"\" {\n\t\tchangeDirCmd = fmt.Sprintf(\"cd %s || exit 1\", shellescape.Quote(workDir))\n\t\t// FIXME: check whether y.Mounts contains the home, not just len > 0\n\t} else if len(y.Mounts) > 0 {\n\t\thostCurrentDir, err := os.Getwd()\n\t\tif err == nil {\n\t\t\tchangeDirCmd = fmt.Sprintf(\"cd %s\", shellescape.Quote(hostCurrentDir))\n\t\t} else {\n\t\t\tchangeDirCmd = \"false\"\n\t\t\tlogrus.WithError(err).Warn(\"failed to get the current directory\")\n\t\t}\n\t\thostHomeDir, err := os.UserHomeDir()\n\t\tif err == nil {\n\t\t\tchangeDirCmd = fmt.Sprintf(\"%s || cd %s\", changeDirCmd, shellescape.Quote(hostHomeDir))\n\t\t} else {\n\t\t\tlogrus.WithError(err).Warn(\"failed to get the home directory\")\n\t\t}\n\t} else {\n\t\tlogrus.Debug(\"the host home does not seem mounted, so the guest shell will have a different cwd\")\n\t}\n\n\tif changeDirCmd == \"\" {\n\t\tchangeDirCmd = \"false\"\n\t}\n\tlogrus.Debugf(\"changeDirCmd=%q\", changeDirCmd)\n\n\tshell, err := cmd.Flags().GetString(\"shell\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif shell == \"\" {\n\t\tshell = `\"$SHELL\"`\n\t} else {\n\t\tshell = shellescape.Quote(shell)\n\t}\n\tscript := fmt.Sprintf(\"%s ; exec %s --login\", changeDirCmd, shell)\n\tif len(args) > 1 {\n\t\tquotedArgs := make([]string, len(args[1:]))\n\t\tparsingEnv := true\n\t\tfor i, arg := range args[1:] {\n\t\t\tif parsingEnv && isEnv(arg) {\n\t\t\t\tquotedArgs[i] = quoteEnv(arg)\n\t\t\t} else {\n\t\t\t\tparsingEnv = false\n\t\t\t\tquotedArgs[i] = shellescape.Quote(arg)\n\t\t\t}\n\t\t}\n\t\tscript += fmt.Sprintf(\n\t\t\t\" -c %s\",\n\t\t\tshellescape.Quote(strings.Join(quotedArgs, \" \")),\n\t\t)\n\t}\n\n\tvar arg0 string\n\tvar arg0Args []string\n\n\tif sshShell := os.Getenv(envShellSSH); sshShell != \"\" {\n\t\tsshShellFields, err := shellwords.Parse(sshShell)\n\t\tswitch {\n\t\tcase err != nil:\n\t\t\tlogrus.WithError(err).Warnf(\"Failed to split %s variable into shell tokens. \"+\n\t\t\t\t\"Falling back to 'ssh' command\", envShellSSH)\n\t\tcase len(sshShellFields) > 0:\n\t\t\targ0 = sshShellFields[0]\n\t\t\tif len(sshShellFields) > 1 {\n\t\t\t\targ0Args = sshShellFields[1:]\n\t\t\t}\n\t\t}\n\t}\n\n\tif arg0 == \"\" {\n\t\targ0, err = exec.LookPath(\"ssh\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tsshOpts, err := sshutil.SSHOpts(inst.Dir, *y.SSH.LoadDotSSHPubKeys, *y.SSH.ForwardAgent, *y.SSH.ForwardX11, *y.SSH.ForwardX11Trusted)\n\tif err != nil {\n\t\treturn err\n\t}\n\tsshArgs := sshutil.SSHArgsFromOpts(sshOpts)\n\tif isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t// required for showing the shell prompt: https://stackoverflow.com/a/626574\n\t\tsshArgs = append(sshArgs, \"-t\")\n\t}\n\tif _, present := os.LookupEnv(\"COLORTERM\"); present {\n\t\t// SendEnv config is cumulative, with already existing options in ssh_config\n\t\tsshArgs = append(sshArgs, \"-o\", \"SendEnv=COLORTERM\")\n\t}\n\tsshArgs = append(sshArgs, []string{\n\t\t\"-q\",\n\t\t\"-p\", strconv.Itoa(inst.SSHLocalPort),\n\t\tinst.SSHAddress,\n\t\t\"--\",\n\t\tscript,\n\t}...)\n\tsshCmd := exec.Command(arg0, append(arg0Args, sshArgs...)...)\n\tsshCmd.Stdin = os.Stdin\n\tsshCmd.Stdout = os.Stdout\n\tsshCmd.Stderr = os.Stderr\n\tlogrus.Debugf(\"executing ssh (may take a long)): %+v\", sshCmd.Args)\n\n\t// TODO: use syscall.Exec directly (results in losing tty?)\n\treturn sshCmd.Run()\n}\n\nfunc shellBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\nfunc isEnv(arg string) bool {\n\treturn len(strings.Split(arg, \"=\")) > 1\n}\n\nfunc quoteEnv(arg string) string {\n\tenv := strings.SplitN(arg, \"=\", 2)\n\tenv[1] = shellescape.Quote(env[1])\n\treturn strings.Join(env, \"=\")\n}\n\n// Path: cmd/limactl/show_ssh.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/dirnames\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst showSSHExample = `\n  \"cmd\" format (default): Full ssh command line.\n    $ limactl show-ssh --format=cmd default\n    ssh -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022 lima-default\n\n  \"args\" format: Similar to the cmd format but omits \"ssh\" and the destination address\n    $ limactl show-ssh --format=args default\n    -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022\n\n  \"options\" format: ssh option key value pairs\n    $ limactl show-ssh --format=options default\n    IdentityFile=\"/Users/example/.lima/_config/user\"\n    User=example\n    Hostname=127.0.0.1\n    Port=60022\n\n  \"config\" format: ~/.ssh/config format\n    $ limactl show-ssh --format=config default\n    Host lima-default\n      IdentityFile \"/Users/example/.lima/_config/user \"\n      User example\n      Hostname 127.0.0.1\n      Port 60022\n\n  To show the config file path:\n    $ limactl ls --format='{{.SSHConfigFile}}' default\n    /Users/example/.lima/default/ssh.config\n`\n\nfunc newShowSSHCommand() *cobra.Command {\n\tlimaHome := \"~/\" + dirnames.DotLima\n\tif s, err := dirnames.LimaDir(); err == nil {\n\t\tlimaHome = s\n\t}\n\tshellCmd := &cobra.Command{\n\t\tUse:   \"show-ssh [flags] INSTANCE\",\n\t\tShort: \"Show the ssh command line (DEPRECATED; use `ssh -F` instead)\",\n\t\tLong: fmt.Sprintf(`Show the ssh command line (DEPRECATED)\n\nWARNING: 'limactl show-ssh' is deprecated.\nInstead, use 'ssh -F %s/default/ssh.config lima-default' .\n`, limaHome),\n\t\tExample:           showSSHExample,\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:              showSSHAction,\n\t\tValidArgsFunction: showSSHBashComplete,\n\t\tSilenceErrors:     true,\n\t\tGroupID:           advancedCommand,\n\t}\n\n\tshellCmd.Flags().StringP(\"format\", \"f\", sshutil.FormatCmd, \"Format: \"+strings.Join(sshutil.Formats, \", \"))\n\t_ = shellCmd.RegisterFlagCompletionFunc(\"format\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn sshutil.Formats, cobra.ShellCompDirectiveNoFileComp\n\t})\n\treturn shellCmd\n}\n\nfunc showSSHAction(cmd *cobra.Command, args []string) error {\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tinstName := args[0]\n\tw := cmd.OutOrStdout()\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t}\n\t\treturn err\n\t}\n\tlogrus.Warnf(\"`limactl show-ssh` is deprecated. Instead, use `ssh -F %s lima-%s`.\",\n\t\tfilepath.Join(inst.Dir, filenames.SSHConfig), inst.Name)\n\ty, err := inst.LoadYAML()\n\tif err != nil {\n\t\treturn err\n\t}\n\topts, err := sshutil.SSHOpts(inst.Dir, *y.SSH.LoadDotSSHPubKeys, *y.SSH.ForwardAgent, *y.SSH.ForwardX11, *y.SSH.ForwardX11Trusted)\n\tif err != nil {\n\t\treturn err\n\t}\n\topts = append(opts, \"Hostname=127.0.0.1\")\n\topts = append(opts, fmt.Sprintf(\"Port=%d\", inst.SSHLocalPort))\n\treturn sshutil.Format(w, instName, format, opts)\n}\n\nfunc showSSHBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/snapshot.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/snapshot\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSnapshotCommand() *cobra.Command {\n\tsnapshotCmd := &cobra.Command{\n\t\tUse:   \"snapshot\",\n\t\tShort: \"Manage instance snapshots\",\n\t\tPersistentPreRun: func(cmd *cobra.Command, args []string) {\n\t\t\tlogrus.Warn(\"`limactl snapshot` is experimental\")\n\t\t},\n\t\tGroupID: advancedCommand,\n\t}\n\tsnapshotCmd.AddCommand(newSnapshotApplyCommand())\n\tsnapshotCmd.AddCommand(newSnapshotCreateCommand())\n\tsnapshotCmd.AddCommand(newSnapshotDeleteCommand())\n\tsnapshotCmd.AddCommand(newSnapshotListCommand())\n\n\treturn snapshotCmd\n}\n\nfunc newSnapshotCreateCommand() *cobra.Command {\n\tcreateCmd := &cobra.Command{\n\t\tUse:               \"create INSTANCE\",\n\t\tAliases:           []string{\"save\"},\n\t\tShort:             \"Create (save) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotCreateAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tcreateCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn createCmd\n}\n\nfunc snapshotCreateAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Save(ctx, inst, tag)\n}\n\nfunc newSnapshotDeleteCommand() *cobra.Command {\n\tdeleteCmd := &cobra.Command{\n\t\tUse:               \"delete INSTANCE\",\n\t\tAliases:           []string{\"del\"},\n\t\tShort:             \"Delete (del) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotDeleteAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tdeleteCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn deleteCmd\n}\n\n\nfunc snapshotDeleteAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Del(ctx, inst, tag)\n}\n\nfunc newSnapshotApplyCommand() *cobra.Command {\n\tapplyCmd := &cobra.Command{\n\t\tUse:               \"apply INSTANCE\",\n\t\tAliases:           []string{\"load\"},\n\t\tShort:             \"Apply (load) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotApplyAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tapplyCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn applyCmd\n}\n\nfunc snapshotApplyAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Load(ctx, inst, tag)\n}\n\nfunc newSnapshotListCommand() *cobra.Command {\n\tlistCmd := &cobra.Command{\n\t\tUse:               \"list INSTANCE\",\n\t\tAliases:           []string{\"ls\"},\n\t\tShort:             \"List existing snapshots\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotListAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tlistCmd.Flags().BoolP(\"quiet\", \"q\", false, \"Only show tags\")\n\n\treturn listCmd\n}\n\nfunc snapshotListAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tquiet, err := cmd.Flags().GetBool(\"quiet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tctx := cmd.Context()\n\tout, err := snapshot.List(ctx, inst)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif quiet {\n\t\tfor i, line := range strings.Split(out, \"\\n\") {\n\t\t\t// \"ID\", \"TAG\", \"VM SIZE\", \"DATE\", \"VM CLOCK\", \"ICOUNT\"\n\t\t\tfields := strings.Fields(line)\n\t\t\tif i == 0 && len(fields) > 1 && fields[1] != \"TAG\" {\n\t\t\t\t// make sure that output matches the expected\n\t\t\t\treturn fmt.Errorf(\"unknown header: %s\", line)\n\t\t\t}\n\t\t\tif i == 0 || line == \"\" {\n\t\t\t\t// skip header and empty line after using split\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttag := fields[1]\n\t\t\tfmt.Printf(\"%s\\n\", tag)\n\t\t}\n\t\treturn nil\n\t}\n\tfmt.Print(out)\n\treturn nil\n}\n\nfunc snapshotBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/start-at-login.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc startAtLoginCommand() *cobra.Command {\n\tstartAtLoginCommand := &cobra.Command{\n\t\tUse:               \"start-at-login INSTANCE\",\n\t\tShort:             \"Register/Unregister an autostart file for the instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              startAtLoginAction,\n\t\tValidArgsFunction: startAtLoginComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\n\tstartAtLoginCommand.Flags().Bool(\n\t\t\"enabled\", true,\n\t\t\"Automatically start the instance when the user logs in\",\n\t)\n\n\treturn startAtLoginCommand\n}\n\nfunc startAtLoginAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\tlogrus.Infof(\"Instance %q not found\", instName)\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\n\tflags := cmd.Flags()\n\tstartAtLogin, err := flags.GetBool(\"enabled\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif startAtLogin {\n\t\tif err := autostart.CreateStartAtLoginEntry(runtime.GOOS, inst.Name, inst.Dir); err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"Can't create an autostart file for instance %q\", inst.Name)\n\t\t} else {\n\t\t\tlogrus.Infof(\"The autostart file %q has been created or updated\", autostart.GetFilePath(runtime.GOOS, inst.Name))\n\t\t}\n\t} else {\n\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"The autostart file %q could not be deleted\", instName)\n\t\t} else if deleted {\n\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc startAtLoginComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/sudoers.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/networks\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSudoersCommand() *cobra.Command {\n\tnetworksMD := \"$PREFIX/share/doc/lima/docs/network.md\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n\t\tnetworksMD = filepath.Join(prefixDir, \"share/doc/lima/docs/network.md\")\n\t}\n\tsudoersCommand := &cobra.Command{\n\t\tUse: \"sudoers [--check [SUDOERSFILE-TO-CHECK]]\",\n\t\tExample: `\nTo generate the /etc/sudoers.d/lima file:\n$ limactl sudoers | sudo tee /etc/sudoers.d/lima\n\nTo validate the existing /etc/sudoers.d/lima file:\n$ limactl sudoers --check /etc/sudoers.d/lima\n`,\n\t\tShort: \"Generate the content of the /etc/sudoers.d/lima file\",\n\t\tLong: fmt.Sprintf(`Generate the content of the /etc/sudoers.d/lima file for enabling vmnet.framework support.\nThe content is written to stdout, NOT to the file.\nThis command must not run as the root.\nSee %s for the usage.`, networksMD),\n\t\tArgs:    WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:    sudoersAction,\n\t\tGroupID: advancedCommand,\n\t}\n\tconfigFile, _ := networks.ConfigFile()\n\tsudoersCommand.Flags().Bool(\"check\", false,\n\t\tfmt.Sprintf(\"check that the sudoers file is up-to-date with %q\", configFile))\n\treturn sudoersCommand\n}\n\nfunc sudoersAction(cmd *cobra.Command, args []string) error {\n\tif runtime.GOOS != \"darwin\" {\n\t\treturn errors.New(\"sudoers command is only supported on macOS right now\")\n\t}\n\tcheck, err := cmd.Flags().GetBool(\"check\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif check {\n\t\treturn verifySudoAccess(args)\n\t}\n\tswitch len(args) {\n\tcase 0:\n\t\t// NOP\n\tcase 1:\n\t\treturn fmt.Errorf(\"the file argument can be specified only for --check mode\")\n\tdefault:\n\t\treturn fmt.Errorf(\"unexpected arguments %v\", args)\n\t}\n\tsudoers, err := networks.Sudoers()\n\tif err != nil {\n\t\treturn err\n\t}\n\tfmt.Print(sudoers)\n\treturn nil\n}\n\nfunc verifySudoAccess(args []string) error {\n\tconfig, err := networks.Config()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif err := config.Validate(); err != nil {\n\t\treturn err\n\t}\n\tvar file string\n\tswitch len(args) {\n\tcase 0:\n\t\tfile = config.Paths.Sudoers\n\t\tif file == \"\" {\n\t\t\tconfigFile, _ := networks.ConfigFile()\n\t\t\treturn fmt.Errorf(\"no sudoers file defined in %q\", configFile)\n\t\t}\n\tcase 1:\n\t\tfile = args[0]\n\tdefault:\n\t\treturn errors.New(\"can check only a single sudoers file\")\n\t}\n\tif err := config.VerifySudoAccess(file); err != nil {\n\t\treturn err\n\t}\n\tfmt.Printf(\"%q is up-to-date (or sudo doesn't require a password)\\n\", file)\n\treturn nil\n}\n\n// Path: cmd/limactl/unprotect.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newUnprotectCommand() *cobra.Command {\n\tunprotectCommand := &cobra.Command{\n\t\tUse:               \"unprotect INSTANCE [INSTANCE, ...]\",\n\t\tShort:             \"Unprotect an instance\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              unprotectAction,\n\t\tValidArgsFunction: unprotectBashComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn unprotectCommand\n}\n\nfunc unprotectAction(_ *cobra.Command, args []string) error {\n\tvar errs []error\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to inspect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tif !inst.Protected {\n\t\t\tlogrus.Warnf(\"Instance %q isn't protected. Skipping.\", instName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := inst.Unprotect(); err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to unprotect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tlogrus.Infof(\"Unprotected %q\", instName)\n\t}\n\treturn errors.Join(errs...)\n}\n\nfunc unprotectBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/usernet.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\n\t\"github.com/lima-vm/lima/pkg/networks/usernet\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newUsernetCommand() *cobra.Command {\n\thostagentCommand := &cobra.Command{\n\t\tUse:    \"usernet\",\n\t\tShort:  \"run usernet\",\n\t\tArgs:   cobra.ExactArgs(0),\n\t\tRunE:   usernetAction,\n\t\tHidden: true,\n\t}\n\thostagentCommand.Flags().StringP(\"pidfile\", \"p\", \"\", \"write pid to file\")\n\thostagentCommand.Flags().StringP(\"endpoint\", \"e\", \"\", \"exposes usernet api(s) on this endpoint\")\n\thostagentCommand.Flags().String(\"listen-qemu\", \"\", \"listen for qemu connections\")\n\thostagentCommand.Flags().String(\"listen\", \"\", \"listen on a Unix socket and receive Bess-compatible FDs as SCM_RIGHTS messages\")\n\thostagentCommand.Flags().String(\"subnet\", \"192.168.5.0/24\", \"sets subnet value for the usernet network\")\n\thostagentCommand.Flags().Int(\"mtu\", 1500, \"mtu\")\n\thostagentCommand.Flags().StringToString(\"leases\", nil, \"pass default static leases for startup. Eg: '192.168.104.1=52:55:55:b3:bc:d9,192.168.104.2=5a:94:ef:e4:0c:df' \")\n\treturn hostagentCommand\n}\n\nfunc usernetAction(cmd *cobra.Command, _ []string) error {\n\tpidfile, err := cmd.Flags().GetString(\"pidfile\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif pidfile != \"\" {\n\t\tif _, err := os.Stat(pidfile); !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"pidfile %q already exists\", pidfile)\n\t\t}\n\t\tif err := os.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())+\"\\n\"), 0o644); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer os.RemoveAll(pidfile)\n\t}\n\tendpoint, err := cmd.Flags().GetString(\"endpoint\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tqemuSocket, err := cmd.Flags().GetString(\"listen-qemu\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfdSocket, err := cmd.Flags().GetString(\"listen\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tsubnet, err := cmd.Flags().GetString(\"subnet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tleases, err := cmd.Flags().GetStringToString(\"leases\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tmtu, err := cmd.Flags().GetInt(\"mtu\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tos.RemoveAll(endpoint)\n\tos.RemoveAll(qemuSocket)\n\tos.RemoveAll(fdSocket)\n\n\treturn usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{\n\t\tMTU:           mtu,\n\t\tEndpoint:      endpoint,\n\t\tQemuSocket:    qemuSocket,\n\t\tFdSocket:      fdSocket,\n\t\tSubnet:        subnet,\n\t\tDefaultLeases: leases,\n\t})\n}\n\n// Path: cmd/limactl/validate.go\npackage main\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/cmd/limactl/guessarg\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/spf13/cobra\"\n\n\t\"github.com/sirupsen/logrus\"\n)\n\nfunc newValidateCommand() *cobra.Command {\n\tvalidateCommand := &cobra.Command{\n\t\tUse:     \"validate FILE.yaml [FILE.yaml, ...]\",\n\t\tShort:   \"Validate YAML files\",\n\t\tArgs:    WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:    validateAction,\n\t\tGroupID: advancedCommand,\n\t}\n\treturn validateCommand\n}\n\nfunc validateAction(_ *cobra.Command, args []string) error {\n\tfor _, f := range args {\n\t\t_, err := store.LoadYAMLByFilePath(f)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"failed to load YAML file %q: %w\", f, err)\n\t\t}\n\t\tif _, err := guessarg.InstNameFromYAMLPath(f); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tlogrus.Infof(\"%q: OK\", f)\n\t}\n\n\treturn nil\n}\n\n", "needle_token_start": 8290, "needle_token_end": 8426, "code_context_ntokens": 12820, "output": ["```go\nfunc snapshotDeleteAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Del(ctx, inst, tag)\n}\n```"]}
{"repo": "lima-vm/lima", "name": "SeemsHTTPURL", "language": "go", "path": "cmd/limactl/guessarg/guessarg.go", "position_ratio": 0.65, "description": "\nFunction Description:\n1. **Purpose**: To determine if a given string represents a URL that uses the HTTP or HTTPS protocol.\n2. **Input**: A single string representing a potential URL.\n3. **Output**: A boolean value indicating whether the input string is a URL using the HTTP or HTTPS protocol.\n4. **Procedure**: The function attempts to parse the input string as a URL. If parsing fails, it returns false. If parsing is successful, it checks if the URL's scheme is either \"http\" or \"https\". It returns true if either condition is met, otherwise it returns false.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/limactl/disk.go\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"os\"\n\t\"text/tabwriter\"\n\n\t\"github.com/docker/go-units\"\n\t\"github.com/lima-vm/lima/pkg/qemu\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDiskCommand() *cobra.Command {\n\tdiskCommand := &cobra.Command{\n\t\tUse:   \"disk\",\n\t\tShort: \"Lima disk management\",\n\t\tExample: `  Create a disk:\n  $ limactl disk create DISK --size SIZE [--format qcow2]\n\n  List existing disks:\n  $ limactl disk ls\n\n  Delete a disk:\n  $ limactl disk delete DISK\n  \n  Resize a disk:\n  $ limactl disk resize DISK --size SIZE`,\n\t\tSilenceUsage:  true,\n\t\tSilenceErrors: true,\n\t\tGroupID:       advancedCommand,\n\t}\n\tdiskCommand.AddCommand(\n\t\tnewDiskCreateCommand(),\n\t\tnewDiskListCommand(),\n\t\tnewDiskDeleteCommand(),\n\t\tnewDiskUnlockCommand(),\n\t\tnewDiskResizeCommand(),\n\t)\n\treturn diskCommand\n}\n\nfunc newDiskCreateCommand() *cobra.Command {\n\tdiskCreateCommand := &cobra.Command{\n\t\tUse: \"create DISK\",\n\t\tExample: `\nTo create a new disk:\n$ limactl disk create DISK --size SIZE [--format qcow2]\n`,\n\t\tShort: \"Create a Lima disk\",\n\t\tArgs:  WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:  diskCreateAction,\n\t}\n\tdiskCreateCommand.Flags().String(\"size\", \"\", \"configure the disk size\")\n\t_ = diskCreateCommand.MarkFlagRequired(\"size\")\n\tdiskCreateCommand.Flags().String(\"format\", \"qcow2\", \"specify the disk format\")\n\treturn diskCreateCommand\n}\n\nfunc diskCreateAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tswitch format {\n\tcase \"qcow2\", \"raw\":\n\tdefault:\n\t\treturn fmt.Errorf(`disk format %q not supported, use \"qcow2\" or \"raw\" instead`, format)\n\t}\n\n\t// only exactly one arg is allowed\n\tname := args[0]\n\n\tdiskDir, err := store.DiskDir(name)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif _, err := os.Stat(diskDir); !errors.Is(err, fs.ErrNotExist) {\n\t\treturn fmt.Errorf(\"disk %q already exists (%q)\", name, diskDir)\n\t}\n\n\tlogrus.Infof(\"Creating %s disk %q with size %s\", format, name, units.BytesSize(float64(diskSize)))\n\n\tif err := os.MkdirAll(diskDir, 0o700); err != nil {\n\t\treturn err\n\t}\n\n\tif err := qemu.CreateDataDisk(diskDir, format, int(diskSize)); err != nil {\n\t\trerr := os.RemoveAll(diskDir)\n\t\tif rerr != nil {\n\t\t\terr = errors.Join(err, fmt.Errorf(\"failed to remove a directory %q: %w\", diskDir, rerr))\n\t\t}\n\t\treturn fmt.Errorf(\"Failed to create %s disk in %q: %w\", format, diskDir, err)\n\t}\n\n\treturn nil\n}\n\nfunc newDiskListCommand() *cobra.Command {\n\tdiskListCommand := &cobra.Command{\n\t\tUse: \"list\",\n\t\tExample: `\nTo list existing disks:\n$ limactl disk list\n`,\n\t\tShort:   \"List existing Lima disks\",\n\t\tAliases: []string{\"ls\"},\n\t\tArgs:    WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:    diskListAction,\n\t}\n\tdiskListCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\treturn diskListCommand\n}\n\nfunc diskMatches(diskName string, disks []string) []string {\n\tmatches := []string{}\n\tfor _, disk := range disks {\n\t\tif disk == diskName {\n\t\t\tmatches = append(matches, disk)\n\t\t}\n\t}\n\treturn matches\n}\n\nfunc diskListAction(cmd *cobra.Command, args []string) error {\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tallDisks, err := store.Disks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdisks := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := diskMatches(arg, allDisks)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tdisks = append(disks, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No disk matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tdisks = allDisks\n\t}\n\n\tif jsonFormat {\n\t\tfor _, diskName := range disks {\n\t\t\tdisk, err := store.InspectDisk(diskName)\n\t\t\tif err != nil {\n\t\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tj, err := json.Marshal(disk)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), string(j))\n\t\t}\n\t\treturn nil\n\t}\n\n\tw := tabwriter.NewWriter(cmd.OutOrStdout(), 4, 8, 4, ' ', 0)\n\tfmt.Fprintln(w, \"NAME\\tSIZE\\tFORMAT\\tDIR\\tIN-USE-BY\")\n\n\tif len(disks) == 0 {\n\t\tlogrus.Warn(\"No disk found. Run `limactl disk create DISK --size SIZE` to create a disk.\")\n\t}\n\n\tfor _, diskName := range disks {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Fprintf(w, \"%s\\t%s\\t%s\\t%s\\t%s\\n\", disk.Name, units.BytesSize(float64(disk.Size)), disk.Format, disk.Dir, disk.Instance)\n\t}\n\n\treturn w.Flush()\n}\n\nfunc newDiskDeleteCommand() *cobra.Command {\n\tdiskDeleteCommand := &cobra.Command{\n\t\tUse: \"delete DISK [DISK, ...]\",\n\t\tExample: `\nTo delete a disk:\n$ limactl disk delete DISK\n\nTo delete multiple disks:\n$ limactl disk delete DISK1 DISK2 ...\n`,\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskDeleteAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskDeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"force delete\")\n\treturn diskDeleteCommand\n}\n\nfunc diskDeleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tinstNames, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar instances []*store.Instance\n\tfor _, instName := range instNames {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tinstances = append(instances, inst)\n\t}\n\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tif !force {\n\t\t\tif disk.Instance != \"\" {\n\t\t\t\treturn fmt.Errorf(\"cannot delete disk %q in use by instance %q\", disk.Name, disk.Instance)\n\t\t\t}\n\t\t\tvar refInstances []string\n\t\t\tfor _, inst := range instances {\n\t\t\t\tif len(inst.AdditionalDisks) > 0 {\n\t\t\t\t\tfor _, d := range inst.AdditionalDisks {\n\t\t\t\t\t\tif d.Name == diskName {\n\t\t\t\t\t\t\trefInstances = append(refInstances, inst.Name)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(refInstances) > 0 {\n\t\t\t\tlogrus.Warnf(\"Skipping deleting disk %q, disk is referenced by one or more non-running instances: %q\",\n\t\t\t\t\tdiskName, refInstances)\n\t\t\t\tlogrus.Warnf(\"To delete anyway, run %q\", forceDeleteCommand(diskName))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif err := deleteDisk(disk); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\nfunc deleteDisk(disk *store.Disk) error {\n\tif err := os.RemoveAll(disk.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", disk.Dir, err)\n\t}\n\treturn nil\n}\n\nfunc forceDeleteCommand(diskName string) string {\n\treturn fmt.Sprintf(\"limactl disk delete --force %v\", diskName)\n}\n\nfunc newDiskUnlockCommand() *cobra.Command {\n\tdiskUnlockCommand := &cobra.Command{\n\t\tUse: \"unlock DISK [DISK, ...]\",\n\t\tExample: `\nEmergency recovery! If an instance is force stopped, it may leave a disk locked while not actually using it.\n\nTo unlock a disk:\n$ limactl disk unlock DISK\n\nTo unlock multiple disks:\n$ limactl disk unlock DISK1 DISK2 ...\n`,\n\t\tShort:             \"Unlock one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskUnlockAction,\n...\n// Path: cmd/limactl/stop.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\thostagentevents \"github.com/lima-vm/lima/pkg/hostagent/events\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newStopCommand() *cobra.Command {\n\tstopCmd := &cobra.Command{\n\t\tUse:               \"stop INSTANCE\",\n\t\tShort:             \"Stop an instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              stopAction,\n\t\tValidArgsFunction: stopBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tstopCmd.Flags().BoolP(\"force\", \"f\", false, \"force stop the instance\")\n\treturn stopCmd\n}\n\nfunc stopAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif force {\n\t\tstopInstanceForcibly(inst)\n\t} else {\n\t\terr = stopInstanceGracefully(inst)\n\t}\n\t// TODO: should we also reconcile networks if graceful stop returned an error?\n\tif err == nil {\n\t\terr = networks.Reconcile(cmd.Context(), \"\")\n\t}\n\treturn err\n}\n\nfunc stopInstanceGracefully(inst *store.Instance) error {\n\tif inst.Status != store.StatusRunning {\n\t\treturn fmt.Errorf(\"expected status %q, got %q (maybe use `limactl stop -f`?)\", store.StatusRunning, inst.Status)\n\t}\n\n\tbegin := time.Now() // used for logrus propagation\n\tlogrus.Infof(\"Sending SIGINT to hostagent process %d\", inst.HostAgentPID)\n\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigInt); err != nil {\n\t\tlogrus.Error(err)\n\t}\n\n\tlogrus.Info(\"Waiting for the host agent and the driver processes to shut down\")\n\treturn waitForHostAgentTermination(context.TODO(), inst, begin)\n}\n\nfunc waitForHostAgentTermination(ctx context.Context, inst *store.Instance, begin time.Time) error {\n\tctx2, cancel := context.WithTimeout(ctx, 3*time.Minute)\n\tdefer cancel()\n\n\tvar receivedExitingEvent bool\n\tonEvent := func(ev hostagentevents.Event) bool {\n\t\tif len(ev.Status.Errors) > 0 {\n\t\t\tlogrus.Errorf(\"%+v\", ev.Status.Errors)\n\t\t}\n\t\tif ev.Status.Exiting {\n\t\t\treceivedExitingEvent = true\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\n\thaStdoutPath := filepath.Join(inst.Dir, filenames.HostAgentStdoutLog)\n\thaStderrPath := filepath.Join(inst.Dir, filenames.HostAgentStderrLog)\n\n\tif err := hostagentevents.Watch(ctx2, haStdoutPath, haStderrPath, begin, onEvent); err != nil {\n\t\treturn err\n\t}\n\n\tif !receivedExitingEvent {\n\t\treturn errors.New(\"did not receive an event with the \\\"exiting\\\" status\")\n\t}\n\n\treturn nil\n}\n\nfunc stopInstanceForcibly(inst *store.Instance) {\n\tif inst.DriverPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the %s driver process %d\", inst.VMType, inst.DriverPID)\n\t\tif err := osutil.SysKill(inst.DriverPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Infof(\"The %s driver process seems already stopped\", inst.VMType)\n\t}\n\n\tfor _, d := range inst.AdditionalDisks {\n\t\tdiskName := d.Name\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.Warnf(\"Disk %q does not exist\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\tlogrus.Warnf(\"Failed to unlock disk %q. To use, run `limactl disk unlock %v`\", diskName, diskName)\n\t\t}\n\t}\n\n\tif inst.HostAgentPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the host agent process %d\", inst.HostAgentPID)\n\t\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"The host agent process seems already stopped\")\n\t}\n\n\tsuffixesToBeRemoved := []string{\".pid\", \".sock\", \".tmp\"}\n\tlogrus.Infof(\"Removing %s under %q\", inst.Dir, strings.ReplaceAll(strings.Join(suffixesToBeRemoved, \" \"), \".\", \"*.\"))\n\tfi, err := os.ReadDir(inst.Dir)\n\tif err != nil {\n\t\tlogrus.Error(err)\n\t\treturn\n\t}\n\tfor _, f := range fi {\n\t\tpath := filepath.Join(inst.Dir, f.Name())\n\t\tfor _, suffix := range suffixesToBeRemoved {\n\t\t\tif strings.HasSuffix(path, suffix) {\n\t\t\t\tlogrus.Infof(\"Removing %q\", path)\n\t\t\t\tif err := os.Remove(path); err != nil {\n\t\t\t\t\tlogrus.Error(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc stopBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/delete.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/stop\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDeleteCommand() *cobra.Command {\n\tdeleteCommand := &cobra.Command{\n\t\tUse:               \"delete INSTANCE [INSTANCE, ...]\",\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete an instance of Lima.\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              deleteAction,\n\t\tValidArgsFunction: deleteBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\tdeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"forcibly kill the processes\")\n\treturn deleteCommand\n}\n\nfunc deleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent instance %q\", instName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif err := deleteInstance(cmd.Context(), inst, force); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete instance %q: %w\", instName, err)\n\t\t}\n\t\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\t\tif err != nil && !errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.WithError(err).Warnf(\"The autostart file for instance %q does not exist\", instName)\n\t\t\t} else if deleted {\n\t\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t\t}\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", instName, inst.Dir)\n\t}\n\treturn networks.Reconcile(cmd.Context(), \"\")\n}\n\nfunc deleteInstance(ctx context.Context, inst *store.Instance, force bool) error {\n\tif inst.Protected {\n\t\treturn fmt.Errorf(\"instance is protected to prohibit accidental removal (Hint: use `limactl unprotect`)\")\n\t}\n\tif !force && inst.Status != store.StatusStopped {\n\t\treturn fmt.Errorf(\"expected status %q, got %q\", store.StatusStopped, inst.Status)\n\t}\n\n\tstopInstanceForcibly(inst)\n\n\tif err := stop.Unregister(ctx, inst); err != nil {\n\t\treturn fmt.Errorf(\"failed to unregister %q: %w\", inst.Dir, err)\n\t}\n\n\tif err := os.RemoveAll(inst.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", inst.Dir, err)\n\t}\n\n\treturn nil\n}\n\nfunc deleteBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/editflags/editflags.go\npackage editflags\n\nimport (\n\t\"fmt\"\n\t\"math/bits\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pbnjay/memory\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n\tflag \"github.com/spf13/pflag\"\n)\n\n// RegisterEdit registers flags related to in-place YAML modification, for `limactl edit`.\nfunc RegisterEdit(cmd *cobra.Command) {\n\tregisterEdit(cmd, \"\")\n}\n\nfunc registerEdit(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\n\tflags.Int(\"cpus\", 0, commentPrefix+\"number of CPUs\") // Similar to colima's --cpu, but the flag name is slightly different (cpu vs cpus)\n\t_ = cmd.RegisterFlagCompletionFunc(\"cpus\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeCPUs(runtime.NumCPU()) {\n\t\t\tres = append(res, strconv.Itoa(f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.IPSlice(\"dns\", nil, commentPrefix+\"specify custom DNS (disable host resolver)\") // colima-compatible\n\n\tflags.Float32(\"memory\", 0, commentPrefix+\"memory in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeMemoryGiB(memory.TotalMemory()) {\n\t\t\tres = append(res, fmt.Sprintf(\"%.1f\", f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.StringSlice(\"mount\", nil, commentPrefix+\"directories to mount, suffix ':w' for writable (Do not specify directories that overlap with the existing mounts)\") // colima-compatible\n\n\tflags.String(\"mount-type\", \"\", commentPrefix+\"mount type (reverse-sshfs, 9p, virtiofs)\") // Similar to colima's --mount-type=(sshfs|9p|virtiofs), but \"reverse-sshfs\" is Lima is called \"sshfs\" in colima\n\t_ = cmd.RegisterFlagCompletionFunc(\"mount-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"reverse-sshfs\", \"9p\", \"virtiofs\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"mount-writable\", false, commentPrefix+\"make all mounts writable\")\n\tflags.Bool(\"mount-inotify\", false, commentPrefix+\"enable inotify for mounts\")\n\n\tflags.StringSlice(\"network\", nil, commentPrefix+\"additional networks, e.g., \\\"vzNAT\\\" or \\\"lima:shared\\\" to assign vmnet IP\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"network\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\t// TODO: retrieve the lima:* network list from networks.yaml\n\t\treturn []string{\"lima:shared\", \"lima:bridged\", \"lima:host\", \"lima:user-v2\", \"vzNAT\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"rosetta\", false, commentPrefix+\"enable Rosetta (for vz instances)\")\n\n\tflags.String(\"set\", \"\", commentPrefix+\"modify the template inplace, using yq syntax\")\n\n\t// negative performance impact: https://gitlab.com/qemu-project/qemu/-/issues/334\n\tflags.Bool(\"video\", false, commentPrefix+\"enable video output (has negative performance impact for QEMU)\")\n}\n\n// RegisterCreate registers flags related to in-place YAML modification, for `limactl create`.\nfunc RegisterCreate(cmd *cobra.Command, commentPrefix string) {\n\tregisterEdit(cmd, commentPrefix)\n\tflags := cmd.Flags()\n\n\tflags.String(\"arch\", \"\", commentPrefix+\"machine architecture (x86_64, aarch64, riscv64)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"arch\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"x86_64\", \"aarch64\", \"riscv64\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"containerd\", \"\", commentPrefix+\"containerd mode (user, system, user+system, none)\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"user\", \"system\", \"user+system\", \"none\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Float32(\"disk\", 0, commentPrefix+\"disk size in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"10\", \"30\", \"50\", \"100\", \"200\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"vm-type\", \"\", commentPrefix+\"virtual machine type (qemu, vz)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"qemu\", \"vz\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"plain\", false, commentPrefix+\"plain mode. Disable mounts, port forwarding, containerd, etc.\")\n}\n\nfunc defaultExprFunc(expr string) func(v *flag.Flag) (string, error) {\n\treturn func(v *flag.Flag) (string, error) {\n\t\treturn fmt.Sprintf(expr, v.Value), nil\n\t}\n}\n\n// YQExpressions returns YQ expressions.\nfunc YQExpressions(flags *flag.FlagSet, newInstance bool) ([]string, error) {\n\ttype def struct {\n\t\tflagName                 string\n\t\texprFunc                 func(*flag.Flag) (string, error)\n\t\tonlyValidForNewInstances bool\n\t\texperimental             bool\n\t}\n\td := defaultExprFunc\n\tdefs := []def{\n\t\t{\"cpus\", d(\".cpus = %s\"), false, false},\n\t\t{\n\t\t\t\"dns\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tipSlice, err := flags.GetIPSlice(\"dns\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.dns += [`\n\t\t\t\tfor i, ip := range ipSlice {\n\t\t\t\t\texpr += fmt.Sprintf(\"%q\", ip)\n\t\t\t\t\tif i < len(ipSlice)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .dns |= unique | .hostResolver.enabled=false`\n\t\t\t\tlogrus.Warnf(\"Disabling HostResolver, as custom DNS addresses (%v) are specified\", ipSlice)\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"memory\", d(\".memory = \\\"%sGiB\\\"\"), false, false},\n\t\t{\n\t\t\t\"mount\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"mount\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.mounts += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\twritable := strings.HasSuffix(s, \":w\")\n\t\t\t\t\tloc := strings.TrimSuffix(s, \":w\")\n\t\t\t\t\texpr += fmt.Sprintf(`{\"location\": %q, \"writable\": %v}`, loc, writable)\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .mounts |= unique_by(.location)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"mount-type\", d(\".mountType = %q\"), false, false},\n\t\t{\"mount-inotify\", d(\".mountInotify = %s\"), false, true},\n\t\t{\"mount-writable\", d(\".mounts[].writable = %s\"), false, false},\n\t\t{\n\t\t\t\"network\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"network\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.networks += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\t// CLI syntax is still experimental (YAML syntax is out of experimental)\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase s == \"vzNAT\":\n\t\t\t\t\t\texpr += `{\"vzNAT\": true}`\n\t\t\t\t\tcase strings.HasPrefix(s, \"lima:\"):\n\t\t\t\t\t\tnetwork := strings.TrimPrefix(s, \"lima:\")\n\t\t\t\t\t\texpr += fmt.Sprintf(`{\"lima\": %q}`, network)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\treturn \"\", fmt.Errorf(\"network name must be \\\"vzNAT\\\" or \\\"lima:*\\\", got %q\", s)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .networks |= unique_by(.lima)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t\"rosetta\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tb, err := flags.GetBool(\"rosetta\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\treturn fmt.Sprintf(\".rosetta.enabled = %v | .rosetta.binfmt = %v\", b, b), nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\ttrue,\n\t\t},\n\t\t{\"set\", d(\"%s\"), false, false},\n\t\t{\n\t\t\t\"video\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tb, err := flags.GetBool(\"video\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tif b {\n\t\t\t\t\treturn \".video.display = \\\"default\\\"\", nil\n\t\t\t\t}\n\t\t\t\treturn \".video.display = \\\"none\\\"\", nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"arch\", d(\".arch = %q\"), true, false},\n\t\t{\n\t\t\t\"containerd\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\ts, err := flags.GetString(\"containerd\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\tswitch s {\n\t\t\t\tcase \"user\":\n\t\t\t\t\treturn `.containerd.user = true | .containerd.system = false`, nil\n\t\t\t\tcase \"system\":\n\t\t\t\t\treturn `.containerd.user = false | .containerd.system = true`, nil\n\t\t\t\tcase \"user+system\", \"system+user\":\n\t\t\t\t\treturn `.containerd.user = true | .containerd.system = true`, nil\n\t\t\t\tcase \"none\":\n\t\t\t\t\treturn `.containerd.user = false | .containerd.system = false`, nil\n\t\t\t\tdefault:\n\t\t\t\t\treturn \"\", fmt.Errorf(`expected one of [\"user\", \"system\", \"user+system\", \"none\"], got %q`, s)\n\t\t\t\t}\n\t\t\t},\n\t\t\ttrue,\n\t\t\tfalse,\n\t\t},\n\t\t{\"disk\", d(\".disk= \\\"%sGiB\\\"\"), true, false},\n\t\t{\"vm-type\", d(\".vmType = %q\"), true, false},\n\t\t{\"plain\", d(\".plain = %s\"), true, false},\n\t}\n\tvar exprs []string\n\tfor _, def := range defs {\n\t\tv := flags.Lookup(def.flagName)\n\t\tif v != nil && v.Changed {\n\t\t\tif def.experimental {\n\t\t\t\tlogrus.Warnf(\"`--%s` is experimental\", def.flagName)\n\t\t\t}\n\t\t\tif def.onlyValidForNewInstances && !newInstance {\n\t\t\t\tlogrus.Warnf(\"`--%s` is not applicable to an existing instance (Hint: create a new instance with `limactl create --%s=%s --name=NAME`)\",\n\t\t\t\t\tdef.flagName, def.flagName, v.Value.String())\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\texpr, err := def.exprFunc(v)\n\t\t\tif err != nil {\n\t\t\t\treturn exprs, fmt.Errorf(\"error while processing flag %q: %w\", def.flagName, err)\n\t\t\t}\n\t\t\texprs = append(exprs, expr)\n\t\t}\n\t}\n\treturn exprs, nil\n}\n\nfunc isPowerOfTwo(x int) bool {\n\treturn bits.OnesCount(uint(x)) == 1\n}\n\nfunc completeCPUs(hostCPUs int) []int {\n\tvar res []int\n\tfor i := 1; i <= hostCPUs; i *= 2 {\n\t\tres = append(res, i)\n\t}\n\tif !isPowerOfTwo(hostCPUs) {\n\t\tres = append(res, hostCPUs)\n\t}\n\treturn res\n}\n\nfunc completeMemoryGiB(hostMemory uint64) []float32 {\n\thostMemoryHalfGiB := int(hostMemory / 2 / 1024 / 1024 / 1024)\n\tvar res []float32\n\tif hostMemoryHalfGiB < 1 {\n\t\tres = append(res, 0.5)\n\t}\n\tfor i := 1; i <= hostMemoryHalfGiB; i *= 2 {\n\t\tres = append(res, float32(i))\n\t}\n\tif hostMemoryHalfGiB > 1 && !isPowerOfTwo(hostMemoryHalfGiB) {\n\t\tres = append(res, float32(hostMemoryHalfGiB))\n\t}\n\treturn res\n}\n\n// Path: cmd/limactl/guessarg/guessarg.go\npackage guessarg\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/containerd/containerd/identifiers\"\n)\n\nfunc SeemsTemplateURL(arg string) (bool, *url.URL) {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false, u\n\t}\n\treturn u.Scheme == \"template\", u\n}\n\n\nfunc SeemsHTTPURL(arg string) bool {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif u.Scheme != \"http\" && u.Scheme != \"https\" {\n\t\treturn false\n\t}\n\treturn true\n}\n\nfunc SeemsFileURL(arg string) bool {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false\n\t}\n\treturn u.Scheme == \"file\"\n}\n\nfunc SeemsYAMLPath(arg string) bool {\n\tif strings.Contains(arg, \"/\") {\n\t\treturn true\n\t}\n\tlower := strings.ToLower(arg)\n\treturn strings.HasSuffix(lower, \".yml\") || strings.HasSuffix(lower, \".yaml\")\n}\n\nfunc InstNameFromURL(urlStr string) (string, error) {\n\tu, err := url.Parse(urlStr)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn InstNameFromYAMLPath(path.Base(u.Path))\n}\n\nfunc InstNameFromYAMLPath(yamlPath string) (string, error) {\n\ts := strings.ToLower(filepath.Base(yamlPath))\n\ts = strings.TrimSuffix(strings.TrimSuffix(s, \".yml\"), \".yaml\")\n\ts = strings.ReplaceAll(s, \".\", \"-\")\n\tif err := identifiers.Validate(s); err != nil {\n\t\treturn \"\", fmt.Errorf(\"filename %q is invalid: %w\", yamlPath, err)\n\t}\n\treturn s, nil\n}\n\n// Path: cmd/limactl/start.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/containerd/containerd/identifiers\"\n\t\"github.com/lima-vm/lima/cmd/limactl/editflags\"\n\t\"github.com/lima-vm/lima/cmd/limactl/guessarg\"\n\t\"github.com/lima-vm/lima/pkg/editutil\"\n\t\"github.com/lima-vm/lima/pkg/ioutilx\"\n\t\"github.com/lima-vm/lima/pkg/limayaml\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/start\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/lima-vm/lima/pkg/templatestore\"\n\t\"github.com/lima-vm/lima/pkg/uiutil\"\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/lima-vm/lima/pkg/yqutil\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc registerCreateFlags(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\tflags.String(\"name\", \"\", commentPrefix+\"override the instance name\")\n\tflags.Bool(\"list-templates\", false, commentPrefix+\"list available templates and exit\")\n\teditflags.RegisterCreate(cmd, commentPrefix)\n}\n\nfunc newCreateCommand() *cobra.Command {\n\tcreateCommand := &cobra.Command{\n\t\tUse: \"create FILE.yaml|URL\",\n\t\tExample: `\nTo create an instance \"default\" from the default Ubuntu template:\n$ limactl create\n\nTo create an instance \"default\" from a template \"docker\":\n$ limactl create --name=default template://docker\n\nTo create an instance \"default\" with modified parameters:\n$ limactl create --cpus=2 --memory=2\n\nTo create an instance \"default\" with yq expressions:\n$ limactl create --set='.cpus = 2 | .memory = \"2GiB\"'\n\nTo see the template list:\n$ limactl create --list-templates\n\nTo create an instance \"default\" from a local file:\n$ limactl create --name=default /usr/local/share/lima/templates/fedora.yaml\n\nTo create an instance \"default\" from a remote URL (use carefully, with a trustable source):\n$ limactl create --name=default https://raw.githubusercontent.com/lima-vm/lima/master/examples/alpine.yaml\n\nTo create an instance \"local\" from a template passed to stdin (--name parameter is required):\n$ cat template.yaml | limactl create --name=local -\n`,\n\t\tShort:             \"Create an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tValidArgsFunction: createBashComplete,\n\t\tRunE:              createAction,\n\t\tGroupID:           basicCommand,\n\t}\n\tregisterCreateFlags(createCommand, \"\")\n\treturn createCommand\n}\n\nfunc newStartCommand() *cobra.Command {\n\tstartCommand := &cobra.Command{\n\t\tUse: \"start NAME|FILE.yaml|URL\",\n\t\tExample: `\nTo create an instance \"default\" (if not created yet) from the default Ubuntu template, and start it:\n$ limactl start\n\nTo create an instance \"default\" from a template \"docker\", and start it:\n$ limactl start --name=default template://docker\n\n'limactl start' also accepts the 'limactl create' flags such as '--set'.\nSee the examples in 'limactl create --help'.\n`,\n\t\tShort:             \"Start an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tValidArgsFunction: startBashComplete,\n\t\tRunE:              startAction,\n\t\tGroupID:           basicCommand,\n\t}\n\tregisterCreateFlags(startCommand, \"[limactl create] \")\n\tif runtime.GOOS != \"windows\" {\n\t\tstartCommand.Flags().Bool(\"foreground\", false, \"run the hostagent in the foreground\")\n\t}\n\tstartCommand.Flags().Duration(\"timeout\", start.DefaultWatchHostAgentEventsTimeout, \"duration to wait for the instance to be running before timing out\")\n\treturn startCommand\n}\n\nfunc loadOrCreateInstance(cmd *cobra.Command, args []string, createOnly bool) (*store.Instance, error) {\n\tvar arg string // can be empty\n\tif len(args) > 0 {\n\t\targ = args[0]\n\t}\n\n\tvar (\n\t\tst  = &creatorState{}\n\t\terr error\n\t)\n\n\tflags := cmd.Flags()\n\n\t// Create an instance, with menu TUI when TTY is available\n\ttty, err := flags.GetBool(\"tty\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tst.instName, err = flags.GetString(\"name\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tconst yBytesLimit = 4 * 1024 * 1024 // 4MiB\n\n\tif ok, u := guessarg.SeemsTemplateURL(arg); ok {\n\t\t// No need to use SecureJoin here. https://github.com/lima-vm/lima/pull/805#discussion_r853411702\n\t\ttemplateName := filepath.Join(u.Host, u.Path)\n\t\tlogrus.Debugf(\"interpreting argument %q as a template name %q\", arg, templateName)\n\t\tif st.instName == \"\" {\n\t\t\t// e.g., templateName = \"deprecated/centos-7\" , st.instName = \"centos-7\"\n\t\t\tst.instName = filepath.Base(templateName)\n\t\t}\n\t\tst.yBytes, err = templatestore.Read(templateName)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsHTTPURL(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromURL(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a http url for instance %q\", arg, st.instName)\n\t\treq, err := http.NewRequestWithContext(cmd.Context(), http.MethodGet, arg, http.NoBody)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tresp, err := http.DefaultClient.Do(req)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer resp.Body.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(resp.Body, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsFileURL(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromURL(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a file url for instance %q\", arg, st.instName)\n\t\tr, err := os.Open(strings.TrimPrefix(arg, \"file://\"))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer r.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(r, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if guessarg.SeemsYAMLPath(arg) {\n\t\tif st.instName == \"\" {\n\t\t\tst.instName, err = guessarg.InstNameFromYAMLPath(arg)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tlogrus.Debugf(\"interpreting argument %q as a file path for instance %q\", arg, st.instName)\n\t\tr, err := os.Open(arg)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer r.Close()\n\t\tst.yBytes, err = ioutilx.ReadAtMaximum(r, yBytesLimit)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if arg == \"-\" {\n\t\tif st.instName == \"\" {\n\t\t\treturn nil, errors.New(\"must pass instance name with --name when reading template from stdin\")\n\t\t}\n\t\tst.yBytes, err = io.ReadAll(os.Stdin)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"unexpected error reading stdin: %w\", err)\n\t\t}\n\t\t// see if the tty was set explicitly or not\n\t\tttySet := cmd.Flags().Changed(\"tty\")\n\t\tif ttySet && tty {\n\t\t\treturn nil, errors.New(\"cannot use --tty=true and read template from stdin together\")\n\t\t}\n\t\ttty = false\n\t} else {\n\t\tif arg == \"\" {\n\t\t\tif st.instName == \"\" {\n\t\t\t\tst.instName = DefaultInstanceName\n\t\t\t}\n\t\t} else {\n\t\t\tlogrus.Debugf(\"interpreting argument %q as an instance name\", arg)\n\t\t\tif st.instName != \"\" && st.instName != arg {\n\t\t\t\treturn nil, fmt.Errorf(\"instance name %q and CLI flag --name=%q cannot be specified together\", arg, st.instName)\n\t\t\t}\n\t\t\tst.instName = arg\n\t\t}\n\t\tif err := identifiers.Validate(st.instName); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"argument must be either an instance name, a YAML file path, or a URL, got %q: %w\", st.instName, err)\n\t\t}\n\t\tinst, err := store.Inspect(st.instName)\n\t\tif err == nil {\n\t\t\tif createOnly {\n\t\t\t\treturn nil, fmt.Errorf(\"Instance %q already exists\", st.instName)\n\t\t\t}\n\t\t\tlogrus.Infof(\"Using the existing instance %q\", st.instName)\n\t\t\tyqExprs, err := editflags.YQExpressions(flags, false)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif len(yqExprs) > 0 {\n\t\t\t\tyq := yqutil.Join(yqExprs)\n\t\t\t\tinst, err = applyYQExpressionToExistingInstance(inst, yq)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"failed to apply yq expression %q to instance %q: %w\", yq, st.instName, err)\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn inst, nil\n\t\t}\n\t\tif !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn nil, err\n\t\t}\n\t\tif arg != \"\" && arg != DefaultInstanceName {\n\t\t\tlogrus.Infof(\"Creating an instance %q from template://default (Not from template://%s)\", st.instName, st.instName)\n\t\t\tlogrus.Warnf(\"This form is deprecated. Use `limactl create --name=%s template://default` instead\", st.instName)\n\t\t}\n\t\t// Read the default template for creating a new instance\n\t\tst.yBytes, err = templatestore.Read(templatestore.Default)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tyqExprs, err := editflags.YQExpressions(flags, true)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tyq := yqutil.Join(yqExprs)\n\tif tty {\n\t\tvar err error\n\t\tst, err = chooseNextCreatorState(st, yq)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"Terminal is not available, proceeding without opening an editor\")\n\t\tif err := modifyInPlace(st, yq); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\tsaveBrokenEditorBuffer := tty\n\treturn createInstance(cmd.Context(), st, saveBrokenEditorBuffer)\n}\n\nfunc applyYQExpressionToExistingInstance(inst *store.Instance, yq string) (*store.Instance, error) {\n\tif strings.TrimSpace(yq) == \"\" {\n\t\treturn inst, nil\n\t}\n\tfilePath := filepath.Join(inst.Dir, filenames.LimaYAML)\n\tyContent, err := os.ReadFile(filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tlogrus.Debugf(\"Applying yq expression %q to an existing instance %q\", yq, inst.Name)\n\tyBytes, err := yqutil.EvaluateExpression(yq, yContent)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filePath, yBytes, 0o644); err != nil {\n\t\treturn nil, err\n\t}\n\t// Reload\n\treturn store.Inspect(inst.Name)\n}\n\nfunc createInstance(ctx context.Context, st *creatorState, saveBrokenEditorBuffer bool) (*store.Instance, error) {\n\tif st.instName == \"\" {\n\t\treturn nil, errors.New(\"got empty st.instName\")\n\t}\n\tif len(st.yBytes) == 0 {\n\t\treturn nil, errors.New(\"got empty st.yBytes\")\n\t}\n\n\tinstDir, err := store.InstanceDir(st.instName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// the full path of the socket name must be less than UNIX_PATH_MAX chars.\n\tmaxSockName := filepath.Join(instDir, filenames.LongestSock)\n\tif len(maxSockName) >= osutil.UnixPathMax {\n\t\treturn nil, fmt.Errorf(\"instance name %q too long: %q must be less than UNIX_PATH_MAX=%d characters, but is %d\",\n\t\t\tst.instName, maxSockName, osutil.UnixPathMax, len(maxSockName))\n\t}\n\tif _, err := os.Stat(instDir); !errors.Is(err, os.ErrNotExist) {\n\t\treturn nil, fmt.Errorf(\"instance %q already exists (%q)\", st.instName, instDir)\n\t}\n\t// limayaml.Load() needs to pass the store file path to limayaml.FillDefault() to calculate default MAC addresses\n\tfilePath := filepath.Join(instDir, filenames.LimaYAML)\n\ty, err := limayaml.Load(st.yBytes, filePath)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif err := limayaml.Validate(*y, true); err != nil {\n\t\tif !saveBrokenEditorBuffer {\n\t\t\treturn nil, err\n\t\t}\n\t\trejectedYAML := \"lima.REJECTED.yaml\"\n\t\tif writeErr := os.WriteFile(rejectedYAML, st.yBytes, 0o644); writeErr != nil {\n\t\t\treturn nil, fmt.Errorf(\"the YAML is invalid, attempted to save the buffer as %q but failed: %w: %w\", rejectedYAML, writeErr, err)\n\t\t}\n\t\treturn nil, fmt.Errorf(\"the YAML is invalid, saved the buffer as %q: %w\", rejectedYAML, err)\n\t}\n\tif err := os.MkdirAll(instDir, 0o700); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filePath, st.yBytes, 0o644); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := os.WriteFile(filepath.Join(instDir, filenames.LimaVersion), []byte(version.Version), 0o444); err != nil {\n\t\treturn nil, err\n\t}\n\n\tinst, err := store.Inspect(st.instName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := start.Register(ctx, inst); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn inst, nil\n}\n\ntype creatorState struct {\n\tinstName string // instance name\n\tyBytes   []byte // yaml bytes\n}\n\nfunc modifyInPlace(st *creatorState, yq string) error {\n\tout, err := yqutil.EvaluateExpression(yq, st.yBytes)\n\tif err != nil {\n\t\treturn err\n\t}\n\tst.yBytes = out\n\treturn nil\n}\n\nfunc chooseNextCreatorState(st *creatorState, yq string) (*creatorState, error) {\n\tfor {\n\t\tif err := modifyInPlace(st, yq); err != nil {\n\t\t\tlogrus.WithError(err).Warn(\"Failed to evaluate yq expression\")\n\t\t\treturn st, err\n\t\t}\n\t\tmessage := fmt.Sprintf(\"Creating an instance %q\", st.instName)\n\t\toptions := []string{\n\t\t\t\"Proceed with the current configuration\",\n\t\t\t\"Open an editor to review or modify the current configuration\",\n\t\t\t\"Choose another template (docker, podman, archlinux, fedora, ...)\",\n\t\t\t\"Exit\",\n\t\t}\n\t\tans, err := uiutil.Select(message, options)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, uiutil.InterruptErr) {\n\t\t\t\tlogrus.Fatal(\"Interrupted by user\")\n\t\t\t}\n\t\t\tlogrus.WithError(err).Warn(\"Failed to open TUI\")\n\t\t\treturn st, nil\n\t\t}\n\t\tswitch ans {\n\t\tcase 0: // \"Proceed with the current configuration\"\n\t\t\treturn st, nil\n\t\tcase 1: // \"Open an editor ...\"\n\t\t\thdr := fmt.Sprintf(\"# Review and modify the following configuration for Lima instance %q.\\n\", st.instName)\n\t\t\tif st.instName == DefaultInstanceName {\n\t\t\t\thdr += \"# - In most cases, you do not need to modify this file.\\n\"\n\t\t\t}\n\t\t\thdr += \"# - To cancel starting Lima, just save this file as an empty file.\\n\"\n\t\t\thdr += \"\\n\"\n\t\t\thdr += editutil.GenerateEditorWarningHeader()\n\t\t\tvar err error\n\t\t\tst.yBytes, err = editutil.OpenEditor(st.yBytes, hdr)\n\t\t\tif err != nil {\n\t\t\t\treturn st, err\n...\n", "needle_token_start": 9844, "needle_token_end": 9916, "code_context_ntokens": 15221, "output": ["```go\nfunc SeemsHTTPURL(arg string) bool {\n\tu, err := url.Parse(arg)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif u.Scheme != \"http\" && u.Scheme != \"https\" {\n\t\treturn false\n\t}\n\treturn true\n}\n```"]}
{"repo": "lima-vm/lima", "name": "stopBashComplete", "language": "go", "path": "cmd/limactl/stop.go", "position_ratio": 0.75, "description": "\nFunction Description:\n1. **Purpose**: Provides command-line completion options for stopping instances.\n2. **Input**: The command being executed, an array of strings representing the current command line arguments, and a string representing the current argument being completed.\n3. **Output**: Returns a list of possible completion strings and a directive that instructs the shell completion behavior.\n4. **Procedure**: Calls another function to retrieve the names of all instances, which are then used as the completion suggestions for the command.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/lima-guestagent/daemon_linux.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"net\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/lima-vm/lima/pkg/guestagent\"\n\t\"github.com/lima-vm/lima/pkg/guestagent/api/server\"\n\t\"github.com/lima-vm/lima/pkg/guestagent/serialport\"\n\t\"github.com/mdlayher/vsock\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDaemonCommand() *cobra.Command {\n\tdaemonCommand := &cobra.Command{\n\t\tUse:   \"daemon\",\n\t\tShort: \"run the daemon\",\n\t\tRunE:  daemonAction,\n\t}\n\tdaemonCommand.Flags().Duration(\"tick\", 3*time.Second, \"tick for polling events\")\n\tdaemonCommand.Flags().Int(\"vsock-port\", 0, \"use vsock server instead a UNIX socket\")\n\tdaemonCommand.Flags().String(\"virtio-port\", \"\", \"use virtio server instead a UNIX socket\")\n\treturn daemonCommand\n}\n\nfunc daemonAction(cmd *cobra.Command, _ []string) error {\n\tsocket := \"/run/lima-guestagent.sock\"\n\ttick, err := cmd.Flags().GetDuration(\"tick\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvSockPort, err := cmd.Flags().GetInt(\"vsock-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvirtioPort, err := cmd.Flags().GetString(\"virtio-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif tick == 0 {\n\t\treturn errors.New(\"tick must be specified\")\n\t}\n\tif os.Geteuid() != 0 {\n\t\treturn errors.New(\"must run as the root\")\n\t}\n\tlogrus.Infof(\"event tick: %v\", tick)\n\n\tnewTicker := func() (<-chan time.Time, func()) {\n\t\t// TODO: use an equivalent of `bpftrace -e 'tracepoint:syscalls:sys_*_bind { printf(\"tick\\n\"); }')`,\n\t\t// without depending on `bpftrace` binary.\n\t\t// The agent binary will need CAP_BPF file cap.\n\t\tticker := time.NewTicker(tick)\n\t\treturn ticker.C, ticker.Stop\n\t}\n\n\tagent, err := guestagent.New(newTicker, tick*20)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = os.RemoveAll(socket)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar l net.Listener\n\tif virtioPort != \"\" {\n\t\tqemuL, err := serialport.Listen(\"/dev/virtio-ports/\" + virtioPort)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = qemuL\n\t\tlogrus.Infof(\"serving the guest agent on qemu serial file: %s\", virtioPort)\n\t} else if vSockPort != 0 {\n\t\tvsockL, err := vsock.Listen(uint32(vSockPort), nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = vsockL\n\t\tlogrus.Infof(\"serving the guest agent on vsock port: %d\", vSockPort)\n\t} else {\n\t\tsocketL, err := net.Listen(\"unix\", socket)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := os.Chmod(socket, 0o777); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = socketL\n\t\tlogrus.Infof(\"serving the guest agent on %q\", socket)\n\t}\n\treturn server.StartServer(l, &server.GuestServer{Agent: agent})\n}\n\n// Path: cmd/lima-guestagent/install_systemd_linux.go\npackage main\n\nimport (\n\t_ \"embed\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/textutil\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newInstallSystemdCommand() *cobra.Command {\n\tinstallSystemdCommand := &cobra.Command{\n\t\tUse:   \"install-systemd\",\n\t\tShort: \"install a systemd unit (user)\",\n\t\tRunE:  installSystemdAction,\n\t}\n\tinstallSystemdCommand.Flags().Int(\"vsock-port\", 0, \"use vsock server on specified port\")\n\tinstallSystemdCommand.Flags().String(\"virtio-port\", \"\", \"use virtio server instead a UNIX socket\")\n\treturn installSystemdCommand\n}\n\nfunc installSystemdAction(cmd *cobra.Command, _ []string) error {\n\tvsockPort, err := cmd.Flags().GetInt(\"vsock-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvirtioPort, err := cmd.Flags().GetString(\"virtio-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tunit, err := generateSystemdUnit(vsockPort, virtioPort)\n\tif err != nil {\n\t\treturn err\n\t}\n\tunitPath := \"/etc/systemd/system/lima-guestagent.service\"\n\tif _, err := os.Stat(unitPath); !errors.Is(err, os.ErrNotExist) {\n\t\tlogrus.Infof(\"File %q already exists, overwriting\", unitPath)\n\t} else {\n\t\tunitDir := filepath.Dir(unitPath)\n\t\tif err := os.MkdirAll(unitDir, 0o755); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err := os.WriteFile(unitPath, unit, 0o644); err != nil {\n\t\treturn err\n\t}\n\tlogrus.Infof(\"Written file %q\", unitPath)\n\targs := [][]string{\n\t\t{\"daemon-reload\"},\n\t\t{\"enable\", \"--now\", \"lima-guestagent.service\"},\n\t\t{\"try-restart\", \"lima-guestagent.service\"},\n\t}\n\tfor _, args := range args {\n\t\tcmd := exec.Command(\"systemctl\", append([]string{\"--system\"}, args...)...)\n\t\tcmd.Stdout = os.Stdout\n\t\tcmd.Stderr = os.Stderr\n\t\tlogrus.Infof(\"Executing: %s\", strings.Join(cmd.Args, \" \"))\n\t\tif err := cmd.Run(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tlogrus.Info(\"Done\")\n\treturn nil\n}\n\n//go:embed lima-guestagent.TEMPLATE.service\nvar systemdUnitTemplate string\n\nfunc generateSystemdUnit(vsockPort int, virtioPort string) ([]byte, error) {\n\tselfExeAbs, err := os.Executable()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar args []string\n\tif vsockPort != 0 {\n\t\targs = append(args, fmt.Sprintf(\"--vsock-port %d\", vsockPort))\n\t}\n\tif virtioPort != \"\" {\n\t\targs = append(args, fmt.Sprintf(\"--virtio-port %s\", virtioPort))\n\t}\n\n\tm := map[string]string{\n\t\t\"Binary\": selfExeAbs,\n\t\t\"Args\":   strings.Join(args, \" \"),\n\t}\n\treturn textutil.ExecuteTemplate(systemdUnitTemplate, m)\n}\n\n// Path: cmd/lima-guestagent/main_linux.go\npackage main\n\nimport (\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc main() {\n\tif err := newApp().Execute(); err != nil {\n\t\tlogrus.Fatal(err)\n\t}\n}\n\nfunc newApp() *cobra.Command {\n\trootCmd := &cobra.Command{\n\t\tUse:     \"lima-guestagent\",\n\t\tShort:   \"Do not launch manually\",\n\t\tVersion: strings.TrimPrefix(version.Version, \"v\"),\n\t}\n\trootCmd.PersistentFlags().Bool(\"debug\", false, \"debug mode\")\n\trootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n\t\tdebug, _ := cmd.Flags().GetBool(\"debug\")\n\t\tif debug {\n\t\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t\t}\n\t\treturn nil\n\t}\n\trootCmd.AddCommand(\n\t\tnewDaemonCommand(),\n\t\tnewInstallSystemdCommand(),\n\t)\n\treturn rootCmd\n}\n\n// Path: cmd/limactl/completion.go\npackage main\n\nimport (\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/templatestore\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc bashCompleteInstanceNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tinstances, err := store.Instances()\n\tif err != nil {\n\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t}\n\treturn instances, cobra.ShellCompDirectiveNoFileComp\n}\n\nfunc bashCompleteTemplateNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tvar comp []string\n\tif templates, err := templatestore.Templates(); err == nil {\n\t\tfor _, f := range templates {\n\t\t\tcomp = append(comp, \"template://\"+f.Name)\n\t\t}\n\t}\n\treturn comp, cobra.ShellCompDirectiveDefault\n}\n\nfunc bashCompleteDiskNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tdisks, err := store.Disks()\n\tif err != nil {\n\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t}\n\treturn disks, cobra.ShellCompDirectiveNoFileComp\n}\n\n// Path: cmd/limactl/copy.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\n\t\"github.com/coreos/go-semver/semver\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst copyHelp = `Copy files between host and guest\n\nPrefix guest filenames with the instance name and a colon.\n\nExample: limactl copy default:/etc/os-release .\n`\n\nfunc newCopyCommand() *cobra.Command {\n\tcopyCommand := &cobra.Command{\n\t\tUse:     \"copy SOURCE ... TARGET\",\n\t\tAliases: []string{\"cp\"},\n\t\tShort:   \"Copy files between host and guest\",\n\t\tLong:    copyHelp,\n\t\tArgs:    WrapArgsError(cobra.MinimumNArgs(2)),\n\t\tRunE:    copyAction,\n\t\tGroupID: advancedCommand,\n\t}\n\n\tcopyCommand.Flags().BoolP(\"recursive\", \"r\", false, \"copy directories recursively\")\n\n\treturn copyCommand\n}\n\nfunc copyAction(cmd *cobra.Command, args []string) error {\n\trecursive, err := cmd.Flags().GetBool(\"recursive\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\targ0, err := exec.LookPath(\"scp\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tu, err := osutil.LimaUser(false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tinstDirs := make(map[string]string)\n\tscpFlags := []string{}\n\tscpArgs := []string{}\n\tdebug, err := cmd.Flags().GetBool(\"debug\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif debug {\n\t\tscpFlags = append(scpFlags, \"-v\")\n\t}\n\tif recursive {\n\t\tscpFlags = append(scpFlags, \"-r\")\n\t}\n\tlegacySSH := false\n\tif sshutil.DetectOpenSSHVersion().LessThan(*semver.New(\"8.0.0\")) {\n\t\tlegacySSH = true\n\t}\n\tfor _, arg := range args {\n\t\tpath := strings.Split(arg, \":\")\n\t\tswitch len(path) {\n\t\tcase 1:\n\t\t\tscpArgs = append(scpArgs, arg)\n\t\tcase 2:\n\t\t\tinstName := path[0]\n\t\t\tinst, err := store.Inspect(instName)\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t\t\t}\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif inst.Status == store.StatusStopped {\n\t\t\t\treturn fmt.Errorf(\"instance %q is stopped, run `limactl start %s` to start the instance\", instName, instName)\n\t\t\t}\n\t\t\tif legacySSH {\n\t\t\t\tscpFlags = append(scpFlags, \"-P\", fmt.Sprintf(\"%d\", inst.SSHLocalPort))\n\t\t\t\tscpArgs = append(scpArgs, fmt.Sprintf(\"%s@127.0.0.1:%s\", u.Username, path[1]))\n\t\t\t} else {\n\t\t\t\tscpArgs = append(scpArgs, fmt.Sprintf(\"scp://%s@127.0.0.1:%d/%s\", u.Username, inst.SSHLocalPort, path[1]))\n\t\t\t}\n\t\t\tinstDirs[instName] = inst.Dir\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"path %q contains multiple colons\", arg)\n\t\t}\n\t}\n\tif legacySSH && len(instDirs) > 1 {\n\t\treturn fmt.Errorf(\"More than one (instance) host is involved in this command, this is only supported for openSSH v8.0 or higher\")\n\t}\n\tscpFlags = append(scpFlags, \"-3\", \"--\")\n\tscpArgs = append(scpFlags, scpArgs...)\n\n\tvar sshOpts []string\n\tif len(instDirs) == 1 {\n\t\t// Only one (instance) host is involved; we can use the instance-specific\n\t\t// arguments such as ControlPath.  This is preferred as we can multiplex\n\t\t// sessions without re-authenticating (MaxSessions permitting).\n\t\tfor _, instDir := range instDirs {\n\t\t\tsshOpts, err = sshutil.SSHOpts(instDir, false, false, false, false)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// Copying among multiple hosts; we can't pass in host-specific options.\n\t\tsshOpts, err = sshutil.CommonOpts(false)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tsshArgs := sshutil.SSHArgsFromOpts(sshOpts)\n\n\tsshCmd := exec.Command(arg0, append(sshArgs, scpArgs...)...)\n\tsshCmd.Stdin = cmd.InOrStdin()\n\tsshCmd.Stdout = cmd.OutOrStdout()\n\tsshCmd.Stderr = cmd.ErrOrStderr()\n\tlogrus.Debugf(\"executing scp (may take a long time)): %+v\", sshCmd.Args)\n\n\t// TODO: use syscall.Exec directly (results in losing tty?)\n\treturn sshCmd.Run()\n}\n\n// Path: cmd/limactl/debug.go\npackage main\n\nimport (\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/lima-vm/lima/pkg/hostagent/dns\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDebugCommand() *cobra.Command {\n\tcmd := &cobra.Command{\n\t\tUse:    \"debug\",\n\t\tShort:  \"Debug utilities\",\n\t\tLong:   \"DO NOT USE! THE COMMAND SYNTAX IS SUBJECT TO CHANGE!\",\n\t\tHidden: true,\n\t}\n\tcmd.AddCommand(newDebugDNSCommand())\n\treturn cmd\n}\n\nfunc newDebugDNSCommand() *cobra.Command {\n\tcmd := &cobra.Command{\n\t\tUse:   \"dns UDPPORT [TCPPORT]\",\n\t\tShort: \"Debug built-in DNS\",\n\t\tLong:  \"DO NOT USE! THE COMMAND SYNTAX IS SUBJECT TO CHANGE!\",\n\t\tArgs:  WrapArgsError(cobra.RangeArgs(1, 2)),\n\t\tRunE:  debugDNSAction,\n\t}\n\tcmd.Flags().BoolP(\"ipv6\", \"6\", false, \"lookup IPv6 addresses too\")\n\treturn cmd\n}\n\nfunc debugDNSAction(cmd *cobra.Command, args []string) error {\n\tipv6, err := cmd.Flags().GetBool(\"ipv6\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tudpLocalPort, err := strconv.Atoi(args[0])\n\tif err != nil {\n\t\treturn err\n\t}\n\ttcpLocalPort := 0\n\tif len(args) > 1 {\n\t\ttcpLocalPort, err = strconv.Atoi(args[1])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tsrvOpts := dns.ServerOptions{\n\t\tUDPPort: udpLocalPort,\n\t\tTCPPort: tcpLocalPort,\n\t\tAddress: \"127.0.0.1\",\n\t\tHandlerOptions: dns.HandlerOptions{\n\t\t\tIPv6:        ipv6,\n\t\t\tStaticHosts: map[string]string{},\n\t\t},\n\t}\n\tsrv, err := dns.Start(srvOpts)\n\tif err != nil {\n\t\treturn err\n\t}\n\tlogrus.Infof(\"Started srv %+v (UDP %d, TCP %d)\", srv, udpLocalPort, tcpLocalPort)\n\tfor {\n\t\ttime.Sleep(time.Hour)\n\t}\n}\n\n// Path: cmd/limactl/disk.go\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"os\"\n\t\"text/tabwriter\"\n\n\t\"github.com/docker/go-units\"\n\t\"github.com/lima-vm/lima/pkg/qemu\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDiskCommand() *cobra.Command {\n\tdiskCommand := &cobra.Command{\n\t\tUse:   \"disk\",\n\t\tShort: \"Lima disk management\",\n\t\tExample: `  Create a disk:\n  $ limactl disk create DISK --size SIZE [--format qcow2]\n\n  List existing disks:\n  $ limactl disk ls\n\n  Delete a disk:\n  $ limactl disk delete DISK\n  \n  Resize a disk:\n  $ limactl disk resize DISK --size SIZE`,\n\t\tSilenceUsage:  true,\n\t\tSilenceErrors: true,\n\t\tGroupID:       advancedCommand,\n\t}\n\tdiskCommand.AddCommand(\n\t\tnewDiskCreateCommand(),\n\t\tnewDiskListCommand(),\n\t\tnewDiskDeleteCommand(),\n\t\tnewDiskUnlockCommand(),\n\t\tnewDiskResizeCommand(),\n\t)\n\treturn diskCommand\n}\n\nfunc newDiskCreateCommand() *cobra.Command {\n\tdiskCreateCommand := &cobra.Command{\n\t\tUse: \"create DISK\",\n\t\tExample: `\nTo create a new disk:\n$ limactl disk create DISK --size SIZE [--format qcow2]\n`,\n\t\tShort: \"Create a Lima disk\",\n\t\tArgs:  WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:  diskCreateAction,\n\t}\n\tdiskCreateCommand.Flags().String(\"size\", \"\", \"configure the disk size\")\n\t_ = diskCreateCommand.MarkFlagRequired(\"size\")\n\tdiskCreateCommand.Flags().String(\"format\", \"qcow2\", \"specify the disk format\")\n\treturn diskCreateCommand\n}\n\nfunc diskCreateAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tswitch format {\n\tcase \"qcow2\", \"raw\":\n\tdefault:\n\t\treturn fmt.Errorf(`disk format %q not supported, use \"qcow2\" or \"raw\" instead`, format)\n\t}\n\n\t// only exactly one arg is allowed\n\tname := args[0]\n\n\tdiskDir, err := store.DiskDir(name)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif _, err := os.Stat(diskDir); !errors.Is(err, fs.ErrNotExist) {\n\t\treturn fmt.Errorf(\"disk %q already exists (%q)\", name, diskDir)\n\t}\n\n\tlogrus.Infof(\"Creating %s disk %q with size %s\", format, name, units.BytesSize(float64(diskSize)))\n\n\tif err := os.MkdirAll(diskDir, 0o700); err != nil {\n\t\treturn err\n\t}\n\n\tif err := qemu.CreateDataDisk(diskDir, format, int(diskSize)); err != nil {\n\t\trerr := os.RemoveAll(diskDir)\n\t\tif rerr != nil {\n\t\t\terr = errors.Join(err, fmt.Errorf(\"failed to remove a directory %q: %w\", diskDir, rerr))\n\t\t}\n\t\treturn fmt.Errorf(\"Failed to create %s disk in %q: %w\", format, diskDir, err)\n\t}\n\n\treturn nil\n}\n\nfunc newDiskListCommand() *cobra.Command {\n\tdiskListCommand := &cobra.Command{\n\t\tUse: \"list\",\n\t\tExample: `\nTo list existing disks:\n$ limactl disk list\n`,\n\t\tShort:   \"List existing Lima disks\",\n\t\tAliases: []string{\"ls\"},\n\t\tArgs:    WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:    diskListAction,\n\t}\n\tdiskListCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\treturn diskListCommand\n}\n\nfunc diskMatches(diskName string, disks []string) []string {\n\tmatches := []string{}\n\tfor _, disk := range disks {\n\t\tif disk == diskName {\n\t\t\tmatches = append(matches, disk)\n\t\t}\n\t}\n\treturn matches\n}\n\nfunc diskListAction(cmd *cobra.Command, args []string) error {\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tallDisks, err := store.Disks()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdisks := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := diskMatches(arg, allDisks)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tdisks = append(disks, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No disk matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tdisks = allDisks\n\t}\n\n\tif jsonFormat {\n\t\tfor _, diskName := range disks {\n\t\t\tdisk, err := store.InspectDisk(diskName)\n\t\t\tif err != nil {\n\t\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tj, err := json.Marshal(disk)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), string(j))\n\t\t}\n\t\treturn nil\n\t}\n\n\tw := tabwriter.NewWriter(cmd.OutOrStdout(), 4, 8, 4, ' ', 0)\n\tfmt.Fprintln(w, \"NAME\\tSIZE\\tFORMAT\\tDIR\\tIN-USE-BY\")\n\n\tif len(disks) == 0 {\n\t\tlogrus.Warn(\"No disk found. Run `limactl disk create DISK --size SIZE` to create a disk.\")\n\t}\n\n\tfor _, diskName := range disks {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Errorf(\"disk %q does not exist?\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tfmt.Fprintf(w, \"%s\\t%s\\t%s\\t%s\\t%s\\n\", disk.Name, units.BytesSize(float64(disk.Size)), disk.Format, disk.Dir, disk.Instance)\n\t}\n\n\treturn w.Flush()\n}\n\nfunc newDiskDeleteCommand() *cobra.Command {\n\tdiskDeleteCommand := &cobra.Command{\n\t\tUse: \"delete DISK [DISK, ...]\",\n\t\tExample: `\nTo delete a disk:\n$ limactl disk delete DISK\n\nTo delete multiple disks:\n$ limactl disk delete DISK1 DISK2 ...\n`,\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskDeleteAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskDeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"force delete\")\n\treturn diskDeleteCommand\n}\n\nfunc diskDeleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tinstNames, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar instances []*store.Instance\n\tfor _, instName := range instNames {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tinstances = append(instances, inst)\n\t}\n\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tif !force {\n\t\t\tif disk.Instance != \"\" {\n\t\t\t\treturn fmt.Errorf(\"cannot delete disk %q in use by instance %q\", disk.Name, disk.Instance)\n\t\t\t}\n\t\t\tvar refInstances []string\n\t\t\tfor _, inst := range instances {\n\t\t\t\tif len(inst.AdditionalDisks) > 0 {\n\t\t\t\t\tfor _, d := range inst.AdditionalDisks {\n\t\t\t\t\t\tif d.Name == diskName {\n\t\t\t\t\t\t\trefInstances = append(refInstances, inst.Name)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif len(refInstances) > 0 {\n\t\t\t\tlogrus.Warnf(\"Skipping deleting disk %q, disk is referenced by one or more non-running instances: %q\",\n\t\t\t\t\tdiskName, refInstances)\n\t\t\t\tlogrus.Warnf(\"To delete anyway, run %q\", forceDeleteCommand(diskName))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif err := deleteDisk(disk); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\nfunc deleteDisk(disk *store.Disk) error {\n\tif err := os.RemoveAll(disk.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", disk.Dir, err)\n\t}\n\treturn nil\n}\n\nfunc forceDeleteCommand(diskName string) string {\n\treturn fmt.Sprintf(\"limactl disk delete --force %v\", diskName)\n}\n\nfunc newDiskUnlockCommand() *cobra.Command {\n\tdiskUnlockCommand := &cobra.Command{\n\t\tUse: \"unlock DISK [DISK, ...]\",\n\t\tExample: `\nEmergency recovery! If an instance is force stopped, it may leave a disk locked while not actually using it.\n\nTo unlock a disk:\n$ limactl disk unlock DISK\n\nTo unlock multiple disks:\n$ limactl disk unlock DISK1 DISK2 ...\n`,\n\t\tShort:             \"Unlock one or more Lima disks\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              diskUnlockAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\treturn diskUnlockCommand\n}\n\nfunc diskUnlockAction(_ *cobra.Command, args []string) error {\n\tfor _, diskName := range args {\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent disk %q\", diskName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif disk.Instance == \"\" {\n\t\t\tlogrus.Warnf(\"Ignoring unlocked disk %q\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\t// if store.Inspect throws an error, the instance does not exist, and it is safe to unlock\n\t\tinst, err := store.Inspect(disk.Instance)\n\t\tif err == nil {\n\t\t\tif len(inst.Errors) > 0 {\n\t\t\t\tlogrus.Warnf(\"Cannot unlock disk %q, attached instance %q has errors: %+v\",\n\t\t\t\t\tdiskName, disk.Instance, inst.Errors)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif inst.Status == store.StatusRunning {\n\t\t\t\tlogrus.Warnf(\"Cannot unlock disk %q used by running instance %q\", diskName, disk.Instance)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to unlock disk %q: %w\", diskName, err)\n\t\t}\n\t\tlogrus.Infof(\"Unlocked disk %q (%q)\", diskName, disk.Dir)\n\t}\n\treturn nil\n}\n\nfunc newDiskResizeCommand() *cobra.Command {\n\tdiskResizeCommand := &cobra.Command{\n\t\tUse: \"resize DISK\",\n\t\tExample: `\nResize a disk:\n$ limactl disk resize DISK --size SIZE`,\n\t\tShort:             \"Resize existing Lima disk\",\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:              diskResizeAction,\n\t\tValidArgsFunction: diskBashComplete,\n\t}\n\tdiskResizeCommand.Flags().String(\"size\", \"\", \"Disk size\")\n\t_ = diskResizeCommand.MarkFlagRequired(\"size\")\n\treturn diskResizeCommand\n}\n\nfunc diskResizeAction(cmd *cobra.Command, args []string) error {\n\tsize, err := cmd.Flags().GetString(\"size\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskSize, err := units.RAMInBytes(size)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdiskName := args[0]\n\tdisk, err := store.InspectDisk(diskName)\n\tif err != nil {\n\t\tif errors.Is(err, fs.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"disk %q does not exists\", diskName)\n\t\t}\n\t\treturn err\n\t}\n\n\t// Shrinking can cause a disk failure\n\tif diskSize < disk.Size {\n\t\treturn fmt.Errorf(\"specified size %q is less than the current disk size %q. Disk shrinking is currently unavailable\", units.BytesSize(float64(diskSize)), units.BytesSize(float64(disk.Size)))\n\t}\n\n\tif disk.Instance != \"\" {\n\t\tinst, err := store.Inspect(disk.Instance)\n\t\tif err == nil {\n\t\t\tif inst.Status == store.StatusRunning {\n\t\t\t\treturn fmt.Errorf(\"cannot resize disk %q used by running instance %q. Please stop the VM instance\", diskName, disk.Instance)\n\t\t\t}\n\t\t}\n\t}\n\tif err := qemu.ResizeDataDisk(disk.Dir, disk.Format, int(diskSize)); err != nil {\n\t\treturn fmt.Errorf(\"failed to resize disk %q: %w\", diskName, err)\n\t}\n\tlogrus.Infof(\"Resized disk %q (%q)\", diskName, disk.Dir)\n\treturn nil\n}\n\nfunc diskBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteDiskNames(cmd)\n}\n\n// Path: cmd/limactl/stop.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\thostagentevents \"github.com/lima-vm/lima/pkg/hostagent/events\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newStopCommand() *cobra.Command {\n\tstopCmd := &cobra.Command{\n\t\tUse:               \"stop INSTANCE\",\n\t\tShort:             \"Stop an instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              stopAction,\n\t\tValidArgsFunction: stopBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tstopCmd.Flags().BoolP(\"force\", \"f\", false, \"force stop the instance\")\n\treturn stopCmd\n}\n\nfunc stopAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif force {\n\t\tstopInstanceForcibly(inst)\n\t} else {\n\t\terr = stopInstanceGracefully(inst)\n\t}\n\t// TODO: should we also reconcile networks if graceful stop returned an error?\n\tif err == nil {\n\t\terr = networks.Reconcile(cmd.Context(), \"\")\n\t}\n\treturn err\n}\n\nfunc stopInstanceGracefully(inst *store.Instance) error {\n\tif inst.Status != store.StatusRunning {\n\t\treturn fmt.Errorf(\"expected status %q, got %q (maybe use `limactl stop -f`?)\", store.StatusRunning, inst.Status)\n\t}\n\n\tbegin := time.Now() // used for logrus propagation\n\tlogrus.Infof(\"Sending SIGINT to hostagent process %d\", inst.HostAgentPID)\n\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigInt); err != nil {\n\t\tlogrus.Error(err)\n\t}\n\n\tlogrus.Info(\"Waiting for the host agent and the driver processes to shut down\")\n\treturn waitForHostAgentTermination(context.TODO(), inst, begin)\n}\n\nfunc waitForHostAgentTermination(ctx context.Context, inst *store.Instance, begin time.Time) error {\n\tctx2, cancel := context.WithTimeout(ctx, 3*time.Minute)\n\tdefer cancel()\n\n\tvar receivedExitingEvent bool\n\tonEvent := func(ev hostagentevents.Event) bool {\n\t\tif len(ev.Status.Errors) > 0 {\n\t\t\tlogrus.Errorf(\"%+v\", ev.Status.Errors)\n\t\t}\n\t\tif ev.Status.Exiting {\n\t\t\treceivedExitingEvent = true\n\t\t\treturn true\n\t\t}\n\t\treturn false\n\t}\n\n\thaStdoutPath := filepath.Join(inst.Dir, filenames.HostAgentStdoutLog)\n\thaStderrPath := filepath.Join(inst.Dir, filenames.HostAgentStderrLog)\n\n\tif err := hostagentevents.Watch(ctx2, haStdoutPath, haStderrPath, begin, onEvent); err != nil {\n\t\treturn err\n\t}\n\n\tif !receivedExitingEvent {\n\t\treturn errors.New(\"did not receive an event with the \\\"exiting\\\" status\")\n\t}\n\n\treturn nil\n}\n\nfunc stopInstanceForcibly(inst *store.Instance) {\n\tif inst.DriverPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the %s driver process %d\", inst.VMType, inst.DriverPID)\n\t\tif err := osutil.SysKill(inst.DriverPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Infof(\"The %s driver process seems already stopped\", inst.VMType)\n\t}\n\n\tfor _, d := range inst.AdditionalDisks {\n\t\tdiskName := d.Name\n\t\tdisk, err := store.InspectDisk(diskName)\n\t\tif err != nil {\n\t\t\tlogrus.Warnf(\"Disk %q does not exist\", diskName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := disk.Unlock(); err != nil {\n\t\t\tlogrus.Warnf(\"Failed to unlock disk %q. To use, run `limactl disk unlock %v`\", diskName, diskName)\n\t\t}\n\t}\n\n\tif inst.HostAgentPID > 0 {\n\t\tlogrus.Infof(\"Sending SIGKILL to the host agent process %d\", inst.HostAgentPID)\n\t\tif err := osutil.SysKill(inst.HostAgentPID, osutil.SigKill); err != nil {\n\t\t\tlogrus.Error(err)\n\t\t}\n\t} else {\n\t\tlogrus.Info(\"The host agent process seems already stopped\")\n\t}\n\n\tsuffixesToBeRemoved := []string{\".pid\", \".sock\", \".tmp\"}\n\tlogrus.Infof(\"Removing %s under %q\", inst.Dir, strings.ReplaceAll(strings.Join(suffixesToBeRemoved, \" \"), \".\", \"*.\"))\n\tfi, err := os.ReadDir(inst.Dir)\n\tif err != nil {\n\t\tlogrus.Error(err)\n\t\treturn\n\t}\n\tfor _, f := range fi {\n\t\tpath := filepath.Join(inst.Dir, f.Name())\n\t\tfor _, suffix := range suffixesToBeRemoved {\n\t\t\tif strings.HasSuffix(path, suffix) {\n\t\t\t\tlogrus.Infof(\"Removing %q\", path)\n\t\t\t\tif err := os.Remove(path); err != nil {\n\t\t\t\t\tlogrus.Error(err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n\nfunc stopBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/delete.go\npackage main\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/stop\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDeleteCommand() *cobra.Command {\n\tdeleteCommand := &cobra.Command{\n\t\tUse:               \"delete INSTANCE [INSTANCE, ...]\",\n\t\tAliases:           []string{\"remove\", \"rm\"},\n\t\tShort:             \"Delete an instance of Lima.\",\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              deleteAction,\n\t\tValidArgsFunction: deleteBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\tdeleteCommand.Flags().BoolP(\"force\", \"f\", false, \"forcibly kill the processes\")\n\treturn deleteCommand\n}\n\nfunc deleteAction(cmd *cobra.Command, args []string) error {\n\tforce, err := cmd.Flags().GetBool(\"force\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.Warnf(\"Ignoring non-existent instance %q\", instName)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif err := deleteInstance(cmd.Context(), inst, force); err != nil {\n\t\t\treturn fmt.Errorf(\"failed to delete instance %q: %w\", instName, err)\n\t\t}\n\t\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\t\tif err != nil && !errors.Is(err, os.ErrNotExist) {\n\t\t\t\tlogrus.WithError(err).Warnf(\"The autostart file for instance %q does not exist\", instName)\n\t\t\t} else if deleted {\n\t\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t\t}\n\t\t}\n\t\tlogrus.Infof(\"Deleted %q (%q)\", instName, inst.Dir)\n\t}\n\treturn networks.Reconcile(cmd.Context(), \"\")\n}\n\nfunc deleteInstance(ctx context.Context, inst *store.Instance, force bool) error {\n\tif inst.Protected {\n\t\treturn fmt.Errorf(\"instance is protected to prohibit accidental removal (Hint: use `limactl unprotect`)\")\n\t}\n\tif !force && inst.Status != store.StatusStopped {\n\t\treturn fmt.Errorf(\"expected status %q, got %q\", store.StatusStopped, inst.Status)\n\t}\n\n\tstopInstanceForcibly(inst)\n\n\tif err := stop.Unregister(ctx, inst); err != nil {\n\t\treturn fmt.Errorf(\"failed to unregister %q: %w\", inst.Dir, err)\n\t}\n\n\tif err := os.RemoveAll(inst.Dir); err != nil {\n\t\treturn fmt.Errorf(\"failed to remove %q: %w\", inst.Dir, err)\n\t}\n\n\treturn nil\n}\n\nfunc deleteBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/editflags/editflags.go\npackage editflags\n\nimport (\n\t\"fmt\"\n\t\"math/bits\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pbnjay/memory\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n\tflag \"github.com/spf13/pflag\"\n)\n\n// RegisterEdit registers flags related to in-place YAML modification, for `limactl edit`.\nfunc RegisterEdit(cmd *cobra.Command) {\n\tregisterEdit(cmd, \"\")\n}\n\nfunc registerEdit(cmd *cobra.Command, commentPrefix string) {\n\tflags := cmd.Flags()\n\n\tflags.Int(\"cpus\", 0, commentPrefix+\"number of CPUs\") // Similar to colima's --cpu, but the flag name is slightly different (cpu vs cpus)\n\t_ = cmd.RegisterFlagCompletionFunc(\"cpus\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeCPUs(runtime.NumCPU()) {\n\t\t\tres = append(res, strconv.Itoa(f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.IPSlice(\"dns\", nil, commentPrefix+\"specify custom DNS (disable host resolver)\") // colima-compatible\n\n\tflags.Float32(\"memory\", 0, commentPrefix+\"memory in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\tvar res []string\n\t\tfor _, f := range completeMemoryGiB(memory.TotalMemory()) {\n\t\t\tres = append(res, fmt.Sprintf(\"%.1f\", f))\n\t\t}\n\t\treturn res, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.StringSlice(\"mount\", nil, commentPrefix+\"directories to mount, suffix ':w' for writable (Do not specify directories that overlap with the existing mounts)\") // colima-compatible\n\n\tflags.String(\"mount-type\", \"\", commentPrefix+\"mount type (reverse-sshfs, 9p, virtiofs)\") // Similar to colima's --mount-type=(sshfs|9p|virtiofs), but \"reverse-sshfs\" is Lima is called \"sshfs\" in colima\n\t_ = cmd.RegisterFlagCompletionFunc(\"mount-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"reverse-sshfs\", \"9p\", \"virtiofs\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"mount-writable\", false, commentPrefix+\"make all mounts writable\")\n\tflags.Bool(\"mount-inotify\", false, commentPrefix+\"enable inotify for mounts\")\n\n\tflags.StringSlice(\"network\", nil, commentPrefix+\"additional networks, e.g., \\\"vzNAT\\\" or \\\"lima:shared\\\" to assign vmnet IP\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"network\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\t// TODO: retrieve the lima:* network list from networks.yaml\n\t\treturn []string{\"lima:shared\", \"lima:bridged\", \"lima:host\", \"lima:user-v2\", \"vzNAT\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"rosetta\", false, commentPrefix+\"enable Rosetta (for vz instances)\")\n\n\tflags.String(\"set\", \"\", commentPrefix+\"modify the template inplace, using yq syntax\")\n\n\t// negative performance impact: https://gitlab.com/qemu-project/qemu/-/issues/334\n\tflags.Bool(\"video\", false, commentPrefix+\"enable video output (has negative performance impact for QEMU)\")\n}\n\n// RegisterCreate registers flags related to in-place YAML modification, for `limactl create`.\nfunc RegisterCreate(cmd *cobra.Command, commentPrefix string) {\n\tregisterEdit(cmd, commentPrefix)\n\tflags := cmd.Flags()\n\n\tflags.String(\"arch\", \"\", commentPrefix+\"machine architecture (x86_64, aarch64, riscv64)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"arch\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"x86_64\", \"aarch64\", \"riscv64\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"containerd\", \"\", commentPrefix+\"containerd mode (user, system, user+system, none)\")\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"user\", \"system\", \"user+system\", \"none\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Float32(\"disk\", 0, commentPrefix+\"disk size in GiB\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"memory\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"10\", \"30\", \"50\", \"100\", \"200\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.String(\"vm-type\", \"\", commentPrefix+\"virtual machine type (qemu, vz)\") // colima-compatible\n\t_ = cmd.RegisterFlagCompletionFunc(\"vm-type\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn []string{\"qemu\", \"vz\"}, cobra.ShellCompDirectiveNoFileComp\n\t})\n\n\tflags.Bool(\"plain\", false, commentPrefix+\"plain mode. Disable mounts, port forwarding, containerd, etc.\")\n}\n\nfunc defaultExprFunc(expr string) func(v *flag.Flag) (string, error) {\n\treturn func(v *flag.Flag) (string, error) {\n\t\treturn fmt.Sprintf(expr, v.Value), nil\n\t}\n}\n\n// YQExpressions returns YQ expressions.\nfunc YQExpressions(flags *flag.FlagSet, newInstance bool) ([]string, error) {\n\ttype def struct {\n\t\tflagName                 string\n\t\texprFunc                 func(*flag.Flag) (string, error)\n\t\tonlyValidForNewInstances bool\n\t\texperimental             bool\n\t}\n\td := defaultExprFunc\n\tdefs := []def{\n\t\t{\"cpus\", d(\".cpus = %s\"), false, false},\n\t\t{\n\t\t\t\"dns\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tipSlice, err := flags.GetIPSlice(\"dns\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.dns += [`\n\t\t\t\tfor i, ip := range ipSlice {\n\t\t\t\t\texpr += fmt.Sprintf(\"%q\", ip)\n\t\t\t\t\tif i < len(ipSlice)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .dns |= unique | .hostResolver.enabled=false`\n\t\t\t\tlogrus.Warnf(\"Disabling HostResolver, as custom DNS addresses (%v) are specified\", ipSlice)\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"memory\", d(\".memory = \\\"%sGiB\\\"\"), false, false},\n\t\t{\n\t\t\t\"mount\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"mount\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.mounts += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\twritable := strings.HasSuffix(s, \":w\")\n\t\t\t\t\tloc := strings.TrimSuffix(s, \":w\")\n\t\t\t\t\texpr += fmt.Sprintf(`{\"location\": %q, \"writable\": %v}`, loc, writable)\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .mounts |= unique_by(.location)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\"mount-type\", d(\".mountType = %q\"), false, false},\n\t\t{\"mount-inotify\", d(\".mountInotify = %s\"), false, true},\n\t\t{\"mount-writable\", d(\".mounts[].writable = %s\"), false, false},\n\t\t{\n\t\t\t\"network\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tss, err := flags.GetStringSlice(\"network\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n\t\t\t\t}\n\t\t\t\texpr := `.networks += [`\n\t\t\t\tfor i, s := range ss {\n\t\t\t\t\t// CLI syntax is still experimental (YAML syntax is out of experimental)\n\t\t\t\t\tswitch {\n\t\t\t\t\tcase s == \"vzNAT\":\n\t\t\t\t\t\texpr += `{\"vzNAT\": true}`\n\t\t\t\t\tcase strings.HasPrefix(s, \"lima:\"):\n\t\t\t\t\t\tnetwork := strings.TrimPrefix(s, \"lima:\")\n\t\t\t\t\t\texpr += fmt.Sprintf(`{\"lima\": %q}`, network)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\treturn \"\", fmt.Errorf(\"network name must be \\\"vzNAT\\\" or \\\"lima:*\\\", got %q\", s)\n\t\t\t\t\t}\n\t\t\t\t\tif i < len(ss)-1 {\n\t\t\t\t\t\texpr += \",\"\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\texpr += `] | .networks |= unique_by(.lima)`\n\t\t\t\treturn expr, nil\n\t\t\t},\n\t\t\tfalse,\n\t\t\tfalse,\n\t\t},\n\t\t{\n\t\t\t\"rosetta\",\n\t\t\tfunc(_ *flag.Flag) (string, error) {\n\t\t\t\tb, err := flags.GetBool(\"rosetta\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn \"\", err\n...\n", "needle_token_start": 10672, "needle_token_end": 10719, "code_context_ntokens": 14533, "output": ["```go\nfunc stopBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n```"]}
{"repo": "lima-vm/lima", "name": "newSnapshotCreateCommand", "language": "go", "path": "cmd/limactl/snapshot.go", "position_ratio": 0.85, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to handle the creation of a snapshot for a virtual machine instance, allowing users to save the current state of the instance for future restoration or duplication.\n2. **Input**: It requires at least one argument, the name of the instance for which the snapshot is to be created. Additionally, it accepts an optional tag to name the snapshot.\n3. **Output**: There is no direct output described in the code snippet, but typically, the function would confirm the successful creation of the snapshot or return an error message if the process fails.\n4. **Procedure**: When invoked, the function first checks for the minimum required argument (the instance name). It then proceeds to execute the snapshot creation action, which involves interfacing with the underlying virtual machine management to save the current state of the specified instance under the provided tag, if any.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/limactl/edit.go\npackage main\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/lima-vm/lima/cmd/limactl/editflags\"\n\t\"github.com/lima-vm/lima/pkg/editutil\"\n\t\"github.com/lima-vm/lima/pkg/limayaml\"\n\tnetworks \"github.com/lima-vm/lima/pkg/networks/reconcile\"\n\t\"github.com/lima-vm/lima/pkg/start\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/lima-vm/lima/pkg/uiutil\"\n\t\"github.com/lima-vm/lima/pkg/yqutil\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newEditCommand() *cobra.Command {\n\teditCommand := &cobra.Command{\n\t\tUse:               \"edit INSTANCE\",\n\t\tShort:             \"Edit an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              editAction,\n\t\tValidArgsFunction: editBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\teditflags.RegisterEdit(editCommand)\n\treturn editCommand\n}\n\n...\n// Path: cmd/limactl/editflags/editflags_test.go\npackage editflags\n\nimport (\n\t\"testing\"\n\n\t\"gotest.tools/v3/assert\"\n)\n\nfunc TestCompleteCPUs(t *testing.T) {\n\tassert.DeepEqual(t, []int{1}, completeCPUs(1))\n\tassert.DeepEqual(t, []int{1, 2}, completeCPUs(2))\n\tassert.DeepEqual(t, []int{1, 2, 4, 8}, completeCPUs(8))\n\tassert.DeepEqual(t, []int{1, 2, 4, 8, 16, 20}, completeCPUs(20))\n}\n\nfunc TestCompleteMemoryGiB(t *testing.T) {\n\tassert.DeepEqual(t, []float32{0.5}, completeMemoryGiB(1<<30))\n\tassert.DeepEqual(t, []float32{1}, completeMemoryGiB(2<<30))\n\tassert.DeepEqual(t, []float32{1, 2}, completeMemoryGiB(4<<30))\n\tassert.DeepEqual(t, []float32{1, 2, 4}, completeMemoryGiB(8<<30))\n\tassert.DeepEqual(t, []float32{1, 2, 4, 8, 10}, completeMemoryGiB(20<<30))\n}\n\n// Path: cmd/limactl/factory-reset.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newFactoryResetCommand() *cobra.Command {\n\tresetCommand := &cobra.Command{\n\t\tUse:               \"factory-reset INSTANCE\",\n\t\tShort:             \"Factory reset an instance of Lima\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              factoryResetAction,\n\t\tValidArgsFunction: factoryResetBashComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn resetCommand\n}\n\nfunc factoryResetAction(_ *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\tlogrus.Infof(\"Instance %q not found\", instName)\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\n\tstopInstanceForcibly(inst)\n\n\tfi, err := os.ReadDir(inst.Dir)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor _, f := range fi {\n\t\tpath := filepath.Join(inst.Dir, f.Name())\n\t\tif !strings.HasSuffix(path, \".yaml\") && !strings.HasSuffix(path, \".yml\") && f.Name() != filenames.VzIdentifier {\n\t\t\tlogrus.Infof(\"Removing %q\", path)\n\t\t\tif err := os.Remove(path); err != nil {\n\t\t\t\tlogrus.Error(err)\n\t\t\t}\n\t\t}\n\t}\n\tlogrus.Infof(\"Instance %q has been factory reset\", instName)\n\treturn nil\n}\n\nfunc factoryResetBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/info.go\npackage main\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/pkg/infoutil\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newInfoCommand() *cobra.Command {\n\tinfoCommand := &cobra.Command{\n\t\tUse:     \"info\",\n\t\tShort:   \"Show diagnostic information\",\n\t\tArgs:    WrapArgsError(cobra.NoArgs),\n\t\tRunE:    infoAction,\n\t\tGroupID: advancedCommand,\n\t}\n\treturn infoCommand\n}\n\nfunc infoAction(cmd *cobra.Command, _ []string) error {\n\tinfo, err := infoutil.GetInfo()\n\tif err != nil {\n\t\treturn err\n\t}\n\tj, err := json.MarshalIndent(info, \"\", \"    \")\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = fmt.Fprintln(cmd.OutOrStdout(), string(j))\n\treturn err\n}\n\n// Path: cmd/limactl/gendoc.go\npackage main\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io/fs\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/cpuguy83/go-md2man/v2/md2man\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/cobra/doc\"\n)\n\nfunc newGenDocCommand() *cobra.Command {\n\tgenmanCommand := &cobra.Command{\n\t\tUse:    \"generate-doc DIR\",\n\t\tShort:  \"Generate cli-reference pages\",\n\t\tArgs:   WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:   gendocAction,\n\t\tHidden: true,\n\t}\n\tgenmanCommand.Flags().String(\"type\", \"man\", \"Output type  (man, docsy)\")\n\tgenmanCommand.Flags().String(\"output\", \"\", \"Output directory\")\n\tgenmanCommand.Flags().String(\"prefix\", \"\", \"Install prefix\")\n\treturn genmanCommand\n}\n\nfunc gendocAction(cmd *cobra.Command, args []string) error {\n\toutput, err := cmd.Flags().GetString(\"output\")\n\tif err != nil {\n\t\treturn err\n\t}\n\toutput, err = filepath.Abs(output)\n\tif err != nil {\n\t\treturn err\n\t}\n\tprefix, err := cmd.Flags().GetString(\"prefix\")\n\tif err != nil {\n\t\treturn err\n\t}\n\toutputType, err := cmd.Flags().GetString(\"type\")\n\tif err != nil {\n\t\treturn err\n\t}\n\thomeDir, err := os.UserHomeDir()\n\tif err != nil {\n\t\treturn err\n\t}\n\tdir := args[0]\n\tswitch outputType {\n\tcase \"man\":\n\t\tif err := genMan(cmd, dir); err != nil {\n\t\t\treturn err\n\t\t}\n\tcase \"docsy\":\n\t\tif err := genDocsy(cmd, dir); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif output != \"\" && prefix != \"\" {\n\t\tif err := replaceAll(dir, output, prefix); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn replaceAll(dir, homeDir, \"~\")\n}\n\nfunc genMan(cmd *cobra.Command, dir string) error {\n\tlogrus.Infof(\"Generating man %q\", dir)\n\t// lima(1)\n\tfilePath := filepath.Join(dir, \"lima.1\")\n\tmd := \"LIMA 1\\n======\" + `\n# NAME\nlima - ` + cmd.Root().Short + `\n# SYNOPSIS\n**lima** [_COMMAND_...]\n# DESCRIPTION\nlima is an alias for \"limactl shell default\".\nThe instance name (\"default\") can be changed by specifying $LIMA_INSTANCE.\n\nThe shell and initial workdir inside the instance can be specified via $LIMA_SHELL\nand $LIMA_WORKDIR.\n# SEE ALSO\n**limactl**(1)\n`\n\tout := md2man.Render([]byte(md))\n\tif err := os.WriteFile(filePath, out, 0o644); err != nil {\n\t\treturn err\n\t}\n\t// limactl(1)\n\theader := &doc.GenManHeader{\n\t\tTitle:   \"LIMACTL\",\n\t\tSection: \"1\",\n\t}\n\treturn doc.GenManTree(cmd.Root(), header, dir)\n}\n\nfunc genDocsy(cmd *cobra.Command, dir string) error {\n\treturn doc.GenMarkdownTreeCustom(cmd.Root(), dir, func(s string) string {\n\t\t// Replace limactl_completion_bash to completion bash for docsy title\n\t\tname := filepath.Base(s)\n\t\tname = strings.ReplaceAll(name, \"limactl_\", \"\")\n\t\tname = strings.ReplaceAll(name, \"_\", \" \")\n\t\tname = strings.TrimSuffix(name, filepath.Ext(name))\n\t\treturn fmt.Sprintf(`---\ntitle: %s\nweight: 3\n---\n`, name)\n\t}, func(s string) string {\n\t\t// Use ../ for move one folder up for docsy\n\t\treturn \"../\" + strings.TrimSuffix(s, filepath.Ext(s))\n\t})\n}\n\n// replaceAll replaces all occurrences of new with old, for all files in dir\nfunc replaceAll(dir, old, new string) error {\n\tlogrus.Infof(\"Replacing %q with %q\", old, new)\n\treturn filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif path == dir {\n\t\t\treturn nil\n\t\t}\n\t\tif info.IsDir() {\n\t\t\treturn filepath.SkipDir\n\t\t}\n\t\tin, err := os.ReadFile(path)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tout := bytes.ReplaceAll(in, []byte(old), []byte(new))\n\t\terr = os.WriteFile(path, out, 0o644)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t})\n}\n\n// Path: cmd/limactl/hostagent.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"syscall\"\n\n\t\"github.com/lima-vm/lima/pkg/hostagent\"\n\t\"github.com/lima-vm/lima/pkg/hostagent/api/server\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newHostagentCommand() *cobra.Command {\n\thostagentCommand := &cobra.Command{\n\t\tUse:    \"hostagent INSTANCE\",\n\t\tShort:  \"run hostagent\",\n\t\tArgs:   WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:   hostagentAction,\n\t\tHidden: true,\n\t}\n\thostagentCommand.Flags().StringP(\"pidfile\", \"p\", \"\", \"write pid to file\")\n\thostagentCommand.Flags().String(\"socket\", \"\", \"hostagent socket\")\n\thostagentCommand.Flags().Bool(\"run-gui\", false, \"run gui synchronously within hostagent\")\n\thostagentCommand.Flags().String(\"nerdctl-archive\", \"\", \"local file path (not URL) of nerdctl-full-VERSION-GOOS-GOARCH.tar.gz\")\n\treturn hostagentCommand\n}\n\nfunc hostagentAction(cmd *cobra.Command, args []string) error {\n\tpidfile, err := cmd.Flags().GetString(\"pidfile\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif pidfile != \"\" {\n\t\tif _, err := os.Stat(pidfile); !errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"pidfile %q already exists\", pidfile)\n\t\t}\n\t\tif err := os.WriteFile(pidfile, []byte(strconv.Itoa(os.Getpid())+\"\\n\"), 0o644); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tdefer os.RemoveAll(pidfile)\n\t}\n\tsocket, err := cmd.Flags().GetString(\"socket\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif socket == \"\" {\n\t\treturn fmt.Errorf(\"socket must be specified (limactl version mismatch?)\")\n\t}\n\n\tinstName := args[0]\n\n\trunGUI, err := cmd.Flags().GetBool(\"run-gui\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif runGUI {\n\t\t// Without this the call to vz.RunGUI fails. Adding it here, as this has to be called before the vz cgo loads.\n\t\truntime.LockOSThread()\n\t\tdefer runtime.UnlockOSThread()\n\t}\n\n\tsignalCh := make(chan os.Signal, 1)\n\tsignal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)\n\n\tstdout := &syncWriter{w: cmd.OutOrStdout()}\n\tstderr := &syncWriter{w: cmd.ErrOrStderr()}\n\n\tinitLogrus(stderr)\n\tvar opts []hostagent.Opt\n\tnerdctlArchive, err := cmd.Flags().GetString(\"nerdctl-archive\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif nerdctlArchive != \"\" {\n\t\topts = append(opts, hostagent.WithNerdctlArchive(nerdctlArchive))\n\t}\n\tha, err := hostagent.New(instName, stdout, signalCh, opts...)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tbackend := &server.Backend{\n\t\tAgent: ha,\n\t}\n\tr := http.NewServeMux()\n\tserver.AddRoutes(r, backend)\n\tsrv := &http.Server{Handler: r}\n\terr = os.RemoveAll(socket)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl, err := net.Listen(\"unix\", socket)\n\tlogrus.Infof(\"hostagent socket created at %s\", socket)\n\tif err != nil {\n\t\treturn err\n\t}\n\tgo func() {\n\t\tdefer os.RemoveAll(socket)\n\t\tdefer srv.Close()\n\t\tif serveErr := srv.Serve(l); serveErr != nil {\n\t\t\tlogrus.WithError(serveErr).Warn(\"hostagent API server exited with an error\")\n\t\t}\n\t}()\n\treturn ha.Run(cmd.Context())\n}\n\n// syncer is implemented by *os.File\ntype syncer interface {\n\tSync() error\n}\n\ntype syncWriter struct {\n\tw io.Writer\n}\n\nfunc (w *syncWriter) Write(p []byte) (int, error) {\n\twritten, err := w.w.Write(p)\n\tif err == nil {\n\t\tif s, ok := w.w.(syncer); ok {\n\t\t\t_ = s.Sync()\n\t\t}\n\t}\n\treturn written, err\n}\n\nfunc initLogrus(stderr io.Writer) {\n\tlogrus.SetOutput(stderr)\n\t// JSON logs are parsed in pkg/hostagent/events.Watcher()\n\tlogrus.SetFormatter(new(logrus.JSONFormatter))\n\t// HostAgent logging is one level more verbose than the start command itself\n\tif logrus.GetLevel() == logrus.DebugLevel {\n\t\tlogrus.SetLevel(logrus.TraceLevel)\n\t} else {\n\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t}\n}\n\n// Path: cmd/limactl/list.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/cheggaaa/pb/v3/termutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc fieldNames() []string {\n\tnames := []string{}\n\tvar data store.FormatData\n\tt := reflect.TypeOf(data)\n\tfor i := 0; i < t.NumField(); i++ {\n\t\tf := t.Field(i)\n\t\tif f.Anonymous {\n\t\t\tfor j := 0; j < f.Type.NumField(); j++ {\n\t\t\t\tnames = append(names, f.Type.Field(j).Name)\n\t\t\t}\n\t\t} else {\n\t\t\tnames = append(names, t.Field(i).Name)\n\t\t}\n\t}\n\treturn names\n}\n\nfunc newListCommand() *cobra.Command {\n\tlistCommand := &cobra.Command{\n\t\tUse:     \"list [flags] [INSTANCE]...\",\n\t\tAliases: []string{\"ls\"},\n\t\tShort:   \"List instances of Lima.\",\n\t\tLong: `List instances of Lima.\n\nThe output can be presented in one of several formats, using the --format <format> flag.\n\n  --format json  - output in json format\n  --format yaml  - output in yaml format\n  --format table - output in table format\n  --format '{{ <go template> }}' - if the format begins and ends with '{{ }}', then it is used as a go template.\n` + store.FormatHelp + `\nThe following legacy flags continue to function:\n  --json - equal to '--format json'`,\n\t\tArgs:              WrapArgsError(cobra.ArbitraryArgs),\n\t\tRunE:              listAction,\n\t\tValidArgsFunction: listBashComplete,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tlistCommand.Flags().StringP(\"format\", \"f\", \"table\", \"output format, one of: json, yaml, table, go-template\")\n\tlistCommand.Flags().Bool(\"list-fields\", false, \"List fields available for format\")\n\tlistCommand.Flags().Bool(\"json\", false, \"JSONify output\")\n\tlistCommand.Flags().BoolP(\"quiet\", \"q\", false, \"Only show names\")\n\tlistCommand.Flags().Bool(\"all-fields\", false, \"Show all fields\")\n\n\treturn listCommand\n}\n\nfunc instanceMatches(arg string, instances []string) []string {\n\tmatches := []string{}\n\tfor _, instance := range instances {\n\t\tif instance == arg {\n\t\t\tmatches = append(matches, instance)\n\t\t}\n\t}\n\treturn matches\n}\n\nfunc listAction(cmd *cobra.Command, args []string) error {\n\tquiet, err := cmd.Flags().GetBool(\"quiet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tlistFields, err := cmd.Flags().GetBool(\"list-fields\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tjsonFormat, err := cmd.Flags().GetBool(\"json\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif jsonFormat {\n\t\tformat = \"json\"\n\t}\n\n\t// conflicts\n\tif jsonFormat && cmd.Flags().Changed(\"format\") {\n\t\treturn errors.New(\"option --json conflicts with option --format\")\n\t}\n\tif listFields && cmd.Flags().Changed(\"format\") {\n\t\treturn errors.New(\"option --list-fields conflicts with option --format\")\n\t}\n\n\tif quiet && format != \"table\" {\n\t\treturn errors.New(\"option --quiet can only be used with '--format table'\")\n\t}\n\n\tif listFields {\n\t\tnames := fieldNames()\n\t\tsort.Strings(names)\n\t\tfmt.Println(strings.Join(names, \"\\n\"))\n\t\treturn nil\n\t}\n\n\tallinstances, err := store.Instances()\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(allinstances) == 0 {\n\t\tlogrus.Warn(\"No instance found. Run `limactl create` to create an instance.\")\n\t\treturn nil\n\t}\n\n\tinstanceNames := []string{}\n\tif len(args) > 0 {\n\t\tfor _, arg := range args {\n\t\t\tmatches := instanceMatches(arg, allinstances)\n\t\t\tif len(matches) > 0 {\n\t\t\t\tinstanceNames = append(instanceNames, matches...)\n\t\t\t} else {\n\t\t\t\tlogrus.Warnf(\"No instance matching %v found.\", arg)\n\t\t\t}\n\t\t}\n\t} else {\n\t\tinstanceNames = allinstances\n\t}\n\n\tif quiet {\n\t\tfor _, instName := range instanceNames {\n\t\t\tfmt.Fprintln(cmd.OutOrStdout(), instName)\n\t\t}\n\t\treturn nil\n\t}\n\n\t// get the state and config for all the requested instances\n\tvar instances []*store.Instance\n\tfor _, instanceName := range instanceNames {\n\t\tinstance, err := store.Inspect(instanceName)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"unable to load instance %s: %w\", instanceName, err)\n\t\t}\n\t\tinstances = append(instances, instance)\n\t}\n\n\tfor _, instance := range instances {\n\t\tif len(instance.Errors) > 0 {\n\t\t\tlogrus.WithField(\"errors\", instance.Errors).Warnf(\"instance %q has errors\", instance.Name)\n\t\t}\n\t}\n\n\tallFields, err := cmd.Flags().GetBool(\"all-fields\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\toptions := store.PrintOptions{AllFields: allFields}\n\tout := cmd.OutOrStdout()\n\tif out == os.Stdout {\n\t\tif isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t\tif w, err := termutil.TerminalWidth(); err == nil {\n\t\t\t\toptions.TerminalWidth = w\n\t\t\t}\n\t\t}\n\t}\n\treturn store.PrintInstances(out, instances, format, &options)\n}\n\nfunc listBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/main.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/osutil\"\n\t\"github.com/lima-vm/lima/pkg/store/dirnames\"\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst (\n\tDefaultInstanceName = \"default\"\n\tbasicCommand        = \"basic\"\n\tadvancedCommand     = \"advanced\"\n)\n\nfunc main() {\n\tif err := newApp().Execute(); err != nil {\n\t\thandleExitCoder(err)\n\t\tlogrus.Fatal(err)\n\t}\n}\n\nfunc newApp() *cobra.Command {\n\ttemplatesDir := \"$PREFIX/share/lima/templates\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n\t\ttemplatesDir = filepath.Join(prefixDir, \"share/lima/templates\")\n\t}\n\n\trootCmd := &cobra.Command{\n\t\tUse:     \"limactl\",\n\t\tShort:   \"Lima: Linux virtual machines\",\n\t\tVersion: strings.TrimPrefix(version.Version, \"v\"),\n\t\tExample: fmt.Sprintf(`  Start the default instance:\n  $ limactl start\n\n  Open a shell:\n  $ lima\n\n  Run a container:\n  $ lima nerdctl run -d --name nginx -p 8080:80 nginx:alpine\n\n  Stop the default instance:\n  $ limactl stop\n\n  See also template YAMLs: %s`, templatesDir),\n\t\tSilenceUsage:      true,\n\t\tSilenceErrors:     true,\n\t\tDisableAutoGenTag: true,\n\t}\n\trootCmd.PersistentFlags().String(\"log-level\", \"\", \"Set the logging level [trace, debug, info, warn, error]\")\n\trootCmd.PersistentFlags().Bool(\"debug\", false, \"debug mode\")\n\t// TODO: \"survey\" does not support using cygwin terminal on windows yet\n\trootCmd.PersistentFlags().Bool(\"tty\", isatty.IsTerminal(os.Stdout.Fd()), \"Enable TUI interactions such as opening an editor. Defaults to true when stdout is a terminal. Set to false for automation.\")\n\trootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n\t\tl, _ := cmd.Flags().GetString(\"log-level\")\n\t\tif l != \"\" {\n\t\t\tlvl, err := logrus.ParseLevel(l)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tlogrus.SetLevel(lvl)\n\t\t}\n\t\tdebug, _ := cmd.Flags().GetBool(\"debug\")\n\t\tif debug {\n\t\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t\t}\n\n\t\tif osutil.IsBeingRosettaTranslated() {\n\t\t\t// running under rosetta would provide inappropriate runtime.GOARCH info, see: https://github.com/lima-vm/lima/issues/543\n\t\t\treturn errors.New(\"limactl is running under rosetta, please reinstall lima with native arch\")\n\t\t}\n\n\t\tif runtime.GOOS == \"windows\" && isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t\tformatter := new(logrus.TextFormatter)\n\t\t\t// the default setting does not recognize cygwin on windows\n\t\t\tformatter.ForceColors = true\n\t\t\tlogrus.StandardLogger().SetFormatter(formatter)\n\t\t}\n\t\tif os.Geteuid() == 0 && cmd.Name() != \"generate-doc\" {\n\t\t\treturn errors.New(\"must not run as the root\")\n\t\t}\n\t\t// Make sure either $HOME or $LIMA_HOME is defined, so we don't need\n\t\t// to check for errors later\n\t\tif _, err := dirnames.LimaDir(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\trootCmd.AddGroup(&cobra.Group{ID: \"basic\", Title: \"Basic Commands:\"})\n\trootCmd.AddGroup(&cobra.Group{ID: \"advanced\", Title: \"Advanced Commands:\"})\n\trootCmd.AddCommand(\n\t\tnewCreateCommand(),\n\t\tnewStartCommand(),\n\t\tnewStopCommand(),\n\t\tnewShellCommand(),\n\t\tnewCopyCommand(),\n\t\tnewListCommand(),\n\t\tnewDeleteCommand(),\n\t\tnewValidateCommand(),\n\t\tnewSudoersCommand(),\n\t\tnewPruneCommand(),\n\t\tnewHostagentCommand(),\n\t\tnewInfoCommand(),\n\t\tnewShowSSHCommand(),\n\t\tnewDebugCommand(),\n\t\tnewEditCommand(),\n\t\tnewFactoryResetCommand(),\n\t\tnewDiskCommand(),\n\t\tnewUsernetCommand(),\n\t\tnewGenDocCommand(),\n\t\tnewSnapshotCommand(),\n\t\tnewProtectCommand(),\n\t\tnewUnprotectCommand(),\n\t)\n\tif runtime.GOOS == \"darwin\" || runtime.GOOS == \"linux\" {\n\t\trootCmd.AddCommand(startAtLoginCommand())\n\t}\n\n\treturn rootCmd\n}\n\ntype ExitCoder interface {\n\terror\n\tExitCode() int\n}\n\nfunc handleExitCoder(err error) {\n\tif err == nil {\n\t\treturn\n\t}\n\n\tif exitErr, ok := err.(ExitCoder); ok {\n\t\tos.Exit(exitErr.ExitCode())\n\t\treturn\n\t}\n}\n\n// WrapArgsError annotates cobra args error with some context, so the error message is more user-friendly\nfunc WrapArgsError(argFn cobra.PositionalArgs) cobra.PositionalArgs {\n\treturn func(cmd *cobra.Command, args []string) error {\n\t\terr := argFn(cmd, args)\n\t\tif err == nil {\n\t\t\treturn nil\n\t\t}\n\n\t\treturn fmt.Errorf(\"%q %s.\\nSee '%s --help'.\\n\\nUsage:  %s\\n\\n%s\",\n\t\t\tcmd.CommandPath(), err.Error(),\n\t\t\tcmd.CommandPath(),\n\t\t\tcmd.UseLine(), cmd.Short,\n\t\t)\n\t}\n}\n\n// Path: cmd/limactl/protect.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newProtectCommand() *cobra.Command {\n\tprotectCommand := &cobra.Command{\n\t\tUse:   \"protect INSTANCE [INSTANCE, ...]\",\n\t\tShort: \"Protect an instance to prohibit accidental removal\",\n\t\tLong: `Protect an instance to prohibit accidental removal via the 'limactl delete' command.\nThe instance is not being protected against removal via '/bin/rm', Finder, etc.`,\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              protectAction,\n\t\tValidArgsFunction: protectBashComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn protectCommand\n}\n\nfunc protectAction(_ *cobra.Command, args []string) error {\n\tvar errs []error\n\tfor _, instName := range args {\n\t\tinst, err := store.Inspect(instName)\n\t\tif err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to inspect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tif inst.Protected {\n\t\t\tlogrus.Warnf(\"Instance %q is already protected. Skipping.\", instName)\n\t\t\tcontinue\n\t\t}\n\t\tif err := inst.Protect(); err != nil {\n\t\t\terrs = append(errs, fmt.Errorf(\"failed to protect instance %q: %w\", instName, err))\n\t\t\tcontinue\n\t\t}\n\t\tlogrus.Infof(\"Protected %q\", instName)\n\t}\n\treturn errors.Join(errs...)\n}\n\nfunc protectBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/prune.go\npackage main\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newPruneCommand() *cobra.Command {\n\tpruneCommand := &cobra.Command{\n\t\tUse:               \"prune\",\n\t\tShort:             \"Prune garbage objects\",\n\t\tArgs:              WrapArgsError(cobra.NoArgs),\n\t\tRunE:              pruneAction,\n\t\tValidArgsFunction: cobra.NoFileCompletions,\n\t\tGroupID:           advancedCommand,\n\t}\n\treturn pruneCommand\n}\n\nfunc pruneAction(_ *cobra.Command, _ []string) error {\n\tucd, err := os.UserCacheDir()\n\tif err != nil {\n\t\treturn err\n\t}\n\tcacheDir := filepath.Join(ucd, \"lima\")\n\tlogrus.Infof(\"Pruning %q\", cacheDir)\n\treturn os.RemoveAll(cacheDir)\n}\n\n// Path: cmd/limactl/shell.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/alessio/shellescape\"\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/mattn/go-isatty\"\n\t\"github.com/mattn/go-shellwords\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\n// Environment variable that allows configuring the command (alias) to execute\n// in place of the 'ssh' executable.\nconst envShellSSH = \"SSH\"\n\nconst shellHelp = `Execute shell in Lima\n\nlima command is provided as an alias for limactl shell $LIMA_INSTANCE. $LIMA_INSTANCE defaults to \"` + DefaultInstanceName + `\".\n\nBy default, the first 'ssh' executable found in the host's PATH is used to connect to the Lima instance.\nA custom ssh alias can be used instead by setting the $` + envShellSSH + ` environment variable.\n\nHint: try --debug to show the detailed logs, if it seems hanging (mostly due to some SSH issue).\n`\n\nfunc newShellCommand() *cobra.Command {\n\tshellCmd := &cobra.Command{\n\t\tUse:               \"shell [flags] INSTANCE [COMMAND...]\",\n\t\tShort:             \"Execute shell in Lima\",\n\t\tLong:              shellHelp,\n\t\tArgs:              WrapArgsError(cobra.MinimumNArgs(1)),\n\t\tRunE:              shellAction,\n\t\tValidArgsFunction: shellBashComplete,\n\t\tSilenceErrors:     true,\n\t\tGroupID:           basicCommand,\n\t}\n\n\tshellCmd.Flags().SetInterspersed(false)\n\n\tshellCmd.Flags().String(\"shell\", \"\", \"shell interpreter, e.g. /bin/bash\")\n\tshellCmd.Flags().String(\"workdir\", \"\", \"working directory\")\n\treturn shellCmd\n}\n\nfunc shellAction(cmd *cobra.Command, args []string) error {\n\t// simulate the behavior of double dash\n\tnewArg := []string{}\n\tif len(args) >= 2 && args[1] == \"--\" {\n\t\tnewArg = append(newArg, args[:1]...)\n\t\tnewArg = append(newArg, args[2:]...)\n\t\targs = newArg\n\t}\n\tinstName := args[0]\n\n\tif len(args) >= 2 {\n\t\tswitch args[1] {\n\t\tcase \"create\", \"start\", \"delete\", \"shell\":\n\t\t\t// `lima start` (alias of `limactl $LIMA_INSTANCE start`) is probably a typo of `limactl start`\n\t\t\tlogrus.Warnf(\"Perhaps you meant `limactl %s`?\", strings.Join(args[1:], \" \"))\n\t\t}\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t}\n\t\treturn err\n\t}\n\tif inst.Status == store.StatusStopped {\n\t\treturn fmt.Errorf(\"instance %q is stopped, run `limactl start %s` to start the instance\", instName, instName)\n\t}\n\ty, err := inst.LoadYAML()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// When workDir is explicitly set, the shell MUST have workDir as the cwd, or exit with an error.\n\t//\n\t// changeDirCmd := \"cd workDir || exit 1\"                  if workDir != \"\"\n\t//              := \"cd hostCurrentDir || cd hostHomeDir\"   if workDir == \"\"\n\tvar changeDirCmd string\n\tworkDir, err := cmd.Flags().GetString(\"workdir\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif workDir != \"\" {\n\t\tchangeDirCmd = fmt.Sprintf(\"cd %s || exit 1\", shellescape.Quote(workDir))\n\t\t// FIXME: check whether y.Mounts contains the home, not just len > 0\n\t} else if len(y.Mounts) > 0 {\n\t\thostCurrentDir, err := os.Getwd()\n\t\tif err == nil {\n\t\t\tchangeDirCmd = fmt.Sprintf(\"cd %s\", shellescape.Quote(hostCurrentDir))\n\t\t} else {\n\t\t\tchangeDirCmd = \"false\"\n\t\t\tlogrus.WithError(err).Warn(\"failed to get the current directory\")\n\t\t}\n\t\thostHomeDir, err := os.UserHomeDir()\n\t\tif err == nil {\n\t\t\tchangeDirCmd = fmt.Sprintf(\"%s || cd %s\", changeDirCmd, shellescape.Quote(hostHomeDir))\n\t\t} else {\n\t\t\tlogrus.WithError(err).Warn(\"failed to get the home directory\")\n\t\t}\n\t} else {\n\t\tlogrus.Debug(\"the host home does not seem mounted, so the guest shell will have a different cwd\")\n\t}\n\n\tif changeDirCmd == \"\" {\n\t\tchangeDirCmd = \"false\"\n\t}\n\tlogrus.Debugf(\"changeDirCmd=%q\", changeDirCmd)\n\n\tshell, err := cmd.Flags().GetString(\"shell\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif shell == \"\" {\n\t\tshell = `\"$SHELL\"`\n\t} else {\n\t\tshell = shellescape.Quote(shell)\n\t}\n\tscript := fmt.Sprintf(\"%s ; exec %s --login\", changeDirCmd, shell)\n\tif len(args) > 1 {\n\t\tquotedArgs := make([]string, len(args[1:]))\n\t\tparsingEnv := true\n\t\tfor i, arg := range args[1:] {\n\t\t\tif parsingEnv && isEnv(arg) {\n\t\t\t\tquotedArgs[i] = quoteEnv(arg)\n\t\t\t} else {\n\t\t\t\tparsingEnv = false\n\t\t\t\tquotedArgs[i] = shellescape.Quote(arg)\n\t\t\t}\n\t\t}\n\t\tscript += fmt.Sprintf(\n\t\t\t\" -c %s\",\n\t\t\tshellescape.Quote(strings.Join(quotedArgs, \" \")),\n\t\t)\n\t}\n\n\tvar arg0 string\n\tvar arg0Args []string\n\n\tif sshShell := os.Getenv(envShellSSH); sshShell != \"\" {\n\t\tsshShellFields, err := shellwords.Parse(sshShell)\n\t\tswitch {\n\t\tcase err != nil:\n\t\t\tlogrus.WithError(err).Warnf(\"Failed to split %s variable into shell tokens. \"+\n\t\t\t\t\"Falling back to 'ssh' command\", envShellSSH)\n\t\tcase len(sshShellFields) > 0:\n\t\t\targ0 = sshShellFields[0]\n\t\t\tif len(sshShellFields) > 1 {\n\t\t\t\targ0Args = sshShellFields[1:]\n\t\t\t}\n\t\t}\n\t}\n\n\tif arg0 == \"\" {\n\t\targ0, err = exec.LookPath(\"ssh\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tsshOpts, err := sshutil.SSHOpts(inst.Dir, *y.SSH.LoadDotSSHPubKeys, *y.SSH.ForwardAgent, *y.SSH.ForwardX11, *y.SSH.ForwardX11Trusted)\n\tif err != nil {\n\t\treturn err\n\t}\n\tsshArgs := sshutil.SSHArgsFromOpts(sshOpts)\n\tif isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {\n\t\t// required for showing the shell prompt: https://stackoverflow.com/a/626574\n\t\tsshArgs = append(sshArgs, \"-t\")\n\t}\n\tif _, present := os.LookupEnv(\"COLORTERM\"); present {\n\t\t// SendEnv config is cumulative, with already existing options in ssh_config\n\t\tsshArgs = append(sshArgs, \"-o\", \"SendEnv=COLORTERM\")\n\t}\n\tsshArgs = append(sshArgs, []string{\n\t\t\"-q\",\n\t\t\"-p\", strconv.Itoa(inst.SSHLocalPort),\n\t\tinst.SSHAddress,\n\t\t\"--\",\n\t\tscript,\n\t}...)\n\tsshCmd := exec.Command(arg0, append(arg0Args, sshArgs...)...)\n\tsshCmd.Stdin = os.Stdin\n\tsshCmd.Stdout = os.Stdout\n\tsshCmd.Stderr = os.Stderr\n\tlogrus.Debugf(\"executing ssh (may take a long)): %+v\", sshCmd.Args)\n\n\t// TODO: use syscall.Exec directly (results in losing tty?)\n\treturn sshCmd.Run()\n}\n\nfunc shellBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\nfunc isEnv(arg string) bool {\n\treturn len(strings.Split(arg, \"=\")) > 1\n}\n\nfunc quoteEnv(arg string) string {\n\tenv := strings.SplitN(arg, \"=\", 2)\n\tenv[1] = shellescape.Quote(env[1])\n\treturn strings.Join(env, \"=\")\n}\n\n// Path: cmd/limactl/show_ssh.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/sshutil\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/store/dirnames\"\n\t\"github.com/lima-vm/lima/pkg/store/filenames\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nconst showSSHExample = `\n  \"cmd\" format (default): Full ssh command line.\n    $ limactl show-ssh --format=cmd default\n    ssh -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022 lima-default\n\n  \"args\" format: Similar to the cmd format but omits \"ssh\" and the destination address\n    $ limactl show-ssh --format=args default\n    -o IdentityFile=\"/Users/example/.lima/_config/user\" -o User=example -o Hostname=127.0.0.1 -o Port=60022\n\n  \"options\" format: ssh option key value pairs\n    $ limactl show-ssh --format=options default\n    IdentityFile=\"/Users/example/.lima/_config/user\"\n    User=example\n    Hostname=127.0.0.1\n    Port=60022\n\n  \"config\" format: ~/.ssh/config format\n    $ limactl show-ssh --format=config default\n    Host lima-default\n      IdentityFile \"/Users/example/.lima/_config/user \"\n      User example\n      Hostname 127.0.0.1\n      Port 60022\n\n  To show the config file path:\n    $ limactl ls --format='{{.SSHConfigFile}}' default\n    /Users/example/.lima/default/ssh.config\n`\n\nfunc newShowSSHCommand() *cobra.Command {\n\tlimaHome := \"~/\" + dirnames.DotLima\n\tif s, err := dirnames.LimaDir(); err == nil {\n\t\tlimaHome = s\n\t}\n\tshellCmd := &cobra.Command{\n\t\tUse:   \"show-ssh [flags] INSTANCE\",\n\t\tShort: \"Show the ssh command line (DEPRECATED; use `ssh -F` instead)\",\n\t\tLong: fmt.Sprintf(`Show the ssh command line (DEPRECATED)\n\nWARNING: 'limactl show-ssh' is deprecated.\nInstead, use 'ssh -F %s/default/ssh.config lima-default' .\n`, limaHome),\n\t\tExample:           showSSHExample,\n\t\tArgs:              WrapArgsError(cobra.ExactArgs(1)),\n\t\tRunE:              showSSHAction,\n\t\tValidArgsFunction: showSSHBashComplete,\n\t\tSilenceErrors:     true,\n\t\tGroupID:           advancedCommand,\n\t}\n\n\tshellCmd.Flags().StringP(\"format\", \"f\", sshutil.FormatCmd, \"Format: \"+strings.Join(sshutil.Formats, \", \"))\n\t_ = shellCmd.RegisterFlagCompletionFunc(\"format\", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {\n\t\treturn sshutil.Formats, cobra.ShellCompDirectiveNoFileComp\n\t})\n\treturn shellCmd\n}\n\nfunc showSSHAction(cmd *cobra.Command, args []string) error {\n\tformat, err := cmd.Flags().GetString(\"format\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tinstName := args[0]\n\tw := cmd.OutOrStdout()\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\treturn fmt.Errorf(\"instance %q does not exist, run `limactl create %s` to create a new instance\", instName, instName)\n\t\t}\n\t\treturn err\n\t}\n\tlogrus.Warnf(\"`limactl show-ssh` is deprecated. Instead, use `ssh -F %s lima-%s`.\",\n\t\tfilepath.Join(inst.Dir, filenames.SSHConfig), inst.Name)\n\ty, err := inst.LoadYAML()\n\tif err != nil {\n\t\treturn err\n\t}\n\topts, err := sshutil.SSHOpts(inst.Dir, *y.SSH.LoadDotSSHPubKeys, *y.SSH.ForwardAgent, *y.SSH.ForwardX11, *y.SSH.ForwardX11Trusted)\n\tif err != nil {\n\t\treturn err\n\t}\n\topts = append(opts, \"Hostname=127.0.0.1\")\n\topts = append(opts, fmt.Sprintf(\"Port=%d\", inst.SSHLocalPort))\n\treturn sshutil.Format(w, instName, format, opts)\n}\n\nfunc showSSHBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/snapshot.go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/snapshot\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSnapshotCommand() *cobra.Command {\n\tsnapshotCmd := &cobra.Command{\n\t\tUse:   \"snapshot\",\n\t\tShort: \"Manage instance snapshots\",\n\t\tPersistentPreRun: func(cmd *cobra.Command, args []string) {\n\t\t\tlogrus.Warn(\"`limactl snapshot` is experimental\")\n\t\t},\n\t\tGroupID: advancedCommand,\n\t}\n\tsnapshotCmd.AddCommand(newSnapshotApplyCommand())\n\tsnapshotCmd.AddCommand(newSnapshotCreateCommand())\n\tsnapshotCmd.AddCommand(newSnapshotDeleteCommand())\n\tsnapshotCmd.AddCommand(newSnapshotListCommand())\n\n\treturn snapshotCmd\n}\n\n\nfunc newSnapshotCreateCommand() *cobra.Command {\n\tcreateCmd := &cobra.Command{\n\t\tUse:               \"create INSTANCE\",\n\t\tAliases:           []string{\"save\"},\n\t\tShort:             \"Create (save) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotCreateAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tcreateCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn createCmd\n}\n\nfunc snapshotCreateAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Save(ctx, inst, tag)\n}\n\nfunc newSnapshotDeleteCommand() *cobra.Command {\n\tdeleteCmd := &cobra.Command{\n\t\tUse:               \"delete INSTANCE\",\n\t\tAliases:           []string{\"del\"},\n\t\tShort:             \"Delete (del) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotDeleteAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tdeleteCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn deleteCmd\n}\n\nfunc snapshotDeleteAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Del(ctx, inst, tag)\n}\n\nfunc newSnapshotApplyCommand() *cobra.Command {\n\tapplyCmd := &cobra.Command{\n\t\tUse:               \"apply INSTANCE\",\n\t\tAliases:           []string{\"load\"},\n\t\tShort:             \"Apply (load) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotApplyAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tapplyCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn applyCmd\n}\n\nfunc snapshotApplyAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Load(ctx, inst, tag)\n}\n\nfunc newSnapshotListCommand() *cobra.Command {\n\tlistCmd := &cobra.Command{\n\t\tUse:               \"list INSTANCE\",\n\t\tAliases:           []string{\"ls\"},\n\t\tShort:             \"List existing snapshots\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotListAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tlistCmd.Flags().BoolP(\"quiet\", \"q\", false, \"Only show tags\")\n\n\treturn listCmd\n}\n\nfunc snapshotListAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tquiet, err := cmd.Flags().GetBool(\"quiet\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tctx := cmd.Context()\n\tout, err := snapshot.List(ctx, inst)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif quiet {\n\t\tfor i, line := range strings.Split(out, \"\\n\") {\n\t\t\t// \"ID\", \"TAG\", \"VM SIZE\", \"DATE\", \"VM CLOCK\", \"ICOUNT\"\n\t\t\tfields := strings.Fields(line)\n\t\t\tif i == 0 && len(fields) > 1 && fields[1] != \"TAG\" {\n\t\t\t\t// make sure that output matches the expected\n\t\t\t\treturn fmt.Errorf(\"unknown header: %s\", line)\n\t\t\t}\n\t\t\tif i == 0 || line == \"\" {\n\t\t\t\t// skip header and empty line after using split\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttag := fields[1]\n\t\t\tfmt.Printf(\"%s\\n\", tag)\n\t\t}\n\t\treturn nil\n\t}\n\tfmt.Print(out)\n\treturn nil\n}\n\nfunc snapshotBashComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/start-at-login.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"os\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/autostart\"\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc startAtLoginCommand() *cobra.Command {\n\tstartAtLoginCommand := &cobra.Command{\n\t\tUse:               \"start-at-login INSTANCE\",\n\t\tShort:             \"Register/Unregister an autostart file for the instance\",\n\t\tArgs:              WrapArgsError(cobra.MaximumNArgs(1)),\n\t\tRunE:              startAtLoginAction,\n\t\tValidArgsFunction: startAtLoginComplete,\n\t\tGroupID:           advancedCommand,\n\t}\n\n\tstartAtLoginCommand.Flags().Bool(\n\t\t\"enabled\", true,\n\t\t\"Automatically start the instance when the user logs in\",\n\t)\n\n\treturn startAtLoginCommand\n}\n\nfunc startAtLoginAction(cmd *cobra.Command, args []string) error {\n\tinstName := DefaultInstanceName\n\tif len(args) > 0 {\n\t\tinstName = args[0]\n\t}\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\tif errors.Is(err, os.ErrNotExist) {\n\t\t\tlogrus.Infof(\"Instance %q not found\", instName)\n\t\t\treturn nil\n\t\t}\n\t\treturn err\n\t}\n\n\tflags := cmd.Flags()\n\tstartAtLogin, err := flags.GetBool(\"enabled\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif startAtLogin {\n\t\tif err := autostart.CreateStartAtLoginEntry(runtime.GOOS, inst.Name, inst.Dir); err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"Can't create an autostart file for instance %q\", inst.Name)\n\t\t} else {\n\t\t\tlogrus.Infof(\"The autostart file %q has been created or updated\", autostart.GetFilePath(runtime.GOOS, inst.Name))\n\t\t}\n\t} else {\n\t\tdeleted, err := autostart.DeleteStartAtLoginEntry(runtime.GOOS, instName)\n\t\tif err != nil {\n\t\t\tlogrus.WithError(err).Warnf(\"The autostart file %q could not be deleted\", instName)\n\t\t} else if deleted {\n\t\t\tlogrus.Infof(\"The autostart file %q has been deleted\", autostart.GetFilePath(runtime.GOOS, instName))\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc startAtLoginComplete(cmd *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {\n\treturn bashCompleteInstanceNames(cmd)\n}\n\n// Path: cmd/limactl/sudoers.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime\"\n\n\t\"github.com/lima-vm/lima/pkg/networks\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newSudoersCommand() *cobra.Command {\n\tnetworksMD := \"$PREFIX/share/doc/lima/docs/network.md\"\n\tif exe, err := os.Executable(); err == nil {\n\t\tbinDir := filepath.Dir(exe)\n\t\tprefixDir := filepath.Dir(binDir)\n...\n", "needle_token_start": 12831, "needle_token_end": 12964, "code_context_ntokens": 15129, "output": ["```go\nfunc newSnapshotCreateCommand() *cobra.Command {\n\tcreateCmd := &cobra.Command{\n\t\tUse:               \"create INSTANCE\",\n\t\tAliases:           []string{\"save\"},\n\t\tShort:             \"Create (save) a snapshot\",\n\t\tArgs:              cobra.MinimumNArgs(1),\n\t\tRunE:              snapshotCreateAction,\n\t\tValidArgsFunction: snapshotBashComplete,\n\t}\n\tcreateCmd.Flags().String(\"tag\", \"\", \"name of the snapshot\")\n\n\treturn createCmd\n}\n\nfunc snapshotCreateAction(cmd *cobra.Command, args []string) error {\n\tinstName := args[0]\n\n\tinst, err := store.Inspect(instName)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttag, err := cmd.Flags().GetString(\"tag\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif tag == \"\" {\n\t\treturn fmt.Errorf(\"expected tag\")\n\t}\n\n\tctx := cmd.Context()\n\treturn snapshot.Save(ctx, inst, tag)\n}\n```"]}
{"repo": "lima-vm/lima", "name": "generateSystemdUnit", "language": "go", "path": "cmd/lima-guestagent/install_systemd_linux.go", "position_ratio": 0.95, "description": "\nFunction Description:\n1. **Purpose**: The function is designed to create a systemd unit configuration for a service that can communicate using either a VSOCK or a VirtIO serial interface, or default to a UNIX socket if neither is specified.\n2. **Input**: It accepts two parameters: an integer representing the VSOCK port and a string for the VirtIO port.\n3. **Output**: Returns a byte array containing the systemd unit file content, or an error if the operation fails.\n4. **Procedure**: The function first determines the executable path of the current process. It then constructs the command-line arguments based on the provided VSOCK and VirtIO port values. Using these values, it populates a template for the systemd unit file. The populated template is processed to replace placeholders with actual values, resulting in the final configuration content.\n\n", "instruction": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:", "template": "instruction\ncode_context\ndescription\ninstruction", "code_context": "// Path: cmd/lima-guestagent/daemon_linux.go\npackage main\n\nimport (\n\t\"errors\"\n\t\"net\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/lima-vm/lima/pkg/guestagent\"\n\t\"github.com/lima-vm/lima/pkg/guestagent/api/server\"\n\t\"github.com/lima-vm/lima/pkg/guestagent/serialport\"\n\t\"github.com/mdlayher/vsock\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newDaemonCommand() *cobra.Command {\n\tdaemonCommand := &cobra.Command{\n\t\tUse:   \"daemon\",\n\t\tShort: \"run the daemon\",\n\t\tRunE:  daemonAction,\n\t}\n\tdaemonCommand.Flags().Duration(\"tick\", 3*time.Second, \"tick for polling events\")\n\tdaemonCommand.Flags().Int(\"vsock-port\", 0, \"use vsock server instead a UNIX socket\")\n\tdaemonCommand.Flags().String(\"virtio-port\", \"\", \"use virtio server instead a UNIX socket\")\n\treturn daemonCommand\n}\n\nfunc daemonAction(cmd *cobra.Command, _ []string) error {\n\tsocket := \"/run/lima-guestagent.sock\"\n\ttick, err := cmd.Flags().GetDuration(\"tick\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvSockPort, err := cmd.Flags().GetInt(\"vsock-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvirtioPort, err := cmd.Flags().GetString(\"virtio-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tif tick == 0 {\n\t\treturn errors.New(\"tick must be specified\")\n\t}\n\tif os.Geteuid() != 0 {\n\t\treturn errors.New(\"must run as the root\")\n\t}\n\tlogrus.Infof(\"event tick: %v\", tick)\n\n\tnewTicker := func() (<-chan time.Time, func()) {\n\t\t// TODO: use an equivalent of `bpftrace -e 'tracepoint:syscalls:sys_*_bind { printf(\"tick\\n\"); }')`,\n\t\t// without depending on `bpftrace` binary.\n\t\t// The agent binary will need CAP_BPF file cap.\n\t\tticker := time.NewTicker(tick)\n\t\treturn ticker.C, ticker.Stop\n\t}\n\n\tagent, err := guestagent.New(newTicker, tick*20)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = os.RemoveAll(socket)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar l net.Listener\n\tif virtioPort != \"\" {\n\t\tqemuL, err := serialport.Listen(\"/dev/virtio-ports/\" + virtioPort)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = qemuL\n\t\tlogrus.Infof(\"serving the guest agent on qemu serial file: %s\", virtioPort)\n\t} else if vSockPort != 0 {\n\t\tvsockL, err := vsock.Listen(uint32(vSockPort), nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = vsockL\n\t\tlogrus.Infof(\"serving the guest agent on vsock port: %d\", vSockPort)\n\t} else {\n\t\tsocketL, err := net.Listen(\"unix\", socket)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := os.Chmod(socket, 0o777); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tl = socketL\n\t\tlogrus.Infof(\"serving the guest agent on %q\", socket)\n\t}\n\treturn server.StartServer(l, &server.GuestServer{Agent: agent})\n}\n\n// Path: cmd/lima-guestagent/install_systemd_linux.go\npackage main\n\nimport (\n\t_ \"embed\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/textutil\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc newInstallSystemdCommand() *cobra.Command {\n\tinstallSystemdCommand := &cobra.Command{\n\t\tUse:   \"install-systemd\",\n\t\tShort: \"install a systemd unit (user)\",\n\t\tRunE:  installSystemdAction,\n\t}\n\tinstallSystemdCommand.Flags().Int(\"vsock-port\", 0, \"use vsock server on specified port\")\n\tinstallSystemdCommand.Flags().String(\"virtio-port\", \"\", \"use virtio server instead a UNIX socket\")\n\treturn installSystemdCommand\n}\n\nfunc installSystemdAction(cmd *cobra.Command, _ []string) error {\n\tvsockPort, err := cmd.Flags().GetInt(\"vsock-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tvirtioPort, err := cmd.Flags().GetString(\"virtio-port\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tunit, err := generateSystemdUnit(vsockPort, virtioPort)\n\tif err != nil {\n\t\treturn err\n\t}\n\tunitPath := \"/etc/systemd/system/lima-guestagent.service\"\n\tif _, err := os.Stat(unitPath); !errors.Is(err, os.ErrNotExist) {\n\t\tlogrus.Infof(\"File %q already exists, overwriting\", unitPath)\n\t} else {\n\t\tunitDir := filepath.Dir(unitPath)\n\t\tif err := os.MkdirAll(unitDir, 0o755); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif err := os.WriteFile(unitPath, unit, 0o644); err != nil {\n\t\treturn err\n\t}\n\tlogrus.Infof(\"Written file %q\", unitPath)\n\targs := [][]string{\n\t\t{\"daemon-reload\"},\n\t\t{\"enable\", \"--now\", \"lima-guestagent.service\"},\n\t\t{\"try-restart\", \"lima-guestagent.service\"},\n\t}\n\tfor _, args := range args {\n\t\tcmd := exec.Command(\"systemctl\", append([]string{\"--system\"}, args...)...)\n\t\tcmd.Stdout = os.Stdout\n\t\tcmd.Stderr = os.Stderr\n\t\tlogrus.Infof(\"Executing: %s\", strings.Join(cmd.Args, \" \"))\n\t\tif err := cmd.Run(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tlogrus.Info(\"Done\")\n\treturn nil\n}\n\n//go:embed lima-guestagent.TEMPLATE.service\nvar systemdUnitTemplate string\n\n\nfunc generateSystemdUnit(vsockPort int, virtioPort string) ([]byte, error) {\n\tselfExeAbs, err := os.Executable()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar args []string\n\tif vsockPort != 0 {\n\t\targs = append(args, fmt.Sprintf(\"--vsock-port %d\", vsockPort))\n\t}\n\tif virtioPort != \"\" {\n\t\targs = append(args, fmt.Sprintf(\"--virtio-port %s\", virtioPort))\n\t}\n\n\tm := map[string]string{\n\t\t\"Binary\": selfExeAbs,\n\t\t\"Args\":   strings.Join(args, \" \"),\n\t}\n\treturn textutil.ExecuteTemplate(systemdUnitTemplate, m)\n}\n\n// Path: cmd/lima-guestagent/main_linux.go\npackage main\n\nimport (\n\t\"strings\"\n\n\t\"github.com/lima-vm/lima/pkg/version\"\n\t\"github.com/sirupsen/logrus\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc main() {\n\tif err := newApp().Execute(); err != nil {\n\t\tlogrus.Fatal(err)\n\t}\n}\n\nfunc newApp() *cobra.Command {\n\trootCmd := &cobra.Command{\n\t\tUse:     \"lima-guestagent\",\n\t\tShort:   \"Do not launch manually\",\n\t\tVersion: strings.TrimPrefix(version.Version, \"v\"),\n\t}\n\trootCmd.PersistentFlags().Bool(\"debug\", false, \"debug mode\")\n\trootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n\t\tdebug, _ := cmd.Flags().GetBool(\"debug\")\n\t\tif debug {\n\t\t\tlogrus.SetLevel(logrus.DebugLevel)\n\t\t}\n\t\treturn nil\n\t}\n\trootCmd.AddCommand(\n\t\tnewDaemonCommand(),\n\t\tnewInstallSystemdCommand(),\n\t)\n\treturn rootCmd\n}\n\n// Path: cmd/limactl/completion.go\npackage main\n\nimport (\n\t\"github.com/lima-vm/lima/pkg/store\"\n\t\"github.com/lima-vm/lima/pkg/templatestore\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc bashCompleteInstanceNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tinstances, err := store.Instances()\n\tif err != nil {\n\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t}\n\treturn instances, cobra.ShellCompDirectiveNoFileComp\n}\n\nfunc bashCompleteTemplateNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tvar comp []string\n\tif templates, err := templatestore.Templates(); err == nil {\n\t\tfor _, f := range templates {\n\t\t\tcomp = append(comp, \"template://\"+f.Name)\n\t\t}\n\t}\n\treturn comp, cobra.ShellCompDirectiveDefault\n}\n\nfunc bashCompleteDiskNames(_ *cobra.Command) ([]string, cobra.ShellCompDirective) {\n\tdisks, err := store.Disks()\n\tif err != nil {\n\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t}\n...\n", "needle_token_start": 1813, "needle_token_end": 2012, "code_context_ntokens": 2680, "output": ["```go\nfunc generateSystemdUnit(vsockPort int, virtioPort string) ([]byte, error) {\n\tselfExeAbs, err := os.Executable()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar args []string\n\tif vsockPort != 0 {\n\t\targs = append(args, fmt.Sprintf(\"--vsock-port %d\", vsockPort))\n\t}\n\tif virtioPort != \"\" {\n\t\targs = append(args, fmt.Sprintf(\"--virtio-port %s\", virtioPort))\n\t}\n\n\tm := map[string]string{\n\t\t\"Binary\": selfExeAbs,\n\t\t\"Args\":   strings.Join(args, \" \"),\n\t}\n\treturn textutil.ExecuteTemplate(systemdUnitTemplate, m)\n}\n```"]}
